4889 files changed, 392247 insertions, 226849 deletions

diff --git a/drivers/Makefile b/drivers/Makefile
index 53abb4a5f736..f0afdfb3c7df 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -29,6 +29,8 @@ obj-$(CONFIG_SFI)		+= sfi/
 # was used and do nothing if so
 obj-$(CONFIG_PNP)		+= pnp/
 obj-y				+= amba/
+
+obj-y				+= clk/
 # Many drivers will want to use DMA so this has to be made available
 # really early.
 obj-$(CONFIG_DMADEVICES)	+= dma/
@@ -142,8 +144,6 @@ obj-$(CONFIG_VHOST)		+= vhost/
 obj-$(CONFIG_VLYNQ)		+= vlynq/
 obj-$(CONFIG_STAGING)		+= staging/
 obj-y				+= platform/
-#common clk code
-obj-y				+= clk/
 
 obj-$(CONFIG_MAILBOX)		+= mailbox/
 obj-$(CONFIG_HWSPINLOCK)	+= hwspinlock/
diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig
index 445ce28475b3..535e7828445a 100644
--- a/drivers/acpi/Kconfig
+++ b/drivers/acpi/Kconfig
@@ -77,6 +77,9 @@ config ACPI_DEBUGGER_USER
 
 endif
 
+config ACPI_SPCR_TABLE
+	bool
+
 config ACPI_SLEEP
 	bool
 	depends on SUSPEND || HIBERNATION
@@ -227,7 +230,6 @@ config ACPI_MCFG
 config ACPI_CPPC_LIB
 	bool
 	depends on ACPI_PROCESSOR
-	depends on !ACPI_CPU_FREQ_PSS
 	select MAILBOX
 	select PCC
 	help
@@ -462,6 +464,9 @@ source "drivers/acpi/nfit/Kconfig"
 source "drivers/acpi/apei/Kconfig"
 source "drivers/acpi/dptf/Kconfig"
 
+config ACPI_WATCHDOG
+	bool
+
 config ACPI_EXTLOG
 	tristate "Extended Error Log support"
 	depends on X86_MCE && X86_LOCAL_APIC
@@ -521,4 +526,8 @@ config ACPI_CONFIGFS
 	  userspace. The configurable ACPI groups will be visible under
 	  /config/acpi, assuming configfs is mounted under /config.
 
+if ARM64
+source "drivers/acpi/arm64/Kconfig"
+endif
+
 endif	# ACPI
diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile
index 5ae9d85c5159..9ed087853dee 100644
--- a/drivers/acpi/Makefile
+++ b/drivers/acpi/Makefile
@@ -56,6 +56,7 @@ acpi-$(CONFIG_ACPI_NUMA)	+= numa.o
 acpi-$(CONFIG_ACPI_PROCFS_POWER) += cm_sbs.o
 acpi-y				+= acpi_lpat.o
 acpi-$(CONFIG_ACPI_GENERIC_GSI) += gsi.o
+acpi-$(CONFIG_ACPI_WATCHDOG)	+= acpi_watchdog.o
 
 # These are (potentially) separate modules
 
@@ -81,6 +82,7 @@ obj-$(CONFIG_ACPI_EC_DEBUGFS)	+= ec_sys.o
 obj-$(CONFIG_ACPI_CUSTOM_METHOD)+= custom_method.o
 obj-$(CONFIG_ACPI_BGRT)		+= bgrt.o
 obj-$(CONFIG_ACPI_CPPC_LIB)	+= cppc_acpi.o
+obj-$(CONFIG_ACPI_SPCR_TABLE)	+= spcr.o
 obj-$(CONFIG_ACPI_DEBUGGER_USER) += acpi_dbg.o
 
 # processor has its own "processor." module_param namespace
@@ -105,3 +107,5 @@ obj-$(CONFIG_ACPI_CONFIGFS)	+= acpi_configfs.o
 
 video-objs			+= acpi_video.o video_detect.o
 obj-y				+= dptf/
+
+obj-$(CONFIG_ARM64)		+= arm64/
diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c
index 1daf9c46df8e..d58fbf7f04e6 100644
--- a/drivers/acpi/acpi_apd.c
+++ b/drivers/acpi/acpi_apd.c
@@ -42,6 +42,7 @@ struct apd_private_data;
 struct apd_device_desc {
 	unsigned int flags;
 	unsigned int fixed_clk_rate;
+	struct property_entry *properties;
 	int (*setup)(struct apd_private_data *pdata);
 };
 
@@ -71,22 +72,35 @@ static int acpi_apd_setup(struct apd_private_data *pdata)
 }
 
 #ifdef CONFIG_X86_AMD_PLATFORM_DEVICE
-static struct apd_device_desc cz_i2c_desc = {
+static const struct apd_device_desc cz_i2c_desc = {
 	.setup = acpi_apd_setup,
 	.fixed_clk_rate = 133000000,
 };
 
-static struct apd_device_desc cz_uart_desc = {
+static struct property_entry uart_properties[] = {
+	PROPERTY_ENTRY_U32("reg-io-width", 4),
+	PROPERTY_ENTRY_U32("reg-shift", 2),
+	PROPERTY_ENTRY_BOOL("snps,uart-16550-compatible"),
+	{ },
+};
+
+static const struct apd_device_desc cz_uart_desc = {
 	.setup = acpi_apd_setup,
 	.fixed_clk_rate = 48000000,
+	.properties = uart_properties,
 };
 #endif
 
 #ifdef CONFIG_ARM64
-static struct apd_device_desc xgene_i2c_desc = {
+static const struct apd_device_desc xgene_i2c_desc = {
 	.setup = acpi_apd_setup,
 	.fixed_clk_rate = 100000000,
 };
+
+static const struct apd_device_desc vulcan_spi_desc = {
+	.setup = acpi_apd_setup,
+	.fixed_clk_rate = 133000000,
+};
 #endif
 
 #else
@@ -125,6 +139,12 @@ static int acpi_apd_create_device(struct acpi_device *adev,
 			goto err_out;
 	}
 
+	if (dev_desc->properties) {
+		ret = device_add_properties(&adev->dev, dev_desc->properties);
+		if (ret)
+			goto err_out;
+	}
+
 	adev->driver_data = pdata;
 	pdev = acpi_create_platform_device(adev);
 	if (!IS_ERR_OR_NULL(pdev))
@@ -149,6 +169,7 @@ static const struct acpi_device_id acpi_apd_device_ids[] = {
 #endif
 #ifdef CONFIG_ARM64
 	{ "APMC0D0F", APD_ADDR(xgene_i2c_desc) },
+	{ "BRCM900D", APD_ADDR(vulcan_spi_desc) },
 #endif
 	{ }
 };
diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c
index 357a0b8f860b..552010288135 100644
--- a/drivers/acpi/acpi_lpss.c
+++ b/drivers/acpi/acpi_lpss.c
@@ -75,6 +75,7 @@ struct lpss_device_desc {
 	const char *clk_con_id;
 	unsigned int prv_offset;
 	size_t prv_size_override;
+	struct property_entry *properties;
 	void (*setup)(struct lpss_private_data *pdata);
 };
 
@@ -163,11 +164,19 @@ static const struct lpss_device_desc lpt_i2c_dev_desc = {
 	.prv_offset = 0x800,
 };
 
+static struct property_entry uart_properties[] = {
+	PROPERTY_ENTRY_U32("reg-io-width", 4),
+	PROPERTY_ENTRY_U32("reg-shift", 2),
+	PROPERTY_ENTRY_BOOL("snps,uart-16550-compatible"),
+	{ },
+};
+
 static const struct lpss_device_desc lpt_uart_dev_desc = {
 	.flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_LTR,
 	.clk_con_id = "baudclk",
 	.prv_offset = 0x800,
 	.setup = lpss_uart_setup,
+	.properties = uart_properties,
 };
 
 static const struct lpss_device_desc lpt_sdio_dev_desc = {
@@ -189,6 +198,7 @@ static const struct lpss_device_desc byt_uart_dev_desc = {
 	.clk_con_id = "baudclk",
 	.prv_offset = 0x800,
 	.setup = lpss_uart_setup,
+	.properties = uart_properties,
 };
 
 static const struct lpss_device_desc bsw_uart_dev_desc = {
@@ -197,6 +207,7 @@ static const struct lpss_device_desc bsw_uart_dev_desc = {
 	.clk_con_id = "baudclk",
 	.prv_offset = 0x800,
 	.setup = lpss_uart_setup,
+	.properties = uart_properties,
 };
 
 static const struct lpss_device_desc byt_spi_dev_desc = {
@@ -440,6 +451,12 @@ static int acpi_lpss_create_device(struct acpi_device *adev,
 		goto err_out;
 	}
 
+	if (dev_desc->properties) {
+		ret = device_add_properties(&adev->dev, dev_desc->properties);
+		if (ret)
+			goto err_out;
+	}
+
 	adev->driver_data = pdata;
 	pdev = acpi_create_platform_device(adev);
 	if (!IS_ERR_OR_NULL(pdev)) {
diff --git a/drivers/acpi/acpi_platform.c b/drivers/acpi/acpi_platform.c
index 159f7f19abce..b200ae1f3c6f 100644
--- a/drivers/acpi/acpi_platform.c
+++ b/drivers/acpi/acpi_platform.c
@@ -17,6 +17,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/dma-mapping.h>
+#include <linux/pci.h>
 #include <linux/platform_device.h>
 
 #include "internal.h"
@@ -30,6 +31,22 @@ static const struct acpi_device_id forbidden_id_list[] = {
 	{"", 0},
 };
 
+static void acpi_platform_fill_resource(struct acpi_device *adev,
+	const struct resource *src, struct resource *dest)
+{
+	struct device *parent;
+
+	*dest = *src;
+
+	/*
+	 * If the device has parent we need to take its resources into
+	 * account as well because this device might consume part of those.
+	 */
+	parent = acpi_get_first_physical_node(adev->parent);
+	if (parent && dev_is_pci(parent))
+		dest->parent = pci_find_resource(to_pci_dev(parent), dest);
+}
+
 /**
  * acpi_create_platform_device - Create platform device for ACPI device node
  * @adev: ACPI device node to create a platform device for.
@@ -70,7 +87,8 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *adev)
 		}
 		count = 0;
 		list_for_each_entry(rentry, &resource_list, node)
-			resources[count++] = *rentry->res;
+			acpi_platform_fill_resource(adev, rentry->res,
+						    &resources[count++]);
 
 		acpi_dev_free_resource_list(&resource_list);
 	}
diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c
index c7ba948d253c..3de3b6b8f0f1 100644
--- a/drivers/acpi/acpi_processor.c
+++ b/drivers/acpi/acpi_processor.c
@@ -182,6 +182,11 @@ int __weak arch_register_cpu(int cpu)
 
 void __weak arch_unregister_cpu(int cpu) {}
 
+int __weak acpi_map_cpu2node(acpi_handle handle, int cpu, int physid)
+{
+	return -ENODEV;
+}
+
 static int acpi_processor_hotadd_init(struct acpi_processor *pr)
 {
 	unsigned long long sta;
@@ -300,8 +305,11 @@ static int acpi_processor_get_info(struct acpi_device *device)
 	 *  Extra Processor objects may be enumerated on MP systems with
 	 *  less than the max # of CPUs. They should be ignored _iff
 	 *  they are physically not present.
+	 *
+	 *  NOTE: Even if the processor has a cpuid, it may not be present
+	 *  because cpuid <-> apicid mapping is persistent now.
 	 */
-	if (invalid_logical_cpuid(pr->id)) {
+	if (invalid_logical_cpuid(pr->id) || !cpu_present(pr->id)) {
 		int ret = acpi_processor_hotadd_init(pr);
 		if (ret)
 			return ret;
@@ -573,8 +581,102 @@ static struct acpi_scan_handler processor_container_handler = {
 	.attach = acpi_processor_container_attach,
 };
 
+/* The number of the unique processor IDs */
+static int nr_unique_ids __initdata;
+
+/* The number of the duplicate processor IDs */
+static int nr_duplicate_ids __initdata;
+
+/* Used to store the unique processor IDs */
+static int unique_processor_ids[] __initdata = {
+	[0 ... NR_CPUS - 1] = -1,
+};
+
+/* Used to store the duplicate processor IDs */
+static int duplicate_processor_ids[] __initdata = {
+	[0 ... NR_CPUS - 1] = -1,
+};
+
+static void __init processor_validated_ids_update(int proc_id)
+{
+	int i;
+
+	if (nr_unique_ids == NR_CPUS||nr_duplicate_ids == NR_CPUS)
+		return;
+
+	/*
+	 * Firstly, compare the proc_id with duplicate IDs, if the proc_id is
+	 * already in the IDs, do nothing.
+	 */
+	for (i = 0; i < nr_duplicate_ids; i++) {
+		if (duplicate_processor_ids[i] == proc_id)
+			return;
+	}
+
+	/*
+	 * Secondly, compare the proc_id with unique IDs, if the proc_id is in
+	 * the IDs, put it in the duplicate IDs.
+	 */
+	for (i = 0; i < nr_unique_ids; i++) {
+		if (unique_processor_ids[i] == proc_id) {
+			duplicate_processor_ids[nr_duplicate_ids] = proc_id;
+			nr_duplicate_ids++;
+			return;
+		}
+	}
+
+	/*
+	 * Lastly, the proc_id is a unique ID, put it in the unique IDs.
+	 */
+	unique_processor_ids[nr_unique_ids] = proc_id;
+	nr_unique_ids++;
+}
+
+static acpi_status __init acpi_processor_ids_walk(acpi_handle handle,
+						  u32 lvl,
+						  void *context,
+						  void **rv)
+{
+	acpi_status status;
+	union acpi_object object = { 0 };
+	struct acpi_buffer buffer = { sizeof(union acpi_object), &object };
+
+	status = acpi_evaluate_object(handle, NULL, NULL, &buffer);
+	if (ACPI_FAILURE(status))
+		acpi_handle_info(handle, "Not get the processor object\n");
+	else
+		processor_validated_ids_update(object.processor.proc_id);
+
+	return AE_OK;
+}
+
+static void __init acpi_processor_check_duplicates(void)
+{
+	/* Search all processor nodes in ACPI namespace */
+	acpi_walk_namespace(ACPI_TYPE_PROCESSOR, ACPI_ROOT_OBJECT,
+						ACPI_UINT32_MAX,
+						acpi_processor_ids_walk,
+						NULL, NULL, NULL);
+}
+
+bool __init acpi_processor_validate_proc_id(int proc_id)
+{
+	int i;
+
+	/*
+	 * compare the proc_id with duplicate IDs, if the proc_id is already
+	 * in the duplicate IDs, return true, otherwise, return false.
+	 */
+	for (i = 0; i < nr_duplicate_ids; i++) {
+		if (duplicate_processor_ids[i] == proc_id)
+			return true;
+	}
+	return false;
+}
+
 void __init acpi_processor_init(void)
 {
+	acpi_processor_check_duplicates();
 	acpi_scan_add_handler_with_hotplug(&processor_handler, "processor");
 	acpi_scan_add_handler(&processor_container_handler);
 }
diff --git a/drivers/acpi/acpi_watchdog.c b/drivers/acpi/acpi_watchdog.c
new file mode 100644
index 000000000000..13caebd679f5
--- /dev/null
+++ b/drivers/acpi/acpi_watchdog.c
@@ -0,0 +1,123 @@
+/*
+ * ACPI watchdog table parsing support.
+ *
+ * Copyright (C) 2016, Intel Corporation
+ * Author: Mika Westerberg <mika.westerberg@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#define pr_fmt(fmt) "ACPI: watchdog: " fmt
+
+#include <linux/acpi.h>
+#include <linux/ioport.h>
+#include <linux/platform_device.h>
+
+#include "internal.h"
+
+/**
+ * Returns true if this system should prefer ACPI based watchdog instead of
+ * the native one (which are typically the same hardware).
+ */
+bool acpi_has_watchdog(void)
+{
+	struct acpi_table_header hdr;
+
+	if (acpi_disabled)
+		return false;
+
+	return ACPI_SUCCESS(acpi_get_table_header(ACPI_SIG_WDAT, 0, &hdr));
+}
+EXPORT_SYMBOL_GPL(acpi_has_watchdog);
+
+void __init acpi_watchdog_init(void)
+{
+	const struct acpi_wdat_entry *entries;
+	const struct acpi_table_wdat *wdat;
+	struct list_head resource_list;
+	struct resource_entry *rentry;
+	struct platform_device *pdev;
+	struct resource *resources;
+	size_t nresources = 0;
+	acpi_status status;
+	int i;
+
+	status = acpi_get_table(ACPI_SIG_WDAT, 0,
+				(struct acpi_table_header **)&wdat);
+	if (ACPI_FAILURE(status)) {
+		/* It is fine if there is no WDAT */
+		return;
+	}
+
+	/* Watchdog disabled by BIOS */
+	if (!(wdat->flags & ACPI_WDAT_ENABLED))
+		return;
+
+	/* Skip legacy PCI WDT devices */
+	if (wdat->pci_segment != 0xff || wdat->pci_bus != 0xff ||
+	    wdat->pci_device != 0xff || wdat->pci_function != 0xff)
+		return;
+
+	INIT_LIST_HEAD(&resource_list);
+
+	entries = (struct acpi_wdat_entry *)(wdat + 1);
+	for (i = 0; i < wdat->entries; i++) {
+		const struct acpi_generic_address *gas;
+		struct resource_entry *rentry;
+		struct resource res;
+		bool found;
+
+		gas = &entries[i].register_region;
+
+		res.start = gas->address;
+		if (gas->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
+			res.flags = IORESOURCE_MEM;
+			res.end = res.start + ALIGN(gas->access_width, 4);
+		} else if (gas->space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
+			res.flags = IORESOURCE_IO;
+			res.end = res.start + gas->access_width;
+		} else {
+			pr_warn("Unsupported address space: %u\n",
+				gas->space_id);
+			goto fail_free_resource_list;
+		}
+
+		found = false;
+		resource_list_for_each_entry(rentry, &resource_list) {
+			if (resource_contains(rentry->res, &res)) {
+				found = true;
+				break;
+			}
+		}
+
+		if (!found) {
+			rentry = resource_list_create_entry(NULL, 0);
+			if (!rentry)
+				goto fail_free_resource_list;
+
+			*rentry->res = res;
+			resource_list_add_tail(rentry, &resource_list);
+			nresources++;
+		}
+	}
+
+	resources = kcalloc(nresources, sizeof(*resources), GFP_KERNEL);
+	if (!resources)
+		goto fail_free_resource_list;
+
+	i = 0;
+	resource_list_for_each_entry(rentry, &resource_list)
+		resources[i++] = *rentry->res;
+
+	pdev = platform_device_register_simple("wdat_wdt", PLATFORM_DEVID_NONE,
+					       resources, nresources);
+	if (IS_ERR(pdev))
+		pr_err("Failed to create platform device\n");
+
+	kfree(resources);
+
+fail_free_resource_list:
+	resource_list_free(&resource_list);
+}
diff --git a/drivers/acpi/acpica/Makefile b/drivers/acpi/acpica/Makefile
index 227bb7bb19d7..32d93edbc479 100644
--- a/drivers/acpi/acpica/Makefile
+++ b/drivers/acpi/acpica/Makefile
@@ -175,6 +175,7 @@ acpi-y +=		\
 	utresrc.o	\
 	utstate.o	\
 	utstring.o	\
+	utstrtoul64.o	\
 	utxface.o	\
 	utxfinit.o	\
 	utxferror.o	\
diff --git a/drivers/acpi/acpica/acapps.h b/drivers/acpi/acpica/acapps.h
index ca2c0607104b..0bd6307e1f3c 100644
--- a/drivers/acpi/acpica/acapps.h
+++ b/drivers/acpi/acpica/acapps.h
@@ -44,7 +44,9 @@
 #ifndef _ACAPPS
 #define _ACAPPS
 
-#include <stdio.h>
+#ifdef ACPI_USE_STANDARD_HEADERS
+#include <sys/stat.h>
+#endif				/* ACPI_USE_STANDARD_HEADERS */
 
 /* Common info for tool signons */
 
@@ -81,13 +83,13 @@
 /* Macros for usage messages */
 
 #define ACPI_USAGE_HEADER(usage) \
-	acpi_os_printf ("Usage: %s\nOptions:\n", usage);
+	printf ("Usage: %s\nOptions:\n", usage);
 
 #define ACPI_USAGE_TEXT(description) \
-	acpi_os_printf (description);
+	printf (description);
 
 #define ACPI_OPTION(name, description) \
-	acpi_os_printf (" %-20s%s\n", name, description);
+	printf ("  %-20s%s\n", name, description);
 
 /* Check for unexpected exceptions */
 
diff --git a/drivers/acpi/acpica/acdebug.h b/drivers/acpi/acpica/acdebug.h
index f6404ea928cb..94737f8845ac 100644
--- a/drivers/acpi/acpica/acdebug.h
+++ b/drivers/acpi/acpica/acdebug.h
@@ -155,7 +155,7 @@ acpi_status acpi_db_disassemble_method(char *name);
 
 void acpi_db_disassemble_aml(char *statements, union acpi_parse_object *op);
 
-void acpi_db_batch_execute(char *count_arg);
+void acpi_db_evaluate_predefined_names(void);
 
 /*
  * dbnames - namespace commands
diff --git a/drivers/acpi/acpica/acevents.h b/drivers/acpi/acpica/acevents.h
index 77af91cf46d4..92fa47c6498c 100644
--- a/drivers/acpi/acpica/acevents.h
+++ b/drivers/acpi/acpica/acevents.h
@@ -86,6 +86,9 @@ acpi_ev_update_gpe_enable_mask(struct acpi_gpe_event_info *gpe_event_info);
 acpi_status acpi_ev_enable_gpe(struct acpi_gpe_event_info *gpe_event_info);
 
 acpi_status
+acpi_ev_mask_gpe(struct acpi_gpe_event_info *gpe_event_info, u8 is_masked);
+
+acpi_status
 acpi_ev_add_gpe_reference(struct acpi_gpe_event_info *gpe_event_info);
 
 acpi_status
diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h
index fded776236e2..750fa824d42c 100644
--- a/drivers/acpi/acpica/acglobal.h
+++ b/drivers/acpi/acpica/acglobal.h
@@ -317,6 +317,7 @@ ACPI_INIT_GLOBAL(u8, acpi_gbl_ignore_noop_operator, FALSE);
 ACPI_INIT_GLOBAL(u8, acpi_gbl_cstyle_disassembly, TRUE);
 ACPI_INIT_GLOBAL(u8, acpi_gbl_force_aml_disassembly, FALSE);
 ACPI_INIT_GLOBAL(u8, acpi_gbl_dm_opt_verbose, TRUE);
+ACPI_INIT_GLOBAL(u8, acpi_gbl_dm_emit_external_opcodes, FALSE);
 
 ACPI_GLOBAL(u8, acpi_gbl_dm_opt_disasm);
 ACPI_GLOBAL(u8, acpi_gbl_dm_opt_listing);
@@ -382,6 +383,7 @@ ACPI_GLOBAL(const char, *acpi_gbl_pld_shape_list[]);
 
 ACPI_INIT_GLOBAL(ACPI_FILE, acpi_gbl_debug_file, NULL);
 ACPI_INIT_GLOBAL(ACPI_FILE, acpi_gbl_output_file, NULL);
+ACPI_INIT_GLOBAL(u8, acpi_gbl_debug_timeout, FALSE);
 
 /* Print buffer */
 
diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h
index 13331d70dea0..dff1207a6078 100644
--- a/drivers/acpi/acpica/aclocal.h
+++ b/drivers/acpi/acpica/aclocal.h
@@ -484,6 +484,7 @@ struct acpi_gpe_event_info {
 	u8 flags;		/* Misc info about this GPE */
 	u8 gpe_number;		/* This GPE */
 	u8 runtime_count;	/* References to a run GPE */
+	u8 disable_for_dispatch;	/* Masked during dispatching */
 };
 
 /* Information about a GPE register pair, one per each status/enable pair in an array */
@@ -494,6 +495,7 @@ struct acpi_gpe_register_info {
 	u16 base_gpe_number;	/* Base GPE number for this register */
 	u8 enable_for_wake;	/* GPEs to keep enabled when sleeping */
 	u8 enable_for_run;	/* GPEs to keep enabled when running */
+	u8 mask_for_run;	/* GPEs to keep masked when running */
 	u8 enable_mask;		/* Current mask of enabled GPEs */
 };
 
diff --git a/drivers/acpi/acpica/acnamesp.h b/drivers/acpi/acpica/acnamesp.h
index f33a4ba8e0cb..bb7fca1c8ba3 100644
--- a/drivers/acpi/acpica/acnamesp.h
+++ b/drivers/acpi/acpica/acnamesp.h
@@ -130,6 +130,9 @@ acpi_status
 acpi_ns_parse_table(u32 table_index, struct acpi_namespace_node *start_node);
 
 acpi_status
+acpi_ns_execute_table(u32 table_index, struct acpi_namespace_node *start_node);
+
+acpi_status
 acpi_ns_one_complete_parse(u32 pass_number,
 			   u32 table_index,
 			   struct acpi_namespace_node *start_node);
@@ -296,6 +299,11 @@ u8
 acpi_ns_pattern_match(struct acpi_namespace_node *obj_node, char *search_for);
 
 acpi_status
+acpi_ns_get_node_unlocked(struct acpi_namespace_node *prefix_node,
+			  const char *external_pathname,
+			  u32 flags, struct acpi_namespace_node **out_node);
+
+acpi_status
 acpi_ns_get_node(struct acpi_namespace_node *prefix_node,
 		 const char *external_pathname,
 		 u32 flags, struct acpi_namespace_node **out_node);
diff --git a/drivers/acpi/acpica/acparser.h b/drivers/acpi/acpica/acparser.h
index fc305775c3d7..939d41113970 100644
--- a/drivers/acpi/acpica/acparser.h
+++ b/drivers/acpi/acpica/acparser.h
@@ -78,6 +78,8 @@ extern const u8 acpi_gbl_long_op_index[];
  */
 acpi_status acpi_ps_execute_method(struct acpi_evaluate_info *info);
 
+acpi_status acpi_ps_execute_table(struct acpi_evaluate_info *info);
+
 /*
  * psargs - Parse AML opcode arguments
  */
diff --git a/drivers/acpi/acpica/actables.h b/drivers/acpi/acpica/actables.h
index cd5a135fcf29..e85953b6fa0e 100644
--- a/drivers/acpi/acpica/actables.h
+++ b/drivers/acpi/acpica/actables.h
@@ -123,6 +123,14 @@ acpi_tb_install_standard_table(acpi_physical_address address,
 
 void acpi_tb_uninstall_table(struct acpi_table_desc *table_desc);
 
+acpi_status
+acpi_tb_load_table(u32 table_index, struct acpi_namespace_node *parent_node);
+
+acpi_status
+acpi_tb_install_and_load_table(struct acpi_table_header *table,
+			       acpi_physical_address address,
+			       u8 flags, u8 override, u32 *table_index);
+
 void acpi_tb_terminate(void);
 
 acpi_status acpi_tb_delete_namespace_by_owner(u32 table_index);
@@ -155,10 +163,6 @@ void
 acpi_tb_install_table_with_override(struct acpi_table_desc *new_table_desc,
 				    u8 override, u32 *table_index);
 
-acpi_status
-acpi_tb_install_fixed_table(acpi_physical_address address,
-			    char *signature, u32 *table_index);
-
 acpi_status acpi_tb_parse_root_table(acpi_physical_address rsdp_address);
 
 /*
diff --git a/drivers/acpi/acpica/acutils.h b/drivers/acpi/acpica/acutils.h
index a7dbb2b882cf..0a1b53c9ee0e 100644
--- a/drivers/acpi/acpica/acutils.h
+++ b/drivers/acpi/acpica/acutils.h
@@ -114,13 +114,25 @@ extern const char *acpi_gbl_pt_decode[];
 /*
  * Common error message prefixes
  */
+#ifndef ACPI_MSG_ERROR
 #define ACPI_MSG_ERROR          "ACPI Error: "
+#endif
+#ifndef ACPI_MSG_EXCEPTION
 #define ACPI_MSG_EXCEPTION      "ACPI Exception: "
+#endif
+#ifndef ACPI_MSG_WARNING
 #define ACPI_MSG_WARNING        "ACPI Warning: "
+#endif
+#ifndef ACPI_MSG_INFO
 #define ACPI_MSG_INFO           "ACPI: "
+#endif
 
+#ifndef ACPI_MSG_BIOS_ERROR
 #define ACPI_MSG_BIOS_ERROR     "ACPI BIOS Error (bug): "
+#endif
+#ifndef ACPI_MSG_BIOS_WARNING
 #define ACPI_MSG_BIOS_WARNING   "ACPI BIOS Warning (bug): "
+#endif
 
 /*
  * Common message suffix
@@ -184,14 +196,15 @@ void acpi_ut_strlwr(char *src_string);
 
 int acpi_ut_stricmp(char *string1, char *string2);
 
-acpi_status
-acpi_ut_strtoul64(char *string,
-		  u32 base, u32 max_integer_byte_width, u64 *ret_integer);
-
-/* Values for max_integer_byte_width above */
+acpi_status acpi_ut_strtoul64(char *string, u32 flags, u64 *ret_integer);
 
-#define ACPI_MAX32_BYTE_WIDTH       4
-#define ACPI_MAX64_BYTE_WIDTH       8
+/*
+ * Values for Flags above
+ * Note: LIMIT values correspond to acpi_gbl_integer_byte_width values (4/8)
+ */
+#define ACPI_STRTOUL_32BIT          0x04	/* 4 bytes */
+#define ACPI_STRTOUL_64BIT          0x08	/* 8 bytes */
+#define ACPI_STRTOUL_BASE16         0x10	/* Default: Base10/16 */
 
 /*
  * utglobal - Global data structures and procedures
@@ -221,6 +234,8 @@ const char *acpi_ut_get_event_name(u32 event_id);
 
 char acpi_ut_hex_to_ascii_char(u64 integer, u32 position);
 
+acpi_status acpi_ut_ascii_to_hex_byte(char *two_ascii_chars, u8 *return_byte);
+
 u8 acpi_ut_ascii_char_to_hex(int hex_char);
 
 u8 acpi_ut_valid_object_type(acpi_object_type type);
@@ -318,6 +333,11 @@ acpi_ut_ptr_exit(u32 line_number,
 		 const char *module_name, u32 component_id, u8 *ptr);
 
 void
+acpi_ut_str_exit(u32 line_number,
+		 const char *function_name,
+		 const char *module_name, u32 component_id, const char *string);
+
+void
 acpi_ut_debug_dump_buffer(u8 *buffer, u32 count, u32 display, u32 component_id);
 
 void acpi_ut_dump_buffer(u8 *buffer, u32 count, u32 display, u32 offset);
@@ -707,25 +727,6 @@ const struct ah_device_id *acpi_ah_match_hardware_id(char *hid);
 const char *acpi_ah_match_uuid(u8 *data);
 
 /*
- * utprint - printf/vprintf output functions
- */
-const char *acpi_ut_scan_number(const char *string, u64 *number_ptr);
-
-const char *acpi_ut_print_number(char *string, u64 number);
-
-int
-acpi_ut_vsnprintf(char *string,
-		  acpi_size size, const char *format, va_list args);
-
-int acpi_ut_snprintf(char *string, acpi_size size, const char *format, ...);
-
-#ifdef ACPI_APPLICATION
-int acpi_ut_file_vprintf(ACPI_FILE file, const char *format, va_list args);
-
-int acpi_ut_file_printf(ACPI_FILE file, const char *format, ...);
-#endif
-
-/*
  * utuuid -- UUID support functions
  */
 #if (defined ACPI_ASL_COMPILER || defined ACPI_EXEC_APP || defined ACPI_HELP_APP)
diff --git a/drivers/acpi/acpica/dbconvert.c b/drivers/acpi/acpica/dbconvert.c
index 7cd07b27f758..147ce8894f76 100644
--- a/drivers/acpi/acpica/dbconvert.c
+++ b/drivers/acpi/acpica/dbconvert.c
@@ -277,9 +277,10 @@ acpi_db_convert_to_object(acpi_object_type type,
 	default:
 
 		object->type = ACPI_TYPE_INTEGER;
-		status =
-		    acpi_ut_strtoul64(string, 16, acpi_gbl_integer_byte_width,
-				      &object->integer.value);
+		status = acpi_ut_strtoul64(string,
+					   (acpi_gbl_integer_byte_width |
+					    ACPI_STRTOUL_BASE16),
+					   &object->integer.value);
 		break;
 	}
 
diff --git a/drivers/acpi/acpica/dbexec.c b/drivers/acpi/acpica/dbexec.c
index 12df2915ad74..fe3da7c31bb7 100644
--- a/drivers/acpi/acpica/dbexec.c
+++ b/drivers/acpi/acpica/dbexec.c
@@ -392,42 +392,48 @@ acpi_db_execute(char *name, char **args, acpi_object_type *types, u32 flags)
 					  acpi_db_execution_walk, NULL, NULL,
 					  NULL);
 		return;
-	} else {
-		name_string = ACPI_ALLOCATE(strlen(name) + 1);
-		if (!name_string) {
-			return;
-		}
+	}
 
-		memset(&acpi_gbl_db_method_info, 0,
-		       sizeof(struct acpi_db_method_info));
+	name_string = ACPI_ALLOCATE(strlen(name) + 1);
+	if (!name_string) {
+		return;
+	}
 
-		strcpy(name_string, name);
-		acpi_ut_strupr(name_string);
-		acpi_gbl_db_method_info.name = name_string;
-		acpi_gbl_db_method_info.args = args;
-		acpi_gbl_db_method_info.types = types;
-		acpi_gbl_db_method_info.flags = flags;
+	memset(&acpi_gbl_db_method_info, 0, sizeof(struct acpi_db_method_info));
+	strcpy(name_string, name);
+	acpi_ut_strupr(name_string);
 
-		return_obj.pointer = NULL;
-		return_obj.length = ACPI_ALLOCATE_BUFFER;
+	/* Subcommand to Execute all predefined names in the namespace */
 
-		status = acpi_db_execute_setup(&acpi_gbl_db_method_info);
-		if (ACPI_FAILURE(status)) {
-			ACPI_FREE(name_string);
-			return;
-		}
+	if (!strncmp(name_string, "PREDEF", 6)) {
+		acpi_db_evaluate_predefined_names();
+		ACPI_FREE(name_string);
+		return;
+	}
 
-		/* Get the NS node, determines existence also */
+	acpi_gbl_db_method_info.name = name_string;
+	acpi_gbl_db_method_info.args = args;
+	acpi_gbl_db_method_info.types = types;
+	acpi_gbl_db_method_info.flags = flags;
 
-		status = acpi_get_handle(NULL, acpi_gbl_db_method_info.pathname,
-					 &acpi_gbl_db_method_info.method);
-		if (ACPI_SUCCESS(status)) {
-			status =
-			    acpi_db_execute_method(&acpi_gbl_db_method_info,
-						   &return_obj);
-		}
+	return_obj.pointer = NULL;
+	return_obj.length = ACPI_ALLOCATE_BUFFER;
+
+	status = acpi_db_execute_setup(&acpi_gbl_db_method_info);
+	if (ACPI_FAILURE(status)) {
 		ACPI_FREE(name_string);
+		return;
+	}
+
+	/* Get the NS node, determines existence also */
+
+	status = acpi_get_handle(NULL, acpi_gbl_db_method_info.pathname,
+				 &acpi_gbl_db_method_info.method);
+	if (ACPI_SUCCESS(status)) {
+		status = acpi_db_execute_method(&acpi_gbl_db_method_info,
+						&return_obj);
 	}
+	ACPI_FREE(name_string);
 
 	/*
 	 * Allow any handlers in separate threads to complete.
diff --git a/drivers/acpi/acpica/dbfileio.c b/drivers/acpi/acpica/dbfileio.c
index 483287942372..6f05b8c271a5 100644
--- a/drivers/acpi/acpica/dbfileio.c
+++ b/drivers/acpi/acpica/dbfileio.c
@@ -46,14 +46,12 @@
 #include "accommon.h"
 #include "acdebug.h"
 #include "actables.h"
-#include <stdio.h>
-#ifdef ACPI_APPLICATION
-#include "acapps.h"
-#endif
 
 #define _COMPONENT          ACPI_CA_DEBUGGER
 ACPI_MODULE_NAME("dbfileio")
 
+#ifdef ACPI_APPLICATION
+#include "acapps.h"
 #ifdef ACPI_DEBUGGER
 /*******************************************************************************
  *
@@ -69,8 +67,6 @@ ACPI_MODULE_NAME("dbfileio")
 void acpi_db_close_debug_file(void)
 {
 
-#ifdef ACPI_APPLICATION
-
 	if (acpi_gbl_debug_file) {
 		fclose(acpi_gbl_debug_file);
 		acpi_gbl_debug_file = NULL;
@@ -78,7 +74,6 @@ void acpi_db_close_debug_file(void)
 		acpi_os_printf("Debug output file %s closed\n",
 			       acpi_gbl_db_debug_filename);
 	}
-#endif
 }
 
 /*******************************************************************************
@@ -96,8 +91,6 @@ void acpi_db_close_debug_file(void)
 void acpi_db_open_debug_file(char *name)
 {
 
-#ifdef ACPI_APPLICATION
-
 	acpi_db_close_debug_file();
 	acpi_gbl_debug_file = fopen(name, "w+");
 	if (!acpi_gbl_debug_file) {
@@ -109,8 +102,6 @@ void acpi_db_open_debug_file(char *name)
 	strncpy(acpi_gbl_db_debug_filename, name,
 		sizeof(acpi_gbl_db_debug_filename));
 	acpi_gbl_db_output_to_file = TRUE;
-
-#endif
 }
 #endif
 
@@ -152,12 +143,13 @@ acpi_status acpi_db_load_tables(struct acpi_new_table_desc *list_head)
 			return (status);
 		}
 
-		fprintf(stderr,
-			"Acpi table [%4.4s] successfully installed and loaded\n",
-			table->signature);
+		acpi_os_printf
+		    ("Acpi table [%4.4s] successfully installed and loaded\n",
+		     table->signature);
 
 		table_list_head = table_list_head->next;
 	}
 
 	return (AE_OK);
 }
+#endif
diff --git a/drivers/acpi/acpica/dbinput.c b/drivers/acpi/acpica/dbinput.c
index 7cd5d2e022da..068214f9cc9d 100644
--- a/drivers/acpi/acpica/dbinput.c
+++ b/drivers/acpi/acpica/dbinput.c
@@ -286,6 +286,8 @@ static const struct acpi_db_command_help acpi_gbl_db_command_help[] = {
 	{1, "     \"Ascii String\"", "String method argument\n"},
 	{1, "     (Hex Byte List)", "Buffer method argument\n"},
 	{1, "     [Package Element List]", "Package method argument\n"},
+	{5, "  Execute predefined",
+	 "Execute all predefined (public) methods\n"},
 	{1, "  Go", "Allow method to run to completion\n"},
 	{1, "  Information", "Display info about the current method\n"},
 	{1, "  Into", "Step into (not over) a method call\n"},
diff --git a/drivers/acpi/acpica/dbmethod.c b/drivers/acpi/acpica/dbmethod.c
index f17a86f6b16b..314b94cf086a 100644
--- a/drivers/acpi/acpica/dbmethod.c
+++ b/drivers/acpi/acpica/dbmethod.c
@@ -52,6 +52,11 @@
 #define _COMPONENT          ACPI_CA_DEBUGGER
 ACPI_MODULE_NAME("dbmethod")
 
+/* Local prototypes */
+static acpi_status
+acpi_db_walk_for_execute(acpi_handle obj_handle,
+			 u32 nesting_level, void *context, void **return_value);
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_db_set_method_breakpoint
@@ -66,6 +71,7 @@ ACPI_MODULE_NAME("dbmethod")
  *              AML offset
  *
  ******************************************************************************/
+
 void
 acpi_db_set_method_breakpoint(char *location,
 			      struct acpi_walk_state *walk_state,
@@ -367,3 +373,129 @@ acpi_status acpi_db_disassemble_method(char *name)
 	acpi_ut_release_owner_id(&obj_desc->method.owner_id);
 	return (AE_OK);
 }
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_db_walk_for_execute
+ *
+ * PARAMETERS:  Callback from walk_namespace
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Batch execution module. Currently only executes predefined
+ *              ACPI names.
+ *
+ ******************************************************************************/
+
+static acpi_status
+acpi_db_walk_for_execute(acpi_handle obj_handle,
+			 u32 nesting_level, void *context, void **return_value)
+{
+	struct acpi_namespace_node *node =
+	    (struct acpi_namespace_node *)obj_handle;
+	struct acpi_db_execute_walk *info =
+	    (struct acpi_db_execute_walk *)context;
+	struct acpi_buffer return_obj;
+	acpi_status status;
+	char *pathname;
+	u32 i;
+	struct acpi_device_info *obj_info;
+	struct acpi_object_list param_objects;
+	union acpi_object params[ACPI_METHOD_NUM_ARGS];
+	const union acpi_predefined_info *predefined;
+
+	predefined = acpi_ut_match_predefined_method(node->name.ascii);
+	if (!predefined) {
+		return (AE_OK);
+	}
+
+	if (node->type == ACPI_TYPE_LOCAL_SCOPE) {
+		return (AE_OK);
+	}
+
+	pathname = acpi_ns_get_external_pathname(node);
+	if (!pathname) {
+		return (AE_OK);
+	}
+
+	/* Get the object info for number of method parameters */
+
+	status = acpi_get_object_info(obj_handle, &obj_info);
+	if (ACPI_FAILURE(status)) {
+		return (status);
+	}
+
+	param_objects.pointer = NULL;
+	param_objects.count = 0;
+
+	if (obj_info->type == ACPI_TYPE_METHOD) {
+
+		/* Setup default parameters */
+
+		for (i = 0; i < obj_info->param_count; i++) {
+			params[i].type = ACPI_TYPE_INTEGER;
+			params[i].integer.value = 1;
+		}
+
+		param_objects.pointer = params;
+		param_objects.count = obj_info->param_count;
+	}
+
+	ACPI_FREE(obj_info);
+	return_obj.pointer = NULL;
+	return_obj.length = ACPI_ALLOCATE_BUFFER;
+
+	/* Do the actual method execution */
+
+	acpi_gbl_method_executing = TRUE;
+
+	status = acpi_evaluate_object(node, NULL, &param_objects, &return_obj);
+
+	acpi_os_printf("%-32s returned %s\n", pathname,
+		       acpi_format_exception(status));
+	acpi_gbl_method_executing = FALSE;
+	ACPI_FREE(pathname);
+
+	/* Ignore status from method execution */
+
+	status = AE_OK;
+
+	/* Update count, check if we have executed enough methods */
+
+	info->count++;
+	if (info->count >= info->max_count) {
+		status = AE_CTRL_TERMINATE;
+	}
+
+	return (status);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_db_evaluate_predefined_names
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Namespace batch execution. Execute predefined names in the
+ *              namespace, up to the max count, if specified.
+ *
+ ******************************************************************************/
+
+void acpi_db_evaluate_predefined_names(void)
+{
+	struct acpi_db_execute_walk info;
+
+	info.count = 0;
+	info.max_count = ACPI_UINT32_MAX;
+
+	/* Search all nodes in namespace */
+
+	(void)acpi_walk_namespace(ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
+				  ACPI_UINT32_MAX, acpi_db_walk_for_execute,
+				  NULL, (void *)&info, NULL);
+
+	acpi_os_printf("Evaluated %u predefined names in the namespace\n",
+		       info.count);
+}
diff --git a/drivers/acpi/acpica/dbobject.c b/drivers/acpi/acpica/dbobject.c
index 1d59e8b6f859..08eaaf350b24 100644
--- a/drivers/acpi/acpica/dbobject.c
+++ b/drivers/acpi/acpica/dbobject.c
@@ -142,11 +142,11 @@ void acpi_db_decode_internal_object(union acpi_operand_object *obj_desc)
 
 	case ACPI_TYPE_STRING:
 
-		acpi_os_printf("(%u) \"%.24s",
+		acpi_os_printf("(%u) \"%.60s",
 			       obj_desc->string.length,
 			       obj_desc->string.pointer);
 
-		if (obj_desc->string.length > 24) {
+		if (obj_desc->string.length > 60) {
 			acpi_os_printf("...");
 		} else {
 			acpi_os_printf("\"");
diff --git a/drivers/acpi/acpica/dsmethod.c b/drivers/acpi/acpica/dsmethod.c
index 47c7b52a519c..32e9ddc0cf2b 100644
--- a/drivers/acpi/acpica/dsmethod.c
+++ b/drivers/acpi/acpica/dsmethod.c
@@ -99,11 +99,14 @@ acpi_ds_auto_serialize_method(struct acpi_namespace_node *node,
 			  "Method auto-serialization parse [%4.4s] %p\n",
 			  acpi_ut_get_node_name(node), node));
 
+	acpi_ex_enter_interpreter();
+
 	/* Create/Init a root op for the method parse tree */
 
 	op = acpi_ps_alloc_op(AML_METHOD_OP, obj_desc->method.aml_start);
 	if (!op) {
-		return_ACPI_STATUS(AE_NO_MEMORY);
+		status = AE_NO_MEMORY;
+		goto unlock;
 	}
 
 	acpi_ps_set_name(op, node->name.integer);
@@ -115,7 +118,8 @@ acpi_ds_auto_serialize_method(struct acpi_namespace_node *node,
 	    acpi_ds_create_walk_state(node->owner_id, NULL, NULL, NULL);
 	if (!walk_state) {
 		acpi_ps_free_op(op);
-		return_ACPI_STATUS(AE_NO_MEMORY);
+		status = AE_NO_MEMORY;
+		goto unlock;
 	}
 
 	status = acpi_ds_init_aml_walk(walk_state, op, node,
@@ -134,6 +138,8 @@ acpi_ds_auto_serialize_method(struct acpi_namespace_node *node,
 	status = acpi_ps_parse_aml(walk_state);
 
 	acpi_ps_delete_parse_tree(op);
+unlock:
+	acpi_ex_exit_interpreter();
 	return_ACPI_STATUS(status);
 }
 
@@ -757,8 +763,10 @@ acpi_ds_terminate_control_method(union acpi_operand_object *method_desc,
 
 			/* Delete any direct children of (created by) this method */
 
+			(void)acpi_ex_exit_interpreter();
 			acpi_ns_delete_namespace_subtree(walk_state->
 							 method_node);
+			(void)acpi_ex_enter_interpreter();
 
 			/*
 			 * Delete any objects that were created by this method
@@ -769,9 +777,11 @@ acpi_ds_terminate_control_method(union acpi_operand_object *method_desc,
 			 */
 			if (method_desc->method.
 			    info_flags & ACPI_METHOD_MODIFIED_NAMESPACE) {
+				(void)acpi_ex_exit_interpreter();
 				acpi_ns_delete_namespace_by_owner(method_desc->
 								  method.
 								  owner_id);
+				(void)acpi_ex_enter_interpreter();
 				method_desc->method.info_flags &=
 				    ~ACPI_METHOD_MODIFIED_NAMESPACE;
 			}
diff --git a/drivers/acpi/acpica/dsutils.c b/drivers/acpi/acpica/dsutils.c
index f393de9f5887..7d8ef52fb88d 100644
--- a/drivers/acpi/acpica/dsutils.c
+++ b/drivers/acpi/acpica/dsutils.c
@@ -565,15 +565,14 @@ acpi_ds_create_operand(struct acpi_walk_state *walk_state,
 					status = AE_OK;
 				} else if (parent_op->common.aml_opcode ==
 					   AML_EXTERNAL_OP) {
-
-					/* TBD: May only be temporary */
-
-					obj_desc =
-					    acpi_ut_create_string_object((acpi_size)name_length);
-
-					strncpy(obj_desc->string.pointer,
-						name_string, name_length);
-					status = AE_OK;
+					/*
+					 * This opcode should never appear here. It is used only
+					 * by AML disassemblers and is surrounded by an If(0)
+					 * by the ASL compiler.
+					 *
+					 * Therefore, if we see it here, it is a serious error.
+					 */
+					status = AE_AML_BAD_OPCODE;
 				} else {
 					/*
 					 * We just plain didn't find it -- which is a
diff --git a/drivers/acpi/acpica/dswexec.c b/drivers/acpi/acpica/dswexec.c
index 402ecc590c56..438597cf6cc5 100644
--- a/drivers/acpi/acpica/dswexec.c
+++ b/drivers/acpi/acpica/dswexec.c
@@ -133,7 +133,8 @@ acpi_ds_get_predicate_value(struct acpi_walk_state *walk_state,
 	 * Result of predicate evaluation must be an Integer
 	 * object. Implicitly convert the argument if necessary.
 	 */
-	status = acpi_ex_convert_to_integer(obj_desc, &local_obj_desc, 16);
+	status = acpi_ex_convert_to_integer(obj_desc, &local_obj_desc,
+					    ACPI_STRTOUL_BASE16);
 	if (ACPI_FAILURE(status)) {
 		goto cleanup;
 	}
diff --git a/drivers/acpi/acpica/dswload2.c b/drivers/acpi/acpica/dswload2.c
index 762db3fa70e0..028b22a3154e 100644
--- a/drivers/acpi/acpica/dswload2.c
+++ b/drivers/acpi/acpica/dswload2.c
@@ -605,16 +605,13 @@ acpi_status acpi_ds_load2_end_op(struct acpi_walk_state *walk_state)
 				if (ACPI_FAILURE(status)) {
 					return_ACPI_STATUS(status);
 				}
-
-				acpi_ex_exit_interpreter();
 			}
 
+			acpi_ex_exit_interpreter();
 			status =
 			    acpi_ev_initialize_region
 			    (acpi_ns_get_attached_object(node), FALSE);
-			if (walk_state->method_node) {
-				acpi_ex_enter_interpreter();
-			}
+			acpi_ex_enter_interpreter();
 
 			if (ACPI_FAILURE(status)) {
 				/*
diff --git a/drivers/acpi/acpica/evgpe.c b/drivers/acpi/acpica/evgpe.c
index 4b4949ce05bc..bdb10bee13ce 100644
--- a/drivers/acpi/acpica/evgpe.c
+++ b/drivers/acpi/acpica/evgpe.c
@@ -130,6 +130,60 @@ acpi_status acpi_ev_enable_gpe(struct acpi_gpe_event_info *gpe_event_info)
 
 /*******************************************************************************
  *
+ * FUNCTION:    acpi_ev_mask_gpe
+ *
+ * PARAMETERS:  gpe_event_info          - GPE to be blocked/unblocked
+ *              is_masked               - Whether the GPE is masked or not
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Unconditionally mask/unmask a GPE during runtime.
+ *
+ ******************************************************************************/
+
+acpi_status
+acpi_ev_mask_gpe(struct acpi_gpe_event_info *gpe_event_info, u8 is_masked)
+{
+	struct acpi_gpe_register_info *gpe_register_info;
+	u32 register_bit;
+
+	ACPI_FUNCTION_TRACE(ev_mask_gpe);
+
+	gpe_register_info = gpe_event_info->register_info;
+	if (!gpe_register_info) {
+		return_ACPI_STATUS(AE_NOT_EXIST);
+	}
+
+	register_bit = acpi_hw_get_gpe_register_bit(gpe_event_info);
+
+	/* Perform the action */
+
+	if (is_masked) {
+		if (register_bit & gpe_register_info->mask_for_run) {
+			return_ACPI_STATUS(AE_BAD_PARAMETER);
+		}
+
+		(void)acpi_hw_low_set_gpe(gpe_event_info, ACPI_GPE_DISABLE);
+		ACPI_SET_BIT(gpe_register_info->mask_for_run, (u8)register_bit);
+	} else {
+		if (!(register_bit & gpe_register_info->mask_for_run)) {
+			return_ACPI_STATUS(AE_BAD_PARAMETER);
+		}
+
+		ACPI_CLEAR_BIT(gpe_register_info->mask_for_run,
+			       (u8)register_bit);
+		if (gpe_event_info->runtime_count
+		    && !gpe_event_info->disable_for_dispatch) {
+			(void)acpi_hw_low_set_gpe(gpe_event_info,
+						  ACPI_GPE_ENABLE);
+		}
+	}
+
+	return_ACPI_STATUS(AE_OK);
+}
+
+/*******************************************************************************
+ *
  * FUNCTION:    acpi_ev_add_gpe_reference
  *
  * PARAMETERS:  gpe_event_info          - Add a reference to this GPE
@@ -674,6 +728,7 @@ acpi_status acpi_ev_finish_gpe(struct acpi_gpe_event_info *gpe_event_info)
 	 * in the event_info.
 	 */
 	(void)acpi_hw_low_set_gpe(gpe_event_info, ACPI_GPE_CONDITIONAL_ENABLE);
+	gpe_event_info->disable_for_dispatch = FALSE;
 	return (AE_OK);
 }
 
@@ -737,6 +792,8 @@ acpi_ev_gpe_dispatch(struct acpi_namespace_node *gpe_device,
 		}
 	}
 
+	gpe_event_info->disable_for_dispatch = TRUE;
+
 	/*
 	 * Dispatch the GPE to either an installed handler or the control
 	 * method associated with this GPE (_Lxx or _Exx). If a handler
diff --git a/drivers/acpi/acpica/evgpeinit.c b/drivers/acpi/acpica/evgpeinit.c
index 7dc75474c897..16ce4835e7d0 100644
--- a/drivers/acpi/acpica/evgpeinit.c
+++ b/drivers/acpi/acpica/evgpeinit.c
@@ -323,7 +323,9 @@ acpi_ev_match_gpe_method(acpi_handle obj_handle,
 	struct acpi_gpe_walk_info *walk_info =
 	    ACPI_CAST_PTR(struct acpi_gpe_walk_info, context);
 	struct acpi_gpe_event_info *gpe_event_info;
+	acpi_status status;
 	u32 gpe_number;
+	u8 temp_gpe_number;
 	char name[ACPI_NAME_SIZE + 1];
 	u8 type;
 
@@ -377,8 +379,8 @@ acpi_ev_match_gpe_method(acpi_handle obj_handle,
 
 	/* 4) The last two characters of the name are the hex GPE Number */
 
-	gpe_number = strtoul(&name[2], NULL, 16);
-	if (gpe_number == ACPI_UINT32_MAX) {
+	status = acpi_ut_ascii_to_hex_byte(&name[2], &temp_gpe_number);
+	if (ACPI_FAILURE(status)) {
 
 		/* Conversion failed; invalid method, just ignore it */
 
@@ -390,6 +392,7 @@ acpi_ev_match_gpe_method(acpi_handle obj_handle,
 
 	/* Ensure that we have a valid GPE number for this GPE block */
 
+	gpe_number = (u32)temp_gpe_number;
 	gpe_event_info =
 	    acpi_ev_low_get_gpe_info(gpe_number, walk_info->gpe_block);
 	if (!gpe_event_info) {
diff --git a/drivers/acpi/acpica/evrgnini.c b/drivers/acpi/acpica/evrgnini.c
index b6ea9c0d0d8c..3843f1fc5dbb 100644
--- a/drivers/acpi/acpica/evrgnini.c
+++ b/drivers/acpi/acpica/evrgnini.c
@@ -553,7 +553,8 @@ acpi_ev_initialize_region(union acpi_operand_object *region_obj,
 				 *
 				 * See acpi_ns_exec_module_code
 				 */
-				if (obj_desc->method.
+				if (!acpi_gbl_parse_table_as_term_list &&
+				    obj_desc->method.
 				    info_flags & ACPI_METHOD_MODULE_LEVEL) {
 					handler_obj =
 					    obj_desc->method.dispatch.handler;
diff --git a/drivers/acpi/acpica/evxfgpe.c b/drivers/acpi/acpica/evxfgpe.c
index 17cfef721d00..d7a3b2775505 100644
--- a/drivers/acpi/acpica/evxfgpe.c
+++ b/drivers/acpi/acpica/evxfgpe.c
@@ -235,11 +235,13 @@ acpi_status acpi_set_gpe(acpi_handle gpe_device, u32 gpe_number, u8 action)
 	case ACPI_GPE_ENABLE:
 
 		status = acpi_hw_low_set_gpe(gpe_event_info, ACPI_GPE_ENABLE);
+		gpe_event_info->disable_for_dispatch = FALSE;
 		break;
 
 	case ACPI_GPE_DISABLE:
 
 		status = acpi_hw_low_set_gpe(gpe_event_info, ACPI_GPE_DISABLE);
+		gpe_event_info->disable_for_dispatch = TRUE;
 		break;
 
 	default:
@@ -257,6 +259,47 @@ ACPI_EXPORT_SYMBOL(acpi_set_gpe)
 
 /*******************************************************************************
  *
+ * FUNCTION:    acpi_mask_gpe
+ *
+ * PARAMETERS:  gpe_device          - Parent GPE Device. NULL for GPE0/GPE1
+ *              gpe_number          - GPE level within the GPE block
+ *              is_masked           - Whether the GPE is masked or not
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Unconditionally mask/unmask the an individual GPE, ex., to
+ *              prevent a GPE flooding.
+ *
+ ******************************************************************************/
+acpi_status acpi_mask_gpe(acpi_handle gpe_device, u32 gpe_number, u8 is_masked)
+{
+	struct acpi_gpe_event_info *gpe_event_info;
+	acpi_status status;
+	acpi_cpu_flags flags;
+
+	ACPI_FUNCTION_TRACE(acpi_mask_gpe);
+
+	flags = acpi_os_acquire_lock(acpi_gbl_gpe_lock);
+
+	/* Ensure that we have a valid GPE number */
+
+	gpe_event_info = acpi_ev_get_gpe_event_info(gpe_device, gpe_number);
+	if (!gpe_event_info) {
+		status = AE_BAD_PARAMETER;
+		goto unlock_and_exit;
+	}
+
+	status = acpi_ev_mask_gpe(gpe_event_info, is_masked);
+
+unlock_and_exit:
+	acpi_os_release_lock(acpi_gbl_gpe_lock, flags);
+	return_ACPI_STATUS(status);
+}
+
+ACPI_EXPORT_SYMBOL(acpi_mask_gpe)
+
+/*******************************************************************************
+ *
  * FUNCTION:    acpi_mark_gpe_for_wake
  *
  * PARAMETERS:  gpe_device          - Parent GPE Device. NULL for GPE0/GPE1
diff --git a/drivers/acpi/acpica/exconcat.c b/drivers/acpi/acpica/exconcat.c
index 2423fe03e879..5429c2a6bc41 100644
--- a/drivers/acpi/acpica/exconcat.c
+++ b/drivers/acpi/acpica/exconcat.c
@@ -156,7 +156,7 @@ acpi_ex_do_concatenate(union acpi_operand_object *operand0,
 
 		status =
 		    acpi_ex_convert_to_integer(local_operand1, &temp_operand1,
-					       16);
+					       ACPI_STRTOUL_BASE16);
 		break;
 
 	case ACPI_TYPE_BUFFER:
diff --git a/drivers/acpi/acpica/exconfig.c b/drivers/acpi/acpica/exconfig.c
index a1d177d58254..718428ba0b89 100644
--- a/drivers/acpi/acpica/exconfig.c
+++ b/drivers/acpi/acpica/exconfig.c
@@ -55,9 +55,7 @@ ACPI_MODULE_NAME("exconfig")
 
 /* Local prototypes */
 static acpi_status
-acpi_ex_add_table(u32 table_index,
-		  struct acpi_namespace_node *parent_node,
-		  union acpi_operand_object **ddb_handle);
+acpi_ex_add_table(u32 table_index, union acpi_operand_object **ddb_handle);
 
 static acpi_status
 acpi_ex_region_read(union acpi_operand_object *obj_desc,
@@ -79,13 +77,9 @@ acpi_ex_region_read(union acpi_operand_object *obj_desc,
  ******************************************************************************/
 
 static acpi_status
-acpi_ex_add_table(u32 table_index,
-		  struct acpi_namespace_node *parent_node,
-		  union acpi_operand_object **ddb_handle)
+acpi_ex_add_table(u32 table_index, union acpi_operand_object **ddb_handle)
 {
 	union acpi_operand_object *obj_desc;
-	acpi_status status;
-	acpi_owner_id owner_id;
 
 	ACPI_FUNCTION_TRACE(ex_add_table);
 
@@ -100,39 +94,8 @@ acpi_ex_add_table(u32 table_index,
 
 	obj_desc->common.flags |= AOPOBJ_DATA_VALID;
 	obj_desc->reference.class = ACPI_REFCLASS_TABLE;
-	*ddb_handle = obj_desc;
-
-	/* Install the new table into the local data structures */
-
 	obj_desc->reference.value = table_index;
-
-	/* Add the table to the namespace */
-
-	status = acpi_ns_load_table(table_index, parent_node);
-	if (ACPI_FAILURE(status)) {
-		acpi_ut_remove_reference(obj_desc);
-		*ddb_handle = NULL;
-		return_ACPI_STATUS(status);
-	}
-
-	/* Execute any module-level code that was found in the table */
-
-	acpi_ex_exit_interpreter();
-	if (acpi_gbl_group_module_level_code) {
-		acpi_ns_exec_module_code_list();
-	}
-	acpi_ex_enter_interpreter();
-
-	/*
-	 * Update GPEs for any new _Lxx/_Exx methods. Ignore errors. The host is
-	 * responsible for discovering any new wake GPEs by running _PRW methods
-	 * that may have been loaded by this table.
-	 */
-	status = acpi_tb_get_owner_id(table_index, &owner_id);
-	if (ACPI_SUCCESS(status)) {
-		acpi_ev_update_gpes(owner_id);
-	}
-
+	*ddb_handle = obj_desc;
 	return_ACPI_STATUS(AE_OK);
 }
 
@@ -159,16 +122,17 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state,
 	struct acpi_namespace_node *start_node;
 	struct acpi_namespace_node *parameter_node = NULL;
 	union acpi_operand_object *ddb_handle;
-	struct acpi_table_header *table;
 	u32 table_index;
 
 	ACPI_FUNCTION_TRACE(ex_load_table_op);
 
 	/* Find the ACPI table in the RSDT/XSDT */
 
+	acpi_ex_exit_interpreter();
 	status = acpi_tb_find_table(operand[0]->string.pointer,
 				    operand[1]->string.pointer,
 				    operand[2]->string.pointer, &table_index);
+	acpi_ex_enter_interpreter();
 	if (ACPI_FAILURE(status)) {
 		if (status != AE_NOT_FOUND) {
 			return_ACPI_STATUS(status);
@@ -197,9 +161,10 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state,
 		 * Find the node referenced by the root_path_string. This is the
 		 * location within the namespace where the table will be loaded.
 		 */
-		status =
-		    acpi_ns_get_node(start_node, operand[3]->string.pointer,
-				     ACPI_NS_SEARCH_PARENT, &parent_node);
+		status = acpi_ns_get_node_unlocked(start_node,
+						   operand[3]->string.pointer,
+						   ACPI_NS_SEARCH_PARENT,
+						   &parent_node);
 		if (ACPI_FAILURE(status)) {
 			return_ACPI_STATUS(status);
 		}
@@ -219,9 +184,10 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state,
 
 		/* Find the node referenced by the parameter_path_string */
 
-		status =
-		    acpi_ns_get_node(start_node, operand[4]->string.pointer,
-				     ACPI_NS_SEARCH_PARENT, &parameter_node);
+		status = acpi_ns_get_node_unlocked(start_node,
+						   operand[4]->string.pointer,
+						   ACPI_NS_SEARCH_PARENT,
+						   &parameter_node);
 		if (ACPI_FAILURE(status)) {
 			return_ACPI_STATUS(status);
 		}
@@ -229,7 +195,15 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state,
 
 	/* Load the table into the namespace */
 
-	status = acpi_ex_add_table(table_index, parent_node, &ddb_handle);
+	ACPI_INFO(("Dynamic OEM Table Load:"));
+	acpi_ex_exit_interpreter();
+	status = acpi_tb_load_table(table_index, parent_node);
+	acpi_ex_enter_interpreter();
+	if (ACPI_FAILURE(status)) {
+		return_ACPI_STATUS(status);
+	}
+
+	status = acpi_ex_add_table(table_index, &ddb_handle);
 	if (ACPI_FAILURE(status)) {
 		return_ACPI_STATUS(status);
 	}
@@ -252,19 +226,6 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state,
 		}
 	}
 
-	status = acpi_get_table_by_index(table_index, &table);
-	if (ACPI_SUCCESS(status)) {
-		ACPI_INFO(("Dynamic OEM Table Load:"));
-		acpi_tb_print_table_header(0, table);
-	}
-
-	/* Invoke table handler if present */
-
-	if (acpi_gbl_table_handler) {
-		(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_LOAD, table,
-					     acpi_gbl_table_handler_context);
-	}
-
 	*return_desc = ddb_handle;
 	return_ACPI_STATUS(status);
 }
@@ -475,13 +436,12 @@ acpi_ex_load_op(union acpi_operand_object *obj_desc,
 	/* Install the new table into the local data structures */
 
 	ACPI_INFO(("Dynamic OEM Table Load:"));
-	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
-
-	status = acpi_tb_install_standard_table(ACPI_PTR_TO_PHYSADDR(table),
-						ACPI_TABLE_ORIGIN_INTERNAL_VIRTUAL,
-						TRUE, TRUE, &table_index);
-
-	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
+	acpi_ex_exit_interpreter();
+	status =
+	    acpi_tb_install_and_load_table(table, ACPI_PTR_TO_PHYSADDR(table),
+					   ACPI_TABLE_ORIGIN_INTERNAL_VIRTUAL,
+					   TRUE, &table_index);
+	acpi_ex_enter_interpreter();
 	if (ACPI_FAILURE(status)) {
 
 		/* Delete allocated table buffer */
@@ -491,25 +451,13 @@ acpi_ex_load_op(union acpi_operand_object *obj_desc,
 	}
 
 	/*
-	 * Note: Now table is "INSTALLED", it must be validated before
-	 * loading.
-	 */
-	status =
-	    acpi_tb_validate_table(&acpi_gbl_root_table_list.
-				   tables[table_index]);
-	if (ACPI_FAILURE(status)) {
-		return_ACPI_STATUS(status);
-	}
-
-	/*
 	 * Add the table to the namespace.
 	 *
 	 * Note: Load the table objects relative to the root of the namespace.
 	 * This appears to go against the ACPI specification, but we do it for
 	 * compatibility with other ACPI implementations.
 	 */
-	status =
-	    acpi_ex_add_table(table_index, acpi_gbl_root_node, &ddb_handle);
+	status = acpi_ex_add_table(table_index, &ddb_handle);
 	if (ACPI_FAILURE(status)) {
 
 		/* On error, table_ptr was deallocated above */
@@ -532,14 +480,6 @@ acpi_ex_load_op(union acpi_operand_object *obj_desc,
 	/* Remove the reference by added by acpi_ex_store above */
 
 	acpi_ut_remove_reference(ddb_handle);
-
-	/* Invoke table handler if present */
-
-	if (acpi_gbl_table_handler) {
-		(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_LOAD, table,
-					     acpi_gbl_table_handler_context);
-	}
-
 	return_ACPI_STATUS(status);
 }
 
@@ -592,10 +532,17 @@ acpi_status acpi_ex_unload_table(union acpi_operand_object *ddb_handle)
 
 	table_index = table_desc->reference.value;
 
+	/*
+	 * Release the interpreter lock so that the table lock won't have
+	 * strict order requirement against it.
+	 */
+	acpi_ex_exit_interpreter();
+
 	/* Ensure the table is still loaded */
 
 	if (!acpi_tb_is_table_loaded(table_index)) {
-		return_ACPI_STATUS(AE_NOT_EXIST);
+		status = AE_NOT_EXIST;
+		goto lock_and_exit;
 	}
 
 	/* Invoke table handler if present */
@@ -613,16 +560,24 @@ acpi_status acpi_ex_unload_table(union acpi_operand_object *ddb_handle)
 
 	status = acpi_tb_delete_namespace_by_owner(table_index);
 	if (ACPI_FAILURE(status)) {
-		return_ACPI_STATUS(status);
+		goto lock_and_exit;
 	}
 
 	(void)acpi_tb_release_owner_id(table_index);
 	acpi_tb_set_table_loaded_flag(table_index, FALSE);
 
+lock_and_exit:
+
+	/* Re-acquire the interpreter lock */
+
+	acpi_ex_enter_interpreter();
+
 	/*
 	 * Invalidate the handle. We do this because the handle may be stored
 	 * in a named object and may not be actually deleted until much later.
 	 */
-	ddb_handle->common.flags &= ~AOPOBJ_DATA_VALID;
-	return_ACPI_STATUS(AE_OK);
+	if (ACPI_SUCCESS(status)) {
+		ddb_handle->common.flags &= ~AOPOBJ_DATA_VALID;
+	}
+	return_ACPI_STATUS(status);
 }
diff --git a/drivers/acpi/acpica/exconvrt.c b/drivers/acpi/acpica/exconvrt.c
index b7e9b3d803e1..588ad1409dbe 100644
--- a/drivers/acpi/acpica/exconvrt.c
+++ b/drivers/acpi/acpica/exconvrt.c
@@ -124,9 +124,9 @@ acpi_ex_convert_to_integer(union acpi_operand_object *obj_desc,
 		 * of ACPI 3.0) is that the to_integer() operator allows both decimal
 		 * and hexadecimal strings (hex prefixed with "0x").
 		 */
-		status = acpi_ut_strtoul64((char *)pointer, flags,
-					   acpi_gbl_integer_byte_width,
-					   &result);
+		status = acpi_ut_strtoul64(ACPI_CAST_PTR(char, pointer),
+					   (acpi_gbl_integer_byte_width |
+					    flags), &result);
 		if (ACPI_FAILURE(status)) {
 			return_ACPI_STATUS(status);
 		}
@@ -632,7 +632,7 @@ acpi_ex_convert_to_target_type(acpi_object_type destination_type,
 			 */
 			status =
 			    acpi_ex_convert_to_integer(source_desc, result_desc,
-						       16);
+						       ACPI_STRTOUL_BASE16);
 			break;
 
 		case ACPI_TYPE_STRING:
diff --git a/drivers/acpi/acpica/exmisc.c b/drivers/acpi/acpica/exmisc.c
index 4f7e667624b3..37c88b424a02 100644
--- a/drivers/acpi/acpica/exmisc.c
+++ b/drivers/acpi/acpica/exmisc.c
@@ -327,8 +327,8 @@ acpi_ex_do_logical_op(u16 opcode,
 	switch (operand0->common.type) {
 	case ACPI_TYPE_INTEGER:
 
-		status =
-		    acpi_ex_convert_to_integer(operand1, &local_operand1, 16);
+		status = acpi_ex_convert_to_integer(operand1, &local_operand1,
+						    ACPI_STRTOUL_BASE16);
 		break;
 
 	case ACPI_TYPE_STRING:
diff --git a/drivers/acpi/acpica/exoparg1.c b/drivers/acpi/acpica/exoparg1.c
index 4e17506a7384..007300433cde 100644
--- a/drivers/acpi/acpica/exoparg1.c
+++ b/drivers/acpi/acpica/exoparg1.c
@@ -521,9 +521,10 @@ acpi_status acpi_ex_opcode_1A_1T_1R(struct acpi_walk_state *walk_state)
 
 	case AML_TO_INTEGER_OP:	/* to_integer (Data, Result) */
 
+		/* Perform "explicit" conversion */
+
 		status =
-		    acpi_ex_convert_to_integer(operand[0], &return_desc,
-					       ACPI_ANY_BASE);
+		    acpi_ex_convert_to_integer(operand[0], &return_desc, 0);
 		if (return_desc == operand[0]) {
 
 			/* No conversion performed, add ref to handle return value */
@@ -890,14 +891,16 @@ acpi_status acpi_ex_opcode_1A_0T_1R(struct acpi_walk_state *walk_state)
 				 *    Field, so we need to resolve the node to a value.
 				 */
 				status =
-				    acpi_ns_get_node(walk_state->scope_info->
-						     scope.node,
-						     operand[0]->string.pointer,
-						     ACPI_NS_SEARCH_PARENT,
-						     ACPI_CAST_INDIRECT_PTR
-						     (struct
-						      acpi_namespace_node,
-						      &return_desc));
+				    acpi_ns_get_node_unlocked(walk_state->
+							      scope_info->scope.
+							      node,
+							      operand[0]->
+							      string.pointer,
+							      ACPI_NS_SEARCH_PARENT,
+							      ACPI_CAST_INDIRECT_PTR
+							      (struct
+							       acpi_namespace_node,
+							       &return_desc));
 				if (ACPI_FAILURE(status)) {
 					goto cleanup;
 				}
diff --git a/drivers/acpi/acpica/exresop.c b/drivers/acpi/acpica/exresop.c
index 27b41fd7542d..f29eba1dc5e9 100644
--- a/drivers/acpi/acpica/exresop.c
+++ b/drivers/acpi/acpica/exresop.c
@@ -410,12 +410,13 @@ acpi_ex_resolve_operands(u16 opcode,
 		case ARGI_INTEGER:
 
 			/*
-			 * Need an operand of type ACPI_TYPE_INTEGER,
-			 * But we can implicitly convert from a STRING or BUFFER
-			 * aka - "Implicit Source Operand Conversion"
+			 * Need an operand of type ACPI_TYPE_INTEGER, but we can
+			 * implicitly convert from a STRING or BUFFER.
+			 *
+			 * Known as "Implicit Source Operand Conversion"
 			 */
-			status =
-			    acpi_ex_convert_to_integer(obj_desc, stack_ptr, 16);
+			status = acpi_ex_convert_to_integer(obj_desc, stack_ptr,
+							    ACPI_STRTOUL_BASE16);
 			if (ACPI_FAILURE(status)) {
 				if (status == AE_TYPE) {
 					ACPI_ERROR((AE_INFO,
diff --git a/drivers/acpi/acpica/extrace.c b/drivers/acpi/acpica/extrace.c
index b52e84841c1a..c9ca82610d77 100644
--- a/drivers/acpi/acpica/extrace.c
+++ b/drivers/acpi/acpica/extrace.c
@@ -201,7 +201,6 @@ acpi_ex_start_trace_method(struct acpi_namespace_node *method_node,
 			   union acpi_operand_object *obj_desc,
 			   struct acpi_walk_state *walk_state)
 {
-	acpi_status status;
 	char *pathname = NULL;
 	u8 enabled = FALSE;
 
@@ -211,11 +210,6 @@ acpi_ex_start_trace_method(struct acpi_namespace_node *method_node,
 		pathname = acpi_ns_get_normalized_pathname(method_node, TRUE);
 	}
 
-	status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE);
-	if (ACPI_FAILURE(status)) {
-		goto exit;
-	}
-
 	enabled = acpi_ex_interpreter_trace_enabled(pathname);
 	if (enabled && !acpi_gbl_trace_method_object) {
 		acpi_gbl_trace_method_object = obj_desc;
@@ -233,9 +227,6 @@ acpi_ex_start_trace_method(struct acpi_namespace_node *method_node,
 		}
 	}
 
-	(void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
-
-exit:
 	if (enabled) {
 		ACPI_TRACE_POINT(ACPI_TRACE_AML_METHOD, TRUE,
 				 obj_desc ? obj_desc->method.aml_start : NULL,
@@ -267,7 +258,6 @@ acpi_ex_stop_trace_method(struct acpi_namespace_node *method_node,
 			  union acpi_operand_object *obj_desc,
 			  struct acpi_walk_state *walk_state)
 {
-	acpi_status status;
 	char *pathname = NULL;
 	u8 enabled;
 
@@ -277,26 +267,14 @@ acpi_ex_stop_trace_method(struct acpi_namespace_node *method_node,
 		pathname = acpi_ns_get_normalized_pathname(method_node, TRUE);
 	}
 
-	status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE);
-	if (ACPI_FAILURE(status)) {
-		goto exit_path;
-	}
-
 	enabled = acpi_ex_interpreter_trace_enabled(NULL);
 
-	(void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
-
 	if (enabled) {
 		ACPI_TRACE_POINT(ACPI_TRACE_AML_METHOD, FALSE,
 				 obj_desc ? obj_desc->method.aml_start : NULL,
 				 pathname);
 	}
 
-	status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE);
-	if (ACPI_FAILURE(status)) {
-		goto exit_path;
-	}
-
 	/* Check whether the tracer should be stopped */
 
 	if (acpi_gbl_trace_method_object == obj_desc) {
@@ -312,9 +290,6 @@ acpi_ex_stop_trace_method(struct acpi_namespace_node *method_node,
 		acpi_gbl_trace_method_object = NULL;
 	}
 
-	(void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
-
-exit_path:
 	if (pathname) {
 		ACPI_FREE(pathname);
 	}
diff --git a/drivers/acpi/acpica/exutils.c b/drivers/acpi/acpica/exutils.c
index 425f13372e68..a8b857a7e9fb 100644
--- a/drivers/acpi/acpica/exutils.c
+++ b/drivers/acpi/acpica/exutils.c
@@ -94,6 +94,10 @@ void acpi_ex_enter_interpreter(void)
 		ACPI_ERROR((AE_INFO,
 			    "Could not acquire AML Interpreter mutex"));
 	}
+	status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE);
+	if (ACPI_FAILURE(status)) {
+		ACPI_ERROR((AE_INFO, "Could not acquire AML Namespace mutex"));
+	}
 
 	return_VOID;
 }
@@ -127,6 +131,10 @@ void acpi_ex_exit_interpreter(void)
 
 	ACPI_FUNCTION_TRACE(ex_exit_interpreter);
 
+	status = acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
+	if (ACPI_FAILURE(status)) {
+		ACPI_ERROR((AE_INFO, "Could not release AML Namespace mutex"));
+	}
 	status = acpi_ut_release_mutex(ACPI_MTX_INTERPRETER);
 	if (ACPI_FAILURE(status)) {
 		ACPI_ERROR((AE_INFO,
diff --git a/drivers/acpi/acpica/hwgpe.c b/drivers/acpi/acpica/hwgpe.c
index bdecd5e76e87..76b0e350f5bb 100644
--- a/drivers/acpi/acpica/hwgpe.c
+++ b/drivers/acpi/acpica/hwgpe.c
@@ -98,7 +98,7 @@ acpi_status
 acpi_hw_low_set_gpe(struct acpi_gpe_event_info *gpe_event_info, u32 action)
 {
 	struct acpi_gpe_register_info *gpe_register_info;
-	acpi_status status;
+	acpi_status status = AE_OK;
 	u32 enable_mask;
 	u32 register_bit;
 
@@ -148,9 +148,14 @@ acpi_hw_low_set_gpe(struct acpi_gpe_event_info *gpe_event_info, u32 action)
 		return (AE_BAD_PARAMETER);
 	}
 
-	/* Write the updated enable mask */
+	if (!(register_bit & gpe_register_info->mask_for_run)) {
 
-	status = acpi_hw_write(enable_mask, &gpe_register_info->enable_address);
+		/* Write the updated enable mask */
+
+		status =
+		    acpi_hw_write(enable_mask,
+				  &gpe_register_info->enable_address);
+	}
 	return (status);
 }
 
@@ -242,6 +247,12 @@ acpi_hw_get_gpe_status(struct acpi_gpe_event_info *gpe_event_info,
 		local_event_status |= ACPI_EVENT_FLAG_ENABLED;
 	}
 
+	/* GPE currently masked? (masked for runtime?) */
+
+	if (register_bit & gpe_register_info->mask_for_run) {
+		local_event_status |= ACPI_EVENT_FLAG_MASKED;
+	}
+
 	/* GPE enabled for wake? */
 
 	if (register_bit & gpe_register_info->enable_for_wake) {
@@ -397,6 +408,7 @@ acpi_hw_enable_runtime_gpe_block(struct acpi_gpe_xrupt_info *gpe_xrupt_info,
 	u32 i;
 	acpi_status status;
 	struct acpi_gpe_register_info *gpe_register_info;
+	u8 enable_mask;
 
 	/* NOTE: assumes that all GPEs are currently disabled */
 
@@ -410,9 +422,10 @@ acpi_hw_enable_runtime_gpe_block(struct acpi_gpe_xrupt_info *gpe_xrupt_info,
 
 		/* Enable all "runtime" GPEs in this register */
 
+		enable_mask = gpe_register_info->enable_for_run &
+		    ~gpe_register_info->mask_for_run;
 		status =
-		    acpi_hw_gpe_enable_write(gpe_register_info->enable_for_run,
-					     gpe_register_info);
+		    acpi_hw_gpe_enable_write(enable_mask, gpe_register_info);
 		if (ACPI_FAILURE(status)) {
 			return (status);
 		}
diff --git a/drivers/acpi/acpica/nsaccess.c b/drivers/acpi/acpica/nsaccess.c
index 426a6307eafa..73f98d3fed25 100644
--- a/drivers/acpi/acpica/nsaccess.c
+++ b/drivers/acpi/acpica/nsaccess.c
@@ -108,9 +108,9 @@ acpi_status acpi_ns_root_initialize(void)
 		}
 
 		status =
-		    acpi_ns_lookup(NULL, (char *)init_val->name, init_val->type,
-				   ACPI_IMODE_LOAD_PASS2, ACPI_NS_NO_UPSEARCH,
-				   NULL, &new_node);
+		    acpi_ns_lookup(NULL, ACPI_CAST_PTR(char, init_val->name),
+				   init_val->type, ACPI_IMODE_LOAD_PASS2,
+				   ACPI_NS_NO_UPSEARCH, NULL, &new_node);
 		if (ACPI_FAILURE(status)) {
 			ACPI_EXCEPTION((AE_INFO, status,
 					"Could not create predefined name %s",
diff --git a/drivers/acpi/acpica/nsconvert.c b/drivers/acpi/acpica/nsconvert.c
index c803bda7915c..2b85dee6d4c0 100644
--- a/drivers/acpi/acpica/nsconvert.c
+++ b/drivers/acpi/acpica/nsconvert.c
@@ -79,7 +79,6 @@ acpi_ns_convert_to_integer(union acpi_operand_object *original_object,
 		/* String-to-Integer conversion */
 
 		status = acpi_ut_strtoul64(original_object->string.pointer,
-					   ACPI_ANY_BASE,
 					   acpi_gbl_integer_byte_width, &value);
 		if (ACPI_FAILURE(status)) {
 			return (status);
diff --git a/drivers/acpi/acpica/nsdump.c b/drivers/acpi/acpica/nsdump.c
index ce1f8605d996..84f35dd27033 100644
--- a/drivers/acpi/acpica/nsdump.c
+++ b/drivers/acpi/acpica/nsdump.c
@@ -338,7 +338,7 @@ acpi_ns_dump_one_object(acpi_handle obj_handle,
 		case ACPI_TYPE_STRING:
 
 			acpi_os_printf("Len %.2X ", obj_desc->string.length);
-			acpi_ut_print_string(obj_desc->string.pointer, 32);
+			acpi_ut_print_string(obj_desc->string.pointer, 80);
 			acpi_os_printf("\n");
 			break;
 
diff --git a/drivers/acpi/acpica/nsload.c b/drivers/acpi/acpica/nsload.c
index b5e2b0ada0ab..334d3c5ba617 100644
--- a/drivers/acpi/acpica/nsload.c
+++ b/drivers/acpi/acpica/nsload.c
@@ -46,6 +46,7 @@
 #include "acnamesp.h"
 #include "acdispat.h"
 #include "actables.h"
+#include "acinterp.h"
 
 #define _COMPONENT          ACPI_NAMESPACE
 ACPI_MODULE_NAME("nsload")
@@ -78,20 +79,6 @@ acpi_ns_load_table(u32 table_index, struct acpi_namespace_node *node)
 
 	ACPI_FUNCTION_TRACE(ns_load_table);
 
-	/*
-	 * Parse the table and load the namespace with all named
-	 * objects found within. Control methods are NOT parsed
-	 * at this time. In fact, the control methods cannot be
-	 * parsed until the entire namespace is loaded, because
-	 * if a control method makes a forward reference (call)
-	 * to another control method, we can't continue parsing
-	 * because we don't know how many arguments to parse next!
-	 */
-	status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE);
-	if (ACPI_FAILURE(status)) {
-		return_ACPI_STATUS(status);
-	}
-
 	/* If table already loaded into namespace, just return */
 
 	if (acpi_tb_is_table_loaded(table_index)) {
@@ -107,6 +94,15 @@ acpi_ns_load_table(u32 table_index, struct acpi_namespace_node *node)
 		goto unlock;
 	}
 
+	/*
+	 * Parse the table and load the namespace with all named
+	 * objects found within. Control methods are NOT parsed
+	 * at this time. In fact, the control methods cannot be
+	 * parsed until the entire namespace is loaded, because
+	 * if a control method makes a forward reference (call)
+	 * to another control method, we can't continue parsing
+	 * because we don't know how many arguments to parse next!
+	 */
 	status = acpi_ns_parse_table(table_index, node);
 	if (ACPI_SUCCESS(status)) {
 		acpi_tb_set_table_loaded_flag(table_index, TRUE);
@@ -120,7 +116,6 @@ acpi_ns_load_table(u32 table_index, struct acpi_namespace_node *node)
 		 * exist. This target of Scope must already exist in the
 		 * namespace, as per the ACPI specification.
 		 */
-		(void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
 		acpi_ns_delete_namespace_by_owner(acpi_gbl_root_table_list.
 						  tables[table_index].owner_id);
 
@@ -129,8 +124,6 @@ acpi_ns_load_table(u32 table_index, struct acpi_namespace_node *node)
 	}
 
 unlock:
-	(void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
-
 	if (ACPI_FAILURE(status)) {
 		return_ACPI_STATUS(status);
 	}
@@ -162,7 +155,8 @@ unlock:
 	 * other ACPI implementations. Optionally, the execution can be deferred
 	 * until later, see acpi_initialize_objects.
 	 */
-	if (!acpi_gbl_group_module_level_code) {
+	if (!acpi_gbl_parse_table_as_term_list
+	    && !acpi_gbl_group_module_level_code) {
 		acpi_ns_exec_module_code_list();
 	}
 
diff --git a/drivers/acpi/acpica/nsparse.c b/drivers/acpi/acpica/nsparse.c
index f631a47724f0..4f14e9205bff 100644
--- a/drivers/acpi/acpica/nsparse.c
+++ b/drivers/acpi/acpica/nsparse.c
@@ -47,12 +47,103 @@
 #include "acparser.h"
 #include "acdispat.h"
 #include "actables.h"
+#include "acinterp.h"
 
 #define _COMPONENT          ACPI_NAMESPACE
 ACPI_MODULE_NAME("nsparse")
 
 /*******************************************************************************
  *
+ * FUNCTION:    ns_execute_table
+ *
+ * PARAMETERS:  table_desc      - An ACPI table descriptor for table to parse
+ *              start_node      - Where to enter the table into the namespace
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Load ACPI/AML table by executing the entire table as a
+ *              term_list.
+ *
+ ******************************************************************************/
+acpi_status
+acpi_ns_execute_table(u32 table_index, struct acpi_namespace_node *start_node)
+{
+	acpi_status status;
+	struct acpi_table_header *table;
+	acpi_owner_id owner_id;
+	struct acpi_evaluate_info *info = NULL;
+	u32 aml_length;
+	u8 *aml_start;
+	union acpi_operand_object *method_obj = NULL;
+
+	ACPI_FUNCTION_TRACE(ns_execute_table);
+
+	status = acpi_get_table_by_index(table_index, &table);
+	if (ACPI_FAILURE(status)) {
+		return_ACPI_STATUS(status);
+	}
+
+	/* Table must consist of at least a complete header */
+
+	if (table->length < sizeof(struct acpi_table_header)) {
+		return_ACPI_STATUS(AE_BAD_HEADER);
+	}
+
+	aml_start = (u8 *)table + sizeof(struct acpi_table_header);
+	aml_length = table->length - sizeof(struct acpi_table_header);
+
+	status = acpi_tb_get_owner_id(table_index, &owner_id);
+	if (ACPI_FAILURE(status)) {
+		return_ACPI_STATUS(status);
+	}
+
+	/* Create, initialize, and link a new temporary method object */
+
+	method_obj = acpi_ut_create_internal_object(ACPI_TYPE_METHOD);
+	if (!method_obj) {
+		return_ACPI_STATUS(AE_NO_MEMORY);
+	}
+
+	/* Allocate the evaluation information block */
+
+	info = ACPI_ALLOCATE_ZEROED(sizeof(struct acpi_evaluate_info));
+	if (!info) {
+		status = AE_NO_MEMORY;
+		goto cleanup;
+	}
+
+	ACPI_DEBUG_PRINT((ACPI_DB_PARSE,
+			  "Create table code block: %p\n", method_obj));
+
+	method_obj->method.aml_start = aml_start;
+	method_obj->method.aml_length = aml_length;
+	method_obj->method.owner_id = owner_id;
+	method_obj->method.info_flags |= ACPI_METHOD_MODULE_LEVEL;
+
+	info->pass_number = ACPI_IMODE_EXECUTE;
+	info->node = start_node;
+	info->obj_desc = method_obj;
+	info->node_flags = info->node->flags;
+	info->full_pathname = acpi_ns_get_normalized_pathname(info->node, TRUE);
+	if (!info->full_pathname) {
+		status = AE_NO_MEMORY;
+		goto cleanup;
+	}
+
+	status = acpi_ps_execute_table(info);
+
+cleanup:
+	if (info) {
+		ACPI_FREE(info->full_pathname);
+		info->full_pathname = NULL;
+	}
+	ACPI_FREE(info);
+	acpi_ut_remove_reference(method_obj);
+	return_ACPI_STATUS(status);
+}
+
+/*******************************************************************************
+ *
  * FUNCTION:    ns_one_complete_parse
  *
  * PARAMETERS:  pass_number             - 1 or 2
@@ -63,6 +154,7 @@ ACPI_MODULE_NAME("nsparse")
  * DESCRIPTION: Perform one complete parse of an ACPI/AML table.
  *
  ******************************************************************************/
+
 acpi_status
 acpi_ns_one_complete_parse(u32 pass_number,
 			   u32 table_index,
@@ -143,7 +235,9 @@ acpi_ns_one_complete_parse(u32 pass_number,
 
 	ACPI_DEBUG_PRINT((ACPI_DB_PARSE,
 			  "*PARSE* pass %u parse\n", pass_number));
+	acpi_ex_enter_interpreter();
 	status = acpi_ps_parse_aml(walk_state);
+	acpi_ex_exit_interpreter();
 
 cleanup:
 	acpi_ps_delete_parse_tree(parse_root);
@@ -170,38 +264,47 @@ acpi_ns_parse_table(u32 table_index, struct acpi_namespace_node *start_node)
 
 	ACPI_FUNCTION_TRACE(ns_parse_table);
 
-	/*
-	 * AML Parse, pass 1
-	 *
-	 * In this pass, we load most of the namespace. Control methods
-	 * are not parsed until later. A parse tree is not created. Instead,
-	 * each Parser Op subtree is deleted when it is finished. This saves
-	 * a great deal of memory, and allows a small cache of parse objects
-	 * to service the entire parse. The second pass of the parse then
-	 * performs another complete parse of the AML.
-	 */
-	ACPI_DEBUG_PRINT((ACPI_DB_PARSE, "**** Start pass 1\n"));
-
-	status = acpi_ns_one_complete_parse(ACPI_IMODE_LOAD_PASS1,
-					    table_index, start_node);
-	if (ACPI_FAILURE(status)) {
-		return_ACPI_STATUS(status);
-	}
+	if (acpi_gbl_parse_table_as_term_list) {
+		ACPI_DEBUG_PRINT((ACPI_DB_PARSE, "**** Start load pass\n"));
 
-	/*
-	 * AML Parse, pass 2
-	 *
-	 * In this pass, we resolve forward references and other things
-	 * that could not be completed during the first pass.
-	 * Another complete parse of the AML is performed, but the
-	 * overhead of this is compensated for by the fact that the
-	 * parse objects are all cached.
-	 */
-	ACPI_DEBUG_PRINT((ACPI_DB_PARSE, "**** Start pass 2\n"));
-	status = acpi_ns_one_complete_parse(ACPI_IMODE_LOAD_PASS2,
-					    table_index, start_node);
-	if (ACPI_FAILURE(status)) {
-		return_ACPI_STATUS(status);
+		status = acpi_ns_execute_table(table_index, start_node);
+		if (ACPI_FAILURE(status)) {
+			return_ACPI_STATUS(status);
+		}
+	} else {
+		/*
+		 * AML Parse, pass 1
+		 *
+		 * In this pass, we load most of the namespace. Control methods
+		 * are not parsed until later. A parse tree is not created.
+		 * Instead, each Parser Op subtree is deleted when it is finished.
+		 * This saves a great deal of memory, and allows a small cache of
+		 * parse objects to service the entire parse. The second pass of
+		 * the parse then performs another complete parse of the AML.
+		 */
+		ACPI_DEBUG_PRINT((ACPI_DB_PARSE, "**** Start pass 1\n"));
+
+		status = acpi_ns_one_complete_parse(ACPI_IMODE_LOAD_PASS1,
+						    table_index, start_node);
+		if (ACPI_FAILURE(status)) {
+			return_ACPI_STATUS(status);
+		}
+
+		/*
+		 * AML Parse, pass 2
+		 *
+		 * In this pass, we resolve forward references and other things
+		 * that could not be completed during the first pass.
+		 * Another complete parse of the AML is performed, but the
+		 * overhead of this is compensated for by the fact that the
+		 * parse objects are all cached.
+		 */
+		ACPI_DEBUG_PRINT((ACPI_DB_PARSE, "**** Start pass 2\n"));
+		status = acpi_ns_one_complete_parse(ACPI_IMODE_LOAD_PASS2,
+						    table_index, start_node);
+		if (ACPI_FAILURE(status)) {
+			return_ACPI_STATUS(status);
+		}
 	}
 
 	return_ACPI_STATUS(status);
diff --git a/drivers/acpi/acpica/nsutils.c b/drivers/acpi/acpica/nsutils.c
index 784a30b76e0f..691814dfed31 100644
--- a/drivers/acpi/acpica/nsutils.c
+++ b/drivers/acpi/acpica/nsutils.c
@@ -662,7 +662,7 @@ u32 acpi_ns_opens_scope(acpi_object_type type)
 
 /*******************************************************************************
  *
- * FUNCTION:    acpi_ns_get_node
+ * FUNCTION:    acpi_ns_get_node_unlocked
  *
  * PARAMETERS:  *pathname   - Name to be found, in external (ASL) format. The
  *                            \ (backslash) and ^ (carat) prefixes, and the
@@ -678,20 +678,21 @@ u32 acpi_ns_opens_scope(acpi_object_type type)
  * DESCRIPTION: Look up a name relative to a given scope and return the
  *              corresponding Node. NOTE: Scope can be null.
  *
- * MUTEX:       Locks namespace
+ * MUTEX:       Doesn't locks namespace
  *
  ******************************************************************************/
 
 acpi_status
-acpi_ns_get_node(struct acpi_namespace_node *prefix_node,
-		 const char *pathname,
-		 u32 flags, struct acpi_namespace_node **return_node)
+acpi_ns_get_node_unlocked(struct acpi_namespace_node *prefix_node,
+			  const char *pathname,
+			  u32 flags, struct acpi_namespace_node **return_node)
 {
 	union acpi_generic_state scope_info;
 	acpi_status status;
 	char *internal_path;
 
-	ACPI_FUNCTION_TRACE_PTR(ns_get_node, ACPI_CAST_PTR(char, pathname));
+	ACPI_FUNCTION_TRACE_PTR(ns_get_node_unlocked,
+				ACPI_CAST_PTR(char, pathname));
 
 	/* Simplest case is a null pathname */
 
@@ -718,13 +719,6 @@ acpi_ns_get_node(struct acpi_namespace_node *prefix_node,
 		return_ACPI_STATUS(status);
 	}
 
-	/* Must lock namespace during lookup */
-
-	status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE);
-	if (ACPI_FAILURE(status)) {
-		goto cleanup;
-	}
-
 	/* Setup lookup scope (search starting point) */
 
 	scope_info.scope.node = prefix_node;
@@ -740,9 +734,49 @@ acpi_ns_get_node(struct acpi_namespace_node *prefix_node,
 				  pathname, acpi_format_exception(status)));
 	}
 
-	(void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
-
-cleanup:
 	ACPI_FREE(internal_path);
 	return_ACPI_STATUS(status);
 }
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ns_get_node
+ *
+ * PARAMETERS:  *pathname   - Name to be found, in external (ASL) format. The
+ *                            \ (backslash) and ^ (carat) prefixes, and the
+ *                            . (period) to separate segments are supported.
+ *              prefix_node  - Root of subtree to be searched, or NS_ALL for the
+ *                            root of the name space. If Name is fully
+ *                            qualified (first s8 is '\'), the passed value
+ *                            of Scope will not be accessed.
+ *              flags       - Used to indicate whether to perform upsearch or
+ *                            not.
+ *              return_node - Where the Node is returned
+ *
+ * DESCRIPTION: Look up a name relative to a given scope and return the
+ *              corresponding Node. NOTE: Scope can be null.
+ *
+ * MUTEX:       Locks namespace
+ *
+ ******************************************************************************/
+
+acpi_status
+acpi_ns_get_node(struct acpi_namespace_node *prefix_node,
+		 const char *pathname,
+		 u32 flags, struct acpi_namespace_node **return_node)
+{
+	acpi_status status;
+
+	ACPI_FUNCTION_TRACE_PTR(ns_get_node, ACPI_CAST_PTR(char, pathname));
+
+	status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE);
+	if (ACPI_FAILURE(status)) {
+		return_ACPI_STATUS(status);
+	}
+
+	status = acpi_ns_get_node_unlocked(prefix_node, pathname,
+					   flags, return_node);
+
+	(void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
+	return_ACPI_STATUS(status);
+}
diff --git a/drivers/acpi/acpica/psparse.c b/drivers/acpi/acpica/psparse.c
index 0a23897d8adf..1ce26d9f8ff6 100644
--- a/drivers/acpi/acpica/psparse.c
+++ b/drivers/acpi/acpica/psparse.c
@@ -537,9 +537,11 @@ acpi_status acpi_ps_parse_aml(struct acpi_walk_state *walk_state)
 
 			/* Either the method parse or actual execution failed */
 
+			acpi_ex_exit_interpreter();
 			ACPI_ERROR_METHOD("Method parse/execution failed",
 					  walk_state->method_node, NULL,
 					  status);
+			acpi_ex_enter_interpreter();
 
 			/* Check for possible multi-thread reentrancy problem */
 
@@ -571,7 +573,9 @@ acpi_status acpi_ps_parse_aml(struct acpi_walk_state *walk_state)
 		 * cleanup to do
 		 */
 		if (((walk_state->parse_flags & ACPI_PARSE_MODE_MASK) ==
-		     ACPI_PARSE_EXECUTE) || (ACPI_FAILURE(status))) {
+		     ACPI_PARSE_EXECUTE &&
+		     !(walk_state->parse_flags & ACPI_PARSE_MODULE_LEVEL)) ||
+		    (ACPI_FAILURE(status))) {
 			acpi_ds_terminate_control_method(walk_state->
 							 method_desc,
 							 walk_state);
diff --git a/drivers/acpi/acpica/psxface.c b/drivers/acpi/acpica/psxface.c
index cf30cd821f5e..f3c87264bd1b 100644
--- a/drivers/acpi/acpica/psxface.c
+++ b/drivers/acpi/acpica/psxface.c
@@ -252,6 +252,90 @@ cleanup:
 
 /*******************************************************************************
  *
+ * FUNCTION:    acpi_ps_execute_table
+ *
+ * PARAMETERS:  info            - Method info block, contains:
+ *              node            - Node to where the is entered into the
+ *                                namespace
+ *              obj_desc        - Pseudo method object describing the AML
+ *                                code of the entire table
+ *              pass_number     - Parse or execute pass
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Execute a table
+ *
+ ******************************************************************************/
+
+acpi_status acpi_ps_execute_table(struct acpi_evaluate_info *info)
+{
+	acpi_status status;
+	union acpi_parse_object *op = NULL;
+	struct acpi_walk_state *walk_state = NULL;
+
+	ACPI_FUNCTION_TRACE(ps_execute_table);
+
+	/* Create and init a Root Node */
+
+	op = acpi_ps_create_scope_op(info->obj_desc->method.aml_start);
+	if (!op) {
+		status = AE_NO_MEMORY;
+		goto cleanup;
+	}
+
+	/* Create and initialize a new walk state */
+
+	walk_state =
+	    acpi_ds_create_walk_state(info->obj_desc->method.owner_id, NULL,
+				      NULL, NULL);
+	if (!walk_state) {
+		status = AE_NO_MEMORY;
+		goto cleanup;
+	}
+
+	status = acpi_ds_init_aml_walk(walk_state, op, info->node,
+				       info->obj_desc->method.aml_start,
+				       info->obj_desc->method.aml_length, info,
+				       info->pass_number);
+	if (ACPI_FAILURE(status)) {
+		goto cleanup;
+	}
+
+	if (info->obj_desc->method.info_flags & ACPI_METHOD_MODULE_LEVEL) {
+		walk_state->parse_flags |= ACPI_PARSE_MODULE_LEVEL;
+	}
+
+	/* Info->Node is the default location to load the table  */
+
+	if (info->node && info->node != acpi_gbl_root_node) {
+		status =
+		    acpi_ds_scope_stack_push(info->node, ACPI_TYPE_METHOD,
+					     walk_state);
+		if (ACPI_FAILURE(status)) {
+			goto cleanup;
+		}
+	}
+
+	/*
+	 * Parse the AML, walk_state will be deleted by parse_aml
+	 */
+	acpi_ex_enter_interpreter();
+	status = acpi_ps_parse_aml(walk_state);
+	acpi_ex_exit_interpreter();
+	walk_state = NULL;
+
+cleanup:
+	if (walk_state) {
+		acpi_ds_delete_walk_state(walk_state);
+	}
+	if (op) {
+		acpi_ps_delete_parse_tree(op);
+	}
+	return_ACPI_STATUS(status);
+}
+
+/*******************************************************************************
+ *
  * FUNCTION:    acpi_ps_update_parameter_list
  *
  * PARAMETERS:  info            - See struct acpi_evaluate_info
diff --git a/drivers/acpi/acpica/tbdata.c b/drivers/acpi/acpica/tbdata.c
index 1388a19e5db8..d9ca8c2aa2d3 100644
--- a/drivers/acpi/acpica/tbdata.c
+++ b/drivers/acpi/acpica/tbdata.c
@@ -45,6 +45,7 @@
 #include "accommon.h"
 #include "acnamesp.h"
 #include "actables.h"
+#include "acevents.h"
 
 #define _COMPONENT          ACPI_TABLES
 ACPI_MODULE_NAME("tbdata")
@@ -613,17 +614,12 @@ acpi_status acpi_tb_delete_namespace_by_owner(u32 table_index)
 	 * lock may block, and also since the execution of a namespace walk
 	 * must be allowed to use the interpreter.
 	 */
-	(void)acpi_ut_release_mutex(ACPI_MTX_INTERPRETER);
 	status = acpi_ut_acquire_write_lock(&acpi_gbl_namespace_rw_lock);
-
-	acpi_ns_delete_namespace_by_owner(owner_id);
 	if (ACPI_FAILURE(status)) {
 		return_ACPI_STATUS(status);
 	}
-
+	acpi_ns_delete_namespace_by_owner(owner_id);
 	acpi_ut_release_write_lock(&acpi_gbl_namespace_rw_lock);
-
-	status = acpi_ut_acquire_mutex(ACPI_MTX_INTERPRETER);
 	return_ACPI_STATUS(status);
 }
 
@@ -771,3 +767,142 @@ void acpi_tb_set_table_loaded_flag(u32 table_index, u8 is_loaded)
 
 	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
 }
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_tb_load_table
+ *
+ * PARAMETERS:  table_index             - Table index
+ *              parent_node             - Where table index is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Load an ACPI table
+ *
+ ******************************************************************************/
+
+acpi_status
+acpi_tb_load_table(u32 table_index, struct acpi_namespace_node *parent_node)
+{
+	struct acpi_table_header *table;
+	acpi_status status;
+	acpi_owner_id owner_id;
+
+	ACPI_FUNCTION_TRACE(tb_load_table);
+
+	/*
+	 * Note: Now table is "INSTALLED", it must be validated before
+	 * using.
+	 */
+	status = acpi_get_table_by_index(table_index, &table);
+	if (ACPI_FAILURE(status)) {
+		return_ACPI_STATUS(status);
+	}
+
+	status = acpi_ns_load_table(table_index, parent_node);
+
+	/* Execute any module-level code that was found in the table */
+
+	if (!acpi_gbl_parse_table_as_term_list
+	    && acpi_gbl_group_module_level_code) {
+		acpi_ns_exec_module_code_list();
+	}
+
+	/*
+	 * Update GPEs for any new _Lxx/_Exx methods. Ignore errors. The host is
+	 * responsible for discovering any new wake GPEs by running _PRW methods
+	 * that may have been loaded by this table.
+	 */
+	status = acpi_tb_get_owner_id(table_index, &owner_id);
+	if (ACPI_SUCCESS(status)) {
+		acpi_ev_update_gpes(owner_id);
+	}
+
+	/* Invoke table handler if present */
+
+	if (acpi_gbl_table_handler) {
+		(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_LOAD, table,
+					     acpi_gbl_table_handler_context);
+	}
+
+	return_ACPI_STATUS(status);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_tb_install_and_load_table
+ *
+ * PARAMETERS:  table                   - Pointer to the table
+ *              address                 - Physical address of the table
+ *              flags                   - Allocation flags of the table
+ *              table_index             - Where table index is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Install and load an ACPI table
+ *
+ ******************************************************************************/
+
+acpi_status
+acpi_tb_install_and_load_table(struct acpi_table_header *table,
+			       acpi_physical_address address,
+			       u8 flags, u8 override, u32 *table_index)
+{
+	acpi_status status;
+	u32 i;
+	acpi_owner_id owner_id;
+
+	ACPI_FUNCTION_TRACE(acpi_load_table);
+
+	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
+
+	/* Install the table and load it into the namespace */
+
+	status = acpi_tb_install_standard_table(address, flags, TRUE,
+						override, &i);
+	if (ACPI_FAILURE(status)) {
+		goto unlock_and_exit;
+	}
+
+	/*
+	 * Note: Now table is "INSTALLED", it must be validated before
+	 * using.
+	 */
+	status = acpi_tb_validate_table(&acpi_gbl_root_table_list.tables[i]);
+	if (ACPI_FAILURE(status)) {
+		goto unlock_and_exit;
+	}
+
+	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
+	status = acpi_ns_load_table(i, acpi_gbl_root_node);
+
+	/* Execute any module-level code that was found in the table */
+
+	if (!acpi_gbl_parse_table_as_term_list
+	    && acpi_gbl_group_module_level_code) {
+		acpi_ns_exec_module_code_list();
+	}
+
+	/*
+	 * Update GPEs for any new _Lxx/_Exx methods. Ignore errors. The host is
+	 * responsible for discovering any new wake GPEs by running _PRW methods
+	 * that may have been loaded by this table.
+	 */
+	status = acpi_tb_get_owner_id(i, &owner_id);
+	if (ACPI_SUCCESS(status)) {
+		acpi_ev_update_gpes(owner_id);
+	}
+
+	/* Invoke table handler if present */
+
+	if (acpi_gbl_table_handler) {
+		(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_LOAD, table,
+					     acpi_gbl_table_handler_context);
+	}
+	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
+
+unlock_and_exit:
+	*table_index = i;
+	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
+	return_ACPI_STATUS(status);
+}
diff --git a/drivers/acpi/acpica/tbfadt.c b/drivers/acpi/acpica/tbfadt.c
index 620806965243..046c4d0394ee 100644
--- a/drivers/acpi/acpica/tbfadt.c
+++ b/drivers/acpi/acpica/tbfadt.c
@@ -344,23 +344,27 @@ void acpi_tb_parse_fadt(void)
 
 	/* Obtain the DSDT and FACS tables via their addresses within the FADT */
 
-	acpi_tb_install_fixed_table((acpi_physical_address)acpi_gbl_FADT.Xdsdt,
-				    ACPI_SIG_DSDT, &acpi_gbl_dsdt_index);
+	acpi_tb_install_standard_table((acpi_physical_address)acpi_gbl_FADT.
+				       Xdsdt,
+				       ACPI_TABLE_ORIGIN_INTERNAL_PHYSICAL,
+				       FALSE, TRUE, &acpi_gbl_dsdt_index);
 
 	/* If Hardware Reduced flag is set, there is no FACS */
 
 	if (!acpi_gbl_reduced_hardware) {
 		if (acpi_gbl_FADT.facs) {
-			acpi_tb_install_fixed_table((acpi_physical_address)
-						    acpi_gbl_FADT.facs,
-						    ACPI_SIG_FACS,
-						    &acpi_gbl_facs_index);
+			acpi_tb_install_standard_table((acpi_physical_address)
+						       acpi_gbl_FADT.facs,
+						       ACPI_TABLE_ORIGIN_INTERNAL_PHYSICAL,
+						       FALSE, TRUE,
+						       &acpi_gbl_facs_index);
 		}
 		if (acpi_gbl_FADT.Xfacs) {
-			acpi_tb_install_fixed_table((acpi_physical_address)
-						    acpi_gbl_FADT.Xfacs,
-						    ACPI_SIG_FACS,
-						    &acpi_gbl_xfacs_index);
+			acpi_tb_install_standard_table((acpi_physical_address)
+						       acpi_gbl_FADT.Xfacs,
+						       ACPI_TABLE_ORIGIN_INTERNAL_PHYSICAL,
+						       FALSE, TRUE,
+						       &acpi_gbl_xfacs_index);
 		}
 	}
 }
@@ -476,17 +480,19 @@ static void acpi_tb_convert_fadt(void)
 	u32 i;
 
 	/*
-	 * For ACPI 1.0 FADTs (revision 1 or 2), ensure that reserved fields which
+	 * For ACPI 1.0 FADTs (revision 1), ensure that reserved fields which
 	 * should be zero are indeed zero. This will workaround BIOSs that
 	 * inadvertently place values in these fields.
 	 *
 	 * The ACPI 1.0 reserved fields that will be zeroed are the bytes located
 	 * at offset 45, 55, 95, and the word located at offset 109, 110.
 	 *
-	 * Note: The FADT revision value is unreliable. Only the length can be
-	 * trusted.
+	 * Note: The FADT revision value is unreliable because of BIOS errors.
+	 * The table length is instead used as the final word on the version.
+	 *
+	 * Note: FADT revision 3 is the ACPI 2.0 version of the FADT.
 	 */
-	if (acpi_gbl_FADT.header.length <= ACPI_FADT_V2_SIZE) {
+	if (acpi_gbl_FADT.header.length <= ACPI_FADT_V3_SIZE) {
 		acpi_gbl_FADT.preferred_profile = 0;
 		acpi_gbl_FADT.pstate_control = 0;
 		acpi_gbl_FADT.cst_control = 0;
@@ -552,78 +558,74 @@ static void acpi_tb_convert_fadt(void)
 		 *
 		 * Address32 zero, Address64 [don't care]   - Use Address64
 		 *
+		 * No override: if acpi_gbl_use32_bit_fadt_addresses is FALSE, and:
 		 * Address32 non-zero, Address64 zero       - Copy/use Address32
 		 * Address32 non-zero == Address64 non-zero - Use Address64
 		 * Address32 non-zero != Address64 non-zero - Warning, use Address64
 		 *
 		 * Override: if acpi_gbl_use32_bit_fadt_addresses is TRUE, and:
+		 * Address32 non-zero, Address64 zero       - Copy/use Address32
+		 * Address32 non-zero == Address64 non-zero - Copy/use Address32
 		 * Address32 non-zero != Address64 non-zero - Warning, copy/use Address32
 		 *
 		 * Note: space_id is always I/O for 32-bit legacy address fields
 		 */
 		if (address32) {
-			if (!address64->address) {
+			if (address64->address) {
+				if (address64->address != (u64)address32) {
+
+					/* Address mismatch */
+
+					ACPI_BIOS_WARNING((AE_INFO,
+							   "32/64X address mismatch in FADT/%s: "
+							   "0x%8.8X/0x%8.8X%8.8X, using %u-bit address",
+							   name, address32,
+							   ACPI_FORMAT_UINT64
+							   (address64->address),
+							   acpi_gbl_use32_bit_fadt_addresses
+							   ? 32 : 64));
+				}
 
-				/* 64-bit address is zero, use 32-bit address */
+				/*
+				 * For each extended field, check for length mismatch
+				 * between the legacy length field and the corresponding
+				 * 64-bit X length field.
+				 * Note: If the legacy length field is > 0xFF bits, ignore
+				 * this check. (GPE registers can be larger than the
+				 * 64-bit GAS structure can accomodate, 0xFF bits).
+				 */
+				if ((ACPI_MUL_8(length) <= ACPI_UINT8_MAX) &&
+				    (address64->bit_width !=
+				     ACPI_MUL_8(length))) {
+					ACPI_BIOS_WARNING((AE_INFO,
+							   "32/64X length mismatch in FADT/%s: %u/%u",
+							   name,
+							   ACPI_MUL_8(length),
+							   address64->
+							   bit_width));
+				}
+			}
 
+			/*
+			 * Hardware register access code always uses the 64-bit fields.
+			 * So if the 64-bit field is zero or is to be overridden,
+			 * initialize it with the 32-bit fields.
+			 * Note that when the 32-bit address favor is specified, the
+			 * 64-bit fields are always re-initialized so that
+			 * access_size/bit_width/bit_offset fields can be correctly
+			 * configured to the values to trigger a 32-bit compatible
+			 * access mode in the hardware register access code.
+			 */
+			if (!address64->address
+			    || acpi_gbl_use32_bit_fadt_addresses) {
 				acpi_tb_init_generic_address(address64,
 							     ACPI_ADR_SPACE_SYSTEM_IO,
-							     *ACPI_ADD_PTR(u8,
-									   &acpi_gbl_FADT,
-									   fadt_info_table
-									   [i].
-									   length),
+							     length,
 							     (u64)address32,
 							     name, flags);
-			} else if (address64->address != (u64)address32) {
-
-				/* Address mismatch */
-
-				ACPI_BIOS_WARNING((AE_INFO,
-						   "32/64X address mismatch in FADT/%s: "
-						   "0x%8.8X/0x%8.8X%8.8X, using %u-bit address",
-						   name, address32,
-						   ACPI_FORMAT_UINT64
-						   (address64->address),
-						   acpi_gbl_use32_bit_fadt_addresses
-						   ? 32 : 64));
-
-				if (acpi_gbl_use32_bit_fadt_addresses) {
-
-					/* 32-bit address override */
-
-					acpi_tb_init_generic_address(address64,
-								     ACPI_ADR_SPACE_SYSTEM_IO,
-								     *ACPI_ADD_PTR
-								     (u8,
-								      &acpi_gbl_FADT,
-								      fadt_info_table
-								      [i].
-								      length),
-								     (u64)
-								     address32,
-								     name,
-								     flags);
-				}
 			}
 		}
 
-		/*
-		 * For each extended field, check for length mismatch between the
-		 * legacy length field and the corresponding 64-bit X length field.
-		 * Note: If the legacy length field is > 0xFF bits, ignore this
-		 * check. (GPE registers can be larger than the 64-bit GAS structure
-		 * can accomodate, 0xFF bits).
-		 */
-		if (address64->address &&
-		    (ACPI_MUL_8(length) <= ACPI_UINT8_MAX) &&
-		    (address64->bit_width != ACPI_MUL_8(length))) {
-			ACPI_BIOS_WARNING((AE_INFO,
-					   "32/64X length mismatch in FADT/%s: %u/%u",
-					   name, ACPI_MUL_8(length),
-					   address64->bit_width));
-		}
-
 		if (fadt_info_table[i].flags & ACPI_FADT_REQUIRED) {
 			/*
 			 * Field is required (Pm1a_event, Pm1a_control).
diff --git a/drivers/acpi/acpica/tbfind.c b/drivers/acpi/acpica/tbfind.c
index e348d616e60f..f6b9b4e4298b 100644
--- a/drivers/acpi/acpica/tbfind.c
+++ b/drivers/acpi/acpica/tbfind.c
@@ -68,7 +68,7 @@ acpi_status
 acpi_tb_find_table(char *signature,
 		   char *oem_id, char *oem_table_id, u32 *table_index)
 {
-	acpi_status status;
+	acpi_status status = AE_OK;
 	struct acpi_table_header header;
 	u32 i;
 
@@ -96,6 +96,7 @@ acpi_tb_find_table(char *signature,
 
 	/* Search for the table */
 
+	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 	for (i = 0; i < acpi_gbl_root_table_list.current_table_count; ++i) {
 		if (memcmp(&(acpi_gbl_root_table_list.tables[i].signature),
 			   header.signature, ACPI_NAME_SIZE)) {
@@ -115,7 +116,7 @@ acpi_tb_find_table(char *signature,
 			    acpi_tb_validate_table(&acpi_gbl_root_table_list.
 						   tables[i]);
 			if (ACPI_FAILURE(status)) {
-				return_ACPI_STATUS(status);
+				goto unlock_and_exit;
 			}
 
 			if (!acpi_gbl_root_table_list.tables[i].pointer) {
@@ -144,9 +145,12 @@ acpi_tb_find_table(char *signature,
 			ACPI_DEBUG_PRINT((ACPI_DB_TABLES,
 					  "Found table [%4.4s]\n",
 					  header.signature));
-			return_ACPI_STATUS(AE_OK);
+			goto unlock_and_exit;
 		}
 	}
+	status = AE_NOT_FOUND;
 
-	return_ACPI_STATUS(AE_NOT_FOUND);
+unlock_and_exit:
+	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
+	return_ACPI_STATUS(status);
 }
diff --git a/drivers/acpi/acpica/tbinstal.c b/drivers/acpi/acpica/tbinstal.c
index 8b13052128fc..5fdf251a9f97 100644
--- a/drivers/acpi/acpica/tbinstal.c
+++ b/drivers/acpi/acpica/tbinstal.c
@@ -157,68 +157,6 @@ acpi_tb_install_table_with_override(struct acpi_table_desc *new_table_desc,
 
 /*******************************************************************************
  *
- * FUNCTION:    acpi_tb_install_fixed_table
- *
- * PARAMETERS:  address                 - Physical address of DSDT or FACS
- *              signature               - Table signature, NULL if no need to
- *                                        match
- *              table_index             - Where the table index is returned
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Install a fixed ACPI table (DSDT/FACS) into the global data
- *              structure.
- *
- ******************************************************************************/
-
-acpi_status
-acpi_tb_install_fixed_table(acpi_physical_address address,
-			    char *signature, u32 *table_index)
-{
-	struct acpi_table_desc new_table_desc;
-	acpi_status status;
-
-	ACPI_FUNCTION_TRACE(tb_install_fixed_table);
-
-	if (!address) {
-		ACPI_ERROR((AE_INFO,
-			    "Null physical address for ACPI table [%s]",
-			    signature));
-		return (AE_NO_MEMORY);
-	}
-
-	/* Fill a table descriptor for validation */
-
-	status = acpi_tb_acquire_temp_table(&new_table_desc, address,
-					    ACPI_TABLE_ORIGIN_INTERNAL_PHYSICAL);
-	if (ACPI_FAILURE(status)) {
-		ACPI_ERROR((AE_INFO,
-			    "Could not acquire table length at %8.8X%8.8X",
-			    ACPI_FORMAT_UINT64(address)));
-		return_ACPI_STATUS(status);
-	}
-
-	/* Validate and verify a table before installation */
-
-	status = acpi_tb_verify_temp_table(&new_table_desc, signature);
-	if (ACPI_FAILURE(status)) {
-		goto release_and_exit;
-	}
-
-	/* Add the table to the global root table list */
-
-	acpi_tb_install_table_with_override(&new_table_desc, TRUE, table_index);
-
-release_and_exit:
-
-	/* Release the temporary table descriptor */
-
-	acpi_tb_release_temp_table(&new_table_desc);
-	return_ACPI_STATUS(status);
-}
-
-/*******************************************************************************
- *
  * FUNCTION:    acpi_tb_install_standard_table
  *
  * PARAMETERS:  address             - Address of the table (might be a virtual
@@ -230,8 +168,7 @@ release_and_exit:
  *
  * RETURN:      Status
  *
- * DESCRIPTION: This function is called to install an ACPI table that is
- *              neither DSDT nor FACS (a "standard" table.)
+ * DESCRIPTION: This function is called to verify and install an ACPI table.
  *              When this function is called by "Load" or "LoadTable" opcodes,
  *              or by acpi_load_table() API, the "Reload" parameter is set.
  *              After sucessfully returning from this function, table is
@@ -364,6 +301,14 @@ acpi_tb_install_standard_table(acpi_physical_address address,
 	acpi_tb_install_table_with_override(&new_table_desc, override,
 					    table_index);
 
+	/* Invoke table handler if present */
+
+	if (acpi_gbl_table_handler) {
+		(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_INSTALL,
+					     new_table_desc.pointer,
+					     acpi_gbl_table_handler_context);
+	}
+
 release_and_exit:
 
 	/* Release the temporary table descriptor */
diff --git a/drivers/acpi/acpica/tbutils.c b/drivers/acpi/acpica/tbutils.c
index e28553914bf5..51eb07cf9898 100644
--- a/drivers/acpi/acpica/tbutils.c
+++ b/drivers/acpi/acpica/tbutils.c
@@ -252,7 +252,8 @@ acpi_tb_get_root_table_entry(u8 *table_entry, u32 table_entry_size)
  *
  ******************************************************************************/
 
-acpi_status __init acpi_tb_parse_root_table(acpi_physical_address rsdp_address)
+acpi_status ACPI_INIT_FUNCTION
+acpi_tb_parse_root_table(acpi_physical_address rsdp_address)
 {
 	struct acpi_table_rsdp *rsdp;
 	u32 table_entry_size;
diff --git a/drivers/acpi/acpica/tbxface.c b/drivers/acpi/acpica/tbxface.c
index 3ecec937e8c9..4ab6b9cd0aec 100644
--- a/drivers/acpi/acpica/tbxface.c
+++ b/drivers/acpi/acpica/tbxface.c
@@ -98,7 +98,7 @@ acpi_status acpi_allocate_root_table(u32 initial_table_count)
  *
  ******************************************************************************/
 
-acpi_status __init
+acpi_status ACPI_INIT_FUNCTION
 acpi_initialize_tables(struct acpi_table_desc *initial_table_array,
 		       u32 initial_table_count, u8 allow_resize)
 {
@@ -164,7 +164,7 @@ ACPI_EXPORT_SYMBOL_INIT(acpi_initialize_tables)
  *              kernel.
  *
  ******************************************************************************/
-acpi_status __init acpi_reallocate_root_table(void)
+acpi_status ACPI_INIT_FUNCTION acpi_reallocate_root_table(void)
 {
 	acpi_status status;
 
diff --git a/drivers/acpi/acpica/tbxfload.c b/drivers/acpi/acpica/tbxfload.c
index ac71abcd32bb..5569f637f669 100644
--- a/drivers/acpi/acpica/tbxfload.c
+++ b/drivers/acpi/acpica/tbxfload.c
@@ -63,7 +63,7 @@ ACPI_MODULE_NAME("tbxfload")
  * DESCRIPTION: Load the ACPI tables from the RSDT/XSDT
  *
  ******************************************************************************/
-acpi_status __init acpi_load_tables(void)
+acpi_status ACPI_INIT_FUNCTION acpi_load_tables(void)
 {
 	acpi_status status;
 
@@ -103,7 +103,8 @@ acpi_status __init acpi_load_tables(void)
 				"While loading namespace from ACPI tables"));
 	}
 
-	if (!acpi_gbl_group_module_level_code) {
+	if (acpi_gbl_parse_table_as_term_list
+	    || !acpi_gbl_group_module_level_code) {
 		/*
 		 * Initialize the objects that remain uninitialized. This
 		 * runs the executable AML that may be part of the
@@ -188,11 +189,11 @@ acpi_status acpi_tb_load_namespace(void)
 	memcpy(&acpi_gbl_original_dsdt_header, acpi_gbl_DSDT,
 	       sizeof(struct acpi_table_header));
 
-	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
-
 	/* Load and parse tables */
 
+	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
 	status = acpi_ns_load_table(acpi_gbl_dsdt_index, acpi_gbl_root_node);
+	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 	if (ACPI_FAILURE(status)) {
 		ACPI_EXCEPTION((AE_INFO, status, "[DSDT] table load failed"));
 		tables_failed++;
@@ -202,7 +203,6 @@ acpi_status acpi_tb_load_namespace(void)
 
 	/* Load any SSDT or PSDT tables. Note: Loop leaves tables locked */
 
-	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 	for (i = 0; i < acpi_gbl_root_table_list.current_table_count; ++i) {
 		table = &acpi_gbl_root_table_list.tables[i];
 
@@ -220,6 +220,7 @@ acpi_status acpi_tb_load_namespace(void)
 
 		(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
 		status = acpi_ns_load_table(i, acpi_gbl_root_node);
+		(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 		if (ACPI_FAILURE(status)) {
 			ACPI_EXCEPTION((AE_INFO, status,
 					"(%4.4s:%8.8s) while loading table",
@@ -235,8 +236,6 @@ acpi_status acpi_tb_load_namespace(void)
 		} else {
 			tables_loaded++;
 		}
-
-		(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 	}
 
 	if (!tables_failed) {
@@ -272,7 +271,7 @@ unlock_and_exit:
  *
  ******************************************************************************/
 
-acpi_status __init
+acpi_status ACPI_INIT_FUNCTION
 acpi_install_table(acpi_physical_address address, u8 physical)
 {
 	acpi_status status;
@@ -324,49 +323,13 @@ acpi_status acpi_load_table(struct acpi_table_header *table)
 		return_ACPI_STATUS(AE_BAD_PARAMETER);
 	}
 
-	/* Must acquire the interpreter lock during this operation */
-
-	status = acpi_ut_acquire_mutex(ACPI_MTX_INTERPRETER);
-	if (ACPI_FAILURE(status)) {
-		return_ACPI_STATUS(status);
-	}
-
 	/* Install the table and load it into the namespace */
 
 	ACPI_INFO(("Host-directed Dynamic ACPI Table Load:"));
-	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
-
-	status = acpi_tb_install_standard_table(ACPI_PTR_TO_PHYSADDR(table),
-						ACPI_TABLE_ORIGIN_EXTERNAL_VIRTUAL,
-						TRUE, FALSE, &table_index);
-
-	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
-	if (ACPI_FAILURE(status)) {
-		goto unlock_and_exit;
-	}
-
-	/*
-	 * Note: Now table is "INSTALLED", it must be validated before
-	 * using.
-	 */
 	status =
-	    acpi_tb_validate_table(&acpi_gbl_root_table_list.
-				   tables[table_index]);
-	if (ACPI_FAILURE(status)) {
-		goto unlock_and_exit;
-	}
-
-	status = acpi_ns_load_table(table_index, acpi_gbl_root_node);
-
-	/* Invoke table handler if present */
-
-	if (acpi_gbl_table_handler) {
-		(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_LOAD, table,
-					     acpi_gbl_table_handler_context);
-	}
-
-unlock_and_exit:
-	(void)acpi_ut_release_mutex(ACPI_MTX_INTERPRETER);
+	    acpi_tb_install_and_load_table(table, ACPI_PTR_TO_PHYSADDR(table),
+					   ACPI_TABLE_ORIGIN_EXTERNAL_VIRTUAL,
+					   FALSE, &table_index);
 	return_ACPI_STATUS(status);
 }
 
@@ -415,9 +378,9 @@ acpi_status acpi_unload_parent_table(acpi_handle object)
 		return_ACPI_STATUS(AE_TYPE);
 	}
 
-	/* Must acquire the interpreter lock during this operation */
+	/* Must acquire the table lock during this operation */
 
-	status = acpi_ut_acquire_mutex(ACPI_MTX_INTERPRETER);
+	status = acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 	if (ACPI_FAILURE(status)) {
 		return_ACPI_STATUS(status);
 	}
@@ -444,8 +407,10 @@ acpi_status acpi_unload_parent_table(acpi_handle object)
 
 		/* Ensure the table is actually loaded */
 
+		(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
 		if (!acpi_tb_is_table_loaded(i)) {
 			status = AE_NOT_EXIST;
+			(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 			break;
 		}
 
@@ -471,10 +436,11 @@ acpi_status acpi_unload_parent_table(acpi_handle object)
 
 		status = acpi_tb_release_owner_id(i);
 		acpi_tb_set_table_loaded_flag(i, FALSE);
+		(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 		break;
 	}
 
-	(void)acpi_ut_release_mutex(ACPI_MTX_INTERPRETER);
+	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
 	return_ACPI_STATUS(status);
 }
 
diff --git a/drivers/acpi/acpica/tbxfroot.c b/drivers/acpi/acpica/tbxfroot.c
index adb6cfc54661..0adb1c78d863 100644
--- a/drivers/acpi/acpica/tbxfroot.c
+++ b/drivers/acpi/acpica/tbxfroot.c
@@ -142,7 +142,8 @@ acpi_status acpi_tb_validate_rsdp(struct acpi_table_rsdp *rsdp)
  *
  ******************************************************************************/
 
-acpi_status __init acpi_find_root_pointer(acpi_physical_address *table_address)
+acpi_status ACPI_INIT_FUNCTION
+acpi_find_root_pointer(acpi_physical_address *table_address)
 {
 	u8 *table_ptr;
 	u8 *mem_rover;
@@ -244,6 +245,8 @@ acpi_status __init acpi_find_root_pointer(acpi_physical_address *table_address)
 	return_ACPI_STATUS(AE_NOT_FOUND);
 }
 
+ACPI_EXPORT_SYMBOL_INIT(acpi_find_root_pointer)
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_tb_scan_memory_for_rsdp
diff --git a/drivers/acpi/acpica/utaddress.c b/drivers/acpi/acpica/utaddress.c
index c986ec66a118..433d822798b6 100644
--- a/drivers/acpi/acpica/utaddress.c
+++ b/drivers/acpi/acpica/utaddress.c
@@ -77,7 +77,6 @@ acpi_ut_add_address_range(acpi_adr_space_type space_id,
 			  u32 length, struct acpi_namespace_node *region_node)
 {
 	struct acpi_address_range *range_info;
-	acpi_status status;
 
 	ACPI_FUNCTION_TRACE(ut_add_address_range);
 
@@ -97,12 +96,6 @@ acpi_ut_add_address_range(acpi_adr_space_type space_id,
 	range_info->end_address = (address + length - 1);
 	range_info->region_node = region_node;
 
-	status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE);
-	if (ACPI_FAILURE(status)) {
-		ACPI_FREE(range_info);
-		return_ACPI_STATUS(status);
-	}
-
 	range_info->next = acpi_gbl_address_range_list[space_id];
 	acpi_gbl_address_range_list[space_id] = range_info;
 
@@ -112,7 +105,6 @@ acpi_ut_add_address_range(acpi_adr_space_type space_id,
 			  ACPI_FORMAT_UINT64(address),
 			  ACPI_FORMAT_UINT64(range_info->end_address)));
 
-	(void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
 	return_ACPI_STATUS(AE_OK);
 }
 
diff --git a/drivers/acpi/acpica/utbuffer.c b/drivers/acpi/acpica/utbuffer.c
index bd31faf5da7c..ff2981275b9a 100644
--- a/drivers/acpi/acpica/utbuffer.c
+++ b/drivers/acpi/acpica/utbuffer.c
@@ -239,8 +239,7 @@ acpi_ut_dump_buffer_to_file(ACPI_FILE file,
 	u8 buf_char;
 
 	if (!buffer) {
-		acpi_ut_file_printf(file,
-				    "Null Buffer Pointer in DumpBuffer!\n");
+		fprintf(file, "Null Buffer Pointer in DumpBuffer!\n");
 		return;
 	}
 
@@ -254,7 +253,7 @@ acpi_ut_dump_buffer_to_file(ACPI_FILE file,
 
 		/* Print current offset */
 
-		acpi_ut_file_printf(file, "%6.4X: ", (base_offset + i));
+		fprintf(file, "%6.4X: ", (base_offset + i));
 
 		/* Print 16 hex chars */
 
@@ -263,8 +262,7 @@ acpi_ut_dump_buffer_to_file(ACPI_FILE file,
 
 				/* Dump fill spaces */
 
-				acpi_ut_file_printf(file, "%*s",
-						    ((display * 2) + 1), " ");
+				fprintf(file, "%*s", ((display * 2) + 1), " ");
 				j += display;
 				continue;
 			}
@@ -273,34 +271,34 @@ acpi_ut_dump_buffer_to_file(ACPI_FILE file,
 			case DB_BYTE_DISPLAY:
 			default:	/* Default is BYTE display */
 
-				acpi_ut_file_printf(file, "%02X ",
-						    buffer[(acpi_size)i + j]);
+				fprintf(file, "%02X ",
+					buffer[(acpi_size)i + j]);
 				break;
 
 			case DB_WORD_DISPLAY:
 
 				ACPI_MOVE_16_TO_32(&temp32,
 						   &buffer[(acpi_size)i + j]);
-				acpi_ut_file_printf(file, "%04X ", temp32);
+				fprintf(file, "%04X ", temp32);
 				break;
 
 			case DB_DWORD_DISPLAY:
 
 				ACPI_MOVE_32_TO_32(&temp32,
 						   &buffer[(acpi_size)i + j]);
-				acpi_ut_file_printf(file, "%08X ", temp32);
+				fprintf(file, "%08X ", temp32);
 				break;
 
 			case DB_QWORD_DISPLAY:
 
 				ACPI_MOVE_32_TO_32(&temp32,
 						   &buffer[(acpi_size)i + j]);
-				acpi_ut_file_printf(file, "%08X", temp32);
+				fprintf(file, "%08X", temp32);
 
 				ACPI_MOVE_32_TO_32(&temp32,
 						   &buffer[(acpi_size)i + j +
 							   4]);
-				acpi_ut_file_printf(file, "%08X ", temp32);
+				fprintf(file, "%08X ", temp32);
 				break;
 			}
 
@@ -311,24 +309,24 @@ acpi_ut_dump_buffer_to_file(ACPI_FILE file,
 		 * Print the ASCII equivalent characters but watch out for the bad
 		 * unprintable ones (printable chars are 0x20 through 0x7E)
 		 */
-		acpi_ut_file_printf(file, " ");
+		fprintf(file, " ");
 		for (j = 0; j < 16; j++) {
 			if (i + j >= count) {
-				acpi_ut_file_printf(file, "\n");
+				fprintf(file, "\n");
 				return;
 			}
 
 			buf_char = buffer[(acpi_size)i + j];
 			if (isprint(buf_char)) {
-				acpi_ut_file_printf(file, "%c", buf_char);
+				fprintf(file, "%c", buf_char);
 			} else {
-				acpi_ut_file_printf(file, ".");
+				fprintf(file, ".");
 			}
 		}
 
 		/* Done with that line. */
 
-		acpi_ut_file_printf(file, "\n");
+		fprintf(file, "\n");
 		i += 16;
 	}
 
diff --git a/drivers/acpi/acpica/utdebug.c b/drivers/acpi/acpica/utdebug.c
index 574422205005..044df9b0356e 100644
--- a/drivers/acpi/acpica/utdebug.c
+++ b/drivers/acpi/acpica/utdebug.c
@@ -562,6 +562,43 @@ acpi_ut_ptr_exit(u32 line_number,
 
 /*******************************************************************************
  *
+ * FUNCTION:    acpi_ut_str_exit
+ *
+ * PARAMETERS:  line_number         - Caller's line number
+ *              function_name       - Caller's procedure name
+ *              module_name         - Caller's module name
+ *              component_id        - Caller's component ID
+ *              string              - String to display
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Function exit trace. Prints only if TRACE_FUNCTIONS bit is
+ *              set in debug_level. Prints exit value also.
+ *
+ ******************************************************************************/
+
+void
+acpi_ut_str_exit(u32 line_number,
+		 const char *function_name,
+		 const char *module_name, u32 component_id, const char *string)
+{
+
+	/* Check if enabled up-front for performance */
+
+	if (ACPI_IS_DEBUG_ENABLED(ACPI_LV_FUNCTIONS, component_id)) {
+		acpi_debug_print(ACPI_LV_FUNCTIONS,
+				 line_number, function_name, module_name,
+				 component_id, "%s %s\n",
+				 acpi_gbl_function_exit_prefix, string);
+	}
+
+	if (acpi_gbl_nesting_level) {
+		acpi_gbl_nesting_level--;
+	}
+}
+
+/*******************************************************************************
+ *
  * FUNCTION:    acpi_trace_point
  *
  * PARAMETERS:  type                - Trace event type
@@ -591,27 +628,3 @@ acpi_trace_point(acpi_trace_event_type type, u8 begin, u8 *aml, char *pathname)
 
 ACPI_EXPORT_SYMBOL(acpi_trace_point)
 #endif
-#ifdef ACPI_APPLICATION
-/*******************************************************************************
- *
- * FUNCTION:    acpi_log_error
- *
- * PARAMETERS:  format              - Printf format field
- *              ...                 - Optional printf arguments
- *
- * RETURN:      None
- *
- * DESCRIPTION: Print error message to the console, used by applications.
- *
- ******************************************************************************/
-void ACPI_INTERNAL_VAR_XFACE acpi_log_error(const char *format, ...)
-{
-	va_list args;
-
-	va_start(args, format);
-	(void)acpi_ut_file_vprintf(ACPI_FILE_ERR, format, args);
-	va_end(args);
-}
-
-ACPI_EXPORT_SYMBOL(acpi_log_error)
-#endif
diff --git a/drivers/acpi/acpica/utdecode.c b/drivers/acpi/acpica/utdecode.c
index efd7988e34cb..15728ad8356b 100644
--- a/drivers/acpi/acpica/utdecode.c
+++ b/drivers/acpi/acpica/utdecode.c
@@ -253,7 +253,7 @@ const char *acpi_ut_get_object_type_name(union acpi_operand_object *obj_desc)
 		return_PTR("Invalid object");
 	}
 
-	return_PTR(acpi_ut_get_type_name(obj_desc->common.type));
+	return_STR(acpi_ut_get_type_name(obj_desc->common.type));
 }
 
 /*******************************************************************************
diff --git a/drivers/acpi/acpica/uthex.c b/drivers/acpi/acpica/uthex.c
index 4354fb800fe4..36d2fc789088 100644
--- a/drivers/acpi/acpica/uthex.c
+++ b/drivers/acpi/acpica/uthex.c
@@ -75,9 +75,41 @@ char acpi_ut_hex_to_ascii_char(u64 integer, u32 position)
 
 /*******************************************************************************
  *
+ * FUNCTION:    acpi_ut_ascii_to_hex_byte
+ *
+ * PARAMETERS:  two_ascii_chars             - Pointer to two ASCII characters
+ *              return_byte                 - Where converted byte is returned
+ *
+ * RETURN:      Status and converted hex byte
+ *
+ * DESCRIPTION: Perform ascii-to-hex translation, exactly two ASCII characters
+ *              to a single converted byte value.
+ *
+ ******************************************************************************/
+
+acpi_status acpi_ut_ascii_to_hex_byte(char *two_ascii_chars, u8 *return_byte)
+{
+
+	/* Both ASCII characters must be valid hex digits */
+
+	if (!isxdigit((int)two_ascii_chars[0]) ||
+	    !isxdigit((int)two_ascii_chars[1])) {
+		return (AE_BAD_HEX_CONSTANT);
+	}
+
+	*return_byte =
+	    acpi_ut_ascii_char_to_hex(two_ascii_chars[1]) |
+	    (acpi_ut_ascii_char_to_hex(two_ascii_chars[0]) << 4);
+
+	return (AE_OK);
+}
+
+/*******************************************************************************
+ *
  * FUNCTION:    acpi_ut_ascii_char_to_hex
  *
- * PARAMETERS:  hex_char                - Hex character in Ascii
+ * PARAMETERS:  hex_char                - Hex character in Ascii. Must be:
+ *                                        0-9 or A-F or a-f
  *
  * RETURN:      The binary value of the ascii/hex character
  *
@@ -88,13 +120,19 @@ char acpi_ut_hex_to_ascii_char(u64 integer, u32 position)
 u8 acpi_ut_ascii_char_to_hex(int hex_char)
 {
 
-	if (hex_char <= 0x39) {
-		return ((u8)(hex_char - 0x30));
+	/* Values 0-9 */
+
+	if (hex_char <= '9') {
+		return ((u8)(hex_char - '0'));
 	}
 
-	if (hex_char <= 0x46) {
+	/* Upper case A-F */
+
+	if (hex_char <= 'F') {
 		return ((u8)(hex_char - 0x37));
 	}
 
+	/* Lower case a-f */
+
 	return ((u8)(hex_char - 0x57));
 }
diff --git a/drivers/acpi/acpica/utinit.c b/drivers/acpi/acpica/utinit.c
index f91f724c487c..1711fdf41709 100644
--- a/drivers/acpi/acpica/utinit.c
+++ b/drivers/acpi/acpica/utinit.c
@@ -206,7 +206,7 @@ acpi_status acpi_ut_init_globals(void)
 	acpi_gbl_next_owner_id_offset = 0;
 	acpi_gbl_debugger_configuration = DEBUGGER_THREADING;
 	acpi_gbl_osi_mutex = NULL;
-	acpi_gbl_max_loop_iterations = 0xFFFF;
+	acpi_gbl_max_loop_iterations = ACPI_MAX_LOOP_COUNT;
 
 	/* Hardware oriented */
 
diff --git a/drivers/acpi/acpica/utnonansi.c b/drivers/acpi/acpica/utnonansi.c
index 3465fe2c5a5c..2514239282c2 100644
--- a/drivers/acpi/acpica/utnonansi.c
+++ b/drivers/acpi/acpica/utnonansi.c
@@ -48,8 +48,8 @@
 ACPI_MODULE_NAME("utnonansi")
 
 /*
- * Non-ANSI C library functions - strlwr, strupr, stricmp, and a 64-bit
- * version of strtoul.
+ * Non-ANSI C library functions - strlwr, strupr, stricmp, and "safe"
+ * string functions.
  */
 /*******************************************************************************
  *
@@ -200,356 +200,3 @@ acpi_ut_safe_strncat(char *dest,
 	return (FALSE);
 }
 #endif
-
-/*******************************************************************************
- *
- * FUNCTION:    acpi_ut_strtoul64
- *
- * PARAMETERS:  string                  - Null terminated string
- *              base                    - Radix of the string: 16 or 10 or
- *                                        ACPI_ANY_BASE
- *              max_integer_byte_width  - Maximum allowable integer,in bytes:
- *                                        4 or 8 (32 or 64 bits)
- *              ret_integer             - Where the converted integer is
- *                                        returned
- *
- * RETURN:      Status and Converted value
- *
- * DESCRIPTION: Convert a string into an unsigned value. Performs either a
- *              32-bit or 64-bit conversion, depending on the input integer
- *              size (often the current mode of the interpreter).
- *
- * NOTES:       Negative numbers are not supported, as they are not supported
- *              by ACPI.
- *
- *              acpi_gbl_integer_byte_width should be set to the proper width.
- *              For the core ACPICA code, this width depends on the DSDT
- *              version. For iASL, the default byte width is always 8 for the
- *              parser, but error checking is performed later to flag cases
- *              where a 64-bit constant is defined in a 32-bit DSDT/SSDT.
- *
- *              Does not support Octal strings, not needed at this time.
- *
- ******************************************************************************/
-
-acpi_status
-acpi_ut_strtoul64(char *string,
-		  u32 base, u32 max_integer_byte_width, u64 *ret_integer)
-{
-	u32 this_digit = 0;
-	u64 return_value = 0;
-	u64 quotient;
-	u64 dividend;
-	u8 valid_digits = 0;
-	u8 sign_of0x = 0;
-	u8 term = 0;
-
-	ACPI_FUNCTION_TRACE_STR(ut_strtoul64, string);
-
-	switch (base) {
-	case ACPI_ANY_BASE:
-	case 10:
-	case 16:
-
-		break;
-
-	default:
-
-		/* Invalid Base */
-
-		return_ACPI_STATUS(AE_BAD_PARAMETER);
-	}
-
-	if (!string) {
-		goto error_exit;
-	}
-
-	/* Skip over any white space in the buffer */
-
-	while ((*string) && (isspace((int)*string) || *string == '\t')) {
-		string++;
-	}
-
-	if (base == ACPI_ANY_BASE) {
-		/*
-		 * Base equal to ACPI_ANY_BASE means 'Either decimal or hex'.
-		 * We need to determine if it is decimal or hexadecimal.
-		 */
-		if ((*string == '0') && (tolower((int)*(string + 1)) == 'x')) {
-			sign_of0x = 1;
-			base = 16;
-
-			/* Skip over the leading '0x' */
-			string += 2;
-		} else {
-			base = 10;
-		}
-	}
-
-	/* Any string left? Check that '0x' is not followed by white space. */
-
-	if (!(*string) || isspace((int)*string) || *string == '\t') {
-		if (base == ACPI_ANY_BASE) {
-			goto error_exit;
-		} else {
-			goto all_done;
-		}
-	}
-
-	/*
-	 * Perform a 32-bit or 64-bit conversion, depending upon the input
-	 * byte width
-	 */
-	dividend = (max_integer_byte_width <= ACPI_MAX32_BYTE_WIDTH) ?
-	    ACPI_UINT32_MAX : ACPI_UINT64_MAX;
-
-	/* Main loop: convert the string to a 32- or 64-bit integer */
-
-	while (*string) {
-		if (isdigit((int)*string)) {
-
-			/* Convert ASCII 0-9 to Decimal value */
-
-			this_digit = ((u8)*string) - '0';
-		} else if (base == 10) {
-
-			/* Digit is out of range; possible in to_integer case only */
-
-			term = 1;
-		} else {
-			this_digit = (u8)toupper((int)*string);
-			if (isxdigit((int)this_digit)) {
-
-				/* Convert ASCII Hex char to value */
-
-				this_digit = this_digit - 'A' + 10;
-			} else {
-				term = 1;
-			}
-		}
-
-		if (term) {
-			if (base == ACPI_ANY_BASE) {
-				goto error_exit;
-			} else {
-				break;
-			}
-		} else if ((valid_digits == 0) && (this_digit == 0)
-			   && !sign_of0x) {
-
-			/* Skip zeros */
-			string++;
-			continue;
-		}
-
-		valid_digits++;
-
-		if (sign_of0x && ((valid_digits > 16) ||
-				  ((valid_digits > 8)
-				   && (max_integer_byte_width <=
-				       ACPI_MAX32_BYTE_WIDTH)))) {
-			/*
-			 * This is to_integer operation case.
-			 * No restrictions for string-to-integer conversion,
-			 * see ACPI spec.
-			 */
-			goto error_exit;
-		}
-
-		/* Divide the digit into the correct position */
-
-		(void)acpi_ut_short_divide((dividend - (u64)this_digit), base,
-					   &quotient, NULL);
-
-		if (return_value > quotient) {
-			if (base == ACPI_ANY_BASE) {
-				goto error_exit;
-			} else {
-				break;
-			}
-		}
-
-		return_value *= base;
-		return_value += this_digit;
-		string++;
-	}
-
-	/* All done, normal exit */
-
-all_done:
-
-	ACPI_DEBUG_PRINT((ACPI_DB_EXEC, "Converted value: %8.8X%8.8X\n",
-			  ACPI_FORMAT_UINT64(return_value)));
-
-	*ret_integer = return_value;
-	return_ACPI_STATUS(AE_OK);
-
-error_exit:
-
-	/* Base was set/validated above (10 or 16) */
-
-	if (base == 10) {
-		return_ACPI_STATUS(AE_BAD_DECIMAL_CONSTANT);
-	} else {
-		return_ACPI_STATUS(AE_BAD_HEX_CONSTANT);
-	}
-}
-
-#ifdef _OBSOLETE_FUNCTIONS
-/* Removed: 01/2016 */
-
-/*******************************************************************************
- *
- * FUNCTION:    strtoul64
- *
- * PARAMETERS:  string              - Null terminated string
- *              terminater          - Where a pointer to the terminating byte
- *                                    is returned
- *              base                - Radix of the string
- *
- * RETURN:      Converted value
- *
- * DESCRIPTION: Convert a string into an unsigned value.
- *
- ******************************************************************************/
-
-acpi_status strtoul64(char *string, u32 base, u64 *ret_integer)
-{
-	u32 index;
-	u32 sign;
-	u64 return_value = 0;
-	acpi_status status = AE_OK;
-
-	*ret_integer = 0;
-
-	switch (base) {
-	case 0:
-	case 8:
-	case 10:
-	case 16:
-
-		break;
-
-	default:
-		/*
-		 * The specified Base parameter is not in the domain of
-		 * this function:
-		 */
-		return (AE_BAD_PARAMETER);
-	}
-
-	/* Skip over any white space in the buffer: */
-
-	while (isspace((int)*string) || *string == '\t') {
-		++string;
-	}
-
-	/*
-	 * The buffer may contain an optional plus or minus sign.
-	 * If it does, then skip over it but remember what is was:
-	 */
-	if (*string == '-') {
-		sign = ACPI_SIGN_NEGATIVE;
-		++string;
-	} else if (*string == '+') {
-		++string;
-		sign = ACPI_SIGN_POSITIVE;
-	} else {
-		sign = ACPI_SIGN_POSITIVE;
-	}
-
-	/*
-	 * If the input parameter Base is zero, then we need to
-	 * determine if it is octal, decimal, or hexadecimal:
-	 */
-	if (base == 0) {
-		if (*string == '0') {
-			if (tolower((int)*(++string)) == 'x') {
-				base = 16;
-				++string;
-			} else {
-				base = 8;
-			}
-		} else {
-			base = 10;
-		}
-	}
-
-	/*
-	 * For octal and hexadecimal bases, skip over the leading
-	 * 0 or 0x, if they are present.
-	 */
-	if (base == 8 && *string == '0') {
-		string++;
-	}
-
-	if (base == 16 && *string == '0' && tolower((int)*(++string)) == 'x') {
-		string++;
-	}
-
-	/* Main loop: convert the string to an unsigned long */
-
-	while (*string) {
-		if (isdigit((int)*string)) {
-			index = ((u8)*string) - '0';
-		} else {
-			index = (u8)toupper((int)*string);
-			if (isupper((int)index)) {
-				index = index - 'A' + 10;
-			} else {
-				goto error_exit;
-			}
-		}
-
-		if (index >= base) {
-			goto error_exit;
-		}
-
-		/* Check to see if value is out of range: */
-
-		if (return_value > ((ACPI_UINT64_MAX - (u64)index) / (u64)base)) {
-			goto error_exit;
-		} else {
-			return_value *= base;
-			return_value += index;
-		}
-
-		++string;
-	}
-
-	/* If a minus sign was present, then "the conversion is negated": */
-
-	if (sign == ACPI_SIGN_NEGATIVE) {
-		return_value = (ACPI_UINT32_MAX - return_value) + 1;
-	}
-
-	*ret_integer = return_value;
-	return (status);
-
-error_exit:
-	switch (base) {
-	case 8:
-
-		status = AE_BAD_OCTAL_CONSTANT;
-		break;
-
-	case 10:
-
-		status = AE_BAD_DECIMAL_CONSTANT;
-		break;
-
-	case 16:
-
-		status = AE_BAD_HEX_CONSTANT;
-		break;
-
-	default:
-
-		/* Base validated above */
-
-		break;
-	}
-
-	return (status);
-}
-#endif
diff --git a/drivers/acpi/acpica/utosi.c b/drivers/acpi/acpica/utosi.c
index 3f5fed670271..f0484b058c44 100644
--- a/drivers/acpi/acpica/utosi.c
+++ b/drivers/acpi/acpica/utosi.c
@@ -390,11 +390,22 @@ struct acpi_interface_info *acpi_ut_get_interface(acpi_string interface_name)
  * PARAMETERS:  walk_state          - Current walk state
  *
  * RETURN:      Status
+ *              Integer: TRUE (0) if input string is matched
+ *                       FALSE (-1) if string is not matched
  *
  * DESCRIPTION: Implementation of the _OSI predefined control method. When
  *              an invocation of _OSI is encountered in the system AML,
  *              control is transferred to this function.
  *
+ * (August 2016)
+ * Note:  _OSI is now defined to return "Ones" to indicate a match, for
+ * compatibility with other ACPI implementations. On a 32-bit DSDT, Ones
+ * is 0xFFFFFFFF. On a 64-bit DSDT, Ones is 0xFFFFFFFFFFFFFFFF
+ * (ACPI_UINT64_MAX).
+ *
+ * This function always returns ACPI_UINT64_MAX for TRUE, and later code
+ * will truncate this to 32 bits if necessary.
+ *
  ******************************************************************************/
 
 acpi_status acpi_ut_osi_implementation(struct acpi_walk_state *walk_state)
@@ -404,7 +415,7 @@ acpi_status acpi_ut_osi_implementation(struct acpi_walk_state *walk_state)
 	struct acpi_interface_info *interface_info;
 	acpi_interface_handler interface_handler;
 	acpi_status status;
-	u32 return_value;
+	u64 return_value;
 
 	ACPI_FUNCTION_TRACE(ut_osi_implementation);
 
@@ -444,7 +455,7 @@ acpi_status acpi_ut_osi_implementation(struct acpi_walk_state *walk_state)
 			acpi_gbl_osi_data = interface_info->value;
 		}
 
-		return_value = ACPI_UINT32_MAX;
+		return_value = ACPI_UINT64_MAX;
 	}
 
 	acpi_os_release_mutex(acpi_gbl_osi_mutex);
@@ -456,9 +467,10 @@ acpi_status acpi_ut_osi_implementation(struct acpi_walk_state *walk_state)
 	 */
 	interface_handler = acpi_gbl_interface_handler;
 	if (interface_handler) {
-		return_value =
-		    interface_handler(string_desc->string.pointer,
-				      return_value);
+		if (interface_handler
+		    (string_desc->string.pointer, (u32)return_value)) {
+			return_value = ACPI_UINT64_MAX;
+		}
 	}
 
 	ACPI_DEBUG_PRINT_RAW((ACPI_DB_INFO,
diff --git a/drivers/acpi/acpica/utpredef.c b/drivers/acpi/acpica/utpredef.c
index 770a1775b264..ce18346b6144 100644
--- a/drivers/acpi/acpica/utpredef.c
+++ b/drivers/acpi/acpica/utpredef.c
@@ -176,8 +176,6 @@ void acpi_ut_get_expected_return_types(char *buffer, u32 expected_btypes)
  ******************************************************************************/
 
 #if (defined ACPI_ASL_COMPILER || defined ACPI_HELP_APP)
-#include <stdio.h>
-#include <string.h>
 
 /* Local prototypes */
 
diff --git a/drivers/acpi/acpica/utprint.c b/drivers/acpi/acpica/utprint.c
index dd084cf52502..40eba804d49c 100644
--- a/drivers/acpi/acpica/utprint.c
+++ b/drivers/acpi/acpica/utprint.c
@@ -336,7 +336,7 @@ static char *acpi_ut_format_number(char *string,
 
 /*******************************************************************************
  *
- * FUNCTION:    acpi_ut_vsnprintf
+ * FUNCTION:    vsnprintf
  *
  * PARAMETERS:  string              - String with boundary
  *              size                - Boundary of the string
@@ -349,9 +349,7 @@ static char *acpi_ut_format_number(char *string,
  *
  ******************************************************************************/
 
-int
-acpi_ut_vsnprintf(char *string,
-		  acpi_size size, const char *format, va_list args)
+int vsnprintf(char *string, acpi_size size, const char *format, va_list args)
 {
 	u8 base;
 	u8 type;
@@ -586,7 +584,7 @@ acpi_ut_vsnprintf(char *string,
 
 /*******************************************************************************
  *
- * FUNCTION:    acpi_ut_snprintf
+ * FUNCTION:    snprintf
  *
  * PARAMETERS:  string              - String with boundary
  *              size                - Boundary of the string
@@ -598,13 +596,38 @@ acpi_ut_vsnprintf(char *string,
  *
  ******************************************************************************/
 
-int acpi_ut_snprintf(char *string, acpi_size size, const char *format, ...)
+int snprintf(char *string, acpi_size size, const char *format, ...)
 {
 	va_list args;
 	int length;
 
 	va_start(args, format);
-	length = acpi_ut_vsnprintf(string, size, format, args);
+	length = vsnprintf(string, size, format, args);
+	va_end(args);
+
+	return (length);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    sprintf
+ *
+ * PARAMETERS:  string              - String with boundary
+ *              Format, ...         - Standard printf format
+ *
+ * RETURN:      Number of bytes actually written.
+ *
+ * DESCRIPTION: Formatted output to a string.
+ *
+ ******************************************************************************/
+
+int sprintf(char *string, const char *format, ...)
+{
+	va_list args;
+	int length;
+
+	va_start(args, format);
+	length = vsnprintf(string, ACPI_UINT32_MAX, format, args);
 	va_end(args);
 
 	return (length);
@@ -613,7 +636,59 @@ int acpi_ut_snprintf(char *string, acpi_size size, const char *format, ...)
 #ifdef ACPI_APPLICATION
 /*******************************************************************************
  *
- * FUNCTION:    acpi_ut_file_vprintf
+ * FUNCTION:    vprintf
+ *
+ * PARAMETERS:  format              - Standard printf format
+ *              args                - Argument list
+ *
+ * RETURN:      Number of bytes actually written.
+ *
+ * DESCRIPTION: Formatted output to stdout using argument list pointer.
+ *
+ ******************************************************************************/
+
+int vprintf(const char *format, va_list args)
+{
+	acpi_cpu_flags flags;
+	int length;
+
+	flags = acpi_os_acquire_lock(acpi_gbl_print_lock);
+	length = vsnprintf(acpi_gbl_print_buffer,
+			   sizeof(acpi_gbl_print_buffer), format, args);
+
+	(void)fwrite(acpi_gbl_print_buffer, length, 1, ACPI_FILE_OUT);
+	acpi_os_release_lock(acpi_gbl_print_lock, flags);
+
+	return (length);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    printf
+ *
+ * PARAMETERS:  Format, ...         - Standard printf format
+ *
+ * RETURN:      Number of bytes actually written.
+ *
+ * DESCRIPTION: Formatted output to stdout.
+ *
+ ******************************************************************************/
+
+int printf(const char *format, ...)
+{
+	va_list args;
+	int length;
+
+	va_start(args, format);
+	length = vprintf(format, args);
+	va_end(args);
+
+	return (length);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    vfprintf
  *
  * PARAMETERS:  file                - File descriptor
  *              format              - Standard printf format
@@ -625,16 +700,16 @@ int acpi_ut_snprintf(char *string, acpi_size size, const char *format, ...)
  *
  ******************************************************************************/
 
-int acpi_ut_file_vprintf(ACPI_FILE file, const char *format, va_list args)
+int vfprintf(FILE * file, const char *format, va_list args)
 {
 	acpi_cpu_flags flags;
 	int length;
 
 	flags = acpi_os_acquire_lock(acpi_gbl_print_lock);
-	length = acpi_ut_vsnprintf(acpi_gbl_print_buffer,
-				   sizeof(acpi_gbl_print_buffer), format, args);
+	length = vsnprintf(acpi_gbl_print_buffer,
+			   sizeof(acpi_gbl_print_buffer), format, args);
 
-	(void)acpi_os_write_file(file, acpi_gbl_print_buffer, length, 1);
+	(void)fwrite(acpi_gbl_print_buffer, length, 1, file);
 	acpi_os_release_lock(acpi_gbl_print_lock, flags);
 
 	return (length);
@@ -642,7 +717,7 @@ int acpi_ut_file_vprintf(ACPI_FILE file, const char *format, va_list args)
 
 /*******************************************************************************
  *
- * FUNCTION:    acpi_ut_file_printf
+ * FUNCTION:    fprintf
  *
  * PARAMETERS:  file                - File descriptor
  *              Format, ...         - Standard printf format
@@ -653,13 +728,13 @@ int acpi_ut_file_vprintf(ACPI_FILE file, const char *format, va_list args)
  *
  ******************************************************************************/
 
-int acpi_ut_file_printf(ACPI_FILE file, const char *format, ...)
+int fprintf(FILE * file, const char *format, ...)
 {
 	va_list args;
 	int length;
 
 	va_start(args, format);
-	length = acpi_ut_file_vprintf(file, format, args);
+	length = vfprintf(file, format, args);
 	va_end(args);
 
 	return (length);
diff --git a/drivers/acpi/acpica/utstrtoul64.c b/drivers/acpi/acpica/utstrtoul64.c
new file mode 100644
index 000000000000..b4f341c98a95
--- /dev/null
+++ b/drivers/acpi/acpica/utstrtoul64.c
@@ -0,0 +1,348 @@
+/*******************************************************************************
+ *
+ * Module Name: utstrtoul64 - string to 64-bit integer support
+ *
+ ******************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2016, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+#include <acpi/acpi.h>
+#include "accommon.h"
+
+/*******************************************************************************
+ *
+ * The functions in this module satisfy the need for 64-bit string-to-integer
+ * conversions on both 32-bit and 64-bit platforms.
+ *
+ ******************************************************************************/
+
+#define _COMPONENT          ACPI_UTILITIES
+ACPI_MODULE_NAME("utstrtoul64")
+
+/* Local prototypes */
+static u64 acpi_ut_strtoul_base10(char *string, u32 flags);
+
+static u64 acpi_ut_strtoul_base16(char *string, u32 flags);
+
+/*******************************************************************************
+ *
+ * String conversion rules as written in the ACPI specification. The error
+ * conditions and behavior are different depending on the type of conversion.
+ *
+ *
+ * Implicit data type conversion: string-to-integer
+ * --------------------------------------------------
+ *
+ * Base is always 16. This is the ACPI_STRTOUL_BASE16 case.
+ *
+ * Example:
+ *      Add ("BA98", Arg0, Local0)
+ *
+ * The integer is initialized to the value zero.
+ * The ASCII string is interpreted as a hexadecimal constant.
+ *
+ *  1)  A "0x" prefix is not allowed. However, ACPICA allows this for
+ *      compatibility with previous ACPICA. (NO ERROR)
+ *
+ *  2)  Terminates when the size of an integer is reached (32 or 64 bits).
+ *      (NO ERROR)
+ *
+ *  3)  The first non-hex character terminates the conversion without error.
+ *      (NO ERROR)
+ *
+ *  4)  Conversion of a null (zero-length) string to an integer is not
+ *      allowed. However, ACPICA allows this for compatibility with previous
+ *      ACPICA. This conversion returns the value 0. (NO ERROR)
+ *
+ *
+ * Explicit data type conversion:  to_integer() with string operand
+ * ---------------------------------------------------------------
+ *
+ * Base is either 10 (default) or 16 (with 0x prefix)
+ *
+ * Examples:
+ *      to_integer ("1000")
+ *      to_integer ("0xABCD")
+ *
+ *  1)  Can be (must be) either a decimal or hexadecimal numeric string.
+ *      A hex value must be prefixed by "0x" or it is interpreted as a decimal.
+ *
+ *  2)  The value must not exceed the maximum of an integer value. ACPI spec
+ *      states the behavior is "unpredictable", so ACPICA matches the behavior
+ *      of the implicit conversion case.(NO ERROR)
+ *
+ *  3)  Behavior on the first non-hex character is not specified by the ACPI
+ *      spec, so ACPICA matches the behavior of the implicit conversion case
+ *      and terminates. (NO ERROR)
+ *
+ *  4)  A null (zero-length) string is illegal.
+ *      However, ACPICA allows this for compatibility with previous ACPICA.
+ *      This conversion returns the value 0. (NO ERROR)
+ *
+ ******************************************************************************/
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_strtoul64
+ *
+ * PARAMETERS:  string                  - Null terminated input string
+ *              flags                   - Conversion info, see below
+ *              return_value            - Where the converted integer is
+ *                                        returned
+ *
+ * RETURN:      Status and Converted value
+ *
+ * DESCRIPTION: Convert a string into an unsigned value. Performs either a
+ *              32-bit or 64-bit conversion, depending on the input integer
+ *              size in Flags (often the current mode of the interpreter).
+ *
+ * Values for Flags:
+ *      ACPI_STRTOUL_32BIT      - Max integer value is 32 bits
+ *      ACPI_STRTOUL_64BIT      - Max integer value is 64 bits
+ *      ACPI_STRTOUL_BASE16     - Input string is hexadecimal. Default
+ *                                is 10/16 based on string prefix (0x).
+ *
+ * NOTES:
+ *   Negative numbers are not supported, as they are not supported by ACPI.
+ *
+ *   Supports only base 16 or base 10 strings/values. Does not
+ *   support Octal strings, as these are not supported by ACPI.
+ *
+ * Current users of this support:
+ *
+ *  interpreter - Implicit and explicit conversions, GPE method names
+ *  debugger    - Command line input string conversion
+ *  iASL        - Main parser, conversion of constants to integers
+ *  iASL        - Data Table Compiler parser (constant math expressions)
+ *  iASL        - Preprocessor (constant math expressions)
+ *  acpi_dump   - Input table addresses
+ *  acpi_exec   - Testing of the acpi_ut_strtoul64 function
+ *
+ * Note concerning callers:
+ *   acpi_gbl_integer_byte_width can be used to set the 32/64 limit. If used,
+ *   this global should be set to the proper width. For the core ACPICA code,
+ *   this width depends on the DSDT version. For iASL, the default byte
+ *   width is always 8 for the parser, but error checking is performed later
+ *   to flag cases where a 64-bit constant is defined in a 32-bit DSDT/SSDT.
+ *
+ ******************************************************************************/
+
+acpi_status acpi_ut_strtoul64(char *string, u32 flags, u64 *return_value)
+{
+	acpi_status status = AE_OK;
+	u32 base;
+
+	ACPI_FUNCTION_TRACE_STR(ut_strtoul64, string);
+
+	/* Parameter validation */
+
+	if (!string || !return_value) {
+		return_ACPI_STATUS(AE_BAD_PARAMETER);
+	}
+
+	*return_value = 0;
+
+	/* Check for zero-length string, returns 0 */
+
+	if (*string == 0) {
+		return_ACPI_STATUS(AE_OK);
+	}
+
+	/* Skip over any white space at start of string */
+
+	while (isspace((int)*string)) {
+		string++;
+	}
+
+	/* End of string? return 0 */
+
+	if (*string == 0) {
+		return_ACPI_STATUS(AE_OK);
+	}
+
+	/*
+	 * 1) The "0x" prefix indicates base 16. Per the ACPI specification,
+	 * the "0x" prefix is only allowed for implicit (non-strict) conversions.
+	 * However, we always allow it for compatibility with older ACPICA.
+	 */
+	if ((*string == ACPI_ASCII_ZERO) &&
+	    (tolower((int)*(string + 1)) == 'x')) {
+		string += 2;	/* Go past the 0x */
+		if (*string == 0) {
+			return_ACPI_STATUS(AE_OK);	/* Return value 0 */
+		}
+
+		base = 16;
+	}
+
+	/* 2) Force to base 16 (implicit conversion case) */
+
+	else if (flags & ACPI_STRTOUL_BASE16) {
+		base = 16;
+	}
+
+	/* 3) Default fallback is to Base 10 */
+
+	else {
+		base = 10;
+	}
+
+	/* Skip all leading zeros */
+
+	while (*string == ACPI_ASCII_ZERO) {
+		string++;
+		if (*string == 0) {
+			return_ACPI_STATUS(AE_OK);	/* Return value 0 */
+		}
+	}
+
+	/* Perform the base 16 or 10 conversion */
+
+	if (base == 16) {
+		*return_value = acpi_ut_strtoul_base16(string, flags);
+	} else {
+		*return_value = acpi_ut_strtoul_base10(string, flags);
+	}
+
+	return_ACPI_STATUS(status);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_strtoul_base10
+ *
+ * PARAMETERS:  string                  - Null terminated input string
+ *              flags                   - Conversion info
+ *
+ * RETURN:      64-bit converted integer
+ *
+ * DESCRIPTION: Performs a base 10 conversion of the input string to an
+ *              integer value, either 32 or 64 bits.
+ *              Note: String must be valid and non-null.
+ *
+ ******************************************************************************/
+
+static u64 acpi_ut_strtoul_base10(char *string, u32 flags)
+{
+	int ascii_digit;
+	u64 next_value;
+	u64 return_value = 0;
+
+	/* Main loop: convert each ASCII byte in the input string */
+
+	while (*string) {
+		ascii_digit = *string;
+		if (!isdigit(ascii_digit)) {
+
+			/* Not ASCII 0-9, terminate */
+
+			goto exit;
+		}
+
+		/* Convert and insert (add) the decimal digit */
+
+		next_value =
+		    (return_value * 10) + (ascii_digit - ACPI_ASCII_ZERO);
+
+		/* Check for overflow (32 or 64 bit) - return current converted value */
+
+		if (((flags & ACPI_STRTOUL_32BIT) && (next_value > ACPI_UINT32_MAX)) || (next_value < return_value)) {	/* 64-bit overflow case */
+			goto exit;
+		}
+
+		return_value = next_value;
+		string++;
+	}
+
+exit:
+	return (return_value);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_strtoul_base16
+ *
+ * PARAMETERS:  string                  - Null terminated input string
+ *              flags                   - conversion info
+ *
+ * RETURN:      64-bit converted integer
+ *
+ * DESCRIPTION: Performs a base 16 conversion of the input string to an
+ *              integer value, either 32 or 64 bits.
+ *              Note: String must be valid and non-null.
+ *
+ ******************************************************************************/
+
+static u64 acpi_ut_strtoul_base16(char *string, u32 flags)
+{
+	int ascii_digit;
+	u32 valid_digits = 1;
+	u64 return_value = 0;
+
+	/* Main loop: convert each ASCII byte in the input string */
+
+	while (*string) {
+
+		/* Check for overflow (32 or 64 bit) - return current converted value */
+
+		if ((valid_digits > 16) ||
+		    ((valid_digits > 8) && (flags & ACPI_STRTOUL_32BIT))) {
+			goto exit;
+		}
+
+		ascii_digit = *string;
+		if (!isxdigit(ascii_digit)) {
+
+			/* Not Hex ASCII A-F, a-f, or 0-9, terminate */
+
+			goto exit;
+		}
+
+		/* Convert and insert the hex digit */
+
+		return_value =
+		    (return_value << 4) |
+		    acpi_ut_ascii_char_to_hex(ascii_digit);
+
+		string++;
+		valid_digits++;
+	}
+
+exit:
+	return (return_value);
+}
diff --git a/drivers/acpi/acpica/uttrack.c b/drivers/acpi/acpica/uttrack.c
index 0df07dfa53b6..df31d71ce596 100644
--- a/drivers/acpi/acpica/uttrack.c
+++ b/drivers/acpi/acpica/uttrack.c
@@ -95,13 +95,11 @@ acpi_ut_create_list(const char *list_name,
 {
 	struct acpi_memory_list *cache;
 
-	cache = acpi_os_allocate(sizeof(struct acpi_memory_list));
+	cache = acpi_os_allocate_zeroed(sizeof(struct acpi_memory_list));
 	if (!cache) {
 		return (AE_NO_MEMORY);
 	}
 
-	memset(cache, 0, sizeof(struct acpi_memory_list));
-
 	cache->list_name = list_name;
 	cache->object_size = object_size;
 
diff --git a/drivers/acpi/acpica/utxface.c b/drivers/acpi/acpica/utxface.c
index d9e6aac7dc83..ec503c862961 100644
--- a/drivers/acpi/acpica/utxface.c
+++ b/drivers/acpi/acpica/utxface.c
@@ -61,7 +61,7 @@ ACPI_MODULE_NAME("utxface")
  * DESCRIPTION: Shutdown the ACPICA subsystem and release all resources.
  *
  ******************************************************************************/
-acpi_status __init acpi_terminate(void)
+acpi_status ACPI_INIT_FUNCTION acpi_terminate(void)
 {
 	acpi_status status;
 
diff --git a/drivers/acpi/acpica/utxfinit.c b/drivers/acpi/acpica/utxfinit.c
index 75b5f27da267..a5ca0f57cd08 100644
--- a/drivers/acpi/acpica/utxfinit.c
+++ b/drivers/acpi/acpica/utxfinit.c
@@ -69,7 +69,7 @@ void ae_do_object_overrides(void);
  *
  ******************************************************************************/
 
-acpi_status __init acpi_initialize_subsystem(void)
+acpi_status ACPI_INIT_FUNCTION acpi_initialize_subsystem(void)
 {
 	acpi_status status;
 
@@ -141,7 +141,7 @@ ACPI_EXPORT_SYMBOL_INIT(acpi_initialize_subsystem)
  *              Puts system into ACPI mode if it isn't already.
  *
  ******************************************************************************/
-acpi_status __init acpi_enable_subsystem(u32 flags)
+acpi_status ACPI_INIT_FUNCTION acpi_enable_subsystem(u32 flags)
 {
 	acpi_status status = AE_OK;
 
@@ -239,7 +239,7 @@ ACPI_EXPORT_SYMBOL_INIT(acpi_enable_subsystem)
  *              objects and executing AML code for Regions, buffers, etc.
  *
  ******************************************************************************/
-acpi_status __init acpi_initialize_objects(u32 flags)
+acpi_status ACPI_INIT_FUNCTION acpi_initialize_objects(u32 flags)
 {
 	acpi_status status = AE_OK;
 
@@ -265,7 +265,8 @@ acpi_status __init acpi_initialize_objects(u32 flags)
 	 * all of the tables have been loaded. It is a legacy option and is
 	 * not compatible with other ACPI implementations. See acpi_ns_load_table.
 	 */
-	if (acpi_gbl_group_module_level_code) {
+	if (!acpi_gbl_parse_table_as_term_list
+	    && acpi_gbl_group_module_level_code) {
 		acpi_ns_exec_module_code_list();
 
 		/*
diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c
index f096ab3cb54d..ec4f507b524f 100644
--- a/drivers/acpi/apei/erst.c
+++ b/drivers/acpi/apei/erst.c
@@ -938,7 +938,7 @@ static int erst_clearer(enum pstore_type_id type, u64 id, int count,
 static struct pstore_info erst_info = {
 	.owner		= THIS_MODULE,
 	.name		= "erst",
-	.flags		= PSTORE_FLAGS_FRAGILE,
+	.flags		= PSTORE_FLAGS_DMESG,
 	.open		= erst_open_pstore,
 	.close		= erst_close_pstore,
 	.read		= erst_reader,
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index 60746ef904e4..f0a029e68d3e 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -457,7 +457,7 @@ static void ghes_do_proc(struct ghes *ghes,
 
 				devfn = PCI_DEVFN(pcie_err->device_id.device,
 						  pcie_err->device_id.function);
-				aer_severity = cper_severity_to_aer(sev);
+				aer_severity = cper_severity_to_aer(gdata->error_severity);
 
 				/*
 				 * If firmware reset the component to contain
diff --git a/drivers/acpi/arm64/Kconfig b/drivers/acpi/arm64/Kconfig
new file mode 100644
index 000000000000..4616da4c15be
--- /dev/null
+++ b/drivers/acpi/arm64/Kconfig
@@ -0,0 +1,6 @@
+#
+# ACPI Configuration for ARM64
+#
+
+config ACPI_IORT
+	bool
diff --git a/drivers/acpi/arm64/Makefile b/drivers/acpi/arm64/Makefile
new file mode 100644
index 000000000000..72331f2ce0e9
--- /dev/null
+++ b/drivers/acpi/arm64/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_ACPI_IORT) 	+= iort.o
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
new file mode 100644
index 000000000000..6b81746cd13c
--- /dev/null
+++ b/drivers/acpi/arm64/iort.c
@@ -0,0 +1,368 @@
+/*
+ * Copyright (C) 2016, Semihalf
+ *	Author: Tomasz Nowicki <tn@semihalf.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * This file implements early detection/parsing of I/O mapping
+ * reported to OS through firmware via I/O Remapping Table (IORT)
+ * IORT document number: ARM DEN 0049A
+ */
+
+#define pr_fmt(fmt)	"ACPI: IORT: " fmt
+
+#include <linux/acpi_iort.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+
+struct iort_its_msi_chip {
+	struct list_head	list;
+	struct fwnode_handle	*fw_node;
+	u32			translation_id;
+};
+
+typedef acpi_status (*iort_find_node_callback)
+	(struct acpi_iort_node *node, void *context);
+
+/* Root pointer to the mapped IORT table */
+static struct acpi_table_header *iort_table;
+
+static LIST_HEAD(iort_msi_chip_list);
+static DEFINE_SPINLOCK(iort_msi_chip_lock);
+
+/**
+ * iort_register_domain_token() - register domain token and related ITS ID
+ * to the list from where we can get it back later on.
+ * @trans_id: ITS ID.
+ * @fw_node: Domain token.
+ *
+ * Returns: 0 on success, -ENOMEM if no memory when allocating list element
+ */
+int iort_register_domain_token(int trans_id, struct fwnode_handle *fw_node)
+{
+	struct iort_its_msi_chip *its_msi_chip;
+
+	its_msi_chip = kzalloc(sizeof(*its_msi_chip), GFP_KERNEL);
+	if (!its_msi_chip)
+		return -ENOMEM;
+
+	its_msi_chip->fw_node = fw_node;
+	its_msi_chip->translation_id = trans_id;
+
+	spin_lock(&iort_msi_chip_lock);
+	list_add(&its_msi_chip->list, &iort_msi_chip_list);
+	spin_unlock(&iort_msi_chip_lock);
+
+	return 0;
+}
+
+/**
+ * iort_deregister_domain_token() - Deregister domain token based on ITS ID
+ * @trans_id: ITS ID.
+ *
+ * Returns: none.
+ */
+void iort_deregister_domain_token(int trans_id)
+{
+	struct iort_its_msi_chip *its_msi_chip, *t;
+
+	spin_lock(&iort_msi_chip_lock);
+	list_for_each_entry_safe(its_msi_chip, t, &iort_msi_chip_list, list) {
+		if (its_msi_chip->translation_id == trans_id) {
+			list_del(&its_msi_chip->list);
+			kfree(its_msi_chip);
+			break;
+		}
+	}
+	spin_unlock(&iort_msi_chip_lock);
+}
+
+/**
+ * iort_find_domain_token() - Find domain token based on given ITS ID
+ * @trans_id: ITS ID.
+ *
+ * Returns: domain token when find on the list, NULL otherwise
+ */
+struct fwnode_handle *iort_find_domain_token(int trans_id)
+{
+	struct fwnode_handle *fw_node = NULL;
+	struct iort_its_msi_chip *its_msi_chip;
+
+	spin_lock(&iort_msi_chip_lock);
+	list_for_each_entry(its_msi_chip, &iort_msi_chip_list, list) {
+		if (its_msi_chip->translation_id == trans_id) {
+			fw_node = its_msi_chip->fw_node;
+			break;
+		}
+	}
+	spin_unlock(&iort_msi_chip_lock);
+
+	return fw_node;
+}
+
+static struct acpi_iort_node *iort_scan_node(enum acpi_iort_node_type type,
+					     iort_find_node_callback callback,
+					     void *context)
+{
+	struct acpi_iort_node *iort_node, *iort_end;
+	struct acpi_table_iort *iort;
+	int i;
+
+	if (!iort_table)
+		return NULL;
+
+	/* Get the first IORT node */
+	iort = (struct acpi_table_iort *)iort_table;
+	iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort,
+				 iort->node_offset);
+	iort_end = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
+				iort_table->length);
+
+	for (i = 0; i < iort->node_count; i++) {
+		if (WARN_TAINT(iort_node >= iort_end, TAINT_FIRMWARE_WORKAROUND,
+			       "IORT node pointer overflows, bad table!\n"))
+			return NULL;
+
+		if (iort_node->type == type &&
+		    ACPI_SUCCESS(callback(iort_node, context)))
+				return iort_node;
+
+		iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort_node,
+					 iort_node->length);
+	}
+
+	return NULL;
+}
+
+static acpi_status iort_match_node_callback(struct acpi_iort_node *node,
+					    void *context)
+{
+	struct device *dev = context;
+	acpi_status status;
+
+	if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT) {
+		struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
+		struct acpi_device *adev = to_acpi_device_node(dev->fwnode);
+		struct acpi_iort_named_component *ncomp;
+
+		if (!adev) {
+			status = AE_NOT_FOUND;
+			goto out;
+		}
+
+		status = acpi_get_name(adev->handle, ACPI_FULL_PATHNAME, &buf);
+		if (ACPI_FAILURE(status)) {
+			dev_warn(dev, "Can't get device full path name\n");
+			goto out;
+		}
+
+		ncomp = (struct acpi_iort_named_component *)node->node_data;
+		status = !strcmp(ncomp->device_name, buf.pointer) ?
+							AE_OK : AE_NOT_FOUND;
+		acpi_os_free(buf.pointer);
+	} else if (node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
+		struct acpi_iort_root_complex *pci_rc;
+		struct pci_bus *bus;
+
+		bus = to_pci_bus(dev);
+		pci_rc = (struct acpi_iort_root_complex *)node->node_data;
+
+		/*
+		 * It is assumed that PCI segment numbers maps one-to-one
+		 * with root complexes. Each segment number can represent only
+		 * one root complex.
+		 */
+		status = pci_rc->pci_segment_number == pci_domain_nr(bus) ?
+							AE_OK : AE_NOT_FOUND;
+	} else {
+		status = AE_NOT_FOUND;
+	}
+out:
+	return status;
+}
+
+static int iort_id_map(struct acpi_iort_id_mapping *map, u8 type, u32 rid_in,
+		       u32 *rid_out)
+{
+	/* Single mapping does not care for input id */
+	if (map->flags & ACPI_IORT_ID_SINGLE_MAPPING) {
+		if (type == ACPI_IORT_NODE_NAMED_COMPONENT ||
+		    type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
+			*rid_out = map->output_base;
+			return 0;
+		}
+
+		pr_warn(FW_BUG "[map %p] SINGLE MAPPING flag not allowed for node type %d, skipping ID map\n",
+			map, type);
+		return -ENXIO;
+	}
+
+	if (rid_in < map->input_base ||
+	    (rid_in >= map->input_base + map->id_count))
+		return -ENXIO;
+
+	*rid_out = map->output_base + (rid_in - map->input_base);
+	return 0;
+}
+
+static struct acpi_iort_node *iort_node_map_rid(struct acpi_iort_node *node,
+						u32 rid_in, u32 *rid_out,
+						u8 type)
+{
+	u32 rid = rid_in;
+
+	/* Parse the ID mapping tree to find specified node type */
+	while (node) {
+		struct acpi_iort_id_mapping *map;
+		int i;
+
+		if (node->type == type) {
+			if (rid_out)
+				*rid_out = rid;
+			return node;
+		}
+
+		if (!node->mapping_offset || !node->mapping_count)
+			goto fail_map;
+
+		map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
+				   node->mapping_offset);
+
+		/* Firmware bug! */
+		if (!map->output_reference) {
+			pr_err(FW_BUG "[node %p type %d] ID map has NULL parent reference\n",
+			       node, node->type);
+			goto fail_map;
+		}
+
+		/* Do the RID translation */
+		for (i = 0; i < node->mapping_count; i++, map++) {
+			if (!iort_id_map(map, node->type, rid, &rid))
+				break;
+		}
+
+		if (i == node->mapping_count)
+			goto fail_map;
+
+		node = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
+				    map->output_reference);
+	}
+
+fail_map:
+	/* Map input RID to output RID unchanged on mapping failure*/
+	if (rid_out)
+		*rid_out = rid_in;
+
+	return NULL;
+}
+
+static struct acpi_iort_node *iort_find_dev_node(struct device *dev)
+{
+	struct pci_bus *pbus;
+
+	if (!dev_is_pci(dev))
+		return iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
+				      iort_match_node_callback, dev);
+
+	/* Find a PCI root bus */
+	pbus = to_pci_dev(dev)->bus;
+	while (!pci_is_root_bus(pbus))
+		pbus = pbus->parent;
+
+	return iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
+			      iort_match_node_callback, &pbus->dev);
+}
+
+/**
+ * iort_msi_map_rid() - Map a MSI requester ID for a device
+ * @dev: The device for which the mapping is to be done.
+ * @req_id: The device requester ID.
+ *
+ * Returns: mapped MSI RID on success, input requester ID otherwise
+ */
+u32 iort_msi_map_rid(struct device *dev, u32 req_id)
+{
+	struct acpi_iort_node *node;
+	u32 dev_id;
+
+	node = iort_find_dev_node(dev);
+	if (!node)
+		return req_id;
+
+	iort_node_map_rid(node, req_id, &dev_id, ACPI_IORT_NODE_ITS_GROUP);
+	return dev_id;
+}
+
+/**
+ * iort_dev_find_its_id() - Find the ITS identifier for a device
+ * @dev: The device.
+ * @idx: Index of the ITS identifier list.
+ * @its_id: ITS identifier.
+ *
+ * Returns: 0 on success, appropriate error value otherwise
+ */
+static int iort_dev_find_its_id(struct device *dev, u32 req_id,
+				unsigned int idx, int *its_id)
+{
+	struct acpi_iort_its_group *its;
+	struct acpi_iort_node *node;
+
+	node = iort_find_dev_node(dev);
+	if (!node)
+		return -ENXIO;
+
+	node = iort_node_map_rid(node, req_id, NULL, ACPI_IORT_NODE_ITS_GROUP);
+	if (!node)
+		return -ENXIO;
+
+	/* Move to ITS specific data */
+	its = (struct acpi_iort_its_group *)node->node_data;
+	if (idx > its->its_count) {
+		dev_err(dev, "requested ITS ID index [%d] is greater than available [%d]\n",
+			idx, its->its_count);
+		return -ENXIO;
+	}
+
+	*its_id = its->identifiers[idx];
+	return 0;
+}
+
+/**
+ * iort_get_device_domain() - Find MSI domain related to a device
+ * @dev: The device.
+ * @req_id: Requester ID for the device.
+ *
+ * Returns: the MSI domain for this device, NULL otherwise
+ */
+struct irq_domain *iort_get_device_domain(struct device *dev, u32 req_id)
+{
+	struct fwnode_handle *handle;
+	int its_id;
+
+	if (iort_dev_find_its_id(dev, req_id, 0, &its_id))
+		return NULL;
+
+	handle = iort_find_domain_token(its_id);
+	if (!handle)
+		return NULL;
+
+	return irq_find_matching_fwnode(handle, DOMAIN_BUS_PCI_MSI);
+}
+
+void __init acpi_iort_init(void)
+{
+	acpi_status status;
+
+	status = acpi_get_table(ACPI_SIG_IORT, 0, &iort_table);
+	if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
+		const char *msg = acpi_format_exception(status);
+		pr_err("Failed to get table, %s\n", msg);
+	}
+}
diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
index ab234791a0ba..93ecae55fe6a 100644
--- a/drivers/acpi/battery.c
+++ b/drivers/acpi/battery.c
@@ -733,15 +733,17 @@ static int acpi_battery_update(struct acpi_battery *battery, bool resume)
 			return result;
 		acpi_battery_init_alarm(battery);
 	}
+
+	result = acpi_battery_get_state(battery);
+	if (result)
+		return result;
+	acpi_battery_quirks(battery);
+
 	if (!battery->bat) {
 		result = sysfs_add_battery(battery);
 		if (result)
 			return result;
 	}
-	result = acpi_battery_get_state(battery);
-	if (result)
-		return result;
-	acpi_battery_quirks(battery);
 
 	/*
 	 * Wakeup the system if battery is critical low
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index 85b7d07fe5c8..56190d00fd87 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -36,6 +36,7 @@
 #ifdef CONFIG_X86
 #include <asm/mpspec.h>
 #endif
+#include <linux/acpi_iort.h>
 #include <linux/pci.h>
 #include <acpi/apei.h>
 #include <linux/dmi.h>
@@ -985,7 +986,8 @@ void __init acpi_early_init(void)
 		goto error0;
 	}
 
-	if (acpi_gbl_group_module_level_code) {
+	if (!acpi_gbl_parse_table_as_term_list &&
+	    acpi_gbl_group_module_level_code) {
 		status = acpi_load_tables();
 		if (ACPI_FAILURE(status)) {
 			printk(KERN_ERR PREFIX
@@ -1074,7 +1076,8 @@ static int __init acpi_bus_init(void)
 	status = acpi_ec_ecdt_probe();
 	/* Ignore result. Not having an ECDT is not fatal. */
 
-	if (!acpi_gbl_group_module_level_code) {
+	if (acpi_gbl_parse_table_as_term_list ||
+	    !acpi_gbl_group_module_level_code) {
 		status = acpi_load_tables();
 		if (ACPI_FAILURE(status)) {
 			printk(KERN_ERR PREFIX
@@ -1186,6 +1189,7 @@ static int __init acpi_init(void)
 	}
 
 	pci_mmcfg_late_init();
+	acpi_iort_init();
 	acpi_scan_init();
 	acpi_ec_init();
 	acpi_debugfs_init();
@@ -1193,6 +1197,7 @@ static int __init acpi_init(void)
 	acpi_wakeup_device_init();
 	acpi_debugger_init();
 	acpi_setup_sb_notify_handler();
+	acpi_set_processor_mapping();
 	return 0;
 }
 
diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c
index 31abb0bdd4f2..e19f530f1083 100644
--- a/drivers/acpi/button.c
+++ b/drivers/acpi/button.c
@@ -19,6 +19,8 @@
  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  */
 
+#define pr_fmt(fmt) "ACPI : button: " fmt
+
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/init.h>
@@ -104,6 +106,8 @@ struct acpi_button {
 	struct input_dev *input;
 	char phys[32];			/* for input device */
 	unsigned long pushed;
+	int last_state;
+	ktime_t last_time;
 	bool suspended;
 };
 
@@ -111,6 +115,10 @@ static BLOCKING_NOTIFIER_HEAD(acpi_lid_notifier);
 static struct acpi_device *lid_device;
 static u8 lid_init_state = ACPI_BUTTON_LID_INIT_METHOD;
 
+static unsigned long lid_report_interval __read_mostly = 500;
+module_param(lid_report_interval, ulong, 0644);
+MODULE_PARM_DESC(lid_report_interval, "Interval (ms) between lid key events");
+
 /* --------------------------------------------------------------------------
                               FS Interface (/proc)
    -------------------------------------------------------------------------- */
@@ -134,10 +142,79 @@ static int acpi_lid_notify_state(struct acpi_device *device, int state)
 {
 	struct acpi_button *button = acpi_driver_data(device);
 	int ret;
+	ktime_t next_report;
+	bool do_update;
+
+	/*
+	 * In lid_init_state=ignore mode, if user opens/closes lid
+	 * frequently with "open" missing, and "last_time" is also updated
+	 * frequently, "close" cannot be delivered to the userspace.
+	 * So "last_time" is only updated after a timeout or an actual
+	 * switch.
+	 */
+	if (lid_init_state != ACPI_BUTTON_LID_INIT_IGNORE ||
+	    button->last_state != !!state)
+		do_update = true;
+	else
+		do_update = false;
+
+	next_report = ktime_add(button->last_time,
+				ms_to_ktime(lid_report_interval));
+	if (button->last_state == !!state &&
+	    ktime_after(ktime_get(), next_report)) {
+		/* Complain the buggy firmware */
+		pr_warn_once("The lid device is not compliant to SW_LID.\n");
 
-	/* input layer checks if event is redundant */
-	input_report_switch(button->input, SW_LID, !state);
-	input_sync(button->input);
+		/*
+		 * Send the unreliable complement switch event:
+		 *
+		 * On most platforms, the lid device is reliable. However
+		 * there are exceptions:
+		 * 1. Platforms returning initial lid state as "close" by
+		 *    default after booting/resuming:
+		 *     https://bugzilla.kernel.org/show_bug.cgi?id=89211
+		 *     https://bugzilla.kernel.org/show_bug.cgi?id=106151
+		 * 2. Platforms never reporting "open" events:
+		 *     https://bugzilla.kernel.org/show_bug.cgi?id=106941
+		 * On these buggy platforms, the usage model of the ACPI
+		 * lid device actually is:
+		 * 1. The initial returning value of _LID may not be
+		 *    reliable.
+		 * 2. The open event may not be reliable.
+		 * 3. The close event is reliable.
+		 *
+		 * But SW_LID is typed as input switch event, the input
+		 * layer checks if the event is redundant. Hence if the
+		 * state is not switched, the userspace cannot see this
+		 * platform triggered reliable event. By inserting a
+		 * complement switch event, it then is guaranteed that the
+		 * platform triggered reliable one can always be seen by
+		 * the userspace.
+		 */
+		if (lid_init_state == ACPI_BUTTON_LID_INIT_IGNORE) {
+			do_update = true;
+			/*
+			 * Do generate complement switch event for "close"
+			 * as "close" is reliable and wrong "open" won't
+			 * trigger unexpected behaviors.
+			 * Do not generate complement switch event for
+			 * "open" as "open" is not reliable and wrong
+			 * "close" will trigger unexpected behaviors.
+			 */
+			if (!state) {
+				input_report_switch(button->input,
+						    SW_LID, state);
+				input_sync(button->input);
+			}
+		}
+	}
+	/* Send the platform triggered reliable event */
+	if (do_update) {
+		input_report_switch(button->input, SW_LID, !state);
+		input_sync(button->input);
+		button->last_state = !!state;
+		button->last_time = ktime_get();
+	}
 
 	if (state)
 		pm_wakeup_event(&device->dev, 0);
@@ -411,6 +488,8 @@ static int acpi_button_add(struct acpi_device *device)
 		strcpy(name, ACPI_BUTTON_DEVICE_NAME_LID);
 		sprintf(class, "%s/%s",
 			ACPI_BUTTON_CLASS, ACPI_BUTTON_SUBCLASS_LID);
+		button->last_state = !!acpi_lid_evaluate_state(device);
+		button->last_time = ktime_get();
 	} else {
 		printk(KERN_ERR PREFIX "Unsupported hid [%s]\n", hid);
 		error = -ENODEV;
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index 2e981732805b..d0d0504b7c89 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -40,15 +40,48 @@
 #include <linux/cpufreq.h>
 #include <linux/delay.h>
 #include <linux/ktime.h>
+#include <linux/rwsem.h>
+#include <linux/wait.h>
 
 #include <acpi/cppc_acpi.h>
-/*
- * Lock to provide mutually exclusive access to the PCC
- * channel. e.g. When the remote updates the shared region
- * with new data, the reader needs to be protected from
- * other CPUs activity on the same channel.
- */
-static DEFINE_SPINLOCK(pcc_lock);
+
+struct cppc_pcc_data {
+	struct mbox_chan *pcc_channel;
+	void __iomem *pcc_comm_addr;
+	int pcc_subspace_idx;
+	bool pcc_channel_acquired;
+	ktime_t deadline;
+	unsigned int pcc_mpar, pcc_mrtt, pcc_nominal;
+
+	bool pending_pcc_write_cmd;	/* Any pending/batched PCC write cmds? */
+	bool platform_owns_pcc;		/* Ownership of PCC subspace */
+	unsigned int pcc_write_cnt;	/* Running count of PCC write commands */
+
+	/*
+	 * Lock to provide controlled access to the PCC channel.
+	 *
+	 * For performance critical usecases(currently cppc_set_perf)
+	 *	We need to take read_lock and check if channel belongs to OSPM
+	 * before reading or writing to PCC subspace
+	 *	We need to take write_lock before transferring the channel
+	 * ownership to the platform via a Doorbell
+	 *	This allows us to batch a number of CPPC requests if they happen
+	 * to originate in about the same time
+	 *
+	 * For non-performance critical usecases(init)
+	 *	Take write_lock for all purposes which gives exclusive access
+	 */
+	struct rw_semaphore pcc_lock;
+
+	/* Wait queue for CPUs whose requests were batched */
+	wait_queue_head_t pcc_write_wait_q;
+};
+
+/* Structure to represent the single PCC channel */
+static struct cppc_pcc_data pcc_data = {
+	.pcc_subspace_idx = -1,
+	.platform_owns_pcc = true,
+};
 
 /*
  * The cpc_desc structure contains the ACPI register details
@@ -59,18 +92,25 @@ static DEFINE_SPINLOCK(pcc_lock);
  */
 static DEFINE_PER_CPU(struct cpc_desc *, cpc_desc_ptr);
 
-/* This layer handles all the PCC specifics for CPPC. */
-static struct mbox_chan *pcc_channel;
-static void __iomem *pcc_comm_addr;
-static u64 comm_base_addr;
-static int pcc_subspace_idx = -1;
-static bool pcc_channel_acquired;
-static ktime_t deadline;
-static unsigned int pcc_mpar, pcc_mrtt;
-
 /* pcc mapped address + header size + offset within PCC subspace */
-#define GET_PCC_VADDR(offs) (pcc_comm_addr + 0x8 + (offs))
-
+#define GET_PCC_VADDR(offs) (pcc_data.pcc_comm_addr + 0x8 + (offs))
+
+/* Check if a CPC regsiter is in PCC */
+#define CPC_IN_PCC(cpc) ((cpc)->type == ACPI_TYPE_BUFFER &&		\
+				(cpc)->cpc_entry.reg.space_id ==	\
+				ACPI_ADR_SPACE_PLATFORM_COMM)
+
+/* Evalutes to True if reg is a NULL register descriptor */
+#define IS_NULL_REG(reg) ((reg)->space_id ==  ACPI_ADR_SPACE_SYSTEM_MEMORY && \
+				(reg)->address == 0 &&			\
+				(reg)->bit_width == 0 &&		\
+				(reg)->bit_offset == 0 &&		\
+				(reg)->access_width == 0)
+
+/* Evalutes to True if an optional cpc field is supported */
+#define CPC_SUPPORTED(cpc) ((cpc)->type == ACPI_TYPE_INTEGER ?		\
+				!!(cpc)->cpc_entry.int_value :		\
+				!IS_NULL_REG(&(cpc)->cpc_entry.reg))
 /*
  * Arbitrary Retries in case the remote processor is slow to respond
  * to PCC commands. Keeping it high enough to cover emulators where
@@ -78,11 +118,79 @@ static unsigned int pcc_mpar, pcc_mrtt;
  */
 #define NUM_RETRIES 500
 
-static int check_pcc_chan(void)
+struct cppc_attr {
+	struct attribute attr;
+	ssize_t (*show)(struct kobject *kobj,
+			struct attribute *attr, char *buf);
+	ssize_t (*store)(struct kobject *kobj,
+			struct attribute *attr, const char *c, ssize_t count);
+};
+
+#define define_one_cppc_ro(_name)		\
+static struct cppc_attr _name =			\
+__ATTR(_name, 0444, show_##_name, NULL)
+
+#define to_cpc_desc(a) container_of(a, struct cpc_desc, kobj)
+
+static ssize_t show_feedback_ctrs(struct kobject *kobj,
+		struct attribute *attr, char *buf)
+{
+	struct cpc_desc *cpc_ptr = to_cpc_desc(kobj);
+	struct cppc_perf_fb_ctrs fb_ctrs = {0};
+
+	cppc_get_perf_ctrs(cpc_ptr->cpu_id, &fb_ctrs);
+
+	return scnprintf(buf, PAGE_SIZE, "ref:%llu del:%llu\n",
+			fb_ctrs.reference, fb_ctrs.delivered);
+}
+define_one_cppc_ro(feedback_ctrs);
+
+static ssize_t show_reference_perf(struct kobject *kobj,
+		struct attribute *attr, char *buf)
+{
+	struct cpc_desc *cpc_ptr = to_cpc_desc(kobj);
+	struct cppc_perf_fb_ctrs fb_ctrs = {0};
+
+	cppc_get_perf_ctrs(cpc_ptr->cpu_id, &fb_ctrs);
+
+	return scnprintf(buf, PAGE_SIZE, "%llu\n",
+			fb_ctrs.reference_perf);
+}
+define_one_cppc_ro(reference_perf);
+
+static ssize_t show_wraparound_time(struct kobject *kobj,
+				struct attribute *attr, char *buf)
+{
+	struct cpc_desc *cpc_ptr = to_cpc_desc(kobj);
+	struct cppc_perf_fb_ctrs fb_ctrs = {0};
+
+	cppc_get_perf_ctrs(cpc_ptr->cpu_id, &fb_ctrs);
+
+	return scnprintf(buf, PAGE_SIZE, "%llu\n", fb_ctrs.ctr_wrap_time);
+
+}
+define_one_cppc_ro(wraparound_time);
+
+static struct attribute *cppc_attrs[] = {
+	&feedback_ctrs.attr,
+	&reference_perf.attr,
+	&wraparound_time.attr,
+	NULL
+};
+
+static struct kobj_type cppc_ktype = {
+	.sysfs_ops = &kobj_sysfs_ops,
+	.default_attrs = cppc_attrs,
+};
+
+static int check_pcc_chan(bool chk_err_bit)
 {
-	int ret = -EIO;
-	struct acpi_pcct_shared_memory __iomem *generic_comm_base = pcc_comm_addr;
-	ktime_t next_deadline = ktime_add(ktime_get(), deadline);
+	int ret = -EIO, status = 0;
+	struct acpi_pcct_shared_memory __iomem *generic_comm_base = pcc_data.pcc_comm_addr;
+	ktime_t next_deadline = ktime_add(ktime_get(), pcc_data.deadline);
+
+	if (!pcc_data.platform_owns_pcc)
+		return 0;
 
 	/* Retry in case the remote processor was too slow to catch up. */
 	while (!ktime_after(ktime_get(), next_deadline)) {
@@ -91,8 +199,11 @@ static int check_pcc_chan(void)
 		 * platform and should have set the command completion bit when
 		 * PCC can be used by OSPM
 		 */
-		if (readw_relaxed(&generic_comm_base->status) & PCC_CMD_COMPLETE) {
+		status = readw_relaxed(&generic_comm_base->status);
+		if (status & PCC_CMD_COMPLETE_MASK) {
 			ret = 0;
+			if (chk_err_bit && (status & PCC_ERROR_MASK))
+				ret = -EIO;
 			break;
 		}
 		/*
@@ -102,14 +213,23 @@ static int check_pcc_chan(void)
 		udelay(3);
 	}
 
+	if (likely(!ret))
+		pcc_data.platform_owns_pcc = false;
+	else
+		pr_err("PCC check channel failed. Status=%x\n", status);
+
 	return ret;
 }
 
+/*
+ * This function transfers the ownership of the PCC to the platform
+ * So it must be called while holding write_lock(pcc_lock)
+ */
 static int send_pcc_cmd(u16 cmd)
 {
-	int ret = -EIO;
+	int ret = -EIO, i;
 	struct acpi_pcct_shared_memory *generic_comm_base =
-		(struct acpi_pcct_shared_memory *) pcc_comm_addr;
+		(struct acpi_pcct_shared_memory *) pcc_data.pcc_comm_addr;
 	static ktime_t last_cmd_cmpl_time, last_mpar_reset;
 	static int mpar_count;
 	unsigned int time_delta;
@@ -119,20 +239,29 @@ static int send_pcc_cmd(u16 cmd)
 	 * the channel before writing to PCC space
 	 */
 	if (cmd == CMD_READ) {
-		ret = check_pcc_chan();
+		/*
+		 * If there are pending cpc_writes, then we stole the channel
+		 * before write completion, so first send a WRITE command to
+		 * platform
+		 */
+		if (pcc_data.pending_pcc_write_cmd)
+			send_pcc_cmd(CMD_WRITE);
+
+		ret = check_pcc_chan(false);
 		if (ret)
-			return ret;
-	}
+			goto end;
+	} else /* CMD_WRITE */
+		pcc_data.pending_pcc_write_cmd = FALSE;
 
 	/*
 	 * Handle the Minimum Request Turnaround Time(MRTT)
 	 * "The minimum amount of time that OSPM must wait after the completion
 	 * of a command before issuing the next command, in microseconds"
 	 */
-	if (pcc_mrtt) {
+	if (pcc_data.pcc_mrtt) {
 		time_delta = ktime_us_delta(ktime_get(), last_cmd_cmpl_time);
-		if (pcc_mrtt > time_delta)
-			udelay(pcc_mrtt - time_delta);
+		if (pcc_data.pcc_mrtt > time_delta)
+			udelay(pcc_data.pcc_mrtt - time_delta);
 	}
 
 	/*
@@ -146,15 +275,16 @@ static int send_pcc_cmd(u16 cmd)
 	 * not send the request to the platform after hitting the MPAR limit in
 	 * any 60s window
 	 */
-	if (pcc_mpar) {
+	if (pcc_data.pcc_mpar) {
 		if (mpar_count == 0) {
 			time_delta = ktime_ms_delta(ktime_get(), last_mpar_reset);
 			if (time_delta < 60 * MSEC_PER_SEC) {
 				pr_debug("PCC cmd not sent due to MPAR limit");
-				return -EIO;
+				ret = -EIO;
+				goto end;
 			}
 			last_mpar_reset = ktime_get();
-			mpar_count = pcc_mpar;
+			mpar_count = pcc_data.pcc_mpar;
 		}
 		mpar_count--;
 	}
@@ -165,33 +295,43 @@ static int send_pcc_cmd(u16 cmd)
 	/* Flip CMD COMPLETE bit */
 	writew_relaxed(0, &generic_comm_base->status);
 
+	pcc_data.platform_owns_pcc = true;
+
 	/* Ring doorbell */
-	ret = mbox_send_message(pcc_channel, &cmd);
+	ret = mbox_send_message(pcc_data.pcc_channel, &cmd);
 	if (ret < 0) {
 		pr_err("Err sending PCC mbox message. cmd:%d, ret:%d\n",
 				cmd, ret);
-		return ret;
+		goto end;
 	}
 
-	/*
-	 * For READs we need to ensure the cmd completed to ensure
-	 * the ensuing read()s can proceed. For WRITEs we dont care
-	 * because the actual write()s are done before coming here
-	 * and the next READ or WRITE will check if the channel
-	 * is busy/free at the entry of this call.
-	 *
-	 * If Minimum Request Turnaround Time is non-zero, we need
-	 * to record the completion time of both READ and WRITE
-	 * command for proper handling of MRTT, so we need to check
-	 * for pcc_mrtt in addition to CMD_READ
-	 */
-	if (cmd == CMD_READ || pcc_mrtt) {
-		ret = check_pcc_chan();
-		if (pcc_mrtt)
-			last_cmd_cmpl_time = ktime_get();
+	/* wait for completion and check for PCC errro bit */
+	ret = check_pcc_chan(true);
+
+	if (pcc_data.pcc_mrtt)
+		last_cmd_cmpl_time = ktime_get();
+
+	if (pcc_data.pcc_channel->mbox->txdone_irq)
+		mbox_chan_txdone(pcc_data.pcc_channel, ret);
+	else
+		mbox_client_txdone(pcc_data.pcc_channel, ret);
+
+end:
+	if (cmd == CMD_WRITE) {
+		if (unlikely(ret)) {
+			for_each_possible_cpu(i) {
+				struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i);
+				if (!desc)
+					continue;
+
+				if (desc->write_cmd_id == pcc_data.pcc_write_cnt)
+					desc->write_cmd_status = ret;
+			}
+		}
+		pcc_data.pcc_write_cnt++;
+		wake_up_all(&pcc_data.pcc_write_wait_q);
 	}
 
-	mbox_client_txdone(pcc_channel, ret);
 	return ret;
 }
 
@@ -272,13 +412,13 @@ end:
  *
  *	Return: 0 for success or negative value for err.
  */
-int acpi_get_psd_map(struct cpudata **all_cpu_data)
+int acpi_get_psd_map(struct cppc_cpudata **all_cpu_data)
 {
 	int count_target;
 	int retval = 0;
 	unsigned int i, j;
 	cpumask_var_t covered_cpus;
-	struct cpudata *pr, *match_pr;
+	struct cppc_cpudata *pr, *match_pr;
 	struct acpi_psd_package *pdomain;
 	struct acpi_psd_package *match_pdomain;
 	struct cpc_desc *cpc_ptr, *match_cpc_ptr;
@@ -394,14 +534,13 @@ EXPORT_SYMBOL_GPL(acpi_get_psd_map);
 static int register_pcc_channel(int pcc_subspace_idx)
 {
 	struct acpi_pcct_hw_reduced *cppc_ss;
-	unsigned int len;
 	u64 usecs_lat;
 
 	if (pcc_subspace_idx >= 0) {
-		pcc_channel = pcc_mbox_request_channel(&cppc_mbox_cl,
+		pcc_data.pcc_channel = pcc_mbox_request_channel(&cppc_mbox_cl,
 				pcc_subspace_idx);
 
-		if (IS_ERR(pcc_channel)) {
+		if (IS_ERR(pcc_data.pcc_channel)) {
 			pr_err("Failed to find PCC communication channel\n");
 			return -ENODEV;
 		}
@@ -412,7 +551,7 @@ static int register_pcc_channel(int pcc_subspace_idx)
 		 * PCC channels) and stored pointers to the
 		 * subspace communication region in con_priv.
 		 */
-		cppc_ss = pcc_channel->con_priv;
+		cppc_ss = (pcc_data.pcc_channel)->con_priv;
 
 		if (!cppc_ss) {
 			pr_err("No PCC subspace found for CPPC\n");
@@ -420,35 +559,42 @@ static int register_pcc_channel(int pcc_subspace_idx)
 		}
 
 		/*
-		 * This is the shared communication region
-		 * for the OS and Platform to communicate over.
-		 */
-		comm_base_addr = cppc_ss->base_address;
-		len = cppc_ss->length;
-
-		/*
 		 * cppc_ss->latency is just a Nominal value. In reality
 		 * the remote processor could be much slower to reply.
 		 * So add an arbitrary amount of wait on top of Nominal.
 		 */
 		usecs_lat = NUM_RETRIES * cppc_ss->latency;
-		deadline = ns_to_ktime(usecs_lat * NSEC_PER_USEC);
-		pcc_mrtt = cppc_ss->min_turnaround_time;
-		pcc_mpar = cppc_ss->max_access_rate;
+		pcc_data.deadline = ns_to_ktime(usecs_lat * NSEC_PER_USEC);
+		pcc_data.pcc_mrtt = cppc_ss->min_turnaround_time;
+		pcc_data.pcc_mpar = cppc_ss->max_access_rate;
+		pcc_data.pcc_nominal = cppc_ss->latency;
 
-		pcc_comm_addr = acpi_os_ioremap(comm_base_addr, len);
-		if (!pcc_comm_addr) {
+		pcc_data.pcc_comm_addr = acpi_os_ioremap(cppc_ss->base_address, cppc_ss->length);
+		if (!pcc_data.pcc_comm_addr) {
 			pr_err("Failed to ioremap PCC comm region mem\n");
 			return -ENOMEM;
 		}
 
 		/* Set flag so that we dont come here for each CPU. */
-		pcc_channel_acquired = true;
+		pcc_data.pcc_channel_acquired = true;
 	}
 
 	return 0;
 }
 
+/**
+ * cpc_ffh_supported() - check if FFH reading supported
+ *
+ * Check if the architecture has support for functional fixed hardware
+ * read/write capability.
+ *
+ * Return: true for supported, false for not supported
+ */
+bool __weak cpc_ffh_supported(void)
+{
+	return false;
+}
+
 /*
  * An example CPC table looks like the following.
  *
@@ -507,6 +653,7 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 	union acpi_object *out_obj, *cpc_obj;
 	struct cpc_desc *cpc_ptr;
 	struct cpc_reg *gas_t;
+	struct device *cpu_dev;
 	acpi_handle handle = pr->handle;
 	unsigned int num_ent, i, cpc_rev;
 	acpi_status status;
@@ -545,6 +692,8 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 		goto out_free;
 	}
 
+	cpc_ptr->num_entries = num_ent;
+
 	/* Second entry should be revision. */
 	cpc_obj = &out_obj->package.elements[1];
 	if (cpc_obj->type == ACPI_TYPE_INTEGER)	{
@@ -579,16 +728,27 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 			 * so extract it only once.
 			 */
 			if (gas_t->space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
-				if (pcc_subspace_idx < 0)
-					pcc_subspace_idx = gas_t->access_width;
-				else if (pcc_subspace_idx != gas_t->access_width) {
+				if (pcc_data.pcc_subspace_idx < 0)
+					pcc_data.pcc_subspace_idx = gas_t->access_width;
+				else if (pcc_data.pcc_subspace_idx != gas_t->access_width) {
 					pr_debug("Mismatched PCC ids.\n");
 					goto out_free;
 				}
-			} else if (gas_t->space_id != ACPI_ADR_SPACE_SYSTEM_MEMORY) {
-				/* Support only PCC and SYS MEM type regs */
-				pr_debug("Unsupported register type: %d\n", gas_t->space_id);
-				goto out_free;
+			} else if (gas_t->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
+				if (gas_t->address) {
+					void __iomem *addr;
+
+					addr = ioremap(gas_t->address, gas_t->bit_width/8);
+					if (!addr)
+						goto out_free;
+					cpc_ptr->cpc_regs[i-2].sys_mem_vaddr = addr;
+				}
+			} else {
+				if (gas_t->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE || !cpc_ffh_supported()) {
+					/* Support only PCC ,SYS MEM and FFH type regs */
+					pr_debug("Unsupported register type: %d\n", gas_t->space_id);
+					goto out_free;
+				}
 			}
 
 			cpc_ptr->cpc_regs[i-2].type = ACPI_TYPE_BUFFER;
@@ -607,10 +767,13 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 		goto out_free;
 
 	/* Register PCC channel once for all CPUs. */
-	if (!pcc_channel_acquired) {
-		ret = register_pcc_channel(pcc_subspace_idx);
+	if (!pcc_data.pcc_channel_acquired) {
+		ret = register_pcc_channel(pcc_data.pcc_subspace_idx);
 		if (ret)
 			goto out_free;
+
+		init_rwsem(&pcc_data.pcc_lock);
+		init_waitqueue_head(&pcc_data.pcc_write_wait_q);
 	}
 
 	/* Plug PSD data into this CPUs CPC descriptor. */
@@ -619,10 +782,27 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 	/* Everything looks okay */
 	pr_debug("Parsed CPC struct for CPU: %d\n", pr->id);
 
+	/* Add per logical CPU nodes for reading its feedback counters. */
+	cpu_dev = get_cpu_device(pr->id);
+	if (!cpu_dev)
+		goto out_free;
+
+	ret = kobject_init_and_add(&cpc_ptr->kobj, &cppc_ktype, &cpu_dev->kobj,
+			"acpi_cppc");
+	if (ret)
+		goto out_free;
+
 	kfree(output.pointer);
 	return 0;
 
 out_free:
+	/* Free all the mapped sys mem areas for this CPU */
+	for (i = 2; i < cpc_ptr->num_entries; i++) {
+		void __iomem *addr = cpc_ptr->cpc_regs[i-2].sys_mem_vaddr;
+
+		if (addr)
+			iounmap(addr);
+	}
 	kfree(cpc_ptr);
 
 out_buf_free:
@@ -640,26 +820,82 @@ EXPORT_SYMBOL_GPL(acpi_cppc_processor_probe);
 void acpi_cppc_processor_exit(struct acpi_processor *pr)
 {
 	struct cpc_desc *cpc_ptr;
+	unsigned int i;
+	void __iomem *addr;
+
 	cpc_ptr = per_cpu(cpc_desc_ptr, pr->id);
+
+	/* Free all the mapped sys mem areas for this CPU */
+	for (i = 2; i < cpc_ptr->num_entries; i++) {
+		addr = cpc_ptr->cpc_regs[i-2].sys_mem_vaddr;
+		if (addr)
+			iounmap(addr);
+	}
+
+	kobject_put(&cpc_ptr->kobj);
 	kfree(cpc_ptr);
 }
 EXPORT_SYMBOL_GPL(acpi_cppc_processor_exit);
 
+/**
+ * cpc_read_ffh() - Read FFH register
+ * @cpunum:	cpu number to read
+ * @reg:	cppc register information
+ * @val:	place holder for return value
+ *
+ * Read bit_width bits from a specified address and bit_offset
+ *
+ * Return: 0 for success and error code
+ */
+int __weak cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val)
+{
+	return -ENOTSUPP;
+}
+
+/**
+ * cpc_write_ffh() - Write FFH register
+ * @cpunum:	cpu number to write
+ * @reg:	cppc register information
+ * @val:	value to write
+ *
+ * Write value of bit_width bits to a specified address and bit_offset
+ *
+ * Return: 0 for success and error code
+ */
+int __weak cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val)
+{
+	return -ENOTSUPP;
+}
+
 /*
  * Since cpc_read and cpc_write are called while holding pcc_lock, it should be
  * as fast as possible. We have already mapped the PCC subspace during init, so
  * we can directly write to it.
  */
 
-static int cpc_read(struct cpc_reg *reg, u64 *val)
+static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val)
 {
 	int ret_val = 0;
+	void __iomem *vaddr = 0;
+	struct cpc_reg *reg = &reg_res->cpc_entry.reg;
+
+	if (reg_res->type == ACPI_TYPE_INTEGER) {
+		*val = reg_res->cpc_entry.int_value;
+		return ret_val;
+	}
 
 	*val = 0;
-	if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
-		void __iomem *vaddr = GET_PCC_VADDR(reg->address);
+	if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM)
+		vaddr = GET_PCC_VADDR(reg->address);
+	else if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY)
+		vaddr = reg_res->sys_mem_vaddr;
+	else if (reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE)
+		return cpc_read_ffh(cpu, reg, val);
+	else
+		return acpi_os_read_memory((acpi_physical_address)reg->address,
+				val, reg->bit_width);
 
-		switch (reg->bit_width) {
+	switch (reg->bit_width) {
 		case 8:
 			*val = readb_relaxed(vaddr);
 			break;
@@ -674,23 +910,30 @@ static int cpc_read(struct cpc_reg *reg, u64 *val)
 			break;
 		default:
 			pr_debug("Error: Cannot read %u bit width from PCC\n",
-				reg->bit_width);
+					reg->bit_width);
 			ret_val = -EFAULT;
-		}
-	} else
-		ret_val = acpi_os_read_memory((acpi_physical_address)reg->address,
-					val, reg->bit_width);
+	}
+
 	return ret_val;
 }
 
-static int cpc_write(struct cpc_reg *reg, u64 val)
+static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val)
 {
 	int ret_val = 0;
+	void __iomem *vaddr = 0;
+	struct cpc_reg *reg = &reg_res->cpc_entry.reg;
+
+	if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM)
+		vaddr = GET_PCC_VADDR(reg->address);
+	else if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY)
+		vaddr = reg_res->sys_mem_vaddr;
+	else if (reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE)
+		return cpc_write_ffh(cpu, reg, val);
+	else
+		return acpi_os_write_memory((acpi_physical_address)reg->address,
+				val, reg->bit_width);
 
-	if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
-		void __iomem *vaddr = GET_PCC_VADDR(reg->address);
-
-		switch (reg->bit_width) {
+	switch (reg->bit_width) {
 		case 8:
 			writeb_relaxed(val, vaddr);
 			break;
@@ -705,13 +948,11 @@ static int cpc_write(struct cpc_reg *reg, u64 val)
 			break;
 		default:
 			pr_debug("Error: Cannot write %u bit width to PCC\n",
-				reg->bit_width);
+					reg->bit_width);
 			ret_val = -EFAULT;
 			break;
-		}
-	} else
-		ret_val = acpi_os_write_memory((acpi_physical_address)reg->address,
-				val, reg->bit_width);
+	}
+
 	return ret_val;
 }
 
@@ -727,8 +968,8 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum);
 	struct cpc_register_resource *highest_reg, *lowest_reg, *ref_perf,
 								 *nom_perf;
-	u64 high, low, ref, nom;
-	int ret = 0;
+	u64 high, low, nom;
+	int ret = 0, regs_in_pcc = 0;
 
 	if (!cpc_desc) {
 		pr_debug("No CPC descriptor for CPU:%d\n", cpunum);
@@ -740,13 +981,11 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 	ref_perf = &cpc_desc->cpc_regs[REFERENCE_PERF];
 	nom_perf = &cpc_desc->cpc_regs[NOMINAL_PERF];
 
-	spin_lock(&pcc_lock);
-
 	/* Are any of the regs PCC ?*/
-	if ((highest_reg->cpc_entry.reg.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) ||
-			(lowest_reg->cpc_entry.reg.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) ||
-			(ref_perf->cpc_entry.reg.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) ||
-			(nom_perf->cpc_entry.reg.space_id == ACPI_ADR_SPACE_PLATFORM_COMM)) {
+	if (CPC_IN_PCC(highest_reg) || CPC_IN_PCC(lowest_reg) ||
+		CPC_IN_PCC(ref_perf) || CPC_IN_PCC(nom_perf)) {
+		regs_in_pcc = 1;
+		down_write(&pcc_data.pcc_lock);
 		/* Ring doorbell once to update PCC subspace */
 		if (send_pcc_cmd(CMD_READ) < 0) {
 			ret = -EIO;
@@ -754,26 +993,21 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 		}
 	}
 
-	cpc_read(&highest_reg->cpc_entry.reg, &high);
+	cpc_read(cpunum, highest_reg, &high);
 	perf_caps->highest_perf = high;
 
-	cpc_read(&lowest_reg->cpc_entry.reg, &low);
+	cpc_read(cpunum, lowest_reg, &low);
 	perf_caps->lowest_perf = low;
 
-	cpc_read(&ref_perf->cpc_entry.reg, &ref);
-	perf_caps->reference_perf = ref;
-
-	cpc_read(&nom_perf->cpc_entry.reg, &nom);
+	cpc_read(cpunum, nom_perf, &nom);
 	perf_caps->nominal_perf = nom;
 
-	if (!ref)
-		perf_caps->reference_perf = perf_caps->nominal_perf;
-
 	if (!high || !low || !nom)
 		ret = -EFAULT;
 
 out_err:
-	spin_unlock(&pcc_lock);
+	if (regs_in_pcc)
+		up_write(&pcc_data.pcc_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(cppc_get_perf_caps);
@@ -788,9 +1022,10 @@ EXPORT_SYMBOL_GPL(cppc_get_perf_caps);
 int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs)
 {
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum);
-	struct cpc_register_resource *delivered_reg, *reference_reg;
-	u64 delivered, reference;
-	int ret = 0;
+	struct cpc_register_resource *delivered_reg, *reference_reg,
+		*ref_perf_reg, *ctr_wrap_reg;
+	u64 delivered, reference, ref_perf, ctr_wrap_time;
+	int ret = 0, regs_in_pcc = 0;
 
 	if (!cpc_desc) {
 		pr_debug("No CPC descriptor for CPU:%d\n", cpunum);
@@ -799,12 +1034,21 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs)
 
 	delivered_reg = &cpc_desc->cpc_regs[DELIVERED_CTR];
 	reference_reg = &cpc_desc->cpc_regs[REFERENCE_CTR];
+	ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF];
+	ctr_wrap_reg = &cpc_desc->cpc_regs[CTR_WRAP_TIME];
 
-	spin_lock(&pcc_lock);
+	/*
+	 * If refernce perf register is not supported then we should
+	 * use the nominal perf value
+	 */
+	if (!CPC_SUPPORTED(ref_perf_reg))
+		ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF];
 
 	/* Are any of the regs PCC ?*/
-	if ((delivered_reg->cpc_entry.reg.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) ||
-			(reference_reg->cpc_entry.reg.space_id == ACPI_ADR_SPACE_PLATFORM_COMM)) {
+	if (CPC_IN_PCC(delivered_reg) || CPC_IN_PCC(reference_reg) ||
+		CPC_IN_PCC(ctr_wrap_reg) || CPC_IN_PCC(ref_perf_reg)) {
+		down_write(&pcc_data.pcc_lock);
+		regs_in_pcc = 1;
 		/* Ring doorbell once to update PCC subspace */
 		if (send_pcc_cmd(CMD_READ) < 0) {
 			ret = -EIO;
@@ -812,25 +1056,31 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs)
 		}
 	}
 
-	cpc_read(&delivered_reg->cpc_entry.reg, &delivered);
-	cpc_read(&reference_reg->cpc_entry.reg, &reference);
+	cpc_read(cpunum, delivered_reg, &delivered);
+	cpc_read(cpunum, reference_reg, &reference);
+	cpc_read(cpunum, ref_perf_reg, &ref_perf);
 
-	if (!delivered || !reference) {
+	/*
+	 * Per spec, if ctr_wrap_time optional register is unsupported, then the
+	 * performance counters are assumed to never wrap during the lifetime of
+	 * platform
+	 */
+	ctr_wrap_time = (u64)(~((u64)0));
+	if (CPC_SUPPORTED(ctr_wrap_reg))
+		cpc_read(cpunum, ctr_wrap_reg, &ctr_wrap_time);
+
+	if (!delivered || !reference ||	!ref_perf) {
 		ret = -EFAULT;
 		goto out_err;
 	}
 
 	perf_fb_ctrs->delivered = delivered;
 	perf_fb_ctrs->reference = reference;
-
-	perf_fb_ctrs->delivered -= perf_fb_ctrs->prev_delivered;
-	perf_fb_ctrs->reference -= perf_fb_ctrs->prev_reference;
-
-	perf_fb_ctrs->prev_delivered = delivered;
-	perf_fb_ctrs->prev_reference = reference;
-
+	perf_fb_ctrs->reference_perf = ref_perf;
+	perf_fb_ctrs->ctr_wrap_time = ctr_wrap_time;
 out_err:
-	spin_unlock(&pcc_lock);
+	if (regs_in_pcc)
+		up_write(&pcc_data.pcc_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(cppc_get_perf_ctrs);
@@ -855,30 +1105,142 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 
 	desired_reg = &cpc_desc->cpc_regs[DESIRED_PERF];
 
-	spin_lock(&pcc_lock);
-
-	/* If this is PCC reg, check if channel is free before writing */
-	if (desired_reg->cpc_entry.reg.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
-		ret = check_pcc_chan();
-		if (ret)
-			goto busy_channel;
+	/*
+	 * This is Phase-I where we want to write to CPC registers
+	 * -> We want all CPUs to be able to execute this phase in parallel
+	 *
+	 * Since read_lock can be acquired by multiple CPUs simultaneously we
+	 * achieve that goal here
+	 */
+	if (CPC_IN_PCC(desired_reg)) {
+		down_read(&pcc_data.pcc_lock);	/* BEGIN Phase-I */
+		if (pcc_data.platform_owns_pcc) {
+			ret = check_pcc_chan(false);
+			if (ret) {
+				up_read(&pcc_data.pcc_lock);
+				return ret;
+			}
+		}
+		/*
+		 * Update the pending_write to make sure a PCC CMD_READ will not
+		 * arrive and steal the channel during the switch to write lock
+		 */
+		pcc_data.pending_pcc_write_cmd = true;
+		cpc_desc->write_cmd_id = pcc_data.pcc_write_cnt;
+		cpc_desc->write_cmd_status = 0;
 	}
 
 	/*
 	 * Skip writing MIN/MAX until Linux knows how to come up with
 	 * useful values.
 	 */
-	cpc_write(&desired_reg->cpc_entry.reg, perf_ctrls->desired_perf);
+	cpc_write(cpu, desired_reg, perf_ctrls->desired_perf);
 
-	/* Is this a PCC reg ?*/
-	if (desired_reg->cpc_entry.reg.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
-		/* Ring doorbell so Remote can get our perf request. */
-		if (send_pcc_cmd(CMD_WRITE) < 0)
-			ret = -EIO;
+	if (CPC_IN_PCC(desired_reg))
+		up_read(&pcc_data.pcc_lock);	/* END Phase-I */
+	/*
+	 * This is Phase-II where we transfer the ownership of PCC to Platform
+	 *
+	 * Short Summary: Basically if we think of a group of cppc_set_perf
+	 * requests that happened in short overlapping interval. The last CPU to
+	 * come out of Phase-I will enter Phase-II and ring the doorbell.
+	 *
+	 * We have the following requirements for Phase-II:
+	 *     1. We want to execute Phase-II only when there are no CPUs
+	 * currently executing in Phase-I
+	 *     2. Once we start Phase-II we want to avoid all other CPUs from
+	 * entering Phase-I.
+	 *     3. We want only one CPU among all those who went through Phase-I
+	 * to run phase-II
+	 *
+	 * If write_trylock fails to get the lock and doesn't transfer the
+	 * PCC ownership to the platform, then one of the following will be TRUE
+	 *     1. There is at-least one CPU in Phase-I which will later execute
+	 * write_trylock, so the CPUs in Phase-I will be responsible for
+	 * executing the Phase-II.
+	 *     2. Some other CPU has beaten this CPU to successfully execute the
+	 * write_trylock and has already acquired the write_lock. We know for a
+	 * fact it(other CPU acquiring the write_lock) couldn't have happened
+	 * before this CPU's Phase-I as we held the read_lock.
+	 *     3. Some other CPU executing pcc CMD_READ has stolen the
+	 * down_write, in which case, send_pcc_cmd will check for pending
+	 * CMD_WRITE commands by checking the pending_pcc_write_cmd.
+	 * So this CPU can be certain that its request will be delivered
+	 *    So in all cases, this CPU knows that its request will be delivered
+	 * by another CPU and can return
+	 *
+	 * After getting the down_write we still need to check for
+	 * pending_pcc_write_cmd to take care of the following scenario
+	 *    The thread running this code could be scheduled out between
+	 * Phase-I and Phase-II. Before it is scheduled back on, another CPU
+	 * could have delivered the request to Platform by triggering the
+	 * doorbell and transferred the ownership of PCC to platform. So this
+	 * avoids triggering an unnecessary doorbell and more importantly before
+	 * triggering the doorbell it makes sure that the PCC channel ownership
+	 * is still with OSPM.
+	 *   pending_pcc_write_cmd can also be cleared by a different CPU, if
+	 * there was a pcc CMD_READ waiting on down_write and it steals the lock
+	 * before the pcc CMD_WRITE is completed. pcc_send_cmd checks for this
+	 * case during a CMD_READ and if there are pending writes it delivers
+	 * the write command before servicing the read command
+	 */
+	if (CPC_IN_PCC(desired_reg)) {
+		if (down_write_trylock(&pcc_data.pcc_lock)) {	/* BEGIN Phase-II */
+			/* Update only if there are pending write commands */
+			if (pcc_data.pending_pcc_write_cmd)
+				send_pcc_cmd(CMD_WRITE);
+			up_write(&pcc_data.pcc_lock);		/* END Phase-II */
+		} else
+			/* Wait until pcc_write_cnt is updated by send_pcc_cmd */
+			wait_event(pcc_data.pcc_write_wait_q,
+				cpc_desc->write_cmd_id != pcc_data.pcc_write_cnt);
+
+		/* send_pcc_cmd updates the status in case of failure */
+		ret = cpc_desc->write_cmd_status;
 	}
-busy_channel:
-	spin_unlock(&pcc_lock);
-
 	return ret;
 }
 EXPORT_SYMBOL_GPL(cppc_set_perf);
+
+/**
+ * cppc_get_transition_latency - returns frequency transition latency in ns
+ *
+ * ACPI CPPC does not explicitly specifiy how a platform can specify the
+ * transition latency for perfromance change requests. The closest we have
+ * is the timing information from the PCCT tables which provides the info
+ * on the number and frequency of PCC commands the platform can handle.
+ */
+unsigned int cppc_get_transition_latency(int cpu_num)
+{
+	/*
+	 * Expected transition latency is based on the PCCT timing values
+	 * Below are definition from ACPI spec:
+	 * pcc_nominal- Expected latency to process a command, in microseconds
+	 * pcc_mpar   - The maximum number of periodic requests that the subspace
+	 *              channel can support, reported in commands per minute. 0
+	 *              indicates no limitation.
+	 * pcc_mrtt   - The minimum amount of time that OSPM must wait after the
+	 *              completion of a command before issuing the next command,
+	 *              in microseconds.
+	 */
+	unsigned int latency_ns = 0;
+	struct cpc_desc *cpc_desc;
+	struct cpc_register_resource *desired_reg;
+
+	cpc_desc = per_cpu(cpc_desc_ptr, cpu_num);
+	if (!cpc_desc)
+		return CPUFREQ_ETERNAL;
+
+	desired_reg = &cpc_desc->cpc_regs[DESIRED_PERF];
+	if (!CPC_IN_PCC(desired_reg))
+		return CPUFREQ_ETERNAL;
+
+	if (pcc_data.pcc_mpar)
+		latency_ns = 60 * (1000 * 1000 * 1000 / pcc_data.pcc_mpar);
+
+	latency_ns = max(latency_ns, pcc_data.pcc_nominal * 1000);
+	latency_ns = max(latency_ns, pcc_data.pcc_mrtt * 1000);
+
+	return latency_ns;
+}
+EXPORT_SYMBOL_GPL(cppc_get_transition_latency);
diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
index e7bd57cc550a..680531062160 100644
--- a/drivers/acpi/ec.c
+++ b/drivers/acpi/ec.c
@@ -104,10 +104,12 @@ enum ec_command {
 #define ACPI_EC_MAX_QUERIES	16	/* Maximum number of parallel queries */
 
 enum {
+	EC_FLAGS_QUERY_ENABLED,		/* Query is enabled */
 	EC_FLAGS_QUERY_PENDING,		/* Query is pending */
 	EC_FLAGS_QUERY_GUARDING,	/* Guard for SCI_EVT check */
 	EC_FLAGS_GPE_HANDLER_INSTALLED,	/* GPE handler installed */
 	EC_FLAGS_EC_HANDLER_INSTALLED,	/* OpReg handler installed */
+	EC_FLAGS_EVT_HANDLER_INSTALLED, /* _Qxx handlers installed */
 	EC_FLAGS_STARTED,		/* Driver is started */
 	EC_FLAGS_STOPPED,		/* Driver is stopped */
 	EC_FLAGS_COMMAND_STORM,		/* GPE storms occurred to the
@@ -145,6 +147,10 @@ static unsigned int ec_storm_threshold  __read_mostly = 8;
 module_param(ec_storm_threshold, uint, 0644);
 MODULE_PARM_DESC(ec_storm_threshold, "Maxim false GPE numbers not considered as GPE storm");
 
+static bool ec_freeze_events __read_mostly = true;
+module_param(ec_freeze_events, bool, 0644);
+MODULE_PARM_DESC(ec_freeze_events, "Disabling event handling during suspend/resume");
+
 struct acpi_ec_query_handler {
 	struct list_head node;
 	acpi_ec_query_func func;
@@ -179,6 +185,7 @@ static void acpi_ec_event_processor(struct work_struct *work);
 
 struct acpi_ec *boot_ec, *first_ec;
 EXPORT_SYMBOL(first_ec);
+static bool boot_ec_is_ecdt = false;
 static struct workqueue_struct *ec_query_wq;
 
 static int EC_FLAGS_CLEAR_ON_RESUME; /* Needs acpi_ec_clear() on boot/resume */
@@ -239,6 +246,30 @@ static bool acpi_ec_started(struct acpi_ec *ec)
 	       !test_bit(EC_FLAGS_STOPPED, &ec->flags);
 }
 
+static bool acpi_ec_event_enabled(struct acpi_ec *ec)
+{
+	/*
+	 * There is an OSPM early stage logic. During the early stages
+	 * (boot/resume), OSPMs shouldn't enable the event handling, only
+	 * the EC transactions are allowed to be performed.
+	 */
+	if (!test_bit(EC_FLAGS_QUERY_ENABLED, &ec->flags))
+		return false;
+	/*
+	 * However, disabling the event handling is experimental for late
+	 * stage (suspend), and is controlled by the boot parameter of
+	 * "ec_freeze_events":
+	 * 1. true:  The EC event handling is disabled before entering
+	 *           the noirq stage.
+	 * 2. false: The EC event handling is automatically disabled as
+	 *           soon as the EC driver is stopped.
+	 */
+	if (ec_freeze_events)
+		return acpi_ec_started(ec);
+	else
+		return test_bit(EC_FLAGS_STARTED, &ec->flags);
+}
+
 static bool acpi_ec_flushed(struct acpi_ec *ec)
 {
 	return ec->reference_count == 1;
@@ -429,7 +460,8 @@ static bool acpi_ec_submit_flushable_request(struct acpi_ec *ec)
 
 static void acpi_ec_submit_query(struct acpi_ec *ec)
 {
-	if (!test_and_set_bit(EC_FLAGS_QUERY_PENDING, &ec->flags)) {
+	if (acpi_ec_event_enabled(ec) &&
+	    !test_and_set_bit(EC_FLAGS_QUERY_PENDING, &ec->flags)) {
 		ec_dbg_evt("Command(%s) submitted/blocked",
 			   acpi_ec_cmd_string(ACPI_EC_COMMAND_QUERY));
 		ec->nr_pending_queries++;
@@ -446,6 +478,86 @@ static void acpi_ec_complete_query(struct acpi_ec *ec)
 	}
 }
 
+static inline void __acpi_ec_enable_event(struct acpi_ec *ec)
+{
+	if (!test_and_set_bit(EC_FLAGS_QUERY_ENABLED, &ec->flags))
+		ec_log_drv("event unblocked");
+	if (!test_bit(EC_FLAGS_QUERY_PENDING, &ec->flags))
+		advance_transaction(ec);
+}
+
+static inline void __acpi_ec_disable_event(struct acpi_ec *ec)
+{
+	if (test_and_clear_bit(EC_FLAGS_QUERY_ENABLED, &ec->flags))
+		ec_log_drv("event blocked");
+}
+
+/*
+ * Process _Q events that might have accumulated in the EC.
+ * Run with locked ec mutex.
+ */
+static void acpi_ec_clear(struct acpi_ec *ec)
+{
+	int i, status;
+	u8 value = 0;
+
+	for (i = 0; i < ACPI_EC_CLEAR_MAX; i++) {
+		status = acpi_ec_query(ec, &value);
+		if (status || !value)
+			break;
+	}
+	if (unlikely(i == ACPI_EC_CLEAR_MAX))
+		pr_warn("Warning: Maximum of %d stale EC events cleared\n", i);
+	else
+		pr_info("%d stale EC events cleared\n", i);
+}
+
+static void acpi_ec_enable_event(struct acpi_ec *ec)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&ec->lock, flags);
+	if (acpi_ec_started(ec))
+		__acpi_ec_enable_event(ec);
+	spin_unlock_irqrestore(&ec->lock, flags);
+
+	/* Drain additional events if hardware requires that */
+	if (EC_FLAGS_CLEAR_ON_RESUME)
+		acpi_ec_clear(ec);
+}
+
+static bool acpi_ec_query_flushed(struct acpi_ec *ec)
+{
+	bool flushed;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ec->lock, flags);
+	flushed = !ec->nr_pending_queries;
+	spin_unlock_irqrestore(&ec->lock, flags);
+	return flushed;
+}
+
+static void __acpi_ec_flush_event(struct acpi_ec *ec)
+{
+	/*
+	 * When ec_freeze_events is true, we need to flush events in
+	 * the proper position before entering the noirq stage.
+	 */
+	wait_event(ec->wait, acpi_ec_query_flushed(ec));
+	if (ec_query_wq)
+		flush_workqueue(ec_query_wq);
+}
+
+static void acpi_ec_disable_event(struct acpi_ec *ec)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&ec->lock, flags);
+	__acpi_ec_disable_event(ec);
+	spin_unlock_irqrestore(&ec->lock, flags);
+	__acpi_ec_flush_event(ec);
+}
+
 static bool acpi_ec_guard_event(struct acpi_ec *ec)
 {
 	bool guarded = true;
@@ -832,27 +944,6 @@ acpi_handle ec_get_handle(void)
 }
 EXPORT_SYMBOL(ec_get_handle);
 
-/*
- * Process _Q events that might have accumulated in the EC.
- * Run with locked ec mutex.
- */
-static void acpi_ec_clear(struct acpi_ec *ec)
-{
-	int i, status;
-	u8 value = 0;
-
-	for (i = 0; i < ACPI_EC_CLEAR_MAX; i++) {
-		status = acpi_ec_query(ec, &value);
-		if (status || !value)
-			break;
-	}
-
-	if (unlikely(i == ACPI_EC_CLEAR_MAX))
-		pr_warn("Warning: Maximum of %d stale EC events cleared\n", i);
-	else
-		pr_info("%d stale EC events cleared\n", i);
-}
-
 static void acpi_ec_start(struct acpi_ec *ec, bool resuming)
 {
 	unsigned long flags;
@@ -896,7 +987,8 @@ static void acpi_ec_stop(struct acpi_ec *ec, bool suspending)
 		if (!suspending) {
 			acpi_ec_complete_request(ec);
 			ec_dbg_ref(ec, "Decrease driver");
-		}
+		} else if (!ec_freeze_events)
+			__acpi_ec_disable_event(ec);
 		clear_bit(EC_FLAGS_STARTED, &ec->flags);
 		clear_bit(EC_FLAGS_STOPPED, &ec->flags);
 		ec_log_drv("EC stopped");
@@ -919,20 +1011,6 @@ void acpi_ec_block_transactions(void)
 
 void acpi_ec_unblock_transactions(void)
 {
-	struct acpi_ec *ec = first_ec;
-
-	if (!ec)
-		return;
-
-	/* Allow transactions to be carried out again */
-	acpi_ec_start(ec, true);
-
-	if (EC_FLAGS_CLEAR_ON_RESUME)
-		acpi_ec_clear(ec);
-}
-
-void acpi_ec_unblock_transactions_early(void)
-{
 	/*
 	 * Allow transactions to happen again (this function is called from
 	 * atomic context during wakeup, so we don't need to acquire the mutex).
@@ -1228,13 +1306,21 @@ acpi_ec_space_handler(u32 function, acpi_physical_address address,
 static acpi_status
 ec_parse_io_ports(struct acpi_resource *resource, void *context);
 
-static struct acpi_ec *make_acpi_ec(void)
+static void acpi_ec_free(struct acpi_ec *ec)
+{
+	if (first_ec == ec)
+		first_ec = NULL;
+	if (boot_ec == ec)
+		boot_ec = NULL;
+	kfree(ec);
+}
+
+static struct acpi_ec *acpi_ec_alloc(void)
 {
 	struct acpi_ec *ec = kzalloc(sizeof(struct acpi_ec), GFP_KERNEL);
 
 	if (!ec)
 		return NULL;
-	ec->flags = 1 << EC_FLAGS_QUERY_PENDING;
 	mutex_init(&ec->mutex);
 	init_waitqueue_head(&ec->wait);
 	INIT_LIST_HEAD(&ec->list);
@@ -1290,7 +1376,12 @@ ec_parse_device(acpi_handle handle, u32 Level, void *context, void **retval)
 	return AE_CTRL_TERMINATE;
 }
 
-static int ec_install_handlers(struct acpi_ec *ec)
+/*
+ * Note: This function returns an error code only when the address space
+ *       handler is not installed, which means "not able to handle
+ *       transactions".
+ */
+static int ec_install_handlers(struct acpi_ec *ec, bool handle_events)
 {
 	acpi_status status;
 
@@ -1319,6 +1410,16 @@ static int ec_install_handlers(struct acpi_ec *ec)
 		set_bit(EC_FLAGS_EC_HANDLER_INSTALLED, &ec->flags);
 	}
 
+	if (!handle_events)
+		return 0;
+
+	if (!test_bit(EC_FLAGS_EVT_HANDLER_INSTALLED, &ec->flags)) {
+		/* Find and register all query methods */
+		acpi_walk_namespace(ACPI_TYPE_METHOD, ec->handle, 1,
+				    acpi_ec_register_query_methods,
+				    NULL, ec, NULL);
+		set_bit(EC_FLAGS_EVT_HANDLER_INSTALLED, &ec->flags);
+	}
 	if (!test_bit(EC_FLAGS_GPE_HANDLER_INSTALLED, &ec->flags)) {
 		status = acpi_install_gpe_raw_handler(NULL, ec->gpe,
 					  ACPI_GPE_EDGE_TRIGGERED,
@@ -1329,6 +1430,9 @@ static int ec_install_handlers(struct acpi_ec *ec)
 			if (test_bit(EC_FLAGS_STARTED, &ec->flags) &&
 			    ec->reference_count >= 1)
 				acpi_ec_enable_gpe(ec, true);
+
+			/* EC is fully operational, allow queries */
+			acpi_ec_enable_event(ec);
 		}
 	}
 
@@ -1363,23 +1467,104 @@ static void ec_remove_handlers(struct acpi_ec *ec)
 			pr_err("failed to remove gpe handler\n");
 		clear_bit(EC_FLAGS_GPE_HANDLER_INSTALLED, &ec->flags);
 	}
+	if (test_bit(EC_FLAGS_EVT_HANDLER_INSTALLED, &ec->flags)) {
+		acpi_ec_remove_query_handlers(ec, true, 0);
+		clear_bit(EC_FLAGS_EVT_HANDLER_INSTALLED, &ec->flags);
+	}
 }
 
-static struct acpi_ec *acpi_ec_alloc(void)
+static int acpi_ec_setup(struct acpi_ec *ec, bool handle_events)
 {
-	struct acpi_ec *ec;
+	int ret;
 
-	/* Check for boot EC */
-	if (boot_ec) {
-		ec = boot_ec;
-		boot_ec = NULL;
-		ec_remove_handlers(ec);
-		if (first_ec == ec)
-			first_ec = NULL;
-	} else {
-		ec = make_acpi_ec();
+	ret = ec_install_handlers(ec, handle_events);
+	if (ret)
+		return ret;
+
+	/* First EC capable of handling transactions */
+	if (!first_ec) {
+		first_ec = ec;
+		acpi_handle_info(first_ec->handle, "Used as first EC\n");
 	}
-	return ec;
+
+	acpi_handle_info(ec->handle,
+			 "GPE=0x%lx, EC_CMD/EC_SC=0x%lx, EC_DATA=0x%lx\n",
+			 ec->gpe, ec->command_addr, ec->data_addr);
+	return ret;
+}
+
+static int acpi_config_boot_ec(struct acpi_ec *ec, acpi_handle handle,
+			       bool handle_events, bool is_ecdt)
+{
+	int ret;
+
+	/*
+	 * Changing the ACPI handle results in a re-configuration of the
+	 * boot EC. And if it happens after the namespace initialization,
+	 * it causes _REG evaluations.
+	 */
+	if (boot_ec && boot_ec->handle != handle)
+		ec_remove_handlers(boot_ec);
+
+	/* Unset old boot EC */
+	if (boot_ec != ec)
+		acpi_ec_free(boot_ec);
+
+	/*
+	 * ECDT device creation is split into acpi_ec_ecdt_probe() and
+	 * acpi_ec_ecdt_start(). This function takes care of completing the
+	 * ECDT parsing logic as the handle update should be performed
+	 * between the installation/uninstallation of the handlers.
+	 */
+	if (ec->handle != handle)
+		ec->handle = handle;
+
+	ret = acpi_ec_setup(ec, handle_events);
+	if (ret)
+		return ret;
+
+	/* Set new boot EC */
+	if (!boot_ec) {
+		boot_ec = ec;
+		boot_ec_is_ecdt = is_ecdt;
+	}
+
+	acpi_handle_info(boot_ec->handle,
+			 "Used as boot %s EC to handle transactions%s\n",
+			 is_ecdt ? "ECDT" : "DSDT",
+			 handle_events ? " and events" : "");
+	return ret;
+}
+
+static bool acpi_ec_ecdt_get_handle(acpi_handle *phandle)
+{
+	struct acpi_table_ecdt *ecdt_ptr;
+	acpi_status status;
+	acpi_handle handle;
+
+	status = acpi_get_table(ACPI_SIG_ECDT, 1,
+				(struct acpi_table_header **)&ecdt_ptr);
+	if (ACPI_FAILURE(status))
+		return false;
+
+	status = acpi_get_handle(NULL, ecdt_ptr->id, &handle);
+	if (ACPI_FAILURE(status))
+		return false;
+
+	*phandle = handle;
+	return true;
+}
+
+static bool acpi_is_boot_ec(struct acpi_ec *ec)
+{
+	if (!boot_ec)
+		return false;
+	if (ec->handle == boot_ec->handle &&
+	    ec->gpe == boot_ec->gpe &&
+	    ec->command_addr == boot_ec->command_addr &&
+	    ec->data_addr == boot_ec->data_addr)
+		return true;
+	return false;
 }
 
 static int acpi_ec_add(struct acpi_device *device)
@@ -1395,16 +1580,21 @@ static int acpi_ec_add(struct acpi_device *device)
 		return -ENOMEM;
 	if (ec_parse_device(device->handle, 0, ec, NULL) !=
 		AE_CTRL_TERMINATE) {
-			kfree(ec);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto err_alloc;
 	}
 
-	/* Find and register all query methods */
-	acpi_walk_namespace(ACPI_TYPE_METHOD, ec->handle, 1,
-			    acpi_ec_register_query_methods, NULL, ec, NULL);
+	if (acpi_is_boot_ec(ec)) {
+		boot_ec_is_ecdt = false;
+		acpi_handle_debug(ec->handle, "duplicated.\n");
+		acpi_ec_free(ec);
+		ec = boot_ec;
+		ret = acpi_config_boot_ec(ec, ec->handle, true, false);
+	} else
+		ret = acpi_ec_setup(ec, true);
+	if (ret)
+		goto err_query;
 
-	if (!first_ec)
-		first_ec = ec;
 	device->driver_data = ec;
 
 	ret = !!request_region(ec->data_addr, 1, "EC data");
@@ -1412,20 +1602,17 @@ static int acpi_ec_add(struct acpi_device *device)
 	ret = !!request_region(ec->command_addr, 1, "EC cmd");
 	WARN(!ret, "Could not request EC cmd io port 0x%lx", ec->command_addr);
 
-	pr_info("GPE = 0x%lx, I/O: command/status = 0x%lx, data = 0x%lx\n",
-			  ec->gpe, ec->command_addr, ec->data_addr);
-
-	ret = ec_install_handlers(ec);
-
 	/* Reprobe devices depending on the EC */
 	acpi_walk_dep_device_list(ec->handle);
+	acpi_handle_debug(ec->handle, "enumerated.\n");
+	return 0;
 
-	/* EC is fully operational, allow queries */
-	clear_bit(EC_FLAGS_QUERY_PENDING, &ec->flags);
-
-	/* Clear stale _Q events if hardware might require that */
-	if (EC_FLAGS_CLEAR_ON_RESUME)
-		acpi_ec_clear(ec);
+err_query:
+	if (ec != boot_ec)
+		acpi_ec_remove_query_handlers(ec, true, 0);
+err_alloc:
+	if (ec != boot_ec)
+		acpi_ec_free(ec);
 	return ret;
 }
 
@@ -1437,14 +1624,13 @@ static int acpi_ec_remove(struct acpi_device *device)
 		return -EINVAL;
 
 	ec = acpi_driver_data(device);
-	ec_remove_handlers(ec);
-	acpi_ec_remove_query_handlers(ec, true, 0);
 	release_region(ec->data_addr, 1);
 	release_region(ec->command_addr, 1);
 	device->driver_data = NULL;
-	if (ec == first_ec)
-		first_ec = NULL;
-	kfree(ec);
+	if (ec != boot_ec) {
+		ec_remove_handlers(ec);
+		acpi_ec_free(ec);
+	}
 	return 0;
 }
 
@@ -1486,9 +1672,8 @@ int __init acpi_ec_dsdt_probe(void)
 	if (!ec)
 		return -ENOMEM;
 	/*
-	 * Finding EC from DSDT if there is no ECDT EC available. When this
-	 * function is invoked, ACPI tables have been fully loaded, we can
-	 * walk namespace now.
+	 * At this point, the namespace is initialized, so start to find
+	 * the namespace objects.
 	 */
 	status = acpi_get_devices(ec_device_ids[0].id,
 				  ec_parse_device, ec, NULL);
@@ -1496,16 +1681,47 @@ int __init acpi_ec_dsdt_probe(void)
 		ret = -ENODEV;
 		goto error;
 	}
-	ret = ec_install_handlers(ec);
-
+	/*
+	 * When the DSDT EC is available, always re-configure boot EC to
+	 * have _REG evaluated. _REG can only be evaluated after the
+	 * namespace initialization.
+	 * At this point, the GPE is not fully initialized, so do not to
+	 * handle the events.
+	 */
+	ret = acpi_config_boot_ec(ec, ec->handle, false, false);
 error:
 	if (ret)
-		kfree(ec);
-	else
-		first_ec = boot_ec = ec;
+		acpi_ec_free(ec);
 	return ret;
 }
 
+/*
+ * If the DSDT EC is not functioning, we still need to prepare a fully
+ * functioning ECDT EC first in order to handle the events.
+ * https://bugzilla.kernel.org/show_bug.cgi?id=115021
+ */
+int __init acpi_ec_ecdt_start(void)
+{
+	acpi_handle handle;
+
+	if (!boot_ec)
+		return -ENODEV;
+	/*
+	 * The DSDT EC should have already been started in
+	 * acpi_ec_add().
+	 */
+	if (!boot_ec_is_ecdt)
+		return -ENODEV;
+
+	/*
+	 * At this point, the namespace and the GPE is initialized, so
+	 * start to find the namespace objects and handle the events.
+	 */
+	if (!acpi_ec_ecdt_get_handle(&handle))
+		return -ENODEV;
+	return acpi_config_boot_ec(boot_ec, handle, true, true);
+}
+
 #if 0
 /*
  * Some EC firmware variations refuses to respond QR_EC when SCI_EVT is not
@@ -1600,7 +1816,6 @@ int __init acpi_ec_ecdt_probe(void)
 		goto error;
 	}
 
-	pr_info("EC description table is found, configuring boot EC\n");
 	if (EC_FLAGS_CORRECT_ECDT) {
 		ec->command_addr = ecdt_ptr->data.address;
 		ec->data_addr = ecdt_ptr->control.address;
@@ -1609,16 +1824,90 @@ int __init acpi_ec_ecdt_probe(void)
 		ec->data_addr = ecdt_ptr->data.address;
 	}
 	ec->gpe = ecdt_ptr->gpe;
-	ec->handle = ACPI_ROOT_OBJECT;
-	ret = ec_install_handlers(ec);
+
+	/*
+	 * At this point, the namespace is not initialized, so do not find
+	 * the namespace objects, or handle the events.
+	 */
+	ret = acpi_config_boot_ec(ec, ACPI_ROOT_OBJECT, false, true);
 error:
 	if (ret)
-		kfree(ec);
-	else
-		first_ec = boot_ec = ec;
+		acpi_ec_free(ec);
 	return ret;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static void acpi_ec_enter_noirq(struct acpi_ec *ec)
+{
+	unsigned long flags;
+
+	if (ec == first_ec) {
+		spin_lock_irqsave(&ec->lock, flags);
+		ec->saved_busy_polling = ec_busy_polling;
+		ec->saved_polling_guard = ec_polling_guard;
+		ec_busy_polling = true;
+		ec_polling_guard = 0;
+		ec_log_drv("interrupt blocked");
+		spin_unlock_irqrestore(&ec->lock, flags);
+	}
+}
+
+static void acpi_ec_leave_noirq(struct acpi_ec *ec)
+{
+	unsigned long flags;
+
+	if (ec == first_ec) {
+		spin_lock_irqsave(&ec->lock, flags);
+		ec_busy_polling = ec->saved_busy_polling;
+		ec_polling_guard = ec->saved_polling_guard;
+		ec_log_drv("interrupt unblocked");
+		spin_unlock_irqrestore(&ec->lock, flags);
+	}
+}
+
+static int acpi_ec_suspend_noirq(struct device *dev)
+{
+	struct acpi_ec *ec =
+		acpi_driver_data(to_acpi_device(dev));
+
+	acpi_ec_enter_noirq(ec);
+	return 0;
+}
+
+static int acpi_ec_resume_noirq(struct device *dev)
+{
+	struct acpi_ec *ec =
+		acpi_driver_data(to_acpi_device(dev));
+
+	acpi_ec_leave_noirq(ec);
+	return 0;
+}
+
+static int acpi_ec_suspend(struct device *dev)
+{
+	struct acpi_ec *ec =
+		acpi_driver_data(to_acpi_device(dev));
+
+	if (ec_freeze_events)
+		acpi_ec_disable_event(ec);
+	return 0;
+}
+
+static int acpi_ec_resume(struct device *dev)
+{
+	struct acpi_ec *ec =
+		acpi_driver_data(to_acpi_device(dev));
+
+	acpi_ec_enable_event(ec);
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops acpi_ec_pm = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(acpi_ec_suspend_noirq, acpi_ec_resume_noirq)
+	SET_SYSTEM_SLEEP_PM_OPS(acpi_ec_suspend, acpi_ec_resume)
+};
+
 static int param_set_event_clearing(const char *val, struct kernel_param *kp)
 {
 	int result = 0;
@@ -1664,6 +1953,7 @@ static struct acpi_driver acpi_ec_driver = {
 		.add = acpi_ec_add,
 		.remove = acpi_ec_remove,
 		},
+	.drv.pm = &acpi_ec_pm,
 };
 
 static inline int acpi_ec_query_init(void)
diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
index 940218ff0193..1b41a2739dac 100644
--- a/drivers/acpi/internal.h
+++ b/drivers/acpi/internal.h
@@ -40,10 +40,8 @@ int acpi_sysfs_init(void);
 void acpi_container_init(void);
 void acpi_memory_hotplug_init(void);
 #ifdef	CONFIG_ACPI_HOTPLUG_IOAPIC
-int acpi_ioapic_add(struct acpi_pci_root *root);
 int acpi_ioapic_remove(struct acpi_pci_root *root);
 #else
-static inline int acpi_ioapic_add(struct acpi_pci_root *root) { return 0; }
 static inline int acpi_ioapic_remove(struct acpi_pci_root *root) { return 0; }
 #endif
 #ifdef CONFIG_ACPI_DOCK
@@ -116,7 +114,6 @@ bool acpi_device_is_present(struct acpi_device *adev);
 bool acpi_device_is_battery(struct acpi_device *adev);
 bool acpi_device_is_first_physical_node(struct acpi_device *adev,
 					const struct device *dev);
-struct device *acpi_get_first_physical_node(struct acpi_device *adev);
 
 /* --------------------------------------------------------------------------
                      Device Matching and Notification
@@ -174,6 +171,8 @@ struct acpi_ec {
 	struct work_struct work;
 	unsigned long timestamp;
 	unsigned long nr_pending_queries;
+	bool saved_busy_polling;
+	unsigned int saved_polling_guard;
 };
 
 extern struct acpi_ec *first_ec;
@@ -185,9 +184,9 @@ typedef int (*acpi_ec_query_func) (void *data);
 int acpi_ec_init(void);
 int acpi_ec_ecdt_probe(void);
 int acpi_ec_dsdt_probe(void);
+int acpi_ec_ecdt_start(void);
 void acpi_ec_block_transactions(void);
 void acpi_ec_unblock_transactions(void);
-void acpi_ec_unblock_transactions_early(void);
 int acpi_ec_add_query_handler(struct acpi_ec *ec, u8 query_bit,
 			      acpi_handle handle, acpi_ec_query_func func,
 			      void *data);
@@ -225,4 +224,14 @@ static inline void suspend_nvs_restore(void) {}
 void acpi_init_properties(struct acpi_device *adev);
 void acpi_free_properties(struct acpi_device *adev);
 
+/*--------------------------------------------------------------------------
+				Watchdog
+  -------------------------------------------------------------------------- */
+
+#ifdef CONFIG_ACPI_WATCHDOG
+void acpi_watchdog_init(void);
+#else
+static inline void acpi_watchdog_init(void) {}
+#endif
+
 #endif /* _ACPI_INTERNAL_H_ */
diff --git a/drivers/acpi/ioapic.c b/drivers/acpi/ioapic.c
index ccdc8db16bb8..6d7ce6e12aaa 100644
--- a/drivers/acpi/ioapic.c
+++ b/drivers/acpi/ioapic.c
@@ -46,7 +46,7 @@ static acpi_status setup_res(struct acpi_resource *acpi_res, void *data)
 	struct resource_win win;
 
 	res->flags = 0;
-	if (acpi_dev_filter_resource_type(acpi_res, IORESOURCE_MEM) == 0)
+	if (acpi_dev_filter_resource_type(acpi_res, IORESOURCE_MEM))
 		return AE_OK;
 
 	if (!acpi_dev_resource_memory(acpi_res, res)) {
@@ -97,7 +97,7 @@ static acpi_status handle_ioapic_add(acpi_handle handle, u32 lvl,
 	unsigned long long gsi_base;
 	struct acpi_pci_ioapic *ioapic;
 	struct pci_dev *dev = NULL;
-	struct resource *res = NULL;
+	struct resource *res = NULL, *pci_res = NULL, *crs_res;
 	char *type = NULL;
 
 	if (!acpi_is_ioapic(handle, &type))
@@ -137,23 +137,30 @@ static acpi_status handle_ioapic_add(acpi_handle handle, u32 lvl,
 		pci_set_master(dev);
 		if (pci_request_region(dev, 0, type))
 			goto exit_disable;
-		res = &dev->resource[0];
+		pci_res = &dev->resource[0];
 		ioapic->pdev = dev;
 	} else {
 		pci_dev_put(dev);
 		dev = NULL;
+	}
 
-		res = &ioapic->res;
-		acpi_walk_resources(handle, METHOD_NAME__CRS, setup_res, res);
-		if (res->flags == 0) {
-			acpi_handle_warn(handle, "failed to get resource\n");
-			goto exit_free;
-		} else if (request_resource(&iomem_resource, res)) {
-			acpi_handle_warn(handle, "failed to insert resource\n");
-			goto exit_free;
-		}
+	crs_res = &ioapic->res;
+	acpi_walk_resources(handle, METHOD_NAME__CRS, setup_res, crs_res);
+	crs_res->name = type;
+	crs_res->flags |= IORESOURCE_BUSY;
+	if (crs_res->flags == 0) {
+		acpi_handle_warn(handle, "failed to get resource\n");
+		goto exit_release;
+	} else if (insert_resource(&iomem_resource, crs_res)) {
+		acpi_handle_warn(handle, "failed to insert resource\n");
+		goto exit_release;
 	}
 
+	/* try pci resource first, then "_CRS" resource */
+	res = pci_res;
+	if (!res || !res->flags)
+		res = crs_res;
+
 	if (acpi_register_ioapic(handle, res->start, (u32)gsi_base)) {
 		acpi_handle_warn(handle, "failed to register IOAPIC\n");
 		goto exit_release;
@@ -174,14 +181,13 @@ done:
 exit_release:
 	if (dev)
 		pci_release_region(dev, 0);
-	else
-		release_resource(res);
+	if (ioapic->res.flags && ioapic->res.parent)
+		release_resource(&ioapic->res);
 exit_disable:
 	if (dev)
 		pci_disable_device(dev);
 exit_put:
 	pci_dev_put(dev);
-exit_free:
 	kfree(ioapic);
 exit:
 	mutex_unlock(&ioapic_list_lock);
@@ -189,13 +195,13 @@ exit:
 	return AE_OK;
 }
 
-int acpi_ioapic_add(struct acpi_pci_root *root)
+int acpi_ioapic_add(acpi_handle root_handle)
 {
 	acpi_status status, retval = AE_OK;
 
-	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, root->device->handle,
+	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, root_handle,
 				     UINT_MAX, handle_ioapic_add, NULL,
-				     root->device->handle, (void **)&retval);
+				     root_handle, (void **)&retval);
 
 	return ACPI_SUCCESS(status) && ACPI_SUCCESS(retval) ? 0 : -ENODEV;
 }
@@ -217,9 +223,9 @@ int acpi_ioapic_remove(struct acpi_pci_root *root)
 			pci_release_region(ioapic->pdev, 0);
 			pci_disable_device(ioapic->pdev);
 			pci_dev_put(ioapic->pdev);
-		} else if (ioapic->res.flags && ioapic->res.parent) {
-			release_resource(&ioapic->res);
 		}
+		if (ioapic->res.flags && ioapic->res.parent)
+			release_resource(&ioapic->res);
 		list_del(&ioapic->list);
 		kfree(ioapic);
 	}
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index 8c234dd9b8bc..71a7d07c28c9 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -94,54 +94,50 @@ static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
 	return to_acpi_device(acpi_desc->dev);
 }
 
-static int xlat_status(void *buf, unsigned int cmd)
+static int xlat_status(void *buf, unsigned int cmd, u32 status)
 {
 	struct nd_cmd_clear_error *clear_err;
 	struct nd_cmd_ars_status *ars_status;
-	struct nd_cmd_ars_start *ars_start;
-	struct nd_cmd_ars_cap *ars_cap;
 	u16 flags;
 
 	switch (cmd) {
 	case ND_CMD_ARS_CAP:
-		ars_cap = buf;
-		if ((ars_cap->status & 0xffff) == NFIT_ARS_CAP_NONE)
+		if ((status & 0xffff) == NFIT_ARS_CAP_NONE)
 			return -ENOTTY;
 
 		/* Command failed */
-		if (ars_cap->status & 0xffff)
+		if (status & 0xffff)
 			return -EIO;
 
 		/* No supported scan types for this range */
 		flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE;
-		if ((ars_cap->status >> 16 & flags) == 0)
+		if ((status >> 16 & flags) == 0)
 			return -ENOTTY;
 		break;
 	case ND_CMD_ARS_START:
-		ars_start = buf;
 		/* ARS is in progress */
-		if ((ars_start->status & 0xffff) == NFIT_ARS_START_BUSY)
+		if ((status & 0xffff) == NFIT_ARS_START_BUSY)
 			return -EBUSY;
 
 		/* Command failed */
-		if (ars_start->status & 0xffff)
+		if (status & 0xffff)
 			return -EIO;
 		break;
 	case ND_CMD_ARS_STATUS:
 		ars_status = buf;
 		/* Command failed */
-		if (ars_status->status & 0xffff)
+		if (status & 0xffff)
 			return -EIO;
 		/* Check extended status (Upper two bytes) */
-		if (ars_status->status == NFIT_ARS_STATUS_DONE)
+		if (status == NFIT_ARS_STATUS_DONE)
 			return 0;
 
 		/* ARS is in progress */
-		if (ars_status->status == NFIT_ARS_STATUS_BUSY)
+		if (status == NFIT_ARS_STATUS_BUSY)
 			return -EBUSY;
 
 		/* No ARS performed for the current boot */
-		if (ars_status->status == NFIT_ARS_STATUS_NONE)
+		if (status == NFIT_ARS_STATUS_NONE)
 			return -EAGAIN;
 
 		/*
@@ -149,19 +145,19 @@ static int xlat_status(void *buf, unsigned int cmd)
 		 * agent wants the scan to stop.  If we didn't overflow
 		 * then just continue with the returned results.
 		 */
-		if (ars_status->status == NFIT_ARS_STATUS_INTR) {
+		if (status == NFIT_ARS_STATUS_INTR) {
 			if (ars_status->flags & NFIT_ARS_F_OVERFLOW)
 				return -ENOSPC;
 			return 0;
 		}
 
 		/* Unknown status */
-		if (ars_status->status >> 16)
+		if (status >> 16)
 			return -EIO;
 		break;
 	case ND_CMD_CLEAR_ERROR:
 		clear_err = buf;
-		if (clear_err->status & 0xffff)
+		if (status & 0xffff)
 			return -EIO;
 		if (!clear_err->cleared)
 			return -EIO;
@@ -172,6 +168,9 @@ static int xlat_status(void *buf, unsigned int cmd)
 		break;
 	}
 
+	/* all other non-zero status results in an error */
+	if (status)
+		return -EIO;
 	return 0;
 }
 
@@ -186,10 +185,10 @@ static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
 	struct nd_cmd_pkg *call_pkg = NULL;
 	const char *cmd_name, *dimm_name;
 	unsigned long cmd_mask, dsm_mask;
+	u32 offset, fw_status = 0;
 	acpi_handle handle;
 	unsigned int func;
 	const u8 *uuid;
-	u32 offset;
 	int rc, i;
 
 	func = cmd;
@@ -317,6 +316,15 @@ static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
 				out_obj->buffer.pointer + offset, out_size);
 		offset += out_size;
 	}
+
+	/*
+	 * Set fw_status for all the commands with a known format to be
+	 * later interpreted by xlat_status().
+	 */
+	if (i >= 1 && ((cmd >= ND_CMD_ARS_CAP && cmd <= ND_CMD_CLEAR_ERROR)
+			|| (cmd >= ND_CMD_SMART && cmd <= ND_CMD_VENDOR)))
+		fw_status = *(u32 *) out_obj->buffer.pointer;
+
 	if (offset + in_buf.buffer.length < buf_len) {
 		if (i >= 1) {
 			/*
@@ -325,7 +333,7 @@ static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
 			 */
 			rc = buf_len - offset - in_buf.buffer.length;
 			if (cmd_rc)
-				*cmd_rc = xlat_status(buf, cmd);
+				*cmd_rc = xlat_status(buf, cmd, fw_status);
 		} else {
 			dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
 					__func__, dimm_name, cmd_name, buf_len,
@@ -335,7 +343,7 @@ static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
 	} else {
 		rc = 0;
 		if (cmd_rc)
-			*cmd_rc = xlat_status(buf, cmd);
+			*cmd_rc = xlat_status(buf, cmd, fw_status);
 	}
 
  out:
@@ -878,6 +886,58 @@ static ssize_t revision_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(revision);
 
+static ssize_t hw_error_scrub_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
+	struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
+	struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
+
+	return sprintf(buf, "%d\n", acpi_desc->scrub_mode);
+}
+
+/*
+ * The 'hw_error_scrub' attribute can have the following values written to it:
+ * '0': Switch to the default mode where an exception will only insert
+ *      the address of the memory error into the poison and badblocks lists.
+ * '1': Enable a full scrub to happen if an exception for a memory error is
+ *      received.
+ */
+static ssize_t hw_error_scrub_store(struct device *dev,
+		struct device_attribute *attr, const char *buf, size_t size)
+{
+	struct nvdimm_bus_descriptor *nd_desc;
+	ssize_t rc;
+	long val;
+
+	rc = kstrtol(buf, 0, &val);
+	if (rc)
+		return rc;
+
+	device_lock(dev);
+	nd_desc = dev_get_drvdata(dev);
+	if (nd_desc) {
+		struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
+
+		switch (val) {
+		case HW_ERROR_SCRUB_ON:
+			acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON;
+			break;
+		case HW_ERROR_SCRUB_OFF:
+			acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF;
+			break;
+		default:
+			rc = -EINVAL;
+			break;
+		}
+	}
+	device_unlock(dev);
+	if (rc)
+		return rc;
+	return size;
+}
+static DEVICE_ATTR_RW(hw_error_scrub);
+
 /*
  * This shows the number of full Address Range Scrubs that have been
  * completed since driver load time. Userspace can wait on this using
@@ -950,6 +1010,7 @@ static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n)
 static struct attribute *acpi_nfit_attributes[] = {
 	&dev_attr_revision.attr,
 	&dev_attr_scrub.attr,
+	&dev_attr_hw_error_scrub.attr,
 	NULL,
 };
 
@@ -1248,6 +1309,44 @@ static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
 	return NULL;
 }
 
+void __acpi_nvdimm_notify(struct device *dev, u32 event)
+{
+	struct nfit_mem *nfit_mem;
+	struct acpi_nfit_desc *acpi_desc;
+
+	dev_dbg(dev->parent, "%s: %s: event: %d\n", dev_name(dev), __func__,
+			event);
+
+	if (event != NFIT_NOTIFY_DIMM_HEALTH) {
+		dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev),
+				event);
+		return;
+	}
+
+	acpi_desc = dev_get_drvdata(dev->parent);
+	if (!acpi_desc)
+		return;
+
+	/*
+	 * If we successfully retrieved acpi_desc, then we know nfit_mem data
+	 * is still valid.
+	 */
+	nfit_mem = dev_get_drvdata(dev);
+	if (nfit_mem && nfit_mem->flags_attr)
+		sysfs_notify_dirent(nfit_mem->flags_attr);
+}
+EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify);
+
+static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data)
+{
+	struct acpi_device *adev = data;
+	struct device *dev = &adev->dev;
+
+	device_lock(dev->parent);
+	__acpi_nvdimm_notify(dev, event);
+	device_unlock(dev->parent);
+}
+
 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
 		struct nfit_mem *nfit_mem, u32 device_handle)
 {
@@ -1272,6 +1371,13 @@ static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
 		return force_enable_dimms ? 0 : -ENODEV;
 	}
 
+	if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle,
+		ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) {
+		dev_err(dev, "%s: notification registration failed\n",
+				dev_name(&adev_dimm->dev));
+		return -ENXIO;
+	}
+
 	/*
 	 * Until standardization materializes we need to consider 4
 	 * different command sets.  Note, that checking for function0 (bit0)
@@ -1310,18 +1416,41 @@ static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
 	return 0;
 }
 
+static void shutdown_dimm_notify(void *data)
+{
+	struct acpi_nfit_desc *acpi_desc = data;
+	struct nfit_mem *nfit_mem;
+
+	mutex_lock(&acpi_desc->init_mutex);
+	/*
+	 * Clear out the nfit_mem->flags_attr and shut down dimm event
+	 * notifications.
+	 */
+	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
+		struct acpi_device *adev_dimm = nfit_mem->adev;
+
+		if (nfit_mem->flags_attr) {
+			sysfs_put(nfit_mem->flags_attr);
+			nfit_mem->flags_attr = NULL;
+		}
+		if (adev_dimm)
+			acpi_remove_notify_handler(adev_dimm->handle,
+					ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify);
+	}
+	mutex_unlock(&acpi_desc->init_mutex);
+}
+
 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
 {
 	struct nfit_mem *nfit_mem;
-	int dimm_count = 0;
+	int dimm_count = 0, rc;
+	struct nvdimm *nvdimm;
 
 	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
 		struct acpi_nfit_flush_address *flush;
 		unsigned long flags = 0, cmd_mask;
-		struct nvdimm *nvdimm;
 		u32 device_handle;
 		u16 mem_flags;
-		int rc;
 
 		device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 		nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
@@ -1374,7 +1503,30 @@ static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
 
 	}
 
-	return nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
+	rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
+	if (rc)
+		return rc;
+
+	/*
+	 * Now that dimms are successfully registered, and async registration
+	 * is flushed, attempt to enable event notification.
+	 */
+	list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
+		struct kernfs_node *nfit_kernfs;
+
+		nvdimm = nfit_mem->nvdimm;
+		nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit");
+		if (nfit_kernfs)
+			nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs,
+					"flags");
+		sysfs_put(nfit_kernfs);
+		if (!nfit_mem->flags_attr)
+			dev_warn(acpi_desc->dev, "%s: notifications disabled\n",
+					nvdimm_name(nvdimm));
+	}
+
+	return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify,
+			acpi_desc);
 }
 
 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
@@ -1483,9 +1635,9 @@ static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
 	if (!info)
 		return -ENOMEM;
 	for (i = 0; i < nr; i++) {
-		struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
+		struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
 		struct nfit_set_info_map *map = &info->mapping[i];
-		struct nvdimm *nvdimm = nd_mapping->nvdimm;
+		struct nvdimm *nvdimm = mapping->nvdimm;
 		struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 		struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
 				spa->range_index, i);
@@ -1527,11 +1679,12 @@ static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
 {
 	struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
 	u64 offset = nfit_blk->stat_offset + mmio->size * bw;
+	const u32 STATUS_MASK = 0x80000037;
 
 	if (mmio->num_lines)
 		offset = to_interleave_offset(offset, mmio);
 
-	return readl(mmio->addr.base + offset);
+	return readl(mmio->addr.base + offset) & STATUS_MASK;
 }
 
 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
@@ -1908,7 +2061,7 @@ static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc,
 }
 
 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
-		struct nd_mapping *nd_mapping, struct nd_region_desc *ndr_desc,
+		struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc,
 		struct acpi_nfit_memory_map *memdev,
 		struct nfit_spa *nfit_spa)
 {
@@ -1925,12 +2078,12 @@ static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
 		return -ENODEV;
 	}
 
-	nd_mapping->nvdimm = nvdimm;
+	mapping->nvdimm = nvdimm;
 	switch (nfit_spa_type(spa)) {
 	case NFIT_SPA_PM:
 	case NFIT_SPA_VOLATILE:
-		nd_mapping->start = memdev->address;
-		nd_mapping->size = memdev->region_size;
+		mapping->start = memdev->address;
+		mapping->size = memdev->region_size;
 		break;
 	case NFIT_SPA_DCR:
 		nfit_mem = nvdimm_provider_data(nvdimm);
@@ -1938,13 +2091,13 @@ static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
 			dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
 					spa->range_index, nvdimm_name(nvdimm));
 		} else {
-			nd_mapping->size = nfit_mem->bdw->capacity;
-			nd_mapping->start = nfit_mem->bdw->start_address;
+			mapping->size = nfit_mem->bdw->capacity;
+			mapping->start = nfit_mem->bdw->start_address;
 			ndr_desc->num_lanes = nfit_mem->bdw->windows;
 			blk_valid = 1;
 		}
 
-		ndr_desc->nd_mapping = nd_mapping;
+		ndr_desc->mapping = mapping;
 		ndr_desc->num_mappings = blk_valid;
 		ndbr_desc = to_blk_region_desc(ndr_desc);
 		ndbr_desc->enable = acpi_nfit_blk_region_enable;
@@ -1970,7 +2123,7 @@ static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa)
 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
 		struct nfit_spa *nfit_spa)
 {
-	static struct nd_mapping nd_mappings[ND_MAX_MAPPINGS];
+	static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS];
 	struct acpi_nfit_system_address *spa = nfit_spa->spa;
 	struct nd_blk_region_desc ndbr_desc;
 	struct nd_region_desc *ndr_desc;
@@ -1989,7 +2142,7 @@ static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
 	}
 
 	memset(&res, 0, sizeof(res));
-	memset(&nd_mappings, 0, sizeof(nd_mappings));
+	memset(&mappings, 0, sizeof(mappings));
 	memset(&ndbr_desc, 0, sizeof(ndbr_desc));
 	res.start = spa->address;
 	res.end = res.start + spa->length - 1;
@@ -2005,7 +2158,7 @@ static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
 
 	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 		struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
-		struct nd_mapping *nd_mapping;
+		struct nd_mapping_desc *mapping;
 
 		if (memdev->range_index != spa->range_index)
 			continue;
@@ -2014,14 +2167,14 @@ static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
 					spa->range_index, ND_MAX_MAPPINGS);
 			return -ENXIO;
 		}
-		nd_mapping = &nd_mappings[count++];
-		rc = acpi_nfit_init_mapping(acpi_desc, nd_mapping, ndr_desc,
+		mapping = &mappings[count++];
+		rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc,
 				memdev, nfit_spa);
 		if (rc)
 			goto out;
 	}
 
-	ndr_desc->nd_mapping = nd_mappings;
+	ndr_desc->mapping = mappings;
 	ndr_desc->num_mappings = count;
 	rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
 	if (rc)
@@ -2669,29 +2822,30 @@ static int acpi_nfit_remove(struct acpi_device *adev)
 	return 0;
 }
 
-static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
+void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event)
 {
-	struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev);
+	struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
 	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
-	struct device *dev = &adev->dev;
 	union acpi_object *obj;
 	acpi_status status;
 	int ret;
 
 	dev_dbg(dev, "%s: event: %d\n", __func__, event);
 
-	device_lock(dev);
+	if (event != NFIT_NOTIFY_UPDATE)
+		return;
+
 	if (!dev->driver) {
 		/* dev->driver may be null if we're being removed */
 		dev_dbg(dev, "%s: no driver found for dev\n", __func__);
-		goto out_unlock;
+		return;
 	}
 
 	if (!acpi_desc) {
 		acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
 		if (!acpi_desc)
-			goto out_unlock;
-		acpi_nfit_desc_init(acpi_desc, &adev->dev);
+			return;
+		acpi_nfit_desc_init(acpi_desc, dev);
 	} else {
 		/*
 		 * Finish previous registration before considering new
@@ -2701,10 +2855,10 @@ static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
 	}
 
 	/* Evaluate _FIT */
-	status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
+	status = acpi_evaluate_object(handle, "_FIT", NULL, &buf);
 	if (ACPI_FAILURE(status)) {
 		dev_err(dev, "failed to evaluate _FIT\n");
-		goto out_unlock;
+		return;
 	}
 
 	obj = buf.pointer;
@@ -2716,9 +2870,14 @@ static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
 	} else
 		dev_err(dev, "Invalid _FIT\n");
 	kfree(buf.pointer);
+}
+EXPORT_SYMBOL_GPL(__acpi_nfit_notify);
 
- out_unlock:
-	device_unlock(dev);
+static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
+{
+	device_lock(&adev->dev);
+	__acpi_nfit_notify(&adev->dev, adev->handle, event);
+	device_unlock(&adev->dev);
 }
 
 static const struct acpi_device_id acpi_nfit_ids[] = {
diff --git a/drivers/acpi/nfit/mce.c b/drivers/acpi/nfit/mce.c
index 4c745bf389fe..e5ce81c38eed 100644
--- a/drivers/acpi/nfit/mce.c
+++ b/drivers/acpi/nfit/mce.c
@@ -14,6 +14,7 @@
  */
 #include <linux/notifier.h>
 #include <linux/acpi.h>
+#include <linux/nd.h>
 #include <asm/mce.h>
 #include "nfit.h"
 
@@ -42,7 +43,7 @@ static int nfit_handle_mce(struct notifier_block *nb, unsigned long val,
 		list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 			struct acpi_nfit_system_address *spa = nfit_spa->spa;
 
-			if (nfit_spa_type(spa) == NFIT_SPA_PM)
+			if (nfit_spa_type(spa) != NFIT_SPA_PM)
 				continue;
 			/* find the spa that covers the mce addr */
 			if (spa->address > mce->addr)
@@ -62,12 +63,25 @@ static int nfit_handle_mce(struct notifier_block *nb, unsigned long val,
 		}
 		mutex_unlock(&acpi_desc->init_mutex);
 
-		/*
-		 * We can ignore an -EBUSY here because if an ARS is already
-		 * in progress, just let that be the last authoritative one
-		 */
-		if (found_match)
+		if (!found_match)
+			continue;
+
+		/* If this fails due to an -ENOMEM, there is little we can do */
+		nvdimm_bus_add_poison(acpi_desc->nvdimm_bus,
+				ALIGN(mce->addr, L1_CACHE_BYTES),
+				L1_CACHE_BYTES);
+		nvdimm_region_notify(nfit_spa->nd_region,
+				NVDIMM_REVALIDATE_POISON);
+
+		if (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON) {
+			/*
+			 * We can ignore an -EBUSY here because if an ARS is
+			 * already in progress, just let that be the last
+			 * authoritative one
+			 */
 			acpi_nfit_ars_rescan(acpi_desc);
+		}
+		break;
 	}
 
 	mutex_unlock(&acpi_desc_lock);
diff --git a/drivers/acpi/nfit/nfit.h b/drivers/acpi/nfit/nfit.h
index e894ded24d99..14296f5267c8 100644
--- a/drivers/acpi/nfit/nfit.h
+++ b/drivers/acpi/nfit/nfit.h
@@ -78,6 +78,14 @@ enum {
 	NFIT_ARS_TIMEOUT = 90,
 };
 
+enum nfit_root_notifiers {
+	NFIT_NOTIFY_UPDATE = 0x80,
+};
+
+enum nfit_dimm_notifiers {
+	NFIT_NOTIFY_DIMM_HEALTH = 0x81,
+};
+
 struct nfit_spa {
 	struct list_head list;
 	struct nd_region *nd_region;
@@ -124,6 +132,7 @@ struct nfit_mem {
 	struct acpi_nfit_system_address *spa_bdw;
 	struct acpi_nfit_interleave *idt_dcr;
 	struct acpi_nfit_interleave *idt_bdw;
+	struct kernfs_node *flags_attr;
 	struct nfit_flush *nfit_flush;
 	struct list_head list;
 	struct acpi_device *adev;
@@ -152,6 +161,7 @@ struct acpi_nfit_desc {
 	struct list_head list;
 	struct kernfs_node *scrub_count_state;
 	unsigned int scrub_count;
+	unsigned int scrub_mode;
 	unsigned int cancel:1;
 	unsigned long dimm_cmd_force_en;
 	unsigned long bus_cmd_force_en;
@@ -159,6 +169,11 @@ struct acpi_nfit_desc {
 			void *iobuf, u64 len, int rw);
 };
 
+enum scrub_mode {
+	HW_ERROR_SCRUB_OFF,
+	HW_ERROR_SCRUB_ON,
+};
+
 enum nd_blk_mmio_selector {
 	BDW,
 	DCR,
@@ -223,5 +238,7 @@ static inline struct acpi_nfit_desc *to_acpi_desc(
 
 const u8 *to_nfit_uuid(enum nfit_uuids id);
 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *nfit, acpi_size sz);
+void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event);
+void __acpi_nvdimm_notify(struct device *dev, u32 event);
 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev);
 #endif /* __NFIT_H__ */
diff --git a/drivers/acpi/pci_irq.c b/drivers/acpi/pci_irq.c
index 2c45dd3acc17..c576a6fe4ebb 100644
--- a/drivers/acpi/pci_irq.c
+++ b/drivers/acpi/pci_irq.c
@@ -411,7 +411,15 @@ int acpi_pci_irq_enable(struct pci_dev *dev)
 	int gsi;
 	u8 pin;
 	int triggering = ACPI_LEVEL_SENSITIVE;
-	int polarity = ACPI_ACTIVE_LOW;
+	/*
+	 * On ARM systems with the GIC interrupt model, level interrupts
+	 * are always polarity high by specification; PCI legacy
+	 * IRQs lines are inverted before reaching the interrupt
+	 * controller and must therefore be considered active high
+	 * as default.
+	 */
+	int polarity = acpi_irq_model == ACPI_IRQ_MODEL_GIC ?
+				      ACPI_ACTIVE_HIGH : ACPI_ACTIVE_LOW;
 	char *link = NULL;
 	char link_desc[16];
 	int rc;
diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c
index d144168d4ef9..bf601d4df8cf 100644
--- a/drivers/acpi/pci_root.c
+++ b/drivers/acpi/pci_root.c
@@ -614,7 +614,17 @@ static int acpi_pci_root_add(struct acpi_device *device,
 	if (hotadd) {
 		pcibios_resource_survey_bus(root->bus);
 		pci_assign_unassigned_root_bus_resources(root->bus);
-		acpi_ioapic_add(root);
+		/*
+		 * This is only called for the hotadd case. For the boot-time
+		 * case, we need to wait until after PCI initialization in
+		 * order to deal with IOAPICs mapped in on a PCI BAR.
+		 *
+		 * This is currently x86-specific, because acpi_ioapic_add()
+		 * is an empty function without CONFIG_ACPI_HOTPLUG_IOAPIC.
+		 * And CONFIG_ACPI_HOTPLUG_IOAPIC depends on CONFIG_X86_IO_APIC
+		 * (see drivers/acpi/Kconfig).
+		 */
+		acpi_ioapic_add(root->device->handle);
 	}
 
 	pci_lock_rescan_remove();
diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c
index 9125d7d96372..5c78ee1860b0 100644
--- a/drivers/acpi/processor_core.c
+++ b/drivers/acpi/processor_core.c
@@ -32,12 +32,12 @@ static struct acpi_table_madt *get_madt_table(void)
 }
 
 static int map_lapic_id(struct acpi_subtable_header *entry,
-		 u32 acpi_id, phys_cpuid_t *apic_id)
+		 u32 acpi_id, phys_cpuid_t *apic_id, bool ignore_disabled)
 {
 	struct acpi_madt_local_apic *lapic =
 		container_of(entry, struct acpi_madt_local_apic, header);
 
-	if (!(lapic->lapic_flags & ACPI_MADT_ENABLED))
+	if (ignore_disabled && !(lapic->lapic_flags & ACPI_MADT_ENABLED))
 		return -ENODEV;
 
 	if (lapic->processor_id != acpi_id)
@@ -48,12 +48,13 @@ static int map_lapic_id(struct acpi_subtable_header *entry,
 }
 
 static int map_x2apic_id(struct acpi_subtable_header *entry,
-		int device_declaration, u32 acpi_id, phys_cpuid_t *apic_id)
+		int device_declaration, u32 acpi_id, phys_cpuid_t *apic_id,
+		bool ignore_disabled)
 {
 	struct acpi_madt_local_x2apic *apic =
 		container_of(entry, struct acpi_madt_local_x2apic, header);
 
-	if (!(apic->lapic_flags & ACPI_MADT_ENABLED))
+	if (ignore_disabled && !(apic->lapic_flags & ACPI_MADT_ENABLED))
 		return -ENODEV;
 
 	if (device_declaration && (apic->uid == acpi_id)) {
@@ -65,12 +66,13 @@ static int map_x2apic_id(struct acpi_subtable_header *entry,
 }
 
 static int map_lsapic_id(struct acpi_subtable_header *entry,
-		int device_declaration, u32 acpi_id, phys_cpuid_t *apic_id)
+		int device_declaration, u32 acpi_id, phys_cpuid_t *apic_id,
+		bool ignore_disabled)
 {
 	struct acpi_madt_local_sapic *lsapic =
 		container_of(entry, struct acpi_madt_local_sapic, header);
 
-	if (!(lsapic->lapic_flags & ACPI_MADT_ENABLED))
+	if (ignore_disabled && !(lsapic->lapic_flags & ACPI_MADT_ENABLED))
 		return -ENODEV;
 
 	if (device_declaration) {
@@ -87,12 +89,13 @@ static int map_lsapic_id(struct acpi_subtable_header *entry,
  * Retrieve the ARM CPU physical identifier (MPIDR)
  */
 static int map_gicc_mpidr(struct acpi_subtable_header *entry,
-		int device_declaration, u32 acpi_id, phys_cpuid_t *mpidr)
+		int device_declaration, u32 acpi_id, phys_cpuid_t *mpidr,
+		bool ignore_disabled)
 {
 	struct acpi_madt_generic_interrupt *gicc =
 	    container_of(entry, struct acpi_madt_generic_interrupt, header);
 
-	if (!(gicc->flags & ACPI_MADT_ENABLED))
+	if (ignore_disabled && !(gicc->flags & ACPI_MADT_ENABLED))
 		return -ENODEV;
 
 	/* device_declaration means Device object in DSDT, in the
@@ -109,7 +112,7 @@ static int map_gicc_mpidr(struct acpi_subtable_header *entry,
 }
 
 static phys_cpuid_t map_madt_entry(struct acpi_table_madt *madt,
-				   int type, u32 acpi_id)
+				   int type, u32 acpi_id, bool ignore_disabled)
 {
 	unsigned long madt_end, entry;
 	phys_cpuid_t phys_id = PHYS_CPUID_INVALID;	/* CPU hardware ID */
@@ -127,16 +130,20 @@ static phys_cpuid_t map_madt_entry(struct acpi_table_madt *madt,
 		struct acpi_subtable_header *header =
 			(struct acpi_subtable_header *)entry;
 		if (header->type == ACPI_MADT_TYPE_LOCAL_APIC) {
-			if (!map_lapic_id(header, acpi_id, &phys_id))
+			if (!map_lapic_id(header, acpi_id, &phys_id,
+					  ignore_disabled))
 				break;
 		} else if (header->type == ACPI_MADT_TYPE_LOCAL_X2APIC) {
-			if (!map_x2apic_id(header, type, acpi_id, &phys_id))
+			if (!map_x2apic_id(header, type, acpi_id, &phys_id,
+					   ignore_disabled))
 				break;
 		} else if (header->type == ACPI_MADT_TYPE_LOCAL_SAPIC) {
-			if (!map_lsapic_id(header, type, acpi_id, &phys_id))
+			if (!map_lsapic_id(header, type, acpi_id, &phys_id,
+					   ignore_disabled))
 				break;
 		} else if (header->type == ACPI_MADT_TYPE_GENERIC_INTERRUPT) {
-			if (!map_gicc_mpidr(header, type, acpi_id, &phys_id))
+			if (!map_gicc_mpidr(header, type, acpi_id, &phys_id,
+					    ignore_disabled))
 				break;
 		}
 		entry += header->length;
@@ -156,14 +163,15 @@ phys_cpuid_t __init acpi_map_madt_entry(u32 acpi_id)
 	if (!madt)
 		return PHYS_CPUID_INVALID;
 
-	rv = map_madt_entry(madt, 1, acpi_id);
+	rv = map_madt_entry(madt, 1, acpi_id, true);
 
 	early_acpi_os_unmap_memory(madt, tbl_size);
 
 	return rv;
 }
 
-static phys_cpuid_t map_mat_entry(acpi_handle handle, int type, u32 acpi_id)
+static phys_cpuid_t map_mat_entry(acpi_handle handle, int type, u32 acpi_id,
+				  bool ignore_disabled)
 {
 	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 	union acpi_object *obj;
@@ -184,30 +192,38 @@ static phys_cpuid_t map_mat_entry(acpi_handle handle, int type, u32 acpi_id)
 
 	header = (struct acpi_subtable_header *)obj->buffer.pointer;
 	if (header->type == ACPI_MADT_TYPE_LOCAL_APIC)
-		map_lapic_id(header, acpi_id, &phys_id);
+		map_lapic_id(header, acpi_id, &phys_id, ignore_disabled);
 	else if (header->type == ACPI_MADT_TYPE_LOCAL_SAPIC)
-		map_lsapic_id(header, type, acpi_id, &phys_id);
+		map_lsapic_id(header, type, acpi_id, &phys_id, ignore_disabled);
 	else if (header->type == ACPI_MADT_TYPE_LOCAL_X2APIC)
-		map_x2apic_id(header, type, acpi_id, &phys_id);
+		map_x2apic_id(header, type, acpi_id, &phys_id, ignore_disabled);
 	else if (header->type == ACPI_MADT_TYPE_GENERIC_INTERRUPT)
-		map_gicc_mpidr(header, type, acpi_id, &phys_id);
+		map_gicc_mpidr(header, type, acpi_id, &phys_id,
+			       ignore_disabled);
 
 exit:
 	kfree(buffer.pointer);
 	return phys_id;
 }
 
-phys_cpuid_t acpi_get_phys_id(acpi_handle handle, int type, u32 acpi_id)
+static phys_cpuid_t __acpi_get_phys_id(acpi_handle handle, int type,
+				       u32 acpi_id, bool ignore_disabled)
 {
 	phys_cpuid_t phys_id;
 
-	phys_id = map_mat_entry(handle, type, acpi_id);
+	phys_id = map_mat_entry(handle, type, acpi_id, ignore_disabled);
 	if (invalid_phys_cpuid(phys_id))
-		phys_id = map_madt_entry(get_madt_table(), type, acpi_id);
+		phys_id = map_madt_entry(get_madt_table(), type, acpi_id,
+					   ignore_disabled);
 
 	return phys_id;
 }
 
+phys_cpuid_t acpi_get_phys_id(acpi_handle handle, int type, u32 acpi_id)
+{
+	return __acpi_get_phys_id(handle, type, acpi_id, true);
+}
+
 int acpi_map_cpuid(phys_cpuid_t phys_id, u32 acpi_id)
 {
 #ifdef CONFIG_SMP
@@ -264,6 +280,79 @@ int acpi_get_cpuid(acpi_handle handle, int type, u32 acpi_id)
 }
 EXPORT_SYMBOL_GPL(acpi_get_cpuid);
 
+#ifdef CONFIG_ACPI_HOTPLUG_CPU
+static bool __init
+map_processor(acpi_handle handle, phys_cpuid_t *phys_id, int *cpuid)
+{
+	int type, id;
+	u32 acpi_id;
+	acpi_status status;
+	acpi_object_type acpi_type;
+	unsigned long long tmp;
+	union acpi_object object = { 0 };
+	struct acpi_buffer buffer = { sizeof(union acpi_object), &object };
+
+	status = acpi_get_type(handle, &acpi_type);
+	if (ACPI_FAILURE(status))
+		return false;
+
+	switch (acpi_type) {
+	case ACPI_TYPE_PROCESSOR:
+		status = acpi_evaluate_object(handle, NULL, NULL, &buffer);
+		if (ACPI_FAILURE(status))
+			return false;
+		acpi_id = object.processor.proc_id;
+
+		/* validate the acpi_id */
+		if(acpi_processor_validate_proc_id(acpi_id))
+			return false;
+		break;
+	case ACPI_TYPE_DEVICE:
+		status = acpi_evaluate_integer(handle, "_UID", NULL, &tmp);
+		if (ACPI_FAILURE(status))
+			return false;
+		acpi_id = tmp;
+		break;
+	default:
+		return false;
+	}
+
+	type = (acpi_type == ACPI_TYPE_DEVICE) ? 1 : 0;
+
+	*phys_id = __acpi_get_phys_id(handle, type, acpi_id, false);
+	id = acpi_map_cpuid(*phys_id, acpi_id);
+
+	if (id < 0)
+		return false;
+	*cpuid = id;
+	return true;
+}
+
+static acpi_status __init
+set_processor_node_mapping(acpi_handle handle, u32 lvl, void *context,
+			   void **rv)
+{
+	phys_cpuid_t phys_id;
+	int cpu_id;
+
+	if (!map_processor(handle, &phys_id, &cpu_id))
+		return AE_ERROR;
+
+	acpi_map_cpu2node(handle, cpu_id, phys_id);
+	return AE_OK;
+}
+
+void __init acpi_set_processor_mapping(void)
+{
+	/* Set persistent cpu <-> node mapping for all processors. */
+	acpi_walk_namespace(ACPI_TYPE_PROCESSOR, ACPI_ROOT_OBJECT,
+			    ACPI_UINT32_MAX, set_processor_node_mapping,
+			    NULL, NULL, NULL);
+}
+#else
+void __init acpi_set_processor_mapping(void) {}
+#endif /* CONFIG_ACPI_HOTPLUG_CPU */
+
 #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC
 static int get_ioapic_id(struct acpi_subtable_header *entry, u32 gsi_base,
 			 u64 *phys_addr, int *ioapic_id)
diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c
index 0553aeebb228..9d5f0c7ed3f7 100644
--- a/drivers/acpi/processor_driver.c
+++ b/drivers/acpi/processor_driver.c
@@ -110,55 +110,46 @@ static void acpi_processor_notify(acpi_handle handle, u32 event, void *data)
 
 static int __acpi_processor_start(struct acpi_device *device);
 
-static int acpi_cpu_soft_notify(struct notifier_block *nfb,
-					  unsigned long action, void *hcpu)
+static int acpi_soft_cpu_online(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
 	struct acpi_processor *pr = per_cpu(processors, cpu);
 	struct acpi_device *device;
-	action &= ~CPU_TASKS_FROZEN;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_DEAD:
-		break;
-	default:
-		return NOTIFY_DONE;
-	}
 
 	if (!pr || acpi_bus_get_device(pr->handle, &device))
-		return NOTIFY_DONE;
-
-	if (action == CPU_ONLINE) {
-		/*
-		 * CPU got physically hotplugged and onlined for the first time:
-		 * Initialize missing things.
-		 */
-		if (pr->flags.need_hotplug_init) {
-			int ret;
-
-			pr_info("Will online and init hotplugged CPU: %d\n",
-				pr->id);
-			pr->flags.need_hotplug_init = 0;
-			ret = __acpi_processor_start(device);
-			WARN(ret, "Failed to start CPU: %d\n", pr->id);
-		} else {
-			/* Normal CPU soft online event. */
-			acpi_processor_ppc_has_changed(pr, 0);
-			acpi_processor_hotplug(pr);
-			acpi_processor_reevaluate_tstate(pr, action);
-			acpi_processor_tstate_has_changed(pr);
-		}
-	} else if (action == CPU_DEAD) {
-		/* Invalidate flag.throttling after the CPU is offline. */
-		acpi_processor_reevaluate_tstate(pr, action);
+		return 0;
+	/*
+	 * CPU got physically hotplugged and onlined for the first time:
+	 * Initialize missing things.
+	 */
+	if (pr->flags.need_hotplug_init) {
+		int ret;
+
+		pr_info("Will online and init hotplugged CPU: %d\n",
+			pr->id);
+		pr->flags.need_hotplug_init = 0;
+		ret = __acpi_processor_start(device);
+		WARN(ret, "Failed to start CPU: %d\n", pr->id);
+	} else {
+		/* Normal CPU soft online event. */
+		acpi_processor_ppc_has_changed(pr, 0);
+		acpi_processor_hotplug(pr);
+		acpi_processor_reevaluate_tstate(pr, false);
+		acpi_processor_tstate_has_changed(pr);
 	}
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block acpi_cpu_notifier = {
-	    .notifier_call = acpi_cpu_soft_notify,
-};
+static int acpi_soft_cpu_dead(unsigned int cpu)
+{
+	struct acpi_processor *pr = per_cpu(processors, cpu);
+	struct acpi_device *device;
+
+	if (!pr || acpi_bus_get_device(pr->handle, &device))
+		return 0;
+
+	acpi_processor_reevaluate_tstate(pr, true);
+	return 0;
+}
 
 #ifdef CONFIG_ACPI_CPU_FREQ_PSS
 static int acpi_pss_perf_init(struct acpi_processor *pr,
@@ -245,8 +236,8 @@ static int __acpi_processor_start(struct acpi_device *device)
 		return 0;
 
 	result = acpi_cppc_processor_probe(pr);
-	if (result)
-		return -ENODEV;
+	if (result && !IS_ENABLED(CONFIG_ACPI_CPU_FREQ_PSS))
+		dev_warn(&device->dev, "CPPC data invalid or not present\n");
 
 	if (!cpuidle_get_driver() || cpuidle_get_driver() == &acpi_idle_driver)
 		acpi_processor_power_init(pr);
@@ -303,7 +294,7 @@ static int acpi_processor_stop(struct device *dev)
  * This is needed for the powernow-k8 driver, that works even without
  * ACPI, but needs symbols from this driver
  */
-
+static enum cpuhp_state hp_online;
 static int __init acpi_processor_driver_init(void)
 {
 	int result = 0;
@@ -315,11 +306,22 @@ static int __init acpi_processor_driver_init(void)
 	if (result < 0)
 		return result;
 
-	register_hotcpu_notifier(&acpi_cpu_notifier);
+	result = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					   "acpi/cpu-drv:online",
+					   acpi_soft_cpu_online, NULL);
+	if (result < 0)
+		goto err;
+	hp_online = result;
+	cpuhp_setup_state_nocalls(CPUHP_ACPI_CPUDRV_DEAD, "acpi/cpu-drv:dead",
+				  NULL, acpi_soft_cpu_dead);
+
 	acpi_thermal_cpufreq_init();
 	acpi_processor_ppc_init();
 	acpi_processor_throttling_init();
 	return 0;
+err:
+	driver_unregister(&acpi_processor_driver);
+	return result;
 }
 
 static void __exit acpi_processor_driver_exit(void)
@@ -329,7 +331,8 @@ static void __exit acpi_processor_driver_exit(void)
 
 	acpi_processor_ppc_exit();
 	acpi_thermal_cpufreq_exit();
-	unregister_hotcpu_notifier(&acpi_cpu_notifier);
+	cpuhp_remove_state_nocalls(hp_online);
+	cpuhp_remove_state_nocalls(CPUHP_ACPI_CPUDRV_DEAD);
 	driver_unregister(&acpi_processor_driver);
 }
 
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index cea52528aa18..2237d3f24f0e 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -31,6 +31,7 @@
 #include <linux/sched.h>       /* need_resched() */
 #include <linux/tick.h>
 #include <linux/cpuidle.h>
+#include <linux/cpu.h>
 #include <acpi/processor.h>
 
 /*
@@ -115,7 +116,7 @@ static const struct dmi_system_id processor_power_dmi_table[] = {
  * Callers should disable interrupts before the call and enable
  * interrupts after return.
  */
-static void acpi_safe_halt(void)
+static void __cpuidle acpi_safe_halt(void)
 {
 	if (!tif_need_resched()) {
 		safe_halt();
@@ -645,7 +646,7 @@ static int acpi_idle_bm_check(void)
  *
  * Caller disables interrupt before call and enables interrupt after return.
  */
-static void acpi_idle_do_entry(struct acpi_processor_cx *cx)
+static void __cpuidle acpi_idle_do_entry(struct acpi_processor_cx *cx)
 {
 	if (cx->entry_method == ACPI_CSTATE_FFH) {
 		/* Call into architectural FFH based C-state */
diff --git a/drivers/acpi/processor_throttling.c b/drivers/acpi/processor_throttling.c
index c72e64893d03..d51ca1c05619 100644
--- a/drivers/acpi/processor_throttling.c
+++ b/drivers/acpi/processor_throttling.c
@@ -375,11 +375,11 @@ int acpi_processor_tstate_has_changed(struct acpi_processor *pr)
  *	3. TSD domain
  */
 void acpi_processor_reevaluate_tstate(struct acpi_processor *pr,
-					unsigned long action)
+					bool is_dead)
 {
 	int result = 0;
 
-	if (action == CPU_DEAD) {
+	if (is_dead) {
 		/* When one CPU is offline, the T-state throttling
 		 * will be invalidated.
 		 */
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index ad9fc84a8601..035ac646d8db 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -2002,6 +2002,7 @@ int __init acpi_scan_init(void)
 	acpi_pnp_init();
 	acpi_int340x_thermal_init();
 	acpi_amba_init();
+	acpi_watchdog_init();
 
 	acpi_scan_add_handler(&generic_device_handler);
 
@@ -2044,6 +2045,7 @@ int __init acpi_scan_init(void)
 	}
 
 	acpi_update_all_gpes();
+	acpi_ec_ecdt_start();
 
 	acpi_scan_initialized = true;
 
@@ -2054,7 +2056,7 @@ int __init acpi_scan_init(void)
 
 static struct acpi_probe_entry *ape;
 static int acpi_probe_count;
-static DEFINE_SPINLOCK(acpi_probe_lock);
+static DEFINE_MUTEX(acpi_probe_mutex);
 
 static int __init acpi_match_madt(struct acpi_subtable_header *header,
 				  const unsigned long end)
@@ -2073,7 +2075,7 @@ int __init __acpi_probe_device_table(struct acpi_probe_entry *ap_head, int nr)
 	if (acpi_disabled)
 		return 0;
 
-	spin_lock(&acpi_probe_lock);
+	mutex_lock(&acpi_probe_mutex);
 	for (ape = ap_head; nr; ape++, nr--) {
 		if (ACPI_COMPARE_NAME(ACPI_SIG_MADT, ape->id)) {
 			acpi_probe_count = 0;
@@ -2086,7 +2088,7 @@ int __init __acpi_probe_device_table(struct acpi_probe_entry *ap_head, int nr)
 				count++;
 		}
 	}
-	spin_unlock(&acpi_probe_lock);
+	mutex_unlock(&acpi_probe_mutex);
 
 	return count;
 }
diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c
index 2b38c1bb0446..deb0ff78eba8 100644
--- a/drivers/acpi/sleep.c
+++ b/drivers/acpi/sleep.c
@@ -572,7 +572,7 @@ static int acpi_suspend_enter(suspend_state_t pm_state)
 
 		acpi_get_event_status(ACPI_EVENT_POWER_BUTTON, &pwr_btn_status);
 
-		if (pwr_btn_status & ACPI_EVENT_FLAG_SET) {
+		if (pwr_btn_status & ACPI_EVENT_FLAG_STATUS_SET) {
 			acpi_clear_event(ACPI_EVENT_POWER_BUTTON);
 			/* Flag for later */
 			pwr_btn_event_pending = true;
@@ -586,7 +586,7 @@ static int acpi_suspend_enter(suspend_state_t pm_state)
 	 */
 	acpi_disable_all_gpes();
 	/* Allow EC transactions to happen. */
-	acpi_ec_unblock_transactions_early();
+	acpi_ec_unblock_transactions();
 
 	suspend_nvs_restore();
 
@@ -784,7 +784,7 @@ static void acpi_hibernation_leave(void)
 	/* Restore the NVS memory area */
 	suspend_nvs_restore();
 	/* Allow EC transactions to happen. */
-	acpi_ec_unblock_transactions_early();
+	acpi_ec_unblock_transactions();
 }
 
 static void acpi_pm_thaw(void)
diff --git a/drivers/acpi/spcr.c b/drivers/acpi/spcr.c
new file mode 100644
index 000000000000..e8d7bc7d4da8
--- /dev/null
+++ b/drivers/acpi/spcr.c
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2012, Intel Corporation
+ * Copyright (c) 2015, Red Hat, Inc.
+ * Copyright (c) 2015, 2016 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#define pr_fmt(fmt) "ACPI: SPCR: " fmt
+
+#include <linux/acpi.h>
+#include <linux/console.h>
+#include <linux/kernel.h>
+#include <linux/serial_core.h>
+
+/**
+ * parse_spcr() - parse ACPI SPCR table and add preferred console
+ *
+ * @earlycon: set up earlycon for the console specified by the table
+ *
+ * For the architectures with support for ACPI, CONFIG_ACPI_SPCR_TABLE may be
+ * defined to parse ACPI SPCR table.  As a result of the parsing preferred
+ * console is registered and if @earlycon is true, earlycon is set up.
+ *
+ * When CONFIG_ACPI_SPCR_TABLE is defined, this function should be called
+ * from arch inintialization code as soon as the DT/ACPI decision is made.
+ *
+ */
+int __init parse_spcr(bool earlycon)
+{
+	static char opts[64];
+	struct acpi_table_spcr *table;
+	acpi_size table_size;
+	acpi_status status;
+	char *uart;
+	char *iotype;
+	int baud_rate;
+	int err;
+
+	if (acpi_disabled)
+		return -ENODEV;
+
+	status = acpi_get_table_with_size(ACPI_SIG_SPCR, 0,
+					  (struct acpi_table_header **)&table,
+					  &table_size);
+
+	if (ACPI_FAILURE(status))
+		return -ENOENT;
+
+	if (table->header.revision < 2) {
+		err = -ENOENT;
+		pr_err("wrong table version\n");
+		goto done;
+	}
+
+	iotype = table->serial_port.space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY ?
+			"mmio" : "io";
+
+	switch (table->interface_type) {
+	case ACPI_DBG2_ARM_SBSA_32BIT:
+		iotype = "mmio32";
+		/* fall through */
+	case ACPI_DBG2_ARM_PL011:
+	case ACPI_DBG2_ARM_SBSA_GENERIC:
+	case ACPI_DBG2_BCM2835:
+		uart = "pl011";
+		break;
+	case ACPI_DBG2_16550_COMPATIBLE:
+	case ACPI_DBG2_16550_SUBSET:
+		uart = "uart";
+		break;
+	default:
+		err = -ENOENT;
+		goto done;
+	}
+
+	switch (table->baud_rate) {
+	case 3:
+		baud_rate = 9600;
+		break;
+	case 4:
+		baud_rate = 19200;
+		break;
+	case 6:
+		baud_rate = 57600;
+		break;
+	case 7:
+		baud_rate = 115200;
+		break;
+	default:
+		err = -ENOENT;
+		goto done;
+	}
+
+	snprintf(opts, sizeof(opts), "%s,%s,0x%llx,%d", uart, iotype,
+		 table->serial_port.address, baud_rate);
+
+	pr_info("console: %s\n", opts);
+
+	if (earlycon)
+		setup_earlycon(opts);
+
+	err = add_preferred_console(uart, 0, opts + strlen(uart) + 1);
+
+done:
+	early_acpi_os_unmap_memory((void __iomem *)table, table_size);
+	return err;
+}
diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c
index 358165e9f5b8..703c26e7022c 100644
--- a/drivers/acpi/sysfs.c
+++ b/drivers/acpi/sysfs.c
@@ -314,10 +314,14 @@ static struct kobject *tables_kobj;
 static struct kobject *dynamic_tables_kobj;
 static struct kobject *hotplug_kobj;
 
+#define ACPI_MAX_TABLE_INSTANCES	999
+#define ACPI_INST_SIZE			4 /* including trailing 0 */
+
 struct acpi_table_attr {
 	struct bin_attribute attr;
-	char name[8];
+	char name[ACPI_NAME_SIZE];
 	int instance;
+	char filename[ACPI_NAME_SIZE+ACPI_INST_SIZE];
 	struct list_head node;
 };
 
@@ -329,14 +333,9 @@ static ssize_t acpi_table_show(struct file *filp, struct kobject *kobj,
 	    container_of(bin_attr, struct acpi_table_attr, attr);
 	struct acpi_table_header *table_header = NULL;
 	acpi_status status;
-	char name[ACPI_NAME_SIZE];
-
-	if (strncmp(table_attr->name, "NULL", 4))
-		memcpy(name, table_attr->name, ACPI_NAME_SIZE);
-	else
-		memcpy(name, "\0\0\0\0", 4);
 
-	status = acpi_get_table(name, table_attr->instance, &table_header);
+	status = acpi_get_table(table_attr->name, table_attr->instance,
+				&table_header);
 	if (ACPI_FAILURE(status))
 		return -ENODEV;
 
@@ -344,38 +343,45 @@ static ssize_t acpi_table_show(struct file *filp, struct kobject *kobj,
 				       table_header, table_header->length);
 }
 
-static void acpi_table_attr_init(struct acpi_table_attr *table_attr,
-				 struct acpi_table_header *table_header)
+static int acpi_table_attr_init(struct kobject *tables_obj,
+				struct acpi_table_attr *table_attr,
+				struct acpi_table_header *table_header)
 {
 	struct acpi_table_header *header = NULL;
 	struct acpi_table_attr *attr = NULL;
+	char instance_str[ACPI_INST_SIZE];
 
 	sysfs_attr_init(&table_attr->attr.attr);
-	if (table_header->signature[0] != '\0')
-		memcpy(table_attr->name, table_header->signature,
-		       ACPI_NAME_SIZE);
-	else
-		memcpy(table_attr->name, "NULL", 4);
+	ACPI_MOVE_NAME(table_attr->name, table_header->signature);
 
 	list_for_each_entry(attr, &acpi_table_attr_list, node) {
-		if (!memcmp(table_attr->name, attr->name, ACPI_NAME_SIZE))
+		if (ACPI_COMPARE_NAME(table_attr->name, attr->name))
 			if (table_attr->instance < attr->instance)
 				table_attr->instance = attr->instance;
 	}
 	table_attr->instance++;
+	if (table_attr->instance > ACPI_MAX_TABLE_INSTANCES) {
+		pr_warn("%4.4s: too many table instances\n",
+			table_attr->name);
+		return -ERANGE;
+	}
 
+	ACPI_MOVE_NAME(table_attr->filename, table_header->signature);
+	table_attr->filename[ACPI_NAME_SIZE] = '\0';
 	if (table_attr->instance > 1 || (table_attr->instance == 1 &&
 					 !acpi_get_table
-					 (table_header->signature, 2, &header)))
-		sprintf(table_attr->name + ACPI_NAME_SIZE, "%d",
-			table_attr->instance);
+					 (table_header->signature, 2, &header))) {
+		snprintf(instance_str, sizeof(instance_str), "%u",
+			 table_attr->instance);
+		strcat(table_attr->filename, instance_str);
+	}
 
 	table_attr->attr.size = table_header->length;
 	table_attr->attr.read = acpi_table_show;
-	table_attr->attr.attr.name = table_attr->name;
+	table_attr->attr.attr.name = table_attr->filename;
 	table_attr->attr.attr.mode = 0400;
 
-	return;
+	return sysfs_create_bin_file(tables_obj, &table_attr->attr);
 }
 
 acpi_status acpi_sysfs_table_handler(u32 event, void *table, void *context)
@@ -383,21 +389,22 @@ acpi_status acpi_sysfs_table_handler(u32 event, void *table, void *context)
 	struct acpi_table_attr *table_attr;
 
 	switch (event) {
-	case ACPI_TABLE_EVENT_LOAD:
+	case ACPI_TABLE_EVENT_INSTALL:
 		table_attr =
 		    kzalloc(sizeof(struct acpi_table_attr), GFP_KERNEL);
 		if (!table_attr)
 			return AE_NO_MEMORY;
 
-		acpi_table_attr_init(table_attr, table);
-		if (sysfs_create_bin_file(dynamic_tables_kobj,
-					  &table_attr->attr)) {
+		if (acpi_table_attr_init(dynamic_tables_kobj,
+					 table_attr, table)) {
 			kfree(table_attr);
 			return AE_ERROR;
-		} else
-			list_add_tail(&table_attr->node, &acpi_table_attr_list);
+		}
+		list_add_tail(&table_attr->node, &acpi_table_attr_list);
 		break;
+	case ACPI_TABLE_EVENT_LOAD:
 	case ACPI_TABLE_EVENT_UNLOAD:
+	case ACPI_TABLE_EVENT_UNINSTALL:
 		/*
 		 * we do not need to do anything right now
 		 * because the table is not deleted from the
@@ -435,13 +442,12 @@ static int acpi_tables_sysfs_init(void)
 		if (ACPI_FAILURE(status))
 			continue;
 
-		table_attr = NULL;
 		table_attr = kzalloc(sizeof(*table_attr), GFP_KERNEL);
 		if (!table_attr)
 			return -ENOMEM;
 
-		acpi_table_attr_init(table_attr, table_header);
-		ret = sysfs_create_bin_file(tables_kobj, &table_attr->attr);
+		ret = acpi_table_attr_init(tables_kobj,
+					   table_attr, table_header);
 		if (ret) {
 			kfree(table_attr);
 			return ret;
@@ -597,14 +603,27 @@ static ssize_t counter_show(struct kobject *kobj,
 	if (result)
 		goto end;
 
+	if (status & ACPI_EVENT_FLAG_ENABLE_SET)
+		size += sprintf(buf + size, "  EN");
+	else
+		size += sprintf(buf + size, "    ");
+	if (status & ACPI_EVENT_FLAG_STATUS_SET)
+		size += sprintf(buf + size, " STS");
+	else
+		size += sprintf(buf + size, "    ");
+
 	if (!(status & ACPI_EVENT_FLAG_HAS_HANDLER))
-		size += sprintf(buf + size, "   invalid");
+		size += sprintf(buf + size, " invalid     ");
 	else if (status & ACPI_EVENT_FLAG_ENABLED)
-		size += sprintf(buf + size, "   enabled");
+		size += sprintf(buf + size, " enabled     ");
 	else if (status & ACPI_EVENT_FLAG_WAKE_ENABLED)
-		size += sprintf(buf + size, "   wake_enabled");
+		size += sprintf(buf + size, " wake_enabled");
+	else
+		size += sprintf(buf + size, " disabled    ");
+	if (status & ACPI_EVENT_FLAG_MASKED)
+		size += sprintf(buf + size, " masked  ");
 	else
-		size += sprintf(buf + size, "   disabled");
+		size += sprintf(buf + size, " unmasked");
 
 end:
 	size += sprintf(buf + size, "\n");
@@ -655,8 +674,12 @@ static ssize_t counter_set(struct kobject *kobj,
 			 !(status & ACPI_EVENT_FLAG_ENABLED))
 			result = acpi_enable_gpe(handle, index);
 		else if (!strcmp(buf, "clear\n") &&
-			 (status & ACPI_EVENT_FLAG_SET))
+			 (status & ACPI_EVENT_FLAG_STATUS_SET))
 			result = acpi_clear_gpe(handle, index);
+		else if (!strcmp(buf, "mask\n"))
+			result = acpi_mask_gpe(handle, index, TRUE);
+		else if (!strcmp(buf, "unmask\n"))
+			result = acpi_mask_gpe(handle, index, FALSE);
 		else if (!kstrtoul(buf, 0, &tmp))
 			all_counters[index].count = tmp;
 		else
@@ -664,13 +687,13 @@ static ssize_t counter_set(struct kobject *kobj,
 	} else if (index < num_gpes + ACPI_NUM_FIXED_EVENTS) {
 		int event = index - num_gpes;
 		if (!strcmp(buf, "disable\n") &&
-		    (status & ACPI_EVENT_FLAG_ENABLED))
+		    (status & ACPI_EVENT_FLAG_ENABLE_SET))
 			result = acpi_disable_event(event, ACPI_NOT_ISR);
 		else if (!strcmp(buf, "enable\n") &&
-			 !(status & ACPI_EVENT_FLAG_ENABLED))
+			 !(status & ACPI_EVENT_FLAG_ENABLE_SET))
 			result = acpi_enable_event(event, ACPI_NOT_ISR);
 		else if (!strcmp(buf, "clear\n") &&
-			 (status & ACPI_EVENT_FLAG_SET))
+			 (status & ACPI_EVENT_FLAG_STATUS_SET))
 			result = acpi_clear_event(event);
 		else if (!kstrtoul(buf, 0, &tmp))
 			all_counters[index].count = tmp;
diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c
index 9f0ad6ebb368..cdd56c4657e0 100644
--- a/drivers/acpi/tables.c
+++ b/drivers/acpi/tables.c
@@ -35,7 +35,6 @@
 #include <linux/earlycpio.h>
 #include <linux/memblock.h>
 #include <linux/initrd.h>
-#include <linux/acpi.h>
 #include "internal.h"
 
 #ifdef CONFIG_ACPI_CUSTOM_DSDT
@@ -246,6 +245,7 @@ acpi_parse_entries_array(char *id, unsigned long table_size,
 	struct acpi_subtable_header *entry;
 	unsigned long table_end;
 	int count = 0;
+	int errs = 0;
 	int i;
 
 	if (acpi_disabled)
@@ -278,10 +278,12 @@ acpi_parse_entries_array(char *id, unsigned long table_size,
 			if (entry->type != proc[i].id)
 				continue;
 			if (!proc[i].handler ||
-			     proc[i].handler(entry, table_end))
-				return -EINVAL;
+			     (!errs && proc[i].handler(entry, table_end))) {
+				errs++;
+				continue;
+			}
 
-			proc->count++;
+			proc[i].count++;
 			break;
 		}
 		if (i != proc_num)
@@ -301,11 +303,11 @@ acpi_parse_entries_array(char *id, unsigned long table_size,
 	}
 
 	if (max_entries && count > max_entries) {
-		pr_warn("[%4.4s:0x%02x] ignored %i entries of %i found\n",
-			id, proc->id, count - max_entries, count);
+		pr_warn("[%4.4s:0x%02x] found the maximum %i entries\n",
+			id, proc->id, count);
 	}
 
-	return count;
+	return errs ? -EINVAL : count;
 }
 
 int __init
diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c
index a5b5c87e2114..a56fa2a1e9aa 100644
--- a/drivers/amba/bus.c
+++ b/drivers/amba/bus.c
@@ -19,6 +19,7 @@
 #include <linux/amba/bus.h>
 #include <linux/sizes.h>
 #include <linux/limits.h>
+#include <linux/clk/clk-conf.h>
 
 #include <asm/irq.h>
 
@@ -237,6 +238,10 @@ static int amba_probe(struct device *dev)
 	int ret;
 
 	do {
+		ret = of_clk_set_defaults(dev->of_node, false);
+		if (ret < 0)
+			break;
+
 		ret = dev_pm_domain_attach(dev, true);
 		if (ret == -EPROBE_DEFER)
 			break;
diff --git a/drivers/android/binder.c b/drivers/android/binder.c
index 16288e777ec3..562af94bec35 100644
--- a/drivers/android/binder.c
+++ b/drivers/android/binder.c
@@ -59,7 +59,6 @@ static struct dentry *binder_debugfs_dir_entry_proc;
 static struct binder_node *binder_context_mgr_node;
 static kuid_t binder_context_mgr_uid = INVALID_UID;
 static int binder_last_id;
-static struct workqueue_struct *binder_deferred_workqueue;
 
 #define BINDER_DEBUG_ENTRY(name) \
 static int binder_##name##_open(struct inode *inode, struct file *file) \
@@ -3227,7 +3226,7 @@ binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer)
 	if (hlist_unhashed(&proc->deferred_work_node)) {
 		hlist_add_head(&proc->deferred_work_node,
 				&binder_deferred_list);
-		queue_work(binder_deferred_workqueue, &binder_deferred_work);
+		schedule_work(&binder_deferred_work);
 	}
 	mutex_unlock(&binder_deferred_lock);
 }
@@ -3679,10 +3678,6 @@ static int __init binder_init(void)
 {
 	int ret;
 
-	binder_deferred_workqueue = create_singlethread_workqueue("binder");
-	if (!binder_deferred_workqueue)
-		return -ENOMEM;
-
 	binder_debugfs_dir_entry_root = debugfs_create_dir("binder", NULL);
 	if (binder_debugfs_dir_entry_root)
 		binder_debugfs_dir_entry_proc = debugfs_create_dir("proc",
diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
index 7461a587b39b..dcf2c724fd06 100644
--- a/drivers/ata/libahci.c
+++ b/drivers/ata/libahci.c
@@ -2524,7 +2524,7 @@ static int ahci_host_activate_multi_irqs(struct ata_host *host,
 
 		/* Do not receive interrupts sent by dummy ports */
 		if (!pp) {
-			disable_irq(irq + i);
+			disable_irq(irq);
 			continue;
 		}
 
diff --git a/drivers/ata/pata_ninja32.c b/drivers/ata/pata_ninja32.c
index 633aa2934a18..44f97ad3c88d 100644
--- a/drivers/ata/pata_ninja32.c
+++ b/drivers/ata/pata_ninja32.c
@@ -144,7 +144,7 @@ static int ninja32_init_one(struct pci_dev *dev, const struct pci_device_id *id)
 	ap->ioaddr.altstatus_addr = base + 0x1E;
 	ap->ioaddr.bmdma_addr = base;
 	ata_sff_std_ports(&ap->ioaddr);
-	ap->pflags = ATA_PFLAG_PIO32 | ATA_PFLAG_PIO32CHANGE;
+	ap->pflags |= ATA_PFLAG_PIO32 | ATA_PFLAG_PIO32CHANGE;
 
 	ninja32_program(base);
 	/* FIXME: Should we disable them at remove ? */
diff --git a/drivers/atm/eni.c b/drivers/atm/eni.c
index 6339efd32697..f2aaf9e32a36 100644
--- a/drivers/atm/eni.c
+++ b/drivers/atm/eni.c
@@ -1845,8 +1845,9 @@ static int eni_start(struct atm_dev *dev)
 	/* initialize memory management */
 	buffer_mem = eni_dev->mem - (buf - eni_dev->ram);
 	eni_dev->free_list_size = buffer_mem/MID_MIN_BUF_SIZE/2;
-	eni_dev->free_list = kmalloc(
-	    sizeof(struct eni_free)*(eni_dev->free_list_size+1),GFP_KERNEL);
+	eni_dev->free_list = kmalloc_array(eni_dev->free_list_size + 1,
+					   sizeof(*eni_dev->free_list),
+					   GFP_KERNEL);
 	if (!eni_dev->free_list) {
 		printk(KERN_ERR DEV_LABEL "(itf %d): couldn't get free page\n",
 		    dev->number);
diff --git a/drivers/atm/fore200e.c b/drivers/atm/fore200e.c
index 75dde903b238..81aaa505862c 100644
--- a/drivers/atm/fore200e.c
+++ b/drivers/atm/fore200e.c
@@ -2489,7 +2489,7 @@ static int fore200e_load_and_start_fw(struct fore200e *fore200e)
 {
     const struct firmware *firmware;
     struct device *device;
-    struct fw_header *fw_header;
+    const struct fw_header *fw_header;
     const __le32 *fw_data;
     u32 fw_size;
     u32 __iomem *load_addr;
@@ -2511,9 +2511,9 @@ static int fore200e_load_and_start_fw(struct fore200e *fore200e)
 	return err;
     }
 
-    fw_data = (__le32 *) firmware->data;
+    fw_data = (const __le32 *)firmware->data;
     fw_size = firmware->size / sizeof(u32);
-    fw_header = (struct fw_header *) firmware->data;
+    fw_header = (const struct fw_header *)firmware->data;
     load_addr = fore200e->virt_base + le32_to_cpu(fw_header->load_offset);
 
     DPRINTK(2, "device %s firmware being loaded at 0x%p (%d words)\n",
diff --git a/drivers/atm/he.c b/drivers/atm/he.c
index 0f5cb37636bc..31b513a23ae0 100644
--- a/drivers/atm/he.c
+++ b/drivers/atm/he.c
@@ -779,8 +779,9 @@ static int he_init_group(struct he_dev *he_dev, int group)
 		  G0_RBPS_BS + (group * 32));
 
 	/* bitmap table */
-	he_dev->rbpl_table = kmalloc(BITS_TO_LONGS(RBPL_TABLE_SIZE)
-				     * sizeof(unsigned long), GFP_KERNEL);
+	he_dev->rbpl_table = kmalloc_array(BITS_TO_LONGS(RBPL_TABLE_SIZE),
+					   sizeof(*he_dev->rbpl_table),
+					   GFP_KERNEL);
 	if (!he_dev->rbpl_table) {
 		hprintk("unable to allocate rbpl bitmap table\n");
 		return -ENOMEM;
@@ -788,8 +789,9 @@ static int he_init_group(struct he_dev *he_dev, int group)
 	bitmap_zero(he_dev->rbpl_table, RBPL_TABLE_SIZE);
 
 	/* rbpl_virt 64-bit pointers */
-	he_dev->rbpl_virt = kmalloc(RBPL_TABLE_SIZE
-				    * sizeof(struct he_buff *), GFP_KERNEL);
+	he_dev->rbpl_virt = kmalloc_array(RBPL_TABLE_SIZE,
+					  sizeof(*he_dev->rbpl_virt),
+					  GFP_KERNEL);
 	if (!he_dev->rbpl_virt) {
 		hprintk("unable to allocate rbpl virt table\n");
 		goto out_free_rbpl_table;
diff --git a/drivers/atm/iphase.c b/drivers/atm/iphase.c
index 809dd1e02091..b2756765950e 100644
--- a/drivers/atm/iphase.c
+++ b/drivers/atm/iphase.c
@@ -1885,9 +1885,9 @@ static int open_tx(struct atm_vcc *vcc)
                 if ((ret = ia_cbr_setup (iadev, vcc)) < 0) {     
                     return ret;
                 }
-       } 
-	else  
-           printk("iadev:  Non UBR, ABR and CBR traffic not supportedn"); 
+	} else {
+		printk("iadev:  Non UBR, ABR and CBR traffic not supported\n");
+	}
         
         iadev->testTable[vcc->vci]->vc_status |= VC_ACTIVE;
 	IF_EVENT(printk("ia open_tx returning \n");)  
@@ -1975,7 +1975,9 @@ static int tx_init(struct atm_dev *dev)
 		buf_desc_ptr++;		  
 		tx_pkt_start += iadev->tx_buf_sz;  
 	}  
-        iadev->tx_buf = kmalloc(iadev->num_tx_desc*sizeof(struct cpcs_trailer_desc), GFP_KERNEL);
+	iadev->tx_buf = kmalloc_array(iadev->num_tx_desc,
+				      sizeof(*iadev->tx_buf),
+				      GFP_KERNEL);
         if (!iadev->tx_buf) {
             printk(KERN_ERR DEV_LABEL " couldn't get mem\n");
 	    goto err_free_dle;
@@ -1995,8 +1997,9 @@ static int tx_init(struct atm_dev *dev)
 						       sizeof(*cpcs),
 						       DMA_TO_DEVICE);
         }
-        iadev->desc_tbl = kmalloc(iadev->num_tx_desc *
-                                   sizeof(struct desc_tbl_t), GFP_KERNEL);
+	iadev->desc_tbl = kmalloc_array(iadev->num_tx_desc,
+					sizeof(*iadev->desc_tbl),
+					GFP_KERNEL);
 	if (!iadev->desc_tbl) {
 		printk(KERN_ERR DEV_LABEL " couldn't get mem\n");
 		goto err_free_all_tx_bufs;
@@ -2124,7 +2127,9 @@ static int tx_init(struct atm_dev *dev)
 	memset((caddr_t)(iadev->seg_ram+i),  0, iadev->num_vc*4);
 	vc = (struct main_vc *)iadev->MAIN_VC_TABLE_ADDR;  
 	evc = (struct ext_vc *)iadev->EXT_VC_TABLE_ADDR;  
-        iadev->testTable = kmalloc(sizeof(long)*iadev->num_vc, GFP_KERNEL); 
+	iadev->testTable = kmalloc_array(iadev->num_vc,
+					 sizeof(*iadev->testTable),
+					 GFP_KERNEL);
         if (!iadev->testTable) {
            printk("Get freepage  failed\n");
 	   goto err_free_desc_tbl;
diff --git a/drivers/atm/nicstar.c b/drivers/atm/nicstar.c
index 700ed15c2362..c7296b583787 100644
--- a/drivers/atm/nicstar.c
+++ b/drivers/atm/nicstar.c
@@ -370,7 +370,8 @@ static int ns_init_card(int i, struct pci_dev *pcidev)
 		return error;
         }
 
-	if ((card = kmalloc(sizeof(ns_dev), GFP_KERNEL)) == NULL) {
+	card = kmalloc(sizeof(*card), GFP_KERNEL);
+	if (!card) {
 		printk
 		    ("nicstar%d: can't allocate memory for device structure.\n",
 		     i);
@@ -611,7 +612,7 @@ static int ns_init_card(int i, struct pci_dev *pcidev)
 	for (j = 0; j < card->rct_size; j++)
 		ns_write_sram(card, j * 4, u32d, 4);
 
-	memset(card->vcmap, 0, NS_MAX_RCTSIZE * sizeof(vc_map));
+	memset(card->vcmap, 0, sizeof(card->vcmap));
 
 	for (j = 0; j < NS_FRSCD_NUM; j++)
 		card->scd2vc[j] = NULL;
@@ -862,7 +863,7 @@ static scq_info *get_scq(ns_dev *card, int size, u32 scd)
 	if (size != VBR_SCQSIZE && size != CBR_SCQSIZE)
 		return NULL;
 
-	scq = kmalloc(sizeof(scq_info), GFP_KERNEL);
+	scq = kmalloc(sizeof(*scq), GFP_KERNEL);
 	if (!scq)
 		return NULL;
         scq->org = dma_alloc_coherent(&card->pcidev->dev,
@@ -871,8 +872,9 @@ static scq_info *get_scq(ns_dev *card, int size, u32 scd)
 		kfree(scq);
 		return NULL;
 	}
-	scq->skb = kmalloc(sizeof(struct sk_buff *) *
-			   (size / NS_SCQE_SIZE), GFP_KERNEL);
+	scq->skb = kmalloc_array(size / NS_SCQE_SIZE,
+				 sizeof(*scq->skb),
+				 GFP_KERNEL);
 	if (!scq->skb) {
 		dma_free_coherent(&card->pcidev->dev,
 				  2 * size, scq->org, scq->dma);
@@ -2021,7 +2023,8 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
 
 		cell = skb->data;
 		for (i = ns_rsqe_cellcount(rsqe); i; i--) {
-			if ((sb = dev_alloc_skb(NS_SMSKBSIZE)) == NULL) {
+			sb = dev_alloc_skb(NS_SMSKBSIZE);
+			if (!sb) {
 				printk
 				    ("nicstar%d: Can't allocate buffers for aal0.\n",
 				     card->index);
diff --git a/drivers/atm/zatm.c b/drivers/atm/zatm.c
index cecfb943762f..d3dc95484161 100644
--- a/drivers/atm/zatm.c
+++ b/drivers/atm/zatm.c
@@ -598,12 +598,13 @@ static void close_rx(struct atm_vcc *vcc)
 static int start_rx(struct atm_dev *dev)
 {
 	struct zatm_dev *zatm_dev;
-	int size,i;
+	int i;
 
-DPRINTK("start_rx\n");
+	DPRINTK("start_rx\n");
 	zatm_dev = ZATM_DEV(dev);
-	size = sizeof(struct atm_vcc *)*zatm_dev->chans;
-	zatm_dev->rx_map =  kzalloc(size,GFP_KERNEL);
+	zatm_dev->rx_map = kcalloc(zatm_dev->chans,
+				   sizeof(*zatm_dev->rx_map),
+				   GFP_KERNEL);
 	if (!zatm_dev->rx_map) return -ENOMEM;
 	/* set VPI/VCI split (use all VCIs and give what's left to VPIs) */
 	zpokel(zatm_dev,(1 << dev->ci_range.vci_bits)-1,uPD98401_VRR);
@@ -998,8 +999,9 @@ static int start_tx(struct atm_dev *dev)
 
 	DPRINTK("start_tx\n");
 	zatm_dev = ZATM_DEV(dev);
-	zatm_dev->tx_map = kmalloc(sizeof(struct atm_vcc *)*
-	    zatm_dev->chans,GFP_KERNEL);
+	zatm_dev->tx_map = kmalloc_array(zatm_dev->chans,
+					 sizeof(*zatm_dev->tx_map),
+					 GFP_KERNEL);
 	if (!zatm_dev->tx_map) return -ENOMEM;
 	zatm_dev->tx_bw = ATM_OC3_PCR;
 	zatm_dev->free_shapers = (1 << NR_SHAPERS)-1;
@@ -1398,7 +1400,7 @@ static int zatm_open(struct atm_vcc *vcc)
 	DPRINTK(DEV_LABEL "(itf %d): open %d.%d\n",vcc->dev->number,vcc->vpi,
 	    vcc->vci);
 	if (!test_bit(ATM_VF_PARTIAL,&vcc->flags)) {
-		zatm_vcc = kmalloc(sizeof(struct zatm_vcc),GFP_KERNEL);
+		zatm_vcc = kmalloc(sizeof(*zatm_vcc), GFP_KERNEL);
 		if (!zatm_vcc) {
 			clear_bit(ATM_VF_ADDR,&vcc->flags);
 			return -ENOMEM;
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index 98504ec99c7d..fdf44cac08e6 100644
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -212,6 +212,16 @@ config DEBUG_DEVRES
 
 	  If you are unsure about this, Say N here.
 
+config DEBUG_TEST_DRIVER_REMOVE
+	bool "Test driver remove calls during probe"
+	depends on DEBUG_KERNEL
+	help
+	  Say Y here if you want the Driver core to test driver remove functions
+	  by calling probe, remove, probe. This tests the remove path without
+	  having to unbind the driver or unload the driver module.
+
+	  If you are unsure about this, say N here.
+
 config SYS_HYPERVISOR
 	bool
 	default n
diff --git a/drivers/base/attribute_container.c b/drivers/base/attribute_container.c
index 2ba4cac080c5..95e3ef82f3b7 100644
--- a/drivers/base/attribute_container.c
+++ b/drivers/base/attribute_container.c
@@ -243,7 +243,7 @@ attribute_container_remove_device(struct device *dev,
  * @dev:  The generic device to run the trigger for
  * @fn	  the function to execute for each classdev.
  *
- * This funcion is for executing a trigger when you need to know both
+ * This function is for executing a trigger when you need to know both
  * the container and the classdev.  If you only care about the
  * container, then use attribute_container_trigger() instead.
  */
diff --git a/drivers/base/core.c b/drivers/base/core.c
index 0a8bdade53f2..ce057a568673 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -836,11 +836,29 @@ static struct kobject *get_device_parent(struct device *dev,
 	return NULL;
 }
 
+static inline bool live_in_glue_dir(struct kobject *kobj,
+				    struct device *dev)
+{
+	if (!kobj || !dev->class ||
+	    kobj->kset != &dev->class->p->glue_dirs)
+		return false;
+	return true;
+}
+
+static inline struct kobject *get_glue_dir(struct device *dev)
+{
+	return dev->kobj.parent;
+}
+
+/*
+ * make sure cleaning up dir as the last step, we need to make
+ * sure .release handler of kobject is run with holding the
+ * global lock
+ */
 static void cleanup_glue_dir(struct device *dev, struct kobject *glue_dir)
 {
 	/* see if we live in a "glue" directory */
-	if (!glue_dir || !dev->class ||
-	    glue_dir->kset != &dev->class->p->glue_dirs)
+	if (!live_in_glue_dir(glue_dir, dev))
 		return;
 
 	mutex_lock(&gdp_mutex);
@@ -848,11 +866,6 @@ static void cleanup_glue_dir(struct device *dev, struct kobject *glue_dir)
 	mutex_unlock(&gdp_mutex);
 }
 
-static void cleanup_device_parent(struct device *dev)
-{
-	cleanup_glue_dir(dev, dev->kobj.parent);
-}
-
 static int device_add_class_symlinks(struct device *dev)
 {
 	struct device_node *of_node = dev_of_node(dev);
@@ -1028,6 +1041,7 @@ int device_add(struct device *dev)
 	struct kobject *kobj;
 	struct class_interface *class_intf;
 	int error = -EINVAL;
+	struct kobject *glue_dir = NULL;
 
 	dev = get_device(dev);
 	if (!dev)
@@ -1072,8 +1086,10 @@ int device_add(struct device *dev)
 	/* first, register with generic layer. */
 	/* we require the name to be set before, and pass NULL */
 	error = kobject_add(&dev->kobj, dev->kobj.parent, NULL);
-	if (error)
+	if (error) {
+		glue_dir = get_glue_dir(dev);
 		goto Error;
+	}
 
 	/* notify platform of device entry */
 	if (platform_notify)
@@ -1154,9 +1170,10 @@ done:
 	device_remove_file(dev, &dev_attr_uevent);
  attrError:
 	kobject_uevent(&dev->kobj, KOBJ_REMOVE);
+	glue_dir = get_glue_dir(dev);
 	kobject_del(&dev->kobj);
  Error:
-	cleanup_device_parent(dev);
+	cleanup_glue_dir(dev, glue_dir);
 	put_device(parent);
 name_error:
 	kfree(dev->p);
@@ -1232,6 +1249,7 @@ EXPORT_SYMBOL_GPL(put_device);
 void device_del(struct device *dev)
 {
 	struct device *parent = dev->parent;
+	struct kobject *glue_dir = NULL;
 	struct class_interface *class_intf;
 
 	/* Notify clients of device removal.  This call must come
@@ -1266,6 +1284,7 @@ void device_del(struct device *dev)
 	bus_remove_device(dev);
 	device_pm_remove(dev);
 	driver_deferred_probe_del(dev);
+	device_remove_properties(dev);
 
 	/* Notify the platform of the removal, in case they
 	 * need to do anything...
@@ -1276,8 +1295,9 @@ void device_del(struct device *dev)
 		blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
 					     BUS_NOTIFY_REMOVED_DEVICE, dev);
 	kobject_uevent(&dev->kobj, KOBJ_REMOVE);
-	cleanup_device_parent(dev);
+	glue_dir = get_glue_dir(dev);
 	kobject_del(&dev->kobj);
+	cleanup_glue_dir(dev, glue_dir);
 	put_device(parent);
 }
 EXPORT_SYMBOL_GPL(device_del);
diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index 691eeea2f19a..4c28e1a09786 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -371,12 +371,13 @@ int register_cpu(struct cpu *cpu, int num)
 	if (cpu->hotpluggable)
 		cpu->dev.groups = hotplugable_cpu_attr_groups;
 	error = device_register(&cpu->dev);
-	if (!error)
-		per_cpu(cpu_sys_devices, num) = &cpu->dev;
-	if (!error)
-		register_cpu_under_node(num, cpu_to_node(num));
+	if (error)
+		return error;
 
-	return error;
+	per_cpu(cpu_sys_devices, num) = &cpu->dev;
+	register_cpu_under_node(num, cpu_to_node(num));
+
+	return 0;
 }
 
 struct device *get_cpu_device(unsigned cpu)
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 16688f50729c..d22a7260f42b 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -51,7 +51,6 @@
 static DEFINE_MUTEX(deferred_probe_mutex);
 static LIST_HEAD(deferred_probe_pending_list);
 static LIST_HEAD(deferred_probe_active_list);
-static struct workqueue_struct *deferred_wq;
 static atomic_t deferred_trigger_count = ATOMIC_INIT(0);
 
 /*
@@ -175,7 +174,7 @@ static void driver_deferred_probe_trigger(void)
 	 * Kick the re-probe thread.  It may already be scheduled, but it is
 	 * safe to kick it again.
 	 */
-	queue_work(deferred_wq, &deferred_probe_work);
+	schedule_work(&deferred_probe_work);
 }
 
 /**
@@ -211,14 +210,10 @@ void device_unblock_probing(void)
  */
 static int deferred_probe_initcall(void)
 {
-	deferred_wq = create_singlethread_workqueue("deferwq");
-	if (WARN_ON(!deferred_wq))
-		return -ENOMEM;
-
 	driver_deferred_probe_enable = true;
 	driver_deferred_probe_trigger();
 	/* Sort as many dependencies as possible before exiting initcalls */
-	flush_workqueue(deferred_wq);
+	flush_work(&deferred_probe_work);
 	return 0;
 }
 late_initcall(deferred_probe_initcall);
@@ -329,6 +324,7 @@ static int really_probe(struct device *dev, struct device_driver *drv)
 {
 	int ret = -EPROBE_DEFER;
 	int local_trigger_count = atomic_read(&deferred_trigger_count);
+	bool test_remove = IS_ENABLED(CONFIG_DEBUG_TEST_DRIVER_REMOVE);
 
 	if (defer_all_probes) {
 		/*
@@ -346,6 +342,7 @@ static int really_probe(struct device *dev, struct device_driver *drv)
 		 drv->bus->name, __func__, drv->name, dev_name(dev));
 	WARN_ON(!list_empty(&dev->devres_head));
 
+re_probe:
 	dev->driver = drv;
 
 	/* If using pinctrl, bind pins now before probing */
@@ -383,6 +380,25 @@ static int really_probe(struct device *dev, struct device_driver *drv)
 			goto probe_failed;
 	}
 
+	if (test_remove) {
+		test_remove = false;
+
+		if (dev->bus && dev->bus->remove)
+			dev->bus->remove(dev);
+		else if (drv->remove)
+			drv->remove(dev);
+
+		devres_release_all(dev);
+		driver_sysfs_remove(dev);
+		dev->driver = NULL;
+		dev_set_drvdata(dev, NULL);
+		if (dev->pm_domain && dev->pm_domain->dismiss)
+			dev->pm_domain->dismiss(dev);
+		pm_runtime_reinit(dev);
+
+		goto re_probe;
+	}
+
 	pinctrl_init_done(dev);
 
 	if (dev->pm_domain && dev->pm_domain->sync)
@@ -460,8 +476,7 @@ int driver_probe_done(void)
 void wait_for_device_probe(void)
 {
 	/* wait for the deferred probe workqueue to finish */
-	if (driver_deferred_probe_enable)
-		flush_workqueue(deferred_wq);
+	flush_work(&deferred_probe_work);
 
 	/* wait for the known devices to complete their probing */
 	wait_event(probe_waitqueue, atomic_read(&probe_count) == 0);
diff --git a/drivers/base/dma-coherent.c b/drivers/base/dma-coherent.c
index bdf28f7dd5e8..640a7e63c453 100644
--- a/drivers/base/dma-coherent.c
+++ b/drivers/base/dma-coherent.c
@@ -165,6 +165,7 @@ int dma_alloc_from_coherent(struct device *dev, ssize_t size,
 	int order = get_order(size);
 	unsigned long flags;
 	int pageno;
+	int dma_memory_map;
 
 	if (!dev)
 		return 0;
@@ -187,11 +188,12 @@ int dma_alloc_from_coherent(struct device *dev, ssize_t size,
 	 */
 	*dma_handle = mem->device_base + (pageno << PAGE_SHIFT);
 	*ret = mem->virt_base + (pageno << PAGE_SHIFT);
-	if (mem->flags & DMA_MEMORY_MAP)
+	dma_memory_map = (mem->flags & DMA_MEMORY_MAP);
+	spin_unlock_irqrestore(&mem->spinlock, flags);
+	if (dma_memory_map)
 		memset(*ret, 0, size);
 	else
 		memset_io(*ret, 0, size);
-	spin_unlock_irqrestore(&mem->spinlock, flags);
 
 	return 1;
 
@@ -261,8 +263,8 @@ int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma,
 		   (mem->virt_base + (mem->size << PAGE_SHIFT))) {
 		unsigned long off = vma->vm_pgoff;
 		int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;
-		int user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
-		int count = size >> PAGE_SHIFT;
+		int user_count = vma_pages(vma);
+		int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 
 		*ret = -ENXIO;
 		if (off < count && user_count <= count - off) {
diff --git a/drivers/base/dma-mapping.c b/drivers/base/dma-mapping.c
index d799662f19eb..8f8b68c80986 100644
--- a/drivers/base/dma-mapping.c
+++ b/drivers/base/dma-mapping.c
@@ -198,10 +198,13 @@ int dmam_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
 
 	rc = dma_declare_coherent_memory(dev, phys_addr, device_addr, size,
 					 flags);
-	if (rc == 0)
+	if (rc) {
 		devres_add(dev, res);
-	else
+		rc = 0;
+	} else {
 		devres_free(res);
+		rc = -ENOMEM;
+	}
 
 	return rc;
 }
@@ -247,7 +250,7 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 {
 	int ret = -ENXIO;
 #if defined(CONFIG_MMU) && !defined(CONFIG_ARCH_NO_COHERENT_DMA_MMAP)
-	unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	unsigned long user_count = vma_pages(vma);
 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 	unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr));
 	unsigned long off = vma->vm_pgoff;
@@ -334,7 +337,7 @@ void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
 		return;
 	}
 
-	unmap_kernel_range((unsigned long)cpu_addr, size);
+	unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
 	vunmap(cpu_addr);
 }
 #endif
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index dc75de9059cd..62c63c0c5c22 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -361,8 +361,11 @@ store_mem_state(struct device *dev,
 err:
 	unlock_device_hotplug();
 
-	if (ret)
+	if (ret < 0)
 		return ret;
+	if (ret)
+		return -EINVAL;
+
 	return count;
 }
 
diff --git a/drivers/base/pinctrl.c b/drivers/base/pinctrl.c
index 076297592754..5917b4b5fb99 100644
--- a/drivers/base/pinctrl.c
+++ b/drivers/base/pinctrl.c
@@ -91,9 +91,13 @@ cleanup_alloc:
 	devm_kfree(dev, dev->pins);
 	dev->pins = NULL;
 
-	/* Only return deferrals */
-	if (ret != -EPROBE_DEFER)
-		ret = 0;
+	/* Return deferrals */
+	if (ret == -EPROBE_DEFER)
+		return ret;
+	/* Return serious errors */
+	if (ret == -EINVAL)
+		return ret;
+	/* We ignore errors like -ENOENT meaning no pinctrl state */
 
-	return ret;
+	return 0;
 }
diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index 279e53989374..be6a599bc0c1 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -142,13 +142,12 @@ static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq,
 	}
 
 	for (i = 0; i < nvec; i++) {
-		desc = alloc_msi_entry(dev);
+		desc = alloc_msi_entry(dev, 1, NULL);
 		if (!desc)
 			break;
 
 		desc->platform.msi_priv_data = data;
 		desc->platform.msi_index = base + i;
-		desc->nvec_used = 1;
 		desc->irq = virq ? virq + i : 0;
 
 		list_add_tail(&desc->list, dev_to_msi_list(dev));
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index 6482d47deb50..c4af00385502 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -97,7 +97,7 @@ int platform_get_irq(struct platform_device *dev, unsigned int num)
 		int ret;
 
 		ret = of_irq_get(dev->dev.of_node, num);
-		if (ret >= 0 || ret == -EPROBE_DEFER)
+		if (ret > 0 || ret == -EPROBE_DEFER)
 			return ret;
 	}
 
@@ -108,9 +108,14 @@ int platform_get_irq(struct platform_device *dev, unsigned int num)
 	 * IORESOURCE_BITS correspond 1-to-1 to the IRQF_TRIGGER*
 	 * settings.
 	 */
-	if (r && r->flags & IORESOURCE_BITS)
-		irqd_set_trigger_type(irq_get_irq_data(r->start),
-				      r->flags & IORESOURCE_BITS);
+	if (r && r->flags & IORESOURCE_BITS) {
+		struct irq_data *irqd;
+
+		irqd = irq_get_irq_data(r->start);
+		if (!irqd)
+			return -ENXIO;
+		irqd_set_trigger_type(irqd, r->flags & IORESOURCE_BITS);
+	}
 
 	return r ? r->start : -ENXIO;
 #endif
@@ -175,7 +180,7 @@ int platform_get_irq_byname(struct platform_device *dev, const char *name)
 		int ret;
 
 		ret = of_irq_get_byname(dev->dev.of_node, name);
-		if (ret >= 0 || ret == -EPROBE_DEFER)
+		if (ret > 0 || ret == -EPROBE_DEFER)
 			return ret;
 	}
 
@@ -434,6 +439,7 @@ void platform_device_del(struct platform_device *pdev)
 	int i;
 
 	if (pdev) {
+		device_remove_properties(&pdev->dev);
 		device_del(&pdev->dev);
 
 		if (pdev->id_auto) {
@@ -446,8 +452,6 @@ void platform_device_del(struct platform_device *pdev)
 			if (r->parent)
 				release_resource(r);
 		}
-
-		device_remove_properties(&pdev->dev);
 	}
 }
 EXPORT_SYMBOL_GPL(platform_device_del);
diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index a1f2aff33997..e023066e4215 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -45,7 +45,7 @@ static DEFINE_MUTEX(gpd_list_lock);
  * and checks that the PM domain pointer is a real generic PM domain.
  * Any failure results in NULL being returned.
  */
-struct generic_pm_domain *pm_genpd_lookup_dev(struct device *dev)
+static struct generic_pm_domain *genpd_lookup_dev(struct device *dev)
 {
 	struct generic_pm_domain *genpd = NULL, *gpd;
 
@@ -586,7 +586,7 @@ static int __init genpd_poweroff_unused(void)
 }
 late_initcall(genpd_poweroff_unused);
 
-#ifdef CONFIG_PM_SLEEP
+#if defined(CONFIG_PM_SLEEP) || defined(CONFIG_PM_GENERIC_DOMAINS_OF)
 
 /**
  * pm_genpd_present - Check if the given PM domain has been initialized.
@@ -606,6 +606,10 @@ static bool pm_genpd_present(const struct generic_pm_domain *genpd)
 	return false;
 }
 
+#endif
+
+#ifdef CONFIG_PM_SLEEP
+
 static bool genpd_dev_active_wakeup(struct generic_pm_domain *genpd,
 				    struct device *dev)
 {
@@ -613,9 +617,8 @@ static bool genpd_dev_active_wakeup(struct generic_pm_domain *genpd,
 }
 
 /**
- * pm_genpd_sync_poweroff - Synchronously power off a PM domain and its masters.
+ * genpd_sync_poweroff - Synchronously power off a PM domain and its masters.
  * @genpd: PM domain to power off, if possible.
- * @timed: True if latency measurements are allowed.
  *
  * Check if the given PM domain can be powered off (during system suspend or
  * hibernation) and do that if so.  Also, in that case propagate to its masters.
@@ -625,8 +628,7 @@ static bool genpd_dev_active_wakeup(struct generic_pm_domain *genpd,
  * executed sequentially, so it is guaranteed that it will never run twice in
  * parallel).
  */
-static void pm_genpd_sync_poweroff(struct generic_pm_domain *genpd,
-				   bool timed)
+static void genpd_sync_poweroff(struct generic_pm_domain *genpd)
 {
 	struct gpd_link *link;
 
@@ -639,28 +641,26 @@ static void pm_genpd_sync_poweroff(struct generic_pm_domain *genpd,
 
 	/* Choose the deepest state when suspending */
 	genpd->state_idx = genpd->state_count - 1;
-	genpd_power_off(genpd, timed);
+	genpd_power_off(genpd, false);
 
 	genpd->status = GPD_STATE_POWER_OFF;
 
 	list_for_each_entry(link, &genpd->slave_links, slave_node) {
 		genpd_sd_counter_dec(link->master);
-		pm_genpd_sync_poweroff(link->master, timed);
+		genpd_sync_poweroff(link->master);
 	}
 }
 
 /**
- * pm_genpd_sync_poweron - Synchronously power on a PM domain and its masters.
+ * genpd_sync_poweron - Synchronously power on a PM domain and its masters.
  * @genpd: PM domain to power on.
- * @timed: True if latency measurements are allowed.
  *
  * This function is only called in "noirq" and "syscore" stages of system power
  * transitions, so it need not acquire locks (all of the "noirq" callbacks are
  * executed sequentially, so it is guaranteed that it will never run twice in
  * parallel).
  */
-static void pm_genpd_sync_poweron(struct generic_pm_domain *genpd,
-				  bool timed)
+static void genpd_sync_poweron(struct generic_pm_domain *genpd)
 {
 	struct gpd_link *link;
 
@@ -668,11 +668,11 @@ static void pm_genpd_sync_poweron(struct generic_pm_domain *genpd,
 		return;
 
 	list_for_each_entry(link, &genpd->slave_links, slave_node) {
-		pm_genpd_sync_poweron(link->master, timed);
+		genpd_sync_poweron(link->master);
 		genpd_sd_counter_inc(link->master);
 	}
 
-	genpd_power_on(genpd, timed);
+	genpd_power_on(genpd, false);
 
 	genpd->status = GPD_STATE_ACTIVE;
 }
@@ -784,7 +784,7 @@ static int pm_genpd_suspend_noirq(struct device *dev)
 	 * the same PM domain, so it is not necessary to use locking here.
 	 */
 	genpd->suspended_count++;
-	pm_genpd_sync_poweroff(genpd, true);
+	genpd_sync_poweroff(genpd);
 
 	return 0;
 }
@@ -814,7 +814,7 @@ static int pm_genpd_resume_noirq(struct device *dev)
 	 * guaranteed that this function will never run twice in parallel for
 	 * the same PM domain, so it is not necessary to use locking here.
 	 */
-	pm_genpd_sync_poweron(genpd, true);
+	genpd_sync_poweron(genpd);
 	genpd->suspended_count--;
 
 	if (genpd->dev_ops.stop && genpd->dev_ops.start)
@@ -902,12 +902,12 @@ static int pm_genpd_restore_noirq(struct device *dev)
 	if (genpd->suspended_count++ == 0)
 		/*
 		 * The boot kernel might put the domain into arbitrary state,
-		 * so make it appear as powered off to pm_genpd_sync_poweron(),
+		 * so make it appear as powered off to genpd_sync_poweron(),
 		 * so that it tries to power it on in case it was really off.
 		 */
 		genpd->status = GPD_STATE_POWER_OFF;
 
-	pm_genpd_sync_poweron(genpd, true);
+	genpd_sync_poweron(genpd);
 
 	if (genpd->dev_ops.stop && genpd->dev_ops.start)
 		ret = pm_runtime_force_resume(dev);
@@ -962,9 +962,9 @@ static void genpd_syscore_switch(struct device *dev, bool suspend)
 
 	if (suspend) {
 		genpd->suspended_count++;
-		pm_genpd_sync_poweroff(genpd, false);
+		genpd_sync_poweroff(genpd);
 	} else {
-		pm_genpd_sync_poweron(genpd, false);
+		genpd_sync_poweron(genpd);
 		genpd->suspended_count--;
 	}
 }
@@ -1056,14 +1056,8 @@ static void genpd_free_dev_data(struct device *dev,
 	dev_pm_put_subsys_data(dev);
 }
 
-/**
- * __pm_genpd_add_device - Add a device to an I/O PM domain.
- * @genpd: PM domain to add the device to.
- * @dev: Device to be added.
- * @td: Set of PM QoS timing parameters to attach to the device.
- */
-int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
-			  struct gpd_timing_data *td)
+static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
+			    struct gpd_timing_data *td)
 {
 	struct generic_pm_domain_data *gpd_data;
 	int ret = 0;
@@ -1103,15 +1097,28 @@ int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(__pm_genpd_add_device);
 
 /**
- * pm_genpd_remove_device - Remove a device from an I/O PM domain.
- * @genpd: PM domain to remove the device from.
- * @dev: Device to be removed.
+ * __pm_genpd_add_device - Add a device to an I/O PM domain.
+ * @genpd: PM domain to add the device to.
+ * @dev: Device to be added.
+ * @td: Set of PM QoS timing parameters to attach to the device.
  */
-int pm_genpd_remove_device(struct generic_pm_domain *genpd,
-			   struct device *dev)
+int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
+			  struct gpd_timing_data *td)
+{
+	int ret;
+
+	mutex_lock(&gpd_list_lock);
+	ret = genpd_add_device(genpd, dev, td);
+	mutex_unlock(&gpd_list_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(__pm_genpd_add_device);
+
+static int genpd_remove_device(struct generic_pm_domain *genpd,
+			       struct device *dev)
 {
 	struct generic_pm_domain_data *gpd_data;
 	struct pm_domain_data *pdd;
@@ -1119,10 +1126,6 @@ int pm_genpd_remove_device(struct generic_pm_domain *genpd,
 
 	dev_dbg(dev, "%s()\n", __func__);
 
-	if (!genpd || genpd != pm_genpd_lookup_dev(dev))
-		return -EINVAL;
-
-	/* The above validation also means we have existing domain_data. */
 	pdd = dev->power.subsys_data->domain_data;
 	gpd_data = to_gpd_data(pdd);
 	dev_pm_qos_remove_notifier(dev, &gpd_data->nb);
@@ -1154,15 +1157,24 @@ int pm_genpd_remove_device(struct generic_pm_domain *genpd,
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(pm_genpd_remove_device);
 
 /**
- * pm_genpd_add_subdomain - Add a subdomain to an I/O PM domain.
- * @genpd: Master PM domain to add the subdomain to.
- * @subdomain: Subdomain to be added.
+ * pm_genpd_remove_device - Remove a device from an I/O PM domain.
+ * @genpd: PM domain to remove the device from.
+ * @dev: Device to be removed.
  */
-int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
-			   struct generic_pm_domain *subdomain)
+int pm_genpd_remove_device(struct generic_pm_domain *genpd,
+			   struct device *dev)
+{
+	if (!genpd || genpd != genpd_lookup_dev(dev))
+		return -EINVAL;
+
+	return genpd_remove_device(genpd, dev);
+}
+EXPORT_SYMBOL_GPL(pm_genpd_remove_device);
+
+static int genpd_add_subdomain(struct generic_pm_domain *genpd,
+			       struct generic_pm_domain *subdomain)
 {
 	struct gpd_link *link, *itr;
 	int ret = 0;
@@ -1205,6 +1217,23 @@ int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
 		kfree(link);
 	return ret;
 }
+
+/**
+ * pm_genpd_add_subdomain - Add a subdomain to an I/O PM domain.
+ * @genpd: Master PM domain to add the subdomain to.
+ * @subdomain: Subdomain to be added.
+ */
+int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
+			   struct generic_pm_domain *subdomain)
+{
+	int ret;
+
+	mutex_lock(&gpd_list_lock);
+	ret = genpd_add_subdomain(genpd, subdomain);
+	mutex_unlock(&gpd_list_lock);
+
+	return ret;
+}
 EXPORT_SYMBOL_GPL(pm_genpd_add_subdomain);
 
 /**
@@ -1278,27 +1307,17 @@ int pm_genpd_init(struct generic_pm_domain *genpd,
 	genpd->device_count = 0;
 	genpd->max_off_time_ns = -1;
 	genpd->max_off_time_changed = true;
+	genpd->provider = NULL;
+	genpd->has_provider = false;
 	genpd->domain.ops.runtime_suspend = genpd_runtime_suspend;
 	genpd->domain.ops.runtime_resume = genpd_runtime_resume;
 	genpd->domain.ops.prepare = pm_genpd_prepare;
-	genpd->domain.ops.suspend = pm_generic_suspend;
-	genpd->domain.ops.suspend_late = pm_generic_suspend_late;
 	genpd->domain.ops.suspend_noirq = pm_genpd_suspend_noirq;
 	genpd->domain.ops.resume_noirq = pm_genpd_resume_noirq;
-	genpd->domain.ops.resume_early = pm_generic_resume_early;
-	genpd->domain.ops.resume = pm_generic_resume;
-	genpd->domain.ops.freeze = pm_generic_freeze;
-	genpd->domain.ops.freeze_late = pm_generic_freeze_late;
 	genpd->domain.ops.freeze_noirq = pm_genpd_freeze_noirq;
 	genpd->domain.ops.thaw_noirq = pm_genpd_thaw_noirq;
-	genpd->domain.ops.thaw_early = pm_generic_thaw_early;
-	genpd->domain.ops.thaw = pm_generic_thaw;
-	genpd->domain.ops.poweroff = pm_generic_poweroff;
-	genpd->domain.ops.poweroff_late = pm_generic_poweroff_late;
 	genpd->domain.ops.poweroff_noirq = pm_genpd_suspend_noirq;
 	genpd->domain.ops.restore_noirq = pm_genpd_restore_noirq;
-	genpd->domain.ops.restore_early = pm_generic_restore_early;
-	genpd->domain.ops.restore = pm_generic_restore;
 	genpd->domain.ops.complete = pm_genpd_complete;
 
 	if (genpd->flags & GENPD_FLAG_PM_CLK) {
@@ -1328,7 +1347,71 @@ int pm_genpd_init(struct generic_pm_domain *genpd,
 }
 EXPORT_SYMBOL_GPL(pm_genpd_init);
 
+static int genpd_remove(struct generic_pm_domain *genpd)
+{
+	struct gpd_link *l, *link;
+
+	if (IS_ERR_OR_NULL(genpd))
+		return -EINVAL;
+
+	mutex_lock(&genpd->lock);
+
+	if (genpd->has_provider) {
+		mutex_unlock(&genpd->lock);
+		pr_err("Provider present, unable to remove %s\n", genpd->name);
+		return -EBUSY;
+	}
+
+	if (!list_empty(&genpd->master_links) || genpd->device_count) {
+		mutex_unlock(&genpd->lock);
+		pr_err("%s: unable to remove %s\n", __func__, genpd->name);
+		return -EBUSY;
+	}
+
+	list_for_each_entry_safe(link, l, &genpd->slave_links, slave_node) {
+		list_del(&link->master_node);
+		list_del(&link->slave_node);
+		kfree(link);
+	}
+
+	list_del(&genpd->gpd_list_node);
+	mutex_unlock(&genpd->lock);
+	cancel_work_sync(&genpd->power_off_work);
+	pr_debug("%s: removed %s\n", __func__, genpd->name);
+
+	return 0;
+}
+
+/**
+ * pm_genpd_remove - Remove a generic I/O PM domain
+ * @genpd: Pointer to PM domain that is to be removed.
+ *
+ * To remove the PM domain, this function:
+ *  - Removes the PM domain as a subdomain to any parent domains,
+ *    if it was added.
+ *  - Removes the PM domain from the list of registered PM domains.
+ *
+ * The PM domain will only be removed, if the associated provider has
+ * been removed, it is not a parent to any other PM domain and has no
+ * devices associated with it.
+ */
+int pm_genpd_remove(struct generic_pm_domain *genpd)
+{
+	int ret;
+
+	mutex_lock(&gpd_list_lock);
+	ret = genpd_remove(genpd);
+	mutex_unlock(&gpd_list_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pm_genpd_remove);
+
 #ifdef CONFIG_PM_GENERIC_DOMAINS_OF
+
+typedef struct generic_pm_domain *(*genpd_xlate_t)(struct of_phandle_args *args,
+						   void *data);
+
 /*
  * Device Tree based PM domain providers.
  *
@@ -1340,8 +1423,8 @@ EXPORT_SYMBOL_GPL(pm_genpd_init);
  * maps a PM domain specifier retrieved from the device tree to a PM domain.
  *
  * Two simple mapping functions have been provided for convenience:
- *  - __of_genpd_xlate_simple() for 1:1 device tree node to PM domain mapping.
- *  - __of_genpd_xlate_onecell() for mapping of multiple PM domains per node by
+ *  - genpd_xlate_simple() for 1:1 device tree node to PM domain mapping.
+ *  - genpd_xlate_onecell() for mapping of multiple PM domains per node by
  *    index.
  */
 
@@ -1366,7 +1449,7 @@ static LIST_HEAD(of_genpd_providers);
 static DEFINE_MUTEX(of_genpd_mutex);
 
 /**
- * __of_genpd_xlate_simple() - Xlate function for direct node-domain mapping
+ * genpd_xlate_simple() - Xlate function for direct node-domain mapping
  * @genpdspec: OF phandle args to map into a PM domain
  * @data: xlate function private data - pointer to struct generic_pm_domain
  *
@@ -1374,7 +1457,7 @@ static DEFINE_MUTEX(of_genpd_mutex);
  * have their own device tree nodes. The private data of xlate function needs
  * to be a valid pointer to struct generic_pm_domain.
  */
-struct generic_pm_domain *__of_genpd_xlate_simple(
+static struct generic_pm_domain *genpd_xlate_simple(
 					struct of_phandle_args *genpdspec,
 					void *data)
 {
@@ -1382,10 +1465,9 @@ struct generic_pm_domain *__of_genpd_xlate_simple(
 		return ERR_PTR(-EINVAL);
 	return data;
 }
-EXPORT_SYMBOL_GPL(__of_genpd_xlate_simple);
 
 /**
- * __of_genpd_xlate_onecell() - Xlate function using a single index.
+ * genpd_xlate_onecell() - Xlate function using a single index.
  * @genpdspec: OF phandle args to map into a PM domain
  * @data: xlate function private data - pointer to struct genpd_onecell_data
  *
@@ -1394,7 +1476,7 @@ EXPORT_SYMBOL_GPL(__of_genpd_xlate_simple);
  * A single cell is used as an index into an array of PM domains specified in
  * the genpd_onecell_data struct when registering the provider.
  */
-struct generic_pm_domain *__of_genpd_xlate_onecell(
+static struct generic_pm_domain *genpd_xlate_onecell(
 					struct of_phandle_args *genpdspec,
 					void *data)
 {
@@ -1414,16 +1496,15 @@ struct generic_pm_domain *__of_genpd_xlate_onecell(
 
 	return genpd_data->domains[idx];
 }
-EXPORT_SYMBOL_GPL(__of_genpd_xlate_onecell);
 
 /**
- * __of_genpd_add_provider() - Register a PM domain provider for a node
+ * genpd_add_provider() - Register a PM domain provider for a node
  * @np: Device node pointer associated with the PM domain provider.
  * @xlate: Callback for decoding PM domain from phandle arguments.
  * @data: Context pointer for @xlate callback.
  */
-int __of_genpd_add_provider(struct device_node *np, genpd_xlate_t xlate,
-			void *data)
+static int genpd_add_provider(struct device_node *np, genpd_xlate_t xlate,
+			      void *data)
 {
 	struct of_genpd_provider *cp;
 
@@ -1442,7 +1523,83 @@ int __of_genpd_add_provider(struct device_node *np, genpd_xlate_t xlate,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(__of_genpd_add_provider);
+
+/**
+ * of_genpd_add_provider_simple() - Register a simple PM domain provider
+ * @np: Device node pointer associated with the PM domain provider.
+ * @genpd: Pointer to PM domain associated with the PM domain provider.
+ */
+int of_genpd_add_provider_simple(struct device_node *np,
+				 struct generic_pm_domain *genpd)
+{
+	int ret = -EINVAL;
+
+	if (!np || !genpd)
+		return -EINVAL;
+
+	mutex_lock(&gpd_list_lock);
+
+	if (pm_genpd_present(genpd))
+		ret = genpd_add_provider(np, genpd_xlate_simple, genpd);
+
+	if (!ret) {
+		genpd->provider = &np->fwnode;
+		genpd->has_provider = true;
+	}
+
+	mutex_unlock(&gpd_list_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(of_genpd_add_provider_simple);
+
+/**
+ * of_genpd_add_provider_onecell() - Register a onecell PM domain provider
+ * @np: Device node pointer associated with the PM domain provider.
+ * @data: Pointer to the data associated with the PM domain provider.
+ */
+int of_genpd_add_provider_onecell(struct device_node *np,
+				  struct genpd_onecell_data *data)
+{
+	unsigned int i;
+	int ret = -EINVAL;
+
+	if (!np || !data)
+		return -EINVAL;
+
+	mutex_lock(&gpd_list_lock);
+
+	for (i = 0; i < data->num_domains; i++) {
+		if (!data->domains[i])
+			continue;
+		if (!pm_genpd_present(data->domains[i]))
+			goto error;
+
+		data->domains[i]->provider = &np->fwnode;
+		data->domains[i]->has_provider = true;
+	}
+
+	ret = genpd_add_provider(np, genpd_xlate_onecell, data);
+	if (ret < 0)
+		goto error;
+
+	mutex_unlock(&gpd_list_lock);
+
+	return 0;
+
+error:
+	while (i--) {
+		if (!data->domains[i])
+			continue;
+		data->domains[i]->provider = NULL;
+		data->domains[i]->has_provider = false;
+	}
+
+	mutex_unlock(&gpd_list_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(of_genpd_add_provider_onecell);
 
 /**
  * of_genpd_del_provider() - Remove a previously registered PM domain provider
@@ -1451,10 +1608,21 @@ EXPORT_SYMBOL_GPL(__of_genpd_add_provider);
 void of_genpd_del_provider(struct device_node *np)
 {
 	struct of_genpd_provider *cp;
+	struct generic_pm_domain *gpd;
 
+	mutex_lock(&gpd_list_lock);
 	mutex_lock(&of_genpd_mutex);
 	list_for_each_entry(cp, &of_genpd_providers, link) {
 		if (cp->node == np) {
+			/*
+			 * For each PM domain associated with the
+			 * provider, set the 'has_provider' to false
+			 * so that the PM domain can be safely removed.
+			 */
+			list_for_each_entry(gpd, &gpd_list, gpd_list_node)
+				if (gpd->provider == &np->fwnode)
+					gpd->has_provider = false;
+
 			list_del(&cp->link);
 			of_node_put(cp->node);
 			kfree(cp);
@@ -1462,11 +1630,12 @@ void of_genpd_del_provider(struct device_node *np)
 		}
 	}
 	mutex_unlock(&of_genpd_mutex);
+	mutex_unlock(&gpd_list_lock);
 }
 EXPORT_SYMBOL_GPL(of_genpd_del_provider);
 
 /**
- * of_genpd_get_from_provider() - Look-up PM domain
+ * genpd_get_from_provider() - Look-up PM domain
  * @genpdspec: OF phandle args to use for look-up
  *
  * Looks for a PM domain provider under the node specified by @genpdspec and if
@@ -1476,7 +1645,7 @@ EXPORT_SYMBOL_GPL(of_genpd_del_provider);
  * Returns a valid pointer to struct generic_pm_domain on success or ERR_PTR()
  * on failure.
  */
-struct generic_pm_domain *of_genpd_get_from_provider(
+static struct generic_pm_domain *genpd_get_from_provider(
 					struct of_phandle_args *genpdspec)
 {
 	struct generic_pm_domain *genpd = ERR_PTR(-ENOENT);
@@ -1499,7 +1668,109 @@ struct generic_pm_domain *of_genpd_get_from_provider(
 
 	return genpd;
 }
-EXPORT_SYMBOL_GPL(of_genpd_get_from_provider);
+
+/**
+ * of_genpd_add_device() - Add a device to an I/O PM domain
+ * @genpdspec: OF phandle args to use for look-up PM domain
+ * @dev: Device to be added.
+ *
+ * Looks-up an I/O PM domain based upon phandle args provided and adds
+ * the device to the PM domain. Returns a negative error code on failure.
+ */
+int of_genpd_add_device(struct of_phandle_args *genpdspec, struct device *dev)
+{
+	struct generic_pm_domain *genpd;
+	int ret;
+
+	mutex_lock(&gpd_list_lock);
+
+	genpd = genpd_get_from_provider(genpdspec);
+	if (IS_ERR(genpd)) {
+		ret = PTR_ERR(genpd);
+		goto out;
+	}
+
+	ret = genpd_add_device(genpd, dev, NULL);
+
+out:
+	mutex_unlock(&gpd_list_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(of_genpd_add_device);
+
+/**
+ * of_genpd_add_subdomain - Add a subdomain to an I/O PM domain.
+ * @parent_spec: OF phandle args to use for parent PM domain look-up
+ * @subdomain_spec: OF phandle args to use for subdomain look-up
+ *
+ * Looks-up a parent PM domain and subdomain based upon phandle args
+ * provided and adds the subdomain to the parent PM domain. Returns a
+ * negative error code on failure.
+ */
+int of_genpd_add_subdomain(struct of_phandle_args *parent_spec,
+			   struct of_phandle_args *subdomain_spec)
+{
+	struct generic_pm_domain *parent, *subdomain;
+	int ret;
+
+	mutex_lock(&gpd_list_lock);
+
+	parent = genpd_get_from_provider(parent_spec);
+	if (IS_ERR(parent)) {
+		ret = PTR_ERR(parent);
+		goto out;
+	}
+
+	subdomain = genpd_get_from_provider(subdomain_spec);
+	if (IS_ERR(subdomain)) {
+		ret = PTR_ERR(subdomain);
+		goto out;
+	}
+
+	ret = genpd_add_subdomain(parent, subdomain);
+
+out:
+	mutex_unlock(&gpd_list_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(of_genpd_add_subdomain);
+
+/**
+ * of_genpd_remove_last - Remove the last PM domain registered for a provider
+ * @provider: Pointer to device structure associated with provider
+ *
+ * Find the last PM domain that was added by a particular provider and
+ * remove this PM domain from the list of PM domains. The provider is
+ * identified by the 'provider' device structure that is passed. The PM
+ * domain will only be removed, if the provider associated with domain
+ * has been removed.
+ *
+ * Returns a valid pointer to struct generic_pm_domain on success or
+ * ERR_PTR() on failure.
+ */
+struct generic_pm_domain *of_genpd_remove_last(struct device_node *np)
+{
+	struct generic_pm_domain *gpd, *genpd = ERR_PTR(-ENOENT);
+	int ret;
+
+	if (IS_ERR_OR_NULL(np))
+		return ERR_PTR(-EINVAL);
+
+	mutex_lock(&gpd_list_lock);
+	list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
+		if (gpd->provider == &np->fwnode) {
+			ret = genpd_remove(gpd);
+			genpd = ret ? ERR_PTR(ret) : gpd;
+			break;
+		}
+	}
+	mutex_unlock(&gpd_list_lock);
+
+	return genpd;
+}
+EXPORT_SYMBOL_GPL(of_genpd_remove_last);
 
 /**
  * genpd_dev_pm_detach - Detach a device from its PM domain.
@@ -1515,14 +1786,14 @@ static void genpd_dev_pm_detach(struct device *dev, bool power_off)
 	unsigned int i;
 	int ret = 0;
 
-	pd = pm_genpd_lookup_dev(dev);
-	if (!pd)
+	pd = dev_to_genpd(dev);
+	if (IS_ERR(pd))
 		return;
 
 	dev_dbg(dev, "removing from PM domain %s\n", pd->name);
 
 	for (i = 1; i < GENPD_RETRY_MAX_MS; i <<= 1) {
-		ret = pm_genpd_remove_device(pd, dev);
+		ret = genpd_remove_device(pd, dev);
 		if (ret != -EAGAIN)
 			break;
 
@@ -1596,9 +1867,11 @@ int genpd_dev_pm_attach(struct device *dev)
 			return -ENOENT;
 	}
 
-	pd = of_genpd_get_from_provider(&pd_args);
+	mutex_lock(&gpd_list_lock);
+	pd = genpd_get_from_provider(&pd_args);
 	of_node_put(pd_args.np);
 	if (IS_ERR(pd)) {
+		mutex_unlock(&gpd_list_lock);
 		dev_dbg(dev, "%s() failed to find PM domain: %ld\n",
 			__func__, PTR_ERR(pd));
 		return -EPROBE_DEFER;
@@ -1607,13 +1880,14 @@ int genpd_dev_pm_attach(struct device *dev)
 	dev_dbg(dev, "adding to PM domain %s\n", pd->name);
 
 	for (i = 1; i < GENPD_RETRY_MAX_MS; i <<= 1) {
-		ret = pm_genpd_add_device(pd, dev);
+		ret = genpd_add_device(pd, dev, NULL);
 		if (ret != -EAGAIN)
 			break;
 
 		mdelay(i);
 		cond_resched();
 	}
+	mutex_unlock(&gpd_list_lock);
 
 	if (ret < 0) {
 		dev_err(dev, "failed to add to PM domain %s: %d",
@@ -1636,7 +1910,7 @@ EXPORT_SYMBOL_GPL(genpd_dev_pm_attach);
 
 /***        debugfs support        ***/
 
-#ifdef CONFIG_PM_ADVANCED_DEBUG
+#ifdef CONFIG_DEBUG_FS
 #include <linux/pm.h>
 #include <linux/device.h>
 #include <linux/debugfs.h>
@@ -1784,4 +2058,4 @@ static void __exit pm_genpd_debug_exit(void)
 	debugfs_remove_recursive(pm_genpd_debugfs_dir);
 }
 __exitcall(pm_genpd_debug_exit);
-#endif /* CONFIG_PM_ADVANCED_DEBUG */
+#endif /* CONFIG_DEBUG_FS */
diff --git a/drivers/base/power/opp/core.c b/drivers/base/power/opp/core.c
index df0c70963d9e..4c7c6da7a989 100644
--- a/drivers/base/power/opp/core.c
+++ b/drivers/base/power/opp/core.c
@@ -584,7 +584,6 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq)
 	struct clk *clk;
 	unsigned long freq, old_freq;
 	unsigned long u_volt, u_volt_min, u_volt_max;
-	unsigned long ou_volt, ou_volt_min, ou_volt_max;
 	int ret;
 
 	if (unlikely(!target_freq)) {
@@ -620,11 +619,7 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq)
 	}
 
 	old_opp = _find_freq_ceil(opp_table, &old_freq);
-	if (!IS_ERR(old_opp)) {
-		ou_volt = old_opp->u_volt;
-		ou_volt_min = old_opp->u_volt_min;
-		ou_volt_max = old_opp->u_volt_max;
-	} else {
+	if (IS_ERR(old_opp)) {
 		dev_err(dev, "%s: failed to find current OPP for freq %lu (%ld)\n",
 			__func__, old_freq, PTR_ERR(old_opp));
 	}
@@ -683,7 +678,8 @@ restore_freq:
 restore_voltage:
 	/* This shouldn't harm even if the voltages weren't updated earlier */
 	if (!IS_ERR(old_opp))
-		_set_opp_voltage(dev, reg, ou_volt, ou_volt_min, ou_volt_max);
+		_set_opp_voltage(dev, reg, old_opp->u_volt,
+				 old_opp->u_volt_min, old_opp->u_volt_max);
 
 	return ret;
 }
diff --git a/drivers/base/power/opp/of.c b/drivers/base/power/opp/of.c
index 1dfd3dd92624..5552211e6fcd 100644
--- a/drivers/base/power/opp/of.c
+++ b/drivers/base/power/opp/of.c
@@ -71,8 +71,18 @@ static bool _opp_is_supported(struct device *dev, struct opp_table *opp_table,
 	u32 version;
 	int ret;
 
-	if (!opp_table->supported_hw)
-		return true;
+	if (!opp_table->supported_hw) {
+		/*
+		 * In the case that no supported_hw has been set by the
+		 * platform but there is an opp-supported-hw value set for
+		 * an OPP then the OPP should not be enabled as there is
+		 * no way to see if the hardware supports it.
+		 */
+		if (of_find_property(np, "opp-supported-hw", NULL))
+			return false;
+		else
+			return true;
+	}
 
 	while (count--) {
 		ret = of_property_read_u32_index(np, "opp-supported-hw", count,
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index e097d355cc04..82a081ea4317 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -301,7 +301,7 @@ static int rpm_idle(struct device *dev, int rpmflags)
 	int (*callback)(struct device *);
 	int retval;
 
-	trace_rpm_idle(dev, rpmflags);
+	trace_rpm_idle_rcuidle(dev, rpmflags);
 	retval = rpm_check_suspend_allowed(dev);
 	if (retval < 0)
 		;	/* Conditions are wrong. */
@@ -337,7 +337,7 @@ static int rpm_idle(struct device *dev, int rpmflags)
 			dev->power.request_pending = true;
 			queue_work(pm_wq, &dev->power.work);
 		}
-		trace_rpm_return_int(dev, _THIS_IP_, 0);
+		trace_rpm_return_int_rcuidle(dev, _THIS_IP_, 0);
 		return 0;
 	}
 
@@ -352,7 +352,7 @@ static int rpm_idle(struct device *dev, int rpmflags)
 	wake_up_all(&dev->power.wait_queue);
 
  out:
-	trace_rpm_return_int(dev, _THIS_IP_, retval);
+	trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
 	return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO);
 }
 
@@ -419,7 +419,7 @@ static int rpm_suspend(struct device *dev, int rpmflags)
 	struct device *parent = NULL;
 	int retval;
 
-	trace_rpm_suspend(dev, rpmflags);
+	trace_rpm_suspend_rcuidle(dev, rpmflags);
 
  repeat:
 	retval = rpm_check_suspend_allowed(dev);
@@ -549,7 +549,7 @@ static int rpm_suspend(struct device *dev, int rpmflags)
 	}
 
  out:
-	trace_rpm_return_int(dev, _THIS_IP_, retval);
+	trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
 
 	return retval;
 
@@ -601,7 +601,7 @@ static int rpm_resume(struct device *dev, int rpmflags)
 	struct device *parent = NULL;
 	int retval = 0;
 
-	trace_rpm_resume(dev, rpmflags);
+	trace_rpm_resume_rcuidle(dev, rpmflags);
 
  repeat:
 	if (dev->power.runtime_error)
@@ -764,7 +764,7 @@ static int rpm_resume(struct device *dev, int rpmflags)
 		spin_lock_irq(&dev->power.lock);
 	}
 
-	trace_rpm_return_int(dev, _THIS_IP_, retval);
+	trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
 
 	return retval;
 }
diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h
index a0380338946a..2a4435d76028 100644
--- a/drivers/base/regmap/internal.h
+++ b/drivers/base/regmap/internal.h
@@ -105,8 +105,8 @@ struct regmap {
 
 	bool defer_caching;
 
-	u8 read_flag_mask;
-	u8 write_flag_mask;
+	unsigned long read_flag_mask;
+	unsigned long write_flag_mask;
 
 	/* number of bits to (left) shift the reg value when formatting*/
 	int reg_shift;
@@ -173,6 +173,7 @@ struct regcache_ops {
 	int (*drop)(struct regmap *map, unsigned int min, unsigned int max);
 };
 
+bool regmap_cached(struct regmap *map, unsigned int reg);
 bool regmap_writeable(struct regmap *map, unsigned int reg);
 bool regmap_readable(struct regmap *map, unsigned int reg);
 bool regmap_volatile(struct regmap *map, unsigned int reg);
diff --git a/drivers/base/regmap/regcache-rbtree.c b/drivers/base/regmap/regcache-rbtree.c
index aa56af87d941..b11af3f2c1db 100644
--- a/drivers/base/regmap/regcache-rbtree.c
+++ b/drivers/base/regmap/regcache-rbtree.c
@@ -404,6 +404,7 @@ static int regcache_rbtree_write(struct regmap *map, unsigned int reg,
 		unsigned int new_base_reg, new_top_reg;
 		unsigned int min, max;
 		unsigned int max_dist;
+		unsigned int dist, best_dist = UINT_MAX;
 
 		max_dist = map->reg_stride * sizeof(*rbnode_tmp) /
 			map->cache_word_size;
@@ -423,24 +424,41 @@ static int regcache_rbtree_write(struct regmap *map, unsigned int reg,
 				&base_reg, &top_reg);
 
 			if (base_reg <= max && top_reg >= min) {
-				new_base_reg = min(reg, base_reg);
-				new_top_reg = max(reg, top_reg);
-			} else {
-				if (max < base_reg)
-					node = node->rb_left;
+				if (reg < base_reg)
+					dist = base_reg - reg;
+				else if (reg > top_reg)
+					dist = reg - top_reg;
 				else
-					node = node->rb_right;
-
-				continue;
+					dist = 0;
+				if (dist < best_dist) {
+					rbnode = rbnode_tmp;
+					best_dist = dist;
+					new_base_reg = min(reg, base_reg);
+					new_top_reg = max(reg, top_reg);
+				}
 			}
 
-			ret = regcache_rbtree_insert_to_block(map, rbnode_tmp,
+			/*
+			 * Keep looking, we want to choose the closest block,
+			 * otherwise we might end up creating overlapping
+			 * blocks, which breaks the rbtree.
+			 */
+			if (reg < base_reg)
+				node = node->rb_left;
+			else if (reg > top_reg)
+				node = node->rb_right;
+			else
+				break;
+		}
+
+		if (rbnode) {
+			ret = regcache_rbtree_insert_to_block(map, rbnode,
 							      new_base_reg,
 							      new_top_reg, reg,
 							      value);
 			if (ret)
 				return ret;
-			rbtree_ctx->cached_rbnode = rbnode_tmp;
+			rbtree_ctx->cached_rbnode = rbnode;
 			return 0;
 		}
 
diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c
index df7ff7290821..4e582561e1e7 100644
--- a/drivers/base/regmap/regcache.c
+++ b/drivers/base/regmap/regcache.c
@@ -38,10 +38,11 @@ static int regcache_hw_init(struct regmap *map)
 
 	/* calculate the size of reg_defaults */
 	for (count = 0, i = 0; i < map->num_reg_defaults_raw; i++)
-		if (!regmap_volatile(map, i * map->reg_stride))
+		if (regmap_readable(map, i * map->reg_stride) &&
+		    !regmap_volatile(map, i * map->reg_stride))
 			count++;
 
-	/* all registers are volatile, so just bypass */
+	/* all registers are unreadable or volatile, so just bypass */
 	if (!count) {
 		map->cache_bypass = true;
 		return 0;
diff --git a/drivers/base/regmap/regmap-debugfs.c b/drivers/base/regmap/regmap-debugfs.c
index 1ee3d40861c7..36ce3511c733 100644
--- a/drivers/base/regmap/regmap-debugfs.c
+++ b/drivers/base/regmap/regmap-debugfs.c
@@ -77,6 +77,17 @@ static void regmap_debugfs_free_dump_cache(struct regmap *map)
 	}
 }
 
+static bool regmap_printable(struct regmap *map, unsigned int reg)
+{
+	if (regmap_precious(map, reg))
+		return false;
+
+	if (!regmap_readable(map, reg) && !regmap_cached(map, reg))
+		return false;
+
+	return true;
+}
+
 /*
  * Work out where the start offset maps into register numbers, bearing
  * in mind that we suppress hidden registers.
@@ -105,8 +116,7 @@ static unsigned int regmap_debugfs_get_dump_start(struct regmap *map,
 	if (list_empty(&map->debugfs_off_cache)) {
 		for (; i <= map->max_register; i += map->reg_stride) {
 			/* Skip unprinted registers, closing off cache entry */
-			if (!regmap_readable(map, i) ||
-			    regmap_precious(map, i)) {
+			if (!regmap_printable(map, i)) {
 				if (c) {
 					c->max = p - 1;
 					c->max_reg = i - map->reg_stride;
@@ -204,7 +214,7 @@ static ssize_t regmap_read_debugfs(struct regmap *map, unsigned int from,
 	start_reg = regmap_debugfs_get_dump_start(map, from, *ppos, &p);
 
 	for (i = start_reg; i <= to; i += map->reg_stride) {
-		if (!regmap_readable(map, i))
+		if (!regmap_readable(map, i) && !regmap_cached(map, i))
 			continue;
 
 		if (regmap_precious(map, i))
diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c
index 51fa7d66a393..ae63bb0875ea 100644
--- a/drivers/base/regmap/regmap.c
+++ b/drivers/base/regmap/regmap.c
@@ -93,6 +93,29 @@ bool regmap_writeable(struct regmap *map, unsigned int reg)
 	return true;
 }
 
+bool regmap_cached(struct regmap *map, unsigned int reg)
+{
+	int ret;
+	unsigned int val;
+
+	if (map->cache == REGCACHE_NONE)
+		return false;
+
+	if (!map->cache_ops)
+		return false;
+
+	if (map->max_register && reg > map->max_register)
+		return false;
+
+	map->lock(map->lock_arg);
+	ret = regcache_read(map, reg, &val);
+	map->unlock(map->lock_arg);
+	if (ret)
+		return false;
+
+	return true;
+}
+
 bool regmap_readable(struct regmap *map, unsigned int reg)
 {
 	if (!map->reg_read)
@@ -749,6 +772,9 @@ struct regmap *__regmap_init(struct device *dev,
 		case REGMAP_ENDIAN_BIG:
 			map->format.format_reg = regmap_format_16_be;
 			break;
+		case REGMAP_ENDIAN_LITTLE:
+			map->format.format_reg = regmap_format_16_le;
+			break;
 		case REGMAP_ENDIAN_NATIVE:
 			map->format.format_reg = regmap_format_16_native;
 			break;
@@ -768,6 +794,9 @@ struct regmap *__regmap_init(struct device *dev,
 		case REGMAP_ENDIAN_BIG:
 			map->format.format_reg = regmap_format_32_be;
 			break;
+		case REGMAP_ENDIAN_LITTLE:
+			map->format.format_reg = regmap_format_32_le;
+			break;
 		case REGMAP_ENDIAN_NATIVE:
 			map->format.format_reg = regmap_format_32_native;
 			break;
@@ -782,6 +811,9 @@ struct regmap *__regmap_init(struct device *dev,
 		case REGMAP_ENDIAN_BIG:
 			map->format.format_reg = regmap_format_64_be;
 			break;
+		case REGMAP_ENDIAN_LITTLE:
+			map->format.format_reg = regmap_format_64_le;
+			break;
 		case REGMAP_ENDIAN_NATIVE:
 			map->format.format_reg = regmap_format_64_native;
 			break;
@@ -1296,12 +1328,26 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg,
 	return 0;
 }
 
+static void regmap_set_work_buf_flag_mask(struct regmap *map, int max_bytes,
+					  unsigned long mask)
+{
+	u8 *buf;
+	int i;
+
+	if (!mask || !map->work_buf)
+		return;
+
+	buf = map->work_buf;
+
+	for (i = 0; i < max_bytes; i++)
+		buf[i] |= (mask >> (8 * i)) & 0xff;
+}
+
 int _regmap_raw_write(struct regmap *map, unsigned int reg,
 		      const void *val, size_t val_len)
 {
 	struct regmap_range_node *range;
 	unsigned long flags;
-	u8 *u8 = map->work_buf;
 	void *work_val = map->work_buf + map->format.reg_bytes +
 		map->format.pad_bytes;
 	void *buf;
@@ -1370,8 +1416,8 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg,
 	}
 
 	map->format.format_reg(map->work_buf, reg, map->reg_shift);
-
-	u8[0] |= map->write_flag_mask;
+	regmap_set_work_buf_flag_mask(map, map->format.reg_bytes,
+				      map->write_flag_mask);
 
 	/*
 	 * Essentially all I/O mechanisms will be faster with a single
@@ -1474,6 +1520,12 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg,
 		ret = map->bus->write(map->bus_context, buf, len);
 
 		kfree(buf);
+	} else if (ret != 0 && !map->cache_bypass && map->format.parse_val) {
+		/* regcache_drop_region() takes lock that we already have,
+		 * thus call map->cache_ops->drop() directly
+		 */
+		if (map->cache_ops && map->cache_ops->drop)
+			map->cache_ops->drop(map, reg, reg + 1);
 	}
 
 	trace_regmap_hw_write_done(map, reg, val_len / map->format.val_bytes);
@@ -2245,7 +2297,6 @@ static int _regmap_raw_read(struct regmap *map, unsigned int reg, void *val,
 			    unsigned int val_len)
 {
 	struct regmap_range_node *range;
-	u8 *u8 = map->work_buf;
 	int ret;
 
 	WARN_ON(!map->bus);
@@ -2262,15 +2313,8 @@ static int _regmap_raw_read(struct regmap *map, unsigned int reg, void *val,
 	}
 
 	map->format.format_reg(map->work_buf, reg, map->reg_shift);
-
-	/*
-	 * Some buses or devices flag reads by setting the high bits in the
-	 * register address; since it's always the high bits for all
-	 * current formats we can do this here rather than in
-	 * formatting.  This may break if we get interesting formats.
-	 */
-	u8[0] |= map->read_flag_mask;
-
+	regmap_set_work_buf_flag_mask(map, map->format.reg_bytes,
+				      map->read_flag_mask);
 	trace_regmap_hw_read_start(map, reg, val_len / map->format.val_bytes);
 
 	ret = map->bus->read(map->bus_context, map->work_buf,
diff --git a/drivers/base/soc.c b/drivers/base/soc.c
index 75b98aad6faf..b63f23e6ad61 100644
--- a/drivers/base/soc.c
+++ b/drivers/base/soc.c
@@ -6,7 +6,6 @@
  */
 
 #include <linux/sysfs.h>
-#include <linux/module.h>
 #include <linux/init.h>
 #include <linux/stat.h>
 #include <linux/slab.h>
@@ -160,11 +159,3 @@ static int __init soc_bus_register(void)
 	return bus_register(&soc_bus_type);
 }
 core_initcall(soc_bus_register);
-
-static void __exit soc_bus_unregister(void)
-{
-	ida_destroy(&soc_ida);
-
-	bus_unregister(&soc_bus_type);
-}
-module_exit(soc_bus_unregister);
diff --git a/drivers/bcma/driver_chipcommon.c b/drivers/bcma/driver_chipcommon.c
index 921ce1834673..b4f6520e74f0 100644
--- a/drivers/bcma/driver_chipcommon.c
+++ b/drivers/bcma/driver_chipcommon.c
@@ -36,12 +36,31 @@ u32 bcma_chipco_get_alp_clock(struct bcma_drv_cc *cc)
 }
 EXPORT_SYMBOL_GPL(bcma_chipco_get_alp_clock);
 
+static bool bcma_core_cc_has_pmu_watchdog(struct bcma_drv_cc *cc)
+{
+	struct bcma_bus *bus = cc->core->bus;
+
+	if (cc->capabilities & BCMA_CC_CAP_PMU) {
+		if (bus->chipinfo.id == BCMA_CHIP_ID_BCM53573) {
+			WARN(bus->chipinfo.rev <= 1, "No watchdog available\n");
+			/* 53573B0 and 53573B1 have bugged PMU watchdog. It can
+			 * be enabled but timer can't be bumped. Use CC one
+			 * instead.
+			 */
+			return false;
+		}
+		return true;
+	} else {
+		return false;
+	}
+}
+
 static u32 bcma_chipco_watchdog_get_max_timer(struct bcma_drv_cc *cc)
 {
 	struct bcma_bus *bus = cc->core->bus;
 	u32 nb;
 
-	if (cc->capabilities & BCMA_CC_CAP_PMU) {
+	if (bcma_core_cc_has_pmu_watchdog(cc)) {
 		if (bus->chipinfo.id == BCMA_CHIP_ID_BCM4706)
 			nb = 32;
 		else if (cc->core->id.rev < 26)
@@ -95,9 +114,16 @@ static int bcma_chipco_watchdog_ticks_per_ms(struct bcma_drv_cc *cc)
 
 int bcma_chipco_watchdog_register(struct bcma_drv_cc *cc)
 {
+	struct bcma_bus *bus = cc->core->bus;
 	struct bcm47xx_wdt wdt = {};
 	struct platform_device *pdev;
 
+	if (bus->chipinfo.id == BCMA_CHIP_ID_BCM53573 &&
+	    bus->chipinfo.rev <= 1) {
+		pr_debug("No watchdog on 53573A0 / 53573A1\n");
+		return 0;
+	}
+
 	wdt.driver_data = cc;
 	wdt.timer_set = bcma_chipco_watchdog_timer_set_wdt;
 	wdt.timer_set_ms = bcma_chipco_watchdog_timer_set_ms_wdt;
@@ -105,7 +131,7 @@ int bcma_chipco_watchdog_register(struct bcma_drv_cc *cc)
 		bcma_chipco_watchdog_get_max_timer(cc) / cc->ticks_per_ms;
 
 	pdev = platform_device_register_data(NULL, "bcm47xx-wdt",
-					     cc->core->bus->num, &wdt,
+					     bus->num, &wdt,
 					     sizeof(wdt));
 	if (IS_ERR(pdev))
 		return PTR_ERR(pdev);
@@ -217,7 +243,7 @@ u32 bcma_chipco_watchdog_timer_set(struct bcma_drv_cc *cc, u32 ticks)
 	u32 maxt;
 
 	maxt = bcma_chipco_watchdog_get_max_timer(cc);
-	if (cc->capabilities & BCMA_CC_CAP_PMU) {
+	if (bcma_core_cc_has_pmu_watchdog(cc)) {
 		if (ticks == 1)
 			ticks = 2;
 		else if (ticks > maxt)
diff --git a/drivers/bcma/main.c b/drivers/bcma/main.c
index 1f635471f318..2c1798e38abd 100644
--- a/drivers/bcma/main.c
+++ b/drivers/bcma/main.c
@@ -209,6 +209,8 @@ static void bcma_of_fill_device(struct platform_device *parent,
 		core->dev.of_node = node;
 
 	core->irq = bcma_of_get_irq(parent, core, 0);
+
+	of_dma_configure(&core->dev, node);
 }
 
 unsigned int bcma_core_irq(struct bcma_device *core, int num)
@@ -248,12 +250,12 @@ void bcma_prepare_core(struct bcma_bus *bus, struct bcma_device *core)
 		core->irq = bus->host_pci->irq;
 		break;
 	case BCMA_HOSTTYPE_SOC:
-		core->dev.dma_mask = &core->dev.coherent_dma_mask;
-		if (bus->host_pdev) {
+		if (IS_ENABLED(CONFIG_OF) && bus->host_pdev) {
 			core->dma_dev = &bus->host_pdev->dev;
 			core->dev.parent = &bus->host_pdev->dev;
 			bcma_of_fill_device(bus->host_pdev, core);
 		} else {
+			core->dev.dma_mask = &core->dev.coherent_dma_mask;
 			core->dma_dev = &core->dev;
 		}
 		break;
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
index b71a9c767009..e3d8e4ced4a2 100644
--- a/drivers/block/floppy.c
+++ b/drivers/block/floppy.c
@@ -3706,22 +3706,21 @@ static int floppy_open(struct block_device *bdev, fmode_t mode)
 	if (UFDCS->rawcmd == 1)
 		UFDCS->rawcmd = 2;
 
-	if (mode & (FMODE_READ|FMODE_WRITE)) {
-		UDRS->last_checked = 0;
-		clear_bit(FD_OPEN_SHOULD_FAIL_BIT, &UDRS->flags);
-		check_disk_change(bdev);
-		if (test_bit(FD_DISK_CHANGED_BIT, &UDRS->flags))
-			goto out;
-		if (test_bit(FD_OPEN_SHOULD_FAIL_BIT, &UDRS->flags))
+	if (!(mode & FMODE_NDELAY)) {
+		if (mode & (FMODE_READ|FMODE_WRITE)) {
+			UDRS->last_checked = 0;
+			clear_bit(FD_OPEN_SHOULD_FAIL_BIT, &UDRS->flags);
+			check_disk_change(bdev);
+			if (test_bit(FD_DISK_CHANGED_BIT, &UDRS->flags))
+				goto out;
+			if (test_bit(FD_OPEN_SHOULD_FAIL_BIT, &UDRS->flags))
+				goto out;
+		}
+		res = -EROFS;
+		if ((mode & FMODE_WRITE) &&
+		    !test_bit(FD_DISK_WRITABLE_BIT, &UDRS->flags))
 			goto out;
 	}
-
-	res = -EROFS;
-
-	if ((mode & FMODE_WRITE) &&
-			!test_bit(FD_DISK_WRITABLE_BIT, &UDRS->flags))
-		goto out;
-
 	mutex_unlock(&open_lock);
 	mutex_unlock(&floppy_mutex);
 	return 0;
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index c9f2107f7095..fa1b7a90ba11 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -840,13 +840,13 @@ static void loop_config_discard(struct loop_device *lo)
 
 static void loop_unprepare_queue(struct loop_device *lo)
 {
-	flush_kthread_worker(&lo->worker);
+	kthread_flush_worker(&lo->worker);
 	kthread_stop(lo->worker_task);
 }
 
 static int loop_prepare_queue(struct loop_device *lo)
 {
-	init_kthread_worker(&lo->worker);
+	kthread_init_worker(&lo->worker);
 	lo->worker_task = kthread_run(kthread_worker_fn,
 			&lo->worker, "loop%d", lo->lo_number);
 	if (IS_ERR(lo->worker_task))
@@ -1658,7 +1658,7 @@ static int loop_queue_rq(struct blk_mq_hw_ctx *hctx,
 		break;
 	}
 
-	queue_kthread_work(&lo->worker, &cmd->work);
+	kthread_queue_work(&lo->worker, &cmd->work);
 
 	return BLK_MQ_RQ_QUEUE_OK;
 }
@@ -1696,14 +1696,13 @@ static int loop_init_request(void *data, struct request *rq,
 	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
 
 	cmd->rq = rq;
-	init_kthread_work(&cmd->work, loop_queue_work);
+	kthread_init_work(&cmd->work, loop_queue_work);
 
 	return 0;
 }
 
 static struct blk_mq_ops loop_mq_ops = {
 	.queue_rq       = loop_queue_rq,
-	.map_queue      = blk_mq_map_queue,
 	.init_request	= loop_init_request,
 };
 
diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c
index 2aca98e8e427..3cfd879267b2 100644
--- a/drivers/block/mtip32xx/mtip32xx.c
+++ b/drivers/block/mtip32xx/mtip32xx.c
@@ -3686,7 +3686,7 @@ static int mtip_block_open(struct block_device *dev, fmode_t mode)
 	return -ENODEV;
 }
 
-void mtip_block_release(struct gendisk *disk, fmode_t mode)
+static void mtip_block_release(struct gendisk *disk, fmode_t mode)
 {
 }
 
@@ -3895,7 +3895,6 @@ exit_handler:
 
 static struct blk_mq_ops mtip_mq_ops = {
 	.queue_rq	= mtip_queue_rq,
-	.map_queue	= blk_mq_map_queue,
 	.init_request	= mtip_init_cmd,
 	.exit_request	= mtip_free_cmd,
 	.complete	= mtip_softirq_done_fn,
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index a9e398019f38..ba405b55329f 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -34,33 +34,29 @@
 #include <linux/kthread.h>
 #include <linux/types.h>
 #include <linux/debugfs.h>
+#include <linux/blk-mq.h>
 
 #include <asm/uaccess.h>
 #include <asm/types.h>
 
 #include <linux/nbd.h>
 
+#define NBD_TIMEDOUT			0
+#define NBD_DISCONNECT_REQUESTED	1
+
 struct nbd_device {
 	u32 flags;
+	unsigned long runtime_flags;
 	struct socket * sock;	/* If == NULL, device is not ready, yet	*/
 	int magic;
 
-	spinlock_t queue_lock;
-	struct list_head queue_head;	/* Requests waiting result */
-	struct request *active_req;
-	wait_queue_head_t active_wq;
-	struct list_head waiting_queue;	/* Requests to be sent */
-	wait_queue_head_t waiting_wq;
+	struct blk_mq_tag_set tag_set;
 
 	struct mutex tx_lock;
 	struct gendisk *disk;
 	int blksize;
 	loff_t bytesize;
-	int xmit_timeout;
-	bool timedout;
-	bool disconnect; /* a disconnect has been requested by user */
 
-	struct timer_list timeout_timer;
 	/* protects initialization and shutdown of the socket */
 	spinlock_t sock_lock;
 	struct task_struct *task_recv;
@@ -71,6 +67,11 @@ struct nbd_device {
 #endif
 };
 
+struct nbd_cmd {
+	struct nbd_device *nbd;
+	struct list_head list;
+};
+
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 static struct dentry *nbd_dbg_dir;
 #endif
@@ -83,18 +84,6 @@ static unsigned int nbds_max = 16;
 static struct nbd_device *nbd_dev;
 static int max_part;
 
-/*
- * Use just one lock (or at most 1 per NIC). Two arguments for this:
- * 1. Each NIC is essentially a synchronization point for all servers
- *    accessed through that NIC so there's no need to have more locks
- *    than NICs anyway.
- * 2. More locks lead to more "Dirty cache line bouncing" which will slow
- *    down each lock to the point where they're actually slower than just
- *    a single lock.
- * Thanks go to Jens Axboe and Al Viro for their LKML emails explaining this!
- */
-static DEFINE_SPINLOCK(nbd_lock);
-
 static inline struct device *nbd_to_dev(struct nbd_device *nbd)
 {
 	return disk_to_dev(nbd->disk);
@@ -153,18 +142,16 @@ static int nbd_size_set(struct nbd_device *nbd, struct block_device *bdev,
 	return 0;
 }
 
-static void nbd_end_request(struct nbd_device *nbd, struct request *req)
+static void nbd_end_request(struct nbd_cmd *cmd)
 {
+	struct nbd_device *nbd = cmd->nbd;
+	struct request *req = blk_mq_rq_from_pdu(cmd);
 	int error = req->errors ? -EIO : 0;
-	struct request_queue *q = req->q;
-	unsigned long flags;
 
-	dev_dbg(nbd_to_dev(nbd), "request %p: %s\n", req,
+	dev_dbg(nbd_to_dev(nbd), "request %p: %s\n", cmd,
 		error ? "failed" : "done");
 
-	spin_lock_irqsave(q->queue_lock, flags);
-	__blk_end_request_all(req, error);
-	spin_unlock_irqrestore(q->queue_lock, flags);
+	blk_mq_complete_request(req, error);
 }
 
 /*
@@ -172,40 +159,49 @@ static void nbd_end_request(struct nbd_device *nbd, struct request *req)
  */
 static void sock_shutdown(struct nbd_device *nbd)
 {
-	spin_lock_irq(&nbd->sock_lock);
+	struct socket *sock;
+
+	spin_lock(&nbd->sock_lock);
 
 	if (!nbd->sock) {
 		spin_unlock_irq(&nbd->sock_lock);
 		return;
 	}
 
+	sock = nbd->sock;
 	dev_warn(disk_to_dev(nbd->disk), "shutting down socket\n");
-	kernel_sock_shutdown(nbd->sock, SHUT_RDWR);
-	sockfd_put(nbd->sock);
 	nbd->sock = NULL;
-	spin_unlock_irq(&nbd->sock_lock);
+	spin_unlock(&nbd->sock_lock);
 
-	del_timer(&nbd->timeout_timer);
+	kernel_sock_shutdown(sock, SHUT_RDWR);
+	sockfd_put(sock);
 }
 
-static void nbd_xmit_timeout(unsigned long arg)
+static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
+						 bool reserved)
 {
-	struct nbd_device *nbd = (struct nbd_device *)arg;
-	unsigned long flags;
-
-	if (list_empty(&nbd->queue_head))
-		return;
+	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);
+	struct nbd_device *nbd = cmd->nbd;
+	struct socket *sock = NULL;
 
-	spin_lock_irqsave(&nbd->sock_lock, flags);
+	spin_lock(&nbd->sock_lock);
 
-	nbd->timedout = true;
+	set_bit(NBD_TIMEDOUT, &nbd->runtime_flags);
 
-	if (nbd->sock)
-		kernel_sock_shutdown(nbd->sock, SHUT_RDWR);
+	if (nbd->sock) {
+		sock = nbd->sock;
+		get_file(sock->file);
+	}
 
-	spin_unlock_irqrestore(&nbd->sock_lock, flags);
+	spin_unlock(&nbd->sock_lock);
+	if (sock) {
+		kernel_sock_shutdown(sock, SHUT_RDWR);
+		sockfd_put(sock);
+	}
 
+	req->errors++;
 	dev_err(nbd_to_dev(nbd), "Connection timed out, shutting down connection\n");
+	return BLK_EH_HANDLED;
 }
 
 /*
@@ -255,9 +251,6 @@ static int sock_xmit(struct nbd_device *nbd, int send, void *buf, int size,
 
 	tsk_restore_flags(current, pflags, PF_MEMALLOC);
 
-	if (!send && nbd->xmit_timeout)
-		mod_timer(&nbd->timeout_timer, jiffies + nbd->xmit_timeout);
-
 	return result;
 }
 
@@ -273,8 +266,9 @@ static inline int sock_send_bvec(struct nbd_device *nbd, struct bio_vec *bvec,
 }
 
 /* always call with the tx_lock held */
-static int nbd_send_req(struct nbd_device *nbd, struct request *req)
+static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd)
 {
+	struct request *req = blk_mq_rq_from_pdu(cmd);
 	int result, flags;
 	struct nbd_request request;
 	unsigned long size = blk_rq_bytes(req);
@@ -298,10 +292,10 @@ static int nbd_send_req(struct nbd_device *nbd, struct request *req)
 		request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9);
 		request.len = htonl(size);
 	}
-	memcpy(request.handle, &req, sizeof(req));
+	memcpy(request.handle, &req->tag, sizeof(req->tag));
 
 	dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n",
-		req, nbdcmd_to_ascii(type),
+		cmd, nbdcmd_to_ascii(type),
 		(unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req));
 	result = sock_xmit(nbd, 1, &request, sizeof(request),
 			(type == NBD_CMD_WRITE) ? MSG_MORE : 0);
@@ -323,7 +317,7 @@ static int nbd_send_req(struct nbd_device *nbd, struct request *req)
 			if (!rq_iter_last(bvec, iter))
 				flags = MSG_MORE;
 			dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n",
-				req, bvec.bv_len);
+				cmd, bvec.bv_len);
 			result = sock_send_bvec(nbd, &bvec, flags);
 			if (result <= 0) {
 				dev_err(disk_to_dev(nbd->disk),
@@ -336,29 +330,6 @@ static int nbd_send_req(struct nbd_device *nbd, struct request *req)
 	return 0;
 }
 
-static struct request *nbd_find_request(struct nbd_device *nbd,
-					struct request *xreq)
-{
-	struct request *req, *tmp;
-	int err;
-
-	err = wait_event_interruptible(nbd->active_wq, nbd->active_req != xreq);
-	if (unlikely(err))
-		return ERR_PTR(err);
-
-	spin_lock(&nbd->queue_lock);
-	list_for_each_entry_safe(req, tmp, &nbd->queue_head, queuelist) {
-		if (req != xreq)
-			continue;
-		list_del_init(&req->queuelist);
-		spin_unlock(&nbd->queue_lock);
-		return req;
-	}
-	spin_unlock(&nbd->queue_lock);
-
-	return ERR_PTR(-ENOENT);
-}
-
 static inline int sock_recv_bvec(struct nbd_device *nbd, struct bio_vec *bvec)
 {
 	int result;
@@ -370,11 +341,14 @@ static inline int sock_recv_bvec(struct nbd_device *nbd, struct bio_vec *bvec)
 }
 
 /* NULL returned = something went wrong, inform userspace */
-static struct request *nbd_read_stat(struct nbd_device *nbd)
+static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd)
 {
 	int result;
 	struct nbd_reply reply;
-	struct request *req;
+	struct nbd_cmd *cmd;
+	struct request *req = NULL;
+	u16 hwq;
+	int tag;
 
 	reply.magic = 0;
 	result = sock_xmit(nbd, 0, &reply, sizeof(reply), MSG_WAITALL);
@@ -390,25 +364,27 @@ static struct request *nbd_read_stat(struct nbd_device *nbd)
 		return ERR_PTR(-EPROTO);
 	}
 
-	req = nbd_find_request(nbd, *(struct request **)reply.handle);
-	if (IS_ERR(req)) {
-		result = PTR_ERR(req);
-		if (result != -ENOENT)
-			return ERR_PTR(result);
+	memcpy(&tag, reply.handle, sizeof(int));
 
-		dev_err(disk_to_dev(nbd->disk), "Unexpected reply (%p)\n",
-			reply.handle);
-		return ERR_PTR(-EBADR);
+	hwq = blk_mq_unique_tag_to_hwq(tag);
+	if (hwq < nbd->tag_set.nr_hw_queues)
+		req = blk_mq_tag_to_rq(nbd->tag_set.tags[hwq],
+				       blk_mq_unique_tag_to_tag(tag));
+	if (!req || !blk_mq_request_started(req)) {
+		dev_err(disk_to_dev(nbd->disk), "Unexpected reply (%d) %p\n",
+			tag, req);
+		return ERR_PTR(-ENOENT);
 	}
+	cmd = blk_mq_rq_to_pdu(req);
 
 	if (ntohl(reply.error)) {
 		dev_err(disk_to_dev(nbd->disk), "Other side returned error (%d)\n",
 			ntohl(reply.error));
 		req->errors++;
-		return req;
+		return cmd;
 	}
 
-	dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", req);
+	dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", cmd);
 	if (rq_data_dir(req) != WRITE) {
 		struct req_iterator iter;
 		struct bio_vec bvec;
@@ -419,13 +395,13 @@ static struct request *nbd_read_stat(struct nbd_device *nbd)
 				dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n",
 					result);
 				req->errors++;
-				return req;
+				return cmd;
 			}
 			dev_dbg(nbd_to_dev(nbd), "request %p: got %d bytes data\n",
-				req, bvec.bv_len);
+				cmd, bvec.bv_len);
 		}
 	}
-	return req;
+	return cmd;
 }
 
 static ssize_t pid_show(struct device *dev,
@@ -444,7 +420,7 @@ static struct device_attribute pid_attr = {
 
 static int nbd_thread_recv(struct nbd_device *nbd, struct block_device *bdev)
 {
-	struct request *req;
+	struct nbd_cmd *cmd;
 	int ret;
 
 	BUG_ON(nbd->magic != NBD_MAGIC);
@@ -460,13 +436,13 @@ static int nbd_thread_recv(struct nbd_device *nbd, struct block_device *bdev)
 	nbd_size_update(nbd, bdev);
 
 	while (1) {
-		req = nbd_read_stat(nbd);
-		if (IS_ERR(req)) {
-			ret = PTR_ERR(req);
+		cmd = nbd_read_stat(nbd);
+		if (IS_ERR(cmd)) {
+			ret = PTR_ERR(cmd);
 			break;
 		}
 
-		nbd_end_request(nbd, req);
+		nbd_end_request(cmd);
 	}
 
 	nbd_size_clear(nbd, bdev);
@@ -475,44 +451,37 @@ static int nbd_thread_recv(struct nbd_device *nbd, struct block_device *bdev)
 	return ret;
 }
 
-static void nbd_clear_que(struct nbd_device *nbd)
+static void nbd_clear_req(struct request *req, void *data, bool reserved)
 {
-	struct request *req;
+	struct nbd_cmd *cmd;
+
+	if (!blk_mq_request_started(req))
+		return;
+	cmd = blk_mq_rq_to_pdu(req);
+	req->errors++;
+	nbd_end_request(cmd);
+}
 
+static void nbd_clear_que(struct nbd_device *nbd)
+{
 	BUG_ON(nbd->magic != NBD_MAGIC);
 
 	/*
 	 * Because we have set nbd->sock to NULL under the tx_lock, all
-	 * modifications to the list must have completed by now.  For
-	 * the same reason, the active_req must be NULL.
-	 *
-	 * As a consequence, we don't need to take the spin lock while
-	 * purging the list here.
+	 * modifications to the list must have completed by now.
 	 */
 	BUG_ON(nbd->sock);
-	BUG_ON(nbd->active_req);
 
-	while (!list_empty(&nbd->queue_head)) {
-		req = list_entry(nbd->queue_head.next, struct request,
-				 queuelist);
-		list_del_init(&req->queuelist);
-		req->errors++;
-		nbd_end_request(nbd, req);
-	}
-
-	while (!list_empty(&nbd->waiting_queue)) {
-		req = list_entry(nbd->waiting_queue.next, struct request,
-				 queuelist);
-		list_del_init(&req->queuelist);
-		req->errors++;
-		nbd_end_request(nbd, req);
-	}
+	blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_clear_req, NULL);
 	dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n");
 }
 
 
-static void nbd_handle_req(struct nbd_device *nbd, struct request *req)
+static void nbd_handle_cmd(struct nbd_cmd *cmd)
 {
+	struct request *req = blk_mq_rq_from_pdu(cmd);
+	struct nbd_device *nbd = cmd->nbd;
+
 	if (req->cmd_type != REQ_TYPE_FS)
 		goto error_out;
 
@@ -526,6 +495,7 @@ static void nbd_handle_req(struct nbd_device *nbd, struct request *req)
 	req->errors = 0;
 
 	mutex_lock(&nbd->tx_lock);
+	nbd->task_send = current;
 	if (unlikely(!nbd->sock)) {
 		mutex_unlock(&nbd->tx_lock);
 		dev_err(disk_to_dev(nbd->disk),
@@ -533,106 +503,30 @@ static void nbd_handle_req(struct nbd_device *nbd, struct request *req)
 		goto error_out;
 	}
 
-	nbd->active_req = req;
-
-	if (nbd->xmit_timeout && list_empty_careful(&nbd->queue_head))
-		mod_timer(&nbd->timeout_timer, jiffies + nbd->xmit_timeout);
-
-	if (nbd_send_req(nbd, req) != 0) {
+	if (nbd_send_cmd(nbd, cmd) != 0) {
 		dev_err(disk_to_dev(nbd->disk), "Request send failed\n");
 		req->errors++;
-		nbd_end_request(nbd, req);
-	} else {
-		spin_lock(&nbd->queue_lock);
-		list_add_tail(&req->queuelist, &nbd->queue_head);
-		spin_unlock(&nbd->queue_lock);
+		nbd_end_request(cmd);
 	}
 
-	nbd->active_req = NULL;
+	nbd->task_send = NULL;
 	mutex_unlock(&nbd->tx_lock);
-	wake_up_all(&nbd->active_wq);
 
 	return;
 
 error_out:
 	req->errors++;
-	nbd_end_request(nbd, req);
+	nbd_end_request(cmd);
 }
 
-static int nbd_thread_send(void *data)
+static int nbd_queue_rq(struct blk_mq_hw_ctx *hctx,
+			const struct blk_mq_queue_data *bd)
 {
-	struct nbd_device *nbd = data;
-	struct request *req;
-
-	nbd->task_send = current;
-
-	set_user_nice(current, MIN_NICE);
-	while (!kthread_should_stop() || !list_empty(&nbd->waiting_queue)) {
-		/* wait for something to do */
-		wait_event_interruptible(nbd->waiting_wq,
-					 kthread_should_stop() ||
-					 !list_empty(&nbd->waiting_queue));
-
-		/* extract request */
-		if (list_empty(&nbd->waiting_queue))
-			continue;
-
-		spin_lock_irq(&nbd->queue_lock);
-		req = list_entry(nbd->waiting_queue.next, struct request,
-				 queuelist);
-		list_del_init(&req->queuelist);
-		spin_unlock_irq(&nbd->queue_lock);
-
-		/* handle request */
-		nbd_handle_req(nbd, req);
-	}
+	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
 
-	nbd->task_send = NULL;
-
-	return 0;
-}
-
-/*
- * We always wait for result of write, for now. It would be nice to make it optional
- * in future
- * if ((rq_data_dir(req) == WRITE) && (nbd->flags & NBD_WRITE_NOCHK))
- *   { printk( "Warning: Ignoring result!\n"); nbd_end_request( req ); }
- */
-
-static void nbd_request_handler(struct request_queue *q)
-		__releases(q->queue_lock) __acquires(q->queue_lock)
-{
-	struct request *req;
-	
-	while ((req = blk_fetch_request(q)) != NULL) {
-		struct nbd_device *nbd;
-
-		spin_unlock_irq(q->queue_lock);
-
-		nbd = req->rq_disk->private_data;
-
-		BUG_ON(nbd->magic != NBD_MAGIC);
-
-		dev_dbg(nbd_to_dev(nbd), "request %p: dequeued (flags=%x)\n",
-			req, req->cmd_type);
-
-		if (unlikely(!nbd->sock)) {
-			dev_err_ratelimited(disk_to_dev(nbd->disk),
-					    "Attempted send on closed socket\n");
-			req->errors++;
-			nbd_end_request(nbd, req);
-			spin_lock_irq(q->queue_lock);
-			continue;
-		}
-
-		spin_lock_irq(&nbd->queue_lock);
-		list_add_tail(&req->queuelist, &nbd->waiting_queue);
-		spin_unlock_irq(&nbd->queue_lock);
-
-		wake_up(&nbd->waiting_wq);
-
-		spin_lock_irq(q->queue_lock);
-	}
+	blk_mq_start_request(bd->rq);
+	nbd_handle_cmd(cmd);
+	return BLK_MQ_RQ_QUEUE_OK;
 }
 
 static int nbd_set_socket(struct nbd_device *nbd, struct socket *sock)
@@ -657,15 +551,13 @@ out:
 /* Reset all properties of an NBD device */
 static void nbd_reset(struct nbd_device *nbd)
 {
-	nbd->disconnect = false;
-	nbd->timedout = false;
+	nbd->runtime_flags = 0;
 	nbd->blksize = 1024;
 	nbd->bytesize = 0;
 	set_capacity(nbd->disk, 0);
 	nbd->flags = 0;
-	nbd->xmit_timeout = 0;
+	nbd->tag_set.timeout = 0;
 	queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue);
-	del_timer_sync(&nbd->timeout_timer);
 }
 
 static void nbd_bdev_reset(struct block_device *bdev)
@@ -700,33 +592,37 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 {
 	switch (cmd) {
 	case NBD_DISCONNECT: {
-		struct request sreq;
+		struct request *sreq;
 
 		dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n");
 		if (!nbd->sock)
 			return -EINVAL;
 
+		sreq = blk_mq_alloc_request(bdev_get_queue(bdev), WRITE, 0);
+		if (!sreq)
+			return -ENOMEM;
+
 		mutex_unlock(&nbd->tx_lock);
 		fsync_bdev(bdev);
 		mutex_lock(&nbd->tx_lock);
-		blk_rq_init(NULL, &sreq);
-		sreq.cmd_type = REQ_TYPE_DRV_PRIV;
+		sreq->cmd_type = REQ_TYPE_DRV_PRIV;
 
 		/* Check again after getting mutex back.  */
-		if (!nbd->sock)
+		if (!nbd->sock) {
+			blk_mq_free_request(sreq);
 			return -EINVAL;
+		}
 
-		nbd->disconnect = true;
+		set_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags);
 
-		nbd_send_req(nbd, &sreq);
+		nbd_send_cmd(nbd, blk_mq_rq_to_pdu(sreq));
+		blk_mq_free_request(sreq);
 		return 0;
 	}
  
 	case NBD_CLEAR_SOCK:
 		sock_shutdown(nbd);
 		nbd_clear_que(nbd);
-		BUG_ON(!list_empty(&nbd->queue_head));
-		BUG_ON(!list_empty(&nbd->waiting_queue));
 		kill_bdev(bdev);
 		return 0;
 
@@ -758,13 +654,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 		return nbd_size_set(nbd, bdev, nbd->blksize, arg);
 
 	case NBD_SET_TIMEOUT:
-		nbd->xmit_timeout = arg * HZ;
-		if (arg)
-			mod_timer(&nbd->timeout_timer,
-				  jiffies + nbd->xmit_timeout);
-		else
-			del_timer_sync(&nbd->timeout_timer);
-
+		nbd->tag_set.timeout = arg * HZ;
 		return 0;
 
 	case NBD_SET_FLAGS:
@@ -772,7 +662,6 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 		return 0;
 
 	case NBD_DO_IT: {
-		struct task_struct *thread;
 		int error;
 
 		if (nbd->task_recv)
@@ -786,18 +675,9 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 
 		nbd_parse_flags(nbd, bdev);
 
-		thread = kthread_run(nbd_thread_send, nbd, "%s",
-				     nbd_name(nbd));
-		if (IS_ERR(thread)) {
-			mutex_lock(&nbd->tx_lock);
-			nbd->task_recv = NULL;
-			return PTR_ERR(thread);
-		}
-
 		nbd_dev_dbg_init(nbd);
 		error = nbd_thread_recv(nbd, bdev);
 		nbd_dev_dbg_close(nbd);
-		kthread_stop(thread);
 
 		mutex_lock(&nbd->tx_lock);
 		nbd->task_recv = NULL;
@@ -807,9 +687,10 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 		kill_bdev(bdev);
 		nbd_bdev_reset(bdev);
 
-		if (nbd->disconnect) /* user requested, ignore socket errors */
+		/* user requested, ignore socket errors */
+		if (test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags))
 			error = 0;
-		if (nbd->timedout)
+		if (test_bit(NBD_TIMEDOUT, &nbd->runtime_flags))
 			error = -ETIMEDOUT;
 
 		nbd_reset(nbd);
@@ -825,10 +706,10 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 		return 0;
 
 	case NBD_PRINT_DEBUG:
-		dev_info(disk_to_dev(nbd->disk),
-			"next = %p, prev = %p, head = %p\n",
-			nbd->queue_head.next, nbd->queue_head.prev,
-			&nbd->queue_head);
+		/*
+		 * For compatibility only, we no longer keep a list of
+		 * outstanding requests.
+		 */
 		return 0;
 	}
 	return -ENOTTY;
@@ -935,7 +816,7 @@ static int nbd_dev_dbg_init(struct nbd_device *nbd)
 
 	debugfs_create_file("tasks", 0444, dir, nbd, &nbd_dbg_tasks_ops);
 	debugfs_create_u64("size_bytes", 0444, dir, &nbd->bytesize);
-	debugfs_create_u32("timeout", 0444, dir, &nbd->xmit_timeout);
+	debugfs_create_u32("timeout", 0444, dir, &nbd->tag_set.timeout);
 	debugfs_create_u32("blocksize", 0444, dir, &nbd->blksize);
 	debugfs_create_file("flags", 0444, dir, nbd, &nbd_dbg_flags_ops);
 
@@ -987,6 +868,23 @@ static void nbd_dbg_close(void)
 
 #endif
 
+static int nbd_init_request(void *data, struct request *rq,
+			    unsigned int hctx_idx, unsigned int request_idx,
+			    unsigned int numa_node)
+{
+	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(rq);
+
+	cmd->nbd = data;
+	INIT_LIST_HEAD(&cmd->list);
+	return 0;
+}
+
+static struct blk_mq_ops nbd_mq_ops = {
+	.queue_rq	= nbd_queue_rq,
+	.init_request	= nbd_init_request,
+	.timeout	= nbd_xmit_timeout,
+};
+
 /*
  * And here should be modules and kernel interface 
  *  (Just smiley confuses emacs :-)
@@ -1035,16 +933,34 @@ static int __init nbd_init(void)
 		if (!disk)
 			goto out;
 		nbd_dev[i].disk = disk;
+
+		nbd_dev[i].tag_set.ops = &nbd_mq_ops;
+		nbd_dev[i].tag_set.nr_hw_queues = 1;
+		nbd_dev[i].tag_set.queue_depth = 128;
+		nbd_dev[i].tag_set.numa_node = NUMA_NO_NODE;
+		nbd_dev[i].tag_set.cmd_size = sizeof(struct nbd_cmd);
+		nbd_dev[i].tag_set.flags = BLK_MQ_F_SHOULD_MERGE |
+			BLK_MQ_F_SG_MERGE | BLK_MQ_F_BLOCKING;
+		nbd_dev[i].tag_set.driver_data = &nbd_dev[i];
+
+		err = blk_mq_alloc_tag_set(&nbd_dev[i].tag_set);
+		if (err) {
+			put_disk(disk);
+			goto out;
+		}
+
 		/*
 		 * The new linux 2.5 block layer implementation requires
 		 * every gendisk to have its very own request_queue struct.
 		 * These structs are big so we dynamically allocate them.
 		 */
-		disk->queue = blk_init_queue(nbd_request_handler, &nbd_lock);
+		disk->queue = blk_mq_init_queue(&nbd_dev[i].tag_set);
 		if (!disk->queue) {
+			blk_mq_free_tag_set(&nbd_dev[i].tag_set);
 			put_disk(disk);
 			goto out;
 		}
+
 		/*
 		 * Tell the block layer that we are not a rotational device
 		 */
@@ -1069,16 +985,8 @@ static int __init nbd_init(void)
 	for (i = 0; i < nbds_max; i++) {
 		struct gendisk *disk = nbd_dev[i].disk;
 		nbd_dev[i].magic = NBD_MAGIC;
-		INIT_LIST_HEAD(&nbd_dev[i].waiting_queue);
-		spin_lock_init(&nbd_dev[i].queue_lock);
 		spin_lock_init(&nbd_dev[i].sock_lock);
-		INIT_LIST_HEAD(&nbd_dev[i].queue_head);
 		mutex_init(&nbd_dev[i].tx_lock);
-		init_timer(&nbd_dev[i].timeout_timer);
-		nbd_dev[i].timeout_timer.function = nbd_xmit_timeout;
-		nbd_dev[i].timeout_timer.data = (unsigned long)&nbd_dev[i];
-		init_waitqueue_head(&nbd_dev[i].active_wq);
-		init_waitqueue_head(&nbd_dev[i].waiting_wq);
 		disk->major = NBD_MAJOR;
 		disk->first_minor = i << part_shift;
 		disk->fops = &nbd_fops;
@@ -1091,6 +999,7 @@ static int __init nbd_init(void)
 	return 0;
 out:
 	while (i--) {
+		blk_mq_free_tag_set(&nbd_dev[i].tag_set);
 		blk_cleanup_queue(nbd_dev[i].disk->queue);
 		put_disk(nbd_dev[i].disk);
 	}
@@ -1110,6 +1019,7 @@ static void __exit nbd_cleanup(void)
 		if (disk) {
 			del_gendisk(disk);
 			blk_cleanup_queue(disk->queue);
+			blk_mq_free_tag_set(&nbd_dev[i].tag_set);
 			put_disk(disk);
 		}
 	}
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c
index 75a7f88d6717..ba6f4a2e73db 100644
--- a/drivers/block/null_blk.c
+++ b/drivers/block/null_blk.c
@@ -34,6 +34,7 @@ struct nullb {
 	unsigned int index;
 	struct request_queue *q;
 	struct gendisk *disk;
+	struct nvm_dev *ndev;
 	struct blk_mq_tag_set tag_set;
 	struct hrtimer timer;
 	unsigned int queue_depth;
@@ -393,7 +394,6 @@ static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 
 static struct blk_mq_ops null_mq_ops = {
 	.queue_rq       = null_queue_rq,
-	.map_queue      = blk_mq_map_queue,
 	.init_hctx	= null_init_hctx,
 	.complete	= null_softirq_done_fn,
 };
@@ -414,23 +414,6 @@ static void cleanup_queues(struct nullb *nullb)
 	kfree(nullb->queues);
 }
 
-static void null_del_dev(struct nullb *nullb)
-{
-	list_del_init(&nullb->list);
-
-	if (use_lightnvm)
-		nvm_unregister(nullb->disk_name);
-	else
-		del_gendisk(nullb->disk);
-	blk_cleanup_queue(nullb->q);
-	if (queue_mode == NULL_Q_MQ)
-		blk_mq_free_tag_set(&nullb->tag_set);
-	if (!use_lightnvm)
-		put_disk(nullb->disk);
-	cleanup_queues(nullb);
-	kfree(nullb);
-}
-
 #ifdef CONFIG_NVM
 
 static void null_lnvm_end_io(struct request *rq, int error)
@@ -564,10 +547,58 @@ static struct nvm_dev_ops null_lnvm_dev_ops = {
 	/* Simulate nvme protocol restriction */
 	.max_phys_sect		= 64,
 };
+
+static int null_nvm_register(struct nullb *nullb)
+{
+	struct nvm_dev *dev;
+	int rv;
+
+	dev = nvm_alloc_dev(0);
+	if (!dev)
+		return -ENOMEM;
+
+	dev->q = nullb->q;
+	memcpy(dev->name, nullb->disk_name, DISK_NAME_LEN);
+	dev->ops = &null_lnvm_dev_ops;
+
+	rv = nvm_register(dev);
+	if (rv) {
+		kfree(dev);
+		return rv;
+	}
+	nullb->ndev = dev;
+	return 0;
+}
+
+static void null_nvm_unregister(struct nullb *nullb)
+{
+	nvm_unregister(nullb->ndev);
+}
 #else
-static struct nvm_dev_ops null_lnvm_dev_ops;
+static int null_nvm_register(struct nullb *nullb)
+{
+	return -EINVAL;
+}
+static void null_nvm_unregister(struct nullb *nullb) {}
 #endif /* CONFIG_NVM */
 
+static void null_del_dev(struct nullb *nullb)
+{
+	list_del_init(&nullb->list);
+
+	if (use_lightnvm)
+		null_nvm_unregister(nullb);
+	else
+		del_gendisk(nullb->disk);
+	blk_cleanup_queue(nullb->q);
+	if (queue_mode == NULL_Q_MQ)
+		blk_mq_free_tag_set(&nullb->tag_set);
+	if (!use_lightnvm)
+		put_disk(nullb->disk);
+	cleanup_queues(nullb);
+	kfree(nullb);
+}
+
 static int null_open(struct block_device *bdev, fmode_t mode)
 {
 	return 0;
@@ -640,11 +671,32 @@ static int init_driver_queues(struct nullb *nullb)
 	return 0;
 }
 
-static int null_add_dev(void)
+static int null_gendisk_register(struct nullb *nullb)
 {
 	struct gendisk *disk;
-	struct nullb *nullb;
 	sector_t size;
+
+	disk = nullb->disk = alloc_disk_node(1, home_node);
+	if (!disk)
+		return -ENOMEM;
+	size = gb * 1024 * 1024 * 1024ULL;
+	set_capacity(disk, size >> 9);
+
+	disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
+	disk->major		= null_major;
+	disk->first_minor	= nullb->index;
+	disk->fops		= &null_fops;
+	disk->private_data	= nullb;
+	disk->queue		= nullb->q;
+	strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
+
+	add_disk(disk);
+	return 0;
+}
+
+static int null_add_dev(void)
+{
+	struct nullb *nullb;
 	int rv;
 
 	nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, home_node);
@@ -716,42 +768,19 @@ static int null_add_dev(void)
 
 	sprintf(nullb->disk_name, "nullb%d", nullb->index);
 
-	if (use_lightnvm) {
-		rv = nvm_register(nullb->q, nullb->disk_name,
-							&null_lnvm_dev_ops);
-		if (rv)
-			goto out_cleanup_blk_queue;
-		goto done;
-	}
-
-	disk = nullb->disk = alloc_disk_node(1, home_node);
-	if (!disk) {
-		rv = -ENOMEM;
-		goto out_cleanup_lightnvm;
-	}
-	size = gb * 1024 * 1024 * 1024ULL;
-	set_capacity(disk, size >> 9);
-
-	disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
-	disk->major		= null_major;
-	disk->first_minor	= nullb->index;
-	disk->fops		= &null_fops;
-	disk->private_data	= nullb;
-	disk->queue		= nullb->q;
-	strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
+	if (use_lightnvm)
+		rv = null_nvm_register(nullb);
+	else
+		rv = null_gendisk_register(nullb);
 
-	add_disk(disk);
+	if (rv)
+		goto out_cleanup_blk_queue;
 
-done:
 	mutex_lock(&lock);
 	list_add_tail(&nullb->list, &nullb_list);
 	mutex_unlock(&lock);
 
 	return 0;
-
-out_cleanup_lightnvm:
-	if (use_lightnvm)
-		nvm_unregister(nullb->disk_name);
 out_cleanup_blk_queue:
 	blk_cleanup_queue(nullb->q);
 out_cleanup_tags:
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 1a04af6d2421..abb71628ab61 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -31,6 +31,7 @@
 #include <linux/ceph/libceph.h>
 #include <linux/ceph/osd_client.h>
 #include <linux/ceph/mon_client.h>
+#include <linux/ceph/cls_lock_client.h>
 #include <linux/ceph/decode.h>
 #include <linux/parser.h>
 #include <linux/bsearch.h>
@@ -114,12 +115,17 @@ static int atomic_dec_return_safe(atomic_t *v)
 
 #define RBD_OBJ_PREFIX_LEN_MAX	64
 
+#define RBD_NOTIFY_TIMEOUT	5	/* seconds */
+#define RBD_RETRY_DELAY		msecs_to_jiffies(1000)
+
 /* Feature bits */
 
 #define RBD_FEATURE_LAYERING	(1<<0)
 #define RBD_FEATURE_STRIPINGV2	(1<<1)
-#define RBD_FEATURES_ALL \
-	    (RBD_FEATURE_LAYERING | RBD_FEATURE_STRIPINGV2)
+#define RBD_FEATURE_EXCLUSIVE_LOCK (1<<2)
+#define RBD_FEATURES_ALL	(RBD_FEATURE_LAYERING |		\
+				 RBD_FEATURE_STRIPINGV2 |	\
+				 RBD_FEATURE_EXCLUSIVE_LOCK)
 
 /* Features supported by this (client software) implementation. */
 
@@ -128,11 +134,8 @@ static int atomic_dec_return_safe(atomic_t *v)
 /*
  * An RBD device name will be "rbd#", where the "rbd" comes from
  * RBD_DRV_NAME above, and # is a unique integer identifier.
- * MAX_INT_FORMAT_WIDTH is used in ensuring DEV_NAME_LEN is big
- * enough to hold all possible device names.
  */
 #define DEV_NAME_LEN		32
-#define MAX_INT_FORMAT_WIDTH	((5 * sizeof (int)) / 2 + 1)
 
 /*
  * block device image metadata (in-memory version)
@@ -322,6 +325,24 @@ struct rbd_img_request {
 #define for_each_obj_request_safe(ireq, oreq, n) \
 	list_for_each_entry_safe_reverse(oreq, n, &(ireq)->obj_requests, links)
 
+enum rbd_watch_state {
+	RBD_WATCH_STATE_UNREGISTERED,
+	RBD_WATCH_STATE_REGISTERED,
+	RBD_WATCH_STATE_ERROR,
+};
+
+enum rbd_lock_state {
+	RBD_LOCK_STATE_UNLOCKED,
+	RBD_LOCK_STATE_LOCKED,
+	RBD_LOCK_STATE_RELEASING,
+};
+
+/* WatchNotify::ClientId */
+struct rbd_client_id {
+	u64 gid;
+	u64 handle;
+};
+
 struct rbd_mapping {
 	u64                     size;
 	u64                     features;
@@ -349,13 +370,29 @@ struct rbd_device {
 	unsigned long		flags;		/* possibly lock protected */
 	struct rbd_spec		*spec;
 	struct rbd_options	*opts;
+	char			*config_info;	/* add{,_single_major} string */
 
 	struct ceph_object_id	header_oid;
 	struct ceph_object_locator header_oloc;
 
-	struct ceph_file_layout	layout;
+	struct ceph_file_layout	layout;		/* used for all rbd requests */
 
+	struct mutex		watch_mutex;
+	enum rbd_watch_state	watch_state;
 	struct ceph_osd_linger_request *watch_handle;
+	u64			watch_cookie;
+	struct delayed_work	watch_dwork;
+
+	struct rw_semaphore	lock_rwsem;
+	enum rbd_lock_state	lock_state;
+	struct rbd_client_id	owner_cid;
+	struct work_struct	acquired_lock_work;
+	struct work_struct	released_lock_work;
+	struct delayed_work	lock_dwork;
+	struct work_struct	unlock_work;
+	wait_queue_head_t	lock_waitq;
+
+	struct workqueue_struct	*task_wq;
 
 	struct rbd_spec		*parent_spec;
 	u64			parent_overlap;
@@ -439,6 +476,29 @@ static int minor_to_rbd_dev_id(int minor)
 	return minor >> RBD_SINGLE_MAJOR_PART_SHIFT;
 }
 
+static bool rbd_is_lock_supported(struct rbd_device *rbd_dev)
+{
+	return (rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK) &&
+	       rbd_dev->spec->snap_id == CEPH_NOSNAP &&
+	       !rbd_dev->mapping.read_only;
+}
+
+static bool __rbd_is_lock_owner(struct rbd_device *rbd_dev)
+{
+	return rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED ||
+	       rbd_dev->lock_state == RBD_LOCK_STATE_RELEASING;
+}
+
+static bool rbd_is_lock_owner(struct rbd_device *rbd_dev)
+{
+	bool is_lock_owner;
+
+	down_read(&rbd_dev->lock_rwsem);
+	is_lock_owner = __rbd_is_lock_owner(rbd_dev);
+	up_read(&rbd_dev->lock_rwsem);
+	return is_lock_owner;
+}
+
 static BUS_ATTR(add, S_IWUSR, NULL, rbd_add);
 static BUS_ATTR(remove, S_IWUSR, NULL, rbd_remove);
 static BUS_ATTR(add_single_major, S_IWUSR, NULL, rbd_add_single_major);
@@ -735,6 +795,7 @@ enum {
 	/* string args above */
 	Opt_read_only,
 	Opt_read_write,
+	Opt_lock_on_read,
 	Opt_err
 };
 
@@ -746,16 +807,19 @@ static match_table_t rbd_opts_tokens = {
 	{Opt_read_only, "ro"},		/* Alternate spelling */
 	{Opt_read_write, "read_write"},
 	{Opt_read_write, "rw"},		/* Alternate spelling */
+	{Opt_lock_on_read, "lock_on_read"},
 	{Opt_err, NULL}
 };
 
 struct rbd_options {
 	int	queue_depth;
 	bool	read_only;
+	bool	lock_on_read;
 };
 
 #define RBD_QUEUE_DEPTH_DEFAULT	BLKDEV_MAX_RQ
 #define RBD_READ_ONLY_DEFAULT	false
+#define RBD_LOCK_ON_READ_DEFAULT false
 
 static int parse_rbd_opts_token(char *c, void *private)
 {
@@ -791,6 +855,9 @@ static int parse_rbd_opts_token(char *c, void *private)
 	case Opt_read_write:
 		rbd_opts->read_only = false;
 		break;
+	case Opt_lock_on_read:
+		rbd_opts->lock_on_read = true;
+		break;
 	default:
 		/* libceph prints "bad option" msg */
 		return -EINVAL;
@@ -919,7 +986,6 @@ static int rbd_header_from_disk(struct rbd_device *rbd_dev,
 	char *snap_names = NULL;
 	u64 *snap_sizes = NULL;
 	u32 snap_count;
-	size_t size;
 	int ret = -ENOMEM;
 	u32 i;
 
@@ -957,9 +1023,9 @@ static int rbd_header_from_disk(struct rbd_device *rbd_dev,
 			goto out_err;
 
 		/* ...as well as the array of their sizes. */
-
-		size = snap_count * sizeof (*header->snap_sizes);
-		snap_sizes = kmalloc(size, GFP_KERNEL);
+		snap_sizes = kmalloc_array(snap_count,
+					   sizeof(*header->snap_sizes),
+					   GFP_KERNEL);
 		if (!snap_sizes)
 			goto out_err;
 
@@ -1551,11 +1617,18 @@ static bool obj_request_type_valid(enum obj_request_type type)
 	}
 }
 
-static int rbd_obj_request_submit(struct ceph_osd_client *osdc,
-				struct rbd_obj_request *obj_request)
+static void rbd_img_obj_callback(struct rbd_obj_request *obj_request);
+
+static void rbd_obj_request_submit(struct rbd_obj_request *obj_request)
 {
-	dout("%s %p\n", __func__, obj_request);
-	return ceph_osdc_start_request(osdc, obj_request->osd_req, false);
+	struct ceph_osd_request *osd_req = obj_request->osd_req;
+
+	dout("%s %p osd_req %p\n", __func__, obj_request, osd_req);
+	if (obj_request_img_data_test(obj_request)) {
+		WARN_ON(obj_request->callback != rbd_img_obj_callback);
+		rbd_img_request_get(obj_request->img_request);
+	}
+	ceph_osdc_start_request(osd_req->r_osdc, osd_req, false);
 }
 
 static void rbd_obj_request_end(struct rbd_obj_request *obj_request)
@@ -1745,6 +1818,22 @@ static void rbd_obj_request_complete(struct rbd_obj_request *obj_request)
 		complete_all(&obj_request->completion);
 }
 
+static void rbd_obj_request_error(struct rbd_obj_request *obj_request, int err)
+{
+	obj_request->result = err;
+	obj_request->xferred = 0;
+	/*
+	 * kludge - mirror rbd_obj_request_submit() to match a put in
+	 * rbd_img_obj_callback()
+	 */
+	if (obj_request_img_data_test(obj_request)) {
+		WARN_ON(obj_request->callback != rbd_img_obj_callback);
+		rbd_img_request_get(obj_request->img_request);
+	}
+	obj_request_done_set(obj_request);
+	rbd_obj_request_complete(obj_request);
+}
+
 static void rbd_osd_read_callback(struct rbd_obj_request *obj_request)
 {
 	struct rbd_img_request *img_request = NULL;
@@ -1877,11 +1966,10 @@ static void rbd_osd_req_callback(struct ceph_osd_request *osd_req)
 
 static void rbd_osd_req_format_read(struct rbd_obj_request *obj_request)
 {
-	struct rbd_img_request *img_request = obj_request->img_request;
 	struct ceph_osd_request *osd_req = obj_request->osd_req;
 
-	if (img_request)
-		osd_req->r_snapid = img_request->snap_id;
+	rbd_assert(obj_request_img_data_test(obj_request));
+	osd_req->r_snapid = obj_request->img_request->snap_id;
 }
 
 static void rbd_osd_req_format_write(struct rbd_obj_request *obj_request)
@@ -2074,7 +2162,9 @@ static void rbd_obj_request_destroy(struct kref *kref)
 			bio_chain_put(obj_request->bio_list);
 		break;
 	case OBJ_REQUEST_PAGES:
-		if (obj_request->pages)
+		/* img_data requests don't own their page array */
+		if (obj_request->pages &&
+		    !obj_request_img_data_test(obj_request))
 			ceph_release_page_vector(obj_request->pages,
 						obj_request->page_count);
 		break;
@@ -2295,13 +2385,6 @@ static bool rbd_img_obj_end_request(struct rbd_obj_request *obj_request)
 		xferred = obj_request->length;
 	}
 
-	/* Image object requests don't own their page array */
-
-	if (obj_request->type == OBJ_REQUEST_PAGES) {
-		obj_request->pages = NULL;
-		obj_request->page_count = 0;
-	}
-
 	if (img_request_child_test(img_request)) {
 		rbd_assert(img_request->obj_request != NULL);
 		more = obj_request->which < img_request->obj_request_count - 1;
@@ -2520,8 +2603,6 @@ static int rbd_img_request_fill(struct rbd_img_request *img_request,
 
 		rbd_img_obj_request_fill(obj_request, osd_req, op_type, 0);
 
-		rbd_img_request_get(img_request);
-
 		img_offset += length;
 		resid -= length;
 	}
@@ -2579,7 +2660,6 @@ rbd_img_obj_parent_read_full_callback(struct rbd_img_request *img_request)
 {
 	struct rbd_obj_request *orig_request;
 	struct ceph_osd_request *osd_req;
-	struct ceph_osd_client *osdc;
 	struct rbd_device *rbd_dev;
 	struct page **pages;
 	enum obj_operation_type op_type;
@@ -2603,7 +2683,7 @@ rbd_img_obj_parent_read_full_callback(struct rbd_img_request *img_request)
 	rbd_assert(obj_request_type_valid(orig_request->type));
 	img_result = img_request->result;
 	parent_length = img_request->length;
-	rbd_assert(parent_length == img_request->xferred);
+	rbd_assert(img_result || parent_length == img_request->xferred);
 	rbd_img_request_put(img_request);
 
 	rbd_assert(orig_request->img_request);
@@ -2616,13 +2696,9 @@ rbd_img_obj_parent_read_full_callback(struct rbd_img_request *img_request)
 	 * and re-submit the original write request.
 	 */
 	if (!rbd_dev->parent_overlap) {
-		struct ceph_osd_client *osdc;
-
 		ceph_release_page_vector(pages, page_count);
-		osdc = &rbd_dev->rbd_client->client->osdc;
-		img_result = rbd_obj_request_submit(osdc, orig_request);
-		if (!img_result)
-			return;
+		rbd_obj_request_submit(orig_request);
+		return;
 	}
 
 	if (img_result)
@@ -2656,17 +2732,12 @@ rbd_img_obj_parent_read_full_callback(struct rbd_img_request *img_request)
 
 	/* All set, send it off. */
 
-	osdc = &rbd_dev->rbd_client->client->osdc;
-	img_result = rbd_obj_request_submit(osdc, orig_request);
-	if (!img_result)
-		return;
-out_err:
-	/* Record the error code and complete the request */
+	rbd_obj_request_submit(orig_request);
+	return;
 
-	orig_request->result = img_result;
-	orig_request->xferred = 0;
-	obj_request_done_set(orig_request);
-	rbd_obj_request_complete(orig_request);
+out_err:
+	ceph_release_page_vector(pages, page_count);
+	rbd_obj_request_error(orig_request, img_result);
 }
 
 /*
@@ -2680,26 +2751,19 @@ out_err:
  * When the read completes, this page array will be transferred to
  * the original object request for the copyup operation.
  *
- * If an error occurs, record it as the result of the original
- * object request and mark it done so it gets completed.
+ * If an error occurs, it is recorded as the result of the original
+ * object request in rbd_img_obj_exists_callback().
  */
 static int rbd_img_obj_parent_read_full(struct rbd_obj_request *obj_request)
 {
-	struct rbd_img_request *img_request = NULL;
+	struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;
 	struct rbd_img_request *parent_request = NULL;
-	struct rbd_device *rbd_dev;
 	u64 img_offset;
 	u64 length;
 	struct page **pages = NULL;
 	u32 page_count;
 	int result;
 
-	rbd_assert(obj_request_img_data_test(obj_request));
-	rbd_assert(obj_request_type_valid(obj_request->type));
-
-	img_request = obj_request->img_request;
-	rbd_assert(img_request != NULL);
-	rbd_dev = img_request->rbd_dev;
 	rbd_assert(rbd_dev->parent != NULL);
 
 	/*
@@ -2740,10 +2804,11 @@ static int rbd_img_obj_parent_read_full(struct rbd_obj_request *obj_request)
 	result = rbd_img_request_fill(parent_request, OBJ_REQUEST_PAGES, pages);
 	if (result)
 		goto out_err;
+
 	parent_request->copyup_pages = pages;
 	parent_request->copyup_page_count = page_count;
-
 	parent_request->callback = rbd_img_obj_parent_read_full_callback;
+
 	result = rbd_img_request_submit(parent_request);
 	if (!result)
 		return 0;
@@ -2757,10 +2822,6 @@ out_err:
 		ceph_release_page_vector(pages, page_count);
 	if (parent_request)
 		rbd_img_request_put(parent_request);
-	obj_request->result = result;
-	obj_request->xferred = 0;
-	obj_request_done_set(obj_request);
-
 	return result;
 }
 
@@ -2793,17 +2854,13 @@ static void rbd_img_obj_exists_callback(struct rbd_obj_request *obj_request)
 
 	/*
 	 * If the overlap has become 0 (most likely because the
-	 * image has been flattened) we need to free the pages
-	 * and re-submit the original write request.
+	 * image has been flattened) we need to re-submit the
+	 * original request.
 	 */
 	rbd_dev = orig_request->img_request->rbd_dev;
 	if (!rbd_dev->parent_overlap) {
-		struct ceph_osd_client *osdc;
-
-		osdc = &rbd_dev->rbd_client->client->osdc;
-		result = rbd_obj_request_submit(osdc, orig_request);
-		if (!result)
-			return;
+		rbd_obj_request_submit(orig_request);
+		return;
 	}
 
 	/*
@@ -2816,31 +2873,45 @@ static void rbd_img_obj_exists_callback(struct rbd_obj_request *obj_request)
 		obj_request_existence_set(orig_request, true);
 	} else if (result == -ENOENT) {
 		obj_request_existence_set(orig_request, false);
-	} else if (result) {
-		orig_request->result = result;
-		goto out;
+	} else {
+		goto fail_orig_request;
 	}
 
 	/*
 	 * Resubmit the original request now that we have recorded
 	 * whether the target object exists.
 	 */
-	orig_request->result = rbd_img_obj_request_submit(orig_request);
-out:
-	if (orig_request->result)
-		rbd_obj_request_complete(orig_request);
+	result = rbd_img_obj_request_submit(orig_request);
+	if (result)
+		goto fail_orig_request;
+
+	return;
+
+fail_orig_request:
+	rbd_obj_request_error(orig_request, result);
 }
 
 static int rbd_img_obj_exists_submit(struct rbd_obj_request *obj_request)
 {
+	struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;
 	struct rbd_obj_request *stat_request;
-	struct rbd_device *rbd_dev;
-	struct ceph_osd_client *osdc;
-	struct page **pages = NULL;
+	struct page **pages;
 	u32 page_count;
 	size_t size;
 	int ret;
 
+	stat_request = rbd_obj_request_create(obj_request->object_name, 0, 0,
+					      OBJ_REQUEST_PAGES);
+	if (!stat_request)
+		return -ENOMEM;
+
+	stat_request->osd_req = rbd_osd_req_create(rbd_dev, OBJ_OP_READ, 1,
+						   stat_request);
+	if (!stat_request->osd_req) {
+		ret = -ENOMEM;
+		goto fail_stat_request;
+	}
+
 	/*
 	 * The response data for a STAT call consists of:
 	 *     le64 length;
@@ -2852,52 +2923,33 @@ static int rbd_img_obj_exists_submit(struct rbd_obj_request *obj_request)
 	size = sizeof (__le64) + sizeof (__le32) + sizeof (__le32);
 	page_count = (u32)calc_pages_for(0, size);
 	pages = ceph_alloc_page_vector(page_count, GFP_KERNEL);
-	if (IS_ERR(pages))
-		return PTR_ERR(pages);
+	if (IS_ERR(pages)) {
+		ret = PTR_ERR(pages);
+		goto fail_stat_request;
+	}
 
-	ret = -ENOMEM;
-	stat_request = rbd_obj_request_create(obj_request->object_name, 0, 0,
-							OBJ_REQUEST_PAGES);
-	if (!stat_request)
-		goto out;
+	osd_req_op_init(stat_request->osd_req, 0, CEPH_OSD_OP_STAT, 0);
+	osd_req_op_raw_data_in_pages(stat_request->osd_req, 0, pages, size, 0,
+				     false, false);
 
 	rbd_obj_request_get(obj_request);
 	stat_request->obj_request = obj_request;
 	stat_request->pages = pages;
 	stat_request->page_count = page_count;
-
-	rbd_assert(obj_request->img_request);
-	rbd_dev = obj_request->img_request->rbd_dev;
-	stat_request->osd_req = rbd_osd_req_create(rbd_dev, OBJ_OP_READ, 1,
-						   stat_request);
-	if (!stat_request->osd_req)
-		goto out;
 	stat_request->callback = rbd_img_obj_exists_callback;
 
-	osd_req_op_init(stat_request->osd_req, 0, CEPH_OSD_OP_STAT, 0);
-	osd_req_op_raw_data_in_pages(stat_request->osd_req, 0, pages, size, 0,
-					false, false);
-	rbd_osd_req_format_read(stat_request);
-
-	osdc = &rbd_dev->rbd_client->client->osdc;
-	ret = rbd_obj_request_submit(osdc, stat_request);
-out:
-	if (ret)
-		rbd_obj_request_put(obj_request);
+	rbd_obj_request_submit(stat_request);
+	return 0;
 
+fail_stat_request:
+	rbd_obj_request_put(stat_request);
 	return ret;
 }
 
 static bool img_obj_request_simple(struct rbd_obj_request *obj_request)
 {
-	struct rbd_img_request *img_request;
-	struct rbd_device *rbd_dev;
-
-	rbd_assert(obj_request_img_data_test(obj_request));
-
-	img_request = obj_request->img_request;
-	rbd_assert(img_request);
-	rbd_dev = img_request->rbd_dev;
+	struct rbd_img_request *img_request = obj_request->img_request;
+	struct rbd_device *rbd_dev = img_request->rbd_dev;
 
 	/* Reads */
 	if (!img_request_write_test(img_request) &&
@@ -2936,14 +2988,13 @@ static bool img_obj_request_simple(struct rbd_obj_request *obj_request)
 
 static int rbd_img_obj_request_submit(struct rbd_obj_request *obj_request)
 {
-	if (img_obj_request_simple(obj_request)) {
-		struct rbd_device *rbd_dev;
-		struct ceph_osd_client *osdc;
-
-		rbd_dev = obj_request->img_request->rbd_dev;
-		osdc = &rbd_dev->rbd_client->client->osdc;
+	rbd_assert(obj_request_img_data_test(obj_request));
+	rbd_assert(obj_request_type_valid(obj_request->type));
+	rbd_assert(obj_request->img_request);
 
-		return rbd_obj_request_submit(osdc, obj_request);
+	if (img_obj_request_simple(obj_request)) {
+		rbd_obj_request_submit(obj_request);
+		return 0;
 	}
 
 	/*
@@ -3006,12 +3057,8 @@ static void rbd_img_parent_read_callback(struct rbd_img_request *img_request)
 	rbd_assert(obj_request->img_request);
 	rbd_dev = obj_request->img_request->rbd_dev;
 	if (!rbd_dev->parent_overlap) {
-		struct ceph_osd_client *osdc;
-
-		osdc = &rbd_dev->rbd_client->client->osdc;
-		img_result = rbd_obj_request_submit(osdc, obj_request);
-		if (!img_result)
-			return;
+		rbd_obj_request_submit(obj_request);
+		return;
 	}
 
 	obj_request->result = img_result;
@@ -3084,65 +3131,724 @@ out_err:
 	obj_request_done_set(obj_request);
 }
 
-static int rbd_dev_header_watch_sync(struct rbd_device *rbd_dev);
-static void __rbd_dev_header_unwatch_sync(struct rbd_device *rbd_dev);
+static const struct rbd_client_id rbd_empty_cid;
 
-static void rbd_watch_cb(void *arg, u64 notify_id, u64 cookie,
-			 u64 notifier_id, void *data, size_t data_len)
+static bool rbd_cid_equal(const struct rbd_client_id *lhs,
+			  const struct rbd_client_id *rhs)
+{
+	return lhs->gid == rhs->gid && lhs->handle == rhs->handle;
+}
+
+static struct rbd_client_id rbd_get_cid(struct rbd_device *rbd_dev)
+{
+	struct rbd_client_id cid;
+
+	mutex_lock(&rbd_dev->watch_mutex);
+	cid.gid = ceph_client_gid(rbd_dev->rbd_client->client);
+	cid.handle = rbd_dev->watch_cookie;
+	mutex_unlock(&rbd_dev->watch_mutex);
+	return cid;
+}
+
+/*
+ * lock_rwsem must be held for write
+ */
+static void rbd_set_owner_cid(struct rbd_device *rbd_dev,
+			      const struct rbd_client_id *cid)
+{
+	dout("%s rbd_dev %p %llu-%llu -> %llu-%llu\n", __func__, rbd_dev,
+	     rbd_dev->owner_cid.gid, rbd_dev->owner_cid.handle,
+	     cid->gid, cid->handle);
+	rbd_dev->owner_cid = *cid; /* struct */
+}
+
+static void format_lock_cookie(struct rbd_device *rbd_dev, char *buf)
+{
+	mutex_lock(&rbd_dev->watch_mutex);
+	sprintf(buf, "%s %llu", RBD_LOCK_COOKIE_PREFIX, rbd_dev->watch_cookie);
+	mutex_unlock(&rbd_dev->watch_mutex);
+}
+
+/*
+ * lock_rwsem must be held for write
+ */
+static int rbd_lock(struct rbd_device *rbd_dev)
+{
+	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+	struct rbd_client_id cid = rbd_get_cid(rbd_dev);
+	char cookie[32];
+	int ret;
+
+	WARN_ON(__rbd_is_lock_owner(rbd_dev));
+
+	format_lock_cookie(rbd_dev, cookie);
+	ret = ceph_cls_lock(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
+			    RBD_LOCK_NAME, CEPH_CLS_LOCK_EXCLUSIVE, cookie,
+			    RBD_LOCK_TAG, "", 0);
+	if (ret)
+		return ret;
+
+	rbd_dev->lock_state = RBD_LOCK_STATE_LOCKED;
+	rbd_set_owner_cid(rbd_dev, &cid);
+	queue_work(rbd_dev->task_wq, &rbd_dev->acquired_lock_work);
+	return 0;
+}
+
+/*
+ * lock_rwsem must be held for write
+ */
+static int rbd_unlock(struct rbd_device *rbd_dev)
+{
+	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+	char cookie[32];
+	int ret;
+
+	WARN_ON(!__rbd_is_lock_owner(rbd_dev));
+
+	rbd_dev->lock_state = RBD_LOCK_STATE_UNLOCKED;
+
+	format_lock_cookie(rbd_dev, cookie);
+	ret = ceph_cls_unlock(osdc, &rbd_dev->header_oid, &rbd_dev->header_oloc,
+			      RBD_LOCK_NAME, cookie);
+	if (ret && ret != -ENOENT) {
+		rbd_warn(rbd_dev, "cls_unlock failed: %d", ret);
+		return ret;
+	}
+
+	rbd_set_owner_cid(rbd_dev, &rbd_empty_cid);
+	queue_work(rbd_dev->task_wq, &rbd_dev->released_lock_work);
+	return 0;
+}
+
+static int __rbd_notify_op_lock(struct rbd_device *rbd_dev,
+				enum rbd_notify_op notify_op,
+				struct page ***preply_pages,
+				size_t *preply_len)
+{
+	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+	struct rbd_client_id cid = rbd_get_cid(rbd_dev);
+	int buf_size = 4 + 8 + 8 + CEPH_ENCODING_START_BLK_LEN;
+	char buf[buf_size];
+	void *p = buf;
+
+	dout("%s rbd_dev %p notify_op %d\n", __func__, rbd_dev, notify_op);
+
+	/* encode *LockPayload NotifyMessage (op + ClientId) */
+	ceph_start_encoding(&p, 2, 1, buf_size - CEPH_ENCODING_START_BLK_LEN);
+	ceph_encode_32(&p, notify_op);
+	ceph_encode_64(&p, cid.gid);
+	ceph_encode_64(&p, cid.handle);
+
+	return ceph_osdc_notify(osdc, &rbd_dev->header_oid,
+				&rbd_dev->header_oloc, buf, buf_size,
+				RBD_NOTIFY_TIMEOUT, preply_pages, preply_len);
+}
+
+static void rbd_notify_op_lock(struct rbd_device *rbd_dev,
+			       enum rbd_notify_op notify_op)
+{
+	struct page **reply_pages;
+	size_t reply_len;
+
+	__rbd_notify_op_lock(rbd_dev, notify_op, &reply_pages, &reply_len);
+	ceph_release_page_vector(reply_pages, calc_pages_for(0, reply_len));
+}
+
+static void rbd_notify_acquired_lock(struct work_struct *work)
+{
+	struct rbd_device *rbd_dev = container_of(work, struct rbd_device,
+						  acquired_lock_work);
+
+	rbd_notify_op_lock(rbd_dev, RBD_NOTIFY_OP_ACQUIRED_LOCK);
+}
+
+static void rbd_notify_released_lock(struct work_struct *work)
+{
+	struct rbd_device *rbd_dev = container_of(work, struct rbd_device,
+						  released_lock_work);
+
+	rbd_notify_op_lock(rbd_dev, RBD_NOTIFY_OP_RELEASED_LOCK);
+}
+
+static int rbd_request_lock(struct rbd_device *rbd_dev)
+{
+	struct page **reply_pages;
+	size_t reply_len;
+	bool lock_owner_responded = false;
+	int ret;
+
+	dout("%s rbd_dev %p\n", __func__, rbd_dev);
+
+	ret = __rbd_notify_op_lock(rbd_dev, RBD_NOTIFY_OP_REQUEST_LOCK,
+				   &reply_pages, &reply_len);
+	if (ret && ret != -ETIMEDOUT) {
+		rbd_warn(rbd_dev, "failed to request lock: %d", ret);
+		goto out;
+	}
+
+	if (reply_len > 0 && reply_len <= PAGE_SIZE) {
+		void *p = page_address(reply_pages[0]);
+		void *const end = p + reply_len;
+		u32 n;
+
+		ceph_decode_32_safe(&p, end, n, e_inval); /* num_acks */
+		while (n--) {
+			u8 struct_v;
+			u32 len;
+
+			ceph_decode_need(&p, end, 8 + 8, e_inval);
+			p += 8 + 8; /* skip gid and cookie */
+
+			ceph_decode_32_safe(&p, end, len, e_inval);
+			if (!len)
+				continue;
+
+			if (lock_owner_responded) {
+				rbd_warn(rbd_dev,
+					 "duplicate lock owners detected");
+				ret = -EIO;
+				goto out;
+			}
+
+			lock_owner_responded = true;
+			ret = ceph_start_decoding(&p, end, 1, "ResponseMessage",
+						  &struct_v, &len);
+			if (ret) {
+				rbd_warn(rbd_dev,
+					 "failed to decode ResponseMessage: %d",
+					 ret);
+				goto e_inval;
+			}
+
+			ret = ceph_decode_32(&p);
+		}
+	}
+
+	if (!lock_owner_responded) {
+		rbd_warn(rbd_dev, "no lock owners detected");
+		ret = -ETIMEDOUT;
+	}
+
+out:
+	ceph_release_page_vector(reply_pages, calc_pages_for(0, reply_len));
+	return ret;
+
+e_inval:
+	ret = -EINVAL;
+	goto out;
+}
+
+static void wake_requests(struct rbd_device *rbd_dev, bool wake_all)
+{
+	dout("%s rbd_dev %p wake_all %d\n", __func__, rbd_dev, wake_all);
+
+	cancel_delayed_work(&rbd_dev->lock_dwork);
+	if (wake_all)
+		wake_up_all(&rbd_dev->lock_waitq);
+	else
+		wake_up(&rbd_dev->lock_waitq);
+}
+
+static int get_lock_owner_info(struct rbd_device *rbd_dev,
+			       struct ceph_locker **lockers, u32 *num_lockers)
 {
-	struct rbd_device *rbd_dev = arg;
 	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+	u8 lock_type;
+	char *lock_tag;
+	int ret;
+
+	dout("%s rbd_dev %p\n", __func__, rbd_dev);
+
+	ret = ceph_cls_lock_info(osdc, &rbd_dev->header_oid,
+				 &rbd_dev->header_oloc, RBD_LOCK_NAME,
+				 &lock_type, &lock_tag, lockers, num_lockers);
+	if (ret)
+		return ret;
+
+	if (*num_lockers == 0) {
+		dout("%s rbd_dev %p no lockers detected\n", __func__, rbd_dev);
+		goto out;
+	}
+
+	if (strcmp(lock_tag, RBD_LOCK_TAG)) {
+		rbd_warn(rbd_dev, "locked by external mechanism, tag %s",
+			 lock_tag);
+		ret = -EBUSY;
+		goto out;
+	}
+
+	if (lock_type == CEPH_CLS_LOCK_SHARED) {
+		rbd_warn(rbd_dev, "shared lock type detected");
+		ret = -EBUSY;
+		goto out;
+	}
+
+	if (strncmp((*lockers)[0].id.cookie, RBD_LOCK_COOKIE_PREFIX,
+		    strlen(RBD_LOCK_COOKIE_PREFIX))) {
+		rbd_warn(rbd_dev, "locked by external mechanism, cookie %s",
+			 (*lockers)[0].id.cookie);
+		ret = -EBUSY;
+		goto out;
+	}
+
+out:
+	kfree(lock_tag);
+	return ret;
+}
+
+static int find_watcher(struct rbd_device *rbd_dev,
+			const struct ceph_locker *locker)
+{
+	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+	struct ceph_watch_item *watchers;
+	u32 num_watchers;
+	u64 cookie;
+	int i;
+	int ret;
+
+	ret = ceph_osdc_list_watchers(osdc, &rbd_dev->header_oid,
+				      &rbd_dev->header_oloc, &watchers,
+				      &num_watchers);
+	if (ret)
+		return ret;
+
+	sscanf(locker->id.cookie, RBD_LOCK_COOKIE_PREFIX " %llu", &cookie);
+	for (i = 0; i < num_watchers; i++) {
+		if (!memcmp(&watchers[i].addr, &locker->info.addr,
+			    sizeof(locker->info.addr)) &&
+		    watchers[i].cookie == cookie) {
+			struct rbd_client_id cid = {
+				.gid = le64_to_cpu(watchers[i].name.num),
+				.handle = cookie,
+			};
+
+			dout("%s rbd_dev %p found cid %llu-%llu\n", __func__,
+			     rbd_dev, cid.gid, cid.handle);
+			rbd_set_owner_cid(rbd_dev, &cid);
+			ret = 1;
+			goto out;
+		}
+	}
+
+	dout("%s rbd_dev %p no watchers\n", __func__, rbd_dev);
+	ret = 0;
+out:
+	kfree(watchers);
+	return ret;
+}
+
+/*
+ * lock_rwsem must be held for write
+ */
+static int rbd_try_lock(struct rbd_device *rbd_dev)
+{
+	struct ceph_client *client = rbd_dev->rbd_client->client;
+	struct ceph_locker *lockers;
+	u32 num_lockers;
+	int ret;
+
+	for (;;) {
+		ret = rbd_lock(rbd_dev);
+		if (ret != -EBUSY)
+			return ret;
+
+		/* determine if the current lock holder is still alive */
+		ret = get_lock_owner_info(rbd_dev, &lockers, &num_lockers);
+		if (ret)
+			return ret;
+
+		if (num_lockers == 0)
+			goto again;
+
+		ret = find_watcher(rbd_dev, lockers);
+		if (ret) {
+			if (ret > 0)
+				ret = 0; /* have to request lock */
+			goto out;
+		}
+
+		rbd_warn(rbd_dev, "%s%llu seems dead, breaking lock",
+			 ENTITY_NAME(lockers[0].id.name));
+
+		ret = ceph_monc_blacklist_add(&client->monc,
+					      &lockers[0].info.addr);
+		if (ret) {
+			rbd_warn(rbd_dev, "blacklist of %s%llu failed: %d",
+				 ENTITY_NAME(lockers[0].id.name), ret);
+			goto out;
+		}
+
+		ret = ceph_cls_break_lock(&client->osdc, &rbd_dev->header_oid,
+					  &rbd_dev->header_oloc, RBD_LOCK_NAME,
+					  lockers[0].id.cookie,
+					  &lockers[0].id.name);
+		if (ret && ret != -ENOENT)
+			goto out;
+
+again:
+		ceph_free_lockers(lockers, num_lockers);
+	}
+
+out:
+	ceph_free_lockers(lockers, num_lockers);
+	return ret;
+}
+
+/*
+ * ret is set only if lock_state is RBD_LOCK_STATE_UNLOCKED
+ */
+static enum rbd_lock_state rbd_try_acquire_lock(struct rbd_device *rbd_dev,
+						int *pret)
+{
+	enum rbd_lock_state lock_state;
+
+	down_read(&rbd_dev->lock_rwsem);
+	dout("%s rbd_dev %p read lock_state %d\n", __func__, rbd_dev,
+	     rbd_dev->lock_state);
+	if (__rbd_is_lock_owner(rbd_dev)) {
+		lock_state = rbd_dev->lock_state;
+		up_read(&rbd_dev->lock_rwsem);
+		return lock_state;
+	}
+
+	up_read(&rbd_dev->lock_rwsem);
+	down_write(&rbd_dev->lock_rwsem);
+	dout("%s rbd_dev %p write lock_state %d\n", __func__, rbd_dev,
+	     rbd_dev->lock_state);
+	if (!__rbd_is_lock_owner(rbd_dev)) {
+		*pret = rbd_try_lock(rbd_dev);
+		if (*pret)
+			rbd_warn(rbd_dev, "failed to acquire lock: %d", *pret);
+	}
+
+	lock_state = rbd_dev->lock_state;
+	up_write(&rbd_dev->lock_rwsem);
+	return lock_state;
+}
+
+static void rbd_acquire_lock(struct work_struct *work)
+{
+	struct rbd_device *rbd_dev = container_of(to_delayed_work(work),
+					    struct rbd_device, lock_dwork);
+	enum rbd_lock_state lock_state;
 	int ret;
 
-	dout("%s rbd_dev %p cookie %llu notify_id %llu\n", __func__, rbd_dev,
-	     cookie, notify_id);
+	dout("%s rbd_dev %p\n", __func__, rbd_dev);
+again:
+	lock_state = rbd_try_acquire_lock(rbd_dev, &ret);
+	if (lock_state != RBD_LOCK_STATE_UNLOCKED || ret == -EBLACKLISTED) {
+		if (lock_state == RBD_LOCK_STATE_LOCKED)
+			wake_requests(rbd_dev, true);
+		dout("%s rbd_dev %p lock_state %d ret %d - done\n", __func__,
+		     rbd_dev, lock_state, ret);
+		return;
+	}
+
+	ret = rbd_request_lock(rbd_dev);
+	if (ret == -ETIMEDOUT) {
+		goto again; /* treat this as a dead client */
+	} else if (ret < 0) {
+		rbd_warn(rbd_dev, "error requesting lock: %d", ret);
+		mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork,
+				 RBD_RETRY_DELAY);
+	} else {
+		/*
+		 * lock owner acked, but resend if we don't see them
+		 * release the lock
+		 */
+		dout("%s rbd_dev %p requeueing lock_dwork\n", __func__,
+		     rbd_dev);
+		mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork,
+		    msecs_to_jiffies(2 * RBD_NOTIFY_TIMEOUT * MSEC_PER_SEC));
+	}
+}
+
+/*
+ * lock_rwsem must be held for write
+ */
+static bool rbd_release_lock(struct rbd_device *rbd_dev)
+{
+	dout("%s rbd_dev %p read lock_state %d\n", __func__, rbd_dev,
+	     rbd_dev->lock_state);
+	if (rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED)
+		return false;
 
+	rbd_dev->lock_state = RBD_LOCK_STATE_RELEASING;
+	downgrade_write(&rbd_dev->lock_rwsem);
 	/*
-	 * Until adequate refresh error handling is in place, there is
-	 * not much we can do here, except warn.
+	 * Ensure that all in-flight IO is flushed.
 	 *
-	 * See http://tracker.ceph.com/issues/5040
+	 * FIXME: ceph_osdc_sync() flushes the entire OSD client, which
+	 * may be shared with other devices.
 	 */
-	ret = rbd_dev_refresh(rbd_dev);
-	if (ret)
-		rbd_warn(rbd_dev, "refresh failed: %d", ret);
+	ceph_osdc_sync(&rbd_dev->rbd_client->client->osdc);
+	up_read(&rbd_dev->lock_rwsem);
+
+	down_write(&rbd_dev->lock_rwsem);
+	dout("%s rbd_dev %p write lock_state %d\n", __func__, rbd_dev,
+	     rbd_dev->lock_state);
+	if (rbd_dev->lock_state != RBD_LOCK_STATE_RELEASING)
+		return false;
+
+	if (!rbd_unlock(rbd_dev))
+		/*
+		 * Give others a chance to grab the lock - we would re-acquire
+		 * almost immediately if we got new IO during ceph_osdc_sync()
+		 * otherwise.  We need to ack our own notifications, so this
+		 * lock_dwork will be requeued from rbd_wait_state_locked()
+		 * after wake_requests() in rbd_handle_released_lock().
+		 */
+		cancel_delayed_work(&rbd_dev->lock_dwork);
+
+	return true;
+}
+
+static void rbd_release_lock_work(struct work_struct *work)
+{
+	struct rbd_device *rbd_dev = container_of(work, struct rbd_device,
+						  unlock_work);
+
+	down_write(&rbd_dev->lock_rwsem);
+	rbd_release_lock(rbd_dev);
+	up_write(&rbd_dev->lock_rwsem);
+}
+
+static void rbd_handle_acquired_lock(struct rbd_device *rbd_dev, u8 struct_v,
+				     void **p)
+{
+	struct rbd_client_id cid = { 0 };
+
+	if (struct_v >= 2) {
+		cid.gid = ceph_decode_64(p);
+		cid.handle = ceph_decode_64(p);
+	}
+
+	dout("%s rbd_dev %p cid %llu-%llu\n", __func__, rbd_dev, cid.gid,
+	     cid.handle);
+	if (!rbd_cid_equal(&cid, &rbd_empty_cid)) {
+		down_write(&rbd_dev->lock_rwsem);
+		if (rbd_cid_equal(&cid, &rbd_dev->owner_cid)) {
+			/*
+			 * we already know that the remote client is
+			 * the owner
+			 */
+			up_write(&rbd_dev->lock_rwsem);
+			return;
+		}
+
+		rbd_set_owner_cid(rbd_dev, &cid);
+		downgrade_write(&rbd_dev->lock_rwsem);
+	} else {
+		down_read(&rbd_dev->lock_rwsem);
+	}
+
+	if (!__rbd_is_lock_owner(rbd_dev))
+		wake_requests(rbd_dev, false);
+	up_read(&rbd_dev->lock_rwsem);
+}
+
+static void rbd_handle_released_lock(struct rbd_device *rbd_dev, u8 struct_v,
+				     void **p)
+{
+	struct rbd_client_id cid = { 0 };
+
+	if (struct_v >= 2) {
+		cid.gid = ceph_decode_64(p);
+		cid.handle = ceph_decode_64(p);
+	}
+
+	dout("%s rbd_dev %p cid %llu-%llu\n", __func__, rbd_dev, cid.gid,
+	     cid.handle);
+	if (!rbd_cid_equal(&cid, &rbd_empty_cid)) {
+		down_write(&rbd_dev->lock_rwsem);
+		if (!rbd_cid_equal(&cid, &rbd_dev->owner_cid)) {
+			dout("%s rbd_dev %p unexpected owner, cid %llu-%llu != owner_cid %llu-%llu\n",
+			     __func__, rbd_dev, cid.gid, cid.handle,
+			     rbd_dev->owner_cid.gid, rbd_dev->owner_cid.handle);
+			up_write(&rbd_dev->lock_rwsem);
+			return;
+		}
+
+		rbd_set_owner_cid(rbd_dev, &rbd_empty_cid);
+		downgrade_write(&rbd_dev->lock_rwsem);
+	} else {
+		down_read(&rbd_dev->lock_rwsem);
+	}
+
+	if (!__rbd_is_lock_owner(rbd_dev))
+		wake_requests(rbd_dev, false);
+	up_read(&rbd_dev->lock_rwsem);
+}
+
+static bool rbd_handle_request_lock(struct rbd_device *rbd_dev, u8 struct_v,
+				    void **p)
+{
+	struct rbd_client_id my_cid = rbd_get_cid(rbd_dev);
+	struct rbd_client_id cid = { 0 };
+	bool need_to_send;
+
+	if (struct_v >= 2) {
+		cid.gid = ceph_decode_64(p);
+		cid.handle = ceph_decode_64(p);
+	}
+
+	dout("%s rbd_dev %p cid %llu-%llu\n", __func__, rbd_dev, cid.gid,
+	     cid.handle);
+	if (rbd_cid_equal(&cid, &my_cid))
+		return false;
+
+	down_read(&rbd_dev->lock_rwsem);
+	need_to_send = __rbd_is_lock_owner(rbd_dev);
+	if (rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED) {
+		if (!rbd_cid_equal(&rbd_dev->owner_cid, &rbd_empty_cid)) {
+			dout("%s rbd_dev %p queueing unlock_work\n", __func__,
+			     rbd_dev);
+			queue_work(rbd_dev->task_wq, &rbd_dev->unlock_work);
+		}
+	}
+	up_read(&rbd_dev->lock_rwsem);
+	return need_to_send;
+}
+
+static void __rbd_acknowledge_notify(struct rbd_device *rbd_dev,
+				     u64 notify_id, u64 cookie, s32 *result)
+{
+	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
+	int buf_size = 4 + CEPH_ENCODING_START_BLK_LEN;
+	char buf[buf_size];
+	int ret;
+
+	if (result) {
+		void *p = buf;
+
+		/* encode ResponseMessage */
+		ceph_start_encoding(&p, 1, 1,
+				    buf_size - CEPH_ENCODING_START_BLK_LEN);
+		ceph_encode_32(&p, *result);
+	} else {
+		buf_size = 0;
+	}
 
 	ret = ceph_osdc_notify_ack(osdc, &rbd_dev->header_oid,
 				   &rbd_dev->header_oloc, notify_id, cookie,
-				   NULL, 0);
+				   buf, buf_size);
 	if (ret)
-		rbd_warn(rbd_dev, "notify_ack ret %d", ret);
+		rbd_warn(rbd_dev, "acknowledge_notify failed: %d", ret);
 }
 
-static void rbd_watch_errcb(void *arg, u64 cookie, int err)
+static void rbd_acknowledge_notify(struct rbd_device *rbd_dev, u64 notify_id,
+				   u64 cookie)
+{
+	dout("%s rbd_dev %p\n", __func__, rbd_dev);
+	__rbd_acknowledge_notify(rbd_dev, notify_id, cookie, NULL);
+}
+
+static void rbd_acknowledge_notify_result(struct rbd_device *rbd_dev,
+					  u64 notify_id, u64 cookie, s32 result)
+{
+	dout("%s rbd_dev %p result %d\n", __func__, rbd_dev, result);
+	__rbd_acknowledge_notify(rbd_dev, notify_id, cookie, &result);
+}
+
+static void rbd_watch_cb(void *arg, u64 notify_id, u64 cookie,
+			 u64 notifier_id, void *data, size_t data_len)
 {
 	struct rbd_device *rbd_dev = arg;
+	void *p = data;
+	void *const end = p + data_len;
+	u8 struct_v;
+	u32 len;
+	u32 notify_op;
 	int ret;
 
-	rbd_warn(rbd_dev, "encountered watch error: %d", err);
+	dout("%s rbd_dev %p cookie %llu notify_id %llu data_len %zu\n",
+	     __func__, rbd_dev, cookie, notify_id, data_len);
+	if (data_len) {
+		ret = ceph_start_decoding(&p, end, 1, "NotifyMessage",
+					  &struct_v, &len);
+		if (ret) {
+			rbd_warn(rbd_dev, "failed to decode NotifyMessage: %d",
+				 ret);
+			return;
+		}
 
-	__rbd_dev_header_unwatch_sync(rbd_dev);
+		notify_op = ceph_decode_32(&p);
+	} else {
+		/* legacy notification for header updates */
+		notify_op = RBD_NOTIFY_OP_HEADER_UPDATE;
+		len = 0;
+	}
 
-	ret = rbd_dev_header_watch_sync(rbd_dev);
-	if (ret) {
-		rbd_warn(rbd_dev, "failed to reregister watch: %d", ret);
-		return;
+	dout("%s rbd_dev %p notify_op %u\n", __func__, rbd_dev, notify_op);
+	switch (notify_op) {
+	case RBD_NOTIFY_OP_ACQUIRED_LOCK:
+		rbd_handle_acquired_lock(rbd_dev, struct_v, &p);
+		rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
+		break;
+	case RBD_NOTIFY_OP_RELEASED_LOCK:
+		rbd_handle_released_lock(rbd_dev, struct_v, &p);
+		rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
+		break;
+	case RBD_NOTIFY_OP_REQUEST_LOCK:
+		if (rbd_handle_request_lock(rbd_dev, struct_v, &p))
+			/*
+			 * send ResponseMessage(0) back so the client
+			 * can detect a missing owner
+			 */
+			rbd_acknowledge_notify_result(rbd_dev, notify_id,
+						      cookie, 0);
+		else
+			rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
+		break;
+	case RBD_NOTIFY_OP_HEADER_UPDATE:
+		ret = rbd_dev_refresh(rbd_dev);
+		if (ret)
+			rbd_warn(rbd_dev, "refresh failed: %d", ret);
+
+		rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
+		break;
+	default:
+		if (rbd_is_lock_owner(rbd_dev))
+			rbd_acknowledge_notify_result(rbd_dev, notify_id,
+						      cookie, -EOPNOTSUPP);
+		else
+			rbd_acknowledge_notify(rbd_dev, notify_id, cookie);
+		break;
 	}
+}
 
-	ret = rbd_dev_refresh(rbd_dev);
-	if (ret)
-		rbd_warn(rbd_dev, "reregisteration refresh failed: %d", ret);
+static void __rbd_unregister_watch(struct rbd_device *rbd_dev);
+
+static void rbd_watch_errcb(void *arg, u64 cookie, int err)
+{
+	struct rbd_device *rbd_dev = arg;
+
+	rbd_warn(rbd_dev, "encountered watch error: %d", err);
+
+	down_write(&rbd_dev->lock_rwsem);
+	rbd_set_owner_cid(rbd_dev, &rbd_empty_cid);
+	up_write(&rbd_dev->lock_rwsem);
+
+	mutex_lock(&rbd_dev->watch_mutex);
+	if (rbd_dev->watch_state == RBD_WATCH_STATE_REGISTERED) {
+		__rbd_unregister_watch(rbd_dev);
+		rbd_dev->watch_state = RBD_WATCH_STATE_ERROR;
+
+		queue_delayed_work(rbd_dev->task_wq, &rbd_dev->watch_dwork, 0);
+	}
+	mutex_unlock(&rbd_dev->watch_mutex);
 }
 
 /*
- * Initiate a watch request, synchronously.
+ * watch_mutex must be locked
  */
-static int rbd_dev_header_watch_sync(struct rbd_device *rbd_dev)
+static int __rbd_register_watch(struct rbd_device *rbd_dev)
 {
 	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
 	struct ceph_osd_linger_request *handle;
 
 	rbd_assert(!rbd_dev->watch_handle);
+	dout("%s rbd_dev %p\n", __func__, rbd_dev);
 
 	handle = ceph_osdc_watch(osdc, &rbd_dev->header_oid,
 				 &rbd_dev->header_oloc, rbd_watch_cb,
@@ -3154,13 +3860,16 @@ static int rbd_dev_header_watch_sync(struct rbd_device *rbd_dev)
 	return 0;
 }
 
-static void __rbd_dev_header_unwatch_sync(struct rbd_device *rbd_dev)
+/*
+ * watch_mutex must be locked
+ */
+static void __rbd_unregister_watch(struct rbd_device *rbd_dev)
 {
 	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
 	int ret;
 
-	if (!rbd_dev->watch_handle)
-		return;
+	rbd_assert(rbd_dev->watch_handle);
+	dout("%s rbd_dev %p\n", __func__, rbd_dev);
 
 	ret = ceph_osdc_unwatch(osdc, rbd_dev->watch_handle);
 	if (ret)
@@ -3169,17 +3878,100 @@ static void __rbd_dev_header_unwatch_sync(struct rbd_device *rbd_dev)
 	rbd_dev->watch_handle = NULL;
 }
 
-/*
- * Tear down a watch request, synchronously.
- */
-static void rbd_dev_header_unwatch_sync(struct rbd_device *rbd_dev)
+static int rbd_register_watch(struct rbd_device *rbd_dev)
 {
-	__rbd_dev_header_unwatch_sync(rbd_dev);
+	int ret;
+
+	mutex_lock(&rbd_dev->watch_mutex);
+	rbd_assert(rbd_dev->watch_state == RBD_WATCH_STATE_UNREGISTERED);
+	ret = __rbd_register_watch(rbd_dev);
+	if (ret)
+		goto out;
+
+	rbd_dev->watch_state = RBD_WATCH_STATE_REGISTERED;
+	rbd_dev->watch_cookie = rbd_dev->watch_handle->linger_id;
+
+out:
+	mutex_unlock(&rbd_dev->watch_mutex);
+	return ret;
+}
+
+static void cancel_tasks_sync(struct rbd_device *rbd_dev)
+{
+	dout("%s rbd_dev %p\n", __func__, rbd_dev);
+
+	cancel_delayed_work_sync(&rbd_dev->watch_dwork);
+	cancel_work_sync(&rbd_dev->acquired_lock_work);
+	cancel_work_sync(&rbd_dev->released_lock_work);
+	cancel_delayed_work_sync(&rbd_dev->lock_dwork);
+	cancel_work_sync(&rbd_dev->unlock_work);
+}
+
+static void rbd_unregister_watch(struct rbd_device *rbd_dev)
+{
+	WARN_ON(waitqueue_active(&rbd_dev->lock_waitq));
+	cancel_tasks_sync(rbd_dev);
+
+	mutex_lock(&rbd_dev->watch_mutex);
+	if (rbd_dev->watch_state == RBD_WATCH_STATE_REGISTERED)
+		__rbd_unregister_watch(rbd_dev);
+	rbd_dev->watch_state = RBD_WATCH_STATE_UNREGISTERED;
+	mutex_unlock(&rbd_dev->watch_mutex);
 
-	dout("%s flushing notifies\n", __func__);
 	ceph_osdc_flush_notifies(&rbd_dev->rbd_client->client->osdc);
 }
 
+static void rbd_reregister_watch(struct work_struct *work)
+{
+	struct rbd_device *rbd_dev = container_of(to_delayed_work(work),
+					    struct rbd_device, watch_dwork);
+	bool was_lock_owner = false;
+	int ret;
+
+	dout("%s rbd_dev %p\n", __func__, rbd_dev);
+
+	down_write(&rbd_dev->lock_rwsem);
+	if (rbd_dev->lock_state == RBD_LOCK_STATE_LOCKED)
+		was_lock_owner = rbd_release_lock(rbd_dev);
+
+	mutex_lock(&rbd_dev->watch_mutex);
+	if (rbd_dev->watch_state != RBD_WATCH_STATE_ERROR)
+		goto fail_unlock;
+
+	ret = __rbd_register_watch(rbd_dev);
+	if (ret) {
+		rbd_warn(rbd_dev, "failed to reregister watch: %d", ret);
+		if (ret != -EBLACKLISTED)
+			queue_delayed_work(rbd_dev->task_wq,
+					   &rbd_dev->watch_dwork,
+					   RBD_RETRY_DELAY);
+		goto fail_unlock;
+	}
+
+	rbd_dev->watch_state = RBD_WATCH_STATE_REGISTERED;
+	rbd_dev->watch_cookie = rbd_dev->watch_handle->linger_id;
+	mutex_unlock(&rbd_dev->watch_mutex);
+
+	ret = rbd_dev_refresh(rbd_dev);
+	if (ret)
+		rbd_warn(rbd_dev, "reregisteration refresh failed: %d", ret);
+
+	if (was_lock_owner) {
+		ret = rbd_try_lock(rbd_dev);
+		if (ret)
+			rbd_warn(rbd_dev, "reregisteration lock failed: %d",
+				 ret);
+	}
+
+	up_write(&rbd_dev->lock_rwsem);
+	wake_requests(rbd_dev, true);
+	return;
+
+fail_unlock:
+	mutex_unlock(&rbd_dev->watch_mutex);
+	up_write(&rbd_dev->lock_rwsem);
+}
+
 /*
  * Synchronous osd object method call.  Returns the number of bytes
  * returned in the outbound buffer, or a negative error code.
@@ -3193,7 +3985,6 @@ static int rbd_obj_method_sync(struct rbd_device *rbd_dev,
 			     void *inbound,
 			     size_t inbound_size)
 {
-	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
 	struct rbd_obj_request *obj_request;
 	struct page **pages;
 	u32 page_count;
@@ -3242,11 +4033,8 @@ static int rbd_obj_method_sync(struct rbd_device *rbd_dev,
 	osd_req_op_cls_response_data_pages(obj_request->osd_req, 0,
 					obj_request->pages, inbound_size,
 					0, false, false);
-	rbd_osd_req_format_read(obj_request);
 
-	ret = rbd_obj_request_submit(osdc, obj_request);
-	if (ret)
-		goto out;
+	rbd_obj_request_submit(obj_request);
 	ret = rbd_obj_request_wait(obj_request);
 	if (ret)
 		goto out;
@@ -3267,6 +4055,29 @@ out:
 	return ret;
 }
 
+/*
+ * lock_rwsem must be held for read
+ */
+static void rbd_wait_state_locked(struct rbd_device *rbd_dev)
+{
+	DEFINE_WAIT(wait);
+
+	do {
+		/*
+		 * Note the use of mod_delayed_work() in rbd_acquire_lock()
+		 * and cancel_delayed_work() in wake_requests().
+		 */
+		dout("%s rbd_dev %p queueing lock_dwork\n", __func__, rbd_dev);
+		queue_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork, 0);
+		prepare_to_wait_exclusive(&rbd_dev->lock_waitq, &wait,
+					  TASK_UNINTERRUPTIBLE);
+		up_read(&rbd_dev->lock_rwsem);
+		schedule();
+		down_read(&rbd_dev->lock_rwsem);
+	} while (rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED);
+	finish_wait(&rbd_dev->lock_waitq, &wait);
+}
+
 static void rbd_queue_workfn(struct work_struct *work)
 {
 	struct request *rq = blk_mq_rq_from_pdu(work);
@@ -3277,6 +4088,7 @@ static void rbd_queue_workfn(struct work_struct *work)
 	u64 length = blk_rq_bytes(rq);
 	enum obj_operation_type op_type;
 	u64 mapping_size;
+	bool must_be_locked;
 	int result;
 
 	if (rq->cmd_type != REQ_TYPE_FS) {
@@ -3338,6 +4150,10 @@ static void rbd_queue_workfn(struct work_struct *work)
 	if (op_type != OBJ_OP_READ) {
 		snapc = rbd_dev->header.snapc;
 		ceph_get_snap_context(snapc);
+		must_be_locked = rbd_is_lock_supported(rbd_dev);
+	} else {
+		must_be_locked = rbd_dev->opts->lock_on_read &&
+					rbd_is_lock_supported(rbd_dev);
 	}
 	up_read(&rbd_dev->header_rwsem);
 
@@ -3348,11 +4164,17 @@ static void rbd_queue_workfn(struct work_struct *work)
 		goto err_rq;
 	}
 
+	if (must_be_locked) {
+		down_read(&rbd_dev->lock_rwsem);
+		if (rbd_dev->lock_state != RBD_LOCK_STATE_LOCKED)
+			rbd_wait_state_locked(rbd_dev);
+	}
+
 	img_request = rbd_img_request_create(rbd_dev, offset, length, op_type,
 					     snapc);
 	if (!img_request) {
 		result = -ENOMEM;
-		goto err_rq;
+		goto err_unlock;
 	}
 	img_request->rq = rq;
 	snapc = NULL; /* img_request consumes a ref */
@@ -3370,10 +4192,15 @@ static void rbd_queue_workfn(struct work_struct *work)
 	if (result)
 		goto err_img_request;
 
+	if (must_be_locked)
+		up_read(&rbd_dev->lock_rwsem);
 	return;
 
 err_img_request:
 	rbd_img_request_put(img_request);
+err_unlock:
+	if (must_be_locked)
+		up_read(&rbd_dev->lock_rwsem);
 err_rq:
 	if (result)
 		rbd_warn(rbd_dev, "%s %llx at %llx result %d",
@@ -3415,7 +4242,6 @@ static int rbd_obj_read_sync(struct rbd_device *rbd_dev,
 				u64 offset, u64 length, void *buf)
 
 {
-	struct ceph_osd_client *osdc = &rbd_dev->rbd_client->client->osdc;
 	struct rbd_obj_request *obj_request;
 	struct page **pages = NULL;
 	u32 page_count;
@@ -3448,11 +4274,8 @@ static int rbd_obj_read_sync(struct rbd_device *rbd_dev,
 					obj_request->length,
 					obj_request->offset & ~PAGE_MASK,
 					false, false);
-	rbd_osd_req_format_read(obj_request);
 
-	ret = rbd_obj_request_submit(osdc, obj_request);
-	if (ret)
-		goto out;
+	rbd_obj_request_submit(obj_request);
 	ret = rbd_obj_request_wait(obj_request);
 	if (ret)
 		goto out;
@@ -3621,7 +4444,6 @@ static int rbd_init_request(void *data, struct request *rq,
 
 static struct blk_mq_ops rbd_mq_ops = {
 	.queue_rq	= rbd_queue_rq,
-	.map_queue	= blk_mq_map_queue,
 	.init_request	= rbd_init_request,
 };
 
@@ -3752,13 +4574,40 @@ static ssize_t rbd_minor_show(struct device *dev,
 	return sprintf(buf, "%d\n", rbd_dev->minor);
 }
 
+static ssize_t rbd_client_addr_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
+	struct ceph_entity_addr *client_addr =
+	    ceph_client_addr(rbd_dev->rbd_client->client);
+
+	return sprintf(buf, "%pISpc/%u\n", &client_addr->in_addr,
+		       le32_to_cpu(client_addr->nonce));
+}
+
 static ssize_t rbd_client_id_show(struct device *dev,
 				  struct device_attribute *attr, char *buf)
 {
 	struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
 
 	return sprintf(buf, "client%lld\n",
-			ceph_client_id(rbd_dev->rbd_client->client));
+		       ceph_client_gid(rbd_dev->rbd_client->client));
+}
+
+static ssize_t rbd_cluster_fsid_show(struct device *dev,
+				     struct device_attribute *attr, char *buf)
+{
+	struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
+
+	return sprintf(buf, "%pU\n", &rbd_dev->rbd_client->client->fsid);
+}
+
+static ssize_t rbd_config_info_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
+
+	return sprintf(buf, "%s\n", rbd_dev->config_info);
 }
 
 static ssize_t rbd_pool_show(struct device *dev,
@@ -3810,6 +4659,14 @@ static ssize_t rbd_snap_show(struct device *dev,
 	return sprintf(buf, "%s\n", rbd_dev->spec->snap_name);
 }
 
+static ssize_t rbd_snap_id_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
+
+	return sprintf(buf, "%llu\n", rbd_dev->spec->snap_id);
+}
+
 /*
  * For a v2 image, shows the chain of parent images, separated by empty
  * lines.  For v1 images or if there is no parent, shows "(no parent
@@ -3862,13 +4719,17 @@ static DEVICE_ATTR(size, S_IRUGO, rbd_size_show, NULL);
 static DEVICE_ATTR(features, S_IRUGO, rbd_features_show, NULL);
 static DEVICE_ATTR(major, S_IRUGO, rbd_major_show, NULL);
 static DEVICE_ATTR(minor, S_IRUGO, rbd_minor_show, NULL);
+static DEVICE_ATTR(client_addr, S_IRUGO, rbd_client_addr_show, NULL);
 static DEVICE_ATTR(client_id, S_IRUGO, rbd_client_id_show, NULL);
+static DEVICE_ATTR(cluster_fsid, S_IRUGO, rbd_cluster_fsid_show, NULL);
+static DEVICE_ATTR(config_info, S_IRUSR, rbd_config_info_show, NULL);
 static DEVICE_ATTR(pool, S_IRUGO, rbd_pool_show, NULL);
 static DEVICE_ATTR(pool_id, S_IRUGO, rbd_pool_id_show, NULL);
 static DEVICE_ATTR(name, S_IRUGO, rbd_name_show, NULL);
 static DEVICE_ATTR(image_id, S_IRUGO, rbd_image_id_show, NULL);
 static DEVICE_ATTR(refresh, S_IWUSR, NULL, rbd_image_refresh);
 static DEVICE_ATTR(current_snap, S_IRUGO, rbd_snap_show, NULL);
+static DEVICE_ATTR(snap_id, S_IRUGO, rbd_snap_id_show, NULL);
 static DEVICE_ATTR(parent, S_IRUGO, rbd_parent_show, NULL);
 
 static struct attribute *rbd_attrs[] = {
@@ -3876,12 +4737,16 @@ static struct attribute *rbd_attrs[] = {
 	&dev_attr_features.attr,
 	&dev_attr_major.attr,
 	&dev_attr_minor.attr,
+	&dev_attr_client_addr.attr,
 	&dev_attr_client_id.attr,
+	&dev_attr_cluster_fsid.attr,
+	&dev_attr_config_info.attr,
 	&dev_attr_pool.attr,
 	&dev_attr_pool_id.attr,
 	&dev_attr_name.attr,
 	&dev_attr_image_id.attr,
 	&dev_attr_current_snap.attr,
+	&dev_attr_snap_id.attr,
 	&dev_attr_parent.attr,
 	&dev_attr_refresh.attr,
 	NULL
@@ -3944,17 +4809,32 @@ static void rbd_spec_free(struct kref *kref)
 	kfree(spec);
 }
 
-static void rbd_dev_release(struct device *dev)
+static void rbd_dev_free(struct rbd_device *rbd_dev)
 {
-	struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
-	bool need_put = !!rbd_dev->opts;
+	WARN_ON(rbd_dev->watch_state != RBD_WATCH_STATE_UNREGISTERED);
+	WARN_ON(rbd_dev->lock_state != RBD_LOCK_STATE_UNLOCKED);
 
 	ceph_oid_destroy(&rbd_dev->header_oid);
+	ceph_oloc_destroy(&rbd_dev->header_oloc);
+	kfree(rbd_dev->config_info);
 
 	rbd_put_client(rbd_dev->rbd_client);
 	rbd_spec_put(rbd_dev->spec);
 	kfree(rbd_dev->opts);
 	kfree(rbd_dev);
+}
+
+static void rbd_dev_release(struct device *dev)
+{
+	struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
+	bool need_put = !!rbd_dev->opts;
+
+	if (need_put) {
+		destroy_workqueue(rbd_dev->task_wq);
+		ida_simple_remove(&rbd_dev_id_ida, rbd_dev->dev_id);
+	}
+
+	rbd_dev_free(rbd_dev);
 
 	/*
 	 * This is racy, but way better than putting module outside of
@@ -3965,25 +4845,34 @@ static void rbd_dev_release(struct device *dev)
 		module_put(THIS_MODULE);
 }
 
-static struct rbd_device *rbd_dev_create(struct rbd_client *rbdc,
-					 struct rbd_spec *spec,
-					 struct rbd_options *opts)
+static struct rbd_device *__rbd_dev_create(struct rbd_client *rbdc,
+					   struct rbd_spec *spec)
 {
 	struct rbd_device *rbd_dev;
 
-	rbd_dev = kzalloc(sizeof (*rbd_dev), GFP_KERNEL);
+	rbd_dev = kzalloc(sizeof(*rbd_dev), GFP_KERNEL);
 	if (!rbd_dev)
 		return NULL;
 
 	spin_lock_init(&rbd_dev->lock);
-	rbd_dev->flags = 0;
-	atomic_set(&rbd_dev->parent_ref, 0);
 	INIT_LIST_HEAD(&rbd_dev->node);
 	init_rwsem(&rbd_dev->header_rwsem);
 
 	ceph_oid_init(&rbd_dev->header_oid);
 	ceph_oloc_init(&rbd_dev->header_oloc);
 
+	mutex_init(&rbd_dev->watch_mutex);
+	rbd_dev->watch_state = RBD_WATCH_STATE_UNREGISTERED;
+	INIT_DELAYED_WORK(&rbd_dev->watch_dwork, rbd_reregister_watch);
+
+	init_rwsem(&rbd_dev->lock_rwsem);
+	rbd_dev->lock_state = RBD_LOCK_STATE_UNLOCKED;
+	INIT_WORK(&rbd_dev->acquired_lock_work, rbd_notify_acquired_lock);
+	INIT_WORK(&rbd_dev->released_lock_work, rbd_notify_released_lock);
+	INIT_DELAYED_WORK(&rbd_dev->lock_dwork, rbd_acquire_lock);
+	INIT_WORK(&rbd_dev->unlock_work, rbd_release_lock_work);
+	init_waitqueue_head(&rbd_dev->lock_waitq);
+
 	rbd_dev->dev.bus = &rbd_bus_type;
 	rbd_dev->dev.type = &rbd_device_type;
 	rbd_dev->dev.parent = &rbd_root_dev;
@@ -3991,9 +4880,6 @@ static struct rbd_device *rbd_dev_create(struct rbd_client *rbdc,
 
 	rbd_dev->rbd_client = rbdc;
 	rbd_dev->spec = spec;
-	rbd_dev->opts = opts;
-
-	/* Initialize the layout used for all rbd requests */
 
 	rbd_dev->layout.stripe_unit = 1 << RBD_MAX_OBJ_ORDER;
 	rbd_dev->layout.stripe_count = 1;
@@ -4001,15 +4887,48 @@ static struct rbd_device *rbd_dev_create(struct rbd_client *rbdc,
 	rbd_dev->layout.pool_id = spec->pool_id;
 	RCU_INIT_POINTER(rbd_dev->layout.pool_ns, NULL);
 
-	/*
-	 * If this is a mapping rbd_dev (as opposed to a parent one),
-	 * pin our module.  We have a ref from do_rbd_add(), so use
-	 * __module_get().
-	 */
-	if (rbd_dev->opts)
-		__module_get(THIS_MODULE);
+	return rbd_dev;
+}
+
+/*
+ * Create a mapping rbd_dev.
+ */
+static struct rbd_device *rbd_dev_create(struct rbd_client *rbdc,
+					 struct rbd_spec *spec,
+					 struct rbd_options *opts)
+{
+	struct rbd_device *rbd_dev;
+
+	rbd_dev = __rbd_dev_create(rbdc, spec);
+	if (!rbd_dev)
+		return NULL;
+
+	rbd_dev->opts = opts;
+
+	/* get an id and fill in device name */
+	rbd_dev->dev_id = ida_simple_get(&rbd_dev_id_ida, 0,
+					 minor_to_rbd_dev_id(1 << MINORBITS),
+					 GFP_KERNEL);
+	if (rbd_dev->dev_id < 0)
+		goto fail_rbd_dev;
 
+	sprintf(rbd_dev->name, RBD_DRV_NAME "%d", rbd_dev->dev_id);
+	rbd_dev->task_wq = alloc_ordered_workqueue("%s-tasks", WQ_MEM_RECLAIM,
+						   rbd_dev->name);
+	if (!rbd_dev->task_wq)
+		goto fail_dev_id;
+
+	/* we have a ref from do_rbd_add() */
+	__module_get(THIS_MODULE);
+
+	dout("%s rbd_dev %p dev_id %d\n", __func__, rbd_dev, rbd_dev->dev_id);
 	return rbd_dev;
+
+fail_dev_id:
+	ida_simple_remove(&rbd_dev_id_ida, rbd_dev->dev_id);
+fail_rbd_dev:
+	rbd_dev_free(rbd_dev);
+	return NULL;
 }
 
 static void rbd_dev_destroy(struct rbd_device *rbd_dev)
@@ -4645,46 +5564,6 @@ static int rbd_dev_header_info(struct rbd_device *rbd_dev)
 }
 
 /*
- * Get a unique rbd identifier for the given new rbd_dev, and add
- * the rbd_dev to the global list.
- */
-static int rbd_dev_id_get(struct rbd_device *rbd_dev)
-{
-	int new_dev_id;
-
-	new_dev_id = ida_simple_get(&rbd_dev_id_ida,
-				    0, minor_to_rbd_dev_id(1 << MINORBITS),
-				    GFP_KERNEL);
-	if (new_dev_id < 0)
-		return new_dev_id;
-
-	rbd_dev->dev_id = new_dev_id;
-
-	spin_lock(&rbd_dev_list_lock);
-	list_add_tail(&rbd_dev->node, &rbd_dev_list);
-	spin_unlock(&rbd_dev_list_lock);
-
-	dout("rbd_dev %p given dev id %d\n", rbd_dev, rbd_dev->dev_id);
-
-	return 0;
-}
-
-/*
- * Remove an rbd_dev from the global list, and record that its
- * identifier is no longer in use.
- */
-static void rbd_dev_id_put(struct rbd_device *rbd_dev)
-{
-	spin_lock(&rbd_dev_list_lock);
-	list_del_init(&rbd_dev->node);
-	spin_unlock(&rbd_dev_list_lock);
-
-	ida_simple_remove(&rbd_dev_id_ida, rbd_dev->dev_id);
-
-	dout("rbd_dev %p released dev id %d\n", rbd_dev, rbd_dev->dev_id);
-}
-
-/*
  * Skips over white space at *buf, and updates *buf to point to the
  * first found non-space character (if any). Returns the length of
  * the token (string of non-white space characters) found.  Note
@@ -4859,6 +5738,7 @@ static int rbd_add_parse_args(const char *buf,
 
 	rbd_opts->read_only = RBD_READ_ONLY_DEFAULT;
 	rbd_opts->queue_depth = RBD_QUEUE_DEPTH_DEFAULT;
+	rbd_opts->lock_on_read = RBD_LOCK_ON_READ_DEFAULT;
 
 	copts = ceph_parse_options(options, mon_addrs,
 					mon_addrs + mon_addrs_size - 1,
@@ -5076,8 +5956,7 @@ static int rbd_dev_probe_parent(struct rbd_device *rbd_dev, int depth)
 		goto out_err;
 	}
 
-	parent = rbd_dev_create(rbd_dev->rbd_client, rbd_dev->parent_spec,
-				NULL);
+	parent = __rbd_dev_create(rbd_dev->rbd_client, rbd_dev->parent_spec);
 	if (!parent) {
 		ret = -ENOMEM;
 		goto out_err;
@@ -5112,22 +5991,12 @@ static int rbd_dev_device_setup(struct rbd_device *rbd_dev)
 {
 	int ret;
 
-	/* Get an id and fill in device name. */
-
-	ret = rbd_dev_id_get(rbd_dev);
-	if (ret)
-		goto err_out_unlock;
-
-	BUILD_BUG_ON(DEV_NAME_LEN
-			< sizeof (RBD_DRV_NAME) + MAX_INT_FORMAT_WIDTH);
-	sprintf(rbd_dev->name, "%s%d", RBD_DRV_NAME, rbd_dev->dev_id);
-
 	/* Record our major and minor device numbers. */
 
 	if (!single_major) {
 		ret = register_blkdev(0, rbd_dev->name);
 		if (ret < 0)
-			goto err_out_id;
+			goto err_out_unlock;
 
 		rbd_dev->major = ret;
 		rbd_dev->minor = 0;
@@ -5159,9 +6028,14 @@ static int rbd_dev_device_setup(struct rbd_device *rbd_dev)
 	set_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags);
 	up_write(&rbd_dev->header_rwsem);
 
+	spin_lock(&rbd_dev_list_lock);
+	list_add_tail(&rbd_dev->node, &rbd_dev_list);
+	spin_unlock(&rbd_dev_list_lock);
+
 	add_disk(rbd_dev->disk);
-	pr_info("%s: added with size 0x%llx\n", rbd_dev->disk->disk_name,
-		(unsigned long long) rbd_dev->mapping.size);
+	pr_info("%s: capacity %llu features 0x%llx\n", rbd_dev->disk->disk_name,
+		(unsigned long long)get_capacity(rbd_dev->disk) << SECTOR_SHIFT,
+		rbd_dev->header.features);
 
 	return ret;
 
@@ -5172,8 +6046,6 @@ err_out_disk:
 err_out_blkdev:
 	if (!single_major)
 		unregister_blkdev(rbd_dev->major, rbd_dev->name);
-err_out_id:
-	rbd_dev_id_put(rbd_dev);
 err_out_unlock:
 	up_write(&rbd_dev->header_rwsem);
 	return ret;
@@ -5234,7 +6106,7 @@ static int rbd_dev_image_probe(struct rbd_device *rbd_dev, int depth)
 		goto err_out_format;
 
 	if (!depth) {
-		ret = rbd_dev_header_watch_sync(rbd_dev);
+		ret = rbd_register_watch(rbd_dev);
 		if (ret) {
 			if (ret == -ENOENT)
 				pr_info("image %s/%s does not exist\n",
@@ -5293,7 +6165,7 @@ err_out_probe:
 	rbd_dev_unprobe(rbd_dev);
 err_out_watch:
 	if (!depth)
-		rbd_dev_header_unwatch_sync(rbd_dev);
+		rbd_unregister_watch(rbd_dev);
 err_out_format:
 	rbd_dev->image_format = 0;
 	kfree(rbd_dev->spec->image_id);
@@ -5336,15 +6208,6 @@ static ssize_t do_rbd_add(struct bus_type *bus,
 	}
 	spec->pool_id = (u64)rc;
 
-	/* The ceph file layout needs to fit pool id in 32 bits */
-
-	if (spec->pool_id > (u64)U32_MAX) {
-		rbd_warn(NULL, "pool id too large (%llu > %u)",
-				(unsigned long long)spec->pool_id, U32_MAX);
-		rc = -EIO;
-		goto err_out_client;
-	}
-
 	rbd_dev = rbd_dev_create(rbdc, spec, rbd_opts);
 	if (!rbd_dev) {
 		rc = -ENOMEM;
@@ -5354,10 +6217,18 @@ static ssize_t do_rbd_add(struct bus_type *bus,
 	spec = NULL;		/* rbd_dev now owns this */
 	rbd_opts = NULL;	/* rbd_dev now owns this */
 
+	rbd_dev->config_info = kstrdup(buf, GFP_KERNEL);
+	if (!rbd_dev->config_info) {
+		rc = -ENOMEM;
+		goto err_out_rbd_dev;
+	}
+
 	down_write(&rbd_dev->header_rwsem);
 	rc = rbd_dev_image_probe(rbd_dev, 0);
-	if (rc < 0)
+	if (rc < 0) {
+		up_write(&rbd_dev->header_rwsem);
 		goto err_out_rbd_dev;
+	}
 
 	/* If we are mapping a snapshot it must be marked read-only */
 
@@ -5369,11 +6240,11 @@ static ssize_t do_rbd_add(struct bus_type *bus,
 	rc = rbd_dev_device_setup(rbd_dev);
 	if (rc) {
 		/*
-		 * rbd_dev_header_unwatch_sync() can't be moved into
+		 * rbd_unregister_watch() can't be moved into
 		 * rbd_dev_image_release() without refactoring, see
 		 * commit 1f3ef78861ac.
 		 */
-		rbd_dev_header_unwatch_sync(rbd_dev);
+		rbd_unregister_watch(rbd_dev);
 		rbd_dev_image_release(rbd_dev);
 		goto out;
 	}
@@ -5384,7 +6255,6 @@ out:
 	return rc;
 
 err_out_rbd_dev:
-	up_write(&rbd_dev->header_rwsem);
 	rbd_dev_destroy(rbd_dev);
 err_out_client:
 	rbd_put_client(rbdc);
@@ -5414,12 +6284,16 @@ static ssize_t rbd_add_single_major(struct bus_type *bus,
 static void rbd_dev_device_release(struct rbd_device *rbd_dev)
 {
 	rbd_free_disk(rbd_dev);
+
+	spin_lock(&rbd_dev_list_lock);
+	list_del_init(&rbd_dev->node);
+	spin_unlock(&rbd_dev_list_lock);
+
 	clear_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags);
 	device_del(&rbd_dev->dev);
 	rbd_dev_mapping_clear(rbd_dev);
 	if (!single_major)
 		unregister_blkdev(rbd_dev->major, rbd_dev->name);
-	rbd_dev_id_put(rbd_dev);
 }
 
 static void rbd_dev_remove_parent(struct rbd_device *rbd_dev)
@@ -5455,18 +6329,26 @@ static ssize_t do_rbd_remove(struct bus_type *bus,
 	struct rbd_device *rbd_dev = NULL;
 	struct list_head *tmp;
 	int dev_id;
-	unsigned long ul;
+	char opt_buf[6];
 	bool already = false;
+	bool force = false;
 	int ret;
 
-	ret = kstrtoul(buf, 10, &ul);
-	if (ret)
-		return ret;
-
-	/* convert to int; abort if we lost anything in the conversion */
-	dev_id = (int)ul;
-	if (dev_id != ul)
+	dev_id = -1;
+	opt_buf[0] = '\0';
+	sscanf(buf, "%d %5s", &dev_id, opt_buf);
+	if (dev_id < 0) {
+		pr_err("dev_id out of range\n");
 		return -EINVAL;
+	}
+	if (opt_buf[0] != '\0') {
+		if (!strcmp(opt_buf, "force")) {
+			force = true;
+		} else {
+			pr_err("bad remove option at '%s'\n", opt_buf);
+			return -EINVAL;
+		}
+	}
 
 	ret = -ENOENT;
 	spin_lock(&rbd_dev_list_lock);
@@ -5479,7 +6361,7 @@ static ssize_t do_rbd_remove(struct bus_type *bus,
 	}
 	if (!ret) {
 		spin_lock_irq(&rbd_dev->lock);
-		if (rbd_dev->open_count)
+		if (rbd_dev->open_count && !force)
 			ret = -EBUSY;
 		else
 			already = test_and_set_bit(RBD_DEV_FLAG_REMOVING,
@@ -5490,7 +6372,20 @@ static ssize_t do_rbd_remove(struct bus_type *bus,
 	if (ret < 0 || already)
 		return ret;
 
-	rbd_dev_header_unwatch_sync(rbd_dev);
+	if (force) {
+		/*
+		 * Prevent new IO from being queued and wait for existing
+		 * IO to complete/fail.
+		 */
+		blk_mq_freeze_queue(rbd_dev->disk->queue);
+		blk_set_queue_dying(rbd_dev->disk->queue);
+	}
+
+	down_write(&rbd_dev->lock_rwsem);
+	if (__rbd_is_lock_owner(rbd_dev))
+		rbd_unlock(rbd_dev);
+	up_write(&rbd_dev->lock_rwsem);
+	rbd_unregister_watch(rbd_dev);
 
 	/*
 	 * Don't free anything from rbd_dev->disk until after all
diff --git a/drivers/block/rbd_types.h b/drivers/block/rbd_types.h
index 49d77cbcf8bd..94f367db27b0 100644
--- a/drivers/block/rbd_types.h
+++ b/drivers/block/rbd_types.h
@@ -28,6 +28,17 @@
 #define RBD_DATA_PREFIX        "rbd_data."
 #define RBD_ID_PREFIX          "rbd_id."
 
+#define RBD_LOCK_NAME          "rbd_lock"
+#define RBD_LOCK_TAG           "internal"
+#define RBD_LOCK_COOKIE_PREFIX "auto"
+
+enum rbd_notify_op {
+	RBD_NOTIFY_OP_ACQUIRED_LOCK      = 0,
+	RBD_NOTIFY_OP_RELEASED_LOCK      = 1,
+	RBD_NOTIFY_OP_REQUEST_LOCK       = 2,
+	RBD_NOTIFY_OP_HEADER_UPDATE      = 3,
+};
+
 /*
  * For format version 1, rbd image 'foo' consists of objects
  *   foo.rbd		- image metadata
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 1523e05c46fc..2dc5c96c186a 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -391,22 +391,16 @@ static int init_vq(struct virtio_blk *vblk)
 		num_vqs = 1;
 
 	vblk->vqs = kmalloc(sizeof(*vblk->vqs) * num_vqs, GFP_KERNEL);
-	if (!vblk->vqs) {
-		err = -ENOMEM;
-		goto out;
-	}
+	if (!vblk->vqs)
+		return -ENOMEM;
 
 	names = kmalloc(sizeof(*names) * num_vqs, GFP_KERNEL);
-	if (!names)
-		goto err_names;
-
 	callbacks = kmalloc(sizeof(*callbacks) * num_vqs, GFP_KERNEL);
-	if (!callbacks)
-		goto err_callbacks;
-
 	vqs = kmalloc(sizeof(*vqs) * num_vqs, GFP_KERNEL);
-	if (!vqs)
-		goto err_vqs;
+	if (!names || !callbacks || !vqs) {
+		err = -ENOMEM;
+		goto out;
+	}
 
 	for (i = 0; i < num_vqs; i++) {
 		callbacks[i] = virtblk_done;
@@ -417,7 +411,7 @@ static int init_vq(struct virtio_blk *vblk)
 	/* Discover virtqueues and write information to configuration.  */
 	err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names);
 	if (err)
-		goto err_find_vqs;
+		goto out;
 
 	for (i = 0; i < num_vqs; i++) {
 		spin_lock_init(&vblk->vqs[i].lock);
@@ -425,16 +419,12 @@ static int init_vq(struct virtio_blk *vblk)
 	}
 	vblk->num_vqs = num_vqs;
 
- err_find_vqs:
+out:
 	kfree(vqs);
- err_vqs:
 	kfree(callbacks);
- err_callbacks:
 	kfree(names);
- err_names:
 	if (err)
 		kfree(vblk->vqs);
- out:
 	return err;
 }
 
@@ -552,7 +542,6 @@ static int virtblk_init_request(void *data, struct request *rq,
 
 static struct blk_mq_ops virtio_mq_ops = {
 	.queue_rq	= virtio_queue_rq,
-	.map_queue	= blk_mq_map_queue,
 	.complete	= virtblk_request_done,
 	.init_request	= virtblk_init_request,
 };
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
index be4fea6a5dd3..9908597c5209 100644
--- a/drivers/block/xen-blkfront.c
+++ b/drivers/block/xen-blkfront.c
@@ -189,6 +189,8 @@ struct blkfront_info
 	struct mutex mutex;
 	struct xenbus_device *xbdev;
 	struct gendisk *gd;
+	u16 sector_size;
+	unsigned int physical_sector_size;
 	int vdevice;
 	blkif_vdev_t handle;
 	enum blkif_state connected;
@@ -907,12 +909,47 @@ out_busy:
 
 static struct blk_mq_ops blkfront_mq_ops = {
 	.queue_rq = blkif_queue_rq,
-	.map_queue = blk_mq_map_queue,
 };
 
+static void blkif_set_queue_limits(struct blkfront_info *info)
+{
+	struct request_queue *rq = info->rq;
+	struct gendisk *gd = info->gd;
+	unsigned int segments = info->max_indirect_segments ? :
+				BLKIF_MAX_SEGMENTS_PER_REQUEST;
+
+	queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq);
+
+	if (info->feature_discard) {
+		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, rq);
+		blk_queue_max_discard_sectors(rq, get_capacity(gd));
+		rq->limits.discard_granularity = info->discard_granularity;
+		rq->limits.discard_alignment = info->discard_alignment;
+		if (info->feature_secdiscard)
+			queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, rq);
+	}
+
+	/* Hard sector size and max sectors impersonate the equiv. hardware. */
+	blk_queue_logical_block_size(rq, info->sector_size);
+	blk_queue_physical_block_size(rq, info->physical_sector_size);
+	blk_queue_max_hw_sectors(rq, (segments * XEN_PAGE_SIZE) / 512);
+
+	/* Each segment in a request is up to an aligned page in size. */
+	blk_queue_segment_boundary(rq, PAGE_SIZE - 1);
+	blk_queue_max_segment_size(rq, PAGE_SIZE);
+
+	/* Ensure a merged request will fit in a single I/O ring slot. */
+	blk_queue_max_segments(rq, segments / GRANTS_PER_PSEG);
+
+	/* Make sure buffer addresses are sector-aligned. */
+	blk_queue_dma_alignment(rq, 511);
+
+	/* Make sure we don't use bounce buffers. */
+	blk_queue_bounce_limit(rq, BLK_BOUNCE_ANY);
+}
+
 static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size,
-				unsigned int physical_sector_size,
-				unsigned int segments)
+				unsigned int physical_sector_size)
 {
 	struct request_queue *rq;
 	struct blkfront_info *info = gd->private_data;
@@ -944,36 +981,11 @@ static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size,
 	}
 
 	rq->queuedata = info;
-	queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq);
-
-	if (info->feature_discard) {
-		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, rq);
-		blk_queue_max_discard_sectors(rq, get_capacity(gd));
-		rq->limits.discard_granularity = info->discard_granularity;
-		rq->limits.discard_alignment = info->discard_alignment;
-		if (info->feature_secdiscard)
-			queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, rq);
-	}
-
-	/* Hard sector size and max sectors impersonate the equiv. hardware. */
-	blk_queue_logical_block_size(rq, sector_size);
-	blk_queue_physical_block_size(rq, physical_sector_size);
-	blk_queue_max_hw_sectors(rq, (segments * XEN_PAGE_SIZE) / 512);
-
-	/* Each segment in a request is up to an aligned page in size. */
-	blk_queue_segment_boundary(rq, PAGE_SIZE - 1);
-	blk_queue_max_segment_size(rq, PAGE_SIZE);
-
-	/* Ensure a merged request will fit in a single I/O ring slot. */
-	blk_queue_max_segments(rq, segments / GRANTS_PER_PSEG);
-
-	/* Make sure buffer addresses are sector-aligned. */
-	blk_queue_dma_alignment(rq, 511);
-
-	/* Make sure we don't use bounce buffers. */
-	blk_queue_bounce_limit(rq, BLK_BOUNCE_ANY);
-
-	gd->queue = rq;
+	info->rq = gd->queue = rq;
+	info->gd = gd;
+	info->sector_size = sector_size;
+	info->physical_sector_size = physical_sector_size;
+	blkif_set_queue_limits(info);
 
 	return 0;
 }
@@ -1136,16 +1148,11 @@ static int xlvbd_alloc_gendisk(blkif_sector_t capacity,
 	gd->private_data = info;
 	set_capacity(gd, capacity);
 
-	if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size,
-				 info->max_indirect_segments ? :
-				 BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
+	if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size)) {
 		del_gendisk(gd);
 		goto release;
 	}
 
-	info->rq = gd->queue;
-	info->gd = gd;
-
 	xlvbd_flush(info);
 
 	if (vdisk_info & VDISK_READONLY)
@@ -1315,7 +1322,7 @@ free_shadow:
 			rinfo->ring_ref[i] = GRANT_INVALID_REF;
 		}
 	}
-	free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * PAGE_SIZE));
+	free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * XEN_PAGE_SIZE));
 	rinfo->ring.sring = NULL;
 
 	if (rinfo->irq)
@@ -2007,8 +2014,10 @@ static int blkif_recover(struct blkfront_info *info)
 	struct split_bio *split_bio;
 
 	blkfront_gather_backend_features(info);
+	/* Reset limits changed by blk_mq_update_nr_hw_queues(). */
+	blkif_set_queue_limits(info);
 	segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST;
-	blk_queue_max_segments(info->rq, segs);
+	blk_queue_max_segments(info->rq, segs / GRANTS_PER_PSEG);
 
 	for (r_index = 0; r_index < info->nr_rings; r_index++) {
 		struct blkfront_ring_info *rinfo = &info->rinfo[r_index];
@@ -2432,7 +2441,7 @@ static void blkfront_connect(struct blkfront_info *info)
 	if (err) {
 		xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s",
 				 info->xbdev->otherend);
-		return;
+		goto fail;
 	}
 
 	xenbus_switch_state(info->xbdev, XenbusStateConnected);
@@ -2445,6 +2454,11 @@ static void blkfront_connect(struct blkfront_info *info)
 	device_add_disk(&info->xbdev->dev, info->gd);
 
 	info->is_ready = 1;
+	return;
+
+fail:
+	blkif_free(info, 0);
+	return;
 }
 
 /**
diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig
index cf50fd2e96df..3cc9bff9d99d 100644
--- a/drivers/bluetooth/Kconfig
+++ b/drivers/bluetooth/Kconfig
@@ -180,6 +180,17 @@ config BT_HCIUART_AG6XX
 
 	  Say Y here to compile support for Intel AG6XX protocol.
 
+config BT_HCIUART_MRVL
+	bool "Marvell protocol support"
+	depends on BT_HCIUART
+	select BT_HCIUART_H4
+	help
+	  Marvell is serial protocol for communication between Bluetooth
+	  device and host. This protocol is required for most Marvell Bluetooth
+	  devices with UART interface.
+
+	  Say Y here to compile support for HCI MRVL protocol.
+
 config BT_HCIBCM203X
 	tristate "HCI BCM203x USB driver"
 	depends on USB
@@ -331,4 +342,16 @@ config BT_WILINK
 	  Say Y here to compile support for Texas Instrument's WiLink7 driver
 	  into the kernel or say M to compile it as module (btwilink).
 
+config BT_QCOMSMD
+	tristate "Qualcomm SMD based HCI support"
+	depends on QCOM_SMD && QCOM_WCNSS_CTRL
+	select BT_QCA
+	help
+	  Qualcomm SMD based HCI driver.
+	  This driver is used to bridge HCI data onto the shared memory
+	  channels to the WCNSS core.
+
+	  Say Y here to compile support for HCI over Qualcomm SMD into the
+	  kernel or say M to compile as a module.
+
 endmenu
diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile
index 9c18939fc5c9..b1fc29a697b7 100644
--- a/drivers/bluetooth/Makefile
+++ b/drivers/bluetooth/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_BT_ATH3K)		+= ath3k.o
 obj-$(CONFIG_BT_MRVL)		+= btmrvl.o
 obj-$(CONFIG_BT_MRVL_SDIO)	+= btmrvl_sdio.o
 obj-$(CONFIG_BT_WILINK)		+= btwilink.o
+obj-$(CONFIG_BT_QCOMSMD)	+= btqcomsmd.o
 obj-$(CONFIG_BT_BCM)		+= btbcm.o
 obj-$(CONFIG_BT_RTL)		+= btrtl.o
 obj-$(CONFIG_BT_QCA)		+= btqca.o
@@ -37,6 +38,7 @@ hci_uart-$(CONFIG_BT_HCIUART_INTEL)	+= hci_intel.o
 hci_uart-$(CONFIG_BT_HCIUART_BCM)	+= hci_bcm.o
 hci_uart-$(CONFIG_BT_HCIUART_QCA)	+= hci_qca.o
 hci_uart-$(CONFIG_BT_HCIUART_AG6XX)	+= hci_ag6xx.o
+hci_uart-$(CONFIG_BT_HCIUART_MRVL)	+= hci_mrvl.o
 hci_uart-objs				:= $(hci_uart-y)
 
 ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/bluetooth/bcm203x.c b/drivers/bluetooth/bcm203x.c
index 5b0ef7bbe8ac..5ce6d4176dc3 100644
--- a/drivers/bluetooth/bcm203x.c
+++ b/drivers/bluetooth/bcm203x.c
@@ -185,10 +185,8 @@ static int bcm203x_probe(struct usb_interface *intf, const struct usb_device_id
 	data->state = BCM203X_LOAD_MINIDRV;
 
 	data->urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!data->urb) {
-		BT_ERR("Can't allocate URB");
+	if (!data->urb)
 		return -ENOMEM;
-	}
 
 	if (request_firmware(&firmware, "BCM2033-MD.hex", &udev->dev) < 0) {
 		BT_ERR("Mini driver request failed");
diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c
index 4a6208168850..28afd5d585f9 100644
--- a/drivers/bluetooth/btqca.c
+++ b/drivers/bluetooth/btqca.c
@@ -55,8 +55,8 @@ static int rome_patch_ver_req(struct hci_dev *hdev, u32 *rome_version)
 	}
 
 	edl = (struct edl_event_hdr *)(skb->data);
-	if (!edl || !edl->data) {
-		BT_ERR("%s: TLV with no header or no data", hdev->name);
+	if (!edl) {
+		BT_ERR("%s: TLV with no header", hdev->name);
 		err = -EILSEQ;
 		goto out;
 	}
@@ -224,8 +224,8 @@ static int rome_tlv_send_segment(struct hci_dev *hdev, int idx, int seg_size,
 	}
 
 	edl = (struct edl_event_hdr *)(skb->data);
-	if (!edl || !edl->data) {
-		BT_ERR("%s: TLV with no header or no data", hdev->name);
+	if (!edl) {
+		BT_ERR("%s: TLV with no header", hdev->name);
 		err = -EILSEQ;
 		goto out;
 	}
diff --git a/drivers/bluetooth/btqcomsmd.c b/drivers/bluetooth/btqcomsmd.c
new file mode 100644
index 000000000000..08c2c93887c1
--- /dev/null
+++ b/drivers/bluetooth/btqcomsmd.c
@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2016, Linaro Ltd.
+ * Copyright (c) 2015, Sony Mobile Communications Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/soc/qcom/smd.h>
+#include <linux/soc/qcom/wcnss_ctrl.h>
+#include <linux/platform_device.h>
+
+#include <net/bluetooth/bluetooth.h>
+#include <net/bluetooth/hci_core.h>
+
+#include "btqca.h"
+
+struct btqcomsmd {
+	struct hci_dev *hdev;
+
+	struct qcom_smd_channel *acl_channel;
+	struct qcom_smd_channel *cmd_channel;
+};
+
+static int btqcomsmd_recv(struct hci_dev *hdev, unsigned int type,
+			   const void *data, size_t count)
+{
+	struct sk_buff *skb;
+
+	/* Use GFP_ATOMIC as we're in IRQ context */
+	skb = bt_skb_alloc(count, GFP_ATOMIC);
+	if (!skb) {
+		hdev->stat.err_rx++;
+		return -ENOMEM;
+	}
+
+	hci_skb_pkt_type(skb) = type;
+	memcpy(skb_put(skb, count), data, count);
+
+	return hci_recv_frame(hdev, skb);
+}
+
+static int btqcomsmd_acl_callback(struct qcom_smd_channel *channel,
+				  const void *data, size_t count)
+{
+	struct btqcomsmd *btq = qcom_smd_get_drvdata(channel);
+
+	btq->hdev->stat.byte_rx += count;
+	return btqcomsmd_recv(btq->hdev, HCI_ACLDATA_PKT, data, count);
+}
+
+static int btqcomsmd_cmd_callback(struct qcom_smd_channel *channel,
+				  const void *data, size_t count)
+{
+	struct btqcomsmd *btq = qcom_smd_get_drvdata(channel);
+
+	return btqcomsmd_recv(btq->hdev, HCI_EVENT_PKT, data, count);
+}
+
+static int btqcomsmd_send(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btqcomsmd *btq = hci_get_drvdata(hdev);
+	int ret;
+
+	switch (hci_skb_pkt_type(skb)) {
+	case HCI_ACLDATA_PKT:
+		ret = qcom_smd_send(btq->acl_channel, skb->data, skb->len);
+		hdev->stat.acl_tx++;
+		hdev->stat.byte_tx += skb->len;
+		break;
+	case HCI_COMMAND_PKT:
+		ret = qcom_smd_send(btq->cmd_channel, skb->data, skb->len);
+		hdev->stat.cmd_tx++;
+		break;
+	default:
+		ret = -EILSEQ;
+		break;
+	}
+
+	kfree_skb(skb);
+
+	return ret;
+}
+
+static int btqcomsmd_open(struct hci_dev *hdev)
+{
+	return 0;
+}
+
+static int btqcomsmd_close(struct hci_dev *hdev)
+{
+	return 0;
+}
+
+static int btqcomsmd_probe(struct platform_device *pdev)
+{
+	struct btqcomsmd *btq;
+	struct hci_dev *hdev;
+	void *wcnss;
+	int ret;
+
+	btq = devm_kzalloc(&pdev->dev, sizeof(*btq), GFP_KERNEL);
+	if (!btq)
+		return -ENOMEM;
+
+	wcnss = dev_get_drvdata(pdev->dev.parent);
+
+	btq->acl_channel = qcom_wcnss_open_channel(wcnss, "APPS_RIVA_BT_ACL",
+						   btqcomsmd_acl_callback);
+	if (IS_ERR(btq->acl_channel))
+		return PTR_ERR(btq->acl_channel);
+
+	btq->cmd_channel = qcom_wcnss_open_channel(wcnss, "APPS_RIVA_BT_CMD",
+						   btqcomsmd_cmd_callback);
+	if (IS_ERR(btq->cmd_channel))
+		return PTR_ERR(btq->cmd_channel);
+
+	qcom_smd_set_drvdata(btq->acl_channel, btq);
+	qcom_smd_set_drvdata(btq->cmd_channel, btq);
+
+	hdev = hci_alloc_dev();
+	if (!hdev)
+		return -ENOMEM;
+
+	hci_set_drvdata(hdev, btq);
+	btq->hdev = hdev;
+	SET_HCIDEV_DEV(hdev, &pdev->dev);
+
+	hdev->bus = HCI_SMD;
+	hdev->open = btqcomsmd_open;
+	hdev->close = btqcomsmd_close;
+	hdev->send = btqcomsmd_send;
+	hdev->set_bdaddr = qca_set_bdaddr_rome;
+
+	ret = hci_register_dev(hdev);
+	if (ret < 0) {
+		hci_free_dev(hdev);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, btq);
+
+	return 0;
+}
+
+static int btqcomsmd_remove(struct platform_device *pdev)
+{
+	struct btqcomsmd *btq = platform_get_drvdata(pdev);
+
+	hci_unregister_dev(btq->hdev);
+	hci_free_dev(btq->hdev);
+
+	return 0;
+}
+
+static const struct of_device_id btqcomsmd_of_match[] = {
+	{ .compatible = "qcom,wcnss-bt", },
+	{ },
+};
+
+static struct platform_driver btqcomsmd_driver = {
+	.probe = btqcomsmd_probe,
+	.remove = btqcomsmd_remove,
+	.driver  = {
+		.name  = "btqcomsmd",
+		.of_match_table = btqcomsmd_of_match,
+	},
+};
+
+module_platform_driver(btqcomsmd_driver);
+
+MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@sonymobile.com>");
+MODULE_DESCRIPTION("Qualcomm SMD HCI driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c
index 84288938f7f2..fc9b25703c67 100644
--- a/drivers/bluetooth/btrtl.c
+++ b/drivers/bluetooth/btrtl.c
@@ -33,6 +33,7 @@
 #define RTL_ROM_LMP_8723B	0x8723
 #define RTL_ROM_LMP_8821A	0x8821
 #define RTL_ROM_LMP_8761A	0x8761
+#define RTL_ROM_LMP_8822B	0x8822
 
 static int rtl_read_rom_version(struct hci_dev *hdev, u8 *version)
 {
@@ -78,11 +79,15 @@ static int rtl8723b_parse_firmware(struct hci_dev *hdev, u16 lmp_subver,
 	const unsigned char *patch_length_base, *patch_offset_base;
 	u32 patch_offset = 0;
 	u16 patch_length, num_patches;
-	const u16 project_id_to_lmp_subver[] = {
-		RTL_ROM_LMP_8723A,
-		RTL_ROM_LMP_8723B,
-		RTL_ROM_LMP_8821A,
-		RTL_ROM_LMP_8761A
+	static const struct {
+		__u16 lmp_subver;
+		__u8 id;
+	} project_id_to_lmp_subver[] = {
+		{ RTL_ROM_LMP_8723A, 0 },
+		{ RTL_ROM_LMP_8723B, 1 },
+		{ RTL_ROM_LMP_8821A, 2 },
+		{ RTL_ROM_LMP_8761A, 3 },
+		{ RTL_ROM_LMP_8822B, 8 },
 	};
 
 	ret = rtl_read_rom_version(hdev, &rom_version);
@@ -134,14 +139,20 @@ static int rtl8723b_parse_firmware(struct hci_dev *hdev, u16 lmp_subver,
 		return -EINVAL;
 	}
 
-	if (project_id >= ARRAY_SIZE(project_id_to_lmp_subver)) {
+	/* Find project_id in table */
+	for (i = 0; i < ARRAY_SIZE(project_id_to_lmp_subver); i++) {
+		if (project_id == project_id_to_lmp_subver[i].id)
+			break;
+	}
+
+	if (i >= ARRAY_SIZE(project_id_to_lmp_subver)) {
 		BT_ERR("%s: unknown project id %d", hdev->name, project_id);
 		return -EINVAL;
 	}
 
-	if (lmp_subver != project_id_to_lmp_subver[project_id]) {
+	if (lmp_subver != project_id_to_lmp_subver[i].lmp_subver) {
 		BT_ERR("%s: firmware is for %x but this is a %x", hdev->name,
-		       project_id_to_lmp_subver[project_id], lmp_subver);
+		       project_id_to_lmp_subver[i].lmp_subver, lmp_subver);
 		return -EINVAL;
 	}
 
@@ -257,6 +268,26 @@ out:
 	return ret;
 }
 
+static int rtl_load_config(struct hci_dev *hdev, const char *name, u8 **buff)
+{
+	const struct firmware *fw;
+	int ret;
+
+	BT_INFO("%s: rtl: loading %s", hdev->name, name);
+	ret = request_firmware(&fw, name, &hdev->dev);
+	if (ret < 0) {
+		BT_ERR("%s: Failed to load %s", hdev->name, name);
+		return ret;
+	}
+
+	ret = fw->size;
+	*buff = kmemdup(fw->data, ret, GFP_KERNEL);
+
+	release_firmware(fw);
+
+	return ret;
+}
+
 static int btrtl_setup_rtl8723a(struct hci_dev *hdev)
 {
 	const struct firmware *fw;
@@ -296,25 +327,74 @@ static int btrtl_setup_rtl8723b(struct hci_dev *hdev, u16 lmp_subver,
 	unsigned char *fw_data = NULL;
 	const struct firmware *fw;
 	int ret;
+	int cfg_sz;
+	u8 *cfg_buff = NULL;
+	u8 *tbuff;
+	char *cfg_name = NULL;
+
+	switch (lmp_subver) {
+	case RTL_ROM_LMP_8723B:
+		cfg_name = "rtl_bt/rtl8723b_config.bin";
+		break;
+	case RTL_ROM_LMP_8821A:
+		cfg_name = "rtl_bt/rtl8821a_config.bin";
+		break;
+	case RTL_ROM_LMP_8761A:
+		cfg_name = "rtl_bt/rtl8761a_config.bin";
+		break;
+	case RTL_ROM_LMP_8822B:
+		cfg_name = "rtl_bt/rtl8822b_config.bin";
+		break;
+	default:
+		BT_ERR("%s: rtl: no config according to lmp_subver %04x",
+		       hdev->name, lmp_subver);
+		break;
+	}
+
+	if (cfg_name) {
+		cfg_sz = rtl_load_config(hdev, cfg_name, &cfg_buff);
+		if (cfg_sz < 0)
+			cfg_sz = 0;
+	} else
+		cfg_sz = 0;
 
 	BT_INFO("%s: rtl: loading %s", hdev->name, fw_name);
 	ret = request_firmware(&fw, fw_name, &hdev->dev);
 	if (ret < 0) {
 		BT_ERR("%s: Failed to load %s", hdev->name, fw_name);
-		return ret;
+		goto err_req_fw;
 	}
 
 	ret = rtl8723b_parse_firmware(hdev, lmp_subver, fw, &fw_data);
 	if (ret < 0)
 		goto out;
 
+	if (cfg_sz) {
+		tbuff = kzalloc(ret + cfg_sz, GFP_KERNEL);
+		if (!tbuff) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		memcpy(tbuff, fw_data, ret);
+		kfree(fw_data);
+
+		memcpy(tbuff + ret, cfg_buff, cfg_sz);
+		ret += cfg_sz;
+
+		fw_data = tbuff;
+	}
+
+	BT_INFO("cfg_sz %d, total size %d", cfg_sz, ret);
+
 	ret = rtl_download_firmware(hdev, fw_data, ret);
-	kfree(fw_data);
-	if (ret < 0)
-		goto out;
 
 out:
 	release_firmware(fw);
+	kfree(fw_data);
+err_req_fw:
+	if (cfg_sz)
+		kfree(cfg_buff);
 	return ret;
 }
 
@@ -377,6 +457,9 @@ int btrtl_setup_realtek(struct hci_dev *hdev)
 	case RTL_ROM_LMP_8761A:
 		return btrtl_setup_rtl8723b(hdev, lmp_subver,
 					    "rtl_bt/rtl8761a_fw.bin");
+	case RTL_ROM_LMP_8822B:
+		return btrtl_setup_rtl8723b(hdev, lmp_subver,
+					    "rtl_bt/rtl8822b_fw.bin");
 	default:
 		BT_INFO("rtl: assuming no firmware upload needed.");
 		return 0;
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index 811f9b97e360..2f633df9f4e6 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -62,6 +62,7 @@ static struct usb_driver btusb_driver;
 #define BTUSB_REALTEK		0x20000
 #define BTUSB_BCM2045		0x40000
 #define BTUSB_IFNUM_2		0x80000
+#define BTUSB_CW6622		0x100000
 
 static const struct usb_device_id btusb_table[] = {
 	/* Generic Bluetooth USB device */
@@ -248,9 +249,11 @@ static const struct usb_device_id blacklist_table[] = {
 
 	/* QCA ROME chipset */
 	{ USB_DEVICE(0x0cf3, 0xe007), .driver_info = BTUSB_QCA_ROME },
+	{ USB_DEVICE(0x0cf3, 0xe009), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0cf3, 0xe300), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0cf3, 0xe360), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0489, 0xe092), .driver_info = BTUSB_QCA_ROME },
+	{ USB_DEVICE(0x04ca, 0x3011), .driver_info = BTUSB_QCA_ROME },
 
 	/* Broadcom BCM2035 */
 	{ USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 },
@@ -290,7 +293,8 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x0400, 0x080a), .driver_info = BTUSB_BROKEN_ISOC },
 
 	/* CONWISE Technology based adapters with buggy SCO support */
-	{ USB_DEVICE(0x0e5e, 0x6622), .driver_info = BTUSB_BROKEN_ISOC },
+	{ USB_DEVICE(0x0e5e, 0x6622),
+	  .driver_info = BTUSB_BROKEN_ISOC | BTUSB_CW6622},
 
 	/* Roper Class 1 Bluetooth Dongle (Silicon Wave based) */
 	{ USB_DEVICE(0x1310, 0x0001), .driver_info = BTUSB_SWAVE },
@@ -310,6 +314,7 @@ static const struct usb_device_id blacklist_table[] = {
 	/* Marvell Bluetooth devices */
 	{ USB_DEVICE(0x1286, 0x2044), .driver_info = BTUSB_MARVELL },
 	{ USB_DEVICE(0x1286, 0x2046), .driver_info = BTUSB_MARVELL },
+	{ USB_DEVICE(0x1286, 0x204e), .driver_info = BTUSB_MARVELL },
 
 	/* Intel Bluetooth devices */
 	{ USB_DEVICE(0x8087, 0x07da), .driver_info = BTUSB_CSR },
@@ -1038,6 +1043,10 @@ static int btusb_open(struct hci_dev *hdev)
 
 	BT_DBG("%s", hdev->name);
 
+	err = usb_autopm_get_interface(data->intf);
+	if (err < 0)
+		return err;
+
 	/* Patching USB firmware files prior to starting any URBs of HCI path
 	 * It is more safe to use USB bulk channel for downloading USB patch
 	 */
@@ -1047,10 +1056,6 @@ static int btusb_open(struct hci_dev *hdev)
 			return err;
 	}
 
-	err = usb_autopm_get_interface(data->intf);
-	if (err < 0)
-		return err;
-
 	data->intf->needs_remote_wakeup = 1;
 
 	if (test_and_set_bit(BTUSB_INTR_RUNNING, &data->flags))
@@ -2221,9 +2226,8 @@ static int btusb_setup_intel_new(struct hci_dev *hdev)
 	err = wait_on_bit_timeout(&data->flags, BTUSB_DOWNLOADING,
 				  TASK_INTERRUPTIBLE,
 				  msecs_to_jiffies(5000));
-	if (err == 1) {
+	if (err == -EINTR) {
 		BT_ERR("%s: Firmware loading interrupted", hdev->name);
-		err = -EINTR;
 		goto done;
 	}
 
@@ -2275,7 +2279,7 @@ done:
 				  TASK_INTERRUPTIBLE,
 				  msecs_to_jiffies(1000));
 
-	if (err == 1) {
+	if (err == -EINTR) {
 		BT_ERR("%s: Device boot interrupted", hdev->name);
 		return -EINTR;
 	}
@@ -2845,6 +2849,9 @@ static int btusb_probe(struct usb_interface *intf,
 	hdev->send   = btusb_send_frame;
 	hdev->notify = btusb_notify;
 
+	if (id->driver_info & BTUSB_CW6622)
+		set_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks);
+
 	if (id->driver_info & BTUSB_BCM2045)
 		set_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks);
 
diff --git a/drivers/bluetooth/btwilink.c b/drivers/bluetooth/btwilink.c
index 485281b3f167..ef51c9c864c5 100644
--- a/drivers/bluetooth/btwilink.c
+++ b/drivers/bluetooth/btwilink.c
@@ -245,6 +245,7 @@ static int ti_st_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
 {
 	struct ti_st *hst;
 	long len;
+	int pkt_type;
 
 	hst = hci_get_drvdata(hdev);
 
@@ -258,6 +259,7 @@ static int ti_st_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
 	 * Freeing skb memory is taken care in shared transport layer,
 	 * so don't free skb memory here.
 	 */
+	pkt_type = hci_skb_pkt_type(skb);
 	len = hst->st_write(skb);
 	if (len < 0) {
 		kfree_skb(skb);
@@ -268,7 +270,7 @@ static int ti_st_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
 
 	/* ST accepted our skb. So, Go ahead and do rest */
 	hdev->stat.byte_tx += len;
-	ti_st_tx_complete(hst, hci_skb_pkt_type(skb));
+	ti_st_tx_complete(hst, pkt_type);
 
 	return 0;
 }
diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c
index 1c97eda8bae3..5ccb90ef0146 100644
--- a/drivers/bluetooth/hci_bcm.c
+++ b/drivers/bluetooth/hci_bcm.c
@@ -798,7 +798,7 @@ static int bcm_remove(struct platform_device *pdev)
 
 static const struct hci_uart_proto bcm_proto = {
 	.id		= HCI_UART_BCM,
-	.name		= "BCM",
+	.name		= "Broadcom",
 	.manufacturer	= 15,
 	.init_speed	= 115200,
 	.oper_speed	= 4000000,
diff --git a/drivers/bluetooth/hci_bcsp.c b/drivers/bluetooth/hci_bcsp.c
index d7d23ceba4d1..a2c921faaa12 100644
--- a/drivers/bluetooth/hci_bcsp.c
+++ b/drivers/bluetooth/hci_bcsp.c
@@ -90,7 +90,8 @@ struct bcsp_struct {
 /* ---- BCSP CRC calculation ---- */
 
 /* Table for calculating CRC for polynomial 0x1021, LSB processed first,
-initial value 0xffff, bits shifted in reverse order. */
+ * initial value 0xffff, bits shifted in reverse order.
+ */
 
 static const u16 crc_table[] = {
 	0x0000, 0x1081, 0x2102, 0x3183,
@@ -174,7 +175,7 @@ static int bcsp_enqueue(struct hci_uart *hu, struct sk_buff *skb)
 }
 
 static struct sk_buff *bcsp_prepare_pkt(struct bcsp_struct *bcsp, u8 *data,
-		int len, int pkt_type)
+					int len, int pkt_type)
 {
 	struct sk_buff *nskb;
 	u8 hdr[4], chan;
@@ -213,6 +214,7 @@ static struct sk_buff *bcsp_prepare_pkt(struct bcsp_struct *bcsp, u8 *data,
 		/* Vendor specific commands */
 		if (hci_opcode_ogf(__le16_to_cpu(opcode)) == 0x3f) {
 			u8 desc = *(data + HCI_COMMAND_HDR_SIZE);
+
 			if ((desc & 0xf0) == 0xc0) {
 				data += HCI_COMMAND_HDR_SIZE + 1;
 				len  -= HCI_COMMAND_HDR_SIZE + 1;
@@ -271,8 +273,8 @@ static struct sk_buff *bcsp_prepare_pkt(struct bcsp_struct *bcsp, u8 *data,
 	/* Put CRC */
 	if (bcsp->use_crc) {
 		bcsp_txmsg_crc = bitrev16(bcsp_txmsg_crc);
-		bcsp_slip_one_byte(nskb, (u8) ((bcsp_txmsg_crc >> 8) & 0x00ff));
-		bcsp_slip_one_byte(nskb, (u8) (bcsp_txmsg_crc & 0x00ff));
+		bcsp_slip_one_byte(nskb, (u8)((bcsp_txmsg_crc >> 8) & 0x00ff));
+		bcsp_slip_one_byte(nskb, (u8)(bcsp_txmsg_crc & 0x00ff));
 	}
 
 	bcsp_slip_msgdelim(nskb);
@@ -287,7 +289,8 @@ static struct sk_buff *bcsp_dequeue(struct hci_uart *hu)
 	struct sk_buff *skb;
 
 	/* First of all, check for unreliable messages in the queue,
-	   since they have priority */
+	 * since they have priority
+	 */
 
 	skb = skb_dequeue(&bcsp->unrel);
 	if (skb != NULL) {
@@ -414,7 +417,7 @@ static void bcsp_handle_le_pkt(struct hci_uart *hu)
 
 	/* spot "conf" pkts and reply with a "conf rsp" pkt */
 	if (bcsp->rx_skb->data[1] >> 4 == 4 && bcsp->rx_skb->data[2] == 0 &&
-			!memcmp(&bcsp->rx_skb->data[4], conf_pkt, 4)) {
+	    !memcmp(&bcsp->rx_skb->data[4], conf_pkt, 4)) {
 		struct sk_buff *nskb = alloc_skb(4, GFP_ATOMIC);
 
 		BT_DBG("Found a LE conf pkt");
@@ -428,7 +431,7 @@ static void bcsp_handle_le_pkt(struct hci_uart *hu)
 	}
 	/* Spot "sync" pkts. If we find one...disaster! */
 	else if (bcsp->rx_skb->data[1] >> 4 == 4 && bcsp->rx_skb->data[2] == 0 &&
-			!memcmp(&bcsp->rx_skb->data[4], sync_pkt, 4)) {
+		 !memcmp(&bcsp->rx_skb->data[4], sync_pkt, 4)) {
 		BT_ERR("Found a LE sync pkt, card has reset");
 	}
 }
@@ -446,7 +449,7 @@ static inline void bcsp_unslip_one_byte(struct bcsp_struct *bcsp, unsigned char
 		default:
 			memcpy(skb_put(bcsp->rx_skb, 1), &byte, 1);
 			if ((bcsp->rx_skb->data[0] & 0x40) != 0 &&
-					bcsp->rx_state != BCSP_W4_CRC)
+			    bcsp->rx_state != BCSP_W4_CRC)
 				bcsp_crc_update(&bcsp->message_crc, byte);
 			bcsp->rx_count--;
 		}
@@ -457,7 +460,7 @@ static inline void bcsp_unslip_one_byte(struct bcsp_struct *bcsp, unsigned char
 		case 0xdc:
 			memcpy(skb_put(bcsp->rx_skb, 1), &c0, 1);
 			if ((bcsp->rx_skb->data[0] & 0x40) != 0 &&
-					bcsp->rx_state != BCSP_W4_CRC)
+			    bcsp->rx_state != BCSP_W4_CRC)
 				bcsp_crc_update(&bcsp->message_crc, 0xc0);
 			bcsp->rx_esc_state = BCSP_ESCSTATE_NOESC;
 			bcsp->rx_count--;
@@ -466,7 +469,7 @@ static inline void bcsp_unslip_one_byte(struct bcsp_struct *bcsp, unsigned char
 		case 0xdd:
 			memcpy(skb_put(bcsp->rx_skb, 1), &db, 1);
 			if ((bcsp->rx_skb->data[0] & 0x40) != 0 &&
-					bcsp->rx_state != BCSP_W4_CRC)
+			    bcsp->rx_state != BCSP_W4_CRC)
 				bcsp_crc_update(&bcsp->message_crc, 0xdb);
 			bcsp->rx_esc_state = BCSP_ESCSTATE_NOESC;
 			bcsp->rx_count--;
@@ -485,13 +488,28 @@ static inline void bcsp_unslip_one_byte(struct bcsp_struct *bcsp, unsigned char
 static void bcsp_complete_rx_pkt(struct hci_uart *hu)
 {
 	struct bcsp_struct *bcsp = hu->priv;
-	int pass_up;
+	int pass_up = 0;
 
 	if (bcsp->rx_skb->data[0] & 0x80) {	/* reliable pkt */
 		BT_DBG("Received seqno %u from card", bcsp->rxseq_txack);
-		bcsp->rxseq_txack++;
-		bcsp->rxseq_txack %= 0x8;
-		bcsp->txack_req    = 1;
+
+		/* check the rx sequence number is as expected */
+		if ((bcsp->rx_skb->data[0] & 0x07) == bcsp->rxseq_txack) {
+			bcsp->rxseq_txack++;
+			bcsp->rxseq_txack %= 0x8;
+		} else {
+			/* handle re-transmitted packet or
+			 * when packet was missed
+			 */
+			BT_ERR("Out-of-order packet arrived, got %u expected %u",
+			       bcsp->rx_skb->data[0] & 0x07, bcsp->rxseq_txack);
+
+			/* do not process out-of-order packet payload */
+			pass_up = 2;
+		}
+
+		/* send current txack value to all received reliable packets */
+		bcsp->txack_req = 1;
 
 		/* If needed, transmit an ack pkt */
 		hci_uart_tx_wakeup(hu);
@@ -500,26 +518,33 @@ static void bcsp_complete_rx_pkt(struct hci_uart *hu)
 	bcsp->rxack = (bcsp->rx_skb->data[0] >> 3) & 0x07;
 	BT_DBG("Request for pkt %u from card", bcsp->rxack);
 
+	/* handle received ACK indications,
+	 * including those from out-of-order packets
+	 */
 	bcsp_pkt_cull(bcsp);
-	if ((bcsp->rx_skb->data[1] & 0x0f) == 6 &&
-			bcsp->rx_skb->data[0] & 0x80) {
-		hci_skb_pkt_type(bcsp->rx_skb) = HCI_ACLDATA_PKT;
-		pass_up = 1;
-	} else if ((bcsp->rx_skb->data[1] & 0x0f) == 5 &&
-			bcsp->rx_skb->data[0] & 0x80) {
-		hci_skb_pkt_type(bcsp->rx_skb) = HCI_EVENT_PKT;
-		pass_up = 1;
-	} else if ((bcsp->rx_skb->data[1] & 0x0f) == 7) {
-		hci_skb_pkt_type(bcsp->rx_skb) = HCI_SCODATA_PKT;
-		pass_up = 1;
-	} else if ((bcsp->rx_skb->data[1] & 0x0f) == 1 &&
-			!(bcsp->rx_skb->data[0] & 0x80)) {
-		bcsp_handle_le_pkt(hu);
-		pass_up = 0;
-	} else
-		pass_up = 0;
-
-	if (!pass_up) {
+
+	if (pass_up != 2) {
+		if ((bcsp->rx_skb->data[1] & 0x0f) == 6 &&
+		    (bcsp->rx_skb->data[0] & 0x80)) {
+			hci_skb_pkt_type(bcsp->rx_skb) = HCI_ACLDATA_PKT;
+			pass_up = 1;
+		} else if ((bcsp->rx_skb->data[1] & 0x0f) == 5 &&
+			   (bcsp->rx_skb->data[0] & 0x80)) {
+			hci_skb_pkt_type(bcsp->rx_skb) = HCI_EVENT_PKT;
+			pass_up = 1;
+		} else if ((bcsp->rx_skb->data[1] & 0x0f) == 7) {
+			hci_skb_pkt_type(bcsp->rx_skb) = HCI_SCODATA_PKT;
+			pass_up = 1;
+		} else if ((bcsp->rx_skb->data[1] & 0x0f) == 1 &&
+			   !(bcsp->rx_skb->data[0] & 0x80)) {
+			bcsp_handle_le_pkt(hu);
+			pass_up = 0;
+		} else {
+			pass_up = 0;
+		}
+	}
+
+	if (pass_up == 0) {
 		struct hci_event_hdr hdr;
 		u8 desc = (bcsp->rx_skb->data[1] & 0x0f);
 
@@ -537,18 +562,23 @@ static void bcsp_complete_rx_pkt(struct hci_uart *hu)
 				hci_recv_frame(hu->hdev, bcsp->rx_skb);
 			} else {
 				BT_ERR("Packet for unknown channel (%u %s)",
-					bcsp->rx_skb->data[1] & 0x0f,
-					bcsp->rx_skb->data[0] & 0x80 ?
-					"reliable" : "unreliable");
+				       bcsp->rx_skb->data[1] & 0x0f,
+				       bcsp->rx_skb->data[0] & 0x80 ?
+				       "reliable" : "unreliable");
 				kfree_skb(bcsp->rx_skb);
 			}
 		} else
 			kfree_skb(bcsp->rx_skb);
-	} else {
+	} else if (pass_up == 1) {
 		/* Pull out BCSP hdr */
 		skb_pull(bcsp->rx_skb, 4);
 
 		hci_recv_frame(hu->hdev, bcsp->rx_skb);
+	} else {
+		/* ignore packet payload of already ACKed re-transmitted
+		 * packets or when a packet was missed in the BCSP window
+		 */
+		kfree_skb(bcsp->rx_skb);
 	}
 
 	bcsp->rx_state = BCSP_W4_PKT_DELIMITER;
@@ -567,7 +597,7 @@ static int bcsp_recv(struct hci_uart *hu, const void *data, int count)
 	const unsigned char *ptr;
 
 	BT_DBG("hu %p count %d rx_state %d rx_count %ld",
-		hu, count, bcsp->rx_state, bcsp->rx_count);
+	       hu, count, bcsp->rx_state, bcsp->rx_count);
 
 	ptr = data;
 	while (count) {
@@ -586,24 +616,14 @@ static int bcsp_recv(struct hci_uart *hu, const void *data, int count)
 
 		switch (bcsp->rx_state) {
 		case BCSP_W4_BCSP_HDR:
-			if ((0xff & (u8) ~ (bcsp->rx_skb->data[0] + bcsp->rx_skb->data[1] +
-					bcsp->rx_skb->data[2])) != bcsp->rx_skb->data[3]) {
+			if ((0xff & (u8)~(bcsp->rx_skb->data[0] + bcsp->rx_skb->data[1] +
+			    bcsp->rx_skb->data[2])) != bcsp->rx_skb->data[3]) {
 				BT_ERR("Error in BCSP hdr checksum");
 				kfree_skb(bcsp->rx_skb);
 				bcsp->rx_state = BCSP_W4_PKT_DELIMITER;
 				bcsp->rx_count = 0;
 				continue;
 			}
-			if (bcsp->rx_skb->data[0] & 0x80	/* reliable pkt */
-						&& (bcsp->rx_skb->data[0] & 0x07) != bcsp->rxseq_txack) {
-				BT_ERR("Out-of-order packet arrived, got %u expected %u",
-					bcsp->rx_skb->data[0] & 0x07, bcsp->rxseq_txack);
-
-				kfree_skb(bcsp->rx_skb);
-				bcsp->rx_state = BCSP_W4_PKT_DELIMITER;
-				bcsp->rx_count = 0;
-				continue;
-			}
 			bcsp->rx_state = BCSP_W4_DATA;
 			bcsp->rx_count = (bcsp->rx_skb->data[1] >> 4) +
 					(bcsp->rx_skb->data[2] << 4);	/* May be 0 */
@@ -620,8 +640,8 @@ static int bcsp_recv(struct hci_uart *hu, const void *data, int count)
 		case BCSP_W4_CRC:
 			if (bitrev16(bcsp->message_crc) != bscp_get_crc(bcsp)) {
 				BT_ERR("Checksum failed: computed %04x received %04x",
-					bitrev16(bcsp->message_crc),
-					bscp_get_crc(bcsp));
+				       bitrev16(bcsp->message_crc),
+				       bscp_get_crc(bcsp));
 
 				kfree_skb(bcsp->rx_skb);
 				bcsp->rx_state = BCSP_W4_PKT_DELIMITER;
@@ -679,7 +699,7 @@ static int bcsp_recv(struct hci_uart *hu, const void *data, int count)
 	/* Arrange to retransmit all messages in the relq. */
 static void bcsp_timed_event(unsigned long arg)
 {
-	struct hci_uart *hu = (struct hci_uart *) arg;
+	struct hci_uart *hu = (struct hci_uart *)arg;
 	struct bcsp_struct *bcsp = hu->priv;
 	struct sk_buff *skb;
 	unsigned long flags;
@@ -715,7 +735,7 @@ static int bcsp_open(struct hci_uart *hu)
 
 	init_timer(&bcsp->tbcsp);
 	bcsp->tbcsp.function = bcsp_timed_event;
-	bcsp->tbcsp.data     = (u_long) hu;
+	bcsp->tbcsp.data     = (u_long)hu;
 
 	bcsp->rx_state = BCSP_W4_PKT_DELIMITER;
 
diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c
index ed0a4201b551..9e271286c5e5 100644
--- a/drivers/bluetooth/hci_intel.c
+++ b/drivers/bluetooth/hci_intel.c
@@ -128,7 +128,7 @@ static int intel_wait_booting(struct hci_uart *hu)
 				  TASK_INTERRUPTIBLE,
 				  msecs_to_jiffies(1000));
 
-	if (err == 1) {
+	if (err == -EINTR) {
 		bt_dev_err(hu->hdev, "Device boot interrupted");
 		return -EINTR;
 	}
@@ -151,7 +151,7 @@ static int intel_wait_lpm_transaction(struct hci_uart *hu)
 				  TASK_INTERRUPTIBLE,
 				  msecs_to_jiffies(1000));
 
-	if (err == 1) {
+	if (err == -EINTR) {
 		bt_dev_err(hu->hdev, "LPM transaction interrupted");
 		return -EINTR;
 	}
@@ -813,7 +813,7 @@ static int intel_setup(struct hci_uart *hu)
 	err = wait_on_bit_timeout(&intel->flags, STATE_DOWNLOADING,
 				  TASK_INTERRUPTIBLE,
 				  msecs_to_jiffies(5000));
-	if (err == 1) {
+	if (err == -EINTR) {
 		bt_dev_err(hdev, "Firmware loading interrupted");
 		err = -EINTR;
 		goto done;
diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c
index dda97398c59a..9497c469efd2 100644
--- a/drivers/bluetooth/hci_ldisc.c
+++ b/drivers/bluetooth/hci_ldisc.c
@@ -697,34 +697,36 @@ static int hci_uart_tty_ioctl(struct tty_struct *tty, struct file *file,
 	case HCIUARTSETPROTO:
 		if (!test_and_set_bit(HCI_UART_PROTO_SET, &hu->flags)) {
 			err = hci_uart_set_proto(hu, arg);
-			if (err) {
+			if (err)
 				clear_bit(HCI_UART_PROTO_SET, &hu->flags);
-				return err;
-			}
 		} else
-			return -EBUSY;
+			err = -EBUSY;
 		break;
 
 	case HCIUARTGETPROTO:
 		if (test_bit(HCI_UART_PROTO_SET, &hu->flags))
-			return hu->proto->id;
-		return -EUNATCH;
+			err = hu->proto->id;
+		else
+			err = -EUNATCH;
+		break;
 
 	case HCIUARTGETDEVICE:
 		if (test_bit(HCI_UART_REGISTERED, &hu->flags))
-			return hu->hdev->id;
-		return -EUNATCH;
+			err = hu->hdev->id;
+		else
+			err = -EUNATCH;
+		break;
 
 	case HCIUARTSETFLAGS:
 		if (test_bit(HCI_UART_PROTO_SET, &hu->flags))
-			return -EBUSY;
-		err = hci_uart_set_flags(hu, arg);
-		if (err)
-			return err;
+			err = -EBUSY;
+		else
+			err = hci_uart_set_flags(hu, arg);
 		break;
 
 	case HCIUARTGETFLAGS:
-		return hu->hdev_flags;
+		err = hu->hdev_flags;
+		break;
 
 	default:
 		err = n_tty_ioctl_helper(tty, file, cmd, arg);
@@ -810,6 +812,9 @@ static int __init hci_uart_init(void)
 #ifdef CONFIG_BT_HCIUART_AG6XX
 	ag6xx_init();
 #endif
+#ifdef CONFIG_BT_HCIUART_MRVL
+	mrvl_init();
+#endif
 
 	return 0;
 }
@@ -845,6 +850,9 @@ static void __exit hci_uart_exit(void)
 #ifdef CONFIG_BT_HCIUART_AG6XX
 	ag6xx_deinit();
 #endif
+#ifdef CONFIG_BT_HCIUART_MRVL
+	mrvl_deinit();
+#endif
 
 	/* Release tty registration of line discipline */
 	err = tty_unregister_ldisc(N_HCI);
diff --git a/drivers/bluetooth/hci_mrvl.c b/drivers/bluetooth/hci_mrvl.c
new file mode 100644
index 000000000000..bbc4b39b1dbf
--- /dev/null
+++ b/drivers/bluetooth/hci_mrvl.c
@@ -0,0 +1,387 @@
+/*
+ *
+ *  Bluetooth HCI UART driver for marvell devices
+ *
+ *  Copyright (C) 2016  Marvell International Ltd.
+ *  Copyright (C) 2016  Intel Corporation
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/skbuff.h>
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/tty.h>
+
+#include <net/bluetooth/bluetooth.h>
+#include <net/bluetooth/hci_core.h>
+
+#include "hci_uart.h"
+
+#define HCI_FW_REQ_PKT 0xA5
+#define HCI_CHIP_VER_PKT 0xAA
+
+#define MRVL_ACK 0x5A
+#define MRVL_NAK 0xBF
+#define MRVL_RAW_DATA 0x1F
+
+enum {
+	STATE_CHIP_VER_PENDING,
+	STATE_FW_REQ_PENDING,
+};
+
+struct mrvl_data {
+	struct sk_buff *rx_skb;
+	struct sk_buff_head txq;
+	struct sk_buff_head rawq;
+	unsigned long flags;
+	unsigned int tx_len;
+	u8 id, rev;
+};
+
+struct hci_mrvl_pkt {
+	__le16 lhs;
+	__le16 rhs;
+} __packed;
+#define HCI_MRVL_PKT_SIZE 4
+
+static int mrvl_open(struct hci_uart *hu)
+{
+	struct mrvl_data *mrvl;
+
+	BT_DBG("hu %p", hu);
+
+	mrvl = kzalloc(sizeof(*mrvl), GFP_KERNEL);
+	if (!mrvl)
+		return -ENOMEM;
+
+	skb_queue_head_init(&mrvl->txq);
+	skb_queue_head_init(&mrvl->rawq);
+
+	set_bit(STATE_CHIP_VER_PENDING, &mrvl->flags);
+
+	hu->priv = mrvl;
+	return 0;
+}
+
+static int mrvl_close(struct hci_uart *hu)
+{
+	struct mrvl_data *mrvl = hu->priv;
+
+	BT_DBG("hu %p", hu);
+
+	skb_queue_purge(&mrvl->txq);
+	skb_queue_purge(&mrvl->rawq);
+	kfree_skb(mrvl->rx_skb);
+	kfree(mrvl);
+
+	hu->priv = NULL;
+	return 0;
+}
+
+static int mrvl_flush(struct hci_uart *hu)
+{
+	struct mrvl_data *mrvl = hu->priv;
+
+	BT_DBG("hu %p", hu);
+
+	skb_queue_purge(&mrvl->txq);
+	skb_queue_purge(&mrvl->rawq);
+
+	return 0;
+}
+
+static struct sk_buff *mrvl_dequeue(struct hci_uart *hu)
+{
+	struct mrvl_data *mrvl = hu->priv;
+	struct sk_buff *skb;
+
+	skb = skb_dequeue(&mrvl->txq);
+	if (!skb) {
+		/* Any raw data ? */
+		skb = skb_dequeue(&mrvl->rawq);
+	} else {
+		/* Prepend skb with frame type */
+		memcpy(skb_push(skb, 1), &bt_cb(skb)->pkt_type, 1);
+	}
+
+	return skb;
+}
+
+static int mrvl_enqueue(struct hci_uart *hu, struct sk_buff *skb)
+{
+	struct mrvl_data *mrvl = hu->priv;
+
+	skb_queue_tail(&mrvl->txq, skb);
+	return 0;
+}
+
+static void mrvl_send_ack(struct hci_uart *hu, unsigned char type)
+{
+	struct mrvl_data *mrvl = hu->priv;
+	struct sk_buff *skb;
+
+	/* No H4 payload, only 1 byte header */
+	skb = bt_skb_alloc(0, GFP_ATOMIC);
+	if (!skb) {
+		bt_dev_err(hu->hdev, "Unable to alloc ack/nak packet");
+		return;
+	}
+	hci_skb_pkt_type(skb) = type;
+
+	skb_queue_tail(&mrvl->txq, skb);
+	hci_uart_tx_wakeup(hu);
+}
+
+static int mrvl_recv_fw_req(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct hci_mrvl_pkt *pkt = (void *)skb->data;
+	struct hci_uart *hu = hci_get_drvdata(hdev);
+	struct mrvl_data *mrvl = hu->priv;
+	int ret = 0;
+
+	if ((pkt->lhs ^ pkt->rhs) != 0xffff) {
+		bt_dev_err(hdev, "Corrupted mrvl header");
+		mrvl_send_ack(hu, MRVL_NAK);
+		ret = -EINVAL;
+		goto done;
+	}
+	mrvl_send_ack(hu, MRVL_ACK);
+
+	if (!test_bit(STATE_FW_REQ_PENDING, &mrvl->flags)) {
+		bt_dev_err(hdev, "Received unexpected firmware request");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	mrvl->tx_len = le16_to_cpu(pkt->lhs);
+
+	clear_bit(STATE_FW_REQ_PENDING, &mrvl->flags);
+	smp_mb__after_atomic();
+	wake_up_bit(&mrvl->flags, STATE_FW_REQ_PENDING);
+
+done:
+	kfree_skb(skb);
+	return ret;
+}
+
+static int mrvl_recv_chip_ver(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct hci_mrvl_pkt *pkt = (void *)skb->data;
+	struct hci_uart *hu = hci_get_drvdata(hdev);
+	struct mrvl_data *mrvl = hu->priv;
+	u16 version = le16_to_cpu(pkt->lhs);
+	int ret = 0;
+
+	if ((pkt->lhs ^ pkt->rhs) != 0xffff) {
+		bt_dev_err(hdev, "Corrupted mrvl header");
+		mrvl_send_ack(hu, MRVL_NAK);
+		ret = -EINVAL;
+		goto done;
+	}
+	mrvl_send_ack(hu, MRVL_ACK);
+
+	if (!test_bit(STATE_CHIP_VER_PENDING, &mrvl->flags)) {
+		bt_dev_err(hdev, "Received unexpected chip version");
+		goto done;
+	}
+
+	mrvl->id = version;
+	mrvl->rev = version >> 8;
+
+	bt_dev_info(hdev, "Controller id = %x, rev = %x", mrvl->id, mrvl->rev);
+
+	clear_bit(STATE_CHIP_VER_PENDING, &mrvl->flags);
+	smp_mb__after_atomic();
+	wake_up_bit(&mrvl->flags, STATE_CHIP_VER_PENDING);
+
+done:
+	kfree_skb(skb);
+	return ret;
+}
+
+#define HCI_RECV_CHIP_VER \
+	.type = HCI_CHIP_VER_PKT, \
+	.hlen = HCI_MRVL_PKT_SIZE, \
+	.loff = 0, \
+	.lsize = 0, \
+	.maxlen = HCI_MRVL_PKT_SIZE
+
+#define HCI_RECV_FW_REQ \
+	.type = HCI_FW_REQ_PKT, \
+	.hlen = HCI_MRVL_PKT_SIZE, \
+	.loff = 0, \
+	.lsize = 0, \
+	.maxlen = HCI_MRVL_PKT_SIZE
+
+static const struct h4_recv_pkt mrvl_recv_pkts[] = {
+	{ H4_RECV_ACL,       .recv = hci_recv_frame     },
+	{ H4_RECV_SCO,       .recv = hci_recv_frame     },
+	{ H4_RECV_EVENT,     .recv = hci_recv_frame     },
+	{ HCI_RECV_FW_REQ,   .recv = mrvl_recv_fw_req   },
+	{ HCI_RECV_CHIP_VER, .recv = mrvl_recv_chip_ver },
+};
+
+static int mrvl_recv(struct hci_uart *hu, const void *data, int count)
+{
+	struct mrvl_data *mrvl = hu->priv;
+
+	if (!test_bit(HCI_UART_REGISTERED, &hu->flags))
+		return -EUNATCH;
+
+	mrvl->rx_skb = h4_recv_buf(hu->hdev, mrvl->rx_skb, data, count,
+				    mrvl_recv_pkts,
+				    ARRAY_SIZE(mrvl_recv_pkts));
+	if (IS_ERR(mrvl->rx_skb)) {
+		int err = PTR_ERR(mrvl->rx_skb);
+		bt_dev_err(hu->hdev, "Frame reassembly failed (%d)", err);
+		mrvl->rx_skb = NULL;
+		return err;
+	}
+
+	return count;
+}
+
+static int mrvl_load_firmware(struct hci_dev *hdev, const char *name)
+{
+	struct hci_uart *hu = hci_get_drvdata(hdev);
+	struct mrvl_data *mrvl = hu->priv;
+	const struct firmware *fw = NULL;
+	const u8 *fw_ptr, *fw_max;
+	int err;
+
+	err = request_firmware(&fw, name, &hdev->dev);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to load firmware file %s", name);
+		return err;
+	}
+
+	fw_ptr = fw->data;
+	fw_max = fw->data + fw->size;
+
+	bt_dev_info(hdev, "Loading %s", name);
+
+	set_bit(STATE_FW_REQ_PENDING, &mrvl->flags);
+
+	while (fw_ptr <= fw_max) {
+		struct sk_buff *skb;
+
+		/* Controller drives the firmware load by sending firmware
+		 * request packets containing the expected fragment size.
+		 */
+		err = wait_on_bit_timeout(&mrvl->flags, STATE_FW_REQ_PENDING,
+					  TASK_INTERRUPTIBLE,
+					  msecs_to_jiffies(2000));
+		if (err == 1) {
+			bt_dev_err(hdev, "Firmware load interrupted");
+			err = -EINTR;
+			break;
+		} else if (err) {
+			bt_dev_err(hdev, "Firmware request timeout");
+			err = -ETIMEDOUT;
+			break;
+		}
+
+		bt_dev_dbg(hdev, "Firmware request, expecting %d bytes",
+			   mrvl->tx_len);
+
+		if (fw_ptr == fw_max) {
+			/* Controller requests a null size once firmware is
+			 * fully loaded. If controller expects more data, there
+			 * is an issue.
+			 */
+			if (!mrvl->tx_len) {
+				bt_dev_info(hdev, "Firmware loading complete");
+			} else {
+				bt_dev_err(hdev, "Firmware loading failure");
+				err = -EINVAL;
+			}
+			break;
+		}
+
+		if (fw_ptr + mrvl->tx_len > fw_max) {
+			mrvl->tx_len = fw_max - fw_ptr;
+			bt_dev_dbg(hdev, "Adjusting tx_len to %d",
+				   mrvl->tx_len);
+		}
+
+		skb = bt_skb_alloc(mrvl->tx_len, GFP_KERNEL);
+		if (!skb) {
+			bt_dev_err(hdev, "Failed to alloc mem for FW packet");
+			err = -ENOMEM;
+			break;
+		}
+		bt_cb(skb)->pkt_type = MRVL_RAW_DATA;
+
+		memcpy(skb_put(skb, mrvl->tx_len), fw_ptr, mrvl->tx_len);
+		fw_ptr += mrvl->tx_len;
+
+		set_bit(STATE_FW_REQ_PENDING, &mrvl->flags);
+
+		skb_queue_tail(&mrvl->rawq, skb);
+		hci_uart_tx_wakeup(hu);
+	}
+
+	release_firmware(fw);
+	return err;
+}
+
+static int mrvl_setup(struct hci_uart *hu)
+{
+	int err;
+
+	hci_uart_set_flow_control(hu, true);
+
+	err = mrvl_load_firmware(hu->hdev, "mrvl/helper_uart_3000000.bin");
+	if (err) {
+		bt_dev_err(hu->hdev, "Unable to download firmware helper");
+		return -EINVAL;
+	}
+
+	hci_uart_set_baudrate(hu, 3000000);
+	hci_uart_set_flow_control(hu, false);
+
+	err = mrvl_load_firmware(hu->hdev, "mrvl/uart8897_bt.bin");
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static const struct hci_uart_proto mrvl_proto = {
+	.id		= HCI_UART_MRVL,
+	.name		= "Marvell",
+	.init_speed	= 115200,
+	.open		= mrvl_open,
+	.close		= mrvl_close,
+	.flush		= mrvl_flush,
+	.setup		= mrvl_setup,
+	.recv		= mrvl_recv,
+	.enqueue	= mrvl_enqueue,
+	.dequeue	= mrvl_dequeue,
+};
+
+int __init mrvl_init(void)
+{
+	return hci_uart_register_proto(&mrvl_proto);
+}
+
+int __exit mrvl_deinit(void)
+{
+	return hci_uart_unregister_proto(&mrvl_proto);
+}
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index 683c2b642057..6c867fbc56a7 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -397,7 +397,7 @@ static int qca_open(struct hci_uart *hu)
 	skb_queue_head_init(&qca->txq);
 	skb_queue_head_init(&qca->tx_wait_q);
 	spin_lock_init(&qca->hci_ibs_lock);
-	qca->workqueue = create_singlethread_workqueue("qca_wq");
+	qca->workqueue = alloc_ordered_workqueue("qca_wq", 0);
 	if (!qca->workqueue) {
 		BT_ERR("QCA Workqueue not initialized properly");
 		kfree(qca);
diff --git a/drivers/bluetooth/hci_uart.h b/drivers/bluetooth/hci_uart.h
index 839bad1d8152..070139513e65 100644
--- a/drivers/bluetooth/hci_uart.h
+++ b/drivers/bluetooth/hci_uart.h
@@ -35,7 +35,7 @@
 #define HCIUARTGETFLAGS		_IOR('U', 204, int)
 
 /* UART protocols */
-#define HCI_UART_MAX_PROTO	10
+#define HCI_UART_MAX_PROTO	12
 
 #define HCI_UART_H4	0
 #define HCI_UART_BCSP	1
@@ -47,6 +47,8 @@
 #define HCI_UART_BCM	7
 #define HCI_UART_QCA	8
 #define HCI_UART_AG6XX	9
+#define HCI_UART_NOKIA	10
+#define HCI_UART_MRVL	11
 
 #define HCI_UART_RAW_DEVICE	0
 #define HCI_UART_RESET_ON_INIT	1
@@ -189,3 +191,8 @@ int qca_deinit(void);
 int ag6xx_init(void);
 int ag6xx_deinit(void);
 #endif
+
+#ifdef CONFIG_BT_HCIUART_MRVL
+int mrvl_init(void);
+int mrvl_deinit(void);
+#endif
diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c
index 3ff229b2e7f3..c4a75a18dcae 100644
--- a/drivers/bluetooth/hci_vhci.c
+++ b/drivers/bluetooth/hci_vhci.c
@@ -377,21 +377,7 @@ static struct miscdevice vhci_miscdev = {
 	.fops	= &vhci_fops,
 	.minor	= VHCI_MINOR,
 };
-
-static int __init vhci_init(void)
-{
-	BT_INFO("Virtual HCI driver ver %s", VERSION);
-
-	return misc_register(&vhci_miscdev);
-}
-
-static void __exit vhci_exit(void)
-{
-	misc_deregister(&vhci_miscdev);
-}
-
-module_init(vhci_init);
-module_exit(vhci_exit);
+module_misc_device(vhci_miscdev);
 
 module_param(amp, bool, 0644);
 MODULE_PARM_DESC(amp, "Create AMP controller device");
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 3b205e212337..7010dcac9328 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -108,6 +108,14 @@ config OMAP_OCP2SCP
 	  OCP2SCP and in OMAP5, both USB PHY and SATA PHY is connected via
 	  OCP2SCP.
 
+config QCOM_EBI2
+	bool "Qualcomm External Bus Interface 2 (EBI2)"
+	depends on HAS_IOMEM
+	help
+	  Say y here to enable support for the Qualcomm External Bus
+	  Interface 2, which can be used to connect things like NAND Flash,
+	  SRAM, ethernet adapters, FPGAs and LCD displays.
+
 config SIMPLE_PM_BUS
 	bool "Simple Power-Managed Bus Driver"
 	depends on OF && PM
@@ -132,12 +140,8 @@ config SUNXI_RSB
 	  with various RSB based devices, such as AXP223, AXP8XX PMICs,
 	  and AC100/AC200 ICs.
 
-# TODO: This uses pm_clk_*() symbols that aren't exported in v4.7 and hence
-# the driver will fail to build as a module. However there are patches to
-# address that queued for v4.8, so this can be turned into a tristate symbol
-# after v4.8-rc1.
 config TEGRA_ACONNECT
-	bool "Tegra ACONNECT Bus Driver"
+	tristate "Tegra ACONNECT Bus Driver"
 	depends on ARCH_TEGRA_210_SOC
 	depends on OF && PM
 	select PM_CLK
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index ac84cc4348e3..c6cfa6b2606e 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -15,6 +15,7 @@ obj-$(CONFIG_MVEBU_MBUS) 	+= mvebu-mbus.o
 obj-$(CONFIG_OMAP_INTERCONNECT)	+= omap_l3_smx.o omap_l3_noc.o
 
 obj-$(CONFIG_OMAP_OCP2SCP)	+= omap-ocp2scp.o
+obj-$(CONFIG_QCOM_EBI2)		+= qcom-ebi2.o
 obj-$(CONFIG_SUNXI_RSB)		+= sunxi-rsb.o
 obj-$(CONFIG_SIMPLE_PM_BUS)	+= simple-pm-bus.o
 obj-$(CONFIG_TEGRA_ACONNECT)	+= tegra-aconnect.o
diff --git a/drivers/bus/arm-cci.c b/drivers/bus/arm-cci.c
index 5755907f836f..890082315054 100644
--- a/drivers/bus/arm-cci.c
+++ b/drivers/bus/arm-cci.c
@@ -144,15 +144,12 @@ struct cci_pmu {
 	int num_cntrs;
 	atomic_t active_events;
 	struct mutex reserve_mutex;
-	struct list_head entry;
+	struct hlist_node node;
 	cpumask_t cpus;
 };
 
 #define to_cci_pmu(c)	(container_of(c, struct cci_pmu, pmu))
 
-static DEFINE_MUTEX(cci_pmu_mutex);
-static LIST_HEAD(cci_pmu_list);
-
 enum cci_models {
 #ifdef CONFIG_ARM_CCI400_PMU
 	CCI400_R0,
@@ -551,7 +548,7 @@ static struct attribute *cci5xx_pmu_event_attrs[] = {
 	CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB),
 	CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC),
 	CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD),
-	CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snopp_rq_stall_tt_full, 0xE),
+	CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_stall_tt_full, 0xE),
 	CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF),
 	NULL
 };
@@ -1506,25 +1503,21 @@ static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev)
 	return perf_pmu_register(&cci_pmu->pmu, name, -1);
 }
 
-static int cci_pmu_offline_cpu(unsigned int cpu)
+static int cci_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 {
-	struct cci_pmu *cci_pmu;
+	struct cci_pmu *cci_pmu = hlist_entry_safe(node, struct cci_pmu, node);
 	unsigned int target;
 
-	mutex_lock(&cci_pmu_mutex);
-	list_for_each_entry(cci_pmu, &cci_pmu_list, entry) {
-		if (!cpumask_test_and_clear_cpu(cpu, &cci_pmu->cpus))
-			continue;
-		target = cpumask_any_but(cpu_online_mask, cpu);
-		if (target >= nr_cpu_ids)
-			continue;
-		/*
-		 * TODO: migrate context once core races on event->ctx have
-		 * been fixed.
-		 */
-		cpumask_set_cpu(target, &cci_pmu->cpus);
-	}
-	mutex_unlock(&cci_pmu_mutex);
+	if (!cpumask_test_and_clear_cpu(cpu, &cci_pmu->cpus))
+		return 0;
+	target = cpumask_any_but(cpu_online_mask, cpu);
+	if (target >= nr_cpu_ids)
+		return 0;
+	/*
+	 * TODO: migrate context once core races on event->ctx have
+	 * been fixed.
+	 */
+	cpumask_set_cpu(target, &cci_pmu->cpus);
 	return 0;
 }
 
@@ -1768,10 +1761,8 @@ static int cci_pmu_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	mutex_lock(&cci_pmu_mutex);
-	list_add(&cci_pmu->entry, &cci_pmu_list);
-	mutex_unlock(&cci_pmu_mutex);
-
+	cpuhp_state_add_instance_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE,
+					 &cci_pmu->node);
 	pr_info("ARM %s PMU driver probed", cci_pmu->model->name);
 	return 0;
 }
@@ -1804,9 +1795,9 @@ static int __init cci_platform_init(void)
 {
 	int ret;
 
-	ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE,
-					"AP_PERF_ARM_CCI_ONLINE", NULL,
-					cci_pmu_offline_cpu);
+	ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_CCI_ONLINE,
+				      "AP_PERF_ARM_CCI_ONLINE", NULL,
+				      cci_pmu_offline_cpu);
 	if (ret)
 		return ret;
 
diff --git a/drivers/bus/arm-ccn.c b/drivers/bus/arm-ccn.c
index 97a9185af433..d1074d9b38ba 100644
--- a/drivers/bus/arm-ccn.c
+++ b/drivers/bus/arm-ccn.c
@@ -167,7 +167,7 @@ struct arm_ccn_dt {
 	struct hrtimer hrtimer;
 
 	cpumask_t cpu;
-	struct list_head entry;
+	struct hlist_node node;
 
 	struct pmu pmu;
 };
@@ -187,11 +187,9 @@ struct arm_ccn {
 	struct arm_ccn_component *xp;
 
 	struct arm_ccn_dt dt;
+	int mn_id;
 };
 
-static DEFINE_MUTEX(arm_ccn_mutex);
-static LIST_HEAD(arm_ccn_list);
-
 static int arm_ccn_node_to_xp(int node)
 {
 	return node / CCN_NUM_XP_PORTS;
@@ -212,6 +210,7 @@ static int arm_ccn_node_to_xp_port(int node)
 #define CCN_CONFIG_TYPE(_config)	(((_config) >> 8) & 0xff)
 #define CCN_CONFIG_EVENT(_config)	(((_config) >> 16) & 0xff)
 #define CCN_CONFIG_PORT(_config)	(((_config) >> 24) & 0x3)
+#define CCN_CONFIG_BUS(_config)		(((_config) >> 24) & 0x3)
 #define CCN_CONFIG_VC(_config)		(((_config) >> 26) & 0x7)
 #define CCN_CONFIG_DIR(_config)		(((_config) >> 29) & 0x1)
 #define CCN_CONFIG_MASK(_config)	(((_config) >> 30) & 0xf)
@@ -241,6 +240,7 @@ static CCN_FORMAT_ATTR(xp, "config:0-7");
 static CCN_FORMAT_ATTR(type, "config:8-15");
 static CCN_FORMAT_ATTR(event, "config:16-23");
 static CCN_FORMAT_ATTR(port, "config:24-25");
+static CCN_FORMAT_ATTR(bus, "config:24-25");
 static CCN_FORMAT_ATTR(vc, "config:26-28");
 static CCN_FORMAT_ATTR(dir, "config:29-29");
 static CCN_FORMAT_ATTR(mask, "config:30-33");
@@ -253,6 +253,7 @@ static struct attribute *arm_ccn_pmu_format_attrs[] = {
 	&arm_ccn_pmu_format_attr_type.attr.attr,
 	&arm_ccn_pmu_format_attr_event.attr.attr,
 	&arm_ccn_pmu_format_attr_port.attr.attr,
+	&arm_ccn_pmu_format_attr_bus.attr.attr,
 	&arm_ccn_pmu_format_attr_vc.attr.attr,
 	&arm_ccn_pmu_format_attr_dir.attr.attr,
 	&arm_ccn_pmu_format_attr_mask.attr.attr,
@@ -328,6 +329,7 @@ struct arm_ccn_pmu_event {
 static ssize_t arm_ccn_pmu_event_show(struct device *dev,
 		struct device_attribute *attr, char *buf)
 {
+	struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
 	struct arm_ccn_pmu_event *event = container_of(attr,
 			struct arm_ccn_pmu_event, attr);
 	ssize_t res;
@@ -349,10 +351,17 @@ static ssize_t arm_ccn_pmu_event_show(struct device *dev,
 		break;
 	case CCN_TYPE_XP:
 		res += snprintf(buf + res, PAGE_SIZE - res,
-				",xp=?,port=?,vc=?,dir=?");
+				",xp=?,vc=?");
 		if (event->event == CCN_EVENT_WATCHPOINT)
 			res += snprintf(buf + res, PAGE_SIZE - res,
-					",cmp_l=?,cmp_h=?,mask=?");
+					",port=?,dir=?,cmp_l=?,cmp_h=?,mask=?");
+		else
+			res += snprintf(buf + res, PAGE_SIZE - res,
+					",bus=?");
+
+		break;
+	case CCN_TYPE_MN:
+		res += snprintf(buf + res, PAGE_SIZE - res, ",node=%d", ccn->mn_id);
 		break;
 	default:
 		res += snprintf(buf + res, PAGE_SIZE - res, ",node=?");
@@ -383,9 +392,9 @@ static umode_t arm_ccn_pmu_events_is_visible(struct kobject *kobj,
 }
 
 static struct arm_ccn_pmu_event arm_ccn_pmu_events[] = {
-	CCN_EVENT_MN(eobarrier, "dir=0,vc=0,cmp_h=0x1c00", CCN_IDX_MASK_OPCODE),
-	CCN_EVENT_MN(ecbarrier, "dir=0,vc=0,cmp_h=0x1e00", CCN_IDX_MASK_OPCODE),
-	CCN_EVENT_MN(dvmop, "dir=0,vc=0,cmp_h=0x2800", CCN_IDX_MASK_OPCODE),
+	CCN_EVENT_MN(eobarrier, "dir=1,vc=0,cmp_h=0x1c00", CCN_IDX_MASK_OPCODE),
+	CCN_EVENT_MN(ecbarrier, "dir=1,vc=0,cmp_h=0x1e00", CCN_IDX_MASK_OPCODE),
+	CCN_EVENT_MN(dvmop, "dir=1,vc=0,cmp_h=0x2800", CCN_IDX_MASK_OPCODE),
 	CCN_EVENT_HNI(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY),
 	CCN_EVENT_HNI(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY),
 	CCN_EVENT_HNI(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY),
@@ -733,9 +742,10 @@ static int arm_ccn_pmu_event_init(struct perf_event *event)
 
 	if (has_branch_stack(event) || event->attr.exclude_user ||
 			event->attr.exclude_kernel || event->attr.exclude_hv ||
-			event->attr.exclude_idle) {
+			event->attr.exclude_idle || event->attr.exclude_host ||
+			event->attr.exclude_guest) {
 		dev_warn(ccn->dev, "Can't exclude execution levels!\n");
-		return -EOPNOTSUPP;
+		return -EINVAL;
 	}
 
 	if (event->cpu < 0) {
@@ -759,6 +769,12 @@ static int arm_ccn_pmu_event_init(struct perf_event *event)
 
 	/* Validate node/xp vs topology */
 	switch (type) {
+	case CCN_TYPE_MN:
+		if (node_xp != ccn->mn_id) {
+			dev_warn(ccn->dev, "Invalid MN ID %d!\n", node_xp);
+			return -EINVAL;
+		}
+		break;
 	case CCN_TYPE_XP:
 		if (node_xp >= ccn->num_xps) {
 			dev_warn(ccn->dev, "Invalid XP ID %d!\n", node_xp);
@@ -886,6 +902,10 @@ static void arm_ccn_pmu_xp_dt_config(struct perf_event *event, int enable)
 	struct arm_ccn_component *xp;
 	u32 val, dt_cfg;
 
+	/* Nothing to do for cycle counter */
+	if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER)
+		return;
+
 	if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP)
 		xp = &ccn->xp[CCN_CONFIG_XP(event->attr.config)];
 	else
@@ -917,38 +937,17 @@ static void arm_ccn_pmu_event_start(struct perf_event *event, int flags)
 			arm_ccn_pmu_read_counter(ccn, hw->idx));
 	hw->state = 0;
 
-	/*
-	 * Pin the timer, so that the overflows are handled by the chosen
-	 * event->cpu (this is the same one as presented in "cpumask"
-	 * attribute).
-	 */
-	if (!ccn->irq)
-		hrtimer_start(&ccn->dt.hrtimer, arm_ccn_pmu_timer_period(),
-				HRTIMER_MODE_REL_PINNED);
-
 	/* Set the DT bus input, engaging the counter */
 	arm_ccn_pmu_xp_dt_config(event, 1);
 }
 
 static void arm_ccn_pmu_event_stop(struct perf_event *event, int flags)
 {
-	struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
 	struct hw_perf_event *hw = &event->hw;
-	u64 timeout;
 
 	/* Disable counting, setting the DT bus to pass-through mode */
 	arm_ccn_pmu_xp_dt_config(event, 0);
 
-	if (!ccn->irq)
-		hrtimer_cancel(&ccn->dt.hrtimer);
-
-	/* Let the DT bus drain */
-	timeout = arm_ccn_pmu_read_counter(ccn, CCN_IDX_PMU_CYCLE_COUNTER) +
-			ccn->num_xps;
-	while (arm_ccn_pmu_read_counter(ccn, CCN_IDX_PMU_CYCLE_COUNTER) <
-			timeout)
-		cpu_relax();
-
 	if (flags & PERF_EF_UPDATE)
 		arm_ccn_pmu_event_update(event);
 
@@ -988,7 +987,7 @@ static void arm_ccn_pmu_xp_watchpoint_config(struct perf_event *event)
 
 	/* Comparison values */
 	writel(cmp_l & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_L(wp));
-	writel((cmp_l >> 32) & 0xefffffff,
+	writel((cmp_l >> 32) & 0x7fffffff,
 			source->base + CCN_XP_DT_CMP_VAL_L(wp) + 4);
 	writel(cmp_h & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_H(wp));
 	writel((cmp_h >> 32) & 0x0fffffff,
@@ -996,7 +995,7 @@ static void arm_ccn_pmu_xp_watchpoint_config(struct perf_event *event)
 
 	/* Mask */
 	writel(mask_l & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_L(wp));
-	writel((mask_l >> 32) & 0xefffffff,
+	writel((mask_l >> 32) & 0x7fffffff,
 			source->base + CCN_XP_DT_CMP_MASK_L(wp) + 4);
 	writel(mask_h & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_H(wp));
 	writel((mask_h >> 32) & 0x0fffffff,
@@ -1014,7 +1013,7 @@ static void arm_ccn_pmu_xp_event_config(struct perf_event *event)
 	hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(hw->config_base);
 
 	id = (CCN_CONFIG_VC(event->attr.config) << 4) |
-			(CCN_CONFIG_PORT(event->attr.config) << 3) |
+			(CCN_CONFIG_BUS(event->attr.config) << 3) |
 			(CCN_CONFIG_EVENT(event->attr.config) << 0);
 
 	val = readl(source->base + CCN_XP_PMU_EVENT_SEL);
@@ -1099,15 +1098,31 @@ static void arm_ccn_pmu_event_config(struct perf_event *event)
 	spin_unlock(&ccn->dt.config_lock);
 }
 
+static int arm_ccn_pmu_active_counters(struct arm_ccn *ccn)
+{
+	return bitmap_weight(ccn->dt.pmu_counters_mask,
+			     CCN_NUM_PMU_EVENT_COUNTERS + 1);
+}
+
 static int arm_ccn_pmu_event_add(struct perf_event *event, int flags)
 {
 	int err;
 	struct hw_perf_event *hw = &event->hw;
+	struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
 
 	err = arm_ccn_pmu_event_alloc(event);
 	if (err)
 		return err;
 
+	/*
+	 * Pin the timer, so that the overflows are handled by the chosen
+	 * event->cpu (this is the same one as presented in "cpumask"
+	 * attribute).
+	 */
+	if (!ccn->irq && arm_ccn_pmu_active_counters(ccn) == 1)
+		hrtimer_start(&ccn->dt.hrtimer, arm_ccn_pmu_timer_period(),
+			      HRTIMER_MODE_REL_PINNED);
+
 	arm_ccn_pmu_event_config(event);
 
 	hw->state = PERF_HES_STOPPED;
@@ -1120,9 +1135,14 @@ static int arm_ccn_pmu_event_add(struct perf_event *event, int flags)
 
 static void arm_ccn_pmu_event_del(struct perf_event *event, int flags)
 {
+	struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
+
 	arm_ccn_pmu_event_stop(event, PERF_EF_UPDATE);
 
 	arm_ccn_pmu_event_release(event);
+
+	if (!ccn->irq && arm_ccn_pmu_active_counters(ccn) == 0)
+		hrtimer_cancel(&ccn->dt.hrtimer);
 }
 
 static void arm_ccn_pmu_event_read(struct perf_event *event)
@@ -1130,6 +1150,24 @@ static void arm_ccn_pmu_event_read(struct perf_event *event)
 	arm_ccn_pmu_event_update(event);
 }
 
+static void arm_ccn_pmu_enable(struct pmu *pmu)
+{
+	struct arm_ccn *ccn = pmu_to_arm_ccn(pmu);
+
+	u32 val = readl(ccn->dt.base + CCN_DT_PMCR);
+	val |= CCN_DT_PMCR__PMU_EN;
+	writel(val, ccn->dt.base + CCN_DT_PMCR);
+}
+
+static void arm_ccn_pmu_disable(struct pmu *pmu)
+{
+	struct arm_ccn *ccn = pmu_to_arm_ccn(pmu);
+
+	u32 val = readl(ccn->dt.base + CCN_DT_PMCR);
+	val &= ~CCN_DT_PMCR__PMU_EN;
+	writel(val, ccn->dt.base + CCN_DT_PMCR);
+}
+
 static irqreturn_t arm_ccn_pmu_overflow_handler(struct arm_ccn_dt *dt)
 {
 	u32 pmovsr = readl(dt->base + CCN_DT_PMOVSR);
@@ -1173,30 +1211,24 @@ static enum hrtimer_restart arm_ccn_pmu_timer_handler(struct hrtimer *hrtimer)
 }
 
 
-static int arm_ccn_pmu_offline_cpu(unsigned int cpu)
+static int arm_ccn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 {
-	struct arm_ccn_dt *dt;
+	struct arm_ccn_dt *dt = hlist_entry_safe(node, struct arm_ccn_dt, node);
+	struct arm_ccn *ccn = container_of(dt, struct arm_ccn, dt);
 	unsigned int target;
 
-	mutex_lock(&arm_ccn_mutex);
-	list_for_each_entry(dt, &arm_ccn_list, entry) {
-		struct arm_ccn *ccn = container_of(dt, struct arm_ccn, dt);
-
-		if (!cpumask_test_and_clear_cpu(cpu, &dt->cpu))
-			continue;
-		target = cpumask_any_but(cpu_online_mask, cpu);
-		if (target >= nr_cpu_ids)
-			continue;
-		perf_pmu_migrate_context(&dt->pmu, cpu, target);
-		cpumask_set_cpu(target, &dt->cpu);
-		if (ccn->irq)
-			WARN_ON(irq_set_affinity_hint(ccn->irq, &dt->cpu) != 0);
-	}
-	mutex_unlock(&arm_ccn_mutex);
+	if (!cpumask_test_and_clear_cpu(cpu, &dt->cpu))
+		return 0;
+	target = cpumask_any_but(cpu_online_mask, cpu);
+	if (target >= nr_cpu_ids)
+		return 0;
+	perf_pmu_migrate_context(&dt->pmu, cpu, target);
+	cpumask_set_cpu(target, &dt->cpu);
+	if (ccn->irq)
+		WARN_ON(irq_set_affinity_hint(ccn->irq, &dt->cpu) != 0);
 	return 0;
 }
 
-
 static DEFINE_IDA(arm_ccn_pmu_ida);
 
 static int arm_ccn_pmu_init(struct arm_ccn *ccn)
@@ -1252,6 +1284,8 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn)
 		.start = arm_ccn_pmu_event_start,
 		.stop = arm_ccn_pmu_event_stop,
 		.read = arm_ccn_pmu_event_read,
+		.pmu_enable = arm_ccn_pmu_enable,
+		.pmu_disable = arm_ccn_pmu_disable,
 	};
 
 	/* No overflow interrupt? Have to use a timer instead. */
@@ -1278,9 +1312,8 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn)
 	if (err)
 		goto error_pmu_register;
 
-	mutex_lock(&arm_ccn_mutex);
-	list_add(&ccn->dt.entry, &arm_ccn_list);
-	mutex_unlock(&arm_ccn_mutex);
+	cpuhp_state_add_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
+					 &ccn->dt.node);
 	return 0;
 
 error_pmu_register:
@@ -1296,10 +1329,8 @@ static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn)
 {
 	int i;
 
-	mutex_lock(&arm_ccn_mutex);
-	list_del(&ccn->dt.entry);
-	mutex_unlock(&arm_ccn_mutex);
-
+	cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
+					    &ccn->dt.node);
 	if (ccn->irq)
 		irq_set_affinity_hint(ccn->irq, NULL);
 	for (i = 0; i < ccn->num_xps; i++)
@@ -1361,6 +1392,8 @@ static int arm_ccn_init_nodes(struct arm_ccn *ccn, int region,
 
 	switch (type) {
 	case CCN_TYPE_MN:
+		ccn->mn_id = id;
+		return 0;
 	case CCN_TYPE_DT:
 		return 0;
 	case CCN_TYPE_XP:
@@ -1471,8 +1504,9 @@ static int arm_ccn_probe(struct platform_device *pdev)
 		/* Can set 'disable' bits, so can acknowledge interrupts */
 		writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE,
 				ccn->base + CCN_MN_ERRINT_STATUS);
-		err = devm_request_irq(ccn->dev, irq, arm_ccn_irq_handler, 0,
-				dev_name(ccn->dev), ccn);
+		err = devm_request_irq(ccn->dev, irq, arm_ccn_irq_handler,
+				       IRQF_NOBALANCING | IRQF_NO_THREAD,
+				       dev_name(ccn->dev), ccn);
 		if (err)
 			return err;
 
@@ -1527,9 +1561,9 @@ static int __init arm_ccn_init(void)
 {
 	int i, ret;
 
-	ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
-					"AP_PERF_ARM_CCN_ONLINE", NULL,
-					arm_ccn_pmu_offline_cpu);
+	ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_CCN_ONLINE,
+				      "AP_PERF_ARM_CCN_ONLINE", NULL,
+				      arm_ccn_pmu_offline_cpu);
 	if (ret)
 		return ret;
 
@@ -1541,7 +1575,7 @@ static int __init arm_ccn_init(void)
 
 static void __exit arm_ccn_exit(void)
 {
-	cpuhp_remove_state_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE);
+	cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CCN_ONLINE);
 	platform_driver_unregister(&arm_ccn_driver);
 }
 
diff --git a/drivers/bus/mips_cdmm.c b/drivers/bus/mips_cdmm.c
index cad49bc38b3e..1b14256376d2 100644
--- a/drivers/bus/mips_cdmm.c
+++ b/drivers/bus/mips_cdmm.c
@@ -596,19 +596,20 @@ BUILD_PERDEV_HELPER(cpu_down)       /* int mips_cdmm_cpu_down_helper(...) */
 BUILD_PERDEV_HELPER(cpu_up)         /* int mips_cdmm_cpu_up_helper(...) */
 
 /**
- * mips_cdmm_bus_down() - Tear down the CDMM bus.
- * @data:	Pointer to unsigned int CPU number.
+ * mips_cdmm_cpu_down_prep() - Callback for CPUHP DOWN_PREP:
+ *			       Tear down the CDMM bus.
+ * @cpu:	unsigned int CPU number.
  *
  * This function is executed on the hotplugged CPU and calls the CDMM
  * driver cpu_down callback for all devices on that CPU.
  */
-static long mips_cdmm_bus_down(void *data)
+static int mips_cdmm_cpu_down_prep(unsigned int cpu)
 {
 	struct mips_cdmm_bus *bus;
 	long ret;
 
 	/* Inform all the devices on the bus */
-	ret = bus_for_each_dev(&mips_cdmm_bustype, NULL, data,
+	ret = bus_for_each_dev(&mips_cdmm_bustype, NULL, &cpu,
 			       mips_cdmm_cpu_down_helper);
 
 	/*
@@ -623,8 +624,8 @@ static long mips_cdmm_bus_down(void *data)
 }
 
 /**
- * mips_cdmm_bus_up() - Bring up the CDMM bus.
- * @data:	Pointer to unsigned int CPU number.
+ * mips_cdmm_cpu_online() - Callback for CPUHP ONLINE: Bring up the CDMM bus.
+ * @cpu:	unsigned int CPU number.
  *
  * This work_on_cpu callback function is executed on a given CPU to discover
  * CDMM devices on that CPU, or to call the CDMM driver cpu_up callback for all
@@ -634,7 +635,7 @@ static long mips_cdmm_bus_down(void *data)
  * initialisation. When CPUs are brought online the function is
  * invoked directly on the hotplugged CPU.
  */
-static long mips_cdmm_bus_up(void *data)
+static int mips_cdmm_cpu_online(unsigned int cpu)
 {
 	struct mips_cdmm_bus *bus;
 	long ret;
@@ -651,51 +652,13 @@ static long mips_cdmm_bus_up(void *data)
 		mips_cdmm_bus_discover(bus);
 	else
 		/* Inform all the devices on the bus */
-		ret = bus_for_each_dev(&mips_cdmm_bustype, NULL, data,
+		ret = bus_for_each_dev(&mips_cdmm_bustype, NULL, &cpu,
 				       mips_cdmm_cpu_up_helper);
 
 	return ret;
 }
 
 /**
- * mips_cdmm_cpu_notify() - Take action when a CPU is going online or offline.
- * @nb:		CPU notifier block .
- * @action:	Event that has taken place (CPU_*).
- * @data:	CPU number.
- *
- * This notifier is used to keep the CDMM buses updated as CPUs are offlined and
- * onlined. When CPUs go offline or come back online, so does their CDMM bus, so
- * devices must be informed. Also when CPUs come online for the first time the
- * devices on the CDMM bus need discovering.
- *
- * Returns:	NOTIFY_OK if event was used.
- *		NOTIFY_DONE if we didn't care.
- */
-static int mips_cdmm_cpu_notify(struct notifier_block *nb,
-				unsigned long action, void *data)
-{
-	unsigned int cpu = (unsigned int)data;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		mips_cdmm_bus_up(&cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-		mips_cdmm_bus_down(&cpu);
-		break;
-	default:
-		return NOTIFY_DONE;
-	}
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block mips_cdmm_cpu_nb = {
-	.notifier_call = mips_cdmm_cpu_notify,
-};
-
-/**
  * mips_cdmm_init() - Initialise CDMM bus.
  *
  * Initialise CDMM bus, discover CDMM devices for online CPUs, and arrange for
@@ -703,7 +666,6 @@ static struct notifier_block mips_cdmm_cpu_nb = {
  */
 static int __init mips_cdmm_init(void)
 {
-	unsigned int cpu;
 	int ret;
 
 	/* Register the bus */
@@ -712,19 +674,11 @@ static int __init mips_cdmm_init(void)
 		return ret;
 
 	/* We want to be notified about new CPUs */
-	ret = register_cpu_notifier(&mips_cdmm_cpu_nb);
-	if (ret) {
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "bus/cdmm:online",
+				mips_cdmm_cpu_online, mips_cdmm_cpu_down_prep);
+	if (ret < 0)
 		pr_warn("cdmm: Failed to register CPU notifier\n");
-		goto out;
-	}
-
-	/* Discover devices on CDMM of online CPUs */
-	for_each_online_cpu(cpu)
-		work_on_cpu(cpu, mips_cdmm_bus_up, &cpu);
 
-	return 0;
-out:
-	bus_unregister(&mips_cdmm_bustype);
 	return ret;
 }
 subsys_initcall(mips_cdmm_init);
diff --git a/drivers/bus/qcom-ebi2.c b/drivers/bus/qcom-ebi2.c
new file mode 100644
index 000000000000..a6444244c411
--- /dev/null
+++ b/drivers/bus/qcom-ebi2.c
@@ -0,0 +1,408 @@
+/*
+ * Qualcomm External Bus Interface 2 (EBI2) driver
+ * an older version of the Qualcomm Parallel Interface Controller (QPIC)
+ *
+ * Copyright (C) 2016 Linaro Ltd.
+ *
+ * Author: Linus Walleij <linus.walleij@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2, as
+ * published by the Free Software Foundation.
+ *
+ * See the device tree bindings for this block for more details on the
+ * hardware.
+ */
+
+#include <linux/module.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/bitops.h>
+
+/*
+ * CS0, CS1, CS4 and CS5 are two bits wide, CS2 and CS3 are one bit.
+ */
+#define EBI2_CS0_ENABLE_MASK BIT(0)|BIT(1)
+#define EBI2_CS1_ENABLE_MASK BIT(2)|BIT(3)
+#define EBI2_CS2_ENABLE_MASK BIT(4)
+#define EBI2_CS3_ENABLE_MASK BIT(5)
+#define EBI2_CS4_ENABLE_MASK BIT(6)|BIT(7)
+#define EBI2_CS5_ENABLE_MASK BIT(8)|BIT(9)
+#define EBI2_CSN_MASK GENMASK(9, 0)
+
+#define EBI2_XMEM_CFG 0x0000 /* Power management etc */
+
+/*
+ * SLOW CSn CFG
+ *
+ * Bits 31-28: RECOVERY recovery cycles (0 = 1, 1 = 2 etc) this is the time the
+ *             memory continues to drive the data bus after OE is de-asserted.
+ *             Inserted when reading one CS and switching to another CS or read
+ *             followed by write on the same CS. Valid values 0 thru 15.
+ * Bits 27-24: WR_HOLD write hold cycles, these are extra cycles inserted after
+ *             every write minimum 1. The data out is driven from the time WE is
+ *             asserted until CS is asserted. With a hold of 1, the CS stays
+ *             active for 1 extra cycle etc. Valid values 0 thru 15.
+ * Bits 23-16: WR_DELTA initial latency for write cycles inserted for the first
+ *             write to a page or burst memory
+ * Bits 15-8:  RD_DELTA initial latency for read cycles inserted for the first
+ *             read to a page or burst memory
+ * Bits 7-4:   WR_WAIT number of wait cycles for every write access, 0=1 cycle
+ *             so 1 thru 16 cycles.
+ * Bits 3-0:   RD_WAIT number of wait cycles for every read access, 0=1 cycle
+ *             so 1 thru 16 cycles.
+ */
+#define EBI2_XMEM_CS0_SLOW_CFG 0x0008
+#define EBI2_XMEM_CS1_SLOW_CFG 0x000C
+#define EBI2_XMEM_CS2_SLOW_CFG 0x0010
+#define EBI2_XMEM_CS3_SLOW_CFG 0x0014
+#define EBI2_XMEM_CS4_SLOW_CFG 0x0018
+#define EBI2_XMEM_CS5_SLOW_CFG 0x001C
+
+#define EBI2_XMEM_RECOVERY_SHIFT	28
+#define EBI2_XMEM_WR_HOLD_SHIFT		24
+#define EBI2_XMEM_WR_DELTA_SHIFT	16
+#define EBI2_XMEM_RD_DELTA_SHIFT	8
+#define EBI2_XMEM_WR_WAIT_SHIFT		4
+#define EBI2_XMEM_RD_WAIT_SHIFT		0
+
+/*
+ * FAST CSn CFG
+ * Bits 31-28: ?
+ * Bits 27-24: RD_HOLD: the length in cycles of the first segment of a read
+ *             transfer. For a single read trandfer this will be the time
+ *             from CS assertion to OE assertion.
+ * Bits 18-24: ?
+ * Bits 17-16: ADV_OE_RECOVERY, the number of cycles elapsed before an OE
+ *             assertion, with respect to the cycle where ADV is asserted.
+ *             2 means 2 cycles between ADV and OE. Values 0, 1, 2 or 3.
+ * Bits 5:     ADDR_HOLD_ENA, The address is held for an extra cycle to meet
+ *             hold time requirements with ADV assertion.
+ *
+ * The manual mentions "write precharge cycles" and "precharge cycles".
+ * We have not been able to figure out which bit fields these correspond to
+ * in the hardware, or what valid values exist. The current hypothesis is that
+ * this is something just used on the FAST chip selects. There is also a "byte
+ * device enable" flag somewhere for 8bit memories.
+ */
+#define EBI2_XMEM_CS0_FAST_CFG 0x0028
+#define EBI2_XMEM_CS1_FAST_CFG 0x002C
+#define EBI2_XMEM_CS2_FAST_CFG 0x0030
+#define EBI2_XMEM_CS3_FAST_CFG 0x0034
+#define EBI2_XMEM_CS4_FAST_CFG 0x0038
+#define EBI2_XMEM_CS5_FAST_CFG 0x003C
+
+#define EBI2_XMEM_RD_HOLD_SHIFT		24
+#define EBI2_XMEM_ADV_OE_RECOVERY_SHIFT	16
+#define EBI2_XMEM_ADDR_HOLD_ENA_SHIFT	5
+
+/**
+ * struct cs_data - struct with info on a chipselect setting
+ * @enable_mask: mask to enable the chipselect in the EBI2 config
+ * @slow_cfg0: offset to XMEMC slow CS config
+ * @fast_cfg1: offset to XMEMC fast CS config
+ */
+struct cs_data {
+	u32 enable_mask;
+	u16 slow_cfg;
+	u16 fast_cfg;
+};
+
+static const struct cs_data cs_info[] = {
+	{
+		/* CS0 */
+		.enable_mask = EBI2_CS0_ENABLE_MASK,
+		.slow_cfg = EBI2_XMEM_CS0_SLOW_CFG,
+		.fast_cfg = EBI2_XMEM_CS0_FAST_CFG,
+	},
+	{
+		/* CS1 */
+		.enable_mask = EBI2_CS1_ENABLE_MASK,
+		.slow_cfg = EBI2_XMEM_CS1_SLOW_CFG,
+		.fast_cfg = EBI2_XMEM_CS1_FAST_CFG,
+	},
+	{
+		/* CS2 */
+		.enable_mask = EBI2_CS2_ENABLE_MASK,
+		.slow_cfg = EBI2_XMEM_CS2_SLOW_CFG,
+		.fast_cfg = EBI2_XMEM_CS2_FAST_CFG,
+	},
+	{
+		/* CS3 */
+		.enable_mask = EBI2_CS3_ENABLE_MASK,
+		.slow_cfg = EBI2_XMEM_CS3_SLOW_CFG,
+		.fast_cfg = EBI2_XMEM_CS3_FAST_CFG,
+	},
+	{
+		/* CS4 */
+		.enable_mask = EBI2_CS4_ENABLE_MASK,
+		.slow_cfg = EBI2_XMEM_CS4_SLOW_CFG,
+		.fast_cfg = EBI2_XMEM_CS4_FAST_CFG,
+	},
+	{
+		/* CS5 */
+		.enable_mask = EBI2_CS5_ENABLE_MASK,
+		.slow_cfg = EBI2_XMEM_CS5_SLOW_CFG,
+		.fast_cfg = EBI2_XMEM_CS5_FAST_CFG,
+	},
+};
+
+/**
+ * struct ebi2_xmem_prop - describes an XMEM config property
+ * @prop: the device tree binding name
+ * @max: maximum value for the property
+ * @slowreg: true if this property is in the SLOW CS config register
+ * else it is assumed to be in the FAST config register
+ * @shift: the bit field start in the SLOW or FAST register for this
+ * property
+ */
+struct ebi2_xmem_prop {
+	const char *prop;
+	u32 max;
+	bool slowreg;
+	u16 shift;
+};
+
+static const struct ebi2_xmem_prop xmem_props[] = {
+	{
+		.prop = "qcom,xmem-recovery-cycles",
+		.max = 15,
+		.slowreg = true,
+		.shift = EBI2_XMEM_RECOVERY_SHIFT,
+	},
+	{
+		.prop = "qcom,xmem-write-hold-cycles",
+		.max = 15,
+		.slowreg = true,
+		.shift = EBI2_XMEM_WR_HOLD_SHIFT,
+	},
+	{
+		.prop = "qcom,xmem-write-delta-cycles",
+		.max = 255,
+		.slowreg = true,
+		.shift = EBI2_XMEM_WR_DELTA_SHIFT,
+	},
+	{
+		.prop = "qcom,xmem-read-delta-cycles",
+		.max = 255,
+		.slowreg = true,
+		.shift = EBI2_XMEM_RD_DELTA_SHIFT,
+	},
+	{
+		.prop = "qcom,xmem-write-wait-cycles",
+		.max = 15,
+		.slowreg = true,
+		.shift = EBI2_XMEM_WR_WAIT_SHIFT,
+	},
+	{
+		.prop = "qcom,xmem-read-wait-cycles",
+		.max = 15,
+		.slowreg = true,
+		.shift = EBI2_XMEM_RD_WAIT_SHIFT,
+	},
+	{
+		.prop = "qcom,xmem-address-hold-enable",
+		.max = 1, /* boolean prop */
+		.slowreg = false,
+		.shift = EBI2_XMEM_ADDR_HOLD_ENA_SHIFT,
+	},
+	{
+		.prop = "qcom,xmem-adv-to-oe-recovery-cycles",
+		.max = 3,
+		.slowreg = false,
+		.shift = EBI2_XMEM_ADV_OE_RECOVERY_SHIFT,
+	},
+	{
+		.prop = "qcom,xmem-read-hold-cycles",
+		.max = 15,
+		.slowreg = false,
+		.shift = EBI2_XMEM_RD_HOLD_SHIFT,
+	},
+};
+
+static void qcom_ebi2_setup_chipselect(struct device_node *np,
+				       struct device *dev,
+				       void __iomem *ebi2_base,
+				       void __iomem *ebi2_xmem,
+				       u32 csindex)
+{
+	const struct cs_data *csd;
+	u32 slowcfg, fastcfg;
+	u32 val;
+	int ret;
+	int i;
+
+	csd = &cs_info[csindex];
+	val = readl(ebi2_base);
+	val |= csd->enable_mask;
+	writel(val, ebi2_base);
+	dev_dbg(dev, "enabled CS%u\n", csindex);
+
+	/* Next set up the XMEMC */
+	slowcfg = 0;
+	fastcfg = 0;
+
+	for (i = 0; i < ARRAY_SIZE(xmem_props); i++) {
+		const struct ebi2_xmem_prop *xp = &xmem_props[i];
+
+		/* All are regular u32 values */
+		ret = of_property_read_u32(np, xp->prop, &val);
+		if (ret) {
+			dev_dbg(dev, "could not read %s for CS%d\n",
+				xp->prop, csindex);
+			continue;
+		}
+
+		/* First check boolean props */
+		if (xp->max == 1 && val) {
+			if (xp->slowreg)
+				slowcfg |= BIT(xp->shift);
+			else
+				fastcfg |= BIT(xp->shift);
+			dev_dbg(dev, "set %s flag\n", xp->prop);
+			continue;
+		}
+
+		/* We're dealing with an u32 */
+		if (val > xp->max) {
+			dev_err(dev,
+				"too high value for %s: %u, capped at %u\n",
+				xp->prop, val, xp->max);
+			val = xp->max;
+		}
+		if (xp->slowreg)
+			slowcfg |= (val << xp->shift);
+		else
+			fastcfg |= (val << xp->shift);
+		dev_dbg(dev, "set %s to %u\n", xp->prop, val);
+	}
+
+	dev_info(dev, "CS%u: SLOW CFG 0x%08x, FAST CFG 0x%08x\n",
+		 csindex, slowcfg, fastcfg);
+
+	if (slowcfg)
+		writel(slowcfg, ebi2_xmem + csd->slow_cfg);
+	if (fastcfg)
+		writel(fastcfg, ebi2_xmem + csd->fast_cfg);
+}
+
+static int qcom_ebi2_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct device_node *child;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	void __iomem *ebi2_base;
+	void __iomem *ebi2_xmem;
+	struct clk *ebi2xclk;
+	struct clk *ebi2clk;
+	bool have_children = false;
+	u32 val;
+	int ret;
+
+	ebi2xclk = devm_clk_get(dev, "ebi2x");
+	if (IS_ERR(ebi2xclk))
+		return PTR_ERR(ebi2xclk);
+
+	ret = clk_prepare_enable(ebi2xclk);
+	if (ret) {
+		dev_err(dev, "could not enable EBI2X clk (%d)\n", ret);
+		return ret;
+	}
+
+	ebi2clk = devm_clk_get(dev, "ebi2");
+	if (IS_ERR(ebi2clk)) {
+		ret = PTR_ERR(ebi2clk);
+		goto err_disable_2x_clk;
+	}
+
+	ret = clk_prepare_enable(ebi2clk);
+	if (ret) {
+		dev_err(dev, "could not enable EBI2 clk\n");
+		goto err_disable_2x_clk;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	ebi2_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(ebi2_base)) {
+		ret = PTR_ERR(ebi2_base);
+		goto err_disable_clk;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	ebi2_xmem = devm_ioremap_resource(dev, res);
+	if (IS_ERR(ebi2_xmem)) {
+		ret = PTR_ERR(ebi2_xmem);
+		goto err_disable_clk;
+	}
+
+	/* Allegedly this turns the power save mode off */
+	writel(0UL, ebi2_xmem + EBI2_XMEM_CFG);
+
+	/* Disable all chipselects */
+	val = readl(ebi2_base);
+	val &= ~EBI2_CSN_MASK;
+	writel(val, ebi2_base);
+
+	/* Walk over the child nodes and see what chipselects we use */
+	for_each_available_child_of_node(np, child) {
+		u32 csindex;
+
+		/* Figure out the chipselect */
+		ret = of_property_read_u32(child, "reg", &csindex);
+		if (ret)
+			return ret;
+
+		if (csindex > 5) {
+			dev_err(dev,
+				"invalid chipselect %u, we only support 0-5\n",
+				csindex);
+			continue;
+		}
+
+		qcom_ebi2_setup_chipselect(child,
+					   dev,
+					   ebi2_base,
+					   ebi2_xmem,
+					   csindex);
+
+		/* We have at least one child */
+		have_children = true;
+	}
+
+	if (have_children)
+		return of_platform_default_populate(np, NULL, dev);
+	return 0;
+
+err_disable_clk:
+	clk_disable_unprepare(ebi2clk);
+err_disable_2x_clk:
+	clk_disable_unprepare(ebi2xclk);
+
+	return ret;
+}
+
+static const struct of_device_id qcom_ebi2_of_match[] = {
+	{ .compatible = "qcom,msm8660-ebi2", },
+	{ .compatible = "qcom,apq8060-ebi2", },
+	{ }
+};
+
+static struct platform_driver qcom_ebi2_driver = {
+	.probe = qcom_ebi2_probe,
+	.driver = {
+		.name = "qcom-ebi2",
+		.of_match_table = qcom_ebi2_of_match,
+	},
+};
+module_platform_driver(qcom_ebi2_driver);
+MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
+MODULE_DESCRIPTION("Qualcomm EBI2 driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/bus/tegra-aconnect.c b/drivers/bus/tegra-aconnect.c
index 7e4104b74fa8..084ae286fa23 100644
--- a/drivers/bus/tegra-aconnect.c
+++ b/drivers/bus/tegra-aconnect.c
@@ -15,24 +15,6 @@
 #include <linux/pm_clock.h>
 #include <linux/pm_runtime.h>
 
-static int tegra_aconnect_add_clock(struct device *dev, char *name)
-{
-	struct clk *clk;
-	int ret;
-
-	clk = clk_get(dev, name);
-	if (IS_ERR(clk)) {
-		dev_err(dev, "%s clock not found\n", name);
-		return PTR_ERR(clk);
-	}
-
-	ret = pm_clk_add_clk(dev, clk);
-	if (ret)
-		clk_put(clk);
-
-	return ret;
-}
-
 static int tegra_aconnect_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -44,11 +26,11 @@ static int tegra_aconnect_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	ret = tegra_aconnect_add_clock(&pdev->dev, "ape");
+	ret = of_pm_clk_add_clk(&pdev->dev, "ape");
 	if (ret)
 		goto clk_destroy;
 
-	ret = tegra_aconnect_add_clock(&pdev->dev, "apb2ape");
+	ret = of_pm_clk_add_clk(&pdev->dev, "apb2ape");
 	if (ret)
 		goto clk_destroy;
 
diff --git a/drivers/bus/vexpress-config.c b/drivers/bus/vexpress-config.c
index c3cb76b363c6..9efdf1de4035 100644
--- a/drivers/bus/vexpress-config.c
+++ b/drivers/bus/vexpress-config.c
@@ -178,6 +178,7 @@ static int vexpress_config_populate(struct device_node *node)
 
 	parent = class_find_device(vexpress_config_class, NULL, bridge,
 			vexpress_config_node_match);
+	of_node_put(bridge);
 	if (WARN_ON(!parent))
 		return -ENODEV;
 
diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c
index 44311296ec02..0f7d28a98b9a 100644
--- a/drivers/char/agp/intel-gtt.c
+++ b/drivers/char/agp/intel-gtt.c
@@ -845,6 +845,8 @@ void intel_gtt_insert_page(dma_addr_t addr,
 			   unsigned int flags)
 {
 	intel_private.driver->write_entry(addr, pg, flags);
+	if (intel_private.driver->chipset_flush)
+		intel_private.driver->chipset_flush();
 }
 EXPORT_SYMBOL(intel_gtt_insert_page);
 
diff --git a/drivers/char/bfin-otp.c b/drivers/char/bfin-otp.c
index 44660f1c4849..35d46da754d7 100644
--- a/drivers/char/bfin-otp.c
+++ b/drivers/char/bfin-otp.c
@@ -230,45 +230,7 @@ static struct miscdevice bfin_otp_misc_device = {
 	.name     = DRIVER_NAME,
 	.fops     = &bfin_otp_fops,
 };
-
-/**
- *	bfin_otp_init - Initialize module
- *
- *	Registers the device and notifier handler. Actual device
- *	initialization is handled by bfin_otp_open().
- */
-static int __init bfin_otp_init(void)
-{
-	int ret;
-
-	stampit();
-
-	ret = misc_register(&bfin_otp_misc_device);
-	if (ret) {
-		pr_init(KERN_ERR PFX "unable to register a misc device\n");
-		return ret;
-	}
-
-	pr_init(KERN_INFO PFX "initialized\n");
-
-	return 0;
-}
-
-/**
- *	bfin_otp_exit - Deinitialize module
- *
- *	Unregisters the device and notifier handler. Actual device
- *	deinitialization is handled by bfin_otp_close().
- */
-static void __exit bfin_otp_exit(void)
-{
-	stampit();
-
-	misc_deregister(&bfin_otp_misc_device);
-}
-
-module_init(bfin_otp_init);
-module_exit(bfin_otp_exit);
+module_misc_device(bfin_otp_misc_device);
 
 MODULE_AUTHOR("Mike Frysinger <vapier@gentoo.org>");
 MODULE_DESCRIPTION("Blackfin OTP Memory Interface");
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index 56ad5a5936a9..200dab5136a7 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -244,7 +244,7 @@ config HW_RANDOM_TX4939
 
 config HW_RANDOM_MXC_RNGA
 	tristate "Freescale i.MX RNGA Random Number Generator"
-	depends on ARCH_HAS_RNGA
+	depends on SOC_IMX31
 	default HW_RANDOM
 	---help---
 	  This driver provides kernel-side support for the Random Number
@@ -410,6 +410,19 @@ config HW_RANDOM_MESON
 
 	  If unsure, say Y.
 
+config HW_RANDOM_CAVIUM
+       tristate "Cavium ThunderX Random Number Generator support"
+       depends on HW_RANDOM && PCI && (ARM64 || (COMPILE_TEST && 64BIT))
+       default HW_RANDOM
+       ---help---
+         This driver provides kernel-side support for the Random Number
+         Generator hardware found on Cavium SoCs.
+
+         To compile this driver as a module, choose M here: the
+         module will be called cavium_rng.
+
+         If unsure, say Y.
+
 endif # HW_RANDOM
 
 config UML_RANDOM
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index 04bb0b03356f..5f52b1e4e7be 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -35,3 +35,4 @@ obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o
 obj-$(CONFIG_HW_RANDOM_STM32) += stm32-rng.o
 obj-$(CONFIG_HW_RANDOM_PIC32) += pic32-rng.o
 obj-$(CONFIG_HW_RANDOM_MESON) += meson-rng.o
+obj-$(CONFIG_HW_RANDOM_CAVIUM) += cavium-rng.o cavium-rng-vf.o
diff --git a/drivers/char/hw_random/amd-rng.c b/drivers/char/hw_random/amd-rng.c
index 48f6a83cdd61..4a99ac756f08 100644
--- a/drivers/char/hw_random/amd-rng.c
+++ b/drivers/char/hw_random/amd-rng.c
@@ -24,16 +24,18 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/hw_random.h>
 #include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/hw_random.h>
-#include <linux/delay.h>
-#include <asm/io.h>
 
+#define DRV_NAME "AMD768-HWRNG"
 
-#define PFX	KBUILD_MODNAME ": "
-
+#define RNGDATA		0x00
+#define RNGDONE		0x04
+#define PMBASE_OFFSET	0xF0
+#define PMBASE_SIZE	8
 
 /*
  * Data for PCI driver interface
@@ -50,72 +52,84 @@ static const struct pci_device_id pci_tbl[] = {
 };
 MODULE_DEVICE_TABLE(pci, pci_tbl);
 
-static struct pci_dev *amd_pdev;
-
+struct amd768_priv {
+	void __iomem *iobase;
+	struct pci_dev *pcidev;
+};
 
-static int amd_rng_data_present(struct hwrng *rng, int wait)
+static int amd_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
 {
-	u32 pmbase = (u32)rng->priv;
-	int data, i;
-
-	for (i = 0; i < 20; i++) {
-		data = !!(inl(pmbase + 0xF4) & 1);
-		if (data || !wait)
-			break;
-		udelay(10);
+	u32 *data = buf;
+	struct amd768_priv *priv = (struct amd768_priv *)rng->priv;
+	size_t read = 0;
+	/* We will wait at maximum one time per read */
+	int timeout = max / 4 + 1;
+
+	/*
+	 * RNG data is available when RNGDONE is set to 1
+	 * New random numbers are generated approximately 128 microseconds
+	 * after RNGDATA is read
+	 */
+	while (read < max) {
+		if (ioread32(priv->iobase + RNGDONE) == 0) {
+			if (wait) {
+				/* Delay given by datasheet */
+				usleep_range(128, 196);
+				if (timeout-- == 0)
+					return read;
+			} else {
+				return 0;
+			}
+		} else {
+			*data = ioread32(priv->iobase + RNGDATA);
+			data++;
+			read += 4;
+		}
 	}
-	return data;
-}
 
-static int amd_rng_data_read(struct hwrng *rng, u32 *data)
-{
-	u32 pmbase = (u32)rng->priv;
-
-	*data = inl(pmbase + 0xF0);
-
-	return 4;
+	return read;
 }
 
 static int amd_rng_init(struct hwrng *rng)
 {
+	struct amd768_priv *priv = (struct amd768_priv *)rng->priv;
 	u8 rnen;
 
-	pci_read_config_byte(amd_pdev, 0x40, &rnen);
-	rnen |= (1 << 7);	/* RNG on */
-	pci_write_config_byte(amd_pdev, 0x40, rnen);
+	pci_read_config_byte(priv->pcidev, 0x40, &rnen);
+	rnen |= BIT(7);	/* RNG on */
+	pci_write_config_byte(priv->pcidev, 0x40, rnen);
 
-	pci_read_config_byte(amd_pdev, 0x41, &rnen);
-	rnen |= (1 << 7);	/* PMIO enable */
-	pci_write_config_byte(amd_pdev, 0x41, rnen);
+	pci_read_config_byte(priv->pcidev, 0x41, &rnen);
+	rnen |= BIT(7);	/* PMIO enable */
+	pci_write_config_byte(priv->pcidev, 0x41, rnen);
 
 	return 0;
 }
 
 static void amd_rng_cleanup(struct hwrng *rng)
 {
+	struct amd768_priv *priv = (struct amd768_priv *)rng->priv;
 	u8 rnen;
 
-	pci_read_config_byte(amd_pdev, 0x40, &rnen);
-	rnen &= ~(1 << 7);	/* RNG off */
-	pci_write_config_byte(amd_pdev, 0x40, rnen);
+	pci_read_config_byte(priv->pcidev, 0x40, &rnen);
+	rnen &= ~BIT(7);	/* RNG off */
+	pci_write_config_byte(priv->pcidev, 0x40, rnen);
 }
 
-
 static struct hwrng amd_rng = {
 	.name		= "amd",
 	.init		= amd_rng_init,
 	.cleanup	= amd_rng_cleanup,
-	.data_present	= amd_rng_data_present,
-	.data_read	= amd_rng_data_read,
+	.read		= amd_rng_read,
 };
 
-
 static int __init mod_init(void)
 {
 	int err = -ENODEV;
 	struct pci_dev *pdev = NULL;
 	const struct pci_device_id *ent;
 	u32 pmbase;
+	struct amd768_priv *priv;
 
 	for_each_pci_dev(pdev) {
 		ent = pci_match_id(pci_tbl, pdev);
@@ -123,42 +137,44 @@ static int __init mod_init(void)
 			goto found;
 	}
 	/* Device not found. */
-	goto out;
+	return -ENODEV;
 
 found:
 	err = pci_read_config_dword(pdev, 0x58, &pmbase);
 	if (err)
-		goto out;
-	err = -EIO;
+		return err;
+
 	pmbase &= 0x0000FF00;
 	if (pmbase == 0)
-		goto out;
-	if (!request_region(pmbase + 0xF0, 8, "AMD HWRNG")) {
-		dev_err(&pdev->dev, "AMD HWRNG region 0x%x already in use!\n",
+		return -EIO;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	if (!devm_request_region(&pdev->dev, pmbase + PMBASE_OFFSET,
+				PMBASE_SIZE, DRV_NAME)) {
+		dev_err(&pdev->dev, DRV_NAME " region 0x%x already in use!\n",
 			pmbase + 0xF0);
-		err = -EBUSY;
-		goto out;
+		return -EBUSY;
 	}
-	amd_rng.priv = (unsigned long)pmbase;
-	amd_pdev = pdev;
-
-	pr_info("AMD768 RNG detected\n");
-	err = hwrng_register(&amd_rng);
-	if (err) {
-		pr_err(PFX "RNG registering failed (%d)\n",
-		       err);
-		release_region(pmbase + 0xF0, 8);
-		goto out;
+
+	priv->iobase = devm_ioport_map(&pdev->dev, pmbase + PMBASE_OFFSET,
+			PMBASE_SIZE);
+	if (!priv->iobase) {
+		pr_err(DRV_NAME "Cannot map ioport\n");
+		return -ENOMEM;
 	}
-out:
-	return err;
+
+	amd_rng.priv = (unsigned long)priv;
+	priv->pcidev = pdev;
+
+	pr_info(DRV_NAME " detected\n");
+	return devm_hwrng_register(&pdev->dev, &amd_rng);
 }
 
 static void __exit mod_exit(void)
 {
-	u32 pmbase = (unsigned long)amd_rng.priv;
-	release_region(pmbase + 0xF0, 8);
-	hwrng_unregister(&amd_rng);
 }
 
 module_init(mod_init);
diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c
index af2149273fe0..574211a49549 100644
--- a/drivers/char/hw_random/bcm2835-rng.c
+++ b/drivers/char/hw_random/bcm2835-rng.c
@@ -92,9 +92,10 @@ static int bcm2835_rng_probe(struct platform_device *pdev)
 	bcm2835_rng_ops.priv = (unsigned long)rng_base;
 
 	rng_id = of_match_node(bcm2835_rng_of_match, np);
-	if (!rng_id)
+	if (!rng_id) {
+		iounmap(rng_base);
 		return -EINVAL;
-
+	}
 	/* Check for rng init function, execute it */
 	rng_setup = rng_id->data;
 	if (rng_setup)
diff --git a/drivers/char/hw_random/cavium-rng-vf.c b/drivers/char/hw_random/cavium-rng-vf.c
new file mode 100644
index 000000000000..066ae0e78d63
--- /dev/null
+++ b/drivers/char/hw_random/cavium-rng-vf.c
@@ -0,0 +1,99 @@
+/*
+ * Hardware Random Number Generator support for Cavium, Inc.
+ * Thunder processor family.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2016 Cavium, Inc.
+ */
+
+#include <linux/hw_random.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+
+struct cavium_rng {
+	struct hwrng ops;
+	void __iomem *result;
+};
+
+/* Read data from the RNG unit */
+static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait)
+{
+	struct cavium_rng *p = container_of(rng, struct cavium_rng, ops);
+	unsigned int size = max;
+
+	while (size >= 8) {
+		*((u64 *)dat) = readq(p->result);
+		size -= 8;
+		dat += 8;
+	}
+	while (size > 0) {
+		*((u8 *)dat) = readb(p->result);
+		size--;
+		dat++;
+	}
+	return max;
+}
+
+/* Map Cavium RNG to an HWRNG object */
+static int cavium_rng_probe_vf(struct	pci_dev		*pdev,
+			 const struct	pci_device_id	*id)
+{
+	struct	cavium_rng *rng;
+	int	ret;
+
+	rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL);
+	if (!rng)
+		return -ENOMEM;
+
+	/* Map the RNG result */
+	rng->result = pcim_iomap(pdev, 0, 0);
+	if (!rng->result) {
+		dev_err(&pdev->dev, "Error iomap failed retrieving result.\n");
+		return -ENOMEM;
+	}
+
+	rng->ops.name    = "cavium rng";
+	rng->ops.read    = cavium_rng_read;
+	rng->ops.quality = 1000;
+
+	pci_set_drvdata(pdev, rng);
+
+	ret = hwrng_register(&rng->ops);
+	if (ret) {
+		dev_err(&pdev->dev, "Error registering device as HWRNG.\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+/* Remove the VF */
+void  cavium_rng_remove_vf(struct pci_dev *pdev)
+{
+	struct cavium_rng *rng;
+
+	rng = pci_get_drvdata(pdev);
+	hwrng_unregister(&rng->ops);
+}
+
+static const struct pci_device_id cavium_rng_vf_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa033), 0, 0, 0},
+	{0,},
+};
+MODULE_DEVICE_TABLE(pci, cavium_rng_vf_id_table);
+
+static struct pci_driver cavium_rng_vf_driver = {
+	.name		= "cavium_rng_vf",
+	.id_table	= cavium_rng_vf_id_table,
+	.probe		= cavium_rng_probe_vf,
+	.remove		= cavium_rng_remove_vf,
+};
+module_pci_driver(cavium_rng_vf_driver);
+
+MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/char/hw_random/cavium-rng.c b/drivers/char/hw_random/cavium-rng.c
new file mode 100644
index 000000000000..a944e0a47f42
--- /dev/null
+++ b/drivers/char/hw_random/cavium-rng.c
@@ -0,0 +1,94 @@
+/*
+ * Hardware Random Number Generator support for Cavium Inc.
+ * Thunder processor family.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2016 Cavium, Inc.
+ */
+
+#include <linux/hw_random.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+
+#define THUNDERX_RNM_ENT_EN     0x1
+#define THUNDERX_RNM_RNG_EN     0x2
+
+struct cavium_rng_pf {
+	void __iomem *control_status;
+};
+
+/* Enable the RNG hardware and activate the VF */
+static int cavium_rng_probe(struct pci_dev *pdev,
+			const struct pci_device_id *id)
+{
+	struct	cavium_rng_pf *rng;
+	int	iov_err;
+
+	rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL);
+	if (!rng)
+		return -ENOMEM;
+
+	/*Map the RNG control */
+	rng->control_status = pcim_iomap(pdev, 0, 0);
+	if (!rng->control_status) {
+		dev_err(&pdev->dev,
+			"Error iomap failed retrieving control_status.\n");
+		return -ENOMEM;
+	}
+
+	/* Enable the RNG hardware and entropy source */
+	writeq(THUNDERX_RNM_RNG_EN | THUNDERX_RNM_ENT_EN,
+		rng->control_status);
+
+	pci_set_drvdata(pdev, rng);
+
+	/* Enable the Cavium RNG as a VF */
+	iov_err = pci_enable_sriov(pdev, 1);
+	if (iov_err != 0) {
+		/* Disable the RNG hardware and entropy source */
+		writeq(0, rng->control_status);
+		dev_err(&pdev->dev,
+			"Error initializing RNG virtual function,(%i).\n",
+			iov_err);
+		return iov_err;
+	}
+
+	return 0;
+}
+
+/* Disable VF and RNG Hardware */
+void  cavium_rng_remove(struct pci_dev *pdev)
+{
+	struct cavium_rng_pf *rng;
+
+	rng = pci_get_drvdata(pdev);
+
+	/* Remove the VF */
+	pci_disable_sriov(pdev);
+
+	/* Disable the RNG hardware and entropy source */
+	writeq(0, rng->control_status);
+}
+
+static const struct pci_device_id cavium_rng_pf_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa018), 0, 0, 0}, /* Thunder RNM */
+	{0,},
+};
+
+MODULE_DEVICE_TABLE(pci, cavium_rng_pf_id_table);
+
+static struct pci_driver cavium_rng_pf_driver = {
+	.name		= "cavium_rng_pf",
+	.id_table	= cavium_rng_pf_id_table,
+	.probe		= cavium_rng_probe,
+	.remove		= cavium_rng_remove,
+};
+
+module_pci_driver(cavium_rng_pf_driver);
+MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 9203f2d130c0..482794526e8c 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -449,22 +449,6 @@ int hwrng_register(struct hwrng *rng)
 		goto out;
 
 	mutex_lock(&rng_mutex);
-
-	/* kmalloc makes this safe for virt_to_page() in virtio_rng.c */
-	err = -ENOMEM;
-	if (!rng_buffer) {
-		rng_buffer = kmalloc(rng_buffer_size(), GFP_KERNEL);
-		if (!rng_buffer)
-			goto out_unlock;
-	}
-	if (!rng_fillbuf) {
-		rng_fillbuf = kmalloc(rng_buffer_size(), GFP_KERNEL);
-		if (!rng_fillbuf) {
-			kfree(rng_buffer);
-			goto out_unlock;
-		}
-	}
-
 	/* Must not register two RNGs with the same name. */
 	err = -EEXIST;
 	list_for_each_entry(tmp, &rng_list, list) {
@@ -573,7 +557,26 @@ EXPORT_SYMBOL_GPL(devm_hwrng_unregister);
 
 static int __init hwrng_modinit(void)
 {
-	return register_miscdev();
+	int ret = -ENOMEM;
+
+	/* kmalloc makes this safe for virt_to_page() in virtio_rng.c */
+	rng_buffer = kmalloc(rng_buffer_size(), GFP_KERNEL);
+	if (!rng_buffer)
+		return -ENOMEM;
+
+	rng_fillbuf = kmalloc(rng_buffer_size(), GFP_KERNEL);
+	if (!rng_fillbuf) {
+		kfree(rng_buffer);
+		return -ENOMEM;
+	}
+
+	ret = register_miscdev();
+	if (ret) {
+		kfree(rng_fillbuf);
+		kfree(rng_buffer);
+	}
+
+	return ret;
 }
 
 static void __exit hwrng_modexit(void)
diff --git a/drivers/char/hw_random/geode-rng.c b/drivers/char/hw_random/geode-rng.c
index 0d0579fe465e..e7a245942029 100644
--- a/drivers/char/hw_random/geode-rng.c
+++ b/drivers/char/hw_random/geode-rng.c
@@ -24,15 +24,12 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/hw_random.h>
+#include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/hw_random.h>
-#include <linux/delay.h>
-#include <asm/io.h>
-
-
-#define PFX	KBUILD_MODNAME ": "
 
 #define GEODE_RNG_DATA_REG   0x50
 #define GEODE_RNG_STATUS_REG 0x54
@@ -85,7 +82,6 @@ static struct hwrng geode_rng = {
 
 static int __init mod_init(void)
 {
-	int err = -ENODEV;
 	struct pci_dev *pdev = NULL;
 	const struct pci_device_id *ent;
 	void __iomem *mem;
@@ -93,43 +89,27 @@ static int __init mod_init(void)
 
 	for_each_pci_dev(pdev) {
 		ent = pci_match_id(pci_tbl, pdev);
-		if (ent)
-			goto found;
-	}
-	/* Device not found. */
-	goto out;
-
-found:
-	rng_base = pci_resource_start(pdev, 0);
-	if (rng_base == 0)
-		goto out;
-	err = -ENOMEM;
-	mem = ioremap(rng_base, 0x58);
-	if (!mem)
-		goto out;
-	geode_rng.priv = (unsigned long)mem;
-
-	pr_info("AMD Geode RNG detected\n");
-	err = hwrng_register(&geode_rng);
-	if (err) {
-		pr_err(PFX "RNG registering failed (%d)\n",
-		       err);
-		goto err_unmap;
+		if (ent) {
+			rng_base = pci_resource_start(pdev, 0);
+			if (rng_base == 0)
+				return -ENODEV;
+
+			mem = devm_ioremap(&pdev->dev, rng_base, 0x58);
+			if (!mem)
+				return -ENOMEM;
+			geode_rng.priv = (unsigned long)mem;
+
+			pr_info("AMD Geode RNG detected\n");
+			return devm_hwrng_register(&pdev->dev, &geode_rng);
+		}
 	}
-out:
-	return err;
 
-err_unmap:
-	iounmap(mem);
-	goto out;
+	/* Device not found. */
+	return -ENODEV;
 }
 
 static void __exit mod_exit(void)
 {
-	void __iomem *mem = (void __iomem *)geode_rng.priv;
-
-	hwrng_unregister(&geode_rng);
-	iounmap(mem);
 }
 
 module_init(mod_init);
diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c
index 0cfd81bcaeac..58bef39f7286 100644
--- a/drivers/char/hw_random/meson-rng.c
+++ b/drivers/char/hw_random/meson-rng.c
@@ -76,9 +76,6 @@ static int meson_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
 	struct meson_rng_data *data =
 			container_of(rng, struct meson_rng_data, rng);
 
-	if (max < sizeof(u32))
-		return 0;
-
 	*(u32 *)buf = readl_relaxed(data->base + RNG_DATA);
 
 	return sizeof(u32);
diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c
index 01d4be2c354b..f5c26a5f6875 100644
--- a/drivers/char/hw_random/omap-rng.c
+++ b/drivers/char/hw_random/omap-rng.c
@@ -385,7 +385,7 @@ static int omap_rng_probe(struct platform_device *pdev)
 
 	pm_runtime_enable(&pdev->dev);
 	ret = pm_runtime_get_sync(&pdev->dev);
-	if (ret) {
+	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret);
 		pm_runtime_put_noidle(&pdev->dev);
 		goto err_ioremap;
@@ -443,7 +443,7 @@ static int __maybe_unused omap_rng_resume(struct device *dev)
 	int ret;
 
 	ret = pm_runtime_get_sync(dev);
-	if (ret) {
+	if (ret < 0) {
 		dev_err(dev, "Failed to runtime_get device: %d\n", ret);
 		pm_runtime_put_noidle(dev);
 		return ret;
diff --git a/drivers/char/hw_random/omap3-rom-rng.c b/drivers/char/hw_random/omap3-rom-rng.c
index 8da14f1a1f56..37a58d78aab3 100644
--- a/drivers/char/hw_random/omap3-rom-rng.c
+++ b/drivers/char/hw_random/omap3-rom-rng.c
@@ -71,12 +71,7 @@ static int omap3_rom_rng_get_random(void *buf, unsigned int count)
 	return 0;
 }
 
-static int omap3_rom_rng_data_present(struct hwrng *rng, int wait)
-{
-	return 1;
-}
-
-static int omap3_rom_rng_data_read(struct hwrng *rng, u32 *data)
+static int omap3_rom_rng_read(struct hwrng *rng, void *data, size_t max, bool w)
 {
 	int r;
 
@@ -88,8 +83,7 @@ static int omap3_rom_rng_data_read(struct hwrng *rng, u32 *data)
 
 static struct hwrng omap3_rom_rng_ops = {
 	.name		= "omap3-rom",
-	.data_present	= omap3_rom_rng_data_present,
-	.data_read	= omap3_rom_rng_data_read,
+	.read		= omap3_rom_rng_read,
 };
 
 static int omap3_rom_rng_probe(struct platform_device *pdev)
diff --git a/drivers/char/hw_random/pasemi-rng.c b/drivers/char/hw_random/pasemi-rng.c
index 699b7259f5d7..545df485bcc4 100644
--- a/drivers/char/hw_random/pasemi-rng.c
+++ b/drivers/char/hw_random/pasemi-rng.c
@@ -26,7 +26,7 @@
 #include <linux/delay.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
-#include <asm/io.h>
+#include <linux/io.h>
 
 #define SDCRNG_CTL_REG			0x00
 #define   SDCRNG_CTL_FVLD_M		0x0000f000
@@ -95,42 +95,20 @@ static struct hwrng pasemi_rng = {
 	.data_read	= pasemi_rng_data_read,
 };
 
-static int rng_probe(struct platform_device *ofdev)
+static int rng_probe(struct platform_device *pdev)
 {
 	void __iomem *rng_regs;
-	struct device_node *rng_np = ofdev->dev.of_node;
-	struct resource res;
-	int err = 0;
+	struct resource *res;
 
-	err = of_address_to_resource(rng_np, 0, &res);
-	if (err)
-		return -ENODEV;
-
-	rng_regs = ioremap(res.start, 0x100);
-
-	if (!rng_regs)
-		return -ENOMEM;
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	rng_regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(rng_regs))
+		return PTR_ERR(rng_regs);
 
 	pasemi_rng.priv = (unsigned long)rng_regs;
 
 	pr_info("Registering PA Semi RNG\n");
-
-	err = hwrng_register(&pasemi_rng);
-
-	if (err)
-		iounmap(rng_regs);
-
-	return err;
-}
-
-static int rng_remove(struct platform_device *dev)
-{
-	void __iomem *rng_regs = (void __iomem *)pasemi_rng.priv;
-
-	hwrng_unregister(&pasemi_rng);
-	iounmap(rng_regs);
-
-	return 0;
+	return devm_hwrng_register(&pdev->dev, &pasemi_rng);
 }
 
 static const struct of_device_id rng_match[] = {
@@ -146,7 +124,6 @@ static struct platform_driver rng_driver = {
 		.of_match_table = rng_match,
 	},
 	.probe		= rng_probe,
-	.remove		= rng_remove,
 };
 
 module_platform_driver(rng_driver);
diff --git a/drivers/char/hw_random/pic32-rng.c b/drivers/char/hw_random/pic32-rng.c
index 108897bea2d0..11dc9b7c09ce 100644
--- a/drivers/char/hw_random/pic32-rng.c
+++ b/drivers/char/hw_random/pic32-rng.c
@@ -143,7 +143,6 @@ static struct platform_driver pic32_rng_driver = {
 	.remove		= pic32_rng_remove,
 	.driver		= {
 		.name	= "pic32-rng",
-		.owner	= THIS_MODULE,
 		.of_match_table = of_match_ptr(pic32_rng_of_match),
 	},
 };
diff --git a/drivers/char/hw_random/st-rng.c b/drivers/char/hw_random/st-rng.c
index 1d35363d23c5..938ec10e733d 100644
--- a/drivers/char/hw_random/st-rng.c
+++ b/drivers/char/hw_random/st-rng.c
@@ -54,9 +54,6 @@ static int st_rng_read(struct hwrng *rng, void *data, size_t max, bool wait)
 	u32 status;
 	int i;
 
-	if (max < sizeof(u16))
-		return -EINVAL;
-
 	/* Wait until FIFO is full - max 4uS*/
 	for (i = 0; i < ST_RNG_FILL_FIFO_TIMEOUT; i++) {
 		status = readl_relaxed(ddata->base + ST_RNG_STATUS_REG);
@@ -111,6 +108,7 @@ static int st_rng_probe(struct platform_device *pdev)
 	ret = hwrng_register(&ddata->ops);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register HW RNG\n");
+		clk_disable_unprepare(clk);
 		return ret;
 	}
 
diff --git a/drivers/char/hw_random/tx4939-rng.c b/drivers/char/hw_random/tx4939-rng.c
index a7b694913416..1093583b579c 100644
--- a/drivers/char/hw_random/tx4939-rng.c
+++ b/drivers/char/hw_random/tx4939-rng.c
@@ -144,22 +144,13 @@ static int __init tx4939_rng_probe(struct platform_device *dev)
 	}
 
 	platform_set_drvdata(dev, rngdev);
-	return hwrng_register(&rngdev->rng);
-}
-
-static int __exit tx4939_rng_remove(struct platform_device *dev)
-{
-	struct tx4939_rng *rngdev = platform_get_drvdata(dev);
-
-	hwrng_unregister(&rngdev->rng);
-	return 0;
+	return devm_hwrng_register(&dev->dev, &rngdev->rng);
 }
 
 static struct platform_driver tx4939_rng_driver = {
 	.driver		= {
 		.name	= "tx4939-rng",
 	},
-	.remove = tx4939_rng_remove,
 };
 
 module_platform_driver_probe(tx4939_rng_driver, tx4939_rng_probe);
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index a33163dbb913..5bb1985ec484 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -381,6 +381,9 @@ static ssize_t read_kmem(struct file *file, char __user *buf,
 	char *kbuf; /* k-addr because vread() takes vmlist_lock rwlock */
 	int err = 0;
 
+	if (!pfn_valid(PFN_DOWN(p)))
+		return -EIO;
+
 	read = 0;
 	if (p < (unsigned long) high_memory) {
 		low_count = count;
@@ -509,6 +512,9 @@ static ssize_t write_kmem(struct file *file, const char __user *buf,
 	char *kbuf; /* k-addr because vwrite() takes vmlist_lock rwlock */
 	int err = 0;
 
+	if (!pfn_valid(PFN_DOWN(p)))
+		return -EIO;
+
 	if (p < (unsigned long) high_memory) {
 		unsigned long to_write = min_t(unsigned long, count,
 					       (unsigned long)high_memory - p);
diff --git a/drivers/char/mwave/3780i.c b/drivers/char/mwave/3780i.c
index 28740046bc83..972c40a19629 100644
--- a/drivers/char/mwave/3780i.c
+++ b/drivers/char/mwave/3780i.c
@@ -124,7 +124,7 @@ static void dsp3780I_WriteGenCfg(unsigned short usDspBaseIO, unsigned uIndex,
 		MKBYTE(rSlaveControl));
 
 	rSlaveControl_Save = rSlaveControl;
-	rSlaveControl.ConfigMode = TRUE;
+	rSlaveControl.ConfigMode = true;
 
 	PRINTK_2(TRACE_3780I,
 		"3780i::dsp3780i_WriteGenCfg entry rSlaveControl+ConfigMode %x\n",
@@ -155,7 +155,7 @@ unsigned char dsp3780I_ReadGenCfg(unsigned short usDspBaseIO,
 
 	MKBYTE(rSlaveControl) = InByteDsp(DSP_IsaSlaveControl);
 	rSlaveControl_Save = rSlaveControl;
-	rSlaveControl.ConfigMode = TRUE;
+	rSlaveControl.ConfigMode = true;
 	OutByteDsp(DSP_IsaSlaveControl, MKBYTE(rSlaveControl));
 	OutByteDsp(DSP_ConfigAddress, (unsigned char) uIndex);
 	ucValue = InByteDsp(DSP_ConfigData);
@@ -230,7 +230,7 @@ int dsp3780I_EnableDSP(DSP_3780I_CONFIG_SETTINGS * pSettings,
 			rUartCfg1.BaseIO = 3;
 			break;
 		}
-		rUartCfg2.Enable = TRUE;
+		rUartCfg2.Enable = true;
 	}
 
 	rHBridgeCfg1.Reserved = rHBridgeCfg2.Reserved = 0;
@@ -238,7 +238,7 @@ int dsp3780I_EnableDSP(DSP_3780I_CONFIG_SETTINGS * pSettings,
 	rHBridgeCfg1.IrqPulse = pSettings->bDspIrqPulse;
 	rHBridgeCfg1.Irq = (unsigned char) pIrqMap[pSettings->usDspIrq];
 	rHBridgeCfg1.AccessMode = 1;
-	rHBridgeCfg2.Enable = TRUE;
+	rHBridgeCfg2.Enable = true;
 
 
 	rBusmasterCfg2.Reserved = 0;
@@ -278,8 +278,8 @@ int dsp3780I_EnableDSP(DSP_3780I_CONFIG_SETTINGS * pSettings,
 	* soft-reset active for 10ms.
 	*/
 	rSlaveControl.ClockControl = 0;
-	rSlaveControl.SoftReset = TRUE;
-	rSlaveControl.ConfigMode = FALSE;
+	rSlaveControl.SoftReset = true;
+	rSlaveControl.ConfigMode = false;
 	rSlaveControl.Reserved = 0;
 
 	PRINTK_4(TRACE_3780I,
@@ -302,7 +302,7 @@ int dsp3780I_EnableDSP(DSP_3780I_CONFIG_SETTINGS * pSettings,
 	for (i = 0; i < 11; i++)
 		udelay(2000);
 
-	rSlaveControl.SoftReset = FALSE;
+	rSlaveControl.SoftReset = false;
 	OutWordDsp(DSP_IsaSlaveControl, MKWORD(rSlaveControl));
 
 	MKWORD(tval) = InWordDsp(DSP_IsaSlaveControl);
@@ -326,10 +326,10 @@ int dsp3780I_EnableDSP(DSP_3780I_CONFIG_SETTINGS * pSettings,
 	}
 
 
-	rHBridgeControl.EnableDspInt = FALSE;
-	rHBridgeControl.MemAutoInc = TRUE;
-	rHBridgeControl.IoAutoInc = FALSE;
-	rHBridgeControl.DiagnosticMode = FALSE;
+	rHBridgeControl.EnableDspInt = false;
+	rHBridgeControl.MemAutoInc = true;
+	rHBridgeControl.IoAutoInc = false;
+	rHBridgeControl.DiagnosticMode = false;
 
 	PRINTK_3(TRACE_3780I,
 		"3780i::dsp3780i_EnableDSP DSP_HBridgeControl %x rHBridgeControl %x\n",
@@ -345,7 +345,7 @@ int dsp3780I_EnableDSP(DSP_3780I_CONFIG_SETTINGS * pSettings,
 	ChipID = ReadMsaCfg(DSP_ChipID);
 
 	PRINTK_2(TRACE_3780I,
-		"3780i::dsp3780I_EnableDSP exiting bRC=TRUE, ChipID %x\n",
+		"3780i::dsp3780I_EnableDSP exiting bRC=true, ChipID %x\n",
 		ChipID);
 
 	return 0;
@@ -361,8 +361,8 @@ int dsp3780I_DisableDSP(DSP_3780I_CONFIG_SETTINGS * pSettings)
 	PRINTK_1(TRACE_3780I, "3780i::dsp3780i_DisableDSP entry\n");
 
 	rSlaveControl.ClockControl = 0;
-	rSlaveControl.SoftReset = TRUE;
-	rSlaveControl.ConfigMode = FALSE;
+	rSlaveControl.SoftReset = true;
+	rSlaveControl.ConfigMode = false;
 	rSlaveControl.Reserved = 0;
 	spin_lock_irqsave(&dsp_lock, flags);
 	OutWordDsp(DSP_IsaSlaveControl, MKWORD(rSlaveControl));
@@ -398,14 +398,14 @@ int dsp3780I_Reset(DSP_3780I_CONFIG_SETTINGS * pSettings)
 	PRINTK_2(TRACE_3780I, "3780i::dsp3780i_Reset rHBridgeControl %x\n",
 		MKWORD(rHBridgeControl));
 
-	rHBridgeControl.EnableDspInt = FALSE;
+	rHBridgeControl.EnableDspInt = false;
 	OutWordDsp(DSP_HBridgeControl, MKWORD(rHBridgeControl));
 	spin_unlock_irqrestore(&dsp_lock, flags);
 
 	/* Reset the core via the boot domain register */
-	rBootDomain.ResetCore = TRUE;
-	rBootDomain.Halt = TRUE;
-	rBootDomain.NMI = TRUE;
+	rBootDomain.ResetCore = true;
+	rBootDomain.Halt = true;
+	rBootDomain.NMI = true;
 	rBootDomain.Reserved = 0;
 
 	PRINTK_2(TRACE_3780I, "3780i::dsp3780i_Reset rBootDomain %x\n",
@@ -438,26 +438,26 @@ int dsp3780I_Run(DSP_3780I_CONFIG_SETTINGS * pSettings)
 
 
 	/* Transition the core to a running state */
-	rBootDomain.ResetCore = TRUE;
-	rBootDomain.Halt = FALSE;
-	rBootDomain.NMI = TRUE;
+	rBootDomain.ResetCore = true;
+	rBootDomain.Halt = false;
+	rBootDomain.NMI = true;
 	rBootDomain.Reserved = 0;
 	WriteMsaCfg(DSP_MspBootDomain, MKWORD(rBootDomain));
 
 	udelay(5);
 
-	rBootDomain.ResetCore = FALSE;
+	rBootDomain.ResetCore = false;
 	WriteMsaCfg(DSP_MspBootDomain, MKWORD(rBootDomain));
 	udelay(5);
 
-	rBootDomain.NMI = FALSE;
+	rBootDomain.NMI = false;
 	WriteMsaCfg(DSP_MspBootDomain, MKWORD(rBootDomain));
 	udelay(5);
 
 	/* Enable DSP to PC interrupt */
 	spin_lock_irqsave(&dsp_lock, flags);
 	MKWORD(rHBridgeControl) = InWordDsp(DSP_HBridgeControl);
-	rHBridgeControl.EnableDspInt = TRUE;
+	rHBridgeControl.EnableDspInt = true;
 
 	PRINTK_2(TRACE_3780I, "3780i::dsp3780i_Run rHBridgeControl %x\n",
 		MKWORD(rHBridgeControl));
@@ -466,7 +466,7 @@ int dsp3780I_Run(DSP_3780I_CONFIG_SETTINGS * pSettings)
 	spin_unlock_irqrestore(&dsp_lock, flags);
 
 
-	PRINTK_1(TRACE_3780I, "3780i::dsp3780i_Run exit bRC=TRUE\n");
+	PRINTK_1(TRACE_3780I, "3780i::dsp3780i_Run exit bRC=true\n");
 
 	return 0;
 }
@@ -508,7 +508,7 @@ int dsp3780I_ReadDStore(unsigned short usDspBaseIO, void __user *pvBuffer,
 
 
 	PRINTK_1(TRACE_3780I,
-		"3780I::dsp3780I_ReadDStore exit bRC=TRUE\n");
+		"3780I::dsp3780I_ReadDStore exit bRC=true\n");
 
 	return 0;
 }
@@ -550,7 +550,7 @@ int dsp3780I_ReadAndClearDStore(unsigned short usDspBaseIO,
 
 
 	PRINTK_1(TRACE_3780I,
-		"3780I::dsp3780I_ReadAndClearDStore exit bRC=TRUE\n");
+		"3780I::dsp3780I_ReadAndClearDStore exit bRC=true\n");
 
 	return 0;
 }
@@ -592,7 +592,7 @@ int dsp3780I_WriteDStore(unsigned short usDspBaseIO, void __user *pvBuffer,
 
 
 	PRINTK_1(TRACE_3780I,
-		"3780I::dsp3780D_WriteDStore exit bRC=TRUE\n");
+		"3780I::dsp3780D_WriteDStore exit bRC=true\n");
 
 	return 0;
 }
@@ -640,7 +640,7 @@ int dsp3780I_ReadIStore(unsigned short usDspBaseIO, void __user *pvBuffer,
 	}
 
 	PRINTK_1(TRACE_3780I,
-		"3780I::dsp3780I_ReadIStore exit bRC=TRUE\n");
+		"3780I::dsp3780I_ReadIStore exit bRC=true\n");
 
 	return 0;
 }
@@ -689,7 +689,7 @@ int dsp3780I_WriteIStore(unsigned short usDspBaseIO, void __user *pvBuffer,
 	}
 
 	PRINTK_1(TRACE_3780I,
-		"3780I::dsp3780I_WriteIStore exit bRC=TRUE\n");
+		"3780I::dsp3780I_WriteIStore exit bRC=true\n");
 
 	return 0;
 }
@@ -713,7 +713,7 @@ int dsp3780I_GetIPCSource(unsigned short usDspBaseIO,
 	*/
 	spin_lock_irqsave(&dsp_lock, flags);
 	MKWORD(rHBridgeControl) = InWordDsp(DSP_HBridgeControl);
-	rHBridgeControl.EnableDspInt = FALSE;
+	rHBridgeControl.EnableDspInt = false;
 	OutWordDsp(DSP_HBridgeControl, MKWORD(rHBridgeControl));
 
 	*pusIPCSource = InWordDsp(DSP_Interrupt);
@@ -725,7 +725,7 @@ int dsp3780I_GetIPCSource(unsigned short usDspBaseIO,
 
 	OutWordDsp(DSP_Interrupt, (unsigned short) ~(*pusIPCSource));
 
-	rHBridgeControl.EnableDspInt = TRUE;
+	rHBridgeControl.EnableDspInt = true;
 	OutWordDsp(DSP_HBridgeControl, MKWORD(rHBridgeControl));
 	spin_unlock_irqrestore(&dsp_lock, flags);
 
diff --git a/drivers/char/mwave/3780i.h b/drivers/char/mwave/3780i.h
index fba6ab1160ce..9ccb6b270b07 100644
--- a/drivers/char/mwave/3780i.h
+++ b/drivers/char/mwave/3780i.h
@@ -101,7 +101,7 @@ typedef struct {
 } DSP_UART_CFG_1;
 
 typedef struct {
-	unsigned char Enable:1;	/* RW: Enable I/O and IRQ: 0=FALSE, 1=TRUE */
+	unsigned char Enable:1;	/* RW: Enable I/O and IRQ: 0=false, 1=true */
 	unsigned char Reserved:7;	/* 0: Reserved */
 } DSP_UART_CFG_2;
 
@@ -114,7 +114,7 @@ typedef struct {
 } DSP_HBRIDGE_CFG_1;
 
 typedef struct {
-	unsigned char Enable:1;	/* RW: enable I/O and IRQ: 0=FALSE, 1=TRUE */
+	unsigned char Enable:1;	/* RW: enable I/O and IRQ: 0=false, 1=true */
 	unsigned char Reserved:7;	/* 0: Reserved */
 } DSP_HBRIDGE_CFG_2;
 
@@ -133,12 +133,12 @@ typedef struct {
 
 
 typedef struct {
-	unsigned char GateIOCHRDY:1;	/* RW: Enable IOCHRDY gating: 0=FALSE, 1=TRUE */
+	unsigned char GateIOCHRDY:1;	/* RW: Enable IOCHRDY gating: 0=false, 1=true */
 	unsigned char Reserved:7;	/* 0: Reserved */
 } DSP_ISA_PROT_CFG;
 
 typedef struct {
-	unsigned char Enable:1;	/* RW: Enable low power suspend/resume 0=FALSE, 1=TRUE */
+	unsigned char Enable:1;	/* RW: Enable low power suspend/resume 0=false, 1=true */
 	unsigned char Reserved:7;	/* 0: Reserved */
 } DSP_POWER_MGMT_CFG;
 
diff --git a/drivers/char/mwave/mwavedd.c b/drivers/char/mwave/mwavedd.c
index 164544afd680..3a3ff2eb6cba 100644
--- a/drivers/char/mwave/mwavedd.c
+++ b/drivers/char/mwave/mwavedd.c
@@ -296,8 +296,8 @@ static long mwave_ioctl(struct file *file, unsigned int iocmd,
 				pDrvData->IPCs[ipcnum].usIntCount);
 
 			mutex_lock(&mwave_mutex);
-			pDrvData->IPCs[ipcnum].bIsHere = FALSE;
-			pDrvData->IPCs[ipcnum].bIsEnabled = TRUE;
+			pDrvData->IPCs[ipcnum].bIsHere = false;
+			pDrvData->IPCs[ipcnum].bIsEnabled = true;
 			mutex_unlock(&mwave_mutex);
 	
 			PRINTK_2(TRACE_MWAVE,
@@ -324,7 +324,7 @@ static long mwave_ioctl(struct file *file, unsigned int iocmd,
 				pDrvData->IPCs[ipcnum].usIntCount);
 	
 			mutex_lock(&mwave_mutex);
-			if (pDrvData->IPCs[ipcnum].bIsEnabled == TRUE) {
+			if (pDrvData->IPCs[ipcnum].bIsEnabled == true) {
 				DECLARE_WAITQUEUE(wait, current);
 
 				PRINTK_2(TRACE_MWAVE,
@@ -332,7 +332,7 @@ static long mwave_ioctl(struct file *file, unsigned int iocmd,
 					" ipc %x going to sleep\n",
 					ipcnum);
 				add_wait_queue(&pDrvData->IPCs[ipcnum].ipc_wait_queue, &wait);
-				pDrvData->IPCs[ipcnum].bIsHere = TRUE;
+				pDrvData->IPCs[ipcnum].bIsHere = true;
 				set_current_state(TASK_INTERRUPTIBLE);
 				/* check whether an event was signalled by */
 				/* the interrupt handler while we were gone */
@@ -355,7 +355,7 @@ static long mwave_ioctl(struct file *file, unsigned int iocmd,
 						" application\n",
 						ipcnum);
 				}
-				pDrvData->IPCs[ipcnum].bIsHere = FALSE;
+				pDrvData->IPCs[ipcnum].bIsHere = false;
 				remove_wait_queue(&pDrvData->IPCs[ipcnum].ipc_wait_queue, &wait);
 				set_current_state(TASK_RUNNING);
 				PRINTK_2(TRACE_MWAVE,
@@ -384,9 +384,9 @@ static long mwave_ioctl(struct file *file, unsigned int iocmd,
 				return -EINVAL;
 			}
 			mutex_lock(&mwave_mutex);
-			if (pDrvData->IPCs[ipcnum].bIsEnabled == TRUE) {
-				pDrvData->IPCs[ipcnum].bIsEnabled = FALSE;
-				if (pDrvData->IPCs[ipcnum].bIsHere == TRUE) {
+			if (pDrvData->IPCs[ipcnum].bIsEnabled == true) {
+				pDrvData->IPCs[ipcnum].bIsEnabled = false;
+				if (pDrvData->IPCs[ipcnum].bIsHere == true) {
 					wake_up_interruptible(&pDrvData->IPCs[ipcnum].ipc_wait_queue);
 				}
 			}
@@ -541,7 +541,7 @@ static void mwave_exit(void)
 
 	if (pDrvData->device_registered) {
 		device_unregister(&mwave_device);
-		pDrvData->device_registered = FALSE;
+		pDrvData->device_registered = false;
 	}
 #endif
 
@@ -576,16 +576,16 @@ static int __init mwave_init(void)
 
 	memset(&mwave_s_mdd, 0, sizeof(MWAVE_DEVICE_DATA));
 
-	pDrvData->bBDInitialized = FALSE;
-	pDrvData->bResourcesClaimed = FALSE;
-	pDrvData->bDSPEnabled = FALSE;
-	pDrvData->bDSPReset = FALSE;
-	pDrvData->bMwaveDevRegistered = FALSE;
+	pDrvData->bBDInitialized = false;
+	pDrvData->bResourcesClaimed = false;
+	pDrvData->bDSPEnabled = false;
+	pDrvData->bDSPReset = false;
+	pDrvData->bMwaveDevRegistered = false;
 	pDrvData->sLine = -1;
 
 	for (i = 0; i < ARRAY_SIZE(pDrvData->IPCs); i++) {
-		pDrvData->IPCs[i].bIsEnabled = FALSE;
-		pDrvData->IPCs[i].bIsHere = FALSE;
+		pDrvData->IPCs[i].bIsEnabled = false;
+		pDrvData->IPCs[i].bIsHere = false;
 		pDrvData->IPCs[i].usIntCount = 0;	/* no ints received yet */
 		init_waitqueue_head(&pDrvData->IPCs[i].ipc_wait_queue);
 	}
@@ -601,7 +601,7 @@ static int __init mwave_init(void)
 				" Failed to initialize board data\n");
 		goto cleanup_error;
 	}
-	pDrvData->bBDInitialized = TRUE;
+	pDrvData->bBDInitialized = true;
 
 	retval = tp3780I_CalcResources(&pDrvData->rBDData);
 	PRINTK_2(TRACE_MWAVE,
@@ -626,7 +626,7 @@ static int __init mwave_init(void)
 				" Failed to claim resources\n");
 		goto cleanup_error;
 	}
-	pDrvData->bResourcesClaimed = TRUE;
+	pDrvData->bResourcesClaimed = true;
 
 	retval = tp3780I_EnableDSP(&pDrvData->rBDData);
 	PRINTK_2(TRACE_MWAVE,
@@ -639,7 +639,7 @@ static int __init mwave_init(void)
 				" Failed to enable DSP\n");
 		goto cleanup_error;
 	}
-	pDrvData->bDSPEnabled = TRUE;
+	pDrvData->bDSPEnabled = true;
 
 	if (misc_register(&mwave_misc_dev) < 0) {
 		PRINTK_ERROR(KERN_ERR_MWAVE
@@ -647,7 +647,7 @@ static int __init mwave_init(void)
 				" Failed to register misc device\n");
 		goto cleanup_error;
 	}
-	pDrvData->bMwaveDevRegistered = TRUE;
+	pDrvData->bMwaveDevRegistered = true;
 
 	pDrvData->sLine = register_serial_portandirq(
 		pDrvData->rBDData.rDspSettings.usUartBaseIO,
@@ -668,7 +668,7 @@ static int __init mwave_init(void)
 
 	if (device_register(&mwave_device))
 		goto cleanup_error;
-	pDrvData->device_registered = TRUE;
+	pDrvData->device_registered = true;
 	for (i = 0; i < ARRAY_SIZE(mwave_dev_attrs); i++) {
 		if(device_create_file(&mwave_device, mwave_dev_attrs[i])) {
 			PRINTK_ERROR(KERN_ERR_MWAVE
diff --git a/drivers/char/mwave/mwavedd.h b/drivers/char/mwave/mwavedd.h
index 7e0d530e2e07..37e0a4926080 100644
--- a/drivers/char/mwave/mwavedd.h
+++ b/drivers/char/mwave/mwavedd.h
@@ -125,8 +125,8 @@ extern int mwave_uart_io;
 
 typedef struct _MWAVE_IPC {
 	unsigned short usIntCount;	/* 0=none, 1=first, 2=greater than 1st */
-	BOOLEAN bIsEnabled;
-	BOOLEAN bIsHere;
+	bool bIsEnabled;
+	bool bIsHere;
 	/* entry spin lock */
 	wait_queue_head_t ipc_wait_queue;
 } MWAVE_IPC;
@@ -135,12 +135,12 @@ typedef struct _MWAVE_DEVICE_DATA {
 	THINKPAD_BD_DATA rBDData;	/* board driver's data area */
 	unsigned long ulIPCSource_ISR;	/* IPC source bits for recently processed intr, set during ISR processing */
 	unsigned long ulIPCSource_DPC;	/* IPC source bits for recently processed intr, set during DPC processing */
-	BOOLEAN bBDInitialized;
-	BOOLEAN bResourcesClaimed;
-	BOOLEAN bDSPEnabled;
-	BOOLEAN bDSPReset;
+	bool bBDInitialized;
+	bool bResourcesClaimed;
+	bool bDSPEnabled;
+	bool bDSPReset;
 	MWAVE_IPC IPCs[16];
-	BOOLEAN bMwaveDevRegistered;
+	bool bMwaveDevRegistered;
 	short sLine;
 	int nr_registered_attrs;
 	int device_registered;
diff --git a/drivers/char/mwave/smapi.c b/drivers/char/mwave/smapi.c
index 6187fd14b3fe..8c5411a8f33f 100644
--- a/drivers/char/mwave/smapi.c
+++ b/drivers/char/mwave/smapi.c
@@ -493,7 +493,7 @@ exit_smapi_request_error:
 }
 
 
-int smapi_set_DSP_power_state(BOOLEAN bOn)
+int smapi_set_DSP_power_state(bool bOn)
 {
 	int bRC = -EIO;
 	unsigned short usAX, usBX, usCX, usDX, usDI, usSI;
@@ -556,7 +556,7 @@ int smapi_init(void)
 			PRINTK_ERROR("smapi::smapi_init, ERROR unable to read from SMAPI port\n");
 		} else {
 			PRINTK_2(TRACE_SMAPI,
-				"smapi::smapi_init, exit TRUE g_usSmapiPort %x\n",
+				"smapi::smapi_init, exit true g_usSmapiPort %x\n",
 				g_usSmapiPort);
 			retval = 0;
 			//SmapiQuerySystemID();
diff --git a/drivers/char/mwave/smapi.h b/drivers/char/mwave/smapi.h
index 64b2ec1420e3..ebc206b000b9 100644
--- a/drivers/char/mwave/smapi.h
+++ b/drivers/char/mwave/smapi.h
@@ -49,10 +49,6 @@
 #ifndef _LINUX_SMAPI_H
 #define _LINUX_SMAPI_H
 
-#define TRUE 1
-#define FALSE 0
-#define BOOLEAN int
-
 typedef struct {
 	int bDSPPresent;
 	int bDSPEnabled;
@@ -74,7 +70,7 @@ typedef struct {
 int smapi_init(void);
 int smapi_query_DSP_cfg(SMAPI_DSP_SETTINGS * pSettings);
 int smapi_set_DSP_cfg(void);
-int smapi_set_DSP_power_state(BOOLEAN bOn);
+int smapi_set_DSP_power_state(bool bOn);
 
 
 #endif
diff --git a/drivers/char/mwave/tp3780i.c b/drivers/char/mwave/tp3780i.c
index 04e6d6a27994..5e1618a76b2a 100644
--- a/drivers/char/mwave/tp3780i.c
+++ b/drivers/char/mwave/tp3780i.c
@@ -80,13 +80,13 @@ static void EnableSRAM(THINKPAD_BD_DATA * pBDData)
 	WriteMsaCfg(DSP_GpioModeControl_15_8, MKWORD(rGpioMode));
 
 	MKWORD(rGpioDriverEnable) = 0;
-	rGpioDriverEnable.Enable10 = TRUE;
-	rGpioDriverEnable.Mask10 = TRUE;
+	rGpioDriverEnable.Enable10 = true;
+	rGpioDriverEnable.Mask10 = true;
 	WriteMsaCfg(DSP_GpioDriverEnable_15_8, MKWORD(rGpioDriverEnable));
 
 	MKWORD(rGpioOutputData) = 0;
 	rGpioOutputData.Latch10 = 0;
-	rGpioOutputData.Mask10 = TRUE;
+	rGpioOutputData.Mask10 = true;
 	WriteMsaCfg(DSP_GpioOutputData_15_8, MKWORD(rGpioOutputData));
 
 	PRINTK_1(TRACE_TP3780I, "tp3780i::EnableSRAM exit\n");
@@ -127,7 +127,7 @@ static irqreturn_t DspInterrupt(int irq, void *dev_id)
 				PRINTK_2(TRACE_TP3780I,
 					"tp3780i::DspInterrupt usIntCount %x\n",
 					pDrvData->IPCs[usPCNum - 1].usIntCount);
-				if (pDrvData->IPCs[usPCNum - 1].bIsEnabled == TRUE) {
+				if (pDrvData->IPCs[usPCNum - 1].bIsEnabled == true) {
 					PRINTK_2(TRACE_TP3780I,
 						"tp3780i::DspInterrupt, waking up usPCNum %x\n",
 						usPCNum - 1);
@@ -160,8 +160,8 @@ int tp3780I_InitializeBoardData(THINKPAD_BD_DATA * pBDData)
 
 	PRINTK_2(TRACE_TP3780I, "tp3780i::tp3780I_InitializeBoardData entry pBDData %p\n", pBDData);
 
-	pBDData->bDSPEnabled = FALSE;
-	pSettings->bInterruptClaimed = FALSE;
+	pBDData->bDSPEnabled = false;
+	pSettings->bInterruptClaimed = false;
 
 	retval = smapi_init();
 	if (retval) {
@@ -269,7 +269,7 @@ int tp3780I_ReleaseResources(THINKPAD_BD_DATA * pBDData)
 
 	if (pSettings->bInterruptClaimed) {
 		free_irq(pSettings->usDspIrq, NULL);
-		pSettings->bInterruptClaimed = FALSE;
+		pSettings->bInterruptClaimed = false;
 	}
 
 	PRINTK_2(TRACE_TP3780I,
@@ -283,7 +283,7 @@ int tp3780I_ReleaseResources(THINKPAD_BD_DATA * pBDData)
 int tp3780I_EnableDSP(THINKPAD_BD_DATA * pBDData)
 {
 	DSP_3780I_CONFIG_SETTINGS *pSettings = &pBDData->rDspSettings;
-	BOOLEAN bDSPPoweredUp = FALSE, bInterruptAllocated = FALSE;
+	bool bDSPPoweredUp = false, bInterruptAllocated = false;
 
 	PRINTK_2(TRACE_TP3780I, "tp3780i::tp3780I_EnableDSP entry pBDData %p\n", pBDData);
 
@@ -336,14 +336,14 @@ int tp3780I_EnableDSP(THINKPAD_BD_DATA * pBDData)
 		}
 	}
 
-	pSettings->bDspIrqActiveLow = pSettings->bDspIrqPulse = TRUE;
-	pSettings->bUartIrqActiveLow = pSettings->bUartIrqPulse = TRUE;
+	pSettings->bDspIrqActiveLow = pSettings->bDspIrqPulse = true;
+	pSettings->bUartIrqActiveLow = pSettings->bUartIrqPulse = true;
 
 	if (pBDData->bShareDspIrq) {
-		pSettings->bDspIrqActiveLow = FALSE;
+		pSettings->bDspIrqActiveLow = false;
 	}
 	if (pBDData->bShareUartIrq) {
-		pSettings->bUartIrqActiveLow = FALSE;
+		pSettings->bUartIrqActiveLow = false;
 	}
 
 	pSettings->usNumTransfers = TP_CFG_NumTransfers;
@@ -373,16 +373,16 @@ int tp3780I_EnableDSP(THINKPAD_BD_DATA * pBDData)
 		PRINTK_3(TRACE_TP3780I,
 			"tp3780i::tp3780I_EnableDSP, got interrupt %x bShareDspIrq %x\n",
 			pSettings->usDspIrq, pBDData->bShareDspIrq);
-		bInterruptAllocated = TRUE;
-		pSettings->bInterruptClaimed = TRUE;
+		bInterruptAllocated = true;
+		pSettings->bInterruptClaimed = true;
 	}
 
-	smapi_set_DSP_power_state(FALSE);
-	if (smapi_set_DSP_power_state(TRUE)) {
-		PRINTK_ERROR(KERN_ERR_MWAVE "tp3780i::tp3780I_EnableDSP: Error: smapi_set_DSP_power_state(TRUE) failed\n");
+	smapi_set_DSP_power_state(false);
+	if (smapi_set_DSP_power_state(true)) {
+		PRINTK_ERROR(KERN_ERR_MWAVE "tp3780i::tp3780I_EnableDSP: Error: smapi_set_DSP_power_state(true) failed\n");
 		goto exit_cleanup;
 	} else {
-		bDSPPoweredUp = TRUE;
+		bDSPPoweredUp = true;
 	}
 
 	if (dsp3780I_EnableDSP(pSettings, s_ausThinkpadIrqToField, s_ausThinkpadDmaToField)) {
@@ -392,7 +392,7 @@ int tp3780I_EnableDSP(THINKPAD_BD_DATA * pBDData)
 
 	EnableSRAM(pBDData);
 
-	pBDData->bDSPEnabled = TRUE;
+	pBDData->bDSPEnabled = true;
 
 	PRINTK_1(TRACE_TP3780I, "tp3780i::tp3780I_EnableDSP exit\n");
 
@@ -401,10 +401,10 @@ int tp3780I_EnableDSP(THINKPAD_BD_DATA * pBDData)
 exit_cleanup:
 	PRINTK_ERROR("tp3780i::tp3780I_EnableDSP: Cleaning up\n");
 	if (bDSPPoweredUp)
-		smapi_set_DSP_power_state(FALSE);
+		smapi_set_DSP_power_state(false);
 	if (bInterruptAllocated) {
 		free_irq(pSettings->usDspIrq, NULL);
-		pSettings->bInterruptClaimed = FALSE;
+		pSettings->bInterruptClaimed = false;
 	}
 	return -EIO;
 }
@@ -421,10 +421,10 @@ int tp3780I_DisableDSP(THINKPAD_BD_DATA * pBDData)
 		dsp3780I_DisableDSP(&pBDData->rDspSettings);
 		if (pSettings->bInterruptClaimed) {
 			free_irq(pSettings->usDspIrq, NULL);
-			pSettings->bInterruptClaimed = FALSE;
+			pSettings->bInterruptClaimed = false;
 		}
-		smapi_set_DSP_power_state(FALSE);
-		pBDData->bDSPEnabled = FALSE;
+		smapi_set_DSP_power_state(false);
+		pBDData->bDSPEnabled = false;
 	}
 
 	PRINTK_2(TRACE_TP3780I, "tp3780i::tp3780I_DisableDSP exit retval %x\n", retval);
@@ -516,7 +516,7 @@ int tp3780I_ReadWriteDspDStore(THINKPAD_BD_DATA * pBDData, unsigned int uOpcode,
 	int retval = 0;
 	DSP_3780I_CONFIG_SETTINGS *pSettings = &pBDData->rDspSettings;
 	unsigned short usDspBaseIO = pSettings->usDspBaseIO;
-	BOOLEAN bRC = 0;
+	bool bRC = 0;
 
 	PRINTK_6(TRACE_TP3780I,
 		"tp3780i::tp3780I_ReadWriteDspDStore entry pBDData %p, uOpcode %x, pvBuffer %p, uCount %x, ulDSPAddr %lx\n",
@@ -552,7 +552,7 @@ int tp3780I_ReadWriteDspIStore(THINKPAD_BD_DATA * pBDData, unsigned int uOpcode,
 	int retval = 0;
 	DSP_3780I_CONFIG_SETTINGS *pSettings = &pBDData->rDspSettings;
 	unsigned short usDspBaseIO = pSettings->usDspBaseIO;
-	BOOLEAN bRC = 0;
+	bool bRC = 0;
 
 	PRINTK_6(TRACE_TP3780I,
 		"tp3780i::tp3780I_ReadWriteDspIStore entry pBDData %p, uOpcode %x, pvBuffer %p, uCount %x, ulDSPAddr %lx\n",
diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c
index f8a483c67b07..d23368874710 100644
--- a/drivers/char/ppdev.c
+++ b/drivers/char/ppdev.c
@@ -286,7 +286,7 @@ static int register_device(int minor, struct pp_struct *pp)
 	struct parport *port;
 	struct pardevice *pdev = NULL;
 	char *name;
-	int fl;
+	struct pardev_cb ppdev_cb;
 
 	name = kasprintf(GFP_KERNEL, CHRDEV "%x", minor);
 	if (name == NULL)
@@ -299,9 +299,11 @@ static int register_device(int minor, struct pp_struct *pp)
 		return -ENXIO;
 	}
 
-	fl = (pp->flags & PP_EXCL) ? PARPORT_FLAG_EXCL : 0;
-	pdev = parport_register_device(port, name, NULL,
-				       NULL, pp_irq, fl, pp);
+	memset(&ppdev_cb, 0, sizeof(ppdev_cb));
+	ppdev_cb.irq_func = pp_irq;
+	ppdev_cb.flags = (pp->flags & PP_EXCL) ? PARPORT_FLAG_EXCL : 0;
+	ppdev_cb.private = pp;
+	pdev = parport_register_dev_model(port, name, &ppdev_cb, minor);
 	parport_put_port(port);
 
 	if (!pdev) {
@@ -799,10 +801,23 @@ static void pp_detach(struct parport *port)
 	device_destroy(ppdev_class, MKDEV(PP_MAJOR, port->number));
 }
 
+static int pp_probe(struct pardevice *par_dev)
+{
+	struct device_driver *drv = par_dev->dev.driver;
+	int len = strlen(drv->name);
+
+	if (strncmp(par_dev->name, drv->name, len))
+		return -ENODEV;
+
+	return 0;
+}
+
 static struct parport_driver pp_driver = {
 	.name		= CHRDEV,
-	.attach		= pp_attach,
+	.probe		= pp_probe,
+	.match_port	= pp_attach,
 	.detach		= pp_detach,
+	.devmodel	= true,
 };
 
 static int __init ppdev_init(void)
diff --git a/drivers/char/random.c b/drivers/char/random.c
index 3efb3bf0ab83..d131e152c8ce 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -2100,23 +2100,37 @@ unsigned long get_random_long(void)
 }
 EXPORT_SYMBOL(get_random_long);
 
-/*
- * randomize_range() returns a start address such that
+/**
+ * randomize_page - Generate a random, page aligned address
+ * @start:	The smallest acceptable address the caller will take.
+ * @range:	The size of the area, starting at @start, within which the
+ *		random address must fall.
+ *
+ * If @start + @range would overflow, @range is capped.
  *
- *    [...... <range> .....]
- *  start                  end
+ * NOTE: Historical use of randomize_range, which this replaces, presumed that
+ * @start was already page aligned.  We now align it regardless.
  *
- * a <range> with size "len" starting at the return value is inside in the
- * area defined by [start, end], but is otherwise randomized.
+ * Return: A page aligned address within [start, start + range).  On error,
+ * @start is returned.
  */
 unsigned long
-randomize_range(unsigned long start, unsigned long end, unsigned long len)
+randomize_page(unsigned long start, unsigned long range)
 {
-	unsigned long range = end - len - start;
+	if (!PAGE_ALIGNED(start)) {
+		range -= PAGE_ALIGN(start) - start;
+		start = PAGE_ALIGN(start);
+	}
 
-	if (end <= start + len)
-		return 0;
-	return PAGE_ALIGN(get_random_int() % range + start);
+	if (start > ULONG_MAX - range)
+		range = ULONG_MAX - start;
+
+	range >>= PAGE_SHIFT;
+
+	if (range == 0)
+		return start;
+
+	return start + (get_random_long() % range << PAGE_SHIFT);
 }
 
 /* Interface for in-kernel drivers of true hardware RNGs.
diff --git a/drivers/char/snsc.c b/drivers/char/snsc.c
index 94006f9c2e43..10e56323f390 100644
--- a/drivers/char/snsc.c
+++ b/drivers/char/snsc.c
@@ -385,13 +385,18 @@ scdrv_init(void)
 
 	event_nasid = ia64_sn_get_console_nasid();
 
+	snsc_class = class_create(THIS_MODULE, SYSCTL_BASENAME);
+	if (IS_ERR(snsc_class)) {
+		printk("%s: failed to allocate class\n", __func__);
+		return PTR_ERR(snsc_class);
+	}
+
 	if (alloc_chrdev_region(&first_dev, 0, num_cnodes,
 				SYSCTL_BASENAME) < 0) {
 		printk("%s: failed to register SN system controller device\n",
 		       __func__);
 		return -ENODEV;
 	}
-	snsc_class = class_create(THIS_MODULE, SYSCTL_BASENAME);
 
 	for (cnode = 0; cnode < num_cnodes; cnode++) {
 			geoid = cnodeid_get_geoid(cnode);
diff --git a/drivers/char/sonypi.c b/drivers/char/sonypi.c
index e496daefe9e0..719c5b4eed39 100644
--- a/drivers/char/sonypi.c
+++ b/drivers/char/sonypi.c
@@ -934,7 +934,7 @@ static ssize_t sonypi_misc_read(struct file *file, char __user *buf,
 
 	if (ret > 0) {
 		struct inode *inode = file_inode(file);
-		inode->i_atime = current_fs_time(inode->i_sb);
+		inode->i_atime = current_time(inode);
 	}
 
 	return ret;
diff --git a/drivers/char/tb0219.c b/drivers/char/tb0219.c
index 480a777db577..7c19d9b22785 100644
--- a/drivers/char/tb0219.c
+++ b/drivers/char/tb0219.c
@@ -21,6 +21,7 @@
 #include <linux/fs.h>
 #include <linux/init.h>
 #include <linux/module.h>
+#include <linux/uaccess.h>
 
 #include <asm/io.h>
 #include <asm/reboot.h>
diff --git a/drivers/char/tile-srom.c b/drivers/char/tile-srom.c
index 69f6b4acc377..398800edb2cc 100644
--- a/drivers/char/tile-srom.c
+++ b/drivers/char/tile-srom.c
@@ -331,13 +331,11 @@ static const struct file_operations srom_fops = {
 /**
  * srom_setup_minor() - Initialize per-minor information.
  * @srom: Per-device SROM state.
- * @index: Device to set up.
+ * @devhdl: Partition device handle.
  */
-static int srom_setup_minor(struct srom_dev *srom, int index)
+static int srom_setup_minor(struct srom_dev *srom, int devhdl)
 {
-	struct device *dev;
-	int devhdl = srom->hv_devhdl;
-
+	srom->hv_devhdl = devhdl;
 	mutex_init(&srom->lock);
 
 	if (_srom_read(devhdl, &srom->total_size,
@@ -350,9 +348,7 @@ static int srom_setup_minor(struct srom_dev *srom, int index)
 		       SROM_PAGE_SIZE_OFF, sizeof(srom->page_size)) < 0)
 		return -EIO;
 
-	dev = device_create(srom_class, &srom_parent->dev,
-			    MKDEV(srom_major, index), srom, "%d", index);
-	return PTR_ERR_OR_ZERO(dev);
+	return 0;
 }
 
 /** srom_init() - Initialize the driver's module. */
@@ -365,7 +361,7 @@ static int srom_init(void)
 	 * Start with a plausible number of partitions; the krealloc() call
 	 * below will yield about log(srom_devs) additional allocations.
 	 */
-	srom_devices = kzalloc(4 * sizeof(struct srom_dev), GFP_KERNEL);
+	srom_devices = kmalloc(4 * sizeof(struct srom_dev), GFP_KERNEL);
 
 	/* Discover the number of srom partitions. */
 	for (i = 0; ; i++) {
@@ -373,7 +369,7 @@ static int srom_init(void)
 		char buf[20];
 		struct srom_dev *new_srom_devices =
 			krealloc(srom_devices, (i+1) * sizeof(struct srom_dev),
-				 GFP_KERNEL | __GFP_ZERO);
+				 GFP_KERNEL);
 		if (!new_srom_devices) {
 			result = -ENOMEM;
 			goto fail_mem;
@@ -387,7 +383,9 @@ static int srom_init(void)
 					  i, devhdl);
 			break;
 		}
-		srom_devices[i].hv_devhdl = devhdl;
+		result = srom_setup_minor(&srom_devices[i], devhdl);
+		if (result != 0)
+			goto fail_mem;
 	}
 	srom_devs = i;
 
@@ -431,9 +429,13 @@ static int srom_init(void)
 	srom_class->dev_groups = srom_dev_groups;
 	srom_class->devnode = srom_devnode;
 
-	/* Do per-partition initialization */
+	/* Create per-partition devices */
 	for (i = 0; i < srom_devs; i++) {
-		result = srom_setup_minor(srom_devices + i, i);
+		struct device *dev =
+			device_create(srom_class, &srom_parent->dev,
+				      MKDEV(srom_major, i), srom_devices + i,
+				      "%d", i);
+		result = PTR_ERR_OR_ZERO(dev);
 		if (result < 0)
 			goto fail_class;
 	}
diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c
index c2ee30451e41..6f060c76217b 100644
--- a/drivers/char/tpm/st33zp24/st33zp24.c
+++ b/drivers/char/tpm/st33zp24/st33zp24.c
@@ -589,8 +589,6 @@ int st33zp24_probe(void *phy_id, const struct st33zp24_phy_ops *ops,
 		chip->flags |= TPM_CHIP_FLAG_IRQ;
 
 		disable_irq_nosync(tpm_dev->irq);
-
-		tpm_gen_interrupt(chip);
 	}
 
 	return tpm_chip_register(chip);
diff --git a/drivers/char/tpm/tpm-dev.c b/drivers/char/tpm/tpm-dev.c
index f5d452151c6b..912ad30be585 100644
--- a/drivers/char/tpm/tpm-dev.c
+++ b/drivers/char/tpm/tpm-dev.c
@@ -145,7 +145,7 @@ static ssize_t tpm_write(struct file *file, const char __user *buf,
 		return -EPIPE;
 	}
 	out_size = tpm_transmit(priv->chip, priv->data_buffer,
-				sizeof(priv->data_buffer));
+				sizeof(priv->data_buffer), 0);
 
 	tpm_put_ops(priv->chip);
 	if (out_size < 0) {
diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c
index 1abe2d7a2610..8de61876f633 100644
--- a/drivers/char/tpm/tpm-interface.c
+++ b/drivers/char/tpm/tpm-interface.c
@@ -330,13 +330,16 @@ EXPORT_SYMBOL_GPL(tpm_calc_ordinal_duration);
 /*
  * Internal kernel interface to transmit TPM commands
  */
-ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf,
-		     size_t bufsiz)
+ssize_t tpm_transmit(struct tpm_chip *chip, const u8 *buf, size_t bufsiz,
+		     unsigned int flags)
 {
 	ssize_t rc;
 	u32 count, ordinal;
 	unsigned long stop;
 
+	if (bufsiz < TPM_HEADER_SIZE)
+		return -EINVAL;
+
 	if (bufsiz > TPM_BUFSIZE)
 		bufsiz = TPM_BUFSIZE;
 
@@ -350,7 +353,8 @@ ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf,
 		return -E2BIG;
 	}
 
-	mutex_lock(&chip->tpm_mutex);
+	if (!(flags & TPM_TRANSMIT_UNLOCKED))
+		mutex_lock(&chip->tpm_mutex);
 
 	rc = chip->ops->send(chip, (u8 *) buf, count);
 	if (rc < 0) {
@@ -393,20 +397,21 @@ out_recv:
 		dev_err(&chip->dev,
 			"tpm_transmit: tpm_recv: error %zd\n", rc);
 out:
-	mutex_unlock(&chip->tpm_mutex);
+	if (!(flags & TPM_TRANSMIT_UNLOCKED))
+		mutex_unlock(&chip->tpm_mutex);
 	return rc;
 }
 
 #define TPM_DIGEST_SIZE 20
 #define TPM_RET_CODE_IDX 6
 
-ssize_t tpm_transmit_cmd(struct tpm_chip *chip, void *cmd,
-			 int len, const char *desc)
+ssize_t tpm_transmit_cmd(struct tpm_chip *chip, const void *cmd,
+			 int len, unsigned int flags, const char *desc)
 {
-	struct tpm_output_header *header;
+	const struct tpm_output_header *header;
 	int err;
 
-	len = tpm_transmit(chip, (u8 *) cmd, len);
+	len = tpm_transmit(chip, (const u8 *)cmd, len, flags);
 	if (len <  0)
 		return len;
 	else if (len < TPM_HEADER_SIZE)
@@ -453,26 +458,13 @@ ssize_t tpm_getcap(struct tpm_chip *chip, __be32 subcap_id, cap_t *cap,
 		tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
 		tpm_cmd.params.getcap_in.subcap = subcap_id;
 	}
-	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, desc);
+	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0,
+			      desc);
 	if (!rc)
 		*cap = tpm_cmd.params.getcap_out.cap;
 	return rc;
 }
-
-void tpm_gen_interrupt(struct tpm_chip *chip)
-{
-	struct	tpm_cmd_t tpm_cmd;
-	ssize_t rc;
-
-	tpm_cmd.header.in = tpm_getcap_header;
-	tpm_cmd.params.getcap_in.cap = TPM_CAP_PROP;
-	tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
-	tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT;
-
-	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE,
-			      "attempting to determine the timeouts");
-}
-EXPORT_SYMBOL_GPL(tpm_gen_interrupt);
+EXPORT_SYMBOL_GPL(tpm_getcap);
 
 #define TPM_ORD_STARTUP cpu_to_be32(153)
 #define TPM_ST_CLEAR cpu_to_be16(1)
@@ -490,7 +482,7 @@ static int tpm_startup(struct tpm_chip *chip, __be16 startup_type)
 	start_cmd.header.in = tpm_startup_header;
 
 	start_cmd.params.startup_in.startup_type = startup_type;
-	return tpm_transmit_cmd(chip, &start_cmd, TPM_INTERNAL_RESULT_SIZE,
+	return tpm_transmit_cmd(chip, &start_cmd, TPM_INTERNAL_RESULT_SIZE, 0,
 				"attempting to start the TPM");
 }
 
@@ -521,7 +513,8 @@ int tpm_get_timeouts(struct tpm_chip *chip)
 	tpm_cmd.params.getcap_in.cap = TPM_CAP_PROP;
 	tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
 	tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT;
-	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, NULL);
+	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0,
+			      NULL);
 
 	if (rc == TPM_ERR_INVALID_POSTINIT) {
 		/* The TPM is not started, we are the first to talk to it.
@@ -535,7 +528,7 @@ int tpm_get_timeouts(struct tpm_chip *chip)
 		tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
 		tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT;
 		rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE,
-				  NULL);
+				      0, NULL);
 	}
 	if (rc) {
 		dev_err(&chip->dev,
@@ -596,7 +589,7 @@ duration:
 	tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
 	tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_DURATION;
 
-	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE,
+	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0,
 			      "attempting to determine the durations");
 	if (rc)
 		return rc;
@@ -633,7 +626,7 @@ EXPORT_SYMBOL_GPL(tpm_get_timeouts);
 #define TPM_ORD_CONTINUE_SELFTEST 83
 #define CONTINUE_SELFTEST_RESULT_SIZE 10
 
-static struct tpm_input_header continue_selftest_header = {
+static const struct tpm_input_header continue_selftest_header = {
 	.tag = TPM_TAG_RQU_COMMAND,
 	.length = cpu_to_be32(10),
 	.ordinal = cpu_to_be32(TPM_ORD_CONTINUE_SELFTEST),
@@ -652,14 +645,14 @@ static int tpm_continue_selftest(struct tpm_chip *chip)
 	struct tpm_cmd_t cmd;
 
 	cmd.header.in = continue_selftest_header;
-	rc = tpm_transmit_cmd(chip, &cmd, CONTINUE_SELFTEST_RESULT_SIZE,
+	rc = tpm_transmit_cmd(chip, &cmd, CONTINUE_SELFTEST_RESULT_SIZE, 0,
 			      "continue selftest");
 	return rc;
 }
 
 #define TPM_ORDINAL_PCRREAD cpu_to_be32(21)
 #define READ_PCR_RESULT_SIZE 30
-static struct tpm_input_header pcrread_header = {
+static const struct tpm_input_header pcrread_header = {
 	.tag = TPM_TAG_RQU_COMMAND,
 	.length = cpu_to_be32(14),
 	.ordinal = TPM_ORDINAL_PCRREAD
@@ -672,7 +665,7 @@ int tpm_pcr_read_dev(struct tpm_chip *chip, int pcr_idx, u8 *res_buf)
 
 	cmd.header.in = pcrread_header;
 	cmd.params.pcrread_in.pcr_idx = cpu_to_be32(pcr_idx);
-	rc = tpm_transmit_cmd(chip, &cmd, READ_PCR_RESULT_SIZE,
+	rc = tpm_transmit_cmd(chip, &cmd, READ_PCR_RESULT_SIZE, 0,
 			      "attempting to read a pcr value");
 
 	if (rc == 0)
@@ -745,7 +738,7 @@ EXPORT_SYMBOL_GPL(tpm_pcr_read);
  */
 #define TPM_ORD_PCR_EXTEND cpu_to_be32(20)
 #define EXTEND_PCR_RESULT_SIZE 34
-static struct tpm_input_header pcrextend_header = {
+static const struct tpm_input_header pcrextend_header = {
 	.tag = TPM_TAG_RQU_COMMAND,
 	.length = cpu_to_be32(34),
 	.ordinal = TPM_ORD_PCR_EXTEND
@@ -770,7 +763,7 @@ int tpm_pcr_extend(u32 chip_num, int pcr_idx, const u8 *hash)
 	cmd.header.in = pcrextend_header;
 	cmd.params.pcrextend_in.pcr_idx = cpu_to_be32(pcr_idx);
 	memcpy(cmd.params.pcrextend_in.hash, hash, TPM_DIGEST_SIZE);
-	rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE,
+	rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, 0,
 			      "attempting extend a PCR value");
 
 	tpm_put_ops(chip);
@@ -792,7 +785,7 @@ int tpm_do_selftest(struct tpm_chip *chip)
 	unsigned int loops;
 	unsigned int delay_msec = 100;
 	unsigned long duration;
-	struct tpm_cmd_t cmd;
+	u8 dummy[TPM_DIGEST_SIZE];
 
 	duration = tpm_calc_ordinal_duration(chip, TPM_ORD_CONTINUE_SELFTEST);
 
@@ -807,9 +800,8 @@ int tpm_do_selftest(struct tpm_chip *chip)
 
 	do {
 		/* Attempt to read a PCR value */
-		cmd.header.in = pcrread_header;
-		cmd.params.pcrread_in.pcr_idx = cpu_to_be32(0);
-		rc = tpm_transmit(chip, (u8 *) &cmd, READ_PCR_RESULT_SIZE);
+		rc = tpm_pcr_read_dev(chip, 0, dummy);
+
 		/* Some buggy TPMs will not respond to tpm_tis_ready() for
 		 * around 300ms while the self test is ongoing, keep trying
 		 * until the self test duration expires. */
@@ -824,7 +816,6 @@ int tpm_do_selftest(struct tpm_chip *chip)
 		if (rc < TPM_HEADER_SIZE)
 			return -EFAULT;
 
-		rc = be32_to_cpu(cmd.header.out.return_code);
 		if (rc == TPM_ERR_DISABLED || rc == TPM_ERR_DEACTIVATED) {
 			dev_info(&chip->dev,
 				 "TPM is disabled/deactivated (0x%X)\n", rc);
@@ -879,7 +870,7 @@ int tpm_send(u32 chip_num, void *cmd, size_t buflen)
 	if (chip == NULL)
 		return -ENODEV;
 
-	rc = tpm_transmit_cmd(chip, cmd, buflen, "attempting tpm_cmd");
+	rc = tpm_transmit_cmd(chip, cmd, buflen, 0, "attempting tpm_cmd");
 
 	tpm_put_ops(chip);
 	return rc;
@@ -949,7 +940,7 @@ EXPORT_SYMBOL_GPL(wait_for_tpm_stat);
 #define TPM_ORD_SAVESTATE cpu_to_be32(152)
 #define SAVESTATE_RESULT_SIZE 10
 
-static struct tpm_input_header savestate_header = {
+static const struct tpm_input_header savestate_header = {
 	.tag = TPM_TAG_RQU_COMMAND,
 	.length = cpu_to_be32(10),
 	.ordinal = TPM_ORD_SAVESTATE
@@ -981,14 +972,15 @@ int tpm_pm_suspend(struct device *dev)
 		cmd.params.pcrextend_in.pcr_idx = cpu_to_be32(tpm_suspend_pcr);
 		memcpy(cmd.params.pcrextend_in.hash, dummy_hash,
 		       TPM_DIGEST_SIZE);
-		rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE,
+		rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, 0,
 				      "extending dummy pcr before suspend");
 	}
 
 	/* now do the actual savestate */
 	for (try = 0; try < TPM_RETRY; try++) {
 		cmd.header.in = savestate_header;
-		rc = tpm_transmit_cmd(chip, &cmd, SAVESTATE_RESULT_SIZE, NULL);
+		rc = tpm_transmit_cmd(chip, &cmd, SAVESTATE_RESULT_SIZE, 0,
+				      NULL);
 
 		/*
 		 * If the TPM indicates that it is too busy to respond to
@@ -1032,7 +1024,7 @@ int tpm_pm_resume(struct device *dev)
 EXPORT_SYMBOL_GPL(tpm_pm_resume);
 
 #define TPM_GETRANDOM_RESULT_SIZE	18
-static struct tpm_input_header tpm_getrandom_header = {
+static const struct tpm_input_header tpm_getrandom_header = {
 	.tag = TPM_TAG_RQU_COMMAND,
 	.length = cpu_to_be32(14),
 	.ordinal = TPM_ORD_GET_RANDOM
@@ -1072,8 +1064,8 @@ int tpm_get_random(u32 chip_num, u8 *out, size_t max)
 		tpm_cmd.params.getrandom_in.num_bytes = cpu_to_be32(num_bytes);
 
 		err = tpm_transmit_cmd(chip, &tpm_cmd,
-				   TPM_GETRANDOM_RESULT_SIZE + num_bytes,
-				   "attempting get random");
+				       TPM_GETRANDOM_RESULT_SIZE + num_bytes,
+				       0, "attempting get random");
 		if (err)
 			break;
 
diff --git a/drivers/char/tpm/tpm-sysfs.c b/drivers/char/tpm/tpm-sysfs.c
index b46cf70c8b16..a76ab4af9fb2 100644
--- a/drivers/char/tpm/tpm-sysfs.c
+++ b/drivers/char/tpm/tpm-sysfs.c
@@ -22,7 +22,7 @@
 
 #define READ_PUBEK_RESULT_SIZE 314
 #define TPM_ORD_READPUBEK cpu_to_be32(124)
-static struct tpm_input_header tpm_readpubek_header = {
+static const struct tpm_input_header tpm_readpubek_header = {
 	.tag = TPM_TAG_RQU_COMMAND,
 	.length = cpu_to_be32(30),
 	.ordinal = TPM_ORD_READPUBEK
@@ -39,7 +39,7 @@ static ssize_t pubek_show(struct device *dev, struct device_attribute *attr,
 	struct tpm_chip *chip = to_tpm_chip(dev);
 
 	tpm_cmd.header.in = tpm_readpubek_header;
-	err = tpm_transmit_cmd(chip, &tpm_cmd, READ_PUBEK_RESULT_SIZE,
+	err = tpm_transmit_cmd(chip, &tpm_cmd, READ_PUBEK_RESULT_SIZE, 0,
 			       "attempting to read the PUBEK");
 	if (err)
 		goto out;
diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
index 3e32d5bd2dc6..4d183c97f6a6 100644
--- a/drivers/char/tpm/tpm.h
+++ b/drivers/char/tpm/tpm.h
@@ -476,32 +476,35 @@ extern dev_t tpm_devt;
 extern const struct file_operations tpm_fops;
 extern struct idr dev_nums_idr;
 
+enum tpm_transmit_flags {
+	TPM_TRANSMIT_UNLOCKED	= BIT(0),
+};
+
+ssize_t tpm_transmit(struct tpm_chip *chip, const u8 *buf, size_t bufsiz,
+		     unsigned int flags);
+ssize_t tpm_transmit_cmd(struct tpm_chip *chip, const void *cmd, int len,
+			 unsigned int flags, const char *desc);
 ssize_t tpm_getcap(struct tpm_chip *chip, __be32 subcap_id, cap_t *cap,
 		   const char *desc);
-ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf,
-		     size_t bufsiz);
-ssize_t tpm_transmit_cmd(struct tpm_chip *chip, void *cmd, int len,
-			 const char *desc);
-extern int tpm_get_timeouts(struct tpm_chip *);
-extern void tpm_gen_interrupt(struct tpm_chip *);
+int tpm_get_timeouts(struct tpm_chip *);
 int tpm1_auto_startup(struct tpm_chip *chip);
-extern int tpm_do_selftest(struct tpm_chip *);
-extern unsigned long tpm_calc_ordinal_duration(struct tpm_chip *, u32);
-extern int tpm_pm_suspend(struct device *);
-extern int tpm_pm_resume(struct device *);
-extern int wait_for_tpm_stat(struct tpm_chip *, u8, unsigned long,
-			     wait_queue_head_t *, bool);
+int tpm_do_selftest(struct tpm_chip *chip);
+unsigned long tpm_calc_ordinal_duration(struct tpm_chip *chip, u32 ordinal);
+int tpm_pm_suspend(struct device *dev);
+int tpm_pm_resume(struct device *dev);
+int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask, unsigned long timeout,
+		      wait_queue_head_t *queue, bool check_cancel);
 
 struct tpm_chip *tpm_chip_find_get(int chip_num);
 __must_check int tpm_try_get_ops(struct tpm_chip *chip);
 void tpm_put_ops(struct tpm_chip *chip);
 
-extern struct tpm_chip *tpm_chip_alloc(struct device *dev,
-				       const struct tpm_class_ops *ops);
-extern struct tpm_chip *tpmm_chip_alloc(struct device *pdev,
-				       const struct tpm_class_ops *ops);
-extern int tpm_chip_register(struct tpm_chip *chip);
-extern void tpm_chip_unregister(struct tpm_chip *chip);
+struct tpm_chip *tpm_chip_alloc(struct device *dev,
+				const struct tpm_class_ops *ops);
+struct tpm_chip *tpmm_chip_alloc(struct device *pdev,
+				 const struct tpm_class_ops *ops);
+int tpm_chip_register(struct tpm_chip *chip);
+void tpm_chip_unregister(struct tpm_chip *chip);
 
 void tpm_sysfs_add_device(struct tpm_chip *chip);
 
@@ -528,8 +531,7 @@ ssize_t tpm2_get_tpm_pt(struct tpm_chip *chip, u32 property_id,
 			u32 *value, const char *desc);
 
 int tpm2_auto_startup(struct tpm_chip *chip);
-extern void tpm2_shutdown(struct tpm_chip *chip, u16 shutdown_type);
-extern unsigned long tpm2_calc_ordinal_duration(struct tpm_chip *, u32);
-extern int tpm2_gen_interrupt(struct tpm_chip *chip);
-extern int tpm2_probe(struct tpm_chip *chip);
+void tpm2_shutdown(struct tpm_chip *chip, u16 shutdown_type);
+unsigned long tpm2_calc_ordinal_duration(struct tpm_chip *chip, u32 ordinal);
+int tpm2_probe(struct tpm_chip *chip);
 #endif
diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c
index 08c7e23ed535..7df55d58c939 100644
--- a/drivers/char/tpm/tpm2-cmd.c
+++ b/drivers/char/tpm/tpm2-cmd.c
@@ -282,7 +282,7 @@ int tpm2_pcr_read(struct tpm_chip *chip, int pcr_idx, u8 *res_buf)
 	       sizeof(cmd.params.pcrread_in.pcr_select));
 	cmd.params.pcrread_in.pcr_select[pcr_idx >> 3] = 1 << (pcr_idx & 0x7);
 
-	rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd),
+	rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0,
 			      "attempting to read a pcr value");
 	if (rc == 0) {
 		buf = cmd.params.pcrread_out.digest;
@@ -330,7 +330,7 @@ int tpm2_pcr_extend(struct tpm_chip *chip, int pcr_idx, const u8 *hash)
 	cmd.params.pcrextend_in.hash_alg = cpu_to_be16(TPM2_ALG_SHA1);
 	memcpy(cmd.params.pcrextend_in.digest, hash, TPM_DIGEST_SIZE);
 
-	rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd),
+	rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0,
 			      "attempting extend a PCR value");
 
 	return rc;
@@ -376,7 +376,7 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *out, size_t max)
 		cmd.header.in = tpm2_getrandom_header;
 		cmd.params.getrandom_in.size = cpu_to_be16(num_bytes);
 
-		err = tpm_transmit_cmd(chip, &cmd, sizeof(cmd),
+		err = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0,
 				       "attempting get random");
 		if (err)
 			break;
@@ -434,12 +434,12 @@ static void tpm2_buf_append_auth(struct tpm_buf *buf, u32 session_handle,
 }
 
 /**
- * tpm2_seal_trusted() - seal a trusted key
- * @chip_num: A specific chip number for the request or TPM_ANY_NUM
- * @options: authentication values and other options
+ * tpm2_seal_trusted() - seal the payload of a trusted key
+ * @chip_num: TPM chip to use
  * @payload: the key data in clear and encrypted form
+ * @options: authentication values and other options
  *
- * Returns < 0 on error and 0 on success.
+ * Return: < 0 on error and 0 on success.
  */
 int tpm2_seal_trusted(struct tpm_chip *chip,
 		      struct trusted_key_payload *payload,
@@ -512,7 +512,7 @@ int tpm2_seal_trusted(struct tpm_chip *chip,
 		goto out;
 	}
 
-	rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "sealing data");
+	rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, 0, "sealing data");
 	if (rc)
 		goto out;
 
@@ -538,10 +538,18 @@ out:
 	return rc;
 }
 
-static int tpm2_load(struct tpm_chip *chip,
-		     struct trusted_key_payload *payload,
-		     struct trusted_key_options *options,
-		     u32 *blob_handle)
+/**
+ * tpm2_load_cmd() - execute a TPM2_Load command
+ * @chip_num: TPM chip to use
+ * @payload: the key data in clear and encrypted form
+ * @options: authentication values and other options
+ *
+ * Return: same as with tpm_transmit_cmd
+ */
+static int tpm2_load_cmd(struct tpm_chip *chip,
+			 struct trusted_key_payload *payload,
+			 struct trusted_key_options *options,
+			 u32 *blob_handle, unsigned int flags)
 {
 	struct tpm_buf buf;
 	unsigned int private_len;
@@ -576,7 +584,7 @@ static int tpm2_load(struct tpm_chip *chip,
 		goto out;
 	}
 
-	rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "loading blob");
+	rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, flags, "loading blob");
 	if (!rc)
 		*blob_handle = be32_to_cpup(
 			(__be32 *) &buf.data[TPM_HEADER_SIZE]);
@@ -590,7 +598,16 @@ out:
 	return rc;
 }
 
-static void tpm2_flush_context(struct tpm_chip *chip, u32 handle)
+/**
+ * tpm2_flush_context_cmd() - execute a TPM2_FlushContext command
+ * @chip_num: TPM chip to use
+ * @payload: the key data in clear and encrypted form
+ * @options: authentication values and other options
+ *
+ * Return: same as with tpm_transmit_cmd
+ */
+static void tpm2_flush_context_cmd(struct tpm_chip *chip, u32 handle,
+				   unsigned int flags)
 {
 	struct tpm_buf buf;
 	int rc;
@@ -604,7 +621,8 @@ static void tpm2_flush_context(struct tpm_chip *chip, u32 handle)
 
 	tpm_buf_append_u32(&buf, handle);
 
-	rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "flushing context");
+	rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, flags,
+			      "flushing context");
 	if (rc)
 		dev_warn(&chip->dev, "0x%08x was not flushed, rc=%d\n", handle,
 			 rc);
@@ -612,10 +630,18 @@ static void tpm2_flush_context(struct tpm_chip *chip, u32 handle)
 	tpm_buf_destroy(&buf);
 }
 
-static int tpm2_unseal(struct tpm_chip *chip,
-		       struct trusted_key_payload *payload,
-		       struct trusted_key_options *options,
-		       u32 blob_handle)
+/**
+ * tpm2_unseal_cmd() - execute a TPM2_Unload command
+ * @chip_num: TPM chip to use
+ * @payload: the key data in clear and encrypted form
+ * @options: authentication values and other options
+ *
+ * Return: same as with tpm_transmit_cmd
+ */
+static int tpm2_unseal_cmd(struct tpm_chip *chip,
+			   struct trusted_key_payload *payload,
+			   struct trusted_key_options *options,
+			   u32 blob_handle, unsigned int flags)
 {
 	struct tpm_buf buf;
 	u16 data_len;
@@ -635,7 +661,7 @@ static int tpm2_unseal(struct tpm_chip *chip,
 			     options->blobauth /* hmac */,
 			     TPM_DIGEST_SIZE);
 
-	rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "unsealing");
+	rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, flags, "unsealing");
 	if (rc > 0)
 		rc = -EPERM;
 
@@ -654,12 +680,12 @@ static int tpm2_unseal(struct tpm_chip *chip,
 }
 
 /**
- * tpm_unseal_trusted() - unseal a trusted key
- * @chip_num: A specific chip number for the request or TPM_ANY_NUM
- * @options: authentication values and other options
+ * tpm_unseal_trusted() - unseal the payload of a trusted key
+ * @chip_num: TPM chip to use
  * @payload: the key data in clear and encrypted form
+ * @options: authentication values and other options
  *
- * Returns < 0 on error and 0 on success.
+ * Return: < 0 on error and 0 on success.
  */
 int tpm2_unseal_trusted(struct tpm_chip *chip,
 			struct trusted_key_payload *payload,
@@ -668,14 +694,17 @@ int tpm2_unseal_trusted(struct tpm_chip *chip,
 	u32 blob_handle;
 	int rc;
 
-	rc = tpm2_load(chip, payload, options, &blob_handle);
+	mutex_lock(&chip->tpm_mutex);
+	rc = tpm2_load_cmd(chip, payload, options, &blob_handle,
+			   TPM_TRANSMIT_UNLOCKED);
 	if (rc)
-		return rc;
-
-	rc = tpm2_unseal(chip, payload, options, blob_handle);
-
-	tpm2_flush_context(chip, blob_handle);
+		goto out;
 
+	rc = tpm2_unseal_cmd(chip, payload, options, blob_handle,
+			     TPM_TRANSMIT_UNLOCKED);
+	tpm2_flush_context_cmd(chip, blob_handle, TPM_TRANSMIT_UNLOCKED);
+out:
+	mutex_unlock(&chip->tpm_mutex);
 	return rc;
 }
 
@@ -701,12 +730,13 @@ ssize_t tpm2_get_tpm_pt(struct tpm_chip *chip, u32 property_id,  u32 *value,
 	cmd.params.get_tpm_pt_in.property_id = cpu_to_be32(property_id);
 	cmd.params.get_tpm_pt_in.property_cnt = cpu_to_be32(1);
 
-	rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), desc);
+	rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, desc);
 	if (!rc)
 		*value = be32_to_cpu(cmd.params.get_tpm_pt_out.value);
 
 	return rc;
 }
+EXPORT_SYMBOL_GPL(tpm2_get_tpm_pt);
 
 #define TPM2_STARTUP_IN_SIZE \
 	(sizeof(struct tpm_input_header) + \
@@ -735,7 +765,7 @@ static int tpm2_startup(struct tpm_chip *chip, u16 startup_type)
 	cmd.header.in = tpm2_startup_header;
 
 	cmd.params.startup_in.startup_type = cpu_to_be16(startup_type);
-	return tpm_transmit_cmd(chip, &cmd, sizeof(cmd),
+	return tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0,
 				"attempting to start the TPM");
 }
 
@@ -763,7 +793,7 @@ void tpm2_shutdown(struct tpm_chip *chip, u16 shutdown_type)
 	cmd.header.in = tpm2_shutdown_header;
 	cmd.params.startup_in.startup_type = cpu_to_be16(shutdown_type);
 
-	rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), "stopping the TPM");
+	rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "stopping the TPM");
 
 	/* In places where shutdown command is sent there's no much we can do
 	 * except print the error code on a system failure.
@@ -828,7 +858,7 @@ static int tpm2_start_selftest(struct tpm_chip *chip, bool full)
 	cmd.header.in = tpm2_selftest_header;
 	cmd.params.selftest_in.full_test = full;
 
-	rc = tpm_transmit_cmd(chip, &cmd, TPM2_SELF_TEST_IN_SIZE,
+	rc = tpm_transmit_cmd(chip, &cmd, TPM2_SELF_TEST_IN_SIZE, 0,
 			      "continue selftest");
 
 	/* At least some prototype chips seem to give RC_TESTING error
@@ -880,7 +910,7 @@ static int tpm2_do_selftest(struct tpm_chip *chip)
 		cmd.params.pcrread_in.pcr_select[1] = 0x00;
 		cmd.params.pcrread_in.pcr_select[2] = 0x00;
 
-		rc = tpm_transmit_cmd(chip, (u8 *) &cmd, sizeof(cmd), NULL);
+		rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, NULL);
 		if (rc < 0)
 			break;
 
@@ -895,23 +925,6 @@ static int tpm2_do_selftest(struct tpm_chip *chip)
 }
 
 /**
- * tpm2_gen_interrupt() - generate an interrupt
- * @chip: TPM chip to use
- *
- * 0 is returned when the operation is successful. If a negative number is
- * returned it remarks a POSIX error code. If a positive number is returned
- * it remarks a TPM error.
- */
-int tpm2_gen_interrupt(struct tpm_chip *chip)
-{
-	u32 dummy;
-
-	return tpm2_get_tpm_pt(chip, 0x100, &dummy,
-			       "attempting to generate an interrupt");
-}
-EXPORT_SYMBOL_GPL(tpm2_gen_interrupt);
-
-/**
  * tpm2_probe() - probe TPM 2.0
  * @chip: TPM chip to use
  *
@@ -928,11 +941,9 @@ int tpm2_probe(struct tpm_chip *chip)
 	cmd.params.get_tpm_pt_in.property_id = cpu_to_be32(0x100);
 	cmd.params.get_tpm_pt_in.property_cnt = cpu_to_be32(1);
 
-	rc = tpm_transmit(chip, (const char *) &cmd, sizeof(cmd));
+	rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd),  0, NULL);
 	if (rc <  0)
 		return rc;
-	else if (rc < TPM_HEADER_SIZE)
-		return -EFAULT;
 
 	if (be16_to_cpu(cmd.header.out.tag) == TPM2_ST_NO_SESSIONS)
 		chip->flags |= TPM_CHIP_FLAG_TPM2;
@@ -957,7 +968,7 @@ int tpm2_auto_startup(struct tpm_chip *chip)
 		goto out;
 
 	rc = tpm2_do_selftest(chip);
-	if (rc != TPM2_RC_INITIALIZE) {
+	if (rc != 0 && rc != TPM2_RC_INITIALIZE) {
 		dev_err(&chip->dev, "TPM self test failed\n");
 		goto out;
 	}
@@ -974,7 +985,6 @@ int tpm2_auto_startup(struct tpm_chip *chip)
 		}
 	}
 
-	return rc;
 out:
 	if (rc > 0)
 		rc = -ENODEV;
diff --git a/drivers/char/tpm/tpm_crb.c b/drivers/char/tpm/tpm_crb.c
index 018c382554ba..a7c870af916c 100644
--- a/drivers/char/tpm/tpm_crb.c
+++ b/drivers/char/tpm/tpm_crb.c
@@ -19,7 +19,6 @@
 #include <linux/highmem.h>
 #include <linux/rculist.h>
 #include <linux/module.h>
-#include <linux/platform_device.h>
 #include "tpm.h"
 
 #define ACPI_SIG_TPM2 "TPM2"
@@ -34,14 +33,14 @@ enum crb_defaults {
 	CRB_ACPI_START_INDEX = 1,
 };
 
-enum crb_ca_request {
-	CRB_CA_REQ_GO_IDLE	= BIT(0),
-	CRB_CA_REQ_CMD_READY	= BIT(1),
+enum crb_ctrl_req {
+	CRB_CTRL_REQ_CMD_READY	= BIT(0),
+	CRB_CTRL_REQ_GO_IDLE	= BIT(1),
 };
 
-enum crb_ca_status {
-	CRB_CA_STS_ERROR	= BIT(0),
-	CRB_CA_STS_TPM_IDLE	= BIT(1),
+enum crb_ctrl_sts {
+	CRB_CTRL_STS_ERROR	= BIT(0),
+	CRB_CTRL_STS_TPM_IDLE	= BIT(1),
 };
 
 enum crb_start {
@@ -67,7 +66,7 @@ struct crb_control_area {
 } __packed;
 
 enum crb_status {
-	CRB_STS_COMPLETE	= BIT(0),
+	CRB_DRV_STS_COMPLETE	= BIT(0),
 };
 
 enum crb_flags {
@@ -81,6 +80,7 @@ struct crb_priv {
 	struct crb_control_area __iomem *cca;
 	u8 __iomem *cmd;
 	u8 __iomem *rsp;
+	u32 cmd_size;
 };
 
 static SIMPLE_DEV_PM_OPS(crb_pm, tpm_pm_suspend, tpm_pm_resume);
@@ -92,7 +92,7 @@ static u8 crb_status(struct tpm_chip *chip)
 
 	if ((ioread32(&priv->cca->start) & CRB_START_INVOKE) !=
 	    CRB_START_INVOKE)
-		sts |= CRB_STS_COMPLETE;
+		sts |= CRB_DRV_STS_COMPLETE;
 
 	return sts;
 }
@@ -106,7 +106,7 @@ static int crb_recv(struct tpm_chip *chip, u8 *buf, size_t count)
 	if (count < 6)
 		return -EIO;
 
-	if (ioread32(&priv->cca->sts) & CRB_CA_STS_ERROR)
+	if (ioread32(&priv->cca->sts) & CRB_CTRL_STS_ERROR)
 		return -EIO;
 
 	memcpy_fromio(buf, priv->rsp, 6);
@@ -142,11 +142,14 @@ static int crb_send(struct tpm_chip *chip, u8 *buf, size_t len)
 	struct crb_priv *priv = dev_get_drvdata(&chip->dev);
 	int rc = 0;
 
-	if (len > ioread32(&priv->cca->cmd_size)) {
-		dev_err(&chip->dev,
-			"invalid command count value %x %zx\n",
-			(unsigned int) len,
-			(size_t) ioread32(&priv->cca->cmd_size));
+	/* Zero the cancel register so that the next command will not get
+	 * canceled.
+	 */
+	iowrite32(0, &priv->cca->cancel);
+
+	if (len > priv->cmd_size) {
+		dev_err(&chip->dev, "invalid command count value %zd %d\n",
+			len, priv->cmd_size);
 		return -E2BIG;
 	}
 
@@ -156,7 +159,7 @@ static int crb_send(struct tpm_chip *chip, u8 *buf, size_t len)
 	wmb();
 
 	if (priv->flags & CRB_FL_CRB_START)
-		iowrite32(cpu_to_le32(CRB_START_INVOKE), &priv->cca->start);
+		iowrite32(CRB_START_INVOKE, &priv->cca->start);
 
 	if (priv->flags & CRB_FL_ACPI_START)
 		rc = crb_do_acpi_start(chip);
@@ -168,15 +171,10 @@ static void crb_cancel(struct tpm_chip *chip)
 {
 	struct crb_priv *priv = dev_get_drvdata(&chip->dev);
 
-	iowrite32(cpu_to_le32(CRB_CANCEL_INVOKE), &priv->cca->cancel);
-
-	/* Make sure that cmd is populated before issuing cancel. */
-	wmb();
+	iowrite32(CRB_CANCEL_INVOKE, &priv->cca->cancel);
 
 	if ((priv->flags & CRB_FL_ACPI_START) && crb_do_acpi_start(chip))
 		dev_err(&chip->dev, "ACPI Start failed\n");
-
-	iowrite32(0, &priv->cca->cancel);
 }
 
 static bool crb_req_canceled(struct tpm_chip *chip, u8 status)
@@ -194,8 +192,8 @@ static const struct tpm_class_ops tpm_crb = {
 	.send = crb_send,
 	.cancel = crb_cancel,
 	.req_canceled = crb_req_canceled,
-	.req_complete_mask = CRB_STS_COMPLETE,
-	.req_complete_val = CRB_STS_COMPLETE,
+	.req_complete_mask = CRB_DRV_STS_COMPLETE,
+	.req_complete_val = CRB_DRV_STS_COMPLETE,
 };
 
 static int crb_init(struct acpi_device *device, struct crb_priv *priv)
@@ -265,8 +263,7 @@ static int crb_map_io(struct acpi_device *device, struct crb_priv *priv,
 	acpi_dev_free_resource_list(&resources);
 
 	if (resource_type(&io_res) != IORESOURCE_MEM) {
-		dev_err(dev,
-			FW_BUG "TPM2 ACPI table does not define a memory resource\n");
+		dev_err(dev, FW_BUG "TPM2 ACPI table does not define a memory resource\n");
 		return -EINVAL;
 	}
 
@@ -302,6 +299,7 @@ static int crb_map_io(struct acpi_device *device, struct crb_priv *priv,
 		dev_err(dev, FW_BUG "overlapping command and response buffer sizes are not identical");
 		return -EINVAL;
 	}
+	priv->cmd_size = cmd_size;
 
 	priv->rsp = priv->cmd;
 	return 0;
diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c
index d66f51b3648e..e3bf31b37138 100644
--- a/drivers/char/tpm/tpm_tis_core.c
+++ b/drivers/char/tpm/tpm_tis_core.c
@@ -440,7 +440,6 @@ static int probe_itpm(struct tpm_chip *chip)
 		0x00, 0x00, 0x00, 0xf1
 	};
 	size_t len = sizeof(cmd_getticks);
-	bool itpm;
 	u16 vendor;
 
 	rc = tpm_tis_read16(priv, TPM_DID_VID(0), &vendor);
@@ -451,8 +450,6 @@ static int probe_itpm(struct tpm_chip *chip)
 	if (vendor != TPM_VID_INTEL)
 		return 0;
 
-	itpm = false;
-
 	rc = tpm_tis_send_data(chip, cmd_getticks, len);
 	if (rc == 0)
 		goto out;
@@ -460,8 +457,6 @@ static int probe_itpm(struct tpm_chip *chip)
 	tpm_tis_ready(chip);
 	release_locality(chip, priv->locality, 0);
 
-	itpm = true;
-
 	rc = tpm_tis_send_data(chip, cmd_getticks, len);
 	if (rc == 0) {
 		dev_info(&chip->dev, "Detected an iTPM.\n");
@@ -526,6 +521,18 @@ static irqreturn_t tis_int_handler(int dummy, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+static int tpm_tis_gen_interrupt(struct tpm_chip *chip)
+{
+	const char *desc = "attempting to generate an interrupt";
+	u32 cap2;
+	cap_t cap;
+
+	if (chip->flags & TPM_CHIP_FLAG_TPM2)
+		return tpm2_get_tpm_pt(chip, 0x100, &cap2, desc);
+	else
+		return tpm_getcap(chip, TPM_CAP_PROP_TIS_TIMEOUT, &cap, desc);
+}
+
 /* Register the IRQ and issue a command that will cause an interrupt. If an
  * irq is seen then leave the chip setup for IRQ operation, otherwise reverse
  * everything and leave in polling mode. Returns 0 on success.
@@ -575,10 +582,9 @@ static int tpm_tis_probe_irq_single(struct tpm_chip *chip, u32 intmask,
 	/* Generate an interrupt by having the core call through to
 	 * tpm_tis_send
 	 */
-	if (chip->flags & TPM_CHIP_FLAG_TPM2)
-		tpm2_gen_interrupt(chip);
-	else
-		tpm_gen_interrupt(chip);
+	rc = tpm_tis_gen_interrupt(chip);
+	if (rc < 0)
+		return rc;
 
 	/* tpm_tis_send will either confirm the interrupt is working or it
 	 * will call disable_irq which undoes all of the above.
diff --git a/drivers/char/ttyprintk.c b/drivers/char/ttyprintk.c
index b098d2d0b7c4..67549ce88cc9 100644
--- a/drivers/char/ttyprintk.c
+++ b/drivers/char/ttyprintk.c
@@ -31,60 +31,53 @@ static struct ttyprintk_port tpk_port;
  * printk messages (also suitable for logging service):
  * - any cr is replaced by nl
  * - adds a ttyprintk source tag in front of each line
- * - too long message is fragmeted, with '\'nl between fragments
- * - TPK_STR_SIZE isn't really the write_room limiting factor, bcause
+ * - too long message is fragmented, with '\'nl between fragments
+ * - TPK_STR_SIZE isn't really the write_room limiting factor, because
  *   it is emptied on the fly during preformatting.
  */
 #define TPK_STR_SIZE 508 /* should be bigger then max expected line length */
 #define TPK_MAX_ROOM 4096 /* we could assume 4K for instance */
-static const char *tpk_tag = "[U] "; /* U for User */
 static int tpk_curr;
 
+static char tpk_buffer[TPK_STR_SIZE + 4];
+
+static void tpk_flush(void)
+{
+	if (tpk_curr > 0) {
+		tpk_buffer[tpk_curr] = '\0';
+		pr_info("[U] %s\n", tpk_buffer);
+		tpk_curr = 0;
+	}
+}
+
 static int tpk_printk(const unsigned char *buf, int count)
 {
-	static char tmp[TPK_STR_SIZE + 4];
 	int i = tpk_curr;
 
 	if (buf == NULL) {
-		/* flush tmp[] */
-		if (tpk_curr > 0) {
-			/* non nl or cr terminated message - add nl */
-			tmp[tpk_curr + 0] = '\n';
-			tmp[tpk_curr + 1] = '\0';
-			printk(KERN_INFO "%s%s", tpk_tag, tmp);
-			tpk_curr = 0;
-		}
+		tpk_flush();
 		return i;
 	}
 
 	for (i = 0; i < count; i++) {
-		tmp[tpk_curr] = buf[i];
-		if (tpk_curr < TPK_STR_SIZE) {
-			switch (buf[i]) {
-			case '\r':
-				/* replace cr with nl */
-				tmp[tpk_curr + 0] = '\n';
-				tmp[tpk_curr + 1] = '\0';
-				printk(KERN_INFO "%s%s", tpk_tag, tmp);
-				tpk_curr = 0;
-				if ((i + 1) < count && buf[i + 1] == '\n')
-					i++;
-				break;
-			case '\n':
-				tmp[tpk_curr + 1] = '\0';
-				printk(KERN_INFO "%s%s", tpk_tag, tmp);
-				tpk_curr = 0;
-				break;
-			default:
-				tpk_curr++;
-			}
-		} else {
+		if (tpk_curr >= TPK_STR_SIZE) {
 			/* end of tmp buffer reached: cut the message in two */
-			tmp[tpk_curr + 1] = '\\';
-			tmp[tpk_curr + 2] = '\n';
-			tmp[tpk_curr + 3] = '\0';
-			printk(KERN_INFO "%s%s", tpk_tag, tmp);
-			tpk_curr = 0;
+			tpk_buffer[tpk_curr++] = '\\';
+			tpk_flush();
+		}
+
+		switch (buf[i]) {
+		case '\r':
+			tpk_flush();
+			if ((i + 1) < count && buf[i + 1] == '\n')
+				i++;
+			break;
+		case '\n':
+			tpk_flush();
+			break;
+		default:
+			tpk_buffer[tpk_curr++] = buf[i];
+			break;
 		}
 	}
 
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index d2406fe25533..d433b1db1fdd 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -38,7 +38,6 @@
 #include <linux/workqueue.h>
 #include <linux/module.h>
 #include <linux/dma-mapping.h>
-#include <linux/kconfig.h>
 #include "../tty/hvc/hvc_console.h"
 
 #define is_rproc_enabled IS_ENABLED(CONFIG_REMOTEPROC)
@@ -165,6 +164,12 @@ struct ports_device {
 	 */
 	struct virtqueue *c_ivq, *c_ovq;
 
+	/*
+	 * A control packet buffer for guest->host requests, protected
+	 * by c_ovq_lock.
+	 */
+	struct virtio_console_control cpkt;
+
 	/* Array of per-port IO virtqueues */
 	struct virtqueue **in_vqs, **out_vqs;
 
@@ -560,28 +565,29 @@ static ssize_t __send_control_msg(struct ports_device *portdev, u32 port_id,
 				  unsigned int event, unsigned int value)
 {
 	struct scatterlist sg[1];
-	struct virtio_console_control cpkt;
 	struct virtqueue *vq;
 	unsigned int len;
 
 	if (!use_multiport(portdev))
 		return 0;
 
-	cpkt.id = cpu_to_virtio32(portdev->vdev, port_id);
-	cpkt.event = cpu_to_virtio16(portdev->vdev, event);
-	cpkt.value = cpu_to_virtio16(portdev->vdev, value);
-
 	vq = portdev->c_ovq;
 
-	sg_init_one(sg, &cpkt, sizeof(cpkt));
-
 	spin_lock(&portdev->c_ovq_lock);
-	if (virtqueue_add_outbuf(vq, sg, 1, &cpkt, GFP_ATOMIC) == 0) {
+
+	portdev->cpkt.id = cpu_to_virtio32(portdev->vdev, port_id);
+	portdev->cpkt.event = cpu_to_virtio16(portdev->vdev, event);
+	portdev->cpkt.value = cpu_to_virtio16(portdev->vdev, value);
+
+	sg_init_one(sg, &portdev->cpkt, sizeof(struct virtio_console_control));
+
+	if (virtqueue_add_outbuf(vq, sg, 1, &portdev->cpkt, GFP_ATOMIC) == 0) {
 		virtqueue_kick(vq);
 		while (!virtqueue_get_buf(vq, &len)
 			&& !virtqueue_is_broken(vq))
 			cpu_relax();
 	}
+
 	spin_unlock(&portdev->c_ovq_lock);
 	return 0;
 }
@@ -882,7 +888,7 @@ static int pipe_to_sg(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
 		return 0;
 
 	/* Try lock this page */
-	if (buf->ops->steal(pipe, buf) == 0) {
+	if (pipe_buf_steal(pipe, buf) == 0) {
 		/* Get reference and unlock page for moving */
 		get_page(buf->page);
 		unlock_page(buf->page);
diff --git a/drivers/char/xillybus/xillybus_core.c b/drivers/char/xillybus/xillybus_core.c
index dcd19f3f182e..b6c9cdead7f3 100644
--- a/drivers/char/xillybus/xillybus_core.c
+++ b/drivers/char/xillybus/xillybus_core.c
@@ -655,10 +655,10 @@ static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
 
 	version = channel->wr_buffers[0]->addr;
 
-	/* Check version number. Accept anything below 0x82 for now. */
+	/* Check version number. Reject anything above 0x82. */
 	if (*version > 0x82) {
 		dev_err(endpoint->dev,
-			"No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgarde. Aborting.\n",
+			"No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n",
 			*version);
 		return -ENODEV;
 	}
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index e2d9bd760c84..6a8ac04bedeb 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -31,22 +31,12 @@ config COMMON_CLK_WM831X
 
 source "drivers/clk/versatile/Kconfig"
 
-config COMMON_CLK_MAX_GEN
-        bool
-
 config COMMON_CLK_MAX77686
-	tristate "Clock driver for Maxim 77686 MFD"
-	depends on MFD_MAX77686
-	select COMMON_CLK_MAX_GEN
-	---help---
-	  This driver supports Maxim 77686 crystal oscillator clock. 
-
-config COMMON_CLK_MAX77802
-	tristate "Clock driver for Maxim 77802 PMIC"
-	depends on MFD_MAX77686
-	select COMMON_CLK_MAX_GEN
+	tristate "Clock driver for Maxim 77620/77686/77802 MFD"
+	depends on MFD_MAX77686 || MFD_MAX77620
 	---help---
-	  This driver supports Maxim 77802 crystal oscillator clock.
+	  This driver supports Maxim 77620/77686/77802 crystal oscillator
+	  clock.
 
 config COMMON_CLK_RK808
 	tristate "Clock driver for RK808/RK818"
@@ -210,6 +200,7 @@ config COMMON_CLK_OXNAS
 
 source "drivers/clk/bcm/Kconfig"
 source "drivers/clk/hisilicon/Kconfig"
+source "drivers/clk/mediatek/Kconfig"
 source "drivers/clk/meson/Kconfig"
 source "drivers/clk/mvebu/Kconfig"
 source "drivers/clk/qcom/Kconfig"
@@ -218,5 +209,6 @@ source "drivers/clk/samsung/Kconfig"
 source "drivers/clk/sunxi-ng/Kconfig"
 source "drivers/clk/tegra/Kconfig"
 source "drivers/clk/ti/Kconfig"
+source "drivers/clk/uniphier/Kconfig"
 
 endmenu
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index 3b6f9cf3464a..925081ec14c0 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -26,10 +26,7 @@ obj-$(CONFIG_ARCH_CLPS711X)		+= clk-clps711x.o
 obj-$(CONFIG_COMMON_CLK_CS2000_CP)	+= clk-cs2000-cp.o
 obj-$(CONFIG_ARCH_EFM32)		+= clk-efm32gg.o
 obj-$(CONFIG_ARCH_HIGHBANK)		+= clk-highbank.o
-obj-$(CONFIG_MACH_LOONGSON32)		+= clk-ls1x.o
-obj-$(CONFIG_COMMON_CLK_MAX_GEN)	+= clk-max-gen.o
 obj-$(CONFIG_COMMON_CLK_MAX77686)	+= clk-max77686.o
-obj-$(CONFIG_COMMON_CLK_MAX77802)	+= clk-max77802.o
 obj-$(CONFIG_ARCH_MB86S7X)		+= clk-mb86s7x.o
 obj-$(CONFIG_ARCH_MOXART)		+= clk-moxart.o
 obj-$(CONFIG_ARCH_NOMADIK)		+= clk-nomadik.o
@@ -63,6 +60,7 @@ obj-$(CONFIG_ARCH_HISI)			+= hisilicon/
 obj-$(CONFIG_ARCH_MXC)			+= imx/
 obj-$(CONFIG_MACH_INGENIC)		+= ingenic/
 obj-$(CONFIG_COMMON_CLK_KEYSTONE)	+= keystone/
+obj-$(CONFIG_MACH_LOONGSON32)		+= loongson1/
 obj-$(CONFIG_ARCH_MEDIATEK)		+= mediatek/
 obj-$(CONFIG_COMMON_CLK_AMLOGIC)	+= meson/
 obj-$(CONFIG_MACH_PIC32)		+= microchip/
@@ -86,6 +84,7 @@ obj-$(CONFIG_ARCH_SUNXI)		+= sunxi/
 obj-$(CONFIG_ARCH_SUNXI)		+= sunxi-ng/
 obj-$(CONFIG_ARCH_TEGRA)		+= tegra/
 obj-y					+= ti/
+obj-$(CONFIG_CLK_UNIPHIER)		+= uniphier/
 obj-$(CONFIG_ARCH_U8500)		+= ux500/
 obj-$(CONFIG_COMMON_CLK_VERSATILE)	+= versatile/
 obj-$(CONFIG_X86)			+= x86/
diff --git a/drivers/clk/at91/clk-generated.c b/drivers/clk/at91/clk-generated.c
index 7f6bec8837ea..4e1cd5aa69d8 100644
--- a/drivers/clk/at91/clk-generated.c
+++ b/drivers/clk/at91/clk-generated.c
@@ -233,14 +233,16 @@ static void clk_generated_startup(struct clk_generated *gck)
 					>> AT91_PMC_PCR_GCKDIV_OFFSET;
 }
 
-static struct clk * __init
-at91_clk_register_generated(struct regmap *regmap,  spinlock_t *lock, const char
-			    *name, const char **parent_names, u8 num_parents,
-			    u8 id, const struct clk_range *range)
+static struct clk_hw * __init
+at91_clk_register_generated(struct regmap *regmap, spinlock_t *lock,
+			    const char *name, const char **parent_names,
+			    u8 num_parents, u8 id,
+			    const struct clk_range *range)
 {
 	struct clk_generated *gck;
-	struct clk *clk = NULL;
 	struct clk_init_data init;
+	struct clk_hw *hw;
+	int ret;
 
 	gck = kzalloc(sizeof(*gck), GFP_KERNEL);
 	if (!gck)
@@ -258,13 +260,15 @@ at91_clk_register_generated(struct regmap *regmap,  spinlock_t *lock, const char
 	gck->lock = lock;
 	gck->range = *range;
 
-	clk = clk_register(NULL, &gck->hw);
-	if (IS_ERR(clk))
+	hw = &gck->hw;
+	ret = clk_hw_register(NULL, &gck->hw);
+	if (ret) {
 		kfree(gck);
-	else
+		hw = ERR_PTR(ret);
+	} else
 		clk_generated_startup(gck);
 
-	return clk;
+	return hw;
 }
 
 static void __init of_sama5d2_clk_generated_setup(struct device_node *np)
@@ -272,7 +276,7 @@ static void __init of_sama5d2_clk_generated_setup(struct device_node *np)
 	int num;
 	u32 id;
 	const char *name;
-	struct clk *clk;
+	struct clk_hw *hw;
 	unsigned int num_parents;
 	const char *parent_names[GENERATED_SOURCE_MAX];
 	struct device_node *gcknp;
@@ -306,13 +310,13 @@ static void __init of_sama5d2_clk_generated_setup(struct device_node *np)
 		of_at91_get_clk_range(gcknp, "atmel,clk-output-range",
 				      &range);
 
-		clk = at91_clk_register_generated(regmap, &pmc_pcr_lock, name,
+		hw = at91_clk_register_generated(regmap, &pmc_pcr_lock, name,
 						  parent_names, num_parents,
 						  id, &range);
-		if (IS_ERR(clk))
+		if (IS_ERR(hw))
 			continue;
 
-		of_clk_add_provider(gcknp, of_clk_src_simple_get, clk);
+		of_clk_add_hw_provider(gcknp, of_clk_hw_simple_get, hw);
 	}
 }
 CLK_OF_DECLARE(of_sama5d2_clk_generated_setup, "atmel,sama5d2-clk-generated",
diff --git a/drivers/clk/at91/clk-h32mx.c b/drivers/clk/at91/clk-h32mx.c
index 8e20c8a76db7..e0daa4a31f88 100644
--- a/drivers/clk/at91/clk-h32mx.c
+++ b/drivers/clk/at91/clk-h32mx.c
@@ -92,7 +92,7 @@ static void __init of_sama5d4_clk_h32mx_setup(struct device_node *np)
 	struct clk_init_data init;
 	const char *parent_name;
 	struct regmap *regmap;
-	struct clk *clk;
+	int ret;
 
 	regmap = syscon_node_to_regmap(of_get_parent(np));
 	if (IS_ERR(regmap))
@@ -113,13 +113,13 @@ static void __init of_sama5d4_clk_h32mx_setup(struct device_node *np)
 	h32mxclk->hw.init = &init;
 	h32mxclk->regmap = regmap;
 
-	clk = clk_register(NULL, &h32mxclk->hw);
-	if (IS_ERR(clk)) {
+	ret = clk_hw_register(NULL, &h32mxclk->hw);
+	if (ret) {
 		kfree(h32mxclk);
 		return;
 	}
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, &h32mxclk->hw);
 }
 CLK_OF_DECLARE(of_sama5d4_clk_h32mx_setup, "atmel,sama5d4-clk-h32mx",
 	       of_sama5d4_clk_h32mx_setup);
diff --git a/drivers/clk/at91/clk-main.c b/drivers/clk/at91/clk-main.c
index 58b5baca670c..c813c27f2e58 100644
--- a/drivers/clk/at91/clk-main.c
+++ b/drivers/clk/at91/clk-main.c
@@ -128,15 +128,16 @@ static const struct clk_ops main_osc_ops = {
 	.is_prepared = clk_main_osc_is_prepared,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_main_osc(struct regmap *regmap,
 			   const char *name,
 			   const char *parent_name,
 			   bool bypass)
 {
 	struct clk_main_osc *osc;
-	struct clk *clk = NULL;
 	struct clk_init_data init;
+	struct clk_hw *hw;
+	int ret;
 
 	if (!name || !parent_name)
 		return ERR_PTR(-EINVAL);
@@ -160,16 +161,19 @@ at91_clk_register_main_osc(struct regmap *regmap,
 				   AT91_PMC_MOSCEN,
 				   AT91_PMC_OSCBYPASS | AT91_PMC_KEY);
 
-	clk = clk_register(NULL, &osc->hw);
-	if (IS_ERR(clk))
+	hw = &osc->hw;
+	ret = clk_hw_register(NULL, &osc->hw);
+	if (ret) {
 		kfree(osc);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init of_at91rm9200_clk_main_osc_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *name = np->name;
 	const char *parent_name;
 	struct regmap *regmap;
@@ -183,11 +187,11 @@ static void __init of_at91rm9200_clk_main_osc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	clk = at91_clk_register_main_osc(regmap, name, parent_name, bypass);
-	if (IS_ERR(clk))
+	hw = at91_clk_register_main_osc(regmap, name, parent_name, bypass);
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(at91rm9200_clk_main_osc, "atmel,at91rm9200-clk-main-osc",
 	       of_at91rm9200_clk_main_osc_setup);
@@ -271,14 +275,15 @@ static const struct clk_ops main_rc_osc_ops = {
 	.recalc_accuracy = clk_main_rc_osc_recalc_accuracy,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_main_rc_osc(struct regmap *regmap,
 			      const char *name,
 			      u32 frequency, u32 accuracy)
 {
 	struct clk_main_rc_osc *osc;
-	struct clk *clk = NULL;
 	struct clk_init_data init;
+	struct clk_hw *hw;
+	int ret;
 
 	if (!name || !frequency)
 		return ERR_PTR(-EINVAL);
@@ -298,16 +303,19 @@ at91_clk_register_main_rc_osc(struct regmap *regmap,
 	osc->frequency = frequency;
 	osc->accuracy = accuracy;
 
-	clk = clk_register(NULL, &osc->hw);
-	if (IS_ERR(clk))
+	hw = &osc->hw;
+	ret = clk_hw_register(NULL, hw);
+	if (ret) {
 		kfree(osc);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init of_at91sam9x5_clk_main_rc_osc_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	u32 frequency = 0;
 	u32 accuracy = 0;
 	const char *name = np->name;
@@ -321,11 +329,11 @@ static void __init of_at91sam9x5_clk_main_rc_osc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	clk = at91_clk_register_main_rc_osc(regmap, name, frequency, accuracy);
-	if (IS_ERR(clk))
+	hw = at91_clk_register_main_rc_osc(regmap, name, frequency, accuracy);
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(at91sam9x5_clk_main_rc_osc, "atmel,at91sam9x5-clk-main-rc-osc",
 	       of_at91sam9x5_clk_main_rc_osc_setup);
@@ -395,14 +403,15 @@ static const struct clk_ops rm9200_main_ops = {
 	.recalc_rate = clk_rm9200_main_recalc_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_rm9200_main(struct regmap *regmap,
 			      const char *name,
 			      const char *parent_name)
 {
 	struct clk_rm9200_main *clkmain;
-	struct clk *clk = NULL;
 	struct clk_init_data init;
+	struct clk_hw *hw;
+	int ret;
 
 	if (!name)
 		return ERR_PTR(-EINVAL);
@@ -423,16 +432,19 @@ at91_clk_register_rm9200_main(struct regmap *regmap,
 	clkmain->hw.init = &init;
 	clkmain->regmap = regmap;
 
-	clk = clk_register(NULL, &clkmain->hw);
-	if (IS_ERR(clk))
+	hw = &clkmain->hw;
+	ret = clk_hw_register(NULL, &clkmain->hw);
+	if (ret) {
 		kfree(clkmain);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init of_at91rm9200_clk_main_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *parent_name;
 	const char *name = np->name;
 	struct regmap *regmap;
@@ -444,11 +456,11 @@ static void __init of_at91rm9200_clk_main_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	clk = at91_clk_register_rm9200_main(regmap, name, parent_name);
-	if (IS_ERR(clk))
+	hw = at91_clk_register_rm9200_main(regmap, name, parent_name);
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(at91rm9200_clk_main, "atmel,at91rm9200-clk-main",
 	       of_at91rm9200_clk_main_setup);
@@ -529,16 +541,17 @@ static const struct clk_ops sam9x5_main_ops = {
 	.get_parent = clk_sam9x5_main_get_parent,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_sam9x5_main(struct regmap *regmap,
 			      const char *name,
 			      const char **parent_names,
 			      int num_parents)
 {
 	struct clk_sam9x5_main *clkmain;
-	struct clk *clk = NULL;
 	struct clk_init_data init;
 	unsigned int status;
+	struct clk_hw *hw;
+	int ret;
 
 	if (!name)
 		return ERR_PTR(-EINVAL);
@@ -561,16 +574,19 @@ at91_clk_register_sam9x5_main(struct regmap *regmap,
 	regmap_read(clkmain->regmap, AT91_CKGR_MOR, &status);
 	clkmain->parent = status & AT91_PMC_MOSCEN ? 1 : 0;
 
-	clk = clk_register(NULL, &clkmain->hw);
-	if (IS_ERR(clk))
+	hw = &clkmain->hw;
+	ret = clk_hw_register(NULL, &clkmain->hw);
+	if (ret) {
 		kfree(clkmain);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init of_at91sam9x5_clk_main_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *parent_names[2];
 	unsigned int num_parents;
 	const char *name = np->name;
@@ -587,12 +603,12 @@ static void __init of_at91sam9x5_clk_main_setup(struct device_node *np)
 
 	of_property_read_string(np, "clock-output-names", &name);
 
-	clk = at91_clk_register_sam9x5_main(regmap, name, parent_names,
+	hw = at91_clk_register_sam9x5_main(regmap, name, parent_names,
 					    num_parents);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(at91sam9x5_clk_main, "atmel,at91sam9x5-clk-main",
 	       of_at91sam9x5_clk_main_setup);
diff --git a/drivers/clk/at91/clk-master.c b/drivers/clk/at91/clk-master.c
index d1021e106191..e9cba9fc26d7 100644
--- a/drivers/clk/at91/clk-master.c
+++ b/drivers/clk/at91/clk-master.c
@@ -120,7 +120,7 @@ static const struct clk_ops master_ops = {
 	.get_parent = clk_master_get_parent,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_master(struct regmap *regmap,
 		const char *name, int num_parents,
 		const char **parent_names,
@@ -128,8 +128,9 @@ at91_clk_register_master(struct regmap *regmap,
 		const struct clk_master_characteristics *characteristics)
 {
 	struct clk_master *master;
-	struct clk *clk = NULL;
 	struct clk_init_data init;
+	struct clk_hw *hw;
+	int ret;
 
 	if (!name || !num_parents || !parent_names)
 		return ERR_PTR(-EINVAL);
@@ -149,12 +150,14 @@ at91_clk_register_master(struct regmap *regmap,
 	master->characteristics = characteristics;
 	master->regmap = regmap;
 
-	clk = clk_register(NULL, &master->hw);
-	if (IS_ERR(clk)) {
+	hw = &master->hw;
+	ret = clk_hw_register(NULL, &master->hw);
+	if (ret) {
 		kfree(master);
+		hw = ERR_PTR(ret);
 	}
 
-	return clk;
+	return hw;
 }
 
 
@@ -198,7 +201,7 @@ static void __init
 of_at91_clk_master_setup(struct device_node *np,
 			 const struct clk_master_layout *layout)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	unsigned int num_parents;
 	const char *parent_names[MASTER_SOURCE_MAX];
 	const char *name = np->name;
@@ -221,13 +224,13 @@ of_at91_clk_master_setup(struct device_node *np,
 	if (IS_ERR(regmap))
 		return;
 
-	clk = at91_clk_register_master(regmap, name, num_parents,
+	hw = at91_clk_register_master(regmap, name, num_parents,
 				       parent_names, layout,
 				       characteristics);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto out_free_characteristics;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 	return;
 
 out_free_characteristics:
diff --git a/drivers/clk/at91/clk-peripheral.c b/drivers/clk/at91/clk-peripheral.c
index fd160728e990..dc29fd979d3f 100644
--- a/drivers/clk/at91/clk-peripheral.c
+++ b/drivers/clk/at91/clk-peripheral.c
@@ -104,13 +104,14 @@ static const struct clk_ops peripheral_ops = {
 	.is_enabled = clk_peripheral_is_enabled,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_peripheral(struct regmap *regmap, const char *name,
 			     const char *parent_name, u32 id)
 {
 	struct clk_peripheral *periph;
-	struct clk *clk = NULL;
 	struct clk_init_data init;
+	struct clk_hw *hw;
+	int ret;
 
 	if (!name || !parent_name || id > PERIPHERAL_ID_MAX)
 		return ERR_PTR(-EINVAL);
@@ -129,11 +130,14 @@ at91_clk_register_peripheral(struct regmap *regmap, const char *name,
 	periph->hw.init = &init;
 	periph->regmap = regmap;
 
-	clk = clk_register(NULL, &periph->hw);
-	if (IS_ERR(clk))
+	hw = &periph->hw;
+	ret = clk_hw_register(NULL, &periph->hw);
+	if (ret) {
 		kfree(periph);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void clk_sam9x5_peripheral_autodiv(struct clk_sam9x5_peripheral *periph)
@@ -327,14 +331,15 @@ static const struct clk_ops sam9x5_peripheral_ops = {
 	.set_rate = clk_sam9x5_peripheral_set_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_sam9x5_peripheral(struct regmap *regmap, spinlock_t *lock,
 				    const char *name, const char *parent_name,
 				    u32 id, const struct clk_range *range)
 {
 	struct clk_sam9x5_peripheral *periph;
-	struct clk *clk = NULL;
 	struct clk_init_data init;
+	struct clk_hw *hw;
+	int ret;
 
 	if (!name || !parent_name)
 		return ERR_PTR(-EINVAL);
@@ -357,13 +362,15 @@ at91_clk_register_sam9x5_peripheral(struct regmap *regmap, spinlock_t *lock,
 	periph->auto_div = true;
 	periph->range = *range;
 
-	clk = clk_register(NULL, &periph->hw);
-	if (IS_ERR(clk))
+	hw = &periph->hw;
+	ret = clk_hw_register(NULL, &periph->hw);
+	if (ret) {
 		kfree(periph);
-	else
+		hw = ERR_PTR(ret);
+	} else
 		clk_sam9x5_peripheral_autodiv(periph);
 
-	return clk;
+	return hw;
 }
 
 static void __init
@@ -371,7 +378,7 @@ of_at91_clk_periph_setup(struct device_node *np, u8 type)
 {
 	int num;
 	u32 id;
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *parent_name;
 	const char *name;
 	struct device_node *periphclknp;
@@ -400,7 +407,7 @@ of_at91_clk_periph_setup(struct device_node *np, u8 type)
 			name = periphclknp->name;
 
 		if (type == PERIPHERAL_AT91RM9200) {
-			clk = at91_clk_register_peripheral(regmap, name,
+			hw = at91_clk_register_peripheral(regmap, name,
 							   parent_name, id);
 		} else {
 			struct clk_range range = CLK_RANGE(0, 0);
@@ -409,17 +416,17 @@ of_at91_clk_periph_setup(struct device_node *np, u8 type)
 					      "atmel,clk-output-range",
 					      &range);
 
-			clk = at91_clk_register_sam9x5_peripheral(regmap,
+			hw = at91_clk_register_sam9x5_peripheral(regmap,
 								  &pmc_pcr_lock,
 								  name,
 								  parent_name,
 								  id, &range);
 		}
 
-		if (IS_ERR(clk))
+		if (IS_ERR(hw))
 			continue;
 
-		of_clk_add_provider(periphclknp, of_clk_src_simple_get, clk);
+		of_clk_add_hw_provider(periphclknp, of_clk_hw_simple_get, hw);
 	}
 }
 
diff --git a/drivers/clk/at91/clk-pll.c b/drivers/clk/at91/clk-pll.c
index fb2e0b56d4b7..45ad168e1496 100644
--- a/drivers/clk/at91/clk-pll.c
+++ b/drivers/clk/at91/clk-pll.c
@@ -296,17 +296,18 @@ static const struct clk_ops pll_ops = {
 	.set_rate = clk_pll_set_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_pll(struct regmap *regmap, const char *name,
 		      const char *parent_name, u8 id,
 		      const struct clk_pll_layout *layout,
 		      const struct clk_pll_characteristics *characteristics)
 {
 	struct clk_pll *pll;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
 	int offset = PLL_REG(id);
 	unsigned int pllr;
+	int ret;
 
 	if (id > PLL_MAX_ID)
 		return ERR_PTR(-EINVAL);
@@ -330,12 +331,14 @@ at91_clk_register_pll(struct regmap *regmap, const char *name,
 	pll->div = PLL_DIV(pllr);
 	pll->mul = PLL_MUL(pllr, layout);
 
-	clk = clk_register(NULL, &pll->hw);
-	if (IS_ERR(clk)) {
+	hw = &pll->hw;
+	ret = clk_hw_register(NULL, &pll->hw);
+	if (ret) {
 		kfree(pll);
+		hw = ERR_PTR(ret);
 	}
 
-	return clk;
+	return hw;
 }
 
 
@@ -465,7 +468,7 @@ of_at91_clk_pll_setup(struct device_node *np,
 		      const struct clk_pll_layout *layout)
 {
 	u32 id;
-	struct clk *clk;
+	struct clk_hw *hw;
 	struct regmap *regmap;
 	const char *parent_name;
 	const char *name = np->name;
@@ -486,12 +489,12 @@ of_at91_clk_pll_setup(struct device_node *np,
 	if (!characteristics)
 		return;
 
-	clk = at91_clk_register_pll(regmap, name, parent_name, id, layout,
+	hw = at91_clk_register_pll(regmap, name, parent_name, id, layout,
 				    characteristics);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto out_free_characteristics;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 	return;
 
 out_free_characteristics:
diff --git a/drivers/clk/at91/clk-plldiv.c b/drivers/clk/at91/clk-plldiv.c
index 2bed26481027..b4afaf22f3fd 100644
--- a/drivers/clk/at91/clk-plldiv.c
+++ b/drivers/clk/at91/clk-plldiv.c
@@ -75,13 +75,14 @@ static const struct clk_ops plldiv_ops = {
 	.set_rate = clk_plldiv_set_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_plldiv(struct regmap *regmap, const char *name,
 			 const char *parent_name)
 {
 	struct clk_plldiv *plldiv;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	plldiv = kzalloc(sizeof(*plldiv), GFP_KERNEL);
 	if (!plldiv)
@@ -96,18 +97,20 @@ at91_clk_register_plldiv(struct regmap *regmap, const char *name,
 	plldiv->hw.init = &init;
 	plldiv->regmap = regmap;
 
-	clk = clk_register(NULL, &plldiv->hw);
-
-	if (IS_ERR(clk))
+	hw = &plldiv->hw;
+	ret = clk_hw_register(NULL, &plldiv->hw);
+	if (ret) {
 		kfree(plldiv);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init
 of_at91sam9x5_clk_plldiv_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *parent_name;
 	const char *name = np->name;
 	struct regmap *regmap;
@@ -120,12 +123,11 @@ of_at91sam9x5_clk_plldiv_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	clk = at91_clk_register_plldiv(regmap, name, parent_name);
-	if (IS_ERR(clk))
+	hw = at91_clk_register_plldiv(regmap, name, parent_name);
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
-	return;
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(at91sam9x5_clk_plldiv, "atmel,at91sam9x5-clk-plldiv",
 	       of_at91sam9x5_clk_plldiv_setup);
diff --git a/drivers/clk/at91/clk-programmable.c b/drivers/clk/at91/clk-programmable.c
index 25d5906640c3..190122e64a3a 100644
--- a/drivers/clk/at91/clk-programmable.c
+++ b/drivers/clk/at91/clk-programmable.c
@@ -170,15 +170,16 @@ static const struct clk_ops programmable_ops = {
 	.set_rate = clk_programmable_set_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_programmable(struct regmap *regmap,
 			       const char *name, const char **parent_names,
 			       u8 num_parents, u8 id,
 			       const struct clk_programmable_layout *layout)
 {
 	struct clk_programmable *prog;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	if (id > PROG_ID_MAX)
 		return ERR_PTR(-EINVAL);
@@ -198,11 +199,14 @@ at91_clk_register_programmable(struct regmap *regmap,
 	prog->hw.init = &init;
 	prog->regmap = regmap;
 
-	clk = clk_register(NULL, &prog->hw);
-	if (IS_ERR(clk))
+	hw = &prog->hw;
+	ret = clk_hw_register(NULL, &prog->hw);
+	if (ret) {
 		kfree(prog);
+		hw = &prog->hw;
+	}
 
-	return clk;
+	return hw;
 }
 
 static const struct clk_programmable_layout at91rm9200_programmable_layout = {
@@ -229,7 +233,7 @@ of_at91_clk_prog_setup(struct device_node *np,
 {
 	int num;
 	u32 id;
-	struct clk *clk;
+	struct clk_hw *hw;
 	unsigned int num_parents;
 	const char *parent_names[PROG_SOURCE_MAX];
 	const char *name;
@@ -257,13 +261,13 @@ of_at91_clk_prog_setup(struct device_node *np,
 		if (of_property_read_string(np, "clock-output-names", &name))
 			name = progclknp->name;
 
-		clk = at91_clk_register_programmable(regmap, name,
+		hw = at91_clk_register_programmable(regmap, name,
 						     parent_names, num_parents,
 						     id, layout);
-		if (IS_ERR(clk))
+		if (IS_ERR(hw))
 			continue;
 
-		of_clk_add_provider(progclknp, of_clk_src_simple_get, clk);
+		of_clk_add_hw_provider(progclknp, of_clk_hw_simple_get, hw);
 	}
 }
 
diff --git a/drivers/clk/at91/clk-slow.c b/drivers/clk/at91/clk-slow.c
index 61090b1146cf..560a8b9abf93 100644
--- a/drivers/clk/at91/clk-slow.c
+++ b/drivers/clk/at91/clk-slow.c
@@ -13,42 +13,11 @@
 #include <linux/clk-provider.h>
 #include <linux/clkdev.h>
 #include <linux/clk/at91_pmc.h>
-#include <linux/delay.h>
 #include <linux/of.h>
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 
 #include "pmc.h"
-#include "sckc.h"
-
-#define SLOW_CLOCK_FREQ		32768
-#define SLOWCK_SW_CYCLES	5
-#define SLOWCK_SW_TIME_USEC	((SLOWCK_SW_CYCLES * USEC_PER_SEC) / \
-				 SLOW_CLOCK_FREQ)
-
-#define	AT91_SCKC_CR			0x00
-#define		AT91_SCKC_RCEN		(1 << 0)
-#define		AT91_SCKC_OSC32EN	(1 << 1)
-#define		AT91_SCKC_OSC32BYP	(1 << 2)
-#define		AT91_SCKC_OSCSEL	(1 << 3)
-
-struct clk_slow_osc {
-	struct clk_hw hw;
-	void __iomem *sckcr;
-	unsigned long startup_usec;
-};
-
-#define to_clk_slow_osc(hw) container_of(hw, struct clk_slow_osc, hw)
-
-struct clk_slow_rc_osc {
-	struct clk_hw hw;
-	void __iomem *sckcr;
-	unsigned long frequency;
-	unsigned long accuracy;
-	unsigned long startup_usec;
-};
-
-#define to_clk_slow_rc_osc(hw) container_of(hw, struct clk_slow_rc_osc, hw)
 
 struct clk_sam9260_slow {
 	struct clk_hw hw;
@@ -57,328 +26,6 @@ struct clk_sam9260_slow {
 
 #define to_clk_sam9260_slow(hw) container_of(hw, struct clk_sam9260_slow, hw)
 
-struct clk_sam9x5_slow {
-	struct clk_hw hw;
-	void __iomem *sckcr;
-	u8 parent;
-};
-
-#define to_clk_sam9x5_slow(hw) container_of(hw, struct clk_sam9x5_slow, hw)
-
-static int clk_slow_osc_prepare(struct clk_hw *hw)
-{
-	struct clk_slow_osc *osc = to_clk_slow_osc(hw);
-	void __iomem *sckcr = osc->sckcr;
-	u32 tmp = readl(sckcr);
-
-	if (tmp & AT91_SCKC_OSC32BYP)
-		return 0;
-
-	writel(tmp | AT91_SCKC_OSC32EN, sckcr);
-
-	usleep_range(osc->startup_usec, osc->startup_usec + 1);
-
-	return 0;
-}
-
-static void clk_slow_osc_unprepare(struct clk_hw *hw)
-{
-	struct clk_slow_osc *osc = to_clk_slow_osc(hw);
-	void __iomem *sckcr = osc->sckcr;
-	u32 tmp = readl(sckcr);
-
-	if (tmp & AT91_SCKC_OSC32BYP)
-		return;
-
-	writel(tmp & ~AT91_SCKC_OSC32EN, sckcr);
-}
-
-static int clk_slow_osc_is_prepared(struct clk_hw *hw)
-{
-	struct clk_slow_osc *osc = to_clk_slow_osc(hw);
-	void __iomem *sckcr = osc->sckcr;
-	u32 tmp = readl(sckcr);
-
-	if (tmp & AT91_SCKC_OSC32BYP)
-		return 1;
-
-	return !!(tmp & AT91_SCKC_OSC32EN);
-}
-
-static const struct clk_ops slow_osc_ops = {
-	.prepare = clk_slow_osc_prepare,
-	.unprepare = clk_slow_osc_unprepare,
-	.is_prepared = clk_slow_osc_is_prepared,
-};
-
-static struct clk * __init
-at91_clk_register_slow_osc(void __iomem *sckcr,
-			   const char *name,
-			   const char *parent_name,
-			   unsigned long startup,
-			   bool bypass)
-{
-	struct clk_slow_osc *osc;
-	struct clk *clk = NULL;
-	struct clk_init_data init;
-
-	if (!sckcr || !name || !parent_name)
-		return ERR_PTR(-EINVAL);
-
-	osc = kzalloc(sizeof(*osc), GFP_KERNEL);
-	if (!osc)
-		return ERR_PTR(-ENOMEM);
-
-	init.name = name;
-	init.ops = &slow_osc_ops;
-	init.parent_names = &parent_name;
-	init.num_parents = 1;
-	init.flags = CLK_IGNORE_UNUSED;
-
-	osc->hw.init = &init;
-	osc->sckcr = sckcr;
-	osc->startup_usec = startup;
-
-	if (bypass)
-		writel((readl(sckcr) & ~AT91_SCKC_OSC32EN) | AT91_SCKC_OSC32BYP,
-		       sckcr);
-
-	clk = clk_register(NULL, &osc->hw);
-	if (IS_ERR(clk))
-		kfree(osc);
-
-	return clk;
-}
-
-void __init of_at91sam9x5_clk_slow_osc_setup(struct device_node *np,
-					     void __iomem *sckcr)
-{
-	struct clk *clk;
-	const char *parent_name;
-	const char *name = np->name;
-	u32 startup;
-	bool bypass;
-
-	parent_name = of_clk_get_parent_name(np, 0);
-	of_property_read_string(np, "clock-output-names", &name);
-	of_property_read_u32(np, "atmel,startup-time-usec", &startup);
-	bypass = of_property_read_bool(np, "atmel,osc-bypass");
-
-	clk = at91_clk_register_slow_osc(sckcr, name, parent_name, startup,
-					 bypass);
-	if (IS_ERR(clk))
-		return;
-
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
-}
-
-static unsigned long clk_slow_rc_osc_recalc_rate(struct clk_hw *hw,
-						 unsigned long parent_rate)
-{
-	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
-
-	return osc->frequency;
-}
-
-static unsigned long clk_slow_rc_osc_recalc_accuracy(struct clk_hw *hw,
-						     unsigned long parent_acc)
-{
-	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
-
-	return osc->accuracy;
-}
-
-static int clk_slow_rc_osc_prepare(struct clk_hw *hw)
-{
-	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
-	void __iomem *sckcr = osc->sckcr;
-
-	writel(readl(sckcr) | AT91_SCKC_RCEN, sckcr);
-
-	usleep_range(osc->startup_usec, osc->startup_usec + 1);
-
-	return 0;
-}
-
-static void clk_slow_rc_osc_unprepare(struct clk_hw *hw)
-{
-	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
-	void __iomem *sckcr = osc->sckcr;
-
-	writel(readl(sckcr) & ~AT91_SCKC_RCEN, sckcr);
-}
-
-static int clk_slow_rc_osc_is_prepared(struct clk_hw *hw)
-{
-	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
-
-	return !!(readl(osc->sckcr) & AT91_SCKC_RCEN);
-}
-
-static const struct clk_ops slow_rc_osc_ops = {
-	.prepare = clk_slow_rc_osc_prepare,
-	.unprepare = clk_slow_rc_osc_unprepare,
-	.is_prepared = clk_slow_rc_osc_is_prepared,
-	.recalc_rate = clk_slow_rc_osc_recalc_rate,
-	.recalc_accuracy = clk_slow_rc_osc_recalc_accuracy,
-};
-
-static struct clk * __init
-at91_clk_register_slow_rc_osc(void __iomem *sckcr,
-			      const char *name,
-			      unsigned long frequency,
-			      unsigned long accuracy,
-			      unsigned long startup)
-{
-	struct clk_slow_rc_osc *osc;
-	struct clk *clk = NULL;
-	struct clk_init_data init;
-
-	if (!sckcr || !name)
-		return ERR_PTR(-EINVAL);
-
-	osc = kzalloc(sizeof(*osc), GFP_KERNEL);
-	if (!osc)
-		return ERR_PTR(-ENOMEM);
-
-	init.name = name;
-	init.ops = &slow_rc_osc_ops;
-	init.parent_names = NULL;
-	init.num_parents = 0;
-	init.flags = CLK_IGNORE_UNUSED;
-
-	osc->hw.init = &init;
-	osc->sckcr = sckcr;
-	osc->frequency = frequency;
-	osc->accuracy = accuracy;
-	osc->startup_usec = startup;
-
-	clk = clk_register(NULL, &osc->hw);
-	if (IS_ERR(clk))
-		kfree(osc);
-
-	return clk;
-}
-
-void __init of_at91sam9x5_clk_slow_rc_osc_setup(struct device_node *np,
-						void __iomem *sckcr)
-{
-	struct clk *clk;
-	u32 frequency = 0;
-	u32 accuracy = 0;
-	u32 startup = 0;
-	const char *name = np->name;
-
-	of_property_read_string(np, "clock-output-names", &name);
-	of_property_read_u32(np, "clock-frequency", &frequency);
-	of_property_read_u32(np, "clock-accuracy", &accuracy);
-	of_property_read_u32(np, "atmel,startup-time-usec", &startup);
-
-	clk = at91_clk_register_slow_rc_osc(sckcr, name, frequency, accuracy,
-					    startup);
-	if (IS_ERR(clk))
-		return;
-
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
-}
-
-static int clk_sam9x5_slow_set_parent(struct clk_hw *hw, u8 index)
-{
-	struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw);
-	void __iomem *sckcr = slowck->sckcr;
-	u32 tmp;
-
-	if (index > 1)
-		return -EINVAL;
-
-	tmp = readl(sckcr);
-
-	if ((!index && !(tmp & AT91_SCKC_OSCSEL)) ||
-	    (index && (tmp & AT91_SCKC_OSCSEL)))
-		return 0;
-
-	if (index)
-		tmp |= AT91_SCKC_OSCSEL;
-	else
-		tmp &= ~AT91_SCKC_OSCSEL;
-
-	writel(tmp, sckcr);
-
-	usleep_range(SLOWCK_SW_TIME_USEC, SLOWCK_SW_TIME_USEC + 1);
-
-	return 0;
-}
-
-static u8 clk_sam9x5_slow_get_parent(struct clk_hw *hw)
-{
-	struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw);
-
-	return !!(readl(slowck->sckcr) & AT91_SCKC_OSCSEL);
-}
-
-static const struct clk_ops sam9x5_slow_ops = {
-	.set_parent = clk_sam9x5_slow_set_parent,
-	.get_parent = clk_sam9x5_slow_get_parent,
-};
-
-static struct clk * __init
-at91_clk_register_sam9x5_slow(void __iomem *sckcr,
-			      const char *name,
-			      const char **parent_names,
-			      int num_parents)
-{
-	struct clk_sam9x5_slow *slowck;
-	struct clk *clk = NULL;
-	struct clk_init_data init;
-
-	if (!sckcr || !name || !parent_names || !num_parents)
-		return ERR_PTR(-EINVAL);
-
-	slowck = kzalloc(sizeof(*slowck), GFP_KERNEL);
-	if (!slowck)
-		return ERR_PTR(-ENOMEM);
-
-	init.name = name;
-	init.ops = &sam9x5_slow_ops;
-	init.parent_names = parent_names;
-	init.num_parents = num_parents;
-	init.flags = 0;
-
-	slowck->hw.init = &init;
-	slowck->sckcr = sckcr;
-	slowck->parent = !!(readl(sckcr) & AT91_SCKC_OSCSEL);
-
-	clk = clk_register(NULL, &slowck->hw);
-	if (IS_ERR(clk))
-		kfree(slowck);
-
-	return clk;
-}
-
-void __init of_at91sam9x5_clk_slow_setup(struct device_node *np,
-					 void __iomem *sckcr)
-{
-	struct clk *clk;
-	const char *parent_names[2];
-	unsigned int num_parents;
-	const char *name = np->name;
-
-	num_parents = of_clk_get_parent_count(np);
-	if (num_parents == 0 || num_parents > 2)
-		return;
-
-	of_clk_parent_fill(np, parent_names, num_parents);
-
-	of_property_read_string(np, "clock-output-names", &name);
-
-	clk = at91_clk_register_sam9x5_slow(sckcr, name, parent_names,
-					    num_parents);
-	if (IS_ERR(clk))
-		return;
-
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
-}
-
 static u8 clk_sam9260_slow_get_parent(struct clk_hw *hw)
 {
 	struct clk_sam9260_slow *slowck = to_clk_sam9260_slow(hw);
@@ -393,15 +40,16 @@ static const struct clk_ops sam9260_slow_ops = {
 	.get_parent = clk_sam9260_slow_get_parent,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_sam9260_slow(struct regmap *regmap,
 			       const char *name,
 			       const char **parent_names,
 			       int num_parents)
 {
 	struct clk_sam9260_slow *slowck;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	if (!name)
 		return ERR_PTR(-EINVAL);
@@ -422,16 +70,19 @@ at91_clk_register_sam9260_slow(struct regmap *regmap,
 	slowck->hw.init = &init;
 	slowck->regmap = regmap;
 
-	clk = clk_register(NULL, &slowck->hw);
-	if (IS_ERR(clk))
+	hw = &slowck->hw;
+	ret = clk_hw_register(NULL, &slowck->hw);
+	if (ret) {
 		kfree(slowck);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init of_at91sam9260_clk_slow_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *parent_names[2];
 	unsigned int num_parents;
 	const char *name = np->name;
@@ -448,12 +99,12 @@ static void __init of_at91sam9260_clk_slow_setup(struct device_node *np)
 
 	of_property_read_string(np, "clock-output-names", &name);
 
-	clk = at91_clk_register_sam9260_slow(regmap, name, parent_names,
+	hw = at91_clk_register_sam9260_slow(regmap, name, parent_names,
 					     num_parents);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 
 CLK_OF_DECLARE(at91sam9260_clk_slow, "atmel,at91sam9260-clk-slow",
diff --git a/drivers/clk/at91/clk-smd.c b/drivers/clk/at91/clk-smd.c
index 3c04b069d5b8..965c662b90a5 100644
--- a/drivers/clk/at91/clk-smd.c
+++ b/drivers/clk/at91/clk-smd.c
@@ -111,13 +111,14 @@ static const struct clk_ops at91sam9x5_smd_ops = {
 	.set_rate = at91sam9x5_clk_smd_set_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91sam9x5_clk_register_smd(struct regmap *regmap, const char *name,
 			    const char **parent_names, u8 num_parents)
 {
 	struct at91sam9x5_clk_smd *smd;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	smd = kzalloc(sizeof(*smd), GFP_KERNEL);
 	if (!smd)
@@ -132,16 +133,19 @@ at91sam9x5_clk_register_smd(struct regmap *regmap, const char *name,
 	smd->hw.init = &init;
 	smd->regmap = regmap;
 
-	clk = clk_register(NULL, &smd->hw);
-	if (IS_ERR(clk))
+	hw = &smd->hw;
+	ret = clk_hw_register(NULL, &smd->hw);
+	if (ret) {
 		kfree(smd);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init of_at91sam9x5_clk_smd_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	unsigned int num_parents;
 	const char *parent_names[SMD_SOURCE_MAX];
 	const char *name = np->name;
@@ -159,12 +163,12 @@ static void __init of_at91sam9x5_clk_smd_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	clk = at91sam9x5_clk_register_smd(regmap, name, parent_names,
+	hw = at91sam9x5_clk_register_smd(regmap, name, parent_names,
 					  num_parents);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(at91sam9x5_clk_smd, "atmel,at91sam9x5-clk-smd",
 	       of_at91sam9x5_clk_smd_setup);
diff --git a/drivers/clk/at91/clk-system.c b/drivers/clk/at91/clk-system.c
index 8f35d8172909..86a36809765d 100644
--- a/drivers/clk/at91/clk-system.c
+++ b/drivers/clk/at91/clk-system.c
@@ -88,13 +88,14 @@ static const struct clk_ops system_ops = {
 	.is_prepared = clk_system_is_prepared,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_system(struct regmap *regmap, const char *name,
 			 const char *parent_name, u8 id)
 {
 	struct clk_system *sys;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	if (!parent_name || id > SYSTEM_MAX_ID)
 		return ERR_PTR(-EINVAL);
@@ -113,18 +114,21 @@ at91_clk_register_system(struct regmap *regmap, const char *name,
 	sys->hw.init = &init;
 	sys->regmap = regmap;
 
-	clk = clk_register(NULL, &sys->hw);
-	if (IS_ERR(clk))
+	hw = &sys->hw;
+	ret = clk_hw_register(NULL, &sys->hw);
+	if (ret) {
 		kfree(sys);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init of_at91rm9200_clk_sys_setup(struct device_node *np)
 {
 	int num;
 	u32 id;
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *name;
 	struct device_node *sysclknp;
 	const char *parent_name;
@@ -147,11 +151,11 @@ static void __init of_at91rm9200_clk_sys_setup(struct device_node *np)
 
 		parent_name = of_clk_get_parent_name(sysclknp, 0);
 
-		clk = at91_clk_register_system(regmap, name, parent_name, id);
-		if (IS_ERR(clk))
+		hw = at91_clk_register_system(regmap, name, parent_name, id);
+		if (IS_ERR(hw))
 			continue;
 
-		of_clk_add_provider(sysclknp, of_clk_src_simple_get, clk);
+		of_clk_add_hw_provider(sysclknp, of_clk_hw_simple_get, hw);
 	}
 }
 CLK_OF_DECLARE(at91rm9200_clk_sys, "atmel,at91rm9200-clk-system",
diff --git a/drivers/clk/at91/clk-usb.c b/drivers/clk/at91/clk-usb.c
index d80bdb0a8b02..791770a563fc 100644
--- a/drivers/clk/at91/clk-usb.c
+++ b/drivers/clk/at91/clk-usb.c
@@ -192,13 +192,14 @@ static const struct clk_ops at91sam9n12_usb_ops = {
 	.set_rate = at91sam9x5_clk_usb_set_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91sam9x5_clk_register_usb(struct regmap *regmap, const char *name,
 			    const char **parent_names, u8 num_parents)
 {
 	struct at91sam9x5_clk_usb *usb;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	usb = kzalloc(sizeof(*usb), GFP_KERNEL);
 	if (!usb)
@@ -214,20 +215,24 @@ at91sam9x5_clk_register_usb(struct regmap *regmap, const char *name,
 	usb->hw.init = &init;
 	usb->regmap = regmap;
 
-	clk = clk_register(NULL, &usb->hw);
-	if (IS_ERR(clk))
+	hw = &usb->hw;
+	ret = clk_hw_register(NULL, &usb->hw);
+	if (ret) {
 		kfree(usb);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
-static struct clk * __init
+static struct clk_hw * __init
 at91sam9n12_clk_register_usb(struct regmap *regmap, const char *name,
 			     const char *parent_name)
 {
 	struct at91sam9x5_clk_usb *usb;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	usb = kzalloc(sizeof(*usb), GFP_KERNEL);
 	if (!usb)
@@ -242,11 +247,14 @@ at91sam9n12_clk_register_usb(struct regmap *regmap, const char *name,
 	usb->hw.init = &init;
 	usb->regmap = regmap;
 
-	clk = clk_register(NULL, &usb->hw);
-	if (IS_ERR(clk))
+	hw = &usb->hw;
+	ret = clk_hw_register(NULL, &usb->hw);
+	if (ret) {
 		kfree(usb);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static unsigned long at91rm9200_clk_usb_recalc_rate(struct clk_hw *hw,
@@ -334,13 +342,14 @@ static const struct clk_ops at91rm9200_usb_ops = {
 	.set_rate = at91rm9200_clk_usb_set_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91rm9200_clk_register_usb(struct regmap *regmap, const char *name,
 			    const char *parent_name, const u32 *divisors)
 {
 	struct at91rm9200_clk_usb *usb;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	usb = kzalloc(sizeof(*usb), GFP_KERNEL);
 	if (!usb)
@@ -356,16 +365,19 @@ at91rm9200_clk_register_usb(struct regmap *regmap, const char *name,
 	usb->regmap = regmap;
 	memcpy(usb->divisors, divisors, sizeof(usb->divisors));
 
-	clk = clk_register(NULL, &usb->hw);
-	if (IS_ERR(clk))
+	hw = &usb->hw;
+	ret = clk_hw_register(NULL, &usb->hw);
+	if (ret) {
 		kfree(usb);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init of_at91sam9x5_clk_usb_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	unsigned int num_parents;
 	const char *parent_names[USB_SOURCE_MAX];
 	const char *name = np->name;
@@ -383,19 +395,19 @@ static void __init of_at91sam9x5_clk_usb_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	clk = at91sam9x5_clk_register_usb(regmap, name, parent_names,
-					  num_parents);
-	if (IS_ERR(clk))
+	hw = at91sam9x5_clk_register_usb(regmap, name, parent_names,
+					 num_parents);
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(at91sam9x5_clk_usb, "atmel,at91sam9x5-clk-usb",
 	       of_at91sam9x5_clk_usb_setup);
 
 static void __init of_at91sam9n12_clk_usb_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *parent_name;
 	const char *name = np->name;
 	struct regmap *regmap;
@@ -410,18 +422,18 @@ static void __init of_at91sam9n12_clk_usb_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	clk = at91sam9n12_clk_register_usb(regmap, name, parent_name);
-	if (IS_ERR(clk))
+	hw = at91sam9n12_clk_register_usb(regmap, name, parent_name);
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(at91sam9n12_clk_usb, "atmel,at91sam9n12-clk-usb",
 	       of_at91sam9n12_clk_usb_setup);
 
 static void __init of_at91rm9200_clk_usb_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *parent_name;
 	const char *name = np->name;
 	u32 divisors[4] = {0, 0, 0, 0};
@@ -440,12 +452,11 @@ static void __init of_at91rm9200_clk_usb_setup(struct device_node *np)
 	regmap = syscon_node_to_regmap(of_get_parent(np));
 	if (IS_ERR(regmap))
 		return;
-
-	clk = at91rm9200_clk_register_usb(regmap, name, parent_name, divisors);
-	if (IS_ERR(clk))
+	hw = at91rm9200_clk_register_usb(regmap, name, parent_name, divisors);
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(at91rm9200_clk_usb, "atmel,at91rm9200-clk-usb",
 	       of_at91rm9200_clk_usb_setup);
diff --git a/drivers/clk/at91/clk-utmi.c b/drivers/clk/at91/clk-utmi.c
index 61fcf399e58c..aadabd9d1e2b 100644
--- a/drivers/clk/at91/clk-utmi.c
+++ b/drivers/clk/at91/clk-utmi.c
@@ -77,13 +77,14 @@ static const struct clk_ops utmi_ops = {
 	.recalc_rate = clk_utmi_recalc_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 at91_clk_register_utmi(struct regmap *regmap,
 		       const char *name, const char *parent_name)
 {
 	struct clk_utmi *utmi;
-	struct clk *clk = NULL;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	utmi = kzalloc(sizeof(*utmi), GFP_KERNEL);
 	if (!utmi)
@@ -98,16 +99,19 @@ at91_clk_register_utmi(struct regmap *regmap,
 	utmi->hw.init = &init;
 	utmi->regmap = regmap;
 
-	clk = clk_register(NULL, &utmi->hw);
-	if (IS_ERR(clk))
+	hw = &utmi->hw;
+	ret = clk_hw_register(NULL, &utmi->hw);
+	if (ret) {
 		kfree(utmi);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
 static void __init of_at91sam9x5_clk_utmi_setup(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *parent_name;
 	const char *name = np->name;
 	struct regmap *regmap;
@@ -120,11 +124,11 @@ static void __init of_at91sam9x5_clk_utmi_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	clk = at91_clk_register_utmi(regmap, name, parent_name);
-	if (IS_ERR(clk))
+	hw = at91_clk_register_utmi(regmap, name, parent_name);
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
 	return;
 }
 CLK_OF_DECLARE(at91sam9x5_clk_utmi, "atmel,at91sam9x5-clk-utmi",
diff --git a/drivers/clk/at91/sckc.c b/drivers/clk/at91/sckc.c
index 1184d76a7ab7..ab6ecefc49ad 100644
--- a/drivers/clk/at91/sckc.c
+++ b/drivers/clk/at91/sckc.c
@@ -12,11 +12,382 @@
 
 #include <linux/clk-provider.h>
 #include <linux/clkdev.h>
+#include <linux/delay.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/io.h>
 
-#include "sckc.h"
+#define SLOW_CLOCK_FREQ		32768
+#define SLOWCK_SW_CYCLES	5
+#define SLOWCK_SW_TIME_USEC	((SLOWCK_SW_CYCLES * USEC_PER_SEC) / \
+				 SLOW_CLOCK_FREQ)
+
+#define	AT91_SCKC_CR			0x00
+#define		AT91_SCKC_RCEN		(1 << 0)
+#define		AT91_SCKC_OSC32EN	(1 << 1)
+#define		AT91_SCKC_OSC32BYP	(1 << 2)
+#define		AT91_SCKC_OSCSEL	(1 << 3)
+
+struct clk_slow_osc {
+	struct clk_hw hw;
+	void __iomem *sckcr;
+	unsigned long startup_usec;
+};
+
+#define to_clk_slow_osc(hw) container_of(hw, struct clk_slow_osc, hw)
+
+struct clk_sama5d4_slow_osc {
+	struct clk_hw hw;
+	void __iomem *sckcr;
+	unsigned long startup_usec;
+	bool prepared;
+};
+
+#define to_clk_sama5d4_slow_osc(hw) container_of(hw, struct clk_sama5d4_slow_osc, hw)
+
+struct clk_slow_rc_osc {
+	struct clk_hw hw;
+	void __iomem *sckcr;
+	unsigned long frequency;
+	unsigned long accuracy;
+	unsigned long startup_usec;
+};
+
+#define to_clk_slow_rc_osc(hw) container_of(hw, struct clk_slow_rc_osc, hw)
+
+struct clk_sam9x5_slow {
+	struct clk_hw hw;
+	void __iomem *sckcr;
+	u8 parent;
+};
+
+#define to_clk_sam9x5_slow(hw) container_of(hw, struct clk_sam9x5_slow, hw)
+
+static int clk_slow_osc_prepare(struct clk_hw *hw)
+{
+	struct clk_slow_osc *osc = to_clk_slow_osc(hw);
+	void __iomem *sckcr = osc->sckcr;
+	u32 tmp = readl(sckcr);
+
+	if (tmp & (AT91_SCKC_OSC32BYP | AT91_SCKC_OSC32EN))
+		return 0;
+
+	writel(tmp | AT91_SCKC_OSC32EN, sckcr);
+
+	usleep_range(osc->startup_usec, osc->startup_usec + 1);
+
+	return 0;
+}
+
+static void clk_slow_osc_unprepare(struct clk_hw *hw)
+{
+	struct clk_slow_osc *osc = to_clk_slow_osc(hw);
+	void __iomem *sckcr = osc->sckcr;
+	u32 tmp = readl(sckcr);
+
+	if (tmp & AT91_SCKC_OSC32BYP)
+		return;
+
+	writel(tmp & ~AT91_SCKC_OSC32EN, sckcr);
+}
+
+static int clk_slow_osc_is_prepared(struct clk_hw *hw)
+{
+	struct clk_slow_osc *osc = to_clk_slow_osc(hw);
+	void __iomem *sckcr = osc->sckcr;
+	u32 tmp = readl(sckcr);
+
+	if (tmp & AT91_SCKC_OSC32BYP)
+		return 1;
+
+	return !!(tmp & AT91_SCKC_OSC32EN);
+}
+
+static const struct clk_ops slow_osc_ops = {
+	.prepare = clk_slow_osc_prepare,
+	.unprepare = clk_slow_osc_unprepare,
+	.is_prepared = clk_slow_osc_is_prepared,
+};
+
+static struct clk_hw * __init
+at91_clk_register_slow_osc(void __iomem *sckcr,
+			   const char *name,
+			   const char *parent_name,
+			   unsigned long startup,
+			   bool bypass)
+{
+	struct clk_slow_osc *osc;
+	struct clk_hw *hw;
+	struct clk_init_data init;
+	int ret;
+
+	if (!sckcr || !name || !parent_name)
+		return ERR_PTR(-EINVAL);
+
+	osc = kzalloc(sizeof(*osc), GFP_KERNEL);
+	if (!osc)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &slow_osc_ops;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+	init.flags = CLK_IGNORE_UNUSED;
+
+	osc->hw.init = &init;
+	osc->sckcr = sckcr;
+	osc->startup_usec = startup;
+
+	if (bypass)
+		writel((readl(sckcr) & ~AT91_SCKC_OSC32EN) | AT91_SCKC_OSC32BYP,
+		       sckcr);
+
+	hw = &osc->hw;
+	ret = clk_hw_register(NULL, &osc->hw);
+	if (ret) {
+		kfree(osc);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
+static void __init
+of_at91sam9x5_clk_slow_osc_setup(struct device_node *np, void __iomem *sckcr)
+{
+	struct clk_hw *hw;
+	const char *parent_name;
+	const char *name = np->name;
+	u32 startup;
+	bool bypass;
+
+	parent_name = of_clk_get_parent_name(np, 0);
+	of_property_read_string(np, "clock-output-names", &name);
+	of_property_read_u32(np, "atmel,startup-time-usec", &startup);
+	bypass = of_property_read_bool(np, "atmel,osc-bypass");
+
+	hw = at91_clk_register_slow_osc(sckcr, name, parent_name, startup,
+					 bypass);
+	if (IS_ERR(hw))
+		return;
+
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
+}
+
+static unsigned long clk_slow_rc_osc_recalc_rate(struct clk_hw *hw,
+						 unsigned long parent_rate)
+{
+	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
+
+	return osc->frequency;
+}
+
+static unsigned long clk_slow_rc_osc_recalc_accuracy(struct clk_hw *hw,
+						     unsigned long parent_acc)
+{
+	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
+
+	return osc->accuracy;
+}
+
+static int clk_slow_rc_osc_prepare(struct clk_hw *hw)
+{
+	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
+	void __iomem *sckcr = osc->sckcr;
+
+	writel(readl(sckcr) | AT91_SCKC_RCEN, sckcr);
+
+	usleep_range(osc->startup_usec, osc->startup_usec + 1);
+
+	return 0;
+}
+
+static void clk_slow_rc_osc_unprepare(struct clk_hw *hw)
+{
+	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
+	void __iomem *sckcr = osc->sckcr;
+
+	writel(readl(sckcr) & ~AT91_SCKC_RCEN, sckcr);
+}
+
+static int clk_slow_rc_osc_is_prepared(struct clk_hw *hw)
+{
+	struct clk_slow_rc_osc *osc = to_clk_slow_rc_osc(hw);
+
+	return !!(readl(osc->sckcr) & AT91_SCKC_RCEN);
+}
+
+static const struct clk_ops slow_rc_osc_ops = {
+	.prepare = clk_slow_rc_osc_prepare,
+	.unprepare = clk_slow_rc_osc_unprepare,
+	.is_prepared = clk_slow_rc_osc_is_prepared,
+	.recalc_rate = clk_slow_rc_osc_recalc_rate,
+	.recalc_accuracy = clk_slow_rc_osc_recalc_accuracy,
+};
+
+static struct clk_hw * __init
+at91_clk_register_slow_rc_osc(void __iomem *sckcr,
+			      const char *name,
+			      unsigned long frequency,
+			      unsigned long accuracy,
+			      unsigned long startup)
+{
+	struct clk_slow_rc_osc *osc;
+	struct clk_hw *hw;
+	struct clk_init_data init;
+	int ret;
+
+	if (!sckcr || !name)
+		return ERR_PTR(-EINVAL);
+
+	osc = kzalloc(sizeof(*osc), GFP_KERNEL);
+	if (!osc)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &slow_rc_osc_ops;
+	init.parent_names = NULL;
+	init.num_parents = 0;
+	init.flags = CLK_IGNORE_UNUSED;
+
+	osc->hw.init = &init;
+	osc->sckcr = sckcr;
+	osc->frequency = frequency;
+	osc->accuracy = accuracy;
+	osc->startup_usec = startup;
+
+	hw = &osc->hw;
+	ret = clk_hw_register(NULL, &osc->hw);
+	if (ret) {
+		kfree(osc);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
+static void __init
+of_at91sam9x5_clk_slow_rc_osc_setup(struct device_node *np, void __iomem *sckcr)
+{
+	struct clk_hw *hw;
+	u32 frequency = 0;
+	u32 accuracy = 0;
+	u32 startup = 0;
+	const char *name = np->name;
+
+	of_property_read_string(np, "clock-output-names", &name);
+	of_property_read_u32(np, "clock-frequency", &frequency);
+	of_property_read_u32(np, "clock-accuracy", &accuracy);
+	of_property_read_u32(np, "atmel,startup-time-usec", &startup);
+
+	hw = at91_clk_register_slow_rc_osc(sckcr, name, frequency, accuracy,
+					    startup);
+	if (IS_ERR(hw))
+		return;
+
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
+}
+
+static int clk_sam9x5_slow_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw);
+	void __iomem *sckcr = slowck->sckcr;
+	u32 tmp;
+
+	if (index > 1)
+		return -EINVAL;
+
+	tmp = readl(sckcr);
+
+	if ((!index && !(tmp & AT91_SCKC_OSCSEL)) ||
+	    (index && (tmp & AT91_SCKC_OSCSEL)))
+		return 0;
+
+	if (index)
+		tmp |= AT91_SCKC_OSCSEL;
+	else
+		tmp &= ~AT91_SCKC_OSCSEL;
+
+	writel(tmp, sckcr);
+
+	usleep_range(SLOWCK_SW_TIME_USEC, SLOWCK_SW_TIME_USEC + 1);
+
+	return 0;
+}
+
+static u8 clk_sam9x5_slow_get_parent(struct clk_hw *hw)
+{
+	struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw);
+
+	return !!(readl(slowck->sckcr) & AT91_SCKC_OSCSEL);
+}
+
+static const struct clk_ops sam9x5_slow_ops = {
+	.set_parent = clk_sam9x5_slow_set_parent,
+	.get_parent = clk_sam9x5_slow_get_parent,
+};
+
+static struct clk_hw * __init
+at91_clk_register_sam9x5_slow(void __iomem *sckcr,
+			      const char *name,
+			      const char **parent_names,
+			      int num_parents)
+{
+	struct clk_sam9x5_slow *slowck;
+	struct clk_hw *hw;
+	struct clk_init_data init;
+	int ret;
+
+	if (!sckcr || !name || !parent_names || !num_parents)
+		return ERR_PTR(-EINVAL);
+
+	slowck = kzalloc(sizeof(*slowck), GFP_KERNEL);
+	if (!slowck)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &sam9x5_slow_ops;
+	init.parent_names = parent_names;
+	init.num_parents = num_parents;
+	init.flags = 0;
+
+	slowck->hw.init = &init;
+	slowck->sckcr = sckcr;
+	slowck->parent = !!(readl(sckcr) & AT91_SCKC_OSCSEL);
+
+	hw = &slowck->hw;
+	ret = clk_hw_register(NULL, &slowck->hw);
+	if (ret) {
+		kfree(slowck);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
+static void __init
+of_at91sam9x5_clk_slow_setup(struct device_node *np, void __iomem *sckcr)
+{
+	struct clk_hw *hw;
+	const char *parent_names[2];
+	unsigned int num_parents;
+	const char *name = np->name;
+
+	num_parents = of_clk_get_parent_count(np);
+	if (num_parents == 0 || num_parents > 2)
+		return;
+
+	of_clk_parent_fill(np, parent_names, num_parents);
+
+	of_property_read_string(np, "clock-output-names", &name);
+
+	hw = at91_clk_register_sam9x5_slow(sckcr, name, parent_names,
+					    num_parents);
+	if (IS_ERR(hw))
+		return;
+
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
+}
 
 static const struct of_device_id sckc_clk_ids[] __initconst = {
 	/* Slow clock */
@@ -55,3 +426,94 @@ static void __init of_at91sam9x5_sckc_setup(struct device_node *np)
 }
 CLK_OF_DECLARE(at91sam9x5_clk_sckc, "atmel,at91sam9x5-sckc",
 	       of_at91sam9x5_sckc_setup);
+
+static int clk_sama5d4_slow_osc_prepare(struct clk_hw *hw)
+{
+	struct clk_sama5d4_slow_osc *osc = to_clk_sama5d4_slow_osc(hw);
+
+	if (osc->prepared)
+		return 0;
+
+	/*
+	 * Assume that if it has already been selected (for example by the
+	 * bootloader), enough time has aready passed.
+	 */
+	if ((readl(osc->sckcr) & AT91_SCKC_OSCSEL)) {
+		osc->prepared = true;
+		return 0;
+	}
+
+	usleep_range(osc->startup_usec, osc->startup_usec + 1);
+	osc->prepared = true;
+
+	return 0;
+}
+
+static int clk_sama5d4_slow_osc_is_prepared(struct clk_hw *hw)
+{
+	struct clk_sama5d4_slow_osc *osc = to_clk_sama5d4_slow_osc(hw);
+
+	return osc->prepared;
+}
+
+static const struct clk_ops sama5d4_slow_osc_ops = {
+	.prepare = clk_sama5d4_slow_osc_prepare,
+	.is_prepared = clk_sama5d4_slow_osc_is_prepared,
+};
+
+static void __init of_sama5d4_sckc_setup(struct device_node *np)
+{
+	void __iomem *regbase = of_iomap(np, 0);
+	struct clk_hw *hw;
+	struct clk_sama5d4_slow_osc *osc;
+	struct clk_init_data init;
+	const char *xtal_name;
+	const char *parent_names[2] = { "slow_rc_osc", "slow_osc" };
+	bool bypass;
+	int ret;
+
+	if (!regbase)
+		return;
+
+	hw = clk_hw_register_fixed_rate_with_accuracy(NULL, parent_names[0],
+						      NULL, 0, 32768,
+						      250000000);
+	if (IS_ERR(hw))
+		return;
+
+	xtal_name = of_clk_get_parent_name(np, 0);
+
+	bypass = of_property_read_bool(np, "atmel,osc-bypass");
+
+	osc = kzalloc(sizeof(*osc), GFP_KERNEL);
+	if (!osc)
+		return;
+
+	init.name = parent_names[1];
+	init.ops = &sama5d4_slow_osc_ops;
+	init.parent_names = &xtal_name;
+	init.num_parents = 1;
+	init.flags = CLK_IGNORE_UNUSED;
+
+	osc->hw.init = &init;
+	osc->sckcr = regbase;
+	osc->startup_usec = 1200000;
+
+	if (bypass)
+		writel((readl(regbase) | AT91_SCKC_OSC32BYP), regbase);
+
+	hw = &osc->hw;
+	ret = clk_hw_register(NULL, &osc->hw);
+	if (ret) {
+		kfree(osc);
+		return;
+	}
+
+	hw = at91_clk_register_sam9x5_slow(regbase, "slowck", parent_names, 2);
+	if (IS_ERR(hw))
+		return;
+
+	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
+}
+CLK_OF_DECLARE(sama5d4_clk_sckc, "atmel,sama5d4-sckc",
+	       of_sama5d4_sckc_setup);
diff --git a/drivers/clk/at91/sckc.h b/drivers/clk/at91/sckc.h
deleted file mode 100644
index 836fcf59820f..000000000000
--- a/drivers/clk/at91/sckc.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * drivers/clk/at91/sckc.h
- *
- *  Copyright (C) 2013 Boris BREZILLON <b.brezillon@overkiz.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#ifndef __AT91_SCKC_H_
-#define __AT91_SCKC_H_
-
-extern void __init of_at91sam9x5_clk_slow_osc_setup(struct device_node *np,
-						    void __iomem *sckcr);
-extern void __init of_at91sam9x5_clk_slow_rc_osc_setup(struct device_node *np,
-						       void __iomem *sckcr);
-extern void __init of_at91sam9x5_clk_slow_setup(struct device_node *np,
-						void __iomem *sckcr);
-
-#endif /* __AT91_SCKC_H_ */
diff --git a/drivers/clk/axis/clk-artpec6.c b/drivers/clk/axis/clk-artpec6.c
index ffc988b098e4..da1a073c2236 100644
--- a/drivers/clk/axis/clk-artpec6.c
+++ b/drivers/clk/axis/clk-artpec6.c
@@ -113,8 +113,8 @@ static void of_artpec6_clkctrl_setup(struct device_node *np)
 	of_clk_add_provider(np, of_clk_src_onecell_get, &clkdata->clk_data);
 }
 
-CLK_OF_DECLARE(artpec6_clkctrl, "axis,artpec6-clkctrl",
-	       of_artpec6_clkctrl_setup);
+CLK_OF_DECLARE_DRIVER(artpec6_clkctrl, "axis,artpec6-clkctrl",
+		      of_artpec6_clkctrl_setup);
 
 static int artpec6_clkctrl_probe(struct platform_device *pdev)
 {
diff --git a/drivers/clk/bcm/Kconfig b/drivers/clk/bcm/Kconfig
index f2878459199a..f21e9b7afd1a 100644
--- a/drivers/clk/bcm/Kconfig
+++ b/drivers/clk/bcm/Kconfig
@@ -19,8 +19,36 @@ config CLK_BCM_KONA
 	  in the BCM281xx and BCM21664 families.
 
 config COMMON_CLK_IPROC
-	bool
+	bool "Broadcom iProc clock support"
+	depends on ARCH_BCM_IPROC || COMPILE_TEST
 	depends on COMMON_CLK
+	default ARCH_BCM_IPROC
 	help
 	  Enable common clock framework support for Broadcom SoCs
 	  based on the iProc architecture
+
+if COMMON_CLK_IPROC
+
+config CLK_BCM_CYGNUS
+	bool "Broadcom Cygnus clock support"
+	depends on ARCH_BCM_CYGNUS || COMPILE_TEST
+	default ARCH_BCM_CYGNUS
+	help
+	  Enable common clock framework support for the Broadcom Cygnus SoC
+
+config CLK_BCM_NSP
+	bool "Broadcom Northstar/Northstar Plus clock support"
+	depends on ARCH_BCM_5301X || ARCH_BCM_NSP || COMPILE_TEST
+	default ARCH_BCM_5301X || ARCH_BCM_NSP
+	help
+	  Enable common clock framework support for the Broadcom Northstar and
+	  Northstar Plus SoCs
+
+config CLK_BCM_NS2
+	bool "Broadcom Northstar 2 clock support"
+	depends on ARCH_BCM_IPROC || COMPILE_TEST
+	default ARCH_BCM_IPROC
+	help
+	  Enable common clock framework support for the Broadcom Northstar 2 SoC
+
+endif
diff --git a/drivers/clk/bcm/Makefile b/drivers/clk/bcm/Makefile
index 1d79bd2c36f0..d9dc848f18c9 100644
--- a/drivers/clk/bcm/Makefile
+++ b/drivers/clk/bcm/Makefile
@@ -6,7 +6,7 @@ obj-$(CONFIG_CLK_BCM_KONA)	+= clk-bcm21664.o
 obj-$(CONFIG_COMMON_CLK_IPROC)	+= clk-iproc-armpll.o clk-iproc-pll.o clk-iproc-asiu.o
 obj-$(CONFIG_ARCH_BCM2835)	+= clk-bcm2835.o
 obj-$(CONFIG_ARCH_BCM2835)	+= clk-bcm2835-aux.o
-obj-$(CONFIG_COMMON_CLK_IPROC)	+= clk-ns2.o
-obj-$(CONFIG_ARCH_BCM_CYGNUS)	+= clk-cygnus.o
-obj-$(CONFIG_ARCH_BCM_NSP)	+= clk-nsp.o
-obj-$(CONFIG_ARCH_BCM_5301X)	+= clk-nsp.o
+obj-$(CONFIG_ARCH_BCM_53573)	+= clk-bcm53573-ilp.o
+obj-$(CONFIG_CLK_BCM_CYGNUS)	+= clk-cygnus.o
+obj-$(CONFIG_CLK_BCM_NSP)	+= clk-nsp.o
+obj-$(CONFIG_CLK_BCM_NS2)	+= clk-ns2.o
diff --git a/drivers/clk/bcm/clk-bcm2835-aux.c b/drivers/clk/bcm/clk-bcm2835-aux.c
index 3a177ade6e6c..bd750cf2238d 100644
--- a/drivers/clk/bcm/clk-bcm2835-aux.c
+++ b/drivers/clk/bcm/clk-bcm2835-aux.c
@@ -25,7 +25,7 @@
 static int bcm2835_aux_clk_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct clk_onecell_data *onecell;
+	struct clk_hw_onecell_data *onecell;
 	const char *parent;
 	struct clk *parent_clk;
 	struct resource *res;
@@ -41,28 +41,24 @@ static int bcm2835_aux_clk_probe(struct platform_device *pdev)
 	if (IS_ERR(reg))
 		return PTR_ERR(reg);
 
-	onecell = devm_kmalloc(dev, sizeof(*onecell), GFP_KERNEL);
+	onecell = devm_kmalloc(dev, sizeof(*onecell) + sizeof(*onecell->hws) *
+			       BCM2835_AUX_CLOCK_COUNT, GFP_KERNEL);
 	if (!onecell)
 		return -ENOMEM;
-	onecell->clk_num = BCM2835_AUX_CLOCK_COUNT;
-	onecell->clks = devm_kcalloc(dev, BCM2835_AUX_CLOCK_COUNT,
-				     sizeof(*onecell->clks), GFP_KERNEL);
-	if (!onecell->clks)
-		return -ENOMEM;
+	onecell->num = BCM2835_AUX_CLOCK_COUNT;
 
 	gate = reg + BCM2835_AUXENB;
-	onecell->clks[BCM2835_AUX_CLOCK_UART] =
-		clk_register_gate(dev, "aux_uart", parent, 0, gate, 0, 0, NULL);
-
-	onecell->clks[BCM2835_AUX_CLOCK_SPI1] =
-		clk_register_gate(dev, "aux_spi1", parent, 0, gate, 1, 0, NULL);
+	onecell->hws[BCM2835_AUX_CLOCK_UART] =
+		clk_hw_register_gate(dev, "aux_uart", parent, 0, gate, 0, 0, NULL);
 
-	onecell->clks[BCM2835_AUX_CLOCK_SPI2] =
-		clk_register_gate(dev, "aux_spi2", parent, 0, gate, 2, 0, NULL);
+	onecell->hws[BCM2835_AUX_CLOCK_SPI1] =
+		clk_hw_register_gate(dev, "aux_spi1", parent, 0, gate, 1, 0, NULL);
 
-	of_clk_add_provider(pdev->dev.of_node, of_clk_src_onecell_get, onecell);
+	onecell->hws[BCM2835_AUX_CLOCK_SPI2] =
+		clk_hw_register_gate(dev, "aux_spi2", parent, 0, gate, 2, 0, NULL);
 
-	return 0;
+	return of_clk_add_hw_provider(pdev->dev.of_node, of_clk_hw_onecell_get,
+				      onecell);
 }
 
 static const struct of_device_id bcm2835_aux_clk_of_match[] = {
diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c
index 7a7970865c2d..b68bf573dcfb 100644
--- a/drivers/clk/bcm/clk-bcm2835.c
+++ b/drivers/clk/bcm/clk-bcm2835.c
@@ -36,6 +36,7 @@
 
 #include <linux/clk-provider.h>
 #include <linux/clkdev.h>
+#include <linux/clk.h>
 #include <linux/clk/bcm2835.h>
 #include <linux/debugfs.h>
 #include <linux/module.h>
@@ -302,8 +303,8 @@ struct bcm2835_cprman {
 	spinlock_t regs_lock; /* spinlock for all clocks */
 	const char *osc_name;
 
-	struct clk_onecell_data onecell;
-	struct clk *clks[];
+	/* Must be last */
+	struct clk_hw_onecell_data onecell;
 };
 
 static inline void cprman_write(struct bcm2835_cprman *cprman, u32 reg, u32 val)
@@ -344,24 +345,24 @@ static int bcm2835_debugfs_regset(struct bcm2835_cprman *cprman, u32 base,
  */
 void __init bcm2835_init_clocks(void)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	int ret;
 
-	clk = clk_register_fixed_rate(NULL, "apb_pclk", NULL, 0, 126000000);
-	if (IS_ERR(clk))
+	hw = clk_hw_register_fixed_rate(NULL, "apb_pclk", NULL, 0, 126000000);
+	if (IS_ERR(hw))
 		pr_err("apb_pclk not registered\n");
 
-	clk = clk_register_fixed_rate(NULL, "uart0_pclk", NULL, 0, 3000000);
-	if (IS_ERR(clk))
+	hw = clk_hw_register_fixed_rate(NULL, "uart0_pclk", NULL, 0, 3000000);
+	if (IS_ERR(hw))
 		pr_err("uart0_pclk not registered\n");
-	ret = clk_register_clkdev(clk, NULL, "20201000.uart");
+	ret = clk_hw_register_clkdev(hw, NULL, "20201000.uart");
 	if (ret)
 		pr_err("uart0_pclk alias not registered\n");
 
-	clk = clk_register_fixed_rate(NULL, "uart1_pclk", NULL, 0, 125000000);
-	if (IS_ERR(clk))
+	hw = clk_hw_register_fixed_rate(NULL, "uart1_pclk", NULL, 0, 125000000);
+	if (IS_ERR(hw))
 		pr_err("uart1_pclk not registered\n");
-	ret = clk_register_clkdev(clk, NULL, "20215000.uart");
+	ret = clk_hw_register_clkdev(hw, NULL, "20215000.uart");
 	if (ret)
 		pr_err("uart1_pclk alias not registered\n");
 }
@@ -443,6 +444,8 @@ struct bcm2835_clock_data {
 	/* Number of fractional bits in the divider */
 	u32 frac_bits;
 
+	u32 flags;
+
 	bool is_vpu_clock;
 	bool is_mash_clock;
 };
@@ -1006,16 +1009,28 @@ static int bcm2835_clock_set_rate(struct clk_hw *hw,
 	return 0;
 }
 
+static bool
+bcm2835_clk_is_pllc(struct clk_hw *hw)
+{
+	if (!hw)
+		return false;
+
+	return strncmp(clk_hw_get_name(hw), "pllc", 4) == 0;
+}
+
 static int bcm2835_clock_determine_rate(struct clk_hw *hw,
 					struct clk_rate_request *req)
 {
 	struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
 	struct clk_hw *parent, *best_parent = NULL;
+	bool current_parent_is_pllc;
 	unsigned long rate, best_rate = 0;
 	unsigned long prate, best_prate = 0;
 	size_t i;
 	u32 div;
 
+	current_parent_is_pllc = bcm2835_clk_is_pllc(clk_hw_get_parent(hw));
+
 	/*
 	 * Select parent clock that results in the closest but lower rate
 	 */
@@ -1023,6 +1038,17 @@ static int bcm2835_clock_determine_rate(struct clk_hw *hw,
 		parent = clk_hw_get_parent_by_index(hw, i);
 		if (!parent)
 			continue;
+
+		/*
+		 * Don't choose a PLLC-derived clock as our parent
+		 * unless it had been manually set that way.  PLLC's
+		 * frequency gets adjusted by the firmware due to
+		 * over-temp or under-voltage conditions, without
+		 * prior notification to our clock consumer.
+		 */
+		if (bcm2835_clk_is_pllc(parent) && !current_parent_is_pllc)
+			continue;
+
 		prate = clk_hw_get_rate(parent);
 		div = bcm2835_clock_choose_div(hw, req->rate, prate, true);
 		rate = bcm2835_clock_rate_from_divisor(clock, prate, div);
@@ -1121,11 +1147,12 @@ static const struct clk_ops bcm2835_vpu_clock_clk_ops = {
 	.debug_init = bcm2835_clock_debug_init,
 };
 
-static struct clk *bcm2835_register_pll(struct bcm2835_cprman *cprman,
-					const struct bcm2835_pll_data *data)
+static struct clk_hw *bcm2835_register_pll(struct bcm2835_cprman *cprman,
+					   const struct bcm2835_pll_data *data)
 {
 	struct bcm2835_pll *pll;
 	struct clk_init_data init;
+	int ret;
 
 	memset(&init, 0, sizeof(init));
 
@@ -1144,17 +1171,20 @@ static struct clk *bcm2835_register_pll(struct bcm2835_cprman *cprman,
 	pll->data = data;
 	pll->hw.init = &init;
 
-	return devm_clk_register(cprman->dev, &pll->hw);
+	ret = devm_clk_hw_register(cprman->dev, &pll->hw);
+	if (ret)
+		return NULL;
+	return &pll->hw;
 }
 
-static struct clk *
+static struct clk_hw *
 bcm2835_register_pll_divider(struct bcm2835_cprman *cprman,
 			     const struct bcm2835_pll_divider_data *data)
 {
 	struct bcm2835_pll_divider *divider;
 	struct clk_init_data init;
-	struct clk *clk;
 	const char *divider_name;
+	int ret;
 
 	if (data->fixed_divider != 1) {
 		divider_name = devm_kasprintf(cprman->dev, GFP_KERNEL,
@@ -1188,32 +1218,33 @@ bcm2835_register_pll_divider(struct bcm2835_cprman *cprman,
 	divider->cprman = cprman;
 	divider->data = data;
 
-	clk = devm_clk_register(cprman->dev, &divider->div.hw);
-	if (IS_ERR(clk))
-		return clk;
+	ret = devm_clk_hw_register(cprman->dev, &divider->div.hw);
+	if (ret)
+		return ERR_PTR(ret);
 
 	/*
 	 * PLLH's channels have a fixed divide by 10 afterwards, which
 	 * is what our consumers are actually using.
 	 */
 	if (data->fixed_divider != 1) {
-		return clk_register_fixed_factor(cprman->dev, data->name,
-						 divider_name,
-						 CLK_SET_RATE_PARENT,
-						 1,
-						 data->fixed_divider);
+		return clk_hw_register_fixed_factor(cprman->dev, data->name,
+						    divider_name,
+						    CLK_SET_RATE_PARENT,
+						    1,
+						    data->fixed_divider);
 	}
 
-	return clk;
+	return &divider->div.hw;
 }
 
-static struct clk *bcm2835_register_clock(struct bcm2835_cprman *cprman,
+static struct clk_hw *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 					  const struct bcm2835_clock_data *data)
 {
 	struct bcm2835_clock *clock;
 	struct clk_init_data init;
 	const char *parents[1 << CM_SRC_BITS];
 	size_t i;
+	int ret;
 
 	/*
 	 * Replace our "xosc" references with the oscillator's
@@ -1230,13 +1261,19 @@ static struct clk *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 	init.parent_names = parents;
 	init.num_parents = data->num_mux_parents;
 	init.name = data->name;
-	init.flags = CLK_IGNORE_UNUSED;
+	init.flags = data->flags | CLK_IGNORE_UNUSED;
 
 	if (data->is_vpu_clock) {
 		init.ops = &bcm2835_vpu_clock_clk_ops;
 	} else {
 		init.ops = &bcm2835_clock_clk_ops;
 		init.flags |= CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE;
+
+		/* If the clock wasn't actually enabled at boot, it's not
+		 * critical.
+		 */
+		if (!(cprman_read(cprman, data->ctl_reg) & CM_ENABLE))
+			init.flags &= ~CLK_IS_CRITICAL;
 	}
 
 	clock = devm_kzalloc(cprman->dev, sizeof(*clock), GFP_KERNEL);
@@ -1247,7 +1284,10 @@ static struct clk *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 	clock->data = data;
 	clock->hw.init = &init;
 
-	return devm_clk_register(cprman->dev, &clock->hw);
+	ret = devm_clk_hw_register(cprman->dev, &clock->hw);
+	if (ret)
+		return ERR_PTR(ret);
+	return &clock->hw;
 }
 
 static struct clk *bcm2835_register_gate(struct bcm2835_cprman *cprman,
@@ -1259,8 +1299,8 @@ static struct clk *bcm2835_register_gate(struct bcm2835_cprman *cprman,
 				 CM_GATE_BIT, 0, &cprman->regs_lock);
 }
 
-typedef struct clk *(*bcm2835_clk_register)(struct bcm2835_cprman *cprman,
-					    const void *data);
+typedef struct clk_hw *(*bcm2835_clk_register)(struct bcm2835_cprman *cprman,
+					       const void *data);
 struct bcm2835_clk_desc {
 	bcm2835_clk_register clk_register;
 	const void *data;
@@ -1649,6 +1689,7 @@ static const struct bcm2835_clk_desc clk_desc_array[] = {
 		.div_reg = CM_VPUDIV,
 		.int_bits = 12,
 		.frac_bits = 8,
+		.flags = CLK_IS_CRITICAL,
 		.is_vpu_clock = true),
 
 	/* clocks with per parent mux */
@@ -1705,13 +1746,15 @@ static const struct bcm2835_clk_desc clk_desc_array[] = {
 		.div_reg = CM_GP1DIV,
 		.int_bits = 12,
 		.frac_bits = 12,
+		.flags = CLK_IS_CRITICAL,
 		.is_mash_clock = true),
 	[BCM2835_CLOCK_GP2]	= REGISTER_PER_CLK(
 		.name = "gp2",
 		.ctl_reg = CM_GP2CTL,
 		.div_reg = CM_GP2DIV,
 		.int_bits = 12,
-		.frac_bits = 12),
+		.frac_bits = 12,
+		.flags = CLK_IS_CRITICAL),
 
 	/* HDMI state machine */
 	[BCM2835_CLOCK_HSM]	= REGISTER_PER_CLK(
@@ -1790,18 +1833,38 @@ static const struct bcm2835_clk_desc clk_desc_array[] = {
 		.ctl_reg = CM_PERIICTL),
 };
 
+/*
+ * Permanently take a reference on the parent of the SDRAM clock.
+ *
+ * While the SDRAM is being driven by its dedicated PLL most of the
+ * time, there is a little loop running in the firmware that
+ * periodically switches the SDRAM to using our CM clock to do PVT
+ * recalibration, with the assumption that the previously configured
+ * SDRAM parent is still enabled and running.
+ */
+static int bcm2835_mark_sdc_parent_critical(struct clk *sdc)
+{
+	struct clk *parent = clk_get_parent(sdc);
+
+	if (IS_ERR(parent))
+		return PTR_ERR(parent);
+
+	return clk_prepare_enable(parent);
+}
+
 static int bcm2835_clk_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct clk **clks;
+	struct clk_hw **hws;
 	struct bcm2835_cprman *cprman;
 	struct resource *res;
 	const struct bcm2835_clk_desc *desc;
 	const size_t asize = ARRAY_SIZE(clk_desc_array);
 	size_t i;
+	int ret;
 
-	cprman = devm_kzalloc(dev,
-			      sizeof(*cprman) + asize * sizeof(*clks),
+	cprman = devm_kzalloc(dev, sizeof(*cprman) +
+			      sizeof(*cprman->onecell.hws) * asize,
 			      GFP_KERNEL);
 	if (!cprman)
 		return -ENOMEM;
@@ -1819,18 +1882,21 @@ static int bcm2835_clk_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, cprman);
 
-	cprman->onecell.clk_num = asize;
-	cprman->onecell.clks = cprman->clks;
-	clks = cprman->clks;
+	cprman->onecell.num = asize;
+	hws = cprman->onecell.hws;
 
 	for (i = 0; i < asize; i++) {
 		desc = &clk_desc_array[i];
 		if (desc->clk_register && desc->data)
-			clks[i] = desc->clk_register(cprman, desc->data);
+			hws[i] = desc->clk_register(cprman, desc->data);
 	}
 
-	return of_clk_add_provider(dev->of_node, of_clk_src_onecell_get,
-				   &cprman->onecell);
+	ret = bcm2835_mark_sdc_parent_critical(hws[BCM2835_CLOCK_SDRAM]->clk);
+	if (ret)
+		return ret;
+
+	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get,
+				      &cprman->onecell);
 }
 
 static const struct of_device_id bcm2835_clk_of_match[] = {
diff --git a/drivers/clk/bcm/clk-bcm53573-ilp.c b/drivers/clk/bcm/clk-bcm53573-ilp.c
new file mode 100644
index 000000000000..36eb3716ffb0
--- /dev/null
+++ b/drivers/clk/bcm/clk-bcm53573-ilp.c
@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#define PMU_XTAL_FREQ_RATIO			0x66c
+#define  XTAL_ALP_PER_4ILP			0x00001fff
+#define  XTAL_CTL_EN				0x80000000
+#define PMU_SLOW_CLK_PERIOD			0x6dc
+
+struct bcm53573_ilp {
+	struct clk_hw hw;
+	struct regmap *regmap;
+};
+
+static int bcm53573_ilp_enable(struct clk_hw *hw)
+{
+	struct bcm53573_ilp *ilp = container_of(hw, struct bcm53573_ilp, hw);
+
+	regmap_write(ilp->regmap, PMU_SLOW_CLK_PERIOD, 0x10199);
+	regmap_write(ilp->regmap, 0x674, 0x10000);
+
+	return 0;
+}
+
+static void bcm53573_ilp_disable(struct clk_hw *hw)
+{
+	struct bcm53573_ilp *ilp = container_of(hw, struct bcm53573_ilp, hw);
+
+	regmap_write(ilp->regmap, PMU_SLOW_CLK_PERIOD, 0);
+	regmap_write(ilp->regmap, 0x674, 0);
+}
+
+static unsigned long bcm53573_ilp_recalc_rate(struct clk_hw *hw,
+					      unsigned long parent_rate)
+{
+	struct bcm53573_ilp *ilp = container_of(hw, struct bcm53573_ilp, hw);
+	struct regmap *regmap = ilp->regmap;
+	u32 last_val, cur_val;
+	int sum = 0, num = 0, loop_num = 0;
+	int avg;
+
+	/* Enable measurement */
+	regmap_write(regmap, PMU_XTAL_FREQ_RATIO, XTAL_CTL_EN);
+
+	/* Read initial value */
+	regmap_read(regmap, PMU_XTAL_FREQ_RATIO, &last_val);
+	last_val &= XTAL_ALP_PER_4ILP;
+
+	/*
+	 * At minimum we should loop for a bit to let hardware do the
+	 * measurement. This isn't very accurate however, so for a better
+	 * precision lets try getting 20 different values for and use average.
+	 */
+	while (num < 20) {
+		regmap_read(regmap, PMU_XTAL_FREQ_RATIO, &cur_val);
+		cur_val &= XTAL_ALP_PER_4ILP;
+
+		if (cur_val != last_val) {
+			/* Got different value, use it */
+			sum += cur_val;
+			num++;
+			loop_num = 0;
+			last_val = cur_val;
+		} else if (++loop_num > 5000) {
+			/* Same value over and over, give up */
+			sum += cur_val;
+			num++;
+			break;
+		}
+
+		cpu_relax();
+	}
+
+	/* Disable measurement to save power */
+	regmap_write(regmap, PMU_XTAL_FREQ_RATIO, 0x0);
+
+	avg = sum / num;
+
+	return parent_rate * 4 / avg;
+}
+
+static const struct clk_ops bcm53573_ilp_clk_ops = {
+	.enable = bcm53573_ilp_enable,
+	.disable = bcm53573_ilp_disable,
+	.recalc_rate = bcm53573_ilp_recalc_rate,
+};
+
+static void bcm53573_ilp_init(struct device_node *np)
+{
+	struct bcm53573_ilp *ilp;
+	struct clk_init_data init = { };
+	const char *parent_name;
+	int err;
+
+	ilp = kzalloc(sizeof(*ilp), GFP_KERNEL);
+	if (!ilp)
+		return;
+
+	parent_name = of_clk_get_parent_name(np, 0);
+	if (!parent_name) {
+		err = -ENOENT;
+		goto err_free_ilp;
+	}
+
+	ilp->regmap = syscon_node_to_regmap(of_get_parent(np));
+	if (IS_ERR(ilp->regmap)) {
+		err = PTR_ERR(ilp->regmap);
+		goto err_free_ilp;
+	}
+
+	init.name = np->name;
+	init.ops = &bcm53573_ilp_clk_ops;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	ilp->hw.init = &init;
+	err = clk_hw_register(NULL, &ilp->hw);
+	if (err)
+		goto err_free_ilp;
+
+	err = of_clk_add_hw_provider(np, of_clk_hw_simple_get, &ilp->hw);
+	if (err)
+		goto err_clk_hw_unregister;
+
+	return;
+
+err_clk_hw_unregister:
+	clk_hw_unregister(&ilp->hw);
+err_free_ilp:
+	kfree(ilp);
+	pr_err("Failed to init ILP clock: %d\n", err);
+}
+
+/* We need it very early for arch code, before device model gets ready */
+CLK_OF_DECLARE(bcm53573_ilp_clk, "brcm,bcm53573-ilp", bcm53573_ilp_init);
diff --git a/drivers/clk/bcm/clk-kona-setup.c b/drivers/clk/bcm/clk-kona-setup.c
index 526b0b0e9a9f..c37a7f0e83aa 100644
--- a/drivers/clk/bcm/clk-kona-setup.c
+++ b/drivers/clk/bcm/clk-kona-setup.c
@@ -586,8 +586,8 @@ static u32 *parent_process(const char *clocks[],
 	}
 
 	/* There is at least one parent, so allocate a selector array */
-
-	parent_sel = kmalloc(parent_count * sizeof(*parent_sel), GFP_KERNEL);
+	parent_sel = kmalloc_array(parent_count, sizeof(*parent_sel),
+				   GFP_KERNEL);
 	if (!parent_sel) {
 		pr_err("%s: error allocating %u parent selectors\n", __func__,
 				parent_count);
@@ -696,77 +696,69 @@ static void bcm_clk_teardown(struct kona_clk *bcm_clk)
 	bcm_clk->type = bcm_clk_none;
 }
 
-static void kona_clk_teardown(struct clk *clk)
+static void kona_clk_teardown(struct clk_hw *hw)
 {
-	struct clk_hw *hw;
 	struct kona_clk *bcm_clk;
 
-	if (!clk)
+	if (!hw)
 		return;
 
-	hw = __clk_get_hw(clk);
-	if (!hw) {
-		pr_err("%s: clk %p has null hw pointer\n", __func__, clk);
-		return;
-	}
-	clk_unregister(clk);
+	clk_hw_unregister(hw);
 
 	bcm_clk = to_kona_clk(hw);
 	bcm_clk_teardown(bcm_clk);
 }
 
-struct clk *kona_clk_setup(struct kona_clk *bcm_clk)
+static int kona_clk_setup(struct kona_clk *bcm_clk)
 {
+	int ret;
 	struct clk_init_data *init_data = &bcm_clk->init_data;
-	struct clk *clk = NULL;
 
 	switch (bcm_clk->type) {
 	case bcm_clk_peri:
-		if (peri_clk_setup(bcm_clk->u.data, init_data))
-			return NULL;
+		ret = peri_clk_setup(bcm_clk->u.data, init_data);
+		if (ret)
+			return ret;
 		break;
 	default:
 		pr_err("%s: clock type %d invalid for %s\n", __func__,
 			(int)bcm_clk->type, init_data->name);
-		return NULL;
+		return -EINVAL;
 	}
 
 	/* Make sure everything makes sense before we set it up */
 	if (!kona_clk_valid(bcm_clk)) {
 		pr_err("%s: clock data invalid for %s\n", __func__,
 			init_data->name);
+		ret = -EINVAL;
 		goto out_teardown;
 	}
 
 	bcm_clk->hw.init = init_data;
-	clk = clk_register(NULL, &bcm_clk->hw);
-	if (IS_ERR(clk)) {
-		pr_err("%s: error registering clock %s (%ld)\n", __func__,
-			init_data->name, PTR_ERR(clk));
+	ret = clk_hw_register(NULL, &bcm_clk->hw);
+	if (ret) {
+		pr_err("%s: error registering clock %s (%d)\n", __func__,
+			init_data->name, ret);
 		goto out_teardown;
 	}
-	BUG_ON(!clk);
 
-	return clk;
+	return 0;
 out_teardown:
 	bcm_clk_teardown(bcm_clk);
 
-	return NULL;
+	return ret;
 }
 
 static void ccu_clks_teardown(struct ccu_data *ccu)
 {
 	u32 i;
 
-	for (i = 0; i < ccu->clk_data.clk_num; i++)
-		kona_clk_teardown(ccu->clk_data.clks[i]);
-	kfree(ccu->clk_data.clks);
+	for (i = 0; i < ccu->clk_num; i++)
+		kona_clk_teardown(&ccu->kona_clks[i].hw);
 }
 
 static void kona_ccu_teardown(struct ccu_data *ccu)
 {
-	kfree(ccu->clk_data.clks);
-	ccu->clk_data.clks = NULL;
 	if (!ccu->base)
 		return;
 
@@ -793,6 +785,20 @@ static bool ccu_data_valid(struct ccu_data *ccu)
 	return true;
 }
 
+static struct clk_hw *
+of_clk_kona_onecell_get(struct of_phandle_args *clkspec, void *data)
+{
+	struct ccu_data *ccu = data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= ccu->clk_num) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return &ccu->kona_clks[idx].hw;
+}
+
 /*
  * Set up a CCU.  Call the provided ccu_clks_setup callback to
  * initialize the array of clocks provided by the CCU.
@@ -805,18 +811,6 @@ void __init kona_dt_ccu_setup(struct ccu_data *ccu,
 	unsigned int i;
 	int ret;
 
-	if (ccu->clk_data.clk_num) {
-		size_t size;
-
-		size = ccu->clk_data.clk_num * sizeof(*ccu->clk_data.clks);
-		ccu->clk_data.clks = kzalloc(size, GFP_KERNEL);
-		if (!ccu->clk_data.clks) {
-			pr_err("%s: unable to allocate %u clocks for %s\n",
-				__func__, ccu->clk_data.clk_num, node->name);
-			return;
-		}
-	}
-
 	ret = of_address_to_resource(node, 0, &res);
 	if (ret) {
 		pr_err("%s: no valid CCU registers found for %s\n", __func__,
@@ -851,13 +845,13 @@ void __init kona_dt_ccu_setup(struct ccu_data *ccu,
 	 * the clock framework clock array (in ccu->data).  Then
 	 * register as a provider for these clocks.
 	 */
-	for (i = 0; i < ccu->clk_data.clk_num; i++) {
+	for (i = 0; i < ccu->clk_num; i++) {
 		if (!ccu->kona_clks[i].ccu)
 			continue;
-		ccu->clk_data.clks[i] = kona_clk_setup(&ccu->kona_clks[i]);
+		kona_clk_setup(&ccu->kona_clks[i]);
 	}
 
-	ret = of_clk_add_provider(node, of_clk_src_onecell_get, &ccu->clk_data);
+	ret = of_clk_add_hw_provider(node, of_clk_kona_onecell_get, ccu);
 	if (ret) {
 		pr_err("%s: error adding ccu %s as provider (%d)\n", __func__,
 				node->name, ret);
diff --git a/drivers/clk/bcm/clk-kona.c b/drivers/clk/bcm/clk-kona.c
index 3a15347b4233..eee64b9e5d10 100644
--- a/drivers/clk/bcm/clk-kona.c
+++ b/drivers/clk/bcm/clk-kona.c
@@ -1256,19 +1256,18 @@ bool __init kona_ccu_init(struct ccu_data *ccu)
 {
 	unsigned long flags;
 	unsigned int which;
-	struct clk **clks = ccu->clk_data.clks;
 	struct kona_clk *kona_clks = ccu->kona_clks;
 	bool success = true;
 
 	flags = ccu_lock(ccu);
 	__ccu_write_enable(ccu);
 
-	for (which = 0; which < ccu->clk_data.clk_num; which++) {
-		struct kona_clk *bcm_clk;
+	for (which = 0; which < ccu->clk_num; which++) {
+		struct kona_clk *bcm_clk = &kona_clks[which];
 
-		if (!clks[which])
+		if (!bcm_clk->ccu)
 			continue;
-		bcm_clk = &kona_clks[which];
+
 		success &= __kona_clk_init(bcm_clk);
 	}
 
diff --git a/drivers/clk/bcm/clk-kona.h b/drivers/clk/bcm/clk-kona.h
index 906576ec97b6..f4b39bb5558a 100644
--- a/drivers/clk/bcm/clk-kona.h
+++ b/drivers/clk/bcm/clk-kona.h
@@ -481,7 +481,7 @@ struct ccu_data {
 	bool write_enabled;	/* write access is currently enabled */
 	struct ccu_policy policy;
 	struct device_node *node;
-	struct clk_onecell_data clk_data;
+	size_t clk_num;
 	const char *name;
 	u32 range;		/* byte range of address space */
 	struct kona_clk kona_clks[];	/* must be last */
@@ -491,9 +491,7 @@ struct ccu_data {
 #define KONA_CCU_COMMON(_prefix, _name, _ccuname)			    \
 	.name		= #_name "_ccu",				    \
 	.lock		= __SPIN_LOCK_UNLOCKED(_name ## _ccu_data.lock),    \
-	.clk_data	= {						    \
-		.clk_num = _prefix ## _ ## _ccuname ## _CCU_CLOCK_COUNT,    \
-	}
+	.clk_num	= _prefix ## _ ## _ccuname ## _CCU_CLOCK_COUNT
 
 /* Exported globals */
 
@@ -505,7 +503,6 @@ extern u64 scaled_div_max(struct bcm_clk_div *div);
 extern u64 scaled_div_build(struct bcm_clk_div *div, u32 div_value,
 				u32 billionths);
 
-extern struct clk *kona_clk_setup(struct kona_clk *bcm_clk);
 extern void __init kona_dt_ccu_setup(struct ccu_data *ccu,
 				struct device_node *node);
 extern bool __init kona_ccu_init(struct ccu_data *ccu);
diff --git a/drivers/clk/berlin/berlin2-avpll.c b/drivers/clk/berlin/berlin2-avpll.c
index fd0f26c38465..cfcae468e989 100644
--- a/drivers/clk/berlin/berlin2-avpll.c
+++ b/drivers/clk/berlin/berlin2-avpll.c
@@ -188,7 +188,7 @@ static const struct clk_ops berlin2_avpll_vco_ops = {
 	.recalc_rate	= berlin2_avpll_vco_recalc_rate,
 };
 
-struct clk * __init berlin2_avpll_vco_register(void __iomem *base,
+int __init berlin2_avpll_vco_register(void __iomem *base,
 			       const char *name, const char *parent_name,
 			       u8 vco_flags, unsigned long flags)
 {
@@ -197,7 +197,7 @@ struct clk * __init berlin2_avpll_vco_register(void __iomem *base,
 
 	vco = kzalloc(sizeof(*vco), GFP_KERNEL);
 	if (!vco)
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 
 	vco->base = base;
 	vco->flags = vco_flags;
@@ -208,7 +208,7 @@ struct clk * __init berlin2_avpll_vco_register(void __iomem *base,
 	init.num_parents = 1;
 	init.flags = flags;
 
-	return clk_register(NULL, &vco->hw);
+	return clk_hw_register(NULL, &vco->hw);
 }
 
 struct berlin2_avpll_channel {
@@ -364,7 +364,7 @@ static const struct clk_ops berlin2_avpll_channel_ops = {
  */
 static const u8 quirk_index[] __initconst = { 0, 6, 5, 4, 3, 2, 1, 7 };
 
-struct clk * __init berlin2_avpll_channel_register(void __iomem *base,
+int __init berlin2_avpll_channel_register(void __iomem *base,
 			   const char *name, u8 index, const char *parent_name,
 			   u8 ch_flags, unsigned long flags)
 {
@@ -373,7 +373,7 @@ struct clk * __init berlin2_avpll_channel_register(void __iomem *base,
 
 	ch = kzalloc(sizeof(*ch), GFP_KERNEL);
 	if (!ch)
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 
 	ch->base = base;
 	if (ch_flags & BERLIN2_AVPLL_SCRAMBLE_QUIRK)
@@ -389,5 +389,5 @@ struct clk * __init berlin2_avpll_channel_register(void __iomem *base,
 	init.num_parents = 1;
 	init.flags = flags;
 
-	return clk_register(NULL, &ch->hw);
+	return clk_hw_register(NULL, &ch->hw);
 }
diff --git a/drivers/clk/berlin/berlin2-avpll.h b/drivers/clk/berlin/berlin2-avpll.h
index a37f5068d299..17e311153b42 100644
--- a/drivers/clk/berlin/berlin2-avpll.h
+++ b/drivers/clk/berlin/berlin2-avpll.h
@@ -19,17 +19,13 @@
 #ifndef __BERLIN2_AVPLL_H
 #define __BERLIN2_AVPLL_H
 
-struct clk;
-
 #define BERLIN2_AVPLL_BIT_QUIRK		BIT(0)
 #define BERLIN2_AVPLL_SCRAMBLE_QUIRK	BIT(1)
 
-struct clk * __init
-berlin2_avpll_vco_register(void __iomem *base, const char *name,
+int berlin2_avpll_vco_register(void __iomem *base, const char *name,
 	   const char *parent_name, u8 vco_flags, unsigned long flags);
 
-struct clk * __init
-berlin2_avpll_channel_register(void __iomem *base, const char *name,
+int berlin2_avpll_channel_register(void __iomem *base, const char *name,
 		       u8 index, const char *parent_name, u8 ch_flags,
 		       unsigned long flags);
 
diff --git a/drivers/clk/berlin/berlin2-div.c b/drivers/clk/berlin/berlin2-div.c
index 81ff97f8aa0b..41ab2d392c57 100644
--- a/drivers/clk/berlin/berlin2-div.c
+++ b/drivers/clk/berlin/berlin2-div.c
@@ -234,7 +234,7 @@ static const struct clk_ops berlin2_div_mux_ops = {
 	.get_parent	= berlin2_div_get_parent,
 };
 
-struct clk * __init
+struct clk_hw * __init
 berlin2_div_register(const struct berlin2_div_map *map,
 		     void __iomem *base, const char *name, u8 div_flags,
 		     const char **parent_names, int num_parents,
@@ -259,7 +259,7 @@ berlin2_div_register(const struct berlin2_div_map *map,
 	if ((div_flags & BERLIN2_DIV_HAS_MUX) == 0)
 		mux_ops = NULL;
 
-	return clk_register_composite(NULL, name, parent_names, num_parents,
+	return clk_hw_register_composite(NULL, name, parent_names, num_parents,
 				      &div->hw, mux_ops, &div->hw, rate_ops,
 				      &div->hw, gate_ops, flags);
 }
diff --git a/drivers/clk/berlin/berlin2-div.h b/drivers/clk/berlin/berlin2-div.h
index 15e3384f3116..e835ddf8374a 100644
--- a/drivers/clk/berlin/berlin2-div.h
+++ b/drivers/clk/berlin/berlin2-div.h
@@ -19,7 +19,7 @@
 #ifndef __BERLIN2_DIV_H
 #define __BERLIN2_DIV_H
 
-struct clk;
+struct clk_hw;
 
 #define BERLIN2_DIV_HAS_GATE		BIT(0)
 #define BERLIN2_DIV_HAS_MUX		BIT(1)
@@ -80,7 +80,7 @@ struct berlin2_div_data {
 	u8 div_flags;
 };
 
-struct clk * __init
+struct clk_hw *
 berlin2_div_register(const struct berlin2_div_map *map,
 	     void __iomem *base,  const char *name, u8 div_flags,
 	     const char **parent_names, int num_parents,
diff --git a/drivers/clk/berlin/berlin2-pll.c b/drivers/clk/berlin/berlin2-pll.c
index 1c2294d3ba85..4ffbe80f6323 100644
--- a/drivers/clk/berlin/berlin2-pll.c
+++ b/drivers/clk/berlin/berlin2-pll.c
@@ -84,7 +84,7 @@ static const struct clk_ops berlin2_pll_ops = {
 	.recalc_rate	= berlin2_pll_recalc_rate,
 };
 
-struct clk * __init
+int __init
 berlin2_pll_register(const struct berlin2_pll_map *map,
 		     void __iomem *base, const char *name,
 		     const char *parent_name, unsigned long flags)
@@ -94,7 +94,7 @@ berlin2_pll_register(const struct berlin2_pll_map *map,
 
 	pll = kzalloc(sizeof(*pll), GFP_KERNEL);
 	if (!pll)
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 
 	/* copy pll_map to allow __initconst */
 	memcpy(&pll->map, map, sizeof(*map));
@@ -106,5 +106,5 @@ berlin2_pll_register(const struct berlin2_pll_map *map,
 	init.num_parents = 1;
 	init.flags = flags;
 
-	return clk_register(NULL, &pll->hw);
+	return clk_hw_register(NULL, &pll->hw);
 }
diff --git a/drivers/clk/berlin/berlin2-pll.h b/drivers/clk/berlin/berlin2-pll.h
index 8831ce27ac1e..583e024b9bed 100644
--- a/drivers/clk/berlin/berlin2-pll.h
+++ b/drivers/clk/berlin/berlin2-pll.h
@@ -19,8 +19,6 @@
 #ifndef __BERLIN2_PLL_H
 #define __BERLIN2_PLL_H
 
-struct clk;
-
 struct berlin2_pll_map {
 	const u8 vcodiv[16];
 	u8 mult;
@@ -29,9 +27,8 @@ struct berlin2_pll_map {
 	u8 divsel_shift;
 };
 
-struct clk * __init
-berlin2_pll_register(const struct berlin2_pll_map *map,
-		     void __iomem *base, const char *name,
-		     const char *parent_name, unsigned long flags);
+int berlin2_pll_register(const struct berlin2_pll_map *map,
+			 void __iomem *base, const char *name,
+			 const char *parent_name, unsigned long flags);
 
 #endif /* __BERLIN2_PLL_H */
diff --git a/drivers/clk/berlin/bg2.c b/drivers/clk/berlin/bg2.c
index 23e0e3be6c37..edf3b96b3b73 100644
--- a/drivers/clk/berlin/bg2.c
+++ b/drivers/clk/berlin/bg2.c
@@ -92,8 +92,7 @@
  */
 
 #define	MAX_CLKS 41
-static struct clk *clks[MAX_CLKS];
-static struct clk_onecell_data clk_data;
+static struct clk_hw_onecell_data *clk_data;
 static DEFINE_SPINLOCK(lock);
 static void __iomem *gbase;
 
@@ -505,8 +504,17 @@ static void __init berlin2_clock_setup(struct device_node *np)
 	struct device_node *parent_np = of_get_parent(np);
 	const char *parent_names[9];
 	struct clk *clk;
+	struct clk_hw *hw;
+	struct clk_hw **hws;
 	u8 avpll_flags = 0;
-	int n;
+	int n, ret;
+
+	clk_data = kzalloc(sizeof(*clk_data) +
+			   sizeof(*clk_data->hws) * MAX_CLKS, GFP_KERNEL);
+	if (!clk_data)
+		return;
+	clk_data->num = MAX_CLKS;
+	hws = clk_data->hws;
 
 	gbase = of_iomap(parent_np, 0);
 	if (!gbase)
@@ -526,118 +534,118 @@ static void __init berlin2_clock_setup(struct device_node *np)
 	}
 
 	/* simple register PLLs */
-	clk = berlin2_pll_register(&bg2_pll_map, gbase + REG_SYSPLLCTL0,
+	ret = berlin2_pll_register(&bg2_pll_map, gbase + REG_SYSPLLCTL0,
 				   clk_names[SYSPLL], clk_names[REFCLK], 0);
-	if (IS_ERR(clk))
+	if (ret)
 		goto bg2_fail;
 
-	clk = berlin2_pll_register(&bg2_pll_map, gbase + REG_MEMPLLCTL0,
+	ret = berlin2_pll_register(&bg2_pll_map, gbase + REG_MEMPLLCTL0,
 				   clk_names[MEMPLL], clk_names[REFCLK], 0);
-	if (IS_ERR(clk))
+	if (ret)
 		goto bg2_fail;
 
-	clk = berlin2_pll_register(&bg2_pll_map, gbase + REG_CPUPLLCTL0,
+	ret = berlin2_pll_register(&bg2_pll_map, gbase + REG_CPUPLLCTL0,
 				   clk_names[CPUPLL], clk_names[REFCLK], 0);
-	if (IS_ERR(clk))
+	if (ret)
 		goto bg2_fail;
 
 	if (of_device_is_compatible(np, "marvell,berlin2-global-register"))
 		avpll_flags |= BERLIN2_AVPLL_SCRAMBLE_QUIRK;
 
 	/* audio/video VCOs */
-	clk = berlin2_avpll_vco_register(gbase + REG_AVPLLCTL0, "avpll_vcoA",
+	ret = berlin2_avpll_vco_register(gbase + REG_AVPLLCTL0, "avpll_vcoA",
 			 clk_names[REFCLK], avpll_flags, 0);
-	if (IS_ERR(clk))
+	if (ret)
 		goto bg2_fail;
 
 	for (n = 0; n < 8; n++) {
-		clk = berlin2_avpll_channel_register(gbase + REG_AVPLLCTL0,
+		ret = berlin2_avpll_channel_register(gbase + REG_AVPLLCTL0,
 			     clk_names[AVPLL_A1 + n], n, "avpll_vcoA",
 			     avpll_flags, 0);
-		if (IS_ERR(clk))
+		if (ret)
 			goto bg2_fail;
 	}
 
-	clk = berlin2_avpll_vco_register(gbase + REG_AVPLLCTL31, "avpll_vcoB",
+	ret = berlin2_avpll_vco_register(gbase + REG_AVPLLCTL31, "avpll_vcoB",
 				 clk_names[REFCLK], BERLIN2_AVPLL_BIT_QUIRK |
 				 avpll_flags, 0);
-	if (IS_ERR(clk))
+	if (ret)
 		goto bg2_fail;
 
 	for (n = 0; n < 8; n++) {
-		clk = berlin2_avpll_channel_register(gbase + REG_AVPLLCTL31,
+		ret = berlin2_avpll_channel_register(gbase + REG_AVPLLCTL31,
 			     clk_names[AVPLL_B1 + n], n, "avpll_vcoB",
 			     BERLIN2_AVPLL_BIT_QUIRK | avpll_flags, 0);
-		if (IS_ERR(clk))
+		if (ret)
 			goto bg2_fail;
 	}
 
 	/* reference clock bypass switches */
 	parent_names[0] = clk_names[SYSPLL];
 	parent_names[1] = clk_names[REFCLK];
-	clk = clk_register_mux(NULL, "syspll_byp", parent_names, 2,
+	hw = clk_hw_register_mux(NULL, "syspll_byp", parent_names, 2,
 			       0, gbase + REG_CLKSWITCH0, 0, 1, 0, &lock);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto bg2_fail;
-	clk_names[SYSPLL] = __clk_get_name(clk);
+	clk_names[SYSPLL] = clk_hw_get_name(hw);
 
 	parent_names[0] = clk_names[MEMPLL];
 	parent_names[1] = clk_names[REFCLK];
-	clk = clk_register_mux(NULL, "mempll_byp", parent_names, 2,
+	hw = clk_hw_register_mux(NULL, "mempll_byp", parent_names, 2,
 			       0, gbase + REG_CLKSWITCH0, 1, 1, 0, &lock);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto bg2_fail;
-	clk_names[MEMPLL] = __clk_get_name(clk);
+	clk_names[MEMPLL] = clk_hw_get_name(hw);
 
 	parent_names[0] = clk_names[CPUPLL];
 	parent_names[1] = clk_names[REFCLK];
-	clk = clk_register_mux(NULL, "cpupll_byp", parent_names, 2,
+	hw = clk_hw_register_mux(NULL, "cpupll_byp", parent_names, 2,
 			       0, gbase + REG_CLKSWITCH0, 2, 1, 0, &lock);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto bg2_fail;
-	clk_names[CPUPLL] = __clk_get_name(clk);
+	clk_names[CPUPLL] = clk_hw_get_name(hw);
 
 	/* clock muxes */
 	parent_names[0] = clk_names[AVPLL_B3];
 	parent_names[1] = clk_names[AVPLL_A3];
-	clk = clk_register_mux(NULL, clk_names[AUDIO1_PLL], parent_names, 2,
+	hw = clk_hw_register_mux(NULL, clk_names[AUDIO1_PLL], parent_names, 2,
 			       0, gbase + REG_CLKSELECT2, 29, 1, 0, &lock);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto bg2_fail;
 
 	parent_names[0] = clk_names[VIDEO0_PLL];
 	parent_names[1] = clk_names[VIDEO_EXT0];
-	clk = clk_register_mux(NULL, clk_names[VIDEO0_IN], parent_names, 2,
+	hw = clk_hw_register_mux(NULL, clk_names[VIDEO0_IN], parent_names, 2,
 			       0, gbase + REG_CLKSELECT3, 4, 1, 0, &lock);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto bg2_fail;
 
 	parent_names[0] = clk_names[VIDEO1_PLL];
 	parent_names[1] = clk_names[VIDEO_EXT0];
-	clk = clk_register_mux(NULL, clk_names[VIDEO1_IN], parent_names, 2,
+	hw = clk_hw_register_mux(NULL, clk_names[VIDEO1_IN], parent_names, 2,
 			       0, gbase + REG_CLKSELECT3, 6, 1, 0, &lock);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto bg2_fail;
 
 	parent_names[0] = clk_names[AVPLL_A2];
 	parent_names[1] = clk_names[AVPLL_B2];
-	clk = clk_register_mux(NULL, clk_names[VIDEO1_PLL], parent_names, 2,
+	hw = clk_hw_register_mux(NULL, clk_names[VIDEO1_PLL], parent_names, 2,
 			       0, gbase + REG_CLKSELECT3, 7, 1, 0, &lock);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto bg2_fail;
 
 	parent_names[0] = clk_names[VIDEO2_PLL];
 	parent_names[1] = clk_names[VIDEO_EXT0];
-	clk = clk_register_mux(NULL, clk_names[VIDEO2_IN], parent_names, 2,
+	hw = clk_hw_register_mux(NULL, clk_names[VIDEO2_IN], parent_names, 2,
 			       0, gbase + REG_CLKSELECT3, 9, 1, 0, &lock);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto bg2_fail;
 
 	parent_names[0] = clk_names[AVPLL_B1];
 	parent_names[1] = clk_names[AVPLL_A5];
-	clk = clk_register_mux(NULL, clk_names[VIDEO2_PLL], parent_names, 2,
+	hw = clk_hw_register_mux(NULL, clk_names[VIDEO2_PLL], parent_names, 2,
 			       0, gbase + REG_CLKSELECT3, 10, 1, 0, &lock);
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		goto bg2_fail;
 
 	/* clock divider cells */
@@ -648,7 +656,7 @@ static void __init berlin2_clock_setup(struct device_node *np)
 		for (k = 0; k < dd->num_parents; k++)
 			parent_names[k] = clk_names[dd->parent_ids[k]];
 
-		clks[CLKID_SYS + n] = berlin2_div_register(&dd->map, gbase,
+		hws[CLKID_SYS + n] = berlin2_div_register(&dd->map, gbase,
 				dd->name, dd->div_flags, parent_names,
 				dd->num_parents, dd->flags, &lock);
 	}
@@ -657,18 +665,18 @@ static void __init berlin2_clock_setup(struct device_node *np)
 	for (n = 0; n < ARRAY_SIZE(bg2_gates); n++) {
 		const struct berlin2_gate_data *gd = &bg2_gates[n];
 
-		clks[CLKID_GETH0 + n] = clk_register_gate(NULL, gd->name,
+		hws[CLKID_GETH0 + n] = clk_hw_register_gate(NULL, gd->name,
 			    gd->parent_name, gd->flags, gbase + REG_CLKENABLE,
 			    gd->bit_idx, 0, &lock);
 	}
 
 	/* twdclk is derived from cpu/3 */
-	clks[CLKID_TWD] =
-		clk_register_fixed_factor(NULL, "twd", "cpu", 0, 1, 3);
+	hws[CLKID_TWD] =
+		clk_hw_register_fixed_factor(NULL, "twd", "cpu", 0, 1, 3);
 
 	/* check for errors on leaf clocks */
 	for (n = 0; n < MAX_CLKS; n++) {
-		if (!IS_ERR(clks[n]))
+		if (!IS_ERR(hws[n]))
 			continue;
 
 		pr_err("%s: Unable to register leaf clock %d\n",
@@ -677,9 +685,7 @@ static void __init berlin2_clock_setup(struct device_node *np)
 	}
 
 	/* register clk-provider */
-	clk_data.clks = clks;
-	clk_data.clk_num = MAX_CLKS;
-	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+	of_clk_add_hw_provider(np, of_clk_hw_onecell_get, &clk_data);
 
 	return;
 
diff --git a/drivers/clk/berlin/bg2q.c b/drivers/clk/berlin/bg2q.c
index f144547cf76c..0718e831475f 100644
--- a/drivers/clk/berlin/bg2q.c
+++ b/drivers/clk/berlin/bg2q.c
@@ -46,8 +46,7 @@
 #define REG_SDIO1XIN_CLKCTL	0x015c
 
 #define	MAX_CLKS 28
-static struct clk *clks[MAX_CLKS];
-static struct clk_onecell_data clk_data;
+static struct clk_hw_onecell_data *clk_data;
 static DEFINE_SPINLOCK(lock);
 static void __iomem *gbase;
 static void __iomem *cpupll_base;
@@ -293,7 +292,15 @@ static void __init berlin2q_clock_setup(struct device_node *np)
 	struct device_node *parent_np = of_get_parent(np);
 	const char *parent_names[9];
 	struct clk *clk;
-	int n;
+	struct clk_hw **hws;
+	int n, ret;
+
+	clk_data = kzalloc(sizeof(*clk_data) +
+			   sizeof(*clk_data->hws) * MAX_CLKS, GFP_KERNEL);
+	if (!clk_data)
+		return;
+	clk_data->num = MAX_CLKS;
+	hws = clk_data->hws;
 
 	gbase = of_iomap(parent_np, 0);
 	if (!gbase) {
@@ -317,14 +324,14 @@ static void __init berlin2q_clock_setup(struct device_node *np)
 	}
 
 	/* simple register PLLs */
-	clk = berlin2_pll_register(&bg2q_pll_map, gbase + REG_SYSPLLCTL0,
+	ret = berlin2_pll_register(&bg2q_pll_map, gbase + REG_SYSPLLCTL0,
 				   clk_names[SYSPLL], clk_names[REFCLK], 0);
-	if (IS_ERR(clk))
+	if (ret)
 		goto bg2q_fail;
 
-	clk = berlin2_pll_register(&bg2q_pll_map, cpupll_base,
+	ret = berlin2_pll_register(&bg2q_pll_map, cpupll_base,
 				   clk_names[CPUPLL], clk_names[REFCLK], 0);
-	if (IS_ERR(clk))
+	if (ret)
 		goto bg2q_fail;
 
 	/* TODO: add BG2Q AVPLL */
@@ -342,7 +349,7 @@ static void __init berlin2q_clock_setup(struct device_node *np)
 		for (k = 0; k < dd->num_parents; k++)
 			parent_names[k] = clk_names[dd->parent_ids[k]];
 
-		clks[CLKID_SYS + n] = berlin2_div_register(&dd->map, gbase,
+		hws[CLKID_SYS + n] = berlin2_div_register(&dd->map, gbase,
 				dd->name, dd->div_flags, parent_names,
 				dd->num_parents, dd->flags, &lock);
 	}
@@ -351,22 +358,22 @@ static void __init berlin2q_clock_setup(struct device_node *np)
 	for (n = 0; n < ARRAY_SIZE(bg2q_gates); n++) {
 		const struct berlin2_gate_data *gd = &bg2q_gates[n];
 
-		clks[CLKID_GFX2DAXI + n] = clk_register_gate(NULL, gd->name,
+		hws[CLKID_GFX2DAXI + n] = clk_hw_register_gate(NULL, gd->name,
 			    gd->parent_name, gd->flags, gbase + REG_CLKENABLE,
 			    gd->bit_idx, 0, &lock);
 	}
 
 	/* cpuclk divider is fixed to 1 */
-	clks[CLKID_CPU] =
-		clk_register_fixed_factor(NULL, "cpu", clk_names[CPUPLL],
+	hws[CLKID_CPU] =
+		clk_hw_register_fixed_factor(NULL, "cpu", clk_names[CPUPLL],
 					  0, 1, 1);
 	/* twdclk is derived from cpu/3 */
-	clks[CLKID_TWD] =
-		clk_register_fixed_factor(NULL, "twd", "cpu", 0, 1, 3);
+	hws[CLKID_TWD] =
+		clk_hw_register_fixed_factor(NULL, "twd", "cpu", 0, 1, 3);
 
 	/* check for errors on leaf clocks */
 	for (n = 0; n < MAX_CLKS; n++) {
-		if (!IS_ERR(clks[n]))
+		if (!IS_ERR(hws[n]))
 			continue;
 
 		pr_err("%s: Unable to register leaf clock %d\n",
@@ -375,9 +382,7 @@ static void __init berlin2q_clock_setup(struct device_node *np)
 	}
 
 	/* register clk-provider */
-	clk_data.clks = clks;
-	clk_data.clk_num = MAX_CLKS;
-	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+	of_clk_add_hw_provider(np, of_clk_hw_onecell_get, &clk_data);
 
 	return;
 
diff --git a/drivers/clk/clk-asm9260.c b/drivers/clk/clk-asm9260.c
index 90897af8d9f7..ea8568536193 100644
--- a/drivers/clk/clk-asm9260.c
+++ b/drivers/clk/clk-asm9260.c
@@ -68,8 +68,7 @@
 #define HW_LCDCLKDIV		0x01fc
 #define HW_ADCANACLKDIV		0x0200
 
-static struct clk *clks[MAX_CLKS];
-static struct clk_onecell_data clk_data;
+static struct clk_hw_onecell_data *clk_data;
 static DEFINE_SPINLOCK(asm9260_clk_lock);
 
 struct asm9260_div_clk {
@@ -267,12 +266,20 @@ static struct asm9260_mux_clock asm9260_mux_clks[] __initdata = {
 
 static void __init asm9260_acc_init(struct device_node *np)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
+	struct clk_hw **hws;
 	const char *ref_clk, *pll_clk = "pll";
 	u32 rate;
 	int n;
 	u32 accuracy = 0;
 
+	clk_data = kzalloc(sizeof(*clk_data) +
+			   sizeof(*clk_data->hws) * MAX_CLKS, GFP_KERNEL);
+	if (!clk_data)
+		return;
+	clk_data->num = MAX_CLKS;
+	hws = clk_data->hws;
+
 	base = of_io_request_and_map(np, 0, np->name);
 	if (IS_ERR(base))
 		panic("%s: unable to map resource", np->name);
@@ -282,10 +289,10 @@ static void __init asm9260_acc_init(struct device_node *np)
 
 	ref_clk = of_clk_get_parent_name(np, 0);
 	accuracy = clk_get_accuracy(__clk_lookup(ref_clk));
-	clk = clk_register_fixed_rate_with_accuracy(NULL, pll_clk,
+	hw = clk_hw_register_fixed_rate_with_accuracy(NULL, pll_clk,
 			ref_clk, 0, rate, accuracy);
 
-	if (IS_ERR(clk))
+	if (IS_ERR(hw))
 		panic("%s: can't register REFCLK. Check DT!", np->name);
 
 	for (n = 0; n < ARRAY_SIZE(asm9260_mux_clks); n++) {
@@ -293,7 +300,7 @@ static void __init asm9260_acc_init(struct device_node *np)
 
 		mc->parent_names[0] = ref_clk;
 		mc->parent_names[1] = pll_clk;
-		clk = clk_register_mux_table(NULL, mc->name, mc->parent_names,
+		hw = clk_hw_register_mux_table(NULL, mc->name, mc->parent_names,
 				mc->num_parents, mc->flags, base + mc->offset,
 				0, mc->mask, 0, mc->table, &asm9260_clk_lock);
 	}
@@ -302,7 +309,7 @@ static void __init asm9260_acc_init(struct device_node *np)
 	for (n = 0; n < ARRAY_SIZE(asm9260_mux_gates); n++) {
 		const struct asm9260_gate_data *gd = &asm9260_mux_gates[n];
 
-		clk = clk_register_gate(NULL, gd->name,
+		hw = clk_hw_register_gate(NULL, gd->name,
 			gd->parent_name, gd->flags | CLK_SET_RATE_PARENT,
 			base + gd->reg, gd->bit_idx, 0, &asm9260_clk_lock);
 	}
@@ -311,7 +318,7 @@ static void __init asm9260_acc_init(struct device_node *np)
 	for (n = 0; n < ARRAY_SIZE(asm9260_div_clks); n++) {
 		const struct asm9260_div_clk *dc = &asm9260_div_clks[n];
 
-		clks[dc->idx] = clk_register_divider(NULL, dc->name,
+		hws[dc->idx] = clk_hw_register_divider(NULL, dc->name,
 				dc->parent_name, CLK_SET_RATE_PARENT,
 				base + dc->reg, 0, 8, CLK_DIVIDER_ONE_BASED,
 				&asm9260_clk_lock);
@@ -321,14 +328,14 @@ static void __init asm9260_acc_init(struct device_node *np)
 	for (n = 0; n < ARRAY_SIZE(asm9260_ahb_gates); n++) {
 		const struct asm9260_gate_data *gd = &asm9260_ahb_gates[n];
 
-		clks[gd->idx] = clk_register_gate(NULL, gd->name,
+		hws[gd->idx] = clk_hw_register_gate(NULL, gd->name,
 				gd->parent_name, gd->flags, base + gd->reg,
 				gd->bit_idx, 0, &asm9260_clk_lock);
 	}
 
 	/* check for errors on leaf clocks */
 	for (n = 0; n < MAX_CLKS; n++) {
-		if (!IS_ERR(clks[n]))
+		if (!IS_ERR(hws[n]))
 			continue;
 
 		pr_err("%s: Unable to register leaf clock %d\n",
@@ -337,9 +344,7 @@ static void __init asm9260_acc_init(struct device_node *np)
 	}
 
 	/* register clk-provider */
-	clk_data.clks = clks;
-	clk_data.clk_num = MAX_CLKS;
-	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+	of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data);
 	return;
 fail:
 	iounmap(base);
diff --git a/drivers/clk/clk-axi-clkgen.c b/drivers/clk/clk-axi-clkgen.c
index 3294db3b4e4e..5e918e7afaba 100644
--- a/drivers/clk/clk-axi-clkgen.c
+++ b/drivers/clk/clk-axi-clkgen.c
@@ -392,8 +392,8 @@ static int axi_clkgen_probe(struct platform_device *pdev)
 	const char *parent_names[2];
 	const char *clk_name;
 	struct resource *mem;
-	struct clk *clk;
 	unsigned int i;
+	int ret;
 
 	if (!pdev->dev.of_node)
 		return -ENODEV;
@@ -433,12 +433,12 @@ static int axi_clkgen_probe(struct platform_device *pdev)
 	axi_clkgen_mmcm_enable(axi_clkgen, false);
 
 	axi_clkgen->clk_hw.init = &init;
-	clk = devm_clk_register(&pdev->dev, &axi_clkgen->clk_hw);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = devm_clk_hw_register(&pdev->dev, &axi_clkgen->clk_hw);
+	if (ret)
+		return ret;
 
-	return of_clk_add_provider(pdev->dev.of_node, of_clk_src_simple_get,
-				    clk);
+	return of_clk_add_hw_provider(pdev->dev.of_node, of_clk_hw_simple_get,
+				      &axi_clkgen->clk_hw);
 }
 
 static int axi_clkgen_remove(struct platform_device *pdev)
diff --git a/drivers/clk/clk-axm5516.c b/drivers/clk/clk-axm5516.c
index c7c91a5ecf8b..5d7ae333257e 100644
--- a/drivers/clk/clk-axm5516.c
+++ b/drivers/clk/clk-axm5516.c
@@ -516,6 +516,19 @@ static struct axxia_clk *axmclk_clocks[] = {
 	[AXXIA_CLK_MMC]      = &clk_mmc_mux.aclk,
 };
 
+static struct clk_hw *
+of_clk_axmclk_get(struct of_phandle_args *clkspec, void *unused)
+{
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= ARRAY_SIZE(axmclk_clocks)) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return &axmclk_clocks[idx]->hw;
+}
+
 static const struct regmap_config axmclk_regmap_config = {
 	.reg_bits	= 32,
 	.reg_stride	= 4,
@@ -530,21 +543,14 @@ static const struct of_device_id axmclk_match_table[] = {
 };
 MODULE_DEVICE_TABLE(of, axmclk_match_table);
 
-struct axmclk_priv {
-	struct clk_onecell_data onecell;
-	struct clk *clks[];
-};
-
 static int axmclk_probe(struct platform_device *pdev)
 {
 	void __iomem *base;
 	struct resource *res;
 	int i, ret;
 	struct device *dev = &pdev->dev;
-	struct clk *clk;
 	struct regmap *regmap;
 	size_t num_clks;
-	struct axmclk_priv *priv;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	base = devm_ioremap_resource(dev, res);
@@ -557,29 +563,18 @@ static int axmclk_probe(struct platform_device *pdev)
 
 	num_clks = ARRAY_SIZE(axmclk_clocks);
 	pr_info("axmclk: supporting %zu clocks\n", num_clks);
-	priv = devm_kzalloc(dev, sizeof(*priv) + sizeof(*priv->clks) * num_clks,
-			    GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->onecell.clks = priv->clks;
-	priv->onecell.clk_num = num_clks;
 
 	/* Update each entry with the allocated regmap and register the clock
 	 * with the common clock framework
 	 */
 	for (i = 0; i < num_clks; i++) {
 		axmclk_clocks[i]->regmap = regmap;
-		clk = devm_clk_register(dev, &axmclk_clocks[i]->hw);
-		if (IS_ERR(clk))
-			return PTR_ERR(clk);
-		priv->clks[i] = clk;
+		ret = devm_clk_hw_register(dev, &axmclk_clocks[i]->hw);
+		if (ret)
+			return ret;
 	}
 
-	ret = of_clk_add_provider(dev->of_node,
-				  of_clk_src_onecell_get, &priv->onecell);
-
-	return ret;
+	return of_clk_add_hw_provider(dev->of_node, of_clk_axmclk_get, NULL);
 }
 
 static int axmclk_remove(struct platform_device *pdev)
diff --git a/drivers/clk/clk-cdce706.c b/drivers/clk/clk-cdce706.c
index 01877f64eff6..f21d9092564f 100644
--- a/drivers/clk/clk-cdce706.c
+++ b/drivers/clk/clk-cdce706.c
@@ -71,7 +71,6 @@ struct cdce706_hw_data {
 	struct cdce706_dev_data *dev_data;
 	unsigned idx;
 	unsigned parent;
-	struct clk *clk;
 	struct clk_hw hw;
 	unsigned div;
 	unsigned mul;
@@ -81,8 +80,6 @@ struct cdce706_hw_data {
 struct cdce706_dev_data {
 	struct i2c_client *client;
 	struct regmap *regmap;
-	struct clk_onecell_data onecell;
-	struct clk *clks[6];
 	struct clk *clkin_clk[2];
 	const char *clkin_name[2];
 	struct cdce706_hw_data clkin[1];
@@ -455,18 +452,19 @@ static int cdce706_register_hw(struct cdce706_dev_data *cdce,
 			       struct clk_init_data *init)
 {
 	unsigned i;
+	int ret;
 
 	for (i = 0; i < num_hw; ++i, ++hw) {
 		init->name = clk_names[i];
 		hw->dev_data = cdce;
 		hw->idx = i;
 		hw->hw.init = init;
-		hw->clk = devm_clk_register(&cdce->client->dev,
+		ret = devm_clk_hw_register(&cdce->client->dev,
 					    &hw->hw);
-		if (IS_ERR(hw->clk)) {
+		if (ret) {
 			dev_err(&cdce->client->dev, "Failed to register %s\n",
 				clk_names[i]);
-			return PTR_ERR(hw->clk);
+			return ret;
 		}
 	}
 	return 0;
@@ -613,13 +611,23 @@ static int cdce706_register_clkouts(struct cdce706_dev_data *cdce)
 			cdce->clkout[i].parent);
 	}
 
-	ret = cdce706_register_hw(cdce, cdce->clkout,
-				  ARRAY_SIZE(cdce->clkout),
-				  cdce706_clkout_name, &init);
-	for (i = 0; i < ARRAY_SIZE(cdce->clkout); ++i)
-		cdce->clks[i] = cdce->clkout[i].clk;
+	return cdce706_register_hw(cdce, cdce->clkout,
+				   ARRAY_SIZE(cdce->clkout),
+				   cdce706_clkout_name, &init);
+}
 
-	return ret;
+static struct clk_hw *
+of_clk_cdce_get(struct of_phandle_args *clkspec, void *data)
+{
+	struct cdce706_dev_data *cdce = data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= ARRAY_SIZE(cdce->clkout)) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return &cdce->clkout[idx].hw;
 }
 
 static int cdce706_probe(struct i2c_client *client,
@@ -657,12 +665,8 @@ static int cdce706_probe(struct i2c_client *client,
 	ret = cdce706_register_clkouts(cdce);
 	if (ret < 0)
 		return ret;
-	cdce->onecell.clks = cdce->clks;
-	cdce->onecell.clk_num = ARRAY_SIZE(cdce->clks);
-	ret = of_clk_add_provider(client->dev.of_node, of_clk_src_onecell_get,
-				  &cdce->onecell);
-
-	return ret;
+	return of_clk_add_hw_provider(client->dev.of_node, of_clk_cdce_get,
+				      cdce);
 }
 
 static int cdce706_remove(struct i2c_client *client)
diff --git a/drivers/clk/clk-cdce925.c b/drivers/clk/clk-cdce925.c
index 089bf88ffa8d..b8459c14a1b7 100644
--- a/drivers/clk/clk-cdce925.c
+++ b/drivers/clk/clk-cdce925.c
@@ -62,8 +62,6 @@ struct clk_cdce925_chip {
 	struct i2c_client *i2c_client;
 	struct clk_cdce925_pll pll[NUMBER_OF_PLLS];
 	struct clk_cdce925_output clk[NUMBER_OF_OUTPUTS];
-	struct clk *dt_clk[NUMBER_OF_OUTPUTS];
-	struct clk_onecell_data onecell;
 };
 
 /* ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** */
@@ -557,6 +555,20 @@ static int cdce925_regmap_i2c_read(void *context,
 		return -EIO;
 }
 
+static struct clk_hw *
+of_clk_cdce925_get(struct of_phandle_args *clkspec, void *_data)
+{
+	struct clk_cdce925_chip *data = _data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= ARRAY_SIZE(data->clk)) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return &data->clk[idx].hw;
+}
+
 /* The CDCE925 uses a funky way to read/write registers. Bulk mode is
  * just weird, so just use the single byte mode exclusively. */
 static struct regmap_bus regmap_cdce925_bus = {
@@ -572,7 +584,6 @@ static int cdce925_probe(struct i2c_client *client,
 	const char *parent_name;
 	const char *pll_clk_name[NUMBER_OF_PLLS] = {NULL,};
 	struct clk_init_data init;
-	struct clk *clk;
 	u32 value;
 	int i;
 	int err;
@@ -622,10 +633,9 @@ static int cdce925_probe(struct i2c_client *client,
 		data->pll[i].chip = data;
 		data->pll[i].hw.init = &init;
 		data->pll[i].index = i;
-		clk = devm_clk_register(&client->dev, &data->pll[i].hw);
-		if (IS_ERR(clk)) {
+		err = devm_clk_hw_register(&client->dev, &data->pll[i].hw);
+		if (err) {
 			dev_err(&client->dev, "Failed register PLL %d\n", i);
-			err = PTR_ERR(clk);
 			goto error;
 		}
 		sprintf(child_name, "PLL%d", i+1);
@@ -634,7 +644,7 @@ static int cdce925_probe(struct i2c_client *client,
 			continue;
 		if (!of_property_read_u32(np_output,
 			"clock-frequency", &value)) {
-			err = clk_set_rate(clk, value);
+			err = clk_set_rate(data->pll[i].hw.clk, value);
 			if (err)
 				dev_err(&client->dev,
 					"unable to set PLL frequency %ud\n",
@@ -663,14 +673,12 @@ static int cdce925_probe(struct i2c_client *client,
 	data->clk[0].hw.init = &init;
 	data->clk[0].index = 0;
 	data->clk[0].pdiv = 1;
-	clk = devm_clk_register(&client->dev, &data->clk[0].hw);
+	err = devm_clk_hw_register(&client->dev, &data->clk[0].hw);
 	kfree(init.name); /* clock framework made a copy of the name */
-	if (IS_ERR(clk)) {
+	if (err) {
 		dev_err(&client->dev, "clock registration Y1 failed\n");
-		err = PTR_ERR(clk);
 		goto error;
 	}
-	data->dt_clk[0] = clk;
 
 	/* Register output clocks Y2 .. Y5*/
 	init.ops = &cdce925_clk_ops;
@@ -695,21 +703,17 @@ static int cdce925_probe(struct i2c_client *client,
 			init.parent_names = &pll_clk_name[1];
 			break;
 		}
-		clk = devm_clk_register(&client->dev, &data->clk[i].hw);
+		err = devm_clk_hw_register(&client->dev, &data->clk[i].hw);
 		kfree(init.name); /* clock framework made a copy of the name */
-		if (IS_ERR(clk)) {
+		if (err) {
 			dev_err(&client->dev, "clock registration failed\n");
-			err = PTR_ERR(clk);
 			goto error;
 		}
-		data->dt_clk[i] = clk;
 	}
 
 	/* Register the output clocks */
-	data->onecell.clk_num = NUMBER_OF_OUTPUTS;
-	data->onecell.clks = data->dt_clk;
-	err = of_clk_add_provider(client->dev.of_node, of_clk_src_onecell_get,
-		&data->onecell);
+	err = of_clk_add_hw_provider(client->dev.of_node, of_clk_cdce925_get,
+				  data);
 	if (err)
 		dev_err(&client->dev, "unable to add OF clock provider\n");
 
diff --git a/drivers/clk/clk-clps711x.c b/drivers/clk/clk-clps711x.c
index adaf109f2fe2..9193f64561f6 100644
--- a/drivers/clk/clk-clps711x.c
+++ b/drivers/clk/clk-clps711x.c
@@ -40,9 +40,8 @@ static const struct clk_div_table timer_div_table[] = {
 };
 
 struct clps711x_clk {
-	struct clk_onecell_data	clk_data;
-	spinlock_t		lock;
-	struct clk		*clks[CLPS711X_CLK_MAX];
+	spinlock_t			lock;
+	struct clk_hw_onecell_data	clk_data;
 };
 
 static struct clps711x_clk * __init _clps711x_clk_init(void __iomem *base,
@@ -55,7 +54,9 @@ static struct clps711x_clk * __init _clps711x_clk_init(void __iomem *base,
 	if (!base)
 		return ERR_PTR(-ENOMEM);
 
-	clps711x_clk = kzalloc(sizeof(*clps711x_clk), GFP_KERNEL);
+	clps711x_clk = kzalloc(sizeof(*clps711x_clk) +
+			sizeof(*clps711x_clk->clk_data.hws) * CLPS711X_CLK_MAX,
+			GFP_KERNEL);
 	if (!clps711x_clk)
 		return ERR_PTR(-ENOMEM);
 
@@ -106,40 +107,40 @@ static struct clps711x_clk * __init _clps711x_clk_init(void __iomem *base,
 	tmp |= SYSCON1_TC2M | SYSCON1_TC2S;
 	writel(tmp, base + CLPS711X_SYSCON1);
 
-	clps711x_clk->clks[CLPS711X_CLK_DUMMY] =
-		clk_register_fixed_rate(NULL, "dummy", NULL, 0, 0);
-	clps711x_clk->clks[CLPS711X_CLK_CPU] =
-		clk_register_fixed_rate(NULL, "cpu", NULL, 0, f_cpu);
-	clps711x_clk->clks[CLPS711X_CLK_BUS] =
-		clk_register_fixed_rate(NULL, "bus", NULL, 0, f_bus);
-	clps711x_clk->clks[CLPS711X_CLK_PLL] =
-		clk_register_fixed_rate(NULL, "pll", NULL, 0, f_pll);
-	clps711x_clk->clks[CLPS711X_CLK_TIMERREF] =
-		clk_register_fixed_rate(NULL, "timer_ref", NULL, 0, f_tim);
-	clps711x_clk->clks[CLPS711X_CLK_TIMER1] =
-		clk_register_divider_table(NULL, "timer1", "timer_ref", 0,
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_DUMMY] =
+		clk_hw_register_fixed_rate(NULL, "dummy", NULL, 0, 0);
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_CPU] =
+		clk_hw_register_fixed_rate(NULL, "cpu", NULL, 0, f_cpu);
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_BUS] =
+		clk_hw_register_fixed_rate(NULL, "bus", NULL, 0, f_bus);
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_PLL] =
+		clk_hw_register_fixed_rate(NULL, "pll", NULL, 0, f_pll);
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_TIMERREF] =
+		clk_hw_register_fixed_rate(NULL, "timer_ref", NULL, 0, f_tim);
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_TIMER1] =
+		clk_hw_register_divider_table(NULL, "timer1", "timer_ref", 0,
 					   base + CLPS711X_SYSCON1, 5, 1, 0,
 					   timer_div_table, &clps711x_clk->lock);
-	clps711x_clk->clks[CLPS711X_CLK_TIMER2] =
-		clk_register_divider_table(NULL, "timer2", "timer_ref", 0,
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_TIMER2] =
+		clk_hw_register_divider_table(NULL, "timer2", "timer_ref", 0,
 					   base + CLPS711X_SYSCON1, 7, 1, 0,
 					   timer_div_table, &clps711x_clk->lock);
-	clps711x_clk->clks[CLPS711X_CLK_PWM] =
-		clk_register_fixed_rate(NULL, "pwm", NULL, 0, f_pwm);
-	clps711x_clk->clks[CLPS711X_CLK_SPIREF] =
-		clk_register_fixed_rate(NULL, "spi_ref", NULL, 0, f_spi);
-	clps711x_clk->clks[CLPS711X_CLK_SPI] =
-		clk_register_divider_table(NULL, "spi", "spi_ref", 0,
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_PWM] =
+		clk_hw_register_fixed_rate(NULL, "pwm", NULL, 0, f_pwm);
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_SPIREF] =
+		clk_hw_register_fixed_rate(NULL, "spi_ref", NULL, 0, f_spi);
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_SPI] =
+		clk_hw_register_divider_table(NULL, "spi", "spi_ref", 0,
 					   base + CLPS711X_SYSCON1, 16, 2, 0,
 					   spi_div_table, &clps711x_clk->lock);
-	clps711x_clk->clks[CLPS711X_CLK_UART] =
-		clk_register_fixed_factor(NULL, "uart", "bus", 0, 1, 10);
-	clps711x_clk->clks[CLPS711X_CLK_TICK] =
-		clk_register_fixed_rate(NULL, "tick", NULL, 0, 64);
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_UART] =
+		clk_hw_register_fixed_factor(NULL, "uart", "bus", 0, 1, 10);
+	clps711x_clk->clk_data.hws[CLPS711X_CLK_TICK] =
+		clk_hw_register_fixed_rate(NULL, "tick", NULL, 0, 64);
 	for (i = 0; i < CLPS711X_CLK_MAX; i++)
-		if (IS_ERR(clps711x_clk->clks[i]))
+		if (IS_ERR(clps711x_clk->clk_data.hws[i]))
 			pr_err("clk %i: register failed with %ld\n",
-			       i, PTR_ERR(clps711x_clk->clks[i]));
+			       i, PTR_ERR(clps711x_clk->clk_data.hws[i]));
 
 	return clps711x_clk;
 }
@@ -153,17 +154,17 @@ void __init clps711x_clk_init(void __iomem *base)
 	BUG_ON(IS_ERR(clps711x_clk));
 
 	/* Clocksource */
-	clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_TIMER1],
+	clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_TIMER1],
 			    NULL, "clps711x-timer.0");
-	clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_TIMER2],
+	clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_TIMER2],
 			    NULL, "clps711x-timer.1");
 
 	/* Drivers */
-	clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_PWM],
+	clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_PWM],
 			    NULL, "clps711x-pwm");
-	clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_UART],
+	clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_UART],
 			    NULL, "clps711x-uart.0");
-	clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_UART],
+	clk_hw_register_clkdev(clps711x_clk->clk_data.hws[CLPS711X_CLK_UART],
 			    NULL, "clps711x-uart.1");
 }
 
@@ -179,10 +180,9 @@ static void __init clps711x_clk_init_dt(struct device_node *np)
 	clps711x_clk = _clps711x_clk_init(base, fref);
 	BUG_ON(IS_ERR(clps711x_clk));
 
-	clps711x_clk->clk_data.clks = clps711x_clk->clks;
-	clps711x_clk->clk_data.clk_num = CLPS711X_CLK_MAX;
-	of_clk_add_provider(np, of_clk_src_onecell_get,
-			    &clps711x_clk->clk_data);
+	clps711x_clk->clk_data.num = CLPS711X_CLK_MAX;
+	of_clk_add_hw_provider(np, of_clk_hw_onecell_get,
+			       &clps711x_clk->clk_data);
 }
 CLK_OF_DECLARE(clps711x, "cirrus,ep7209-clk", clps711x_clk_init_dt);
 #endif
diff --git a/drivers/clk/clk-cs2000-cp.c b/drivers/clk/clk-cs2000-cp.c
index 7379de8dc894..021f3daf34e1 100644
--- a/drivers/clk/clk-cs2000-cp.c
+++ b/drivers/clk/clk-cs2000-cp.c
@@ -59,7 +59,6 @@ struct cs2000_priv {
 	struct i2c_client *client;
 	struct clk *clk_in;
 	struct clk *ref_clk;
-	struct clk *clk_out;
 };
 
 static const struct of_device_id cs2000_of_match[] = {
@@ -371,7 +370,6 @@ static int cs2000_clk_register(struct cs2000_priv *priv)
 	struct device_node *np = dev->of_node;
 	struct clk_init_data init;
 	const char *name = np->name;
-	struct clk *clk;
 	static const char *parent_names[CLK_MAX];
 	int ch = 0; /* it uses ch0 only at this point */
 	int rate;
@@ -400,18 +398,16 @@ static int cs2000_clk_register(struct cs2000_priv *priv)
 
 	priv->hw.init = &init;
 
-	clk = clk_register(dev, &priv->hw);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = clk_hw_register(dev, &priv->hw);
+	if (ret)
+		return ret;
 
-	ret = of_clk_add_provider(np, of_clk_src_simple_get, clk);
+	ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, &priv->hw);
 	if (ret < 0) {
-		clk_unregister(clk);
+		clk_hw_unregister(&priv->hw);
 		return ret;
 	}
 
-	priv->clk_out = clk;
-
 	return 0;
 }
 
@@ -454,7 +450,7 @@ static int cs2000_remove(struct i2c_client *client)
 
 	of_clk_del_provider(np);
 
-	clk_unregister(priv->clk_out);
+	clk_hw_unregister(&priv->hw);
 
 	return 0;
 }
diff --git a/drivers/clk/clk-divider.c b/drivers/clk/clk-divider.c
index a0f55bc1ad3d..96386ffc8483 100644
--- a/drivers/clk/clk-divider.c
+++ b/drivers/clk/clk-divider.c
@@ -352,7 +352,7 @@ static long clk_divider_round_rate(struct clk_hw *hw, unsigned long rate,
 
 	/* if read only, just return current value */
 	if (divider->flags & CLK_DIVIDER_READ_ONLY) {
-		bestdiv = readl(divider->reg) >> divider->shift;
+		bestdiv = clk_readl(divider->reg) >> divider->shift;
 		bestdiv &= div_mask(divider->width);
 		bestdiv = _get_div(divider->table, bestdiv, divider->flags,
 			divider->width);
diff --git a/drivers/clk/clk-efm32gg.c b/drivers/clk/clk-efm32gg.c
index 22e4c659704e..8802a2dd56ac 100644
--- a/drivers/clk/clk-efm32gg.c
+++ b/drivers/clk/clk-efm32gg.c
@@ -10,24 +10,31 @@
 #include <linux/clk-provider.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/slab.h>
 
 #include <dt-bindings/clock/efm32-cmu.h>
 
 #define CMU_HFPERCLKEN0		0x44
+#define CMU_MAX_CLKS		37
 
-static struct clk *clk[37];
-static struct clk_onecell_data clk_data = {
-	.clks = clk,
-	.clk_num = ARRAY_SIZE(clk),
-};
+static struct clk_hw_onecell_data *clk_data;
 
 static void __init efm32gg_cmu_init(struct device_node *np)
 {
 	int i;
 	void __iomem *base;
+	struct clk_hw **hws;
 
-	for (i = 0; i < ARRAY_SIZE(clk); ++i)
-		clk[i] = ERR_PTR(-ENOENT);
+	clk_data = kzalloc(sizeof(*clk_data) +
+			   sizeof(*clk_data->hws) * CMU_MAX_CLKS, GFP_KERNEL);
+
+	if (!clk_data)
+		return;
+
+	hws = clk_data->hws;
+
+	for (i = 0; i < CMU_MAX_CLKS; ++i)
+		hws[i] = ERR_PTR(-ENOENT);
 
 	base = of_iomap(np, 0);
 	if (!base) {
@@ -35,46 +42,46 @@ static void __init efm32gg_cmu_init(struct device_node *np)
 		return;
 	}
 
-	clk[clk_HFXO] = clk_register_fixed_rate(NULL, "HFXO", NULL,
-			0, 48000000);
+	hws[clk_HFXO] = clk_hw_register_fixed_rate(NULL, "HFXO", NULL, 0,
+						   48000000);
 
-	clk[clk_HFPERCLKUSART0] = clk_register_gate(NULL, "HFPERCLK.USART0",
+	hws[clk_HFPERCLKUSART0] = clk_hw_register_gate(NULL, "HFPERCLK.USART0",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 0, 0, NULL);
-	clk[clk_HFPERCLKUSART1] = clk_register_gate(NULL, "HFPERCLK.USART1",
+	hws[clk_HFPERCLKUSART1] = clk_hw_register_gate(NULL, "HFPERCLK.USART1",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 1, 0, NULL);
-	clk[clk_HFPERCLKUSART2] = clk_register_gate(NULL, "HFPERCLK.USART2",
+	hws[clk_HFPERCLKUSART2] = clk_hw_register_gate(NULL, "HFPERCLK.USART2",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 2, 0, NULL);
-	clk[clk_HFPERCLKUART0] = clk_register_gate(NULL, "HFPERCLK.UART0",
+	hws[clk_HFPERCLKUART0] = clk_hw_register_gate(NULL, "HFPERCLK.UART0",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 3, 0, NULL);
-	clk[clk_HFPERCLKUART1] = clk_register_gate(NULL, "HFPERCLK.UART1",
+	hws[clk_HFPERCLKUART1] = clk_hw_register_gate(NULL, "HFPERCLK.UART1",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 4, 0, NULL);
-	clk[clk_HFPERCLKTIMER0] = clk_register_gate(NULL, "HFPERCLK.TIMER0",
+	hws[clk_HFPERCLKTIMER0] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER0",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 5, 0, NULL);
-	clk[clk_HFPERCLKTIMER1] = clk_register_gate(NULL, "HFPERCLK.TIMER1",
+	hws[clk_HFPERCLKTIMER1] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER1",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 6, 0, NULL);
-	clk[clk_HFPERCLKTIMER2] = clk_register_gate(NULL, "HFPERCLK.TIMER2",
+	hws[clk_HFPERCLKTIMER2] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER2",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 7, 0, NULL);
-	clk[clk_HFPERCLKTIMER3] = clk_register_gate(NULL, "HFPERCLK.TIMER3",
+	hws[clk_HFPERCLKTIMER3] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER3",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 8, 0, NULL);
-	clk[clk_HFPERCLKACMP0] = clk_register_gate(NULL, "HFPERCLK.ACMP0",
+	hws[clk_HFPERCLKACMP0] = clk_hw_register_gate(NULL, "HFPERCLK.ACMP0",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 9, 0, NULL);
-	clk[clk_HFPERCLKACMP1] = clk_register_gate(NULL, "HFPERCLK.ACMP1",
+	hws[clk_HFPERCLKACMP1] = clk_hw_register_gate(NULL, "HFPERCLK.ACMP1",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 10, 0, NULL);
-	clk[clk_HFPERCLKI2C0] = clk_register_gate(NULL, "HFPERCLK.I2C0",
+	hws[clk_HFPERCLKI2C0] = clk_hw_register_gate(NULL, "HFPERCLK.I2C0",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 11, 0, NULL);
-	clk[clk_HFPERCLKI2C1] = clk_register_gate(NULL, "HFPERCLK.I2C1",
+	hws[clk_HFPERCLKI2C1] = clk_hw_register_gate(NULL, "HFPERCLK.I2C1",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 12, 0, NULL);
-	clk[clk_HFPERCLKGPIO] = clk_register_gate(NULL, "HFPERCLK.GPIO",
+	hws[clk_HFPERCLKGPIO] = clk_hw_register_gate(NULL, "HFPERCLK.GPIO",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 13, 0, NULL);
-	clk[clk_HFPERCLKVCMP] = clk_register_gate(NULL, "HFPERCLK.VCMP",
+	hws[clk_HFPERCLKVCMP] = clk_hw_register_gate(NULL, "HFPERCLK.VCMP",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 14, 0, NULL);
-	clk[clk_HFPERCLKPRS] = clk_register_gate(NULL, "HFPERCLK.PRS",
+	hws[clk_HFPERCLKPRS] = clk_hw_register_gate(NULL, "HFPERCLK.PRS",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 15, 0, NULL);
-	clk[clk_HFPERCLKADC0] = clk_register_gate(NULL, "HFPERCLK.ADC0",
+	hws[clk_HFPERCLKADC0] = clk_hw_register_gate(NULL, "HFPERCLK.ADC0",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 16, 0, NULL);
-	clk[clk_HFPERCLKDAC0] = clk_register_gate(NULL, "HFPERCLK.DAC0",
+	hws[clk_HFPERCLKDAC0] = clk_hw_register_gate(NULL, "HFPERCLK.DAC0",
 			"HFXO", 0, base + CMU_HFPERCLKEN0, 17, 0, NULL);
 
-	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
+	of_clk_add_hw_provider(np, of_clk_hw_onecell_get, &clk_data);
 }
 CLK_OF_DECLARE(efm32ggcmu, "efm32gg,cmu", efm32gg_cmu_init);
diff --git a/drivers/clk/clk-fixed-factor.c b/drivers/clk/clk-fixed-factor.c
index 4db3be214077..a5d402de5584 100644
--- a/drivers/clk/clk-fixed-factor.c
+++ b/drivers/clk/clk-fixed-factor.c
@@ -12,6 +12,7 @@
 #include <linux/slab.h>
 #include <linux/err.h>
 #include <linux/of.h>
+#include <linux/platform_device.h>
 
 /*
  * DOC: basic fixed multiplier and divider clock that cannot gate
@@ -147,27 +148,25 @@ static const struct of_device_id set_rate_parent_matches[] = {
 	{ /* Sentinel */ },
 };
 
-/**
- * of_fixed_factor_clk_setup() - Setup function for simple fixed factor clock
- */
-void __init of_fixed_factor_clk_setup(struct device_node *node)
+static struct clk *_of_fixed_factor_clk_setup(struct device_node *node)
 {
 	struct clk *clk;
 	const char *clk_name = node->name;
 	const char *parent_name;
 	unsigned long flags = 0;
 	u32 div, mult;
+	int ret;
 
 	if (of_property_read_u32(node, "clock-div", &div)) {
 		pr_err("%s Fixed factor clock <%s> must have a clock-div property\n",
 			__func__, node->name);
-		return;
+		return ERR_PTR(-EIO);
 	}
 
 	if (of_property_read_u32(node, "clock-mult", &mult)) {
 		pr_err("%s Fixed factor clock <%s> must have a clock-mult property\n",
 			__func__, node->name);
-		return;
+		return ERR_PTR(-EIO);
 	}
 
 	of_property_read_string(node, "clock-output-names", &clk_name);
@@ -178,10 +177,67 @@ void __init of_fixed_factor_clk_setup(struct device_node *node)
 
 	clk = clk_register_fixed_factor(NULL, clk_name, parent_name, flags,
 					mult, div);
-	if (!IS_ERR(clk))
-		of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (IS_ERR(clk))
+		return clk;
+
+	ret = of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (ret) {
+		clk_unregister(clk);
+		return ERR_PTR(ret);
+	}
+
+	return clk;
+}
+
+/**
+ * of_fixed_factor_clk_setup() - Setup function for simple fixed factor clock
+ */
+void __init of_fixed_factor_clk_setup(struct device_node *node)
+{
+	_of_fixed_factor_clk_setup(node);
 }
-EXPORT_SYMBOL_GPL(of_fixed_factor_clk_setup);
 CLK_OF_DECLARE(fixed_factor_clk, "fixed-factor-clock",
 		of_fixed_factor_clk_setup);
+
+static int of_fixed_factor_clk_remove(struct platform_device *pdev)
+{
+	struct clk *clk = platform_get_drvdata(pdev);
+
+	clk_unregister_fixed_factor(clk);
+
+	return 0;
+}
+
+static int of_fixed_factor_clk_probe(struct platform_device *pdev)
+{
+	struct clk *clk;
+
+	/*
+	 * This function is not executed when of_fixed_factor_clk_setup
+	 * succeeded.
+	 */
+	clk = _of_fixed_factor_clk_setup(pdev->dev.of_node);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	platform_set_drvdata(pdev, clk);
+
+	return 0;
+}
+
+static const struct of_device_id of_fixed_factor_clk_ids[] = {
+	{ .compatible = "fixed-factor-clock" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, of_fixed_factor_clk_ids);
+
+static struct platform_driver of_fixed_factor_clk_driver = {
+	.driver = {
+		.name = "of_fixed_factor_clk",
+		.of_match_table = of_fixed_factor_clk_ids,
+	},
+	.probe = of_fixed_factor_clk_probe,
+	.remove = of_fixed_factor_clk_remove,
+};
+builtin_platform_driver(of_fixed_factor_clk_driver);
 #endif
diff --git a/drivers/clk/clk-fixed-rate.c b/drivers/clk/clk-fixed-rate.c
index 2edb39342a02..b5c46b3f8764 100644
--- a/drivers/clk/clk-fixed-rate.c
+++ b/drivers/clk/clk-fixed-rate.c
@@ -15,6 +15,7 @@
 #include <linux/io.h>
 #include <linux/err.h>
 #include <linux/of.h>
+#include <linux/platform_device.h>
 
 /*
  * DOC: basic fixed-rate clock that cannot gate
@@ -157,18 +158,16 @@ void clk_hw_unregister_fixed_rate(struct clk_hw *hw)
 EXPORT_SYMBOL_GPL(clk_hw_unregister_fixed_rate);
 
 #ifdef CONFIG_OF
-/**
- * of_fixed_clk_setup() - Setup function for simple fixed rate clock
- */
-void of_fixed_clk_setup(struct device_node *node)
+static struct clk *_of_fixed_clk_setup(struct device_node *node)
 {
 	struct clk *clk;
 	const char *clk_name = node->name;
 	u32 rate;
 	u32 accuracy = 0;
+	int ret;
 
 	if (of_property_read_u32(node, "clock-frequency", &rate))
-		return;
+		return ERR_PTR(-EIO);
 
 	of_property_read_u32(node, "clock-accuracy", &accuracy);
 
@@ -176,9 +175,66 @@ void of_fixed_clk_setup(struct device_node *node)
 
 	clk = clk_register_fixed_rate_with_accuracy(NULL, clk_name, NULL,
 						    0, rate, accuracy);
-	if (!IS_ERR(clk))
-		of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (IS_ERR(clk))
+		return clk;
+
+	ret = of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (ret) {
+		clk_unregister(clk);
+		return ERR_PTR(ret);
+	}
+
+	return clk;
+}
+
+/**
+ * of_fixed_clk_setup() - Setup function for simple fixed rate clock
+ */
+void __init of_fixed_clk_setup(struct device_node *node)
+{
+	_of_fixed_clk_setup(node);
 }
-EXPORT_SYMBOL_GPL(of_fixed_clk_setup);
 CLK_OF_DECLARE(fixed_clk, "fixed-clock", of_fixed_clk_setup);
+
+static int of_fixed_clk_remove(struct platform_device *pdev)
+{
+	struct clk *clk = platform_get_drvdata(pdev);
+
+	clk_unregister_fixed_rate(clk);
+
+	return 0;
+}
+
+static int of_fixed_clk_probe(struct platform_device *pdev)
+{
+	struct clk *clk;
+
+	/*
+	 * This function is not executed when of_fixed_clk_setup
+	 * succeeded.
+	 */
+	clk = _of_fixed_clk_setup(pdev->dev.of_node);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	platform_set_drvdata(pdev, clk);
+
+	return 0;
+}
+
+static const struct of_device_id of_fixed_clk_ids[] = {
+	{ .compatible = "fixed-clock" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, of_fixed_clk_ids);
+
+static struct platform_driver of_fixed_clk_driver = {
+	.driver = {
+		.name = "of_fixed_clk",
+		.of_match_table = of_fixed_clk_ids,
+	},
+	.probe = of_fixed_clk_probe,
+	.remove = of_fixed_clk_remove,
+};
+builtin_platform_driver(of_fixed_clk_driver);
 #endif
diff --git a/drivers/clk/clk-ls1x.c b/drivers/clk/clk-ls1x.c
deleted file mode 100644
index 5097831387ff..000000000000
--- a/drivers/clk/clk-ls1x.c
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Copyright (c) 2012 Zhang, Keguang <keguang.zhang@gmail.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#include <linux/clkdev.h>
-#include <linux/clk-provider.h>
-#include <linux/io.h>
-#include <linux/slab.h>
-#include <linux/err.h>
-
-#include <loongson1.h>
-
-#define OSC		(33 * 1000000)
-#define DIV_APB		2
-
-static DEFINE_SPINLOCK(_lock);
-
-static int ls1x_pll_clk_enable(struct clk_hw *hw)
-{
-	return 0;
-}
-
-static void ls1x_pll_clk_disable(struct clk_hw *hw)
-{
-}
-
-static unsigned long ls1x_pll_recalc_rate(struct clk_hw *hw,
-					  unsigned long parent_rate)
-{
-	u32 pll, rate;
-
-	pll = __raw_readl(LS1X_CLK_PLL_FREQ);
-	rate = 12 + (pll & 0x3f) + (((pll >> 8) & 0x3ff) >> 10);
-	rate *= OSC;
-	rate >>= 1;
-
-	return rate;
-}
-
-static const struct clk_ops ls1x_pll_clk_ops = {
-	.enable = ls1x_pll_clk_enable,
-	.disable = ls1x_pll_clk_disable,
-	.recalc_rate = ls1x_pll_recalc_rate,
-};
-
-static struct clk *__init clk_register_pll(struct device *dev,
-					   const char *name,
-					   const char *parent_name,
-					   unsigned long flags)
-{
-	struct clk_hw *hw;
-	struct clk *clk;
-	struct clk_init_data init;
-
-	/* allocate the divider */
-	hw = kzalloc(sizeof(struct clk_hw), GFP_KERNEL);
-	if (!hw) {
-		pr_err("%s: could not allocate clk_hw\n", __func__);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	init.name = name;
-	init.ops = &ls1x_pll_clk_ops;
-	init.flags = flags | CLK_IS_BASIC;
-	init.parent_names = (parent_name ? &parent_name : NULL);
-	init.num_parents = (parent_name ? 1 : 0);
-	hw->init = &init;
-
-	/* register the clock */
-	clk = clk_register(dev, hw);
-
-	if (IS_ERR(clk))
-		kfree(hw);
-
-	return clk;
-}
-
-static const char * const cpu_parents[] = { "cpu_clk_div", "osc_33m_clk", };
-static const char * const ahb_parents[] = { "ahb_clk_div", "osc_33m_clk", };
-static const char * const dc_parents[] = { "dc_clk_div", "osc_33m_clk", };
-
-void __init ls1x_clk_init(void)
-{
-	struct clk *clk;
-
-	clk = clk_register_fixed_rate(NULL, "osc_33m_clk", NULL, 0, OSC);
-	clk_register_clkdev(clk, "osc_33m_clk", NULL);
-
-	/* clock derived from 33 MHz OSC clk */
-	clk = clk_register_pll(NULL, "pll_clk", "osc_33m_clk", 0);
-	clk_register_clkdev(clk, "pll_clk", NULL);
-
-	/* clock derived from PLL clk */
-	/*                                 _____
-	 *         _______________________|     |
-	 * OSC ___/                       | MUX |___ CPU CLK
-	 *        \___ PLL ___ CPU DIV ___|     |
-	 *                                |_____|
-	 */
-	clk = clk_register_divider(NULL, "cpu_clk_div", "pll_clk",
-				   CLK_GET_RATE_NOCACHE, LS1X_CLK_PLL_DIV,
-				   DIV_CPU_SHIFT, DIV_CPU_WIDTH,
-				   CLK_DIVIDER_ONE_BASED |
-				   CLK_DIVIDER_ROUND_CLOSEST, &_lock);
-	clk_register_clkdev(clk, "cpu_clk_div", NULL);
-	clk = clk_register_mux(NULL, "cpu_clk", cpu_parents,
-			       ARRAY_SIZE(cpu_parents),
-			       CLK_SET_RATE_NO_REPARENT, LS1X_CLK_PLL_DIV,
-			       BYPASS_CPU_SHIFT, BYPASS_CPU_WIDTH, 0, &_lock);
-	clk_register_clkdev(clk, "cpu_clk", NULL);
-
-	/*                                 _____
-	 *         _______________________|     |
-	 * OSC ___/                       | MUX |___ DC  CLK
-	 *        \___ PLL ___ DC  DIV ___|     |
-	 *                                |_____|
-	 */
-	clk = clk_register_divider(NULL, "dc_clk_div", "pll_clk",
-				   0, LS1X_CLK_PLL_DIV, DIV_DC_SHIFT,
-				   DIV_DC_WIDTH, CLK_DIVIDER_ONE_BASED, &_lock);
-	clk_register_clkdev(clk, "dc_clk_div", NULL);
-	clk = clk_register_mux(NULL, "dc_clk", dc_parents,
-			       ARRAY_SIZE(dc_parents),
-			       CLK_SET_RATE_NO_REPARENT, LS1X_CLK_PLL_DIV,
-			       BYPASS_DC_SHIFT, BYPASS_DC_WIDTH, 0, &_lock);
-	clk_register_clkdev(clk, "dc_clk", NULL);
-
-	/*                                 _____
-	 *         _______________________|     |
-	 * OSC ___/                       | MUX |___ DDR CLK
-	 *        \___ PLL ___ DDR DIV ___|     |
-	 *                                |_____|
-	 */
-	clk = clk_register_divider(NULL, "ahb_clk_div", "pll_clk",
-				   0, LS1X_CLK_PLL_DIV, DIV_DDR_SHIFT,
-				   DIV_DDR_WIDTH, CLK_DIVIDER_ONE_BASED,
-				   &_lock);
-	clk_register_clkdev(clk, "ahb_clk_div", NULL);
-	clk = clk_register_mux(NULL, "ahb_clk", ahb_parents,
-			       ARRAY_SIZE(ahb_parents),
-			       CLK_SET_RATE_NO_REPARENT, LS1X_CLK_PLL_DIV,
-			       BYPASS_DDR_SHIFT, BYPASS_DDR_WIDTH, 0, &_lock);
-	clk_register_clkdev(clk, "ahb_clk", NULL);
-	clk_register_clkdev(clk, "stmmaceth", NULL);
-
-	/* clock derived from AHB clk */
-	/* APB clk is always half of the AHB clk */
-	clk = clk_register_fixed_factor(NULL, "apb_clk", "ahb_clk", 0, 1,
-					DIV_APB);
-	clk_register_clkdev(clk, "apb_clk", NULL);
-	clk_register_clkdev(clk, "ls1x_i2c", NULL);
-	clk_register_clkdev(clk, "ls1x_pwmtimer", NULL);
-	clk_register_clkdev(clk, "ls1x_spi", NULL);
-	clk_register_clkdev(clk, "ls1x_wdt", NULL);
-	clk_register_clkdev(clk, "serial8250", NULL);
-}
diff --git a/drivers/clk/clk-max-gen.c b/drivers/clk/clk-max-gen.c
deleted file mode 100644
index 35af9cb6da4f..000000000000
--- a/drivers/clk/clk-max-gen.c
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * clk-max-gen.c - Generic clock driver for Maxim PMICs clocks
- *
- * Copyright (C) 2014 Google, Inc
- *
- * Copyright (C) 2012 Samsung Electornics
- * Jonghwa Lee <jonghwa3.lee@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * This driver is based on clk-max77686.c
- *
- */
-
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/err.h>
-#include <linux/regmap.h>
-#include <linux/platform_device.h>
-#include <linux/clk-provider.h>
-#include <linux/mutex.h>
-#include <linux/clkdev.h>
-#include <linux/of.h>
-#include <linux/export.h>
-
-#include "clk-max-gen.h"
-
-struct max_gen_clk {
-	struct regmap *regmap;
-	u32 mask;
-	u32 reg;
-	struct clk_hw hw;
-};
-
-static struct max_gen_clk *to_max_gen_clk(struct clk_hw *hw)
-{
-	return container_of(hw, struct max_gen_clk, hw);
-}
-
-static int max_gen_clk_prepare(struct clk_hw *hw)
-{
-	struct max_gen_clk *max_gen = to_max_gen_clk(hw);
-
-	return regmap_update_bits(max_gen->regmap, max_gen->reg,
-				  max_gen->mask, max_gen->mask);
-}
-
-static void max_gen_clk_unprepare(struct clk_hw *hw)
-{
-	struct max_gen_clk *max_gen = to_max_gen_clk(hw);
-
-	regmap_update_bits(max_gen->regmap, max_gen->reg,
-			   max_gen->mask, ~max_gen->mask);
-}
-
-static int max_gen_clk_is_prepared(struct clk_hw *hw)
-{
-	struct max_gen_clk *max_gen = to_max_gen_clk(hw);
-	int ret;
-	u32 val;
-
-	ret = regmap_read(max_gen->regmap, max_gen->reg, &val);
-
-	if (ret < 0)
-		return -EINVAL;
-
-	return val & max_gen->mask;
-}
-
-static unsigned long max_gen_recalc_rate(struct clk_hw *hw,
-					 unsigned long parent_rate)
-{
-	return 32768;
-}
-
-struct clk_ops max_gen_clk_ops = {
-	.prepare	= max_gen_clk_prepare,
-	.unprepare	= max_gen_clk_unprepare,
-	.is_prepared	= max_gen_clk_is_prepared,
-	.recalc_rate	= max_gen_recalc_rate,
-};
-EXPORT_SYMBOL_GPL(max_gen_clk_ops);
-
-static struct clk *max_gen_clk_register(struct device *dev,
-					struct max_gen_clk *max_gen)
-{
-	struct clk *clk;
-	struct clk_hw *hw = &max_gen->hw;
-	int ret;
-
-	clk = devm_clk_register(dev, hw);
-	if (IS_ERR(clk))
-		return clk;
-
-	ret = clk_register_clkdev(clk, hw->init->name, NULL);
-
-	if (ret)
-		return ERR_PTR(ret);
-
-	return clk;
-}
-
-int max_gen_clk_probe(struct platform_device *pdev, struct regmap *regmap,
-		      u32 reg, struct clk_init_data *clks_init, int num_init)
-{
-	int i, ret;
-	struct max_gen_clk *max_gen_clks;
-	struct clk **clocks;
-	struct device *dev = pdev->dev.parent;
-	const char *clk_name;
-	struct clk_init_data *init;
-
-	clocks = devm_kzalloc(dev, sizeof(struct clk *) * num_init, GFP_KERNEL);
-	if (!clocks)
-		return -ENOMEM;
-
-	max_gen_clks = devm_kzalloc(dev, sizeof(struct max_gen_clk)
-				    * num_init, GFP_KERNEL);
-	if (!max_gen_clks)
-		return -ENOMEM;
-
-	for (i = 0; i < num_init; i++) {
-		max_gen_clks[i].regmap = regmap;
-		max_gen_clks[i].mask = 1 << i;
-		max_gen_clks[i].reg = reg;
-
-		init = devm_kzalloc(dev, sizeof(*init), GFP_KERNEL);
-		if (!init)
-			return -ENOMEM;
-
-		if (dev->of_node &&
-		    !of_property_read_string_index(dev->of_node,
-						   "clock-output-names",
-						   i, &clk_name))
-			init->name = clk_name;
-		else
-			init->name = clks_init[i].name;
-
-		init->ops = clks_init[i].ops;
-		init->flags = clks_init[i].flags;
-
-		max_gen_clks[i].hw.init = init;
-
-		clocks[i] = max_gen_clk_register(dev, &max_gen_clks[i]);
-		if (IS_ERR(clocks[i])) {
-			ret = PTR_ERR(clocks[i]);
-			dev_err(dev, "failed to register %s\n",
-				max_gen_clks[i].hw.init->name);
-			return ret;
-		}
-	}
-
-	platform_set_drvdata(pdev, clocks);
-
-	if (dev->of_node) {
-		struct clk_onecell_data *of_data;
-
-		of_data = devm_kzalloc(dev, sizeof(*of_data), GFP_KERNEL);
-		if (!of_data)
-			return -ENOMEM;
-
-		of_data->clks = clocks;
-		of_data->clk_num = num_init;
-		ret = of_clk_add_provider(dev->of_node, of_clk_src_onecell_get,
-					  of_data);
-
-		if (ret) {
-			dev_err(dev, "failed to register OF clock provider\n");
-			return ret;
-		}
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(max_gen_clk_probe);
-
-int max_gen_clk_remove(struct platform_device *pdev, int num_init)
-{
-	struct device *dev = pdev->dev.parent;
-
-	if (dev->of_node)
-		of_clk_del_provider(dev->of_node);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(max_gen_clk_remove);
diff --git a/drivers/clk/clk-max-gen.h b/drivers/clk/clk-max-gen.h
deleted file mode 100644
index 997e86fc3f4d..000000000000
--- a/drivers/clk/clk-max-gen.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * clk-max-gen.h - Generic clock driver for Maxim PMICs clocks
- *
- * Copyright (C) 2014 Google, Inc
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#ifndef __CLK_MAX_GEN_H__
-#define __CLK_MAX_GEN_H__
-
-#include <linux/types.h>
-#include <linux/device.h>
-#include <linux/clkdev.h>
-#include <linux/regmap.h>
-#include <linux/platform_device.h>
-
-int max_gen_clk_probe(struct platform_device *pdev, struct regmap *regmap,
-		      u32 reg, struct clk_init_data *clks_init, int num_init);
-int max_gen_clk_remove(struct platform_device *pdev, int num_init);
-extern struct clk_ops max_gen_clk_ops;
-
-#endif /* __CLK_MAX_GEN_H__ */
diff --git a/drivers/clk/clk-max77686.c b/drivers/clk/clk-max77686.c
index 9b6f2772e948..b637f5979023 100644
--- a/drivers/clk/clk-max77686.c
+++ b/drivers/clk/clk-max77686.c
@@ -1,5 +1,5 @@
 /*
- * clk-max77686.c - Clock driver for Maxim 77686
+ * clk-max77686.c - Clock driver for Maxim 77686/MAX77802
  *
  * Copyright (C) 2012 Samsung Electornics
  * Jonghwa Lee <jonghwa3.lee@samsung.com>
@@ -25,46 +25,284 @@
 #include <linux/err.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/mfd/max77620.h>
 #include <linux/mfd/max77686.h>
 #include <linux/mfd/max77686-private.h>
 #include <linux/clk-provider.h>
 #include <linux/mutex.h>
 #include <linux/clkdev.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
 
 #include <dt-bindings/clock/maxim,max77686.h>
-#include "clk-max-gen.h"
+#include <dt-bindings/clock/maxim,max77802.h>
+#include <dt-bindings/clock/maxim,max77620.h>
 
-static struct clk_init_data max77686_clks_init[MAX77686_CLKS_NUM] = {
+#define MAX77802_CLOCK_LOW_JITTER_SHIFT 0x3
+
+enum max77686_chip_name {
+	CHIP_MAX77686,
+	CHIP_MAX77802,
+	CHIP_MAX77620,
+};
+
+struct max77686_hw_clk_info {
+	const char *name;
+	u32 clk_reg;
+	u32 clk_enable_mask;
+	u32 flags;
+};
+
+struct max77686_clk_init_data {
+	struct regmap *regmap;
+	struct clk_hw hw;
+	struct clk_init_data clk_idata;
+	const struct max77686_hw_clk_info *clk_info;
+};
+
+struct max77686_clk_driver_data {
+	enum max77686_chip_name chip;
+	struct max77686_clk_init_data *max_clk_data;
+	size_t num_clks;
+};
+
+static const struct
+max77686_hw_clk_info max77686_hw_clks_info[MAX77686_CLKS_NUM] = {
 	[MAX77686_CLK_AP] = {
 		.name = "32khz_ap",
-		.ops = &max_gen_clk_ops,
+		.clk_reg = MAX77686_REG_32KHZ,
+		.clk_enable_mask = BIT(MAX77686_CLK_AP),
 	},
 	[MAX77686_CLK_CP] = {
 		.name = "32khz_cp",
-		.ops = &max_gen_clk_ops,
+		.clk_reg = MAX77686_REG_32KHZ,
+		.clk_enable_mask = BIT(MAX77686_CLK_CP),
 	},
 	[MAX77686_CLK_PMIC] = {
 		.name = "32khz_pmic",
-		.ops = &max_gen_clk_ops,
+		.clk_reg = MAX77686_REG_32KHZ,
+		.clk_enable_mask = BIT(MAX77686_CLK_PMIC),
+	},
+};
+
+static const struct
+max77686_hw_clk_info max77802_hw_clks_info[MAX77802_CLKS_NUM] = {
+	[MAX77802_CLK_32K_AP] = {
+		.name = "32khz_ap",
+		.clk_reg = MAX77802_REG_32KHZ,
+		.clk_enable_mask = BIT(MAX77802_CLK_32K_AP),
+	},
+	[MAX77802_CLK_32K_CP] = {
+		.name = "32khz_cp",
+		.clk_reg = MAX77802_REG_32KHZ,
+		.clk_enable_mask = BIT(MAX77802_CLK_32K_CP),
+	},
+};
+
+static const struct
+max77686_hw_clk_info max77620_hw_clks_info[MAX77620_CLKS_NUM] = {
+	[MAX77620_CLK_32K_OUT0] = {
+		.name = "32khz_out0",
+		.clk_reg = MAX77620_REG_CNFG1_32K,
+		.clk_enable_mask = MAX77620_CNFG1_32K_OUT0_EN,
 	},
 };
 
+static struct max77686_clk_init_data *to_max77686_clk_init_data(
+				struct clk_hw *hw)
+{
+	return container_of(hw, struct max77686_clk_init_data, hw);
+}
+
+static int max77686_clk_prepare(struct clk_hw *hw)
+{
+	struct max77686_clk_init_data *max77686 = to_max77686_clk_init_data(hw);
+
+	return regmap_update_bits(max77686->regmap, max77686->clk_info->clk_reg,
+				  max77686->clk_info->clk_enable_mask,
+				  max77686->clk_info->clk_enable_mask);
+}
+
+static void max77686_clk_unprepare(struct clk_hw *hw)
+{
+	struct max77686_clk_init_data *max77686 = to_max77686_clk_init_data(hw);
+
+	regmap_update_bits(max77686->regmap, max77686->clk_info->clk_reg,
+			   max77686->clk_info->clk_enable_mask,
+			   ~max77686->clk_info->clk_enable_mask);
+}
+
+static int max77686_clk_is_prepared(struct clk_hw *hw)
+{
+	struct max77686_clk_init_data *max77686 = to_max77686_clk_init_data(hw);
+	int ret;
+	u32 val;
+
+	ret = regmap_read(max77686->regmap, max77686->clk_info->clk_reg, &val);
+
+	if (ret < 0)
+		return -EINVAL;
+
+	return val & max77686->clk_info->clk_enable_mask;
+}
+
+static unsigned long max77686_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	return 32768;
+}
+
+static struct clk_ops max77686_clk_ops = {
+	.prepare	= max77686_clk_prepare,
+	.unprepare	= max77686_clk_unprepare,
+	.is_prepared	= max77686_clk_is_prepared,
+	.recalc_rate	= max77686_recalc_rate,
+};
+
+static struct clk_hw *
+of_clk_max77686_get(struct of_phandle_args *clkspec, void *data)
+{
+	struct max77686_clk_driver_data *drv_data = data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= drv_data->num_clks) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return &drv_data->max_clk_data[idx].hw;
+}
+
 static int max77686_clk_probe(struct platform_device *pdev)
 {
-	struct max77686_dev *iodev = dev_get_drvdata(pdev->dev.parent);
+	struct device *dev = &pdev->dev;
+	struct device *parent = dev->parent;
+	const struct platform_device_id *id = platform_get_device_id(pdev);
+	struct max77686_clk_driver_data *drv_data;
+	const struct max77686_hw_clk_info *hw_clks;
+	struct regmap *regmap;
+	int i, ret, num_clks;
+
+	drv_data = devm_kzalloc(dev, sizeof(*drv_data), GFP_KERNEL);
+	if (!drv_data)
+		return -ENOMEM;
+
+	regmap = dev_get_regmap(parent, NULL);
+	if (!regmap) {
+		dev_err(dev, "Failed to get rtc regmap\n");
+		return -ENODEV;
+	}
+
+	drv_data->chip = id->driver_data;
+
+	switch (drv_data->chip) {
+	case CHIP_MAX77686:
+		num_clks = MAX77686_CLKS_NUM;
+		hw_clks = max77686_hw_clks_info;
+		break;
+
+	case CHIP_MAX77802:
+		num_clks = MAX77802_CLKS_NUM;
+		hw_clks = max77802_hw_clks_info;
+		break;
+
+	case CHIP_MAX77620:
+		num_clks = MAX77620_CLKS_NUM;
+		hw_clks = max77620_hw_clks_info;
+		break;
 
-	return max_gen_clk_probe(pdev, iodev->regmap, MAX77686_REG_32KHZ,
-				 max77686_clks_init, MAX77686_CLKS_NUM);
+	default:
+		dev_err(dev, "Unknown Chip ID\n");
+		return -EINVAL;
+	}
+
+	drv_data->max_clk_data = devm_kcalloc(dev, num_clks,
+					      sizeof(*drv_data->max_clk_data),
+					      GFP_KERNEL);
+	if (!drv_data->max_clk_data)
+		return -ENOMEM;
+
+	for (i = 0; i < num_clks; i++) {
+		struct max77686_clk_init_data *max_clk_data;
+		const char *clk_name;
+
+		max_clk_data = &drv_data->max_clk_data[i];
+
+		max_clk_data->regmap = regmap;
+		max_clk_data->clk_info = &hw_clks[i];
+		max_clk_data->clk_idata.flags = hw_clks[i].flags;
+		max_clk_data->clk_idata.ops = &max77686_clk_ops;
+
+		if (parent->of_node &&
+		    !of_property_read_string_index(parent->of_node,
+						   "clock-output-names",
+						   i, &clk_name))
+			max_clk_data->clk_idata.name = clk_name;
+		else
+			max_clk_data->clk_idata.name = hw_clks[i].name;
+
+		max_clk_data->hw.init = &max_clk_data->clk_idata;
+
+		ret = devm_clk_hw_register(dev, &max_clk_data->hw);
+		if (ret) {
+			dev_err(dev, "Failed to clock register: %d\n", ret);
+			return ret;
+		}
+
+		ret = clk_hw_register_clkdev(&max_clk_data->hw,
+					     max_clk_data->clk_idata.name, NULL);
+		if (ret < 0) {
+			dev_err(dev, "Failed to clkdev register: %d\n", ret);
+			return ret;
+		}
+	}
+
+	if (parent->of_node) {
+		ret = of_clk_add_hw_provider(parent->of_node, of_clk_max77686_get,
+					     drv_data);
+
+		if (ret < 0) {
+			dev_err(dev, "Failed to register OF clock provider: %d\n",
+				ret);
+			return ret;
+		}
+	}
+
+	/* MAX77802: Enable low-jitter mode on the 32khz clocks. */
+	if (drv_data->chip == CHIP_MAX77802) {
+		ret = regmap_update_bits(regmap, MAX77802_REG_32KHZ,
+					 1 << MAX77802_CLOCK_LOW_JITTER_SHIFT,
+					 1 << MAX77802_CLOCK_LOW_JITTER_SHIFT);
+		if (ret < 0) {
+			dev_err(dev, "Failed to config low-jitter: %d\n", ret);
+			goto remove_of_clk_provider;
+		}
+	}
+
+	return 0;
+
+remove_of_clk_provider:
+	if (parent->of_node)
+		of_clk_del_provider(parent->of_node);
+
+	return ret;
 }
 
 static int max77686_clk_remove(struct platform_device *pdev)
 {
-	return max_gen_clk_remove(pdev, MAX77686_CLKS_NUM);
+	struct device *parent = pdev->dev.parent;
+
+	if (parent->of_node)
+		of_clk_del_provider(parent->of_node);
+
+	return 0;
 }
 
 static const struct platform_device_id max77686_clk_id[] = {
-	{ "max77686-clk", 0},
-	{ },
+	{ "max77686-clk", .driver_data = CHIP_MAX77686, },
+	{ "max77802-clk", .driver_data = CHIP_MAX77802, },
+	{ "max77620-clock", .driver_data = CHIP_MAX77620, },
+	{},
 };
 MODULE_DEVICE_TABLE(platform, max77686_clk_id);
 
diff --git a/drivers/clk/clk-max77802.c b/drivers/clk/clk-max77802.c
deleted file mode 100644
index 355dd2e522c3..000000000000
--- a/drivers/clk/clk-max77802.c
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * clk-max77802.c - Clock driver for Maxim 77802
- *
- * Copyright (C) 2014 Google, Inc
- *
- * Copyright (C) 2012 Samsung Electornics
- * Jonghwa Lee <jonghwa3.lee@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * This driver is based on clk-max77686.c
- */
-
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/err.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/mfd/max77686-private.h>
-#include <linux/clk-provider.h>
-#include <linux/mutex.h>
-#include <linux/clkdev.h>
-
-#include <dt-bindings/clock/maxim,max77802.h>
-#include "clk-max-gen.h"
-
-#define MAX77802_CLOCK_OPMODE_MASK	0x1
-#define MAX77802_CLOCK_LOW_JITTER_SHIFT 0x3
-
-static struct clk_init_data max77802_clks_init[MAX77802_CLKS_NUM] = {
-	[MAX77802_CLK_32K_AP] = {
-		.name = "32khz_ap",
-		.ops = &max_gen_clk_ops,
-	},
-	[MAX77802_CLK_32K_CP] = {
-		.name = "32khz_cp",
-		.ops = &max_gen_clk_ops,
-	},
-};
-
-static int max77802_clk_probe(struct platform_device *pdev)
-{
-	struct max77686_dev *iodev = dev_get_drvdata(pdev->dev.parent);
-	int ret;
-
-	ret = max_gen_clk_probe(pdev, iodev->regmap, MAX77802_REG_32KHZ,
-				max77802_clks_init, MAX77802_CLKS_NUM);
-
-	if (ret) {
-		dev_err(&pdev->dev, "generic probe failed %d\n", ret);
-		return ret;
-	}
-
-	/* Enable low-jitter mode on the 32khz clocks. */
-	ret = regmap_update_bits(iodev->regmap, MAX77802_REG_32KHZ,
-				 1 << MAX77802_CLOCK_LOW_JITTER_SHIFT,
-				 1 << MAX77802_CLOCK_LOW_JITTER_SHIFT);
-	if (ret < 0)
-		dev_err(&pdev->dev, "failed to enable low-jitter mode\n");
-
-	return ret;
-}
-
-static int max77802_clk_remove(struct platform_device *pdev)
-{
-	return max_gen_clk_remove(pdev, MAX77802_CLKS_NUM);
-}
-
-static const struct platform_device_id max77802_clk_id[] = {
-	{ "max77802-clk", 0},
-	{ },
-};
-MODULE_DEVICE_TABLE(platform, max77802_clk_id);
-
-static struct platform_driver max77802_clk_driver = {
-	.driver = {
-		.name  = "max77802-clk",
-	},
-	.probe = max77802_clk_probe,
-	.remove = max77802_clk_remove,
-	.id_table = max77802_clk_id,
-};
-
-module_platform_driver(max77802_clk_driver);
-
-MODULE_DESCRIPTION("MAXIM 77802 Clock Driver");
-MODULE_AUTHOR("Javier Martinez Canillas <javier@osg.samsung.com");
-MODULE_LICENSE("GPL");
diff --git a/drivers/clk/clk-mb86s7x.c b/drivers/clk/clk-mb86s7x.c
index e0817754ca3e..2a83a3ff1d09 100644
--- a/drivers/clk/clk-mb86s7x.c
+++ b/drivers/clk/clk-mb86s7x.c
@@ -327,10 +327,11 @@ static struct clk_ops clk_clc_ops = {
 	.set_rate = clc_set_rate,
 };
 
-struct clk *mb86s7x_clclk_register(struct device *cpu_dev)
+static struct clk_hw *mb86s7x_clclk_register(struct device *cpu_dev)
 {
 	struct clk_init_data init;
 	struct cl_clk *clc;
+	int ret;
 
 	clc = kzalloc(sizeof(*clc), GFP_KERNEL);
 	if (!clc)
@@ -344,14 +345,17 @@ struct clk *mb86s7x_clclk_register(struct device *cpu_dev)
 	init.flags = CLK_GET_RATE_NOCACHE;
 	init.num_parents = 0;
 
-	return devm_clk_register(cpu_dev, &clc->hw);
+	ret = devm_clk_hw_register(cpu_dev, &clc->hw);
+	if (ret)
+		return ERR_PTR(ret);
+	return &clc->hw;
 }
 
 static int mb86s7x_clclk_of_init(void)
 {
 	int cpu, ret = -ENODEV;
 	struct device_node *np;
-	struct clk *clk;
+	struct clk_hw *hw;
 
 	np = of_find_compatible_node(NULL, NULL, "fujitsu,mb86s70-scb-1.0");
 	if (!np || !of_device_is_available(np))
@@ -365,12 +369,12 @@ static int mb86s7x_clclk_of_init(void)
 			continue;
 		}
 
-		clk = mb86s7x_clclk_register(cpu_dev);
-		if (IS_ERR(clk)) {
+		hw = mb86s7x_clclk_register(cpu_dev);
+		if (IS_ERR(hw)) {
 			pr_err("failed to register cpu%d clock\n", cpu);
 			continue;
 		}
-		if (clk_register_clkdev(clk, NULL, dev_name(cpu_dev))) {
+		if (clk_hw_register_clkdev(hw, NULL, dev_name(cpu_dev))) {
 			pr_err("failed to register cpu%d clock lookup\n", cpu);
 			continue;
 		}
diff --git a/drivers/clk/clk-moxart.c b/drivers/clk/clk-moxart.c
index f37f719643ec..b86dac851116 100644
--- a/drivers/clk/clk-moxart.c
+++ b/drivers/clk/clk-moxart.c
@@ -19,7 +19,8 @@
 static void __init moxart_of_pll_clk_init(struct device_node *node)
 {
 	static void __iomem *base;
-	struct clk *clk, *ref_clk;
+	struct clk_hw *hw;
+	struct clk *ref_clk;
 	unsigned int mul;
 	const char *name = node->name;
 	const char *parent_name;
@@ -42,14 +43,14 @@ static void __init moxart_of_pll_clk_init(struct device_node *node)
 		return;
 	}
 
-	clk = clk_register_fixed_factor(NULL, name, parent_name, 0, mul, 1);
-	if (IS_ERR(clk)) {
+	hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, mul, 1);
+	if (IS_ERR(hw)) {
 		pr_err("%s: failed to register clock\n", node->full_name);
 		return;
 	}
 
-	clk_register_clkdev(clk, NULL, name);
-	of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	clk_hw_register_clkdev(hw, NULL, name);
+	of_clk_add_hw_provider(node, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(moxart_pll_clock, "moxa,moxart-pll-clock",
 	       moxart_of_pll_clk_init);
@@ -57,7 +58,8 @@ CLK_OF_DECLARE(moxart_pll_clock, "moxa,moxart-pll-clock",
 static void __init moxart_of_apb_clk_init(struct device_node *node)
 {
 	static void __iomem *base;
-	struct clk *clk, *pll_clk;
+	struct clk_hw *hw;
+	struct clk *pll_clk;
 	unsigned int div, val;
 	unsigned int div_idx[] = { 2, 3, 4, 6, 8};
 	const char *name = node->name;
@@ -85,14 +87,14 @@ static void __init moxart_of_apb_clk_init(struct device_node *node)
 		return;
 	}
 
-	clk = clk_register_fixed_factor(NULL, name, parent_name, 0, 1, div);
-	if (IS_ERR(clk)) {
+	hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, 1, div);
+	if (IS_ERR(hw)) {
 		pr_err("%s: failed to register clock\n", node->full_name);
 		return;
 	}
 
-	clk_register_clkdev(clk, NULL, name);
-	of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	clk_hw_register_clkdev(hw, NULL, name);
+	of_clk_add_hw_provider(node, of_clk_hw_simple_get, hw);
 }
 CLK_OF_DECLARE(moxart_apb_clock, "moxa,moxart-apb-clock",
 	       moxart_of_apb_clk_init);
diff --git a/drivers/clk/clk-nspire.c b/drivers/clk/clk-nspire.c
index 64f196a90816..f861011d5d21 100644
--- a/drivers/clk/clk-nspire.c
+++ b/drivers/clk/clk-nspire.c
@@ -69,7 +69,7 @@ static void __init nspire_ahbdiv_setup(struct device_node *node,
 {
 	u32 val;
 	void __iomem *io;
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *clk_name = node->name;
 	const char *parent_name;
 	struct nspire_clk_info info;
@@ -85,10 +85,10 @@ static void __init nspire_ahbdiv_setup(struct device_node *node,
 	of_property_read_string(node, "clock-output-names", &clk_name);
 	parent_name = of_clk_get_parent_name(node, 0);
 
-	clk = clk_register_fixed_factor(NULL, clk_name, parent_name, 0,
-					1, info.base_ahb_ratio);
-	if (!IS_ERR(clk))
-		of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	hw = clk_hw_register_fixed_factor(NULL, clk_name, parent_name, 0,
+					  1, info.base_ahb_ratio);
+	if (!IS_ERR(hw))
+		of_clk_add_hw_provider(node, of_clk_hw_simple_get, hw);
 }
 
 static void __init nspire_ahbdiv_setup_cx(struct device_node *node)
@@ -111,7 +111,7 @@ static void __init nspire_clk_setup(struct device_node *node,
 {
 	u32 val;
 	void __iomem *io;
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *clk_name = node->name;
 	struct nspire_clk_info info;
 
@@ -125,9 +125,10 @@ static void __init nspire_clk_setup(struct device_node *node,
 
 	of_property_read_string(node, "clock-output-names", &clk_name);
 
-	clk = clk_register_fixed_rate(NULL, clk_name, NULL, 0, info.base_clock);
-	if (!IS_ERR(clk))
-		of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	hw = clk_hw_register_fixed_rate(NULL, clk_name, NULL, 0,
+					info.base_clock);
+	if (!IS_ERR(hw))
+		of_clk_add_hw_provider(node, of_clk_hw_simple_get, hw);
 	else
 		return;
 
diff --git a/drivers/clk/clk-palmas.c b/drivers/clk/clk-palmas.c
index 8328863cb0e0..31f590cea493 100644
--- a/drivers/clk/clk-palmas.c
+++ b/drivers/clk/clk-palmas.c
@@ -41,7 +41,6 @@ struct palmas_clk32k_desc {
 
 struct palmas_clock_info {
 	struct device *dev;
-	struct clk *clk;
 	struct clk_hw hw;
 	struct palmas *palmas;
 	const struct palmas_clk32k_desc *clk_desc;
@@ -218,7 +217,7 @@ static int palmas_clks_init_configure(struct palmas_clock_info *cinfo)
 	}
 
 	if (cinfo->ext_control_pin) {
-		ret = clk_prepare(cinfo->clk);
+		ret = clk_prepare(cinfo->hw.clk);
 		if (ret < 0) {
 			dev_err(cinfo->dev, "Clock prep failed, %d\n", ret);
 			return ret;
@@ -242,7 +241,6 @@ static int palmas_clks_probe(struct platform_device *pdev)
 	struct device_node *node = pdev->dev.of_node;
 	const struct palmas_clks_of_match_data *match_data;
 	struct palmas_clock_info *cinfo;
-	struct clk *clk;
 	int ret;
 
 	match_data = of_device_get_match_data(&pdev->dev);
@@ -261,22 +259,20 @@ static int palmas_clks_probe(struct platform_device *pdev)
 
 	cinfo->clk_desc = &match_data->desc;
 	cinfo->hw.init = &match_data->init;
-	clk = devm_clk_register(&pdev->dev, &cinfo->hw);
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
+	ret = devm_clk_hw_register(&pdev->dev, &cinfo->hw);
+	if (ret) {
 		dev_err(&pdev->dev, "Fail to register clock %s, %d\n",
 			match_data->desc.clk_name, ret);
 		return ret;
 	}
 
-	cinfo->clk = clk;
 	ret = palmas_clks_init_configure(cinfo);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Clock config failed, %d\n", ret);
 		return ret;
 	}
 
-	ret = of_clk_add_provider(node, of_clk_src_simple_get, cinfo->clk);
+	ret = of_clk_add_hw_provider(node, of_clk_hw_simple_get, &cinfo->hw);
 	if (ret < 0)
 		dev_err(&pdev->dev, "Fail to add clock driver, %d\n", ret);
 	return ret;
diff --git a/drivers/clk/clk-pwm.c b/drivers/clk/clk-pwm.c
index 1630a1f085f7..8cb9d117fdbf 100644
--- a/drivers/clk/clk-pwm.c
+++ b/drivers/clk/clk-pwm.c
@@ -61,7 +61,6 @@ static int clk_pwm_probe(struct platform_device *pdev)
 	struct pwm_device *pwm;
 	struct pwm_args pargs;
 	const char *clk_name;
-	struct clk *clk;
 	int ret;
 
 	clk_pwm = devm_kzalloc(&pdev->dev, sizeof(*clk_pwm), GFP_KERNEL);
@@ -107,11 +106,11 @@ static int clk_pwm_probe(struct platform_device *pdev)
 
 	clk_pwm->pwm = pwm;
 	clk_pwm->hw.init = &init;
-	clk = devm_clk_register(&pdev->dev, &clk_pwm->hw);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = devm_clk_hw_register(&pdev->dev, &clk_pwm->hw);
+	if (ret)
+		return ret;
 
-	return of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	return of_clk_add_hw_provider(node, of_clk_hw_simple_get, &clk_pwm->hw);
 }
 
 static int clk_pwm_remove(struct platform_device *pdev)
diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c
index 58566a17944a..20b105584f82 100644
--- a/drivers/clk/clk-qoriq.c
+++ b/drivers/clk/clk-qoriq.c
@@ -766,7 +766,11 @@ static struct clk * __init create_one_cmux(struct clockgen *cg, int idx)
 	if (!hwc)
 		return NULL;
 
-	hwc->reg = cg->regs + 0x20 * idx;
+	if (cg->info.flags & CG_VER3)
+		hwc->reg = cg->regs + 0x70000 + 0x20 * idx;
+	else
+		hwc->reg = cg->regs + 0x20 * idx;
+
 	hwc->info = cg->info.cmux_groups[cg->info.cmux_to_group[idx]];
 
 	/*
diff --git a/drivers/clk/clk-rk808.c b/drivers/clk/clk-rk808.c
index 74383039761e..6461f2820a5b 100644
--- a/drivers/clk/clk-rk808.c
+++ b/drivers/clk/clk-rk808.c
@@ -22,11 +22,8 @@
 #include <linux/mfd/rk808.h>
 #include <linux/i2c.h>
 
-#define RK808_NR_OUTPUT 2
-
 struct rk808_clkout {
 	struct rk808 *rk808;
-	struct clk_onecell_data clk_data;
 	struct clk_hw		clkout1_hw;
 	struct clk_hw		clkout2_hw;
 };
@@ -85,14 +82,28 @@ static const struct clk_ops rk808_clkout2_ops = {
 	.recalc_rate = rk808_clkout_recalc_rate,
 };
 
+static struct clk_hw *
+of_clk_rk808_get(struct of_phandle_args *clkspec, void *data)
+{
+	struct rk808_clkout *rk808_clkout = data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= 2) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return idx ? &rk808_clkout->clkout2_hw : &rk808_clkout->clkout1_hw;
+}
+
 static int rk808_clkout_probe(struct platform_device *pdev)
 {
 	struct rk808 *rk808 = dev_get_drvdata(pdev->dev.parent);
 	struct i2c_client *client = rk808->i2c;
 	struct device_node *node = client->dev.of_node;
 	struct clk_init_data init = {};
-	struct clk **clk_table;
 	struct rk808_clkout *rk808_clkout;
+	int ret;
 
 	rk808_clkout = devm_kzalloc(&client->dev,
 				    sizeof(*rk808_clkout), GFP_KERNEL);
@@ -101,11 +112,6 @@ static int rk808_clkout_probe(struct platform_device *pdev)
 
 	rk808_clkout->rk808 = rk808;
 
-	clk_table = devm_kcalloc(&client->dev, RK808_NR_OUTPUT,
-				 sizeof(struct clk *), GFP_KERNEL);
-	if (!clk_table)
-		return -ENOMEM;
-
 	init.parent_names = NULL;
 	init.num_parents = 0;
 	init.name = "rk808-clkout1";
@@ -116,10 +122,9 @@ static int rk808_clkout_probe(struct platform_device *pdev)
 	of_property_read_string_index(node, "clock-output-names",
 				      0, &init.name);
 
-	clk_table[0] = devm_clk_register(&client->dev,
-					 &rk808_clkout->clkout1_hw);
-	if (IS_ERR(clk_table[0]))
-		return PTR_ERR(clk_table[0]);
+	ret = devm_clk_hw_register(&client->dev, &rk808_clkout->clkout1_hw);
+	if (ret)
+		return ret;
 
 	init.name = "rk808-clkout2";
 	init.ops = &rk808_clkout2_ops;
@@ -129,16 +134,11 @@ static int rk808_clkout_probe(struct platform_device *pdev)
 	of_property_read_string_index(node, "clock-output-names",
 				      1, &init.name);
 
-	clk_table[1] = devm_clk_register(&client->dev,
-					 &rk808_clkout->clkout2_hw);
-	if (IS_ERR(clk_table[1]))
-		return PTR_ERR(clk_table[1]);
-
-	rk808_clkout->clk_data.clks = clk_table;
-	rk808_clkout->clk_data.clk_num = RK808_NR_OUTPUT;
+	ret = devm_clk_hw_register(&client->dev, &rk808_clkout->clkout2_hw);
+	if (ret)
+		return ret;
 
-	return of_clk_add_provider(node, of_clk_src_onecell_get,
-				   &rk808_clkout->clk_data);
+	return of_clk_add_hw_provider(node, of_clk_rk808_get, rk808_clkout);
 }
 
 static int rk808_clkout_remove(struct platform_device *pdev)
diff --git a/drivers/clk/clk-scpi.c b/drivers/clk/clk-scpi.c
index 6962ee5d1e9a..2a3e9d8e88b0 100644
--- a/drivers/clk/clk-scpi.c
+++ b/drivers/clk/clk-scpi.c
@@ -146,13 +146,13 @@ static const struct of_device_id scpi_clk_match[] = {
 	{}
 };
 
-static struct clk *
+static int
 scpi_clk_ops_init(struct device *dev, const struct of_device_id *match,
 		  struct scpi_clk *sclk, const char *name)
 {
 	struct clk_init_data init;
-	struct clk *clk;
 	unsigned long min = 0, max = 0;
+	int ret;
 
 	init.name = name;
 	init.flags = 0;
@@ -164,18 +164,18 @@ scpi_clk_ops_init(struct device *dev, const struct of_device_id *match,
 	if (init.ops == &scpi_dvfs_ops) {
 		sclk->info = sclk->scpi_ops->dvfs_get_info(sclk->id);
 		if (IS_ERR(sclk->info))
-			return NULL;
+			return PTR_ERR(sclk->info);
 	} else if (init.ops == &scpi_clk_ops) {
 		if (sclk->scpi_ops->clk_get_range(sclk->id, &min, &max) || !max)
-			return NULL;
+			return -EINVAL;
 	} else {
-		return NULL;
+		return -EINVAL;
 	}
 
-	clk = devm_clk_register(dev, &sclk->hw);
-	if (!IS_ERR(clk) && max)
+	ret = devm_clk_hw_register(dev, &sclk->hw);
+	if (!ret && max)
 		clk_hw_set_rate_range(&sclk->hw, min, max);
-	return clk;
+	return ret;
 }
 
 struct scpi_clk_data {
@@ -183,7 +183,7 @@ struct scpi_clk_data {
 	unsigned int clk_num;
 };
 
-static struct clk *
+static struct clk_hw *
 scpi_of_clk_src_get(struct of_phandle_args *clkspec, void *data)
 {
 	struct scpi_clk *sclk;
@@ -193,7 +193,7 @@ scpi_of_clk_src_get(struct of_phandle_args *clkspec, void *data)
 	for (count = 0; count < clk_data->clk_num; count++) {
 		sclk = clk_data->clk[count];
 		if (idx == sclk->id)
-			return sclk->hw.clk;
+			return &sclk->hw;
 	}
 
 	return ERR_PTR(-EINVAL);
@@ -202,8 +202,7 @@ scpi_of_clk_src_get(struct of_phandle_args *clkspec, void *data)
 static int scpi_clk_add(struct device *dev, struct device_node *np,
 			const struct of_device_id *match)
 {
-	struct clk **clks;
-	int idx, count;
+	int idx, count, err;
 	struct scpi_clk_data *clk_data;
 
 	count = of_property_count_strings(np, "clock-output-names");
@@ -222,10 +221,6 @@ static int scpi_clk_add(struct device *dev, struct device_node *np,
 	if (!clk_data->clk)
 		return -ENOMEM;
 
-	clks = devm_kcalloc(dev, count, sizeof(*clks), GFP_KERNEL);
-	if (!clks)
-		return -ENOMEM;
-
 	for (idx = 0; idx < count; idx++) {
 		struct scpi_clk *sclk;
 		const char *name;
@@ -249,15 +244,15 @@ static int scpi_clk_add(struct device *dev, struct device_node *np,
 
 		sclk->id = val;
 
-		clks[idx] = scpi_clk_ops_init(dev, match, sclk, name);
-		if (IS_ERR_OR_NULL(clks[idx]))
+		err = scpi_clk_ops_init(dev, match, sclk, name);
+		if (err)
 			dev_err(dev, "failed to register clock '%s'\n", name);
 		else
 			dev_dbg(dev, "Registered clock '%s'\n", name);
 		clk_data->clk[idx] = sclk;
 	}
 
-	return of_clk_add_provider(np, scpi_of_clk_src_get, clk_data);
+	return of_clk_add_hw_provider(np, scpi_of_clk_src_get, clk_data);
 }
 
 static int scpi_clocks_remove(struct platform_device *pdev)
diff --git a/drivers/clk/clk-si514.c b/drivers/clk/clk-si514.c
index ceef25b0990b..09b6718956bd 100644
--- a/drivers/clk/clk-si514.c
+++ b/drivers/clk/clk-si514.c
@@ -305,7 +305,6 @@ static int si514_probe(struct i2c_client *client,
 {
 	struct clk_si514 *data;
 	struct clk_init_data init;
-	struct clk *clk;
 	int err;
 
 	data = devm_kzalloc(&client->dev, sizeof(*data), GFP_KERNEL);
@@ -330,13 +329,13 @@ static int si514_probe(struct i2c_client *client,
 
 	i2c_set_clientdata(client, data);
 
-	clk = devm_clk_register(&client->dev, &data->hw);
-	if (IS_ERR(clk)) {
+	err = devm_clk_hw_register(&client->dev, &data->hw);
+	if (err) {
 		dev_err(&client->dev, "clock registration failed\n");
-		return PTR_ERR(clk);
+		return err;
 	}
-	err = of_clk_add_provider(client->dev.of_node, of_clk_src_simple_get,
-			clk);
+	err = of_clk_add_hw_provider(client->dev.of_node, of_clk_hw_simple_get,
+				     &data->hw);
 	if (err) {
 		dev_err(&client->dev, "unable to add clk provider\n");
 		return err;
diff --git a/drivers/clk/clk-si5351.c b/drivers/clk/clk-si5351.c
index b1bc12c045d3..b051db43fae1 100644
--- a/drivers/clk/clk-si5351.c
+++ b/drivers/clk/clk-si5351.c
@@ -54,7 +54,6 @@ struct si5351_driver_data {
 	enum si5351_variant	variant;
 	struct i2c_client	*client;
 	struct regmap		*regmap;
-	struct clk_onecell_data onecell;
 
 	struct clk		*pxtal;
 	const char		*pxtal_name;
@@ -66,6 +65,7 @@ struct si5351_driver_data {
 	struct si5351_hw_data	pll[2];
 	struct si5351_hw_data	*msynth;
 	struct si5351_hw_data	*clkout;
+	size_t			num_clkout;
 };
 
 static const char * const si5351_input_names[] = {
@@ -1307,11 +1307,31 @@ put_child:
 	of_node_put(child);
 	return -EINVAL;
 }
+
+static struct clk_hw *
+si53351_of_clk_get(struct of_phandle_args *clkspec, void *data)
+{
+	struct si5351_driver_data *drvdata = data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= drvdata->num_clkout) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return &drvdata->clkout[idx].hw;
+}
 #else
 static int si5351_dt_parse(struct i2c_client *client, enum si5351_variant variant)
 {
 	return 0;
 }
+
+static struct clk_hw *
+si53351_of_clk_get(struct of_phandle_args *clkspec, void *data)
+{
+	return NULL;
+}
 #endif /* CONFIG_OF */
 
 static int si5351_i2c_probe(struct i2c_client *client,
@@ -1321,7 +1341,6 @@ static int si5351_i2c_probe(struct i2c_client *client,
 	struct si5351_platform_data *pdata;
 	struct si5351_driver_data *drvdata;
 	struct clk_init_data init;
-	struct clk *clk;
 	const char *parent_names[4];
 	u8 num_parents, num_clocks;
 	int ret, n;
@@ -1438,10 +1457,9 @@ static int si5351_i2c_probe(struct i2c_client *client,
 		init.num_parents = 1;
 	}
 	drvdata->xtal.init = &init;
-	clk = devm_clk_register(&client->dev, &drvdata->xtal);
-	if (IS_ERR(clk)) {
+	ret = devm_clk_hw_register(&client->dev, &drvdata->xtal);
+	if (ret) {
 		dev_err(&client->dev, "unable to register %s\n", init.name);
-		ret = PTR_ERR(clk);
 		goto err_clk;
 	}
 
@@ -1456,11 +1474,10 @@ static int si5351_i2c_probe(struct i2c_client *client,
 			init.num_parents = 1;
 		}
 		drvdata->clkin.init = &init;
-		clk = devm_clk_register(&client->dev, &drvdata->clkin);
-		if (IS_ERR(clk)) {
+		ret = devm_clk_hw_register(&client->dev, &drvdata->clkin);
+		if (ret) {
 			dev_err(&client->dev, "unable to register %s\n",
 				init.name);
-			ret = PTR_ERR(clk);
 			goto err_clk;
 		}
 	}
@@ -1480,10 +1497,9 @@ static int si5351_i2c_probe(struct i2c_client *client,
 	init.flags = 0;
 	init.parent_names = parent_names;
 	init.num_parents = num_parents;
-	clk = devm_clk_register(&client->dev, &drvdata->pll[0].hw);
-	if (IS_ERR(clk)) {
+	ret = devm_clk_hw_register(&client->dev, &drvdata->pll[0].hw);
+	if (ret) {
 		dev_err(&client->dev, "unable to register %s\n", init.name);
-		ret = PTR_ERR(clk);
 		goto err_clk;
 	}
 
@@ -1505,10 +1521,9 @@ static int si5351_i2c_probe(struct i2c_client *client,
 		init.parent_names = parent_names;
 		init.num_parents = num_parents;
 	}
-	clk = devm_clk_register(&client->dev, &drvdata->pll[1].hw);
-	if (IS_ERR(clk)) {
+	ret = devm_clk_hw_register(&client->dev, &drvdata->pll[1].hw);
+	if (ret) {
 		dev_err(&client->dev, "unable to register %s\n", init.name);
-		ret = PTR_ERR(clk);
 		goto err_clk;
 	}
 
@@ -1524,13 +1539,9 @@ static int si5351_i2c_probe(struct i2c_client *client,
 				       sizeof(*drvdata->msynth), GFP_KERNEL);
 	drvdata->clkout = devm_kzalloc(&client->dev, num_clocks *
 				       sizeof(*drvdata->clkout), GFP_KERNEL);
+	drvdata->num_clkout = num_clocks;
 
-	drvdata->onecell.clk_num = num_clocks;
-	drvdata->onecell.clks = devm_kzalloc(&client->dev,
-		num_clocks * sizeof(*drvdata->onecell.clks), GFP_KERNEL);
-
-	if (WARN_ON(!drvdata->msynth || !drvdata->clkout ||
-		    !drvdata->onecell.clks)) {
+	if (WARN_ON(!drvdata->msynth || !drvdata->clkout)) {
 		ret = -ENOMEM;
 		goto err_clk;
 	}
@@ -1547,11 +1558,11 @@ static int si5351_i2c_probe(struct i2c_client *client,
 			init.flags |= CLK_SET_RATE_PARENT;
 		init.parent_names = parent_names;
 		init.num_parents = 2;
-		clk = devm_clk_register(&client->dev, &drvdata->msynth[n].hw);
-		if (IS_ERR(clk)) {
+		ret = devm_clk_hw_register(&client->dev,
+					   &drvdata->msynth[n].hw);
+		if (ret) {
 			dev_err(&client->dev, "unable to register %s\n",
 				init.name);
-			ret = PTR_ERR(clk);
 			goto err_clk;
 		}
 	}
@@ -1575,19 +1586,19 @@ static int si5351_i2c_probe(struct i2c_client *client,
 			init.flags |= CLK_SET_RATE_PARENT;
 		init.parent_names = parent_names;
 		init.num_parents = num_parents;
-		clk = devm_clk_register(&client->dev, &drvdata->clkout[n].hw);
-		if (IS_ERR(clk)) {
+		ret = devm_clk_hw_register(&client->dev,
+					   &drvdata->clkout[n].hw);
+		if (ret) {
 			dev_err(&client->dev, "unable to register %s\n",
 				init.name);
-			ret = PTR_ERR(clk);
 			goto err_clk;
 		}
-		drvdata->onecell.clks[n] = clk;
 
 		/* set initial clkout rate */
 		if (pdata->clkout[n].rate != 0) {
 			int ret;
-			ret = clk_set_rate(clk, pdata->clkout[n].rate);
+			ret = clk_set_rate(drvdata->clkout[n].hw.clk,
+					   pdata->clkout[n].rate);
 			if (ret != 0) {
 				dev_err(&client->dev, "Cannot set rate : %d\n",
 					ret);
@@ -1595,8 +1606,8 @@ static int si5351_i2c_probe(struct i2c_client *client,
 		}
 	}
 
-	ret = of_clk_add_provider(client->dev.of_node, of_clk_src_onecell_get,
-				  &drvdata->onecell);
+	ret = of_clk_add_hw_provider(client->dev.of_node, si53351_of_clk_get,
+				     drvdata);
 	if (ret) {
 		dev_err(&client->dev, "unable to add clk provider\n");
 		goto err_clk;
diff --git a/drivers/clk/clk-si570.c b/drivers/clk/clk-si570.c
index d56648521a95..646af1d1898d 100644
--- a/drivers/clk/clk-si570.c
+++ b/drivers/clk/clk-si570.c
@@ -408,7 +408,6 @@ static int si570_probe(struct i2c_client *client,
 {
 	struct clk_si570 *data;
 	struct clk_init_data init;
-	struct clk *clk;
 	u32 initial_fout, factory_fout, stability;
 	int err;
 	enum clk_si570_variant variant = id->driver_data;
@@ -462,13 +461,13 @@ static int si570_probe(struct i2c_client *client,
 	if (err)
 		return err;
 
-	clk = devm_clk_register(&client->dev, &data->hw);
-	if (IS_ERR(clk)) {
+	err = devm_clk_hw_register(&client->dev, &data->hw);
+	if (err) {
 		dev_err(&client->dev, "clock registration failed\n");
-		return PTR_ERR(clk);
+		return err;
 	}
-	err = of_clk_add_provider(client->dev.of_node, of_clk_src_simple_get,
-			clk);
+	err = of_clk_add_hw_provider(client->dev.of_node, of_clk_hw_simple_get,
+				     &data->hw);
 	if (err) {
 		dev_err(&client->dev, "unable to add clk provider\n");
 		return err;
@@ -477,7 +476,7 @@ static int si570_probe(struct i2c_client *client,
 	/* Read the requested initial output frequency from device tree */
 	if (!of_property_read_u32(client->dev.of_node, "clock-frequency",
 				&initial_fout)) {
-		err = clk_set_rate(clk, initial_fout);
+		err = clk_set_rate(data->hw.clk, initial_fout);
 		if (err) {
 			of_clk_del_provider(client->dev.of_node);
 			return err;
diff --git a/drivers/clk/clk-twl6040.c b/drivers/clk/clk-twl6040.c
index 697c66757400..7b222a5db931 100644
--- a/drivers/clk/clk-twl6040.c
+++ b/drivers/clk/clk-twl6040.c
@@ -26,60 +26,73 @@
 #include <linux/mfd/twl6040.h>
 #include <linux/clk-provider.h>
 
-struct twl6040_clk {
+struct twl6040_pdmclk {
 	struct twl6040 *twl6040;
 	struct device *dev;
-	struct clk_hw mcpdm_fclk;
-	struct clk *clk;
+	struct clk_hw pdmclk_hw;
 	int enabled;
 };
 
-static int twl6040_bitclk_is_enabled(struct clk_hw *hw)
+static int twl6040_pdmclk_is_prepared(struct clk_hw *hw)
 {
-	struct twl6040_clk *twl6040_clk = container_of(hw, struct twl6040_clk,
-						       mcpdm_fclk);
-	return twl6040_clk->enabled;
+	struct twl6040_pdmclk *pdmclk = container_of(hw, struct twl6040_pdmclk,
+						     pdmclk_hw);
+
+	return pdmclk->enabled;
 }
 
-static int twl6040_bitclk_prepare(struct clk_hw *hw)
+static int twl6040_pdmclk_prepare(struct clk_hw *hw)
 {
-	struct twl6040_clk *twl6040_clk = container_of(hw, struct twl6040_clk,
-						       mcpdm_fclk);
+	struct twl6040_pdmclk *pdmclk = container_of(hw, struct twl6040_pdmclk,
+						     pdmclk_hw);
 	int ret;
 
-	ret = twl6040_power(twl6040_clk->twl6040, 1);
+	ret = twl6040_power(pdmclk->twl6040, 1);
 	if (!ret)
-		twl6040_clk->enabled = 1;
+		pdmclk->enabled = 1;
 
 	return ret;
 }
 
-static void twl6040_bitclk_unprepare(struct clk_hw *hw)
+static void twl6040_pdmclk_unprepare(struct clk_hw *hw)
 {
-	struct twl6040_clk *twl6040_clk = container_of(hw, struct twl6040_clk,
-						       mcpdm_fclk);
+	struct twl6040_pdmclk *pdmclk = container_of(hw, struct twl6040_pdmclk,
+						     pdmclk_hw);
 	int ret;
 
-	ret = twl6040_power(twl6040_clk->twl6040, 0);
+	ret = twl6040_power(pdmclk->twl6040, 0);
 	if (!ret)
-		twl6040_clk->enabled = 0;
+		pdmclk->enabled = 0;
+
 }
 
-static const struct clk_ops twl6040_mcpdm_ops = {
-	.is_enabled = twl6040_bitclk_is_enabled,
-	.prepare = twl6040_bitclk_prepare,
-	.unprepare = twl6040_bitclk_unprepare,
+static unsigned long twl6040_pdmclk_recalc_rate(struct clk_hw *hw,
+						unsigned long parent_rate)
+{
+	struct twl6040_pdmclk *pdmclk = container_of(hw, struct twl6040_pdmclk,
+						     pdmclk_hw);
+
+	return twl6040_get_sysclk(pdmclk->twl6040);
+}
+
+static const struct clk_ops twl6040_pdmclk_ops = {
+	.is_prepared = twl6040_pdmclk_is_prepared,
+	.prepare = twl6040_pdmclk_prepare,
+	.unprepare = twl6040_pdmclk_unprepare,
+	.recalc_rate = twl6040_pdmclk_recalc_rate,
 };
 
-static struct clk_init_data wm831x_clkout_init = {
-	.name = "mcpdm_fclk",
-	.ops = &twl6040_mcpdm_ops,
+static struct clk_init_data twl6040_pdmclk_init = {
+	.name = "pdmclk",
+	.ops = &twl6040_pdmclk_ops,
+	.flags = CLK_GET_RATE_NOCACHE,
 };
 
-static int twl6040_clk_probe(struct platform_device *pdev)
+static int twl6040_pdmclk_probe(struct platform_device *pdev)
 {
 	struct twl6040 *twl6040 = dev_get_drvdata(pdev->dev.parent);
-	struct twl6040_clk *clkdata;
+	struct twl6040_pdmclk *clkdata;
+	int ret;
 
 	clkdata = devm_kzalloc(&pdev->dev, sizeof(*clkdata), GFP_KERNEL);
 	if (!clkdata)
@@ -88,26 +101,28 @@ static int twl6040_clk_probe(struct platform_device *pdev)
 	clkdata->dev = &pdev->dev;
 	clkdata->twl6040 = twl6040;
 
-	clkdata->mcpdm_fclk.init = &wm831x_clkout_init;
-	clkdata->clk = devm_clk_register(&pdev->dev, &clkdata->mcpdm_fclk);
-	if (IS_ERR(clkdata->clk))
-		return PTR_ERR(clkdata->clk);
+	clkdata->pdmclk_hw.init = &twl6040_pdmclk_init;
+	ret = devm_clk_hw_register(&pdev->dev, &clkdata->pdmclk_hw);
+	if (ret)
+		return ret;
 
 	platform_set_drvdata(pdev, clkdata);
 
-	return 0;
+	return of_clk_add_hw_provider(pdev->dev.parent->of_node,
+				      of_clk_hw_simple_get,
+				      &clkdata->pdmclk_hw);
 }
 
-static struct platform_driver twl6040_clk_driver = {
+static struct platform_driver twl6040_pdmclk_driver = {
 	.driver = {
-		.name = "twl6040-clk",
+		.name = "twl6040-pdmclk",
 	},
-	.probe = twl6040_clk_probe,
+	.probe = twl6040_pdmclk_probe,
 };
 
-module_platform_driver(twl6040_clk_driver);
+module_platform_driver(twl6040_pdmclk_driver);
 
 MODULE_DESCRIPTION("TWL6040 clock driver for McPDM functional clock");
 MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
-MODULE_ALIAS("platform:twl6040-clk");
+MODULE_ALIAS("platform:twl6040-pdmclk");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/clk-vt8500.c b/drivers/clk/clk-vt8500.c
index 37368a399ff9..4161a6f25741 100644
--- a/drivers/clk/clk-vt8500.c
+++ b/drivers/clk/clk-vt8500.c
@@ -232,7 +232,7 @@ static const struct clk_ops vt8500_gated_divisor_clk_ops = {
 static __init void vtwm_device_clk_init(struct device_node *node)
 {
 	u32 en_reg, div_reg;
-	struct clk *clk;
+	struct clk_hw *hw;
 	struct clk_device *dev_clk;
 	const char *clk_name = node->name;
 	const char *parent_name;
@@ -301,13 +301,14 @@ static __init void vtwm_device_clk_init(struct device_node *node)
 
 	dev_clk->hw.init = &init;
 
-	clk = clk_register(NULL, &dev_clk->hw);
-	if (WARN_ON(IS_ERR(clk))) {
+	hw = &dev_clk->hw;
+	rc = clk_hw_register(NULL, hw);
+	if (WARN_ON(rc)) {
 		kfree(dev_clk);
 		return;
 	}
-	rc = of_clk_add_provider(node, of_clk_src_simple_get, clk);
-	clk_register_clkdev(clk, clk_name, NULL);
+	rc = of_clk_add_hw_provider(node, of_clk_hw_simple_get, hw);
+	clk_hw_register_clkdev(hw, clk_name, NULL);
 }
 CLK_OF_DECLARE(vt8500_device, "via,vt8500-device-clock", vtwm_device_clk_init);
 
@@ -681,7 +682,7 @@ static const struct clk_ops vtwm_pll_ops = {
 static __init void vtwm_pll_clk_init(struct device_node *node, int pll_type)
 {
 	u32 reg;
-	struct clk *clk;
+	struct clk_hw *hw;
 	struct clk_pll *pll_clk;
 	const char *clk_name = node->name;
 	const char *parent_name;
@@ -714,13 +715,14 @@ static __init void vtwm_pll_clk_init(struct device_node *node, int pll_type)
 
 	pll_clk->hw.init = &init;
 
-	clk = clk_register(NULL, &pll_clk->hw);
-	if (WARN_ON(IS_ERR(clk))) {
+	hw = &pll_clk->hw;
+	rc = clk_hw_register(NULL, &pll_clk->hw);
+	if (WARN_ON(rc)) {
 		kfree(pll_clk);
 		return;
 	}
-	rc = of_clk_add_provider(node, of_clk_src_simple_get, clk);
-	clk_register_clkdev(clk, clk_name, NULL);
+	rc = of_clk_add_hw_provider(node, of_clk_hw_simple_get, hw);
+	clk_hw_register_clkdev(hw, clk_name, NULL);
 }
 
 
diff --git a/drivers/clk/clk-wm831x.c b/drivers/clk/clk-wm831x.c
index 88def4b2761c..f4fdac55727c 100644
--- a/drivers/clk/clk-wm831x.c
+++ b/drivers/clk/clk-wm831x.c
@@ -24,9 +24,6 @@ struct wm831x_clk {
 	struct clk_hw xtal_hw;
 	struct clk_hw fll_hw;
 	struct clk_hw clkout_hw;
-	struct clk *xtal;
-	struct clk *fll;
-	struct clk *clkout;
 	bool xtal_ena;
 };
 
@@ -370,19 +367,19 @@ static int wm831x_clk_probe(struct platform_device *pdev)
 	clkdata->xtal_ena = ret & WM831X_XTAL_ENA;
 
 	clkdata->xtal_hw.init = &wm831x_xtal_init;
-	clkdata->xtal = devm_clk_register(&pdev->dev, &clkdata->xtal_hw);
-	if (IS_ERR(clkdata->xtal))
-		return PTR_ERR(clkdata->xtal);
+	ret = devm_clk_hw_register(&pdev->dev, &clkdata->xtal_hw);
+	if (ret)
+		return ret;
 
 	clkdata->fll_hw.init = &wm831x_fll_init;
-	clkdata->fll = devm_clk_register(&pdev->dev, &clkdata->fll_hw);
-	if (IS_ERR(clkdata->fll))
-		return PTR_ERR(clkdata->fll);
+	ret = devm_clk_hw_register(&pdev->dev, &clkdata->fll_hw);
+	if (ret)
+		return ret;
 
 	clkdata->clkout_hw.init = &wm831x_clkout_init;
-	clkdata->clkout = devm_clk_register(&pdev->dev, &clkdata->clkout_hw);
-	if (IS_ERR(clkdata->clkout))
-		return PTR_ERR(clkdata->clkout);
+	ret = devm_clk_hw_register(&pdev->dev, &clkdata->clkout_hw);
+	if (ret)
+		return ret;
 
 	platform_set_drvdata(pdev, clkdata);
 
diff --git a/drivers/clk/clk-xgene.c b/drivers/clk/clk-xgene.c
index 343313250c58..5daddf5ecc4b 100644
--- a/drivers/clk/clk-xgene.c
+++ b/drivers/clk/clk-xgene.c
@@ -217,6 +217,226 @@ static void xgene_pcppllclk_init(struct device_node *np)
 	xgene_pllclk_init(np, PLL_TYPE_PCP);
 }
 
+/**
+ * struct xgene_clk_pmd - PMD clock
+ *
+ * @hw:		handle between common and hardware-specific interfaces
+ * @reg:	register containing the fractional scale multiplier (scaler)
+ * @shift:	shift to the unit bit field
+ * @denom:	1/denominator unit
+ * @lock:	register lock
+ * Flags:
+ * XGENE_CLK_PMD_SCALE_INVERTED - By default the scaler is the value read
+ *	from the register plus one. For example,
+ *		0 for (0 + 1) / denom,
+ *		1 for (1 + 1) / denom and etc.
+ *	If this flag is set, it is
+ *		0 for (denom - 0) / denom,
+ *		1 for (denom - 1) / denom and etc.
+ *
+ */
+struct xgene_clk_pmd {
+	struct clk_hw	hw;
+	void __iomem	*reg;
+	u8		shift;
+	u32		mask;
+	u64		denom;
+	u32		flags;
+	spinlock_t	*lock;
+};
+
+#define to_xgene_clk_pmd(_hw) container_of(_hw, struct xgene_clk_pmd, hw)
+
+#define XGENE_CLK_PMD_SCALE_INVERTED	BIT(0)
+#define XGENE_CLK_PMD_SHIFT		8
+#define XGENE_CLK_PMD_WIDTH		3
+
+static unsigned long xgene_clk_pmd_recalc_rate(struct clk_hw *hw,
+					       unsigned long parent_rate)
+{
+	struct xgene_clk_pmd *fd = to_xgene_clk_pmd(hw);
+	unsigned long flags = 0;
+	u64 ret, scale;
+	u32 val;
+
+	if (fd->lock)
+		spin_lock_irqsave(fd->lock, flags);
+	else
+		__acquire(fd->lock);
+
+	val = clk_readl(fd->reg);
+
+	if (fd->lock)
+		spin_unlock_irqrestore(fd->lock, flags);
+	else
+		__release(fd->lock);
+
+	ret = (u64)parent_rate;
+
+	scale = (val & fd->mask) >> fd->shift;
+	if (fd->flags & XGENE_CLK_PMD_SCALE_INVERTED)
+		scale = fd->denom - scale;
+	else
+		scale++;
+
+	/* freq = parent_rate * scaler / denom */
+	do_div(ret, fd->denom);
+	ret *= scale;
+	if (ret == 0)
+		ret = (u64)parent_rate;
+
+	return ret;
+}
+
+static long xgene_clk_pmd_round_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long *parent_rate)
+{
+	struct xgene_clk_pmd *fd = to_xgene_clk_pmd(hw);
+	u64 ret, scale;
+
+	if (!rate || rate >= *parent_rate)
+		return *parent_rate;
+
+	/* freq = parent_rate * scaler / denom */
+	ret = rate * fd->denom;
+	scale = DIV_ROUND_UP_ULL(ret, *parent_rate);
+
+	ret = (u64)*parent_rate * scale;
+	do_div(ret, fd->denom);
+
+	return ret;
+}
+
+static int xgene_clk_pmd_set_rate(struct clk_hw *hw, unsigned long rate,
+				  unsigned long parent_rate)
+{
+	struct xgene_clk_pmd *fd = to_xgene_clk_pmd(hw);
+	unsigned long flags = 0;
+	u64 scale, ret;
+	u32 val;
+
+	/*
+	 * Compute the scaler:
+	 *
+	 * freq = parent_rate * scaler / denom, or
+	 * scaler = freq * denom / parent_rate
+	 */
+	ret = rate * fd->denom;
+	scale = DIV_ROUND_UP_ULL(ret, (u64)parent_rate);
+
+	/* Check if inverted */
+	if (fd->flags & XGENE_CLK_PMD_SCALE_INVERTED)
+		scale = fd->denom - scale;
+	else
+		scale--;
+
+	if (fd->lock)
+		spin_lock_irqsave(fd->lock, flags);
+	else
+		__acquire(fd->lock);
+
+	val = clk_readl(fd->reg);
+	val &= ~fd->mask;
+	val |= (scale << fd->shift);
+	clk_writel(val, fd->reg);
+
+	if (fd->lock)
+		spin_unlock_irqrestore(fd->lock, flags);
+	else
+		__release(fd->lock);
+
+	return 0;
+}
+
+static const struct clk_ops xgene_clk_pmd_ops = {
+	.recalc_rate = xgene_clk_pmd_recalc_rate,
+	.round_rate = xgene_clk_pmd_round_rate,
+	.set_rate = xgene_clk_pmd_set_rate,
+};
+
+static struct clk *
+xgene_register_clk_pmd(struct device *dev,
+		       const char *name, const char *parent_name,
+		       unsigned long flags, void __iomem *reg, u8 shift,
+		       u8 width, u64 denom, u32 clk_flags, spinlock_t *lock)
+{
+	struct xgene_clk_pmd *fd;
+	struct clk_init_data init;
+	struct clk *clk;
+
+	fd = kzalloc(sizeof(*fd), GFP_KERNEL);
+	if (!fd)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &xgene_clk_pmd_ops;
+	init.flags = flags;
+	init.parent_names = parent_name ? &parent_name : NULL;
+	init.num_parents = parent_name ? 1 : 0;
+
+	fd->reg = reg;
+	fd->shift = shift;
+	fd->mask = (BIT(width) - 1) << shift;
+	fd->denom = denom;
+	fd->flags = clk_flags;
+	fd->lock = lock;
+	fd->hw.init = &init;
+
+	clk = clk_register(dev, &fd->hw);
+	if (IS_ERR(clk)) {
+		pr_err("%s: could not register clk %s\n", __func__, name);
+		kfree(fd);
+		return NULL;
+	}
+
+	return clk;
+}
+
+static void xgene_pmdclk_init(struct device_node *np)
+{
+	const char *clk_name = np->full_name;
+	void __iomem *csr_reg;
+	struct resource res;
+	struct clk *clk;
+	u64 denom;
+	u32 flags = 0;
+	int rc;
+
+	/* Check if the entry is disabled */
+	if (!of_device_is_available(np))
+		return;
+
+	/* Parse the DTS register for resource */
+	rc = of_address_to_resource(np, 0, &res);
+	if (rc != 0) {
+		pr_err("no DTS register for %s\n", np->full_name);
+		return;
+	}
+	csr_reg = of_iomap(np, 0);
+	if (!csr_reg) {
+		pr_err("Unable to map resource for %s\n", np->full_name);
+		return;
+	}
+	of_property_read_string(np, "clock-output-names", &clk_name);
+
+	denom = BIT(XGENE_CLK_PMD_WIDTH);
+	flags |= XGENE_CLK_PMD_SCALE_INVERTED;
+
+	clk = xgene_register_clk_pmd(NULL, clk_name,
+				     of_clk_get_parent_name(np, 0), 0,
+				     csr_reg, XGENE_CLK_PMD_SHIFT,
+				     XGENE_CLK_PMD_WIDTH, denom,
+				     flags, &clk_lock);
+	if (!IS_ERR(clk)) {
+		of_clk_add_provider(np, of_clk_src_simple_get, clk);
+		clk_register_clkdev(clk, clk_name, NULL);
+		pr_debug("Add %s clock\n", clk_name);
+	} else {
+		if (csr_reg)
+			iounmap(csr_reg);
+	}
+}
+
 /* IP Clock */
 struct xgene_dev_parameters {
 	void __iomem *csr_reg;		/* CSR for IP clock */
@@ -543,6 +763,7 @@ err:
 
 CLK_OF_DECLARE(xgene_socpll_clock, "apm,xgene-socpll-clock", xgene_socpllclk_init);
 CLK_OF_DECLARE(xgene_pcppll_clock, "apm,xgene-pcppll-clock", xgene_pcppllclk_init);
+CLK_OF_DECLARE(xgene_pmd_clock, "apm,xgene-pmd-clock", xgene_pmdclk_init);
 CLK_OF_DECLARE(xgene_socpll_v2_clock, "apm,xgene-socpll-v2-clock",
 	       xgene_socpllclk_init);
 CLK_OF_DECLARE(xgene_pcppll_v2_clock, "apm,xgene-pcppll-v2-clock",
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index 820a939fb6bb..0fb39fe217d1 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -1908,10 +1908,6 @@ int clk_set_phase(struct clk *clk, int degrees)
 
 	clk_prepare_lock();
 
-	/* bail early if nothing to do */
-	if (degrees == clk->core->phase)
-		goto out;
-
 	trace_clk_set_phase(clk->core, degrees);
 
 	if (clk->core->ops->set_phase)
@@ -1922,7 +1918,6 @@ int clk_set_phase(struct clk *clk, int degrees)
 	if (!ret)
 		clk->core->phase = degrees;
 
-out:
 	clk_prepare_unlock();
 
 	return ret;
@@ -2449,8 +2444,16 @@ static int __clk_core_init(struct clk_core *core)
 	hlist_for_each_entry_safe(orphan, tmp2, &clk_orphan_list, child_node) {
 		struct clk_core *parent = __clk_init_parent(orphan);
 
-		if (parent)
-			clk_core_reparent(orphan, parent);
+		/*
+		 * we could call __clk_set_parent, but that would result in a
+		 * redundant call to the .set_rate op, if it exists
+		 */
+		if (parent) {
+			__clk_set_parent_before(orphan, parent);
+			__clk_set_parent_after(orphan, parent, NULL);
+			__clk_recalc_accuracies(orphan);
+			__clk_recalc_rates(orphan, 0);
+		}
 	}
 
 	/*
@@ -2491,7 +2494,7 @@ struct clk *__clk_create_clk(struct clk_hw *hw, const char *dev_id,
 
 	/* This is to allow this function to be chained to others */
 	if (IS_ERR_OR_NULL(hw))
-		return (struct clk *) hw;
+		return ERR_CAST(hw);
 
 	clk = kzalloc(sizeof(*clk), GFP_KERNEL);
 	if (!clk)
@@ -3166,19 +3169,14 @@ __of_clk_get_hw_from_provider(struct of_clk_provider *provider,
 			      struct of_phandle_args *clkspec)
 {
 	struct clk *clk;
-	struct clk_hw *hw = ERR_PTR(-EPROBE_DEFER);
 
-	if (provider->get_hw) {
-		hw = provider->get_hw(clkspec, provider->data);
-	} else if (provider->get) {
-		clk = provider->get(clkspec, provider->data);
-		if (!IS_ERR(clk))
-			hw = __clk_get_hw(clk);
-		else
-			hw = ERR_CAST(clk);
-	}
+	if (provider->get_hw)
+		return provider->get_hw(clkspec, provider->data);
 
-	return hw;
+	clk = provider->get(clkspec, provider->data);
+	if (IS_ERR(clk))
+		return ERR_CAST(clk);
+	return __clk_get_hw(clk);
 }
 
 struct clk *__of_clk_get_from_provider(struct of_phandle_args *clkspec,
@@ -3186,7 +3184,7 @@ struct clk *__of_clk_get_from_provider(struct of_phandle_args *clkspec,
 {
 	struct of_clk_provider *provider;
 	struct clk *clk = ERR_PTR(-EPROBE_DEFER);
-	struct clk_hw *hw = ERR_PTR(-EPROBE_DEFER);
+	struct clk_hw *hw;
 
 	if (!clkspec)
 		return ERR_PTR(-EINVAL);
@@ -3194,12 +3192,13 @@ struct clk *__of_clk_get_from_provider(struct of_phandle_args *clkspec,
 	/* Check if we have such a provider in our array */
 	mutex_lock(&of_clk_mutex);
 	list_for_each_entry(provider, &of_clk_providers, link) {
-		if (provider->node == clkspec->np)
+		if (provider->node == clkspec->np) {
 			hw = __of_clk_get_hw_from_provider(provider, clkspec);
-		if (!IS_ERR(hw)) {
 			clk = __clk_create_clk(hw, dev_id, con_id);
+		}
 
-			if (!IS_ERR(clk) && !__clk_get(clk)) {
+		if (!IS_ERR(clk)) {
+			if (!__clk_get(clk)) {
 				__clk_free_clk(clk);
 				clk = ERR_PTR(-ENOENT);
 			}
@@ -3451,6 +3450,10 @@ void __init of_clk_init(const struct of_device_id *matches)
 					&clk_provider_list, node) {
 			if (force || parent_ready(clk_provider->np)) {
 
+				/* Don't populate platform devices */
+				of_node_set_flag(clk_provider->np,
+						 OF_POPULATED);
+
 				clk_provider->clk_init_cb(clk_provider->np);
 				of_clk_set_defaults(clk_provider->np, true);
 
diff --git a/drivers/clk/h8300/clk-div.c b/drivers/clk/h8300/clk-div.c
index 4bf44a25d950..715b882205a8 100644
--- a/drivers/clk/h8300/clk-div.c
+++ b/drivers/clk/h8300/clk-div.c
@@ -14,7 +14,7 @@ static DEFINE_SPINLOCK(clklock);
 static void __init h8300_div_clk_setup(struct device_node *node)
 {
 	unsigned int num_parents;
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *clk_name = node->name;
 	const char *parent_name;
 	void __iomem *divcr = NULL;
@@ -38,15 +38,15 @@ static void __init h8300_div_clk_setup(struct device_node *node)
 
 	parent_name = of_clk_get_parent_name(node, 0);
 	of_property_read_u32(node, "renesas,width", &width);
-	clk = clk_register_divider(NULL, clk_name, parent_name,
+	hw = clk_hw_register_divider(NULL, clk_name, parent_name,
 				   CLK_SET_RATE_GATE, divcr, offset, width,
 				   CLK_DIVIDER_POWER_OF_TWO, &clklock);
-	if (!IS_ERR(clk)) {
-		of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	if (!IS_ERR(hw)) {
+		of_clk_add_hw_provider(node, of_clk_hw_simple_get, hw);
 		return;
 	}
 	pr_err("%s: failed to register %s div clock (%ld)\n",
-	       __func__, clk_name, PTR_ERR(clk));
+	       __func__, clk_name, PTR_ERR(hw));
 error:
 	if (divcr)
 		iounmap(divcr);
diff --git a/drivers/clk/h8300/clk-h8s2678.c b/drivers/clk/h8300/clk-h8s2678.c
index c9c2fd575ef7..a26312460621 100644
--- a/drivers/clk/h8300/clk-h8s2678.c
+++ b/drivers/clk/h8300/clk-h8s2678.c
@@ -84,11 +84,11 @@ static const struct clk_ops pll_ops = {
 static void __init h8s2678_pll_clk_setup(struct device_node *node)
 {
 	unsigned int num_parents;
-	struct clk *clk;
 	const char *clk_name = node->name;
 	const char *parent_name;
 	struct pll_clock *pll_clock;
 	struct clk_init_data init;
+	int ret;
 
 	num_parents = of_clk_get_parent_count(node);
 	if (!num_parents) {
@@ -121,14 +121,14 @@ static void __init h8s2678_pll_clk_setup(struct device_node *node)
 	init.num_parents = 1;
 	pll_clock->hw.init = &init;
 
-	clk = clk_register(NULL, &pll_clock->hw);
-	if (IS_ERR(clk)) {
-		pr_err("%s: failed to register %s div clock (%ld)\n",
-		       __func__, clk_name, PTR_ERR(clk));
+	ret = clk_hw_register(NULL, &pll_clock->hw);
+	if (ret) {
+		pr_err("%s: failed to register %s div clock (%d)\n",
+		       __func__, clk_name, ret);
 		goto unmap_pllcr;
 	}
 
-	of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	of_clk_add_hw_provider(node, of_clk_hw_simple_get, &pll_clock->hw);
 	return;
 
 unmap_pllcr:
diff --git a/drivers/clk/imx/clk-imx1.c b/drivers/clk/imx/clk-imx1.c
index 99cf802fa51f..eaa462ad09e8 100644
--- a/drivers/clk/imx/clk-imx1.c
+++ b/drivers/clk/imx/clk-imx1.c
@@ -45,10 +45,13 @@ static void __iomem *ccm __initdata;
 #define CCM_PCDR	(ccm + 0x0020)
 #define SCM_GCCR	(ccm + 0x0810)
 
-static void __init _mx1_clocks_init(unsigned long fref)
+static void __init mx1_clocks_init_dt(struct device_node *np)
 {
+	ccm = of_iomap(np, 0);
+	BUG_ON(!ccm);
+
 	clk[IMX1_CLK_DUMMY] = imx_clk_fixed("dummy", 0);
-	clk[IMX1_CLK_CLK32] = imx_obtain_fixed_clock("clk32", fref);
+	clk[IMX1_CLK_CLK32] = imx_obtain_fixed_clock("clk32", 32768);
 	clk[IMX1_CLK_CLK16M_EXT] = imx_clk_fixed("clk16m_ext", 16000000);
 	clk[IMX1_CLK_CLK16M] = imx_clk_gate("clk16m", "clk16m_ext", CCM_CSCR, 17);
 	clk[IMX1_CLK_CLK32_PREMULT] = imx_clk_fixed_factor("clk32_premult", "clk32", 512, 1);
@@ -74,45 +77,6 @@ static void __init _mx1_clocks_init(unsigned long fref)
 	clk[IMX1_CLK_USBD_GATE] = imx_clk_gate("usbd_gate", "clk48m", SCM_GCCR, 0);
 
 	imx_check_clocks(clk, ARRAY_SIZE(clk));
-}
-
-int __init mx1_clocks_init(unsigned long fref)
-{
-	ccm = ioremap(MX1_CCM_BASE_ADDR, SZ_4K);
-	BUG_ON(!ccm);
-
-	_mx1_clocks_init(fref);
-
-	clk_register_clkdev(clk[IMX1_CLK_PER1], "per", "imx-gpt.0");
-	clk_register_clkdev(clk[IMX1_CLK_HCLK], "ipg", "imx-gpt.0");
-	clk_register_clkdev(clk[IMX1_CLK_DMA_GATE], "ahb", "imx1-dma");
-	clk_register_clkdev(clk[IMX1_CLK_HCLK], "ipg", "imx1-dma");
-	clk_register_clkdev(clk[IMX1_CLK_PER1], "per", "imx1-uart.0");
-	clk_register_clkdev(clk[IMX1_CLK_HCLK], "ipg", "imx1-uart.0");
-	clk_register_clkdev(clk[IMX1_CLK_PER1], "per", "imx1-uart.1");
-	clk_register_clkdev(clk[IMX1_CLK_HCLK], "ipg", "imx1-uart.1");
-	clk_register_clkdev(clk[IMX1_CLK_PER1], "per", "imx1-uart.2");
-	clk_register_clkdev(clk[IMX1_CLK_UART3_GATE], "ipg", "imx1-uart.2");
-	clk_register_clkdev(clk[IMX1_CLK_HCLK], NULL, "imx1-i2c.0");
-	clk_register_clkdev(clk[IMX1_CLK_PER2], "per", "imx1-cspi.0");
-	clk_register_clkdev(clk[IMX1_CLK_DUMMY], "ipg", "imx1-cspi.0");
-	clk_register_clkdev(clk[IMX1_CLK_PER2], "per", "imx1-cspi.1");
-	clk_register_clkdev(clk[IMX1_CLK_DUMMY], "ipg", "imx1-cspi.1");
-	clk_register_clkdev(clk[IMX1_CLK_PER2], "per", "imx1-fb.0");
-	clk_register_clkdev(clk[IMX1_CLK_DUMMY], "ipg", "imx1-fb.0");
-	clk_register_clkdev(clk[IMX1_CLK_DUMMY], "ahb", "imx1-fb.0");
-
-	mxc_timer_init(MX1_TIM1_BASE_ADDR, MX1_TIM1_INT, GPT_TYPE_IMX1);
-
-	return 0;
-}
-
-static void __init mx1_clocks_init_dt(struct device_node *np)
-{
-	ccm = of_iomap(np, 0);
-	BUG_ON(!ccm);
-
-	_mx1_clocks_init(32768);
 
 	clk_data.clks = clk;
 	clk_data.clk_num = ARRAY_SIZE(clk);
diff --git a/drivers/clk/imx/clk-imx35.c b/drivers/clk/imx/clk-imx35.c
index b0978d3b83e2..203cad6c9aab 100644
--- a/drivers/clk/imx/clk-imx35.c
+++ b/drivers/clk/imx/clk-imx35.c
@@ -66,20 +66,22 @@ static const char *std_sel[] = {"ppll", "arm"};
 static const char *ipg_per_sel[] = {"ahb_per_div", "arm_per_div"};
 
 enum mx35_clks {
-	ckih, mpll, ppll, mpll_075, arm, hsp, hsp_div, hsp_sel, ahb, ipg,
-	arm_per_div, ahb_per_div, ipg_per, uart_sel, uart_div, esdhc_sel,
-	esdhc1_div, esdhc2_div, esdhc3_div, spdif_sel, spdif_div_pre,
-	spdif_div_post, ssi_sel, ssi1_div_pre, ssi1_div_post, ssi2_div_pre,
-	ssi2_div_post, usb_sel, usb_div, nfc_div, asrc_gate, pata_gate,
-	audmux_gate, can1_gate, can2_gate, cspi1_gate, cspi2_gate, ect_gate,
-	edio_gate, emi_gate, epit1_gate, epit2_gate, esai_gate, esdhc1_gate,
-	esdhc2_gate, esdhc3_gate, fec_gate, gpio1_gate, gpio2_gate, gpio3_gate,
-	gpt_gate, i2c1_gate, i2c2_gate, i2c3_gate, iomuxc_gate, ipu_gate,
-	kpp_gate, mlb_gate, mshc_gate, owire_gate, pwm_gate, rngc_gate,
-	rtc_gate, rtic_gate, scc_gate, sdma_gate, spba_gate, spdif_gate,
-	ssi1_gate, ssi2_gate, uart1_gate, uart2_gate, uart3_gate, usbotg_gate,
-	wdog_gate, max_gate, admux_gate, csi_gate, csi_div, csi_sel, iim_gate,
-	gpu2d_gate, ckil, clk_max
+	/*  0 */ ckih, mpll, ppll, mpll_075, arm, hsp, hsp_div, hsp_sel, ahb,
+	/*  9 */ ipg, arm_per_div, ahb_per_div, ipg_per, uart_sel, uart_div,
+	/* 15 */ esdhc_sel, esdhc1_div, esdhc2_div, esdhc3_div, spdif_sel,
+	/* 20 */ spdif_div_pre, spdif_div_post, ssi_sel, ssi1_div_pre,
+	/* 24 */ ssi1_div_post, ssi2_div_pre, ssi2_div_post, usb_sel, usb_div,
+	/* 29 */ nfc_div, asrc_gate, pata_gate, audmux_gate, can1_gate,
+	/* 34 */ can2_gate, cspi1_gate, cspi2_gate, ect_gate, edio_gate,
+	/* 39 */ emi_gate, epit1_gate, epit2_gate, esai_gate, esdhc1_gate,
+	/* 44 */ esdhc2_gate, esdhc3_gate, fec_gate, gpio1_gate, gpio2_gate,
+	/* 49 */ gpio3_gate, gpt_gate, i2c1_gate, i2c2_gate, i2c3_gate,
+	/* 54 */ iomuxc_gate, ipu_gate, kpp_gate, mlb_gate, mshc_gate,
+	/* 59 */ owire_gate, pwm_gate, rngc_gate, rtc_gate, rtic_gate, scc_gate,
+	/* 65 */ sdma_gate, spba_gate, spdif_gate, ssi1_gate, ssi2_gate,
+	/* 70 */ uart1_gate, uart2_gate, uart3_gate, usbotg_gate, wdog_gate,
+	/* 75 */ max_gate, admux_gate, csi_gate, csi_div, csi_sel, iim_gate,
+	/* 81 */ gpu2d_gate, ckil, clk_max
 };
 
 static struct clk *clk[clk_max];
@@ -115,7 +117,7 @@ static void __init _mx35_clocks_init(void)
 	}
 
 	clk[ckih] = imx_clk_fixed("ckih", 24000000);
-	clk[ckil] = imx_clk_fixed("ckih", 32768);
+	clk[ckil] = imx_clk_fixed("ckil", 32768);
 	clk[mpll] = imx_clk_pllv1(IMX_PLLV1_IMX35, "mpll", "ckih", base + MX35_CCM_MPCTL);
 	clk[ppll] = imx_clk_pllv1(IMX_PLLV1_IMX35, "ppll", "ckih", base + MX35_CCM_PPCTL);
 
diff --git a/drivers/clk/imx/clk-imx51-imx53.c b/drivers/clk/imx/clk-imx51-imx53.c
index 29d4c44ef356..1e3c9ea5f9dc 100644
--- a/drivers/clk/imx/clk-imx51-imx53.c
+++ b/drivers/clk/imx/clk-imx51-imx53.c
@@ -126,6 +126,7 @@ static const char *spdif0_com_sel[] = { "spdif0_podf", "ssi1_root_gate", };
 static const char *mx51_spdif1_com_sel[] = { "spdif1_podf", "ssi2_root_gate", };
 static const char *step_sels[] = { "lp_apm", };
 static const char *cpu_podf_sels[] = { "pll1_sw", "step_sel" };
+static const char *ieee1588_sels[] = { "pll3_sw", "pll4_sw", "dummy" /* usbphy2_clk */, "dummy" /* fec_phy_clk */ };
 
 static struct clk *clk[IMX5_CLK_END];
 static struct clk_onecell_data clk_data;
@@ -543,6 +544,25 @@ static void __init mx53_clocks_init(struct device_node *np)
 	clk[IMX5_CLK_I2C3_GATE]		= imx_clk_gate2("i2c3_gate", "per_root", MXC_CCM_CCGR1, 22);
 	clk[IMX5_CLK_SATA_GATE]		= imx_clk_gate2("sata_gate", "ipg", MXC_CCM_CCGR4, 2);
 
+	clk[IMX5_CLK_FIRI_SEL]		= imx_clk_mux("firi_sel", MXC_CCM_CSCMR2, 12, 2,
+						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
+	clk[IMX5_CLK_FIRI_PRED]		= imx_clk_divider("firi_pred", "firi_sel", MXC_CCM_CSCDR3, 6, 3);
+	clk[IMX5_CLK_FIRI_PODF]		= imx_clk_divider("firi_podf", "firi_pred", MXC_CCM_CSCDR3, 0, 6);
+	clk[IMX5_CLK_FIRI_SERIAL_GATE]	= imx_clk_gate2("firi_serial_gate", "firi_podf", MXC_CCM_CCGR1, 28);
+	clk[IMX5_CLK_FIRI_IPG_GATE]	= imx_clk_gate2("firi_ipg_gate", "ipg", MXC_CCM_CCGR1, 26);
+
+	clk[IMX5_CLK_CSI0_MCLK1_SEL]	= imx_clk_mux("csi0_mclk1_sel", MXC_CCM_CSCMR2, 22, 2,
+						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
+	clk[IMX5_CLK_CSI0_MCLK1_PRED]	= imx_clk_divider("csi0_mclk1_pred", "csi0_mclk1_sel", MXC_CCM_CSCDR4, 6, 3);
+	clk[IMX5_CLK_CSI0_MCLK1_PODF]	= imx_clk_divider("csi0_mclk1_podf", "csi0_mclk1_pred", MXC_CCM_CSCDR4, 0, 6);
+	clk[IMX5_CLK_CSI0_MCLK1_GATE]	= imx_clk_gate2("csi0_mclk1_serial_gate", "csi0_mclk1_podf", MXC_CCM_CCGR6, 4);
+
+	clk[IMX5_CLK_IEEE1588_SEL]	= imx_clk_mux("ieee1588_sel", MXC_CCM_CSCMR2, 14, 2,
+						ieee1588_sels, ARRAY_SIZE(ieee1588_sels));
+	clk[IMX5_CLK_IEEE1588_PRED]	= imx_clk_divider("ieee1588_pred", "ieee1588_sel", MXC_CCM_CSCDR2, 6, 3);
+	clk[IMX5_CLK_IEEE1588_PODF]	= imx_clk_divider("ieee1588_podf", "ieee1588_pred", MXC_CCM_CSCDR2, 0, 6);
+	clk[IMX5_CLK_IEEE1588_GATE]	= imx_clk_gate2("ieee1588_serial_gate", "ieee1588_podf", MXC_CCM_CCGR7, 6);
+
 	clk[IMX5_CLK_CKO1_SEL]		= imx_clk_mux("cko1_sel", MXC_CCM_CCOSR, 0, 4,
 						mx53_cko1_sel, ARRAY_SIZE(mx53_cko1_sel));
 	clk[IMX5_CLK_CKO1_PODF]		= imx_clk_divider("cko1_podf", "cko1_sel", MXC_CCM_CCOSR, 4, 3);
diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c
index ba1c1ae72ac2..ce8ea10407e4 100644
--- a/drivers/clk/imx/clk-imx6q.c
+++ b/drivers/clk/imx/clk-imx6q.c
@@ -318,11 +318,16 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
 		clk[IMX6QDL_CLK_IPG_PER_SEL] = imx_clk_mux("ipg_per_sel", base + 0x1c, 6, 1, ipg_per_sels, ARRAY_SIZE(ipg_per_sels));
 		clk[IMX6QDL_CLK_UART_SEL] = imx_clk_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels));
 		clk[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_mux("gpu2d_core_sel", base + 0x18, 16, 2, gpu2d_core_sels_2, ARRAY_SIZE(gpu2d_core_sels_2));
+	} else if (clk_on_imx6dl()) {
+		clk[IMX6QDL_CLK_MLB_SEL] = imx_clk_mux("mlb_sel",   base + 0x18, 16, 2, gpu2d_core_sels,   ARRAY_SIZE(gpu2d_core_sels));
 	} else {
 		clk[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_mux("gpu2d_core_sel",   base + 0x18, 16, 2, gpu2d_core_sels,   ARRAY_SIZE(gpu2d_core_sels));
 	}
 	clk[IMX6QDL_CLK_GPU3D_CORE_SEL]   = imx_clk_mux("gpu3d_core_sel",   base + 0x18, 4,  2, gpu3d_core_sels,   ARRAY_SIZE(gpu3d_core_sels));
-	clk[IMX6QDL_CLK_GPU3D_SHADER_SEL] = imx_clk_mux("gpu3d_shader_sel", base + 0x18, 8,  2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels));
+	if (clk_on_imx6dl())
+		clk[IMX6QDL_CLK_GPU2D_CORE_SEL] = imx_clk_mux("gpu2d_core_sel", base + 0x18, 8,  2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels));
+	else
+		clk[IMX6QDL_CLK_GPU3D_SHADER_SEL] = imx_clk_mux("gpu3d_shader_sel", base + 0x18, 8,  2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels));
 	clk[IMX6QDL_CLK_IPU1_SEL]         = imx_clk_mux("ipu1_sel",         base + 0x3c, 9,  2, ipu_sels,          ARRAY_SIZE(ipu_sels));
 	clk[IMX6QDL_CLK_IPU2_SEL]         = imx_clk_mux("ipu2_sel",         base + 0x3c, 14, 2, ipu_sels,          ARRAY_SIZE(ipu_sels));
 	clk[IMX6QDL_CLK_LDB_DI0_SEL]      = imx_clk_mux_flags("ldb_di0_sel", base + 0x2c, 9,  3, ldb_di_sels,      ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT);
@@ -400,9 +405,15 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
 		clk[IMX6QDL_CLK_LDB_DI0_DIV_3_5] = imx_clk_fixed_factor("ldb_di0_div_3_5", "ldb_di0_sel", 2, 7);
 		clk[IMX6QDL_CLK_LDB_DI1_DIV_3_5] = imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7);
 	}
-	clk[IMX6QDL_CLK_GPU2D_CORE_PODF]  = imx_clk_divider("gpu2d_core_podf",  "gpu2d_core_sel",    base + 0x18, 23, 3);
+	if (clk_on_imx6dl())
+		clk[IMX6QDL_CLK_MLB_PODF]  = imx_clk_divider("mlb_podf",  "mlb_sel",    base + 0x18, 23, 3);
+	else
+		clk[IMX6QDL_CLK_GPU2D_CORE_PODF]  = imx_clk_divider("gpu2d_core_podf",  "gpu2d_core_sel",    base + 0x18, 23, 3);
 	clk[IMX6QDL_CLK_GPU3D_CORE_PODF]  = imx_clk_divider("gpu3d_core_podf",  "gpu3d_core_sel",    base + 0x18, 26, 3);
-	clk[IMX6QDL_CLK_GPU3D_SHADER]     = imx_clk_divider("gpu3d_shader",     "gpu3d_shader_sel",  base + 0x18, 29, 3);
+	if (clk_on_imx6dl())
+		clk[IMX6QDL_CLK_GPU2D_CORE_PODF]  = imx_clk_divider("gpu2d_core_podf",     "gpu2d_core_sel",  base + 0x18, 29, 3);
+	else
+		clk[IMX6QDL_CLK_GPU3D_SHADER]     = imx_clk_divider("gpu3d_shader",     "gpu3d_shader_sel",  base + 0x18, 29, 3);
 	clk[IMX6QDL_CLK_IPU1_PODF]        = imx_clk_divider("ipu1_podf",        "ipu1_sel",          base + 0x3c, 11, 3);
 	clk[IMX6QDL_CLK_IPU2_PODF]        = imx_clk_divider("ipu2_podf",        "ipu2_sel",          base + 0x3c, 16, 3);
 	clk[IMX6QDL_CLK_LDB_DI0_PODF]     = imx_clk_divider_flags("ldb_di0_podf", "ldb_di0_div_3_5", base + 0x20, 10, 1, 0);
@@ -473,14 +484,7 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
 	clk[IMX6QDL_CLK_ESAI_MEM]     = imx_clk_gate2_shared("esai_mem", "ahb",             base + 0x6c, 16, &share_count_esai);
 	clk[IMX6QDL_CLK_GPT_IPG]      = imx_clk_gate2("gpt_ipg",       "ipg",               base + 0x6c, 20);
 	clk[IMX6QDL_CLK_GPT_IPG_PER]  = imx_clk_gate2("gpt_ipg_per",   "ipg_per",           base + 0x6c, 22);
-	if (clk_on_imx6dl())
-		/*
-		 * The multiplexer and divider of imx6q clock gpu3d_shader get
-		 * redefined/reused as gpu2d_core_sel and gpu2d_core_podf on imx6dl.
-		 */
-		clk[IMX6QDL_CLK_GPU2D_CORE] = imx_clk_gate2("gpu2d_core", "gpu3d_shader", base + 0x6c, 24);
-	else
-		clk[IMX6QDL_CLK_GPU2D_CORE] = imx_clk_gate2("gpu2d_core", "gpu2d_core_podf", base + 0x6c, 24);
+	clk[IMX6QDL_CLK_GPU2D_CORE] = imx_clk_gate2("gpu2d_core", "gpu2d_core_podf", base + 0x6c, 24);
 	clk[IMX6QDL_CLK_GPU3D_CORE]   = imx_clk_gate2("gpu3d_core",    "gpu3d_core_podf",   base + 0x6c, 26);
 	clk[IMX6QDL_CLK_HDMI_IAHB]    = imx_clk_gate2("hdmi_iahb",     "ahb",               base + 0x70, 0);
 	clk[IMX6QDL_CLK_HDMI_ISFR]    = imx_clk_gate2("hdmi_isfr",     "video_27m",         base + 0x70, 4);
@@ -511,7 +515,7 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
 		 * The multiplexer and divider of the imx6q clock gpu2d get
 		 * redefined/reused as mlb_sys_sel and mlb_sys_clk_podf on imx6dl.
 		 */
-		clk[IMX6QDL_CLK_MLB] = imx_clk_gate2("mlb",            "gpu2d_core_podf",   base + 0x74, 18);
+		clk[IMX6QDL_CLK_MLB] = imx_clk_gate2("mlb",            "mlb_podf",   base + 0x74, 18);
 	else
 		clk[IMX6QDL_CLK_MLB] = imx_clk_gate2("mlb",            "axi",               base + 0x74, 18);
 	clk[IMX6QDL_CLK_MMDC_CH0_AXI] = imx_clk_gate2("mmdc_ch0_axi",  "mmdc_ch0_axi_podf", base + 0x74, 20);
@@ -629,6 +633,24 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
 	if (IS_ENABLED(CONFIG_PCI_IMX6))
 		clk_set_parent(clk[IMX6QDL_CLK_LVDS1_SEL], clk[IMX6QDL_CLK_SATA_REF_100M]);
 
+	/*
+	 * Initialize the GPU clock muxes, so that the maximum specified clock
+	 * rates for the respective SoC are not exceeded.
+	 */
+	if (clk_on_imx6dl()) {
+		clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL],
+			       clk[IMX6QDL_CLK_PLL2_PFD1_594M]);
+		clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL],
+			       clk[IMX6QDL_CLK_PLL2_PFD1_594M]);
+	} else if (clk_on_imx6q()) {
+		clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL],
+			       clk[IMX6QDL_CLK_MMDC_CH0_AXI]);
+		clk_set_parent(clk[IMX6QDL_CLK_GPU3D_SHADER_SEL],
+			       clk[IMX6QDL_CLK_PLL2_PFD1_594M]);
+		clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL],
+			       clk[IMX6QDL_CLK_PLL3_USB_OTG]);
+	}
+
 	imx_register_uart_clocks(uart_clks);
 }
 CLK_OF_DECLARE(imx6q, "fsl,imx6q-ccm", imx6q_clocks_init);
diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c
index 6ed4f8fa0667..e7c7353a86fc 100644
--- a/drivers/clk/imx/clk-imx7d.c
+++ b/drivers/clk/imx/clk-imx7d.c
@@ -22,43 +22,63 @@
 
 #include "clk.h"
 
+static u32 share_count_sai1;
+static u32 share_count_sai2;
+static u32 share_count_sai3;
+
+static struct clk_div_table test_div_table[] = {
+	{ .val = 3, .div = 1, },
+	{ .val = 2, .div = 1, },
+	{ .val = 1, .div = 2, },
+	{ .val = 0, .div = 4, },
+	{ }
+};
+
+static struct clk_div_table post_div_table[] = {
+	{ .val = 3, .div = 4, },
+	{ .val = 2, .div = 1, },
+	{ .val = 1, .div = 2, },
+	{ .val = 0, .div = 1, },
+	{ }
+};
+
 static struct clk *clks[IMX7D_CLK_END];
 static const char *arm_a7_sel[] = { "osc", "pll_arm_main_clk",
 	"pll_enet_500m_clk", "pll_dram_main_clk",
-	"pll_sys_main_clk", "pll_sys_pfd0_392m_clk", "pll_audio_main_clk",
+	"pll_sys_main_clk", "pll_sys_pfd0_392m_clk", "pll_audio_post_div",
 	"pll_usb_main_clk", };
 
 static const char *arm_m4_sel[] = { "osc", "pll_sys_main_240m_clk",
 	"pll_enet_250m_clk", "pll_sys_pfd2_270m_clk",
-	"pll_dram_533m_clk", "pll_audio_main_clk", "pll_video_main_clk",
+	"pll_dram_533m_clk", "pll_audio_post_div", "pll_video_main_clk",
 	"pll_usb_main_clk", };
 
 static const char *arm_m0_sel[] = { "osc", "pll_sys_main_120m_clk",
 	"pll_enet_125m_clk", "pll_sys_pfd2_135m_clk",
-	"pll_dram_533m_clk", "pll_audio_main_clk", "pll_video_main_clk",
+	"pll_dram_533m_clk", "pll_audio_post_div", "pll_video_main_clk",
 	"pll_usb_main_clk", };
 
 static const char *axi_sel[] = { "osc", "pll_sys_pfd1_332m_clk",
 	"pll_dram_533m_clk", "pll_enet_250m_clk", "pll_sys_pfd5_clk",
-	"pll_audio_main_clk", "pll_video_main_clk", "pll_sys_pfd7_clk", };
+	"pll_audio_post_div", "pll_video_main_clk", "pll_sys_pfd7_clk", };
 
 static const char *disp_axi_sel[] = { "osc", "pll_sys_pfd1_332m_clk",
 	"pll_dram_533m_clk", "pll_enet_250m_clk", "pll_sys_pfd6_clk",
-	"pll_sys_pfd7_clk", "pll_audio_main_clk", "pll_video_main_clk", };
+	"pll_sys_pfd7_clk", "pll_audio_post_div", "pll_video_main_clk", };
 
 static const char *enet_axi_sel[] = { "osc", "pll_sys_pfd2_270m_clk",
 	"pll_dram_533m_clk", "pll_enet_250m_clk",
-	"pll_sys_main_240m_clk", "pll_audio_main_clk", "pll_video_main_clk",
+	"pll_sys_main_240m_clk", "pll_audio_post_div", "pll_video_main_clk",
 	"pll_sys_pfd4_clk", };
 
 static const char *nand_usdhc_bus_sel[] = { "osc", "pll_sys_pfd2_270m_clk",
 	"pll_dram_533m_clk", "pll_sys_main_240m_clk",
 	"pll_sys_pfd2_135m_clk", "pll_sys_pfd6_clk", "pll_enet_250m_clk",
-	"pll_audio_main_clk", };
+	"pll_audio_post_div", };
 
 static const char *ahb_channel_sel[] = { "osc", "pll_sys_pfd2_270m_clk",
 	"pll_dram_533m_clk", "pll_sys_pfd0_392m_clk",
-	"pll_enet_125m_clk", "pll_usb_main_clk", "pll_audio_main_clk",
+	"pll_enet_125m_clk", "pll_usb_main_clk", "pll_audio_post_div",
 	"pll_video_main_clk", };
 
 static const char *dram_phym_sel[] = { "pll_dram_main_clk",
@@ -69,13 +89,13 @@ static const char *dram_sel[] = { "pll_dram_main_clk",
 
 static const char *dram_phym_alt_sel[] = { "osc", "pll_dram_533m_clk",
 	"pll_sys_main_clk", "pll_enet_500m_clk",
-	"pll_usb_main_clk", "pll_sys_pfd7_clk", "pll_audio_main_clk",
+	"pll_usb_main_clk", "pll_sys_pfd7_clk", "pll_audio_post_div",
 	"pll_video_main_clk", };
 
 static const char *dram_alt_sel[] = { "osc", "pll_dram_533m_clk",
 	"pll_sys_main_clk", "pll_enet_500m_clk",
 	"pll_enet_250m_clk", "pll_sys_pfd0_392m_clk",
-	"pll_audio_main_clk", "pll_sys_pfd2_270m_clk", };
+	"pll_audio_post_div", "pll_sys_pfd2_270m_clk", };
 
 static const char *usb_hsic_sel[] = { "osc", "pll_sys_main_clk",
 	"pll_usb_main_clk", "pll_sys_pfd3_clk", "pll_sys_pfd4_clk",
@@ -101,53 +121,53 @@ static const char *lcdif_pixel_sel[] = { "osc", "pll_sys_pfd5_clk",
 
 static const char *mipi_dsi_sel[] = { "osc", "pll_sys_pfd5_clk",
 	"pll_sys_pfd3_clk", "pll_sys_main_clk", "pll_sys_pfd0_196m_clk",
-	"pll_dram_533m_clk", "pll_video_main_clk", "pll_audio_main_clk", };
+	"pll_dram_533m_clk", "pll_video_main_clk", "pll_audio_post_div", };
 
 static const char *mipi_csi_sel[] = { "osc", "pll_sys_pfd4_clk",
 	"pll_sys_pfd3_clk", "pll_sys_main_clk", "pll_sys_pfd0_196m_clk",
-	"pll_dram_533m_clk", "pll_video_main_clk", "pll_audio_main_clk", };
+	"pll_dram_533m_clk", "pll_video_main_clk", "pll_audio_post_div", };
 
 static const char *mipi_dphy_sel[] = { "osc", "pll_sys_main_120m_clk",
 	"pll_dram_533m_clk", "pll_sys_pfd5_clk", "ref_1m_clk", "ext_clk_2",
 	"pll_video_main_clk", "ext_clk_3", };
 
 static const char *sai1_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
-	"pll_audio_main_clk", "pll_dram_533m_clk", "pll_video_main_clk",
+	"pll_audio_post_div", "pll_dram_533m_clk", "pll_video_main_clk",
 	"pll_sys_pfd4_clk", "pll_enet_125m_clk", "ext_clk_2", };
 
 static const char *sai2_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
-	"pll_audio_main_clk", "pll_dram_533m_clk", "pll_video_main_clk",
+	"pll_audio_post_div", "pll_dram_533m_clk", "pll_video_main_clk",
 	"pll_sys_pfd4_clk", "pll_enet_125m_clk", "ext_clk_2", };
 
 static const char *sai3_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
-	"pll_audio_main_clk", "pll_dram_533m_clk", "pll_video_main_clk",
+	"pll_audio_post_div", "pll_dram_533m_clk", "pll_video_main_clk",
 	"pll_sys_pfd4_clk", "pll_enet_125m_clk", "ext_clk_3", };
 
 static const char *spdif_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
-	"pll_audio_main_clk", "pll_dram_533m_clk", "pll_video_main_clk",
+	"pll_audio_post_div", "pll_dram_533m_clk", "pll_video_main_clk",
 	"pll_sys_pfd4_clk", "pll_enet_125m_clk", "ext_3_clk", };
 
 static const char *enet1_ref_sel[] = { "osc", "pll_enet_125m_clk",
 	"pll_enet_50m_clk", "pll_enet_25m_clk",
-	"pll_sys_main_120m_clk", "pll_audio_main_clk", "pll_video_main_clk",
+	"pll_sys_main_120m_clk", "pll_audio_post_div", "pll_video_main_clk",
 	"ext_clk_4", };
 
 static const char *enet1_time_sel[] = { "osc", "pll_enet_100m_clk",
-	"pll_audio_main_clk", "ext_clk_1", "ext_clk_2", "ext_clk_3",
+	"pll_audio_post_div", "ext_clk_1", "ext_clk_2", "ext_clk_3",
 	"ext_clk_4", "pll_video_main_clk", };
 
 static const char *enet2_ref_sel[] = { "osc", "pll_enet_125m_clk",
 	"pll_enet_50m_clk", "pll_enet_25m_clk",
-	"pll_sys_main_120m_clk", "pll_audio_main_clk", "pll_video_main_clk",
+	"pll_sys_main_120m_clk", "pll_audio_post_div", "pll_video_main_clk",
 	"ext_clk_4", };
 
 static const char *enet2_time_sel[] = { "osc", "pll_enet_100m_clk",
-	"pll_audio_main_clk", "ext_clk_1", "ext_clk_2", "ext_clk_3",
+	"pll_audio_post_div", "ext_clk_1", "ext_clk_2", "ext_clk_3",
 	"ext_clk_4", "pll_video_main_clk", };
 
 static const char *enet_phy_ref_sel[] = { "osc", "pll_enet_25m_clk",
 	"pll_enet_50m_clk", "pll_enet_125m_clk",
-	"pll_dram_533m_clk", "pll_audio_main_clk", "pll_video_main_clk",
+	"pll_dram_533m_clk", "pll_audio_post_div", "pll_video_main_clk",
 	"pll_sys_pfd3_clk", };
 
 static const char *eim_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
@@ -188,22 +208,22 @@ static const char *can2_sel[] = { "osc", "pll_sys_main_120m_clk",
 
 static const char *i2c1_sel[] = { "osc", "pll_sys_main_120m_clk",
 	"pll_enet_50m_clk", "pll_dram_533m_clk",
-	"pll_audio_main_clk", "pll_video_main_clk", "pll_usb_main_clk",
+	"pll_audio_post_div", "pll_video_main_clk", "pll_usb_main_clk",
 	"pll_sys_pfd2_135m_clk", };
 
 static const char *i2c2_sel[] = { "osc", "pll_sys_main_120m_clk",
 	"pll_enet_50m_clk", "pll_dram_533m_clk",
-	"pll_audio_main_clk", "pll_video_main_clk", "pll_usb_main_clk",
+	"pll_audio_post_div", "pll_video_main_clk", "pll_usb_main_clk",
 	"pll_sys_pfd2_135m_clk", };
 
 static const char *i2c3_sel[] = { "osc", "pll_sys_main_120m_clk",
 	"pll_enet_50m_clk", "pll_dram_533m_clk",
-	"pll_audio_main_clk", "pll_video_main_clk", "pll_usb_main_clk",
+	"pll_audio_post_div", "pll_video_main_clk", "pll_usb_main_clk",
 	"pll_sys_pfd2_135m_clk", };
 
 static const char *i2c4_sel[] = { "osc", "pll_sys_main_120m_clk",
 	"pll_enet_50m_clk", "pll_dram_533m_clk",
-	"pll_audio_main_clk", "pll_video_main_clk", "pll_usb_main_clk",
+	"pll_audio_post_div", "pll_video_main_clk", "pll_usb_main_clk",
 	"pll_sys_pfd2_135m_clk", };
 
 static const char *uart1_sel[] = { "osc", "pll_sys_main_240m_clk",
@@ -262,32 +282,32 @@ static const char *ecspi4_sel[] = { "osc", "pll_sys_main_240m_clk",
 	"pll_usb_main_clk", };
 
 static const char *pwm1_sel[] = { "osc", "pll_enet_100m_clk",
-	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_main_clk",
+	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_post_div",
 	"ext_clk_1", "ref_1m_clk", "pll_video_main_clk", };
 
 static const char *pwm2_sel[] = { "osc", "pll_enet_100m_clk",
-	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_main_clk",
+	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_post_div",
 	"ext_clk_1", "ref_1m_clk", "pll_video_main_clk", };
 
 static const char *pwm3_sel[] = { "osc", "pll_enet_100m_clk",
-	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_main_clk",
+	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_post_div",
 	"ext_clk_2", "ref_1m_clk", "pll_video_main_clk", };
 
 static const char *pwm4_sel[] = { "osc", "pll_enet_100m_clk",
-	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_main_clk",
+	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_post_div",
 	"ext_clk_2", "ref_1m_clk", "pll_video_main_clk", };
 
 static const char *flextimer1_sel[] = { "osc", "pll_enet_100m_clk",
-	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_main_clk",
+	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_post_div",
 	"ext_clk_3", "ref_1m_clk", "pll_video_main_clk", };
 
 static const char *flextimer2_sel[] = { "osc", "pll_enet_100m_clk",
-	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_main_clk",
+	"pll_sys_main_120m_clk", "pll_enet_40m_clk", "pll_audio_post_div",
 	"ext_clk_3", "ref_1m_clk", "pll_video_main_clk", };
 
 static const char *sim1_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
 	"pll_sys_main_120m_clk", "pll_dram_533m_clk",
-	"pll_usb_main_clk", "pll_audio_main_clk", "pll_enet_125m_clk",
+	"pll_usb_main_clk", "pll_audio_post_div", "pll_enet_125m_clk",
 	"pll_sys_pfd7_clk", };
 
 static const char *sim2_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
@@ -297,19 +317,19 @@ static const char *sim2_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
 
 static const char *gpt1_sel[] = { "osc", "pll_enet_100m_clk",
 	"pll_sys_pfd0_392m_clk", "pll_enet_40m_clk", "pll_video_main_clk",
-	"ref_1m_clk", "pll_audio_main_clk", "ext_clk_1", };
+	"ref_1m_clk", "pll_audio_post_div", "ext_clk_1", };
 
 static const char *gpt2_sel[] = { "osc", "pll_enet_100m_clk",
 	"pll_sys_pfd0_392m_clk", "pll_enet_40m_clk", "pll_video_main_clk",
-	"ref_1m_clk", "pll_audio_main_clk", "ext_clk_2", };
+	"ref_1m_clk", "pll_audio_post_div", "ext_clk_2", };
 
 static const char *gpt3_sel[] = { "osc", "pll_enet_100m_clk",
 	"pll_sys_pfd0_392m_clk", "pll_enet_40m_clk", "pll_video_main_clk",
-	"ref_1m_clk", "pll_audio_main_clk", "ext_clk_3", };
+	"ref_1m_clk", "pll_audio_post_div", "ext_clk_3", };
 
 static const char *gpt4_sel[] = { "osc", "pll_enet_100m_clk",
 	"pll_sys_pfd0_392m_clk", "pll_enet_40m_clk", "pll_video_main_clk",
-	"ref_1m_clk", "pll_audio_main_clk", "ext_clk_4", };
+	"ref_1m_clk", "pll_audio_post_div", "ext_clk_4", };
 
 static const char *trace_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
 	"pll_sys_main_120m_clk", "pll_dram_533m_clk",
@@ -323,12 +343,12 @@ static const char *wdog_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
 
 static const char *csi_mclk_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
 	"pll_sys_main_120m_clk", "pll_dram_533m_clk",
-	"pll_enet_125m_clk", "pll_audio_main_clk", "pll_video_main_clk",
+	"pll_enet_125m_clk", "pll_audio_post_div", "pll_video_main_clk",
 	"pll_usb_main_clk", };
 
 static const char *audio_mclk_sel[] = { "osc", "pll_sys_pfd2_135m_clk",
 	"pll_sys_main_120m_clk", "pll_dram_533m_clk",
-	"pll_enet_125m_clk", "pll_audio_main_clk", "pll_video_main_clk",
+	"pll_enet_125m_clk", "pll_audio_post_div", "pll_video_main_clk",
 	"pll_usb_main_clk", };
 
 static const char *wrclk_sel[] = { "osc", "pll_enet_40m_clk",
@@ -342,13 +362,13 @@ static const char *clko1_sel[] = { "osc", "pll_sys_main_clk",
 
 static const char *clko2_sel[] = { "osc", "pll_sys_main_240m_clk",
 	"pll_sys_pfd0_392m_clk", "pll_sys_pfd1_166m_clk", "pll_sys_pfd4_clk",
-	"pll_audio_main_clk", "pll_video_main_clk", "ckil", };
+	"pll_audio_post_div", "pll_video_main_clk", "ckil", };
 
 static const char *lvds1_sel[] = { "pll_arm_main_clk",
 	"pll_sys_main_clk", "pll_sys_pfd0_392m_clk", "pll_sys_pfd1_332m_clk",
 	"pll_sys_pfd2_270m_clk", "pll_sys_pfd3_clk", "pll_sys_pfd4_clk",
 	"pll_sys_pfd5_clk", "pll_sys_pfd6_clk", "pll_sys_pfd7_clk",
-	"pll_audio_main_clk", "pll_video_main_clk", "pll_enet_500m_clk",
+	"pll_audio_post_div", "pll_video_main_clk", "pll_enet_500m_clk",
 	"pll_enet_250m_clk", "pll_enet_125m_clk", "pll_enet_100m_clk",
 	"pll_enet_50m_clk", "pll_enet_40m_clk", "pll_enet_25m_clk",
 	"pll_dram_main_clk", };
@@ -430,6 +450,11 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node)
 	clks[IMX7D_PLL_AUDIO_MAIN_CLK] = imx_clk_gate("pll_audio_main_clk", "pll_audio_main_bypass", base + 0xf0, 13);
 	clks[IMX7D_PLL_VIDEO_MAIN_CLK] = imx_clk_gate("pll_video_main_clk", "pll_video_main_bypass", base + 0x130, 13);
 
+	clks[IMX7D_PLL_AUDIO_TEST_DIV]  = clk_register_divider_table(NULL, "pll_audio_test_div", "pll_audio_main_clk",
+				CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xf0, 19, 2, 0, test_div_table, &imx_ccm_lock);
+	clks[IMX7D_PLL_AUDIO_POST_DIV] = clk_register_divider_table(NULL, "pll_audio_post_div", "pll_audio_test_div",
+				CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE, base + 0xf0, 22, 2, 0, post_div_table, &imx_ccm_lock);
+
 	clks[IMX7D_PLL_SYS_PFD0_392M_CLK] = imx_clk_pfd("pll_sys_pfd0_392m_clk", "pll_sys_main_clk", base + 0xc0, 0);
 	clks[IMX7D_PLL_SYS_PFD1_332M_CLK] = imx_clk_pfd("pll_sys_pfd1_332m_clk", "pll_sys_main_clk", base + 0xc0, 1);
 	clks[IMX7D_PLL_SYS_PFD2_270M_CLK] = imx_clk_pfd("pll_sys_pfd2_270m_clk", "pll_sys_main_clk", base + 0xc0, 2);
@@ -779,6 +804,7 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node)
 	clks[IMX7D_DRAM_PHYM_ALT_ROOT_CLK] = imx_clk_gate4("dram_phym_alt_root_clk", "dram_phym_alt_post_div", base + 0x4130, 0);
 	clks[IMX7D_DRAM_ALT_ROOT_CLK] = imx_clk_gate4("dram_alt_root_clk", "dram_alt_post_div", base + 0x4130, 0);
 	clks[IMX7D_USB_HSIC_ROOT_CLK] = imx_clk_gate4("usb_hsic_root_clk", "usb_hsic_post_div", base + 0x4420, 0);
+	clks[IMX7D_SDMA_CORE_CLK] = imx_clk_gate4("sdma_root_clk", "ahb_root_clk", base + 0x4480, 0);
 	clks[IMX7D_PCIE_CTRL_ROOT_CLK] = imx_clk_gate4("pcie_ctrl_root_clk", "pcie_ctrl_post_div", base + 0x4600, 0);
 	clks[IMX7D_PCIE_PHY_ROOT_CLK] = imx_clk_gate4("pcie_phy_root_clk", "pcie_phy_post_div", base + 0x4600, 0);
 	clks[IMX7D_EPDC_PIXEL_ROOT_CLK] = imx_clk_gate4("epdc_pixel_root_clk", "epdc_pixel_post_div", base + 0x44a0, 0);
@@ -786,9 +812,12 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node)
 	clks[IMX7D_MIPI_DSI_ROOT_CLK] = imx_clk_gate4("mipi_dsi_root_clk", "mipi_dsi_post_div", base + 0x4650, 0);
 	clks[IMX7D_MIPI_CSI_ROOT_CLK] = imx_clk_gate4("mipi_csi_root_clk", "mipi_csi_post_div", base + 0x4640, 0);
 	clks[IMX7D_MIPI_DPHY_ROOT_CLK] = imx_clk_gate4("mipi_dphy_root_clk", "mipi_dphy_post_div", base + 0x4660, 0);
-	clks[IMX7D_SAI1_ROOT_CLK] = imx_clk_gate4("sai1_root_clk", "sai1_post_div", base + 0x48c0, 0);
-	clks[IMX7D_SAI2_ROOT_CLK] = imx_clk_gate4("sai2_root_clk", "sai2_post_div", base + 0x48d0, 0);
-	clks[IMX7D_SAI3_ROOT_CLK] = imx_clk_gate4("sai3_root_clk", "sai3_post_div", base + 0x48e0, 0);
+	clks[IMX7D_SAI1_ROOT_CLK] = imx_clk_gate2_shared2("sai1_root_clk", "sai1_post_div", base + 0x48c0, 0, &share_count_sai1);
+	clks[IMX7D_SAI1_IPG_CLK]  = imx_clk_gate2_shared2("sai1_ipg_clk",  "ipg_root_clk",  base + 0x48c0, 0, &share_count_sai1);
+	clks[IMX7D_SAI2_ROOT_CLK] = imx_clk_gate2_shared2("sai2_root_clk", "sai2_post_div", base + 0x48d0, 0, &share_count_sai2);
+	clks[IMX7D_SAI2_IPG_CLK]  = imx_clk_gate2_shared2("sai2_ipg_clk",  "ipg_root_clk",  base + 0x48d0, 0, &share_count_sai2);
+	clks[IMX7D_SAI3_ROOT_CLK] = imx_clk_gate2_shared2("sai3_root_clk", "sai3_post_div", base + 0x48e0, 0, &share_count_sai3);
+	clks[IMX7D_SAI3_IPG_CLK]  = imx_clk_gate2_shared2("sai3_ipg_clk",  "ipg_root_clk",  base + 0x48e0, 0, &share_count_sai3);
 	clks[IMX7D_SPDIF_ROOT_CLK] = imx_clk_gate4("spdif_root_clk", "spdif_post_div", base + 0x44d0, 0);
 	clks[IMX7D_ENET1_REF_ROOT_CLK] = imx_clk_gate4("enet1_ref_root_clk", "enet1_ref_post_div", base + 0x44e0, 0);
 	clks[IMX7D_ENET1_TIME_ROOT_CLK] = imx_clk_gate4("enet1_time_root_clk", "enet1_time_post_div", base + 0x44f0, 0);
@@ -860,8 +889,6 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node)
 	/* use old gpt clk setting, gpt1 root clk must be twice as gpt counter freq */
 	clk_set_parent(clks[IMX7D_GPT1_ROOT_SRC], clks[IMX7D_OSC_24M_CLK]);
 
-	clk_set_parent(clks[IMX7D_ENET_AXI_ROOT_SRC], clks[IMX7D_PLL_ENET_MAIN_250M_CLK]);
-
 	/* set uart module clock's parent clock source that must be great then 80MHz */
 	clk_set_parent(clks[IMX7D_UART1_ROOT_SRC], clks[IMX7D_OSC_24M_CLK]);
 
diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h
index a81c0385ed64..3799ff82a9b4 100644
--- a/drivers/clk/imx/clk.h
+++ b/drivers/clk/imx/clk.h
@@ -134,6 +134,15 @@ static inline struct clk *imx_clk_gate2_shared(const char *name,
 			shift, 0x3, 0, &imx_ccm_lock, share_count);
 }
 
+static inline struct clk *imx_clk_gate2_shared2(const char *name,
+		const char *parent, void __iomem *reg, u8 shift,
+		unsigned int *share_count)
+{
+	return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT |
+				  CLK_OPS_PARENT_ENABLE, reg, shift, 0x3, 0,
+				  &imx_ccm_lock, share_count);
+}
+
 static inline struct clk *imx_clk_gate2_cgr(const char *name,
 		const char *parent, void __iomem *reg, u8 shift, u8 cgr_val)
 {
diff --git a/drivers/clk/loongson1/Makefile b/drivers/clk/loongson1/Makefile
new file mode 100644
index 000000000000..b7f6a16390e0
--- /dev/null
+++ b/drivers/clk/loongson1/Makefile
@@ -0,0 +1,3 @@
+obj-y				+= clk.o
+obj-$(CONFIG_LOONGSON1_LS1B)	+= clk-loongson1b.o
+obj-$(CONFIG_LOONGSON1_LS1C)	+= clk-loongson1c.o
diff --git a/drivers/clk/loongson1/clk-loongson1b.c b/drivers/clk/loongson1/clk-loongson1b.c
new file mode 100644
index 000000000000..f36a97e993c0
--- /dev/null
+++ b/drivers/clk/loongson1/clk-loongson1b.c
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2012-2016 Zhang, Keguang <keguang.zhang@gmail.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
+#include <linux/io.h>
+#include <linux/err.h>
+
+#include <loongson1.h>
+#include "clk.h"
+
+#define OSC		(33 * 1000000)
+#define DIV_APB		2
+
+static DEFINE_SPINLOCK(_lock);
+
+static unsigned long ls1x_pll_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	u32 pll, rate;
+
+	pll = __raw_readl(LS1X_CLK_PLL_FREQ);
+	rate = 12 + (pll & GENMASK(5, 0));
+	rate *= OSC;
+	rate >>= 1;
+
+	return rate;
+}
+
+static const struct clk_ops ls1x_pll_clk_ops = {
+	.recalc_rate = ls1x_pll_recalc_rate,
+};
+
+static const char *const cpu_parents[] = { "cpu_clk_div", "osc_clk", };
+static const char *const ahb_parents[] = { "ahb_clk_div", "osc_clk", };
+static const char *const dc_parents[] = { "dc_clk_div", "osc_clk", };
+
+void __init ls1x_clk_init(void)
+{
+	struct clk_hw *hw;
+
+	hw = clk_hw_register_fixed_rate(NULL, "osc_clk", NULL, 0, OSC);
+	clk_hw_register_clkdev(hw, "osc_clk", NULL);
+
+	/* clock derived from 33 MHz OSC clk */
+	hw = clk_hw_register_pll(NULL, "pll_clk", "osc_clk",
+				 &ls1x_pll_clk_ops, 0);
+	clk_hw_register_clkdev(hw, "pll_clk", NULL);
+
+	/* clock derived from PLL clk */
+	/*                                 _____
+	 *         _______________________|     |
+	 * OSC ___/                       | MUX |___ CPU CLK
+	 *        \___ PLL ___ CPU DIV ___|     |
+	 *                                |_____|
+	 */
+	hw = clk_hw_register_divider(NULL, "cpu_clk_div", "pll_clk",
+				   CLK_GET_RATE_NOCACHE, LS1X_CLK_PLL_DIV,
+				   DIV_CPU_SHIFT, DIV_CPU_WIDTH,
+				   CLK_DIVIDER_ONE_BASED |
+				   CLK_DIVIDER_ROUND_CLOSEST, &_lock);
+	clk_hw_register_clkdev(hw, "cpu_clk_div", NULL);
+	hw = clk_hw_register_mux(NULL, "cpu_clk", cpu_parents,
+			       ARRAY_SIZE(cpu_parents),
+			       CLK_SET_RATE_NO_REPARENT, LS1X_CLK_PLL_DIV,
+			       BYPASS_CPU_SHIFT, BYPASS_CPU_WIDTH, 0, &_lock);
+	clk_hw_register_clkdev(hw, "cpu_clk", NULL);
+
+	/*                                 _____
+	 *         _______________________|     |
+	 * OSC ___/                       | MUX |___ DC  CLK
+	 *        \___ PLL ___ DC  DIV ___|     |
+	 *                                |_____|
+	 */
+	hw = clk_hw_register_divider(NULL, "dc_clk_div", "pll_clk",
+				   0, LS1X_CLK_PLL_DIV, DIV_DC_SHIFT,
+				   DIV_DC_WIDTH, CLK_DIVIDER_ONE_BASED, &_lock);
+	clk_hw_register_clkdev(hw, "dc_clk_div", NULL);
+	hw = clk_hw_register_mux(NULL, "dc_clk", dc_parents,
+			       ARRAY_SIZE(dc_parents),
+			       CLK_SET_RATE_NO_REPARENT, LS1X_CLK_PLL_DIV,
+			       BYPASS_DC_SHIFT, BYPASS_DC_WIDTH, 0, &_lock);
+	clk_hw_register_clkdev(hw, "dc_clk", NULL);
+
+	/*                                 _____
+	 *         _______________________|     |
+	 * OSC ___/                       | MUX |___ DDR CLK
+	 *        \___ PLL ___ DDR DIV ___|     |
+	 *                                |_____|
+	 */
+	hw = clk_hw_register_divider(NULL, "ahb_clk_div", "pll_clk",
+				   0, LS1X_CLK_PLL_DIV, DIV_DDR_SHIFT,
+				   DIV_DDR_WIDTH, CLK_DIVIDER_ONE_BASED,
+				   &_lock);
+	clk_hw_register_clkdev(hw, "ahb_clk_div", NULL);
+	hw = clk_hw_register_mux(NULL, "ahb_clk", ahb_parents,
+			       ARRAY_SIZE(ahb_parents),
+			       CLK_SET_RATE_NO_REPARENT, LS1X_CLK_PLL_DIV,
+			       BYPASS_DDR_SHIFT, BYPASS_DDR_WIDTH, 0, &_lock);
+	clk_hw_register_clkdev(hw, "ahb_clk", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-dma", NULL);
+	clk_hw_register_clkdev(hw, "stmmaceth", NULL);
+
+	/* clock derived from AHB clk */
+	/* APB clk is always half of the AHB clk */
+	hw = clk_hw_register_fixed_factor(NULL, "apb_clk", "ahb_clk", 0, 1,
+					DIV_APB);
+	clk_hw_register_clkdev(hw, "apb_clk", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-ac97", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-i2c", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-nand", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-pwmtimer", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-spi", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-wdt", NULL);
+	clk_hw_register_clkdev(hw, "serial8250", NULL);
+}
diff --git a/drivers/clk/loongson1/clk-loongson1c.c b/drivers/clk/loongson1/clk-loongson1c.c
new file mode 100644
index 000000000000..3466f7320b40
--- /dev/null
+++ b/drivers/clk/loongson1/clk-loongson1c.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2016 Yang Ling <gnaygnil@gmail.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
+
+#include <loongson1.h>
+#include "clk.h"
+
+#define OSC		(24 * 1000000)
+#define DIV_APB		1
+
+static DEFINE_SPINLOCK(_lock);
+
+static unsigned long ls1x_pll_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	u32 pll, rate;
+
+	pll = __raw_readl(LS1X_CLK_PLL_FREQ);
+	rate = ((pll >> 8) & 0xff) + ((pll >> 16) & 0xff);
+	rate *= OSC;
+	rate >>= 2;
+
+	return rate;
+}
+
+static const struct clk_ops ls1x_pll_clk_ops = {
+	.recalc_rate = ls1x_pll_recalc_rate,
+};
+
+static const struct clk_div_table ahb_div_table[] = {
+	[0] = { .val = 0, .div = 2 },
+	[1] = { .val = 1, .div = 4 },
+	[2] = { .val = 2, .div = 3 },
+	[3] = { .val = 3, .div = 3 },
+};
+
+void __init ls1x_clk_init(void)
+{
+	struct clk_hw *hw;
+
+	hw = clk_hw_register_fixed_rate(NULL, "osc_clk", NULL, 0, OSC);
+	clk_hw_register_clkdev(hw, "osc_clk", NULL);
+
+	/* clock derived from 24 MHz OSC clk */
+	hw = clk_hw_register_pll(NULL, "pll_clk", "osc_clk",
+				&ls1x_pll_clk_ops, 0);
+	clk_hw_register_clkdev(hw, "pll_clk", NULL);
+
+	hw = clk_hw_register_divider(NULL, "cpu_clk_div", "pll_clk",
+				   CLK_GET_RATE_NOCACHE, LS1X_CLK_PLL_DIV,
+				   DIV_CPU_SHIFT, DIV_CPU_WIDTH,
+				   CLK_DIVIDER_ONE_BASED |
+				   CLK_DIVIDER_ROUND_CLOSEST, &_lock);
+	clk_hw_register_clkdev(hw, "cpu_clk_div", NULL);
+	hw = clk_hw_register_fixed_factor(NULL, "cpu_clk", "cpu_clk_div",
+					0, 1, 1);
+	clk_hw_register_clkdev(hw, "cpu_clk", NULL);
+
+	hw = clk_hw_register_divider(NULL, "dc_clk_div", "pll_clk",
+				   0, LS1X_CLK_PLL_DIV, DIV_DC_SHIFT,
+				   DIV_DC_WIDTH, CLK_DIVIDER_ONE_BASED, &_lock);
+	clk_hw_register_clkdev(hw, "dc_clk_div", NULL);
+	hw = clk_hw_register_fixed_factor(NULL, "dc_clk", "dc_clk_div",
+					0, 1, 1);
+	clk_hw_register_clkdev(hw, "dc_clk", NULL);
+
+	hw = clk_hw_register_divider_table(NULL, "ahb_clk_div", "cpu_clk_div",
+				0, LS1X_CLK_PLL_FREQ, DIV_DDR_SHIFT,
+				DIV_DDR_WIDTH, CLK_DIVIDER_ALLOW_ZERO,
+				ahb_div_table, &_lock);
+	clk_hw_register_clkdev(hw, "ahb_clk_div", NULL);
+	hw = clk_hw_register_fixed_factor(NULL, "ahb_clk", "ahb_clk_div",
+					0, 1, 1);
+	clk_hw_register_clkdev(hw, "ahb_clk", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-dma", NULL);
+	clk_hw_register_clkdev(hw, "stmmaceth", NULL);
+
+	/* clock derived from AHB clk */
+	hw = clk_hw_register_fixed_factor(NULL, "apb_clk", "ahb_clk", 0, 1,
+					DIV_APB);
+	clk_hw_register_clkdev(hw, "apb_clk", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-ac97", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-i2c", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-nand", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-pwmtimer", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-spi", NULL);
+	clk_hw_register_clkdev(hw, "ls1x-wdt", NULL);
+	clk_hw_register_clkdev(hw, "serial8250", NULL);
+}
diff --git a/drivers/clk/loongson1/clk.c b/drivers/clk/loongson1/clk.c
new file mode 100644
index 000000000000..cfcfd143fccb
--- /dev/null
+++ b/drivers/clk/loongson1/clk.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2012-2016 Zhang, Keguang <keguang.zhang@gmail.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/slab.h>
+
+struct clk_hw *__init clk_hw_register_pll(struct device *dev,
+					  const char *name,
+					  const char *parent_name,
+					  const struct clk_ops *ops,
+					  unsigned long flags)
+{
+	int ret;
+	struct clk_hw *hw;
+	struct clk_init_data init;
+
+	/* allocate the divider */
+	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
+	if (!hw)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = ops;
+	init.flags = flags | CLK_IS_BASIC;
+	init.parent_names = (parent_name ? &parent_name : NULL);
+	init.num_parents = (parent_name ? 1 : 0);
+	hw->init = &init;
+
+	/* register the clock */
+	ret = clk_hw_register(dev, hw);
+	if (ret) {
+		kfree(hw);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
diff --git a/drivers/clk/loongson1/clk.h b/drivers/clk/loongson1/clk.h
new file mode 100644
index 000000000000..085d74b5d496
--- /dev/null
+++ b/drivers/clk/loongson1/clk.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2012-2016 Zhang, Keguang <keguang.zhang@gmail.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#ifndef __LOONGSON1_CLK_H
+#define __LOONGSON1_CLK_H
+
+struct clk_hw *clk_hw_register_pll(struct device *dev,
+				   const char *name,
+				   const char *parent_name,
+				   const struct clk_ops *ops,
+				   unsigned long flags);
+
+#endif /* __LOONGSON1_CLK_H */
diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig
new file mode 100644
index 000000000000..380c372d528e
--- /dev/null
+++ b/drivers/clk/mediatek/Kconfig
@@ -0,0 +1,21 @@
+#
+# MediaTek SoC drivers
+#
+config COMMON_CLK_MEDIATEK
+	bool
+	---help---
+	  Mediatek SoCs' clock support.
+
+config COMMON_CLK_MT8135
+	bool "Clock driver for Mediatek MT8135"
+	select COMMON_CLK_MEDIATEK
+	default ARCH_MEDIATEK
+	---help---
+	  This driver supports Mediatek MT8135 clocks.
+
+config COMMON_CLK_MT8173
+	bool "Clock driver for Mediatek MT8173"
+	select COMMON_CLK_MEDIATEK
+	default ARCH_MEDIATEK
+	---help---
+	  This driver supports Mediatek MT8173 clocks.
diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile
index 95fdfacb2ebf..32e7222e7305 100644
--- a/drivers/clk/mediatek/Makefile
+++ b/drivers/clk/mediatek/Makefile
@@ -1,4 +1,4 @@
-obj-y += clk-mtk.o clk-pll.o clk-gate.o clk-apmixed.o
+obj-$(CONFIG_COMMON_CLK_MEDIATEK) += clk-mtk.o clk-pll.o clk-gate.o clk-apmixed.o
 obj-$(CONFIG_RESET_CONTROLLER) += reset.o
-obj-y += clk-mt8135.o
-obj-y += clk-mt8173.o
+obj-$(CONFIG_COMMON_CLK_MT8135) += clk-mt8135.o
+obj-$(CONFIG_COMMON_CLK_MT8173) += clk-mt8173.o
diff --git a/drivers/clk/mediatek/clk-gate.c b/drivers/clk/mediatek/clk-gate.c
index 2a76901bf04b..d8787bf444eb 100644
--- a/drivers/clk/mediatek/clk-gate.c
+++ b/drivers/clk/mediatek/clk-gate.c
@@ -97,7 +97,7 @@ const struct clk_ops mtk_clk_gate_ops_setclr_inv = {
 	.disable	= mtk_cg_disable_inv,
 };
 
-struct clk * __init mtk_clk_register_gate(
+struct clk *mtk_clk_register_gate(
 		const char *name,
 		const char *parent_name,
 		struct regmap *regmap,
diff --git a/drivers/clk/mediatek/clk-mt8173.c b/drivers/clk/mediatek/clk-mt8173.c
index 10c986018a08..0ac3aee87726 100644
--- a/drivers/clk/mediatek/clk-mt8173.c
+++ b/drivers/clk/mediatek/clk-mt8173.c
@@ -1074,8 +1074,10 @@ static void __init mtk_apmixedsys_init(struct device_node *node)
 	}
 
 	mt8173_pll_clk_data = clk_data = mtk_alloc_clk_data(CLK_APMIXED_NR_CLK);
-	if (!clk_data)
+	if (!clk_data) {
+		iounmap(base);
 		return;
+	}
 
 	mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data);
 
diff --git a/drivers/clk/mediatek/clk-mtk.c b/drivers/clk/mediatek/clk-mtk.c
index 5ada644e6200..bb30f7063569 100644
--- a/drivers/clk/mediatek/clk-mtk.c
+++ b/drivers/clk/mediatek/clk-mtk.c
@@ -24,7 +24,7 @@
 #include "clk-mtk.h"
 #include "clk-gate.h"
 
-struct clk_onecell_data * __init mtk_alloc_clk_data(unsigned int clk_num)
+struct clk_onecell_data *mtk_alloc_clk_data(unsigned int clk_num)
 {
 	int i;
 	struct clk_onecell_data *clk_data;
@@ -49,7 +49,7 @@ err_out:
 	return NULL;
 }
 
-void __init mtk_clk_register_fixed_clks(const struct mtk_fixed_clk *clks,
+void mtk_clk_register_fixed_clks(const struct mtk_fixed_clk *clks,
 		int num, struct clk_onecell_data *clk_data)
 {
 	int i;
@@ -72,7 +72,7 @@ void __init mtk_clk_register_fixed_clks(const struct mtk_fixed_clk *clks,
 	}
 }
 
-void __init mtk_clk_register_factors(const struct mtk_fixed_factor *clks,
+void mtk_clk_register_factors(const struct mtk_fixed_factor *clks,
 		int num, struct clk_onecell_data *clk_data)
 {
 	int i;
@@ -95,7 +95,7 @@ void __init mtk_clk_register_factors(const struct mtk_fixed_factor *clks,
 	}
 }
 
-int __init mtk_clk_register_gates(struct device_node *node,
+int mtk_clk_register_gates(struct device_node *node,
 		const struct mtk_gate *clks,
 		int num, struct clk_onecell_data *clk_data)
 {
@@ -135,7 +135,7 @@ int __init mtk_clk_register_gates(struct device_node *node,
 	return 0;
 }
 
-struct clk * __init mtk_clk_register_composite(const struct mtk_composite *mc,
+struct clk *mtk_clk_register_composite(const struct mtk_composite *mc,
 		void __iomem *base, spinlock_t *lock)
 {
 	struct clk *clk;
@@ -222,7 +222,7 @@ err_out:
 	return ERR_PTR(ret);
 }
 
-void __init mtk_clk_register_composites(const struct mtk_composite *mcs,
+void mtk_clk_register_composites(const struct mtk_composite *mcs,
 		int num, void __iomem *base, spinlock_t *lock,
 		struct clk_onecell_data *clk_data)
 {
diff --git a/drivers/clk/mediatek/clk-pll.c b/drivers/clk/mediatek/clk-pll.c
index 966cab1348da..0c2deac17ce9 100644
--- a/drivers/clk/mediatek/clk-pll.c
+++ b/drivers/clk/mediatek/clk-pll.c
@@ -313,7 +313,7 @@ static struct clk *mtk_clk_register_pll(const struct mtk_pll_data *data,
 	return clk;
 }
 
-void __init mtk_clk_register_plls(struct device_node *node,
+void mtk_clk_register_plls(struct device_node *node,
 		const struct mtk_pll_data *plls, int num_plls, struct clk_onecell_data *clk_data)
 {
 	void __iomem *base;
diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile
index 197e40175166..349583405b7c 100644
--- a/drivers/clk/meson/Makefile
+++ b/drivers/clk/meson/Makefile
@@ -3,5 +3,5 @@
 #
 
 obj-$(CONFIG_COMMON_CLK_AMLOGIC) += clk-pll.o clk-cpu.o clk-mpll.o
-obj-$(CONFIG_COMMON_CLK_MESON8B) += meson8b-clkc.o
-obj-$(CONFIG_COMMON_CLK_GXBB)	 += gxbb.o
+obj-$(CONFIG_COMMON_CLK_MESON8B) += meson8b.o
+obj-$(CONFIG_COMMON_CLK_GXBB)	 += gxbb.o gxbb-aoclk.o
diff --git a/drivers/clk/meson/clkc.h b/drivers/clk/meson/clkc.h
index 53326c32e853..9bb70e7a7d6a 100644
--- a/drivers/clk/meson/clkc.h
+++ b/drivers/clk/meson/clkc.h
@@ -98,7 +98,7 @@ struct meson_clk_mpll {
 };
 
 #define MESON_GATE(_name, _reg, _bit)					\
-struct clk_gate gxbb_##_name = { 						\
+struct clk_gate _name = { 						\
 	.reg = (void __iomem *) _reg, 					\
 	.bit_idx = (_bit), 						\
 	.lock = &clk_lock,						\
diff --git a/drivers/clk/meson/gxbb-aoclk.c b/drivers/clk/meson/gxbb-aoclk.c
new file mode 100644
index 000000000000..b45c5fba7e35
--- /dev/null
+++ b/drivers/clk/meson/gxbb-aoclk.c
@@ -0,0 +1,191 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright (c) 2016 BayLibre, SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * BSD LICENSE
+ *
+ * Copyright (c) 2016 BayLibre, SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ *   * Neither the name of Intel Corporation nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/reset-controller.h>
+#include <linux/init.h>
+#include <dt-bindings/clock/gxbb-aoclkc.h>
+#include <dt-bindings/reset/gxbb-aoclkc.h>
+
+static DEFINE_SPINLOCK(gxbb_aoclk_lock);
+
+struct gxbb_aoclk_reset_controller {
+	struct reset_controller_dev reset;
+	unsigned int *data;
+	void __iomem *base;
+};
+
+static int gxbb_aoclk_do_reset(struct reset_controller_dev *rcdev,
+			       unsigned long id)
+{
+	struct gxbb_aoclk_reset_controller *reset =
+		container_of(rcdev, struct gxbb_aoclk_reset_controller, reset);
+
+	writel(BIT(reset->data[id]), reset->base);
+
+	return 0;
+}
+
+static const struct reset_control_ops gxbb_aoclk_reset_ops = {
+	.reset = gxbb_aoclk_do_reset,
+};
+
+#define GXBB_AO_GATE(_name, _bit)					\
+static struct clk_gate _name##_ao = {					\
+	.reg = (void __iomem *)0,					\
+	.bit_idx = (_bit),						\
+	.lock = &gxbb_aoclk_lock,					\
+	.hw.init = &(struct clk_init_data) {				\
+		.name = #_name "_ao",					\
+		.ops = &clk_gate_ops,					\
+		.parent_names = (const char *[]){ "clk81" },		\
+		.num_parents = 1,					\
+		.flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED),	\
+	},								\
+}
+
+GXBB_AO_GATE(remote, 0);
+GXBB_AO_GATE(i2c_master, 1);
+GXBB_AO_GATE(i2c_slave, 2);
+GXBB_AO_GATE(uart1, 3);
+GXBB_AO_GATE(uart2, 5);
+GXBB_AO_GATE(ir_blaster, 6);
+
+static unsigned int gxbb_aoclk_reset[] = {
+	[RESET_AO_REMOTE] = 16,
+	[RESET_AO_I2C_MASTER] = 18,
+	[RESET_AO_I2C_SLAVE] = 19,
+	[RESET_AO_UART1] = 17,
+	[RESET_AO_UART2] = 22,
+	[RESET_AO_IR_BLASTER] = 23,
+};
+
+static struct clk_gate *gxbb_aoclk_gate[] = {
+	[CLKID_AO_REMOTE] = &remote_ao,
+	[CLKID_AO_I2C_MASTER] = &i2c_master_ao,
+	[CLKID_AO_I2C_SLAVE] = &i2c_slave_ao,
+	[CLKID_AO_UART1] = &uart1_ao,
+	[CLKID_AO_UART2] = &uart2_ao,
+	[CLKID_AO_IR_BLASTER] = &ir_blaster_ao,
+};
+
+static struct clk_hw_onecell_data gxbb_aoclk_onecell_data = {
+	.hws = {
+		[CLKID_AO_REMOTE] = &remote_ao.hw,
+		[CLKID_AO_I2C_MASTER] = &i2c_master_ao.hw,
+		[CLKID_AO_I2C_SLAVE] = &i2c_slave_ao.hw,
+		[CLKID_AO_UART1] = &uart1_ao.hw,
+		[CLKID_AO_UART2] = &uart2_ao.hw,
+		[CLKID_AO_IR_BLASTER] = &ir_blaster_ao.hw,
+	},
+	.num = ARRAY_SIZE(gxbb_aoclk_gate),
+};
+
+static int gxbb_aoclkc_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	void __iomem *base;
+	int ret, clkid;
+	struct device *dev = &pdev->dev;
+	struct gxbb_aoclk_reset_controller *rstc;
+
+	rstc = devm_kzalloc(dev, sizeof(*rstc), GFP_KERNEL);
+	if (!rstc)
+		return -ENOMEM;
+
+	/* Generic clocks */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	/* Reset Controller */
+	rstc->base = base;
+	rstc->data = gxbb_aoclk_reset;
+	rstc->reset.ops = &gxbb_aoclk_reset_ops;
+	rstc->reset.nr_resets = ARRAY_SIZE(gxbb_aoclk_reset);
+	rstc->reset.of_node = dev->of_node;
+	ret = devm_reset_controller_register(dev, &rstc->reset);
+
+	/*
+	 * Populate base address and register all clks
+	 */
+	for (clkid = 0; clkid < gxbb_aoclk_onecell_data.num; clkid++) {
+		gxbb_aoclk_gate[clkid]->reg = base;
+
+		ret = devm_clk_hw_register(dev,
+					gxbb_aoclk_onecell_data.hws[clkid]);
+		if (ret)
+			return ret;
+	}
+
+	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get,
+			&gxbb_aoclk_onecell_data);
+}
+
+static const struct of_device_id gxbb_aoclkc_match_table[] = {
+	{ .compatible = "amlogic,gxbb-aoclkc" },
+	{ }
+};
+
+static struct platform_driver gxbb_aoclkc_driver = {
+	.probe		= gxbb_aoclkc_probe,
+	.driver		= {
+		.name	= "gxbb-aoclkc",
+		.of_match_table = gxbb_aoclkc_match_table,
+	},
+};
+builtin_platform_driver(gxbb_aoclkc_driver);
diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c
index a4c6684b3019..9d9af446bafc 100644
--- a/drivers/clk/meson/gxbb.c
+++ b/drivers/clk/meson/gxbb.c
@@ -565,90 +565,93 @@ static struct clk_gate gxbb_clk81 = {
 };
 
 /* Everything Else (EE) domain gates */
-static MESON_GATE(ddr, HHI_GCLK_MPEG0, 0);
-static MESON_GATE(dos, HHI_GCLK_MPEG0, 1);
-static MESON_GATE(isa, HHI_GCLK_MPEG0, 5);
-static MESON_GATE(pl301, HHI_GCLK_MPEG0, 6);
-static MESON_GATE(periphs, HHI_GCLK_MPEG0, 7);
-static MESON_GATE(spicc, HHI_GCLK_MPEG0, 8);
-static MESON_GATE(i2c, HHI_GCLK_MPEG0, 9);
-static MESON_GATE(sar_adc, HHI_GCLK_MPEG0, 10);
-static MESON_GATE(smart_card, HHI_GCLK_MPEG0, 11);
-static MESON_GATE(rng0, HHI_GCLK_MPEG0, 12);
-static MESON_GATE(uart0, HHI_GCLK_MPEG0, 13);
-static MESON_GATE(sdhc, HHI_GCLK_MPEG0, 14);
-static MESON_GATE(stream, HHI_GCLK_MPEG0, 15);
-static MESON_GATE(async_fifo, HHI_GCLK_MPEG0, 16);
-static MESON_GATE(sdio, HHI_GCLK_MPEG0, 17);
-static MESON_GATE(abuf, HHI_GCLK_MPEG0, 18);
-static MESON_GATE(hiu_iface, HHI_GCLK_MPEG0, 19);
-static MESON_GATE(assist_misc, HHI_GCLK_MPEG0, 23);
-static MESON_GATE(spi, HHI_GCLK_MPEG0, 30);
-
-static MESON_GATE(i2s_spdif, HHI_GCLK_MPEG1, 2);
-static MESON_GATE(eth, HHI_GCLK_MPEG1, 3);
-static MESON_GATE(demux, HHI_GCLK_MPEG1, 4);
-static MESON_GATE(aiu_glue, HHI_GCLK_MPEG1, 6);
-static MESON_GATE(iec958, HHI_GCLK_MPEG1, 7);
-static MESON_GATE(i2s_out, HHI_GCLK_MPEG1, 8);
-static MESON_GATE(amclk, HHI_GCLK_MPEG1, 9);
-static MESON_GATE(aififo2, HHI_GCLK_MPEG1, 10);
-static MESON_GATE(mixer, HHI_GCLK_MPEG1, 11);
-static MESON_GATE(mixer_iface, HHI_GCLK_MPEG1, 12);
-static MESON_GATE(adc, HHI_GCLK_MPEG1, 13);
-static MESON_GATE(blkmv, HHI_GCLK_MPEG1, 14);
-static MESON_GATE(aiu, HHI_GCLK_MPEG1, 15);
-static MESON_GATE(uart1, HHI_GCLK_MPEG1, 16);
-static MESON_GATE(g2d, HHI_GCLK_MPEG1, 20);
-static MESON_GATE(usb0, HHI_GCLK_MPEG1, 21);
-static MESON_GATE(usb1, HHI_GCLK_MPEG1, 22);
-static MESON_GATE(reset, HHI_GCLK_MPEG1, 23);
-static MESON_GATE(nand, HHI_GCLK_MPEG1, 24);
-static MESON_GATE(dos_parser, HHI_GCLK_MPEG1, 25);
-static MESON_GATE(usb, HHI_GCLK_MPEG1, 26);
-static MESON_GATE(vdin1, HHI_GCLK_MPEG1, 28);
-static MESON_GATE(ahb_arb0, HHI_GCLK_MPEG1, 29);
-static MESON_GATE(efuse, HHI_GCLK_MPEG1, 30);
-static MESON_GATE(boot_rom, HHI_GCLK_MPEG1, 31);
-
-static MESON_GATE(ahb_data_bus, HHI_GCLK_MPEG2, 1);
-static MESON_GATE(ahb_ctrl_bus, HHI_GCLK_MPEG2, 2);
-static MESON_GATE(hdmi_intr_sync, HHI_GCLK_MPEG2, 3);
-static MESON_GATE(hdmi_pclk, HHI_GCLK_MPEG2, 4);
-static MESON_GATE(usb1_ddr_bridge, HHI_GCLK_MPEG2, 8);
-static MESON_GATE(usb0_ddr_bridge, HHI_GCLK_MPEG2, 9);
-static MESON_GATE(mmc_pclk, HHI_GCLK_MPEG2, 11);
-static MESON_GATE(dvin, HHI_GCLK_MPEG2, 12);
-static MESON_GATE(uart2, HHI_GCLK_MPEG2, 15);
-static MESON_GATE(sana, HHI_GCLK_MPEG2, 22);
-static MESON_GATE(vpu_intr, HHI_GCLK_MPEG2, 25);
-static MESON_GATE(sec_ahb_ahb3_bridge, HHI_GCLK_MPEG2, 26);
-static MESON_GATE(clk81_a53, HHI_GCLK_MPEG2, 29);
-
-static MESON_GATE(vclk2_venci0, HHI_GCLK_OTHER, 1);
-static MESON_GATE(vclk2_venci1, HHI_GCLK_OTHER, 2);
-static MESON_GATE(vclk2_vencp0, HHI_GCLK_OTHER, 3);
-static MESON_GATE(vclk2_vencp1, HHI_GCLK_OTHER, 4);
-static MESON_GATE(gclk_venci_int0, HHI_GCLK_OTHER, 8);
-static MESON_GATE(gclk_vencp_int, HHI_GCLK_OTHER, 9);
-static MESON_GATE(dac_clk, HHI_GCLK_OTHER, 10);
-static MESON_GATE(aoclk_gate, HHI_GCLK_OTHER, 14);
-static MESON_GATE(iec958_gate, HHI_GCLK_OTHER, 16);
-static MESON_GATE(enc480p, HHI_GCLK_OTHER, 20);
-static MESON_GATE(rng1, HHI_GCLK_OTHER, 21);
-static MESON_GATE(gclk_venci_int1, HHI_GCLK_OTHER, 22);
-static MESON_GATE(vclk2_venclmcc, HHI_GCLK_OTHER, 24);
-static MESON_GATE(vclk2_vencl, HHI_GCLK_OTHER, 25);
-static MESON_GATE(vclk_other, HHI_GCLK_OTHER, 26);
-static MESON_GATE(edp, HHI_GCLK_OTHER, 31);
+static MESON_GATE(gxbb_ddr, HHI_GCLK_MPEG0, 0);
+static MESON_GATE(gxbb_dos, HHI_GCLK_MPEG0, 1);
+static MESON_GATE(gxbb_isa, HHI_GCLK_MPEG0, 5);
+static MESON_GATE(gxbb_pl301, HHI_GCLK_MPEG0, 6);
+static MESON_GATE(gxbb_periphs, HHI_GCLK_MPEG0, 7);
+static MESON_GATE(gxbb_spicc, HHI_GCLK_MPEG0, 8);
+static MESON_GATE(gxbb_i2c, HHI_GCLK_MPEG0, 9);
+static MESON_GATE(gxbb_sar_adc, HHI_GCLK_MPEG0, 10);
+static MESON_GATE(gxbb_smart_card, HHI_GCLK_MPEG0, 11);
+static MESON_GATE(gxbb_rng0, HHI_GCLK_MPEG0, 12);
+static MESON_GATE(gxbb_uart0, HHI_GCLK_MPEG0, 13);
+static MESON_GATE(gxbb_sdhc, HHI_GCLK_MPEG0, 14);
+static MESON_GATE(gxbb_stream, HHI_GCLK_MPEG0, 15);
+static MESON_GATE(gxbb_async_fifo, HHI_GCLK_MPEG0, 16);
+static MESON_GATE(gxbb_sdio, HHI_GCLK_MPEG0, 17);
+static MESON_GATE(gxbb_abuf, HHI_GCLK_MPEG0, 18);
+static MESON_GATE(gxbb_hiu_iface, HHI_GCLK_MPEG0, 19);
+static MESON_GATE(gxbb_assist_misc, HHI_GCLK_MPEG0, 23);
+static MESON_GATE(gxbb_emmc_a, HHI_GCLK_MPEG0, 24);
+static MESON_GATE(gxbb_emmc_b, HHI_GCLK_MPEG0, 25);
+static MESON_GATE(gxbb_emmc_c, HHI_GCLK_MPEG0, 26);
+static MESON_GATE(gxbb_spi, HHI_GCLK_MPEG0, 30);
+
+static MESON_GATE(gxbb_i2s_spdif, HHI_GCLK_MPEG1, 2);
+static MESON_GATE(gxbb_eth, HHI_GCLK_MPEG1, 3);
+static MESON_GATE(gxbb_demux, HHI_GCLK_MPEG1, 4);
+static MESON_GATE(gxbb_aiu_glue, HHI_GCLK_MPEG1, 6);
+static MESON_GATE(gxbb_iec958, HHI_GCLK_MPEG1, 7);
+static MESON_GATE(gxbb_i2s_out, HHI_GCLK_MPEG1, 8);
+static MESON_GATE(gxbb_amclk, HHI_GCLK_MPEG1, 9);
+static MESON_GATE(gxbb_aififo2, HHI_GCLK_MPEG1, 10);
+static MESON_GATE(gxbb_mixer, HHI_GCLK_MPEG1, 11);
+static MESON_GATE(gxbb_mixer_iface, HHI_GCLK_MPEG1, 12);
+static MESON_GATE(gxbb_adc, HHI_GCLK_MPEG1, 13);
+static MESON_GATE(gxbb_blkmv, HHI_GCLK_MPEG1, 14);
+static MESON_GATE(gxbb_aiu, HHI_GCLK_MPEG1, 15);
+static MESON_GATE(gxbb_uart1, HHI_GCLK_MPEG1, 16);
+static MESON_GATE(gxbb_g2d, HHI_GCLK_MPEG1, 20);
+static MESON_GATE(gxbb_usb0, HHI_GCLK_MPEG1, 21);
+static MESON_GATE(gxbb_usb1, HHI_GCLK_MPEG1, 22);
+static MESON_GATE(gxbb_reset, HHI_GCLK_MPEG1, 23);
+static MESON_GATE(gxbb_nand, HHI_GCLK_MPEG1, 24);
+static MESON_GATE(gxbb_dos_parser, HHI_GCLK_MPEG1, 25);
+static MESON_GATE(gxbb_usb, HHI_GCLK_MPEG1, 26);
+static MESON_GATE(gxbb_vdin1, HHI_GCLK_MPEG1, 28);
+static MESON_GATE(gxbb_ahb_arb0, HHI_GCLK_MPEG1, 29);
+static MESON_GATE(gxbb_efuse, HHI_GCLK_MPEG1, 30);
+static MESON_GATE(gxbb_boot_rom, HHI_GCLK_MPEG1, 31);
+
+static MESON_GATE(gxbb_ahb_data_bus, HHI_GCLK_MPEG2, 1);
+static MESON_GATE(gxbb_ahb_ctrl_bus, HHI_GCLK_MPEG2, 2);
+static MESON_GATE(gxbb_hdmi_intr_sync, HHI_GCLK_MPEG2, 3);
+static MESON_GATE(gxbb_hdmi_pclk, HHI_GCLK_MPEG2, 4);
+static MESON_GATE(gxbb_usb1_ddr_bridge, HHI_GCLK_MPEG2, 8);
+static MESON_GATE(gxbb_usb0_ddr_bridge, HHI_GCLK_MPEG2, 9);
+static MESON_GATE(gxbb_mmc_pclk, HHI_GCLK_MPEG2, 11);
+static MESON_GATE(gxbb_dvin, HHI_GCLK_MPEG2, 12);
+static MESON_GATE(gxbb_uart2, HHI_GCLK_MPEG2, 15);
+static MESON_GATE(gxbb_sana, HHI_GCLK_MPEG2, 22);
+static MESON_GATE(gxbb_vpu_intr, HHI_GCLK_MPEG2, 25);
+static MESON_GATE(gxbb_sec_ahb_ahb3_bridge, HHI_GCLK_MPEG2, 26);
+static MESON_GATE(gxbb_clk81_a53, HHI_GCLK_MPEG2, 29);
+
+static MESON_GATE(gxbb_vclk2_venci0, HHI_GCLK_OTHER, 1);
+static MESON_GATE(gxbb_vclk2_venci1, HHI_GCLK_OTHER, 2);
+static MESON_GATE(gxbb_vclk2_vencp0, HHI_GCLK_OTHER, 3);
+static MESON_GATE(gxbb_vclk2_vencp1, HHI_GCLK_OTHER, 4);
+static MESON_GATE(gxbb_gclk_venci_int0, HHI_GCLK_OTHER, 8);
+static MESON_GATE(gxbb_gclk_vencp_int, HHI_GCLK_OTHER, 9);
+static MESON_GATE(gxbb_dac_clk, HHI_GCLK_OTHER, 10);
+static MESON_GATE(gxbb_aoclk_gate, HHI_GCLK_OTHER, 14);
+static MESON_GATE(gxbb_iec958_gate, HHI_GCLK_OTHER, 16);
+static MESON_GATE(gxbb_enc480p, HHI_GCLK_OTHER, 20);
+static MESON_GATE(gxbb_rng1, HHI_GCLK_OTHER, 21);
+static MESON_GATE(gxbb_gclk_venci_int1, HHI_GCLK_OTHER, 22);
+static MESON_GATE(gxbb_vclk2_venclmcc, HHI_GCLK_OTHER, 24);
+static MESON_GATE(gxbb_vclk2_vencl, HHI_GCLK_OTHER, 25);
+static MESON_GATE(gxbb_vclk_other, HHI_GCLK_OTHER, 26);
+static MESON_GATE(gxbb_edp, HHI_GCLK_OTHER, 31);
 
 /* Always On (AO) domain gates */
 
-static MESON_GATE(ao_media_cpu, HHI_GCLK_AO, 0);
-static MESON_GATE(ao_ahb_sram, HHI_GCLK_AO, 1);
-static MESON_GATE(ao_ahb_bus, HHI_GCLK_AO, 2);
-static MESON_GATE(ao_iface, HHI_GCLK_AO, 3);
-static MESON_GATE(ao_i2c, HHI_GCLK_AO, 4);
+static MESON_GATE(gxbb_ao_media_cpu, HHI_GCLK_AO, 0);
+static MESON_GATE(gxbb_ao_ahb_sram, HHI_GCLK_AO, 1);
+static MESON_GATE(gxbb_ao_ahb_bus, HHI_GCLK_AO, 2);
+static MESON_GATE(gxbb_ao_iface, HHI_GCLK_AO, 3);
+static MESON_GATE(gxbb_ao_i2c, HHI_GCLK_AO, 4);
 
 /* Array of all clocks provided by this provider */
 
@@ -748,6 +751,9 @@ static struct clk_hw_onecell_data gxbb_hw_onecell_data = {
 		[CLKID_AO_AHB_BUS]	    = &gxbb_ao_ahb_bus.hw,
 		[CLKID_AO_IFACE]	    = &gxbb_ao_iface.hw,
 		[CLKID_AO_I2C]		    = &gxbb_ao_i2c.hw,
+		[CLKID_SD_EMMC_A]	    = &gxbb_emmc_a.hw,
+		[CLKID_SD_EMMC_B]	    = &gxbb_emmc_b.hw,
+		[CLKID_SD_EMMC_C]	    = &gxbb_emmc_c.hw,
 	},
 	.num = NR_CLKS,
 };
@@ -847,6 +853,9 @@ static struct clk_gate *gxbb_clk_gates[] = {
 	&gxbb_ao_ahb_bus,
 	&gxbb_ao_iface,
 	&gxbb_ao_i2c,
+	&gxbb_emmc_a,
+	&gxbb_emmc_b,
+	&gxbb_emmc_c,
 };
 
 static int gxbb_clkc_probe(struct platform_device *pdev)
@@ -937,8 +946,4 @@ static struct platform_driver gxbb_driver = {
 	},
 };
 
-static int __init gxbb_clkc_init(void)
-{
-	return platform_driver_register(&gxbb_driver);
-}
-device_initcall(gxbb_clkc_init);
+builtin_platform_driver(gxbb_driver);
diff --git a/drivers/clk/meson/gxbb.h b/drivers/clk/meson/gxbb.h
index a2adf3448b59..0252939ba58f 100644
--- a/drivers/clk/meson/gxbb.h
+++ b/drivers/clk/meson/gxbb.h
@@ -170,11 +170,11 @@
  */
 #define CLKID_SYS_PLL		  0
 /* CLKID_CPUCLK */
-#define CLKID_HDMI_PLL		  2
+/* CLKID_HDMI_PLL */
 #define CLKID_FIXED_PLL		  3
-#define CLKID_FCLK_DIV2		  4
-#define CLKID_FCLK_DIV3		  5
-#define CLKID_FCLK_DIV4		  6
+/* CLKID_FCLK_DIV2 */
+/* CLKID_FCLK_DIV3 */
+/* CLKID_FCLK_DIV4 */
 #define CLKID_FCLK_DIV5		  7
 #define CLKID_FCLK_DIV7		  8
 #define CLKID_GP0_PLL		  9
@@ -183,14 +183,14 @@
 /* CLKID_CLK81 */
 #define CLKID_MPLL0		  13
 #define CLKID_MPLL1		  14
-#define CLKID_MPLL2		  15
+/* CLKID_MPLL2 */
 #define CLKID_DDR		  16
 #define CLKID_DOS		  17
 #define CLKID_ISA		  18
 #define CLKID_PL301		  19
 #define CLKID_PERIPHS		  20
 #define CLKID_SPICC		  21
-#define CLKID_I2C		  22
+/* CLKID_I2C */
 #define CLKID_SAR_ADC		  23
 #define CLKID_SMART_CARD	  24
 #define CLKID_RNG0		  25
@@ -202,7 +202,7 @@
 #define CLKID_ABUF		  31
 #define CLKID_HIU_IFACE		  32
 #define CLKID_ASSIST_MISC	  33
-#define CLKID_SPI		  34
+/* CLKID_SPI */
 #define CLKID_I2S_SPDIF		  35
 #define CLKID_ETH		  36
 #define CLKID_DEMUX		  37
@@ -218,12 +218,12 @@
 #define CLKID_AIU		  47
 #define CLKID_UART1		  48
 #define CLKID_G2D		  49
-#define CLKID_USB0		  50
-#define CLKID_USB1		  51
+/* CLKID_USB0 */
+/* CLKID_USB1 */
 #define CLKID_RESET		  52
 #define CLKID_NAND		  53
 #define CLKID_DOS_PARSER	  54
-#define CLKID_USB		  55
+/* CLKID_USB */
 #define CLKID_VDIN1		  56
 #define CLKID_AHB_ARB0		  57
 #define CLKID_EFUSE		  58
@@ -232,8 +232,8 @@
 #define CLKID_AHB_CTRL_BUS	  61
 #define CLKID_HDMI_INTR_SYNC	  62
 #define CLKID_HDMI_PCLK		  63
-#define CLKID_USB1_DDR_BRIDGE	  64
-#define CLKID_USB0_DDR_BRIDGE	  65
+/* CLKID_USB1_DDR_BRIDGE */
+/* CLKID_USB0_DDR_BRIDGE */
 #define CLKID_MMC_PCLK		  66
 #define CLKID_DVIN		  67
 #define CLKID_UART2		  68
@@ -261,9 +261,12 @@
 #define CLKID_AO_AHB_SRAM	  90
 #define CLKID_AO_AHB_BUS	  91
 #define CLKID_AO_IFACE		  92
-#define CLKID_AO_I2C		  93
+/* CLKID_AO_I2C */
+/* CLKID_SD_EMMC_A */
+/* CLKID_SD_EMMC_B */
+/* CLKID_SD_EMMC_C */
 
-#define NR_CLKS			  94
+#define NR_CLKS			  97
 
 /* include the CLKIDs that have been made part of the stable DT binding */
 #include <dt-bindings/clock/gxbb-clkc.h>
diff --git a/drivers/clk/meson/meson8b-clkc.c b/drivers/clk/meson/meson8b.c
index 4c9413cdf373..3f1be46cbb33 100644
--- a/drivers/clk/meson/meson8b-clkc.c
+++ b/drivers/clk/meson/meson8b.c
@@ -23,27 +23,11 @@
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/of_address.h>
-#include <dt-bindings/clock/meson8b-clkc.h>
 #include <linux/platform_device.h>
 #include <linux/init.h>
 
 #include "clkc.h"
-
-/*
- * Clock controller register offsets
- *
- * Register offsets from the HardKernel[0] data sheet are listed in comment
- * blocks below. Those offsets must be multiplied by 4 before adding them to
- * the base address to get the right value
- *
- * [0] http://dn.odroid.com/S805/Datasheet/S805_Datasheet%20V0.8%2020150126.pdf
- */
-#define MESON8B_REG_SYS_CPU_CNTL1	0x015c /* 0x57 offset in data sheet */
-#define MESON8B_REG_HHI_MPEG		0x0174 /* 0x5d offset in data sheet */
-#define MESON8B_REG_MALI		0x01b0 /* 0x6c offset in data sheet */
-#define MESON8B_REG_PLL_FIXED		0x0280
-#define MESON8B_REG_PLL_SYS		0x0300
-#define MESON8B_REG_PLL_VID		0x0320
+#include "meson8b.h"
 
 static DEFINE_SPINLOCK(clk_lock);
 
@@ -128,17 +112,17 @@ static struct clk_fixed_rate meson8b_xtal = {
 
 static struct meson_clk_pll meson8b_fixed_pll = {
 	.m = {
-		.reg_off = MESON8B_REG_PLL_FIXED,
+		.reg_off = HHI_MPLL_CNTL,
 		.shift   = 0,
 		.width   = 9,
 	},
 	.n = {
-		.reg_off = MESON8B_REG_PLL_FIXED,
+		.reg_off = HHI_MPLL_CNTL,
 		.shift   = 9,
 		.width   = 5,
 	},
 	.od = {
-		.reg_off = MESON8B_REG_PLL_FIXED,
+		.reg_off = HHI_MPLL_CNTL,
 		.shift   = 16,
 		.width   = 2,
 	},
@@ -154,17 +138,17 @@ static struct meson_clk_pll meson8b_fixed_pll = {
 
 static struct meson_clk_pll meson8b_vid_pll = {
 	.m = {
-		.reg_off = MESON8B_REG_PLL_VID,
+		.reg_off = HHI_VID_PLL_CNTL,
 		.shift   = 0,
 		.width   = 9,
 	},
 	.n = {
-		.reg_off = MESON8B_REG_PLL_VID,
+		.reg_off = HHI_VID_PLL_CNTL,
 		.shift   = 9,
 		.width   = 5,
 	},
 	.od = {
-		.reg_off = MESON8B_REG_PLL_VID,
+		.reg_off = HHI_VID_PLL_CNTL,
 		.shift   = 16,
 		.width   = 2,
 	},
@@ -180,17 +164,17 @@ static struct meson_clk_pll meson8b_vid_pll = {
 
 static struct meson_clk_pll meson8b_sys_pll = {
 	.m = {
-		.reg_off = MESON8B_REG_PLL_SYS,
+		.reg_off = HHI_SYS_PLL_CNTL,
 		.shift   = 0,
 		.width   = 9,
 	},
 	.n = {
-		.reg_off = MESON8B_REG_PLL_SYS,
+		.reg_off = HHI_SYS_PLL_CNTL,
 		.shift   = 9,
 		.width   = 5,
 	},
 	.od = {
-		.reg_off = MESON8B_REG_PLL_SYS,
+		.reg_off = HHI_SYS_PLL_CNTL,
 		.shift   = 16,
 		.width   = 2,
 	},
@@ -267,7 +251,7 @@ static struct clk_fixed_factor meson8b_fclk_div7 = {
  * forthcoming coordinated clock rates feature
  */
 static struct meson_clk_cpu meson8b_cpu_clk = {
-	.reg_off = MESON8B_REG_SYS_CPU_CNTL1,
+	.reg_off = HHI_SYS_CPU_CLK_CNTL1,
 	.div_table = cpu_div_table,
 	.clk_nb.notifier_call = meson_clk_cpu_notifier_cb,
 	.hw.init = &(struct clk_init_data){
@@ -281,7 +265,7 @@ static struct meson_clk_cpu meson8b_cpu_clk = {
 static u32 mux_table_clk81[]	= { 6, 5, 7 };
 
 struct clk_mux meson8b_mpeg_clk_sel = {
-	.reg = (void *)MESON8B_REG_HHI_MPEG,
+	.reg = (void *)HHI_MPEG_CLK_CNTL,
 	.mask = 0x7,
 	.shift = 12,
 	.flags = CLK_MUX_READ_ONLY,
@@ -303,7 +287,7 @@ struct clk_mux meson8b_mpeg_clk_sel = {
 };
 
 struct clk_divider meson8b_mpeg_clk_div = {
-	.reg = (void *)MESON8B_REG_HHI_MPEG,
+	.reg = (void *)HHI_MPEG_CLK_CNTL,
 	.shift = 0,
 	.width = 7,
 	.lock = &clk_lock,
@@ -317,7 +301,7 @@ struct clk_divider meson8b_mpeg_clk_div = {
 };
 
 struct clk_gate meson8b_clk81 = {
-	.reg = (void *)MESON8B_REG_HHI_MPEG,
+	.reg = (void *)HHI_MPEG_CLK_CNTL,
 	.bit_idx = 7,
 	.lock = &clk_lock,
 	.hw.init = &(struct clk_init_data){
@@ -329,6 +313,92 @@ struct clk_gate meson8b_clk81 = {
 	},
 };
 
+/* Everything Else (EE) domain gates */
+
+static MESON_GATE(meson8b_ddr, HHI_GCLK_MPEG0, 0);
+static MESON_GATE(meson8b_dos, HHI_GCLK_MPEG0, 1);
+static MESON_GATE(meson8b_isa, HHI_GCLK_MPEG0, 5);
+static MESON_GATE(meson8b_pl301, HHI_GCLK_MPEG0, 6);
+static MESON_GATE(meson8b_periphs, HHI_GCLK_MPEG0, 7);
+static MESON_GATE(meson8b_spicc, HHI_GCLK_MPEG0, 8);
+static MESON_GATE(meson8b_i2c, HHI_GCLK_MPEG0, 9);
+static MESON_GATE(meson8b_sar_adc, HHI_GCLK_MPEG0, 10);
+static MESON_GATE(meson8b_smart_card, HHI_GCLK_MPEG0, 11);
+static MESON_GATE(meson8b_rng0, HHI_GCLK_MPEG0, 12);
+static MESON_GATE(meson8b_uart0, HHI_GCLK_MPEG0, 13);
+static MESON_GATE(meson8b_sdhc, HHI_GCLK_MPEG0, 14);
+static MESON_GATE(meson8b_stream, HHI_GCLK_MPEG0, 15);
+static MESON_GATE(meson8b_async_fifo, HHI_GCLK_MPEG0, 16);
+static MESON_GATE(meson8b_sdio, HHI_GCLK_MPEG0, 17);
+static MESON_GATE(meson8b_abuf, HHI_GCLK_MPEG0, 18);
+static MESON_GATE(meson8b_hiu_iface, HHI_GCLK_MPEG0, 19);
+static MESON_GATE(meson8b_assist_misc, HHI_GCLK_MPEG0, 23);
+static MESON_GATE(meson8b_spi, HHI_GCLK_MPEG0, 30);
+
+static MESON_GATE(meson8b_i2s_spdif, HHI_GCLK_MPEG1, 2);
+static MESON_GATE(meson8b_eth, HHI_GCLK_MPEG1, 3);
+static MESON_GATE(meson8b_demux, HHI_GCLK_MPEG1, 4);
+static MESON_GATE(meson8b_aiu_glue, HHI_GCLK_MPEG1, 6);
+static MESON_GATE(meson8b_iec958, HHI_GCLK_MPEG1, 7);
+static MESON_GATE(meson8b_i2s_out, HHI_GCLK_MPEG1, 8);
+static MESON_GATE(meson8b_amclk, HHI_GCLK_MPEG1, 9);
+static MESON_GATE(meson8b_aififo2, HHI_GCLK_MPEG1, 10);
+static MESON_GATE(meson8b_mixer, HHI_GCLK_MPEG1, 11);
+static MESON_GATE(meson8b_mixer_iface, HHI_GCLK_MPEG1, 12);
+static MESON_GATE(meson8b_adc, HHI_GCLK_MPEG1, 13);
+static MESON_GATE(meson8b_blkmv, HHI_GCLK_MPEG1, 14);
+static MESON_GATE(meson8b_aiu, HHI_GCLK_MPEG1, 15);
+static MESON_GATE(meson8b_uart1, HHI_GCLK_MPEG1, 16);
+static MESON_GATE(meson8b_g2d, HHI_GCLK_MPEG1, 20);
+static MESON_GATE(meson8b_usb0, HHI_GCLK_MPEG1, 21);
+static MESON_GATE(meson8b_usb1, HHI_GCLK_MPEG1, 22);
+static MESON_GATE(meson8b_reset, HHI_GCLK_MPEG1, 23);
+static MESON_GATE(meson8b_nand, HHI_GCLK_MPEG1, 24);
+static MESON_GATE(meson8b_dos_parser, HHI_GCLK_MPEG1, 25);
+static MESON_GATE(meson8b_usb, HHI_GCLK_MPEG1, 26);
+static MESON_GATE(meson8b_vdin1, HHI_GCLK_MPEG1, 28);
+static MESON_GATE(meson8b_ahb_arb0, HHI_GCLK_MPEG1, 29);
+static MESON_GATE(meson8b_efuse, HHI_GCLK_MPEG1, 30);
+static MESON_GATE(meson8b_boot_rom, HHI_GCLK_MPEG1, 31);
+
+static MESON_GATE(meson8b_ahb_data_bus, HHI_GCLK_MPEG2, 1);
+static MESON_GATE(meson8b_ahb_ctrl_bus, HHI_GCLK_MPEG2, 2);
+static MESON_GATE(meson8b_hdmi_intr_sync, HHI_GCLK_MPEG2, 3);
+static MESON_GATE(meson8b_hdmi_pclk, HHI_GCLK_MPEG2, 4);
+static MESON_GATE(meson8b_usb1_ddr_bridge, HHI_GCLK_MPEG2, 8);
+static MESON_GATE(meson8b_usb0_ddr_bridge, HHI_GCLK_MPEG2, 9);
+static MESON_GATE(meson8b_mmc_pclk, HHI_GCLK_MPEG2, 11);
+static MESON_GATE(meson8b_dvin, HHI_GCLK_MPEG2, 12);
+static MESON_GATE(meson8b_uart2, HHI_GCLK_MPEG2, 15);
+static MESON_GATE(meson8b_sana, HHI_GCLK_MPEG2, 22);
+static MESON_GATE(meson8b_vpu_intr, HHI_GCLK_MPEG2, 25);
+static MESON_GATE(meson8b_sec_ahb_ahb3_bridge, HHI_GCLK_MPEG2, 26);
+static MESON_GATE(meson8b_clk81_a9, HHI_GCLK_MPEG2, 29);
+
+static MESON_GATE(meson8b_vclk2_venci0, HHI_GCLK_OTHER, 1);
+static MESON_GATE(meson8b_vclk2_venci1, HHI_GCLK_OTHER, 2);
+static MESON_GATE(meson8b_vclk2_vencp0, HHI_GCLK_OTHER, 3);
+static MESON_GATE(meson8b_vclk2_vencp1, HHI_GCLK_OTHER, 4);
+static MESON_GATE(meson8b_gclk_venci_int, HHI_GCLK_OTHER, 8);
+static MESON_GATE(meson8b_gclk_vencp_int, HHI_GCLK_OTHER, 9);
+static MESON_GATE(meson8b_dac_clk, HHI_GCLK_OTHER, 10);
+static MESON_GATE(meson8b_aoclk_gate, HHI_GCLK_OTHER, 14);
+static MESON_GATE(meson8b_iec958_gate, HHI_GCLK_OTHER, 16);
+static MESON_GATE(meson8b_enc480p, HHI_GCLK_OTHER, 20);
+static MESON_GATE(meson8b_rng1, HHI_GCLK_OTHER, 21);
+static MESON_GATE(meson8b_gclk_vencl_int, HHI_GCLK_OTHER, 22);
+static MESON_GATE(meson8b_vclk2_venclmcc, HHI_GCLK_OTHER, 24);
+static MESON_GATE(meson8b_vclk2_vencl, HHI_GCLK_OTHER, 25);
+static MESON_GATE(meson8b_vclk2_other, HHI_GCLK_OTHER, 26);
+static MESON_GATE(meson8b_edp, HHI_GCLK_OTHER, 31);
+
+/* Always On (AO) domain gates */
+
+static MESON_GATE(meson8b_ao_media_cpu, HHI_GCLK_AO, 0);
+static MESON_GATE(meson8b_ao_ahb_sram, HHI_GCLK_AO, 1);
+static MESON_GATE(meson8b_ao_ahb_bus, HHI_GCLK_AO, 2);
+static MESON_GATE(meson8b_ao_iface, HHI_GCLK_AO, 3);
+
 static struct clk_hw_onecell_data meson8b_hw_onecell_data = {
 	.hws = {
 		[CLKID_XTAL] = &meson8b_xtal.hw,
@@ -344,6 +414,83 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = {
 		[CLKID_MPEG_SEL] = &meson8b_mpeg_clk_sel.hw,
 		[CLKID_MPEG_DIV] = &meson8b_mpeg_clk_div.hw,
 		[CLKID_CLK81] = &meson8b_clk81.hw,
+		[CLKID_DDR]		    = &meson8b_ddr.hw,
+		[CLKID_DOS]		    = &meson8b_dos.hw,
+		[CLKID_ISA]		    = &meson8b_isa.hw,
+		[CLKID_PL301]		    = &meson8b_pl301.hw,
+		[CLKID_PERIPHS]		    = &meson8b_periphs.hw,
+		[CLKID_SPICC]		    = &meson8b_spicc.hw,
+		[CLKID_I2C]		    = &meson8b_i2c.hw,
+		[CLKID_SAR_ADC]		    = &meson8b_sar_adc.hw,
+		[CLKID_SMART_CARD]	    = &meson8b_smart_card.hw,
+		[CLKID_RNG0]		    = &meson8b_rng0.hw,
+		[CLKID_UART0]		    = &meson8b_uart0.hw,
+		[CLKID_SDHC]		    = &meson8b_sdhc.hw,
+		[CLKID_STREAM]		    = &meson8b_stream.hw,
+		[CLKID_ASYNC_FIFO]	    = &meson8b_async_fifo.hw,
+		[CLKID_SDIO]		    = &meson8b_sdio.hw,
+		[CLKID_ABUF]		    = &meson8b_abuf.hw,
+		[CLKID_HIU_IFACE]	    = &meson8b_hiu_iface.hw,
+		[CLKID_ASSIST_MISC]	    = &meson8b_assist_misc.hw,
+		[CLKID_SPI]		    = &meson8b_spi.hw,
+		[CLKID_I2S_SPDIF]	    = &meson8b_i2s_spdif.hw,
+		[CLKID_ETH]		    = &meson8b_eth.hw,
+		[CLKID_DEMUX]		    = &meson8b_demux.hw,
+		[CLKID_AIU_GLUE]	    = &meson8b_aiu_glue.hw,
+		[CLKID_IEC958]		    = &meson8b_iec958.hw,
+		[CLKID_I2S_OUT]		    = &meson8b_i2s_out.hw,
+		[CLKID_AMCLK]		    = &meson8b_amclk.hw,
+		[CLKID_AIFIFO2]		    = &meson8b_aififo2.hw,
+		[CLKID_MIXER]		    = &meson8b_mixer.hw,
+		[CLKID_MIXER_IFACE]	    = &meson8b_mixer_iface.hw,
+		[CLKID_ADC]		    = &meson8b_adc.hw,
+		[CLKID_BLKMV]		    = &meson8b_blkmv.hw,
+		[CLKID_AIU]		    = &meson8b_aiu.hw,
+		[CLKID_UART1]		    = &meson8b_uart1.hw,
+		[CLKID_G2D]		    = &meson8b_g2d.hw,
+		[CLKID_USB0]		    = &meson8b_usb0.hw,
+		[CLKID_USB1]		    = &meson8b_usb1.hw,
+		[CLKID_RESET]		    = &meson8b_reset.hw,
+		[CLKID_NAND]		    = &meson8b_nand.hw,
+		[CLKID_DOS_PARSER]	    = &meson8b_dos_parser.hw,
+		[CLKID_USB]		    = &meson8b_usb.hw,
+		[CLKID_VDIN1]		    = &meson8b_vdin1.hw,
+		[CLKID_AHB_ARB0]	    = &meson8b_ahb_arb0.hw,
+		[CLKID_EFUSE]		    = &meson8b_efuse.hw,
+		[CLKID_BOOT_ROM]	    = &meson8b_boot_rom.hw,
+		[CLKID_AHB_DATA_BUS]	    = &meson8b_ahb_data_bus.hw,
+		[CLKID_AHB_CTRL_BUS]	    = &meson8b_ahb_ctrl_bus.hw,
+		[CLKID_HDMI_INTR_SYNC]	    = &meson8b_hdmi_intr_sync.hw,
+		[CLKID_HDMI_PCLK]	    = &meson8b_hdmi_pclk.hw,
+		[CLKID_USB1_DDR_BRIDGE]	    = &meson8b_usb1_ddr_bridge.hw,
+		[CLKID_USB0_DDR_BRIDGE]	    = &meson8b_usb0_ddr_bridge.hw,
+		[CLKID_MMC_PCLK]	    = &meson8b_mmc_pclk.hw,
+		[CLKID_DVIN]		    = &meson8b_dvin.hw,
+		[CLKID_UART2]		    = &meson8b_uart2.hw,
+		[CLKID_SANA]		    = &meson8b_sana.hw,
+		[CLKID_VPU_INTR]	    = &meson8b_vpu_intr.hw,
+		[CLKID_SEC_AHB_AHB3_BRIDGE] = &meson8b_sec_ahb_ahb3_bridge.hw,
+		[CLKID_CLK81_A9]	    = &meson8b_clk81_a9.hw,
+		[CLKID_VCLK2_VENCI0]	    = &meson8b_vclk2_venci0.hw,
+		[CLKID_VCLK2_VENCI1]	    = &meson8b_vclk2_venci1.hw,
+		[CLKID_VCLK2_VENCP0]	    = &meson8b_vclk2_vencp0.hw,
+		[CLKID_VCLK2_VENCP1]	    = &meson8b_vclk2_vencp1.hw,
+		[CLKID_GCLK_VENCI_INT]	    = &meson8b_gclk_venci_int.hw,
+		[CLKID_GCLK_VENCP_INT]	    = &meson8b_gclk_vencp_int.hw,
+		[CLKID_DAC_CLK]		    = &meson8b_dac_clk.hw,
+		[CLKID_AOCLK_GATE]	    = &meson8b_aoclk_gate.hw,
+		[CLKID_IEC958_GATE]	    = &meson8b_iec958_gate.hw,
+		[CLKID_ENC480P]		    = &meson8b_enc480p.hw,
+		[CLKID_RNG1]		    = &meson8b_rng1.hw,
+		[CLKID_GCLK_VENCL_INT]	    = &meson8b_gclk_vencl_int.hw,
+		[CLKID_VCLK2_VENCLMCC]	    = &meson8b_vclk2_venclmcc.hw,
+		[CLKID_VCLK2_VENCL]	    = &meson8b_vclk2_vencl.hw,
+		[CLKID_VCLK2_OTHER]	    = &meson8b_vclk2_other.hw,
+		[CLKID_EDP]		    = &meson8b_edp.hw,
+		[CLKID_AO_MEDIA_CPU]	    = &meson8b_ao_media_cpu.hw,
+		[CLKID_AO_AHB_SRAM]	    = &meson8b_ao_ahb_sram.hw,
+		[CLKID_AO_AHB_BUS]	    = &meson8b_ao_ahb_bus.hw,
+		[CLKID_AO_IFACE]	    = &meson8b_ao_iface.hw,
 	},
 	.num = CLK_NR_CLKS,
 };
@@ -354,6 +501,87 @@ static struct meson_clk_pll *const meson8b_clk_plls[] = {
 	&meson8b_sys_pll,
 };
 
+static struct clk_gate *meson8b_clk_gates[] = {
+	&meson8b_clk81,
+	&meson8b_ddr,
+	&meson8b_dos,
+	&meson8b_isa,
+	&meson8b_pl301,
+	&meson8b_periphs,
+	&meson8b_spicc,
+	&meson8b_i2c,
+	&meson8b_sar_adc,
+	&meson8b_smart_card,
+	&meson8b_rng0,
+	&meson8b_uart0,
+	&meson8b_sdhc,
+	&meson8b_stream,
+	&meson8b_async_fifo,
+	&meson8b_sdio,
+	&meson8b_abuf,
+	&meson8b_hiu_iface,
+	&meson8b_assist_misc,
+	&meson8b_spi,
+	&meson8b_i2s_spdif,
+	&meson8b_eth,
+	&meson8b_demux,
+	&meson8b_aiu_glue,
+	&meson8b_iec958,
+	&meson8b_i2s_out,
+	&meson8b_amclk,
+	&meson8b_aififo2,
+	&meson8b_mixer,
+	&meson8b_mixer_iface,
+	&meson8b_adc,
+	&meson8b_blkmv,
+	&meson8b_aiu,
+	&meson8b_uart1,
+	&meson8b_g2d,
+	&meson8b_usb0,
+	&meson8b_usb1,
+	&meson8b_reset,
+	&meson8b_nand,
+	&meson8b_dos_parser,
+	&meson8b_usb,
+	&meson8b_vdin1,
+	&meson8b_ahb_arb0,
+	&meson8b_efuse,
+	&meson8b_boot_rom,
+	&meson8b_ahb_data_bus,
+	&meson8b_ahb_ctrl_bus,
+	&meson8b_hdmi_intr_sync,
+	&meson8b_hdmi_pclk,
+	&meson8b_usb1_ddr_bridge,
+	&meson8b_usb0_ddr_bridge,
+	&meson8b_mmc_pclk,
+	&meson8b_dvin,
+	&meson8b_uart2,
+	&meson8b_sana,
+	&meson8b_vpu_intr,
+	&meson8b_sec_ahb_ahb3_bridge,
+	&meson8b_clk81_a9,
+	&meson8b_vclk2_venci0,
+	&meson8b_vclk2_venci1,
+	&meson8b_vclk2_vencp0,
+	&meson8b_vclk2_vencp1,
+	&meson8b_gclk_venci_int,
+	&meson8b_gclk_vencp_int,
+	&meson8b_dac_clk,
+	&meson8b_aoclk_gate,
+	&meson8b_iec958_gate,
+	&meson8b_enc480p,
+	&meson8b_rng1,
+	&meson8b_gclk_vencl_int,
+	&meson8b_vclk2_venclmcc,
+	&meson8b_vclk2_vencl,
+	&meson8b_vclk2_other,
+	&meson8b_edp,
+	&meson8b_ao_media_cpu,
+	&meson8b_ao_ahb_sram,
+	&meson8b_ao_ahb_bus,
+	&meson8b_ao_iface,
+};
+
 static int meson8b_clkc_probe(struct platform_device *pdev)
 {
 	void __iomem *clk_base;
@@ -381,6 +609,11 @@ static int meson8b_clkc_probe(struct platform_device *pdev)
 	meson8b_mpeg_clk_div.reg = clk_base + (u32)meson8b_mpeg_clk_div.reg;
 	meson8b_clk81.reg = clk_base + (u32)meson8b_clk81.reg;
 
+	/* Populate base address for gates */
+	for (i = 0; i < ARRAY_SIZE(meson8b_clk_gates); i++)
+		meson8b_clk_gates[i]->reg = clk_base +
+			(u32)meson8b_clk_gates[i]->reg;
+
 	/*
 	 * register all clks
 	 * CLKID_UNUSED = 0, so skip it and start with CLKID_XTAL = 1
@@ -440,8 +673,4 @@ static struct platform_driver meson8b_driver = {
 	},
 };
 
-static int __init meson8b_clkc_init(void)
-{
-	return platform_driver_register(&meson8b_driver);
-}
-device_initcall(meson8b_clkc_init);
+builtin_platform_driver(meson8b_driver);
diff --git a/drivers/clk/meson/meson8b.h b/drivers/clk/meson/meson8b.h
new file mode 100644
index 000000000000..010e9582888d
--- /dev/null
+++ b/drivers/clk/meson/meson8b.h
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2015 Endless Mobile, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * Copyright (c) 2016 BayLibre, Inc.
+ * Michael Turquette <mturquette@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __MESON8B_H
+#define __MESON8B_H
+
+/*
+ * Clock controller register offsets
+ *
+ * Register offsets from the HardKernel[0] data sheet are listed in comment
+ * blocks below. Those offsets must be multiplied by 4 before adding them to
+ * the base address to get the right value
+ *
+ * [0] http://dn.odroid.com/S805/Datasheet/S805_Datasheet%20V0.8%2020150126.pdf
+ */
+#define HHI_GCLK_MPEG0			0x140 /* 0x50 offset in data sheet */
+#define HHI_GCLK_MPEG1			0x144 /* 0x51 offset in data sheet */
+#define HHI_GCLK_MPEG2			0x148 /* 0x52 offset in data sheet */
+#define HHI_GCLK_OTHER			0x150 /* 0x54 offset in data sheet */
+#define HHI_GCLK_AO			0x154 /* 0x55 offset in data sheet */
+#define HHI_SYS_CPU_CLK_CNTL1		0x15c /* 0x57 offset in data sheet */
+#define HHI_MPEG_CLK_CNTL		0x174 /* 0x5d offset in data sheet */
+#define HHI_MPLL_CNTL			0x280 /* 0xa0 offset in data sheet */
+#define HHI_SYS_PLL_CNTL		0x300 /* 0xc0 offset in data sheet */
+#define HHI_VID_PLL_CNTL		0x320 /* 0xc8 offset in data sheet */
+
+/*
+ * CLKID index values
+ *
+ * These indices are entirely contrived and do not map onto the hardware.
+ * Migrate them out of this header and into the DT header file when they need
+ * to be exposed to client nodes in DT: include/dt-bindings/clock/meson8b-clkc.h
+ */
+
+/* CLKID_UNUSED */
+/* CLKID_XTAL */
+/* CLKID_PLL_FIXED */
+/* CLKID_PLL_VID */
+/* CLKID_PLL_SYS */
+/* CLKID_FCLK_DIV2 */
+/* CLKID_FCLK_DIV3 */
+/* CLKID_FCLK_DIV4 */
+/* CLKID_FCLK_DIV5 */
+/* CLKID_FCLK_DIV7 */
+/* CLKID_CLK81 */
+/* CLKID_MALI */
+/* CLKID_CPUCLK */
+/* CLKID_ZERO */
+/* CLKID_MPEG_SEL */
+/* CLKID_MPEG_DIV */
+#define CLKID_DDR		16
+#define CLKID_DOS		17
+#define CLKID_ISA		18
+#define CLKID_PL301		19
+#define CLKID_PERIPHS		20
+#define CLKID_SPICC		21
+#define CLKID_I2C		22
+#define CLKID_SAR_ADC		23
+#define CLKID_SMART_CARD	24
+#define CLKID_RNG0		25
+#define CLKID_UART0		26
+#define CLKID_SDHC		27
+#define CLKID_STREAM		28
+#define CLKID_ASYNC_FIFO	29
+#define CLKID_SDIO		30
+#define CLKID_ABUF		31
+#define CLKID_HIU_IFACE		32
+#define CLKID_ASSIST_MISC	33
+#define CLKID_SPI		34
+#define CLKID_I2S_SPDIF		35
+#define CLKID_ETH		36
+#define CLKID_DEMUX		37
+#define CLKID_AIU_GLUE		38
+#define CLKID_IEC958		39
+#define CLKID_I2S_OUT		40
+#define CLKID_AMCLK		41
+#define CLKID_AIFIFO2		42
+#define CLKID_MIXER		43
+#define CLKID_MIXER_IFACE	44
+#define CLKID_ADC		45
+#define CLKID_BLKMV		46
+#define CLKID_AIU		47
+#define CLKID_UART1		48
+#define CLKID_G2D		49
+#define CLKID_USB0		50
+#define CLKID_USB1		51
+#define CLKID_RESET		52
+#define CLKID_NAND		53
+#define CLKID_DOS_PARSER	54
+#define CLKID_USB		55
+#define CLKID_VDIN1		56
+#define CLKID_AHB_ARB0		57
+#define CLKID_EFUSE		58
+#define CLKID_BOOT_ROM		59
+#define CLKID_AHB_DATA_BUS	60
+#define CLKID_AHB_CTRL_BUS	61
+#define CLKID_HDMI_INTR_SYNC	62
+#define CLKID_HDMI_PCLK		63
+#define CLKID_USB1_DDR_BRIDGE	64
+#define CLKID_USB0_DDR_BRIDGE	65
+#define CLKID_MMC_PCLK		66
+#define CLKID_DVIN		67
+#define CLKID_UART2		68
+#define CLKID_SANA		69
+#define CLKID_VPU_INTR		70
+#define CLKID_SEC_AHB_AHB3_BRIDGE	71
+#define CLKID_CLK81_A9		72
+#define CLKID_VCLK2_VENCI0	73
+#define CLKID_VCLK2_VENCI1	74
+#define CLKID_VCLK2_VENCP0	75
+#define CLKID_VCLK2_VENCP1	76
+#define CLKID_GCLK_VENCI_INT	77
+#define CLKID_GCLK_VENCP_INT	78
+#define CLKID_DAC_CLK		79
+#define CLKID_AOCLK_GATE	80
+#define CLKID_IEC958_GATE	81
+#define CLKID_ENC480P		82
+#define CLKID_RNG1		83
+#define CLKID_GCLK_VENCL_INT	84
+#define CLKID_VCLK2_VENCLMCC	85
+#define CLKID_VCLK2_VENCL	86
+#define CLKID_VCLK2_OTHER	87
+#define CLKID_EDP		88
+#define CLKID_AO_MEDIA_CPU	89
+#define CLKID_AO_AHB_SRAM	90
+#define CLKID_AO_AHB_BUS	91
+#define CLKID_AO_IFACE		92
+
+#define CLK_NR_CLKS		93
+
+/* include the CLKIDs that have been made part of the stable DT binding */
+#include <dt-bindings/clock/meson8b-clkc.h>
+
+#endif /* __MESON8B_H */
diff --git a/drivers/clk/microchip/clk-core.c b/drivers/clk/microchip/clk-core.c
index ca85cea17839..c3b301463425 100644
--- a/drivers/clk/microchip/clk-core.c
+++ b/drivers/clk/microchip/clk-core.c
@@ -199,9 +199,9 @@ static int pbclk_set_rate(struct clk_hw *hw, unsigned long rate,
 
 	spin_unlock_irqrestore(&pb->core->reg_lock, flags);
 
-	/* wait again, for pbdivready */
-	err = readl_poll_timeout_atomic(pb->ctrl_reg, v, v & PB_DIV_READY,
-					1, LOCK_TIMEOUT_US);
+	/* wait again for DIV_READY */
+	err = readl_poll_timeout(pb->ctrl_reg, v, v & PB_DIV_READY,
+				 1, LOCK_TIMEOUT_US);
 	if (err)
 		return err;
 
diff --git a/drivers/clk/microchip/clk-pic32mzda.c b/drivers/clk/microchip/clk-pic32mzda.c
index 51f54380474b..9f734779be92 100644
--- a/drivers/clk/microchip/clk-pic32mzda.c
+++ b/drivers/clk/microchip/clk-pic32mzda.c
@@ -118,6 +118,7 @@ static const struct pic32_sec_osc_data sosc_clk = {
 	.status_reg = 0x1d0,
 	.enable_mask = BIT(1),
 	.status_mask = BIT(4),
+	.fixed_rate = 32768,
 	.init_data = {
 		.name = "sosc_clk",
 		.parent_names = NULL,
diff --git a/drivers/clk/mmp/clk-mmp2.c b/drivers/clk/mmp/clk-mmp2.c
index 383f6a4f64f0..038023483b98 100644
--- a/drivers/clk/mmp/clk-mmp2.c
+++ b/drivers/clk/mmp/clk-mmp2.c
@@ -16,6 +16,7 @@
 #include <linux/io.h>
 #include <linux/delay.h>
 #include <linux/err.h>
+#include <linux/clk/mmp.h>
 
 #include "clk.h"
 
diff --git a/drivers/clk/mvebu/Kconfig b/drivers/clk/mvebu/Kconfig
index 3165da77d525..fddc8ac5faff 100644
--- a/drivers/clk/mvebu/Kconfig
+++ b/drivers/clk/mvebu/Kconfig
@@ -24,6 +24,9 @@ config ARMADA_39X_CLK
 	bool
 	select MVEBU_CLK_COMMON
 
+config ARMADA_37XX_CLK
+       bool
+
 config ARMADA_XP_CLK
 	bool
 	select MVEBU_CLK_COMMON
diff --git a/drivers/clk/mvebu/Makefile b/drivers/clk/mvebu/Makefile
index 7172ef65693d..d9ae97fb43c4 100644
--- a/drivers/clk/mvebu/Makefile
+++ b/drivers/clk/mvebu/Makefile
@@ -6,6 +6,9 @@ obj-$(CONFIG_ARMADA_370_CLK)	+= armada-370.o
 obj-$(CONFIG_ARMADA_375_CLK)	+= armada-375.o
 obj-$(CONFIG_ARMADA_38X_CLK)	+= armada-38x.o
 obj-$(CONFIG_ARMADA_39X_CLK)	+= armada-39x.o
+obj-$(CONFIG_ARMADA_37XX_CLK)	+= armada-37xx-xtal.o
+obj-$(CONFIG_ARMADA_37XX_CLK)	+= armada-37xx-tbg.o
+obj-$(CONFIG_ARMADA_37XX_CLK)	+= armada-37xx-periph.o
 obj-$(CONFIG_ARMADA_XP_CLK)	+= armada-xp.o
 obj-$(CONFIG_ARMADA_AP806_SYSCON) += ap806-system-controller.o
 obj-$(CONFIG_ARMADA_CP110_SYSCON) += cp110-system-controller.o
diff --git a/drivers/clk/mvebu/armada-37xx-periph.c b/drivers/clk/mvebu/armada-37xx-periph.c
new file mode 100644
index 000000000000..45905fc0d75b
--- /dev/null
+++ b/drivers/clk/mvebu/armada-37xx-periph.c
@@ -0,0 +1,447 @@
+/*
+ * Marvell Armada 37xx SoC Peripheral clocks
+ *
+ * Copyright (C) 2016 Marvell
+ *
+ * Gregory CLEMENT <gregory.clement@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2 or later. This program is licensed "as is"
+ * without any warranty of any kind, whether express or implied.
+ *
+ * Most of the peripheral clocks can be modelled like this:
+ *             _____    _______    _______
+ * TBG-A-P  --|     |  |       |  |       |   ______
+ * TBG-B-P  --| Mux |--| /div1 |--| /div2 |--| Gate |--> perip_clk
+ * TBG-A-S  --|     |  |       |  |       |  |______|
+ * TBG-B-S  --|_____|  |_______|  |_______|
+ *
+ * However some clocks may use only one or two block or and use the
+ * xtal clock as parent.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define TBG_SEL		0x0
+#define DIV_SEL0	0x4
+#define DIV_SEL1	0x8
+#define DIV_SEL2	0xC
+#define CLK_SEL		0x10
+#define CLK_DIS		0x14
+
+struct clk_periph_driver_data {
+	struct clk_hw_onecell_data *hw_data;
+	spinlock_t lock;
+};
+
+struct clk_double_div {
+	struct clk_hw hw;
+	void __iomem *reg1;
+	u8 shift1;
+	void __iomem *reg2;
+	u8 shift2;
+};
+
+#define to_clk_double_div(_hw) container_of(_hw, struct clk_double_div, hw)
+
+struct clk_periph_data {
+	const char *name;
+	const char * const *parent_names;
+	int num_parents;
+	struct clk_hw *mux_hw;
+	struct clk_hw *rate_hw;
+	struct clk_hw *gate_hw;
+	bool is_double_div;
+};
+
+static const struct clk_div_table clk_table6[] = {
+	{ .val = 1, .div = 1, },
+	{ .val = 2, .div = 2, },
+	{ .val = 3, .div = 3, },
+	{ .val = 4, .div = 4, },
+	{ .val = 5, .div = 5, },
+	{ .val = 6, .div = 6, },
+	{ .val = 0, .div = 0, }, /* last entry */
+};
+
+static const struct clk_div_table clk_table1[] = {
+	{ .val = 0, .div = 1, },
+	{ .val = 1, .div = 2, },
+	{ .val = 0, .div = 0, }, /* last entry */
+};
+
+static const struct clk_div_table clk_table2[] = {
+	{ .val = 0, .div = 2, },
+	{ .val = 1, .div = 4, },
+	{ .val = 0, .div = 0, }, /* last entry */
+};
+static const struct clk_ops clk_double_div_ops;
+
+#define PERIPH_GATE(_name, _bit)		\
+struct clk_gate gate_##_name = {		\
+	.reg = (void *)CLK_DIS,			\
+	.bit_idx = _bit,			\
+	.hw.init = &(struct clk_init_data){	\
+		.ops =  &clk_gate_ops,		\
+	}					\
+};
+
+#define PERIPH_MUX(_name, _shift)		\
+struct clk_mux mux_##_name = {			\
+	.reg = (void *)TBG_SEL,			\
+	.shift = _shift,			\
+	.mask = 3,				\
+	.hw.init = &(struct clk_init_data){	\
+		.ops =  &clk_mux_ro_ops,	\
+	}					\
+};
+
+#define PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2)	\
+struct clk_double_div rate_##_name = {		\
+	.reg1 = (void *)_reg1,			\
+	.reg2 = (void *)_reg2,			\
+	.shift1 = _shift1,			\
+	.shift2 = _shift2,			\
+	.hw.init = &(struct clk_init_data){	\
+		.ops =  &clk_double_div_ops,	\
+	}					\
+};
+
+#define PERIPH_DIV(_name, _reg, _shift, _table)	\
+struct clk_divider rate_##_name = {		\
+	.reg = (void *)_reg,			\
+	.table = _table,			\
+	.shift = _shift,			\
+	.hw.init = &(struct clk_init_data){	\
+		.ops =  &clk_divider_ro_ops,	\
+	}					\
+};
+
+#define PERIPH_CLK_FULL_DD(_name, _bit, _shift, _reg1, _reg2, _shift1, _shift2)\
+static PERIPH_GATE(_name, _bit);			    \
+static PERIPH_MUX(_name, _shift);			    \
+static PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2);
+
+#define PERIPH_CLK_FULL(_name, _bit, _shift, _reg, _shift1, _table)	\
+static PERIPH_GATE(_name, _bit);			    \
+static PERIPH_MUX(_name, _shift);			    \
+static PERIPH_DIV(_name, _reg, _shift1, _table);
+
+#define PERIPH_CLK_GATE_DIV(_name, _bit,  _reg, _shift, _table)	\
+static PERIPH_GATE(_name, _bit);			\
+static PERIPH_DIV(_name, _reg, _shift, _table);
+
+#define PERIPH_CLK_MUX_DIV(_name, _shift,  _reg, _shift_div, _table)	\
+static PERIPH_MUX(_name, _shift);			    \
+static PERIPH_DIV(_name, _reg, _shift_div, _table);
+
+#define PERIPH_CLK_MUX_DD(_name, _shift, _reg1, _reg2, _shift1, _shift2)\
+static PERIPH_MUX(_name, _shift);			    \
+static PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2);
+
+#define REF_CLK_FULL(_name)				\
+	{ .name = #_name,				\
+	  .parent_names = (const char *[]){ "TBG-A-P",	\
+	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
+	  .num_parents = 4,				\
+	  .mux_hw = &mux_##_name.hw,			\
+	  .gate_hw = &gate_##_name.hw,			\
+	  .rate_hw = &rate_##_name.hw,			\
+	}
+
+#define REF_CLK_FULL_DD(_name)				\
+	{ .name = #_name,				\
+	  .parent_names = (const char *[]){ "TBG-A-P",	\
+	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
+	  .num_parents = 4,				\
+	  .mux_hw = &mux_##_name.hw,			\
+	  .gate_hw = &gate_##_name.hw,			\
+	  .rate_hw = &rate_##_name.hw,			\
+	  .is_double_div = true,			\
+	}
+
+#define REF_CLK_GATE(_name, _parent_name)			\
+	{ .name = #_name,					\
+	  .parent_names = (const char *[]){ _parent_name},	\
+	  .num_parents = 1,					\
+	  .gate_hw = &gate_##_name.hw,				\
+	}
+
+#define REF_CLK_GATE_DIV(_name, _parent_name)			\
+	{ .name = #_name,					\
+	  .parent_names = (const char *[]){ _parent_name},	\
+	  .num_parents = 1,					\
+	  .gate_hw = &gate_##_name.hw,				\
+	  .rate_hw = &rate_##_name.hw,				\
+	}
+
+#define REF_CLK_MUX_DIV(_name)				\
+	{ .name = #_name,				\
+	  .parent_names = (const char *[]){ "TBG-A-P",	\
+	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
+	  .num_parents = 4,				\
+	  .mux_hw = &mux_##_name.hw,			\
+	  .rate_hw = &rate_##_name.hw,			\
+	}
+
+#define REF_CLK_MUX_DD(_name)				\
+	{ .name = #_name,				\
+	  .parent_names = (const char *[]){ "TBG-A-P",	\
+	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
+	  .num_parents = 4,				\
+	  .mux_hw = &mux_##_name.hw,			\
+	  .rate_hw = &rate_##_name.hw,			\
+	  .is_double_div = true,			\
+	}
+
+/* NB periph clocks */
+PERIPH_CLK_FULL_DD(mmc, 2, 0, DIV_SEL2, DIV_SEL2, 16, 13);
+PERIPH_CLK_FULL_DD(sata_host, 3, 2, DIV_SEL2, DIV_SEL2, 10, 7);
+PERIPH_CLK_FULL_DD(sec_at, 6, 4, DIV_SEL1, DIV_SEL1, 3, 0);
+PERIPH_CLK_FULL_DD(sec_dap, 7, 6, DIV_SEL1, DIV_SEL1, 9, 6);
+PERIPH_CLK_FULL_DD(tscem, 8, 8, DIV_SEL1, DIV_SEL1, 15, 12);
+PERIPH_CLK_FULL(tscem_tmx, 10, 10, DIV_SEL1, 18, clk_table6);
+static PERIPH_GATE(avs, 11);
+PERIPH_CLK_FULL_DD(pwm, 13, 14, DIV_SEL0, DIV_SEL0, 3, 0);
+PERIPH_CLK_FULL_DD(sqf, 12, 12, DIV_SEL1, DIV_SEL1, 27, 24);
+static PERIPH_GATE(i2c_2, 16);
+static PERIPH_GATE(i2c_1, 17);
+PERIPH_CLK_GATE_DIV(ddr_phy, 19, DIV_SEL0, 18, clk_table2);
+PERIPH_CLK_FULL_DD(ddr_fclk, 21, 16, DIV_SEL0, DIV_SEL0, 15, 12);
+PERIPH_CLK_FULL(trace, 22, 18, DIV_SEL0, 20, clk_table6);
+PERIPH_CLK_FULL(counter, 23, 20, DIV_SEL0, 23, clk_table6);
+PERIPH_CLK_FULL_DD(eip97, 24, 24, DIV_SEL2, DIV_SEL2, 22, 19);
+PERIPH_CLK_MUX_DIV(cpu, 22, DIV_SEL0, 28, clk_table6);
+
+static struct clk_periph_data data_nb[] ={
+	REF_CLK_FULL_DD(mmc),
+	REF_CLK_FULL_DD(sata_host),
+	REF_CLK_FULL_DD(sec_at),
+	REF_CLK_FULL_DD(sec_dap),
+	REF_CLK_FULL_DD(tscem),
+	REF_CLK_FULL(tscem_tmx),
+	REF_CLK_GATE(avs, "xtal"),
+	REF_CLK_FULL_DD(sqf),
+	REF_CLK_FULL_DD(pwm),
+	REF_CLK_GATE(i2c_2, "xtal"),
+	REF_CLK_GATE(i2c_1, "xtal"),
+	REF_CLK_GATE_DIV(ddr_phy, "TBG-A-S"),
+	REF_CLK_FULL_DD(ddr_fclk),
+	REF_CLK_FULL(trace),
+	REF_CLK_FULL(counter),
+	REF_CLK_FULL_DD(eip97),
+	REF_CLK_MUX_DIV(cpu),
+	{ },
+};
+
+/* SB periph clocks */
+PERIPH_CLK_MUX_DD(gbe_50, 6, DIV_SEL2, DIV_SEL2, 6, 9);
+PERIPH_CLK_MUX_DD(gbe_core, 8, DIV_SEL1, DIV_SEL1, 18, 21);
+PERIPH_CLK_MUX_DD(gbe_125, 10, DIV_SEL1, DIV_SEL1, 6, 9);
+static PERIPH_GATE(gbe1_50, 0);
+static PERIPH_GATE(gbe0_50, 1);
+static PERIPH_GATE(gbe1_125, 2);
+static PERIPH_GATE(gbe0_125, 3);
+PERIPH_CLK_GATE_DIV(gbe1_core, 4, DIV_SEL1, 13, clk_table1);
+PERIPH_CLK_GATE_DIV(gbe0_core, 5, DIV_SEL1, 14, clk_table1);
+PERIPH_CLK_GATE_DIV(gbe_bm, 12, DIV_SEL1, 0, clk_table1);
+PERIPH_CLK_FULL_DD(sdio, 11, 14, DIV_SEL0, DIV_SEL0, 3, 6);
+PERIPH_CLK_FULL_DD(usb32_usb2_sys, 16, 16, DIV_SEL0, DIV_SEL0, 9, 12);
+PERIPH_CLK_FULL_DD(usb32_ss_sys, 17, 18, DIV_SEL0, DIV_SEL0, 15, 18);
+
+static struct clk_periph_data data_sb[] = {
+	REF_CLK_MUX_DD(gbe_50),
+	REF_CLK_MUX_DD(gbe_core),
+	REF_CLK_MUX_DD(gbe_125),
+	REF_CLK_GATE(gbe1_50, "gbe_50"),
+	REF_CLK_GATE(gbe0_50, "gbe_50"),
+	REF_CLK_GATE(gbe1_125, "gbe_125"),
+	REF_CLK_GATE(gbe0_125, "gbe_125"),
+	REF_CLK_GATE_DIV(gbe1_core, "gbe_core"),
+	REF_CLK_GATE_DIV(gbe0_core, "gbe_core"),
+	REF_CLK_GATE_DIV(gbe_bm, "gbe_core"),
+	REF_CLK_FULL_DD(sdio),
+	REF_CLK_FULL_DD(usb32_usb2_sys),
+	REF_CLK_FULL_DD(usb32_ss_sys),
+	{ },
+};
+
+static unsigned int get_div(void __iomem *reg, int shift)
+{
+	u32 val;
+
+	val = (readl(reg) >> shift) & 0x7;
+	if (val > 6)
+		return 0;
+	return val;
+}
+
+static unsigned long clk_double_div_recalc_rate(struct clk_hw *hw,
+		unsigned long parent_rate)
+{
+	struct clk_double_div *double_div = to_clk_double_div(hw);
+	unsigned int div;
+
+	div = get_div(double_div->reg1, double_div->shift1);
+	div *= get_div(double_div->reg2, double_div->shift2);
+
+	return DIV_ROUND_UP_ULL((u64)parent_rate, div);
+}
+
+static const struct clk_ops clk_double_div_ops = {
+	.recalc_rate = clk_double_div_recalc_rate,
+};
+
+static const struct of_device_id armada_3700_periph_clock_of_match[] = {
+	{ .compatible = "marvell,armada-3700-periph-clock-nb",
+	  .data = data_nb, },
+	{ .compatible = "marvell,armada-3700-periph-clock-sb",
+	.data = data_sb, },
+	{ }
+};
+static int armada_3700_add_composite_clk(const struct clk_periph_data *data,
+					 void __iomem *reg, spinlock_t *lock,
+					 struct device *dev, struct clk_hw *hw)
+{
+	const struct clk_ops *mux_ops = NULL, *gate_ops = NULL,
+		*rate_ops = NULL;
+	struct clk_hw *mux_hw = NULL, *gate_hw = NULL, *rate_hw = NULL;
+
+	if (data->mux_hw) {
+		struct clk_mux *mux;
+
+		mux_hw = data->mux_hw;
+		mux = to_clk_mux(mux_hw);
+		mux->lock = lock;
+		mux_ops = mux_hw->init->ops;
+		mux->reg = reg + (u64)mux->reg;
+	}
+
+	if (data->gate_hw) {
+		struct clk_gate *gate;
+
+		gate_hw = data->gate_hw;
+		gate = to_clk_gate(gate_hw);
+		gate->lock = lock;
+		gate_ops = gate_hw->init->ops;
+		gate->reg = reg + (u64)gate->reg;
+	}
+
+	if (data->rate_hw) {
+		rate_hw = data->rate_hw;
+		rate_ops = rate_hw->init->ops;
+		if (data->is_double_div) {
+			struct clk_double_div *rate;
+
+			rate =  to_clk_double_div(rate_hw);
+			rate->reg1 = reg + (u64)rate->reg1;
+			rate->reg2 = reg + (u64)rate->reg2;
+		} else {
+			struct clk_divider *rate = to_clk_divider(rate_hw);
+			const struct clk_div_table *clkt;
+			int table_size = 0;
+
+			rate->reg = reg + (u64)rate->reg;
+			for (clkt = rate->table; clkt->div; clkt++)
+				table_size++;
+			rate->width = order_base_2(table_size);
+			rate->lock = lock;
+		}
+	}
+
+	hw = clk_hw_register_composite(dev, data->name, data->parent_names,
+				       data->num_parents, mux_hw,
+				       mux_ops, rate_hw, rate_ops,
+				       gate_hw, gate_ops, CLK_IGNORE_UNUSED);
+
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+
+	return 0;
+}
+
+static int armada_3700_periph_clock_probe(struct platform_device *pdev)
+{
+	struct clk_periph_driver_data *driver_data;
+	struct device_node *np = pdev->dev.of_node;
+	const struct clk_periph_data *data;
+	struct device *dev = &pdev->dev;
+	int num_periph = 0, i, ret;
+	struct resource *res;
+	void __iomem *reg;
+
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return -ENODEV;
+
+	while (data[num_periph].name)
+		num_periph++;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	reg = devm_ioremap_resource(dev, res);
+	if (IS_ERR(reg))
+		return PTR_ERR(reg);
+
+	driver_data = devm_kzalloc(dev, sizeof(*driver_data), GFP_KERNEL);
+	if (!driver_data)
+		return -ENOMEM;
+
+	driver_data->hw_data = devm_kzalloc(dev, sizeof(*driver_data->hw_data) +
+			    sizeof(*driver_data->hw_data->hws) * num_periph,
+			    GFP_KERNEL);
+	if (!driver_data->hw_data)
+		return -ENOMEM;
+	driver_data->hw_data->num = num_periph;
+
+	spin_lock_init(&driver_data->lock);
+
+	for (i = 0; i < num_periph; i++) {
+		struct clk_hw *hw = driver_data->hw_data->hws[i];
+
+		if (armada_3700_add_composite_clk(&data[i], reg,
+						  &driver_data->lock, dev, hw))
+			dev_err(dev, "Can't register periph clock %s\n",
+			       data[i].name);
+
+	}
+
+	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get,
+				  driver_data->hw_data);
+	if (ret) {
+		for (i = 0; i < num_periph; i++)
+			clk_hw_unregister(driver_data->hw_data->hws[i]);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, driver_data);
+	return 0;
+}
+
+static int armada_3700_periph_clock_remove(struct platform_device *pdev)
+{
+	struct clk_periph_driver_data *data = platform_get_drvdata(pdev);
+	struct clk_hw_onecell_data *hw_data = data->hw_data;
+	int i;
+
+	of_clk_del_provider(pdev->dev.of_node);
+
+	for (i = 0; i < hw_data->num; i++)
+		clk_hw_unregister(hw_data->hws[i]);
+
+	return 0;
+}
+
+static struct platform_driver armada_3700_periph_clock_driver = {
+	.probe = armada_3700_periph_clock_probe,
+	.remove = armada_3700_periph_clock_remove,
+	.driver		= {
+		.name	= "marvell-armada-3700-periph-clock",
+		.of_match_table = armada_3700_periph_clock_of_match,
+	},
+};
+
+builtin_platform_driver(armada_3700_periph_clock_driver);
diff --git a/drivers/clk/mvebu/armada-37xx-tbg.c b/drivers/clk/mvebu/armada-37xx-tbg.c
new file mode 100644
index 000000000000..aa80db11f543
--- /dev/null
+++ b/drivers/clk/mvebu/armada-37xx-tbg.c
@@ -0,0 +1,158 @@
+/*
+ * Marvell Armada 37xx SoC Time Base Generator clocks
+ *
+ * Copyright (C) 2016 Marvell
+ *
+ * Gregory CLEMENT <gregory.clement@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2 or later. This program is licensed "as is"
+ * without any warranty of any kind, whether express or implied.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define NUM_TBG	    4
+
+#define TBG_CTRL0		0x4
+#define TBG_CTRL1		0x8
+#define TBG_CTRL7		0x20
+#define TBG_CTRL8		0x30
+
+#define TBG_DIV_MASK		0x1FF
+
+#define TBG_A_REFDIV		0
+#define TBG_B_REFDIV		16
+
+#define TBG_A_FBDIV		2
+#define TBG_B_FBDIV		18
+
+#define TBG_A_VCODIV_SE		0
+#define TBG_B_VCODIV_SE		16
+
+#define TBG_A_VCODIV_DIFF	1
+#define TBG_B_VCODIV_DIFF	17
+
+struct tbg_def {
+	char *name;
+	u32 refdiv_offset;
+	u32 fbdiv_offset;
+	u32 vcodiv_reg;
+	u32 vcodiv_offset;
+};
+
+static const struct tbg_def tbg[NUM_TBG] = {
+	{"TBG-A-P", TBG_A_REFDIV, TBG_A_FBDIV, TBG_CTRL8, TBG_A_VCODIV_DIFF},
+	{"TBG-B-P", TBG_B_REFDIV, TBG_B_FBDIV, TBG_CTRL8, TBG_B_VCODIV_DIFF},
+	{"TBG-A-S", TBG_A_REFDIV, TBG_A_FBDIV, TBG_CTRL1, TBG_A_VCODIV_SE},
+	{"TBG-B-S", TBG_B_REFDIV, TBG_B_FBDIV, TBG_CTRL1, TBG_B_VCODIV_SE},
+};
+
+static unsigned int tbg_get_mult(void __iomem *reg, const struct tbg_def *ptbg)
+{
+	u32 val;
+
+	val = readl(reg + TBG_CTRL0);
+
+	return ((val >> ptbg->fbdiv_offset) & TBG_DIV_MASK) << 2;
+}
+
+static unsigned int tbg_get_div(void __iomem *reg, const struct tbg_def *ptbg)
+{
+	u32 val;
+	unsigned int div;
+
+	val = readl(reg + TBG_CTRL7);
+
+	div = (val >> ptbg->refdiv_offset) & TBG_DIV_MASK;
+	if (div == 0)
+		div = 1;
+	val = readl(reg + ptbg->vcodiv_reg);
+
+	div *= 1 << ((val >>  ptbg->vcodiv_offset) & TBG_DIV_MASK);
+
+	return div;
+}
+
+
+static int armada_3700_tbg_clock_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct clk_hw_onecell_data *hw_tbg_data;
+	struct device *dev = &pdev->dev;
+	const char *parent_name;
+	struct resource *res;
+	struct clk *parent;
+	void __iomem *reg;
+	int i, ret;
+
+	hw_tbg_data = devm_kzalloc(&pdev->dev, sizeof(*hw_tbg_data)
+				   + sizeof(*hw_tbg_data->hws) * NUM_TBG,
+				   GFP_KERNEL);
+	if (!hw_tbg_data)
+		return -ENOMEM;
+	hw_tbg_data->num = NUM_TBG;
+	platform_set_drvdata(pdev, hw_tbg_data);
+
+	parent = devm_clk_get(dev, NULL);
+	if (IS_ERR(parent)) {
+		dev_err(dev, "Could get the clock parent\n");
+		return -EINVAL;
+	}
+	parent_name = __clk_get_name(parent);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	reg = devm_ioremap_resource(dev, res);
+	if (IS_ERR(reg))
+		return PTR_ERR(reg);
+
+	for (i = 0; i < NUM_TBG; i++) {
+		const char *name;
+		unsigned int mult, div;
+
+		name = tbg[i].name;
+		mult = tbg_get_mult(reg, &tbg[i]);
+		div = tbg_get_div(reg, &tbg[i]);
+		hw_tbg_data->hws[i] = clk_hw_register_fixed_factor(NULL, name,
+						parent_name, 0, mult, div);
+		if (IS_ERR(hw_tbg_data->hws[i]))
+			dev_err(dev, "Can't register TBG clock %s\n", name);
+	}
+
+	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, hw_tbg_data);
+
+	return ret;
+}
+
+static int armada_3700_tbg_clock_remove(struct platform_device *pdev)
+{
+	int i;
+	struct clk_hw_onecell_data *hw_tbg_data = platform_get_drvdata(pdev);
+
+	of_clk_del_provider(pdev->dev.of_node);
+	for (i = 0; i < hw_tbg_data->num; i++)
+		clk_hw_unregister_fixed_factor(hw_tbg_data->hws[i]);
+
+	return 0;
+}
+
+static const struct of_device_id armada_3700_tbg_clock_of_match[] = {
+	{ .compatible = "marvell,armada-3700-tbg-clock", },
+	{ }
+};
+
+static struct platform_driver armada_3700_tbg_clock_driver = {
+	.probe = armada_3700_tbg_clock_probe,
+	.remove = armada_3700_tbg_clock_remove,
+	.driver		= {
+		.name	= "marvell-armada-3700-tbg-clock",
+		.of_match_table = armada_3700_tbg_clock_of_match,
+	},
+};
+
+builtin_platform_driver(armada_3700_tbg_clock_driver);
diff --git a/drivers/clk/mvebu/armada-37xx-xtal.c b/drivers/clk/mvebu/armada-37xx-xtal.c
new file mode 100644
index 000000000000..612d65ede10a
--- /dev/null
+++ b/drivers/clk/mvebu/armada-37xx-xtal.c
@@ -0,0 +1,91 @@
+/*
+ * Marvell Armada 37xx SoC xtal clocks
+ *
+ * Copyright (C) 2016 Marvell
+ *
+ * Gregory CLEMENT <gregory.clement@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2.  This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#define NB_GPIO1_LATCH	0xC
+#define XTAL_MODE	    BIT(31)
+
+static int armada_3700_xtal_clock_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	const char *xtal_name = "xtal";
+	struct device_node *parent;
+	struct regmap *regmap;
+	struct clk_hw *xtal_hw;
+	unsigned int rate;
+	u32 reg;
+	int ret;
+
+	xtal_hw = devm_kzalloc(&pdev->dev, sizeof(*xtal_hw), GFP_KERNEL);
+	if (!xtal_hw)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, xtal_hw);
+
+	parent = np->parent;
+	if (!parent) {
+		dev_err(&pdev->dev, "no parent\n");
+		return -ENODEV;
+	}
+
+	regmap = syscon_node_to_regmap(parent);
+	if (IS_ERR(regmap)) {
+		dev_err(&pdev->dev, "cannot get regmap\n");
+		return PTR_ERR(regmap);
+	}
+
+	ret = regmap_read(regmap, NB_GPIO1_LATCH, &reg);
+	if (ret) {
+		dev_err(&pdev->dev, "cannot read from regmap\n");
+		return ret;
+	}
+
+	if (reg & XTAL_MODE)
+		rate = 40000000;
+	else
+		rate = 25000000;
+
+	of_property_read_string_index(np, "clock-output-names", 0, &xtal_name);
+	xtal_hw = clk_hw_register_fixed_rate(NULL, xtal_name, NULL, 0, rate);
+	if (IS_ERR(xtal_hw))
+		return PTR_ERR(xtal_hw);
+	ret = of_clk_add_hw_provider(np, of_clk_hw_simple_get, xtal_hw);
+
+	return ret;
+}
+
+static int armada_3700_xtal_clock_remove(struct platform_device *pdev)
+{
+	of_clk_del_provider(pdev->dev.of_node);
+
+	return 0;
+}
+
+static const struct of_device_id armada_3700_xtal_clock_of_match[] = {
+	{ .compatible = "marvell,armada-3700-xtal-clock", },
+	{ }
+};
+
+static struct platform_driver armada_3700_xtal_clock_driver = {
+	.probe = armada_3700_xtal_clock_probe,
+	.remove = armada_3700_xtal_clock_remove,
+	.driver		= {
+		.name	= "marvell-armada-3700-xtal-clock",
+		.of_match_table = armada_3700_xtal_clock_of_match,
+	},
+};
+
+builtin_platform_driver(armada_3700_xtal_clock_driver);
diff --git a/drivers/clk/mvebu/armada-39x.c b/drivers/clk/mvebu/armada-39x.c
index efb974df9822..4fdfd32247a9 100644
--- a/drivers/clk/mvebu/armada-39x.c
+++ b/drivers/clk/mvebu/armada-39x.c
@@ -142,6 +142,8 @@ static const struct clk_gating_soc_desc armada_39x_gating_desc[] __initconst = {
 	{ "pex3", NULL, 7 },
 	{ "pex0", NULL, 8 },
 	{ "usb3h0", NULL, 9 },
+	{ "usb3h1", NULL, 10 },
+	{ "sata0", NULL, 15 },
 	{ "sdio", NULL, 17 },
 	{ "xor0", NULL, 22 },
 	{ "xor1", NULL, 28 },
diff --git a/drivers/clk/mvebu/cp110-system-controller.c b/drivers/clk/mvebu/cp110-system-controller.c
index 7fa42d6b2b92..f2303da7fda7 100644
--- a/drivers/clk/mvebu/cp110-system-controller.c
+++ b/drivers/clk/mvebu/cp110-system-controller.c
@@ -81,13 +81,6 @@ enum {
 #define CP110_GATE_EIP150		25
 #define CP110_GATE_EIP197		26
 
-static struct clk *cp110_clks[CP110_CLK_NUM];
-
-static struct clk_onecell_data cp110_clk_data = {
-	.clks = cp110_clks,
-	.clk_num = CP110_CLK_NUM,
-};
-
 struct cp110_gate_clk {
 	struct clk_hw hw;
 	struct regmap *regmap;
@@ -142,6 +135,8 @@ static struct clk *cp110_register_gate(const char *name,
 	if (!gate)
 		return ERR_PTR(-ENOMEM);
 
+	memset(&init, 0, sizeof(init));
+
 	init.name = name;
 	init.ops = &cp110_gate_ops;
 	init.parent_names = &parent_name;
@@ -194,7 +189,8 @@ static int cp110_syscon_clk_probe(struct platform_device *pdev)
 	struct regmap *regmap;
 	struct device_node *np = pdev->dev.of_node;
 	const char *ppv2_name, *apll_name, *core_name, *eip_name, *nand_name;
-	struct clk *clk;
+	struct clk_onecell_data *cp110_clk_data;
+	struct clk *clk, **cp110_clks;
 	u32 nand_clk_ctrl;
 	int i, ret;
 
@@ -207,6 +203,20 @@ static int cp110_syscon_clk_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	cp110_clks = devm_kcalloc(&pdev->dev, sizeof(struct clk *),
+				  CP110_CLK_NUM, GFP_KERNEL);
+	if (!cp110_clks)
+		return -ENOMEM;
+
+	cp110_clk_data = devm_kzalloc(&pdev->dev,
+				      sizeof(*cp110_clk_data),
+				      GFP_KERNEL);
+	if (!cp110_clk_data)
+		return -ENOMEM;
+
+	cp110_clk_data->clks = cp110_clks;
+	cp110_clk_data->clk_num = CP110_CLK_NUM;
+
 	/* Register the APLL which is the root of the clk tree */
 	of_property_read_string_index(np, "core-clock-output-names",
 				      CP110_CORE_APLL, &apll_name);
@@ -334,10 +344,12 @@ static int cp110_syscon_clk_probe(struct platform_device *pdev)
 		cp110_clks[CP110_MAX_CORE_CLOCKS + i] = clk;
 	}
 
-	ret = of_clk_add_provider(np, cp110_of_clk_get, &cp110_clk_data);
+	ret = of_clk_add_provider(np, cp110_of_clk_get, cp110_clk_data);
 	if (ret)
 		goto fail_clk_add;
 
+	platform_set_drvdata(pdev, cp110_clks);
+
 	return 0;
 
 fail_clk_add:
@@ -364,6 +376,7 @@ fail0:
 
 static int cp110_syscon_clk_remove(struct platform_device *pdev)
 {
+	struct clk **cp110_clks = platform_get_drvdata(pdev);
 	int i;
 
 	of_clk_del_provider(pdev->dev.of_node);
diff --git a/drivers/clk/mvebu/orion.c b/drivers/clk/mvebu/orion.c
index fd129566c1ce..a6e5bee23385 100644
--- a/drivers/clk/mvebu/orion.c
+++ b/drivers/clk/mvebu/orion.c
@@ -21,6 +21,76 @@ static const struct coreclk_ratio orion_coreclk_ratios[] __initconst = {
 };
 
 /*
+ * Orion 5181
+ */
+
+#define SAR_MV88F5181_TCLK_FREQ      8
+#define SAR_MV88F5181_TCLK_FREQ_MASK 0x3
+
+static u32 __init mv88f5181_get_tclk_freq(void __iomem *sar)
+{
+	u32 opt = (readl(sar) >> SAR_MV88F5181_TCLK_FREQ) &
+		SAR_MV88F5181_TCLK_FREQ_MASK;
+	if (opt == 0)
+		return 133333333;
+	else if (opt == 1)
+		return 150000000;
+	else if (opt == 2)
+		return 166666667;
+	else
+		return 0;
+}
+
+#define SAR_MV88F5181_CPU_FREQ       4
+#define SAR_MV88F5181_CPU_FREQ_MASK  0xf
+
+static u32 __init mv88f5181_get_cpu_freq(void __iomem *sar)
+{
+	u32 opt = (readl(sar) >> SAR_MV88F5181_CPU_FREQ) &
+		SAR_MV88F5181_CPU_FREQ_MASK;
+	if (opt == 0)
+		return 333333333;
+	else if (opt == 1 || opt == 2)
+		return 400000000;
+	else if (opt == 3)
+		return 500000000;
+	else
+		return 0;
+}
+
+static void __init mv88f5181_get_clk_ratio(void __iomem *sar, int id,
+					   int *mult, int *div)
+{
+	u32 opt = (readl(sar) >> SAR_MV88F5181_CPU_FREQ) &
+		SAR_MV88F5181_CPU_FREQ_MASK;
+	if (opt == 0 || opt == 1) {
+		*mult = 1;
+		*div  = 2;
+	} else if (opt == 2 || opt == 3) {
+		*mult = 1;
+		*div  = 3;
+	} else {
+		*mult = 0;
+		*div  = 1;
+	}
+}
+
+static const struct coreclk_soc_desc mv88f5181_coreclks = {
+	.get_tclk_freq = mv88f5181_get_tclk_freq,
+	.get_cpu_freq = mv88f5181_get_cpu_freq,
+	.get_clk_ratio = mv88f5181_get_clk_ratio,
+	.ratios = orion_coreclk_ratios,
+	.num_ratios = ARRAY_SIZE(orion_coreclk_ratios),
+};
+
+static void __init mv88f5181_clk_init(struct device_node *np)
+{
+	return mvebu_coreclk_setup(np, &mv88f5181_coreclks);
+}
+
+CLK_OF_DECLARE(mv88f5181_clk, "marvell,mv88f5181-core-clock", mv88f5181_clk_init);
+
+/*
  * Orion 5182
  */
 
diff --git a/drivers/clk/nxp/clk-lpc18xx-creg.c b/drivers/clk/nxp/clk-lpc18xx-creg.c
index 9e35749dafdf..c6e802e7e6ec 100644
--- a/drivers/clk/nxp/clk-lpc18xx-creg.c
+++ b/drivers/clk/nxp/clk-lpc18xx-creg.c
@@ -184,7 +184,8 @@ static void __init lpc18xx_creg_clk_init(struct device_node *np)
 
 	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_creg_early_data);
 }
-CLK_OF_DECLARE(lpc18xx_creg_clk, "nxp,lpc1850-creg-clk", lpc18xx_creg_clk_init);
+CLK_OF_DECLARE_DRIVER(lpc18xx_creg_clk, "nxp,lpc1850-creg-clk",
+		      lpc18xx_creg_clk_init);
 
 static struct clk *clk_creg[CREG_CLK_MAX];
 static struct clk_onecell_data clk_creg_data = {
diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c
index 90d740a2fc0d..34c97353cdeb 100644
--- a/drivers/clk/nxp/clk-lpc32xx.c
+++ b/drivers/clk/nxp/clk-lpc32xx.c
@@ -1513,6 +1513,7 @@ static void __init lpc32xx_clk_init(struct device_node *np)
 	if (IS_ERR(clk_regmap)) {
 		pr_err("failed to regmap system control block: %ld\n",
 			PTR_ERR(clk_regmap));
+		iounmap(base);
 		return;
 	}
 
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index 95e3b3e0fa1c..0146d3c2547f 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -87,6 +87,23 @@ config MSM_LCC_8960
 	  Say Y if you want to use audio devices such as i2s, pcm,
 	  SLIMBus, etc.
 
+config MDM_GCC_9615
+	tristate "MDM9615 Global Clock Controller"
+	depends on COMMON_CLK_QCOM
+	help
+	  Support for the global clock controller on mdm9615 devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  i2c, USB, SD/eMMC, etc.
+
+config MDM_LCC_9615
+	tristate "MDM9615 LPASS Clock Controller"
+	select MDM_GCC_9615
+	depends on COMMON_CLK_QCOM
+	help
+	  Support for the LPASS clock controller on mdm9615 devices.
+	  Say Y if you want to use audio devices such as i2s, pcm,
+	  SLIMBus, etc.
+
 config MSM_MMCC_8960
 	tristate "MSM8960 Multimedia Clock Controller"
 	select MSM_GCC_8960
@@ -117,6 +134,7 @@ config MSM_MMCC_8974
 
 config MSM_GCC_8996
 	tristate "MSM8996 Global Clock Controller"
+	select QCOM_GDSC
 	depends on COMMON_CLK_QCOM
 	help
 	  Support for the global clock controller on msm8996 devices.
@@ -126,6 +144,7 @@ config MSM_GCC_8996
 config MSM_MMCC_8996
 	tristate "MSM8996 Multimedia Clock Controller"
 	select MSM_GCC_8996
+	select QCOM_GDSC
 	depends on COMMON_CLK_QCOM
 	help
 	  Support for the multimedia clock controller on msm8996 devices.
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index 2a25f4e75f49..1fb1f5476cb0 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -12,17 +12,20 @@ clk-qcom-y += clk-regmap-mux.o
 clk-qcom-y += reset.o
 clk-qcom-$(CONFIG_QCOM_GDSC) += gdsc.o
 
+# Keep alphabetically sorted by config
 obj-$(CONFIG_APQ_GCC_8084) += gcc-apq8084.o
 obj-$(CONFIG_APQ_MMCC_8084) += mmcc-apq8084.o
 obj-$(CONFIG_IPQ_GCC_4019) += gcc-ipq4019.o
 obj-$(CONFIG_IPQ_GCC_806X) += gcc-ipq806x.o
 obj-$(CONFIG_IPQ_LCC_806X) += lcc-ipq806x.o
+obj-$(CONFIG_MDM_GCC_9615) += gcc-mdm9615.o
+obj-$(CONFIG_MDM_LCC_9615) += lcc-mdm9615.o
 obj-$(CONFIG_MSM_GCC_8660) += gcc-msm8660.o
 obj-$(CONFIG_MSM_GCC_8916) += gcc-msm8916.o
 obj-$(CONFIG_MSM_GCC_8960) += gcc-msm8960.o
-obj-$(CONFIG_MSM_LCC_8960) += lcc-msm8960.o
 obj-$(CONFIG_MSM_GCC_8974) += gcc-msm8974.o
 obj-$(CONFIG_MSM_GCC_8996) += gcc-msm8996.o
+obj-$(CONFIG_MSM_LCC_8960) += lcc-msm8960.o
 obj-$(CONFIG_MSM_MMCC_8960) += mmcc-msm8960.o
 obj-$(CONFIG_MSM_MMCC_8974) += mmcc-msm8974.o
 obj-$(CONFIG_MSM_MMCC_8996) += mmcc-msm8996.o
diff --git a/drivers/clk/qcom/clk-regmap.c b/drivers/clk/qcom/clk-regmap.c
index a58ba39a900c..1c856d330733 100644
--- a/drivers/clk/qcom/clk-regmap.c
+++ b/drivers/clk/qcom/clk-regmap.c
@@ -101,14 +101,13 @@ EXPORT_SYMBOL_GPL(clk_disable_regmap);
  * clk_regmap struct via this function so that the regmap is initialized
  * and so that the clock is registered with the common clock framework.
  */
-struct clk *devm_clk_register_regmap(struct device *dev,
-				     struct clk_regmap *rclk)
+int devm_clk_register_regmap(struct device *dev, struct clk_regmap *rclk)
 {
 	if (dev && dev_get_regmap(dev, NULL))
 		rclk->regmap = dev_get_regmap(dev, NULL);
 	else if (dev && dev->parent)
 		rclk->regmap = dev_get_regmap(dev->parent, NULL);
 
-	return devm_clk_register(dev, &rclk->hw);
+	return devm_clk_hw_register(dev, &rclk->hw);
 }
 EXPORT_SYMBOL_GPL(devm_clk_register_regmap);
diff --git a/drivers/clk/qcom/clk-regmap.h b/drivers/clk/qcom/clk-regmap.h
index 491a63d537df..90d95cd11ec6 100644
--- a/drivers/clk/qcom/clk-regmap.h
+++ b/drivers/clk/qcom/clk-regmap.h
@@ -39,7 +39,6 @@ struct clk_regmap {
 int clk_is_enabled_regmap(struct clk_hw *hw);
 int clk_enable_regmap(struct clk_hw *hw);
 void clk_disable_regmap(struct clk_hw *hw);
-struct clk *
-devm_clk_register_regmap(struct device *dev, struct clk_regmap *rclk);
+int devm_clk_register_regmap(struct device *dev, struct clk_regmap *rclk);
 
 #endif
diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c
index f7c226ab4307..fffcbaf0fba7 100644
--- a/drivers/clk/qcom/common.c
+++ b/drivers/clk/qcom/common.c
@@ -27,8 +27,8 @@
 
 struct qcom_cc {
 	struct qcom_reset_controller reset;
-	struct clk_onecell_data data;
-	struct clk *clks[];
+	struct clk_regmap **rclks;
+	size_t num_rclks;
 };
 
 const
@@ -102,8 +102,8 @@ static int _qcom_cc_register_board_clk(struct device *dev, const char *path,
 	struct device_node *clocks_node;
 	struct clk_fixed_factor *factor;
 	struct clk_fixed_rate *fixed;
-	struct clk *clk;
 	struct clk_init_data init_data = { };
+	int ret;
 
 	clocks_node = of_find_node_by_path("/clocks");
 	if (clocks_node)
@@ -121,9 +121,9 @@ static int _qcom_cc_register_board_clk(struct device *dev, const char *path,
 		init_data.name = path;
 		init_data.ops = &clk_fixed_rate_ops;
 
-		clk = devm_clk_register(dev, &fixed->hw);
-		if (IS_ERR(clk))
-			return PTR_ERR(clk);
+		ret = devm_clk_hw_register(dev, &fixed->hw);
+		if (ret)
+			return ret;
 	}
 	of_node_put(node);
 
@@ -141,9 +141,9 @@ static int _qcom_cc_register_board_clk(struct device *dev, const char *path,
 		init_data.flags = 0;
 		init_data.ops = &clk_fixed_factor_ops;
 
-		clk = devm_clk_register(dev, &factor->hw);
-		if (IS_ERR(clk))
-			return PTR_ERR(clk);
+		ret = devm_clk_hw_register(dev, &factor->hw);
+		if (ret)
+			return ret;
 	}
 
 	return 0;
@@ -174,42 +174,48 @@ int qcom_cc_register_sleep_clk(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(qcom_cc_register_sleep_clk);
 
+static struct clk_hw *qcom_cc_clk_hw_get(struct of_phandle_args *clkspec,
+					 void *data)
+{
+	struct qcom_cc *cc = data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= cc->num_rclks) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return cc->rclks[idx] ? &cc->rclks[idx]->hw : ERR_PTR(-ENOENT);
+}
+
 int qcom_cc_really_probe(struct platform_device *pdev,
 			 const struct qcom_cc_desc *desc, struct regmap *regmap)
 {
 	int i, ret;
 	struct device *dev = &pdev->dev;
-	struct clk *clk;
-	struct clk_onecell_data *data;
-	struct clk **clks;
 	struct qcom_reset_controller *reset;
 	struct qcom_cc *cc;
 	struct gdsc_desc *scd;
 	size_t num_clks = desc->num_clks;
 	struct clk_regmap **rclks = desc->clks;
 
-	cc = devm_kzalloc(dev, sizeof(*cc) + sizeof(*clks) * num_clks,
-			  GFP_KERNEL);
+	cc = devm_kzalloc(dev, sizeof(*cc), GFP_KERNEL);
 	if (!cc)
 		return -ENOMEM;
 
-	clks = cc->clks;
-	data = &cc->data;
-	data->clks = clks;
-	data->clk_num = num_clks;
+	cc->rclks = rclks;
+	cc->num_rclks = num_clks;
 
 	for (i = 0; i < num_clks; i++) {
-		if (!rclks[i]) {
-			clks[i] = ERR_PTR(-ENOENT);
+		if (!rclks[i])
 			continue;
-		}
-		clk = devm_clk_register_regmap(dev, rclks[i]);
-		if (IS_ERR(clk))
-			return PTR_ERR(clk);
-		clks[i] = clk;
+
+		ret = devm_clk_register_regmap(dev, rclks[i]);
+		if (ret)
+			return ret;
 	}
 
-	ret = of_clk_add_provider(dev->of_node, of_clk_src_onecell_get, data);
+	ret = of_clk_add_hw_provider(dev->of_node, qcom_cc_clk_hw_get, cc);
 	if (ret)
 		return ret;
 
diff --git a/drivers/clk/qcom/gcc-ipq4019.c b/drivers/clk/qcom/gcc-ipq4019.c
index 3cd1af0af0d9..b593065de8db 100644
--- a/drivers/clk/qcom/gcc-ipq4019.c
+++ b/drivers/clk/qcom/gcc-ipq4019.c
@@ -1332,7 +1332,6 @@ static struct platform_driver gcc_ipq4019_driver = {
 	.probe		= gcc_ipq4019_probe,
 	.driver		= {
 		.name	= "qcom,gcc-ipq4019",
-		.owner	= THIS_MODULE,
 		.of_match_table = gcc_ipq4019_match_table,
 	},
 };
diff --git a/drivers/clk/qcom/gcc-mdm9615.c b/drivers/clk/qcom/gcc-mdm9615.c
new file mode 100644
index 000000000000..581a17f67379
--- /dev/null
+++ b/drivers/clk/qcom/gcc-mdm9615.c
@@ -0,0 +1,1727 @@
+/*
+ * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved.
+ * Copyright (c) BayLibre, SAS.
+ * Author : Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+
+#include <dt-bindings/clock/qcom,gcc-mdm9615.h>
+#include <dt-bindings/reset/qcom,gcc-mdm9615.h>
+
+#include "common.h"
+#include "clk-regmap.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-branch.h"
+#include "reset.h"
+
+static struct clk_fixed_factor cxo = {
+	.mult = 1,
+	.div = 1,
+	.hw.init = &(struct clk_init_data){
+		.name = "cxo",
+		.parent_names = (const char *[]){ "cxo_board" },
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_pll pll0 = {
+	.l_reg = 0x30c4,
+	.m_reg = 0x30c8,
+	.n_reg = 0x30cc,
+	.config_reg = 0x30d4,
+	.mode_reg = 0x30c0,
+	.status_reg = 0x30d8,
+	.status_bit = 16,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pll0",
+		.parent_names = (const char *[]){ "cxo" },
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap pll0_vote = {
+	.enable_reg = 0x34c0,
+	.enable_mask = BIT(0),
+	.hw.init = &(struct clk_init_data){
+		.name = "pll0_vote",
+		.parent_names = (const char *[]){ "pll8" },
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static struct clk_regmap pll4_vote = {
+	.enable_reg = 0x34c0,
+	.enable_mask = BIT(4),
+	.hw.init = &(struct clk_init_data){
+		.name = "pll4_vote",
+		.parent_names = (const char *[]){ "pll4" },
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static struct clk_pll pll8 = {
+	.l_reg = 0x3144,
+	.m_reg = 0x3148,
+	.n_reg = 0x314c,
+	.config_reg = 0x3154,
+	.mode_reg = 0x3140,
+	.status_reg = 0x3158,
+	.status_bit = 16,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pll8",
+		.parent_names = (const char *[]){ "cxo" },
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap pll8_vote = {
+	.enable_reg = 0x34c0,
+	.enable_mask = BIT(8),
+	.hw.init = &(struct clk_init_data){
+		.name = "pll8_vote",
+		.parent_names = (const char *[]){ "pll8" },
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static struct clk_pll pll14 = {
+	.l_reg = 0x31c4,
+	.m_reg = 0x31c8,
+	.n_reg = 0x31cc,
+	.config_reg = 0x31d4,
+	.mode_reg = 0x31c0,
+	.status_reg = 0x31d8,
+	.status_bit = 16,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pll14",
+		.parent_names = (const char *[]){ "cxo" },
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap pll14_vote = {
+	.enable_reg = 0x34c0,
+	.enable_mask = BIT(11),
+	.hw.init = &(struct clk_init_data){
+		.name = "pll14_vote",
+		.parent_names = (const char *[]){ "pll14" },
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+enum {
+	P_CXO,
+	P_PLL8,
+	P_PLL14,
+};
+
+static const struct parent_map gcc_cxo_pll8_map[] = {
+	{ P_CXO, 0 },
+	{ P_PLL8, 3 }
+};
+
+static const char * const gcc_cxo_pll8[] = {
+	"cxo",
+	"pll8_vote",
+};
+
+static const struct parent_map gcc_cxo_pll14_map[] = {
+	{ P_CXO, 0 },
+	{ P_PLL14, 4 }
+};
+
+static const char * const gcc_cxo_pll14[] = {
+	"cxo",
+	"pll14_vote",
+};
+
+static const struct parent_map gcc_cxo_map[] = {
+	{ P_CXO, 0 },
+};
+
+static const char * const gcc_cxo[] = {
+	"cxo",
+};
+
+static struct freq_tbl clk_tbl_gsbi_uart[] = {
+	{  1843200, P_PLL8, 2,  6, 625 },
+	{  3686400, P_PLL8, 2, 12, 625 },
+	{  7372800, P_PLL8, 2, 24, 625 },
+	{ 14745600, P_PLL8, 2, 48, 625 },
+	{ 16000000, P_PLL8, 4,  1,   6 },
+	{ 24000000, P_PLL8, 4,  1,   4 },
+	{ 32000000, P_PLL8, 4,  1,   3 },
+	{ 40000000, P_PLL8, 1,  5,  48 },
+	{ 46400000, P_PLL8, 1, 29, 240 },
+	{ 48000000, P_PLL8, 4,  1,   2 },
+	{ 51200000, P_PLL8, 1,  2,  15 },
+	{ 56000000, P_PLL8, 1,  7,  48 },
+	{ 58982400, P_PLL8, 1, 96, 625 },
+	{ 64000000, P_PLL8, 2,  1,   3 },
+	{ }
+};
+
+static struct clk_rcg gsbi1_uart_src = {
+	.ns_reg = 0x29d4,
+	.md_reg = 0x29d0,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 16,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_uart,
+	.clkr = {
+		.enable_reg = 0x29d4,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi1_uart_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi1_uart_clk = {
+	.halt_reg = 0x2fcc,
+	.halt_bit = 10,
+	.clkr = {
+		.enable_reg = 0x29d4,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi1_uart_clk",
+			.parent_names = (const char *[]){
+				"gsbi1_uart_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gsbi2_uart_src = {
+	.ns_reg = 0x29f4,
+	.md_reg = 0x29f0,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 16,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_uart,
+	.clkr = {
+		.enable_reg = 0x29f4,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi2_uart_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi2_uart_clk = {
+	.halt_reg = 0x2fcc,
+	.halt_bit = 6,
+	.clkr = {
+		.enable_reg = 0x29f4,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi2_uart_clk",
+			.parent_names = (const char *[]){
+				"gsbi2_uart_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gsbi3_uart_src = {
+	.ns_reg = 0x2a14,
+	.md_reg = 0x2a10,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 16,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_uart,
+	.clkr = {
+		.enable_reg = 0x2a14,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi3_uart_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi3_uart_clk = {
+	.halt_reg = 0x2fcc,
+	.halt_bit = 2,
+	.clkr = {
+		.enable_reg = 0x2a14,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi3_uart_clk",
+			.parent_names = (const char *[]){
+				"gsbi3_uart_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gsbi4_uart_src = {
+	.ns_reg = 0x2a34,
+	.md_reg = 0x2a30,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 16,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_uart,
+	.clkr = {
+		.enable_reg = 0x2a34,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi4_uart_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi4_uart_clk = {
+	.halt_reg = 0x2fd0,
+	.halt_bit = 26,
+	.clkr = {
+		.enable_reg = 0x2a34,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi4_uart_clk",
+			.parent_names = (const char *[]){
+				"gsbi4_uart_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gsbi5_uart_src = {
+	.ns_reg = 0x2a54,
+	.md_reg = 0x2a50,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 16,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_uart,
+	.clkr = {
+		.enable_reg = 0x2a54,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi5_uart_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi5_uart_clk = {
+	.halt_reg = 0x2fd0,
+	.halt_bit = 22,
+	.clkr = {
+		.enable_reg = 0x2a54,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi5_uart_clk",
+			.parent_names = (const char *[]){
+				"gsbi5_uart_src",
+			},
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct freq_tbl clk_tbl_gsbi_qup[] = {
+	{   960000, P_CXO,  4, 1,  5 },
+	{  4800000, P_CXO,  4, 0,  1 },
+	{  9600000, P_CXO,  2, 0,  1 },
+	{ 15060000, P_PLL8, 1, 2, 51 },
+	{ 24000000, P_PLL8, 4, 1,  4 },
+	{ 25600000, P_PLL8, 1, 1, 15 },
+	{ 48000000, P_PLL8, 4, 1,  2 },
+	{ 51200000, P_PLL8, 1, 2, 15 },
+	{ }
+};
+
+static struct clk_rcg gsbi1_qup_src = {
+	.ns_reg = 0x29cc,
+	.md_reg = 0x29c8,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_qup,
+	.clkr = {
+		.enable_reg = 0x29cc,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi1_qup_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi1_qup_clk = {
+	.halt_reg = 0x2fcc,
+	.halt_bit = 9,
+	.clkr = {
+		.enable_reg = 0x29cc,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi1_qup_clk",
+			.parent_names = (const char *[]){ "gsbi1_qup_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gsbi2_qup_src = {
+	.ns_reg = 0x29ec,
+	.md_reg = 0x29e8,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_qup,
+	.clkr = {
+		.enable_reg = 0x29ec,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi2_qup_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi2_qup_clk = {
+	.halt_reg = 0x2fcc,
+	.halt_bit = 4,
+	.clkr = {
+		.enable_reg = 0x29ec,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi2_qup_clk",
+			.parent_names = (const char *[]){ "gsbi2_qup_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gsbi3_qup_src = {
+	.ns_reg = 0x2a0c,
+	.md_reg = 0x2a08,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_qup,
+	.clkr = {
+		.enable_reg = 0x2a0c,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi3_qup_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi3_qup_clk = {
+	.halt_reg = 0x2fcc,
+	.halt_bit = 0,
+	.clkr = {
+		.enable_reg = 0x2a0c,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi3_qup_clk",
+			.parent_names = (const char *[]){ "gsbi3_qup_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gsbi4_qup_src = {
+	.ns_reg = 0x2a2c,
+	.md_reg = 0x2a28,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_qup,
+	.clkr = {
+		.enable_reg = 0x2a2c,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi4_qup_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi4_qup_clk = {
+	.halt_reg = 0x2fd0,
+	.halt_bit = 24,
+	.clkr = {
+		.enable_reg = 0x2a2c,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi4_qup_clk",
+			.parent_names = (const char *[]){ "gsbi4_qup_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gsbi5_qup_src = {
+	.ns_reg = 0x2a4c,
+	.md_reg = 0x2a48,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_gsbi_qup,
+	.clkr = {
+		.enable_reg = 0x2a4c,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi5_qup_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	},
+};
+
+static struct clk_branch gsbi5_qup_clk = {
+	.halt_reg = 0x2fd0,
+	.halt_bit = 20,
+	.clkr = {
+		.enable_reg = 0x2a4c,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi5_qup_clk",
+			.parent_names = (const char *[]){ "gsbi5_qup_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl clk_tbl_gp[] = {
+	{ 9600000, P_CXO,  2, 0, 0 },
+	{ 19200000, P_CXO,  1, 0, 0 },
+	{ }
+};
+
+static struct clk_rcg gp0_src = {
+	.ns_reg = 0x2d24,
+	.md_reg = 0x2d00,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_map,
+	},
+	.freq_tbl = clk_tbl_gp,
+	.clkr = {
+		.enable_reg = 0x2d24,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gp0_src",
+			.parent_names = gcc_cxo,
+			.num_parents = 1,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_PARENT_GATE,
+		},
+	}
+};
+
+static struct clk_branch gp0_clk = {
+	.halt_reg = 0x2fd8,
+	.halt_bit = 7,
+	.clkr = {
+		.enable_reg = 0x2d24,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gp0_clk",
+			.parent_names = (const char *[]){ "gp0_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gp1_src = {
+	.ns_reg = 0x2d44,
+	.md_reg = 0x2d40,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_map,
+	},
+	.freq_tbl = clk_tbl_gp,
+	.clkr = {
+		.enable_reg = 0x2d44,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gp1_src",
+			.parent_names = gcc_cxo,
+			.num_parents = 1,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	}
+};
+
+static struct clk_branch gp1_clk = {
+	.halt_reg = 0x2fd8,
+	.halt_bit = 6,
+	.clkr = {
+		.enable_reg = 0x2d44,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gp1_clk",
+			.parent_names = (const char *[]){ "gp1_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg gp2_src = {
+	.ns_reg = 0x2d64,
+	.md_reg = 0x2d60,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_map,
+	},
+	.freq_tbl = clk_tbl_gp,
+	.clkr = {
+		.enable_reg = 0x2d64,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "gp2_src",
+			.parent_names = gcc_cxo,
+			.num_parents = 1,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	}
+};
+
+static struct clk_branch gp2_clk = {
+	.halt_reg = 0x2fd8,
+	.halt_bit = 5,
+	.clkr = {
+		.enable_reg = 0x2d64,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gp2_clk",
+			.parent_names = (const char *[]){ "gp2_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_branch pmem_clk = {
+	.hwcg_reg = 0x25a0,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fc8,
+	.halt_bit = 20,
+	.clkr = {
+		.enable_reg = 0x25a0,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "pmem_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_rcg prng_src = {
+	.ns_reg = 0x2e80,
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 4,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "prng_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+		},
+	},
+};
+
+static struct clk_branch prng_clk = {
+	.halt_reg = 0x2fd8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.halt_bit = 10,
+	.clkr = {
+		.enable_reg = 0x3080,
+		.enable_mask = BIT(10),
+		.hw.init = &(struct clk_init_data){
+			.name = "prng_clk",
+			.parent_names = (const char *[]){ "prng_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static const struct freq_tbl clk_tbl_sdc[] = {
+	{    144000, P_CXO,   1, 1, 133 },
+	{    400000, P_PLL8,  4, 1, 240 },
+	{  16000000, P_PLL8,  4, 1,   6 },
+	{  17070000, P_PLL8,  1, 2,  45 },
+	{  20210000, P_PLL8,  1, 1,  19 },
+	{  24000000, P_PLL8,  4, 1,   4 },
+	{  38400000, P_PLL8,  2, 1,   5 },
+	{  48000000, P_PLL8,  4, 1,   2 },
+	{  64000000, P_PLL8,  3, 1,   2 },
+	{  76800000, P_PLL8,  1, 1,   5 },
+	{ }
+};
+
+static struct clk_rcg sdc1_src = {
+	.ns_reg = 0x282c,
+	.md_reg = 0x2828,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_sdc,
+	.clkr = {
+		.enable_reg = 0x282c,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "sdc1_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	}
+};
+
+static struct clk_branch sdc1_clk = {
+	.halt_reg = 0x2fc8,
+	.halt_bit = 6,
+	.clkr = {
+		.enable_reg = 0x282c,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "sdc1_clk",
+			.parent_names = (const char *[]){ "sdc1_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg sdc2_src = {
+	.ns_reg = 0x284c,
+	.md_reg = 0x2848,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_sdc,
+	.clkr = {
+		.enable_reg = 0x284c,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "sdc2_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	}
+};
+
+static struct clk_branch sdc2_clk = {
+	.halt_reg = 0x2fc8,
+	.halt_bit = 5,
+	.clkr = {
+		.enable_reg = 0x284c,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "sdc2_clk",
+			.parent_names = (const char *[]){ "sdc2_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl clk_tbl_usb[] = {
+	{ 60000000, P_PLL8, 1, 5, 32 },
+	{ }
+};
+
+static struct clk_rcg usb_hs1_xcvr_src = {
+	.ns_reg = 0x290c,
+	.md_reg = 0x2908,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_usb,
+	.clkr = {
+		.enable_reg = 0x290c,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "usb_hs1_xcvr_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	}
+};
+
+static struct clk_branch usb_hs1_xcvr_clk = {
+	.halt_reg = 0x2fc8,
+	.halt_bit = 0,
+	.clkr = {
+		.enable_reg = 0x290c,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "usb_hs1_xcvr_clk",
+			.parent_names = (const char *[]){ "usb_hs1_xcvr_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg usb_hsic_xcvr_fs_src = {
+	.ns_reg = 0x2928,
+	.md_reg = 0x2924,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_usb,
+	.clkr = {
+		.enable_reg = 0x2928,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "usb_hsic_xcvr_fs_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	}
+};
+
+static struct clk_branch usb_hsic_xcvr_fs_clk = {
+	.halt_reg = 0x2fc8,
+	.halt_bit = 9,
+	.clkr = {
+		.enable_reg = 0x2928,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "usb_hsic_xcvr_fs_clk",
+			.parent_names =
+				(const char *[]){ "usb_hsic_xcvr_fs_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl clk_tbl_usb_hs1_system[] = {
+	{ 60000000, P_PLL8, 1, 5, 32 },
+	{ }
+};
+
+static struct clk_rcg usb_hs1_system_src = {
+	.ns_reg = 0x36a4,
+	.md_reg = 0x36a0,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_usb_hs1_system,
+	.clkr = {
+		.enable_reg = 0x36a4,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "usb_hs1_system_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	}
+};
+
+static struct clk_branch usb_hs1_system_clk = {
+	.halt_reg = 0x2fc8,
+	.halt_bit = 4,
+	.clkr = {
+		.enable_reg = 0x36a4,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.parent_names =
+				(const char *[]){ "usb_hs1_system_src" },
+			.num_parents = 1,
+			.name = "usb_hs1_system_clk",
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+		},
+	},
+};
+
+static const struct freq_tbl clk_tbl_usb_hsic_system[] = {
+	{ 64000000, P_PLL8, 1, 1, 6 },
+	{ }
+};
+
+static struct clk_rcg usb_hsic_system_src = {
+	.ns_reg = 0x2b58,
+	.md_reg = 0x2b54,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll8_map,
+	},
+	.freq_tbl = clk_tbl_usb_hsic_system,
+	.clkr = {
+		.enable_reg = 0x2b58,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "usb_hsic_system_src",
+			.parent_names = gcc_cxo_pll8,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	}
+};
+
+static struct clk_branch usb_hsic_system_clk = {
+	.halt_reg = 0x2fc8,
+	.halt_bit = 7,
+	.clkr = {
+		.enable_reg = 0x2b58,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.parent_names =
+				(const char *[]){ "usb_hsic_system_src" },
+			.num_parents = 1,
+			.name = "usb_hsic_system_clk",
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl clk_tbl_usb_hsic_hsic[] = {
+	{ 48000000, P_PLL14, 1, 0, 0 },
+	{ }
+};
+
+static struct clk_rcg usb_hsic_hsic_src = {
+	.ns_reg = 0x2b50,
+	.md_reg = 0x2b4c,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = gcc_cxo_pll14_map,
+	},
+	.freq_tbl = clk_tbl_usb_hsic_hsic,
+	.clkr = {
+		.enable_reg = 0x2b50,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "usb_hsic_hsic_src",
+			.parent_names = gcc_cxo_pll14,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	}
+};
+
+static struct clk_branch usb_hsic_hsic_clk = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x2b50,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.parent_names = (const char *[]){ "usb_hsic_hsic_src" },
+			.num_parents = 1,
+			.name = "usb_hsic_hsic_clk",
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_branch usb_hsic_hsio_cal_clk = {
+	.halt_reg = 0x2fc8,
+	.halt_bit = 8,
+	.clkr = {
+		.enable_reg = 0x2b48,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.parent_names = (const char *[]){ "cxo" },
+			.num_parents = 1,
+			.name = "usb_hsic_hsio_cal_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch ce1_core_clk = {
+	.hwcg_reg = 0x2724,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fd4,
+	.halt_bit = 27,
+	.clkr = {
+		.enable_reg = 0x2724,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "ce1_core_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch ce1_h_clk = {
+	.halt_reg = 0x2fd4,
+	.halt_bit = 1,
+	.clkr = {
+		.enable_reg = 0x2720,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "ce1_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch dma_bam_h_clk = {
+	.hwcg_reg = 0x25c0,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fc8,
+	.halt_bit = 12,
+	.clkr = {
+		.enable_reg = 0x25c0,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "dma_bam_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch gsbi1_h_clk = {
+	.hwcg_reg = 0x29c0,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fcc,
+	.halt_bit = 11,
+	.clkr = {
+		.enable_reg = 0x29c0,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi1_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch gsbi2_h_clk = {
+	.hwcg_reg = 0x29e0,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fcc,
+	.halt_bit = 7,
+	.clkr = {
+		.enable_reg = 0x29e0,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi2_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch gsbi3_h_clk = {
+	.hwcg_reg = 0x2a00,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fcc,
+	.halt_bit = 3,
+	.clkr = {
+		.enable_reg = 0x2a00,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi3_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch gsbi4_h_clk = {
+	.hwcg_reg = 0x2a20,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fd0,
+	.halt_bit = 27,
+	.clkr = {
+		.enable_reg = 0x2a20,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi4_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch gsbi5_h_clk = {
+	.hwcg_reg = 0x2a40,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fd0,
+	.halt_bit = 23,
+	.clkr = {
+		.enable_reg = 0x2a40,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gsbi5_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch usb_hs1_h_clk = {
+	.hwcg_reg = 0x2900,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fc8,
+	.halt_bit = 1,
+	.clkr = {
+		.enable_reg = 0x2900,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "usb_hs1_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch usb_hsic_h_clk = {
+	.halt_reg = 0x2fcc,
+	.halt_bit = 28,
+	.clkr = {
+		.enable_reg = 0x2920,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "usb_hsic_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch sdc1_h_clk = {
+	.hwcg_reg = 0x2820,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fc8,
+	.halt_bit = 11,
+	.clkr = {
+		.enable_reg = 0x2820,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "sdc1_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch sdc2_h_clk = {
+	.hwcg_reg = 0x2840,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fc8,
+	.halt_bit = 10,
+	.clkr = {
+		.enable_reg = 0x2840,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "sdc2_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch adm0_clk = {
+	.halt_reg = 0x2fdc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.halt_bit = 14,
+	.clkr = {
+		.enable_reg = 0x3080,
+		.enable_mask = BIT(2),
+		.hw.init = &(struct clk_init_data){
+			.name = "adm0_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch adm0_pbus_clk = {
+	.hwcg_reg = 0x2208,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fdc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.halt_bit = 13,
+	.clkr = {
+		.enable_reg = 0x3080,
+		.enable_mask = BIT(3),
+		.hw.init = &(struct clk_init_data){
+			.name = "adm0_pbus_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch pmic_arb0_h_clk = {
+	.halt_reg = 0x2fd8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.halt_bit = 22,
+	.clkr = {
+		.enable_reg = 0x3080,
+		.enable_mask = BIT(8),
+		.hw.init = &(struct clk_init_data){
+			.name = "pmic_arb0_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch pmic_arb1_h_clk = {
+	.halt_reg = 0x2fd8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.halt_bit = 21,
+	.clkr = {
+		.enable_reg = 0x3080,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "pmic_arb1_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch pmic_ssbi2_clk = {
+	.halt_reg = 0x2fd8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.halt_bit = 23,
+	.clkr = {
+		.enable_reg = 0x3080,
+		.enable_mask = BIT(7),
+		.hw.init = &(struct clk_init_data){
+			.name = "pmic_ssbi2_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_branch rpm_msg_ram_h_clk = {
+	.hwcg_reg = 0x27e0,
+	.hwcg_bit = 6,
+	.halt_reg = 0x2fd8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.halt_bit = 12,
+	.clkr = {
+		.enable_reg = 0x3080,
+		.enable_mask = BIT(6),
+		.hw.init = &(struct clk_init_data){
+			.name = "rpm_msg_ram_h_clk",
+			.ops = &clk_branch_ops,
+		},
+	},
+};
+
+static struct clk_hw *gcc_mdm9615_hws[] = {
+	&cxo.hw,
+};
+
+static struct clk_regmap *gcc_mdm9615_clks[] = {
+	[PLL0] = &pll0.clkr,
+	[PLL0_VOTE] = &pll0_vote,
+	[PLL4_VOTE] = &pll4_vote,
+	[PLL8] = &pll8.clkr,
+	[PLL8_VOTE] = &pll8_vote,
+	[PLL14] = &pll14.clkr,
+	[PLL14_VOTE] = &pll14_vote,
+	[GSBI1_UART_SRC] = &gsbi1_uart_src.clkr,
+	[GSBI1_UART_CLK] = &gsbi1_uart_clk.clkr,
+	[GSBI2_UART_SRC] = &gsbi2_uart_src.clkr,
+	[GSBI2_UART_CLK] = &gsbi2_uart_clk.clkr,
+	[GSBI3_UART_SRC] = &gsbi3_uart_src.clkr,
+	[GSBI3_UART_CLK] = &gsbi3_uart_clk.clkr,
+	[GSBI4_UART_SRC] = &gsbi4_uart_src.clkr,
+	[GSBI4_UART_CLK] = &gsbi4_uart_clk.clkr,
+	[GSBI5_UART_SRC] = &gsbi5_uart_src.clkr,
+	[GSBI5_UART_CLK] = &gsbi5_uart_clk.clkr,
+	[GSBI1_QUP_SRC] = &gsbi1_qup_src.clkr,
+	[GSBI1_QUP_CLK] = &gsbi1_qup_clk.clkr,
+	[GSBI2_QUP_SRC] = &gsbi2_qup_src.clkr,
+	[GSBI2_QUP_CLK] = &gsbi2_qup_clk.clkr,
+	[GSBI3_QUP_SRC] = &gsbi3_qup_src.clkr,
+	[GSBI3_QUP_CLK] = &gsbi3_qup_clk.clkr,
+	[GSBI4_QUP_SRC] = &gsbi4_qup_src.clkr,
+	[GSBI4_QUP_CLK] = &gsbi4_qup_clk.clkr,
+	[GSBI5_QUP_SRC] = &gsbi5_qup_src.clkr,
+	[GSBI5_QUP_CLK] = &gsbi5_qup_clk.clkr,
+	[GP0_SRC] = &gp0_src.clkr,
+	[GP0_CLK] = &gp0_clk.clkr,
+	[GP1_SRC] = &gp1_src.clkr,
+	[GP1_CLK] = &gp1_clk.clkr,
+	[GP2_SRC] = &gp2_src.clkr,
+	[GP2_CLK] = &gp2_clk.clkr,
+	[PMEM_A_CLK] = &pmem_clk.clkr,
+	[PRNG_SRC] = &prng_src.clkr,
+	[PRNG_CLK] = &prng_clk.clkr,
+	[SDC1_SRC] = &sdc1_src.clkr,
+	[SDC1_CLK] = &sdc1_clk.clkr,
+	[SDC2_SRC] = &sdc2_src.clkr,
+	[SDC2_CLK] = &sdc2_clk.clkr,
+	[USB_HS1_XCVR_SRC] = &usb_hs1_xcvr_src.clkr,
+	[USB_HS1_XCVR_CLK] = &usb_hs1_xcvr_clk.clkr,
+	[USB_HS1_SYSTEM_CLK_SRC] = &usb_hs1_system_src.clkr,
+	[USB_HS1_SYSTEM_CLK] = &usb_hs1_system_clk.clkr,
+	[USB_HSIC_XCVR_FS_SRC] = &usb_hsic_xcvr_fs_src.clkr,
+	[USB_HSIC_XCVR_FS_CLK] = &usb_hsic_xcvr_fs_clk.clkr,
+	[USB_HSIC_SYSTEM_CLK_SRC] = &usb_hsic_system_src.clkr,
+	[USB_HSIC_SYSTEM_CLK] = &usb_hsic_system_clk.clkr,
+	[USB_HSIC_HSIC_CLK_SRC] = &usb_hsic_hsic_src.clkr,
+	[USB_HSIC_HSIC_CLK] = &usb_hsic_hsic_clk.clkr,
+	[USB_HSIC_HSIO_CAL_CLK] = &usb_hsic_hsio_cal_clk.clkr,
+	[CE1_CORE_CLK] = &ce1_core_clk.clkr,
+	[CE1_H_CLK] = &ce1_h_clk.clkr,
+	[DMA_BAM_H_CLK] = &dma_bam_h_clk.clkr,
+	[GSBI1_H_CLK] = &gsbi1_h_clk.clkr,
+	[GSBI2_H_CLK] = &gsbi2_h_clk.clkr,
+	[GSBI3_H_CLK] = &gsbi3_h_clk.clkr,
+	[GSBI4_H_CLK] = &gsbi4_h_clk.clkr,
+	[GSBI5_H_CLK] = &gsbi5_h_clk.clkr,
+	[USB_HS1_H_CLK] = &usb_hs1_h_clk.clkr,
+	[USB_HSIC_H_CLK] = &usb_hsic_h_clk.clkr,
+	[SDC1_H_CLK] = &sdc1_h_clk.clkr,
+	[SDC2_H_CLK] = &sdc2_h_clk.clkr,
+	[ADM0_CLK] = &adm0_clk.clkr,
+	[ADM0_PBUS_CLK] = &adm0_pbus_clk.clkr,
+	[PMIC_ARB0_H_CLK] = &pmic_arb0_h_clk.clkr,
+	[PMIC_ARB1_H_CLK] = &pmic_arb1_h_clk.clkr,
+	[PMIC_SSBI2_CLK] = &pmic_ssbi2_clk.clkr,
+	[RPM_MSG_RAM_H_CLK] = &rpm_msg_ram_h_clk.clkr,
+};
+
+static const struct qcom_reset_map gcc_mdm9615_resets[] = {
+	[DMA_BAM_RESET] = { 0x25c0, 7 },
+	[CE1_H_RESET] = { 0x2720, 7 },
+	[CE1_CORE_RESET] = { 0x2724, 7 },
+	[SDC1_RESET] = { 0x2830 },
+	[SDC2_RESET] = { 0x2850 },
+	[ADM0_C2_RESET] = { 0x220c, 4 },
+	[ADM0_C1_RESET] = { 0x220c, 3 },
+	[ADM0_C0_RESET] = { 0x220c, 2 },
+	[ADM0_PBUS_RESET] = { 0x220c, 1 },
+	[ADM0_RESET] = { 0x220c },
+	[USB_HS1_RESET] = { 0x2910 },
+	[USB_HSIC_RESET] = { 0x2934 },
+	[GSBI1_RESET] = { 0x29dc },
+	[GSBI2_RESET] = { 0x29fc },
+	[GSBI3_RESET] = { 0x2a1c },
+	[GSBI4_RESET] = { 0x2a3c },
+	[GSBI5_RESET] = { 0x2a5c },
+	[PDM_RESET] = { 0x2CC0, 12 },
+};
+
+static const struct regmap_config gcc_mdm9615_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0x3660,
+	.fast_io	= true,
+};
+
+static const struct qcom_cc_desc gcc_mdm9615_desc = {
+	.config = &gcc_mdm9615_regmap_config,
+	.clks = gcc_mdm9615_clks,
+	.num_clks = ARRAY_SIZE(gcc_mdm9615_clks),
+	.resets = gcc_mdm9615_resets,
+	.num_resets = ARRAY_SIZE(gcc_mdm9615_resets),
+};
+
+static const struct of_device_id gcc_mdm9615_match_table[] = {
+	{ .compatible = "qcom,gcc-mdm9615" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_mdm9615_match_table);
+
+static int gcc_mdm9615_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct regmap *regmap;
+	int ret;
+	int i;
+
+	regmap = qcom_cc_map(pdev, &gcc_mdm9615_desc);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	for (i = 0; i < ARRAY_SIZE(gcc_mdm9615_hws); i++) {
+		ret = devm_clk_hw_register(dev, gcc_mdm9615_hws[i]);
+		if (ret)
+			return ret;
+	}
+
+	return qcom_cc_really_probe(pdev, &gcc_mdm9615_desc, regmap);
+}
+
+static struct platform_driver gcc_mdm9615_driver = {
+	.probe		= gcc_mdm9615_probe,
+	.driver		= {
+		.name	= "gcc-mdm9615",
+		.of_match_table = gcc_mdm9615_match_table,
+	},
+};
+
+static int __init gcc_mdm9615_init(void)
+{
+	return platform_driver_register(&gcc_mdm9615_driver);
+}
+core_initcall(gcc_mdm9615_init);
+
+static void __exit gcc_mdm9615_exit(void)
+{
+	platform_driver_unregister(&gcc_mdm9615_driver);
+}
+module_exit(gcc_mdm9615_exit);
+
+MODULE_DESCRIPTION("QCOM GCC MDM9615 Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:gcc-mdm9615");
diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c
index bbf732bbc3fd..fe03e6fbc7df 100644
--- a/drivers/clk/qcom/gcc-msm8996.c
+++ b/drivers/clk/qcom/gcc-msm8996.c
@@ -2592,9 +2592,9 @@ static struct clk_branch gcc_pcie_2_aux_clk = {
 };
 
 static struct clk_branch gcc_pcie_2_pipe_clk = {
-	.halt_reg = 0x6e108,
+	.halt_reg = 0x6e018,
 	.clkr = {
-		.enable_reg = 0x6e108,
+		.enable_reg = 0x6e018,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
 			.name = "gcc_pcie_2_pipe_clk",
@@ -3329,6 +3329,8 @@ static const struct qcom_reset_map gcc_msm8996_resets[] = {
 	[GCC_USB_20_BCR] = { 0x12000 },
 	[GCC_QUSB2PHY_PRIM_BCR] = { 0x12038 },
 	[GCC_QUSB2PHY_SEC_BCR] = { 0x1203c },
+	[GCC_USB3_PHY_BCR] = { 0x50020 },
+	[GCC_USB3PHY_PHY_BCR] = { 0x50024 },
 	[GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x6a000 },
 	[GCC_SDCC1_BCR] = { 0x13000 },
 	[GCC_SDCC2_BCR] = { 0x14000 },
@@ -3404,6 +3406,8 @@ static const struct qcom_reset_map gcc_msm8996_resets[] = {
 	[GCC_PCIE_2_BCR] = { 0x6e000 },
 	[GCC_PCIE_2_PHY_BCR] = { 0x6e038 },
 	[GCC_PCIE_PHY_BCR] = { 0x6f000 },
+	[GCC_PCIE_PHY_COM_BCR] = { 0x6f014 },
+	[GCC_PCIE_PHY_COM_NOCSR_BCR] = { 0x6f00c },
 	[GCC_DCD_BCR] = { 0x70000 },
 	[GCC_OBT_ODT_BCR] = { 0x73000 },
 	[GCC_UFS_BCR] = { 0x75000 },
@@ -3447,9 +3451,8 @@ MODULE_DEVICE_TABLE(of, gcc_msm8996_match_table);
 
 static int gcc_msm8996_probe(struct platform_device *pdev)
 {
-	struct clk *clk;
 	struct device *dev = &pdev->dev;
-	int i;
+	int i, ret;
 	struct regmap *regmap;
 
 	regmap = qcom_cc_map(pdev, &gcc_msm8996_desc);
@@ -3463,9 +3466,9 @@ static int gcc_msm8996_probe(struct platform_device *pdev)
 	regmap_update_bits(regmap, 0x52008, BIT(21), BIT(21));
 
 	for (i = 0; i < ARRAY_SIZE(gcc_msm8996_hws); i++) {
-		clk = devm_clk_register(dev, gcc_msm8996_hws[i]);
-		if (IS_ERR(clk))
-			return PTR_ERR(clk);
+		ret = devm_clk_hw_register(dev, gcc_msm8996_hws[i]);
+		if (ret)
+			return ret;
 	}
 
 	return qcom_cc_really_probe(pdev, &gcc_msm8996_desc, regmap);
diff --git a/drivers/clk/qcom/lcc-mdm9615.c b/drivers/clk/qcom/lcc-mdm9615.c
new file mode 100644
index 000000000000..3237ef4c1197
--- /dev/null
+++ b/drivers/clk/qcom/lcc-mdm9615.c
@@ -0,0 +1,580 @@
+/*
+ * Copyright (c) 2014, The Linux Foundation. All rights reserved.
+ * Copyright (c) BayLibre, SAS.
+ * Author : Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,lcc-mdm9615.h>
+
+#include "common.h"
+#include "clk-regmap.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-branch.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+
+static struct clk_pll pll4 = {
+	.l_reg = 0x4,
+	.m_reg = 0x8,
+	.n_reg = 0xc,
+	.config_reg = 0x14,
+	.mode_reg = 0x0,
+	.status_reg = 0x18,
+	.status_bit = 16,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pll4",
+		.parent_names = (const char *[]){ "cxo" },
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+enum {
+	P_CXO,
+	P_PLL4,
+};
+
+static const struct parent_map lcc_cxo_pll4_map[] = {
+	{ P_CXO, 0 },
+	{ P_PLL4, 2 }
+};
+
+static const char * const lcc_cxo_pll4[] = {
+	"cxo",
+	"pll4_vote",
+};
+
+static struct freq_tbl clk_tbl_aif_osr_492[] = {
+	{   512000, P_PLL4, 4, 1, 240 },
+	{   768000, P_PLL4, 4, 1, 160 },
+	{  1024000, P_PLL4, 4, 1, 120 },
+	{  1536000, P_PLL4, 4, 1,  80 },
+	{  2048000, P_PLL4, 4, 1,  60 },
+	{  3072000, P_PLL4, 4, 1,  40 },
+	{  4096000, P_PLL4, 4, 1,  30 },
+	{  6144000, P_PLL4, 4, 1,  20 },
+	{  8192000, P_PLL4, 4, 1,  15 },
+	{ 12288000, P_PLL4, 4, 1,  10 },
+	{ 24576000, P_PLL4, 4, 1,   5 },
+	{ 27000000, P_CXO,  1, 0,   0 },
+	{ }
+};
+
+static struct freq_tbl clk_tbl_aif_osr_393[] = {
+	{   512000, P_PLL4, 4, 1, 192 },
+	{   768000, P_PLL4, 4, 1, 128 },
+	{  1024000, P_PLL4, 4, 1,  96 },
+	{  1536000, P_PLL4, 4, 1,  64 },
+	{  2048000, P_PLL4, 4, 1,  48 },
+	{  3072000, P_PLL4, 4, 1,  32 },
+	{  4096000, P_PLL4, 4, 1,  24 },
+	{  6144000, P_PLL4, 4, 1,  16 },
+	{  8192000, P_PLL4, 4, 1,  12 },
+	{ 12288000, P_PLL4, 4, 1,   8 },
+	{ 24576000, P_PLL4, 4, 1,   4 },
+	{ 27000000, P_CXO,  1, 0,   0 },
+	{ }
+};
+
+static struct clk_rcg mi2s_osr_src = {
+	.ns_reg = 0x48,
+	.md_reg = 0x4c,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 24,
+		.m_val_shift = 8,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = lcc_cxo_pll4_map,
+	},
+	.freq_tbl = clk_tbl_aif_osr_393,
+	.clkr = {
+		.enable_reg = 0x48,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "mi2s_osr_src",
+			.parent_names = lcc_cxo_pll4,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	},
+};
+
+static const char * const lcc_mi2s_parents[] = {
+	"mi2s_osr_src",
+};
+
+static struct clk_branch mi2s_osr_clk = {
+	.halt_reg = 0x50,
+	.halt_bit = 1,
+	.halt_check = BRANCH_HALT_ENABLE,
+	.clkr = {
+		.enable_reg = 0x48,
+		.enable_mask = BIT(17),
+		.hw.init = &(struct clk_init_data){
+			.name = "mi2s_osr_clk",
+			.parent_names = lcc_mi2s_parents,
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_regmap_div mi2s_div_clk = {
+	.reg = 0x48,
+	.shift = 10,
+	.width = 4,
+	.clkr = {
+		.enable_reg = 0x48,
+		.enable_mask = BIT(15),
+		.hw.init = &(struct clk_init_data){
+			.name = "mi2s_div_clk",
+			.parent_names = lcc_mi2s_parents,
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+		},
+	},
+};
+
+static struct clk_branch mi2s_bit_div_clk = {
+	.halt_reg = 0x50,
+	.halt_bit = 0,
+	.halt_check = BRANCH_HALT_ENABLE,
+	.clkr = {
+		.enable_reg = 0x48,
+		.enable_mask = BIT(15),
+		.hw.init = &(struct clk_init_data){
+			.name = "mi2s_bit_div_clk",
+			.parent_names = (const char *[]){ "mi2s_div_clk" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_regmap_mux mi2s_bit_clk = {
+	.reg = 0x48,
+	.shift = 14,
+	.width = 1,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "mi2s_bit_clk",
+			.parent_names = (const char *[]){
+				"mi2s_bit_div_clk",
+				"mi2s_codec_clk",
+			},
+			.num_parents = 2,
+			.ops = &clk_regmap_mux_closest_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+#define CLK_AIF_OSR_DIV(prefix, _ns, _md, hr)			\
+static struct clk_rcg prefix##_osr_src = {			\
+	.ns_reg = _ns,						\
+	.md_reg = _md,						\
+	.mn = {							\
+		.mnctr_en_bit = 8,				\
+		.mnctr_reset_bit = 7,				\
+		.mnctr_mode_shift = 5,				\
+		.n_val_shift = 24,				\
+		.m_val_shift = 8,				\
+		.width = 8,					\
+	},							\
+	.p = {							\
+		.pre_div_shift = 3,				\
+		.pre_div_width = 2,				\
+	},							\
+	.s = {							\
+		.src_sel_shift = 0,				\
+		.parent_map = lcc_cxo_pll4_map,			\
+	},							\
+	.freq_tbl = clk_tbl_aif_osr_393,			\
+	.clkr = {						\
+		.enable_reg = _ns,				\
+		.enable_mask = BIT(9),				\
+		.hw.init = &(struct clk_init_data){		\
+			.name = #prefix "_osr_src",		\
+			.parent_names = lcc_cxo_pll4,		\
+			.num_parents = 2,			\
+			.ops = &clk_rcg_ops,			\
+			.flags = CLK_SET_RATE_GATE,		\
+		},						\
+	},							\
+};								\
+								\
+static const char * const lcc_##prefix##_parents[] = {		\
+	#prefix "_osr_src",					\
+};								\
+								\
+static struct clk_branch prefix##_osr_clk = {			\
+	.halt_reg = hr,						\
+	.halt_bit = 1,						\
+	.halt_check = BRANCH_HALT_ENABLE,			\
+	.clkr = {						\
+		.enable_reg = _ns,				\
+		.enable_mask = BIT(21),				\
+		.hw.init = &(struct clk_init_data){		\
+			.name = #prefix "_osr_clk",		\
+			.parent_names = lcc_##prefix##_parents,	\
+			.num_parents = 1,			\
+			.ops = &clk_branch_ops,			\
+			.flags = CLK_SET_RATE_PARENT,		\
+		},						\
+	},							\
+};								\
+								\
+static struct clk_regmap_div prefix##_div_clk = {		\
+	.reg = _ns,						\
+	.shift = 10,						\
+	.width = 8,						\
+	.clkr = {						\
+		.hw.init = &(struct clk_init_data){		\
+			.name = #prefix "_div_clk",		\
+			.parent_names = lcc_##prefix##_parents,	\
+			.num_parents = 1,			\
+			.ops = &clk_regmap_div_ops,		\
+		},						\
+	},							\
+};								\
+								\
+static struct clk_branch prefix##_bit_div_clk = {		\
+	.halt_reg = hr,						\
+	.halt_bit = 0,						\
+	.halt_check = BRANCH_HALT_ENABLE,			\
+	.clkr = {						\
+		.enable_reg = _ns,				\
+		.enable_mask = BIT(19),				\
+		.hw.init = &(struct clk_init_data){		\
+			.name = #prefix "_bit_div_clk",		\
+			.parent_names = (const char *[]){	\
+				#prefix "_div_clk"		\
+			},					\
+			.num_parents = 1,			\
+			.ops = &clk_branch_ops,			\
+			.flags = CLK_SET_RATE_PARENT,		\
+		},						\
+	},							\
+};								\
+								\
+static struct clk_regmap_mux prefix##_bit_clk = {		\
+	.reg = _ns,						\
+	.shift = 18,						\
+	.width = 1,						\
+	.clkr = {						\
+		.hw.init = &(struct clk_init_data){		\
+			.name = #prefix "_bit_clk",		\
+			.parent_names = (const char *[]){	\
+				#prefix "_bit_div_clk",		\
+				#prefix "_codec_clk",		\
+			},					\
+			.num_parents = 2,			\
+			.ops = &clk_regmap_mux_closest_ops,	\
+			.flags = CLK_SET_RATE_PARENT,		\
+		},						\
+	},							\
+}
+
+CLK_AIF_OSR_DIV(codec_i2s_mic, 0x60, 0x64, 0x68);
+CLK_AIF_OSR_DIV(spare_i2s_mic, 0x78, 0x7c, 0x80);
+CLK_AIF_OSR_DIV(codec_i2s_spkr, 0x6c, 0x70, 0x74);
+CLK_AIF_OSR_DIV(spare_i2s_spkr, 0x84, 0x88, 0x8c);
+
+static struct freq_tbl clk_tbl_pcm_492[] = {
+	{   256000, P_PLL4, 4, 1, 480 },
+	{   512000, P_PLL4, 4, 1, 240 },
+	{   768000, P_PLL4, 4, 1, 160 },
+	{  1024000, P_PLL4, 4, 1, 120 },
+	{  1536000, P_PLL4, 4, 1,  80 },
+	{  2048000, P_PLL4, 4, 1,  60 },
+	{  3072000, P_PLL4, 4, 1,  40 },
+	{  4096000, P_PLL4, 4, 1,  30 },
+	{  6144000, P_PLL4, 4, 1,  20 },
+	{  8192000, P_PLL4, 4, 1,  15 },
+	{ 12288000, P_PLL4, 4, 1,  10 },
+	{ 24576000, P_PLL4, 4, 1,   5 },
+	{ 27000000, P_CXO,  1, 0,   0 },
+	{ }
+};
+
+static struct freq_tbl clk_tbl_pcm_393[] = {
+	{   256000, P_PLL4, 4, 1, 384 },
+	{   512000, P_PLL4, 4, 1, 192 },
+	{   768000, P_PLL4, 4, 1, 128 },
+	{  1024000, P_PLL4, 4, 1,  96 },
+	{  1536000, P_PLL4, 4, 1,  64 },
+	{  2048000, P_PLL4, 4, 1,  48 },
+	{  3072000, P_PLL4, 4, 1,  32 },
+	{  4096000, P_PLL4, 4, 1,  24 },
+	{  6144000, P_PLL4, 4, 1,  16 },
+	{  8192000, P_PLL4, 4, 1,  12 },
+	{ 12288000, P_PLL4, 4, 1,   8 },
+	{ 24576000, P_PLL4, 4, 1,   4 },
+	{ 27000000, P_CXO,  1, 0,   0 },
+	{ }
+};
+
+static struct clk_rcg pcm_src = {
+	.ns_reg = 0x54,
+	.md_reg = 0x58,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 16,
+		.m_val_shift = 16,
+		.width = 16,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = lcc_cxo_pll4_map,
+	},
+	.freq_tbl = clk_tbl_pcm_393,
+	.clkr = {
+		.enable_reg = 0x54,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "pcm_src",
+			.parent_names = lcc_cxo_pll4,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	},
+};
+
+static struct clk_branch pcm_clk_out = {
+	.halt_reg = 0x5c,
+	.halt_bit = 0,
+	.halt_check = BRANCH_HALT_ENABLE,
+	.clkr = {
+		.enable_reg = 0x54,
+		.enable_mask = BIT(11),
+		.hw.init = &(struct clk_init_data){
+			.name = "pcm_clk_out",
+			.parent_names = (const char *[]){ "pcm_src" },
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_regmap_mux pcm_clk = {
+	.reg = 0x54,
+	.shift = 10,
+	.width = 1,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "pcm_clk",
+			.parent_names = (const char *[]){
+				"pcm_clk_out",
+				"pcm_codec_clk",
+			},
+			.num_parents = 2,
+			.ops = &clk_regmap_mux_closest_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg slimbus_src = {
+	.ns_reg = 0xcc,
+	.md_reg = 0xd0,
+	.mn = {
+		.mnctr_en_bit = 8,
+		.mnctr_reset_bit = 7,
+		.mnctr_mode_shift = 5,
+		.n_val_shift = 24,
+		.m_val_shift = 8,
+		.width = 8,
+	},
+	.p = {
+		.pre_div_shift = 3,
+		.pre_div_width = 2,
+	},
+	.s = {
+		.src_sel_shift = 0,
+		.parent_map = lcc_cxo_pll4_map,
+	},
+	.freq_tbl = clk_tbl_aif_osr_393,
+	.clkr = {
+		.enable_reg = 0xcc,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "slimbus_src",
+			.parent_names = lcc_cxo_pll4,
+			.num_parents = 2,
+			.ops = &clk_rcg_ops,
+			.flags = CLK_SET_RATE_GATE,
+		},
+	},
+};
+
+static const char * const lcc_slimbus_parents[] = {
+	"slimbus_src",
+};
+
+static struct clk_branch audio_slimbus_clk = {
+	.halt_reg = 0xd4,
+	.halt_bit = 0,
+	.halt_check = BRANCH_HALT_ENABLE,
+	.clkr = {
+		.enable_reg = 0xcc,
+		.enable_mask = BIT(10),
+		.hw.init = &(struct clk_init_data){
+			.name = "audio_slimbus_clk",
+			.parent_names = lcc_slimbus_parents,
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_branch sps_slimbus_clk = {
+	.halt_reg = 0xd4,
+	.halt_bit = 1,
+	.halt_check = BRANCH_HALT_ENABLE,
+	.clkr = {
+		.enable_reg = 0xcc,
+		.enable_mask = BIT(12),
+		.hw.init = &(struct clk_init_data){
+			.name = "sps_slimbus_clk",
+			.parent_names = lcc_slimbus_parents,
+			.num_parents = 1,
+			.ops = &clk_branch_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_regmap *lcc_mdm9615_clks[] = {
+	[PLL4] = &pll4.clkr,
+	[MI2S_OSR_SRC] = &mi2s_osr_src.clkr,
+	[MI2S_OSR_CLK] = &mi2s_osr_clk.clkr,
+	[MI2S_DIV_CLK] = &mi2s_div_clk.clkr,
+	[MI2S_BIT_DIV_CLK] = &mi2s_bit_div_clk.clkr,
+	[MI2S_BIT_CLK] = &mi2s_bit_clk.clkr,
+	[PCM_SRC] = &pcm_src.clkr,
+	[PCM_CLK_OUT] = &pcm_clk_out.clkr,
+	[PCM_CLK] = &pcm_clk.clkr,
+	[SLIMBUS_SRC] = &slimbus_src.clkr,
+	[AUDIO_SLIMBUS_CLK] = &audio_slimbus_clk.clkr,
+	[SPS_SLIMBUS_CLK] = &sps_slimbus_clk.clkr,
+	[CODEC_I2S_MIC_OSR_SRC] = &codec_i2s_mic_osr_src.clkr,
+	[CODEC_I2S_MIC_OSR_CLK] = &codec_i2s_mic_osr_clk.clkr,
+	[CODEC_I2S_MIC_DIV_CLK] = &codec_i2s_mic_div_clk.clkr,
+	[CODEC_I2S_MIC_BIT_DIV_CLK] = &codec_i2s_mic_bit_div_clk.clkr,
+	[CODEC_I2S_MIC_BIT_CLK] = &codec_i2s_mic_bit_clk.clkr,
+	[SPARE_I2S_MIC_OSR_SRC] = &spare_i2s_mic_osr_src.clkr,
+	[SPARE_I2S_MIC_OSR_CLK] = &spare_i2s_mic_osr_clk.clkr,
+	[SPARE_I2S_MIC_DIV_CLK] = &spare_i2s_mic_div_clk.clkr,
+	[SPARE_I2S_MIC_BIT_DIV_CLK] = &spare_i2s_mic_bit_div_clk.clkr,
+	[SPARE_I2S_MIC_BIT_CLK] = &spare_i2s_mic_bit_clk.clkr,
+	[CODEC_I2S_SPKR_OSR_SRC] = &codec_i2s_spkr_osr_src.clkr,
+	[CODEC_I2S_SPKR_OSR_CLK] = &codec_i2s_spkr_osr_clk.clkr,
+	[CODEC_I2S_SPKR_DIV_CLK] = &codec_i2s_spkr_div_clk.clkr,
+	[CODEC_I2S_SPKR_BIT_DIV_CLK] = &codec_i2s_spkr_bit_div_clk.clkr,
+	[CODEC_I2S_SPKR_BIT_CLK] = &codec_i2s_spkr_bit_clk.clkr,
+	[SPARE_I2S_SPKR_OSR_SRC] = &spare_i2s_spkr_osr_src.clkr,
+	[SPARE_I2S_SPKR_OSR_CLK] = &spare_i2s_spkr_osr_clk.clkr,
+	[SPARE_I2S_SPKR_DIV_CLK] = &spare_i2s_spkr_div_clk.clkr,
+	[SPARE_I2S_SPKR_BIT_DIV_CLK] = &spare_i2s_spkr_bit_div_clk.clkr,
+	[SPARE_I2S_SPKR_BIT_CLK] = &spare_i2s_spkr_bit_clk.clkr,
+};
+
+static const struct regmap_config lcc_mdm9615_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0xfc,
+	.fast_io	= true,
+};
+
+static const struct qcom_cc_desc lcc_mdm9615_desc = {
+	.config = &lcc_mdm9615_regmap_config,
+	.clks = lcc_mdm9615_clks,
+	.num_clks = ARRAY_SIZE(lcc_mdm9615_clks),
+};
+
+static const struct of_device_id lcc_mdm9615_match_table[] = {
+	{ .compatible = "qcom,lcc-mdm9615" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lcc_mdm9615_match_table);
+
+static int lcc_mdm9615_probe(struct platform_device *pdev)
+{
+	u32 val;
+	struct regmap *regmap;
+
+	regmap = qcom_cc_map(pdev, &lcc_mdm9615_desc);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	/* Use the correct frequency plan depending on speed of PLL4 */
+	regmap_read(regmap, 0x4, &val);
+	if (val == 0x12) {
+		slimbus_src.freq_tbl = clk_tbl_aif_osr_492;
+		mi2s_osr_src.freq_tbl = clk_tbl_aif_osr_492;
+		codec_i2s_mic_osr_src.freq_tbl = clk_tbl_aif_osr_492;
+		spare_i2s_mic_osr_src.freq_tbl = clk_tbl_aif_osr_492;
+		codec_i2s_spkr_osr_src.freq_tbl = clk_tbl_aif_osr_492;
+		spare_i2s_spkr_osr_src.freq_tbl = clk_tbl_aif_osr_492;
+		pcm_src.freq_tbl = clk_tbl_pcm_492;
+	}
+	/* Enable PLL4 source on the LPASS Primary PLL Mux */
+	regmap_write(regmap, 0xc4, 0x1);
+
+	return qcom_cc_really_probe(pdev, &lcc_mdm9615_desc, regmap);
+}
+
+static struct platform_driver lcc_mdm9615_driver = {
+	.probe		= lcc_mdm9615_probe,
+	.driver		= {
+		.name	= "lcc-mdm9615",
+		.of_match_table = lcc_mdm9615_match_table,
+	},
+};
+module_platform_driver(lcc_mdm9615_driver);
+
+MODULE_DESCRIPTION("QCOM LCC MDM9615 Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:lcc-mdm9615");
diff --git a/drivers/clk/qcom/mmcc-msm8996.c b/drivers/clk/qcom/mmcc-msm8996.c
index 847dd9dadeca..ca97e1151797 100644
--- a/drivers/clk/qcom/mmcc-msm8996.c
+++ b/drivers/clk/qcom/mmcc-msm8996.c
@@ -2888,6 +2888,14 @@ static struct clk_hw *mmcc_msm8996_hws[] = {
 	&gpll0_div.hw,
 };
 
+static struct gdsc mmagic_bimc_gdsc = {
+	.gdscr = 0x529c,
+	.pd = {
+		.name = "mmagic_bimc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
 static struct gdsc mmagic_video_gdsc = {
 	.gdscr = 0x119c,
 	.gds_hw_ctrl = 0x120c,
@@ -3201,6 +3209,7 @@ static struct clk_regmap *mmcc_msm8996_clocks[] = {
 };
 
 static struct gdsc *mmcc_msm8996_gdscs[] = {
+	[MMAGIC_BIMC_GDSC] = &mmagic_bimc_gdsc,
 	[MMAGIC_VIDEO_GDSC] = &mmagic_video_gdsc,
 	[MMAGIC_MDSS_GDSC] = &mmagic_mdss_gdsc,
 	[MMAGIC_CAMSS_GDSC] = &mmagic_camss_gdsc,
@@ -3305,9 +3314,8 @@ MODULE_DEVICE_TABLE(of, mmcc_msm8996_match_table);
 
 static int mmcc_msm8996_probe(struct platform_device *pdev)
 {
-	struct clk *clk;
 	struct device *dev = &pdev->dev;
-	int i;
+	int i, ret;
 	struct regmap *regmap;
 
 	regmap = qcom_cc_map(pdev, &mmcc_msm8996_desc);
@@ -3320,9 +3328,9 @@ static int mmcc_msm8996_probe(struct platform_device *pdev)
 	regmap_update_bits(regmap, 0x5054, BIT(15), 0);
 
 	for (i = 0; i < ARRAY_SIZE(mmcc_msm8996_hws); i++) {
-		clk = devm_clk_register(dev, mmcc_msm8996_hws[i]);
-		if (IS_ERR(clk))
-			return PTR_ERR(clk);
+		ret = devm_clk_hw_register(dev, mmcc_msm8996_hws[i]);
+		if (ret)
+			return ret;
 	}
 
 	return qcom_cc_really_probe(pdev, &mmcc_msm8996_desc, regmap);
diff --git a/drivers/clk/renesas/clk-mstp.c b/drivers/clk/renesas/clk-mstp.c
index 5093a250650d..9375777776d9 100644
--- a/drivers/clk/renesas/clk-mstp.c
+++ b/drivers/clk/renesas/clk-mstp.c
@@ -167,7 +167,7 @@ static void __init cpg_mstp_clocks_init(struct device_node *np)
 	unsigned int i;
 
 	group = kzalloc(sizeof(*group), GFP_KERNEL);
-	clks = kmalloc(MSTP_MAX_CLOCKS * sizeof(*clks), GFP_KERNEL);
+	clks = kmalloc_array(MSTP_MAX_CLOCKS, sizeof(*clks), GFP_KERNEL);
 	if (group == NULL || clks == NULL) {
 		kfree(group);
 		kfree(clks);
diff --git a/drivers/clk/renesas/clk-rz.c b/drivers/clk/renesas/clk-rz.c
index f6312c62f16b..5adb934326d1 100644
--- a/drivers/clk/renesas/clk-rz.c
+++ b/drivers/clk/renesas/clk-rz.c
@@ -25,10 +25,31 @@ struct rz_cpg {
 #define CPG_FRQCR	0x10
 #define CPG_FRQCR2	0x14
 
+#define PPR0		0xFCFE3200
+#define PIBC0		0xFCFE7000
+
+#define MD_CLK(x)	((x >> 2) & 1)	/* P0_2 */
+
 /* -----------------------------------------------------------------------------
  * Initialization
  */
 
+static u16 __init rz_cpg_read_mode_pins(void)
+{
+	void __iomem *ppr0, *pibc0;
+	u16 modes;
+
+	ppr0 = ioremap_nocache(PPR0, 2);
+	pibc0 = ioremap_nocache(PIBC0, 2);
+	BUG_ON(!ppr0 || !pibc0);
+	iowrite16(4, pibc0);	/* enable input buffer */
+	modes = ioread16(ppr0);
+	iounmap(ppr0);
+	iounmap(pibc0);
+
+	return modes;
+}
+
 static struct clk * __init
 rz_cpg_register_clock(struct device_node *np, struct rz_cpg *cpg, const char *name)
 {
@@ -37,8 +58,7 @@ rz_cpg_register_clock(struct device_node *np, struct rz_cpg *cpg, const char *na
 	static const unsigned frqcr_tab[4] = { 3, 2, 0, 1 };
 
 	if (strcmp(name, "pll") == 0) {
-		/* FIXME: cpg_mode should be read from GPIO. But no GPIO support yet */
-		unsigned cpg_mode = 0; /* hardcoded to EXTAL for now */
+		unsigned int cpg_mode = MD_CLK(rz_cpg_read_mode_pins());
 		const char *parent_name = of_clk_get_parent_name(np, cpg_mode);
 
 		mult = cpg_mode ? (32 / 4) : 30;
diff --git a/drivers/clk/renesas/r8a7795-cpg-mssr.c b/drivers/clk/renesas/r8a7795-cpg-mssr.c
index d359c92e13a6..f255e451e8ca 100644
--- a/drivers/clk/renesas/r8a7795-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7795-cpg-mssr.c
@@ -69,6 +69,7 @@ static const struct cpg_core_clk r8a7795_core_clks[] __initconst = {
 	DEF_FIXED(".s1",        CLK_S1,            CLK_PLL1_DIV2,  3, 1),
 	DEF_FIXED(".s2",        CLK_S2,            CLK_PLL1_DIV2,  4, 1),
 	DEF_FIXED(".s3",        CLK_S3,            CLK_PLL1_DIV2,  6, 1),
+	DEF_FIXED(".sdsrc",     CLK_SDSRC,         CLK_PLL1_DIV2,  2, 1),
 
 	/* Core Clock Outputs */
 	DEF_FIXED("ztr",        R8A7795_CLK_ZTR,   CLK_PLL1_DIV2,  6, 1),
@@ -87,10 +88,10 @@ static const struct cpg_core_clk r8a7795_core_clks[] __initconst = {
 	DEF_FIXED("s3d2",       R8A7795_CLK_S3D2,  CLK_S3,         2, 1),
 	DEF_FIXED("s3d4",       R8A7795_CLK_S3D4,  CLK_S3,         4, 1),
 
-	DEF_GEN3_SD("sd0",      R8A7795_CLK_SD0,   CLK_PLL1_DIV2, 0x0074),
-	DEF_GEN3_SD("sd1",      R8A7795_CLK_SD1,   CLK_PLL1_DIV2, 0x0078),
-	DEF_GEN3_SD("sd2",      R8A7795_CLK_SD2,   CLK_PLL1_DIV2, 0x0268),
-	DEF_GEN3_SD("sd3",      R8A7795_CLK_SD3,   CLK_PLL1_DIV2, 0x026c),
+	DEF_GEN3_SD("sd0",      R8A7795_CLK_SD0,   CLK_SDSRC,     0x0074),
+	DEF_GEN3_SD("sd1",      R8A7795_CLK_SD1,   CLK_SDSRC,     0x0078),
+	DEF_GEN3_SD("sd2",      R8A7795_CLK_SD2,   CLK_SDSRC,     0x0268),
+	DEF_GEN3_SD("sd3",      R8A7795_CLK_SD3,   CLK_SDSRC,     0x026c),
 
 	DEF_FIXED("cl",         R8A7795_CLK_CL,    CLK_PLL1_DIV2, 48, 1),
 	DEF_FIXED("cp",         R8A7795_CLK_CP,    CLK_EXTAL,      2, 1),
@@ -122,6 +123,10 @@ static const struct mssr_mod_clk r8a7795_mod_clks[] __initconst = {
 	DEF_MOD("sys-dmac2",		 217,	R8A7795_CLK_S3D1),
 	DEF_MOD("sys-dmac1",		 218,	R8A7795_CLK_S3D1),
 	DEF_MOD("sys-dmac0",		 219,	R8A7795_CLK_S3D1),
+	DEF_MOD("cmt3",			 300,	R8A7795_CLK_R),
+	DEF_MOD("cmt2",			 301,	R8A7795_CLK_R),
+	DEF_MOD("cmt1",			 302,	R8A7795_CLK_R),
+	DEF_MOD("cmt0",			 303,	R8A7795_CLK_R),
 	DEF_MOD("scif2",		 310,	R8A7795_CLK_S3D4),
 	DEF_MOD("sdif3",		 311,	R8A7795_CLK_SD3),
 	DEF_MOD("sdif2",		 312,	R8A7795_CLK_SD2),
diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c
index c84b549c14d2..eb347ed265f2 100644
--- a/drivers/clk/renesas/r8a7796-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c
@@ -45,6 +45,7 @@ enum clk_ids {
 	CLK_S3,
 	CLK_SDSRC,
 	CLK_SSPSRC,
+	CLK_RINT,
 
 	/* Module Clocks */
 	MOD_CLK_BASE
@@ -69,6 +70,7 @@ static const struct cpg_core_clk r8a7796_core_clks[] __initconst = {
 	DEF_FIXED(".s1",        CLK_S1,            CLK_PLL1_DIV2,  3, 1),
 	DEF_FIXED(".s2",        CLK_S2,            CLK_PLL1_DIV2,  4, 1),
 	DEF_FIXED(".s3",        CLK_S3,            CLK_PLL1_DIV2,  6, 1),
+	DEF_FIXED(".sdsrc",     CLK_SDSRC,         CLK_PLL1_DIV2,  2, 1),
 
 	/* Core Clock Outputs */
 	DEF_FIXED("ztr",        R8A7796_CLK_ZTR,   CLK_PLL1_DIV2,  6, 1),
@@ -92,13 +94,42 @@ static const struct cpg_core_clk r8a7796_core_clks[] __initconst = {
 	DEF_FIXED("s3d2",       R8A7796_CLK_S3D2,  CLK_S3,         2, 1),
 	DEF_FIXED("s3d4",       R8A7796_CLK_S3D4,  CLK_S3,         4, 1),
 
+	DEF_GEN3_SD("sd0",      R8A7796_CLK_SD0,   CLK_SDSRC,    0x0074),
+	DEF_GEN3_SD("sd1",      R8A7796_CLK_SD1,   CLK_SDSRC,    0x0078),
+	DEF_GEN3_SD("sd2",      R8A7796_CLK_SD2,   CLK_SDSRC,    0x0268),
+	DEF_GEN3_SD("sd3",      R8A7796_CLK_SD3,   CLK_SDSRC,    0x026c),
+
 	DEF_FIXED("cl",         R8A7796_CLK_CL,    CLK_PLL1_DIV2, 48, 1),
 	DEF_FIXED("cp",         R8A7796_CLK_CP,    CLK_EXTAL,      2, 1),
+
+	DEF_DIV6_RO("osc",      R8A7796_CLK_OSC,   CLK_EXTAL, CPG_RCKCR,  8),
+	DEF_DIV6_RO("r_int",    CLK_RINT,          CLK_EXTAL, CPG_RCKCR, 32),
+
+	DEF_BASE("r",           R8A7796_CLK_R,     CLK_TYPE_GEN3_R, CLK_RINT),
 };
 
 static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
+	DEF_MOD("cmt3",			 300,	R8A7796_CLK_R),
+	DEF_MOD("cmt2",			 301,	R8A7796_CLK_R),
+	DEF_MOD("cmt1",			 302,	R8A7796_CLK_R),
+	DEF_MOD("cmt0",			 303,	R8A7796_CLK_R),
 	DEF_MOD("scif2",		 310,	R8A7796_CLK_S3D4),
+	DEF_MOD("sdif3",		 311,	R8A7796_CLK_SD3),
+	DEF_MOD("sdif2",		 312,	R8A7796_CLK_SD2),
+	DEF_MOD("sdif1",		 313,	R8A7796_CLK_SD1),
+	DEF_MOD("sdif0",		 314,	R8A7796_CLK_SD0),
+	DEF_MOD("rwdt0",		 402,	R8A7796_CLK_R),
 	DEF_MOD("intc-ap",		 408,	R8A7796_CLK_S3D1),
+	DEF_MOD("thermal",		 522,	R8A7796_CLK_CP),
+	DEF_MOD("etheravb",		 812,	R8A7796_CLK_S0D6),
+	DEF_MOD("gpio7",		 905,	R8A7796_CLK_S3D4),
+	DEF_MOD("gpio6",		 906,	R8A7796_CLK_S3D4),
+	DEF_MOD("gpio5",		 907,	R8A7796_CLK_S3D4),
+	DEF_MOD("gpio4",		 908,	R8A7796_CLK_S3D4),
+	DEF_MOD("gpio3",		 909,	R8A7796_CLK_S3D4),
+	DEF_MOD("gpio2",		 910,	R8A7796_CLK_S3D4),
+	DEF_MOD("gpio1",		 911,	R8A7796_CLK_S3D4),
+	DEF_MOD("gpio0",		 912,	R8A7796_CLK_S3D4),
 };
 
 static const unsigned int r8a7796_crit_mod_clks[] __initconst = {
diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile
index f47a2fa962d2..b5f2c8ed12e1 100644
--- a/drivers/clk/rockchip/Makefile
+++ b/drivers/clk/rockchip/Makefile
@@ -8,6 +8,7 @@ obj-y	+= clk-pll.o
 obj-y	+= clk-cpu.o
 obj-y	+= clk-inverter.o
 obj-y	+= clk-mmc-phase.o
+obj-y	+= clk-ddr.o
 obj-$(CONFIG_RESET_CONTROLLER)	+= softrst.o
 
 obj-y	+= clk-rk3036.o
diff --git a/drivers/clk/rockchip/clk-ddr.c b/drivers/clk/rockchip/clk-ddr.c
new file mode 100644
index 000000000000..8feba93672c5
--- /dev/null
+++ b/drivers/clk/rockchip/clk-ddr.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2016 Rockchip Electronics Co. Ltd.
+ * Author: Lin Huang <hl@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/arm-smccc.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <soc/rockchip/rockchip_sip.h>
+#include "clk.h"
+
+struct rockchip_ddrclk {
+	struct clk_hw	hw;
+	void __iomem	*reg_base;
+	int		mux_offset;
+	int		mux_shift;
+	int		mux_width;
+	int		div_shift;
+	int		div_width;
+	int		ddr_flag;
+	spinlock_t	*lock;
+};
+
+#define to_rockchip_ddrclk_hw(hw) container_of(hw, struct rockchip_ddrclk, hw)
+
+static int rockchip_ddrclk_sip_set_rate(struct clk_hw *hw, unsigned long drate,
+					unsigned long prate)
+{
+	struct rockchip_ddrclk *ddrclk = to_rockchip_ddrclk_hw(hw);
+	unsigned long flags;
+	struct arm_smccc_res res;
+
+	spin_lock_irqsave(ddrclk->lock, flags);
+	arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, drate, 0,
+		      ROCKCHIP_SIP_CONFIG_DRAM_SET_RATE,
+		      0, 0, 0, 0, &res);
+	spin_unlock_irqrestore(ddrclk->lock, flags);
+
+	return res.a0;
+}
+
+static unsigned long
+rockchip_ddrclk_sip_recalc_rate(struct clk_hw *hw,
+				unsigned long parent_rate)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0,
+		      ROCKCHIP_SIP_CONFIG_DRAM_GET_RATE,
+		      0, 0, 0, 0, &res);
+
+	return res.a0;
+}
+
+static long rockchip_ddrclk_sip_round_rate(struct clk_hw *hw,
+					   unsigned long rate,
+					   unsigned long *prate)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, rate, 0,
+		      ROCKCHIP_SIP_CONFIG_DRAM_ROUND_RATE,
+		      0, 0, 0, 0, &res);
+
+	return res.a0;
+}
+
+static u8 rockchip_ddrclk_get_parent(struct clk_hw *hw)
+{
+	struct rockchip_ddrclk *ddrclk = to_rockchip_ddrclk_hw(hw);
+	int num_parents = clk_hw_get_num_parents(hw);
+	u32 val;
+
+	val = clk_readl(ddrclk->reg_base +
+			ddrclk->mux_offset) >> ddrclk->mux_shift;
+	val &= GENMASK(ddrclk->mux_width - 1, 0);
+
+	if (val >= num_parents)
+		return -EINVAL;
+
+	return val;
+}
+
+static const struct clk_ops rockchip_ddrclk_sip_ops = {
+	.recalc_rate = rockchip_ddrclk_sip_recalc_rate,
+	.set_rate = rockchip_ddrclk_sip_set_rate,
+	.round_rate = rockchip_ddrclk_sip_round_rate,
+	.get_parent = rockchip_ddrclk_get_parent,
+};
+
+struct clk *rockchip_clk_register_ddrclk(const char *name, int flags,
+					 const char *const *parent_names,
+					 u8 num_parents, int mux_offset,
+					 int mux_shift, int mux_width,
+					 int div_shift, int div_width,
+					 int ddr_flag, void __iomem *reg_base,
+					 spinlock_t *lock)
+{
+	struct rockchip_ddrclk *ddrclk;
+	struct clk_init_data init;
+	struct clk *clk;
+
+	ddrclk = kzalloc(sizeof(*ddrclk), GFP_KERNEL);
+	if (!ddrclk)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.parent_names = parent_names;
+	init.num_parents = num_parents;
+
+	init.flags = flags;
+	init.flags |= CLK_SET_RATE_NO_REPARENT;
+
+	switch (ddr_flag) {
+	case ROCKCHIP_DDRCLK_SIP:
+		init.ops = &rockchip_ddrclk_sip_ops;
+		break;
+	default:
+		pr_err("%s: unsupported ddrclk type %d\n", __func__, ddr_flag);
+		kfree(ddrclk);
+		return ERR_PTR(-EINVAL);
+	}
+
+	ddrclk->reg_base = reg_base;
+	ddrclk->lock = lock;
+	ddrclk->hw.init = &init;
+	ddrclk->mux_offset = mux_offset;
+	ddrclk->mux_shift = mux_shift;
+	ddrclk->mux_width = mux_width;
+	ddrclk->div_shift = div_shift;
+	ddrclk->div_width = div_width;
+	ddrclk->ddr_flag = ddr_flag;
+
+	clk = clk_register(NULL, &ddrclk->hw);
+	if (IS_ERR(clk)) {
+		pr_err("%s: could not register ddrclk %s\n", __func__,	name);
+		kfree(ddrclk);
+		return NULL;
+	}
+
+	return clk;
+}
diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c
index db81e454166b..9c1373e81683 100644
--- a/drivers/clk/rockchip/clk-pll.c
+++ b/drivers/clk/rockchip/clk-pll.c
@@ -837,7 +837,7 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx,
 		u8 num_parents, int con_offset, int grf_lock_offset,
 		int lock_shift, int mode_offset, int mode_shift,
 		struct rockchip_pll_rate_table *rate_table,
-		u8 clk_pll_flags)
+		unsigned long flags, u8 clk_pll_flags)
 {
 	const char *pll_parents[3];
 	struct clk_init_data init;
@@ -892,7 +892,7 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx,
 	init.name = pll_name;
 
 	/* keep all plls untouched for now */
-	init.flags = CLK_IGNORE_UNUSED;
+	init.flags = flags | CLK_IGNORE_UNUSED;
 
 	init.parent_names = &parent_names[0];
 	init.num_parents = 1;
diff --git a/drivers/clk/rockchip/clk-rk3399.c b/drivers/clk/rockchip/clk-rk3399.c
index c109d80e7a8a..8387c7a40bda 100644
--- a/drivers/clk/rockchip/clk-rk3399.c
+++ b/drivers/clk/rockchip/clk-rk3399.c
@@ -100,8 +100,10 @@ static struct rockchip_pll_rate_table rk3399_pll_rates[] = {
 	RK3036_PLL_RATE( 297000000, 1, 99, 4, 2, 1, 0),
 	RK3036_PLL_RATE( 216000000, 1, 72, 4, 2, 1, 0),
 	RK3036_PLL_RATE( 148500000, 1, 99, 4, 4, 1, 0),
+	RK3036_PLL_RATE( 106500000, 1, 71, 4, 4, 1, 0),
 	RK3036_PLL_RATE(  96000000, 1, 64, 4, 4, 1, 0),
 	RK3036_PLL_RATE(  74250000, 2, 99, 4, 4, 1, 0),
+	RK3036_PLL_RATE(  65000000, 1, 65, 6, 4, 1, 0),
 	RK3036_PLL_RATE(  54000000, 1, 54, 6, 4, 1, 0),
 	RK3036_PLL_RATE(  27000000, 1, 27, 6, 4, 1, 0),
 	{ /* sentinel */ },
@@ -118,6 +120,10 @@ PNAME(mux_armclkb_p)				= { "clk_core_b_lpll_src",
 						    "clk_core_b_bpll_src",
 						    "clk_core_b_dpll_src",
 						    "clk_core_b_gpll_src" };
+PNAME(mux_ddrclk_p)				= { "clk_ddrc_lpll_src",
+						    "clk_ddrc_bpll_src",
+						    "clk_ddrc_dpll_src",
+						    "clk_ddrc_gpll_src" };
 PNAME(mux_aclk_cci_p)				= { "cpll_aclk_cci_src",
 						    "gpll_aclk_cci_src",
 						    "npll_aclk_cci_src",
@@ -373,6 +379,7 @@ static struct rockchip_cpuclk_rate_table rk3399_cpuclkb_rates[] __initdata = {
 	RK3399_CPUCLKB_RATE(2184000000, 1, 11, 11),
 	RK3399_CPUCLKB_RATE(2088000000, 1, 10, 10),
 	RK3399_CPUCLKB_RATE(2040000000, 1, 10, 10),
+	RK3399_CPUCLKB_RATE(2016000000, 1, 9, 9),
 	RK3399_CPUCLKB_RATE(1992000000, 1, 9, 9),
 	RK3399_CPUCLKB_RATE(1896000000, 1, 9, 9),
 	RK3399_CPUCLKB_RATE(1800000000, 1, 8, 8),
@@ -578,7 +585,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	COMPOSITE(0, "clk_spdif_div", mux_pll_src_cpll_gpll_p, 0,
 			RK3399_CLKSEL_CON(32), 7, 1, MFLAGS, 0, 7, DFLAGS,
 			RK3399_CLKGATE_CON(8), 13, GFLAGS),
-	COMPOSITE_FRACMUX(0, "clk_spdif_frac", "clk_spdif_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "clk_spdif_frac", "clk_spdif_div", 0,
 			RK3399_CLKSEL_CON(99), 0,
 			RK3399_CLKGATE_CON(8), 14, GFLAGS,
 			&rk3399_spdif_fracmux),
@@ -592,7 +599,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	COMPOSITE(0, "clk_i2s0_div", mux_pll_src_cpll_gpll_p, 0,
 			RK3399_CLKSEL_CON(28), 7, 1, MFLAGS, 0, 7, DFLAGS,
 			RK3399_CLKGATE_CON(8), 3, GFLAGS),
-	COMPOSITE_FRACMUX(0, "clk_i2s0_frac", "clk_i2s0_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "clk_i2s0_frac", "clk_i2s0_div", 0,
 			RK3399_CLKSEL_CON(96), 0,
 			RK3399_CLKGATE_CON(8), 4, GFLAGS,
 			&rk3399_i2s0_fracmux),
@@ -602,7 +609,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	COMPOSITE(0, "clk_i2s1_div", mux_pll_src_cpll_gpll_p, 0,
 			RK3399_CLKSEL_CON(29), 7, 1, MFLAGS, 0, 7, DFLAGS,
 			RK3399_CLKGATE_CON(8), 6, GFLAGS),
-	COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_div", 0,
 			RK3399_CLKSEL_CON(97), 0,
 			RK3399_CLKGATE_CON(8), 7, GFLAGS,
 			&rk3399_i2s1_fracmux),
@@ -612,7 +619,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	COMPOSITE(0, "clk_i2s2_div", mux_pll_src_cpll_gpll_p, 0,
 			RK3399_CLKSEL_CON(30), 7, 1, MFLAGS, 0, 7, DFLAGS,
 			RK3399_CLKGATE_CON(8), 9, GFLAGS),
-	COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_div", 0,
 			RK3399_CLKSEL_CON(98), 0,
 			RK3399_CLKGATE_CON(8), 10, GFLAGS,
 			&rk3399_i2s2_fracmux),
@@ -631,7 +638,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "clk_uart0_div", "clk_uart0_src", 0,
 			RK3399_CLKSEL_CON(33), 0, 7, DFLAGS,
 			RK3399_CLKGATE_CON(9), 0, GFLAGS),
-	COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_div", 0,
 			RK3399_CLKSEL_CON(100), 0,
 			RK3399_CLKGATE_CON(9), 1, GFLAGS,
 			&rk3399_uart0_fracmux),
@@ -641,7 +648,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "clk_uart1_div", "clk_uart_src", 0,
 			RK3399_CLKSEL_CON(34), 0, 7, DFLAGS,
 			RK3399_CLKGATE_CON(9), 2, GFLAGS),
-	COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_div", 0,
 			RK3399_CLKSEL_CON(101), 0,
 			RK3399_CLKGATE_CON(9), 3, GFLAGS,
 			&rk3399_uart1_fracmux),
@@ -649,7 +656,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "clk_uart2_div", "clk_uart_src", 0,
 			RK3399_CLKSEL_CON(35), 0, 7, DFLAGS,
 			RK3399_CLKGATE_CON(9), 4, GFLAGS),
-	COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_div", 0,
 			RK3399_CLKSEL_CON(102), 0,
 			RK3399_CLKGATE_CON(9), 5, GFLAGS,
 			&rk3399_uart2_fracmux),
@@ -657,7 +664,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "clk_uart3_div", "clk_uart_src", 0,
 			RK3399_CLKSEL_CON(36), 0, 7, DFLAGS,
 			RK3399_CLKGATE_CON(9), 6, GFLAGS),
-	COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_div", 0,
 			RK3399_CLKSEL_CON(103), 0,
 			RK3399_CLKGATE_CON(9), 7, GFLAGS,
 			&rk3399_uart3_fracmux),
@@ -833,9 +840,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 
 	/* perihp */
 	GATE(0, "cpll_aclk_perihp_src", "cpll", CLK_IGNORE_UNUSED,
-			RK3399_CLKGATE_CON(5), 0, GFLAGS),
-	GATE(0, "gpll_aclk_perihp_src", "gpll", CLK_IGNORE_UNUSED,
 			RK3399_CLKGATE_CON(5), 1, GFLAGS),
+	GATE(0, "gpll_aclk_perihp_src", "gpll", CLK_IGNORE_UNUSED,
+			RK3399_CLKGATE_CON(5), 0, GFLAGS),
 	COMPOSITE(ACLK_PERIHP, "aclk_perihp", mux_aclk_perihp_p, CLK_IGNORE_UNUSED,
 			RK3399_CLKSEL_CON(14), 7, 1, MFLAGS, 0, 5, DFLAGS,
 			RK3399_CLKGATE_CON(5), 2, GFLAGS),
@@ -846,9 +853,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 			RK3399_CLKSEL_CON(14), 12, 2, DFLAGS,
 			RK3399_CLKGATE_CON(5), 4, GFLAGS),
 
-	GATE(ACLK_PERF_PCIE, "aclk_perf_pcie", "aclk_perihp", CLK_IGNORE_UNUSED,
+	GATE(ACLK_PERF_PCIE, "aclk_perf_pcie", "aclk_perihp", 0,
 			RK3399_CLKGATE_CON(20), 2, GFLAGS),
-	GATE(ACLK_PCIE, "aclk_pcie", "aclk_perihp", CLK_IGNORE_UNUSED,
+	GATE(ACLK_PCIE, "aclk_pcie", "aclk_perihp", 0,
 			RK3399_CLKGATE_CON(20), 10, GFLAGS),
 	GATE(0, "aclk_perihp_noc", "aclk_perihp", CLK_IGNORE_UNUSED,
 			RK3399_CLKGATE_CON(20), 12, GFLAGS),
@@ -923,9 +930,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 			RK3399_CLKGATE_CON(6), 14, GFLAGS),
 
 	GATE(0, "cpll_aclk_emmc_src", "cpll", CLK_IGNORE_UNUSED,
-			RK3399_CLKGATE_CON(6), 12, GFLAGS),
-	GATE(0, "gpll_aclk_emmc_src", "gpll", CLK_IGNORE_UNUSED,
 			RK3399_CLKGATE_CON(6), 13, GFLAGS),
+	GATE(0, "gpll_aclk_emmc_src", "gpll", CLK_IGNORE_UNUSED,
+			RK3399_CLKGATE_CON(6), 12, GFLAGS),
 	COMPOSITE_NOGATE(ACLK_EMMC, "aclk_emmc", mux_aclk_emmc_p, CLK_IGNORE_UNUSED,
 			RK3399_CLKSEL_CON(21), 7, 1, MFLAGS, 0, 5, DFLAGS),
 	GATE(ACLK_EMMC_CORE, "aclk_emmccore", "aclk_emmc", CLK_IGNORE_UNUSED,
@@ -1071,7 +1078,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	/* vio */
 	COMPOSITE(ACLK_VIO, "aclk_vio", mux_pll_src_cpll_gpll_ppll_p, CLK_IGNORE_UNUSED,
 			RK3399_CLKSEL_CON(42), 6, 2, MFLAGS, 0, 5, DFLAGS,
-			RK3399_CLKGATE_CON(11), 10, GFLAGS),
+			RK3399_CLKGATE_CON(11), 0, GFLAGS),
 	COMPOSITE_NOMUX(PCLK_VIO, "pclk_vio", "aclk_vio", 0,
 			RK3399_CLKSEL_CON(43), 0, 5, DFLAGS,
 			RK3399_CLKGATE_CON(11), 1, GFLAGS),
@@ -1161,7 +1168,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 			RK3399_CLKSEL_CON(49), 8, 2, MFLAGS, 0, 8, DFLAGS,
 			RK3399_CLKGATE_CON(10), 12, GFLAGS),
 
-	COMPOSITE_FRACMUX_NOGATE(0, "dclk_vop0_frac", "dclk_vop0_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX_NOGATE(DCLK_VOP0_FRAC, "dclk_vop0_frac", "dclk_vop0_div", 0,
 			RK3399_CLKSEL_CON(106), 0,
 			&rk3399_dclk_vop0_fracmux),
 
@@ -1191,7 +1198,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 			RK3399_CLKSEL_CON(50), 8, 2, MFLAGS, 0, 8, DFLAGS,
 			RK3399_CLKGATE_CON(10), 13, GFLAGS),
 
-	COMPOSITE_FRACMUX_NOGATE(0, "dclk_vop1_frac", "dclk_vop1_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX_NOGATE(DCLK_VOP1_FRAC, "dclk_vop1_frac", "dclk_vop1_div", 0,
 			RK3399_CLKSEL_CON(107), 0,
 			&rk3399_dclk_vop1_fracmux),
 
@@ -1305,7 +1312,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	/* testout */
 	MUX(0, "clk_test_pre", mux_pll_src_cpll_gpll_p, CLK_SET_RATE_PARENT,
 			RK3399_CLKSEL_CON(58), 7, 1, MFLAGS),
-	COMPOSITE_FRAC(0, "clk_test_frac", "clk_test_pre", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRAC(0, "clk_test_frac", "clk_test_pre", 0,
 			RK3399_CLKSEL_CON(105), 0,
 			RK3399_CLKGATE_CON(13), 9, GFLAGS),
 
@@ -1377,6 +1384,18 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "clk_test", "clk_test_pre", CLK_IGNORE_UNUSED,
 			RK3368_CLKSEL_CON(58), 0, 5, DFLAGS,
 			RK3368_CLKGATE_CON(13), 11, GFLAGS),
+
+	/* ddrc */
+	GATE(0, "clk_ddrc_lpll_src", "lpll", 0, RK3399_CLKGATE_CON(3),
+	     0, GFLAGS),
+	GATE(0, "clk_ddrc_bpll_src", "bpll", 0, RK3399_CLKGATE_CON(3),
+	     1, GFLAGS),
+	GATE(0, "clk_ddrc_dpll_src", "dpll", 0, RK3399_CLKGATE_CON(3),
+	     2, GFLAGS),
+	GATE(0, "clk_ddrc_gpll_src", "gpll", 0, RK3399_CLKGATE_CON(3),
+	     3, GFLAGS),
+	COMPOSITE_DDRCLK(SCLK_DDRC, "sclk_ddrc", mux_ddrclk_p, 0,
+		       RK3399_CLKSEL_CON(6), 4, 2, 0, 0, ROCKCHIP_DDRCLK_SIP),
 };
 
 static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = {
@@ -1398,7 +1417,7 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = {
 			RK3399_PMU_CLKSEL_CON(1), 13, 1, MFLAGS, 8, 5, DFLAGS,
 			RK3399_PMU_CLKGATE_CON(0), 8, GFLAGS),
 
-	COMPOSITE_FRACMUX_NOGATE(0, "clk_wifi_frac", "clk_wifi_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX_NOGATE(0, "clk_wifi_frac", "clk_wifi_div", 0,
 			RK3399_PMU_CLKSEL_CON(7), 0,
 			&rk3399_pmuclk_wifi_fracmux),
 
@@ -1426,7 +1445,7 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = {
 			RK3399_PMU_CLKSEL_CON(5), 10, 1, MFLAGS, 0, 7, DFLAGS,
 			RK3399_PMU_CLKGATE_CON(0), 5, GFLAGS),
 
-	COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_div", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_div", 0,
 			RK3399_PMU_CLKSEL_CON(6), 0,
 			RK3399_PMU_CLKGATE_CON(0), 6, GFLAGS,
 			&rk3399_uart4_pmu_fracmux),
@@ -1468,6 +1487,9 @@ static const char *const rk3399_cru_critical_clocks[] __initconst = {
 	"aclk_cci_pre",
 	"aclk_gic",
 	"aclk_gic_noc",
+	"aclk_hdcp_noc",
+	"hclk_hdcp_noc",
+	"pclk_hdcp_noc",
 	"pclk_perilp0",
 	"pclk_perilp0",
 	"hclk_perilp0",
@@ -1484,9 +1506,14 @@ static const char *const rk3399_cru_critical_clocks[] __initconst = {
 	"hclk_perilp1",
 	"hclk_perilp1_noc",
 	"aclk_dmac0_perilp",
+	"aclk_emmc_noc",
 	"gpll_hclk_perilp1_src",
 	"gpll_aclk_perilp0_src",
 	"gpll_aclk_perihp_src",
+	"aclk_vio_noc",
+
+	/* ddrc */
+	"sclk_ddrc"
 };
 
 static const char *const rk3399_pmucru_critical_clocks[] __initconst = {
diff --git a/drivers/clk/rockchip/clk-rockchip.c b/drivers/clk/rockchip/clk-rockchip.c
index 4cf838d52ef6..2c9bb81144c9 100644
--- a/drivers/clk/rockchip/clk-rockchip.c
+++ b/drivers/clk/rockchip/clk-rockchip.c
@@ -49,14 +49,19 @@ static void __init rk2928_gate_clk_init(struct device_node *node)
 	}
 
 	reg = of_iomap(node, 0);
+	if (!reg)
+		return;
 
 	clk_data = kzalloc(sizeof(struct clk_onecell_data), GFP_KERNEL);
-	if (!clk_data)
+	if (!clk_data) {
+		iounmap(reg);
 		return;
+	}
 
 	clk_data->clks = kzalloc(qty * sizeof(struct clk *), GFP_KERNEL);
 	if (!clk_data->clks) {
 		kfree(clk_data);
+		iounmap(reg);
 		return;
 	}
 
diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c
index 7ffd134995f2..b886be30f34f 100644
--- a/drivers/clk/rockchip/clk.c
+++ b/drivers/clk/rockchip/clk.c
@@ -385,7 +385,7 @@ void __init rockchip_clk_register_plls(struct rockchip_clk_provider *ctx,
 				list->con_offset, grf_lock_offset,
 				list->lock_shift, list->mode_offset,
 				list->mode_shift, list->rate_table,
-				list->pll_flags);
+				list->flags, list->pll_flags);
 		if (IS_ERR(clk)) {
 			pr_err("%s: failed to register clock %s\n", __func__,
 				list->name);
@@ -484,6 +484,15 @@ void __init rockchip_clk_register_branches(
 				list->gate_offset, list->gate_shift,
 				list->gate_flags, flags, &ctx->lock);
 			break;
+		case branch_ddrclk:
+			clk = rockchip_clk_register_ddrclk(
+				list->name, list->flags,
+				list->parent_names, list->num_parents,
+				list->muxdiv_offset, list->mux_shift,
+				list->mux_width, list->div_shift,
+				list->div_width, list->div_flags,
+				ctx->reg_base, &ctx->lock);
+			break;
 		}
 
 		/* none of the cases above matched */
diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h
index 2194ffa8c9fd..1653edd792a5 100644
--- a/drivers/clk/rockchip/clk.h
+++ b/drivers/clk/rockchip/clk.h
@@ -238,7 +238,7 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx,
 		u8 num_parents, int con_offset, int grf_lock_offset,
 		int lock_shift, int mode_offset, int mode_shift,
 		struct rockchip_pll_rate_table *rate_table,
-		u8 clk_pll_flags);
+		unsigned long flags, u8 clk_pll_flags);
 
 struct rockchip_cpuclk_clksel {
 	int reg;
@@ -281,6 +281,20 @@ struct clk *rockchip_clk_register_mmc(const char *name,
 				const char *const *parent_names, u8 num_parents,
 				void __iomem *reg, int shift);
 
+/*
+ * DDRCLK flags, including method of setting the rate
+ * ROCKCHIP_DDRCLK_SIP: use SIP call to bl31 to change ddrclk rate.
+ */
+#define ROCKCHIP_DDRCLK_SIP		BIT(0)
+
+struct clk *rockchip_clk_register_ddrclk(const char *name, int flags,
+					 const char *const *parent_names,
+					 u8 num_parents, int mux_offset,
+					 int mux_shift, int mux_width,
+					 int div_shift, int div_width,
+					 int ddr_flags, void __iomem *reg_base,
+					 spinlock_t *lock);
+
 #define ROCKCHIP_INVERTER_HIWORD_MASK	BIT(0)
 
 struct clk *rockchip_clk_register_inverter(const char *name,
@@ -299,6 +313,7 @@ enum rockchip_clk_branch_type {
 	branch_mmc,
 	branch_inverter,
 	branch_factor,
+	branch_ddrclk,
 };
 
 struct rockchip_clk_branch {
@@ -488,6 +503,24 @@ struct rockchip_clk_branch {
 		.child		= ch,				\
 	}
 
+#define COMPOSITE_DDRCLK(_id, cname, pnames, f, mo, ms, mw,	\
+			 ds, dw, df)				\
+	{							\
+		.id		= _id,				\
+		.branch_type	= branch_ddrclk,		\
+		.name		= cname,			\
+		.parent_names	= pnames,			\
+		.num_parents	= ARRAY_SIZE(pnames),		\
+		.flags		= f,				\
+		.muxdiv_offset  = mo,                           \
+		.mux_shift      = ms,                           \
+		.mux_width      = mw,                           \
+		.div_shift      = ds,                           \
+		.div_width      = dw,                           \
+		.div_flags	= df,				\
+		.gate_offset    = -1,                           \
+	}
+
 #define MUX(_id, cname, pnames, f, o, s, w, mf)			\
 	{							\
 		.id		= _id,				\
diff --git a/drivers/clk/samsung/clk-exynos-audss.c b/drivers/clk/samsung/clk-exynos-audss.c
index bdf8b971f332..51d152f735cc 100644
--- a/drivers/clk/samsung/clk-exynos-audss.c
+++ b/drivers/clk/samsung/clk-exynos-audss.c
@@ -14,18 +14,13 @@
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/syscore_ops.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 
 #include <dt-bindings/clock/exynos-audss-clk.h>
 
-enum exynos_audss_clk_type {
-	TYPE_EXYNOS4210,
-	TYPE_EXYNOS5250,
-	TYPE_EXYNOS5420,
-};
-
 static DEFINE_SPINLOCK(lock);
 static struct clk **clk_table;
 static void __iomem *reg_base;
@@ -44,9 +39,9 @@ static struct clk *epll;
 
 #ifdef CONFIG_PM_SLEEP
 static unsigned long reg_save[][2] = {
-	{ASS_CLK_SRC,  0},
-	{ASS_CLK_DIV,  0},
-	{ASS_CLK_GATE, 0},
+	{ ASS_CLK_SRC,  0 },
+	{ ASS_CLK_DIV,  0 },
+	{ ASS_CLK_GATE, 0 },
 };
 
 static int exynos_audss_clk_suspend(void)
@@ -73,14 +68,43 @@ static struct syscore_ops exynos_audss_clk_syscore_ops = {
 };
 #endif /* CONFIG_PM_SLEEP */
 
+struct exynos_audss_clk_drvdata {
+	unsigned int has_adma_clk:1;
+	unsigned int has_mst_clk:1;
+	unsigned int enable_epll:1;
+	unsigned int num_clks;
+};
+
+static const struct exynos_audss_clk_drvdata exynos4210_drvdata = {
+	.num_clks	= EXYNOS_AUDSS_MAX_CLKS - 1,
+};
+
+static const struct exynos_audss_clk_drvdata exynos5410_drvdata = {
+	.num_clks	= EXYNOS_AUDSS_MAX_CLKS - 1,
+	.has_mst_clk	= 1,
+};
+
+static const struct exynos_audss_clk_drvdata exynos5420_drvdata = {
+	.num_clks	= EXYNOS_AUDSS_MAX_CLKS,
+	.has_adma_clk	= 1,
+	.enable_epll	= 1,
+};
+
 static const struct of_device_id exynos_audss_clk_of_match[] = {
-	{ .compatible = "samsung,exynos4210-audss-clock",
-	  .data = (void *)TYPE_EXYNOS4210, },
-	{ .compatible = "samsung,exynos5250-audss-clock",
-	  .data = (void *)TYPE_EXYNOS5250, },
-	{ .compatible = "samsung,exynos5420-audss-clock",
-	  .data = (void *)TYPE_EXYNOS5420, },
-	{},
+	{
+		.compatible	= "samsung,exynos4210-audss-clock",
+		.data		= &exynos4210_drvdata,
+	}, {
+		.compatible	= "samsung,exynos5250-audss-clock",
+		.data		= &exynos4210_drvdata,
+	}, {
+		.compatible	= "samsung,exynos5410-audss-clock",
+		.data		= &exynos5410_drvdata,
+	}, {
+		.compatible	= "samsung,exynos5420-audss-clock",
+		.data		= &exynos5420_drvdata,
+	},
+	{ },
 };
 
 static void exynos_audss_clk_teardown(void)
@@ -106,19 +130,17 @@ static void exynos_audss_clk_teardown(void)
 /* register exynos_audss clocks */
 static int exynos_audss_clk_probe(struct platform_device *pdev)
 {
-	int i, ret = 0;
-	struct resource *res;
 	const char *mout_audss_p[] = {"fin_pll", "fout_epll"};
 	const char *mout_i2s_p[] = {"mout_audss", "cdclk0", "sclk_audio0"};
 	const char *sclk_pcm_p = "sclk_pcm0";
 	struct clk *pll_ref, *pll_in, *cdclk, *sclk_audio, *sclk_pcm_in;
-	const struct of_device_id *match;
-	enum exynos_audss_clk_type variant;
+	const struct exynos_audss_clk_drvdata *variant;
+	struct resource *res;
+	int i, ret = 0;
 
-	match = of_match_node(exynos_audss_clk_of_match, pdev->dev.of_node);
-	if (!match)
+	variant = of_device_get_match_data(&pdev->dev);
+	if (!variant)
 		return -EINVAL;
-	variant = (enum exynos_audss_clk_type)match->data;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	reg_base = devm_ioremap_resource(&pdev->dev, res);
@@ -126,7 +148,7 @@ static int exynos_audss_clk_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "failed to map audss registers\n");
 		return PTR_ERR(reg_base);
 	}
-	/* EPLL don't have to be enabled for boards other than Exynos5420 */
+
 	epll = ERR_PTR(-ENODEV);
 
 	clk_table = devm_kzalloc(&pdev->dev,
@@ -136,10 +158,7 @@ static int exynos_audss_clk_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	clk_data.clks = clk_table;
-	if (variant == TYPE_EXYNOS5420)
-		clk_data.clk_num = EXYNOS_AUDSS_MAX_CLKS;
-	else
-		clk_data.clk_num = EXYNOS_AUDSS_MAX_CLKS - 1;
+	clk_data.clk_num = variant->num_clks;
 
 	pll_ref = devm_clk_get(&pdev->dev, "pll_ref");
 	pll_in = devm_clk_get(&pdev->dev, "pll_in");
@@ -148,13 +167,13 @@ static int exynos_audss_clk_probe(struct platform_device *pdev)
 	if (!IS_ERR(pll_in)) {
 		mout_audss_p[1] = __clk_get_name(pll_in);
 
-		if (variant == TYPE_EXYNOS5420) {
+		if (variant->enable_epll) {
 			epll = pll_in;
 
 			ret = clk_prepare_enable(epll);
 			if (ret) {
 				dev_err(&pdev->dev,
-						"failed to prepare the epll clock\n");
+					"failed to prepare the epll clock\n");
 				return ret;
 			}
 		}
@@ -210,7 +229,7 @@ static int exynos_audss_clk_probe(struct platform_device *pdev)
 				sclk_pcm_p, CLK_SET_RATE_PARENT,
 				reg_base + ASS_CLK_GATE, 5, 0, &lock);
 
-	if (variant == TYPE_EXYNOS5420) {
+	if (variant->has_adma_clk) {
 		clk_table[EXYNOS_ADMA] = clk_register_gate(NULL, "adma",
 				"dout_srp", CLK_SET_RATE_PARENT,
 				reg_base + ASS_CLK_GATE, 9, 0, &lock);
@@ -234,9 +253,6 @@ static int exynos_audss_clk_probe(struct platform_device *pdev)
 #ifdef CONFIG_PM_SLEEP
 	register_syscore_ops(&exynos_audss_clk_syscore_ops);
 #endif
-
-	dev_info(&pdev->dev, "setup completed\n");
-
 	return 0;
 
 unregister:
diff --git a/drivers/clk/samsung/clk-exynos5260.c b/drivers/clk/samsung/clk-exynos5260.c
index a43642c36039..fd1d9bfc151b 100644
--- a/drivers/clk/samsung/clk-exynos5260.c
+++ b/drivers/clk/samsung/clk-exynos5260.c
@@ -131,21 +131,21 @@ static const struct samsung_gate_clock aud_gate_clks[] __initconst = {
 			EN_IP_AUD, 4, 0, 0),
 };
 
+static const struct samsung_cmu_info aud_cmu __initconst = {
+	.mux_clks	= aud_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(aud_mux_clks),
+	.div_clks	= aud_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(aud_div_clks),
+	.gate_clks	= aud_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(aud_gate_clks),
+	.nr_clk_ids	= AUD_NR_CLK,
+	.clk_regs	= aud_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(aud_clk_regs),
+};
+
 static void __init exynos5260_clk_aud_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.mux_clks = aud_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(aud_mux_clks);
-	cmu.div_clks = aud_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(aud_div_clks);
-	cmu.gate_clks = aud_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(aud_gate_clks);
-	cmu.nr_clk_ids = AUD_NR_CLK;
-	cmu.clk_regs = aud_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(aud_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &aud_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_aud, "samsung,exynos5260-clock-aud",
@@ -321,21 +321,21 @@ static const struct samsung_gate_clock disp_gate_clks[] __initconst = {
 			EN_IP_DISP, 25, 0, 0),
 };
 
+static const struct samsung_cmu_info disp_cmu __initconst = {
+	.mux_clks	= disp_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(disp_mux_clks),
+	.div_clks	= disp_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(disp_div_clks),
+	.gate_clks	= disp_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(disp_gate_clks),
+	.nr_clk_ids	= DISP_NR_CLK,
+	.clk_regs	= disp_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(disp_clk_regs),
+};
+
 static void __init exynos5260_clk_disp_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.mux_clks = disp_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(disp_mux_clks);
-	cmu.div_clks = disp_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(disp_div_clks);
-	cmu.gate_clks = disp_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(disp_gate_clks);
-	cmu.nr_clk_ids = DISP_NR_CLK;
-	cmu.clk_regs = disp_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(disp_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &disp_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_disp, "samsung,exynos5260-clock-disp",
@@ -385,21 +385,21 @@ static const struct samsung_pll_clock egl_pll_clks[] __initconst = {
 		pll2550_24mhz_tbl),
 };
 
+static const struct samsung_cmu_info egl_cmu __initconst = {
+	.pll_clks	= egl_pll_clks,
+	.nr_pll_clks	= ARRAY_SIZE(egl_pll_clks),
+	.mux_clks	= egl_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(egl_mux_clks),
+	.div_clks	= egl_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(egl_div_clks),
+	.nr_clk_ids	= EGL_NR_CLK,
+	.clk_regs	= egl_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(egl_clk_regs),
+};
+
 static void __init exynos5260_clk_egl_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.pll_clks = egl_pll_clks;
-	cmu.nr_pll_clks =  ARRAY_SIZE(egl_pll_clks);
-	cmu.mux_clks = egl_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(egl_mux_clks);
-	cmu.div_clks = egl_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(egl_div_clks);
-	cmu.nr_clk_ids = EGL_NR_CLK;
-	cmu.clk_regs = egl_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(egl_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &egl_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_egl, "samsung,exynos5260-clock-egl",
@@ -487,19 +487,19 @@ static const struct samsung_gate_clock fsys_gate_clks[] __initconst = {
 			EN_IP_FSYS_SECURE_SMMU_RTIC, 12, 0, 0),
 };
 
+static const struct samsung_cmu_info fsys_cmu __initconst = {
+	.mux_clks	= fsys_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(fsys_mux_clks),
+	.gate_clks	= fsys_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(fsys_gate_clks),
+	.nr_clk_ids	= FSYS_NR_CLK,
+	.clk_regs	= fsys_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(fsys_clk_regs),
+};
+
 static void __init exynos5260_clk_fsys_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.mux_clks = fsys_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(fsys_mux_clks);
-	cmu.gate_clks = fsys_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(fsys_gate_clks);
-	cmu.nr_clk_ids = FSYS_NR_CLK;
-	cmu.clk_regs = fsys_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(fsys_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &fsys_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_fsys, "samsung,exynos5260-clock-fsys",
@@ -576,21 +576,21 @@ static const struct samsung_gate_clock g2d_gate_clks[] __initconst = {
 			EN_IP_G2D_SECURE_SMMU_G2D, 15, 0, 0),
 };
 
+static const struct samsung_cmu_info g2d_cmu __initconst = {
+	.mux_clks	= g2d_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(g2d_mux_clks),
+	.div_clks	= g2d_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(g2d_div_clks),
+	.gate_clks	= g2d_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(g2d_gate_clks),
+	.nr_clk_ids	= G2D_NR_CLK,
+	.clk_regs	= g2d_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(g2d_clk_regs),
+};
+
 static void __init exynos5260_clk_g2d_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.mux_clks = g2d_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(g2d_mux_clks);
-	cmu.div_clks = g2d_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(g2d_div_clks);
-	cmu.gate_clks = g2d_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(g2d_gate_clks);
-	cmu.nr_clk_ids = G2D_NR_CLK;
-	cmu.clk_regs = g2d_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(g2d_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &g2d_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_g2d, "samsung,exynos5260-clock-g2d",
@@ -637,23 +637,23 @@ static const struct samsung_pll_clock g3d_pll_clks[] __initconst = {
 		pll2550_24mhz_tbl),
 };
 
+static const struct samsung_cmu_info g3d_cmu __initconst = {
+	.pll_clks	= g3d_pll_clks,
+	.nr_pll_clks	= ARRAY_SIZE(g3d_pll_clks),
+	.mux_clks	= g3d_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(g3d_mux_clks),
+	.div_clks	= g3d_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(g3d_div_clks),
+	.gate_clks	= g3d_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(g3d_gate_clks),
+	.nr_clk_ids	= G3D_NR_CLK,
+	.clk_regs	= g3d_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(g3d_clk_regs),
+};
+
 static void __init exynos5260_clk_g3d_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.pll_clks = g3d_pll_clks;
-	cmu.nr_pll_clks =  ARRAY_SIZE(g3d_pll_clks);
-	cmu.mux_clks = g3d_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(g3d_mux_clks);
-	cmu.div_clks = g3d_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(g3d_div_clks);
-	cmu.gate_clks = g3d_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(g3d_gate_clks);
-	cmu.nr_clk_ids = G3D_NR_CLK;
-	cmu.clk_regs = g3d_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(g3d_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &g3d_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_g3d, "samsung,exynos5260-clock-g3d",
@@ -772,21 +772,21 @@ static const struct samsung_gate_clock gscl_gate_clks[] __initconst = {
 			EN_IP_GSCL_SECURE_SMMU_MSCL1, 20, 0, 0),
 };
 
+static const struct samsung_cmu_info gscl_cmu __initconst = {
+	.mux_clks	= gscl_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(gscl_mux_clks),
+	.div_clks	= gscl_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(gscl_div_clks),
+	.gate_clks	= gscl_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(gscl_gate_clks),
+	.nr_clk_ids	= GSCL_NR_CLK,
+	.clk_regs	= gscl_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(gscl_clk_regs),
+};
+
 static void __init exynos5260_clk_gscl_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.mux_clks = gscl_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(gscl_mux_clks);
-	cmu.div_clks = gscl_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(gscl_div_clks);
-	cmu.gate_clks = gscl_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(gscl_gate_clks);
-	cmu.nr_clk_ids = GSCL_NR_CLK;
-	cmu.clk_regs = gscl_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(gscl_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &gscl_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_gscl, "samsung,exynos5260-clock-gscl",
@@ -891,21 +891,21 @@ static const struct samsung_gate_clock isp_gate_clks[] __initconst = {
 			EN_SCLK_ISP, 9, CLK_SET_RATE_PARENT, 0),
 };
 
+static const struct samsung_cmu_info isp_cmu __initconst = {
+	.mux_clks	= isp_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(isp_mux_clks),
+	.div_clks	= isp_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(isp_div_clks),
+	.gate_clks	= isp_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(isp_gate_clks),
+	.nr_clk_ids	= ISP_NR_CLK,
+	.clk_regs	= isp_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(isp_clk_regs),
+};
+
 static void __init exynos5260_clk_isp_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.mux_clks = isp_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(isp_mux_clks);
-	cmu.div_clks = isp_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(isp_div_clks);
-	cmu.gate_clks = isp_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(isp_gate_clks);
-	cmu.nr_clk_ids = ISP_NR_CLK;
-	cmu.clk_regs = isp_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(isp_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &isp_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_isp, "samsung,exynos5260-clock-isp",
@@ -955,21 +955,21 @@ static const struct samsung_pll_clock kfc_pll_clks[] __initconst = {
 		pll2550_24mhz_tbl),
 };
 
+static const struct samsung_cmu_info kfc_cmu __initconst = {
+	.pll_clks	= kfc_pll_clks,
+	.nr_pll_clks	= ARRAY_SIZE(kfc_pll_clks),
+	.mux_clks	= kfc_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(kfc_mux_clks),
+	.div_clks	= kfc_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(kfc_div_clks),
+	.nr_clk_ids	= KFC_NR_CLK,
+	.clk_regs	= kfc_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(kfc_clk_regs),
+};
+
 static void __init exynos5260_clk_kfc_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.pll_clks = kfc_pll_clks;
-	cmu.nr_pll_clks =  ARRAY_SIZE(kfc_pll_clks);
-	cmu.mux_clks = kfc_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(kfc_mux_clks);
-	cmu.div_clks = kfc_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(kfc_div_clks);
-	cmu.nr_clk_ids = KFC_NR_CLK;
-	cmu.clk_regs = kfc_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(kfc_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &kfc_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_kfc, "samsung,exynos5260-clock-kfc",
@@ -1011,21 +1011,21 @@ static const struct samsung_gate_clock mfc_gate_clks[] __initconst = {
 			EN_IP_MFC_SECURE_SMMU2_MFC, 7, 0, 0),
 };
 
+static const struct samsung_cmu_info mfc_cmu __initconst = {
+	.mux_clks	= mfc_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(mfc_mux_clks),
+	.div_clks	= mfc_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(mfc_div_clks),
+	.gate_clks	= mfc_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(mfc_gate_clks),
+	.nr_clk_ids	= MFC_NR_CLK,
+	.clk_regs	= mfc_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(mfc_clk_regs),
+};
+
 static void __init exynos5260_clk_mfc_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.mux_clks = mfc_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(mfc_mux_clks);
-	cmu.div_clks = mfc_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(mfc_div_clks);
-	cmu.gate_clks = mfc_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(mfc_gate_clks);
-	cmu.nr_clk_ids = MFC_NR_CLK;
-	cmu.clk_regs = mfc_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(mfc_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &mfc_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_mfc, "samsung,exynos5260-clock-mfc",
@@ -1158,23 +1158,23 @@ static const struct samsung_pll_clock mif_pll_clks[] __initconst = {
 		pll2550_24mhz_tbl),
 };
 
+static const struct samsung_cmu_info mif_cmu __initconst = {
+	.pll_clks	= mif_pll_clks,
+	.nr_pll_clks	= ARRAY_SIZE(mif_pll_clks),
+	.mux_clks	= mif_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(mif_mux_clks),
+	.div_clks	= mif_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(mif_div_clks),
+	.gate_clks	= mif_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(mif_gate_clks),
+	.nr_clk_ids	= MIF_NR_CLK,
+	.clk_regs	= mif_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(mif_clk_regs),
+};
+
 static void __init exynos5260_clk_mif_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.pll_clks = mif_pll_clks;
-	cmu.nr_pll_clks =  ARRAY_SIZE(mif_pll_clks);
-	cmu.mux_clks = mif_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(mif_mux_clks);
-	cmu.div_clks = mif_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(mif_div_clks);
-	cmu.gate_clks = mif_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(mif_gate_clks);
-	cmu.nr_clk_ids = MIF_NR_CLK;
-	cmu.clk_regs = mif_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(mif_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &mif_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_mif, "samsung,exynos5260-clock-mif",
@@ -1366,21 +1366,21 @@ static const struct samsung_gate_clock peri_gate_clks[] __initconst = {
 		EN_IP_PERI_SECURE_TZPC, 20, 0, 0),
 };
 
+static const struct samsung_cmu_info peri_cmu __initconst = {
+	.mux_clks	= peri_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(peri_mux_clks),
+	.div_clks	= peri_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(peri_div_clks),
+	.gate_clks	= peri_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(peri_gate_clks),
+	.nr_clk_ids	= PERI_NR_CLK,
+	.clk_regs	= peri_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(peri_clk_regs),
+};
+
 static void __init exynos5260_clk_peri_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.mux_clks = peri_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(peri_mux_clks);
-	cmu.div_clks = peri_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(peri_div_clks);
-	cmu.gate_clks = peri_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(peri_gate_clks);
-	cmu.nr_clk_ids = PERI_NR_CLK;
-	cmu.clk_regs = peri_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(peri_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &peri_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_peri, "samsung,exynos5260-clock-peri",
@@ -1818,25 +1818,25 @@ static const struct samsung_pll_clock top_pll_clks[] __initconst = {
 		pll2650_24mhz_tbl),
 };
 
+static const struct samsung_cmu_info top_cmu __initconst = {
+	.pll_clks	= top_pll_clks,
+	.nr_pll_clks	= ARRAY_SIZE(top_pll_clks),
+	.mux_clks	= top_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(top_mux_clks),
+	.div_clks	= top_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(top_div_clks),
+	.gate_clks	= top_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(top_gate_clks),
+	.fixed_clks	= fixed_rate_clks,
+	.nr_fixed_clks	= ARRAY_SIZE(fixed_rate_clks),
+	.nr_clk_ids	= TOP_NR_CLK,
+	.clk_regs	= top_clk_regs,
+	.nr_clk_regs	= ARRAY_SIZE(top_clk_regs),
+};
+
 static void __init exynos5260_clk_top_init(struct device_node *np)
 {
-	struct samsung_cmu_info cmu = { NULL };
-
-	cmu.pll_clks = top_pll_clks;
-	cmu.nr_pll_clks =  ARRAY_SIZE(top_pll_clks);
-	cmu.mux_clks = top_mux_clks;
-	cmu.nr_mux_clks = ARRAY_SIZE(top_mux_clks);
-	cmu.div_clks = top_div_clks;
-	cmu.nr_div_clks = ARRAY_SIZE(top_div_clks);
-	cmu.gate_clks = top_gate_clks;
-	cmu.nr_gate_clks = ARRAY_SIZE(top_gate_clks);
-	cmu.fixed_clks = fixed_rate_clks;
-	cmu.nr_fixed_clks = ARRAY_SIZE(fixed_rate_clks);
-	cmu.nr_clk_ids = TOP_NR_CLK;
-	cmu.clk_regs = top_clk_regs;
-	cmu.nr_clk_regs = ARRAY_SIZE(top_clk_regs);
-
-	samsung_cmu_register_one(np, &cmu);
+	samsung_cmu_register_one(np, &top_cmu);
 }
 
 CLK_OF_DECLARE(exynos5260_clk_top, "samsung,exynos5260-clock-top",
diff --git a/drivers/clk/samsung/clk-exynos5410.c b/drivers/clk/samsung/clk-exynos5410.c
index 54ec486a5e45..fc471a49e8f4 100644
--- a/drivers/clk/samsung/clk-exynos5410.c
+++ b/drivers/clk/samsung/clk-exynos5410.c
@@ -14,6 +14,7 @@
 #include <linux/clk-provider.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/clk.h>
 
 #include "clk.h"
 
@@ -21,6 +22,8 @@
 #define APLL_CON0               0x100
 #define CPLL_LOCK               0x10020
 #define CPLL_CON0               0x10120
+#define EPLL_LOCK               0x10040
+#define EPLL_CON0               0x10130
 #define MPLL_LOCK               0x4000
 #define MPLL_CON0               0x4100
 #define BPLL_LOCK               0x20010
@@ -58,7 +61,7 @@
 
 /* list of PLLs */
 enum exynos5410_plls {
-	apll, cpll, mpll,
+	apll, cpll, epll, mpll,
 	bpll, kpll,
 	nr_plls                 /* number of PLLs */
 };
@@ -67,6 +70,7 @@ enum exynos5410_plls {
 PNAME(apll_p)		= { "fin_pll", "fout_apll", };
 PNAME(bpll_p)		= { "fin_pll", "fout_bpll", };
 PNAME(cpll_p)		= { "fin_pll", "fout_cpll" };
+PNAME(epll_p)		= { "fin_pll", "fout_epll" };
 PNAME(mpll_p)		= { "fin_pll", "fout_mpll", };
 PNAME(kpll_p)		= { "fin_pll", "fout_kpll", };
 
@@ -95,6 +99,8 @@ static const struct samsung_mux_clock exynos5410_mux_clks[] __initconst = {
 	MUX(0, "sclk_bpll", bpll_p, SRC_CDREX, 0, 1),
 	MUX(0, "sclk_bpll_muxed", bpll_user_p, SRC_TOP2, 24, 1),
 
+	MUX(0, "sclk_epll", epll_p, SRC_TOP2, 12, 1),
+
 	MUX(0, "sclk_cpll", cpll_p, SRC_TOP2, 8, 1),
 
 	MUX(0, "sclk_mpll_bpll", mpll_bpll_p, SRC_TOP1, 20, 1),
@@ -176,6 +182,8 @@ static const struct samsung_gate_clock exynos5410_gate_clks[] __initconst = {
 	GATE(CLK_MMC0, "sdmmc0", "aclk200", GATE_BUS_FSYS0, 12, 0, 0),
 	GATE(CLK_MMC1, "sdmmc1", "aclk200", GATE_BUS_FSYS0, 13, 0, 0),
 	GATE(CLK_MMC2, "sdmmc2", "aclk200", GATE_BUS_FSYS0, 14, 0, 0),
+	GATE(CLK_PDMA1, "pdma1", "aclk200", GATE_BUS_FSYS0, 2, 0, 0),
+	GATE(CLK_PDMA0, "pdma0", "aclk200", GATE_BUS_FSYS0, 1, 0, 0),
 
 	GATE(CLK_SCLK_USBPHY301, "sclk_usbphy301", "dout_usbphy301",
 	     GATE_TOP_SCLK_FSYS, 7, CLK_SET_RATE_PARENT, 0),
@@ -217,11 +225,26 @@ static const struct samsung_gate_clock exynos5410_gate_clks[] __initconst = {
 	GATE(CLK_USBD301, "usbd301", "aclk200_fsys", GATE_IP_FSYS, 20, 0, 0),
 };
 
-static const struct samsung_pll_clock exynos5410_plls[nr_plls] __initconst = {
+static const struct samsung_pll_rate_table exynos5410_pll2550x_24mhz_tbl[] __initconst = {
+	PLL_36XX_RATE(400000000U, 200, 3, 2, 0),
+	PLL_36XX_RATE(333000000U, 111, 2, 2, 0),
+	PLL_36XX_RATE(300000000U, 100, 2, 2, 0),
+	PLL_36XX_RATE(266000000U, 266, 3, 3, 0),
+	PLL_36XX_RATE(200000000U, 200, 3, 3, 0),
+	PLL_36XX_RATE(192000000U, 192, 3, 3, 0),
+	PLL_36XX_RATE(166000000U, 166, 3, 3, 0),
+	PLL_36XX_RATE(133000000U, 266, 3, 4, 0),
+	PLL_36XX_RATE(100000000U, 200, 3, 4, 0),
+	PLL_36XX_RATE(66000000U,  176, 2, 5, 0),
+};
+
+static struct samsung_pll_clock exynos5410_plls[nr_plls] __initdata = {
 	[apll] = PLL(pll_35xx, CLK_FOUT_APLL, "fout_apll", "fin_pll", APLL_LOCK,
 		APLL_CON0, NULL),
 	[cpll] = PLL(pll_35xx, CLK_FOUT_CPLL, "fout_cpll", "fin_pll", CPLL_LOCK,
 		CPLL_CON0, NULL),
+	[epll] = PLL(pll_2650x, CLK_FOUT_EPLL, "fout_epll", "fin_pll", EPLL_LOCK,
+		EPLL_CON0, NULL),
 	[mpll] = PLL(pll_35xx, CLK_FOUT_MPLL, "fout_mpll", "fin_pll", MPLL_LOCK,
 		MPLL_CON0, NULL),
 	[bpll] = PLL(pll_35xx, CLK_FOUT_BPLL, "fout_bpll", "fin_pll", BPLL_LOCK,
@@ -230,29 +253,27 @@ static const struct samsung_pll_clock exynos5410_plls[nr_plls] __initconst = {
 		KPLL_CON0, NULL),
 };
 
+static const struct samsung_cmu_info cmu __initconst = {
+	.pll_clks	= exynos5410_plls,
+	.nr_pll_clks	= ARRAY_SIZE(exynos5410_plls),
+	.mux_clks	= exynos5410_mux_clks,
+	.nr_mux_clks	= ARRAY_SIZE(exynos5410_mux_clks),
+	.div_clks	= exynos5410_div_clks,
+	.nr_div_clks	= ARRAY_SIZE(exynos5410_div_clks),
+	.gate_clks	= exynos5410_gate_clks,
+	.nr_gate_clks	= ARRAY_SIZE(exynos5410_gate_clks),
+	.nr_clk_ids	= CLK_NR_CLKS,
+};
+
 /* register exynos5410 clocks */
 static void __init exynos5410_clk_init(struct device_node *np)
 {
-	struct samsung_clk_provider *ctx;
-	void __iomem *reg_base;
-
-	reg_base = of_iomap(np, 0);
-	if (!reg_base)
-		panic("%s: failed to map registers\n", __func__);
-
-	ctx = samsung_clk_init(np, reg_base, CLK_NR_CLKS);
-
-	samsung_clk_register_pll(ctx, exynos5410_plls,
-			ARRAY_SIZE(exynos5410_plls), reg_base);
+	struct clk *xxti = of_clk_get(np, 0);
 
-	samsung_clk_register_mux(ctx, exynos5410_mux_clks,
-			ARRAY_SIZE(exynos5410_mux_clks));
-	samsung_clk_register_div(ctx, exynos5410_div_clks,
-			ARRAY_SIZE(exynos5410_div_clks));
-	samsung_clk_register_gate(ctx, exynos5410_gate_clks,
-			ARRAY_SIZE(exynos5410_gate_clks));
+	if (!IS_ERR(xxti) && clk_get_rate(xxti) == 24 * MHZ)
+		exynos5410_plls[epll].rate_table = exynos5410_pll2550x_24mhz_tbl;
 
-	samsung_clk_of_add_provider(np, ctx);
+	samsung_cmu_register_one(np, &cmu);
 
 	pr_debug("Exynos5410: clock setup completed.\n");
 }
diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c
index bb196ca21a77..8c8b495cbf0d 100644
--- a/drivers/clk/samsung/clk-exynos5420.c
+++ b/drivers/clk/samsung/clk-exynos5420.c
@@ -131,6 +131,9 @@
 #define TOP_SPARE2		0x10b08
 #define BPLL_LOCK		0x20010
 #define BPLL_CON0		0x20110
+#define SRC_CDREX		0x20200
+#define DIV_CDREX0		0x20500
+#define DIV_CDREX1		0x20504
 #define KPLL_LOCK		0x28000
 #define KPLL_CON0		0x28100
 #define SRC_KFC			0x28200
@@ -244,6 +247,9 @@ static const unsigned long exynos5x_clk_regs[] __initconst = {
 	GATE_TOP_SCLK_FSYS,
 	GATE_TOP_SCLK_PERIC,
 	TOP_SPARE2,
+	SRC_CDREX,
+	DIV_CDREX0,
+	DIV_CDREX1,
 	SRC_KFC,
 	DIV_KFC0,
 };
@@ -448,6 +454,8 @@ PNAME(mout_maudio0_p) = {"fin_pll", "maudio_clk", "mout_sclk_dpll",
 			 "mout_sclk_epll", "mout_sclk_rpll"};
 PNAME(mout_mau_epll_clk_p) = {"mout_sclk_epll", "mout_sclk_dpll",
 				"mout_sclk_mpll", "mout_sclk_spll"};
+PNAME(mout_mclk_cdrex_p) = {"mout_bpll", "mout_mx_mspll_ccore"};
+
 /* List of parents specific to exynos5800 */
 PNAME(mout_epll2_5800_p)	= { "mout_sclk_epll", "ff_dout_epll2" };
 PNAME(mout_group1_5800_p)	= { "mout_sclk_cpll", "mout_sclk_dpll",
@@ -465,6 +473,9 @@ PNAME(mout_group6_5800_p)	= { "mout_sclk_ipll", "mout_sclk_dpll",
 PNAME(mout_group7_5800_p)	= { "mout_sclk_cpll", "mout_sclk_dpll",
 					"mout_sclk_mpll", "mout_sclk_spll",
 					"mout_epll2", "mout_sclk_ipll" };
+PNAME(mout_mx_mspll_ccore_p)	= {"sclk_bpll", "mout_sclk_dpll",
+					"mout_sclk_mpll", "ff_dout_spll2",
+					"mout_sclk_spll", "mout_sclk_epll"};
 PNAME(mout_mau_epll_clk_5800_p)	= { "mout_sclk_epll", "mout_sclk_dpll",
 					"mout_sclk_mpll",
 					"ff_dout_spll2" };
@@ -523,6 +534,8 @@ static const struct samsung_mux_clock exynos5800_mux_clks[] __initconst = {
 	MUX(0, "mout_aclk300_disp1", mout_group5_5800_p, SRC_TOP2, 24, 2),
 	MUX(0, "mout_aclk300_gscl", mout_group5_5800_p, SRC_TOP2, 28, 2),
 
+	MUX(CLK_MOUT_MX_MSPLL_CCORE, "mout_mx_mspll_ccore",
+			mout_mx_mspll_ccore_p, SRC_TOP7, 16, 2),
 	MUX(0, "mout_mau_epll_clk", mout_mau_epll_clk_5800_p, SRC_TOP7,
 			20, 2),
 	MUX(0, "sclk_bpll", mout_bpll_p, SRC_TOP7, 24, 1),
@@ -601,6 +614,8 @@ static const struct samsung_mux_clock exynos5420_mux_clks[] __initconst = {
 	MUX(0, "mout_aclk300_disp1", mout_group1_p, SRC_TOP2, 24, 2),
 	MUX(0, "mout_aclk300_gscl", mout_group1_p, SRC_TOP2, 28, 2),
 
+	MUX(CLK_MOUT_MX_MSPLL_CCORE, "mout_mx_mspll_ccore",
+			mout_group5_5800_p, SRC_TOP7, 16, 2),
 	MUX(0, "mout_mau_epll_clk", mout_mau_epll_clk_p, SRC_TOP7, 20, 2),
 
 	MUX(0, "mout_fimd1", mout_group3_p, SRC_DISP10, 4, 1),
@@ -744,6 +759,12 @@ static const struct samsung_mux_clock exynos5x_mux_clks[] __initconst = {
 
 	MUX(0, "mout_fimd1_final", mout_fimd1_final_p, TOP_SPARE2, 8, 1),
 
+	/* CDREX block */
+	MUX_F(CLK_MOUT_MCLK_CDREX, "mout_mclk_cdrex", mout_mclk_cdrex_p,
+			SRC_CDREX, 4, 1, CLK_SET_RATE_PARENT, 0),
+	MUX_F(CLK_MOUT_BPLL, "mout_bpll", mout_bpll_p, SRC_CDREX, 0, 1,
+			CLK_SET_RATE_PARENT, 0),
+
 	/* MAU Block */
 	MUX(CLK_MOUT_MAUDIO0, "mout_maudio0", mout_maudio0_p, SRC_MAU, 28, 3),
 
@@ -836,6 +857,21 @@ static const struct samsung_div_clock exynos5x_div_clks[] __initconst = {
 	DIV(CLK_DOUT_ACLK400_DISP1, "dout_aclk400_disp1",
 			"mout_aclk400_disp1", DIV_TOP2, 4, 3),
 
+	/* CDREX Block */
+	DIV(CLK_DOUT_PCLK_CDREX, "dout_pclk_cdrex", "dout_aclk_cdrex1",
+			DIV_CDREX0, 28, 3),
+	DIV_F(CLK_DOUT_SCLK_CDREX, "dout_sclk_cdrex", "mout_mclk_cdrex",
+			DIV_CDREX0, 24, 3, CLK_SET_RATE_PARENT, 0),
+	DIV(CLK_DOUT_ACLK_CDREX1, "dout_aclk_cdrex1", "dout_clk2x_phy0",
+			DIV_CDREX0, 16, 3),
+	DIV(CLK_DOUT_CCLK_DREX0, "dout_cclk_drex0", "dout_clk2x_phy0",
+			DIV_CDREX0, 8, 3),
+	DIV(CLK_DOUT_CLK2X_PHY0, "dout_clk2x_phy0", "dout_sclk_cdrex",
+			DIV_CDREX0, 3, 5),
+
+	DIV(CLK_DOUT_PCLK_CORE_MEM, "dout_pclk_core_mem", "mout_mclk_cdrex",
+			DIV_CDREX1, 8, 3),
+
 	/* Audio Block */
 	DIV(0, "dout_maudio0", "mout_maudio0", DIV_MAU, 20, 4),
 	DIV(0, "dout_maupcm0", "dout_maudio0", DIV_MAU, 24, 8),
@@ -1364,6 +1400,7 @@ static void __init exynos5x_clk_init(struct device_node *np,
 	if (_get_rate("fin_pll") == 24 * MHZ) {
 		exynos5x_plls[apll].rate_table = exynos5420_pll2550x_24mhz_tbl;
 		exynos5x_plls[kpll].rate_table = exynos5420_pll2550x_24mhz_tbl;
+		exynos5x_plls[bpll].rate_table = exynos5420_pll2550x_24mhz_tbl;
 	}
 
 	samsung_clk_register_pll(ctx, exynos5x_plls, ARRAY_SIZE(exynos5x_plls),
diff --git a/drivers/clk/samsung/clk-exynos5440.c b/drivers/clk/samsung/clk-exynos5440.c
index a57d01b99b76..a80f3ef20801 100644
--- a/drivers/clk/samsung/clk-exynos5440.c
+++ b/drivers/clk/samsung/clk-exynos5440.c
@@ -112,6 +112,11 @@ static struct notifier_block exynos5440_clk_restart_handler = {
 	.priority = 128,
 };
 
+static const struct samsung_pll_clock exynos5440_plls[] __initconst = {
+	PLL(pll_2550x, CLK_CPLLA, "cplla", "xtal", 0, 0x4c, NULL),
+	PLL(pll_2550x, CLK_CPLLB, "cpllb", "xtal", 0, 0x50, NULL),
+};
+
 /* register exynos5440 clocks */
 static void __init exynos5440_clk_init(struct device_node *np)
 {
@@ -129,8 +134,8 @@ static void __init exynos5440_clk_init(struct device_node *np)
 	samsung_clk_of_register_fixed_ext(ctx, exynos5440_fixed_rate_ext_clks,
 		ARRAY_SIZE(exynos5440_fixed_rate_ext_clks), ext_clk_match);
 
-	samsung_clk_register_pll2550x("cplla", "xtal", reg_base + 0x1c, 0x10);
-	samsung_clk_register_pll2550x("cpllb", "xtal", reg_base + 0x20, 0x10);
+	samsung_clk_register_pll(ctx, exynos5440_plls,
+			ARRAY_SIZE(exynos5440_plls), ctx->reg_base);
 
 	samsung_clk_register_fixed_rate(ctx, exynos5440_fixed_rate_clks,
 			ARRAY_SIZE(exynos5440_fixed_rate_clks));
diff --git a/drivers/clk/samsung/clk-pll.c b/drivers/clk/samsung/clk-pll.c
index 48139bd510f1..9617825daabb 100644
--- a/drivers/clk/samsung/clk-pll.c
+++ b/drivers/clk/samsung/clk-pll.c
@@ -890,22 +890,14 @@ static const struct clk_ops samsung_s3c2440_mpll_clk_ops = {
 #define PLL2550X_M_SHIFT      (4)
 #define PLL2550X_S_SHIFT      (0)
 
-struct samsung_clk_pll2550x {
-	struct clk_hw		hw;
-	const void __iomem	*reg_base;
-	unsigned long		offset;
-};
-
-#define to_clk_pll2550x(_hw) container_of(_hw, struct samsung_clk_pll2550x, hw)
-
 static unsigned long samsung_pll2550x_recalc_rate(struct clk_hw *hw,
 				unsigned long parent_rate)
 {
-	struct samsung_clk_pll2550x *pll = to_clk_pll2550x(hw);
+	struct samsung_clk_pll *pll = to_clk_pll(hw);
 	u32 r, p, m, s, pll_stat;
 	u64 fvco = parent_rate;
 
-	pll_stat = readl_relaxed(pll->reg_base + pll->offset * 3);
+	pll_stat = readl_relaxed(pll->con_reg);
 	r = (pll_stat >> PLL2550X_R_SHIFT) & PLL2550X_R_MASK;
 	if (!r)
 		return 0;
@@ -923,43 +915,6 @@ static const struct clk_ops samsung_pll2550x_clk_ops = {
 	.recalc_rate = samsung_pll2550x_recalc_rate,
 };
 
-struct clk * __init samsung_clk_register_pll2550x(const char *name,
-			const char *pname, const void __iomem *reg_base,
-			const unsigned long offset)
-{
-	struct samsung_clk_pll2550x *pll;
-	struct clk *clk;
-	struct clk_init_data init;
-
-	pll = kzalloc(sizeof(*pll), GFP_KERNEL);
-	if (!pll) {
-		pr_err("%s: could not allocate pll clk %s\n", __func__, name);
-		return NULL;
-	}
-
-	init.name = name;
-	init.ops = &samsung_pll2550x_clk_ops;
-	init.flags = CLK_GET_RATE_NOCACHE;
-	init.parent_names = &pname;
-	init.num_parents = 1;
-
-	pll->hw.init = &init;
-	pll->reg_base = reg_base;
-	pll->offset = offset;
-
-	clk = clk_register(NULL, &pll->hw);
-	if (IS_ERR(clk)) {
-		pr_err("%s: failed to register pll clock %s\n", __func__,
-				name);
-		kfree(pll);
-	}
-
-	if (clk_register_clkdev(clk, name, NULL))
-		pr_err("%s: failed to register lookup for %s", __func__, name);
-
-	return clk;
-}
-
 /*
  * PLL2550xx Clock Type
  */
@@ -1063,6 +1018,102 @@ static const struct clk_ops samsung_pll2550xx_clk_min_ops = {
 };
 
 /*
+ * PLL2650x Clock Type
+ */
+
+/* Maximum lock time can be 3000 * PDIV cycles */
+#define PLL2650X_LOCK_FACTOR		3000
+
+#define PLL2650X_M_MASK			0x1ff
+#define PLL2650X_P_MASK			0x3f
+#define PLL2650X_S_MASK			0x7
+#define PLL2650X_K_MASK			0xffff
+#define PLL2650X_LOCK_STAT_MASK		0x1
+#define PLL2650X_M_SHIFT		16
+#define PLL2650X_P_SHIFT		8
+#define PLL2650X_S_SHIFT		0
+#define PLL2650X_K_SHIFT		0
+#define PLL2650X_LOCK_STAT_SHIFT	29
+#define PLL2650X_PLL_ENABLE_SHIFT	31
+
+static unsigned long samsung_pll2650x_recalc_rate(struct clk_hw *hw,
+				unsigned long parent_rate)
+{
+	struct samsung_clk_pll *pll = to_clk_pll(hw);
+	u64 fout = parent_rate;
+	u32 mdiv, pdiv, sdiv, pll_con0, pll_con1;
+	s16 kdiv;
+
+	pll_con0 = readl_relaxed(pll->con_reg);
+	mdiv = (pll_con0 >> PLL2650X_M_SHIFT) & PLL2650X_M_MASK;
+	pdiv = (pll_con0 >> PLL2650X_P_SHIFT) & PLL2650X_P_MASK;
+	sdiv = (pll_con0 >> PLL2650X_S_SHIFT) & PLL2650X_S_MASK;
+
+	pll_con1 = readl_relaxed(pll->con_reg + 4);
+	kdiv = (s16)((pll_con1 >> PLL2650X_K_SHIFT) & PLL2650X_K_MASK);
+
+	fout *= (mdiv << 16) + kdiv;
+	do_div(fout, (pdiv << sdiv));
+	fout >>= 16;
+
+	return (unsigned long)fout;
+}
+
+static int samsung_pll2650x_set_rate(struct clk_hw *hw, unsigned long drate,
+					unsigned long prate)
+{
+	struct samsung_clk_pll *pll = to_clk_pll(hw);
+	const struct samsung_pll_rate_table *rate;
+	u32 con0, con1;
+
+	/* Get required rate settings from table */
+	rate = samsung_get_pll_settings(pll, drate);
+	if (!rate) {
+		pr_err("%s: Invalid rate : %lu for pll clk %s\n", __func__,
+			drate, clk_hw_get_name(hw));
+		return -EINVAL;
+	}
+
+	con0 = readl_relaxed(pll->con_reg);
+	con1 = readl_relaxed(pll->con_reg + 4);
+
+	/* Set PLL lock time. */
+	writel_relaxed(rate->pdiv * PLL2650X_LOCK_FACTOR, pll->lock_reg);
+
+	/* Change PLL PMS values */
+	con0 &= ~((PLL2650X_M_MASK << PLL2650X_M_SHIFT) |
+			(PLL2650X_P_MASK << PLL2650X_P_SHIFT) |
+			(PLL2650X_S_MASK << PLL2650X_S_SHIFT));
+	con0 |= (rate->mdiv << PLL2650X_M_SHIFT) |
+			(rate->pdiv << PLL2650X_P_SHIFT) |
+			(rate->sdiv << PLL2650X_S_SHIFT);
+	con0 |= (1 << PLL2650X_PLL_ENABLE_SHIFT);
+	writel_relaxed(con0, pll->con_reg);
+
+	con1 &= ~(PLL2650X_K_MASK << PLL2650X_K_SHIFT);
+	con1 |= ((rate->kdiv & PLL2650X_K_MASK) << PLL2650X_K_SHIFT);
+	writel_relaxed(con1, pll->con_reg + 4);
+
+	do {
+		cpu_relax();
+		con0 = readl_relaxed(pll->con_reg);
+	} while (!(con0 & (PLL2650X_LOCK_STAT_MASK
+			<< PLL2650X_LOCK_STAT_SHIFT)));
+
+	return 0;
+}
+
+static const struct clk_ops samsung_pll2650x_clk_ops = {
+	.recalc_rate = samsung_pll2650x_recalc_rate,
+	.round_rate = samsung_pll_round_rate,
+	.set_rate = samsung_pll2650x_set_rate,
+};
+
+static const struct clk_ops samsung_pll2650x_clk_min_ops = {
+	.recalc_rate = samsung_pll2650x_recalc_rate,
+};
+
+/*
  * PLL2650XX Clock Type
  */
 
@@ -1263,12 +1314,21 @@ static void __init _samsung_clk_register_pll(struct samsung_clk_provider *ctx,
 		else
 			init.ops = &samsung_s3c2440_mpll_clk_ops;
 		break;
+	case pll_2550x:
+		init.ops = &samsung_pll2550x_clk_ops;
+		break;
 	case pll_2550xx:
 		if (!pll->rate_table)
 			init.ops = &samsung_pll2550xx_clk_min_ops;
 		else
 			init.ops = &samsung_pll2550xx_clk_ops;
 		break;
+	case pll_2650x:
+		if (!pll->rate_table)
+			init.ops = &samsung_pll2650x_clk_min_ops;
+		else
+			init.ops = &samsung_pll2650x_clk_ops;
+		break;
 	case pll_2650xx:
 		if (!pll->rate_table)
 			init.ops = &samsung_pll2650xx_clk_min_ops;
diff --git a/drivers/clk/samsung/clk-pll.h b/drivers/clk/samsung/clk-pll.h
index 213de9af8b4f..a1ca0233cb4b 100644
--- a/drivers/clk/samsung/clk-pll.h
+++ b/drivers/clk/samsung/clk-pll.h
@@ -31,7 +31,9 @@ enum samsung_pll_type {
 	pll_s3c2410_mpll,
 	pll_s3c2410_upll,
 	pll_s3c2440_mpll,
+	pll_2550x,
 	pll_2550xx,
+	pll_2650x,
 	pll_2650xx,
 	pll_1450x,
 	pll_1451x,
diff --git a/drivers/clk/st/clk-flexgen.c b/drivers/clk/st/clk-flexgen.c
index 546bd79c8e3a..a485f3b284b9 100644
--- a/drivers/clk/st/clk-flexgen.c
+++ b/drivers/clk/st/clk-flexgen.c
@@ -15,6 +15,11 @@
 #include <linux/of.h>
 #include <linux/of_address.h>
 
+struct clkgen_data {
+	unsigned long flags;
+	bool mode;
+};
+
 struct flexgen {
 	struct clk_hw hw;
 
@@ -28,9 +33,14 @@ struct flexgen {
 	struct clk_gate fgate;
 	/* Final divisor */
 	struct clk_divider fdiv;
+	/* Asynchronous mode control */
+	struct clk_gate sync;
+	/* hw control flags */
+	bool control_mode;
 };
 
 #define to_flexgen(_hw) container_of(_hw, struct flexgen, hw)
+#define to_clk_gate(_hw) container_of(_hw, struct clk_gate, hw)
 
 static int flexgen_enable(struct clk_hw *hw)
 {
@@ -139,12 +149,21 @@ static int flexgen_set_rate(struct clk_hw *hw, unsigned long rate,
 	struct flexgen *flexgen = to_flexgen(hw);
 	struct clk_hw *pdiv_hw = &flexgen->pdiv.hw;
 	struct clk_hw *fdiv_hw = &flexgen->fdiv.hw;
+	struct clk_hw *sync_hw = &flexgen->sync.hw;
+	struct clk_gate *config = to_clk_gate(sync_hw);
 	unsigned long div = 0;
 	int ret = 0;
+	u32 reg;
 
 	__clk_hw_set_clk(pdiv_hw, hw);
 	__clk_hw_set_clk(fdiv_hw, hw);
 
+	if (flexgen->control_mode) {
+		reg = readl(config->reg);
+		reg &= ~BIT(config->bit_idx);
+		writel(reg, config->reg);
+	}
+
 	div = clk_best_div(parent_rate, rate);
 
 	/*
@@ -178,7 +197,7 @@ static const struct clk_ops flexgen_ops = {
 static struct clk *clk_register_flexgen(const char *name,
 				const char **parent_names, u8 num_parents,
 				void __iomem *reg, spinlock_t *lock, u32 idx,
-				unsigned long flexgen_flags) {
+				unsigned long flexgen_flags, bool mode) {
 	struct flexgen *fgxbar;
 	struct clk *clk;
 	struct clk_init_data init;
@@ -227,6 +246,13 @@ static struct clk *clk_register_flexgen(const char *name,
 	fgxbar->fdiv.reg = fdiv_reg;
 	fgxbar->fdiv.width = 6;
 
+	/* Final divider sync config */
+	fgxbar->sync.lock = lock;
+	fgxbar->sync.reg = fdiv_reg;
+	fgxbar->sync.bit_idx = 7;
+
+	fgxbar->control_mode = mode;
+
 	fgxbar->hw.init = &init;
 
 	clk = clk_register(NULL, &fgxbar->hw);
@@ -259,6 +285,27 @@ static const char ** __init flexgen_get_parents(struct device_node *np,
 	return parents;
 }
 
+static const struct clkgen_data clkgen_audio = {
+	.flags = CLK_SET_RATE_PARENT,
+};
+
+static const struct clkgen_data clkgen_video = {
+	.flags = CLK_SET_RATE_PARENT,
+	.mode = 1,
+};
+
+static const struct of_device_id flexgen_of_match[] = {
+	{
+		.compatible = "st,flexgen-audio",
+		.data = &clkgen_audio,
+	},
+	{
+		.compatible = "st,flexgen-video",
+		.data = &clkgen_video,
+	},
+	{}
+};
+
 static void __init st_of_flexgen_setup(struct device_node *np)
 {
 	struct device_node *pnode;
@@ -267,7 +314,11 @@ static void __init st_of_flexgen_setup(struct device_node *np)
 	const char **parents;
 	int num_parents, i;
 	spinlock_t *rlock = NULL;
+	const struct of_device_id *match;
+	struct clkgen_data *data = NULL;
+	unsigned long flex_flags = 0;
 	int ret;
+	bool clk_mode = 0;
 
 	pnode = of_get_parent(np);
 	if (!pnode)
@@ -281,6 +332,13 @@ static void __init st_of_flexgen_setup(struct device_node *np)
 	if (!parents)
 		return;
 
+	match = of_match_node(flexgen_of_match, np);
+	if (match) {
+		data = (struct clkgen_data *)match->data;
+		flex_flags = data->flags;
+		clk_mode = data->mode;
+	}
+
 	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
 	if (!clk_data)
 		goto err;
@@ -307,7 +365,6 @@ static void __init st_of_flexgen_setup(struct device_node *np)
 	for (i = 0; i < clk_data->clk_num; i++) {
 		struct clk *clk;
 		const char *clk_name;
-		unsigned long flex_flags = 0;
 
 		if (of_property_read_string_index(np, "clock-output-names",
 						  i, &clk_name)) {
@@ -323,7 +380,7 @@ static void __init st_of_flexgen_setup(struct device_node *np)
 			continue;
 
 		clk = clk_register_flexgen(clk_name, parents, num_parents,
-					   reg, rlock, i, flex_flags);
+					   reg, rlock, i, flex_flags, clk_mode);
 
 		if (IS_ERR(clk))
 			goto err;
diff --git a/drivers/clk/st/clkgen-fsyn.c b/drivers/clk/st/clkgen-fsyn.c
index 09afeb85109c..14819d919df1 100644
--- a/drivers/clk/st/clkgen-fsyn.c
+++ b/drivers/clk/st/clkgen-fsyn.c
@@ -42,79 +42,6 @@ struct stm_fs {
 	unsigned long nsdiv;
 };
 
-static const struct stm_fs fs216c65_rtbl[] = {
-	{ .mdiv = 0x1f, .pe = 0x0,	.sdiv = 0x7,	.nsdiv = 0 },	/* 312.5 Khz */
-	{ .mdiv = 0x17, .pe = 0x25ed,	.sdiv = 0x1,	.nsdiv = 0 },	/* 27    MHz */
-	{ .mdiv = 0x1a, .pe = 0x7b36,	.sdiv = 0x2,	.nsdiv = 1 },	/* 36.87 MHz */
-	{ .mdiv = 0x13, .pe = 0x0,	.sdiv = 0x2,	.nsdiv = 1 },	/* 48    MHz */
-	{ .mdiv = 0x11, .pe = 0x1c72,	.sdiv = 0x1,	.nsdiv = 1 },	/* 108   MHz */
-};
-
-static const struct stm_fs fs432c65_rtbl[] = {
-	{ .mdiv = 0x1f, .pe = 0x0,	.sdiv = 0x7,	.nsdiv = 0 },	/* 625     Khz */
-	{ .mdiv = 0x13, .pe = 0x777c,	.sdiv = 0x4,	.nsdiv = 1 },	/* 25.175  MHz */
-	{ .mdiv = 0x19, .pe = 0x4d35,	.sdiv = 0x2,	.nsdiv = 0 },	/* 25.200  MHz */
-	{ .mdiv = 0x11, .pe = 0x1c72,	.sdiv = 0x4,	.nsdiv = 1 },	/* 27.000  MHz */
-	{ .mdiv = 0x17, .pe = 0x28f5,	.sdiv = 0x2,	.nsdiv = 0 },	/* 27.027  MHz */
-	{ .mdiv = 0x16, .pe = 0x3359,	.sdiv = 0x2,	.nsdiv = 0 },	/* 28.320  MHz */
-	{ .mdiv = 0x1f, .pe = 0x2083,	.sdiv = 0x3,	.nsdiv = 1 },	/* 30.240  MHz */
-	{ .mdiv = 0x1e, .pe = 0x430d,	.sdiv = 0x3,	.nsdiv = 1 },	/* 31.500  MHz */
-	{ .mdiv = 0x17, .pe = 0x0,	.sdiv = 0x3,	.nsdiv = 1 },	/* 40.000  MHz */
-	{ .mdiv = 0x19, .pe = 0x121a,	.sdiv = 0x1,	.nsdiv = 0 },	/* 49.500  MHz */
-	{ .mdiv = 0x13, .pe = 0x6667,	.sdiv = 0x3,	.nsdiv = 1 },	/* 50.000  MHz */
-	{ .mdiv = 0x10, .pe = 0x1ee6,	.sdiv = 0x3,	.nsdiv = 1 },	/* 57.284  MHz */
-	{ .mdiv = 0x1d, .pe = 0x3b14,	.sdiv = 0x2,	.nsdiv = 1 },	/* 65.000  MHz */
-	{ .mdiv = 0x12, .pe = 0x7c65,	.sdiv = 0x1,	.nsdiv = 0 },	/* 71.000  MHz */
-	{ .mdiv = 0x19, .pe = 0xecd,	.sdiv = 0x2,	.nsdiv = 1 },	/* 74.176  MHz */
-	{ .mdiv = 0x19, .pe = 0x121a,	.sdiv = 0x2,	.nsdiv = 1 },	/* 74.250  MHz */
-	{ .mdiv = 0x19, .pe = 0x3334,	.sdiv = 0x2,	.nsdiv = 1 },	/* 75.000  MHz */
-	{ .mdiv = 0x18, .pe = 0x5138,	.sdiv = 0x2,	.nsdiv = 1 },	/* 78.800  MHz */
-	{ .mdiv = 0x1d, .pe = 0x77d,	.sdiv = 0x0,	.nsdiv = 0 },	/* 85.500  MHz */
-	{ .mdiv = 0x1c, .pe = 0x13d5,	.sdiv = 0x0,	.nsdiv = 0 },	/* 88.750  MHz */
-	{ .mdiv = 0x11, .pe = 0x1c72,	.sdiv = 0x2,	.nsdiv = 1 },	/* 108.000 MHz */
-	{ .mdiv = 0x17, .pe = 0x28f5,	.sdiv = 0x0,	.nsdiv = 0 },	/* 108.108 MHz */
-	{ .mdiv = 0x10, .pe = 0x6e26,	.sdiv = 0x2,	.nsdiv = 1 },	/* 118.963 MHz */
-	{ .mdiv = 0x15, .pe = 0x3e63,	.sdiv = 0x0,	.nsdiv = 0 },	/* 119.000 MHz */
-	{ .mdiv = 0x1c, .pe = 0x471d,	.sdiv = 0x1,	.nsdiv = 1 },	/* 135.000 MHz */
-	{ .mdiv = 0x19, .pe = 0xecd,	.sdiv = 0x1,	.nsdiv = 1 },	/* 148.352 MHz */
-	{ .mdiv = 0x19, .pe = 0x121a,	.sdiv = 0x1,	.nsdiv = 1 },	/* 148.500 MHz */
-	{ .mdiv = 0x19, .pe = 0x121a,	.sdiv = 0x0,	.nsdiv = 1 },	/* 297     MHz */
-};
-
-static const struct stm_fs fs660c32_rtbl[] = {
-	{ .mdiv = 0x14, .pe = 0x376b,	.sdiv = 0x4,	.nsdiv = 1 },	/* 25.175  MHz */
-	{ .mdiv = 0x14, .pe = 0x30c3,	.sdiv = 0x4,	.nsdiv = 1 },	/* 25.200  MHz */
-	{ .mdiv = 0x10, .pe = 0x71c7,	.sdiv = 0x4,	.nsdiv = 1 },	/* 27.000  MHz */
-	{ .mdiv = 0x00, .pe = 0x47af,	.sdiv = 0x3,	.nsdiv = 0 },	/* 27.027  MHz */
-	{ .mdiv = 0x0e, .pe = 0x4e1a,	.sdiv = 0x4,	.nsdiv = 1 },	/* 28.320  MHz */
-	{ .mdiv = 0x0b, .pe = 0x534d,	.sdiv = 0x4,	.nsdiv = 1 },	/* 30.240  MHz */
-	{ .mdiv = 0x17, .pe = 0x6fbf,	.sdiv = 0x2,	.nsdiv = 0 },	/* 31.500  MHz */
-	{ .mdiv = 0x01, .pe = 0x0,	.sdiv = 0x4,	.nsdiv = 1 },	/* 40.000  MHz */
-	{ .mdiv = 0x15, .pe = 0x2aab,	.sdiv = 0x3,	.nsdiv = 1 },	/* 49.500  MHz */
-	{ .mdiv = 0x14, .pe = 0x6666,	.sdiv = 0x3,	.nsdiv = 1 },	/* 50.000  MHz */
-	{ .mdiv = 0x1d, .pe = 0x395f,	.sdiv = 0x1,	.nsdiv = 0 },	/* 57.284  MHz */
-	{ .mdiv = 0x08, .pe = 0x4ec5,	.sdiv = 0x3,	.nsdiv = 1 },	/* 65.000  MHz */
-	{ .mdiv = 0x05, .pe = 0x1770,	.sdiv = 0x3,	.nsdiv = 1 },	/* 71.000  MHz */
-	{ .mdiv = 0x03, .pe = 0x4ba7,	.sdiv = 0x3,	.nsdiv = 1 },	/* 74.176  MHz */
-	{ .mdiv = 0x0f, .pe = 0x3426,	.sdiv = 0x1,	.nsdiv = 0 },	/* 74.250  MHz */
-	{ .mdiv = 0x0e, .pe = 0x7777,	.sdiv = 0x1,	.nsdiv = 0 },	/* 75.000  MHz */
-	{ .mdiv = 0x01, .pe = 0x4053,	.sdiv = 0x3,	.nsdiv = 1 },	/* 78.800  MHz */
-	{ .mdiv = 0x09, .pe = 0x15b5,	.sdiv = 0x1,	.nsdiv = 0 },	/* 85.500  MHz */
-	{ .mdiv = 0x1b, .pe = 0x3f19,	.sdiv = 0x2,	.nsdiv = 1 },	/* 88.750  MHz */
-	{ .mdiv = 0x10, .pe = 0x71c7,	.sdiv = 0x2,	.nsdiv = 1 },	/* 108.000 MHz */
-	{ .mdiv = 0x00, .pe = 0x47af,	.sdiv = 0x1,	.nsdiv = 0 },	/* 108.108 MHz */
-	{ .mdiv = 0x0c, .pe = 0x3118,	.sdiv = 0x2,	.nsdiv = 1 },	/* 118.963 MHz */
-	{ .mdiv = 0x0c, .pe = 0x2f54,	.sdiv = 0x2,	.nsdiv = 1 },	/* 119.000 MHz */
-	{ .mdiv = 0x07, .pe = 0xe39,	.sdiv = 0x2,	.nsdiv = 1 },	/* 135.000 MHz */
-	{ .mdiv = 0x03, .pe = 0x4ba7,	.sdiv = 0x2,	.nsdiv = 1 },	/* 148.352 MHz */
-	{ .mdiv = 0x0f, .pe = 0x3426,	.sdiv = 0x0,	.nsdiv = 0 },	/* 148.500 MHz */
-	{ .mdiv = 0x03, .pe = 0x4ba7,	.sdiv = 0x1,	.nsdiv = 1 },	/* 296.704 MHz */
-	{ .mdiv = 0x03, .pe = 0x471c,	.sdiv = 0x1,	.nsdiv = 1 },	/* 297.000 MHz */
-	{ .mdiv = 0x00, .pe = 0x295f,	.sdiv = 0x1,	.nsdiv = 1 },	/* 326.700 MHz */
-	{ .mdiv = 0x1f, .pe = 0x3633,	.sdiv = 0x0,	.nsdiv = 1 },	/* 333.000 MHz */
-	{ .mdiv = 0x1c, .pe = 0x0,	.sdiv = 0x0,	.nsdiv = 1 },	/* 352.000 Mhz */
-};
-
 struct clkgen_quadfs_data {
 	bool reset_present;
 	bool bwfilter_present;
@@ -138,174 +65,18 @@ struct clkgen_quadfs_data {
 	struct clkgen_field nsdiv[QUADFS_MAX_CHAN];
 
 	const struct clk_ops *pll_ops;
-	const struct stm_fs *rtbl;
-	u8 rtbl_cnt;
+	int  (*get_params)(unsigned long, unsigned long, struct stm_fs *);
 	int  (*get_rate)(unsigned long , const struct stm_fs *,
 			unsigned long *);
 };
 
-static const struct clk_ops st_quadfs_pll_c65_ops;
 static const struct clk_ops st_quadfs_pll_c32_ops;
-static const struct clk_ops st_quadfs_fs216c65_ops;
-static const struct clk_ops st_quadfs_fs432c65_ops;
 static const struct clk_ops st_quadfs_fs660c32_ops;
 
-static int clk_fs216c65_get_rate(unsigned long, const struct stm_fs *,
-		unsigned long *);
-static int clk_fs432c65_get_rate(unsigned long, const struct stm_fs *,
-		unsigned long *);
+static int clk_fs660c32_dig_get_params(unsigned long input,
+		unsigned long output, struct stm_fs *fs);
 static int clk_fs660c32_dig_get_rate(unsigned long, const struct stm_fs *,
 		unsigned long *);
-/*
- * Values for all of the standalone instances of this clock
- * generator found in STiH415 and STiH416 SYSCFG register banks. Note
- * that the individual channel standby control bits (nsb) are in the
- * first register along with the PLL control bits.
- */
-static const struct clkgen_quadfs_data st_fs216c65_416 = {
-	/* 416 specific */
-	.npda	= CLKGEN_FIELD(0x0, 0x1, 14),
-	.nsb	= { CLKGEN_FIELD(0x0, 0x1, 10),
-		    CLKGEN_FIELD(0x0, 0x1, 11),
-		    CLKGEN_FIELD(0x0, 0x1, 12),
-		    CLKGEN_FIELD(0x0, 0x1, 13) },
-	.nsdiv_present = true,
-	.nsdiv	= { CLKGEN_FIELD(0x0, 0x1, 18),
-		    CLKGEN_FIELD(0x0, 0x1, 19),
-		    CLKGEN_FIELD(0x0, 0x1, 20),
-		    CLKGEN_FIELD(0x0, 0x1, 21) },
-	.mdiv	= { CLKGEN_FIELD(0x4, 0x1f, 0),
-		    CLKGEN_FIELD(0x14, 0x1f, 0),
-		    CLKGEN_FIELD(0x24, 0x1f, 0),
-		    CLKGEN_FIELD(0x34, 0x1f, 0) },
-	.en	= { CLKGEN_FIELD(0x10, 0x1, 0),
-		    CLKGEN_FIELD(0x20, 0x1, 0),
-		    CLKGEN_FIELD(0x30, 0x1, 0),
-		    CLKGEN_FIELD(0x40, 0x1, 0) },
-	.ndiv	= CLKGEN_FIELD(0x0, 0x1, 15),
-	.bwfilter_present = true,
-	.ref_bw = CLKGEN_FIELD(0x0, 0x3, 16),
-	.pe	= { CLKGEN_FIELD(0x8, 0xffff, 0),
-		    CLKGEN_FIELD(0x18, 0xffff, 0),
-		    CLKGEN_FIELD(0x28, 0xffff, 0),
-		    CLKGEN_FIELD(0x38, 0xffff, 0) },
-	.sdiv	= { CLKGEN_FIELD(0xC, 0x7, 0),
-		    CLKGEN_FIELD(0x1C, 0x7, 0),
-		    CLKGEN_FIELD(0x2C, 0x7, 0),
-		    CLKGEN_FIELD(0x3C, 0x7, 0) },
-	.pll_ops	= &st_quadfs_pll_c65_ops,
-	.rtbl		= fs216c65_rtbl,
-	.rtbl_cnt	= ARRAY_SIZE(fs216c65_rtbl),
-	.get_rate	= clk_fs216c65_get_rate,
-};
-
-static const struct clkgen_quadfs_data st_fs432c65_416 = {
-	.npda	= CLKGEN_FIELD(0x0, 0x1, 14),
-	.nsb	= { CLKGEN_FIELD(0x0, 0x1, 10),
-		    CLKGEN_FIELD(0x0, 0x1, 11),
-		    CLKGEN_FIELD(0x0, 0x1, 12),
-		    CLKGEN_FIELD(0x0, 0x1, 13) },
-	.nsdiv_present = true,
-	.nsdiv	= { CLKGEN_FIELD(0x0, 0x1, 18),
-		   CLKGEN_FIELD(0x0, 0x1, 19),
-		   CLKGEN_FIELD(0x0, 0x1, 20),
-		   CLKGEN_FIELD(0x0, 0x1, 21) },
-	.mdiv	= { CLKGEN_FIELD(0x4, 0x1f, 0),
-		    CLKGEN_FIELD(0x14, 0x1f, 0),
-		    CLKGEN_FIELD(0x24, 0x1f, 0),
-		    CLKGEN_FIELD(0x34, 0x1f, 0) },
-	.en	= { CLKGEN_FIELD(0x10, 0x1, 0),
-		    CLKGEN_FIELD(0x20, 0x1, 0),
-		    CLKGEN_FIELD(0x30, 0x1, 0),
-		    CLKGEN_FIELD(0x40, 0x1, 0) },
-	.ndiv	= CLKGEN_FIELD(0x0, 0x1, 15),
-	.bwfilter_present = true,
-	.ref_bw = CLKGEN_FIELD(0x0, 0x3, 16),
-	.pe	= { CLKGEN_FIELD(0x8, 0xffff, 0),
-		    CLKGEN_FIELD(0x18, 0xffff, 0),
-		    CLKGEN_FIELD(0x28, 0xffff, 0),
-		    CLKGEN_FIELD(0x38, 0xffff, 0) },
-	.sdiv	= { CLKGEN_FIELD(0xC, 0x7, 0),
-		    CLKGEN_FIELD(0x1C, 0x7, 0),
-		    CLKGEN_FIELD(0x2C, 0x7, 0),
-		    CLKGEN_FIELD(0x3C, 0x7, 0) },
-	.pll_ops	= &st_quadfs_pll_c65_ops,
-	.rtbl		= fs432c65_rtbl,
-	.rtbl_cnt	= ARRAY_SIZE(fs432c65_rtbl),
-	.get_rate	= clk_fs432c65_get_rate,
-};
-
-static const struct clkgen_quadfs_data st_fs660c32_E_416 = {
-	.npda	= CLKGEN_FIELD(0x0, 0x1, 14),
-	.nsb	= { CLKGEN_FIELD(0x0, 0x1, 10),
-		    CLKGEN_FIELD(0x0, 0x1, 11),
-		    CLKGEN_FIELD(0x0, 0x1, 12),
-		    CLKGEN_FIELD(0x0, 0x1, 13) },
-	.nsdiv_present = true,
-	.nsdiv	= { CLKGEN_FIELD(0x0, 0x1, 18),
-		    CLKGEN_FIELD(0x0, 0x1, 19),
-		    CLKGEN_FIELD(0x0, 0x1, 20),
-		    CLKGEN_FIELD(0x0, 0x1, 21) },
-	.mdiv	= { CLKGEN_FIELD(0x4, 0x1f, 0),
-		    CLKGEN_FIELD(0x14, 0x1f, 0),
-		    CLKGEN_FIELD(0x24, 0x1f, 0),
-		    CLKGEN_FIELD(0x34, 0x1f, 0) },
-	.en	= { CLKGEN_FIELD(0x10, 0x1, 0),
-		    CLKGEN_FIELD(0x20, 0x1, 0),
-		    CLKGEN_FIELD(0x30, 0x1, 0),
-		    CLKGEN_FIELD(0x40, 0x1, 0) },
-	.ndiv	= CLKGEN_FIELD(0x0, 0x7, 15),
-	.pe	= { CLKGEN_FIELD(0x8, 0x7fff, 0),
-		    CLKGEN_FIELD(0x18, 0x7fff, 0),
-		    CLKGEN_FIELD(0x28, 0x7fff, 0),
-		    CLKGEN_FIELD(0x38, 0x7fff, 0) },
-	.sdiv	= { CLKGEN_FIELD(0xC, 0xf, 0),
-		    CLKGEN_FIELD(0x1C, 0xf, 0),
-		    CLKGEN_FIELD(0x2C, 0xf, 0),
-		    CLKGEN_FIELD(0x3C, 0xf, 0) },
-	.lockstatus_present = true,
-	.lock_status = CLKGEN_FIELD(0xAC, 0x1, 0),
-	.pll_ops	= &st_quadfs_pll_c32_ops,
-	.rtbl		= fs660c32_rtbl,
-	.rtbl_cnt	= ARRAY_SIZE(fs660c32_rtbl),
-	.get_rate	= clk_fs660c32_dig_get_rate,
-};
-
-static const struct clkgen_quadfs_data st_fs660c32_F_416 = {
-	.npda	= CLKGEN_FIELD(0x0, 0x1, 14),
-	.nsb	= { CLKGEN_FIELD(0x0, 0x1, 10),
-		    CLKGEN_FIELD(0x0, 0x1, 11),
-		    CLKGEN_FIELD(0x0, 0x1, 12),
-		    CLKGEN_FIELD(0x0, 0x1, 13) },
-	.nsdiv_present = true,
-	.nsdiv	= { CLKGEN_FIELD(0x0, 0x1, 18),
-		    CLKGEN_FIELD(0x0, 0x1, 19),
-		    CLKGEN_FIELD(0x0, 0x1, 20),
-		    CLKGEN_FIELD(0x0, 0x1, 21) },
-	.mdiv	= { CLKGEN_FIELD(0x4, 0x1f, 0),
-		    CLKGEN_FIELD(0x14, 0x1f, 0),
-		    CLKGEN_FIELD(0x24, 0x1f, 0),
-		    CLKGEN_FIELD(0x34, 0x1f, 0) },
-	.en	= { CLKGEN_FIELD(0x10, 0x1, 0),
-		    CLKGEN_FIELD(0x20, 0x1, 0),
-		    CLKGEN_FIELD(0x30, 0x1, 0),
-		    CLKGEN_FIELD(0x40, 0x1, 0) },
-	.ndiv	= CLKGEN_FIELD(0x0, 0x7, 15),
-	.pe	= { CLKGEN_FIELD(0x8, 0x7fff, 0),
-		    CLKGEN_FIELD(0x18, 0x7fff, 0),
-		    CLKGEN_FIELD(0x28, 0x7fff, 0),
-		    CLKGEN_FIELD(0x38, 0x7fff, 0) },
-	.sdiv	= { CLKGEN_FIELD(0xC, 0xf, 0),
-		    CLKGEN_FIELD(0x1C, 0xf, 0),
-		    CLKGEN_FIELD(0x2C, 0xf, 0),
-		    CLKGEN_FIELD(0x3C, 0xf, 0) },
-	.lockstatus_present = true,
-	.lock_status = CLKGEN_FIELD(0xEC, 0x1, 0),
-	.pll_ops	= &st_quadfs_pll_c32_ops,
-	.rtbl		= fs660c32_rtbl,
-	.rtbl_cnt	= ARRAY_SIZE(fs660c32_rtbl),
-	.get_rate	= clk_fs660c32_dig_get_rate,
-};
 
 static const struct clkgen_quadfs_data st_fs660c32_C = {
 	.nrst_present = true,
@@ -345,8 +116,7 @@ static const struct clkgen_quadfs_data st_fs660c32_C = {
 	.powerup_polarity = 1,
 	.standby_polarity = 1,
 	.pll_ops	= &st_quadfs_pll_c32_ops,
-	.rtbl		= fs660c32_rtbl,
-	.rtbl_cnt	= ARRAY_SIZE(fs660c32_rtbl),
+	.get_params	= clk_fs660c32_dig_get_params,
 	.get_rate	= clk_fs660c32_dig_get_rate,
 };
 
@@ -388,8 +158,7 @@ static const struct clkgen_quadfs_data st_fs660c32_D = {
 	.powerup_polarity = 1,
 	.standby_polarity = 1,
 	.pll_ops	= &st_quadfs_pll_c32_ops,
-	.rtbl		= fs660c32_rtbl,
-	.rtbl_cnt	= ARRAY_SIZE(fs660c32_rtbl),
+	.get_params	= clk_fs660c32_dig_get_params,
 	.get_rate	= clk_fs660c32_dig_get_rate,};
 
 /**
@@ -605,12 +374,6 @@ static int quadfs_pll_fs660c32_set_rate(struct clk_hw *hw, unsigned long rate,
 	return 0;
 }
 
-static const struct clk_ops st_quadfs_pll_c65_ops = {
-	.enable		= quadfs_pll_enable,
-	.disable	= quadfs_pll_disable,
-	.is_enabled	= quadfs_pll_is_enabled,
-};
-
 static const struct clk_ops st_quadfs_pll_c32_ops = {
 	.enable		= quadfs_pll_enable,
 	.disable	= quadfs_pll_disable,
@@ -797,48 +560,6 @@ static int quadfs_fsynth_is_enabled(struct clk_hw *hw)
 	return fs->data->standby_polarity ? !nsb : !!nsb;
 }
 
-#define P15			(uint64_t)(1 << 15)
-
-static int clk_fs216c65_get_rate(unsigned long input, const struct stm_fs *fs,
-		unsigned long *rate)
-{
-	uint64_t res;
-	unsigned long ns;
-	unsigned long nd = 8; /* ndiv stuck at 0 => val = 8 */
-	unsigned long s;
-	long m;
-
-	m = fs->mdiv - 32;
-	s = 1 << (fs->sdiv + 1);
-	ns = (fs->nsdiv ? 1 : 3);
-
-	res = (uint64_t)(s * ns * P15 * (uint64_t)(m + 33));
-	res = res - (s * ns * fs->pe);
-	*rate = div64_u64(P15 * nd * input * 32, res);
-
-	return 0;
-}
-
-static int clk_fs432c65_get_rate(unsigned long input, const struct stm_fs *fs,
-		unsigned long *rate)
-{
-	uint64_t res;
-	unsigned long nd = 16; /* ndiv value; stuck at 0 (30Mhz input) */
-	long m;
-	unsigned long sd;
-	unsigned long ns;
-
-	m = fs->mdiv - 32;
-	sd = 1 << (fs->sdiv + 1);
-	ns = (fs->nsdiv ? 1 : 3);
-
-	res = (uint64_t)(sd * ns * P15 * (uint64_t)(m + 33));
-	res = res - (sd * ns * fs->pe);
-	*rate = div64_u64(P15 * nd * input * 32, res);
-
-	return 0;
-}
-
 #define P20		(uint64_t)(1 << 20)
 
 static int clk_fs660c32_dig_get_rate(unsigned long input,
@@ -864,6 +585,107 @@ static int clk_fs660c32_dig_get_rate(unsigned long input,
 	return 0;
 }
 
+
+static int clk_fs660c32_get_pe(int m, int si, unsigned long *deviation,
+		signed long input, unsigned long output, uint64_t *p,
+		struct stm_fs *fs)
+{
+	unsigned long new_freq, new_deviation;
+	struct stm_fs fs_tmp;
+	uint64_t val;
+
+	val = (uint64_t)output << si;
+
+	*p = (uint64_t)input * P20 - (32LL  + (uint64_t)m) * val * (P20 / 32LL);
+
+	*p = div64_u64(*p, val);
+
+	if (*p > 32767LL)
+		return 1;
+
+	fs_tmp.mdiv = (unsigned long) m;
+	fs_tmp.pe = (unsigned long)*p;
+	fs_tmp.sdiv = si;
+	fs_tmp.nsdiv = 1;
+
+	clk_fs660c32_dig_get_rate(input, &fs_tmp, &new_freq);
+
+	new_deviation = abs(output - new_freq);
+
+	if (new_deviation < *deviation) {
+		fs->mdiv = m;
+		fs->pe = (unsigned long)*p;
+		fs->sdiv = si;
+		fs->nsdiv = 1;
+		*deviation = new_deviation;
+	}
+	return 0;
+}
+
+static int clk_fs660c32_dig_get_params(unsigned long input,
+		unsigned long output, struct stm_fs *fs)
+{
+	int si;	/* sdiv_reg (8 downto 0) */
+	int m; /* md value */
+	unsigned long new_freq, new_deviation;
+	/* initial condition to say: "infinite deviation" */
+	unsigned long deviation = ~0;
+	uint64_t p, p1, p2;	/* pe value */
+	int r1, r2;
+
+	struct stm_fs fs_tmp;
+
+	for (si = 0; (si <= 8) && deviation; si++) {
+
+		/* Boundary test to avoid useless iteration */
+		r1 = clk_fs660c32_get_pe(0, si, &deviation,
+				input, output, &p1, fs);
+		r2 = clk_fs660c32_get_pe(31, si, &deviation,
+				input, output, &p2, fs);
+
+		/* No solution */
+		if (r1 && r2 && (p1 > p2))
+			continue;
+
+		/* Try to find best deviation */
+		for (m = 1; (m < 31) && deviation; m++)
+			clk_fs660c32_get_pe(m, si, &deviation,
+					input, output, &p, fs);
+
+	}
+
+	if (deviation == ~0) /* No solution found */
+		return -1;
+
+	/* pe fine tuning if deviation not 0: +/- 2 around computed pe value */
+	if (deviation) {
+		fs_tmp.mdiv = fs->mdiv;
+		fs_tmp.sdiv = fs->sdiv;
+		fs_tmp.nsdiv = fs->nsdiv;
+
+		if (fs->pe > 2)
+			p2 = fs->pe - 2;
+		else
+			p2 = 0;
+
+		for (; p2 < 32768ll && (p2 <= (fs->pe + 2)); p2++) {
+			fs_tmp.pe = (unsigned long)p2;
+
+			clk_fs660c32_dig_get_rate(input, &fs_tmp, &new_freq);
+
+			new_deviation = abs(output - new_freq);
+
+			/* Check if this is a better solution */
+			if (new_deviation < deviation) {
+				fs->pe = (unsigned long)p2;
+				deviation = new_deviation;
+
+			}
+		}
+	}
+	return 0;
+}
+
 static int quadfs_fsynt_get_hw_value_for_recalc(struct st_clk_quadfs_fsynth *fs,
 		struct stm_fs *params)
 {
@@ -899,38 +721,14 @@ static long quadfs_find_best_rate(struct clk_hw *hw, unsigned long drate,
 	struct st_clk_quadfs_fsynth *fs = to_quadfs_fsynth(hw);
 	int (*clk_fs_get_rate)(unsigned long ,
 				const struct stm_fs *, unsigned long *);
-	struct stm_fs prev_params;
-	unsigned long prev_rate, rate = 0;
-	unsigned long diff_rate, prev_diff_rate = ~0;
-	int index;
+	int (*clk_fs_get_params)(unsigned long, unsigned long, struct stm_fs *);
+	unsigned long rate = 0;
 
 	clk_fs_get_rate = fs->data->get_rate;
+	clk_fs_get_params = fs->data->get_params;
 
-	for (index = 0; index < fs->data->rtbl_cnt; index++) {
-		prev_rate = rate;
-
-		*params = fs->data->rtbl[index];
-		prev_params = *params;
-
-		clk_fs_get_rate(prate, &fs->data->rtbl[index], &rate);
-
-		diff_rate = abs(drate - rate);
-
-		if (diff_rate > prev_diff_rate) {
-			rate = prev_rate;
-			*params = prev_params;
-			break;
-		}
-
-		prev_diff_rate = diff_rate;
-
-		if (drate == rate)
-			return rate;
-	}
-
-
-	if (index == fs->data->rtbl_cnt)
-		*params = prev_params;
+	if (!clk_fs_get_params(prate, drate, params))
+		clk_fs_get_rate(prate, params, &rate);
 
 	return rate;
 }
@@ -1063,34 +861,6 @@ static struct clk * __init st_clk_register_quadfs_fsynth(
 	return clk;
 }
 
-static const struct of_device_id quadfs_of_match[] = {
-	{
-		.compatible = "st,stih416-quadfs216",
-		.data = &st_fs216c65_416
-	},
-	{
-		.compatible = "st,stih416-quadfs432",
-		.data = &st_fs432c65_416
-	},
-	{
-		.compatible = "st,stih416-quadfs660-E",
-		.data = &st_fs660c32_E_416
-	},
-	{
-		.compatible = "st,stih416-quadfs660-F",
-		.data = &st_fs660c32_F_416
-	},
-	{
-		.compatible = "st,stih407-quadfs660-C",
-		.data = &st_fs660c32_C
-	},
-	{
-		.compatible = "st,stih407-quadfs660-D",
-		.data = &st_fs660c32_D
-	},
-	{}
-};
-
 static void __init st_of_create_quadfs_fsynths(
 		struct device_node *np, const char *pll_name,
 		struct clkgen_quadfs_data *quadfs, void __iomem *reg,
@@ -1150,18 +920,14 @@ static void __init st_of_create_quadfs_fsynths(
 	of_clk_add_provider(np, of_clk_src_onecell_get, clk_data);
 }
 
-static void __init st_of_quadfs_setup(struct device_node *np)
+static void __init st_of_quadfs_setup(struct device_node *np,
+		struct clkgen_quadfs_data *data)
 {
-	const struct of_device_id *match;
 	struct clk *clk;
 	const char *pll_name, *clk_parent_name;
 	void __iomem *reg;
 	spinlock_t *lock;
 
-	match = of_match_node(quadfs_of_match, np);
-	if (WARN_ON(!match))
-		return;
-
 	reg = of_iomap(np, 0);
 	if (!reg)
 		return;
@@ -1180,8 +946,8 @@ static void __init st_of_quadfs_setup(struct device_node *np)
 
 	spin_lock_init(lock);
 
-	clk = st_clk_register_quadfs_pll(pll_name, clk_parent_name,
-			(struct clkgen_quadfs_data *) match->data, reg, lock);
+	clk = st_clk_register_quadfs_pll(pll_name, clk_parent_name, data,
+			reg, lock);
 	if (IS_ERR(clk))
 		goto err_exit;
 	else
@@ -1190,11 +956,20 @@ static void __init st_of_quadfs_setup(struct device_node *np)
 			__clk_get_name(clk_get_parent(clk)),
 			(unsigned int)clk_get_rate(clk));
 
-	st_of_create_quadfs_fsynths(np, pll_name,
-				    (struct clkgen_quadfs_data *)match->data,
-				    reg, lock);
+	st_of_create_quadfs_fsynths(np, pll_name, data, reg, lock);
 
 err_exit:
 	kfree(pll_name); /* No longer need local copy of the PLL name */
 }
-CLK_OF_DECLARE(quadfs, "st,quadfs", st_of_quadfs_setup);
+
+static void __init st_of_quadfs660C_setup(struct device_node *np)
+{
+	st_of_quadfs_setup(np, (struct clkgen_quadfs_data *) &st_fs660c32_C);
+}
+CLK_OF_DECLARE(quadfs660C, "st,quadfs-pll", st_of_quadfs660C_setup);
+
+static void __init st_of_quadfs660D_setup(struct device_node *np)
+{
+	st_of_quadfs_setup(np, (struct clkgen_quadfs_data *) &st_fs660c32_D);
+}
+CLK_OF_DECLARE(quadfs660D, "st,quadfs", st_of_quadfs660D_setup);
diff --git a/drivers/clk/st/clkgen-mux.c b/drivers/clk/st/clkgen-mux.c
index b1e10ffe7a44..c514d39760cb 100644
--- a/drivers/clk/st/clkgen-mux.c
+++ b/drivers/clk/st/clkgen-mux.c
@@ -19,9 +19,6 @@
 #include <linux/clk-provider.h>
 #include "clkgen.h"
 
-static DEFINE_SPINLOCK(clkgena_divmux_lock);
-static DEFINE_SPINLOCK(clkgenf_lock);
-
 static const char ** __init clkgen_mux_get_parents(struct device_node *np,
 						       int *num_parents)
 {
@@ -40,498 +37,6 @@ static const char ** __init clkgen_mux_get_parents(struct device_node *np,
 	return parents;
 }
 
-/**
- * DOC: Clock mux with a programmable divider on each of its three inputs.
- *      The mux has an input setting which effectively gates its output.
- *
- * Traits of this clock:
- * prepare - clk_(un)prepare only ensures parent is (un)prepared
- * enable - clk_enable and clk_disable are functional & control gating
- * rate - set rate is supported
- * parent - set/get parent
- */
-
-#define NUM_INPUTS 3
-
-struct clkgena_divmux {
-	struct clk_hw hw;
-	/* Subclassed mux and divider structures */
-	struct clk_mux mux;
-	struct clk_divider div[NUM_INPUTS];
-	/* Enable/running feedback register bits for each input */
-	void __iomem *feedback_reg[NUM_INPUTS];
-	int feedback_bit_idx;
-
-	u8              muxsel;
-};
-
-#define to_clkgena_divmux(_hw) container_of(_hw, struct clkgena_divmux, hw)
-
-struct clkgena_divmux_data {
-	int num_outputs;
-	int mux_offset;
-	int mux_offset2;
-	int mux_start_bit;
-	int div_offsets[NUM_INPUTS];
-	int fb_offsets[NUM_INPUTS];
-	int fb_start_bit_idx;
-};
-
-#define CKGAX_CLKOPSRC_SWITCH_OFF 0x3
-
-static int clkgena_divmux_is_running(struct clkgena_divmux *mux)
-{
-	u32 regval = readl(mux->feedback_reg[mux->muxsel]);
-	u32 running = regval & BIT(mux->feedback_bit_idx);
-	return !!running;
-}
-
-static int clkgena_divmux_enable(struct clk_hw *hw)
-{
-	struct clkgena_divmux *genamux = to_clkgena_divmux(hw);
-	struct clk_hw *mux_hw = &genamux->mux.hw;
-	unsigned long timeout;
-	int ret = 0;
-
-	__clk_hw_set_clk(mux_hw, hw);
-
-	ret = clk_mux_ops.set_parent(mux_hw, genamux->muxsel);
-	if (ret)
-		return ret;
-
-	timeout = jiffies + msecs_to_jiffies(10);
-
-	while (!clkgena_divmux_is_running(genamux)) {
-		if (time_after(jiffies, timeout))
-			return -ETIMEDOUT;
-		cpu_relax();
-	}
-
-	return 0;
-}
-
-static void clkgena_divmux_disable(struct clk_hw *hw)
-{
-	struct clkgena_divmux *genamux = to_clkgena_divmux(hw);
-	struct clk_hw *mux_hw = &genamux->mux.hw;
-
-	__clk_hw_set_clk(mux_hw, hw);
-
-	clk_mux_ops.set_parent(mux_hw, CKGAX_CLKOPSRC_SWITCH_OFF);
-}
-
-static int clkgena_divmux_is_enabled(struct clk_hw *hw)
-{
-	struct clkgena_divmux *genamux = to_clkgena_divmux(hw);
-	struct clk_hw *mux_hw = &genamux->mux.hw;
-
-	__clk_hw_set_clk(mux_hw, hw);
-
-	return (s8)clk_mux_ops.get_parent(mux_hw) > 0;
-}
-
-static u8 clkgena_divmux_get_parent(struct clk_hw *hw)
-{
-	struct clkgena_divmux *genamux = to_clkgena_divmux(hw);
-	struct clk_hw *mux_hw = &genamux->mux.hw;
-
-	__clk_hw_set_clk(mux_hw, hw);
-
-	genamux->muxsel = clk_mux_ops.get_parent(mux_hw);
-	if ((s8)genamux->muxsel < 0) {
-		pr_debug("%s: %s: Invalid parent, setting to default.\n",
-		      __func__, clk_hw_get_name(hw));
-		genamux->muxsel = 0;
-	}
-
-	return genamux->muxsel;
-}
-
-static int clkgena_divmux_set_parent(struct clk_hw *hw, u8 index)
-{
-	struct clkgena_divmux *genamux = to_clkgena_divmux(hw);
-
-	if (index >= CKGAX_CLKOPSRC_SWITCH_OFF)
-		return -EINVAL;
-
-	genamux->muxsel = index;
-
-	/*
-	 * If the mux is already enabled, call enable directly to set the
-	 * new mux position and wait for it to start running again. Otherwise
-	 * do nothing.
-	 */
-	if (clkgena_divmux_is_enabled(hw))
-		clkgena_divmux_enable(hw);
-
-	return 0;
-}
-
-static unsigned long clkgena_divmux_recalc_rate(struct clk_hw *hw,
-		unsigned long parent_rate)
-{
-	struct clkgena_divmux *genamux = to_clkgena_divmux(hw);
-	struct clk_hw *div_hw = &genamux->div[genamux->muxsel].hw;
-
-	__clk_hw_set_clk(div_hw, hw);
-
-	return clk_divider_ops.recalc_rate(div_hw, parent_rate);
-}
-
-static int clkgena_divmux_set_rate(struct clk_hw *hw, unsigned long rate,
-				unsigned long parent_rate)
-{
-	struct clkgena_divmux *genamux = to_clkgena_divmux(hw);
-	struct clk_hw *div_hw = &genamux->div[genamux->muxsel].hw;
-
-	__clk_hw_set_clk(div_hw, hw);
-
-	return clk_divider_ops.set_rate(div_hw, rate, parent_rate);
-}
-
-static long clkgena_divmux_round_rate(struct clk_hw *hw, unsigned long rate,
-				   unsigned long *prate)
-{
-	struct clkgena_divmux *genamux = to_clkgena_divmux(hw);
-	struct clk_hw *div_hw = &genamux->div[genamux->muxsel].hw;
-
-	__clk_hw_set_clk(div_hw, hw);
-
-	return clk_divider_ops.round_rate(div_hw, rate, prate);
-}
-
-static const struct clk_ops clkgena_divmux_ops = {
-	.enable = clkgena_divmux_enable,
-	.disable = clkgena_divmux_disable,
-	.is_enabled = clkgena_divmux_is_enabled,
-	.get_parent = clkgena_divmux_get_parent,
-	.set_parent = clkgena_divmux_set_parent,
-	.round_rate = clkgena_divmux_round_rate,
-	.recalc_rate = clkgena_divmux_recalc_rate,
-	.set_rate = clkgena_divmux_set_rate,
-};
-
-/**
- * clk_register_genamux - register a genamux clock with the clock framework
- */
-static struct clk * __init clk_register_genamux(const char *name,
-				const char **parent_names, u8 num_parents,
-				void __iomem *reg,
-				const struct clkgena_divmux_data *muxdata,
-				u32 idx)
-{
-	/*
-	 * Fixed constants across all ClockgenA variants
-	 */
-	const int mux_width = 2;
-	const int divider_width = 5;
-	struct clkgena_divmux *genamux;
-	struct clk *clk;
-	struct clk_init_data init;
-	int i;
-
-	genamux = kzalloc(sizeof(*genamux), GFP_KERNEL);
-	if (!genamux)
-		return ERR_PTR(-ENOMEM);
-
-	init.name = name;
-	init.ops = &clkgena_divmux_ops;
-	init.flags = CLK_IS_BASIC | CLK_GET_RATE_NOCACHE;
-	init.parent_names = parent_names;
-	init.num_parents = num_parents;
-
-	genamux->mux.lock  = &clkgena_divmux_lock;
-	genamux->mux.mask = BIT(mux_width) - 1;
-	genamux->mux.shift = muxdata->mux_start_bit + (idx * mux_width);
-	if (genamux->mux.shift > 31) {
-		/*
-		 * We have spilled into the second mux register so
-		 * adjust the register address and the bit shift accordingly
-		 */
-		genamux->mux.reg = reg + muxdata->mux_offset2;
-		genamux->mux.shift -= 32;
-	} else {
-		genamux->mux.reg   = reg + muxdata->mux_offset;
-	}
-
-	for (i = 0; i < NUM_INPUTS; i++) {
-		/*
-		 * Divider config for each input
-		 */
-		void __iomem *divbase = reg + muxdata->div_offsets[i];
-		genamux->div[i].width = divider_width;
-		genamux->div[i].reg = divbase + (idx * sizeof(u32));
-
-		/*
-		 * Mux enabled/running feedback register for each input.
-		 */
-		genamux->feedback_reg[i] = reg + muxdata->fb_offsets[i];
-	}
-
-	genamux->feedback_bit_idx = muxdata->fb_start_bit_idx + idx;
-	genamux->hw.init = &init;
-
-	clk = clk_register(NULL, &genamux->hw);
-	if (IS_ERR(clk)) {
-		kfree(genamux);
-		goto err;
-	}
-
-	pr_debug("%s: parent %s rate %lu\n",
-			__clk_get_name(clk),
-			__clk_get_name(clk_get_parent(clk)),
-			clk_get_rate(clk));
-err:
-	return clk;
-}
-
-static struct clkgena_divmux_data st_divmux_c65hs = {
-	.num_outputs = 4,
-	.mux_offset = 0x14,
-	.mux_start_bit = 0,
-	.div_offsets = { 0x800, 0x900, 0xb00 },
-	.fb_offsets = { 0x18, 0x1c, 0x20 },
-	.fb_start_bit_idx = 0,
-};
-
-static struct clkgena_divmux_data st_divmux_c65ls = {
-	.num_outputs = 14,
-	.mux_offset = 0x14,
-	.mux_offset2 = 0x24,
-	.mux_start_bit = 8,
-	.div_offsets = { 0x810, 0xa10, 0xb10 },
-	.fb_offsets = { 0x18, 0x1c, 0x20 },
-	.fb_start_bit_idx = 4,
-};
-
-static struct clkgena_divmux_data st_divmux_c32odf0 = {
-	.num_outputs = 8,
-	.mux_offset = 0x1c,
-	.mux_start_bit = 0,
-	.div_offsets = { 0x800, 0x900, 0xa60 },
-	.fb_offsets = { 0x2c, 0x24, 0x28 },
-	.fb_start_bit_idx = 0,
-};
-
-static struct clkgena_divmux_data st_divmux_c32odf1 = {
-	.num_outputs = 8,
-	.mux_offset = 0x1c,
-	.mux_start_bit = 16,
-	.div_offsets = { 0x820, 0x980, 0xa80 },
-	.fb_offsets = { 0x2c, 0x24, 0x28 },
-	.fb_start_bit_idx = 8,
-};
-
-static struct clkgena_divmux_data st_divmux_c32odf2 = {
-	.num_outputs = 8,
-	.mux_offset = 0x20,
-	.mux_start_bit = 0,
-	.div_offsets = { 0x840, 0xa20, 0xb10 },
-	.fb_offsets = { 0x2c, 0x24, 0x28 },
-	.fb_start_bit_idx = 16,
-};
-
-static struct clkgena_divmux_data st_divmux_c32odf3 = {
-	.num_outputs = 8,
-	.mux_offset = 0x20,
-	.mux_start_bit = 16,
-	.div_offsets = { 0x860, 0xa40, 0xb30 },
-	.fb_offsets = { 0x2c, 0x24, 0x28 },
-	.fb_start_bit_idx = 24,
-};
-
-static const struct of_device_id clkgena_divmux_of_match[] = {
-	{
-		.compatible = "st,clkgena-divmux-c65-hs",
-		.data = &st_divmux_c65hs,
-	},
-	{
-		.compatible = "st,clkgena-divmux-c65-ls",
-		.data = &st_divmux_c65ls,
-	},
-	{
-		.compatible = "st,clkgena-divmux-c32-odf0",
-		.data = &st_divmux_c32odf0,
-	},
-	{
-		.compatible = "st,clkgena-divmux-c32-odf1",
-		.data = &st_divmux_c32odf1,
-	},
-	{
-		.compatible = "st,clkgena-divmux-c32-odf2",
-		.data = &st_divmux_c32odf2,
-	},
-	{
-		.compatible = "st,clkgena-divmux-c32-odf3",
-		.data = &st_divmux_c32odf3,
-	},
-	{}
-};
-
-static void __iomem * __init clkgen_get_register_base(struct device_node *np)
-{
-	struct device_node *pnode;
-	void __iomem *reg;
-
-	pnode = of_get_parent(np);
-	if (!pnode)
-		return NULL;
-
-	reg = of_iomap(pnode, 0);
-
-	of_node_put(pnode);
-	return reg;
-}
-
-static void __init st_of_clkgena_divmux_setup(struct device_node *np)
-{
-	const struct of_device_id *match;
-	const struct clkgena_divmux_data *data;
-	struct clk_onecell_data *clk_data;
-	void __iomem *reg;
-	const char **parents;
-	int num_parents = 0, i;
-
-	match = of_match_node(clkgena_divmux_of_match, np);
-	if (WARN_ON(!match))
-		return;
-
-	data = match->data;
-
-	reg = clkgen_get_register_base(np);
-	if (!reg)
-		return;
-
-	parents = clkgen_mux_get_parents(np, &num_parents);
-	if (IS_ERR(parents))
-		goto err_parents;
-
-	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
-	if (!clk_data)
-		goto err_alloc;
-
-	clk_data->clk_num = data->num_outputs;
-	clk_data->clks = kcalloc(clk_data->clk_num, sizeof(struct clk *),
-				 GFP_KERNEL);
-
-	if (!clk_data->clks)
-		goto err_alloc_clks;
-
-	for (i = 0; i < clk_data->clk_num; i++) {
-		struct clk *clk;
-		const char *clk_name;
-
-		if (of_property_read_string_index(np, "clock-output-names",
-						  i, &clk_name))
-			break;
-
-		/*
-		 * If we read an empty clock name then the output is unused
-		 */
-		if (*clk_name == '\0')
-			continue;
-
-		clk = clk_register_genamux(clk_name, parents, num_parents,
-					   reg, data, i);
-
-		if (IS_ERR(clk))
-			goto err;
-
-		clk_data->clks[i] = clk;
-	}
-
-	kfree(parents);
-
-	of_clk_add_provider(np, of_clk_src_onecell_get, clk_data);
-	return;
-err:
-	kfree(clk_data->clks);
-err_alloc_clks:
-	kfree(clk_data);
-err_alloc:
-	kfree(parents);
-err_parents:
-	iounmap(reg);
-}
-CLK_OF_DECLARE(clkgenadivmux, "st,clkgena-divmux", st_of_clkgena_divmux_setup);
-
-struct clkgena_prediv_data {
-	u32 offset;
-	u8 shift;
-	struct clk_div_table *table;
-};
-
-static struct clk_div_table prediv_table16[] = {
-	{ .val = 0, .div = 1 },
-	{ .val = 1, .div = 16 },
-	{ .div = 0 },
-};
-
-static struct clkgena_prediv_data prediv_c65_data = {
-	.offset = 0x4c,
-	.shift = 31,
-	.table = prediv_table16,
-};
-
-static struct clkgena_prediv_data prediv_c32_data = {
-	.offset = 0x50,
-	.shift = 1,
-	.table = prediv_table16,
-};
-
-static const struct of_device_id clkgena_prediv_of_match[] = {
-	{ .compatible = "st,clkgena-prediv-c65", .data = &prediv_c65_data },
-	{ .compatible = "st,clkgena-prediv-c32", .data = &prediv_c32_data },
-	{}
-};
-
-static void __init st_of_clkgena_prediv_setup(struct device_node *np)
-{
-	const struct of_device_id *match;
-	void __iomem *reg;
-	const char *parent_name, *clk_name;
-	struct clk *clk;
-	const struct clkgena_prediv_data *data;
-
-	match = of_match_node(clkgena_prediv_of_match, np);
-	if (!match) {
-		pr_err("%s: No matching data\n", __func__);
-		return;
-	}
-
-	data = match->data;
-
-	reg = clkgen_get_register_base(np);
-	if (!reg)
-		return;
-
-	parent_name = of_clk_get_parent_name(np, 0);
-	if (!parent_name)
-		goto err;
-
-	if (of_property_read_string_index(np, "clock-output-names",
-					  0, &clk_name))
-		goto err;
-
-	clk = clk_register_divider_table(NULL, clk_name, parent_name,
-					 CLK_GET_RATE_NOCACHE,
-					 reg + data->offset, data->shift, 1,
-					 0, data->table, NULL);
-	if (IS_ERR(clk))
-		goto err;
-
-	of_clk_add_provider(np, of_clk_src_simple_get, clk);
-	pr_debug("%s: parent %s rate %u\n",
-		__clk_get_name(clk),
-		__clk_get_name(clk_get_parent(clk)),
-		(unsigned int)clk_get_rate(clk));
-
-	return;
-err:
-	iounmap(reg);
-}
-CLK_OF_DECLARE(clkgenaprediv, "st,clkgena-prediv", st_of_clkgena_prediv_setup);
-
 struct clkgen_mux_data {
 	u32 offset;
 	u8 shift;
@@ -541,49 +46,6 @@ struct clkgen_mux_data {
 	u8 mux_flags;
 };
 
-static struct clkgen_mux_data clkgen_mux_c_vcc_hd_416 = {
-	.offset = 0,
-	.shift = 0,
-	.width = 1,
-};
-
-static struct clkgen_mux_data clkgen_mux_f_vcc_fvdp_416 = {
-	.offset = 0,
-	.shift = 0,
-	.width = 1,
-};
-
-static struct clkgen_mux_data clkgen_mux_f_vcc_hva_416 = {
-	.offset = 0,
-	.shift = 0,
-	.width = 1,
-};
-
-static struct clkgen_mux_data clkgen_mux_f_vcc_hd_416 = {
-	.offset = 0,
-	.shift = 16,
-	.width = 1,
-	.lock = &clkgenf_lock,
-};
-
-static struct clkgen_mux_data clkgen_mux_c_vcc_sd_416 = {
-	.offset = 0,
-	.shift = 17,
-	.width = 1,
-	.lock = &clkgenf_lock,
-};
-
-static struct clkgen_mux_data stih415_a9_mux_data = {
-	.offset = 0,
-	.shift = 1,
-	.width = 2,
-	.lock = &clkgen_a9_lock,
-};
-static struct clkgen_mux_data stih416_a9_mux_data = {
-	.offset = 0,
-	.shift = 0,
-	.width = 2,
-};
 static struct clkgen_mux_data stih407_a9_mux_data = {
 	.offset = 0x1a4,
 	.shift = 0,
@@ -591,58 +53,13 @@ static struct clkgen_mux_data stih407_a9_mux_data = {
 	.lock = &clkgen_a9_lock,
 };
 
-static const struct of_device_id mux_of_match[] = {
-	{
-		.compatible = "st,stih416-clkgenc-vcc-hd",
-		.data = &clkgen_mux_c_vcc_hd_416,
-	},
-	{
-		.compatible = "st,stih416-clkgenf-vcc-fvdp",
-		.data = &clkgen_mux_f_vcc_fvdp_416,
-	},
-	{
-		.compatible = "st,stih416-clkgenf-vcc-hva",
-		.data = &clkgen_mux_f_vcc_hva_416,
-	},
-	{
-		.compatible = "st,stih416-clkgenf-vcc-hd",
-		.data = &clkgen_mux_f_vcc_hd_416,
-	},
-	{
-		.compatible = "st,stih416-clkgenf-vcc-sd",
-		.data = &clkgen_mux_c_vcc_sd_416,
-	},
-	{
-		.compatible = "st,stih415-clkgen-a9-mux",
-		.data = &stih415_a9_mux_data,
-	},
-	{
-		.compatible = "st,stih416-clkgen-a9-mux",
-		.data = &stih416_a9_mux_data,
-	},
-	{
-		.compatible = "st,stih407-clkgen-a9-mux",
-		.data = &stih407_a9_mux_data,
-	},
-	{}
-};
-
-static void __init st_of_clkgen_mux_setup(struct device_node *np)
+static void __init st_of_clkgen_mux_setup(struct device_node *np,
+		struct clkgen_mux_data *data)
 {
-	const struct of_device_id *match;
 	struct clk *clk;
 	void __iomem *reg;
 	const char **parents;
-	int num_parents;
-	const struct clkgen_mux_data *data;
-
-	match = of_match_node(mux_of_match, np);
-	if (!match) {
-		pr_err("%s: No matching data\n", __func__);
-		return;
-	}
-
-	data = match->data;
+	int num_parents = 0;
 
 	reg = of_iomap(np, 0);
 	if (!reg) {
@@ -679,161 +96,10 @@ err:
 err_parents:
 	iounmap(reg);
 }
-CLK_OF_DECLARE(clkgen_mux, "st,clkgen-mux", st_of_clkgen_mux_setup);
-
-#define VCC_MAX_CHANNELS 16
-
-#define VCC_GATE_OFFSET 0x0
-#define VCC_MUX_OFFSET 0x4
-#define VCC_DIV_OFFSET 0x8
-
-struct clkgen_vcc_data {
-	spinlock_t *lock;
-	unsigned long clk_flags;
-};
-
-static struct clkgen_vcc_data st_clkgenc_vcc_416 = {
-	.clk_flags = CLK_SET_RATE_PARENT,
-};
-
-static struct clkgen_vcc_data st_clkgenf_vcc_416 = {
-	.lock = &clkgenf_lock,
-};
-
-static const struct of_device_id vcc_of_match[] = {
-	{ .compatible = "st,stih416-clkgenc", .data = &st_clkgenc_vcc_416 },
-	{ .compatible = "st,stih416-clkgenf", .data = &st_clkgenf_vcc_416 },
-	{}
-};
 
-static void __init st_of_clkgen_vcc_setup(struct device_node *np)
+static void __init st_of_clkgen_a9_mux_setup(struct device_node *np)
 {
-	const struct of_device_id *match;
-	void __iomem *reg;
-	const char **parents;
-	int num_parents, i;
-	struct clk_onecell_data *clk_data;
-	const struct clkgen_vcc_data *data;
-
-	match = of_match_node(vcc_of_match, np);
-	if (WARN_ON(!match))
-		return;
-	data = match->data;
-
-	reg = of_iomap(np, 0);
-	if (!reg)
-		return;
-
-	parents = clkgen_mux_get_parents(np, &num_parents);
-	if (IS_ERR(parents))
-		goto err_parents;
-
-	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
-	if (!clk_data)
-		goto err_alloc;
-
-	clk_data->clk_num = VCC_MAX_CHANNELS;
-	clk_data->clks = kcalloc(clk_data->clk_num, sizeof(struct clk *),
-				 GFP_KERNEL);
-
-	if (!clk_data->clks)
-		goto err_alloc_clks;
-
-	for (i = 0; i < clk_data->clk_num; i++) {
-		struct clk *clk;
-		const char *clk_name;
-		struct clk_gate *gate;
-		struct clk_divider *div;
-		struct clk_mux *mux;
-
-		if (of_property_read_string_index(np, "clock-output-names",
-						  i, &clk_name))
-			break;
-
-		/*
-		 * If we read an empty clock name then the output is unused
-		 */
-		if (*clk_name == '\0')
-			continue;
-
-		gate = kzalloc(sizeof(*gate), GFP_KERNEL);
-		if (!gate)
-			goto err;
-
-		div = kzalloc(sizeof(*div), GFP_KERNEL);
-		if (!div) {
-			kfree(gate);
-			goto err;
-		}
-
-		mux = kzalloc(sizeof(*mux), GFP_KERNEL);
-		if (!mux) {
-			kfree(gate);
-			kfree(div);
-			goto err;
-		}
-
-		gate->reg = reg + VCC_GATE_OFFSET;
-		gate->bit_idx = i;
-		gate->flags = CLK_GATE_SET_TO_DISABLE;
-		gate->lock = data->lock;
-
-		div->reg = reg + VCC_DIV_OFFSET;
-		div->shift = 2 * i;
-		div->width = 2;
-		div->flags = CLK_DIVIDER_POWER_OF_TWO |
-			CLK_DIVIDER_ROUND_CLOSEST;
-
-		mux->reg = reg + VCC_MUX_OFFSET;
-		mux->shift = 2 * i;
-		mux->mask = 0x3;
-
-		clk = clk_register_composite(NULL, clk_name, parents,
-					     num_parents,
-					     &mux->hw, &clk_mux_ops,
-					     &div->hw, &clk_divider_ops,
-					     &gate->hw, &clk_gate_ops,
-					     data->clk_flags |
-					     CLK_GET_RATE_NOCACHE);
-		if (IS_ERR(clk)) {
-			kfree(gate);
-			kfree(div);
-			kfree(mux);
-			goto err;
-		}
-
-		pr_debug("%s: parent %s rate %u\n",
-			__clk_get_name(clk),
-			__clk_get_name(clk_get_parent(clk)),
-			(unsigned int)clk_get_rate(clk));
-
-		clk_data->clks[i] = clk;
-	}
-
-	kfree(parents);
-
-	of_clk_add_provider(np, of_clk_src_onecell_get, clk_data);
-	return;
-
-err:
-	for (i = 0; i < clk_data->clk_num; i++) {
-		struct clk_composite *composite;
-
-		if (!clk_data->clks[i])
-			continue;
-
-		composite = to_clk_composite(__clk_get_hw(clk_data->clks[i]));
-		kfree(to_clk_gate(composite->gate_hw));
-		kfree(to_clk_divider(composite->rate_hw));
-		kfree(to_clk_mux(composite->mux_hw));
-	}
-
-	kfree(clk_data->clks);
-err_alloc_clks:
-	kfree(clk_data);
-err_alloc:
-	kfree(parents);
-err_parents:
-	iounmap(reg);
+	st_of_clkgen_mux_setup(np, &stih407_a9_mux_data);
 }
-CLK_OF_DECLARE(clkgen_vcc, "st,clkgen-vcc", st_of_clkgen_vcc_setup);
+CLK_OF_DECLARE(clkgen_a9mux, "st,stih407-clkgen-a9-mux",
+		st_of_clkgen_a9_mux_setup);
diff --git a/drivers/clk/st/clkgen-pll.c b/drivers/clk/st/clkgen-pll.c
index 0b5990e82e0d..25bda48a5d35 100644
--- a/drivers/clk/st/clkgen-pll.c
+++ b/drivers/clk/st/clkgen-pll.c
@@ -26,14 +26,6 @@ static DEFINE_SPINLOCK(clkgena_c32_odf_lock);
 DEFINE_SPINLOCK(clkgen_a9_lock);
 
 /*
- * Common PLL configuration register bits for PLL800 and PLL1600 C65
- */
-#define C65_MDIV_PLL800_MASK	(0xff)
-#define C65_MDIV_PLL1600_MASK	(0x7)
-#define C65_NDIV_MASK		(0xff)
-#define C65_PDIV_MASK		(0x7)
-
-/*
  * PLL configuration register bits for PLL3200 C32
  */
 #define C32_NDIV_MASK (0xff)
@@ -70,144 +62,10 @@ struct clkgen_pll_data {
 	const struct clk_ops *ops;
 };
 
-static const struct clk_ops st_pll1600c65_ops;
-static const struct clk_ops st_pll800c65_ops;
 static const struct clk_ops stm_pll3200c32_ops;
 static const struct clk_ops stm_pll3200c32_a9_ops;
-static const struct clk_ops st_pll1200c32_ops;
 static const struct clk_ops stm_pll4600c28_ops;
 
-static const struct clkgen_pll_data st_pll1600c65_ax = {
-	.pdn_status	= CLKGEN_FIELD(0x0, 0x1,			19),
-	.pdn_ctrl	= CLKGEN_FIELD(0x10,	0x1,			0),
-	.locked_status	= CLKGEN_FIELD(0x0, 0x1,			31),
-	.mdiv		= CLKGEN_FIELD(0x0, C65_MDIV_PLL1600_MASK,	0),
-	.ndiv		= CLKGEN_FIELD(0x0, C65_NDIV_MASK,		8),
-	.ops		= &st_pll1600c65_ops
-};
-
-static const struct clkgen_pll_data st_pll800c65_ax = {
-	.pdn_status	= CLKGEN_FIELD(0x0,	0x1,			19),
-	.pdn_ctrl	= CLKGEN_FIELD(0xC,	0x1,			1),
-	.locked_status	= CLKGEN_FIELD(0x0,	0x1,			31),
-	.mdiv		= CLKGEN_FIELD(0x0,	C65_MDIV_PLL800_MASK,	0),
-	.ndiv		= CLKGEN_FIELD(0x0,	C65_NDIV_MASK,		8),
-	.pdiv		= CLKGEN_FIELD(0x0,	C65_PDIV_MASK,		16),
-	.ops		= &st_pll800c65_ops
-};
-
-static const struct clkgen_pll_data st_pll3200c32_a1x_0 = {
-	.pdn_status	= CLKGEN_FIELD(0x0,	0x1,			31),
-	.pdn_ctrl	= CLKGEN_FIELD(0x18,	0x1,			0),
-	.locked_status	= CLKGEN_FIELD(0x4,	0x1,			31),
-	.ndiv		= CLKGEN_FIELD(0x0,	C32_NDIV_MASK,		0x0),
-	.idf		= CLKGEN_FIELD(0x4,	C32_IDF_MASK,		0x0),
-	.num_odfs = 4,
-	.odf =	{	CLKGEN_FIELD(0x54,	C32_ODF_MASK,		4),
-			CLKGEN_FIELD(0x54,	C32_ODF_MASK,		10),
-			CLKGEN_FIELD(0x54,	C32_ODF_MASK,		16),
-			CLKGEN_FIELD(0x54,	C32_ODF_MASK,		22) },
-	.odf_gate = {	CLKGEN_FIELD(0x54,	0x1,			0),
-			CLKGEN_FIELD(0x54,	0x1,			1),
-			CLKGEN_FIELD(0x54,	0x1,			2),
-			CLKGEN_FIELD(0x54,	0x1,			3) },
-	.ops		= &stm_pll3200c32_ops,
-};
-
-static const struct clkgen_pll_data st_pll3200c32_a1x_1 = {
-	.pdn_status	= CLKGEN_FIELD(0xC,	0x1,			31),
-	.pdn_ctrl	= CLKGEN_FIELD(0x18,	0x1,			1),
-	.locked_status	= CLKGEN_FIELD(0x10,	0x1,			31),
-	.ndiv		= CLKGEN_FIELD(0xC,	C32_NDIV_MASK,		0x0),
-	.idf		= CLKGEN_FIELD(0x10,	C32_IDF_MASK,		0x0),
-	.num_odfs = 4,
-	.odf = {	CLKGEN_FIELD(0x58,	C32_ODF_MASK,		4),
-			CLKGEN_FIELD(0x58,	C32_ODF_MASK,		10),
-			CLKGEN_FIELD(0x58,	C32_ODF_MASK,		16),
-			CLKGEN_FIELD(0x58,	C32_ODF_MASK,		22) },
-	.odf_gate = {	CLKGEN_FIELD(0x58,	0x1,			0),
-			CLKGEN_FIELD(0x58,	0x1,			1),
-			CLKGEN_FIELD(0x58,	0x1,			2),
-			CLKGEN_FIELD(0x58,	0x1,			3) },
-	.ops		= &stm_pll3200c32_ops,
-};
-
-/* 415 specific */
-static const struct clkgen_pll_data st_pll3200c32_a9_415 = {
-	.pdn_status	= CLKGEN_FIELD(0x0,	0x1,			0),
-	.pdn_ctrl	= CLKGEN_FIELD(0x0,	0x1,			0),
-	.locked_status	= CLKGEN_FIELD(0x6C,	0x1,			0),
-	.ndiv		= CLKGEN_FIELD(0x0,	C32_NDIV_MASK,		9),
-	.idf		= CLKGEN_FIELD(0x0,	C32_IDF_MASK,		22),
-	.num_odfs = 1,
-	.odf =		{ CLKGEN_FIELD(0x0,	C32_ODF_MASK,		3) },
-	.odf_gate =	{ CLKGEN_FIELD(0x0,	0x1,			28) },
-	.ops		= &stm_pll3200c32_ops,
-};
-
-static const struct clkgen_pll_data st_pll3200c32_ddr_415 = {
-	.pdn_status	= CLKGEN_FIELD(0x0,	0x1,			0),
-	.pdn_ctrl	= CLKGEN_FIELD(0x0,	0x1,			0),
-	.locked_status	= CLKGEN_FIELD(0x100,	0x1,			0),
-	.ndiv		= CLKGEN_FIELD(0x8,	C32_NDIV_MASK,		0),
-	.idf		= CLKGEN_FIELD(0x0,	C32_IDF_MASK,		25),
-	.num_odfs = 2,
-	.odf		= { CLKGEN_FIELD(0x8,	C32_ODF_MASK,		8),
-			    CLKGEN_FIELD(0x8,	C32_ODF_MASK,		14) },
-	.odf_gate	= { CLKGEN_FIELD(0x4,	0x1,			28),
-			    CLKGEN_FIELD(0x4,	0x1,			29) },
-	.ops		= &stm_pll3200c32_ops,
-};
-
-static const struct clkgen_pll_data st_pll1200c32_gpu_415 = {
-	.pdn_status	= CLKGEN_FIELD(0x4,	0x1,			0),
-	.pdn_ctrl	= CLKGEN_FIELD(0x4,	0x1,			0),
-	.locked_status	= CLKGEN_FIELD(0x168,	0x1,			0),
-	.ldf		= CLKGEN_FIELD(0x0,	C32_LDF_MASK,		3),
-	.idf		= CLKGEN_FIELD(0x0,	C32_IDF_MASK,		0),
-	.num_odfs = 0,
-	.odf		= { CLKGEN_FIELD(0x0,	C32_ODF_MASK,		10) },
-	.ops		= &st_pll1200c32_ops,
-};
-
-/* 416 specific */
-static const struct clkgen_pll_data st_pll3200c32_a9_416 = {
-	.pdn_status	= CLKGEN_FIELD(0x0,	0x1,			0),
-	.pdn_ctrl	= CLKGEN_FIELD(0x0,	0x1,			0),
-	.locked_status	= CLKGEN_FIELD(0x6C,	0x1,			0),
-	.ndiv		= CLKGEN_FIELD(0x8,	C32_NDIV_MASK,		0),
-	.idf		= CLKGEN_FIELD(0x0,	C32_IDF_MASK,		25),
-	.num_odfs = 1,
-	.odf		= { CLKGEN_FIELD(0x8,	C32_ODF_MASK,		8) },
-	.odf_gate	= { CLKGEN_FIELD(0x4,	0x1,			28) },
-	.ops		= &stm_pll3200c32_ops,
-};
-
-static const struct clkgen_pll_data st_pll3200c32_ddr_416 = {
-	.pdn_status	= CLKGEN_FIELD(0x0,	0x1,			0),
-	.pdn_ctrl	= CLKGEN_FIELD(0x0,	0x1,			0),
-	.locked_status	= CLKGEN_FIELD(0x10C,	0x1,			0),
-	.ndiv		= CLKGEN_FIELD(0x8,	C32_NDIV_MASK,		0),
-	.idf		= CLKGEN_FIELD(0x0,	C32_IDF_MASK,		25),
-	.num_odfs = 2,
-	.odf		= { CLKGEN_FIELD(0x8,	C32_ODF_MASK,		8),
-			    CLKGEN_FIELD(0x8,	C32_ODF_MASK,		14) },
-	.odf_gate	= { CLKGEN_FIELD(0x4,	0x1,			28),
-			    CLKGEN_FIELD(0x4,	0x1,			29) },
-	.ops		= &stm_pll3200c32_ops,
-};
-
-static const struct clkgen_pll_data st_pll1200c32_gpu_416 = {
-	.pdn_status	= CLKGEN_FIELD(0x8E4,	0x1,			3),
-	.pdn_ctrl	= CLKGEN_FIELD(0x8E4,	0x1,			3),
-	.locked_status	= CLKGEN_FIELD(0x90C,	0x1,			0),
-	.ldf		= CLKGEN_FIELD(0x0,	C32_LDF_MASK,		3),
-	.idf		= CLKGEN_FIELD(0x0,	C32_IDF_MASK,		0),
-	.num_odfs = 0,
-	.odf		= { CLKGEN_FIELD(0x0,	C32_ODF_MASK,		10) },
-	.ops		= &st_pll1200c32_ops,
-};
-
 static const struct clkgen_pll_data st_pll3200c32_407_a0 = {
 	/* 407 A0 */
 	.pdn_status	= CLKGEN_FIELD(0x2a0,	0x1,			8),
@@ -410,57 +268,6 @@ static void clkgen_pll_disable(struct clk_hw *hw)
 		spin_unlock_irqrestore(pll->lock, flags);
 }
 
-static unsigned long recalc_stm_pll800c65(struct clk_hw *hw,
-		unsigned long parent_rate)
-{
-	struct clkgen_pll *pll = to_clkgen_pll(hw);
-	unsigned long mdiv, ndiv, pdiv;
-	unsigned long rate;
-	uint64_t res;
-
-	if (!clkgen_pll_is_enabled(hw) || !clkgen_pll_is_locked(hw))
-		return 0;
-
-	pdiv = CLKGEN_READ(pll, pdiv);
-	mdiv = CLKGEN_READ(pll, mdiv);
-	ndiv = CLKGEN_READ(pll, ndiv);
-
-	if (!mdiv)
-		mdiv++; /* mdiv=0 or 1 => MDIV=1 */
-
-	res = (uint64_t)2 * (uint64_t)parent_rate * (uint64_t)ndiv;
-	rate = (unsigned long)div64_u64(res, mdiv * (1 << pdiv));
-
-	pr_debug("%s:%s rate %lu\n", clk_hw_get_name(hw), __func__, rate);
-
-	return rate;
-
-}
-
-static unsigned long recalc_stm_pll1600c65(struct clk_hw *hw,
-		unsigned long parent_rate)
-{
-	struct clkgen_pll *pll = to_clkgen_pll(hw);
-	unsigned long mdiv, ndiv;
-	unsigned long rate;
-
-	if (!clkgen_pll_is_enabled(hw) || !clkgen_pll_is_locked(hw))
-		return 0;
-
-	mdiv = CLKGEN_READ(pll, mdiv);
-	ndiv = CLKGEN_READ(pll, ndiv);
-
-	if (!mdiv)
-		mdiv = 1;
-
-	/* Note: input is divided by 1000 to avoid overflow */
-	rate = ((2 * (parent_rate / 1000) * ndiv) / mdiv) * 1000;
-
-	pr_debug("%s:%s rate %lu\n", clk_hw_get_name(hw), __func__, rate);
-
-	return rate;
-}
-
 static int clk_pll3200c32_get_params(unsigned long input, unsigned long output,
 			  struct stm_pll *pll)
 {
@@ -608,33 +415,6 @@ static int set_rate_stm_pll3200c32(struct clk_hw *hw, unsigned long rate,
 	return 0;
 }
 
-static unsigned long recalc_stm_pll1200c32(struct clk_hw *hw,
-		unsigned long parent_rate)
-{
-	struct clkgen_pll *pll = to_clkgen_pll(hw);
-	unsigned long odf, ldf, idf;
-	unsigned long rate;
-
-	if (!clkgen_pll_is_enabled(hw) || !clkgen_pll_is_locked(hw))
-		return 0;
-
-	odf = CLKGEN_READ(pll, odf[0]);
-	ldf = CLKGEN_READ(pll, ldf);
-	idf = CLKGEN_READ(pll, idf);
-
-	if (!idf) /* idf==0 means 1 */
-		idf = 1;
-	if (!odf) /* odf==0 means 1 */
-		odf = 1;
-
-	/* Note: input is divided by 1000 to avoid overflow */
-	rate = (((parent_rate / 1000) * ldf) / (odf * idf)) * 1000;
-
-	pr_debug("%s:%s rate %lu\n", clk_hw_get_name(hw), __func__, rate);
-
-	return rate;
-}
-
 /* PLL output structure
  * FVCO >> /2 >> FVCOBY2 (no output)
  *                 |> Divider (ODF) >> PHI
@@ -792,20 +572,6 @@ static int set_rate_stm_pll4600c28(struct clk_hw *hw, unsigned long rate,
 	return 0;
 }
 
-static const struct clk_ops st_pll1600c65_ops = {
-	.enable		= clkgen_pll_enable,
-	.disable	= clkgen_pll_disable,
-	.is_enabled	= clkgen_pll_is_enabled,
-	.recalc_rate	= recalc_stm_pll1600c65,
-};
-
-static const struct clk_ops st_pll800c65_ops = {
-	.enable		= clkgen_pll_enable,
-	.disable	= clkgen_pll_disable,
-	.is_enabled	= clkgen_pll_is_enabled,
-	.recalc_rate	= recalc_stm_pll800c65,
-};
-
 static const struct clk_ops stm_pll3200c32_ops = {
 	.enable		= clkgen_pll_enable,
 	.disable	= clkgen_pll_disable,
@@ -822,13 +588,6 @@ static const struct clk_ops stm_pll3200c32_a9_ops = {
 	.set_rate	= set_rate_stm_pll3200c32,
 };
 
-static const struct clk_ops st_pll1200c32_ops = {
-	.enable		= clkgen_pll_enable,
-	.disable	= clkgen_pll_disable,
-	.is_enabled	= clkgen_pll_is_enabled,
-	.recalc_rate	= recalc_stm_pll1200c32,
-};
-
 static const struct clk_ops stm_pll4600c28_ops = {
 	.enable		= clkgen_pll_enable,
 	.disable	= clkgen_pll_disable,
@@ -877,22 +636,6 @@ static struct clk * __init clkgen_pll_register(const char *parent_name,
 	return clk;
 }
 
-static struct clk * __init clkgen_c65_lsdiv_register(const char *parent_name,
-						     const char *clk_name)
-{
-	struct clk *clk;
-
-	clk = clk_register_fixed_factor(NULL, clk_name, parent_name, 0, 1, 2);
-	if (IS_ERR(clk))
-		return clk;
-
-	pr_debug("%s: parent %s rate %lu\n",
-			__clk_get_name(clk),
-			__clk_get_name(clk_get_parent(clk)),
-			clk_get_rate(clk));
-	return clk;
-}
-
 static void __iomem * __init clkgen_get_register_base(
 				struct device_node *np)
 {
@@ -909,89 +652,6 @@ static void __iomem * __init clkgen_get_register_base(
 	return reg;
 }
 
-#define CLKGENAx_PLL0_OFFSET 0x0
-#define CLKGENAx_PLL1_OFFSET 0x4
-
-static void __init clkgena_c65_pll_setup(struct device_node *np)
-{
-	const int num_pll_outputs = 3;
-	struct clk_onecell_data *clk_data;
-	const char *parent_name;
-	void __iomem *reg;
-	const char *clk_name;
-
-	parent_name = of_clk_get_parent_name(np, 0);
-	if (!parent_name)
-		return;
-
-	reg = clkgen_get_register_base(np);
-	if (!reg)
-		return;
-
-	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
-	if (!clk_data)
-		return;
-
-	clk_data->clk_num = num_pll_outputs;
-	clk_data->clks = kzalloc(clk_data->clk_num * sizeof(struct clk *),
-				 GFP_KERNEL);
-
-	if (!clk_data->clks)
-		goto err;
-
-	if (of_property_read_string_index(np, "clock-output-names",
-					  0, &clk_name))
-		goto err;
-
-	/*
-	 * PLL0 HS (high speed) output
-	 */
-	clk_data->clks[0] = clkgen_pll_register(parent_name,
-			(struct clkgen_pll_data *) &st_pll1600c65_ax,
-			reg + CLKGENAx_PLL0_OFFSET, 0, clk_name, NULL);
-
-	if (IS_ERR(clk_data->clks[0]))
-		goto err;
-
-	if (of_property_read_string_index(np, "clock-output-names",
-					  1, &clk_name))
-		goto err;
-
-	/*
-	 * PLL0 LS (low speed) output, which is a fixed divide by 2 of the
-	 * high speed output.
-	 */
-	clk_data->clks[1] = clkgen_c65_lsdiv_register(__clk_get_name
-						      (clk_data->clks[0]),
-						      clk_name);
-
-	if (IS_ERR(clk_data->clks[1]))
-		goto err;
-
-	if (of_property_read_string_index(np, "clock-output-names",
-					  2, &clk_name))
-		goto err;
-
-	/*
-	 * PLL1 output
-	 */
-	clk_data->clks[2] = clkgen_pll_register(parent_name,
-			(struct clkgen_pll_data *) &st_pll800c65_ax,
-			reg + CLKGENAx_PLL1_OFFSET, 0, clk_name, NULL);
-
-	if (IS_ERR(clk_data->clks[2]))
-		goto err;
-
-	of_clk_add_provider(np, of_clk_src_onecell_get, clk_data);
-	return;
-
-err:
-	kfree(clk_data->clks);
-	kfree(clk_data);
-}
-CLK_OF_DECLARE(clkgena_c65_plls,
-	       "st,clkgena-plls-c65", clkgena_c65_pll_setup);
-
 static struct clk * __init clkgen_odf_register(const char *parent_name,
 					       void __iomem *reg,
 					       struct clkgen_pll_data *pll_data,
@@ -1042,72 +702,17 @@ static struct clk * __init clkgen_odf_register(const char *parent_name,
 	return clk;
 }
 
-static const struct of_device_id c32_pll_of_match[] = {
-	{
-		.compatible = "st,plls-c32-a1x-0",
-		.data = &st_pll3200c32_a1x_0,
-	},
-	{
-		.compatible = "st,plls-c32-a1x-1",
-		.data = &st_pll3200c32_a1x_1,
-	},
-	{
-		.compatible = "st,stih415-plls-c32-a9",
-		.data = &st_pll3200c32_a9_415,
-	},
-	{
-		.compatible = "st,stih415-plls-c32-ddr",
-		.data = &st_pll3200c32_ddr_415,
-	},
-	{
-		.compatible = "st,stih416-plls-c32-a9",
-		.data = &st_pll3200c32_a9_416,
-	},
-	{
-		.compatible = "st,stih416-plls-c32-ddr",
-		.data = &st_pll3200c32_ddr_416,
-	},
-	{
-		.compatible = "st,stih407-plls-c32-a0",
-		.data = &st_pll3200c32_407_a0,
-	},
-	{
-		.compatible = "st,plls-c32-cx_0",
-		.data = &st_pll3200c32_cx_0,
-	},
-	{
-		.compatible = "st,plls-c32-cx_1",
-		.data = &st_pll3200c32_cx_1,
-	},
-	{
-		.compatible = "st,stih407-plls-c32-a9",
-		.data = &st_pll3200c32_407_a9,
-	},
-	{
-		.compatible = "st,stih418-plls-c28-a9",
-		.data = &st_pll4600c28_418_a9,
-	},
-	{}
-};
 
-static void __init clkgen_c32_pll_setup(struct device_node *np)
+static void __init clkgen_c32_pll_setup(struct device_node *np,
+		struct clkgen_pll_data *data)
 {
-	const struct of_device_id *match;
 	struct clk *clk;
 	const char *parent_name, *pll_name;
 	void __iomem *pll_base;
 	int num_odfs, odf;
 	struct clk_onecell_data *clk_data;
-	struct clkgen_pll_data	*data;
 	unsigned long pll_flags = 0;
 
-	match = of_match_node(c32_pll_of_match, np);
-	if (!match) {
-		pr_err("%s: No matching data\n", __func__);
-		return;
-	}
-
-	data = (struct clkgen_pll_data *) match->data;
 
 	parent_name = of_clk_get_parent_name(np, 0);
 	if (!parent_name)
@@ -1166,59 +771,30 @@ err:
 	kfree(clk_data->clks);
 	kfree(clk_data);
 }
-CLK_OF_DECLARE(clkgen_c32_pll, "st,clkgen-plls-c32", clkgen_c32_pll_setup);
-
-static const struct of_device_id c32_gpu_pll_of_match[] = {
-	{
-		.compatible = "st,stih415-gpu-pll-c32",
-		.data = &st_pll1200c32_gpu_415,
-	},
-	{
-		.compatible = "st,stih416-gpu-pll-c32",
-		.data = &st_pll1200c32_gpu_416,
-	},
-	{}
-};
-
-static void __init clkgengpu_c32_pll_setup(struct device_node *np)
+static void __init clkgen_c32_pll0_setup(struct device_node *np)
 {
-	const struct of_device_id *match;
-	struct clk *clk;
-	const char *parent_name;
-	void __iomem *reg;
-	const char *clk_name;
-	struct clkgen_pll_data	*data;
-
-	match = of_match_node(c32_gpu_pll_of_match, np);
-	if (!match) {
-		pr_err("%s: No matching data\n", __func__);
-		return;
-	}
-
-	data = (struct clkgen_pll_data *)match->data;
-
-	parent_name = of_clk_get_parent_name(np, 0);
-	if (!parent_name)
-		return;
-
-	reg = clkgen_get_register_base(np);
-	if (!reg)
-		return;
-
-	if (of_property_read_string_index(np, "clock-output-names",
-					  0, &clk_name))
-		return;
+	clkgen_c32_pll_setup(np,
+			(struct clkgen_pll_data *) &st_pll3200c32_cx_0);
+}
+CLK_OF_DECLARE(c32_pll0, "st,clkgen-pll0", clkgen_c32_pll0_setup);
 
-	/*
-	 * PLL 1200MHz output
-	 */
-	clk = clkgen_pll_register(parent_name, data, reg,
-				  0, clk_name, data->lock);
+static void __init clkgen_c32_pll1_setup(struct device_node *np)
+{
+	clkgen_c32_pll_setup(np,
+			(struct clkgen_pll_data *) &st_pll3200c32_cx_1);
+}
+CLK_OF_DECLARE(c32_pll1, "st,clkgen-pll1", clkgen_c32_pll1_setup);
 
-	if (!IS_ERR(clk))
-		of_clk_add_provider(np, of_clk_src_simple_get, clk);
+static void __init clkgen_c32_plla9_setup(struct device_node *np)
+{
+	clkgen_c32_pll_setup(np,
+			(struct clkgen_pll_data *) &st_pll3200c32_407_a9);
+}
+CLK_OF_DECLARE(c32_plla9, "st,stih407-clkgen-plla9", clkgen_c32_plla9_setup);
 
-	return;
+static void __init clkgen_c28_plla9_setup(struct device_node *np)
+{
+	clkgen_c32_pll_setup(np,
+			(struct clkgen_pll_data *) &st_pll4600c28_418_a9);
 }
-CLK_OF_DECLARE(clkgengpu_c32_pll,
-	       "st,clkgengpu-pll-c32", clkgengpu_c32_pll_setup);
+CLK_OF_DECLARE(c28_plla9, "st,stih418-clkgen-plla9", clkgen_c28_plla9_setup);
diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig
index 2afcbd39e41e..254d9526c018 100644
--- a/drivers/clk/sunxi-ng/Kconfig
+++ b/drivers/clk/sunxi-ng/Kconfig
@@ -1,5 +1,6 @@
 config SUNXI_CCU
 	bool "Clock support for Allwinner SoCs"
+	depends on ARCH_SUNXI || COMPILE_TEST
 	default ARCH_SUNXI
 
 if SUNXI_CCU
@@ -19,6 +20,10 @@ config SUNXI_CCU_GATE
 config SUNXI_CCU_MUX
 	bool
 
+config SUNXI_CCU_MULT
+	bool
+	select SUNXI_CCU_MUX
+
 config SUNXI_CCU_PHASE
 	bool
 
@@ -51,6 +56,40 @@ config SUNXI_CCU_MP
 
 # SoC Drivers
 
+config SUN6I_A31_CCU
+	bool "Support for the Allwinner A31/A31s CCU"
+	select SUNXI_CCU_DIV
+	select SUNXI_CCU_NK
+	select SUNXI_CCU_NKM
+	select SUNXI_CCU_NM
+	select SUNXI_CCU_MP
+	select SUNXI_CCU_PHASE
+	default MACH_SUN6I
+
+config SUN8I_A23_CCU
+	bool "Support for the Allwinner A23 CCU"
+	select SUNXI_CCU_DIV
+	select SUNXI_CCU_MULT
+	select SUNXI_CCU_NK
+	select SUNXI_CCU_NKM
+	select SUNXI_CCU_NKMP
+	select SUNXI_CCU_NM
+	select SUNXI_CCU_MP
+	select SUNXI_CCU_PHASE
+	default MACH_SUN8I
+
+config SUN8I_A33_CCU
+	bool "Support for the Allwinner A33 CCU"
+	select SUNXI_CCU_DIV
+	select SUNXI_CCU_MULT
+	select SUNXI_CCU_NK
+	select SUNXI_CCU_NKM
+	select SUNXI_CCU_NKMP
+	select SUNXI_CCU_NM
+	select SUNXI_CCU_MP
+	select SUNXI_CCU_PHASE
+	default MACH_SUN8I
+
 config SUN8I_H3_CCU
 	bool "Support for the Allwinner H3 CCU"
 	select SUNXI_CCU_DIV
diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile
index 633ce642ffae..106cba27c331 100644
--- a/drivers/clk/sunxi-ng/Makefile
+++ b/drivers/clk/sunxi-ng/Makefile
@@ -7,6 +7,7 @@ obj-$(CONFIG_SUNXI_CCU_DIV)	+= ccu_div.o
 obj-$(CONFIG_SUNXI_CCU_FRAC)	+= ccu_frac.o
 obj-$(CONFIG_SUNXI_CCU_GATE)	+= ccu_gate.o
 obj-$(CONFIG_SUNXI_CCU_MUX)	+= ccu_mux.o
+obj-$(CONFIG_SUNXI_CCU_MULT)	+= ccu_mult.o
 obj-$(CONFIG_SUNXI_CCU_PHASE)	+= ccu_phase.o
 
 # Multi-factor clocks
@@ -17,4 +18,7 @@ obj-$(CONFIG_SUNXI_CCU_NM)	+= ccu_nm.o
 obj-$(CONFIG_SUNXI_CCU_MP)	+= ccu_mp.o
 
 # SoC support
+obj-$(CONFIG_SUN6I_A31_CCU)	+= ccu-sun6i-a31.o
+obj-$(CONFIG_SUN8I_A23_CCU)	+= ccu-sun8i-a23.o
+obj-$(CONFIG_SUN8I_A33_CCU)	+= ccu-sun8i-a33.o
 obj-$(CONFIG_SUN8I_H3_CCU)	+= ccu-sun8i-h3.o
diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
new file mode 100644
index 000000000000..79596463e0d9
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
@@ -0,0 +1,1239 @@
+/*
+ * Copyright (c) 2016 Chen-Yu Tsai
+ *
+ * Chen-Yu Tsai <wens@csie.org>
+ *
+ * Based on ccu-sun8i-h3.c by Maxime Ripard.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+
+#include "ccu_common.h"
+#include "ccu_reset.h"
+
+#include "ccu_div.h"
+#include "ccu_gate.h"
+#include "ccu_mp.h"
+#include "ccu_mult.h"
+#include "ccu_mux.h"
+#include "ccu_nk.h"
+#include "ccu_nkm.h"
+#include "ccu_nkmp.h"
+#include "ccu_nm.h"
+#include "ccu_phase.h"
+
+#include "ccu-sun6i-a31.h"
+
+static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_cpu_clk, "pll-cpu",
+				     "osc24M", 0x000,
+				     8, 5,	/* N */
+				     4, 2,	/* K */
+				     0, 2,	/* M */
+				     BIT(31),	/* gate */
+				     BIT(28),	/* lock */
+				     0);
+
+/*
+ * The Audio PLL is supposed to have 4 outputs: 3 fixed factors from
+ * the base (2x, 4x and 8x), and one variable divider (the one true
+ * pll audio).
+ *
+ * We don't have any need for the variable divider for now, so we just
+ * hardcode it to match with the clock names
+ */
+#define SUN6I_A31_PLL_AUDIO_REG	0x008
+
+static SUNXI_CCU_NM_WITH_GATE_LOCK(pll_audio_base_clk, "pll-audio-base",
+				   "osc24M", 0x008,
+				   8, 7,	/* N */
+				   0, 5,	/* M */
+				   BIT(31),	/* gate */
+				   BIT(28),	/* lock */
+				   CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_video0_clk, "pll-video0",
+					"osc24M", 0x010,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_ve_clk, "pll-ve",
+					"osc24M", 0x018,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_ddr_clk, "pll-ddr",
+				    "osc24M", 0x020,
+				    8, 5,	/* N */
+				    4, 2,	/* K */
+				    0, 2,	/* M */
+				    BIT(31),	/* gate */
+				    BIT(28),	/* lock */
+				    CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NK_WITH_GATE_LOCK_POSTDIV(pll_periph_clk, "pll-periph",
+					   "osc24M", 0x028,
+					   8, 5,	/* N */
+					   4, 2,	/* K */
+					   BIT(31),	/* gate */
+					   BIT(28),	/* lock */
+					   2,		/* post-div */
+					   CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_video1_clk, "pll-video1",
+					"osc24M", 0x030,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_gpu_clk, "pll-gpu",
+					"osc24M", 0x038,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+/*
+ * The MIPI PLL has 2 modes: "MIPI" and "HDMI".
+ *
+ * The MIPI mode is a standard NKM-style clock. The HDMI mode is an
+ * integer / fractional clock with switchable multipliers and dividers.
+ * This is not supported here. We hardcode the PLL to MIPI mode.
+ */
+#define SUN6I_A31_PLL_MIPI_REG	0x040
+
+static const char * const pll_mipi_parents[] = { "pll-video0", "pll-video1" };
+static SUNXI_CCU_NKM_WITH_MUX_GATE_LOCK(pll_mipi_clk, "pll-mipi",
+					pll_mipi_parents, 0x040,
+					8, 4,	/* N */
+					4, 2,	/* K */
+					0, 4,	/* M */
+					21, 0,	/* mux */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll9_clk, "pll9",
+					"osc24M", 0x044,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll10_clk, "pll10",
+					"osc24M", 0x048,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static const char * const cpux_parents[] = { "osc32k", "osc24M",
+					     "pll-cpu", "pll-cpu" };
+static SUNXI_CCU_MUX(cpu_clk, "cpu", cpux_parents,
+		     0x050, 16, 2, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL);
+
+static struct clk_div_table axi_div_table[] = {
+	{ .val = 0, .div = 1 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 3 },
+	{ .val = 3, .div = 4 },
+	{ .val = 4, .div = 4 },
+	{ .val = 5, .div = 4 },
+	{ .val = 6, .div = 4 },
+	{ .val = 7, .div = 4 },
+	{ /* Sentinel */ },
+};
+
+static SUNXI_CCU_DIV_TABLE(axi_clk, "axi", "cpu",
+			   0x050, 0, 3, axi_div_table, 0);
+
+static const char * const ahb1_parents[] = { "osc32k", "osc24M",
+					     "axi", "pll-periph" };
+
+static struct ccu_div ahb1_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO),
+
+	.mux		= {
+		.shift	= 12,
+		.width	= 2,
+
+		.variable_prediv	= {
+			.index	= 3,
+			.shift	= 6,
+			.width	= 2,
+		},
+	},
+
+	.common		= {
+		.reg		= 0x054,
+		.features	= CCU_FEATURE_VARIABLE_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("ahb1",
+						      ahb1_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct clk_div_table apb1_div_table[] = {
+	{ .val = 0, .div = 2 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 4 },
+	{ .val = 3, .div = 8 },
+	{ /* Sentinel */ },
+};
+
+static SUNXI_CCU_DIV_TABLE(apb1_clk, "apb1", "ahb1",
+			   0x054, 8, 2, apb1_div_table, 0);
+
+static const char * const apb2_parents[] = { "osc32k", "osc24M",
+					     "pll-periph", "pll-periph" };
+static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", apb2_parents, 0x058,
+			     0, 5,	/* M */
+			     16, 2,	/* P */
+			     24, 2,	/* mux */
+			     0);
+
+static SUNXI_CCU_GATE(ahb1_mipidsi_clk,	"ahb1-mipidsi",	"ahb1",
+		      0x060, BIT(1), 0);
+static SUNXI_CCU_GATE(ahb1_ss_clk,	"ahb1-ss",	"ahb1",
+		      0x060, BIT(5), 0);
+static SUNXI_CCU_GATE(ahb1_dma_clk,	"ahb1-dma",	"ahb1",
+		      0x060, BIT(6), 0);
+static SUNXI_CCU_GATE(ahb1_mmc0_clk,	"ahb1-mmc0",	"ahb1",
+		      0x060, BIT(8), 0);
+static SUNXI_CCU_GATE(ahb1_mmc1_clk,	"ahb1-mmc1",	"ahb1",
+		      0x060, BIT(9), 0);
+static SUNXI_CCU_GATE(ahb1_mmc2_clk,	"ahb1-mmc2",	"ahb1",
+		      0x060, BIT(10), 0);
+static SUNXI_CCU_GATE(ahb1_mmc3_clk,	"ahb1-mmc3",	"ahb1",
+		      0x060, BIT(12), 0);
+static SUNXI_CCU_GATE(ahb1_nand1_clk,	"ahb1-nand1",	"ahb1",
+		      0x060, BIT(13), 0);
+static SUNXI_CCU_GATE(ahb1_nand0_clk,	"ahb1-nand0",	"ahb1",
+		      0x060, BIT(13), 0);
+static SUNXI_CCU_GATE(ahb1_sdram_clk,	"ahb1-sdram",	"ahb1",
+		      0x060, BIT(14), 0);
+static SUNXI_CCU_GATE(ahb1_emac_clk,	"ahb1-emac",	"ahb1",
+		      0x060, BIT(17), 0);
+static SUNXI_CCU_GATE(ahb1_ts_clk,	"ahb1-ts",	"ahb1",
+		      0x060, BIT(18), 0);
+static SUNXI_CCU_GATE(ahb1_hstimer_clk,	"ahb1-hstimer",	"ahb1",
+		      0x060, BIT(19), 0);
+static SUNXI_CCU_GATE(ahb1_spi0_clk,	"ahb1-spi0",	"ahb1",
+		      0x060, BIT(20), 0);
+static SUNXI_CCU_GATE(ahb1_spi1_clk,	"ahb1-spi1",	"ahb1",
+		      0x060, BIT(21), 0);
+static SUNXI_CCU_GATE(ahb1_spi2_clk,	"ahb1-spi2",	"ahb1",
+		      0x060, BIT(22), 0);
+static SUNXI_CCU_GATE(ahb1_spi3_clk,	"ahb1-spi3",	"ahb1",
+		      0x060, BIT(23), 0);
+static SUNXI_CCU_GATE(ahb1_otg_clk,	"ahb1-otg",	"ahb1",
+		      0x060, BIT(24), 0);
+static SUNXI_CCU_GATE(ahb1_ehci0_clk,	"ahb1-ehci0",	"ahb1",
+		      0x060, BIT(26), 0);
+static SUNXI_CCU_GATE(ahb1_ehci1_clk,	"ahb1-ehci1",	"ahb1",
+		      0x060, BIT(27), 0);
+static SUNXI_CCU_GATE(ahb1_ohci0_clk,	"ahb1-ohci0",	"ahb1",
+		      0x060, BIT(29), 0);
+static SUNXI_CCU_GATE(ahb1_ohci1_clk,	"ahb1-ohci1",	"ahb1",
+		      0x060, BIT(30), 0);
+static SUNXI_CCU_GATE(ahb1_ohci2_clk,	"ahb1-ohci2",	"ahb1",
+		      0x060, BIT(31), 0);
+
+static SUNXI_CCU_GATE(ahb1_ve_clk,	"ahb1-ve",	"ahb1",
+		      0x064, BIT(0), 0);
+static SUNXI_CCU_GATE(ahb1_lcd0_clk,	"ahb1-lcd0",	"ahb1",
+		      0x064, BIT(4), 0);
+static SUNXI_CCU_GATE(ahb1_lcd1_clk,	"ahb1-lcd1",	"ahb1",
+		      0x064, BIT(5), 0);
+static SUNXI_CCU_GATE(ahb1_csi_clk,	"ahb1-csi",	"ahb1",
+		      0x064, BIT(8), 0);
+static SUNXI_CCU_GATE(ahb1_hdmi_clk,	"ahb1-hdmi",	"ahb1",
+		      0x064, BIT(11), 0);
+static SUNXI_CCU_GATE(ahb1_be0_clk,	"ahb1-be0",	"ahb1",
+		      0x064, BIT(12), 0);
+static SUNXI_CCU_GATE(ahb1_be1_clk,	"ahb1-be1",	"ahb1",
+		      0x064, BIT(13), 0);
+static SUNXI_CCU_GATE(ahb1_fe0_clk,	"ahb1-fe0",	"ahb1",
+		      0x064, BIT(14), 0);
+static SUNXI_CCU_GATE(ahb1_fe1_clk,	"ahb1-fe1",	"ahb1",
+		      0x064, BIT(15), 0);
+static SUNXI_CCU_GATE(ahb1_mp_clk,	"ahb1-mp",	"ahb1",
+		      0x064, BIT(18), 0);
+static SUNXI_CCU_GATE(ahb1_gpu_clk,	"ahb1-gpu",	"ahb1",
+		      0x064, BIT(20), 0);
+static SUNXI_CCU_GATE(ahb1_deu0_clk,	"ahb1-deu0",	"ahb1",
+		      0x064, BIT(23), 0);
+static SUNXI_CCU_GATE(ahb1_deu1_clk,	"ahb1-deu1",	"ahb1",
+		      0x064, BIT(24), 0);
+static SUNXI_CCU_GATE(ahb1_drc0_clk,	"ahb1-drc0",	"ahb1",
+		      0x064, BIT(25), 0);
+static SUNXI_CCU_GATE(ahb1_drc1_clk,	"ahb1-drc1",	"ahb1",
+		      0x064, BIT(26), 0);
+
+static SUNXI_CCU_GATE(apb1_codec_clk,	"apb1-codec",	"apb1",
+		      0x068, BIT(0), 0);
+static SUNXI_CCU_GATE(apb1_spdif_clk,	"apb1-spdif",	"apb1",
+		      0x068, BIT(1), 0);
+static SUNXI_CCU_GATE(apb1_digital_mic_clk,	"apb1-digital-mic",	"apb1",
+		      0x068, BIT(4), 0);
+static SUNXI_CCU_GATE(apb1_pio_clk,	"apb1-pio",	"apb1",
+		      0x068, BIT(5), 0);
+static SUNXI_CCU_GATE(apb1_daudio0_clk,	"apb1-daudio0",	"apb1",
+		      0x068, BIT(12), 0);
+static SUNXI_CCU_GATE(apb1_daudio1_clk,	"apb1-daudio1",	"apb1",
+		      0x068, BIT(13), 0);
+
+static SUNXI_CCU_GATE(apb2_i2c0_clk,	"apb2-i2c0",	"apb2",
+		      0x06c, BIT(0), 0);
+static SUNXI_CCU_GATE(apb2_i2c1_clk,	"apb2-i2c1",	"apb2",
+		      0x06c, BIT(1), 0);
+static SUNXI_CCU_GATE(apb2_i2c2_clk,	"apb2-i2c2",	"apb2",
+		      0x06c, BIT(2), 0);
+static SUNXI_CCU_GATE(apb2_i2c3_clk,	"apb2-i2c3",	"apb2",
+		      0x06c, BIT(3), 0);
+static SUNXI_CCU_GATE(apb2_uart0_clk,	"apb2-uart0",	"apb2",
+		      0x06c, BIT(16), 0);
+static SUNXI_CCU_GATE(apb2_uart1_clk,	"apb2-uart1",	"apb2",
+		      0x06c, BIT(17), 0);
+static SUNXI_CCU_GATE(apb2_uart2_clk,	"apb2-uart2",	"apb2",
+		      0x06c, BIT(18), 0);
+static SUNXI_CCU_GATE(apb2_uart3_clk,	"apb2-uart3",	"apb2",
+		      0x06c, BIT(19), 0);
+static SUNXI_CCU_GATE(apb2_uart4_clk,	"apb2-uart4",	"apb2",
+		      0x06c, BIT(20), 0);
+static SUNXI_CCU_GATE(apb2_uart5_clk,	"apb2-uart5",	"apb2",
+		      0x06c, BIT(21), 0);
+
+static const char * const mod0_default_parents[] = { "osc24M", "pll-periph" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(nand0_clk, "nand0", mod0_default_parents,
+				  0x080,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(nand1_clk, "nand1", mod0_default_parents,
+				  0x084,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc0_clk, "mmc0", mod0_default_parents,
+				  0x088,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc0_sample_clk, "mmc0_sample", "mmc0",
+		       0x088, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc0_output_clk, "mmc0_output", "mmc0",
+		       0x088, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc1_clk, "mmc1", mod0_default_parents,
+				  0x08c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc1_sample_clk, "mmc1_sample", "mmc1",
+		       0x08c, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc1_output_clk, "mmc1_output", "mmc1",
+		       0x08c, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc2_clk, "mmc2", mod0_default_parents,
+				  0x090,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc2_sample_clk, "mmc2_sample", "mmc2",
+		       0x090, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc2_output_clk, "mmc2_output", "mmc2",
+		       0x090, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc3_clk, "mmc3", mod0_default_parents,
+				  0x094,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc3_sample_clk, "mmc3_sample", "mmc3",
+		       0x094, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc3_output_clk, "mmc3_output", "mmc3",
+		       0x094, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(ts_clk, "ts", mod0_default_parents, 0x098,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(ss_clk, "ss", mod0_default_parents, 0x09c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", mod0_default_parents, 0x0a0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi2_clk, "spi2", mod0_default_parents, 0x0a8,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi3_clk, "spi3", mod0_default_parents, 0x0ac,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const daudio_parents[] = { "pll-audio-8x", "pll-audio-4x",
+					       "pll-audio-2x", "pll-audio" };
+static SUNXI_CCU_MUX_WITH_GATE(daudio0_clk, "daudio0", daudio_parents,
+			       0x0b0, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+static SUNXI_CCU_MUX_WITH_GATE(daudio1_clk, "daudio1", daudio_parents,
+			       0x0b4, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_M_WITH_GATE(spdif_clk, "spdif", "pll-audio",
+			     0x0c0, 0, 4, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(usb_phy0_clk,	"usb-phy0",	"osc24M",
+		      0x0cc, BIT(8), 0);
+static SUNXI_CCU_GATE(usb_phy1_clk,	"usb-phy1",	"osc24M",
+		      0x0cc, BIT(9), 0);
+static SUNXI_CCU_GATE(usb_phy2_clk,	"usb-phy2",	"osc24M",
+		      0x0cc, BIT(10), 0);
+static SUNXI_CCU_GATE(usb_ohci0_clk,	"usb-ohci0",	"osc24M",
+		      0x0cc, BIT(16), 0);
+static SUNXI_CCU_GATE(usb_ohci1_clk,	"usb-ohci1",	"osc24M",
+		      0x0cc, BIT(17), 0);
+static SUNXI_CCU_GATE(usb_ohci2_clk,	"usb-ohci2",	"osc24M",
+		      0x0cc, BIT(18), 0);
+
+/* TODO emac clk not supported yet */
+
+static const char * const dram_parents[] = { "pll-ddr", "pll-periph" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(mdfs_clk, "mdfs", dram_parents, 0x0f0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  CLK_IS_CRITICAL);
+
+static SUNXI_CCU_M_WITH_MUX(sdram0_clk, "sdram0", dram_parents,
+			    0x0f4, 0, 4, 4, 1, CLK_IS_CRITICAL);
+static SUNXI_CCU_M_WITH_MUX(sdram1_clk, "sdram1", dram_parents,
+			    0x0f4, 8, 4, 12, 1, CLK_IS_CRITICAL);
+
+static SUNXI_CCU_GATE(dram_ve_clk,	"dram-ve",	"mdfs",
+		      0x100, BIT(0), 0);
+static SUNXI_CCU_GATE(dram_csi_isp_clk,	"dram-csi-isp",	"mdfs",
+		      0x100, BIT(1), 0);
+static SUNXI_CCU_GATE(dram_ts_clk,	"dram-ts",	"mdfs",
+		      0x100, BIT(3), 0);
+static SUNXI_CCU_GATE(dram_drc0_clk,	"dram-drc0",	"mdfs",
+		      0x100, BIT(16), 0);
+static SUNXI_CCU_GATE(dram_drc1_clk,	"dram-drc1",	"mdfs",
+		      0x100, BIT(17), 0);
+static SUNXI_CCU_GATE(dram_deu0_clk,	"dram-deu0",	"mdfs",
+		      0x100, BIT(18), 0);
+static SUNXI_CCU_GATE(dram_deu1_clk,	"dram-deu1",	"mdfs",
+		      0x100, BIT(19), 0);
+static SUNXI_CCU_GATE(dram_fe0_clk,	"dram-fe0",	"mdfs",
+		      0x100, BIT(24), 0);
+static SUNXI_CCU_GATE(dram_fe1_clk,	"dram-fe1",	"mdfs",
+		      0x100, BIT(25), 0);
+static SUNXI_CCU_GATE(dram_be0_clk,	"dram-be0",	"mdfs",
+		      0x100, BIT(26), 0);
+static SUNXI_CCU_GATE(dram_be1_clk,	"dram-be1",	"mdfs",
+		      0x100, BIT(27), 0);
+static SUNXI_CCU_GATE(dram_mp_clk,	"dram-mp",	"mdfs",
+		      0x100, BIT(28), 0);
+
+static const char * const de_parents[] = { "pll-video0", "pll-video1",
+					   "pll-periph-2x", "pll-gpu",
+					   "pll9", "pll10" };
+static SUNXI_CCU_M_WITH_MUX_GATE(be0_clk, "be0", de_parents,
+				 0x104, 0, 4, 24, 3, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(be1_clk, "be1", de_parents,
+				 0x108, 0, 4, 24, 3, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(fe0_clk, "fe0", de_parents,
+				 0x10c, 0, 4, 24, 3, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(fe1_clk, "fe1", de_parents,
+				 0x110, 0, 4, 24, 3, BIT(31), 0);
+
+static const char * const mp_parents[] = { "pll-video0", "pll-video1",
+					   "pll9", "pll10" };
+static SUNXI_CCU_M_WITH_MUX_GATE(mp_clk, "mp", mp_parents,
+				 0x114, 0, 4, 24, 3, BIT(31), 0);
+
+static const char * const lcd_ch0_parents[] = { "pll-video0", "pll-video1",
+						"pll-video0-2x",
+						"pll-video1-2x", "pll-mipi" };
+static SUNXI_CCU_MUX_WITH_GATE(lcd0_ch0_clk, "lcd0-ch0", lcd_ch0_parents,
+			       0x118, 24, 2, BIT(31), CLK_SET_RATE_PARENT);
+static SUNXI_CCU_MUX_WITH_GATE(lcd1_ch0_clk, "lcd1-ch0", lcd_ch0_parents,
+			       0x11c, 24, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static const char * const lcd_ch1_parents[] = { "pll-video0", "pll-video1",
+						"pll-video0-2x",
+						"pll-video1-2x" };
+static SUNXI_CCU_M_WITH_MUX_GATE(lcd0_ch1_clk, "lcd0-ch1", lcd_ch1_parents,
+				 0x12c, 0, 4, 24, 3, BIT(31),
+				 CLK_SET_RATE_PARENT);
+static SUNXI_CCU_M_WITH_MUX_GATE(lcd1_ch1_clk, "lcd1-ch1", lcd_ch1_parents,
+				 0x12c, 0, 4, 24, 3, BIT(31),
+				 CLK_SET_RATE_PARENT);
+
+static const char * const csi_sclk_parents[] = { "pll-video0", "pll-video1",
+						 "pll9", "pll10", "pll-mipi",
+						 "pll-ve" };
+static SUNXI_CCU_M_WITH_MUX_GATE(csi0_sclk_clk, "csi0-sclk", csi_sclk_parents,
+				 0x134, 16, 4, 24, 3, BIT(31), 0);
+
+static const char * const csi_mclk_parents[] = { "pll-video0", "pll-video1",
+						 "osc24M" };
+static const u8 csi_mclk_table[] = { 0, 1, 5 };
+static struct ccu_div csi0_mclk_clk = {
+	.enable		= BIT(15),
+	.div		= _SUNXI_CCU_DIV(0, 4),
+	.mux		= _SUNXI_CCU_MUX_TABLE(8, 3, csi_mclk_table),
+	.common		= {
+		.reg		= 0x134,
+		.hw.init	= CLK_HW_INIT_PARENTS("csi0-mclk",
+						      csi_mclk_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct ccu_div csi1_mclk_clk = {
+	.enable		= BIT(15),
+	.div		= _SUNXI_CCU_DIV(0, 4),
+	.mux		= _SUNXI_CCU_MUX_TABLE(8, 3, csi_mclk_table),
+	.common		= {
+		.reg		= 0x138,
+		.hw.init	= CLK_HW_INIT_PARENTS("csi1-mclk",
+						      csi_mclk_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve",
+			     0x13c, 16, 3, BIT(31), 0);
+
+static SUNXI_CCU_GATE(codec_clk,	"codec",	"pll-audio",
+		      0x140, BIT(31), CLK_SET_RATE_PARENT);
+static SUNXI_CCU_GATE(avs_clk,		"avs",		"osc24M",
+		      0x144, BIT(31), 0);
+static SUNXI_CCU_GATE(digital_mic_clk,	"digital-mic",	"pll-audio",
+		      0x148, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(hdmi_clk, "hdmi", lcd_ch1_parents,
+				 0x150, 0, 4, 24, 2, BIT(31),
+				 CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(hdmi_ddc_clk, "hdmi-ddc", "osc24M", 0x150, BIT(31), 0);
+
+static SUNXI_CCU_GATE(ps_clk, "ps", "lcd1-ch1", 0x140, BIT(31), 0);
+
+static const char * const mbus_parents[] = { "osc24M", "pll-periph",
+					     "pll-ddr" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(mbus0_clk, "mbus0", mbus_parents, 0x15c,
+				  0, 3,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  CLK_IS_CRITICAL);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mbus1_clk, "mbus1", mbus_parents, 0x160,
+				  0, 3,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  CLK_IS_CRITICAL);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(mipi_dsi_clk, "mipi-dsi", lcd_ch1_parents,
+				 0x168, 16, 3, 24, 2, BIT(31),
+				 CLK_SET_RATE_PARENT);
+static SUNXI_CCU_M_WITH_MUX_GATE(mipi_dsi_dphy_clk, "mipi-dsi-dphy",
+				 lcd_ch1_parents, 0x168, 0, 3, 8, 2,
+				 BIT(15), CLK_SET_RATE_PARENT);
+static SUNXI_CCU_M_WITH_MUX_GATE(mipi_csi_dphy_clk, "mipi-csi-dphy",
+				 lcd_ch1_parents, 0x16c, 0, 3, 8, 2,
+				 BIT(15), 0);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(iep_drc0_clk, "iep-drc0", de_parents,
+				 0x180, 0, 3, 24, 2, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(iep_drc1_clk, "iep-drc1", de_parents,
+				 0x184, 0, 3, 24, 2, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(iep_deu0_clk, "iep-deu0", de_parents,
+				 0x188, 0, 3, 24, 2, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(iep_deu1_clk, "iep-deu1", de_parents,
+				 0x18c, 0, 3, 24, 2, BIT(31), 0);
+
+static const char * const gpu_parents[] = { "pll-gpu", "pll-periph-2x",
+					    "pll-video0", "pll-video1",
+					    "pll9", "pll10" };
+static const struct ccu_mux_fixed_prediv gpu_predivs[] = {
+	{ .index = 1, .div = 3, },
+};
+
+static struct ccu_div gpu_core_clk = {
+	.enable		= BIT(31),
+	.div		= _SUNXI_CCU_DIV(0, 3),
+	.mux		= {
+		.shift		= 24,
+		.width		= 3,
+		.fixed_predivs	= gpu_predivs,
+		.n_predivs	= ARRAY_SIZE(gpu_predivs),
+	},
+	.common		= {
+		.reg		= 0x1a0,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("gpu-core",
+						      gpu_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct ccu_div gpu_memory_clk = {
+	.enable		= BIT(31),
+	.div		= _SUNXI_CCU_DIV(0, 3),
+	.mux		= {
+		.shift		= 24,
+		.width		= 3,
+		.fixed_predivs	= gpu_predivs,
+		.n_predivs	= ARRAY_SIZE(gpu_predivs),
+	},
+	.common		= {
+		.reg		= 0x1a4,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("gpu-memory",
+						      gpu_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct ccu_div gpu_hyd_clk = {
+	.enable		= BIT(31),
+	.div		= _SUNXI_CCU_DIV(0, 3),
+	.mux		= {
+		.shift		= 24,
+		.width		= 3,
+		.fixed_predivs	= gpu_predivs,
+		.n_predivs	= ARRAY_SIZE(gpu_predivs),
+	},
+	.common		= {
+		.reg		= 0x1a8,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("gpu-hyd",
+						      gpu_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static SUNXI_CCU_M_WITH_MUX_GATE(ats_clk, "ats", mod0_default_parents, 0x1b0,
+				 0, 3,		/* M */
+				 24, 2,		/* mux */
+				 BIT(31),	/* gate */
+				 0);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(trace_clk, "trace", mod0_default_parents,
+				 0x1b0,
+				 0, 3,		/* M */
+				 24, 2,		/* mux */
+				 BIT(31),	/* gate */
+				 0);
+
+static const char * const clk_out_parents[] = { "osc24M", "osc32k", "osc24M",
+						"axi", "ahb1" };
+static const u8 clk_out_table[] = { 0, 1, 2, 11, 13 };
+
+static const struct ccu_mux_fixed_prediv clk_out_predivs[] = {
+	{ .index = 0, .div = 750, },
+	{ .index = 3, .div = 4, },
+	{ .index = 4, .div = 4, },
+};
+
+static struct ccu_mp out_a_clk = {
+	.enable		= BIT(31),
+	.m		= _SUNXI_CCU_DIV(8, 5),
+	.p		= _SUNXI_CCU_DIV(20, 2),
+	.mux		= {
+		.shift		= 24,
+		.width		= 4,
+		.table		= clk_out_table,
+		.fixed_predivs	= clk_out_predivs,
+		.n_predivs	= ARRAY_SIZE(clk_out_predivs),
+	},
+	.common		= {
+		.reg		= 0x300,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("out-a",
+						      clk_out_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct ccu_mp out_b_clk = {
+	.enable		= BIT(31),
+	.m		= _SUNXI_CCU_DIV(8, 5),
+	.p		= _SUNXI_CCU_DIV(20, 2),
+	.mux		= {
+		.shift		= 24,
+		.width		= 4,
+		.table		= clk_out_table,
+		.fixed_predivs	= clk_out_predivs,
+		.n_predivs	= ARRAY_SIZE(clk_out_predivs),
+	},
+	.common		= {
+		.reg		= 0x304,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("out-b",
+						      clk_out_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct ccu_mp out_c_clk = {
+	.enable		= BIT(31),
+	.m		= _SUNXI_CCU_DIV(8, 5),
+	.p		= _SUNXI_CCU_DIV(20, 2),
+	.mux		= {
+		.shift		= 24,
+		.width		= 4,
+		.table		= clk_out_table,
+		.fixed_predivs	= clk_out_predivs,
+		.n_predivs	= ARRAY_SIZE(clk_out_predivs),
+	},
+	.common		= {
+		.reg		= 0x308,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("out-c",
+						      clk_out_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct ccu_common *sun6i_a31_ccu_clks[] = {
+	&pll_cpu_clk.common,
+	&pll_audio_base_clk.common,
+	&pll_video0_clk.common,
+	&pll_ve_clk.common,
+	&pll_ddr_clk.common,
+	&pll_periph_clk.common,
+	&pll_video1_clk.common,
+	&pll_gpu_clk.common,
+	&pll_mipi_clk.common,
+	&pll9_clk.common,
+	&pll10_clk.common,
+	&cpu_clk.common,
+	&axi_clk.common,
+	&ahb1_clk.common,
+	&apb1_clk.common,
+	&apb2_clk.common,
+	&ahb1_mipidsi_clk.common,
+	&ahb1_ss_clk.common,
+	&ahb1_dma_clk.common,
+	&ahb1_mmc0_clk.common,
+	&ahb1_mmc1_clk.common,
+	&ahb1_mmc2_clk.common,
+	&ahb1_mmc3_clk.common,
+	&ahb1_nand1_clk.common,
+	&ahb1_nand0_clk.common,
+	&ahb1_sdram_clk.common,
+	&ahb1_emac_clk.common,
+	&ahb1_ts_clk.common,
+	&ahb1_hstimer_clk.common,
+	&ahb1_spi0_clk.common,
+	&ahb1_spi1_clk.common,
+	&ahb1_spi2_clk.common,
+	&ahb1_spi3_clk.common,
+	&ahb1_otg_clk.common,
+	&ahb1_ehci0_clk.common,
+	&ahb1_ehci1_clk.common,
+	&ahb1_ohci0_clk.common,
+	&ahb1_ohci1_clk.common,
+	&ahb1_ohci2_clk.common,
+	&ahb1_ve_clk.common,
+	&ahb1_lcd0_clk.common,
+	&ahb1_lcd1_clk.common,
+	&ahb1_csi_clk.common,
+	&ahb1_hdmi_clk.common,
+	&ahb1_be0_clk.common,
+	&ahb1_be1_clk.common,
+	&ahb1_fe0_clk.common,
+	&ahb1_fe1_clk.common,
+	&ahb1_mp_clk.common,
+	&ahb1_gpu_clk.common,
+	&ahb1_deu0_clk.common,
+	&ahb1_deu1_clk.common,
+	&ahb1_drc0_clk.common,
+	&ahb1_drc1_clk.common,
+	&apb1_codec_clk.common,
+	&apb1_spdif_clk.common,
+	&apb1_digital_mic_clk.common,
+	&apb1_pio_clk.common,
+	&apb1_daudio0_clk.common,
+	&apb1_daudio1_clk.common,
+	&apb2_i2c0_clk.common,
+	&apb2_i2c1_clk.common,
+	&apb2_i2c2_clk.common,
+	&apb2_i2c3_clk.common,
+	&apb2_uart0_clk.common,
+	&apb2_uart1_clk.common,
+	&apb2_uart2_clk.common,
+	&apb2_uart3_clk.common,
+	&apb2_uart4_clk.common,
+	&apb2_uart5_clk.common,
+	&nand0_clk.common,
+	&nand1_clk.common,
+	&mmc0_clk.common,
+	&mmc0_sample_clk.common,
+	&mmc0_output_clk.common,
+	&mmc1_clk.common,
+	&mmc1_sample_clk.common,
+	&mmc1_output_clk.common,
+	&mmc2_clk.common,
+	&mmc2_sample_clk.common,
+	&mmc2_output_clk.common,
+	&mmc3_clk.common,
+	&mmc3_sample_clk.common,
+	&mmc3_output_clk.common,
+	&ts_clk.common,
+	&ss_clk.common,
+	&spi0_clk.common,
+	&spi1_clk.common,
+	&spi2_clk.common,
+	&spi3_clk.common,
+	&daudio0_clk.common,
+	&daudio1_clk.common,
+	&spdif_clk.common,
+	&usb_phy0_clk.common,
+	&usb_phy1_clk.common,
+	&usb_phy2_clk.common,
+	&usb_ohci0_clk.common,
+	&usb_ohci1_clk.common,
+	&usb_ohci2_clk.common,
+	&mdfs_clk.common,
+	&sdram0_clk.common,
+	&sdram1_clk.common,
+	&dram_ve_clk.common,
+	&dram_csi_isp_clk.common,
+	&dram_ts_clk.common,
+	&dram_drc0_clk.common,
+	&dram_drc1_clk.common,
+	&dram_deu0_clk.common,
+	&dram_deu1_clk.common,
+	&dram_fe0_clk.common,
+	&dram_fe1_clk.common,
+	&dram_be0_clk.common,
+	&dram_be1_clk.common,
+	&dram_mp_clk.common,
+	&be0_clk.common,
+	&be1_clk.common,
+	&fe0_clk.common,
+	&fe1_clk.common,
+	&mp_clk.common,
+	&lcd0_ch0_clk.common,
+	&lcd1_ch0_clk.common,
+	&lcd0_ch1_clk.common,
+	&lcd1_ch1_clk.common,
+	&csi0_sclk_clk.common,
+	&csi0_mclk_clk.common,
+	&csi1_mclk_clk.common,
+	&ve_clk.common,
+	&codec_clk.common,
+	&avs_clk.common,
+	&digital_mic_clk.common,
+	&hdmi_clk.common,
+	&hdmi_ddc_clk.common,
+	&ps_clk.common,
+	&mbus0_clk.common,
+	&mbus1_clk.common,
+	&mipi_dsi_clk.common,
+	&mipi_dsi_dphy_clk.common,
+	&mipi_csi_dphy_clk.common,
+	&iep_drc0_clk.common,
+	&iep_drc1_clk.common,
+	&iep_deu0_clk.common,
+	&iep_deu1_clk.common,
+	&gpu_core_clk.common,
+	&gpu_memory_clk.common,
+	&gpu_hyd_clk.common,
+	&ats_clk.common,
+	&trace_clk.common,
+	&out_a_clk.common,
+	&out_b_clk.common,
+	&out_c_clk.common,
+};
+
+/* We hardcode the divider to 4 for now */
+static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
+			"pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
+			"pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
+			"pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
+			"pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_periph_2x_clk, "pll-periph-2x",
+			"pll-periph", 1, 2, 0);
+static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x",
+			"pll-video0", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x",
+			"pll-video1", 1, 2, CLK_SET_RATE_PARENT);
+
+static struct clk_hw_onecell_data sun6i_a31_hw_clks = {
+	.hws	= {
+		[CLK_PLL_CPU]		= &pll_cpu_clk.common.hw,
+		[CLK_PLL_AUDIO_BASE]	= &pll_audio_base_clk.common.hw,
+		[CLK_PLL_AUDIO]		= &pll_audio_clk.hw,
+		[CLK_PLL_AUDIO_2X]	= &pll_audio_2x_clk.hw,
+		[CLK_PLL_AUDIO_4X]	= &pll_audio_4x_clk.hw,
+		[CLK_PLL_AUDIO_8X]	= &pll_audio_8x_clk.hw,
+		[CLK_PLL_VIDEO0]	= &pll_video0_clk.common.hw,
+		[CLK_PLL_VIDEO0_2X]	= &pll_video0_2x_clk.hw,
+		[CLK_PLL_VE]		= &pll_ve_clk.common.hw,
+		[CLK_PLL_DDR]		= &pll_ddr_clk.common.hw,
+		[CLK_PLL_PERIPH]	= &pll_periph_clk.common.hw,
+		[CLK_PLL_PERIPH_2X]	= &pll_periph_2x_clk.hw,
+		[CLK_PLL_VIDEO1]	= &pll_video1_clk.common.hw,
+		[CLK_PLL_VIDEO1_2X]	= &pll_video1_2x_clk.hw,
+		[CLK_PLL_GPU]		= &pll_gpu_clk.common.hw,
+		[CLK_PLL_MIPI]		= &pll_mipi_clk.common.hw,
+		[CLK_PLL9]		= &pll9_clk.common.hw,
+		[CLK_PLL10]		= &pll10_clk.common.hw,
+		[CLK_CPU]		= &cpu_clk.common.hw,
+		[CLK_AXI]		= &axi_clk.common.hw,
+		[CLK_AHB1]		= &ahb1_clk.common.hw,
+		[CLK_APB1]		= &apb1_clk.common.hw,
+		[CLK_APB2]		= &apb2_clk.common.hw,
+		[CLK_AHB1_MIPIDSI]	= &ahb1_mipidsi_clk.common.hw,
+		[CLK_AHB1_SS]		= &ahb1_ss_clk.common.hw,
+		[CLK_AHB1_DMA]		= &ahb1_dma_clk.common.hw,
+		[CLK_AHB1_MMC0]		= &ahb1_mmc0_clk.common.hw,
+		[CLK_AHB1_MMC1]		= &ahb1_mmc1_clk.common.hw,
+		[CLK_AHB1_MMC2]		= &ahb1_mmc2_clk.common.hw,
+		[CLK_AHB1_MMC3]		= &ahb1_mmc3_clk.common.hw,
+		[CLK_AHB1_NAND1]	= &ahb1_nand1_clk.common.hw,
+		[CLK_AHB1_NAND0]	= &ahb1_nand0_clk.common.hw,
+		[CLK_AHB1_SDRAM]	= &ahb1_sdram_clk.common.hw,
+		[CLK_AHB1_EMAC]		= &ahb1_emac_clk.common.hw,
+		[CLK_AHB1_TS]		= &ahb1_ts_clk.common.hw,
+		[CLK_AHB1_HSTIMER]	= &ahb1_hstimer_clk.common.hw,
+		[CLK_AHB1_SPI0]		= &ahb1_spi0_clk.common.hw,
+		[CLK_AHB1_SPI1]		= &ahb1_spi1_clk.common.hw,
+		[CLK_AHB1_SPI2]		= &ahb1_spi2_clk.common.hw,
+		[CLK_AHB1_SPI3]		= &ahb1_spi3_clk.common.hw,
+		[CLK_AHB1_OTG]		= &ahb1_otg_clk.common.hw,
+		[CLK_AHB1_EHCI0]	= &ahb1_ehci0_clk.common.hw,
+		[CLK_AHB1_EHCI1]	= &ahb1_ehci1_clk.common.hw,
+		[CLK_AHB1_OHCI0]	= &ahb1_ohci0_clk.common.hw,
+		[CLK_AHB1_OHCI1]	= &ahb1_ohci1_clk.common.hw,
+		[CLK_AHB1_OHCI2]	= &ahb1_ohci2_clk.common.hw,
+		[CLK_AHB1_VE]		= &ahb1_ve_clk.common.hw,
+		[CLK_AHB1_LCD0]		= &ahb1_lcd0_clk.common.hw,
+		[CLK_AHB1_LCD1]		= &ahb1_lcd1_clk.common.hw,
+		[CLK_AHB1_CSI]		= &ahb1_csi_clk.common.hw,
+		[CLK_AHB1_HDMI]		= &ahb1_hdmi_clk.common.hw,
+		[CLK_AHB1_BE0]		= &ahb1_be0_clk.common.hw,
+		[CLK_AHB1_BE1]		= &ahb1_be1_clk.common.hw,
+		[CLK_AHB1_FE0]		= &ahb1_fe0_clk.common.hw,
+		[CLK_AHB1_FE1]		= &ahb1_fe1_clk.common.hw,
+		[CLK_AHB1_MP]		= &ahb1_mp_clk.common.hw,
+		[CLK_AHB1_GPU]		= &ahb1_gpu_clk.common.hw,
+		[CLK_AHB1_DEU0]		= &ahb1_deu0_clk.common.hw,
+		[CLK_AHB1_DEU1]		= &ahb1_deu1_clk.common.hw,
+		[CLK_AHB1_DRC0]		= &ahb1_drc0_clk.common.hw,
+		[CLK_AHB1_DRC1]		= &ahb1_drc1_clk.common.hw,
+		[CLK_APB1_CODEC]	= &apb1_codec_clk.common.hw,
+		[CLK_APB1_SPDIF]	= &apb1_spdif_clk.common.hw,
+		[CLK_APB1_DIGITAL_MIC]	= &apb1_digital_mic_clk.common.hw,
+		[CLK_APB1_PIO]		= &apb1_pio_clk.common.hw,
+		[CLK_APB1_DAUDIO0]	= &apb1_daudio0_clk.common.hw,
+		[CLK_APB1_DAUDIO1]	= &apb1_daudio1_clk.common.hw,
+		[CLK_APB2_I2C0]		= &apb2_i2c0_clk.common.hw,
+		[CLK_APB2_I2C1]		= &apb2_i2c1_clk.common.hw,
+		[CLK_APB2_I2C2]		= &apb2_i2c2_clk.common.hw,
+		[CLK_APB2_I2C3]		= &apb2_i2c3_clk.common.hw,
+		[CLK_APB2_UART0]	= &apb2_uart0_clk.common.hw,
+		[CLK_APB2_UART1]	= &apb2_uart1_clk.common.hw,
+		[CLK_APB2_UART2]	= &apb2_uart2_clk.common.hw,
+		[CLK_APB2_UART3]	= &apb2_uart3_clk.common.hw,
+		[CLK_APB2_UART4]	= &apb2_uart4_clk.common.hw,
+		[CLK_APB2_UART5]	= &apb2_uart5_clk.common.hw,
+		[CLK_NAND0]		= &nand0_clk.common.hw,
+		[CLK_NAND1]		= &nand1_clk.common.hw,
+		[CLK_MMC0]		= &mmc0_clk.common.hw,
+		[CLK_MMC0_SAMPLE]	= &mmc0_sample_clk.common.hw,
+		[CLK_MMC0_OUTPUT]	= &mmc0_output_clk.common.hw,
+		[CLK_MMC1]		= &mmc1_clk.common.hw,
+		[CLK_MMC1_SAMPLE]	= &mmc1_sample_clk.common.hw,
+		[CLK_MMC1_OUTPUT]	= &mmc1_output_clk.common.hw,
+		[CLK_MMC2]		= &mmc2_clk.common.hw,
+		[CLK_MMC2_SAMPLE]	= &mmc2_sample_clk.common.hw,
+		[CLK_MMC2_OUTPUT]	= &mmc2_output_clk.common.hw,
+		[CLK_MMC3]		= &mmc3_clk.common.hw,
+		[CLK_MMC3_SAMPLE]	= &mmc3_sample_clk.common.hw,
+		[CLK_MMC3_OUTPUT]	= &mmc3_output_clk.common.hw,
+		[CLK_TS]		= &ts_clk.common.hw,
+		[CLK_SS]		= &ss_clk.common.hw,
+		[CLK_SPI0]		= &spi0_clk.common.hw,
+		[CLK_SPI1]		= &spi1_clk.common.hw,
+		[CLK_SPI2]		= &spi2_clk.common.hw,
+		[CLK_SPI3]		= &spi3_clk.common.hw,
+		[CLK_DAUDIO0]		= &daudio0_clk.common.hw,
+		[CLK_DAUDIO1]		= &daudio1_clk.common.hw,
+		[CLK_SPDIF]		= &spdif_clk.common.hw,
+		[CLK_USB_PHY0]		= &usb_phy0_clk.common.hw,
+		[CLK_USB_PHY1]		= &usb_phy1_clk.common.hw,
+		[CLK_USB_PHY2]		= &usb_phy2_clk.common.hw,
+		[CLK_USB_OHCI0]		= &usb_ohci0_clk.common.hw,
+		[CLK_USB_OHCI1]		= &usb_ohci1_clk.common.hw,
+		[CLK_USB_OHCI2]		= &usb_ohci2_clk.common.hw,
+		[CLK_MDFS]		= &mdfs_clk.common.hw,
+		[CLK_SDRAM0]		= &sdram0_clk.common.hw,
+		[CLK_SDRAM1]		= &sdram1_clk.common.hw,
+		[CLK_DRAM_VE]		= &dram_ve_clk.common.hw,
+		[CLK_DRAM_CSI_ISP]	= &dram_csi_isp_clk.common.hw,
+		[CLK_DRAM_TS]		= &dram_ts_clk.common.hw,
+		[CLK_DRAM_DRC0]		= &dram_drc0_clk.common.hw,
+		[CLK_DRAM_DRC1]		= &dram_drc1_clk.common.hw,
+		[CLK_DRAM_DEU0]		= &dram_deu0_clk.common.hw,
+		[CLK_DRAM_DEU1]		= &dram_deu1_clk.common.hw,
+		[CLK_DRAM_FE0]		= &dram_fe0_clk.common.hw,
+		[CLK_DRAM_FE1]		= &dram_fe1_clk.common.hw,
+		[CLK_DRAM_BE0]		= &dram_be0_clk.common.hw,
+		[CLK_DRAM_BE1]		= &dram_be1_clk.common.hw,
+		[CLK_DRAM_MP]		= &dram_mp_clk.common.hw,
+		[CLK_BE0]		= &be0_clk.common.hw,
+		[CLK_BE1]		= &be1_clk.common.hw,
+		[CLK_FE0]		= &fe0_clk.common.hw,
+		[CLK_FE1]		= &fe1_clk.common.hw,
+		[CLK_MP]		= &mp_clk.common.hw,
+		[CLK_LCD0_CH0]		= &lcd0_ch0_clk.common.hw,
+		[CLK_LCD1_CH0]		= &lcd1_ch0_clk.common.hw,
+		[CLK_LCD0_CH1]		= &lcd0_ch1_clk.common.hw,
+		[CLK_LCD1_CH1]		= &lcd1_ch1_clk.common.hw,
+		[CLK_CSI0_SCLK]		= &csi0_sclk_clk.common.hw,
+		[CLK_CSI0_MCLK]		= &csi0_mclk_clk.common.hw,
+		[CLK_CSI1_MCLK]		= &csi1_mclk_clk.common.hw,
+		[CLK_VE]		= &ve_clk.common.hw,
+		[CLK_CODEC]		= &codec_clk.common.hw,
+		[CLK_AVS]		= &avs_clk.common.hw,
+		[CLK_DIGITAL_MIC]	= &digital_mic_clk.common.hw,
+		[CLK_HDMI]		= &hdmi_clk.common.hw,
+		[CLK_HDMI_DDC]		= &hdmi_ddc_clk.common.hw,
+		[CLK_PS]		= &ps_clk.common.hw,
+		[CLK_MBUS0]		= &mbus0_clk.common.hw,
+		[CLK_MBUS1]		= &mbus1_clk.common.hw,
+		[CLK_MIPI_DSI]		= &mipi_dsi_clk.common.hw,
+		[CLK_MIPI_DSI_DPHY]	= &mipi_dsi_dphy_clk.common.hw,
+		[CLK_MIPI_CSI_DPHY]	= &mipi_csi_dphy_clk.common.hw,
+		[CLK_IEP_DRC0]		= &iep_drc0_clk.common.hw,
+		[CLK_IEP_DRC1]		= &iep_drc1_clk.common.hw,
+		[CLK_IEP_DEU0]		= &iep_deu0_clk.common.hw,
+		[CLK_IEP_DEU1]		= &iep_deu1_clk.common.hw,
+		[CLK_GPU_CORE]		= &gpu_core_clk.common.hw,
+		[CLK_GPU_MEMORY]	= &gpu_memory_clk.common.hw,
+		[CLK_GPU_HYD]		= &gpu_hyd_clk.common.hw,
+		[CLK_ATS]		= &ats_clk.common.hw,
+		[CLK_TRACE]		= &trace_clk.common.hw,
+		[CLK_OUT_A]		= &out_a_clk.common.hw,
+		[CLK_OUT_B]		= &out_b_clk.common.hw,
+		[CLK_OUT_C]		= &out_c_clk.common.hw,
+	},
+	.num	= CLK_NUMBER,
+};
+
+static struct ccu_reset_map sun6i_a31_ccu_resets[] = {
+	[RST_USB_PHY0]		= { 0x0cc, BIT(0) },
+	[RST_USB_PHY1]		= { 0x0cc, BIT(1) },
+	[RST_USB_PHY2]		= { 0x0cc, BIT(2) },
+
+	[RST_AHB1_MIPI_DSI]	= { 0x2c0, BIT(1) },
+	[RST_AHB1_SS]		= { 0x2c0, BIT(5) },
+	[RST_AHB1_DMA]		= { 0x2c0, BIT(6) },
+	[RST_AHB1_MMC0]		= { 0x2c0, BIT(8) },
+	[RST_AHB1_MMC1]		= { 0x2c0, BIT(9) },
+	[RST_AHB1_MMC2]		= { 0x2c0, BIT(10) },
+	[RST_AHB1_MMC3]		= { 0x2c0, BIT(11) },
+	[RST_AHB1_NAND1]	= { 0x2c0, BIT(12) },
+	[RST_AHB1_NAND0]	= { 0x2c0, BIT(13) },
+	[RST_AHB1_SDRAM]	= { 0x2c0, BIT(14) },
+	[RST_AHB1_EMAC]		= { 0x2c0, BIT(17) },
+	[RST_AHB1_TS]		= { 0x2c0, BIT(18) },
+	[RST_AHB1_HSTIMER]	= { 0x2c0, BIT(19) },
+	[RST_AHB1_SPI0]		= { 0x2c0, BIT(20) },
+	[RST_AHB1_SPI1]		= { 0x2c0, BIT(21) },
+	[RST_AHB1_SPI2]		= { 0x2c0, BIT(22) },
+	[RST_AHB1_SPI3]		= { 0x2c0, BIT(23) },
+	[RST_AHB1_OTG]		= { 0x2c0, BIT(24) },
+	[RST_AHB1_EHCI0]	= { 0x2c0, BIT(26) },
+	[RST_AHB1_EHCI1]	= { 0x2c0, BIT(27) },
+	[RST_AHB1_OHCI0]	= { 0x2c0, BIT(29) },
+	[RST_AHB1_OHCI1]	= { 0x2c0, BIT(30) },
+	[RST_AHB1_OHCI2]	= { 0x2c0, BIT(31) },
+
+	[RST_AHB1_VE]		= { 0x2c4, BIT(0) },
+	[RST_AHB1_LCD0]		= { 0x2c4, BIT(4) },
+	[RST_AHB1_LCD1]		= { 0x2c4, BIT(5) },
+	[RST_AHB1_CSI]		= { 0x2c4, BIT(8) },
+	[RST_AHB1_HDMI]		= { 0x2c4, BIT(11) },
+	[RST_AHB1_BE0]		= { 0x2c4, BIT(12) },
+	[RST_AHB1_BE1]		= { 0x2c4, BIT(13) },
+	[RST_AHB1_FE0]		= { 0x2c4, BIT(14) },
+	[RST_AHB1_FE1]		= { 0x2c4, BIT(15) },
+	[RST_AHB1_MP]		= { 0x2c4, BIT(18) },
+	[RST_AHB1_GPU]		= { 0x2c4, BIT(20) },
+	[RST_AHB1_DEU0]		= { 0x2c4, BIT(23) },
+	[RST_AHB1_DEU1]		= { 0x2c4, BIT(24) },
+	[RST_AHB1_DRC0]		= { 0x2c4, BIT(25) },
+	[RST_AHB1_DRC1]		= { 0x2c4, BIT(26) },
+	[RST_AHB1_LVDS]		= { 0x2c8, BIT(0) },
+
+	[RST_APB1_CODEC]	= { 0x2d0, BIT(0) },
+	[RST_APB1_SPDIF]	= { 0x2d0, BIT(1) },
+	[RST_APB1_DIGITAL_MIC]	= { 0x2d0, BIT(4) },
+	[RST_APB1_DAUDIO0]	= { 0x2d0, BIT(12) },
+	[RST_APB1_DAUDIO1]	= { 0x2d0, BIT(13) },
+
+	[RST_APB2_I2C0]		= { 0x2d8, BIT(0) },
+	[RST_APB2_I2C1]		= { 0x2d8, BIT(1) },
+	[RST_APB2_I2C2]		= { 0x2d8, BIT(2) },
+	[RST_APB2_I2C3]		= { 0x2d8, BIT(3) },
+	[RST_APB2_UART0]	= { 0x2d8, BIT(16) },
+	[RST_APB2_UART1]	= { 0x2d8, BIT(17) },
+	[RST_APB2_UART2]	= { 0x2d8, BIT(18) },
+	[RST_APB2_UART3]	= { 0x2d8, BIT(19) },
+	[RST_APB2_UART4]	= { 0x2d8, BIT(20) },
+	[RST_APB2_UART5]	= { 0x2d8, BIT(21) },
+};
+
+static const struct sunxi_ccu_desc sun6i_a31_ccu_desc = {
+	.ccu_clks	= sun6i_a31_ccu_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun6i_a31_ccu_clks),
+
+	.hw_clks	= &sun6i_a31_hw_clks,
+
+	.resets		= sun6i_a31_ccu_resets,
+	.num_resets	= ARRAY_SIZE(sun6i_a31_ccu_resets),
+};
+
+static struct ccu_mux_nb sun6i_a31_cpu_nb = {
+	.common		= &cpu_clk.common,
+	.cm		= &cpu_clk.mux,
+	.delay_us	= 1, /* > 8 clock cycles at 24 MHz */
+	.bypass_index	= 1, /* index of 24 MHz oscillator */
+};
+
+static void __init sun6i_a31_ccu_setup(struct device_node *node)
+{
+	void __iomem *reg;
+	u32 val;
+
+	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(reg)) {
+		pr_err("%s: Could not map the clock registers\n",
+		       of_node_full_name(node));
+		return;
+	}
+
+	/* Force the PLL-Audio-1x divider to 4 */
+	val = readl(reg + SUN6I_A31_PLL_AUDIO_REG);
+	val &= ~GENMASK(19, 16);
+	writel(val | (3 << 16), reg + SUN6I_A31_PLL_AUDIO_REG);
+
+	/* Force PLL-MIPI to MIPI mode */
+	val = readl(reg + SUN6I_A31_PLL_MIPI_REG);
+	val &= BIT(16);
+	writel(val, reg + SUN6I_A31_PLL_MIPI_REG);
+
+	sunxi_ccu_probe(node, reg, &sun6i_a31_ccu_desc);
+
+	ccu_mux_notifier_register(pll_cpu_clk.common.hw.clk,
+				  &sun6i_a31_cpu_nb);
+}
+CLK_OF_DECLARE(sun6i_a31_ccu, "allwinner,sun6i-a31-ccu",
+	       sun6i_a31_ccu_setup);
diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.h b/drivers/clk/sunxi-ng/ccu-sun6i-a31.h
new file mode 100644
index 000000000000..4e434011e9e7
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2016 Chen-Yu Tsai
+ *
+ * Chen-Yu Tsai <wens@csie.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _CCU_SUN6I_A31_H_
+#define _CCU_SUN6I_A31_H_
+
+#include <dt-bindings/clock/sun6i-a31-ccu.h>
+#include <dt-bindings/reset/sun6i-a31-ccu.h>
+
+#define CLK_PLL_CPU		0
+#define CLK_PLL_AUDIO_BASE	1
+#define CLK_PLL_AUDIO		2
+#define CLK_PLL_AUDIO_2X	3
+#define CLK_PLL_AUDIO_4X	4
+#define CLK_PLL_AUDIO_8X	5
+#define CLK_PLL_VIDEO0		6
+#define CLK_PLL_VIDEO0_2X	7
+#define CLK_PLL_VE		8
+#define CLK_PLL_DDR		9
+
+/* The PLL_PERIPH clock is exported */
+
+#define CLK_PLL_PERIPH_2X	11
+#define CLK_PLL_VIDEO1		12
+#define CLK_PLL_VIDEO1_2X	13
+#define CLK_PLL_GPU		14
+#define CLK_PLL_MIPI		15
+#define CLK_PLL9		16
+#define CLK_PLL10		17
+
+/* The CPUX clock is exported */
+
+#define CLK_AXI			19
+#define CLK_AHB1		20
+#define CLK_APB1		21
+#define CLK_APB2		22
+
+/* All the bus gates are exported */
+
+/* The first bunch of module clocks are exported */
+
+/* EMAC clock is not implemented */
+
+#define CLK_MDFS		107
+#define CLK_SDRAM0		108
+#define CLK_SDRAM1		109
+
+/* All the DRAM gates are exported */
+
+/* Some more module clocks are exported */
+
+#define CLK_MBUS0		141
+#define CLK_MBUS1		142
+
+/* Some more module clocks and external clock outputs are exported */
+
+#define CLK_NUMBER		(CLK_OUT_C + 1)
+
+#endif /* _CCU_SUN6I_A31_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23-a33.h b/drivers/clk/sunxi-ng/ccu-sun8i-a23-a33.h
new file mode 100644
index 000000000000..62c0f8d49ef8
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23-a33.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2016 Maxime Ripard
+ *
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _CCU_SUN8I_A23_A33_H_
+#define _CCU_SUN8I_A23_A33_H_
+
+#include <dt-bindings/clock/sun8i-a23-a33-ccu.h>
+#include <dt-bindings/reset/sun8i-a23-a33-ccu.h>
+
+#define CLK_PLL_CPUX		0
+#define CLK_PLL_AUDIO_BASE	1
+#define CLK_PLL_AUDIO		2
+#define CLK_PLL_AUDIO_2X	3
+#define CLK_PLL_AUDIO_4X	4
+#define CLK_PLL_AUDIO_8X	5
+#define CLK_PLL_VIDEO		6
+#define CLK_PLL_VIDEO_2X	7
+#define CLK_PLL_VE		8
+#define CLK_PLL_DDR0		9
+#define CLK_PLL_PERIPH		10
+#define CLK_PLL_PERIPH_2X	11
+#define CLK_PLL_GPU		12
+#define CLK_PLL_MIPI		13
+#define CLK_PLL_HSIC		14
+#define CLK_PLL_DE		15
+#define CLK_PLL_DDR1		16
+#define CLK_PLL_DDR		17
+
+/* The CPUX clock is exported */
+
+#define CLK_AXI			19
+#define CLK_AHB1		20
+#define CLK_APB1		21
+#define CLK_APB2		22
+
+/* All the bus gates are exported */
+
+/* The first part of the mod clocks is exported */
+
+#define CLK_DRAM		79
+
+/* Some more module clocks are exported */
+
+#define CLK_MBUS		95
+
+/* And the last module clocks are exported */
+
+#define CLK_NUMBER		(CLK_ATS + 1)
+
+#endif /* _CCU_SUN8I_A23_A33_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
new file mode 100644
index 000000000000..2646d980087b
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
@@ -0,0 +1,737 @@
+/*
+ * Copyright (c) 2016 Maxime Ripard. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+
+#include "ccu_common.h"
+#include "ccu_reset.h"
+
+#include "ccu_div.h"
+#include "ccu_gate.h"
+#include "ccu_mp.h"
+#include "ccu_mult.h"
+#include "ccu_nk.h"
+#include "ccu_nkm.h"
+#include "ccu_nkmp.h"
+#include "ccu_nm.h"
+#include "ccu_phase.h"
+
+#include "ccu-sun8i-a23-a33.h"
+
+
+static struct ccu_nkmp pll_cpux_clk = {
+	.enable = BIT(31),
+	.lock	= BIT(28),
+
+	.n	= _SUNXI_CCU_MULT(8, 5),
+	.k	= _SUNXI_CCU_MULT(4, 2),
+	.m	= _SUNXI_CCU_DIV(0, 2),
+	.p	= _SUNXI_CCU_DIV_MAX(16, 2, 4),
+
+	.common	= {
+		.reg		= 0x000,
+		.hw.init	= CLK_HW_INIT("pll-cpux", "osc24M",
+					      &ccu_nkmp_ops,
+					      0),
+	},
+};
+
+/*
+ * The Audio PLL is supposed to have 4 outputs: 3 fixed factors from
+ * the base (2x, 4x and 8x), and one variable divider (the one true
+ * pll audio).
+ *
+ * We don't have any need for the variable divider for now, so we just
+ * hardcode it to match with the clock names
+ */
+#define SUN8I_A23_PLL_AUDIO_REG	0x008
+
+static SUNXI_CCU_NM_WITH_GATE_LOCK(pll_audio_base_clk, "pll-audio-base",
+				   "osc24M", 0x008,
+				   8, 7,		/* N */
+				   0, 5,		/* M */
+				   BIT(31),		/* gate */
+				   BIT(28),		/* lock */
+				   CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_video_clk, "pll-video",
+					"osc24M", 0x010,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_ve_clk, "pll-ve",
+					"osc24M", 0x018,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_ddr_clk, "pll-ddr",
+				    "osc24M", 0x020,
+				    8, 5,		/* N */
+				    4, 2,		/* K */
+				    0, 2,		/* M */
+				    BIT(31),		/* gate */
+				    BIT(28),		/* lock */
+				    0);
+
+static SUNXI_CCU_NK_WITH_GATE_LOCK_POSTDIV(pll_periph_clk, "pll-periph",
+					   "osc24M", 0x028,
+					   8, 5,	/* N */
+					   4, 2,	/* K */
+					   BIT(31),	/* gate */
+					   BIT(28),	/* lock */
+					   2,		/* post-div */
+					   CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_gpu_clk, "pll-gpu",
+					"osc24M", 0x038,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+/*
+ * The MIPI PLL has 2 modes: "MIPI" and "HDMI".
+ *
+ * The MIPI mode is a standard NKM-style clock. The HDMI mode is an
+ * integer / fractional clock with switchable multipliers and dividers.
+ * This is not supported here. We hardcode the PLL to MIPI mode.
+ */
+#define SUN8I_A23_PLL_MIPI_REG	0x040
+static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_mipi_clk, "pll-mipi",
+				    "pll-video", 0x040,
+				    8, 4,		/* N */
+				    4, 2,		/* K */
+				    0, 4,		/* M */
+				    BIT(31),		/* gate */
+				    BIT(28),		/* lock */
+				    CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_hsic_clk, "pll-hsic",
+					"osc24M", 0x044,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_de_clk, "pll-de",
+					"osc24M", 0x048,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static const char * const cpux_parents[] = { "osc32k", "osc24M",
+					     "pll-cpux" , "pll-cpux" };
+static SUNXI_CCU_MUX(cpux_clk, "cpux", cpux_parents,
+		     0x050, 16, 2, CLK_IS_CRITICAL);
+
+static SUNXI_CCU_M(axi_clk, "axi", "cpux", 0x050, 0, 2, 0);
+
+static const char * const ahb1_parents[] = { "osc32k", "osc24M",
+					     "axi" , "pll-periph" };
+static struct ccu_div ahb1_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO),
+
+	.mux		= {
+		.shift	= 12,
+		.width	= 2,
+
+		.variable_prediv	= {
+			.index	= 3,
+			.shift	= 6,
+			.width	= 2,
+		},
+	},
+
+	.common		= {
+		.reg		= 0x054,
+		.features	= CCU_FEATURE_VARIABLE_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("ahb1",
+						      ahb1_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct clk_div_table apb1_div_table[] = {
+	{ .val = 0, .div = 2 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 4 },
+	{ .val = 3, .div = 8 },
+	{ /* Sentinel */ },
+};
+static SUNXI_CCU_DIV_TABLE(apb1_clk, "apb1", "ahb1",
+			   0x054, 8, 2, apb1_div_table, 0);
+
+static const char * const apb2_parents[] = { "osc32k", "osc24M",
+					     "pll-periph" , "pll-periph" };
+static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", apb2_parents, 0x058,
+			     0, 5,	/* M */
+			     16, 2,	/* P */
+			     24, 2,	/* mux */
+			     0);
+
+static SUNXI_CCU_GATE(bus_mipi_dsi_clk,	"bus-mipi-dsi",	"ahb1",
+		      0x060, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_dma_clk,	"bus-dma",	"ahb1",
+		      0x060, BIT(6), 0);
+static SUNXI_CCU_GATE(bus_mmc0_clk,	"bus-mmc0",	"ahb1",
+		      0x060, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_mmc1_clk,	"bus-mmc1",	"ahb1",
+		      0x060, BIT(9), 0);
+static SUNXI_CCU_GATE(bus_mmc2_clk,	"bus-mmc2",	"ahb1",
+		      0x060, BIT(10), 0);
+static SUNXI_CCU_GATE(bus_nand_clk,	"bus-nand",	"ahb1",
+		      0x060, BIT(13), 0);
+static SUNXI_CCU_GATE(bus_dram_clk,	"bus-dram",	"ahb1",
+		      0x060, BIT(14), 0);
+static SUNXI_CCU_GATE(bus_hstimer_clk,	"bus-hstimer",	"ahb1",
+		      0x060, BIT(19), 0);
+static SUNXI_CCU_GATE(bus_spi0_clk,	"bus-spi0",	"ahb1",
+		      0x060, BIT(20), 0);
+static SUNXI_CCU_GATE(bus_spi1_clk,	"bus-spi1",	"ahb1",
+		      0x060, BIT(21), 0);
+static SUNXI_CCU_GATE(bus_otg_clk,	"bus-otg",	"ahb1",
+		      0x060, BIT(24), 0);
+static SUNXI_CCU_GATE(bus_ehci_clk,	"bus-ehci",	"ahb1",
+		      0x060, BIT(26), 0);
+static SUNXI_CCU_GATE(bus_ohci_clk,	"bus-ohci",	"ahb1",
+		      0x060, BIT(29), 0);
+
+static SUNXI_CCU_GATE(bus_ve_clk,	"bus-ve",	"ahb1",
+		      0x064, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_lcd_clk,	"bus-lcd",	"ahb1",
+		      0x064, BIT(4), 0);
+static SUNXI_CCU_GATE(bus_csi_clk,	"bus-csi",	"ahb1",
+		      0x064, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_de_be_clk,	"bus-de-be",	"ahb1",
+		      0x064, BIT(12), 0);
+static SUNXI_CCU_GATE(bus_de_fe_clk,	"bus-de-fe",	"ahb1",
+		      0x064, BIT(14), 0);
+static SUNXI_CCU_GATE(bus_gpu_clk,	"bus-gpu",	"ahb1",
+		      0x064, BIT(20), 0);
+static SUNXI_CCU_GATE(bus_msgbox_clk,	"bus-msgbox",	"ahb1",
+		      0x064, BIT(21), 0);
+static SUNXI_CCU_GATE(bus_spinlock_clk,	"bus-spinlock",	"ahb1",
+		      0x064, BIT(22), 0);
+static SUNXI_CCU_GATE(bus_drc_clk,	"bus-drc",	"ahb1",
+		      0x064, BIT(25), 0);
+
+static SUNXI_CCU_GATE(bus_codec_clk,	"bus-codec",	"apb1",
+		      0x068, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_pio_clk,	"bus-pio",	"apb1",
+		      0x068, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_i2s0_clk,	"bus-i2s0",	"apb1",
+		      0x068, BIT(12), 0);
+static SUNXI_CCU_GATE(bus_i2s1_clk,	"bus-i2s1",	"apb1",
+		      0x068, BIT(13), 0);
+
+static SUNXI_CCU_GATE(bus_i2c0_clk,	"bus-i2c0",	"apb2",
+		      0x06c, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_i2c1_clk,	"bus-i2c1",	"apb2",
+		      0x06c, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_i2c2_clk,	"bus-i2c2",	"apb2",
+		      0x06c, BIT(2), 0);
+static SUNXI_CCU_GATE(bus_uart0_clk,	"bus-uart0",	"apb2",
+		      0x06c, BIT(16), 0);
+static SUNXI_CCU_GATE(bus_uart1_clk,	"bus-uart1",	"apb2",
+		      0x06c, BIT(17), 0);
+static SUNXI_CCU_GATE(bus_uart2_clk,	"bus-uart2",	"apb2",
+		      0x06c, BIT(18), 0);
+static SUNXI_CCU_GATE(bus_uart3_clk,	"bus-uart3",	"apb2",
+		      0x06c, BIT(19), 0);
+static SUNXI_CCU_GATE(bus_uart4_clk,	"bus-uart4",	"apb2",
+		      0x06c, BIT(20), 0);
+
+static const char * const mod0_default_parents[] = { "osc24M", "pll-periph" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(nand_clk, "nand", mod0_default_parents, 0x080,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc0_clk, "mmc0", mod0_default_parents, 0x088,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc0_sample_clk, "mmc0_sample", "mmc0",
+		       0x088, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc0_output_clk, "mmc0_output", "mmc0",
+		       0x088, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc1_clk, "mmc1", mod0_default_parents, 0x08c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc1_sample_clk, "mmc1_sample", "mmc1",
+		       0x08c, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc1_output_clk, "mmc1_output", "mmc1",
+		       0x08c, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc2_clk, "mmc2", mod0_default_parents, 0x090,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc2_sample_clk, "mmc2_sample", "mmc2",
+		       0x090, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc2_output_clk, "mmc2_output", "mmc2",
+		       0x090, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", mod0_default_parents, 0x0a0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const i2s_parents[] = { "pll-audio-8x", "pll-audio-4x",
+					    "pll-audio-2x", "pll-audio" };
+static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", i2s_parents,
+			       0x0b0, 16, 2, BIT(31), 0);
+
+static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", i2s_parents,
+			       0x0b4, 16, 2, BIT(31), 0);
+
+/* TODO: the parent for most of the USB clocks is not known */
+static SUNXI_CCU_GATE(usb_phy0_clk,	"usb-phy0",	"osc24M",
+		      0x0cc, BIT(8), 0);
+static SUNXI_CCU_GATE(usb_phy1_clk,	"usb-phy1",	"osc24M",
+		      0x0cc, BIT(9), 0);
+static SUNXI_CCU_GATE(usb_hsic_clk,	"usb-hsic",	"pll-hsic",
+		      0x0cc, BIT(10), 0);
+static SUNXI_CCU_GATE(usb_hsic_12M_clk,	"usb-hsic-12M",	"osc24M",
+		      0x0cc, BIT(11), 0);
+static SUNXI_CCU_GATE(usb_ohci_clk,	"usb-ohci",	"osc24M",
+		      0x0cc, BIT(16), 0);
+
+static SUNXI_CCU_GATE(dram_ve_clk,	"dram-ve",	"pll-ddr",
+		      0x100, BIT(0), 0);
+static SUNXI_CCU_GATE(dram_csi_clk,	"dram-csi",	"pll-ddr",
+		      0x100, BIT(1), 0);
+static SUNXI_CCU_GATE(dram_drc_clk,	"dram-drc",	"pll-ddr",
+		      0x100, BIT(16), 0);
+static SUNXI_CCU_GATE(dram_de_fe_clk,	"dram-de-fe",	"pll-ddr",
+		      0x100, BIT(24), 0);
+static SUNXI_CCU_GATE(dram_de_be_clk,	"dram-de-be",	"pll-ddr",
+		      0x100, BIT(26), 0);
+
+static const char * const de_parents[] = { "pll-video", "pll-periph-2x",
+					   "pll-gpu", "pll-de" };
+static const u8 de_table[] = { 0, 2, 3, 5 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(de_be_clk, "de-be",
+				       de_parents, de_table,
+				       0x104, 0, 4, 24, 3, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(de_fe_clk, "de-fe",
+				       de_parents, de_table,
+				       0x10c, 0, 4, 24, 3, BIT(31), 0);
+
+static const char * const lcd_ch0_parents[] = { "pll-video", "pll-video-2x",
+						"pll-mipi" };
+static const u8 lcd_ch0_table[] = { 0, 2, 4 };
+static SUNXI_CCU_MUX_TABLE_WITH_GATE(lcd_ch0_clk, "lcd-ch0",
+				     lcd_ch0_parents, lcd_ch0_table,
+				     0x118, 24, 3, BIT(31),
+				     CLK_SET_RATE_PARENT);
+
+static const char * const lcd_ch1_parents[] = { "pll-video", "pll-video-2x" };
+static const u8 lcd_ch1_table[] = { 0, 2 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(lcd_ch1_clk, "lcd-ch1",
+				       lcd_ch1_parents, lcd_ch1_table,
+				       0x12c, 0, 4, 24, 2, BIT(31), 0);
+
+static const char * const csi_sclk_parents[] = { "pll-video", "pll-de",
+						 "pll-mipi", "pll-ve" };
+static const u8 csi_sclk_table[] = { 0, 3, 4, 5 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(csi_sclk_clk, "csi-sclk",
+				       csi_sclk_parents, csi_sclk_table,
+				       0x134, 16, 4, 24, 3, BIT(31), 0);
+
+static const char * const csi_mclk_parents[] = { "pll-video", "pll-de",
+						 "osc24M" };
+static const u8 csi_mclk_table[] = { 0, 3, 5 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(csi_mclk_clk, "csi-mclk",
+				       csi_mclk_parents, csi_mclk_table,
+				       0x134, 0, 5, 8, 3, BIT(15), 0);
+
+static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve",
+			     0x13c, 16, 3, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(ac_dig_clk,	"ac-dig",	"pll-audio",
+		      0x140, BIT(31), 0);
+static SUNXI_CCU_GATE(avs_clk,		"avs",		"osc24M",
+		      0x144, BIT(31), 0);
+
+static const char * const mbus_parents[] = { "osc24M", "pll-periph-2x",
+					     "pll-ddr" };
+static SUNXI_CCU_M_WITH_MUX_GATE(mbus_clk, "mbus", mbus_parents,
+				 0x15c, 0, 3, 24, 2, BIT(31), CLK_IS_CRITICAL);
+
+static const char * const dsi_sclk_parents[] = { "pll-video", "pll-video-2x" };
+static const u8 dsi_sclk_table[] = { 0, 2 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(dsi_sclk_clk, "dsi-sclk",
+				       dsi_sclk_parents, dsi_sclk_table,
+				       0x168, 16, 4, 24, 2, BIT(31), 0);
+
+static const char * const dsi_dphy_parents[] = { "pll-video", "pll-periph" };
+static const u8 dsi_dphy_table[] = { 0, 2 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(dsi_dphy_clk, "dsi-dphy",
+				       dsi_dphy_parents, dsi_dphy_table,
+				       0x168, 0, 4, 8, 2, BIT(15), 0);
+
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(drc_clk, "drc",
+				       de_parents, de_table,
+				       0x180, 0, 4, 24, 3, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_GATE(gpu_clk, "gpu", "pll-gpu",
+			     0x1a0, 0, 3, BIT(31), 0);
+
+static const char * const ats_parents[] = { "osc24M", "pll-periph" };
+static SUNXI_CCU_M_WITH_MUX_GATE(ats_clk, "ats", ats_parents,
+				 0x1b0, 0, 3, 24, 2, BIT(31), 0);
+
+static struct ccu_common *sun8i_a23_ccu_clks[] = {
+	&pll_cpux_clk.common,
+	&pll_audio_base_clk.common,
+	&pll_video_clk.common,
+	&pll_ve_clk.common,
+	&pll_ddr_clk.common,
+	&pll_periph_clk.common,
+	&pll_gpu_clk.common,
+	&pll_mipi_clk.common,
+	&pll_hsic_clk.common,
+	&pll_de_clk.common,
+	&cpux_clk.common,
+	&axi_clk.common,
+	&ahb1_clk.common,
+	&apb1_clk.common,
+	&apb2_clk.common,
+	&bus_mipi_dsi_clk.common,
+	&bus_dma_clk.common,
+	&bus_mmc0_clk.common,
+	&bus_mmc1_clk.common,
+	&bus_mmc2_clk.common,
+	&bus_nand_clk.common,
+	&bus_dram_clk.common,
+	&bus_hstimer_clk.common,
+	&bus_spi0_clk.common,
+	&bus_spi1_clk.common,
+	&bus_otg_clk.common,
+	&bus_ehci_clk.common,
+	&bus_ohci_clk.common,
+	&bus_ve_clk.common,
+	&bus_lcd_clk.common,
+	&bus_csi_clk.common,
+	&bus_de_fe_clk.common,
+	&bus_de_be_clk.common,
+	&bus_gpu_clk.common,
+	&bus_msgbox_clk.common,
+	&bus_spinlock_clk.common,
+	&bus_drc_clk.common,
+	&bus_codec_clk.common,
+	&bus_pio_clk.common,
+	&bus_i2s0_clk.common,
+	&bus_i2s1_clk.common,
+	&bus_i2c0_clk.common,
+	&bus_i2c1_clk.common,
+	&bus_i2c2_clk.common,
+	&bus_uart0_clk.common,
+	&bus_uart1_clk.common,
+	&bus_uart2_clk.common,
+	&bus_uart3_clk.common,
+	&bus_uart4_clk.common,
+	&nand_clk.common,
+	&mmc0_clk.common,
+	&mmc0_sample_clk.common,
+	&mmc0_output_clk.common,
+	&mmc1_clk.common,
+	&mmc1_sample_clk.common,
+	&mmc1_output_clk.common,
+	&mmc2_clk.common,
+	&mmc2_sample_clk.common,
+	&mmc2_output_clk.common,
+	&spi0_clk.common,
+	&spi1_clk.common,
+	&i2s0_clk.common,
+	&i2s1_clk.common,
+	&usb_phy0_clk.common,
+	&usb_phy1_clk.common,
+	&usb_hsic_clk.common,
+	&usb_hsic_12M_clk.common,
+	&usb_ohci_clk.common,
+	&dram_ve_clk.common,
+	&dram_csi_clk.common,
+	&dram_drc_clk.common,
+	&dram_de_fe_clk.common,
+	&dram_de_be_clk.common,
+	&de_be_clk.common,
+	&de_fe_clk.common,
+	&lcd_ch0_clk.common,
+	&lcd_ch1_clk.common,
+	&csi_sclk_clk.common,
+	&csi_mclk_clk.common,
+	&ve_clk.common,
+	&ac_dig_clk.common,
+	&avs_clk.common,
+	&mbus_clk.common,
+	&dsi_sclk_clk.common,
+	&dsi_dphy_clk.common,
+	&drc_clk.common,
+	&gpu_clk.common,
+	&ats_clk.common,
+};
+
+/* We hardcode the divider to 4 for now */
+static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
+			"pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
+			"pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
+			"pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
+			"pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_periph_2x_clk, "pll-periph-2x",
+			"pll-periph", 1, 2, 0);
+static CLK_FIXED_FACTOR(pll_video_2x_clk, "pll-video-2x",
+			"pll-video", 1, 2, 0);
+
+static struct clk_hw_onecell_data sun8i_a23_hw_clks = {
+	.hws	= {
+		[CLK_PLL_CPUX]		= &pll_cpux_clk.common.hw,
+		[CLK_PLL_AUDIO_BASE]	= &pll_audio_base_clk.common.hw,
+		[CLK_PLL_AUDIO]		= &pll_audio_clk.hw,
+		[CLK_PLL_AUDIO_2X]	= &pll_audio_2x_clk.hw,
+		[CLK_PLL_AUDIO_4X]	= &pll_audio_4x_clk.hw,
+		[CLK_PLL_AUDIO_8X]	= &pll_audio_8x_clk.hw,
+		[CLK_PLL_VIDEO]		= &pll_video_clk.common.hw,
+		[CLK_PLL_VIDEO_2X]	= &pll_video_2x_clk.hw,
+		[CLK_PLL_VE]		= &pll_ve_clk.common.hw,
+		[CLK_PLL_DDR0]		= &pll_ddr_clk.common.hw,
+		[CLK_PLL_PERIPH]	= &pll_periph_clk.common.hw,
+		[CLK_PLL_PERIPH_2X]	= &pll_periph_2x_clk.hw,
+		[CLK_PLL_GPU]		= &pll_gpu_clk.common.hw,
+		[CLK_PLL_MIPI]		= &pll_mipi_clk.common.hw,
+		[CLK_PLL_HSIC]		= &pll_hsic_clk.common.hw,
+		[CLK_PLL_DE]		= &pll_de_clk.common.hw,
+		[CLK_CPUX]		= &cpux_clk.common.hw,
+		[CLK_AXI]		= &axi_clk.common.hw,
+		[CLK_AHB1]		= &ahb1_clk.common.hw,
+		[CLK_APB1]		= &apb1_clk.common.hw,
+		[CLK_APB2]		= &apb2_clk.common.hw,
+		[CLK_BUS_MIPI_DSI]	= &bus_mipi_dsi_clk.common.hw,
+		[CLK_BUS_DMA]		= &bus_dma_clk.common.hw,
+		[CLK_BUS_MMC0]		= &bus_mmc0_clk.common.hw,
+		[CLK_BUS_MMC1]		= &bus_mmc1_clk.common.hw,
+		[CLK_BUS_MMC2]		= &bus_mmc2_clk.common.hw,
+		[CLK_BUS_NAND]		= &bus_nand_clk.common.hw,
+		[CLK_BUS_DRAM]		= &bus_dram_clk.common.hw,
+		[CLK_BUS_HSTIMER]	= &bus_hstimer_clk.common.hw,
+		[CLK_BUS_SPI0]		= &bus_spi0_clk.common.hw,
+		[CLK_BUS_SPI1]		= &bus_spi1_clk.common.hw,
+		[CLK_BUS_OTG]		= &bus_otg_clk.common.hw,
+		[CLK_BUS_EHCI]		= &bus_ehci_clk.common.hw,
+		[CLK_BUS_OHCI]		= &bus_ohci_clk.common.hw,
+		[CLK_BUS_VE]		= &bus_ve_clk.common.hw,
+		[CLK_BUS_LCD]		= &bus_lcd_clk.common.hw,
+		[CLK_BUS_CSI]		= &bus_csi_clk.common.hw,
+		[CLK_BUS_DE_BE]		= &bus_de_be_clk.common.hw,
+		[CLK_BUS_DE_FE]		= &bus_de_fe_clk.common.hw,
+		[CLK_BUS_GPU]		= &bus_gpu_clk.common.hw,
+		[CLK_BUS_MSGBOX]	= &bus_msgbox_clk.common.hw,
+		[CLK_BUS_SPINLOCK]	= &bus_spinlock_clk.common.hw,
+		[CLK_BUS_DRC]		= &bus_drc_clk.common.hw,
+		[CLK_BUS_CODEC]		= &bus_codec_clk.common.hw,
+		[CLK_BUS_PIO]		= &bus_pio_clk.common.hw,
+		[CLK_BUS_I2S0]		= &bus_i2s0_clk.common.hw,
+		[CLK_BUS_I2S1]		= &bus_i2s1_clk.common.hw,
+		[CLK_BUS_I2C0]		= &bus_i2c0_clk.common.hw,
+		[CLK_BUS_I2C1]		= &bus_i2c1_clk.common.hw,
+		[CLK_BUS_I2C2]		= &bus_i2c2_clk.common.hw,
+		[CLK_BUS_UART0]		= &bus_uart0_clk.common.hw,
+		[CLK_BUS_UART1]		= &bus_uart1_clk.common.hw,
+		[CLK_BUS_UART2]		= &bus_uart2_clk.common.hw,
+		[CLK_BUS_UART3]		= &bus_uart3_clk.common.hw,
+		[CLK_BUS_UART4]		= &bus_uart4_clk.common.hw,
+		[CLK_NAND]		= &nand_clk.common.hw,
+		[CLK_MMC0]		= &mmc0_clk.common.hw,
+		[CLK_MMC0_SAMPLE]	= &mmc0_sample_clk.common.hw,
+		[CLK_MMC0_OUTPUT]	= &mmc0_output_clk.common.hw,
+		[CLK_MMC1]		= &mmc1_clk.common.hw,
+		[CLK_MMC1_SAMPLE]	= &mmc1_sample_clk.common.hw,
+		[CLK_MMC1_OUTPUT]	= &mmc1_output_clk.common.hw,
+		[CLK_MMC2]		= &mmc2_clk.common.hw,
+		[CLK_MMC2_SAMPLE]	= &mmc2_sample_clk.common.hw,
+		[CLK_MMC2_OUTPUT]	= &mmc2_output_clk.common.hw,
+		[CLK_SPI0]		= &spi0_clk.common.hw,
+		[CLK_SPI1]		= &spi1_clk.common.hw,
+		[CLK_I2S0]		= &i2s0_clk.common.hw,
+		[CLK_I2S1]		= &i2s1_clk.common.hw,
+		[CLK_USB_PHY0]		= &usb_phy0_clk.common.hw,
+		[CLK_USB_PHY1]		= &usb_phy1_clk.common.hw,
+		[CLK_USB_HSIC]		= &usb_hsic_clk.common.hw,
+		[CLK_USB_HSIC_12M]	= &usb_hsic_12M_clk.common.hw,
+		[CLK_USB_OHCI]		= &usb_ohci_clk.common.hw,
+		[CLK_DRAM_VE]		= &dram_ve_clk.common.hw,
+		[CLK_DRAM_CSI]		= &dram_csi_clk.common.hw,
+		[CLK_DRAM_DRC]		= &dram_drc_clk.common.hw,
+		[CLK_DRAM_DE_FE]	= &dram_de_fe_clk.common.hw,
+		[CLK_DRAM_DE_BE]	= &dram_de_be_clk.common.hw,
+		[CLK_DE_BE]		= &de_be_clk.common.hw,
+		[CLK_DE_FE]		= &de_fe_clk.common.hw,
+		[CLK_LCD_CH0]		= &lcd_ch0_clk.common.hw,
+		[CLK_LCD_CH1]		= &lcd_ch1_clk.common.hw,
+		[CLK_CSI_SCLK]		= &csi_sclk_clk.common.hw,
+		[CLK_CSI_MCLK]		= &csi_mclk_clk.common.hw,
+		[CLK_VE]		= &ve_clk.common.hw,
+		[CLK_AC_DIG]		= &ac_dig_clk.common.hw,
+		[CLK_AVS]		= &avs_clk.common.hw,
+		[CLK_MBUS]		= &mbus_clk.common.hw,
+		[CLK_DSI_SCLK]		= &dsi_sclk_clk.common.hw,
+		[CLK_DSI_DPHY]		= &dsi_dphy_clk.common.hw,
+		[CLK_DRC]		= &drc_clk.common.hw,
+		[CLK_GPU]		= &gpu_clk.common.hw,
+		[CLK_ATS]		= &ats_clk.common.hw,
+	},
+	.num	= CLK_NUMBER,
+};
+
+static struct ccu_reset_map sun8i_a23_ccu_resets[] = {
+	[RST_USB_PHY0]		=  { 0x0cc, BIT(0) },
+	[RST_USB_PHY1]		=  { 0x0cc, BIT(1) },
+	[RST_USB_HSIC]		=  { 0x0cc, BIT(2) },
+
+	[RST_MBUS]		=  { 0x0fc, BIT(31) },
+
+	[RST_BUS_MIPI_DSI]	=  { 0x2c0, BIT(1) },
+	[RST_BUS_DMA]		=  { 0x2c0, BIT(6) },
+	[RST_BUS_MMC0]		=  { 0x2c0, BIT(8) },
+	[RST_BUS_MMC1]		=  { 0x2c0, BIT(9) },
+	[RST_BUS_MMC2]		=  { 0x2c0, BIT(10) },
+	[RST_BUS_NAND]		=  { 0x2c0, BIT(13) },
+	[RST_BUS_DRAM]		=  { 0x2c0, BIT(14) },
+	[RST_BUS_HSTIMER]	=  { 0x2c0, BIT(19) },
+	[RST_BUS_SPI0]		=  { 0x2c0, BIT(20) },
+	[RST_BUS_SPI1]		=  { 0x2c0, BIT(21) },
+	[RST_BUS_OTG]		=  { 0x2c0, BIT(24) },
+	[RST_BUS_EHCI]		=  { 0x2c0, BIT(26) },
+	[RST_BUS_OHCI]		=  { 0x2c0, BIT(29) },
+
+	[RST_BUS_VE]		=  { 0x2c4, BIT(0) },
+	[RST_BUS_LCD]		=  { 0x2c4, BIT(4) },
+	[RST_BUS_CSI]		=  { 0x2c4, BIT(8) },
+	[RST_BUS_DE_BE]		=  { 0x2c4, BIT(12) },
+	[RST_BUS_DE_FE]		=  { 0x2c4, BIT(14) },
+	[RST_BUS_GPU]		=  { 0x2c4, BIT(20) },
+	[RST_BUS_MSGBOX]	=  { 0x2c4, BIT(21) },
+	[RST_BUS_SPINLOCK]	=  { 0x2c4, BIT(22) },
+	[RST_BUS_DRC]		=  { 0x2c4, BIT(25) },
+
+	[RST_BUS_LVDS]		=  { 0x2c8, BIT(0) },
+
+	[RST_BUS_CODEC]		=  { 0x2d0, BIT(0) },
+	[RST_BUS_I2S0]		=  { 0x2d0, BIT(12) },
+	[RST_BUS_I2S1]		=  { 0x2d0, BIT(13) },
+
+	[RST_BUS_I2C0]		=  { 0x2d8, BIT(0) },
+	[RST_BUS_I2C1]		=  { 0x2d8, BIT(1) },
+	[RST_BUS_I2C2]		=  { 0x2d8, BIT(2) },
+	[RST_BUS_UART0]		=  { 0x2d8, BIT(16) },
+	[RST_BUS_UART1]		=  { 0x2d8, BIT(17) },
+	[RST_BUS_UART2]		=  { 0x2d8, BIT(18) },
+	[RST_BUS_UART3]		=  { 0x2d8, BIT(19) },
+	[RST_BUS_UART4]		=  { 0x2d8, BIT(20) },
+};
+
+static const struct sunxi_ccu_desc sun8i_a23_ccu_desc = {
+	.ccu_clks	= sun8i_a23_ccu_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun8i_a23_ccu_clks),
+
+	.hw_clks	= &sun8i_a23_hw_clks,
+
+	.resets		= sun8i_a23_ccu_resets,
+	.num_resets	= ARRAY_SIZE(sun8i_a23_ccu_resets),
+};
+
+static void __init sun8i_a23_ccu_setup(struct device_node *node)
+{
+	void __iomem *reg;
+	u32 val;
+
+	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(reg)) {
+		pr_err("%s: Could not map the clock registers\n",
+		       of_node_full_name(node));
+		return;
+	}
+
+	/* Force the PLL-Audio-1x divider to 4 */
+	val = readl(reg + SUN8I_A23_PLL_AUDIO_REG);
+	val &= ~GENMASK(19, 16);
+	writel(val | (3 << 16), reg + SUN8I_A23_PLL_AUDIO_REG);
+
+	/* Force PLL-MIPI to MIPI mode */
+	val = readl(reg + SUN8I_A23_PLL_MIPI_REG);
+	val &= ~BIT(16);
+	writel(val, reg + SUN8I_A23_PLL_MIPI_REG);
+
+	sunxi_ccu_probe(node, reg, &sun8i_a23_ccu_desc);
+}
+CLK_OF_DECLARE(sun8i_a23_ccu, "allwinner,sun8i-a23-ccu",
+	       sun8i_a23_ccu_setup);
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
new file mode 100644
index 000000000000..96b40ca57697
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
@@ -0,0 +1,780 @@
+/*
+ * Copyright (c) 2016 Maxime Ripard. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+
+#include "ccu_common.h"
+#include "ccu_reset.h"
+
+#include "ccu_div.h"
+#include "ccu_gate.h"
+#include "ccu_mp.h"
+#include "ccu_mult.h"
+#include "ccu_nk.h"
+#include "ccu_nkm.h"
+#include "ccu_nkmp.h"
+#include "ccu_nm.h"
+#include "ccu_phase.h"
+
+#include "ccu-sun8i-a23-a33.h"
+
+static struct ccu_nkmp pll_cpux_clk = {
+	.enable = BIT(31),
+	.lock	= BIT(28),
+
+	.n	= _SUNXI_CCU_MULT(8, 5),
+	.k	= _SUNXI_CCU_MULT(4, 2),
+	.m	= _SUNXI_CCU_DIV(0, 2),
+	.p	= _SUNXI_CCU_DIV_MAX(16, 2, 4),
+
+	.common	= {
+		.reg		= 0x000,
+		.hw.init	= CLK_HW_INIT("pll-cpux", "osc24M",
+					      &ccu_nkmp_ops,
+					      0),
+	},
+};
+
+/*
+ * The Audio PLL is supposed to have 4 outputs: 3 fixed factors from
+ * the base (2x, 4x and 8x), and one variable divider (the one true
+ * pll audio).
+ *
+ * We don't have any need for the variable divider for now, so we just
+ * hardcode it to match with the clock names
+ */
+#define SUN8I_A33_PLL_AUDIO_REG	0x008
+
+static SUNXI_CCU_NM_WITH_GATE_LOCK(pll_audio_base_clk, "pll-audio-base",
+				   "osc24M", 0x008,
+				   8, 7,		/* N */
+				   0, 5,		/* M */
+				   BIT(31),		/* gate */
+				   BIT(28),		/* lock */
+				   CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_video_clk, "pll-video",
+					"osc24M", 0x010,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_ve_clk, "pll-ve",
+					"osc24M", 0x018,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_ddr0_clk, "pll-ddr0",
+				    "osc24M", 0x020,
+				    8, 5,		/* N */
+				    4, 2,		/* K */
+				    0, 2,		/* M */
+				    BIT(31),		/* gate */
+				    BIT(28),		/* lock */
+				    0);
+
+static SUNXI_CCU_NK_WITH_GATE_LOCK_POSTDIV(pll_periph_clk, "pll-periph",
+					   "osc24M", 0x028,
+					   8, 5,	/* N */
+					   4, 2,	/* K */
+					   BIT(31),	/* gate */
+					   BIT(28),	/* lock */
+					   2,		/* post-div */
+					   CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_gpu_clk, "pll-gpu",
+					"osc24M", 0x038,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+/*
+ * The MIPI PLL has 2 modes: "MIPI" and "HDMI".
+ *
+ * The MIPI mode is a standard NKM-style clock. The HDMI mode is an
+ * integer / fractional clock with switchable multipliers and dividers.
+ * This is not supported here. We hardcode the PLL to MIPI mode.
+ */
+#define SUN8I_A33_PLL_MIPI_REG	0x040
+static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_mipi_clk, "pll-mipi",
+				    "pll-video", 0x040,
+				    8, 4,		/* N */
+				    4, 2,		/* K */
+				    0, 4,		/* M */
+				    BIT(31),		/* gate */
+				    BIT(28),		/* lock */
+				    CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_hsic_clk, "pll-hsic",
+					"osc24M", 0x044,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_de_clk, "pll-de",
+					"osc24M", 0x048,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+/* TODO: Fix N */
+static SUNXI_CCU_N_WITH_GATE_LOCK(pll_ddr1_clk, "pll-ddr1",
+				  "osc24M", 0x04c,
+				  8, 6,			/* N */
+				  BIT(31),		/* gate */
+				  BIT(28),		/* lock */
+				  CLK_SET_RATE_UNGATE);
+
+static const char * const cpux_parents[] = { "osc32k", "osc24M",
+					     "pll-cpux" , "pll-cpux" };
+static SUNXI_CCU_MUX(cpux_clk, "cpux", cpux_parents,
+		     0x050, 16, 2, CLK_IS_CRITICAL);
+
+static SUNXI_CCU_M(axi_clk, "axi", "cpux", 0x050, 0, 2, 0);
+
+static const char * const ahb1_parents[] = { "osc32k", "osc24M",
+					     "axi" , "pll-periph" };
+static struct ccu_div ahb1_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO),
+
+	.mux		= {
+		.shift	= 12,
+		.width	= 2,
+
+		.variable_prediv	= {
+			.index	= 3,
+			.shift	= 6,
+			.width	= 2,
+		},
+	},
+
+	.common		= {
+		.reg		= 0x054,
+		.features	= CCU_FEATURE_VARIABLE_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("ahb1",
+						      ahb1_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct clk_div_table apb1_div_table[] = {
+	{ .val = 0, .div = 2 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 4 },
+	{ .val = 3, .div = 8 },
+	{ /* Sentinel */ },
+};
+static SUNXI_CCU_DIV_TABLE(apb1_clk, "apb1", "ahb1",
+			   0x054, 8, 2, apb1_div_table, 0);
+
+static const char * const apb2_parents[] = { "osc32k", "osc24M",
+					     "pll-periph" , "pll-periph" };
+static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", apb2_parents, 0x058,
+			     0, 5,	/* M */
+			     16, 2,	/* P */
+			     24, 2,	/* mux */
+			     0);
+
+static SUNXI_CCU_GATE(bus_mipi_dsi_clk,	"bus-mipi-dsi",	"ahb1",
+		      0x060, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_ss_clk,	"bus-ss",	"ahb1",
+		      0x060, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_dma_clk,	"bus-dma",	"ahb1",
+		      0x060, BIT(6), 0);
+static SUNXI_CCU_GATE(bus_mmc0_clk,	"bus-mmc0",	"ahb1",
+		      0x060, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_mmc1_clk,	"bus-mmc1",	"ahb1",
+		      0x060, BIT(9), 0);
+static SUNXI_CCU_GATE(bus_mmc2_clk,	"bus-mmc2",	"ahb1",
+		      0x060, BIT(10), 0);
+static SUNXI_CCU_GATE(bus_nand_clk,	"bus-nand",	"ahb1",
+		      0x060, BIT(13), 0);
+static SUNXI_CCU_GATE(bus_dram_clk,	"bus-dram",	"ahb1",
+		      0x060, BIT(14), 0);
+static SUNXI_CCU_GATE(bus_hstimer_clk,	"bus-hstimer",	"ahb1",
+		      0x060, BIT(19), 0);
+static SUNXI_CCU_GATE(bus_spi0_clk,	"bus-spi0",	"ahb1",
+		      0x060, BIT(20), 0);
+static SUNXI_CCU_GATE(bus_spi1_clk,	"bus-spi1",	"ahb1",
+		      0x060, BIT(21), 0);
+static SUNXI_CCU_GATE(bus_otg_clk,	"bus-otg",	"ahb1",
+		      0x060, BIT(24), 0);
+static SUNXI_CCU_GATE(bus_ehci_clk,	"bus-ehci",	"ahb1",
+		      0x060, BIT(26), 0);
+static SUNXI_CCU_GATE(bus_ohci_clk,	"bus-ohci",	"ahb1",
+		      0x060, BIT(29), 0);
+
+static SUNXI_CCU_GATE(bus_ve_clk,	"bus-ve",	"ahb1",
+		      0x064, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_lcd_clk,	"bus-lcd",	"ahb1",
+		      0x064, BIT(4), 0);
+static SUNXI_CCU_GATE(bus_csi_clk,	"bus-csi",	"ahb1",
+		      0x064, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_de_be_clk,	"bus-de-be",	"ahb1",
+		      0x064, BIT(12), 0);
+static SUNXI_CCU_GATE(bus_de_fe_clk,	"bus-de-fe",	"ahb1",
+		      0x064, BIT(14), 0);
+static SUNXI_CCU_GATE(bus_gpu_clk,	"bus-gpu",	"ahb1",
+		      0x064, BIT(20), 0);
+static SUNXI_CCU_GATE(bus_msgbox_clk,	"bus-msgbox",	"ahb1",
+		      0x064, BIT(21), 0);
+static SUNXI_CCU_GATE(bus_spinlock_clk,	"bus-spinlock",	"ahb1",
+		      0x064, BIT(22), 0);
+static SUNXI_CCU_GATE(bus_drc_clk,	"bus-drc",	"ahb1",
+		      0x064, BIT(25), 0);
+static SUNXI_CCU_GATE(bus_sat_clk,	"bus-sat",	"ahb1",
+		      0x064, BIT(26), 0);
+
+static SUNXI_CCU_GATE(bus_codec_clk,	"bus-codec",	"apb1",
+		      0x068, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_pio_clk,	"bus-pio",	"apb1",
+		      0x068, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_i2s0_clk,	"bus-i2s0",	"apb1",
+		      0x068, BIT(12), 0);
+static SUNXI_CCU_GATE(bus_i2s1_clk,	"bus-i2s1",	"apb1",
+		      0x068, BIT(13), 0);
+
+static SUNXI_CCU_GATE(bus_i2c0_clk,	"bus-i2c0",	"apb2",
+		      0x06c, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_i2c1_clk,	"bus-i2c1",	"apb2",
+		      0x06c, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_i2c2_clk,	"bus-i2c2",	"apb2",
+		      0x06c, BIT(2), 0);
+static SUNXI_CCU_GATE(bus_uart0_clk,	"bus-uart0",	"apb2",
+		      0x06c, BIT(16), 0);
+static SUNXI_CCU_GATE(bus_uart1_clk,	"bus-uart1",	"apb2",
+		      0x06c, BIT(17), 0);
+static SUNXI_CCU_GATE(bus_uart2_clk,	"bus-uart2",	"apb2",
+		      0x06c, BIT(18), 0);
+static SUNXI_CCU_GATE(bus_uart3_clk,	"bus-uart3",	"apb2",
+		      0x06c, BIT(19), 0);
+static SUNXI_CCU_GATE(bus_uart4_clk,	"bus-uart4",	"apb2",
+		      0x06c, BIT(20), 0);
+
+static const char * const mod0_default_parents[] = { "osc24M", "pll-periph" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(nand_clk, "nand", mod0_default_parents, 0x080,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc0_clk, "mmc0", mod0_default_parents, 0x088,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc0_sample_clk, "mmc0_sample", "mmc0",
+		       0x088, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc0_output_clk, "mmc0_output", "mmc0",
+		       0x088, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc1_clk, "mmc1", mod0_default_parents, 0x08c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc1_sample_clk, "mmc1_sample", "mmc1",
+		       0x08c, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc1_output_clk, "mmc1_output", "mmc1",
+		       0x08c, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc2_clk, "mmc2", mod0_default_parents, 0x090,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_PHASE(mmc2_sample_clk, "mmc2_sample", "mmc2",
+		       0x090, 20, 3, 0);
+static SUNXI_CCU_PHASE(mmc2_output_clk, "mmc2_output", "mmc2",
+		       0x090, 8, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(ss_clk, "ss", mod0_default_parents, 0x09c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", mod0_default_parents, 0x0a0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const i2s_parents[] = { "pll-audio-8x", "pll-audio-4x",
+					    "pll-audio-2x", "pll-audio" };
+static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", i2s_parents,
+			       0x0b0, 16, 2, BIT(31), 0);
+
+static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", i2s_parents,
+			       0x0b4, 16, 2, BIT(31), 0);
+
+/* TODO: the parent for most of the USB clocks is not known */
+static SUNXI_CCU_GATE(usb_phy0_clk,	"usb-phy0",	"osc24M",
+		      0x0cc, BIT(8), 0);
+static SUNXI_CCU_GATE(usb_phy1_clk,	"usb-phy1",	"osc24M",
+		      0x0cc, BIT(9), 0);
+static SUNXI_CCU_GATE(usb_hsic_clk,	"usb-hsic",	"pll-hsic",
+		      0x0cc, BIT(10), 0);
+static SUNXI_CCU_GATE(usb_hsic_12M_clk,	"usb-hsic-12M",	"osc24M",
+		      0x0cc, BIT(11), 0);
+static SUNXI_CCU_GATE(usb_ohci_clk,	"usb-ohci",	"osc24M",
+		      0x0cc, BIT(16), 0);
+
+static SUNXI_CCU_M(dram_clk, "dram", "pll-ddr",
+		   0x0f4, 0, 4, CLK_IS_CRITICAL);
+
+static const char * const pll_ddr_parents[] = { "pll-ddr0", "pll-ddr1" };
+static SUNXI_CCU_MUX(pll_ddr_clk, "pll-ddr", pll_ddr_parents,
+		     0x0f8, 16, 1, 0);
+
+static SUNXI_CCU_GATE(dram_ve_clk,	"dram-ve",	"dram",
+		      0x100, BIT(0), 0);
+static SUNXI_CCU_GATE(dram_csi_clk,	"dram-csi",	"dram",
+		      0x100, BIT(1), 0);
+static SUNXI_CCU_GATE(dram_drc_clk,	"dram-drc",	"dram",
+		      0x100, BIT(16), 0);
+static SUNXI_CCU_GATE(dram_de_fe_clk,	"dram-de-fe",	"dram",
+		      0x100, BIT(24), 0);
+static SUNXI_CCU_GATE(dram_de_be_clk,	"dram-de-be",	"dram",
+		      0x100, BIT(26), 0);
+
+static const char * const de_parents[] = { "pll-video", "pll-periph-2x",
+					   "pll-gpu", "pll-de" };
+static const u8 de_table[] = { 0, 2, 3, 5 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(de_be_clk, "de-be",
+				       de_parents, de_table,
+				       0x104, 0, 4, 24, 3, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(de_fe_clk, "de-fe",
+				       de_parents, de_table,
+				       0x10c, 0, 4, 24, 3, BIT(31), 0);
+
+static const char * const lcd_ch0_parents[] = { "pll-video", "pll-video-2x",
+						"pll-mipi" };
+static const u8 lcd_ch0_table[] = { 0, 2, 4 };
+static SUNXI_CCU_MUX_TABLE_WITH_GATE(lcd_ch0_clk, "lcd-ch0",
+				     lcd_ch0_parents, lcd_ch0_table,
+				     0x118, 24, 3, BIT(31),
+				     CLK_SET_RATE_PARENT);
+
+static const char * const lcd_ch1_parents[] = { "pll-video", "pll-video-2x" };
+static const u8 lcd_ch1_table[] = { 0, 2 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(lcd_ch1_clk, "lcd-ch1",
+				       lcd_ch1_parents, lcd_ch1_table,
+				       0x12c, 0, 4, 24, 2, BIT(31), 0);
+
+static const char * const csi_sclk_parents[] = { "pll-video", "pll-de",
+						 "pll-mipi", "pll-ve" };
+static const u8 csi_sclk_table[] = { 0, 3, 4, 5 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(csi_sclk_clk, "csi-sclk",
+				       csi_sclk_parents, csi_sclk_table,
+				       0x134, 16, 4, 24, 3, BIT(31), 0);
+
+static const char * const csi_mclk_parents[] = { "pll-video", "pll-de",
+						 "osc24M" };
+static const u8 csi_mclk_table[] = { 0, 3, 5 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(csi_mclk_clk, "csi-mclk",
+				       csi_mclk_parents, csi_mclk_table,
+				       0x134, 0, 5, 8, 3, BIT(15), 0);
+
+static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve",
+			     0x13c, 16, 3, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(ac_dig_clk,	"ac-dig",	"pll-audio",
+		      0x140, BIT(31), 0);
+static SUNXI_CCU_GATE(ac_dig_4x_clk,	"ac-dig-4x",	"pll-audio-4x",
+		      0x140, BIT(30), 0);
+static SUNXI_CCU_GATE(avs_clk,		"avs",		"osc24M",
+		      0x144, BIT(31), 0);
+
+static const char * const mbus_parents[] = { "osc24M", "pll-periph-2x",
+					     "pll-ddr0", "pll-ddr1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(mbus_clk, "mbus", mbus_parents,
+				 0x15c, 0, 3, 24, 2, BIT(31), CLK_IS_CRITICAL);
+
+static const char * const dsi_sclk_parents[] = { "pll-video", "pll-video-2x" };
+static const u8 dsi_sclk_table[] = { 0, 2 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(dsi_sclk_clk, "dsi-sclk",
+				       dsi_sclk_parents, dsi_sclk_table,
+				       0x168, 16, 4, 24, 2, BIT(31), 0);
+
+static const char * const dsi_dphy_parents[] = { "pll-video", "pll-periph" };
+static const u8 dsi_dphy_table[] = { 0, 2 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(dsi_dphy_clk, "dsi-dphy",
+				       dsi_dphy_parents, dsi_dphy_table,
+				       0x168, 0, 4, 8, 2, BIT(15), 0);
+
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(drc_clk, "drc",
+				       de_parents, de_table,
+				       0x180, 0, 4, 24, 3, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_GATE(gpu_clk, "gpu", "pll-gpu",
+			     0x1a0, 0, 3, BIT(31), 0);
+
+static const char * const ats_parents[] = { "osc24M", "pll-periph" };
+static SUNXI_CCU_M_WITH_MUX_GATE(ats_clk, "ats", ats_parents,
+				 0x1b0, 0, 3, 24, 2, BIT(31), 0);
+
+static struct ccu_common *sun8i_a33_ccu_clks[] = {
+	&pll_cpux_clk.common,
+	&pll_audio_base_clk.common,
+	&pll_video_clk.common,
+	&pll_ve_clk.common,
+	&pll_ddr0_clk.common,
+	&pll_periph_clk.common,
+	&pll_gpu_clk.common,
+	&pll_mipi_clk.common,
+	&pll_hsic_clk.common,
+	&pll_de_clk.common,
+	&pll_ddr1_clk.common,
+	&pll_ddr_clk.common,
+	&cpux_clk.common,
+	&axi_clk.common,
+	&ahb1_clk.common,
+	&apb1_clk.common,
+	&apb2_clk.common,
+	&bus_mipi_dsi_clk.common,
+	&bus_ss_clk.common,
+	&bus_dma_clk.common,
+	&bus_mmc0_clk.common,
+	&bus_mmc1_clk.common,
+	&bus_mmc2_clk.common,
+	&bus_nand_clk.common,
+	&bus_dram_clk.common,
+	&bus_hstimer_clk.common,
+	&bus_spi0_clk.common,
+	&bus_spi1_clk.common,
+	&bus_otg_clk.common,
+	&bus_ehci_clk.common,
+	&bus_ohci_clk.common,
+	&bus_ve_clk.common,
+	&bus_lcd_clk.common,
+	&bus_csi_clk.common,
+	&bus_de_fe_clk.common,
+	&bus_de_be_clk.common,
+	&bus_gpu_clk.common,
+	&bus_msgbox_clk.common,
+	&bus_spinlock_clk.common,
+	&bus_drc_clk.common,
+	&bus_sat_clk.common,
+	&bus_codec_clk.common,
+	&bus_pio_clk.common,
+	&bus_i2s0_clk.common,
+	&bus_i2s1_clk.common,
+	&bus_i2c0_clk.common,
+	&bus_i2c1_clk.common,
+	&bus_i2c2_clk.common,
+	&bus_uart0_clk.common,
+	&bus_uart1_clk.common,
+	&bus_uart2_clk.common,
+	&bus_uart3_clk.common,
+	&bus_uart4_clk.common,
+	&nand_clk.common,
+	&mmc0_clk.common,
+	&mmc0_sample_clk.common,
+	&mmc0_output_clk.common,
+	&mmc1_clk.common,
+	&mmc1_sample_clk.common,
+	&mmc1_output_clk.common,
+	&mmc2_clk.common,
+	&mmc2_sample_clk.common,
+	&mmc2_output_clk.common,
+	&ss_clk.common,
+	&spi0_clk.common,
+	&spi1_clk.common,
+	&i2s0_clk.common,
+	&i2s1_clk.common,
+	&usb_phy0_clk.common,
+	&usb_phy1_clk.common,
+	&usb_hsic_clk.common,
+	&usb_hsic_12M_clk.common,
+	&usb_ohci_clk.common,
+	&dram_clk.common,
+	&dram_ve_clk.common,
+	&dram_csi_clk.common,
+	&dram_drc_clk.common,
+	&dram_de_fe_clk.common,
+	&dram_de_be_clk.common,
+	&de_be_clk.common,
+	&de_fe_clk.common,
+	&lcd_ch0_clk.common,
+	&lcd_ch1_clk.common,
+	&csi_sclk_clk.common,
+	&csi_mclk_clk.common,
+	&ve_clk.common,
+	&ac_dig_clk.common,
+	&ac_dig_4x_clk.common,
+	&avs_clk.common,
+	&mbus_clk.common,
+	&dsi_sclk_clk.common,
+	&dsi_dphy_clk.common,
+	&drc_clk.common,
+	&gpu_clk.common,
+	&ats_clk.common,
+};
+
+/* We hardcode the divider to 4 for now */
+static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
+			"pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
+			"pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
+			"pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
+			"pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_periph_2x_clk, "pll-periph-2x",
+			"pll-periph", 1, 2, 0);
+static CLK_FIXED_FACTOR(pll_video_2x_clk, "pll-video-2x",
+			"pll-video", 1, 2, 0);
+
+static struct clk_hw_onecell_data sun8i_a33_hw_clks = {
+	.hws	= {
+		[CLK_PLL_CPUX]		= &pll_cpux_clk.common.hw,
+		[CLK_PLL_AUDIO_BASE]	= &pll_audio_base_clk.common.hw,
+		[CLK_PLL_AUDIO]		= &pll_audio_clk.hw,
+		[CLK_PLL_AUDIO_2X]	= &pll_audio_2x_clk.hw,
+		[CLK_PLL_AUDIO_4X]	= &pll_audio_4x_clk.hw,
+		[CLK_PLL_AUDIO_8X]	= &pll_audio_8x_clk.hw,
+		[CLK_PLL_VIDEO]		= &pll_video_clk.common.hw,
+		[CLK_PLL_VIDEO_2X]	= &pll_video_2x_clk.hw,
+		[CLK_PLL_VE]		= &pll_ve_clk.common.hw,
+		[CLK_PLL_DDR0]		= &pll_ddr0_clk.common.hw,
+		[CLK_PLL_PERIPH]	= &pll_periph_clk.common.hw,
+		[CLK_PLL_PERIPH_2X]	= &pll_periph_2x_clk.hw,
+		[CLK_PLL_GPU]		= &pll_gpu_clk.common.hw,
+		[CLK_PLL_MIPI]		= &pll_mipi_clk.common.hw,
+		[CLK_PLL_HSIC]		= &pll_hsic_clk.common.hw,
+		[CLK_PLL_DE]		= &pll_de_clk.common.hw,
+		[CLK_PLL_DDR1]		= &pll_ddr1_clk.common.hw,
+		[CLK_PLL_DDR]		= &pll_ddr_clk.common.hw,
+		[CLK_CPUX]		= &cpux_clk.common.hw,
+		[CLK_AXI]		= &axi_clk.common.hw,
+		[CLK_AHB1]		= &ahb1_clk.common.hw,
+		[CLK_APB1]		= &apb1_clk.common.hw,
+		[CLK_APB2]		= &apb2_clk.common.hw,
+		[CLK_BUS_MIPI_DSI]	= &bus_mipi_dsi_clk.common.hw,
+		[CLK_BUS_SS]		= &bus_ss_clk.common.hw,
+		[CLK_BUS_DMA]		= &bus_dma_clk.common.hw,
+		[CLK_BUS_MMC0]		= &bus_mmc0_clk.common.hw,
+		[CLK_BUS_MMC1]		= &bus_mmc1_clk.common.hw,
+		[CLK_BUS_MMC2]		= &bus_mmc2_clk.common.hw,
+		[CLK_BUS_NAND]		= &bus_nand_clk.common.hw,
+		[CLK_BUS_DRAM]		= &bus_dram_clk.common.hw,
+		[CLK_BUS_HSTIMER]	= &bus_hstimer_clk.common.hw,
+		[CLK_BUS_SPI0]		= &bus_spi0_clk.common.hw,
+		[CLK_BUS_SPI1]		= &bus_spi1_clk.common.hw,
+		[CLK_BUS_OTG]		= &bus_otg_clk.common.hw,
+		[CLK_BUS_EHCI]		= &bus_ehci_clk.common.hw,
+		[CLK_BUS_OHCI]		= &bus_ohci_clk.common.hw,
+		[CLK_BUS_VE]		= &bus_ve_clk.common.hw,
+		[CLK_BUS_LCD]		= &bus_lcd_clk.common.hw,
+		[CLK_BUS_CSI]		= &bus_csi_clk.common.hw,
+		[CLK_BUS_DE_BE]		= &bus_de_be_clk.common.hw,
+		[CLK_BUS_DE_FE]		= &bus_de_fe_clk.common.hw,
+		[CLK_BUS_GPU]		= &bus_gpu_clk.common.hw,
+		[CLK_BUS_MSGBOX]	= &bus_msgbox_clk.common.hw,
+		[CLK_BUS_SPINLOCK]	= &bus_spinlock_clk.common.hw,
+		[CLK_BUS_DRC]		= &bus_drc_clk.common.hw,
+		[CLK_BUS_SAT]		= &bus_sat_clk.common.hw,
+		[CLK_BUS_CODEC]		= &bus_codec_clk.common.hw,
+		[CLK_BUS_PIO]		= &bus_pio_clk.common.hw,
+		[CLK_BUS_I2S0]		= &bus_i2s0_clk.common.hw,
+		[CLK_BUS_I2S1]		= &bus_i2s1_clk.common.hw,
+		[CLK_BUS_I2C0]		= &bus_i2c0_clk.common.hw,
+		[CLK_BUS_I2C1]		= &bus_i2c1_clk.common.hw,
+		[CLK_BUS_I2C2]		= &bus_i2c2_clk.common.hw,
+		[CLK_BUS_UART0]		= &bus_uart0_clk.common.hw,
+		[CLK_BUS_UART1]		= &bus_uart1_clk.common.hw,
+		[CLK_BUS_UART2]		= &bus_uart2_clk.common.hw,
+		[CLK_BUS_UART3]		= &bus_uart3_clk.common.hw,
+		[CLK_BUS_UART4]		= &bus_uart4_clk.common.hw,
+		[CLK_NAND]		= &nand_clk.common.hw,
+		[CLK_MMC0]		= &mmc0_clk.common.hw,
+		[CLK_MMC0_SAMPLE]	= &mmc0_sample_clk.common.hw,
+		[CLK_MMC0_OUTPUT]	= &mmc0_output_clk.common.hw,
+		[CLK_MMC1]		= &mmc1_clk.common.hw,
+		[CLK_MMC1_SAMPLE]	= &mmc1_sample_clk.common.hw,
+		[CLK_MMC1_OUTPUT]	= &mmc1_output_clk.common.hw,
+		[CLK_MMC2]		= &mmc2_clk.common.hw,
+		[CLK_MMC2_SAMPLE]	= &mmc2_sample_clk.common.hw,
+		[CLK_MMC2_OUTPUT]	= &mmc2_output_clk.common.hw,
+		[CLK_SS]		= &ss_clk.common.hw,
+		[CLK_SPI0]		= &spi0_clk.common.hw,
+		[CLK_SPI1]		= &spi1_clk.common.hw,
+		[CLK_I2S0]		= &i2s0_clk.common.hw,
+		[CLK_I2S1]		= &i2s1_clk.common.hw,
+		[CLK_USB_PHY0]		= &usb_phy0_clk.common.hw,
+		[CLK_USB_PHY1]		= &usb_phy1_clk.common.hw,
+		[CLK_USB_HSIC]		= &usb_hsic_clk.common.hw,
+		[CLK_USB_HSIC_12M]	= &usb_hsic_12M_clk.common.hw,
+		[CLK_USB_OHCI]		= &usb_ohci_clk.common.hw,
+		[CLK_DRAM]		= &dram_clk.common.hw,
+		[CLK_DRAM_VE]		= &dram_ve_clk.common.hw,
+		[CLK_DRAM_CSI]		= &dram_csi_clk.common.hw,
+		[CLK_DRAM_DRC]		= &dram_drc_clk.common.hw,
+		[CLK_DRAM_DE_FE]	= &dram_de_fe_clk.common.hw,
+		[CLK_DRAM_DE_BE]	= &dram_de_be_clk.common.hw,
+		[CLK_DE_BE]		= &de_be_clk.common.hw,
+		[CLK_DE_FE]		= &de_fe_clk.common.hw,
+		[CLK_LCD_CH0]		= &lcd_ch0_clk.common.hw,
+		[CLK_LCD_CH1]		= &lcd_ch1_clk.common.hw,
+		[CLK_CSI_SCLK]		= &csi_sclk_clk.common.hw,
+		[CLK_CSI_MCLK]		= &csi_mclk_clk.common.hw,
+		[CLK_VE]		= &ve_clk.common.hw,
+		[CLK_AC_DIG]		= &ac_dig_clk.common.hw,
+		[CLK_AC_DIG_4X]		= &ac_dig_4x_clk.common.hw,
+		[CLK_AVS]		= &avs_clk.common.hw,
+		[CLK_MBUS]		= &mbus_clk.common.hw,
+		[CLK_DSI_SCLK]		= &dsi_sclk_clk.common.hw,
+		[CLK_DSI_DPHY]		= &dsi_dphy_clk.common.hw,
+		[CLK_DRC]		= &drc_clk.common.hw,
+		[CLK_GPU]		= &gpu_clk.common.hw,
+		[CLK_ATS]		= &ats_clk.common.hw,
+	},
+	.num	= CLK_NUMBER,
+};
+
+static struct ccu_reset_map sun8i_a33_ccu_resets[] = {
+	[RST_USB_PHY0]		=  { 0x0cc, BIT(0) },
+	[RST_USB_PHY1]		=  { 0x0cc, BIT(1) },
+	[RST_USB_HSIC]		=  { 0x0cc, BIT(2) },
+
+	[RST_MBUS]		=  { 0x0fc, BIT(31) },
+
+	[RST_BUS_MIPI_DSI]	=  { 0x2c0, BIT(1) },
+	[RST_BUS_SS]		=  { 0x2c0, BIT(5) },
+	[RST_BUS_DMA]		=  { 0x2c0, BIT(6) },
+	[RST_BUS_MMC0]		=  { 0x2c0, BIT(8) },
+	[RST_BUS_MMC1]		=  { 0x2c0, BIT(9) },
+	[RST_BUS_MMC2]		=  { 0x2c0, BIT(10) },
+	[RST_BUS_NAND]		=  { 0x2c0, BIT(13) },
+	[RST_BUS_DRAM]		=  { 0x2c0, BIT(14) },
+	[RST_BUS_HSTIMER]	=  { 0x2c0, BIT(19) },
+	[RST_BUS_SPI0]		=  { 0x2c0, BIT(20) },
+	[RST_BUS_SPI1]		=  { 0x2c0, BIT(21) },
+	[RST_BUS_OTG]		=  { 0x2c0, BIT(24) },
+	[RST_BUS_EHCI]		=  { 0x2c0, BIT(26) },
+	[RST_BUS_OHCI]		=  { 0x2c0, BIT(29) },
+
+	[RST_BUS_VE]		=  { 0x2c4, BIT(0) },
+	[RST_BUS_LCD]		=  { 0x2c4, BIT(4) },
+	[RST_BUS_CSI]		=  { 0x2c4, BIT(8) },
+	[RST_BUS_DE_BE]		=  { 0x2c4, BIT(12) },
+	[RST_BUS_DE_FE]		=  { 0x2c4, BIT(14) },
+	[RST_BUS_GPU]		=  { 0x2c4, BIT(20) },
+	[RST_BUS_MSGBOX]	=  { 0x2c4, BIT(21) },
+	[RST_BUS_SPINLOCK]	=  { 0x2c4, BIT(22) },
+	[RST_BUS_DRC]		=  { 0x2c4, BIT(25) },
+	[RST_BUS_SAT]		=  { 0x2c4, BIT(26) },
+
+	[RST_BUS_LVDS]		=  { 0x2c8, BIT(0) },
+
+	[RST_BUS_CODEC]		=  { 0x2d0, BIT(0) },
+	[RST_BUS_I2S0]		=  { 0x2d0, BIT(12) },
+	[RST_BUS_I2S1]		=  { 0x2d0, BIT(13) },
+
+	[RST_BUS_I2C0]		=  { 0x2d8, BIT(0) },
+	[RST_BUS_I2C1]		=  { 0x2d8, BIT(1) },
+	[RST_BUS_I2C2]		=  { 0x2d8, BIT(2) },
+	[RST_BUS_UART0]		=  { 0x2d8, BIT(16) },
+	[RST_BUS_UART1]		=  { 0x2d8, BIT(17) },
+	[RST_BUS_UART2]		=  { 0x2d8, BIT(18) },
+	[RST_BUS_UART3]		=  { 0x2d8, BIT(19) },
+	[RST_BUS_UART4]		=  { 0x2d8, BIT(20) },
+};
+
+static const struct sunxi_ccu_desc sun8i_a33_ccu_desc = {
+	.ccu_clks	= sun8i_a33_ccu_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun8i_a33_ccu_clks),
+
+	.hw_clks	= &sun8i_a33_hw_clks,
+
+	.resets		= sun8i_a33_ccu_resets,
+	.num_resets	= ARRAY_SIZE(sun8i_a33_ccu_resets),
+};
+
+static void __init sun8i_a33_ccu_setup(struct device_node *node)
+{
+	void __iomem *reg;
+	u32 val;
+
+	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(reg)) {
+		pr_err("%s: Could not map the clock registers\n",
+		       of_node_full_name(node));
+		return;
+	}
+
+	/* Force the PLL-Audio-1x divider to 4 */
+	val = readl(reg + SUN8I_A33_PLL_AUDIO_REG);
+	val &= ~GENMASK(19, 16);
+	writel(val | (3 << 16), reg + SUN8I_A33_PLL_AUDIO_REG);
+
+	/* Force PLL-MIPI to MIPI mode */
+	val = readl(reg + SUN8I_A33_PLL_MIPI_REG);
+	val &= ~BIT(16);
+	writel(val, reg + SUN8I_A33_PLL_MIPI_REG);
+
+	sunxi_ccu_probe(node, reg, &sun8i_a33_ccu_desc);
+}
+CLK_OF_DECLARE(sun8i_a33_ccu, "allwinner,sun8i-a33-ccu",
+	       sun8i_a33_ccu_setup);
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
index 9af359544110..4d70590f05e3 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
@@ -184,15 +184,15 @@ static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", apb2_parents, 0x058,
 			     0);
 
 static const char * const ahb2_parents[] = { "ahb1" , "pll-periph0" };
+static const struct ccu_mux_fixed_prediv ahb2_fixed_predivs[] = {
+	{ .index = 1, .div = 2 },
+};
 static struct ccu_mux ahb2_clk = {
 	.mux		= {
 		.shift	= 0,
 		.width	= 1,
-
-		.fixed_prediv	= {
-			.index	= 1,
-			.div	= 2,
-		},
+		.fixed_predivs	= ahb2_fixed_predivs,
+		.n_predivs	= ARRAY_SIZE(ahb2_fixed_predivs),
 	},
 
 	.common		= {
@@ -783,14 +783,14 @@ static struct ccu_reset_map sun8i_h3_ccu_resets[] = {
 	[RST_BUS_I2S1]		=  { 0x2d0, BIT(13) },
 	[RST_BUS_I2S2]		=  { 0x2d0, BIT(14) },
 
-	[RST_BUS_I2C0]		=  { 0x2d4, BIT(0) },
-	[RST_BUS_I2C1]		=  { 0x2d4, BIT(1) },
-	[RST_BUS_I2C2]		=  { 0x2d4, BIT(2) },
-	[RST_BUS_UART0]		=  { 0x2d4, BIT(16) },
-	[RST_BUS_UART1]		=  { 0x2d4, BIT(17) },
-	[RST_BUS_UART2]		=  { 0x2d4, BIT(18) },
-	[RST_BUS_UART3]		=  { 0x2d4, BIT(19) },
-	[RST_BUS_SCR]		=  { 0x2d4, BIT(20) },
+	[RST_BUS_I2C0]		=  { 0x2d8, BIT(0) },
+	[RST_BUS_I2C1]		=  { 0x2d8, BIT(1) },
+	[RST_BUS_I2C2]		=  { 0x2d8, BIT(2) },
+	[RST_BUS_UART0]		=  { 0x2d8, BIT(16) },
+	[RST_BUS_UART1]		=  { 0x2d8, BIT(17) },
+	[RST_BUS_UART2]		=  { 0x2d8, BIT(18) },
+	[RST_BUS_UART3]		=  { 0x2d8, BIT(19) },
+	[RST_BUS_SCR]		=  { 0x2d8, BIT(20) },
 };
 
 static const struct sunxi_ccu_desc sun8i_h3_ccu_desc = {
diff --git a/drivers/clk/sunxi-ng/ccu_common.c b/drivers/clk/sunxi-ng/ccu_common.c
index fc17b5295e16..51d4bac97ab3 100644
--- a/drivers/clk/sunxi-ng/ccu_common.c
+++ b/drivers/clk/sunxi-ng/ccu_common.c
@@ -31,7 +31,7 @@ void ccu_helper_wait_for_lock(struct ccu_common *common, u32 lock)
 		return;
 
 	WARN_ON(readl_relaxed_poll_timeout(common->base + common->reg, reg,
-					   !(reg & lock), 100, 70000));
+					   reg & lock, 100, 70000));
 }
 
 int sunxi_ccu_probe(struct device_node *node, void __iomem *reg,
diff --git a/drivers/clk/sunxi-ng/ccu_div.h b/drivers/clk/sunxi-ng/ccu_div.h
index 653ade5769b3..34c338832c0d 100644
--- a/drivers/clk/sunxi-ng/ccu_div.h
+++ b/drivers/clk/sunxi-ng/ccu_div.h
@@ -19,10 +19,29 @@
 #include "ccu_common.h"
 #include "ccu_mux.h"
 
+/**
+ * struct _ccu_div - Internal divider description
+ * @shift: Bit offset of the divider in its register
+ * @width: Width of the divider field in its register
+ * @max: Maximum value allowed for that divider. This is the
+ *       arithmetic value, not the maximum value to be set in the
+ *       register.
+ * @flags: clk_divider flags to apply on this divider
+ * @table: Divider table pointer (if applicable)
+ *
+ * That structure represents a single divider, and is meant to be
+ * embedded in other structures representing the various clock
+ * classes.
+ *
+ * It is basically a wrapper around the clk_divider functions
+ * arguments.
+ */
 struct _ccu_div {
 	u8			shift;
 	u8			width;
 
+	u32			max;
+
 	u32			flags;
 
 	struct clk_div_table	*table;
@@ -36,14 +55,25 @@ struct _ccu_div {
 		.table	= _table,					\
 	}
 
-#define _SUNXI_CCU_DIV_FLAGS(_shift, _width, _flags)			\
-	_SUNXI_CCU_DIV_TABLE_FLAGS(_shift, _width, NULL, _flags)
-
 #define _SUNXI_CCU_DIV_TABLE(_shift, _width, _table)			\
 	_SUNXI_CCU_DIV_TABLE_FLAGS(_shift, _width, _table, 0)
 
+#define _SUNXI_CCU_DIV_MAX_FLAGS(_shift, _width, _max, _flags) \
+	{								\
+		.shift	= _shift,					\
+		.width	= _width,					\
+		.flags	= _flags,					\
+		.max	= _max,						\
+	}
+
+#define _SUNXI_CCU_DIV_FLAGS(_shift, _width, _flags)			\
+	_SUNXI_CCU_DIV_MAX_FLAGS(_shift, _width, 0, _flags)
+
+#define _SUNXI_CCU_DIV_MAX(_shift, _width, _max)			\
+	_SUNXI_CCU_DIV_MAX_FLAGS(_shift, _width, _max, 0)
+
 #define _SUNXI_CCU_DIV(_shift, _width)					\
-	_SUNXI_CCU_DIV_TABLE_FLAGS(_shift, _width, NULL, 0)
+	_SUNXI_CCU_DIV_FLAGS(_shift, _width, 0)
 
 struct ccu_div {
 	u32			enable;
@@ -77,13 +107,16 @@ struct ccu_div {
 				      _shift, _width, _table, 0,	\
 				      _flags)
 
-#define SUNXI_CCU_M_WITH_MUX_GATE(_struct, _name, _parents, _reg,	\
-				  _mshift, _mwidth, _muxshift, _muxwidth, \
-				  _gate, _flags)			\
+#define SUNXI_CCU_M_WITH_MUX_TABLE_GATE(_struct, _name,			\
+					_parents, _table,		\
+					_reg,				\
+					_mshift, _mwidth,		\
+					_muxshift, _muxwidth,		\
+					_gate, _flags)			\
 	struct ccu_div _struct = {					\
 		.enable	= _gate,					\
 		.div	= _SUNXI_CCU_DIV(_mshift, _mwidth),		\
-		.mux	= SUNXI_CLK_MUX(_muxshift, _muxwidth),		\
+		.mux	= _SUNXI_CCU_MUX_TABLE(_muxshift, _muxwidth, _table), \
 		.common	= {						\
 			.reg		= _reg,				\
 			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
@@ -93,12 +126,23 @@ struct ccu_div {
 		},							\
 	}
 
+#define SUNXI_CCU_M_WITH_MUX_GATE(_struct, _name, _parents, _reg,	\
+				  _mshift, _mwidth, _muxshift, _muxwidth, \
+				  _gate, _flags)			\
+	SUNXI_CCU_M_WITH_MUX_TABLE_GATE(_struct, _name,			\
+					_parents, NULL,			\
+					_reg, _mshift, _mwidth,		\
+					_muxshift, _muxwidth,		\
+					_gate, _flags)
+
 #define SUNXI_CCU_M_WITH_MUX(_struct, _name, _parents, _reg,		\
 			     _mshift, _mwidth, _muxshift, _muxwidth,	\
 			     _flags)					\
-	SUNXI_CCU_M_WITH_MUX_GATE(_struct, _name, _parents, _reg,	\
-				  _mshift, _mwidth, _muxshift, _muxwidth, \
-				  0, _flags)
+	SUNXI_CCU_M_WITH_MUX_TABLE_GATE(_struct, _name,			\
+					_parents, NULL,			\
+					_reg, _mshift, _mwidth,		\
+					_muxshift, _muxwidth,		\
+					0, _flags)
 
 
 #define SUNXI_CCU_M_WITH_GATE(_struct, _name, _parent, _reg,		\
diff --git a/drivers/clk/sunxi-ng/ccu_mp.c b/drivers/clk/sunxi-ng/ccu_mp.c
index cbf33ef5faa9..ebb1b31568a5 100644
--- a/drivers/clk/sunxi-ng/ccu_mp.c
+++ b/drivers/clk/sunxi-ng/ccu_mp.c
@@ -21,9 +21,9 @@ static void ccu_mp_find_best(unsigned long parent, unsigned long rate,
 	unsigned int best_m = 0, best_p = 0;
 	unsigned int _m, _p;
 
-	for (_p = 0; _p <= max_p; _p++) {
+	for (_p = 1; _p <= max_p; _p <<= 1) {
 		for (_m = 1; _m <= max_m; _m++) {
-			unsigned long tmp_rate = (parent >> _p) / _m;
+			unsigned long tmp_rate = parent / _p / _m;
 
 			if (tmp_rate > rate)
 				continue;
@@ -46,13 +46,15 @@ static unsigned long ccu_mp_round_rate(struct ccu_mux_internal *mux,
 				       void *data)
 {
 	struct ccu_mp *cmp = data;
+	unsigned int max_m, max_p;
 	unsigned int m, p;
 
-	ccu_mp_find_best(parent_rate, rate,
-			 1 << cmp->m.width, (1 << cmp->p.width) - 1,
-			 &m, &p);
+	max_m = cmp->m.max ?: 1 << cmp->m.width;
+	max_p = cmp->p.max ?: 1 << ((1 << cmp->p.width) - 1);
 
-	return (parent_rate >> p) / m;
+	ccu_mp_find_best(parent_rate, rate, max_m, max_p, &m, &p);
+
+	return parent_rate / p / m;
 }
 
 static void ccu_mp_disable(struct clk_hw *hw)
@@ -108,13 +110,14 @@ static int ccu_mp_set_rate(struct clk_hw *hw, unsigned long rate,
 {
 	struct ccu_mp *cmp = hw_to_ccu_mp(hw);
 	unsigned long flags;
+	unsigned int max_m, max_p;
 	unsigned int m, p;
 	u32 reg;
 
-	ccu_mp_find_best(parent_rate, rate,
-			 1 << cmp->m.width, (1 << cmp->p.width) - 1,
-			 &m, &p);
+	max_m = cmp->m.max ?: 1 << cmp->m.width;
+	max_p = cmp->p.max ?: 1 << ((1 << cmp->p.width) - 1);
 
+	ccu_mp_find_best(parent_rate, rate, max_m, max_p, &m, &p);
 
 	spin_lock_irqsave(cmp->common.lock, flags);
 
@@ -122,7 +125,7 @@ static int ccu_mp_set_rate(struct clk_hw *hw, unsigned long rate,
 	reg &= ~GENMASK(cmp->m.width + cmp->m.shift - 1, cmp->m.shift);
 	reg &= ~GENMASK(cmp->p.width + cmp->p.shift - 1, cmp->p.shift);
 
-	writel(reg | (p << cmp->p.shift) | ((m - 1) << cmp->m.shift),
+	writel(reg | (ilog2(p) << cmp->p.shift) | ((m - 1) << cmp->m.shift),
 	       cmp->common.base + cmp->common.reg);
 
 	spin_unlock_irqrestore(cmp->common.lock, flags);
diff --git a/drivers/clk/sunxi-ng/ccu_mp.h b/drivers/clk/sunxi-ng/ccu_mp.h
index 3cf12bf95962..edf9215ea8cc 100644
--- a/drivers/clk/sunxi-ng/ccu_mp.h
+++ b/drivers/clk/sunxi-ng/ccu_mp.h
@@ -44,7 +44,7 @@ struct ccu_mp {
 		.enable	= _gate,					\
 		.m	= _SUNXI_CCU_DIV(_mshift, _mwidth),		\
 		.p	= _SUNXI_CCU_DIV(_pshift, _pwidth),		\
-		.mux	= SUNXI_CLK_MUX(_muxshift, _muxwidth),		\
+		.mux	= _SUNXI_CCU_MUX(_muxshift, _muxwidth),		\
 		.common	= {						\
 			.reg		= _reg,				\
 			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
diff --git a/drivers/clk/sunxi-ng/ccu_mult.c b/drivers/clk/sunxi-ng/ccu_mult.c
new file mode 100644
index 000000000000..010e9424691d
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu_mult.c
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2016 Maxime Ripard
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#include <linux/clk-provider.h>
+
+#include "ccu_gate.h"
+#include "ccu_mult.h"
+
+static void ccu_mult_find_best(unsigned long parent, unsigned long rate,
+			       unsigned int max_n, unsigned int *n)
+{
+	*n = rate / parent;
+}
+
+static unsigned long ccu_mult_round_rate(struct ccu_mux_internal *mux,
+					unsigned long parent_rate,
+					unsigned long rate,
+					void *data)
+{
+	struct ccu_mult *cm = data;
+	unsigned int n;
+
+	ccu_mult_find_best(parent_rate, rate, 1 << cm->mult.width, &n);
+
+	return parent_rate * n;
+}
+
+static void ccu_mult_disable(struct clk_hw *hw)
+{
+	struct ccu_mult *cm = hw_to_ccu_mult(hw);
+
+	return ccu_gate_helper_disable(&cm->common, cm->enable);
+}
+
+static int ccu_mult_enable(struct clk_hw *hw)
+{
+	struct ccu_mult *cm = hw_to_ccu_mult(hw);
+
+	return ccu_gate_helper_enable(&cm->common, cm->enable);
+}
+
+static int ccu_mult_is_enabled(struct clk_hw *hw)
+{
+	struct ccu_mult *cm = hw_to_ccu_mult(hw);
+
+	return ccu_gate_helper_is_enabled(&cm->common, cm->enable);
+}
+
+static unsigned long ccu_mult_recalc_rate(struct clk_hw *hw,
+					unsigned long parent_rate)
+{
+	struct ccu_mult *cm = hw_to_ccu_mult(hw);
+	unsigned long val;
+	u32 reg;
+
+	reg = readl(cm->common.base + cm->common.reg);
+	val = reg >> cm->mult.shift;
+	val &= (1 << cm->mult.width) - 1;
+
+	ccu_mux_helper_adjust_parent_for_prediv(&cm->common, &cm->mux, -1,
+						&parent_rate);
+
+	return parent_rate * (val + 1);
+}
+
+static int ccu_mult_determine_rate(struct clk_hw *hw,
+				struct clk_rate_request *req)
+{
+	struct ccu_mult *cm = hw_to_ccu_mult(hw);
+
+	return ccu_mux_helper_determine_rate(&cm->common, &cm->mux,
+					     req, ccu_mult_round_rate, cm);
+}
+
+static int ccu_mult_set_rate(struct clk_hw *hw, unsigned long rate,
+			   unsigned long parent_rate)
+{
+	struct ccu_mult *cm = hw_to_ccu_mult(hw);
+	unsigned long flags;
+	unsigned int n;
+	u32 reg;
+
+	ccu_mux_helper_adjust_parent_for_prediv(&cm->common, &cm->mux, -1,
+						&parent_rate);
+
+	ccu_mult_find_best(parent_rate, rate, 1 << cm->mult.width, &n);
+
+	spin_lock_irqsave(cm->common.lock, flags);
+
+	reg = readl(cm->common.base + cm->common.reg);
+	reg &= ~GENMASK(cm->mult.width + cm->mult.shift - 1, cm->mult.shift);
+
+	writel(reg | ((n - 1) << cm->mult.shift),
+	       cm->common.base + cm->common.reg);
+
+	spin_unlock_irqrestore(cm->common.lock, flags);
+
+	return 0;
+}
+
+static u8 ccu_mult_get_parent(struct clk_hw *hw)
+{
+	struct ccu_mult *cm = hw_to_ccu_mult(hw);
+
+	return ccu_mux_helper_get_parent(&cm->common, &cm->mux);
+}
+
+static int ccu_mult_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct ccu_mult *cm = hw_to_ccu_mult(hw);
+
+	return ccu_mux_helper_set_parent(&cm->common, &cm->mux, index);
+}
+
+const struct clk_ops ccu_mult_ops = {
+	.disable	= ccu_mult_disable,
+	.enable		= ccu_mult_enable,
+	.is_enabled	= ccu_mult_is_enabled,
+
+	.get_parent	= ccu_mult_get_parent,
+	.set_parent	= ccu_mult_set_parent,
+
+	.determine_rate	= ccu_mult_determine_rate,
+	.recalc_rate	= ccu_mult_recalc_rate,
+	.set_rate	= ccu_mult_set_rate,
+};
diff --git a/drivers/clk/sunxi-ng/ccu_mult.h b/drivers/clk/sunxi-ng/ccu_mult.h
index 609db6610880..5d2c8dc14073 100644
--- a/drivers/clk/sunxi-ng/ccu_mult.h
+++ b/drivers/clk/sunxi-ng/ccu_mult.h
@@ -1,6 +1,9 @@
 #ifndef _CCU_MULT_H_
 #define _CCU_MULT_H_
 
+#include "ccu_common.h"
+#include "ccu_mux.h"
+
 struct _ccu_mult {
 	u8	shift;
 	u8	width;
@@ -12,4 +15,36 @@ struct _ccu_mult {
 		.width	= _width,		\
 	}
 
+struct ccu_mult {
+	u32			enable;
+
+	struct _ccu_mult	mult;
+	struct ccu_mux_internal	mux;
+	struct ccu_common	common;
+};
+
+#define SUNXI_CCU_N_WITH_GATE_LOCK(_struct, _name, _parent, _reg,	\
+				   _mshift, _mwidth, _gate, _lock,	\
+				   _flags)				\
+	struct ccu_mult _struct = {					\
+		.enable	= _gate,					\
+		.mult	= _SUNXI_CCU_MULT(_mshift, _mwidth),		\
+		.common	= {						\
+			.reg		= _reg,				\
+			.hw.init	= CLK_HW_INIT(_name,		\
+						      _parent,		\
+						      &ccu_mult_ops,	\
+						      _flags),		\
+		},							\
+	}
+
+static inline struct ccu_mult *hw_to_ccu_mult(struct clk_hw *hw)
+{
+	struct ccu_common *common = hw_to_ccu_common(hw);
+
+	return container_of(common, struct ccu_mult, common);
+}
+
+extern const struct clk_ops ccu_mult_ops;
+
 #endif /* _CCU_MULT_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu_mux.c b/drivers/clk/sunxi-ng/ccu_mux.c
index 58fc36e7dcce..a43ad52a957d 100644
--- a/drivers/clk/sunxi-ng/ccu_mux.c
+++ b/drivers/clk/sunxi-ng/ccu_mux.c
@@ -8,7 +8,9 @@
  * the License, or (at your option) any later version.
  */
 
+#include <linux/clk.h>
 #include <linux/clk-provider.h>
+#include <linux/delay.h>
 
 #include "ccu_gate.h"
 #include "ccu_mux.h"
@@ -18,8 +20,9 @@ void ccu_mux_helper_adjust_parent_for_prediv(struct ccu_common *common,
 					     int parent_index,
 					     unsigned long *parent_rate)
 {
-	u8 prediv = 1;
+	u16 prediv = 1;
 	u32 reg;
+	int i;
 
 	if (!((common->features & CCU_FEATURE_FIXED_PREDIV) ||
 	      (common->features & CCU_FEATURE_VARIABLE_PREDIV)))
@@ -32,8 +35,9 @@ void ccu_mux_helper_adjust_parent_for_prediv(struct ccu_common *common,
 	}
 
 	if (common->features & CCU_FEATURE_FIXED_PREDIV)
-		if (parent_index == cm->fixed_prediv.index)
-			prediv = cm->fixed_prediv.div;
+		for (i = 0; i < cm->n_predivs; i++)
+			if (parent_index == cm->fixed_predivs[i].index)
+				prediv = cm->fixed_predivs[i].div;
 
 	if (common->features & CCU_FEATURE_VARIABLE_PREDIV)
 		if (parent_index == cm->variable_prediv.index) {
@@ -107,6 +111,15 @@ u8 ccu_mux_helper_get_parent(struct ccu_common *common,
 	parent = reg >> cm->shift;
 	parent &= (1 << cm->width) - 1;
 
+	if (cm->table) {
+		int num_parents = clk_hw_get_num_parents(&common->hw);
+		int i;
+
+		for (i = 0; i < num_parents; i++)
+			if (cm->table[i] == parent)
+				return i;
+	}
+
 	return parent;
 }
 
@@ -117,6 +130,9 @@ int ccu_mux_helper_set_parent(struct ccu_common *common,
 	unsigned long flags;
 	u32 reg;
 
+	if (cm->table)
+		index = cm->table[index];
+
 	spin_lock_irqsave(common->lock, flags);
 
 	reg = readl(common->base + common->reg);
@@ -185,3 +201,37 @@ const struct clk_ops ccu_mux_ops = {
 	.determine_rate	= __clk_mux_determine_rate,
 	.recalc_rate	= ccu_mux_recalc_rate,
 };
+
+/*
+ * This clock notifier is called when the frequency of the of the parent
+ * PLL clock is to be changed. The idea is to switch the parent to a
+ * stable clock, such as the main oscillator, while the PLL frequency
+ * stabilizes.
+ */
+static int ccu_mux_notifier_cb(struct notifier_block *nb,
+			       unsigned long event, void *data)
+{
+	struct ccu_mux_nb *mux = to_ccu_mux_nb(nb);
+	int ret = 0;
+
+	if (event == PRE_RATE_CHANGE) {
+		mux->original_index = ccu_mux_helper_get_parent(mux->common,
+								mux->cm);
+		ret = ccu_mux_helper_set_parent(mux->common, mux->cm,
+						mux->bypass_index);
+	} else if (event == POST_RATE_CHANGE) {
+		ret = ccu_mux_helper_set_parent(mux->common, mux->cm,
+						mux->original_index);
+	}
+
+	udelay(mux->delay_us);
+
+	return notifier_from_errno(ret);
+}
+
+int ccu_mux_notifier_register(struct clk *clk, struct ccu_mux_nb *mux_nb)
+{
+	mux_nb->clk_nb.notifier_call = ccu_mux_notifier_cb;
+
+	return clk_notifier_register(clk, &mux_nb->clk_nb);
+}
diff --git a/drivers/clk/sunxi-ng/ccu_mux.h b/drivers/clk/sunxi-ng/ccu_mux.h
index 945082631e7d..47aba3a48245 100644
--- a/drivers/clk/sunxi-ng/ccu_mux.h
+++ b/drivers/clk/sunxi-ng/ccu_mux.h
@@ -5,14 +5,18 @@
 
 #include "ccu_common.h"
 
+struct ccu_mux_fixed_prediv {
+	u8	index;
+	u16	div;
+};
+
 struct ccu_mux_internal {
-	u8	shift;
-	u8	width;
+	u8		shift;
+	u8		width;
+	const u8	*table;
 
-	struct {
-		u8	index;
-		u8	div;
-	} fixed_prediv;
+	const struct ccu_mux_fixed_prediv	*fixed_predivs;
+	u8		n_predivs;
 
 	struct {
 		u8	index;
@@ -21,12 +25,16 @@ struct ccu_mux_internal {
 	} variable_prediv;
 };
 
-#define SUNXI_CLK_MUX(_shift, _width)	\
-	{					\
-		.shift	= _shift,		\
-		.width	= _width,		\
+#define _SUNXI_CCU_MUX_TABLE(_shift, _width, _table)	\
+	{						\
+		.shift	= _shift,			\
+		.width	= _width,			\
+		.table	= _table,			\
 	}
 
+#define _SUNXI_CCU_MUX(_shift, _width) \
+	_SUNXI_CCU_MUX_TABLE(_shift, _width, NULL)
+
 struct ccu_mux {
 	u16			reg;
 	u32			enable;
@@ -35,9 +43,12 @@ struct ccu_mux {
 	struct ccu_common	common;
 };
 
-#define SUNXI_CCU_MUX(_struct, _name, _parents, _reg, _shift, _width, _flags) \
+#define SUNXI_CCU_MUX_TABLE_WITH_GATE(_struct, _name, _parents, _table,	\
+				     _reg, _shift, _width, _gate,	\
+				     _flags)				\
 	struct ccu_mux _struct = {					\
-		.mux	= SUNXI_CLK_MUX(_shift, _width),		\
+		.enable	= _gate,					\
+		.mux	= _SUNXI_CCU_MUX_TABLE(_shift, _width, _table),	\
 		.common	= {						\
 			.reg		= _reg,				\
 			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
@@ -49,17 +60,14 @@ struct ccu_mux {
 
 #define SUNXI_CCU_MUX_WITH_GATE(_struct, _name, _parents, _reg,		\
 				_shift, _width, _gate, _flags)		\
-	struct ccu_mux _struct = {					\
-		.enable	= _gate,					\
-		.mux	= SUNXI_CLK_MUX(_shift, _width),		\
-		.common	= {						\
-			.reg		= _reg,				\
-			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
-							      _parents, \
-							      &ccu_mux_ops, \
-							      _flags),	\
-		}							\
-	}
+	SUNXI_CCU_MUX_TABLE_WITH_GATE(_struct, _name, _parents, NULL,	\
+				      _reg, _shift, _width, _gate,	\
+				      _flags)
+
+#define SUNXI_CCU_MUX(_struct, _name, _parents, _reg, _shift, _width,	\
+		      _flags)						\
+	SUNXI_CCU_MUX_TABLE_WITH_GATE(_struct, _name, _parents, NULL,	\
+				      _reg, _shift, _width, 0, _flags)
 
 static inline struct ccu_mux *hw_to_ccu_mux(struct clk_hw *hw)
 {
@@ -88,4 +96,18 @@ int ccu_mux_helper_set_parent(struct ccu_common *common,
 			      struct ccu_mux_internal *cm,
 			      u8 index);
 
+struct ccu_mux_nb {
+	struct notifier_block	clk_nb;
+	struct ccu_common	*common;
+	struct ccu_mux_internal	*cm;
+
+	u32	delay_us;	/* How many us to wait after reparenting */
+	u8	bypass_index;	/* Which parent to temporarily use */
+	u8	original_index;	/* This is set by the notifier callback */
+};
+
+#define to_ccu_mux_nb(_nb) container_of(_nb, struct ccu_mux_nb, clk_nb)
+
+int ccu_mux_notifier_register(struct clk *clk, struct ccu_mux_nb *mux_nb);
+
 #endif /* _CCU_MUX_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu_nk.c b/drivers/clk/sunxi-ng/ccu_nk.c
index 4470ffc8cf0d..d6fafb397489 100644
--- a/drivers/clk/sunxi-ng/ccu_nk.c
+++ b/drivers/clk/sunxi-ng/ccu_nk.c
@@ -14,9 +14,9 @@
 #include "ccu_gate.h"
 #include "ccu_nk.h"
 
-void ccu_nk_find_best(unsigned long parent, unsigned long rate,
-		      unsigned int max_n, unsigned int max_k,
-		      unsigned int *n, unsigned int *k)
+static void ccu_nk_find_best(unsigned long parent, unsigned long rate,
+			     unsigned int max_n, unsigned int max_k,
+			     unsigned int *n, unsigned int *k)
 {
 	unsigned long best_rate = 0;
 	unsigned int best_k = 0, best_n = 0;
diff --git a/drivers/clk/sunxi-ng/ccu_nkm.c b/drivers/clk/sunxi-ng/ccu_nkm.c
index 2071822b1e9c..059fdc3b4f96 100644
--- a/drivers/clk/sunxi-ng/ccu_nkm.c
+++ b/drivers/clk/sunxi-ng/ccu_nkm.c
@@ -93,19 +93,30 @@ static unsigned long ccu_nkm_recalc_rate(struct clk_hw *hw,
 	return parent_rate * (n + 1) * (k + 1) / (m + 1);
 }
 
-static long ccu_nkm_round_rate(struct clk_hw *hw, unsigned long rate,
-			      unsigned long *parent_rate)
+static unsigned long ccu_nkm_round_rate(struct ccu_mux_internal *mux,
+					unsigned long parent_rate,
+					unsigned long rate,
+					void *data)
 {
-	struct ccu_nkm *nkm = hw_to_ccu_nkm(hw);
+	struct ccu_nkm *nkm = data;
 	struct _ccu_nkm _nkm;
 
 	_nkm.max_n = 1 << nkm->n.width;
 	_nkm.max_k = 1 << nkm->k.width;
-	_nkm.max_m = 1 << nkm->m.width;
+	_nkm.max_m = nkm->m.max ?: 1 << nkm->m.width;
 
-	ccu_nkm_find_best(*parent_rate, rate, &_nkm);
+	ccu_nkm_find_best(parent_rate, rate, &_nkm);
 
-	return *parent_rate * _nkm.n * _nkm.k / _nkm.m;
+	return parent_rate * _nkm.n * _nkm.k / _nkm.m;
+}
+
+static int ccu_nkm_determine_rate(struct clk_hw *hw,
+				  struct clk_rate_request *req)
+{
+	struct ccu_nkm *nkm = hw_to_ccu_nkm(hw);
+
+	return ccu_mux_helper_determine_rate(&nkm->common, &nkm->mux,
+					     req, ccu_nkm_round_rate, nkm);
 }
 
 static int ccu_nkm_set_rate(struct clk_hw *hw, unsigned long rate,
@@ -118,7 +129,7 @@ static int ccu_nkm_set_rate(struct clk_hw *hw, unsigned long rate,
 
 	_nkm.max_n = 1 << nkm->n.width;
 	_nkm.max_k = 1 << nkm->k.width;
-	_nkm.max_m = 1 << nkm->m.width;
+	_nkm.max_m = nkm->m.max ?: 1 << nkm->m.width;
 
 	ccu_nkm_find_best(parent_rate, rate, &_nkm);
 
@@ -142,12 +153,29 @@ static int ccu_nkm_set_rate(struct clk_hw *hw, unsigned long rate,
 	return 0;
 }
 
+static u8 ccu_nkm_get_parent(struct clk_hw *hw)
+{
+	struct ccu_nkm *nkm = hw_to_ccu_nkm(hw);
+
+	return ccu_mux_helper_get_parent(&nkm->common, &nkm->mux);
+}
+
+static int ccu_nkm_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct ccu_nkm *nkm = hw_to_ccu_nkm(hw);
+
+	return ccu_mux_helper_set_parent(&nkm->common, &nkm->mux, index);
+}
+
 const struct clk_ops ccu_nkm_ops = {
 	.disable	= ccu_nkm_disable,
 	.enable		= ccu_nkm_enable,
 	.is_enabled	= ccu_nkm_is_enabled,
 
+	.get_parent	= ccu_nkm_get_parent,
+	.set_parent	= ccu_nkm_set_parent,
+
+	.determine_rate	= ccu_nkm_determine_rate,
 	.recalc_rate	= ccu_nkm_recalc_rate,
-	.round_rate	= ccu_nkm_round_rate,
 	.set_rate	= ccu_nkm_set_rate,
 };
diff --git a/drivers/clk/sunxi-ng/ccu_nkm.h b/drivers/clk/sunxi-ng/ccu_nkm.h
index 1936ac1c6b37..35493fddd8ab 100644
--- a/drivers/clk/sunxi-ng/ccu_nkm.h
+++ b/drivers/clk/sunxi-ng/ccu_nkm.h
@@ -32,10 +32,33 @@ struct ccu_nkm {
 	struct _ccu_mult	n;
 	struct _ccu_mult	k;
 	struct _ccu_div		m;
+	struct ccu_mux_internal	mux;
 
 	struct ccu_common	common;
 };
 
+#define SUNXI_CCU_NKM_WITH_MUX_GATE_LOCK(_struct, _name, _parents, _reg, \
+					 _nshift, _nwidth,		\
+					 _kshift, _kwidth,		\
+					 _mshift, _mwidth,		\
+					 _muxshift, _muxwidth,		\
+					 _gate, _lock, _flags)		\
+	struct ccu_nkm _struct = {					\
+		.enable		= _gate,				\
+		.lock		= _lock,				\
+		.k		= _SUNXI_CCU_MULT(_kshift, _kwidth),	\
+		.n		= _SUNXI_CCU_MULT(_nshift, _nwidth),	\
+		.m		= _SUNXI_CCU_DIV(_mshift, _mwidth),	\
+		.mux		= _SUNXI_CCU_MUX(_muxshift, _muxwidth),	\
+		.common		= {					\
+			.reg		= _reg,				\
+			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
+						      _parents,		\
+						      &ccu_nkm_ops,	\
+						      _flags),		\
+		},							\
+	}
+
 #define SUNXI_CCU_NKM_WITH_GATE_LOCK(_struct, _name, _parent, _reg,	\
 				     _nshift, _nwidth,			\
 				     _kshift, _kwidth,			\
diff --git a/drivers/clk/sunxi-ng/ccu_nkmp.c b/drivers/clk/sunxi-ng/ccu_nkmp.c
index 9f2b98e19dc9..9769dee99511 100644
--- a/drivers/clk/sunxi-ng/ccu_nkmp.c
+++ b/drivers/clk/sunxi-ng/ccu_nkmp.c
@@ -29,14 +29,14 @@ static void ccu_nkmp_find_best(unsigned long parent, unsigned long rate,
 	unsigned long _n, _k, _m, _p;
 
 	for (_k = 1; _k <= nkmp->max_k; _k++) {
-		for (_p = 0; _p <= nkmp->max_p; _p++) {
+		for (_p = 1; _p <= nkmp->max_p; _p <<= 1) {
 			unsigned long tmp_rate;
 
-			rational_best_approximation(rate / _k, parent >> _p,
+			rational_best_approximation(rate / _k, parent / _p,
 						    nkmp->max_n, nkmp->max_m,
 						    &_n, &_m);
 
-			tmp_rate = (parent * _n * _k >> _p) / _m;
+			tmp_rate = parent * _n * _k / (_m * _p);
 
 			if (tmp_rate > rate)
 				continue;
@@ -110,13 +110,12 @@ static long ccu_nkmp_round_rate(struct clk_hw *hw, unsigned long rate,
 
 	_nkmp.max_n = 1 << nkmp->n.width;
 	_nkmp.max_k = 1 << nkmp->k.width;
-	_nkmp.max_m = 1 << nkmp->m.width;
-	_nkmp.max_p = (1 << nkmp->p.width) - 1;
+	_nkmp.max_m = nkmp->m.max ?: 1 << nkmp->m.width;
+	_nkmp.max_p = nkmp->p.max ?: 1 << ((1 << nkmp->p.width) - 1);
 
-	ccu_nkmp_find_best(*parent_rate, rate,
-			   &_nkmp);
+	ccu_nkmp_find_best(*parent_rate, rate, &_nkmp);
 
-	return (*parent_rate * _nkmp.n * _nkmp.k >> _nkmp.p) / _nkmp.m;
+	return *parent_rate * _nkmp.n * _nkmp.k / (_nkmp.m * _nkmp.p);
 }
 
 static int ccu_nkmp_set_rate(struct clk_hw *hw, unsigned long rate,
@@ -129,8 +128,8 @@ static int ccu_nkmp_set_rate(struct clk_hw *hw, unsigned long rate,
 
 	_nkmp.max_n = 1 << nkmp->n.width;
 	_nkmp.max_k = 1 << nkmp->k.width;
-	_nkmp.max_m = 1 << nkmp->m.width;
-	_nkmp.max_p = (1 << nkmp->p.width) - 1;
+	_nkmp.max_m = nkmp->m.max ?: 1 << nkmp->m.width;
+	_nkmp.max_p = nkmp->p.max ?: 1 << ((1 << nkmp->p.width) - 1);
 
 	ccu_nkmp_find_best(parent_rate, rate, &_nkmp);
 
@@ -145,7 +144,7 @@ static int ccu_nkmp_set_rate(struct clk_hw *hw, unsigned long rate,
 	reg |= (_nkmp.n - 1) << nkmp->n.shift;
 	reg |= (_nkmp.k - 1) << nkmp->k.shift;
 	reg |= (_nkmp.m - 1) << nkmp->m.shift;
-	reg |= _nkmp.p << nkmp->p.shift;
+	reg |= ilog2(_nkmp.p) << nkmp->p.shift;
 
 	writel(reg, nkmp->common.base + nkmp->common.reg);
 
diff --git a/drivers/clk/sunxi-ng/ccu_nm.c b/drivers/clk/sunxi-ng/ccu_nm.c
index e35ddd8eec8b..b61bdd8c7a7f 100644
--- a/drivers/clk/sunxi-ng/ccu_nm.c
+++ b/drivers/clk/sunxi-ng/ccu_nm.c
@@ -61,11 +61,13 @@ static long ccu_nm_round_rate(struct clk_hw *hw, unsigned long rate,
 			      unsigned long *parent_rate)
 {
 	struct ccu_nm *nm = hw_to_ccu_nm(hw);
+	unsigned long max_n, max_m;
 	unsigned long n, m;
 
-	rational_best_approximation(rate, *parent_rate,
-				    1 << nm->n.width, 1 << nm->m.width,
-				    &n, &m);
+	max_n = 1 << nm->n.width;
+	max_m = nm->m.max ?: 1 << nm->m.width;
+
+	rational_best_approximation(rate, *parent_rate, max_n, max_m, &n, &m);
 
 	return *parent_rate * n / m;
 }
@@ -75,6 +77,7 @@ static int ccu_nm_set_rate(struct clk_hw *hw, unsigned long rate,
 {
 	struct ccu_nm *nm = hw_to_ccu_nm(hw);
 	unsigned long flags;
+	unsigned long max_n, max_m;
 	unsigned long n, m;
 	u32 reg;
 
@@ -83,9 +86,10 @@ static int ccu_nm_set_rate(struct clk_hw *hw, unsigned long rate,
 	else
 		ccu_frac_helper_disable(&nm->common, &nm->frac);
 
-	rational_best_approximation(rate, parent_rate,
-				    1 << nm->n.width, 1 << nm->m.width,
-				    &n, &m);
+	max_n = 1 << nm->n.width;
+	max_m = nm->m.max ?: 1 << nm->m.width;
+
+	rational_best_approximation(rate, parent_rate, max_n, max_m, &n, &m);
 
 	spin_lock_irqsave(nm->common.lock, flags);
 
diff --git a/drivers/clk/sunxi/clk-a10-pll2.c b/drivers/clk/sunxi/clk-a10-pll2.c
index 0ee1f363e4be..d8eab90ae661 100644
--- a/drivers/clk/sunxi/clk-a10-pll2.c
+++ b/drivers/clk/sunxi/clk-a10-pll2.c
@@ -73,7 +73,7 @@ static void __init sun4i_pll2_setup(struct device_node *node,
 					  SUN4I_PLL2_PRE_DIV_WIDTH,
 					  CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO,
 					  &sun4i_a10_pll2_lock);
-	if (!prediv_clk) {
+	if (IS_ERR(prediv_clk)) {
 		pr_err("Couldn't register the prediv clock\n");
 		goto err_free_array;
 	}
@@ -106,7 +106,7 @@ static void __init sun4i_pll2_setup(struct device_node *node,
 					  &mult->hw, &clk_multiplier_ops,
 					  &gate->hw, &clk_gate_ops,
 					  CLK_SET_RATE_PARENT);
-	if (!base_clk) {
+	if (IS_ERR(base_clk)) {
 		pr_err("Couldn't register the base multiplier clock\n");
 		goto err_free_multiplier;
 	}
diff --git a/drivers/clk/sunxi/clk-mod0.c b/drivers/clk/sunxi/clk-mod0.c
index b38d71cec74c..e54266cc1c51 100644
--- a/drivers/clk/sunxi/clk-mod0.c
+++ b/drivers/clk/sunxi/clk-mod0.c
@@ -91,7 +91,8 @@ static void __init sun4i_a10_mod0_setup(struct device_node *node)
 	sunxi_factors_register(node, &sun4i_a10_mod0_data,
 			       &sun4i_a10_mod0_lock, reg);
 }
-CLK_OF_DECLARE(sun4i_a10_mod0, "allwinner,sun4i-a10-mod0-clk", sun4i_a10_mod0_setup);
+CLK_OF_DECLARE_DRIVER(sun4i_a10_mod0, "allwinner,sun4i-a10-mod0-clk",
+		      sun4i_a10_mod0_setup);
 
 static int sun4i_a10_mod0_clk_probe(struct platform_device *pdev)
 {
diff --git a/drivers/clk/sunxi/clk-sun8i-apb0.c b/drivers/clk/sunxi/clk-sun8i-apb0.c
index a5666e1d0ce7..ea1eed24778c 100644
--- a/drivers/clk/sunxi/clk-sun8i-apb0.c
+++ b/drivers/clk/sunxi/clk-sun8i-apb0.c
@@ -82,8 +82,8 @@ err_unmap:
 	of_address_to_resource(node, 0, &res);
 	release_mem_region(res.start, resource_size(&res));
 }
-CLK_OF_DECLARE(sun8i_a23_apb0, "allwinner,sun8i-a23-apb0-clk",
-	       sun8i_a23_apb0_setup);
+CLK_OF_DECLARE_DRIVER(sun8i_a23_apb0, "allwinner,sun8i-a23-apb0-clk",
+		      sun8i_a23_apb0_setup);
 
 static int sun8i_a23_apb0_clk_probe(struct platform_device *pdev)
 {
diff --git a/drivers/clk/sunxi/clk-sun8i-mbus.c b/drivers/clk/sunxi/clk-sun8i-mbus.c
index 411d3033a96e..b200ebf159ee 100644
--- a/drivers/clk/sunxi/clk-sun8i-mbus.c
+++ b/drivers/clk/sunxi/clk-sun8i-mbus.c
@@ -48,7 +48,7 @@ static void __init sun8i_a23_mbus_setup(struct device_node *node)
 		return;
 
 	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
-	if (!reg) {
+	if (IS_ERR(reg)) {
 		pr_err("Could not get registers for sun8i-mbus-clk\n");
 		goto err_free_parents;
 	}
diff --git a/drivers/clk/tegra/clk-tegra114.c b/drivers/clk/tegra/clk-tegra114.c
index 64da7b79a6e4..933b5dd698b8 100644
--- a/drivers/clk/tegra/clk-tegra114.c
+++ b/drivers/clk/tegra/clk-tegra114.c
@@ -428,7 +428,7 @@ static struct tegra_clk_pll_params pll_d_params = {
 	.div_nmp = &pllp_nmp,
 	.freq_table = pll_d_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
-		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static struct tegra_clk_pll_params pll_d2_params = {
@@ -446,7 +446,7 @@ static struct tegra_clk_pll_params pll_d2_params = {
 	.div_nmp = &pllp_nmp,
 	.freq_table = pll_d_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON |
-		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+		 TEGRA_PLL_HAS_LOCK_ENABLE,
 };
 
 static const struct pdiv_map pllu_p[] = {
diff --git a/drivers/clk/uniphier/Kconfig b/drivers/clk/uniphier/Kconfig
new file mode 100644
index 000000000000..5512377bd62b
--- /dev/null
+++ b/drivers/clk/uniphier/Kconfig
@@ -0,0 +1,9 @@
+config CLK_UNIPHIER
+	bool "Clock driver for UniPhier SoCs"
+	depends on ARCH_UNIPHIER || COMPILE_TEST
+	depends on OF && MFD_SYSCON
+	default ARCH_UNIPHIER
+	help
+	  Support for clock controllers on UniPhier SoCs.
+	  Say Y if you want to control clocks provided by System Control
+	  block, Media I/O block, Peripheral Block.
diff --git a/drivers/clk/uniphier/Makefile b/drivers/clk/uniphier/Makefile
new file mode 100644
index 000000000000..f27b360329ca
--- /dev/null
+++ b/drivers/clk/uniphier/Makefile
@@ -0,0 +1,8 @@
+obj-y	+= clk-uniphier-core.o
+obj-y	+= clk-uniphier-fixed-factor.o
+obj-y	+= clk-uniphier-fixed-rate.o
+obj-y	+= clk-uniphier-gate.o
+obj-y	+= clk-uniphier-mux.o
+obj-y	+= clk-uniphier-sys.o
+obj-y	+= clk-uniphier-mio.o
+obj-y	+= clk-uniphier-peri.o
diff --git a/drivers/clk/uniphier/clk-uniphier-core.c b/drivers/clk/uniphier/clk-uniphier-core.c
new file mode 100644
index 000000000000..5ffb898d0839
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier-core.c
@@ -0,0 +1,214 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/init.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#include "clk-uniphier.h"
+
+static struct clk_hw *uniphier_clk_register(struct device *dev,
+					    struct regmap *regmap,
+					const struct uniphier_clk_data *data)
+{
+	switch (data->type) {
+	case UNIPHIER_CLK_TYPE_FIXED_FACTOR:
+		return uniphier_clk_register_fixed_factor(dev, data->name,
+							  &data->data.factor);
+	case UNIPHIER_CLK_TYPE_FIXED_RATE:
+		return uniphier_clk_register_fixed_rate(dev, data->name,
+							&data->data.rate);
+	case UNIPHIER_CLK_TYPE_GATE:
+		return uniphier_clk_register_gate(dev, regmap, data->name,
+						  &data->data.gate);
+	case UNIPHIER_CLK_TYPE_MUX:
+		return uniphier_clk_register_mux(dev, regmap, data->name,
+						 &data->data.mux);
+	default:
+		dev_err(dev, "unsupported clock type\n");
+		return ERR_PTR(-EINVAL);
+	}
+}
+
+static int uniphier_clk_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct clk_hw_onecell_data *hw_data;
+	const struct uniphier_clk_data *p, *data;
+	struct regmap *regmap;
+	struct device_node *parent;
+	int clk_num = 0;
+
+	data = of_device_get_match_data(dev);
+	if (WARN_ON(!data))
+		return -EINVAL;
+
+	parent = of_get_parent(dev->of_node); /* parent should be syscon node */
+	regmap = syscon_node_to_regmap(parent);
+	of_node_put(parent);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "failed to get regmap (error %ld)\n",
+			PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
+	for (p = data; p->name; p++)
+		clk_num = max(clk_num, p->idx + 1);
+
+	hw_data = devm_kzalloc(dev,
+			sizeof(*hw_data) + clk_num * sizeof(struct clk_hw *),
+			GFP_KERNEL);
+	if (!hw_data)
+		return -ENOMEM;
+
+	hw_data->num = clk_num;
+
+	/* avoid returning NULL for unused idx */
+	for (; clk_num >= 0; clk_num--)
+		hw_data->hws[clk_num] = ERR_PTR(-EINVAL);
+
+	for (p = data; p->name; p++) {
+		struct clk_hw *hw;
+
+		dev_dbg(dev, "register %s (index=%d)\n", p->name, p->idx);
+		hw = uniphier_clk_register(dev, regmap, p);
+		if (IS_ERR(hw)) {
+			dev_err(dev, "failed to register %s (error %ld)\n",
+				p->name, PTR_ERR(hw));
+			return PTR_ERR(hw);
+		}
+
+		if (p->idx >= 0)
+			hw_data->hws[p->idx] = hw;
+	}
+
+	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get,
+				      hw_data);
+}
+
+static int uniphier_clk_remove(struct platform_device *pdev)
+{
+	of_clk_del_provider(pdev->dev.of_node);
+
+	return 0;
+}
+
+static const struct of_device_id uniphier_clk_match[] = {
+	/* System clock */
+	{
+		.compatible = "socionext,uniphier-ld4-clock",
+		.data = uniphier_ld4_sys_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro4-clock",
+		.data = uniphier_pro4_sys_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-sld8-clock",
+		.data = uniphier_sld8_sys_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro5-clock",
+		.data = uniphier_pro5_sys_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pxs2-clock",
+		.data = uniphier_pxs2_sys_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld11-clock",
+		.data = uniphier_ld11_sys_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld20-clock",
+		.data = uniphier_ld20_sys_clk_data,
+	},
+	/* Media I/O clock */
+	{
+		.compatible = "socionext,uniphier-sld3-mio-clock",
+		.data = uniphier_sld3_mio_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld4-mio-clock",
+		.data = uniphier_sld3_mio_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro4-mio-clock",
+		.data = uniphier_sld3_mio_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-sld8-mio-clock",
+		.data = uniphier_sld3_mio_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro5-mio-clock",
+		.data = uniphier_pro5_mio_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pxs2-mio-clock",
+		.data = uniphier_pro5_mio_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld11-mio-clock",
+		.data = uniphier_sld3_mio_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld20-mio-clock",
+		.data = uniphier_pro5_mio_clk_data,
+	},
+	/* Peripheral clock */
+	{
+		.compatible = "socionext,uniphier-ld4-peri-clock",
+		.data = uniphier_ld4_peri_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro4-peri-clock",
+		.data = uniphier_pro4_peri_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-sld8-peri-clock",
+		.data = uniphier_ld4_peri_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro5-peri-clock",
+		.data = uniphier_pro4_peri_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pxs2-peri-clock",
+		.data = uniphier_pro4_peri_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld11-peri-clock",
+		.data = uniphier_pro4_peri_clk_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld20-peri-clock",
+		.data = uniphier_pro4_peri_clk_data,
+	},
+	{ /* sentinel */ }
+};
+
+static struct platform_driver uniphier_clk_driver = {
+	.probe = uniphier_clk_probe,
+	.remove = uniphier_clk_remove,
+	.driver = {
+		.name = "uniphier-clk",
+		.of_match_table = uniphier_clk_match,
+	},
+};
+builtin_platform_driver(uniphier_clk_driver);
diff --git a/drivers/clk/uniphier/clk-uniphier-fixed-factor.c b/drivers/clk/uniphier/clk-uniphier-fixed-factor.c
new file mode 100644
index 000000000000..da2d9f47ef9f
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier-fixed-factor.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+
+#include "clk-uniphier.h"
+
+struct clk_hw *uniphier_clk_register_fixed_factor(struct device *dev,
+						  const char *name,
+			const struct uniphier_clk_fixed_factor_data *data)
+{
+	struct clk_fixed_factor *fix;
+	struct clk_init_data init;
+	int ret;
+
+	fix = devm_kzalloc(dev, sizeof(*fix), GFP_KERNEL);
+	if (!fix)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &clk_fixed_factor_ops;
+	init.flags = data->parent_name ? CLK_SET_RATE_PARENT : 0;
+	init.parent_names = data->parent_name ? &data->parent_name : NULL;
+	init.num_parents = data->parent_name ? 1 : 0;
+
+	fix->mult = data->mult;
+	fix->div = data->div;
+	fix->hw.init = &init;
+
+	ret = devm_clk_hw_register(dev, &fix->hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &fix->hw;
+}
diff --git a/drivers/clk/uniphier/clk-uniphier-fixed-rate.c b/drivers/clk/uniphier/clk-uniphier-fixed-rate.c
new file mode 100644
index 000000000000..0ad0d46173c0
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier-fixed-rate.c
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+
+#include "clk-uniphier.h"
+
+struct clk_hw *uniphier_clk_register_fixed_rate(struct device *dev,
+						const char *name,
+				const struct uniphier_clk_fixed_rate_data *data)
+{
+	struct clk_fixed_rate *fixed;
+	struct clk_init_data init;
+	int ret;
+
+	/* allocate fixed-rate clock */
+	fixed = devm_kzalloc(dev, sizeof(*fixed), GFP_KERNEL);
+	if (!fixed)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &clk_fixed_rate_ops;
+	init.parent_names = NULL;
+	init.num_parents = 0;
+
+	fixed->fixed_rate = data->fixed_rate;
+	fixed->hw.init = &init;
+
+	ret = devm_clk_hw_register(dev, &fixed->hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &fixed->hw;
+}
diff --git a/drivers/clk/uniphier/clk-uniphier-gate.c b/drivers/clk/uniphier/clk-uniphier-gate.c
new file mode 100644
index 000000000000..49142d44446d
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier-gate.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/regmap.h>
+
+#include "clk-uniphier.h"
+
+struct uniphier_clk_gate {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	unsigned int reg;
+	unsigned int bit;
+};
+
+#define to_uniphier_clk_gate(_hw) \
+				container_of(_hw, struct uniphier_clk_gate, hw)
+
+static int uniphier_clk_gate_endisable(struct clk_hw *hw, int enable)
+{
+	struct uniphier_clk_gate *gate = to_uniphier_clk_gate(hw);
+
+	return regmap_write_bits(gate->regmap, gate->reg, BIT(gate->bit),
+				 enable ? BIT(gate->bit) : 0);
+}
+
+static int uniphier_clk_gate_enable(struct clk_hw *hw)
+{
+	return uniphier_clk_gate_endisable(hw, 1);
+}
+
+static void uniphier_clk_gate_disable(struct clk_hw *hw)
+{
+	if (uniphier_clk_gate_endisable(hw, 0) < 0)
+		pr_warn("failed to disable clk\n");
+}
+
+static int uniphier_clk_gate_is_enabled(struct clk_hw *hw)
+{
+	struct uniphier_clk_gate *gate = to_uniphier_clk_gate(hw);
+	unsigned int val;
+
+	if (regmap_read(gate->regmap, gate->reg, &val) < 0)
+		pr_warn("is_enabled() may return wrong result\n");
+
+	return !!(val & BIT(gate->bit));
+}
+
+static const struct clk_ops uniphier_clk_gate_ops = {
+	.enable = uniphier_clk_gate_enable,
+	.disable = uniphier_clk_gate_disable,
+	.is_enabled = uniphier_clk_gate_is_enabled,
+};
+
+struct clk_hw *uniphier_clk_register_gate(struct device *dev,
+					  struct regmap *regmap,
+					  const char *name,
+				const struct uniphier_clk_gate_data *data)
+{
+	struct uniphier_clk_gate *gate;
+	struct clk_init_data init;
+	int ret;
+
+	gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL);
+	if (!gate)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &uniphier_clk_gate_ops;
+	init.flags = data->parent_name ? CLK_SET_RATE_PARENT : 0;
+	init.parent_names = data->parent_name ? &data->parent_name : NULL;
+	init.num_parents = data->parent_name ? 1 : 0;
+
+	gate->regmap = regmap;
+	gate->reg = data->reg;
+	gate->bit = data->bit;
+	gate->hw.init = &init;
+
+	ret = devm_clk_hw_register(dev, &gate->hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &gate->hw;
+}
diff --git a/drivers/clk/uniphier/clk-uniphier-mio.c b/drivers/clk/uniphier/clk-uniphier-mio.c
new file mode 100644
index 000000000000..6aa7ec768d0b
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier-mio.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "clk-uniphier.h"
+
+#define UNIPHIER_MIO_CLK_SD_FIXED					\
+	UNIPHIER_CLK_FACTOR("sd-44m", -1, "sd-133m", 1, 3),		\
+	UNIPHIER_CLK_FACTOR("sd-33m", -1, "sd-200m", 1, 6),		\
+	UNIPHIER_CLK_FACTOR("sd-50m", -1, "sd-200m", 1, 4),		\
+	UNIPHIER_CLK_FACTOR("sd-67m", -1, "sd-200m", 1, 3),		\
+	UNIPHIER_CLK_FACTOR("sd-100m", -1, "sd-200m", 1, 2),		\
+	UNIPHIER_CLK_FACTOR("sd-40m", -1, "sd-200m", 1, 5),		\
+	UNIPHIER_CLK_FACTOR("sd-25m", -1, "sd-200m", 1, 8),		\
+	UNIPHIER_CLK_FACTOR("sd-22m", -1, "sd-133m", 1, 6)
+
+#define UNIPHIER_MIO_CLK_SD(_idx, ch)					\
+	{								\
+		.name = "sd" #ch "-sel",				\
+		.type = UNIPHIER_CLK_TYPE_MUX,				\
+		.idx = -1,						\
+		.data.mux = {						\
+			.parent_names = {				\
+				"sd-44m",				\
+				"sd-33m",				\
+				"sd-50m",				\
+				"sd-67m",				\
+				"sd-100m",				\
+				"sd-40m",				\
+				"sd-25m",				\
+				"sd-22m",				\
+			},						\
+			.num_parents = 8,				\
+			.reg = 0x30 + 0x200 * (ch),			\
+			.masks = {					\
+				0x00031000,				\
+				0x00031000,				\
+				0x00031000,				\
+				0x00031000,				\
+				0x00001300,				\
+				0x00001300,				\
+				0x00001300,				\
+				0x00001300,				\
+			},						\
+			.vals = {					\
+				0x00000000,				\
+				0x00010000,				\
+				0x00020000,				\
+				0x00030000,				\
+				0x00001000,				\
+				0x00001100,				\
+				0x00001200,				\
+				0x00001300,				\
+			},						\
+		},							\
+	},								\
+	UNIPHIER_CLK_GATE("sd" #ch, (_idx), "sd" #ch "-sel", 0x20 + 0x200 * (ch), 8)
+
+#define UNIPHIER_MIO_CLK_USB2(idx, ch)					\
+	UNIPHIER_CLK_GATE("usb2" #ch, (idx), "usb2", 0x20 + 0x200 * (ch), 28)
+
+#define UNIPHIER_MIO_CLK_USB2_PHY(idx, ch)				\
+	UNIPHIER_CLK_GATE("usb2" #ch "-phy", (idx), "usb2", 0x20 + 0x200 * (ch), 29)
+
+#define UNIPHIER_MIO_CLK_DMAC(idx)					\
+	UNIPHIER_CLK_GATE("miodmac", (idx), "stdmac", 0x20, 25)
+
+const struct uniphier_clk_data uniphier_sld3_mio_clk_data[] = {
+	UNIPHIER_MIO_CLK_SD_FIXED,
+	UNIPHIER_MIO_CLK_SD(0, 0),
+	UNIPHIER_MIO_CLK_SD(1, 1),
+	UNIPHIER_MIO_CLK_SD(2, 2),
+	UNIPHIER_MIO_CLK_DMAC(7),
+	UNIPHIER_MIO_CLK_USB2(8, 0),
+	UNIPHIER_MIO_CLK_USB2(9, 1),
+	UNIPHIER_MIO_CLK_USB2(10, 2),
+	UNIPHIER_MIO_CLK_USB2(11, 3),
+	UNIPHIER_MIO_CLK_USB2_PHY(12, 0),
+	UNIPHIER_MIO_CLK_USB2_PHY(13, 1),
+	UNIPHIER_MIO_CLK_USB2_PHY(14, 2),
+	UNIPHIER_MIO_CLK_USB2_PHY(15, 3),
+	{ /* sentinel */ }
+};
+
+const struct uniphier_clk_data uniphier_pro5_mio_clk_data[] = {
+	UNIPHIER_MIO_CLK_SD_FIXED,
+	UNIPHIER_MIO_CLK_SD(0, 0),
+	UNIPHIER_MIO_CLK_SD(1, 1),
+	{ /* sentinel */ }
+};
diff --git a/drivers/clk/uniphier/clk-uniphier-mux.c b/drivers/clk/uniphier/clk-uniphier-mux.c
new file mode 100644
index 000000000000..15a2f2cbe0d9
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier-mux.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/regmap.h>
+
+#include "clk-uniphier.h"
+
+struct uniphier_clk_mux {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	unsigned int reg;
+	const unsigned int *masks;
+	const unsigned int *vals;
+};
+
+#define to_uniphier_clk_mux(_hw) container_of(_hw, struct uniphier_clk_mux, hw)
+
+static int uniphier_clk_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct uniphier_clk_mux *mux = to_uniphier_clk_mux(hw);
+
+	return regmap_write_bits(mux->regmap, mux->reg, mux->masks[index],
+				 mux->vals[index]);
+}
+
+static u8 uniphier_clk_mux_get_parent(struct clk_hw *hw)
+{
+	struct uniphier_clk_mux *mux = to_uniphier_clk_mux(hw);
+	int num_parents = clk_hw_get_num_parents(hw);
+	int ret;
+	u32 val;
+	u8 i;
+
+	ret = regmap_read(mux->regmap, mux->reg, &val);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < num_parents; i++)
+		if ((mux->masks[i] & val) == mux->vals[i])
+			return i;
+
+	return -EINVAL;
+}
+
+static const struct clk_ops uniphier_clk_mux_ops = {
+	.determine_rate = __clk_mux_determine_rate,
+	.set_parent = uniphier_clk_mux_set_parent,
+	.get_parent = uniphier_clk_mux_get_parent,
+};
+
+struct clk_hw *uniphier_clk_register_mux(struct device *dev,
+					 struct regmap *regmap,
+					 const char *name,
+				const struct uniphier_clk_mux_data *data)
+{
+	struct uniphier_clk_mux *mux;
+	struct clk_init_data init;
+	int ret;
+
+	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &uniphier_clk_mux_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	init.parent_names = data->parent_names;
+	init.num_parents = data->num_parents,
+
+	mux->regmap = regmap;
+	mux->reg = data->reg;
+	mux->masks = data->masks;
+	mux->vals = data->vals;
+	mux->hw.init = &init;
+
+	ret = devm_clk_hw_register(dev, &mux->hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &mux->hw;
+}
diff --git a/drivers/clk/uniphier/clk-uniphier-peri.c b/drivers/clk/uniphier/clk-uniphier-peri.c
new file mode 100644
index 000000000000..521c80e9a06f
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier-peri.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "clk-uniphier.h"
+
+#define UNIPHIER_PERI_CLK_UART(idx, ch)					\
+	UNIPHIER_CLK_GATE("uart" #ch, (idx), "uart", 0x24, 19 + (ch))
+
+#define UNIPHIER_PERI_CLK_I2C_COMMON					\
+	UNIPHIER_CLK_GATE("i2c-common", -1, "i2c", 0x20, 1)
+
+#define UNIPHIER_PERI_CLK_I2C(idx, ch)					\
+	UNIPHIER_CLK_GATE("i2c" #ch, (idx), "i2c-common", 0x24, 5 + (ch))
+
+#define UNIPHIER_PERI_CLK_FI2C(idx, ch)					\
+	UNIPHIER_CLK_GATE("i2c" #ch, (idx), "i2c", 0x24, 24 + (ch))
+
+const struct uniphier_clk_data uniphier_ld4_peri_clk_data[] = {
+	UNIPHIER_PERI_CLK_UART(0, 0),
+	UNIPHIER_PERI_CLK_UART(1, 1),
+	UNIPHIER_PERI_CLK_UART(2, 2),
+	UNIPHIER_PERI_CLK_UART(3, 3),
+	UNIPHIER_PERI_CLK_I2C_COMMON,
+	UNIPHIER_PERI_CLK_I2C(4, 0),
+	UNIPHIER_PERI_CLK_I2C(5, 1),
+	UNIPHIER_PERI_CLK_I2C(6, 2),
+	UNIPHIER_PERI_CLK_I2C(7, 3),
+	UNIPHIER_PERI_CLK_I2C(8, 4),
+	{ /* sentinel */ }
+};
+
+const struct uniphier_clk_data uniphier_pro4_peri_clk_data[] = {
+	UNIPHIER_PERI_CLK_UART(0, 0),
+	UNIPHIER_PERI_CLK_UART(1, 1),
+	UNIPHIER_PERI_CLK_UART(2, 2),
+	UNIPHIER_PERI_CLK_UART(3, 3),
+	UNIPHIER_PERI_CLK_FI2C(4, 0),
+	UNIPHIER_PERI_CLK_FI2C(5, 1),
+	UNIPHIER_PERI_CLK_FI2C(6, 2),
+	UNIPHIER_PERI_CLK_FI2C(7, 3),
+	UNIPHIER_PERI_CLK_FI2C(8, 4),
+	UNIPHIER_PERI_CLK_FI2C(9, 5),
+	UNIPHIER_PERI_CLK_FI2C(10, 6),
+	{ /* sentinel */ }
+};
diff --git a/drivers/clk/uniphier/clk-uniphier-sys.c b/drivers/clk/uniphier/clk-uniphier-sys.c
new file mode 100644
index 000000000000..5d029991047d
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier-sys.c
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/stddef.h>
+
+#include "clk-uniphier.h"
+
+#define UNIPHIER_SLD3_SYS_CLK_SD					\
+	UNIPHIER_CLK_FACTOR("sd-200m", -1, "spll", 1, 8),		\
+	UNIPHIER_CLK_FACTOR("sd-133m", -1, "vpll27a", 1, 2)
+
+#define UNIPHIER_PRO5_SYS_CLK_SD					\
+	UNIPHIER_CLK_FACTOR("sd-200m", -1, "spll", 1, 12),		\
+	UNIPHIER_CLK_FACTOR("sd-133m", -1, "spll", 1, 18)
+
+#define UNIPHIER_LD20_SYS_CLK_SD					\
+	UNIPHIER_CLK_FACTOR("sd-200m", -1, "spll", 1, 10),		\
+	UNIPHIER_CLK_FACTOR("sd-133m", -1, "spll", 1, 15)
+
+#define UNIPHIER_SLD3_SYS_CLK_STDMAC(idx)				\
+	UNIPHIER_CLK_GATE("stdmac", (idx), NULL, 0x2104, 10)
+
+#define UNIPHIER_LD11_SYS_CLK_STDMAC(idx)				\
+	UNIPHIER_CLK_GATE("stdmac", (idx), NULL, 0x210c, 8)
+
+#define UNIPHIER_PRO4_SYS_CLK_GIO(idx)					\
+	UNIPHIER_CLK_GATE("gio", (idx), NULL, 0x2104, 6)
+
+#define UNIPHIER_PRO4_SYS_CLK_USB3(idx, ch)				\
+	UNIPHIER_CLK_GATE("usb3" #ch, (idx), NULL, 0x2104, 16 + (ch))
+
+const struct uniphier_clk_data uniphier_sld3_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 65, 1),		/* 1597.44 MHz */
+	UNIPHIER_CLK_FACTOR("upll", -1, "ref", 6000, 512),	/* 288 MHz */
+	UNIPHIER_CLK_FACTOR("a2pll", -1, "ref", 24, 1),		/* 589.824 MHz */
+	UNIPHIER_CLK_FACTOR("vpll27a", -1, "ref", 5625, 512),	/* 270 MHz */
+	UNIPHIER_CLK_FACTOR("uart", 0, "a2pll", 1, 16),
+	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 16),
+	UNIPHIER_SLD3_SYS_CLK_SD,
+	UNIPHIER_CLK_FACTOR("usb2", -1, "upll", 1, 12),
+	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),
+	{ /* sentinel */ }
+};
+
+const struct uniphier_clk_data uniphier_ld4_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 65, 1),		/* 1597.44 MHz */
+	UNIPHIER_CLK_FACTOR("upll", -1, "ref", 6000, 512),	/* 288 MHz */
+	UNIPHIER_CLK_FACTOR("a2pll", -1, "ref", 24, 1),		/* 589.824 MHz */
+	UNIPHIER_CLK_FACTOR("vpll27a", -1, "ref", 5625, 512),	/* 270 MHz */
+	UNIPHIER_CLK_FACTOR("uart", 0, "a2pll", 1, 16),
+	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 16),
+	UNIPHIER_SLD3_SYS_CLK_SD,
+	UNIPHIER_CLK_FACTOR("usb2", -1, "upll", 1, 12),
+	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),		/* Ether, HSC, MIO */
+	{ /* sentinel */ }
+};
+
+const struct uniphier_clk_data uniphier_pro4_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 64, 1),		/* 1600 MHz */
+	UNIPHIER_CLK_FACTOR("upll", -1, "ref", 288, 25),	/* 288 MHz */
+	UNIPHIER_CLK_FACTOR("a2pll", -1, "upll", 256, 125),	/* 589.824 MHz */
+	UNIPHIER_CLK_FACTOR("vpll27a", -1, "ref", 270, 25),	/* 270 MHz */
+	UNIPHIER_CLK_FACTOR("uart", 0, "a2pll", 1, 8),
+	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 32),
+	UNIPHIER_SLD3_SYS_CLK_SD,
+	UNIPHIER_CLK_FACTOR("usb2", -1, "upll", 1, 12),
+	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),		/* HSC, MIO, RLE */
+	UNIPHIER_PRO4_SYS_CLK_GIO(12),			/* Ether, SATA, USB3 */
+	UNIPHIER_PRO4_SYS_CLK_USB3(14, 0),
+	UNIPHIER_PRO4_SYS_CLK_USB3(15, 1),
+	{ /* sentinel */ }
+};
+
+const struct uniphier_clk_data uniphier_sld8_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 64, 1),		/* 1600 MHz */
+	UNIPHIER_CLK_FACTOR("upll", -1, "ref", 288, 25),	/* 288 MHz */
+	UNIPHIER_CLK_FACTOR("vpll27a", -1, "ref", 270, 25),	/* 270 MHz */
+	UNIPHIER_CLK_FACTOR("uart", 0, "spll", 1, 20),
+	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 16),
+	UNIPHIER_SLD3_SYS_CLK_SD,
+	UNIPHIER_CLK_FACTOR("usb2", -1, "upll", 1, 12),
+	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),		/* Ether, HSC, MIO */
+	{ /* sentinel */ }
+};
+
+const struct uniphier_clk_data uniphier_pro5_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 120, 1),		/* 2400 MHz */
+	UNIPHIER_CLK_FACTOR("dapll1", -1, "ref", 128, 1),	/* 2560 MHz */
+	UNIPHIER_CLK_FACTOR("dapll2", -1, "ref", 144, 125),	/* 2949.12 MHz */
+	UNIPHIER_CLK_FACTOR("uart", 0, "dapll2", 1, 40),
+	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 48),
+	UNIPHIER_PRO5_SYS_CLK_SD,
+	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),			/* HSC */
+	UNIPHIER_PRO4_SYS_CLK_GIO(12),				/* PCIe, USB3 */
+	UNIPHIER_PRO4_SYS_CLK_USB3(14, 0),
+	UNIPHIER_PRO4_SYS_CLK_USB3(15, 1),
+	{ /* sentinel */ }
+};
+
+const struct uniphier_clk_data uniphier_pxs2_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 96, 1),		/* 2400 MHz */
+	UNIPHIER_CLK_FACTOR("uart", 0, "spll", 1, 27),
+	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 48),
+	UNIPHIER_PRO5_SYS_CLK_SD,
+	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),			/* HSC, RLE */
+	/* GIO is always clock-enabled: no function for 0x2104 bit6 */
+	UNIPHIER_PRO4_SYS_CLK_USB3(14, 0),
+	UNIPHIER_PRO4_SYS_CLK_USB3(15, 1),
+	/* The document mentions 0x2104 bit 18, but not functional */
+	UNIPHIER_CLK_GATE("usb30-phy", 16, NULL, 0x2104, 19),
+	UNIPHIER_CLK_GATE("usb31-phy", 20, NULL, 0x2104, 20),
+	{ /* sentinel */ }
+};
+
+const struct uniphier_clk_data uniphier_ld11_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 80, 1),		/* 2000 MHz */
+	UNIPHIER_CLK_FACTOR("uart", 0, "spll", 1, 34),
+	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 40),
+	UNIPHIER_LD11_SYS_CLK_STDMAC(8),			/* HSC, MIO */
+	UNIPHIER_CLK_FACTOR("usb2", -1, "ref", 24, 25),
+	{ /* sentinel */ }
+};
+
+const struct uniphier_clk_data uniphier_ld20_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 80, 1),		/* 2000 MHz */
+	UNIPHIER_CLK_FACTOR("uart", 0, "spll", 1, 34),
+	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 40),
+	UNIPHIER_LD20_SYS_CLK_SD,
+	UNIPHIER_LD11_SYS_CLK_STDMAC(8),			/* HSC */
+	/* GIO is always clock-enabled: no function for 0x210c bit5 */
+	/*
+	 * clock for USB Link is enabled by the logic "OR" of bit 14 and bit 15.
+	 * We do not use bit 15 here.
+	 */
+	UNIPHIER_CLK_GATE("usb30", 14, NULL, 0x210c, 14),
+	UNIPHIER_CLK_GATE("usb30-phy0", 16, NULL, 0x210c, 12),
+	UNIPHIER_CLK_GATE("usb30-phy1", 17, NULL, 0x210c, 13),
+	{ /* sentinel */ }
+};
diff --git a/drivers/clk/uniphier/clk-uniphier.h b/drivers/clk/uniphier/clk-uniphier.h
new file mode 100644
index 000000000000..3ae184062388
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier.h
@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __CLK_UNIPHIER_H__
+#define __CLK_UNIPHIER_H__
+
+struct clk_hw;
+struct device;
+struct regmap;
+
+#define UNIPHIER_CLK_MUX_MAX_PARENTS	8
+
+enum uniphier_clk_type {
+	UNIPHIER_CLK_TYPE_FIXED_FACTOR,
+	UNIPHIER_CLK_TYPE_FIXED_RATE,
+	UNIPHIER_CLK_TYPE_GATE,
+	UNIPHIER_CLK_TYPE_MUX,
+};
+
+struct uniphier_clk_fixed_factor_data {
+	const char *parent_name;
+	unsigned int mult;
+	unsigned int div;
+};
+
+struct uniphier_clk_fixed_rate_data {
+	unsigned long fixed_rate;
+};
+
+struct uniphier_clk_gate_data {
+	const char *parent_name;
+	unsigned int reg;
+	unsigned int bit;
+};
+
+struct uniphier_clk_mux_data {
+	const char *parent_names[UNIPHIER_CLK_MUX_MAX_PARENTS];
+	unsigned int num_parents;
+	unsigned int reg;
+	unsigned int masks[UNIPHIER_CLK_MUX_MAX_PARENTS];
+	unsigned int vals[UNIPHIER_CLK_MUX_MAX_PARENTS];
+};
+
+struct uniphier_clk_data {
+	const char *name;
+	enum uniphier_clk_type type;
+	int idx;
+	union {
+		struct uniphier_clk_fixed_factor_data factor;
+		struct uniphier_clk_fixed_rate_data rate;
+		struct uniphier_clk_gate_data gate;
+		struct uniphier_clk_mux_data mux;
+	} data;
+};
+
+#define UNIPHIER_CLK_FACTOR(_name, _idx, _parent, _mult, _div)	\
+	{							\
+		.name = (_name),				\
+		.type = UNIPHIER_CLK_TYPE_FIXED_FACTOR,		\
+		.idx = (_idx),					\
+		.data.factor = {				\
+			.parent_name = (_parent),		\
+			.mult = (_mult),			\
+			.div = (_div),				\
+		},						\
+	}
+
+
+#define UNIPHIER_CLK_GATE(_name, _idx, _parent, _reg, _bit)	\
+	{							\
+		.name = (_name),				\
+		.type = UNIPHIER_CLK_TYPE_GATE,			\
+		.idx = (_idx),					\
+		.data.gate = {					\
+			.parent_name = (_parent),		\
+			.reg = (_reg),				\
+			.bit = (_bit),				\
+		},						\
+	}
+
+
+struct clk_hw *uniphier_clk_register_fixed_factor(struct device *dev,
+						  const char *name,
+			const struct uniphier_clk_fixed_factor_data *data);
+struct clk_hw *uniphier_clk_register_fixed_rate(struct device *dev,
+						const char *name,
+			const struct uniphier_clk_fixed_rate_data *data);
+struct clk_hw *uniphier_clk_register_gate(struct device *dev,
+					  struct regmap *regmap,
+					  const char *name,
+				const struct uniphier_clk_gate_data *data);
+struct clk_hw *uniphier_clk_register_mux(struct device *dev,
+					 struct regmap *regmap,
+					 const char *name,
+				const struct uniphier_clk_mux_data *data);
+
+extern const struct uniphier_clk_data uniphier_sld3_sys_clk_data[];
+extern const struct uniphier_clk_data uniphier_ld4_sys_clk_data[];
+extern const struct uniphier_clk_data uniphier_pro4_sys_clk_data[];
+extern const struct uniphier_clk_data uniphier_sld8_sys_clk_data[];
+extern const struct uniphier_clk_data uniphier_pro5_sys_clk_data[];
+extern const struct uniphier_clk_data uniphier_pxs2_sys_clk_data[];
+extern const struct uniphier_clk_data uniphier_ld11_sys_clk_data[];
+extern const struct uniphier_clk_data uniphier_ld20_sys_clk_data[];
+extern const struct uniphier_clk_data uniphier_sld3_mio_clk_data[];
+extern const struct uniphier_clk_data uniphier_pro5_mio_clk_data[];
+extern const struct uniphier_clk_data uniphier_ld4_peri_clk_data[];
+extern const struct uniphier_clk_data uniphier_pro4_peri_clk_data[];
+
+#endif /* __CLK_UNIPHIER_H__ */
diff --git a/drivers/clk/versatile/clk-icst.c b/drivers/clk/versatile/clk-icst.c
index 5e9b65278e4c..4faa94440779 100644
--- a/drivers/clk/versatile/clk-icst.c
+++ b/drivers/clk/versatile/clk-icst.c
@@ -27,6 +27,26 @@
 /* Magic unlocking token used on all Versatile boards */
 #define VERSATILE_LOCK_VAL	0xA05F
 
+#define VERSATILE_AUX_OSC_BITS 0x7FFFF
+#define INTEGRATOR_AP_CM_BITS 0xFF
+#define INTEGRATOR_AP_SYS_BITS 0xFF
+#define INTEGRATOR_CP_CM_CORE_BITS 0x7FF
+#define INTEGRATOR_CP_CM_MEM_BITS 0x7FF000
+
+#define INTEGRATOR_AP_PCI_25_33_MHZ BIT(8)
+
+/**
+ * enum icst_control_type - the type of ICST control register
+ */
+enum icst_control_type {
+	ICST_VERSATILE, /* The standard type, all control bits available */
+	ICST_INTEGRATOR_AP_CM, /* Only 8 bits of VDW available */
+	ICST_INTEGRATOR_AP_SYS, /* Only 8 bits of VDW available */
+	ICST_INTEGRATOR_AP_PCI, /* Odd bit pattern storage */
+	ICST_INTEGRATOR_CP_CM_CORE, /* Only 8 bits of VDW and 3 bits of OD */
+	ICST_INTEGRATOR_CP_CM_MEM, /* Only 8 bits of VDW and 3 bits of OD */
+};
+
 /**
  * struct clk_icst - ICST VCO clock wrapper
  * @hw: corresponding clock hardware entry
@@ -34,6 +54,7 @@
  * @lockreg: VCO lock register address
  * @params: parameters for this ICST instance
  * @rate: current rate
+ * @ctype: the type of control register for the ICST
  */
 struct clk_icst {
 	struct clk_hw hw;
@@ -42,6 +63,7 @@ struct clk_icst {
 	u32 lockreg_off;
 	struct icst_params *params;
 	unsigned long rate;
+	enum icst_control_type ctype;
 };
 
 #define to_icst(_hw) container_of(_hw, struct clk_icst, hw)
@@ -59,6 +81,76 @@ static int vco_get(struct clk_icst *icst, struct icst_vco *vco)
 	ret = regmap_read(icst->map, icst->vcoreg_off, &val);
 	if (ret)
 		return ret;
+
+	/*
+	 * The Integrator/AP core clock can only access the low eight
+	 * bits of the v PLL divider. Bit 8 is tied low and always zero,
+	 * r is hardwired to 22 and output divider s is hardwired to 1
+	 * (divide by 2) according to the document
+	 * "Integrator CM926EJ-S, CM946E-S, CM966E-S, CM1026EJ-S and
+	 * CM1136JF-S User Guide" ARM DUI 0138E, page 3-13 thru 3-14.
+	 */
+	if (icst->ctype == ICST_INTEGRATOR_AP_CM) {
+		vco->v = val & INTEGRATOR_AP_CM_BITS;
+		vco->r = 22;
+		vco->s = 1;
+		return 0;
+	}
+
+	/*
+	 * The Integrator/AP system clock on the base board can only
+	 * access the low eight bits of the v PLL divider. Bit 8 is tied low
+	 * and always zero, r is hardwired to 46, and the output divider is
+	 * hardwired to 3 (divide by 4) according to the document
+	 * "Integrator AP ASIC Development Motherboard" ARM DUI 0098B,
+	 * page 3-16.
+	 */
+	if (icst->ctype == ICST_INTEGRATOR_AP_SYS) {
+		vco->v = val & INTEGRATOR_AP_SYS_BITS;
+		vco->r = 46;
+		vco->s = 3;
+		return 0;
+	}
+
+	/*
+	 * The Integrator/AP PCI clock is using an odd pattern to create
+	 * the child clock, basically a single bit called DIVX/Y is used
+	 * to select between two different hardwired values: setting the
+	 * bit to 0 yields v = 17, r = 22 and OD = 1, whereas setting the
+	 * bit to 1 yields v = 14, r = 14 and OD = 1 giving the frequencies
+	 * 33 or 25 MHz respectively.
+	 */
+	if (icst->ctype == ICST_INTEGRATOR_AP_PCI) {
+		bool divxy = !!(val & INTEGRATOR_AP_PCI_25_33_MHZ);
+
+		vco->v = divxy ? 17 : 14;
+		vco->r = divxy ? 22 : 14;
+		vco->s = 1;
+		return 0;
+	}
+
+	/*
+	 * The Integrator/CP core clock can access the low eight bits
+	 * of the v PLL divider. Bit 8 is tied low and always zero,
+	 * r is hardwired to 22 and the output divider s is accessible
+	 * in bits 8 thru 10 according to the document
+	 * "Integrator/CM940T, CM920T, CM740T, and CM720T User Guide"
+	 * ARM DUI 0157A, page 3-20 thru 3-23 and 4-10.
+	 */
+	if (icst->ctype == ICST_INTEGRATOR_CP_CM_CORE) {
+		vco->v = val & 0xFF;
+		vco->r = 22;
+		vco->s = (val >> 8) & 7;
+		return 0;
+	}
+
+	if (icst->ctype == ICST_INTEGRATOR_CP_CM_MEM) {
+		vco->v = (val >> 12) & 0xFF;
+		vco->r = 22;
+		vco->s = (val >> 20) & 7;
+		return 0;
+	}
+
 	vco->v = val & 0x1ff;
 	vco->r = (val >> 9) & 0x7f;
 	vco->s = (val >> 16) & 03;
@@ -72,22 +164,62 @@ static int vco_get(struct clk_icst *icst, struct icst_vco *vco)
  */
 static int vco_set(struct clk_icst *icst, struct icst_vco vco)
 {
+	u32 mask;
 	u32 val;
 	int ret;
 
-	ret = regmap_read(icst->map, icst->vcoreg_off, &val);
-	if (ret)
-		return ret;
+	/* Mask the bits used by the VCO */
+	switch (icst->ctype) {
+	case ICST_INTEGRATOR_AP_CM:
+		mask = INTEGRATOR_AP_CM_BITS;
+		val = vco.v & 0xFF;
+		if (vco.v & 0x100)
+			pr_err("ICST error: tried to set bit 8 of VDW\n");
+		if (vco.s != 1)
+			pr_err("ICST error: tried to use VOD != 1\n");
+		if (vco.r != 22)
+			pr_err("ICST error: tried to use RDW != 22\n");
+		break;
+	case ICST_INTEGRATOR_AP_SYS:
+		mask = INTEGRATOR_AP_SYS_BITS;
+		val = vco.v & 0xFF;
+		if (vco.v & 0x100)
+			pr_err("ICST error: tried to set bit 8 of VDW\n");
+		if (vco.s != 3)
+			pr_err("ICST error: tried to use VOD != 1\n");
+		if (vco.r != 46)
+			pr_err("ICST error: tried to use RDW != 22\n");
+		break;
+	case ICST_INTEGRATOR_CP_CM_CORE:
+		mask = INTEGRATOR_CP_CM_CORE_BITS; /* Uses 12 bits */
+		val = (vco.v & 0xFF) | vco.s << 8;
+		if (vco.v & 0x100)
+			pr_err("ICST error: tried to set bit 8 of VDW\n");
+		if (vco.r != 22)
+			pr_err("ICST error: tried to use RDW != 22\n");
+		break;
+	case ICST_INTEGRATOR_CP_CM_MEM:
+		mask = INTEGRATOR_CP_CM_MEM_BITS; /* Uses 12 bits */
+		val = ((vco.v & 0xFF) << 12) | (vco.s << 20);
+		if (vco.v & 0x100)
+			pr_err("ICST error: tried to set bit 8 of VDW\n");
+		if (vco.r != 22)
+			pr_err("ICST error: tried to use RDW != 22\n");
+		break;
+	default:
+		/* Regular auxilary oscillator */
+		mask = VERSATILE_AUX_OSC_BITS;
+		val = vco.v | (vco.r << 9) | (vco.s << 16);
+		break;
+	}
 
-	/* Mask the 18 bits used by the VCO */
-	val &= ~0x7ffff;
-	val |= vco.v | (vco.r << 9) | (vco.s << 16);
+	pr_debug("ICST: new val = 0x%08x\n", val);
 
 	/* This magic unlocks the VCO so it can be controlled */
 	ret = regmap_write(icst->map, icst->lockreg_off, VERSATILE_LOCK_VAL);
 	if (ret)
 		return ret;
-	ret = regmap_write(icst->map, icst->vcoreg_off, val);
+	ret = regmap_update_bits(icst->map, icst->vcoreg_off, mask, val);
 	if (ret)
 		return ret;
 	/* This locks the VCO again */
@@ -121,6 +253,46 @@ static long icst_round_rate(struct clk_hw *hw, unsigned long rate,
 	struct clk_icst *icst = to_icst(hw);
 	struct icst_vco vco;
 
+	if (icst->ctype == ICST_INTEGRATOR_AP_CM ||
+	    icst->ctype == ICST_INTEGRATOR_CP_CM_CORE) {
+		if (rate <= 12000000)
+			return 12000000;
+		if (rate >= 160000000)
+			return 160000000;
+		/* Slam to closest megahertz */
+		return DIV_ROUND_CLOSEST(rate, 1000000) * 1000000;
+	}
+
+	if (icst->ctype == ICST_INTEGRATOR_CP_CM_MEM) {
+		if (rate <= 6000000)
+			return 6000000;
+		if (rate >= 66000000)
+			return 66000000;
+		/* Slam to closest 0.5 megahertz */
+		return DIV_ROUND_CLOSEST(rate, 500000) * 500000;
+	}
+
+	if (icst->ctype == ICST_INTEGRATOR_AP_SYS) {
+		/* Divides between 3 and 50 MHz in steps of 0.25 MHz */
+		if (rate <= 3000000)
+			return 3000000;
+		if (rate >= 50000000)
+			return 5000000;
+		/* Slam to closest 0.25 MHz */
+		return DIV_ROUND_CLOSEST(rate, 250000) * 250000;
+	}
+
+	if (icst->ctype == ICST_INTEGRATOR_AP_PCI) {
+		/*
+		 * If we're below or less than halfway from 25 to 33 MHz
+		 * select 25 MHz
+		 */
+		if (rate <= 25000000 || rate < 29000000)
+			return 25000000;
+		/* Else just return the default frequency */
+		return 33000000;
+	}
+
 	vco = icst_hz_to_vco(icst->params, rate);
 	return icst_hz(icst->params, vco);
 }
@@ -131,6 +303,36 @@ static int icst_set_rate(struct clk_hw *hw, unsigned long rate,
 	struct clk_icst *icst = to_icst(hw);
 	struct icst_vco vco;
 
+	if (icst->ctype == ICST_INTEGRATOR_AP_PCI) {
+		/* This clock is especially primitive */
+		unsigned int val;
+		int ret;
+
+		if (rate == 25000000) {
+			val = 0;
+		} else if (rate == 33000000) {
+			val = INTEGRATOR_AP_PCI_25_33_MHZ;
+		} else {
+			pr_err("ICST: cannot set PCI frequency %lu\n",
+			       rate);
+			return -EINVAL;
+		}
+		ret = regmap_write(icst->map, icst->lockreg_off,
+				   VERSATILE_LOCK_VAL);
+		if (ret)
+			return ret;
+		ret = regmap_update_bits(icst->map, icst->vcoreg_off,
+					 INTEGRATOR_AP_PCI_25_33_MHZ,
+					 val);
+		if (ret)
+			return ret;
+		/* This locks the VCO again */
+		ret = regmap_write(icst->map, icst->lockreg_off, 0);
+		if (ret)
+			return ret;
+		return 0;
+	}
+
 	if (parent_rate)
 		icst->params->ref = parent_rate;
 	vco = icst_hz_to_vco(icst->params, rate);
@@ -148,7 +350,8 @@ static struct clk *icst_clk_setup(struct device *dev,
 				  const struct clk_icst_desc *desc,
 				  const char *name,
 				  const char *parent_name,
-				  struct regmap *map)
+				  struct regmap *map,
+				  enum icst_control_type ctype)
 {
 	struct clk *clk;
 	struct clk_icst *icst;
@@ -178,6 +381,7 @@ static struct clk *icst_clk_setup(struct device *dev,
 	icst->params = pclone;
 	icst->vcoreg_off = desc->vco_offset;
 	icst->lockreg_off = desc->lock_offset;
+	icst->ctype = ctype;
 
 	clk = clk_register(dev, &icst->hw);
 	if (IS_ERR(clk)) {
@@ -206,7 +410,8 @@ struct clk *icst_clk_register(struct device *dev,
 		pr_err("could not initialize ICST regmap\n");
 		return ERR_CAST(map);
 	}
-	return icst_clk_setup(dev, desc, name, parent_name, map);
+	return icst_clk_setup(dev, desc, name, parent_name, map,
+			      ICST_VERSATILE);
 }
 EXPORT_SYMBOL_GPL(icst_clk_register);
 
@@ -239,6 +444,56 @@ static const struct icst_params icst307_params = {
 	.idx2s		= icst307_idx2s,
 };
 
+/**
+ * The core modules on the Integrator/AP and Integrator/CP have
+ * especially crippled ICST525 control.
+ */
+static const struct icst_params icst525_apcp_cm_params = {
+	.vco_max	= ICST525_VCO_MAX_5V,
+	.vco_min	= ICST525_VCO_MIN,
+	/* Minimum 12 MHz, VDW = 4 */
+	.vd_min		= 12,
+	/*
+	 * Maximum 160 MHz, VDW = 152 for all core modules, but
+	 * CM926EJ-S, CM1026EJ-S and CM1136JF-S can actually
+	 * go to 200 MHz (max VDW = 192).
+	 */
+	.vd_max		= 192,
+	/* r is hardcoded to 22 and this is the actual divisor, +2 */
+	.rd_min		= 24,
+	.rd_max		= 24,
+	.s2div		= icst525_s2div,
+	.idx2s		= icst525_idx2s,
+};
+
+static const struct icst_params icst525_ap_sys_params = {
+	.vco_max	= ICST525_VCO_MAX_5V,
+	.vco_min	= ICST525_VCO_MIN,
+	/* Minimum 3 MHz, VDW = 4 */
+	.vd_min		= 3,
+	/* Maximum 50 MHz, VDW = 192 */
+	.vd_max		= 50,
+	/* r is hardcoded to 46 and this is the actual divisor, +2 */
+	.rd_min		= 48,
+	.rd_max		= 48,
+	.s2div		= icst525_s2div,
+	.idx2s		= icst525_idx2s,
+};
+
+static const struct icst_params icst525_ap_pci_params = {
+	.vco_max	= ICST525_VCO_MAX_5V,
+	.vco_min	= ICST525_VCO_MIN,
+	/* Minimum 25 MHz */
+	.vd_min		= 25,
+	/* Maximum 33 MHz */
+	.vd_max		= 33,
+	/* r is hardcoded to 14 or 22 and this is the actual divisors +2 */
+	.rd_min		= 16,
+	.rd_max		= 24,
+	.s2div		= icst525_s2div,
+	.idx2s		= icst525_idx2s,
+};
+
 static void __init of_syscon_icst_setup(struct device_node *np)
 {
 	struct device_node *parent;
@@ -247,6 +502,7 @@ static void __init of_syscon_icst_setup(struct device_node *np)
 	const char *name = np->name;
 	const char *parent_name;
 	struct clk *regclk;
+	enum icst_control_type ctype;
 
 	/* We do not release this reference, we are using it perpetually */
 	parent = of_get_parent(np);
@@ -269,11 +525,28 @@ static void __init of_syscon_icst_setup(struct device_node *np)
 		return;
 	}
 
-	if (of_device_is_compatible(np, "arm,syscon-icst525"))
+	if (of_device_is_compatible(np, "arm,syscon-icst525")) {
 		icst_desc.params = &icst525_params;
-	else if (of_device_is_compatible(np, "arm,syscon-icst307"))
+		ctype = ICST_VERSATILE;
+	} else if (of_device_is_compatible(np, "arm,syscon-icst307")) {
 		icst_desc.params = &icst307_params;
-	else {
+		ctype = ICST_VERSATILE;
+	} else if (of_device_is_compatible(np, "arm,syscon-icst525-integratorap-cm")) {
+		icst_desc.params = &icst525_apcp_cm_params;
+		ctype = ICST_INTEGRATOR_AP_CM;
+	} else if (of_device_is_compatible(np, "arm,syscon-icst525-integratorap-sys")) {
+		icst_desc.params = &icst525_ap_sys_params;
+		ctype = ICST_INTEGRATOR_AP_SYS;
+	} else if (of_device_is_compatible(np, "arm,syscon-icst525-integratorap-pci")) {
+		icst_desc.params = &icst525_ap_pci_params;
+		ctype = ICST_INTEGRATOR_AP_PCI;
+	} else if (of_device_is_compatible(np, "arm,syscon-icst525-integratorcp-cm-core")) {
+		icst_desc.params = &icst525_apcp_cm_params;
+		ctype = ICST_INTEGRATOR_CP_CM_CORE;
+	} else if (of_device_is_compatible(np, "arm,syscon-icst525-integratorcp-cm-mem")) {
+		icst_desc.params = &icst525_apcp_cm_params;
+		ctype = ICST_INTEGRATOR_CP_CM_MEM;
+	} else {
 		pr_err("unknown ICST clock %s\n", name);
 		return;
 	}
@@ -281,7 +554,7 @@ static void __init of_syscon_icst_setup(struct device_node *np)
 	/* Parent clock name is not the same as node parent */
 	parent_name = of_clk_get_parent_name(np, 0);
 
-	regclk = icst_clk_setup(NULL, &icst_desc, name, parent_name, map);
+	regclk = icst_clk_setup(NULL, &icst_desc, name, parent_name, map, ctype);
 	if (IS_ERR(regclk)) {
 		pr_err("error setting up syscon ICST clock %s\n", name);
 		return;
@@ -294,5 +567,14 @@ CLK_OF_DECLARE(arm_syscon_icst525_clk,
 	       "arm,syscon-icst525", of_syscon_icst_setup);
 CLK_OF_DECLARE(arm_syscon_icst307_clk,
 	       "arm,syscon-icst307", of_syscon_icst_setup);
-
+CLK_OF_DECLARE(arm_syscon_integratorap_cm_clk,
+	       "arm,syscon-icst525-integratorap-cm", of_syscon_icst_setup);
+CLK_OF_DECLARE(arm_syscon_integratorap_sys_clk,
+	       "arm,syscon-icst525-integratorap-sys", of_syscon_icst_setup);
+CLK_OF_DECLARE(arm_syscon_integratorap_pci_clk,
+	       "arm,syscon-icst525-integratorap-pci", of_syscon_icst_setup);
+CLK_OF_DECLARE(arm_syscon_integratorcp_cm_core_clk,
+	       "arm,syscon-icst525-integratorcp-cm-core", of_syscon_icst_setup);
+CLK_OF_DECLARE(arm_syscon_integratorcp_cm_mem_clk,
+	       "arm,syscon-icst525-integratorcp-cm-mem", of_syscon_icst_setup);
 #endif
diff --git a/drivers/clk/zte/Makefile b/drivers/clk/zte/Makefile
index 74005aa322a2..83374bfc4c07 100644
--- a/drivers/clk/zte/Makefile
+++ b/drivers/clk/zte/Makefile
@@ -1,2 +1,3 @@
 obj-y := clk.o
 obj-$(CONFIG_SOC_ZX296702) += clk-zx296702.o
+obj-$(CONFIG_ARCH_ZX) += clk-zx296718.o
diff --git a/drivers/clk/zte/clk-zx296718.c b/drivers/clk/zte/clk-zx296718.c
new file mode 100644
index 000000000000..707d62956e9b
--- /dev/null
+++ b/drivers/clk/zte/clk-zx296718.c
@@ -0,0 +1,924 @@
+/*
+ * Copyright (C) 2015 - 2016 ZTE Corporation.
+ * Copyright (C) 2016 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#include <dt-bindings/clock/zx296718-clock.h>
+#include "clk.h"
+
+/* TOP CRM */
+#define TOP_CLK_MUX0	0x04
+#define TOP_CLK_MUX1	0x08
+#define TOP_CLK_MUX2	0x0c
+#define TOP_CLK_MUX3	0x10
+#define TOP_CLK_MUX4	0x14
+#define TOP_CLK_MUX5	0x18
+#define TOP_CLK_MUX6	0x1c
+#define TOP_CLK_MUX7	0x20
+#define TOP_CLK_MUX9	0x28
+
+
+#define TOP_CLK_GATE0	0x34
+#define TOP_CLK_GATE1	0x38
+#define TOP_CLK_GATE2	0x3c
+#define TOP_CLK_GATE3	0x40
+#define TOP_CLK_GATE4	0x44
+#define TOP_CLK_GATE5	0x48
+#define TOP_CLK_GATE6	0x4c
+
+#define TOP_CLK_DIV0	0x58
+
+#define PLL_CPU_REG	0x80
+#define PLL_VGA_REG	0xb0
+#define PLL_DDR_REG	0xa0
+
+/* LSP0 CRM */
+#define LSP0_TIMER3_CLK	0x4
+#define LSP0_TIMER4_CLK	0x8
+#define LSP0_TIMER5_CLK	0xc
+#define LSP0_UART3_CLK	0x10
+#define LSP0_UART1_CLK	0x14
+#define LSP0_UART2_CLK	0x18
+#define LSP0_SPIFC0_CLK	0x1c
+#define LSP0_I2C4_CLK	0x20
+#define LSP0_I2C5_CLK	0x24
+#define LSP0_SSP0_CLK	0x28
+#define LSP0_SSP1_CLK	0x2c
+#define LSP0_USIM0_CLK	0x30
+#define LSP0_GPIO_CLK	0x34
+#define LSP0_I2C3_CLK	0x38
+
+/* LSP1 CRM */
+#define LSP1_UART4_CLK	0x08
+#define LSP1_UART5_CLK	0x0c
+#define LSP1_PWM_CLK	0x10
+#define LSP1_I2C2_CLK	0x14
+#define LSP1_SSP2_CLK	0x1c
+#define LSP1_SSP3_CLK	0x20
+#define LSP1_SSP4_CLK	0x24
+#define LSP1_USIM1_CLK	0x28
+
+/* audio lsp */
+#define AUDIO_I2S0_DIV_CFG1	0x10
+#define AUDIO_I2S0_DIV_CFG2	0x14
+#define AUDIO_I2S0_CLK		0x18
+#define AUDIO_I2S1_DIV_CFG1	0x20
+#define AUDIO_I2S1_DIV_CFG2	0x24
+#define AUDIO_I2S1_CLK		0x28
+#define AUDIO_I2S2_DIV_CFG1	0x30
+#define AUDIO_I2S2_DIV_CFG2	0x34
+#define AUDIO_I2S2_CLK		0x38
+#define AUDIO_I2S3_DIV_CFG1	0x40
+#define AUDIO_I2S3_DIV_CFG2	0x44
+#define AUDIO_I2S3_CLK		0x48
+#define AUDIO_I2C0_CLK		0x50
+#define AUDIO_SPDIF0_DIV_CFG1	0x60
+#define AUDIO_SPDIF0_DIV_CFG2	0x64
+#define AUDIO_SPDIF0_CLK	0x68
+#define AUDIO_SPDIF1_DIV_CFG1	0x70
+#define AUDIO_SPDIF1_DIV_CFG2	0x74
+#define AUDIO_SPDIF1_CLK	0x78
+#define AUDIO_TIMER_CLK		0x80
+#define AUDIO_TDM_CLK		0x90
+#define AUDIO_TS_CLK		0xa0
+
+static DEFINE_SPINLOCK(clk_lock);
+
+static struct zx_pll_config pll_cpu_table[] = {
+	PLL_RATE(1312000000, 0x00103621, 0x04aaaaaa),
+	PLL_RATE(1407000000, 0x00103a21, 0x04aaaaaa),
+	PLL_RATE(1503000000, 0x00103e21, 0x04aaaaaa),
+	PLL_RATE(1600000000, 0x00104221, 0x04aaaaaa),
+};
+
+PNAME(osc) = {
+	"osc24m",
+	"osc32k",
+};
+
+PNAME(dbg_wclk_p) = {
+	"clk334m",
+	"clk466m",
+	"clk396m",
+	"clk250m",
+};
+
+PNAME(a72_coreclk_p) = {
+	"osc24m",
+	"pll_mm0_1188m",
+	"pll_mm1_1296m",
+	"clk1000m",
+	"clk648m",
+	"clk1600m",
+	"pll_audio_1800m",
+	"pll_vga_1800m",
+};
+
+PNAME(cpu_periclk_p) = {
+	"osc24m",
+	"clk500m",
+	"clk594m",
+	"clk466m",
+	"clk294m",
+	"clk334m",
+	"clk250m",
+	"clk125m",
+};
+
+PNAME(a53_coreclk_p) = {
+	"osc24m",
+	"clk1000m",
+	"pll_mm0_1188m",
+	"clk648m",
+	"clk500m",
+	"clk800m",
+	"clk1600m",
+	"pll_audio_1800m",
+};
+
+PNAME(sec_wclk_p) = {
+	"osc24m",
+	"clk396m",
+	"clk334m",
+	"clk297m",
+	"clk250m",
+	"clk198m",
+	"clk148m5",
+	"clk99m",
+};
+
+PNAME(sd_nand_wclk_p) = {
+	"osc24m",
+	"clk49m5",
+	"clk99m",
+	"clk198m",
+	"clk167m",
+	"clk148m5",
+	"clk125m",
+	"clk216m",
+};
+
+PNAME(emmc_wclk_p) = {
+	"osc24m",
+	"clk198m",
+	"clk99m",
+	"clk396m",
+	"clk334m",
+	"clk297m",
+	"clk250m",
+	"clk148m5",
+};
+
+PNAME(clk32_p) = {
+	"osc32k",
+	"clk32k768",
+};
+
+PNAME(usb_ref24m_p) = {
+	"osc32k",
+	"clk32k768",
+};
+
+PNAME(sys_noc_alck_p) = {
+	"osc24m",
+	"clk250m",
+	"clk198m",
+	"clk148m5",
+	"clk108m",
+	"clk54m",
+	"clk216m",
+	"clk240m",
+};
+
+PNAME(vde_aclk_p) = {
+	"clk334m",
+	"clk594m",
+	"clk500m",
+	"clk432m",
+	"clk480m",
+	"clk297m",
+	"clk_vga",  /*600MHz*/
+	"clk294m",
+};
+
+PNAME(vce_aclk_p) = {
+	"clk334m",
+	"clk594m",
+	"clk500m",
+	"clk432m",
+	"clk396m",
+	"clk297m",
+	"clk_vga",  /*600MHz*/
+	"clk294m",
+};
+
+PNAME(hde_aclk_p) = {
+	"clk334m",
+	"clk594m",
+	"clk500m",
+	"clk432m",
+	"clk396m",
+	"clk297m",
+	"clk_vga",  /*600MHz*/
+	"clk294m",
+};
+
+PNAME(gpu_aclk_p) = {
+	"clk334m",
+	"clk648m",
+	"clk594m",
+	"clk500m",
+	"clk396m",
+	"clk297m",
+	"clk_vga",  /*600MHz*/
+	"clk294m",
+};
+
+PNAME(sappu_aclk_p) = {
+	"clk396m",
+	"clk500m",
+	"clk250m",
+	"clk148m5",
+};
+
+PNAME(sappu_wclk_p) = {
+	"clk198m",
+	"clk396m",
+	"clk334m",
+	"clk297m",
+	"clk250m",
+	"clk148m5",
+	"clk125m",
+	"clk99m",
+};
+
+PNAME(vou_aclk_p) = {
+	"clk334m",
+	"clk594m",
+	"clk500m",
+	"clk432m",
+	"clk396m",
+	"clk297m",
+	"clk_vga",  /*600MHz*/
+	"clk294m",
+};
+
+PNAME(vou_main_wclk_p) = {
+	"clk108m",
+	"clk594m",
+	"clk297m",
+	"clk148m5",
+	"clk74m25",
+	"clk54m",
+	"clk27m",
+	"clk_vga",
+};
+
+PNAME(vou_aux_wclk_p) = {
+	"clk108m",
+	"clk148m5",
+	"clk74m25",
+	"clk54m",
+	"clk27m",
+	"clk_vga",
+	"clk54m_mm0",
+	"clk"
+};
+
+PNAME(vou_ppu_wclk_p) = {
+	"clk334m",
+	"clk432m",
+	"clk396m",
+	"clk297m",
+	"clk250m",
+	"clk125m",
+	"clk198m",
+	"clk99m",
+};
+
+PNAME(vga_i2c_wclk_p) = {
+	"osc24m",
+	"clk99m",
+};
+
+PNAME(viu_m0_aclk_p) = {
+	"clk334m",
+	"clk432m",
+	"clk396m",
+	"clk297m",
+	"clk250m",
+	"clk125m",
+	"clk198m",
+	"osc24m",
+};
+
+PNAME(viu_m1_aclk_p) = {
+	"clk198m",
+	"clk250m",
+	"clk297m",
+	"clk125m",
+	"clk396m",
+	"clk334m",
+	"clk148m5",
+	"osc24m",
+};
+
+PNAME(viu_clk_p) = {
+	"clk198m",
+	"clk334m",
+	"clk297m",
+	"clk250m",
+	"clk396m",
+	"clk125m",
+	"clk99m",
+	"clk148m5",
+};
+
+PNAME(viu_jpeg_clk_p) = {
+	"clk334m",
+	"clk480m",
+	"clk432m",
+	"clk396m",
+	"clk297m",
+	"clk250m",
+	"clk125m",
+	"clk198m",
+};
+
+PNAME(ts_sys_clk_p) = {
+	"clk192m",
+	"clk167m",
+	"clk125m",
+	"clk99m",
+};
+
+PNAME(wdt_ares_p) = {
+	"osc24m",
+	"clk32k"
+};
+
+static struct clk_zx_pll zx296718_pll_clk[] = {
+	ZX296718_PLL("pll_cpu",	"osc24m",	PLL_CPU_REG,	pll_cpu_table),
+};
+
+static struct zx_clk_fixed_factor top_ffactor_clk[] = {
+	FFACTOR(0, "clk4m",		"osc24m", 1, 6,  0),
+	FFACTOR(0, "clk2m",		"osc24m", 1, 12, 0),
+	/* pll cpu */
+	FFACTOR(0, "clk1600m",		"pll_cpu", 1, 1, CLK_SET_RATE_PARENT),
+	FFACTOR(0, "clk800m",		"pll_cpu", 1, 2, CLK_SET_RATE_PARENT),
+	/* pll mac */
+	FFACTOR(0, "clk25m",		"pll_mac", 1, 40, 0),
+	FFACTOR(0, "clk125m",		"pll_mac", 1, 8, 0),
+	FFACTOR(0, "clk250m",		"pll_mac", 1, 4, 0),
+	FFACTOR(0, "clk50m",		"pll_mac", 1, 20, 0),
+	FFACTOR(0, "clk500m",		"pll_mac", 1, 2, 0),
+	FFACTOR(0, "clk1000m",		"pll_mac", 1, 1, 0),
+	FFACTOR(0, "clk334m",		"pll_mac", 1, 3, 0),
+	FFACTOR(0, "clk167m",		"pll_mac", 1, 6, 0),
+	/* pll mm */
+	FFACTOR(0, "clk54m_mm0",	"pll_mm0", 1, 22, 0),
+	FFACTOR(0, "clk74m25",		"pll_mm0", 1, 16, 0),
+	FFACTOR(0, "clk148m5",		"pll_mm0", 1, 8, 0),
+	FFACTOR(0, "clk297m",		"pll_mm0", 1, 4, 0),
+	FFACTOR(0, "clk594m",		"pll_mm0", 1, 2, 0),
+	FFACTOR(0, "pll_mm0_1188m",	"pll_mm0", 1, 1, 0),
+	FFACTOR(0, "clk396m",		"pll_mm0", 1, 3, 0),
+	FFACTOR(0, "clk198m",		"pll_mm0", 1, 6, 0),
+	FFACTOR(0, "clk99m",		"pll_mm0", 1, 12, 0),
+	FFACTOR(0, "clk49m5",		"pll_mm0", 1, 24, 0),
+	/* pll mm */
+	FFACTOR(0, "clk324m",		"pll_mm1", 1, 4, 0),
+	FFACTOR(0, "clk648m",		"pll_mm1", 1, 2, 0),
+	FFACTOR(0, "pll_mm1_1296m",	"pll_mm1", 1, 1, 0),
+	FFACTOR(0, "clk216m",		"pll_mm1", 1, 6, 0),
+	FFACTOR(0, "clk432m",		"pll_mm1", 1, 3, 0),
+	FFACTOR(0, "clk108m",		"pll_mm1", 1, 12, 0),
+	FFACTOR(0, "clk72m",		"pll_mm1", 1, 18, 0),
+	FFACTOR(0, "clk27m",		"pll_mm1", 1, 48, 0),
+	FFACTOR(0, "clk54m",		"pll_mm1", 1, 24, 0),
+	/* vga */
+	FFACTOR(0, "pll_vga_1800m",	"pll_vga", 1, 1, 0),
+	FFACTOR(0, "clk_vga",		"pll_vga", 1, 2, 0),
+	/* pll ddr */
+	FFACTOR(0, "clk466m",		"pll_ddr", 1, 2, 0),
+
+	/* pll audio */
+	FFACTOR(0, "pll_audio_1800m",	"pll_audio", 1, 1, 0),
+	FFACTOR(0, "clk32k768",		"pll_audio", 1, 27000, 0),
+	FFACTOR(0, "clk16m384",		"pll_audio", 1, 54, 0),
+	FFACTOR(0, "clk294m",		"pll_audio", 1, 3, 0),
+
+	/* pll hsic*/
+	FFACTOR(0, "clk240m",		"pll_hsic", 1, 4, 0),
+	FFACTOR(0, "clk480m",		"pll_hsic", 1, 2, 0),
+	FFACTOR(0, "clk192m",		"pll_hsic", 1, 5, 0),
+	FFACTOR(0, "clk_pll_24m",	"pll_hsic", 1, 40, 0),
+	FFACTOR(0, "emmc_mux_div2",	"emmc_mux", 1, 2, CLK_SET_RATE_PARENT),
+};
+
+static struct clk_div_table noc_div_table[] = {
+	{ .val = 1, .div = 2, },
+	{ .val = 3, .div = 4, },
+};
+static struct zx_clk_div top_div_clk[] = {
+	DIV_T(0, "sys_noc_hclk", "sys_noc_aclk", TOP_CLK_DIV0, 0, 2, 0, noc_div_table),
+	DIV_T(0, "sys_noc_pclk", "sys_noc_aclk", TOP_CLK_DIV0, 4, 2, 0, noc_div_table),
+};
+
+static struct zx_clk_mux top_mux_clk[] = {
+	MUX(0, "dbg_mux",	 dbg_wclk_p,	  TOP_CLK_MUX0, 12, 2),
+	MUX(0, "a72_mux",	 a72_coreclk_p,	  TOP_CLK_MUX0, 8, 3),
+	MUX(0, "cpu_peri_mux",	 cpu_periclk_p,	  TOP_CLK_MUX0, 4, 3),
+	MUX_F(0, "a53_mux",	 a53_coreclk_p,	  TOP_CLK_MUX0, 0, 3, CLK_SET_RATE_PARENT, 0),
+	MUX(0, "sys_noc_aclk",	 sys_noc_alck_p,  TOP_CLK_MUX1, 0, 3),
+	MUX(0, "sec_mux",	 sec_wclk_p,	  TOP_CLK_MUX2, 16, 3),
+	MUX(0, "sd1_mux",	 sd_nand_wclk_p,  TOP_CLK_MUX2, 12, 3),
+	MUX(0, "sd0_mux",	 sd_nand_wclk_p,  TOP_CLK_MUX2, 8, 3),
+	MUX(0, "emmc_mux",	 emmc_wclk_p,	  TOP_CLK_MUX2, 4, 3),
+	MUX(0, "nand_mux",	 sd_nand_wclk_p,  TOP_CLK_MUX2, 0, 3),
+	MUX(0, "usb_ref24m_mux", usb_ref24m_p,	  TOP_CLK_MUX9, 16, 1),
+	MUX(0, "clk32k",	 clk32_p,	  TOP_CLK_MUX9, 12, 1),
+	MUX_F(0, "wdt_mux",	 wdt_ares_p,	  TOP_CLK_MUX9, 8, 1, CLK_SET_RATE_PARENT, 0),
+	MUX(0, "timer_mux",	 osc,		  TOP_CLK_MUX9, 4, 1),
+	MUX(0, "vde_mux",	 vde_aclk_p,	  TOP_CLK_MUX4,  0, 3),
+	MUX(0, "vce_mux",	 vce_aclk_p,	  TOP_CLK_MUX4,  4, 3),
+	MUX(0, "hde_mux",	 hde_aclk_p,	  TOP_CLK_MUX4,  8, 3),
+	MUX(0, "gpu_mux",	 gpu_aclk_p,	  TOP_CLK_MUX5,  0, 3),
+	MUX(0, "sappu_a_mux",	 sappu_aclk_p,	  TOP_CLK_MUX5,  4, 2),
+	MUX(0, "sappu_w_mux",	 sappu_wclk_p,	  TOP_CLK_MUX5,  8, 3),
+	MUX(0, "vou_a_mux",	 vou_aclk_p,	  TOP_CLK_MUX7,  0, 3),
+	MUX(0, "vou_main_w_mux", vou_main_wclk_p, TOP_CLK_MUX7,  4, 3),
+	MUX(0, "vou_aux_w_mux",	 vou_aux_wclk_p,  TOP_CLK_MUX7,  8, 3),
+	MUX(0, "vou_ppu_w_mux",	 vou_ppu_wclk_p,  TOP_CLK_MUX7, 12, 3),
+	MUX(0, "vga_i2c_mux",	 vga_i2c_wclk_p,  TOP_CLK_MUX7, 16, 1),
+	MUX(0, "viu_m0_a_mux",	 viu_m0_aclk_p,	  TOP_CLK_MUX6,  0, 3),
+	MUX(0, "viu_m1_a_mux",	 viu_m1_aclk_p,	  TOP_CLK_MUX6,  4, 3),
+	MUX(0, "viu_w_mux",	 viu_clk_p,	  TOP_CLK_MUX6,  8, 3),
+	MUX(0, "viu_jpeg_w_mux", viu_jpeg_clk_p,  TOP_CLK_MUX6, 12, 3),
+	MUX(0, "ts_sys_mux",	 ts_sys_clk_p,    TOP_CLK_MUX6, 16, 2),
+};
+
+static struct zx_clk_gate top_gate_clk[] = {
+	GATE(CPU_DBG_GATE,    "dbg_wclk",        "dbg_mux",        TOP_CLK_GATE0, 4, CLK_SET_RATE_PARENT, 0),
+	GATE(A72_GATE,        "a72_coreclk",     "a72_mux",        TOP_CLK_GATE0, 3, CLK_SET_RATE_PARENT, 0),
+	GATE(CPU_PERI_GATE,   "cpu_peri",        "cpu_peri_mux",   TOP_CLK_GATE0, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(A53_GATE,        "a53_coreclk",     "a53_mux",        TOP_CLK_GATE0, 0, CLK_SET_RATE_PARENT, 0),
+	GATE(SD1_WCLK,        "sd1_wclk",        "sd1_mux",        TOP_CLK_GATE1, 13, CLK_SET_RATE_PARENT, 0),
+	GATE(SD0_WCLK,        "sd0_wclk",        "sd0_mux",        TOP_CLK_GATE1, 9, CLK_SET_RATE_PARENT, 0),
+	GATE(EMMC_WCLK,       "emmc_wclk",       "emmc_mux_div2",  TOP_CLK_GATE0, 5, CLK_SET_RATE_PARENT, 0),
+	GATE(EMMC_NAND_AXI,   "emmc_nand_aclk",  "sys_noc_aclk",   TOP_CLK_GATE1, 4, CLK_SET_RATE_PARENT, 0),
+	GATE(NAND_WCLK,       "nand_wclk",       "nand_mux",       TOP_CLK_GATE0, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(EMMC_NAND_AHB,   "emmc_nand_hclk",  "sys_noc_hclk",   TOP_CLK_GATE1, 0, CLK_SET_RATE_PARENT, 0),
+	GATE(0,               "lsp1_pclk",       "sys_noc_pclk",   TOP_CLK_GATE2, 31, 0,                  0),
+	GATE(LSP1_148M5,      "lsp1_148m5",      "clk148m5",       TOP_CLK_GATE2, 30, 0,                  0),
+	GATE(LSP1_99M,        "lsp1_99m",        "clk99m",         TOP_CLK_GATE2, 29, 0,                  0),
+	GATE(LSP1_24M,        "lsp1_24m",        "osc24m",         TOP_CLK_GATE2, 28, 0,                  0),
+	GATE(LSP0_74M25,      "lsp0_74m25",      "clk74m25",       TOP_CLK_GATE2, 25, 0,                  0),
+	GATE(0,               "lsp0_pclk",       "sys_noc_pclk",   TOP_CLK_GATE2, 24, 0,                  0),
+	GATE(LSP0_32K,        "lsp0_32k",        "osc32k",         TOP_CLK_GATE2, 23, 0,                  0),
+	GATE(LSP0_148M5,      "lsp0_148m5",      "clk148m5",       TOP_CLK_GATE2, 22, 0,                  0),
+	GATE(LSP0_99M,        "lsp0_99m",        "clk99m",         TOP_CLK_GATE2, 21, 0,                  0),
+	GATE(LSP0_24M,        "lsp0_24m",        "osc24m",         TOP_CLK_GATE2, 20, 0,                  0),
+	GATE(AUDIO_99M,       "audio_99m",       "clk99m",         TOP_CLK_GATE5, 27, 0,                  0),
+	GATE(AUDIO_24M,       "audio_24m",       "osc24m",         TOP_CLK_GATE5, 28, 0,                  0),
+	GATE(AUDIO_16M384,    "audio_16m384",    "clk16m384",      TOP_CLK_GATE5, 29, 0,                  0),
+	GATE(AUDIO_32K,       "audio_32k",       "clk32k",         TOP_CLK_GATE5, 30, 0,                  0),
+	GATE(WDT_WCLK,        "wdt_wclk",        "wdt_mux",        TOP_CLK_GATE6, 9, CLK_SET_RATE_PARENT, 0),
+	GATE(TIMER_WCLK,      "timer_wclk",      "timer_mux",      TOP_CLK_GATE6, 5, CLK_SET_RATE_PARENT, 0),
+	GATE(VDE_ACLK,        "vde_aclk",        "vde_mux",        TOP_CLK_GATE3, 0,  CLK_SET_RATE_PARENT, 0),
+	GATE(VCE_ACLK,        "vce_aclk",        "vce_mux",        TOP_CLK_GATE3, 4,  CLK_SET_RATE_PARENT, 0),
+	GATE(HDE_ACLK,        "hde_aclk",        "hde_mux",        TOP_CLK_GATE3, 8,  CLK_SET_RATE_PARENT, 0),
+	GATE(GPU_ACLK,        "gpu_aclk",        "gpu_mux",        TOP_CLK_GATE3, 16, CLK_SET_RATE_PARENT, 0),
+	GATE(SAPPU_ACLK,      "sappu_aclk",      "sappu_a_mux",    TOP_CLK_GATE3, 20, CLK_SET_RATE_PARENT, 0),
+	GATE(SAPPU_WCLK,      "sappu_wclk",      "sappu_w_mux",    TOP_CLK_GATE3, 22, CLK_SET_RATE_PARENT, 0),
+	GATE(VOU_ACLK,        "vou_aclk",        "vou_a_mux",      TOP_CLK_GATE4, 16, CLK_SET_RATE_PARENT, 0),
+	GATE(VOU_MAIN_WCLK,   "vou_main_wclk",   "vou_main_w_mux", TOP_CLK_GATE4, 18, CLK_SET_RATE_PARENT, 0),
+	GATE(VOU_AUX_WCLK,    "vou_aux_wclk",    "vou_aux_w_mux",  TOP_CLK_GATE4, 19, CLK_SET_RATE_PARENT, 0),
+	GATE(VOU_PPU_WCLK,    "vou_ppu_wclk",    "vou_ppu_w_mux",  TOP_CLK_GATE4, 20, CLK_SET_RATE_PARENT, 0),
+	GATE(MIPI_CFG_CLK,    "mipi_cfg_clk",    "osc24m",         TOP_CLK_GATE4, 21, 0,                   0),
+	GATE(VGA_I2C_WCLK,    "vga_i2c_wclk",    "vga_i2c_mux",    TOP_CLK_GATE4, 23, CLK_SET_RATE_PARENT, 0),
+	GATE(MIPI_REF_CLK,    "mipi_ref_clk",    "clk27m",         TOP_CLK_GATE4, 24, 0,                   0),
+	GATE(HDMI_OSC_CEC,    "hdmi_osc_cec",    "clk2m",          TOP_CLK_GATE4, 22, 0,                   0),
+	GATE(HDMI_OSC_CLK,    "hdmi_osc_clk",    "clk240m",        TOP_CLK_GATE4, 25, 0,                   0),
+	GATE(HDMI_XCLK,       "hdmi_xclk",       "osc24m",         TOP_CLK_GATE4, 26, 0,                   0),
+	GATE(VIU_M0_ACLK,     "viu_m0_aclk",     "viu_m0_a_mux",   TOP_CLK_GATE4, 0,  CLK_SET_RATE_PARENT, 0),
+	GATE(VIU_M1_ACLK,     "viu_m1_aclk",     "viu_m1_a_mux",   TOP_CLK_GATE4, 1,  CLK_SET_RATE_PARENT, 0),
+	GATE(VIU_WCLK,        "viu_wclk",        "viu_w_mux",      TOP_CLK_GATE4, 2,  CLK_SET_RATE_PARENT, 0),
+	GATE(VIU_JPEG_WCLK,   "viu_jpeg_wclk",   "viu_jpeg_w_mux", TOP_CLK_GATE4, 3,  CLK_SET_RATE_PARENT, 0),
+	GATE(VIU_CFG_CLK,     "viu_cfg_clk",     "osc24m",         TOP_CLK_GATE4, 6,  0,                   0),
+	GATE(TS_SYS_WCLK,     "ts_sys_wclk",     "ts_sys_mux",     TOP_CLK_GATE5, 2,  CLK_SET_RATE_PARENT, 0),
+	GATE(TS_SYS_108M,     "ts_sys_108m",     "clk108m",        TOP_CLK_GATE5, 3,  0,                   0),
+	GATE(USB20_HCLK,      "usb20_hclk",      "sys_noc_hclk",   TOP_CLK_GATE2, 12, 0,                   0),
+	GATE(USB20_PHY_CLK,   "usb20_phy_clk",   "usb_ref24m_mux", TOP_CLK_GATE2, 13, 0,                   0),
+	GATE(USB21_HCLK,      "usb21_hclk",      "sys_noc_hclk",   TOP_CLK_GATE2, 14, 0,                   0),
+	GATE(USB21_PHY_CLK,   "usb21_phy_clk",   "usb_ref24m_mux", TOP_CLK_GATE2, 15, 0,                   0),
+	GATE(GMAC_RMIICLK,    "gmac_rmii_clk",   "clk50m",         TOP_CLK_GATE2, 3, 0,                    0),
+	GATE(GMAC_PCLK,       "gmac_pclk",       "clk198m",        TOP_CLK_GATE2, 1, 0,                    0),
+	GATE(GMAC_ACLK,       "gmac_aclk",       "clk49m5",        TOP_CLK_GATE2, 0, 0,                    0),
+	GATE(GMAC_RFCLK,      "gmac_refclk",     "clk25m",         TOP_CLK_GATE2, 4, 0,                    0),
+	GATE(SD1_AHB,         "sd1_hclk",        "sys_noc_hclk",   TOP_CLK_GATE1, 12,  0,                  0),
+	GATE(SD0_AHB,         "sd0_hclk",        "sys_noc_hclk",   TOP_CLK_GATE1, 8,  0,                   0),
+	GATE(TEMPSENSOR_GATE, "tempsensor_gate", "clk4m",          TOP_CLK_GATE5, 31,  0,                  0),
+};
+
+static struct clk_hw_onecell_data top_hw_onecell_data = {
+	.num = TOP_NR_CLKS,
+	.hws = {
+		[TOP_NR_CLKS - 1] = NULL,
+	},
+};
+
+static int __init top_clocks_init(struct device_node *np)
+{
+	void __iomem *reg_base;
+	int i, ret;
+
+	reg_base = of_iomap(np, 0);
+	if (!reg_base) {
+		pr_err("%s: Unable to map clk base\n", __func__);
+		return -ENXIO;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(zx296718_pll_clk); i++) {
+		zx296718_pll_clk[i].reg_base += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &zx296718_pll_clk[i].hw);
+		if (ret) {
+			pr_warn("top clk %s init error!\n",
+				zx296718_pll_clk[i].hw.init->name);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(top_ffactor_clk); i++) {
+		if (top_ffactor_clk[i].id)
+			top_hw_onecell_data.hws[top_ffactor_clk[i].id] =
+					&top_ffactor_clk[i].factor.hw;
+
+		ret = clk_hw_register(NULL, &top_ffactor_clk[i].factor.hw);
+		if (ret) {
+			pr_warn("top clk %s init error!\n",
+				top_ffactor_clk[i].factor.hw.init->name);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(top_mux_clk); i++) {
+		if (top_mux_clk[i].id)
+			top_hw_onecell_data.hws[top_mux_clk[i].id] =
+					&top_mux_clk[i].mux.hw;
+
+		top_mux_clk[i].mux.reg += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &top_mux_clk[i].mux.hw);
+		if (ret) {
+			pr_warn("top clk %s init error!\n",
+				top_mux_clk[i].mux.hw.init->name);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(top_gate_clk); i++) {
+		if (top_gate_clk[i].id)
+			top_hw_onecell_data.hws[top_gate_clk[i].id] =
+					&top_gate_clk[i].gate.hw;
+
+		top_gate_clk[i].gate.reg += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &top_gate_clk[i].gate.hw);
+		if (ret) {
+			pr_warn("top clk %s init error!\n",
+				top_gate_clk[i].gate.hw.init->name);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(top_div_clk); i++) {
+		if (top_div_clk[i].id)
+			top_hw_onecell_data.hws[top_div_clk[i].id] =
+					&top_div_clk[i].div.hw;
+
+		top_div_clk[i].div.reg += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &top_div_clk[i].div.hw);
+		if (ret) {
+			pr_warn("top clk %s init error!\n",
+				top_div_clk[i].div.hw.init->name);
+		}
+	}
+
+	if (of_clk_add_hw_provider(np, of_clk_hw_onecell_get, &top_hw_onecell_data))
+		panic("could not register clk provider\n");
+	pr_info("top clk init over, nr:%d\n", TOP_NR_CLKS);
+
+	return 0;
+}
+
+static struct clk_div_table common_even_div_table[] = {
+	{ .val = 0, .div = 1, },
+	{ .val = 1, .div = 2, },
+	{ .val = 3, .div = 4, },
+	{ .val = 5, .div = 6, },
+	{ .val = 7, .div = 8, },
+	{ .val = 9, .div = 10, },
+	{ .val = 11, .div = 12, },
+	{ .val = 13, .div = 14, },
+	{ .val = 15, .div = 16, },
+};
+
+static struct clk_div_table common_div_table[] = {
+	{ .val = 0, .div = 1, },
+	{ .val = 1, .div = 2, },
+	{ .val = 2, .div = 3, },
+	{ .val = 3, .div = 4, },
+	{ .val = 4, .div = 5, },
+	{ .val = 5, .div = 6, },
+	{ .val = 6, .div = 7, },
+	{ .val = 7, .div = 8, },
+	{ .val = 8, .div = 9, },
+	{ .val = 9, .div = 10, },
+	{ .val = 10, .div = 11, },
+	{ .val = 11, .div = 12, },
+	{ .val = 12, .div = 13, },
+	{ .val = 13, .div = 14, },
+	{ .val = 14, .div = 15, },
+	{ .val = 15, .div = 16, },
+};
+
+PNAME(lsp0_wclk_common_p) = {
+	"lsp0_24m",
+	"lsp0_99m",
+};
+
+PNAME(lsp0_wclk_timer3_p) = {
+	"timer3_div",
+	"lsp0_32k"
+};
+
+PNAME(lsp0_wclk_timer4_p) = {
+	"timer4_div",
+	"lsp0_32k"
+};
+
+PNAME(lsp0_wclk_timer5_p) = {
+	"timer5_div",
+	"lsp0_32k"
+};
+
+PNAME(lsp0_wclk_spifc0_p) = {
+	"lsp0_148m5",
+	"lsp0_24m",
+	"lsp0_99m",
+	"lsp0_74m25"
+};
+
+PNAME(lsp0_wclk_ssp_p) = {
+	"lsp0_148m5",
+	"lsp0_99m",
+	"lsp0_24m",
+};
+
+static struct zx_clk_mux lsp0_mux_clk[] = {
+	MUX(0, "timer3_wclk_mux", lsp0_wclk_timer3_p, LSP0_TIMER3_CLK, 4, 1),
+	MUX(0, "timer4_wclk_mux", lsp0_wclk_timer4_p, LSP0_TIMER4_CLK, 4, 1),
+	MUX(0, "timer5_wclk_mux", lsp0_wclk_timer5_p, LSP0_TIMER5_CLK, 4, 1),
+	MUX(0, "uart3_wclk_mux",  lsp0_wclk_common_p, LSP0_UART3_CLK,  4, 1),
+	MUX(0, "uart1_wclk_mux",  lsp0_wclk_common_p, LSP0_UART1_CLK,  4, 1),
+	MUX(0, "uart2_wclk_mux",  lsp0_wclk_common_p, LSP0_UART2_CLK,  4, 1),
+	MUX(0, "spifc0_wclk_mux", lsp0_wclk_spifc0_p, LSP0_SPIFC0_CLK, 4, 2),
+	MUX(0, "i2c4_wclk_mux",   lsp0_wclk_common_p, LSP0_I2C4_CLK,   4, 1),
+	MUX(0, "i2c5_wclk_mux",   lsp0_wclk_common_p, LSP0_I2C5_CLK,   4, 1),
+	MUX(0, "ssp0_wclk_mux",   lsp0_wclk_ssp_p,    LSP0_SSP0_CLK,   4, 1),
+	MUX(0, "ssp1_wclk_mux",   lsp0_wclk_ssp_p,    LSP0_SSP1_CLK,   4, 1),
+	MUX(0, "i2c3_wclk_mux",   lsp0_wclk_common_p, LSP0_I2C3_CLK,   4, 1),
+};
+
+static struct zx_clk_gate lsp0_gate_clk[] = {
+	GATE(LSP0_TIMER3_WCLK, "timer3_wclk", "timer3_wclk_mux", LSP0_TIMER3_CLK, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_TIMER4_WCLK, "timer4_wclk", "timer4_wclk_mux", LSP0_TIMER4_CLK, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_TIMER5_WCLK, "timer5_wclk", "timer5_wclk_mux", LSP0_TIMER5_CLK, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_UART3_WCLK,  "uart3_wclk",  "uart3_wclk_mux",  LSP0_UART3_CLK,  1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_UART1_WCLK,  "uart1_wclk",  "uart1_wclk_mux",  LSP0_UART1_CLK,  1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_UART2_WCLK,  "uart2_wclk",  "uart2_wclk_mux",  LSP0_UART2_CLK,  1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_SPIFC0_WCLK, "spifc0_wclk", "spifc0_wclk_mux", LSP0_SPIFC0_CLK, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_I2C4_WCLK,   "i2c4_wclk",   "i2c4_wclk_mux",   LSP0_I2C4_CLK,   1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_I2C5_WCLK,   "i2c5_wclk",   "i2c5_wclk_mux",   LSP0_I2C5_CLK,   1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_SSP0_WCLK,   "ssp0_wclk",   "ssp0_div",        LSP0_SSP0_CLK,   1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_SSP1_WCLK,   "ssp1_wclk",   "ssp1_div",        LSP0_SSP1_CLK,   1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP0_I2C3_WCLK,   "i2c3_wclk",   "i2c3_wclk_mux",   LSP0_I2C3_CLK,   1, CLK_SET_RATE_PARENT, 0),
+};
+
+static struct zx_clk_div lsp0_div_clk[] = {
+	DIV_T(0, "timer3_div", "lsp0_24m", LSP0_TIMER3_CLK,  12, 4, 0, common_even_div_table),
+	DIV_T(0, "timer4_div", "lsp0_24m", LSP0_TIMER4_CLK,  12, 4, 0, common_even_div_table),
+	DIV_T(0, "timer5_div", "lsp0_24m", LSP0_TIMER5_CLK,  12, 4, 0, common_even_div_table),
+	DIV_T(0, "ssp0_div", "ssp0_wclk_mux", LSP0_SSP0_CLK, 12, 4, 0, common_even_div_table),
+	DIV_T(0, "ssp1_div", "ssp1_wclk_mux", LSP0_SSP1_CLK, 12, 4, 0, common_even_div_table),
+};
+
+static struct clk_hw_onecell_data lsp0_hw_onecell_data = {
+	.num = LSP0_NR_CLKS,
+	.hws = {
+		[LSP0_NR_CLKS - 1] = NULL,
+	},
+};
+
+static int __init lsp0_clocks_init(struct device_node *np)
+{
+	void __iomem *reg_base;
+	int i, ret;
+
+	reg_base = of_iomap(np, 0);
+	if (!reg_base) {
+		pr_err("%s: Unable to map clk base\n", __func__);
+		return -ENXIO;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(lsp0_mux_clk); i++) {
+		if (lsp0_mux_clk[i].id)
+			lsp0_hw_onecell_data.hws[lsp0_mux_clk[i].id] =
+					&lsp0_mux_clk[i].mux.hw;
+
+		lsp0_mux_clk[i].mux.reg += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &lsp0_mux_clk[i].mux.hw);
+		if (ret) {
+			pr_warn("lsp0 clk %s init error!\n",
+				lsp0_mux_clk[i].mux.hw.init->name);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(lsp0_gate_clk); i++) {
+		if (lsp0_gate_clk[i].id)
+			lsp0_hw_onecell_data.hws[lsp0_gate_clk[i].id] =
+					&lsp0_gate_clk[i].gate.hw;
+
+		lsp0_gate_clk[i].gate.reg += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &lsp0_gate_clk[i].gate.hw);
+		if (ret) {
+			pr_warn("lsp0 clk %s init error!\n",
+				lsp0_gate_clk[i].gate.hw.init->name);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(lsp0_div_clk); i++) {
+		if (lsp0_div_clk[i].id)
+			lsp0_hw_onecell_data.hws[lsp0_div_clk[i].id] =
+					&lsp0_div_clk[i].div.hw;
+
+		lsp0_div_clk[i].div.reg += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &lsp0_div_clk[i].div.hw);
+		if (ret) {
+			pr_warn("lsp0 clk %s init error!\n",
+				lsp0_div_clk[i].div.hw.init->name);
+		}
+	}
+
+	if (of_clk_add_hw_provider(np, of_clk_hw_onecell_get, &lsp0_hw_onecell_data))
+		panic("could not register clk provider\n");
+	pr_info("lsp0-clk init over:%d\n", LSP0_NR_CLKS);
+
+	return 0;
+}
+
+PNAME(lsp1_wclk_common_p) = {
+	"lsp1_24m",
+	"lsp1_99m",
+};
+
+PNAME(lsp1_wclk_ssp_p) = {
+	"lsp1_148m5",
+	"lsp1_99m",
+	"lsp1_24m",
+};
+
+static struct zx_clk_mux lsp1_mux_clk[] = {
+	MUX(0, "uart4_wclk_mux", lsp1_wclk_common_p, LSP1_UART4_CLK, 4, 1),
+	MUX(0, "uart5_wclk_mux", lsp1_wclk_common_p, LSP1_UART5_CLK, 4, 1),
+	MUX(0, "pwm_wclk_mux",   lsp1_wclk_common_p, LSP1_PWM_CLK,   4, 1),
+	MUX(0, "i2c2_wclk_mux",  lsp1_wclk_common_p, LSP1_I2C2_CLK,  4, 1),
+	MUX(0, "ssp2_wclk_mux",  lsp1_wclk_ssp_p,    LSP1_SSP2_CLK,  4, 2),
+	MUX(0, "ssp3_wclk_mux",  lsp1_wclk_ssp_p,    LSP1_SSP3_CLK,  4, 2),
+	MUX(0, "ssp4_wclk_mux",  lsp1_wclk_ssp_p,    LSP1_SSP4_CLK,  4, 2),
+	MUX(0, "usim1_wclk_mux", lsp1_wclk_common_p, LSP1_USIM1_CLK, 4, 1),
+};
+
+static struct zx_clk_div lsp1_div_clk[] = {
+	DIV_T(0, "pwm_div",  "pwm_wclk_mux",  LSP1_PWM_CLK,  12, 4, CLK_SET_RATE_PARENT, common_div_table),
+	DIV_T(0, "ssp2_div", "ssp2_wclk_mux", LSP1_SSP2_CLK, 12, 4, CLK_SET_RATE_PARENT, common_even_div_table),
+	DIV_T(0, "ssp3_div", "ssp3_wclk_mux", LSP1_SSP3_CLK, 12, 4, CLK_SET_RATE_PARENT, common_even_div_table),
+	DIV_T(0, "ssp4_div", "ssp4_wclk_mux", LSP1_SSP4_CLK, 12, 4, CLK_SET_RATE_PARENT, common_even_div_table),
+};
+
+static struct zx_clk_gate lsp1_gate_clk[] = {
+	GATE(LSP1_UART4_WCLK, "lsp1_uart4_wclk", "uart4_wclk_mux", LSP1_UART4_CLK, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP1_UART5_WCLK, "lsp1_uart5_wclk", "uart5_wclk_mux", LSP1_UART5_CLK, 1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP1_PWM_WCLK,   "lsp1_pwm_wclk",   "pwm_div",        LSP1_PWM_CLK,   1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP1_PWM_PCLK,   "lsp1_pwm_pclk",   "lsp1_pclk",      LSP1_PWM_CLK,   0, 0,		   0),
+	GATE(LSP1_I2C2_WCLK,  "lsp1_i2c2_wclk",  "i2c2_wclk_mux",  LSP1_I2C2_CLK,  1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP1_SSP2_WCLK,  "lsp1_ssp2_wclk",  "ssp2_div",       LSP1_SSP2_CLK,  1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP1_SSP3_WCLK,  "lsp1_ssp3_wclk",  "ssp3_div",       LSP1_SSP3_CLK,  1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP1_SSP4_WCLK,  "lsp1_ssp4_wclk",  "ssp4_div",       LSP1_SSP4_CLK,  1, CLK_SET_RATE_PARENT, 0),
+	GATE(LSP1_USIM1_WCLK, "lsp1_usim1_wclk", "usim1_wclk_mux", LSP1_USIM1_CLK, 1, CLK_SET_RATE_PARENT, 0),
+};
+
+static struct clk_hw_onecell_data lsp1_hw_onecell_data = {
+	.num = LSP1_NR_CLKS,
+	.hws = {
+		[LSP1_NR_CLKS - 1] = NULL,
+	},
+};
+
+static int __init lsp1_clocks_init(struct device_node *np)
+{
+	void __iomem *reg_base;
+	int i, ret;
+
+	reg_base = of_iomap(np, 0);
+	if (!reg_base) {
+		pr_err("%s: Unable to map clk base\n", __func__);
+		return -ENXIO;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(lsp1_mux_clk); i++) {
+		if (lsp1_mux_clk[i].id)
+			lsp1_hw_onecell_data.hws[lsp1_mux_clk[i].id] =
+					&lsp0_mux_clk[i].mux.hw;
+
+		lsp1_mux_clk[i].mux.reg += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &lsp1_mux_clk[i].mux.hw);
+		if (ret) {
+			pr_warn("lsp1 clk %s init error!\n",
+				lsp1_mux_clk[i].mux.hw.init->name);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(lsp1_gate_clk); i++) {
+		if (lsp1_gate_clk[i].id)
+			lsp1_hw_onecell_data.hws[lsp1_gate_clk[i].id] =
+					&lsp1_gate_clk[i].gate.hw;
+
+		lsp1_gate_clk[i].gate.reg += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &lsp1_gate_clk[i].gate.hw);
+		if (ret) {
+			pr_warn("lsp1 clk %s init error!\n",
+				lsp1_gate_clk[i].gate.hw.init->name);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(lsp1_div_clk); i++) {
+		if (lsp1_div_clk[i].id)
+			lsp1_hw_onecell_data.hws[lsp1_div_clk[i].id] =
+					&lsp1_div_clk[i].div.hw;
+
+		lsp1_div_clk[i].div.reg += (uintptr_t)reg_base;
+		ret = clk_hw_register(NULL, &lsp1_div_clk[i].div.hw);
+		if (ret) {
+			pr_warn("lsp1 clk %s init error!\n",
+				lsp1_div_clk[i].div.hw.init->name);
+		}
+	}
+
+	if (of_clk_add_hw_provider(np, of_clk_hw_onecell_get, &lsp1_hw_onecell_data))
+		panic("could not register clk provider\n");
+	pr_info("lsp1-clk init over, nr:%d\n", LSP1_NR_CLKS);
+
+	return 0;
+}
+
+static const struct of_device_id zx_clkc_match_table[] = {
+	{ .compatible = "zte,zx296718-topcrm", .data = &top_clocks_init },
+	{ .compatible = "zte,zx296718-lsp0crm", .data = &lsp0_clocks_init },
+	{ .compatible = "zte,zx296718-lsp1crm", .data = &lsp1_clocks_init },
+	{ }
+};
+
+static int zx_clkc_probe(struct platform_device *pdev)
+{
+	int (*init_fn)(struct device_node *np);
+	struct device_node *np = pdev->dev.of_node;
+
+	init_fn = of_device_get_match_data(&pdev->dev);
+	if (!init_fn) {
+		dev_err(&pdev->dev, "Error: No device match found\n");
+		return -ENODEV;
+	}
+
+	return init_fn(np);
+}
+
+static struct platform_driver zx_clk_driver = {
+	.probe		= zx_clkc_probe,
+	.driver		= {
+		.name	= "zx296718-clkc",
+		.of_match_table = zx_clkc_match_table,
+	},
+};
+
+static int __init zx_clk_init(void)
+{
+	return platform_driver_register(&zx_clk_driver);
+}
+core_initcall(zx_clk_init);
diff --git a/drivers/clk/zte/clk.c b/drivers/clk/zte/clk.c
index 7c73c538c43d..c4c1251bc1e7 100644
--- a/drivers/clk/zte/clk.c
+++ b/drivers/clk/zte/clk.c
@@ -21,8 +21,8 @@
 #define to_clk_zx_audio(_hw) container_of(_hw, struct clk_zx_audio, hw)
 
 #define CFG0_CFG1_OFFSET 4
-#define LOCK_FLAG BIT(30)
-#define POWER_DOWN BIT(31)
+#define LOCK_FLAG 30
+#define POWER_DOWN 31
 
 static int rate_to_idx(struct clk_zx_pll *zx_pll, unsigned long rate)
 {
@@ -50,8 +50,8 @@ static int hw_to_idx(struct clk_zx_pll *zx_pll)
 	hw_cfg1 = readl_relaxed(zx_pll->reg_base + CFG0_CFG1_OFFSET);
 
 	/* For matching the value in lookup table */
-	hw_cfg0 &= ~LOCK_FLAG;
-	hw_cfg0 |= POWER_DOWN;
+	hw_cfg0 &= ~BIT(zx_pll->lock_bit);
+	hw_cfg0 |= BIT(zx_pll->pd_bit);
 
 	for (i = 0; i < zx_pll->count; i++) {
 		if (hw_cfg0 == config[i].cfg0 && hw_cfg1 == config[i].cfg1)
@@ -108,10 +108,10 @@ static int zx_pll_enable(struct clk_hw *hw)
 	u32 reg;
 
 	reg = readl_relaxed(zx_pll->reg_base);
-	writel_relaxed(reg & ~POWER_DOWN, zx_pll->reg_base);
+	writel_relaxed(reg & ~BIT(zx_pll->pd_bit), zx_pll->reg_base);
 
 	return readl_relaxed_poll_timeout(zx_pll->reg_base, reg,
-					  reg & LOCK_FLAG, 0, 100);
+					  reg & BIT(zx_pll->lock_bit), 0, 100);
 }
 
 static void zx_pll_disable(struct clk_hw *hw)
@@ -120,7 +120,7 @@ static void zx_pll_disable(struct clk_hw *hw)
 	u32 reg;
 
 	reg = readl_relaxed(zx_pll->reg_base);
-	writel_relaxed(reg | POWER_DOWN, zx_pll->reg_base);
+	writel_relaxed(reg | BIT(zx_pll->pd_bit), zx_pll->reg_base);
 }
 
 static int zx_pll_is_enabled(struct clk_hw *hw)
@@ -130,10 +130,10 @@ static int zx_pll_is_enabled(struct clk_hw *hw)
 
 	reg = readl_relaxed(zx_pll->reg_base);
 
-	return !(reg & POWER_DOWN);
+	return !(reg & BIT(zx_pll->pd_bit));
 }
 
-static const struct clk_ops zx_pll_ops = {
+const struct clk_ops zx_pll_ops = {
 	.recalc_rate = zx_pll_recalc_rate,
 	.round_rate = zx_pll_round_rate,
 	.set_rate = zx_pll_set_rate,
@@ -141,6 +141,7 @@ static const struct clk_ops zx_pll_ops = {
 	.disable = zx_pll_disable,
 	.is_enabled = zx_pll_is_enabled,
 };
+EXPORT_SYMBOL(zx_pll_ops);
 
 struct clk *clk_register_zx_pll(const char *name, const char *parent_name,
 				unsigned long flags, void __iomem *reg_base,
@@ -164,6 +165,8 @@ struct clk *clk_register_zx_pll(const char *name, const char *parent_name,
 	zx_pll->reg_base = reg_base;
 	zx_pll->lookup_table = lookup_table;
 	zx_pll->count = count;
+	zx_pll->lock_bit = LOCK_FLAG;
+	zx_pll->pd_bit = POWER_DOWN;
 	zx_pll->lock = lock;
 	zx_pll->hw.init = &init;
 
diff --git a/drivers/clk/zte/clk.h b/drivers/clk/zte/clk.h
index 65ae08b818d3..0df3474b2cf3 100644
--- a/drivers/clk/zte/clk.h
+++ b/drivers/clk/zte/clk.h
@@ -12,6 +12,26 @@
 #include <linux/clk-provider.h>
 #include <linux/spinlock.h>
 
+#define PNAME(x) static const char *x[]
+
+#define CLK_HW_INIT(_name, _parent, _ops, _flags)			\
+	&(struct clk_init_data) {					\
+		.flags		= _flags,				\
+		.name		= _name,				\
+		.parent_names	= (const char *[]) { _parent },		\
+		.num_parents	= 1,					\
+		.ops		= _ops,					\
+	}
+
+#define CLK_HW_INIT_PARENTS(_name, _parents, _ops, _flags)		\
+	&(struct clk_init_data) {					\
+		.flags		= _flags,				\
+		.name		= _name,				\
+		.parent_names	= _parents,				\
+		.num_parents	= ARRAY_SIZE(_parents),			\
+		.ops		= _ops,					\
+	}
+
 struct zx_pll_config {
 	unsigned long rate;
 	u32 cfg0;
@@ -24,8 +44,115 @@ struct clk_zx_pll {
 	const struct zx_pll_config *lookup_table; /* order by rate asc */
 	int count;
 	spinlock_t *lock;
+	u8 pd_bit;		/* power down bit */
+	u8 lock_bit;		/* pll lock flag bit */
+};
+
+#define PLL_RATE(_rate, _cfg0, _cfg1)	\
+{					\
+	.rate = _rate,			\
+	.cfg0 = _cfg0,			\
+	.cfg1 = _cfg1,			\
+}
+
+#define ZX_PLL(_name, _parent, _reg, _table, _pd, _lock)		\
+{									\
+	.reg_base	= (void __iomem *) _reg,			\
+	.lookup_table	= _table,					\
+	.count		= ARRAY_SIZE(_table),				\
+	.pd_bit		= _pd,						\
+	.lock_bit	= _lock,					\
+	.hw.init	 = CLK_HW_INIT(_name, _parent, &zx_pll_ops,	\
+				CLK_GET_RATE_NOCACHE),			\
+}
+
+#define ZX296718_PLL(_name, _parent, _reg, _table)			\
+ZX_PLL(_name, _parent, _reg, _table, 0, 30)
+
+struct zx_clk_gate {
+	struct clk_gate gate;
+	u16		id;
+};
+
+#define GATE(_id, _name, _parent, _reg, _bit, _flag, _gflags)		\
+{									\
+	.gate = {							\
+		.reg = (void __iomem *) _reg,				\
+		.bit_idx = (_bit),					\
+		.flags = _gflags,					\
+		.lock = &clk_lock,					\
+		.hw.init = CLK_HW_INIT(_name,				\
+					_parent,			\
+					&clk_gate_ops,			\
+					_flag | CLK_IGNORE_UNUSED),	\
+	},								\
+	.id	= _id,							\
+}
+
+struct zx_clk_fixed_factor {
+	struct clk_fixed_factor factor;
+	u16	id;
+};
+
+#define FFACTOR(_id, _name, _parent, _mult, _div, _flag)		\
+{									\
+	.factor = {							\
+		.div		= _div,					\
+		.mult		= _mult,				\
+		.hw.init	= CLK_HW_INIT(_name,			\
+					      _parent,			\
+					      &clk_fixed_factor_ops,	\
+					      _flag),			\
+	},								\
+	.id = _id,							\
+}
+
+struct zx_clk_mux {
+	struct clk_mux mux;
+	u16	id;
+};
+
+#define MUX_F(_id, _name, _parent, _reg, _shift, _width, _flag, _mflag)	\
+{									\
+	.mux = {							\
+		.reg		= (void __iomem *) _reg,		\
+		.mask		= BIT(_width) - 1,			\
+		.shift		= _shift,				\
+		.flags		= _mflag,				\
+		.lock		= &clk_lock,				\
+		.hw.init	= CLK_HW_INIT_PARENTS(_name,		\
+						      _parent,		\
+						      &clk_mux_ops,	\
+						      _flag),		\
+	},								\
+	.id = _id,							\
+}
+
+#define MUX(_id, _name, _parent, _reg, _shift, _width)			\
+MUX_F(_id, _name, _parent, _reg, _shift, _width, 0, 0)
+
+struct zx_clk_div {
+	struct clk_divider div;
+	u16	id;
 };
 
+#define DIV_T(_id, _name, _parent, _reg, _shift, _width, _flag, _table)	\
+{									\
+	.div = {							\
+		.reg		= (void __iomem *) _reg,		\
+		.shift		= _shift,				\
+		.width		= _width,				\
+		.flags		= 0,					\
+		.table		= _table,				\
+		.lock		= &clk_lock,				\
+		.hw.init	= CLK_HW_INIT(_name,			\
+					      _parent,			\
+					      &clk_divider_ops,		\
+					      _flag),			\
+	},								\
+	.id = _id,							\
+}
+
 struct clk *clk_register_zx_pll(const char *name, const char *parent_name,
 	unsigned long flags, void __iomem *reg_base,
 	const struct zx_pll_config *lookup_table, int count, spinlock_t *lock);
@@ -38,4 +165,6 @@ struct clk_zx_audio {
 struct clk *clk_register_zx_audio(const char *name,
 				  const char * const parent_name,
 				  unsigned long flags, void __iomem *reg_base);
+
+extern const struct clk_ops zx_pll_ops;
 #endif
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index 567788664723..245190839359 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -305,6 +305,16 @@ config ARM_ARCH_TIMER_EVTSTREAM
 	  This must be disabled for hardware validation purposes to detect any
 	  hardware anomalies of missing events.
 
+config FSL_ERRATUM_A008585
+	bool "Workaround for Freescale/NXP Erratum A-008585"
+	default y
+	depends on ARM_ARCH_TIMER && ARM64
+	help
+	  This option enables a workaround for Freescale/NXP Erratum
+	  A-008585 ("ARM generic timer may contain an erroneous
+	  value").  The workaround will only be active if the
+	  fsl,erratum-a008585 property is found in the timer node.
+
 config ARM_GLOBAL_TIMER
 	bool "Support for the ARM global timer" if COMPILE_TEST
 	select CLKSRC_OF if OF
@@ -351,7 +361,7 @@ config CLKSRC_METAG_GENERIC
 
 config CLKSRC_EXYNOS_MCT
 	bool "Exynos multi core timer driver" if COMPILE_TEST
-	depends on ARM
+	depends on ARM || ARM64
 	help
 	  Support for Multi Core Timer controller on Exynos SoCs.
 
diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
index 28bce3f4f81d..73c487da6d2a 100644
--- a/drivers/clocksource/arm_arch_timer.c
+++ b/drivers/clocksource/arm_arch_timer.c
@@ -8,6 +8,9 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
+
+#define pr_fmt(fmt)	"arm_arch_timer: " fmt
+
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/device.h>
@@ -91,6 +94,43 @@ early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
  * Architected system timer support.
  */
 
+#ifdef CONFIG_FSL_ERRATUM_A008585
+DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
+EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
+
+static int fsl_a008585_enable = -1;
+
+static int __init early_fsl_a008585_cfg(char *buf)
+{
+	int ret;
+	bool val;
+
+	ret = strtobool(buf, &val);
+	if (ret)
+		return ret;
+
+	fsl_a008585_enable = val;
+	return 0;
+}
+early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg);
+
+u32 __fsl_a008585_read_cntp_tval_el0(void)
+{
+	return __fsl_a008585_read_reg(cntp_tval_el0);
+}
+
+u32 __fsl_a008585_read_cntv_tval_el0(void)
+{
+	return __fsl_a008585_read_reg(cntv_tval_el0);
+}
+
+u64 __fsl_a008585_read_cntvct_el0(void)
+{
+	return __fsl_a008585_read_reg(cntvct_el0);
+}
+EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0);
+#endif /* CONFIG_FSL_ERRATUM_A008585 */
+
 static __always_inline
 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
 			  struct clock_event_device *clk)
@@ -240,6 +280,40 @@ static __always_inline void set_next_event(const int access, unsigned long evt,
 	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 }
 
+#ifdef CONFIG_FSL_ERRATUM_A008585
+static __always_inline void fsl_a008585_set_next_event(const int access,
+		unsigned long evt, struct clock_event_device *clk)
+{
+	unsigned long ctrl;
+	u64 cval = evt + arch_counter_get_cntvct();
+
+	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
+	ctrl |= ARCH_TIMER_CTRL_ENABLE;
+	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
+
+	if (access == ARCH_TIMER_PHYS_ACCESS)
+		write_sysreg(cval, cntp_cval_el0);
+	else if (access == ARCH_TIMER_VIRT_ACCESS)
+		write_sysreg(cval, cntv_cval_el0);
+
+	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
+}
+
+static int fsl_a008585_set_next_event_virt(unsigned long evt,
+					   struct clock_event_device *clk)
+{
+	fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
+	return 0;
+}
+
+static int fsl_a008585_set_next_event_phys(unsigned long evt,
+					   struct clock_event_device *clk)
+{
+	fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
+	return 0;
+}
+#endif /* CONFIG_FSL_ERRATUM_A008585 */
+
 static int arch_timer_set_next_event_virt(unsigned long evt,
 					  struct clock_event_device *clk)
 {
@@ -268,6 +342,19 @@ static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 	return 0;
 }
 
+static void fsl_a008585_set_sne(struct clock_event_device *clk)
+{
+#ifdef CONFIG_FSL_ERRATUM_A008585
+	if (!static_branch_unlikely(&arch_timer_read_ool_enabled))
+		return;
+
+	if (arch_timer_uses_ppi == VIRT_PPI)
+		clk->set_next_event = fsl_a008585_set_next_event_virt;
+	else
+		clk->set_next_event = fsl_a008585_set_next_event_phys;
+#endif
+}
+
 static void __arch_timer_setup(unsigned type,
 			       struct clock_event_device *clk)
 {
@@ -296,6 +383,8 @@ static void __arch_timer_setup(unsigned type,
 		default:
 			BUG();
 		}
+
+		fsl_a008585_set_sne(clk);
 	} else {
 		clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 		clk->name = "arch_mem_timer";
@@ -370,16 +459,33 @@ static bool arch_timer_has_nonsecure_ppi(void)
 		arch_timer_ppi[PHYS_NONSECURE_PPI]);
 }
 
+static u32 check_ppi_trigger(int irq)
+{
+	u32 flags = irq_get_trigger_type(irq);
+
+	if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
+		pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
+		pr_warn("WARNING: Please fix your firmware\n");
+		flags = IRQF_TRIGGER_LOW;
+	}
+
+	return flags;
+}
+
 static int arch_timer_starting_cpu(unsigned int cpu)
 {
 	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
+	u32 flags;
 
 	__arch_timer_setup(ARCH_CP15_TIMER, clk);
 
-	enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], 0);
+	flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
+	enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 
-	if (arch_timer_has_nonsecure_ppi())
-		enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
+	if (arch_timer_has_nonsecure_ppi()) {
+		flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
+		enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
+	}
 
 	arch_counter_set_user_access();
 	if (evtstrm_enable)
@@ -495,15 +601,19 @@ static void __init arch_counter_register(unsigned type)
 			arch_timer_read_counter = arch_counter_get_cntvct;
 		else
 			arch_timer_read_counter = arch_counter_get_cntpct;
-	} else {
-		arch_timer_read_counter = arch_counter_get_cntvct_mem;
 
-		/* If the clocksource name is "arch_sys_counter" the
-		 * VDSO will attempt to read the CP15-based counter.
-		 * Ensure this does not happen when CP15-based
-		 * counter is not available.
+		clocksource_counter.archdata.vdso_direct = true;
+
+#ifdef CONFIG_FSL_ERRATUM_A008585
+		/*
+		 * Don't use the vdso fastpath if errata require using
+		 * the out-of-line counter accessor.
 		 */
-		clocksource_counter.name = "arch_mem_counter";
+		if (static_branch_unlikely(&arch_timer_read_ool_enabled))
+			clocksource_counter.archdata.vdso_direct = false;
+#endif
+	} else {
+		arch_timer_read_counter = arch_counter_get_cntvct_mem;
 	}
 
 	start_count = arch_timer_read_counter();
@@ -780,6 +890,15 @@ static int __init arch_timer_of_init(struct device_node *np)
 
 	arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 
+#ifdef CONFIG_FSL_ERRATUM_A008585
+	if (fsl_a008585_enable < 0)
+		fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585");
+	if (fsl_a008585_enable) {
+		static_branch_enable(&arch_timer_read_ool_enabled);
+		pr_info("Enabling workaround for FSL erratum A-008585\n");
+	}
+#endif
+
 	/*
 	 * If we cannot rely on firmware initializing the timer registers then
 	 * we should use the physical timers instead.
diff --git a/drivers/clocksource/bcm_kona_timer.c b/drivers/clocksource/bcm_kona_timer.c
index 7e3fd375a627..92f6e4deee74 100644
--- a/drivers/clocksource/bcm_kona_timer.c
+++ b/drivers/clocksource/bcm_kona_timer.c
@@ -66,10 +66,10 @@ static void kona_timer_disable_and_clear(void __iomem *base)
 
 }
 
-static void
+static int
 kona_timer_get_counter(void __iomem *timer_base, uint32_t *msw, uint32_t *lsw)
 {
-	int loop_limit = 4;
+	int loop_limit = 3;
 
 	/*
 	 * Read 64-bit free running counter
@@ -83,18 +83,19 @@ kona_timer_get_counter(void __iomem *timer_base, uint32_t *msw, uint32_t *lsw)
 	 *      if new hi-word is equal to previously read hi-word then stop.
 	 */
 
-	while (--loop_limit) {
+	do {
 		*msw = readl(timer_base + KONA_GPTIMER_STCHI_OFFSET);
 		*lsw = readl(timer_base + KONA_GPTIMER_STCLO_OFFSET);
 		if (*msw == readl(timer_base + KONA_GPTIMER_STCHI_OFFSET))
 			break;
-	}
+	} while (--loop_limit);
 	if (!loop_limit) {
 		pr_err("bcm_kona_timer: getting counter failed.\n");
 		pr_err(" Timer will be impacted\n");
+		return -ETIMEDOUT;
 	}
 
-	return;
+	return 0;
 }
 
 static int kona_timer_set_next_event(unsigned long clc,
@@ -112,8 +113,11 @@ static int kona_timer_set_next_event(unsigned long clc,
 
 	uint32_t lsw, msw;
 	uint32_t reg;
+	int ret;
 
-	kona_timer_get_counter(timers.tmr_regs, &msw, &lsw);
+	ret = kona_timer_get_counter(timers.tmr_regs, &msw, &lsw);
+	if (ret)
+		return ret;
 
 	/* Load the "next" event tick value */
 	writel(lsw + clc, timers.tmr_regs + KONA_GPTIMER_STCM0_OFFSET);
diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c
index 41840d02c331..8f3488b80896 100644
--- a/drivers/clocksource/exynos_mct.c
+++ b/drivers/clocksource/exynos_mct.c
@@ -223,6 +223,7 @@ static u64 notrace exynos4_read_sched_clock(void)
 	return exynos4_read_count_32();
 }
 
+#if defined(CONFIG_ARM)
 static struct delay_timer exynos4_delay_timer;
 
 static cycles_t exynos4_read_current_timer(void)
@@ -231,14 +232,17 @@ static cycles_t exynos4_read_current_timer(void)
 			 "cycles_t needs to move to 32-bit for ARM64 usage");
 	return exynos4_read_count_32();
 }
+#endif
 
 static int __init exynos4_clocksource_init(void)
 {
 	exynos4_mct_frc_start();
 
+#if defined(CONFIG_ARM)
 	exynos4_delay_timer.read_current_timer = &exynos4_read_current_timer;
 	exynos4_delay_timer.freq = clk_rate;
 	register_current_timer_delay(&exynos4_delay_timer);
+#endif
 
 	if (clocksource_register_hz(&mct_frc, clk_rate))
 		panic("%s: can't register clocksource\n", mct_frc.name);
diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c
index d91e8725917c..7a960cd01104 100644
--- a/drivers/clocksource/mips-gic-timer.c
+++ b/drivers/clocksource/mips-gic-timer.c
@@ -109,12 +109,15 @@ static int gic_clockevent_init(void)
 {
 	int ret;
 
-	if (!cpu_has_counter || !gic_frequency)
+	if (!gic_frequency)
 		return -ENXIO;
 
 	ret = setup_percpu_irq(gic_timer_irq, &gic_compare_irqaction);
-	if (ret < 0)
+	if (ret < 0) {
+		pr_err("GIC timer IRQ %d setup failed: %d\n",
+		       gic_timer_irq, ret);
 		return ret;
+	}
 
 	cpuhp_setup_state(CPUHP_AP_MIPS_GIC_TIMER_STARTING,
 			  "AP_MIPS_GIC_TIMER_STARTING", gic_starting_cpu,
@@ -164,7 +167,7 @@ void __init gic_clocksource_init(unsigned int frequency)
 	gic_start_count();
 }
 
-static void __init gic_clocksource_of_init(struct device_node *node)
+static int __init gic_clocksource_of_init(struct device_node *node)
 {
 	struct clk *clk;
 	int ret;
diff --git a/drivers/clocksource/moxart_timer.c b/drivers/clocksource/moxart_timer.c
index 841454417acd..2a8f4705c734 100644
--- a/drivers/clocksource/moxart_timer.c
+++ b/drivers/clocksource/moxart_timer.c
@@ -21,6 +21,7 @@
 #include <linux/io.h>
 #include <linux/clocksource.h>
 #include <linux/bitops.h>
+#include <linux/slab.h>
 
 #define TIMER1_BASE		0x00
 #define TIMER2_BASE		0x10
@@ -36,75 +37,109 @@
 #define TIMER_INTR_MASK		0x38
 
 /*
- * TIMER_CR flags:
+ * Moxart TIMER_CR flags:
  *
- * TIMEREG_CR_*_CLOCK	0: PCLK, 1: EXT1CLK
- * TIMEREG_CR_*_INT	overflow interrupt enable bit
+ * MOXART_CR_*_CLOCK	0: PCLK, 1: EXT1CLK
+ * MOXART_CR_*_INT	overflow interrupt enable bit
  */
-#define TIMEREG_CR_1_ENABLE	BIT(0)
-#define TIMEREG_CR_1_CLOCK	BIT(1)
-#define TIMEREG_CR_1_INT	BIT(2)
-#define TIMEREG_CR_2_ENABLE	BIT(3)
-#define TIMEREG_CR_2_CLOCK	BIT(4)
-#define TIMEREG_CR_2_INT	BIT(5)
-#define TIMEREG_CR_3_ENABLE	BIT(6)
-#define TIMEREG_CR_3_CLOCK	BIT(7)
-#define TIMEREG_CR_3_INT	BIT(8)
-#define TIMEREG_CR_COUNT_UP	BIT(9)
-
-#define TIMER1_ENABLE		(TIMEREG_CR_2_ENABLE | TIMEREG_CR_1_ENABLE)
-#define TIMER1_DISABLE		(TIMEREG_CR_2_ENABLE)
-
-static void __iomem *base;
-static unsigned int clock_count_per_tick;
+#define MOXART_CR_1_ENABLE	BIT(0)
+#define MOXART_CR_1_CLOCK	BIT(1)
+#define MOXART_CR_1_INT	BIT(2)
+#define MOXART_CR_2_ENABLE	BIT(3)
+#define MOXART_CR_2_CLOCK	BIT(4)
+#define MOXART_CR_2_INT	BIT(5)
+#define MOXART_CR_3_ENABLE	BIT(6)
+#define MOXART_CR_3_CLOCK	BIT(7)
+#define MOXART_CR_3_INT	BIT(8)
+#define MOXART_CR_COUNT_UP	BIT(9)
+
+#define MOXART_TIMER1_ENABLE	(MOXART_CR_2_ENABLE | MOXART_CR_1_ENABLE)
+#define MOXART_TIMER1_DISABLE	(MOXART_CR_2_ENABLE)
+
+/*
+ * The ASpeed variant of the IP block has a different layout
+ * for the control register
+ */
+#define ASPEED_CR_1_ENABLE	BIT(0)
+#define ASPEED_CR_1_CLOCK	BIT(1)
+#define ASPEED_CR_1_INT		BIT(2)
+#define ASPEED_CR_2_ENABLE	BIT(4)
+#define ASPEED_CR_2_CLOCK	BIT(5)
+#define ASPEED_CR_2_INT		BIT(6)
+#define ASPEED_CR_3_ENABLE	BIT(8)
+#define ASPEED_CR_3_CLOCK	BIT(9)
+#define ASPEED_CR_3_INT		BIT(10)
+
+#define ASPEED_TIMER1_ENABLE   (ASPEED_CR_2_ENABLE | ASPEED_CR_1_ENABLE)
+#define ASPEED_TIMER1_DISABLE  (ASPEED_CR_2_ENABLE)
+
+struct moxart_timer {
+	void __iomem *base;
+	unsigned int t1_disable_val;
+	unsigned int t1_enable_val;
+	unsigned int count_per_tick;
+	struct clock_event_device clkevt;
+};
+
+static inline struct moxart_timer *to_moxart(struct clock_event_device *evt)
+{
+	return container_of(evt, struct moxart_timer, clkevt);
+}
+
+static inline void moxart_disable(struct clock_event_device *evt)
+{
+	struct moxart_timer *timer = to_moxart(evt);
+
+	writel(timer->t1_disable_val, timer->base + TIMER_CR);
+}
+
+static inline void moxart_enable(struct clock_event_device *evt)
+{
+	struct moxart_timer *timer = to_moxart(evt);
+
+	writel(timer->t1_enable_val, timer->base + TIMER_CR);
+}
 
 static int moxart_shutdown(struct clock_event_device *evt)
 {
-	writel(TIMER1_DISABLE, base + TIMER_CR);
+	moxart_disable(evt);
 	return 0;
 }
 
 static int moxart_set_oneshot(struct clock_event_device *evt)
 {
-	writel(TIMER1_DISABLE, base + TIMER_CR);
-	writel(~0, base + TIMER1_BASE + REG_LOAD);
+	moxart_disable(evt);
+	writel(~0, to_moxart(evt)->base + TIMER1_BASE + REG_LOAD);
 	return 0;
 }
 
 static int moxart_set_periodic(struct clock_event_device *evt)
 {
-	writel(clock_count_per_tick, base + TIMER1_BASE + REG_LOAD);
-	writel(TIMER1_ENABLE, base + TIMER_CR);
+	struct moxart_timer *timer = to_moxart(evt);
+
+	moxart_disable(evt);
+	writel(timer->count_per_tick, timer->base + TIMER1_BASE + REG_LOAD);
+	writel(0, timer->base + TIMER1_BASE + REG_MATCH1);
+	moxart_enable(evt);
 	return 0;
 }
 
 static int moxart_clkevt_next_event(unsigned long cycles,
-				    struct clock_event_device *unused)
+				    struct clock_event_device *evt)
 {
+	struct moxart_timer *timer = to_moxart(evt);
 	u32 u;
 
-	writel(TIMER1_DISABLE, base + TIMER_CR);
+	moxart_disable(evt);
 
-	u = readl(base + TIMER1_BASE + REG_COUNT) - cycles;
-	writel(u, base + TIMER1_BASE + REG_MATCH1);
+	u = readl(timer->base + TIMER1_BASE + REG_COUNT) - cycles;
+	writel(u, timer->base + TIMER1_BASE + REG_MATCH1);
 
-	writel(TIMER1_ENABLE, base + TIMER_CR);
+	moxart_enable(evt);
 
 	return 0;
 }
 
-static struct clock_event_device moxart_clockevent = {
-	.name			= "moxart_timer",
-	.rating			= 200,
-	.features		= CLOCK_EVT_FEAT_PERIODIC |
-				  CLOCK_EVT_FEAT_ONESHOT,
-	.set_state_shutdown	= moxart_shutdown,
-	.set_state_periodic	= moxart_set_periodic,
-	.set_state_oneshot	= moxart_set_oneshot,
-	.tick_resume		= moxart_set_oneshot,
-	.set_next_event		= moxart_clkevt_next_event,
-};
-
 static irqreturn_t moxart_timer_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *evt = dev_id;
@@ -112,21 +147,19 @@ static irqreturn_t moxart_timer_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static struct irqaction moxart_timer_irq = {
-	.name		= "moxart-timer",
-	.flags		= IRQF_TIMER,
-	.handler	= moxart_timer_interrupt,
-	.dev_id		= &moxart_clockevent,
-};
-
 static int __init moxart_timer_init(struct device_node *node)
 {
 	int ret, irq;
 	unsigned long pclk;
 	struct clk *clk;
+	struct moxart_timer *timer;
+
+	timer = kzalloc(sizeof(*timer), GFP_KERNEL);
+	if (!timer)
+		return -ENOMEM;
 
-	base = of_iomap(node, 0);
-	if (!base) {
+	timer->base = of_iomap(node, 0);
+	if (!timer->base) {
 		pr_err("%s: of_iomap failed\n", node->full_name);
 		return -ENXIO;
 	}
@@ -137,12 +170,6 @@ static int __init moxart_timer_init(struct device_node *node)
 		return -EINVAL;
 	}
 
-	ret = setup_irq(irq, &moxart_timer_irq);
-	if (ret) {
-		pr_err("%s: setup_irq failed\n", node->full_name);
-		return ret;
-	}
-
 	clk = of_clk_get(node, 0);
 	if (IS_ERR(clk))  {
 		pr_err("%s: of_clk_get failed\n", node->full_name);
@@ -151,7 +178,32 @@ static int __init moxart_timer_init(struct device_node *node)
 
 	pclk = clk_get_rate(clk);
 
-	ret = clocksource_mmio_init(base + TIMER2_BASE + REG_COUNT,
+	if (of_device_is_compatible(node, "moxa,moxart-timer")) {
+		timer->t1_enable_val = MOXART_TIMER1_ENABLE;
+		timer->t1_disable_val = MOXART_TIMER1_DISABLE;
+	} else if (of_device_is_compatible(node, "aspeed,ast2400-timer")) {
+		timer->t1_enable_val = ASPEED_TIMER1_ENABLE;
+		timer->t1_disable_val = ASPEED_TIMER1_DISABLE;
+	} else {
+		pr_err("%s: unknown platform\n", node->full_name);
+		return -EINVAL;
+	}
+
+	timer->count_per_tick = DIV_ROUND_CLOSEST(pclk, HZ);
+
+	timer->clkevt.name = node->name;
+	timer->clkevt.rating = 200;
+	timer->clkevt.features = CLOCK_EVT_FEAT_PERIODIC |
+					CLOCK_EVT_FEAT_ONESHOT;
+	timer->clkevt.set_state_shutdown = moxart_shutdown;
+	timer->clkevt.set_state_periodic = moxart_set_periodic;
+	timer->clkevt.set_state_oneshot = moxart_set_oneshot;
+	timer->clkevt.tick_resume = moxart_set_oneshot;
+	timer->clkevt.set_next_event = moxart_clkevt_next_event;
+	timer->clkevt.cpumask = cpumask_of(0);
+	timer->clkevt.irq = irq;
+
+	ret = clocksource_mmio_init(timer->base + TIMER2_BASE + REG_COUNT,
 				    "moxart_timer", pclk, 200, 32,
 				    clocksource_mmio_readl_down);
 	if (ret) {
@@ -159,13 +211,26 @@ static int __init moxart_timer_init(struct device_node *node)
 		return ret;
 	}
 
-	clock_count_per_tick = DIV_ROUND_CLOSEST(pclk, HZ);
+	ret = request_irq(irq, moxart_timer_interrupt, IRQF_TIMER,
+			  node->name, &timer->clkevt);
+	if (ret) {
+		pr_err("%s: setup_irq failed\n", node->full_name);
+		return ret;
+	}
 
-	writel(~0, base + TIMER2_BASE + REG_LOAD);
-	writel(TIMEREG_CR_2_ENABLE, base + TIMER_CR);
+	/* Clear match registers */
+	writel(0, timer->base + TIMER1_BASE + REG_MATCH1);
+	writel(0, timer->base + TIMER1_BASE + REG_MATCH2);
+	writel(0, timer->base + TIMER2_BASE + REG_MATCH1);
+	writel(0, timer->base + TIMER2_BASE + REG_MATCH2);
 
-	moxart_clockevent.cpumask = cpumask_of(0);
-	moxart_clockevent.irq = irq;
+	/*
+	 * Start timer 2 rolling as our main wall clock source, keep timer 1
+	 * disabled
+	 */
+	writel(0, timer->base + TIMER_CR);
+	writel(~0, timer->base + TIMER2_BASE + REG_LOAD);
+	writel(timer->t1_disable_val, timer->base + TIMER_CR);
 
 	/*
 	 * documentation is not publicly available:
@@ -173,9 +238,9 @@ static int __init moxart_timer_init(struct device_node *node)
 	 * max_delta 0xfffffffe should be ok because count
 	 * register size is u32
 	 */
-	clockevents_config_and_register(&moxart_clockevent, pclk,
-					0x4, 0xfffffffe);
+	clockevents_config_and_register(&timer->clkevt, pclk, 0x4, 0xfffffffe);
 
 	return 0;
 }
 CLOCKSOURCE_OF_DECLARE(moxart, "moxa,moxart-timer", moxart_timer_init);
+CLOCKSOURCE_OF_DECLARE(aspeed, "aspeed,ast2400-timer", moxart_timer_init);
diff --git a/drivers/clocksource/pxa_timer.c b/drivers/clocksource/pxa_timer.c
index 937e10b84d58..3e1cb512f3ce 100644
--- a/drivers/clocksource/pxa_timer.c
+++ b/drivers/clocksource/pxa_timer.c
@@ -21,6 +21,8 @@
 #include <linux/of_irq.h>
 #include <linux/sched_clock.h>
 
+#include <clocksource/pxa.h>
+
 #include <asm/div64.h>
 
 #define OSMR0		0x00	/* OS Timer 0 Match Register */
diff --git a/drivers/clocksource/sun4i_timer.c b/drivers/clocksource/sun4i_timer.c
index 97669ee4df2a..c83452cacb41 100644
--- a/drivers/clocksource/sun4i_timer.c
+++ b/drivers/clocksource/sun4i_timer.c
@@ -123,12 +123,16 @@ static struct clock_event_device sun4i_clockevent = {
 	.set_next_event = sun4i_clkevt_next_event,
 };
 
+static void sun4i_timer_clear_interrupt(void)
+{
+	writel(TIMER_IRQ_EN(0), timer_base + TIMER_IRQ_ST_REG);
+}
 
 static irqreturn_t sun4i_timer_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *evt = (struct clock_event_device *)dev_id;
 
-	writel(0x1, timer_base + TIMER_IRQ_ST_REG);
+	sun4i_timer_clear_interrupt();
 	evt->event_handler(evt);
 
 	return IRQ_HANDLED;
@@ -208,6 +212,9 @@ static int __init sun4i_timer_init(struct device_node *node)
 	/* Make sure timer is stopped before playing with interrupts */
 	sun4i_clkevt_time_stop(0);
 
+	/* clear timer0 interrupt */
+	sun4i_timer_clear_interrupt();
+
 	sun4i_clockevent.cpumask = cpu_possible_mask;
 	sun4i_clockevent.irq = irq;
 
diff --git a/drivers/clocksource/time-armada-370-xp.c b/drivers/clocksource/time-armada-370-xp.c
index 719b478d136e..3c39e6f45971 100644
--- a/drivers/clocksource/time-armada-370-xp.c
+++ b/drivers/clocksource/time-armada-370-xp.c
@@ -338,7 +338,6 @@ static int __init armada_xp_timer_init(struct device_node *np)
 	struct clk *clk = of_clk_get_by_name(np, "fixed");
 	int ret;
 
-	clk = of_clk_get(np, 0);
 	if (IS_ERR(clk)) {
 		pr_err("Failed to get clock");
 		return PTR_ERR(clk);
diff --git a/drivers/clocksource/time-pistachio.c b/drivers/clocksource/time-pistachio.c
index a7d9a08e4b0e..a8e6c7df853d 100644
--- a/drivers/clocksource/time-pistachio.c
+++ b/drivers/clocksource/time-pistachio.c
@@ -202,10 +202,10 @@ static int __init pistachio_clksrc_of_init(struct device_node *node)
 	rate = clk_get_rate(fast_clk);
 
 	/* Disable irq's for clocksource usage */
-	gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 0);
-	gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 1);
-	gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 2);
-	gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 3);
+	gpt_writel(pcs_gpt.base, 0, TIMER_IRQ_MASK, 0);
+	gpt_writel(pcs_gpt.base, 0, TIMER_IRQ_MASK, 1);
+	gpt_writel(pcs_gpt.base, 0, TIMER_IRQ_MASK, 2);
+	gpt_writel(pcs_gpt.base, 0, TIMER_IRQ_MASK, 3);
 
 	/* Enable timer block */
 	writel(TIMER_ME_GLOBAL, pcs_gpt.base);
diff --git a/drivers/clocksource/timer-atmel-pit.c b/drivers/clocksource/timer-atmel-pit.c
index 1ffac0cb0cb7..6555821bbdae 100644
--- a/drivers/clocksource/timer-atmel-pit.c
+++ b/drivers/clocksource/timer-atmel-pit.c
@@ -149,24 +149,13 @@ static irqreturn_t at91sam926x_pit_interrupt(int irq, void *dev_id)
 {
 	struct pit_data *data = dev_id;
 
-	/*
-	 * irqs should be disabled here, but as the irq is shared they are only
-	 * guaranteed to be off if the timer irq is registered first.
-	 */
-	WARN_ON_ONCE(!irqs_disabled());
-
 	/* The PIT interrupt may be disabled, and is shared */
 	if (clockevent_state_periodic(&data->clkevt) &&
 	    (pit_read(data->base, AT91_PIT_SR) & AT91_PIT_PITS)) {
-		unsigned nr_ticks;
-
 		/* Get number of ticks performed before irq, and ack it */
-		nr_ticks = PIT_PICNT(pit_read(data->base, AT91_PIT_PIVR));
-		do {
-			data->cnt += data->cycle;
-			data->clkevt.event_handler(&data->clkevt);
-			nr_ticks--;
-		} while (nr_ticks);
+		data->cnt += data->cycle * PIT_PICNT(pit_read(data->base,
+							      AT91_PIT_PIVR));
+		data->clkevt.event_handler(&data->clkevt);
 
 		return IRQ_HANDLED;
 	}
@@ -177,11 +166,41 @@ static irqreturn_t at91sam926x_pit_interrupt(int irq, void *dev_id)
 /*
  * Set up both clocksource and clockevent support.
  */
-static int __init at91sam926x_pit_common_init(struct pit_data *data)
+static int __init at91sam926x_pit_dt_init(struct device_node *node)
 {
-	unsigned long	pit_rate;
-	unsigned	bits;
-	int		ret;
+	unsigned long   pit_rate;
+	unsigned        bits;
+	int             ret;
+	struct pit_data *data;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->base = of_iomap(node, 0);
+	if (!data->base) {
+		pr_err("Could not map PIT address\n");
+		return -ENXIO;
+	}
+
+	data->mck = of_clk_get(node, 0);
+	if (IS_ERR(data->mck)) {
+		pr_err("Unable to get mck clk\n");
+		return PTR_ERR(data->mck);
+	}
+
+	ret = clk_prepare_enable(data->mck);
+	if (ret) {
+		pr_err("Unable to enable mck\n");
+		return ret;
+	}
+
+	/* Get the interrupts property */
+	data->irq = irq_of_parse_and_map(node, 0);
+	if (!data->irq) {
+		pr_err("Unable to get IRQ from DT\n");
+		return -EINVAL;
+	}
 
 	/*
 	 * Use our actual MCK to figure out how many MCK/16 ticks per
@@ -236,39 +255,5 @@ static int __init at91sam926x_pit_common_init(struct pit_data *data)
 
 	return 0;
 }
-
-static int __init at91sam926x_pit_dt_init(struct device_node *node)
-{
-	struct pit_data *data;
-
-	data = kzalloc(sizeof(*data), GFP_KERNEL);
-	if (!data)
-		return -ENOMEM;
-
-	data->base = of_iomap(node, 0);
-	if (!data->base) {
-		pr_err("Could not map PIT address\n");
-		return -ENXIO;
-	}
-
-	data->mck = of_clk_get(node, 0);
-	if (IS_ERR(data->mck))
-		/* Fallback on clkdev for !CCF-based boards */
-		data->mck = clk_get(NULL, "mck");
-
-	if (IS_ERR(data->mck)) {
-		pr_err("Unable to get mck clk\n");
-		return PTR_ERR(data->mck);
-	}
-
-	/* Get the interrupts property */
-	data->irq = irq_of_parse_and_map(node, 0);
-	if (!data->irq) {
-		pr_err("Unable to get IRQ from DT\n");
-		return -EINVAL;
-	}
-
-	return at91sam926x_pit_common_init(data);
-}
 CLOCKSOURCE_OF_DECLARE(at91sam926x_pit, "atmel,at91sam9260-pit",
 		       at91sam926x_pit_dt_init);
diff --git a/drivers/clocksource/timer-oxnas-rps.c b/drivers/clocksource/timer-oxnas-rps.c
index bd887e2a8cf8..d630bf417773 100644
--- a/drivers/clocksource/timer-oxnas-rps.c
+++ b/drivers/clocksource/timer-oxnas-rps.c
@@ -295,3 +295,5 @@ err_alloc:
 
 CLOCKSOURCE_OF_DECLARE(ox810se_rps,
 		       "oxsemi,ox810se-rps-timer", oxnas_rps_timer_init);
+CLOCKSOURCE_OF_DECLARE(ox820_rps,
+		       "oxsemi,ox820se-rps-timer", oxnas_rps_timer_init);
diff --git a/drivers/clocksource/timer-ti-32k.c b/drivers/clocksource/timer-ti-32k.c
index 92b7e390f6c8..cf5b14e442e4 100644
--- a/drivers/clocksource/timer-ti-32k.c
+++ b/drivers/clocksource/timer-ti-32k.c
@@ -65,7 +65,7 @@ static inline struct ti_32k *to_ti_32k(struct clocksource *cs)
 	return container_of(cs, struct ti_32k, cs);
 }
 
-static cycle_t ti_32k_read_cycles(struct clocksource *cs)
+static cycle_t notrace ti_32k_read_cycles(struct clocksource *cs)
 {
 	struct ti_32k *ti = to_ti_32k(cs);
 
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index 74919aa81dcb..d8b164a7c4e5 100644
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -194,7 +194,7 @@ config CPU_FREQ_GOV_CONSERVATIVE
 	  If in doubt, say N.
 
 config CPU_FREQ_GOV_SCHEDUTIL
-	tristate "'schedutil' cpufreq policy governor"
+	bool "'schedutil' cpufreq policy governor"
 	depends on CPU_FREQ && SMP
 	select CPU_FREQ_GOV_ATTR_SET
 	select IRQ_WORK
@@ -208,9 +208,6 @@ config CPU_FREQ_GOV_SCHEDUTIL
 	  frequency tipping point is at utilization/capacity equal to 80% in
 	  both cases.
 
-	  To compile this driver as a module, choose M here: the module will
-	  be called cpufreq_schedutil.
-
 	  If in doubt, say N.
 
 comment "CPU frequency scaling drivers"
@@ -225,7 +222,7 @@ config CPUFREQ_DT
 	help
 	  This adds a generic DT based cpufreq driver for frequency management.
 	  It supports both uniprocessor (UP) and symmetric multiprocessor (SMP)
-	  systems which share clock and voltage across all CPUs.
+	  systems.
 
 	  If in doubt, say N.
 
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 8882b8e2ecd0..1b2f28f69a81 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -19,10 +19,19 @@
 #include <linux/delay.h>
 #include <linux/cpu.h>
 #include <linux/cpufreq.h>
+#include <linux/dmi.h>
 #include <linux/vmalloc.h>
 
+#include <asm/unaligned.h>
+
 #include <acpi/cppc_acpi.h>
 
+/* Minimum struct length needed for the DMI processor entry we want */
+#define DMI_ENTRY_PROCESSOR_MIN_LENGTH	48
+
+/* Offest in the DMI processor structure for the max frequency */
+#define DMI_PROCESSOR_MAX_SPEED  0x14
+
 /*
  * These structs contain information parsed from per CPU
  * ACPI _CPC structures.
@@ -30,19 +39,52 @@
  * performance capabilities, desired performance level
  * requested etc.
  */
-static struct cpudata **all_cpu_data;
+static struct cppc_cpudata **all_cpu_data;
+
+/* Capture the max KHz from DMI */
+static u64 cppc_dmi_max_khz;
+
+/* Callback function used to retrieve the max frequency from DMI */
+static void cppc_find_dmi_mhz(const struct dmi_header *dm, void *private)
+{
+	const u8 *dmi_data = (const u8 *)dm;
+	u16 *mhz = (u16 *)private;
+
+	if (dm->type == DMI_ENTRY_PROCESSOR &&
+	    dm->length >= DMI_ENTRY_PROCESSOR_MIN_LENGTH) {
+		u16 val = (u16)get_unaligned((const u16 *)
+				(dmi_data + DMI_PROCESSOR_MAX_SPEED));
+		*mhz = val > *mhz ? val : *mhz;
+	}
+}
+
+/* Look up the max frequency in DMI */
+static u64 cppc_get_dmi_max_khz(void)
+{
+	u16 mhz = 0;
+
+	dmi_walk(cppc_find_dmi_mhz, &mhz);
+
+	/*
+	 * Real stupid fallback value, just in case there is no
+	 * actual value set.
+	 */
+	mhz = mhz ? mhz : 1;
+
+	return (1000 * mhz);
+}
 
 static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
 		unsigned int target_freq,
 		unsigned int relation)
 {
-	struct cpudata *cpu;
+	struct cppc_cpudata *cpu;
 	struct cpufreq_freqs freqs;
 	int ret = 0;
 
 	cpu = all_cpu_data[policy->cpu];
 
-	cpu->perf_ctrls.desired_perf = target_freq;
+	cpu->perf_ctrls.desired_perf = (u64)target_freq * policy->max / cppc_dmi_max_khz;
 	freqs.old = policy->cur;
 	freqs.new = target_freq;
 
@@ -66,7 +108,7 @@ static int cppc_verify_policy(struct cpufreq_policy *policy)
 static void cppc_cpufreq_stop_cpu(struct cpufreq_policy *policy)
 {
 	int cpu_num = policy->cpu;
-	struct cpudata *cpu = all_cpu_data[cpu_num];
+	struct cppc_cpudata *cpu = all_cpu_data[cpu_num];
 	int ret;
 
 	cpu->perf_ctrls.desired_perf = cpu->perf_caps.lowest_perf;
@@ -79,7 +121,7 @@ static void cppc_cpufreq_stop_cpu(struct cpufreq_policy *policy)
 
 static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 {
-	struct cpudata *cpu;
+	struct cppc_cpudata *cpu;
 	unsigned int cpu_num = policy->cpu;
 	int ret = 0;
 
@@ -94,10 +136,13 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 		return ret;
 	}
 
-	policy->min = cpu->perf_caps.lowest_perf;
-	policy->max = cpu->perf_caps.highest_perf;
+	cppc_dmi_max_khz = cppc_get_dmi_max_khz();
+
+	policy->min = cpu->perf_caps.lowest_perf * cppc_dmi_max_khz / cpu->perf_caps.highest_perf;
+	policy->max = cppc_dmi_max_khz;
 	policy->cpuinfo.min_freq = policy->min;
 	policy->cpuinfo.max_freq = policy->max;
+	policy->cpuinfo.transition_latency = cppc_get_transition_latency(cpu_num);
 	policy->shared_type = cpu->shared_type;
 
 	if (policy->shared_type == CPUFREQ_SHARED_TYPE_ANY)
@@ -112,7 +157,8 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 	cpu->cur_policy = policy;
 
 	/* Set policy->cur to max now. The governors will adjust later. */
-	policy->cur = cpu->perf_ctrls.desired_perf = cpu->perf_caps.highest_perf;
+	policy->cur = cppc_dmi_max_khz;
+	cpu->perf_ctrls.desired_perf = cpu->perf_caps.highest_perf;
 
 	ret = cppc_set_perf(cpu_num, &cpu->perf_ctrls);
 	if (ret)
@@ -134,7 +180,7 @@ static struct cpufreq_driver cppc_cpufreq_driver = {
 static int __init cppc_cpufreq_init(void)
 {
 	int i, ret = 0;
-	struct cpudata *cpu;
+	struct cppc_cpudata *cpu;
 
 	if (acpi_disabled)
 		return -ENODEV;
@@ -144,7 +190,7 @@ static int __init cppc_cpufreq_init(void)
 		return -ENOMEM;
 
 	for_each_possible_cpu(i) {
-		all_cpu_data[i] = kzalloc(sizeof(struct cpudata), GFP_KERNEL);
+		all_cpu_data[i] = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL);
 		if (!all_cpu_data[i])
 			goto out;
 
@@ -175,7 +221,7 @@ out:
 
 static void __exit cppc_cpufreq_exit(void)
 {
-	struct cpudata *cpu;
+	struct cppc_cpudata *cpu;
 	int i;
 
 	cpufreq_unregister_driver(&cppc_cpufreq_driver);
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index 0bb44d5b5df4..71267626456b 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -11,6 +11,8 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 
+#include "cpufreq-dt.h"
+
 static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "allwinner,sun4i-a10", },
 	{ .compatible = "allwinner,sun5i-a10s", },
@@ -40,6 +42,7 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "samsung,exynos5250", },
 #ifndef CONFIG_BL_SWITCHER
 	{ .compatible = "samsung,exynos5420", },
+	{ .compatible = "samsung,exynos5433", },
 	{ .compatible = "samsung,exynos5800", },
 #endif
 
@@ -51,6 +54,7 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "renesas,r8a7779", },
 	{ .compatible = "renesas,r8a7790", },
 	{ .compatible = "renesas,r8a7791", },
+	{ .compatible = "renesas,r8a7792", },
 	{ .compatible = "renesas,r8a7793", },
 	{ .compatible = "renesas,r8a7794", },
 	{ .compatible = "renesas,sh73a0", },
@@ -68,12 +72,16 @@ static const struct of_device_id machines[] __initconst = {
 
 	{ .compatible = "sigma,tango4" },
 
+	{ .compatible = "ti,am33xx", },
+	{ .compatible = "ti,dra7", },
 	{ .compatible = "ti,omap2", },
 	{ .compatible = "ti,omap3", },
 	{ .compatible = "ti,omap4", },
 	{ .compatible = "ti,omap5", },
 
 	{ .compatible = "xlnx,zynq-7000", },
+
+	{ }
 };
 
 static int __init cpufreq_dt_platdev_init(void)
@@ -89,7 +97,8 @@ static int __init cpufreq_dt_platdev_init(void)
 	if (!match)
 		return -ENODEV;
 
-	return PTR_ERR_OR_ZERO(platform_device_register_simple("cpufreq-dt", -1,
-							       NULL, 0));
+	return PTR_ERR_OR_ZERO(platform_device_register_data(NULL, "cpufreq-dt",
+			       -1, match->data,
+			       sizeof(struct cpufreq_dt_platform_data)));
 }
 device_initcall(cpufreq_dt_platdev_init);
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
index 3957de801ae8..5c07ae05d69a 100644
--- a/drivers/cpufreq/cpufreq-dt.c
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -25,6 +25,8 @@
 #include <linux/slab.h>
 #include <linux/thermal.h>
 
+#include "cpufreq-dt.h"
+
 struct private_data {
 	struct device *cpu_dev;
 	struct thermal_cooling_device *cdev;
@@ -353,6 +355,7 @@ static struct cpufreq_driver dt_cpufreq_driver = {
 
 static int dt_cpufreq_probe(struct platform_device *pdev)
 {
+	struct cpufreq_dt_platform_data *data = dev_get_platdata(&pdev->dev);
 	int ret;
 
 	/*
@@ -366,7 +369,8 @@ static int dt_cpufreq_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	dt_cpufreq_driver.driver_data = dev_get_platdata(&pdev->dev);
+	if (data && data->have_governor_per_policy)
+		dt_cpufreq_driver.flags |= CPUFREQ_HAVE_GOVERNOR_PER_POLICY;
 
 	ret = cpufreq_register_driver(&dt_cpufreq_driver);
 	if (ret)
diff --git a/drivers/cpufreq/cpufreq-dt.h b/drivers/cpufreq/cpufreq-dt.h
new file mode 100644
index 000000000000..54d774e46c43
--- /dev/null
+++ b/drivers/cpufreq/cpufreq-dt.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2016 Linaro
+ * Viresh Kumar <viresh.kumar@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __CPUFREQ_DT_H__
+#define __CPUFREQ_DT_H__
+
+#include <linux/types.h>
+
+struct cpufreq_dt_platform_data {
+	bool have_governor_per_policy;
+};
+
+#endif /* __CPUFREQ_DT_H__ */
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 3dd4884c6f9e..6e6c1fb60fbc 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -916,58 +916,18 @@ static struct kobj_type ktype_cpufreq = {
 	.release	= cpufreq_sysfs_release,
 };
 
-static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
+static int add_cpu_dev_symlink(struct cpufreq_policy *policy,
+			       struct device *dev)
 {
-	struct device *cpu_dev;
-
-	pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);
-
-	if (!policy)
-		return 0;
-
-	cpu_dev = get_cpu_device(cpu);
-	if (WARN_ON(!cpu_dev))
-		return 0;
-
-	return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
-}
-
-static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
-{
-	struct device *cpu_dev;
-
-	pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);
-
-	cpu_dev = get_cpu_device(cpu);
-	if (WARN_ON(!cpu_dev))
-		return;
-
-	sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
-}
-
-/* Add/remove symlinks for all related CPUs */
-static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
-{
-	unsigned int j;
-	int ret = 0;
-
-	/* Some related CPUs might not be present (physically hotplugged) */
-	for_each_cpu(j, policy->real_cpus) {
-		ret = add_cpu_dev_symlink(policy, j);
-		if (ret)
-			break;
-	}
-
-	return ret;
+	dev_dbg(dev, "%s: Adding symlink\n", __func__);
+	return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 }
 
-static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
+static void remove_cpu_dev_symlink(struct cpufreq_policy *policy,
+				   struct device *dev)
 {
-	unsigned int j;
-
-	/* Some related CPUs might not be present (physically hotplugged) */
-	for_each_cpu(j, policy->real_cpus)
-		remove_cpu_dev_symlink(policy, j);
+	dev_dbg(dev, "%s: Removing symlink\n", __func__);
+	sysfs_remove_link(&dev->kobj, "cpufreq");
 }
 
 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
@@ -999,7 +959,7 @@ static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
 			return ret;
 	}
 
-	return cpufreq_add_dev_symlink(policy);
+	return 0;
 }
 
 __weak struct cpufreq_governor *cpufreq_default_governor(void)
@@ -1073,13 +1033,9 @@ static void handle_update(struct work_struct *work)
 
 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
 {
-	struct device *dev = get_cpu_device(cpu);
 	struct cpufreq_policy *policy;
 	int ret;
 
-	if (WARN_ON(!dev))
-		return NULL;
-
 	policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 	if (!policy)
 		return NULL;
@@ -1133,7 +1089,6 @@ static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)
 
 	down_write(&policy->rwsem);
 	cpufreq_stats_free_table(policy);
-	cpufreq_remove_dev_symlink(policy);
 	kobj = &policy->kobj;
 	cmp = &policy->kobj_unregister;
 	up_write(&policy->rwsem);
@@ -1215,8 +1170,8 @@ static int cpufreq_online(unsigned int cpu)
 	if (new_policy) {
 		/* related_cpus should at least include policy->cpus. */
 		cpumask_copy(policy->related_cpus, policy->cpus);
-		/* Remember CPUs present at the policy creation time. */
-		cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask);
+		/* Clear mask of registered CPUs */
+		cpumask_clear(policy->real_cpus);
 	}
 
 	/*
@@ -1331,6 +1286,8 @@ out_free_policy:
 	return ret;
 }
 
+static int cpufreq_offline(unsigned int cpu);
+
 /**
  * cpufreq_add_dev - the cpufreq interface for a CPU device.
  * @dev: CPU device.
@@ -1340,25 +1297,31 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 {
 	struct cpufreq_policy *policy;
 	unsigned cpu = dev->id;
+	int ret;
 
 	dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
 
-	if (cpu_online(cpu))
-		return cpufreq_online(cpu);
+	if (cpu_online(cpu)) {
+		ret = cpufreq_online(cpu);
+		if (ret)
+			return ret;
+	}
 
-	/*
-	 * A hotplug notifier will follow and we will handle it as CPU online
-	 * then.  For now, just create the sysfs link, unless there is no policy
-	 * or the link is already present.
-	 */
+	/* Create sysfs link on CPU registration */
 	policy = per_cpu(cpufreq_cpu_data, cpu);
 	if (!policy || cpumask_test_and_set_cpu(cpu, policy->real_cpus))
 		return 0;
 
-	return add_cpu_dev_symlink(policy, cpu);
+	ret = add_cpu_dev_symlink(policy, dev);
+	if (ret) {
+		cpumask_clear_cpu(cpu, policy->real_cpus);
+		cpufreq_offline(cpu);
+	}
+
+	return ret;
 }
 
-static void cpufreq_offline(unsigned int cpu)
+static int cpufreq_offline(unsigned int cpu)
 {
 	struct cpufreq_policy *policy;
 	int ret;
@@ -1368,7 +1331,7 @@ static void cpufreq_offline(unsigned int cpu)
 	policy = cpufreq_cpu_get_raw(cpu);
 	if (!policy) {
 		pr_debug("%s: No cpu_data found\n", __func__);
-		return;
+		return 0;
 	}
 
 	down_write(&policy->rwsem);
@@ -1417,6 +1380,7 @@ static void cpufreq_offline(unsigned int cpu)
 
 unlock:
 	up_write(&policy->rwsem);
+	return 0;
 }
 
 /**
@@ -1436,7 +1400,7 @@ static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
 		cpufreq_offline(cpu);
 
 	cpumask_clear_cpu(cpu, policy->real_cpus);
-	remove_cpu_dev_symlink(policy, cpu);
+	remove_cpu_dev_symlink(policy, dev);
 
 	if (cpumask_empty(policy->real_cpus))
 		cpufreq_policy_free(policy, true);
@@ -2332,28 +2296,6 @@ unlock:
 }
 EXPORT_SYMBOL(cpufreq_update_policy);
 
-static int cpufreq_cpu_callback(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		cpufreq_online(cpu);
-		break;
-
-	case CPU_DOWN_PREPARE:
-		cpufreq_offline(cpu);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block __refdata cpufreq_cpu_notifier = {
-	.notifier_call = cpufreq_cpu_callback,
-};
-
 /*********************************************************************
  *               BOOST						     *
  *********************************************************************/
@@ -2455,6 +2397,7 @@ EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
 /*********************************************************************
  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
  *********************************************************************/
+static enum cpuhp_state hp_online;
 
 /**
  * cpufreq_register_driver - register a CPU Frequency driver
@@ -2517,7 +2460,14 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data)
 		goto err_if_unreg;
 	}
 
-	register_hotcpu_notifier(&cpufreq_cpu_notifier);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "cpufreq:online",
+					cpufreq_online,
+					cpufreq_offline);
+	if (ret < 0)
+		goto err_if_unreg;
+	hp_online = ret;
+	ret = 0;
+
 	pr_debug("driver %s up and running\n", driver_data->name);
 	goto out;
 
@@ -2556,7 +2506,7 @@ int cpufreq_unregister_driver(struct cpufreq_driver *driver)
 	get_online_cpus();
 	subsys_interface_unregister(&cpufreq_interface);
 	remove_boost_sysfs_file();
-	unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
+	cpuhp_remove_state_nocalls(hp_online);
 
 	write_lock_irqsave(&cpufreq_driver_lock, flags);
 
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index e415349ab31b..642dd0f183a8 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -260,7 +260,7 @@ static void dbs_irq_work(struct irq_work *irq_work)
 }
 
 static void dbs_update_util_handler(struct update_util_data *data, u64 time,
-				    unsigned long util, unsigned long max)
+				    unsigned int flags)
 {
 	struct cpu_dbs_info *cdbs = container_of(data, struct cpu_dbs_info, update_util);
 	struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index be9eade147f2..806f2039571e 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -181,6 +181,8 @@ struct _pid {
  * @cpu:		CPU number for this instance data
  * @update_util:	CPUFreq utility callback information
  * @update_util_set:	CPUFreq utility callback is set
+ * @iowait_boost:	iowait-related boost fraction
+ * @last_update:	Time of the last update.
  * @pstate:		Stores P state limits for this CPU
  * @vid:		Stores VID limits for this CPU
  * @pid:		Stores PID parameters for this CPU
@@ -206,6 +208,7 @@ struct cpudata {
 	struct vid_data vid;
 	struct _pid pid;
 
+	u64	last_update;
 	u64	last_sample_time;
 	u64	prev_aperf;
 	u64	prev_mperf;
@@ -216,6 +219,7 @@ struct cpudata {
 	struct acpi_processor_performance acpi_perf_data;
 	bool valid_pss_table;
 #endif
+	unsigned int iowait_boost;
 };
 
 static struct cpudata **all_cpu_data;
@@ -229,6 +233,7 @@ static struct cpudata **all_cpu_data;
  * @p_gain_pct:		PID proportional gain
  * @i_gain_pct:		PID integral gain
  * @d_gain_pct:		PID derivative gain
+ * @boost_iowait:	Whether or not to use iowait boosting.
  *
  * Stores per CPU model static PID configuration data.
  */
@@ -240,6 +245,7 @@ struct pstate_adjust_policy {
 	int p_gain_pct;
 	int d_gain_pct;
 	int i_gain_pct;
+	bool boost_iowait;
 };
 
 /**
@@ -1029,7 +1035,7 @@ static struct cpu_defaults core_params = {
 	},
 };
 
-static struct cpu_defaults silvermont_params = {
+static const struct cpu_defaults silvermont_params = {
 	.pid_policy = {
 		.sample_rate_ms = 10,
 		.deadband = 0,
@@ -1037,6 +1043,7 @@ static struct cpu_defaults silvermont_params = {
 		.p_gain_pct = 14,
 		.d_gain_pct = 0,
 		.i_gain_pct = 4,
+		.boost_iowait = true,
 	},
 	.funcs = {
 		.get_max = atom_get_max_pstate,
@@ -1050,7 +1057,7 @@ static struct cpu_defaults silvermont_params = {
 	},
 };
 
-static struct cpu_defaults airmont_params = {
+static const struct cpu_defaults airmont_params = {
 	.pid_policy = {
 		.sample_rate_ms = 10,
 		.deadband = 0,
@@ -1058,6 +1065,7 @@ static struct cpu_defaults airmont_params = {
 		.p_gain_pct = 14,
 		.d_gain_pct = 0,
 		.i_gain_pct = 4,
+		.boost_iowait = true,
 	},
 	.funcs = {
 		.get_max = atom_get_max_pstate,
@@ -1071,7 +1079,7 @@ static struct cpu_defaults airmont_params = {
 	},
 };
 
-static struct cpu_defaults knl_params = {
+static const struct cpu_defaults knl_params = {
 	.pid_policy = {
 		.sample_rate_ms = 10,
 		.deadband = 0,
@@ -1091,7 +1099,7 @@ static struct cpu_defaults knl_params = {
 	},
 };
 
-static struct cpu_defaults bxt_params = {
+static const struct cpu_defaults bxt_params = {
 	.pid_policy = {
 		.sample_rate_ms = 10,
 		.deadband = 0,
@@ -1099,6 +1107,7 @@ static struct cpu_defaults bxt_params = {
 		.p_gain_pct = 14,
 		.d_gain_pct = 0,
 		.i_gain_pct = 4,
+		.boost_iowait = true,
 	},
 	.funcs = {
 		.get_max = core_get_max_pstate,
@@ -1222,36 +1231,18 @@ static inline int32_t get_avg_pstate(struct cpudata *cpu)
 static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
 {
 	struct sample *sample = &cpu->sample;
-	u64 cummulative_iowait, delta_iowait_us;
-	u64 delta_iowait_mperf;
-	u64 mperf, now;
-	int32_t cpu_load;
+	int32_t busy_frac, boost;
 
-	cummulative_iowait = get_cpu_iowait_time_us(cpu->cpu, &now);
+	busy_frac = div_fp(sample->mperf, sample->tsc);
 
-	/*
-	 * Convert iowait time into number of IO cycles spent at max_freq.
-	 * IO is considered as busy only for the cpu_load algorithm. For
-	 * performance this is not needed since we always try to reach the
-	 * maximum P-State, so we are already boosting the IOs.
-	 */
-	delta_iowait_us = cummulative_iowait - cpu->prev_cummulative_iowait;
-	delta_iowait_mperf = div64_u64(delta_iowait_us * cpu->pstate.scaling *
-		cpu->pstate.max_pstate, MSEC_PER_SEC);
+	boost = cpu->iowait_boost;
+	cpu->iowait_boost >>= 1;
 
-	mperf = cpu->sample.mperf + delta_iowait_mperf;
-	cpu->prev_cummulative_iowait = cummulative_iowait;
+	if (busy_frac < boost)
+		busy_frac = boost;
 
-	/*
-	 * The load can be estimated as the ratio of the mperf counter
-	 * running at a constant frequency during active periods
-	 * (C0) and the time stamp counter running at the same frequency
-	 * also during C-states.
-	 */
-	cpu_load = div64_u64(int_tofp(100) * mperf, sample->tsc);
-	cpu->sample.busy_scaled = cpu_load;
-
-	return get_avg_pstate(cpu) - pid_calc(&cpu->pid, cpu_load);
+	sample->busy_scaled = busy_frac * 100;
+	return get_avg_pstate(cpu) - pid_calc(&cpu->pid, sample->busy_scaled);
 }
 
 static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
@@ -1325,15 +1316,29 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
 		sample->mperf,
 		sample->aperf,
 		sample->tsc,
-		get_avg_frequency(cpu));
+		get_avg_frequency(cpu),
+		fp_toint(cpu->iowait_boost * 100));
 }
 
 static void intel_pstate_update_util(struct update_util_data *data, u64 time,
-				     unsigned long util, unsigned long max)
+				     unsigned int flags)
 {
 	struct cpudata *cpu = container_of(data, struct cpudata, update_util);
-	u64 delta_ns = time - cpu->sample.time;
+	u64 delta_ns;
+
+	if (pid_params.boost_iowait) {
+		if (flags & SCHED_CPUFREQ_IOWAIT) {
+			cpu->iowait_boost = int_tofp(1);
+		} else if (cpu->iowait_boost) {
+			/* Clear iowait_boost if the CPU may have been idle. */
+			delta_ns = time - cpu->last_update;
+			if (delta_ns > TICK_NSEC)
+				cpu->iowait_boost = 0;
+		}
+		cpu->last_update = time;
+	}
 
+	delta_ns = time - cpu->sample.time;
 	if ((s64)delta_ns >= pid_params.sample_rate_ns) {
 		bool sample_taken = intel_pstate_sample(cpu, time);
 
diff --git a/drivers/cpufreq/kirkwood-cpufreq.c b/drivers/cpufreq/kirkwood-cpufreq.c
index be42f103db60..1b9bcd76c60e 100644
--- a/drivers/cpufreq/kirkwood-cpufreq.c
+++ b/drivers/cpufreq/kirkwood-cpufreq.c
@@ -123,7 +123,7 @@ static int kirkwood_cpufreq_probe(struct platform_device *pdev)
 
 	priv.cpu_clk = of_clk_get_by_name(np, "cpu_clk");
 	if (IS_ERR(priv.cpu_clk)) {
-		dev_err(priv.dev, "Unable to get cpuclk");
+		dev_err(priv.dev, "Unable to get cpuclk\n");
 		return PTR_ERR(priv.cpu_clk);
 	}
 
@@ -132,7 +132,7 @@ static int kirkwood_cpufreq_probe(struct platform_device *pdev)
 
 	priv.ddr_clk = of_clk_get_by_name(np, "ddrclk");
 	if (IS_ERR(priv.ddr_clk)) {
-		dev_err(priv.dev, "Unable to get ddrclk");
+		dev_err(priv.dev, "Unable to get ddrclk\n");
 		err = PTR_ERR(priv.ddr_clk);
 		goto out_cpu;
 	}
@@ -142,7 +142,7 @@ static int kirkwood_cpufreq_probe(struct platform_device *pdev)
 
 	priv.powersave_clk = of_clk_get_by_name(np, "powersave");
 	if (IS_ERR(priv.powersave_clk)) {
-		dev_err(priv.dev, "Unable to get powersave");
+		dev_err(priv.dev, "Unable to get powersave\n");
 		err = PTR_ERR(priv.powersave_clk);
 		goto out_ddr;
 	}
@@ -155,7 +155,7 @@ static int kirkwood_cpufreq_probe(struct platform_device *pdev)
 	if (!err)
 		return 0;
 
-	dev_err(priv.dev, "Failed to register cpufreq driver");
+	dev_err(priv.dev, "Failed to register cpufreq driver\n");
 
 	clk_disable_unprepare(priv.powersave_clk);
 out_ddr:
diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c
index 87796e0864e9..d3ffde806629 100644
--- a/drivers/cpufreq/powernv-cpufreq.c
+++ b/drivers/cpufreq/powernv-cpufreq.c
@@ -145,11 +145,30 @@ static struct powernv_pstate_info {
 /* Use following macros for conversions between pstate_id and index */
 static inline int idx_to_pstate(unsigned int i)
 {
+	if (unlikely(i >= powernv_pstate_info.nr_pstates)) {
+		pr_warn_once("index %u is out of bound\n", i);
+		return powernv_freqs[powernv_pstate_info.nominal].driver_data;
+	}
+
 	return powernv_freqs[i].driver_data;
 }
 
 static inline unsigned int pstate_to_idx(int pstate)
 {
+	int min = powernv_freqs[powernv_pstate_info.min].driver_data;
+	int max = powernv_freqs[powernv_pstate_info.max].driver_data;
+
+	if (min > 0) {
+		if (unlikely((pstate < max) || (pstate > min))) {
+			pr_warn_once("pstate %d is out of bound\n", pstate);
+			return powernv_pstate_info.nominal;
+		}
+	} else {
+		if (unlikely((pstate > max) || (pstate < min))) {
+			pr_warn_once("pstate %d is out of bound\n", pstate);
+			return powernv_pstate_info.nominal;
+		}
+	}
 	/*
 	 * abs() is deliberately used so that is works with
 	 * both monotonically increasing and decreasing
@@ -593,7 +612,7 @@ void gpstate_timer_handler(unsigned long data)
 	} else {
 		gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
 						 gpstates->highest_lpstate_idx,
-						 freq_data.pstate_id);
+						 gpstates->last_lpstate_idx);
 	}
 
 	/*
diff --git a/drivers/cpufreq/sa1110-cpufreq.c b/drivers/cpufreq/sa1110-cpufreq.c
index b5befc211172..2bac9b6cfeea 100644
--- a/drivers/cpufreq/sa1110-cpufreq.c
+++ b/drivers/cpufreq/sa1110-cpufreq.c
@@ -159,7 +159,7 @@ sdram_calculate_timing(struct sdram_info *sd, u_int cpu_khz,
 	 * half speed or use delayed read latching (errata 13).
 	 */
 	if ((ns_to_cycles(sdram->tck, sd_khz) > 1) ||
-	    (CPU_REVISION < CPU_SA1110_B2 && sd_khz < 62000))
+	    (read_cpuid_revision() < ARM_CPU_REV_SA1110_B2 && sd_khz < 62000))
 		sd_khz /= 2;
 
 	sd->mdcnfg = MDCNFG & 0x007f007f;
diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c
index e8a7bf57b31b..ea7a4e1b68c2 100644
--- a/drivers/cpufreq/scpi-cpufreq.c
+++ b/drivers/cpufreq/scpi-cpufreq.c
@@ -105,7 +105,6 @@ static int scpi_cpufreq_remove(struct platform_device *pdev)
 static struct platform_driver scpi_cpufreq_platdrv = {
 	.driver = {
 		.name	= "scpi-cpufreq",
-		.owner	= THIS_MODULE,
 	},
 	.probe		= scpi_cpufreq_probe,
 	.remove		= scpi_cpufreq_remove,
diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c
index 04042038ec4b..b366e6d830ea 100644
--- a/drivers/cpufreq/sti-cpufreq.c
+++ b/drivers/cpufreq/sti-cpufreq.c
@@ -163,7 +163,7 @@ static int sti_cpufreq_set_opp_info(void)
 
 	reg_fields = sti_cpufreq_match();
 	if (!reg_fields) {
-		dev_err(dev, "This SoC doesn't support voltage scaling");
+		dev_err(dev, "This SoC doesn't support voltage scaling\n");
 		return -ENODEV;
 	}
 
diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c
index d5657d50ac40..71e586d7df71 100644
--- a/drivers/cpuidle/coupled.c
+++ b/drivers/cpuidle/coupled.c
@@ -749,65 +749,52 @@ static void cpuidle_coupled_allow_idle(struct cpuidle_coupled *coupled)
 	put_cpu();
 }
 
-/**
- * cpuidle_coupled_cpu_notify - notifier called during hotplug transitions
- * @nb: notifier block
- * @action: hotplug transition
- * @hcpu: target cpu number
- *
- * Called when a cpu is brought on or offline using hotplug.  Updates the
- * coupled cpu set appropriately
- */
-static int cpuidle_coupled_cpu_notify(struct notifier_block *nb,
-		unsigned long action, void *hcpu)
+static int coupled_cpu_online(unsigned int cpu)
 {
-	int cpu = (unsigned long)hcpu;
 	struct cpuidle_device *dev;
 
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_UP_PREPARE:
-	case CPU_DOWN_PREPARE:
-	case CPU_ONLINE:
-	case CPU_DEAD:
-	case CPU_UP_CANCELED:
-	case CPU_DOWN_FAILED:
-		break;
-	default:
-		return NOTIFY_OK;
-	}
-
 	mutex_lock(&cpuidle_lock);
 
 	dev = per_cpu(cpuidle_devices, cpu);
-	if (!dev || !dev->coupled)
-		goto out;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_UP_PREPARE:
-	case CPU_DOWN_PREPARE:
-		cpuidle_coupled_prevent_idle(dev->coupled);
-		break;
-	case CPU_ONLINE:
-	case CPU_DEAD:
+	if (dev && dev->coupled) {
 		cpuidle_coupled_update_online_cpus(dev->coupled);
-		/* Fall through */
-	case CPU_UP_CANCELED:
-	case CPU_DOWN_FAILED:
 		cpuidle_coupled_allow_idle(dev->coupled);
-		break;
 	}
 
-out:
 	mutex_unlock(&cpuidle_lock);
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block cpuidle_coupled_cpu_notifier = {
-	.notifier_call = cpuidle_coupled_cpu_notify,
-};
+static int coupled_cpu_up_prepare(unsigned int cpu)
+{
+	struct cpuidle_device *dev;
+
+	mutex_lock(&cpuidle_lock);
+
+	dev = per_cpu(cpuidle_devices, cpu);
+	if (dev && dev->coupled)
+		cpuidle_coupled_prevent_idle(dev->coupled);
+
+	mutex_unlock(&cpuidle_lock);
+	return 0;
+}
 
 static int __init cpuidle_coupled_init(void)
 {
-	return register_cpu_notifier(&cpuidle_coupled_cpu_notifier);
+	int ret;
+
+	ret = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE,
+					"cpuidle/coupled:prepare",
+					coupled_cpu_up_prepare,
+					coupled_cpu_online);
+	if (ret)
+		return ret;
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"cpuidle/coupled:online",
+					coupled_cpu_online,
+					coupled_cpu_up_prepare);
+	if (ret < 0)
+		cpuhp_remove_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE);
+	return ret;
 }
 core_initcall(cpuidle_coupled_init);
diff --git a/drivers/cpuidle/cpuidle-arm.c b/drivers/cpuidle/cpuidle-arm.c
index 4ba3d3fe142f..f440d385ed34 100644
--- a/drivers/cpuidle/cpuidle-arm.c
+++ b/drivers/cpuidle/cpuidle-arm.c
@@ -121,6 +121,7 @@ static int __init arm_idle_init(void)
 		dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 		if (!dev) {
 			pr_err("Failed to allocate cpuidle device\n");
+			ret = -ENOMEM;
 			goto out_fail;
 		}
 		dev->cpu = cpu;
diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c
index f7ca891b5b59..7fe442ca38f4 100644
--- a/drivers/cpuidle/cpuidle-powernv.c
+++ b/drivers/cpuidle/cpuidle-powernv.c
@@ -119,40 +119,30 @@ static struct cpuidle_state powernv_states[CPUIDLE_STATE_MAX] = {
 		.enter = snooze_loop },
 };
 
-static int powernv_cpuidle_add_cpu_notifier(struct notifier_block *n,
-			unsigned long action, void *hcpu)
+static int powernv_cpuidle_cpu_online(unsigned int cpu)
 {
-	int hotcpu = (unsigned long)hcpu;
-	struct cpuidle_device *dev =
-				per_cpu(cpuidle_devices, hotcpu);
+	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
 
 	if (dev && cpuidle_get_driver()) {
-		switch (action) {
-		case CPU_ONLINE:
-		case CPU_ONLINE_FROZEN:
-			cpuidle_pause_and_lock();
-			cpuidle_enable_device(dev);
-			cpuidle_resume_and_unlock();
-			break;
+		cpuidle_pause_and_lock();
+		cpuidle_enable_device(dev);
+		cpuidle_resume_and_unlock();
+	}
+	return 0;
+}
 
-		case CPU_DEAD:
-		case CPU_DEAD_FROZEN:
-			cpuidle_pause_and_lock();
-			cpuidle_disable_device(dev);
-			cpuidle_resume_and_unlock();
-			break;
+static int powernv_cpuidle_cpu_dead(unsigned int cpu)
+{
+	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
 
-		default:
-			return NOTIFY_DONE;
-		}
+	if (dev && cpuidle_get_driver()) {
+		cpuidle_pause_and_lock();
+		cpuidle_disable_device(dev);
+		cpuidle_resume_and_unlock();
 	}
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block setup_hotplug_notifier = {
-	.notifier_call = powernv_cpuidle_add_cpu_notifier,
-};
-
 /*
  * powernv_cpuidle_driver_init()
  */
@@ -355,7 +345,14 @@ static int __init powernv_processor_idle_init(void)
 		return retval;
 	}
 
-	register_cpu_notifier(&setup_hotplug_notifier);
+	retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					   "cpuidle/powernv:online",
+					   powernv_cpuidle_cpu_online, NULL);
+	WARN_ON(retval < 0);
+	retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD,
+					   "cpuidle/powernv:dead", NULL,
+					   powernv_cpuidle_cpu_dead);
+	WARN_ON(retval < 0);
 	printk(KERN_DEBUG "powernv_idle_driver registered\n");
 	return 0;
 }
diff --git a/drivers/cpuidle/cpuidle-pseries.c b/drivers/cpuidle/cpuidle-pseries.c
index 07135e009d8b..166ccd711ec9 100644
--- a/drivers/cpuidle/cpuidle-pseries.c
+++ b/drivers/cpuidle/cpuidle-pseries.c
@@ -171,40 +171,30 @@ static struct cpuidle_state shared_states[] = {
 		.enter = &shared_cede_loop },
 };
 
-static int pseries_cpuidle_add_cpu_notifier(struct notifier_block *n,
-			unsigned long action, void *hcpu)
+static int pseries_cpuidle_cpu_online(unsigned int cpu)
 {
-	int hotcpu = (unsigned long)hcpu;
-	struct cpuidle_device *dev =
-				per_cpu(cpuidle_devices, hotcpu);
+	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
 
 	if (dev && cpuidle_get_driver()) {
-		switch (action) {
-		case CPU_ONLINE:
-		case CPU_ONLINE_FROZEN:
-			cpuidle_pause_and_lock();
-			cpuidle_enable_device(dev);
-			cpuidle_resume_and_unlock();
-			break;
+		cpuidle_pause_and_lock();
+		cpuidle_enable_device(dev);
+		cpuidle_resume_and_unlock();
+	}
+	return 0;
+}
 
-		case CPU_DEAD:
-		case CPU_DEAD_FROZEN:
-			cpuidle_pause_and_lock();
-			cpuidle_disable_device(dev);
-			cpuidle_resume_and_unlock();
-			break;
+static int pseries_cpuidle_cpu_dead(unsigned int cpu)
+{
+	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
 
-		default:
-			return NOTIFY_DONE;
-		}
+	if (dev && cpuidle_get_driver()) {
+		cpuidle_pause_and_lock();
+		cpuidle_disable_device(dev);
+		cpuidle_resume_and_unlock();
 	}
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block setup_hotplug_notifier = {
-	.notifier_call = pseries_cpuidle_add_cpu_notifier,
-};
-
 /*
  * pseries_cpuidle_driver_init()
  */
@@ -273,7 +263,14 @@ static int __init pseries_processor_idle_init(void)
 		return retval;
 	}
 
-	register_cpu_notifier(&setup_hotplug_notifier);
+	retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					   "cpuidle/pseries:online",
+					   pseries_cpuidle_cpu_online, NULL);
+	WARN_ON(retval < 0);
+	retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD,
+					   "cpuidle/pseries:DEAD", NULL,
+					   pseries_cpuidle_cpu_dead);
+	WARN_ON(retval < 0);
 	printk(KERN_DEBUG "pseries_idle_driver registered\n");
 	return 0;
 }
diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c
index 389ade4572be..ab264d393233 100644
--- a/drivers/cpuidle/driver.c
+++ b/drivers/cpuidle/driver.c
@@ -14,6 +14,7 @@
 #include <linux/cpuidle.h>
 #include <linux/cpumask.h>
 #include <linux/tick.h>
+#include <linux/cpu.h>
 
 #include "cpuidle.h"
 
@@ -178,8 +179,8 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv)
 }
 
 #ifdef CONFIG_ARCH_HAS_CPU_RELAX
-static int poll_idle(struct cpuidle_device *dev,
-		struct cpuidle_driver *drv, int index)
+static int __cpuidle poll_idle(struct cpuidle_device *dev,
+			       struct cpuidle_driver *drv, int index)
 {
 	local_irq_enable();
 	if (!current_set_polling_and_test()) {
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 1af94e2d1a25..4d2b81f2b223 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -318,6 +318,9 @@ config CRYPTO_DEV_OMAP_AES
 	select CRYPTO_AES
 	select CRYPTO_BLKCIPHER
 	select CRYPTO_ENGINE
+	select CRYPTO_CBC
+	select CRYPTO_ECB
+	select CRYPTO_CTR
 	help
 	  OMAP processors have AES module accelerator. Select this if you
 	  want to use the OMAP module for AES algorithms.
@@ -550,4 +553,6 @@ config CRYPTO_DEV_ROCKCHIP
 	  This driver interfaces with the hardware crypto accelerator.
 	  Supporting cbc/ecb chainmode, and aes/des/des3_ede cipher mode.
 
+source "drivers/crypto/chelsio/Kconfig"
+
 endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 3c6432dd09d9..ad7250fa1348 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -31,3 +31,4 @@ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
 obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
 obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/
 obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
+obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index ea8189f4b021..156aad167cd6 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -111,6 +111,42 @@
 #else
 #define debug(format, arg...)
 #endif
+
+#ifdef DEBUG
+#include <linux/highmem.h>
+
+static void dbg_dump_sg(const char *level, const char *prefix_str,
+			int prefix_type, int rowsize, int groupsize,
+			struct scatterlist *sg, size_t tlen, bool ascii,
+			bool may_sleep)
+{
+	struct scatterlist *it;
+	void *it_page;
+	size_t len;
+	void *buf;
+
+	for (it = sg; it != NULL && tlen > 0 ; it = sg_next(sg)) {
+		/*
+		 * make sure the scatterlist's page
+		 * has a valid virtual memory mapping
+		 */
+		it_page = kmap_atomic(sg_page(it));
+		if (unlikely(!it_page)) {
+			printk(KERN_ERR "dbg_dump_sg: kmap failed\n");
+			return;
+		}
+
+		buf = it_page + it->offset;
+		len = min(tlen, it->length);
+		print_hex_dump(level, prefix_str, prefix_type, rowsize,
+			       groupsize, buf, len, ascii);
+		tlen -= len;
+
+		kunmap_atomic(it_page);
+	}
+}
+#endif
+
 static struct list_head alg_list;
 
 struct caam_alg_entry {
@@ -227,8 +263,9 @@ static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
 	if (is_rfc3686) {
 		nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
 			       enckeylen);
-		append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
-				    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
+		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
+				   LDST_CLASS_IND_CCB |
+				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
 		append_move(desc,
 			    MOVE_SRC_OUTFIFO |
 			    MOVE_DEST_CLASS1CTX |
@@ -441,6 +478,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 			       OP_ALG_AAI_CTR_MOD128);
 	const bool is_rfc3686 = alg->caam.rfc3686;
 
+	if (!ctx->authsize)
+		return 0;
+
 	/* NULL encryption / decryption */
 	if (!ctx->enckeylen)
 		return aead_null_set_sh_desc(aead);
@@ -497,11 +537,10 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 
 	/* Load Counter into CONTEXT1 reg */
 	if (is_rfc3686)
-		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
-				    LDST_CLASS_1_CCB |
-				    LDST_SRCDST_BYTE_CONTEXT |
-				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				     LDST_OFFSET_SHIFT));
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
 
 	/* Class 1 operation */
 	append_operation(desc, ctx->class1_alg_type |
@@ -553,7 +592,10 @@ skip_enc:
 
 	/* Read and write assoclen bytes */
 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+	if (alg->caam.geniv)
+		append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize);
+	else
+		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 
 	/* Skip assoc data */
 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
@@ -562,13 +604,20 @@ skip_enc:
 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 			     KEY_VLF);
 
+	if (alg->caam.geniv) {
+		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
+				LDST_SRCDST_BYTE_CONTEXT |
+				(ctx1_iv_off << LDST_OFFSET_SHIFT));
+		append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
+			    (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
+	}
+
 	/* Load Counter into CONTEXT1 reg */
 	if (is_rfc3686)
-		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
-				    LDST_CLASS_1_CCB |
-				    LDST_SRCDST_BYTE_CONTEXT |
-				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				     LDST_OFFSET_SHIFT));
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
 
 	/* Choose operation */
 	if (ctr_mode)
@@ -614,7 +663,7 @@ skip_enc:
 		keys_fit_inline = true;
 
 	/* aead_givencrypt shared descriptor */
-	desc = ctx->sh_desc_givenc;
+	desc = ctx->sh_desc_enc;
 
 	/* Note: Context registers are saved. */
 	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
@@ -645,13 +694,13 @@ copy_iv:
 	append_operation(desc, ctx->class2_alg_type |
 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 
-	/* ivsize + cryptlen = seqoutlen - authsize */
-	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
-
 	/* Read and write assoclen bytes */
 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 
+	/* ivsize + cryptlen = seqoutlen - authsize */
+	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
+
 	/* Skip assoc data */
 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
 
@@ -669,11 +718,10 @@ copy_iv:
 
 	/* Load Counter into CONTEXT1 reg */
 	if (is_rfc3686)
-		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
-				    LDST_CLASS_1_CCB |
-				    LDST_SRCDST_BYTE_CONTEXT |
-				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				     LDST_OFFSET_SHIFT));
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
 
 	/* Class 1 operation */
 	append_operation(desc, ctx->class1_alg_type |
@@ -697,7 +745,7 @@ copy_iv:
 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
-	if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
+	if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
@@ -1464,7 +1512,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 	int ret = 0;
 	u32 *key_jump_cmd;
 	u32 *desc;
-	u32 *nonce;
+	u8 *nonce;
 	u32 geniv;
 	u32 ctx1_iv_off = 0;
 	const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
@@ -1517,9 +1565,10 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 
 	/* Load nonce into CONTEXT1 reg */
 	if (is_rfc3686) {
-		nonce = (u32 *)(key + keylen);
-		append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
-				    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
+		nonce = (u8 *)key + keylen;
+		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
+				   LDST_CLASS_IND_CCB |
+				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
 		append_move(desc, MOVE_WAITCOMP |
 			    MOVE_SRC_OUTFIFO |
 			    MOVE_DEST_CLASS1CTX |
@@ -1535,11 +1584,10 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 
 	/* Load counter into CONTEXT1 reg */
 	if (is_rfc3686)
-		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
-				    LDST_CLASS_1_CCB |
-				    LDST_SRCDST_BYTE_CONTEXT |
-				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				     LDST_OFFSET_SHIFT));
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
 
 	/* Load operation */
 	append_operation(desc, ctx->class1_alg_type |
@@ -1576,9 +1624,10 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 
 	/* Load nonce into CONTEXT1 reg */
 	if (is_rfc3686) {
-		nonce = (u32 *)(key + keylen);
-		append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
-				    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
+		nonce = (u8 *)key + keylen;
+		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
+				   LDST_CLASS_IND_CCB |
+				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
 		append_move(desc, MOVE_WAITCOMP |
 			    MOVE_SRC_OUTFIFO |
 			    MOVE_DEST_CLASS1CTX |
@@ -1594,11 +1643,10 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 
 	/* Load counter into CONTEXT1 reg */
 	if (is_rfc3686)
-		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
-				    LDST_CLASS_1_CCB |
-				    LDST_SRCDST_BYTE_CONTEXT |
-				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				     LDST_OFFSET_SHIFT));
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
 
 	/* Choose operation */
 	if (ctr_mode)
@@ -1639,9 +1687,10 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 
 	/* Load Nonce into CONTEXT1 reg */
 	if (is_rfc3686) {
-		nonce = (u32 *)(key + keylen);
-		append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
-				    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
+		nonce = (u8 *)key + keylen;
+		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
+				   LDST_CLASS_IND_CCB |
+				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
 		append_move(desc, MOVE_WAITCOMP |
 			    MOVE_SRC_OUTFIFO |
 			    MOVE_DEST_CLASS1CTX |
@@ -1671,11 +1720,10 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 
 	/* Load Counter into CONTEXT1 reg */
 	if (is_rfc3686)
-		append_load_imm_u32(desc, (u32)1, LDST_IMM |
-				    LDST_CLASS_1_CCB |
-				    LDST_SRCDST_BYTE_CONTEXT |
-				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				     LDST_OFFSET_SHIFT));
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
 
 	if (ctx1_iv_off)
 		append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
@@ -1981,9 +2029,9 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 	print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       edesc->src_nents > 1 ? 100 : ivsize, 1);
-	print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
-		       edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
+	dbg_dump_sg(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
+		    DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
+		    edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true);
 #endif
 
 	ablkcipher_unmap(jrdev, edesc, req);
@@ -2013,9 +2061,9 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 	print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       ivsize, 1);
-	print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
-		       edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
+	dbg_dump_sg(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
+		    DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
+		    edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true);
 #endif
 
 	ablkcipher_unmap(jrdev, edesc, req);
@@ -2147,7 +2195,7 @@ static void init_authenc_job(struct aead_request *req,
 
 	init_aead_job(req, edesc, all_contig, encrypt);
 
-	if (ivsize && (is_rfc3686 || !(alg->caam.geniv && encrypt)))
+	if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
 		append_load_as_imm(desc, req->iv, ivsize,
 				   LDST_CLASS_1_CCB |
 				   LDST_SRCDST_BYTE_CONTEXT |
@@ -2170,12 +2218,15 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
 	int len, sec4_sg_index = 0;
 
 #ifdef DEBUG
+	bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
+					      CRYPTO_TFM_REQ_MAY_SLEEP)) != 0);
 	print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       ivsize, 1);
-	print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
-		       edesc->src_nents ? 100 : req->nbytes, 1);
+	printk(KERN_ERR "asked=%d, nbytes%d\n", (int)edesc->src_nents ? 100 : req->nbytes, req->nbytes);
+	dbg_dump_sg(KERN_ERR, "src    @"__stringify(__LINE__)": ",
+		    DUMP_PREFIX_ADDRESS, 16, 4, req->src,
+		    edesc->src_nents ? 100 : req->nbytes, 1, may_sleep);
 #endif
 
 	len = desc_len(sh_desc);
@@ -2227,12 +2278,14 @@ static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
 	int len, sec4_sg_index = 0;
 
 #ifdef DEBUG
+	bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
+					      CRYPTO_TFM_REQ_MAY_SLEEP)) != 0);
 	print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       ivsize, 1);
-	print_hex_dump(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
-		       edesc->src_nents ? 100 : req->nbytes, 1);
+	dbg_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
+		    DUMP_PREFIX_ADDRESS, 16, 4, req->src,
+		    edesc->src_nents ? 100 : req->nbytes, 1, may_sleep);
 #endif
 
 	len = desc_len(sh_desc);
@@ -2502,18 +2555,20 @@ static int aead_decrypt(struct aead_request *req)
 	u32 *desc;
 	int ret = 0;
 
+#ifdef DEBUG
+	bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
+					      CRYPTO_TFM_REQ_MAY_SLEEP)) != 0);
+	dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
+		    DUMP_PREFIX_ADDRESS, 16, 4, req->src,
+		    req->assoclen + req->cryptlen, 1, may_sleep);
+#endif
+
 	/* allocate extended descriptor */
 	edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
 				 &all_contig, false);
 	if (IS_ERR(edesc))
 		return PTR_ERR(edesc);
 
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
-		       req->assoclen + req->cryptlen, 1);
-#endif
-
 	/* Create and submit job descriptor*/
 	init_authenc_job(req, edesc, all_contig, false);
 #ifdef DEBUG
@@ -2534,20 +2589,6 @@ static int aead_decrypt(struct aead_request *req)
 	return ret;
 }
 
-static int aead_givdecrypt(struct aead_request *req)
-{
-	struct crypto_aead *aead = crypto_aead_reqtfm(req);
-	unsigned int ivsize = crypto_aead_ivsize(aead);
-
-	if (req->cryptlen < ivsize)
-		return -EINVAL;
-
-	req->cryptlen -= ivsize;
-	req->assoclen += ivsize;
-
-	return aead_decrypt(req);
-}
-
 /*
  * allocate and map the ablkcipher extended descriptor for ablkcipher
  */
@@ -3207,7 +3248,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = AES_BLOCK_SIZE,
 			.maxauthsize = MD5_DIGEST_SIZE,
 		},
@@ -3253,7 +3294,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = AES_BLOCK_SIZE,
 			.maxauthsize = SHA1_DIGEST_SIZE,
 		},
@@ -3299,7 +3340,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = AES_BLOCK_SIZE,
 			.maxauthsize = SHA224_DIGEST_SIZE,
 		},
@@ -3345,7 +3386,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = AES_BLOCK_SIZE,
 			.maxauthsize = SHA256_DIGEST_SIZE,
 		},
@@ -3391,7 +3432,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = AES_BLOCK_SIZE,
 			.maxauthsize = SHA384_DIGEST_SIZE,
 		},
@@ -3437,7 +3478,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = AES_BLOCK_SIZE,
 			.maxauthsize = SHA512_DIGEST_SIZE,
 		},
@@ -3483,7 +3524,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES3_EDE_BLOCK_SIZE,
 			.maxauthsize = MD5_DIGEST_SIZE,
 		},
@@ -3531,7 +3572,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES3_EDE_BLOCK_SIZE,
 			.maxauthsize = SHA1_DIGEST_SIZE,
 		},
@@ -3579,7 +3620,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES3_EDE_BLOCK_SIZE,
 			.maxauthsize = SHA224_DIGEST_SIZE,
 		},
@@ -3627,7 +3668,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES3_EDE_BLOCK_SIZE,
 			.maxauthsize = SHA256_DIGEST_SIZE,
 		},
@@ -3675,7 +3716,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES3_EDE_BLOCK_SIZE,
 			.maxauthsize = SHA384_DIGEST_SIZE,
 		},
@@ -3723,7 +3764,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES3_EDE_BLOCK_SIZE,
 			.maxauthsize = SHA512_DIGEST_SIZE,
 		},
@@ -3769,7 +3810,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES_BLOCK_SIZE,
 			.maxauthsize = MD5_DIGEST_SIZE,
 		},
@@ -3815,7 +3856,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES_BLOCK_SIZE,
 			.maxauthsize = SHA1_DIGEST_SIZE,
 		},
@@ -3861,7 +3902,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES_BLOCK_SIZE,
 			.maxauthsize = SHA224_DIGEST_SIZE,
 		},
@@ -3907,7 +3948,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES_BLOCK_SIZE,
 			.maxauthsize = SHA256_DIGEST_SIZE,
 		},
@@ -3953,7 +3994,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES_BLOCK_SIZE,
 			.maxauthsize = SHA384_DIGEST_SIZE,
 		},
@@ -3999,7 +4040,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = DES_BLOCK_SIZE,
 			.maxauthsize = SHA512_DIGEST_SIZE,
 		},
@@ -4048,7 +4089,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = CTR_RFC3686_IV_SIZE,
 			.maxauthsize = MD5_DIGEST_SIZE,
 		},
@@ -4099,7 +4140,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = CTR_RFC3686_IV_SIZE,
 			.maxauthsize = SHA1_DIGEST_SIZE,
 		},
@@ -4150,7 +4191,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = CTR_RFC3686_IV_SIZE,
 			.maxauthsize = SHA224_DIGEST_SIZE,
 		},
@@ -4201,7 +4242,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = CTR_RFC3686_IV_SIZE,
 			.maxauthsize = SHA256_DIGEST_SIZE,
 		},
@@ -4252,7 +4293,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = CTR_RFC3686_IV_SIZE,
 			.maxauthsize = SHA384_DIGEST_SIZE,
 		},
@@ -4303,7 +4344,7 @@ static struct caam_aead_alg driver_aeads[] = {
 			.setkey = aead_setkey,
 			.setauthsize = aead_setauthsize,
 			.encrypt = aead_encrypt,
-			.decrypt = aead_givdecrypt,
+			.decrypt = aead_decrypt,
 			.ivsize = CTR_RFC3686_IV_SIZE,
 			.maxauthsize = SHA512_DIGEST_SIZE,
 		},
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index f1ecc8df8d41..660dc206969f 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -99,17 +99,17 @@ static struct list_head hash_list;
 
 /* ahash per-session context */
 struct caam_hash_ctx {
-	struct device *jrdev;
-	u32 sh_desc_update[DESC_HASH_MAX_USED_LEN];
-	u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN];
-	u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN];
-	u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN];
-	u32 sh_desc_finup[DESC_HASH_MAX_USED_LEN];
-	dma_addr_t sh_desc_update_dma;
+	u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
+	u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
+	u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
+	u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
+	u32 sh_desc_finup[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
+	dma_addr_t sh_desc_update_dma ____cacheline_aligned;
 	dma_addr_t sh_desc_update_first_dma;
 	dma_addr_t sh_desc_fin_dma;
 	dma_addr_t sh_desc_digest_dma;
 	dma_addr_t sh_desc_finup_dma;
+	struct device *jrdev;
 	u32 alg_type;
 	u32 alg_op;
 	u8 key[CAAM_MAX_HASH_KEY_SIZE];
@@ -187,15 +187,6 @@ static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev,
 	return buf_dma;
 }
 
-/* Map req->src and put it in link table */
-static inline void src_map_to_sec4_sg(struct device *jrdev,
-				      struct scatterlist *src, int src_nents,
-				      struct sec4_sg_entry *sec4_sg)
-{
-	dma_map_sg(jrdev, src, src_nents, DMA_TO_DEVICE);
-	sg_to_sec4_sg_last(src, src_nents, sec4_sg, 0);
-}
-
 /*
  * Only put buffer in link table if it contains data, which is possible,
  * since a buffer has previously been used, and needs to be unmapped,
@@ -449,7 +440,7 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 	u32 *desc;
 	struct split_key_result result;
 	dma_addr_t src_dma, dst_dma;
-	int ret = 0;
+	int ret;
 
 	desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
 	if (!desc) {
@@ -526,7 +517,7 @@ static int ahash_setkey(struct crypto_ahash *ahash,
 	struct device *jrdev = ctx->jrdev;
 	int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
 	int digestsize = crypto_ahash_digestsize(ahash);
-	int ret = 0;
+	int ret;
 	u8 *hashed_key = NULL;
 
 #ifdef DEBUG
@@ -534,14 +525,15 @@ static int ahash_setkey(struct crypto_ahash *ahash,
 #endif
 
 	if (keylen > blocksize) {
-		hashed_key = kmalloc(sizeof(u8) * digestsize, GFP_KERNEL |
-				     GFP_DMA);
+		hashed_key = kmalloc_array(digestsize,
+					   sizeof(*hashed_key),
+					   GFP_KERNEL | GFP_DMA);
 		if (!hashed_key)
 			return -ENOMEM;
 		ret = hash_digest_key(ctx, key, &keylen, hashed_key,
 				      digestsize);
 		if (ret)
-			goto badkey;
+			goto bad_free_key;
 		key = hashed_key;
 	}
 
@@ -559,14 +551,14 @@ static int ahash_setkey(struct crypto_ahash *ahash,
 
 	ret = gen_split_hash_key(ctx, key, keylen);
 	if (ret)
-		goto badkey;
+		goto bad_free_key;
 
 	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len,
 				      DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
 		dev_err(jrdev, "unable to map key i/o memory\n");
 		ret = -ENOMEM;
-		goto map_err;
+		goto error_free_key;
 	}
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
@@ -579,11 +571,10 @@ static int ahash_setkey(struct crypto_ahash *ahash,
 		dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len,
 				 DMA_TO_DEVICE);
 	}
-
-map_err:
+ error_free_key:
 	kfree(hashed_key);
 	return ret;
-badkey:
+ bad_free_key:
 	kfree(hashed_key);
 	crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
 	return -EINVAL;
@@ -595,16 +586,16 @@ badkey:
  * @sec4_sg_dma: physical mapped address of h/w link table
  * @src_nents: number of segments in input scatterlist
  * @sec4_sg_bytes: length of dma mapped sec4_sg space
- * @sec4_sg: pointer to h/w link table
  * @hw_desc: the h/w job descriptor followed by any referenced link tables
+ * @sec4_sg: h/w link table
  */
 struct ahash_edesc {
 	dma_addr_t dst_dma;
 	dma_addr_t sec4_sg_dma;
 	int src_nents;
 	int sec4_sg_bytes;
-	struct sec4_sg_entry *sec4_sg;
-	u32 hw_desc[0];
+	u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
+	struct sec4_sg_entry sec4_sg[0];
 };
 
 static inline void ahash_unmap(struct device *dev,
@@ -774,6 +765,65 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
 	req->base.complete(&req->base, err);
 }
 
+/*
+ * Allocate an enhanced descriptor, which contains the hardware descriptor
+ * and space for hardware scatter table containing sg_num entries.
+ */
+static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx,
+					     int sg_num, u32 *sh_desc,
+					     dma_addr_t sh_desc_dma,
+					     gfp_t flags)
+{
+	struct ahash_edesc *edesc;
+	unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
+
+	edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
+	if (!edesc) {
+		dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
+		return NULL;
+	}
+
+	init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
+			     HDR_SHARE_DEFER | HDR_REVERSE);
+
+	return edesc;
+}
+
+static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
+			       struct ahash_edesc *edesc,
+			       struct ahash_request *req, int nents,
+			       unsigned int first_sg,
+			       unsigned int first_bytes, size_t to_hash)
+{
+	dma_addr_t src_dma;
+	u32 options;
+
+	if (nents > 1 || first_sg) {
+		struct sec4_sg_entry *sg = edesc->sec4_sg;
+		unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
+
+		sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
+
+		src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
+		if (dma_mapping_error(ctx->jrdev, src_dma)) {
+			dev_err(ctx->jrdev, "unable to map S/G table\n");
+			return -ENOMEM;
+		}
+
+		edesc->sec4_sg_bytes = sgsize;
+		edesc->sec4_sg_dma = src_dma;
+		options = LDST_SGF;
+	} else {
+		src_dma = sg_dma_address(req->src);
+		options = 0;
+	}
+
+	append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
+			  options);
+
+	return 0;
+}
+
 /* submit update job descriptor */
 static int ahash_update_ctx(struct ahash_request *req)
 {
@@ -789,12 +839,10 @@ static int ahash_update_ctx(struct ahash_request *req)
 	int *next_buflen = state->current_buf ? &state->buflen_0 :
 			   &state->buflen_1, last_buflen;
 	int in_len = *buflen + req->nbytes, to_hash;
-	u32 *sh_desc = ctx->sh_desc_update, *desc;
-	dma_addr_t ptr = ctx->sh_desc_update_dma;
-	int src_nents, sec4_sg_bytes, sec4_sg_src_index;
+	u32 *desc;
+	int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
 	struct ahash_edesc *edesc;
 	int ret = 0;
-	int sh_len;
 
 	last_buflen = *next_buflen;
 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
@@ -807,40 +855,51 @@ static int ahash_update_ctx(struct ahash_request *req)
 			dev_err(jrdev, "Invalid number of src SG.\n");
 			return src_nents;
 		}
+
+		if (src_nents) {
+			mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
+						  DMA_TO_DEVICE);
+			if (!mapped_nents) {
+				dev_err(jrdev, "unable to DMA map source\n");
+				return -ENOMEM;
+			}
+		} else {
+			mapped_nents = 0;
+		}
+
 		sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
-		sec4_sg_bytes = (sec4_sg_src_index + src_nents) *
+		sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
 				 sizeof(struct sec4_sg_entry);
 
 		/*
 		 * allocate space for base edesc and hw desc commands,
 		 * link tables
 		 */
-		edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN +
-				sec4_sg_bytes, GFP_DMA | flags);
+		edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
+					  ctx->sh_desc_update,
+					  ctx->sh_desc_update_dma, flags);
 		if (!edesc) {
-			dev_err(jrdev,
-				"could not allocate extended descriptor\n");
+			dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 			return -ENOMEM;
 		}
 
 		edesc->src_nents = src_nents;
 		edesc->sec4_sg_bytes = sec4_sg_bytes;
-		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
-				 DESC_JOB_IO_LEN;
 
 		ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 					 edesc->sec4_sg, DMA_BIDIRECTIONAL);
 		if (ret)
-			return ret;
+			goto unmap_ctx;
 
 		state->buf_dma = try_buf_map_to_sec4_sg(jrdev,
 							edesc->sec4_sg + 1,
 							buf, state->buf_dma,
 							*buflen, last_buflen);
 
-		if (src_nents) {
-			src_map_to_sec4_sg(jrdev, req->src, src_nents,
-					   edesc->sec4_sg + sec4_sg_src_index);
+		if (mapped_nents) {
+			sg_to_sec4_sg_last(req->src, mapped_nents,
+					   edesc->sec4_sg + sec4_sg_src_index,
+					   0);
 			if (*next_buflen)
 				scatterwalk_map_and_copy(next_buf, req->src,
 							 to_hash - *buflen,
@@ -852,17 +911,15 @@ static int ahash_update_ctx(struct ahash_request *req)
 
 		state->current_buf = !state->current_buf;
 
-		sh_len = desc_len(sh_desc);
 		desc = edesc->hw_desc;
-		init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER |
-				     HDR_REVERSE);
 
 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 						     sec4_sg_bytes,
 						     DMA_TO_DEVICE);
 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 			dev_err(jrdev, "unable to map S/G table\n");
-			return -ENOMEM;
+			ret = -ENOMEM;
+			goto unmap_ctx;
 		}
 
 		append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
@@ -877,13 +934,10 @@ static int ahash_update_ctx(struct ahash_request *req)
 #endif
 
 		ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
-		if (!ret) {
-			ret = -EINPROGRESS;
-		} else {
-			ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,
-					   DMA_BIDIRECTIONAL);
-			kfree(edesc);
-		}
+		if (ret)
+			goto unmap_ctx;
+
+		ret = -EINPROGRESS;
 	} else if (*next_buflen) {
 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
 					 req->nbytes, 0);
@@ -899,6 +953,10 @@ static int ahash_update_ctx(struct ahash_request *req)
 #endif
 
 	return ret;
+ unmap_ctx:
+	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
+	kfree(edesc);
+	return ret;
 }
 
 static int ahash_final_ctx(struct ahash_request *req)
@@ -913,38 +971,31 @@ static int ahash_final_ctx(struct ahash_request *req)
 	int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
 	int last_buflen = state->current_buf ? state->buflen_0 :
 			  state->buflen_1;
-	u32 *sh_desc = ctx->sh_desc_fin, *desc;
-	dma_addr_t ptr = ctx->sh_desc_fin_dma;
+	u32 *desc;
 	int sec4_sg_bytes, sec4_sg_src_index;
 	int digestsize = crypto_ahash_digestsize(ahash);
 	struct ahash_edesc *edesc;
-	int ret = 0;
-	int sh_len;
+	int ret;
 
 	sec4_sg_src_index = 1 + (buflen ? 1 : 0);
 	sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
 
 	/* allocate space for base edesc and hw desc commands, link tables */
-	edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes,
-			GFP_DMA | flags);
-	if (!edesc) {
-		dev_err(jrdev, "could not allocate extended descriptor\n");
+	edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
+				  ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
+				  flags);
+	if (!edesc)
 		return -ENOMEM;
-	}
 
-	sh_len = desc_len(sh_desc);
 	desc = edesc->hw_desc;
-	init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
 
 	edesc->sec4_sg_bytes = sec4_sg_bytes;
-	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
-			 DESC_JOB_IO_LEN;
 	edesc->src_nents = 0;
 
 	ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 				 edesc->sec4_sg, DMA_TO_DEVICE);
 	if (ret)
-		return ret;
+		goto unmap_ctx;
 
 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,
 						buf, state->buf_dma, buflen,
@@ -956,7 +1007,8 @@ static int ahash_final_ctx(struct ahash_request *req)
 					    sec4_sg_bytes, DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 		dev_err(jrdev, "unable to map S/G table\n");
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto unmap_ctx;
 	}
 
 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
@@ -966,7 +1018,8 @@ static int ahash_final_ctx(struct ahash_request *req)
 						digestsize);
 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 		dev_err(jrdev, "unable to map dst\n");
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto unmap_ctx;
 	}
 
 #ifdef DEBUG
@@ -975,13 +1028,13 @@ static int ahash_final_ctx(struct ahash_request *req)
 #endif
 
 	ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
-	if (!ret) {
-		ret = -EINPROGRESS;
-	} else {
-		ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
-		kfree(edesc);
-	}
+	if (ret)
+		goto unmap_ctx;
 
+	return -EINPROGRESS;
+ unmap_ctx:
+	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
+	kfree(edesc);
 	return ret;
 }
 
@@ -997,68 +1050,66 @@ static int ahash_finup_ctx(struct ahash_request *req)
 	int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
 	int last_buflen = state->current_buf ? state->buflen_0 :
 			  state->buflen_1;
-	u32 *sh_desc = ctx->sh_desc_finup, *desc;
-	dma_addr_t ptr = ctx->sh_desc_finup_dma;
-	int sec4_sg_bytes, sec4_sg_src_index;
-	int src_nents;
+	u32 *desc;
+	int sec4_sg_src_index;
+	int src_nents, mapped_nents;
 	int digestsize = crypto_ahash_digestsize(ahash);
 	struct ahash_edesc *edesc;
-	int ret = 0;
-	int sh_len;
+	int ret;
 
 	src_nents = sg_nents_for_len(req->src, req->nbytes);
 	if (src_nents < 0) {
 		dev_err(jrdev, "Invalid number of src SG.\n");
 		return src_nents;
 	}
+
+	if (src_nents) {
+		mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
+					  DMA_TO_DEVICE);
+		if (!mapped_nents) {
+			dev_err(jrdev, "unable to DMA map source\n");
+			return -ENOMEM;
+		}
+	} else {
+		mapped_nents = 0;
+	}
+
 	sec4_sg_src_index = 1 + (buflen ? 1 : 0);
-	sec4_sg_bytes = (sec4_sg_src_index + src_nents) *
-			 sizeof(struct sec4_sg_entry);
 
 	/* allocate space for base edesc and hw desc commands, link tables */
-	edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes,
-			GFP_DMA | flags);
+	edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
+				  ctx->sh_desc_finup, ctx->sh_desc_finup_dma,
+				  flags);
 	if (!edesc) {
-		dev_err(jrdev, "could not allocate extended descriptor\n");
+		dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 		return -ENOMEM;
 	}
 
-	sh_len = desc_len(sh_desc);
 	desc = edesc->hw_desc;
-	init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
 
 	edesc->src_nents = src_nents;
-	edesc->sec4_sg_bytes = sec4_sg_bytes;
-	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
-			 DESC_JOB_IO_LEN;
 
 	ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
 				 edesc->sec4_sg, DMA_TO_DEVICE);
 	if (ret)
-		return ret;
+		goto unmap_ctx;
 
 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,
 						buf, state->buf_dma, buflen,
 						last_buflen);
 
-	src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg +
-			   sec4_sg_src_index);
-
-	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
-					    sec4_sg_bytes, DMA_TO_DEVICE);
-	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
-		dev_err(jrdev, "unable to map S/G table\n");
-		return -ENOMEM;
-	}
-
-	append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
-			       buflen + req->nbytes, LDST_SGF);
+	ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
+				  sec4_sg_src_index, ctx->ctx_len + buflen,
+				  req->nbytes);
+	if (ret)
+		goto unmap_ctx;
 
 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
 						digestsize);
 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 		dev_err(jrdev, "unable to map dst\n");
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto unmap_ctx;
 	}
 
 #ifdef DEBUG
@@ -1067,13 +1118,13 @@ static int ahash_finup_ctx(struct ahash_request *req)
 #endif
 
 	ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
-	if (!ret) {
-		ret = -EINPROGRESS;
-	} else {
-		ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
-		kfree(edesc);
-	}
+	if (ret)
+		goto unmap_ctx;
 
+	return -EINPROGRESS;
+ unmap_ctx:
+	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
+	kfree(edesc);
 	return ret;
 }
 
@@ -1084,60 +1135,56 @@ static int ahash_digest(struct ahash_request *req)
 	struct device *jrdev = ctx->jrdev;
 	gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
-	u32 *sh_desc = ctx->sh_desc_digest, *desc;
-	dma_addr_t ptr = ctx->sh_desc_digest_dma;
+	u32 *desc;
 	int digestsize = crypto_ahash_digestsize(ahash);
-	int src_nents, sec4_sg_bytes;
-	dma_addr_t src_dma;
+	int src_nents, mapped_nents;
 	struct ahash_edesc *edesc;
-	int ret = 0;
-	u32 options;
-	int sh_len;
+	int ret;
 
-	src_nents = sg_count(req->src, req->nbytes);
+	src_nents = sg_nents_for_len(req->src, req->nbytes);
 	if (src_nents < 0) {
 		dev_err(jrdev, "Invalid number of src SG.\n");
 		return src_nents;
 	}
-	dma_map_sg(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE);
-	sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry);
+
+	if (src_nents) {
+		mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
+					  DMA_TO_DEVICE);
+		if (!mapped_nents) {
+			dev_err(jrdev, "unable to map source for DMA\n");
+			return -ENOMEM;
+		}
+	} else {
+		mapped_nents = 0;
+	}
 
 	/* allocate space for base edesc and hw desc commands, link tables */
-	edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes + DESC_JOB_IO_LEN,
-			GFP_DMA | flags);
+	edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0,
+				  ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
+				  flags);
 	if (!edesc) {
-		dev_err(jrdev, "could not allocate extended descriptor\n");
+		dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 		return -ENOMEM;
 	}
-	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
-			  DESC_JOB_IO_LEN;
-	edesc->sec4_sg_bytes = sec4_sg_bytes;
-	edesc->src_nents = src_nents;
 
-	sh_len = desc_len(sh_desc);
-	desc = edesc->hw_desc;
-	init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
+	edesc->src_nents = src_nents;
 
-	if (src_nents) {
-		sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
-		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
-					    sec4_sg_bytes, DMA_TO_DEVICE);
-		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
-			dev_err(jrdev, "unable to map S/G table\n");
-			return -ENOMEM;
-		}
-		src_dma = edesc->sec4_sg_dma;
-		options = LDST_SGF;
-	} else {
-		src_dma = sg_dma_address(req->src);
-		options = 0;
+	ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
+				  req->nbytes);
+	if (ret) {
+		ahash_unmap(jrdev, edesc, req, digestsize);
+		kfree(edesc);
+		return ret;
 	}
-	append_seq_in_ptr(desc, src_dma, req->nbytes, options);
+
+	desc = edesc->hw_desc;
 
 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
 						digestsize);
 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 		dev_err(jrdev, "unable to map dst\n");
+		ahash_unmap(jrdev, edesc, req, digestsize);
+		kfree(edesc);
 		return -ENOMEM;
 	}
 
@@ -1168,29 +1215,23 @@ static int ahash_final_no_ctx(struct ahash_request *req)
 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
 	u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
 	int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
-	u32 *sh_desc = ctx->sh_desc_digest, *desc;
-	dma_addr_t ptr = ctx->sh_desc_digest_dma;
+	u32 *desc;
 	int digestsize = crypto_ahash_digestsize(ahash);
 	struct ahash_edesc *edesc;
-	int ret = 0;
-	int sh_len;
+	int ret;
 
 	/* allocate space for base edesc and hw desc commands, link tables */
-	edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | flags);
-	if (!edesc) {
-		dev_err(jrdev, "could not allocate extended descriptor\n");
+	edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest,
+				  ctx->sh_desc_digest_dma, flags);
+	if (!edesc)
 		return -ENOMEM;
-	}
 
-	edesc->sec4_sg_bytes = 0;
-	sh_len = desc_len(sh_desc);
 	desc = edesc->hw_desc;
-	init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
 
 	state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, state->buf_dma)) {
 		dev_err(jrdev, "unable to map src\n");
-		return -ENOMEM;
+		goto unmap;
 	}
 
 	append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
@@ -1199,7 +1240,7 @@ static int ahash_final_no_ctx(struct ahash_request *req)
 						digestsize);
 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 		dev_err(jrdev, "unable to map dst\n");
-		return -ENOMEM;
+		goto unmap;
 	}
 	edesc->src_nents = 0;
 
@@ -1217,6 +1258,11 @@ static int ahash_final_no_ctx(struct ahash_request *req)
 	}
 
 	return ret;
+ unmap:
+	ahash_unmap(jrdev, edesc, req, digestsize);
+	kfree(edesc);
+	return -ENOMEM;
+
 }
 
 /* submit ahash update if it the first job descriptor after update */
@@ -1234,48 +1280,58 @@ static int ahash_update_no_ctx(struct ahash_request *req)
 	int *next_buflen = state->current_buf ? &state->buflen_0 :
 			   &state->buflen_1;
 	int in_len = *buflen + req->nbytes, to_hash;
-	int sec4_sg_bytes, src_nents;
+	int sec4_sg_bytes, src_nents, mapped_nents;
 	struct ahash_edesc *edesc;
-	u32 *desc, *sh_desc = ctx->sh_desc_update_first;
-	dma_addr_t ptr = ctx->sh_desc_update_first_dma;
+	u32 *desc;
 	int ret = 0;
-	int sh_len;
 
 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
 	to_hash = in_len - *next_buflen;
 
 	if (to_hash) {
 		src_nents = sg_nents_for_len(req->src,
-					     req->nbytes - (*next_buflen));
+					     req->nbytes - *next_buflen);
 		if (src_nents < 0) {
 			dev_err(jrdev, "Invalid number of src SG.\n");
 			return src_nents;
 		}
-		sec4_sg_bytes = (1 + src_nents) *
+
+		if (src_nents) {
+			mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
+						  DMA_TO_DEVICE);
+			if (!mapped_nents) {
+				dev_err(jrdev, "unable to DMA map source\n");
+				return -ENOMEM;
+			}
+		} else {
+			mapped_nents = 0;
+		}
+
+		sec4_sg_bytes = (1 + mapped_nents) *
 				sizeof(struct sec4_sg_entry);
 
 		/*
 		 * allocate space for base edesc and hw desc commands,
 		 * link tables
 		 */
-		edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN +
-				sec4_sg_bytes, GFP_DMA | flags);
+		edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
+					  ctx->sh_desc_update_first,
+					  ctx->sh_desc_update_first_dma,
+					  flags);
 		if (!edesc) {
-			dev_err(jrdev,
-				"could not allocate extended descriptor\n");
+			dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 			return -ENOMEM;
 		}
 
 		edesc->src_nents = src_nents;
 		edesc->sec4_sg_bytes = sec4_sg_bytes;
-		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
-				 DESC_JOB_IO_LEN;
 		edesc->dst_dma = 0;
 
 		state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg,
 						    buf, *buflen);
-		src_map_to_sec4_sg(jrdev, req->src, src_nents,
-				   edesc->sec4_sg + 1);
+		sg_to_sec4_sg_last(req->src, mapped_nents,
+				   edesc->sec4_sg + 1, 0);
+
 		if (*next_buflen) {
 			scatterwalk_map_and_copy(next_buf, req->src,
 						 to_hash - *buflen,
@@ -1284,24 +1340,22 @@ static int ahash_update_no_ctx(struct ahash_request *req)
 
 		state->current_buf = !state->current_buf;
 
-		sh_len = desc_len(sh_desc);
 		desc = edesc->hw_desc;
-		init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER |
-				     HDR_REVERSE);
 
 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 						    sec4_sg_bytes,
 						    DMA_TO_DEVICE);
 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 			dev_err(jrdev, "unable to map S/G table\n");
-			return -ENOMEM;
+			ret = -ENOMEM;
+			goto unmap_ctx;
 		}
 
 		append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
 
 		ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
 		if (ret)
-			return ret;
+			goto unmap_ctx;
 
 #ifdef DEBUG
 		print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
@@ -1310,16 +1364,13 @@ static int ahash_update_no_ctx(struct ahash_request *req)
 #endif
 
 		ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
-		if (!ret) {
-			ret = -EINPROGRESS;
-			state->update = ahash_update_ctx;
-			state->finup = ahash_finup_ctx;
-			state->final = ahash_final_ctx;
-		} else {
-			ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,
-					DMA_TO_DEVICE);
-			kfree(edesc);
-		}
+		if (ret)
+			goto unmap_ctx;
+
+		ret = -EINPROGRESS;
+		state->update = ahash_update_ctx;
+		state->finup = ahash_finup_ctx;
+		state->final = ahash_final_ctx;
 	} else if (*next_buflen) {
 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
 					 req->nbytes, 0);
@@ -1335,6 +1386,10 @@ static int ahash_update_no_ctx(struct ahash_request *req)
 #endif
 
 	return ret;
+ unmap_ctx:
+	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
+	kfree(edesc);
+	return ret;
 }
 
 /* submit ahash finup if it the first job descriptor after update */
@@ -1350,61 +1405,63 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
 	int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
 	int last_buflen = state->current_buf ? state->buflen_0 :
 			  state->buflen_1;
-	u32 *sh_desc = ctx->sh_desc_digest, *desc;
-	dma_addr_t ptr = ctx->sh_desc_digest_dma;
-	int sec4_sg_bytes, sec4_sg_src_index, src_nents;
+	u32 *desc;
+	int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
 	int digestsize = crypto_ahash_digestsize(ahash);
 	struct ahash_edesc *edesc;
-	int sh_len;
-	int ret = 0;
+	int ret;
 
 	src_nents = sg_nents_for_len(req->src, req->nbytes);
 	if (src_nents < 0) {
 		dev_err(jrdev, "Invalid number of src SG.\n");
 		return src_nents;
 	}
+
+	if (src_nents) {
+		mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
+					  DMA_TO_DEVICE);
+		if (!mapped_nents) {
+			dev_err(jrdev, "unable to DMA map source\n");
+			return -ENOMEM;
+		}
+	} else {
+		mapped_nents = 0;
+	}
+
 	sec4_sg_src_index = 2;
-	sec4_sg_bytes = (sec4_sg_src_index + src_nents) *
+	sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
 			 sizeof(struct sec4_sg_entry);
 
 	/* allocate space for base edesc and hw desc commands, link tables */
-	edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes,
-			GFP_DMA | flags);
+	edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
+				  ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
+				  flags);
 	if (!edesc) {
-		dev_err(jrdev, "could not allocate extended descriptor\n");
+		dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 		return -ENOMEM;
 	}
 
-	sh_len = desc_len(sh_desc);
 	desc = edesc->hw_desc;
-	init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
 
 	edesc->src_nents = src_nents;
 	edesc->sec4_sg_bytes = sec4_sg_bytes;
-	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
-			 DESC_JOB_IO_LEN;
 
 	state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf,
 						state->buf_dma, buflen,
 						last_buflen);
 
-	src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1);
-
-	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
-					    sec4_sg_bytes, DMA_TO_DEVICE);
-	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
+	ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
+				  req->nbytes);
+	if (ret) {
 		dev_err(jrdev, "unable to map S/G table\n");
-		return -ENOMEM;
+		goto unmap;
 	}
 
-	append_seq_in_ptr(desc, edesc->sec4_sg_dma, buflen +
-			       req->nbytes, LDST_SGF);
-
 	edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
 						digestsize);
 	if (dma_mapping_error(jrdev, edesc->dst_dma)) {
 		dev_err(jrdev, "unable to map dst\n");
-		return -ENOMEM;
+		goto unmap;
 	}
 
 #ifdef DEBUG
@@ -1421,6 +1478,11 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
 	}
 
 	return ret;
+ unmap:
+	ahash_unmap(jrdev, edesc, req, digestsize);
+	kfree(edesc);
+	return -ENOMEM;
+
 }
 
 /* submit first update job descriptor after init */
@@ -1436,78 +1498,65 @@ static int ahash_update_first(struct ahash_request *req)
 	int *next_buflen = state->current_buf ?
 		&state->buflen_1 : &state->buflen_0;
 	int to_hash;
-	u32 *sh_desc = ctx->sh_desc_update_first, *desc;
-	dma_addr_t ptr = ctx->sh_desc_update_first_dma;
-	int sec4_sg_bytes, src_nents;
-	dma_addr_t src_dma;
-	u32 options;
+	u32 *desc;
+	int src_nents, mapped_nents;
 	struct ahash_edesc *edesc;
 	int ret = 0;
-	int sh_len;
 
 	*next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
 				      1);
 	to_hash = req->nbytes - *next_buflen;
 
 	if (to_hash) {
-		src_nents = sg_count(req->src, req->nbytes - (*next_buflen));
+		src_nents = sg_nents_for_len(req->src,
+					     req->nbytes - *next_buflen);
 		if (src_nents < 0) {
 			dev_err(jrdev, "Invalid number of src SG.\n");
 			return src_nents;
 		}
-		dma_map_sg(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE);
-		sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry);
+
+		if (src_nents) {
+			mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
+						  DMA_TO_DEVICE);
+			if (!mapped_nents) {
+				dev_err(jrdev, "unable to map source for DMA\n");
+				return -ENOMEM;
+			}
+		} else {
+			mapped_nents = 0;
+		}
 
 		/*
 		 * allocate space for base edesc and hw desc commands,
 		 * link tables
 		 */
-		edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN +
-				sec4_sg_bytes, GFP_DMA | flags);
+		edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ?
+					  mapped_nents : 0,
+					  ctx->sh_desc_update_first,
+					  ctx->sh_desc_update_first_dma,
+					  flags);
 		if (!edesc) {
-			dev_err(jrdev,
-				"could not allocate extended descriptor\n");
+			dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
 			return -ENOMEM;
 		}
 
 		edesc->src_nents = src_nents;
-		edesc->sec4_sg_bytes = sec4_sg_bytes;
-		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
-				 DESC_JOB_IO_LEN;
 		edesc->dst_dma = 0;
 
-		if (src_nents) {
-			sg_to_sec4_sg_last(req->src, src_nents,
-					   edesc->sec4_sg, 0);
-			edesc->sec4_sg_dma = dma_map_single(jrdev,
-							    edesc->sec4_sg,
-							    sec4_sg_bytes,
-							    DMA_TO_DEVICE);
-			if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
-				dev_err(jrdev, "unable to map S/G table\n");
-				return -ENOMEM;
-			}
-			src_dma = edesc->sec4_sg_dma;
-			options = LDST_SGF;
-		} else {
-			src_dma = sg_dma_address(req->src);
-			options = 0;
-		}
+		ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
+					  to_hash);
+		if (ret)
+			goto unmap_ctx;
 
 		if (*next_buflen)
 			scatterwalk_map_and_copy(next_buf, req->src, to_hash,
 						 *next_buflen, 0);
 
-		sh_len = desc_len(sh_desc);
 		desc = edesc->hw_desc;
-		init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER |
-				     HDR_REVERSE);
-
-		append_seq_in_ptr(desc, src_dma, to_hash, options);
 
 		ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
 		if (ret)
-			return ret;
+			goto unmap_ctx;
 
 #ifdef DEBUG
 		print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
@@ -1515,18 +1564,14 @@ static int ahash_update_first(struct ahash_request *req)
 			       desc_bytes(desc), 1);
 #endif
 
-		ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst,
-				      req);
-		if (!ret) {
-			ret = -EINPROGRESS;
-			state->update = ahash_update_ctx;
-			state->finup = ahash_finup_ctx;
-			state->final = ahash_final_ctx;
-		} else {
-			ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len,
-					DMA_TO_DEVICE);
-			kfree(edesc);
-		}
+		ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
+		if (ret)
+			goto unmap_ctx;
+
+		ret = -EINPROGRESS;
+		state->update = ahash_update_ctx;
+		state->finup = ahash_finup_ctx;
+		state->final = ahash_final_ctx;
 	} else if (*next_buflen) {
 		state->update = ahash_update_no_ctx;
 		state->finup = ahash_finup_no_ctx;
@@ -1541,6 +1586,10 @@ static int ahash_update_first(struct ahash_request *req)
 #endif
 
 	return ret;
+ unmap_ctx:
+	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
+	kfree(edesc);
+	return ret;
 }
 
 static int ahash_finup_first(struct ahash_request *req)
@@ -1799,7 +1848,6 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
 					 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
 					 HASH_MSG_LEN + 64,
 					 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
-	int ret = 0;
 
 	/*
 	 * Get a Job ring from Job Ring driver to ensure in-order
@@ -1819,10 +1867,7 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
 
 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 				 sizeof(struct caam_hash_state));
-
-	ret = ahash_set_sh_desc(ahash);
-
-	return ret;
+	return ahash_set_sh_desc(ahash);
 }
 
 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
@@ -1898,6 +1943,7 @@ caam_hash_alloc(struct caam_hash_template *template,
 			 template->name);
 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
 			 template->driver_name);
+		t_alg->ahash_alg.setkey = NULL;
 	}
 	alg->cra_module = THIS_MODULE;
 	alg->cra_init = caam_hash_cra_init;
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 0ec112ee5204..72ff19658985 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -14,6 +14,7 @@
 #include "jr.h"
 #include "desc_constr.h"
 #include "error.h"
+#include "ctrl.h"
 
 bool caam_little_end;
 EXPORT_SYMBOL(caam_little_end);
@@ -826,6 +827,8 @@ static int caam_probe(struct platform_device *pdev)
 
 caam_remove:
 	caam_remove(pdev);
+	return ret;
+
 iounmap_ctrl:
 	iounmap(ctrl);
 disable_caam_emi_slow:
diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h
index 26427c11ad87..513b6646bb36 100644
--- a/drivers/crypto/caam/desc.h
+++ b/drivers/crypto/caam/desc.h
@@ -23,13 +23,7 @@
 #define SEC4_SG_OFFSET_MASK	0x00001fff
 
 struct sec4_sg_entry {
-#if !defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) && \
-	defined(CONFIG_CRYPTO_DEV_FSL_CAAM_IMX)
-	u32 rsvd1;
-	dma_addr_t ptr;
-#else
 	u64 ptr;
-#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_IMX */
 	u32 len;
 	u32 bpid_offset;
 };
diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h
index d3869b95e7b1..a8cd8a78ec1f 100644
--- a/drivers/crypto/caam/desc_constr.h
+++ b/drivers/crypto/caam/desc_constr.h
@@ -325,6 +325,23 @@ static inline void append_##cmd##_imm_##type(u32 *desc, type immediate, \
 APPEND_CMD_RAW_IMM(load, LOAD, u32);
 
 /*
+ * ee - endianness
+ * size - size of immediate type in bytes
+ */
+#define APPEND_CMD_RAW_IMM2(cmd, op, ee, size) \
+static inline void append_##cmd##_imm_##ee##size(u32 *desc, \
+						   u##size immediate, \
+						   u32 options) \
+{ \
+	__##ee##size data = cpu_to_##ee##size(immediate); \
+	PRINT_POS; \
+	append_cmd(desc, CMD_##op | IMMEDIATE | options | sizeof(data)); \
+	append_data(desc, &data, sizeof(data)); \
+}
+
+APPEND_CMD_RAW_IMM2(load, LOAD, be, 32);
+
+/*
  * Append math command. Only the last part of destination and source need to
  * be specified
  */
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index e2bcacc1a921..5d4c05074a5c 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -41,7 +41,6 @@ struct caam_drv_private_jr {
 	struct device		*dev;
 	int ridx;
 	struct caam_job_ring __iomem *rregs;	/* JobR's register space */
-	struct tasklet_struct irqtask;
 	int irq;			/* One per queue */
 
 	/* Number of scatterlist crypt transforms active on the JobR */
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index a81f551ac222..757c27f9953d 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -73,8 +73,6 @@ static int caam_jr_shutdown(struct device *dev)
 
 	ret = caam_reset_hw_jr(dev);
 
-	tasklet_kill(&jrp->irqtask);
-
 	/* Release interrupt */
 	free_irq(jrp->irq, dev);
 
@@ -130,7 +128,7 @@ static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
 
 	/*
 	 * Check the output ring for ready responses, kick
-	 * tasklet if jobs done.
+	 * the threaded irq if jobs done.
 	 */
 	irqstate = rd_reg32(&jrp->rregs->jrintstatus);
 	if (!irqstate)
@@ -152,18 +150,13 @@ static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
 	/* Have valid interrupt at this point, just ACK and trigger */
 	wr_reg32(&jrp->rregs->jrintstatus, irqstate);
 
-	preempt_disable();
-	tasklet_schedule(&jrp->irqtask);
-	preempt_enable();
-
-	return IRQ_HANDLED;
+	return IRQ_WAKE_THREAD;
 }
 
-/* Deferred service handler, run as interrupt-fired tasklet */
-static void caam_jr_dequeue(unsigned long devarg)
+static irqreturn_t caam_jr_threadirq(int irq, void *st_dev)
 {
 	int hw_idx, sw_idx, i, head, tail;
-	struct device *dev = (struct device *)devarg;
+	struct device *dev = st_dev;
 	struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 	void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
 	u32 *userdesc, userstatus;
@@ -237,6 +230,8 @@ static void caam_jr_dequeue(unsigned long devarg)
 
 	/* reenable / unmask IRQs */
 	clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
+
+	return IRQ_HANDLED;
 }
 
 /**
@@ -394,11 +389,10 @@ static int caam_jr_init(struct device *dev)
 
 	jrp = dev_get_drvdata(dev);
 
-	tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
-
 	/* Connect job ring interrupt handler. */
-	error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
-			    dev_name(dev), dev);
+	error = request_threaded_irq(jrp->irq, caam_jr_interrupt,
+				     caam_jr_threadirq, IRQF_SHARED,
+				     dev_name(dev), dev);
 	if (error) {
 		dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
 			jrp->ridx, jrp->irq);
@@ -460,7 +454,6 @@ out_free_inpring:
 out_free_irq:
 	free_irq(jrp->irq, dev);
 out_kill_deq:
-	tasklet_kill(&jrp->irqtask);
 	return error;
 }
 
@@ -513,6 +506,7 @@ static int caam_jr_probe(struct platform_device *pdev)
 	error = caam_jr_init(jrdev); /* now turn on hardware */
 	if (error) {
 		irq_dispose_mapping(jrpriv->irq);
+		iounmap(ctrl);
 		return error;
 	}
 
diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h
index b3c5016f6458..84d2f838a063 100644
--- a/drivers/crypto/caam/regs.h
+++ b/drivers/crypto/caam/regs.h
@@ -196,6 +196,14 @@ static inline u64 rd_reg64(void __iomem *reg)
 #define caam_dma_to_cpu(value) caam32_to_cpu(value)
 #endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT  */
 
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX
+#define cpu_to_caam_dma64(value) \
+		(((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | \
+		 (u64)cpu_to_caam32(upper_32_bits(value)))
+#else
+#define cpu_to_caam_dma64(value) cpu_to_caam64(value)
+#endif
+
 /*
  * jr_outentry
  * Represents each entry in a JobR output ring
diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h
index 19dc64fede0d..41cd5a356d05 100644
--- a/drivers/crypto/caam/sg_sw_sec4.h
+++ b/drivers/crypto/caam/sg_sw_sec4.h
@@ -15,7 +15,7 @@ struct sec4_sg_entry;
 static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
 				      dma_addr_t dma, u32 len, u16 offset)
 {
-	sec4_sg_ptr->ptr = cpu_to_caam_dma(dma);
+	sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma);
 	sec4_sg_ptr->len = cpu_to_caam32(len);
 	sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK);
 #ifdef DEBUG
diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile
index ee4d2741b3ab..346ceb8f17bd 100644
--- a/drivers/crypto/ccp/Makefile
+++ b/drivers/crypto/ccp/Makefile
@@ -2,6 +2,7 @@ obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o
 ccp-objs := ccp-dev.o \
 	    ccp-ops.o \
 	    ccp-dev-v3.o \
+	    ccp-dev-v5.o \
 	    ccp-platform.o \
 	    ccp-dmaengine.o
 ccp-$(CONFIG_PCI) += ccp-pci.o
diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c
index 8f36af62fe95..84a652be4274 100644
--- a/drivers/crypto/ccp/ccp-crypto-sha.c
+++ b/drivers/crypto/ccp/ccp-crypto-sha.c
@@ -4,6 +4,7 @@
  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ * Author: Gary R Hook <gary.hook@amd.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -134,7 +135,22 @@ static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes,
 	rctx->cmd.engine = CCP_ENGINE_SHA;
 	rctx->cmd.u.sha.type = rctx->type;
 	rctx->cmd.u.sha.ctx = &rctx->ctx_sg;
-	rctx->cmd.u.sha.ctx_len = sizeof(rctx->ctx);
+
+	switch (rctx->type) {
+	case CCP_SHA_TYPE_1:
+		rctx->cmd.u.sha.ctx_len = SHA1_DIGEST_SIZE;
+		break;
+	case CCP_SHA_TYPE_224:
+		rctx->cmd.u.sha.ctx_len = SHA224_DIGEST_SIZE;
+		break;
+	case CCP_SHA_TYPE_256:
+		rctx->cmd.u.sha.ctx_len = SHA256_DIGEST_SIZE;
+		break;
+	default:
+		/* Should never get here */
+		break;
+	}
+
 	rctx->cmd.u.sha.src = sg;
 	rctx->cmd.u.sha.src_len = rctx->hash_cnt;
 	rctx->cmd.u.sha.opad = ctx->u.sha.key_len ?
diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c
index d7a710347967..8d2dbacc6161 100644
--- a/drivers/crypto/ccp/ccp-dev-v3.c
+++ b/drivers/crypto/ccp/ccp-dev-v3.c
@@ -4,6 +4,7 @@
  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ * Author: Gary R Hook <gary.hook@amd.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -19,6 +20,61 @@
 
 #include "ccp-dev.h"
 
+static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count)
+{
+	int start;
+	struct ccp_device *ccp = cmd_q->ccp;
+
+	for (;;) {
+		mutex_lock(&ccp->sb_mutex);
+
+		start = (u32)bitmap_find_next_zero_area(ccp->sb,
+							ccp->sb_count,
+							ccp->sb_start,
+							count, 0);
+		if (start <= ccp->sb_count) {
+			bitmap_set(ccp->sb, start, count);
+
+			mutex_unlock(&ccp->sb_mutex);
+			break;
+		}
+
+		ccp->sb_avail = 0;
+
+		mutex_unlock(&ccp->sb_mutex);
+
+		/* Wait for KSB entries to become available */
+		if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
+			return 0;
+	}
+
+	return KSB_START + start;
+}
+
+static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start,
+			 unsigned int count)
+{
+	struct ccp_device *ccp = cmd_q->ccp;
+
+	if (!start)
+		return;
+
+	mutex_lock(&ccp->sb_mutex);
+
+	bitmap_clear(ccp->sb, start - KSB_START, count);
+
+	ccp->sb_avail = 1;
+
+	mutex_unlock(&ccp->sb_mutex);
+
+	wake_up_interruptible_all(&ccp->sb_queue);
+}
+
+static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q)
+{
+	return CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
+}
+
 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
 {
 	struct ccp_cmd_queue *cmd_q = op->cmd_q;
@@ -68,6 +124,9 @@ static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
 			/* On error delete all related jobs from the queue */
 			cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
 			      | op->jobid;
+			if (cmd_q->cmd_error)
+				ccp_log_error(cmd_q->ccp,
+					      cmd_q->cmd_error);
 
 			iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
 
@@ -99,10 +158,10 @@ static int ccp_perform_aes(struct ccp_op *op)
 		| (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
 		| (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
 		| (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
-		| (op->ksb_key << REQ1_KEY_KSB_SHIFT);
+		| (op->sb_key << REQ1_KEY_KSB_SHIFT);
 	cr[1] = op->src.u.dma.length - 1;
 	cr[2] = ccp_addr_lo(&op->src.u.dma);
-	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
+	cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
 		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 		| ccp_addr_hi(&op->src.u.dma);
 	cr[4] = ccp_addr_lo(&op->dst.u.dma);
@@ -129,10 +188,10 @@ static int ccp_perform_xts_aes(struct ccp_op *op)
 	cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
 		| (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
 		| (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
-		| (op->ksb_key << REQ1_KEY_KSB_SHIFT);
+		| (op->sb_key << REQ1_KEY_KSB_SHIFT);
 	cr[1] = op->src.u.dma.length - 1;
 	cr[2] = ccp_addr_lo(&op->src.u.dma);
-	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
+	cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
 		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 		| ccp_addr_hi(&op->src.u.dma);
 	cr[4] = ccp_addr_lo(&op->dst.u.dma);
@@ -158,7 +217,7 @@ static int ccp_perform_sha(struct ccp_op *op)
 		| REQ1_INIT;
 	cr[1] = op->src.u.dma.length - 1;
 	cr[2] = ccp_addr_lo(&op->src.u.dma);
-	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
+	cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
 		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 		| ccp_addr_hi(&op->src.u.dma);
 
@@ -181,11 +240,11 @@ static int ccp_perform_rsa(struct ccp_op *op)
 	/* Fill out the register contents for REQ1 through REQ6 */
 	cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
 		| (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
-		| (op->ksb_key << REQ1_KEY_KSB_SHIFT)
+		| (op->sb_key << REQ1_KEY_KSB_SHIFT)
 		| REQ1_EOM;
 	cr[1] = op->u.rsa.input_len - 1;
 	cr[2] = ccp_addr_lo(&op->src.u.dma);
-	cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
+	cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
 		| (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
 		| ccp_addr_hi(&op->src.u.dma);
 	cr[4] = ccp_addr_lo(&op->dst.u.dma);
@@ -215,10 +274,10 @@ static int ccp_perform_passthru(struct ccp_op *op)
 			| ccp_addr_hi(&op->src.u.dma);
 
 		if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
-			cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT);
+			cr[3] |= (op->sb_key << REQ4_KSB_SHIFT);
 	} else {
-		cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
-		cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
+		cr[2] = op->src.u.sb * CCP_SB_BYTES;
+		cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT);
 	}
 
 	if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
@@ -226,8 +285,8 @@ static int ccp_perform_passthru(struct ccp_op *op)
 		cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
 			| ccp_addr_hi(&op->dst.u.dma);
 	} else {
-		cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
-		cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
+		cr[4] = op->dst.u.sb * CCP_SB_BYTES;
+		cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT);
 	}
 
 	if (op->eom)
@@ -256,35 +315,6 @@ static int ccp_perform_ecc(struct ccp_op *op)
 	return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
 }
 
-static int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
-{
-	struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng);
-	u32 trng_value;
-	int len = min_t(int, sizeof(trng_value), max);
-
-	/*
-	 * Locking is provided by the caller so we can update device
-	 * hwrng-related fields safely
-	 */
-	trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG);
-	if (!trng_value) {
-		/* Zero is returned if not data is available or if a
-		 * bad-entropy error is present. Assume an error if
-		 * we exceed TRNG_RETRIES reads of zero.
-		 */
-		if (ccp->hwrng_retries++ > TRNG_RETRIES)
-			return -EIO;
-
-		return 0;
-	}
-
-	/* Reset the counter and save the rng value */
-	ccp->hwrng_retries = 0;
-	memcpy(data, &trng_value, len);
-
-	return len;
-}
-
 static int ccp_init(struct ccp_device *ccp)
 {
 	struct device *dev = ccp->dev;
@@ -321,9 +351,9 @@ static int ccp_init(struct ccp_device *ccp)
 		cmd_q->dma_pool = dma_pool;
 
 		/* Reserve 2 KSB regions for the queue */
-		cmd_q->ksb_key = KSB_START + ccp->ksb_start++;
-		cmd_q->ksb_ctx = KSB_START + ccp->ksb_start++;
-		ccp->ksb_count -= 2;
+		cmd_q->sb_key = KSB_START + ccp->sb_start++;
+		cmd_q->sb_ctx = KSB_START + ccp->sb_start++;
+		ccp->sb_count -= 2;
 
 		/* Preset some register values and masks that are queue
 		 * number dependent
@@ -335,7 +365,7 @@ static int ccp_init(struct ccp_device *ccp)
 		cmd_q->int_ok = 1 << (i * 2);
 		cmd_q->int_err = 1 << ((i * 2) + 1);
 
-		cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
+		cmd_q->free_slots = ccp_get_free_slots(cmd_q);
 
 		init_waitqueue_head(&cmd_q->int_queue);
 
@@ -375,9 +405,10 @@ static int ccp_init(struct ccp_device *ccp)
 	}
 
 	/* Initialize the queues used to wait for KSB space and suspend */
-	init_waitqueue_head(&ccp->ksb_queue);
+	init_waitqueue_head(&ccp->sb_queue);
 	init_waitqueue_head(&ccp->suspend_queue);
 
+	dev_dbg(dev, "Starting threads...\n");
 	/* Create a kthread for each queue */
 	for (i = 0; i < ccp->cmd_q_count; i++) {
 		struct task_struct *kthread;
@@ -397,29 +428,26 @@ static int ccp_init(struct ccp_device *ccp)
 		wake_up_process(kthread);
 	}
 
-	/* Register the RNG */
-	ccp->hwrng.name = ccp->rngname;
-	ccp->hwrng.read = ccp_trng_read;
-	ret = hwrng_register(&ccp->hwrng);
-	if (ret) {
-		dev_err(dev, "error registering hwrng (%d)\n", ret);
+	dev_dbg(dev, "Enabling interrupts...\n");
+	/* Enable interrupts */
+	iowrite32(qim, ccp->io_regs + IRQ_MASK_REG);
+
+	dev_dbg(dev, "Registering device...\n");
+	ccp_add_device(ccp);
+
+	ret = ccp_register_rng(ccp);
+	if (ret)
 		goto e_kthread;
-	}
 
 	/* Register the DMA engine support */
 	ret = ccp_dmaengine_register(ccp);
 	if (ret)
 		goto e_hwrng;
 
-	ccp_add_device(ccp);
-
-	/* Enable interrupts */
-	iowrite32(qim, ccp->io_regs + IRQ_MASK_REG);
-
 	return 0;
 
 e_hwrng:
-	hwrng_unregister(&ccp->hwrng);
+	ccp_unregister_rng(ccp);
 
 e_kthread:
 	for (i = 0; i < ccp->cmd_q_count; i++)
@@ -441,19 +469,14 @@ static void ccp_destroy(struct ccp_device *ccp)
 	struct ccp_cmd *cmd;
 	unsigned int qim, i;
 
-	/* Remove this device from the list of available units first */
-	ccp_del_device(ccp);
-
 	/* Unregister the DMA engine */
 	ccp_dmaengine_unregister(ccp);
 
 	/* Unregister the RNG */
-	hwrng_unregister(&ccp->hwrng);
+	ccp_unregister_rng(ccp);
 
-	/* Stop the queue kthreads */
-	for (i = 0; i < ccp->cmd_q_count; i++)
-		if (ccp->cmd_q[i].kthread)
-			kthread_stop(ccp->cmd_q[i].kthread);
+	/* Remove this device from the list of available units */
+	ccp_del_device(ccp);
 
 	/* Build queue interrupt mask (two interrupt masks per queue) */
 	qim = 0;
@@ -472,6 +495,11 @@ static void ccp_destroy(struct ccp_device *ccp)
 	}
 	iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG);
 
+	/* Stop the queue kthreads */
+	for (i = 0; i < ccp->cmd_q_count; i++)
+		if (ccp->cmd_q[i].kthread)
+			kthread_stop(ccp->cmd_q[i].kthread);
+
 	ccp->free_irq(ccp);
 
 	for (i = 0; i < ccp->cmd_q_count; i++)
@@ -527,18 +555,24 @@ static irqreturn_t ccp_irq_handler(int irq, void *data)
 }
 
 static const struct ccp_actions ccp3_actions = {
-	.perform_aes = ccp_perform_aes,
-	.perform_xts_aes = ccp_perform_xts_aes,
-	.perform_sha = ccp_perform_sha,
-	.perform_rsa = ccp_perform_rsa,
-	.perform_passthru = ccp_perform_passthru,
-	.perform_ecc = ccp_perform_ecc,
+	.aes = ccp_perform_aes,
+	.xts_aes = ccp_perform_xts_aes,
+	.sha = ccp_perform_sha,
+	.rsa = ccp_perform_rsa,
+	.passthru = ccp_perform_passthru,
+	.ecc = ccp_perform_ecc,
+	.sballoc = ccp_alloc_ksb,
+	.sbfree = ccp_free_ksb,
 	.init = ccp_init,
 	.destroy = ccp_destroy,
+	.get_free_slots = ccp_get_free_slots,
 	.irqhandler = ccp_irq_handler,
 };
 
-struct ccp_vdata ccpv3 = {
+const struct ccp_vdata ccpv3 = {
 	.version = CCP_VERSION(3, 0),
+	.setup = NULL,
 	.perform = &ccp3_actions,
+	.bar = 2,
+	.offset = 0x20000,
 };
diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
new file mode 100644
index 000000000000..faf3cb3ddce2
--- /dev/null
+++ b/drivers/crypto/ccp/ccp-dev-v5.c
@@ -0,0 +1,1017 @@
+/*
+ * AMD Cryptographic Coprocessor (CCP) driver
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Author: Gary R Hook <gary.hook@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/kthread.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/compiler.h>
+#include <linux/ccp.h>
+
+#include "ccp-dev.h"
+
+static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
+{
+	struct ccp_device *ccp;
+	int start;
+
+	/* First look at the map for the queue */
+	if (cmd_q->lsb >= 0) {
+		start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
+							LSB_SIZE,
+							0, count, 0);
+		if (start < LSB_SIZE) {
+			bitmap_set(cmd_q->lsbmap, start, count);
+			return start + cmd_q->lsb * LSB_SIZE;
+		}
+	}
+
+	/* No joy; try to get an entry from the shared blocks */
+	ccp = cmd_q->ccp;
+	for (;;) {
+		mutex_lock(&ccp->sb_mutex);
+
+		start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
+							MAX_LSB_CNT * LSB_SIZE,
+							0,
+							count, 0);
+		if (start <= MAX_LSB_CNT * LSB_SIZE) {
+			bitmap_set(ccp->lsbmap, start, count);
+
+			mutex_unlock(&ccp->sb_mutex);
+			return start * LSB_ITEM_SIZE;
+		}
+
+		ccp->sb_avail = 0;
+
+		mutex_unlock(&ccp->sb_mutex);
+
+		/* Wait for KSB entries to become available */
+		if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
+			return 0;
+	}
+}
+
+static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
+			 unsigned int count)
+{
+	int lsbno = start / LSB_SIZE;
+
+	if (!start)
+		return;
+
+	if (cmd_q->lsb == lsbno) {
+		/* An entry from the private LSB */
+		bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count);
+	} else {
+		/* From the shared LSBs */
+		struct ccp_device *ccp = cmd_q->ccp;
+
+		mutex_lock(&ccp->sb_mutex);
+		bitmap_clear(ccp->lsbmap, start, count);
+		ccp->sb_avail = 1;
+		mutex_unlock(&ccp->sb_mutex);
+		wake_up_interruptible_all(&ccp->sb_queue);
+	}
+}
+
+/* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
+union ccp_function {
+	struct {
+		u16 size:7;
+		u16 encrypt:1;
+		u16 mode:5;
+		u16 type:2;
+	} aes;
+	struct {
+		u16 size:7;
+		u16 encrypt:1;
+		u16 rsvd:5;
+		u16 type:2;
+	} aes_xts;
+	struct {
+		u16 rsvd1:10;
+		u16 type:4;
+		u16 rsvd2:1;
+	} sha;
+	struct {
+		u16 mode:3;
+		u16 size:12;
+	} rsa;
+	struct {
+		u16 byteswap:2;
+		u16 bitwise:3;
+		u16 reflect:2;
+		u16 rsvd:8;
+	} pt;
+	struct  {
+		u16 rsvd:13;
+	} zlib;
+	struct {
+		u16 size:10;
+		u16 type:2;
+		u16 mode:3;
+	} ecc;
+	u16 raw;
+};
+
+#define	CCP_AES_SIZE(p)		((p)->aes.size)
+#define	CCP_AES_ENCRYPT(p)	((p)->aes.encrypt)
+#define	CCP_AES_MODE(p)		((p)->aes.mode)
+#define	CCP_AES_TYPE(p)		((p)->aes.type)
+#define	CCP_XTS_SIZE(p)		((p)->aes_xts.size)
+#define	CCP_XTS_ENCRYPT(p)	((p)->aes_xts.encrypt)
+#define	CCP_SHA_TYPE(p)		((p)->sha.type)
+#define	CCP_RSA_SIZE(p)		((p)->rsa.size)
+#define	CCP_PT_BYTESWAP(p)	((p)->pt.byteswap)
+#define	CCP_PT_BITWISE(p)	((p)->pt.bitwise)
+#define	CCP_ECC_MODE(p)		((p)->ecc.mode)
+#define	CCP_ECC_AFFINE(p)	((p)->ecc.one)
+
+/* Word 0 */
+#define CCP5_CMD_DW0(p)		((p)->dw0)
+#define CCP5_CMD_SOC(p)		(CCP5_CMD_DW0(p).soc)
+#define CCP5_CMD_IOC(p)		(CCP5_CMD_DW0(p).ioc)
+#define CCP5_CMD_INIT(p)	(CCP5_CMD_DW0(p).init)
+#define CCP5_CMD_EOM(p)		(CCP5_CMD_DW0(p).eom)
+#define CCP5_CMD_FUNCTION(p)	(CCP5_CMD_DW0(p).function)
+#define CCP5_CMD_ENGINE(p)	(CCP5_CMD_DW0(p).engine)
+#define CCP5_CMD_PROT(p)	(CCP5_CMD_DW0(p).prot)
+
+/* Word 1 */
+#define CCP5_CMD_DW1(p)		((p)->length)
+#define CCP5_CMD_LEN(p)		(CCP5_CMD_DW1(p))
+
+/* Word 2 */
+#define CCP5_CMD_DW2(p)		((p)->src_lo)
+#define CCP5_CMD_SRC_LO(p)	(CCP5_CMD_DW2(p))
+
+/* Word 3 */
+#define CCP5_CMD_DW3(p)		((p)->dw3)
+#define CCP5_CMD_SRC_MEM(p)	((p)->dw3.src_mem)
+#define CCP5_CMD_SRC_HI(p)	((p)->dw3.src_hi)
+#define CCP5_CMD_LSB_ID(p)	((p)->dw3.lsb_cxt_id)
+#define CCP5_CMD_FIX_SRC(p)	((p)->dw3.fixed)
+
+/* Words 4/5 */
+#define CCP5_CMD_DW4(p)		((p)->dw4)
+#define CCP5_CMD_DST_LO(p)	(CCP5_CMD_DW4(p).dst_lo)
+#define CCP5_CMD_DW5(p)		((p)->dw5.fields.dst_hi)
+#define CCP5_CMD_DST_HI(p)	(CCP5_CMD_DW5(p))
+#define CCP5_CMD_DST_MEM(p)	((p)->dw5.fields.dst_mem)
+#define CCP5_CMD_FIX_DST(p)	((p)->dw5.fields.fixed)
+#define CCP5_CMD_SHA_LO(p)	((p)->dw4.sha_len_lo)
+#define CCP5_CMD_SHA_HI(p)	((p)->dw5.sha_len_hi)
+
+/* Word 6/7 */
+#define CCP5_CMD_DW6(p)		((p)->key_lo)
+#define CCP5_CMD_KEY_LO(p)	(CCP5_CMD_DW6(p))
+#define CCP5_CMD_DW7(p)		((p)->dw7)
+#define CCP5_CMD_KEY_HI(p)	((p)->dw7.key_hi)
+#define CCP5_CMD_KEY_MEM(p)	((p)->dw7.key_mem)
+
+static inline u32 low_address(unsigned long addr)
+{
+	return (u64)addr & 0x0ffffffff;
+}
+
+static inline u32 high_address(unsigned long addr)
+{
+	return ((u64)addr >> 32) & 0x00000ffff;
+}
+
+static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
+{
+	unsigned int head_idx, n;
+	u32 head_lo, queue_start;
+
+	queue_start = low_address(cmd_q->qdma_tail);
+	head_lo = ioread32(cmd_q->reg_head_lo);
+	head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
+
+	n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
+
+	return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
+}
+
+static int ccp5_do_cmd(struct ccp5_desc *desc,
+		       struct ccp_cmd_queue *cmd_q)
+{
+	u32 *mP;
+	__le32 *dP;
+	u32 tail;
+	int	i;
+	int ret = 0;
+
+	if (CCP5_CMD_SOC(desc)) {
+		CCP5_CMD_IOC(desc) = 1;
+		CCP5_CMD_SOC(desc) = 0;
+	}
+	mutex_lock(&cmd_q->q_mutex);
+
+	mP = (u32 *) &cmd_q->qbase[cmd_q->qidx];
+	dP = (__le32 *) desc;
+	for (i = 0; i < 8; i++)
+		mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
+
+	cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
+
+	/* The data used by this command must be flushed to memory */
+	wmb();
+
+	/* Write the new tail address back to the queue register */
+	tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
+	iowrite32(tail, cmd_q->reg_tail_lo);
+
+	/* Turn the queue back on using our cached control register */
+	iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
+	mutex_unlock(&cmd_q->q_mutex);
+
+	if (CCP5_CMD_IOC(desc)) {
+		/* Wait for the job to complete */
+		ret = wait_event_interruptible(cmd_q->int_queue,
+					       cmd_q->int_rcvd);
+		if (ret || cmd_q->cmd_error) {
+			if (cmd_q->cmd_error)
+				ccp_log_error(cmd_q->ccp,
+					      cmd_q->cmd_error);
+			/* A version 5 device doesn't use Job IDs... */
+			if (!ret)
+				ret = -EIO;
+		}
+		cmd_q->int_rcvd = 0;
+	}
+
+	return 0;
+}
+
+static int ccp5_perform_aes(struct ccp_op *op)
+{
+	struct ccp5_desc desc;
+	union ccp_function function;
+	u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
+
+	/* Zero out all the fields of the command desc */
+	memset(&desc, 0, Q_DESC_SIZE);
+
+	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
+
+	CCP5_CMD_SOC(&desc) = op->soc;
+	CCP5_CMD_IOC(&desc) = 1;
+	CCP5_CMD_INIT(&desc) = op->init;
+	CCP5_CMD_EOM(&desc) = op->eom;
+	CCP5_CMD_PROT(&desc) = 0;
+
+	function.raw = 0;
+	CCP_AES_ENCRYPT(&function) = op->u.aes.action;
+	CCP_AES_MODE(&function) = op->u.aes.mode;
+	CCP_AES_TYPE(&function) = op->u.aes.type;
+	if (op->u.aes.mode == CCP_AES_MODE_CFB)
+		CCP_AES_SIZE(&function) = 0x7f;
+
+	CCP5_CMD_FUNCTION(&desc) = function.raw;
+
+	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
+
+	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
+	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
+	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
+	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
+	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
+	CCP5_CMD_KEY_HI(&desc) = 0;
+	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
+	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
+
+	return ccp5_do_cmd(&desc, op->cmd_q);
+}
+
+static int ccp5_perform_xts_aes(struct ccp_op *op)
+{
+	struct ccp5_desc desc;
+	union ccp_function function;
+	u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
+
+	/* Zero out all the fields of the command desc */
+	memset(&desc, 0, Q_DESC_SIZE);
+
+	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
+
+	CCP5_CMD_SOC(&desc) = op->soc;
+	CCP5_CMD_IOC(&desc) = 1;
+	CCP5_CMD_INIT(&desc) = op->init;
+	CCP5_CMD_EOM(&desc) = op->eom;
+	CCP5_CMD_PROT(&desc) = 0;
+
+	function.raw = 0;
+	CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
+	CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
+	CCP5_CMD_FUNCTION(&desc) = function.raw;
+
+	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
+
+	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
+	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
+	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
+	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
+	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
+	CCP5_CMD_KEY_HI(&desc) =  0;
+	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
+	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
+
+	return ccp5_do_cmd(&desc, op->cmd_q);
+}
+
+static int ccp5_perform_sha(struct ccp_op *op)
+{
+	struct ccp5_desc desc;
+	union ccp_function function;
+
+	/* Zero out all the fields of the command desc */
+	memset(&desc, 0, Q_DESC_SIZE);
+
+	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
+
+	CCP5_CMD_SOC(&desc) = op->soc;
+	CCP5_CMD_IOC(&desc) = 1;
+	CCP5_CMD_INIT(&desc) = 1;
+	CCP5_CMD_EOM(&desc) = op->eom;
+	CCP5_CMD_PROT(&desc) = 0;
+
+	function.raw = 0;
+	CCP_SHA_TYPE(&function) = op->u.sha.type;
+	CCP5_CMD_FUNCTION(&desc) = function.raw;
+
+	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
+
+	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
+	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
+	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
+
+	if (op->eom) {
+		CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
+		CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
+	} else {
+		CCP5_CMD_SHA_LO(&desc) = 0;
+		CCP5_CMD_SHA_HI(&desc) = 0;
+	}
+
+	return ccp5_do_cmd(&desc, op->cmd_q);
+}
+
+static int ccp5_perform_rsa(struct ccp_op *op)
+{
+	struct ccp5_desc desc;
+	union ccp_function function;
+
+	/* Zero out all the fields of the command desc */
+	memset(&desc, 0, Q_DESC_SIZE);
+
+	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
+
+	CCP5_CMD_SOC(&desc) = op->soc;
+	CCP5_CMD_IOC(&desc) = 1;
+	CCP5_CMD_INIT(&desc) = 0;
+	CCP5_CMD_EOM(&desc) = 1;
+	CCP5_CMD_PROT(&desc) = 0;
+
+	function.raw = 0;
+	CCP_RSA_SIZE(&function) = op->u.rsa.mod_size;
+	CCP5_CMD_FUNCTION(&desc) = function.raw;
+
+	CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
+
+	/* Source is from external memory */
+	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
+	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
+	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	/* Destination is in external memory */
+	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
+	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
+	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	/* Key (Exponent) is in external memory */
+	CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
+	CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
+	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	return ccp5_do_cmd(&desc, op->cmd_q);
+}
+
+static int ccp5_perform_passthru(struct ccp_op *op)
+{
+	struct ccp5_desc desc;
+	union ccp_function function;
+	struct ccp_dma_info *saddr = &op->src.u.dma;
+	struct ccp_dma_info *daddr = &op->dst.u.dma;
+
+	memset(&desc, 0, Q_DESC_SIZE);
+
+	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
+
+	CCP5_CMD_SOC(&desc) = 0;
+	CCP5_CMD_IOC(&desc) = 1;
+	CCP5_CMD_INIT(&desc) = 0;
+	CCP5_CMD_EOM(&desc) = op->eom;
+	CCP5_CMD_PROT(&desc) = 0;
+
+	function.raw = 0;
+	CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
+	CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
+	CCP5_CMD_FUNCTION(&desc) = function.raw;
+
+	/* Length of source data is always 256 bytes */
+	if (op->src.type == CCP_MEMTYPE_SYSTEM)
+		CCP5_CMD_LEN(&desc) = saddr->length;
+	else
+		CCP5_CMD_LEN(&desc) = daddr->length;
+
+	if (op->src.type == CCP_MEMTYPE_SYSTEM) {
+		CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
+		CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
+		CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+		if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
+			CCP5_CMD_LSB_ID(&desc) = op->sb_key;
+	} else {
+		u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
+
+		CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
+		CCP5_CMD_SRC_HI(&desc) = 0;
+		CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
+	}
+
+	if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
+		CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
+		CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
+		CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+	} else {
+		u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
+
+		CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
+		CCP5_CMD_DST_HI(&desc) = 0;
+		CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
+	}
+
+	return ccp5_do_cmd(&desc, op->cmd_q);
+}
+
+static int ccp5_perform_ecc(struct ccp_op *op)
+{
+	struct ccp5_desc desc;
+	union ccp_function function;
+
+	/* Zero out all the fields of the command desc */
+	memset(&desc, 0, Q_DESC_SIZE);
+
+	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
+
+	CCP5_CMD_SOC(&desc) = 0;
+	CCP5_CMD_IOC(&desc) = 1;
+	CCP5_CMD_INIT(&desc) = 0;
+	CCP5_CMD_EOM(&desc) = 1;
+	CCP5_CMD_PROT(&desc) = 0;
+
+	function.raw = 0;
+	function.ecc.mode = op->u.ecc.function;
+	CCP5_CMD_FUNCTION(&desc) = function.raw;
+
+	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
+
+	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
+	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
+	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
+	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
+	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+
+	return ccp5_do_cmd(&desc, op->cmd_q);
+}
+
+static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
+{
+	int q_mask = 1 << cmd_q->id;
+	int queues = 0;
+	int j;
+
+	/* Build a bit mask to know which LSBs this queue has access to.
+	 * Don't bother with segment 0 as it has special privileges.
+	 */
+	for (j = 1; j < MAX_LSB_CNT; j++) {
+		if (status & q_mask)
+			bitmap_set(cmd_q->lsbmask, j, 1);
+		status >>= LSB_REGION_WIDTH;
+	}
+	queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
+	dev_info(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
+		 cmd_q->id, queues);
+
+	return queues ? 0 : -EINVAL;
+}
+
+
+static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
+					int lsb_cnt, int n_lsbs,
+					unsigned long *lsb_pub)
+{
+	DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
+	int bitno;
+	int qlsb_wgt;
+	int i;
+
+	/* For each queue:
+	 * If the count of potential LSBs available to a queue matches the
+	 * ordinal given to us in lsb_cnt:
+	 * Copy the mask of possible LSBs for this queue into "qlsb";
+	 * For each bit in qlsb, see if the corresponding bit in the
+	 * aggregation mask is set; if so, we have a match.
+	 *     If we have a match, clear the bit in the aggregation to
+	 *     mark it as no longer available.
+	 *     If there is no match, clear the bit in qlsb and keep looking.
+	 */
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
+
+		qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
+
+		if (qlsb_wgt == lsb_cnt) {
+			bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
+
+			bitno = find_first_bit(qlsb, MAX_LSB_CNT);
+			while (bitno < MAX_LSB_CNT) {
+				if (test_bit(bitno, lsb_pub)) {
+					/* We found an available LSB
+					 * that this queue can access
+					 */
+					cmd_q->lsb = bitno;
+					bitmap_clear(lsb_pub, bitno, 1);
+					dev_info(ccp->dev,
+						 "Queue %d gets LSB %d\n",
+						 i, bitno);
+					break;
+				}
+				bitmap_clear(qlsb, bitno, 1);
+				bitno = find_first_bit(qlsb, MAX_LSB_CNT);
+			}
+			if (bitno >= MAX_LSB_CNT)
+				return -EINVAL;
+			n_lsbs--;
+		}
+	}
+	return n_lsbs;
+}
+
+/* For each queue, from the most- to least-constrained:
+ * find an LSB that can be assigned to the queue. If there are N queues that
+ * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
+ * dedicated LSB. Remaining LSB regions become a shared resource.
+ * If we have fewer LSBs than queues, all LSB regions become shared resources.
+ */
+static int ccp_assign_lsbs(struct ccp_device *ccp)
+{
+	DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
+	DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
+	int n_lsbs = 0;
+	int bitno;
+	int i, lsb_cnt;
+	int rc = 0;
+
+	bitmap_zero(lsb_pub, MAX_LSB_CNT);
+
+	/* Create an aggregate bitmap to get a total count of available LSBs */
+	for (i = 0; i < ccp->cmd_q_count; i++)
+		bitmap_or(lsb_pub,
+			  lsb_pub, ccp->cmd_q[i].lsbmask,
+			  MAX_LSB_CNT);
+
+	n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
+
+	if (n_lsbs >= ccp->cmd_q_count) {
+		/* We have enough LSBS to give every queue a private LSB.
+		 * Brute force search to start with the queues that are more
+		 * constrained in LSB choice. When an LSB is privately
+		 * assigned, it is removed from the public mask.
+		 * This is an ugly N squared algorithm with some optimization.
+		 */
+		for (lsb_cnt = 1;
+		     n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
+		     lsb_cnt++) {
+			rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
+							  lsb_pub);
+			if (rc < 0)
+				return -EINVAL;
+			n_lsbs = rc;
+		}
+	}
+
+	rc = 0;
+	/* What's left of the LSBs, according to the public mask, now become
+	 * shared. Any zero bits in the lsb_pub mask represent an LSB region
+	 * that can't be used as a shared resource, so mark the LSB slots for
+	 * them as "in use".
+	 */
+	bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
+
+	bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
+	while (bitno < MAX_LSB_CNT) {
+		bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
+		bitmap_set(qlsb, bitno, 1);
+		bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
+	}
+
+	return rc;
+}
+
+static int ccp5_init(struct ccp_device *ccp)
+{
+	struct device *dev = ccp->dev;
+	struct ccp_cmd_queue *cmd_q;
+	struct dma_pool *dma_pool;
+	char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
+	unsigned int qmr, qim, i;
+	u64 status;
+	u32 status_lo, status_hi;
+	int ret;
+
+	/* Find available queues */
+	qim = 0;
+	qmr = ioread32(ccp->io_regs + Q_MASK_REG);
+	for (i = 0; i < MAX_HW_QUEUES; i++) {
+
+		if (!(qmr & (1 << i)))
+			continue;
+
+		/* Allocate a dma pool for this queue */
+		snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
+			 ccp->name, i);
+		dma_pool = dma_pool_create(dma_pool_name, dev,
+					   CCP_DMAPOOL_MAX_SIZE,
+					   CCP_DMAPOOL_ALIGN, 0);
+		if (!dma_pool) {
+			dev_err(dev, "unable to allocate dma pool\n");
+			ret = -ENOMEM;
+		}
+
+		cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
+		ccp->cmd_q_count++;
+
+		cmd_q->ccp = ccp;
+		cmd_q->id = i;
+		cmd_q->dma_pool = dma_pool;
+		mutex_init(&cmd_q->q_mutex);
+
+		/* Page alignment satisfies our needs for N <= 128 */
+		BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
+		cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
+		cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize,
+						   &cmd_q->qbase_dma,
+						   GFP_KERNEL);
+		if (!cmd_q->qbase) {
+			dev_err(dev, "unable to allocate command queue\n");
+			ret = -ENOMEM;
+			goto e_pool;
+		}
+
+		cmd_q->qidx = 0;
+		/* Preset some register values and masks that are queue
+		 * number dependent
+		 */
+		cmd_q->reg_control = ccp->io_regs +
+				     CMD5_Q_STATUS_INCR * (i + 1);
+		cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
+		cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
+		cmd_q->reg_int_enable = cmd_q->reg_control +
+					CMD5_Q_INT_ENABLE_BASE;
+		cmd_q->reg_interrupt_status = cmd_q->reg_control +
+					      CMD5_Q_INTERRUPT_STATUS_BASE;
+		cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
+		cmd_q->reg_int_status = cmd_q->reg_control +
+					CMD5_Q_INT_STATUS_BASE;
+		cmd_q->reg_dma_status = cmd_q->reg_control +
+					CMD5_Q_DMA_STATUS_BASE;
+		cmd_q->reg_dma_read_status = cmd_q->reg_control +
+					     CMD5_Q_DMA_READ_STATUS_BASE;
+		cmd_q->reg_dma_write_status = cmd_q->reg_control +
+					      CMD5_Q_DMA_WRITE_STATUS_BASE;
+
+		init_waitqueue_head(&cmd_q->int_queue);
+
+		dev_dbg(dev, "queue #%u available\n", i);
+	}
+	if (ccp->cmd_q_count == 0) {
+		dev_notice(dev, "no command queues available\n");
+		ret = -EIO;
+		goto e_pool;
+	}
+	dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
+
+	/* Turn off the queues and disable interrupts until ready */
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		cmd_q = &ccp->cmd_q[i];
+
+		cmd_q->qcontrol = 0; /* Start with nothing */
+		iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
+
+		/* Disable the interrupts */
+		iowrite32(0x00, cmd_q->reg_int_enable);
+		ioread32(cmd_q->reg_int_status);
+		ioread32(cmd_q->reg_status);
+
+		/* Clear the interrupts */
+		iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
+	}
+
+	dev_dbg(dev, "Requesting an IRQ...\n");
+	/* Request an irq */
+	ret = ccp->get_irq(ccp);
+	if (ret) {
+		dev_err(dev, "unable to allocate an IRQ\n");
+		goto e_pool;
+	}
+
+	/* Initialize the queue used to suspend */
+	init_waitqueue_head(&ccp->suspend_queue);
+
+	dev_dbg(dev, "Loading LSB map...\n");
+	/* Copy the private LSB mask to the public registers */
+	status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
+	status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
+	iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
+	iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
+	status = ((u64)status_hi<<30) | (u64)status_lo;
+
+	dev_dbg(dev, "Configuring virtual queues...\n");
+	/* Configure size of each virtual queue accessible to host */
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		u32 dma_addr_lo;
+		u32 dma_addr_hi;
+
+		cmd_q = &ccp->cmd_q[i];
+
+		cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
+		cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
+
+		cmd_q->qdma_tail = cmd_q->qbase_dma;
+		dma_addr_lo = low_address(cmd_q->qdma_tail);
+		iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
+		iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
+
+		dma_addr_hi = high_address(cmd_q->qdma_tail);
+		cmd_q->qcontrol |= (dma_addr_hi << 16);
+		iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
+
+		/* Find the LSB regions accessible to the queue */
+		ccp_find_lsb_regions(cmd_q, status);
+		cmd_q->lsb = -1; /* Unassigned value */
+	}
+
+	dev_dbg(dev, "Assigning LSBs...\n");
+	ret = ccp_assign_lsbs(ccp);
+	if (ret) {
+		dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
+		goto e_irq;
+	}
+
+	/* Optimization: pre-allocate LSB slots for each queue */
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
+		ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
+	}
+
+	dev_dbg(dev, "Starting threads...\n");
+	/* Create a kthread for each queue */
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		struct task_struct *kthread;
+
+		cmd_q = &ccp->cmd_q[i];
+
+		kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
+					 "%s-q%u", ccp->name, cmd_q->id);
+		if (IS_ERR(kthread)) {
+			dev_err(dev, "error creating queue thread (%ld)\n",
+				PTR_ERR(kthread));
+			ret = PTR_ERR(kthread);
+			goto e_kthread;
+		}
+
+		cmd_q->kthread = kthread;
+		wake_up_process(kthread);
+	}
+
+	dev_dbg(dev, "Enabling interrupts...\n");
+	/* Enable interrupts */
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		cmd_q = &ccp->cmd_q[i];
+		iowrite32(ALL_INTERRUPTS, cmd_q->reg_int_enable);
+	}
+
+	dev_dbg(dev, "Registering device...\n");
+	/* Put this on the unit list to make it available */
+	ccp_add_device(ccp);
+
+	ret = ccp_register_rng(ccp);
+	if (ret)
+		goto e_kthread;
+
+	/* Register the DMA engine support */
+	ret = ccp_dmaengine_register(ccp);
+	if (ret)
+		goto e_hwrng;
+
+	return 0;
+
+e_hwrng:
+	ccp_unregister_rng(ccp);
+
+e_kthread:
+	for (i = 0; i < ccp->cmd_q_count; i++)
+		if (ccp->cmd_q[i].kthread)
+			kthread_stop(ccp->cmd_q[i].kthread);
+
+e_irq:
+	ccp->free_irq(ccp);
+
+e_pool:
+	for (i = 0; i < ccp->cmd_q_count; i++)
+		dma_pool_destroy(ccp->cmd_q[i].dma_pool);
+
+	return ret;
+}
+
+static void ccp5_destroy(struct ccp_device *ccp)
+{
+	struct device *dev = ccp->dev;
+	struct ccp_cmd_queue *cmd_q;
+	struct ccp_cmd *cmd;
+	unsigned int i;
+
+	/* Unregister the DMA engine */
+	ccp_dmaengine_unregister(ccp);
+
+	/* Unregister the RNG */
+	ccp_unregister_rng(ccp);
+
+	/* Remove this device from the list of available units first */
+	ccp_del_device(ccp);
+
+	/* Disable and clear interrupts */
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		cmd_q = &ccp->cmd_q[i];
+
+		/* Turn off the run bit */
+		iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
+
+		/* Disable the interrupts */
+		iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
+
+		/* Clear the interrupt status */
+		iowrite32(0x00, cmd_q->reg_int_enable);
+		ioread32(cmd_q->reg_int_status);
+		ioread32(cmd_q->reg_status);
+	}
+
+	/* Stop the queue kthreads */
+	for (i = 0; i < ccp->cmd_q_count; i++)
+		if (ccp->cmd_q[i].kthread)
+			kthread_stop(ccp->cmd_q[i].kthread);
+
+	ccp->free_irq(ccp);
+
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		cmd_q = &ccp->cmd_q[i];
+		dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
+				  cmd_q->qbase_dma);
+	}
+
+	/* Flush the cmd and backlog queue */
+	while (!list_empty(&ccp->cmd)) {
+		/* Invoke the callback directly with an error code */
+		cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
+		list_del(&cmd->entry);
+		cmd->callback(cmd->data, -ENODEV);
+	}
+	while (!list_empty(&ccp->backlog)) {
+		/* Invoke the callback directly with an error code */
+		cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
+		list_del(&cmd->entry);
+		cmd->callback(cmd->data, -ENODEV);
+	}
+}
+
+static irqreturn_t ccp5_irq_handler(int irq, void *data)
+{
+	struct device *dev = data;
+	struct ccp_device *ccp = dev_get_drvdata(dev);
+	u32 status;
+	unsigned int i;
+
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
+
+		status = ioread32(cmd_q->reg_interrupt_status);
+
+		if (status) {
+			cmd_q->int_status = status;
+			cmd_q->q_status = ioread32(cmd_q->reg_status);
+			cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
+
+			/* On error, only save the first error value */
+			if ((status & INT_ERROR) && !cmd_q->cmd_error)
+				cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
+
+			cmd_q->int_rcvd = 1;
+
+			/* Acknowledge the interrupt and wake the kthread */
+			iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
+			wake_up_interruptible(&cmd_q->int_queue);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void ccp5_config(struct ccp_device *ccp)
+{
+	/* Public side */
+	iowrite32(0x00001249, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
+}
+
+static void ccp5other_config(struct ccp_device *ccp)
+{
+	int i;
+	u32 rnd;
+
+	/* We own all of the queues on the NTB CCP */
+
+	iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
+	iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
+	for (i = 0; i < 12; i++) {
+		rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
+		iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
+	}
+
+	iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
+	iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
+	iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
+
+	iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
+	iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
+
+	iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
+
+	ccp5_config(ccp);
+}
+
+/* Version 5 adds some function, but is essentially the same as v5 */
+static const struct ccp_actions ccp5_actions = {
+	.aes = ccp5_perform_aes,
+	.xts_aes = ccp5_perform_xts_aes,
+	.sha = ccp5_perform_sha,
+	.rsa = ccp5_perform_rsa,
+	.passthru = ccp5_perform_passthru,
+	.ecc = ccp5_perform_ecc,
+	.sballoc = ccp_lsb_alloc,
+	.sbfree = ccp_lsb_free,
+	.init = ccp5_init,
+	.destroy = ccp5_destroy,
+	.get_free_slots = ccp5_get_free_slots,
+	.irqhandler = ccp5_irq_handler,
+};
+
+const struct ccp_vdata ccpv5a = {
+	.version = CCP_VERSION(5, 0),
+	.setup = ccp5_config,
+	.perform = &ccp5_actions,
+	.bar = 2,
+	.offset = 0x0,
+};
+
+const struct ccp_vdata ccpv5b = {
+	.version = CCP_VERSION(5, 0),
+	.setup = ccp5other_config,
+	.perform = &ccp5_actions,
+	.bar = 2,
+	.offset = 0x0,
+};
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index 87b9f2bfa623..cafa633aae10 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -4,6 +4,7 @@
  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ * Author: Gary R Hook <gary.hook@amd.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -39,6 +40,59 @@ struct ccp_tasklet_data {
 	struct ccp_cmd *cmd;
 };
 
+/* Human-readable error strings */
+char *ccp_error_codes[] = {
+	"",
+	"ERR 01: ILLEGAL_ENGINE",
+	"ERR 02: ILLEGAL_KEY_ID",
+	"ERR 03: ILLEGAL_FUNCTION_TYPE",
+	"ERR 04: ILLEGAL_FUNCTION_MODE",
+	"ERR 05: ILLEGAL_FUNCTION_ENCRYPT",
+	"ERR 06: ILLEGAL_FUNCTION_SIZE",
+	"ERR 07: Zlib_MISSING_INIT_EOM",
+	"ERR 08: ILLEGAL_FUNCTION_RSVD",
+	"ERR 09: ILLEGAL_BUFFER_LENGTH",
+	"ERR 10: VLSB_FAULT",
+	"ERR 11: ILLEGAL_MEM_ADDR",
+	"ERR 12: ILLEGAL_MEM_SEL",
+	"ERR 13: ILLEGAL_CONTEXT_ID",
+	"ERR 14: ILLEGAL_KEY_ADDR",
+	"ERR 15: 0xF Reserved",
+	"ERR 16: Zlib_ILLEGAL_MULTI_QUEUE",
+	"ERR 17: Zlib_ILLEGAL_JOBID_CHANGE",
+	"ERR 18: CMD_TIMEOUT",
+	"ERR 19: IDMA0_AXI_SLVERR",
+	"ERR 20: IDMA0_AXI_DECERR",
+	"ERR 21: 0x15 Reserved",
+	"ERR 22: IDMA1_AXI_SLAVE_FAULT",
+	"ERR 23: IDMA1_AIXI_DECERR",
+	"ERR 24: 0x18 Reserved",
+	"ERR 25: ZLIBVHB_AXI_SLVERR",
+	"ERR 26: ZLIBVHB_AXI_DECERR",
+	"ERR 27: 0x1B Reserved",
+	"ERR 27: ZLIB_UNEXPECTED_EOM",
+	"ERR 27: ZLIB_EXTRA_DATA",
+	"ERR 30: ZLIB_BTYPE",
+	"ERR 31: ZLIB_UNDEFINED_SYMBOL",
+	"ERR 32: ZLIB_UNDEFINED_DISTANCE_S",
+	"ERR 33: ZLIB_CODE_LENGTH_SYMBOL",
+	"ERR 34: ZLIB _VHB_ILLEGAL_FETCH",
+	"ERR 35: ZLIB_UNCOMPRESSED_LEN",
+	"ERR 36: ZLIB_LIMIT_REACHED",
+	"ERR 37: ZLIB_CHECKSUM_MISMATCH0",
+	"ERR 38: ODMA0_AXI_SLVERR",
+	"ERR 39: ODMA0_AXI_DECERR",
+	"ERR 40: 0x28 Reserved",
+	"ERR 41: ODMA1_AXI_SLVERR",
+	"ERR 42: ODMA1_AXI_DECERR",
+	"ERR 43: LSB_PARITY_ERR",
+};
+
+void ccp_log_error(struct ccp_device *d, int e)
+{
+	dev_err(d->dev, "CCP error: %s (0x%x)\n", ccp_error_codes[e], e);
+}
+
 /* List of CCPs, CCP count, read-write access lock, and access functions
  *
  * Lock structure: get ccp_unit_lock for reading whenever we need to
@@ -58,7 +112,7 @@ static struct ccp_device *ccp_rr;
 
 /* Ever-increasing value to produce unique unit numbers */
 static atomic_t ccp_unit_ordinal;
-unsigned int ccp_increment_unit_ordinal(void)
+static unsigned int ccp_increment_unit_ordinal(void)
 {
 	return atomic_inc_return(&ccp_unit_ordinal);
 }
@@ -118,6 +172,29 @@ void ccp_del_device(struct ccp_device *ccp)
 	write_unlock_irqrestore(&ccp_unit_lock, flags);
 }
 
+
+
+int ccp_register_rng(struct ccp_device *ccp)
+{
+	int ret = 0;
+
+	dev_dbg(ccp->dev, "Registering RNG...\n");
+	/* Register an RNG */
+	ccp->hwrng.name = ccp->rngname;
+	ccp->hwrng.read = ccp_trng_read;
+	ret = hwrng_register(&ccp->hwrng);
+	if (ret)
+		dev_err(ccp->dev, "error registering hwrng (%d)\n", ret);
+
+	return ret;
+}
+
+void ccp_unregister_rng(struct ccp_device *ccp)
+{
+	if (ccp->hwrng.name)
+		hwrng_unregister(&ccp->hwrng);
+}
+
 static struct ccp_device *ccp_get_device(void)
 {
 	unsigned long flags;
@@ -397,9 +474,9 @@ struct ccp_device *ccp_alloc_struct(struct device *dev)
 
 	spin_lock_init(&ccp->cmd_lock);
 	mutex_init(&ccp->req_mutex);
-	mutex_init(&ccp->ksb_mutex);
-	ccp->ksb_count = KSB_COUNT;
-	ccp->ksb_start = 0;
+	mutex_init(&ccp->sb_mutex);
+	ccp->sb_count = KSB_COUNT;
+	ccp->sb_start = 0;
 
 	ccp->ord = ccp_increment_unit_ordinal();
 	snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", ccp->ord);
@@ -408,6 +485,34 @@ struct ccp_device *ccp_alloc_struct(struct device *dev)
 	return ccp;
 }
 
+int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
+{
+	struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng);
+	u32 trng_value;
+	int len = min_t(int, sizeof(trng_value), max);
+
+	/* Locking is provided by the caller so we can update device
+	 * hwrng-related fields safely
+	 */
+	trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG);
+	if (!trng_value) {
+		/* Zero is returned if not data is available or if a
+		 * bad-entropy error is present. Assume an error if
+		 * we exceed TRNG_RETRIES reads of zero.
+		 */
+		if (ccp->hwrng_retries++ > TRNG_RETRIES)
+			return -EIO;
+
+		return 0;
+	}
+
+	/* Reset the counter and save the rng value */
+	ccp->hwrng_retries = 0;
+	memcpy(data, &trng_value, len);
+
+	return len;
+}
+
 #ifdef CONFIG_PM
 bool ccp_queues_suspended(struct ccp_device *ccp)
 {
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index bd41ffceff82..da5f4a678083 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -4,6 +4,7 @@
  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ * Author: Gary R Hook <gary.hook@amd.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -60,7 +61,69 @@
 #define CMD_Q_ERROR(__qs)		((__qs) & 0x0000003f)
 #define CMD_Q_DEPTH(__qs)		(((__qs) >> 12) & 0x0000000f)
 
-/****** REQ0 Related Values ******/
+/* ------------------------ CCP Version 5 Specifics ------------------------ */
+#define CMD5_QUEUE_MASK_OFFSET		0x00
+#define	CMD5_QUEUE_PRIO_OFFSET		0x04
+#define CMD5_REQID_CONFIG_OFFSET	0x08
+#define	CMD5_CMD_TIMEOUT_OFFSET		0x10
+#define LSB_PUBLIC_MASK_LO_OFFSET	0x18
+#define LSB_PUBLIC_MASK_HI_OFFSET	0x1C
+#define LSB_PRIVATE_MASK_LO_OFFSET	0x20
+#define LSB_PRIVATE_MASK_HI_OFFSET	0x24
+
+#define CMD5_Q_CONTROL_BASE		0x0000
+#define CMD5_Q_TAIL_LO_BASE		0x0004
+#define CMD5_Q_HEAD_LO_BASE		0x0008
+#define CMD5_Q_INT_ENABLE_BASE		0x000C
+#define CMD5_Q_INTERRUPT_STATUS_BASE	0x0010
+
+#define CMD5_Q_STATUS_BASE		0x0100
+#define CMD5_Q_INT_STATUS_BASE		0x0104
+#define CMD5_Q_DMA_STATUS_BASE		0x0108
+#define CMD5_Q_DMA_READ_STATUS_BASE	0x010C
+#define CMD5_Q_DMA_WRITE_STATUS_BASE	0x0110
+#define CMD5_Q_ABORT_BASE		0x0114
+#define CMD5_Q_AX_CACHE_BASE		0x0118
+
+#define	CMD5_CONFIG_0_OFFSET		0x6000
+#define	CMD5_TRNG_CTL_OFFSET		0x6008
+#define	CMD5_AES_MASK_OFFSET		0x6010
+#define	CMD5_CLK_GATE_CTL_OFFSET	0x603C
+
+/* Address offset between two virtual queue registers */
+#define CMD5_Q_STATUS_INCR		0x1000
+
+/* Bit masks */
+#define CMD5_Q_RUN			0x1
+#define CMD5_Q_HALT			0x2
+#define CMD5_Q_MEM_LOCATION		0x4
+#define CMD5_Q_SIZE			0x1F
+#define CMD5_Q_SHIFT			3
+#define COMMANDS_PER_QUEUE		16
+#define QUEUE_SIZE_VAL			((ffs(COMMANDS_PER_QUEUE) - 2) & \
+					  CMD5_Q_SIZE)
+#define Q_PTR_MASK			(2 << (QUEUE_SIZE_VAL + 5) - 1)
+#define Q_DESC_SIZE			sizeof(struct ccp5_desc)
+#define Q_SIZE(n)			(COMMANDS_PER_QUEUE*(n))
+
+#define INT_COMPLETION			0x1
+#define INT_ERROR			0x2
+#define INT_QUEUE_STOPPED		0x4
+#define ALL_INTERRUPTS			(INT_COMPLETION| \
+					 INT_ERROR| \
+					 INT_QUEUE_STOPPED)
+
+#define LSB_REGION_WIDTH		5
+#define MAX_LSB_CNT			8
+
+#define LSB_SIZE			16
+#define LSB_ITEM_SIZE			32
+#define PLSB_MAP_SIZE			(LSB_SIZE)
+#define SLSB_MAP_SIZE			(MAX_LSB_CNT * LSB_SIZE)
+
+#define LSB_ENTRY_NUMBER(LSB_ADDR)	(LSB_ADDR / LSB_ITEM_SIZE)
+
+/* ------------------------ CCP Version 3 Specifics ------------------------ */
 #define REQ0_WAIT_FOR_WRITE		0x00000004
 #define REQ0_INT_ON_COMPLETE		0x00000002
 #define REQ0_STOP_ON_COMPLETE		0x00000001
@@ -110,29 +173,30 @@
 #define KSB_START			77
 #define KSB_END				127
 #define KSB_COUNT			(KSB_END - KSB_START + 1)
-#define CCP_KSB_BITS			256
-#define CCP_KSB_BYTES			32
+#define CCP_SB_BITS			256
 
 #define CCP_JOBID_MASK			0x0000003f
 
+/* ------------------------ General CCP Defines ------------------------ */
+
 #define CCP_DMAPOOL_MAX_SIZE		64
 #define CCP_DMAPOOL_ALIGN		BIT(5)
 
 #define CCP_REVERSE_BUF_SIZE		64
 
-#define CCP_AES_KEY_KSB_COUNT		1
-#define CCP_AES_CTX_KSB_COUNT		1
+#define CCP_AES_KEY_SB_COUNT		1
+#define CCP_AES_CTX_SB_COUNT		1
 
-#define CCP_XTS_AES_KEY_KSB_COUNT	1
-#define CCP_XTS_AES_CTX_KSB_COUNT	1
+#define CCP_XTS_AES_KEY_SB_COUNT	1
+#define CCP_XTS_AES_CTX_SB_COUNT	1
 
-#define CCP_SHA_KSB_COUNT		1
+#define CCP_SHA_SB_COUNT		1
 
 #define CCP_RSA_MAX_WIDTH		4096
 
 #define CCP_PASSTHRU_BLOCKSIZE		256
 #define CCP_PASSTHRU_MASKSIZE		32
-#define CCP_PASSTHRU_KSB_COUNT		1
+#define CCP_PASSTHRU_SB_COUNT		1
 
 #define CCP_ECC_MODULUS_BYTES		48      /* 384-bits */
 #define CCP_ECC_MAX_OPERANDS		6
@@ -144,31 +208,12 @@
 #define CCP_ECC_RESULT_OFFSET		60
 #define CCP_ECC_RESULT_SUCCESS		0x0001
 
-struct ccp_op;
-
-/* Structure for computation functions that are device-specific */
-struct ccp_actions {
-	int (*perform_aes)(struct ccp_op *);
-	int (*perform_xts_aes)(struct ccp_op *);
-	int (*perform_sha)(struct ccp_op *);
-	int (*perform_rsa)(struct ccp_op *);
-	int (*perform_passthru)(struct ccp_op *);
-	int (*perform_ecc)(struct ccp_op *);
-	int (*init)(struct ccp_device *);
-	void (*destroy)(struct ccp_device *);
-	irqreturn_t (*irqhandler)(int, void *);
-};
-
-/* Structure to hold CCP version-specific values */
-struct ccp_vdata {
-	unsigned int version;
-	const struct ccp_actions *perform;
-};
-
-extern struct ccp_vdata ccpv3;
+#define CCP_SB_BYTES			32
 
+struct ccp_op;
 struct ccp_device;
 struct ccp_cmd;
+struct ccp_fns;
 
 struct ccp_dma_cmd {
 	struct list_head entry;
@@ -212,9 +257,29 @@ struct ccp_cmd_queue {
 	/* Queue dma pool */
 	struct dma_pool *dma_pool;
 
-	/* Queue reserved KSB regions */
-	u32 ksb_key;
-	u32 ksb_ctx;
+	/* Queue base address (not neccessarily aligned)*/
+	struct ccp5_desc *qbase;
+
+	/* Aligned queue start address (per requirement) */
+	struct mutex q_mutex ____cacheline_aligned;
+	unsigned int qidx;
+
+	/* Version 5 has different requirements for queue memory */
+	unsigned int qsize;
+	dma_addr_t qbase_dma;
+	dma_addr_t qdma_tail;
+
+	/* Per-queue reserved storage block(s) */
+	u32 sb_key;
+	u32 sb_ctx;
+
+	/* Bitmap of LSBs that can be accessed by this queue */
+	DECLARE_BITMAP(lsbmask, MAX_LSB_CNT);
+	/* Private LSB that is assigned to this queue, or -1 if none.
+	 * Bitmap for my private LSB, unused otherwise
+	 */
+	unsigned int lsb;
+	DECLARE_BITMAP(lsbmap, PLSB_MAP_SIZE);
 
 	/* Queue processing thread */
 	struct task_struct *kthread;
@@ -229,8 +294,17 @@ struct ccp_cmd_queue {
 	u32 int_err;
 
 	/* Register addresses for queue */
+	void __iomem *reg_control;
+	void __iomem *reg_tail_lo;
+	void __iomem *reg_head_lo;
+	void __iomem *reg_int_enable;
+	void __iomem *reg_interrupt_status;
 	void __iomem *reg_status;
 	void __iomem *reg_int_status;
+	void __iomem *reg_dma_status;
+	void __iomem *reg_dma_read_status;
+	void __iomem *reg_dma_write_status;
+	u32 qcontrol; /* Cached control register */
 
 	/* Status values from job */
 	u32 int_status;
@@ -253,16 +327,14 @@ struct ccp_device {
 
 	struct device *dev;
 
-	/*
-	 * Bus specific device information
+	/* Bus specific device information
 	 */
 	void *dev_specific;
 	int (*get_irq)(struct ccp_device *ccp);
 	void (*free_irq)(struct ccp_device *ccp);
 	unsigned int irq;
 
-	/*
-	 * I/O area used for device communication. The register mapping
+	/* I/O area used for device communication. The register mapping
 	 * starts at an offset into the mapped bar.
 	 *   The CMD_REQx registers and the Delete_Cmd_Queue_Job register
 	 *   need to be protected while a command queue thread is accessing
@@ -272,8 +344,7 @@ struct ccp_device {
 	void __iomem *io_map;
 	void __iomem *io_regs;
 
-	/*
-	 * Master lists that all cmds are queued on. Because there can be
+	/* Master lists that all cmds are queued on. Because there can be
 	 * more than one CCP command queue that can process a cmd a separate
 	 * backlog list is neeeded so that the backlog completion call
 	 * completes before the cmd is available for execution.
@@ -283,47 +354,54 @@ struct ccp_device {
 	struct list_head cmd;
 	struct list_head backlog;
 
-	/*
-	 * The command queues. These represent the queues available on the
+	/* The command queues. These represent the queues available on the
 	 * CCP that are available for processing cmds
 	 */
 	struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES];
 	unsigned int cmd_q_count;
 
-	/*
-	 * Support for the CCP True RNG
+	/* Support for the CCP True RNG
 	 */
 	struct hwrng hwrng;
 	unsigned int hwrng_retries;
 
-	/*
-	 * Support for the CCP DMA capabilities
+	/* Support for the CCP DMA capabilities
 	 */
 	struct dma_device dma_dev;
 	struct ccp_dma_chan *ccp_dma_chan;
 	struct kmem_cache *dma_cmd_cache;
 	struct kmem_cache *dma_desc_cache;
 
-	/*
-	 * A counter used to generate job-ids for cmds submitted to the CCP
+	/* A counter used to generate job-ids for cmds submitted to the CCP
 	 */
 	atomic_t current_id ____cacheline_aligned;
 
-	/*
-	 * The CCP uses key storage blocks (KSB) to maintain context for certain
-	 * operations. To prevent multiple cmds from using the same KSB range
-	 * a command queue reserves a KSB range for the duration of the cmd.
-	 * Each queue, will however, reserve 2 KSB blocks for operations that
-	 * only require single KSB entries (eg. AES context/iv and key) in order
-	 * to avoid allocation contention.  This will reserve at most 10 KSB
-	 * entries, leaving 40 KSB entries available for dynamic allocation.
+	/* The v3 CCP uses key storage blocks (SB) to maintain context for
+	 * certain operations. To prevent multiple cmds from using the same
+	 * SB range a command queue reserves an SB range for the duration of
+	 * the cmd. Each queue, will however, reserve 2 SB blocks for
+	 * operations that only require single SB entries (eg. AES context/iv
+	 * and key) in order to avoid allocation contention.  This will reserve
+	 * at most 10 SB entries, leaving 40 SB entries available for dynamic
+	 * allocation.
+	 *
+	 * The v5 CCP Local Storage Block (LSB) is broken up into 8
+	 * memrory ranges, each of which can be enabled for access by one
+	 * or more queues. Device initialization takes this into account,
+	 * and attempts to assign one region for exclusive use by each
+	 * available queue; the rest are then aggregated as "public" use.
+	 * If there are fewer regions than queues, all regions are shared
+	 * amongst all queues.
 	 */
-	struct mutex ksb_mutex ____cacheline_aligned;
-	DECLARE_BITMAP(ksb, KSB_COUNT);
-	wait_queue_head_t ksb_queue;
-	unsigned int ksb_avail;
-	unsigned int ksb_count;
-	u32 ksb_start;
+	struct mutex sb_mutex ____cacheline_aligned;
+	DECLARE_BITMAP(sb, KSB_COUNT);
+	wait_queue_head_t sb_queue;
+	unsigned int sb_avail;
+	unsigned int sb_count;
+	u32 sb_start;
+
+	/* Bitmap of shared LSBs, if any */
+	DECLARE_BITMAP(lsbmap, SLSB_MAP_SIZE);
 
 	/* Suspend support */
 	unsigned int suspending;
@@ -335,10 +413,11 @@ struct ccp_device {
 
 enum ccp_memtype {
 	CCP_MEMTYPE_SYSTEM = 0,
-	CCP_MEMTYPE_KSB,
+	CCP_MEMTYPE_SB,
 	CCP_MEMTYPE_LOCAL,
 	CCP_MEMTYPE__LAST,
 };
+#define	CCP_MEMTYPE_LSB	CCP_MEMTYPE_KSB
 
 struct ccp_dma_info {
 	dma_addr_t address;
@@ -379,7 +458,7 @@ struct ccp_mem {
 	enum ccp_memtype type;
 	union {
 		struct ccp_dma_info dma;
-		u32 ksb;
+		u32 sb;
 	} u;
 };
 
@@ -419,13 +498,14 @@ struct ccp_op {
 	u32 jobid;
 	u32 ioc;
 	u32 soc;
-	u32 ksb_key;
-	u32 ksb_ctx;
+	u32 sb_key;
+	u32 sb_ctx;
 	u32 init;
 	u32 eom;
 
 	struct ccp_mem src;
 	struct ccp_mem dst;
+	struct ccp_mem exp;
 
 	union {
 		struct ccp_aes_op aes;
@@ -435,6 +515,7 @@ struct ccp_op {
 		struct ccp_passthru_op passthru;
 		struct ccp_ecc_op ecc;
 	} u;
+	struct ccp_mem key;
 };
 
 static inline u32 ccp_addr_lo(struct ccp_dma_info *info)
@@ -447,6 +528,70 @@ static inline u32 ccp_addr_hi(struct ccp_dma_info *info)
 	return upper_32_bits(info->address + info->offset) & 0x0000ffff;
 }
 
+/**
+ * descriptor for version 5 CPP commands
+ * 8 32-bit words:
+ * word 0: function; engine; control bits
+ * word 1: length of source data
+ * word 2: low 32 bits of source pointer
+ * word 3: upper 16 bits of source pointer; source memory type
+ * word 4: low 32 bits of destination pointer
+ * word 5: upper 16 bits of destination pointer; destination memory type
+ * word 6: low 32 bits of key pointer
+ * word 7: upper 16 bits of key pointer; key memory type
+ */
+struct dword0 {
+	__le32 soc:1;
+	__le32 ioc:1;
+	__le32 rsvd1:1;
+	__le32 init:1;
+	__le32 eom:1;		/* AES/SHA only */
+	__le32 function:15;
+	__le32 engine:4;
+	__le32 prot:1;
+	__le32 rsvd2:7;
+};
+
+struct dword3 {
+	__le32 src_hi:16;
+	__le32 src_mem:2;
+	__le32 lsb_cxt_id:8;
+	__le32 rsvd1:5;
+	__le32 fixed:1;
+};
+
+union dword4 {
+	__le32 dst_lo;		/* NON-SHA	*/
+	__le32 sha_len_lo;	/* SHA		*/
+};
+
+union dword5 {
+	struct {
+		__le32 dst_hi:16;
+		__le32 dst_mem:2;
+		__le32 rsvd1:13;
+		__le32 fixed:1;
+	} fields;
+	__le32 sha_len_hi;
+};
+
+struct dword7 {
+	__le32 key_hi:16;
+	__le32 key_mem:2;
+	__le32 rsvd1:14;
+};
+
+struct ccp5_desc {
+	struct dword0 dw0;
+	__le32 length;
+	__le32 src_lo;
+	struct dword3 dw3;
+	union dword4 dw4;
+	union dword5 dw5;
+	__le32 key_lo;
+	struct dword7 dw7;
+};
+
 int ccp_pci_init(void);
 void ccp_pci_exit(void);
 
@@ -456,13 +601,48 @@ void ccp_platform_exit(void);
 void ccp_add_device(struct ccp_device *ccp);
 void ccp_del_device(struct ccp_device *ccp);
 
+extern void ccp_log_error(struct ccp_device *, int);
+
 struct ccp_device *ccp_alloc_struct(struct device *dev);
 bool ccp_queues_suspended(struct ccp_device *ccp);
 int ccp_cmd_queue_thread(void *data);
+int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait);
 
 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd);
 
+int ccp_register_rng(struct ccp_device *ccp);
+void ccp_unregister_rng(struct ccp_device *ccp);
 int ccp_dmaengine_register(struct ccp_device *ccp);
 void ccp_dmaengine_unregister(struct ccp_device *ccp);
 
+/* Structure for computation functions that are device-specific */
+struct ccp_actions {
+	int (*aes)(struct ccp_op *);
+	int (*xts_aes)(struct ccp_op *);
+	int (*sha)(struct ccp_op *);
+	int (*rsa)(struct ccp_op *);
+	int (*passthru)(struct ccp_op *);
+	int (*ecc)(struct ccp_op *);
+	u32 (*sballoc)(struct ccp_cmd_queue *, unsigned int);
+	void (*sbfree)(struct ccp_cmd_queue *, unsigned int,
+			       unsigned int);
+	unsigned int (*get_free_slots)(struct ccp_cmd_queue *);
+	int (*init)(struct ccp_device *);
+	void (*destroy)(struct ccp_device *);
+	irqreturn_t (*irqhandler)(int, void *);
+};
+
+/* Structure to hold CCP version-specific values */
+struct ccp_vdata {
+	const unsigned int version;
+	void (*setup)(struct ccp_device *);
+	const struct ccp_actions *perform;
+	const unsigned int bar;
+	const unsigned int offset;
+};
+
+extern const struct ccp_vdata ccpv3;
+extern const struct ccp_vdata ccpv5a;
+extern const struct ccp_vdata ccpv5b;
+
 #endif
diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c
index 94f77b0f9ae7..6553912804f7 100644
--- a/drivers/crypto/ccp/ccp-dmaengine.c
+++ b/drivers/crypto/ccp/ccp-dmaengine.c
@@ -299,12 +299,10 @@ static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan,
 {
 	struct ccp_dma_desc *desc;
 
-	desc = kmem_cache_alloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);
+	desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);
 	if (!desc)
 		return NULL;
 
-	memset(desc, 0, sizeof(*desc));
-
 	dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan);
 	desc->tx_desc.flags = flags;
 	desc->tx_desc.tx_submit = ccp_tx_submit;
@@ -650,8 +648,11 @@ int ccp_dmaengine_register(struct ccp_device *ccp)
 	dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
 					     "%s-dmaengine-desc-cache",
 					     ccp->name);
-	if (!dma_cmd_cache_name)
-		return -ENOMEM;
+	if (!dma_desc_cache_name) {
+		ret = -ENOMEM;
+		goto err_cache;
+	}
+
 	ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name,
 						sizeof(struct ccp_dma_desc),
 						sizeof(void *),
diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c
index ffa2891035ac..50fae4442801 100644
--- a/drivers/crypto/ccp/ccp-ops.c
+++ b/drivers/crypto/ccp/ccp-ops.c
@@ -4,6 +4,7 @@
  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ * Author: Gary R Hook <gary.hook@amd.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -20,72 +21,28 @@
 #include "ccp-dev.h"
 
 /* SHA initial context values */
-static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
+static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
 	cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
 	cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
-	cpu_to_be32(SHA1_H4), 0, 0, 0,
+	cpu_to_be32(SHA1_H4),
 };
 
-static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
+static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
 	cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
 	cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
 	cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
 	cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
 };
 
-static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
+static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
 	cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
 	cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
 	cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
 	cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
 };
 
-static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
-{
-	int start;
-
-	for (;;) {
-		mutex_lock(&ccp->ksb_mutex);
-
-		start = (u32)bitmap_find_next_zero_area(ccp->ksb,
-							ccp->ksb_count,
-							ccp->ksb_start,
-							count, 0);
-		if (start <= ccp->ksb_count) {
-			bitmap_set(ccp->ksb, start, count);
-
-			mutex_unlock(&ccp->ksb_mutex);
-			break;
-		}
-
-		ccp->ksb_avail = 0;
-
-		mutex_unlock(&ccp->ksb_mutex);
-
-		/* Wait for KSB entries to become available */
-		if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
-			return 0;
-	}
-
-	return KSB_START + start;
-}
-
-static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
-			 unsigned int count)
-{
-	if (!start)
-		return;
-
-	mutex_lock(&ccp->ksb_mutex);
-
-	bitmap_clear(ccp->ksb, start - KSB_START, count);
-
-	ccp->ksb_avail = 1;
-
-	mutex_unlock(&ccp->ksb_mutex);
-
-	wake_up_interruptible_all(&ccp->ksb_queue);
-}
+#define	CCP_NEW_JOBID(ccp)	((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
+					ccp_gen_jobid(ccp) : 0)
 
 static u32 ccp_gen_jobid(struct ccp_device *ccp)
 {
@@ -231,7 +188,7 @@ static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 				   unsigned int len, unsigned int se_len,
 				   bool sign_extend)
 {
-	unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
+	unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
 	u8 buffer[CCP_REVERSE_BUF_SIZE];
 
 	if (WARN_ON(se_len > sizeof(buffer)))
@@ -241,21 +198,21 @@ static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 	dm_offset = 0;
 	nbytes = len;
 	while (nbytes) {
-		ksb_len = min_t(unsigned int, nbytes, se_len);
-		sg_offset -= ksb_len;
+		sb_len = min_t(unsigned int, nbytes, se_len);
+		sg_offset -= sb_len;
 
-		scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
-		for (i = 0; i < ksb_len; i++)
-			wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
+		scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 0);
+		for (i = 0; i < sb_len; i++)
+			wa->address[dm_offset + i] = buffer[sb_len - i - 1];
 
-		dm_offset += ksb_len;
-		nbytes -= ksb_len;
+		dm_offset += sb_len;
+		nbytes -= sb_len;
 
-		if ((ksb_len != se_len) && sign_extend) {
+		if ((sb_len != se_len) && sign_extend) {
 			/* Must sign-extend to nearest sign-extend length */
 			if (wa->address[dm_offset - 1] & 0x80)
 				memset(wa->address + dm_offset, 0xff,
-				       se_len - ksb_len);
+				       se_len - sb_len);
 		}
 	}
 
@@ -266,22 +223,22 @@ static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 				    struct scatterlist *sg,
 				    unsigned int len)
 {
-	unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
+	unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
 	u8 buffer[CCP_REVERSE_BUF_SIZE];
 
 	sg_offset = 0;
 	dm_offset = len;
 	nbytes = len;
 	while (nbytes) {
-		ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
-		dm_offset -= ksb_len;
+		sb_len = min_t(unsigned int, nbytes, sizeof(buffer));
+		dm_offset -= sb_len;
 
-		for (i = 0; i < ksb_len; i++)
-			buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
-		scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
+		for (i = 0; i < sb_len; i++)
+			buffer[sb_len - i - 1] = wa->address[dm_offset + i];
+		scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 1);
 
-		sg_offset += ksb_len;
-		nbytes -= ksb_len;
+		sg_offset += sb_len;
+		nbytes -= sb_len;
 	}
 }
 
@@ -449,9 +406,9 @@ static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 	}
 }
 
-static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
-				struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
-				u32 byte_swap, bool from)
+static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
+			       struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
+			       u32 byte_swap, bool from)
 {
 	struct ccp_op op;
 
@@ -463,8 +420,8 @@ static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
 
 	if (from) {
 		op.soc = 1;
-		op.src.type = CCP_MEMTYPE_KSB;
-		op.src.u.ksb = ksb;
+		op.src.type = CCP_MEMTYPE_SB;
+		op.src.u.sb = sb;
 		op.dst.type = CCP_MEMTYPE_SYSTEM;
 		op.dst.u.dma.address = wa->dma.address;
 		op.dst.u.dma.length = wa->length;
@@ -472,27 +429,27 @@ static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
 		op.src.type = CCP_MEMTYPE_SYSTEM;
 		op.src.u.dma.address = wa->dma.address;
 		op.src.u.dma.length = wa->length;
-		op.dst.type = CCP_MEMTYPE_KSB;
-		op.dst.u.ksb = ksb;
+		op.dst.type = CCP_MEMTYPE_SB;
+		op.dst.u.sb = sb;
 	}
 
 	op.u.passthru.byte_swap = byte_swap;
 
-	return cmd_q->ccp->vdata->perform->perform_passthru(&op);
+	return cmd_q->ccp->vdata->perform->passthru(&op);
 }
 
-static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
-			   struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
-			   u32 byte_swap)
+static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
+			  struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
+			  u32 byte_swap)
 {
-	return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
+	return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 }
 
-static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
-			     struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
-			     u32 byte_swap)
+static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
+			    struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
+			    u32 byte_swap)
 {
-	return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
+	return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 }
 
 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
@@ -527,54 +484,54 @@ static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 			return -EINVAL;
 	}
 
-	BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
-	BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
+	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
+	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 
 	ret = -EIO;
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
-	op.jobid = ccp_gen_jobid(cmd_q->ccp);
-	op.ksb_key = cmd_q->ksb_key;
-	op.ksb_ctx = cmd_q->ksb_ctx;
+	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
+	op.sb_key = cmd_q->sb_key;
+	op.sb_ctx = cmd_q->sb_ctx;
 	op.init = 1;
 	op.u.aes.type = aes->type;
 	op.u.aes.mode = aes->mode;
 	op.u.aes.action = aes->action;
 
-	/* All supported key sizes fit in a single (32-byte) KSB entry
+	/* All supported key sizes fit in a single (32-byte) SB entry
 	 * and must be in little endian format. Use the 256-bit byte
 	 * swap passthru option to convert from big endian to little
 	 * endian.
 	 */
 	ret = ccp_init_dm_workarea(&key, cmd_q,
-				   CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
+				   CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 				   DMA_TO_DEVICE);
 	if (ret)
 		return ret;
 
-	dm_offset = CCP_KSB_BYTES - aes->key_len;
+	dm_offset = CCP_SB_BYTES - aes->key_len;
 	ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
-	ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
-			      CCP_PASSTHRU_BYTESWAP_256BIT);
+	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
+			     CCP_PASSTHRU_BYTESWAP_256BIT);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_key;
 	}
 
-	/* The AES context fits in a single (32-byte) KSB entry and
+	/* The AES context fits in a single (32-byte) SB entry and
 	 * must be in little endian format. Use the 256-bit byte swap
 	 * passthru option to convert from big endian to little endian.
 	 */
 	ret = ccp_init_dm_workarea(&ctx, cmd_q,
-				   CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
+				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 				   DMA_BIDIRECTIONAL);
 	if (ret)
 		goto e_key;
 
-	dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
+	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 	ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
-	ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
-			      CCP_PASSTHRU_BYTESWAP_256BIT);
+	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
+			     CCP_PASSTHRU_BYTESWAP_256BIT);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_ctx;
@@ -592,9 +549,9 @@ static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 			op.eom = 1;
 
 			/* Push the K1/K2 key to the CCP now */
-			ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
-						op.ksb_ctx,
-						CCP_PASSTHRU_BYTESWAP_256BIT);
+			ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
+					       op.sb_ctx,
+					       CCP_PASSTHRU_BYTESWAP_256BIT);
 			if (ret) {
 				cmd->engine_error = cmd_q->cmd_error;
 				goto e_src;
@@ -602,15 +559,15 @@ static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 
 			ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 					aes->cmac_key_len);
-			ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
-					      CCP_PASSTHRU_BYTESWAP_256BIT);
+			ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
+					     CCP_PASSTHRU_BYTESWAP_256BIT);
 			if (ret) {
 				cmd->engine_error = cmd_q->cmd_error;
 				goto e_src;
 			}
 		}
 
-		ret = cmd_q->ccp->vdata->perform->perform_aes(&op);
+		ret = cmd_q->ccp->vdata->perform->aes(&op);
 		if (ret) {
 			cmd->engine_error = cmd_q->cmd_error;
 			goto e_src;
@@ -622,15 +579,15 @@ static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 	/* Retrieve the AES context - convert from LE to BE using
 	 * 32-byte (256-bit) byteswapping
 	 */
-	ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
-				CCP_PASSTHRU_BYTESWAP_256BIT);
+	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
+			       CCP_PASSTHRU_BYTESWAP_256BIT);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_src;
 	}
 
 	/* ...but we only need AES_BLOCK_SIZE bytes */
-	dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
+	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 	ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 
 e_src:
@@ -680,56 +637,56 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 			return -EINVAL;
 	}
 
-	BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
-	BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
+	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
+	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 
 	ret = -EIO;
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
-	op.jobid = ccp_gen_jobid(cmd_q->ccp);
-	op.ksb_key = cmd_q->ksb_key;
-	op.ksb_ctx = cmd_q->ksb_ctx;
+	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
+	op.sb_key = cmd_q->sb_key;
+	op.sb_ctx = cmd_q->sb_ctx;
 	op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 	op.u.aes.type = aes->type;
 	op.u.aes.mode = aes->mode;
 	op.u.aes.action = aes->action;
 
-	/* All supported key sizes fit in a single (32-byte) KSB entry
+	/* All supported key sizes fit in a single (32-byte) SB entry
 	 * and must be in little endian format. Use the 256-bit byte
 	 * swap passthru option to convert from big endian to little
 	 * endian.
 	 */
 	ret = ccp_init_dm_workarea(&key, cmd_q,
-				   CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
+				   CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 				   DMA_TO_DEVICE);
 	if (ret)
 		return ret;
 
-	dm_offset = CCP_KSB_BYTES - aes->key_len;
+	dm_offset = CCP_SB_BYTES - aes->key_len;
 	ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
-	ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
-			      CCP_PASSTHRU_BYTESWAP_256BIT);
+	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
+			     CCP_PASSTHRU_BYTESWAP_256BIT);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_key;
 	}
 
-	/* The AES context fits in a single (32-byte) KSB entry and
+	/* The AES context fits in a single (32-byte) SB entry and
 	 * must be in little endian format. Use the 256-bit byte swap
 	 * passthru option to convert from big endian to little endian.
 	 */
 	ret = ccp_init_dm_workarea(&ctx, cmd_q,
-				   CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
+				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 				   DMA_BIDIRECTIONAL);
 	if (ret)
 		goto e_key;
 
 	if (aes->mode != CCP_AES_MODE_ECB) {
-		/* Load the AES context - conver to LE */
-		dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
+		/* Load the AES context - convert to LE */
+		dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 		ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
-		ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
-				      CCP_PASSTHRU_BYTESWAP_256BIT);
+		ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
+				     CCP_PASSTHRU_BYTESWAP_256BIT);
 		if (ret) {
 			cmd->engine_error = cmd_q->cmd_error;
 			goto e_ctx;
@@ -772,7 +729,7 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 				op.soc = 1;
 		}
 
-		ret = cmd_q->ccp->vdata->perform->perform_aes(&op);
+		ret = cmd_q->ccp->vdata->perform->aes(&op);
 		if (ret) {
 			cmd->engine_error = cmd_q->cmd_error;
 			goto e_dst;
@@ -785,15 +742,15 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 		/* Retrieve the AES context - convert from LE to BE using
 		 * 32-byte (256-bit) byteswapping
 		 */
-		ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
-					CCP_PASSTHRU_BYTESWAP_256BIT);
+		ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
+				       CCP_PASSTHRU_BYTESWAP_256BIT);
 		if (ret) {
 			cmd->engine_error = cmd_q->cmd_error;
 			goto e_dst;
 		}
 
 		/* ...but we only need AES_BLOCK_SIZE bytes */
-		dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
+		dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 		ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 	}
 
@@ -857,53 +814,53 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
 	if (!xts->key || !xts->iv || !xts->src || !xts->dst)
 		return -EINVAL;
 
-	BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
-	BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
+	BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
+	BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
 
 	ret = -EIO;
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
-	op.jobid = ccp_gen_jobid(cmd_q->ccp);
-	op.ksb_key = cmd_q->ksb_key;
-	op.ksb_ctx = cmd_q->ksb_ctx;
+	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
+	op.sb_key = cmd_q->sb_key;
+	op.sb_ctx = cmd_q->sb_ctx;
 	op.init = 1;
 	op.u.xts.action = xts->action;
 	op.u.xts.unit_size = xts->unit_size;
 
-	/* All supported key sizes fit in a single (32-byte) KSB entry
+	/* All supported key sizes fit in a single (32-byte) SB entry
 	 * and must be in little endian format. Use the 256-bit byte
 	 * swap passthru option to convert from big endian to little
 	 * endian.
 	 */
 	ret = ccp_init_dm_workarea(&key, cmd_q,
-				   CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
+				   CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 				   DMA_TO_DEVICE);
 	if (ret)
 		return ret;
 
-	dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
+	dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
 	ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
 	ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
-	ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
-			      CCP_PASSTHRU_BYTESWAP_256BIT);
+	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
+			     CCP_PASSTHRU_BYTESWAP_256BIT);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_key;
 	}
 
-	/* The AES context fits in a single (32-byte) KSB entry and
+	/* The AES context fits in a single (32-byte) SB entry and
 	 * for XTS is already in little endian format so no byte swapping
 	 * is needed.
 	 */
 	ret = ccp_init_dm_workarea(&ctx, cmd_q,
-				   CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
+				   CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 				   DMA_BIDIRECTIONAL);
 	if (ret)
 		goto e_key;
 
 	ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
-	ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
-			      CCP_PASSTHRU_BYTESWAP_NOOP);
+	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
+			     CCP_PASSTHRU_BYTESWAP_NOOP);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_ctx;
@@ -937,7 +894,7 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
 		if (!src.sg_wa.bytes_left)
 			op.eom = 1;
 
-		ret = cmd_q->ccp->vdata->perform->perform_xts_aes(&op);
+		ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
 		if (ret) {
 			cmd->engine_error = cmd_q->cmd_error;
 			goto e_dst;
@@ -949,15 +906,15 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
 	/* Retrieve the AES context - convert from LE to BE using
 	 * 32-byte (256-bit) byteswapping
 	 */
-	ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
-				CCP_PASSTHRU_BYTESWAP_256BIT);
+	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
+			       CCP_PASSTHRU_BYTESWAP_256BIT);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_dst;
 	}
 
 	/* ...but we only need AES_BLOCK_SIZE bytes */
-	dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
+	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 	ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
 
 e_dst:
@@ -982,163 +939,227 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 	struct ccp_dm_workarea ctx;
 	struct ccp_data src;
 	struct ccp_op op;
+	unsigned int ioffset, ooffset;
+	unsigned int digest_size;
+	int sb_count;
+	const void *init;
+	u64 block_size;
+	int ctx_size;
 	int ret;
 
-	if (sha->ctx_len != CCP_SHA_CTXSIZE)
+	switch (sha->type) {
+	case CCP_SHA_TYPE_1:
+		if (sha->ctx_len < SHA1_DIGEST_SIZE)
+			return -EINVAL;
+		block_size = SHA1_BLOCK_SIZE;
+		break;
+	case CCP_SHA_TYPE_224:
+		if (sha->ctx_len < SHA224_DIGEST_SIZE)
+			return -EINVAL;
+		block_size = SHA224_BLOCK_SIZE;
+		break;
+	case CCP_SHA_TYPE_256:
+		if (sha->ctx_len < SHA256_DIGEST_SIZE)
+			return -EINVAL;
+		block_size = SHA256_BLOCK_SIZE;
+		break;
+	default:
 		return -EINVAL;
+	}
 
 	if (!sha->ctx)
 		return -EINVAL;
 
-	if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
+	if (!sha->final && (sha->src_len & (block_size - 1)))
 		return -EINVAL;
 
-	if (!sha->src_len) {
-		const u8 *sha_zero;
+	/* The version 3 device can't handle zero-length input */
+	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
 
-		/* Not final, just return */
-		if (!sha->final)
-			return 0;
+		if (!sha->src_len) {
+			unsigned int digest_len;
+			const u8 *sha_zero;
 
-		/* CCP can't do a zero length sha operation so the caller
-		 * must buffer the data.
-		 */
-		if (sha->msg_bits)
-			return -EINVAL;
+			/* Not final, just return */
+			if (!sha->final)
+				return 0;
 
-		/* The CCP cannot perform zero-length sha operations so the
-		 * caller is required to buffer data for the final operation.
-		 * However, a sha operation for a message with a total length
-		 * of zero is valid so known values are required to supply
-		 * the result.
-		 */
-		switch (sha->type) {
-		case CCP_SHA_TYPE_1:
-			sha_zero = sha1_zero_message_hash;
-			break;
-		case CCP_SHA_TYPE_224:
-			sha_zero = sha224_zero_message_hash;
-			break;
-		case CCP_SHA_TYPE_256:
-			sha_zero = sha256_zero_message_hash;
-			break;
-		default:
-			return -EINVAL;
-		}
+			/* CCP can't do a zero length sha operation so the
+			 * caller must buffer the data.
+			 */
+			if (sha->msg_bits)
+				return -EINVAL;
+
+			/* The CCP cannot perform zero-length sha operations
+			 * so the caller is required to buffer data for the
+			 * final operation. However, a sha operation for a
+			 * message with a total length of zero is valid so
+			 * known values are required to supply the result.
+			 */
+			switch (sha->type) {
+			case CCP_SHA_TYPE_1:
+				sha_zero = sha1_zero_message_hash;
+				digest_len = SHA1_DIGEST_SIZE;
+				break;
+			case CCP_SHA_TYPE_224:
+				sha_zero = sha224_zero_message_hash;
+				digest_len = SHA224_DIGEST_SIZE;
+				break;
+			case CCP_SHA_TYPE_256:
+				sha_zero = sha256_zero_message_hash;
+				digest_len = SHA256_DIGEST_SIZE;
+				break;
+			default:
+				return -EINVAL;
+			}
 
-		scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
-					 sha->ctx_len, 1);
+			scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
+						 digest_len, 1);
 
-		return 0;
+			return 0;
+		}
 	}
 
-	if (!sha->src)
-		return -EINVAL;
+	/* Set variables used throughout */
+	switch (sha->type) {
+	case CCP_SHA_TYPE_1:
+		digest_size = SHA1_DIGEST_SIZE;
+		init = (void *) ccp_sha1_init;
+		ctx_size = SHA1_DIGEST_SIZE;
+		sb_count = 1;
+		if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
+			ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
+		else
+			ooffset = ioffset = 0;
+		break;
+	case CCP_SHA_TYPE_224:
+		digest_size = SHA224_DIGEST_SIZE;
+		init = (void *) ccp_sha224_init;
+		ctx_size = SHA256_DIGEST_SIZE;
+		sb_count = 1;
+		ioffset = 0;
+		if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
+			ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
+		else
+			ooffset = 0;
+		break;
+	case CCP_SHA_TYPE_256:
+		digest_size = SHA256_DIGEST_SIZE;
+		init = (void *) ccp_sha256_init;
+		ctx_size = SHA256_DIGEST_SIZE;
+		sb_count = 1;
+		ooffset = ioffset = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		goto e_data;
+	}
 
-	BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
+	/* For zero-length plaintext the src pointer is ignored;
+	 * otherwise both parts must be valid
+	 */
+	if (sha->src_len && !sha->src)
+		return -EINVAL;
 
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
-	op.jobid = ccp_gen_jobid(cmd_q->ccp);
-	op.ksb_ctx = cmd_q->ksb_ctx;
+	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
+	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 	op.u.sha.type = sha->type;
 	op.u.sha.msg_bits = sha->msg_bits;
 
-	/* The SHA context fits in a single (32-byte) KSB entry and
-	 * must be in little endian format. Use the 256-bit byte swap
-	 * passthru option to convert from big endian to little endian.
-	 */
-	ret = ccp_init_dm_workarea(&ctx, cmd_q,
-				   CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
+	ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
 				   DMA_BIDIRECTIONAL);
 	if (ret)
 		return ret;
-
 	if (sha->first) {
-		const __be32 *init;
-
 		switch (sha->type) {
 		case CCP_SHA_TYPE_1:
-			init = ccp_sha1_init;
-			break;
 		case CCP_SHA_TYPE_224:
-			init = ccp_sha224_init;
-			break;
 		case CCP_SHA_TYPE_256:
-			init = ccp_sha256_init;
+			memcpy(ctx.address + ioffset, init, ctx_size);
 			break;
 		default:
 			ret = -EINVAL;
 			goto e_ctx;
 		}
-		memcpy(ctx.address, init, CCP_SHA_CTXSIZE);
 	} else {
-		ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
+		/* Restore the context */
+		ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
+				sb_count * CCP_SB_BYTES);
 	}
 
-	ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
-			      CCP_PASSTHRU_BYTESWAP_256BIT);
+	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
+			     CCP_PASSTHRU_BYTESWAP_256BIT);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_ctx;
 	}
 
-	/* Send data to the CCP SHA engine */
-	ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
-			    CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
-	if (ret)
-		goto e_ctx;
+	if (sha->src) {
+		/* Send data to the CCP SHA engine; block_size is set above */
+		ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
+				    block_size, DMA_TO_DEVICE);
+		if (ret)
+			goto e_ctx;
 
-	while (src.sg_wa.bytes_left) {
-		ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
-		if (sha->final && !src.sg_wa.bytes_left)
-			op.eom = 1;
+		while (src.sg_wa.bytes_left) {
+			ccp_prepare_data(&src, NULL, &op, block_size, false);
+			if (sha->final && !src.sg_wa.bytes_left)
+				op.eom = 1;
+
+			ret = cmd_q->ccp->vdata->perform->sha(&op);
+			if (ret) {
+				cmd->engine_error = cmd_q->cmd_error;
+				goto e_data;
+			}
 
-		ret = cmd_q->ccp->vdata->perform->perform_sha(&op);
+			ccp_process_data(&src, NULL, &op);
+		}
+	} else {
+		op.eom = 1;
+		ret = cmd_q->ccp->vdata->perform->sha(&op);
 		if (ret) {
 			cmd->engine_error = cmd_q->cmd_error;
 			goto e_data;
 		}
-
-		ccp_process_data(&src, NULL, &op);
 	}
 
 	/* Retrieve the SHA context - convert from LE to BE using
 	 * 32-byte (256-bit) byteswapping to BE
 	 */
-	ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
-				CCP_PASSTHRU_BYTESWAP_256BIT);
+	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
+			       CCP_PASSTHRU_BYTESWAP_256BIT);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_data;
 	}
 
-	ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
-
-	if (sha->final && sha->opad) {
-		/* HMAC operation, recursively perform final SHA */
-		struct ccp_cmd hmac_cmd;
-		struct scatterlist sg;
-		u64 block_size, digest_size;
-		u8 *hmac_buf;
-
+	if (sha->final) {
+		/* Finishing up, so get the digest */
 		switch (sha->type) {
 		case CCP_SHA_TYPE_1:
-			block_size = SHA1_BLOCK_SIZE;
-			digest_size = SHA1_DIGEST_SIZE;
-			break;
 		case CCP_SHA_TYPE_224:
-			block_size = SHA224_BLOCK_SIZE;
-			digest_size = SHA224_DIGEST_SIZE;
-			break;
 		case CCP_SHA_TYPE_256:
-			block_size = SHA256_BLOCK_SIZE;
-			digest_size = SHA256_DIGEST_SIZE;
+			ccp_get_dm_area(&ctx, ooffset,
+					sha->ctx, 0,
+					digest_size);
 			break;
 		default:
 			ret = -EINVAL;
-			goto e_data;
+			goto e_ctx;
 		}
+	} else {
+		/* Stash the context */
+		ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
+				sb_count * CCP_SB_BYTES);
+	}
+
+	if (sha->final && sha->opad) {
+		/* HMAC operation, recursively perform final SHA */
+		struct ccp_cmd hmac_cmd;
+		struct scatterlist sg;
+		u8 *hmac_buf;
 
 		if (sha->opad_len != block_size) {
 			ret = -EINVAL;
@@ -1153,7 +1174,18 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 		sg_init_one(&sg, hmac_buf, block_size + digest_size);
 
 		scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
-		memcpy(hmac_buf + block_size, ctx.address, digest_size);
+		switch (sha->type) {
+		case CCP_SHA_TYPE_1:
+		case CCP_SHA_TYPE_224:
+		case CCP_SHA_TYPE_256:
+			memcpy(hmac_buf + block_size,
+			       ctx.address + ooffset,
+			       digest_size);
+			break;
+		default:
+			ret = -EINVAL;
+			goto e_ctx;
+		}
 
 		memset(&hmac_cmd, 0, sizeof(hmac_cmd));
 		hmac_cmd.engine = CCP_ENGINE_SHA;
@@ -1176,7 +1208,8 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 	}
 
 e_data:
-	ccp_free_data(&src, cmd_q);
+	if (sha->src)
+		ccp_free_data(&src, cmd_q);
 
 e_ctx:
 	ccp_dm_free(&ctx);
@@ -1190,7 +1223,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 	struct ccp_dm_workarea exp, src;
 	struct ccp_data dst;
 	struct ccp_op op;
-	unsigned int ksb_count, i_len, o_len;
+	unsigned int sb_count, i_len, o_len;
 	int ret;
 
 	if (rsa->key_size > CCP_RSA_MAX_WIDTH)
@@ -1208,16 +1241,17 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 	o_len = ((rsa->key_size + 255) / 256) * 32;
 	i_len = o_len * 2;
 
-	ksb_count = o_len / CCP_KSB_BYTES;
+	sb_count = o_len / CCP_SB_BYTES;
 
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
 	op.jobid = ccp_gen_jobid(cmd_q->ccp);
-	op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
-	if (!op.ksb_key)
+	op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count);
+
+	if (!op.sb_key)
 		return -EIO;
 
-	/* The RSA exponent may span multiple (32-byte) KSB entries and must
+	/* The RSA exponent may span multiple (32-byte) SB entries and must
 	 * be in little endian format. Reverse copy each 32-byte chunk
 	 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
 	 * and each byte within that chunk and do not perform any byte swap
@@ -1225,14 +1259,14 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 	 */
 	ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
 	if (ret)
-		goto e_ksb;
+		goto e_sb;
 
 	ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len,
-				      CCP_KSB_BYTES, false);
+				      CCP_SB_BYTES, false);
 	if (ret)
 		goto e_exp;
-	ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
-			      CCP_PASSTHRU_BYTESWAP_NOOP);
+	ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
+			     CCP_PASSTHRU_BYTESWAP_NOOP);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_exp;
@@ -1247,12 +1281,12 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 		goto e_exp;
 
 	ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,
-				      CCP_KSB_BYTES, false);
+				      CCP_SB_BYTES, false);
 	if (ret)
 		goto e_src;
 	src.address += o_len;	/* Adjust the address for the copy operation */
 	ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
-				      CCP_KSB_BYTES, false);
+				      CCP_SB_BYTES, false);
 	if (ret)
 		goto e_src;
 	src.address -= o_len;	/* Reset the address to original value */
@@ -1274,7 +1308,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 	op.u.rsa.mod_size = rsa->key_size;
 	op.u.rsa.input_len = i_len;
 
-	ret = cmd_q->ccp->vdata->perform->perform_rsa(&op);
+	ret = cmd_q->ccp->vdata->perform->rsa(&op);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_dst;
@@ -1291,8 +1325,8 @@ e_src:
 e_exp:
 	ccp_dm_free(&exp);
 
-e_ksb:
-	ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
+e_sb:
+	cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
 
 	return ret;
 }
@@ -1306,7 +1340,7 @@ static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
 	struct ccp_op op;
 	bool in_place = false;
 	unsigned int i;
-	int ret;
+	int ret = 0;
 
 	if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
 		return -EINVAL;
@@ -1321,26 +1355,26 @@ static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
 			return -EINVAL;
 	}
 
-	BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
+	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
 
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
-	op.jobid = ccp_gen_jobid(cmd_q->ccp);
+	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 
 	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
 		/* Load the mask */
-		op.ksb_key = cmd_q->ksb_key;
+		op.sb_key = cmd_q->sb_key;
 
 		ret = ccp_init_dm_workarea(&mask, cmd_q,
-					   CCP_PASSTHRU_KSB_COUNT *
-					   CCP_KSB_BYTES,
+					   CCP_PASSTHRU_SB_COUNT *
+					   CCP_SB_BYTES,
 					   DMA_TO_DEVICE);
 		if (ret)
 			return ret;
 
 		ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
-		ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
-				      CCP_PASSTHRU_BYTESWAP_NOOP);
+		ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
+				     CCP_PASSTHRU_BYTESWAP_NOOP);
 		if (ret) {
 			cmd->engine_error = cmd_q->cmd_error;
 			goto e_mask;
@@ -1399,7 +1433,7 @@ static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
 		op.dst.u.dma.offset = dst.sg_wa.sg_used;
 		op.dst.u.dma.length = op.src.u.dma.length;
 
-		ret = cmd_q->ccp->vdata->perform->perform_passthru(&op);
+		ret = cmd_q->ccp->vdata->perform->passthru(&op);
 		if (ret) {
 			cmd->engine_error = cmd_q->cmd_error;
 			goto e_dst;
@@ -1448,7 +1482,7 @@ static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
 			return -EINVAL;
 	}
 
-	BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
+	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
 
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
@@ -1456,13 +1490,13 @@ static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
 
 	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
 		/* Load the mask */
-		op.ksb_key = cmd_q->ksb_key;
+		op.sb_key = cmd_q->sb_key;
 
 		mask.length = pt->mask_len;
 		mask.dma.address = pt->mask;
 		mask.dma.length = pt->mask_len;
 
-		ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
+		ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
 				     CCP_PASSTHRU_BYTESWAP_NOOP);
 		if (ret) {
 			cmd->engine_error = cmd_q->cmd_error;
@@ -1484,7 +1518,7 @@ static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
 	op.dst.u.dma.offset = 0;
 	op.dst.u.dma.length = pt->src_len;
 
-	ret = cmd_q->ccp->vdata->perform->perform_passthru(&op);
+	ret = cmd_q->ccp->vdata->perform->passthru(&op);
 	if (ret)
 		cmd->engine_error = cmd_q->cmd_error;
 
@@ -1514,7 +1548,7 @@ static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
-	op.jobid = ccp_gen_jobid(cmd_q->ccp);
+	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 
 	/* Concatenate the modulus and the operands. Both the modulus and
 	 * the operands must be in little endian format.  Since the input
@@ -1575,7 +1609,7 @@ static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 
 	op.u.ecc.function = cmd->u.ecc.function;
 
-	ret = cmd_q->ccp->vdata->perform->perform_ecc(&op);
+	ret = cmd_q->ccp->vdata->perform->ecc(&op);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_dst;
@@ -1639,7 +1673,7 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
-	op.jobid = ccp_gen_jobid(cmd_q->ccp);
+	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 
 	/* Concatenate the modulus and the operands. Both the modulus and
 	 * the operands must be in little endian format.  Since the input
@@ -1677,7 +1711,7 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 		goto e_src;
 	src.address += CCP_ECC_OPERAND_SIZE;
 
-	/* Set the first point Z coordianate to 1 */
+	/* Set the first point Z coordinate to 1 */
 	*src.address = 0x01;
 	src.address += CCP_ECC_OPERAND_SIZE;
 
@@ -1696,7 +1730,7 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 			goto e_src;
 		src.address += CCP_ECC_OPERAND_SIZE;
 
-		/* Set the second point Z coordianate to 1 */
+		/* Set the second point Z coordinate to 1 */
 		*src.address = 0x01;
 		src.address += CCP_ECC_OPERAND_SIZE;
 	} else {
@@ -1739,7 +1773,7 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 
 	op.u.ecc.function = cmd->u.ecc.function;
 
-	ret = cmd_q->ccp->vdata->perform->perform_ecc(&op);
+	ret = cmd_q->ccp->vdata->perform->ecc(&op);
 	if (ret) {
 		cmd->engine_error = cmd_q->cmd_error;
 		goto e_dst;
@@ -1810,7 +1844,7 @@ int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 	cmd->engine_error = 0;
 	cmd_q->cmd_error = 0;
 	cmd_q->int_rcvd = 0;
-	cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
+	cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
 
 	switch (cmd->engine) {
 	case CCP_ENGINE_AES:
diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c
index 0bf262e36b6b..28a9996c1085 100644
--- a/drivers/crypto/ccp/ccp-pci.c
+++ b/drivers/crypto/ccp/ccp-pci.c
@@ -4,6 +4,7 @@
  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ * Author: Gary R Hook <gary.hook@amd.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -25,9 +26,6 @@
 
 #include "ccp-dev.h"
 
-#define IO_BAR				2
-#define IO_OFFSET			0x20000
-
 #define MSIX_VECTORS			2
 
 struct ccp_msix {
@@ -143,10 +141,11 @@ static void ccp_free_irqs(struct ccp_device *ccp)
 			free_irq(ccp_pci->msix[ccp_pci->msix_count].vector,
 				 dev);
 		pci_disable_msix(pdev);
-	} else {
+	} else if (ccp->irq) {
 		free_irq(ccp->irq, dev);
 		pci_disable_msi(pdev);
 	}
+	ccp->irq = 0;
 }
 
 static int ccp_find_mmio_area(struct ccp_device *ccp)
@@ -156,10 +155,11 @@ static int ccp_find_mmio_area(struct ccp_device *ccp)
 	resource_size_t io_len;
 	unsigned long io_flags;
 
-	io_flags = pci_resource_flags(pdev, IO_BAR);
-	io_len = pci_resource_len(pdev, IO_BAR);
-	if ((io_flags & IORESOURCE_MEM) && (io_len >= (IO_OFFSET + 0x800)))
-		return IO_BAR;
+	io_flags = pci_resource_flags(pdev, ccp->vdata->bar);
+	io_len = pci_resource_len(pdev, ccp->vdata->bar);
+	if ((io_flags & IORESOURCE_MEM) &&
+	    (io_len >= (ccp->vdata->offset + 0x800)))
+		return ccp->vdata->bar;
 
 	return -EIO;
 }
@@ -216,7 +216,7 @@ static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		dev_err(dev, "pci_iomap failed\n");
 		goto e_device;
 	}
-	ccp->io_regs = ccp->io_map + IO_OFFSET;
+	ccp->io_regs = ccp->io_map + ccp->vdata->offset;
 
 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
 	if (ret) {
@@ -230,6 +230,9 @@ static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	dev_set_drvdata(dev, ccp);
 
+	if (ccp->vdata->setup)
+		ccp->vdata->setup(ccp);
+
 	ret = ccp->vdata->perform->init(ccp);
 	if (ret)
 		goto e_iomap;
@@ -322,6 +325,8 @@ static int ccp_pci_resume(struct pci_dev *pdev)
 
 static const struct pci_device_id ccp_pci_table[] = {
 	{ PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&ccpv3 },
+	{ PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&ccpv5a },
+	{ PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&ccpv5b },
 	/* Last entry must be zero */
 	{ 0, }
 };
diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
new file mode 100644
index 000000000000..4ce67fb9a880
--- /dev/null
+++ b/drivers/crypto/chelsio/Kconfig
@@ -0,0 +1,19 @@
+config CRYPTO_DEV_CHELSIO
+	tristate "Chelsio Crypto Co-processor Driver"
+	depends on CHELSIO_T4
+	select CRYPTO_SHA1
+	select CRYPTO_SHA256
+	select CRYPTO_SHA512
+	---help---
+	  The Chelsio Crypto Co-processor driver for T6 adapters.
+
+	  For general information about Chelsio and our products, visit
+	  our website at <http://www.chelsio.com>.
+
+	  For customer support, please visit our customer support page at
+	  <http://www.chelsio.com/support.html>.
+
+	  Please send feedback to <linux-bugs@chelsio.com>.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called chcr.
diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile
new file mode 100644
index 000000000000..bebdf06687ad
--- /dev/null
+++ b/drivers/crypto/chelsio/Makefile
@@ -0,0 +1,4 @@
+ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
+
+obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o
+chcr-objs :=  chcr_core.o chcr_algo.o
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
new file mode 100644
index 000000000000..e4ddb921d7b3
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -0,0 +1,1525 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Written and Maintained by:
+ *	Manoj Malviya (manojmalviya@chelsio.com)
+ *	Atul Gupta (atul.gupta@chelsio.com)
+ *	Jitendra Lulla (jlulla@chelsio.com)
+ *	Yeshaswi M R Gowda (yeshaswi@chelsio.com)
+ *	Harsh Jain (harsh@chelsio.com)
+ */
+
+#define pr_fmt(fmt) "chcr:" fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/crypto.h>
+#include <linux/cryptohash.h>
+#include <linux/skbuff.h>
+#include <linux/rtnetlink.h>
+#include <linux/highmem.h>
+#include <linux/scatterlist.h>
+
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/hash.h>
+#include <crypto/sha.h>
+#include <crypto/internal/hash.h>
+
+#include "t4fw_api.h"
+#include "t4_msg.h"
+#include "chcr_core.h"
+#include "chcr_algo.h"
+#include "chcr_crypto.h"
+
+static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
+{
+	return ctx->crypto_ctx->ablkctx;
+}
+
+static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
+{
+	return ctx->crypto_ctx->hmacctx;
+}
+
+static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
+{
+	return ctx->dev->u_ctx;
+}
+
+static inline int is_ofld_imm(const struct sk_buff *skb)
+{
+	return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN);
+}
+
+/*
+ *	sgl_len - calculates the size of an SGL of the given capacity
+ *	@n: the number of SGL entries
+ *	Calculates the number of flits needed for a scatter/gather list that
+ *	can hold the given number of entries.
+ */
+static inline unsigned int sgl_len(unsigned int n)
+{
+	n--;
+	return (3 * n) / 2 + (n & 1) + 2;
+}
+
+/*
+ *	chcr_handle_resp - Unmap the DMA buffers associated with the request
+ *	@req: crypto request
+ */
+int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
+		     int error_status)
+{
+	struct crypto_tfm *tfm = req->tfm;
+	struct chcr_context *ctx = crypto_tfm_ctx(tfm);
+	struct uld_ctx *u_ctx = ULD_CTX(ctx);
+	struct chcr_req_ctx ctx_req;
+	struct cpl_fw6_pld *fw6_pld;
+	unsigned int digestsize, updated_digestsize;
+
+	switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
+	case CRYPTO_ALG_TYPE_BLKCIPHER:
+		ctx_req.req.ablk_req = (struct ablkcipher_request *)req;
+		ctx_req.ctx.ablk_ctx =
+			ablkcipher_request_ctx(ctx_req.req.ablk_req);
+		if (!error_status) {
+			fw6_pld = (struct cpl_fw6_pld *)input;
+			memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2],
+			       AES_BLOCK_SIZE);
+		}
+		dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst,
+			     ABLK_CTX(ctx)->dst_nents, DMA_FROM_DEVICE);
+		if (ctx_req.ctx.ablk_ctx->skb) {
+			kfree_skb(ctx_req.ctx.ablk_ctx->skb);
+			ctx_req.ctx.ablk_ctx->skb = NULL;
+		}
+		break;
+
+	case CRYPTO_ALG_TYPE_AHASH:
+		ctx_req.req.ahash_req = (struct ahash_request *)req;
+		ctx_req.ctx.ahash_ctx =
+			ahash_request_ctx(ctx_req.req.ahash_req);
+		digestsize =
+			crypto_ahash_digestsize(crypto_ahash_reqtfm(
+							ctx_req.req.ahash_req));
+		updated_digestsize = digestsize;
+		if (digestsize == SHA224_DIGEST_SIZE)
+			updated_digestsize = SHA256_DIGEST_SIZE;
+		else if (digestsize == SHA384_DIGEST_SIZE)
+			updated_digestsize = SHA512_DIGEST_SIZE;
+		if (ctx_req.ctx.ahash_ctx->skb)
+			ctx_req.ctx.ahash_ctx->skb = NULL;
+		if (ctx_req.ctx.ahash_ctx->result == 1) {
+			ctx_req.ctx.ahash_ctx->result = 0;
+			memcpy(ctx_req.req.ahash_req->result, input +
+			       sizeof(struct cpl_fw6_pld),
+			       digestsize);
+		} else {
+			memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input +
+			       sizeof(struct cpl_fw6_pld),
+			       updated_digestsize);
+		}
+		kfree(ctx_req.ctx.ahash_ctx->dummy_payload_ptr);
+		ctx_req.ctx.ahash_ctx->dummy_payload_ptr = NULL;
+		break;
+	}
+	return 0;
+}
+
+/*
+ *	calc_tx_flits_ofld - calculate # of flits for an offload packet
+ *	@skb: the packet
+ *	Returns the number of flits needed for the given offload packet.
+ *	These packets are already fully constructed and no additional headers
+ *	will be added.
+ */
+static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
+{
+	unsigned int flits, cnt;
+
+	if (is_ofld_imm(skb))
+		return DIV_ROUND_UP(skb->len, 8);
+
+	flits = skb_transport_offset(skb) / 8;   /* headers */
+	cnt = skb_shinfo(skb)->nr_frags;
+	if (skb_tail_pointer(skb) != skb_transport_header(skb))
+		cnt++;
+	return flits + sgl_len(cnt);
+}
+
+static struct shash_desc *chcr_alloc_shash(unsigned int ds)
+{
+	struct crypto_shash *base_hash = NULL;
+	struct shash_desc *desc;
+
+	switch (ds) {
+	case SHA1_DIGEST_SIZE:
+		base_hash = crypto_alloc_shash("sha1-generic", 0, 0);
+		break;
+	case SHA224_DIGEST_SIZE:
+		base_hash = crypto_alloc_shash("sha224-generic", 0, 0);
+		break;
+	case SHA256_DIGEST_SIZE:
+		base_hash = crypto_alloc_shash("sha256-generic", 0, 0);
+		break;
+	case SHA384_DIGEST_SIZE:
+		base_hash = crypto_alloc_shash("sha384-generic", 0, 0);
+		break;
+	case SHA512_DIGEST_SIZE:
+		base_hash = crypto_alloc_shash("sha512-generic", 0, 0);
+		break;
+	}
+	if (IS_ERR(base_hash)) {
+		pr_err("Can not allocate sha-generic algo.\n");
+		return (void *)base_hash;
+	}
+
+	desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(base_hash),
+		       GFP_KERNEL);
+	if (!desc)
+		return ERR_PTR(-ENOMEM);
+	desc->tfm = base_hash;
+	desc->flags = crypto_shash_get_flags(base_hash);
+	return desc;
+}
+
+static int chcr_compute_partial_hash(struct shash_desc *desc,
+				     char *iopad, char *result_hash,
+				     int digest_size)
+{
+	struct sha1_state sha1_st;
+	struct sha256_state sha256_st;
+	struct sha512_state sha512_st;
+	int error;
+
+	if (digest_size == SHA1_DIGEST_SIZE) {
+		error = crypto_shash_init(desc) ?:
+			crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
+			crypto_shash_export(desc, (void *)&sha1_st);
+		memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
+	} else if (digest_size == SHA224_DIGEST_SIZE) {
+		error = crypto_shash_init(desc) ?:
+			crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
+			crypto_shash_export(desc, (void *)&sha256_st);
+		memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
+
+	} else if (digest_size == SHA256_DIGEST_SIZE) {
+		error = crypto_shash_init(desc) ?:
+			crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
+			crypto_shash_export(desc, (void *)&sha256_st);
+		memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
+
+	} else if (digest_size == SHA384_DIGEST_SIZE) {
+		error = crypto_shash_init(desc) ?:
+			crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
+			crypto_shash_export(desc, (void *)&sha512_st);
+		memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
+
+	} else if (digest_size == SHA512_DIGEST_SIZE) {
+		error = crypto_shash_init(desc) ?:
+			crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
+			crypto_shash_export(desc, (void *)&sha512_st);
+		memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
+	} else {
+		error = -EINVAL;
+		pr_err("Unknown digest size %d\n", digest_size);
+	}
+	return error;
+}
+
+static void chcr_change_order(char *buf, int ds)
+{
+	int i;
+
+	if (ds == SHA512_DIGEST_SIZE) {
+		for (i = 0; i < (ds / sizeof(u64)); i++)
+			*((__be64 *)buf + i) =
+				cpu_to_be64(*((u64 *)buf + i));
+	} else {
+		for (i = 0; i < (ds / sizeof(u32)); i++)
+			*((__be32 *)buf + i) =
+				cpu_to_be32(*((u32 *)buf + i));
+	}
+}
+
+static inline int is_hmac(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct chcr_alg_template *chcr_crypto_alg =
+		container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
+			     alg.hash);
+	if ((chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK) ==
+	    CRYPTO_ALG_SUB_TYPE_HASH_HMAC)
+		return 1;
+	return 0;
+}
+
+static inline unsigned int ch_nents(struct scatterlist *sg,
+				    unsigned int *total_size)
+{
+	unsigned int nents;
+
+	for (nents = 0, *total_size = 0; sg; sg = sg_next(sg)) {
+		nents++;
+		*total_size += sg->length;
+	}
+	return nents;
+}
+
+static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
+			   struct scatterlist *sg,
+			   struct phys_sge_parm *sg_param)
+{
+	struct phys_sge_pairs *to;
+	unsigned int out_buf_size = sg_param->obsize;
+	unsigned int nents = sg_param->nents, i, j, tot_len = 0;
+
+	phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
+				    | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
+	phys_cpl->pcirlxorder_to_noofsgentr =
+		htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
+		      CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
+		      CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
+		      CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
+		      CPL_RX_PHYS_DSGL_DCAID_V(0) |
+		      CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents));
+	phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
+	phys_cpl->rss_hdr_int.qid = htons(sg_param->qid);
+	phys_cpl->rss_hdr_int.hash_val = 0;
+	to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
+				       sizeof(struct cpl_rx_phys_dsgl));
+
+	for (i = 0; nents; to++) {
+		for (j = i; (nents && (j < (8 + i))); j++, nents--) {
+			to->len[j] = htons(sg->length);
+			to->addr[j] = cpu_to_be64(sg_dma_address(sg));
+			if (out_buf_size) {
+				if (tot_len + sg_dma_len(sg) >= out_buf_size) {
+					to->len[j] = htons(out_buf_size -
+							   tot_len);
+					return;
+				}
+				tot_len += sg_dma_len(sg);
+			}
+			sg = sg_next(sg);
+		}
+	}
+}
+
+static inline unsigned
+int map_writesg_phys_cpl(struct device *dev, struct cpl_rx_phys_dsgl *phys_cpl,
+			 struct scatterlist *sg, struct phys_sge_parm *sg_param)
+{
+	if (!sg || !sg_param->nents)
+		return 0;
+
+	sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE);
+	if (sg_param->nents == 0) {
+		pr_err("CHCR : DMA mapping failed\n");
+		return -EINVAL;
+	}
+	write_phys_cpl(phys_cpl, sg, sg_param);
+	return 0;
+}
+
+static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct chcr_alg_template *chcr_crypto_alg =
+		container_of(alg, struct chcr_alg_template, alg.crypto);
+
+	return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
+}
+
+static inline void
+write_sg_data_page_desc(struct sk_buff *skb, unsigned int *frags,
+			struct scatterlist *sg, unsigned int count)
+{
+	struct page *spage;
+	unsigned int page_len;
+
+	skb->len += count;
+	skb->data_len += count;
+	skb->truesize += count;
+	while (count > 0) {
+		if (sg && (!(sg->length)))
+			break;
+		spage = sg_page(sg);
+		get_page(spage);
+		page_len = min(sg->length, count);
+		skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len);
+		(*frags)++;
+		count -= page_len;
+		sg = sg_next(sg);
+	}
+}
+
+static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
+			       struct _key_ctx *key_ctx)
+{
+	if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
+		get_aes_decrypt_key(key_ctx->key, ablkctx->key,
+				    ablkctx->enckey_len << 3);
+		memset(key_ctx->key + ablkctx->enckey_len, 0,
+		       CHCR_AES_MAX_KEY_LEN - ablkctx->enckey_len);
+	} else {
+		memcpy(key_ctx->key,
+		       ablkctx->key + (ablkctx->enckey_len >> 1),
+		       ablkctx->enckey_len >> 1);
+		get_aes_decrypt_key(key_ctx->key + (ablkctx->enckey_len >> 1),
+				    ablkctx->key, ablkctx->enckey_len << 2);
+	}
+	return 0;
+}
+
+static inline void create_wreq(struct chcr_context *ctx,
+			       struct fw_crypto_lookaside_wr *wreq,
+			       void *req, struct sk_buff *skb,
+			       int kctx_len, int hash_sz,
+			       unsigned int phys_dsgl)
+{
+	struct uld_ctx *u_ctx = ULD_CTX(ctx);
+	struct ulp_txpkt *ulptx = (struct ulp_txpkt *)(wreq + 1);
+	struct ulptx_idata *sc_imm = (struct ulptx_idata *)(ulptx + 1);
+	int iv_loc = IV_DSGL;
+	int qid = u_ctx->lldi.rxq_ids[ctx->tx_channel_id];
+	unsigned int immdatalen = 0, nr_frags = 0;
+
+	if (is_ofld_imm(skb)) {
+		immdatalen = skb->data_len;
+		iv_loc = IV_IMMEDIATE;
+	} else {
+		nr_frags = skb_shinfo(skb)->nr_frags;
+	}
+
+	wreq->op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
+						     (kctx_len >> 4));
+	wreq->pld_size_hash_size =
+		htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) |
+		      FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
+	wreq->len16_pkd = htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
+				    (calc_tx_flits_ofld(skb) * 8), 16)));
+	wreq->cookie = cpu_to_be64((uintptr_t)req);
+	wreq->rx_chid_to_rx_q_id =
+		FILL_WR_RX_Q_ID(ctx->dev->tx_channel_id, qid,
+				(hash_sz) ? IV_NOP : iv_loc);
+
+	ulptx->cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id);
+	ulptx->len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
+					 16) - ((sizeof(*wreq)) >> 4)));
+
+	sc_imm->cmd_more = FILL_CMD_MORE(immdatalen);
+	sc_imm->len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + kctx_len +
+				  ((hash_sz) ? DUMMY_BYTES :
+				  (sizeof(struct cpl_rx_phys_dsgl) +
+				   phys_dsgl)) + immdatalen);
+}
+
+/**
+ *	create_cipher_wr - form the WR for cipher operations
+ *	@req: cipher req.
+ *	@ctx: crypto driver context of the request.
+ *	@qid: ingress qid where response of this WR should be received.
+ *	@op_type:	encryption or decryption
+ */
+static struct sk_buff
+*create_cipher_wr(struct crypto_async_request *req_base,
+		  struct chcr_context *ctx, unsigned short qid,
+		  unsigned short op_type)
+{
+	struct ablkcipher_request *req = (struct ablkcipher_request *)req_base;
+	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+	struct uld_ctx *u_ctx = ULD_CTX(ctx);
+	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+	struct sk_buff *skb = NULL;
+	struct _key_ctx *key_ctx;
+	struct fw_crypto_lookaside_wr *wreq;
+	struct cpl_tx_sec_pdu *sec_cpl;
+	struct cpl_rx_phys_dsgl *phys_cpl;
+	struct chcr_blkcipher_req_ctx *req_ctx = ablkcipher_request_ctx(req);
+	struct phys_sge_parm sg_param;
+	unsigned int frags = 0, transhdr_len, phys_dsgl, dst_bufsize = 0;
+	unsigned int ivsize = crypto_ablkcipher_ivsize(tfm), kctx_len;
+
+	if (!req->info)
+		return ERR_PTR(-EINVAL);
+	ablkctx->dst_nents = ch_nents(req->dst, &dst_bufsize);
+	ablkctx->enc = op_type;
+
+	if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
+	    (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE))
+		return ERR_PTR(-EINVAL);
+
+	phys_dsgl = get_space_for_phys_dsgl(ablkctx->dst_nents);
+
+	kctx_len = sizeof(*key_ctx) +
+		(DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
+	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
+	skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)),
+			GFP_ATOMIC);
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+	skb_reserve(skb, sizeof(struct sge_opaque_hdr));
+	wreq = (struct fw_crypto_lookaside_wr *)__skb_put(skb, transhdr_len);
+
+	sec_cpl = (struct cpl_tx_sec_pdu *)((u8 *)wreq + SEC_CPL_OFFSET);
+	sec_cpl->op_ivinsrtofst =
+		FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1, 1);
+
+	sec_cpl->pldlen = htonl(ivsize + req->nbytes);
+	sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(0, 0,
+								ivsize + 1, 0);
+
+	sec_cpl->cipherstop_lo_authinsert =  FILL_SEC_CPL_AUTHINSERT(0, 0,
+								     0, 0);
+	sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0,
+							 ablkctx->ciph_mode,
+							 0, 0, ivsize >> 1, 1);
+	sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
+							  0, 1, phys_dsgl);
+
+	key_ctx = (struct _key_ctx *)((u8 *)sec_cpl + sizeof(*sec_cpl));
+	key_ctx->ctx_hdr = ablkctx->key_ctx_hdr;
+	if (op_type == CHCR_DECRYPT_OP) {
+		if (generate_copy_rrkey(ablkctx, key_ctx))
+			goto map_fail1;
+	} else {
+		if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
+			memcpy(key_ctx->key, ablkctx->key, ablkctx->enckey_len);
+		} else {
+			memcpy(key_ctx->key, ablkctx->key +
+			       (ablkctx->enckey_len >> 1),
+			       ablkctx->enckey_len >> 1);
+			memcpy(key_ctx->key +
+			       (ablkctx->enckey_len >> 1),
+			       ablkctx->key,
+			       ablkctx->enckey_len >> 1);
+		}
+	}
+	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)key_ctx + kctx_len);
+
+	memcpy(ablkctx->iv, req->info, ivsize);
+	sg_init_table(&ablkctx->iv_sg, 1);
+	sg_set_buf(&ablkctx->iv_sg, ablkctx->iv, ivsize);
+	sg_param.nents = ablkctx->dst_nents;
+	sg_param.obsize = dst_bufsize;
+	sg_param.qid = qid;
+	sg_param.align = 1;
+	if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, req->dst,
+				 &sg_param))
+		goto map_fail1;
+
+	skb_set_transport_header(skb, transhdr_len);
+	write_sg_data_page_desc(skb, &frags, &ablkctx->iv_sg, ivsize);
+	write_sg_data_page_desc(skb, &frags, req->src, req->nbytes);
+	create_wreq(ctx, wreq, req, skb, kctx_len, 0, phys_dsgl);
+	req_ctx->skb = skb;
+	skb_get(skb);
+	return skb;
+map_fail1:
+	kfree_skb(skb);
+	return ERR_PTR(-ENOMEM);
+}
+
+static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+			       unsigned int keylen)
+{
+	struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+	struct ablkcipher_alg *alg = crypto_ablkcipher_alg(tfm);
+	unsigned int ck_size, context_size;
+	u16 alignment = 0;
+
+	if ((keylen < alg->min_keysize) || (keylen > alg->max_keysize))
+		goto badkey_err;
+
+	memcpy(ablkctx->key, key, keylen);
+	ablkctx->enckey_len = keylen;
+	if (keylen == AES_KEYSIZE_128) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+	} else if (keylen == AES_KEYSIZE_192) {
+		alignment = 8;
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
+	} else if (keylen == AES_KEYSIZE_256) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
+	} else {
+		goto badkey_err;
+	}
+
+	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
+			keylen + alignment) >> 4;
+
+	ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
+						0, 0, context_size);
+	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
+	return 0;
+badkey_err:
+	crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+	ablkctx->enckey_len = 0;
+	return -EINVAL;
+}
+
+static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
+{
+	int ret = 0;
+	struct sge_ofld_txq *q;
+	struct adapter *adap = netdev2adap(dev);
+
+	local_bh_disable();
+	q = &adap->sge.ofldtxq[idx];
+	spin_lock(&q->sendq.lock);
+	if (q->full)
+		ret = -1;
+	spin_unlock(&q->sendq.lock);
+	local_bh_enable();
+	return ret;
+}
+
+static int chcr_aes_encrypt(struct ablkcipher_request *req)
+{
+	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+	struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+	struct crypto_async_request *req_base = &req->base;
+	struct uld_ctx *u_ctx = ULD_CTX(ctx);
+	struct sk_buff *skb;
+
+	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
+					    ctx->tx_channel_id))) {
+		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return -EBUSY;
+	}
+
+	skb = create_cipher_wr(req_base, ctx,
+			       u_ctx->lldi.rxq_ids[ctx->tx_channel_id],
+			       CHCR_ENCRYPT_OP);
+	if (IS_ERR(skb)) {
+		pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
+		return  PTR_ERR(skb);
+	}
+	skb->dev = u_ctx->lldi.ports[0];
+	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+	chcr_send_wr(skb);
+	return -EINPROGRESS;
+}
+
+static int chcr_aes_decrypt(struct ablkcipher_request *req)
+{
+	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+	struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+	struct crypto_async_request *req_base = &req->base;
+	struct uld_ctx *u_ctx = ULD_CTX(ctx);
+	struct sk_buff *skb;
+
+	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
+					    ctx->tx_channel_id))) {
+		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return -EBUSY;
+	}
+
+	skb = create_cipher_wr(req_base, ctx, u_ctx->lldi.rxq_ids[0],
+			       CHCR_DECRYPT_OP);
+	if (IS_ERR(skb)) {
+		pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
+		return PTR_ERR(skb);
+	}
+	skb->dev = u_ctx->lldi.ports[0];
+	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+	chcr_send_wr(skb);
+	return -EINPROGRESS;
+}
+
+static int chcr_device_init(struct chcr_context *ctx)
+{
+	struct uld_ctx *u_ctx;
+	unsigned int id;
+	int err = 0, rxq_perchan, rxq_idx;
+
+	id = smp_processor_id();
+	if (!ctx->dev) {
+		err = assign_chcr_device(&ctx->dev);
+		if (err) {
+			pr_err("chcr device assignment fails\n");
+			goto out;
+		}
+		u_ctx = ULD_CTX(ctx);
+		rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
+		ctx->dev->tx_channel_id = 0;
+		rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
+		rxq_idx += id % rxq_perchan;
+		spin_lock(&ctx->dev->lock_chcr_dev);
+		ctx->tx_channel_id = rxq_idx;
+		spin_unlock(&ctx->dev->lock_chcr_dev);
+	}
+out:
+	return err;
+}
+
+static int chcr_cra_init(struct crypto_tfm *tfm)
+{
+	tfm->crt_ablkcipher.reqsize =  sizeof(struct chcr_blkcipher_req_ctx);
+	return chcr_device_init(crypto_tfm_ctx(tfm));
+}
+
+static int get_alg_config(struct algo_param *params,
+			  unsigned int auth_size)
+{
+	switch (auth_size) {
+	case SHA1_DIGEST_SIZE:
+		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
+		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
+		params->result_size = SHA1_DIGEST_SIZE;
+		break;
+	case SHA224_DIGEST_SIZE:
+		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
+		params->result_size = SHA256_DIGEST_SIZE;
+		break;
+	case SHA256_DIGEST_SIZE:
+		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
+		params->result_size = SHA256_DIGEST_SIZE;
+		break;
+	case SHA384_DIGEST_SIZE:
+		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
+		params->result_size = SHA512_DIGEST_SIZE;
+		break;
+	case SHA512_DIGEST_SIZE:
+		params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+		params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
+		params->result_size = SHA512_DIGEST_SIZE;
+		break;
+	default:
+		pr_err("chcr : ERROR, unsupported digest size\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static inline int
+write_buffer_data_page_desc(struct chcr_ahash_req_ctx *req_ctx,
+			    struct sk_buff *skb, unsigned int *frags, char *bfr,
+			    u8 bfr_len)
+{
+	void *page_ptr = NULL;
+
+	skb->len += bfr_len;
+	skb->data_len += bfr_len;
+	skb->truesize += bfr_len;
+	page_ptr = kmalloc(CHCR_HASH_MAX_BLOCK_SIZE_128, GFP_ATOMIC | GFP_DMA);
+	if (!page_ptr)
+		return -ENOMEM;
+	get_page(virt_to_page(page_ptr));
+	req_ctx->dummy_payload_ptr = page_ptr;
+	memcpy(page_ptr, bfr, bfr_len);
+	skb_fill_page_desc(skb, *frags, virt_to_page(page_ptr),
+			   offset_in_page(page_ptr), bfr_len);
+	(*frags)++;
+	return 0;
+}
+
+/**
+ *	create_final_hash_wr - Create hash work request
+ *	@req - Cipher req base
+ */
+static struct sk_buff *create_final_hash_wr(struct ahash_request *req,
+					    struct hash_wr_param *param)
+{
+	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+	struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+	struct sk_buff *skb = NULL;
+	struct _key_ctx *key_ctx;
+	struct fw_crypto_lookaside_wr *wreq;
+	struct cpl_tx_sec_pdu *sec_cpl;
+	unsigned int frags = 0, transhdr_len, iopad_alignment = 0;
+	unsigned int digestsize = crypto_ahash_digestsize(tfm);
+	unsigned int kctx_len = sizeof(*key_ctx);
+	u8 hash_size_in_response = 0;
+
+	iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
+	kctx_len += param->alg_prm.result_size + iopad_alignment;
+	if (param->opad_needed)
+		kctx_len += param->alg_prm.result_size + iopad_alignment;
+
+	if (req_ctx->result)
+		hash_size_in_response = digestsize;
+	else
+		hash_size_in_response = param->alg_prm.result_size;
+	transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
+	skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)),
+			GFP_ATOMIC);
+	if (!skb)
+		return skb;
+
+	skb_reserve(skb, sizeof(struct sge_opaque_hdr));
+	wreq = (struct fw_crypto_lookaside_wr *)__skb_put(skb, transhdr_len);
+	memset(wreq, 0, transhdr_len);
+
+	sec_cpl = (struct cpl_tx_sec_pdu *)((u8 *)wreq + SEC_CPL_OFFSET);
+	sec_cpl->op_ivinsrtofst =
+		FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 0, 0);
+	sec_cpl->pldlen = htonl(param->bfr_len + param->sg_len);
+
+	sec_cpl->aadstart_cipherstop_hi =
+		FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
+	sec_cpl->cipherstop_lo_authinsert =
+		FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
+	sec_cpl->seqno_numivs =
+		FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
+					 param->opad_needed, 0, 0);
+
+	sec_cpl->ivgen_hdrlen =
+		FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
+
+	key_ctx = (struct _key_ctx *)((u8 *)sec_cpl + sizeof(*sec_cpl));
+	memcpy(key_ctx->key, req_ctx->partial_hash, param->alg_prm.result_size);
+
+	if (param->opad_needed)
+		memcpy(key_ctx->key + ((param->alg_prm.result_size <= 32) ? 32 :
+				       CHCR_HASH_MAX_DIGEST_SIZE),
+		       hmacctx->opad, param->alg_prm.result_size);
+
+	key_ctx->ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
+					    param->alg_prm.mk_size, 0,
+					    param->opad_needed,
+					    (kctx_len >> 4));
+	sec_cpl->scmd1 = cpu_to_be64((u64)param->scmd1);
+
+	skb_set_transport_header(skb, transhdr_len);
+	if (param->bfr_len != 0)
+		write_buffer_data_page_desc(req_ctx, skb, &frags, req_ctx->bfr,
+					    param->bfr_len);
+	if (param->sg_len != 0)
+		write_sg_data_page_desc(skb, &frags, req->src, param->sg_len);
+
+	create_wreq(ctx, wreq, req, skb, kctx_len, hash_size_in_response,
+		    0);
+	req_ctx->skb = skb;
+	skb_get(skb);
+	return skb;
+}
+
+static int chcr_ahash_update(struct ahash_request *req)
+{
+	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
+	struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+	struct uld_ctx *u_ctx = NULL;
+	struct sk_buff *skb;
+	u8 remainder = 0, bs;
+	unsigned int nbytes = req->nbytes;
+	struct hash_wr_param params;
+
+	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+
+	u_ctx = ULD_CTX(ctx);
+	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
+					    ctx->tx_channel_id))) {
+		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return -EBUSY;
+	}
+
+	if (nbytes + req_ctx->bfr_len >= bs) {
+		remainder = (nbytes + req_ctx->bfr_len) % bs;
+		nbytes = nbytes + req_ctx->bfr_len - remainder;
+	} else {
+		sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->bfr +
+				   req_ctx->bfr_len, nbytes, 0);
+		req_ctx->bfr_len += nbytes;
+		return 0;
+	}
+
+	params.opad_needed = 0;
+	params.more = 1;
+	params.last = 0;
+	params.sg_len = nbytes - req_ctx->bfr_len;
+	params.bfr_len = req_ctx->bfr_len;
+	params.scmd1 = 0;
+	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
+	req_ctx->result = 0;
+	req_ctx->data_len += params.sg_len + params.bfr_len;
+	skb = create_final_hash_wr(req, &params);
+	if (!skb)
+		return -ENOMEM;
+
+	req_ctx->bfr_len = remainder;
+	if (remainder)
+		sg_pcopy_to_buffer(req->src, sg_nents(req->src),
+				   req_ctx->bfr, remainder, req->nbytes -
+				   remainder);
+	skb->dev = u_ctx->lldi.ports[0];
+	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+	chcr_send_wr(skb);
+
+	return -EINPROGRESS;
+}
+
+static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
+{
+	memset(bfr_ptr, 0, bs);
+	*bfr_ptr = 0x80;
+	if (bs == 64)
+		*(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1  << 3);
+	else
+		*(__be64 *)(bfr_ptr + 120) =  cpu_to_be64(scmd1  << 3);
+}
+
+static int chcr_ahash_final(struct ahash_request *req)
+{
+	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
+	struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+	struct hash_wr_param params;
+	struct sk_buff *skb;
+	struct uld_ctx *u_ctx = NULL;
+	u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+
+	u_ctx = ULD_CTX(ctx);
+	if (is_hmac(crypto_ahash_tfm(rtfm)))
+		params.opad_needed = 1;
+	else
+		params.opad_needed = 0;
+	params.sg_len = 0;
+	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
+	req_ctx->result = 1;
+	params.bfr_len = req_ctx->bfr_len;
+	req_ctx->data_len += params.bfr_len + params.sg_len;
+	if (req_ctx->bfr && (req_ctx->bfr_len == 0)) {
+		create_last_hash_block(req_ctx->bfr, bs, req_ctx->data_len);
+		params.last = 0;
+		params.more = 1;
+		params.scmd1 = 0;
+		params.bfr_len = bs;
+
+	} else {
+		params.scmd1 = req_ctx->data_len;
+		params.last = 1;
+		params.more = 0;
+	}
+	skb = create_final_hash_wr(req, &params);
+	skb->dev = u_ctx->lldi.ports[0];
+	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+	chcr_send_wr(skb);
+	return -EINPROGRESS;
+}
+
+static int chcr_ahash_finup(struct ahash_request *req)
+{
+	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
+	struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+	struct uld_ctx *u_ctx = NULL;
+	struct sk_buff *skb;
+	struct hash_wr_param params;
+	u8  bs;
+
+	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+	u_ctx = ULD_CTX(ctx);
+
+	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
+					    ctx->tx_channel_id))) {
+		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return -EBUSY;
+	}
+
+	if (is_hmac(crypto_ahash_tfm(rtfm)))
+		params.opad_needed = 1;
+	else
+		params.opad_needed = 0;
+
+	params.sg_len = req->nbytes;
+	params.bfr_len = req_ctx->bfr_len;
+	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
+	req_ctx->data_len += params.bfr_len + params.sg_len;
+	req_ctx->result = 1;
+	if (req_ctx->bfr && (req_ctx->bfr_len + req->nbytes) == 0) {
+		create_last_hash_block(req_ctx->bfr, bs, req_ctx->data_len);
+		params.last = 0;
+		params.more = 1;
+		params.scmd1 = 0;
+		params.bfr_len = bs;
+	} else {
+		params.scmd1 = req_ctx->data_len;
+		params.last = 1;
+		params.more = 0;
+	}
+
+	skb = create_final_hash_wr(req, &params);
+	if (!skb)
+		return -ENOMEM;
+	skb->dev = u_ctx->lldi.ports[0];
+	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+	chcr_send_wr(skb);
+
+	return -EINPROGRESS;
+}
+
+static int chcr_ahash_digest(struct ahash_request *req)
+{
+	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
+	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
+	struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+	struct uld_ctx *u_ctx = NULL;
+	struct sk_buff *skb;
+	struct hash_wr_param params;
+	u8  bs;
+
+	rtfm->init(req);
+	bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+
+	u_ctx = ULD_CTX(ctx);
+	if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
+					    ctx->tx_channel_id))) {
+		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return -EBUSY;
+	}
+
+	if (is_hmac(crypto_ahash_tfm(rtfm)))
+		params.opad_needed = 1;
+	else
+		params.opad_needed = 0;
+
+	params.last = 0;
+	params.more = 0;
+	params.sg_len = req->nbytes;
+	params.bfr_len = 0;
+	params.scmd1 = 0;
+	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
+	req_ctx->result = 1;
+	req_ctx->data_len += params.bfr_len + params.sg_len;
+
+	if (req_ctx->bfr && req->nbytes == 0) {
+		create_last_hash_block(req_ctx->bfr, bs, 0);
+		params.more = 1;
+		params.bfr_len = bs;
+	}
+
+	skb = create_final_hash_wr(req, &params);
+	if (!skb)
+		return -ENOMEM;
+
+	skb->dev = u_ctx->lldi.ports[0];
+	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+	chcr_send_wr(skb);
+	return -EINPROGRESS;
+}
+
+static int chcr_ahash_export(struct ahash_request *areq, void *out)
+{
+	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+	struct chcr_ahash_req_ctx *state = out;
+
+	state->bfr_len = req_ctx->bfr_len;
+	state->data_len = req_ctx->data_len;
+	memcpy(state->bfr, req_ctx->bfr, CHCR_HASH_MAX_BLOCK_SIZE_128);
+	memcpy(state->partial_hash, req_ctx->partial_hash,
+	       CHCR_HASH_MAX_DIGEST_SIZE);
+	return 0;
+}
+
+static int chcr_ahash_import(struct ahash_request *areq, const void *in)
+{
+	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+	struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
+
+	req_ctx->bfr_len = state->bfr_len;
+	req_ctx->data_len = state->data_len;
+	req_ctx->dummy_payload_ptr = NULL;
+	memcpy(req_ctx->bfr, state->bfr, CHCR_HASH_MAX_BLOCK_SIZE_128);
+	memcpy(req_ctx->partial_hash, state->partial_hash,
+	       CHCR_HASH_MAX_DIGEST_SIZE);
+	return 0;
+}
+
+static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
+			     unsigned int keylen)
+{
+	struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+	struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+	unsigned int digestsize = crypto_ahash_digestsize(tfm);
+	unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+	unsigned int i, err = 0, updated_digestsize;
+
+	/*
+	 * use the key to calculate the ipad and opad. ipad will sent with the
+	 * first request's data. opad will be sent with the final hash result
+	 * ipad in hmacctx->ipad and opad in hmacctx->opad location
+	 */
+	if (!hmacctx->desc)
+		return -EINVAL;
+	if (keylen > bs) {
+		err = crypto_shash_digest(hmacctx->desc, key, keylen,
+					  hmacctx->ipad);
+		if (err)
+			goto out;
+		keylen = digestsize;
+	} else {
+		memcpy(hmacctx->ipad, key, keylen);
+	}
+	memset(hmacctx->ipad + keylen, 0, bs - keylen);
+	memcpy(hmacctx->opad, hmacctx->ipad, bs);
+
+	for (i = 0; i < bs / sizeof(int); i++) {
+		*((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
+		*((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
+	}
+
+	updated_digestsize = digestsize;
+	if (digestsize == SHA224_DIGEST_SIZE)
+		updated_digestsize = SHA256_DIGEST_SIZE;
+	else if (digestsize == SHA384_DIGEST_SIZE)
+		updated_digestsize = SHA512_DIGEST_SIZE;
+	err = chcr_compute_partial_hash(hmacctx->desc, hmacctx->ipad,
+					hmacctx->ipad, digestsize);
+	if (err)
+		goto out;
+	chcr_change_order(hmacctx->ipad, updated_digestsize);
+
+	err = chcr_compute_partial_hash(hmacctx->desc, hmacctx->opad,
+					hmacctx->opad, digestsize);
+	if (err)
+		goto out;
+	chcr_change_order(hmacctx->opad, updated_digestsize);
+out:
+	return err;
+}
+
+static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+			       unsigned int key_len)
+{
+	struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+	int status = 0;
+	unsigned short context_size = 0;
+
+	if ((key_len == (AES_KEYSIZE_128 << 1)) ||
+	    (key_len == (AES_KEYSIZE_256 << 1))) {
+		memcpy(ablkctx->key, key, key_len);
+		ablkctx->enckey_len = key_len;
+		context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
+		ablkctx->key_ctx_hdr =
+			FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
+					 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
+					 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
+					 CHCR_KEYCTX_NO_KEY, 1,
+					 0, context_size);
+		ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
+	} else {
+		crypto_tfm_set_flags((struct crypto_tfm *)tfm,
+				     CRYPTO_TFM_RES_BAD_KEY_LEN);
+		ablkctx->enckey_len = 0;
+		status = -EINVAL;
+	}
+	return status;
+}
+
+static int chcr_sha_init(struct ahash_request *areq)
+{
+	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+	int digestsize =  crypto_ahash_digestsize(tfm);
+
+	req_ctx->data_len = 0;
+	req_ctx->dummy_payload_ptr = NULL;
+	req_ctx->bfr_len = 0;
+	req_ctx->skb = NULL;
+	req_ctx->result = 0;
+	copy_hash_init_values(req_ctx->partial_hash, digestsize);
+	return 0;
+}
+
+static int chcr_sha_cra_init(struct crypto_tfm *tfm)
+{
+	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+				 sizeof(struct chcr_ahash_req_ctx));
+	return chcr_device_init(crypto_tfm_ctx(tfm));
+}
+
+static int chcr_hmac_init(struct ahash_request *areq)
+{
+	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+	struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
+	struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
+	struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+	unsigned int digestsize = crypto_ahash_digestsize(rtfm);
+	unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
+
+	chcr_sha_init(areq);
+	req_ctx->data_len = bs;
+	if (is_hmac(crypto_ahash_tfm(rtfm))) {
+		if (digestsize == SHA224_DIGEST_SIZE)
+			memcpy(req_ctx->partial_hash, hmacctx->ipad,
+			       SHA256_DIGEST_SIZE);
+		else if (digestsize == SHA384_DIGEST_SIZE)
+			memcpy(req_ctx->partial_hash, hmacctx->ipad,
+			       SHA512_DIGEST_SIZE);
+		else
+			memcpy(req_ctx->partial_hash, hmacctx->ipad,
+			       digestsize);
+	}
+	return 0;
+}
+
+static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
+{
+	struct chcr_context *ctx = crypto_tfm_ctx(tfm);
+	struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+	unsigned int digestsize =
+		crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
+
+	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+				 sizeof(struct chcr_ahash_req_ctx));
+	hmacctx->desc = chcr_alloc_shash(digestsize);
+	if (IS_ERR(hmacctx->desc))
+		return PTR_ERR(hmacctx->desc);
+	return chcr_device_init(crypto_tfm_ctx(tfm));
+}
+
+static void chcr_free_shash(struct shash_desc *desc)
+{
+	crypto_free_shash(desc->tfm);
+	kfree(desc);
+}
+
+static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
+{
+	struct chcr_context *ctx = crypto_tfm_ctx(tfm);
+	struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
+
+	if (hmacctx->desc) {
+		chcr_free_shash(hmacctx->desc);
+		hmacctx->desc = NULL;
+	}
+}
+
+static struct chcr_alg_template driver_algs[] = {
+	/* AES-CBC */
+	{
+		.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.is_registered = 0,
+		.alg.crypto = {
+			.cra_name		= "cbc(aes)",
+			.cra_driver_name	= "cbc(aes-chcr)",
+			.cra_priority		= CHCR_CRA_PRIORITY,
+			.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
+				CRYPTO_ALG_ASYNC,
+			.cra_blocksize		= AES_BLOCK_SIZE,
+			.cra_ctxsize		= sizeof(struct chcr_context)
+				+ sizeof(struct ablk_ctx),
+			.cra_alignmask		= 0,
+			.cra_type		= &crypto_ablkcipher_type,
+			.cra_module		= THIS_MODULE,
+			.cra_init		= chcr_cra_init,
+			.cra_exit		= NULL,
+			.cra_u.ablkcipher	= {
+				.min_keysize	= AES_MIN_KEY_SIZE,
+				.max_keysize	= AES_MAX_KEY_SIZE,
+				.ivsize		= AES_BLOCK_SIZE,
+				.setkey			= chcr_aes_cbc_setkey,
+				.encrypt		= chcr_aes_encrypt,
+				.decrypt		= chcr_aes_decrypt,
+			}
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.is_registered = 0,
+		.alg.crypto =   {
+			.cra_name		= "xts(aes)",
+			.cra_driver_name	= "xts(aes-chcr)",
+			.cra_priority		= CHCR_CRA_PRIORITY,
+			.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
+				CRYPTO_ALG_ASYNC,
+			.cra_blocksize		= AES_BLOCK_SIZE,
+			.cra_ctxsize		= sizeof(struct chcr_context) +
+				sizeof(struct ablk_ctx),
+			.cra_alignmask		= 0,
+			.cra_type		= &crypto_ablkcipher_type,
+			.cra_module		= THIS_MODULE,
+			.cra_init		= chcr_cra_init,
+			.cra_exit		= NULL,
+			.cra_u = {
+				.ablkcipher = {
+					.min_keysize	= 2 * AES_MIN_KEY_SIZE,
+					.max_keysize	= 2 * AES_MAX_KEY_SIZE,
+					.ivsize		= AES_BLOCK_SIZE,
+					.setkey		= chcr_aes_xts_setkey,
+					.encrypt	= chcr_aes_encrypt,
+					.decrypt	= chcr_aes_decrypt,
+				}
+			}
+		}
+	},
+	/* SHA */
+	{
+		.type = CRYPTO_ALG_TYPE_AHASH,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA1_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "sha1",
+				.cra_driver_name = "sha1-chcr",
+				.cra_blocksize = SHA1_BLOCK_SIZE,
+			}
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AHASH,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA256_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "sha256",
+				.cra_driver_name = "sha256-chcr",
+				.cra_blocksize = SHA256_BLOCK_SIZE,
+			}
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AHASH,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA224_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "sha224",
+				.cra_driver_name = "sha224-chcr",
+				.cra_blocksize = SHA224_BLOCK_SIZE,
+			}
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AHASH,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA384_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "sha384",
+				.cra_driver_name = "sha384-chcr",
+				.cra_blocksize = SHA384_BLOCK_SIZE,
+			}
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AHASH,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA512_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "sha512",
+				.cra_driver_name = "sha512-chcr",
+				.cra_blocksize = SHA512_BLOCK_SIZE,
+			}
+		}
+	},
+	/* HMAC */
+	{
+		.type = CRYPTO_ALG_TYPE_HMAC,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA1_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "hmac(sha1)",
+				.cra_driver_name = "hmac(sha1-chcr)",
+				.cra_blocksize = SHA1_BLOCK_SIZE,
+			}
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_HMAC,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA224_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "hmac(sha224)",
+				.cra_driver_name = "hmac(sha224-chcr)",
+				.cra_blocksize = SHA224_BLOCK_SIZE,
+			}
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_HMAC,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA256_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "hmac(sha256)",
+				.cra_driver_name = "hmac(sha256-chcr)",
+				.cra_blocksize = SHA256_BLOCK_SIZE,
+			}
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_HMAC,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA384_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "hmac(sha384)",
+				.cra_driver_name = "hmac(sha384-chcr)",
+				.cra_blocksize = SHA384_BLOCK_SIZE,
+			}
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_HMAC,
+		.is_registered = 0,
+		.alg.hash = {
+			.halg.digestsize = SHA512_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "hmac(sha512)",
+				.cra_driver_name = "hmac(sha512-chcr)",
+				.cra_blocksize = SHA512_BLOCK_SIZE,
+			}
+		}
+	},
+};
+
+/*
+ *	chcr_unregister_alg - Deregister crypto algorithms with
+ *	kernel framework.
+ */
+static int chcr_unregister_alg(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
+		switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
+		case CRYPTO_ALG_TYPE_ABLKCIPHER:
+			if (driver_algs[i].is_registered)
+				crypto_unregister_alg(
+						&driver_algs[i].alg.crypto);
+			break;
+		case CRYPTO_ALG_TYPE_AHASH:
+			if (driver_algs[i].is_registered)
+				crypto_unregister_ahash(
+						&driver_algs[i].alg.hash);
+			break;
+		}
+		driver_algs[i].is_registered = 0;
+	}
+	return 0;
+}
+
+#define SZ_AHASH_CTX sizeof(struct chcr_context)
+#define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
+#define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
+#define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC)
+
+/*
+ *	chcr_register_alg - Register crypto algorithms with kernel framework.
+ */
+static int chcr_register_alg(void)
+{
+	struct crypto_alg ai;
+	struct ahash_alg *a_hash;
+	int err = 0, i;
+	char *name = NULL;
+
+	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
+		if (driver_algs[i].is_registered)
+			continue;
+		switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
+		case CRYPTO_ALG_TYPE_ABLKCIPHER:
+			err = crypto_register_alg(&driver_algs[i].alg.crypto);
+			name = driver_algs[i].alg.crypto.cra_driver_name;
+			break;
+		case CRYPTO_ALG_TYPE_AHASH:
+			a_hash = &driver_algs[i].alg.hash;
+			a_hash->update = chcr_ahash_update;
+			a_hash->final = chcr_ahash_final;
+			a_hash->finup = chcr_ahash_finup;
+			a_hash->digest = chcr_ahash_digest;
+			a_hash->export = chcr_ahash_export;
+			a_hash->import = chcr_ahash_import;
+			a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
+			a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
+			a_hash->halg.base.cra_module = THIS_MODULE;
+			a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS;
+			a_hash->halg.base.cra_alignmask = 0;
+			a_hash->halg.base.cra_exit = NULL;
+			a_hash->halg.base.cra_type = &crypto_ahash_type;
+
+			if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
+				a_hash->halg.base.cra_init = chcr_hmac_cra_init;
+				a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
+				a_hash->init = chcr_hmac_init;
+				a_hash->setkey = chcr_ahash_setkey;
+				a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
+			} else {
+				a_hash->init = chcr_sha_init;
+				a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
+				a_hash->halg.base.cra_init = chcr_sha_cra_init;
+			}
+			err = crypto_register_ahash(&driver_algs[i].alg.hash);
+			ai = driver_algs[i].alg.hash.halg.base;
+			name = ai.cra_driver_name;
+			break;
+		}
+		if (err) {
+			pr_err("chcr : %s : Algorithm registration failed\n",
+			       name);
+			goto register_err;
+		} else {
+			driver_algs[i].is_registered = 1;
+		}
+	}
+	return 0;
+
+register_err:
+	chcr_unregister_alg();
+	return err;
+}
+
+/*
+ *	start_crypto - Register the crypto algorithms.
+ *	This should called once when the first device comesup. After this
+ *	kernel will start calling driver APIs for crypto operations.
+ */
+int start_crypto(void)
+{
+	return chcr_register_alg();
+}
+
+/*
+ *	stop_crypto - Deregister all the crypto algorithms with kernel.
+ *	This should be called once when the last device goes down. After this
+ *	kernel will not call the driver API for crypto operations.
+ */
+int stop_crypto(void)
+{
+	chcr_unregister_alg();
+	return 0;
+}
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
new file mode 100644
index 000000000000..ec64fbcdeb49
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -0,0 +1,471 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __CHCR_ALGO_H__
+#define __CHCR_ALGO_H__
+
+/* Crypto key context */
+#define KEY_CONTEXT_CTX_LEN_S           24
+#define KEY_CONTEXT_CTX_LEN_M           0xff
+#define KEY_CONTEXT_CTX_LEN_V(x)        ((x) << KEY_CONTEXT_CTX_LEN_S)
+#define KEY_CONTEXT_CTX_LEN_G(x) \
+	(((x) >> KEY_CONTEXT_CTX_LEN_S) & KEY_CONTEXT_CTX_LEN_M)
+
+#define KEY_CONTEXT_DUAL_CK_S      12
+#define KEY_CONTEXT_DUAL_CK_M      0x1
+#define KEY_CONTEXT_DUAL_CK_V(x)   ((x) << KEY_CONTEXT_DUAL_CK_S)
+#define KEY_CONTEXT_DUAL_CK_G(x)   \
+(((x) >> KEY_CONTEXT_DUAL_CK_S) & KEY_CONTEXT_DUAL_CK_M)
+#define KEY_CONTEXT_DUAL_CK_F      KEY_CONTEXT_DUAL_CK_V(1U)
+
+#define KEY_CONTEXT_SALT_PRESENT_S      10
+#define KEY_CONTEXT_SALT_PRESENT_M      0x1
+#define KEY_CONTEXT_SALT_PRESENT_V(x)   ((x) << KEY_CONTEXT_SALT_PRESENT_S)
+#define KEY_CONTEXT_SALT_PRESENT_G(x)   \
+	(((x) >> KEY_CONTEXT_SALT_PRESENT_S) & \
+	 KEY_CONTEXT_SALT_PRESENT_M)
+#define KEY_CONTEXT_SALT_PRESENT_F      KEY_CONTEXT_SALT_PRESENT_V(1U)
+
+#define KEY_CONTEXT_VALID_S     0
+#define KEY_CONTEXT_VALID_M     0x1
+#define KEY_CONTEXT_VALID_V(x)  ((x) << KEY_CONTEXT_VALID_S)
+#define KEY_CONTEXT_VALID_G(x)  \
+	(((x) >> KEY_CONTEXT_VALID_S) & \
+	 KEY_CONTEXT_VALID_M)
+#define KEY_CONTEXT_VALID_F     KEY_CONTEXT_VALID_V(1U)
+
+#define KEY_CONTEXT_CK_SIZE_S           6
+#define KEY_CONTEXT_CK_SIZE_M           0xf
+#define KEY_CONTEXT_CK_SIZE_V(x)        ((x) << KEY_CONTEXT_CK_SIZE_S)
+#define KEY_CONTEXT_CK_SIZE_G(x)        \
+	(((x) >> KEY_CONTEXT_CK_SIZE_S) & KEY_CONTEXT_CK_SIZE_M)
+
+#define KEY_CONTEXT_MK_SIZE_S           2
+#define KEY_CONTEXT_MK_SIZE_M           0xf
+#define KEY_CONTEXT_MK_SIZE_V(x)        ((x) << KEY_CONTEXT_MK_SIZE_S)
+#define KEY_CONTEXT_MK_SIZE_G(x)        \
+	(((x) >> KEY_CONTEXT_MK_SIZE_S) & KEY_CONTEXT_MK_SIZE_M)
+
+#define KEY_CONTEXT_OPAD_PRESENT_S      11
+#define KEY_CONTEXT_OPAD_PRESENT_M      0x1
+#define KEY_CONTEXT_OPAD_PRESENT_V(x)   ((x) << KEY_CONTEXT_OPAD_PRESENT_S)
+#define KEY_CONTEXT_OPAD_PRESENT_G(x)   \
+	(((x) >> KEY_CONTEXT_OPAD_PRESENT_S) & \
+	 KEY_CONTEXT_OPAD_PRESENT_M)
+#define KEY_CONTEXT_OPAD_PRESENT_F      KEY_CONTEXT_OPAD_PRESENT_V(1U)
+
+#define CHCR_HASH_MAX_DIGEST_SIZE 64
+#define CHCR_MAX_SHA_DIGEST_SIZE 64
+
+#define IPSEC_TRUNCATED_ICV_SIZE 12
+#define TLS_TRUNCATED_HMAC_SIZE 10
+#define CBCMAC_DIGEST_SIZE 16
+#define MAX_HASH_NAME 20
+
+#define SHA1_INIT_STATE_5X4B    5
+#define SHA256_INIT_STATE_8X4B  8
+#define SHA512_INIT_STATE_8X8B  8
+#define SHA1_INIT_STATE         SHA1_INIT_STATE_5X4B
+#define SHA224_INIT_STATE       SHA256_INIT_STATE_8X4B
+#define SHA256_INIT_STATE       SHA256_INIT_STATE_8X4B
+#define SHA384_INIT_STATE       SHA512_INIT_STATE_8X8B
+#define SHA512_INIT_STATE       SHA512_INIT_STATE_8X8B
+
+#define DUMMY_BYTES 16
+
+#define IPAD_DATA 0x36363636
+#define OPAD_DATA 0x5c5c5c5c
+
+#define TRANSHDR_SIZE(alignedkctx_len)\
+	(sizeof(struct ulptx_idata) +\
+	 sizeof(struct ulp_txpkt) +\
+	 sizeof(struct fw_crypto_lookaside_wr) +\
+	 sizeof(struct cpl_tx_sec_pdu) +\
+	 (alignedkctx_len))
+#define CIPHER_TRANSHDR_SIZE(alignedkctx_len, sge_pairs) \
+	(TRANSHDR_SIZE(alignedkctx_len) + sge_pairs +\
+	 sizeof(struct cpl_rx_phys_dsgl))
+#define HASH_TRANSHDR_SIZE(alignedkctx_len)\
+	(TRANSHDR_SIZE(alignedkctx_len) + DUMMY_BYTES)
+
+#define SEC_CPL_OFFSET (sizeof(struct fw_crypto_lookaside_wr) + \
+			sizeof(struct ulp_txpkt) + \
+			sizeof(struct ulptx_idata))
+
+#define FILL_SEC_CPL_OP_IVINSR(id, len, hldr, ofst)      \
+	htonl( \
+	       CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | \
+	       CPL_TX_SEC_PDU_RXCHID_V((id)) | \
+	       CPL_TX_SEC_PDU_ACKFOLLOWS_V(0) | \
+	       CPL_TX_SEC_PDU_ULPTXLPBK_V(1) | \
+	       CPL_TX_SEC_PDU_CPLLEN_V((len)) | \
+	       CPL_TX_SEC_PDU_PLACEHOLDER_V((hldr)) | \
+	       CPL_TX_SEC_PDU_IVINSRTOFST_V((ofst)))
+
+#define  FILL_SEC_CPL_CIPHERSTOP_HI(a_start, a_stop, c_start, c_stop_hi) \
+	htonl( \
+	       CPL_TX_SEC_PDU_AADSTART_V((a_start)) | \
+	       CPL_TX_SEC_PDU_AADSTOP_V((a_stop)) | \
+	       CPL_TX_SEC_PDU_CIPHERSTART_V((c_start)) | \
+	       CPL_TX_SEC_PDU_CIPHERSTOP_HI_V((c_stop_hi)))
+
+#define  FILL_SEC_CPL_AUTHINSERT(c_stop_lo, a_start, a_stop, a_inst) \
+	htonl( \
+	       CPL_TX_SEC_PDU_CIPHERSTOP_LO_V((c_stop_lo)) | \
+		CPL_TX_SEC_PDU_AUTHSTART_V((a_start)) | \
+		CPL_TX_SEC_PDU_AUTHSTOP_V((a_stop)) | \
+		CPL_TX_SEC_PDU_AUTHINSERT_V((a_inst)))
+
+#define  FILL_SEC_CPL_SCMD0_SEQNO(ctrl, seq, cmode, amode, opad, size, nivs)  \
+		htonl( \
+		SCMD_SEQ_NO_CTRL_V(0) | \
+		SCMD_STATUS_PRESENT_V(0) | \
+		SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | \
+		SCMD_ENC_DEC_CTRL_V((ctrl)) | \
+		SCMD_CIPH_AUTH_SEQ_CTRL_V((seq)) | \
+		SCMD_CIPH_MODE_V((cmode)) | \
+		SCMD_AUTH_MODE_V((amode)) | \
+		SCMD_HMAC_CTRL_V((opad)) | \
+		SCMD_IV_SIZE_V((size)) | \
+		SCMD_NUM_IVS_V((nivs)))
+
+#define FILL_SEC_CPL_IVGEN_HDRLEN(last, more, ctx_in, mac, ivdrop, len) htonl( \
+		SCMD_ENB_DBGID_V(0) | \
+		SCMD_IV_GEN_CTRL_V(0) | \
+		SCMD_LAST_FRAG_V((last)) | \
+		SCMD_MORE_FRAGS_V((more)) | \
+		SCMD_TLS_COMPPDU_V(0) | \
+		SCMD_KEY_CTX_INLINE_V((ctx_in)) | \
+		SCMD_TLS_FRAG_ENABLE_V(0) | \
+		SCMD_MAC_ONLY_V((mac)) |  \
+		SCMD_AADIVDROP_V((ivdrop)) | \
+		SCMD_HDR_LEN_V((len)))
+
+#define  FILL_KEY_CTX_HDR(ck_size, mk_size, d_ck, opad, ctx_len) \
+		htonl(KEY_CONTEXT_VALID_V(1) | \
+		      KEY_CONTEXT_CK_SIZE_V((ck_size)) | \
+		      KEY_CONTEXT_MK_SIZE_V(mk_size) | \
+		      KEY_CONTEXT_DUAL_CK_V((d_ck)) | \
+		      KEY_CONTEXT_OPAD_PRESENT_V((opad)) | \
+		      KEY_CONTEXT_SALT_PRESENT_V(1) | \
+		      KEY_CONTEXT_CTX_LEN_V((ctx_len)))
+
+#define FILL_WR_OP_CCTX_SIZE(len, ctx_len) \
+		htonl( \
+			FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \
+			FW_CRYPTO_LOOKASIDE_WR) | \
+			FW_CRYPTO_LOOKASIDE_WR_COMPL_V(0) | \
+			FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((len)) | \
+			FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \
+			FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len)))
+
+#define FILL_WR_RX_Q_ID(cid, qid, wr_iv) \
+		htonl( \
+			FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \
+			FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \
+			FW_CRYPTO_LOOKASIDE_WR_LCB_V(0) | \
+			FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)))
+
+#define FILL_ULPTX_CMD_DEST(cid) \
+	htonl(ULPTX_CMD_V(ULP_TX_PKT) | \
+	      ULP_TXPKT_DEST_V(0) | \
+	      ULP_TXPKT_DATAMODIFY_V(0) | \
+	      ULP_TXPKT_CHANNELID_V((cid)) | \
+	      ULP_TXPKT_RO_V(1) | \
+	      ULP_TXPKT_FID_V(0))
+
+#define KEYCTX_ALIGN_PAD(bs) ({unsigned int _bs = (bs);\
+			      _bs == SHA1_DIGEST_SIZE ? 12 : 0; })
+
+#define FILL_PLD_SIZE_HASH_SIZE(payload_sgl_len, sgl_lengths, total_frags) \
+	htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(payload_sgl_len ? \
+						sgl_lengths[total_frags] : 0) |\
+	      FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(0))
+
+#define FILL_LEN_PKD(calc_tx_flits_ofld, skb) \
+	htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP((\
+					   calc_tx_flits_ofld(skb) * 8), 16)))
+
+#define FILL_CMD_MORE(immdatalen) htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) |\
+					ULP_TX_SC_MORE_V((immdatalen) ? 0 : 1))
+
+#define MAX_NK 8
+#define CRYPTO_MAX_IMM_TX_PKT_LEN 256
+
+struct algo_param {
+	unsigned int auth_mode;
+	unsigned int mk_size;
+	unsigned int result_size;
+};
+
+struct hash_wr_param {
+	unsigned int opad_needed;
+	unsigned int more;
+	unsigned int last;
+	struct algo_param alg_prm;
+	unsigned int sg_len;
+	unsigned int bfr_len;
+	u64 scmd1;
+};
+
+enum {
+	AES_KEYLENGTH_128BIT = 128,
+	AES_KEYLENGTH_192BIT = 192,
+	AES_KEYLENGTH_256BIT = 256
+};
+
+enum {
+	KEYLENGTH_3BYTES = 3,
+	KEYLENGTH_4BYTES = 4,
+	KEYLENGTH_6BYTES = 6,
+	KEYLENGTH_8BYTES = 8
+};
+
+enum {
+	NUMBER_OF_ROUNDS_10 = 10,
+	NUMBER_OF_ROUNDS_12 = 12,
+	NUMBER_OF_ROUNDS_14 = 14,
+};
+
+/*
+ * CCM defines values of 4, 6, 8, 10, 12, 14, and 16 octets,
+ * where they indicate the size of the integrity check value (ICV)
+ */
+enum {
+	AES_CCM_ICV_4   = 4,
+	AES_CCM_ICV_6   = 6,
+	AES_CCM_ICV_8   = 8,
+	AES_CCM_ICV_10  = 10,
+	AES_CCM_ICV_12  = 12,
+	AES_CCM_ICV_14  = 14,
+	AES_CCM_ICV_16 = 16
+};
+
+struct hash_op_params {
+	unsigned char mk_size;
+	unsigned char pad_align;
+	unsigned char auth_mode;
+	char hash_name[MAX_HASH_NAME];
+	unsigned short block_size;
+	unsigned short word_size;
+	unsigned short ipad_size;
+};
+
+struct phys_sge_pairs {
+	__be16 len[8];
+	__be64 addr[8];
+};
+
+struct phys_sge_parm {
+	unsigned int nents;
+	unsigned int obsize;
+	unsigned short qid;
+	unsigned char align;
+};
+
+struct crypto_result {
+	struct completion completion;
+	int err;
+};
+
+static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
+		SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
+};
+
+static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
+		SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
+		SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
+};
+
+static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
+		SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
+		SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
+};
+
+static const u64 sha384_init[SHA512_DIGEST_SIZE / 8] = {
+		SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3,
+		SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7,
+};
+
+static const u64 sha512_init[SHA512_DIGEST_SIZE / 8] = {
+		SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3,
+		SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7,
+};
+
+static inline void copy_hash_init_values(char *key, int digestsize)
+{
+	u8 i;
+	__be32 *dkey = (__be32 *)key;
+	u64 *ldkey = (u64 *)key;
+	__be64 *sha384 = (__be64 *)sha384_init;
+	__be64 *sha512 = (__be64 *)sha512_init;
+
+	switch (digestsize) {
+	case SHA1_DIGEST_SIZE:
+		for (i = 0; i < SHA1_INIT_STATE; i++)
+			dkey[i] = cpu_to_be32(sha1_init[i]);
+		break;
+	case SHA224_DIGEST_SIZE:
+		for (i = 0; i < SHA224_INIT_STATE; i++)
+			dkey[i] = cpu_to_be32(sha224_init[i]);
+		break;
+	case SHA256_DIGEST_SIZE:
+		for (i = 0; i < SHA256_INIT_STATE; i++)
+			dkey[i] = cpu_to_be32(sha256_init[i]);
+		break;
+	case SHA384_DIGEST_SIZE:
+		for (i = 0; i < SHA384_INIT_STATE; i++)
+			ldkey[i] = be64_to_cpu(sha384[i]);
+		break;
+	case SHA512_DIGEST_SIZE:
+		for (i = 0; i < SHA512_INIT_STATE; i++)
+			ldkey[i] = be64_to_cpu(sha512[i]);
+		break;
+	}
+}
+
+static const u8 sgl_lengths[20] = {
+	0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15
+};
+
+/* Number of len fields(8) * size of one addr field */
+#define PHYSDSGL_MAX_LEN_SIZE 16
+
+static inline u16 get_space_for_phys_dsgl(unsigned int sgl_entr)
+{
+	/* len field size + addr field size */
+	return ((sgl_entr >> 3) + ((sgl_entr % 8) ?
+				   1 : 0)) * PHYSDSGL_MAX_LEN_SIZE +
+		(sgl_entr << 3) + ((sgl_entr % 2 ? 1 : 0) << 3);
+}
+
+/* The AES s-transform matrix (s-box). */
+static const u8 aes_sbox[256] = {
+	99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215,
+	171, 118, 202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175,
+	156, 164, 114, 192, 183, 253, 147, 38,  54,  63,  247, 204, 52,  165,
+	229, 241, 113, 216, 49,  21, 4,   199, 35,  195, 24,  150, 5, 154, 7,
+	18,  128, 226, 235, 39,  178, 117, 9,   131, 44,  26,  27,  110, 90,
+	160, 82,  59,  214, 179, 41,  227, 47,  132, 83,  209, 0,   237, 32,
+	252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207, 208, 239, 170,
+	251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168, 81,
+	163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243,
+	210, 205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100,
+	93,  25,  115, 96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184,
+	20,  222, 94,  11,  219, 224, 50,  58,  10,  73,  6,   36,  92,  194,
+	211, 172, 98,  145, 149, 228, 121, 231, 200, 55,  109, 141, 213, 78,
+	169, 108, 86,  244, 234, 101, 122, 174, 8, 186, 120, 37,  46,  28, 166,
+	180, 198, 232, 221, 116, 31,  75,  189, 139, 138, 112, 62,  181, 102,
+	72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158, 225, 248,
+	152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
+	140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,
+	187, 22
+};
+
+static u32 aes_ks_subword(const u32 w)
+{
+	u8 bytes[4];
+
+	*(u32 *)(&bytes[0]) = w;
+	bytes[0] = aes_sbox[bytes[0]];
+	bytes[1] = aes_sbox[bytes[1]];
+	bytes[2] = aes_sbox[bytes[2]];
+	bytes[3] = aes_sbox[bytes[3]];
+	return *(u32 *)(&bytes[0]);
+}
+
+static u32 round_constant[11] = {
+	0x01000000, 0x02000000, 0x04000000, 0x08000000,
+	0x10000000, 0x20000000, 0x40000000, 0x80000000,
+	0x1B000000, 0x36000000, 0x6C000000
+};
+
+/* dec_key - OUTPUT - Reverse round key
+ * key - INPUT - key
+ * keylength - INPUT - length of the key in number of bits
+ */
+static inline void get_aes_decrypt_key(unsigned char *dec_key,
+				       const unsigned char *key,
+				       unsigned int keylength)
+{
+	u32 temp;
+	u32 w_ring[MAX_NK];
+	int i, j, k = 0;
+	u8  nr, nk;
+
+	switch (keylength) {
+	case AES_KEYLENGTH_128BIT:
+		nk = KEYLENGTH_4BYTES;
+		nr = NUMBER_OF_ROUNDS_10;
+		break;
+
+	case AES_KEYLENGTH_192BIT:
+		nk = KEYLENGTH_6BYTES;
+		nr = NUMBER_OF_ROUNDS_12;
+		break;
+	case AES_KEYLENGTH_256BIT:
+		nk = KEYLENGTH_8BYTES;
+		nr = NUMBER_OF_ROUNDS_14;
+		break;
+	default:
+		return;
+	}
+	for (i = 0; i < nk; i++ )
+		w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
+
+	i = 0;
+	temp = w_ring[nk - 1];
+	while(i + nk < (nr + 1) * 4) {
+		if(!(i % nk)) {
+			/* RotWord(temp) */
+			temp = (temp << 8) | (temp >> 24);
+			temp = aes_ks_subword(temp);
+			temp ^= round_constant[i / nk];
+		}
+		else if (nk == 8 && (i % 4 == 0))
+			temp = aes_ks_subword(temp);
+		w_ring[i % nk] ^= temp;
+		temp = w_ring[i % nk];
+		i++;
+	}
+	for (k = 0, j = i % nk; k < nk; k++) {
+		*((u32 *)dec_key + k) = htonl(w_ring[j]);
+		j--;
+		if(j < 0)
+			j += nk;
+	}
+}
+
+#endif /* __CHCR_ALGO_H__ */
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
new file mode 100644
index 000000000000..fb5f9bbfa09c
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -0,0 +1,238 @@
+/**
+ * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux.
+ *
+ * Copyright (C) 2011-2016 Chelsio Communications.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * Written and Maintained by:
+ * Manoj Malviya (manojmalviya@chelsio.com)
+ * Atul Gupta (atul.gupta@chelsio.com)
+ * Jitendra Lulla (jlulla@chelsio.com)
+ * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
+ * Harsh Jain (harsh@chelsio.com)
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/skbuff.h>
+
+#include <crypto/aes.h>
+#include <crypto/hash.h>
+
+#include "t4_msg.h"
+#include "chcr_core.h"
+#include "cxgb4_uld.h"
+
+static LIST_HEAD(uld_ctx_list);
+static DEFINE_MUTEX(dev_mutex);
+static atomic_t dev_count;
+
+typedef int (*chcr_handler_func)(struct chcr_dev *dev, unsigned char *input);
+static int cpl_fw6_pld_handler(struct chcr_dev *dev, unsigned char *input);
+static void *chcr_uld_add(const struct cxgb4_lld_info *lld);
+static int chcr_uld_state_change(void *handle, enum cxgb4_state state);
+
+static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
+	[CPL_FW6_PLD] = cpl_fw6_pld_handler,
+};
+
+static struct cxgb4_uld_info chcr_uld_info = {
+	.name = DRV_MODULE_NAME,
+	.nrxq = MAX_ULD_QSETS,
+	.rxq_size = 1024,
+	.add = chcr_uld_add,
+	.state_change = chcr_uld_state_change,
+	.rx_handler = chcr_uld_rx_handler,
+};
+
+int assign_chcr_device(struct chcr_dev **dev)
+{
+	struct uld_ctx *u_ctx;
+
+	/*
+	 * Which device to use if multiple devices are available TODO
+	 * May be select the device based on round robin. One session
+	 * must go to the same device to maintain the ordering.
+	 */
+	mutex_lock(&dev_mutex); /* TODO ? */
+	u_ctx = list_first_entry(&uld_ctx_list, struct uld_ctx, entry);
+	if (!u_ctx) {
+		mutex_unlock(&dev_mutex);
+		return -ENXIO;
+	}
+
+	*dev = u_ctx->dev;
+	mutex_unlock(&dev_mutex);
+	return 0;
+}
+
+static int chcr_dev_add(struct uld_ctx *u_ctx)
+{
+	struct chcr_dev *dev;
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENXIO;
+
+	spin_lock_init(&dev->lock_chcr_dev);
+	u_ctx->dev = dev;
+	dev->u_ctx = u_ctx;
+	atomic_inc(&dev_count);
+	return 0;
+}
+
+static int chcr_dev_remove(struct uld_ctx *u_ctx)
+{
+	kfree(u_ctx->dev);
+	u_ctx->dev = NULL;
+	atomic_dec(&dev_count);
+	return 0;
+}
+
+static int cpl_fw6_pld_handler(struct chcr_dev *dev,
+			       unsigned char *input)
+{
+	struct crypto_async_request *req;
+	struct cpl_fw6_pld *fw6_pld;
+	u32 ack_err_status = 0;
+	int error_status = 0;
+
+	fw6_pld = (struct cpl_fw6_pld *)input;
+	req = (struct crypto_async_request *)(uintptr_t)be64_to_cpu(
+						    fw6_pld->data[1]);
+
+	ack_err_status =
+		ntohl(*(__be32 *)((unsigned char *)&fw6_pld->data[0] + 4));
+	if (ack_err_status) {
+		if (CHK_MAC_ERR_BIT(ack_err_status) ||
+		    CHK_PAD_ERR_BIT(ack_err_status))
+			error_status = -EINVAL;
+	}
+	/* call completion callback with failure status */
+	if (req) {
+		if (!chcr_handle_resp(req, input, error_status))
+			req->complete(req, error_status);
+		else
+			return -EINVAL;
+	} else {
+		pr_err("Incorrect request address from the firmware\n");
+		return -EFAULT;
+	}
+	return 0;
+}
+
+int chcr_send_wr(struct sk_buff *skb)
+{
+	return cxgb4_ofld_send(skb->dev, skb);
+}
+
+static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
+{
+	struct uld_ctx *u_ctx;
+
+	/* Create the device and add it in the device list */
+	u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
+	if (!u_ctx) {
+		u_ctx = ERR_PTR(-ENOMEM);
+		goto out;
+	}
+	u_ctx->lldi = *lld;
+	mutex_lock(&dev_mutex);
+	list_add_tail(&u_ctx->entry, &uld_ctx_list);
+	mutex_unlock(&dev_mutex);
+out:
+	return u_ctx;
+}
+
+int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
+			const struct pkt_gl *pgl)
+{
+	struct uld_ctx *u_ctx = (struct uld_ctx *)handle;
+	struct chcr_dev *dev = u_ctx->dev;
+	const struct cpl_act_establish *rpl = (struct cpl_act_establish
+					       *)rsp;
+
+	if (rpl->ot.opcode != CPL_FW6_PLD) {
+		pr_err("Unsupported opcode\n");
+		return 0;
+	}
+
+	if (!pgl)
+		work_handlers[rpl->ot.opcode](dev, (unsigned char *)&rsp[1]);
+	else
+		work_handlers[rpl->ot.opcode](dev, pgl->va);
+	return 0;
+}
+
+static int chcr_uld_state_change(void *handle, enum cxgb4_state state)
+{
+	struct uld_ctx *u_ctx = handle;
+	int ret = 0;
+
+	switch (state) {
+	case CXGB4_STATE_UP:
+		if (!u_ctx->dev) {
+			ret = chcr_dev_add(u_ctx);
+			if (ret != 0)
+				return ret;
+		}
+		if (atomic_read(&dev_count) == 1)
+			ret = start_crypto();
+		break;
+
+	case CXGB4_STATE_DETACH:
+		if (u_ctx->dev) {
+			mutex_lock(&dev_mutex);
+			chcr_dev_remove(u_ctx);
+			mutex_unlock(&dev_mutex);
+		}
+		if (!atomic_read(&dev_count))
+			stop_crypto();
+		break;
+
+	case CXGB4_STATE_START_RECOVERY:
+	case CXGB4_STATE_DOWN:
+	default:
+		break;
+	}
+	return ret;
+}
+
+static int __init chcr_crypto_init(void)
+{
+	if (cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info)) {
+		pr_err("ULD register fail: No chcr crypto support in cxgb4");
+		return -1;
+	}
+
+	return 0;
+}
+
+static void __exit chcr_crypto_exit(void)
+{
+	struct uld_ctx *u_ctx, *tmp;
+
+	if (atomic_read(&dev_count))
+		stop_crypto();
+
+	/* Remove all devices from list */
+	mutex_lock(&dev_mutex);
+	list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
+		if (u_ctx->dev)
+			chcr_dev_remove(u_ctx);
+		kfree(u_ctx);
+	}
+	mutex_unlock(&dev_mutex);
+	cxgb4_unregister_uld(CXGB4_ULD_CRYPTO);
+}
+
+module_init(chcr_crypto_init);
+module_exit(chcr_crypto_exit);
+
+MODULE_DESCRIPTION("Crypto Co-processor for Chelsio Terminator cards.");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Chelsio Communications");
+MODULE_VERSION(DRV_VERSION);
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
new file mode 100644
index 000000000000..2a5c671a4232
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -0,0 +1,80 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __CHCR_CORE_H__
+#define __CHCR_CORE_H__
+
+#include <crypto/algapi.h>
+#include "t4_hw.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+
+#define DRV_MODULE_NAME "chcr"
+#define DRV_VERSION "1.0.0.0"
+
+#define MAX_PENDING_REQ_TO_HW 20
+#define CHCR_TEST_RESPONSE_TIMEOUT 1000
+
+#define PAD_ERROR_BIT		1
+#define CHK_PAD_ERR_BIT(x)	(((x) >> PAD_ERROR_BIT) & 1)
+
+#define MAC_ERROR_BIT		0
+#define CHK_MAC_ERR_BIT(x)	(((x) >> MAC_ERROR_BIT) & 1)
+
+struct uld_ctx;
+
+struct chcr_dev {
+	/* Request submited to h/w and waiting for response. */
+	spinlock_t lock_chcr_dev;
+	struct crypto_queue pending_queue;
+	struct uld_ctx *u_ctx;
+	unsigned char tx_channel_id;
+};
+
+struct uld_ctx {
+	struct list_head entry;
+	struct cxgb4_lld_info lldi;
+	struct chcr_dev *dev;
+};
+
+int assign_chcr_device(struct chcr_dev **dev);
+int chcr_send_wr(struct sk_buff *skb);
+int start_crypto(void);
+int stop_crypto(void);
+int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
+			const struct pkt_gl *pgl);
+int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
+		     int err);
+#endif /* __CHCR_CORE_H__ */
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
new file mode 100644
index 000000000000..d7d75605da8b
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -0,0 +1,203 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef __CHCR_CRYPTO_H__
+#define __CHCR_CRYPTO_H__
+
+/* Define following if h/w is not dropping the AAD and IV data before
+ * giving the processed data
+ */
+
+#define CHCR_CRA_PRIORITY 300
+
+#define CHCR_AES_MAX_KEY_LEN  (2 * (AES_MAX_KEY_SIZE)) /* consider xts */
+#define CHCR_MAX_CRYPTO_IV_LEN 16 /* AES IV len */
+
+#define CHCR_MAX_AUTHENC_AES_KEY_LEN 32 /* max aes key length*/
+#define CHCR_MAX_AUTHENC_SHA_KEY_LEN 128 /* max sha key length*/
+
+#define CHCR_GIVENCRYPT_OP 2
+/* CPL/SCMD parameters */
+
+#define CHCR_ENCRYPT_OP 0
+#define CHCR_DECRYPT_OP 1
+
+#define CHCR_SCMD_SEQ_NO_CTRL_32BIT     1
+#define CHCR_SCMD_SEQ_NO_CTRL_48BIT     2
+#define CHCR_SCMD_SEQ_NO_CTRL_64BIT     3
+
+#define CHCR_SCMD_PROTO_VERSION_GENERIC 4
+
+#define CHCR_SCMD_AUTH_CTRL_AUTH_CIPHER 0
+#define CHCR_SCMD_AUTH_CTRL_CIPHER_AUTH 1
+
+#define CHCR_SCMD_CIPHER_MODE_NOP           0
+#define CHCR_SCMD_CIPHER_MODE_AES_CBC       1
+#define CHCR_SCMD_CIPHER_MODE_GENERIC_AES   4
+#define CHCR_SCMD_CIPHER_MODE_AES_XTS       6
+
+#define CHCR_SCMD_AUTH_MODE_NOP             0
+#define CHCR_SCMD_AUTH_MODE_SHA1            1
+#define CHCR_SCMD_AUTH_MODE_SHA224          2
+#define CHCR_SCMD_AUTH_MODE_SHA256          3
+#define CHCR_SCMD_AUTH_MODE_SHA512_224      5
+#define CHCR_SCMD_AUTH_MODE_SHA512_256      6
+#define CHCR_SCMD_AUTH_MODE_SHA512_384      7
+#define CHCR_SCMD_AUTH_MODE_SHA512_512      8
+
+#define CHCR_SCMD_HMAC_CTRL_NOP             0
+#define CHCR_SCMD_HMAC_CTRL_NO_TRUNC        1
+
+#define CHCR_SCMD_IVGEN_CTRL_HW             0
+#define CHCR_SCMD_IVGEN_CTRL_SW             1
+/* This are not really mac key size. They are intermediate values
+ * of sha engine and its size
+ */
+#define CHCR_KEYCTX_MAC_KEY_SIZE_128        0
+#define CHCR_KEYCTX_MAC_KEY_SIZE_160        1
+#define CHCR_KEYCTX_MAC_KEY_SIZE_192        2
+#define CHCR_KEYCTX_MAC_KEY_SIZE_256        3
+#define CHCR_KEYCTX_MAC_KEY_SIZE_512        4
+#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128     0
+#define CHCR_KEYCTX_CIPHER_KEY_SIZE_192     1
+#define CHCR_KEYCTX_CIPHER_KEY_SIZE_256     2
+#define CHCR_KEYCTX_NO_KEY                  15
+
+#define CHCR_CPL_FW4_PLD_IV_OFFSET          (5 * 64) /* bytes. flt #5 and #6 */
+#define CHCR_CPL_FW4_PLD_HASH_RESULT_OFFSET (7 * 64) /* bytes. flt #7 */
+#define CHCR_CPL_FW4_PLD_DATA_SIZE          (4 * 64) /* bytes. flt #4 to #7 */
+
+#define KEY_CONTEXT_HDR_SALT_AND_PAD	    16
+#define flits_to_bytes(x)  (x * 8)
+
+#define IV_NOP                  0
+#define IV_IMMEDIATE            1
+#define IV_DSGL			2
+
+#define CRYPTO_ALG_SUB_TYPE_MASK            0x0f000000
+#define CRYPTO_ALG_SUB_TYPE_HASH_HMAC       0x01000000
+#define CRYPTO_ALG_TYPE_HMAC (CRYPTO_ALG_TYPE_AHASH |\
+			      CRYPTO_ALG_SUB_TYPE_HASH_HMAC)
+
+#define MAX_SALT                4
+#define MAX_SCRATCH_PAD_SIZE    32
+
+#define CHCR_HASH_MAX_BLOCK_SIZE_64  64
+#define CHCR_HASH_MAX_BLOCK_SIZE_128 128
+
+/* Aligned to 128 bit boundary */
+struct _key_ctx {
+	__be32 ctx_hdr;
+	u8 salt[MAX_SALT];
+	__be64 reserverd;
+	unsigned char key[0];
+};
+
+struct ablk_ctx {
+	u8 enc;
+	unsigned int processed_len;
+	__be32 key_ctx_hdr;
+	unsigned int enckey_len;
+	unsigned int dst_nents;
+	struct scatterlist iv_sg;
+	u8 key[CHCR_AES_MAX_KEY_LEN];
+	u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
+	unsigned char ciph_mode;
+};
+
+struct hmac_ctx {
+	struct shash_desc *desc;
+	u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128];
+	u8 opad[CHCR_HASH_MAX_BLOCK_SIZE_128];
+};
+
+struct __crypto_ctx {
+	struct hmac_ctx hmacctx[0];
+	struct ablk_ctx ablkctx[0];
+};
+
+struct chcr_context {
+	struct chcr_dev *dev;
+	unsigned char tx_channel_id;
+	struct __crypto_ctx crypto_ctx[0];
+};
+
+struct chcr_ahash_req_ctx {
+	u32 result;
+	char bfr[CHCR_HASH_MAX_BLOCK_SIZE_128];
+	u8 bfr_len;
+	/* DMA the partial hash in it */
+	u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE];
+	u64 data_len;  /* Data len till time */
+	void *dummy_payload_ptr;
+	/* SKB which is being sent to the hardware for processing */
+	struct sk_buff *skb;
+};
+
+struct chcr_blkcipher_req_ctx {
+	struct sk_buff *skb;
+};
+
+struct chcr_alg_template {
+	u32 type;
+	u32 is_registered;
+	union {
+		struct crypto_alg crypto;
+		struct ahash_alg hash;
+	} alg;
+};
+
+struct chcr_req_ctx {
+	union {
+		struct ahash_request *ahash_req;
+		struct ablkcipher_request *ablk_req;
+	} req;
+	union {
+		struct chcr_ahash_req_ctx *ahash_ctx;
+		struct chcr_blkcipher_req_ctx *ablk_ctx;
+	} ctx;
+};
+
+struct sge_opaque_hdr {
+	void *dev;
+	dma_addr_t addr[MAX_SKB_FRAGS + 1];
+};
+
+typedef struct sk_buff *(*create_wr_t)(struct crypto_async_request *req,
+				       struct chcr_context *ctx,
+				       unsigned short qid,
+				       unsigned short op_type);
+
+#endif /* __CHCR_CRYPTO_H__ */
diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c
index eee2c7e6c299..e09d4055b19e 100644
--- a/drivers/crypto/hifn_795x.c
+++ b/drivers/crypto/hifn_795x.c
@@ -636,20 +636,12 @@ struct hifn_request_context {
 
 static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
 {
-	u32 ret;
-
-	ret = readl(dev->bar[0] + reg);
-
-	return ret;
+	return readl(dev->bar[0] + reg);
 }
 
 static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
 {
-	u32 ret;
-
-	ret = readl(dev->bar[1] + reg);
-
-	return ret;
+	return readl(dev->bar[1] + reg);
 }
 
 static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c
index 68e8aa90fe01..a2e77b87485b 100644
--- a/drivers/crypto/img-hash.c
+++ b/drivers/crypto/img-hash.c
@@ -71,6 +71,7 @@
 #define DRIVER_FLAGS_MD5		BIT(21)
 
 #define IMG_HASH_QUEUE_LENGTH		20
+#define IMG_HASH_DMA_BURST		4
 #define IMG_HASH_DMA_THRESHOLD		64
 
 #ifdef __LITTLE_ENDIAN
@@ -102,8 +103,10 @@ struct img_hash_request_ctx {
 	unsigned long		op;
 
 	size_t			bufcnt;
-	u8 buffer[0] __aligned(sizeof(u32));
 	struct ahash_request	fallback_req;
+
+	/* Zero length buffer must remain last member of struct */
+	u8 buffer[0] __aligned(sizeof(u32));
 };
 
 struct img_hash_ctx {
@@ -340,7 +343,7 @@ static int img_hash_dma_init(struct img_hash_dev *hdev)
 	dma_conf.direction = DMA_MEM_TO_DEV;
 	dma_conf.dst_addr = hdev->bus_addr;
 	dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
-	dma_conf.dst_maxburst = 16;
+	dma_conf.dst_maxburst = IMG_HASH_DMA_BURST;
 	dma_conf.device_fc = false;
 
 	err = dmaengine_slave_config(hdev->dma_lch,  &dma_conf);
@@ -361,7 +364,7 @@ static void img_hash_dma_task(unsigned long d)
 	size_t nbytes, bleft, wsend, len, tbc;
 	struct scatterlist tsg;
 
-	if (!ctx->sg)
+	if (!hdev->req || !ctx->sg)
 		return;
 
 	addr = sg_virt(ctx->sg);
@@ -587,6 +590,32 @@ static int img_hash_finup(struct ahash_request *req)
 	return crypto_ahash_finup(&rctx->fallback_req);
 }
 
+static int img_hash_import(struct ahash_request *req, const void *in)
+{
+	struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
+	rctx->fallback_req.base.flags = req->base.flags
+		& CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	return crypto_ahash_import(&rctx->fallback_req, in);
+}
+
+static int img_hash_export(struct ahash_request *req, void *out)
+{
+	struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
+	rctx->fallback_req.base.flags = req->base.flags
+		& CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	return crypto_ahash_export(&rctx->fallback_req, out);
+}
+
 static int img_hash_digest(struct ahash_request *req)
 {
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
@@ -643,10 +672,9 @@ static int img_hash_digest(struct ahash_request *req)
 	return err;
 }
 
-static int img_hash_cra_init(struct crypto_tfm *tfm)
+static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name)
 {
 	struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm);
-	const char *alg_name = crypto_tfm_alg_name(tfm);
 	int err = -ENOMEM;
 
 	ctx->fallback = crypto_alloc_ahash(alg_name, 0,
@@ -658,6 +686,7 @@ static int img_hash_cra_init(struct crypto_tfm *tfm)
 	}
 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 				 sizeof(struct img_hash_request_ctx) +
+				 crypto_ahash_reqsize(ctx->fallback) +
 				 IMG_HASH_DMA_THRESHOLD);
 
 	return 0;
@@ -666,6 +695,26 @@ err:
 	return err;
 }
 
+static int img_hash_cra_md5_init(struct crypto_tfm *tfm)
+{
+	return img_hash_cra_init(tfm, "md5-generic");
+}
+
+static int img_hash_cra_sha1_init(struct crypto_tfm *tfm)
+{
+	return img_hash_cra_init(tfm, "sha1-generic");
+}
+
+static int img_hash_cra_sha224_init(struct crypto_tfm *tfm)
+{
+	return img_hash_cra_init(tfm, "sha224-generic");
+}
+
+static int img_hash_cra_sha256_init(struct crypto_tfm *tfm)
+{
+	return img_hash_cra_init(tfm, "sha256-generic");
+}
+
 static void img_hash_cra_exit(struct crypto_tfm *tfm)
 {
 	struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm);
@@ -711,9 +760,12 @@ static struct ahash_alg img_algs[] = {
 		.update = img_hash_update,
 		.final = img_hash_final,
 		.finup = img_hash_finup,
+		.export = img_hash_export,
+		.import = img_hash_import,
 		.digest = img_hash_digest,
 		.halg = {
 			.digestsize = MD5_DIGEST_SIZE,
+			.statesize = sizeof(struct md5_state),
 			.base = {
 				.cra_name = "md5",
 				.cra_driver_name = "img-md5",
@@ -723,7 +775,7 @@ static struct ahash_alg img_algs[] = {
 				CRYPTO_ALG_NEED_FALLBACK,
 				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
 				.cra_ctxsize = sizeof(struct img_hash_ctx),
-				.cra_init = img_hash_cra_init,
+				.cra_init = img_hash_cra_md5_init,
 				.cra_exit = img_hash_cra_exit,
 				.cra_module = THIS_MODULE,
 			}
@@ -734,9 +786,12 @@ static struct ahash_alg img_algs[] = {
 		.update = img_hash_update,
 		.final = img_hash_final,
 		.finup = img_hash_finup,
+		.export = img_hash_export,
+		.import = img_hash_import,
 		.digest = img_hash_digest,
 		.halg = {
 			.digestsize = SHA1_DIGEST_SIZE,
+			.statesize = sizeof(struct sha1_state),
 			.base = {
 				.cra_name = "sha1",
 				.cra_driver_name = "img-sha1",
@@ -746,7 +801,7 @@ static struct ahash_alg img_algs[] = {
 				CRYPTO_ALG_NEED_FALLBACK,
 				.cra_blocksize = SHA1_BLOCK_SIZE,
 				.cra_ctxsize = sizeof(struct img_hash_ctx),
-				.cra_init = img_hash_cra_init,
+				.cra_init = img_hash_cra_sha1_init,
 				.cra_exit = img_hash_cra_exit,
 				.cra_module = THIS_MODULE,
 			}
@@ -757,9 +812,12 @@ static struct ahash_alg img_algs[] = {
 		.update = img_hash_update,
 		.final = img_hash_final,
 		.finup = img_hash_finup,
+		.export = img_hash_export,
+		.import = img_hash_import,
 		.digest = img_hash_digest,
 		.halg = {
 			.digestsize = SHA224_DIGEST_SIZE,
+			.statesize = sizeof(struct sha256_state),
 			.base = {
 				.cra_name = "sha224",
 				.cra_driver_name = "img-sha224",
@@ -769,7 +827,7 @@ static struct ahash_alg img_algs[] = {
 				CRYPTO_ALG_NEED_FALLBACK,
 				.cra_blocksize = SHA224_BLOCK_SIZE,
 				.cra_ctxsize = sizeof(struct img_hash_ctx),
-				.cra_init = img_hash_cra_init,
+				.cra_init = img_hash_cra_sha224_init,
 				.cra_exit = img_hash_cra_exit,
 				.cra_module = THIS_MODULE,
 			}
@@ -780,9 +838,12 @@ static struct ahash_alg img_algs[] = {
 		.update = img_hash_update,
 		.final = img_hash_final,
 		.finup = img_hash_finup,
+		.export = img_hash_export,
+		.import = img_hash_import,
 		.digest = img_hash_digest,
 		.halg = {
 			.digestsize = SHA256_DIGEST_SIZE,
+			.statesize = sizeof(struct sha256_state),
 			.base = {
 				.cra_name = "sha256",
 				.cra_driver_name = "img-sha256",
@@ -792,7 +853,7 @@ static struct ahash_alg img_algs[] = {
 				CRYPTO_ALG_NEED_FALLBACK,
 				.cra_blocksize = SHA256_BLOCK_SIZE,
 				.cra_ctxsize = sizeof(struct img_hash_ctx),
-				.cra_init = img_hash_cra_init,
+				.cra_init = img_hash_cra_sha256_init,
 				.cra_exit = img_hash_cra_exit,
 				.cra_module = THIS_MODULE,
 			}
@@ -971,7 +1032,7 @@ static int img_hash_probe(struct platform_device *pdev)
 	err = img_register_algs(hdev);
 	if (err)
 		goto err_algs;
-	dev_dbg(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");
+	dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");
 
 	return 0;
 
@@ -1013,11 +1074,38 @@ static int img_hash_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int img_hash_suspend(struct device *dev)
+{
+	struct img_hash_dev *hdev = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(hdev->hash_clk);
+	clk_disable_unprepare(hdev->sys_clk);
+
+	return 0;
+}
+
+static int img_hash_resume(struct device *dev)
+{
+	struct img_hash_dev *hdev = dev_get_drvdata(dev);
+
+	clk_prepare_enable(hdev->hash_clk);
+	clk_prepare_enable(hdev->sys_clk);
+
+	return 0;
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static const struct dev_pm_ops img_hash_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(img_hash_suspend, img_hash_resume)
+};
+
 static struct platform_driver img_hash_driver = {
 	.probe		= img_hash_probe,
 	.remove		= img_hash_remove,
 	.driver		= {
 		.name	= "img-hash-accelerator",
+		.pm	= &img_hash_pm_ops,
 		.of_match_table	= of_match_ptr(img_hash_match),
 	}
 };
diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c
index 2296934455fc..7868765a70c5 100644
--- a/drivers/crypto/ixp4xx_crypto.c
+++ b/drivers/crypto/ixp4xx_crypto.c
@@ -447,9 +447,8 @@ static int init_ixp_crypto(struct device *dev)
 
 	if (!npe_running(npe_c)) {
 		ret = npe_load_firmware(npe_c, npe_name(npe_c), dev);
-		if (ret) {
-			return ret;
-		}
+		if (ret)
+			goto npe_release;
 		if (npe_recv_message(npe_c, msg, "STATUS_MSG"))
 			goto npe_error;
 	} else {
@@ -473,7 +472,8 @@ static int init_ixp_crypto(struct device *dev)
 	default:
 		printk(KERN_ERR "Firmware of %s lacks crypto support\n",
 			npe_name(npe_c));
-		return -ENODEV;
+		ret = -ENODEV;
+		goto npe_release;
 	}
 	/* buffer_pool will also be used to sometimes store the hmac,
 	 * so assure it is large enough
@@ -512,6 +512,7 @@ npe_error:
 err:
 	dma_pool_destroy(ctx_pool);
 	dma_pool_destroy(buffer_pool);
+npe_release:
 	npe_release(npe_c);
 	return ret;
 }
diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c
index d64af8625d7e..37dadb2a4feb 100644
--- a/drivers/crypto/marvell/cesa.c
+++ b/drivers/crypto/marvell/cesa.c
@@ -166,6 +166,7 @@ static irqreturn_t mv_cesa_int(int irq, void *priv)
 			if (!req)
 				break;
 
+			ctx = crypto_tfm_ctx(req->tfm);
 			mv_cesa_complete_req(ctx, req, 0);
 		}
 	}
diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c
index 82e0f4e6eb1c..9f284682c091 100644
--- a/drivers/crypto/marvell/hash.c
+++ b/drivers/crypto/marvell/hash.c
@@ -374,7 +374,7 @@ static const struct mv_cesa_req_ops mv_cesa_ahash_req_ops = {
 	.complete = mv_cesa_ahash_complete,
 };
 
-static int mv_cesa_ahash_init(struct ahash_request *req,
+static void mv_cesa_ahash_init(struct ahash_request *req,
 			      struct mv_cesa_op_ctx *tmpl, bool algo_le)
 {
 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);
@@ -390,8 +390,6 @@ static int mv_cesa_ahash_init(struct ahash_request *req,
 	creq->op_tmpl = *tmpl;
 	creq->len = 0;
 	creq->algo_le = algo_le;
-
-	return 0;
 }
 
 static inline int mv_cesa_ahash_cra_init(struct crypto_tfm *tfm)
@@ -405,15 +403,16 @@ static inline int mv_cesa_ahash_cra_init(struct crypto_tfm *tfm)
 	return 0;
 }
 
-static int mv_cesa_ahash_cache_req(struct ahash_request *req, bool *cached)
+static bool mv_cesa_ahash_cache_req(struct ahash_request *req)
 {
 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);
+	bool cached = false;
 
-	if (creq->cache_ptr + req->nbytes < 64 && !creq->last_req) {
-		*cached = true;
+	if (creq->cache_ptr + req->nbytes < CESA_MAX_HASH_BLOCK_SIZE && !creq->last_req) {
+		cached = true;
 
 		if (!req->nbytes)
-			return 0;
+			return cached;
 
 		sg_pcopy_to_buffer(req->src, creq->src_nents,
 				   creq->cache + creq->cache_ptr,
@@ -422,7 +421,7 @@ static int mv_cesa_ahash_cache_req(struct ahash_request *req, bool *cached)
 		creq->cache_ptr += req->nbytes;
 	}
 
-	return 0;
+	return cached;
 }
 
 static struct mv_cesa_op_ctx *
@@ -455,7 +454,6 @@ mv_cesa_dma_add_frag(struct mv_cesa_tdma_chain *chain,
 
 static int
 mv_cesa_ahash_dma_add_cache(struct mv_cesa_tdma_chain *chain,
-			    struct mv_cesa_ahash_dma_iter *dma_iter,
 			    struct mv_cesa_ahash_req *creq,
 			    gfp_t flags)
 {
@@ -586,7 +584,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req)
 	 * Add the cache (left-over data from a previous block) first.
 	 * This will never overflow the SRAM size.
 	 */
-	ret = mv_cesa_ahash_dma_add_cache(&basereq->chain, &iter, creq, flags);
+	ret = mv_cesa_ahash_dma_add_cache(&basereq->chain, creq, flags);
 	if (ret)
 		goto err_free_tdma;
 
@@ -668,7 +666,6 @@ err:
 static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached)
 {
 	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);
-	int ret;
 
 	creq->src_nents = sg_nents_for_len(req->src, req->nbytes);
 	if (creq->src_nents < 0) {
@@ -676,17 +673,15 @@ static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached)
 		return creq->src_nents;
 	}
 
-	ret = mv_cesa_ahash_cache_req(req, cached);
-	if (ret)
-		return ret;
+	*cached = mv_cesa_ahash_cache_req(req);
 
 	if (*cached)
 		return 0;
 
 	if (cesa_dev->caps->has_tdma)
-		ret = mv_cesa_ahash_dma_req_init(req);
-
-	return ret;
+		return mv_cesa_ahash_dma_req_init(req);
+	else
+		return 0;
 }
 
 static int mv_cesa_ahash_queue_req(struct ahash_request *req)
@@ -805,13 +800,14 @@ static int mv_cesa_md5_init(struct ahash_request *req)
 	struct mv_cesa_op_ctx tmpl = { };
 
 	mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_MD5);
+
+	mv_cesa_ahash_init(req, &tmpl, true);
+
 	creq->state[0] = MD5_H0;
 	creq->state[1] = MD5_H1;
 	creq->state[2] = MD5_H2;
 	creq->state[3] = MD5_H3;
 
-	mv_cesa_ahash_init(req, &tmpl, true);
-
 	return 0;
 }
 
@@ -873,14 +869,15 @@ static int mv_cesa_sha1_init(struct ahash_request *req)
 	struct mv_cesa_op_ctx tmpl = { };
 
 	mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA1);
+
+	mv_cesa_ahash_init(req, &tmpl, false);
+
 	creq->state[0] = SHA1_H0;
 	creq->state[1] = SHA1_H1;
 	creq->state[2] = SHA1_H2;
 	creq->state[3] = SHA1_H3;
 	creq->state[4] = SHA1_H4;
 
-	mv_cesa_ahash_init(req, &tmpl, false);
-
 	return 0;
 }
 
@@ -942,6 +939,9 @@ static int mv_cesa_sha256_init(struct ahash_request *req)
 	struct mv_cesa_op_ctx tmpl = { };
 
 	mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA256);
+
+	mv_cesa_ahash_init(req, &tmpl, false);
+
 	creq->state[0] = SHA256_H0;
 	creq->state[1] = SHA256_H1;
 	creq->state[2] = SHA256_H2;
@@ -951,8 +951,6 @@ static int mv_cesa_sha256_init(struct ahash_request *req)
 	creq->state[6] = SHA256_H6;
 	creq->state[7] = SHA256_H7;
 
-	mv_cesa_ahash_init(req, &tmpl, false);
-
 	return 0;
 }
 
diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c
index 86a065bcc187..9fd7a5fbaa1b 100644
--- a/drivers/crypto/marvell/tdma.c
+++ b/drivers/crypto/marvell/tdma.c
@@ -261,6 +261,7 @@ struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain,
 	tdma->op = op;
 	tdma->byte_cnt = cpu_to_le32(size | BIT(31));
 	tdma->src = cpu_to_le32(dma_handle);
+	tdma->dst = CESA_SA_CFG_SRAM_OFFSET;
 	tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP;
 
 	return op;
diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c
index e6b658faef63..104e9ce9400a 100644
--- a/drivers/crypto/mv_cesa.c
+++ b/drivers/crypto/mv_cesa.c
@@ -1091,11 +1091,8 @@ static int mv_probe(struct platform_device *pdev)
 
 	cp->max_req_size = cp->sram_size - SRAM_CFG_SPACE;
 
-	if (pdev->dev.of_node)
-		irq = irq_of_parse_and_map(pdev->dev.of_node, 0);
-	else
-		irq = platform_get_irq(pdev, 0);
-	if (irq < 0 || irq == NO_IRQ) {
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
 		ret = irq;
 		goto err;
 	}
diff --git a/drivers/crypto/mxc-scc.c b/drivers/crypto/mxc-scc.c
index ff383ef83871..ee4be1b0d30b 100644
--- a/drivers/crypto/mxc-scc.c
+++ b/drivers/crypto/mxc-scc.c
@@ -668,7 +668,9 @@ static int mxc_scc_probe(struct platform_device *pdev)
 		return PTR_ERR(scc->clk);
 	}
 
-	clk_prepare_enable(scc->clk);
+	ret = clk_prepare_enable(scc->clk);
+	if (ret)
+		return ret;
 
 	/* clear error status register */
 	writel(0x0, scc->base + SCC_SCM_ERROR_STATUS);
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 4ab53a604312..fe32dd95ae4f 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -35,7 +35,8 @@
 #include <linux/interrupt.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/aes.h>
-#include <crypto/algapi.h>
+#include <crypto/engine.h>
+#include <crypto/internal/skcipher.h>
 
 #define DST_MAXBURST			4
 #define DMA_MIN				(DST_MAXBURST * sizeof(u32))
@@ -85,6 +86,8 @@
 #define AES_REG_IRQ_DATA_OUT           BIT(2)
 #define DEFAULT_TIMEOUT		(5*HZ)
 
+#define DEFAULT_AUTOSUSPEND_DELAY	1000
+
 #define FLAGS_MODE_MASK		0x000f
 #define FLAGS_ENCRYPT		BIT(0)
 #define FLAGS_CBC		BIT(1)
@@ -103,6 +106,7 @@ struct omap_aes_ctx {
 	int		keylen;
 	u32		key[AES_KEYSIZE_256 / sizeof(u32)];
 	unsigned long	flags;
+	struct crypto_skcipher	*fallback;
 };
 
 struct omap_aes_reqctx {
@@ -238,11 +242,19 @@ static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
 
 static int omap_aes_hw_init(struct omap_aes_dev *dd)
 {
+	int err;
+
 	if (!(dd->flags & FLAGS_INIT)) {
 		dd->flags |= FLAGS_INIT;
 		dd->err = 0;
 	}
 
+	err = pm_runtime_get_sync(dd->dev);
+	if (err < 0) {
+		dev_err(dd->dev, "failed to get sync: %d\n", err);
+		return err;
+	}
+
 	return 0;
 }
 
@@ -319,20 +331,12 @@ static void omap_aes_dma_stop(struct omap_aes_dev *dd)
 
 static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
 {
-	struct omap_aes_dev *dd = NULL, *tmp;
+	struct omap_aes_dev *dd;
 
 	spin_lock_bh(&list_lock);
-	if (!ctx->dd) {
-		list_for_each_entry(tmp, &dev_list, list) {
-			/* FIXME: take fist available aes core */
-			dd = tmp;
-			break;
-		}
-		ctx->dd = dd;
-	} else {
-		/* already found before */
-		dd = ctx->dd;
-	}
+	dd = list_first_entry(&dev_list, struct omap_aes_dev, list);
+	list_move_tail(&dd->list, &dev_list);
+	ctx->dd = dd;
 	spin_unlock_bh(&list_lock);
 
 	return dd;
@@ -519,7 +523,10 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
 
 	pr_debug("err: %d\n", err);
 
-	crypto_finalize_request(dd->engine, req, err);
+	crypto_finalize_cipher_request(dd->engine, req, err);
+
+	pm_runtime_mark_last_busy(dd->dev);
+	pm_runtime_put_autosuspend(dd->dev);
 }
 
 static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
@@ -592,7 +599,7 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd,
 				 struct ablkcipher_request *req)
 {
 	if (req)
-		return crypto_transfer_request_to_engine(dd->engine, req);
+		return crypto_transfer_cipher_request_to_engine(dd->engine, req);
 
 	return 0;
 }
@@ -602,7 +609,7 @@ static int omap_aes_prepare_req(struct crypto_engine *engine,
 {
 	struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
 			crypto_ablkcipher_reqtfm(req));
-	struct omap_aes_dev *dd = omap_aes_find_dev(ctx);
+	struct omap_aes_dev *dd = ctx->dd;
 	struct omap_aes_reqctx *rctx;
 
 	if (!dd)
@@ -648,7 +655,7 @@ static int omap_aes_crypt_req(struct crypto_engine *engine,
 {
 	struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
 			crypto_ablkcipher_reqtfm(req));
-	struct omap_aes_dev *dd = omap_aes_find_dev(ctx);
+	struct omap_aes_dev *dd = ctx->dd;
 
 	if (!dd)
 		return -ENODEV;
@@ -696,11 +703,29 @@ static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
 			crypto_ablkcipher_reqtfm(req));
 	struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
 	struct omap_aes_dev *dd;
+	int ret;
 
 	pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
 		  !!(mode & FLAGS_ENCRYPT),
 		  !!(mode & FLAGS_CBC));
 
+	if (req->nbytes < 200) {
+		SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+
+		skcipher_request_set_tfm(subreq, ctx->fallback);
+		skcipher_request_set_callback(subreq, req->base.flags, NULL,
+					      NULL);
+		skcipher_request_set_crypt(subreq, req->src, req->dst,
+					   req->nbytes, req->info);
+
+		if (mode & FLAGS_ENCRYPT)
+			ret = crypto_skcipher_encrypt(subreq);
+		else
+			ret = crypto_skcipher_decrypt(subreq);
+
+		skcipher_request_zero(subreq);
+		return ret;
+	}
 	dd = omap_aes_find_dev(ctx);
 	if (!dd)
 		return -ENODEV;
@@ -716,6 +741,7 @@ static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 			   unsigned int keylen)
 {
 	struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+	int ret;
 
 	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 		   keylen != AES_KEYSIZE_256)
@@ -726,6 +752,14 @@ static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 	memcpy(ctx->key, key, keylen);
 	ctx->keylen = keylen;
 
+	crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
+						 CRYPTO_TFM_REQ_MASK);
+
+	ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
+	if (!ret)
+		return 0;
+
 	return 0;
 }
 
@@ -761,22 +795,16 @@ static int omap_aes_ctr_decrypt(struct ablkcipher_request *req)
 
 static int omap_aes_cra_init(struct crypto_tfm *tfm)
 {
-	struct omap_aes_dev *dd = NULL;
-	int err;
+	const char *name = crypto_tfm_alg_name(tfm);
+	const u32 flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
+	struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_skcipher *blk;
 
-	/* Find AES device, currently picks the first device */
-	spin_lock_bh(&list_lock);
-	list_for_each_entry(dd, &dev_list, list) {
-		break;
-	}
-	spin_unlock_bh(&list_lock);
+	blk = crypto_alloc_skcipher(name, 0, flags);
+	if (IS_ERR(blk))
+		return PTR_ERR(blk);
 
-	err = pm_runtime_get_sync(dd->dev);
-	if (err < 0) {
-		dev_err(dd->dev, "%s: failed to get_sync(%d)\n",
-			__func__, err);
-		return err;
-	}
+	ctx->fallback = blk;
 
 	tfm->crt_ablkcipher.reqsize = sizeof(struct omap_aes_reqctx);
 
@@ -785,16 +813,12 @@ static int omap_aes_cra_init(struct crypto_tfm *tfm)
 
 static void omap_aes_cra_exit(struct crypto_tfm *tfm)
 {
-	struct omap_aes_dev *dd = NULL;
+	struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	/* Find AES device, currently picks the first device */
-	spin_lock_bh(&list_lock);
-	list_for_each_entry(dd, &dev_list, list) {
-		break;
-	}
-	spin_unlock_bh(&list_lock);
+	if (ctx->fallback)
+		crypto_free_skcipher(ctx->fallback);
 
-	pm_runtime_put_sync(dd->dev);
+	ctx->fallback = NULL;
 }
 
 /* ********************** ALGS ************************************ */
@@ -806,7 +830,7 @@ static struct crypto_alg algs_ecb_cbc[] = {
 	.cra_priority		= 300,
 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
 				  CRYPTO_ALG_KERN_DRIVER_ONLY |
-				  CRYPTO_ALG_ASYNC,
+				  CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
 	.cra_blocksize		= AES_BLOCK_SIZE,
 	.cra_ctxsize		= sizeof(struct omap_aes_ctx),
 	.cra_alignmask		= 0,
@@ -828,7 +852,7 @@ static struct crypto_alg algs_ecb_cbc[] = {
 	.cra_priority		= 300,
 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
 				  CRYPTO_ALG_KERN_DRIVER_ONLY |
-				  CRYPTO_ALG_ASYNC,
+				  CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
 	.cra_blocksize		= AES_BLOCK_SIZE,
 	.cra_ctxsize		= sizeof(struct omap_aes_ctx),
 	.cra_alignmask		= 0,
@@ -854,7 +878,7 @@ static struct crypto_alg algs_ctr[] = {
 	.cra_priority		= 300,
 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
 				  CRYPTO_ALG_KERN_DRIVER_ONLY |
-				  CRYPTO_ALG_ASYNC,
+				  CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
 	.cra_blocksize		= AES_BLOCK_SIZE,
 	.cra_ctxsize		= sizeof(struct omap_aes_ctx),
 	.cra_alignmask		= 0,
@@ -1140,6 +1164,9 @@ static int omap_aes_probe(struct platform_device *pdev)
 	}
 	dd->phys_base = res.start;
 
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
+
 	pm_runtime_enable(dev);
 	err = pm_runtime_get_sync(dev);
 	if (err < 0) {
@@ -1186,6 +1213,19 @@ static int omap_aes_probe(struct platform_device *pdev)
 	list_add_tail(&dd->list, &dev_list);
 	spin_unlock(&list_lock);
 
+	/* Initialize crypto engine */
+	dd->engine = crypto_engine_alloc_init(dev, 1);
+	if (!dd->engine) {
+		err = -ENOMEM;
+		goto err_engine;
+	}
+
+	dd->engine->prepare_cipher_request = omap_aes_prepare_req;
+	dd->engine->cipher_one_request = omap_aes_crypt_req;
+	err = crypto_engine_start(dd->engine);
+	if (err)
+		goto err_engine;
+
 	for (i = 0; i < dd->pdata->algs_info_size; i++) {
 		if (!dd->pdata->algs_info[i].registered) {
 			for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
@@ -1203,26 +1243,17 @@ static int omap_aes_probe(struct platform_device *pdev)
 		}
 	}
 
-	/* Initialize crypto engine */
-	dd->engine = crypto_engine_alloc_init(dev, 1);
-	if (!dd->engine)
-		goto err_algs;
-
-	dd->engine->prepare_request = omap_aes_prepare_req;
-	dd->engine->crypt_one_request = omap_aes_crypt_req;
-	err = crypto_engine_start(dd->engine);
-	if (err)
-		goto err_engine;
-
 	return 0;
-err_engine:
-	crypto_engine_exit(dd->engine);
 err_algs:
 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
 			crypto_unregister_alg(
 					&dd->pdata->algs_info[i].algs_list[j]);
 
+err_engine:
+	if (dd->engine)
+		crypto_engine_exit(dd->engine);
+
 	omap_aes_dma_cleanup(dd);
 err_irq:
 	tasklet_kill(&dd->done_task);
diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c
index 5691434ffb2d..a6f65532fd16 100644
--- a/drivers/crypto/omap-des.c
+++ b/drivers/crypto/omap-des.c
@@ -39,6 +39,7 @@
 #include <crypto/scatterwalk.h>
 #include <crypto/des.h>
 #include <crypto/algapi.h>
+#include <crypto/engine.h>
 
 #define DST_MAXBURST			2
 
@@ -506,7 +507,7 @@ static void omap_des_finish_req(struct omap_des_dev *dd, int err)
 	pr_debug("err: %d\n", err);
 
 	pm_runtime_put(dd->dev);
-	crypto_finalize_request(dd->engine, req, err);
+	crypto_finalize_cipher_request(dd->engine, req, err);
 }
 
 static int omap_des_crypt_dma_stop(struct omap_des_dev *dd)
@@ -574,7 +575,7 @@ static int omap_des_handle_queue(struct omap_des_dev *dd,
 				 struct ablkcipher_request *req)
 {
 	if (req)
-		return crypto_transfer_request_to_engine(dd->engine, req);
+		return crypto_transfer_cipher_request_to_engine(dd->engine, req);
 
 	return 0;
 }
@@ -1078,6 +1079,19 @@ static int omap_des_probe(struct platform_device *pdev)
 	list_add_tail(&dd->list, &dev_list);
 	spin_unlock(&list_lock);
 
+	/* Initialize des crypto engine */
+	dd->engine = crypto_engine_alloc_init(dev, 1);
+	if (!dd->engine) {
+		err = -ENOMEM;
+		goto err_engine;
+	}
+
+	dd->engine->prepare_cipher_request = omap_des_prepare_req;
+	dd->engine->cipher_one_request = omap_des_crypt_req;
+	err = crypto_engine_start(dd->engine);
+	if (err)
+		goto err_engine;
+
 	for (i = 0; i < dd->pdata->algs_info_size; i++) {
 		for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
 			algp = &dd->pdata->algs_info[i].algs_list[j];
@@ -1093,27 +1107,18 @@ static int omap_des_probe(struct platform_device *pdev)
 		}
 	}
 
-	/* Initialize des crypto engine */
-	dd->engine = crypto_engine_alloc_init(dev, 1);
-	if (!dd->engine)
-		goto err_algs;
-
-	dd->engine->prepare_request = omap_des_prepare_req;
-	dd->engine->crypt_one_request = omap_des_crypt_req;
-	err = crypto_engine_start(dd->engine);
-	if (err)
-		goto err_engine;
-
 	return 0;
 
-err_engine:
-	crypto_engine_exit(dd->engine);
 err_algs:
 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
 			crypto_unregister_alg(
 					&dd->pdata->algs_info[i].algs_list[j]);
 
+err_engine:
+	if (dd->engine)
+		crypto_engine_exit(dd->engine);
+
 	omap_des_dma_cleanup(dd);
 err_irq:
 	tasklet_kill(&dd->done_task);
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 7fe4eef12fe2..d0b16e5e4ee5 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -112,9 +112,10 @@
 #define FLAGS_DMA_READY		6
 #define FLAGS_AUTO_XOR		7
 #define FLAGS_BE32_SHA1		8
+#define FLAGS_SGS_COPIED	9
+#define FLAGS_SGS_ALLOCED	10
 /* context flags */
 #define FLAGS_FINUP		16
-#define FLAGS_SG		17
 
 #define FLAGS_MODE_SHIFT	18
 #define FLAGS_MODE_MASK		(SHA_REG_MODE_ALGO_MASK	<< FLAGS_MODE_SHIFT)
@@ -134,7 +135,8 @@
 #define OMAP_ALIGN_MASK		(sizeof(u32)-1)
 #define OMAP_ALIGNED		__attribute__((aligned(sizeof(u32))))
 
-#define BUFLEN			PAGE_SIZE
+#define BUFLEN			SHA512_BLOCK_SIZE
+#define OMAP_SHA_DMA_THRESHOLD	256
 
 struct omap_sham_dev;
 
@@ -147,12 +149,12 @@ struct omap_sham_reqctx {
 	size_t			digcnt;
 	size_t			bufcnt;
 	size_t			buflen;
-	dma_addr_t		dma_addr;
 
 	/* walk state */
 	struct scatterlist	*sg;
-	struct scatterlist	sgl;
-	unsigned int		offset;	/* offset in current sg */
+	struct scatterlist	sgl[2];
+	int			offset;	/* offset in current sg */
+	int			sg_len;
 	unsigned int		total;	/* total request */
 
 	u8			buffer[0] OMAP_ALIGNED;
@@ -223,6 +225,7 @@ struct omap_sham_dev {
 	struct dma_chan		*dma_lch;
 	struct tasklet_struct	done_task;
 	u8			polling_mode;
+	u8			xmit_buf[BUFLEN];
 
 	unsigned long		flags;
 	struct crypto_queue	queue;
@@ -510,12 +513,14 @@ static int omap_sham_poll_irq_omap4(struct omap_sham_dev *dd)
 			      SHA_REG_IRQSTATUS_INPUT_RDY);
 }
 
-static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
-			      size_t length, int final)
+static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, size_t length,
+			      int final)
 {
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 	int count, len32, bs32, offset = 0;
-	const u32 *buffer = (const u32 *)buf;
+	const u32 *buffer;
+	int mlen;
+	struct sg_mapping_iter mi;
 
 	dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
 						ctx->digcnt, length, final);
@@ -525,6 +530,7 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
 
 	/* should be non-zero before next lines to disable clocks later */
 	ctx->digcnt += length;
+	ctx->total -= length;
 
 	if (final)
 		set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
@@ -534,16 +540,35 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
 	len32 = DIV_ROUND_UP(length, sizeof(u32));
 	bs32 = get_block_size(ctx) / sizeof(u32);
 
+	sg_miter_start(&mi, ctx->sg, ctx->sg_len,
+		       SG_MITER_FROM_SG | SG_MITER_ATOMIC);
+
+	mlen = 0;
+
 	while (len32) {
 		if (dd->pdata->poll_irq(dd))
 			return -ETIMEDOUT;
 
-		for (count = 0; count < min(len32, bs32); count++, offset++)
+		for (count = 0; count < min(len32, bs32); count++, offset++) {
+			if (!mlen) {
+				sg_miter_next(&mi);
+				mlen = mi.length;
+				if (!mlen) {
+					pr_err("sg miter failure.\n");
+					return -EINVAL;
+				}
+				offset = 0;
+				buffer = mi.addr;
+			}
 			omap_sham_write(dd, SHA_REG_DIN(dd, count),
 					buffer[offset]);
+			mlen -= 4;
+		}
 		len32 -= min(len32, bs32);
 	}
 
+	sg_miter_stop(&mi);
+
 	return -EINPROGRESS;
 }
 
@@ -555,22 +580,27 @@ static void omap_sham_dma_callback(void *param)
 	tasklet_schedule(&dd->done_task);
 }
 
-static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
-			      size_t length, int final, int is_sg)
+static int omap_sham_xmit_dma(struct omap_sham_dev *dd, size_t length,
+			      int final)
 {
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 	struct dma_async_tx_descriptor *tx;
 	struct dma_slave_config cfg;
-	int len32, ret, dma_min = get_block_size(ctx);
+	int ret;
 
 	dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
 						ctx->digcnt, length, final);
 
+	if (!dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE)) {
+		dev_err(dd->dev, "dma_map_sg error\n");
+		return -EINVAL;
+	}
+
 	memset(&cfg, 0, sizeof(cfg));
 
 	cfg.dst_addr = dd->phys_base + SHA_REG_DIN(dd, 0);
 	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
-	cfg.dst_maxburst = dma_min / DMA_SLAVE_BUSWIDTH_4_BYTES;
+	cfg.dst_maxburst = get_block_size(ctx) / DMA_SLAVE_BUSWIDTH_4_BYTES;
 
 	ret = dmaengine_slave_config(dd->dma_lch, &cfg);
 	if (ret) {
@@ -578,30 +608,12 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
 		return ret;
 	}
 
-	len32 = DIV_ROUND_UP(length, dma_min) * dma_min;
-
-	if (is_sg) {
-		/*
-		 * The SG entry passed in may not have the 'length' member
-		 * set correctly so use a local SG entry (sgl) with the
-		 * proper value for 'length' instead.  If this is not done,
-		 * the dmaengine may try to DMA the incorrect amount of data.
-		 */
-		sg_init_table(&ctx->sgl, 1);
-		sg_assign_page(&ctx->sgl, sg_page(ctx->sg));
-		ctx->sgl.offset = ctx->sg->offset;
-		sg_dma_len(&ctx->sgl) = len32;
-		sg_dma_address(&ctx->sgl) = sg_dma_address(ctx->sg);
-
-		tx = dmaengine_prep_slave_sg(dd->dma_lch, &ctx->sgl, 1,
-			DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
-	} else {
-		tx = dmaengine_prep_slave_single(dd->dma_lch, dma_addr, len32,
-			DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
-	}
+	tx = dmaengine_prep_slave_sg(dd->dma_lch, ctx->sg, ctx->sg_len,
+				     DMA_MEM_TO_DEV,
+				     DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 
 	if (!tx) {
-		dev_err(dd->dev, "prep_slave_sg/single() failed\n");
+		dev_err(dd->dev, "prep_slave_sg failed\n");
 		return -EINVAL;
 	}
 
@@ -611,6 +623,7 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
 	dd->pdata->write_ctrl(dd, length, final, 1);
 
 	ctx->digcnt += length;
+	ctx->total -= length;
 
 	if (final)
 		set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
@@ -625,189 +638,257 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
 	return -EINPROGRESS;
 }
 
-static size_t omap_sham_append_buffer(struct omap_sham_reqctx *ctx,
-				const u8 *data, size_t length)
+static int omap_sham_copy_sg_lists(struct omap_sham_reqctx *ctx,
+				   struct scatterlist *sg, int bs, int new_len)
 {
-	size_t count = min(length, ctx->buflen - ctx->bufcnt);
+	int n = sg_nents(sg);
+	struct scatterlist *tmp;
+	int offset = ctx->offset;
 
-	count = min(count, ctx->total);
-	if (count <= 0)
-		return 0;
-	memcpy(ctx->buffer + ctx->bufcnt, data, count);
-	ctx->bufcnt += count;
+	if (ctx->bufcnt)
+		n++;
 
-	return count;
-}
+	ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);
+	if (!ctx->sg)
+		return -ENOMEM;
 
-static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx)
-{
-	size_t count;
-	const u8 *vaddr;
+	sg_init_table(ctx->sg, n);
 
-	while (ctx->sg) {
-		vaddr = kmap_atomic(sg_page(ctx->sg));
-		vaddr += ctx->sg->offset;
+	tmp = ctx->sg;
 
-		count = omap_sham_append_buffer(ctx,
-				vaddr + ctx->offset,
-				ctx->sg->length - ctx->offset);
+	ctx->sg_len = 0;
 
-		kunmap_atomic((void *)vaddr);
+	if (ctx->bufcnt) {
+		sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt);
+		tmp = sg_next(tmp);
+		ctx->sg_len++;
+	}
 
-		if (!count)
-			break;
-		ctx->offset += count;
-		ctx->total -= count;
-		if (ctx->offset == ctx->sg->length) {
-			ctx->sg = sg_next(ctx->sg);
-			if (ctx->sg)
-				ctx->offset = 0;
-			else
-				ctx->total = 0;
+	while (sg && new_len) {
+		int len = sg->length - offset;
+
+		if (offset) {
+			offset -= sg->length;
+			if (offset < 0)
+				offset = 0;
+		}
+
+		if (new_len < len)
+			len = new_len;
+
+		if (len > 0) {
+			new_len -= len;
+			sg_set_page(tmp, sg_page(sg), len, sg->offset);
+			if (new_len <= 0)
+				sg_mark_end(tmp);
+			tmp = sg_next(tmp);
+			ctx->sg_len++;
 		}
+
+		sg = sg_next(sg);
 	}
 
+	set_bit(FLAGS_SGS_ALLOCED, &ctx->dd->flags);
+
+	ctx->bufcnt = 0;
+
 	return 0;
 }
 
-static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd,
-					struct omap_sham_reqctx *ctx,
-					size_t length, int final)
+static int omap_sham_copy_sgs(struct omap_sham_reqctx *ctx,
+			      struct scatterlist *sg, int bs, int new_len)
 {
-	int ret;
+	int pages;
+	void *buf;
+	int len;
 
-	ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
-				       DMA_TO_DEVICE);
-	if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
-		dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
-		return -EINVAL;
+	len = new_len + ctx->bufcnt;
+
+	pages = get_order(ctx->total);
+
+	buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
+	if (!buf) {
+		pr_err("Couldn't allocate pages for unaligned cases.\n");
+		return -ENOMEM;
 	}
 
-	ctx->flags &= ~BIT(FLAGS_SG);
+	if (ctx->bufcnt)
+		memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);
 
-	ret = omap_sham_xmit_dma(dd, ctx->dma_addr, length, final, 0);
-	if (ret != -EINPROGRESS)
-		dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
-				 DMA_TO_DEVICE);
+	scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->offset,
+				 ctx->total - ctx->bufcnt, 0);
+	sg_init_table(ctx->sgl, 1);
+	sg_set_buf(ctx->sgl, buf, len);
+	ctx->sg = ctx->sgl;
+	set_bit(FLAGS_SGS_COPIED, &ctx->dd->flags);
+	ctx->sg_len = 1;
+	ctx->bufcnt = 0;
+	ctx->offset = 0;
 
-	return ret;
+	return 0;
 }
 
-static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
+static int omap_sham_align_sgs(struct scatterlist *sg,
+			       int nbytes, int bs, bool final,
+			       struct omap_sham_reqctx *rctx)
 {
-	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
-	unsigned int final;
-	size_t count;
+	int n = 0;
+	bool aligned = true;
+	bool list_ok = true;
+	struct scatterlist *sg_tmp = sg;
+	int new_len;
+	int offset = rctx->offset;
 
-	omap_sham_append_sg(ctx);
+	if (!sg || !sg->length || !nbytes)
+		return 0;
+
+	new_len = nbytes;
 
-	final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
+	if (offset)
+		list_ok = false;
 
-	dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
-					 ctx->bufcnt, ctx->digcnt, final);
+	if (final)
+		new_len = DIV_ROUND_UP(new_len, bs) * bs;
+	else
+		new_len = new_len / bs * bs;
 
-	if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
-		count = ctx->bufcnt;
-		ctx->bufcnt = 0;
-		return omap_sham_xmit_dma_map(dd, ctx, count, final);
+	while (nbytes > 0 && sg_tmp) {
+		n++;
+
+		if (offset < sg_tmp->length) {
+			if (!IS_ALIGNED(offset + sg_tmp->offset, 4)) {
+				aligned = false;
+				break;
+			}
+
+			if (!IS_ALIGNED(sg_tmp->length - offset, bs)) {
+				aligned = false;
+				break;
+			}
+		}
+
+		if (offset) {
+			offset -= sg_tmp->length;
+			if (offset < 0) {
+				nbytes += offset;
+				offset = 0;
+			}
+		} else {
+			nbytes -= sg_tmp->length;
+		}
+
+		sg_tmp = sg_next(sg_tmp);
+
+		if (nbytes < 0) {
+			list_ok = false;
+			break;
+		}
 	}
 
+	if (!aligned)
+		return omap_sham_copy_sgs(rctx, sg, bs, new_len);
+	else if (!list_ok)
+		return omap_sham_copy_sg_lists(rctx, sg, bs, new_len);
+
+	rctx->sg_len = n;
+	rctx->sg = sg;
+
 	return 0;
 }
 
-/* Start address alignment */
-#define SG_AA(sg)	(IS_ALIGNED(sg->offset, sizeof(u32)))
-/* SHA1 block size alignment */
-#define SG_SA(sg, bs)	(IS_ALIGNED(sg->length, bs))
-
-static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
+static int omap_sham_prepare_request(struct ahash_request *req, bool update)
 {
-	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
-	unsigned int length, final, tail;
-	struct scatterlist *sg;
-	int ret, bs;
+	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
+	int bs;
+	int ret;
+	int nbytes;
+	bool final = rctx->flags & BIT(FLAGS_FINUP);
+	int xmit_len, hash_later;
 
-	if (!ctx->total)
+	if (!req)
 		return 0;
 
-	if (ctx->bufcnt || ctx->offset)
-		return omap_sham_update_dma_slow(dd);
-
-	/*
-	 * Don't use the sg interface when the transfer size is less
-	 * than the number of elements in a DMA frame.  Otherwise,
-	 * the dmaengine infrastructure will calculate that it needs
-	 * to transfer 0 frames which ultimately fails.
-	 */
-	if (ctx->total < get_block_size(ctx))
-		return omap_sham_update_dma_slow(dd);
-
-	dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
-			ctx->digcnt, ctx->bufcnt, ctx->total);
+	bs = get_block_size(rctx);
 
-	sg = ctx->sg;
-	bs = get_block_size(ctx);
+	if (update)
+		nbytes = req->nbytes;
+	else
+		nbytes = 0;
 
-	if (!SG_AA(sg))
-		return omap_sham_update_dma_slow(dd);
+	rctx->total = nbytes + rctx->bufcnt;
 
-	if (!sg_is_last(sg) && !SG_SA(sg, bs))
-		/* size is not BLOCK_SIZE aligned */
-		return omap_sham_update_dma_slow(dd);
+	if (!rctx->total)
+		return 0;
 
-	length = min(ctx->total, sg->length);
+	if (nbytes && (!IS_ALIGNED(rctx->bufcnt, bs))) {
+		int len = bs - rctx->bufcnt % bs;
 
-	if (sg_is_last(sg)) {
-		if (!(ctx->flags & BIT(FLAGS_FINUP))) {
-			/* not last sg must be BLOCK_SIZE aligned */
-			tail = length & (bs - 1);
-			/* without finup() we need one block to close hash */
-			if (!tail)
-				tail = bs;
-			length -= tail;
-		}
+		if (len > nbytes)
+			len = nbytes;
+		scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, req->src,
+					 0, len, 0);
+		rctx->bufcnt += len;
+		nbytes -= len;
+		rctx->offset = len;
 	}
 
-	if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
-		dev_err(dd->dev, "dma_map_sg  error\n");
-		return -EINVAL;
-	}
+	if (rctx->bufcnt)
+		memcpy(rctx->dd->xmit_buf, rctx->buffer, rctx->bufcnt);
 
-	ctx->flags |= BIT(FLAGS_SG);
+	ret = omap_sham_align_sgs(req->src, nbytes, bs, final, rctx);
+	if (ret)
+		return ret;
 
-	ctx->total -= length;
-	ctx->offset = length; /* offset where to start slow */
+	xmit_len = rctx->total;
 
-	final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
+	if (!IS_ALIGNED(xmit_len, bs)) {
+		if (final)
+			xmit_len = DIV_ROUND_UP(xmit_len, bs) * bs;
+		else
+			xmit_len = xmit_len / bs * bs;
+	}
 
-	ret = omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final, 1);
-	if (ret != -EINPROGRESS)
-		dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
+	hash_later = rctx->total - xmit_len;
+	if (hash_later < 0)
+		hash_later = 0;
 
-	return ret;
-}
+	if (rctx->bufcnt && nbytes) {
+		/* have data from previous operation and current */
+		sg_init_table(rctx->sgl, 2);
+		sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, rctx->bufcnt);
 
-static int omap_sham_update_cpu(struct omap_sham_dev *dd)
-{
-	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
-	int bufcnt, final;
+		sg_chain(rctx->sgl, 2, req->src);
 
-	if (!ctx->total)
-		return 0;
+		rctx->sg = rctx->sgl;
 
-	omap_sham_append_sg(ctx);
+		rctx->sg_len++;
+	} else if (rctx->bufcnt) {
+		/* have buffered data only */
+		sg_init_table(rctx->sgl, 1);
+		sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, xmit_len);
 
-	final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
+		rctx->sg = rctx->sgl;
 
-	dev_dbg(dd->dev, "cpu: bufcnt: %u, digcnt: %d, final: %d\n",
-		ctx->bufcnt, ctx->digcnt, final);
+		rctx->sg_len = 1;
+	}
 
-	if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
-		bufcnt = ctx->bufcnt;
-		ctx->bufcnt = 0;
-		return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, final);
+	if (hash_later) {
+		if (req->nbytes) {
+			scatterwalk_map_and_copy(rctx->buffer, req->src,
+						 req->nbytes - hash_later,
+						 hash_later, 0);
+		} else {
+			memcpy(rctx->buffer, rctx->buffer + xmit_len,
+			       hash_later);
+		}
+		rctx->bufcnt = hash_later;
+	} else {
+		rctx->bufcnt = 0;
 	}
 
+	if (!final)
+		rctx->total = xmit_len;
+
 	return 0;
 }
 
@@ -815,18 +896,9 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
 {
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 
+	dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
 
-	if (ctx->flags & BIT(FLAGS_SG)) {
-		dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
-		if (ctx->sg->length == ctx->offset) {
-			ctx->sg = sg_next(ctx->sg);
-			if (ctx->sg)
-				ctx->offset = 0;
-		}
-	} else {
-		dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
-				 DMA_TO_DEVICE);
-	}
+	clear_bit(FLAGS_DMA_ACTIVE, &dd->flags);
 
 	return 0;
 }
@@ -887,6 +959,8 @@ static int omap_sham_init(struct ahash_request *req)
 
 	ctx->bufcnt = 0;
 	ctx->digcnt = 0;
+	ctx->total = 0;
+	ctx->offset = 0;
 	ctx->buflen = BUFLEN;
 
 	if (tctx->flags & BIT(FLAGS_HMAC)) {
@@ -909,14 +983,19 @@ static int omap_sham_update_req(struct omap_sham_dev *dd)
 	struct ahash_request *req = dd->req;
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 	int err;
+	bool final = ctx->flags & BIT(FLAGS_FINUP);
 
 	dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n",
 		 ctx->total, ctx->digcnt, (ctx->flags & BIT(FLAGS_FINUP)) != 0);
 
+	if (ctx->total < get_block_size(ctx) ||
+	    ctx->total < OMAP_SHA_DMA_THRESHOLD)
+		ctx->flags |= BIT(FLAGS_CPU);
+
 	if (ctx->flags & BIT(FLAGS_CPU))
-		err = omap_sham_update_cpu(dd);
+		err = omap_sham_xmit_cpu(dd, ctx->total, final);
 	else
-		err = omap_sham_update_dma_start(dd);
+		err = omap_sham_xmit_dma(dd, ctx->total, final);
 
 	/* wait for dma completion before can take more data */
 	dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
@@ -930,7 +1009,7 @@ static int omap_sham_final_req(struct omap_sham_dev *dd)
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 	int err = 0, use_dma = 1;
 
-	if ((ctx->bufcnt <= get_block_size(ctx)) || dd->polling_mode)
+	if ((ctx->total <= get_block_size(ctx)) || dd->polling_mode)
 		/*
 		 * faster to handle last block with cpu or
 		 * use cpu when dma is not present.
@@ -938,9 +1017,9 @@ static int omap_sham_final_req(struct omap_sham_dev *dd)
 		use_dma = 0;
 
 	if (use_dma)
-		err = omap_sham_xmit_dma_map(dd, ctx, ctx->bufcnt, 1);
+		err = omap_sham_xmit_dma(dd, ctx->total, 1);
 	else
-		err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1);
+		err = omap_sham_xmit_cpu(dd, ctx->total, 1);
 
 	ctx->bufcnt = 0;
 
@@ -988,6 +1067,17 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 	struct omap_sham_dev *dd = ctx->dd;
 
+	if (test_bit(FLAGS_SGS_COPIED, &dd->flags))
+		free_pages((unsigned long)sg_virt(ctx->sg),
+			   get_order(ctx->sg->length));
+
+	if (test_bit(FLAGS_SGS_ALLOCED, &dd->flags))
+		kfree(ctx->sg);
+
+	ctx->sg = NULL;
+
+	dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED));
+
 	if (!err) {
 		dd->pdata->copy_hash(req, 1);
 		if (test_bit(FLAGS_FINAL, &dd->flags))
@@ -1005,9 +1095,6 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
 
 	if (req->base.complete)
 		req->base.complete(&req->base, err);
-
-	/* handle new request */
-	tasklet_schedule(&dd->done_task);
 }
 
 static int omap_sham_handle_queue(struct omap_sham_dev *dd,
@@ -1018,6 +1105,7 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd,
 	unsigned long flags;
 	int err = 0, ret = 0;
 
+retry:
 	spin_lock_irqsave(&dd->lock, flags);
 	if (req)
 		ret = ahash_enqueue_request(&dd->queue, req);
@@ -1041,6 +1129,10 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd,
 	dd->req = req;
 	ctx = ahash_request_ctx(req);
 
+	err = omap_sham_prepare_request(req, ctx->op == OP_UPDATE);
+	if (err)
+		goto err1;
+
 	dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
 						ctx->op, req->nbytes);
 
@@ -1061,11 +1153,19 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd,
 		err = omap_sham_final_req(dd);
 	}
 err1:
-	if (err != -EINPROGRESS)
+	dev_dbg(dd->dev, "exit, err: %d\n", err);
+
+	if (err != -EINPROGRESS) {
 		/* done_task will not finish it, so do it here */
 		omap_sham_finish_req(req, err);
+		req = NULL;
 
-	dev_dbg(dd->dev, "exit, err: %d\n", err);
+		/*
+		 * Execute next request immediately if there is anything
+		 * in queue.
+		 */
+		goto retry;
+	}
 
 	return ret;
 }
@@ -1085,34 +1185,15 @@ static int omap_sham_update(struct ahash_request *req)
 {
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 	struct omap_sham_dev *dd = ctx->dd;
-	int bs = get_block_size(ctx);
 
 	if (!req->nbytes)
 		return 0;
 
-	ctx->total = req->nbytes;
-	ctx->sg = req->src;
-	ctx->offset = 0;
-
-	if (ctx->flags & BIT(FLAGS_FINUP)) {
-		if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 240) {
-			/*
-			* OMAP HW accel works only with buffers >= 9
-			* will switch to bypass in final()
-			* final has the same request and data
-			*/
-			omap_sham_append_sg(ctx);
-			return 0;
-		} else if ((ctx->bufcnt + ctx->total <= bs) ||
-			   dd->polling_mode) {
-			/*
-			 * faster to use CPU for short transfers or
-			 * use cpu when dma is not present.
-			 */
-			ctx->flags |= BIT(FLAGS_CPU);
-		}
-	} else if (ctx->bufcnt + ctx->total < ctx->buflen) {
-		omap_sham_append_sg(ctx);
+	if (ctx->total + req->nbytes < ctx->buflen) {
+		scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
+					 0, req->nbytes, 0);
+		ctx->bufcnt += req->nbytes;
+		ctx->total += req->nbytes;
 		return 0;
 	}
 
@@ -1137,9 +1218,20 @@ static int omap_sham_final_shash(struct ahash_request *req)
 {
 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+	int offset = 0;
+
+	/*
+	 * If we are running HMAC on limited hardware support, skip
+	 * the ipad in the beginning of the buffer if we are going for
+	 * software fallback algorithm.
+	 */
+	if (test_bit(FLAGS_HMAC, &ctx->flags) &&
+	    !test_bit(FLAGS_AUTO_XOR, &ctx->dd->flags))
+		offset = get_block_size(ctx);
 
 	return omap_sham_shash_digest(tctx->fallback, req->base.flags,
-				      ctx->buffer, ctx->bufcnt, req->result);
+				      ctx->buffer + offset,
+				      ctx->bufcnt - offset, req->result);
 }
 
 static int omap_sham_final(struct ahash_request *req)
@@ -1154,10 +1246,11 @@ static int omap_sham_final(struct ahash_request *req)
 	/*
 	 * OMAP HW accel works only with buffers >= 9.
 	 * HMAC is always >= 9 because ipad == block size.
-	 * If buffersize is less than 240, we use fallback SW encoding,
-	 * as using DMA + HW in this case doesn't provide any benefit.
+	 * If buffersize is less than DMA_THRESHOLD, we use fallback
+	 * SW encoding, as using DMA + HW in this case doesn't provide
+	 * any benefit.
 	 */
-	if ((ctx->digcnt + ctx->bufcnt) < 240)
+	if (!ctx->digcnt && ctx->bufcnt < OMAP_SHA_DMA_THRESHOLD)
 		return omap_sham_final_shash(req);
 	else if (ctx->bufcnt)
 		return omap_sham_enqueue(req, OP_FINAL);
@@ -1323,6 +1416,25 @@ static void omap_sham_cra_exit(struct crypto_tfm *tfm)
 	}
 }
 
+static int omap_sham_export(struct ahash_request *req, void *out)
+{
+	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
+
+	memcpy(out, rctx, sizeof(*rctx) + rctx->bufcnt);
+
+	return 0;
+}
+
+static int omap_sham_import(struct ahash_request *req, const void *in)
+{
+	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
+	const struct omap_sham_reqctx *ctx_in = in;
+
+	memcpy(rctx, in, sizeof(*rctx) + ctx_in->bufcnt);
+
+	return 0;
+}
+
 static struct ahash_alg algs_sha1_md5[] = {
 {
 	.init		= omap_sham_init,
@@ -1341,7 +1453,7 @@ static struct ahash_alg algs_sha1_md5[] = {
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA1_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
-		.cra_alignmask		= 0,
+		.cra_alignmask		= OMAP_ALIGN_MASK,
 		.cra_module		= THIS_MODULE,
 		.cra_init		= omap_sham_cra_init,
 		.cra_exit		= omap_sham_cra_exit,
@@ -1440,7 +1552,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA224_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
-		.cra_alignmask		= 0,
+		.cra_alignmask		= OMAP_ALIGN_MASK,
 		.cra_module		= THIS_MODULE,
 		.cra_init		= omap_sham_cra_init,
 		.cra_exit		= omap_sham_cra_exit,
@@ -1462,7 +1574,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA256_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
-		.cra_alignmask		= 0,
+		.cra_alignmask		= OMAP_ALIGN_MASK,
 		.cra_module		= THIS_MODULE,
 		.cra_init		= omap_sham_cra_init,
 		.cra_exit		= omap_sham_cra_exit,
@@ -1535,7 +1647,7 @@ static struct ahash_alg algs_sha384_sha512[] = {
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA384_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
-		.cra_alignmask		= 0,
+		.cra_alignmask		= OMAP_ALIGN_MASK,
 		.cra_module		= THIS_MODULE,
 		.cra_init		= omap_sham_cra_init,
 		.cra_exit		= omap_sham_cra_exit,
@@ -1557,7 +1669,7 @@ static struct ahash_alg algs_sha384_sha512[] = {
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA512_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
-		.cra_alignmask		= 0,
+		.cra_alignmask		= OMAP_ALIGN_MASK,
 		.cra_module		= THIS_MODULE,
 		.cra_init		= omap_sham_cra_init,
 		.cra_exit		= omap_sham_cra_exit,
@@ -1624,12 +1736,8 @@ static void omap_sham_done_task(unsigned long data)
 	}
 
 	if (test_bit(FLAGS_CPU, &dd->flags)) {
-		if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
-			/* hash or semi-hash ready */
-			err = omap_sham_update_cpu(dd);
-			if (err != -EINPROGRESS)
-				goto finish;
-		}
+		if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
+			goto finish;
 	} else if (test_bit(FLAGS_DMA_READY, &dd->flags)) {
 		if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) {
 			omap_sham_update_dma_stop(dd);
@@ -1641,8 +1749,6 @@ static void omap_sham_done_task(unsigned long data)
 		if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
 			/* hash or semi-hash ready */
 			clear_bit(FLAGS_DMA_READY, &dd->flags);
-			err = omap_sham_update_dma_start(dd);
-			if (err != -EINPROGRESS)
 				goto finish;
 		}
 	}
@@ -1653,6 +1759,10 @@ finish:
 	dev_dbg(dd->dev, "update done: err: %d\n", err);
 	/* finish curent request */
 	omap_sham_finish_req(dd->req, err);
+
+	/* If we are not busy, process next req */
+	if (!test_bit(FLAGS_BUSY, &dd->flags))
+		omap_sham_handle_queue(dd, NULL);
 }
 
 static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd)
@@ -1977,8 +2087,14 @@ static int omap_sham_probe(struct platform_device *pdev)
 
 	for (i = 0; i < dd->pdata->algs_info_size; i++) {
 		for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
-			err = crypto_register_ahash(
-					&dd->pdata->algs_info[i].algs_list[j]);
+			struct ahash_alg *alg;
+
+			alg = &dd->pdata->algs_info[i].algs_list[j];
+			alg->export = omap_sham_export;
+			alg->import = omap_sham_import;
+			alg->halg.statesize = sizeof(struct omap_sham_reqctx) +
+					      BUFLEN;
+			err = crypto_register_ahash(alg);
 			if (err)
 				goto err_algs;
 
diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h
index 2f2681d3458a..afc9a0a86747 100644
--- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h
+++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h
@@ -55,7 +55,7 @@
 #define ADF_C3XXX_MAX_ACCELERATORS 3
 #define ADF_C3XXX_MAX_ACCELENGINES 6
 #define ADF_C3XXX_ACCELERATORS_REG_OFFSET 16
-#define ADF_C3XXX_ACCELERATORS_MASK 0x3
+#define ADF_C3XXX_ACCELERATORS_MASK 0x7
 #define ADF_C3XXX_ACCELENGINES_MASK 0x3F
 #define ADF_C3XXX_ETR_MAX_BANKS 16
 #define ADF_C3XXX_SMIAPF0_MASK_OFFSET (0x3A000 + 0x28)
diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c
index ce7c4626c983..3744b22f0c46 100644
--- a/drivers/crypto/qat/qat_common/adf_admin.c
+++ b/drivers/crypto/qat/qat_common/adf_admin.c
@@ -146,6 +146,7 @@ struct adf_admin_comms {
 	dma_addr_t phy_addr;
 	dma_addr_t const_tbl_addr;
 	void *virt_addr;
+	void *virt_tbl_addr;
 	void __iomem *mailbox_addr;
 	struct mutex lock;	/* protects adf_admin_comms struct */
 };
@@ -251,17 +252,19 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev)
 		return -ENOMEM;
 	}
 
-	admin->const_tbl_addr = dma_map_single(&GET_DEV(accel_dev),
-					       (void *) const_tab, 1024,
-					       DMA_TO_DEVICE);
-
-	if (unlikely(dma_mapping_error(&GET_DEV(accel_dev),
-				       admin->const_tbl_addr))) {
+	admin->virt_tbl_addr = dma_zalloc_coherent(&GET_DEV(accel_dev),
+						   PAGE_SIZE,
+						   &admin->const_tbl_addr,
+						   GFP_KERNEL);
+	if (!admin->virt_tbl_addr) {
+		dev_err(&GET_DEV(accel_dev), "Failed to allocate const_tbl\n");
 		dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
 				  admin->virt_addr, admin->phy_addr);
 		kfree(admin);
 		return -ENOMEM;
 	}
+
+	memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab));
 	reg_val = (u64)admin->phy_addr;
 	ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32);
 	ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val);
@@ -282,9 +285,10 @@ void adf_exit_admin_comms(struct adf_accel_dev *accel_dev)
 	if (admin->virt_addr)
 		dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
 				  admin->virt_addr, admin->phy_addr);
+	if (admin->virt_tbl_addr)
+		dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
+				  admin->virt_tbl_addr, admin->const_tbl_addr);
 
-	dma_unmap_single(&GET_DEV(accel_dev), admin->const_tbl_addr, 1024,
-			 DMA_TO_DEVICE);
 	mutex_destroy(&admin->lock);
 	kfree(admin);
 	accel_dev->admin = NULL;
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index 769148dbaeb3..20f35df8a01f 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -1260,8 +1260,8 @@ static struct crypto_alg qat_algs[] = { {
 			.setkey = qat_alg_ablkcipher_xts_setkey,
 			.decrypt = qat_alg_ablkcipher_decrypt,
 			.encrypt = qat_alg_ablkcipher_encrypt,
-			.min_keysize = AES_MIN_KEY_SIZE,
-			.max_keysize = AES_MAX_KEY_SIZE,
+			.min_keysize = 2 * AES_MIN_KEY_SIZE,
+			.max_keysize = 2 * AES_MAX_KEY_SIZE,
 			.ivsize = AES_BLOCK_SIZE,
 		},
 	},
diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c
index 9b961b37a282..e2454d90d949 100644
--- a/drivers/crypto/qat/qat_common/qat_uclo.c
+++ b/drivers/crypto/qat/qat_common/qat_uclo.c
@@ -967,10 +967,6 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle)
 	struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 	unsigned int ae;
 
-	obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(uint64_t),
-					GFP_KERNEL);
-	if (!obj_handle->uword_buf)
-		return -ENOMEM;
 	obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff;
 	obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *)
 					     obj_handle->obj_hdr->file_buff;
@@ -982,6 +978,10 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle)
 		pr_err("QAT: UOF incompatible\n");
 		return -EINVAL;
 	}
+	obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(uint64_t),
+					GFP_KERNEL);
+	if (!obj_handle->uword_buf)
+		return -ENOMEM;
 	obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE;
 	if (!obj_handle->obj_hdr->file_buff ||
 	    !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT,
diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c
index af508258d2ea..d0f80c6241f9 100644
--- a/drivers/crypto/rockchip/rk3288_crypto.c
+++ b/drivers/crypto/rockchip/rk3288_crypto.c
@@ -304,11 +304,9 @@ static int rk_crypto_probe(struct platform_device *pdev)
 	usleep_range(10, 20);
 	reset_control_deassert(crypto_info->rst);
 
-	err = devm_add_action(dev, rk_crypto_action, crypto_info);
-	if (err) {
-		reset_control_assert(crypto_info->rst);
+	err = devm_add_action_or_reset(dev, rk_crypto_action, crypto_info);
+	if (err)
 		goto err_crypto;
-	}
 
 	spin_lock_init(&crypto_info->lock);
 
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
index 3830d7c4e138..90efd10d57a1 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
@@ -29,7 +29,8 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq)
 	u32 tx_cnt = 0;
 	u32 spaces;
 	u32 v;
-	int i, err = 0;
+	int err = 0;
+	unsigned int i;
 	unsigned int ileft = areq->nbytes;
 	unsigned int oleft = areq->nbytes;
 	unsigned int todo;
@@ -139,7 +140,8 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
 	u32 tx_cnt = 0;
 	u32 v;
 	u32 spaces;
-	int i, err = 0;
+	int err = 0;
+	unsigned int i;
 	unsigned int ileft = areq->nbytes;
 	unsigned int oleft = areq->nbytes;
 	unsigned int todo;
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
index 107cd2a41cae..3ac6c6c4ad18 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
@@ -172,45 +172,45 @@ static struct sun4i_ss_alg_template ss_algs[] = {
 },
 {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
 	.alg.crypto = {
-			.cra_name = "cbc(des3_ede)",
-			.cra_driver_name = "cbc-des3-sun4i-ss",
-			.cra_priority = 300,
-			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
-			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
-			.cra_ctxsize = sizeof(struct sun4i_req_ctx),
-			.cra_module = THIS_MODULE,
-			.cra_alignmask = 3,
-			.cra_type = &crypto_ablkcipher_type,
-			.cra_init = sun4i_ss_cipher_init,
-			.cra_u.ablkcipher = {
-				.min_keysize    = DES3_EDE_KEY_SIZE,
-				.max_keysize    = DES3_EDE_KEY_SIZE,
-				.ivsize         = DES3_EDE_BLOCK_SIZE,
-				.setkey         = sun4i_ss_des3_setkey,
-				.encrypt        = sun4i_ss_cbc_des3_encrypt,
-				.decrypt        = sun4i_ss_cbc_des3_decrypt,
+		.cra_name = "cbc(des3_ede)",
+		.cra_driver_name = "cbc-des3-sun4i-ss",
+		.cra_priority = 300,
+		.cra_blocksize = DES3_EDE_BLOCK_SIZE,
+		.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.cra_ctxsize = sizeof(struct sun4i_req_ctx),
+		.cra_module = THIS_MODULE,
+		.cra_alignmask = 3,
+		.cra_type = &crypto_ablkcipher_type,
+		.cra_init = sun4i_ss_cipher_init,
+		.cra_u.ablkcipher = {
+			.min_keysize    = DES3_EDE_KEY_SIZE,
+			.max_keysize    = DES3_EDE_KEY_SIZE,
+			.ivsize         = DES3_EDE_BLOCK_SIZE,
+			.setkey         = sun4i_ss_des3_setkey,
+			.encrypt        = sun4i_ss_cbc_des3_encrypt,
+			.decrypt        = sun4i_ss_cbc_des3_decrypt,
 		}
 	}
 },
 {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
 	.alg.crypto = {
-			.cra_name = "ecb(des3_ede)",
-			.cra_driver_name = "ecb-des3-sun4i-ss",
-			.cra_priority = 300,
-			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
-			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
-			.cra_ctxsize = sizeof(struct sun4i_req_ctx),
-			.cra_module = THIS_MODULE,
-			.cra_alignmask = 3,
-			.cra_type = &crypto_ablkcipher_type,
-			.cra_init = sun4i_ss_cipher_init,
-			.cra_u.ablkcipher = {
-				.min_keysize    = DES3_EDE_KEY_SIZE,
-				.max_keysize    = DES3_EDE_KEY_SIZE,
-				.ivsize         = DES3_EDE_BLOCK_SIZE,
-				.setkey         = sun4i_ss_des3_setkey,
-				.encrypt        = sun4i_ss_ecb_des3_encrypt,
-				.decrypt        = sun4i_ss_ecb_des3_decrypt,
+		.cra_name = "ecb(des3_ede)",
+		.cra_driver_name = "ecb-des3-sun4i-ss",
+		.cra_priority = 300,
+		.cra_blocksize = DES3_EDE_BLOCK_SIZE,
+		.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.cra_ctxsize = sizeof(struct sun4i_req_ctx),
+		.cra_module = THIS_MODULE,
+		.cra_alignmask = 3,
+		.cra_type = &crypto_ablkcipher_type,
+		.cra_init = sun4i_ss_cipher_init,
+		.cra_u.ablkcipher = {
+			.min_keysize    = DES3_EDE_KEY_SIZE,
+			.max_keysize    = DES3_EDE_KEY_SIZE,
+			.ivsize         = DES3_EDE_BLOCK_SIZE,
+			.setkey         = sun4i_ss_des3_setkey,
+			.encrypt        = sun4i_ss_ecb_des3_encrypt,
+			.decrypt        = sun4i_ss_ecb_des3_decrypt,
 		}
 	}
 },
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c
index ff8031498809..0de2f62d51ff 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c
@@ -20,6 +20,15 @@
 
 int sun4i_hash_crainit(struct crypto_tfm *tfm)
 {
+	struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);
+	struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
+	struct sun4i_ss_alg_template *algt;
+
+	memset(op, 0, sizeof(struct sun4i_tfm_ctx));
+
+	algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash);
+	op->ss = algt->ss;
+
 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 				 sizeof(struct sun4i_req_ctx));
 	return 0;
@@ -32,13 +41,10 @@ int sun4i_hash_init(struct ahash_request *areq)
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
 	struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
 	struct sun4i_ss_alg_template *algt;
-	struct sun4i_ss_ctx *ss;
 
 	memset(op, 0, sizeof(struct sun4i_req_ctx));
 
 	algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash);
-	ss = algt->ss;
-	op->ss = algt->ss;
 	op->mode = algt->mode;
 
 	return 0;
@@ -129,6 +135,9 @@ int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in)
 	return 0;
 }
 
+#define SS_HASH_UPDATE 1
+#define SS_HASH_FINAL 2
+
 /*
  * sun4i_hash_update: update hash engine
  *
@@ -156,7 +165,7 @@ int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in)
  * write remaining data in op->buf
  * final state op->len=56
  */
-int sun4i_hash_update(struct ahash_request *areq)
+static int sun4i_hash(struct ahash_request *areq)
 {
 	u32 v, ivmode = 0;
 	unsigned int i = 0;
@@ -167,8 +176,9 @@ int sun4i_hash_update(struct ahash_request *areq)
 	 */
 
 	struct sun4i_req_ctx *op = ahash_request_ctx(areq);
-	struct sun4i_ss_ctx *ss = op->ss;
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+	struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+	struct sun4i_ss_ctx *ss = tfmctx->ss;
 	unsigned int in_i = 0; /* advancement in the current SG */
 	unsigned int end;
 	/*
@@ -180,22 +190,30 @@ int sun4i_hash_update(struct ahash_request *areq)
 	u32 spaces, rx_cnt = SS_RX_DEFAULT;
 	size_t copied = 0;
 	struct sg_mapping_iter mi;
+	unsigned int j = 0;
+	int zeros;
+	unsigned int index, padlen;
+	__be64 bits;
+	u32 bf[32];
+	u32 wb = 0;
+	unsigned int nwait, nbw = 0;
+	struct scatterlist *in_sg = areq->src;
 
 	dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x",
 		__func__, crypto_tfm_alg_name(areq->base.tfm),
 		op->byte_count, areq->nbytes, op->mode,
 		op->len, op->hash[0]);
 
-	if (areq->nbytes == 0)
+	if (unlikely(areq->nbytes == 0) && (op->flags & SS_HASH_FINAL) == 0)
 		return 0;
 
 	/* protect against overflow */
-	if (areq->nbytes > UINT_MAX - op->len) {
+	if (unlikely(areq->nbytes > UINT_MAX - op->len)) {
 		dev_err(ss->dev, "Cannot process too large request\n");
 		return -EINVAL;
 	}
 
-	if (op->len + areq->nbytes < 64) {
+	if (op->len + areq->nbytes < 64 && (op->flags & SS_HASH_FINAL) == 0) {
 		/* linearize data to op->buf */
 		copied = sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
 					    op->buf + op->len, areq->nbytes, 0);
@@ -203,14 +221,6 @@ int sun4i_hash_update(struct ahash_request *areq)
 		return 0;
 	}
 
-	end = ((areq->nbytes + op->len) / 64) * 64 - op->len;
-
-	if (end > areq->nbytes || areq->nbytes - end > 63) {
-		dev_err(ss->dev, "ERROR: Bound error %u %u\n",
-			end, areq->nbytes);
-		return -EINVAL;
-	}
-
 	spin_lock_bh(&ss->slock);
 
 	/*
@@ -225,6 +235,34 @@ int sun4i_hash_update(struct ahash_request *areq)
 	/* Enable the device */
 	writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL);
 
+	if ((op->flags & SS_HASH_UPDATE) == 0)
+		goto hash_final;
+
+	/* start of handling data */
+	if ((op->flags & SS_HASH_FINAL) == 0) {
+		end = ((areq->nbytes + op->len) / 64) * 64 - op->len;
+
+		if (end > areq->nbytes || areq->nbytes - end > 63) {
+			dev_err(ss->dev, "ERROR: Bound error %u %u\n",
+				end, areq->nbytes);
+			err = -EINVAL;
+			goto release_ss;
+		}
+	} else {
+		/* Since we have the flag final, we can go up to modulo 4 */
+		end = ((areq->nbytes + op->len) / 4) * 4 - op->len;
+	}
+
+	/* TODO if SGlen % 4 and op->len == 0 then DMA */
+	i = 1;
+	while (in_sg && i == 1) {
+		if ((in_sg->length % 4) != 0)
+			i = 0;
+		in_sg = sg_next(in_sg);
+	}
+	if (i == 1 && op->len == 0)
+		dev_dbg(ss->dev, "We can DMA\n");
+
 	i = 0;
 	sg_miter_start(&mi, areq->src, sg_nents(areq->src),
 		       SG_MITER_FROM_SG | SG_MITER_ATOMIC);
@@ -285,7 +323,11 @@ int sun4i_hash_update(struct ahash_request *areq)
 			}
 		}
 	} while (i < end);
-	/* final linear */
+
+	/*
+	 * Now we have written to the device all that we can,
+	 * store the remaining bytes in op->buf
+	 */
 	if ((areq->nbytes - i) < 64) {
 		while (i < areq->nbytes && in_i < mi.length && op->len < 64) {
 			/* how many bytes we can read from current SG */
@@ -304,13 +346,21 @@ int sun4i_hash_update(struct ahash_request *areq)
 
 	sg_miter_stop(&mi);
 
+	/*
+	 * End of data process
+	 * Now if we have the flag final go to finalize part
+	 * If not, store the partial hash
+	 */
+	if ((op->flags & SS_HASH_FINAL) > 0)
+		goto hash_final;
+
 	writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL);
 	i = 0;
 	do {
 		v = readl(ss->base + SS_CTL);
 		i++;
 	} while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0);
-	if (i >= SS_TIMEOUT) {
+	if (unlikely(i >= SS_TIMEOUT)) {
 		dev_err_ratelimited(ss->dev,
 				    "ERROR: hash end timeout %d>%d ctl=%x len=%u\n",
 				    i, SS_TIMEOUT, v, areq->nbytes);
@@ -318,56 +368,24 @@ int sun4i_hash_update(struct ahash_request *areq)
 		goto release_ss;
 	}
 
-	/* get the partial hash only if something was written */
 	for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++)
 		op->hash[i] = readl(ss->base + SS_MD0 + i * 4);
 
-release_ss:
-	writel(0, ss->base + SS_CTL);
-	spin_unlock_bh(&ss->slock);
-	return err;
-}
+	goto release_ss;
 
 /*
- * sun4i_hash_final: finalize hashing operation
+ * hash_final: finalize hashing operation
  *
  * If we have some remaining bytes, we write them.
  * Then ask the SS for finalizing the hashing operation
  *
  * I do not check RX FIFO size in this function since the size is 32
  * after each enabling and this function neither write more than 32 words.
+ * If we come from the update part, we cannot have more than
+ * 3 remaining bytes to write and SS is fast enough to not care about it.
  */
-int sun4i_hash_final(struct ahash_request *areq)
-{
-	u32 v, ivmode = 0;
-	unsigned int i;
-	unsigned int j = 0;
-	int zeros, err = 0;
-	unsigned int index, padlen;
-	__be64 bits;
-	struct sun4i_req_ctx *op = ahash_request_ctx(areq);
-	struct sun4i_ss_ctx *ss = op->ss;
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
-	u32 bf[32];
-	u32 wb = 0;
-	unsigned int nwait, nbw = 0;
-
-	dev_dbg(ss->dev, "%s: byte=%llu len=%u mode=%x wl=%u h=%x",
-		__func__, op->byte_count, areq->nbytes, op->mode,
-		op->len, op->hash[0]);
 
-	spin_lock_bh(&ss->slock);
-
-	/*
-	 * if we have already written something,
-	 * restore the partial hash state
-	 */
-	if (op->byte_count > 0) {
-		ivmode = SS_IV_ARBITRARY;
-		for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++)
-			writel(op->hash[i], ss->base + SS_IV0 + i * 4);
-	}
-	writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL);
+hash_final:
 
 	/* write the remaining words of the wait buffer */
 	if (op->len > 0) {
@@ -428,7 +446,7 @@ int sun4i_hash_final(struct ahash_request *areq)
 
 	/*
 	 * Wait for SS to finish the hash.
-	 * The timeout could happen only in case of bad overcloking
+	 * The timeout could happen only in case of bad overclocking
 	 * or driver bug.
 	 */
 	i = 0;
@@ -436,7 +454,7 @@ int sun4i_hash_final(struct ahash_request *areq)
 		v = readl(ss->base + SS_CTL);
 		i++;
 	} while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0);
-	if (i >= SS_TIMEOUT) {
+	if (unlikely(i >= SS_TIMEOUT)) {
 		dev_err_ratelimited(ss->dev,
 				    "ERROR: hash end timeout %d>%d ctl=%x len=%u\n",
 				    i, SS_TIMEOUT, v, areq->nbytes);
@@ -463,30 +481,41 @@ release_ss:
 	return err;
 }
 
+int sun4i_hash_final(struct ahash_request *areq)
+{
+	struct sun4i_req_ctx *op = ahash_request_ctx(areq);
+
+	op->flags = SS_HASH_FINAL;
+	return sun4i_hash(areq);
+}
+
+int sun4i_hash_update(struct ahash_request *areq)
+{
+	struct sun4i_req_ctx *op = ahash_request_ctx(areq);
+
+	op->flags = SS_HASH_UPDATE;
+	return sun4i_hash(areq);
+}
+
 /* sun4i_hash_finup: finalize hashing operation after an update */
 int sun4i_hash_finup(struct ahash_request *areq)
 {
-	int err;
-
-	err = sun4i_hash_update(areq);
-	if (err != 0)
-		return err;
+	struct sun4i_req_ctx *op = ahash_request_ctx(areq);
 
-	return sun4i_hash_final(areq);
+	op->flags = SS_HASH_UPDATE | SS_HASH_FINAL;
+	return sun4i_hash(areq);
 }
 
 /* combo of init/update/final functions */
 int sun4i_hash_digest(struct ahash_request *areq)
 {
 	int err;
+	struct sun4i_req_ctx *op = ahash_request_ctx(areq);
 
 	err = sun4i_hash_init(areq);
 	if (err != 0)
 		return err;
 
-	err = sun4i_hash_update(areq);
-	if (err != 0)
-		return err;
-
-	return sun4i_hash_final(areq);
+	op->flags = SS_HASH_UPDATE | SS_HASH_FINAL;
+	return sun4i_hash(areq);
 }
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss.h b/drivers/crypto/sunxi-ss/sun4i-ss.h
index 8e9c05f6e4d4..f04c0f8cf026 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss.h
+++ b/drivers/crypto/sunxi-ss/sun4i-ss.h
@@ -163,7 +163,7 @@ struct sun4i_req_ctx {
 	u32 hash[5]; /* for storing SS_IVx register */
 	char buf[64];
 	unsigned int len;
-	struct sun4i_ss_ctx *ss;
+	int flags;
 };
 
 int sun4i_hash_crainit(struct crypto_tfm *tfm);
diff --git a/drivers/crypto/vmx/Kconfig b/drivers/crypto/vmx/Kconfig
index a83ead109d5f..c3d524ea6998 100644
--- a/drivers/crypto/vmx/Kconfig
+++ b/drivers/crypto/vmx/Kconfig
@@ -1,6 +1,7 @@
 config CRYPTO_DEV_VMX_ENCRYPT
 	tristate "Encryption acceleration support on P8 CPU"
 	depends on CRYPTO_DEV_VMX
+	select CRYPTO_GHASH
 	default m
 	help
 	  Support for VMX cryptographic acceleration instructions on Power8 CPU.
diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c
index cfb25413917c..24353ec336c5 100644
--- a/drivers/crypto/vmx/aes_xts.c
+++ b/drivers/crypto/vmx/aes_xts.c
@@ -129,8 +129,8 @@ static int p8_aes_xts_crypt(struct blkcipher_desc *desc,
 
 		blkcipher_walk_init(&walk, dst, src, nbytes);
 
-		iv = (u8 *)walk.iv;
 		ret = blkcipher_walk_virt(desc, &walk);
+		iv = walk.iv;
 		memset(tweak, 0, AES_BLOCK_SIZE);
 		aes_p8_encrypt(iv, tweak, &ctx->tweak_key);
 
diff --git a/drivers/crypto/vmx/ghash.c b/drivers/crypto/vmx/ghash.c
index 6c999cb01b80..27a94a119009 100644
--- a/drivers/crypto/vmx/ghash.c
+++ b/drivers/crypto/vmx/ghash.c
@@ -26,16 +26,13 @@
 #include <linux/hardirq.h>
 #include <asm/switch_to.h>
 #include <crypto/aes.h>
+#include <crypto/ghash.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/internal/hash.h>
 #include <crypto/b128ops.h>
 
 #define IN_INTERRUPT in_interrupt()
 
-#define GHASH_BLOCK_SIZE (16)
-#define GHASH_DIGEST_SIZE (16)
-#define GHASH_KEY_LEN (16)
-
 void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
 void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
 void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
@@ -55,16 +52,11 @@ struct p8_ghash_desc_ctx {
 
 static int p8_ghash_init_tfm(struct crypto_tfm *tfm)
 {
-	const char *alg;
+	const char *alg = "ghash-generic";
 	struct crypto_shash *fallback;
 	struct crypto_shash *shash_tfm = __crypto_shash_cast(tfm);
 	struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	if (!(alg = crypto_tfm_alg_name(tfm))) {
-		printk(KERN_ERR "Failed to get algorithm name.\n");
-		return -ENOENT;
-	}
-
 	fallback = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
 	if (IS_ERR(fallback)) {
 		printk(KERN_ERR
@@ -78,10 +70,18 @@ static int p8_ghash_init_tfm(struct crypto_tfm *tfm)
 	crypto_shash_set_flags(fallback,
 			       crypto_shash_get_flags((struct crypto_shash
 						       *) tfm));
-	ctx->fallback = fallback;
 
-	shash_tfm->descsize = sizeof(struct p8_ghash_desc_ctx)
-	    + crypto_shash_descsize(fallback);
+	/* Check if the descsize defined in the algorithm is still enough. */
+	if (shash_tfm->descsize < sizeof(struct p8_ghash_desc_ctx)
+	    + crypto_shash_descsize(fallback)) {
+		printk(KERN_ERR
+		       "Desc size of the fallback implementation (%s) does not match the expected value: %lu vs %u\n",
+		       alg,
+		       shash_tfm->descsize - sizeof(struct p8_ghash_desc_ctx),
+		       crypto_shash_descsize(fallback));
+		return -EINVAL;
+	}
+	ctx->fallback = fallback;
 
 	return 0;
 }
@@ -113,7 +113,7 @@ static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
 {
 	struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm));
 
-	if (keylen != GHASH_KEY_LEN)
+	if (keylen != GHASH_BLOCK_SIZE)
 		return -EINVAL;
 
 	preempt_disable();
@@ -211,7 +211,8 @@ struct shash_alg p8_ghash_alg = {
 	.update = p8_ghash_update,
 	.final = p8_ghash_final,
 	.setkey = p8_ghash_setkey,
-	.descsize = sizeof(struct p8_ghash_desc_ctx),
+	.descsize = sizeof(struct p8_ghash_desc_ctx)
+		+ sizeof(struct ghash_desc_ctx),
 	.base = {
 		 .cra_name = "ghash",
 		 .cra_driver_name = "p8_ghash",
diff --git a/drivers/dax/Kconfig b/drivers/dax/Kconfig
index cedab7572de3..daadd20aa936 100644
--- a/drivers/dax/Kconfig
+++ b/drivers/dax/Kconfig
@@ -23,4 +23,9 @@ config DEV_DAX_PMEM
 
 	  Say Y if unsure
 
+config NR_DEV_DAX
+	int "Maximum number of Device-DAX instances"
+	default 32768
+	range 256 2147483647
+
 endif
diff --git a/drivers/dax/dax.c b/drivers/dax/dax.c
index 803f3953b341..0e499bfca41c 100644
--- a/drivers/dax/dax.c
+++ b/drivers/dax/dax.c
@@ -13,15 +13,25 @@
 #include <linux/pagemap.h>
 #include <linux/module.h>
 #include <linux/device.h>
+#include <linux/mount.h>
 #include <linux/pfn_t.h>
+#include <linux/hash.h>
+#include <linux/cdev.h>
 #include <linux/slab.h>
 #include <linux/dax.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
+#include "dax.h"
 
-static int dax_major;
+static dev_t dax_devt;
 static struct class *dax_class;
 static DEFINE_IDA(dax_minor_ida);
+static int nr_dax = CONFIG_NR_DEV_DAX;
+module_param(nr_dax, int, S_IRUGO);
+static struct vfsmount *dax_mnt;
+static struct kmem_cache *dax_cache __read_mostly;
+static struct super_block *dax_superblock __read_mostly;
+MODULE_PARM_DESC(nr_dax, "max number of device-dax instances");
 
 /**
  * struct dax_region - mapping infrastructure for dax devices
@@ -48,7 +58,7 @@ struct dax_region {
  * struct dax_dev - subdivision of a dax region
  * @region - parent region
  * @dev - device backing the character device
- * @kref - enable this data to be tracked in filp->private_data
+ * @cdev - core chardev data
  * @alive - !alive + rcu grace period == no new mappings can be established
  * @id - child id in the region
  * @num_resources - number of physical address extents in this device
@@ -56,41 +66,139 @@ struct dax_region {
  */
 struct dax_dev {
 	struct dax_region *region;
-	struct device *dev;
-	struct kref kref;
+	struct inode *inode;
+	struct device dev;
+	struct cdev cdev;
 	bool alive;
 	int id;
 	int num_resources;
 	struct resource res[0];
 };
 
-static void dax_region_free(struct kref *kref)
+static struct inode *dax_alloc_inode(struct super_block *sb)
 {
-	struct dax_region *dax_region;
+	return kmem_cache_alloc(dax_cache, GFP_KERNEL);
+}
 
-	dax_region = container_of(kref, struct dax_region, kref);
-	kfree(dax_region);
+static void dax_i_callback(struct rcu_head *head)
+{
+	struct inode *inode = container_of(head, struct inode, i_rcu);
+
+	kmem_cache_free(dax_cache, inode);
 }
 
-void dax_region_put(struct dax_region *dax_region)
+static void dax_destroy_inode(struct inode *inode)
 {
-	kref_put(&dax_region->kref, dax_region_free);
+	call_rcu(&inode->i_rcu, dax_i_callback);
 }
-EXPORT_SYMBOL_GPL(dax_region_put);
 
-static void dax_dev_free(struct kref *kref)
+static const struct super_operations dax_sops = {
+	.statfs = simple_statfs,
+	.alloc_inode = dax_alloc_inode,
+	.destroy_inode = dax_destroy_inode,
+	.drop_inode = generic_delete_inode,
+};
+
+static struct dentry *dax_mount(struct file_system_type *fs_type,
+		int flags, const char *dev_name, void *data)
 {
-	struct dax_dev *dax_dev;
+	return mount_pseudo(fs_type, "dax:", &dax_sops, NULL, DAXFS_MAGIC);
+}
 
-	dax_dev = container_of(kref, struct dax_dev, kref);
-	dax_region_put(dax_dev->region);
-	kfree(dax_dev);
+static struct file_system_type dax_type = {
+	.name = "dax",
+	.mount = dax_mount,
+	.kill_sb = kill_anon_super,
+};
+
+static int dax_test(struct inode *inode, void *data)
+{
+	return inode->i_cdev == data;
+}
+
+static int dax_set(struct inode *inode, void *data)
+{
+	inode->i_cdev = data;
+	return 0;
+}
+
+static struct inode *dax_inode_get(struct cdev *cdev, dev_t devt)
+{
+	struct inode *inode;
+
+	inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),
+			dax_test, dax_set, cdev);
+
+	if (!inode)
+		return NULL;
+
+	if (inode->i_state & I_NEW) {
+		inode->i_mode = S_IFCHR;
+		inode->i_flags = S_DAX;
+		inode->i_rdev = devt;
+		mapping_set_gfp_mask(&inode->i_data, GFP_USER);
+		unlock_new_inode(inode);
+	}
+	return inode;
+}
+
+static void init_once(void *inode)
+{
+	inode_init_once(inode);
+}
+
+static int dax_inode_init(void)
+{
+	int rc;
+
+	dax_cache = kmem_cache_create("dax_cache", sizeof(struct inode), 0,
+			(SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
+			 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
+			init_once);
+	if (!dax_cache)
+		return -ENOMEM;
+
+	rc = register_filesystem(&dax_type);
+	if (rc)
+		goto err_register_fs;
+
+	dax_mnt = kern_mount(&dax_type);
+	if (IS_ERR(dax_mnt)) {
+		rc = PTR_ERR(dax_mnt);
+		goto err_mount;
+	}
+	dax_superblock = dax_mnt->mnt_sb;
+
+	return 0;
+
+ err_mount:
+	unregister_filesystem(&dax_type);
+ err_register_fs:
+	kmem_cache_destroy(dax_cache);
+
+	return rc;
 }
 
-static void dax_dev_put(struct dax_dev *dax_dev)
+static void dax_inode_exit(void)
+{
+	kern_unmount(dax_mnt);
+	unregister_filesystem(&dax_type);
+	kmem_cache_destroy(dax_cache);
+}
+
+static void dax_region_free(struct kref *kref)
+{
+	struct dax_region *dax_region;
+
+	dax_region = container_of(kref, struct dax_region, kref);
+	kfree(dax_region);
+}
+
+void dax_region_put(struct dax_region *dax_region)
 {
-	kref_put(&dax_dev->kref, dax_dev_free);
+	kref_put(&dax_region->kref, dax_region_free);
 }
+EXPORT_SYMBOL_GPL(dax_region_put);
 
 struct dax_region *alloc_dax_region(struct device *parent, int region_id,
 		struct resource *res, unsigned int align, void *addr,
@@ -98,8 +206,11 @@ struct dax_region *alloc_dax_region(struct device *parent, int region_id,
 {
 	struct dax_region *dax_region;
 
-	dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
+	if (!IS_ALIGNED(res->start, align)
+			|| !IS_ALIGNED(resource_size(res), align))
+		return NULL;
 
+	dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
 	if (!dax_region)
 		return NULL;
 
@@ -116,10 +227,15 @@ struct dax_region *alloc_dax_region(struct device *parent, int region_id,
 }
 EXPORT_SYMBOL_GPL(alloc_dax_region);
 
+static struct dax_dev *to_dax_dev(struct device *dev)
+{
+	return container_of(dev, struct dax_dev, dev);
+}
+
 static ssize_t size_show(struct device *dev,
 		struct device_attribute *attr, char *buf)
 {
-	struct dax_dev *dax_dev = dev_get_drvdata(dev);
+	struct dax_dev *dax_dev = to_dax_dev(dev);
 	unsigned long long size = 0;
 	int i;
 
@@ -144,180 +260,11 @@ static const struct attribute_group *dax_attribute_groups[] = {
 	NULL,
 };
 
-static void unregister_dax_dev(void *_dev)
-{
-	struct device *dev = _dev;
-	struct dax_dev *dax_dev = dev_get_drvdata(dev);
-	struct dax_region *dax_region = dax_dev->region;
-
-	dev_dbg(dev, "%s\n", __func__);
-
-	/*
-	 * Note, rcu is not protecting the liveness of dax_dev, rcu is
-	 * ensuring that any fault handlers that might have seen
-	 * dax_dev->alive == true, have completed.  Any fault handlers
-	 * that start after synchronize_rcu() has started will abort
-	 * upon seeing dax_dev->alive == false.
-	 */
-	dax_dev->alive = false;
-	synchronize_rcu();
-
-	get_device(dev);
-	device_unregister(dev);
-	ida_simple_remove(&dax_region->ida, dax_dev->id);
-	ida_simple_remove(&dax_minor_ida, MINOR(dev->devt));
-	put_device(dev);
-	dax_dev_put(dax_dev);
-}
-
-int devm_create_dax_dev(struct dax_region *dax_region, struct resource *res,
-		int count)
-{
-	struct device *parent = dax_region->dev;
-	struct dax_dev *dax_dev;
-	struct device *dev;
-	int rc, minor;
-	dev_t dev_t;
-
-	dax_dev = kzalloc(sizeof(*dax_dev) + sizeof(*res) * count, GFP_KERNEL);
-	if (!dax_dev)
-		return -ENOMEM;
-	memcpy(dax_dev->res, res, sizeof(*res) * count);
-	dax_dev->num_resources = count;
-	kref_init(&dax_dev->kref);
-	dax_dev->alive = true;
-	dax_dev->region = dax_region;
-	kref_get(&dax_region->kref);
-
-	dax_dev->id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL);
-	if (dax_dev->id < 0) {
-		rc = dax_dev->id;
-		goto err_id;
-	}
-
-	minor = ida_simple_get(&dax_minor_ida, 0, 0, GFP_KERNEL);
-	if (minor < 0) {
-		rc = minor;
-		goto err_minor;
-	}
-
-	dev_t = MKDEV(dax_major, minor);
-	dev = device_create_with_groups(dax_class, parent, dev_t, dax_dev,
-			dax_attribute_groups, "dax%d.%d", dax_region->id,
-			dax_dev->id);
-	if (IS_ERR(dev)) {
-		rc = PTR_ERR(dev);
-		goto err_create;
-	}
-	dax_dev->dev = dev;
-
-	rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_dev, dev);
-	if (rc)
-		return rc;
-
-	return 0;
-
- err_create:
-	ida_simple_remove(&dax_minor_ida, minor);
- err_minor:
-	ida_simple_remove(&dax_region->ida, dax_dev->id);
- err_id:
-	dax_dev_put(dax_dev);
-
-	return rc;
-}
-EXPORT_SYMBOL_GPL(devm_create_dax_dev);
-
-/* return an unmapped area aligned to the dax region specified alignment */
-static unsigned long dax_dev_get_unmapped_area(struct file *filp,
-		unsigned long addr, unsigned long len, unsigned long pgoff,
-		unsigned long flags)
-{
-	unsigned long off, off_end, off_align, len_align, addr_align, align;
-	struct dax_dev *dax_dev = filp ? filp->private_data : NULL;
-	struct dax_region *dax_region;
-
-	if (!dax_dev || addr)
-		goto out;
-
-	dax_region = dax_dev->region;
-	align = dax_region->align;
-	off = pgoff << PAGE_SHIFT;
-	off_end = off + len;
-	off_align = round_up(off, align);
-
-	if ((off_end <= off_align) || ((off_end - off_align) < align))
-		goto out;
-
-	len_align = len + align;
-	if ((off + len_align) < off)
-		goto out;
-
-	addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
-			pgoff, flags);
-	if (!IS_ERR_VALUE(addr_align)) {
-		addr_align += (off - addr_align) & (align - 1);
-		return addr_align;
-	}
- out:
-	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
-}
-
-static int __match_devt(struct device *dev, const void *data)
-{
-	const dev_t *devt = data;
-
-	return dev->devt == *devt;
-}
-
-static struct device *dax_dev_find(dev_t dev_t)
-{
-	return class_find_device(dax_class, NULL, &dev_t, __match_devt);
-}
-
-static int dax_dev_open(struct inode *inode, struct file *filp)
-{
-	struct dax_dev *dax_dev = NULL;
-	struct device *dev;
-
-	dev = dax_dev_find(inode->i_rdev);
-	if (!dev)
-		return -ENXIO;
-
-	device_lock(dev);
-	dax_dev = dev_get_drvdata(dev);
-	if (dax_dev) {
-		dev_dbg(dev, "%s\n", __func__);
-		filp->private_data = dax_dev;
-		kref_get(&dax_dev->kref);
-		inode->i_flags = S_DAX;
-	}
-	device_unlock(dev);
-
-	if (!dax_dev) {
-		put_device(dev);
-		return -ENXIO;
-	}
-	return 0;
-}
-
-static int dax_dev_release(struct inode *inode, struct file *filp)
-{
-	struct dax_dev *dax_dev = filp->private_data;
-	struct device *dev = dax_dev->dev;
-
-	dev_dbg(dax_dev->dev, "%s\n", __func__);
-	dax_dev_put(dax_dev);
-	put_device(dev);
-
-	return 0;
-}
-
 static int check_vma(struct dax_dev *dax_dev, struct vm_area_struct *vma,
 		const char *func)
 {
 	struct dax_region *dax_region = dax_dev->region;
-	struct device *dev = dax_dev->dev;
+	struct device *dev = &dax_dev->dev;
 	unsigned long mask;
 
 	if (!dax_dev->alive)
@@ -382,7 +329,7 @@ static int __dax_dev_fault(struct dax_dev *dax_dev, struct vm_area_struct *vma,
 		struct vm_fault *vmf)
 {
 	unsigned long vaddr = (unsigned long) vmf->virtual_address;
-	struct device *dev = dax_dev->dev;
+	struct device *dev = &dax_dev->dev;
 	struct dax_region *dax_region;
 	int rc = VM_FAULT_SIGBUS;
 	phys_addr_t phys;
@@ -422,7 +369,7 @@ static int dax_dev_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	struct file *filp = vma->vm_file;
 	struct dax_dev *dax_dev = filp->private_data;
 
-	dev_dbg(dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__,
+	dev_dbg(&dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__,
 			current->comm, (vmf->flags & FAULT_FLAG_WRITE)
 			? "write" : "read", vma->vm_start, vma->vm_end);
 	rcu_read_lock();
@@ -437,7 +384,7 @@ static int __dax_dev_pmd_fault(struct dax_dev *dax_dev,
 		unsigned int flags)
 {
 	unsigned long pmd_addr = addr & PMD_MASK;
-	struct device *dev = dax_dev->dev;
+	struct device *dev = &dax_dev->dev;
 	struct dax_region *dax_region;
 	phys_addr_t phys;
 	pgoff_t pgoff;
@@ -459,7 +406,7 @@ static int __dax_dev_pmd_fault(struct dax_dev *dax_dev,
 	}
 
 	pgoff = linear_page_index(vma, pmd_addr);
-	phys = pgoff_to_phys(dax_dev, pgoff, PAGE_SIZE);
+	phys = pgoff_to_phys(dax_dev, pgoff, PMD_SIZE);
 	if (phys == -1) {
 		dev_dbg(dev, "%s: phys_to_pgoff(%#lx) failed\n", __func__,
 				pgoff);
@@ -479,7 +426,7 @@ static int dax_dev_pmd_fault(struct vm_area_struct *vma, unsigned long addr,
 	struct file *filp = vma->vm_file;
 	struct dax_dev *dax_dev = filp->private_data;
 
-	dev_dbg(dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__,
+	dev_dbg(&dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__,
 			current->comm, (flags & FAULT_FLAG_WRITE)
 			? "write" : "read", vma->vm_start, vma->vm_end);
 
@@ -490,81 +437,257 @@ static int dax_dev_pmd_fault(struct vm_area_struct *vma, unsigned long addr,
 	return rc;
 }
 
-static void dax_dev_vm_open(struct vm_area_struct *vma)
-{
-	struct file *filp = vma->vm_file;
-	struct dax_dev *dax_dev = filp->private_data;
-
-	dev_dbg(dax_dev->dev, "%s\n", __func__);
-	kref_get(&dax_dev->kref);
-}
-
-static void dax_dev_vm_close(struct vm_area_struct *vma)
-{
-	struct file *filp = vma->vm_file;
-	struct dax_dev *dax_dev = filp->private_data;
-
-	dev_dbg(dax_dev->dev, "%s\n", __func__);
-	dax_dev_put(dax_dev);
-}
-
 static const struct vm_operations_struct dax_dev_vm_ops = {
 	.fault = dax_dev_fault,
 	.pmd_fault = dax_dev_pmd_fault,
-	.open = dax_dev_vm_open,
-	.close = dax_dev_vm_close,
 };
 
-static int dax_dev_mmap(struct file *filp, struct vm_area_struct *vma)
+static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
 {
 	struct dax_dev *dax_dev = filp->private_data;
 	int rc;
 
-	dev_dbg(dax_dev->dev, "%s\n", __func__);
+	dev_dbg(&dax_dev->dev, "%s\n", __func__);
 
 	rc = check_vma(dax_dev, vma, __func__);
 	if (rc)
 		return rc;
 
-	kref_get(&dax_dev->kref);
 	vma->vm_ops = &dax_dev_vm_ops;
 	vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
 	return 0;
+}
+
+/* return an unmapped area aligned to the dax region specified alignment */
+static unsigned long dax_get_unmapped_area(struct file *filp,
+		unsigned long addr, unsigned long len, unsigned long pgoff,
+		unsigned long flags)
+{
+	unsigned long off, off_end, off_align, len_align, addr_align, align;
+	struct dax_dev *dax_dev = filp ? filp->private_data : NULL;
+	struct dax_region *dax_region;
+
+	if (!dax_dev || addr)
+		goto out;
+
+	dax_region = dax_dev->region;
+	align = dax_region->align;
+	off = pgoff << PAGE_SHIFT;
+	off_end = off + len;
+	off_align = round_up(off, align);
+
+	if ((off_end <= off_align) || ((off_end - off_align) < align))
+		goto out;
+
+	len_align = len + align;
+	if ((off + len_align) < off)
+		goto out;
 
+	addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
+			pgoff, flags);
+	if (!IS_ERR_VALUE(addr_align)) {
+		addr_align += (off - addr_align) & (align - 1);
+		return addr_align;
+	}
+ out:
+	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
+}
+
+static int dax_open(struct inode *inode, struct file *filp)
+{
+	struct dax_dev *dax_dev;
+
+	dax_dev = container_of(inode->i_cdev, struct dax_dev, cdev);
+	dev_dbg(&dax_dev->dev, "%s\n", __func__);
+	inode->i_mapping = dax_dev->inode->i_mapping;
+	inode->i_mapping->host = dax_dev->inode;
+	filp->f_mapping = inode->i_mapping;
+	filp->private_data = dax_dev;
+	inode->i_flags = S_DAX;
+
+	return 0;
+}
+
+static int dax_release(struct inode *inode, struct file *filp)
+{
+	struct dax_dev *dax_dev = filp->private_data;
+
+	dev_dbg(&dax_dev->dev, "%s\n", __func__);
+	return 0;
 }
 
 static const struct file_operations dax_fops = {
 	.llseek = noop_llseek,
 	.owner = THIS_MODULE,
-	.open = dax_dev_open,
-	.release = dax_dev_release,
-	.get_unmapped_area = dax_dev_get_unmapped_area,
-	.mmap = dax_dev_mmap,
+	.open = dax_open,
+	.release = dax_release,
+	.get_unmapped_area = dax_get_unmapped_area,
+	.mmap = dax_mmap,
 };
 
+static void dax_dev_release(struct device *dev)
+{
+	struct dax_dev *dax_dev = to_dax_dev(dev);
+	struct dax_region *dax_region = dax_dev->region;
+
+	ida_simple_remove(&dax_region->ida, dax_dev->id);
+	ida_simple_remove(&dax_minor_ida, MINOR(dev->devt));
+	dax_region_put(dax_region);
+	iput(dax_dev->inode);
+	kfree(dax_dev);
+}
+
+static void unregister_dax_dev(void *dev)
+{
+	struct dax_dev *dax_dev = to_dax_dev(dev);
+	struct cdev *cdev = &dax_dev->cdev;
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	/*
+	 * Note, rcu is not protecting the liveness of dax_dev, rcu is
+	 * ensuring that any fault handlers that might have seen
+	 * dax_dev->alive == true, have completed.  Any fault handlers
+	 * that start after synchronize_rcu() has started will abort
+	 * upon seeing dax_dev->alive == false.
+	 */
+	dax_dev->alive = false;
+	synchronize_rcu();
+	unmap_mapping_range(dax_dev->inode->i_mapping, 0, 0, 1);
+	cdev_del(cdev);
+	device_unregister(dev);
+}
+
+struct dax_dev *devm_create_dax_dev(struct dax_region *dax_region,
+		struct resource *res, int count)
+{
+	struct device *parent = dax_region->dev;
+	struct dax_dev *dax_dev;
+	int rc = 0, minor, i;
+	struct device *dev;
+	struct cdev *cdev;
+	dev_t dev_t;
+
+	dax_dev = kzalloc(sizeof(*dax_dev) + sizeof(*res) * count, GFP_KERNEL);
+	if (!dax_dev)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < count; i++) {
+		if (!IS_ALIGNED(res[i].start, dax_region->align)
+				|| !IS_ALIGNED(resource_size(&res[i]),
+					dax_region->align)) {
+			rc = -EINVAL;
+			break;
+		}
+		dax_dev->res[i].start = res[i].start;
+		dax_dev->res[i].end = res[i].end;
+	}
+
+	if (i < count)
+		goto err_id;
+
+	dax_dev->id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL);
+	if (dax_dev->id < 0) {
+		rc = dax_dev->id;
+		goto err_id;
+	}
+
+	minor = ida_simple_get(&dax_minor_ida, 0, 0, GFP_KERNEL);
+	if (minor < 0) {
+		rc = minor;
+		goto err_minor;
+	}
+
+	dev_t = MKDEV(MAJOR(dax_devt), minor);
+	dev = &dax_dev->dev;
+	dax_dev->inode = dax_inode_get(&dax_dev->cdev, dev_t);
+	if (!dax_dev->inode) {
+		rc = -ENOMEM;
+		goto err_inode;
+	}
+
+	/* device_initialize() so cdev can reference kobj parent */
+	device_initialize(dev);
+
+	cdev = &dax_dev->cdev;
+	cdev_init(cdev, &dax_fops);
+	cdev->owner = parent->driver->owner;
+	cdev->kobj.parent = &dev->kobj;
+	rc = cdev_add(&dax_dev->cdev, dev_t, 1);
+	if (rc)
+		goto err_cdev;
+
+	/* from here on we're committed to teardown via dax_dev_release() */
+	dax_dev->num_resources = count;
+	dax_dev->alive = true;
+	dax_dev->region = dax_region;
+	kref_get(&dax_region->kref);
+
+	dev->devt = dev_t;
+	dev->class = dax_class;
+	dev->parent = parent;
+	dev->groups = dax_attribute_groups;
+	dev->release = dax_dev_release;
+	dev_set_name(dev, "dax%d.%d", dax_region->id, dax_dev->id);
+	rc = device_add(dev);
+	if (rc) {
+		put_device(dev);
+		return ERR_PTR(rc);
+	}
+
+	rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_dev, dev);
+	if (rc)
+		return ERR_PTR(rc);
+
+	return dax_dev;
+
+ err_cdev:
+	iput(dax_dev->inode);
+ err_inode:
+	ida_simple_remove(&dax_minor_ida, minor);
+ err_minor:
+	ida_simple_remove(&dax_region->ida, dax_dev->id);
+ err_id:
+	kfree(dax_dev);
+
+	return ERR_PTR(rc);
+}
+EXPORT_SYMBOL_GPL(devm_create_dax_dev);
+
 static int __init dax_init(void)
 {
 	int rc;
 
-	rc = register_chrdev(0, "dax", &dax_fops);
-	if (rc < 0)
+	rc = dax_inode_init();
+	if (rc)
 		return rc;
-	dax_major = rc;
+
+	nr_dax = max(nr_dax, 256);
+	rc = alloc_chrdev_region(&dax_devt, 0, nr_dax, "dax");
+	if (rc)
+		goto err_chrdev;
 
 	dax_class = class_create(THIS_MODULE, "dax");
 	if (IS_ERR(dax_class)) {
-		unregister_chrdev(dax_major, "dax");
-		return PTR_ERR(dax_class);
+		rc = PTR_ERR(dax_class);
+		goto err_class;
 	}
 
 	return 0;
+
+ err_class:
+	unregister_chrdev_region(dax_devt, nr_dax);
+ err_chrdev:
+	dax_inode_exit();
+	return rc;
 }
 
 static void __exit dax_exit(void)
 {
 	class_destroy(dax_class);
-	unregister_chrdev(dax_major, "dax");
+	unregister_chrdev_region(dax_devt, nr_dax);
 	ida_destroy(&dax_minor_ida);
+	dax_inode_exit();
 }
 
 MODULE_AUTHOR("Intel Corporation");
diff --git a/drivers/dax/dax.h b/drivers/dax/dax.h
index d8b8f1f25054..ddd829ab58c0 100644
--- a/drivers/dax/dax.h
+++ b/drivers/dax/dax.h
@@ -13,12 +13,13 @@
 #ifndef __DAX_H__
 #define __DAX_H__
 struct device;
+struct dax_dev;
 struct resource;
 struct dax_region;
 void dax_region_put(struct dax_region *dax_region);
 struct dax_region *alloc_dax_region(struct device *parent,
 		int region_id, struct resource *res, unsigned int align,
 		void *addr, unsigned long flags);
-int devm_create_dax_dev(struct dax_region *dax_region, struct resource *res,
-		int count);
+struct dax_dev *devm_create_dax_dev(struct dax_region *dax_region,
+		struct resource *res, int count);
 #endif /* __DAX_H__ */
diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c
index dfb168568af1..9630d8837ba9 100644
--- a/drivers/dax/pmem.c
+++ b/drivers/dax/pmem.c
@@ -24,7 +24,7 @@ struct dax_pmem {
 	struct completion cmp;
 };
 
-struct dax_pmem *to_dax_pmem(struct percpu_ref *ref)
+static struct dax_pmem *to_dax_pmem(struct percpu_ref *ref)
 {
 	return container_of(ref, struct dax_pmem, ref);
 }
@@ -61,6 +61,7 @@ static int dax_pmem_probe(struct device *dev)
 	int rc;
 	void *addr;
 	struct resource res;
+	struct dax_dev *dax_dev;
 	struct nd_pfn_sb *pfn_sb;
 	struct dax_pmem *dax_pmem;
 	struct nd_region *nd_region;
@@ -116,6 +117,9 @@ static int dax_pmem_probe(struct device *dev)
 	if (rc)
 		return rc;
 
+	/* adjust the dax_region resource to the start of data */
+	res.start += le64_to_cpu(pfn_sb->dataoff);
+
 	nd_region = to_nd_region(dev->parent);
 	dax_region = alloc_dax_region(dev, nd_region->id, &res,
 			le32_to_cpu(pfn_sb->align), addr, PFN_DEV|PFN_MAP);
@@ -123,12 +127,12 @@ static int dax_pmem_probe(struct device *dev)
 		return -ENOMEM;
 
 	/* TODO: support for subdividing a dax region... */
-	rc = devm_create_dax_dev(dax_region, &res, 1);
+	dax_dev = devm_create_dax_dev(dax_region, &res, 1);
 
 	/* child dax_dev instances now own the lifetime of the dax_region */
 	dax_region_put(dax_region);
 
-	return rc;
+	return PTR_ERR_OR_ZERO(dax_dev);
 }
 
 static struct nd_device_driver dax_pmem_driver = {
diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig
index a5be56ec57f2..41254e702f1e 100644
--- a/drivers/devfreq/Kconfig
+++ b/drivers/devfreq/Kconfig
@@ -76,7 +76,7 @@ comment "DEVFREQ Drivers"
 
 config ARM_EXYNOS_BUS_DEVFREQ
 	tristate "ARM EXYNOS Generic Memory Bus DEVFREQ Driver"
-	depends on ARCH_EXYNOS
+	depends on ARCH_EXYNOS || COMPILE_TEST
 	select DEVFREQ_GOV_SIMPLE_ONDEMAND
 	select DEVFREQ_GOV_PASSIVE
 	select DEVFREQ_EVENT_EXYNOS_PPMU
@@ -91,14 +91,26 @@ config ARM_EXYNOS_BUS_DEVFREQ
 	  This does not yet operate with optimal voltages.
 
 config ARM_TEGRA_DEVFREQ
-       tristate "Tegra DEVFREQ Driver"
-       depends on ARCH_TEGRA_124_SOC
-       select DEVFREQ_GOV_SIMPLE_ONDEMAND
-       select PM_OPP
-       help
-         This adds the DEVFREQ driver for the Tegra family of SoCs.
-         It reads ACTMON counters of memory controllers and adjusts the
-         operating frequencies and voltages with OPP support.
+	tristate "Tegra DEVFREQ Driver"
+	depends on ARCH_TEGRA_124_SOC
+	select DEVFREQ_GOV_SIMPLE_ONDEMAND
+	select PM_OPP
+	help
+	  This adds the DEVFREQ driver for the Tegra family of SoCs.
+	  It reads ACTMON counters of memory controllers and adjusts the
+	  operating frequencies and voltages with OPP support.
+
+config ARM_RK3399_DMC_DEVFREQ
+	tristate "ARM RK3399 DMC DEVFREQ Driver"
+	depends on ARCH_ROCKCHIP
+	select DEVFREQ_EVENT_ROCKCHIP_DFI
+	select DEVFREQ_GOV_SIMPLE_ONDEMAND
+	select PM_DEVFREQ_EVENT
+	select PM_OPP
+	help
+          This adds the DEVFREQ driver for the RK3399 DMC(Dynamic Memory Controller).
+          It sets the frequency for the memory controller and reads the usage counts
+          from hardware.
 
 source "drivers/devfreq/event/Kconfig"
 
diff --git a/drivers/devfreq/Makefile b/drivers/devfreq/Makefile
index 09f11d9d40d5..fbff40a508a4 100644
--- a/drivers/devfreq/Makefile
+++ b/drivers/devfreq/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_DEVFREQ_GOV_PASSIVE)	+= governor_passive.o
 
 # DEVFREQ Drivers
 obj-$(CONFIG_ARM_EXYNOS_BUS_DEVFREQ)	+= exynos-bus.o
+obj-$(CONFIG_ARM_RK3399_DMC_DEVFREQ)	+= rk3399_dmc.o
 obj-$(CONFIG_ARM_TEGRA_DEVFREQ)		+= tegra-devfreq.o
 
 # DEVFREQ Event Drivers
diff --git a/drivers/devfreq/event/Kconfig b/drivers/devfreq/event/Kconfig
index eb6f74a2b6b9..0fdae8608961 100644
--- a/drivers/devfreq/event/Kconfig
+++ b/drivers/devfreq/event/Kconfig
@@ -15,7 +15,7 @@ if PM_DEVFREQ_EVENT
 
 config DEVFREQ_EVENT_EXYNOS_NOCP
 	tristate "EXYNOS NoC (Network On Chip) Probe DEVFREQ event Driver"
-	depends on ARCH_EXYNOS
+	depends on ARCH_EXYNOS || COMPILE_TEST
 	select PM_OPP
 	help
 	  This add the devfreq-event driver for Exynos SoC. It provides NoC
@@ -23,11 +23,18 @@ config DEVFREQ_EVENT_EXYNOS_NOCP
 
 config DEVFREQ_EVENT_EXYNOS_PPMU
 	tristate "EXYNOS PPMU (Platform Performance Monitoring Unit) DEVFREQ event Driver"
-	depends on ARCH_EXYNOS
+	depends on ARCH_EXYNOS || COMPILE_TEST
 	select PM_OPP
 	help
 	  This add the devfreq-event driver for Exynos SoC. It provides PPMU
 	  (Platform Performance Monitoring Unit) counters to estimate the
 	  utilization of each module.
 
+config DEVFREQ_EVENT_ROCKCHIP_DFI
+	tristate "ROCKCHIP DFI DEVFREQ event Driver"
+	depends on ARCH_ROCKCHIP
+	help
+	  This add the devfreq-event driver for Rockchip SoC. It provides DFI
+	  (DDR Monitor Module) driver to count ddr load.
+
 endif # PM_DEVFREQ_EVENT
diff --git a/drivers/devfreq/event/Makefile b/drivers/devfreq/event/Makefile
index 3d6afd352253..dda7090a47c6 100644
--- a/drivers/devfreq/event/Makefile
+++ b/drivers/devfreq/event/Makefile
@@ -2,3 +2,4 @@
 
 obj-$(CONFIG_DEVFREQ_EVENT_EXYNOS_NOCP) += exynos-nocp.o
 obj-$(CONFIG_DEVFREQ_EVENT_EXYNOS_PPMU) += exynos-ppmu.o
+obj-$(CONFIG_DEVFREQ_EVENT_ROCKCHIP_DFI) += rockchip-dfi.o
diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c
index 845bf25fb9fb..f55cf0eb2a66 100644
--- a/drivers/devfreq/event/exynos-ppmu.c
+++ b/drivers/devfreq/event/exynos-ppmu.c
@@ -406,8 +406,6 @@ static int of_get_devfreq_events(struct device_node *np,
 		of_property_read_string(node, "event-name", &desc[j].name);
 
 		j++;
-
-		of_node_put(node);
 	}
 	info->desc = desc;
 
diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c
new file mode 100644
index 000000000000..43fcc5a7f515
--- /dev/null
+++ b/drivers/devfreq/event/rockchip-dfi.c
@@ -0,0 +1,256 @@
+/*
+ * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd
+ * Author: Lin Huang <hl@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/devfreq-event.h>
+#include <linux/kernel.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/of.h>
+
+#define RK3399_DMC_NUM_CH	2
+
+/* DDRMON_CTRL */
+#define DDRMON_CTRL	0x04
+#define CLR_DDRMON_CTRL	(0x1f0000 << 0)
+#define LPDDR4_EN	(0x10001 << 4)
+#define HARDWARE_EN	(0x10001 << 3)
+#define LPDDR3_EN	(0x10001 << 2)
+#define SOFTWARE_EN	(0x10001 << 1)
+#define SOFTWARE_DIS	(0x10000 << 1)
+#define TIME_CNT_EN	(0x10001 << 0)
+
+#define DDRMON_CH0_COUNT_NUM		0x28
+#define DDRMON_CH0_DFI_ACCESS_NUM	0x2c
+#define DDRMON_CH1_COUNT_NUM		0x3c
+#define DDRMON_CH1_DFI_ACCESS_NUM	0x40
+
+/* pmu grf */
+#define PMUGRF_OS_REG2	0x308
+#define DDRTYPE_SHIFT	13
+#define DDRTYPE_MASK	7
+
+enum {
+	DDR3 = 3,
+	LPDDR3 = 6,
+	LPDDR4 = 7,
+	UNUSED = 0xFF
+};
+
+struct dmc_usage {
+	u32 access;
+	u32 total;
+};
+
+/*
+ * The dfi controller can monitor DDR load. It has an upper and lower threshold
+ * for the operating points. Whenever the usage leaves these bounds an event is
+ * generated to indicate the DDR frequency should be changed.
+ */
+struct rockchip_dfi {
+	struct devfreq_event_dev *edev;
+	struct devfreq_event_desc *desc;
+	struct dmc_usage ch_usage[RK3399_DMC_NUM_CH];
+	struct device *dev;
+	void __iomem *regs;
+	struct regmap *regmap_pmu;
+	struct clk *clk;
+};
+
+static void rockchip_dfi_start_hardware_counter(struct devfreq_event_dev *edev)
+{
+	struct rockchip_dfi *info = devfreq_event_get_drvdata(edev);
+	void __iomem *dfi_regs = info->regs;
+	u32 val;
+	u32 ddr_type;
+
+	/* get ddr type */
+	regmap_read(info->regmap_pmu, PMUGRF_OS_REG2, &val);
+	ddr_type = (val >> DDRTYPE_SHIFT) & DDRTYPE_MASK;
+
+	/* clear DDRMON_CTRL setting */
+	writel_relaxed(CLR_DDRMON_CTRL, dfi_regs + DDRMON_CTRL);
+
+	/* set ddr type to dfi */
+	if (ddr_type == LPDDR3)
+		writel_relaxed(LPDDR3_EN, dfi_regs + DDRMON_CTRL);
+	else if (ddr_type == LPDDR4)
+		writel_relaxed(LPDDR4_EN, dfi_regs + DDRMON_CTRL);
+
+	/* enable count, use software mode */
+	writel_relaxed(SOFTWARE_EN, dfi_regs + DDRMON_CTRL);
+}
+
+static void rockchip_dfi_stop_hardware_counter(struct devfreq_event_dev *edev)
+{
+	struct rockchip_dfi *info = devfreq_event_get_drvdata(edev);
+	void __iomem *dfi_regs = info->regs;
+
+	writel_relaxed(SOFTWARE_DIS, dfi_regs + DDRMON_CTRL);
+}
+
+static int rockchip_dfi_get_busier_ch(struct devfreq_event_dev *edev)
+{
+	struct rockchip_dfi *info = devfreq_event_get_drvdata(edev);
+	u32 tmp, max = 0;
+	u32 i, busier_ch = 0;
+	void __iomem *dfi_regs = info->regs;
+
+	rockchip_dfi_stop_hardware_counter(edev);
+
+	/* Find out which channel is busier */
+	for (i = 0; i < RK3399_DMC_NUM_CH; i++) {
+		info->ch_usage[i].access = readl_relaxed(dfi_regs +
+				DDRMON_CH0_DFI_ACCESS_NUM + i * 20) * 4;
+		info->ch_usage[i].total = readl_relaxed(dfi_regs +
+				DDRMON_CH0_COUNT_NUM + i * 20);
+		tmp = info->ch_usage[i].access;
+		if (tmp > max) {
+			busier_ch = i;
+			max = tmp;
+		}
+	}
+	rockchip_dfi_start_hardware_counter(edev);
+
+	return busier_ch;
+}
+
+static int rockchip_dfi_disable(struct devfreq_event_dev *edev)
+{
+	struct rockchip_dfi *info = devfreq_event_get_drvdata(edev);
+
+	rockchip_dfi_stop_hardware_counter(edev);
+	clk_disable_unprepare(info->clk);
+
+	return 0;
+}
+
+static int rockchip_dfi_enable(struct devfreq_event_dev *edev)
+{
+	struct rockchip_dfi *info = devfreq_event_get_drvdata(edev);
+	int ret;
+
+	ret = clk_prepare_enable(info->clk);
+	if (ret) {
+		dev_err(&edev->dev, "failed to enable dfi clk: %d\n", ret);
+		return ret;
+	}
+
+	rockchip_dfi_start_hardware_counter(edev);
+	return 0;
+}
+
+static int rockchip_dfi_set_event(struct devfreq_event_dev *edev)
+{
+	return 0;
+}
+
+static int rockchip_dfi_get_event(struct devfreq_event_dev *edev,
+				  struct devfreq_event_data *edata)
+{
+	struct rockchip_dfi *info = devfreq_event_get_drvdata(edev);
+	int busier_ch;
+
+	busier_ch = rockchip_dfi_get_busier_ch(edev);
+
+	edata->load_count = info->ch_usage[busier_ch].access;
+	edata->total_count = info->ch_usage[busier_ch].total;
+
+	return 0;
+}
+
+static const struct devfreq_event_ops rockchip_dfi_ops = {
+	.disable = rockchip_dfi_disable,
+	.enable = rockchip_dfi_enable,
+	.get_event = rockchip_dfi_get_event,
+	.set_event = rockchip_dfi_set_event,
+};
+
+static const struct of_device_id rockchip_dfi_id_match[] = {
+	{ .compatible = "rockchip,rk3399-dfi" },
+	{ },
+};
+
+static int rockchip_dfi_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rockchip_dfi *data;
+	struct resource *res;
+	struct devfreq_event_desc *desc;
+	struct device_node *np = pdev->dev.of_node, *node;
+
+	data = devm_kzalloc(dev, sizeof(struct rockchip_dfi), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	data->regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(data->regs))
+		return PTR_ERR(data->regs);
+
+	data->clk = devm_clk_get(dev, "pclk_ddr_mon");
+	if (IS_ERR(data->clk)) {
+		dev_err(dev, "Cannot get the clk dmc_clk\n");
+		return PTR_ERR(data->clk);
+	};
+
+	/* try to find the optional reference to the pmu syscon */
+	node = of_parse_phandle(np, "rockchip,pmu", 0);
+	if (node) {
+		data->regmap_pmu = syscon_node_to_regmap(node);
+		if (IS_ERR(data->regmap_pmu))
+			return PTR_ERR(data->regmap_pmu);
+	}
+	data->dev = dev;
+
+	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	desc->ops = &rockchip_dfi_ops;
+	desc->driver_data = data;
+	desc->name = np->name;
+	data->desc = desc;
+
+	data->edev = devm_devfreq_event_add_edev(&pdev->dev, desc);
+	if (IS_ERR(data->edev)) {
+		dev_err(&pdev->dev,
+			"failed to add devfreq-event device\n");
+		return PTR_ERR(data->edev);
+	}
+
+	platform_set_drvdata(pdev, data);
+
+	return 0;
+}
+
+static struct platform_driver rockchip_dfi_driver = {
+	.probe	= rockchip_dfi_probe,
+	.driver = {
+		.name	= "rockchip-dfi",
+		.of_match_table = rockchip_dfi_id_match,
+	},
+};
+module_platform_driver(rockchip_dfi_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Lin Huang <hl@rock-chips.com>");
+MODULE_DESCRIPTION("Rockchip DFI driver");
diff --git a/drivers/devfreq/rk3399_dmc.c b/drivers/devfreq/rk3399_dmc.c
new file mode 100644
index 000000000000..e24b73d66659
--- /dev/null
+++ b/drivers/devfreq/rk3399_dmc.c
@@ -0,0 +1,470 @@
+/*
+ * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd.
+ * Author: Lin Huang <hl@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/arm-smccc.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/devfreq.h>
+#include <linux/devfreq-event.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/regulator/consumer.h>
+#include <linux/rwsem.h>
+#include <linux/suspend.h>
+
+#include <soc/rockchip/rockchip_sip.h>
+
+struct dram_timing {
+	unsigned int ddr3_speed_bin;
+	unsigned int pd_idle;
+	unsigned int sr_idle;
+	unsigned int sr_mc_gate_idle;
+	unsigned int srpd_lite_idle;
+	unsigned int standby_idle;
+	unsigned int auto_pd_dis_freq;
+	unsigned int dram_dll_dis_freq;
+	unsigned int phy_dll_dis_freq;
+	unsigned int ddr3_odt_dis_freq;
+	unsigned int ddr3_drv;
+	unsigned int ddr3_odt;
+	unsigned int phy_ddr3_ca_drv;
+	unsigned int phy_ddr3_dq_drv;
+	unsigned int phy_ddr3_odt;
+	unsigned int lpddr3_odt_dis_freq;
+	unsigned int lpddr3_drv;
+	unsigned int lpddr3_odt;
+	unsigned int phy_lpddr3_ca_drv;
+	unsigned int phy_lpddr3_dq_drv;
+	unsigned int phy_lpddr3_odt;
+	unsigned int lpddr4_odt_dis_freq;
+	unsigned int lpddr4_drv;
+	unsigned int lpddr4_dq_odt;
+	unsigned int lpddr4_ca_odt;
+	unsigned int phy_lpddr4_ca_drv;
+	unsigned int phy_lpddr4_ck_cs_drv;
+	unsigned int phy_lpddr4_dq_drv;
+	unsigned int phy_lpddr4_odt;
+};
+
+struct rk3399_dmcfreq {
+	struct device *dev;
+	struct devfreq *devfreq;
+	struct devfreq_simple_ondemand_data ondemand_data;
+	struct clk *dmc_clk;
+	struct devfreq_event_dev *edev;
+	struct mutex lock;
+	struct dram_timing timing;
+
+	/*
+	 * DDR Converser of Frequency (DCF) is used to implement DDR frequency
+	 * conversion without the participation of CPU, we will implement and
+	 * control it in arm trust firmware.
+	 */
+	wait_queue_head_t	wait_dcf_queue;
+	int irq;
+	int wait_dcf_flag;
+	struct regulator *vdd_center;
+	unsigned long rate, target_rate;
+	unsigned long volt, target_volt;
+	struct dev_pm_opp *curr_opp;
+};
+
+static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq,
+				 u32 flags)
+{
+	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
+	struct dev_pm_opp *opp;
+	unsigned long old_clk_rate = dmcfreq->rate;
+	unsigned long target_volt, target_rate;
+	int err;
+
+	rcu_read_lock();
+	opp = devfreq_recommended_opp(dev, freq, flags);
+	if (IS_ERR(opp)) {
+		rcu_read_unlock();
+		return PTR_ERR(opp);
+	}
+
+	target_rate = dev_pm_opp_get_freq(opp);
+	target_volt = dev_pm_opp_get_voltage(opp);
+
+	dmcfreq->rate = dev_pm_opp_get_freq(dmcfreq->curr_opp);
+	dmcfreq->volt = dev_pm_opp_get_voltage(dmcfreq->curr_opp);
+
+	rcu_read_unlock();
+
+	if (dmcfreq->rate == target_rate)
+		return 0;
+
+	mutex_lock(&dmcfreq->lock);
+
+	/*
+	 * If frequency scaling from low to high, adjust voltage first.
+	 * If frequency scaling from high to low, adjust frequency first.
+	 */
+	if (old_clk_rate < target_rate) {
+		err = regulator_set_voltage(dmcfreq->vdd_center, target_volt,
+					    target_volt);
+		if (err) {
+			dev_err(dev, "Cannot to set voltage %lu uV\n",
+				target_volt);
+			goto out;
+		}
+	}
+	dmcfreq->wait_dcf_flag = 1;
+
+	err = clk_set_rate(dmcfreq->dmc_clk, target_rate);
+	if (err) {
+		dev_err(dev, "Cannot to set frequency %lu (%d)\n",
+			target_rate, err);
+		regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt,
+				      dmcfreq->volt);
+		goto out;
+	}
+
+	/*
+	 * Wait until bcf irq happen, it means freq scaling finish in
+	 * arm trust firmware, use 100ms as timeout time.
+	 */
+	if (!wait_event_timeout(dmcfreq->wait_dcf_queue,
+				!dmcfreq->wait_dcf_flag, HZ / 10))
+		dev_warn(dev, "Timeout waiting for dcf interrupt\n");
+
+	/*
+	 * Check the dpll rate,
+	 * There only two result we will get,
+	 * 1. Ddr frequency scaling fail, we still get the old rate.
+	 * 2. Ddr frequency scaling sucessful, we get the rate we set.
+	 */
+	dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk);
+
+	/* If get the incorrect rate, set voltage to old value. */
+	if (dmcfreq->rate != target_rate) {
+		dev_err(dev, "Get wrong ddr frequency, Request frequency %lu,\
+			Current frequency %lu\n", target_rate, dmcfreq->rate);
+		regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt,
+				      dmcfreq->volt);
+		goto out;
+	} else if (old_clk_rate > target_rate)
+		err = regulator_set_voltage(dmcfreq->vdd_center, target_volt,
+					    target_volt);
+	if (err)
+		dev_err(dev, "Cannot to set vol %lu uV\n", target_volt);
+
+	dmcfreq->curr_opp = opp;
+out:
+	mutex_unlock(&dmcfreq->lock);
+	return err;
+}
+
+static int rk3399_dmcfreq_get_dev_status(struct device *dev,
+					 struct devfreq_dev_status *stat)
+{
+	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
+	struct devfreq_event_data edata;
+	int ret = 0;
+
+	ret = devfreq_event_get_event(dmcfreq->edev, &edata);
+	if (ret < 0)
+		return ret;
+
+	stat->current_frequency = dmcfreq->rate;
+	stat->busy_time = edata.load_count;
+	stat->total_time = edata.total_count;
+
+	return ret;
+}
+
+static int rk3399_dmcfreq_get_cur_freq(struct device *dev, unsigned long *freq)
+{
+	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
+
+	*freq = dmcfreq->rate;
+
+	return 0;
+}
+
+static struct devfreq_dev_profile rk3399_devfreq_dmc_profile = {
+	.polling_ms	= 200,
+	.target		= rk3399_dmcfreq_target,
+	.get_dev_status	= rk3399_dmcfreq_get_dev_status,
+	.get_cur_freq	= rk3399_dmcfreq_get_cur_freq,
+};
+
+static __maybe_unused int rk3399_dmcfreq_suspend(struct device *dev)
+{
+	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
+	int ret = 0;
+
+	ret = devfreq_event_disable_edev(dmcfreq->edev);
+	if (ret < 0) {
+		dev_err(dev, "failed to disable the devfreq-event devices\n");
+		return ret;
+	}
+
+	ret = devfreq_suspend_device(dmcfreq->devfreq);
+	if (ret < 0) {
+		dev_err(dev, "failed to suspend the devfreq devices\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static __maybe_unused int rk3399_dmcfreq_resume(struct device *dev)
+{
+	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
+	int ret = 0;
+
+	ret = devfreq_event_enable_edev(dmcfreq->edev);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable the devfreq-event devices\n");
+		return ret;
+	}
+
+	ret = devfreq_resume_device(dmcfreq->devfreq);
+	if (ret < 0) {
+		dev_err(dev, "failed to resume the devfreq devices\n");
+		return ret;
+	}
+	return ret;
+}
+
+static SIMPLE_DEV_PM_OPS(rk3399_dmcfreq_pm, rk3399_dmcfreq_suspend,
+			 rk3399_dmcfreq_resume);
+
+static irqreturn_t rk3399_dmc_irq(int irq, void *dev_id)
+{
+	struct rk3399_dmcfreq *dmcfreq = dev_id;
+	struct arm_smccc_res res;
+
+	dmcfreq->wait_dcf_flag = 0;
+	wake_up(&dmcfreq->wait_dcf_queue);
+
+	/* Clear the DCF interrupt */
+	arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0,
+		      ROCKCHIP_SIP_CONFIG_DRAM_CLR_IRQ,
+		      0, 0, 0, 0, &res);
+
+	return IRQ_HANDLED;
+}
+
+static int of_get_ddr_timings(struct dram_timing *timing,
+			      struct device_node *np)
+{
+	int ret = 0;
+
+	ret = of_property_read_u32(np, "rockchip,ddr3_speed_bin",
+				   &timing->ddr3_speed_bin);
+	ret |= of_property_read_u32(np, "rockchip,pd_idle",
+				    &timing->pd_idle);
+	ret |= of_property_read_u32(np, "rockchip,sr_idle",
+				    &timing->sr_idle);
+	ret |= of_property_read_u32(np, "rockchip,sr_mc_gate_idle",
+				    &timing->sr_mc_gate_idle);
+	ret |= of_property_read_u32(np, "rockchip,srpd_lite_idle",
+				    &timing->srpd_lite_idle);
+	ret |= of_property_read_u32(np, "rockchip,standby_idle",
+				    &timing->standby_idle);
+	ret |= of_property_read_u32(np, "rockchip,auto_pd_dis_freq",
+				    &timing->auto_pd_dis_freq);
+	ret |= of_property_read_u32(np, "rockchip,dram_dll_dis_freq",
+				    &timing->dram_dll_dis_freq);
+	ret |= of_property_read_u32(np, "rockchip,phy_dll_dis_freq",
+				    &timing->phy_dll_dis_freq);
+	ret |= of_property_read_u32(np, "rockchip,ddr3_odt_dis_freq",
+				    &timing->ddr3_odt_dis_freq);
+	ret |= of_property_read_u32(np, "rockchip,ddr3_drv",
+				    &timing->ddr3_drv);
+	ret |= of_property_read_u32(np, "rockchip,ddr3_odt",
+				    &timing->ddr3_odt);
+	ret |= of_property_read_u32(np, "rockchip,phy_ddr3_ca_drv",
+				    &timing->phy_ddr3_ca_drv);
+	ret |= of_property_read_u32(np, "rockchip,phy_ddr3_dq_drv",
+				    &timing->phy_ddr3_dq_drv);
+	ret |= of_property_read_u32(np, "rockchip,phy_ddr3_odt",
+				    &timing->phy_ddr3_odt);
+	ret |= of_property_read_u32(np, "rockchip,lpddr3_odt_dis_freq",
+				    &timing->lpddr3_odt_dis_freq);
+	ret |= of_property_read_u32(np, "rockchip,lpddr3_drv",
+				    &timing->lpddr3_drv);
+	ret |= of_property_read_u32(np, "rockchip,lpddr3_odt",
+				    &timing->lpddr3_odt);
+	ret |= of_property_read_u32(np, "rockchip,phy_lpddr3_ca_drv",
+				    &timing->phy_lpddr3_ca_drv);
+	ret |= of_property_read_u32(np, "rockchip,phy_lpddr3_dq_drv",
+				    &timing->phy_lpddr3_dq_drv);
+	ret |= of_property_read_u32(np, "rockchip,phy_lpddr3_odt",
+				    &timing->phy_lpddr3_odt);
+	ret |= of_property_read_u32(np, "rockchip,lpddr4_odt_dis_freq",
+				    &timing->lpddr4_odt_dis_freq);
+	ret |= of_property_read_u32(np, "rockchip,lpddr4_drv",
+				    &timing->lpddr4_drv);
+	ret |= of_property_read_u32(np, "rockchip,lpddr4_dq_odt",
+				    &timing->lpddr4_dq_odt);
+	ret |= of_property_read_u32(np, "rockchip,lpddr4_ca_odt",
+				    &timing->lpddr4_ca_odt);
+	ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_ca_drv",
+				    &timing->phy_lpddr4_ca_drv);
+	ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_ck_cs_drv",
+				    &timing->phy_lpddr4_ck_cs_drv);
+	ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_dq_drv",
+				    &timing->phy_lpddr4_dq_drv);
+	ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_odt",
+				    &timing->phy_lpddr4_odt);
+
+	return ret;
+}
+
+static int rk3399_dmcfreq_probe(struct platform_device *pdev)
+{
+	struct arm_smccc_res res;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = pdev->dev.of_node;
+	struct rk3399_dmcfreq *data;
+	int ret, irq, index, size;
+	uint32_t *timing;
+	struct dev_pm_opp *opp;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "Cannot get the dmc interrupt resource\n");
+		return -EINVAL;
+	}
+	data = devm_kzalloc(dev, sizeof(struct rk3399_dmcfreq), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	mutex_init(&data->lock);
+
+	data->vdd_center = devm_regulator_get(dev, "center");
+	if (IS_ERR(data->vdd_center)) {
+		dev_err(dev, "Cannot get the regulator \"center\"\n");
+		return PTR_ERR(data->vdd_center);
+	}
+
+	data->dmc_clk = devm_clk_get(dev, "dmc_clk");
+	if (IS_ERR(data->dmc_clk)) {
+		dev_err(dev, "Cannot get the clk dmc_clk\n");
+		return PTR_ERR(data->dmc_clk);
+	};
+
+	data->irq = irq;
+	ret = devm_request_irq(dev, irq, rk3399_dmc_irq, 0,
+			       dev_name(dev), data);
+	if (ret) {
+		dev_err(dev, "Failed to request dmc irq: %d\n", ret);
+		return ret;
+	}
+
+	init_waitqueue_head(&data->wait_dcf_queue);
+	data->wait_dcf_flag = 0;
+
+	data->edev = devfreq_event_get_edev_by_phandle(dev, 0);
+	if (IS_ERR(data->edev))
+		return -EPROBE_DEFER;
+
+	ret = devfreq_event_enable_edev(data->edev);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable devfreq-event devices\n");
+		return ret;
+	}
+
+	/*
+	 * Get dram timing and pass it to arm trust firmware,
+	 * the dram drvier in arm trust firmware will get these
+	 * timing and to do dram initial.
+	 */
+	if (!of_get_ddr_timings(&data->timing, np)) {
+		timing = &data->timing.ddr3_speed_bin;
+		size = sizeof(struct dram_timing) / 4;
+		for (index = 0; index < size; index++) {
+			arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, *timing++, index,
+				      ROCKCHIP_SIP_CONFIG_DRAM_SET_PARAM,
+				      0, 0, 0, 0, &res);
+			if (res.a0) {
+				dev_err(dev, "Failed to set dram param: %ld\n",
+					res.a0);
+				return -EINVAL;
+			}
+		}
+	}
+
+	arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0,
+		      ROCKCHIP_SIP_CONFIG_DRAM_INIT,
+		      0, 0, 0, 0, &res);
+
+	/*
+	 * We add a devfreq driver to our parent since it has a device tree node
+	 * with operating points.
+	 */
+	if (dev_pm_opp_of_add_table(dev)) {
+		dev_err(dev, "Invalid operating-points in device tree.\n");
+		rcu_read_unlock();
+		return -EINVAL;
+	}
+
+	of_property_read_u32(np, "upthreshold",
+			     &data->ondemand_data.upthreshold);
+	of_property_read_u32(np, "downdifferential",
+			     &data->ondemand_data.downdifferential);
+
+	data->rate = clk_get_rate(data->dmc_clk);
+
+	rcu_read_lock();
+	opp = devfreq_recommended_opp(dev, &data->rate, 0);
+	if (IS_ERR(opp)) {
+		rcu_read_unlock();
+		return PTR_ERR(opp);
+	}
+	rcu_read_unlock();
+	data->curr_opp = opp;
+
+	rk3399_devfreq_dmc_profile.initial_freq = data->rate;
+
+	data->devfreq = devfreq_add_device(dev,
+					   &rk3399_devfreq_dmc_profile,
+					   "simple_ondemand",
+					   &data->ondemand_data);
+	if (IS_ERR(data->devfreq))
+		return PTR_ERR(data->devfreq);
+	devm_devfreq_register_opp_notifier(dev, data->devfreq);
+
+	data->dev = dev;
+	platform_set_drvdata(pdev, data);
+
+	return 0;
+}
+
+static const struct of_device_id rk3399dmc_devfreq_of_match[] = {
+	{ .compatible = "rockchip,rk3399-dmc" },
+	{ },
+};
+
+static struct platform_driver rk3399_dmcfreq_driver = {
+	.probe	= rk3399_dmcfreq_probe,
+	.driver = {
+		.name	= "rk3399-dmc-freq",
+		.pm	= &rk3399_dmcfreq_pm,
+		.of_match_table = rk3399dmc_devfreq_of_match,
+	},
+};
+module_platform_driver(rk3399_dmcfreq_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Lin Huang <hl@rock-chips.com>");
+MODULE_DESCRIPTION("RK3399 dmcfreq driver with devfreq framework");
diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig
index 25bcfa0b474f..2585821b24ab 100644
--- a/drivers/dma-buf/Kconfig
+++ b/drivers/dma-buf/Kconfig
@@ -17,4 +17,17 @@ config SYNC_FILE
 	  Files fds, to the DRM driver for example. More details at
 	  Documentation/sync_file.txt.
 
+config SW_SYNC
+	bool "Sync File Validation Framework"
+	default n
+	depends on SYNC_FILE
+	depends on DEBUG_FS
+	---help---
+	  A sync object driver that uses a 32bit counter to coordinate
+	  synchronization.  Useful when there is no hardware primitive backing
+	  the synchronization.
+
+	  WARNING: improper use of this can result in deadlocking kernel
+	  drivers from userspace. Intended for test and debug only.
+
 endmenu
diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
index f353db213a81..210a10bfad2b 100644
--- a/drivers/dma-buf/Makefile
+++ b/drivers/dma-buf/Makefile
@@ -1,2 +1,3 @@
 obj-y := dma-buf.o fence.o reservation.o seqno-fence.o fence-array.o
 obj-$(CONFIG_SYNC_FILE)		+= sync_file.o
+obj-$(CONFIG_SW_SYNC)		+= sw_sync.o sync_debug.o
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index ddaee60ae52a..cf04d249a6a4 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -586,6 +586,22 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *attach,
 }
 EXPORT_SYMBOL_GPL(dma_buf_unmap_attachment);
 
+static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
+				      enum dma_data_direction direction)
+{
+	bool write = (direction == DMA_BIDIRECTIONAL ||
+		      direction == DMA_TO_DEVICE);
+	struct reservation_object *resv = dmabuf->resv;
+	long ret;
+
+	/* Wait on any implicit rendering fences */
+	ret = reservation_object_wait_timeout_rcu(resv, write, true,
+						  MAX_SCHEDULE_TIMEOUT);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
 
 /**
  * dma_buf_begin_cpu_access - Must be called before accessing a dma_buf from the
@@ -608,6 +624,13 @@ int dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
 	if (dmabuf->ops->begin_cpu_access)
 		ret = dmabuf->ops->begin_cpu_access(dmabuf, direction);
 
+	/* Ensure that all fences are waited upon - but we first allow
+	 * the native handler the chance to do so more efficiently if it
+	 * chooses. A double invocation here will be reasonably cheap no-op.
+	 */
+	if (ret == 0)
+		ret = __dma_buf_begin_cpu_access(dmabuf, direction);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(dma_buf_begin_cpu_access);
diff --git a/drivers/dma-buf/fence-array.c b/drivers/dma-buf/fence-array.c
index a8731c853da6..f1989fcaf354 100644
--- a/drivers/dma-buf/fence-array.c
+++ b/drivers/dma-buf/fence-array.c
@@ -99,6 +99,7 @@ const struct fence_ops fence_array_ops = {
 	.wait = fence_default_wait,
 	.release = fence_array_release,
 };
+EXPORT_SYMBOL(fence_array_ops);
 
 /**
  * fence_array_create - Create a custom fence array
@@ -106,14 +107,14 @@ const struct fence_ops fence_array_ops = {
  * @fences:		[in]	array containing the fences
  * @context:		[in]	fence context to use
  * @seqno:		[in]	sequence number to use
- * @signal_on_any	[in]	signal on any fence in the array
+ * @signal_on_any:	[in]	signal on any fence in the array
  *
  * Allocate a fence_array object and initialize the base fence with fence_init().
  * In case of error it returns NULL.
  *
- * The caller should allocte the fences array with num_fences size
+ * The caller should allocate the fences array with num_fences size
  * and fill it with the fences it wants to add to the object. Ownership of this
- * array is take and fence_put() is used on each fence on release.
+ * array is taken and fence_put() is used on each fence on release.
  *
  * If @signal_on_any is true the fence array signals if any fence in the array
  * signals, otherwise it signals when all fences in the array signal.
diff --git a/drivers/dma-buf/reservation.c b/drivers/dma-buf/reservation.c
index 9566a62ad8e3..723d8af988e5 100644
--- a/drivers/dma-buf/reservation.c
+++ b/drivers/dma-buf/reservation.c
@@ -205,7 +205,7 @@ done:
  * @fence: the shared fence to add
  *
  * Add a fence to a shared slot, obj->lock must be held, and
- * reservation_object_reserve_shared_fence has been called.
+ * reservation_object_reserve_shared() has been called.
  */
 void reservation_object_add_shared_fence(struct reservation_object *obj,
 					 struct fence *fence)
diff --git a/drivers/staging/android/sw_sync.c b/drivers/dma-buf/sw_sync.c
index 115c9174705f..62e8e6dc7953 100644
--- a/drivers/staging/android/sw_sync.c
+++ b/drivers/dma-buf/sw_sync.c
@@ -1,5 +1,5 @@
 /*
- * drivers/dma-buf/sw_sync.c
+ * Sync File validation framework
  *
  * Copyright (C) 2012 Google, Inc.
  *
@@ -23,8 +23,38 @@
 #include "sync_debug.h"
 
 #define CREATE_TRACE_POINTS
-#include "trace/sync.h"
+#include "sync_trace.h"
 
+/*
+ * SW SYNC validation framework
+ *
+ * A sync object driver that uses a 32bit counter to coordinate
+ * synchronization.  Useful when there is no hardware primitive backing
+ * the synchronization.
+ *
+ * To start the framework just open:
+ *
+ * <debugfs>/sync/sw_sync
+ *
+ * That will create a sync timeline, all fences created under this timeline
+ * file descriptor will belong to the this timeline.
+ *
+ * The 'sw_sync' file can be opened many times as to create different
+ * timelines.
+ *
+ * Fences can be created with SW_SYNC_IOC_CREATE_FENCE ioctl with struct
+ * sw_sync_ioctl_create_fence as parameter.
+ *
+ * To increment the timeline counter, SW_SYNC_IOC_INC ioctl should be used
+ * with the increment as u32. This will update the last signaled value
+ * from the timeline and signal any fence that has a seqno smaller or equal
+ * to it.
+ *
+ * struct sw_sync_ioctl_create_fence
+ * @value:	the seqno to initialise the fence with
+ * @name:	the name of the new sync point
+ * @fence:	return the fd of the new sync_file with the created fence
+ */
 struct sw_sync_create_fence_data {
 	__u32	value;
 	char	name[32];
@@ -35,6 +65,7 @@ struct sw_sync_create_fence_data {
 
 #define SW_SYNC_IOC_CREATE_FENCE	_IOWR(SW_SYNC_IOC_MAGIC, 0,\
 		struct sw_sync_create_fence_data)
+
 #define SW_SYNC_IOC_INC			_IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
 
 static const struct fence_ops timeline_fence_ops;
@@ -176,7 +207,7 @@ static void timeline_fence_release(struct fence *fence)
 
 	spin_lock_irqsave(fence->lock, flags);
 	list_del(&pt->child_list);
-	if (WARN_ON_ONCE(!list_empty(&pt->active_list)))
+	if (!list_empty(&pt->active_list))
 		list_del(&pt->active_list);
 	spin_unlock_irqrestore(fence->lock, flags);
 
diff --git a/drivers/staging/android/sync_debug.c b/drivers/dma-buf/sync_debug.c
index 4c5a85595a85..2dd4c3db6caa 100644
--- a/drivers/staging/android/sync_debug.c
+++ b/drivers/dma-buf/sync_debug.c
@@ -1,5 +1,5 @@
 /*
- * drivers/base/sync.c
+ * Sync File validation framework and debug information
  *
  * Copyright (C) 2012 Google, Inc.
  *
@@ -135,10 +135,16 @@ static void sync_print_sync_file(struct seq_file *s,
 	int i;
 
 	seq_printf(s, "[%p] %s: %s\n", sync_file, sync_file->name,
-		   sync_status_str(atomic_read(&sync_file->status)));
+		   sync_status_str(!fence_is_signaled(sync_file->fence)));
 
-	for (i = 0; i < sync_file->num_fences; ++i)
-		sync_print_fence(s, sync_file->cbs[i].fence, true);
+	if (fence_is_array(sync_file->fence)) {
+		struct fence_array *array = to_fence_array(sync_file->fence);
+
+		for (i = 0; i < array->num_fences; ++i)
+			sync_print_fence(s, array->fences[i], true);
+	} else {
+		sync_print_fence(s, sync_file->fence, true);
+	}
 }
 
 static int sync_debugfs_show(struct seq_file *s, void *unused)
diff --git a/drivers/staging/android/sync_debug.h b/drivers/dma-buf/sync_debug.h
index fab66396d421..d269aa6783aa 100644
--- a/drivers/staging/android/sync_debug.h
+++ b/drivers/dma-buf/sync_debug.h
@@ -1,5 +1,5 @@
 /*
- * include/linux/sync.h
+ * Sync File validation framework and debug infomation
  *
  * Copyright (C) 2012 Google, Inc.
  *
diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c
index 9aaa608dfe01..b29a9e817320 100644
--- a/drivers/dma-buf/sync_file.c
+++ b/drivers/dma-buf/sync_file.c
@@ -28,11 +28,11 @@
 
 static const struct file_operations sync_file_fops;
 
-static struct sync_file *sync_file_alloc(int size)
+static struct sync_file *sync_file_alloc(void)
 {
 	struct sync_file *sync_file;
 
-	sync_file = kzalloc(size, GFP_KERNEL);
+	sync_file = kzalloc(sizeof(*sync_file), GFP_KERNEL);
 	if (!sync_file)
 		return NULL;
 
@@ -45,6 +45,8 @@ static struct sync_file *sync_file_alloc(int size)
 
 	init_waitqueue_head(&sync_file->wq);
 
+	INIT_LIST_HEAD(&sync_file->cb.node);
+
 	return sync_file;
 
 err:
@@ -54,14 +56,11 @@ err:
 
 static void fence_check_cb_func(struct fence *f, struct fence_cb *cb)
 {
-	struct sync_file_cb *check;
 	struct sync_file *sync_file;
 
-	check = container_of(cb, struct sync_file_cb, cb);
-	sync_file = check->sync_file;
+	sync_file = container_of(cb, struct sync_file, cb);
 
-	if (atomic_dec_and_test(&sync_file->status))
-		wake_up_all(&sync_file->wq);
+	wake_up_all(&sync_file->wq);
 }
 
 /**
@@ -76,23 +75,17 @@ struct sync_file *sync_file_create(struct fence *fence)
 {
 	struct sync_file *sync_file;
 
-	sync_file = sync_file_alloc(offsetof(struct sync_file, cbs[1]));
+	sync_file = sync_file_alloc();
 	if (!sync_file)
 		return NULL;
 
-	sync_file->num_fences = 1;
-	atomic_set(&sync_file->status, 1);
+	sync_file->fence = fence;
+
 	snprintf(sync_file->name, sizeof(sync_file->name), "%s-%s%llu-%d",
 		 fence->ops->get_driver_name(fence),
 		 fence->ops->get_timeline_name(fence), fence->context,
 		 fence->seqno);
 
-	sync_file->cbs[0].fence = fence;
-	sync_file->cbs[0].sync_file = sync_file;
-	if (fence_add_callback(fence, &sync_file->cbs[0].cb,
-			       fence_check_cb_func))
-		atomic_dec(&sync_file->status);
-
 	return sync_file;
 }
 EXPORT_SYMBOL(sync_file_create);
@@ -121,14 +114,73 @@ err:
 	return NULL;
 }
 
-static void sync_file_add_pt(struct sync_file *sync_file, int *i,
-			     struct fence *fence)
+/**
+ * sync_file_get_fence - get the fence related to the sync_file fd
+ * @fd:		sync_file fd to get the fence from
+ *
+ * Ensures @fd references a valid sync_file and returns a fence that
+ * represents all fence in the sync_file. On error NULL is returned.
+ */
+struct fence *sync_file_get_fence(int fd)
+{
+	struct sync_file *sync_file;
+	struct fence *fence;
+
+	sync_file = sync_file_fdget(fd);
+	if (!sync_file)
+		return NULL;
+
+	fence = fence_get(sync_file->fence);
+	fput(sync_file->file);
+
+	return fence;
+}
+EXPORT_SYMBOL(sync_file_get_fence);
+
+static int sync_file_set_fence(struct sync_file *sync_file,
+			       struct fence **fences, int num_fences)
+{
+	struct fence_array *array;
+
+	/*
+	 * The reference for the fences in the new sync_file and held
+	 * in add_fence() during the merge procedure, so for num_fences == 1
+	 * we already own a new reference to the fence. For num_fence > 1
+	 * we own the reference of the fence_array creation.
+	 */
+	if (num_fences == 1) {
+		sync_file->fence = fences[0];
+		kfree(fences);
+	} else {
+		array = fence_array_create(num_fences, fences,
+					   fence_context_alloc(1), 1, false);
+		if (!array)
+			return -ENOMEM;
+
+		sync_file->fence = &array->base;
+	}
+
+	return 0;
+}
+
+static struct fence **get_fences(struct sync_file *sync_file, int *num_fences)
+{
+	if (fence_is_array(sync_file->fence)) {
+		struct fence_array *array = to_fence_array(sync_file->fence);
+
+		*num_fences = array->num_fences;
+		return array->fences;
+	}
+
+	*num_fences = 1;
+	return &sync_file->fence;
+}
+
+static void add_fence(struct fence **fences, int *i, struct fence *fence)
 {
-	sync_file->cbs[*i].fence = fence;
-	sync_file->cbs[*i].sync_file = sync_file;
+	fences[*i] = fence;
 
-	if (!fence_add_callback(fence, &sync_file->cbs[*i].cb,
-				fence_check_cb_func)) {
+	if (!fence_is_signaled(fence)) {
 		fence_get(fence);
 		(*i)++;
 	}
@@ -147,16 +199,24 @@ static void sync_file_add_pt(struct sync_file *sync_file, int *i,
 static struct sync_file *sync_file_merge(const char *name, struct sync_file *a,
 					 struct sync_file *b)
 {
-	int num_fences = a->num_fences + b->num_fences;
 	struct sync_file *sync_file;
-	int i, i_a, i_b;
-	unsigned long size = offsetof(struct sync_file, cbs[num_fences]);
+	struct fence **fences, **nfences, **a_fences, **b_fences;
+	int i, i_a, i_b, num_fences, a_num_fences, b_num_fences;
 
-	sync_file = sync_file_alloc(size);
+	sync_file = sync_file_alloc();
 	if (!sync_file)
 		return NULL;
 
-	atomic_set(&sync_file->status, num_fences);
+	a_fences = get_fences(a, &a_num_fences);
+	b_fences = get_fences(b, &b_num_fences);
+	if (a_num_fences > INT_MAX - b_num_fences)
+		return NULL;
+
+	num_fences = a_num_fences + b_num_fences;
+
+	fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL);
+	if (!fences)
+		goto err;
 
 	/*
 	 * Assume sync_file a and b are both ordered and have no
@@ -165,55 +225,69 @@ static struct sync_file *sync_file_merge(const char *name, struct sync_file *a,
 	 * If a sync_file can only be created with sync_file_merge
 	 * and sync_file_create, this is a reasonable assumption.
 	 */
-	for (i = i_a = i_b = 0; i_a < a->num_fences && i_b < b->num_fences; ) {
-		struct fence *pt_a = a->cbs[i_a].fence;
-		struct fence *pt_b = b->cbs[i_b].fence;
+	for (i = i_a = i_b = 0; i_a < a_num_fences && i_b < b_num_fences; ) {
+		struct fence *pt_a = a_fences[i_a];
+		struct fence *pt_b = b_fences[i_b];
 
 		if (pt_a->context < pt_b->context) {
-			sync_file_add_pt(sync_file, &i, pt_a);
+			add_fence(fences, &i, pt_a);
 
 			i_a++;
 		} else if (pt_a->context > pt_b->context) {
-			sync_file_add_pt(sync_file, &i, pt_b);
+			add_fence(fences, &i, pt_b);
 
 			i_b++;
 		} else {
 			if (pt_a->seqno - pt_b->seqno <= INT_MAX)
-				sync_file_add_pt(sync_file, &i, pt_a);
+				add_fence(fences, &i, pt_a);
 			else
-				sync_file_add_pt(sync_file, &i, pt_b);
+				add_fence(fences, &i, pt_b);
 
 			i_a++;
 			i_b++;
 		}
 	}
 
-	for (; i_a < a->num_fences; i_a++)
-		sync_file_add_pt(sync_file, &i, a->cbs[i_a].fence);
+	for (; i_a < a_num_fences; i_a++)
+		add_fence(fences, &i, a_fences[i_a]);
+
+	for (; i_b < b_num_fences; i_b++)
+		add_fence(fences, &i, b_fences[i_b]);
 
-	for (; i_b < b->num_fences; i_b++)
-		sync_file_add_pt(sync_file, &i, b->cbs[i_b].fence);
+	if (i == 0)
+		fences[i++] = fence_get(a_fences[0]);
 
-	if (num_fences > i)
-		atomic_sub(num_fences - i, &sync_file->status);
-	sync_file->num_fences = i;
+	if (num_fences > i) {
+		nfences = krealloc(fences, i * sizeof(*fences),
+				  GFP_KERNEL);
+		if (!nfences)
+			goto err;
+
+		fences = nfences;
+	}
+
+	if (sync_file_set_fence(sync_file, fences, i) < 0) {
+		kfree(fences);
+		goto err;
+	}
 
 	strlcpy(sync_file->name, name, sizeof(sync_file->name));
 	return sync_file;
+
+err:
+	fput(sync_file->file);
+	return NULL;
+
 }
 
 static void sync_file_free(struct kref *kref)
 {
 	struct sync_file *sync_file = container_of(kref, struct sync_file,
 						     kref);
-	int i;
-
-	for (i = 0; i < sync_file->num_fences; ++i) {
-		fence_remove_callback(sync_file->cbs[i].fence,
-				      &sync_file->cbs[i].cb);
-		fence_put(sync_file->cbs[i].fence);
-	}
 
+	if (test_bit(POLL_ENABLED, &sync_file->fence->flags))
+		fence_remove_callback(sync_file->fence, &sync_file->cb);
+	fence_put(sync_file->fence);
 	kfree(sync_file);
 }
 
@@ -228,17 +302,17 @@ static int sync_file_release(struct inode *inode, struct file *file)
 static unsigned int sync_file_poll(struct file *file, poll_table *wait)
 {
 	struct sync_file *sync_file = file->private_data;
-	int status;
 
 	poll_wait(file, &sync_file->wq, wait);
 
-	status = atomic_read(&sync_file->status);
+	if (!poll_does_not_wait(wait) &&
+	    !test_and_set_bit(POLL_ENABLED, &sync_file->fence->flags)) {
+		if (fence_add_callback(sync_file->fence, &sync_file->cb,
+				       fence_check_cb_func) < 0)
+			wake_up_all(&sync_file->wq);
+	}
 
-	if (!status)
-		return POLLIN;
-	if (status < 0)
-		return POLLERR;
-	return 0;
+	return fence_is_signaled(sync_file->fence) ? POLLIN : 0;
 }
 
 static long sync_file_ioctl_merge(struct sync_file *sync_file,
@@ -315,8 +389,9 @@ static long sync_file_ioctl_fence_info(struct sync_file *sync_file,
 {
 	struct sync_file_info info;
 	struct sync_fence_info *fence_info = NULL;
+	struct fence **fences;
 	__u32 size;
-	int ret, i;
+	int num_fences, ret, i;
 
 	if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
 		return -EFAULT;
@@ -324,6 +399,8 @@ static long sync_file_ioctl_fence_info(struct sync_file *sync_file,
 	if (info.flags || info.pad)
 		return -EINVAL;
 
+	fences = get_fences(sync_file, &num_fences);
+
 	/*
 	 * Passing num_fences = 0 means that userspace doesn't want to
 	 * retrieve any sync_fence_info. If num_fences = 0 we skip filling
@@ -333,16 +410,16 @@ static long sync_file_ioctl_fence_info(struct sync_file *sync_file,
 	if (!info.num_fences)
 		goto no_fences;
 
-	if (info.num_fences < sync_file->num_fences)
+	if (info.num_fences < num_fences)
 		return -EINVAL;
 
-	size = sync_file->num_fences * sizeof(*fence_info);
+	size = num_fences * sizeof(*fence_info);
 	fence_info = kzalloc(size, GFP_KERNEL);
 	if (!fence_info)
 		return -ENOMEM;
 
-	for (i = 0; i < sync_file->num_fences; ++i)
-		sync_fill_fence_info(sync_file->cbs[i].fence, &fence_info[i]);
+	for (i = 0; i < num_fences; i++)
+		sync_fill_fence_info(fences[i], &fence_info[i]);
 
 	if (copy_to_user(u64_to_user_ptr(info.sync_fence_info), fence_info,
 			 size)) {
@@ -352,11 +429,8 @@ static long sync_file_ioctl_fence_info(struct sync_file *sync_file,
 
 no_fences:
 	strlcpy(info.name, sync_file->name, sizeof(info.name));
-	info.status = atomic_read(&sync_file->status);
-	if (info.status >= 0)
-		info.status = !info.status;
-
-	info.num_fences = sync_file->num_fences;
+	info.status = fence_is_signaled(sync_file->fence);
+	info.num_fences = num_fences;
 
 	if (copy_to_user((void __user *)arg, &info, sizeof(info)))
 		ret = -EFAULT;
diff --git a/drivers/staging/android/trace/sync.h b/drivers/dma-buf/sync_trace.h
index 6b5ce9640ddd..d13d59ff1b85 100644
--- a/drivers/staging/android/trace/sync.h
+++ b/drivers/dma-buf/sync_trace.h
@@ -1,11 +1,11 @@
 #undef TRACE_SYSTEM
-#define TRACE_INCLUDE_PATH ../../drivers/staging/android/trace
-#define TRACE_SYSTEM sync
+#define TRACE_INCLUDE_PATH ../../drivers/dma-buf
+#define TRACE_SYSTEM sync_trace
 
 #if !defined(_TRACE_SYNC_H) || defined(TRACE_HEADER_MULTI_READ)
 #define _TRACE_SYNC_H
 
-#include "../sync_debug.h"
+#include "sync_debug.h"
 #include <linux/tracepoint.h>
 
 TRACE_EVENT(sync_timeline,
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 739f797b40d9..af63a6bcf564 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -102,7 +102,7 @@ config AXI_DMAC
 config COH901318
 	bool "ST-Ericsson COH901318 DMA support"
 	select DMA_ENGINE
-	depends on ARCH_U300
+	depends on ARCH_U300 || COMPILE_TEST
 	help
 	  Enable support for ST-Ericsson COH 901 318 DMA.
 
@@ -114,13 +114,13 @@ config DMA_BCM2835
 
 config DMA_JZ4740
 	tristate "JZ4740 DMA support"
-	depends on MACH_JZ4740
+	depends on MACH_JZ4740 || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
 
 config DMA_JZ4780
 	tristate "JZ4780 DMA support"
-	depends on MACH_JZ4780
+	depends on MACH_JZ4780 || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
 	help
@@ -130,14 +130,14 @@ config DMA_JZ4780
 
 config DMA_OMAP
 	tristate "OMAP DMA support"
-	depends on ARCH_OMAP
+	depends on ARCH_OMAP || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
-	select TI_DMA_CROSSBAR if SOC_DRA7XX
+	select TI_DMA_CROSSBAR if (SOC_DRA7XX || COMPILE_TEST)
 
 config DMA_SA11X0
 	tristate "SA-11x0 DMA support"
-	depends on ARCH_SA1100
+	depends on ARCH_SA1100 || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
 	help
@@ -150,7 +150,6 @@ config DMA_SUN4I
 	depends on MACH_SUN4I || MACH_SUN5I || MACH_SUN7I
 	default (MACH_SUN4I || MACH_SUN5I || MACH_SUN7I)
 	select DMA_ENGINE
-	select DMA_OF
 	select DMA_VIRTUAL_CHANNELS
 	help
 	  Enable support for the DMA controller present in the sun4i,
@@ -167,7 +166,7 @@ config DMA_SUN6I
 
 config EP93XX_DMA
 	bool "Cirrus Logic EP93xx DMA support"
-	depends on ARCH_EP93XX
+	depends on ARCH_EP93XX || COMPILE_TEST
 	select DMA_ENGINE
 	help
 	  Enable support for the Cirrus Logic EP93xx M2P/M2M DMA controller.
@@ -279,7 +278,7 @@ config INTEL_MIC_X100_DMA
 
 config K3_DMA
 	tristate "Hisilicon K3 DMA support"
-	depends on ARCH_HI3xxx
+	depends on ARCH_HI3xxx || ARCH_HISI || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
 	help
@@ -297,16 +296,16 @@ config LPC18XX_DMAMUX
 
 config MMP_PDMA
 	bool "MMP PDMA support"
-	depends on (ARCH_MMP || ARCH_PXA)
+	depends on ARCH_MMP || ARCH_PXA || COMPILE_TEST
 	select DMA_ENGINE
 	help
 	  Support the MMP PDMA engine for PXA and MMP platform.
 
 config MMP_TDMA
 	bool "MMP Two-Channel DMA support"
-	depends on ARCH_MMP
+	depends on ARCH_MMP || COMPILE_TEST
 	select DMA_ENGINE
-	select MMP_SRAM
+	select MMP_SRAM if ARCH_MMP
 	help
 	  Support the MMP Two-Channel DMA engine.
 	  This engine used for MMP Audio DMA and pxa910 SQU.
@@ -316,7 +315,6 @@ config MOXART_DMA
 	tristate "MOXART DMA support"
 	depends on ARCH_MOXART
 	select DMA_ENGINE
-	select DMA_OF
 	select DMA_VIRTUAL_CHANNELS
 	help
 	  Enable support for the MOXA ART SoC DMA controller.
@@ -439,9 +437,8 @@ config STE_DMA40
 
 config STM32_DMA
 	bool "STMicroelectronics STM32 DMA support"
-	depends on ARCH_STM32
+	depends on ARCH_STM32 || COMPILE_TEST
 	select DMA_ENGINE
-	select DMA_OF
 	select DMA_VIRTUAL_CHANNELS
 	help
 	  Enable support for the on-chip DMA controller on STMicroelectronics
@@ -451,7 +448,7 @@ config STM32_DMA
 
 config S3C24XX_DMAC
 	bool "Samsung S3C24XX DMA support"
-	depends on ARCH_S3C24XX
+	depends on ARCH_S3C24XX || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
 	help
@@ -483,10 +480,9 @@ config TEGRA20_APB_DMA
 
 config TEGRA210_ADMA
 	bool "NVIDIA Tegra210 ADMA support"
-	depends on ARCH_TEGRA_210_SOC
+	depends on (ARCH_TEGRA_210_SOC || COMPILE_TEST) && PM_CLK
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
-	select PM_CLK
 	help
 	  Support for the NVIDIA Tegra210 ADMA controller driver. The
 	  DMA controller has multiple DMA channels and is used to service
@@ -497,7 +493,7 @@ config TEGRA210_ADMA
 
 config TIMB_DMA
 	tristate "Timberdale FPGA DMA support"
-	depends on MFD_TIMBERDALE
+	depends on MFD_TIMBERDALE || COMPILE_TEST
 	select DMA_ENGINE
 	help
 	  Enable support for the Timberdale FPGA DMA engine.
@@ -515,10 +511,10 @@ config TI_DMA_CROSSBAR
 
 config TI_EDMA
 	bool "TI EDMA support"
-	depends on ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE
+	depends on ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
-	select TI_DMA_CROSSBAR if ARCH_OMAP
+	select TI_DMA_CROSSBAR if (ARCH_OMAP || COMPILE_TEST)
 	default n
 	help
 	  Enable support for the TI EDMA controller. This DMA
@@ -561,7 +557,7 @@ config XILINX_ZYNQMP_DMA
 
 config ZX_DMA
 	tristate "ZTE ZX296702 DMA support"
-	depends on ARCH_ZX
+	depends on ARCH_ZX || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
 	help
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index 53d22eb73b56..a4c8f80db29d 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -473,15 +473,11 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
 	/* for cyclic transfers,
 	 * no need to replay callback function while stopping */
 	if (!atc_chan_is_cyclic(atchan)) {
-		dma_async_tx_callback	callback = txd->callback;
-		void			*param = txd->callback_param;
-
 		/*
 		 * The API requires that no submissions are done from a
 		 * callback, so we don't need to drop the lock here
 		 */
-		if (callback)
-			callback(param);
+		dmaengine_desc_get_callback_invoke(txd, NULL);
 	}
 
 	dma_run_dependencies(txd);
@@ -598,15 +594,12 @@ static void atc_handle_cyclic(struct at_dma_chan *atchan)
 {
 	struct at_desc			*first = atc_first_active(atchan);
 	struct dma_async_tx_descriptor	*txd = &first->txd;
-	dma_async_tx_callback		callback = txd->callback;
-	void				*param = txd->callback_param;
 
 	dev_vdbg(chan2dev(&atchan->chan_common),
 			"new cyclic period llp 0x%08x\n",
 			channel_readl(atchan, DSCR));
 
-	if (callback)
-		callback(param);
+	dmaengine_desc_get_callback_invoke(txd, NULL);
 }
 
 /*--  IRQ & Tasklet  ---------------------------------------------------*/
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index e434ffe7bc5c..b7d7f2d443a1 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -1572,8 +1572,8 @@ static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)
 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);
 	txd = &desc->tx_dma_desc;
 
-	if (txd->callback && (txd->flags & DMA_PREP_INTERRUPT))
-		txd->callback(txd->callback_param);
+	if (txd->flags & DMA_PREP_INTERRUPT)
+		dmaengine_desc_get_callback_invoke(txd, NULL);
 }
 
 static void at_xdmac_tasklet(unsigned long data)
@@ -1616,8 +1616,8 @@ static void at_xdmac_tasklet(unsigned long data)
 
 		if (!at_xdmac_chan_is_cyclic(atchan)) {
 			dma_cookie_complete(txd);
-			if (txd->callback && (txd->flags & DMA_PREP_INTERRUPT))
-				txd->callback(txd->callback_param);
+			if (txd->flags & DMA_PREP_INTERRUPT)
+				dmaengine_desc_get_callback_invoke(txd, NULL);
 		}
 
 		dma_run_dependencies(txd);
@@ -2067,7 +2067,7 @@ err_dma_unregister:
 err_clk_disable:
 	clk_disable_unprepare(atxdmac->clk);
 err_free_irq:
-	free_irq(atxdmac->irq, atxdmac->dma.dev);
+	free_irq(atxdmac->irq, atxdmac);
 	return ret;
 }
 
@@ -2081,7 +2081,7 @@ static int at_xdmac_remove(struct platform_device *pdev)
 	dma_async_device_unregister(&atxdmac->dma);
 	clk_disable_unprepare(atxdmac->clk);
 
-	free_irq(atxdmac->irq, atxdmac->dma.dev);
+	free_irq(atxdmac->irq, atxdmac);
 
 	for (i = 0; i < atxdmac->dma.chancnt; i++) {
 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];
diff --git a/drivers/dma/bestcomm/bestcomm.c b/drivers/dma/bestcomm/bestcomm.c
index 7ce843723003..7a67b8345092 100644
--- a/drivers/dma/bestcomm/bestcomm.c
+++ b/drivers/dma/bestcomm/bestcomm.c
@@ -82,7 +82,7 @@ bcom_task_alloc(int bd_count, int bd_size, int priv_size)
 
 	/* Get IRQ of that task */
 	tsk->irq = irq_of_parse_and_map(bcom_eng->ofnode, tsk->tasknum);
-	if (tsk->irq == NO_IRQ)
+	if (!tsk->irq)
 		goto error;
 
 	/* Init the BDs, if needed */
@@ -104,7 +104,7 @@ bcom_task_alloc(int bd_count, int bd_size, int priv_size)
 
 error:
 	if (tsk) {
-		if (tsk->irq != NO_IRQ)
+		if (tsk->irq)
 			irq_dispose_mapping(tsk->irq);
 		bcom_sram_free(tsk->bd);
 		kfree(tsk->cookie);
diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c
index e4acd63e42aa..74794c9859f6 100644
--- a/drivers/dma/coh901318.c
+++ b/drivers/dma/coh901318.c
@@ -1319,10 +1319,10 @@ static void coh901318_list_print(struct coh901318_chan *cohc,
 	int i = 0;
 
 	while (l) {
-		dev_vdbg(COHC_2_DEV(cohc), "i %d, lli %p, ctrl 0x%x, src 0x%x"
-			 ", dst 0x%x, link 0x%x virt_link_addr 0x%p\n",
-			 i, l, l->control, l->src_addr, l->dst_addr,
-			 l->link_addr, l->virt_link_addr);
+		dev_vdbg(COHC_2_DEV(cohc), "i %d, lli %p, ctrl 0x%x, src 0x%pad"
+			 ", dst 0x%pad, link 0x%pad virt_link_addr 0x%p\n",
+			 i, l, l->control, &l->src_addr, &l->dst_addr,
+			 &l->link_addr, l->virt_link_addr);
 		i++;
 		l = l->virt_link_addr;
 	}
@@ -1335,7 +1335,7 @@ static void coh901318_list_print(struct coh901318_chan *cohc,
 static struct coh901318_base *debugfs_dma_base;
 static struct dentry *dma_dentry;
 
-static int coh901318_debugfs_read(struct file *file, char __user *buf,
+static ssize_t coh901318_debugfs_read(struct file *file, char __user *buf,
 				  size_t count, loff_t *f_pos)
 {
 	u64 started_channels = debugfs_dma_base->pm.started_channels;
@@ -1352,9 +1352,10 @@ static int coh901318_debugfs_read(struct file *file, char __user *buf,
 
 	tmp += sprintf(tmp, "DMA -- enabled dma channels\n");
 
-	for (i = 0; i < U300_DMA_CHANNELS; i++)
-		if (started_channels & (1 << i))
+	for (i = 0; i < U300_DMA_CHANNELS; i++) {
+		if (started_channels & (1ULL << i))
 			tmp += sprintf(tmp, "channel %d\n", i);
+	}
 
 	tmp += sprintf(tmp, "Pool alloc nbr %d\n", pool_count);
 
@@ -1553,15 +1554,8 @@ coh901318_desc_submit(struct coh901318_chan *cohc, struct coh901318_desc *desc)
 static struct coh901318_desc *
 coh901318_first_active_get(struct coh901318_chan *cohc)
 {
-	struct coh901318_desc *d;
-
-	if (list_empty(&cohc->active))
-		return NULL;
-
-	d = list_first_entry(&cohc->active,
-			     struct coh901318_desc,
-			     node);
-	return d;
+	return list_first_entry_or_null(&cohc->active, struct coh901318_desc,
+					node);
 }
 
 static void
@@ -1579,15 +1573,8 @@ coh901318_desc_queue(struct coh901318_chan *cohc, struct coh901318_desc *desc)
 static struct coh901318_desc *
 coh901318_first_queued(struct coh901318_chan *cohc)
 {
-	struct coh901318_desc *d;
-
-	if (list_empty(&cohc->queue))
-		return NULL;
-
-	d = list_first_entry(&cohc->queue,
-			     struct coh901318_desc,
-			     node);
-	return d;
+	return list_first_entry_or_null(&cohc->queue, struct coh901318_desc,
+					node);
 }
 
 static inline u32 coh901318_get_bytes_in_lli(struct coh901318_lli *in_lli)
@@ -1766,7 +1753,7 @@ static int coh901318_resume(struct dma_chan *chan)
 
 bool coh901318_filter_id(struct dma_chan *chan, void *chan_id)
 {
-	unsigned int ch_nr = (unsigned int) chan_id;
+	unsigned long ch_nr = (unsigned long) chan_id;
 
 	if (ch_nr == to_coh901318_chan(chan)->id)
 		return true;
@@ -1888,8 +1875,7 @@ static void dma_tasklet(unsigned long data)
 	struct coh901318_chan *cohc = (struct coh901318_chan *) data;
 	struct coh901318_desc *cohd_fin;
 	unsigned long flags;
-	dma_async_tx_callback callback;
-	void *callback_param;
+	struct dmaengine_desc_callback cb;
 
 	dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d"
 		 " nbr_active_done %ld\n", __func__,
@@ -1904,8 +1890,7 @@ static void dma_tasklet(unsigned long data)
 		goto err;
 
 	/* locate callback to client */
-	callback = cohd_fin->desc.callback;
-	callback_param = cohd_fin->desc.callback_param;
+	dmaengine_desc_get_callback(&cohd_fin->desc, &cb);
 
 	/* sign this job as completed on the channel */
 	dma_cookie_complete(&cohd_fin->desc);
@@ -1920,8 +1905,7 @@ static void dma_tasklet(unsigned long data)
 	spin_unlock_irqrestore(&cohc->lock, flags);
 
 	/* Call the callback when we're done */
-	if (callback)
-		callback(callback_param);
+	dmaengine_desc_callback_invoke(&cb, NULL);
 
 	spin_lock_irqsave(&cohc->lock, flags);
 
@@ -2247,8 +2231,8 @@ coh901318_prep_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 	spin_lock_irqsave(&cohc->lock, flg);
 
 	dev_vdbg(COHC_2_DEV(cohc),
-		 "[%s] channel %d src 0x%x dest 0x%x size %d\n",
-		 __func__, cohc->id, src, dest, size);
+		 "[%s] channel %d src 0x%pad dest 0x%pad size %zu\n",
+		 __func__, cohc->id, &src, &dest, size);
 
 	if (flags & DMA_PREP_INTERRUPT)
 		/* Trigger interrupt after last lli */
@@ -2744,8 +2728,8 @@ static int __init coh901318_probe(struct platform_device *pdev)
 		goto err_register_of_dma;
 
 	platform_set_drvdata(pdev, base);
-	dev_info(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%08x\n",
-		(u32) base->virtbase);
+	dev_info(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%p\n",
+		base->virtbase);
 
 	return err;
 
diff --git a/drivers/dma/coh901318_lli.c b/drivers/dma/coh901318_lli.c
index 702112d547c8..d612b2e5abc4 100644
--- a/drivers/dma/coh901318_lli.c
+++ b/drivers/dma/coh901318_lli.c
@@ -75,7 +75,7 @@ coh901318_lli_alloc(struct coh901318_pool *pool, unsigned int len)
 	lli = head;
 	lli->phy_this = phy;
 	lli->link_addr = 0x00000000;
-	lli->virt_link_addr = 0x00000000U;
+	lli->virt_link_addr = NULL;
 
 	for (i = 1; i < len; i++) {
 		lli_prev = lli;
@@ -88,7 +88,7 @@ coh901318_lli_alloc(struct coh901318_pool *pool, unsigned int len)
 		DEBUGFS_POOL_COUNTER_ADD(pool, 1);
 		lli->phy_this = phy;
 		lli->link_addr = 0x00000000;
-		lli->virt_link_addr = 0x00000000U;
+		lli->virt_link_addr = NULL;
 
 		lli_prev->link_addr = phy;
 		lli_prev->virt_link_addr = lli;
diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c
index 4b2317426c8e..bac5f023013b 100644
--- a/drivers/dma/cppi41.c
+++ b/drivers/dma/cppi41.c
@@ -108,6 +108,8 @@ struct cppi41_channel {
 	unsigned td_queued:1;
 	unsigned td_seen:1;
 	unsigned td_desc_seen:1;
+
+	struct list_head node;		/* Node for pending list */
 };
 
 struct cppi41_desc {
@@ -146,6 +148,9 @@ struct cppi41_dd {
 	const struct chan_queues *queues_tx;
 	struct chan_queues td_queue;
 
+	struct list_head pending;	/* Pending queued transfers */
+	spinlock_t lock;		/* Lock for pending list */
+
 	/* context for suspend/resume */
 	unsigned int dma_tdfdq;
 };
@@ -331,7 +336,11 @@ static irqreturn_t cppi41_irq(int irq, void *data)
 
 			c->residue = pd_trans_len(c->desc->pd6) - len;
 			dma_cookie_complete(&c->txd);
-			c->txd.callback(c->txd.callback_param);
+			dmaengine_desc_get_callback_invoke(&c->txd, NULL);
+
+			/* Paired with cppi41_dma_issue_pending */
+			pm_runtime_mark_last_busy(cdd->ddev.dev);
+			pm_runtime_put_autosuspend(cdd->ddev.dev);
 		}
 	}
 	return IRQ_HANDLED;
@@ -349,6 +358,12 @@ static dma_cookie_t cppi41_tx_submit(struct dma_async_tx_descriptor *tx)
 static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct cppi41_channel *c = to_cpp41_chan(chan);
+	struct cppi41_dd *cdd = c->cdd;
+	int error;
+
+	error = pm_runtime_get_sync(cdd->ddev.dev);
+	if (error < 0)
+		return error;
 
 	dma_cookie_init(chan);
 	dma_async_tx_descriptor_init(&c->txd, chan);
@@ -357,11 +372,26 @@ static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan)
 	if (!c->is_tx)
 		cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
 
+	pm_runtime_mark_last_busy(cdd->ddev.dev);
+	pm_runtime_put_autosuspend(cdd->ddev.dev);
+
 	return 0;
 }
 
 static void cppi41_dma_free_chan_resources(struct dma_chan *chan)
 {
+	struct cppi41_channel *c = to_cpp41_chan(chan);
+	struct cppi41_dd *cdd = c->cdd;
+	int error;
+
+	error = pm_runtime_get_sync(cdd->ddev.dev);
+	if (error < 0)
+		return;
+
+	WARN_ON(!list_empty(&cdd->pending));
+
+	pm_runtime_mark_last_busy(cdd->ddev.dev);
+	pm_runtime_put_autosuspend(cdd->ddev.dev);
 }
 
 static enum dma_status cppi41_dma_tx_status(struct dma_chan *chan,
@@ -386,21 +416,6 @@ static void push_desc_queue(struct cppi41_channel *c)
 	u32 desc_phys;
 	u32 reg;
 
-	desc_phys = lower_32_bits(c->desc_phys);
-	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
-	WARN_ON(cdd->chan_busy[desc_num]);
-	cdd->chan_busy[desc_num] = c;
-
-	reg = (sizeof(struct cppi41_desc) - 24) / 4;
-	reg |= desc_phys;
-	cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
-}
-
-static void cppi41_dma_issue_pending(struct dma_chan *chan)
-{
-	struct cppi41_channel *c = to_cpp41_chan(chan);
-	u32 reg;
-
 	c->residue = 0;
 
 	reg = GCR_CHAN_ENABLE;
@@ -418,7 +433,46 @@ static void cppi41_dma_issue_pending(struct dma_chan *chan)
 	 * before starting the dma engine.
 	 */
 	__iowmb();
-	push_desc_queue(c);
+
+	desc_phys = lower_32_bits(c->desc_phys);
+	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
+	WARN_ON(cdd->chan_busy[desc_num]);
+	cdd->chan_busy[desc_num] = c;
+
+	reg = (sizeof(struct cppi41_desc) - 24) / 4;
+	reg |= desc_phys;
+	cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
+}
+
+static void pending_desc(struct cppi41_channel *c)
+{
+	struct cppi41_dd *cdd = c->cdd;
+	unsigned long flags;
+
+	spin_lock_irqsave(&cdd->lock, flags);
+	list_add_tail(&c->node, &cdd->pending);
+	spin_unlock_irqrestore(&cdd->lock, flags);
+}
+
+static void cppi41_dma_issue_pending(struct dma_chan *chan)
+{
+	struct cppi41_channel *c = to_cpp41_chan(chan);
+	struct cppi41_dd *cdd = c->cdd;
+	int error;
+
+	/* PM runtime paired with dmaengine_desc_get_callback_invoke */
+	error = pm_runtime_get(cdd->ddev.dev);
+	if ((error != -EINPROGRESS) && error < 0) {
+		dev_err(cdd->ddev.dev, "Failed to pm_runtime_get: %i\n",
+			error);
+
+		return;
+	}
+
+	if (likely(pm_runtime_active(cdd->ddev.dev)))
+		push_desc_queue(c);
+	else
+		pending_desc(c);
 }
 
 static u32 get_host_pd0(u32 length)
@@ -940,12 +994,18 @@ static int cppi41_dma_probe(struct platform_device *pdev)
 	cdd->ctrl_mem = of_iomap(dev->of_node, 1);
 	cdd->sched_mem = of_iomap(dev->of_node, 2);
 	cdd->qmgr_mem = of_iomap(dev->of_node, 3);
+	spin_lock_init(&cdd->lock);
+	INIT_LIST_HEAD(&cdd->pending);
+
+	platform_set_drvdata(pdev, cdd);
 
 	if (!cdd->usbss_mem || !cdd->ctrl_mem || !cdd->sched_mem ||
 			!cdd->qmgr_mem)
 		return -ENXIO;
 
 	pm_runtime_enable(dev);
+	pm_runtime_set_autosuspend_delay(dev, 100);
+	pm_runtime_use_autosuspend(dev);
 	ret = pm_runtime_get_sync(dev);
 	if (ret < 0)
 		goto err_get_sync;
@@ -985,7 +1045,9 @@ static int cppi41_dma_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_of;
 
-	platform_set_drvdata(pdev, cdd);
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+
 	return 0;
 err_of:
 	dma_async_device_unregister(&cdd->ddev);
@@ -996,7 +1058,8 @@ err_irq:
 err_chans:
 	deinit_cppi41(dev, cdd);
 err_init_cppi:
-	pm_runtime_put(dev);
+	pm_runtime_dont_use_autosuspend(dev);
+	pm_runtime_put_sync(dev);
 err_get_sync:
 	pm_runtime_disable(dev);
 	iounmap(cdd->usbss_mem);
@@ -1021,13 +1084,13 @@ static int cppi41_dma_remove(struct platform_device *pdev)
 	iounmap(cdd->ctrl_mem);
 	iounmap(cdd->sched_mem);
 	iounmap(cdd->qmgr_mem);
-	pm_runtime_put(&pdev->dev);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
+	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int cppi41_suspend(struct device *dev)
+static int __maybe_unused cppi41_suspend(struct device *dev)
 {
 	struct cppi41_dd *cdd = dev_get_drvdata(dev);
 
@@ -1038,7 +1101,7 @@ static int cppi41_suspend(struct device *dev)
 	return 0;
 }
 
-static int cppi41_resume(struct device *dev)
+static int __maybe_unused cppi41_resume(struct device *dev)
 {
 	struct cppi41_dd *cdd = dev_get_drvdata(dev);
 	struct cppi41_channel *c;
@@ -1062,9 +1125,38 @@ static int cppi41_resume(struct device *dev)
 
 	return 0;
 }
-#endif
 
-static SIMPLE_DEV_PM_OPS(cppi41_pm_ops, cppi41_suspend, cppi41_resume);
+static int __maybe_unused cppi41_runtime_suspend(struct device *dev)
+{
+	struct cppi41_dd *cdd = dev_get_drvdata(dev);
+
+	WARN_ON(!list_empty(&cdd->pending));
+
+	return 0;
+}
+
+static int __maybe_unused cppi41_runtime_resume(struct device *dev)
+{
+	struct cppi41_dd *cdd = dev_get_drvdata(dev);
+	struct cppi41_channel *c, *_c;
+	unsigned long flags;
+
+	spin_lock_irqsave(&cdd->lock, flags);
+	list_for_each_entry_safe(c, _c, &cdd->pending, node) {
+		push_desc_queue(c);
+		list_del(&c->node);
+	}
+	spin_unlock_irqrestore(&cdd->lock, flags);
+
+	return 0;
+}
+
+static const struct dev_pm_ops cppi41_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(cppi41_suspend, cppi41_resume)
+	SET_RUNTIME_PM_OPS(cppi41_runtime_suspend,
+			   cppi41_runtime_resume,
+			   NULL)
+};
 
 static struct platform_driver cpp41_dma_driver = {
 	.probe  = cppi41_dma_probe,
diff --git a/drivers/dma/dma-jz4740.c b/drivers/dma/dma-jz4740.c
index 9689b36c005a..d50273fed715 100644
--- a/drivers/dma/dma-jz4740.c
+++ b/drivers/dma/dma-jz4740.c
@@ -21,8 +21,6 @@
 #include <linux/irq.h>
 #include <linux/clk.h>
 
-#include <asm/mach-jz4740/dma.h>
-
 #include "virt-dma.h"
 
 #define JZ_DMA_NR_CHANS 6
diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c
index dade7c47ff18..7373b7a555ec 100644
--- a/drivers/dma/dma-jz4780.c
+++ b/drivers/dma/dma-jz4780.c
@@ -324,8 +324,10 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg(
 					      sg_dma_address(&sgl[i]),
 					      sg_dma_len(&sgl[i]),
 					      direction);
-		if (err < 0)
+		if (err < 0) {
+			jz4780_dma_desc_free(&jzchan->desc->vdesc);
 			return NULL;
+		}
 
 		desc->desc[i].dcm |= JZ_DMA_DCM_TIE;
 
@@ -368,8 +370,10 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic(
 	for (i = 0; i < periods; i++) {
 		err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr,
 					      period_len, direction);
-		if (err < 0)
+		if (err < 0) {
+			jz4780_dma_desc_free(&jzchan->desc->vdesc);
 			return NULL;
+		}
 
 		buf_addr += period_len;
 
@@ -396,7 +400,7 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic(
 	return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);
 }
 
-struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy(
+static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy(
 	struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 	size_t len, unsigned long flags)
 {
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 8c9f45fd55fc..6b535262ac5d 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -997,6 +997,13 @@ int dma_async_device_register(struct dma_device *device)
 		}
 		chan->client_count = 0;
 	}
+
+	if (!chancnt) {
+		dev_err(device->dev, "%s: device has no channels!\n", __func__);
+		rc = -ENODEV;
+		goto err_out;
+	}
+
 	device->chancnt = chancnt;
 
 	mutex_lock(&dma_list_mutex);
diff --git a/drivers/dma/dmaengine.h b/drivers/dma/dmaengine.h
index 17f983a4e9ba..882ff9448c3b 100644
--- a/drivers/dma/dmaengine.h
+++ b/drivers/dma/dmaengine.h
@@ -86,4 +86,88 @@ static inline void dma_set_residue(struct dma_tx_state *state, u32 residue)
 		state->residue = residue;
 }
 
+struct dmaengine_desc_callback {
+	dma_async_tx_callback callback;
+	dma_async_tx_callback_result callback_result;
+	void *callback_param;
+};
+
+/**
+ * dmaengine_desc_get_callback - get the passed in callback function
+ * @tx: tx descriptor
+ * @cb: temp struct to hold the callback info
+ *
+ * Fill the passed in cb struct with what's available in the passed in
+ * tx descriptor struct
+ * No locking is required.
+ */
+static inline void
+dmaengine_desc_get_callback(struct dma_async_tx_descriptor *tx,
+			    struct dmaengine_desc_callback *cb)
+{
+	cb->callback = tx->callback;
+	cb->callback_result = tx->callback_result;
+	cb->callback_param = tx->callback_param;
+}
+
+/**
+ * dmaengine_desc_callback_invoke - call the callback function in cb struct
+ * @cb: temp struct that is holding the callback info
+ * @result: transaction result
+ *
+ * Call the callback function provided in the cb struct with the parameter
+ * in the cb struct.
+ * Locking is dependent on the driver.
+ */
+static inline void
+dmaengine_desc_callback_invoke(struct dmaengine_desc_callback *cb,
+			       const struct dmaengine_result *result)
+{
+	struct dmaengine_result dummy_result = {
+		.result = DMA_TRANS_NOERROR,
+		.residue = 0
+	};
+
+	if (cb->callback_result) {
+		if (!result)
+			result = &dummy_result;
+		cb->callback_result(cb->callback_param, result);
+	} else if (cb->callback) {
+		cb->callback(cb->callback_param);
+	}
+}
+
+/**
+ * dmaengine_desc_get_callback_invoke - get the callback in tx descriptor and
+ * 					then immediately call the callback.
+ * @tx: dma async tx descriptor
+ * @result: transaction result
+ *
+ * Call dmaengine_desc_get_callback() and dmaengine_desc_callback_invoke()
+ * in a single function since no work is necessary in between for the driver.
+ * Locking is dependent on the driver.
+ */
+static inline void
+dmaengine_desc_get_callback_invoke(struct dma_async_tx_descriptor *tx,
+				   const struct dmaengine_result *result)
+{
+	struct dmaengine_desc_callback cb;
+
+	dmaengine_desc_get_callback(tx, &cb);
+	dmaengine_desc_callback_invoke(&cb, result);
+}
+
+/**
+ * dmaengine_desc_callback_valid - verify the callback is valid in cb
+ * @cb: callback info struct
+ *
+ * Return a bool that verifies whether callback in cb is valid or not.
+ * No locking is required.
+ */
+static inline bool
+dmaengine_desc_callback_valid(struct dmaengine_desc_callback *cb)
+{
+	return (cb->callback) ? true : false;
+}
+
 #endif
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index 1245db5438e1..cf76fc6149e5 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -56,10 +56,10 @@ module_param(sg_buffers, uint, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(sg_buffers,
 		"Number of scatter gather buffers (default: 1)");
 
-static unsigned int dmatest = 1;
+static unsigned int dmatest;
 module_param(dmatest, uint, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(dmatest,
-		"dmatest 0-memcpy 1-slave_sg (default: 1)");
+		"dmatest 0-memcpy 1-slave_sg (default: 0)");
 
 static unsigned int xor_sources = 3;
 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
@@ -426,7 +426,9 @@ static int dmatest_func(void *data)
 	int			src_cnt;
 	int			dst_cnt;
 	int			i;
-	ktime_t			ktime;
+	ktime_t			ktime, start, diff;
+	ktime_t			filltime = ktime_set(0, 0);
+	ktime_t			comparetime = ktime_set(0, 0);
 	s64			runtime = 0;
 	unsigned long long	total_len = 0;
 
@@ -503,7 +505,7 @@ static int dmatest_func(void *data)
 		total_tests++;
 
 		/* honor alignment restrictions */
-		if (thread->type == DMA_MEMCPY)
+		if (thread->type == DMA_MEMCPY || thread->type == DMA_SG)
 			align = dev->copy_align;
 		else if (thread->type == DMA_XOR)
 			align = dev->xor_align;
@@ -531,6 +533,7 @@ static int dmatest_func(void *data)
 			src_off = 0;
 			dst_off = 0;
 		} else {
+			start = ktime_get();
 			src_off = dmatest_random() % (params->buf_size - len + 1);
 			dst_off = dmatest_random() % (params->buf_size - len + 1);
 
@@ -541,6 +544,9 @@ static int dmatest_func(void *data)
 					  params->buf_size);
 			dmatest_init_dsts(thread->dsts, dst_off, len,
 					  params->buf_size);
+
+			diff = ktime_sub(ktime_get(), start);
+			filltime = ktime_add(filltime, diff);
 		}
 
 		um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
@@ -683,6 +689,7 @@ static int dmatest_func(void *data)
 			continue;
 		}
 
+		start = ktime_get();
 		pr_debug("%s: verifying source buffer...\n", current->comm);
 		error_count = dmatest_verify(thread->srcs, 0, src_off,
 				0, PATTERN_SRC, true);
@@ -703,6 +710,9 @@ static int dmatest_func(void *data)
 				params->buf_size, dst_off + len,
 				PATTERN_DST, false);
 
+		diff = ktime_sub(ktime_get(), start);
+		comparetime = ktime_add(comparetime, diff);
+
 		if (error_count) {
 			result("data error", total_tests, src_off, dst_off,
 			       len, error_count);
@@ -712,7 +722,10 @@ static int dmatest_func(void *data)
 				       dst_off, len, 0);
 		}
 	}
-	runtime = ktime_us_delta(ktime_get(), ktime);
+	ktime = ktime_sub(ktime_get(), ktime);
+	ktime = ktime_sub(ktime, comparetime);
+	ktime = ktime_sub(ktime, filltime);
+	runtime = ktime_to_us(ktime);
 
 	ret = 0;
 err_dstbuf:
diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c
index edf053f73a49..c2c0a613cb7a 100644
--- a/drivers/dma/dw/core.c
+++ b/drivers/dma/dw/core.c
@@ -46,9 +46,9 @@
 		u8 _dmsize = _is_slave ? _sconfig->dst_maxburst :	\
 			DW_DMA_MSIZE_16;			\
 		u8 _dms = (_dwc->direction == DMA_MEM_TO_DEV) ?		\
-			_dwc->p_master : _dwc->m_master;		\
+			_dwc->dws.p_master : _dwc->dws.m_master;	\
 		u8 _sms = (_dwc->direction == DMA_DEV_TO_MEM) ?		\
-			_dwc->p_master : _dwc->m_master;		\
+			_dwc->dws.p_master : _dwc->dws.m_master;	\
 								\
 		(DWC_CTLL_DST_MSIZE(_dmsize)			\
 		 | DWC_CTLL_SRC_MSIZE(_smsize)			\
@@ -143,12 +143,16 @@ static void dwc_initialize(struct dw_dma_chan *dwc)
 	struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 	u32 cfghi = DWC_CFGH_FIFO_MODE;
 	u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority);
+	bool hs_polarity = dwc->dws.hs_polarity;
 
 	if (test_bit(DW_DMA_IS_INITIALIZED, &dwc->flags))
 		return;
 
-	cfghi |= DWC_CFGH_DST_PER(dwc->dst_id);
-	cfghi |= DWC_CFGH_SRC_PER(dwc->src_id);
+	cfghi |= DWC_CFGH_DST_PER(dwc->dws.dst_id);
+	cfghi |= DWC_CFGH_SRC_PER(dwc->dws.src_id);
+
+	/* Set polarity of handshake interface */
+	cfglo |= hs_polarity ? DWC_CFGL_HS_DST_POL | DWC_CFGL_HS_SRC_POL : 0;
 
 	channel_writel(dwc, CFG_LO, cfglo);
 	channel_writel(dwc, CFG_HI, cfghi);
@@ -209,7 +213,7 @@ static inline void dwc_do_single_block(struct dw_dma_chan *dwc,
 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
 {
 	struct dw_dma	*dw = to_dw_dma(dwc->chan.device);
-	u8		lms = DWC_LLP_LMS(dwc->m_master);
+	u8		lms = DWC_LLP_LMS(dwc->dws.m_master);
 	unsigned long	was_soft_llp;
 
 	/* ASSERT:  channel is idle */
@@ -270,20 +274,19 @@ static void
 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
 		bool callback_required)
 {
-	dma_async_tx_callback		callback = NULL;
-	void				*param = NULL;
 	struct dma_async_tx_descriptor	*txd = &desc->txd;
 	struct dw_desc			*child;
 	unsigned long			flags;
+	struct dmaengine_desc_callback	cb;
 
 	dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
 
 	spin_lock_irqsave(&dwc->lock, flags);
 	dma_cookie_complete(txd);
-	if (callback_required) {
-		callback = txd->callback;
-		param = txd->callback_param;
-	}
+	if (callback_required)
+		dmaengine_desc_get_callback(txd, &cb);
+	else
+		memset(&cb, 0, sizeof(cb));
 
 	/* async_tx_ack */
 	list_for_each_entry(child, &desc->tx_list, desc_node)
@@ -292,8 +295,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
 	dwc_desc_put(dwc, desc);
 	spin_unlock_irqrestore(&dwc->lock, flags);
 
-	if (callback)
-		callback(param);
+	dmaengine_desc_callback_invoke(&cb, NULL);
 }
 
 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
@@ -662,7 +664,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 	struct dw_desc		*prev;
 	size_t			xfer_count;
 	size_t			offset;
-	u8			m_master = dwc->m_master;
+	u8			m_master = dwc->dws.m_master;
 	unsigned int		src_width;
 	unsigned int		dst_width;
 	unsigned int		data_width = dw->pdata->data_width[m_master];
@@ -740,7 +742,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 	struct dw_desc		*prev;
 	struct dw_desc		*first;
 	u32			ctllo;
-	u8			m_master = dwc->m_master;
+	u8			m_master = dwc->dws.m_master;
 	u8			lms = DWC_LLP_LMS(m_master);
 	dma_addr_t		reg;
 	unsigned int		reg_width;
@@ -895,12 +897,7 @@ bool dw_dma_filter(struct dma_chan *chan, void *param)
 		return false;
 
 	/* We have to copy data since dws can be temporary storage */
-
-	dwc->src_id = dws->src_id;
-	dwc->dst_id = dws->dst_id;
-
-	dwc->m_master = dws->m_master;
-	dwc->p_master = dws->p_master;
+	memcpy(&dwc->dws, dws, sizeof(struct dw_dma_slave));
 
 	return true;
 }
@@ -1167,11 +1164,7 @@ static void dwc_free_chan_resources(struct dma_chan *chan)
 	spin_lock_irqsave(&dwc->lock, flags);
 
 	/* Clear custom channel configuration */
-	dwc->src_id = 0;
-	dwc->dst_id = 0;
-
-	dwc->m_master = 0;
-	dwc->p_master = 0;
+	memset(&dwc->dws, 0, sizeof(struct dw_dma_slave));
 
 	clear_bit(DW_DMA_IS_INITIALIZED, &dwc->flags);
 
@@ -1264,7 +1257,7 @@ struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
 	struct dw_cyclic_desc		*retval = NULL;
 	struct dw_desc			*desc;
 	struct dw_desc			*last = NULL;
-	u8				lms = DWC_LLP_LMS(dwc->m_master);
+	u8				lms = DWC_LLP_LMS(dwc->dws.m_master);
 	unsigned long			was_cyclic;
 	unsigned int			reg_width;
 	unsigned int			periods;
@@ -1576,11 +1569,7 @@ int dw_dma_probe(struct dw_dma_chip *chip)
 				(dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0;
 		} else {
 			dwc->block_size = pdata->block_size;
-
-			/* Check if channel supports multi block transfer */
-			channel_writel(dwc, LLP, DWC_LLP_LOC(0xffffffff));
-			dwc->nollp = DWC_LLP_LOC(channel_readl(dwc, LLP)) == 0;
-			channel_writel(dwc, LLP, 0);
+			dwc->nollp = pdata->is_nollp;
 		}
 	}
 
diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h
index 4b7bd7834046..f65dd104479f 100644
--- a/drivers/dma/dw/regs.h
+++ b/drivers/dma/dw/regs.h
@@ -245,10 +245,7 @@ struct dw_dma_chan {
 	bool			nollp;
 
 	/* custom slave configuration */
-	u8			src_id;
-	u8			dst_id;
-	u8			m_master;
-	u8			p_master;
+	struct dw_dma_slave	dws;
 
 	/* configuration passed via .device_config */
 	struct dma_slave_config dma_sconfig;
diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c
index 3d277fa76c1a..e18a58068bca 100644
--- a/drivers/dma/edma.c
+++ b/drivers/dma/edma.c
@@ -263,22 +263,29 @@ static const struct edmacc_param dummy_paramset = {
 
 #define EDMA_BINDING_LEGACY	0
 #define EDMA_BINDING_TPCC	1
+static const u32 edma_binding_type[] = {
+	[EDMA_BINDING_LEGACY] = EDMA_BINDING_LEGACY,
+	[EDMA_BINDING_TPCC] = EDMA_BINDING_TPCC,
+};
+
 static const struct of_device_id edma_of_ids[] = {
 	{
 		.compatible = "ti,edma3",
-		.data = (void *)EDMA_BINDING_LEGACY,
+		.data = &edma_binding_type[EDMA_BINDING_LEGACY],
 	},
 	{
 		.compatible = "ti,edma3-tpcc",
-		.data = (void *)EDMA_BINDING_TPCC,
+		.data = &edma_binding_type[EDMA_BINDING_TPCC],
 	},
 	{}
 };
+MODULE_DEVICE_TABLE(of, edma_of_ids);
 
 static const struct of_device_id edma_tptc_of_ids[] = {
 	{ .compatible = "ti,edma3-tptc", },
 	{}
 };
+MODULE_DEVICE_TABLE(of, edma_tptc_of_ids);
 
 static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
 {
@@ -405,18 +412,12 @@ static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no,
 	edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
 }
 
-static inline void set_bits(int offset, int len, unsigned long *p)
+static inline void edma_set_bits(int offset, int len, unsigned long *p)
 {
 	for (; len > 0; len--)
 		set_bit(offset + (len - 1), p);
 }
 
-static inline void clear_bits(int offset, int len, unsigned long *p)
-{
-	for (; len > 0; len--)
-		clear_bit(offset + (len - 1), p);
-}
-
 static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
 					  int priority)
 {
@@ -464,13 +465,15 @@ static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
 	memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
 }
 
-static void edma_read_slot(struct edma_cc *ecc, unsigned slot,
+static int edma_read_slot(struct edma_cc *ecc, unsigned slot,
 			   struct edmacc_param *param)
 {
 	slot = EDMA_CHAN_SLOT(slot);
 	if (slot >= ecc->num_slots)
-		return;
+		return -EINVAL;
 	memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
+
+	return 0;
 }
 
 /**
@@ -1476,13 +1479,15 @@ static void edma_error_handler(struct edma_chan *echan)
 	struct edma_cc *ecc = echan->ecc;
 	struct device *dev = echan->vchan.chan.device->dev;
 	struct edmacc_param p;
+	int err;
 
 	if (!echan->edesc)
 		return;
 
 	spin_lock(&echan->vchan.lock);
 
-	edma_read_slot(ecc, echan->slot[0], &p);
+	err = edma_read_slot(ecc, echan->slot[0], &p);
+
 	/*
 	 * Issue later based on missed flag which will be sure
 	 * to happen as:
@@ -1495,7 +1500,7 @@ static void edma_error_handler(struct edma_chan *echan)
 	 * lead to some nasty recursion when we are in a NULL
 	 * slot. So we avoid doing so and set the missed flag.
 	 */
-	if (p.a_b_cnt == 0 && p.ccnt == 0) {
+	if (err || (p.a_b_cnt == 0 && p.ccnt == 0)) {
 		dev_dbg(dev, "Error on null slot, setting miss\n");
 		echan->missed = 1;
 	} else {
@@ -2019,8 +2024,7 @@ static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
 {
 	struct edma_soc_info *info;
 	struct property *prop;
-	size_t sz;
-	int ret;
+	int sz, ret;
 
 	info = devm_kzalloc(dev, sizeof(struct edma_soc_info), GFP_KERNEL);
 	if (!info)
@@ -2182,7 +2186,7 @@ static int edma_probe(struct platform_device *pdev)
 		const struct of_device_id *match;
 
 		match = of_match_node(edma_of_ids, node);
-		if (match && (u32)match->data == EDMA_BINDING_TPCC)
+		if (match && (*(u32 *)match->data) == EDMA_BINDING_TPCC)
 			legacy_mode = false;
 
 		info = edma_setup_info_from_dt(dev, legacy_mode);
@@ -2260,7 +2264,7 @@ static int edma_probe(struct platform_device *pdev)
 			for (i = 0; rsv_slots[i][0] != -1; i++) {
 				off = rsv_slots[i][0];
 				ln = rsv_slots[i][1];
-				set_bits(off, ln, ecc->slot_inuse);
+				edma_set_bits(off, ln, ecc->slot_inuse);
 			}
 		}
 	}
diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c
index 21f08cc3352b..d37e8dda8079 100644
--- a/drivers/dma/ep93xx_dma.c
+++ b/drivers/dma/ep93xx_dma.c
@@ -262,10 +262,8 @@ static void ep93xx_dma_set_active(struct ep93xx_dma_chan *edmac,
 static struct ep93xx_dma_desc *
 ep93xx_dma_get_active(struct ep93xx_dma_chan *edmac)
 {
-	if (list_empty(&edmac->active))
-		return NULL;
-
-	return list_first_entry(&edmac->active, struct ep93xx_dma_desc, node);
+	return list_first_entry_or_null(&edmac->active,
+					struct ep93xx_dma_desc, node);
 }
 
 /**
@@ -739,10 +737,10 @@ static void ep93xx_dma_tasklet(unsigned long data)
 {
 	struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data;
 	struct ep93xx_dma_desc *desc, *d;
-	dma_async_tx_callback callback = NULL;
-	void *callback_param = NULL;
+	struct dmaengine_desc_callback cb;
 	LIST_HEAD(list);
 
+	memset(&cb, 0, sizeof(cb));
 	spin_lock_irq(&edmac->lock);
 	/*
 	 * If dma_terminate_all() was called before we get to run, the active
@@ -757,8 +755,7 @@ static void ep93xx_dma_tasklet(unsigned long data)
 				dma_cookie_complete(&desc->txd);
 			list_splice_init(&edmac->active, &list);
 		}
-		callback = desc->txd.callback;
-		callback_param = desc->txd.callback_param;
+		dmaengine_desc_get_callback(&desc->txd, &cb);
 	}
 	spin_unlock_irq(&edmac->lock);
 
@@ -771,8 +768,7 @@ static void ep93xx_dma_tasklet(unsigned long data)
 		ep93xx_dma_desc_put(edmac, desc);
 	}
 
-	if (callback)
-		callback(callback_param);
+	dmaengine_desc_callback_invoke(&cb, NULL);
 }
 
 static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id)
@@ -1047,11 +1043,11 @@ ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 
 	first = NULL;
 	for_each_sg(sgl, sg, sg_len, i) {
-		size_t sg_len = sg_dma_len(sg);
+		size_t len = sg_dma_len(sg);
 
-		if (sg_len > DMA_MAX_CHAN_BYTES) {
-			dev_warn(chan2dev(edmac), "too big transfer size %d\n",
-				 sg_len);
+		if (len > DMA_MAX_CHAN_BYTES) {
+			dev_warn(chan2dev(edmac), "too big transfer size %zu\n",
+				 len);
 			goto fail;
 		}
 
@@ -1068,7 +1064,7 @@ ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 			desc->src_addr = edmac->runtime_addr;
 			desc->dst_addr = sg_dma_address(sg);
 		}
-		desc->size = sg_len;
+		desc->size = len;
 
 		if (!first)
 			first = desc;
@@ -1125,7 +1121,7 @@ ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
 	}
 
 	if (period_len > DMA_MAX_CHAN_BYTES) {
-		dev_warn(chan2dev(edmac), "too big period length %d\n",
+		dev_warn(chan2dev(edmac), "too big period length %zu\n",
 			 period_len);
 		return NULL;
 	}
diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c
index aad167eaaee8..db2f9e1653a2 100644
--- a/drivers/dma/fsl_raid.c
+++ b/drivers/dma/fsl_raid.c
@@ -134,17 +134,9 @@ static void fsl_re_issue_pending(struct dma_chan *chan)
 
 static void fsl_re_desc_done(struct fsl_re_desc *desc)
 {
-	dma_async_tx_callback callback;
-	void *callback_param;
-
 	dma_cookie_complete(&desc->async_tx);
-
-	callback = desc->async_tx.callback;
-	callback_param = desc->async_tx.callback_param;
-	if (callback)
-		callback(callback_param);
-
 	dma_descriptor_unmap(&desc->async_tx);
+	dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
 }
 
 static void fsl_re_cleanup_descs(struct fsl_re_chan *re_chan)
@@ -670,7 +662,7 @@ static int fsl_re_chan_probe(struct platform_device *ofdev,
 
 	/* read irq property from dts */
 	chan->irq = irq_of_parse_and_map(np, 0);
-	if (chan->irq == NO_IRQ) {
+	if (!chan->irq) {
 		dev_err(dev, "No IRQ defined for JR %d\n", q);
 		ret = -ENODEV;
 		goto err_free;
@@ -836,6 +828,7 @@ static int fsl_re_probe(struct platform_device *ofdev)
 		rc = of_property_read_u32(np, "reg", &off);
 		if (rc) {
 			dev_err(dev, "Reg property not found in JQ node\n");
+			of_node_put(np);
 			return -ENODEV;
 		}
 		/* Find out the Job Rings present under each JQ */
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index 911b7177eb50..51c75bf2b9b6 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -516,13 +516,9 @@ static dma_cookie_t fsldma_run_tx_complete_actions(struct fsldma_chan *chan,
 	if (txd->cookie > 0) {
 		ret = txd->cookie;
 
-		/* Run the link descriptor callback function */
-		if (txd->callback) {
-			chan_dbg(chan, "LD %p callback\n", desc);
-			txd->callback(txd->callback_param);
-		}
-
 		dma_descriptor_unmap(txd);
+		/* Run the link descriptor callback function */
+		dmaengine_desc_get_callback_invoke(txd, NULL);
 	}
 
 	/* Run any dependencies */
@@ -1153,7 +1149,7 @@ static void fsldma_free_irqs(struct fsldma_device *fdev)
 	struct fsldma_chan *chan;
 	int i;
 
-	if (fdev->irq != NO_IRQ) {
+	if (fdev->irq) {
 		dev_dbg(fdev->dev, "free per-controller IRQ\n");
 		free_irq(fdev->irq, fdev);
 		return;
@@ -1161,7 +1157,7 @@ static void fsldma_free_irqs(struct fsldma_device *fdev)
 
 	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
 		chan = fdev->chan[i];
-		if (chan && chan->irq != NO_IRQ) {
+		if (chan && chan->irq) {
 			chan_dbg(chan, "free per-channel IRQ\n");
 			free_irq(chan->irq, chan);
 		}
@@ -1175,7 +1171,7 @@ static int fsldma_request_irqs(struct fsldma_device *fdev)
 	int i;
 
 	/* if we have a per-controller IRQ, use that */
-	if (fdev->irq != NO_IRQ) {
+	if (fdev->irq) {
 		dev_dbg(fdev->dev, "request per-controller IRQ\n");
 		ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED,
 				  "fsldma-controller", fdev);
@@ -1188,7 +1184,7 @@ static int fsldma_request_irqs(struct fsldma_device *fdev)
 		if (!chan)
 			continue;
 
-		if (chan->irq == NO_IRQ) {
+		if (!chan->irq) {
 			chan_err(chan, "interrupts property missing in device tree\n");
 			ret = -ENODEV;
 			goto out_unwind;
@@ -1211,7 +1207,7 @@ out_unwind:
 		if (!chan)
 			continue;
 
-		if (chan->irq == NO_IRQ)
+		if (!chan->irq)
 			continue;
 
 		free_irq(chan->irq, chan);
@@ -1311,7 +1307,7 @@ static int fsl_dma_chan_probe(struct fsldma_device *fdev,
 	list_add_tail(&chan->common.device_node, &fdev->common.channels);
 
 	dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible,
-		 chan->irq != NO_IRQ ? chan->irq : fdev->irq);
+		 chan->irq ? chan->irq : fdev->irq);
 
 	return 0;
 
@@ -1351,7 +1347,7 @@ static int fsldma_of_probe(struct platform_device *op)
 	if (!fdev->regs) {
 		dev_err(&op->dev, "unable to ioremap registers\n");
 		err = -ENOMEM;
-		goto out_free_fdev;
+		goto out_free;
 	}
 
 	/* map the channel IRQ if it exists, but don't hookup the handler yet */
@@ -1416,6 +1412,8 @@ static int fsldma_of_probe(struct platform_device *op)
 
 out_free_fdev:
 	irq_dispose_mapping(fdev->irq);
+	iounmap(fdev->regs);
+out_free:
 	kfree(fdev);
 out_return:
 	return err;
diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c
index c5f21efd6090..29d04ca71d52 100644
--- a/drivers/dma/hsu/hsu.c
+++ b/drivers/dma/hsu/hsu.c
@@ -200,10 +200,9 @@ EXPORT_SYMBOL_GPL(hsu_dma_get_status);
  *      is not a normal timeout interrupt, ie. hsu_dma_get_status() returned 0.
  *
  *      Return:
- *      IRQ_NONE for invalid channel number, IRQ_HANDLED otherwise.
+ *      0 for invalid channel number, 1 otherwise.
  */
-irqreturn_t hsu_dma_do_irq(struct hsu_dma_chip *chip, unsigned short nr,
-			   u32 status)
+int hsu_dma_do_irq(struct hsu_dma_chip *chip, unsigned short nr, u32 status)
 {
 	struct hsu_dma_chan *hsuc;
 	struct hsu_dma_desc *desc;
@@ -211,7 +210,7 @@ irqreturn_t hsu_dma_do_irq(struct hsu_dma_chip *chip, unsigned short nr,
 
 	/* Sanity check */
 	if (nr >= chip->hsu->nr_channels)
-		return IRQ_NONE;
+		return 0;
 
 	hsuc = &chip->hsu->chan[nr];
 
@@ -230,7 +229,7 @@ irqreturn_t hsu_dma_do_irq(struct hsu_dma_chip *chip, unsigned short nr,
 	}
 	spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
 
-	return IRQ_HANDLED;
+	return 1;
 }
 EXPORT_SYMBOL_GPL(hsu_dma_do_irq);
 
diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c
index 9916058531d9..b51639f045ed 100644
--- a/drivers/dma/hsu/pci.c
+++ b/drivers/dma/hsu/pci.c
@@ -29,7 +29,7 @@ static irqreturn_t hsu_pci_irq(int irq, void *dev)
 	u32 dmaisr;
 	u32 status;
 	unsigned short i;
-	irqreturn_t ret = IRQ_NONE;
+	int ret = 0;
 	int err;
 
 	dmaisr = readl(chip->regs + HSU_PCI_DMAISR);
@@ -37,14 +37,14 @@ static irqreturn_t hsu_pci_irq(int irq, void *dev)
 		if (dmaisr & 0x1) {
 			err = hsu_dma_get_status(chip, i, &status);
 			if (err > 0)
-				ret |= IRQ_HANDLED;
+				ret |= 1;
 			else if (err == 0)
 				ret |= hsu_dma_do_irq(chip, i, status);
 		}
 		dmaisr >>= 1;
 	}
 
-	return ret;
+	return IRQ_RETVAL(ret);
 }
 
 static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c
index a4c53be482cf..624f1e1e9c55 100644
--- a/drivers/dma/img-mdc-dma.c
+++ b/drivers/dma/img-mdc-dma.c
@@ -861,7 +861,6 @@ static int mdc_dma_probe(struct platform_device *pdev)
 {
 	struct mdc_dma *mdma;
 	struct resource *res;
-	const struct of_device_id *match;
 	unsigned int i;
 	u32 val;
 	int ret;
@@ -871,8 +870,7 @@ static int mdc_dma_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	platform_set_drvdata(pdev, mdma);
 
-	match = of_match_device(mdc_dma_of_match, &pdev->dev);
-	mdma->soc = match->data;
+	mdma->soc = of_device_get_match_data(&pdev->dev);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	mdma->regs = devm_ioremap_resource(&pdev->dev, res);
diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c
index a960608c0a4d..ab0fb804fb1e 100644
--- a/drivers/dma/imx-dma.c
+++ b/drivers/dma/imx-dma.c
@@ -663,9 +663,7 @@ static void imxdma_tasklet(unsigned long data)
 out:
 	spin_unlock_irqrestore(&imxdma->lock, flags);
 
-	if (desc->desc.callback)
-		desc->desc.callback(desc->desc.callback_param);
-
+	dmaengine_desc_get_callback_invoke(&desc->desc, NULL);
 }
 
 static int imxdma_terminate_all(struct dma_chan *chan)
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index 03ec76fc22ff..b9629b2bfc05 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -184,7 +184,7 @@
 struct sdma_mode_count {
 	u32 count   : 16; /* size of the buffer pointed by this BD */
 	u32 status  :  8; /* E,R,I,C,W,D status bits stored here */
-	u32 command :  8; /* command mostlky used for channel 0 */
+	u32 command :  8; /* command mostly used for channel 0 */
 };
 
 /*
@@ -479,6 +479,24 @@ static struct sdma_driver_data sdma_imx6q = {
 	.script_addrs = &sdma_script_imx6q,
 };
 
+static struct sdma_script_start_addrs sdma_script_imx7d = {
+	.ap_2_ap_addr = 644,
+	.uart_2_mcu_addr = 819,
+	.mcu_2_app_addr = 749,
+	.uartsh_2_mcu_addr = 1034,
+	.mcu_2_shp_addr = 962,
+	.app_2_mcu_addr = 685,
+	.shp_2_mcu_addr = 893,
+	.spdif_2_mcu_addr = 1102,
+	.mcu_2_spdif_addr = 1136,
+};
+
+static struct sdma_driver_data sdma_imx7d = {
+	.chnenbl0 = SDMA_CHNENBL0_IMX35,
+	.num_events = 48,
+	.script_addrs = &sdma_script_imx7d,
+};
+
 static const struct platform_device_id sdma_devtypes[] = {
 	{
 		.name = "imx25-sdma",
@@ -499,6 +517,9 @@ static const struct platform_device_id sdma_devtypes[] = {
 		.name = "imx6q-sdma",
 		.driver_data = (unsigned long)&sdma_imx6q,
 	}, {
+		.name = "imx7d-sdma",
+		.driver_data = (unsigned long)&sdma_imx7d,
+	}, {
 		/* sentinel */
 	}
 };
@@ -511,6 +532,7 @@ static const struct of_device_id sdma_dt_ids[] = {
 	{ .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, },
 	{ .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },
 	{ .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },
+	{ .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, sdma_dt_ids);
@@ -648,15 +670,11 @@ static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
 	writel_relaxed(val, sdma->regs + chnenbl);
 }
 
-static void sdma_handle_channel_loop(struct sdma_channel *sdmac)
-{
-	if (sdmac->desc.callback)
-		sdmac->desc.callback(sdmac->desc.callback_param);
-}
-
 static void sdma_update_channel_loop(struct sdma_channel *sdmac)
 {
 	struct sdma_buffer_descriptor *bd;
+	int error = 0;
+	enum dma_status	old_status = sdmac->status;
 
 	/*
 	 * loop mode. Iterate over descriptors, re-setup them and
@@ -668,17 +686,41 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac)
 		if (bd->mode.status & BD_DONE)
 			break;
 
-		if (bd->mode.status & BD_RROR)
+		if (bd->mode.status & BD_RROR) {
+			bd->mode.status &= ~BD_RROR;
 			sdmac->status = DMA_ERROR;
+			error = -EIO;
+		}
 
+	       /*
+		* We use bd->mode.count to calculate the residue, since contains
+		* the number of bytes present in the current buffer descriptor.
+		*/
+
+		sdmac->chn_real_count = bd->mode.count;
 		bd->mode.status |= BD_DONE;
+		bd->mode.count = sdmac->period_len;
+
+		/*
+		 * The callback is called from the interrupt context in order
+		 * to reduce latency and to avoid the risk of altering the
+		 * SDMA transaction status by the time the client tasklet is
+		 * executed.
+		 */
+
+		dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL);
+
 		sdmac->buf_tail++;
 		sdmac->buf_tail %= sdmac->num_bd;
+
+		if (error)
+			sdmac->status = old_status;
 	}
 }
 
-static void mxc_sdma_handle_channel_normal(struct sdma_channel *sdmac)
+static void mxc_sdma_handle_channel_normal(unsigned long data)
 {
+	struct sdma_channel *sdmac = (struct sdma_channel *) data;
 	struct sdma_buffer_descriptor *bd;
 	int i, error = 0;
 
@@ -701,18 +743,8 @@ static void mxc_sdma_handle_channel_normal(struct sdma_channel *sdmac)
 		sdmac->status = DMA_COMPLETE;
 
 	dma_cookie_complete(&sdmac->desc);
-	if (sdmac->desc.callback)
-		sdmac->desc.callback(sdmac->desc.callback_param);
-}
-
-static void sdma_tasklet(unsigned long data)
-{
-	struct sdma_channel *sdmac = (struct sdma_channel *) data;
 
-	if (sdmac->flags & IMX_DMA_SG_LOOP)
-		sdma_handle_channel_loop(sdmac);
-	else
-		mxc_sdma_handle_channel_normal(sdmac);
+	dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL);
 }
 
 static irqreturn_t sdma_int_handler(int irq, void *dev_id)
@@ -731,8 +763,8 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id)
 
 		if (sdmac->flags & IMX_DMA_SG_LOOP)
 			sdma_update_channel_loop(sdmac);
-
-		tasklet_schedule(&sdmac->tasklet);
+		else
+			tasklet_schedule(&sdmac->tasklet);
 
 		__clear_bit(channel, &stat);
 	}
@@ -1353,7 +1385,8 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan,
 	u32 residue;
 
 	if (sdmac->flags & IMX_DMA_SG_LOOP)
-		residue = (sdmac->num_bd - sdmac->buf_tail) * sdmac->period_len;
+		residue = (sdmac->num_bd - sdmac->buf_tail) *
+			   sdmac->period_len - sdmac->chn_real_count;
 	else
 		residue = sdmac->chn_count - sdmac->chn_real_count;
 
@@ -1375,6 +1408,7 @@ static void sdma_issue_pending(struct dma_chan *chan)
 #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1	34
 #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2	38
 #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3	41
+#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4	42
 
 static void sdma_add_scripts(struct sdma_engine *sdma,
 		const struct sdma_script_start_addrs *addr)
@@ -1424,6 +1458,9 @@ static void sdma_load_firmware(const struct firmware *fw, void *context)
 	case 3:
 		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3;
 		break;
+	case 4:
+		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4;
+		break;
 	default:
 		dev_err(sdma->dev, "unknown firmware version\n");
 		goto err_firmware;
@@ -1732,7 +1769,7 @@ static int sdma_probe(struct platform_device *pdev)
 		dma_cookie_init(&sdmac->chan);
 		sdmac->channel = i;
 
-		tasklet_init(&sdmac->tasklet, sdma_tasklet,
+		tasklet_init(&sdmac->tasklet, mxc_sdma_handle_channel_normal,
 			     (unsigned long) sdmac);
 		/*
 		 * Add the channel to the DMAC list. Do not add channel 0 though
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index bd09961443b1..49386ce04bf5 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -38,8 +38,54 @@
 
 #include "../dmaengine.h"
 
+static char *chanerr_str[] = {
+	"DMA Transfer Destination Address Error",
+	"Next Descriptor Address Error",
+	"Descriptor Error",
+	"Chan Address Value Error",
+	"CHANCMD Error",
+	"Chipset Uncorrectable Data Integrity Error",
+	"DMA Uncorrectable Data Integrity Error",
+	"Read Data Error",
+	"Write Data Error",
+	"Descriptor Control Error",
+	"Descriptor Transfer Size Error",
+	"Completion Address Error",
+	"Interrupt Configuration Error",
+	"Super extended descriptor Address Error",
+	"Unaffiliated Error",
+	"CRC or XOR P Error",
+	"XOR Q Error",
+	"Descriptor Count Error",
+	"DIF All F detect Error",
+	"Guard Tag verification Error",
+	"Application Tag verification Error",
+	"Reference Tag verification Error",
+	"Bundle Bit Error",
+	"Result DIF All F detect Error",
+	"Result Guard Tag verification Error",
+	"Result Application Tag verification Error",
+	"Result Reference Tag verification Error",
+	NULL
+};
+
 static void ioat_eh(struct ioatdma_chan *ioat_chan);
 
+static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr)
+{
+	int i;
+
+	for (i = 0; i < 32; i++) {
+		if ((chanerr >> i) & 1) {
+			if (chanerr_str[i]) {
+				dev_err(to_dev(ioat_chan), "Err(%d): %s\n",
+					i, chanerr_str[i]);
+			} else
+				break;
+		}
+	}
+}
+
 /**
  * ioat_dma_do_interrupt - handler used for single vector interrupt mode
  * @irq: interrupt id
@@ -568,12 +614,14 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
 
 		tx = &desc->txd;
 		if (tx->cookie) {
+			struct dmaengine_result res;
+
 			dma_cookie_complete(tx);
 			dma_descriptor_unmap(tx);
-			if (tx->callback) {
-				tx->callback(tx->callback_param);
-				tx->callback = NULL;
-			}
+			res.result = DMA_TRANS_NOERROR;
+			dmaengine_desc_get_callback_invoke(tx, NULL);
+			tx->callback = NULL;
+			tx->callback_result = NULL;
 		}
 
 		if (tx->phys == phys_complete)
@@ -622,7 +670,8 @@ static void ioat_cleanup(struct ioatdma_chan *ioat_chan)
 	if (is_ioat_halted(*ioat_chan->completion)) {
 		u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 
-		if (chanerr & IOAT_CHANERR_HANDLE_MASK) {
+		if (chanerr &
+		    (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) {
 			mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
 			ioat_eh(ioat_chan);
 		}
@@ -652,6 +701,61 @@ static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
 	__ioat_restart_chan(ioat_chan);
 }
 
+
+static void ioat_abort_descs(struct ioatdma_chan *ioat_chan)
+{
+	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
+	struct ioat_ring_ent *desc;
+	u16 active;
+	int idx = ioat_chan->tail, i;
+
+	/*
+	 * We assume that the failed descriptor has been processed.
+	 * Now we are just returning all the remaining submitted
+	 * descriptors to abort.
+	 */
+	active = ioat_ring_active(ioat_chan);
+
+	/* we skip the failed descriptor that tail points to */
+	for (i = 1; i < active; i++) {
+		struct dma_async_tx_descriptor *tx;
+
+		smp_read_barrier_depends();
+		prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
+		desc = ioat_get_ring_ent(ioat_chan, idx + i);
+
+		tx = &desc->txd;
+		if (tx->cookie) {
+			struct dmaengine_result res;
+
+			dma_cookie_complete(tx);
+			dma_descriptor_unmap(tx);
+			res.result = DMA_TRANS_ABORTED;
+			dmaengine_desc_get_callback_invoke(tx, &res);
+			tx->callback = NULL;
+			tx->callback_result = NULL;
+		}
+
+		/* skip extended descriptors */
+		if (desc_has_ext(desc)) {
+			WARN_ON(i + 1 >= active);
+			i++;
+		}
+
+		/* cleanup super extended descriptors */
+		if (desc->sed) {
+			ioat_free_sed(ioat_dma, desc->sed);
+			desc->sed = NULL;
+		}
+	}
+
+	smp_mb(); /* finish all descriptor reads before incrementing tail */
+	ioat_chan->tail = idx + active;
+
+	desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
+	ioat_chan->last_completion = *ioat_chan->completion = desc->txd.phys;
+}
+
 static void ioat_eh(struct ioatdma_chan *ioat_chan)
 {
 	struct pci_dev *pdev = to_pdev(ioat_chan);
@@ -662,6 +766,8 @@ static void ioat_eh(struct ioatdma_chan *ioat_chan)
 	u32 err_handled = 0;
 	u32 chanerr_int;
 	u32 chanerr;
+	bool abort = false;
+	struct dmaengine_result res;
 
 	/* cleanup so tail points to descriptor that caused the error */
 	if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
@@ -697,30 +803,55 @@ static void ioat_eh(struct ioatdma_chan *ioat_chan)
 		break;
 	}
 
+	if (chanerr & IOAT_CHANERR_RECOVER_MASK) {
+		if (chanerr & IOAT_CHANERR_READ_DATA_ERR) {
+			res.result = DMA_TRANS_READ_FAILED;
+			err_handled |= IOAT_CHANERR_READ_DATA_ERR;
+		} else if (chanerr & IOAT_CHANERR_WRITE_DATA_ERR) {
+			res.result = DMA_TRANS_WRITE_FAILED;
+			err_handled |= IOAT_CHANERR_WRITE_DATA_ERR;
+		}
+
+		abort = true;
+	} else
+		res.result = DMA_TRANS_NOERROR;
+
 	/* fault on unhandled error or spurious halt */
 	if (chanerr ^ err_handled || chanerr == 0) {
 		dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n",
 			__func__, chanerr, err_handled);
+		dev_err(to_dev(ioat_chan), "Errors handled:\n");
+		ioat_print_chanerrs(ioat_chan, err_handled);
+		dev_err(to_dev(ioat_chan), "Errors not handled:\n");
+		ioat_print_chanerrs(ioat_chan, (chanerr & ~err_handled));
+
 		BUG();
-	} else { /* cleanup the faulty descriptor */
-		tx = &desc->txd;
-		if (tx->cookie) {
-			dma_cookie_complete(tx);
-			dma_descriptor_unmap(tx);
-			if (tx->callback) {
-				tx->callback(tx->callback_param);
-				tx->callback = NULL;
-			}
-		}
 	}
 
-	writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-	pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
+	/* cleanup the faulty descriptor since we are continuing */
+	tx = &desc->txd;
+	if (tx->cookie) {
+		dma_cookie_complete(tx);
+		dma_descriptor_unmap(tx);
+		dmaengine_desc_get_callback_invoke(tx, &res);
+		tx->callback = NULL;
+		tx->callback_result = NULL;
+	}
 
 	/* mark faulting descriptor as complete */
 	*ioat_chan->completion = desc->txd.phys;
 
 	spin_lock_bh(&ioat_chan->prep_lock);
+	/* we need abort all descriptors */
+	if (abort) {
+		ioat_abort_descs(ioat_chan);
+		/* clean up the channel, we could be in weird state */
+		ioat_reset_hw(ioat_chan);
+	}
+
+	writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
+	pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
+
 	ioat_restart_channel(ioat_chan);
 	spin_unlock_bh(&ioat_chan->prep_lock);
 }
@@ -753,10 +884,28 @@ void ioat_timer_event(unsigned long data)
 		chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 		dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n",
 			__func__, chanerr);
-		if (test_bit(IOAT_RUN, &ioat_chan->state))
-			BUG_ON(is_ioat_bug(chanerr));
-		else /* we never got off the ground */
-			return;
+		dev_err(to_dev(ioat_chan), "Errors:\n");
+		ioat_print_chanerrs(ioat_chan, chanerr);
+
+		if (test_bit(IOAT_RUN, &ioat_chan->state)) {
+			spin_lock_bh(&ioat_chan->cleanup_lock);
+			spin_lock_bh(&ioat_chan->prep_lock);
+			set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
+			spin_unlock_bh(&ioat_chan->prep_lock);
+
+			ioat_abort_descs(ioat_chan);
+			dev_warn(to_dev(ioat_chan), "Reset channel...\n");
+			ioat_reset_hw(ioat_chan);
+			dev_warn(to_dev(ioat_chan), "Restart channel...\n");
+			ioat_restart_channel(ioat_chan);
+
+			spin_lock_bh(&ioat_chan->prep_lock);
+			clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
+			spin_unlock_bh(&ioat_chan->prep_lock);
+			spin_unlock_bh(&ioat_chan->cleanup_lock);
+		}
+
+		return;
 	}
 
 	spin_lock_bh(&ioat_chan->cleanup_lock);
@@ -780,14 +929,26 @@ void ioat_timer_event(unsigned long data)
 		u32 chanerr;
 
 		chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-		dev_warn(to_dev(ioat_chan), "Restarting channel...\n");
-		dev_warn(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n",
-			 status, chanerr);
-		dev_warn(to_dev(ioat_chan), "Active descriptors: %d\n",
-			 ioat_ring_active(ioat_chan));
+		dev_err(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n",
+			status, chanerr);
+		dev_err(to_dev(ioat_chan), "Errors:\n");
+		ioat_print_chanerrs(ioat_chan, chanerr);
+
+		dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n",
+			ioat_ring_active(ioat_chan));
 
 		spin_lock_bh(&ioat_chan->prep_lock);
+		set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
+		spin_unlock_bh(&ioat_chan->prep_lock);
+
+		ioat_abort_descs(ioat_chan);
+		dev_warn(to_dev(ioat_chan), "Resetting channel...\n");
+		ioat_reset_hw(ioat_chan);
+		dev_warn(to_dev(ioat_chan), "Restarting channel...\n");
 		ioat_restart_channel(ioat_chan);
+
+		spin_lock_bh(&ioat_chan->prep_lock);
+		clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
 		spin_unlock_bh(&ioat_chan->prep_lock);
 		spin_unlock_bh(&ioat_chan->cleanup_lock);
 		return;
diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c
index 7145f7716a92..015f7110b96d 100644
--- a/drivers/dma/ioat/init.c
+++ b/drivers/dma/ioat/init.c
@@ -828,7 +828,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
 
 	dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
 	if (dma_mapping_error(dev, dest_dma))
-		goto dma_unmap;
+		goto free_resources;
 
 	for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
 		dma_srcs[i] = DMA_ERROR_CODE;
diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h
index 70534981a49b..48fa4cf9f64a 100644
--- a/drivers/dma/ioat/registers.h
+++ b/drivers/dma/ioat/registers.h
@@ -240,6 +240,8 @@
 #define IOAT_CHANERR_DESCRIPTOR_COUNT_ERR	0x40000
 
 #define IOAT_CHANERR_HANDLE_MASK (IOAT_CHANERR_XOR_P_OR_CRC_ERR | IOAT_CHANERR_XOR_Q_ERR)
+#define IOAT_CHANERR_RECOVER_MASK (IOAT_CHANERR_READ_DATA_ERR | \
+				   IOAT_CHANERR_WRITE_DATA_ERR)
 
 #define IOAT_CHANERR_MASK_OFFSET		0x2C	/* 32-bit Channel Error Register */
 
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index f039cfadf17b..a410657f7bcd 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -71,8 +71,7 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
 		/* call the callback (must not sleep or submit new
 		 * operations to this channel)
 		 */
-		if (tx->callback)
-			tx->callback(tx->callback_param);
+		dmaengine_desc_get_callback_invoke(tx, NULL);
 
 		dma_descriptor_unmap(tx);
 		if (desc->group_head)
diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c
index b54f62de9232..ed76044ce4b9 100644
--- a/drivers/dma/ipu/ipu_idmac.c
+++ b/drivers/dma/ipu/ipu_idmac.c
@@ -1160,11 +1160,10 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
 	struct scatterlist **sg, *sgnext, *sgnew = NULL;
 	/* Next transfer descriptor */
 	struct idmac_tx_desc *desc, *descnew;
-	dma_async_tx_callback callback;
-	void *callback_param;
 	bool done = false;
 	u32 ready0, ready1, curbuf, err;
 	unsigned long flags;
+	struct dmaengine_desc_callback cb;
 
 	/* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
 
@@ -1278,12 +1277,12 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
 
 	if (likely(sgnew) &&
 	    ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) {
-		callback = descnew->txd.callback;
-		callback_param = descnew->txd.callback_param;
+		dmaengine_desc_get_callback(&descnew->txd, &cb);
+
 		list_del_init(&descnew->list);
 		spin_unlock(&ichan->lock);
-		if (callback)
-			callback(callback_param);
+
+		dmaengine_desc_callback_invoke(&cb, NULL);
 		spin_lock(&ichan->lock);
 	}
 
@@ -1292,13 +1291,12 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
 	if (done)
 		dma_cookie_complete(&desc->txd);
 
-	callback = desc->txd.callback;
-	callback_param = desc->txd.callback_param;
+	dmaengine_desc_get_callback(&desc->txd, &cb);
 
 	spin_unlock(&ichan->lock);
 
-	if (done && (desc->txd.flags & DMA_PREP_INTERRUPT) && callback)
-		callback(callback_param);
+	if (done && (desc->txd.flags & DMA_PREP_INTERRUPT))
+		dmaengine_desc_callback_invoke(&cb, NULL);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/dma/ipu/ipu_irq.c b/drivers/dma/ipu/ipu_irq.c
index 2bf37e68ad0f..dd184b50e5b4 100644
--- a/drivers/dma/ipu/ipu_irq.c
+++ b/drivers/dma/ipu/ipu_irq.c
@@ -286,22 +286,21 @@ static void ipu_irq_handler(struct irq_desc *desc)
 		raw_spin_unlock(&bank_lock);
 		while ((line = ffs(status))) {
 			struct ipu_irq_map *map;
-			unsigned int irq = NO_IRQ;
+			unsigned int irq;
 
 			line--;
 			status &= ~(1UL << line);
 
 			raw_spin_lock(&bank_lock);
 			map = src2map(32 * i + line);
-			if (map)
-				irq = map->irq;
-			raw_spin_unlock(&bank_lock);
-
 			if (!map) {
+				raw_spin_unlock(&bank_lock);
 				pr_err("IPU: Interrupt on unmapped source %u bank %d\n",
 				       line, i);
 				continue;
 			}
+			irq = map->irq;
+			raw_spin_unlock(&bank_lock);
 			generic_handle_irq(irq);
 		}
 	}
diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c
index 39de8980128c..aabcb7934b05 100644
--- a/drivers/dma/k3dma.c
+++ b/drivers/dma/k3dma.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013 Linaro Ltd.
+ * Copyright (c) 2013 - 2015 Linaro Ltd.
  * Copyright (c) 2013 Hisilicon Limited.
  *
  * This program is free software; you can redistribute it and/or modify
@@ -8,6 +8,8 @@
  */
 #include <linux/sched.h>
 #include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
 #include <linux/dmaengine.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -25,22 +27,28 @@
 
 #define DRIVER_NAME		"k3-dma"
 #define DMA_MAX_SIZE		0x1ffc
+#define DMA_CYCLIC_MAX_PERIOD	0x1000
+#define LLI_BLOCK_SIZE		(4 * PAGE_SIZE)
 
 #define INT_STAT		0x00
 #define INT_TC1			0x04
+#define INT_TC2			0x08
 #define INT_ERR1		0x0c
 #define INT_ERR2		0x10
 #define INT_TC1_MASK		0x18
+#define INT_TC2_MASK		0x1c
 #define INT_ERR1_MASK		0x20
 #define INT_ERR2_MASK		0x24
 #define INT_TC1_RAW		0x600
-#define INT_ERR1_RAW		0x608
-#define INT_ERR2_RAW		0x610
+#define INT_TC2_RAW		0x608
+#define INT_ERR1_RAW		0x610
+#define INT_ERR2_RAW		0x618
 #define CH_PRI			0x688
 #define CH_STAT			0x690
 #define CX_CUR_CNT		0x704
 #define CX_LLI			0x800
-#define CX_CNT			0x810
+#define CX_CNT1			0x80c
+#define CX_CNT0			0x810
 #define CX_SRC			0x814
 #define CX_DST			0x818
 #define CX_CFG			0x81c
@@ -49,6 +57,7 @@
 
 #define CX_LLI_CHAIN_EN		0x2
 #define CX_CFG_EN		0x1
+#define CX_CFG_NODEIRQ		BIT(1)
 #define CX_CFG_MEM2PER		(0x1 << 2)
 #define CX_CFG_PER2MEM		(0x2 << 2)
 #define CX_CFG_SRCINCR		(0x1 << 31)
@@ -68,7 +77,7 @@ struct k3_dma_desc_sw {
 	dma_addr_t		desc_hw_lli;
 	size_t			desc_num;
 	size_t			size;
-	struct k3_desc_hw	desc_hw[0];
+	struct k3_desc_hw	*desc_hw;
 };
 
 struct k3_dma_phy;
@@ -81,6 +90,7 @@ struct k3_dma_chan {
 	enum dma_transfer_direction dir;
 	dma_addr_t		dev_addr;
 	enum dma_status		status;
+	bool			cyclic;
 };
 
 struct k3_dma_phy {
@@ -100,6 +110,7 @@ struct k3_dma_dev {
 	struct k3_dma_phy	*phy;
 	struct k3_dma_chan	*chans;
 	struct clk		*clk;
+	struct dma_pool		*pool;
 	u32			dma_channels;
 	u32			dma_requests;
 	unsigned int		irq;
@@ -135,6 +146,7 @@ static void k3_dma_terminate_chan(struct k3_dma_phy *phy, struct k3_dma_dev *d)
 
 	val = 0x1 << phy->idx;
 	writel_relaxed(val, d->base + INT_TC1_RAW);
+	writel_relaxed(val, d->base + INT_TC2_RAW);
 	writel_relaxed(val, d->base + INT_ERR1_RAW);
 	writel_relaxed(val, d->base + INT_ERR2_RAW);
 }
@@ -142,7 +154,7 @@ static void k3_dma_terminate_chan(struct k3_dma_phy *phy, struct k3_dma_dev *d)
 static void k3_dma_set_desc(struct k3_dma_phy *phy, struct k3_desc_hw *hw)
 {
 	writel_relaxed(hw->lli, phy->base + CX_LLI);
-	writel_relaxed(hw->count, phy->base + CX_CNT);
+	writel_relaxed(hw->count, phy->base + CX_CNT0);
 	writel_relaxed(hw->saddr, phy->base + CX_SRC);
 	writel_relaxed(hw->daddr, phy->base + CX_DST);
 	writel_relaxed(AXI_CFG_DEFAULT, phy->base + AXI_CFG);
@@ -176,11 +188,13 @@ static void k3_dma_enable_dma(struct k3_dma_dev *d, bool on)
 
 		/* unmask irq */
 		writel_relaxed(0xffff, d->base + INT_TC1_MASK);
+		writel_relaxed(0xffff, d->base + INT_TC2_MASK);
 		writel_relaxed(0xffff, d->base + INT_ERR1_MASK);
 		writel_relaxed(0xffff, d->base + INT_ERR2_MASK);
 	} else {
 		/* mask irq */
 		writel_relaxed(0x0, d->base + INT_TC1_MASK);
+		writel_relaxed(0x0, d->base + INT_TC2_MASK);
 		writel_relaxed(0x0, d->base + INT_ERR1_MASK);
 		writel_relaxed(0x0, d->base + INT_ERR2_MASK);
 	}
@@ -193,22 +207,31 @@ static irqreturn_t k3_dma_int_handler(int irq, void *dev_id)
 	struct k3_dma_chan *c;
 	u32 stat = readl_relaxed(d->base + INT_STAT);
 	u32 tc1  = readl_relaxed(d->base + INT_TC1);
+	u32 tc2  = readl_relaxed(d->base + INT_TC2);
 	u32 err1 = readl_relaxed(d->base + INT_ERR1);
 	u32 err2 = readl_relaxed(d->base + INT_ERR2);
 	u32 i, irq_chan = 0;
 
 	while (stat) {
 		i = __ffs(stat);
-		stat &= (stat - 1);
-		if (likely(tc1 & BIT(i))) {
+		stat &= ~BIT(i);
+		if (likely(tc1 & BIT(i)) || (tc2 & BIT(i))) {
+			unsigned long flags;
+
 			p = &d->phy[i];
 			c = p->vchan;
-			if (c) {
-				unsigned long flags;
-
+			if (c && (tc1 & BIT(i))) {
 				spin_lock_irqsave(&c->vc.lock, flags);
 				vchan_cookie_complete(&p->ds_run->vd);
+				WARN_ON_ONCE(p->ds_done);
 				p->ds_done = p->ds_run;
+				p->ds_run = NULL;
+				spin_unlock_irqrestore(&c->vc.lock, flags);
+			}
+			if (c && (tc2 & BIT(i))) {
+				spin_lock_irqsave(&c->vc.lock, flags);
+				if (p->ds_run != NULL)
+					vchan_cyclic_callback(&p->ds_run->vd);
 				spin_unlock_irqrestore(&c->vc.lock, flags);
 			}
 			irq_chan |= BIT(i);
@@ -218,14 +241,17 @@ static irqreturn_t k3_dma_int_handler(int irq, void *dev_id)
 	}
 
 	writel_relaxed(irq_chan, d->base + INT_TC1_RAW);
+	writel_relaxed(irq_chan, d->base + INT_TC2_RAW);
 	writel_relaxed(err1, d->base + INT_ERR1_RAW);
 	writel_relaxed(err2, d->base + INT_ERR2_RAW);
 
-	if (irq_chan) {
+	if (irq_chan)
 		tasklet_schedule(&d->task);
+
+	if (irq_chan || err1 || err2)
 		return IRQ_HANDLED;
-	} else
-		return IRQ_NONE;
+
+	return IRQ_NONE;
 }
 
 static int k3_dma_start_txd(struct k3_dma_chan *c)
@@ -247,14 +273,14 @@ static int k3_dma_start_txd(struct k3_dma_chan *c)
 		 * so vc->desc_issued only contains desc pending
 		 */
 		list_del(&ds->vd.node);
+
+		WARN_ON_ONCE(c->phy->ds_run);
+		WARN_ON_ONCE(c->phy->ds_done);
 		c->phy->ds_run = ds;
-		c->phy->ds_done = NULL;
 		/* start dma */
 		k3_dma_set_desc(c->phy, &ds->desc_hw[0]);
 		return 0;
 	}
-	c->phy->ds_done = NULL;
-	c->phy->ds_run = NULL;
 	return -EAGAIN;
 }
 
@@ -351,7 +377,7 @@ static enum dma_status k3_dma_tx_status(struct dma_chan *chan,
 	 * its total size.
 	 */
 	vd = vchan_find_desc(&c->vc, cookie);
-	if (vd) {
+	if (vd && !c->cyclic) {
 		bytes = container_of(vd, struct k3_dma_desc_sw, vd)->size;
 	} else if ((!p) || (!p->ds_run)) {
 		bytes = 0;
@@ -361,7 +387,8 @@ static enum dma_status k3_dma_tx_status(struct dma_chan *chan,
 
 		bytes = k3_dma_get_curr_cnt(d, p);
 		clli = k3_dma_get_curr_lli(p);
-		index = (clli - ds->desc_hw_lli) / sizeof(struct k3_desc_hw);
+		index = ((clli - ds->desc_hw_lli) /
+				sizeof(struct k3_desc_hw)) + 1;
 		for (; index < ds->desc_num; index++) {
 			bytes += ds->desc_hw[index].count;
 			/* end of lli */
@@ -402,9 +429,10 @@ static void k3_dma_issue_pending(struct dma_chan *chan)
 static void k3_dma_fill_desc(struct k3_dma_desc_sw *ds, dma_addr_t dst,
 			dma_addr_t src, size_t len, u32 num, u32 ccfg)
 {
-	if ((num + 1) < ds->desc_num)
+	if (num != ds->desc_num - 1)
 		ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) *
 			sizeof(struct k3_desc_hw);
+
 	ds->desc_hw[num].lli |= CX_LLI_CHAIN_EN;
 	ds->desc_hw[num].count = len;
 	ds->desc_hw[num].saddr = src;
@@ -412,6 +440,35 @@ static void k3_dma_fill_desc(struct k3_dma_desc_sw *ds, dma_addr_t dst,
 	ds->desc_hw[num].config = ccfg;
 }
 
+static struct k3_dma_desc_sw *k3_dma_alloc_desc_resource(int num,
+							struct dma_chan *chan)
+{
+	struct k3_dma_chan *c = to_k3_chan(chan);
+	struct k3_dma_desc_sw *ds;
+	struct k3_dma_dev *d = to_k3_dma(chan->device);
+	int lli_limit = LLI_BLOCK_SIZE / sizeof(struct k3_desc_hw);
+
+	if (num > lli_limit) {
+		dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n",
+			&c->vc, num, lli_limit);
+		return NULL;
+	}
+
+	ds = kzalloc(sizeof(*ds), GFP_NOWAIT);
+	if (!ds)
+		return NULL;
+
+	ds->desc_hw = dma_pool_alloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
+	if (!ds->desc_hw) {
+		dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc);
+		kfree(ds);
+		return NULL;
+	}
+	memset(ds->desc_hw, 0, sizeof(struct k3_desc_hw) * num);
+	ds->desc_num = num;
+	return ds;
+}
+
 static struct dma_async_tx_descriptor *k3_dma_prep_memcpy(
 	struct dma_chan *chan,	dma_addr_t dst, dma_addr_t src,
 	size_t len, unsigned long flags)
@@ -425,13 +482,13 @@ static struct dma_async_tx_descriptor *k3_dma_prep_memcpy(
 		return NULL;
 
 	num = DIV_ROUND_UP(len, DMA_MAX_SIZE);
-	ds = kzalloc(sizeof(*ds) + num * sizeof(ds->desc_hw[0]), GFP_ATOMIC);
+
+	ds = k3_dma_alloc_desc_resource(num, chan);
 	if (!ds)
 		return NULL;
 
-	ds->desc_hw_lli = __virt_to_phys((unsigned long)&ds->desc_hw[0]);
+	c->cyclic = 0;
 	ds->size = len;
-	ds->desc_num = num;
 	num = 0;
 
 	if (!c->ccfg) {
@@ -474,18 +531,17 @@ static struct dma_async_tx_descriptor *k3_dma_prep_slave_sg(
 	if (sgl == NULL)
 		return NULL;
 
+	c->cyclic = 0;
+
 	for_each_sg(sgl, sg, sglen, i) {
 		avail = sg_dma_len(sg);
 		if (avail > DMA_MAX_SIZE)
 			num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1;
 	}
 
-	ds = kzalloc(sizeof(*ds) + num * sizeof(ds->desc_hw[0]), GFP_ATOMIC);
+	ds = k3_dma_alloc_desc_resource(num, chan);
 	if (!ds)
 		return NULL;
-
-	ds->desc_hw_lli = __virt_to_phys((unsigned long)&ds->desc_hw[0]);
-	ds->desc_num = num;
 	num = 0;
 
 	for_each_sg(sgl, sg, sglen, i) {
@@ -516,6 +572,73 @@ static struct dma_async_tx_descriptor *k3_dma_prep_slave_sg(
 	return vchan_tx_prep(&c->vc, &ds->vd, flags);
 }
 
+static struct dma_async_tx_descriptor *
+k3_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
+		       size_t buf_len, size_t period_len,
+		       enum dma_transfer_direction dir,
+		       unsigned long flags)
+{
+	struct k3_dma_chan *c = to_k3_chan(chan);
+	struct k3_dma_desc_sw *ds;
+	size_t len, avail, total = 0;
+	dma_addr_t addr, src = 0, dst = 0;
+	int num = 1, since = 0;
+	size_t modulo = DMA_CYCLIC_MAX_PERIOD;
+	u32 en_tc2 = 0;
+
+	dev_dbg(chan->device->dev, "%s: buf %pad, dst %pad, buf len %zu, period_len = %zu, dir %d\n",
+	       __func__, &buf_addr, &to_k3_chan(chan)->dev_addr,
+	       buf_len, period_len, (int)dir);
+
+	avail = buf_len;
+	if (avail > modulo)
+		num += DIV_ROUND_UP(avail, modulo) - 1;
+
+	ds = k3_dma_alloc_desc_resource(num, chan);
+	if (!ds)
+		return NULL;
+
+	c->cyclic = 1;
+	addr = buf_addr;
+	avail = buf_len;
+	total = avail;
+	num = 0;
+
+	if (period_len < modulo)
+		modulo = period_len;
+
+	do {
+		len = min_t(size_t, avail, modulo);
+
+		if (dir == DMA_MEM_TO_DEV) {
+			src = addr;
+			dst = c->dev_addr;
+		} else if (dir == DMA_DEV_TO_MEM) {
+			src = c->dev_addr;
+			dst = addr;
+		}
+		since += len;
+		if (since >= period_len) {
+			/* descriptor asks for TC2 interrupt on completion */
+			en_tc2 = CX_CFG_NODEIRQ;
+			since -= period_len;
+		} else
+			en_tc2 = 0;
+
+		k3_dma_fill_desc(ds, dst, src, len, num++, c->ccfg | en_tc2);
+
+		addr += len;
+		avail -= len;
+	} while (avail);
+
+	/* "Cyclic" == end of link points back to start of link */
+	ds->desc_hw[num - 1].lli |= ds->desc_hw_lli;
+
+	ds->size = total;
+
+	return vchan_tx_prep(&c->vc, &ds->vd, flags);
+}
+
 static int k3_dma_config(struct dma_chan *chan,
 			 struct dma_slave_config *cfg)
 {
@@ -551,7 +674,7 @@ static int k3_dma_config(struct dma_chan *chan,
 	c->ccfg |= (val << 12) | (val << 16);
 
 	if ((maxburst == 0) || (maxburst > 16))
-		val = 16;
+		val = 15;
 	else
 		val = maxburst - 1;
 	c->ccfg |= (val << 20) | (val << 24);
@@ -563,6 +686,16 @@ static int k3_dma_config(struct dma_chan *chan,
 	return 0;
 }
 
+static void k3_dma_free_desc(struct virt_dma_desc *vd)
+{
+	struct k3_dma_desc_sw *ds =
+		container_of(vd, struct k3_dma_desc_sw, vd);
+	struct k3_dma_dev *d = to_k3_dma(vd->tx.chan->device);
+
+	dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli);
+	kfree(ds);
+}
+
 static int k3_dma_terminate_all(struct dma_chan *chan)
 {
 	struct k3_dma_chan *c = to_k3_chan(chan);
@@ -586,7 +719,15 @@ static int k3_dma_terminate_all(struct dma_chan *chan)
 		k3_dma_terminate_chan(p, d);
 		c->phy = NULL;
 		p->vchan = NULL;
-		p->ds_run = p->ds_done = NULL;
+		if (p->ds_run) {
+			k3_dma_free_desc(&p->ds_run->vd);
+			p->ds_run = NULL;
+		}
+		if (p->ds_done) {
+			k3_dma_free_desc(&p->ds_done->vd);
+			p->ds_done = NULL;
+		}
+
 	}
 	spin_unlock_irqrestore(&c->vc.lock, flags);
 	vchan_dma_desc_free_list(&c->vc, &head);
@@ -639,14 +780,6 @@ static int k3_dma_transfer_resume(struct dma_chan *chan)
 	return 0;
 }
 
-static void k3_dma_free_desc(struct virt_dma_desc *vd)
-{
-	struct k3_dma_desc_sw *ds =
-		container_of(vd, struct k3_dma_desc_sw, vd);
-
-	kfree(ds);
-}
-
 static const struct of_device_id k3_pdma_dt_ids[] = {
 	{ .compatible = "hisilicon,k3-dma-1.0", },
 	{}
@@ -706,6 +839,12 @@ static int k3_dma_probe(struct platform_device *op)
 
 	d->irq = irq;
 
+	/* A DMA memory pool for LLIs, align on 32-byte boundary */
+	d->pool = dmam_pool_create(DRIVER_NAME, &op->dev,
+					LLI_BLOCK_SIZE, 32, 0);
+	if (!d->pool)
+		return -ENOMEM;
+
 	/* init phy channel */
 	d->phy = devm_kzalloc(&op->dev,
 		d->dma_channels * sizeof(struct k3_dma_phy), GFP_KERNEL);
@@ -722,11 +861,13 @@ static int k3_dma_probe(struct platform_device *op)
 	INIT_LIST_HEAD(&d->slave.channels);
 	dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 	dma_cap_set(DMA_MEMCPY, d->slave.cap_mask);
+	dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
 	d->slave.dev = &op->dev;
 	d->slave.device_free_chan_resources = k3_dma_free_chan_resources;
 	d->slave.device_tx_status = k3_dma_tx_status;
 	d->slave.device_prep_dma_memcpy = k3_dma_prep_memcpy;
 	d->slave.device_prep_slave_sg = k3_dma_prep_slave_sg;
+	d->slave.device_prep_dma_cyclic = k3_dma_prep_dma_cyclic;
 	d->slave.device_issue_pending = k3_dma_issue_pending;
 	d->slave.device_config = k3_dma_config;
 	d->slave.device_pause = k3_dma_transfer_pause;
diff --git a/drivers/dma/mic_x100_dma.c b/drivers/dma/mic_x100_dma.c
index 1502b24b7c7d..818255844a3c 100644
--- a/drivers/dma/mic_x100_dma.c
+++ b/drivers/dma/mic_x100_dma.c
@@ -104,10 +104,8 @@ static void mic_dma_cleanup(struct mic_dma_chan *ch)
 		tx = &ch->tx_array[last_tail];
 		if (tx->cookie) {
 			dma_cookie_complete(tx);
-			if (tx->callback) {
-				tx->callback(tx->callback_param);
-				tx->callback = NULL;
-			}
+			dmaengine_desc_get_callback_invoke(tx, NULL);
+			tx->callback = NULL;
 		}
 		last_tail = mic_dma_hw_ring_inc(last_tail);
 	}
diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c
index f4b25fb0d040..eb3a1f42ab06 100644
--- a/drivers/dma/mmp_pdma.c
+++ b/drivers/dma/mmp_pdma.c
@@ -864,19 +864,15 @@ static void dma_do_tasklet(unsigned long data)
 	struct mmp_pdma_desc_sw *desc, *_desc;
 	LIST_HEAD(chain_cleanup);
 	unsigned long flags;
+	struct dmaengine_desc_callback cb;
 
 	if (chan->cyclic_first) {
-		dma_async_tx_callback cb = NULL;
-		void *cb_data = NULL;
-
 		spin_lock_irqsave(&chan->desc_lock, flags);
 		desc = chan->cyclic_first;
-		cb = desc->async_tx.callback;
-		cb_data = desc->async_tx.callback_param;
+		dmaengine_desc_get_callback(&desc->async_tx, &cb);
 		spin_unlock_irqrestore(&chan->desc_lock, flags);
 
-		if (cb)
-			cb(cb_data);
+		dmaengine_desc_callback_invoke(&cb, NULL);
 
 		return;
 	}
@@ -921,8 +917,8 @@ static void dma_do_tasklet(unsigned long data)
 		/* Remove from the list of transactions */
 		list_del(&desc->node);
 		/* Run the link descriptor callback function */
-		if (txd->callback)
-			txd->callback(txd->callback_param);
+		dmaengine_desc_get_callback(txd, &cb);
+		dmaengine_desc_callback_invoke(&cb, NULL);
 
 		dma_pool_free(chan->desc_pool, desc, txd->phys);
 	}
diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c
index b3441f57a364..13c68b6434ce 100644
--- a/drivers/dma/mmp_tdma.c
+++ b/drivers/dma/mmp_tdma.c
@@ -349,9 +349,7 @@ static void dma_do_tasklet(unsigned long data)
 {
 	struct mmp_tdma_chan *tdmac = (struct mmp_tdma_chan *)data;
 
-	if (tdmac->desc.callback)
-		tdmac->desc.callback(tdmac->desc.callback_param);
-
+	dmaengine_desc_get_callback_invoke(&tdmac->desc, NULL);
 }
 
 static void mmp_tdma_free_descriptor(struct mmp_tdma_chan *tdmac)
@@ -433,7 +431,7 @@ static struct dma_async_tx_descriptor *mmp_tdma_prep_dma_cyclic(
 
 	if (period_len > TDMA_MAX_XFER_BYTES) {
 		dev_err(tdmac->dev,
-				"maximum period size exceeded: %d > %d\n",
+				"maximum period size exceeded: %zu > %d\n",
 				period_len, TDMA_MAX_XFER_BYTES);
 		goto err_out;
 	}
diff --git a/drivers/dma/moxart-dma.c b/drivers/dma/moxart-dma.c
index a6e642792e5a..e1a5c2242f6f 100644
--- a/drivers/dma/moxart-dma.c
+++ b/drivers/dma/moxart-dma.c
@@ -579,7 +579,7 @@ static int moxart_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	irq = irq_of_parse_and_map(node, 0);
-	if (irq == NO_IRQ) {
+	if (!irq) {
 		dev_err(dev, "no IRQ resource\n");
 		return -EINVAL;
 	}
diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c
index fa86592c7ae1..dde713461a95 100644
--- a/drivers/dma/mpc512x_dma.c
+++ b/drivers/dma/mpc512x_dma.c
@@ -411,8 +411,7 @@ static void mpc_dma_process_completed(struct mpc_dma *mdma)
 		list_for_each_entry(mdesc, &list, node) {
 			desc = &mdesc->desc;
 
-			if (desc->callback)
-				desc->callback(desc->callback_param);
+			dmaengine_desc_get_callback_invoke(desc, NULL);
 
 			last_cookie = desc->cookie;
 			dma_run_dependencies(desc);
@@ -926,7 +925,7 @@ static int mpc_dma_probe(struct platform_device *op)
 	}
 
 	mdma->irq = irq_of_parse_and_map(dn, 0);
-	if (mdma->irq == NO_IRQ) {
+	if (!mdma->irq) {
 		dev_err(dev, "Error mapping IRQ!\n");
 		retval = -EINVAL;
 		goto err;
@@ -935,7 +934,7 @@ static int mpc_dma_probe(struct platform_device *op)
 	if (of_device_is_compatible(dn, "fsl,mpc8308-dma")) {
 		mdma->is_mpc8308 = 1;
 		mdma->irq2 = irq_of_parse_and_map(dn, 1);
-		if (mdma->irq2 == NO_IRQ) {
+		if (!mdma->irq2) {
 			dev_err(dev, "Error mapping IRQ!\n");
 			retval = -EINVAL;
 			goto err_dispose1;
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index f4c9f98ec35e..23f75285a4d9 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -206,14 +206,11 @@ mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
 	if (desc->async_tx.cookie > 0) {
 		cookie = desc->async_tx.cookie;
 
+		dma_descriptor_unmap(&desc->async_tx);
 		/* call the callback (must not sleep or submit new
 		 * operations to this channel)
 		 */
-		if (desc->async_tx.callback)
-			desc->async_tx.callback(
-				desc->async_tx.callback_param);
-
-		dma_descriptor_unmap(&desc->async_tx);
+		dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
 	}
 
 	/* run dependent operations */
@@ -470,12 +467,90 @@ static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
 	return mv_chan->slots_allocated ? : -ENOMEM;
 }
 
+/*
+ * Check if source or destination is an PCIe/IO address (non-SDRAM) and add
+ * a new MBus window if necessary. Use a cache for these check so that
+ * the MMIO mapped registers don't have to be accessed for this check
+ * to speed up this process.
+ */
+static int mv_xor_add_io_win(struct mv_xor_chan *mv_chan, u32 addr)
+{
+	struct mv_xor_device *xordev = mv_chan->xordev;
+	void __iomem *base = mv_chan->mmr_high_base;
+	u32 win_enable;
+	u32 size;
+	u8 target, attr;
+	int ret;
+	int i;
+
+	/* Nothing needs to get done for the Armada 3700 */
+	if (xordev->xor_type == XOR_ARMADA_37XX)
+		return 0;
+
+	/*
+	 * Loop over the cached windows to check, if the requested area
+	 * is already mapped. If this the case, nothing needs to be done
+	 * and we can return.
+	 */
+	for (i = 0; i < WINDOW_COUNT; i++) {
+		if (addr >= xordev->win_start[i] &&
+		    addr <= xordev->win_end[i]) {
+			/* Window is already mapped */
+			return 0;
+		}
+	}
+
+	/*
+	 * The window is not mapped, so we need to create the new mapping
+	 */
+
+	/* If no IO window is found that addr has to be located in SDRAM */
+	ret = mvebu_mbus_get_io_win_info(addr, &size, &target, &attr);
+	if (ret < 0)
+		return 0;
+
+	/*
+	 * Mask the base addr 'addr' according to 'size' read back from the
+	 * MBus window. Otherwise we might end up with an address located
+	 * somewhere in the middle of this area here.
+	 */
+	size -= 1;
+	addr &= ~size;
+
+	/*
+	 * Reading one of both enabled register is enough, as they are always
+	 * programmed to the identical values
+	 */
+	win_enable = readl(base + WINDOW_BAR_ENABLE(0));
+
+	/* Set 'i' to the first free window to write the new values to */
+	i = ffs(~win_enable) - 1;
+	if (i >= WINDOW_COUNT)
+		return -ENOMEM;
+
+	writel((addr & 0xffff0000) | (attr << 8) | target,
+	       base + WINDOW_BASE(i));
+	writel(size & 0xffff0000, base + WINDOW_SIZE(i));
+
+	/* Fill the caching variables for later use */
+	xordev->win_start[i] = addr;
+	xordev->win_end[i] = addr + size;
+
+	win_enable |= (1 << i);
+	win_enable |= 3 << (16 + (2 * i));
+	writel(win_enable, base + WINDOW_BAR_ENABLE(0));
+	writel(win_enable, base + WINDOW_BAR_ENABLE(1));
+
+	return 0;
+}
+
 static struct dma_async_tx_descriptor *
 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
 		    unsigned int src_cnt, size_t len, unsigned long flags)
 {
 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 	struct mv_xor_desc_slot *sw_desc;
+	int ret;
 
 	if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
 		return NULL;
@@ -486,6 +561,11 @@ mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
 		"%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
 		__func__, src_cnt, len, &dest, flags);
 
+	/* Check if a new window needs to get added for 'dest' */
+	ret = mv_xor_add_io_win(mv_chan, dest);
+	if (ret)
+		return NULL;
+
 	sw_desc = mv_chan_alloc_slot(mv_chan);
 	if (sw_desc) {
 		sw_desc->type = DMA_XOR;
@@ -493,8 +573,13 @@ mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
 		mv_desc_init(sw_desc, dest, len, flags);
 		if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
 			mv_desc_set_mode(sw_desc);
-		while (src_cnt--)
+		while (src_cnt--) {
+			/* Check if a new window needs to get added for 'src' */
+			ret = mv_xor_add_io_win(mv_chan, src[src_cnt]);
+			if (ret)
+				return NULL;
 			mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
+		}
 	}
 
 	dev_dbg(mv_chan_to_devp(mv_chan),
@@ -959,6 +1044,7 @@ mv_xor_channel_add(struct mv_xor_device *xordev,
 		mv_chan->op_in_desc = XOR_MODE_IN_DESC;
 
 	dma_dev = &mv_chan->dmadev;
+	mv_chan->xordev = xordev;
 
 	/*
 	 * These source and destination dummy buffers are used to implement
@@ -1086,6 +1172,10 @@ mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
 		       dram->mbus_dram_target_id, base + WINDOW_BASE(i));
 		writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
 
+		/* Fill the caching variables for later use */
+		xordev->win_start[i] = cs->base;
+		xordev->win_end[i] = cs->base + cs->size - 1;
+
 		win_enable |= (1 << i);
 		win_enable |= 3 << (16 + (2 * i));
 	}
diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h
index bf56e082e7cd..88eeab222a23 100644
--- a/drivers/dma/mv_xor.h
+++ b/drivers/dma/mv_xor.h
@@ -80,12 +80,17 @@
 #define WINDOW_BAR_ENABLE(chan)	(0x40 + ((chan) << 2))
 #define WINDOW_OVERRIDE_CTRL(chan)	(0xA0 + ((chan) << 2))
 
+#define WINDOW_COUNT		8
+
 struct mv_xor_device {
 	void __iomem	     *xor_base;
 	void __iomem	     *xor_high_base;
 	struct clk	     *clk;
 	struct mv_xor_chan   *channels[MV_XOR_MAX_CHANNELS];
 	int		     xor_type;
+
+	u32                  win_start[WINDOW_COUNT];
+	u32                  win_end[WINDOW_COUNT];
 };
 
 /**
@@ -127,6 +132,8 @@ struct mv_xor_chan {
 	char			dummy_dst[MV_XOR_MIN_BYTE_COUNT];
 	dma_addr_t		dummy_src_addr, dummy_dst_addr;
 	u32                     saved_config_reg, saved_int_mask_reg;
+
+	struct mv_xor_device	*xordev;
 };
 
 /**
diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c
index 60de35251da5..e217268c7098 100644
--- a/drivers/dma/mxs-dma.c
+++ b/drivers/dma/mxs-dma.c
@@ -326,8 +326,7 @@ static void mxs_dma_tasklet(unsigned long data)
 {
 	struct mxs_dma_chan *mxs_chan = (struct mxs_dma_chan *) data;
 
-	if (mxs_chan->desc.callback)
-		mxs_chan->desc.callback(mxs_chan->desc.callback_param);
+	dmaengine_desc_get_callback_invoke(&mxs_chan->desc, NULL);
 }
 
 static int mxs_dma_irq_to_chan(struct mxs_dma_engine *mxs_dma, int irq)
@@ -429,12 +428,10 @@ static int mxs_dma_alloc_chan_resources(struct dma_chan *chan)
 		goto err_alloc;
 	}
 
-	if (mxs_chan->chan_irq != NO_IRQ) {
-		ret = request_irq(mxs_chan->chan_irq, mxs_dma_int_handler,
-					0, "mxs-dma", mxs_dma);
-		if (ret)
-			goto err_irq;
-	}
+	ret = request_irq(mxs_chan->chan_irq, mxs_dma_int_handler,
+			  0, "mxs-dma", mxs_dma);
+	if (ret)
+		goto err_irq;
 
 	ret = clk_prepare_enable(mxs_dma->clk);
 	if (ret)
diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c
index 08c45c185549..09de71519d37 100644
--- a/drivers/dma/nbpfaxi.c
+++ b/drivers/dma/nbpfaxi.c
@@ -1102,8 +1102,7 @@ static void nbpf_chan_tasklet(unsigned long data)
 {
 	struct nbpf_channel *chan = (struct nbpf_channel *)data;
 	struct nbpf_desc *desc, *tmp;
-	dma_async_tx_callback callback;
-	void *param;
+	struct dmaengine_desc_callback cb;
 
 	while (!list_empty(&chan->done)) {
 		bool found = false, must_put, recycling = false;
@@ -1151,14 +1150,12 @@ static void nbpf_chan_tasklet(unsigned long data)
 			must_put = false;
 		}
 
-		callback = desc->async_tx.callback;
-		param = desc->async_tx.callback_param;
+		dmaengine_desc_get_callback(&desc->async_tx, &cb);
 
 		/* ack and callback completed descriptor */
 		spin_unlock_irq(&chan->lock);
 
-		if (callback)
-			callback(param);
+		dmaengine_desc_callback_invoke(&cb, NULL);
 
 		if (must_put)
 			nbpf_desc_put(desc);
diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c
index d99ca2b511c4..7ca27d4b1c54 100644
--- a/drivers/dma/omap-dma.c
+++ b/drivers/dma/omap-dma.c
@@ -8,6 +8,7 @@
 #include <linux/delay.h>
 #include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -32,10 +33,12 @@ struct omap_dmadev {
 	const struct omap_dma_reg *reg_map;
 	struct omap_system_dma_plat_info *plat;
 	bool legacy;
+	bool ll123_supported;
+	struct dma_pool *desc_pool;
 	unsigned dma_requests;
 	spinlock_t irq_lock;
 	uint32_t irq_enable_mask;
-	struct omap_chan *lch_map[OMAP_SDMA_CHANNELS];
+	struct omap_chan **lch_map;
 };
 
 struct omap_chan {
@@ -55,16 +58,40 @@ struct omap_chan {
 	unsigned sgidx;
 };
 
+#define DESC_NXT_SV_REFRESH	(0x1 << 24)
+#define DESC_NXT_SV_REUSE	(0x2 << 24)
+#define DESC_NXT_DV_REFRESH	(0x1 << 26)
+#define DESC_NXT_DV_REUSE	(0x2 << 26)
+#define DESC_NTYPE_TYPE2	(0x2 << 29)
+
+/* Type 2 descriptor with Source or Destination address update */
+struct omap_type2_desc {
+	uint32_t next_desc;
+	uint32_t en;
+	uint32_t addr; /* src or dst */
+	uint16_t fn;
+	uint16_t cicr;
+	int16_t cdei;
+	int16_t csei;
+	int32_t cdfi;
+	int32_t csfi;
+} __packed;
+
 struct omap_sg {
 	dma_addr_t addr;
 	uint32_t en;		/* number of elements (24-bit) */
 	uint32_t fn;		/* number of frames (16-bit) */
 	int32_t fi;		/* for double indexing */
 	int16_t ei;		/* for double indexing */
+
+	/* Linked list */
+	struct omap_type2_desc *t2_desc;
+	dma_addr_t t2_desc_paddr;
 };
 
 struct omap_desc {
 	struct virt_dma_desc vd;
+	bool using_ll;
 	enum dma_transfer_direction dir;
 	dma_addr_t dev_addr;
 
@@ -81,6 +108,9 @@ struct omap_desc {
 };
 
 enum {
+	CAPS_0_SUPPORT_LL123	= BIT(20),	/* Linked List type1/2/3 */
+	CAPS_0_SUPPORT_LL4	= BIT(21),	/* Linked List type4 */
+
 	CCR_FS			= BIT(5),
 	CCR_READ_PRIORITY	= BIT(6),
 	CCR_ENABLE		= BIT(7),
@@ -151,6 +181,19 @@ enum {
 	CICR_SUPER_BLOCK_IE	= BIT(14),	/* OMAP2+ only */
 
 	CLNK_CTRL_ENABLE_LNK	= BIT(15),
+
+	CDP_DST_VALID_INC	= 0 << 0,
+	CDP_DST_VALID_RELOAD	= 1 << 0,
+	CDP_DST_VALID_REUSE	= 2 << 0,
+	CDP_SRC_VALID_INC	= 0 << 2,
+	CDP_SRC_VALID_RELOAD	= 1 << 2,
+	CDP_SRC_VALID_REUSE	= 2 << 2,
+	CDP_NTYPE_TYPE1		= 1 << 4,
+	CDP_NTYPE_TYPE2		= 2 << 4,
+	CDP_NTYPE_TYPE3		= 3 << 4,
+	CDP_TMODE_NORMAL	= 0 << 8,
+	CDP_TMODE_LLIST		= 1 << 8,
+	CDP_FAST		= BIT(10),
 };
 
 static const unsigned es_bytes[] = {
@@ -180,7 +223,64 @@ static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor
 
 static void omap_dma_desc_free(struct virt_dma_desc *vd)
 {
-	kfree(container_of(vd, struct omap_desc, vd));
+	struct omap_desc *d = to_omap_dma_desc(&vd->tx);
+
+	if (d->using_ll) {
+		struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
+		int i;
+
+		for (i = 0; i < d->sglen; i++) {
+			if (d->sg[i].t2_desc)
+				dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
+					      d->sg[i].t2_desc_paddr);
+		}
+	}
+
+	kfree(d);
+}
+
+static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
+				     enum dma_transfer_direction dir, bool last)
+{
+	struct omap_sg *sg = &d->sg[idx];
+	struct omap_type2_desc *t2_desc = sg->t2_desc;
+
+	if (idx)
+		d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
+	if (last)
+		t2_desc->next_desc = 0xfffffffc;
+
+	t2_desc->en = sg->en;
+	t2_desc->addr = sg->addr;
+	t2_desc->fn = sg->fn & 0xffff;
+	t2_desc->cicr = d->cicr;
+	if (!last)
+		t2_desc->cicr &= ~CICR_BLOCK_IE;
+
+	switch (dir) {
+	case DMA_DEV_TO_MEM:
+		t2_desc->cdei = sg->ei;
+		t2_desc->csei = d->ei;
+		t2_desc->cdfi = sg->fi;
+		t2_desc->csfi = d->fi;
+
+		t2_desc->en |= DESC_NXT_DV_REFRESH;
+		t2_desc->en |= DESC_NXT_SV_REUSE;
+		break;
+	case DMA_MEM_TO_DEV:
+		t2_desc->cdei = d->ei;
+		t2_desc->csei = sg->ei;
+		t2_desc->cdfi = d->fi;
+		t2_desc->csfi = sg->fi;
+
+		t2_desc->en |= DESC_NXT_SV_REFRESH;
+		t2_desc->en |= DESC_NXT_DV_REUSE;
+		break;
+	default:
+		return;
+	}
+
+	t2_desc->en |= DESC_NTYPE_TYPE2;
 }
 
 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
@@ -285,6 +385,7 @@ static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 {
 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
+	uint16_t cicr = d->cicr;
 
 	if (__dma_omap15xx(od->plat->dma_attr))
 		omap_dma_chan_write(c, CPC, 0);
@@ -293,8 +394,27 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 
 	omap_dma_clear_csr(c);
 
+	if (d->using_ll) {
+		uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
+
+		if (d->dir == DMA_DEV_TO_MEM)
+			cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
+		else
+			cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
+		omap_dma_chan_write(c, CDP, cdp);
+
+		omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
+		omap_dma_chan_write(c, CCDN, 0);
+		omap_dma_chan_write(c, CCFN, 0xffff);
+		omap_dma_chan_write(c, CCEN, 0xffffff);
+
+		cicr &= ~CICR_BLOCK_IE;
+	} else if (od->ll123_supported) {
+		omap_dma_chan_write(c, CDP, 0);
+	}
+
 	/* Enable interrupts */
-	omap_dma_chan_write(c, CICR, d->cicr);
+	omap_dma_chan_write(c, CICR, cicr);
 
 	/* Enable channel */
 	omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
@@ -365,10 +485,9 @@ static void omap_dma_stop(struct omap_chan *c)
 	c->running = false;
 }
 
-static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d,
-	unsigned idx)
+static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
 {
-	struct omap_sg *sg = d->sg + idx;
+	struct omap_sg *sg = d->sg + c->sgidx;
 	unsigned cxsa, cxei, cxfi;
 
 	if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
@@ -388,6 +507,7 @@ static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d,
 	omap_dma_chan_write(c, CFN, sg->fn);
 
 	omap_dma_start(c, d);
+	c->sgidx++;
 }
 
 static void omap_dma_start_desc(struct omap_chan *c)
@@ -433,7 +553,7 @@ static void omap_dma_start_desc(struct omap_chan *c)
 	omap_dma_chan_write(c, CSDP, d->csdp);
 	omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
 
-	omap_dma_start_sg(c, d, 0);
+	omap_dma_start_sg(c, d);
 }
 
 static void omap_dma_callback(int ch, u16 status, void *data)
@@ -445,15 +565,13 @@ static void omap_dma_callback(int ch, u16 status, void *data)
 	spin_lock_irqsave(&c->vc.lock, flags);
 	d = c->desc;
 	if (d) {
-		if (!c->cyclic) {
-			if (++c->sgidx < d->sglen) {
-				omap_dma_start_sg(c, d, c->sgidx);
-			} else {
-				omap_dma_start_desc(c);
-				vchan_cookie_complete(&d->vd);
-			}
-		} else {
+		if (c->cyclic) {
 			vchan_cyclic_callback(&d->vd);
+		} else if (d->using_ll || c->sgidx == d->sglen) {
+			omap_dma_start_desc(c);
+			vchan_cookie_complete(&d->vd);
+		} else {
+			omap_dma_start_sg(c, d);
 		}
 	}
 	spin_unlock_irqrestore(&c->vc.lock, flags);
@@ -503,6 +621,7 @@ static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 	struct omap_chan *c = to_omap_dma_chan(chan);
+	struct device *dev = od->ddev.dev;
 	int ret;
 
 	if (od->legacy) {
@@ -513,8 +632,7 @@ static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 				       &c->dma_ch);
 	}
 
-	dev_dbg(od->ddev.dev, "allocating channel %u for %u\n",
-		c->dma_ch, c->dma_sig);
+	dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
 
 	if (ret >= 0) {
 		omap_dma_assign(od, c, c->dma_ch);
@@ -570,7 +688,8 @@ static void omap_dma_free_chan_resources(struct dma_chan *chan)
 	vchan_free_chan_resources(&c->vc);
 	omap_free_dma(c->dma_ch);
 
-	dev_dbg(od->ddev.dev, "freeing channel for %u\n", c->dma_sig);
+	dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
+		c->dma_sig);
 	c->dma_sig = 0;
 }
 
@@ -744,6 +863,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 	struct omap_desc *d;
 	dma_addr_t dev_addr;
 	unsigned i, es, en, frame_bytes;
+	bool ll_failed = false;
 	u32 burst;
 
 	if (dir == DMA_DEV_TO_MEM) {
@@ -784,13 +904,16 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 	d->es = es;
 
 	d->ccr = c->ccr | CCR_SYNC_FRAME;
-	if (dir == DMA_DEV_TO_MEM)
+	if (dir == DMA_DEV_TO_MEM) {
 		d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
-	else
+		d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
+	} else {
 		d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
+		d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
+	}
 
 	d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
-	d->csdp = es;
+	d->csdp |= es;
 
 	if (dma_omap1()) {
 		d->cicr |= CICR_TOUT_IE;
@@ -819,14 +942,47 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 	 */
 	en = burst;
 	frame_bytes = es_bytes[es] * en;
+
+	if (sglen >= 2)
+		d->using_ll = od->ll123_supported;
+
 	for_each_sg(sgl, sgent, sglen, i) {
-		d->sg[i].addr = sg_dma_address(sgent);
-		d->sg[i].en = en;
-		d->sg[i].fn = sg_dma_len(sgent) / frame_bytes;
+		struct omap_sg *osg = &d->sg[i];
+
+		osg->addr = sg_dma_address(sgent);
+		osg->en = en;
+		osg->fn = sg_dma_len(sgent) / frame_bytes;
+
+		if (d->using_ll) {
+			osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
+						      &osg->t2_desc_paddr);
+			if (!osg->t2_desc) {
+				dev_err(chan->device->dev,
+					"t2_desc[%d] allocation failed\n", i);
+				ll_failed = true;
+				d->using_ll = false;
+				continue;
+			}
+
+			omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
+		}
 	}
 
 	d->sglen = sglen;
 
+	/* Release the dma_pool entries if one allocation failed */
+	if (ll_failed) {
+		for (i = 0; i < d->sglen; i++) {
+			struct omap_sg *osg = &d->sg[i];
+
+			if (osg->t2_desc) {
+				dma_pool_free(od->desc_pool, osg->t2_desc,
+					      osg->t2_desc_paddr);
+				osg->t2_desc = NULL;
+			}
+		}
+	}
+
 	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
 }
 
@@ -1225,16 +1381,24 @@ static int omap_dma_probe(struct platform_device *pdev)
 	spin_lock_init(&od->lock);
 	spin_lock_init(&od->irq_lock);
 
-	od->dma_requests = OMAP_SDMA_REQUESTS;
-	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
-						      "dma-requests",
-						      &od->dma_requests)) {
+	if (!pdev->dev.of_node) {
+		od->dma_requests = od->plat->dma_attr->lch_count;
+		if (unlikely(!od->dma_requests))
+			od->dma_requests = OMAP_SDMA_REQUESTS;
+	} else if (of_property_read_u32(pdev->dev.of_node, "dma-requests",
+					&od->dma_requests)) {
 		dev_info(&pdev->dev,
 			 "Missing dma-requests property, using %u.\n",
 			 OMAP_SDMA_REQUESTS);
+		od->dma_requests = OMAP_SDMA_REQUESTS;
 	}
 
-	for (i = 0; i < OMAP_SDMA_CHANNELS; i++) {
+	od->lch_map = devm_kcalloc(&pdev->dev, od->dma_requests,
+				   sizeof(*od->lch_map), GFP_KERNEL);
+	if (!od->lch_map)
+		return -ENOMEM;
+
+	for (i = 0; i < od->dma_requests; i++) {
 		rc = omap_dma_chan_init(od);
 		if (rc) {
 			omap_dma_free(od);
@@ -1257,10 +1421,25 @@ static int omap_dma_probe(struct platform_device *pdev)
 			return rc;
 	}
 
+	if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
+		od->ll123_supported = true;
+
 	od->ddev.filter.map = od->plat->slave_map;
 	od->ddev.filter.mapcnt = od->plat->slavecnt;
 	od->ddev.filter.fn = omap_dma_filter_fn;
 
+	if (od->ll123_supported) {
+		od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
+						&pdev->dev,
+						sizeof(struct omap_type2_desc),
+						4, 0);
+		if (!od->desc_pool) {
+			dev_err(&pdev->dev,
+				"unable to allocate descriptor pool\n");
+			od->ll123_supported = false;
+		}
+	}
+
 	rc = dma_async_device_register(&od->ddev);
 	if (rc) {
 		pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
@@ -1284,7 +1463,8 @@ static int omap_dma_probe(struct platform_device *pdev)
 		}
 	}
 
-	dev_info(&pdev->dev, "OMAP DMA engine driver\n");
+	dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
+		 od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
 
 	return rc;
 }
@@ -1307,6 +1487,9 @@ static int omap_dma_remove(struct platform_device *pdev)
 		omap_dma_glbl_write(od, IRQENABLE_L0, 0);
 	}
 
+	if (od->ll123_supported)
+		dma_pool_destroy(od->desc_pool);
+
 	omap_dma_free(od);
 
 	return 0;
diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c
index 113605f6fe20..df95727dc2fb 100644
--- a/drivers/dma/pch_dma.c
+++ b/drivers/dma/pch_dma.c
@@ -357,14 +357,13 @@ static void pdc_chain_complete(struct pch_dma_chan *pd_chan,
 			       struct pch_dma_desc *desc)
 {
 	struct dma_async_tx_descriptor *txd = &desc->txd;
-	dma_async_tx_callback callback = txd->callback;
-	void *param = txd->callback_param;
+	struct dmaengine_desc_callback cb;
 
+	dmaengine_desc_get_callback(txd, &cb);
 	list_splice_init(&desc->tx_list, &pd_chan->free_list);
 	list_move(&desc->desc_node, &pd_chan->free_list);
 
-	if (callback)
-		callback(param);
+	dmaengine_desc_callback_invoke(&cb, NULL);
 }
 
 static void pdc_complete_all(struct pch_dma_chan *pd_chan)
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 4fc3ffbd5ca0..030fe05ed43b 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -2039,14 +2039,12 @@ static void pl330_tasklet(unsigned long data)
 	}
 
 	while (!list_empty(&pch->completed_list)) {
-		dma_async_tx_callback callback;
-		void *callback_param;
+		struct dmaengine_desc_callback cb;
 
 		desc = list_first_entry(&pch->completed_list,
 					struct dma_pl330_desc, node);
 
-		callback = desc->txd.callback;
-		callback_param = desc->txd.callback_param;
+		dmaengine_desc_get_callback(&desc->txd, &cb);
 
 		if (pch->cyclic) {
 			desc->status = PREP;
@@ -2064,9 +2062,9 @@ static void pl330_tasklet(unsigned long data)
 
 		dma_descriptor_unmap(&desc->txd);
 
-		if (callback) {
+		if (dmaengine_desc_callback_valid(&cb)) {
 			spin_unlock_irqrestore(&pch->lock, flags);
-			callback(callback_param);
+			dmaengine_desc_callback_invoke(&cb, NULL);
 			spin_lock_irqsave(&pch->lock, flags);
 		}
 	}
@@ -2274,7 +2272,7 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 {
 	enum dma_status ret;
 	unsigned long flags;
-	struct dma_pl330_desc *desc, *running = NULL;
+	struct dma_pl330_desc *desc, *running = NULL, *last_enq = NULL;
 	struct dma_pl330_chan *pch = to_pchan(chan);
 	unsigned int transferred, residual = 0;
 
@@ -2287,10 +2285,13 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 		goto out;
 
 	spin_lock_irqsave(&pch->lock, flags);
+	spin_lock(&pch->thread->dmac->lock);
 
 	if (pch->thread->req_running != -1)
 		running = pch->thread->req[pch->thread->req_running].desc;
 
+	last_enq = pch->thread->req[pch->thread->lstenq].desc;
+
 	/* Check in pending list */
 	list_for_each_entry(desc, &pch->work_list, node) {
 		if (desc->status == DONE)
@@ -2298,6 +2299,15 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 		else if (running && desc == running)
 			transferred =
 				pl330_get_current_xferred_count(pch, desc);
+		else if (desc->status == BUSY)
+			/*
+			 * Busy but not running means either just enqueued,
+			 * or finished and not yet marked done
+			 */
+			if (desc == last_enq)
+				transferred = 0;
+			else
+				transferred = desc->bytes_requested;
 		else
 			transferred = 0;
 		residual += desc->bytes_requested - transferred;
@@ -2318,6 +2328,7 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 		if (desc->last)
 			residual = 0;
 	}
+	spin_unlock(&pch->thread->dmac->lock);
 	spin_unlock_irqrestore(&pch->lock, flags);
 
 out:
diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c
index da3688b94bdc..b1535b1fe95c 100644
--- a/drivers/dma/ppc4xx/adma.c
+++ b/drivers/dma/ppc4xx/adma.c
@@ -1482,14 +1482,11 @@ static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
 		cookie = desc->async_tx.cookie;
 		desc->async_tx.cookie = 0;
 
+		dma_descriptor_unmap(&desc->async_tx);
 		/* call the callback (must not sleep or submit new
 		 * operations to this channel)
 		 */
-		if (desc->async_tx.callback)
-			desc->async_tx.callback(
-				desc->async_tx.callback_param);
-
-		dma_descriptor_unmap(&desc->async_tx);
+		dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
 	}
 
 	/* run dependent operations */
@@ -3891,7 +3888,7 @@ static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
 	np = ofdev->dev.of_node;
 	if (adev->id != PPC440SPE_XOR_ID) {
 		adev->err_irq = irq_of_parse_and_map(np, 1);
-		if (adev->err_irq == NO_IRQ) {
+		if (!adev->err_irq) {
 			dev_warn(adev->dev, "no err irq resource?\n");
 			*initcode = PPC_ADMA_INIT_IRQ2;
 			adev->err_irq = -ENXIO;
@@ -3902,7 +3899,7 @@ static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
 	}
 
 	adev->irq = irq_of_parse_and_map(np, 0);
-	if (adev->irq == NO_IRQ) {
+	if (!adev->irq) {
 		dev_err(adev->dev, "no irq resource\n");
 		*initcode = PPC_ADMA_INIT_IRQ1;
 		ret = -ENXIO;
diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c
index dc7850a422b8..3f56f9ca4482 100644
--- a/drivers/dma/pxa_dma.c
+++ b/drivers/dma/pxa_dma.c
@@ -638,7 +638,7 @@ static bool pxad_try_hotchain(struct virt_dma_chan *vc,
 		vd_last_issued = list_entry(vc->desc_issued.prev,
 					    struct virt_dma_desc, node);
 		pxad_desc_chain(vd_last_issued, vd);
-		if (is_chan_running(chan) || is_desc_completed(vd_last_issued))
+		if (is_chan_running(chan) || is_desc_completed(vd))
 			return true;
 	}
 
@@ -671,6 +671,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
 	struct virt_dma_desc *vd, *tmp;
 	unsigned int dcsr;
 	unsigned long flags;
+	bool vd_completed;
 	dma_cookie_t last_started = 0;
 
 	BUG_ON(!chan);
@@ -681,15 +682,17 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
 	list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {
+		vd_completed = is_desc_completed(vd);
 		dev_dbg(&chan->vc.chan.dev->device,
-			"%s(): checking txd %p[%x]: completed=%d\n",
-			__func__, vd, vd->tx.cookie, is_desc_completed(vd));
+			"%s(): checking txd %p[%x]: completed=%d dcsr=0x%x\n",
+			__func__, vd, vd->tx.cookie, vd_completed,
+			dcsr);
 		last_started = vd->tx.cookie;
 		if (to_pxad_sw_desc(vd)->cyclic) {
 			vchan_cyclic_callback(vd);
 			break;
 		}
-		if (is_desc_completed(vd)) {
+		if (vd_completed) {
 			list_del(&vd->node);
 			vchan_cookie_complete(vd);
 		} else {
diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c
index b2374cd91e45..e244e10a94b5 100644
--- a/drivers/dma/qcom/hidma.c
+++ b/drivers/dma/qcom/hidma.c
@@ -111,6 +111,7 @@ static void hidma_process_completed(struct hidma_chan *mchan)
 	struct dma_async_tx_descriptor *desc;
 	dma_cookie_t last_cookie;
 	struct hidma_desc *mdesc;
+	struct hidma_desc *next;
 	unsigned long irqflags;
 	struct list_head list;
 
@@ -122,28 +123,36 @@ static void hidma_process_completed(struct hidma_chan *mchan)
 	spin_unlock_irqrestore(&mchan->lock, irqflags);
 
 	/* Execute callbacks and run dependencies */
-	list_for_each_entry(mdesc, &list, node) {
+	list_for_each_entry_safe(mdesc, next, &list, node) {
 		enum dma_status llstat;
+		struct dmaengine_desc_callback cb;
+		struct dmaengine_result result;
 
 		desc = &mdesc->desc;
+		last_cookie = desc->cookie;
 
 		spin_lock_irqsave(&mchan->lock, irqflags);
 		dma_cookie_complete(desc);
 		spin_unlock_irqrestore(&mchan->lock, irqflags);
 
 		llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch);
-		if (desc->callback && (llstat == DMA_COMPLETE))
-			desc->callback(desc->callback_param);
+		dmaengine_desc_get_callback(desc, &cb);
 
-		last_cookie = desc->cookie;
 		dma_run_dependencies(desc);
-	}
 
-	/* Free descriptors */
-	spin_lock_irqsave(&mchan->lock, irqflags);
-	list_splice_tail_init(&list, &mchan->free);
-	spin_unlock_irqrestore(&mchan->lock, irqflags);
+		spin_lock_irqsave(&mchan->lock, irqflags);
+		list_move(&mdesc->node, &mchan->free);
+
+		if (llstat == DMA_COMPLETE) {
+			mchan->last_success = last_cookie;
+			result.result = DMA_TRANS_NOERROR;
+		} else
+			result.result = DMA_TRANS_ABORTED;
+
+		spin_unlock_irqrestore(&mchan->lock, irqflags);
 
+		dmaengine_desc_callback_invoke(&cb, &result);
+	}
 }
 
 /*
@@ -238,6 +247,19 @@ static void hidma_issue_pending(struct dma_chan *dmach)
 		hidma_ll_start(dmadev->lldev);
 }
 
+static inline bool hidma_txn_is_success(dma_cookie_t cookie,
+		dma_cookie_t last_success, dma_cookie_t last_used)
+{
+	if (last_success <= last_used) {
+		if ((cookie <= last_success) || (cookie > last_used))
+			return true;
+	} else {
+		if ((cookie <= last_success) && (cookie > last_used))
+			return true;
+	}
+	return false;
+}
+
 static enum dma_status hidma_tx_status(struct dma_chan *dmach,
 				       dma_cookie_t cookie,
 				       struct dma_tx_state *txstate)
@@ -246,8 +268,13 @@ static enum dma_status hidma_tx_status(struct dma_chan *dmach,
 	enum dma_status ret;
 
 	ret = dma_cookie_status(dmach, cookie, txstate);
-	if (ret == DMA_COMPLETE)
-		return ret;
+	if (ret == DMA_COMPLETE) {
+		bool is_success;
+
+		is_success = hidma_txn_is_success(cookie, mchan->last_success,
+						  dmach->cookie);
+		return is_success ? ret : DMA_ERROR;
+	}
 
 	if (mchan->paused && (ret == DMA_IN_PROGRESS)) {
 		unsigned long flags;
@@ -398,6 +425,7 @@ static int hidma_terminate_channel(struct dma_chan *chan)
 	hidma_process_completed(mchan);
 
 	spin_lock_irqsave(&mchan->lock, irqflags);
+	mchan->last_success = 0;
 	list_splice_init(&mchan->active, &list);
 	list_splice_init(&mchan->prepared, &list);
 	list_splice_init(&mchan->completed, &list);
@@ -413,14 +441,9 @@ static int hidma_terminate_channel(struct dma_chan *chan)
 	/* return all user requests */
 	list_for_each_entry_safe(mdesc, tmp, &list, node) {
 		struct dma_async_tx_descriptor *txd = &mdesc->desc;
-		dma_async_tx_callback callback = mdesc->desc.callback;
-		void *param = mdesc->desc.callback_param;
 
 		dma_descriptor_unmap(txd);
-
-		if (callback)
-			callback(param);
-
+		dmaengine_desc_get_callback_invoke(txd, NULL);
 		dma_run_dependencies(txd);
 
 		/* move myself to free_list */
diff --git a/drivers/dma/qcom/hidma.h b/drivers/dma/qcom/hidma.h
index db413a5efc4e..e52e20716303 100644
--- a/drivers/dma/qcom/hidma.h
+++ b/drivers/dma/qcom/hidma.h
@@ -72,7 +72,6 @@ struct hidma_lldev {
 
 	u32 tre_write_offset;           /* TRE write location              */
 	struct tasklet_struct task;	/* task delivering notifications   */
-	struct tasklet_struct rst_task;	/* task to reset HW                */
 	DECLARE_KFIFO_PTR(handoff_fifo,
 		struct hidma_tre *);    /* pending TREs FIFO               */
 };
@@ -89,6 +88,7 @@ struct hidma_chan {
 	bool				allocated;
 	char				dbg_name[16];
 	u32				dma_sig;
+	dma_cookie_t			last_success;
 
 	/*
 	 * active descriptor on this channel
diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c
index ad20dfb64c71..3224f24c577b 100644
--- a/drivers/dma/qcom/hidma_ll.c
+++ b/drivers/dma/qcom/hidma_ll.c
@@ -381,27 +381,6 @@ static int hidma_ll_reset(struct hidma_lldev *lldev)
 }
 
 /*
- * Abort all transactions and perform a reset.
- */
-static void hidma_ll_abort(unsigned long arg)
-{
-	struct hidma_lldev *lldev = (struct hidma_lldev *)arg;
-	u8 err_code = HIDMA_EVRE_STATUS_ERROR;
-	u8 err_info = 0xFF;
-	int rc;
-
-	hidma_cleanup_pending_tre(lldev, err_info, err_code);
-
-	/* reset the channel for recovery */
-	rc = hidma_ll_setup(lldev);
-	if (rc) {
-		dev_err(lldev->dev, "channel reinitialize failed after error\n");
-		return;
-	}
-	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
-}
-
-/*
  * The interrupt handler for HIDMA will try to consume as many pending
  * EVRE from the event queue as possible. Each EVRE has an associated
  * TRE that holds the user interface parameters. EVRE reports the
@@ -454,13 +433,18 @@ irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
 
 	while (cause) {
 		if (cause & HIDMA_ERR_INT_MASK) {
-			dev_err(lldev->dev, "error 0x%x, resetting...\n",
+			dev_err(lldev->dev, "error 0x%x, disabling...\n",
 					cause);
 
 			/* Clear out pending interrupts */
 			writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
 
-			tasklet_schedule(&lldev->rst_task);
+			/* No further submissions. */
+			hidma_ll_disable(lldev);
+
+			/* Driver completes the txn and intimates the client.*/
+			hidma_cleanup_pending_tre(lldev, 0xFF,
+						  HIDMA_EVRE_STATUS_ERROR);
 			goto out;
 		}
 
@@ -808,7 +792,6 @@ struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres,
 		return NULL;
 
 	spin_lock_init(&lldev->lock);
-	tasklet_init(&lldev->rst_task, hidma_ll_abort, (unsigned long)lldev);
 	tasklet_init(&lldev->task, hidma_ll_tre_complete, (unsigned long)lldev);
 	lldev->initialized = 1;
 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
@@ -831,7 +814,6 @@ int hidma_ll_uninit(struct hidma_lldev *lldev)
 
 	required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres;
 	tasklet_kill(&lldev->task);
-	tasklet_kill(&lldev->rst_task);
 	memset(lldev->trepool, 0, required_bytes);
 	lldev->trepool = NULL;
 	lldev->pending_tre_count = 0;
diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c
index ce67075589f5..3c579abbabb7 100644
--- a/drivers/dma/s3c24xx-dma.c
+++ b/drivers/dma/s3c24xx-dma.c
@@ -823,11 +823,11 @@ static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy(
 	struct s3c24xx_sg *dsg;
 	int src_mod, dest_mod;
 
-	dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %d bytes from %s\n",
+	dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %zu bytes from %s\n",
 			len, s3cchan->name);
 
 	if ((len & S3C24XX_DCON_TC_MASK) != len) {
-		dev_err(&s3cdma->pdev->dev, "memcpy size %d to large\n", len);
+		dev_err(&s3cdma->pdev->dev, "memcpy size %zu to large\n", len);
 		return NULL;
 	}
 
@@ -1301,6 +1301,9 @@ static int s3c24xx_dma_probe(struct platform_device *pdev)
 	s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic;
 	s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config;
 	s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all;
+	s3cdma->slave.filter.map = pdata->slave_map;
+	s3cdma->slave.filter.mapcnt = pdata->slavecnt;
+	s3cdma->slave.filter.fn = s3c24xx_dma_filter;
 
 	/* Register as many memcpy channels as there are physical channels */
 	ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy,
@@ -1418,7 +1421,7 @@ bool s3c24xx_dma_filter(struct dma_chan *chan, void *param)
 
 	s3cchan = to_s3c24xx_dma_chan(chan);
 
-	return s3cchan->id == (int)param;
+	return s3cchan->id == (uintptr_t)param;
 }
 EXPORT_SYMBOL(s3c24xx_dma_filter);
 
diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c
index 43db255050d2..1adeb3265085 100644
--- a/drivers/dma/sa11x0-dma.c
+++ b/drivers/dma/sa11x0-dma.c
@@ -463,7 +463,7 @@ static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 			dma_addr_t addr = sa11x0_dma_pos(p);
 			unsigned i;
 
-			dev_vdbg(d->slave.dev, "tx_status: addr:%x\n", addr);
+			dev_vdbg(d->slave.dev, "tx_status: addr:%pad\n", &addr);
 
 			for (i = 0; i < txd->sglen; i++) {
 				dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
@@ -491,7 +491,7 @@ static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 	}
 	spin_unlock_irqrestore(&c->vc.lock, flags);
 
-	dev_vdbg(d->slave.dev, "tx_status: bytes 0x%zx\n", state->residue);
+	dev_vdbg(d->slave.dev, "tx_status: bytes 0x%x\n", state->residue);
 
 	return ret;
 }
@@ -551,8 +551,8 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 		if (len > DMA_MAX_SIZE)
 			j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
 		if (addr & DMA_ALIGN) {
-			dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %08x\n",
-				&c->vc, addr);
+			dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %pad\n",
+				&c->vc, &addr);
 			return NULL;
 		}
 	}
@@ -599,7 +599,7 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 	txd->size = size;
 	txd->sglen = j;
 
-	dev_dbg(chan->device->dev, "vchan %p: txd %p: size %u nr %u\n",
+	dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n",
 		&c->vc, &txd->vd, txd->size, txd->sglen);
 
 	return vchan_tx_prep(&c->vc, &txd->vd, flags);
@@ -693,8 +693,8 @@ static int sa11x0_dma_device_config(struct dma_chan *chan,
 	if (maxburst == 8)
 		ddar |= DDAR_BS;
 
-	dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %x width %u burst %u\n",
-		&c->vc, addr, width, maxburst);
+	dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %pad width %u burst %u\n",
+		&c->vc, &addr, width, maxburst);
 
 	c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
 
diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c
index 0dd953884d1d..2e441d0ccd79 100644
--- a/drivers/dma/sh/rcar-dmac.c
+++ b/drivers/dma/sh/rcar-dmac.c
@@ -118,14 +118,34 @@ struct rcar_dmac_desc_page {
 	sizeof(struct rcar_dmac_xfer_chunk))
 
 /*
+ * struct rcar_dmac_chan_slave - Slave configuration
+ * @slave_addr: slave memory address
+ * @xfer_size: size (in bytes) of hardware transfers
+ */
+struct rcar_dmac_chan_slave {
+	phys_addr_t slave_addr;
+	unsigned int xfer_size;
+};
+
+/*
+ * struct rcar_dmac_chan_map - Map of slave device phys to dma address
+ * @addr: slave dma address
+ * @dir: direction of mapping
+ * @slave: slave configuration that is mapped
+ */
+struct rcar_dmac_chan_map {
+	dma_addr_t addr;
+	enum dma_data_direction dir;
+	struct rcar_dmac_chan_slave slave;
+};
+
+/*
  * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel
  * @chan: base DMA channel object
  * @iomem: channel I/O memory base
  * @index: index of this channel in the controller
- * @src_xfer_size: size (in bytes) of hardware transfers on the source side
- * @dst_xfer_size: size (in bytes) of hardware transfers on the destination side
- * @src_slave_addr: slave source memory address
- * @dst_slave_addr: slave destination memory address
+ * @src: slave memory address and size on the source side
+ * @dst: slave memory address and size on the destination side
  * @mid_rid: hardware MID/RID for the DMA client using this channel
  * @lock: protects the channel CHCR register and the desc members
  * @desc.free: list of free descriptors
@@ -142,10 +162,9 @@ struct rcar_dmac_chan {
 	void __iomem *iomem;
 	unsigned int index;
 
-	unsigned int src_xfer_size;
-	unsigned int dst_xfer_size;
-	dma_addr_t src_slave_addr;
-	dma_addr_t dst_slave_addr;
+	struct rcar_dmac_chan_slave src;
+	struct rcar_dmac_chan_slave dst;
+	struct rcar_dmac_chan_map map;
 	int mid_rid;
 
 	spinlock_t lock;
@@ -793,13 +812,13 @@ static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan,
 	case DMA_DEV_TO_MEM:
 		chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_FIXED
 		     | RCAR_DMACHCR_RS_DMARS;
-		xfer_size = chan->src_xfer_size;
+		xfer_size = chan->src.xfer_size;
 		break;
 
 	case DMA_MEM_TO_DEV:
 		chcr = RCAR_DMACHCR_DM_FIXED | RCAR_DMACHCR_SM_INC
 		     | RCAR_DMACHCR_RS_DMARS;
-		xfer_size = chan->dst_xfer_size;
+		xfer_size = chan->dst.xfer_size;
 		break;
 
 	case DMA_MEM_TO_MEM:
@@ -1023,13 +1042,65 @@ rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 				      DMA_MEM_TO_MEM, flags, false);
 }
 
+static int rcar_dmac_map_slave_addr(struct dma_chan *chan,
+				    enum dma_transfer_direction dir)
+{
+	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
+	struct rcar_dmac_chan_map *map = &rchan->map;
+	phys_addr_t dev_addr;
+	size_t dev_size;
+	enum dma_data_direction dev_dir;
+
+	if (dir == DMA_DEV_TO_MEM) {
+		dev_addr = rchan->src.slave_addr;
+		dev_size = rchan->src.xfer_size;
+		dev_dir = DMA_TO_DEVICE;
+	} else {
+		dev_addr = rchan->dst.slave_addr;
+		dev_size = rchan->dst.xfer_size;
+		dev_dir = DMA_FROM_DEVICE;
+	}
+
+	/* Reuse current map if possible. */
+	if (dev_addr == map->slave.slave_addr &&
+	    dev_size == map->slave.xfer_size &&
+	    dev_dir == map->dir)
+		return 0;
+
+	/* Remove old mapping if present. */
+	if (map->slave.xfer_size)
+		dma_unmap_resource(chan->device->dev, map->addr,
+				   map->slave.xfer_size, map->dir, 0);
+	map->slave.xfer_size = 0;
+
+	/* Create new slave address map. */
+	map->addr = dma_map_resource(chan->device->dev, dev_addr, dev_size,
+				     dev_dir, 0);
+
+	if (dma_mapping_error(chan->device->dev, map->addr)) {
+		dev_err(chan->device->dev,
+			"chan%u: failed to map %zx@%pap", rchan->index,
+			dev_size, &dev_addr);
+		return -EIO;
+	}
+
+	dev_dbg(chan->device->dev, "chan%u: map %zx@%pap to %pad dir: %s\n",
+		rchan->index, dev_size, &dev_addr, &map->addr,
+		dev_dir == DMA_TO_DEVICE ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE");
+
+	map->slave.slave_addr = dev_addr;
+	map->slave.xfer_size = dev_size;
+	map->dir = dev_dir;
+
+	return 0;
+}
+
 static struct dma_async_tx_descriptor *
 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 			unsigned int sg_len, enum dma_transfer_direction dir,
 			unsigned long flags, void *context)
 {
 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
-	dma_addr_t dev_addr;
 
 	/* Someone calling slave DMA on a generic channel? */
 	if (rchan->mid_rid < 0 || !sg_len) {
@@ -1039,9 +1110,10 @@ rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		return NULL;
 	}
 
-	dev_addr = dir == DMA_DEV_TO_MEM
-		 ? rchan->src_slave_addr : rchan->dst_slave_addr;
-	return rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, dev_addr,
+	if (rcar_dmac_map_slave_addr(chan, dir))
+		return NULL;
+
+	return rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr,
 				      dir, flags, false);
 }
 
@@ -1055,7 +1127,6 @@ rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
 	struct dma_async_tx_descriptor *desc;
 	struct scatterlist *sgl;
-	dma_addr_t dev_addr;
 	unsigned int sg_len;
 	unsigned int i;
 
@@ -1067,6 +1138,9 @@ rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 		return NULL;
 	}
 
+	if (rcar_dmac_map_slave_addr(chan, dir))
+		return NULL;
+
 	sg_len = buf_len / period_len;
 	if (sg_len > RCAR_DMAC_MAX_SG_LEN) {
 		dev_err(chan->device->dev,
@@ -1094,9 +1168,7 @@ rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 		sg_dma_len(&sgl[i]) = period_len;
 	}
 
-	dev_addr = dir == DMA_DEV_TO_MEM
-		 ? rchan->src_slave_addr : rchan->dst_slave_addr;
-	desc = rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, dev_addr,
+	desc = rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr,
 				      dir, flags, true);
 
 	kfree(sgl);
@@ -1112,10 +1184,10 @@ static int rcar_dmac_device_config(struct dma_chan *chan,
 	 * We could lock this, but you shouldn't be configuring the
 	 * channel, while using it...
 	 */
-	rchan->src_slave_addr = cfg->src_addr;
-	rchan->dst_slave_addr = cfg->dst_addr;
-	rchan->src_xfer_size = cfg->src_addr_width;
-	rchan->dst_xfer_size = cfg->dst_addr_width;
+	rchan->src.slave_addr = cfg->src_addr;
+	rchan->dst.slave_addr = cfg->dst_addr;
+	rchan->src.xfer_size = cfg->src_addr_width;
+	rchan->dst.xfer_size = cfg->dst_addr_width;
 
 	return 0;
 }
@@ -1389,21 +1461,18 @@ static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev)
 {
 	struct rcar_dmac_chan *chan = dev;
 	struct rcar_dmac_desc *desc;
+	struct dmaengine_desc_callback cb;
 
 	spin_lock_irq(&chan->lock);
 
 	/* For cyclic transfers notify the user after every chunk. */
 	if (chan->desc.running && chan->desc.running->cyclic) {
-		dma_async_tx_callback callback;
-		void *callback_param;
-
 		desc = chan->desc.running;
-		callback = desc->async_tx.callback;
-		callback_param = desc->async_tx.callback_param;
+		dmaengine_desc_get_callback(&desc->async_tx, &cb);
 
-		if (callback) {
+		if (dmaengine_desc_callback_valid(&cb)) {
 			spin_unlock_irq(&chan->lock);
-			callback(callback_param);
+			dmaengine_desc_callback_invoke(&cb, NULL);
 			spin_lock_irq(&chan->lock);
 		}
 	}
@@ -1418,14 +1487,15 @@ static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev)
 		dma_cookie_complete(&desc->async_tx);
 		list_del(&desc->node);
 
-		if (desc->async_tx.callback) {
+		dmaengine_desc_get_callback(&desc->async_tx, &cb);
+		if (dmaengine_desc_callback_valid(&cb)) {
 			spin_unlock_irq(&chan->lock);
 			/*
 			 * We own the only reference to this descriptor, we can
 			 * safely dereference it without holding the channel
 			 * lock.
 			 */
-			desc->async_tx.callback(desc->async_tx.callback_param);
+			dmaengine_desc_callback_invoke(&cb, NULL);
 			spin_lock_irq(&chan->lock);
 		}
 
diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c
index 10fcabad80f3..12fa48e380cf 100644
--- a/drivers/dma/sh/shdma-base.c
+++ b/drivers/dma/sh/shdma-base.c
@@ -330,10 +330,11 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all)
 	bool head_acked = false;
 	dma_cookie_t cookie = 0;
 	dma_async_tx_callback callback = NULL;
-	void *param = NULL;
+	struct dmaengine_desc_callback cb;
 	unsigned long flags;
 	LIST_HEAD(cyclic_list);
 
+	memset(&cb, 0, sizeof(cb));
 	spin_lock_irqsave(&schan->chan_lock, flags);
 	list_for_each_entry_safe(desc, _desc, &schan->ld_queue, node) {
 		struct dma_async_tx_descriptor *tx = &desc->async_tx;
@@ -367,8 +368,8 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all)
 		/* Call callback on the last chunk */
 		if (desc->mark == DESC_COMPLETED && tx->callback) {
 			desc->mark = DESC_WAITING;
+			dmaengine_desc_get_callback(tx, &cb);
 			callback = tx->callback;
-			param = tx->callback_param;
 			dev_dbg(schan->dev, "descriptor #%d@%p on %d callback\n",
 				tx->cookie, tx, schan->id);
 			BUG_ON(desc->chunks != 1);
@@ -430,8 +431,7 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all)
 
 	spin_unlock_irqrestore(&schan->chan_lock, flags);
 
-	if (callback)
-		callback(param);
+	dmaengine_desc_callback_invoke(&cb, NULL);
 
 	return callback;
 }
@@ -885,9 +885,9 @@ bool shdma_reset(struct shdma_dev *sdev)
 		/* Complete all  */
 		list_for_each_entry(sdesc, &dl, node) {
 			struct dma_async_tx_descriptor *tx = &sdesc->async_tx;
+
 			sdesc->mark = DESC_IDLE;
-			if (tx->callback)
-				tx->callback(tx->callback_param);
+			dmaengine_desc_get_callback_invoke(tx, NULL);
 		}
 
 		spin_lock(&schan->chan_lock);
diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c
index 749f1bd5d65d..06ecdc38cee0 100644
--- a/drivers/dma/sh/usb-dmac.c
+++ b/drivers/dma/sh/usb-dmac.c
@@ -600,27 +600,30 @@ static irqreturn_t usb_dmac_isr_channel(int irq, void *dev)
 {
 	struct usb_dmac_chan *chan = dev;
 	irqreturn_t ret = IRQ_NONE;
-	u32 mask = USB_DMACHCR_TE;
-	u32 check_bits = USB_DMACHCR_TE | USB_DMACHCR_SP;
+	u32 mask = 0;
 	u32 chcr;
+	bool xfer_end = false;
 
 	spin_lock(&chan->vc.lock);
 
 	chcr = usb_dmac_chan_read(chan, USB_DMACHCR);
-	if (chcr & check_bits)
-		mask |= USB_DMACHCR_DE | check_bits;
+	if (chcr & (USB_DMACHCR_TE | USB_DMACHCR_SP)) {
+		mask |= USB_DMACHCR_DE | USB_DMACHCR_TE | USB_DMACHCR_SP;
+		if (chcr & USB_DMACHCR_DE)
+			xfer_end = true;
+		ret |= IRQ_HANDLED;
+	}
 	if (chcr & USB_DMACHCR_NULL) {
 		/* An interruption of TE will happen after we set FTE */
 		mask |= USB_DMACHCR_NULL;
 		chcr |= USB_DMACHCR_FTE;
 		ret |= IRQ_HANDLED;
 	}
-	usb_dmac_chan_write(chan, USB_DMACHCR, chcr & ~mask);
+	if (mask)
+		usb_dmac_chan_write(chan, USB_DMACHCR, chcr & ~mask);
 
-	if (chcr & check_bits) {
+	if (xfer_end)
 		usb_dmac_isr_transfer_end(chan);
-		ret |= IRQ_HANDLED;
-	}
 
 	spin_unlock(&chan->vc.lock);
 
diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c
index d8bc3f2a71db..8f62edad51be 100644
--- a/drivers/dma/sirf-dma.c
+++ b/drivers/dma/sirf-dma.c
@@ -360,9 +360,7 @@ static void sirfsoc_dma_process_completed(struct sirfsoc_dma *sdma)
 			list_for_each_entry(sdesc, &list, node) {
 				desc = &sdesc->desc;
 
-				if (desc->callback)
-					desc->callback(desc->callback_param);
-
+				dmaengine_desc_get_callback_invoke(desc, NULL);
 				last_cookie = desc->cookie;
 				dma_run_dependencies(desc);
 			}
@@ -388,8 +386,7 @@ static void sirfsoc_dma_process_completed(struct sirfsoc_dma *sdma)
 
 			desc = &sdesc->desc;
 			while (happened_cyclic != schan->completed_cyclic) {
-				if (desc->callback)
-					desc->callback(desc->callback_param);
+				dmaengine_desc_get_callback_invoke(desc, NULL);
 				schan->completed_cyclic++;
 			}
 		}
@@ -869,7 +866,7 @@ static int sirfsoc_dma_probe(struct platform_device *op)
 	}
 
 	sdma->irq = irq_of_parse_and_map(dn, 0);
-	if (sdma->irq == NO_IRQ) {
+	if (!sdma->irq) {
 		dev_err(dev, "Error mapping IRQ!\n");
 		return -EINVAL;
 	}
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index 8b18e44a02d5..8684d11b29bb 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -874,7 +874,7 @@ static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
 	}
 
 	if (curr_lcla < 0)
-		goto out;
+		goto set_current;
 
 	for (; lli_current < lli_len; lli_current++) {
 		unsigned int lcla_offset = chan->phy_chan->num * 1024 +
@@ -925,8 +925,7 @@ static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
 			break;
 		}
 	}
-
-out:
+ set_current:
 	desc->lli_current = lli_current;
 }
 
@@ -941,15 +940,7 @@ static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
 
 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
 {
-	struct d40_desc *d;
-
-	if (list_empty(&d40c->active))
-		return NULL;
-
-	d = list_first_entry(&d40c->active,
-			     struct d40_desc,
-			     node);
-	return d;
+	return list_first_entry_or_null(&d40c->active, struct d40_desc, node);
 }
 
 /* remove desc from current queue and add it to the pending_queue */
@@ -962,36 +953,18 @@ static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
 
 static struct d40_desc *d40_first_pending(struct d40_chan *d40c)
 {
-	struct d40_desc *d;
-
-	if (list_empty(&d40c->pending_queue))
-		return NULL;
-
-	d = list_first_entry(&d40c->pending_queue,
-			     struct d40_desc,
-			     node);
-	return d;
+	return list_first_entry_or_null(&d40c->pending_queue, struct d40_desc,
+					node);
 }
 
 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
 {
-	struct d40_desc *d;
-
-	if (list_empty(&d40c->queue))
-		return NULL;
-
-	d = list_first_entry(&d40c->queue,
-			     struct d40_desc,
-			     node);
-	return d;
+	return list_first_entry_or_null(&d40c->queue, struct d40_desc, node);
 }
 
 static struct d40_desc *d40_first_done(struct d40_chan *d40c)
 {
-	if (list_empty(&d40c->done))
-		return NULL;
-
-	return list_first_entry(&d40c->done, struct d40_desc, node);
+	return list_first_entry_or_null(&d40c->done, struct d40_desc, node);
 }
 
 static int d40_psize_2_burst_size(bool is_log, int psize)
@@ -1083,7 +1056,7 @@ static int __d40_execute_command_phy(struct d40_chan *d40c,
 			D40_CHAN_POS(d40c->phy_chan->num);
 
 		if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
-			goto done;
+			goto unlock;
 	}
 
 	wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
@@ -1119,7 +1092,7 @@ static int __d40_execute_command_phy(struct d40_chan *d40c,
 		}
 
 	}
-done:
+ unlock:
 	spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
 	return ret;
 }
@@ -1596,8 +1569,7 @@ static void dma_tasklet(unsigned long data)
 	struct d40_desc *d40d;
 	unsigned long flags;
 	bool callback_active;
-	dma_async_tx_callback callback;
-	void *callback_param;
+	struct dmaengine_desc_callback cb;
 
 	spin_lock_irqsave(&d40c->lock, flags);
 
@@ -1607,7 +1579,7 @@ static void dma_tasklet(unsigned long data)
 		/* Check if we have reached here for cyclic job */
 		d40d = d40_first_active_get(d40c);
 		if (d40d == NULL || !d40d->cyclic)
-			goto err;
+			goto check_pending_tx;
 	}
 
 	if (!d40d->cyclic)
@@ -1624,8 +1596,7 @@ static void dma_tasklet(unsigned long data)
 
 	/* Callback to client */
 	callback_active = !!(d40d->txd.flags & DMA_PREP_INTERRUPT);
-	callback = d40d->txd.callback;
-	callback_param = d40d->txd.callback_param;
+	dmaengine_desc_get_callback(&d40d->txd, &cb);
 
 	if (!d40d->cyclic) {
 		if (async_tx_test_ack(&d40d->txd)) {
@@ -1646,12 +1617,11 @@ static void dma_tasklet(unsigned long data)
 
 	spin_unlock_irqrestore(&d40c->lock, flags);
 
-	if (callback_active && callback)
-		callback(callback_param);
+	if (callback_active)
+		dmaengine_desc_callback_invoke(&cb, NULL);
 
 	return;
-
-err:
+ check_pending_tx:
 	/* Rescue manouver if receiving double interrupts */
 	if (d40c->pending_tx > 0)
 		d40c->pending_tx--;
@@ -1780,42 +1750,40 @@ static bool d40_alloc_mask_set(struct d40_phy_res *phy,
 		    phy->allocated_dst == D40_ALLOC_FREE) {
 			phy->allocated_dst = D40_ALLOC_PHY;
 			phy->allocated_src = D40_ALLOC_PHY;
-			goto found;
+			goto found_unlock;
 		} else
-			goto not_found;
+			goto not_found_unlock;
 	}
 
 	/* Logical channel */
 	if (is_src) {
 		if (phy->allocated_src == D40_ALLOC_PHY)
-			goto not_found;
+			goto not_found_unlock;
 
 		if (phy->allocated_src == D40_ALLOC_FREE)
 			phy->allocated_src = D40_ALLOC_LOG_FREE;
 
 		if (!(phy->allocated_src & BIT(log_event_line))) {
 			phy->allocated_src |= BIT(log_event_line);
-			goto found;
+			goto found_unlock;
 		} else
-			goto not_found;
+			goto not_found_unlock;
 	} else {
 		if (phy->allocated_dst == D40_ALLOC_PHY)
-			goto not_found;
+			goto not_found_unlock;
 
 		if (phy->allocated_dst == D40_ALLOC_FREE)
 			phy->allocated_dst = D40_ALLOC_LOG_FREE;
 
 		if (!(phy->allocated_dst & BIT(log_event_line))) {
 			phy->allocated_dst |= BIT(log_event_line);
-			goto found;
-		} else
-			goto not_found;
+			goto found_unlock;
+		}
 	}
-
-not_found:
+ not_found_unlock:
 	spin_unlock_irqrestore(&phy->lock, flags);
 	return false;
-found:
+ found_unlock:
 	spin_unlock_irqrestore(&phy->lock, flags);
 	return true;
 }
@@ -1831,7 +1799,7 @@ static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
 		phy->allocated_dst = D40_ALLOC_FREE;
 		phy->allocated_src = D40_ALLOC_FREE;
 		is_free = true;
-		goto out;
+		goto unlock;
 	}
 
 	/* Logical channel */
@@ -1847,8 +1815,7 @@ static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
 
 	is_free = ((phy->allocated_src | phy->allocated_dst) ==
 		   D40_ALLOC_FREE);
-
-out:
+ unlock:
 	spin_unlock_irqrestore(&phy->lock, flags);
 
 	return is_free;
@@ -2047,7 +2014,7 @@ static int d40_free_dma(struct d40_chan *d40c)
 	res = d40_channel_execute_command(d40c, D40_DMA_STOP);
 	if (res) {
 		chan_err(d40c, "stop failed\n");
-		goto out;
+		goto mark_last_busy;
 	}
 
 	d40_alloc_mask_free(phy, is_src, chan_is_logical(d40c) ? event : 0);
@@ -2065,8 +2032,7 @@ static int d40_free_dma(struct d40_chan *d40c)
 	d40c->busy = false;
 	d40c->phy_chan = NULL;
 	d40c->configured = false;
-out:
-
+ mark_last_busy:
 	pm_runtime_mark_last_busy(d40c->base->dev);
 	pm_runtime_put_autosuspend(d40c->base->dev);
 	return res;
@@ -2094,8 +2060,7 @@ static bool d40_is_paused(struct d40_chan *d40c)
 			D40_CHAN_POS(d40c->phy_chan->num);
 		if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
 			is_paused = true;
-
-		goto _exit;
+		goto unlock;
 	}
 
 	if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV ||
@@ -2105,7 +2070,7 @@ static bool d40_is_paused(struct d40_chan *d40c)
 		status = readl(chanbase + D40_CHAN_REG_SSLNK);
 	} else {
 		chan_err(d40c, "Unknown direction\n");
-		goto _exit;
+		goto unlock;
 	}
 
 	status = (status & D40_EVENTLINE_MASK(event)) >>
@@ -2113,7 +2078,7 @@ static bool d40_is_paused(struct d40_chan *d40c)
 
 	if (status != D40_DMA_RUN)
 		is_paused = true;
-_exit:
+ unlock:
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return is_paused;
 
@@ -2198,7 +2163,7 @@ static struct d40_desc *
 d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
 	      unsigned int sg_len, unsigned long dma_flags)
 {
-	struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
+	struct stedma40_chan_cfg *cfg;
 	struct d40_desc *desc;
 	int ret;
 
@@ -2206,17 +2171,18 @@ d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
 	if (!desc)
 		return NULL;
 
+	cfg = &chan->dma_cfg;
 	desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
 					cfg->dst_info.data_width);
 	if (desc->lli_len < 0) {
 		chan_err(chan, "Unaligned size\n");
-		goto err;
+		goto free_desc;
 	}
 
 	ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
 	if (ret < 0) {
 		chan_err(chan, "Could not allocate lli\n");
-		goto err;
+		goto free_desc;
 	}
 
 	desc->lli_current = 0;
@@ -2226,8 +2192,7 @@ d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
 	dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
 
 	return desc;
-
-err:
+ free_desc:
 	d40_desc_free(chan, desc);
 	return NULL;
 }
@@ -2238,8 +2203,8 @@ d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
 	    enum dma_transfer_direction direction, unsigned long dma_flags)
 {
 	struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
-	dma_addr_t src_dev_addr = 0;
-	dma_addr_t dst_dev_addr = 0;
+	dma_addr_t src_dev_addr;
+	dma_addr_t dst_dev_addr;
 	struct d40_desc *desc;
 	unsigned long flags;
 	int ret;
@@ -2253,11 +2218,13 @@ d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
 
 	desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
 	if (desc == NULL)
-		goto err;
+		goto unlock;
 
 	if (sg_next(&sg_src[sg_len - 1]) == sg_src)
 		desc->cyclic = true;
 
+	src_dev_addr = 0;
+	dst_dev_addr = 0;
 	if (direction == DMA_DEV_TO_MEM)
 		src_dev_addr = chan->runtime_addr;
 	else if (direction == DMA_MEM_TO_DEV)
@@ -2273,7 +2240,7 @@ d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
 	if (ret) {
 		chan_err(chan, "Failed to prepare %s sg job: %d\n",
 			 chan_is_logical(chan) ? "log" : "phy", ret);
-		goto err;
+		goto free_desc;
 	}
 
 	/*
@@ -2285,10 +2252,9 @@ d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
 	spin_unlock_irqrestore(&chan->lock, flags);
 
 	return &desc->txd;
-
-err:
-	if (desc)
-		d40_desc_free(chan, desc);
+ free_desc:
+	d40_desc_free(chan, desc);
+ unlock:
 	spin_unlock_irqrestore(&chan->lock, flags);
 	return NULL;
 }
@@ -2426,7 +2392,7 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
 		err = d40_config_memcpy(d40c);
 		if (err) {
 			chan_err(d40c, "Failed to configure memcpy channel\n");
-			goto fail;
+			goto mark_last_busy;
 		}
 	}
 
@@ -2434,7 +2400,7 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
 	if (err) {
 		chan_err(d40c, "Failed to allocate channel\n");
 		d40c->configured = false;
-		goto fail;
+		goto mark_last_busy;
 	}
 
 	pm_runtime_get_sync(d40c->base->dev);
@@ -2468,7 +2434,7 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
 	 */
 	if (is_free_phy)
 		d40_config_write(d40c);
-fail:
+ mark_last_busy:
 	pm_runtime_mark_last_busy(d40c->base->dev);
 	pm_runtime_put_autosuspend(d40c->base->dev);
 	spin_unlock_irqrestore(&d40c->lock, flags);
@@ -2891,7 +2857,7 @@ static int __init d40_dmaengine_init(struct d40_base *base,
 
 	if (err) {
 		d40_err(base->dev, "Failed to register slave channels\n");
-		goto failure1;
+		goto exit;
 	}
 
 	d40_chan_init(base, &base->dma_memcpy, base->log_chans,
@@ -2908,7 +2874,7 @@ static int __init d40_dmaengine_init(struct d40_base *base,
 	if (err) {
 		d40_err(base->dev,
 			"Failed to register memcpy only channels\n");
-		goto failure2;
+		goto unregister_slave;
 	}
 
 	d40_chan_init(base, &base->dma_both, base->phy_chans,
@@ -2926,14 +2892,14 @@ static int __init d40_dmaengine_init(struct d40_base *base,
 	if (err) {
 		d40_err(base->dev,
 			"Failed to register logical and physical capable channels\n");
-		goto failure3;
+		goto unregister_memcpy;
 	}
 	return 0;
-failure3:
+ unregister_memcpy:
 	dma_async_device_unregister(&base->dma_memcpy);
-failure2:
+ unregister_slave:
 	dma_async_device_unregister(&base->dma_slave);
-failure1:
+ exit:
 	return err;
 }
 
@@ -3144,11 +3110,11 @@ static int __init d40_phy_res_init(struct d40_base *base)
 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
 {
 	struct stedma40_platform_data *plat_data = dev_get_platdata(&pdev->dev);
-	struct clk *clk = NULL;
-	void __iomem *virtbase = NULL;
-	struct resource *res = NULL;
-	struct d40_base *base = NULL;
-	int num_log_chans = 0;
+	struct clk *clk;
+	void __iomem *virtbase;
+	struct resource *res;
+	struct d40_base *base;
+	int num_log_chans;
 	int num_phy_chans;
 	int num_memcpy_chans;
 	int clk_ret = -EINVAL;
@@ -3160,27 +3126,27 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
 	clk = clk_get(&pdev->dev, NULL);
 	if (IS_ERR(clk)) {
 		d40_err(&pdev->dev, "No matching clock found\n");
-		goto failure;
+		goto check_prepare_enabled;
 	}
 
 	clk_ret = clk_prepare_enable(clk);
 	if (clk_ret) {
 		d40_err(&pdev->dev, "Failed to prepare/enable clock\n");
-		goto failure;
+		goto disable_unprepare;
 	}
 
 	/* Get IO for DMAC base address */
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
 	if (!res)
-		goto failure;
+		goto disable_unprepare;
 
 	if (request_mem_region(res->start, resource_size(res),
 			       D40_NAME " I/O base") == NULL)
-		goto failure;
+		goto release_region;
 
 	virtbase = ioremap(res->start, resource_size(res));
 	if (!virtbase)
-		goto failure;
+		goto release_region;
 
 	/* This is just a regular AMBA PrimeCell ID actually */
 	for (pid = 0, i = 0; i < 4; i++)
@@ -3192,13 +3158,13 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
 
 	if (cid != AMBA_CID) {
 		d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID\n");
-		goto failure;
+		goto unmap_io;
 	}
 	if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) {
 		d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
 			AMBA_MANF_BITS(pid),
 			AMBA_VENDOR_ST);
-		goto failure;
+		goto unmap_io;
 	}
 	/*
 	 * HW revision:
@@ -3212,7 +3178,7 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
 	rev = AMBA_REV_BITS(pid);
 	if (rev < 2) {
 		d40_err(&pdev->dev, "hardware revision: %d is not supported", rev);
-		goto failure;
+		goto unmap_io;
 	}
 
 	/* The number of physical channels on this HW */
@@ -3238,7 +3204,7 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
 		       sizeof(struct d40_chan), GFP_KERNEL);
 
 	if (base == NULL)
-		goto failure;
+		goto unmap_io;
 
 	base->rev = rev;
 	base->clk = clk;
@@ -3283,65 +3249,66 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
 		base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4a);
 	}
 
-	base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
+	base->phy_res = kcalloc(num_phy_chans,
+				sizeof(*base->phy_res),
 				GFP_KERNEL);
 	if (!base->phy_res)
-		goto failure;
+		goto free_base;
 
-	base->lookup_phy_chans = kzalloc(num_phy_chans *
-					 sizeof(struct d40_chan *),
+	base->lookup_phy_chans = kcalloc(num_phy_chans,
+					 sizeof(*base->lookup_phy_chans),
 					 GFP_KERNEL);
 	if (!base->lookup_phy_chans)
-		goto failure;
+		goto free_phy_res;
 
-	base->lookup_log_chans = kzalloc(num_log_chans *
-					 sizeof(struct d40_chan *),
+	base->lookup_log_chans = kcalloc(num_log_chans,
+					 sizeof(*base->lookup_log_chans),
 					 GFP_KERNEL);
 	if (!base->lookup_log_chans)
-		goto failure;
+		goto free_phy_chans;
 
-	base->reg_val_backup_chan = kmalloc(base->num_phy_chans *
-					    sizeof(d40_backup_regs_chan),
-					    GFP_KERNEL);
+	base->reg_val_backup_chan = kmalloc_array(base->num_phy_chans,
+						  sizeof(d40_backup_regs_chan),
+						  GFP_KERNEL);
 	if (!base->reg_val_backup_chan)
-		goto failure;
+		goto free_log_chans;
 
-	base->lcla_pool.alloc_map =
-		kzalloc(num_phy_chans * sizeof(struct d40_desc *)
-			* D40_LCLA_LINK_PER_EVENT_GRP, GFP_KERNEL);
+	base->lcla_pool.alloc_map = kcalloc(num_phy_chans
+					    * D40_LCLA_LINK_PER_EVENT_GRP,
+					    sizeof(*base->lcla_pool.alloc_map),
+					    GFP_KERNEL);
 	if (!base->lcla_pool.alloc_map)
-		goto failure;
+		goto free_backup_chan;
 
 	base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
 					    0, SLAB_HWCACHE_ALIGN,
 					    NULL);
 	if (base->desc_slab == NULL)
-		goto failure;
+		goto free_map;
 
 	return base;
-
-failure:
+ free_map:
+	kfree(base->lcla_pool.alloc_map);
+ free_backup_chan:
+	kfree(base->reg_val_backup_chan);
+ free_log_chans:
+	kfree(base->lookup_log_chans);
+ free_phy_chans:
+	kfree(base->lookup_phy_chans);
+ free_phy_res:
+	kfree(base->phy_res);
+ free_base:
+	kfree(base);
+ unmap_io:
+	iounmap(virtbase);
+ release_region:
+	release_mem_region(res->start, resource_size(res));
+ check_prepare_enabled:
 	if (!clk_ret)
+ disable_unprepare:
 		clk_disable_unprepare(clk);
 	if (!IS_ERR(clk))
 		clk_put(clk);
-	if (virtbase)
-		iounmap(virtbase);
-	if (res)
-		release_mem_region(res->start,
-				   resource_size(res));
-	if (virtbase)
-		iounmap(virtbase);
-
-	if (base) {
-		kfree(base->lcla_pool.alloc_map);
-		kfree(base->reg_val_backup_chan);
-		kfree(base->lookup_log_chans);
-		kfree(base->lookup_phy_chans);
-		kfree(base->phy_res);
-		kfree(base);
-	}
-
 	return NULL;
 }
 
@@ -3404,20 +3371,18 @@ static int __init d40_lcla_allocate(struct d40_base *base)
 	struct d40_lcla_pool *pool = &base->lcla_pool;
 	unsigned long *page_list;
 	int i, j;
-	int ret = 0;
+	int ret;
 
 	/*
 	 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
 	 * To full fill this hardware requirement without wasting 256 kb
 	 * we allocate pages until we get an aligned one.
 	 */
-	page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
-			    GFP_KERNEL);
-
-	if (!page_list) {
-		ret = -ENOMEM;
-		goto failure;
-	}
+	page_list = kmalloc_array(MAX_LCLA_ALLOC_ATTEMPTS,
+				  sizeof(*page_list),
+				  GFP_KERNEL);
+	if (!page_list)
+		return -ENOMEM;
 
 	/* Calculating how many pages that are required */
 	base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
@@ -3433,7 +3398,7 @@ static int __init d40_lcla_allocate(struct d40_base *base)
 
 			for (j = 0; j < i; j++)
 				free_pages(page_list[j], base->lcla_pool.pages);
-			goto failure;
+			goto free_page_list;
 		}
 
 		if ((virt_to_phys((void *)page_list[i]) &
@@ -3460,7 +3425,7 @@ static int __init d40_lcla_allocate(struct d40_base *base)
 							 GFP_KERNEL);
 		if (!base->lcla_pool.base_unaligned) {
 			ret = -ENOMEM;
-			goto failure;
+			goto free_page_list;
 		}
 
 		base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
@@ -3473,12 +3438,13 @@ static int __init d40_lcla_allocate(struct d40_base *base)
 	if (dma_mapping_error(base->dev, pool->dma_addr)) {
 		pool->dma_addr = 0;
 		ret = -ENOMEM;
-		goto failure;
+		goto free_page_list;
 	}
 
 	writel(virt_to_phys(base->lcla_pool.base),
 	       base->virtbase + D40_DREG_LCLA);
-failure:
+	ret = 0;
+ free_page_list:
 	kfree(page_list);
 	return ret;
 }
@@ -3490,9 +3456,7 @@ static int __init d40_of_probe(struct platform_device *pdev,
 	int num_phy = 0, num_memcpy = 0, num_disabled = 0;
 	const __be32 *list;
 
-	pdata = devm_kzalloc(&pdev->dev,
-			     sizeof(struct stedma40_platform_data),
-			     GFP_KERNEL);
+	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata)
 		return -ENOMEM;
 
@@ -3574,7 +3538,7 @@ static int __init d40_probe(struct platform_device *pdev)
 	if (!res) {
 		ret = -ENOENT;
 		d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
-		goto failure;
+		goto destroy_cache;
 	}
 	base->lcpa_size = resource_size(res);
 	base->phy_lcpa = res->start;
@@ -3583,7 +3547,7 @@ static int __init d40_probe(struct platform_device *pdev)
 			       D40_NAME " I/O lcpa") == NULL) {
 		ret = -EBUSY;
 		d40_err(&pdev->dev, "Failed to request LCPA region %pR\n", res);
-		goto failure;
+		goto destroy_cache;
 	}
 
 	/* We make use of ESRAM memory for this. */
@@ -3599,7 +3563,7 @@ static int __init d40_probe(struct platform_device *pdev)
 	if (!base->lcpa_base) {
 		ret = -ENOMEM;
 		d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
-		goto failure;
+		goto destroy_cache;
 	}
 	/* If lcla has to be located in ESRAM we don't need to allocate */
 	if (base->plat_data->use_esram_lcla) {
@@ -3609,14 +3573,14 @@ static int __init d40_probe(struct platform_device *pdev)
 			ret = -ENOENT;
 			d40_err(&pdev->dev,
 				"No \"lcla_esram\" memory resource\n");
-			goto failure;
+			goto destroy_cache;
 		}
 		base->lcla_pool.base = ioremap(res->start,
 						resource_size(res));
 		if (!base->lcla_pool.base) {
 			ret = -ENOMEM;
 			d40_err(&pdev->dev, "Failed to ioremap LCLA region\n");
-			goto failure;
+			goto destroy_cache;
 		}
 		writel(res->start, base->virtbase + D40_DREG_LCLA);
 
@@ -3624,7 +3588,7 @@ static int __init d40_probe(struct platform_device *pdev)
 		ret = d40_lcla_allocate(base);
 		if (ret) {
 			d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
-			goto failure;
+			goto destroy_cache;
 		}
 	}
 
@@ -3635,7 +3599,7 @@ static int __init d40_probe(struct platform_device *pdev)
 	ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
 	if (ret) {
 		d40_err(&pdev->dev, "No IRQ defined\n");
-		goto failure;
+		goto destroy_cache;
 	}
 
 	if (base->plat_data->use_esram_lcla) {
@@ -3645,7 +3609,7 @@ static int __init d40_probe(struct platform_device *pdev)
 			d40_err(&pdev->dev, "Failed to get lcpa_regulator\n");
 			ret = PTR_ERR(base->lcpa_regulator);
 			base->lcpa_regulator = NULL;
-			goto failure;
+			goto destroy_cache;
 		}
 
 		ret = regulator_enable(base->lcpa_regulator);
@@ -3654,7 +3618,7 @@ static int __init d40_probe(struct platform_device *pdev)
 				"Failed to enable lcpa_regulator\n");
 			regulator_put(base->lcpa_regulator);
 			base->lcpa_regulator = NULL;
-			goto failure;
+			goto destroy_cache;
 		}
 	}
 
@@ -3669,13 +3633,13 @@ static int __init d40_probe(struct platform_device *pdev)
 
 	ret = d40_dmaengine_init(base, num_reserved_chans);
 	if (ret)
-		goto failure;
+		goto destroy_cache;
 
 	base->dev->dma_parms = &base->dma_parms;
 	ret = dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE);
 	if (ret) {
 		d40_err(&pdev->dev, "Failed to set dma max seg size\n");
-		goto failure;
+		goto destroy_cache;
 	}
 
 	d40_hw_init(base);
@@ -3689,8 +3653,7 @@ static int __init d40_probe(struct platform_device *pdev)
 
 	dev_info(base->dev, "initialized\n");
 	return 0;
-
-failure:
+ destroy_cache:
 	kmem_cache_destroy(base->desc_slab);
 	if (base->virtbase)
 		iounmap(base->virtbase);
@@ -3732,7 +3695,7 @@ failure:
 	kfree(base->lookup_phy_chans);
 	kfree(base->phy_res);
 	kfree(base);
-report_failure:
+ report_failure:
 	d40_err(&pdev->dev, "probe failed\n");
 	return ret;
 }
diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c
index 047476a1383d..307547f4848d 100644
--- a/drivers/dma/stm32-dma.c
+++ b/drivers/dma/stm32-dma.c
@@ -954,7 +954,7 @@ static void stm32_dma_desc_free(struct virt_dma_desc *vdesc)
 	kfree(container_of(vdesc, struct stm32_dma_desc, vdesc));
 }
 
-void stm32_dma_set_config(struct stm32_dma_chan *chan,
+static void stm32_dma_set_config(struct stm32_dma_chan *chan,
 			  struct stm32_dma_cfg *cfg)
 {
 	stm32_dma_clear_reg(&chan->chan_reg);
diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c
index 3835fcde3545..83461994e418 100644
--- a/drivers/dma/sun6i-dma.c
+++ b/drivers/dma/sun6i-dma.c
@@ -1011,6 +1011,12 @@ static struct sun6i_dma_config sun8i_a23_dma_cfg = {
 	.nr_max_vchans   = 37,
 };
 
+static struct sun6i_dma_config sun8i_a83t_dma_cfg = {
+	.nr_max_channels = 8,
+	.nr_max_requests = 28,
+	.nr_max_vchans   = 39,
+};
+
 /*
  * The H3 has 12 physical channels, a maximum DRQ port id of 27,
  * and a total of 34 usable source and destination endpoints.
@@ -1025,6 +1031,7 @@ static struct sun6i_dma_config sun8i_h3_dma_cfg = {
 static const struct of_device_id sun6i_dma_match[] = {
 	{ .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg },
 	{ .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg },
+	{ .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg },
 	{ .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg },
 	{ /* sentinel */ }
 };
diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c
index 6ab9eb98588a..3722b9d8d9fe 100644
--- a/drivers/dma/tegra20-apb-dma.c
+++ b/drivers/dma/tegra20-apb-dma.c
@@ -655,8 +655,7 @@ static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc,
 static void tegra_dma_tasklet(unsigned long data)
 {
 	struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data;
-	dma_async_tx_callback callback = NULL;
-	void *callback_param = NULL;
+	struct dmaengine_desc_callback cb;
 	struct tegra_dma_desc *dma_desc;
 	unsigned long flags;
 	int cb_count;
@@ -666,13 +665,12 @@ static void tegra_dma_tasklet(unsigned long data)
 		dma_desc  = list_first_entry(&tdc->cb_desc,
 					typeof(*dma_desc), cb_node);
 		list_del(&dma_desc->cb_node);
-		callback = dma_desc->txd.callback;
-		callback_param = dma_desc->txd.callback_param;
+		dmaengine_desc_get_callback(&dma_desc->txd, &cb);
 		cb_count = dma_desc->cb_count;
 		dma_desc->cb_count = 0;
 		spin_unlock_irqrestore(&tdc->lock, flags);
-		while (cb_count-- && callback)
-			callback(callback_param);
+		while (cb_count--)
+			dmaengine_desc_callback_invoke(&cb, NULL);
 		spin_lock_irqsave(&tdc->lock, flags);
 	}
 	spin_unlock_irqrestore(&tdc->lock, flags);
diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c
index c4b121c4559d..b10cbaa82ff5 100644
--- a/drivers/dma/tegra210-adma.c
+++ b/drivers/dma/tegra210-adma.c
@@ -670,7 +670,6 @@ static int tegra_adma_probe(struct platform_device *pdev)
 	const struct tegra_adma_chip_data *cdata;
 	struct tegra_adma *tdma;
 	struct resource	*res;
-	struct clk *clk;
 	int ret, i;
 
 	cdata = of_device_get_match_data(&pdev->dev);
@@ -697,18 +696,9 @@ static int tegra_adma_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	clk = clk_get(&pdev->dev, "d_audio");
-	if (IS_ERR(clk)) {
-		dev_err(&pdev->dev, "ADMA clock not found\n");
-		ret = PTR_ERR(clk);
-		goto clk_destroy;
-	}
-
-	ret = pm_clk_add_clk(&pdev->dev, clk);
-	if (ret) {
-		clk_put(clk);
+	ret = of_pm_clk_add_clk(&pdev->dev, "d_audio");
+	if (ret)
 		goto clk_destroy;
-	}
 
 	pm_runtime_enable(&pdev->dev);
 
diff --git a/drivers/dma/ti-dma-crossbar.c b/drivers/dma/ti-dma-crossbar.c
index 5ae294b256a7..3f24aeb48c0e 100644
--- a/drivers/dma/ti-dma-crossbar.c
+++ b/drivers/dma/ti-dma-crossbar.c
@@ -18,15 +18,19 @@
 
 #define TI_XBAR_DRA7		0
 #define TI_XBAR_AM335X		1
+static const u32 ti_xbar_type[] = {
+	[TI_XBAR_DRA7] = TI_XBAR_DRA7,
+	[TI_XBAR_AM335X] = TI_XBAR_AM335X,
+};
 
 static const struct of_device_id ti_dma_xbar_match[] = {
 	{
 		.compatible = "ti,dra7-dma-crossbar",
-		.data = (void *)TI_XBAR_DRA7,
+		.data = &ti_xbar_type[TI_XBAR_DRA7],
 	},
 	{
 		.compatible = "ti,am335x-edma-crossbar",
-		.data = (void *)TI_XBAR_AM335X,
+		.data = &ti_xbar_type[TI_XBAR_AM335X],
 	},
 	{},
 };
@@ -190,9 +194,6 @@ static int ti_am335x_xbar_probe(struct platform_device *pdev)
 #define TI_DRA7_XBAR_OUTPUTS	127
 #define TI_DRA7_XBAR_INPUTS	256
 
-#define TI_XBAR_EDMA_OFFSET	0
-#define TI_XBAR_SDMA_OFFSET	1
-
 struct ti_dra7_xbar_data {
 	void __iomem *iomem;
 
@@ -280,18 +281,25 @@ static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec,
 	return map;
 }
 
+#define TI_XBAR_EDMA_OFFSET	0
+#define TI_XBAR_SDMA_OFFSET	1
+static const u32 ti_dma_offset[] = {
+	[TI_XBAR_EDMA_OFFSET] = 0,
+	[TI_XBAR_SDMA_OFFSET] = 1,
+};
+
 static const struct of_device_id ti_dra7_master_match[] = {
 	{
 		.compatible = "ti,omap4430-sdma",
-		.data = (void *)TI_XBAR_SDMA_OFFSET,
+		.data = &ti_dma_offset[TI_XBAR_SDMA_OFFSET],
 	},
 	{
 		.compatible = "ti,edma3",
-		.data = (void *)TI_XBAR_EDMA_OFFSET,
+		.data = &ti_dma_offset[TI_XBAR_EDMA_OFFSET],
 	},
 	{
 		.compatible = "ti,edma3-tpcc",
-		.data = (void *)TI_XBAR_EDMA_OFFSET,
+		.data = &ti_dma_offset[TI_XBAR_EDMA_OFFSET],
 	},
 	{},
 };
@@ -311,7 +319,7 @@ static int ti_dra7_xbar_probe(struct platform_device *pdev)
 	struct property *prop;
 	struct resource *res;
 	u32 safe_val;
-	size_t sz;
+	int sz;
 	void __iomem *iomem;
 	int i, ret;
 
@@ -395,7 +403,7 @@ static int ti_dra7_xbar_probe(struct platform_device *pdev)
 
 	xbar->dmarouter.dev = &pdev->dev;
 	xbar->dmarouter.route_free = ti_dra7_xbar_free;
-	xbar->dma_offset = (u32)match->data;
+	xbar->dma_offset = *(u32 *)match->data;
 
 	mutex_init(&xbar->mutex);
 	platform_set_drvdata(pdev, xbar);
@@ -428,7 +436,7 @@ static int ti_dma_xbar_probe(struct platform_device *pdev)
 	if (unlikely(!match))
 		return -EINVAL;
 
-	switch ((u32)match->data) {
+	switch (*(u32 *)match->data) {
 	case TI_XBAR_DRA7:
 		ret = ti_dra7_xbar_probe(pdev);
 		break;
diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c
index e82745aa42a8..896bafb7a532 100644
--- a/drivers/dma/timb_dma.c
+++ b/drivers/dma/timb_dma.c
@@ -226,8 +226,7 @@ static void __td_start_dma(struct timb_dma_chan *td_chan)
 
 static void __td_finish(struct timb_dma_chan *td_chan)
 {
-	dma_async_tx_callback		callback;
-	void				*param;
+	struct dmaengine_desc_callback	cb;
 	struct dma_async_tx_descriptor	*txd;
 	struct timb_dma_desc		*td_desc;
 
@@ -252,8 +251,7 @@ static void __td_finish(struct timb_dma_chan *td_chan)
 	dma_cookie_complete(txd);
 	td_chan->ongoing = false;
 
-	callback = txd->callback;
-	param = txd->callback_param;
+	dmaengine_desc_get_callback(txd, &cb);
 
 	list_move(&td_desc->desc_node, &td_chan->free_list);
 
@@ -262,8 +260,7 @@ static void __td_finish(struct timb_dma_chan *td_chan)
 	 * The API requires that no submissions are done from a
 	 * callback, so we don't need to drop the lock here
 	 */
-	if (callback)
-		callback(param);
+	dmaengine_desc_callback_invoke(&cb, NULL);
 }
 
 static u32 __td_ier_mask(struct timb_dma *td)
diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c
index 7632290e7c14..4d8c7b9078fd 100644
--- a/drivers/dma/txx9dmac.c
+++ b/drivers/dma/txx9dmac.c
@@ -403,16 +403,14 @@ static void
 txx9dmac_descriptor_complete(struct txx9dmac_chan *dc,
 			     struct txx9dmac_desc *desc)
 {
-	dma_async_tx_callback callback;
-	void *param;
+	struct dmaengine_desc_callback cb;
 	struct dma_async_tx_descriptor *txd = &desc->txd;
 
 	dev_vdbg(chan2dev(&dc->chan), "descriptor %u %p complete\n",
 		 txd->cookie, desc);
 
 	dma_cookie_complete(txd);
-	callback = txd->callback;
-	param = txd->callback_param;
+	dmaengine_desc_get_callback(txd, &cb);
 
 	txx9dmac_sync_desc_for_cpu(dc, desc);
 	list_splice_init(&desc->tx_list, &dc->free_list);
@@ -423,8 +421,7 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc,
 	 * The API requires that no submissions are done from a
 	 * callback, so we don't need to drop the lock here
 	 */
-	if (callback)
-		callback(param);
+	dmaengine_desc_callback_invoke(&cb, NULL);
 	dma_run_dependencies(txd);
 }
 
diff --git a/drivers/dma/virt-dma.c b/drivers/dma/virt-dma.c
index a35c211857dd..e47fc9b0944f 100644
--- a/drivers/dma/virt-dma.c
+++ b/drivers/dma/virt-dma.c
@@ -87,8 +87,7 @@ static void vchan_complete(unsigned long arg)
 {
 	struct virt_dma_chan *vc = (struct virt_dma_chan *)arg;
 	struct virt_dma_desc *vd;
-	dma_async_tx_callback cb = NULL;
-	void *cb_data = NULL;
+	struct dmaengine_desc_callback cb;
 	LIST_HEAD(head);
 
 	spin_lock_irq(&vc->lock);
@@ -96,18 +95,17 @@ static void vchan_complete(unsigned long arg)
 	vd = vc->cyclic;
 	if (vd) {
 		vc->cyclic = NULL;
-		cb = vd->tx.callback;
-		cb_data = vd->tx.callback_param;
+		dmaengine_desc_get_callback(&vd->tx, &cb);
+	} else {
+		memset(&cb, 0, sizeof(cb));
 	}
 	spin_unlock_irq(&vc->lock);
 
-	if (cb)
-		cb(cb_data);
+	dmaengine_desc_callback_invoke(&cb, NULL);
 
 	while (!list_empty(&head)) {
 		vd = list_first_entry(&head, struct virt_dma_desc, node);
-		cb = vd->tx.callback;
-		cb_data = vd->tx.callback_param;
+		dmaengine_desc_get_callback(&vd->tx, &cb);
 
 		list_del(&vd->node);
 		if (dmaengine_desc_test_reuse(&vd->tx))
@@ -115,8 +113,7 @@ static void vchan_complete(unsigned long arg)
 		else
 			vc->desc_free(vd);
 
-		if (cb)
-			cb(cb_data);
+		dmaengine_desc_callback_invoke(&cb, NULL);
 	}
 }
 
diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h
index d9731ca5e262..3f776a46a29c 100644
--- a/drivers/dma/virt-dma.h
+++ b/drivers/dma/virt-dma.h
@@ -45,6 +45,8 @@ static inline struct virt_dma_chan *to_virt_chan(struct dma_chan *chan)
 void vchan_dma_desc_free_list(struct virt_dma_chan *vc, struct list_head *head);
 void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev);
 struct virt_dma_desc *vchan_find_desc(struct virt_dma_chan *, dma_cookie_t);
+extern dma_cookie_t vchan_tx_submit(struct dma_async_tx_descriptor *);
+extern int vchan_tx_desc_free(struct dma_async_tx_descriptor *);
 
 /**
  * vchan_tx_prep - prepare a descriptor
@@ -55,8 +57,6 @@ struct virt_dma_desc *vchan_find_desc(struct virt_dma_chan *, dma_cookie_t);
 static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan *vc,
 	struct virt_dma_desc *vd, unsigned long tx_flags)
 {
-	extern dma_cookie_t vchan_tx_submit(struct dma_async_tx_descriptor *);
-	extern int vchan_tx_desc_free(struct dma_async_tx_descriptor *);
 	unsigned long flags;
 
 	dma_async_tx_descriptor_init(&vd->tx, &vc->chan);
@@ -123,10 +123,8 @@ static inline void vchan_cyclic_callback(struct virt_dma_desc *vd)
  */
 static inline struct virt_dma_desc *vchan_next_desc(struct virt_dma_chan *vc)
 {
-	if (list_empty(&vc->desc_issued))
-		return NULL;
-
-	return list_first_entry(&vc->desc_issued, struct virt_dma_desc, node);
+	return list_first_entry_or_null(&vc->desc_issued,
+					struct virt_dma_desc, node);
 }
 
 /**
diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c
index 9cb93c5b655d..8b693b712d0f 100644
--- a/drivers/dma/xgene-dma.c
+++ b/drivers/dma/xgene-dma.c
@@ -606,12 +606,10 @@ static void xgene_dma_run_tx_complete_actions(struct xgene_dma_chan *chan,
 		return;
 
 	dma_cookie_complete(tx);
+	dma_descriptor_unmap(tx);
 
 	/* Run the link descriptor callback function */
-	if (tx->callback)
-		tx->callback(tx->callback_param);
-
-	dma_descriptor_unmap(tx);
+	dmaengine_desc_get_callback_invoke(tx, NULL);
 
 	/* Run any dependencies */
 	dma_run_dependencies(tx);
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index 4e223d094433..8288fe4d17c3 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -755,8 +755,7 @@ static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan)
 	spin_lock_irqsave(&chan->lock, flags);
 
 	list_for_each_entry_safe(desc, next, &chan->done_list, node) {
-		dma_async_tx_callback callback;
-		void *callback_param;
+		struct dmaengine_desc_callback cb;
 
 		if (desc->cyclic) {
 			xilinx_dma_chan_handle_cyclic(chan, desc, &flags);
@@ -767,11 +766,10 @@ static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan)
 		list_del(&desc->node);
 
 		/* Run the link descriptor callback function */
-		callback = desc->async_tx.callback;
-		callback_param = desc->async_tx.callback_param;
-		if (callback) {
+		dmaengine_desc_get_callback(&desc->async_tx, &cb);
+		if (dmaengine_desc_callback_valid(&cb)) {
 			spin_unlock_irqrestore(&chan->lock, flags);
-			callback(callback_param);
+			dmaengine_desc_callback_invoke(&cb, NULL);
 			spin_lock_irqsave(&chan->lock, flags);
 		}
 
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index d0c1dab9b435..82d85cce81f8 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -251,6 +251,14 @@ config EDAC_SBRIDGE
 	  Support for error detection and correction the Intel
 	  Sandy Bridge, Ivy Bridge and Haswell Integrated Memory Controllers.
 
+config EDAC_SKX
+	tristate "Intel Skylake server Integrated MC"
+	depends on EDAC_MM_EDAC && PCI && X86_64 && X86_MCE_INTEL
+	depends on PCI_MMCONFIG
+	help
+	  Support for error detection and correction the Intel
+	  Skylake server Integrated Memory Controllers.
+
 config EDAC_MPC85XX
 	tristate "Freescale MPC83xx / MPC85xx"
 	depends on EDAC_MM_EDAC && FSL_SOC
@@ -258,6 +266,13 @@ config EDAC_MPC85XX
 	  Support for error detection and correction on the Freescale
 	  MPC8349, MPC8560, MPC8540, MPC8548, T4240
 
+config EDAC_LAYERSCAPE
+	tristate "Freescale Layerscape DDR"
+	depends on EDAC_MM_EDAC && ARCH_LAYERSCAPE
+	help
+	  Support for error detection and correction on Freescale memory
+	  controllers on Layerscape SoCs.
+
 config EDAC_MV64X60
 	tristate "Marvell MV64x60"
 	depends on EDAC_MM_EDAC && MV64X60
@@ -398,6 +413,41 @@ config EDAC_ALTERA_ETHERNET
 	  Support for error detection and correction on the
 	  Altera Ethernet FIFO Memory for Altera SoCs.
 
+config EDAC_ALTERA_NAND
+	bool "Altera NAND FIFO ECC"
+	depends on EDAC_ALTERA=y && MTD_NAND_DENALI
+	help
+	  Support for error detection and correction on the
+	  Altera NAND FIFO Memory for Altera SoCs.
+
+config EDAC_ALTERA_DMA
+	bool "Altera DMA FIFO ECC"
+	depends on EDAC_ALTERA=y && PL330_DMA=y
+	help
+	  Support for error detection and correction on the
+	  Altera DMA FIFO Memory for Altera SoCs.
+
+config EDAC_ALTERA_USB
+	bool "Altera USB FIFO ECC"
+	depends on EDAC_ALTERA=y && USB_DWC2
+	help
+	  Support for error detection and correction on the
+	  Altera USB FIFO Memory for Altera SoCs.
+
+config EDAC_ALTERA_QSPI
+	bool "Altera QSPI FIFO ECC"
+	depends on EDAC_ALTERA=y && SPI_CADENCE_QUADSPI
+	help
+	  Support for error detection and correction on the
+	  Altera QSPI FIFO Memory for Altera SoCs.
+
+config EDAC_ALTERA_SDMMC
+	bool "Altera SDMMC FIFO ECC"
+	depends on EDAC_ALTERA=y && MMC_DW
+	help
+	  Support for error detection and correction on the
+	  Altera SDMMC FIFO Memory for Altera SoCs.
+
 config EDAC_SYNOPSYS
 	tristate "Synopsys DDR Memory Controller"
 	depends on EDAC_MM_EDAC && ARCH_ZYNQ
diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile
index f9e4a3e0e6e9..88e472e8b9a9 100644
--- a/drivers/edac/Makefile
+++ b/drivers/edac/Makefile
@@ -31,6 +31,7 @@ obj-$(CONFIG_EDAC_I5400)		+= i5400_edac.o
 obj-$(CONFIG_EDAC_I7300)		+= i7300_edac.o
 obj-$(CONFIG_EDAC_I7CORE)		+= i7core_edac.o
 obj-$(CONFIG_EDAC_SBRIDGE)		+= sb_edac.o
+obj-$(CONFIG_EDAC_SKX)			+= skx_edac.o
 obj-$(CONFIG_EDAC_E7XXX)		+= e7xxx_edac.o
 obj-$(CONFIG_EDAC_E752X)		+= e752x_edac.o
 obj-$(CONFIG_EDAC_I82443BXGX)		+= i82443bxgx_edac.o
@@ -50,7 +51,13 @@ amd64_edac_mod-$(CONFIG_EDAC_AMD64_ERROR_INJECTION) += amd64_edac_inj.o
 obj-$(CONFIG_EDAC_AMD64)		+= amd64_edac_mod.o
 
 obj-$(CONFIG_EDAC_PASEMI)		+= pasemi_edac.o
-obj-$(CONFIG_EDAC_MPC85XX)		+= mpc85xx_edac.o
+
+mpc85xx_edac_mod-y			:= fsl_ddr_edac.o mpc85xx_edac.o
+obj-$(CONFIG_EDAC_MPC85XX)		+= mpc85xx_edac_mod.o
+
+layerscape_edac_mod-y			:= fsl_ddr_edac.o layerscape_edac.o
+obj-$(CONFIG_EDAC_LAYERSCAPE)		+= layerscape_edac_mod.o
+
 obj-$(CONFIG_EDAC_MV64X60)		+= mv64x60_edac.o
 obj-$(CONFIG_EDAC_CELL)			+= cell_edac.o
 obj-$(CONFIG_EDAC_PPC4XX)		+= ppc4xx_edac.o
diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
index 2398d0701f5b..58d3e2b39b5b 100644
--- a/drivers/edac/altera_edac.c
+++ b/drivers/edac/altera_edac.c
@@ -203,7 +203,7 @@ static void altr_sdr_mc_create_debugfs_nodes(struct mem_ctl_info *mci)
 	if (!mci->debugfs)
 		return;
 
-	edac_debugfs_create_file("inject_ctrl", S_IWUSR, mci->debugfs, mci,
+	edac_debugfs_create_file("altr_trigger", S_IWUSR, mci->debugfs, mci,
 				 &altr_sdr_mc_debug_inject_fops);
 }
 
@@ -680,7 +680,7 @@ static void altr_create_edacdev_dbgfs(struct edac_device_ctl_info *edac_dci,
 	if (!drvdata->debugfs_dir)
 		return;
 
-	if (!edac_debugfs_create_file(priv->dbgfs_name, S_IWUSR,
+	if (!edac_debugfs_create_file("altr_trigger", S_IWUSR,
 				      drvdata->debugfs_dir, edac_dci,
 				      priv->inject_fops))
 		debugfs_remove_recursive(drvdata->debugfs_dir);
@@ -1108,7 +1108,6 @@ static const struct edac_device_prv_data ocramecc_data = {
 	.setup = altr_check_ecc_deps,
 	.ce_clear_mask = (ALTR_OCR_ECC_EN | ALTR_OCR_ECC_SERR),
 	.ue_clear_mask = (ALTR_OCR_ECC_EN | ALTR_OCR_ECC_DERR),
-	.dbgfs_name = "altr_ocram_trigger",
 	.alloc_mem = ocram_alloc_mem,
 	.free_mem = ocram_free_mem,
 	.ecc_enable_mask = ALTR_OCR_ECC_EN,
@@ -1125,7 +1124,6 @@ static const struct edac_device_prv_data a10_ocramecc_data = {
 	.ce_clear_mask = ALTR_A10_ECC_SERRPENA,
 	.ue_clear_mask = ALTR_A10_ECC_DERRPENA,
 	.irq_status_mask = A10_SYSMGR_ECC_INTSTAT_OCRAM,
-	.dbgfs_name = "altr_ocram_trigger",
 	.ecc_enable_mask = ALTR_A10_OCRAM_ECC_EN_CTL,
 	.ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST,
 	.ce_set_mask = ALTR_A10_ECC_TSERRA,
@@ -1228,7 +1226,6 @@ static const struct edac_device_prv_data l2ecc_data = {
 	.setup = altr_l2_check_deps,
 	.ce_clear_mask = 0,
 	.ue_clear_mask = 0,
-	.dbgfs_name = "altr_l2_trigger",
 	.alloc_mem = l2_alloc_mem,
 	.free_mem = l2_free_mem,
 	.ecc_enable_mask = ALTR_L2_ECC_EN,
@@ -1244,7 +1241,6 @@ static const struct edac_device_prv_data a10_l2ecc_data = {
 	.ce_clear_mask = ALTR_A10_L2_ECC_SERR_CLR,
 	.ue_clear_mask = ALTR_A10_L2_ECC_MERR_CLR,
 	.irq_status_mask = A10_SYSMGR_ECC_INTSTAT_L2,
-	.dbgfs_name = "altr_l2_trigger",
 	.alloc_mem = l2_alloc_mem,
 	.free_mem = l2_free_mem,
 	.ecc_enable_mask = ALTR_A10_L2_ECC_EN_CTL,
@@ -1266,7 +1262,6 @@ static const struct edac_device_prv_data a10_enetecc_data = {
 	.setup = altr_check_ecc_deps,
 	.ce_clear_mask = ALTR_A10_ECC_SERRPENA,
 	.ue_clear_mask = ALTR_A10_ECC_DERRPENA,
-	.dbgfs_name = "altr_trigger",
 	.ecc_enable_mask = ALTR_A10_COMMON_ECC_EN_CTL,
 	.ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST,
 	.ce_set_mask = ALTR_A10_ECC_TSERRA,
@@ -1285,6 +1280,292 @@ early_initcall(socfpga_init_ethernet_ecc);
 
 #endif	/* CONFIG_EDAC_ALTERA_ETHERNET */
 
+/********************** NAND Device Functions **********************/
+
+#ifdef CONFIG_EDAC_ALTERA_NAND
+
+static const struct edac_device_prv_data a10_nandecc_data = {
+	.setup = altr_check_ecc_deps,
+	.ce_clear_mask = ALTR_A10_ECC_SERRPENA,
+	.ue_clear_mask = ALTR_A10_ECC_DERRPENA,
+	.ecc_enable_mask = ALTR_A10_COMMON_ECC_EN_CTL,
+	.ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST,
+	.ce_set_mask = ALTR_A10_ECC_TSERRA,
+	.ue_set_mask = ALTR_A10_ECC_TDERRA,
+	.set_err_ofst = ALTR_A10_ECC_INTTEST_OFST,
+	.ecc_irq_handler = altr_edac_a10_ecc_irq,
+	.inject_fops = &altr_edac_a10_device_inject_fops,
+};
+
+static int __init socfpga_init_nand_ecc(void)
+{
+	return altr_init_a10_ecc_device_type("altr,socfpga-nand-ecc");
+}
+
+early_initcall(socfpga_init_nand_ecc);
+
+#endif	/* CONFIG_EDAC_ALTERA_NAND */
+
+/********************** DMA Device Functions **********************/
+
+#ifdef CONFIG_EDAC_ALTERA_DMA
+
+static const struct edac_device_prv_data a10_dmaecc_data = {
+	.setup = altr_check_ecc_deps,
+	.ce_clear_mask = ALTR_A10_ECC_SERRPENA,
+	.ue_clear_mask = ALTR_A10_ECC_DERRPENA,
+	.ecc_enable_mask = ALTR_A10_COMMON_ECC_EN_CTL,
+	.ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST,
+	.ce_set_mask = ALTR_A10_ECC_TSERRA,
+	.ue_set_mask = ALTR_A10_ECC_TDERRA,
+	.set_err_ofst = ALTR_A10_ECC_INTTEST_OFST,
+	.ecc_irq_handler = altr_edac_a10_ecc_irq,
+	.inject_fops = &altr_edac_a10_device_inject_fops,
+};
+
+static int __init socfpga_init_dma_ecc(void)
+{
+	return altr_init_a10_ecc_device_type("altr,socfpga-dma-ecc");
+}
+
+early_initcall(socfpga_init_dma_ecc);
+
+#endif	/* CONFIG_EDAC_ALTERA_DMA */
+
+/********************** USB Device Functions **********************/
+
+#ifdef CONFIG_EDAC_ALTERA_USB
+
+static const struct edac_device_prv_data a10_usbecc_data = {
+	.setup = altr_check_ecc_deps,
+	.ce_clear_mask = ALTR_A10_ECC_SERRPENA,
+	.ue_clear_mask = ALTR_A10_ECC_DERRPENA,
+	.ecc_enable_mask = ALTR_A10_COMMON_ECC_EN_CTL,
+	.ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST,
+	.ce_set_mask = ALTR_A10_ECC_TSERRA,
+	.ue_set_mask = ALTR_A10_ECC_TDERRA,
+	.set_err_ofst = ALTR_A10_ECC_INTTEST_OFST,
+	.ecc_irq_handler = altr_edac_a10_ecc_irq,
+	.inject_fops = &altr_edac_a10_device_inject_fops,
+};
+
+static int __init socfpga_init_usb_ecc(void)
+{
+	return altr_init_a10_ecc_device_type("altr,socfpga-usb-ecc");
+}
+
+early_initcall(socfpga_init_usb_ecc);
+
+#endif	/* CONFIG_EDAC_ALTERA_USB */
+
+/********************** QSPI Device Functions **********************/
+
+#ifdef CONFIG_EDAC_ALTERA_QSPI
+
+static const struct edac_device_prv_data a10_qspiecc_data = {
+	.setup = altr_check_ecc_deps,
+	.ce_clear_mask = ALTR_A10_ECC_SERRPENA,
+	.ue_clear_mask = ALTR_A10_ECC_DERRPENA,
+	.ecc_enable_mask = ALTR_A10_COMMON_ECC_EN_CTL,
+	.ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST,
+	.ce_set_mask = ALTR_A10_ECC_TSERRA,
+	.ue_set_mask = ALTR_A10_ECC_TDERRA,
+	.set_err_ofst = ALTR_A10_ECC_INTTEST_OFST,
+	.ecc_irq_handler = altr_edac_a10_ecc_irq,
+	.inject_fops = &altr_edac_a10_device_inject_fops,
+};
+
+static int __init socfpga_init_qspi_ecc(void)
+{
+	return altr_init_a10_ecc_device_type("altr,socfpga-qspi-ecc");
+}
+
+early_initcall(socfpga_init_qspi_ecc);
+
+#endif	/* CONFIG_EDAC_ALTERA_QSPI */
+
+/********************* SDMMC Device Functions **********************/
+
+#ifdef CONFIG_EDAC_ALTERA_SDMMC
+
+static const struct edac_device_prv_data a10_sdmmceccb_data;
+static int altr_portb_setup(struct altr_edac_device_dev *device)
+{
+	struct edac_device_ctl_info *dci;
+	struct altr_edac_device_dev *altdev;
+	char *ecc_name = "sdmmcb-ecc";
+	int edac_idx, rc;
+	struct device_node *np;
+	const struct edac_device_prv_data *prv = &a10_sdmmceccb_data;
+
+	rc = altr_check_ecc_deps(device);
+	if (rc)
+		return rc;
+
+	np = of_find_compatible_node(NULL, NULL, "altr,socfpga-sdmmc-ecc");
+	if (!np) {
+		edac_printk(KERN_WARNING, EDAC_DEVICE, "SDMMC node not found\n");
+		return -ENODEV;
+	}
+
+	/* Create the PortB EDAC device */
+	edac_idx = edac_device_alloc_index();
+	dci = edac_device_alloc_ctl_info(sizeof(*altdev), ecc_name, 1,
+					 ecc_name, 1, 0, NULL, 0, edac_idx);
+	if (!dci) {
+		edac_printk(KERN_ERR, EDAC_DEVICE,
+			    "%s: Unable to allocate PortB EDAC device\n",
+			    ecc_name);
+		return -ENOMEM;
+	}
+
+	/* Initialize the PortB EDAC device structure from PortA structure */
+	altdev = dci->pvt_info;
+	*altdev = *device;
+
+	if (!devres_open_group(&altdev->ddev, altr_portb_setup, GFP_KERNEL))
+		return -ENOMEM;
+
+	/* Update PortB specific values */
+	altdev->edac_dev_name = ecc_name;
+	altdev->edac_idx = edac_idx;
+	altdev->edac_dev = dci;
+	altdev->data = prv;
+	dci->dev = &altdev->ddev;
+	dci->ctl_name = "Altera ECC Manager";
+	dci->mod_name = ecc_name;
+	dci->dev_name = ecc_name;
+
+	/* Update the IRQs for PortB */
+	altdev->sb_irq = irq_of_parse_and_map(np, 2);
+	if (!altdev->sb_irq) {
+		edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB SBIRQ alloc\n");
+		rc = -ENODEV;
+		goto err_release_group_1;
+	}
+	rc = devm_request_irq(&altdev->ddev, altdev->sb_irq,
+			      prv->ecc_irq_handler,
+			      IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
+			      ecc_name, altdev);
+	if (rc) {
+		edac_printk(KERN_ERR, EDAC_DEVICE, "PortB SBERR IRQ error\n");
+		goto err_release_group_1;
+	}
+
+	altdev->db_irq = irq_of_parse_and_map(np, 3);
+	if (!altdev->db_irq) {
+		edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB DBIRQ alloc\n");
+		rc = -ENODEV;
+		goto err_release_group_1;
+	}
+	rc = devm_request_irq(&altdev->ddev, altdev->db_irq,
+			      prv->ecc_irq_handler,
+			      IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
+			      ecc_name, altdev);
+	if (rc) {
+		edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n");
+		goto err_release_group_1;
+	}
+
+	rc = edac_device_add_device(dci);
+	if (rc) {
+		edac_printk(KERN_ERR, EDAC_DEVICE,
+			    "edac_device_add_device portB failed\n");
+		rc = -ENOMEM;
+		goto err_release_group_1;
+	}
+	altr_create_edacdev_dbgfs(dci, prv);
+
+	list_add(&altdev->next, &altdev->edac->a10_ecc_devices);
+
+	devres_remove_group(&altdev->ddev, altr_portb_setup);
+
+	return 0;
+
+err_release_group_1:
+	edac_device_free_ctl_info(dci);
+	devres_release_group(&altdev->ddev, altr_portb_setup);
+	edac_printk(KERN_ERR, EDAC_DEVICE,
+		    "%s:Error setting up EDAC device: %d\n", ecc_name, rc);
+	return rc;
+}
+
+static irqreturn_t altr_edac_a10_ecc_irq_portb(int irq, void *dev_id)
+{
+	struct altr_edac_device_dev *ad = dev_id;
+	void __iomem  *base = ad->base;
+	const struct edac_device_prv_data *priv = ad->data;
+
+	if (irq == ad->sb_irq) {
+		writel(priv->ce_clear_mask,
+		       base + ALTR_A10_ECC_INTSTAT_OFST);
+		edac_device_handle_ce(ad->edac_dev, 0, 0, ad->edac_dev_name);
+		return IRQ_HANDLED;
+	} else if (irq == ad->db_irq) {
+		writel(priv->ue_clear_mask,
+		       base + ALTR_A10_ECC_INTSTAT_OFST);
+		edac_device_handle_ue(ad->edac_dev, 0, 0, ad->edac_dev_name);
+		return IRQ_HANDLED;
+	}
+
+	WARN_ONCE(1, "Unhandled IRQ%d on Port B.", irq);
+
+	return IRQ_NONE;
+}
+
+static const struct edac_device_prv_data a10_sdmmcecca_data = {
+	.setup = altr_portb_setup,
+	.ce_clear_mask = ALTR_A10_ECC_SERRPENA,
+	.ue_clear_mask = ALTR_A10_ECC_DERRPENA,
+	.ecc_enable_mask = ALTR_A10_COMMON_ECC_EN_CTL,
+	.ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST,
+	.ce_set_mask = ALTR_A10_ECC_SERRPENA,
+	.ue_set_mask = ALTR_A10_ECC_DERRPENA,
+	.set_err_ofst = ALTR_A10_ECC_INTTEST_OFST,
+	.ecc_irq_handler = altr_edac_a10_ecc_irq,
+	.inject_fops = &altr_edac_a10_device_inject_fops,
+};
+
+static const struct edac_device_prv_data a10_sdmmceccb_data = {
+	.setup = altr_portb_setup,
+	.ce_clear_mask = ALTR_A10_ECC_SERRPENB,
+	.ue_clear_mask = ALTR_A10_ECC_DERRPENB,
+	.ecc_enable_mask = ALTR_A10_COMMON_ECC_EN_CTL,
+	.ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST,
+	.ce_set_mask = ALTR_A10_ECC_TSERRB,
+	.ue_set_mask = ALTR_A10_ECC_TDERRB,
+	.set_err_ofst = ALTR_A10_ECC_INTTEST_OFST,
+	.ecc_irq_handler = altr_edac_a10_ecc_irq_portb,
+	.inject_fops = &altr_edac_a10_device_inject_fops,
+};
+
+static int __init socfpga_init_sdmmc_ecc(void)
+{
+	int rc = -ENODEV;
+	struct device_node *child = of_find_compatible_node(NULL, NULL,
+						"altr,socfpga-sdmmc-ecc");
+	if (!child) {
+		edac_printk(KERN_WARNING, EDAC_DEVICE, "SDMMC node not found\n");
+		return -ENODEV;
+	}
+
+	if (!of_device_is_available(child))
+		goto exit;
+
+	if (validate_parent_available(child))
+		goto exit;
+
+	rc = altr_init_a10_ecc_block(child, ALTR_A10_SDMMC_IRQ_MASK,
+				     a10_sdmmcecca_data.ecc_enable_mask, 1);
+exit:
+	of_node_put(child);
+	return rc;
+}
+
+early_initcall(socfpga_init_sdmmc_ecc);
+
+#endif	/* CONFIG_EDAC_ALTERA_SDMMC */
+
 /********************* Arria10 EDAC Device Functions *************************/
 static const struct of_device_id altr_edac_a10_device_of_match[] = {
 #ifdef CONFIG_EDAC_ALTERA_L2C
@@ -1298,6 +1579,21 @@ static const struct of_device_id altr_edac_a10_device_of_match[] = {
 	{ .compatible = "altr,socfpga-eth-mac-ecc",
 	  .data = &a10_enetecc_data },
 #endif
+#ifdef CONFIG_EDAC_ALTERA_NAND
+	{ .compatible = "altr,socfpga-nand-ecc", .data = &a10_nandecc_data },
+#endif
+#ifdef CONFIG_EDAC_ALTERA_DMA
+	{ .compatible = "altr,socfpga-dma-ecc", .data = &a10_dmaecc_data },
+#endif
+#ifdef CONFIG_EDAC_ALTERA_USB
+	{ .compatible = "altr,socfpga-usb-ecc", .data = &a10_usbecc_data },
+#endif
+#ifdef CONFIG_EDAC_ALTERA_QSPI
+	{ .compatible = "altr,socfpga-qspi-ecc", .data = &a10_qspiecc_data },
+#endif
+#ifdef CONFIG_EDAC_ALTERA_SDMMC
+	{ .compatible = "altr,socfpga-sdmmc-ecc", .data = &a10_sdmmcecca_data },
+#endif
 	{},
 };
 MODULE_DEVICE_TABLE(of, altr_edac_a10_device_of_match);
@@ -1451,11 +1747,11 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac,
 		rc = -ENODEV;
 		goto err_release_group1;
 	}
-	rc = devm_request_irq(edac->dev, altdev->sb_irq,
-			      prv->ecc_irq_handler,
-			      IRQF_SHARED, ecc_name, altdev);
+	rc = devm_request_irq(edac->dev, altdev->sb_irq, prv->ecc_irq_handler,
+			      IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
+			      ecc_name, altdev);
 	if (rc) {
-		edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n");
+		edac_printk(KERN_ERR, EDAC_DEVICE, "No SBERR IRQ resource\n");
 		goto err_release_group1;
 	}
 
@@ -1465,9 +1761,9 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac,
 		rc = -ENODEV;
 		goto err_release_group1;
 	}
-	rc = devm_request_irq(edac->dev, altdev->db_irq,
-			      prv->ecc_irq_handler,
-			      IRQF_SHARED, ecc_name, altdev);
+	rc = devm_request_irq(edac->dev, altdev->db_irq, prv->ecc_irq_handler,
+			      IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
+			      ecc_name, altdev);
 	if (rc) {
 		edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n");
 		goto err_release_group1;
@@ -1526,7 +1822,7 @@ static int a10_eccmgr_irqdomain_map(struct irq_domain *d, unsigned int irq,
 	return 0;
 }
 
-struct irq_domain_ops a10_eccmgr_ic_ops = {
+static struct irq_domain_ops a10_eccmgr_ic_ops = {
 	.map = a10_eccmgr_irqdomain_map,
 	.xlate = irq_domain_xlate_twocell,
 };
@@ -1584,15 +1880,19 @@ static int altr_edac_a10_probe(struct platform_device *pdev)
 	for_each_child_of_node(pdev->dev.of_node, child) {
 		if (!of_device_is_available(child))
 			continue;
-		if (of_device_is_compatible(child, "altr,socfpga-a10-l2-ecc"))
-			altr_edac_a10_device_add(edac, child);
-		else if ((of_device_is_compatible(child,
-					"altr,socfpga-a10-ocram-ecc")) ||
-			 (of_device_is_compatible(child,
-					"altr,socfpga-eth-mac-ecc")))
+
+		if (of_device_is_compatible(child, "altr,socfpga-a10-l2-ecc") || 
+		    of_device_is_compatible(child, "altr,socfpga-a10-ocram-ecc") ||
+		    of_device_is_compatible(child, "altr,socfpga-eth-mac-ecc") ||
+		    of_device_is_compatible(child, "altr,socfpga-nand-ecc") ||
+		    of_device_is_compatible(child, "altr,socfpga-dma-ecc") ||
+		    of_device_is_compatible(child, "altr,socfpga-usb-ecc") ||
+		    of_device_is_compatible(child, "altr,socfpga-qspi-ecc") ||
+		    of_device_is_compatible(child, "altr,socfpga-sdmmc-ecc"))
+
 			altr_edac_a10_device_add(edac, child);
-		else if (of_device_is_compatible(child,
-						 "altr,sdram-edac-a10"))
+
+		else if (of_device_is_compatible(child, "altr,sdram-edac-a10"))
 			of_platform_populate(pdev->dev.of_node,
 					     altr_sdram_ctrl_of_match,
 					     NULL, &pdev->dev);
diff --git a/drivers/edac/altera_edac.h b/drivers/edac/altera_edac.h
index 687d8e754d36..cbc96290f743 100644
--- a/drivers/edac/altera_edac.h
+++ b/drivers/edac/altera_edac.h
@@ -250,6 +250,8 @@ struct altr_sdram_mc_data {
 #define ALTR_A10_ECC_INTTEST_OFST       0x24
 #define ALTR_A10_ECC_TSERRA             BIT(0)
 #define ALTR_A10_ECC_TDERRA             BIT(8)
+#define ALTR_A10_ECC_TSERRB             BIT(16)
+#define ALTR_A10_ECC_TDERRB             BIT(24)
 
 /* ECC Manager Defines */
 #define A10_SYSMGR_ECC_INTMASK_SET_OFST   0x94
@@ -288,6 +290,9 @@ struct altr_sdram_mc_data {
 /* Arria 10 Ethernet ECC Management Group Defines */
 #define ALTR_A10_COMMON_ECC_EN_CTL      BIT(0)
 
+/* Arria 10 SDMMC ECC Management Group Defines */
+#define ALTR_A10_SDMMC_IRQ_MASK         (BIT(16) | BIT(15))
+
 /* A10 ECC Controller memory initialization timeout */
 #define ALTR_A10_ECC_INIT_WATCHDOG_10US      10000
 
@@ -298,7 +303,6 @@ struct edac_device_prv_data {
 	int ce_clear_mask;
 	int ue_clear_mask;
 	int irq_status_mask;
-	char dbgfs_name[20];
 	void * (*alloc_mem)(size_t size, void **other);
 	void (*free_mem)(void *p, size_t size, void *other);
 	int ecc_enable_mask;
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index 8c0ec2128907..ee181c53626f 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -1425,11 +1425,17 @@ static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
 
 		if (intlv_addr & 0x2) {
 			u8 shift = intlv_addr & 0x1 ? 9 : 6;
-			u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
+			u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
 
 			return ((sys_addr >> shift) & 1) ^ temp;
 		}
 
+		if (intlv_addr & 0x4) {
+			u8 shift = intlv_addr & 0x1 ? 9 : 8;
+
+			return (sys_addr >> shift) & 1;
+		}
+
 		return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
 	}
 
@@ -1726,8 +1732,11 @@ static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
 	if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
 		return -EINVAL;
 
-	channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
-					     num_dcts_intlv, dct_sel);
+	if (pvt->model >= 0x60)
+		channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
+	else
+		channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
+						     num_dcts_intlv, dct_sel);
 
 	/* Verify we stay within the MAX number of channels allowed */
 	if (channel > 3)
@@ -2961,6 +2970,15 @@ static void setup_pci_device(void)
 	}
 }
 
+static const struct x86_cpu_id amd64_cpuids[] = {
+	{ X86_VENDOR_AMD, 0xF,	X86_MODEL_ANY,	X86_FEATURE_ANY, 0 },
+	{ X86_VENDOR_AMD, 0x10, X86_MODEL_ANY,	X86_FEATURE_ANY, 0 },
+	{ X86_VENDOR_AMD, 0x15, X86_MODEL_ANY,	X86_FEATURE_ANY, 0 },
+	{ X86_VENDOR_AMD, 0x16, X86_MODEL_ANY,	X86_FEATURE_ANY, 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
+
 static int __init amd64_edac_init(void)
 {
 	int err = -ENODEV;
diff --git a/drivers/edac/fsl_ddr_edac.c b/drivers/edac/fsl_ddr_edac.c
new file mode 100644
index 000000000000..9774f52f0c3e
--- /dev/null
+++ b/drivers/edac/fsl_ddr_edac.c
@@ -0,0 +1,633 @@
+/*
+ * Freescale Memory Controller kernel module
+ *
+ * Support Power-based SoCs including MPC85xx, MPC86xx, MPC83xx and
+ * ARM-based Layerscape SoCs including LS2xxx. Originally split
+ * out from mpc85xx_edac EDAC driver.
+ *
+ * Parts Copyrighted (c) 2013 by Freescale Semiconductor, Inc.
+ *
+ * Author: Dave Jiang <djiang@mvista.com>
+ *
+ * 2006-2007 (c) MontaVista Software, Inc. This file is licensed under
+ * the terms of the GNU General Public License version 2. This program
+ * is licensed "as is" without any warranty of any kind, whether express
+ * or implied.
+ */
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/ctype.h>
+#include <linux/io.h>
+#include <linux/mod_devicetable.h>
+#include <linux/edac.h>
+#include <linux/smp.h>
+#include <linux/gfp.h>
+
+#include <linux/of_platform.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include "edac_module.h"
+#include "edac_core.h"
+#include "fsl_ddr_edac.h"
+
+#define EDAC_MOD_STR	"fsl_ddr_edac"
+
+static int edac_mc_idx;
+
+static u32 orig_ddr_err_disable;
+static u32 orig_ddr_err_sbe;
+static bool little_endian;
+
+static inline u32 ddr_in32(void __iomem *addr)
+{
+	return little_endian ? ioread32(addr) : ioread32be(addr);
+}
+
+static inline void ddr_out32(void __iomem *addr, u32 value)
+{
+	if (little_endian)
+		iowrite32(value, addr);
+	else
+		iowrite32be(value, addr);
+}
+
+/************************ MC SYSFS parts ***********************************/
+
+#define to_mci(k) container_of(k, struct mem_ctl_info, dev)
+
+static ssize_t fsl_mc_inject_data_hi_show(struct device *dev,
+					  struct device_attribute *mattr,
+					  char *data)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+	return sprintf(data, "0x%08x",
+		       ddr_in32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_HI));
+}
+
+static ssize_t fsl_mc_inject_data_lo_show(struct device *dev,
+					  struct device_attribute *mattr,
+					      char *data)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+	return sprintf(data, "0x%08x",
+		       ddr_in32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_LO));
+}
+
+static ssize_t fsl_mc_inject_ctrl_show(struct device *dev,
+				       struct device_attribute *mattr,
+					   char *data)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+	return sprintf(data, "0x%08x",
+		       ddr_in32(pdata->mc_vbase + FSL_MC_ECC_ERR_INJECT));
+}
+
+static ssize_t fsl_mc_inject_data_hi_store(struct device *dev,
+					   struct device_attribute *mattr,
+					       const char *data, size_t count)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+	unsigned long val;
+	int rc;
+
+	if (isdigit(*data)) {
+		rc = kstrtoul(data, 0, &val);
+		if (rc)
+			return rc;
+
+		ddr_out32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_HI, val);
+		return count;
+	}
+	return 0;
+}
+
+static ssize_t fsl_mc_inject_data_lo_store(struct device *dev,
+					   struct device_attribute *mattr,
+					       const char *data, size_t count)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+	unsigned long val;
+	int rc;
+
+	if (isdigit(*data)) {
+		rc = kstrtoul(data, 0, &val);
+		if (rc)
+			return rc;
+
+		ddr_out32(pdata->mc_vbase + FSL_MC_DATA_ERR_INJECT_LO, val);
+		return count;
+	}
+	return 0;
+}
+
+static ssize_t fsl_mc_inject_ctrl_store(struct device *dev,
+					struct device_attribute *mattr,
+					       const char *data, size_t count)
+{
+	struct mem_ctl_info *mci = to_mci(dev);
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+	unsigned long val;
+	int rc;
+
+	if (isdigit(*data)) {
+		rc = kstrtoul(data, 0, &val);
+		if (rc)
+			return rc;
+
+		ddr_out32(pdata->mc_vbase + FSL_MC_ECC_ERR_INJECT, val);
+		return count;
+	}
+	return 0;
+}
+
+DEVICE_ATTR(inject_data_hi, S_IRUGO | S_IWUSR,
+	    fsl_mc_inject_data_hi_show, fsl_mc_inject_data_hi_store);
+DEVICE_ATTR(inject_data_lo, S_IRUGO | S_IWUSR,
+	    fsl_mc_inject_data_lo_show, fsl_mc_inject_data_lo_store);
+DEVICE_ATTR(inject_ctrl, S_IRUGO | S_IWUSR,
+	    fsl_mc_inject_ctrl_show, fsl_mc_inject_ctrl_store);
+
+static struct attribute *fsl_ddr_dev_attrs[] = {
+	&dev_attr_inject_data_hi.attr,
+	&dev_attr_inject_data_lo.attr,
+	&dev_attr_inject_ctrl.attr,
+	NULL
+};
+
+ATTRIBUTE_GROUPS(fsl_ddr_dev);
+
+/**************************** MC Err device ***************************/
+
+/*
+ * Taken from table 8-55 in the MPC8641 User's Manual and/or 9-61 in the
+ * MPC8572 User's Manual.  Each line represents a syndrome bit column as a
+ * 64-bit value, but split into an upper and lower 32-bit chunk.  The labels
+ * below correspond to Freescale's manuals.
+ */
+static unsigned int ecc_table[16] = {
+	/* MSB           LSB */
+	/* [0:31]    [32:63] */
+	0xf00fe11e, 0xc33c0ff7,	/* Syndrome bit 7 */
+	0x00ff00ff, 0x00fff0ff,
+	0x0f0f0f0f, 0x0f0fff00,
+	0x11113333, 0x7777000f,
+	0x22224444, 0x8888222f,
+	0x44448888, 0xffff4441,
+	0x8888ffff, 0x11118882,
+	0xffff1111, 0x22221114,	/* Syndrome bit 0 */
+};
+
+/*
+ * Calculate the correct ECC value for a 64-bit value specified by high:low
+ */
+static u8 calculate_ecc(u32 high, u32 low)
+{
+	u32 mask_low;
+	u32 mask_high;
+	int bit_cnt;
+	u8 ecc = 0;
+	int i;
+	int j;
+
+	for (i = 0; i < 8; i++) {
+		mask_high = ecc_table[i * 2];
+		mask_low = ecc_table[i * 2 + 1];
+		bit_cnt = 0;
+
+		for (j = 0; j < 32; j++) {
+			if ((mask_high >> j) & 1)
+				bit_cnt ^= (high >> j) & 1;
+			if ((mask_low >> j) & 1)
+				bit_cnt ^= (low >> j) & 1;
+		}
+
+		ecc |= bit_cnt << i;
+	}
+
+	return ecc;
+}
+
+/*
+ * Create the syndrome code which is generated if the data line specified by
+ * 'bit' failed.  Eg generate an 8-bit codes seen in Table 8-55 in the MPC8641
+ * User's Manual and 9-61 in the MPC8572 User's Manual.
+ */
+static u8 syndrome_from_bit(unsigned int bit) {
+	int i;
+	u8 syndrome = 0;
+
+	/*
+	 * Cycle through the upper or lower 32-bit portion of each value in
+	 * ecc_table depending on if 'bit' is in the upper or lower half of
+	 * 64-bit data.
+	 */
+	for (i = bit < 32; i < 16; i += 2)
+		syndrome |= ((ecc_table[i] >> (bit % 32)) & 1) << (i / 2);
+
+	return syndrome;
+}
+
+/*
+ * Decode data and ecc syndrome to determine what went wrong
+ * Note: This can only decode single-bit errors
+ */
+static void sbe_ecc_decode(u32 cap_high, u32 cap_low, u32 cap_ecc,
+		       int *bad_data_bit, int *bad_ecc_bit)
+{
+	int i;
+	u8 syndrome;
+
+	*bad_data_bit = -1;
+	*bad_ecc_bit = -1;
+
+	/*
+	 * Calculate the ECC of the captured data and XOR it with the captured
+	 * ECC to find an ECC syndrome value we can search for
+	 */
+	syndrome = calculate_ecc(cap_high, cap_low) ^ cap_ecc;
+
+	/* Check if a data line is stuck... */
+	for (i = 0; i < 64; i++) {
+		if (syndrome == syndrome_from_bit(i)) {
+			*bad_data_bit = i;
+			return;
+		}
+	}
+
+	/* If data is correct, check ECC bits for errors... */
+	for (i = 0; i < 8; i++) {
+		if ((syndrome >> i) & 0x1) {
+			*bad_ecc_bit = i;
+			return;
+		}
+	}
+}
+
+#define make64(high, low) (((u64)(high) << 32) | (low))
+
+static void fsl_mc_check(struct mem_ctl_info *mci)
+{
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+	struct csrow_info *csrow;
+	u32 bus_width;
+	u32 err_detect;
+	u32 syndrome;
+	u64 err_addr;
+	u32 pfn;
+	int row_index;
+	u32 cap_high;
+	u32 cap_low;
+	int bad_data_bit;
+	int bad_ecc_bit;
+
+	err_detect = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DETECT);
+	if (!err_detect)
+		return;
+
+	fsl_mc_printk(mci, KERN_ERR, "Err Detect Register: %#8.8x\n",
+		      err_detect);
+
+	/* no more processing if not ECC bit errors */
+	if (!(err_detect & (DDR_EDE_SBE | DDR_EDE_MBE))) {
+		ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, err_detect);
+		return;
+	}
+
+	syndrome = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_ECC);
+
+	/* Mask off appropriate bits of syndrome based on bus width */
+	bus_width = (ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG) &
+		     DSC_DBW_MASK) ? 32 : 64;
+	if (bus_width == 64)
+		syndrome &= 0xff;
+	else
+		syndrome &= 0xffff;
+
+	err_addr = make64(
+		ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_EXT_ADDRESS),
+		ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_ADDRESS));
+	pfn = err_addr >> PAGE_SHIFT;
+
+	for (row_index = 0; row_index < mci->nr_csrows; row_index++) {
+		csrow = mci->csrows[row_index];
+		if ((pfn >= csrow->first_page) && (pfn <= csrow->last_page))
+			break;
+	}
+
+	cap_high = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_DATA_HI);
+	cap_low = ddr_in32(pdata->mc_vbase + FSL_MC_CAPTURE_DATA_LO);
+
+	/*
+	 * Analyze single-bit errors on 64-bit wide buses
+	 * TODO: Add support for 32-bit wide buses
+	 */
+	if ((err_detect & DDR_EDE_SBE) && (bus_width == 64)) {
+		sbe_ecc_decode(cap_high, cap_low, syndrome,
+				&bad_data_bit, &bad_ecc_bit);
+
+		if (bad_data_bit != -1)
+			fsl_mc_printk(mci, KERN_ERR,
+				"Faulty Data bit: %d\n", bad_data_bit);
+		if (bad_ecc_bit != -1)
+			fsl_mc_printk(mci, KERN_ERR,
+				"Faulty ECC bit: %d\n", bad_ecc_bit);
+
+		fsl_mc_printk(mci, KERN_ERR,
+			"Expected Data / ECC:\t%#8.8x_%08x / %#2.2x\n",
+			cap_high ^ (1 << (bad_data_bit - 32)),
+			cap_low ^ (1 << bad_data_bit),
+			syndrome ^ (1 << bad_ecc_bit));
+	}
+
+	fsl_mc_printk(mci, KERN_ERR,
+			"Captured Data / ECC:\t%#8.8x_%08x / %#2.2x\n",
+			cap_high, cap_low, syndrome);
+	fsl_mc_printk(mci, KERN_ERR, "Err addr: %#8.8llx\n", err_addr);
+	fsl_mc_printk(mci, KERN_ERR, "PFN: %#8.8x\n", pfn);
+
+	/* we are out of range */
+	if (row_index == mci->nr_csrows)
+		fsl_mc_printk(mci, KERN_ERR, "PFN out of range!\n");
+
+	if (err_detect & DDR_EDE_SBE)
+		edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
+				     pfn, err_addr & ~PAGE_MASK, syndrome,
+				     row_index, 0, -1,
+				     mci->ctl_name, "");
+
+	if (err_detect & DDR_EDE_MBE)
+		edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
+				     pfn, err_addr & ~PAGE_MASK, syndrome,
+				     row_index, 0, -1,
+				     mci->ctl_name, "");
+
+	ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, err_detect);
+}
+
+static irqreturn_t fsl_mc_isr(int irq, void *dev_id)
+{
+	struct mem_ctl_info *mci = dev_id;
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+	u32 err_detect;
+
+	err_detect = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DETECT);
+	if (!err_detect)
+		return IRQ_NONE;
+
+	fsl_mc_check(mci);
+
+	return IRQ_HANDLED;
+}
+
+static void fsl_ddr_init_csrows(struct mem_ctl_info *mci)
+{
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+	struct csrow_info *csrow;
+	struct dimm_info *dimm;
+	u32 sdram_ctl;
+	u32 sdtype;
+	enum mem_type mtype;
+	u32 cs_bnds;
+	int index;
+
+	sdram_ctl = ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG);
+
+	sdtype = sdram_ctl & DSC_SDTYPE_MASK;
+	if (sdram_ctl & DSC_RD_EN) {
+		switch (sdtype) {
+		case 0x02000000:
+			mtype = MEM_RDDR;
+			break;
+		case 0x03000000:
+			mtype = MEM_RDDR2;
+			break;
+		case 0x07000000:
+			mtype = MEM_RDDR3;
+			break;
+		case 0x05000000:
+			mtype = MEM_RDDR4;
+			break;
+		default:
+			mtype = MEM_UNKNOWN;
+			break;
+		}
+	} else {
+		switch (sdtype) {
+		case 0x02000000:
+			mtype = MEM_DDR;
+			break;
+		case 0x03000000:
+			mtype = MEM_DDR2;
+			break;
+		case 0x07000000:
+			mtype = MEM_DDR3;
+			break;
+		case 0x05000000:
+			mtype = MEM_DDR4;
+			break;
+		default:
+			mtype = MEM_UNKNOWN;
+			break;
+		}
+	}
+
+	for (index = 0; index < mci->nr_csrows; index++) {
+		u32 start;
+		u32 end;
+
+		csrow = mci->csrows[index];
+		dimm = csrow->channels[0]->dimm;
+
+		cs_bnds = ddr_in32(pdata->mc_vbase + FSL_MC_CS_BNDS_0 +
+				   (index * FSL_MC_CS_BNDS_OFS));
+
+		start = (cs_bnds & 0xffff0000) >> 16;
+		end   = (cs_bnds & 0x0000ffff);
+
+		if (start == end)
+			continue;	/* not populated */
+
+		start <<= (24 - PAGE_SHIFT);
+		end   <<= (24 - PAGE_SHIFT);
+		end    |= (1 << (24 - PAGE_SHIFT)) - 1;
+
+		csrow->first_page = start;
+		csrow->last_page = end;
+
+		dimm->nr_pages = end + 1 - start;
+		dimm->grain = 8;
+		dimm->mtype = mtype;
+		dimm->dtype = DEV_UNKNOWN;
+		if (sdram_ctl & DSC_X32_EN)
+			dimm->dtype = DEV_X32;
+		dimm->edac_mode = EDAC_SECDED;
+	}
+}
+
+int fsl_mc_err_probe(struct platform_device *op)
+{
+	struct mem_ctl_info *mci;
+	struct edac_mc_layer layers[2];
+	struct fsl_mc_pdata *pdata;
+	struct resource r;
+	u32 sdram_ctl;
+	int res;
+
+	if (!devres_open_group(&op->dev, fsl_mc_err_probe, GFP_KERNEL))
+		return -ENOMEM;
+
+	layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
+	layers[0].size = 4;
+	layers[0].is_virt_csrow = true;
+	layers[1].type = EDAC_MC_LAYER_CHANNEL;
+	layers[1].size = 1;
+	layers[1].is_virt_csrow = false;
+	mci = edac_mc_alloc(edac_mc_idx, ARRAY_SIZE(layers), layers,
+			    sizeof(*pdata));
+	if (!mci) {
+		devres_release_group(&op->dev, fsl_mc_err_probe);
+		return -ENOMEM;
+	}
+
+	pdata = mci->pvt_info;
+	pdata->name = "fsl_mc_err";
+	mci->pdev = &op->dev;
+	pdata->edac_idx = edac_mc_idx++;
+	dev_set_drvdata(mci->pdev, mci);
+	mci->ctl_name = pdata->name;
+	mci->dev_name = pdata->name;
+
+	/*
+	 * Get the endianness of DDR controller registers.
+	 * Default is big endian.
+	 */
+	little_endian = of_property_read_bool(op->dev.of_node, "little-endian");
+
+	res = of_address_to_resource(op->dev.of_node, 0, &r);
+	if (res) {
+		pr_err("%s: Unable to get resource for MC err regs\n",
+		       __func__);
+		goto err;
+	}
+
+	if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r),
+				     pdata->name)) {
+		pr_err("%s: Error while requesting mem region\n",
+		       __func__);
+		res = -EBUSY;
+		goto err;
+	}
+
+	pdata->mc_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r));
+	if (!pdata->mc_vbase) {
+		pr_err("%s: Unable to setup MC err regs\n", __func__);
+		res = -ENOMEM;
+		goto err;
+	}
+
+	sdram_ctl = ddr_in32(pdata->mc_vbase + FSL_MC_DDR_SDRAM_CFG);
+	if (!(sdram_ctl & DSC_ECC_EN)) {
+		/* no ECC */
+		pr_warn("%s: No ECC DIMMs discovered\n", __func__);
+		res = -ENODEV;
+		goto err;
+	}
+
+	edac_dbg(3, "init mci\n");
+	mci->mtype_cap = MEM_FLAG_DDR | MEM_FLAG_RDDR |
+			 MEM_FLAG_DDR2 | MEM_FLAG_RDDR2 |
+			 MEM_FLAG_DDR3 | MEM_FLAG_RDDR3 |
+			 MEM_FLAG_DDR4 | MEM_FLAG_RDDR4;
+	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
+	mci->edac_cap = EDAC_FLAG_SECDED;
+	mci->mod_name = EDAC_MOD_STR;
+
+	if (edac_op_state == EDAC_OPSTATE_POLL)
+		mci->edac_check = fsl_mc_check;
+
+	mci->ctl_page_to_phys = NULL;
+
+	mci->scrub_mode = SCRUB_SW_SRC;
+
+	fsl_ddr_init_csrows(mci);
+
+	/* store the original error disable bits */
+	orig_ddr_err_disable = ddr_in32(pdata->mc_vbase + FSL_MC_ERR_DISABLE);
+	ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DISABLE, 0);
+
+	/* clear all error bits */
+	ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DETECT, ~0);
+
+	res = edac_mc_add_mc_with_groups(mci, fsl_ddr_dev_groups);
+	if (res) {
+		edac_dbg(3, "failed edac_mc_add_mc()\n");
+		goto err;
+	}
+
+	if (edac_op_state == EDAC_OPSTATE_INT) {
+		ddr_out32(pdata->mc_vbase + FSL_MC_ERR_INT_EN,
+			  DDR_EIE_MBEE | DDR_EIE_SBEE);
+
+		/* store the original error management threshold */
+		orig_ddr_err_sbe = ddr_in32(pdata->mc_vbase +
+					    FSL_MC_ERR_SBE) & 0xff0000;
+
+		/* set threshold to 1 error per interrupt */
+		ddr_out32(pdata->mc_vbase + FSL_MC_ERR_SBE, 0x10000);
+
+		/* register interrupts */
+		pdata->irq = platform_get_irq(op, 0);
+		res = devm_request_irq(&op->dev, pdata->irq,
+				       fsl_mc_isr,
+				       IRQF_SHARED,
+				       "[EDAC] MC err", mci);
+		if (res < 0) {
+			pr_err("%s: Unable to request irq %d for FSL DDR DRAM ERR\n",
+			       __func__, pdata->irq);
+			res = -ENODEV;
+			goto err2;
+		}
+
+		pr_info(EDAC_MOD_STR " acquired irq %d for MC\n",
+		       pdata->irq);
+	}
+
+	devres_remove_group(&op->dev, fsl_mc_err_probe);
+	edac_dbg(3, "success\n");
+	pr_info(EDAC_MOD_STR " MC err registered\n");
+
+	return 0;
+
+err2:
+	edac_mc_del_mc(&op->dev);
+err:
+	devres_release_group(&op->dev, fsl_mc_err_probe);
+	edac_mc_free(mci);
+	return res;
+}
+
+int fsl_mc_err_remove(struct platform_device *op)
+{
+	struct mem_ctl_info *mci = dev_get_drvdata(&op->dev);
+	struct fsl_mc_pdata *pdata = mci->pvt_info;
+
+	edac_dbg(0, "\n");
+
+	if (edac_op_state == EDAC_OPSTATE_INT) {
+		ddr_out32(pdata->mc_vbase + FSL_MC_ERR_INT_EN, 0);
+	}
+
+	ddr_out32(pdata->mc_vbase + FSL_MC_ERR_DISABLE,
+		  orig_ddr_err_disable);
+	ddr_out32(pdata->mc_vbase + FSL_MC_ERR_SBE, orig_ddr_err_sbe);
+
+	edac_mc_del_mc(&op->dev);
+	edac_mc_free(mci);
+	return 0;
+}
diff --git a/drivers/edac/fsl_ddr_edac.h b/drivers/edac/fsl_ddr_edac.h
new file mode 100644
index 000000000000..4ccee292eff1
--- /dev/null
+++ b/drivers/edac/fsl_ddr_edac.h
@@ -0,0 +1,79 @@
+/*
+ * Freescale Memory Controller kernel module
+ *
+ * Support  Power-based SoCs including MPC85xx, MPC86xx, MPC83xx and
+ * ARM-based Layerscape SoCs including LS2xxx. Originally split
+ * out from mpc85xx_edac EDAC driver.
+ *
+ * Author: Dave Jiang <djiang@mvista.com>
+ *
+ * 2006-2007 (c) MontaVista Software, Inc. This file is licensed under
+ * the terms of the GNU General Public License version 2. This program
+ * is licensed "as is" without any warranty of any kind, whether express
+ * or implied.
+ *
+ */
+#ifndef _FSL_DDR_EDAC_H_
+#define _FSL_DDR_EDAC_H_
+
+#define fsl_mc_printk(mci, level, fmt, arg...) \
+	edac_mc_chipset_printk(mci, level, "FSL_DDR", fmt, ##arg)
+
+/*
+ * DRAM error defines
+ */
+
+/* DDR_SDRAM_CFG */
+#define FSL_MC_DDR_SDRAM_CFG	0x0110
+#define FSL_MC_CS_BNDS_0		0x0000
+#define FSL_MC_CS_BNDS_OFS		0x0008
+
+#define FSL_MC_DATA_ERR_INJECT_HI	0x0e00
+#define FSL_MC_DATA_ERR_INJECT_LO	0x0e04
+#define FSL_MC_ECC_ERR_INJECT	0x0e08
+#define FSL_MC_CAPTURE_DATA_HI	0x0e20
+#define FSL_MC_CAPTURE_DATA_LO	0x0e24
+#define FSL_MC_CAPTURE_ECC		0x0e28
+#define FSL_MC_ERR_DETECT		0x0e40
+#define FSL_MC_ERR_DISABLE		0x0e44
+#define FSL_MC_ERR_INT_EN		0x0e48
+#define FSL_MC_CAPTURE_ATRIBUTES	0x0e4c
+#define FSL_MC_CAPTURE_ADDRESS	0x0e50
+#define FSL_MC_CAPTURE_EXT_ADDRESS	0x0e54
+#define FSL_MC_ERR_SBE		0x0e58
+
+#define DSC_MEM_EN	0x80000000
+#define DSC_ECC_EN	0x20000000
+#define DSC_RD_EN	0x10000000
+#define DSC_DBW_MASK	0x00180000
+#define DSC_DBW_32	0x00080000
+#define DSC_DBW_64	0x00000000
+
+#define DSC_SDTYPE_MASK		0x07000000
+#define DSC_X32_EN	0x00000020
+
+/* Err_Int_En */
+#define DDR_EIE_MSEE	0x1	/* memory select */
+#define DDR_EIE_SBEE	0x4	/* single-bit ECC error */
+#define DDR_EIE_MBEE	0x8	/* multi-bit ECC error */
+
+/* Err_Detect */
+#define DDR_EDE_MSE		0x1	/* memory select */
+#define DDR_EDE_SBE		0x4	/* single-bit ECC error */
+#define DDR_EDE_MBE		0x8	/* multi-bit ECC error */
+#define DDR_EDE_MME		0x80000000	/* multiple memory errors */
+
+/* Err_Disable */
+#define DDR_EDI_MSED	0x1	/* memory select disable */
+#define	DDR_EDI_SBED	0x4	/* single-bit ECC error disable */
+#define	DDR_EDI_MBED	0x8	/* multi-bit ECC error disable */
+
+struct fsl_mc_pdata {
+	char *name;
+	int edac_idx;
+	void __iomem *mc_vbase;
+	int irq;
+};
+int fsl_mc_err_probe(struct platform_device *op);
+int fsl_mc_err_remove(struct platform_device *op);
+#endif
diff --git a/drivers/edac/layerscape_edac.c b/drivers/edac/layerscape_edac.c
new file mode 100644
index 000000000000..6c59d897ad12
--- /dev/null
+++ b/drivers/edac/layerscape_edac.c
@@ -0,0 +1,73 @@
+/*
+ * Freescale Memory Controller kernel module
+ *
+ * Author: York Sun <york.sun@nxp.com>
+ *
+ * Copyright 2016 NXP Semiconductor
+ *
+ * Derived from mpc85xx_edac.c
+ * Author: Dave Jiang <djiang@mvista.com>
+ *
+ * 2006-2007 (c) MontaVista Software, Inc. This file is licensed under
+ * the terms of the GNU General Public License version 2. This program
+ * is licensed "as is" without any warranty of any kind, whether express
+ * or implied.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include "edac_core.h"
+#include "fsl_ddr_edac.h"
+
+static const struct of_device_id fsl_ddr_mc_err_of_match[] = {
+	{ .compatible = "fsl,qoriq-memory-controller", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, fsl_ddr_mc_err_of_match);
+
+static struct platform_driver fsl_ddr_mc_err_driver = {
+	.probe = fsl_mc_err_probe,
+	.remove = fsl_mc_err_remove,
+	.driver = {
+		.name = "fsl_ddr_mc_err",
+		.of_match_table = fsl_ddr_mc_err_of_match,
+	},
+};
+
+static int __init fsl_ddr_mc_init(void)
+{
+	int res;
+
+	/* make sure error reporting method is sane */
+	switch (edac_op_state) {
+	case EDAC_OPSTATE_POLL:
+	case EDAC_OPSTATE_INT:
+		break;
+	default:
+		edac_op_state = EDAC_OPSTATE_INT;
+		break;
+	}
+
+	res = platform_driver_register(&fsl_ddr_mc_err_driver);
+	if (res) {
+		pr_err("MC fails to register\n");
+		return res;
+	}
+
+	return 0;
+}
+
+module_init(fsl_ddr_mc_init);
+
+static void __exit fsl_ddr_mc_exit(void)
+{
+	platform_driver_unregister(&fsl_ddr_mc_err_driver);
+}
+
+module_exit(fsl_ddr_mc_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("NXP Semiconductor");
+module_param(edac_op_state, int, 0444);
+MODULE_PARM_DESC(edac_op_state,
+		 "EDAC Error Reporting state: 0=Poll, 2=Interrupt");
diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c
index 9b6800a79c7f..daaac2c79ca7 100644
--- a/drivers/edac/mce_amd.c
+++ b/drivers/edac/mce_amd.c
@@ -148,12 +148,12 @@ static const char * const mc6_mce_desc[] = {
 };
 
 /* Scalable MCA error strings */
-static const char * const f17h_ls_mce_desc[] = {
+static const char * const smca_ls_mce_desc[] = {
 	"Load queue parity",
 	"Store queue parity",
 	"Miss address buffer payload parity",
 	"L1 TLB parity",
-	"",						/* reserved */
+	"Reserved",
 	"DC tag error type 6",
 	"DC tag error type 1",
 	"Internal error type 1",
@@ -172,7 +172,7 @@ static const char * const f17h_ls_mce_desc[] = {
 	"L2 fill data error",
 };
 
-static const char * const f17h_if_mce_desc[] = {
+static const char * const smca_if_mce_desc[] = {
 	"microtag probe port parity error",
 	"IC microtag or full tag multi-hit error",
 	"IC full tag parity",
@@ -185,19 +185,22 @@ static const char * const f17h_if_mce_desc[] = {
 	"BPQ snoop parity on Thread 1",
 	"L1 BTB multi-match error",
 	"L2 BTB multi-match error",
+	"L2 Cache Response Poison error",
+	"System Read Data error",
 };
 
-static const char * const f17h_l2_mce_desc[] = {
+static const char * const smca_l2_mce_desc[] = {
 	"L2M tag multi-way-hit error",
 	"L2M tag ECC error",
 	"L2M data ECC error",
 	"HW assert",
 };
 
-static const char * const f17h_de_mce_desc[] = {
+static const char * const smca_de_mce_desc[] = {
 	"uop cache tag parity error",
 	"uop cache data parity error",
 	"Insn buffer parity error",
+	"uop queue parity error",
 	"Insn dispatch queue parity error",
 	"Fetch address FIFO parity",
 	"Patch RAM data parity",
@@ -205,7 +208,7 @@ static const char * const f17h_de_mce_desc[] = {
 	"uop buffer parity"
 };
 
-static const char * const f17h_ex_mce_desc[] = {
+static const char * const smca_ex_mce_desc[] = {
 	"Watchdog timeout error",
 	"Phy register file parity",
 	"Flag register file parity",
@@ -214,18 +217,22 @@ static const char * const f17h_ex_mce_desc[] = {
 	"EX payload parity",
 	"Checkpoint queue parity",
 	"Retire dispatch queue parity",
+	"Retire status queue parity error",
+	"Scheduling queue parity error",
+	"Branch buffer queue parity error",
 };
 
-static const char * const f17h_fp_mce_desc[] = {
+static const char * const smca_fp_mce_desc[] = {
 	"Physical register file parity",
 	"Freelist parity error",
 	"Schedule queue parity",
 	"NSQ parity error",
 	"Retire queue parity",
 	"Status register file parity",
+	"Hardware assertion",
 };
 
-static const char * const f17h_l3_mce_desc[] = {
+static const char * const smca_l3_mce_desc[] = {
 	"Shadow tag macro ECC error",
 	"Shadow tag macro multi-way-hit error",
 	"L3M tag ECC error",
@@ -236,7 +243,7 @@ static const char * const f17h_l3_mce_desc[] = {
 	"L3 HW assert",
 };
 
-static const char * const f17h_cs_mce_desc[] = {
+static const char * const smca_cs_mce_desc[] = {
 	"Illegal request from transport layer",
 	"Address violation",
 	"Security violation",
@@ -248,14 +255,14 @@ static const char * const f17h_cs_mce_desc[] = {
 	"ECC error on probe filter access",
 };
 
-static const char * const f17h_pie_mce_desc[] = {
+static const char * const smca_pie_mce_desc[] = {
 	"HW assert",
 	"Internal PIE register security violation",
 	"Error on GMI link",
 	"Poison data written to internal PIE register",
 };
 
-static const char * const f17h_umc_mce_desc[] = {
+static const char * const smca_umc_mce_desc[] = {
 	"DRAM ECC error",
 	"Data poison error on DRAM",
 	"SDP parity error",
@@ -264,18 +271,39 @@ static const char * const f17h_umc_mce_desc[] = {
 	"Write data CRC error",
 };
 
-static const char * const f17h_pb_mce_desc[] = {
+static const char * const smca_pb_mce_desc[] = {
 	"Parameter Block RAM ECC error",
 };
 
-static const char * const f17h_psp_mce_desc[] = {
+static const char * const smca_psp_mce_desc[] = {
 	"PSP RAM ECC or parity error",
 };
 
-static const char * const f17h_smu_mce_desc[] = {
+static const char * const smca_smu_mce_desc[] = {
 	"SMU RAM ECC or parity error",
 };
 
+struct smca_mce_desc {
+	const char * const *descs;
+	unsigned int num_descs;
+};
+
+static struct smca_mce_desc smca_mce_descs[] = {
+	[SMCA_LS]	= { smca_ls_mce_desc,	ARRAY_SIZE(smca_ls_mce_desc)	},
+	[SMCA_IF]	= { smca_if_mce_desc,	ARRAY_SIZE(smca_if_mce_desc)	},
+	[SMCA_L2_CACHE]	= { smca_l2_mce_desc,	ARRAY_SIZE(smca_l2_mce_desc)	},
+	[SMCA_DE]	= { smca_de_mce_desc,	ARRAY_SIZE(smca_de_mce_desc)	},
+	[SMCA_EX]	= { smca_ex_mce_desc,	ARRAY_SIZE(smca_ex_mce_desc)	},
+	[SMCA_FP]	= { smca_fp_mce_desc,	ARRAY_SIZE(smca_fp_mce_desc)	},
+	[SMCA_L3_CACHE]	= { smca_l3_mce_desc,	ARRAY_SIZE(smca_l3_mce_desc)	},
+	[SMCA_CS]	= { smca_cs_mce_desc,	ARRAY_SIZE(smca_cs_mce_desc)	},
+	[SMCA_PIE]	= { smca_pie_mce_desc,	ARRAY_SIZE(smca_pie_mce_desc)	},
+	[SMCA_UMC]	= { smca_umc_mce_desc,	ARRAY_SIZE(smca_umc_mce_desc)	},
+	[SMCA_PB]	= { smca_pb_mce_desc,	ARRAY_SIZE(smca_pb_mce_desc)	},
+	[SMCA_PSP]	= { smca_psp_mce_desc,	ARRAY_SIZE(smca_psp_mce_desc)	},
+	[SMCA_SMU]	= { smca_smu_mce_desc,	ARRAY_SIZE(smca_smu_mce_desc)	},
+};
+
 static bool f12h_mc0_mce(u16 ec, u8 xec)
 {
 	bool ret = false;
@@ -820,175 +848,35 @@ static void decode_mc6_mce(struct mce *m)
 	pr_emerg(HW_ERR "Corrupted MC6 MCE info?\n");
 }
 
-static void decode_f17h_core_errors(const char *ip_name, u8 xec,
-				   unsigned int mca_type)
-{
-	const char * const *error_desc_array;
-	size_t len;
-
-	pr_emerg(HW_ERR "%s Error: ", ip_name);
-
-	switch (mca_type) {
-	case SMCA_LS:
-		error_desc_array = f17h_ls_mce_desc;
-		len = ARRAY_SIZE(f17h_ls_mce_desc) - 1;
-
-		if (xec == 0x4) {
-			pr_cont("Unrecognized LS MCA error code.\n");
-			return;
-		}
-		break;
-
-	case SMCA_IF:
-		error_desc_array = f17h_if_mce_desc;
-		len = ARRAY_SIZE(f17h_if_mce_desc) - 1;
-		break;
-
-	case SMCA_L2_CACHE:
-		error_desc_array = f17h_l2_mce_desc;
-		len = ARRAY_SIZE(f17h_l2_mce_desc) - 1;
-		break;
-
-	case SMCA_DE:
-		error_desc_array = f17h_de_mce_desc;
-		len = ARRAY_SIZE(f17h_de_mce_desc) - 1;
-		break;
-
-	case SMCA_EX:
-		error_desc_array = f17h_ex_mce_desc;
-		len = ARRAY_SIZE(f17h_ex_mce_desc) - 1;
-		break;
-
-	case SMCA_FP:
-		error_desc_array = f17h_fp_mce_desc;
-		len = ARRAY_SIZE(f17h_fp_mce_desc) - 1;
-		break;
-
-	case SMCA_L3_CACHE:
-		error_desc_array = f17h_l3_mce_desc;
-		len = ARRAY_SIZE(f17h_l3_mce_desc) - 1;
-		break;
-
-	default:
-		pr_cont("Corrupted MCA core error info.\n");
-		return;
-	}
-
-	if (xec > len) {
-		pr_cont("Unrecognized %s MCA bank error code.\n",
-			 amd_core_mcablock_names[mca_type]);
-		return;
-	}
-
-	pr_cont("%s.\n", error_desc_array[xec]);
-}
-
-static void decode_df_errors(u8 xec, unsigned int mca_type)
-{
-	const char * const *error_desc_array;
-	size_t len;
-
-	pr_emerg(HW_ERR "Data Fabric Error: ");
-
-	switch (mca_type) {
-	case  SMCA_CS:
-		error_desc_array = f17h_cs_mce_desc;
-		len = ARRAY_SIZE(f17h_cs_mce_desc) - 1;
-		break;
-
-	case SMCA_PIE:
-		error_desc_array = f17h_pie_mce_desc;
-		len = ARRAY_SIZE(f17h_pie_mce_desc) - 1;
-		break;
-
-	default:
-		pr_cont("Corrupted MCA Data Fabric info.\n");
-		return;
-	}
-
-	if (xec > len) {
-		pr_cont("Unrecognized %s MCA bank error code.\n",
-			 amd_df_mcablock_names[mca_type]);
-		return;
-	}
-
-	pr_cont("%s.\n", error_desc_array[xec]);
-}
-
 /* Decode errors according to Scalable MCA specification */
 static void decode_smca_errors(struct mce *m)
 {
-	u32 addr = MSR_AMD64_SMCA_MCx_IPID(m->bank);
-	unsigned int hwid, mca_type, i;
-	u8 xec = XEC(m->status, xec_mask);
-	const char * const *error_desc_array;
+	struct smca_hwid_mcatype *type;
+	unsigned int bank_type;
 	const char *ip_name;
-	u32 low, high;
-	size_t len;
+	u8 xec = XEC(m->status, xec_mask);
 
-	if (rdmsr_safe(addr, &low, &high)) {
-		pr_emerg("Invalid IP block specified, error information is unreliable.\n");
+	if (m->bank >= ARRAY_SIZE(smca_banks))
 		return;
-	}
-
-	hwid = high & MCI_IPID_HWID;
-	mca_type = (high & MCI_IPID_MCATYPE) >> 16;
-
-	pr_emerg(HW_ERR "MC%d IPID value: 0x%08x%08x\n", m->bank, high, low);
-
-	/*
-	 * Based on hwid and mca_type values, decode errors from respective IPs.
-	 * Note: mca_type values make sense only in the context of an hwid.
-	 */
-	for (i = 0; i < ARRAY_SIZE(amd_hwids); i++)
-		if (amd_hwids[i].hwid == hwid)
-			break;
-
-	switch (i) {
-	case SMCA_F17H_CORE:
-		ip_name = (mca_type == SMCA_L3_CACHE) ?
-			  "L3 Cache" : "F17h Core";
-		return decode_f17h_core_errors(ip_name, xec, mca_type);
-		break;
 
-	case SMCA_DF:
-		return decode_df_errors(xec, mca_type);
-		break;
-
-	case SMCA_UMC:
-		error_desc_array = f17h_umc_mce_desc;
-		len = ARRAY_SIZE(f17h_umc_mce_desc) - 1;
-		break;
-
-	case SMCA_PB:
-		error_desc_array = f17h_pb_mce_desc;
-		len = ARRAY_SIZE(f17h_pb_mce_desc) - 1;
-		break;
+	if (boot_cpu_data.x86 >= 0x17 && m->bank == 4)
+		pr_emerg(HW_ERR "Bank 4 is reserved on Fam17h.\n");
 
-	case SMCA_PSP:
-		error_desc_array = f17h_psp_mce_desc;
-		len = ARRAY_SIZE(f17h_psp_mce_desc) - 1;
-		break;
-
-	case SMCA_SMU:
-		error_desc_array = f17h_smu_mce_desc;
-		len = ARRAY_SIZE(f17h_smu_mce_desc) - 1;
-		break;
-
-	default:
-		pr_emerg(HW_ERR "HWID:%d does not match any existing IPs.\n", hwid);
+	type = smca_banks[m->bank].type;
+	if (!type)
 		return;
-	}
 
-	ip_name = amd_hwids[i].name;
-	pr_emerg(HW_ERR "%s Error: ", ip_name);
+	bank_type = type->bank_type;
+	ip_name = smca_bank_names[bank_type].long_name;
 
-	if (xec > len) {
-		pr_cont("Unrecognized %s MCA bank error code.\n", ip_name);
-		return;
-	}
+	pr_emerg(HW_ERR "%s Extended Error Code: %d\n", ip_name, xec);
 
-	pr_cont("%s.\n", error_desc_array[xec]);
+	/* Only print the decode of valid error codes */
+	if (xec < smca_mce_descs[bank_type].num_descs &&
+			(type->xec_bitmap & BIT_ULL(xec))) {
+		pr_emerg(HW_ERR "%s Error: ", ip_name);
+		pr_cont("%s.\n", smca_mce_descs[bank_type].descs[xec]);
+	}
 }
 
 static inline void amd_decode_err_code(u16 ec)
@@ -1078,6 +966,8 @@ int amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 		u32 low, high;
 		u32 addr = MSR_AMD64_SMCA_MCx_CONFIG(m->bank);
 
+		pr_cont("|%s", ((m->status & MCI_STATUS_SYNDV) ? "SyndV" : "-"));
+
 		if (!rdmsr_safe(addr, &low, &high) &&
 		    (low & MCI_CONFIG_MCAX))
 			pr_cont("|%s", ((m->status & MCI_STATUS_TCC) ? "TCC" : "-"));
@@ -1091,12 +981,20 @@ int amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 	pr_cont("]: 0x%016llx\n", m->status);
 
 	if (m->status & MCI_STATUS_ADDRV)
-		pr_emerg(HW_ERR "MC%d Error Address: 0x%016llx\n", m->bank, m->addr);
+		pr_emerg(HW_ERR "Error Addr: 0x%016llx", m->addr);
 
 	if (boot_cpu_has(X86_FEATURE_SMCA)) {
+		if (m->status & MCI_STATUS_SYNDV)
+			pr_cont(", Syndrome: 0x%016llx", m->synd);
+
+		pr_cont(", IPID: 0x%016llx", m->ipid);
+
+		pr_cont("\n");
+
 		decode_smca_errors(m);
 		goto err_code;
-	}
+	} else
+		pr_cont("\n");
 
 	if (!fam_ops)
 		goto err_code;
diff --git a/drivers/edac/mpc85xx_edac.c b/drivers/edac/mpc85xx_edac.c
index ca63d0da8889..ff0567526ee3 100644
--- a/drivers/edac/mpc85xx_edac.c
+++ b/drivers/edac/mpc85xx_edac.c
@@ -27,15 +27,12 @@
 #include "edac_module.h"
 #include "edac_core.h"
 #include "mpc85xx_edac.h"
+#include "fsl_ddr_edac.h"
 
 static int edac_dev_idx;
 #ifdef CONFIG_PCI
 static int edac_pci_idx;
 #endif
-static int edac_mc_idx;
-
-static u32 orig_ddr_err_disable;
-static u32 orig_ddr_err_sbe;
 
 /*
  * PCI Err defines
@@ -46,103 +43,6 @@ static u32 orig_pci_err_en;
 #endif
 
 static u32 orig_l2_err_disable;
-#ifdef CONFIG_FSL_SOC_BOOKE
-static u32 orig_hid1[2];
-#endif
-
-/************************ MC SYSFS parts ***********************************/
-
-#define to_mci(k) container_of(k, struct mem_ctl_info, dev)
-
-static ssize_t mpc85xx_mc_inject_data_hi_show(struct device *dev,
-					      struct device_attribute *mattr,
-					      char *data)
-{
-	struct mem_ctl_info *mci = to_mci(dev);
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-	return sprintf(data, "0x%08x",
-		       in_be32(pdata->mc_vbase +
-			       MPC85XX_MC_DATA_ERR_INJECT_HI));
-}
-
-static ssize_t mpc85xx_mc_inject_data_lo_show(struct device *dev,
-					      struct device_attribute *mattr,
-					      char *data)
-{
-	struct mem_ctl_info *mci = to_mci(dev);
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-	return sprintf(data, "0x%08x",
-		       in_be32(pdata->mc_vbase +
-			       MPC85XX_MC_DATA_ERR_INJECT_LO));
-}
-
-static ssize_t mpc85xx_mc_inject_ctrl_show(struct device *dev,
-					   struct device_attribute *mattr,
-					   char *data)
-{
-	struct mem_ctl_info *mci = to_mci(dev);
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-	return sprintf(data, "0x%08x",
-		       in_be32(pdata->mc_vbase + MPC85XX_MC_ECC_ERR_INJECT));
-}
-
-static ssize_t mpc85xx_mc_inject_data_hi_store(struct device *dev,
-					       struct device_attribute *mattr,
-					       const char *data, size_t count)
-{
-	struct mem_ctl_info *mci = to_mci(dev);
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-	if (isdigit(*data)) {
-		out_be32(pdata->mc_vbase + MPC85XX_MC_DATA_ERR_INJECT_HI,
-			 simple_strtoul(data, NULL, 0));
-		return count;
-	}
-	return 0;
-}
-
-static ssize_t mpc85xx_mc_inject_data_lo_store(struct device *dev,
-					       struct device_attribute *mattr,
-					       const char *data, size_t count)
-{
-	struct mem_ctl_info *mci = to_mci(dev);
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-	if (isdigit(*data)) {
-		out_be32(pdata->mc_vbase + MPC85XX_MC_DATA_ERR_INJECT_LO,
-			 simple_strtoul(data, NULL, 0));
-		return count;
-	}
-	return 0;
-}
-
-static ssize_t mpc85xx_mc_inject_ctrl_store(struct device *dev,
-					       struct device_attribute *mattr,
-					       const char *data, size_t count)
-{
-	struct mem_ctl_info *mci = to_mci(dev);
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-	if (isdigit(*data)) {
-		out_be32(pdata->mc_vbase + MPC85XX_MC_ECC_ERR_INJECT,
-			 simple_strtoul(data, NULL, 0));
-		return count;
-	}
-	return 0;
-}
-
-DEVICE_ATTR(inject_data_hi, S_IRUGO | S_IWUSR,
-	    mpc85xx_mc_inject_data_hi_show, mpc85xx_mc_inject_data_hi_store);
-DEVICE_ATTR(inject_data_lo, S_IRUGO | S_IWUSR,
-	    mpc85xx_mc_inject_data_lo_show, mpc85xx_mc_inject_data_lo_store);
-DEVICE_ATTR(inject_ctrl, S_IRUGO | S_IWUSR,
-	    mpc85xx_mc_inject_ctrl_show, mpc85xx_mc_inject_ctrl_store);
-
-static struct attribute *mpc85xx_dev_attrs[] = {
-	&dev_attr_inject_data_hi.attr,
-	&dev_attr_inject_data_lo.attr,
-	&dev_attr_inject_ctrl.attr,
-	NULL
-};
-
-ATTRIBUTE_GROUPS(mpc85xx_dev);
 
 /**************************** PCI Err device ***************************/
 #ifdef CONFIG_PCI
@@ -160,18 +60,18 @@ static void mpc85xx_pci_check(struct edac_pci_ctl_info *pci)
 		return;
 	}
 
-	printk(KERN_ERR "PCI error(s) detected\n");
-	printk(KERN_ERR "PCI/X ERR_DR register: %#08x\n", err_detect);
+	pr_err("PCI error(s) detected\n");
+	pr_err("PCI/X ERR_DR register: %#08x\n", err_detect);
 
-	printk(KERN_ERR "PCI/X ERR_ATTRIB register: %#08x\n",
+	pr_err("PCI/X ERR_ATTRIB register: %#08x\n",
 	       in_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_ATTRIB));
-	printk(KERN_ERR "PCI/X ERR_ADDR register: %#08x\n",
+	pr_err("PCI/X ERR_ADDR register: %#08x\n",
 	       in_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_ADDR));
-	printk(KERN_ERR "PCI/X ERR_EXT_ADDR register: %#08x\n",
+	pr_err("PCI/X ERR_EXT_ADDR register: %#08x\n",
 	       in_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_EXT_ADDR));
-	printk(KERN_ERR "PCI/X ERR_DL register: %#08x\n",
+	pr_err("PCI/X ERR_DL register: %#08x\n",
 	       in_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_DL));
-	printk(KERN_ERR "PCI/X ERR_DH register: %#08x\n",
+	pr_err("PCI/X ERR_DH register: %#08x\n",
 	       in_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_DH));
 
 	/* clear error bits */
@@ -187,14 +87,14 @@ static void mpc85xx_pci_check(struct edac_pci_ctl_info *pci)
 static void mpc85xx_pcie_check(struct edac_pci_ctl_info *pci)
 {
 	struct mpc85xx_pci_pdata *pdata = pci->pvt_info;
-	u32 err_detect;
+	u32 err_detect, err_cap_stat;
 
 	err_detect = in_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_DR);
+	err_cap_stat = in_be32(pdata->pci_vbase + MPC85XX_PCI_GAS_TIMR);
 
 	pr_err("PCIe error(s) detected\n");
 	pr_err("PCIe ERR_DR register: 0x%08x\n", err_detect);
-	pr_err("PCIe ERR_CAP_STAT register: 0x%08x\n",
-			in_be32(pdata->pci_vbase + MPC85XX_PCI_GAS_TIMR));
+	pr_err("PCIe ERR_CAP_STAT register: 0x%08x\n", err_cap_stat);
 	pr_err("PCIe ERR_CAP_R0 register: 0x%08x\n",
 			in_be32(pdata->pci_vbase + MPC85XX_PCIE_ERR_CAP_R0));
 	pr_err("PCIe ERR_CAP_R1 register: 0x%08x\n",
@@ -206,6 +106,9 @@ static void mpc85xx_pcie_check(struct edac_pci_ctl_info *pci)
 
 	/* clear error bits */
 	out_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_DR, err_detect);
+
+	/* reset error capture */
+	out_be32(pdata->pci_vbase + MPC85XX_PCI_GAS_TIMR, err_cap_stat | 0x1);
 }
 
 static int mpc85xx_pcie_find_capability(struct device_node *np)
@@ -267,7 +170,6 @@ static int mpc85xx_pci_err_probe(struct platform_device *op)
 
 	pdata = pci->pvt_info;
 	pdata->name = "mpc85xx_pci_err";
-	pdata->irq = NO_IRQ;
 
 	plat_data = op->dev.platform_data;
 	if (!plat_data) {
@@ -297,8 +199,7 @@ static int mpc85xx_pci_err_probe(struct platform_device *op)
 
 	res = of_address_to_resource(of_node, 0, &r);
 	if (res) {
-		printk(KERN_ERR "%s: Unable to get resource for "
-		       "PCI err regs\n", __func__);
+		pr_err("%s: Unable to get resource for PCI err regs\n", __func__);
 		goto err;
 	}
 
@@ -307,15 +208,14 @@ static int mpc85xx_pci_err_probe(struct platform_device *op)
 
 	if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r),
 					pdata->name)) {
-		printk(KERN_ERR "%s: Error while requesting mem region\n",
-		       __func__);
+		pr_err("%s: Error while requesting mem region\n", __func__);
 		res = -EBUSY;
 		goto err;
 	}
 
 	pdata->pci_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r));
 	if (!pdata->pci_vbase) {
-		printk(KERN_ERR "%s: Unable to setup PCI err regs\n", __func__);
+		pr_err("%s: Unable to setup PCI err regs\n", __func__);
 		res = -ENOMEM;
 		goto err;
 	}
@@ -344,6 +244,9 @@ static int mpc85xx_pci_err_probe(struct platform_device *op)
 	/* clear error bits */
 	out_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_DR, ~0);
 
+	/* reset error capture */
+	out_be32(pdata->pci_vbase + MPC85XX_PCI_GAS_TIMR, 0x1);
+
 	if (edac_pci_add_device(pci, pdata->edac_idx) > 0) {
 		edac_dbg(3, "failed edac_pci_add_device()\n");
 		goto err;
@@ -356,15 +259,14 @@ static int mpc85xx_pci_err_probe(struct platform_device *op)
 				       IRQF_SHARED,
 				       "[EDAC] PCI err", pci);
 		if (res < 0) {
-			printk(KERN_ERR
-			       "%s: Unable to request irq %d for "
-			       "MPC85xx PCI err\n", __func__, pdata->irq);
+			pr_err("%s: Unable to request irq %d for MPC85xx PCI err\n",
+				__func__, pdata->irq);
 			irq_dispose_mapping(pdata->irq);
 			res = -ENODEV;
 			goto err2;
 		}
 
-		printk(KERN_INFO EDAC_MOD_STR " acquired irq %d for PCI Err\n",
+		pr_info(EDAC_MOD_STR " acquired irq %d for PCI Err\n",
 		       pdata->irq);
 	}
 
@@ -386,7 +288,7 @@ static int mpc85xx_pci_err_probe(struct platform_device *op)
 
 	devres_remove_group(&op->dev, mpc85xx_pci_err_probe);
 	edac_dbg(3, "success\n");
-	printk(KERN_INFO EDAC_MOD_STR " PCI err registered\n");
+	pr_info(EDAC_MOD_STR " PCI err registered\n");
 
 	return 0;
 
@@ -529,17 +431,17 @@ static void mpc85xx_l2_check(struct edac_device_ctl_info *edac_dev)
 	if (!(err_detect & L2_EDE_MASK))
 		return;
 
-	printk(KERN_ERR "ECC Error in CPU L2 cache\n");
-	printk(KERN_ERR "L2 Error Detect Register: 0x%08x\n", err_detect);
-	printk(KERN_ERR "L2 Error Capture Data High Register: 0x%08x\n",
+	pr_err("ECC Error in CPU L2 cache\n");
+	pr_err("L2 Error Detect Register: 0x%08x\n", err_detect);
+	pr_err("L2 Error Capture Data High Register: 0x%08x\n",
 	       in_be32(pdata->l2_vbase + MPC85XX_L2_CAPTDATAHI));
-	printk(KERN_ERR "L2 Error Capture Data Lo Register: 0x%08x\n",
+	pr_err("L2 Error Capture Data Lo Register: 0x%08x\n",
 	       in_be32(pdata->l2_vbase + MPC85XX_L2_CAPTDATALO));
-	printk(KERN_ERR "L2 Error Syndrome Register: 0x%08x\n",
+	pr_err("L2 Error Syndrome Register: 0x%08x\n",
 	       in_be32(pdata->l2_vbase + MPC85XX_L2_CAPTECC));
-	printk(KERN_ERR "L2 Error Attributes Capture Register: 0x%08x\n",
+	pr_err("L2 Error Attributes Capture Register: 0x%08x\n",
 	       in_be32(pdata->l2_vbase + MPC85XX_L2_ERRATTR));
-	printk(KERN_ERR "L2 Error Address Capture Register: 0x%08x\n",
+	pr_err("L2 Error Address Capture Register: 0x%08x\n",
 	       in_be32(pdata->l2_vbase + MPC85XX_L2_ERRADDR));
 
 	/* clear error detect register */
@@ -588,7 +490,6 @@ static int mpc85xx_l2_err_probe(struct platform_device *op)
 
 	pdata = edac_dev->pvt_info;
 	pdata->name = "mpc85xx_l2_err";
-	pdata->irq = NO_IRQ;
 	edac_dev->dev = &op->dev;
 	dev_set_drvdata(edac_dev->dev, edac_dev);
 	edac_dev->ctl_name = pdata->name;
@@ -596,8 +497,7 @@ static int mpc85xx_l2_err_probe(struct platform_device *op)
 
 	res = of_address_to_resource(op->dev.of_node, 0, &r);
 	if (res) {
-		printk(KERN_ERR "%s: Unable to get resource for "
-		       "L2 err regs\n", __func__);
+		pr_err("%s: Unable to get resource for L2 err regs\n", __func__);
 		goto err;
 	}
 
@@ -606,15 +506,14 @@ static int mpc85xx_l2_err_probe(struct platform_device *op)
 
 	if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r),
 				     pdata->name)) {
-		printk(KERN_ERR "%s: Error while requesting mem region\n",
-		       __func__);
+		pr_err("%s: Error while requesting mem region\n", __func__);
 		res = -EBUSY;
 		goto err;
 	}
 
 	pdata->l2_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r));
 	if (!pdata->l2_vbase) {
-		printk(KERN_ERR "%s: Unable to setup L2 err regs\n", __func__);
+		pr_err("%s: Unable to setup L2 err regs\n", __func__);
 		res = -ENOMEM;
 		goto err;
 	}
@@ -646,16 +545,14 @@ static int mpc85xx_l2_err_probe(struct platform_device *op)
 				       mpc85xx_l2_isr, IRQF_SHARED,
 				       "[EDAC] L2 err", edac_dev);
 		if (res < 0) {
-			printk(KERN_ERR
-			       "%s: Unable to request irq %d for "
-			       "MPC85xx L2 err\n", __func__, pdata->irq);
+			pr_err("%s: Unable to request irq %d for MPC85xx L2 err\n",
+				__func__, pdata->irq);
 			irq_dispose_mapping(pdata->irq);
 			res = -ENODEV;
 			goto err2;
 		}
 
-		printk(KERN_INFO EDAC_MOD_STR " acquired irq %d for L2 Err\n",
-		       pdata->irq);
+		pr_info(EDAC_MOD_STR " acquired irq %d for L2 Err\n", pdata->irq);
 
 		edac_dev->op_state = OP_RUNNING_INTERRUPT;
 
@@ -665,7 +562,7 @@ static int mpc85xx_l2_err_probe(struct platform_device *op)
 	devres_remove_group(&op->dev, mpc85xx_l2_err_probe);
 
 	edac_dbg(3, "success\n");
-	printk(KERN_INFO EDAC_MOD_STR " L2 err registered\n");
+	pr_info(EDAC_MOD_STR " L2 err registered\n");
 
 	return 0;
 
@@ -729,466 +626,6 @@ static struct platform_driver mpc85xx_l2_err_driver = {
 	},
 };
 
-/**************************** MC Err device ***************************/
-
-/*
- * Taken from table 8-55 in the MPC8641 User's Manual and/or 9-61 in the
- * MPC8572 User's Manual.  Each line represents a syndrome bit column as a
- * 64-bit value, but split into an upper and lower 32-bit chunk.  The labels
- * below correspond to Freescale's manuals.
- */
-static unsigned int ecc_table[16] = {
-	/* MSB           LSB */
-	/* [0:31]    [32:63] */
-	0xf00fe11e, 0xc33c0ff7,	/* Syndrome bit 7 */
-	0x00ff00ff, 0x00fff0ff,
-	0x0f0f0f0f, 0x0f0fff00,
-	0x11113333, 0x7777000f,
-	0x22224444, 0x8888222f,
-	0x44448888, 0xffff4441,
-	0x8888ffff, 0x11118882,
-	0xffff1111, 0x22221114,	/* Syndrome bit 0 */
-};
-
-/*
- * Calculate the correct ECC value for a 64-bit value specified by high:low
- */
-static u8 calculate_ecc(u32 high, u32 low)
-{
-	u32 mask_low;
-	u32 mask_high;
-	int bit_cnt;
-	u8 ecc = 0;
-	int i;
-	int j;
-
-	for (i = 0; i < 8; i++) {
-		mask_high = ecc_table[i * 2];
-		mask_low = ecc_table[i * 2 + 1];
-		bit_cnt = 0;
-
-		for (j = 0; j < 32; j++) {
-			if ((mask_high >> j) & 1)
-				bit_cnt ^= (high >> j) & 1;
-			if ((mask_low >> j) & 1)
-				bit_cnt ^= (low >> j) & 1;
-		}
-
-		ecc |= bit_cnt << i;
-	}
-
-	return ecc;
-}
-
-/*
- * Create the syndrome code which is generated if the data line specified by
- * 'bit' failed.  Eg generate an 8-bit codes seen in Table 8-55 in the MPC8641
- * User's Manual and 9-61 in the MPC8572 User's Manual.
- */
-static u8 syndrome_from_bit(unsigned int bit) {
-	int i;
-	u8 syndrome = 0;
-
-	/*
-	 * Cycle through the upper or lower 32-bit portion of each value in
-	 * ecc_table depending on if 'bit' is in the upper or lower half of
-	 * 64-bit data.
-	 */
-	for (i = bit < 32; i < 16; i += 2)
-		syndrome |= ((ecc_table[i] >> (bit % 32)) & 1) << (i / 2);
-
-	return syndrome;
-}
-
-/*
- * Decode data and ecc syndrome to determine what went wrong
- * Note: This can only decode single-bit errors
- */
-static void sbe_ecc_decode(u32 cap_high, u32 cap_low, u32 cap_ecc,
-		       int *bad_data_bit, int *bad_ecc_bit)
-{
-	int i;
-	u8 syndrome;
-
-	*bad_data_bit = -1;
-	*bad_ecc_bit = -1;
-
-	/*
-	 * Calculate the ECC of the captured data and XOR it with the captured
-	 * ECC to find an ECC syndrome value we can search for
-	 */
-	syndrome = calculate_ecc(cap_high, cap_low) ^ cap_ecc;
-
-	/* Check if a data line is stuck... */
-	for (i = 0; i < 64; i++) {
-		if (syndrome == syndrome_from_bit(i)) {
-			*bad_data_bit = i;
-			return;
-		}
-	}
-
-	/* If data is correct, check ECC bits for errors... */
-	for (i = 0; i < 8; i++) {
-		if ((syndrome >> i) & 0x1) {
-			*bad_ecc_bit = i;
-			return;
-		}
-	}
-}
-
-#define make64(high, low) (((u64)(high) << 32) | (low))
-
-static void mpc85xx_mc_check(struct mem_ctl_info *mci)
-{
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-	struct csrow_info *csrow;
-	u32 bus_width;
-	u32 err_detect;
-	u32 syndrome;
-	u64 err_addr;
-	u32 pfn;
-	int row_index;
-	u32 cap_high;
-	u32 cap_low;
-	int bad_data_bit;
-	int bad_ecc_bit;
-
-	err_detect = in_be32(pdata->mc_vbase + MPC85XX_MC_ERR_DETECT);
-	if (!err_detect)
-		return;
-
-	mpc85xx_mc_printk(mci, KERN_ERR, "Err Detect Register: %#8.8x\n",
-			  err_detect);
-
-	/* no more processing if not ECC bit errors */
-	if (!(err_detect & (DDR_EDE_SBE | DDR_EDE_MBE))) {
-		out_be32(pdata->mc_vbase + MPC85XX_MC_ERR_DETECT, err_detect);
-		return;
-	}
-
-	syndrome = in_be32(pdata->mc_vbase + MPC85XX_MC_CAPTURE_ECC);
-
-	/* Mask off appropriate bits of syndrome based on bus width */
-	bus_width = (in_be32(pdata->mc_vbase + MPC85XX_MC_DDR_SDRAM_CFG) &
-			DSC_DBW_MASK) ? 32 : 64;
-	if (bus_width == 64)
-		syndrome &= 0xff;
-	else
-		syndrome &= 0xffff;
-
-	err_addr = make64(
-		in_be32(pdata->mc_vbase + MPC85XX_MC_CAPTURE_EXT_ADDRESS),
-		in_be32(pdata->mc_vbase + MPC85XX_MC_CAPTURE_ADDRESS));
-	pfn = err_addr >> PAGE_SHIFT;
-
-	for (row_index = 0; row_index < mci->nr_csrows; row_index++) {
-		csrow = mci->csrows[row_index];
-		if ((pfn >= csrow->first_page) && (pfn <= csrow->last_page))
-			break;
-	}
-
-	cap_high = in_be32(pdata->mc_vbase + MPC85XX_MC_CAPTURE_DATA_HI);
-	cap_low = in_be32(pdata->mc_vbase + MPC85XX_MC_CAPTURE_DATA_LO);
-
-	/*
-	 * Analyze single-bit errors on 64-bit wide buses
-	 * TODO: Add support for 32-bit wide buses
-	 */
-	if ((err_detect & DDR_EDE_SBE) && (bus_width == 64)) {
-		sbe_ecc_decode(cap_high, cap_low, syndrome,
-				&bad_data_bit, &bad_ecc_bit);
-
-		if (bad_data_bit != -1)
-			mpc85xx_mc_printk(mci, KERN_ERR,
-				"Faulty Data bit: %d\n", bad_data_bit);
-		if (bad_ecc_bit != -1)
-			mpc85xx_mc_printk(mci, KERN_ERR,
-				"Faulty ECC bit: %d\n", bad_ecc_bit);
-
-		mpc85xx_mc_printk(mci, KERN_ERR,
-			"Expected Data / ECC:\t%#8.8x_%08x / %#2.2x\n",
-			cap_high ^ (1 << (bad_data_bit - 32)),
-			cap_low ^ (1 << bad_data_bit),
-			syndrome ^ (1 << bad_ecc_bit));
-	}
-
-	mpc85xx_mc_printk(mci, KERN_ERR,
-			"Captured Data / ECC:\t%#8.8x_%08x / %#2.2x\n",
-			cap_high, cap_low, syndrome);
-	mpc85xx_mc_printk(mci, KERN_ERR, "Err addr: %#8.8llx\n", err_addr);
-	mpc85xx_mc_printk(mci, KERN_ERR, "PFN: %#8.8x\n", pfn);
-
-	/* we are out of range */
-	if (row_index == mci->nr_csrows)
-		mpc85xx_mc_printk(mci, KERN_ERR, "PFN out of range!\n");
-
-	if (err_detect & DDR_EDE_SBE)
-		edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
-				     pfn, err_addr & ~PAGE_MASK, syndrome,
-				     row_index, 0, -1,
-				     mci->ctl_name, "");
-
-	if (err_detect & DDR_EDE_MBE)
-		edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
-				     pfn, err_addr & ~PAGE_MASK, syndrome,
-				     row_index, 0, -1,
-				     mci->ctl_name, "");
-
-	out_be32(pdata->mc_vbase + MPC85XX_MC_ERR_DETECT, err_detect);
-}
-
-static irqreturn_t mpc85xx_mc_isr(int irq, void *dev_id)
-{
-	struct mem_ctl_info *mci = dev_id;
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-	u32 err_detect;
-
-	err_detect = in_be32(pdata->mc_vbase + MPC85XX_MC_ERR_DETECT);
-	if (!err_detect)
-		return IRQ_NONE;
-
-	mpc85xx_mc_check(mci);
-
-	return IRQ_HANDLED;
-}
-
-static void mpc85xx_init_csrows(struct mem_ctl_info *mci)
-{
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-	struct csrow_info *csrow;
-	struct dimm_info *dimm;
-	u32 sdram_ctl;
-	u32 sdtype;
-	enum mem_type mtype;
-	u32 cs_bnds;
-	int index;
-
-	sdram_ctl = in_be32(pdata->mc_vbase + MPC85XX_MC_DDR_SDRAM_CFG);
-
-	sdtype = sdram_ctl & DSC_SDTYPE_MASK;
-	if (sdram_ctl & DSC_RD_EN) {
-		switch (sdtype) {
-		case DSC_SDTYPE_DDR:
-			mtype = MEM_RDDR;
-			break;
-		case DSC_SDTYPE_DDR2:
-			mtype = MEM_RDDR2;
-			break;
-		case DSC_SDTYPE_DDR3:
-			mtype = MEM_RDDR3;
-			break;
-		default:
-			mtype = MEM_UNKNOWN;
-			break;
-		}
-	} else {
-		switch (sdtype) {
-		case DSC_SDTYPE_DDR:
-			mtype = MEM_DDR;
-			break;
-		case DSC_SDTYPE_DDR2:
-			mtype = MEM_DDR2;
-			break;
-		case DSC_SDTYPE_DDR3:
-			mtype = MEM_DDR3;
-			break;
-		default:
-			mtype = MEM_UNKNOWN;
-			break;
-		}
-	}
-
-	for (index = 0; index < mci->nr_csrows; index++) {
-		u32 start;
-		u32 end;
-
-		csrow = mci->csrows[index];
-		dimm = csrow->channels[0]->dimm;
-
-		cs_bnds = in_be32(pdata->mc_vbase + MPC85XX_MC_CS_BNDS_0 +
-				  (index * MPC85XX_MC_CS_BNDS_OFS));
-
-		start = (cs_bnds & 0xffff0000) >> 16;
-		end   = (cs_bnds & 0x0000ffff);
-
-		if (start == end)
-			continue;	/* not populated */
-
-		start <<= (24 - PAGE_SHIFT);
-		end   <<= (24 - PAGE_SHIFT);
-		end    |= (1 << (24 - PAGE_SHIFT)) - 1;
-
-		csrow->first_page = start;
-		csrow->last_page = end;
-
-		dimm->nr_pages = end + 1 - start;
-		dimm->grain = 8;
-		dimm->mtype = mtype;
-		dimm->dtype = DEV_UNKNOWN;
-		if (sdram_ctl & DSC_X32_EN)
-			dimm->dtype = DEV_X32;
-		dimm->edac_mode = EDAC_SECDED;
-	}
-}
-
-static int mpc85xx_mc_err_probe(struct platform_device *op)
-{
-	struct mem_ctl_info *mci;
-	struct edac_mc_layer layers[2];
-	struct mpc85xx_mc_pdata *pdata;
-	struct resource r;
-	u32 sdram_ctl;
-	int res;
-
-	if (!devres_open_group(&op->dev, mpc85xx_mc_err_probe, GFP_KERNEL))
-		return -ENOMEM;
-
-	layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
-	layers[0].size = 4;
-	layers[0].is_virt_csrow = true;
-	layers[1].type = EDAC_MC_LAYER_CHANNEL;
-	layers[1].size = 1;
-	layers[1].is_virt_csrow = false;
-	mci = edac_mc_alloc(edac_mc_idx, ARRAY_SIZE(layers), layers,
-			    sizeof(*pdata));
-	if (!mci) {
-		devres_release_group(&op->dev, mpc85xx_mc_err_probe);
-		return -ENOMEM;
-	}
-
-	pdata = mci->pvt_info;
-	pdata->name = "mpc85xx_mc_err";
-	pdata->irq = NO_IRQ;
-	mci->pdev = &op->dev;
-	pdata->edac_idx = edac_mc_idx++;
-	dev_set_drvdata(mci->pdev, mci);
-	mci->ctl_name = pdata->name;
-	mci->dev_name = pdata->name;
-
-	res = of_address_to_resource(op->dev.of_node, 0, &r);
-	if (res) {
-		printk(KERN_ERR "%s: Unable to get resource for MC err regs\n",
-		       __func__);
-		goto err;
-	}
-
-	if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r),
-				     pdata->name)) {
-		printk(KERN_ERR "%s: Error while requesting mem region\n",
-		       __func__);
-		res = -EBUSY;
-		goto err;
-	}
-
-	pdata->mc_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r));
-	if (!pdata->mc_vbase) {
-		printk(KERN_ERR "%s: Unable to setup MC err regs\n", __func__);
-		res = -ENOMEM;
-		goto err;
-	}
-
-	sdram_ctl = in_be32(pdata->mc_vbase + MPC85XX_MC_DDR_SDRAM_CFG);
-	if (!(sdram_ctl & DSC_ECC_EN)) {
-		/* no ECC */
-		printk(KERN_WARNING "%s: No ECC DIMMs discovered\n", __func__);
-		res = -ENODEV;
-		goto err;
-	}
-
-	edac_dbg(3, "init mci\n");
-	mci->mtype_cap = MEM_FLAG_RDDR | MEM_FLAG_RDDR2 |
-	    MEM_FLAG_DDR | MEM_FLAG_DDR2;
-	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
-	mci->edac_cap = EDAC_FLAG_SECDED;
-	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = MPC85XX_REVISION;
-
-	if (edac_op_state == EDAC_OPSTATE_POLL)
-		mci->edac_check = mpc85xx_mc_check;
-
-	mci->ctl_page_to_phys = NULL;
-
-	mci->scrub_mode = SCRUB_SW_SRC;
-
-	mpc85xx_init_csrows(mci);
-
-	/* store the original error disable bits */
-	orig_ddr_err_disable =
-	    in_be32(pdata->mc_vbase + MPC85XX_MC_ERR_DISABLE);
-	out_be32(pdata->mc_vbase + MPC85XX_MC_ERR_DISABLE, 0);
-
-	/* clear all error bits */
-	out_be32(pdata->mc_vbase + MPC85XX_MC_ERR_DETECT, ~0);
-
-	if (edac_mc_add_mc_with_groups(mci, mpc85xx_dev_groups)) {
-		edac_dbg(3, "failed edac_mc_add_mc()\n");
-		goto err;
-	}
-
-	if (edac_op_state == EDAC_OPSTATE_INT) {
-		out_be32(pdata->mc_vbase + MPC85XX_MC_ERR_INT_EN,
-			 DDR_EIE_MBEE | DDR_EIE_SBEE);
-
-		/* store the original error management threshold */
-		orig_ddr_err_sbe = in_be32(pdata->mc_vbase +
-					   MPC85XX_MC_ERR_SBE) & 0xff0000;
-
-		/* set threshold to 1 error per interrupt */
-		out_be32(pdata->mc_vbase + MPC85XX_MC_ERR_SBE, 0x10000);
-
-		/* register interrupts */
-		pdata->irq = irq_of_parse_and_map(op->dev.of_node, 0);
-		res = devm_request_irq(&op->dev, pdata->irq,
-				       mpc85xx_mc_isr,
-				       IRQF_SHARED,
-				       "[EDAC] MC err", mci);
-		if (res < 0) {
-			printk(KERN_ERR "%s: Unable to request irq %d for "
-			       "MPC85xx DRAM ERR\n", __func__, pdata->irq);
-			irq_dispose_mapping(pdata->irq);
-			res = -ENODEV;
-			goto err2;
-		}
-
-		printk(KERN_INFO EDAC_MOD_STR " acquired irq %d for MC\n",
-		       pdata->irq);
-	}
-
-	devres_remove_group(&op->dev, mpc85xx_mc_err_probe);
-	edac_dbg(3, "success\n");
-	printk(KERN_INFO EDAC_MOD_STR " MC err registered\n");
-
-	return 0;
-
-err2:
-	edac_mc_del_mc(&op->dev);
-err:
-	devres_release_group(&op->dev, mpc85xx_mc_err_probe);
-	edac_mc_free(mci);
-	return res;
-}
-
-static int mpc85xx_mc_err_remove(struct platform_device *op)
-{
-	struct mem_ctl_info *mci = dev_get_drvdata(&op->dev);
-	struct mpc85xx_mc_pdata *pdata = mci->pvt_info;
-
-	edac_dbg(0, "\n");
-
-	if (edac_op_state == EDAC_OPSTATE_INT) {
-		out_be32(pdata->mc_vbase + MPC85XX_MC_ERR_INT_EN, 0);
-		irq_dispose_mapping(pdata->irq);
-	}
-
-	out_be32(pdata->mc_vbase + MPC85XX_MC_ERR_DISABLE,
-		 orig_ddr_err_disable);
-	out_be32(pdata->mc_vbase + MPC85XX_MC_ERR_SBE, orig_ddr_err_sbe);
-
-	edac_mc_del_mc(&op->dev);
-	edac_mc_free(mci);
-	return 0;
-}
-
 static const struct of_device_id mpc85xx_mc_err_of_match[] = {
 /* deprecate the fsl,85.. forms in the future, 2.6.30? */
 	{ .compatible = "fsl,8540-memory-controller", },
@@ -1217,22 +654,14 @@ static const struct of_device_id mpc85xx_mc_err_of_match[] = {
 MODULE_DEVICE_TABLE(of, mpc85xx_mc_err_of_match);
 
 static struct platform_driver mpc85xx_mc_err_driver = {
-	.probe = mpc85xx_mc_err_probe,
-	.remove = mpc85xx_mc_err_remove,
+	.probe = fsl_mc_err_probe,
+	.remove = fsl_mc_err_remove,
 	.driver = {
 		.name = "mpc85xx_mc_err",
 		.of_match_table = mpc85xx_mc_err_of_match,
 	},
 };
 
-#ifdef CONFIG_FSL_SOC_BOOKE
-static void __init mpc85xx_mc_clear_rfxe(void *data)
-{
-	orig_hid1[smp_processor_id()] = mfspr(SPRN_HID1);
-	mtspr(SPRN_HID1, (orig_hid1[smp_processor_id()] & ~HID1_RFXE));
-}
-#endif
-
 static struct platform_driver * const drivers[] = {
 	&mpc85xx_mc_err_driver,
 	&mpc85xx_l2_err_driver,
@@ -1246,8 +675,7 @@ static int __init mpc85xx_mc_init(void)
 	int res = 0;
 	u32 __maybe_unused pvr = 0;
 
-	printk(KERN_INFO "Freescale(R) MPC85xx EDAC driver, "
-	       "(C) 2006 Montavista Software\n");
+	pr_info("Freescale(R) MPC85xx EDAC driver, (C) 2006 Montavista Software\n");
 
 	/* make sure error reporting method is sane */
 	switch (edac_op_state) {
@@ -1261,44 +689,15 @@ static int __init mpc85xx_mc_init(void)
 
 	res = platform_register_drivers(drivers, ARRAY_SIZE(drivers));
 	if (res)
-		printk(KERN_WARNING EDAC_MOD_STR "drivers fail to register\n");
-
-#ifdef CONFIG_FSL_SOC_BOOKE
-	pvr = mfspr(SPRN_PVR);
-
-	if ((PVR_VER(pvr) == PVR_VER_E500V1) ||
-	    (PVR_VER(pvr) == PVR_VER_E500V2)) {
-		/*
-		 * need to clear HID1[RFXE] to disable machine check int
-		 * so we can catch it
-		 */
-		if (edac_op_state == EDAC_OPSTATE_INT)
-			on_each_cpu(mpc85xx_mc_clear_rfxe, NULL, 0);
-	}
-#endif
+		pr_warn(EDAC_MOD_STR "drivers fail to register\n");
 
 	return 0;
 }
 
 module_init(mpc85xx_mc_init);
 
-#ifdef CONFIG_FSL_SOC_BOOKE
-static void __exit mpc85xx_mc_restore_hid1(void *data)
-{
-	mtspr(SPRN_HID1, orig_hid1[smp_processor_id()]);
-}
-#endif
-
 static void __exit mpc85xx_mc_exit(void)
 {
-#ifdef CONFIG_FSL_SOC_BOOKE
-	u32 pvr = mfspr(SPRN_PVR);
-
-	if ((PVR_VER(pvr) == PVR_VER_E500V1) ||
-	    (PVR_VER(pvr) == PVR_VER_E500V2)) {
-		on_each_cpu(mpc85xx_mc_restore_hid1, NULL, 0);
-	}
-#endif
 	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
 }
 
diff --git a/drivers/edac/mpc85xx_edac.h b/drivers/edac/mpc85xx_edac.h
index 9352e88d53e5..3f6fb16ad34f 100644
--- a/drivers/edac/mpc85xx_edac.h
+++ b/drivers/edac/mpc85xx_edac.h
@@ -17,65 +17,6 @@
 #define mpc85xx_printk(level, fmt, arg...) \
 	edac_printk(level, "MPC85xx", fmt, ##arg)
 
-#define mpc85xx_mc_printk(mci, level, fmt, arg...) \
-	edac_mc_chipset_printk(mci, level, "MPC85xx", fmt, ##arg)
-
-/*
- * DRAM error defines
- */
-
-/* DDR_SDRAM_CFG */
-#define MPC85XX_MC_DDR_SDRAM_CFG	0x0110
-#define MPC85XX_MC_CS_BNDS_0		0x0000
-#define MPC85XX_MC_CS_BNDS_1		0x0008
-#define MPC85XX_MC_CS_BNDS_2		0x0010
-#define MPC85XX_MC_CS_BNDS_3		0x0018
-#define MPC85XX_MC_CS_BNDS_OFS		0x0008
-
-#define MPC85XX_MC_DATA_ERR_INJECT_HI	0x0e00
-#define MPC85XX_MC_DATA_ERR_INJECT_LO	0x0e04
-#define MPC85XX_MC_ECC_ERR_INJECT	0x0e08
-#define MPC85XX_MC_CAPTURE_DATA_HI	0x0e20
-#define MPC85XX_MC_CAPTURE_DATA_LO	0x0e24
-#define MPC85XX_MC_CAPTURE_ECC		0x0e28
-#define MPC85XX_MC_ERR_DETECT		0x0e40
-#define MPC85XX_MC_ERR_DISABLE		0x0e44
-#define MPC85XX_MC_ERR_INT_EN		0x0e48
-#define MPC85XX_MC_CAPTURE_ATRIBUTES	0x0e4c
-#define MPC85XX_MC_CAPTURE_ADDRESS	0x0e50
-#define MPC85XX_MC_CAPTURE_EXT_ADDRESS	0x0e54
-#define MPC85XX_MC_ERR_SBE		0x0e58
-
-#define DSC_MEM_EN	0x80000000
-#define DSC_ECC_EN	0x20000000
-#define DSC_RD_EN	0x10000000
-#define DSC_DBW_MASK	0x00180000
-#define DSC_DBW_32	0x00080000
-#define DSC_DBW_64	0x00000000
-
-#define DSC_SDTYPE_MASK		0x07000000
-
-#define DSC_SDTYPE_DDR		0x02000000
-#define DSC_SDTYPE_DDR2		0x03000000
-#define DSC_SDTYPE_DDR3		0x07000000
-#define DSC_X32_EN	0x00000020
-
-/* Err_Int_En */
-#define DDR_EIE_MSEE	0x1	/* memory select */
-#define DDR_EIE_SBEE	0x4	/* single-bit ECC error */
-#define DDR_EIE_MBEE	0x8	/* multi-bit ECC error */
-
-/* Err_Detect */
-#define DDR_EDE_MSE		0x1	/* memory select */
-#define DDR_EDE_SBE		0x4	/* single-bit ECC error */
-#define DDR_EDE_MBE		0x8	/* multi-bit ECC error */
-#define DDR_EDE_MME		0x80000000	/* multiple memory errors */
-
-/* Err_Disable */
-#define DDR_EDI_MSED	0x1	/* memory select disable */
-#define	DDR_EDI_SBED	0x4	/* single-bit ECC error disable */
-#define	DDR_EDI_MBED	0x8	/* multi-bit ECC error disable */
-
 /*
  * L2 Err defines
  */
@@ -149,13 +90,6 @@
 #define MPC85XX_PCIE_ERR_CAP_R2		0x0030
 #define MPC85XX_PCIE_ERR_CAP_R3		0x0034
 
-struct mpc85xx_mc_pdata {
-	char *name;
-	int edac_idx;
-	void __iomem *mc_vbase;
-	int irq;
-};
-
 struct mpc85xx_l2_pdata {
 	char *name;
 	int edac_idx;
diff --git a/drivers/edac/mv64x60_edac.c b/drivers/edac/mv64x60_edac.c
index 6c54127e6eae..cb9b8577acbc 100644
--- a/drivers/edac/mv64x60_edac.c
+++ b/drivers/edac/mv64x60_edac.c
@@ -118,7 +118,6 @@ static int mv64x60_pci_err_probe(struct platform_device *pdev)
 
 	pdata->pci_hose = pdev->id;
 	pdata->name = "mpc85xx_pci_err";
-	pdata->irq = NO_IRQ;
 	platform_set_drvdata(pdev, pci);
 	pci->dev = &pdev->dev;
 	pci->dev_name = dev_name(&pdev->dev);
@@ -291,7 +290,6 @@ static int mv64x60_sram_err_probe(struct platform_device *pdev)
 
 	pdata = edac_dev->pvt_info;
 	pdata->name = "mv64x60_sram_err";
-	pdata->irq = NO_IRQ;
 	edac_dev->dev = &pdev->dev;
 	platform_set_drvdata(pdev, edac_dev);
 	edac_dev->dev_name = dev_name(&pdev->dev);
@@ -459,7 +457,6 @@ static int mv64x60_cpu_err_probe(struct platform_device *pdev)
 
 	pdata = edac_dev->pvt_info;
 	pdata->name = "mv64x60_cpu_err";
-	pdata->irq = NO_IRQ;
 	edac_dev->dev = &pdev->dev;
 	platform_set_drvdata(pdev, edac_dev);
 	edac_dev->dev_name = dev_name(&pdev->dev);
@@ -727,7 +724,6 @@ static int mv64x60_mc_err_probe(struct platform_device *pdev)
 	mci->pdev = &pdev->dev;
 	platform_set_drvdata(pdev, mci);
 	pdata->name = "mv64x60_mc_err";
-	pdata->irq = NO_IRQ;
 	mci->dev_name = dev_name(&pdev->dev);
 	pdata->edac_idx = edac_mc_idx++;
 
diff --git a/drivers/edac/ppc4xx_edac.c b/drivers/edac/ppc4xx_edac.c
index d3a64ba61fa3..691ce25e9010 100644
--- a/drivers/edac/ppc4xx_edac.c
+++ b/drivers/edac/ppc4xx_edac.c
@@ -1029,8 +1029,6 @@ static int ppc4xx_edac_mc_init(struct mem_ctl_info *mci,
 	pdata			= mci->pvt_info;
 
 	pdata->dcr_host		= *dcr_host;
-	pdata->irqs.sec		= NO_IRQ;
-	pdata->irqs.ded		= NO_IRQ;
 
 	/* Initialize controller capabilities and configuration */
 
@@ -1111,7 +1109,7 @@ static int ppc4xx_edac_register_irq(struct platform_device *op,
 	ded_irq = irq_of_parse_and_map(np, INTMAP_ECCDED_INDEX);
 	sec_irq = irq_of_parse_and_map(np, INTMAP_ECCSEC_INDEX);
 
-	if (ded_irq == NO_IRQ || sec_irq == NO_IRQ) {
+	if (!ded_irq || !sec_irq) {
 		ppc4xx_edac_mc_printk(KERN_ERR, mci,
 				      "Unable to map interrupts.\n");
 		status = -ENODEV;
diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
index 4fb2eb7c800d..54775221a01f 100644
--- a/drivers/edac/sb_edac.c
+++ b/drivers/edac/sb_edac.c
@@ -552,9 +552,9 @@ static const struct pci_id_table pci_dev_descr_haswell_table[] = {
 /* Knight's Landing Support */
 /*
  * KNL's memory channels are swizzled between memory controllers.
- * MC0 is mapped to CH3,5,6 and MC1 is mapped to CH0,1,2
+ * MC0 is mapped to CH3,4,5 and MC1 is mapped to CH0,1,2
  */
-#define knl_channel_remap(channel) ((channel + 3) % 6)
+#define knl_channel_remap(mc, chan) ((mc) ? (chan) : (chan) + 3)
 
 /* Memory controller, TAD tables, error injection - 2-8-0, 2-9-0 (2 of these) */
 #define PCI_DEVICE_ID_INTEL_KNL_IMC_MC       0x7840
@@ -1286,7 +1286,7 @@ static u32 knl_get_mc_route(int entry, u32 reg)
 	mc = GET_BITFIELD(reg, entry*3, (entry*3)+2);
 	chan = GET_BITFIELD(reg, (entry*2) + 18, (entry*2) + 18 + 1);
 
-	return knl_channel_remap(mc*3 + chan);
+	return knl_channel_remap(mc, chan);
 }
 
 /*
@@ -2474,7 +2474,7 @@ static int sbridge_mci_bind_devs(struct mem_ctl_info *mci,
 
 	/* Check if everything were registered */
 	if (!pvt->pci_sad0 || !pvt->pci_sad1 || !pvt->pci_ha0 ||
-	    !pvt-> pci_tad || !pvt->pci_ras  || !pvt->pci_ta)
+	    !pvt->pci_ras || !pvt->pci_ta)
 		goto enodev;
 
 	if (saw_chan_mask != 0x0f)
@@ -2563,8 +2563,7 @@ static int ibridge_mci_bind_devs(struct mem_ctl_info *mci,
 
 	/* Check if everything were registered */
 	if (!pvt->pci_sad0 || !pvt->pci_ha0 || !pvt->pci_br0 ||
-	    !pvt->pci_br1 || !pvt->pci_tad || !pvt->pci_ras  ||
-	    !pvt->pci_ta)
+	    !pvt->pci_br1 || !pvt->pci_ras || !pvt->pci_ta)
 		goto enodev;
 
 	if (saw_chan_mask != 0x0f && /* -EN */
@@ -2997,8 +2996,15 @@ static void sbridge_mce_output_error(struct mem_ctl_info *mci,
 		} else {
 			char A = *("A");
 
-			channel = knl_channel_remap(channel);
+			/*
+			 * Reported channel is in range 0-2, so we can't map it
+			 * back to mc. To figure out mc we check machine check
+			 * bank register that reported this error.
+			 * bank15 means mc0 and bank16 means mc1.
+			 */
+			channel = knl_channel_remap(m->bank == 16, channel);
 			channel_mask = 1 << channel;
+
 			snprintf(msg, sizeof(msg),
 				"%s%s err_code:%04x:%04x channel:%d (DIMM_%c)",
 				overflow ? " OVERFLOW" : "",
diff --git a/drivers/edac/skx_edac.c b/drivers/edac/skx_edac.c
new file mode 100644
index 000000000000..0ff4878c2aa1
--- /dev/null
+++ b/drivers/edac/skx_edac.c
@@ -0,0 +1,1121 @@
+/*
+ * EDAC driver for Intel(R) Xeon(R) Skylake processors
+ * Copyright (c) 2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/edac.h>
+#include <linux/mmzone.h>
+#include <linux/smp.h>
+#include <linux/bitmap.h>
+#include <linux/math64.h>
+#include <linux/mod_devicetable.h>
+#include <asm/cpu_device_id.h>
+#include <asm/processor.h>
+#include <asm/mce.h>
+
+#include "edac_core.h"
+
+#define SKX_REVISION    " Ver: 1.0 "
+
+/*
+ * Debug macros
+ */
+#define skx_printk(level, fmt, arg...)			\
+	edac_printk(level, "skx", fmt, ##arg)
+
+#define skx_mc_printk(mci, level, fmt, arg...)		\
+	edac_mc_chipset_printk(mci, level, "skx", fmt, ##arg)
+
+/*
+ * Get a bit field at register value <v>, from bit <lo> to bit <hi>
+ */
+#define GET_BITFIELD(v, lo, hi) \
+	(((v) & GENMASK_ULL((hi), (lo))) >> (lo))
+
+static LIST_HEAD(skx_edac_list);
+
+static u64 skx_tolm, skx_tohm;
+
+#define NUM_IMC			2	/* memory controllers per socket */
+#define NUM_CHANNELS		3	/* channels per memory controller */
+#define NUM_DIMMS		2	/* Max DIMMS per channel */
+
+#define	MASK26	0x3FFFFFF		/* Mask for 2^26 */
+#define MASK29	0x1FFFFFFF		/* Mask for 2^29 */
+
+/*
+ * Each cpu socket contains some pci devices that provide global
+ * information, and also some that are local to each of the two
+ * memory controllers on the die.
+ */
+struct skx_dev {
+	struct list_head	list;
+	u8			bus[4];
+	struct pci_dev	*sad_all;
+	struct pci_dev	*util_all;
+	u32	mcroute;
+	struct skx_imc {
+		struct mem_ctl_info *mci;
+		u8	mc;	/* system wide mc# */
+		u8	lmc;	/* socket relative mc# */
+		u8	src_id, node_id;
+		struct skx_channel {
+			struct pci_dev *cdev;
+			struct skx_dimm {
+				u8	close_pg;
+				u8	bank_xor_enable;
+				u8	fine_grain_bank;
+				u8	rowbits;
+				u8	colbits;
+			} dimms[NUM_DIMMS];
+		} chan[NUM_CHANNELS];
+	} imc[NUM_IMC];
+};
+static int skx_num_sockets;
+
+struct skx_pvt {
+	struct skx_imc	*imc;
+};
+
+struct decoded_addr {
+	struct skx_dev *dev;
+	u64	addr;
+	int	socket;
+	int	imc;
+	int	channel;
+	u64	chan_addr;
+	int	sktways;
+	int	chanways;
+	int	dimm;
+	int	rank;
+	int	channel_rank;
+	u64	rank_address;
+	int	row;
+	int	column;
+	int	bank_address;
+	int	bank_group;
+};
+
+static struct skx_dev *get_skx_dev(u8 bus, u8 idx)
+{
+	struct skx_dev *d;
+
+	list_for_each_entry(d, &skx_edac_list, list) {
+		if (d->bus[idx] == bus)
+			return d;
+	}
+
+	return NULL;
+}
+
+enum munittype {
+	CHAN0, CHAN1, CHAN2, SAD_ALL, UTIL_ALL, SAD
+};
+
+struct munit {
+	u16	did;
+	u16	devfn[NUM_IMC];
+	u8	busidx;
+	u8	per_socket;
+	enum munittype mtype;
+};
+
+/*
+ * List of PCI device ids that we need together with some device
+ * number and function numbers to tell which memory controller the
+ * device belongs to.
+ */
+static const struct munit skx_all_munits[] = {
+	{ 0x2054, { }, 1, 1, SAD_ALL },
+	{ 0x2055, { }, 1, 1, UTIL_ALL },
+	{ 0x2040, { PCI_DEVFN(10, 0), PCI_DEVFN(12, 0) }, 2, 2, CHAN0 },
+	{ 0x2044, { PCI_DEVFN(10, 4), PCI_DEVFN(12, 4) }, 2, 2, CHAN1 },
+	{ 0x2048, { PCI_DEVFN(11, 0), PCI_DEVFN(13, 0) }, 2, 2, CHAN2 },
+	{ 0x208e, { }, 1, 0, SAD },
+	{ }
+};
+
+/*
+ * We use the per-socket device 0x2016 to count how many sockets are present,
+ * and to detemine which PCI buses are associated with each socket. Allocate
+ * and build the full list of all the skx_dev structures that we need here.
+ */
+static int get_all_bus_mappings(void)
+{
+	struct pci_dev *pdev, *prev;
+	struct skx_dev *d;
+	u32 reg;
+	int ndev = 0;
+
+	prev = NULL;
+	for (;;) {
+		pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x2016, prev);
+		if (!pdev)
+			break;
+		ndev++;
+		d = kzalloc(sizeof(*d), GFP_KERNEL);
+		if (!d) {
+			pci_dev_put(pdev);
+			return -ENOMEM;
+		}
+		pci_read_config_dword(pdev, 0xCC, &reg);
+		d->bus[0] =  GET_BITFIELD(reg, 0, 7);
+		d->bus[1] =  GET_BITFIELD(reg, 8, 15);
+		d->bus[2] =  GET_BITFIELD(reg, 16, 23);
+		d->bus[3] =  GET_BITFIELD(reg, 24, 31);
+		edac_dbg(2, "busses: %x, %x, %x, %x\n",
+			 d->bus[0], d->bus[1], d->bus[2], d->bus[3]);
+		list_add_tail(&d->list, &skx_edac_list);
+		skx_num_sockets++;
+		prev = pdev;
+	}
+
+	return ndev;
+}
+
+static int get_all_munits(const struct munit *m)
+{
+	struct pci_dev *pdev, *prev;
+	struct skx_dev *d;
+	u32 reg;
+	int i = 0, ndev = 0;
+
+	prev = NULL;
+	for (;;) {
+		pdev = pci_get_device(PCI_VENDOR_ID_INTEL, m->did, prev);
+		if (!pdev)
+			break;
+		ndev++;
+		if (m->per_socket == NUM_IMC) {
+			for (i = 0; i < NUM_IMC; i++)
+				if (m->devfn[i] == pdev->devfn)
+					break;
+			if (i == NUM_IMC)
+				goto fail;
+		}
+		d = get_skx_dev(pdev->bus->number, m->busidx);
+		if (!d)
+			goto fail;
+
+		/* Be sure that the device is enabled */
+		if (unlikely(pci_enable_device(pdev) < 0)) {
+			skx_printk(KERN_ERR,
+				"Couldn't enable %04x:%04x\n", PCI_VENDOR_ID_INTEL, m->did);
+			goto fail;
+		}
+
+		switch (m->mtype) {
+		case CHAN0: case CHAN1: case CHAN2:
+			pci_dev_get(pdev);
+			d->imc[i].chan[m->mtype].cdev = pdev;
+			break;
+		case SAD_ALL:
+			pci_dev_get(pdev);
+			d->sad_all = pdev;
+			break;
+		case UTIL_ALL:
+			pci_dev_get(pdev);
+			d->util_all = pdev;
+			break;
+		case SAD:
+			/*
+			 * one of these devices per core, including cores
+			 * that don't exist on this SKU. Ignore any that
+			 * read a route table of zero, make sure all the
+			 * non-zero values match.
+			 */
+			pci_read_config_dword(pdev, 0xB4, &reg);
+			if (reg != 0) {
+				if (d->mcroute == 0)
+					d->mcroute = reg;
+				else if (d->mcroute != reg) {
+					skx_printk(KERN_ERR,
+						"mcroute mismatch\n");
+					goto fail;
+				}
+			}
+			ndev--;
+			break;
+		}
+
+		prev = pdev;
+	}
+
+	return ndev;
+fail:
+	pci_dev_put(pdev);
+	return -ENODEV;
+}
+
+const struct x86_cpu_id skx_cpuids[] = {
+	{ X86_VENDOR_INTEL, 6, 0x55, 0, 0 },	/* Skylake */
+	{ }
+};
+MODULE_DEVICE_TABLE(x86cpu, skx_cpuids);
+
+static u8 get_src_id(struct skx_dev *d)
+{
+	u32 reg;
+
+	pci_read_config_dword(d->util_all, 0xF0, &reg);
+
+	return GET_BITFIELD(reg, 12, 14);
+}
+
+static u8 skx_get_node_id(struct skx_dev *d)
+{
+	u32 reg;
+
+	pci_read_config_dword(d->util_all, 0xF4, &reg);
+
+	return GET_BITFIELD(reg, 0, 2);
+}
+
+static int get_dimm_attr(u32 reg, int lobit, int hibit, int add, int minval,
+			 int maxval, char *name)
+{
+	u32 val = GET_BITFIELD(reg, lobit, hibit);
+
+	if (val < minval || val > maxval) {
+		edac_dbg(2, "bad %s = %d (raw=%x)\n", name, val, reg);
+		return -EINVAL;
+	}
+	return val + add;
+}
+
+#define IS_DIMM_PRESENT(mtr)		GET_BITFIELD((mtr), 15, 15)
+
+#define numrank(reg) get_dimm_attr((reg), 12, 13, 0, 1, 2, "ranks")
+#define numrow(reg) get_dimm_attr((reg), 2, 4, 12, 1, 6, "rows")
+#define numcol(reg) get_dimm_attr((reg), 0, 1, 10, 0, 2, "cols")
+
+static int get_width(u32 mtr)
+{
+	switch (GET_BITFIELD(mtr, 8, 9)) {
+	case 0:
+		return DEV_X4;
+	case 1:
+		return DEV_X8;
+	case 2:
+		return DEV_X16;
+	}
+	return DEV_UNKNOWN;
+}
+
+static int skx_get_hi_lo(void)
+{
+	struct pci_dev *pdev;
+	u32 reg;
+
+	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x2034, NULL);
+	if (!pdev) {
+		edac_dbg(0, "Can't get tolm/tohm\n");
+		return -ENODEV;
+	}
+
+	pci_read_config_dword(pdev, 0xD0, &reg);
+	skx_tolm = reg;
+	pci_read_config_dword(pdev, 0xD4, &reg);
+	skx_tohm = reg;
+	pci_read_config_dword(pdev, 0xD8, &reg);
+	skx_tohm |= (u64)reg << 32;
+
+	pci_dev_put(pdev);
+	edac_dbg(2, "tolm=%llx tohm=%llx\n", skx_tolm, skx_tohm);
+
+	return 0;
+}
+
+static int get_dimm_info(u32 mtr, u32 amap, struct dimm_info *dimm,
+			 struct skx_imc *imc, int chan, int dimmno)
+{
+	int  banks = 16, ranks, rows, cols, npages;
+	u64 size;
+
+	if (!IS_DIMM_PRESENT(mtr))
+		return 0;
+	ranks = numrank(mtr);
+	rows = numrow(mtr);
+	cols = numcol(mtr);
+
+	/*
+	 * Compute size in 8-byte (2^3) words, then shift to MiB (2^20)
+	 */
+	size = ((1ull << (rows + cols + ranks)) * banks) >> (20 - 3);
+	npages = MiB_TO_PAGES(size);
+
+	edac_dbg(0, "mc#%d: channel %d, dimm %d, %lld Mb (%d pages) bank: %d, rank: %d, row: %#x, col: %#x\n",
+		 imc->mc, chan, dimmno, size, npages,
+		 banks, ranks, rows, cols);
+
+	imc->chan[chan].dimms[dimmno].close_pg = GET_BITFIELD(mtr, 0, 0);
+	imc->chan[chan].dimms[dimmno].bank_xor_enable = GET_BITFIELD(mtr, 9, 9);
+	imc->chan[chan].dimms[dimmno].fine_grain_bank = GET_BITFIELD(amap, 0, 0);
+	imc->chan[chan].dimms[dimmno].rowbits = rows;
+	imc->chan[chan].dimms[dimmno].colbits = cols;
+
+	dimm->nr_pages = npages;
+	dimm->grain = 32;
+	dimm->dtype = get_width(mtr);
+	dimm->mtype = MEM_DDR4;
+	dimm->edac_mode = EDAC_SECDED; /* likely better than this */
+	snprintf(dimm->label, sizeof(dimm->label), "CPU_SrcID#%u_MC#%u_Chan#%u_DIMM#%u",
+		 imc->src_id, imc->lmc, chan, dimmno);
+
+	return 1;
+}
+
+#define SKX_GET_MTMTR(dev, reg) \
+	pci_read_config_dword((dev), 0x87c, &reg)
+
+static bool skx_check_ecc(struct pci_dev *pdev)
+{
+	u32 mtmtr;
+
+	SKX_GET_MTMTR(pdev, mtmtr);
+
+	return !!GET_BITFIELD(mtmtr, 2, 2);
+}
+
+static int skx_get_dimm_config(struct mem_ctl_info *mci)
+{
+	struct skx_pvt *pvt = mci->pvt_info;
+	struct skx_imc *imc = pvt->imc;
+	struct dimm_info *dimm;
+	int i, j;
+	u32 mtr, amap;
+	int ndimms;
+
+	for (i = 0; i < NUM_CHANNELS; i++) {
+		ndimms = 0;
+		pci_read_config_dword(imc->chan[i].cdev, 0x8C, &amap);
+		for (j = 0; j < NUM_DIMMS; j++) {
+			dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms,
+					     mci->n_layers, i, j, 0);
+			pci_read_config_dword(imc->chan[i].cdev,
+					0x80 + 4*j, &mtr);
+			ndimms += get_dimm_info(mtr, amap, dimm, imc, i, j);
+		}
+		if (ndimms && !skx_check_ecc(imc->chan[0].cdev)) {
+			skx_printk(KERN_ERR, "ECC is disabled on imc %d\n", imc->mc);
+			return -ENODEV;
+		}
+	}
+
+	return 0;
+}
+
+static void skx_unregister_mci(struct skx_imc *imc)
+{
+	struct mem_ctl_info *mci = imc->mci;
+
+	if (!mci)
+		return;
+
+	edac_dbg(0, "MC%d: mci = %p\n", imc->mc, mci);
+
+	/* Remove MC sysfs nodes */
+	edac_mc_del_mc(mci->pdev);
+
+	edac_dbg(1, "%s: free mci struct\n", mci->ctl_name);
+	kfree(mci->ctl_name);
+	edac_mc_free(mci);
+}
+
+static int skx_register_mci(struct skx_imc *imc)
+{
+	struct mem_ctl_info *mci;
+	struct edac_mc_layer layers[2];
+	struct pci_dev *pdev = imc->chan[0].cdev;
+	struct skx_pvt *pvt;
+	int rc;
+
+	/* allocate a new MC control structure */
+	layers[0].type = EDAC_MC_LAYER_CHANNEL;
+	layers[0].size = NUM_CHANNELS;
+	layers[0].is_virt_csrow = false;
+	layers[1].type = EDAC_MC_LAYER_SLOT;
+	layers[1].size = NUM_DIMMS;
+	layers[1].is_virt_csrow = true;
+	mci = edac_mc_alloc(imc->mc, ARRAY_SIZE(layers), layers,
+			    sizeof(struct skx_pvt));
+
+	if (unlikely(!mci))
+		return -ENOMEM;
+
+	edac_dbg(0, "MC#%d: mci = %p\n", imc->mc, mci);
+
+	/* Associate skx_dev and mci for future usage */
+	imc->mci = mci;
+	pvt = mci->pvt_info;
+	pvt->imc = imc;
+
+	mci->ctl_name = kasprintf(GFP_KERNEL, "Skylake Socket#%d IMC#%d",
+				  imc->node_id, imc->lmc);
+	mci->mtype_cap = MEM_FLAG_DDR4;
+	mci->edac_ctl_cap = EDAC_FLAG_NONE;
+	mci->edac_cap = EDAC_FLAG_NONE;
+	mci->mod_name = "skx_edac.c";
+	mci->dev_name = pci_name(imc->chan[0].cdev);
+	mci->mod_ver = SKX_REVISION;
+	mci->ctl_page_to_phys = NULL;
+
+	rc = skx_get_dimm_config(mci);
+	if (rc < 0)
+		goto fail;
+
+	/* record ptr to the generic device */
+	mci->pdev = &pdev->dev;
+
+	/* add this new MC control structure to EDAC's list of MCs */
+	if (unlikely(edac_mc_add_mc(mci))) {
+		edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
+		rc = -EINVAL;
+		goto fail;
+	}
+
+	return 0;
+
+fail:
+	kfree(mci->ctl_name);
+	edac_mc_free(mci);
+	imc->mci = NULL;
+	return rc;
+}
+
+#define	SKX_MAX_SAD 24
+
+#define SKX_GET_SAD(d, i, reg)	\
+	pci_read_config_dword((d)->sad_all, 0x60 + 8 * (i), &reg)
+#define SKX_GET_ILV(d, i, reg)	\
+	pci_read_config_dword((d)->sad_all, 0x64 + 8 * (i), &reg)
+
+#define	SKX_SAD_MOD3MODE(sad)	GET_BITFIELD((sad), 30, 31)
+#define	SKX_SAD_MOD3(sad)	GET_BITFIELD((sad), 27, 27)
+#define SKX_SAD_LIMIT(sad)	(((u64)GET_BITFIELD((sad), 7, 26) << 26) | MASK26)
+#define	SKX_SAD_MOD3ASMOD2(sad)	GET_BITFIELD((sad), 5, 6)
+#define	SKX_SAD_ATTR(sad)	GET_BITFIELD((sad), 3, 4)
+#define	SKX_SAD_INTERLEAVE(sad)	GET_BITFIELD((sad), 1, 2)
+#define SKX_SAD_ENABLE(sad)	GET_BITFIELD((sad), 0, 0)
+
+#define SKX_ILV_REMOTE(tgt)	(((tgt) & 8) == 0)
+#define SKX_ILV_TARGET(tgt)	((tgt) & 7)
+
+static bool skx_sad_decode(struct decoded_addr *res)
+{
+	struct skx_dev *d = list_first_entry(&skx_edac_list, typeof(*d), list);
+	u64 addr = res->addr;
+	int i, idx, tgt, lchan, shift;
+	u32 sad, ilv;
+	u64 limit, prev_limit;
+	int remote = 0;
+
+	/* Simple sanity check for I/O space or out of range */
+	if (addr >= skx_tohm || (addr >= skx_tolm && addr < BIT_ULL(32))) {
+		edac_dbg(0, "Address %llx out of range\n", addr);
+		return false;
+	}
+
+restart:
+	prev_limit = 0;
+	for (i = 0; i < SKX_MAX_SAD; i++) {
+		SKX_GET_SAD(d, i, sad);
+		limit = SKX_SAD_LIMIT(sad);
+		if (SKX_SAD_ENABLE(sad)) {
+			if (addr >= prev_limit && addr <= limit)
+				goto sad_found;
+		}
+		prev_limit = limit + 1;
+	}
+	edac_dbg(0, "No SAD entry for %llx\n", addr);
+	return false;
+
+sad_found:
+	SKX_GET_ILV(d, i, ilv);
+
+	switch (SKX_SAD_INTERLEAVE(sad)) {
+	case 0:
+		idx = GET_BITFIELD(addr, 6, 8);
+		break;
+	case 1:
+		idx = GET_BITFIELD(addr, 8, 10);
+		break;
+	case 2:
+		idx = GET_BITFIELD(addr, 12, 14);
+		break;
+	case 3:
+		idx = GET_BITFIELD(addr, 30, 32);
+		break;
+	}
+
+	tgt = GET_BITFIELD(ilv, 4 * idx, 4 * idx + 3);
+
+	/* If point to another node, find it and start over */
+	if (SKX_ILV_REMOTE(tgt)) {
+		if (remote) {
+			edac_dbg(0, "Double remote!\n");
+			return false;
+		}
+		remote = 1;
+		list_for_each_entry(d, &skx_edac_list, list) {
+			if (d->imc[0].src_id == SKX_ILV_TARGET(tgt))
+				goto restart;
+		}
+		edac_dbg(0, "Can't find node %d\n", SKX_ILV_TARGET(tgt));
+		return false;
+	}
+
+	if (SKX_SAD_MOD3(sad) == 0)
+		lchan = SKX_ILV_TARGET(tgt);
+	else {
+		switch (SKX_SAD_MOD3MODE(sad)) {
+		case 0:
+			shift = 6;
+			break;
+		case 1:
+			shift = 8;
+			break;
+		case 2:
+			shift = 12;
+			break;
+		default:
+			edac_dbg(0, "illegal mod3mode\n");
+			return false;
+		}
+		switch (SKX_SAD_MOD3ASMOD2(sad)) {
+		case 0:
+			lchan = (addr >> shift) % 3;
+			break;
+		case 1:
+			lchan = (addr >> shift) % 2;
+			break;
+		case 2:
+			lchan = (addr >> shift) % 2;
+			lchan = (lchan << 1) | ~lchan;
+			break;
+		case 3:
+			lchan = ((addr >> shift) % 2) << 1;
+			break;
+		}
+		lchan = (lchan << 1) | (SKX_ILV_TARGET(tgt) & 1);
+	}
+
+	res->dev = d;
+	res->socket = d->imc[0].src_id;
+	res->imc = GET_BITFIELD(d->mcroute, lchan * 3, lchan * 3 + 2);
+	res->channel = GET_BITFIELD(d->mcroute, lchan * 2 + 18, lchan * 2 + 19);
+
+	edac_dbg(2, "%llx: socket=%d imc=%d channel=%d\n",
+		 res->addr, res->socket, res->imc, res->channel);
+	return true;
+}
+
+#define	SKX_MAX_TAD 8
+
+#define SKX_GET_TADBASE(d, mc, i, reg)			\
+	pci_read_config_dword((d)->imc[mc].chan[0].cdev, 0x850 + 4 * (i), &reg)
+#define SKX_GET_TADWAYNESS(d, mc, i, reg)		\
+	pci_read_config_dword((d)->imc[mc].chan[0].cdev, 0x880 + 4 * (i), &reg)
+#define SKX_GET_TADCHNILVOFFSET(d, mc, ch, i, reg)	\
+	pci_read_config_dword((d)->imc[mc].chan[ch].cdev, 0x90 + 4 * (i), &reg)
+
+#define	SKX_TAD_BASE(b)		((u64)GET_BITFIELD((b), 12, 31) << 26)
+#define SKX_TAD_SKT_GRAN(b)	GET_BITFIELD((b), 4, 5)
+#define SKX_TAD_CHN_GRAN(b)	GET_BITFIELD((b), 6, 7)
+#define	SKX_TAD_LIMIT(b)	(((u64)GET_BITFIELD((b), 12, 31) << 26) | MASK26)
+#define	SKX_TAD_OFFSET(b)	((u64)GET_BITFIELD((b), 4, 23) << 26)
+#define	SKX_TAD_SKTWAYS(b)	(1 << GET_BITFIELD((b), 10, 11))
+#define	SKX_TAD_CHNWAYS(b)	(GET_BITFIELD((b), 8, 9) + 1)
+
+/* which bit used for both socket and channel interleave */
+static int skx_granularity[] = { 6, 8, 12, 30 };
+
+static u64 skx_do_interleave(u64 addr, int shift, int ways, u64 lowbits)
+{
+	addr >>= shift;
+	addr /= ways;
+	addr <<= shift;
+
+	return addr | (lowbits & ((1ull << shift) - 1));
+}
+
+static bool skx_tad_decode(struct decoded_addr *res)
+{
+	int i;
+	u32 base, wayness, chnilvoffset;
+	int skt_interleave_bit, chn_interleave_bit;
+	u64 channel_addr;
+
+	for (i = 0; i < SKX_MAX_TAD; i++) {
+		SKX_GET_TADBASE(res->dev, res->imc, i, base);
+		SKX_GET_TADWAYNESS(res->dev, res->imc, i, wayness);
+		if (SKX_TAD_BASE(base) <= res->addr && res->addr <= SKX_TAD_LIMIT(wayness))
+			goto tad_found;
+	}
+	edac_dbg(0, "No TAD entry for %llx\n", res->addr);
+	return false;
+
+tad_found:
+	res->sktways = SKX_TAD_SKTWAYS(wayness);
+	res->chanways = SKX_TAD_CHNWAYS(wayness);
+	skt_interleave_bit = skx_granularity[SKX_TAD_SKT_GRAN(base)];
+	chn_interleave_bit = skx_granularity[SKX_TAD_CHN_GRAN(base)];
+
+	SKX_GET_TADCHNILVOFFSET(res->dev, res->imc, res->channel, i, chnilvoffset);
+	channel_addr = res->addr - SKX_TAD_OFFSET(chnilvoffset);
+
+	if (res->chanways == 3 && skt_interleave_bit > chn_interleave_bit) {
+		/* Must handle channel first, then socket */
+		channel_addr = skx_do_interleave(channel_addr, chn_interleave_bit,
+						 res->chanways, channel_addr);
+		channel_addr = skx_do_interleave(channel_addr, skt_interleave_bit,
+						 res->sktways, channel_addr);
+	} else {
+		/* Handle socket then channel. Preserve low bits from original address */
+		channel_addr = skx_do_interleave(channel_addr, skt_interleave_bit,
+						 res->sktways, res->addr);
+		channel_addr = skx_do_interleave(channel_addr, chn_interleave_bit,
+						 res->chanways, res->addr);
+	}
+
+	res->chan_addr = channel_addr;
+
+	edac_dbg(2, "%llx: chan_addr=%llx sktways=%d chanways=%d\n",
+		 res->addr, res->chan_addr, res->sktways, res->chanways);
+	return true;
+}
+
+#define SKX_MAX_RIR 4
+
+#define SKX_GET_RIRWAYNESS(d, mc, ch, i, reg)		\
+	pci_read_config_dword((d)->imc[mc].chan[ch].cdev,	\
+			      0x108 + 4 * (i), &reg)
+#define SKX_GET_RIRILV(d, mc, ch, idx, i, reg)		\
+	pci_read_config_dword((d)->imc[mc].chan[ch].cdev,	\
+			      0x120 + 16 * idx + 4 * (i), &reg)
+
+#define	SKX_RIR_VALID(b) GET_BITFIELD((b), 31, 31)
+#define	SKX_RIR_LIMIT(b) (((u64)GET_BITFIELD((b), 1, 11) << 29) | MASK29)
+#define	SKX_RIR_WAYS(b) (1 << GET_BITFIELD((b), 28, 29))
+#define	SKX_RIR_CHAN_RANK(b) GET_BITFIELD((b), 16, 19)
+#define	SKX_RIR_OFFSET(b) ((u64)(GET_BITFIELD((b), 2, 15) << 26))
+
+static bool skx_rir_decode(struct decoded_addr *res)
+{
+	int i, idx, chan_rank;
+	int shift;
+	u32 rirway, rirlv;
+	u64 rank_addr, prev_limit = 0, limit;
+
+	if (res->dev->imc[res->imc].chan[res->channel].dimms[0].close_pg)
+		shift = 6;
+	else
+		shift = 13;
+
+	for (i = 0; i < SKX_MAX_RIR; i++) {
+		SKX_GET_RIRWAYNESS(res->dev, res->imc, res->channel, i, rirway);
+		limit = SKX_RIR_LIMIT(rirway);
+		if (SKX_RIR_VALID(rirway)) {
+			if (prev_limit <= res->chan_addr &&
+			    res->chan_addr <= limit)
+				goto rir_found;
+		}
+		prev_limit = limit;
+	}
+	edac_dbg(0, "No RIR entry for %llx\n", res->addr);
+	return false;
+
+rir_found:
+	rank_addr = res->chan_addr >> shift;
+	rank_addr /= SKX_RIR_WAYS(rirway);
+	rank_addr <<= shift;
+	rank_addr |= res->chan_addr & GENMASK_ULL(shift - 1, 0);
+
+	res->rank_address = rank_addr;
+	idx = (res->chan_addr >> shift) % SKX_RIR_WAYS(rirway);
+
+	SKX_GET_RIRILV(res->dev, res->imc, res->channel, idx, i, rirlv);
+	res->rank_address = rank_addr - SKX_RIR_OFFSET(rirlv);
+	chan_rank = SKX_RIR_CHAN_RANK(rirlv);
+	res->channel_rank = chan_rank;
+	res->dimm = chan_rank / 4;
+	res->rank = chan_rank % 4;
+
+	edac_dbg(2, "%llx: dimm=%d rank=%d chan_rank=%d rank_addr=%llx\n",
+		 res->addr, res->dimm, res->rank,
+		 res->channel_rank, res->rank_address);
+	return true;
+}
+
+static u8 skx_close_row[] = {
+	15, 16, 17, 18, 20, 21, 22, 28, 10, 11, 12, 13, 29, 30, 31, 32, 33
+};
+static u8 skx_close_column[] = {
+	3, 4, 5, 14, 19, 23, 24, 25, 26, 27
+};
+static u8 skx_open_row[] = {
+	14, 15, 16, 20, 28, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31, 32, 33
+};
+static u8 skx_open_column[] = {
+	3, 4, 5, 6, 7, 8, 9, 10, 11, 12
+};
+static u8 skx_open_fine_column[] = {
+	3, 4, 5, 7, 8, 9, 10, 11, 12, 13
+};
+
+static int skx_bits(u64 addr, int nbits, u8 *bits)
+{
+	int i, res = 0;
+
+	for (i = 0; i < nbits; i++)
+		res |= ((addr >> bits[i]) & 1) << i;
+	return res;
+}
+
+static int skx_bank_bits(u64 addr, int b0, int b1, int do_xor, int x0, int x1)
+{
+	int ret = GET_BITFIELD(addr, b0, b0) | (GET_BITFIELD(addr, b1, b1) << 1);
+
+	if (do_xor)
+		ret ^= GET_BITFIELD(addr, x0, x0) | (GET_BITFIELD(addr, x1, x1) << 1);
+
+	return ret;
+}
+
+static bool skx_mad_decode(struct decoded_addr *r)
+{
+	struct skx_dimm *dimm = &r->dev->imc[r->imc].chan[r->channel].dimms[r->dimm];
+	int bg0 = dimm->fine_grain_bank ? 6 : 13;
+
+	if (dimm->close_pg) {
+		r->row = skx_bits(r->rank_address, dimm->rowbits, skx_close_row);
+		r->column = skx_bits(r->rank_address, dimm->colbits, skx_close_column);
+		r->column |= 0x400; /* C10 is autoprecharge, always set */
+		r->bank_address = skx_bank_bits(r->rank_address, 8, 9, dimm->bank_xor_enable, 22, 28);
+		r->bank_group = skx_bank_bits(r->rank_address, 6, 7, dimm->bank_xor_enable, 20, 21);
+	} else {
+		r->row = skx_bits(r->rank_address, dimm->rowbits, skx_open_row);
+		if (dimm->fine_grain_bank)
+			r->column = skx_bits(r->rank_address, dimm->colbits, skx_open_fine_column);
+		else
+			r->column = skx_bits(r->rank_address, dimm->colbits, skx_open_column);
+		r->bank_address = skx_bank_bits(r->rank_address, 18, 19, dimm->bank_xor_enable, 22, 23);
+		r->bank_group = skx_bank_bits(r->rank_address, bg0, 17, dimm->bank_xor_enable, 20, 21);
+	}
+	r->row &= (1u << dimm->rowbits) - 1;
+
+	edac_dbg(2, "%llx: row=%x col=%x bank_addr=%d bank_group=%d\n",
+		 r->addr, r->row, r->column, r->bank_address,
+		 r->bank_group);
+	return true;
+}
+
+static bool skx_decode(struct decoded_addr *res)
+{
+
+	return skx_sad_decode(res) && skx_tad_decode(res) &&
+		skx_rir_decode(res) && skx_mad_decode(res);
+}
+
+#ifdef CONFIG_EDAC_DEBUG
+/*
+ * Debug feature. Make /sys/kernel/debug/skx_edac_test/addr.
+ * Write an address to this file to exercise the address decode
+ * logic in this driver.
+ */
+static struct dentry *skx_test;
+static u64 skx_fake_addr;
+
+static int debugfs_u64_set(void *data, u64 val)
+{
+	struct decoded_addr res;
+
+	res.addr = val;
+	skx_decode(&res);
+
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");
+
+static struct dentry *mydebugfs_create(const char *name, umode_t mode,
+				       struct dentry *parent, u64 *value)
+{
+	return debugfs_create_file(name, mode, parent, value, &fops_u64_wo);
+}
+
+static void setup_skx_debug(void)
+{
+	skx_test = debugfs_create_dir("skx_edac_test", NULL);
+	mydebugfs_create("addr", S_IWUSR, skx_test, &skx_fake_addr);
+}
+
+static void teardown_skx_debug(void)
+{
+	debugfs_remove_recursive(skx_test);
+}
+#else
+static void setup_skx_debug(void)
+{
+}
+
+static void teardown_skx_debug(void)
+{
+}
+#endif /*CONFIG_EDAC_DEBUG*/
+
+static void skx_mce_output_error(struct mem_ctl_info *mci,
+				 const struct mce *m,
+				 struct decoded_addr *res)
+{
+	enum hw_event_mc_err_type tp_event;
+	char *type, *optype, msg[256];
+	bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0);
+	bool overflow = GET_BITFIELD(m->status, 62, 62);
+	bool uncorrected_error = GET_BITFIELD(m->status, 61, 61);
+	bool recoverable;
+	u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52);
+	u32 mscod = GET_BITFIELD(m->status, 16, 31);
+	u32 errcode = GET_BITFIELD(m->status, 0, 15);
+	u32 optypenum = GET_BITFIELD(m->status, 4, 6);
+
+	recoverable = GET_BITFIELD(m->status, 56, 56);
+
+	if (uncorrected_error) {
+		if (ripv) {
+			type = "FATAL";
+			tp_event = HW_EVENT_ERR_FATAL;
+		} else {
+			type = "NON_FATAL";
+			tp_event = HW_EVENT_ERR_UNCORRECTED;
+		}
+	} else {
+		type = "CORRECTED";
+		tp_event = HW_EVENT_ERR_CORRECTED;
+	}
+
+	/*
+	 * According with Table 15-9 of the Intel Architecture spec vol 3A,
+	 * memory errors should fit in this mask:
+	 *	000f 0000 1mmm cccc (binary)
+	 * where:
+	 *	f = Correction Report Filtering Bit. If 1, subsequent errors
+	 *	    won't be shown
+	 *	mmm = error type
+	 *	cccc = channel
+	 * If the mask doesn't match, report an error to the parsing logic
+	 */
+	if (!((errcode & 0xef80) == 0x80)) {
+		optype = "Can't parse: it is not a mem";
+	} else {
+		switch (optypenum) {
+		case 0:
+			optype = "generic undef request error";
+			break;
+		case 1:
+			optype = "memory read error";
+			break;
+		case 2:
+			optype = "memory write error";
+			break;
+		case 3:
+			optype = "addr/cmd error";
+			break;
+		case 4:
+			optype = "memory scrubbing error";
+			break;
+		default:
+			optype = "reserved";
+			break;
+		}
+	}
+
+	snprintf(msg, sizeof(msg),
+		 "%s%s err_code:%04x:%04x socket:%d imc:%d rank:%d bg:%d ba:%d row:%x col:%x",
+		 overflow ? " OVERFLOW" : "",
+		 (uncorrected_error && recoverable) ? " recoverable" : "",
+		 mscod, errcode,
+		 res->socket, res->imc, res->rank,
+		 res->bank_group, res->bank_address, res->row, res->column);
+
+	edac_dbg(0, "%s\n", msg);
+
+	/* Call the helper to output message */
+	edac_mc_handle_error(tp_event, mci, core_err_cnt,
+			     m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0,
+			     res->channel, res->dimm, -1,
+			     optype, msg);
+}
+
+static int skx_mce_check_error(struct notifier_block *nb, unsigned long val,
+			       void *data)
+{
+	struct mce *mce = (struct mce *)data;
+	struct decoded_addr res;
+	struct mem_ctl_info *mci;
+	char *type;
+
+	if (get_edac_report_status() == EDAC_REPORTING_DISABLED)
+		return NOTIFY_DONE;
+
+	/* ignore unless this is memory related with an address */
+	if ((mce->status & 0xefff) >> 7 != 1 || !(mce->status & MCI_STATUS_ADDRV))
+		return NOTIFY_DONE;
+
+	res.addr = mce->addr;
+	if (!skx_decode(&res))
+		return NOTIFY_DONE;
+	mci = res.dev->imc[res.imc].mci;
+
+	if (mce->mcgstatus & MCG_STATUS_MCIP)
+		type = "Exception";
+	else
+		type = "Event";
+
+	skx_mc_printk(mci, KERN_DEBUG, "HANDLING MCE MEMORY ERROR\n");
+
+	skx_mc_printk(mci, KERN_DEBUG, "CPU %d: Machine Check %s: %Lx "
+			  "Bank %d: %016Lx\n", mce->extcpu, type,
+			  mce->mcgstatus, mce->bank, mce->status);
+	skx_mc_printk(mci, KERN_DEBUG, "TSC %llx ", mce->tsc);
+	skx_mc_printk(mci, KERN_DEBUG, "ADDR %llx ", mce->addr);
+	skx_mc_printk(mci, KERN_DEBUG, "MISC %llx ", mce->misc);
+
+	skx_mc_printk(mci, KERN_DEBUG, "PROCESSOR %u:%x TIME %llu SOCKET "
+			  "%u APIC %x\n", mce->cpuvendor, mce->cpuid,
+			  mce->time, mce->socketid, mce->apicid);
+
+	skx_mce_output_error(mci, mce, &res);
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block skx_mce_dec = {
+	.notifier_call = skx_mce_check_error,
+};
+
+static void skx_remove(void)
+{
+	int i, j;
+	struct skx_dev *d, *tmp;
+
+	edac_dbg(0, "\n");
+
+	list_for_each_entry_safe(d, tmp, &skx_edac_list, list) {
+		list_del(&d->list);
+		for (i = 0; i < NUM_IMC; i++) {
+			skx_unregister_mci(&d->imc[i]);
+			for (j = 0; j < NUM_CHANNELS; j++)
+				pci_dev_put(d->imc[i].chan[j].cdev);
+		}
+		pci_dev_put(d->util_all);
+		pci_dev_put(d->sad_all);
+
+		kfree(d);
+	}
+}
+
+/*
+ * skx_init:
+ *	make sure we are running on the correct cpu model
+ *	search for all the devices we need
+ *	check which DIMMs are present.
+ */
+int __init skx_init(void)
+{
+	const struct x86_cpu_id *id;
+	const struct munit *m;
+	int rc = 0, i;
+	u8 mc = 0, src_id, node_id;
+	struct skx_dev *d;
+
+	edac_dbg(2, "\n");
+
+	id = x86_match_cpu(skx_cpuids);
+	if (!id)
+		return -ENODEV;
+
+	rc = skx_get_hi_lo();
+	if (rc)
+		return rc;
+
+	rc = get_all_bus_mappings();
+	if (rc < 0)
+		goto fail;
+	if (rc == 0) {
+		edac_dbg(2, "No memory controllers found\n");
+		return -ENODEV;
+	}
+
+	for (m = skx_all_munits; m->did; m++) {
+		rc = get_all_munits(m);
+		if (rc < 0)
+			goto fail;
+		if (rc != m->per_socket * skx_num_sockets) {
+			edac_dbg(2, "Expected %d, got %d of %x\n",
+				 m->per_socket * skx_num_sockets, rc, m->did);
+			rc = -ENODEV;
+			goto fail;
+		}
+	}
+
+	list_for_each_entry(d, &skx_edac_list, list) {
+		src_id = get_src_id(d);
+		node_id = skx_get_node_id(d);
+		edac_dbg(2, "src_id=%d node_id=%d\n", src_id, node_id);
+		for (i = 0; i < NUM_IMC; i++) {
+			d->imc[i].mc = mc++;
+			d->imc[i].lmc = i;
+			d->imc[i].src_id = src_id;
+			d->imc[i].node_id = node_id;
+			rc = skx_register_mci(&d->imc[i]);
+			if (rc < 0)
+				goto fail;
+		}
+	}
+
+	/* Ensure that the OPSTATE is set correctly for POLL or NMI */
+	opstate_init();
+
+	setup_skx_debug();
+
+	mce_register_decode_chain(&skx_mce_dec);
+
+	return 0;
+fail:
+	skx_remove();
+	return rc;
+}
+
+static void __exit skx_exit(void)
+{
+	edac_dbg(2, "\n");
+	mce_unregister_decode_chain(&skx_mce_dec);
+	skx_remove();
+	teardown_skx_debug();
+}
+
+module_init(skx_init);
+module_exit(skx_exit);
+
+module_param(edac_op_state, int, 0444);
+MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Tony Luck");
+MODULE_DESCRIPTION("MC Driver for Intel Skylake server processors");
diff --git a/drivers/edac/wq.c b/drivers/edac/wq.c
index 1b8c07e44fd8..2a9a11ae2461 100644
--- a/drivers/edac/wq.c
+++ b/drivers/edac/wq.c
@@ -27,7 +27,7 @@ EXPORT_SYMBOL_GPL(edac_stop_work);
 
 int edac_workqueue_setup(void)
 {
-	wq = create_singlethread_workqueue("edac-poller");
+	wq = alloc_ordered_workqueue("edac-poller", WQ_MEM_RECLAIM);
 	if (!wq)
 		return -ENODEV;
 	else
diff --git a/drivers/extcon/Kconfig b/drivers/extcon/Kconfig
index 3d89e60a3e71..04788d92ea52 100644
--- a/drivers/extcon/Kconfig
+++ b/drivers/extcon/Kconfig
@@ -96,6 +96,12 @@ config EXTCON_PALMAS
 	  Say Y here to enable support for USB peripheral and USB host
 	  detection by palmas usb.
 
+config EXTCON_QCOM_SPMI_MISC
+	tristate "Qualcomm USB extcon support"
+	help
+	  Say Y here to enable SPMI PMIC based USB cable detection
+	  support on Qualcomm PMICs such as PM8941.
+
 config EXTCON_RT8973A
 	tristate "Richtek RT8973A EXTCON support"
 	depends on I2C
diff --git a/drivers/extcon/Makefile b/drivers/extcon/Makefile
index 972c813c375b..31a0a999c4fb 100644
--- a/drivers/extcon/Makefile
+++ b/drivers/extcon/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_EXTCON_MAX77693)	+= extcon-max77693.o
 obj-$(CONFIG_EXTCON_MAX77843)	+= extcon-max77843.o
 obj-$(CONFIG_EXTCON_MAX8997)	+= extcon-max8997.o
 obj-$(CONFIG_EXTCON_PALMAS)	+= extcon-palmas.o
+obj-$(CONFIG_EXTCON_QCOM_SPMI_MISC) += extcon-qcom-spmi-misc.o
 obj-$(CONFIG_EXTCON_RT8973A)	+= extcon-rt8973a.o
 obj-$(CONFIG_EXTCON_SM5502)	+= extcon-sm5502.o
 obj-$(CONFIG_EXTCON_USB_GPIO)	+= extcon-usb-gpio.o
diff --git a/drivers/extcon/extcon-adc-jack.c b/drivers/extcon/extcon-adc-jack.c
index 44e48aa78a84..bc538708c753 100644
--- a/drivers/extcon/extcon-adc-jack.c
+++ b/drivers/extcon/extcon-adc-jack.c
@@ -3,6 +3,9 @@
  *
  * Analog Jack extcon driver with ADC-based detection capability.
  *
+ * Copyright (C) 2016 Samsung Electronics
+ * Chanwoo Choi <cw00.choi@samsung.com>
+ *
  * Copyright (C) 2012 Samsung Electronics
  * MyungJoo Ham <myungjoo.ham@samsung.com>
  *
@@ -58,7 +61,7 @@ static void adc_jack_handler(struct work_struct *work)
 	struct adc_jack_data *data = container_of(to_delayed_work(work),
 			struct adc_jack_data,
 			handler);
-	u32 state = 0;
+	struct adc_jack_cond *def;
 	int ret, adc_val;
 	int i;
 
@@ -70,17 +73,18 @@ static void adc_jack_handler(struct work_struct *work)
 
 	/* Get state from adc value with adc_conditions */
 	for (i = 0; i < data->num_conditions; i++) {
-		struct adc_jack_cond *def = &data->adc_conditions[i];
-		if (!def->state)
-			break;
+		def = &data->adc_conditions[i];
 		if (def->min_adc <= adc_val && def->max_adc >= adc_val) {
-			state = def->state;
-			break;
+			extcon_set_state_sync(data->edev, def->id, true);
+			return;
 		}
 	}
-	/* if no def has met, it means state = 0 (no cables attached) */
 
-	extcon_set_state(data->edev, state);
+	/* Set the detached state if adc value is not included in the range */
+	for (i = 0; i < data->num_conditions; i++) {
+		def = &data->adc_conditions[i];
+		extcon_set_state_sync(data->edev, def->id, false);
+	}
 }
 
 static irqreturn_t adc_jack_irq_thread(int irq, void *_data)
@@ -114,16 +118,14 @@ static int adc_jack_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 
-	if (!pdata->adc_conditions ||
-			!pdata->adc_conditions[0].state) {
+	if (!pdata->adc_conditions) {
 		dev_err(&pdev->dev, "error: adc_conditions not defined.\n");
 		return -EINVAL;
 	}
 	data->adc_conditions = pdata->adc_conditions;
 
 	/* Check the length of array and set num_conditions */
-	for (i = 0; data->adc_conditions[i].state; i++)
-		;
+	for (i = 0; data->adc_conditions[i].id != EXTCON_NONE; i++);
 	data->num_conditions = i;
 
 	data->chan = iio_channel_get(&pdev->dev, pdata->consumer_channel);
@@ -158,6 +160,7 @@ static int adc_jack_probe(struct platform_device *pdev)
 	if (data->wakeup_source)
 		device_init_wakeup(&pdev->dev, 1);
 
+	adc_jack_handler(&data->handler.work);
 	return 0;
 }
 
diff --git a/drivers/extcon/extcon-arizona.c b/drivers/extcon/extcon-arizona.c
index 1d8e0a57bd51..56e6c4c7c60d 100644
--- a/drivers/extcon/extcon-arizona.c
+++ b/drivers/extcon/extcon-arizona.c
@@ -183,7 +183,7 @@ static void arizona_extcon_hp_clamp(struct arizona_extcon_info *info,
 		if (clamp)
 			val = ARIZONA_RMV_SHRT_HP1L;
 		break;
-	};
+	}
 
 	snd_soc_dapm_mutex_lock(arizona->dapm);
 
@@ -614,7 +614,7 @@ static irqreturn_t arizona_hpdet_irq(int irq, void *data)
 	}
 
 	/* If the cable was removed while measuring ignore the result */
-	ret = extcon_get_cable_state_(info->edev, EXTCON_MECHANICAL);
+	ret = extcon_get_state(info->edev, EXTCON_MECHANICAL);
 	if (ret < 0) {
 		dev_err(arizona->dev, "Failed to check cable state: %d\n",
 			ret);
@@ -649,7 +649,7 @@ static irqreturn_t arizona_hpdet_irq(int irq, void *data)
 	else
 		report = EXTCON_JACK_HEADPHONE;
 
-	ret = extcon_set_cable_state_(info->edev, report, true);
+	ret = extcon_set_state_sync(info->edev, report, true);
 	if (ret != 0)
 		dev_err(arizona->dev, "Failed to report HP/line: %d\n",
 			ret);
@@ -732,7 +732,7 @@ err:
 			   ARIZONA_ACCDET_MODE_MASK, ARIZONA_ACCDET_MODE_MIC);
 
 	/* Just report headphone */
-	ret = extcon_set_cable_state_(info->edev, EXTCON_JACK_HEADPHONE, true);
+	ret = extcon_set_state_sync(info->edev, EXTCON_JACK_HEADPHONE, true);
 	if (ret != 0)
 		dev_err(arizona->dev, "Failed to report headphone: %d\n", ret);
 
@@ -789,7 +789,7 @@ err:
 			   ARIZONA_ACCDET_MODE_MASK, ARIZONA_ACCDET_MODE_MIC);
 
 	/* Just report headphone */
-	ret = extcon_set_cable_state_(info->edev, EXTCON_JACK_HEADPHONE, true);
+	ret = extcon_set_state_sync(info->edev, EXTCON_JACK_HEADPHONE, true);
 	if (ret != 0)
 		dev_err(arizona->dev, "Failed to report headphone: %d\n", ret);
 
@@ -829,7 +829,7 @@ static void arizona_micd_detect(struct work_struct *work)
 	mutex_lock(&info->lock);
 
 	/* If the cable was removed while measuring ignore the result */
-	ret = extcon_get_cable_state_(info->edev, EXTCON_MECHANICAL);
+	ret = extcon_get_state(info->edev, EXTCON_MECHANICAL);
 	if (ret < 0) {
 		dev_err(arizona->dev, "Failed to check cable state: %d\n",
 				ret);
@@ -914,7 +914,7 @@ static void arizona_micd_detect(struct work_struct *work)
 
 		arizona_identify_headphone(info);
 
-		ret = extcon_set_cable_state_(info->edev,
+		ret = extcon_set_state_sync(info->edev,
 					      EXTCON_JACK_MICROPHONE, true);
 		if (ret != 0)
 			dev_err(arizona->dev, "Headset report failed: %d\n",
@@ -1108,7 +1108,7 @@ static irqreturn_t arizona_jackdet(int irq, void *data)
 
 	if (info->last_jackdet == present) {
 		dev_dbg(arizona->dev, "Detected jack\n");
-		ret = extcon_set_cable_state_(info->edev,
+		ret = extcon_set_state_sync(info->edev,
 					      EXTCON_MECHANICAL, true);
 
 		if (ret != 0)
@@ -1149,10 +1149,13 @@ static irqreturn_t arizona_jackdet(int irq, void *data)
 					 info->micd_ranges[i].key, 0);
 		input_sync(info->input);
 
-		ret = extcon_update_state(info->edev, 0xffffffff, 0);
-		if (ret != 0)
-			dev_err(arizona->dev, "Removal report failed: %d\n",
-				ret);
+		for (i = 0; i < ARRAY_SIZE(arizona_cable) - 1; i++) {
+			ret = extcon_set_state_sync(info->edev,
+					arizona_cable[i], false);
+			if (ret != 0)
+				dev_err(arizona->dev,
+					"Removal report failed: %d\n", ret);
+		}
 
 		regmap_update_bits(arizona->regmap,
 				   ARIZONA_JACK_DETECT_DEBOUNCE,
diff --git a/drivers/extcon/extcon-axp288.c b/drivers/extcon/extcon-axp288.c
index fd55c2f2080a..42f41e808292 100644
--- a/drivers/extcon/extcon-axp288.c
+++ b/drivers/extcon/extcon-axp288.c
@@ -189,19 +189,19 @@ static int axp288_handle_chrg_det_event(struct axp288_extcon_info *info)
 
 	switch (chrg_type) {
 	case DET_STAT_SDP:
-		dev_dbg(info->dev, "sdp cable is connecetd\n");
+		dev_dbg(info->dev, "sdp cable is connected\n");
 		notify_otg = true;
 		notify_charger = true;
 		cable = EXTCON_CHG_USB_SDP;
 		break;
 	case DET_STAT_CDP:
-		dev_dbg(info->dev, "cdp cable is connecetd\n");
+		dev_dbg(info->dev, "cdp cable is connected\n");
 		notify_otg = true;
 		notify_charger = true;
 		cable = EXTCON_CHG_USB_CDP;
 		break;
 	case DET_STAT_DCP:
-		dev_dbg(info->dev, "dcp cable is connecetd\n");
+		dev_dbg(info->dev, "dcp cable is connected\n");
 		notify_charger = true;
 		cable = EXTCON_CHG_USB_DCP;
 		break;
@@ -226,7 +226,7 @@ notify_otg:
 	}
 
 	if (notify_charger)
-		extcon_set_cable_state_(info->edev, cable, vbus_attach);
+		extcon_set_state_sync(info->edev, cable, vbus_attach);
 
 	/* Clear the flags on disconnect event */
 	if (!vbus_attach)
diff --git a/drivers/extcon/extcon-gpio.c b/drivers/extcon/extcon-gpio.c
index d023789f0fda..ebed22f22d75 100644
--- a/drivers/extcon/extcon-gpio.c
+++ b/drivers/extcon/extcon-gpio.c
@@ -49,7 +49,8 @@ static void gpio_extcon_work(struct work_struct *work)
 	state = gpiod_get_value_cansleep(data->id_gpiod);
 	if (data->pdata->gpio_active_low)
 		state = !state;
-	extcon_set_state(data->edev, state);
+
+	extcon_set_state_sync(data->edev, data->pdata->extcon_id, state);
 }
 
 static irqreturn_t gpio_irq_handler(int irq, void *dev_id)
diff --git a/drivers/extcon/extcon-max14577.c b/drivers/extcon/extcon-max14577.c
index 852a7112f451..12e26c4e7763 100644
--- a/drivers/extcon/extcon-max14577.c
+++ b/drivers/extcon/extcon-max14577.c
@@ -3,7 +3,7 @@
  *
  * Copyright (C) 2013,2014 Samsung Electronics
  * Chanwoo Choi <cw00.choi@samsung.com>
- * Krzysztof Kozlowski <k.kozlowski@samsung.com>
+ * Krzysztof Kozlowski <krzk@kernel.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -357,7 +357,7 @@ static int max14577_muic_jig_handler(struct max14577_muic_info *info,
 	if (ret < 0)
 		return ret;
 
-	extcon_set_cable_state_(info->edev, EXTCON_JIG, attached);
+	extcon_set_state_sync(info->edev, EXTCON_JIG, attached);
 
 	return 0;
 }
@@ -454,24 +454,24 @@ static int max14577_muic_chg_handler(struct max14577_muic_info *info)
 		if (ret < 0)
 			return ret;
 
-		extcon_set_cable_state_(info->edev, EXTCON_USB, attached);
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SDP,
+		extcon_set_state_sync(info->edev, EXTCON_USB, attached);
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SDP,
 					attached);
 		break;
 	case MAX14577_CHARGER_TYPE_DEDICATED_CHG:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_DCP,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_DCP,
 					attached);
 		break;
 	case MAX14577_CHARGER_TYPE_DOWNSTREAM_PORT:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_CDP,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_CDP,
 					attached);
 		break;
 	case MAX14577_CHARGER_TYPE_SPECIAL_500MA:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SLOW,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SLOW,
 					attached);
 		break;
 	case MAX14577_CHARGER_TYPE_SPECIAL_1A:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_FAST,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_FAST,
 					attached);
 		break;
 	case MAX14577_CHARGER_TYPE_NONE:
@@ -791,6 +791,6 @@ static struct platform_driver max14577_muic_driver = {
 module_platform_driver(max14577_muic_driver);
 
 MODULE_DESCRIPTION("Maxim 14577/77836 Extcon driver");
-MODULE_AUTHOR("Chanwoo Choi <cw00.choi@samsung.com>, Krzysztof Kozlowski <k.kozlowski@samsung.com>");
+MODULE_AUTHOR("Chanwoo Choi <cw00.choi@samsung.com>, Krzysztof Kozlowski <krzk@kernel.org>");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:extcon-max14577");
diff --git a/drivers/extcon/extcon-max3355.c b/drivers/extcon/extcon-max3355.c
index c24abec5d06c..533e16a952b8 100644
--- a/drivers/extcon/extcon-max3355.c
+++ b/drivers/extcon/extcon-max3355.c
@@ -39,16 +39,16 @@ static irqreturn_t max3355_id_irq(int irq, void *dev_id)
 		 * As we don't have event for USB peripheral cable attached,
 		 * we simulate USB peripheral attach here.
 		 */
-		extcon_set_cable_state_(data->edev, EXTCON_USB_HOST, false);
-		extcon_set_cable_state_(data->edev, EXTCON_USB, true);
+		extcon_set_state_sync(data->edev, EXTCON_USB_HOST, false);
+		extcon_set_state_sync(data->edev, EXTCON_USB, true);
 	} else {
 		/*
 		 * ID = 0 means USB HOST cable attached.
 		 * As we don't have event for USB peripheral cable detached,
 		 * we simulate USB peripheral detach here.
 		 */
-		extcon_set_cable_state_(data->edev, EXTCON_USB, false);
-		extcon_set_cable_state_(data->edev, EXTCON_USB_HOST, true);
+		extcon_set_state_sync(data->edev, EXTCON_USB, false);
+		extcon_set_state_sync(data->edev, EXTCON_USB_HOST, true);
 	}
 
 	return IRQ_HANDLED;
diff --git a/drivers/extcon/extcon-max77693.c b/drivers/extcon/extcon-max77693.c
index f17cb76b567c..68dbcb814b2f 100644
--- a/drivers/extcon/extcon-max77693.c
+++ b/drivers/extcon/extcon-max77693.c
@@ -505,8 +505,8 @@ static int max77693_muic_dock_handler(struct max77693_muic_info *info,
 		if (ret < 0)
 			return ret;
 
-		extcon_set_cable_state_(info->edev, EXTCON_DOCK, attached);
-		extcon_set_cable_state_(info->edev, EXTCON_DISP_MHL, attached);
+		extcon_set_state_sync(info->edev, EXTCON_DOCK, attached);
+		extcon_set_state_sync(info->edev, EXTCON_DISP_MHL, attached);
 		goto out;
 	case MAX77693_MUIC_ADC_AUDIO_MODE_REMOTE:	/* Dock-Desk */
 		dock_id = EXTCON_DOCK;
@@ -514,8 +514,8 @@ static int max77693_muic_dock_handler(struct max77693_muic_info *info,
 	case MAX77693_MUIC_ADC_AV_CABLE_NOLOAD:		/* Dock-Audio */
 		dock_id = EXTCON_DOCK;
 		if (!attached) {
-			extcon_set_cable_state_(info->edev, EXTCON_USB, false);
-			extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SDP,
+			extcon_set_state_sync(info->edev, EXTCON_USB, false);
+			extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SDP,
 						false);
 		}
 		break;
@@ -530,7 +530,7 @@ static int max77693_muic_dock_handler(struct max77693_muic_info *info,
 					attached);
 	if (ret < 0)
 		return ret;
-	extcon_set_cable_state_(info->edev, dock_id, attached);
+	extcon_set_state_sync(info->edev, dock_id, attached);
 
 out:
 	return 0;
@@ -596,7 +596,7 @@ static int max77693_muic_adc_ground_handler(struct max77693_muic_info *info)
 						attached);
 		if (ret < 0)
 			return ret;
-		extcon_set_cable_state_(info->edev, EXTCON_USB_HOST, attached);
+		extcon_set_state_sync(info->edev, EXTCON_USB_HOST, attached);
 		break;
 	case MAX77693_MUIC_GND_AV_CABLE_LOAD:
 		/* Audio Video Cable with load, PATH:AUDIO */
@@ -604,14 +604,14 @@ static int max77693_muic_adc_ground_handler(struct max77693_muic_info *info)
 						attached);
 		if (ret < 0)
 			return ret;
-		extcon_set_cable_state_(info->edev, EXTCON_USB, attached);
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SDP,
+		extcon_set_state_sync(info->edev, EXTCON_USB, attached);
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SDP,
 					attached);
 		break;
 	case MAX77693_MUIC_GND_MHL:
 	case MAX77693_MUIC_GND_MHL_VB:
 		/* MHL or MHL with USB/TA cable */
-		extcon_set_cable_state_(info->edev, EXTCON_DISP_MHL, attached);
+		extcon_set_state_sync(info->edev, EXTCON_DISP_MHL, attached);
 		break;
 	default:
 		dev_err(info->dev, "failed to detect %s cable of gnd type\n",
@@ -653,7 +653,7 @@ static int max77693_muic_jig_handler(struct max77693_muic_info *info,
 	if (ret < 0)
 		return ret;
 
-	extcon_set_cable_state_(info->edev, EXTCON_JIG, attached);
+	extcon_set_state_sync(info->edev, EXTCON_JIG, attached);
 
 	return 0;
 }
@@ -807,10 +807,10 @@ static int max77693_muic_chg_handler(struct max77693_muic_info *info)
 			 * - Support charging through micro-usb port without
 			 *   data connection
 			 */
-			extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_DCP,
+			extcon_set_state_sync(info->edev, EXTCON_CHG_USB_DCP,
 						attached);
 			if (!cable_attached)
-				extcon_set_cable_state_(info->edev,
+				extcon_set_state_sync(info->edev,
 					EXTCON_DISP_MHL, cable_attached);
 			break;
 		}
@@ -834,13 +834,13 @@ static int max77693_muic_chg_handler(struct max77693_muic_info *info)
 			 * - Support charging through micro-usb port without
 			 *   data connection.
 			 */
-			extcon_set_cable_state_(info->edev, EXTCON_USB,
+			extcon_set_state_sync(info->edev, EXTCON_USB,
 						attached);
-			extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SDP,
+			extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SDP,
 						attached);
 
 			if (!cable_attached)
-				extcon_set_cable_state_(info->edev, EXTCON_DOCK,
+				extcon_set_state_sync(info->edev, EXTCON_DOCK,
 							cable_attached);
 			break;
 		case MAX77693_MUIC_ADC_RESERVED_ACC_3:		/* Dock-Smart */
@@ -869,9 +869,9 @@ static int max77693_muic_chg_handler(struct max77693_muic_info *info)
 			if (ret < 0)
 				return ret;
 
-			extcon_set_cable_state_(info->edev, EXTCON_DOCK,
+			extcon_set_state_sync(info->edev, EXTCON_DOCK,
 						attached);
-			extcon_set_cable_state_(info->edev, EXTCON_DISP_MHL,
+			extcon_set_state_sync(info->edev, EXTCON_DISP_MHL,
 						attached);
 			break;
 		}
@@ -905,28 +905,28 @@ static int max77693_muic_chg_handler(struct max77693_muic_info *info)
 			if (ret < 0)
 				return ret;
 
-			extcon_set_cable_state_(info->edev, EXTCON_USB,
+			extcon_set_state_sync(info->edev, EXTCON_USB,
 						attached);
-			extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SDP,
+			extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SDP,
 						attached);
 			break;
 		case MAX77693_CHARGER_TYPE_DEDICATED_CHG:
 			/* Only TA cable */
-			extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_DCP,
+			extcon_set_state_sync(info->edev, EXTCON_CHG_USB_DCP,
 						attached);
 			break;
 		}
 		break;
 	case MAX77693_CHARGER_TYPE_DOWNSTREAM_PORT:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_CDP,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_CDP,
 					attached);
 		break;
 	case MAX77693_CHARGER_TYPE_APPLE_500MA:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SLOW,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SLOW,
 					attached);
 		break;
 	case MAX77693_CHARGER_TYPE_APPLE_1A_2A:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_FAST,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_FAST,
 					attached);
 		break;
 	case MAX77693_CHARGER_TYPE_DEAD_BATTERY:
diff --git a/drivers/extcon/extcon-max77843.c b/drivers/extcon/extcon-max77843.c
index b188bd650efa..5d11fdf36e94 100644
--- a/drivers/extcon/extcon-max77843.c
+++ b/drivers/extcon/extcon-max77843.c
@@ -346,7 +346,7 @@ static int max77843_muic_adc_gnd_handler(struct max77843_muic_info *info)
 		if (ret < 0)
 			return ret;
 
-		extcon_set_cable_state_(info->edev, EXTCON_USB_HOST, attached);
+		extcon_set_state_sync(info->edev, EXTCON_USB_HOST, attached);
 		break;
 	case MAX77843_MUIC_GND_MHL_VB:
 	case MAX77843_MUIC_GND_MHL:
@@ -356,7 +356,7 @@ static int max77843_muic_adc_gnd_handler(struct max77843_muic_info *info)
 		if (ret < 0)
 			return ret;
 
-		extcon_set_cable_state_(info->edev, EXTCON_DISP_MHL, attached);
+		extcon_set_state_sync(info->edev, EXTCON_DISP_MHL, attached);
 		break;
 	default:
 		dev_err(info->dev, "failed to detect %s accessory(gnd:0x%x)\n",
@@ -392,7 +392,7 @@ static int max77843_muic_jig_handler(struct max77843_muic_info *info,
 	if (ret < 0)
 		return ret;
 
-	extcon_set_cable_state_(info->edev, EXTCON_JIG, attached);
+	extcon_set_state_sync(info->edev, EXTCON_JIG, attached);
 
 	return 0;
 }
@@ -486,8 +486,8 @@ static int max77843_muic_chg_handler(struct max77843_muic_info *info)
 		if (ret < 0)
 			return ret;
 
-		extcon_set_cable_state_(info->edev, EXTCON_USB, attached);
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SDP,
+		extcon_set_state_sync(info->edev, EXTCON_USB, attached);
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SDP,
 					attached);
 		break;
 	case MAX77843_MUIC_CHG_DOWNSTREAM:
@@ -497,7 +497,7 @@ static int max77843_muic_chg_handler(struct max77843_muic_info *info)
 		if (ret < 0)
 			return ret;
 
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_CDP,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_CDP,
 					attached);
 		break;
 	case MAX77843_MUIC_CHG_DEDICATED:
@@ -507,7 +507,7 @@ static int max77843_muic_chg_handler(struct max77843_muic_info *info)
 		if (ret < 0)
 			return ret;
 
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_DCP,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_DCP,
 					attached);
 		break;
 	case MAX77843_MUIC_CHG_SPECIAL_500MA:
@@ -517,7 +517,7 @@ static int max77843_muic_chg_handler(struct max77843_muic_info *info)
 		if (ret < 0)
 			return ret;
 
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SLOW,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SLOW,
 					attached);
 		break;
 	case MAX77843_MUIC_CHG_SPECIAL_1A:
@@ -527,7 +527,7 @@ static int max77843_muic_chg_handler(struct max77843_muic_info *info)
 		if (ret < 0)
 			return ret;
 
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_FAST,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_FAST,
 					attached);
 		break;
 	case MAX77843_MUIC_CHG_GND:
@@ -536,10 +536,10 @@ static int max77843_muic_chg_handler(struct max77843_muic_info *info)
 
 		/* Charger cable on MHL accessory is attach or detach */
 		if (gnd_type == MAX77843_MUIC_GND_MHL_VB)
-			extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_DCP,
+			extcon_set_state_sync(info->edev, EXTCON_CHG_USB_DCP,
 						true);
 		else if (gnd_type == MAX77843_MUIC_GND_MHL)
-			extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_DCP,
+			extcon_set_state_sync(info->edev, EXTCON_CHG_USB_DCP,
 						false);
 		break;
 	case MAX77843_MUIC_CHG_NONE:
diff --git a/drivers/extcon/extcon-max8997.c b/drivers/extcon/extcon-max8997.c
index 9a89320d09a8..4a0612fb9c07 100644
--- a/drivers/extcon/extcon-max8997.c
+++ b/drivers/extcon/extcon-max8997.c
@@ -331,11 +331,11 @@ static int max8997_muic_handle_usb(struct max8997_muic_info *info,
 
 	switch (usb_type) {
 	case MAX8997_USB_HOST:
-		extcon_set_cable_state_(info->edev, EXTCON_USB_HOST, attached);
+		extcon_set_state_sync(info->edev, EXTCON_USB_HOST, attached);
 		break;
 	case MAX8997_USB_DEVICE:
-		extcon_set_cable_state_(info->edev, EXTCON_USB, attached);
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SDP,
+		extcon_set_state_sync(info->edev, EXTCON_USB, attached);
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SDP,
 					attached);
 		break;
 	default:
@@ -361,7 +361,7 @@ static int max8997_muic_handle_dock(struct max8997_muic_info *info,
 	switch (cable_type) {
 	case MAX8997_MUIC_ADC_AV_CABLE_NOLOAD:
 	case MAX8997_MUIC_ADC_FACTORY_MODE_UART_ON:
-		extcon_set_cable_state_(info->edev, EXTCON_DOCK, attached);
+		extcon_set_state_sync(info->edev, EXTCON_DOCK, attached);
 		break;
 	default:
 		dev_err(info->dev, "failed to detect %s dock device\n",
@@ -384,7 +384,7 @@ static int max8997_muic_handle_jig_uart(struct max8997_muic_info *info,
 		return ret;
 	}
 
-	extcon_set_cable_state_(info->edev, EXTCON_JIG, attached);
+	extcon_set_state_sync(info->edev, EXTCON_JIG, attached);
 
 	return 0;
 }
@@ -406,7 +406,7 @@ static int max8997_muic_adc_handler(struct max8997_muic_info *info)
 			return ret;
 		break;
 	case MAX8997_MUIC_ADC_MHL:
-		extcon_set_cable_state_(info->edev, EXTCON_DISP_MHL, attached);
+		extcon_set_state_sync(info->edev, EXTCON_DISP_MHL, attached);
 		break;
 	case MAX8997_MUIC_ADC_FACTORY_MODE_USB_OFF:
 	case MAX8997_MUIC_ADC_FACTORY_MODE_USB_ON:
@@ -489,19 +489,19 @@ static int max8997_muic_chg_handler(struct max8997_muic_info *info)
 		}
 		break;
 	case MAX8997_CHARGER_TYPE_DOWNSTREAM_PORT:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_CDP,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_CDP,
 					attached);
 		break;
 	case MAX8997_CHARGER_TYPE_DEDICATED_CHG:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_DCP,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_DCP,
 					attached);
 		break;
 	case MAX8997_CHARGER_TYPE_500MA:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SLOW,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SLOW,
 					attached);
 		break;
 	case MAX8997_CHARGER_TYPE_1A:
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_FAST,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_FAST,
 					attached);
 		break;
 	default:
diff --git a/drivers/extcon/extcon-palmas.c b/drivers/extcon/extcon-palmas.c
index caff46c0e214..634ba70782de 100644
--- a/drivers/extcon/extcon-palmas.c
+++ b/drivers/extcon/extcon-palmas.c
@@ -61,7 +61,7 @@ static irqreturn_t palmas_vbus_irq_handler(int irq, void *_palmas_usb)
 	if (vbus_line_state & PALMAS_INT3_LINE_STATE_VBUS) {
 		if (palmas_usb->linkstat != PALMAS_USB_STATE_VBUS) {
 			palmas_usb->linkstat = PALMAS_USB_STATE_VBUS;
-			extcon_set_cable_state_(edev, EXTCON_USB, true);
+			extcon_set_state_sync(edev, EXTCON_USB, true);
 			dev_info(palmas_usb->dev, "USB cable is attached\n");
 		} else {
 			dev_dbg(palmas_usb->dev,
@@ -70,7 +70,7 @@ static irqreturn_t palmas_vbus_irq_handler(int irq, void *_palmas_usb)
 	} else if (!(vbus_line_state & PALMAS_INT3_LINE_STATE_VBUS)) {
 		if (palmas_usb->linkstat == PALMAS_USB_STATE_VBUS) {
 			palmas_usb->linkstat = PALMAS_USB_STATE_DISCONNECT;
-			extcon_set_cable_state_(edev, EXTCON_USB, false);
+			extcon_set_state_sync(edev, EXTCON_USB, false);
 			dev_info(palmas_usb->dev, "USB cable is detached\n");
 		} else {
 			dev_dbg(palmas_usb->dev,
@@ -98,7 +98,7 @@ static irqreturn_t palmas_id_irq_handler(int irq, void *_palmas_usb)
 			PALMAS_USB_ID_INT_LATCH_CLR,
 			PALMAS_USB_ID_INT_EN_HI_CLR_ID_GND);
 		palmas_usb->linkstat = PALMAS_USB_STATE_ID;
-		extcon_set_cable_state_(edev, EXTCON_USB_HOST, true);
+		extcon_set_state_sync(edev, EXTCON_USB_HOST, true);
 		dev_info(palmas_usb->dev, "USB-HOST cable is attached\n");
 	} else if ((set & PALMAS_USB_ID_INT_SRC_ID_FLOAT) &&
 				(id_src & PALMAS_USB_ID_INT_SRC_ID_FLOAT)) {
@@ -106,17 +106,17 @@ static irqreturn_t palmas_id_irq_handler(int irq, void *_palmas_usb)
 			PALMAS_USB_ID_INT_LATCH_CLR,
 			PALMAS_USB_ID_INT_EN_HI_CLR_ID_FLOAT);
 		palmas_usb->linkstat = PALMAS_USB_STATE_DISCONNECT;
-		extcon_set_cable_state_(edev, EXTCON_USB_HOST, false);
+		extcon_set_state_sync(edev, EXTCON_USB_HOST, false);
 		dev_info(palmas_usb->dev, "USB-HOST cable is detached\n");
 	} else if ((palmas_usb->linkstat == PALMAS_USB_STATE_ID) &&
 				(!(set & PALMAS_USB_ID_INT_SRC_ID_GND))) {
 		palmas_usb->linkstat = PALMAS_USB_STATE_DISCONNECT;
-		extcon_set_cable_state_(edev, EXTCON_USB_HOST, false);
+		extcon_set_state_sync(edev, EXTCON_USB_HOST, false);
 		dev_info(palmas_usb->dev, "USB-HOST cable is detached\n");
 	} else if ((palmas_usb->linkstat == PALMAS_USB_STATE_DISCONNECT) &&
 				(id_src & PALMAS_USB_ID_INT_SRC_ID_GND)) {
 		palmas_usb->linkstat = PALMAS_USB_STATE_ID;
-		extcon_set_cable_state_(edev, EXTCON_USB_HOST, true);
+		extcon_set_state_sync(edev, EXTCON_USB_HOST, true);
 		dev_info(palmas_usb->dev, " USB-HOST cable is attached\n");
 	}
 
@@ -137,10 +137,10 @@ static void palmas_gpio_id_detect(struct work_struct *work)
 	id = gpiod_get_value_cansleep(palmas_usb->id_gpiod);
 
 	if (id) {
-		extcon_set_cable_state_(edev, EXTCON_USB_HOST, false);
+		extcon_set_state_sync(edev, EXTCON_USB_HOST, false);
 		dev_info(palmas_usb->dev, "USB-HOST cable is detached\n");
 	} else {
-		extcon_set_cable_state_(edev, EXTCON_USB_HOST, true);
+		extcon_set_state_sync(edev, EXTCON_USB_HOST, true);
 		dev_info(palmas_usb->dev, "USB-HOST cable is attached\n");
 	}
 }
diff --git a/drivers/extcon/extcon-qcom-spmi-misc.c b/drivers/extcon/extcon-qcom-spmi-misc.c
new file mode 100644
index 000000000000..ca957a5f4291
--- /dev/null
+++ b/drivers/extcon/extcon-qcom-spmi-misc.c
@@ -0,0 +1,170 @@
+/**
+ * extcon-qcom-spmi-misc.c - Qualcomm USB extcon driver to support USB ID
+ *				detection based on extcon-usb-gpio.c.
+ *
+ * Copyright (C) 2016 Linaro, Ltd.
+ * Stephen Boyd <stephen.boyd@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/extcon.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+
+#define USB_ID_DEBOUNCE_MS	5	/* ms */
+
+struct qcom_usb_extcon_info {
+	struct extcon_dev *edev;
+	int irq;
+	struct delayed_work wq_detcable;
+	unsigned long debounce_jiffies;
+};
+
+static const unsigned int qcom_usb_extcon_cable[] = {
+	EXTCON_USB_HOST,
+	EXTCON_NONE,
+};
+
+static void qcom_usb_extcon_detect_cable(struct work_struct *work)
+{
+	bool id;
+	int ret;
+	struct qcom_usb_extcon_info *info = container_of(to_delayed_work(work),
+						    struct qcom_usb_extcon_info,
+						    wq_detcable);
+
+	/* check ID and update cable state */
+	ret = irq_get_irqchip_state(info->irq, IRQCHIP_STATE_LINE_LEVEL, &id);
+	if (ret)
+		return;
+
+	extcon_set_state(info->edev, EXTCON_USB_HOST, !id);
+}
+
+static irqreturn_t qcom_usb_irq_handler(int irq, void *dev_id)
+{
+	struct qcom_usb_extcon_info *info = dev_id;
+
+	queue_delayed_work(system_power_efficient_wq, &info->wq_detcable,
+			   info->debounce_jiffies);
+
+	return IRQ_HANDLED;
+}
+
+static int qcom_usb_extcon_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct qcom_usb_extcon_info *info;
+	int ret;
+
+	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	info->edev = devm_extcon_dev_allocate(dev, qcom_usb_extcon_cable);
+	if (IS_ERR(info->edev)) {
+		dev_err(dev, "failed to allocate extcon device\n");
+		return -ENOMEM;
+	}
+
+	ret = devm_extcon_dev_register(dev, info->edev);
+	if (ret < 0) {
+		dev_err(dev, "failed to register extcon device\n");
+		return ret;
+	}
+
+	info->debounce_jiffies = msecs_to_jiffies(USB_ID_DEBOUNCE_MS);
+	INIT_DELAYED_WORK(&info->wq_detcable, qcom_usb_extcon_detect_cable);
+
+	info->irq = platform_get_irq_byname(pdev, "usb_id");
+	if (info->irq < 0)
+		return info->irq;
+
+	ret = devm_request_threaded_irq(dev, info->irq, NULL,
+					qcom_usb_irq_handler,
+					IRQF_TRIGGER_RISING |
+					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+					pdev->name, info);
+	if (ret < 0) {
+		dev_err(dev, "failed to request handler for ID IRQ\n");
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, info);
+	device_init_wakeup(dev, 1);
+
+	/* Perform initial detection */
+	qcom_usb_extcon_detect_cable(&info->wq_detcable.work);
+
+	return 0;
+}
+
+static int qcom_usb_extcon_remove(struct platform_device *pdev)
+{
+	struct qcom_usb_extcon_info *info = platform_get_drvdata(pdev);
+
+	cancel_delayed_work_sync(&info->wq_detcable);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int qcom_usb_extcon_suspend(struct device *dev)
+{
+	struct qcom_usb_extcon_info *info = dev_get_drvdata(dev);
+	int ret = 0;
+
+	if (device_may_wakeup(dev))
+		ret = enable_irq_wake(info->irq);
+
+	return ret;
+}
+
+static int qcom_usb_extcon_resume(struct device *dev)
+{
+	struct qcom_usb_extcon_info *info = dev_get_drvdata(dev);
+	int ret = 0;
+
+	if (device_may_wakeup(dev))
+		ret = disable_irq_wake(info->irq);
+
+	return ret;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(qcom_usb_extcon_pm_ops,
+			 qcom_usb_extcon_suspend, qcom_usb_extcon_resume);
+
+static const struct of_device_id qcom_usb_extcon_dt_match[] = {
+	{ .compatible = "qcom,pm8941-misc", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qcom_usb_extcon_dt_match);
+
+static struct platform_driver qcom_usb_extcon_driver = {
+	.probe		= qcom_usb_extcon_probe,
+	.remove		= qcom_usb_extcon_remove,
+	.driver		= {
+		.name	= "extcon-pm8941-misc",
+		.pm	= &qcom_usb_extcon_pm_ops,
+		.of_match_table = qcom_usb_extcon_dt_match,
+	},
+};
+module_platform_driver(qcom_usb_extcon_driver);
+
+MODULE_DESCRIPTION("QCOM USB ID extcon driver");
+MODULE_AUTHOR("Stephen Boyd <stephen.boyd@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/extcon/extcon-rt8973a.c b/drivers/extcon/extcon-rt8973a.c
index 97e074d70eca..174c388739ea 100644
--- a/drivers/extcon/extcon-rt8973a.c
+++ b/drivers/extcon/extcon-rt8973a.c
@@ -398,9 +398,9 @@ static int rt8973a_muic_cable_handler(struct rt8973a_muic_info *info,
 		return ret;
 
 	/* Change the state of external accessory */
-	extcon_set_cable_state_(info->edev, id, attached);
+	extcon_set_state_sync(info->edev, id, attached);
 	if (id == EXTCON_USB)
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SDP,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SDP,
 					attached);
 
 	return 0;
diff --git a/drivers/extcon/extcon-sm5502.c b/drivers/extcon/extcon-sm5502.c
index df769a17e736..b22325688503 100644
--- a/drivers/extcon/extcon-sm5502.c
+++ b/drivers/extcon/extcon-sm5502.c
@@ -411,9 +411,9 @@ static int sm5502_muic_cable_handler(struct sm5502_muic_info *info,
 		return ret;
 
 	/* Change the state of external accessory */
-	extcon_set_cable_state_(info->edev, id, attached);
+	extcon_set_state_sync(info->edev, id, attached);
 	if (id == EXTCON_USB)
-		extcon_set_cable_state_(info->edev, EXTCON_CHG_USB_SDP,
+		extcon_set_state_sync(info->edev, EXTCON_CHG_USB_SDP,
 					attached);
 
 	return 0;
diff --git a/drivers/extcon/extcon-usb-gpio.c b/drivers/extcon/extcon-usb-gpio.c
index 2512660dc4b9..a27d350f69e3 100644
--- a/drivers/extcon/extcon-usb-gpio.c
+++ b/drivers/extcon/extcon-usb-gpio.c
@@ -63,16 +63,16 @@ static void usb_extcon_detect_cable(struct work_struct *work)
 		 * As we don't have event for USB peripheral cable attached,
 		 * we simulate USB peripheral attach here.
 		 */
-		extcon_set_cable_state_(info->edev, EXTCON_USB_HOST, false);
-		extcon_set_cable_state_(info->edev, EXTCON_USB, true);
+		extcon_set_state_sync(info->edev, EXTCON_USB_HOST, false);
+		extcon_set_state_sync(info->edev, EXTCON_USB, true);
 	} else {
 		/*
 		 * ID = 0 means USB HOST cable attached.
 		 * As we don't have event for USB peripheral cable detached,
 		 * we simulate USB peripheral detach here.
 		 */
-		extcon_set_cable_state_(info->edev, EXTCON_USB, false);
-		extcon_set_cable_state_(info->edev, EXTCON_USB_HOST, true);
+		extcon_set_state_sync(info->edev, EXTCON_USB, false);
+		extcon_set_state_sync(info->edev, EXTCON_USB_HOST, true);
 	}
 }
 
diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c
index 8682efc0f57b..78298460d168 100644
--- a/drivers/extcon/extcon.c
+++ b/drivers/extcon/extcon.c
@@ -38,43 +38,159 @@
 #define SUPPORTED_CABLE_MAX	32
 #define CABLE_NAME_MAX		30
 
-static const char *extcon_name[] =  {
-	[EXTCON_NONE]			= "NONE",
+struct __extcon_info {
+	unsigned int type;
+	unsigned int id;
+	const char *name;
+
+} extcon_info[] = {
+	[EXTCON_NONE] = {
+		.type = EXTCON_TYPE_MISC,
+		.id = EXTCON_NONE,
+		.name = "NONE",
+	},
 
 	/* USB external connector */
-	[EXTCON_USB]			= "USB",
-	[EXTCON_USB_HOST]		= "USB-HOST",
+	[EXTCON_USB] = {
+		.type = EXTCON_TYPE_USB,
+		.id = EXTCON_USB,
+		.name = "USB",
+	},
+	[EXTCON_USB_HOST] = {
+		.type = EXTCON_TYPE_USB,
+		.id = EXTCON_USB_HOST,
+		.name = "USB_HOST",
+	},
 
 	/* Charging external connector */
-	[EXTCON_CHG_USB_SDP]		= "SDP",
-	[EXTCON_CHG_USB_DCP]		= "DCP",
-	[EXTCON_CHG_USB_CDP]		= "CDP",
-	[EXTCON_CHG_USB_ACA]		= "ACA",
-	[EXTCON_CHG_USB_FAST]		= "FAST-CHARGER",
-	[EXTCON_CHG_USB_SLOW]		= "SLOW-CHARGER",
+	[EXTCON_CHG_USB_SDP] = {
+		.type = EXTCON_TYPE_CHG | EXTCON_TYPE_USB,
+		.id = EXTCON_CHG_USB_SDP,
+		.name = "SDP",
+	},
+	[EXTCON_CHG_USB_DCP] = {
+		.type = EXTCON_TYPE_CHG | EXTCON_TYPE_USB,
+		.id = EXTCON_CHG_USB_DCP,
+		.name = "DCP",
+	},
+	[EXTCON_CHG_USB_CDP] = {
+		.type = EXTCON_TYPE_CHG | EXTCON_TYPE_USB,
+		.id = EXTCON_CHG_USB_CDP,
+		.name = "CDP",
+	},
+	[EXTCON_CHG_USB_ACA] = {
+		.type = EXTCON_TYPE_CHG | EXTCON_TYPE_USB,
+		.id = EXTCON_CHG_USB_ACA,
+		.name = "ACA",
+	},
+	[EXTCON_CHG_USB_FAST] = {
+		.type = EXTCON_TYPE_CHG | EXTCON_TYPE_USB,
+		.id = EXTCON_CHG_USB_FAST,
+		.name = "FAST-CHARGER",
+	},
+	[EXTCON_CHG_USB_SLOW] = {
+		.type = EXTCON_TYPE_CHG | EXTCON_TYPE_USB,
+		.id = EXTCON_CHG_USB_SLOW,
+		.name = "SLOW-CHARGER",
+	},
+	[EXTCON_CHG_WPT] = {
+		.type = EXTCON_TYPE_CHG,
+		.id = EXTCON_CHG_WPT,
+		.name = "WPT",
+	},
 
 	/* Jack external connector */
-	[EXTCON_JACK_MICROPHONE]	= "MICROPHONE",
-	[EXTCON_JACK_HEADPHONE]		= "HEADPHONE",
-	[EXTCON_JACK_LINE_IN]		= "LINE-IN",
-	[EXTCON_JACK_LINE_OUT]		= "LINE-OUT",
-	[EXTCON_JACK_VIDEO_IN]		= "VIDEO-IN",
-	[EXTCON_JACK_VIDEO_OUT]		= "VIDEO-OUT",
-	[EXTCON_JACK_SPDIF_IN]		= "SPDIF-IN",
-	[EXTCON_JACK_SPDIF_OUT]		= "SPDIF-OUT",
+	[EXTCON_JACK_MICROPHONE] = {
+		.type = EXTCON_TYPE_JACK,
+		.id = EXTCON_JACK_MICROPHONE,
+		.name = "MICROPHONE",
+	},
+	[EXTCON_JACK_HEADPHONE] = {
+		.type = EXTCON_TYPE_JACK,
+		.id = EXTCON_JACK_HEADPHONE,
+		.name = "HEADPHONE",
+	},
+	[EXTCON_JACK_LINE_IN] = {
+		.type = EXTCON_TYPE_JACK,
+		.id = EXTCON_JACK_LINE_IN,
+		.name = "LINE-IN",
+	},
+	[EXTCON_JACK_LINE_OUT] = {
+		.type = EXTCON_TYPE_JACK,
+		.id = EXTCON_JACK_LINE_OUT,
+		.name = "LINE-OUT",
+	},
+	[EXTCON_JACK_VIDEO_IN] = {
+		.type = EXTCON_TYPE_JACK,
+		.id = EXTCON_JACK_VIDEO_IN,
+		.name = "VIDEO-IN",
+	},
+	[EXTCON_JACK_VIDEO_OUT] = {
+		.type = EXTCON_TYPE_JACK,
+		.id = EXTCON_JACK_VIDEO_OUT,
+		.name = "VIDEO-OUT",
+	},
+	[EXTCON_JACK_SPDIF_IN] = {
+		.type = EXTCON_TYPE_JACK,
+		.id = EXTCON_JACK_SPDIF_IN,
+		.name = "SPDIF-IN",
+	},
+	[EXTCON_JACK_SPDIF_OUT] = {
+		.type = EXTCON_TYPE_JACK,
+		.id = EXTCON_JACK_SPDIF_OUT,
+		.name = "SPDIF-OUT",
+	},
 
 	/* Display external connector */
-	[EXTCON_DISP_HDMI]		= "HDMI",
-	[EXTCON_DISP_MHL]		= "MHL",
-	[EXTCON_DISP_DVI]		= "DVI",
-	[EXTCON_DISP_VGA]		= "VGA",
+	[EXTCON_DISP_HDMI] = {
+		.type = EXTCON_TYPE_DISP,
+		.id = EXTCON_DISP_HDMI,
+		.name = "HDMI",
+	},
+	[EXTCON_DISP_MHL] = {
+		.type = EXTCON_TYPE_DISP,
+		.id = EXTCON_DISP_MHL,
+		.name = "MHL",
+	},
+	[EXTCON_DISP_DVI] = {
+		.type = EXTCON_TYPE_DISP,
+		.id = EXTCON_DISP_DVI,
+		.name = "DVI",
+	},
+	[EXTCON_DISP_VGA] = {
+		.type = EXTCON_TYPE_DISP,
+		.id = EXTCON_DISP_VGA,
+		.name = "VGA",
+	},
+	[EXTCON_DISP_DP] = {
+		.type = EXTCON_TYPE_DISP | EXTCON_TYPE_USB,
+		.id = EXTCON_DISP_DP,
+		.name = "DP",
+	},
+	[EXTCON_DISP_HMD] = {
+		.type = EXTCON_TYPE_DISP | EXTCON_TYPE_USB,
+		.id = EXTCON_DISP_HMD,
+		.name = "HMD",
+	},
 
 	/* Miscellaneous external connector */
-	[EXTCON_DOCK]			= "DOCK",
-	[EXTCON_JIG]			= "JIG",
-	[EXTCON_MECHANICAL]		= "MECHANICAL",
-
-	NULL,
+	[EXTCON_DOCK] = {
+		.type = EXTCON_TYPE_MISC,
+		.id = EXTCON_DOCK,
+		.name = "DOCK",
+	},
+	[EXTCON_JIG] = {
+		.type = EXTCON_TYPE_MISC,
+		.id = EXTCON_JIG,
+		.name = "JIG",
+	},
+	[EXTCON_MECHANICAL] = {
+		.type = EXTCON_TYPE_MISC,
+		.id = EXTCON_MECHANICAL,
+		.name = "MECHANICAL",
+	},
+
+	{ /* sentinel */ }
 };
 
 /**
@@ -95,6 +211,16 @@ struct extcon_cable {
 	struct device_attribute attr_state;
 
 	struct attribute *attrs[3]; /* to be fed to attr_g.attrs */
+
+	union extcon_property_value usb_propval[EXTCON_PROP_USB_CNT];
+	union extcon_property_value chg_propval[EXTCON_PROP_CHG_CNT];
+	union extcon_property_value jack_propval[EXTCON_PROP_JACK_CNT];
+	union extcon_property_value disp_propval[EXTCON_PROP_DISP_CNT];
+
+	unsigned long usb_bits[BITS_TO_LONGS(EXTCON_PROP_USB_CNT)];
+	unsigned long chg_bits[BITS_TO_LONGS(EXTCON_PROP_CHG_CNT)];
+	unsigned long jack_bits[BITS_TO_LONGS(EXTCON_PROP_JACK_CNT)];
+	unsigned long disp_bits[BITS_TO_LONGS(EXTCON_PROP_DISP_CNT)];
 };
 
 static struct class *extcon_class;
@@ -147,14 +273,93 @@ static int find_cable_index_by_id(struct extcon_dev *edev, const unsigned int id
 	return -EINVAL;
 }
 
-static bool is_extcon_changed(u32 prev, u32 new, int idx, bool *attached)
+static int get_extcon_type(unsigned int prop)
 {
-	if (((prev >> idx) & 0x1) != ((new >> idx) & 0x1)) {
-		*attached = ((new >> idx) & 0x1) ? true : false;
-		return true;
+	switch (prop) {
+	case EXTCON_PROP_USB_MIN ... EXTCON_PROP_USB_MAX:
+		return EXTCON_TYPE_USB;
+	case EXTCON_PROP_CHG_MIN ... EXTCON_PROP_CHG_MAX:
+		return EXTCON_TYPE_CHG;
+	case EXTCON_PROP_JACK_MIN ... EXTCON_PROP_JACK_MAX:
+		return EXTCON_TYPE_JACK;
+	case EXTCON_PROP_DISP_MIN ... EXTCON_PROP_DISP_MAX:
+		return EXTCON_TYPE_DISP;
+	default:
+		return -EINVAL;
 	}
+}
+
+static bool is_extcon_attached(struct extcon_dev *edev, unsigned int index)
+{
+	return !!(edev->state & BIT(index));
+}
+
+static bool is_extcon_changed(struct extcon_dev *edev, int index,
+				bool new_state)
+{
+	int state = !!(edev->state & BIT(index));
+	return (state != new_state);
+}
+
+static bool is_extcon_property_supported(unsigned int id, unsigned int prop)
+{
+	int type;
+
+	/* Check whether the property is supported or not. */
+	type = get_extcon_type(prop);
+	if (type < 0)
+		return false;
 
-	return false;
+	/* Check whether a specific extcon id supports the property or not. */
+	return !!(extcon_info[id].type & type);
+}
+
+static int is_extcon_property_capability(struct extcon_dev *edev,
+				unsigned int id, int index,unsigned int prop)
+{
+	struct extcon_cable *cable;
+	int type, ret;
+
+	/* Check whether the property is supported or not. */
+	type = get_extcon_type(prop);
+	if (type < 0)
+		return type;
+
+	cable = &edev->cables[index];
+
+	switch (type) {
+	case EXTCON_TYPE_USB:
+		ret = test_bit(prop - EXTCON_PROP_USB_MIN, cable->usb_bits);
+		break;
+	case EXTCON_TYPE_CHG:
+		ret = test_bit(prop - EXTCON_PROP_CHG_MIN, cable->chg_bits);
+		break;
+	case EXTCON_TYPE_JACK:
+		ret = test_bit(prop - EXTCON_PROP_JACK_MIN, cable->jack_bits);
+		break;
+	case EXTCON_TYPE_DISP:
+		ret = test_bit(prop - EXTCON_PROP_DISP_MIN, cable->disp_bits);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static void init_property(struct extcon_dev *edev, unsigned int id, int index)
+{
+	unsigned int type = extcon_info[id].type;
+	struct extcon_cable *cable = &edev->cables[index];
+
+	if (EXTCON_TYPE_USB & type)
+		memset(cable->usb_propval, 0, sizeof(cable->usb_propval));
+	if (EXTCON_TYPE_CHG & type)
+		memset(cable->chg_propval, 0, sizeof(cable->chg_propval));
+	if (EXTCON_TYPE_JACK & type)
+		memset(cable->jack_propval, 0, sizeof(cable->jack_propval));
+	if (EXTCON_TYPE_DISP & type)
+		memset(cable->disp_propval, 0, sizeof(cable->disp_propval));
 }
 
 static ssize_t state_show(struct device *dev, struct device_attribute *attr,
@@ -168,32 +373,13 @@ static ssize_t state_show(struct device *dev, struct device_attribute *attr,
 
 	for (i = 0; i < edev->max_supported; i++) {
 		count += sprintf(buf + count, "%s=%d\n",
-				extcon_name[edev->supported_cable[i]],
+				extcon_info[edev->supported_cable[i]].name,
 				 !!(edev->state & (1 << i)));
 	}
 
 	return count;
 }
-
-static ssize_t state_store(struct device *dev, struct device_attribute *attr,
-			   const char *buf, size_t count)
-{
-	u32 state;
-	ssize_t ret = 0;
-	struct extcon_dev *edev = dev_get_drvdata(dev);
-
-	ret = sscanf(buf, "0x%x", &state);
-	if (ret == 0)
-		ret = -EINVAL;
-	else
-		ret = extcon_set_state(edev, state);
-
-	if (ret < 0)
-		return ret;
-
-	return count;
-}
-static DEVICE_ATTR_RW(state);
+static DEVICE_ATTR_RO(state);
 
 static ssize_t name_show(struct device *dev, struct device_attribute *attr,
 		char *buf)
@@ -212,7 +398,7 @@ static ssize_t cable_name_show(struct device *dev,
 	int i = cable->cable_index;
 
 	return sprintf(buf, "%s\n",
-			extcon_name[cable->edev->supported_cable[i]]);
+			extcon_info[cable->edev->supported_cable[i]].name);
 }
 
 static ssize_t cable_state_show(struct device *dev,
@@ -224,26 +410,17 @@ static ssize_t cable_state_show(struct device *dev,
 	int i = cable->cable_index;
 
 	return sprintf(buf, "%d\n",
-		       extcon_get_cable_state_(cable->edev,
-					       cable->edev->supported_cable[i]));
+		extcon_get_state(cable->edev, cable->edev->supported_cable[i]));
 }
 
 /**
- * extcon_update_state() - Update the cable attach states of the extcon device
- *			   only for the masked bits.
- * @edev:	the extcon device
- * @mask:	the bit mask to designate updated bits.
- * @state:	new cable attach status for @edev
- *
- * Changing the state sends uevent with environment variable containing
- * the name of extcon device (envp[0]) and the state output (envp[1]).
- * Tizen uses this format for extcon device to get events from ports.
- * Android uses this format as well.
+ * extcon_sync()	- Synchronize the states for both the attached/detached
+ * @edev:		the extcon device that has the cable.
  *
- * Note that the notifier provides which bits are changed in the state
- * variable with the val parameter (second) to the callback.
+ * This function send a notification to synchronize the all states of a
+ * specific external connector
  */
-int extcon_update_state(struct extcon_dev *edev, u32 mask, u32 state)
+int extcon_sync(struct extcon_dev *edev, unsigned int id)
 {
 	char name_buf[120];
 	char state_buf[120];
@@ -252,100 +429,102 @@ int extcon_update_state(struct extcon_dev *edev, u32 mask, u32 state)
 	int env_offset = 0;
 	int length;
 	int index;
+	int state;
 	unsigned long flags;
-	bool attached;
 
 	if (!edev)
 		return -EINVAL;
 
-	spin_lock_irqsave(&edev->lock, flags);
+	index = find_cable_index_by_id(edev, id);
+	if (index < 0)
+		return index;
 
-	if (edev->state != ((edev->state & ~mask) | (state & mask))) {
-		u32 old_state;
+	spin_lock_irqsave(&edev->lock, flags);
 
-		if (check_mutually_exclusive(edev, (edev->state & ~mask) |
-						   (state & mask))) {
-			spin_unlock_irqrestore(&edev->lock, flags);
-			return -EPERM;
-		}
+	state = !!(edev->state & BIT(index));
+	raw_notifier_call_chain(&edev->nh[index], state, edev);
 
-		old_state = edev->state;
-		edev->state &= ~mask;
-		edev->state |= state & mask;
+	/* This could be in interrupt handler */
+	prop_buf = (char *)get_zeroed_page(GFP_ATOMIC);
+	if (!prop_buf) {
+		/* Unlock early before uevent */
+		spin_unlock_irqrestore(&edev->lock, flags);
 
-		for (index = 0; index < edev->max_supported; index++) {
-			if (is_extcon_changed(old_state, edev->state, index,
-					      &attached))
-				raw_notifier_call_chain(&edev->nh[index],
-							attached, edev);
-		}
+		dev_err(&edev->dev, "out of memory in extcon_set_state\n");
+		kobject_uevent(&edev->dev.kobj, KOBJ_CHANGE);
 
-		/* This could be in interrupt handler */
-		prop_buf = (char *)get_zeroed_page(GFP_ATOMIC);
-		if (prop_buf) {
-			length = name_show(&edev->dev, NULL, prop_buf);
-			if (length > 0) {
-				if (prop_buf[length - 1] == '\n')
-					prop_buf[length - 1] = 0;
-				snprintf(name_buf, sizeof(name_buf),
-					"NAME=%s", prop_buf);
-				envp[env_offset++] = name_buf;
-			}
-			length = state_show(&edev->dev, NULL, prop_buf);
-			if (length > 0) {
-				if (prop_buf[length - 1] == '\n')
-					prop_buf[length - 1] = 0;
-				snprintf(state_buf, sizeof(state_buf),
-					"STATE=%s", prop_buf);
-				envp[env_offset++] = state_buf;
-			}
-			envp[env_offset] = NULL;
-			/* Unlock early before uevent */
-			spin_unlock_irqrestore(&edev->lock, flags);
+		return 0;
+	}
 
-			kobject_uevent_env(&edev->dev.kobj, KOBJ_CHANGE, envp);
-			free_page((unsigned long)prop_buf);
-		} else {
-			/* Unlock early before uevent */
-			spin_unlock_irqrestore(&edev->lock, flags);
+	length = name_show(&edev->dev, NULL, prop_buf);
+	if (length > 0) {
+		if (prop_buf[length - 1] == '\n')
+			prop_buf[length - 1] = 0;
+		snprintf(name_buf, sizeof(name_buf), "NAME=%s", prop_buf);
+		envp[env_offset++] = name_buf;
+	}
 
-			dev_err(&edev->dev, "out of memory in extcon_set_state\n");
-			kobject_uevent(&edev->dev.kobj, KOBJ_CHANGE);
-		}
-	} else {
-		/* No changes */
-		spin_unlock_irqrestore(&edev->lock, flags);
+	length = state_show(&edev->dev, NULL, prop_buf);
+	if (length > 0) {
+		if (prop_buf[length - 1] == '\n')
+			prop_buf[length - 1] = 0;
+		snprintf(state_buf, sizeof(state_buf), "STATE=%s", prop_buf);
+		envp[env_offset++] = state_buf;
 	}
+	envp[env_offset] = NULL;
+
+	/* Unlock early before uevent */
+	spin_unlock_irqrestore(&edev->lock, flags);
+	kobject_uevent_env(&edev->dev.kobj, KOBJ_CHANGE, envp);
+	free_page((unsigned long)prop_buf);
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(extcon_update_state);
+EXPORT_SYMBOL_GPL(extcon_sync);
 
 /**
- * extcon_set_state() - Set the cable attach states of the extcon device.
- * @edev:	the extcon device
- * @state:	new cable attach status for @edev
- *
- * Note that notifier provides which bits are changed in the state
- * variable with the val parameter (second) to the callback.
+ * extcon_get_state() - Get the state of a external connector.
+ * @edev:	the extcon device that has the cable.
+ * @id:		the unique id of each external connector in extcon enumeration.
  */
-int extcon_set_state(struct extcon_dev *edev, u32 state)
+int extcon_get_state(struct extcon_dev *edev, const unsigned int id)
 {
+	int index, state;
+	unsigned long flags;
+
 	if (!edev)
 		return -EINVAL;
 
-	return extcon_update_state(edev, 0xffffffff, state);
+	index = find_cable_index_by_id(edev, id);
+	if (index < 0)
+		return index;
+
+	spin_lock_irqsave(&edev->lock, flags);
+	state = is_extcon_attached(edev, index);
+	spin_unlock_irqrestore(&edev->lock, flags);
+
+	return state;
 }
-EXPORT_SYMBOL_GPL(extcon_set_state);
+EXPORT_SYMBOL_GPL(extcon_get_state);
 
 /**
- * extcon_get_cable_state_() - Get the status of a specific cable.
- * @edev:	the extcon device that has the cable.
- * @id:		the unique id of each external connector in extcon enumeration.
+ * extcon_set_state() - Set the state of a external connector.
+ *			without a notification.
+ * @edev:		the extcon device that has the cable.
+ * @id:			the unique id of each external connector
+ *			in extcon enumeration.
+ * @state:		the new cable status. The default semantics is
+ *			true: attached / false: detached.
+ *
+ * This function only set the state of a external connector without
+ * a notification. To synchronize the data of a external connector,
+ * use extcon_set_state_sync() and extcon_sync().
  */
-int extcon_get_cable_state_(struct extcon_dev *edev, const unsigned int id)
+int extcon_set_state(struct extcon_dev *edev, unsigned int id,
+				bool cable_state)
 {
-	int index;
+	unsigned long flags;
+	int index, ret = 0;
 
 	if (!edev)
 		return -EINVAL;
@@ -354,41 +533,338 @@ int extcon_get_cable_state_(struct extcon_dev *edev, const unsigned int id)
 	if (index < 0)
 		return index;
 
-	if (edev->max_supported && edev->max_supported <= index)
-		return -EINVAL;
+	spin_lock_irqsave(&edev->lock, flags);
+
+	/* Check whether the external connector's state is changed. */
+	if (!is_extcon_changed(edev, index, cable_state))
+		goto out;
+
+	if (check_mutually_exclusive(edev,
+		(edev->state & ~BIT(index)) | (cable_state & BIT(index)))) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	/*
+	 * Initialize the value of extcon property before setting
+	 * the detached state for an external connector.
+	 */
+	if (!cable_state)
+		init_property(edev, id, index);
+
+	/* Update the state for a external connector. */
+	if (cable_state)
+		edev->state |= BIT(index);
+	else
+		edev->state &= ~(BIT(index));
+out:
+	spin_unlock_irqrestore(&edev->lock, flags);
 
-	return !!(edev->state & (1 << index));
+	return ret;
 }
-EXPORT_SYMBOL_GPL(extcon_get_cable_state_);
+EXPORT_SYMBOL_GPL(extcon_set_state);
 
 /**
- * extcon_set_cable_state_() - Set the status of a specific cable.
+ * extcon_set_state_sync() - Set the state of a external connector
+ *			with a notification.
  * @edev:		the extcon device that has the cable.
  * @id:			the unique id of each external connector
  *			in extcon enumeration.
  * @state:		the new cable status. The default semantics is
  *			true: attached / false: detached.
+ *
+ * This function set the state of external connector and synchronize the data
+ * by usning a notification.
  */
-int extcon_set_cable_state_(struct extcon_dev *edev, unsigned int id,
+int extcon_set_state_sync(struct extcon_dev *edev, unsigned int id,
 				bool cable_state)
 {
-	u32 state;
+	int ret, index;
+	unsigned long flags;
+
+	index = find_cable_index_by_id(edev, id);
+	if (index < 0)
+		return index;
+
+	/* Check whether the external connector's state is changed. */
+	spin_lock_irqsave(&edev->lock, flags);
+	ret = is_extcon_changed(edev, index, cable_state);
+	spin_unlock_irqrestore(&edev->lock, flags);
+	if (!ret)
+		return 0;
+
+	ret = extcon_set_state(edev, id, cable_state);
+	if (ret < 0)
+		return ret;
+
+	return extcon_sync(edev, id);
+}
+EXPORT_SYMBOL_GPL(extcon_set_state_sync);
+
+/**
+ * extcon_get_property() - Get the property value of a specific cable.
+ * @edev:		the extcon device that has the cable.
+ * @id:			the unique id of each external connector
+ *			in extcon enumeration.
+ * @prop:		the property id among enum extcon_property.
+ * @prop_val:		the pointer which store the value of property.
+ *
+ * When getting the property value of external connector, the external connector
+ * should be attached. If detached state, function just return 0 without
+ * property value. Also, the each property should be included in the list of
+ * supported properties according to the type of external connectors.
+ *
+ * Returns 0 if success or error number if fail
+ */
+int extcon_get_property(struct extcon_dev *edev, unsigned int id,
+				unsigned int prop,
+				union extcon_property_value *prop_val)
+{
+	struct extcon_cable *cable;
+	unsigned long flags;
+	int index, ret = 0;
+
+	*prop_val = (union extcon_property_value)(0);
+
+	if (!edev)
+		return -EINVAL;
+
+	/* Check whether the property is supported or not */
+	if (!is_extcon_property_supported(id, prop))
+		return -EINVAL;
+
+	/* Find the cable index of external connector by using id */
+	index = find_cable_index_by_id(edev, id);
+	if (index < 0)
+		return index;
+
+	spin_lock_irqsave(&edev->lock, flags);
+
+	/* Check whether the property is available or not. */
+	if (!is_extcon_property_capability(edev, id, index, prop)) {
+		spin_unlock_irqrestore(&edev->lock, flags);
+		return -EPERM;
+	}
+
+	/*
+	 * Check whether the external connector is attached.
+	 * If external connector is detached, the user can not
+	 * get the property value.
+	 */
+	if (!is_extcon_attached(edev, index)) {
+		spin_unlock_irqrestore(&edev->lock, flags);
+		return 0;
+	}
+
+	cable = &edev->cables[index];
+
+	/* Get the property value according to extcon type */
+	switch (prop) {
+	case EXTCON_PROP_USB_MIN ... EXTCON_PROP_USB_MAX:
+		*prop_val = cable->usb_propval[prop - EXTCON_PROP_USB_MIN];
+		break;
+	case EXTCON_PROP_CHG_MIN ... EXTCON_PROP_CHG_MAX:
+		*prop_val = cable->chg_propval[prop - EXTCON_PROP_CHG_MIN];
+		break;
+	case EXTCON_PROP_JACK_MIN ... EXTCON_PROP_JACK_MAX:
+		*prop_val = cable->jack_propval[prop - EXTCON_PROP_JACK_MIN];
+		break;
+	case EXTCON_PROP_DISP_MIN ... EXTCON_PROP_DISP_MAX:
+		*prop_val = cable->disp_propval[prop - EXTCON_PROP_DISP_MIN];
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	spin_unlock_irqrestore(&edev->lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(extcon_get_property);
+
+/**
+ * extcon_set_property() - Set the property value of a specific cable.
+ * @edev:		the extcon device that has the cable.
+ * @id:			the unique id of each external connector
+ *			in extcon enumeration.
+ * @prop:		the property id among enum extcon_property.
+ * @prop_val:		the pointer including the new value of property.
+ *
+ * The each property should be included in the list of supported properties
+ * according to the type of external connectors.
+ *
+ * Returns 0 if success or error number if fail
+ */
+int extcon_set_property(struct extcon_dev *edev, unsigned int id,
+				unsigned int prop,
+				union extcon_property_value prop_val)
+{
+	struct extcon_cable *cable;
+	unsigned long flags;
+	int index, ret = 0;
+
+	if (!edev)
+		return -EINVAL;
+
+	/* Check whether the property is supported or not */
+	if (!is_extcon_property_supported(id, prop))
+		return -EINVAL;
+
+	/* Find the cable index of external connector by using id */
+	index = find_cable_index_by_id(edev, id);
+	if (index < 0)
+		return index;
+
+	spin_lock_irqsave(&edev->lock, flags);
+
+	/* Check whether the property is available or not. */
+	if (!is_extcon_property_capability(edev, id, index, prop)) {
+		spin_unlock_irqrestore(&edev->lock, flags);
+		return -EPERM;
+	}
+
+	cable = &edev->cables[index];
+
+	/* Set the property value according to extcon type */
+	switch (prop) {
+	case EXTCON_PROP_USB_MIN ... EXTCON_PROP_USB_MAX:
+		cable->usb_propval[prop - EXTCON_PROP_USB_MIN] = prop_val;
+		break;
+	case EXTCON_PROP_CHG_MIN ... EXTCON_PROP_CHG_MAX:
+		cable->chg_propval[prop - EXTCON_PROP_CHG_MIN] = prop_val;
+		break;
+	case EXTCON_PROP_JACK_MIN ... EXTCON_PROP_JACK_MAX:
+		cable->jack_propval[prop - EXTCON_PROP_JACK_MIN] = prop_val;
+		break;
+	case EXTCON_PROP_DISP_MIN ... EXTCON_PROP_DISP_MAX:
+		cable->disp_propval[prop - EXTCON_PROP_DISP_MIN] = prop_val;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	spin_unlock_irqrestore(&edev->lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(extcon_set_property);
+
+/**
+ * extcon_set_property_sync() - Set the property value of a specific cable
+			with a notification.
+ * @prop_val:		the pointer including the new value of property.
+ *
+ * When setting the property value of external connector, the external connector
+ * should be attached. The each property should be included in the list of
+ * supported properties according to the type of external connectors.
+ *
+ * Returns 0 if success or error number if fail
+ */
+int extcon_set_property_sync(struct extcon_dev *edev, unsigned int id,
+				unsigned int prop,
+				union extcon_property_value prop_val)
+{
+	int ret;
+
+	ret = extcon_set_property(edev, id, prop, prop_val);
+	if (ret < 0)
+		return ret;
+
+	return extcon_sync(edev, id);
+}
+EXPORT_SYMBOL_GPL(extcon_set_property_sync);
+
+/**
+ * extcon_get_property_capability() - Get the capability of property
+ *			of an external connector.
+ * @edev:		the extcon device that has the cable.
+ * @id:			the unique id of each external connector
+ *			in extcon enumeration.
+ * @prop:		the property id among enum extcon_property.
+ *
+ * Returns 1 if the property is available or 0 if not available.
+ */
+int extcon_get_property_capability(struct extcon_dev *edev, unsigned int id,
+					unsigned int prop)
+{
 	int index;
 
 	if (!edev)
 		return -EINVAL;
 
+	/* Check whether the property is supported or not */
+	if (!is_extcon_property_supported(id, prop))
+		return -EINVAL;
+
+	/* Find the cable index of external connector by using id */
 	index = find_cable_index_by_id(edev, id);
 	if (index < 0)
 		return index;
 
-	if (edev->max_supported && edev->max_supported <= index)
+	return is_extcon_property_capability(edev, id, index, prop);
+}
+EXPORT_SYMBOL_GPL(extcon_get_property_capability);
+
+/**
+ * extcon_set_property_capability() - Set the capability of a property
+ *			of an external connector.
+ * @edev:		the extcon device that has the cable.
+ * @id:			the unique id of each external connector
+ *			in extcon enumeration.
+ * @prop:		the property id among enum extcon_property.
+ *
+ * This function set the capability of a property for an external connector
+ * to mark the bit in capability bitmap which mean the available state of
+ * a property.
+ *
+ * Returns 0 if success or error number if fail
+ */
+int extcon_set_property_capability(struct extcon_dev *edev, unsigned int id,
+					unsigned int prop)
+{
+	struct extcon_cable *cable;
+	int index, type, ret = 0;
+
+	if (!edev)
 		return -EINVAL;
 
-	state = cable_state ? (1 << index) : 0;
-	return extcon_update_state(edev, 1 << index, state);
+	/* Check whether the property is supported or not. */
+	if (!is_extcon_property_supported(id, prop))
+		return -EINVAL;
+
+	/* Find the cable index of external connector by using id. */
+	index = find_cable_index_by_id(edev, id);
+	if (index < 0)
+		return index;
+
+	type = get_extcon_type(prop);
+	if (type < 0)
+		return type;
+
+	cable = &edev->cables[index];
+
+	switch (type) {
+	case EXTCON_TYPE_USB:
+		__set_bit(prop - EXTCON_PROP_USB_MIN, cable->usb_bits);
+		break;
+	case EXTCON_TYPE_CHG:
+		__set_bit(prop - EXTCON_PROP_CHG_MIN, cable->chg_bits);
+		break;
+	case EXTCON_TYPE_JACK:
+		__set_bit(prop - EXTCON_PROP_JACK_MIN, cable->jack_bits);
+		break;
+	case EXTCON_TYPE_DISP:
+		__set_bit(prop - EXTCON_PROP_DISP_MIN, cable->disp_bits);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
 }
-EXPORT_SYMBOL_GPL(extcon_set_cable_state_);
+EXPORT_SYMBOL_GPL(extcon_set_property_capability);
 
 /**
  * extcon_get_extcon_dev() - Get the extcon device instance from the name
@@ -428,7 +904,7 @@ int extcon_register_notifier(struct extcon_dev *edev, unsigned int id,
 			     struct notifier_block *nb)
 {
 	unsigned long flags;
-	int ret, idx;
+	int ret, idx = -EINVAL;
 
 	if (!nb)
 		return -EINVAL;
@@ -846,13 +1322,13 @@ struct extcon_dev *extcon_get_edev_by_phandle(struct device *dev, int index)
 		return ERR_PTR(-EINVAL);
 
 	if (!dev->of_node) {
-		dev_err(dev, "device does not have a device node entry\n");
+		dev_dbg(dev, "device does not have a device node entry\n");
 		return ERR_PTR(-EINVAL);
 	}
 
 	node = of_parse_phandle(dev->of_node, "extcon", index);
 	if (!node) {
-		dev_err(dev, "failed to get phandle in %s node\n",
+		dev_dbg(dev, "failed to get phandle in %s node\n",
 			dev->of_node->full_name);
 		return ERR_PTR(-ENODEV);
 	}
diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig
index 0e22f241403b..bca172d42c74 100644
--- a/drivers/firmware/Kconfig
+++ b/drivers/firmware/Kconfig
@@ -209,5 +209,6 @@ config HAVE_ARM_SMCCC
 source "drivers/firmware/broadcom/Kconfig"
 source "drivers/firmware/google/Kconfig"
 source "drivers/firmware/efi/Kconfig"
+source "drivers/firmware/meson/Kconfig"
 
 endmenu
diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile
index 44a59dcfc398..898ac41fa8b3 100644
--- a/drivers/firmware/Makefile
+++ b/drivers/firmware/Makefile
@@ -22,6 +22,7 @@ obj-$(CONFIG_QCOM_SCM_32)	+= qcom_scm-32.o
 CFLAGS_qcom_scm-32.o :=$(call as-instr,.arch armv7-a\n.arch_extension sec,-DREQUIRES_SEC=1) -march=armv7-a
 
 obj-y				+= broadcom/
+obj-y				+= meson/
 obj-$(CONFIG_GOOGLE_FIRMWARE)	+= google/
 obj-$(CONFIG_EFI)		+= efi/
 obj-$(CONFIG_UEFI_CPER)		+= efi/
diff --git a/drivers/firmware/arm_scpi.c b/drivers/firmware/arm_scpi.c
index 438893762076..ce2bc2a38101 100644
--- a/drivers/firmware/arm_scpi.c
+++ b/drivers/firmware/arm_scpi.c
@@ -709,9 +709,10 @@ static int scpi_probe(struct platform_device *pdev)
 		struct mbox_client *cl = &pchan->cl;
 		struct device_node *shmem = of_parse_phandle(np, "shmem", idx);
 
-		if (of_address_to_resource(shmem, 0, &res)) {
+		ret = of_address_to_resource(shmem, 0, &res);
+		of_node_put(shmem);
+		if (ret) {
 			dev_err(dev, "failed to get SCPI payload mem resource\n");
-			ret = -EINVAL;
 			goto err;
 		}
 
diff --git a/drivers/firmware/dcdbas.c b/drivers/firmware/dcdbas.c
index 829eec8959f2..2fe1a130189f 100644
--- a/drivers/firmware/dcdbas.c
+++ b/drivers/firmware/dcdbas.c
@@ -23,6 +23,7 @@
 #include <linux/platform_device.h>
 #include <linux/dma-mapping.h>
 #include <linux/errno.h>
+#include <linux/cpu.h>
 #include <linux/gfp.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -238,33 +239,14 @@ static ssize_t host_control_on_shutdown_store(struct device *dev,
 	return count;
 }
 
-/**
- * dcdbas_smi_request: generate SMI request
- *
- * Called with smi_data_lock.
- */
-int dcdbas_smi_request(struct smi_cmd *smi_cmd)
+static int raise_smi(void *par)
 {
-	cpumask_var_t old_mask;
-	int ret = 0;
-
-	if (smi_cmd->magic != SMI_CMD_MAGIC) {
-		dev_info(&dcdbas_pdev->dev, "%s: invalid magic value\n",
-			 __func__);
-		return -EBADR;
-	}
+	struct smi_cmd *smi_cmd = par;
 
-	/* SMI requires CPU 0 */
-	if (!alloc_cpumask_var(&old_mask, GFP_KERNEL))
-		return -ENOMEM;
-
-	cpumask_copy(old_mask, &current->cpus_allowed);
-	set_cpus_allowed_ptr(current, cpumask_of(0));
 	if (smp_processor_id() != 0) {
 		dev_dbg(&dcdbas_pdev->dev, "%s: failed to get CPU 0\n",
 			__func__);
-		ret = -EBUSY;
-		goto out;
+		return -EBUSY;
 	}
 
 	/* generate SMI */
@@ -280,9 +262,28 @@ int dcdbas_smi_request(struct smi_cmd *smi_cmd)
 		: "memory"
 	);
 
-out:
-	set_cpus_allowed_ptr(current, old_mask);
-	free_cpumask_var(old_mask);
+	return 0;
+}
+/**
+ * dcdbas_smi_request: generate SMI request
+ *
+ * Called with smi_data_lock.
+ */
+int dcdbas_smi_request(struct smi_cmd *smi_cmd)
+{
+	int ret;
+
+	if (smi_cmd->magic != SMI_CMD_MAGIC) {
+		dev_info(&dcdbas_pdev->dev, "%s: invalid magic value\n",
+			 __func__);
+		return -EBADR;
+	}
+
+	/* SMI requires CPU 0 */
+	get_online_cpus();
+	ret = smp_call_on_cpu(0, raise_smi, smi_cmd, true);
+	put_online_cpus();
+
 	return ret;
 }
 
diff --git a/drivers/firmware/dmi-id.c b/drivers/firmware/dmi-id.c
index 94a58a082b99..44c01390d035 100644
--- a/drivers/firmware/dmi-id.c
+++ b/drivers/firmware/dmi-id.c
@@ -229,14 +229,14 @@ static int __init dmi_id_init(void)
 
 	ret = device_register(dmi_dev);
 	if (ret)
-		goto fail_free_dmi_dev;
+		goto fail_put_dmi_dev;
 
 	return 0;
 
-fail_free_dmi_dev:
-	kfree(dmi_dev);
-fail_class_unregister:
+fail_put_dmi_dev:
+	put_device(dmi_dev);
 
+fail_class_unregister:
 	class_unregister(&dmi_class);
 
 	return ret;
diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig
index 6394152f648f..c981be17d3c0 100644
--- a/drivers/firmware/efi/Kconfig
+++ b/drivers/firmware/efi/Kconfig
@@ -112,6 +112,23 @@ config EFI_CAPSULE_LOADER
 
 	  Most users should say N.
 
+config EFI_TEST
+	tristate "EFI Runtime Service Tests Support"
+	depends on EFI
+	default n
+	help
+	  This driver uses the efi.<service> function pointers directly instead
+	  of going through the efivar API, because it is not trying to test the
+	  kernel subsystem, just for testing the UEFI runtime service
+	  interfaces which are provided by the firmware. This driver is used
+	  by the Firmware Test Suite (FWTS) for testing the UEFI runtime
+	  interfaces readiness of the firmware.
+	  Details for FWTS are available from:
+	  <https://wiki.ubuntu.com/FirmwareTestSuite>
+
+	  Say Y here to enable the runtime services support via /dev/efi_test.
+	  If unsure, say N.
+
 endmenu
 
 config UEFI_CPER
diff --git a/drivers/firmware/efi/Makefile b/drivers/firmware/efi/Makefile
index a219640f881f..c8a439f6d715 100644
--- a/drivers/firmware/efi/Makefile
+++ b/drivers/firmware/efi/Makefile
@@ -10,7 +10,7 @@
 KASAN_SANITIZE_runtime-wrappers.o	:= n
 
 obj-$(CONFIG_EFI)			+= efi.o vars.o reboot.o memattr.o
-obj-$(CONFIG_EFI)			+= capsule.o
+obj-$(CONFIG_EFI)			+= capsule.o memmap.o
 obj-$(CONFIG_EFI_VARS)			+= efivars.o
 obj-$(CONFIG_EFI_ESRT)			+= esrt.o
 obj-$(CONFIG_EFI_VARS_PSTORE)		+= efi-pstore.o
@@ -20,6 +20,7 @@ obj-$(CONFIG_EFI_RUNTIME_WRAPPERS)	+= runtime-wrappers.o
 obj-$(CONFIG_EFI_STUB)			+= libstub/
 obj-$(CONFIG_EFI_FAKE_MEMMAP)		+= fake_mem.o
 obj-$(CONFIG_EFI_BOOTLOADER_CONTROL)	+= efibc.o
+obj-$(CONFIG_EFI_TEST)			+= test/
 
 arm-obj-$(CONFIG_EFI)			:= arm-init.o arm-runtime.o
 obj-$(CONFIG_ARM)			+= $(arm-obj-y)
diff --git a/drivers/firmware/efi/arm-init.c b/drivers/firmware/efi/arm-init.c
index c49d50e68aee..8efe13075c92 100644
--- a/drivers/firmware/efi/arm-init.c
+++ b/drivers/firmware/efi/arm-init.c
@@ -26,9 +26,9 @@
 
 u64 efi_system_table;
 
-static int __init is_normal_ram(efi_memory_desc_t *md)
+static int __init is_memory(efi_memory_desc_t *md)
 {
-	if (md->attribute & EFI_MEMORY_WB)
+	if (md->attribute & (EFI_MEMORY_WB|EFI_MEMORY_WT|EFI_MEMORY_WC))
 		return 1;
 	return 0;
 }
@@ -152,9 +152,9 @@ out:
 }
 
 /*
- * Return true for RAM regions we want to permanently reserve.
+ * Return true for regions that can be used as System RAM.
  */
-static __init int is_reserve_region(efi_memory_desc_t *md)
+static __init int is_usable_memory(efi_memory_desc_t *md)
 {
 	switch (md->type) {
 	case EFI_LOADER_CODE:
@@ -163,18 +163,22 @@ static __init int is_reserve_region(efi_memory_desc_t *md)
 	case EFI_BOOT_SERVICES_DATA:
 	case EFI_CONVENTIONAL_MEMORY:
 	case EFI_PERSISTENT_MEMORY:
-		return 0;
+		/*
+		 * According to the spec, these regions are no longer reserved
+		 * after calling ExitBootServices(). However, we can only use
+		 * them as System RAM if they can be mapped writeback cacheable.
+		 */
+		return (md->attribute & EFI_MEMORY_WB);
 	default:
 		break;
 	}
-	return is_normal_ram(md);
+	return false;
 }
 
 static __init void reserve_regions(void)
 {
 	efi_memory_desc_t *md;
 	u64 paddr, npages, size;
-	int resv;
 
 	if (efi_enabled(EFI_DBG))
 		pr_info("Processing EFI memory map:\n");
@@ -191,32 +195,29 @@ static __init void reserve_regions(void)
 		paddr = md->phys_addr;
 		npages = md->num_pages;
 
-		resv = is_reserve_region(md);
 		if (efi_enabled(EFI_DBG)) {
 			char buf[64];
 
-			pr_info("  0x%012llx-0x%012llx %s%s\n",
+			pr_info("  0x%012llx-0x%012llx %s\n",
 				paddr, paddr + (npages << EFI_PAGE_SHIFT) - 1,
-				efi_md_typeattr_format(buf, sizeof(buf), md),
-				resv ? "*" : "");
+				efi_md_typeattr_format(buf, sizeof(buf), md));
 		}
 
 		memrange_efi_to_native(&paddr, &npages);
 		size = npages << PAGE_SHIFT;
 
-		if (is_normal_ram(md))
+		if (is_memory(md)) {
 			early_init_dt_add_memory_arch(paddr, size);
 
-		if (resv)
-			memblock_mark_nomap(paddr, size);
-
+			if (!is_usable_memory(md))
+				memblock_mark_nomap(paddr, size);
+		}
 	}
-
-	set_bit(EFI_MEMMAP, &efi.flags);
 }
 
 void __init efi_init(void)
 {
+	struct efi_memory_map_data data;
 	struct efi_fdt_params params;
 
 	/* Grab UEFI information placed in FDT by stub */
@@ -225,9 +226,12 @@ void __init efi_init(void)
 
 	efi_system_table = params.system_table;
 
-	efi.memmap.phys_map = params.mmap;
-	efi.memmap.map = early_memremap_ro(params.mmap, params.mmap_size);
-	if (efi.memmap.map == NULL) {
+	data.desc_version = params.desc_ver;
+	data.desc_size = params.desc_size;
+	data.size = params.mmap_size;
+	data.phys_map = params.mmap;
+
+	if (efi_memmap_init_early(&data) < 0) {
 		/*
 		* If we are booting via UEFI, the UEFI memory map is the only
 		* description of memory we have, so there is little point in
@@ -235,9 +239,6 @@ void __init efi_init(void)
 		*/
 		panic("Unable to map EFI memory map.\n");
 	}
-	efi.memmap.map_end = efi.memmap.map + params.mmap_size;
-	efi.memmap.desc_size = params.desc_size;
-	efi.memmap.desc_version = params.desc_ver;
 
 	WARN(efi.memmap.desc_version != 1,
 	     "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
@@ -248,7 +249,8 @@ void __init efi_init(void)
 
 	reserve_regions();
 	efi_memattr_init();
-	early_memunmap(efi.memmap.map, params.mmap_size);
+	efi_esrt_init();
+	efi_memmap_unmap();
 
 	memblock_reserve(params.mmap & PAGE_MASK,
 			 PAGE_ALIGN(params.mmap_size +
diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c
index c394b81fe452..7c75a8d9091a 100644
--- a/drivers/firmware/efi/arm-runtime.c
+++ b/drivers/firmware/efi/arm-runtime.c
@@ -39,6 +39,26 @@ static struct mm_struct efi_mm = {
 	.mmlist			= LIST_HEAD_INIT(efi_mm.mmlist),
 };
 
+#ifdef CONFIG_ARM64_PTDUMP
+#include <asm/ptdump.h>
+
+static struct ptdump_info efi_ptdump_info = {
+	.mm		= &efi_mm,
+	.markers	= (struct addr_marker[]){
+		{ 0,		"UEFI runtime start" },
+		{ TASK_SIZE_64,	"UEFI runtime end" }
+	},
+	.base_addr	= 0,
+};
+
+static int __init ptdump_init(void)
+{
+	return ptdump_register(&efi_ptdump_info, "efi_page_tables");
+}
+device_initcall(ptdump_init);
+
+#endif
+
 static bool __init efi_virtmap_init(void)
 {
 	efi_memory_desc_t *md;
@@ -114,14 +134,12 @@ static int __init arm_enable_runtime_services(void)
 
 	pr_info("Remapping and enabling EFI services.\n");
 
-	mapsize = efi.memmap.map_end - efi.memmap.map;
+	mapsize = efi.memmap.desc_size * efi.memmap.nr_map;
 
-	efi.memmap.map = memremap(efi.memmap.phys_map, mapsize, MEMREMAP_WB);
-	if (!efi.memmap.map) {
+	if (efi_memmap_init_late(efi.memmap.phys_map, mapsize)) {
 		pr_err("Failed to remap EFI memory map\n");
 		return -ENOMEM;
 	}
-	efi.memmap.map_end = efi.memmap.map + mapsize;
 
 	if (!efi_virtmap_init()) {
 		pr_err("UEFI virtual mapping missing or invalid -- runtime services will not be available\n");
diff --git a/drivers/firmware/efi/capsule-loader.c b/drivers/firmware/efi/capsule-loader.c
index c99c24bc79b0..9ae6c116c474 100644
--- a/drivers/firmware/efi/capsule-loader.c
+++ b/drivers/firmware/efi/capsule-loader.c
@@ -16,6 +16,7 @@
 #include <linux/slab.h>
 #include <linux/mutex.h>
 #include <linux/efi.h>
+#include <linux/vmalloc.h>
 
 #define NO_FURTHER_WRITE_ACTION -1
 
@@ -108,14 +109,15 @@ static ssize_t efi_capsule_submit_update(struct capsule_info *cap_info)
 	int ret;
 	void *cap_hdr_temp;
 
-	cap_hdr_temp = kmap(cap_info->pages[0]);
+	cap_hdr_temp = vmap(cap_info->pages, cap_info->index,
+			VM_MAP, PAGE_KERNEL);
 	if (!cap_hdr_temp) {
-		pr_debug("%s: kmap() failed\n", __func__);
+		pr_debug("%s: vmap() failed\n", __func__);
 		return -EFAULT;
 	}
 
 	ret = efi_capsule_update(cap_hdr_temp, cap_info->pages);
-	kunmap(cap_info->pages[0]);
+	vunmap(cap_hdr_temp);
 	if (ret) {
 		pr_err("%s: efi_capsule_update() failed\n", __func__);
 		return ret;
diff --git a/drivers/firmware/efi/capsule.c b/drivers/firmware/efi/capsule.c
index 53b9fd2293ee..6eedff45e6d7 100644
--- a/drivers/firmware/efi/capsule.c
+++ b/drivers/firmware/efi/capsule.c
@@ -190,9 +190,9 @@ efi_capsule_update_locked(efi_capsule_header_t *capsule,
  * map the capsule described by @capsule with its data in @pages and
  * send it to the firmware via the UpdateCapsule() runtime service.
  *
- * @capsule must be a virtual mapping of the first page in @pages
- * (@pages[0]) in the kernel address space. That is, a
- * capsule_header_t that describes the entire contents of the capsule
+ * @capsule must be a virtual mapping of the complete capsule update in the
+ * kernel address space, as the capsule can be consumed immediately.
+ * A capsule_header_t that describes the entire contents of the capsule
  * must be at the start of the first data page.
  *
  * Even though this function will validate that the firmware supports
diff --git a/drivers/firmware/efi/efi-pstore.c b/drivers/firmware/efi/efi-pstore.c
index 30a24d09ea6c..f402ba2eed46 100644
--- a/drivers/firmware/efi/efi-pstore.c
+++ b/drivers/firmware/efi/efi-pstore.c
@@ -125,16 +125,19 @@ static void efi_pstore_scan_sysfs_enter(struct efivar_entry *pos,
  * @entry: deleting entry
  * @turn_off_scanning: Check if a scanning flag should be turned off
  */
-static inline void __efi_pstore_scan_sysfs_exit(struct efivar_entry *entry,
+static inline int __efi_pstore_scan_sysfs_exit(struct efivar_entry *entry,
 						bool turn_off_scanning)
 {
 	if (entry->deleting) {
 		list_del(&entry->list);
 		efivar_entry_iter_end();
 		efivar_unregister(entry);
-		efivar_entry_iter_begin();
+		if (efivar_entry_iter_begin())
+			return -EINTR;
 	} else if (turn_off_scanning)
 		entry->scanning = false;
+
+	return 0;
 }
 
 /**
@@ -144,13 +147,18 @@ static inline void __efi_pstore_scan_sysfs_exit(struct efivar_entry *entry,
  * @head: list head
  * @stop: a flag checking if scanning will stop
  */
-static void efi_pstore_scan_sysfs_exit(struct efivar_entry *pos,
+static int efi_pstore_scan_sysfs_exit(struct efivar_entry *pos,
 				       struct efivar_entry *next,
 				       struct list_head *head, bool stop)
 {
-	__efi_pstore_scan_sysfs_exit(pos, true);
+	int ret = __efi_pstore_scan_sysfs_exit(pos, true);
+
+	if (ret)
+		return ret;
+
 	if (stop)
-		__efi_pstore_scan_sysfs_exit(next, &next->list != head);
+		ret = __efi_pstore_scan_sysfs_exit(next, &next->list != head);
+	return ret;
 }
 
 /**
@@ -172,13 +180,17 @@ static int efi_pstore_sysfs_entry_iter(void *data, struct efivar_entry **pos)
 	struct efivar_entry *entry, *n;
 	struct list_head *head = &efivar_sysfs_list;
 	int size = 0;
+	int ret;
 
 	if (!*pos) {
 		list_for_each_entry_safe(entry, n, head, list) {
 			efi_pstore_scan_sysfs_enter(entry, n, head);
 
 			size = efi_pstore_read_func(entry, data);
-			efi_pstore_scan_sysfs_exit(entry, n, head, size < 0);
+			ret = efi_pstore_scan_sysfs_exit(entry, n, head,
+							 size < 0);
+			if (ret)
+				return ret;
 			if (size)
 				break;
 		}
@@ -190,7 +202,9 @@ static int efi_pstore_sysfs_entry_iter(void *data, struct efivar_entry **pos)
 		efi_pstore_scan_sysfs_enter((*pos), n, head);
 
 		size = efi_pstore_read_func((*pos), data);
-		efi_pstore_scan_sysfs_exit((*pos), n, head, size < 0);
+		ret = efi_pstore_scan_sysfs_exit((*pos), n, head, size < 0);
+		if (ret)
+			return ret;
 		if (size)
 			break;
 	}
@@ -232,7 +246,10 @@ static ssize_t efi_pstore_read(u64 *id, enum pstore_type_id *type,
 	if (!*data.buf)
 		return -ENOMEM;
 
-	efivar_entry_iter_begin();
+	if (efivar_entry_iter_begin()) {
+		kfree(*data.buf);
+		return -EINTR;
+	}
 	size = efi_pstore_sysfs_entry_iter(&data,
 					   (struct efivar_entry **)&psi->data);
 	efivar_entry_iter_end();
@@ -347,7 +364,8 @@ static int efi_pstore_erase(enum pstore_type_id type, u64 id, int count,
 	edata.time = time;
 	edata.name = efi_name;
 
-	efivar_entry_iter_begin();
+	if (efivar_entry_iter_begin())
+		return -EINTR;
 	found = __efivar_entry_iter(efi_pstore_erase_func, &efivar_sysfs_list, &edata, &entry);
 
 	if (found && !entry->scanning) {
@@ -362,7 +380,7 @@ static int efi_pstore_erase(enum pstore_type_id type, u64 id, int count,
 static struct pstore_info efi_pstore_info = {
 	.owner		= THIS_MODULE,
 	.name		= "efi",
-	.flags		= PSTORE_FLAGS_FRAGILE,
+	.flags		= PSTORE_FLAGS_DMESG,
 	.open		= efi_pstore_open,
 	.close		= efi_pstore_close,
 	.read		= efi_pstore_read,
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index 5a2631af7410..1ac199cd75e7 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -27,6 +27,7 @@
 #include <linux/slab.h>
 #include <linux/acpi.h>
 #include <linux/ucs2_string.h>
+#include <linux/memblock.h>
 
 #include <asm/early_ioremap.h>
 
@@ -347,56 +348,31 @@ subsys_initcall(efisubsys_init);
 
 /*
  * Find the efi memory descriptor for a given physical address.  Given a
- * physicall address, determine if it exists within an EFI Memory Map entry,
+ * physical address, determine if it exists within an EFI Memory Map entry,
  * and if so, populate the supplied memory descriptor with the appropriate
  * data.
  */
 int __init efi_mem_desc_lookup(u64 phys_addr, efi_memory_desc_t *out_md)
 {
-	struct efi_memory_map *map = &efi.memmap;
-	phys_addr_t p, e;
+	efi_memory_desc_t *md;
 
 	if (!efi_enabled(EFI_MEMMAP)) {
 		pr_err_once("EFI_MEMMAP is not enabled.\n");
 		return -EINVAL;
 	}
 
-	if (!map) {
-		pr_err_once("efi.memmap is not set.\n");
-		return -EINVAL;
-	}
 	if (!out_md) {
 		pr_err_once("out_md is null.\n");
 		return -EINVAL;
         }
-	if (WARN_ON_ONCE(!map->phys_map))
-		return -EINVAL;
-	if (WARN_ON_ONCE(map->nr_map == 0) || WARN_ON_ONCE(map->desc_size == 0))
-		return -EINVAL;
 
-	e = map->phys_map + map->nr_map * map->desc_size;
-	for (p = map->phys_map; p < e; p += map->desc_size) {
-		efi_memory_desc_t *md;
+	for_each_efi_memory_desc(md) {
 		u64 size;
 		u64 end;
 
-		/*
-		 * If a driver calls this after efi_free_boot_services,
-		 * ->map will be NULL, and the target may also not be mapped.
-		 * So just always get our own virtual map on the CPU.
-		 *
-		 */
-		md = early_memremap(p, sizeof (*md));
-		if (!md) {
-			pr_err_once("early_memremap(%pa, %zu) failed.\n",
-				    &p, sizeof (*md));
-			return -ENOMEM;
-		}
-
 		if (!(md->attribute & EFI_MEMORY_RUNTIME) &&
 		    md->type != EFI_BOOT_SERVICES_DATA &&
 		    md->type != EFI_RUNTIME_SERVICES_DATA) {
-			early_memunmap(md, sizeof (*md));
 			continue;
 		}
 
@@ -404,11 +380,8 @@ int __init efi_mem_desc_lookup(u64 phys_addr, efi_memory_desc_t *out_md)
 		end = md->phys_addr + size;
 		if (phys_addr >= md->phys_addr && phys_addr < end) {
 			memcpy(out_md, md, sizeof(*out_md));
-			early_memunmap(md, sizeof (*md));
 			return 0;
 		}
-
-		early_memunmap(md, sizeof (*md));
 	}
 	pr_err_once("requested map not found.\n");
 	return -ENOENT;
@@ -424,6 +397,35 @@ u64 __init efi_mem_desc_end(efi_memory_desc_t *md)
 	return end;
 }
 
+void __init __weak efi_arch_mem_reserve(phys_addr_t addr, u64 size) {}
+
+/**
+ * efi_mem_reserve - Reserve an EFI memory region
+ * @addr: Physical address to reserve
+ * @size: Size of reservation
+ *
+ * Mark a region as reserved from general kernel allocation and
+ * prevent it being released by efi_free_boot_services().
+ *
+ * This function should be called drivers once they've parsed EFI
+ * configuration tables to figure out where their data lives, e.g.
+ * efi_esrt_init().
+ */
+void __init efi_mem_reserve(phys_addr_t addr, u64 size)
+{
+	if (!memblock_is_region_reserved(addr, size))
+		memblock_reserve(addr, size);
+
+	/*
+	 * Some architectures (x86) reserve all boot services ranges
+	 * until efi_free_boot_services() because of buggy firmware
+	 * implementations. This means the above memblock_reserve() is
+	 * superfluous on x86 and instead what it needs to do is
+	 * ensure the @start, @size is not freed.
+	 */
+	efi_arch_mem_reserve(addr, size);
+}
+
 static __initdata efi_config_table_type_t common_tables[] = {
 	{ACPI_20_TABLE_GUID, "ACPI 2.0", &efi.acpi20},
 	{ACPI_TABLE_GUID, "ACPI", &efi.acpi},
@@ -657,9 +659,12 @@ static int __init fdt_find_uefi_params(unsigned long node, const char *uname,
 		}
 
 		if (subnode) {
-			node = of_get_flat_dt_subnode_by_name(node, subnode);
-			if (node < 0)
+			int err = of_get_flat_dt_subnode_by_name(node, subnode);
+
+			if (err < 0)
 				return 0;
+
+			node = err;
 		}
 
 		return __find_uefi_params(node, info, dt_params[i].params);
@@ -808,6 +813,9 @@ int efi_status_to_err(efi_status_t status)
 	case EFI_NOT_FOUND:
 		err = -ENOENT;
 		break;
+	case EFI_ABORTED:
+		err = -EINTR;
+		break;
 	default:
 		err = -EINVAL;
 	}
diff --git a/drivers/firmware/efi/efivars.c b/drivers/firmware/efi/efivars.c
index 116b244dee68..3e626fd9bd4e 100644
--- a/drivers/firmware/efi/efivars.c
+++ b/drivers/firmware/efi/efivars.c
@@ -510,7 +510,8 @@ static ssize_t efivar_delete(struct file *filp, struct kobject *kobj,
 		vendor = del_var->VendorGuid;
 	}
 
-	efivar_entry_iter_begin();
+	if (efivar_entry_iter_begin())
+		return -EINTR;
 	entry = efivar_entry_find(name, vendor, &efivar_sysfs_list, true);
 	if (!entry)
 		err = -EINVAL;
@@ -575,7 +576,10 @@ efivar_create_sysfs_entry(struct efivar_entry *new_var)
 		return ret;
 
 	kobject_uevent(&new_var->kobj, KOBJ_ADD);
-	efivar_entry_add(new_var, &efivar_sysfs_list);
+	if (efivar_entry_add(new_var, &efivar_sysfs_list)) {
+		efivar_unregister(new_var);
+		return -EINTR;
+	}
 
 	return 0;
 }
@@ -690,7 +694,10 @@ static int efivars_sysfs_callback(efi_char16_t *name, efi_guid_t vendor,
 
 static int efivar_sysfs_destroy(struct efivar_entry *entry, void *data)
 {
-	efivar_entry_remove(entry);
+	int err = efivar_entry_remove(entry);
+
+	if (err)
+		return err;
 	efivar_unregister(entry);
 	return 0;
 }
@@ -698,7 +705,14 @@ static int efivar_sysfs_destroy(struct efivar_entry *entry, void *data)
 static void efivars_sysfs_exit(void)
 {
 	/* Remove all entries and destroy */
-	__efivar_entry_iter(efivar_sysfs_destroy, &efivar_sysfs_list, NULL, NULL);
+	int err;
+
+	err = __efivar_entry_iter(efivar_sysfs_destroy, &efivar_sysfs_list,
+				  NULL, NULL);
+	if (err) {
+		pr_err("efivars: Failed to destroy sysfs entries\n");
+		return;
+	}
 
 	if (efivars_new_var)
 		sysfs_remove_bin_file(&efivars_kset->kobj, efivars_new_var);
diff --git a/drivers/firmware/efi/esrt.c b/drivers/firmware/efi/esrt.c
index 75feb3f5829b..14914074f716 100644
--- a/drivers/firmware/efi/esrt.c
+++ b/drivers/firmware/efi/esrt.c
@@ -16,6 +16,7 @@
 #include <linux/device.h>
 #include <linux/efi.h>
 #include <linux/init.h>
+#include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/kobject.h>
 #include <linux/list.h>
@@ -235,7 +236,7 @@ static struct attribute_group esrt_attr_group = {
 };
 
 /*
- * remap the table, copy it to kmalloced pages, and unmap it.
+ * remap the table, validate it, mark it reserved and unmap it.
  */
 void __init efi_esrt_init(void)
 {
@@ -335,7 +336,7 @@ void __init efi_esrt_init(void)
 
 	end = esrt_data + size;
 	pr_info("Reserving ESRT space from %pa to %pa.\n", &esrt_data, &end);
-	memblock_reserve(esrt_data, esrt_data_size);
+	efi_mem_reserve(esrt_data, esrt_data_size);
 
 	pr_debug("esrt-init: loaded.\n");
 err_memunmap:
@@ -382,28 +383,18 @@ static void cleanup_entry_list(void)
 static int __init esrt_sysfs_init(void)
 {
 	int error;
-	struct efi_system_resource_table __iomem *ioesrt;
 
 	pr_debug("esrt-sysfs: loading.\n");
 	if (!esrt_data || !esrt_data_size)
 		return -ENOSYS;
 
-	ioesrt = ioremap(esrt_data, esrt_data_size);
-	if (!ioesrt) {
-		pr_err("ioremap(%pa, %zu) failed.\n", &esrt_data,
-		       esrt_data_size);
-		return -ENOMEM;
-	}
-
-	esrt = kmalloc(esrt_data_size, GFP_KERNEL);
+	esrt = memremap(esrt_data, esrt_data_size, MEMREMAP_WB);
 	if (!esrt) {
-		pr_err("kmalloc failed. (wanted %zu bytes)\n", esrt_data_size);
-		iounmap(ioesrt);
+		pr_err("memremap(%pa, %zu) failed.\n", &esrt_data,
+		       esrt_data_size);
 		return -ENOMEM;
 	}
 
-	memcpy_fromio(esrt, ioesrt, esrt_data_size);
-
 	esrt_kobj = kobject_create_and_add("esrt", efi_kobj);
 	if (!esrt_kobj) {
 		pr_err("Firmware table registration failed.\n");
@@ -429,8 +420,6 @@ static int __init esrt_sysfs_init(void)
 	if (error)
 		goto err_cleanup_list;
 
-	memblock_remove(esrt_data, esrt_data_size);
-
 	pr_debug("esrt-sysfs: loaded.\n");
 
 	return 0;
diff --git a/drivers/firmware/efi/fake_mem.c b/drivers/firmware/efi/fake_mem.c
index 48430aba13c1..520a40e5e0e4 100644
--- a/drivers/firmware/efi/fake_mem.c
+++ b/drivers/firmware/efi/fake_mem.c
@@ -35,17 +35,13 @@
 
 #define EFI_MAX_FAKEMEM CONFIG_EFI_MAX_FAKE_MEM
 
-struct fake_mem {
-	struct range range;
-	u64 attribute;
-};
-static struct fake_mem fake_mems[EFI_MAX_FAKEMEM];
+static struct efi_mem_range fake_mems[EFI_MAX_FAKEMEM];
 static int nr_fake_mem;
 
 static int __init cmp_fake_mem(const void *x1, const void *x2)
 {
-	const struct fake_mem *m1 = x1;
-	const struct fake_mem *m2 = x2;
+	const struct efi_mem_range *m1 = x1;
+	const struct efi_mem_range *m2 = x2;
 
 	if (m1->range.start < m2->range.start)
 		return -1;
@@ -56,40 +52,21 @@ static int __init cmp_fake_mem(const void *x1, const void *x2)
 
 void __init efi_fake_memmap(void)
 {
-	u64 start, end, m_start, m_end, m_attr;
 	int new_nr_map = efi.memmap.nr_map;
 	efi_memory_desc_t *md;
 	phys_addr_t new_memmap_phy;
 	void *new_memmap;
-	void *old, *new;
 	int i;
 
-	if (!nr_fake_mem || !efi_enabled(EFI_MEMMAP))
+	if (!nr_fake_mem)
 		return;
 
 	/* count up the number of EFI memory descriptor */
-	for_each_efi_memory_desc(md) {
-		start = md->phys_addr;
-		end = start + (md->num_pages << EFI_PAGE_SHIFT) - 1;
-
-		for (i = 0; i < nr_fake_mem; i++) {
-			/* modifying range */
-			m_start = fake_mems[i].range.start;
-			m_end = fake_mems[i].range.end;
-
-			if (m_start <= start) {
-				/* split into 2 parts */
-				if (start < m_end && m_end < end)
-					new_nr_map++;
-			}
-			if (start < m_start && m_start < end) {
-				/* split into 3 parts */
-				if (m_end < end)
-					new_nr_map += 2;
-				/* split into 2 parts */
-				if (end <= m_end)
-					new_nr_map++;
-			}
+	for (i = 0; i < nr_fake_mem; i++) {
+		for_each_efi_memory_desc(md) {
+			struct range *r = &fake_mems[i].range;
+
+			new_nr_map += efi_memmap_split_count(md, r);
 		}
 	}
 
@@ -107,85 +84,13 @@ void __init efi_fake_memmap(void)
 		return;
 	}
 
-	for (old = efi.memmap.map, new = new_memmap;
-	     old < efi.memmap.map_end;
-	     old += efi.memmap.desc_size, new += efi.memmap.desc_size) {
-
-		/* copy original EFI memory descriptor */
-		memcpy(new, old, efi.memmap.desc_size);
-		md = new;
-		start = md->phys_addr;
-		end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1;
-
-		for (i = 0; i < nr_fake_mem; i++) {
-			/* modifying range */
-			m_start = fake_mems[i].range.start;
-			m_end = fake_mems[i].range.end;
-			m_attr = fake_mems[i].attribute;
-
-			if (m_start <= start && end <= m_end)
-				md->attribute |= m_attr;
-
-			if (m_start <= start &&
-			    (start < m_end && m_end < end)) {
-				/* first part */
-				md->attribute |= m_attr;
-				md->num_pages = (m_end - md->phys_addr + 1) >>
-					EFI_PAGE_SHIFT;
-				/* latter part */
-				new += efi.memmap.desc_size;
-				memcpy(new, old, efi.memmap.desc_size);
-				md = new;
-				md->phys_addr = m_end + 1;
-				md->num_pages = (end - md->phys_addr + 1) >>
-					EFI_PAGE_SHIFT;
-			}
-
-			if ((start < m_start && m_start < end) && m_end < end) {
-				/* first part */
-				md->num_pages = (m_start - md->phys_addr) >>
-					EFI_PAGE_SHIFT;
-				/* middle part */
-				new += efi.memmap.desc_size;
-				memcpy(new, old, efi.memmap.desc_size);
-				md = new;
-				md->attribute |= m_attr;
-				md->phys_addr = m_start;
-				md->num_pages = (m_end - m_start + 1) >>
-					EFI_PAGE_SHIFT;
-				/* last part */
-				new += efi.memmap.desc_size;
-				memcpy(new, old, efi.memmap.desc_size);
-				md = new;
-				md->phys_addr = m_end + 1;
-				md->num_pages = (end - m_end) >>
-					EFI_PAGE_SHIFT;
-			}
-
-			if ((start < m_start && m_start < end) &&
-			    (end <= m_end)) {
-				/* first part */
-				md->num_pages = (m_start - md->phys_addr) >>
-					EFI_PAGE_SHIFT;
-				/* latter part */
-				new += efi.memmap.desc_size;
-				memcpy(new, old, efi.memmap.desc_size);
-				md = new;
-				md->phys_addr = m_start;
-				md->num_pages = (end - md->phys_addr + 1) >>
-					EFI_PAGE_SHIFT;
-				md->attribute |= m_attr;
-			}
-		}
-	}
+	for (i = 0; i < nr_fake_mem; i++)
+		efi_memmap_insert(&efi.memmap, new_memmap, &fake_mems[i]);
 
 	/* swap into new EFI memmap */
-	efi_unmap_memmap();
-	efi.memmap.map = new_memmap;
-	efi.memmap.phys_map = new_memmap_phy;
-	efi.memmap.nr_map = new_nr_map;
-	efi.memmap.map_end = efi.memmap.map + efi.memmap.nr_map * efi.memmap.desc_size;
-	set_bit(EFI_MEMMAP, &efi.flags);
+	early_memunmap(new_memmap, efi.memmap.desc_size * new_nr_map);
+
+	efi_memmap_install(new_memmap_phy, new_nr_map);
 
 	/* print new EFI memmap */
 	efi_print_memmap();
@@ -223,7 +128,7 @@ static int __init setup_fake_mem(char *p)
 			p++;
 	}
 
-	sort(fake_mems, nr_fake_mem, sizeof(struct fake_mem),
+	sort(fake_mems, nr_fake_mem, sizeof(struct efi_mem_range),
 	     cmp_fake_mem, NULL);
 
 	for (i = 0; i < nr_fake_mem; i++)
diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c
index 3bd127f95315..aded10662020 100644
--- a/drivers/firmware/efi/libstub/efi-stub-helper.c
+++ b/drivers/firmware/efi/libstub/efi-stub-helper.c
@@ -41,6 +41,8 @@ static unsigned long __chunk_size = EFI_READ_CHUNK_SIZE;
 #define EFI_ALLOC_ALIGN		EFI_PAGE_SIZE
 #endif
 
+#define EFI_MMAP_NR_SLACK_SLOTS	8
+
 struct file_info {
 	efi_file_handle_t *handle;
 	u64 size;
@@ -63,49 +65,62 @@ void efi_printk(efi_system_table_t *sys_table_arg, char *str)
 	}
 }
 
+static inline bool mmap_has_headroom(unsigned long buff_size,
+				     unsigned long map_size,
+				     unsigned long desc_size)
+{
+	unsigned long slack = buff_size - map_size;
+
+	return slack / desc_size >= EFI_MMAP_NR_SLACK_SLOTS;
+}
+
 efi_status_t efi_get_memory_map(efi_system_table_t *sys_table_arg,
-				efi_memory_desc_t **map,
-				unsigned long *map_size,
-				unsigned long *desc_size,
-				u32 *desc_ver,
-				unsigned long *key_ptr)
+				struct efi_boot_memmap *map)
 {
 	efi_memory_desc_t *m = NULL;
 	efi_status_t status;
 	unsigned long key;
 	u32 desc_version;
 
-	*map_size = sizeof(*m) * 32;
+	*map->desc_size =	sizeof(*m);
+	*map->map_size =	*map->desc_size * 32;
+	*map->buff_size =	*map->map_size;
 again:
-	/*
-	 * Add an additional efi_memory_desc_t because we're doing an
-	 * allocation which may be in a new descriptor region.
-	 */
-	*map_size += sizeof(*m);
 	status = efi_call_early(allocate_pool, EFI_LOADER_DATA,
-				*map_size, (void **)&m);
+				*map->map_size, (void **)&m);
 	if (status != EFI_SUCCESS)
 		goto fail;
 
-	*desc_size = 0;
+	*map->desc_size = 0;
 	key = 0;
-	status = efi_call_early(get_memory_map, map_size, m,
-				&key, desc_size, &desc_version);
-	if (status == EFI_BUFFER_TOO_SMALL) {
+	status = efi_call_early(get_memory_map, map->map_size, m,
+				&key, map->desc_size, &desc_version);
+	if (status == EFI_BUFFER_TOO_SMALL ||
+	    !mmap_has_headroom(*map->buff_size, *map->map_size,
+			       *map->desc_size)) {
 		efi_call_early(free_pool, m);
+		/*
+		 * Make sure there is some entries of headroom so that the
+		 * buffer can be reused for a new map after allocations are
+		 * no longer permitted.  Its unlikely that the map will grow to
+		 * exceed this headroom once we are ready to trigger
+		 * ExitBootServices()
+		 */
+		*map->map_size += *map->desc_size * EFI_MMAP_NR_SLACK_SLOTS;
+		*map->buff_size = *map->map_size;
 		goto again;
 	}
 
 	if (status != EFI_SUCCESS)
 		efi_call_early(free_pool, m);
 
-	if (key_ptr && status == EFI_SUCCESS)
-		*key_ptr = key;
-	if (desc_ver && status == EFI_SUCCESS)
-		*desc_ver = desc_version;
+	if (map->key_ptr && status == EFI_SUCCESS)
+		*map->key_ptr = key;
+	if (map->desc_ver && status == EFI_SUCCESS)
+		*map->desc_ver = desc_version;
 
 fail:
-	*map = m;
+	*map->map = m;
 	return status;
 }
 
@@ -113,13 +128,20 @@ fail:
 unsigned long get_dram_base(efi_system_table_t *sys_table_arg)
 {
 	efi_status_t status;
-	unsigned long map_size;
+	unsigned long map_size, buff_size;
 	unsigned long membase  = EFI_ERROR;
 	struct efi_memory_map map;
 	efi_memory_desc_t *md;
+	struct efi_boot_memmap boot_map;
 
-	status = efi_get_memory_map(sys_table_arg, (efi_memory_desc_t **)&map.map,
-				    &map_size, &map.desc_size, NULL, NULL);
+	boot_map.map =		(efi_memory_desc_t **)&map.map;
+	boot_map.map_size =	&map_size;
+	boot_map.desc_size =	&map.desc_size;
+	boot_map.desc_ver =	NULL;
+	boot_map.key_ptr =	NULL;
+	boot_map.buff_size =	&buff_size;
+
+	status = efi_get_memory_map(sys_table_arg, &boot_map);
 	if (status != EFI_SUCCESS)
 		return membase;
 
@@ -144,15 +166,22 @@ efi_status_t efi_high_alloc(efi_system_table_t *sys_table_arg,
 			    unsigned long size, unsigned long align,
 			    unsigned long *addr, unsigned long max)
 {
-	unsigned long map_size, desc_size;
+	unsigned long map_size, desc_size, buff_size;
 	efi_memory_desc_t *map;
 	efi_status_t status;
 	unsigned long nr_pages;
 	u64 max_addr = 0;
 	int i;
+	struct efi_boot_memmap boot_map;
+
+	boot_map.map =		&map;
+	boot_map.map_size =	&map_size;
+	boot_map.desc_size =	&desc_size;
+	boot_map.desc_ver =	NULL;
+	boot_map.key_ptr =	NULL;
+	boot_map.buff_size =	&buff_size;
 
-	status = efi_get_memory_map(sys_table_arg, &map, &map_size, &desc_size,
-				    NULL, NULL);
+	status = efi_get_memory_map(sys_table_arg, &boot_map);
 	if (status != EFI_SUCCESS)
 		goto fail;
 
@@ -230,14 +259,21 @@ efi_status_t efi_low_alloc(efi_system_table_t *sys_table_arg,
 			   unsigned long size, unsigned long align,
 			   unsigned long *addr)
 {
-	unsigned long map_size, desc_size;
+	unsigned long map_size, desc_size, buff_size;
 	efi_memory_desc_t *map;
 	efi_status_t status;
 	unsigned long nr_pages;
 	int i;
+	struct efi_boot_memmap boot_map;
 
-	status = efi_get_memory_map(sys_table_arg, &map, &map_size, &desc_size,
-				    NULL, NULL);
+	boot_map.map =		&map;
+	boot_map.map_size =	&map_size;
+	boot_map.desc_size =	&desc_size;
+	boot_map.desc_ver =	NULL;
+	boot_map.key_ptr =	NULL;
+	boot_map.buff_size =	&buff_size;
+
+	status = efi_get_memory_map(sys_table_arg, &boot_map);
 	if (status != EFI_SUCCESS)
 		goto fail;
 
@@ -704,3 +740,76 @@ char *efi_convert_cmdline(efi_system_table_t *sys_table_arg,
 	*cmd_line_len = options_bytes;
 	return (char *)cmdline_addr;
 }
+
+/*
+ * Handle calling ExitBootServices according to the requirements set out by the
+ * spec.  Obtains the current memory map, and returns that info after calling
+ * ExitBootServices.  The client must specify a function to perform any
+ * processing of the memory map data prior to ExitBootServices.  A client
+ * specific structure may be passed to the function via priv.  The client
+ * function may be called multiple times.
+ */
+efi_status_t efi_exit_boot_services(efi_system_table_t *sys_table_arg,
+				    void *handle,
+				    struct efi_boot_memmap *map,
+				    void *priv,
+				    efi_exit_boot_map_processing priv_func)
+{
+	efi_status_t status;
+
+	status = efi_get_memory_map(sys_table_arg, map);
+
+	if (status != EFI_SUCCESS)
+		goto fail;
+
+	status = priv_func(sys_table_arg, map, priv);
+	if (status != EFI_SUCCESS)
+		goto free_map;
+
+	status = efi_call_early(exit_boot_services, handle, *map->key_ptr);
+
+	if (status == EFI_INVALID_PARAMETER) {
+		/*
+		 * The memory map changed between efi_get_memory_map() and
+		 * exit_boot_services().  Per the UEFI Spec v2.6, Section 6.4:
+		 * EFI_BOOT_SERVICES.ExitBootServices we need to get the
+		 * updated map, and try again.  The spec implies one retry
+		 * should be sufficent, which is confirmed against the EDK2
+		 * implementation.  Per the spec, we can only invoke
+		 * get_memory_map() and exit_boot_services() - we cannot alloc
+		 * so efi_get_memory_map() cannot be used, and we must reuse
+		 * the buffer.  For all practical purposes, the headroom in the
+		 * buffer should account for any changes in the map so the call
+		 * to get_memory_map() is expected to succeed here.
+		 */
+		*map->map_size = *map->buff_size;
+		status = efi_call_early(get_memory_map,
+					map->map_size,
+					*map->map,
+					map->key_ptr,
+					map->desc_size,
+					map->desc_ver);
+
+		/* exit_boot_services() was called, thus cannot free */
+		if (status != EFI_SUCCESS)
+			goto fail;
+
+		status = priv_func(sys_table_arg, map, priv);
+		/* exit_boot_services() was called, thus cannot free */
+		if (status != EFI_SUCCESS)
+			goto fail;
+
+		status = efi_call_early(exit_boot_services, handle, *map->key_ptr);
+	}
+
+	/* exit_boot_services() was called, thus cannot free */
+	if (status != EFI_SUCCESS)
+		goto fail;
+
+	return EFI_SUCCESS;
+
+free_map:
+	efi_call_early(free_pool, *map->map);
+fail:
+	return status;
+}
diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c
index e58abfa953cc..a6a93116a8f0 100644
--- a/drivers/firmware/efi/libstub/fdt.c
+++ b/drivers/firmware/efi/libstub/fdt.c
@@ -152,6 +152,27 @@ fdt_set_fail:
 #define EFI_FDT_ALIGN EFI_PAGE_SIZE
 #endif
 
+struct exit_boot_struct {
+	efi_memory_desc_t *runtime_map;
+	int *runtime_entry_count;
+};
+
+static efi_status_t exit_boot_func(efi_system_table_t *sys_table_arg,
+				   struct efi_boot_memmap *map,
+				   void *priv)
+{
+	struct exit_boot_struct *p = priv;
+	/*
+	 * Update the memory map with virtual addresses. The function will also
+	 * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME
+	 * entries so that we can pass it straight to SetVirtualAddressMap()
+	 */
+	efi_get_virtmap(*map->map, *map->map_size, *map->desc_size,
+			p->runtime_map, p->runtime_entry_count);
+
+	return EFI_SUCCESS;
+}
+
 /*
  * Allocate memory for a new FDT, then add EFI, commandline, and
  * initrd related fields to the FDT.  This routine increases the
@@ -175,13 +196,22 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
 					    unsigned long fdt_addr,
 					    unsigned long fdt_size)
 {
-	unsigned long map_size, desc_size;
+	unsigned long map_size, desc_size, buff_size;
 	u32 desc_ver;
 	unsigned long mmap_key;
 	efi_memory_desc_t *memory_map, *runtime_map;
 	unsigned long new_fdt_size;
 	efi_status_t status;
 	int runtime_entry_count = 0;
+	struct efi_boot_memmap map;
+	struct exit_boot_struct priv;
+
+	map.map =	&runtime_map;
+	map.map_size =	&map_size;
+	map.desc_size =	&desc_size;
+	map.desc_ver =	&desc_ver;
+	map.key_ptr =	&mmap_key;
+	map.buff_size =	&buff_size;
 
 	/*
 	 * Get a copy of the current memory map that we will use to prepare
@@ -189,8 +219,7 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
 	 * subsequent allocations adding entries, since they could not affect
 	 * the number of EFI_MEMORY_RUNTIME regions.
 	 */
-	status = efi_get_memory_map(sys_table, &runtime_map, &map_size,
-				    &desc_size, &desc_ver, &mmap_key);
+	status = efi_get_memory_map(sys_table, &map);
 	if (status != EFI_SUCCESS) {
 		pr_efi_err(sys_table, "Unable to retrieve UEFI memory map.\n");
 		return status;
@@ -199,6 +228,7 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
 	pr_efi(sys_table,
 	       "Exiting boot services and installing virtual address map...\n");
 
+	map.map = &memory_map;
 	/*
 	 * Estimate size of new FDT, and allocate memory for it. We
 	 * will allocate a bigger buffer if this ends up being too
@@ -218,8 +248,7 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
 		 * we can get the memory map key  needed for
 		 * exit_boot_services().
 		 */
-		status = efi_get_memory_map(sys_table, &memory_map, &map_size,
-					    &desc_size, &desc_ver, &mmap_key);
+		status = efi_get_memory_map(sys_table, &map);
 		if (status != EFI_SUCCESS)
 			goto fail_free_new_fdt;
 
@@ -250,16 +279,11 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
 		}
 	}
 
-	/*
-	 * Update the memory map with virtual addresses. The function will also
-	 * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME
-	 * entries so that we can pass it straight into SetVirtualAddressMap()
-	 */
-	efi_get_virtmap(memory_map, map_size, desc_size, runtime_map,
-			&runtime_entry_count);
-
-	/* Now we are ready to exit_boot_services.*/
-	status = sys_table->boottime->exit_boot_services(handle, mmap_key);
+	sys_table->boottime->free_pool(memory_map);
+	priv.runtime_map = runtime_map;
+	priv.runtime_entry_count = &runtime_entry_count;
+	status = efi_exit_boot_services(sys_table, handle, &map, &priv,
+					exit_boot_func);
 
 	if (status == EFI_SUCCESS) {
 		efi_set_virtual_address_map_t *svam;
diff --git a/drivers/firmware/efi/libstub/random.c b/drivers/firmware/efi/libstub/random.c
index 53f6d3fe6d86..0c9f58c5ba50 100644
--- a/drivers/firmware/efi/libstub/random.c
+++ b/drivers/firmware/efi/libstub/random.c
@@ -73,12 +73,20 @@ efi_status_t efi_random_alloc(efi_system_table_t *sys_table_arg,
 			      unsigned long random_seed)
 {
 	unsigned long map_size, desc_size, total_slots = 0, target_slot;
+	unsigned long buff_size;
 	efi_status_t status;
 	efi_memory_desc_t *memory_map;
 	int map_offset;
+	struct efi_boot_memmap map;
 
-	status = efi_get_memory_map(sys_table_arg, &memory_map, &map_size,
-				    &desc_size, NULL, NULL);
+	map.map =	&memory_map;
+	map.map_size =	&map_size;
+	map.desc_size =	&desc_size;
+	map.desc_ver =	NULL;
+	map.key_ptr =	NULL;
+	map.buff_size =	&buff_size;
+
+	status = efi_get_memory_map(sys_table_arg, &map);
 	if (status != EFI_SUCCESS)
 		return status;
 
diff --git a/drivers/firmware/efi/memmap.c b/drivers/firmware/efi/memmap.c
new file mode 100644
index 000000000000..f03ddecd232b
--- /dev/null
+++ b/drivers/firmware/efi/memmap.c
@@ -0,0 +1,303 @@
+/*
+ * Common EFI memory map functions.
+ */
+
+#define pr_fmt(fmt) "efi: " fmt
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/efi.h>
+#include <linux/io.h>
+#include <asm/early_ioremap.h>
+
+/**
+ * __efi_memmap_init - Common code for mapping the EFI memory map
+ * @data: EFI memory map data
+ * @late: Use early or late mapping function?
+ *
+ * This function takes care of figuring out which function to use to
+ * map the EFI memory map in efi.memmap based on how far into the boot
+ * we are.
+ *
+ * During bootup @late should be %false since we only have access to
+ * the early_memremap*() functions as the vmalloc space isn't setup.
+ * Once the kernel is fully booted we can fallback to the more robust
+ * memremap*() API.
+ *
+ * Returns zero on success, a negative error code on failure.
+ */
+static int __init
+__efi_memmap_init(struct efi_memory_map_data *data, bool late)
+{
+	struct efi_memory_map map;
+	phys_addr_t phys_map;
+
+	if (efi_enabled(EFI_PARAVIRT))
+		return 0;
+
+	phys_map = data->phys_map;
+
+	if (late)
+		map.map = memremap(phys_map, data->size, MEMREMAP_WB);
+	else
+		map.map = early_memremap(phys_map, data->size);
+
+	if (!map.map) {
+		pr_err("Could not map the memory map!\n");
+		return -ENOMEM;
+	}
+
+	map.phys_map = data->phys_map;
+	map.nr_map = data->size / data->desc_size;
+	map.map_end = map.map + data->size;
+
+	map.desc_version = data->desc_version;
+	map.desc_size = data->desc_size;
+	map.late = late;
+
+	set_bit(EFI_MEMMAP, &efi.flags);
+
+	efi.memmap = map;
+
+	return 0;
+}
+
+/**
+ * efi_memmap_init_early - Map the EFI memory map data structure
+ * @data: EFI memory map data
+ *
+ * Use early_memremap() to map the passed in EFI memory map and assign
+ * it to efi.memmap.
+ */
+int __init efi_memmap_init_early(struct efi_memory_map_data *data)
+{
+	/* Cannot go backwards */
+	WARN_ON(efi.memmap.late);
+
+	return __efi_memmap_init(data, false);
+}
+
+void __init efi_memmap_unmap(void)
+{
+	if (!efi.memmap.late) {
+		unsigned long size;
+
+		size = efi.memmap.desc_size * efi.memmap.nr_map;
+		early_memunmap(efi.memmap.map, size);
+	} else {
+		memunmap(efi.memmap.map);
+	}
+
+	efi.memmap.map = NULL;
+	clear_bit(EFI_MEMMAP, &efi.flags);
+}
+
+/**
+ * efi_memmap_init_late - Map efi.memmap with memremap()
+ * @phys_addr: Physical address of the new EFI memory map
+ * @size: Size in bytes of the new EFI memory map
+ *
+ * Setup a mapping of the EFI memory map using ioremap_cache(). This
+ * function should only be called once the vmalloc space has been
+ * setup and is therefore not suitable for calling during early EFI
+ * initialise, e.g. in efi_init(). Additionally, it expects
+ * efi_memmap_init_early() to have already been called.
+ *
+ * The reason there are two EFI memmap initialisation
+ * (efi_memmap_init_early() and this late version) is because the
+ * early EFI memmap should be explicitly unmapped once EFI
+ * initialisation is complete as the fixmap space used to map the EFI
+ * memmap (via early_memremap()) is a scarce resource.
+ *
+ * This late mapping is intended to persist for the duration of
+ * runtime so that things like efi_mem_desc_lookup() and
+ * efi_mem_attributes() always work.
+ *
+ * Returns zero on success, a negative error code on failure.
+ */
+int __init efi_memmap_init_late(phys_addr_t addr, unsigned long size)
+{
+	struct efi_memory_map_data data = {
+		.phys_map = addr,
+		.size = size,
+	};
+
+	/* Did we forget to unmap the early EFI memmap? */
+	WARN_ON(efi.memmap.map);
+
+	/* Were we already called? */
+	WARN_ON(efi.memmap.late);
+
+	/*
+	 * It makes no sense to allow callers to register different
+	 * values for the following fields. Copy them out of the
+	 * existing early EFI memmap.
+	 */
+	data.desc_version = efi.memmap.desc_version;
+	data.desc_size = efi.memmap.desc_size;
+
+	return __efi_memmap_init(&data, true);
+}
+
+/**
+ * efi_memmap_install - Install a new EFI memory map in efi.memmap
+ * @addr: Physical address of the memory map
+ * @nr_map: Number of entries in the memory map
+ *
+ * Unlike efi_memmap_init_*(), this function does not allow the caller
+ * to switch from early to late mappings. It simply uses the existing
+ * mapping function and installs the new memmap.
+ *
+ * Returns zero on success, a negative error code on failure.
+ */
+int __init efi_memmap_install(phys_addr_t addr, unsigned int nr_map)
+{
+	struct efi_memory_map_data data;
+
+	efi_memmap_unmap();
+
+	data.phys_map = addr;
+	data.size = efi.memmap.desc_size * nr_map;
+	data.desc_version = efi.memmap.desc_version;
+	data.desc_size = efi.memmap.desc_size;
+
+	return __efi_memmap_init(&data, efi.memmap.late);
+}
+
+/**
+ * efi_memmap_split_count - Count number of additional EFI memmap entries
+ * @md: EFI memory descriptor to split
+ * @range: Address range (start, end) to split around
+ *
+ * Returns the number of additional EFI memmap entries required to
+ * accomodate @range.
+ */
+int __init efi_memmap_split_count(efi_memory_desc_t *md, struct range *range)
+{
+	u64 m_start, m_end;
+	u64 start, end;
+	int count = 0;
+
+	start = md->phys_addr;
+	end = start + (md->num_pages << EFI_PAGE_SHIFT) - 1;
+
+	/* modifying range */
+	m_start = range->start;
+	m_end = range->end;
+
+	if (m_start <= start) {
+		/* split into 2 parts */
+		if (start < m_end && m_end < end)
+			count++;
+	}
+
+	if (start < m_start && m_start < end) {
+		/* split into 3 parts */
+		if (m_end < end)
+			count += 2;
+		/* split into 2 parts */
+		if (end <= m_end)
+			count++;
+	}
+
+	return count;
+}
+
+/**
+ * efi_memmap_insert - Insert a memory region in an EFI memmap
+ * @old_memmap: The existing EFI memory map structure
+ * @buf: Address of buffer to store new map
+ * @mem: Memory map entry to insert
+ *
+ * It is suggested that you call efi_memmap_split_count() first
+ * to see how large @buf needs to be.
+ */
+void __init efi_memmap_insert(struct efi_memory_map *old_memmap, void *buf,
+			      struct efi_mem_range *mem)
+{
+	u64 m_start, m_end, m_attr;
+	efi_memory_desc_t *md;
+	u64 start, end;
+	void *old, *new;
+
+	/* modifying range */
+	m_start = mem->range.start;
+	m_end = mem->range.end;
+	m_attr = mem->attribute;
+
+	/*
+	 * The EFI memory map deals with regions in EFI_PAGE_SIZE
+	 * units. Ensure that the region described by 'mem' is aligned
+	 * correctly.
+	 */
+	if (!IS_ALIGNED(m_start, EFI_PAGE_SIZE) ||
+	    !IS_ALIGNED(m_end + 1, EFI_PAGE_SIZE)) {
+		WARN_ON(1);
+		return;
+	}
+
+	for (old = old_memmap->map, new = buf;
+	     old < old_memmap->map_end;
+	     old += old_memmap->desc_size, new += old_memmap->desc_size) {
+
+		/* copy original EFI memory descriptor */
+		memcpy(new, old, old_memmap->desc_size);
+		md = new;
+		start = md->phys_addr;
+		end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1;
+
+		if (m_start <= start && end <= m_end)
+			md->attribute |= m_attr;
+
+		if (m_start <= start &&
+		    (start < m_end && m_end < end)) {
+			/* first part */
+			md->attribute |= m_attr;
+			md->num_pages = (m_end - md->phys_addr + 1) >>
+				EFI_PAGE_SHIFT;
+			/* latter part */
+			new += old_memmap->desc_size;
+			memcpy(new, old, old_memmap->desc_size);
+			md = new;
+			md->phys_addr = m_end + 1;
+			md->num_pages = (end - md->phys_addr + 1) >>
+				EFI_PAGE_SHIFT;
+		}
+
+		if ((start < m_start && m_start < end) && m_end < end) {
+			/* first part */
+			md->num_pages = (m_start - md->phys_addr) >>
+				EFI_PAGE_SHIFT;
+			/* middle part */
+			new += old_memmap->desc_size;
+			memcpy(new, old, old_memmap->desc_size);
+			md = new;
+			md->attribute |= m_attr;
+			md->phys_addr = m_start;
+			md->num_pages = (m_end - m_start + 1) >>
+				EFI_PAGE_SHIFT;
+			/* last part */
+			new += old_memmap->desc_size;
+			memcpy(new, old, old_memmap->desc_size);
+			md = new;
+			md->phys_addr = m_end + 1;
+			md->num_pages = (end - m_end) >>
+				EFI_PAGE_SHIFT;
+		}
+
+		if ((start < m_start && m_start < end) &&
+		    (end <= m_end)) {
+			/* first part */
+			md->num_pages = (m_start - md->phys_addr) >>
+				EFI_PAGE_SHIFT;
+			/* latter part */
+			new += old_memmap->desc_size;
+			memcpy(new, old, old_memmap->desc_size);
+			md = new;
+			md->phys_addr = m_start;
+			md->num_pages = (end - md->phys_addr + 1) >>
+				EFI_PAGE_SHIFT;
+			md->attribute |= m_attr;
+		}
+	}
+}
diff --git a/drivers/firmware/efi/runtime-map.c b/drivers/firmware/efi/runtime-map.c
index 5c55227a34c8..8e64b77aeac9 100644
--- a/drivers/firmware/efi/runtime-map.c
+++ b/drivers/firmware/efi/runtime-map.c
@@ -14,10 +14,6 @@
 
 #include <asm/setup.h>
 
-static void *efi_runtime_map;
-static int nr_efi_runtime_map;
-static u32 efi_memdesc_size;
-
 struct efi_runtime_map_entry {
 	efi_memory_desc_t md;
 	struct kobject kobj;   /* kobject for each entry */
@@ -106,7 +102,8 @@ static struct kobj_type __refdata map_ktype = {
 static struct kset *map_kset;
 
 static struct efi_runtime_map_entry *
-add_sysfs_runtime_map_entry(struct kobject *kobj, int nr)
+add_sysfs_runtime_map_entry(struct kobject *kobj, int nr,
+			    efi_memory_desc_t *md)
 {
 	int ret;
 	struct efi_runtime_map_entry *entry;
@@ -124,8 +121,7 @@ add_sysfs_runtime_map_entry(struct kobject *kobj, int nr)
 		return ERR_PTR(-ENOMEM);
 	}
 
-	memcpy(&entry->md, efi_runtime_map + nr * efi_memdesc_size,
-	       sizeof(efi_memory_desc_t));
+	memcpy(&entry->md, md, sizeof(efi_memory_desc_t));
 
 	kobject_init(&entry->kobj, &map_ktype);
 	entry->kobj.kset = map_kset;
@@ -142,12 +138,12 @@ add_sysfs_runtime_map_entry(struct kobject *kobj, int nr)
 
 int efi_get_runtime_map_size(void)
 {
-	return nr_efi_runtime_map * efi_memdesc_size;
+	return efi.memmap.nr_map * efi.memmap.desc_size;
 }
 
 int efi_get_runtime_map_desc_size(void)
 {
-	return efi_memdesc_size;
+	return efi.memmap.desc_size;
 }
 
 int efi_runtime_map_copy(void *buf, size_t bufsz)
@@ -157,38 +153,33 @@ int efi_runtime_map_copy(void *buf, size_t bufsz)
 	if (sz > bufsz)
 		sz = bufsz;
 
-	memcpy(buf, efi_runtime_map, sz);
+	memcpy(buf, efi.memmap.map, sz);
 	return 0;
 }
 
-void efi_runtime_map_setup(void *map, int nr_entries, u32 desc_size)
-{
-	efi_runtime_map = map;
-	nr_efi_runtime_map = nr_entries;
-	efi_memdesc_size = desc_size;
-}
-
 int __init efi_runtime_map_init(struct kobject *efi_kobj)
 {
 	int i, j, ret = 0;
 	struct efi_runtime_map_entry *entry;
+	efi_memory_desc_t *md;
 
-	if (!efi_runtime_map)
+	if (!efi_enabled(EFI_MEMMAP))
 		return 0;
 
-	map_entries = kzalloc(nr_efi_runtime_map * sizeof(entry), GFP_KERNEL);
+	map_entries = kzalloc(efi.memmap.nr_map * sizeof(entry), GFP_KERNEL);
 	if (!map_entries) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
-	for (i = 0; i < nr_efi_runtime_map; i++) {
-		entry = add_sysfs_runtime_map_entry(efi_kobj, i);
+	i = 0;
+	for_each_efi_memory_desc(md) {
+		entry = add_sysfs_runtime_map_entry(efi_kobj, i, md);
 		if (IS_ERR(entry)) {
 			ret = PTR_ERR(entry);
 			goto out_add_entry;
 		}
-		*(map_entries + i) = entry;
+		*(map_entries + i++) = entry;
 	}
 
 	return 0;
diff --git a/drivers/firmware/efi/runtime-wrappers.c b/drivers/firmware/efi/runtime-wrappers.c
index 41958774cde3..ae54870b2788 100644
--- a/drivers/firmware/efi/runtime-wrappers.c
+++ b/drivers/firmware/efi/runtime-wrappers.c
@@ -14,11 +14,13 @@
  * This file is released under the GPLv2.
  */
 
+#define pr_fmt(fmt)	"efi: " fmt
+
 #include <linux/bug.h>
 #include <linux/efi.h>
 #include <linux/irqflags.h>
 #include <linux/mutex.h>
-#include <linux/spinlock.h>
+#include <linux/semaphore.h>
 #include <linux/stringify.h>
 #include <asm/efi.h>
 
@@ -81,20 +83,21 @@ void efi_call_virt_check_flags(unsigned long flags, const char *call)
  * +------------------------------------+-------------------------------+
  *
  * Due to the fact that the EFI pstore may write to the variable store in
- * interrupt context, we need to use a spinlock for at least the groups that
+ * interrupt context, we need to use a lock for at least the groups that
  * contain SetVariable() and QueryVariableInfo(). That leaves little else, as
  * none of the remaining functions are actually ever called at runtime.
- * So let's just use a single spinlock to serialize all Runtime Services calls.
+ * So let's just use a single lock to serialize all Runtime Services calls.
  */
-static DEFINE_SPINLOCK(efi_runtime_lock);
+static DEFINE_SEMAPHORE(efi_runtime_lock);
 
 static efi_status_t virt_efi_get_time(efi_time_t *tm, efi_time_cap_t *tc)
 {
 	efi_status_t status;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(get_time, tm, tc);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -102,9 +105,10 @@ static efi_status_t virt_efi_set_time(efi_time_t *tm)
 {
 	efi_status_t status;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(set_time, tm);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -114,9 +118,10 @@ static efi_status_t virt_efi_get_wakeup_time(efi_bool_t *enabled,
 {
 	efi_status_t status;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(get_wakeup_time, enabled, pending, tm);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -124,9 +129,10 @@ static efi_status_t virt_efi_set_wakeup_time(efi_bool_t enabled, efi_time_t *tm)
 {
 	efi_status_t status;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(set_wakeup_time, enabled, tm);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -138,10 +144,11 @@ static efi_status_t virt_efi_get_variable(efi_char16_t *name,
 {
 	efi_status_t status;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(get_variable, name, vendor, attr, data_size,
 			       data);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -151,9 +158,10 @@ static efi_status_t virt_efi_get_next_variable(unsigned long *name_size,
 {
 	efi_status_t status;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(get_next_variable, name_size, name, vendor);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -165,10 +173,11 @@ static efi_status_t virt_efi_set_variable(efi_char16_t *name,
 {
 	efi_status_t status;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(set_variable, name, vendor, attr, data_size,
 			       data);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -179,12 +188,12 @@ virt_efi_set_variable_nonblocking(efi_char16_t *name, efi_guid_t *vendor,
 {
 	efi_status_t status;
 
-	if (!spin_trylock(&efi_runtime_lock))
+	if (down_trylock(&efi_runtime_lock))
 		return EFI_NOT_READY;
 
 	status = efi_call_virt(set_variable, name, vendor, attr, data_size,
 			       data);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -199,10 +208,11 @@ static efi_status_t virt_efi_query_variable_info(u32 attr,
 	if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION)
 		return EFI_UNSUPPORTED;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(query_variable_info, attr, storage_space,
 			       remaining_space, max_variable_size);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -217,12 +227,12 @@ virt_efi_query_variable_info_nonblocking(u32 attr,
 	if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION)
 		return EFI_UNSUPPORTED;
 
-	if (!spin_trylock(&efi_runtime_lock))
+	if (down_trylock(&efi_runtime_lock))
 		return EFI_NOT_READY;
 
 	status = efi_call_virt(query_variable_info, attr, storage_space,
 			       remaining_space, max_variable_size);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -230,9 +240,10 @@ static efi_status_t virt_efi_get_next_high_mono_count(u32 *count)
 {
 	efi_status_t status;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(get_next_high_mono_count, count);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -241,9 +252,13 @@ static void virt_efi_reset_system(int reset_type,
 				  unsigned long data_size,
 				  efi_char16_t *data)
 {
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock)) {
+		pr_warn("failed to invoke the reset_system() runtime service:\n"
+			"could not get exclusive access to the firmware\n");
+		return;
+	}
 	__efi_call_virt(reset_system, reset_type, status, data_size, data);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 }
 
 static efi_status_t virt_efi_update_capsule(efi_capsule_header_t **capsules,
@@ -255,9 +270,10 @@ static efi_status_t virt_efi_update_capsule(efi_capsule_header_t **capsules,
 	if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION)
 		return EFI_UNSUPPORTED;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(update_capsule, capsules, count, sg_list);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
@@ -271,10 +287,11 @@ static efi_status_t virt_efi_query_capsule_caps(efi_capsule_header_t **capsules,
 	if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION)
 		return EFI_UNSUPPORTED;
 
-	spin_lock(&efi_runtime_lock);
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
 	status = efi_call_virt(query_capsule_caps, capsules, count, max_size,
 			       reset_type);
-	spin_unlock(&efi_runtime_lock);
+	up(&efi_runtime_lock);
 	return status;
 }
 
diff --git a/drivers/firmware/efi/test/Makefile b/drivers/firmware/efi/test/Makefile
new file mode 100644
index 000000000000..bcd4577d40e6
--- /dev/null
+++ b/drivers/firmware/efi/test/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_EFI_TEST)			+= efi_test.o
diff --git a/drivers/firmware/efi/test/efi_test.c b/drivers/firmware/efi/test/efi_test.c
new file mode 100644
index 000000000000..f61bb52be318
--- /dev/null
+++ b/drivers/firmware/efi/test/efi_test.c
@@ -0,0 +1,749 @@
+/*
+ * EFI Test Driver for Runtime Services
+ *
+ * Copyright(C) 2012-2016 Canonical Ltd.
+ *
+ * This driver exports EFI runtime services interfaces into userspace, which
+ * allow to use and test UEFI runtime services provided by firmware.
+ *
+ */
+
+#include <linux/version.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/proc_fs.h>
+#include <linux/efi.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+
+#include "efi_test.h"
+
+MODULE_AUTHOR("Ivan Hu <ivan.hu@canonical.com>");
+MODULE_DESCRIPTION("EFI Test Driver");
+MODULE_LICENSE("GPL");
+
+/*
+ * Count the bytes in 'str', including the terminating NULL.
+ *
+ * Note this function returns the number of *bytes*, not the number of
+ * ucs2 characters.
+ */
+static inline size_t user_ucs2_strsize(efi_char16_t  __user *str)
+{
+	efi_char16_t *s = str, c;
+	size_t len;
+
+	if (!str)
+		return 0;
+
+	/* Include terminating NULL */
+	len = sizeof(efi_char16_t);
+
+	if (get_user(c, s++)) {
+		/* Can't read userspace memory for size */
+		return 0;
+	}
+
+	while (c != 0) {
+		if (get_user(c, s++)) {
+			/* Can't read userspace memory for size */
+			return 0;
+		}
+		len += sizeof(efi_char16_t);
+	}
+	return len;
+}
+
+/*
+ * Allocate a buffer and copy a ucs2 string from user space into it.
+ */
+static inline int
+copy_ucs2_from_user_len(efi_char16_t **dst, efi_char16_t __user *src,
+			size_t len)
+{
+	efi_char16_t *buf;
+
+	if (!src) {
+		*dst = NULL;
+		return 0;
+	}
+
+	if (!access_ok(VERIFY_READ, src, 1))
+		return -EFAULT;
+
+	buf = kmalloc(len, GFP_KERNEL);
+	if (!buf) {
+		*dst = NULL;
+		return -ENOMEM;
+	}
+	*dst = buf;
+
+	if (copy_from_user(*dst, src, len)) {
+		kfree(buf);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/*
+ * Count the bytes in 'str', including the terminating NULL.
+ *
+ * Just a wrap for user_ucs2_strsize
+ */
+static inline int
+get_ucs2_strsize_from_user(efi_char16_t __user *src, size_t *len)
+{
+	if (!access_ok(VERIFY_READ, src, 1))
+		return -EFAULT;
+
+	*len = user_ucs2_strsize(src);
+	if (*len == 0)
+		return -EFAULT;
+
+	return 0;
+}
+
+/*
+ * Calculate the required buffer allocation size and copy a ucs2 string
+ * from user space into it.
+ *
+ * This function differs from copy_ucs2_from_user_len() because it
+ * calculates the size of the buffer to allocate by taking the length of
+ * the string 'src'.
+ *
+ * If a non-zero value is returned, the caller MUST NOT access 'dst'.
+ *
+ * It is the caller's responsibility to free 'dst'.
+ */
+static inline int
+copy_ucs2_from_user(efi_char16_t **dst, efi_char16_t __user *src)
+{
+	size_t len;
+
+	if (!access_ok(VERIFY_READ, src, 1))
+		return -EFAULT;
+
+	len = user_ucs2_strsize(src);
+	if (len == 0)
+		return -EFAULT;
+	return copy_ucs2_from_user_len(dst, src, len);
+}
+
+/*
+ * Copy a ucs2 string to a user buffer.
+ *
+ * This function is a simple wrapper around copy_to_user() that does
+ * nothing if 'src' is NULL, which is useful for reducing the amount of
+ * NULL checking the caller has to do.
+ *
+ * 'len' specifies the number of bytes to copy.
+ */
+static inline int
+copy_ucs2_to_user_len(efi_char16_t __user *dst, efi_char16_t *src, size_t len)
+{
+	if (!src)
+		return 0;
+
+	if (!access_ok(VERIFY_WRITE, dst, 1))
+		return -EFAULT;
+
+	return copy_to_user(dst, src, len);
+}
+
+static long efi_runtime_get_variable(unsigned long arg)
+{
+	struct efi_getvariable __user *getvariable_user;
+	struct efi_getvariable getvariable;
+	unsigned long datasize, prev_datasize, *dz;
+	efi_guid_t vendor_guid, *vd = NULL;
+	efi_status_t status;
+	efi_char16_t *name = NULL;
+	u32 attr, *at;
+	void *data = NULL;
+	int rv = 0;
+
+	getvariable_user = (struct efi_getvariable __user *)arg;
+
+	if (copy_from_user(&getvariable, getvariable_user,
+			   sizeof(getvariable)))
+		return -EFAULT;
+	if (getvariable.data_size &&
+	    get_user(datasize, getvariable.data_size))
+		return -EFAULT;
+	if (getvariable.vendor_guid) {
+		if (copy_from_user(&vendor_guid, getvariable.vendor_guid,
+					sizeof(vendor_guid)))
+			return -EFAULT;
+		vd = &vendor_guid;
+	}
+
+	if (getvariable.variable_name) {
+		rv = copy_ucs2_from_user(&name, getvariable.variable_name);
+		if (rv)
+			return rv;
+	}
+
+	at = getvariable.attributes ? &attr : NULL;
+	dz = getvariable.data_size ? &datasize : NULL;
+
+	if (getvariable.data_size && getvariable.data) {
+		data = kmalloc(datasize, GFP_KERNEL);
+		if (!data) {
+			kfree(name);
+			return -ENOMEM;
+		}
+	}
+
+	prev_datasize = datasize;
+	status = efi.get_variable(name, vd, at, dz, data);
+	kfree(name);
+
+	if (put_user(status, getvariable.status)) {
+		rv = -EFAULT;
+		goto out;
+	}
+
+	if (status != EFI_SUCCESS) {
+		if (status == EFI_BUFFER_TOO_SMALL) {
+			if (dz && put_user(datasize, getvariable.data_size)) {
+				rv = -EFAULT;
+				goto out;
+			}
+		}
+		rv = -EINVAL;
+		goto out;
+	}
+
+	if (prev_datasize < datasize) {
+		rv = -EINVAL;
+		goto out;
+	}
+
+	if (data) {
+		if (copy_to_user(getvariable.data, data, datasize)) {
+			rv = -EFAULT;
+			goto out;
+		}
+	}
+
+	if (at && put_user(attr, getvariable.attributes)) {
+		rv = -EFAULT;
+		goto out;
+	}
+
+	if (dz && put_user(datasize, getvariable.data_size))
+		rv = -EFAULT;
+
+out:
+	kfree(data);
+	return rv;
+
+}
+
+static long efi_runtime_set_variable(unsigned long arg)
+{
+	struct efi_setvariable __user *setvariable_user;
+	struct efi_setvariable setvariable;
+	efi_guid_t vendor_guid;
+	efi_status_t status;
+	efi_char16_t *name = NULL;
+	void *data;
+	int rv = 0;
+
+	setvariable_user = (struct efi_setvariable __user *)arg;
+
+	if (copy_from_user(&setvariable, setvariable_user, sizeof(setvariable)))
+		return -EFAULT;
+	if (copy_from_user(&vendor_guid, setvariable.vendor_guid,
+				sizeof(vendor_guid)))
+		return -EFAULT;
+
+	if (setvariable.variable_name) {
+		rv = copy_ucs2_from_user(&name, setvariable.variable_name);
+		if (rv)
+			return rv;
+	}
+
+	data = kmalloc(setvariable.data_size, GFP_KERNEL);
+	if (!data) {
+		kfree(name);
+		return -ENOMEM;
+	}
+	if (copy_from_user(data, setvariable.data, setvariable.data_size)) {
+		rv = -EFAULT;
+		goto out;
+	}
+
+	status = efi.set_variable(name, &vendor_guid,
+				setvariable.attributes,
+				setvariable.data_size, data);
+
+	if (put_user(status, setvariable.status)) {
+		rv = -EFAULT;
+		goto out;
+	}
+
+	rv = status == EFI_SUCCESS ? 0 : -EINVAL;
+
+out:
+	kfree(data);
+	kfree(name);
+
+	return rv;
+}
+
+static long efi_runtime_get_time(unsigned long arg)
+{
+	struct efi_gettime __user *gettime_user;
+	struct efi_gettime  gettime;
+	efi_status_t status;
+	efi_time_cap_t cap;
+	efi_time_t efi_time;
+
+	gettime_user = (struct efi_gettime __user *)arg;
+	if (copy_from_user(&gettime, gettime_user, sizeof(gettime)))
+		return -EFAULT;
+
+	status = efi.get_time(gettime.time ? &efi_time : NULL,
+			      gettime.capabilities ? &cap : NULL);
+
+	if (put_user(status, gettime.status))
+		return -EFAULT;
+
+	if (status != EFI_SUCCESS)
+		return -EINVAL;
+
+	if (gettime.capabilities) {
+		efi_time_cap_t __user *cap_local;
+
+		cap_local = (efi_time_cap_t *)gettime.capabilities;
+		if (put_user(cap.resolution, &(cap_local->resolution)) ||
+			put_user(cap.accuracy, &(cap_local->accuracy)) ||
+			put_user(cap.sets_to_zero, &(cap_local->sets_to_zero)))
+			return -EFAULT;
+	}
+	if (gettime.time) {
+		if (copy_to_user(gettime.time, &efi_time, sizeof(efi_time_t)))
+			return -EFAULT;
+	}
+
+	return 0;
+}
+
+static long efi_runtime_set_time(unsigned long arg)
+{
+	struct efi_settime __user *settime_user;
+	struct efi_settime settime;
+	efi_status_t status;
+	efi_time_t efi_time;
+
+	settime_user = (struct efi_settime __user *)arg;
+	if (copy_from_user(&settime, settime_user, sizeof(settime)))
+		return -EFAULT;
+	if (copy_from_user(&efi_time, settime.time,
+					sizeof(efi_time_t)))
+		return -EFAULT;
+	status = efi.set_time(&efi_time);
+
+	if (put_user(status, settime.status))
+		return -EFAULT;
+
+	return status == EFI_SUCCESS ? 0 : -EINVAL;
+}
+
+static long efi_runtime_get_waketime(unsigned long arg)
+{
+	struct efi_getwakeuptime __user *getwakeuptime_user;
+	struct efi_getwakeuptime getwakeuptime;
+	efi_bool_t enabled, pending;
+	efi_status_t status;
+	efi_time_t efi_time;
+
+	getwakeuptime_user = (struct efi_getwakeuptime __user *)arg;
+	if (copy_from_user(&getwakeuptime, getwakeuptime_user,
+				sizeof(getwakeuptime)))
+		return -EFAULT;
+
+	status = efi.get_wakeup_time(
+		getwakeuptime.enabled ? (efi_bool_t *)&enabled : NULL,
+		getwakeuptime.pending ? (efi_bool_t *)&pending : NULL,
+		getwakeuptime.time ? &efi_time : NULL);
+
+	if (put_user(status, getwakeuptime.status))
+		return -EFAULT;
+
+	if (status != EFI_SUCCESS)
+		return -EINVAL;
+
+	if (getwakeuptime.enabled && put_user(enabled,
+						getwakeuptime.enabled))
+		return -EFAULT;
+
+	if (getwakeuptime.time) {
+		if (copy_to_user(getwakeuptime.time, &efi_time,
+				sizeof(efi_time_t)))
+			return -EFAULT;
+	}
+
+	return 0;
+}
+
+static long efi_runtime_set_waketime(unsigned long arg)
+{
+	struct efi_setwakeuptime __user *setwakeuptime_user;
+	struct efi_setwakeuptime setwakeuptime;
+	efi_bool_t enabled;
+	efi_status_t status;
+	efi_time_t efi_time;
+
+	setwakeuptime_user = (struct efi_setwakeuptime __user *)arg;
+
+	if (copy_from_user(&setwakeuptime, setwakeuptime_user,
+				sizeof(setwakeuptime)))
+		return -EFAULT;
+
+	enabled = setwakeuptime.enabled;
+	if (setwakeuptime.time) {
+		if (copy_from_user(&efi_time, setwakeuptime.time,
+					sizeof(efi_time_t)))
+			return -EFAULT;
+
+		status = efi.set_wakeup_time(enabled, &efi_time);
+	} else
+		status = efi.set_wakeup_time(enabled, NULL);
+
+	if (put_user(status, setwakeuptime.status))
+		return -EFAULT;
+
+	return status == EFI_SUCCESS ? 0 : -EINVAL;
+}
+
+static long efi_runtime_get_nextvariablename(unsigned long arg)
+{
+	struct efi_getnextvariablename __user *getnextvariablename_user;
+	struct efi_getnextvariablename getnextvariablename;
+	unsigned long name_size, prev_name_size = 0, *ns = NULL;
+	efi_status_t status;
+	efi_guid_t *vd = NULL;
+	efi_guid_t vendor_guid;
+	efi_char16_t *name = NULL;
+	int rv;
+
+	getnextvariablename_user = (struct efi_getnextvariablename __user *)arg;
+
+	if (copy_from_user(&getnextvariablename, getnextvariablename_user,
+			   sizeof(getnextvariablename)))
+		return -EFAULT;
+
+	if (getnextvariablename.variable_name_size) {
+		if (get_user(name_size, getnextvariablename.variable_name_size))
+			return -EFAULT;
+		ns = &name_size;
+		prev_name_size = name_size;
+	}
+
+	if (getnextvariablename.vendor_guid) {
+		if (copy_from_user(&vendor_guid,
+				getnextvariablename.vendor_guid,
+				sizeof(vendor_guid)))
+			return -EFAULT;
+		vd = &vendor_guid;
+	}
+
+	if (getnextvariablename.variable_name) {
+		size_t name_string_size = 0;
+
+		rv = get_ucs2_strsize_from_user(
+				getnextvariablename.variable_name,
+				&name_string_size);
+		if (rv)
+			return rv;
+		/*
+		 * The name_size may be smaller than the real buffer size where
+		 * variable name located in some use cases. The most typical
+		 * case is passing a 0 to get the required buffer size for the
+		 * 1st time call. So we need to copy the content from user
+		 * space for at least the string size of variable name, or else
+		 * the name passed to UEFI may not be terminated as we expected.
+		 */
+		rv = copy_ucs2_from_user_len(&name,
+				getnextvariablename.variable_name,
+				prev_name_size > name_string_size ?
+				prev_name_size : name_string_size);
+		if (rv)
+			return rv;
+	}
+
+	status = efi.get_next_variable(ns, name, vd);
+
+	if (put_user(status, getnextvariablename.status)) {
+		rv = -EFAULT;
+		goto out;
+	}
+
+	if (status != EFI_SUCCESS) {
+		if (status == EFI_BUFFER_TOO_SMALL) {
+			if (ns && put_user(*ns,
+				getnextvariablename.variable_name_size)) {
+				rv = -EFAULT;
+				goto out;
+			}
+		}
+		rv = -EINVAL;
+		goto out;
+	}
+
+	if (name) {
+		if (copy_ucs2_to_user_len(getnextvariablename.variable_name,
+						name, prev_name_size)) {
+			rv = -EFAULT;
+			goto out;
+		}
+	}
+
+	if (ns) {
+		if (put_user(*ns, getnextvariablename.variable_name_size)) {
+			rv = -EFAULT;
+			goto out;
+		}
+	}
+
+	if (vd) {
+		if (copy_to_user(getnextvariablename.vendor_guid, vd,
+							sizeof(efi_guid_t)))
+			rv = -EFAULT;
+	}
+
+out:
+	kfree(name);
+	return rv;
+}
+
+static long efi_runtime_get_nexthighmonocount(unsigned long arg)
+{
+	struct efi_getnexthighmonotoniccount __user *getnexthighmonocount_user;
+	struct efi_getnexthighmonotoniccount getnexthighmonocount;
+	efi_status_t status;
+	u32 count;
+
+	getnexthighmonocount_user = (struct
+			efi_getnexthighmonotoniccount __user *)arg;
+
+	if (copy_from_user(&getnexthighmonocount,
+			   getnexthighmonocount_user,
+			   sizeof(getnexthighmonocount)))
+		return -EFAULT;
+
+	status = efi.get_next_high_mono_count(
+		getnexthighmonocount.high_count ? &count : NULL);
+
+	if (put_user(status, getnexthighmonocount.status))
+		return -EFAULT;
+
+	if (status != EFI_SUCCESS)
+		return -EINVAL;
+
+	if (getnexthighmonocount.high_count &&
+	    put_user(count, getnexthighmonocount.high_count))
+		return -EFAULT;
+
+	return 0;
+}
+
+static long efi_runtime_query_variableinfo(unsigned long arg)
+{
+	struct efi_queryvariableinfo __user *queryvariableinfo_user;
+	struct efi_queryvariableinfo queryvariableinfo;
+	efi_status_t status;
+	u64 max_storage, remaining, max_size;
+
+	queryvariableinfo_user = (struct efi_queryvariableinfo __user *)arg;
+
+	if (copy_from_user(&queryvariableinfo, queryvariableinfo_user,
+			   sizeof(queryvariableinfo)))
+		return -EFAULT;
+
+	status = efi.query_variable_info(queryvariableinfo.attributes,
+					 &max_storage, &remaining, &max_size);
+
+	if (put_user(status, queryvariableinfo.status))
+		return -EFAULT;
+
+	if (status != EFI_SUCCESS)
+		return -EINVAL;
+
+	if (put_user(max_storage,
+		     queryvariableinfo.maximum_variable_storage_size))
+		return -EFAULT;
+
+	if (put_user(remaining,
+		     queryvariableinfo.remaining_variable_storage_size))
+		return -EFAULT;
+
+	if (put_user(max_size, queryvariableinfo.maximum_variable_size))
+		return -EFAULT;
+
+	return 0;
+}
+
+static long efi_runtime_query_capsulecaps(unsigned long arg)
+{
+	struct efi_querycapsulecapabilities __user *qcaps_user;
+	struct efi_querycapsulecapabilities qcaps;
+	efi_capsule_header_t *capsules;
+	efi_status_t status;
+	u64 max_size;
+	int i, reset_type;
+	int rv = 0;
+
+	qcaps_user = (struct efi_querycapsulecapabilities __user *)arg;
+
+	if (copy_from_user(&qcaps, qcaps_user, sizeof(qcaps)))
+		return -EFAULT;
+
+	capsules = kcalloc(qcaps.capsule_count + 1,
+			   sizeof(efi_capsule_header_t), GFP_KERNEL);
+	if (!capsules)
+		return -ENOMEM;
+
+	for (i = 0; i < qcaps.capsule_count; i++) {
+		efi_capsule_header_t *c;
+		/*
+		 * We cannot dereference qcaps.capsule_header_array directly to
+		 * obtain the address of the capsule as it resides in the
+		 * user space
+		 */
+		if (get_user(c, qcaps.capsule_header_array + i)) {
+			rv = -EFAULT;
+			goto out;
+		}
+		if (copy_from_user(&capsules[i], c,
+				sizeof(efi_capsule_header_t))) {
+			rv = -EFAULT;
+			goto out;
+		}
+	}
+
+	qcaps.capsule_header_array = &capsules;
+
+	status = efi.query_capsule_caps((efi_capsule_header_t **)
+					qcaps.capsule_header_array,
+					qcaps.capsule_count,
+					&max_size, &reset_type);
+
+	if (put_user(status, qcaps.status)) {
+		rv = -EFAULT;
+		goto out;
+	}
+
+	if (status != EFI_SUCCESS) {
+		rv = -EINVAL;
+		goto out;
+	}
+
+	if (put_user(max_size, qcaps.maximum_capsule_size)) {
+		rv = -EFAULT;
+		goto out;
+	}
+
+	if (put_user(reset_type, qcaps.reset_type))
+		rv = -EFAULT;
+
+out:
+	kfree(capsules);
+	return rv;
+}
+
+static long efi_test_ioctl(struct file *file, unsigned int cmd,
+							unsigned long arg)
+{
+	switch (cmd) {
+	case EFI_RUNTIME_GET_VARIABLE:
+		return efi_runtime_get_variable(arg);
+
+	case EFI_RUNTIME_SET_VARIABLE:
+		return efi_runtime_set_variable(arg);
+
+	case EFI_RUNTIME_GET_TIME:
+		return efi_runtime_get_time(arg);
+
+	case EFI_RUNTIME_SET_TIME:
+		return efi_runtime_set_time(arg);
+
+	case EFI_RUNTIME_GET_WAKETIME:
+		return efi_runtime_get_waketime(arg);
+
+	case EFI_RUNTIME_SET_WAKETIME:
+		return efi_runtime_set_waketime(arg);
+
+	case EFI_RUNTIME_GET_NEXTVARIABLENAME:
+		return efi_runtime_get_nextvariablename(arg);
+
+	case EFI_RUNTIME_GET_NEXTHIGHMONOTONICCOUNT:
+		return efi_runtime_get_nexthighmonocount(arg);
+
+	case EFI_RUNTIME_QUERY_VARIABLEINFO:
+		return efi_runtime_query_variableinfo(arg);
+
+	case EFI_RUNTIME_QUERY_CAPSULECAPABILITIES:
+		return efi_runtime_query_capsulecaps(arg);
+	}
+
+	return -ENOTTY;
+}
+
+static int efi_test_open(struct inode *inode, struct file *file)
+{
+	/*
+	 * nothing special to do here
+	 * We do accept multiple open files at the same time as we
+	 * synchronize on the per call operation.
+	 */
+	return 0;
+}
+
+static int efi_test_close(struct inode *inode, struct file *file)
+{
+	return 0;
+}
+
+/*
+ *	The various file operations we support.
+ */
+static const struct file_operations efi_test_fops = {
+	.owner		= THIS_MODULE,
+	.unlocked_ioctl	= efi_test_ioctl,
+	.open		= efi_test_open,
+	.release	= efi_test_close,
+	.llseek		= no_llseek,
+};
+
+static struct miscdevice efi_test_dev = {
+	MISC_DYNAMIC_MINOR,
+	"efi_test",
+	&efi_test_fops
+};
+
+static int __init efi_test_init(void)
+{
+	int ret;
+
+	ret = misc_register(&efi_test_dev);
+	if (ret) {
+		pr_err("efi_test: can't misc_register on minor=%d\n",
+			MISC_DYNAMIC_MINOR);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void __exit efi_test_exit(void)
+{
+	misc_deregister(&efi_test_dev);
+}
+
+module_init(efi_test_init);
+module_exit(efi_test_exit);
diff --git a/drivers/firmware/efi/test/efi_test.h b/drivers/firmware/efi/test/efi_test.h
new file mode 100644
index 000000000000..a33a6c633852
--- /dev/null
+++ b/drivers/firmware/efi/test/efi_test.h
@@ -0,0 +1,110 @@
+/*
+ * EFI Test driver Header
+ *
+ * Copyright(C) 2012-2016 Canonical Ltd.
+ *
+ */
+
+#ifndef _DRIVERS_FIRMWARE_EFI_TEST_H_
+#define _DRIVERS_FIRMWARE_EFI_TEST_H_
+
+#include <linux/efi.h>
+
+struct efi_getvariable {
+	efi_char16_t	*variable_name;
+	efi_guid_t	*vendor_guid;
+	u32		*attributes;
+	unsigned long	*data_size;
+	void		*data;
+	efi_status_t	*status;
+} __packed;
+
+struct efi_setvariable {
+	efi_char16_t	*variable_name;
+	efi_guid_t	*vendor_guid;
+	u32		attributes;
+	unsigned long	data_size;
+	void		*data;
+	efi_status_t	*status;
+} __packed;
+
+struct efi_getnextvariablename {
+	unsigned long	*variable_name_size;
+	efi_char16_t	*variable_name;
+	efi_guid_t	*vendor_guid;
+	efi_status_t	*status;
+} __packed;
+
+struct efi_queryvariableinfo {
+	u32		attributes;
+	u64		*maximum_variable_storage_size;
+	u64		*remaining_variable_storage_size;
+	u64		*maximum_variable_size;
+	efi_status_t	*status;
+} __packed;
+
+struct efi_gettime {
+	efi_time_t	*time;
+	efi_time_cap_t	*capabilities;
+	efi_status_t	*status;
+} __packed;
+
+struct efi_settime {
+	efi_time_t	*time;
+	efi_status_t	*status;
+} __packed;
+
+struct efi_getwakeuptime {
+	efi_bool_t	*enabled;
+	efi_bool_t	*pending;
+	efi_time_t	*time;
+	efi_status_t	*status;
+} __packed;
+
+struct efi_setwakeuptime {
+	efi_bool_t	enabled;
+	efi_time_t	*time;
+	efi_status_t	*status;
+} __packed;
+
+struct efi_getnexthighmonotoniccount {
+	u32		*high_count;
+	efi_status_t	*status;
+} __packed;
+
+struct efi_querycapsulecapabilities {
+	efi_capsule_header_t	**capsule_header_array;
+	unsigned long		capsule_count;
+	u64			*maximum_capsule_size;
+	int			*reset_type;
+	efi_status_t		*status;
+} __packed;
+
+#define EFI_RUNTIME_GET_VARIABLE \
+	_IOWR('p', 0x01, struct efi_getvariable)
+#define EFI_RUNTIME_SET_VARIABLE \
+	_IOW('p', 0x02, struct efi_setvariable)
+
+#define EFI_RUNTIME_GET_TIME \
+	_IOR('p', 0x03, struct efi_gettime)
+#define EFI_RUNTIME_SET_TIME \
+	_IOW('p', 0x04, struct efi_settime)
+
+#define EFI_RUNTIME_GET_WAKETIME \
+	_IOR('p', 0x05, struct efi_getwakeuptime)
+#define EFI_RUNTIME_SET_WAKETIME \
+	_IOW('p', 0x06, struct efi_setwakeuptime)
+
+#define EFI_RUNTIME_GET_NEXTVARIABLENAME \
+	_IOWR('p', 0x07, struct efi_getnextvariablename)
+
+#define EFI_RUNTIME_QUERY_VARIABLEINFO \
+	_IOR('p', 0x08, struct efi_queryvariableinfo)
+
+#define EFI_RUNTIME_GET_NEXTHIGHMONOTONICCOUNT \
+	_IOR('p', 0x09, struct efi_getnexthighmonotoniccount)
+
+#define EFI_RUNTIME_QUERY_CAPSULECAPABILITIES \
+	_IOR('p', 0x0A, struct efi_querycapsulecapabilities)
+
+#endif /* _DRIVERS_FIRMWARE_EFI_TEST_H_ */
diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c
index d3b751383286..9336ffdf6e2c 100644
--- a/drivers/firmware/efi/vars.c
+++ b/drivers/firmware/efi/vars.c
@@ -37,6 +37,14 @@
 /* Private pointer to registered efivars */
 static struct efivars *__efivars;
 
+/*
+ * efivars_lock protects three things:
+ * 1) efivarfs_list and efivars_sysfs_list
+ * 2) ->ops calls
+ * 3) (un)registration of __efivars
+ */
+static DEFINE_SEMAPHORE(efivars_lock);
+
 static bool efivar_wq_enabled = true;
 DECLARE_WORK(efivar_work, NULL);
 EXPORT_SYMBOL_GPL(efivar_work);
@@ -434,7 +442,10 @@ int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *),
 		return -ENOMEM;
 	}
 
-	spin_lock_irq(&__efivars->lock);
+	if (down_interruptible(&efivars_lock)) {
+		err = -EINTR;
+		goto free;
+	}
 
 	/*
 	 * Per EFI spec, the maximum storage allocated for both
@@ -450,7 +461,7 @@ int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *),
 		switch (status) {
 		case EFI_SUCCESS:
 			if (duplicates)
-				spin_unlock_irq(&__efivars->lock);
+				up(&efivars_lock);
 
 			variable_name_size = var_name_strnsize(variable_name,
 							       variable_name_size);
@@ -476,8 +487,12 @@ int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *),
 					status = EFI_NOT_FOUND;
 			}
 
-			if (duplicates)
-				spin_lock_irq(&__efivars->lock);
+			if (duplicates) {
+				if (down_interruptible(&efivars_lock)) {
+					err = -EINTR;
+					goto free;
+				}
+			}
 
 			break;
 		case EFI_NOT_FOUND:
@@ -491,8 +506,8 @@ int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *),
 
 	} while (status != EFI_NOT_FOUND);
 
-	spin_unlock_irq(&__efivars->lock);
-
+	up(&efivars_lock);
+free:
 	kfree(variable_name);
 
 	return err;
@@ -503,24 +518,34 @@ EXPORT_SYMBOL_GPL(efivar_init);
  * efivar_entry_add - add entry to variable list
  * @entry: entry to add to list
  * @head: list head
+ *
+ * Returns 0 on success, or a kernel error code on failure.
  */
-void efivar_entry_add(struct efivar_entry *entry, struct list_head *head)
+int efivar_entry_add(struct efivar_entry *entry, struct list_head *head)
 {
-	spin_lock_irq(&__efivars->lock);
+	if (down_interruptible(&efivars_lock))
+		return -EINTR;
 	list_add(&entry->list, head);
-	spin_unlock_irq(&__efivars->lock);
+	up(&efivars_lock);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(efivar_entry_add);
 
 /**
  * efivar_entry_remove - remove entry from variable list
  * @entry: entry to remove from list
+ *
+ * Returns 0 on success, or a kernel error code on failure.
  */
-void efivar_entry_remove(struct efivar_entry *entry)
+int efivar_entry_remove(struct efivar_entry *entry)
 {
-	spin_lock_irq(&__efivars->lock);
+	if (down_interruptible(&efivars_lock))
+		return -EINTR;
 	list_del(&entry->list);
-	spin_unlock_irq(&__efivars->lock);
+	up(&efivars_lock);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(efivar_entry_remove);
 
@@ -537,10 +562,8 @@ EXPORT_SYMBOL_GPL(efivar_entry_remove);
  */
 static void efivar_entry_list_del_unlock(struct efivar_entry *entry)
 {
-	lockdep_assert_held(&__efivars->lock);
-
 	list_del(&entry->list);
-	spin_unlock_irq(&__efivars->lock);
+	up(&efivars_lock);
 }
 
 /**
@@ -563,8 +586,6 @@ int __efivar_entry_delete(struct efivar_entry *entry)
 	const struct efivar_operations *ops = __efivars->ops;
 	efi_status_t status;
 
-	lockdep_assert_held(&__efivars->lock);
-
 	status = ops->set_variable(entry->var.VariableName,
 				   &entry->var.VendorGuid,
 				   0, 0, NULL);
@@ -581,20 +602,22 @@ EXPORT_SYMBOL_GPL(__efivar_entry_delete);
  * variable list. It is the caller's responsibility to free @entry
  * once we return.
  *
- * Returns 0 on success, or a converted EFI status code if
- * set_variable() fails.
+ * Returns 0 on success, -EINTR if we can't grab the semaphore,
+ * converted EFI status code if set_variable() fails.
  */
 int efivar_entry_delete(struct efivar_entry *entry)
 {
 	const struct efivar_operations *ops = __efivars->ops;
 	efi_status_t status;
 
-	spin_lock_irq(&__efivars->lock);
+	if (down_interruptible(&efivars_lock))
+		return -EINTR;
+
 	status = ops->set_variable(entry->var.VariableName,
 				   &entry->var.VendorGuid,
 				   0, 0, NULL);
 	if (!(status == EFI_SUCCESS || status == EFI_NOT_FOUND)) {
-		spin_unlock_irq(&__efivars->lock);
+		up(&efivars_lock);
 		return efi_status_to_err(status);
 	}
 
@@ -620,9 +643,9 @@ EXPORT_SYMBOL_GPL(efivar_entry_delete);
  * If @head is not NULL a lookup is performed to determine whether
  * the entry is already on the list.
  *
- * Returns 0 on success, -EEXIST if a lookup is performed and the entry
- * already exists on the list, or a converted EFI status code if
- * set_variable() fails.
+ * Returns 0 on success, -EINTR if we can't grab the semaphore,
+ * -EEXIST if a lookup is performed and the entry already exists on
+ * the list, or a converted EFI status code if set_variable() fails.
  */
 int efivar_entry_set(struct efivar_entry *entry, u32 attributes,
 		     unsigned long size, void *data, struct list_head *head)
@@ -632,10 +655,10 @@ int efivar_entry_set(struct efivar_entry *entry, u32 attributes,
 	efi_char16_t *name = entry->var.VariableName;
 	efi_guid_t vendor = entry->var.VendorGuid;
 
-	spin_lock_irq(&__efivars->lock);
-
+	if (down_interruptible(&efivars_lock))
+		return -EINTR;
 	if (head && efivar_entry_find(name, vendor, head, false)) {
-		spin_unlock_irq(&__efivars->lock);
+		up(&efivars_lock);
 		return -EEXIST;
 	}
 
@@ -644,7 +667,7 @@ int efivar_entry_set(struct efivar_entry *entry, u32 attributes,
 		status = ops->set_variable(name, &vendor,
 					   attributes, size, data);
 
-	spin_unlock_irq(&__efivars->lock);
+	up(&efivars_lock);
 
 	return efi_status_to_err(status);
 
@@ -658,30 +681,29 @@ EXPORT_SYMBOL_GPL(efivar_entry_set);
  * from crash/panic handlers.
  *
  * Crucially, this function will not block if it cannot acquire
- * __efivars->lock. Instead, it returns -EBUSY.
+ * efivars_lock. Instead, it returns -EBUSY.
  */
 static int
 efivar_entry_set_nonblocking(efi_char16_t *name, efi_guid_t vendor,
 			     u32 attributes, unsigned long size, void *data)
 {
 	const struct efivar_operations *ops = __efivars->ops;
-	unsigned long flags;
 	efi_status_t status;
 
-	if (!spin_trylock_irqsave(&__efivars->lock, flags))
+	if (down_trylock(&efivars_lock))
 		return -EBUSY;
 
 	status = check_var_size_nonblocking(attributes,
 					    size + ucs2_strsize(name, 1024));
 	if (status != EFI_SUCCESS) {
-		spin_unlock_irqrestore(&__efivars->lock, flags);
+		up(&efivars_lock);
 		return -ENOSPC;
 	}
 
 	status = ops->set_variable_nonblocking(name, &vendor, attributes,
 					       size, data);
 
-	spin_unlock_irqrestore(&__efivars->lock, flags);
+	up(&efivars_lock);
 	return efi_status_to_err(status);
 }
 
@@ -706,7 +728,6 @@ int efivar_entry_set_safe(efi_char16_t *name, efi_guid_t vendor, u32 attributes,
 			  bool block, unsigned long size, void *data)
 {
 	const struct efivar_operations *ops = __efivars->ops;
-	unsigned long flags;
 	efi_status_t status;
 
 	if (!ops->query_variable_store)
@@ -727,21 +748,22 @@ int efivar_entry_set_safe(efi_char16_t *name, efi_guid_t vendor, u32 attributes,
 						    size, data);
 
 	if (!block) {
-		if (!spin_trylock_irqsave(&__efivars->lock, flags))
+		if (down_trylock(&efivars_lock))
 			return -EBUSY;
 	} else {
-		spin_lock_irqsave(&__efivars->lock, flags);
+		if (down_interruptible(&efivars_lock))
+			return -EINTR;
 	}
 
 	status = check_var_size(attributes, size + ucs2_strsize(name, 1024));
 	if (status != EFI_SUCCESS) {
-		spin_unlock_irqrestore(&__efivars->lock, flags);
+		up(&efivars_lock);
 		return -ENOSPC;
 	}
 
 	status = ops->set_variable(name, &vendor, attributes, size, data);
 
-	spin_unlock_irqrestore(&__efivars->lock, flags);
+	up(&efivars_lock);
 
 	return efi_status_to_err(status);
 }
@@ -771,8 +793,6 @@ struct efivar_entry *efivar_entry_find(efi_char16_t *name, efi_guid_t guid,
 	int strsize1, strsize2;
 	bool found = false;
 
-	lockdep_assert_held(&__efivars->lock);
-
 	list_for_each_entry_safe(entry, n, head, list) {
 		strsize1 = ucs2_strsize(name, 1024);
 		strsize2 = ucs2_strsize(entry->var.VariableName, 1024);
@@ -814,10 +834,11 @@ int efivar_entry_size(struct efivar_entry *entry, unsigned long *size)
 
 	*size = 0;
 
-	spin_lock_irq(&__efivars->lock);
+	if (down_interruptible(&efivars_lock))
+		return -EINTR;
 	status = ops->get_variable(entry->var.VariableName,
 				   &entry->var.VendorGuid, NULL, size, NULL);
-	spin_unlock_irq(&__efivars->lock);
+	up(&efivars_lock);
 
 	if (status != EFI_BUFFER_TOO_SMALL)
 		return efi_status_to_err(status);
@@ -843,8 +864,6 @@ int __efivar_entry_get(struct efivar_entry *entry, u32 *attributes,
 	const struct efivar_operations *ops = __efivars->ops;
 	efi_status_t status;
 
-	lockdep_assert_held(&__efivars->lock);
-
 	status = ops->get_variable(entry->var.VariableName,
 				   &entry->var.VendorGuid,
 				   attributes, size, data);
@@ -866,11 +885,12 @@ int efivar_entry_get(struct efivar_entry *entry, u32 *attributes,
 	const struct efivar_operations *ops = __efivars->ops;
 	efi_status_t status;
 
-	spin_lock_irq(&__efivars->lock);
+	if (down_interruptible(&efivars_lock))
+		return -EINTR;
 	status = ops->get_variable(entry->var.VariableName,
 				   &entry->var.VendorGuid,
 				   attributes, size, data);
-	spin_unlock_irq(&__efivars->lock);
+	up(&efivars_lock);
 
 	return efi_status_to_err(status);
 }
@@ -917,7 +937,8 @@ int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes,
 	 * set_variable call, and removal of the variable from the efivars
 	 * list (in the case of an authenticated delete).
 	 */
-	spin_lock_irq(&__efivars->lock);
+	if (down_interruptible(&efivars_lock))
+		return -EINTR;
 
 	/*
 	 * Ensure that the available space hasn't shrunk below the safe level
@@ -957,7 +978,7 @@ int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes,
 	if (status == EFI_NOT_FOUND)
 		efivar_entry_list_del_unlock(entry);
 	else
-		spin_unlock_irq(&__efivars->lock);
+		up(&efivars_lock);
 
 	if (status && status != EFI_BUFFER_TOO_SMALL)
 		return efi_status_to_err(status);
@@ -965,7 +986,7 @@ int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes,
 	return 0;
 
 out:
-	spin_unlock_irq(&__efivars->lock);
+	up(&efivars_lock);
 	return err;
 
 }
@@ -978,9 +999,9 @@ EXPORT_SYMBOL_GPL(efivar_entry_set_get_size);
  * efivar_entry_iter_end() is called. This function is usually used in
  * conjunction with __efivar_entry_iter() or efivar_entry_iter().
  */
-void efivar_entry_iter_begin(void)
+int efivar_entry_iter_begin(void)
 {
-	spin_lock_irq(&__efivars->lock);
+	return down_interruptible(&efivars_lock);
 }
 EXPORT_SYMBOL_GPL(efivar_entry_iter_begin);
 
@@ -991,7 +1012,7 @@ EXPORT_SYMBOL_GPL(efivar_entry_iter_begin);
  */
 void efivar_entry_iter_end(void)
 {
-	spin_unlock_irq(&__efivars->lock);
+	up(&efivars_lock);
 }
 EXPORT_SYMBOL_GPL(efivar_entry_iter_end);
 
@@ -1067,7 +1088,9 @@ int efivar_entry_iter(int (*func)(struct efivar_entry *, void *),
 {
 	int err = 0;
 
-	efivar_entry_iter_begin();
+	err = efivar_entry_iter_begin();
+	if (err)
+		return err;
 	err = __efivar_entry_iter(func, head, data, NULL);
 	efivar_entry_iter_end();
 
@@ -1112,12 +1135,18 @@ int efivars_register(struct efivars *efivars,
 		     const struct efivar_operations *ops,
 		     struct kobject *kobject)
 {
-	spin_lock_init(&efivars->lock);
+	if (down_interruptible(&efivars_lock))
+		return -EINTR;
+
 	efivars->ops = ops;
 	efivars->kobject = kobject;
 
 	__efivars = efivars;
 
+	pr_info("Registered efivars operations\n");
+
+	up(&efivars_lock);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(efivars_register);
@@ -1133,6 +1162,9 @@ int efivars_unregister(struct efivars *efivars)
 {
 	int rv;
 
+	if (down_interruptible(&efivars_lock))
+		return -EINTR;
+
 	if (!__efivars) {
 		printk(KERN_ERR "efivars not registered\n");
 		rv = -EINVAL;
@@ -1144,10 +1176,12 @@ int efivars_unregister(struct efivars *efivars)
 		goto out;
 	}
 
+	pr_info("Unregistered efivars operations\n");
 	__efivars = NULL;
 
 	rv = 0;
 out:
+	up(&efivars_lock);
 	return rv;
 }
 EXPORT_SYMBOL_GPL(efivars_unregister);
diff --git a/drivers/firmware/google/gsmi.c b/drivers/firmware/google/gsmi.c
index f1ab05ea56bb..c46387160976 100644
--- a/drivers/firmware/google/gsmi.c
+++ b/drivers/firmware/google/gsmi.c
@@ -910,8 +910,7 @@ out_err:
 	gsmi_buf_free(gsmi_dev.param_buf);
 	gsmi_buf_free(gsmi_dev.data_buf);
 	gsmi_buf_free(gsmi_dev.name_buf);
-	if (gsmi_dev.dma_pool)
-		dma_pool_destroy(gsmi_dev.dma_pool);
+	dma_pool_destroy(gsmi_dev.dma_pool);
 	platform_device_unregister(gsmi_dev.pdev);
 	pr_info("gsmi: failed to load: %d\n", ret);
 	return ret;
diff --git a/drivers/firmware/meson/Kconfig b/drivers/firmware/meson/Kconfig
new file mode 100644
index 000000000000..170d7e8bcdfb
--- /dev/null
+++ b/drivers/firmware/meson/Kconfig
@@ -0,0 +1,9 @@
+#
+# Amlogic Secure Monitor driver
+#
+config MESON_SM
+	bool
+	default ARCH_MESON
+	depends on ARM64_4K_PAGES
+	help
+	  Say y here to enable the Amlogic secure monitor driver
diff --git a/drivers/firmware/meson/Makefile b/drivers/firmware/meson/Makefile
new file mode 100644
index 000000000000..9ab3884f96bc
--- /dev/null
+++ b/drivers/firmware/meson/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_MESON_SM) +=	meson_sm.o
diff --git a/drivers/firmware/meson/meson_sm.c b/drivers/firmware/meson/meson_sm.c
new file mode 100644
index 000000000000..b0d254930ed3
--- /dev/null
+++ b/drivers/firmware/meson/meson_sm.c
@@ -0,0 +1,248 @@
+/*
+ * Amlogic Secure Monitor driver
+ *
+ * Copyright (C) 2016 Endless Mobile, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define pr_fmt(fmt) "meson-sm: " fmt
+
+#include <linux/arm-smccc.h>
+#include <linux/bug.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/printk.h>
+#include <linux/types.h>
+#include <linux/sizes.h>
+
+#include <linux/firmware/meson/meson_sm.h>
+
+struct meson_sm_cmd {
+	unsigned int index;
+	u32 smc_id;
+};
+#define CMD(d, s) { .index = (d), .smc_id = (s), }
+
+struct meson_sm_chip {
+	unsigned int shmem_size;
+	u32 cmd_shmem_in_base;
+	u32 cmd_shmem_out_base;
+	struct meson_sm_cmd cmd[];
+};
+
+struct meson_sm_chip gxbb_chip = {
+	.shmem_size		= SZ_4K,
+	.cmd_shmem_in_base	= 0x82000020,
+	.cmd_shmem_out_base	= 0x82000021,
+	.cmd = {
+		CMD(SM_EFUSE_READ,	0x82000030),
+		CMD(SM_EFUSE_WRITE,	0x82000031),
+		CMD(SM_EFUSE_USER_MAX,	0x82000033),
+		{ /* sentinel */ },
+	},
+};
+
+struct meson_sm_firmware {
+	const struct meson_sm_chip *chip;
+	void __iomem *sm_shmem_in_base;
+	void __iomem *sm_shmem_out_base;
+};
+
+static struct meson_sm_firmware fw;
+
+static u32 meson_sm_get_cmd(const struct meson_sm_chip *chip,
+			    unsigned int cmd_index)
+{
+	const struct meson_sm_cmd *cmd = chip->cmd;
+
+	while (cmd->smc_id && cmd->index != cmd_index)
+		cmd++;
+
+	return cmd->smc_id;
+}
+
+static u32 __meson_sm_call(u32 cmd, u32 arg0, u32 arg1, u32 arg2,
+			   u32 arg3, u32 arg4)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_smc(cmd, arg0, arg1, arg2, arg3, arg4, 0, 0, &res);
+	return res.a0;
+}
+
+static void __iomem *meson_sm_map_shmem(u32 cmd_shmem, unsigned int size)
+{
+	u32 sm_phy_base;
+
+	sm_phy_base = __meson_sm_call(cmd_shmem, 0, 0, 0, 0, 0);
+	if (!sm_phy_base)
+		return 0;
+
+	return ioremap_cache(sm_phy_base, size);
+}
+
+/**
+ * meson_sm_call - generic SMC32 call to the secure-monitor
+ *
+ * @cmd_index:	Index of the SMC32 function ID
+ * @ret:	Returned value
+ * @arg0:	SMC32 Argument 0
+ * @arg1:	SMC32 Argument 1
+ * @arg2:	SMC32 Argument 2
+ * @arg3:	SMC32 Argument 3
+ * @arg4:	SMC32 Argument 4
+ *
+ * Return:	0 on success, a negative value on error
+ */
+int meson_sm_call(unsigned int cmd_index, u32 *ret, u32 arg0,
+		  u32 arg1, u32 arg2, u32 arg3, u32 arg4)
+{
+	u32 cmd, lret;
+
+	if (!fw.chip)
+		return -ENOENT;
+
+	cmd = meson_sm_get_cmd(fw.chip, cmd_index);
+	if (!cmd)
+		return -EINVAL;
+
+	lret = __meson_sm_call(cmd, arg0, arg1, arg2, arg3, arg4);
+
+	if (ret)
+		*ret = lret;
+
+	return 0;
+}
+EXPORT_SYMBOL(meson_sm_call);
+
+/**
+ * meson_sm_call_read - retrieve data from secure-monitor
+ *
+ * @buffer:	Buffer to store the retrieved data
+ * @cmd_index:	Index of the SMC32 function ID
+ * @arg0:	SMC32 Argument 0
+ * @arg1:	SMC32 Argument 1
+ * @arg2:	SMC32 Argument 2
+ * @arg3:	SMC32 Argument 3
+ * @arg4:	SMC32 Argument 4
+ *
+ * Return:	size of read data on success, a negative value on error
+ */
+int meson_sm_call_read(void *buffer, unsigned int cmd_index, u32 arg0,
+		       u32 arg1, u32 arg2, u32 arg3, u32 arg4)
+{
+	u32 size;
+
+	if (!fw.chip)
+		return -ENOENT;
+
+	if (!fw.chip->cmd_shmem_out_base)
+		return -EINVAL;
+
+	if (meson_sm_call(cmd_index, &size, arg0, arg1, arg2, arg3, arg4) < 0)
+		return -EINVAL;
+
+	if (!size || size > fw.chip->shmem_size)
+		return -EINVAL;
+
+	if (buffer)
+		memcpy(buffer, fw.sm_shmem_out_base, size);
+
+	return size;
+}
+EXPORT_SYMBOL(meson_sm_call_read);
+
+/**
+ * meson_sm_call_write - send data to secure-monitor
+ *
+ * @buffer:	Buffer containing data to send
+ * @size:	Size of the data to send
+ * @cmd_index:	Index of the SMC32 function ID
+ * @arg0:	SMC32 Argument 0
+ * @arg1:	SMC32 Argument 1
+ * @arg2:	SMC32 Argument 2
+ * @arg3:	SMC32 Argument 3
+ * @arg4:	SMC32 Argument 4
+ *
+ * Return:	size of sent data on success, a negative value on error
+ */
+int meson_sm_call_write(void *buffer, unsigned int size, unsigned int cmd_index,
+			u32 arg0, u32 arg1, u32 arg2, u32 arg3, u32 arg4)
+{
+	u32 written;
+
+	if (!fw.chip)
+		return -ENOENT;
+
+	if (size > fw.chip->shmem_size)
+		return -EINVAL;
+
+	if (!fw.chip->cmd_shmem_in_base)
+		return -EINVAL;
+
+	memcpy(fw.sm_shmem_in_base, buffer, size);
+
+	if (meson_sm_call(cmd_index, &written, arg0, arg1, arg2, arg3, arg4) < 0)
+		return -EINVAL;
+
+	if (!written)
+		return -EINVAL;
+
+	return written;
+}
+EXPORT_SYMBOL(meson_sm_call_write);
+
+static const struct of_device_id meson_sm_ids[] = {
+	{ .compatible = "amlogic,meson-gxbb-sm", .data = &gxbb_chip },
+	{ /* sentinel */ },
+};
+
+int __init meson_sm_init(void)
+{
+	const struct meson_sm_chip *chip;
+	const struct of_device_id *matched_np;
+	struct device_node *np;
+
+	np = of_find_matching_node_and_match(NULL, meson_sm_ids, &matched_np);
+	if (!np)
+		return -ENODEV;
+
+	chip = matched_np->data;
+	if (!chip) {
+		pr_err("unable to setup secure-monitor data\n");
+		goto out;
+	}
+
+	if (chip->cmd_shmem_in_base) {
+		fw.sm_shmem_in_base = meson_sm_map_shmem(chip->cmd_shmem_in_base,
+							 chip->shmem_size);
+		if (WARN_ON(!fw.sm_shmem_in_base))
+			goto out;
+	}
+
+	if (chip->cmd_shmem_out_base) {
+		fw.sm_shmem_out_base = meson_sm_map_shmem(chip->cmd_shmem_out_base,
+							  chip->shmem_size);
+		if (WARN_ON(!fw.sm_shmem_out_base))
+			goto out_in_base;
+	}
+
+	fw.chip = chip;
+	pr_info("secure-monitor enabled\n");
+
+	return 0;
+
+out_in_base:
+	iounmap(fw.sm_shmem_in_base);
+out:
+	return -EINVAL;
+}
+device_initcall(meson_sm_init);
diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c
index e64a501adbf4..d95c70227c05 100644
--- a/drivers/firmware/qcom_scm.c
+++ b/drivers/firmware/qcom_scm.c
@@ -1,4 +1,7 @@
-/* Copyright (c) 2010,2015, The Linux Foundation. All rights reserved.
+/*
+ * Qualcomm SCM driver
+ *
+ * Copyright (c) 2010,2015, The Linux Foundation. All rights reserved.
  * Copyright (C) 2015 Linaro Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
@@ -12,7 +15,7 @@
  *
  */
 #include <linux/platform_device.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/cpumask.h>
 #include <linux/export.h>
 #include <linux/dma-mapping.h>
@@ -376,8 +379,6 @@ static const struct of_device_id qcom_scm_dt_match[] = {
 	{}
 };
 
-MODULE_DEVICE_TABLE(of, qcom_scm_dt_match);
-
 static struct platform_driver qcom_scm_driver = {
 	.driver = {
 		.name	= "qcom_scm",
@@ -414,14 +415,4 @@ static int __init qcom_scm_init(void)
 
 	return platform_driver_register(&qcom_scm_driver);
 }
-
 subsys_initcall(qcom_scm_init);
-
-static void __exit qcom_scm_exit(void)
-{
-	platform_driver_unregister(&qcom_scm_driver);
-}
-module_exit(qcom_scm_exit);
-
-MODULE_DESCRIPTION("Qualcomm SCM driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig
index d61410299ec0..cd84934774cc 100644
--- a/drivers/fpga/Kconfig
+++ b/drivers/fpga/Kconfig
@@ -21,6 +21,7 @@ config FPGA_MGR_SOCFPGA
 
 config FPGA_MGR_ZYNQ_FPGA
 	tristate "Xilinx Zynq FPGA"
+	depends on ARCH_ZYNQ || COMPILE_TEST
 	depends on HAS_DMA
 	help
 	  FPGA manager driver support for Xilinx Zynq FPGAs.
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index 98dd47a30fc7..26ee00f6bd58 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -10,27 +10,6 @@ config ARCH_HAVE_CUSTOM_GPIO_H
 	  overriding the default implementations.  New uses of this are
 	  strongly discouraged.
 
-config ARCH_WANT_OPTIONAL_GPIOLIB
-	bool
-	help
-	  Select this config option from the architecture Kconfig, if
-	  it is possible to use gpiolib on the architecture, but let the
-	  user decide whether to actually build it or not.
-	  Select this instead of ARCH_REQUIRE_GPIOLIB, if your architecture does
-	  not depend on GPIOs being available, but rather let the user
-	  decide whether he needs it or not.
-
-config ARCH_REQUIRE_GPIOLIB
-	bool
-	select GPIOLIB
-	help
-	  Platforms select gpiolib if they use this infrastructure
-	  for all their GPIOs, usually starting with ones integrated
-	  into SOC processors.
-	  Selecting this from the architecture code will cause the gpiolib
-	  code to always get built in.
-
-
 menuconfig GPIOLIB
 	bool "GPIO Support"
 	select ANON_INODES
@@ -50,6 +29,7 @@ config GPIO_DEVRES
 config OF_GPIO
 	def_bool y
 	depends on OF
+	depends on HAS_IOMEM
 
 config GPIO_ACPI
 	def_bool y
@@ -86,6 +66,7 @@ config GPIO_SYSFS
 	  exported to userspace; this can be useful when debugging.
 
 config GPIO_GENERIC
+	depends on HAS_IOMEM # Only for IOMEM drivers
 	tristate
 
 # put drivers in the right section, in alphabetical order
@@ -95,6 +76,7 @@ config GPIO_MAX730X
 	tristate
 
 menu "Memory mapped GPIO drivers"
+	depends on HAS_IOMEM
 
 config GPIO_74XX_MMIO
 	tristate "GPIO driver for 74xx-ICs with MMIO access"
@@ -127,6 +109,13 @@ config GPIO_AMDPT
 	  driver for GPIO functionality on Promontory IOHub
 	  Require ACPI ASL code to enumerate as a platform device.
 
+config GPIO_ASPEED
+	tristate "Aspeed GPIO support"
+	depends on (ARCH_ASPEED || COMPILE_TEST) && OF_GPIO
+	select GPIOLIB_IRQCHIP
+	help
+	  Say Y here to support Aspeed AST2400 and AST2500 GPIO controllers.
+
 config GPIO_ATH79
 	tristate "Atheros AR71XX/AR724X/AR913X GPIO support"
 	default y if ATH79
@@ -137,6 +126,12 @@ config GPIO_ATH79
 	  Select this option to enable GPIO driver for
 	  Atheros AR71XX/AR724X/AR913X SoC devices.
 
+config GPIO_AXP209
+	tristate "X-Powers AXP209 PMIC GPIO Support"
+	depends on MFD_AXP20X
+	help
+	  Say yes to enable GPIO support for the AXP209 PMIC
+
 config GPIO_BCM_KONA
 	bool "Broadcom Kona GPIO"
 	depends on OF_GPIO && (ARCH_BCM_MOBILE || COMPILE_TEST)
@@ -188,7 +183,7 @@ config GPIO_EP93XX
 config GPIO_ETRAXFS
 	bool "Axis ETRAX FS General I/O"
 	depends on CRIS || COMPILE_TEST
-	depends on OF
+	depends on OF_GPIO
 	select GPIO_GENERIC
 	select GPIOLIB_IRQCHIP
 	help
@@ -214,7 +209,7 @@ config GPIO_GENERIC_PLATFORM
 
 config GPIO_GRGPIO
 	tristate "Aeroflex Gaisler GRGPIO support"
-	depends on OF
+	depends on OF_GPIO
 	select GPIO_GENERIC
 	select IRQ_DOMAIN
 	help
@@ -236,7 +231,8 @@ config GPIO_ICH
 
 config GPIO_IOP
 	tristate "Intel IOP GPIO"
-	depends on ARM && (ARCH_IOP32X || ARCH_IOP33X)
+	depends on ARCH_IOP32X || ARCH_IOP33X || COMPILE_TEST
+	select GPIO_GENERIC
 	help
 	  Say yes here to support the GPIO functionality of a number of Intel
 	  IOP32X or IOP33X.
@@ -286,6 +282,18 @@ config GPIO_MM_LANTIQ
 	  (EBU) found on Lantiq SoCs. The gpios are output only as they are
 	  created by attaching a 16bit latch to the bus.
 
+config GPIO_MOCKUP
+	tristate "GPIO Testing Driver"
+	depends on GPIOLIB
+	select GPIO_SYSFS
+	help
+	  This enables GPIO Testing driver, which provides a way to test GPIO
+	  subsystem through sysfs(or char device) and debugfs. GPIO_SYSFS
+	  must be selected for this test.
+	  User could use it through the script in
+	  tools/testing/selftests/gpio/gpio-mockup.sh. Reference the usage in
+	  it.
+
 config GPIO_MOXART
 	bool "MOXART GPIO support"
 	depends on ARCH_MOXART || COMPILE_TEST
@@ -312,7 +320,7 @@ config GPIO_MPC8XXX
 config GPIO_MVEBU
 	def_bool y
 	depends on PLAT_ORION
-	depends on OF
+	depends on OF_GPIO
 	select GENERIC_IRQ_CHIP
 
 config GPIO_MXC
@@ -405,7 +413,7 @@ config GPIO_TEGRA
 	bool "NVIDIA Tegra GPIO support"
 	default ARCH_TEGRA
 	depends on ARCH_TEGRA || COMPILE_TEST
-	depends on OF
+	depends on OF_GPIO
 	help
 	  Say yes here to support GPIO pins on NVIDIA Tegra SoCs.
 
@@ -573,6 +581,19 @@ config GPIO_F7188X
 	  To compile this driver as a module, choose M here: the module will
 	  be called f7188x-gpio.
 
+config GPIO_GPIO_MM
+	tristate "Diamond Systems GPIO-MM GPIO support"
+	depends on ISA_BUS_API
+	help
+	  Enables GPIO support for the Diamond Systems GPIO-MM and GPIO-MM-12.
+
+	  The Diamond Systems GPIO-MM device features 48 lines of digital I/O
+	  via the emulation of dual 82C55A PPI chips. This driver provides GPIO
+	  support for these 48 channels of digital I/O.
+
+	  The base port addresses for the devices may be configured via the base
+	  array module parameter.
+
 config GPIO_IT87
 	tristate "IT87xx GPIO support"
 	help
@@ -779,6 +800,13 @@ config GPIO_TPIC2810
 	  To compile this driver as a module, choose M here: the module will
 	  be called gpio-tpic2810.
 
+config GPIO_TS4900
+	tristate "Technologic Systems FPGA I2C GPIO"
+	select REGMAP_I2C
+	help
+	  Say yes here to enabled the GPIO driver for Technologic's FPGA core.
+	  Series supported include TS-4100, TS-4900, TS-7970 and TS-7990.
+
 endmenu
 
 menu "MFD GPIO expanders"
@@ -848,6 +876,14 @@ config GPIO_DLN2
 	  This driver can also be built as a module. If so, the module
 	  will be called gpio-dln2.
 
+config HTC_EGPIO
+	bool "HTC EGPIO support"
+	depends on GPIOLIB && ARM
+	help
+	    This driver supports the CPLD egpio chip present on
+	    several HTC phones.  It provides basic support for input
+	    pins, output pins, and irqs.
+
 config GPIO_JANZ_TTL
 	tristate "Janz VMOD-TTL Digital IO Module"
 	depends on MFD_JANZ_CMODIO
@@ -874,6 +910,16 @@ config GPIO_LP3943
 	  LP3943 can be used as a GPIO expander which provides up to 16 GPIOs.
 	  Open drain outputs are required for this usage.
 
+config GPIO_LP873X
+	tristate "TI LP873X GPO"
+	depends on MFD_TI_LP873X
+	help
+	  This driver supports the GPO on TI Lp873x PMICs. 2 GPOs are present
+	  on LP873X PMICs.
+
+	  This driver can also be built as a module. If so, the module will be
+          called gpio-lp873x.
+
 config GPIO_MAX77620
 	tristate "GPIO support for PMIC MAX77620 and MAX20024"
 	depends on MFD_MAX77620
@@ -984,6 +1030,19 @@ config GPIO_UCB1400
 	  This enables support for the Philips UCB1400 GPIO pins.
 	  The UCB1400 is an AC97 audio codec.
 
+config GPIO_WHISKEY_COVE
+	tristate "GPIO support for Whiskey Cove PMIC"
+	depends on INTEL_SOC_PMIC
+	select GPIOLIB_IRQCHIP
+	help
+	  Support for GPIO pins on Whiskey Cove PMIC.
+
+	  Say Yes if you have a Intel SoC based tablet with Whiskey Cove PMIC
+	  inside.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called gpio-wcove.
+
 config GPIO_WM831X
 	tristate "WM831x GPIOs"
 	depends on MFD_WM831X
@@ -1099,7 +1158,7 @@ menu "SPI GPIO expanders"
 
 config GPIO_74X164
 	tristate "74x164 serial-in/parallel-out 8-bits shift register"
-	depends on OF
+	depends on OF_GPIO
 	help
 	  Driver for 74x164 compatible serial-in/parallel-out 8-outputs
 	  shift registers. This driver can be used to provide access
@@ -1130,6 +1189,7 @@ menu "SPI or I2C GPIO expanders"
 
 config GPIO_MCP23S08
 	tristate "Microchip MCP23xxx I/O expander"
+	depends on OF_GPIO
 	select GPIOLIB_IRQCHIP
 	help
 	  SPI/I2C driver for Microchip MCP23S08/MCP23S17/MCP23008/MCP23017
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index 2a035ed8f168..ab28a2daeacc 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -28,6 +28,8 @@ obj-$(CONFIG_GPIO_AMD8111)	+= gpio-amd8111.o
 obj-$(CONFIG_GPIO_AMDPT)	+= gpio-amdpt.o
 obj-$(CONFIG_GPIO_ARIZONA)	+= gpio-arizona.o
 obj-$(CONFIG_GPIO_ATH79)	+= gpio-ath79.o
+obj-$(CONFIG_GPIO_ASPEED)	+= gpio-aspeed.o
+obj-$(CONFIG_GPIO_AXP209)	+= gpio-axp209.o
 obj-$(CONFIG_GPIO_BCM_KONA)	+= gpio-bcm-kona.o
 obj-$(CONFIG_GPIO_BRCMSTB)	+= gpio-brcmstb.o
 obj-$(CONFIG_GPIO_BT8XX)	+= gpio-bt8xx.o
@@ -44,7 +46,9 @@ obj-$(CONFIG_GPIO_EP93XX)	+= gpio-ep93xx.o
 obj-$(CONFIG_GPIO_ETRAXFS)	+= gpio-etraxfs.o
 obj-$(CONFIG_GPIO_F7188X)	+= gpio-f7188x.o
 obj-$(CONFIG_GPIO_GE_FPGA)	+= gpio-ge.o
+obj-$(CONFIG_GPIO_GPIO_MM)	+= gpio-gpio-mm.o
 obj-$(CONFIG_GPIO_GRGPIO)	+= gpio-grgpio.o
+obj-$(CONFIG_HTC_EGPIO)		+= gpio-htc-egpio.o
 obj-$(CONFIG_GPIO_ICH)		+= gpio-ich.o
 obj-$(CONFIG_GPIO_IOP)		+= gpio-iop.o
 obj-$(CONFIG_GPIO_IT87)		+= gpio-it87.o
@@ -56,6 +60,7 @@ obj-$(CONFIG_GPIO_LOONGSON)	+= gpio-loongson.o
 obj-$(CONFIG_GPIO_LP3943)	+= gpio-lp3943.o
 obj-$(CONFIG_GPIO_LPC18XX)	+= gpio-lpc18xx.o
 obj-$(CONFIG_ARCH_LPC32XX)	+= gpio-lpc32xx.o
+obj-$(CONFIG_GPIO_LP873X)	+= gpio-lp873x.o
 obj-$(CONFIG_GPIO_LYNXPOINT)	+= gpio-lynxpoint.o
 obj-$(CONFIG_GPIO_MAX730X)	+= gpio-max730x.o
 obj-$(CONFIG_GPIO_MAX7300)	+= gpio-max7300.o
@@ -70,6 +75,7 @@ obj-$(CONFIG_GPIO_MC9S08DZ60)	+= gpio-mc9s08dz60.o
 obj-$(CONFIG_GPIO_MCP23S08)	+= gpio-mcp23s08.o
 obj-$(CONFIG_GPIO_ML_IOH)	+= gpio-ml-ioh.o
 obj-$(CONFIG_GPIO_MM_LANTIQ)	+= gpio-mm-lantiq.o
+obj-$(CONFIG_GPIO_MOCKUP)      += gpio-mockup.o
 obj-$(CONFIG_GPIO_MOXART)	+= gpio-moxart.o
 obj-$(CONFIG_GPIO_MPC5200)	+= gpio-mpc5200.o
 obj-$(CONFIG_GPIO_MPC8XXX)	+= gpio-mpc8xxx.o
@@ -110,6 +116,7 @@ obj-$(CONFIG_GPIO_TPS6586X)	+= gpio-tps6586x.o
 obj-$(CONFIG_GPIO_TPS65910)	+= gpio-tps65910.o
 obj-$(CONFIG_GPIO_TPS65912)	+= gpio-tps65912.o
 obj-$(CONFIG_GPIO_TS4800)	+= gpio-ts4800.o
+obj-$(CONFIG_GPIO_TS4900)	+= gpio-ts4900.o
 obj-$(CONFIG_GPIO_TS5500)	+= gpio-ts5500.o
 obj-$(CONFIG_GPIO_TWL4030)	+= gpio-twl4030.o
 obj-$(CONFIG_GPIO_TWL6040)	+= gpio-twl6040.o
@@ -120,6 +127,7 @@ obj-$(CONFIG_GPIO_VF610)	+= gpio-vf610.o
 obj-$(CONFIG_GPIO_VIPERBOARD)	+= gpio-viperboard.o
 obj-$(CONFIG_GPIO_VR41XX)	+= gpio-vr41xx.o
 obj-$(CONFIG_GPIO_VX855)	+= gpio-vx855.o
+obj-$(CONFIG_GPIO_WHISKEY_COVE)	+= gpio-wcove.o
 obj-$(CONFIG_GPIO_WM831X)	+= gpio-wm831x.o
 obj-$(CONFIG_GPIO_WM8350)	+= gpio-wm8350.o
 obj-$(CONFIG_GPIO_WM8994)	+= gpio-wm8994.o
diff --git a/drivers/gpio/gpio-altera.c b/drivers/gpio/gpio-altera.c
index 3f87a03abc22..5bddbd507ca9 100644
--- a/drivers/gpio/gpio-altera.c
+++ b/drivers/gpio/gpio-altera.c
@@ -17,6 +17,7 @@
  */
 
 #include <linux/io.h>
+#include <linux/module.h>
 #include <linux/of_gpio.h>
 #include <linux/platform_device.h>
 
diff --git a/drivers/gpio/gpio-arizona.c b/drivers/gpio/gpio-arizona.c
index 991370494922..482462889c8f 100644
--- a/drivers/gpio/gpio-arizona.c
+++ b/drivers/gpio/gpio-arizona.c
@@ -79,7 +79,7 @@ static void arizona_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 			   ARIZONA_GPN_LVL, value);
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "arizona",
 	.owner			= THIS_MODULE,
 	.direction_input	= arizona_gpio_direction_in,
diff --git a/drivers/gpio/gpio-aspeed.c b/drivers/gpio/gpio-aspeed.c
new file mode 100644
index 000000000000..03a5925a423c
--- /dev/null
+++ b/drivers/gpio/gpio-aspeed.c
@@ -0,0 +1,455 @@
+/*
+ * Copyright 2015 IBM Corp.
+ *
+ * Joel Stanley <joel@jms.id.au>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+#include <linux/platform_device.h>
+#include <linux/gpio/driver.h>
+#include <linux/pinctrl/consumer.h>
+
+struct aspeed_gpio {
+	struct gpio_chip chip;
+	spinlock_t lock;
+	void __iomem *base;
+	int irq;
+};
+
+struct aspeed_gpio_bank {
+	uint16_t	val_regs;
+	uint16_t	irq_regs;
+	const char	names[4];
+};
+
+static const struct aspeed_gpio_bank aspeed_gpio_banks[] = {
+	{
+		.val_regs = 0x0000,
+		.irq_regs = 0x0008,
+		.names = { 'A', 'B', 'C', 'D' },
+	},
+	{
+		.val_regs = 0x0020,
+		.irq_regs = 0x0028,
+		.names = { 'E', 'F', 'G', 'H' },
+	},
+	{
+		.val_regs = 0x0070,
+		.irq_regs = 0x0098,
+		.names = { 'I', 'J', 'K', 'L' },
+	},
+	{
+		.val_regs = 0x0078,
+		.irq_regs = 0x00e8,
+		.names = { 'M', 'N', 'O', 'P' },
+	},
+	{
+		.val_regs = 0x0080,
+		.irq_regs = 0x0118,
+		.names = { 'Q', 'R', 'S', 'T' },
+	},
+	{
+		.val_regs = 0x0088,
+		.irq_regs = 0x0148,
+		.names = { 'U', 'V', 'W', 'X' },
+	},
+	/*
+	 * A bank exists for { 'Y', 'Z', "AA", "AB" }, but is not implemented.
+	 * Only half of GPIOs Y support interrupt configuration, and none of Z,
+	 * AA or AB do as they are output only.
+	 */
+};
+
+#define GPIO_BANK(x)	((x) >> 5)
+#define GPIO_OFFSET(x)	((x) & 0x1f)
+#define GPIO_BIT(x)	BIT(GPIO_OFFSET(x))
+
+#define GPIO_DATA	0x00
+#define GPIO_DIR	0x04
+
+#define GPIO_IRQ_ENABLE	0x00
+#define GPIO_IRQ_TYPE0	0x04
+#define GPIO_IRQ_TYPE1	0x08
+#define GPIO_IRQ_TYPE2	0x0c
+#define GPIO_IRQ_STATUS	0x10
+
+static const struct aspeed_gpio_bank *to_bank(unsigned int offset)
+{
+	unsigned int bank = GPIO_BANK(offset);
+
+	WARN_ON(bank > ARRAY_SIZE(aspeed_gpio_banks));
+	return &aspeed_gpio_banks[bank];
+}
+
+static void __iomem *bank_val_reg(struct aspeed_gpio *gpio,
+		const struct aspeed_gpio_bank *bank,
+		unsigned int reg)
+{
+	return gpio->base + bank->val_regs + reg;
+}
+
+static void __iomem *bank_irq_reg(struct aspeed_gpio *gpio,
+		const struct aspeed_gpio_bank *bank,
+		unsigned int reg)
+{
+	return gpio->base + bank->irq_regs + reg;
+}
+
+static int aspeed_gpio_get(struct gpio_chip *gc, unsigned int offset)
+{
+	struct aspeed_gpio *gpio = gpiochip_get_data(gc);
+	const struct aspeed_gpio_bank *bank = to_bank(offset);
+
+	return !!(ioread32(bank_val_reg(gpio, bank, GPIO_DATA))
+			& GPIO_BIT(offset));
+}
+
+static void __aspeed_gpio_set(struct gpio_chip *gc, unsigned int offset,
+			      int val)
+{
+	struct aspeed_gpio *gpio = gpiochip_get_data(gc);
+	const struct aspeed_gpio_bank *bank = to_bank(offset);
+	void __iomem *addr;
+	u32 reg;
+
+	addr = bank_val_reg(gpio, bank, GPIO_DATA);
+	reg = ioread32(addr);
+
+	if (val)
+		reg |= GPIO_BIT(offset);
+	else
+		reg &= ~GPIO_BIT(offset);
+
+	iowrite32(reg, addr);
+}
+
+static void aspeed_gpio_set(struct gpio_chip *gc, unsigned int offset,
+			    int val)
+{
+	struct aspeed_gpio *gpio = gpiochip_get_data(gc);
+	unsigned long flags;
+
+	spin_lock_irqsave(&gpio->lock, flags);
+
+	__aspeed_gpio_set(gc, offset, val);
+
+	spin_unlock_irqrestore(&gpio->lock, flags);
+}
+
+static int aspeed_gpio_dir_in(struct gpio_chip *gc, unsigned int offset)
+{
+	struct aspeed_gpio *gpio = gpiochip_get_data(gc);
+	const struct aspeed_gpio_bank *bank = to_bank(offset);
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&gpio->lock, flags);
+
+	reg = ioread32(bank_val_reg(gpio, bank, GPIO_DIR));
+	iowrite32(reg & ~GPIO_BIT(offset), bank_val_reg(gpio, bank, GPIO_DIR));
+
+	spin_unlock_irqrestore(&gpio->lock, flags);
+
+	return 0;
+}
+
+static int aspeed_gpio_dir_out(struct gpio_chip *gc,
+			       unsigned int offset, int val)
+{
+	struct aspeed_gpio *gpio = gpiochip_get_data(gc);
+	const struct aspeed_gpio_bank *bank = to_bank(offset);
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&gpio->lock, flags);
+
+	reg = ioread32(bank_val_reg(gpio, bank, GPIO_DIR));
+	iowrite32(reg | GPIO_BIT(offset), bank_val_reg(gpio, bank, GPIO_DIR));
+
+	__aspeed_gpio_set(gc, offset, val);
+
+	spin_unlock_irqrestore(&gpio->lock, flags);
+
+	return 0;
+}
+
+static int aspeed_gpio_get_direction(struct gpio_chip *gc, unsigned int offset)
+{
+	struct aspeed_gpio *gpio = gpiochip_get_data(gc);
+	const struct aspeed_gpio_bank *bank = to_bank(offset);
+	unsigned long flags;
+	u32 val;
+
+	spin_lock_irqsave(&gpio->lock, flags);
+
+	val = ioread32(bank_val_reg(gpio, bank, GPIO_DIR)) & GPIO_BIT(offset);
+
+	spin_unlock_irqrestore(&gpio->lock, flags);
+
+	return !val;
+
+}
+
+static inline int irqd_to_aspeed_gpio_data(struct irq_data *d,
+		struct aspeed_gpio **gpio,
+		const struct aspeed_gpio_bank **bank,
+		u32 *bit)
+{
+	int offset;
+
+	offset = irqd_to_hwirq(d);
+
+	*gpio = irq_data_get_irq_chip_data(d);
+	*bank = to_bank(offset);
+	*bit = GPIO_BIT(offset);
+
+	return 0;
+}
+
+static void aspeed_gpio_irq_ack(struct irq_data *d)
+{
+	const struct aspeed_gpio_bank *bank;
+	struct aspeed_gpio *gpio;
+	unsigned long flags;
+	void __iomem *status_addr;
+	u32 bit;
+	int rc;
+
+	rc = irqd_to_aspeed_gpio_data(d, &gpio, &bank, &bit);
+	if (rc)
+		return;
+
+	status_addr = bank_irq_reg(gpio, bank, GPIO_IRQ_STATUS);
+
+	spin_lock_irqsave(&gpio->lock, flags);
+	iowrite32(bit, status_addr);
+	spin_unlock_irqrestore(&gpio->lock, flags);
+}
+
+static void aspeed_gpio_irq_set_mask(struct irq_data *d, bool set)
+{
+	const struct aspeed_gpio_bank *bank;
+	struct aspeed_gpio *gpio;
+	unsigned long flags;
+	u32 reg, bit;
+	void __iomem *addr;
+	int rc;
+
+	rc = irqd_to_aspeed_gpio_data(d, &gpio, &bank, &bit);
+	if (rc)
+		return;
+
+	addr = bank_irq_reg(gpio, bank, GPIO_IRQ_ENABLE);
+
+	spin_lock_irqsave(&gpio->lock, flags);
+
+	reg = ioread32(addr);
+	if (set)
+		reg |= bit;
+	else
+		reg &= bit;
+	iowrite32(reg, addr);
+
+	spin_unlock_irqrestore(&gpio->lock, flags);
+}
+
+static void aspeed_gpio_irq_mask(struct irq_data *d)
+{
+	aspeed_gpio_irq_set_mask(d, false);
+}
+
+static void aspeed_gpio_irq_unmask(struct irq_data *d)
+{
+	aspeed_gpio_irq_set_mask(d, true);
+}
+
+static int aspeed_gpio_set_type(struct irq_data *d, unsigned int type)
+{
+	u32 type0 = 0;
+	u32 type1 = 0;
+	u32 type2 = 0;
+	u32 bit, reg;
+	const struct aspeed_gpio_bank *bank;
+	irq_flow_handler_t handler;
+	struct aspeed_gpio *gpio;
+	unsigned long flags;
+	void __iomem *addr;
+	int rc;
+
+	rc = irqd_to_aspeed_gpio_data(d, &gpio, &bank, &bit);
+	if (rc)
+		return -EINVAL;
+
+	switch (type & IRQ_TYPE_SENSE_MASK) {
+	case IRQ_TYPE_EDGE_BOTH:
+		type2 |= bit;
+	case IRQ_TYPE_EDGE_RISING:
+		type0 |= bit;
+	case IRQ_TYPE_EDGE_FALLING:
+		handler = handle_edge_irq;
+		break;
+	case IRQ_TYPE_LEVEL_HIGH:
+		type0 |= bit;
+	case IRQ_TYPE_LEVEL_LOW:
+		type1 |= bit;
+		handler = handle_level_irq;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&gpio->lock, flags);
+
+	addr = bank_irq_reg(gpio, bank, GPIO_IRQ_TYPE0);
+	reg = ioread32(addr);
+	reg = (reg & ~bit) | type0;
+	iowrite32(reg, addr);
+
+	addr = bank_irq_reg(gpio, bank, GPIO_IRQ_TYPE1);
+	reg = ioread32(addr);
+	reg = (reg & ~bit) | type1;
+	iowrite32(reg, addr);
+
+	addr = bank_irq_reg(gpio, bank, GPIO_IRQ_TYPE2);
+	reg = ioread32(addr);
+	reg = (reg & ~bit) | type2;
+	iowrite32(reg, addr);
+
+	spin_unlock_irqrestore(&gpio->lock, flags);
+
+	irq_set_handler_locked(d, handler);
+
+	return 0;
+}
+
+static void aspeed_gpio_irq_handler(struct irq_desc *desc)
+{
+	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
+	struct irq_chip *ic = irq_desc_get_chip(desc);
+	struct aspeed_gpio *data = gpiochip_get_data(gc);
+	unsigned int i, p, girq;
+	unsigned long reg;
+
+	chained_irq_enter(ic, desc);
+
+	for (i = 0; i < ARRAY_SIZE(aspeed_gpio_banks); i++) {
+		const struct aspeed_gpio_bank *bank = &aspeed_gpio_banks[i];
+
+		reg = ioread32(bank_irq_reg(data, bank, GPIO_IRQ_STATUS));
+
+		for_each_set_bit(p, &reg, 32) {
+			girq = irq_find_mapping(gc->irqdomain, i * 32 + p);
+			generic_handle_irq(girq);
+		}
+
+	}
+
+	chained_irq_exit(ic, desc);
+}
+
+static struct irq_chip aspeed_gpio_irqchip = {
+	.name		= "aspeed-gpio",
+	.irq_ack	= aspeed_gpio_irq_ack,
+	.irq_mask	= aspeed_gpio_irq_mask,
+	.irq_unmask	= aspeed_gpio_irq_unmask,
+	.irq_set_type	= aspeed_gpio_set_type,
+};
+
+static int aspeed_gpio_setup_irqs(struct aspeed_gpio *gpio,
+		struct platform_device *pdev)
+{
+	int rc;
+
+	rc = platform_get_irq(pdev, 0);
+	if (rc < 0)
+		return rc;
+
+	gpio->irq = rc;
+
+	rc = gpiochip_irqchip_add(&gpio->chip, &aspeed_gpio_irqchip,
+			0, handle_bad_irq, IRQ_TYPE_NONE);
+	if (rc) {
+		dev_info(&pdev->dev, "Could not add irqchip\n");
+		return rc;
+	}
+
+	gpiochip_set_chained_irqchip(&gpio->chip, &aspeed_gpio_irqchip,
+				     gpio->irq, aspeed_gpio_irq_handler);
+
+	return 0;
+}
+
+static int aspeed_gpio_request(struct gpio_chip *chip, unsigned int offset)
+{
+	return pinctrl_request_gpio(chip->base + offset);
+}
+
+static void aspeed_gpio_free(struct gpio_chip *chip, unsigned int offset)
+{
+	pinctrl_free_gpio(chip->base + offset);
+}
+
+static int __init aspeed_gpio_probe(struct platform_device *pdev)
+{
+	struct aspeed_gpio *gpio;
+	struct resource *res;
+	int rc;
+
+	gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
+	if (!gpio)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	gpio->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(gpio->base))
+		return PTR_ERR(gpio->base);
+
+	spin_lock_init(&gpio->lock);
+
+	gpio->chip.ngpio = ARRAY_SIZE(aspeed_gpio_banks) * 32;
+
+	gpio->chip.parent = &pdev->dev;
+	gpio->chip.direction_input = aspeed_gpio_dir_in;
+	gpio->chip.direction_output = aspeed_gpio_dir_out;
+	gpio->chip.get_direction = aspeed_gpio_get_direction;
+	gpio->chip.request = aspeed_gpio_request;
+	gpio->chip.free = aspeed_gpio_free;
+	gpio->chip.get = aspeed_gpio_get;
+	gpio->chip.set = aspeed_gpio_set;
+	gpio->chip.label = dev_name(&pdev->dev);
+	gpio->chip.base = -1;
+
+	rc = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio);
+	if (rc < 0)
+		return rc;
+
+	return aspeed_gpio_setup_irqs(gpio, pdev);
+}
+
+static const struct of_device_id aspeed_gpio_of_table[] = {
+	{ .compatible = "aspeed,ast2400-gpio" },
+	{ .compatible = "aspeed,ast2500-gpio" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, aspeed_gpio_of_table);
+
+static struct platform_driver aspeed_gpio_driver = {
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.of_match_table = aspeed_gpio_of_table,
+	},
+};
+
+module_platform_driver_probe(aspeed_gpio_driver, aspeed_gpio_probe);
+
+MODULE_DESCRIPTION("Aspeed GPIO Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-ath79.c b/drivers/gpio/gpio-ath79.c
index c4f4cddc7c1a..9457e2022bf6 100644
--- a/drivers/gpio/gpio-ath79.c
+++ b/drivers/gpio/gpio-ath79.c
@@ -15,6 +15,7 @@
 #include <linux/platform_data/gpio-ath79.h>
 #include <linux/of_device.h>
 #include <linux/interrupt.h>
+#include <linux/module.h>
 #include <linux/irq.h>
 
 #define AR71XX_GPIO_REG_OE		0x00
diff --git a/drivers/gpio/gpio-axp209.c b/drivers/gpio/gpio-axp209.c
new file mode 100644
index 000000000000..d9c2a517c6df
--- /dev/null
+++ b/drivers/gpio/gpio-axp209.c
@@ -0,0 +1,192 @@
+/*
+ * AXP20x GPIO driver
+ *
+ * Copyright (C) 2016 Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under  the terms of the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/gpio/driver.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/axp20x.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#define AXP20X_GPIO_FUNCTIONS		0x7
+#define AXP20X_GPIO_FUNCTION_OUT_LOW	0
+#define AXP20X_GPIO_FUNCTION_OUT_HIGH	1
+#define AXP20X_GPIO_FUNCTION_INPUT	2
+
+struct axp20x_gpio {
+	struct gpio_chip	chip;
+	struct regmap		*regmap;
+};
+
+static int axp20x_gpio_get_reg(unsigned offset)
+{
+	switch (offset) {
+	case 0:
+		return AXP20X_GPIO0_CTRL;
+	case 1:
+		return AXP20X_GPIO1_CTRL;
+	case 2:
+		return AXP20X_GPIO2_CTRL;
+	}
+
+	return -EINVAL;
+}
+
+static int axp20x_gpio_input(struct gpio_chip *chip, unsigned offset)
+{
+	struct axp20x_gpio *gpio = gpiochip_get_data(chip);
+	int reg;
+
+	reg = axp20x_gpio_get_reg(offset);
+	if (reg < 0)
+		return reg;
+
+	return regmap_update_bits(gpio->regmap, reg,
+				  AXP20X_GPIO_FUNCTIONS,
+				  AXP20X_GPIO_FUNCTION_INPUT);
+}
+
+static int axp20x_gpio_get(struct gpio_chip *chip, unsigned offset)
+{
+	struct axp20x_gpio *gpio = gpiochip_get_data(chip);
+	unsigned int val;
+	int reg, ret;
+
+	reg = axp20x_gpio_get_reg(offset);
+	if (reg < 0)
+		return reg;
+
+	ret = regmap_read(gpio->regmap, reg, &val);
+	if (ret)
+		return ret;
+
+	return !!(val & BIT(offset + 4));
+}
+
+static int axp20x_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
+{
+	struct axp20x_gpio *gpio = gpiochip_get_data(chip);
+	unsigned int val;
+	int reg, ret;
+
+	reg = axp20x_gpio_get_reg(offset);
+	if (reg < 0)
+		return reg;
+
+	ret = regmap_read(gpio->regmap, reg, &val);
+	if (ret)
+		return ret;
+
+	/*
+	 * This shouldn't really happen if the pin is in use already,
+	 * or if it's not in use yet, it doesn't matter since we're
+	 * going to change the value soon anyway. Default to output.
+	 */
+	if ((val & AXP20X_GPIO_FUNCTIONS) > 2)
+		return 0;
+
+	/*
+	 * The GPIO directions are the three lowest values.
+	 * 2 is input, 0 and 1 are output
+	 */
+	return val & 2;
+}
+
+static int axp20x_gpio_output(struct gpio_chip *chip, unsigned offset,
+			      int value)
+{
+	struct axp20x_gpio *gpio = gpiochip_get_data(chip);
+	int reg;
+
+	reg = axp20x_gpio_get_reg(offset);
+	if (reg < 0)
+		return reg;
+
+	return regmap_update_bits(gpio->regmap, reg,
+				  AXP20X_GPIO_FUNCTIONS,
+				  value ? AXP20X_GPIO_FUNCTION_OUT_HIGH
+				  : AXP20X_GPIO_FUNCTION_OUT_LOW);
+}
+
+static void axp20x_gpio_set(struct gpio_chip *chip, unsigned offset,
+			    int value)
+{
+	axp20x_gpio_output(chip, offset, value);
+}
+
+static int axp20x_gpio_probe(struct platform_device *pdev)
+{
+	struct axp20x_dev *axp20x = dev_get_drvdata(pdev->dev.parent);
+	struct axp20x_gpio *gpio;
+	int ret;
+
+	if (!of_device_is_available(pdev->dev.of_node))
+		return -ENODEV;
+
+	if (!axp20x) {
+		dev_err(&pdev->dev, "Parent drvdata not set\n");
+		return -EINVAL;
+	}
+
+	gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
+	if (!gpio)
+		return -ENOMEM;
+
+	gpio->chip.base			= -1;
+	gpio->chip.can_sleep		= true;
+	gpio->chip.parent		= &pdev->dev;
+	gpio->chip.label		= dev_name(&pdev->dev);
+	gpio->chip.owner		= THIS_MODULE;
+	gpio->chip.get			= axp20x_gpio_get;
+	gpio->chip.get_direction	= axp20x_gpio_get_direction;
+	gpio->chip.set			= axp20x_gpio_set;
+	gpio->chip.direction_input	= axp20x_gpio_input;
+	gpio->chip.direction_output	= axp20x_gpio_output;
+	gpio->chip.ngpio		= 3;
+
+	gpio->regmap = axp20x->regmap;
+
+	ret = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to register GPIO chip\n");
+		return ret;
+	}
+
+	dev_info(&pdev->dev, "AXP209 GPIO driver loaded\n");
+
+	return 0;
+}
+
+static const struct of_device_id axp20x_gpio_match[] = {
+	{ .compatible = "x-powers,axp209-gpio" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, axp20x_gpio_match);
+
+static struct platform_driver axp20x_gpio_driver = {
+	.probe		= axp20x_gpio_probe,
+	.driver = {
+		.name		= "axp20x-gpio",
+		.of_match_table	= axp20x_gpio_match,
+	},
+};
+
+module_platform_driver(axp20x_gpio_driver);
+
+MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
+MODULE_DESCRIPTION("AXP20x PMIC GPIO driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-bcm-kona.c b/drivers/gpio/gpio-bcm-kona.c
index 953e4b829e32..3d1cf018e8e7 100644
--- a/drivers/gpio/gpio-bcm-kona.c
+++ b/drivers/gpio/gpio-bcm-kona.c
@@ -308,7 +308,7 @@ static int bcm_kona_gpio_set_debounce(struct gpio_chip *chip, unsigned gpio,
 	return 0;
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label = "bcm-kona-gpio",
 	.owner = THIS_MODULE,
 	.request = bcm_kona_gpio_request,
diff --git a/drivers/gpio/gpio-da9052.c b/drivers/gpio/gpio-da9052.c
index e29553b7ccdb..dd8977cf3e85 100644
--- a/drivers/gpio/gpio-da9052.c
+++ b/drivers/gpio/gpio-da9052.c
@@ -184,7 +184,7 @@ static int da9052_gpio_to_irq(struct gpio_chip *gc, u32 offset)
 	return irq;
 }
 
-static struct gpio_chip reference_gp = {
+static const struct gpio_chip reference_gp = {
 	.label = "da9052-gpio",
 	.owner = THIS_MODULE,
 	.get = da9052_gpio_get,
diff --git a/drivers/gpio/gpio-da9055.c b/drivers/gpio/gpio-da9055.c
index 2c2c18dc6c4f..82053b52cba0 100644
--- a/drivers/gpio/gpio-da9055.c
+++ b/drivers/gpio/gpio-da9055.c
@@ -121,7 +121,7 @@ static int da9055_gpio_to_irq(struct gpio_chip *gc, u32 offset)
 				  DA9055_IRQ_GPI0 + offset);
 }
 
-static struct gpio_chip reference_gp = {
+static const struct gpio_chip reference_gp = {
 	.label = "da9055-gpio",
 	.owner = THIS_MODULE,
 	.get = da9055_gpio_get,
diff --git a/drivers/gpio/gpio-f7188x.c b/drivers/gpio/gpio-f7188x.c
index 600be8418707..e8accde62aa7 100644
--- a/drivers/gpio/gpio-f7188x.c
+++ b/drivers/gpio/gpio-f7188x.c
@@ -214,8 +214,7 @@ static struct f7188x_gpio_bank f81866_gpio_bank[] = {
 static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 {
 	int err;
-	struct f7188x_gpio_bank *bank =
-		container_of(chip, struct f7188x_gpio_bank, chip);
+	struct f7188x_gpio_bank *bank = gpiochip_get_data(chip);
 	struct f7188x_sio *sio = bank->data->sio;
 	u8 dir;
 
diff --git a/drivers/gpio/gpio-gpio-mm.c b/drivers/gpio/gpio-gpio-mm.c
new file mode 100644
index 000000000000..1e7def9449ce
--- /dev/null
+++ b/drivers/gpio/gpio-gpio-mm.c
@@ -0,0 +1,267 @@
+/*
+ * GPIO driver for the Diamond Systems GPIO-MM
+ * Copyright (C) 2016 William Breathitt Gray
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * This driver supports the following Diamond Systems devices: GPIO-MM and
+ * GPIO-MM-12.
+ */
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/gpio/driver.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/isa.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/spinlock.h>
+
+#define GPIOMM_EXTENT 8
+#define MAX_NUM_GPIOMM max_num_isa_dev(GPIOMM_EXTENT)
+
+static unsigned int base[MAX_NUM_GPIOMM];
+static unsigned int num_gpiomm;
+module_param_array(base, uint, &num_gpiomm, 0);
+MODULE_PARM_DESC(base, "Diamond Systems GPIO-MM base addresses");
+
+/**
+ * struct gpiomm_gpio - GPIO device private data structure
+ * @chip:	instance of the gpio_chip
+ * @io_state:	bit I/O state (whether bit is set to input or output)
+ * @out_state:	output bits state
+ * @control:	Control registers state
+ * @lock:	synchronization lock to prevent I/O race conditions
+ * @base:	base port address of the GPIO device
+ */
+struct gpiomm_gpio {
+	struct gpio_chip chip;
+	unsigned char io_state[6];
+	unsigned char out_state[6];
+	unsigned char control[2];
+	spinlock_t lock;
+	unsigned int base;
+};
+
+static int gpiomm_gpio_get_direction(struct gpio_chip *chip,
+	unsigned int offset)
+{
+	struct gpiomm_gpio *const gpiommgpio = gpiochip_get_data(chip);
+	const unsigned int port = offset / 8;
+	const unsigned int mask = BIT(offset % 8);
+
+	return !!(gpiommgpio->io_state[port] & mask);
+}
+
+static int gpiomm_gpio_direction_input(struct gpio_chip *chip,
+	unsigned int offset)
+{
+	struct gpiomm_gpio *const gpiommgpio = gpiochip_get_data(chip);
+	const unsigned int io_port = offset / 8;
+	const unsigned int control_port = io_port / 3;
+	const unsigned int control_addr = gpiommgpio->base + 3 + control_port*4;
+	unsigned long flags;
+	unsigned int control;
+
+	spin_lock_irqsave(&gpiommgpio->lock, flags);
+
+	/* Check if configuring Port C */
+	if (io_port == 2 || io_port == 5) {
+		/* Port C can be configured by nibble */
+		if (offset % 8 > 3) {
+			gpiommgpio->io_state[io_port] |= 0xF0;
+			gpiommgpio->control[control_port] |= BIT(3);
+		} else {
+			gpiommgpio->io_state[io_port] |= 0x0F;
+			gpiommgpio->control[control_port] |= BIT(0);
+		}
+	} else {
+		gpiommgpio->io_state[io_port] |= 0xFF;
+		if (io_port == 0 || io_port == 3)
+			gpiommgpio->control[control_port] |= BIT(4);
+		else
+			gpiommgpio->control[control_port] |= BIT(1);
+	}
+
+	control = BIT(7) | gpiommgpio->control[control_port];
+	outb(control, control_addr);
+
+	spin_unlock_irqrestore(&gpiommgpio->lock, flags);
+
+	return 0;
+}
+
+static int gpiomm_gpio_direction_output(struct gpio_chip *chip,
+	unsigned int offset, int value)
+{
+	struct gpiomm_gpio *const gpiommgpio = gpiochip_get_data(chip);
+	const unsigned int io_port = offset / 8;
+	const unsigned int control_port = io_port / 3;
+	const unsigned int mask = BIT(offset % 8);
+	const unsigned int control_addr = gpiommgpio->base + 3 + control_port*4;
+	const unsigned int out_port = (io_port > 2) ? io_port + 1 : io_port;
+	unsigned long flags;
+	unsigned int control;
+
+	spin_lock_irqsave(&gpiommgpio->lock, flags);
+
+	/* Check if configuring Port C */
+	if (io_port == 2 || io_port == 5) {
+		/* Port C can be configured by nibble */
+		if (offset % 8 > 3) {
+			gpiommgpio->io_state[io_port] &= 0x0F;
+			gpiommgpio->control[control_port] &= ~BIT(3);
+		} else {
+			gpiommgpio->io_state[io_port] &= 0xF0;
+			gpiommgpio->control[control_port] &= ~BIT(0);
+		}
+	} else {
+		gpiommgpio->io_state[io_port] &= 0x00;
+		if (io_port == 0 || io_port == 3)
+			gpiommgpio->control[control_port] &= ~BIT(4);
+		else
+			gpiommgpio->control[control_port] &= ~BIT(1);
+	}
+
+	if (value)
+		gpiommgpio->out_state[io_port] |= mask;
+	else
+		gpiommgpio->out_state[io_port] &= ~mask;
+
+	control = BIT(7) | gpiommgpio->control[control_port];
+	outb(control, control_addr);
+
+	outb(gpiommgpio->out_state[io_port], gpiommgpio->base + out_port);
+
+	spin_unlock_irqrestore(&gpiommgpio->lock, flags);
+
+	return 0;
+}
+
+static int gpiomm_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct gpiomm_gpio *const gpiommgpio = gpiochip_get_data(chip);
+	const unsigned int port = offset / 8;
+	const unsigned int mask = BIT(offset % 8);
+	const unsigned int in_port = (port > 2) ? port + 1 : port;
+	unsigned long flags;
+	unsigned int port_state;
+
+	spin_lock_irqsave(&gpiommgpio->lock, flags);
+
+	/* ensure that GPIO is set for input */
+	if (!(gpiommgpio->io_state[port] & mask)) {
+		spin_unlock_irqrestore(&gpiommgpio->lock, flags);
+		return -EINVAL;
+	}
+
+	port_state = inb(gpiommgpio->base + in_port);
+
+	spin_unlock_irqrestore(&gpiommgpio->lock, flags);
+
+	return !!(port_state & mask);
+}
+
+static void gpiomm_gpio_set(struct gpio_chip *chip, unsigned int offset,
+	int value)
+{
+	struct gpiomm_gpio *const gpiommgpio = gpiochip_get_data(chip);
+	const unsigned int port = offset / 8;
+	const unsigned int mask = BIT(offset % 8);
+	const unsigned int out_port = (port > 2) ? port + 1 : port;
+	unsigned long flags;
+
+	spin_lock_irqsave(&gpiommgpio->lock, flags);
+
+	if (value)
+		gpiommgpio->out_state[port] |= mask;
+	else
+		gpiommgpio->out_state[port] &= ~mask;
+
+	outb(gpiommgpio->out_state[port], gpiommgpio->base + out_port);
+
+	spin_unlock_irqrestore(&gpiommgpio->lock, flags);
+}
+
+static int gpiomm_probe(struct device *dev, unsigned int id)
+{
+	struct gpiomm_gpio *gpiommgpio;
+	const char *const name = dev_name(dev);
+	int err;
+
+	gpiommgpio = devm_kzalloc(dev, sizeof(*gpiommgpio), GFP_KERNEL);
+	if (!gpiommgpio)
+		return -ENOMEM;
+
+	if (!devm_request_region(dev, base[id], GPIOMM_EXTENT, name)) {
+		dev_err(dev, "Unable to lock port addresses (0x%X-0x%X)\n",
+			base[id], base[id] + GPIOMM_EXTENT);
+		return -EBUSY;
+	}
+
+	gpiommgpio->chip.label = name;
+	gpiommgpio->chip.parent = dev;
+	gpiommgpio->chip.owner = THIS_MODULE;
+	gpiommgpio->chip.base = -1;
+	gpiommgpio->chip.ngpio = 48;
+	gpiommgpio->chip.get_direction = gpiomm_gpio_get_direction;
+	gpiommgpio->chip.direction_input = gpiomm_gpio_direction_input;
+	gpiommgpio->chip.direction_output = gpiomm_gpio_direction_output;
+	gpiommgpio->chip.get = gpiomm_gpio_get;
+	gpiommgpio->chip.set = gpiomm_gpio_set;
+	gpiommgpio->base = base[id];
+
+	spin_lock_init(&gpiommgpio->lock);
+
+	dev_set_drvdata(dev, gpiommgpio);
+
+	err = gpiochip_add_data(&gpiommgpio->chip, gpiommgpio);
+	if (err) {
+		dev_err(dev, "GPIO registering failed (%d)\n", err);
+		return err;
+	}
+
+	/* initialize all GPIO as output */
+	outb(0x80, base[id] + 3);
+	outb(0x00, base[id]);
+	outb(0x00, base[id] + 1);
+	outb(0x00, base[id] + 2);
+	outb(0x80, base[id] + 7);
+	outb(0x00, base[id] + 4);
+	outb(0x00, base[id] + 5);
+	outb(0x00, base[id] + 6);
+
+	return 0;
+}
+
+static int gpiomm_remove(struct device *dev, unsigned int id)
+{
+	struct gpiomm_gpio *const gpiommgpio = dev_get_drvdata(dev);
+
+	gpiochip_remove(&gpiommgpio->chip);
+
+	return 0;
+}
+
+static struct isa_driver gpiomm_driver = {
+	.probe = gpiomm_probe,
+	.driver = {
+		.name = "gpio-mm"
+	},
+	.remove = gpiomm_remove
+};
+
+module_isa_driver(gpiomm_driver, num_gpiomm);
+
+MODULE_AUTHOR("William Breathitt Gray <vilhelm.gray@gmail.com>");
+MODULE_DESCRIPTION("Diamond Systems GPIO-MM GPIO driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/htc-egpio.c b/drivers/gpio/gpio-htc-egpio.c
index 513cfc5c8fb6..0b4df6051097 100644
--- a/drivers/mfd/htc-egpio.c
+++ b/drivers/gpio/gpio-htc-egpio.c
@@ -14,10 +14,10 @@
 #include <linux/irq.h>
 #include <linux/io.h>
 #include <linux/spinlock.h>
+#include <linux/platform_data/gpio-htc-egpio.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-#include <linux/mfd/htc-egpio.h>
 
 struct egpio_chip {
 	int              reg_start;
diff --git a/drivers/gpio/gpio-iop.c b/drivers/gpio/gpio-iop.c
index 860c535922fd..98c7ff2a76e7 100644
--- a/drivers/gpio/gpio-iop.c
+++ b/drivers/gpio/gpio-iop.c
@@ -10,111 +10,40 @@
  * your option) any later version.
  */
 
-#include <linux/device.h>
-#include <linux/init.h>
-#include <linux/types.h>
-#include <linux/errno.h>
-#include <linux/gpio.h>
-#include <linux/export.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/gpio/driver.h>
 #include <linux/platform_device.h>
-#include <linux/bitops.h>
-#include <linux/io.h>
 
-#define IOP3XX_N_GPIOS	8
-
-#define GPIO_IN			0
-#define GPIO_OUT		1
-#define GPIO_LOW		0
-#define GPIO_HIGH		1
-
-/* Memory base offset */
-static void __iomem *base;
-
-#define IOP3XX_GPIO_REG(reg)	(base + (reg))
-#define IOP3XX_GPOE		IOP3XX_GPIO_REG(0x0000)
-#define IOP3XX_GPID		IOP3XX_GPIO_REG(0x0004)
-#define IOP3XX_GPOD		IOP3XX_GPIO_REG(0x0008)
-
-static void gpio_line_config(int line, int direction)
-{
-	unsigned long flags;
-	u32 val;
-
-	local_irq_save(flags);
-	val = readl(IOP3XX_GPOE);
-	if (direction == GPIO_IN) {
-		val |= BIT(line);
-	} else if (direction == GPIO_OUT) {
-		val &= ~BIT(line);
-	}
-	writel(val, IOP3XX_GPOE);
-	local_irq_restore(flags);
-}
-
-static int gpio_line_get(int line)
-{
-	return !!(readl(IOP3XX_GPID) & BIT(line));
-}
-
-static void gpio_line_set(int line, int value)
-{
-	unsigned long flags;
-	u32 val;
-
-	local_irq_save(flags);
-	val = readl(IOP3XX_GPOD);
-	if (value == GPIO_LOW) {
-		val &= ~BIT(line);
-	} else if (value == GPIO_HIGH) {
-		val |= BIT(line);
-	}
-	writel(val, IOP3XX_GPOD);
-	local_irq_restore(flags);
-}
-
-static int iop3xx_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
-{
-	gpio_line_config(gpio, GPIO_IN);
-	return 0;
-}
-
-static int iop3xx_gpio_direction_output(struct gpio_chip *chip, unsigned gpio, int level)
-{
-	gpio_line_set(gpio, level);
-	gpio_line_config(gpio, GPIO_OUT);
-	return 0;
-}
-
-static int iop3xx_gpio_get_value(struct gpio_chip *chip, unsigned gpio)
-{
-	return gpio_line_get(gpio);
-}
-
-static void iop3xx_gpio_set_value(struct gpio_chip *chip, unsigned gpio, int value)
-{
-	gpio_line_set(gpio, value);
-}
-
-static struct gpio_chip iop3xx_chip = {
-	.label			= "iop3xx",
-	.direction_input	= iop3xx_gpio_direction_input,
-	.get			= iop3xx_gpio_get_value,
-	.direction_output	= iop3xx_gpio_direction_output,
-	.set			= iop3xx_gpio_set_value,
-	.base			= 0,
-	.ngpio			= IOP3XX_N_GPIOS,
-};
+#define IOP3XX_GPOE	0x0000
+#define IOP3XX_GPID	0x0004
+#define IOP3XX_GPOD	0x0008
 
 static int iop3xx_gpio_probe(struct platform_device *pdev)
 {
 	struct resource *res;
+	struct gpio_chip *gc;
+	void __iomem *base;
+	int err;
+
+	gc = devm_kzalloc(&pdev->dev, sizeof(*gc), GFP_KERNEL);
+	if (!gc)
+		return -ENOMEM;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	return devm_gpiochip_add_data(&pdev->dev, &iop3xx_chip, NULL);
+	err = bgpio_init(gc, &pdev->dev, 1, base + IOP3XX_GPID,
+			 base + IOP3XX_GPOD, NULL, NULL, base + IOP3XX_GPOE, 0);
+	if (err)
+		return err;
+
+	gc->base = 0;
+	gc->owner = THIS_MODULE;
+
+	return devm_gpiochip_add_data(&pdev->dev, gc, NULL);
 }
 
 static struct platform_driver iop3xx_gpio_driver = {
diff --git a/drivers/gpio/gpio-it87.c b/drivers/gpio/gpio-it87.c
index 63a962d18cd6..45d29e488dbb 100644
--- a/drivers/gpio/gpio-it87.c
+++ b/drivers/gpio/gpio-it87.c
@@ -273,7 +273,7 @@ exit:
 	return rc;
 }
 
-static struct gpio_chip it87_template_chip = {
+static const struct gpio_chip it87_template_chip = {
 	.label			= KBUILD_MODNAME,
 	.owner			= THIS_MODULE,
 	.request		= it87_gpio_request,
diff --git a/drivers/gpio/gpio-loongson1.c b/drivers/gpio/gpio-loongson1.c
index 10c09bdd8514..72b64039241a 100644
--- a/drivers/gpio/gpio-loongson1.c
+++ b/drivers/gpio/gpio-loongson1.c
@@ -8,6 +8,7 @@
  * warranty of any kind, whether express or implied.
  */
 
+#include <linux/module.h>
 #include <linux/gpio/driver.h>
 #include <linux/platform_device.h>
 
@@ -55,11 +56,6 @@ static int ls1x_gpio_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(dev, "failed to get I/O memory\n");
-		return -EINVAL;
-	}
-
 	gpio_reg_base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(gpio_reg_base))
 		return PTR_ERR(gpio_reg_base);
diff --git a/drivers/gpio/gpio-lp873x.c b/drivers/gpio/gpio-lp873x.c
new file mode 100644
index 000000000000..218c706359aa
--- /dev/null
+++ b/drivers/gpio/gpio-lp873x.c
@@ -0,0 +1,183 @@
+/*
+ * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/
+ *	Keerthy <j-keerthy@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether expressed or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License version 2 for more details.
+ *
+ * Based on the TPS65218 driver
+ */
+
+#include <linux/gpio.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <linux/mfd/lp873x.h>
+
+#define BITS_PER_GPO		0x4
+#define LP873X_GPO_CTRL_OD	0x2
+
+struct lp873x_gpio {
+	struct gpio_chip chip;
+	struct lp873x *lp873;
+};
+
+static int lp873x_gpio_get_direction(struct gpio_chip *chip,
+				     unsigned int offset)
+{
+	/* This device is output only */
+	return 0;
+}
+
+static int lp873x_gpio_direction_input(struct gpio_chip *chip,
+				       unsigned int offset)
+{
+	/* This device is output only */
+	return -EINVAL;
+}
+
+static int lp873x_gpio_direction_output(struct gpio_chip *chip,
+					unsigned int offset, int value)
+{
+	struct lp873x_gpio *gpio = gpiochip_get_data(chip);
+
+	/* Set the initial value */
+	return regmap_update_bits(gpio->lp873->regmap, LP873X_REG_GPO_CTRL,
+				  BIT(offset * BITS_PER_GPO),
+				  value ? BIT(offset * BITS_PER_GPO) : 0);
+}
+
+static int lp873x_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct lp873x_gpio *gpio = gpiochip_get_data(chip);
+	int ret, val;
+
+	ret = regmap_read(gpio->lp873->regmap, LP873X_REG_GPO_CTRL, &val);
+	if (ret < 0)
+		return ret;
+
+	return val & BIT(offset * BITS_PER_GPO);
+}
+
+static void lp873x_gpio_set(struct gpio_chip *chip, unsigned int offset,
+			    int value)
+{
+	struct lp873x_gpio *gpio = gpiochip_get_data(chip);
+
+	regmap_update_bits(gpio->lp873->regmap, LP873X_REG_GPO_CTRL,
+			   BIT(offset * BITS_PER_GPO),
+			   value ? BIT(offset * BITS_PER_GPO) : 0);
+}
+
+static int lp873x_gpio_request(struct gpio_chip *gc, unsigned int offset)
+{
+	struct lp873x_gpio *gpio = gpiochip_get_data(gc);
+	int ret;
+
+	switch (offset) {
+	case 0:
+		/* No MUX Set up Needed for GPO */
+		break;
+	case 1:
+		/* Setup the CLKIN_PIN_SEL MUX to GPO2 */
+		ret = regmap_update_bits(gpio->lp873->regmap, LP873X_REG_CONFIG,
+					 LP873X_CONFIG_CLKIN_PIN_SEL, 0);
+		if (ret)
+			return ret;
+
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int lp873x_gpio_set_single_ended(struct gpio_chip *gc,
+					unsigned int offset,
+					enum single_ended_mode mode)
+{
+	struct lp873x_gpio *gpio = gpiochip_get_data(gc);
+
+	switch (mode) {
+	case LINE_MODE_OPEN_DRAIN:
+		return regmap_update_bits(gpio->lp873->regmap,
+					  LP873X_REG_GPO_CTRL,
+					  BIT(offset * BITS_PER_GPO +
+					  LP873X_GPO_CTRL_OD),
+					  BIT(offset * BITS_PER_GPO +
+					  LP873X_GPO_CTRL_OD));
+	case LINE_MODE_PUSH_PULL:
+		return regmap_update_bits(gpio->lp873->regmap,
+					  LP873X_REG_GPO_CTRL,
+					  BIT(offset * BITS_PER_GPO +
+					  LP873X_GPO_CTRL_OD), 0);
+	default:
+		return -ENOTSUPP;
+	}
+}
+
+static const struct gpio_chip template_chip = {
+	.label			= "lp873x-gpio",
+	.owner			= THIS_MODULE,
+	.request		= lp873x_gpio_request,
+	.get_direction		= lp873x_gpio_get_direction,
+	.direction_input	= lp873x_gpio_direction_input,
+	.direction_output	= lp873x_gpio_direction_output,
+	.get			= lp873x_gpio_get,
+	.set			= lp873x_gpio_set,
+	.set_single_ended	= lp873x_gpio_set_single_ended,
+	.base			= -1,
+	.ngpio			= 2,
+	.can_sleep		= true,
+};
+
+static int lp873x_gpio_probe(struct platform_device *pdev)
+{
+	struct lp873x_gpio *gpio;
+	int ret;
+
+	gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
+	if (!gpio)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, gpio);
+
+	gpio->lp873 = dev_get_drvdata(pdev->dev.parent);
+	gpio->chip = template_chip;
+	gpio->chip.parent = gpio->lp873->dev;
+
+	ret = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct platform_device_id lp873x_gpio_id_table[] = {
+	{ "lp873x-gpio", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(platform, lp873x_gpio_id_table);
+
+static struct platform_driver lp873x_gpio_driver = {
+	.driver = {
+		.name = "lp873x-gpio",
+	},
+	.probe = lp873x_gpio_probe,
+	.id_table = lp873x_gpio_id_table,
+};
+module_platform_driver(lp873x_gpio_driver);
+
+MODULE_AUTHOR("Keerthy <j-keerthy@ti.com>");
+MODULE_DESCRIPTION("LP873X GPIO driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-lpc18xx.c b/drivers/gpio/gpio-lpc18xx.c
index 98832c9f614a..f12e02e1016d 100644
--- a/drivers/gpio/gpio-lpc18xx.c
+++ b/drivers/gpio/gpio-lpc18xx.c
@@ -78,7 +78,7 @@ static int lpc18xx_gpio_direction_output(struct gpio_chip *chip,
 	return lpc18xx_gpio_direction(chip, offset, true);
 }
 
-static struct gpio_chip lpc18xx_chip = {
+static const struct gpio_chip lpc18xx_chip = {
 	.label			= "lpc18xx/43xx-gpio",
 	.request		= gpiochip_generic_request,
 	.free			= gpiochip_generic_free,
diff --git a/drivers/gpio/gpio-lpc32xx.c b/drivers/gpio/gpio-lpc32xx.c
index fc5f197906ac..92b3ae2a6735 100644
--- a/drivers/gpio/gpio-lpc32xx.c
+++ b/drivers/gpio/gpio-lpc32xx.c
@@ -25,7 +25,6 @@
 #include <linux/of_gpio.h>
 #include <linux/platform_device.h>
 #include <linux/module.h>
-#include <linux/platform_data/gpio-lpc32xx.h>
 
 #include <mach/hardware.h>
 #include <mach/platform.h>
@@ -68,6 +67,20 @@
 #define GPI3_PIN_IN_SEL(x, y)			(((x) >> (y)) & 1)
 #define GPO3_PIN_IN_SEL(x, y)			(((x) >> (y)) & 1)
 
+#define LPC32XX_GPIO_P0_MAX	8
+#define LPC32XX_GPIO_P1_MAX	24
+#define LPC32XX_GPIO_P2_MAX	13
+#define LPC32XX_GPIO_P3_MAX	6
+#define LPC32XX_GPI_P3_MAX	29
+#define LPC32XX_GPO_P3_MAX	24
+
+#define LPC32XX_GPIO_P0_GRP	0
+#define LPC32XX_GPIO_P1_GRP	(LPC32XX_GPIO_P0_GRP + LPC32XX_GPIO_P0_MAX)
+#define LPC32XX_GPIO_P2_GRP	(LPC32XX_GPIO_P1_GRP + LPC32XX_GPIO_P1_MAX)
+#define LPC32XX_GPIO_P3_GRP	(LPC32XX_GPIO_P2_GRP + LPC32XX_GPIO_P2_MAX)
+#define LPC32XX_GPI_P3_GRP	(LPC32XX_GPIO_P3_GRP + LPC32XX_GPIO_P3_MAX)
+#define LPC32XX_GPO_P3_GRP	(LPC32XX_GPI_P3_GRP + LPC32XX_GPI_P3_MAX)
+
 struct gpio_regs {
 	void __iomem *inp_state;
 	void __iomem *outp_state;
diff --git a/drivers/gpio/gpio-max730x.c b/drivers/gpio/gpio-max730x.c
index 08807368f007..946d09195598 100644
--- a/drivers/gpio/gpio-max730x.c
+++ b/drivers/gpio/gpio-max730x.c
@@ -192,6 +192,10 @@ int __max730x_probe(struct max7301 *ts)
 	ts->chip.parent = dev;
 	ts->chip.owner = THIS_MODULE;
 
+	ret = gpiochip_add_data(&ts->chip, ts);
+	if (ret)
+		goto exit_destroy;
+
 	/*
 	 * initialize pullups according to platform data and cache the
 	 * register values for later use.
@@ -213,10 +217,6 @@ int __max730x_probe(struct max7301 *ts)
 		}
 	}
 
-	ret = gpiochip_add_data(&ts->chip, ts);
-	if (ret)
-		goto exit_destroy;
-
 	return ret;
 
 exit_destroy:
diff --git a/drivers/gpio/gpio-mcp23s08.c b/drivers/gpio/gpio-mcp23s08.c
index ac22efc1840e..99d37b56c258 100644
--- a/drivers/gpio/gpio-mcp23s08.c
+++ b/drivers/gpio/gpio-mcp23s08.c
@@ -564,7 +564,7 @@ static int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev,
 	mcp->chip.direction_output = mcp23s08_direction_output;
 	mcp->chip.set = mcp23s08_set;
 	mcp->chip.dbg_show = mcp23s08_dbg_show;
-#ifdef CONFIG_OF
+#ifdef CONFIG_OF_GPIO
 	mcp->chip.of_gpio_n_cells = 2;
 	mcp->chip.of_node = dev->of_node;
 #endif
diff --git a/drivers/gpio/gpio-mmio.c b/drivers/gpio/gpio-mmio.c
index 6ec144baeb11..d7d03ad052d0 100644
--- a/drivers/gpio/gpio-mmio.c
+++ b/drivers/gpio/gpio-mmio.c
@@ -573,6 +573,7 @@ static void __iomem *bgpio_map(struct platform_device *pdev,
 
 #ifdef CONFIG_OF
 static const struct of_device_id bgpio_of_match[] = {
+	{ .compatible = "brcm,bcm6345-gpio" },
 	{ .compatible = "wd,mbl-gpio" },
 	{ }
 };
@@ -593,6 +594,9 @@ static struct bgpio_pdata *bgpio_parse_dt(struct platform_device *pdev,
 
 	pdata->base = -1;
 
+	if (of_device_is_big_endian(pdev->dev.of_node))
+		*flags |= BGPIOF_BIG_ENDIAN_BYTE_ORDER;
+
 	if (of_property_read_bool(pdev->dev.of_node, "no-output"))
 		*flags |= BGPIOF_NO_OUTPUT;
 
diff --git a/drivers/gpio/gpio-mockup.c b/drivers/gpio/gpio-mockup.c
new file mode 100644
index 000000000000..1ef85b0c2b1f
--- /dev/null
+++ b/drivers/gpio/gpio-mockup.c
@@ -0,0 +1,214 @@
+/*
+ * GPIO Testing Device Driver
+ *
+ * Copyright (C) 2014  Kamlakant Patel <kamlakant.patel@broadcom.com>
+ * Copyright (C) 2015-2016  Bamvor Jian Zhang <bamvor.zhangjian@linaro.org>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/gpio/driver.h>
+#include <linux/platform_device.h>
+
+#define GPIO_NAME	"gpio-mockup"
+#define	MAX_GC		10
+
+enum direction {
+	OUT,
+	IN
+};
+
+/*
+ * struct gpio_pin_status - structure describing a GPIO status
+ * @dir:       Configures direction of gpio as "in" or "out", 0=in, 1=out
+ * @value:     Configures status of the gpio as 0(low) or 1(high)
+ */
+struct gpio_pin_status {
+	enum direction dir;
+	bool value;
+};
+
+struct mockup_gpio_controller {
+	struct gpio_chip gc;
+	struct gpio_pin_status *stats;
+};
+
+static int gpio_mockup_ranges[MAX_GC << 1];
+static int gpio_mockup_params_nr;
+module_param_array(gpio_mockup_ranges, int, &gpio_mockup_params_nr, 0400);
+
+const char pins_name_start = 'A';
+
+static int mockup_gpio_get(struct gpio_chip *gc, unsigned int offset)
+{
+	struct mockup_gpio_controller *cntr = gpiochip_get_data(gc);
+
+	return cntr->stats[offset].value;
+}
+
+static void mockup_gpio_set(struct gpio_chip *gc, unsigned int offset,
+			    int value)
+{
+	struct mockup_gpio_controller *cntr = gpiochip_get_data(gc);
+
+	cntr->stats[offset].value = !!value;
+}
+
+static int mockup_gpio_dirout(struct gpio_chip *gc, unsigned int offset,
+			      int value)
+{
+	struct mockup_gpio_controller *cntr = gpiochip_get_data(gc);
+
+	mockup_gpio_set(gc, offset, value);
+	cntr->stats[offset].dir = OUT;
+	return 0;
+}
+
+static int mockup_gpio_dirin(struct gpio_chip *gc, unsigned int offset)
+{
+	struct mockup_gpio_controller *cntr = gpiochip_get_data(gc);
+
+	cntr->stats[offset].dir = IN;
+	return 0;
+}
+
+static int mockup_gpio_get_direction(struct gpio_chip *gc, unsigned int offset)
+{
+	struct mockup_gpio_controller *cntr = gpiochip_get_data(gc);
+
+	return cntr->stats[offset].dir;
+}
+
+static int mockup_gpio_add(struct device *dev,
+			   struct mockup_gpio_controller *cntr,
+			   const char *name, int base, int ngpio)
+{
+	int ret;
+
+	cntr->gc.base = base;
+	cntr->gc.ngpio = ngpio;
+	cntr->gc.label = name;
+	cntr->gc.owner = THIS_MODULE;
+	cntr->gc.parent = dev;
+	cntr->gc.get = mockup_gpio_get;
+	cntr->gc.set = mockup_gpio_set;
+	cntr->gc.direction_output = mockup_gpio_dirout;
+	cntr->gc.direction_input = mockup_gpio_dirin;
+	cntr->gc.get_direction = mockup_gpio_get_direction;
+	cntr->stats = devm_kzalloc(dev, sizeof(*cntr->stats) * cntr->gc.ngpio,
+				   GFP_KERNEL);
+	if (!cntr->stats) {
+		ret = -ENOMEM;
+		goto err;
+	}
+	ret = devm_gpiochip_add_data(dev, &cntr->gc, cntr);
+	if (ret)
+		goto err;
+
+	dev_info(dev, "gpio<%d..%d> add successful!", base, base + ngpio);
+	return 0;
+err:
+	dev_err(dev, "gpio<%d..%d> add failed!", base, base + ngpio);
+	return ret;
+}
+
+static int mockup_gpio_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mockup_gpio_controller *cntr;
+	int ret;
+	int i;
+	int base;
+	int ngpio;
+	char chip_name[sizeof(GPIO_NAME) + 3];
+
+	if (gpio_mockup_params_nr < 2)
+		return -EINVAL;
+
+	cntr = devm_kzalloc(dev, sizeof(*cntr) * (gpio_mockup_params_nr >> 1),
+			    GFP_KERNEL);
+	if (!cntr)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, cntr);
+
+	for (i = 0; i < gpio_mockup_params_nr >> 1; i++) {
+		base = gpio_mockup_ranges[i * 2];
+		if (base == -1)
+			ngpio = gpio_mockup_ranges[i * 2 + 1];
+		else
+			ngpio = gpio_mockup_ranges[i * 2 + 1] - base;
+
+		if (ngpio >= 0) {
+			sprintf(chip_name, "%s-%c", GPIO_NAME,
+				pins_name_start + i);
+			ret = mockup_gpio_add(dev, &cntr[i],
+					      chip_name, base, ngpio);
+		} else {
+			ret = -1;
+		}
+		if (ret) {
+			if (base < 0)
+				dev_err(dev, "gpio<%d..%d> add failed\n",
+					base, ngpio);
+			else
+				dev_err(dev, "gpio<%d..%d> add failed\n",
+					base, base + ngpio);
+
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static struct platform_driver mockup_gpio_driver = {
+	.driver = {
+		   .name = GPIO_NAME,
+		   },
+	.probe = mockup_gpio_probe,
+};
+
+static struct platform_device *pdev;
+static int __init mock_device_init(void)
+{
+	int err;
+
+	pdev = platform_device_alloc(GPIO_NAME, -1);
+	if (!pdev)
+		return -ENOMEM;
+
+	err = platform_device_add(pdev);
+	if (err) {
+		platform_device_put(pdev);
+		return err;
+	}
+
+	err = platform_driver_register(&mockup_gpio_driver);
+	if (err) {
+		platform_device_unregister(pdev);
+		return err;
+	}
+
+	return 0;
+}
+
+static void __exit mock_device_exit(void)
+{
+	platform_driver_unregister(&mockup_gpio_driver);
+	platform_device_unregister(pdev);
+}
+
+module_init(mock_device_init);
+module_exit(mock_device_exit);
+
+MODULE_AUTHOR("Kamlakant Patel <kamlakant.patel@broadcom.com>");
+MODULE_AUTHOR("Bamvor Jian Zhang <bamvor.zhangjian@linaro.org>");
+MODULE_DESCRIPTION("GPIO Testing driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-msic.c b/drivers/gpio/gpio-msic.c
index d75649787e6c..1b7ce7f85886 100644
--- a/drivers/gpio/gpio-msic.c
+++ b/drivers/gpio/gpio-msic.c
@@ -20,7 +20,6 @@
  *
  */
 
-#include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
@@ -328,9 +327,4 @@ static int __init platform_msic_gpio_init(void)
 {
 	return platform_driver_register(&platform_msic_gpio_driver);
 }
-
 subsys_initcall(platform_msic_gpio_init);
-
-MODULE_AUTHOR("Mathias Nyman <mathias.nyman@linux.intel.com>");
-MODULE_DESCRIPTION("Intel Medfield MSIC GPIO driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c
index 1b342a3842c8..c1a1e00b8cb0 100644
--- a/drivers/gpio/gpio-mxc.c
+++ b/drivers/gpio/gpio-mxc.c
@@ -2,7 +2,8 @@
  * MXC GPIO support. (c) 2008 Daniel Mack <daniel@caiaq.de>
  * Copyright 2008 Juergen Beisert, kernel@pengutronix.de
  *
- * Based on code from Freescale,
+ * Based on code from Freescale Semiconductor,
+ * Authors: Daniel Mack, Juergen Beisert.
  * Copyright (C) 2004-2010 Freescale Semiconductor, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -33,7 +34,6 @@
 #include <linux/gpio.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
-#include <linux/module.h>
 #include <linux/bug.h>
 
 enum mxc_gpio_hwtype {
@@ -458,6 +458,11 @@ static int mxc_gpio_probe(struct platform_device *pdev)
 	if (err)
 		goto out_bgio;
 
+	if (of_property_read_bool(np, "gpio-ranges")) {
+		port->gc.request = gpiochip_generic_request;
+		port->gc.free = gpiochip_generic_free;
+	}
+
 	port->gc.to_irq = mxc_gpio_to_irq;
 	port->gc.base = (pdev->id < 0) ? of_alias_get_id(np, "gpio") * 32 :
 					     pdev->id * 32;
@@ -510,10 +515,4 @@ static int __init gpio_mxc_init(void)
 {
 	return platform_driver_register(&mxc_gpio_driver);
 }
-postcore_initcall(gpio_mxc_init);
-
-MODULE_AUTHOR("Freescale Semiconductor, "
-	      "Daniel Mack <danielncaiaq.de>, "
-	      "Juergen Beisert <kernel@pengutronix.de>");
-MODULE_DESCRIPTION("Freescale MXC GPIO");
-MODULE_LICENSE("GPL");
+subsys_initcall(gpio_mxc_init);
diff --git a/drivers/gpio/gpio-palmas.c b/drivers/gpio/gpio-palmas.c
index 839474430229..3d818195e351 100644
--- a/drivers/gpio/gpio-palmas.c
+++ b/drivers/gpio/gpio-palmas.c
@@ -152,7 +152,6 @@ static const struct of_device_id of_palmas_gpio_match[] = {
 	{ .compatible = "ti,tps80036-gpio", .data = &tps80036_dev_data,},
 	{ },
 };
-MODULE_DEVICE_TABLE(of, of_palmas_gpio_match);
 
 static int palmas_gpio_probe(struct platform_device *pdev)
 {
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index 02f2a5621bb0..45c8817d068c 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -21,6 +21,7 @@
 #include <asm/unaligned.h>
 #include <linux/of_platform.h>
 #include <linux/acpi.h>
+#include <linux/regulator/consumer.h>
 
 #define PCA953X_INPUT		0
 #define PCA953X_OUTPUT		1
@@ -94,6 +95,24 @@ MODULE_DEVICE_TABLE(acpi, pca953x_acpi_ids);
 
 #define NBANK(chip) DIV_ROUND_UP(chip->gpio_chip.ngpio, BANK_SZ)
 
+struct pca953x_reg_config {
+	int direction;
+	int output;
+	int input;
+};
+
+static const struct pca953x_reg_config pca953x_regs = {
+	.direction = PCA953X_DIRECTION,
+	.output = PCA953X_OUTPUT,
+	.input = PCA953X_INPUT,
+};
+
+static const struct pca953x_reg_config pca957x_regs = {
+	.direction = PCA957X_CFG,
+	.output = PCA957X_OUT,
+	.input = PCA957X_IN,
+};
+
 struct pca953x_chip {
 	unsigned gpio_start;
 	u8 reg_output[MAX_BANK];
@@ -111,8 +130,13 @@ struct pca953x_chip {
 	struct i2c_client *client;
 	struct gpio_chip gpio_chip;
 	const char *const *names;
-	int	chip_type;
 	unsigned long driver_data;
+	struct regulator *regulator;
+
+	const struct pca953x_reg_config *regs;
+
+	int (*write_regs)(struct pca953x_chip *, int, u8 *);
+	int (*read_regs)(struct pca953x_chip *, int, u8 *);
 };
 
 static int pca953x_read_single(struct pca953x_chip *chip, int reg, u32 *val,
@@ -152,38 +176,44 @@ static int pca953x_write_single(struct pca953x_chip *chip, int reg, u32 val,
 	return 0;
 }
 
-static int pca953x_write_regs(struct pca953x_chip *chip, int reg, u8 *val)
+static int pca953x_write_regs_8(struct pca953x_chip *chip, int reg, u8 *val)
 {
-	int ret = 0;
+	return i2c_smbus_write_byte_data(chip->client, reg, *val);
+}
 
-	if (chip->gpio_chip.ngpio <= 8)
-		ret = i2c_smbus_write_byte_data(chip->client, reg, *val);
-	else if (chip->gpio_chip.ngpio >= 24) {
-		int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ);
-		ret = i2c_smbus_write_i2c_block_data(chip->client,
-					(reg << bank_shift) | REG_ADDR_AI,
-					NBANK(chip), val);
-	} else {
-		switch (chip->chip_type) {
-		case PCA953X_TYPE: {
-			__le16 word = cpu_to_le16(get_unaligned((u16 *)val));
+static int pca953x_write_regs_16(struct pca953x_chip *chip, int reg, u8 *val)
+{
+	__le16 word = cpu_to_le16(get_unaligned((u16 *)val));
 
-			ret = i2c_smbus_write_word_data(chip->client, reg << 1,
-							(__force u16)word);
-			break;
-		}
-		case PCA957X_TYPE:
-			ret = i2c_smbus_write_byte_data(chip->client, reg << 1,
-							val[0]);
-			if (ret < 0)
-				break;
-			ret = i2c_smbus_write_byte_data(chip->client,
-							(reg << 1) + 1,
-							val[1]);
-			break;
-		}
-	}
+	return i2c_smbus_write_word_data(chip->client,
+					 reg << 1, (__force u16)word);
+}
+
+static int pca957x_write_regs_16(struct pca953x_chip *chip, int reg, u8 *val)
+{
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(chip->client, reg << 1, val[0]);
+	if (ret < 0)
+		return ret;
+
+	return i2c_smbus_write_byte_data(chip->client, (reg << 1) + 1, val[1]);
+}
+
+static int pca953x_write_regs_24(struct pca953x_chip *chip, int reg, u8 *val)
+{
+	int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ);
+
+	return i2c_smbus_write_i2c_block_data(chip->client,
+					      (reg << bank_shift) | REG_ADDR_AI,
+					      NBANK(chip), val);
+}
 
+static int pca953x_write_regs(struct pca953x_chip *chip, int reg, u8 *val)
+{
+	int ret = 0;
+
+	ret = chip->write_regs(chip, reg, val);
 	if (ret < 0) {
 		dev_err(&chip->client->dev, "failed writing register\n");
 		return ret;
@@ -192,24 +222,41 @@ static int pca953x_write_regs(struct pca953x_chip *chip, int reg, u8 *val)
 	return 0;
 }
 
-static int pca953x_read_regs(struct pca953x_chip *chip, int reg, u8 *val)
+static int pca953x_read_regs_8(struct pca953x_chip *chip, int reg, u8 *val)
 {
 	int ret;
 
-	if (chip->gpio_chip.ngpio <= 8) {
-		ret = i2c_smbus_read_byte_data(chip->client, reg);
-		*val = ret;
-	} else if (chip->gpio_chip.ngpio >= 24) {
-		int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ);
+	ret = i2c_smbus_read_byte_data(chip->client, reg);
+	*val = ret;
 
-		ret = i2c_smbus_read_i2c_block_data(chip->client,
-					(reg << bank_shift) | REG_ADDR_AI,
-					NBANK(chip), val);
-	} else {
-		ret = i2c_smbus_read_word_data(chip->client, reg << 1);
-		val[0] = (u16)ret & 0xFF;
-		val[1] = (u16)ret >> 8;
-	}
+	return ret;
+}
+
+static int pca953x_read_regs_16(struct pca953x_chip *chip, int reg, u8 *val)
+{
+	int ret;
+
+	ret = i2c_smbus_read_word_data(chip->client, reg << 1);
+	val[0] = (u16)ret & 0xFF;
+	val[1] = (u16)ret >> 8;
+
+	return ret;
+}
+
+static int pca953x_read_regs_24(struct pca953x_chip *chip, int reg, u8 *val)
+{
+	int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ);
+
+	return i2c_smbus_read_i2c_block_data(chip->client,
+					     (reg << bank_shift) | REG_ADDR_AI,
+					     NBANK(chip), val);
+}
+
+static int pca953x_read_regs(struct pca953x_chip *chip, int reg, u8 *val)
+{
+	int ret;
+
+	ret = chip->read_regs(chip, reg, val);
 	if (ret < 0) {
 		dev_err(&chip->client->dev, "failed reading register\n");
 		return ret;
@@ -222,20 +269,12 @@ static int pca953x_gpio_direction_input(struct gpio_chip *gc, unsigned off)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u8 reg_val;
-	int ret, offset = 0;
+	int ret;
 
 	mutex_lock(&chip->i2c_lock);
 	reg_val = chip->reg_direction[off / BANK_SZ] | (1u << (off % BANK_SZ));
 
-	switch (chip->chip_type) {
-	case PCA953X_TYPE:
-		offset = PCA953X_DIRECTION;
-		break;
-	case PCA957X_TYPE:
-		offset = PCA957X_CFG;
-		break;
-	}
-	ret = pca953x_write_single(chip, offset, reg_val, off);
+	ret = pca953x_write_single(chip, chip->regs->direction, reg_val, off);
 	if (ret)
 		goto exit;
 
@@ -250,7 +289,7 @@ static int pca953x_gpio_direction_output(struct gpio_chip *gc,
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u8 reg_val;
-	int ret, offset = 0;
+	int ret;
 
 	mutex_lock(&chip->i2c_lock);
 	/* set output level */
@@ -261,15 +300,7 @@ static int pca953x_gpio_direction_output(struct gpio_chip *gc,
 		reg_val = chip->reg_output[off / BANK_SZ]
 			& ~(1u << (off % BANK_SZ));
 
-	switch (chip->chip_type) {
-	case PCA953X_TYPE:
-		offset = PCA953X_OUTPUT;
-		break;
-	case PCA957X_TYPE:
-		offset = PCA957X_OUT;
-		break;
-	}
-	ret = pca953x_write_single(chip, offset, reg_val, off);
+	ret = pca953x_write_single(chip, chip->regs->output, reg_val, off);
 	if (ret)
 		goto exit;
 
@@ -277,15 +308,7 @@ static int pca953x_gpio_direction_output(struct gpio_chip *gc,
 
 	/* then direction */
 	reg_val = chip->reg_direction[off / BANK_SZ] & ~(1u << (off % BANK_SZ));
-	switch (chip->chip_type) {
-	case PCA953X_TYPE:
-		offset = PCA953X_DIRECTION;
-		break;
-	case PCA957X_TYPE:
-		offset = PCA957X_CFG;
-		break;
-	}
-	ret = pca953x_write_single(chip, offset, reg_val, off);
+	ret = pca953x_write_single(chip, chip->regs->direction, reg_val, off);
 	if (ret)
 		goto exit;
 
@@ -299,18 +322,10 @@ static int pca953x_gpio_get_value(struct gpio_chip *gc, unsigned off)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u32 reg_val;
-	int ret, offset = 0;
+	int ret;
 
 	mutex_lock(&chip->i2c_lock);
-	switch (chip->chip_type) {
-	case PCA953X_TYPE:
-		offset = PCA953X_INPUT;
-		break;
-	case PCA957X_TYPE:
-		offset = PCA957X_IN;
-		break;
-	}
-	ret = pca953x_read_single(chip, offset, &reg_val, off);
+	ret = pca953x_read_single(chip, chip->regs->input, &reg_val, off);
 	mutex_unlock(&chip->i2c_lock);
 	if (ret < 0) {
 		/* NOTE:  diagnostic already emitted; that's all we should
@@ -327,7 +342,7 @@ static void pca953x_gpio_set_value(struct gpio_chip *gc, unsigned off, int val)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u8 reg_val;
-	int ret, offset = 0;
+	int ret;
 
 	mutex_lock(&chip->i2c_lock);
 	if (val)
@@ -337,15 +352,7 @@ static void pca953x_gpio_set_value(struct gpio_chip *gc, unsigned off, int val)
 		reg_val = chip->reg_output[off / BANK_SZ]
 			& ~(1u << (off % BANK_SZ));
 
-	switch (chip->chip_type) {
-	case PCA953X_TYPE:
-		offset = PCA953X_OUTPUT;
-		break;
-	case PCA957X_TYPE:
-		offset = PCA957X_OUT;
-		break;
-	}
-	ret = pca953x_write_single(chip, offset, reg_val, off);
+	ret = pca953x_write_single(chip, chip->regs->output, reg_val, off);
 	if (ret)
 		goto exit;
 
@@ -355,35 +362,31 @@ exit:
 }
 
 static void pca953x_gpio_set_multiple(struct gpio_chip *gc,
-		unsigned long *mask, unsigned long *bits)
+				      unsigned long *mask, unsigned long *bits)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
+	unsigned int bank_mask, bank_val;
+	int bank_shift, bank;
 	u8 reg_val[MAX_BANK];
-	int ret, offset = 0;
-	int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ);
-	int bank;
-
-	switch (chip->chip_type) {
-	case PCA953X_TYPE:
-		offset = PCA953X_OUTPUT;
-		break;
-	case PCA957X_TYPE:
-		offset = PCA957X_OUT;
-		break;
-	}
+	int ret;
+
+	bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ);
 
 	memcpy(reg_val, chip->reg_output, NBANK(chip));
 	mutex_lock(&chip->i2c_lock);
-	for(bank=0; bank<NBANK(chip); bank++) {
-		unsigned bankmask = mask[bank / sizeof(*mask)] >>
-				    ((bank % sizeof(*mask)) * 8);
-		if(bankmask) {
-			unsigned bankval  = bits[bank / sizeof(*bits)] >>
-					    ((bank % sizeof(*bits)) * 8);
-			reg_val[bank] = (reg_val[bank] & ~bankmask) | bankval;
+	for (bank = 0; bank < NBANK(chip); bank++) {
+		bank_mask = mask[bank / sizeof(*mask)] >>
+			   ((bank % sizeof(*mask)) * 8);
+		if (bank_mask) {
+			bank_val = bits[bank / sizeof(*bits)] >>
+				  ((bank % sizeof(*bits)) * 8);
+			reg_val[bank] = (reg_val[bank] & ~bank_mask) | bank_val;
 		}
 	}
-	ret = i2c_smbus_write_i2c_block_data(chip->client, offset << bank_shift, NBANK(chip), reg_val);
+
+	ret = i2c_smbus_write_i2c_block_data(chip->client,
+					     chip->regs->output << bank_shift,
+					     NBANK(chip), reg_val);
 	if (ret)
 		goto exit;
 
@@ -515,7 +518,7 @@ static bool pca953x_irq_pending(struct pca953x_chip *chip, u8 *pending)
 	bool pending_seen = false;
 	bool trigger_seen = false;
 	u8 trigger[MAX_BANK];
-	int ret, i, offset = 0;
+	int ret, i;
 
 	if (chip->driver_data & PCA_PCAL) {
 		/* Read the current interrupt status from the device */
@@ -540,15 +543,7 @@ static bool pca953x_irq_pending(struct pca953x_chip *chip, u8 *pending)
 		return pending_seen;
 	}
 
-	switch (chip->chip_type) {
-	case PCA953X_TYPE:
-		offset = PCA953X_INPUT;
-		break;
-	case PCA957X_TYPE:
-		offset = PCA957X_IN;
-		break;
-	}
-	ret = pca953x_read_regs(chip, offset, cur_stat);
+	ret = pca953x_read_regs(chip, chip->regs->input, cur_stat);
 	if (ret)
 		return false;
 
@@ -608,20 +603,13 @@ static int pca953x_irq_setup(struct pca953x_chip *chip,
 			     int irq_base)
 {
 	struct i2c_client *client = chip->client;
-	int ret, i, offset = 0;
+	int ret, i;
 
 	if (client->irq && irq_base != -1
 			&& (chip->driver_data & PCA_INT)) {
 
-		switch (chip->chip_type) {
-		case PCA953X_TYPE:
-			offset = PCA953X_INPUT;
-			break;
-		case PCA957X_TYPE:
-			offset = PCA957X_IN;
-			break;
-		}
-		ret = pca953x_read_regs(chip, offset, chip->irq_stat);
+		ret = pca953x_read_regs(chip,
+					chip->regs->input, chip->irq_stat);
 		if (ret)
 			return ret;
 
@@ -684,12 +672,14 @@ static int device_pca953x_init(struct pca953x_chip *chip, u32 invert)
 	int ret;
 	u8 val[MAX_BANK];
 
-	ret = pca953x_read_regs(chip, PCA953X_OUTPUT, chip->reg_output);
+	chip->regs = &pca953x_regs;
+
+	ret = pca953x_read_regs(chip, chip->regs->output, chip->reg_output);
 	if (ret)
 		goto out;
 
-	ret = pca953x_read_regs(chip, PCA953X_DIRECTION,
-			       chip->reg_direction);
+	ret = pca953x_read_regs(chip, chip->regs->direction,
+				chip->reg_direction);
 	if (ret)
 		goto out;
 
@@ -709,10 +699,13 @@ static int device_pca957x_init(struct pca953x_chip *chip, u32 invert)
 	int ret;
 	u8 val[MAX_BANK];
 
-	ret = pca953x_read_regs(chip, PCA957X_OUT, chip->reg_output);
+	chip->regs = &pca957x_regs;
+
+	ret = pca953x_read_regs(chip, chip->regs->output, chip->reg_output);
 	if (ret)
 		goto out;
-	ret = pca953x_read_regs(chip, PCA957X_CFG, chip->reg_direction);
+	ret = pca953x_read_regs(chip, chip->regs->direction,
+				chip->reg_direction);
 	if (ret)
 		goto out;
 
@@ -739,13 +732,14 @@ out:
 static const struct of_device_id pca953x_dt_ids[];
 
 static int pca953x_probe(struct i2c_client *client,
-				   const struct i2c_device_id *id)
+				   const struct i2c_device_id *i2c_id)
 {
 	struct pca953x_platform_data *pdata;
 	struct pca953x_chip *chip;
 	int irq_base = 0;
 	int ret;
 	u32 invert = 0;
+	struct regulator *reg;
 
 	chip = devm_kzalloc(&client->dev,
 			sizeof(struct pca953x_chip), GFP_KERNEL);
@@ -765,47 +759,77 @@ static int pca953x_probe(struct i2c_client *client,
 
 	chip->client = client;
 
-	if (id) {
-		chip->driver_data = id->driver_data;
+	reg = devm_regulator_get(&client->dev, "vcc");
+	if (IS_ERR(reg)) {
+		ret = PTR_ERR(reg);
+		if (ret != -EPROBE_DEFER)
+			dev_err(&client->dev, "reg get err: %d\n", ret);
+		return ret;
+	}
+	ret = regulator_enable(reg);
+	if (ret) {
+		dev_err(&client->dev, "reg en err: %d\n", ret);
+		return ret;
+	}
+	chip->regulator = reg;
+
+	if (i2c_id) {
+		chip->driver_data = i2c_id->driver_data;
 	} else {
-		const struct acpi_device_id *id;
+		const struct acpi_device_id *acpi_id;
 		const struct of_device_id *match;
 
 		match = of_match_device(pca953x_dt_ids, &client->dev);
 		if (match) {
 			chip->driver_data = (int)(uintptr_t)match->data;
 		} else {
-			id = acpi_match_device(pca953x_acpi_ids, &client->dev);
-			if (!id)
-				return -ENODEV;
+			acpi_id = acpi_match_device(pca953x_acpi_ids, &client->dev);
+			if (!acpi_id) {
+				ret = -ENODEV;
+				goto err_exit;
+			}
 
-			chip->driver_data = id->driver_data;
+			chip->driver_data = acpi_id->driver_data;
 		}
 	}
 
-	chip->chip_type = PCA_CHIP_TYPE(chip->driver_data);
-
 	mutex_init(&chip->i2c_lock);
+	lockdep_set_subclass(&chip->i2c_lock,
+			     i2c_adapter_depth(client->adapter));
 
 	/* initialize cached registers from their original values.
 	 * we can't share this chip with another i2c master.
 	 */
 	pca953x_setup_gpio(chip, chip->driver_data & PCA_GPIO_MASK);
 
-	if (chip->chip_type == PCA953X_TYPE)
+	if (chip->gpio_chip.ngpio <= 8) {
+		chip->write_regs = pca953x_write_regs_8;
+		chip->read_regs = pca953x_read_regs_8;
+	} else if (chip->gpio_chip.ngpio >= 24) {
+		chip->write_regs = pca953x_write_regs_24;
+		chip->read_regs = pca953x_read_regs_24;
+	} else {
+		if (PCA_CHIP_TYPE(chip->driver_data) == PCA953X_TYPE)
+			chip->write_regs = pca953x_write_regs_16;
+		else
+			chip->write_regs = pca957x_write_regs_16;
+		chip->read_regs = pca953x_read_regs_16;
+	}
+
+	if (PCA_CHIP_TYPE(chip->driver_data) == PCA953X_TYPE)
 		ret = device_pca953x_init(chip, invert);
 	else
 		ret = device_pca957x_init(chip, invert);
 	if (ret)
-		return ret;
+		goto err_exit;
 
 	ret = devm_gpiochip_add_data(&client->dev, &chip->gpio_chip, chip);
 	if (ret)
-		return ret;
+		goto err_exit;
 
 	ret = pca953x_irq_setup(chip, irq_base);
 	if (ret)
-		return ret;
+		goto err_exit;
 
 	if (pdata && pdata->setup) {
 		ret = pdata->setup(client, chip->gpio_chip.base,
@@ -816,6 +840,10 @@ static int pca953x_probe(struct i2c_client *client,
 
 	i2c_set_clientdata(client, chip);
 	return 0;
+
+err_exit:
+	regulator_disable(chip->regulator);
+	return ret;
 }
 
 static int pca953x_remove(struct i2c_client *client)
@@ -827,14 +855,16 @@ static int pca953x_remove(struct i2c_client *client)
 	if (pdata && pdata->teardown) {
 		ret = pdata->teardown(client, chip->gpio_chip.base,
 				chip->gpio_chip.ngpio, pdata->context);
-		if (ret < 0) {
+		if (ret < 0)
 			dev_err(&client->dev, "%s failed, %d\n",
 					"teardown", ret);
-			return ret;
-		}
+	} else {
+		ret = 0;
 	}
 
-	return 0;
+	regulator_disable(chip->regulator);
+
+	return ret;
 }
 
 /* convenience to stop overlong match-table lines */
diff --git a/drivers/gpio/gpio-pisosr.c b/drivers/gpio/gpio-pisosr.c
index cb14b8d1d512..f5545049c187 100644
--- a/drivers/gpio/gpio-pisosr.c
+++ b/drivers/gpio/gpio-pisosr.c
@@ -90,7 +90,7 @@ static int pisosr_gpio_get(struct gpio_chip *chip, unsigned offset)
 	return (gpio->buffer[offset / 8] >> (offset % 8)) & 0x1;
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "pisosr-gpio",
 	.owner			= THIS_MODULE,
 	.get_direction		= pisosr_gpio_get_direction,
diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c
index b96e0b466f74..2be48f5eba36 100644
--- a/drivers/gpio/gpio-rcar.c
+++ b/drivers/gpio/gpio-rcar.c
@@ -348,6 +348,10 @@ static const struct of_device_id gpio_rcar_of_table[] = {
 		/* Gen3 GPIO is identical to Gen2. */
 		.data = &gpio_rcar_info_gen2,
 	}, {
+		.compatible = "renesas,gpio-r8a7796",
+		/* Gen3 GPIO is identical to Gen2. */
+		.data = &gpio_rcar_info_gen2,
+	}, {
 		.compatible = "renesas,gpio-rcar",
 		.data = &gpio_rcar_info_gen1,
 	}, {
diff --git a/drivers/gpio/gpio-sa1100.c b/drivers/gpio/gpio-sa1100.c
index 0c99e8fb9af3..8d8ee0ebf14c 100644
--- a/drivers/gpio/gpio-sa1100.c
+++ b/drivers/gpio/gpio-sa1100.c
@@ -155,7 +155,7 @@ static int sa1100_gpio_irqdomain_map(struct irq_domain *d,
 {
 	irq_set_chip_and_handler(irq, &sa1100_gpio_irq_chip,
 				 handle_edge_irq);
-	irq_set_noprobe(irq);
+	irq_set_probe(irq);
 
 	return 0;
 }
diff --git a/drivers/gpio/gpio-sch.c b/drivers/gpio/gpio-sch.c
index eb43ae4835c1..545004445846 100644
--- a/drivers/gpio/gpio-sch.c
+++ b/drivers/gpio/gpio-sch.c
@@ -138,7 +138,7 @@ static int sch_gpio_direction_out(struct gpio_chip *gc, unsigned gpio_num,
 	return 0;
 }
 
-static struct gpio_chip sch_gpio_chip = {
+static const struct gpio_chip sch_gpio_chip = {
 	.label			= "sch_gpio",
 	.owner			= THIS_MODULE,
 	.direction_input	= sch_gpio_direction_in,
diff --git a/drivers/gpio/gpio-spear-spics.c b/drivers/gpio/gpio-spear-spics.c
index 7ffd16495286..22267479ba68 100644
--- a/drivers/gpio/gpio-spear-spics.c
+++ b/drivers/gpio/gpio-spear-spics.c
@@ -12,7 +12,7 @@
 #include <linux/err.h>
 #include <linux/gpio.h>
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/types.h>
@@ -183,7 +183,6 @@ static const struct of_device_id spics_gpio_of_match[] = {
 	{ .compatible = "st,spear-spics-gpio" },
 	{}
 };
-MODULE_DEVICE_TABLE(of, spics_gpio_of_match);
 
 static struct platform_driver spics_gpio_driver = {
 	.probe = spics_gpio_probe,
@@ -198,7 +197,3 @@ static int __init spics_gpio_init(void)
 	return platform_driver_register(&spics_gpio_driver);
 }
 subsys_initcall(spics_gpio_init);
-
-MODULE_AUTHOR("Shiraz Hashim <shiraz.linux.kernel@gmail.com>");
-MODULE_DESCRIPTION("STMicroelectronics SPEAr SPI Chip Select Abstraction");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c
index f675132de10e..e7d422a6b90b 100644
--- a/drivers/gpio/gpio-stmpe.c
+++ b/drivers/gpio/gpio-stmpe.c
@@ -13,6 +13,7 @@
 #include <linux/of.h>
 #include <linux/mfd/stmpe.h>
 #include <linux/seq_file.h>
+#include <linux/bitops.h>
 
 /*
  * These registers are modified under the irq bus lock and cached to avoid
@@ -20,6 +21,8 @@
  */
 enum { REG_RE, REG_FE, REG_IE };
 
+enum { LSB, CSB, MSB };
+
 #define CACHE_NR_REGS	3
 /* No variant has more than 24 GPIOs */
 #define CACHE_NR_BANKS	(24 / 8)
@@ -39,8 +42,8 @@ static int stmpe_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
-	u8 reg = stmpe->regs[STMPE_IDX_GPMR_LSB] - (offset / 8);
-	u8 mask = 1 << (offset % 8);
+	u8 reg = stmpe->regs[STMPE_IDX_GPMR_LSB + (offset / 8)];
+	u8 mask = BIT(offset % 8);
 	int ret;
 
 	ret = stmpe_reg_read(stmpe, reg);
@@ -55,8 +58,8 @@ static void stmpe_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	int which = val ? STMPE_IDX_GPSR_LSB : STMPE_IDX_GPCR_LSB;
-	u8 reg = stmpe->regs[which] - (offset / 8);
-	u8 mask = 1 << (offset % 8);
+	u8 reg = stmpe->regs[which + (offset / 8)];
+	u8 mask = BIT(offset % 8);
 
 	/*
 	 * Some variants have single register for gpio set/clear functionality.
@@ -74,7 +77,7 @@ static int stmpe_gpio_get_direction(struct gpio_chip *chip,
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	u8 reg = stmpe->regs[STMPE_IDX_GPDR_LSB] - (offset / 8);
-	u8 mask = 1 << (offset % 8);
+	u8 mask = BIT(offset % 8);
 	int ret;
 
 	ret = stmpe_reg_read(stmpe, reg);
@@ -89,8 +92,8 @@ static int stmpe_gpio_direction_output(struct gpio_chip *chip,
 {
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
-	u8 reg = stmpe->regs[STMPE_IDX_GPDR_LSB] - (offset / 8);
-	u8 mask = 1 << (offset % 8);
+	u8 reg = stmpe->regs[STMPE_IDX_GPDR_LSB + (offset / 8)];
+	u8 mask = BIT(offset % 8);
 
 	stmpe_gpio_set(chip, offset, val);
 
@@ -102,8 +105,8 @@ static int stmpe_gpio_direction_input(struct gpio_chip *chip,
 {
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
-	u8 reg = stmpe->regs[STMPE_IDX_GPDR_LSB] - (offset / 8);
-	u8 mask = 1 << (offset % 8);
+	u8 reg = stmpe->regs[STMPE_IDX_GPDR_LSB + (offset / 8)];
+	u8 mask = BIT(offset % 8);
 
 	return stmpe_set_bits(stmpe, reg, mask, 0);
 }
@@ -113,13 +116,13 @@ static int stmpe_gpio_request(struct gpio_chip *chip, unsigned offset)
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 
-	if (stmpe_gpio->norequest_mask & (1 << offset))
+	if (stmpe_gpio->norequest_mask & BIT(offset))
 		return -EINVAL;
 
-	return stmpe_set_altfunc(stmpe, 1 << offset, STMPE_BLOCK_GPIO);
+	return stmpe_set_altfunc(stmpe, BIT(offset), STMPE_BLOCK_GPIO);
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "stmpe",
 	.owner			= THIS_MODULE,
 	.get_direction		= stmpe_gpio_get_direction,
@@ -137,13 +140,14 @@ static int stmpe_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
-	int mask = 1 << (offset % 8);
+	int mask = BIT(offset % 8);
 
 	if (type & IRQ_TYPE_LEVEL_LOW || type & IRQ_TYPE_LEVEL_HIGH)
 		return -EINVAL;
 
-	/* STMPE801 doesn't have RE and FE registers */
-	if (stmpe_gpio->stmpe->partnum == STMPE801)
+	/* STMPE801 and STMPE 1600 don't have RE and FE registers */
+	if (stmpe_gpio->stmpe->partnum == STMPE801 ||
+	    stmpe_gpio->stmpe->partnum == STMPE1600)
 		return 0;
 
 	if (type & IRQ_TYPE_EDGE_RISING)
@@ -173,17 +177,24 @@ static void stmpe_gpio_irq_sync_unlock(struct irq_data *d)
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	int num_banks = DIV_ROUND_UP(stmpe->num_gpios, 8);
-	static const u8 regmap[] = {
-		[REG_RE]	= STMPE_IDX_GPRER_LSB,
-		[REG_FE]	= STMPE_IDX_GPFER_LSB,
-		[REG_IE]	= STMPE_IDX_IEGPIOR_LSB,
+	static const u8 regmap[CACHE_NR_REGS][CACHE_NR_BANKS] = {
+		[REG_RE][LSB] = STMPE_IDX_GPRER_LSB,
+		[REG_RE][CSB] = STMPE_IDX_GPRER_CSB,
+		[REG_RE][MSB] = STMPE_IDX_GPRER_MSB,
+		[REG_FE][LSB] = STMPE_IDX_GPFER_LSB,
+		[REG_FE][CSB] = STMPE_IDX_GPFER_CSB,
+		[REG_FE][MSB] = STMPE_IDX_GPFER_MSB,
+		[REG_IE][LSB] = STMPE_IDX_IEGPIOR_LSB,
+		[REG_IE][CSB] = STMPE_IDX_IEGPIOR_CSB,
+		[REG_IE][MSB] = STMPE_IDX_IEGPIOR_MSB,
 	};
 	int i, j;
 
 	for (i = 0; i < CACHE_NR_REGS; i++) {
-		/* STMPE801 doesn't have RE and FE registers */
-		if ((stmpe->partnum == STMPE801) &&
-				(i != REG_IE))
+		/* STMPE801 and STMPE1600 don't have RE and FE registers */
+		if ((stmpe->partnum == STMPE801 ||
+		     stmpe->partnum == STMPE1600) &&
+		     (i != REG_IE))
 			continue;
 
 		for (j = 0; j < num_banks; j++) {
@@ -194,7 +205,7 @@ static void stmpe_gpio_irq_sync_unlock(struct irq_data *d)
 				continue;
 
 			stmpe_gpio->oldregs[i][j] = new;
-			stmpe_reg_write(stmpe, stmpe->regs[regmap[i]] - j, new);
+			stmpe_reg_write(stmpe, stmpe->regs[regmap[i][j]], new);
 		}
 	}
 
@@ -207,7 +218,7 @@ static void stmpe_gpio_irq_mask(struct irq_data *d)
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
-	int mask = 1 << (offset % 8);
+	int mask = BIT(offset % 8);
 
 	stmpe_gpio->regs[REG_IE][regoffset] &= ~mask;
 }
@@ -216,11 +227,21 @@ static void stmpe_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
+	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
-	int mask = 1 << (offset % 8);
+	int mask = BIT(offset % 8);
 
 	stmpe_gpio->regs[REG_IE][regoffset] |= mask;
+
+	/*
+	 * STMPE1600 workaround: to be able to get IRQ from pins,
+	 * a read must be done on GPMR register, or a write in
+	 * GPSR or GPCR registers
+	 */
+	if (stmpe->partnum == STMPE1600)
+		stmpe_reg_read(stmpe,
+			       stmpe->regs[STMPE_IDX_GPMR_LSB + regoffset]);
 }
 
 static void stmpe_dbg_show_one(struct seq_file *s,
@@ -230,10 +251,10 @@ static void stmpe_dbg_show_one(struct seq_file *s,
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	const char *label = gpiochip_is_requested(gc, offset);
-	int num_banks = DIV_ROUND_UP(stmpe->num_gpios, 8);
 	bool val = !!stmpe_gpio_get(gc, offset);
-	u8 dir_reg = stmpe->regs[STMPE_IDX_GPDR_LSB] - (offset / 8);
-	u8 mask = 1 << (offset % 8);
+	u8 bank = offset / 8;
+	u8 dir_reg = stmpe->regs[STMPE_IDX_GPDR_LSB + bank];
+	u8 mask = BIT(offset % 8);
 	int ret;
 	u8 dir;
 
@@ -247,39 +268,72 @@ static void stmpe_dbg_show_one(struct seq_file *s,
 			   gpio, label ?: "(none)",
 			   val ? "hi" : "lo");
 	} else {
-		u8 edge_det_reg = stmpe->regs[STMPE_IDX_GPEDR_MSB] + num_banks - 1 - (offset / 8);
-		u8 rise_reg = stmpe->regs[STMPE_IDX_GPRER_LSB] - (offset / 8);
-		u8 fall_reg = stmpe->regs[STMPE_IDX_GPFER_LSB] - (offset / 8);
-		u8 irqen_reg = stmpe->regs[STMPE_IDX_IEGPIOR_LSB] - (offset / 8);
-		bool edge_det;
-		bool rise;
-		bool fall;
+		u8 edge_det_reg;
+		u8 rise_reg;
+		u8 fall_reg;
+		u8 irqen_reg;
+
+		char *edge_det_values[] = {"edge-inactive",
+					   "edge-asserted",
+					   "not-supported"};
+		char *rise_values[] = {"no-rising-edge-detection",
+				       "rising-edge-detection",
+				       "not-supported"};
+		char *fall_values[] = {"no-falling-edge-detection",
+				       "falling-edge-detection",
+				       "not-supported"};
+		#define NOT_SUPPORTED_IDX 2
+		u8 edge_det = NOT_SUPPORTED_IDX;
+		u8 rise = NOT_SUPPORTED_IDX;
+		u8 fall = NOT_SUPPORTED_IDX;
 		bool irqen;
 
-		ret = stmpe_reg_read(stmpe, edge_det_reg);
-		if (ret < 0)
+		switch (stmpe->partnum) {
+		case STMPE610:
+		case STMPE811:
+		case STMPE1601:
+		case STMPE2401:
+		case STMPE2403:
+			edge_det_reg = stmpe->regs[STMPE_IDX_GPEDR_LSB + bank];
+			ret = stmpe_reg_read(stmpe, edge_det_reg);
+			if (ret < 0)
+				return;
+			edge_det = !!(ret & mask);
+
+		case STMPE1801:
+			rise_reg = stmpe->regs[STMPE_IDX_GPRER_LSB + bank];
+			fall_reg = stmpe->regs[STMPE_IDX_GPFER_LSB + bank];
+
+			ret = stmpe_reg_read(stmpe, rise_reg);
+			if (ret < 0)
+				return;
+			rise = !!(ret & mask);
+			ret = stmpe_reg_read(stmpe, fall_reg);
+			if (ret < 0)
+				return;
+			fall = !!(ret & mask);
+
+		case STMPE801:
+		case STMPE1600:
+			irqen_reg = stmpe->regs[STMPE_IDX_IEGPIOR_LSB + bank];
+			break;
+
+		default:
 			return;
-		edge_det = !!(ret & mask);
-		ret = stmpe_reg_read(stmpe, rise_reg);
-		if (ret < 0)
-			return;
-		rise = !!(ret & mask);
-		ret = stmpe_reg_read(stmpe, fall_reg);
-		if (ret < 0)
-			return;
-		fall = !!(ret & mask);
+		}
+
 		ret = stmpe_reg_read(stmpe, irqen_reg);
 		if (ret < 0)
 			return;
 		irqen = !!(ret & mask);
 
-		seq_printf(s, " gpio-%-3d (%-20.20s) in  %s %s %s%s%s",
+		seq_printf(s, " gpio-%-3d (%-20.20s) in  %s %13s %13s %25s %25s",
 			   gpio, label ?: "(none)",
 			   val ? "hi" : "lo",
-			   edge_det ? "edge-asserted" : "edge-inactive",
-			   irqen ? "IRQ-enabled" : "",
-			   rise ? " rising-edge-detection" : "",
-			   fall ? " falling-edge-detection" : "");
+			   edge_det_values[edge_det],
+			   irqen ? "IRQ-enabled" : "IRQ-disabled",
+			   rise_values[rise],
+			   fall_values[fall]);
 	}
 }
 
@@ -307,18 +361,32 @@ static irqreturn_t stmpe_gpio_irq(int irq, void *dev)
 {
 	struct stmpe_gpio *stmpe_gpio = dev;
 	struct stmpe *stmpe = stmpe_gpio->stmpe;
-	u8 statmsbreg = stmpe->regs[STMPE_IDX_ISGPIOR_MSB];
+	u8 statmsbreg;
 	int num_banks = DIV_ROUND_UP(stmpe->num_gpios, 8);
 	u8 status[num_banks];
 	int ret;
 	int i;
 
+	/*
+	 * the stmpe_block_read() call below, imposes to set statmsbreg
+	 * with the register located at the lowest address. As STMPE1600
+	 * variant is the only one which respect registers address's order
+	 * (LSB regs located at lowest address than MSB ones) whereas all
+	 * the others have a registers layout with MSB located before the
+	 * LSB regs.
+	 */
+	if (stmpe->partnum == STMPE1600)
+		statmsbreg = stmpe->regs[STMPE_IDX_ISGPIOR_LSB];
+	else
+		statmsbreg = stmpe->regs[STMPE_IDX_ISGPIOR_MSB];
+
 	ret = stmpe_block_read(stmpe, statmsbreg, num_banks, status);
 	if (ret < 0)
 		return IRQ_NONE;
 
 	for (i = 0; i < num_banks; i++) {
-		int bank = num_banks - i - 1;
+		int bank = (stmpe_gpio->stmpe->partnum == STMPE1600) ? i :
+			   num_banks - i - 1;
 		unsigned int enabled = stmpe_gpio->regs[REG_IE][bank];
 		unsigned int stat = status[i];
 
@@ -333,15 +401,21 @@ static irqreturn_t stmpe_gpio_irq(int irq, void *dev)
 							 line);
 
 			handle_nested_irq(child_irq);
-			stat &= ~(1 << bit);
+			stat &= ~BIT(bit);
 		}
 
-		stmpe_reg_write(stmpe, statmsbreg + i, status[i]);
-
-		/* Edge detect register is not present on 801 */
-		if (stmpe->partnum != STMPE801)
-			stmpe_reg_write(stmpe, stmpe->regs[STMPE_IDX_GPEDR_MSB]
-					+ i, status[i]);
+		/*
+		 * interrupt status register write has no effect on
+		 * 801/1801/1600, bits are cleared when read.
+		 * Edge detect register is not present on 801/1600/1801
+		 */
+		if (stmpe->partnum != STMPE801 || stmpe->partnum != STMPE1600 ||
+		    stmpe->partnum != STMPE1801) {
+			stmpe_reg_write(stmpe, statmsbreg + i, status[i]);
+			stmpe_reg_write(stmpe,
+					stmpe->regs[STMPE_IDX_GPEDR_LSB + i],
+					status[i]);
+		}
 	}
 
 	return IRQ_HANDLED;
@@ -376,6 +450,8 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
 
 	of_property_read_u32(np, "st,norequest-mask",
 			&stmpe_gpio->norequest_mask);
+	if (stmpe_gpio->norequest_mask)
+		stmpe_gpio->chip.irq_need_valid_mask = true;
 
 	if (irq < 0)
 		dev_info(&pdev->dev,
@@ -400,6 +476,14 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
 			dev_err(&pdev->dev, "unable to get irq: %d\n", ret);
 			goto out_disable;
 		}
+		if (stmpe_gpio->norequest_mask) {
+			int i;
+
+			/* Forbid unused lines to be mapped as IRQs */
+			for (i = 0; i < sizeof(u32); i++)
+				if (stmpe_gpio->norequest_mask & BIT(i))
+					clear_bit(i, stmpe_gpio->chip.irq_valid_mask);
+		}
 		ret =  gpiochip_irqchip_add(&stmpe_gpio->chip,
 					    &stmpe_gpio_irq_chip,
 					    0,
diff --git a/drivers/gpio/gpio-sx150x.c b/drivers/gpio/gpio-sx150x.c
index a177ebd921d5..af95de89db01 100644
--- a/drivers/gpio/gpio-sx150x.c
+++ b/drivers/gpio/gpio-sx150x.c
@@ -236,7 +236,6 @@ static const struct i2c_device_id sx150x_id[] = {
 	{"sx1502q", 3},
 	{}
 };
-MODULE_DEVICE_TABLE(i2c, sx150x_id);
 
 static const struct of_device_id sx150x_of_match[] = {
 	{ .compatible = "semtech,sx1508q" },
@@ -245,7 +244,6 @@ static const struct of_device_id sx150x_of_match[] = {
 	{ .compatible = "semtech,sx1502q" },
 	{},
 };
-MODULE_DEVICE_TABLE(of, sx150x_of_match);
 
 static s32 sx150x_i2c_write(struct i2c_client *client, u8 reg, u8 val)
 {
diff --git a/drivers/gpio/gpio-tc3589x.c b/drivers/gpio/gpio-tc3589x.c
index 8b3659352e49..5a5a6cb00eea 100644
--- a/drivers/gpio/gpio-tc3589x.c
+++ b/drivers/gpio/gpio-tc3589x.c
@@ -34,7 +34,7 @@ struct tc3589x_gpio {
 	u8 oldregs[CACHE_NR_REGS][CACHE_NR_BANKS];
 };
 
-static int tc3589x_gpio_get(struct gpio_chip *chip, unsigned offset)
+static int tc3589x_gpio_get(struct gpio_chip *chip, unsigned int offset)
 {
 	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
@@ -49,24 +49,24 @@ static int tc3589x_gpio_get(struct gpio_chip *chip, unsigned offset)
 	return !!(ret & mask);
 }
 
-static void tc3589x_gpio_set(struct gpio_chip *chip, unsigned offset, int val)
+static void tc3589x_gpio_set(struct gpio_chip *chip, unsigned int offset, int val)
 {
 	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
 	u8 reg = TC3589x_GPIODATA0 + (offset / 8) * 2;
-	unsigned pos = offset % 8;
+	unsigned int pos = offset % 8;
 	u8 data[] = {val ? BIT(pos) : 0, BIT(pos)};
 
 	tc3589x_block_write(tc3589x, reg, ARRAY_SIZE(data), data);
 }
 
 static int tc3589x_gpio_direction_output(struct gpio_chip *chip,
-					 unsigned offset, int val)
+					 unsigned int offset, int val)
 {
 	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
 	u8 reg = TC3589x_GPIODIR0 + offset / 8;
-	unsigned pos = offset % 8;
+	unsigned int pos = offset % 8;
 
 	tc3589x_gpio_set(chip, offset, val);
 
@@ -74,19 +74,35 @@ static int tc3589x_gpio_direction_output(struct gpio_chip *chip,
 }
 
 static int tc3589x_gpio_direction_input(struct gpio_chip *chip,
-					unsigned offset)
+					unsigned int offset)
 {
 	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
 	u8 reg = TC3589x_GPIODIR0 + offset / 8;
-	unsigned pos = offset % 8;
+	unsigned int pos = offset % 8;
 
 	return tc3589x_set_bits(tc3589x, reg, BIT(pos), 0);
 }
 
-static int tc3589x_gpio_single_ended(struct gpio_chip *chip,
-				     unsigned offset,
-				     enum single_ended_mode mode)
+static int tc3589x_gpio_get_direction(struct gpio_chip *chip,
+				      unsigned int offset)
+{
+	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
+	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
+	u8 reg = TC3589x_GPIODIR0 + offset / 8;
+	unsigned int pos = offset % 8;
+	int ret;
+
+	ret = tc3589x_reg_read(tc3589x, reg);
+	if (ret < 0)
+		return ret;
+
+	return !!(ret & BIT(pos));
+}
+
+static int tc3589x_gpio_set_single_ended(struct gpio_chip *chip,
+					 unsigned int offset,
+					 enum single_ended_mode mode)
 {
 	struct tc3589x_gpio *tc3589x_gpio = gpiochip_get_data(chip);
 	struct tc3589x *tc3589x = tc3589x_gpio->tc3589x;
@@ -97,7 +113,7 @@ static int tc3589x_gpio_single_ended(struct gpio_chip *chip,
 	 */
 	u8 odmreg = TC3589x_GPIOODM0 + (offset / 8) * 2;
 	u8 odereg = TC3589x_GPIOODE0 + (offset / 8) * 2;
-	unsigned pos = offset % 8;
+	unsigned int pos = offset % 8;
 	int ret;
 
 	switch(mode) {
@@ -124,14 +140,15 @@ static int tc3589x_gpio_single_ended(struct gpio_chip *chip,
 	return -ENOTSUPP;
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "tc3589x",
 	.owner			= THIS_MODULE,
-	.direction_input	= tc3589x_gpio_direction_input,
 	.get			= tc3589x_gpio_get,
-	.direction_output	= tc3589x_gpio_direction_output,
 	.set			= tc3589x_gpio_set,
-	.set_single_ended	= tc3589x_gpio_single_ended,
+	.direction_output	= tc3589x_gpio_direction_output,
+	.direction_input	= tc3589x_gpio_direction_input,
+	.get_direction		= tc3589x_gpio_get_direction,
+	.set_single_ended	= tc3589x_gpio_set_single_ended,
 	.can_sleep		= true,
 };
 
diff --git a/drivers/gpio/gpio-tpic2810.c b/drivers/gpio/gpio-tpic2810.c
index cace79c1b70a..c8b34d787eed 100644
--- a/drivers/gpio/gpio-tpic2810.c
+++ b/drivers/gpio/gpio-tpic2810.c
@@ -87,7 +87,7 @@ static void tpic2810_set_multiple(struct gpio_chip *chip, unsigned long *mask,
 	tpic2810_set_mask_bits(chip, *mask, *bits);
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "tpic2810",
 	.owner			= THIS_MODULE,
 	.get_direction		= tpic2810_get_direction,
diff --git a/drivers/gpio/gpio-tps65086.c b/drivers/gpio/gpio-tps65086.c
index 8e25f01ac314..b23c4d2429be 100644
--- a/drivers/gpio/gpio-tps65086.c
+++ b/drivers/gpio/gpio-tps65086.c
@@ -72,7 +72,7 @@ static void tps65086_gpio_set(struct gpio_chip *chip, unsigned offset,
 			   BIT(4 + offset), value ? BIT(4 + offset) : 0);
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "tps65086-gpio",
 	.owner			= THIS_MODULE,
 	.get_direction		= tps65086_gpio_get_direction,
diff --git a/drivers/gpio/gpio-tps65218.c b/drivers/gpio/gpio-tps65218.c
index 1c09a19ae10c..d779307a9685 100644
--- a/drivers/gpio/gpio-tps65218.c
+++ b/drivers/gpio/gpio-tps65218.c
@@ -172,7 +172,7 @@ static int tps65218_gpio_set_single_ended(struct gpio_chip *gc,
 	return -ENOTSUPP;
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "gpio-tps65218",
 	.owner			= THIS_MODULE,
 	.request		= tps65218_gpio_request,
@@ -204,7 +204,8 @@ static int tps65218_gpio_probe(struct platform_device *pdev)
 	tps65218_gpio->gpio_chip.of_node = pdev->dev.of_node;
 #endif
 
-	ret = gpiochip_add_data(&tps65218_gpio->gpio_chip, tps65218_gpio);
+	ret = devm_gpiochip_add_data(&pdev->dev, &tps65218_gpio->gpio_chip,
+				     tps65218_gpio);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register gpiochip, %d\n", ret);
 		return ret;
@@ -215,15 +216,6 @@ static int tps65218_gpio_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int tps65218_gpio_remove(struct platform_device *pdev)
-{
-	struct tps65218_gpio *tps65218_gpio = platform_get_drvdata(pdev);
-
-	gpiochip_remove(&tps65218_gpio->gpio_chip);
-
-	return 0;
-}
-
 static const struct of_device_id tps65218_dt_match[] = {
 	{ .compatible = "ti,tps65218-gpio" },
 	{  }
@@ -242,7 +234,6 @@ static struct platform_driver tps65218_gpio_driver = {
 		.of_match_table = of_match_ptr(tps65218_dt_match)
 	},
 	.probe = tps65218_gpio_probe,
-	.remove = tps65218_gpio_remove,
 	.id_table = tps65218_gpio_id_table,
 };
 
diff --git a/drivers/gpio/gpio-tps65912.c b/drivers/gpio/gpio-tps65912.c
index acfd30a13a56..abc0798ef843 100644
--- a/drivers/gpio/gpio-tps65912.c
+++ b/drivers/gpio/gpio-tps65912.c
@@ -90,7 +90,7 @@ static void tps65912_gpio_set(struct gpio_chip *gc, unsigned offset,
 			   GPIO_SET_MASK, value ? GPIO_SET_MASK : 0);
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "tps65912-gpio",
 	.owner			= THIS_MODULE,
 	.get_direction		= tps65912_gpio_get_direction,
diff --git a/drivers/gpio/gpio-ts4800.c b/drivers/gpio/gpio-ts4800.c
index 0c144a72f9af..99256115bea5 100644
--- a/drivers/gpio/gpio-ts4800.c
+++ b/drivers/gpio/gpio-ts4800.c
@@ -9,6 +9,7 @@
  */
 
 #include <linux/gpio/driver.h>
+#include <linux/module.h>
 #include <linux/of_address.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
diff --git a/drivers/gpio/gpio-ts4900.c b/drivers/gpio/gpio-ts4900.c
new file mode 100644
index 000000000000..5bd21725e604
--- /dev/null
+++ b/drivers/gpio/gpio-ts4900.c
@@ -0,0 +1,190 @@
+/*
+ * Digital I/O driver for Technologic Systems I2C FPGA Core
+ *
+ * Copyright (C) 2015 Technologic Systems
+ * Copyright (C) 2016 Savoir-Faire Linux
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether expressed or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License version 2 for more details.
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/i2c.h>
+#include <linux/of_device.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+#define DEFAULT_PIN_NUMBER	32
+/*
+ * Register bits used by the GPIO device
+ * Some boards, such as TS-7970 do not have a separate input bit
+ */
+#define TS4900_GPIO_OE		0x01
+#define TS4900_GPIO_OUT		0x02
+#define TS4900_GPIO_IN		0x04
+#define TS7970_GPIO_IN		0x02
+
+struct ts4900_gpio_priv {
+	struct regmap *regmap;
+	struct gpio_chip gpio_chip;
+	unsigned int input_bit;
+};
+
+static int ts4900_gpio_get_direction(struct gpio_chip *chip,
+				     unsigned int offset)
+{
+	struct ts4900_gpio_priv *priv = gpiochip_get_data(chip);
+	unsigned int reg;
+
+	regmap_read(priv->regmap, offset, &reg);
+
+	return !(reg & TS4900_GPIO_OE);
+}
+
+static int ts4900_gpio_direction_input(struct gpio_chip *chip,
+				       unsigned int offset)
+{
+	struct ts4900_gpio_priv *priv = gpiochip_get_data(chip);
+
+	/*
+	 * This will clear the output enable bit, the other bits are
+	 * dontcare when this is cleared
+	 */
+	return regmap_write(priv->regmap, offset, 0);
+}
+
+static int ts4900_gpio_direction_output(struct gpio_chip *chip,
+					unsigned int offset, int value)
+{
+	struct ts4900_gpio_priv *priv = gpiochip_get_data(chip);
+	int ret;
+
+	if (value)
+		ret = regmap_write(priv->regmap, offset, TS4900_GPIO_OE |
+							 TS4900_GPIO_OUT);
+	else
+		ret = regmap_write(priv->regmap, offset, TS4900_GPIO_OE);
+
+	return ret;
+}
+
+static int ts4900_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct ts4900_gpio_priv *priv = gpiochip_get_data(chip);
+	unsigned int reg;
+
+	regmap_read(priv->regmap, offset, &reg);
+
+	return !!(reg & priv->input_bit);
+}
+
+static void ts4900_gpio_set(struct gpio_chip *chip, unsigned int offset,
+			    int value)
+{
+	struct ts4900_gpio_priv *priv = gpiochip_get_data(chip);
+
+	if (value)
+		regmap_update_bits(priv->regmap, offset, TS4900_GPIO_OUT,
+				   TS4900_GPIO_OUT);
+	else
+		regmap_update_bits(priv->regmap, offset, TS4900_GPIO_OUT, 0);
+}
+
+static const struct regmap_config ts4900_regmap_config = {
+	.reg_bits = 16,
+	.val_bits = 8,
+};
+
+static const struct gpio_chip template_chip = {
+	.label			= "ts4900-gpio",
+	.owner			= THIS_MODULE,
+	.get_direction		= ts4900_gpio_get_direction,
+	.direction_input	= ts4900_gpio_direction_input,
+	.direction_output	= ts4900_gpio_direction_output,
+	.get			= ts4900_gpio_get,
+	.set			= ts4900_gpio_set,
+	.base			= -1,
+	.can_sleep		= true,
+};
+
+static const struct of_device_id ts4900_gpio_of_match_table[] = {
+	{
+		.compatible = "technologic,ts4900-gpio",
+		.data = (void *)TS4900_GPIO_IN,
+	}, {
+		.compatible = "technologic,ts7970-gpio",
+		.data = (void *)TS7970_GPIO_IN,
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, ts4900_gpio_of_match_table);
+
+static int ts4900_gpio_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	const struct of_device_id *match;
+	struct ts4900_gpio_priv *priv;
+	u32 ngpio;
+	int ret;
+
+	match = of_match_device(ts4900_gpio_of_match_table, &client->dev);
+	if (!match)
+		return -EINVAL;
+
+	if (of_property_read_u32(client->dev.of_node, "ngpios", &ngpio))
+		ngpio = DEFAULT_PIN_NUMBER;
+
+	priv = devm_kzalloc(&client->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->gpio_chip = template_chip;
+	priv->gpio_chip.label = "ts4900-gpio";
+	priv->gpio_chip.ngpio = ngpio;
+	priv->gpio_chip.parent = &client->dev;
+	priv->input_bit = (uintptr_t)match->data;
+
+	priv->regmap = devm_regmap_init_i2c(client, &ts4900_regmap_config);
+	if (IS_ERR(priv->regmap)) {
+		ret = PTR_ERR(priv->regmap);
+		dev_err(&client->dev, "Failed to allocate register map: %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = devm_gpiochip_add_data(&client->dev, &priv->gpio_chip, priv);
+	if (ret < 0) {
+		dev_err(&client->dev, "Unable to register gpiochip\n");
+		return ret;
+	}
+
+	i2c_set_clientdata(client, priv);
+
+	return 0;
+}
+
+static const struct i2c_device_id ts4900_gpio_id_table[] = {
+	{ "ts4900-gpio", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(i2c, ts4900_gpio_id_table);
+
+static struct i2c_driver ts4900_gpio_driver = {
+	.driver = {
+		.name = "ts4900-gpio",
+		.of_match_table = ts4900_gpio_of_match_table,
+	},
+	.probe = ts4900_gpio_probe,
+	.id_table = ts4900_gpio_id_table,
+};
+module_i2c_driver(ts4900_gpio_driver);
+
+MODULE_AUTHOR("Technologic Systems");
+MODULE_DESCRIPTION("GPIO interface for Technologic Systems I2C-FPGA core");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-twl4030.c b/drivers/gpio/gpio-twl4030.c
index 4b807b0e0c8e..dfcfbba74416 100644
--- a/drivers/gpio/gpio-twl4030.c
+++ b/drivers/gpio/gpio-twl4030.c
@@ -381,7 +381,7 @@ static int twl_to_irq(struct gpio_chip *chip, unsigned offset)
 		: -EINVAL;
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "twl4030",
 	.owner			= THIS_MODULE,
 	.request		= twl_request,
diff --git a/drivers/gpio/gpio-vf610.c b/drivers/gpio/gpio-vf610.c
index 6284bdbe1e0c..3edb09cb9ee0 100644
--- a/drivers/gpio/gpio-vf610.c
+++ b/drivers/gpio/gpio-vf610.c
@@ -1,5 +1,5 @@
 /*
- * vf610 GPIO support through PORT and GPIO module
+ * Freescale vf610 GPIO support through PORT and GPIO
  *
  * Copyright (c) 2014 Toradex AG.
  *
@@ -23,7 +23,6 @@
 #include <linux/io.h>
 #include <linux/ioport.h>
 #include <linux/irq.h>
-#include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -289,7 +288,3 @@ static int __init gpio_vf610_init(void)
 	return platform_driver_register(&vf610_gpio_driver);
 }
 device_initcall(gpio_vf610_init);
-
-MODULE_AUTHOR("Stefan Agner <stefan@agner.ch>");
-MODULE_DESCRIPTION("Freescale VF610 GPIO");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-wcove.c b/drivers/gpio/gpio-wcove.c
new file mode 100644
index 000000000000..d0ddba7a9d08
--- /dev/null
+++ b/drivers/gpio/gpio-wcove.c
@@ -0,0 +1,470 @@
+/*
+ * Intel Whiskey Cove PMIC GPIO Driver
+ *
+ * This driver is written based on gpio-crystalcove.c
+ *
+ * Copyright (C) 2016 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/bitops.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/gpio/driver.h>
+#include <linux/mfd/intel_soc_pmic.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/seq_file.h>
+
+/*
+ * Whiskey Cove PMIC has 13 physical GPIO pins divided into 3 banks:
+ * Bank 0: Pin 0 - 6
+ * Bank 1: Pin 7 - 10
+ * Bank 2: Pin 11 -12
+ * Each pin has one output control register and one input control register.
+ */
+#define BANK0_NR_PINS		7
+#define BANK1_NR_PINS		4
+#define BANK2_NR_PINS		2
+#define WCOVE_GPIO_NUM		(BANK0_NR_PINS + BANK1_NR_PINS + BANK2_NR_PINS)
+#define WCOVE_VGPIO_NUM		94
+/* GPIO output control registers (one per pin): 0x4e44 - 0x4e50 */
+#define GPIO_OUT_CTRL_BASE	0x4e44
+/* GPIO input control registers (one per pin): 0x4e51 - 0x4e5d */
+#define GPIO_IN_CTRL_BASE	0x4e51
+
+/*
+ * GPIO interrupts are organized in two groups:
+ * Group 0: Bank 0 pins (Pin 0 - 6)
+ * Group 1: Bank 1 and Bank 2 pins (Pin 7 - 12)
+ * Each group has two registers (one bit per pin): status and mask.
+ */
+#define GROUP0_NR_IRQS		7
+#define GROUP1_NR_IRQS		6
+#define IRQ_MASK_BASE		0x4e19
+#define IRQ_STATUS_BASE		0x4e0b
+#define UPDATE_IRQ_TYPE		BIT(0)
+#define UPDATE_IRQ_MASK		BIT(1)
+
+#define CTLI_INTCNT_DIS		(0 << 1)
+#define CTLI_INTCNT_NE		(1 << 1)
+#define CTLI_INTCNT_PE		(2 << 1)
+#define CTLI_INTCNT_BE		(3 << 1)
+
+#define CTLO_DIR_IN		(0 << 5)
+#define CTLO_DIR_OUT		(1 << 5)
+
+#define CTLO_DRV_MASK		(1 << 4)
+#define CTLO_DRV_OD		(0 << 4)
+#define CTLO_DRV_CMOS		(1 << 4)
+
+#define CTLO_DRV_REN		(1 << 3)
+
+#define CTLO_RVAL_2KDOWN	(0 << 1)
+#define CTLO_RVAL_2KUP		(1 << 1)
+#define CTLO_RVAL_50KDOWN	(2 << 1)
+#define CTLO_RVAL_50KUP		(3 << 1)
+
+#define CTLO_INPUT_SET	(CTLO_DRV_CMOS | CTLO_DRV_REN | CTLO_RVAL_2KUP)
+#define CTLO_OUTPUT_SET	(CTLO_DIR_OUT | CTLO_INPUT_SET)
+
+enum ctrl_register {
+	CTRL_IN,
+	CTRL_OUT,
+};
+
+/*
+ * struct wcove_gpio - Whiskey Cove GPIO controller
+ * @buslock: for bus lock/sync and unlock.
+ * @chip: the abstract gpio_chip structure.
+ * @dev: the gpio device
+ * @regmap: the regmap from the parent device.
+ * @regmap_irq_chip: the regmap of the gpio irq chip.
+ * @update: pending IRQ setting update, to be written to the chip upon unlock.
+ * @intcnt: the Interrupt Detect value to be written.
+ * @set_irq_mask: true if the IRQ mask needs to be set, false to clear.
+ */
+struct wcove_gpio {
+	struct mutex buslock;
+	struct gpio_chip chip;
+	struct device *dev;
+	struct regmap *regmap;
+	struct regmap_irq_chip_data *regmap_irq_chip;
+	int update;
+	int intcnt;
+	bool set_irq_mask;
+};
+
+static inline unsigned int to_reg(int gpio, enum ctrl_register reg_type)
+{
+	unsigned int reg;
+	int bank;
+
+	if (gpio < BANK0_NR_PINS)
+		bank = 0;
+	else if (gpio < BANK0_NR_PINS + BANK1_NR_PINS)
+		bank = 1;
+	else
+		bank = 2;
+
+	if (reg_type == CTRL_IN)
+		reg = GPIO_IN_CTRL_BASE + bank;
+	else
+		reg = GPIO_OUT_CTRL_BASE + bank;
+
+	return reg;
+}
+
+static void wcove_update_irq_mask(struct wcove_gpio *wg, int gpio)
+{
+	unsigned int reg, mask;
+
+	if (gpio < GROUP0_NR_IRQS) {
+		reg = IRQ_MASK_BASE;
+		mask = BIT(gpio % GROUP0_NR_IRQS);
+	} else {
+		reg = IRQ_MASK_BASE + 1;
+		mask = BIT((gpio - GROUP0_NR_IRQS) % GROUP1_NR_IRQS);
+	}
+
+	if (wg->set_irq_mask)
+		regmap_update_bits(wg->regmap, reg, mask, mask);
+	else
+		regmap_update_bits(wg->regmap, reg, mask, 0);
+}
+
+static void wcove_update_irq_ctrl(struct wcove_gpio *wg, int gpio)
+{
+	unsigned int reg = to_reg(gpio, CTRL_IN);
+
+	regmap_update_bits(wg->regmap, reg, CTLI_INTCNT_BE, wg->intcnt);
+}
+
+static int wcove_gpio_dir_in(struct gpio_chip *chip, unsigned int gpio)
+{
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+
+	return regmap_write(wg->regmap, to_reg(gpio, CTRL_OUT),
+			    CTLO_INPUT_SET);
+}
+
+static int wcove_gpio_dir_out(struct gpio_chip *chip, unsigned int gpio,
+				    int value)
+{
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+
+	return regmap_write(wg->regmap, to_reg(gpio, CTRL_OUT),
+			    CTLO_OUTPUT_SET | value);
+}
+
+static int wcove_gpio_get_direction(struct gpio_chip *chip, unsigned int gpio)
+{
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(wg->regmap, to_reg(gpio, CTRL_OUT), &val);
+	if (ret)
+		return ret;
+
+	return !(val & CTLO_DIR_OUT);
+}
+
+static int wcove_gpio_get(struct gpio_chip *chip, unsigned int gpio)
+{
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(wg->regmap, to_reg(gpio, CTRL_IN), &val);
+	if (ret)
+		return ret;
+
+	return val & 0x1;
+}
+
+static void wcove_gpio_set(struct gpio_chip *chip,
+				 unsigned int gpio, int value)
+{
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+
+	if (value)
+		regmap_update_bits(wg->regmap, to_reg(gpio, CTRL_OUT), 1, 1);
+	else
+		regmap_update_bits(wg->regmap, to_reg(gpio, CTRL_OUT), 1, 0);
+}
+
+static int wcove_gpio_set_single_ended(struct gpio_chip *chip,
+					unsigned int gpio,
+					enum single_ended_mode mode)
+{
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+
+	switch (mode) {
+	case LINE_MODE_OPEN_DRAIN:
+		return regmap_update_bits(wg->regmap, to_reg(gpio, CTRL_OUT),
+						CTLO_DRV_MASK, CTLO_DRV_OD);
+	case LINE_MODE_PUSH_PULL:
+		return regmap_update_bits(wg->regmap, to_reg(gpio, CTRL_OUT),
+						CTLO_DRV_MASK, CTLO_DRV_CMOS);
+	default:
+		break;
+	}
+
+	return -ENOTSUPP;
+}
+
+static int wcove_irq_type(struct irq_data *data, unsigned int type)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+
+	switch (type) {
+	case IRQ_TYPE_NONE:
+		wg->intcnt = CTLI_INTCNT_DIS;
+		break;
+	case IRQ_TYPE_EDGE_BOTH:
+		wg->intcnt = CTLI_INTCNT_BE;
+		break;
+	case IRQ_TYPE_EDGE_RISING:
+		wg->intcnt = CTLI_INTCNT_PE;
+		break;
+	case IRQ_TYPE_EDGE_FALLING:
+		wg->intcnt = CTLI_INTCNT_NE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	wg->update |= UPDATE_IRQ_TYPE;
+
+	return 0;
+}
+
+static void wcove_bus_lock(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+
+	mutex_lock(&wg->buslock);
+}
+
+static void wcove_bus_sync_unlock(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+	int gpio = data->hwirq;
+
+	if (wg->update & UPDATE_IRQ_TYPE)
+		wcove_update_irq_ctrl(wg, gpio);
+	if (wg->update & UPDATE_IRQ_MASK)
+		wcove_update_irq_mask(wg, gpio);
+	wg->update = 0;
+
+	mutex_unlock(&wg->buslock);
+}
+
+static void wcove_irq_unmask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+
+	wg->set_irq_mask = false;
+	wg->update |= UPDATE_IRQ_MASK;
+}
+
+static void wcove_irq_mask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+
+	wg->set_irq_mask = true;
+	wg->update |= UPDATE_IRQ_MASK;
+}
+
+static struct irq_chip wcove_irqchip = {
+	.name			= "Whiskey Cove",
+	.irq_mask		= wcove_irq_mask,
+	.irq_unmask		= wcove_irq_unmask,
+	.irq_set_type		= wcove_irq_type,
+	.irq_bus_lock		= wcove_bus_lock,
+	.irq_bus_sync_unlock	= wcove_bus_sync_unlock,
+};
+
+static irqreturn_t wcove_gpio_irq_handler(int irq, void *data)
+{
+	struct wcove_gpio *wg = (struct wcove_gpio *)data;
+	unsigned int pending, virq, gpio, mask, offset;
+	u8 p[2];
+
+	if (regmap_bulk_read(wg->regmap, IRQ_STATUS_BASE, p, 2)) {
+		dev_err(wg->dev, "Failed to read irq status register\n");
+		return IRQ_NONE;
+	}
+
+	pending = p[0] | (p[1] << 8);
+	if (!pending)
+		return IRQ_NONE;
+
+	/* Iterate until no interrupt is pending */
+	while (pending) {
+		/* One iteration is for all pending bits */
+		for_each_set_bit(gpio, (const unsigned long *)&pending,
+						 GROUP0_NR_IRQS) {
+			offset = (gpio > GROUP0_NR_IRQS) ? 1 : 0;
+			mask = (offset == 1) ? BIT(gpio - GROUP0_NR_IRQS) :
+								BIT(gpio);
+			virq = irq_find_mapping(wg->chip.irqdomain, gpio);
+			handle_nested_irq(virq);
+			regmap_update_bits(wg->regmap, IRQ_STATUS_BASE + offset,
+								mask, mask);
+		}
+
+		/* Next iteration */
+		if (regmap_bulk_read(wg->regmap, IRQ_STATUS_BASE, p, 2)) {
+			dev_err(wg->dev, "Failed to read irq status\n");
+			break;
+		}
+
+		pending = p[0] | (p[1] << 8);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void wcove_gpio_dbg_show(struct seq_file *s,
+				      struct gpio_chip *chip)
+{
+	unsigned int ctlo, ctli, irq_mask, irq_status;
+	struct wcove_gpio *wg = gpiochip_get_data(chip);
+	int gpio, offset, group, ret = 0;
+
+	for (gpio = 0; gpio < WCOVE_GPIO_NUM; gpio++) {
+		group = gpio < GROUP0_NR_IRQS ? 0 : 1;
+		ret += regmap_read(wg->regmap, to_reg(gpio, CTRL_OUT), &ctlo);
+		ret += regmap_read(wg->regmap, to_reg(gpio, CTRL_IN), &ctli);
+		ret += regmap_read(wg->regmap, IRQ_MASK_BASE + group,
+							&irq_mask);
+		ret += regmap_read(wg->regmap, IRQ_STATUS_BASE + group,
+							&irq_status);
+		if (ret) {
+			pr_err("Failed to read registers: ctrl out/in or irq status/mask\n");
+			break;
+		}
+
+		offset = gpio % 8;
+		seq_printf(s, " gpio-%-2d %s %s %s %s ctlo=%2x,%s %s\n",
+			   gpio, ctlo & CTLO_DIR_OUT ? "out" : "in ",
+			   ctli & 0x1 ? "hi" : "lo",
+			   ctli & CTLI_INTCNT_NE ? "fall" : "    ",
+			   ctli & CTLI_INTCNT_PE ? "rise" : "    ",
+			   ctlo,
+			   irq_mask & BIT(offset) ? "mask  " : "unmask",
+			   irq_status & BIT(offset) ? "pending" : "       ");
+	}
+}
+
+static int wcove_gpio_probe(struct platform_device *pdev)
+{
+	struct intel_soc_pmic *pmic;
+	struct wcove_gpio *wg;
+	int virq, ret, irq;
+	struct device *dev;
+
+	/*
+	 * This gpio platform device is created by a mfd device (see
+	 * drivers/mfd/intel_soc_pmic_bxtwc.c for details). Information
+	 * shared by all sub-devices created by the mfd device, the regmap
+	 * pointer for instance, is stored as driver data of the mfd device
+	 * driver.
+	 */
+	pmic = dev_get_drvdata(pdev->dev.parent);
+	if (!pmic)
+		return -ENODEV;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	dev = &pdev->dev;
+
+	wg = devm_kzalloc(dev, sizeof(*wg), GFP_KERNEL);
+	if (!wg)
+		return -ENOMEM;
+
+	wg->regmap_irq_chip = pmic->irq_chip_data_level2;
+
+	platform_set_drvdata(pdev, wg);
+
+	mutex_init(&wg->buslock);
+	wg->chip.label = KBUILD_MODNAME;
+	wg->chip.direction_input = wcove_gpio_dir_in;
+	wg->chip.direction_output = wcove_gpio_dir_out;
+	wg->chip.get_direction = wcove_gpio_get_direction;
+	wg->chip.get = wcove_gpio_get;
+	wg->chip.set = wcove_gpio_set;
+	wg->chip.set_single_ended = wcove_gpio_set_single_ended,
+	wg->chip.base = -1;
+	wg->chip.ngpio = WCOVE_VGPIO_NUM;
+	wg->chip.can_sleep = true;
+	wg->chip.parent = pdev->dev.parent;
+	wg->chip.dbg_show = wcove_gpio_dbg_show;
+	wg->dev = dev;
+	wg->regmap = pmic->regmap;
+
+	ret = devm_gpiochip_add_data(dev, &wg->chip, wg);
+	if (ret) {
+		dev_err(dev, "Failed to add gpiochip: %d\n", ret);
+		return ret;
+	}
+
+	ret = gpiochip_irqchip_add(&wg->chip, &wcove_irqchip, 0,
+			     handle_simple_irq, IRQ_TYPE_NONE);
+	if (ret) {
+		dev_err(dev, "Failed to add irqchip: %d\n", ret);
+		return ret;
+	}
+
+	virq = regmap_irq_get_virq(wg->regmap_irq_chip, irq);
+	if (virq < 0) {
+		dev_err(dev, "Failed to get virq by irq %d\n", irq);
+		return virq;
+	}
+
+	ret = devm_request_threaded_irq(dev, virq, NULL,
+		wcove_gpio_irq_handler, IRQF_ONESHOT, pdev->name, wg);
+	if (ret) {
+		dev_err(dev, "Failed to request irq %d\n", virq);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * Whiskey Cove PMIC itself is a analog device(but with digital control
+ * interface) providing power management support for other devices in
+ * the accompanied SoC, so we have no .pm for Whiskey Cove GPIO driver.
+ */
+static struct platform_driver wcove_gpio_driver = {
+	.driver = {
+		.name = "bxt_wcove_gpio",
+	},
+	.probe = wcove_gpio_probe,
+};
+
+module_platform_driver(wcove_gpio_driver);
+
+MODULE_AUTHOR("Ajay Thomas <ajay.thomas.david.rajamanickam@intel.com>");
+MODULE_AUTHOR("Bin Gao <bin.gao@intel.com>");
+MODULE_DESCRIPTION("Intel Whiskey Cove GPIO Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:bxt_wcove_gpio");
diff --git a/drivers/gpio/gpio-wm831x.c b/drivers/gpio/gpio-wm831x.c
index 21f97bcd0062..533707f943f4 100644
--- a/drivers/gpio/gpio-wm831x.c
+++ b/drivers/gpio/gpio-wm831x.c
@@ -247,7 +247,7 @@ static void wm831x_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 #define wm831x_gpio_dbg_show NULL
 #endif
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "wm831x",
 	.owner			= THIS_MODULE,
 	.direction_input	= wm831x_gpio_direction_in,
diff --git a/drivers/gpio/gpio-wm8350.c b/drivers/gpio/gpio-wm8350.c
index e9765707d5c1..e46752e73dd9 100644
--- a/drivers/gpio/gpio-wm8350.c
+++ b/drivers/gpio/gpio-wm8350.c
@@ -93,7 +93,7 @@ static int wm8350_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
 	return wm8350->irq_base + WM8350_IRQ_GPIO(offset);
 }
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "wm8350",
 	.owner			= THIS_MODULE,
 	.direction_input	= wm8350_gpio_direction_in,
diff --git a/drivers/gpio/gpio-wm8994.c b/drivers/gpio/gpio-wm8994.c
index 2457aac8592e..68410fda6138 100644
--- a/drivers/gpio/gpio-wm8994.c
+++ b/drivers/gpio/gpio-wm8994.c
@@ -249,7 +249,7 @@ static void wm8994_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 #define wm8994_gpio_dbg_show NULL
 #endif
 
-static struct gpio_chip template_chip = {
+static const struct gpio_chip template_chip = {
 	.label			= "wm8994",
 	.owner			= THIS_MODULE,
 	.request		= wm8994_gpio_request,
diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c
index e72794e463aa..6b4d10d6e10f 100644
--- a/drivers/gpio/gpio-zynq.c
+++ b/drivers/gpio/gpio-zynq.c
@@ -96,6 +96,9 @@
 /* GPIO upper 16 bit mask */
 #define ZYNQ_GPIO_UPPER_MASK 0xFFFF0000
 
+/* For GPIO quirks */
+#define ZYNQ_GPIO_QUIRK_FOO	BIT(0)
+
 /**
  * struct zynq_gpio - gpio device private data structure
  * @chip:	instance of the gpio_chip
@@ -122,6 +125,7 @@ struct zynq_gpio {
 */
 struct zynq_platform_data {
 	const char *label;
+	u32 quirks;
 	u16 ngpio;
 	int max_bank;
 	int bank_min[ZYNQMP_GPIO_MAX_BANK];
@@ -238,13 +242,19 @@ static void zynq_gpio_set_value(struct gpio_chip *chip, unsigned int pin,
 static int zynq_gpio_dir_in(struct gpio_chip *chip, unsigned int pin)
 {
 	u32 reg;
+	bool is_zynq_gpio;
 	unsigned int bank_num, bank_pin_num;
 	struct zynq_gpio *gpio = gpiochip_get_data(chip);
 
+	is_zynq_gpio = gpio->p_data->quirks & ZYNQ_GPIO_QUIRK_FOO;
 	zynq_gpio_get_bank_pin(pin, &bank_num, &bank_pin_num, gpio);
 
-	/* bank 0 pins 7 and 8 are special and cannot be used as inputs */
-	if (bank_num == 0 && (bank_pin_num == 7 || bank_pin_num == 8))
+	/*
+	 * On zynq bank 0 pins 7 and 8 are special and cannot be used
+	 * as inputs.
+	 */
+	if (is_zynq_gpio && bank_num == 0 &&
+		(bank_pin_num == 7 || bank_pin_num == 8))
 		return -EINVAL;
 
 	/* clear the bit in direction mode reg to set the pin as input */
@@ -627,6 +637,7 @@ static const struct zynq_platform_data zynqmp_gpio_def = {
 
 static const struct zynq_platform_data zynq_gpio_def = {
 	.label = "zynq_gpio",
+	.quirks = ZYNQ_GPIO_QUIRK_FOO,
 	.ngpio = ZYNQ_GPIO_NR_GPIOS,
 	.max_bank = ZYNQ_GPIO_MAX_BANK,
 	.bank_min[0] = ZYNQ_GPIO_BANK0_PIN_MIN(),
diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c
index af514618d7fb..58ece201b8e6 100644
--- a/drivers/gpio/gpiolib-acpi.c
+++ b/drivers/gpio/gpiolib-acpi.c
@@ -14,6 +14,7 @@
 #include <linux/gpio.h>
 #include <linux/gpio/consumer.h>
 #include <linux/gpio/driver.h>
+#include <linux/gpio/machine.h>
 #include <linux/export.h>
 #include <linux/acpi.h>
 #include <linux/interrupt.h>
@@ -395,7 +396,7 @@ struct acpi_gpio_lookup {
 	int n;
 };
 
-static int acpi_find_gpio(struct acpi_resource *ares, void *data)
+static int acpi_populate_gpio_lookup(struct acpi_resource *ares, void *data)
 {
 	struct acpi_gpio_lookup *lookup = data;
 
@@ -440,7 +441,8 @@ static int acpi_gpio_resource_lookup(struct acpi_gpio_lookup *lookup,
 
 	INIT_LIST_HEAD(&res_list);
 
-	ret = acpi_dev_get_resources(lookup->adev, &res_list, acpi_find_gpio,
+	ret = acpi_dev_get_resources(lookup->adev, &res_list,
+				     acpi_populate_gpio_lookup,
 				     lookup);
 	if (ret < 0)
 		return ret;
@@ -513,7 +515,7 @@ static int acpi_gpio_property_lookup(struct fwnode_handle *fwnode,
  * Note: if the GPIO resource has multiple entries in the pin list, this
  * function only returns the first.
  */
-struct gpio_desc *acpi_get_gpiod_by_index(struct acpi_device *adev,
+static struct gpio_desc *acpi_get_gpiod_by_index(struct acpi_device *adev,
 					  const char *propname, int index,
 					  struct acpi_gpio_info *info)
 {
@@ -546,6 +548,55 @@ struct gpio_desc *acpi_get_gpiod_by_index(struct acpi_device *adev,
 	return ret ? ERR_PTR(ret) : lookup.desc;
 }
 
+struct gpio_desc *acpi_find_gpio(struct device *dev,
+				 const char *con_id,
+				 unsigned int idx,
+				 enum gpiod_flags flags,
+				 enum gpio_lookup_flags *lookupflags)
+{
+	struct acpi_device *adev = ACPI_COMPANION(dev);
+	struct acpi_gpio_info info;
+	struct gpio_desc *desc;
+	char propname[32];
+	int i;
+
+	/* Try first from _DSD */
+	for (i = 0; i < ARRAY_SIZE(gpio_suffixes); i++) {
+		if (con_id && strcmp(con_id, "gpios")) {
+			snprintf(propname, sizeof(propname), "%s-%s",
+				 con_id, gpio_suffixes[i]);
+		} else {
+			snprintf(propname, sizeof(propname), "%s",
+				 gpio_suffixes[i]);
+		}
+
+		desc = acpi_get_gpiod_by_index(adev, propname, idx, &info);
+		if (!IS_ERR(desc) || (PTR_ERR(desc) == -EPROBE_DEFER))
+			break;
+	}
+
+	/* Then from plain _CRS GPIOs */
+	if (IS_ERR(desc)) {
+		if (!acpi_can_fallback_to_crs(adev, con_id))
+			return ERR_PTR(-ENOENT);
+
+		desc = acpi_get_gpiod_by_index(adev, NULL, idx, &info);
+		if (IS_ERR(desc))
+			return desc;
+
+		if ((flags == GPIOD_OUT_LOW || flags == GPIOD_OUT_HIGH) &&
+		    info.gpioint) {
+			dev_dbg(dev, "refusing GpioInt() entry when doing GPIOD_OUT_* lookup\n");
+			return ERR_PTR(-ENOENT);
+		}
+	}
+
+	if (info.polarity == GPIO_ACTIVE_LOW)
+		*lookupflags |= GPIO_ACTIVE_LOW;
+
+	return desc;
+}
+
 /**
  * acpi_node_get_gpiod() - get a GPIO descriptor from ACPI resources
  * @fwnode: pointer to an ACPI firmware node to get the GPIO information from
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 75e7b3919ea7..ecad3f0e3b77 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -16,7 +16,6 @@
 #include <linux/errno.h>
 #include <linux/module.h>
 #include <linux/io.h>
-#include <linux/io-mapping.h>
 #include <linux/gpio/consumer.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
@@ -114,6 +113,45 @@ int of_get_named_gpio_flags(struct device_node *np, const char *list_name,
 }
 EXPORT_SYMBOL(of_get_named_gpio_flags);
 
+struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
+			       unsigned int idx,
+			       enum gpio_lookup_flags *flags)
+{
+	char prop_name[32]; /* 32 is max size of property name */
+	enum of_gpio_flags of_flags;
+	struct gpio_desc *desc;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(gpio_suffixes); i++) {
+		if (con_id)
+			snprintf(prop_name, sizeof(prop_name), "%s-%s", con_id,
+				 gpio_suffixes[i]);
+		else
+			snprintf(prop_name, sizeof(prop_name), "%s",
+				 gpio_suffixes[i]);
+
+		desc = of_get_named_gpiod_flags(dev->of_node, prop_name, idx,
+						&of_flags);
+		if (!IS_ERR(desc) || (PTR_ERR(desc) != -ENOENT))
+			break;
+	}
+
+	if (IS_ERR(desc))
+		return desc;
+
+	if (of_flags & OF_GPIO_ACTIVE_LOW)
+		*flags |= GPIO_ACTIVE_LOW;
+
+	if (of_flags & OF_GPIO_SINGLE_ENDED) {
+		if (of_flags & OF_GPIO_ACTIVE_LOW)
+			*flags |= GPIO_OPEN_DRAIN;
+		else
+			*flags |= GPIO_OPEN_SOURCE;
+	}
+
+	return desc;
+}
+
 /**
  * of_parse_own_gpio() - Get a GPIO hog descriptor, names and flags for GPIO API
  * @np:		device node to get GPIO from
diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c
index 932e510aec50..4b44dd97c07f 100644
--- a/drivers/gpio/gpiolib-sysfs.c
+++ b/drivers/gpio/gpiolib-sysfs.c
@@ -670,10 +670,10 @@ int gpiod_export_link(struct device *dev, const char *name,
 EXPORT_SYMBOL_GPL(gpiod_export_link);
 
 /**
- * gpiod_unexport - reverse effect of gpio_export()
+ * gpiod_unexport - reverse effect of gpiod_export()
  * @gpio: gpio to make unavailable
  *
- * This is implicit on gpio_free().
+ * This is implicit on gpiod_free().
  */
 void gpiod_unexport(struct gpio_desc *desc)
 {
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index 53ff25ac66d8..f0fc3a0d37c8 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -71,6 +71,8 @@ LIST_HEAD(gpio_devices);
 
 static void gpiochip_free_hogs(struct gpio_chip *chip);
 static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip);
+static int gpiochip_irqchip_init_valid_mask(struct gpio_chip *gpiochip);
+static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gpiochip);
 
 static bool gpiolib_initialized;
 
@@ -1167,6 +1169,10 @@ int gpiochip_add_data(struct gpio_chip *chip, void *data)
 	if (status)
 		goto err_remove_from_list;
 
+	status = gpiochip_irqchip_init_valid_mask(chip);
+	if (status)
+		goto err_remove_from_list;
+
 	status = of_gpiochip_add(chip);
 	if (status)
 		goto err_remove_chip;
@@ -1192,6 +1198,7 @@ err_remove_chip:
 	acpi_gpiochip_remove(chip);
 	gpiochip_free_hogs(chip);
 	of_gpiochip_remove(chip);
+	gpiochip_irqchip_free_valid_mask(chip);
 err_remove_from_list:
 	spin_lock_irqsave(&gpio_lock, flags);
 	list_del(&gdev->list);
@@ -1363,19 +1370,15 @@ struct gpio_chip *gpiochip_find(void *data,
 					     void *data))
 {
 	struct gpio_device *gdev;
-	struct gpio_chip *chip;
+	struct gpio_chip *chip = NULL;
 	unsigned long flags;
 
 	spin_lock_irqsave(&gpio_lock, flags);
 	list_for_each_entry(gdev, &gpio_devices, list)
-		if (gdev->chip && match(gdev->chip, data))
+		if (gdev->chip && match(gdev->chip, data)) {
+			chip = gdev->chip;
 			break;
-
-	/* No match? */
-	if (&gdev->list == &gpio_devices)
-		chip = NULL;
-	else
-		chip = gdev->chip;
+		}
 
 	spin_unlock_irqrestore(&gpio_lock, flags);
 
@@ -1401,6 +1404,40 @@ static struct gpio_chip *find_chip_by_name(const char *name)
  * The following is irqchip helper code for gpiochips.
  */
 
+static int gpiochip_irqchip_init_valid_mask(struct gpio_chip *gpiochip)
+{
+	int i;
+
+	if (!gpiochip->irq_need_valid_mask)
+		return 0;
+
+	gpiochip->irq_valid_mask = kcalloc(BITS_TO_LONGS(gpiochip->ngpio),
+					   sizeof(long), GFP_KERNEL);
+	if (!gpiochip->irq_valid_mask)
+		return -ENOMEM;
+
+	/* Assume by default all GPIOs are valid */
+	for (i = 0; i < gpiochip->ngpio; i++)
+		set_bit(i, gpiochip->irq_valid_mask);
+
+	return 0;
+}
+
+static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gpiochip)
+{
+	kfree(gpiochip->irq_valid_mask);
+	gpiochip->irq_valid_mask = NULL;
+}
+
+static bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gpiochip,
+				       unsigned int offset)
+{
+	/* No mask means all valid */
+	if (likely(!gpiochip->irq_valid_mask))
+		return true;
+	return test_bit(offset, gpiochip->irq_valid_mask);
+}
+
 /**
  * gpiochip_set_chained_irqchip() - sets a chained irqchip to a gpiochip
  * @gpiochip: the gpiochip to set the irqchip chain to
@@ -1442,9 +1479,12 @@ void gpiochip_set_chained_irqchip(struct gpio_chip *gpiochip,
 	}
 
 	/* Set the parent IRQ for all affected IRQs */
-	for (offset = 0; offset < gpiochip->ngpio; offset++)
+	for (offset = 0; offset < gpiochip->ngpio; offset++) {
+		if (!gpiochip_irqchip_irq_valid(gpiochip, offset))
+			continue;
 		irq_set_parent(irq_find_mapping(gpiochip->irqdomain, offset),
 			       parent_irq);
+	}
 }
 EXPORT_SYMBOL_GPL(gpiochip_set_chained_irqchip);
 
@@ -1551,9 +1591,12 @@ static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip)
 
 	/* Remove all IRQ mappings and delete the domain */
 	if (gpiochip->irqdomain) {
-		for (offset = 0; offset < gpiochip->ngpio; offset++)
+		for (offset = 0; offset < gpiochip->ngpio; offset++) {
+			if (!gpiochip_irqchip_irq_valid(gpiochip, offset))
+				continue;
 			irq_dispose_mapping(
 				irq_find_mapping(gpiochip->irqdomain, offset));
+		}
 		irq_domain_remove(gpiochip->irqdomain);
 	}
 
@@ -1562,6 +1605,8 @@ static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip)
 		gpiochip->irqchip->irq_release_resources = NULL;
 		gpiochip->irqchip = NULL;
 	}
+
+	gpiochip_irqchip_free_valid_mask(gpiochip);
 }
 
 /**
@@ -1597,6 +1642,7 @@ int _gpiochip_irqchip_add(struct gpio_chip *gpiochip,
 			  struct lock_class_key *lock_key)
 {
 	struct device_node *of_node;
+	bool irq_base_set = false;
 	unsigned int offset;
 	unsigned irq_base = 0;
 
@@ -1617,6 +1663,20 @@ int _gpiochip_irqchip_add(struct gpio_chip *gpiochip,
 	if (gpiochip->of_node)
 		of_node = gpiochip->of_node;
 #endif
+	/*
+	 * Specifying a default trigger is a terrible idea if DT or ACPI is
+	 * used to configure the interrupts, as you may end-up with
+	 * conflicting triggers. Tell the user, and reset to NONE.
+	 */
+	if (WARN(of_node && type != IRQ_TYPE_NONE,
+		 "%s: Ignoring %d default trigger\n", of_node->full_name, type))
+		type = IRQ_TYPE_NONE;
+	if (has_acpi_companion(gpiochip->parent) && type != IRQ_TYPE_NONE) {
+		acpi_handle_warn(ACPI_HANDLE(gpiochip->parent),
+				 "Ignoring %d default trigger\n", type);
+		type = IRQ_TYPE_NONE;
+	}
+
 	gpiochip->irqchip = irqchip;
 	gpiochip->irq_handler = handler;
 	gpiochip->irq_default_type = type;
@@ -1646,13 +1706,17 @@ int _gpiochip_irqchip_add(struct gpio_chip *gpiochip,
 	 * necessary to allocate descriptors for all IRQs.
 	 */
 	for (offset = 0; offset < gpiochip->ngpio; offset++) {
+		if (!gpiochip_irqchip_irq_valid(gpiochip, offset))
+			continue;
 		irq_base = irq_create_mapping(gpiochip->irqdomain, offset);
-		if (offset == 0)
+		if (!irq_base_set) {
 			/*
 			 * Store the base into the gpiochip to be used when
 			 * unmapping the irqs.
 			 */
 			gpiochip->irq_base = irq_base;
+			irq_base_set = true;
+		}
 	}
 
 	acpi_gpiochip_request_interrupts(gpiochip);
@@ -1664,6 +1728,12 @@ EXPORT_SYMBOL_GPL(_gpiochip_irqchip_add);
 #else /* CONFIG_GPIOLIB_IRQCHIP */
 
 static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip) {}
+static inline int gpiochip_irqchip_init_valid_mask(struct gpio_chip *gpiochip)
+{
+	return 0;
+}
+static inline void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gpiochip)
+{ }
 
 #endif /* CONFIG_GPIOLIB_IRQCHIP */
 
@@ -2813,94 +2883,6 @@ void gpiod_remove_lookup_table(struct gpiod_lookup_table *table)
 	mutex_unlock(&gpio_lookup_lock);
 }
 
-static struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
-				      unsigned int idx,
-				      enum gpio_lookup_flags *flags)
-{
-	char prop_name[32]; /* 32 is max size of property name */
-	enum of_gpio_flags of_flags;
-	struct gpio_desc *desc;
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(gpio_suffixes); i++) {
-		if (con_id)
-			snprintf(prop_name, sizeof(prop_name), "%s-%s", con_id,
-				 gpio_suffixes[i]);
-		else
-			snprintf(prop_name, sizeof(prop_name), "%s",
-				 gpio_suffixes[i]);
-
-		desc = of_get_named_gpiod_flags(dev->of_node, prop_name, idx,
-						&of_flags);
-		if (!IS_ERR(desc) || (PTR_ERR(desc) != -ENOENT))
-			break;
-	}
-
-	if (IS_ERR(desc))
-		return desc;
-
-	if (of_flags & OF_GPIO_ACTIVE_LOW)
-		*flags |= GPIO_ACTIVE_LOW;
-
-	if (of_flags & OF_GPIO_SINGLE_ENDED) {
-		if (of_flags & OF_GPIO_ACTIVE_LOW)
-			*flags |= GPIO_OPEN_DRAIN;
-		else
-			*flags |= GPIO_OPEN_SOURCE;
-	}
-
-	return desc;
-}
-
-static struct gpio_desc *acpi_find_gpio(struct device *dev,
-					const char *con_id,
-					unsigned int idx,
-					enum gpiod_flags flags,
-					enum gpio_lookup_flags *lookupflags)
-{
-	struct acpi_device *adev = ACPI_COMPANION(dev);
-	struct acpi_gpio_info info;
-	struct gpio_desc *desc;
-	char propname[32];
-	int i;
-
-	/* Try first from _DSD */
-	for (i = 0; i < ARRAY_SIZE(gpio_suffixes); i++) {
-		if (con_id && strcmp(con_id, "gpios")) {
-			snprintf(propname, sizeof(propname), "%s-%s",
-				 con_id, gpio_suffixes[i]);
-		} else {
-			snprintf(propname, sizeof(propname), "%s",
-				 gpio_suffixes[i]);
-		}
-
-		desc = acpi_get_gpiod_by_index(adev, propname, idx, &info);
-		if (!IS_ERR(desc) || (PTR_ERR(desc) == -EPROBE_DEFER))
-			break;
-	}
-
-	/* Then from plain _CRS GPIOs */
-	if (IS_ERR(desc)) {
-		if (!acpi_can_fallback_to_crs(adev, con_id))
-			return ERR_PTR(-ENOENT);
-
-		desc = acpi_get_gpiod_by_index(adev, NULL, idx, &info);
-		if (IS_ERR(desc))
-			return desc;
-
-		if ((flags == GPIOD_OUT_LOW || flags == GPIOD_OUT_HIGH) &&
-		    info.gpioint) {
-			dev_dbg(dev, "refusing GpioInt() entry when doing GPIOD_OUT_* lookup\n");
-			return ERR_PTR(-ENOENT);
-		}
-	}
-
-	if (info.polarity == GPIO_ACTIVE_LOW)
-		*lookupflags |= GPIO_ACTIVE_LOW;
-
-	return desc;
-}
-
 static struct gpiod_lookup_table *gpiod_find_lookup_table(struct device *dev)
 {
 	const char *dev_id = dev ? dev_name(dev) : NULL;
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index 2d9ea5e0cab3..346fbda39220 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -20,6 +20,7 @@
 
 enum of_gpio_flags;
 enum gpiod_flags;
+enum gpio_lookup_flags;
 struct acpi_device;
 
 /**
@@ -86,6 +87,32 @@ struct acpi_gpio_info {
 /* gpio suffixes used for ACPI and device tree lookup */
 static const char * const gpio_suffixes[] = { "gpios", "gpio" };
 
+#ifdef CONFIG_OF_GPIO
+struct gpio_desc *of_find_gpio(struct device *dev,
+			       const char *con_id,
+			       unsigned int idx,
+			       enum gpio_lookup_flags *flags);
+struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
+		   const char *list_name, int index, enum of_gpio_flags *flags);
+int of_gpiochip_add(struct gpio_chip *gc);
+void of_gpiochip_remove(struct gpio_chip *gc);
+#else
+static inline struct gpio_desc *of_find_gpio(struct device *dev,
+					     const char *con_id,
+					     unsigned int idx,
+					     enum gpio_lookup_flags *flags)
+{
+	return ERR_PTR(-ENOENT);
+}
+static inline struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
+		   const char *list_name, int index, enum of_gpio_flags *flags)
+{
+	return ERR_PTR(-ENOENT);
+}
+static inline int of_gpiochip_add(struct gpio_chip *gc) { return 0; }
+static inline void of_gpiochip_remove(struct gpio_chip *gc) { }
+#endif /* CONFIG_OF_GPIO */
+
 #ifdef CONFIG_ACPI
 void acpi_gpiochip_add(struct gpio_chip *chip);
 void acpi_gpiochip_remove(struct gpio_chip *chip);
@@ -93,9 +120,11 @@ void acpi_gpiochip_remove(struct gpio_chip *chip);
 void acpi_gpiochip_request_interrupts(struct gpio_chip *chip);
 void acpi_gpiochip_free_interrupts(struct gpio_chip *chip);
 
-struct gpio_desc *acpi_get_gpiod_by_index(struct acpi_device *adev,
-					  const char *propname, int index,
-					  struct acpi_gpio_info *info);
+struct gpio_desc *acpi_find_gpio(struct device *dev,
+				 const char *con_id,
+				 unsigned int idx,
+				 enum gpiod_flags flags,
+				 enum gpio_lookup_flags *lookupflags);
 struct gpio_desc *acpi_node_get_gpiod(struct fwnode_handle *fwnode,
 				      const char *propname, int index,
 				      struct acpi_gpio_info *info);
@@ -114,10 +143,11 @@ static inline void
 acpi_gpiochip_free_interrupts(struct gpio_chip *chip) { }
 
 static inline struct gpio_desc *
-acpi_get_gpiod_by_index(struct acpi_device *adev, const char *propname,
-			int index, struct acpi_gpio_info *info)
+acpi_find_gpio(struct device *dev, const char *con_id,
+	       unsigned int idx, enum gpiod_flags flags,
+	       enum gpio_lookup_flags *lookupflags)
 {
-	return ERR_PTR(-ENOSYS);
+	return ERR_PTR(-ENOENT);
 }
 static inline struct gpio_desc *
 acpi_node_get_gpiod(struct fwnode_handle *fwnode, const char *propname,
@@ -137,9 +167,6 @@ static inline bool acpi_can_fallback_to_crs(struct acpi_device *adev,
 }
 #endif
 
-struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
-		   const char *list_name, int index, enum of_gpio_flags *flags);
-
 struct gpio_desc *gpiochip_get_desc(struct gpio_chip *chip, u16 hwnum);
 void gpiod_set_array_value_complex(bool raw, bool can_sleep,
 				   unsigned int array_size,
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index fc357319de35..483059a22b1b 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -108,33 +108,13 @@ config DRM_KMS_CMA_HELPER
 
 source "drivers/gpu/drm/i2c/Kconfig"
 
-config DRM_TDFX
-	tristate "3dfx Banshee/Voodoo3+"
-	depends on DRM && PCI
-	help
-	  Choose this option if you have a 3dfx Banshee or Voodoo3 (or later),
-	  graphics card.  If M is selected, the module will be called tdfx.
-
 source "drivers/gpu/drm/arm/Kconfig"
 
-config DRM_R128
-	tristate "ATI Rage 128"
-	depends on DRM && PCI
-	select FW_LOADER
-	help
-	  Choose this option if you have an ATI Rage 128 graphics card.  If M
-	  is selected, the module will be called r128.  AGP support for
-	  this card is strongly suggested (unless you have a PCI version).
-
 config DRM_RADEON
 	tristate "ATI Radeon"
 	depends on DRM && PCI
-	select FB_CFB_FILLRECT
-	select FB_CFB_COPYAREA
-	select FB_CFB_IMAGEBLIT
 	select FW_LOADER
         select DRM_KMS_HELPER
-	select DRM_KMS_FB_HELPER
         select DRM_TTM
 	select POWER_SUPPLY
 	select HWMON
@@ -153,12 +133,8 @@ source "drivers/gpu/drm/radeon/Kconfig"
 config DRM_AMDGPU
 	tristate "AMD GPU"
 	depends on DRM && PCI
-	select FB_CFB_FILLRECT
-	select FB_CFB_COPYAREA
-	select FB_CFB_IMAGEBLIT
 	select FW_LOADER
         select DRM_KMS_HELPER
-	select DRM_KMS_FB_HELPER
         select DRM_TTM
 	select POWER_SUPPLY
 	select HWMON
@@ -171,55 +147,11 @@ config DRM_AMDGPU
 	  If M is selected, the module will be called amdgpu.
 
 source "drivers/gpu/drm/amd/amdgpu/Kconfig"
-source "drivers/gpu/drm/amd/powerplay/Kconfig"
-
-source "drivers/gpu/drm/amd/acp/Kconfig"
 
 source "drivers/gpu/drm/nouveau/Kconfig"
 
-config DRM_I810
-	tristate "Intel I810"
-	# !PREEMPT because of missing ioctl locking
-	depends on DRM && AGP && AGP_INTEL && (!PREEMPT || BROKEN)
-	help
-	  Choose this option if you have an Intel I810 graphics card.  If M is
-	  selected, the module will be called i810.  AGP support is required
-	  for this driver to work.
-
 source "drivers/gpu/drm/i915/Kconfig"
 
-config DRM_MGA
-	tristate "Matrox g200/g400"
-	depends on DRM && PCI
-	select FW_LOADER
-	help
-	  Choose this option if you have a Matrox G200, G400 or G450 graphics
-	  card.  If M is selected, the module will be called mga.  AGP
-	  support is required for this driver to work.
-
-config DRM_SIS
-	tristate "SiS video cards"
-	depends on DRM && AGP
-	depends on FB_SIS || FB_SIS=n
-	help
-	  Choose this option if you have a SiS 630 or compatible video
-          chipset. If M is selected the module will be called sis. AGP
-          support is required for this driver to work.
-
-config DRM_VIA
-	tristate "Via unichrome video cards"
-	depends on DRM && PCI
-	help
-	  Choose this option if you have a Via unichrome or compatible video
-	  chipset. If M is selected the module will be called via.
-
-config DRM_SAVAGE
-	tristate "Savage video cards"
-	depends on DRM && PCI
-	help
-	  Choose this option if you have a Savage3D/4/SuperSavage/Pro/Twister
-	  chipset. If M is selected the module will be called savage.
-
 config DRM_VGEM
 	tristate "Virtual GEM provider"
 	depends on DRM
@@ -290,3 +222,81 @@ source "drivers/gpu/drm/arc/Kconfig"
 source "drivers/gpu/drm/hisilicon/Kconfig"
 
 source "drivers/gpu/drm/mediatek/Kconfig"
+
+# Keep legacy drivers last
+
+menuconfig DRM_LEGACY
+	bool "Enable legacy drivers (DANGEROUS)"
+	depends on DRM
+	help
+	  Enable legacy DRI1 drivers. Those drivers expose unsafe and dangerous
+	  APIs to user-space, which can be used to circumvent access
+	  restrictions and other security measures. For backwards compatibility
+	  those drivers are still available, but their use is highly
+	  inadvisable and might harm your system.
+
+	  You are recommended to use the safe modeset-only drivers instead, and
+	  perform 3D emulation in user-space.
+
+	  Unless you have strong reasons to go rogue, say "N".
+
+if DRM_LEGACY
+
+config DRM_TDFX
+	tristate "3dfx Banshee/Voodoo3+"
+	depends on DRM && PCI
+	help
+	  Choose this option if you have a 3dfx Banshee or Voodoo3 (or later),
+	  graphics card.  If M is selected, the module will be called tdfx.
+
+config DRM_R128
+	tristate "ATI Rage 128"
+	depends on DRM && PCI
+	select FW_LOADER
+	help
+	  Choose this option if you have an ATI Rage 128 graphics card.  If M
+	  is selected, the module will be called r128.  AGP support for
+	  this card is strongly suggested (unless you have a PCI version).
+
+config DRM_I810
+	tristate "Intel I810"
+	# !PREEMPT because of missing ioctl locking
+	depends on DRM && AGP && AGP_INTEL && (!PREEMPT || BROKEN)
+	help
+	  Choose this option if you have an Intel I810 graphics card.  If M is
+	  selected, the module will be called i810.  AGP support is required
+	  for this driver to work.
+
+config DRM_MGA
+	tristate "Matrox g200/g400"
+	depends on DRM && PCI
+	select FW_LOADER
+	help
+	  Choose this option if you have a Matrox G200, G400 or G450 graphics
+	  card.  If M is selected, the module will be called mga.  AGP
+	  support is required for this driver to work.
+
+config DRM_SIS
+	tristate "SiS video cards"
+	depends on DRM && AGP
+	depends on FB_SIS || FB_SIS=n
+	help
+	  Choose this option if you have a SiS 630 or compatible video
+	  chipset. If M is selected the module will be called sis. AGP
+	  support is required for this driver to work.
+
+config DRM_VIA
+	tristate "Via unichrome video cards"
+	depends on DRM && PCI
+	help
+	  Choose this option if you have a Via unichrome or compatible video
+	  chipset. If M is selected the module will be called via.
+
+config DRM_SAVAGE
+	tristate "Savage video cards"
+	depends on DRM && PCI
+	help
+	  Choose this option if you have a Savage3D/4/SuperSavage/Pro/Twister
+	  chipset. If M is selected the module will be called savage.
+
+endif # DRM_LEGACY
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 0238bf8bc8c3..25c720454017 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -12,7 +12,10 @@ drm-y       :=	drm_auth.o drm_bufs.o drm_cache.o \
 		drm_info.o drm_debugfs.o drm_encoder_slave.o \
 		drm_trace_points.o drm_global.o drm_prime.o \
 		drm_rect.o drm_vma_manager.o drm_flip_work.o \
-		drm_modeset_lock.o drm_atomic.o drm_bridge.o
+		drm_modeset_lock.o drm_atomic.o drm_bridge.o \
+		drm_framebuffer.o drm_connector.o drm_blend.o \
+		drm_encoder.o drm_mode_object.o drm_property.o \
+		drm_plane.o drm_color_mgmt.o
 
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
 drm-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_gem_cma_helper.o
@@ -24,7 +27,7 @@ drm-$(CONFIG_AGP) += drm_agpsupport.o
 drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_probe_helper.o \
 		drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o \
 		drm_kms_helper_common.o drm_dp_dual_mode_helper.o \
-		drm_simple_kms_helper.o drm_blend.o
+		drm_simple_kms_helper.o drm_modeset_helper.o
 
 drm_kms_helper-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o
 drm_kms_helper-$(CONFIG_DRM_FBDEV_EMULATION) += drm_fb_helper.o
@@ -46,7 +49,7 @@ obj-$(CONFIG_DRM_RADEON)+= radeon/
 obj-$(CONFIG_DRM_AMDGPU)+= amd/amdgpu/
 obj-$(CONFIG_DRM_MGA)	+= mga/
 obj-$(CONFIG_DRM_I810)	+= i810/
-obj-$(CONFIG_DRM_I915)  += i915/
+obj-$(CONFIG_DRM_I915)	+= i915/
 obj-$(CONFIG_DRM_MGAG200) += mgag200/
 obj-$(CONFIG_DRM_VC4)  += vc4/
 obj-$(CONFIG_DRM_CIRRUS_QEMU) += cirrus/
diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig
index 7335c0420c70..61360e27715f 100644
--- a/drivers/gpu/drm/amd/amdgpu/Kconfig
+++ b/drivers/gpu/drm/amd/amdgpu/Kconfig
@@ -1,3 +1,10 @@
+config DRM_AMDGPU_SI
+	bool "Enable amdgpu support for SI parts"
+	depends on DRM_AMDGPU
+	help
+	  Choose this option if you want to enable experimental support
+	  for SI asics.
+
 config DRM_AMDGPU_CIK
 	bool "Enable amdgpu support for CIK parts"
 	depends on DRM_AMDGPU
@@ -25,3 +32,4 @@ config DRM_AMDGPU_GART_DEBUGFS
 	  Selecting this option creates a debugfs file to inspect the mapped
 	  pages. Uses more memory for housekeeping, enable only for debugging.
 
+source "drivers/gpu/drm/amd/acp/Kconfig"
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index c7fcdcedaadb..248a05d02917 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -23,13 +23,16 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	amdgpu_pm.o atombios_dp.o amdgpu_afmt.o amdgpu_trace_points.o \
 	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
 	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
-	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o
+	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
+	amdgpu_gtt_mgr.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
 	ci_smc.o ci_dpm.o dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o \
 	amdgpu_amdkfd_gfx_v7.o
 
+amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
+
 amdgpu-y += \
 	vi.o
 
@@ -50,15 +53,13 @@ amdgpu-y += \
 amdgpu-y += \
 	amdgpu_dpm.o \
 	amdgpu_powerplay.o \
-	cz_smc.o cz_dpm.o \
-	tonga_smc.o tonga_dpm.o \
-	fiji_smc.o fiji_dpm.o \
-	iceland_smc.o iceland_dpm.o
+	cz_smc.o cz_dpm.o
 
 # add DCE block
 amdgpu-y += \
 	dce_v10_0.o \
-	dce_v11_0.o
+	dce_v11_0.o \
+	dce_virtual.o
 
 # add GFX block
 amdgpu-y += \
@@ -110,14 +111,10 @@ amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
 amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
 amdgpu-$(CONFIG_MMU_NOTIFIER) += amdgpu_mn.o
 
-ifneq ($(CONFIG_DRM_AMD_POWERPLAY),)
-
 include $(FULL_AMD_PATH)/powerplay/Makefile
 
 amdgpu-y += $(AMD_POWERPLAY_FILES)
 
-endif
-
 obj-$(CONFIG_DRM_AMDGPU)+= amdgpu.o
 
 CFLAGS_amdgpu_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/amd/amdgpu/ObjectID.h b/drivers/gpu/drm/amd/amdgpu/ObjectID.h
index 06192698bd96..b8d66670bb17 100644
--- a/drivers/gpu/drm/amd/amdgpu/ObjectID.h
+++ b/drivers/gpu/drm/amd/amdgpu/ObjectID.h
@@ -90,6 +90,7 @@
 #define ENCODER_OBJECT_ID_INTERNAL_VCE            0x24
 #define ENCODER_OBJECT_ID_INTERNAL_UNIPHY3        0x25
 #define ENCODER_OBJECT_ID_INTERNAL_AMCLK          0x27
+#define ENCODER_OBJECT_ID_VIRTUAL                 0x28
 
 #define ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO    0xFF
 
@@ -119,6 +120,7 @@
 #define CONNECTOR_OBJECT_ID_eDP                   0x14
 #define CONNECTOR_OBJECT_ID_MXM                   0x15
 #define CONNECTOR_OBJECT_ID_LVDS_eDP              0x16
+#define CONNECTOR_OBJECT_ID_VIRTUAL               0x17
 
 /* deleted */
 
@@ -147,6 +149,7 @@
 #define GRAPH_OBJECT_ENUM_ID5                     0x05
 #define GRAPH_OBJECT_ENUM_ID6                     0x06
 #define GRAPH_OBJECT_ENUM_ID7                     0x07
+#define GRAPH_OBJECT_ENUM_VIRTUAL                 0x08
 
 /****************************************************/
 /* Graphics Object ID Bit definition                */
@@ -408,6 +411,10 @@
                                                   GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                   ENCODER_OBJECT_ID_HDMI_ANX9805 << OBJECT_ID_SHIFT)
 
+#define ENCODER_VIRTUAL_ENUM_VIRTUAL            ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
+                                                  GRAPH_OBJECT_ENUM_VIRTUAL << ENUM_ID_SHIFT |\
+                                                  ENCODER_OBJECT_ID_VIRTUAL << OBJECT_ID_SHIFT)
+
 /****************************************************/
 /* Connector Object ID definition - Shared with BIOS */
 /****************************************************/
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 8ebc5f1eb4c0..039b57e4644c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -51,11 +51,13 @@
 #include "amdgpu_ih.h"
 #include "amdgpu_irq.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_ttm.h"
 #include "amdgpu_gds.h"
 #include "amd_powerplay.h"
 #include "amdgpu_acp.h"
 
 #include "gpu_scheduler.h"
+#include "amdgpu_virt.h"
 
 /*
  * Modules parameters.
@@ -63,6 +65,7 @@
 extern int amdgpu_modeset;
 extern int amdgpu_vram_limit;
 extern int amdgpu_gart_size;
+extern int amdgpu_moverate;
 extern int amdgpu_benchmarking;
 extern int amdgpu_testing;
 extern int amdgpu_audio;
@@ -91,6 +94,9 @@ extern unsigned amdgpu_pcie_lane_cap;
 extern unsigned amdgpu_cg_mask;
 extern unsigned amdgpu_pg_mask;
 extern char *amdgpu_disable_cu;
+extern int amdgpu_sclk_deep_sleep_en;
+extern char *amdgpu_virtual_display;
+extern unsigned amdgpu_pp_feature_mask;
 
 #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS	        3000
 #define AMDGPU_MAX_USEC_TIMEOUT			100000	/* 100 ms */
@@ -105,7 +111,7 @@ extern char *amdgpu_disable_cu;
 #define AMDGPU_MAX_RINGS			16
 #define AMDGPU_MAX_GFX_RINGS			1
 #define AMDGPU_MAX_COMPUTE_RINGS		8
-#define AMDGPU_MAX_VCE_RINGS			2
+#define AMDGPU_MAX_VCE_RINGS			3
 
 /* max number of IP instances */
 #define AMDGPU_MAX_SDMA_INSTANCES		2
@@ -248,10 +254,9 @@ struct amdgpu_vm_pte_funcs {
 			 uint64_t pe, uint64_t src,
 			 unsigned count);
 	/* write pte one entry at a time with addr mapping */
-	void (*write_pte)(struct amdgpu_ib *ib,
-			  const dma_addr_t *pages_addr, uint64_t pe,
-			  uint64_t addr, unsigned count,
-			  uint32_t incr, uint32_t flags);
+	void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe,
+			  uint64_t value, unsigned count,
+			  uint32_t incr);
 	/* for linear pte/pde updates without addr mapping */
 	void (*set_pte_pde)(struct amdgpu_ib *ib,
 			    uint64_t pe,
@@ -316,6 +321,10 @@ struct amdgpu_ring_funcs {
 	/* note usage for clock and power gating */
 	void (*begin_use)(struct amdgpu_ring *ring);
 	void (*end_use)(struct amdgpu_ring *ring);
+	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
+	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
+	unsigned (*get_emit_ib_size) (struct amdgpu_ring *ring);
+	unsigned (*get_dma_frame_size) (struct amdgpu_ring *ring);
 };
 
 /*
@@ -396,46 +405,8 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
 unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring);
 
 /*
- * TTM.
+ * BO.
  */
-
-#define AMDGPU_TTM_LRU_SIZE	20
-
-struct amdgpu_mman_lru {
-	struct list_head		*lru[TTM_NUM_MEM_TYPES];
-	struct list_head		*swap_lru;
-};
-
-struct amdgpu_mman {
-	struct ttm_bo_global_ref        bo_global_ref;
-	struct drm_global_reference	mem_global_ref;
-	struct ttm_bo_device		bdev;
-	bool				mem_global_referenced;
-	bool				initialized;
-
-#if defined(CONFIG_DEBUG_FS)
-	struct dentry			*vram;
-	struct dentry			*gtt;
-#endif
-
-	/* buffer handling */
-	const struct amdgpu_buffer_funcs	*buffer_funcs;
-	struct amdgpu_ring			*buffer_funcs_ring;
-	/* Scheduler entity for buffer moves */
-	struct amd_sched_entity			entity;
-
-	/* custom LRU management */
-	struct amdgpu_mman_lru			log2_size[AMDGPU_TTM_LRU_SIZE];
-};
-
-int amdgpu_copy_buffer(struct amdgpu_ring *ring,
-		       uint64_t src_offset,
-		       uint64_t dst_offset,
-		       uint32_t byte_count,
-		       struct reservation_object *resv,
-		       struct fence **fence);
-int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
-
 struct amdgpu_bo_list_entry {
 	struct amdgpu_bo		*robj;
 	struct ttm_validate_buffer	tv;
@@ -474,8 +445,6 @@ struct amdgpu_bo_va {
 #define AMDGPU_GEM_DOMAIN_MAX		0x3
 
 struct amdgpu_bo {
-	/* Protected by gem.mutex */
-	struct list_head		list;
 	/* Protected by tbo.reserved */
 	u32				prefered_domains;
 	u32				allowed_domains;
@@ -498,10 +467,12 @@ struct amdgpu_bo {
 	struct amdgpu_device		*adev;
 	struct drm_gem_object		gem_base;
 	struct amdgpu_bo		*parent;
+	struct amdgpu_bo		*shadow;
 
 	struct ttm_bo_kmap_obj		dma_buf_vmap;
 	struct amdgpu_mn		*mn;
 	struct list_head		mn_list;
+	struct list_head		shadow_list;
 };
 #define gem_to_amdgpu_bo(gobj) container_of((gobj), struct amdgpu_bo, gem_base)
 
@@ -646,11 +617,12 @@ int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev);
 void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev);
 int amdgpu_gart_init(struct amdgpu_device *adev);
 void amdgpu_gart_fini(struct amdgpu_device *adev);
-void amdgpu_gart_unbind(struct amdgpu_device *adev, unsigned offset,
+void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
 			int pages);
-int amdgpu_gart_bind(struct amdgpu_device *adev, unsigned offset,
+int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
 		     int pages, struct page **pagelist,
 		     dma_addr_t *dma_addr, uint32_t flags);
+int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);
 
 /*
  * GPU MC structures, functions & helpers
@@ -677,6 +649,8 @@ struct amdgpu_mc {
 	uint32_t                fw_version;
 	struct amdgpu_irq_src	vm_fault;
 	uint32_t		vram_type;
+	uint32_t                srbm_soft_reset;
+	struct amdgpu_mode_mc_save save;
 };
 
 /*
@@ -721,13 +695,14 @@ void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
  */
 
 struct amdgpu_flip_work {
-	struct work_struct		flip_work;
+	struct delayed_work		flip_work;
 	struct work_struct		unpin_work;
 	struct amdgpu_device		*adev;
 	int				crtc_id;
+	u32				target_vblank;
 	uint64_t			base;
 	struct drm_pending_vblank_event *event;
-	struct amdgpu_bo		*old_rbo;
+	struct amdgpu_bo		*old_abo;
 	struct fence			*excl;
 	unsigned			shared_count;
 	struct fence			**shared;
@@ -815,13 +790,17 @@ struct amdgpu_ring {
 /* maximum number of VMIDs */
 #define AMDGPU_NUM_VM	16
 
+/* Maximum number of PTEs the hardware can write with one command */
+#define AMDGPU_VM_MAX_UPDATE_SIZE	0x3FFFF
+
 /* number of entries in page table */
 #define AMDGPU_VM_PTE_COUNT (1 << amdgpu_vm_block_size)
 
 /* PTBs (Page Table Blocks) need to be aligned to 32K */
 #define AMDGPU_VM_PTB_ALIGN_SIZE   32768
-#define AMDGPU_VM_PTB_ALIGN_MASK (AMDGPU_VM_PTB_ALIGN_SIZE - 1)
-#define AMDGPU_VM_PTB_ALIGN(a) (((a) + AMDGPU_VM_PTB_ALIGN_MASK) & ~AMDGPU_VM_PTB_ALIGN_MASK)
+
+/* LOG2 number of continuous pages for the fragment field */
+#define AMDGPU_LOG2_PAGES_PER_FRAG 4
 
 #define AMDGPU_PTE_VALID	(1 << 0)
 #define AMDGPU_PTE_SYSTEM	(1 << 1)
@@ -833,10 +812,7 @@ struct amdgpu_ring {
 #define AMDGPU_PTE_READABLE	(1 << 5)
 #define AMDGPU_PTE_WRITEABLE	(1 << 6)
 
-/* PTE (Page Table Entry) fragment field for different page sizes */
-#define AMDGPU_PTE_FRAG_4KB	(0 << 7)
-#define AMDGPU_PTE_FRAG_64KB	(4 << 7)
-#define AMDGPU_LOG2_PAGES_PER_FRAG 4
+#define AMDGPU_PTE_FRAG(x)	((x & 0x1f) << 7)
 
 /* How to programm VM fault handling */
 #define AMDGPU_VM_FAULT_STOP_NEVER	0
@@ -846,6 +822,7 @@ struct amdgpu_ring {
 struct amdgpu_vm_pt {
 	struct amdgpu_bo_list_entry	entry;
 	uint64_t			addr;
+	uint64_t			shadow_addr;
 };
 
 struct amdgpu_vm {
@@ -948,7 +925,6 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 		      struct amdgpu_job *job);
 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job);
 void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id);
-uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr);
 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
 				    struct amdgpu_vm *vm);
 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
@@ -957,7 +933,7 @@ int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			     struct amdgpu_sync *sync);
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			struct amdgpu_bo_va *bo_va,
-			struct ttm_mem_reg *mem);
+			bool clear);
 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
 			     struct amdgpu_bo *bo);
 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
@@ -992,6 +968,7 @@ struct amdgpu_ctx {
 	spinlock_t		ring_lock;
 	struct fence            **fences;
 	struct amdgpu_ctx_ring	rings[AMDGPU_MAX_RINGS];
+	bool preamble_presented;
 };
 
 struct amdgpu_ctx_mgr {
@@ -1195,6 +1172,10 @@ struct amdgpu_gfx {
 	unsigned			ce_ram_size;
 	struct amdgpu_cu_info		cu_info;
 	const struct amdgpu_gfx_funcs	*funcs;
+
+	/* reset mask */
+	uint32_t                        grbm_soft_reset;
+	uint32_t                        srbm_soft_reset;
 };
 
 int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
@@ -1247,11 +1228,16 @@ struct amdgpu_cs_parser {
 	struct fence			*fence;
 	uint64_t			bytes_moved_threshold;
 	uint64_t			bytes_moved;
+	struct amdgpu_bo_list_entry	*evictable;
 
 	/* user fence */
 	struct amdgpu_bo_list_entry	uf_entry;
 };
 
+#define AMDGPU_PREAMBLE_IB_PRESENT          (1 << 0) /* bit set means command submit involves a preamble IB */
+#define AMDGPU_PREAMBLE_IB_PRESENT_FIRST    (1 << 1) /* bit set means preamble IB is first presented in belonging context */
+#define AMDGPU_HAVE_CTX_SWITCH              (1 << 2) /* bit set means context switch occured */
+
 struct amdgpu_job {
 	struct amd_sched_job    base;
 	struct amdgpu_device	*adev;
@@ -1260,9 +1246,10 @@ struct amdgpu_job {
 	struct amdgpu_sync	sync;
 	struct amdgpu_ib	*ibs;
 	struct fence		*fence; /* the hw fence */
+	uint32_t		preamble_status;
 	uint32_t		num_ibs;
 	void			*owner;
-	uint64_t		ctx;
+	uint64_t		fence_ctx; /* the fence_context this job uses */
 	bool                    vm_needs_flush;
 	unsigned		vm_id;
 	uint64_t		vm_pd_addr;
@@ -1683,6 +1670,7 @@ struct amdgpu_uvd {
 	bool			address_64_bit;
 	bool			use_ctx_buf;
 	struct amd_sched_entity entity;
+	uint32_t                srbm_soft_reset;
 };
 
 /*
@@ -1709,6 +1697,8 @@ struct amdgpu_vce {
 	struct amdgpu_irq_src	irq;
 	unsigned		harvest_config;
 	struct amd_sched_entity	entity;
+	uint32_t                srbm_soft_reset;
+	unsigned		num_rings;
 };
 
 /*
@@ -1726,9 +1716,14 @@ struct amdgpu_sdma_instance {
 
 struct amdgpu_sdma {
 	struct amdgpu_sdma_instance instance[AMDGPU_MAX_SDMA_INSTANCES];
+#ifdef CONFIG_DRM_AMDGPU_SI
+	//SI DMA has a difference trap irq number for the second engine
+	struct amdgpu_irq_src	trap_irq_1;
+#endif
 	struct amdgpu_irq_src	trap_irq;
 	struct amdgpu_irq_src	illegal_inst_irq;
 	int			num_instances;
+	uint32_t                    srbm_soft_reset;
 };
 
 /*
@@ -1830,6 +1825,7 @@ struct amdgpu_asic_funcs {
 	bool (*read_disabled_bios)(struct amdgpu_device *adev);
 	bool (*read_bios_from_rom)(struct amdgpu_device *adev,
 				   u8 *bios, u32 length_bytes);
+	void (*detect_hw_virtualization) (struct amdgpu_device *adev);
 	int (*read_register)(struct amdgpu_device *adev, u32 se_num,
 			     u32 sh_num, u32 reg_offset, u32 *value);
 	void (*set_vga_state)(struct amdgpu_device *adev, bool state);
@@ -1839,8 +1835,9 @@ struct amdgpu_asic_funcs {
 	/* MM block clocks */
 	int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk);
 	int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk);
-	/* query virtual capabilities */
-	u32 (*get_virtual_caps)(struct amdgpu_device *adev);
+	/* static power management */
+	int (*get_pcie_lanes)(struct amdgpu_device *adev);
+	void (*set_pcie_lanes)(struct amdgpu_device *adev, int lanes);
 };
 
 /*
@@ -1933,16 +1930,6 @@ struct amdgpu_atcs {
 struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev);
 void amdgpu_cgs_destroy_device(struct cgs_device *cgs_device);
 
-
-/* GPU virtualization */
-#define AMDGPU_VIRT_CAPS_SRIOV_EN       (1 << 0)
-#define AMDGPU_VIRT_CAPS_IS_VF          (1 << 1)
-struct amdgpu_virtualization {
-	bool supports_sr_iov;
-	bool is_virtual;
-	u32 caps;
-};
-
 /*
  * Core structure, functions and helpers.
  */
@@ -1956,6 +1943,8 @@ struct amdgpu_ip_block_status {
 	bool valid;
 	bool sw;
 	bool hw;
+	bool late_initialized;
+	bool hang;
 };
 
 struct amdgpu_device {
@@ -2014,6 +2003,8 @@ struct amdgpu_device {
 	spinlock_t pcie_idx_lock;
 	amdgpu_rreg_t			pcie_rreg;
 	amdgpu_wreg_t			pcie_wreg;
+	amdgpu_rreg_t			pciep_rreg;
+	amdgpu_wreg_t			pciep_wreg;
 	/* protects concurrent UVD register access */
 	spinlock_t uvd_ctx_idx_lock;
 	amdgpu_rreg_t			uvd_ctx_rreg;
@@ -2054,7 +2045,16 @@ struct amdgpu_device {
 	atomic64_t			num_evictions;
 	atomic_t			gpu_reset_counter;
 
+	/* data for buffer migration throttling */
+	struct {
+		spinlock_t		lock;
+		s64			last_update_us;
+		s64			accum_us; /* accumulated microseconds */
+		u32			log2_max_MBps;
+	} mm_stats;
+
 	/* display */
+	bool				enable_virtual_display;
 	struct amdgpu_mode_info		mode_info;
 	struct work_struct		hotplug_work;
 	struct amdgpu_irq_src		crtc_irq;
@@ -2117,6 +2117,14 @@ struct amdgpu_device {
 	struct kfd_dev          *kfd;
 
 	struct amdgpu_virtualization virtualization;
+
+	/* link all shadow bo */
+	struct list_head                shadow_list;
+	struct mutex                    shadow_list_lock;
+	/* link all gtt */
+	spinlock_t			gtt_list_lock;
+	struct list_head                gtt_list;
+
 };
 
 bool amdgpu_device_is_px(struct drm_device *dev);
@@ -2149,6 +2157,8 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
 #define REG_GET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
 #define RREG32_PCIE(reg) adev->pcie_rreg(adev, (reg))
 #define WREG32_PCIE(reg, v) adev->pcie_wreg(adev, (reg), (v))
+#define RREG32_PCIE_PORT(reg) adev->pciep_rreg(adev, (reg))
+#define WREG32_PCIE_PORT(reg, v) adev->pciep_wreg(adev, (reg), (v))
 #define RREG32_SMC(reg) adev->smc_rreg(adev, (reg))
 #define WREG32_SMC(reg, v) adev->smc_wreg(adev, (reg), (v))
 #define RREG32_UVD_CTX(reg) adev->uvd_ctx_rreg(adev, (reg))
@@ -2192,6 +2202,9 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
 #define REG_GET_FIELD(value, reg, field)				\
 	(((value) & REG_FIELD_MASK(reg, field)) >> REG_FIELD_SHIFT(reg, field))
 
+#define WREG32_FIELD(reg, field, val)	\
+	WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
+
 /*
  * BIOS helpers.
  */
@@ -2235,14 +2248,17 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev))
 #define amdgpu_asic_set_uvd_clocks(adev, v, d) (adev)->asic_funcs->set_uvd_clocks((adev), (v), (d))
 #define amdgpu_asic_set_vce_clocks(adev, ev, ec) (adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
-#define amdgpu_asic_get_virtual_caps(adev) ((adev)->asic_funcs->get_virtual_caps((adev)))
+#define amdgpu_get_pcie_lanes(adev) (adev)->asic_funcs->get_pcie_lanes((adev))
+#define amdgpu_set_pcie_lanes(adev, l) (adev)->asic_funcs->set_pcie_lanes((adev), (l))
+#define amdgpu_asic_get_gpu_clock_counter(adev) (adev)->asic_funcs->get_gpu_clock_counter((adev))
 #define amdgpu_asic_read_disabled_bios(adev) (adev)->asic_funcs->read_disabled_bios((adev))
 #define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
+#define amdgpu_asic_detect_hw_virtualization(adev) (adev)->asic_funcs->detect_hw_virtualization((adev))
 #define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))
 #define amdgpu_gart_flush_gpu_tlb(adev, vmid) (adev)->gart.gart_funcs->flush_gpu_tlb((adev), (vmid))
 #define amdgpu_gart_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gart.gart_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))
 #define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))
-#define amdgpu_vm_write_pte(adev, ib, pa, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pa), (pe), (addr), (count), (incr), (flags)))
+#define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
 #define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
 #define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib)))
 #define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r))
@@ -2257,9 +2273,13 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))
 #define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))
 #define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs->emit_hdp_invalidate((r))
+#define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
+#define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
 #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
+#define amdgpu_ring_get_emit_ib_size(r) (r)->funcs->get_emit_ib_size((r))
+#define amdgpu_ring_get_dma_frame_size(r) (r)->funcs->get_dma_frame_size((r))
 #define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))
 #define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))
 #define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))
@@ -2291,6 +2311,11 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))
 #define amdgpu_gfx_select_se_sh(adev, se, sh, instance) (adev)->gfx.funcs->select_se_sh((adev), (se), (sh), (instance))
 
+#define amdgpu_dpm_read_sensor(adev, idx, value) \
+	((adev)->pp_enabled ? \
+		(adev)->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, (idx), (value)) : \
+		-EINVAL)
+
 #define amdgpu_dpm_get_temperature(adev) \
 	((adev)->pp_enabled ?						\
 	      (adev)->powerplay.pp_funcs->get_temperature((adev)->powerplay.pp_handle) : \
@@ -2342,11 +2367,6 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 	      (adev)->powerplay.pp_funcs->powergate_vce((adev)->powerplay.pp_handle, (g)) : \
 	      (adev)->pm.funcs->powergate_vce((adev), (g)))
 
-#define amdgpu_dpm_debugfs_print_current_performance_level(adev, m) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->print_current_performance_level((adev)->powerplay.pp_handle, (m)) : \
-	      (adev)->pm.funcs->debugfs_print_current_performance_level((adev), (m)))
-
 #define amdgpu_dpm_get_current_power_state(adev) \
 	(adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle)
 
@@ -2387,6 +2407,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 
 /* Common functions */
 int amdgpu_gpu_reset(struct amdgpu_device *adev);
+bool amdgpu_need_backup(struct amdgpu_device *adev);
 void amdgpu_pci_config_reset(struct amdgpu_device *adev);
 bool amdgpu_card_posted(struct amdgpu_device *adev);
 void amdgpu_update_display_priority(struct amdgpu_device *adev);
@@ -2395,7 +2416,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data);
 int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
 		       u32 ip_instance, u32 ring,
 		       struct amdgpu_ring **out_ring);
-void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *rbo, u32 domain);
+void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
 bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
@@ -2412,6 +2433,10 @@ uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);
 void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
 void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);
+u64 amdgpu_ttm_get_gtt_mem_size(struct amdgpu_device *adev);
+int amdgpu_ttm_global_init(struct amdgpu_device *adev);
+int amdgpu_ttm_init(struct amdgpu_device *adev);
+void amdgpu_ttm_fini(struct amdgpu_device *adev);
 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
 					     const u32 *registers,
 					     const u32 array_size);
@@ -2423,11 +2448,13 @@ void amdgpu_register_atpx_handler(void);
 void amdgpu_unregister_atpx_handler(void);
 bool amdgpu_has_atpx_dgpu_power_cntl(void);
 bool amdgpu_is_atpx_hybrid(void);
+bool amdgpu_atpx_dgpu_req_power_for_displays(void);
 #else
 static inline void amdgpu_register_atpx_handler(void) {}
 static inline void amdgpu_unregister_atpx_handler(void) {}
 static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; }
 static inline bool amdgpu_is_atpx_hybrid(void) { return false; }
+static inline bool amdgpu_atpx_dgpu_req_power_for_displays(void) { return false; }
 #endif
 
 /*
@@ -2444,8 +2471,8 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 				 struct drm_file *file_priv);
 void amdgpu_driver_preclose_kms(struct drm_device *dev,
 				struct drm_file *file_priv);
-int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon);
-int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon);
+int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon);
+int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon);
 u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe);
 int amdgpu_enable_vblank_kms(struct drm_device *dev, unsigned int pipe);
 void amdgpu_disable_vblank_kms(struct drm_device *dev, unsigned int pipe);
@@ -2491,6 +2518,7 @@ static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
 struct amdgpu_bo_va_mapping *
 amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
 		       uint64_t addr, struct amdgpu_bo **bo);
+int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser);
 
 #include "amdgpu_object.h"
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index 5cd7b736a9de..5796539a0bcb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -25,6 +25,7 @@
 #include <linux/acpi.h>
 #include <linux/slab.h>
 #include <linux/power_supply.h>
+#include <linux/pm_runtime.h>
 #include <acpi/video.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
@@ -333,6 +334,16 @@ int amdgpu_atif_handler(struct amdgpu_device *adev,
 #endif
 		}
 	}
+	if (req.pending & ATIF_DGPU_DISPLAY_EVENT) {
+		if ((adev->flags & AMD_IS_PX) &&
+		    amdgpu_atpx_dgpu_req_power_for_displays()) {
+			pm_runtime_get_sync(adev->ddev->dev);
+			/* Just fire off a uevent and let userspace tell us what to do */
+			drm_helper_hpd_irq_event(adev->ddev);
+			pm_runtime_mark_last_busy(adev->ddev->dev);
+			pm_runtime_put_autosuspend(adev->ddev->dev);
+		}
+	}
 	/* TODO: check other events */
 
 	/* We've handled the event, stop the notifier chain. The ACPI interface
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index d080d0807a5b..dba8a5b25e66 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -143,14 +143,6 @@ int amdgpu_amdkfd_resume(struct amdgpu_device *rdev)
 	return r;
 }
 
-u32 pool_to_domain(enum kgd_memory_pool p)
-{
-	switch (p) {
-	case KGD_POOL_FRAMEBUFFER: return AMDGPU_GEM_DOMAIN_VRAM;
-	default: return AMDGPU_GEM_DOMAIN_GTT;
-	}
-}
-
 int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 			void **mem_obj, uint64_t *gpu_addr,
 			void **cpu_ptr)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index 362bedc9e507..1a0a5f7cccbc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -103,11 +103,11 @@ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
 				uint32_t pipe_id, uint32_t queue_id);
 
 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
-				unsigned int timeout, uint32_t pipe_id,
+				unsigned int utimeout, uint32_t pipe_id,
 				uint32_t queue_id);
 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
-				unsigned int timeout);
+				unsigned int utimeout);
 static int kgd_address_watch_disable(struct kgd_dev *kgd);
 static int kgd_address_watch_execute(struct kgd_dev *kgd,
 					unsigned int watch_point_id,
@@ -437,11 +437,12 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
 }
 
 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
-				unsigned int timeout, uint32_t pipe_id,
+				unsigned int utimeout, uint32_t pipe_id,
 				uint32_t queue_id)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 	uint32_t temp;
+	int timeout = utimeout;
 
 	acquire_queue(kgd, pipe_id, queue_id);
 	WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
@@ -452,9 +453,8 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
 		temp = RREG32(mmCP_HQD_ACTIVE);
 		if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
 			break;
-		if (timeout == 0) {
-			pr_err("kfd: cp queue preemption time out (%dms)\n",
-				temp);
+		if (timeout <= 0) {
+			pr_err("kfd: cp queue preemption time out.\n");
 			release_queue(kgd);
 			return -ETIME;
 		}
@@ -467,12 +467,13 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
 }
 
 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
-				unsigned int timeout)
+				unsigned int utimeout)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 	struct cik_sdma_rlc_registers *m;
 	uint32_t sdma_base_addr;
 	uint32_t temp;
+	int timeout = utimeout;
 
 	m = get_sdma_mqd(mqd);
 	sdma_base_addr = get_sdma_base_addr(m);
@@ -485,7 +486,7 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
 		temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
 		if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT)
 			break;
-		if (timeout == 0)
+		if (timeout <= 0)
 			return -ETIME;
 		msleep(20);
 		timeout -= 20;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index 04b744d64b57..6697612239c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -62,10 +62,10 @@ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
 		uint32_t pipe_id, uint32_t queue_id);
 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
-				unsigned int timeout, uint32_t pipe_id,
+				unsigned int utimeout, uint32_t pipe_id,
 				uint32_t queue_id);
 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
-				unsigned int timeout);
+				unsigned int utimeout);
 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
 static int kgd_address_watch_disable(struct kgd_dev *kgd);
 static int kgd_address_watch_execute(struct kgd_dev *kgd,
@@ -349,11 +349,12 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
 }
 
 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
-				unsigned int timeout, uint32_t pipe_id,
+				unsigned int utimeout, uint32_t pipe_id,
 				uint32_t queue_id)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 	uint32_t temp;
+	int timeout = utimeout;
 
 	acquire_queue(kgd, pipe_id, queue_id);
 
@@ -363,9 +364,8 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
 		temp = RREG32(mmCP_HQD_ACTIVE);
 		if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
 			break;
-		if (timeout == 0) {
-			pr_err("kfd: cp queue preemption time out (%dms)\n",
-				temp);
+		if (timeout <= 0) {
+			pr_err("kfd: cp queue preemption time out.\n");
 			release_queue(kgd);
 			return -ETIME;
 		}
@@ -378,12 +378,13 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
 }
 
 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
-				unsigned int timeout)
+				unsigned int utimeout)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 	struct cik_sdma_rlc_registers *m;
 	uint32_t sdma_base_addr;
 	uint32_t temp;
+	int timeout = utimeout;
 
 	m = get_sdma_mqd(mqd);
 	sdma_base_addr = get_sdma_base_addr(m);
@@ -396,7 +397,7 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
 		temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
 		if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT)
 			break;
-		if (timeout == 0)
+		if (timeout <= 0)
 			return -ETIME;
 		msleep(20);
 		timeout -= 20;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index 983175363b06..8e6bf548d689 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -259,6 +259,33 @@ static const int object_connector_convert[] = {
 	DRM_MODE_CONNECTOR_Unknown
 };
 
+bool amdgpu_atombios_has_dce_engine_info(struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	struct atom_context *ctx = mode_info->atom_context;
+	int index = GetIndexIntoMasterTable(DATA, Object_Header);
+	u16 size, data_offset;
+	u8 frev, crev;
+	ATOM_DISPLAY_OBJECT_PATH_TABLE *path_obj;
+	ATOM_OBJECT_HEADER *obj_header;
+
+	if (!amdgpu_atom_parse_data_header(ctx, index, &size, &frev, &crev, &data_offset))
+		return false;
+
+	if (crev < 2)
+		return false;
+
+	obj_header = (ATOM_OBJECT_HEADER *) (ctx->bios + data_offset);
+	path_obj = (ATOM_DISPLAY_OBJECT_PATH_TABLE *)
+	    (ctx->bios + data_offset +
+	     le16_to_cpu(obj_header->usDisplayPathTableOffset));
+
+	if (path_obj->ucNumOfDispPath)
+		return true;
+	else
+		return false;
+}
+
 bool amdgpu_atombios_get_connector_info_from_object_table(struct amdgpu_device *adev)
 {
 	struct amdgpu_mode_info *mode_info = &adev->mode_info;
@@ -321,6 +348,19 @@ bool amdgpu_atombios_get_connector_info_from_object_table(struct amdgpu_device *
 			    (le16_to_cpu(path->usConnObjectId) &
 			     OBJECT_TYPE_MASK) >> OBJECT_TYPE_SHIFT;
 
+			/* Skip TV/CV support */
+			if ((le16_to_cpu(path->usDeviceTag) ==
+			     ATOM_DEVICE_TV1_SUPPORT) ||
+			    (le16_to_cpu(path->usDeviceTag) ==
+			     ATOM_DEVICE_CV_SUPPORT))
+				continue;
+
+			if (con_obj_id >= ARRAY_SIZE(object_connector_convert)) {
+				DRM_ERROR("invalid con_obj_id %d for device tag 0x%04x\n",
+					  con_obj_id, le16_to_cpu(path->usDeviceTag));
+				continue;
+			}
+
 			connector_type =
 				object_connector_convert[con_obj_id];
 			connector_object_id = con_obj_id;
@@ -951,6 +991,48 @@ int amdgpu_atombios_get_clock_dividers(struct amdgpu_device *adev,
 		return -EINVAL;
 
 	switch (crev) {
+	case 2:
+	case 3:
+	case 5:
+		/* r6xx, r7xx, evergreen, ni, si.
+		 * TODO: add support for asic_type <= CHIP_RV770*/
+		if (clock_type == COMPUTE_ENGINE_PLL_PARAM) {
+			args.v3.ulClockParams = cpu_to_le32((clock_type << 24) | clock);
+
+			amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
+
+			dividers->post_div = args.v3.ucPostDiv;
+			dividers->enable_post_div = (args.v3.ucCntlFlag &
+						     ATOM_PLL_CNTL_FLAG_PLL_POST_DIV_EN) ? true : false;
+			dividers->enable_dithen = (args.v3.ucCntlFlag &
+						   ATOM_PLL_CNTL_FLAG_FRACTION_DISABLE) ? false : true;
+			dividers->whole_fb_div = le16_to_cpu(args.v3.ulFbDiv.usFbDiv);
+			dividers->frac_fb_div = le16_to_cpu(args.v3.ulFbDiv.usFbDivFrac);
+			dividers->ref_div = args.v3.ucRefDiv;
+			dividers->vco_mode = (args.v3.ucCntlFlag &
+					      ATOM_PLL_CNTL_FLAG_MPLL_VCO_MODE) ? 1 : 0;
+		} else {
+			/* for SI we use ComputeMemoryClockParam for memory plls */
+			if (adev->asic_type >= CHIP_TAHITI)
+				return -EINVAL;
+			args.v5.ulClockParams = cpu_to_le32((clock_type << 24) | clock);
+			if (strobe_mode)
+				args.v5.ucInputFlag = ATOM_PLL_INPUT_FLAG_PLL_STROBE_MODE_EN;
+
+			amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
+
+			dividers->post_div = args.v5.ucPostDiv;
+			dividers->enable_post_div = (args.v5.ucCntlFlag &
+						     ATOM_PLL_CNTL_FLAG_PLL_POST_DIV_EN) ? true : false;
+			dividers->enable_dithen = (args.v5.ucCntlFlag &
+						   ATOM_PLL_CNTL_FLAG_FRACTION_DISABLE) ? false : true;
+			dividers->whole_fb_div = le16_to_cpu(args.v5.ulFbDiv.usFbDiv);
+			dividers->frac_fb_div = le16_to_cpu(args.v5.ulFbDiv.usFbDivFrac);
+			dividers->ref_div = args.v5.ucRefDiv;
+			dividers->vco_mode = (args.v5.ucCntlFlag &
+					      ATOM_PLL_CNTL_FLAG_MPLL_VCO_MODE) ? 1 : 0;
+		}
+		break;
 	case 4:
 		/* fusion */
 		args.v4.ulClock = cpu_to_le32(clock);	/* 10 khz */
@@ -1095,6 +1177,32 @@ void amdgpu_atombios_set_engine_dram_timings(struct amdgpu_device *adev,
 	amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
 }
 
+void amdgpu_atombios_get_default_voltages(struct amdgpu_device *adev,
+					  u16 *vddc, u16 *vddci, u16 *mvdd)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
+	u8 frev, crev;
+	u16 data_offset;
+	union firmware_info *firmware_info;
+
+	*vddc = 0;
+	*vddci = 0;
+	*mvdd = 0;
+
+	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
+				   &frev, &crev, &data_offset)) {
+		firmware_info =
+			(union firmware_info *)(mode_info->atom_context->bios +
+						data_offset);
+		*vddc = le16_to_cpu(firmware_info->info_14.usBootUpVDDCVoltage);
+		if ((frev == 2) && (crev >= 2)) {
+			*vddci = le16_to_cpu(firmware_info->info_22.usBootUpVDDCIVoltage);
+			*mvdd = le16_to_cpu(firmware_info->info_22.usBootUpMVDDCVoltage);
+		}
+	}
+}
+
 union set_voltage {
 	struct _SET_VOLTAGE_PS_ALLOCATION alloc;
 	struct _SET_VOLTAGE_PARAMETERS v1;
@@ -1102,6 +1210,52 @@ union set_voltage {
 	struct _SET_VOLTAGE_PARAMETERS_V1_3 v3;
 };
 
+int amdgpu_atombios_get_max_vddc(struct amdgpu_device *adev, u8 voltage_type,
+			     u16 voltage_id, u16 *voltage)
+{
+	union set_voltage args;
+	int index = GetIndexIntoMasterTable(COMMAND, SetVoltage);
+	u8 frev, crev;
+
+	if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev))
+		return -EINVAL;
+
+	switch (crev) {
+	case 1:
+		return -EINVAL;
+	case 2:
+		args.v2.ucVoltageType = SET_VOLTAGE_GET_MAX_VOLTAGE;
+		args.v2.ucVoltageMode = 0;
+		args.v2.usVoltageLevel = 0;
+
+		amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
+
+		*voltage = le16_to_cpu(args.v2.usVoltageLevel);
+		break;
+	case 3:
+		args.v3.ucVoltageType = voltage_type;
+		args.v3.ucVoltageMode = ATOM_GET_VOLTAGE_LEVEL;
+		args.v3.usVoltageLevel = cpu_to_le16(voltage_id);
+
+		amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
+
+		*voltage = le16_to_cpu(args.v3.usVoltageLevel);
+		break;
+	default:
+		DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int amdgpu_atombios_get_leakage_vddc_based_on_leakage_idx(struct amdgpu_device *adev,
+						      u16 *voltage,
+						      u16 leakage_idx)
+{
+	return amdgpu_atombios_get_max_vddc(adev, VOLTAGE_TYPE_VDDC, leakage_idx, voltage);
+}
+
 void amdgpu_atombios_set_voltage(struct amdgpu_device *adev,
 				 u16 voltage_level,
 				 u8 voltage_type)
@@ -1322,6 +1476,50 @@ static ATOM_VOLTAGE_OBJECT_V3 *amdgpu_atombios_lookup_voltage_object_v3(ATOM_VOL
 	return NULL;
 }
 
+int amdgpu_atombios_get_svi2_info(struct amdgpu_device *adev,
+			      u8 voltage_type,
+			      u8 *svd_gpio_id, u8 *svc_gpio_id)
+{
+	int index = GetIndexIntoMasterTable(DATA, VoltageObjectInfo);
+	u8 frev, crev;
+	u16 data_offset, size;
+	union voltage_object_info *voltage_info;
+	union voltage_object *voltage_object = NULL;
+
+	if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, &size,
+				   &frev, &crev, &data_offset)) {
+		voltage_info = (union voltage_object_info *)
+			(adev->mode_info.atom_context->bios + data_offset);
+
+		switch (frev) {
+		case 3:
+			switch (crev) {
+			case 1:
+				voltage_object = (union voltage_object *)
+					amdgpu_atombios_lookup_voltage_object_v3(&voltage_info->v3,
+								      voltage_type,
+								      VOLTAGE_OBJ_SVID2);
+				if (voltage_object) {
+					*svd_gpio_id = voltage_object->v3.asSVID2Obj.ucSVDGpioId;
+					*svc_gpio_id = voltage_object->v3.asSVID2Obj.ucSVCGpioId;
+				} else {
+					return -EINVAL;
+				}
+				break;
+			default:
+				DRM_ERROR("unknown voltage object table\n");
+				return -EINVAL;
+			}
+			break;
+		default:
+			DRM_ERROR("unknown voltage object table\n");
+			return -EINVAL;
+		}
+
+	}
+	return 0;
+}
+
 bool
 amdgpu_atombios_is_voltage_gpio(struct amdgpu_device *adev,
 				u8 voltage_type, u8 voltage_mode)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
index 8c2e69661799..17356151db38 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
@@ -140,6 +140,8 @@ struct amdgpu_i2c_bus_rec amdgpu_atombios_lookup_i2c_gpio(struct amdgpu_device *
 							  uint8_t id);
 void amdgpu_atombios_i2c_init(struct amdgpu_device *adev);
 
+bool amdgpu_atombios_has_dce_engine_info(struct amdgpu_device *adev);
+
 bool amdgpu_atombios_get_connector_info_from_object_table(struct amdgpu_device *adev);
 
 int amdgpu_atombios_get_clock_info(struct amdgpu_device *adev);
@@ -206,5 +208,19 @@ void amdgpu_atombios_scratch_regs_save(struct amdgpu_device *adev);
 void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev);
 
 void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le);
-
+int amdgpu_atombios_get_max_vddc(struct amdgpu_device *adev, u8 voltage_type,
+			     u16 voltage_id, u16 *voltage);
+int amdgpu_atombios_get_leakage_vddc_based_on_leakage_idx(struct amdgpu_device *adev,
+						      u16 *voltage,
+						      u16 leakage_idx);
+void amdgpu_atombios_get_default_voltages(struct amdgpu_device *adev,
+					  u16 *vddc, u16 *vddci, u16 *mvdd);
+int amdgpu_atombios_get_clock_dividers(struct amdgpu_device *adev,
+				       u8 clock_type,
+				       u32 clock,
+				       bool strobe_mode,
+				       struct atom_clock_dividers *dividers);
+int amdgpu_atombios_get_svi2_info(struct amdgpu_device *adev,
+			      u8 voltage_type,
+			      u8 *svd_gpio_id, u8 *svc_gpio_id);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
index 49de92600074..dae35a96a694 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
@@ -29,6 +29,7 @@ struct amdgpu_atpx {
 	acpi_handle handle;
 	struct amdgpu_atpx_functions functions;
 	bool is_hybrid;
+	bool dgpu_req_power_for_displays;
 };
 
 static struct amdgpu_atpx_priv {
@@ -73,6 +74,10 @@ bool amdgpu_is_atpx_hybrid(void) {
 	return amdgpu_atpx_priv.atpx.is_hybrid;
 }
 
+bool amdgpu_atpx_dgpu_req_power_for_displays(void) {
+	return amdgpu_atpx_priv.atpx.dgpu_req_power_for_displays;
+}
+
 /**
  * amdgpu_atpx_call - call an ATPX method
  *
@@ -200,19 +205,14 @@ static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx)
 	atpx->is_hybrid = false;
 	if (valid_bits & ATPX_MS_HYBRID_GFX_SUPPORTED) {
 		printk("ATPX Hybrid Graphics\n");
-#if 1
-		/* This is a temporary hack until the D3 cold support
-		 * makes it upstream.  The ATPX power_control method seems
-		 * to still work on even if the system should be using
-		 * the new standardized hybrid D3 cold ACPI interface.
-		 */
-		atpx->functions.power_cntl = true;
-#else
 		atpx->functions.power_cntl = false;
-#endif
 		atpx->is_hybrid = true;
 	}
 
+	atpx->dgpu_req_power_for_displays = false;
+	if (valid_bits & ATPX_DGPU_REQ_POWER_FOR_DISPLAYS)
+		atpx->dgpu_req_power_for_displays = true;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index 33e47a43ae32..345305235349 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -39,7 +39,8 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
 	start_jiffies = jiffies;
 	for (i = 0; i < n; i++) {
 		struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
-		r = amdgpu_copy_buffer(ring, saddr, daddr, size, NULL, &fence);
+		r = amdgpu_copy_buffer(ring, saddr, daddr, size, NULL, &fence,
+				       false);
 		if (r)
 			goto exit_do_move;
 		r = fence_wait(fence, false);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index bc0440f7a31d..7a8bfa34682f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -616,7 +616,7 @@ static int amdgpu_cgs_irq_put(struct cgs_device *cgs_device, unsigned src_id, un
 	return amdgpu_irq_put(adev, adev->irq.sources[src_id], type);
 }
 
-int amdgpu_cgs_set_clockgating_state(struct cgs_device *cgs_device,
+static int amdgpu_cgs_set_clockgating_state(struct cgs_device *cgs_device,
 				  enum amd_ip_block_type block_type,
 				  enum amd_clockgating_state state)
 {
@@ -637,7 +637,7 @@ int amdgpu_cgs_set_clockgating_state(struct cgs_device *cgs_device,
 	return r;
 }
 
-int amdgpu_cgs_set_powergating_state(struct cgs_device *cgs_device,
+static int amdgpu_cgs_set_powergating_state(struct cgs_device *cgs_device,
 				  enum amd_ip_block_type block_type,
 				  enum amd_powergating_state state)
 {
@@ -711,6 +711,47 @@ static int amdgpu_cgs_rel_firmware(struct cgs_device *cgs_device, enum cgs_ucode
 	return -EINVAL;
 }
 
+static uint16_t amdgpu_get_firmware_version(struct cgs_device *cgs_device,
+					enum cgs_ucode_id type)
+{
+	CGS_FUNC_ADEV;
+	uint16_t fw_version;
+
+	switch (type) {
+		case CGS_UCODE_ID_SDMA0:
+			fw_version = adev->sdma.instance[0].fw_version;
+			break;
+		case CGS_UCODE_ID_SDMA1:
+			fw_version = adev->sdma.instance[1].fw_version;
+			break;
+		case CGS_UCODE_ID_CP_CE:
+			fw_version = adev->gfx.ce_fw_version;
+			break;
+		case CGS_UCODE_ID_CP_PFP:
+			fw_version = adev->gfx.pfp_fw_version;
+			break;
+		case CGS_UCODE_ID_CP_ME:
+			fw_version = adev->gfx.me_fw_version;
+			break;
+		case CGS_UCODE_ID_CP_MEC:
+			fw_version = adev->gfx.mec_fw_version;
+			break;
+		case CGS_UCODE_ID_CP_MEC_JT1:
+			fw_version = adev->gfx.mec_fw_version;
+			break;
+		case CGS_UCODE_ID_CP_MEC_JT2:
+			fw_version = adev->gfx.mec_fw_version;
+			break;
+		case CGS_UCODE_ID_RLC_G:
+			fw_version = adev->gfx.rlc_fw_version;
+			break;
+		default:
+			DRM_ERROR("firmware type %d do not have version\n", type);
+			fw_version = 0;
+	}
+	return fw_version;
+}
+
 static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 					enum cgs_ucode_id type,
 					struct cgs_firmware_info *info)
@@ -741,6 +782,7 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 		info->mc_addr = gpu_addr;
 		info->image_size = data_size;
 		info->version = (uint16_t)le32_to_cpu(header->header.ucode_version);
+		info->fw_version = amdgpu_get_firmware_version(cgs_device, type);
 		info->feature_version = (uint16_t)le32_to_cpu(header->ucode_feature_version);
 	} else {
 		char fw_name[30] = {0};
@@ -848,6 +890,12 @@ static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device,
 	case CGS_SYSTEM_INFO_GFX_SE_INFO:
 		sys_info->value = adev->gfx.config.max_shader_engines;
 		break;
+	case CGS_SYSTEM_INFO_PCIE_SUB_SYS_ID:
+		sys_info->value = adev->pdev->subsystem_device;
+		break;
+	case CGS_SYSTEM_INFO_PCIE_SUB_SYS_VENDOR_ID:
+		sys_info->value = adev->pdev->subsystem_vendor;
+		break;
 	default:
 		return -ENODEV;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index ff0b55a65ca3..2e3a0543760d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -168,12 +168,12 @@ int amdgpu_connector_get_monitor_bpc(struct drm_connector *connector)
 		}
 
 		/* Any defined maximum tmds clock limit we must not exceed? */
-		if (connector->max_tmds_clock > 0) {
+		if (connector->display_info.max_tmds_clock > 0) {
 			/* mode_clock is clock in kHz for mode to be modeset on this connector */
 			mode_clock = amdgpu_connector->pixelclock_for_modeset;
 
 			/* Maximum allowable input clock in kHz */
-			max_tmds_clock = connector->max_tmds_clock * 1000;
+			max_tmds_clock = connector->display_info.max_tmds_clock;
 
 			DRM_DEBUG("%s: hdmi mode dotclock %d kHz, max tmds input clock %d kHz.\n",
 				  connector->name, mode_clock, max_tmds_clock);
@@ -769,8 +769,10 @@ static void amdgpu_connector_destroy(struct drm_connector *connector)
 {
 	struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-	if (amdgpu_connector->ddc_bus->has_aux)
+	if (amdgpu_connector->ddc_bus->has_aux) {
 		drm_dp_aux_unregister(&amdgpu_connector->ddc_bus->aux);
+		amdgpu_connector->ddc_bus->has_aux = false;
+	}
 	amdgpu_connector_free_edid(connector);
 	kfree(amdgpu_connector->con_priv);
 	drm_connector_unregister(connector);
@@ -1504,6 +1506,88 @@ static const struct drm_connector_funcs amdgpu_connector_edp_funcs = {
 	.force = amdgpu_connector_dvi_force,
 };
 
+static struct drm_encoder *
+amdgpu_connector_virtual_encoder(struct drm_connector *connector)
+{
+	int enc_id = connector->encoder_ids[0];
+	struct drm_encoder *encoder;
+	int i;
+	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
+		if (connector->encoder_ids[i] == 0)
+			break;
+
+		encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]);
+		if (!encoder)
+			continue;
+
+		if (encoder->encoder_type == DRM_MODE_ENCODER_VIRTUAL)
+			return encoder;
+	}
+
+	/* pick the first one */
+	if (enc_id)
+		return drm_encoder_find(connector->dev, enc_id);
+	return NULL;
+}
+
+static int amdgpu_connector_virtual_get_modes(struct drm_connector *connector)
+{
+	struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector);
+
+	if (encoder) {
+		amdgpu_connector_add_common_modes(encoder, connector);
+	}
+
+	return 0;
+}
+
+static int amdgpu_connector_virtual_mode_valid(struct drm_connector *connector,
+					   struct drm_display_mode *mode)
+{
+	return MODE_OK;
+}
+
+static int
+amdgpu_connector_virtual_dpms(struct drm_connector *connector, int mode)
+{
+	return 0;
+}
+
+static enum drm_connector_status
+
+amdgpu_connector_virtual_detect(struct drm_connector *connector, bool force)
+{
+	return connector_status_connected;
+}
+
+static int
+amdgpu_connector_virtual_set_property(struct drm_connector *connector,
+				  struct drm_property *property,
+				  uint64_t val)
+{
+	return 0;
+}
+
+static void amdgpu_connector_virtual_force(struct drm_connector *connector)
+{
+	return;
+}
+
+static const struct drm_connector_helper_funcs amdgpu_connector_virtual_helper_funcs = {
+	.get_modes = amdgpu_connector_virtual_get_modes,
+	.mode_valid = amdgpu_connector_virtual_mode_valid,
+	.best_encoder = amdgpu_connector_virtual_encoder,
+};
+
+static const struct drm_connector_funcs amdgpu_connector_virtual_funcs = {
+	.dpms = amdgpu_connector_virtual_dpms,
+	.detect = amdgpu_connector_virtual_detect,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.set_property = amdgpu_connector_virtual_set_property,
+	.destroy = amdgpu_connector_destroy,
+	.force = amdgpu_connector_virtual_force,
+};
+
 void
 amdgpu_connector_add(struct amdgpu_device *adev,
 		      uint32_t connector_id,
@@ -1888,6 +1972,17 @@ amdgpu_connector_add(struct amdgpu_device *adev,
 			connector->interlace_allowed = false;
 			connector->doublescan_allowed = false;
 			break;
+		case DRM_MODE_CONNECTOR_VIRTUAL:
+			amdgpu_dig_connector = kzalloc(sizeof(struct amdgpu_connector_atom_dig), GFP_KERNEL);
+			if (!amdgpu_dig_connector)
+				goto failed;
+			amdgpu_connector->con_priv = amdgpu_dig_connector;
+			drm_connector_init(dev, &amdgpu_connector->base, &amdgpu_connector_virtual_funcs, connector_type);
+			drm_connector_helper_add(&amdgpu_connector->base, &amdgpu_connector_virtual_helper_funcs);
+			subpixel_order = SubPixelHorizontalRGB;
+			connector->interlace_allowed = false;
+			connector->doublescan_allowed = false;
+			break;
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 0307ff5887c5..b0f6e6957536 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -91,6 +91,7 @@ static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
 				      uint32_t *offset)
 {
 	struct drm_gem_object *gobj;
+	unsigned long size;
 
 	gobj = drm_gem_object_lookup(p->filp, data->handle);
 	if (gobj == NULL)
@@ -101,6 +102,11 @@ static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
 	p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
 	p->uf_entry.tv.shared = true;
 	p->uf_entry.user_pages = NULL;
+
+	size = amdgpu_bo_size(p->uf_entry.robj);
+	if (size != PAGE_SIZE || (data->offset + 8) > size)
+		return -EINVAL;
+
 	*offset = data->offset;
 
 	drm_gem_object_unreference_unlocked(gobj);
@@ -235,70 +241,212 @@ free_chunk:
 	return ret;
 }
 
-/* Returns how many bytes TTM can move per IB.
+/* Convert microseconds to bytes. */
+static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
+{
+	if (us <= 0 || !adev->mm_stats.log2_max_MBps)
+		return 0;
+
+	/* Since accum_us is incremented by a million per second, just
+	 * multiply it by the number of MB/s to get the number of bytes.
+	 */
+	return us << adev->mm_stats.log2_max_MBps;
+}
+
+static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
+{
+	if (!adev->mm_stats.log2_max_MBps)
+		return 0;
+
+	return bytes >> adev->mm_stats.log2_max_MBps;
+}
+
+/* Returns how many bytes TTM can move right now. If no bytes can be moved,
+ * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
+ * which means it can go over the threshold once. If that happens, the driver
+ * will be in debt and no other buffer migrations can be done until that debt
+ * is repaid.
+ *
+ * This approach allows moving a buffer of any size (it's important to allow
+ * that).
+ *
+ * The currency is simply time in microseconds and it increases as the clock
+ * ticks. The accumulated microseconds (us) are converted to bytes and
+ * returned.
  */
 static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
 {
-	u64 real_vram_size = adev->mc.real_vram_size;
-	u64 vram_usage = atomic64_read(&adev->vram_usage);
+	s64 time_us, increment_us;
+	u64 max_bytes;
+	u64 free_vram, total_vram, used_vram;
 
-	/* This function is based on the current VRAM usage.
+	/* Allow a maximum of 200 accumulated ms. This is basically per-IB
+	 * throttling.
 	 *
-	 * - If all of VRAM is free, allow relocating the number of bytes that
-	 *   is equal to 1/4 of the size of VRAM for this IB.
+	 * It means that in order to get full max MBps, at least 5 IBs per
+	 * second must be submitted and not more than 200ms apart from each
+	 * other.
+	 */
+	const s64 us_upper_bound = 200000;
 
-	 * - If more than one half of VRAM is occupied, only allow relocating
-	 *   1 MB of data for this IB.
-	 *
-	 * - From 0 to one half of used VRAM, the threshold decreases
-	 *   linearly.
-	 *         __________________
-	 * 1/4 of -|\               |
-	 * VRAM    | \              |
-	 *         |  \             |
-	 *         |   \            |
-	 *         |    \           |
-	 *         |     \          |
-	 *         |      \         |
-	 *         |       \________|1 MB
-	 *         |----------------|
-	 *    VRAM 0 %             100 %
-	 *         used            used
-	 *
-	 * Note: It's a threshold, not a limit. The threshold must be crossed
-	 * for buffer relocations to stop, so any buffer of an arbitrary size
-	 * can be moved as long as the threshold isn't crossed before
-	 * the relocation takes place. We don't want to disable buffer
-	 * relocations completely.
+	if (!adev->mm_stats.log2_max_MBps)
+		return 0;
+
+	total_vram = adev->mc.real_vram_size - adev->vram_pin_size;
+	used_vram = atomic64_read(&adev->vram_usage);
+	free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
+
+	spin_lock(&adev->mm_stats.lock);
+
+	/* Increase the amount of accumulated us. */
+	time_us = ktime_to_us(ktime_get());
+	increment_us = time_us - adev->mm_stats.last_update_us;
+	adev->mm_stats.last_update_us = time_us;
+	adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
+                                      us_upper_bound);
+
+	/* This prevents the short period of low performance when the VRAM
+	 * usage is low and the driver is in debt or doesn't have enough
+	 * accumulated us to fill VRAM quickly.
 	 *
-	 * The idea is that buffers should be placed in VRAM at creation time
-	 * and TTM should only do a minimum number of relocations during
-	 * command submission. In practice, you need to submit at least
-	 * a dozen IBs to move all buffers to VRAM if they are in GTT.
+	 * The situation can occur in these cases:
+	 * - a lot of VRAM is freed by userspace
+	 * - the presence of a big buffer causes a lot of evictions
+	 *   (solution: split buffers into smaller ones)
 	 *
-	 * Also, things can get pretty crazy under memory pressure and actual
-	 * VRAM usage can change a lot, so playing safe even at 50% does
-	 * consistently increase performance.
+	 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
+	 * accum_us to a positive number.
+	 */
+	if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
+		s64 min_us;
+
+		/* Be more aggresive on dGPUs. Try to fill a portion of free
+		 * VRAM now.
+		 */
+		if (!(adev->flags & AMD_IS_APU))
+			min_us = bytes_to_us(adev, free_vram / 4);
+		else
+			min_us = 0; /* Reset accum_us on APUs. */
+
+		adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
+	}
+
+	/* This returns 0 if the driver is in debt to disallow (optional)
+	 * buffer moves.
+	 */
+	max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
+
+	spin_unlock(&adev->mm_stats.lock);
+	return max_bytes;
+}
+
+/* Report how many bytes have really been moved for the last command
+ * submission. This can result in a debt that can stop buffer migrations
+ * temporarily.
+ */
+static void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev,
+					 u64 num_bytes)
+{
+	spin_lock(&adev->mm_stats.lock);
+	adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
+	spin_unlock(&adev->mm_stats.lock);
+}
+
+static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
+				 struct amdgpu_bo *bo)
+{
+	u64 initial_bytes_moved;
+	uint32_t domain;
+	int r;
+
+	if (bo->pin_count)
+		return 0;
+
+	/* Don't move this buffer if we have depleted our allowance
+	 * to move it. Don't move anything if the threshold is zero.
 	 */
+	if (p->bytes_moved < p->bytes_moved_threshold)
+		domain = bo->prefered_domains;
+	else
+		domain = bo->allowed_domains;
+
+retry:
+	amdgpu_ttm_placement_from_domain(bo, domain);
+	initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+	p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
+		initial_bytes_moved;
+
+	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
+		domain = bo->allowed_domains;
+		goto retry;
+	}
 
-	u64 half_vram = real_vram_size >> 1;
-	u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage;
-	u64 bytes_moved_threshold = half_free_vram >> 1;
-	return max(bytes_moved_threshold, 1024*1024ull);
+	return r;
 }
 
-int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
+/* Last resort, try to evict something from the current working set */
+static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
+				struct amdgpu_bo_list_entry *lobj)
+{
+	uint32_t domain = lobj->robj->allowed_domains;
+	int r;
+
+	if (!p->evictable)
+		return false;
+
+	for (;&p->evictable->tv.head != &p->validated;
+	     p->evictable = list_prev_entry(p->evictable, tv.head)) {
+
+		struct amdgpu_bo_list_entry *candidate = p->evictable;
+		struct amdgpu_bo *bo = candidate->robj;
+		u64 initial_bytes_moved;
+		uint32_t other;
+
+		/* If we reached our current BO we can forget it */
+		if (candidate == lobj)
+			break;
+
+		other = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
+
+		/* Check if this BO is in one of the domains we need space for */
+		if (!(other & domain))
+			continue;
+
+		/* Check if we can move this BO somewhere else */
+		other = bo->allowed_domains & ~domain;
+		if (!other)
+			continue;
+
+		/* Good we can try to move this BO somewhere else */
+		amdgpu_ttm_placement_from_domain(bo, other);
+		initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
+		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+		p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
+			initial_bytes_moved;
+
+		if (unlikely(r))
+			break;
+
+		p->evictable = list_prev_entry(p->evictable, tv.head);
+		list_move(&candidate->tv.head, &p->validated);
+
+		return true;
+	}
+
+	return false;
+}
+
+static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
 			    struct list_head *validated)
 {
 	struct amdgpu_bo_list_entry *lobj;
-	u64 initial_bytes_moved;
 	int r;
 
 	list_for_each_entry(lobj, validated, tv.head) {
 		struct amdgpu_bo *bo = lobj->robj;
 		bool binding_userptr = false;
 		struct mm_struct *usermm;
-		uint32_t domain;
 
 		usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
 		if (usermm && usermm != current->mm)
@@ -313,35 +461,19 @@ int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
 			binding_userptr = true;
 		}
 
-		if (bo->pin_count)
-			continue;
-
-		/* Avoid moving this one if we have moved too many buffers
-		 * for this IB already.
-		 *
-		 * Note that this allows moving at least one buffer of
-		 * any size, because it doesn't take the current "bo"
-		 * into account. We don't want to disallow buffer moves
-		 * completely.
-		 */
-		if (p->bytes_moved <= p->bytes_moved_threshold)
-			domain = bo->prefered_domains;
-		else
-			domain = bo->allowed_domains;
-
-	retry:
-		amdgpu_ttm_placement_from_domain(bo, domain);
-		initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
-		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
-		p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
-			       initial_bytes_moved;
+		if (p->evictable == lobj)
+			p->evictable = NULL;
 
-		if (unlikely(r)) {
-			if (r != -ERESTARTSYS && domain != bo->allowed_domains) {
-				domain = bo->allowed_domains;
-				goto retry;
-			}
+		do {
+			r = amdgpu_cs_bo_validate(p, bo);
+		} while (r == -ENOMEM && amdgpu_cs_try_evict(p, lobj));
+		if (r)
 			return r;
+
+		if (bo->shadow) {
+			r = amdgpu_cs_bo_validate(p, bo);
+			if (r)
+				return r;
 		}
 
 		if (binding_userptr) {
@@ -386,8 +518,10 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 
 		r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
 					   &duplicates);
-		if (unlikely(r != 0))
+		if (unlikely(r != 0)) {
+			DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
 			goto error_free_pages;
+		}
 
 		/* Without a BO list we don't have userptr BOs */
 		if (!p->bo_list)
@@ -427,9 +561,10 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 		/* Unreserve everything again. */
 		ttm_eu_backoff_reservation(&p->ticket, &p->validated);
 
-		/* We tried to often, just abort */
+		/* We tried too many times, just abort */
 		if (!--tries) {
 			r = -EDEADLK;
+			DRM_ERROR("deadlock in %s\n", __func__);
 			goto error_free_pages;
 		}
 
@@ -441,11 +576,13 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 							 sizeof(struct page*));
 			if (!e->user_pages) {
 				r = -ENOMEM;
+				DRM_ERROR("calloc failure in %s\n", __func__);
 				goto error_free_pages;
 			}
 
 			r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
 			if (r) {
+				DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
 				drm_free_large(e->user_pages);
 				e->user_pages = NULL;
 				goto error_free_pages;
@@ -460,14 +597,23 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 
 	p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
 	p->bytes_moved = 0;
+	p->evictable = list_last_entry(&p->validated,
+				       struct amdgpu_bo_list_entry,
+				       tv.head);
 
 	r = amdgpu_cs_list_validate(p, &duplicates);
-	if (r)
+	if (r) {
+		DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
 		goto error_validate;
+	}
 
 	r = amdgpu_cs_list_validate(p, &p->validated);
-	if (r)
+	if (r) {
+		DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n");
 		goto error_validate;
+	}
+
+	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved);
 
 	fpriv->vm.last_eviction_counter =
 		atomic64_read(&p->adev->num_evictions);
@@ -499,8 +645,12 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 		}
 	}
 
-	if (p->uf_entry.robj)
-		p->job->uf_addr += amdgpu_bo_gpu_offset(p->uf_entry.robj);
+	if (!r && p->uf_entry.robj) {
+		struct amdgpu_bo *uf = p->uf_entry.robj;
+
+		r = amdgpu_ttm_bind(&uf->tbo, &uf->tbo.mem);
+		p->job->uf_addr += amdgpu_bo_gpu_offset(uf);
+	}
 
 error_validate:
 	if (r) {
@@ -617,7 +767,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
 			if (bo_va == NULL)
 				continue;
 
-			r = amdgpu_vm_bo_update(adev, bo_va, &bo->tbo.mem);
+			r = amdgpu_vm_bo_update(adev, bo_va, false);
 			if (r)
 				return r;
 
@@ -710,6 +860,14 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
 		if (r)
 			return r;
 
+		if (ib->flags & AMDGPU_IB_FLAG_PREAMBLE) {
+			parser->job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT;
+			if (!parser->ctx->preamble_presented) {
+				parser->job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
+				parser->ctx->preamble_presented = true;
+			}
+		}
+
 		if (parser->job->ring && parser->job->ring != ring)
 			return -EINVAL;
 
@@ -849,7 +1007,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 	}
 
 	job->owner = p->filp;
-	job->ctx = entity->fence_context;
+	job->fence_ctx = entity->fence_context;
 	p->fence = fence_get(&job->base.s_fence->finished);
 	cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
 	job->uf_sequence = cs->out.handle;
@@ -1015,3 +1173,29 @@ amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
 
 	return NULL;
 }
+
+/**
+ * amdgpu_cs_sysvm_access_required - make BOs accessible by the system VM
+ *
+ * @parser: command submission parser context
+ *
+ * Helper for UVD/VCE VM emulation, make sure BOs are accessible by the system VM.
+ */
+int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser)
+{
+	unsigned i;
+	int r;
+
+	if (!parser->bo_list)
+		return 0;
+
+	for (i = 0; i < parser->bo_list->num_entries; i++) {
+		struct amdgpu_bo *bo = parser->bo_list->array[i].robj;
+
+		r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
+		if (unlikely(r))
+			return r;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 17e13621fae9..e203e5561107 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -60,6 +60,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ctx *ctx)
 			amd_sched_entity_fini(&adev->rings[j]->sched,
 					      &ctx->rings[j].entity);
 		kfree(ctx->fences);
+		ctx->fences = NULL;
 		return r;
 	}
 	return 0;
@@ -77,6 +78,7 @@ static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
 		for (j = 0; j < amdgpu_sched_jobs; ++j)
 			fence_put(ctx->rings[i].fences[j]);
 	kfree(ctx->fences);
+	ctx->fences = NULL;
 
 	for (i = 0; i < adev->num_rings; i++)
 		amd_sched_entity_fini(&adev->rings[i]->sched,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index df7ab2458e50..7dbe85d67d26 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -41,16 +41,26 @@
 #include "atom.h"
 #include "amdgpu_atombios.h"
 #include "amd_pcie.h"
+#ifdef CONFIG_DRM_AMDGPU_SI
+#include "si.h"
+#endif
 #ifdef CONFIG_DRM_AMDGPU_CIK
 #include "cik.h"
 #endif
 #include "vi.h"
 #include "bif/bif_4_1_d.h"
+#include <linux/pci.h>
+#include <linux/firmware.h>
 
 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
 
 static const char *amdgpu_asic_name[] = {
+	"TAHITI",
+	"PITCAIRN",
+	"VERDE",
+	"OLAND",
+	"HAINAN",
 	"BONAIRE",
 	"KAVERI",
 	"KABINI",
@@ -101,7 +111,7 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 		    bool always_indirect)
 {
 	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
-	
+
 	if ((reg * 4) < adev->rmmio_size && !always_indirect)
 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 	else {
@@ -642,6 +652,46 @@ bool amdgpu_card_posted(struct amdgpu_device *adev)
 
 }
 
+static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev))
+		return false;
+
+	if (amdgpu_passthrough(adev)) {
+		/* for FIJI: In whole GPU pass-through virtualization case
+		 * old smc fw won't clear some registers (e.g. MEM_SIZE, BIOS_SCRATCH)
+		 * so amdgpu_card_posted return false and driver will incorrectly skip vPost.
+		 * but if we force vPost do in pass-through case, the driver reload will hang.
+		 * whether doing vPost depends on amdgpu_card_posted if smc version is above
+		 * 00160e00 for FIJI.
+		 */
+		if (adev->asic_type == CHIP_FIJI) {
+			int err;
+			uint32_t fw_ver;
+			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
+			/* force vPost if error occured */
+			if (err)
+				return true;
+
+			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
+			if (fw_ver >= 0x00160e00)
+				return !amdgpu_card_posted(adev);
+		}
+	} else {
+		/* in bare-metal case, amdgpu_card_posted return false
+		 * after system reboot/boot, and return true if driver
+		 * reloaded.
+		 * we shouldn't do vPost after driver reload otherwise GPU
+		 * could hang.
+		 */
+		if (amdgpu_card_posted(adev))
+			return false;
+	}
+
+	/* we assume vPost is neede for all other cases */
+	return true;
+}
+
 /**
  * amdgpu_dummy_page_init - init dummy page used by the driver
  *
@@ -1026,7 +1076,7 @@ static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switchero
 		/* don't suspend or resume card normally */
 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 
-		amdgpu_resume_kms(dev, true, true);
+		amdgpu_device_resume(dev, true, true);
 
 		dev->pdev->d3_delay = d3_delay;
 
@@ -1036,7 +1086,7 @@ static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switchero
 		printk(KERN_INFO "amdgpu: switched off\n");
 		drm_kms_helper_poll_disable(dev);
 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
-		amdgpu_suspend_kms(dev, true, true);
+		amdgpu_device_suspend(dev, true, true);
 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
 	}
 }
@@ -1181,10 +1231,38 @@ int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
 	return 1;
 }
 
+static void amdgpu_whether_enable_virtual_display(struct amdgpu_device *adev)
+{
+	adev->enable_virtual_display = false;
+
+	if (amdgpu_virtual_display) {
+		struct drm_device *ddev = adev->ddev;
+		const char *pci_address_name = pci_name(ddev->pdev);
+		char *pciaddstr, *pciaddstr_tmp, *pciaddname;
+
+		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
+		pciaddstr_tmp = pciaddstr;
+		while ((pciaddname = strsep(&pciaddstr_tmp, ";"))) {
+			if (!strcmp(pci_address_name, pciaddname)) {
+				adev->enable_virtual_display = true;
+				break;
+			}
+		}
+
+		DRM_INFO("virtual display string:%s, %s:virtual_display:%d\n",
+				 amdgpu_virtual_display, pci_address_name,
+				 adev->enable_virtual_display);
+
+		kfree(pciaddstr);
+	}
+}
+
 static int amdgpu_early_init(struct amdgpu_device *adev)
 {
 	int i, r;
 
+	amdgpu_whether_enable_virtual_display(adev);
+
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
 	case CHIP_TONGA:
@@ -1202,6 +1280,18 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
 		if (r)
 			return r;
 		break;
+#ifdef CONFIG_DRM_AMDGPU_SI
+	case CHIP_VERDE:
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+	case CHIP_OLAND:
+	case CHIP_HAINAN:
+		adev->family = AMDGPU_FAMILY_SI;
+		r = si_set_ip_blocks(adev);
+		if (r)
+			return r;
+		break;
+#endif
 #ifdef CONFIG_DRM_AMDGPU_CIK
 	case CHIP_BONAIRE:
 	case CHIP_HAWAII:
@@ -1318,6 +1408,9 @@ static int amdgpu_late_init(struct amdgpu_device *adev)
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if (!adev->ip_block_status[i].valid)
 			continue;
+		if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_UVD ||
+			adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_VCE)
+			continue;
 		/* enable clockgating to save power */
 		r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
 								    AMD_CG_STATE_GATE);
@@ -1331,6 +1424,7 @@ static int amdgpu_late_init(struct amdgpu_device *adev)
 				DRM_ERROR("late_init of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
 				return r;
 			}
+			adev->ip_block_status[i].late_initialized = true;
 		}
 	}
 
@@ -1376,8 +1470,11 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 	}
 
 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
+		if (!adev->ip_block_status[i].late_initialized)
+			continue;
 		if (adev->ip_blocks[i].funcs->late_fini)
 			adev->ip_blocks[i].funcs->late_fini((void *)adev);
+		adev->ip_block_status[i].late_initialized = false;
 	}
 
 	return 0;
@@ -1433,13 +1530,10 @@ static int amdgpu_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-static bool amdgpu_device_is_virtual(void)
+static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
 {
-#ifdef CONFIG_X86
-	return boot_cpu_has(X86_FEATURE_HYPERVISOR);
-#else
-	return false;
-#endif
+	if (amdgpu_atombios_has_gpu_virtualization_table(adev))
+		adev->virtualization.virtual_caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
 }
 
 /**
@@ -1461,6 +1555,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 {
 	int r, i;
 	bool runtime = false;
+	u32 max_MBps;
 
 	adev->shutdown = false;
 	adev->dev = &pdev->dev;
@@ -1484,6 +1579,8 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	adev->smc_wreg = &amdgpu_invalid_wreg;
 	adev->pcie_rreg = &amdgpu_invalid_rreg;
 	adev->pcie_wreg = &amdgpu_invalid_wreg;
+	adev->pciep_rreg = &amdgpu_invalid_rreg;
+	adev->pciep_wreg = &amdgpu_invalid_wreg;
 	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
 	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
 	adev->didt_rreg = &amdgpu_invalid_rreg;
@@ -1520,9 +1617,22 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	spin_lock_init(&adev->didt_idx_lock);
 	spin_lock_init(&adev->gc_cac_idx_lock);
 	spin_lock_init(&adev->audio_endpt_idx_lock);
+	spin_lock_init(&adev->mm_stats.lock);
+
+	INIT_LIST_HEAD(&adev->shadow_list);
+	mutex_init(&adev->shadow_list_lock);
+
+	INIT_LIST_HEAD(&adev->gtt_list);
+	spin_lock_init(&adev->gtt_list_lock);
+
+	if (adev->asic_type >= CHIP_BONAIRE) {
+		adev->rmmio_base = pci_resource_start(adev->pdev, 5);
+		adev->rmmio_size = pci_resource_len(adev->pdev, 5);
+	} else {
+		adev->rmmio_base = pci_resource_start(adev->pdev, 2);
+		adev->rmmio_size = pci_resource_len(adev->pdev, 2);
+	}
 
-	adev->rmmio_base = pci_resource_start(adev->pdev, 5);
-	adev->rmmio_size = pci_resource_len(adev->pdev, 5);
 	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
 	if (adev->rmmio == NULL) {
 		return -ENOMEM;
@@ -1530,8 +1640,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
 	DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
 
-	/* doorbell bar mapping */
-	amdgpu_doorbell_init(adev);
+	if (adev->asic_type >= CHIP_BONAIRE)
+		/* doorbell bar mapping */
+		amdgpu_doorbell_init(adev);
 
 	/* io port mapping */
 	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
@@ -1579,25 +1690,24 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		goto failed;
 	}
 
-	/* See if the asic supports SR-IOV */
-	adev->virtualization.supports_sr_iov =
-		amdgpu_atombios_has_gpu_virtualization_table(adev);
-
-	/* Check if we are executing in a virtualized environment */
-	adev->virtualization.is_virtual = amdgpu_device_is_virtual();
-	adev->virtualization.caps = amdgpu_asic_get_virtual_caps(adev);
+	/* detect if we are with an SRIOV vbios */
+	amdgpu_device_detect_sriov_bios(adev);
 
 	/* Post card if necessary */
-	if (!amdgpu_card_posted(adev) ||
-	    (adev->virtualization.is_virtual &&
-	     !(adev->virtualization.caps & AMDGPU_VIRT_CAPS_SRIOV_EN))) {
+	if (amdgpu_vpost_needed(adev)) {
 		if (!adev->bios) {
-			dev_err(adev->dev, "Card not posted and no BIOS - ignoring\n");
+			dev_err(adev->dev, "no vBIOS found\n");
 			r = -EINVAL;
 			goto failed;
 		}
-		DRM_INFO("GPU not posted. posting now...\n");
-		amdgpu_atom_asic_init(adev->mode_info.atom_context);
+		DRM_INFO("GPU posting now...\n");
+		r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
+		if (r) {
+			dev_err(adev->dev, "gpu post error!\n");
+			goto failed;
+		}
+	} else {
+		DRM_INFO("GPU post is not needed\n");
 	}
 
 	/* Initialize clocks */
@@ -1628,6 +1738,14 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 
 	adev->accel_working = true;
 
+	/* Initialize the buffer migration limit. */
+	if (amdgpu_moverate >= 0)
+		max_MBps = amdgpu_moverate;
+	else
+		max_MBps = 8; /* Allow 8 MB/s. */
+	/* Get a log2 for easy divisions. */
+	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
+
 	amdgpu_fbdev_init(adev);
 
 	r = amdgpu_ib_pool_init(adev);
@@ -1708,11 +1826,11 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 
 	DRM_INFO("amdgpu: finishing device.\n");
 	adev->shutdown = true;
+	drm_crtc_force_disable_all(adev->ddev);
 	/* evict vram memory */
 	amdgpu_bo_evict_vram(adev);
 	amdgpu_ib_pool_fini(adev);
 	amdgpu_fence_driver_fini(adev);
-	drm_crtc_force_disable_all(adev->ddev);
 	amdgpu_fbdev_fini(adev);
 	r = amdgpu_fini(adev);
 	kfree(adev->ip_block_status);
@@ -1732,7 +1850,8 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 	adev->rio_mem = NULL;
 	iounmap(adev->rmmio);
 	adev->rmmio = NULL;
-	amdgpu_doorbell_fini(adev);
+	if (adev->asic_type >= CHIP_BONAIRE)
+		amdgpu_doorbell_fini(adev);
 	amdgpu_debugfs_regs_cleanup(adev);
 	amdgpu_debugfs_remove_files(adev);
 }
@@ -1742,7 +1861,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
  * Suspend & resume.
  */
 /**
- * amdgpu_suspend_kms - initiate device suspend
+ * amdgpu_device_suspend - initiate device suspend
  *
  * @pdev: drm dev pointer
  * @state: suspend state
@@ -1751,7 +1870,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
  * Returns 0 for success or an error on failure.
  * Called at driver suspend.
  */
-int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
+int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 {
 	struct amdgpu_device *adev;
 	struct drm_crtc *crtc;
@@ -1819,6 +1938,10 @@ int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
 		/* Shut down the device */
 		pci_disable_device(dev->pdev);
 		pci_set_power_state(dev->pdev, PCI_D3hot);
+	} else {
+		r = amdgpu_asic_reset(adev);
+		if (r)
+			DRM_ERROR("amdgpu asic reset failed\n");
 	}
 
 	if (fbcon) {
@@ -1830,7 +1953,7 @@ int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
 }
 
 /**
- * amdgpu_resume_kms - initiate device resume
+ * amdgpu_device_resume - initiate device resume
  *
  * @pdev: drm dev pointer
  *
@@ -1838,7 +1961,7 @@ int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
  * Returns 0 for success or an error on failure.
  * Called at driver resume.
  */
-int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
+int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 {
 	struct drm_connector *connector;
 	struct amdgpu_device *adev = dev->dev_private;
@@ -1848,22 +1971,26 @@ int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
-	if (fbcon) {
+	if (fbcon)
 		console_lock();
-	}
+
 	if (resume) {
 		pci_set_power_state(dev->pdev, PCI_D0);
 		pci_restore_state(dev->pdev);
-		if (pci_enable_device(dev->pdev)) {
+		r = pci_enable_device(dev->pdev);
+		if (r) {
 			if (fbcon)
 				console_unlock();
-			return -1;
+			return r;
 		}
 	}
 
 	/* post card */
-	if (!amdgpu_card_posted(adev))
-		amdgpu_atom_asic_init(adev->mode_info.atom_context);
+	if (!amdgpu_card_posted(adev) || !resume) {
+		r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
+		if (r)
+			DRM_ERROR("amdgpu asic init failed\n");
+	}
 
 	r = amdgpu_resume(adev);
 	if (r)
@@ -1937,6 +2064,126 @@ int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
 	return 0;
 }
 
+static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
+{
+	int i;
+	bool asic_hang = false;
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_block_status[i].valid)
+			continue;
+		if (adev->ip_blocks[i].funcs->check_soft_reset)
+			adev->ip_blocks[i].funcs->check_soft_reset(adev);
+		if (adev->ip_block_status[i].hang) {
+			DRM_INFO("IP block:%d is hang!\n", i);
+			asic_hang = true;
+		}
+	}
+	return asic_hang;
+}
+
+static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
+{
+	int i, r = 0;
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_block_status[i].valid)
+			continue;
+		if (adev->ip_block_status[i].hang &&
+		    adev->ip_blocks[i].funcs->pre_soft_reset) {
+			r = adev->ip_blocks[i].funcs->pre_soft_reset(adev);
+			if (r)
+				return r;
+		}
+	}
+
+	return 0;
+}
+
+static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
+{
+	if (adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang ||
+	    adev->ip_block_status[AMD_IP_BLOCK_TYPE_SMC].hang ||
+	    adev->ip_block_status[AMD_IP_BLOCK_TYPE_ACP].hang ||
+	    adev->ip_block_status[AMD_IP_BLOCK_TYPE_DCE].hang) {
+		DRM_INFO("Some block need full reset!\n");
+		return true;
+	}
+	return false;
+}
+
+static int amdgpu_soft_reset(struct amdgpu_device *adev)
+{
+	int i, r = 0;
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_block_status[i].valid)
+			continue;
+		if (adev->ip_block_status[i].hang &&
+		    adev->ip_blocks[i].funcs->soft_reset) {
+			r = adev->ip_blocks[i].funcs->soft_reset(adev);
+			if (r)
+				return r;
+		}
+	}
+
+	return 0;
+}
+
+static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
+{
+	int i, r = 0;
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_block_status[i].valid)
+			continue;
+		if (adev->ip_block_status[i].hang &&
+		    adev->ip_blocks[i].funcs->post_soft_reset)
+			r = adev->ip_blocks[i].funcs->post_soft_reset(adev);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+bool amdgpu_need_backup(struct amdgpu_device *adev)
+{
+	if (adev->flags & AMD_IS_APU)
+		return false;
+
+	return amdgpu_lockup_timeout > 0 ? true : false;
+}
+
+static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
+					   struct amdgpu_ring *ring,
+					   struct amdgpu_bo *bo,
+					   struct fence **fence)
+{
+	uint32_t domain;
+	int r;
+
+       if (!bo->shadow)
+               return 0;
+
+       r = amdgpu_bo_reserve(bo, false);
+       if (r)
+               return r;
+       domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
+       /* if bo has been evicted, then no need to recover */
+       if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
+               r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
+						 NULL, fence, true);
+               if (r) {
+                       DRM_ERROR("recover page table failed!\n");
+                       goto err;
+               }
+       }
+err:
+       amdgpu_bo_unreserve(bo);
+       return r;
+}
+
 /**
  * amdgpu_gpu_reset - reset the asic
  *
@@ -1949,6 +2196,12 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 {
 	int i, r;
 	int resched;
+	bool need_full_reset;
+
+	if (!amdgpu_check_soft_reset(adev)) {
+		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
+		return 0;
+	}
 
 	atomic_inc(&adev->gpu_reset_counter);
 
@@ -1967,40 +2220,93 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 	/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
 	amdgpu_fence_driver_force_completion(adev);
 
-	/* save scratch */
-	amdgpu_atombios_scratch_regs_save(adev);
-	r = amdgpu_suspend(adev);
+	need_full_reset = amdgpu_need_full_reset(adev);
 
-retry:
-	/* Disable fb access */
-	if (adev->mode_info.num_crtc) {
-		struct amdgpu_mode_mc_save save;
-		amdgpu_display_stop_mc_access(adev, &save);
-		amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC);
+	if (!need_full_reset) {
+		amdgpu_pre_soft_reset(adev);
+		r = amdgpu_soft_reset(adev);
+		amdgpu_post_soft_reset(adev);
+		if (r || amdgpu_check_soft_reset(adev)) {
+			DRM_INFO("soft reset failed, will fallback to full reset!\n");
+			need_full_reset = true;
+		}
 	}
 
-	r = amdgpu_asic_reset(adev);
-	/* post card */
-	amdgpu_atom_asic_init(adev->mode_info.atom_context);
+	if (need_full_reset) {
+		/* save scratch */
+		amdgpu_atombios_scratch_regs_save(adev);
+		r = amdgpu_suspend(adev);
 
-	if (!r) {
-		dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
-		r = amdgpu_resume(adev);
+retry:
+		/* Disable fb access */
+		if (adev->mode_info.num_crtc) {
+			struct amdgpu_mode_mc_save save;
+			amdgpu_display_stop_mc_access(adev, &save);
+			amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC);
+		}
+
+		r = amdgpu_asic_reset(adev);
+		/* post card */
+		amdgpu_atom_asic_init(adev->mode_info.atom_context);
+
+		if (!r) {
+			dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
+			r = amdgpu_resume(adev);
+		}
+		/* restore scratch */
+		amdgpu_atombios_scratch_regs_restore(adev);
 	}
-	/* restore scratch */
-	amdgpu_atombios_scratch_regs_restore(adev);
 	if (!r) {
+		amdgpu_irq_gpu_reset_resume_helper(adev);
+		if (need_full_reset && amdgpu_need_backup(adev)) {
+			r = amdgpu_ttm_recover_gart(adev);
+			if (r)
+				DRM_ERROR("gart recovery failed!!!\n");
+		}
 		r = amdgpu_ib_ring_tests(adev);
 		if (r) {
 			dev_err(adev->dev, "ib ring test failed (%d).\n", r);
 			r = amdgpu_suspend(adev);
+			need_full_reset = true;
 			goto retry;
 		}
+		/**
+		 * recovery vm page tables, since we cannot depend on VRAM is
+		 * consistent after gpu full reset.
+		 */
+		if (need_full_reset && amdgpu_need_backup(adev)) {
+			struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+			struct amdgpu_bo *bo, *tmp;
+			struct fence *fence = NULL, *next = NULL;
+
+			DRM_INFO("recover vram bo from shadow\n");
+			mutex_lock(&adev->shadow_list_lock);
+			list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
+				amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
+				if (fence) {
+					r = fence_wait(fence, false);
+					if (r) {
+						WARN(r, "recovery from shadow isn't comleted\n");
+						break;
+					}
+				}
 
+				fence_put(fence);
+				fence = next;
+			}
+			mutex_unlock(&adev->shadow_list_lock);
+			if (fence) {
+				r = fence_wait(fence, false);
+				if (r)
+					WARN(r, "recovery from shadow isn't comleted\n");
+			}
+			fence_put(fence);
+		}
 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 			struct amdgpu_ring *ring = adev->rings[i];
 			if (!ring)
 				continue;
+
 			amd_sched_job_recovery(&ring->sched);
 			kthread_unpark(ring->sched.thread);
 		}
@@ -2020,7 +2326,6 @@ retry:
 		/* bad news, how to tell it to userspace ? */
 		dev_info(adev->dev, "GPU reset failed\n");
 	}
-	amdgpu_irq_gpu_reset_resume_helper(adev);
 
 	return r;
 }
@@ -2178,22 +2483,26 @@ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 	struct amdgpu_device *adev = f->f_inode->i_private;
 	ssize_t result = 0;
 	int r;
-	bool use_bank;
+	bool pm_pg_lock, use_bank;
 	unsigned instance_bank, sh_bank, se_bank;
 
 	if (size & 0x3 || *pos & 0x3)
 		return -EINVAL;
 
+	/* are we reading registers for which a PG lock is necessary? */
+	pm_pg_lock = (*pos >> 23) & 1;
+
 	if (*pos & (1ULL << 62)) {
 		se_bank = (*pos >> 24) & 0x3FF;
 		sh_bank = (*pos >> 34) & 0x3FF;
 		instance_bank = (*pos >> 44) & 0x3FF;
 		use_bank = 1;
-		*pos &= 0xFFFFFF;
 	} else {
 		use_bank = 0;
 	}
 
+	*pos &= 0x3FFFF;
+
 	if (use_bank) {
 		if (sh_bank >= adev->gfx.config.max_sh_per_se ||
 		    se_bank >= adev->gfx.config.max_shader_engines)
@@ -2203,6 +2512,9 @@ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 					sh_bank, instance_bank);
 	}
 
+	if (pm_pg_lock)
+		mutex_lock(&adev->pm.mutex);
+
 	while (size) {
 		uint32_t value;
 
@@ -2228,6 +2540,9 @@ end:
 		mutex_unlock(&adev->grbm_idx_mutex);
 	}
 
+	if (pm_pg_lock)
+		mutex_unlock(&adev->pm.mutex);
+
 	return result;
 }
 
@@ -2385,7 +2700,7 @@ static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 	while (size) {
 		uint32_t value;
 
-		value = RREG32_SMC(*pos >> 2);
+		value = RREG32_SMC(*pos);
 		r = put_user(value, (uint32_t *)buf);
 		if (r)
 			return r;
@@ -2416,7 +2731,7 @@ static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *
 		if (r)
 			return r;
 
-		WREG32_SMC(*pos >> 2, value);
+		WREG32_SMC(*pos, value);
 
 		result += 4;
 		buf += 4;
@@ -2438,12 +2753,12 @@ static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 	if (size & 0x3 || *pos & 0x3)
 		return -EINVAL;
 
-	config = kmalloc(256 * sizeof(*config), GFP_KERNEL);
+	config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 	if (!config)
 		return -ENOMEM;
 
 	/* version, increment each time something is added */
-	config[no_regs++] = 0;
+	config[no_regs++] = 2;
 	config[no_regs++] = adev->gfx.config.max_shader_engines;
 	config[no_regs++] = adev->gfx.config.max_tile_pipes;
 	config[no_regs++] = adev->gfx.config.max_cu_per_sh;
@@ -2468,6 +2783,15 @@ static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 	config[no_regs++] = adev->gfx.config.gb_addr_config;
 	config[no_regs++] = adev->gfx.config.num_rbs;
 
+	/* rev==1 */
+	config[no_regs++] = adev->rev_id;
+	config[no_regs++] = adev->pg_flags;
+	config[no_regs++] = adev->cg_flags;
+
+	/* rev==2 */
+	config[no_regs++] = adev->family;
+	config[no_regs++] = adev->external_rev_id;
+
 	while (size && (*pos < no_regs * 4)) {
 		uint32_t value;
 
@@ -2488,6 +2812,29 @@ static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 	return result;
 }
 
+static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = f->f_inode->i_private;
+	int idx, r;
+	int32_t value;
+
+	if (size != 4 || *pos & 0x3)
+		return -EINVAL;
+
+	/* convert offset to sensor number */
+	idx = *pos >> 2;
+
+	if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
+		r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &value);
+	else
+		return -EINVAL;
+
+	if (!r)
+		r = put_user(value, (int32_t *)buf);
+
+	return !r ? 4 : r;
+}
 
 static const struct file_operations amdgpu_debugfs_regs_fops = {
 	.owner = THIS_MODULE,
@@ -2520,12 +2867,19 @@ static const struct file_operations amdgpu_debugfs_gca_config_fops = {
 	.llseek = default_llseek
 };
 
+static const struct file_operations amdgpu_debugfs_sensors_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_sensor_read,
+	.llseek = default_llseek
+};
+
 static const struct file_operations *debugfs_regs[] = {
 	&amdgpu_debugfs_regs_fops,
 	&amdgpu_debugfs_regs_didt_fops,
 	&amdgpu_debugfs_regs_pcie_fops,
 	&amdgpu_debugfs_regs_smc_fops,
 	&amdgpu_debugfs_gca_config_fops,
+	&amdgpu_debugfs_sensors_fops,
 };
 
 static const char *debugfs_regs_names[] = {
@@ -2534,6 +2888,7 @@ static const char *debugfs_regs_names[] = {
 	"amdgpu_regs_pcie",
 	"amdgpu_regs_smc",
 	"amdgpu_gca_config",
+	"amdgpu_sensors",
 };
 
 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index 76f96028313d..083e2b429872 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -41,7 +41,7 @@ static void amdgpu_flip_callback(struct fence *f, struct fence_cb *cb)
 		container_of(cb, struct amdgpu_flip_work, cb);
 
 	fence_put(f);
-	schedule_work(&work->flip_work);
+	schedule_work(&work->flip_work.work);
 }
 
 static bool amdgpu_flip_handle_fence(struct amdgpu_flip_work *work,
@@ -63,16 +63,17 @@ static bool amdgpu_flip_handle_fence(struct amdgpu_flip_work *work,
 
 static void amdgpu_flip_work_func(struct work_struct *__work)
 {
+	struct delayed_work *delayed_work =
+		container_of(__work, struct delayed_work, work);
 	struct amdgpu_flip_work *work =
-		container_of(__work, struct amdgpu_flip_work, flip_work);
+		container_of(delayed_work, struct amdgpu_flip_work, flip_work);
 	struct amdgpu_device *adev = work->adev;
 	struct amdgpu_crtc *amdgpuCrtc = adev->mode_info.crtcs[work->crtc_id];
 
 	struct drm_crtc *crtc = &amdgpuCrtc->base;
 	unsigned long flags;
-	unsigned i, repcnt = 4;
-	int vpos, hpos, stat, min_udelay = 0;
-	struct drm_vblank_crtc *vblank = &crtc->dev->vblank[work->crtc_id];
+	unsigned i;
+	int vpos, hpos;
 
 	if (amdgpu_flip_handle_fence(work, &work->excl))
 		return;
@@ -81,55 +82,23 @@ static void amdgpu_flip_work_func(struct work_struct *__work)
 		if (amdgpu_flip_handle_fence(work, &work->shared[i]))
 			return;
 
-	/* We borrow the event spin lock for protecting flip_status */
-	spin_lock_irqsave(&crtc->dev->event_lock, flags);
-
-	/* If this happens to execute within the "virtually extended" vblank
-	 * interval before the start of the real vblank interval then it needs
-	 * to delay programming the mmio flip until the real vblank is entered.
-	 * This prevents completing a flip too early due to the way we fudge
-	 * our vblank counter and vblank timestamps in order to work around the
-	 * problem that the hw fires vblank interrupts before actual start of
-	 * vblank (when line buffer refilling is done for a frame). It
-	 * complements the fudging logic in amdgpu_get_crtc_scanoutpos() for
-	 * timestamping and amdgpu_get_vblank_counter_kms() for vblank counts.
-	 *
-	 * In practice this won't execute very often unless on very fast
-	 * machines because the time window for this to happen is very small.
+	/* Wait until we're out of the vertical blank period before the one
+	 * targeted by the flip
 	 */
-	while (amdgpuCrtc->enabled && --repcnt) {
-		/* GET_DISTANCE_TO_VBLANKSTART returns distance to real vblank
-		 * start in hpos, and to the "fudged earlier" vblank start in
-		 * vpos.
-		 */
-		stat = amdgpu_get_crtc_scanoutpos(adev->ddev, work->crtc_id,
-						  GET_DISTANCE_TO_VBLANKSTART,
-						  &vpos, &hpos, NULL, NULL,
-						  &crtc->hwmode);
-
-		if ((stat & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE)) !=
-		    (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE) ||
-		    !(vpos >= 0 && hpos <= 0))
-			break;
-
-		/* Sleep at least until estimated real start of hw vblank */
-		min_udelay = (-hpos + 1) * max(vblank->linedur_ns / 1000, 5);
-		if (min_udelay > vblank->framedur_ns / 2000) {
-			/* Don't wait ridiculously long - something is wrong */
-			repcnt = 0;
-			break;
-		}
-		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
-		usleep_range(min_udelay, 2 * min_udelay);
-		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+	if (amdgpuCrtc->enabled &&
+	    (amdgpu_get_crtc_scanoutpos(adev->ddev, work->crtc_id, 0,
+					&vpos, &hpos, NULL, NULL,
+					&crtc->hwmode)
+	     & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK)) ==
+	    (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK) &&
+	    (int)(work->target_vblank -
+		  amdgpu_get_vblank_counter_kms(adev->ddev, amdgpuCrtc->crtc_id)) > 0) {
+		schedule_delayed_work(&work->flip_work, usecs_to_jiffies(1000));
+		return;
 	}
 
-	if (!repcnt)
-		DRM_DEBUG_DRIVER("Delay problem on crtc %d: min_udelay %d, "
-				 "framedur %d, linedur %d, stat %d, vpos %d, "
-				 "hpos %d\n", work->crtc_id, min_udelay,
-				 vblank->framedur_ns / 1000,
-				 vblank->linedur_ns / 1000, stat, vpos, hpos);
+	/* We borrow the event spin lock for protecting flip_status */
+	spin_lock_irqsave(&crtc->dev->event_lock, flags);
 
 	/* Do the flip (mmio) */
 	adev->mode_info.funcs->page_flip(adev, work->crtc_id, work->base, work->async);
@@ -154,25 +123,25 @@ static void amdgpu_unpin_work_func(struct work_struct *__work)
 	int r;
 
 	/* unpin of the old buffer */
-	r = amdgpu_bo_reserve(work->old_rbo, false);
+	r = amdgpu_bo_reserve(work->old_abo, false);
 	if (likely(r == 0)) {
-		r = amdgpu_bo_unpin(work->old_rbo);
+		r = amdgpu_bo_unpin(work->old_abo);
 		if (unlikely(r != 0)) {
 			DRM_ERROR("failed to unpin buffer after flip\n");
 		}
-		amdgpu_bo_unreserve(work->old_rbo);
+		amdgpu_bo_unreserve(work->old_abo);
 	} else
 		DRM_ERROR("failed to reserve buffer after flip\n");
 
-	amdgpu_bo_unref(&work->old_rbo);
+	amdgpu_bo_unref(&work->old_abo);
 	kfree(work->shared);
 	kfree(work);
 }
 
-int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
-			  struct drm_framebuffer *fb,
-			  struct drm_pending_vblank_event *event,
-			  uint32_t page_flip_flags)
+int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
+				 struct drm_framebuffer *fb,
+				 struct drm_pending_vblank_event *event,
+				 uint32_t page_flip_flags, uint32_t target)
 {
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
@@ -181,7 +150,7 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
 	struct amdgpu_framebuffer *new_amdgpu_fb;
 	struct drm_gem_object *obj;
 	struct amdgpu_flip_work *work;
-	struct amdgpu_bo *new_rbo;
+	struct amdgpu_bo *new_abo;
 	unsigned long flags;
 	u64 tiling_flags;
 	u64 base;
@@ -191,7 +160,7 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
 	if (work == NULL)
 		return -ENOMEM;
 
-	INIT_WORK(&work->flip_work, amdgpu_flip_work_func);
+	INIT_DELAYED_WORK(&work->flip_work, amdgpu_flip_work_func);
 	INIT_WORK(&work->unpin_work, amdgpu_unpin_work_func);
 
 	work->event = event;
@@ -204,28 +173,28 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
 	obj = old_amdgpu_fb->obj;
 
 	/* take a reference to the old object */
-	work->old_rbo = gem_to_amdgpu_bo(obj);
-	amdgpu_bo_ref(work->old_rbo);
+	work->old_abo = gem_to_amdgpu_bo(obj);
+	amdgpu_bo_ref(work->old_abo);
 
 	new_amdgpu_fb = to_amdgpu_framebuffer(fb);
 	obj = new_amdgpu_fb->obj;
-	new_rbo = gem_to_amdgpu_bo(obj);
+	new_abo = gem_to_amdgpu_bo(obj);
 
 	/* pin the new buffer */
-	r = amdgpu_bo_reserve(new_rbo, false);
+	r = amdgpu_bo_reserve(new_abo, false);
 	if (unlikely(r != 0)) {
-		DRM_ERROR("failed to reserve new rbo buffer before flip\n");
+		DRM_ERROR("failed to reserve new abo buffer before flip\n");
 		goto cleanup;
 	}
 
-	r = amdgpu_bo_pin_restricted(new_rbo, AMDGPU_GEM_DOMAIN_VRAM, 0, 0, &base);
+	r = amdgpu_bo_pin_restricted(new_abo, AMDGPU_GEM_DOMAIN_VRAM, 0, 0, &base);
 	if (unlikely(r != 0)) {
 		r = -EINVAL;
-		DRM_ERROR("failed to pin new rbo buffer before flip\n");
+		DRM_ERROR("failed to pin new abo buffer before flip\n");
 		goto unreserve;
 	}
 
-	r = reservation_object_get_fences_rcu(new_rbo->tbo.resv, &work->excl,
+	r = reservation_object_get_fences_rcu(new_abo->tbo.resv, &work->excl,
 					      &work->shared_count,
 					      &work->shared);
 	if (unlikely(r != 0)) {
@@ -233,16 +202,12 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
 		goto unpin;
 	}
 
-	amdgpu_bo_get_tiling_flags(new_rbo, &tiling_flags);
-	amdgpu_bo_unreserve(new_rbo);
+	amdgpu_bo_get_tiling_flags(new_abo, &tiling_flags);
+	amdgpu_bo_unreserve(new_abo);
 
 	work->base = base;
-
-	r = drm_crtc_vblank_get(crtc);
-	if (r) {
-		DRM_ERROR("failed to get vblank before flip\n");
-		goto pflip_cleanup;
-	}
+	work->target_vblank = target - drm_crtc_vblank_count(crtc) +
+		amdgpu_get_vblank_counter_kms(dev, work->crtc_id);
 
 	/* we borrow the event spin lock for protecting flip_wrok */
 	spin_lock_irqsave(&crtc->dev->event_lock, flags);
@@ -250,7 +215,7 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
 		DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
 		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
 		r = -EBUSY;
-		goto vblank_cleanup;
+		goto pflip_cleanup;
 	}
 
 	amdgpu_crtc->pflip_status = AMDGPU_FLIP_PENDING;
@@ -262,26 +227,23 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
 	/* update crtc fb */
 	crtc->primary->fb = fb;
 	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
-	amdgpu_flip_work_func(&work->flip_work);
+	amdgpu_flip_work_func(&work->flip_work.work);
 	return 0;
 
-vblank_cleanup:
-	drm_crtc_vblank_put(crtc);
-
 pflip_cleanup:
-	if (unlikely(amdgpu_bo_reserve(new_rbo, false) != 0)) {
-		DRM_ERROR("failed to reserve new rbo in error path\n");
+	if (unlikely(amdgpu_bo_reserve(new_abo, false) != 0)) {
+		DRM_ERROR("failed to reserve new abo in error path\n");
 		goto cleanup;
 	}
 unpin:
-	if (unlikely(amdgpu_bo_unpin(new_rbo) != 0)) {
-		DRM_ERROR("failed to unpin new rbo in error path\n");
+	if (unlikely(amdgpu_bo_unpin(new_abo) != 0)) {
+		DRM_ERROR("failed to unpin new abo in error path\n");
 	}
 unreserve:
-	amdgpu_bo_unreserve(new_rbo);
+	amdgpu_bo_unreserve(new_abo);
 
 cleanup:
-	amdgpu_bo_unref(&work->old_rbo);
+	amdgpu_bo_unref(&work->old_abo);
 	fence_put(work->excl);
 	for (i = 0; i < work->shared_count; ++i)
 		fence_put(work->shared[i]);
@@ -335,7 +297,7 @@ int amdgpu_crtc_set_config(struct drm_mode_set *set)
 	return ret;
 }
 
-static const char *encoder_names[38] = {
+static const char *encoder_names[41] = {
 	"NONE",
 	"INTERNAL_LVDS",
 	"INTERNAL_TMDS1",
@@ -374,6 +336,9 @@ static const char *encoder_names[38] = {
 	"TRAVIS",
 	"INTERNAL_VCE",
 	"INTERNAL_UNIPHY3",
+	"HDMI_ANX9805",
+	"INTERNAL_AMCLK",
+	"VIRTUAL",
 };
 
 static const char *hpd_names[6] = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 9aa533cf4ad1..71ed27eb3dde 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -53,13 +53,19 @@
  * - 3.2.0 - GFX8: Uses EOP_TC_WB_ACTION_EN, so UMDs don't have to do the same
  *           at the end of IBs.
  * - 3.3.0 - Add VM support for UVD on supported hardware.
+ * - 3.4.0 - Add AMDGPU_INFO_NUM_EVICTIONS.
+ * - 3.5.0 - Add support for new UVD_NO_OP register.
+ * - 3.6.0 - kmd involves use CONTEXT_CONTROL in ring buffer.
+ * - 3.7.0 - Add support for VCE clock list packet
+ * - 3.8.0 - Add support raster config init in the kernel
  */
 #define KMS_DRIVER_MAJOR	3
-#define KMS_DRIVER_MINOR	3
+#define KMS_DRIVER_MINOR	8
 #define KMS_DRIVER_PATCHLEVEL	0
 
 int amdgpu_vram_limit = 0;
 int amdgpu_gart_size = -1; /* auto */
+int amdgpu_moverate = -1; /* auto */
 int amdgpu_benchmarking = 0;
 int amdgpu_testing = 0;
 int amdgpu_audio = -1;
@@ -84,11 +90,14 @@ int amdgpu_sched_jobs = 32;
 int amdgpu_sched_hw_submission = 2;
 int amdgpu_powerplay = -1;
 int amdgpu_powercontainment = 1;
+int amdgpu_sclk_deep_sleep_en = 1;
 unsigned amdgpu_pcie_gen_cap = 0;
 unsigned amdgpu_pcie_lane_cap = 0;
 unsigned amdgpu_cg_mask = 0xffffffff;
 unsigned amdgpu_pg_mask = 0xffffffff;
 char *amdgpu_disable_cu = NULL;
+char *amdgpu_virtual_display = NULL;
+unsigned amdgpu_pp_feature_mask = 0xffffffff;
 
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -96,6 +105,9 @@ module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
 MODULE_PARM_DESC(gartsize, "Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)");
 module_param_named(gartsize, amdgpu_gart_size, int, 0600);
 
+MODULE_PARM_DESC(moverate, "Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)");
+module_param_named(moverate, amdgpu_moverate, int, 0600);
+
 MODULE_PARM_DESC(benchmark, "Run benchmark");
 module_param_named(benchmark, amdgpu_benchmarking, int, 0444);
 
@@ -162,13 +174,17 @@ module_param_named(sched_jobs, amdgpu_sched_jobs, int, 0444);
 MODULE_PARM_DESC(sched_hw_submission, "the max number of HW submissions (default 2)");
 module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
 
-#ifdef CONFIG_DRM_AMD_POWERPLAY
 MODULE_PARM_DESC(powerplay, "Powerplay component (1 = enable, 0 = disable, -1 = auto (default))");
 module_param_named(powerplay, amdgpu_powerplay, int, 0444);
 
 MODULE_PARM_DESC(powercontainment, "Power Containment (1 = enable (default), 0 = disable)");
 module_param_named(powercontainment, amdgpu_powercontainment, int, 0444);
-#endif
+
+MODULE_PARM_DESC(ppfeaturemask, "all power features enabled (default))");
+module_param_named(ppfeaturemask, amdgpu_pp_feature_mask, int, 0444);
+
+MODULE_PARM_DESC(sclkdeepsleep, "SCLK Deep Sleep (1 = enable (default), 0 = disable)");
+module_param_named(sclkdeepsleep, amdgpu_sclk_deep_sleep_en, int, 0444);
 
 MODULE_PARM_DESC(pcie_gen_cap, "PCIE Gen Caps (0: autodetect (default))");
 module_param_named(pcie_gen_cap, amdgpu_pcie_gen_cap, uint, 0444);
@@ -185,7 +201,84 @@ module_param_named(pg_mask, amdgpu_pg_mask, uint, 0444);
 MODULE_PARM_DESC(disable_cu, "Disable CUs (se.sh.cu,...)");
 module_param_named(disable_cu, amdgpu_disable_cu, charp, 0444);
 
+MODULE_PARM_DESC(virtual_display, "Enable virtual display feature (the virtual_display will be set like xxxx:xx:xx.x;xxxx:xx:xx.x)");
+module_param_named(virtual_display, amdgpu_virtual_display, charp, 0444);
+
 static const struct pci_device_id pciidlist[] = {
+#ifdef  CONFIG_DRM_AMDGPU_SI
+	{0x1002, 0x6780, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x6784, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x6788, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x678A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x6790, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x6791, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x6792, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x6798, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x6799, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x679A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x679B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x679E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x679F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
+	{0x1002, 0x6800, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN|AMD_IS_MOBILITY},
+	{0x1002, 0x6801, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN|AMD_IS_MOBILITY},
+	{0x1002, 0x6802, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN|AMD_IS_MOBILITY},
+	{0x1002, 0x6806, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN},
+	{0x1002, 0x6808, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN},
+	{0x1002, 0x6809, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN},
+	{0x1002, 0x6810, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN},
+	{0x1002, 0x6811, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN},
+	{0x1002, 0x6816, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN},
+	{0x1002, 0x6817, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN},
+	{0x1002, 0x6818, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN},
+	{0x1002, 0x6819, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PITCAIRN},
+	{0x1002, 0x6600, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6601, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6602, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6603, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6604, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6605, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6606, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6607, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6608, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND},
+	{0x1002, 0x6610, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND},
+	{0x1002, 0x6611, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND},
+	{0x1002, 0x6613, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND},
+	{0x1002, 0x6617, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6620, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6621, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6623, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND|AMD_IS_MOBILITY},
+	{0x1002, 0x6631, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_OLAND},
+	{0x1002, 0x6820, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6821, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6822, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6823, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6824, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6825, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6826, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6827, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6828, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x6829, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x682A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x682B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x682C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x682D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x682F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6830, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6831, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE|AMD_IS_MOBILITY},
+	{0x1002, 0x6835, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x6837, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x6838, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x6839, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x683B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x683D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x683F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_VERDE},
+	{0x1002, 0x6660, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|AMD_IS_MOBILITY},
+	{0x1002, 0x6663, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|AMD_IS_MOBILITY},
+	{0x1002, 0x6664, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|AMD_IS_MOBILITY},
+	{0x1002, 0x6665, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|AMD_IS_MOBILITY},
+	{0x1002, 0x6667, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|AMD_IS_MOBILITY},
+	{0x1002, 0x666F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAINAN|AMD_IS_MOBILITY},
+#endif
 #ifdef CONFIG_DRM_AMDGPU_CIK
 	/* Kaveri */
 	{0x1002, 0x1304, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
@@ -341,7 +434,7 @@ static int amdgpu_kick_out_firmware_fb(struct pci_dev *pdev)
 #ifdef CONFIG_X86
 	primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
 #endif
-	remove_conflicting_framebuffers(ap, "amdgpudrmfb", primary);
+	drm_fb_helper_remove_conflicting_framebuffers(ap, "amdgpudrmfb", primary);
 	kfree(ap);
 
 	return 0;
@@ -383,32 +476,70 @@ amdgpu_pci_remove(struct pci_dev *pdev)
 	drm_put_dev(dev);
 }
 
+static void
+amdgpu_pci_shutdown(struct pci_dev *pdev)
+{
+	/* if we are running in a VM, make sure the device
+	 * torn down properly on reboot/shutdown.
+	 * unfortunately we can't detect certain
+	 * hypervisors so just do this all the time.
+	 */
+	amdgpu_pci_remove(pdev);
+}
+
 static int amdgpu_pmops_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
+
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
-	return amdgpu_suspend_kms(drm_dev, true, true);
+	return amdgpu_device_suspend(drm_dev, true, true);
 }
 
 static int amdgpu_pmops_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
-	return amdgpu_resume_kms(drm_dev, true, true);
+
+	/* GPU comes up enabled by the bios on resume */
+	if (amdgpu_device_is_px(drm_dev)) {
+		pm_runtime_disable(dev);
+		pm_runtime_set_active(dev);
+		pm_runtime_enable(dev);
+	}
+
+	return amdgpu_device_resume(drm_dev, true, true);
 }
 
 static int amdgpu_pmops_freeze(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
+
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
-	return amdgpu_suspend_kms(drm_dev, false, true);
+	return amdgpu_device_suspend(drm_dev, false, true);
 }
 
 static int amdgpu_pmops_thaw(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
+
+	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	return amdgpu_device_resume(drm_dev, false, true);
+}
+
+static int amdgpu_pmops_poweroff(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	return amdgpu_device_suspend(drm_dev, true, true);
+}
+
+static int amdgpu_pmops_restore(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
-	return amdgpu_resume_kms(drm_dev, false, true);
+	return amdgpu_device_resume(drm_dev, false, true);
 }
 
 static int amdgpu_pmops_runtime_suspend(struct device *dev)
@@ -426,7 +557,7 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev)
 	drm_kms_helper_poll_disable(drm_dev);
 	vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_OFF);
 
-	ret = amdgpu_suspend_kms(drm_dev, false, false);
+	ret = amdgpu_device_suspend(drm_dev, false, false);
 	pci_save_state(pdev);
 	pci_disable_device(pdev);
 	pci_ignore_hotplug(pdev);
@@ -459,7 +590,7 @@ static int amdgpu_pmops_runtime_resume(struct device *dev)
 		return ret;
 	pci_set_master(pdev);
 
-	ret = amdgpu_resume_kms(drm_dev, false, false);
+	ret = amdgpu_device_resume(drm_dev, false, false);
 	drm_kms_helper_poll_enable(drm_dev);
 	vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_ON);
 	drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
@@ -513,8 +644,8 @@ static const struct dev_pm_ops amdgpu_pm_ops = {
 	.resume = amdgpu_pmops_resume,
 	.freeze = amdgpu_pmops_freeze,
 	.thaw = amdgpu_pmops_thaw,
-	.poweroff = amdgpu_pmops_freeze,
-	.restore = amdgpu_pmops_resume,
+	.poweroff = amdgpu_pmops_poweroff,
+	.restore = amdgpu_pmops_restore,
 	.runtime_suspend = amdgpu_pmops_runtime_suspend,
 	.runtime_resume = amdgpu_pmops_runtime_resume,
 	.runtime_idle = amdgpu_pmops_runtime_idle,
@@ -596,6 +727,7 @@ static struct pci_driver amdgpu_kms_pci_driver = {
 	.id_table = pciidlist,
 	.probe = amdgpu_pci_probe,
 	.remove = amdgpu_pci_remove,
+	.shutdown = amdgpu_pci_shutdown,
 	.driver.pm = &amdgpu_pm_ops,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 919146780a15..9fb8aa4d6bae 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -25,7 +25,7 @@
  */
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/fb.h>
+#include <linux/pm_runtime.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_crtc.h>
@@ -48,8 +48,35 @@ struct amdgpu_fbdev {
 	struct amdgpu_device *adev;
 };
 
+static int
+amdgpufb_open(struct fb_info *info, int user)
+{
+	struct amdgpu_fbdev *rfbdev = info->par;
+	struct amdgpu_device *adev = rfbdev->adev;
+	int ret = pm_runtime_get_sync(adev->ddev->dev);
+	if (ret < 0 && ret != -EACCES) {
+		pm_runtime_mark_last_busy(adev->ddev->dev);
+		pm_runtime_put_autosuspend(adev->ddev->dev);
+		return ret;
+	}
+	return 0;
+}
+
+static int
+amdgpufb_release(struct fb_info *info, int user)
+{
+	struct amdgpu_fbdev *rfbdev = info->par;
+	struct amdgpu_device *adev = rfbdev->adev;
+
+	pm_runtime_mark_last_busy(adev->ddev->dev);
+	pm_runtime_put_autosuspend(adev->ddev->dev);
+	return 0;
+}
+
 static struct fb_ops amdgpufb_ops = {
 	.owner = THIS_MODULE,
+	.fb_open = amdgpufb_open,
+	.fb_release = amdgpufb_release,
 	.fb_check_var = drm_fb_helper_check_var,
 	.fb_set_par = drm_fb_helper_set_par,
 	.fb_fillrect = drm_fb_helper_cfb_fillrect,
@@ -88,14 +115,14 @@ int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tile
 
 static void amdgpufb_destroy_pinned_object(struct drm_gem_object *gobj)
 {
-	struct amdgpu_bo *rbo = gem_to_amdgpu_bo(gobj);
+	struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
 	int ret;
 
-	ret = amdgpu_bo_reserve(rbo, false);
+	ret = amdgpu_bo_reserve(abo, false);
 	if (likely(ret == 0)) {
-		amdgpu_bo_kunmap(rbo);
-		amdgpu_bo_unpin(rbo);
-		amdgpu_bo_unreserve(rbo);
+		amdgpu_bo_kunmap(abo);
+		amdgpu_bo_unpin(abo);
+		amdgpu_bo_unreserve(abo);
 	}
 	drm_gem_object_unreference_unlocked(gobj);
 }
@@ -106,7 +133,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
 {
 	struct amdgpu_device *adev = rfbdev->adev;
 	struct drm_gem_object *gobj = NULL;
-	struct amdgpu_bo *rbo = NULL;
+	struct amdgpu_bo *abo = NULL;
 	bool fb_tiled = false; /* useful for testing */
 	u32 tiling_flags = 0;
 	int ret;
@@ -132,30 +159,30 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
 		       aligned_size);
 		return -ENOMEM;
 	}
-	rbo = gem_to_amdgpu_bo(gobj);
+	abo = gem_to_amdgpu_bo(gobj);
 
 	if (fb_tiled)
 		tiling_flags = AMDGPU_TILING_SET(ARRAY_MODE, GRPH_ARRAY_2D_TILED_THIN1);
 
-	ret = amdgpu_bo_reserve(rbo, false);
+	ret = amdgpu_bo_reserve(abo, false);
 	if (unlikely(ret != 0))
 		goto out_unref;
 
 	if (tiling_flags) {
-		ret = amdgpu_bo_set_tiling_flags(rbo,
+		ret = amdgpu_bo_set_tiling_flags(abo,
 						 tiling_flags);
 		if (ret)
 			dev_err(adev->dev, "FB failed to set tiling flags\n");
 	}
 
 
-	ret = amdgpu_bo_pin_restricted(rbo, AMDGPU_GEM_DOMAIN_VRAM, 0, 0, NULL);
+	ret = amdgpu_bo_pin_restricted(abo, AMDGPU_GEM_DOMAIN_VRAM, 0, 0, NULL);
 	if (ret) {
-		amdgpu_bo_unreserve(rbo);
+		amdgpu_bo_unreserve(abo);
 		goto out_unref;
 	}
-	ret = amdgpu_bo_kmap(rbo, NULL);
-	amdgpu_bo_unreserve(rbo);
+	ret = amdgpu_bo_kmap(abo, NULL);
+	amdgpu_bo_unreserve(abo);
 	if (ret) {
 		goto out_unref;
 	}
@@ -177,7 +204,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 	struct drm_framebuffer *fb = NULL;
 	struct drm_mode_fb_cmd2 mode_cmd;
 	struct drm_gem_object *gobj = NULL;
-	struct amdgpu_bo *rbo = NULL;
+	struct amdgpu_bo *abo = NULL;
 	int ret;
 	unsigned long tmp;
 
@@ -196,7 +223,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 		return ret;
 	}
 
-	rbo = gem_to_amdgpu_bo(gobj);
+	abo = gem_to_amdgpu_bo(gobj);
 
 	/* okay we have an object now allocate the framebuffer */
 	info = drm_fb_helper_alloc_fbi(helper);
@@ -219,7 +246,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 	/* setup helper */
 	rfbdev->helper.fb = fb;
 
-	memset_io(rbo->kptr, 0x0, amdgpu_bo_size(rbo));
+	memset_io(abo->kptr, 0x0, amdgpu_bo_size(abo));
 
 	strcpy(info->fix.id, "amdgpudrmfb");
 
@@ -228,11 +255,11 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 	info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &amdgpufb_ops;
 
-	tmp = amdgpu_bo_gpu_offset(rbo) - adev->mc.vram_start;
+	tmp = amdgpu_bo_gpu_offset(abo) - adev->mc.vram_start;
 	info->fix.smem_start = adev->mc.aper_base + tmp;
-	info->fix.smem_len = amdgpu_bo_size(rbo);
-	info->screen_base = rbo->kptr;
-	info->screen_size = amdgpu_bo_size(rbo);
+	info->fix.smem_len = amdgpu_bo_size(abo);
+	info->screen_base = abo->kptr;
+	info->screen_size = amdgpu_bo_size(abo);
 
 	drm_fb_helper_fill_var(info, &rfbdev->helper, sizes->fb_width, sizes->fb_height);
 
@@ -249,7 +276,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 
 	DRM_INFO("fb mappable at 0x%lX\n",  info->fix.smem_start);
 	DRM_INFO("vram apper at 0x%lX\n",  (unsigned long)adev->mc.aper_base);
-	DRM_INFO("size %lu\n", (unsigned long)amdgpu_bo_size(rbo));
+	DRM_INFO("size %lu\n", (unsigned long)amdgpu_bo_size(abo));
 	DRM_INFO("fb depth is %d\n", fb->depth);
 	DRM_INFO("   pitch is %d\n", fb->pitches[0]);
 
@@ -259,7 +286,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 out_destroy_fbi:
 	drm_fb_helper_release_fbi(helper);
 out_unref:
-	if (rbo) {
+	if (abo) {
 
 	}
 	if (fb && ret) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 0b109aebfec6..3a2e42f4b897 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -454,6 +454,7 @@ void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
 			fence_put(ring->fence_drv.fences[j]);
 		kfree(ring->fence_drv.fences);
+		ring->fence_drv.fences = NULL;
 		ring->fence_drv.initialized = false;
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index 921bce2df0b0..21a1242fc13b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -221,7 +221,7 @@ void amdgpu_gart_table_vram_free(struct amdgpu_device *adev)
  * Unbinds the requested pages from the gart page table and
  * replaces them with the dummy page (all asics).
  */
-void amdgpu_gart_unbind(struct amdgpu_device *adev, unsigned offset,
+void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
 			int pages)
 {
 	unsigned t;
@@ -238,7 +238,7 @@ void amdgpu_gart_unbind(struct amdgpu_device *adev, unsigned offset,
 	t = offset / AMDGPU_GPU_PAGE_SIZE;
 	p = t / (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
 	for (i = 0; i < pages; i++, p++) {
-#ifdef CONFIG_AMDGPU_GART_DEBUGFS
+#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 		adev->gart.pages[p] = NULL;
 #endif
 		page_base = adev->dummy_page.addr;
@@ -268,7 +268,7 @@ void amdgpu_gart_unbind(struct amdgpu_device *adev, unsigned offset,
  * (all asics).
  * Returns 0 for success, -EINVAL for failure.
  */
-int amdgpu_gart_bind(struct amdgpu_device *adev, unsigned offset,
+int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
 		     int pages, struct page **pagelist, dma_addr_t *dma_addr,
 		     uint32_t flags)
 {
@@ -286,7 +286,7 @@ int amdgpu_gart_bind(struct amdgpu_device *adev, unsigned offset,
 	p = t / (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
 
 	for (i = 0; i < pages; i++, p++) {
-#ifdef CONFIG_AMDGPU_GART_DEBUGFS
+#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 		adev->gart.pages[p] = pagelist[i];
 #endif
 		if (adev->gart.ptr) {
@@ -331,7 +331,7 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
 	DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
 		 adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);
 
-#ifdef CONFIG_AMDGPU_GART_DEBUGFS
+#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 	/* Allocate pages table */
 	adev->gart.pages = vzalloc(sizeof(void *) * adev->gart.num_cpu_pages);
 	if (adev->gart.pages == NULL) {
@@ -357,7 +357,7 @@ void amdgpu_gart_fini(struct amdgpu_device *adev)
 		amdgpu_gart_unbind(adev, 0, adev->gart.num_cpu_pages);
 	}
 	adev->gart.ready = false;
-#ifdef CONFIG_AMDGPU_GART_DEBUGFS
+#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 	vfree(adev->gart.pages);
 	adev->gart.pages = NULL;
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
index 503d54098128..e73728d90388 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
@@ -31,14 +31,6 @@
 #define AMDGPU_GWS_SHIFT	PAGE_SHIFT
 #define AMDGPU_OA_SHIFT		PAGE_SHIFT
 
-#define AMDGPU_PL_GDS		TTM_PL_PRIV0
-#define AMDGPU_PL_GWS		TTM_PL_PRIV1
-#define AMDGPU_PL_OA		TTM_PL_PRIV2
-
-#define AMDGPU_PL_FLAG_GDS		TTM_PL_FLAG_PRIV0
-#define AMDGPU_PL_FLAG_GWS		TTM_PL_FLAG_PRIV1
-#define AMDGPU_PL_FLAG_OA		TTM_PL_FLAG_PRIV2
-
 struct amdgpu_ring;
 struct amdgpu_bo;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 88fbed2389c0..a7ea9a3b454e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -118,23 +118,23 @@ void amdgpu_gem_force_release(struct amdgpu_device *adev)
  */
 int amdgpu_gem_object_open(struct drm_gem_object *obj, struct drm_file *file_priv)
 {
-	struct amdgpu_bo *rbo = gem_to_amdgpu_bo(obj);
-	struct amdgpu_device *adev = rbo->adev;
+	struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
+	struct amdgpu_device *adev = abo->adev;
 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
 	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_va *bo_va;
 	int r;
-	r = amdgpu_bo_reserve(rbo, false);
+	r = amdgpu_bo_reserve(abo, false);
 	if (r)
 		return r;
 
-	bo_va = amdgpu_vm_bo_find(vm, rbo);
+	bo_va = amdgpu_vm_bo_find(vm, abo);
 	if (!bo_va) {
-		bo_va = amdgpu_vm_bo_add(adev, vm, rbo);
+		bo_va = amdgpu_vm_bo_add(adev, vm, abo);
 	} else {
 		++bo_va->ref_count;
 	}
-	amdgpu_bo_unreserve(rbo);
+	amdgpu_bo_unreserve(abo);
 	return 0;
 }
 
@@ -528,7 +528,7 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 		goto error_unreserve;
 
 	if (operation == AMDGPU_VA_OP_MAP)
-		r = amdgpu_vm_bo_update(adev, bo_va, &bo_va->bo->tbo.mem);
+		r = amdgpu_vm_bo_update(adev, bo_va, false);
 
 error_unreserve:
 	ttm_eu_backoff_reservation(&ticket, &list);
@@ -547,7 +547,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 	struct drm_gem_object *gobj;
 	struct amdgpu_device *adev = dev->dev_private;
 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
-	struct amdgpu_bo *rbo;
+	struct amdgpu_bo *abo;
 	struct amdgpu_bo_va *bo_va;
 	struct ttm_validate_buffer tv, tv_pd;
 	struct ww_acquire_ctx ticket;
@@ -587,10 +587,10 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 	gobj = drm_gem_object_lookup(filp, args->handle);
 	if (gobj == NULL)
 		return -ENOENT;
-	rbo = gem_to_amdgpu_bo(gobj);
+	abo = gem_to_amdgpu_bo(gobj);
 	INIT_LIST_HEAD(&list);
 	INIT_LIST_HEAD(&duplicates);
-	tv.bo = &rbo->tbo;
+	tv.bo = &abo->tbo;
 	tv.shared = true;
 	list_add(&tv.head, &list);
 
@@ -604,7 +604,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 		return r;
 	}
 
-	bo_va = amdgpu_vm_bo_find(&fpriv->vm, rbo);
+	bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
 	if (!bo_va) {
 		ttm_eu_backoff_reservation(&ticket, &list);
 		drm_gem_object_unreference_unlocked(gobj);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
new file mode 100644
index 000000000000..f86c84427778
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -0,0 +1,239 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+
+#include <drm/drmP.h>
+#include "amdgpu.h"
+
+struct amdgpu_gtt_mgr {
+	struct drm_mm mm;
+	spinlock_t lock;
+	uint64_t available;
+};
+
+/**
+ * amdgpu_gtt_mgr_init - init GTT manager and DRM MM
+ *
+ * @man: TTM memory type manager
+ * @p_size: maximum size of GTT
+ *
+ * Allocate and initialize the GTT manager.
+ */
+static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
+			       unsigned long p_size)
+{
+	struct amdgpu_gtt_mgr *mgr;
+
+	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
+	if (!mgr)
+		return -ENOMEM;
+
+	drm_mm_init(&mgr->mm, 0, p_size);
+	spin_lock_init(&mgr->lock);
+	mgr->available = p_size;
+	man->priv = mgr;
+	return 0;
+}
+
+/**
+ * amdgpu_gtt_mgr_fini - free and destroy GTT manager
+ *
+ * @man: TTM memory type manager
+ *
+ * Destroy and free the GTT manager, returns -EBUSY if ranges are still
+ * allocated inside it.
+ */
+static int amdgpu_gtt_mgr_fini(struct ttm_mem_type_manager *man)
+{
+	struct amdgpu_gtt_mgr *mgr = man->priv;
+
+	spin_lock(&mgr->lock);
+	if (!drm_mm_clean(&mgr->mm)) {
+		spin_unlock(&mgr->lock);
+		return -EBUSY;
+	}
+
+	drm_mm_takedown(&mgr->mm);
+	spin_unlock(&mgr->lock);
+	kfree(mgr);
+	man->priv = NULL;
+	return 0;
+}
+
+/**
+ * amdgpu_gtt_mgr_alloc - allocate new ranges
+ *
+ * @man: TTM memory type manager
+ * @tbo: TTM BO we need this range for
+ * @place: placement flags and restrictions
+ * @mem: the resulting mem object
+ *
+ * Allocate the address space for a node.
+ */
+int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
+			 struct ttm_buffer_object *tbo,
+			 const struct ttm_place *place,
+			 struct ttm_mem_reg *mem)
+{
+	struct amdgpu_gtt_mgr *mgr = man->priv;
+	struct drm_mm_node *node = mem->mm_node;
+	enum drm_mm_search_flags sflags = DRM_MM_SEARCH_BEST;
+	enum drm_mm_allocator_flags aflags = DRM_MM_CREATE_DEFAULT;
+	unsigned long fpfn, lpfn;
+	int r;
+
+	if (node->start != AMDGPU_BO_INVALID_OFFSET)
+		return 0;
+
+	if (place)
+		fpfn = place->fpfn;
+	else
+		fpfn = 0;
+
+	if (place && place->lpfn)
+		lpfn = place->lpfn;
+	else
+		lpfn = man->size;
+
+	if (place && place->flags & TTM_PL_FLAG_TOPDOWN) {
+		sflags = DRM_MM_SEARCH_BELOW;
+		aflags = DRM_MM_CREATE_TOP;
+	}
+
+	spin_lock(&mgr->lock);
+	r = drm_mm_insert_node_in_range_generic(&mgr->mm, node, mem->num_pages,
+						mem->page_alignment, 0,
+						fpfn, lpfn, sflags, aflags);
+	spin_unlock(&mgr->lock);
+
+	if (!r) {
+		mem->start = node->start;
+		if (&tbo->mem == mem)
+			tbo->offset = (tbo->mem.start << PAGE_SHIFT) +
+			    tbo->bdev->man[tbo->mem.mem_type].gpu_offset;
+	}
+
+	return r;
+}
+
+/**
+ * amdgpu_gtt_mgr_new - allocate a new node
+ *
+ * @man: TTM memory type manager
+ * @tbo: TTM BO we need this range for
+ * @place: placement flags and restrictions
+ * @mem: the resulting mem object
+ *
+ * Dummy, allocate the node but no space for it yet.
+ */
+static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
+			      struct ttm_buffer_object *tbo,
+			      const struct ttm_place *place,
+			      struct ttm_mem_reg *mem)
+{
+	struct amdgpu_gtt_mgr *mgr = man->priv;
+	struct drm_mm_node *node;
+	int r;
+
+	spin_lock(&mgr->lock);
+	if (mgr->available < mem->num_pages) {
+		spin_unlock(&mgr->lock);
+		return 0;
+	}
+	mgr->available -= mem->num_pages;
+	spin_unlock(&mgr->lock);
+
+	node = kzalloc(sizeof(*node), GFP_KERNEL);
+	if (!node)
+		return -ENOMEM;
+
+	node->start = AMDGPU_BO_INVALID_OFFSET;
+	mem->mm_node = node;
+
+	if (place->fpfn || place->lpfn || place->flags & TTM_PL_FLAG_TOPDOWN) {
+		r = amdgpu_gtt_mgr_alloc(man, tbo, place, mem);
+		if (unlikely(r)) {
+			kfree(node);
+			mem->mm_node = NULL;
+		}
+	} else {
+		mem->start = node->start;
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_gtt_mgr_del - free ranges
+ *
+ * @man: TTM memory type manager
+ * @tbo: TTM BO we need this range for
+ * @place: placement flags and restrictions
+ * @mem: TTM memory object
+ *
+ * Free the allocated GTT again.
+ */
+static void amdgpu_gtt_mgr_del(struct ttm_mem_type_manager *man,
+			       struct ttm_mem_reg *mem)
+{
+	struct amdgpu_gtt_mgr *mgr = man->priv;
+	struct drm_mm_node *node = mem->mm_node;
+
+	if (!node)
+		return;
+
+	spin_lock(&mgr->lock);
+	if (node->start != AMDGPU_BO_INVALID_OFFSET)
+		drm_mm_remove_node(node);
+	mgr->available += mem->num_pages;
+	spin_unlock(&mgr->lock);
+
+	kfree(node);
+	mem->mm_node = NULL;
+}
+
+/**
+ * amdgpu_gtt_mgr_debug - dump VRAM table
+ *
+ * @man: TTM memory type manager
+ * @prefix: text prefix
+ *
+ * Dump the table content using printk.
+ */
+static void amdgpu_gtt_mgr_debug(struct ttm_mem_type_manager *man,
+				  const char *prefix)
+{
+	struct amdgpu_gtt_mgr *mgr = man->priv;
+
+	spin_lock(&mgr->lock);
+	drm_mm_debug_table(&mgr->mm, prefix);
+	spin_unlock(&mgr->lock);
+}
+
+const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func = {
+	amdgpu_gtt_mgr_init,
+	amdgpu_gtt_mgr_fini,
+	amdgpu_gtt_mgr_new,
+	amdgpu_gtt_mgr_del,
+	amdgpu_gtt_mgr_debug
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
index 31a676376d73..91d367399956 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c
@@ -158,8 +158,8 @@ static const struct i2c_algorithm amdgpu_atombios_i2c_algo = {
 };
 
 struct amdgpu_i2c_chan *amdgpu_i2c_create(struct drm_device *dev,
-					    struct amdgpu_i2c_bus_rec *rec,
-					    const char *name)
+					  const struct amdgpu_i2c_bus_rec *rec,
+					  const char *name)
 {
 	struct amdgpu_i2c_chan *i2c;
 	int ret;
@@ -186,10 +186,8 @@ struct amdgpu_i2c_chan *amdgpu_i2c_create(struct drm_device *dev,
 			 "AMDGPU i2c hw bus %s", name);
 		i2c->adapter.algo = &amdgpu_atombios_i2c_algo;
 		ret = i2c_add_adapter(&i2c->adapter);
-		if (ret) {
-			DRM_ERROR("Failed to register hw i2c %s\n", name);
+		if (ret)
 			goto out_free;
-		}
 	} else {
 		/* set the amdgpu bit adapter */
 		snprintf(i2c->adapter.name, sizeof(i2c->adapter.name),
@@ -222,6 +220,7 @@ void amdgpu_i2c_destroy(struct amdgpu_i2c_chan *i2c)
 {
 	if (!i2c)
 		return;
+	WARN_ON(i2c->has_aux);
 	i2c_del_adapter(&i2c->adapter);
 	kfree(i2c);
 }
@@ -251,8 +250,8 @@ void amdgpu_i2c_fini(struct amdgpu_device *adev)
 
 /* Add additional buses */
 void amdgpu_i2c_add(struct amdgpu_device *adev,
-		     struct amdgpu_i2c_bus_rec *rec,
-		     const char *name)
+		    const struct amdgpu_i2c_bus_rec *rec,
+		    const char *name)
 {
 	struct drm_device *dev = adev->ddev;
 	int i;
@@ -268,7 +267,7 @@ void amdgpu_i2c_add(struct amdgpu_device *adev,
 /* looks up bus based on id */
 struct amdgpu_i2c_chan *
 amdgpu_i2c_lookup(struct amdgpu_device *adev,
-		   struct amdgpu_i2c_bus_rec *i2c_bus)
+		  const struct amdgpu_i2c_bus_rec *i2c_bus)
 {
 	int i;
 
@@ -338,7 +337,7 @@ static void amdgpu_i2c_put_byte(struct amdgpu_i2c_chan *i2c_bus,
 
 /* ddc router switching */
 void
-amdgpu_i2c_router_select_ddc_port(struct amdgpu_connector *amdgpu_connector)
+amdgpu_i2c_router_select_ddc_port(const struct amdgpu_connector *amdgpu_connector)
 {
 	u8 val;
 
@@ -367,7 +366,7 @@ amdgpu_i2c_router_select_ddc_port(struct amdgpu_connector *amdgpu_connector)
 
 /* clock/data router switching */
 void
-amdgpu_i2c_router_select_cd_port(struct amdgpu_connector *amdgpu_connector)
+amdgpu_i2c_router_select_cd_port(const struct amdgpu_connector *amdgpu_connector)
 {
 	u8 val;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.h
index d81e19b53973..63c2ff7499e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.h
@@ -25,20 +25,20 @@
 #define __AMDGPU_I2C_H__
 
 struct amdgpu_i2c_chan *amdgpu_i2c_create(struct drm_device *dev,
-					    struct amdgpu_i2c_bus_rec *rec,
-					    const char *name);
+					  const struct amdgpu_i2c_bus_rec *rec,
+					  const char *name);
 void amdgpu_i2c_destroy(struct amdgpu_i2c_chan *i2c);
 void amdgpu_i2c_init(struct amdgpu_device *adev);
 void amdgpu_i2c_fini(struct amdgpu_device *adev);
 void amdgpu_i2c_add(struct amdgpu_device *adev,
-		     struct amdgpu_i2c_bus_rec *rec,
-		     const char *name);
+		    const struct amdgpu_i2c_bus_rec *rec,
+		    const char *name);
 struct amdgpu_i2c_chan *
 amdgpu_i2c_lookup(struct amdgpu_device *adev,
-		   struct amdgpu_i2c_bus_rec *i2c_bus);
+		  const struct amdgpu_i2c_bus_rec *i2c_bus);
 void
-amdgpu_i2c_router_select_ddc_port(struct amdgpu_connector *amdgpu_connector);
+amdgpu_i2c_router_select_ddc_port(const struct amdgpu_connector *connector);
 void
-amdgpu_i2c_router_select_cd_port(struct amdgpu_connector *amdgpu_connector);
+amdgpu_i2c_router_select_cd_port(const struct amdgpu_connector *connector);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index a31d7ef3032c..6a6c86c9c169 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -124,7 +124,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	bool skip_preamble, need_ctx_switch;
 	unsigned patch_offset = ~0;
 	struct amdgpu_vm *vm;
-	uint64_t ctx;
+	uint64_t fence_ctx;
+	uint32_t status = 0, alloc_size;
 
 	unsigned i;
 	int r = 0;
@@ -135,14 +136,14 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	/* ring tests don't use a job */
 	if (job) {
 		vm = job->vm;
-		ctx = job->ctx;
+		fence_ctx = job->fence_ctx;
 	} else {
 		vm = NULL;
-		ctx = 0;
+		fence_ctx = 0;
 	}
 
 	if (!ring->ready) {
-		dev_err(adev->dev, "couldn't schedule ib\n");
+		dev_err(adev->dev, "couldn't schedule ib on ring <%s>\n", ring->name);
 		return -EINVAL;
 	}
 
@@ -151,7 +152,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 		return -EINVAL;
 	}
 
-	r = amdgpu_ring_alloc(ring, 256 * num_ibs);
+	alloc_size = amdgpu_ring_get_dma_frame_size(ring) +
+		num_ibs * amdgpu_ring_get_emit_ib_size(ring);
+
+	r = amdgpu_ring_alloc(ring, alloc_size);
 	if (r) {
 		dev_err(adev->dev, "scheduling IB failed (%d).\n", r);
 		return r;
@@ -174,13 +178,22 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	/* always set cond_exec_polling to CONTINUE */
 	*ring->cond_exe_cpu_addr = 1;
 
-	skip_preamble = ring->current_ctx == ctx;
-	need_ctx_switch = ring->current_ctx != ctx;
+	skip_preamble = ring->current_ctx == fence_ctx;
+	need_ctx_switch = ring->current_ctx != fence_ctx;
+	if (job && ring->funcs->emit_cntxcntl) {
+		if (need_ctx_switch)
+			status |= AMDGPU_HAVE_CTX_SWITCH;
+		status |= job->preamble_status;
+		amdgpu_ring_emit_cntxcntl(ring, status);
+	}
+
 	for (i = 0; i < num_ibs; ++i) {
 		ib = &ibs[i];
 
 		/* drop preamble IBs if we don't have a context switch */
-		if ((ib->flags & AMDGPU_IB_FLAG_PREAMBLE) && skip_preamble)
+		if ((ib->flags & AMDGPU_IB_FLAG_PREAMBLE) &&
+			skip_preamble &&
+			!(status & AMDGPU_PREAMBLE_IB_PRESENT_FIRST))
 			continue;
 
 		amdgpu_ring_emit_ib(ring, ib, job ? job->vm_id : 0,
@@ -209,7 +222,9 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	if (patch_offset != ~0 && ring->funcs->patch_cond_exec)
 		amdgpu_ring_patch_cond_exec(ring, patch_offset);
 
-	ring->current_ctx = ctx;
+	ring->current_ctx = fence_ctx;
+	if (ring->funcs->emit_switch_buffer)
+		amdgpu_ring_emit_switch_buffer(ring);
 	amdgpu_ring_commit(ring);
 	return 0;
 }
@@ -280,7 +295,7 @@ void amdgpu_ib_pool_fini(struct amdgpu_device *adev)
 int amdgpu_ib_ring_tests(struct amdgpu_device *adev)
 {
 	unsigned i;
-	int r;
+	int r, ret = 0;
 
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 		struct amdgpu_ring *ring = adev->rings[i];
@@ -301,10 +316,11 @@ int amdgpu_ib_ring_tests(struct amdgpu_device *adev)
 			} else {
 				/* still not good, but we can live with it */
 				DRM_ERROR("amdgpu: failed testing IB on ring %d (%d).\n", i, r);
+				ret = r;
 			}
 		}
 	}
-	return 0;
+	return ret;
 }
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
index 534fc04e80fd..3ab4c65ecc8b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
@@ -40,32 +40,15 @@ static int amdgpu_ih_ring_alloc(struct amdgpu_device *adev)
 
 	/* Allocate ring buffer */
 	if (adev->irq.ih.ring_obj == NULL) {
-		r = amdgpu_bo_create(adev, adev->irq.ih.ring_size,
-				     PAGE_SIZE, true,
-				     AMDGPU_GEM_DOMAIN_GTT, 0,
-				     NULL, NULL, &adev->irq.ih.ring_obj);
+		r = amdgpu_bo_create_kernel(adev, adev->irq.ih.ring_size,
+					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+					    &adev->irq.ih.ring_obj,
+					    &adev->irq.ih.gpu_addr,
+					    (void **)&adev->irq.ih.ring);
 		if (r) {
 			DRM_ERROR("amdgpu: failed to create ih ring buffer (%d).\n", r);
 			return r;
 		}
-		r = amdgpu_bo_reserve(adev->irq.ih.ring_obj, false);
-		if (unlikely(r != 0))
-			return r;
-		r = amdgpu_bo_pin(adev->irq.ih.ring_obj,
-				  AMDGPU_GEM_DOMAIN_GTT,
-				  &adev->irq.ih.gpu_addr);
-		if (r) {
-			amdgpu_bo_unreserve(adev->irq.ih.ring_obj);
-			DRM_ERROR("amdgpu: failed to pin ih ring buffer (%d).\n", r);
-			return r;
-		}
-		r = amdgpu_bo_kmap(adev->irq.ih.ring_obj,
-				   (void **)&adev->irq.ih.ring);
-		amdgpu_bo_unreserve(adev->irq.ih.ring_obj);
-		if (r) {
-			DRM_ERROR("amdgpu: failed to map ih ring buffer (%d).\n", r);
-			return r;
-		}
 	}
 	return 0;
 }
@@ -136,8 +119,6 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size,
  */
 void amdgpu_ih_ring_fini(struct amdgpu_device *adev)
 {
-	int r;
-
 	if (adev->irq.ih.use_bus_addr) {
 		if (adev->irq.ih.ring) {
 			/* add 8 bytes for the rptr/wptr shadows and
@@ -149,17 +130,9 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev)
 			adev->irq.ih.ring = NULL;
 		}
 	} else {
-		if (adev->irq.ih.ring_obj) {
-			r = amdgpu_bo_reserve(adev->irq.ih.ring_obj, false);
-			if (likely(r == 0)) {
-				amdgpu_bo_kunmap(adev->irq.ih.ring_obj);
-				amdgpu_bo_unpin(adev->irq.ih.ring_obj);
-				amdgpu_bo_unreserve(adev->irq.ih.ring_obj);
-			}
-			amdgpu_bo_unref(&adev->irq.ih.ring_obj);
-			adev->irq.ih.ring = NULL;
-			adev->irq.ih.ring_obj = NULL;
-		}
+		amdgpu_bo_free_kernel(&adev->irq.ih.ring_obj,
+				      &adev->irq.ih.gpu_addr,
+				      (void **)&adev->irq.ih.ring);
 		amdgpu_wb_free(adev, adev->irq.ih.wptr_offs);
 		amdgpu_wb_free(adev, adev->irq.ih.rptr_offs);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
index 7ef09352e534..f016464035b8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
@@ -70,6 +70,7 @@ struct amdgpu_irq {
 	/* gen irq stuff */
 	struct irq_domain		*domain; /* GPU irq controller domain */
 	unsigned			virq[AMDGPU_MAX_IRQ_SRC_ID];
+	uint32_t                        srbm_soft_reset;
 };
 
 void amdgpu_irq_preinstall(struct drm_device *dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 6674d40eb3ab..8c5807994073 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -91,7 +91,7 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
 		amdgpu_ib_free(job->adev, &job->ibs[i], f);
 }
 
-void amdgpu_job_free_cb(struct amd_sched_job *s_job)
+static void amdgpu_job_free_cb(struct amd_sched_job *s_job)
 {
 	struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
 
@@ -124,7 +124,7 @@ int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
 		return r;
 
 	job->owner = owner;
-	job->ctx = entity->fence_context;
+	job->fence_ctx = entity->fence_context;
 	*f = fence_get(&job->base.s_fence->finished);
 	amdgpu_job_free_resources(job);
 	amd_sched_entity_push_job(&job->base);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index d942654a1de0..c2c7fb140338 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -292,14 +292,14 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 			type = AMD_IP_BLOCK_TYPE_UVD;
 			ring_mask = adev->uvd.ring.ready ? 1 : 0;
 			ib_start_alignment = AMDGPU_GPU_PAGE_SIZE;
-			ib_size_alignment = 8;
+			ib_size_alignment = 16;
 			break;
 		case AMDGPU_HW_IP_VCE:
 			type = AMD_IP_BLOCK_TYPE_VCE;
-			for (i = 0; i < AMDGPU_MAX_VCE_RINGS; i++)
+			for (i = 0; i < adev->vce.num_rings; i++)
 				ring_mask |= ((adev->vce.ring[i].ready ? 1 : 0) << i);
 			ib_start_alignment = AMDGPU_GPU_PAGE_SIZE;
-			ib_size_alignment = 8;
+			ib_size_alignment = 1;
 			break;
 		default:
 			return -EINVAL;
@@ -373,6 +373,9 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 	case AMDGPU_INFO_NUM_BYTES_MOVED:
 		ui64 = atomic64_read(&adev->num_bytes_moved);
 		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
+	case AMDGPU_INFO_NUM_EVICTIONS:
+		ui64 = atomic64_read(&adev->num_evictions);
+		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 	case AMDGPU_INFO_VRAM_USAGE:
 		ui64 = atomic64_read(&adev->vram_usage);
 		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
@@ -539,12 +542,16 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 		return r;
 
 	fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
-	if (unlikely(!fpriv))
-		return -ENOMEM;
+	if (unlikely(!fpriv)) {
+		r = -ENOMEM;
+		goto out_suspend;
+	}
 
 	r = amdgpu_vm_init(adev, &fpriv->vm);
-	if (r)
-		goto error_free;
+	if (r) {
+		kfree(fpriv);
+		goto out_suspend;
+	}
 
 	mutex_init(&fpriv->bo_list_lock);
 	idr_init(&fpriv->bo_list_handles);
@@ -553,12 +560,9 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 
 	file_priv->driver_priv = fpriv;
 
+out_suspend:
 	pm_runtime_mark_last_busy(dev->dev);
 	pm_runtime_put_autosuspend(dev->dev);
-	return 0;
-
-error_free:
-	kfree(fpriv);
 
 	return r;
 }
@@ -597,6 +601,9 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 
 	kfree(fpriv);
 	file_priv->driver_priv = NULL;
+
+	pm_runtime_mark_last_busy(dev->dev);
+	pm_runtime_put_autosuspend(dev->dev);
 }
 
 /**
@@ -611,6 +618,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 void amdgpu_driver_preclose_kms(struct drm_device *dev,
 				struct drm_file *file_priv)
 {
+	pm_runtime_get_sync(dev->dev);
 }
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index 6b1d7d306564..7b0eff7d060b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -39,6 +39,8 @@
 #include <drm/drm_plane_helper.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
+#include <linux/hrtimer.h>
+#include "amdgpu_irq.h"
 
 struct amdgpu_bo;
 struct amdgpu_device;
@@ -339,6 +341,8 @@ struct amdgpu_mode_info {
 	int			num_dig; /* number of dig blocks */
 	int			disp_priority;
 	const struct amdgpu_display_funcs *funcs;
+	struct hrtimer vblank_timer;
+	enum amdgpu_interrupt_state vsync_timer_enabled;
 };
 
 #define AMDGPU_MAX_BL_LEVEL 0xFF
@@ -587,10 +591,10 @@ int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tile
 void amdgpu_print_display_setup(struct drm_device *dev);
 int amdgpu_modeset_create_props(struct amdgpu_device *adev);
 int amdgpu_crtc_set_config(struct drm_mode_set *set);
-int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
-			  struct drm_framebuffer *fb,
-			  struct drm_pending_vblank_event *event,
-			  uint32_t page_flip_flags);
+int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
+				 struct drm_framebuffer *fb,
+				 struct drm_pending_vblank_event *event,
+				 uint32_t page_flip_flags, uint32_t target);
 extern const struct drm_mode_config_funcs amdgpu_mode_funcs;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 6f0873c75a25..aa074fac0c7f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -38,20 +38,17 @@
 #include "amdgpu_trace.h"
 
 
-int amdgpu_ttm_init(struct amdgpu_device *adev);
-void amdgpu_ttm_fini(struct amdgpu_device *adev);
 
 static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev,
 						struct ttm_mem_reg *mem)
 {
-	u64 ret = 0;
-	if (mem->start << PAGE_SHIFT < adev->mc.visible_vram_size) {
-		ret = (u64)((mem->start << PAGE_SHIFT) + mem->size) >
-			   adev->mc.visible_vram_size ?
-			   adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) :
-			   mem->size;
-	}
-	return ret;
+	if (mem->start << PAGE_SHIFT >= adev->mc.visible_vram_size)
+		return 0;
+
+	return ((mem->start << PAGE_SHIFT) + mem->size) >
+		adev->mc.visible_vram_size ?
+		adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) :
+		mem->size;
 }
 
 static void amdgpu_update_memory_usage(struct amdgpu_device *adev,
@@ -99,6 +96,11 @@ static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 
 	drm_gem_object_release(&bo->gem_base);
 	amdgpu_bo_unref(&bo->parent);
+	if (!list_empty(&bo->shadow_list)) {
+		mutex_lock(&bo->adev->shadow_list_lock);
+		list_del_init(&bo->shadow_list);
+		mutex_unlock(&bo->adev->shadow_list_lock);
+	}
 	kfree(bo->metadata);
 	kfree(bo);
 }
@@ -112,90 +114,99 @@ bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
 
 static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
 				      struct ttm_placement *placement,
-				      struct ttm_place *placements,
+				      struct ttm_place *places,
 				      u32 domain, u64 flags)
 {
-	u32 c = 0, i;
-
-	placement->placement = placements;
-	placement->busy_placement = placements;
+	u32 c = 0;
 
 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
+		unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
+
 		if (flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS &&
-			adev->mc.visible_vram_size < adev->mc.real_vram_size) {
-			placements[c].fpfn =
-				adev->mc.visible_vram_size >> PAGE_SHIFT;
-			placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
-				TTM_PL_FLAG_VRAM | TTM_PL_FLAG_TOPDOWN;
+		    !(flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
+		    adev->mc.visible_vram_size < adev->mc.real_vram_size) {
+			places[c].fpfn = visible_pfn;
+			places[c].lpfn = 0;
+			places[c].flags = TTM_PL_FLAG_WC |
+				TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_VRAM |
+				TTM_PL_FLAG_TOPDOWN;
+			c++;
 		}
-		placements[c].fpfn = 0;
-		placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
+
+		places[c].fpfn = 0;
+		places[c].lpfn = 0;
+		places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
 			TTM_PL_FLAG_VRAM;
-		if (!(flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED))
-			placements[c - 1].flags |= TTM_PL_FLAG_TOPDOWN;
+		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
+			places[c].lpfn = visible_pfn;
+		else
+			places[c].flags |= TTM_PL_FLAG_TOPDOWN;
+		c++;
 	}
 
 	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
-		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) {
-			placements[c].fpfn = 0;
-			placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT |
+		places[c].fpfn = 0;
+		places[c].lpfn = 0;
+		places[c].flags = TTM_PL_FLAG_TT;
+		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
+			places[c].flags |= TTM_PL_FLAG_WC |
 				TTM_PL_FLAG_UNCACHED;
-		} else {
-			placements[c].fpfn = 0;
-			placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT;
-		}
+		else
+			places[c].flags |= TTM_PL_FLAG_CACHED;
+		c++;
 	}
 
 	if (domain & AMDGPU_GEM_DOMAIN_CPU) {
-		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) {
-			placements[c].fpfn = 0;
-			placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_SYSTEM |
+		places[c].fpfn = 0;
+		places[c].lpfn = 0;
+		places[c].flags = TTM_PL_FLAG_SYSTEM;
+		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
+			places[c].flags |= TTM_PL_FLAG_WC |
 				TTM_PL_FLAG_UNCACHED;
-		} else {
-			placements[c].fpfn = 0;
-			placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM;
-		}
+		else
+			places[c].flags |= TTM_PL_FLAG_CACHED;
+		c++;
 	}
 
 	if (domain & AMDGPU_GEM_DOMAIN_GDS) {
-		placements[c].fpfn = 0;
-		placements[c++].flags = TTM_PL_FLAG_UNCACHED |
-			AMDGPU_PL_FLAG_GDS;
+		places[c].fpfn = 0;
+		places[c].lpfn = 0;
+		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;
+		c++;
 	}
+
 	if (domain & AMDGPU_GEM_DOMAIN_GWS) {
-		placements[c].fpfn = 0;
-		placements[c++].flags = TTM_PL_FLAG_UNCACHED |
-			AMDGPU_PL_FLAG_GWS;
+		places[c].fpfn = 0;
+		places[c].lpfn = 0;
+		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;
+		c++;
 	}
+
 	if (domain & AMDGPU_GEM_DOMAIN_OA) {
-		placements[c].fpfn = 0;
-		placements[c++].flags = TTM_PL_FLAG_UNCACHED |
-			AMDGPU_PL_FLAG_OA;
+		places[c].fpfn = 0;
+		places[c].lpfn = 0;
+		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;
+		c++;
 	}
 
 	if (!c) {
-		placements[c].fpfn = 0;
-		placements[c++].flags = TTM_PL_MASK_CACHING |
-			TTM_PL_FLAG_SYSTEM;
+		places[c].fpfn = 0;
+		places[c].lpfn = 0;
+		places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
+		c++;
 	}
+
 	placement->num_placement = c;
-	placement->num_busy_placement = c;
+	placement->placement = places;
 
-	for (i = 0; i < c; i++) {
-		if ((flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
-			(placements[i].flags & TTM_PL_FLAG_VRAM) &&
-			!placements[i].fpfn)
-			placements[i].lpfn =
-				adev->mc.visible_vram_size >> PAGE_SHIFT;
-		else
-			placements[i].lpfn = 0;
-	}
+	placement->num_busy_placement = c;
+	placement->busy_placement = places;
 }
 
-void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *rbo, u32 domain)
+void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 {
-	amdgpu_ttm_placement_init(rbo->adev, &rbo->placement,
-				  rbo->placements, domain, rbo->flags);
+	amdgpu_ttm_placement_init(abo->adev, &abo->placement,
+				  abo->placements, domain, abo->flags);
 }
 
 static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
@@ -211,6 +222,98 @@ static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
 	bo->placement.busy_placement = bo->placements;
 }
 
+/**
+ * amdgpu_bo_create_kernel - create BO for kernel use
+ *
+ * @adev: amdgpu device object
+ * @size: size for the new BO
+ * @align: alignment for the new BO
+ * @domain: where to place it
+ * @bo_ptr: resulting BO
+ * @gpu_addr: GPU addr of the pinned BO
+ * @cpu_addr: optional CPU address mapping
+ *
+ * Allocates and pins a BO for kernel internal use.
+ *
+ * Returns 0 on success, negative error code otherwise.
+ */
+int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
+			    unsigned long size, int align,
+			    u32 domain, struct amdgpu_bo **bo_ptr,
+			    u64 *gpu_addr, void **cpu_addr)
+{
+	int r;
+
+	r = amdgpu_bo_create(adev, size, align, true, domain,
+			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+			     NULL, NULL, bo_ptr);
+	if (r) {
+		dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r);
+		return r;
+	}
+
+	r = amdgpu_bo_reserve(*bo_ptr, false);
+	if (r) {
+		dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
+		goto error_free;
+	}
+
+	r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);
+	if (r) {
+		dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
+		goto error_unreserve;
+	}
+
+	if (cpu_addr) {
+		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
+		if (r) {
+			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
+			goto error_unreserve;
+		}
+	}
+
+	amdgpu_bo_unreserve(*bo_ptr);
+
+	return 0;
+
+error_unreserve:
+	amdgpu_bo_unreserve(*bo_ptr);
+
+error_free:
+	amdgpu_bo_unref(bo_ptr);
+
+	return r;
+}
+
+/**
+ * amdgpu_bo_free_kernel - free BO for kernel use
+ *
+ * @bo: amdgpu BO to free
+ *
+ * unmaps and unpin a BO for kernel internal use.
+ */
+void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
+			   void **cpu_addr)
+{
+	if (*bo == NULL)
+		return;
+
+	if (likely(amdgpu_bo_reserve(*bo, false) == 0)) {
+		if (cpu_addr)
+			amdgpu_bo_kunmap(*bo);
+
+		amdgpu_bo_unpin(*bo);
+		amdgpu_bo_unreserve(*bo);
+	}
+	amdgpu_bo_unref(bo);
+
+	if (gpu_addr)
+		*gpu_addr = 0;
+
+	if (cpu_addr)
+		*cpu_addr = NULL;
+}
+
 int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 				unsigned long size, int byte_align,
 				bool kernel, u32 domain, u64 flags,
@@ -249,7 +352,7 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 		return r;
 	}
 	bo->adev = adev;
-	INIT_LIST_HEAD(&bo->list);
+	INIT_LIST_HEAD(&bo->shadow_list);
 	INIT_LIST_HEAD(&bo->va);
 	bo->prefered_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
 					 AMDGPU_GEM_DOMAIN_GTT |
@@ -277,11 +380,79 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 	if (unlikely(r != 0)) {
 		return r;
 	}
+
+	if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
+	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
+		struct fence *fence;
+
+		if (adev->mman.buffer_funcs_ring == NULL ||
+		   !adev->mman.buffer_funcs_ring->ready) {
+			r = -EBUSY;
+			goto fail_free;
+		}
+
+		r = amdgpu_bo_reserve(bo, false);
+		if (unlikely(r != 0))
+			goto fail_free;
+
+		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
+		r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+		if (unlikely(r != 0))
+			goto fail_unreserve;
+
+		amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);
+		amdgpu_bo_fence(bo, fence, false);
+		amdgpu_bo_unreserve(bo);
+		fence_put(bo->tbo.moving);
+		bo->tbo.moving = fence_get(fence);
+		fence_put(fence);
+	}
 	*bo_ptr = bo;
 
 	trace_amdgpu_bo_create(bo);
 
 	return 0;
+
+fail_unreserve:
+	amdgpu_bo_unreserve(bo);
+fail_free:
+	amdgpu_bo_unref(&bo);
+	return r;
+}
+
+static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
+				   unsigned long size, int byte_align,
+				   struct amdgpu_bo *bo)
+{
+	struct ttm_placement placement = {0};
+	struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
+	int r;
+
+	if (bo->shadow)
+		return 0;
+
+	bo->flags |= AMDGPU_GEM_CREATE_SHADOW;
+	memset(&placements, 0,
+	       (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
+
+	amdgpu_ttm_placement_init(adev, &placement,
+				  placements, AMDGPU_GEM_DOMAIN_GTT,
+				  AMDGPU_GEM_CREATE_CPU_GTT_USWC);
+
+	r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
+					AMDGPU_GEM_DOMAIN_GTT,
+					AMDGPU_GEM_CREATE_CPU_GTT_USWC,
+					NULL, &placement,
+					bo->tbo.resv,
+					&bo->shadow);
+	if (!r) {
+		bo->shadow->parent = amdgpu_bo_ref(bo);
+		mutex_lock(&adev->shadow_list_lock);
+		list_add_tail(&bo->shadow_list, &adev->shadow_list);
+		mutex_unlock(&adev->shadow_list_lock);
+	}
+
+	return r;
 }
 
 int amdgpu_bo_create(struct amdgpu_device *adev,
@@ -293,6 +464,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 {
 	struct ttm_placement placement = {0};
 	struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
+	int r;
 
 	memset(&placements, 0,
 	       (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
@@ -300,9 +472,83 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 	amdgpu_ttm_placement_init(adev, &placement,
 				  placements, domain, flags);
 
-	return amdgpu_bo_create_restricted(adev, size, byte_align, kernel,
-					   domain, flags, sg, &placement,
-					   resv, bo_ptr);
+	r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel,
+					domain, flags, sg, &placement,
+					resv, bo_ptr);
+	if (r)
+		return r;
+
+	if (amdgpu_need_backup(adev) && (flags & AMDGPU_GEM_CREATE_SHADOW)) {
+		r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
+		if (r)
+			amdgpu_bo_unref(bo_ptr);
+	}
+
+	return r;
+}
+
+int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
+			       struct amdgpu_ring *ring,
+			       struct amdgpu_bo *bo,
+			       struct reservation_object *resv,
+			       struct fence **fence,
+			       bool direct)
+
+{
+	struct amdgpu_bo *shadow = bo->shadow;
+	uint64_t bo_addr, shadow_addr;
+	int r;
+
+	if (!shadow)
+		return -EINVAL;
+
+	bo_addr = amdgpu_bo_gpu_offset(bo);
+	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
+
+	r = reservation_object_reserve_shared(bo->tbo.resv);
+	if (r)
+		goto err;
+
+	r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr,
+			       amdgpu_bo_size(bo), resv, fence,
+			       direct);
+	if (!r)
+		amdgpu_bo_fence(bo, *fence, true);
+
+err:
+	return r;
+}
+
+int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
+				  struct amdgpu_ring *ring,
+				  struct amdgpu_bo *bo,
+				  struct reservation_object *resv,
+				  struct fence **fence,
+				  bool direct)
+
+{
+	struct amdgpu_bo *shadow = bo->shadow;
+	uint64_t bo_addr, shadow_addr;
+	int r;
+
+	if (!shadow)
+		return -EINVAL;
+
+	bo_addr = amdgpu_bo_gpu_offset(bo);
+	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
+
+	r = reservation_object_reserve_shared(bo->tbo.resv);
+	if (r)
+		goto err;
+
+	r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr,
+			       amdgpu_bo_size(bo), resv, fence,
+			       direct);
+	if (!r)
+		amdgpu_bo_fence(bo, *fence, true);
+
+err:
+	return r;
 }
 
 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
@@ -380,16 +626,17 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 		return -EINVAL;
 
 	if (bo->pin_count) {
+		uint32_t mem_type = bo->tbo.mem.mem_type;
+
+		if (domain != amdgpu_mem_type_to_domain(mem_type))
+			return -EINVAL;
+
 		bo->pin_count++;
 		if (gpu_addr)
 			*gpu_addr = amdgpu_bo_gpu_offset(bo);
 
 		if (max_offset != 0) {
-			u64 domain_start;
-			if (domain == AMDGPU_GEM_DOMAIN_VRAM)
-				domain_start = bo->adev->mc.vram_start;
-			else
-				domain_start = bo->adev->mc.gtt_start;
+			u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;
 			WARN_ON_ONCE(max_offset <
 				     (amdgpu_bo_gpu_offset(bo) - domain_start));
 		}
@@ -401,7 +648,8 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 		/* force to pin into visible video ram */
 		if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
 		    !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
-		    (!max_offset || max_offset > bo->adev->mc.visible_vram_size)) {
+		    (!max_offset || max_offset >
+		     bo->adev->mc.visible_vram_size)) {
 			if (WARN_ON_ONCE(min_offset >
 					 bo->adev->mc.visible_vram_size))
 				return -EINVAL;
@@ -420,19 +668,28 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 	}
 
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
-	if (likely(r == 0)) {
-		bo->pin_count = 1;
-		if (gpu_addr != NULL)
-			*gpu_addr = amdgpu_bo_gpu_offset(bo);
-		if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
-			bo->adev->vram_pin_size += amdgpu_bo_size(bo);
-			if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
-				bo->adev->invisible_pin_size += amdgpu_bo_size(bo);
-		} else
-			bo->adev->gart_pin_size += amdgpu_bo_size(bo);
-	} else {
+	if (unlikely(r)) {
 		dev_err(bo->adev->dev, "%p pin failed\n", bo);
+		goto error;
+	}
+	r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
+	if (unlikely(r)) {
+		dev_err(bo->adev->dev, "%p bind failed\n", bo);
+		goto error;
 	}
+
+	bo->pin_count = 1;
+	if (gpu_addr != NULL)
+		*gpu_addr = amdgpu_bo_gpu_offset(bo);
+	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
+		bo->adev->vram_pin_size += amdgpu_bo_size(bo);
+		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
+			bo->adev->invisible_pin_size += amdgpu_bo_size(bo);
+	} else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
+		bo->adev->gart_pin_size += amdgpu_bo_size(bo);
+	}
+
+error:
 	return r;
 }
 
@@ -457,16 +714,20 @@ int amdgpu_bo_unpin(struct amdgpu_bo *bo)
 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
 	}
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
-	if (likely(r == 0)) {
-		if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
-			bo->adev->vram_pin_size -= amdgpu_bo_size(bo);
-			if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
-				bo->adev->invisible_pin_size -= amdgpu_bo_size(bo);
-		} else
-			bo->adev->gart_pin_size -= amdgpu_bo_size(bo);
-	} else {
+	if (unlikely(r)) {
 		dev_err(bo->adev->dev, "%p validate failed for unpin\n", bo);
+		goto error;
 	}
+
+	if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
+		bo->adev->vram_pin_size -= amdgpu_bo_size(bo);
+		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
+			bo->adev->invisible_pin_size -= amdgpu_bo_size(bo);
+	} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
+		bo->adev->gart_pin_size -= amdgpu_bo_size(bo);
+	}
+
+error:
 	return r;
 }
 
@@ -588,23 +849,23 @@ int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 			   struct ttm_mem_reg *new_mem)
 {
-	struct amdgpu_bo *rbo;
+	struct amdgpu_bo *abo;
 	struct ttm_mem_reg *old_mem = &bo->mem;
 
 	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
 		return;
 
-	rbo = container_of(bo, struct amdgpu_bo, tbo);
-	amdgpu_vm_bo_invalidate(rbo->adev, rbo);
+	abo = container_of(bo, struct amdgpu_bo, tbo);
+	amdgpu_vm_bo_invalidate(abo->adev, abo);
 
 	/* update statistics */
 	if (!new_mem)
 		return;
 
 	/* move_notify is called before move happens */
-	amdgpu_update_memory_usage(rbo->adev, &bo->mem, new_mem);
+	amdgpu_update_memory_usage(abo->adev, &bo->mem, new_mem);
 
-	trace_amdgpu_ttm_bo_move(rbo, new_mem->mem_type, old_mem->mem_type);
+	trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
 }
 
 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
@@ -637,7 +898,8 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 	for (i = 0; i < abo->placement.num_placement; i++) {
 		/* Force into visible VRAM */
 		if ((abo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
-		    (!abo->placements[i].lpfn || abo->placements[i].lpfn > lpfn))
+		    (!abo->placements[i].lpfn ||
+		     abo->placements[i].lpfn > lpfn))
 			abo->placements[i].lpfn = lpfn;
 	}
 	r = ttm_bo_validate(bo, &abo->placement, false, false);
@@ -674,3 +936,24 @@ void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence,
 	else
 		reservation_object_add_excl_fence(resv, fence);
 }
+
+/**
+ * amdgpu_bo_gpu_offset - return GPU offset of bo
+ * @bo:	amdgpu object for which we query the offset
+ *
+ * Returns current GPU offset of the object.
+ *
+ * Note: object should either be pinned or reserved when calling this
+ * function, it might be useful to add check for this for debugging.
+ */
+u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
+{
+	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
+	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
+		     !amdgpu_ttm_is_bound(bo->tbo.ttm));
+	WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
+		     !bo->pin_count);
+	WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
+
+	return bo->tbo.offset;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index bdb01d932548..8255034d73eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -31,6 +31,8 @@
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
 
+#define AMDGPU_BO_INVALID_OFFSET	LONG_MAX
+
 /**
  * amdgpu_mem_type_to_domain - return domain corresponding to mem_type
  * @mem_type:	ttm memory type
@@ -85,21 +87,6 @@ static inline void amdgpu_bo_unreserve(struct amdgpu_bo *bo)
 	ttm_bo_unreserve(&bo->tbo);
 }
 
-/**
- * amdgpu_bo_gpu_offset - return GPU offset of bo
- * @bo:	amdgpu object for which we query the offset
- *
- * Returns current GPU offset of the object.
- *
- * Note: object should either be pinned or reserved when calling this
- * function, it might be useful to add check for this for debugging.
- */
-static inline u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
-{
-	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
-	return bo->tbo.offset;
-}
-
 static inline unsigned long amdgpu_bo_size(struct amdgpu_bo *bo)
 {
 	return bo->tbo.num_pages << PAGE_SHIFT;
@@ -139,6 +126,12 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 				struct ttm_placement *placement,
 			        struct reservation_object *resv,
 				struct amdgpu_bo **bo_ptr);
+int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
+			    unsigned long size, int align,
+			    u32 domain, struct amdgpu_bo **bo_ptr,
+			    u64 *gpu_addr, void **cpu_addr);
+void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
+			   void **cpu_addr);
 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr);
 void amdgpu_bo_kunmap(struct amdgpu_bo *bo);
 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo);
@@ -165,6 +158,19 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence,
 		     bool shared);
+u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo);
+int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
+			       struct amdgpu_ring *ring,
+			       struct amdgpu_bo *bo,
+			       struct reservation_object *resv,
+			       struct fence **fence, bool direct);
+int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
+				  struct amdgpu_ring *ring,
+				  struct amdgpu_bo *bo,
+				  struct reservation_object *resv,
+				  struct fence **fence,
+				  bool direct);
+
 
 /*
  * sub allocation
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c
index d15314957732..8e67c1210d7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pll.c
@@ -25,6 +25,7 @@
 #include "amdgpu.h"
 #include "atom.h"
 #include "atombios_encoders.h"
+#include "amdgpu_pll.h"
 #include <asm/div64.h>
 #include <linux/gcd.h>
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index ff63b88b0ffa..accc908bdc88 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -305,7 +305,7 @@ static ssize_t amdgpu_get_pp_table(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 	char *table = NULL;
-	int size, i;
+	int size;
 
 	if (adev->pp_enabled)
 		size = amdgpu_dpm_get_pp_table(adev, &table);
@@ -315,10 +315,7 @@ static ssize_t amdgpu_get_pp_table(struct device *dev,
 	if (size >= PAGE_SIZE)
 		size = PAGE_SIZE - 1;
 
-	for (i = 0; i < size; i++) {
-		sprintf(buf + i, "%02x", table[i]);
-	}
-	sprintf(buf + i, "\n");
+	memcpy(buf, table, size);
 
 	return size;
 }
@@ -1106,54 +1103,46 @@ force:
 
 void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 {
-	if (adev->pp_enabled)
+	if (adev->pp_enabled || adev->pm.funcs->powergate_uvd) {
+		/* enable/disable UVD */
+		mutex_lock(&adev->pm.mutex);
 		amdgpu_dpm_powergate_uvd(adev, !enable);
-	else {
-		if (adev->pm.funcs->powergate_uvd) {
+		mutex_unlock(&adev->pm.mutex);
+	} else {
+		if (enable) {
 			mutex_lock(&adev->pm.mutex);
-			/* enable/disable UVD */
-			amdgpu_dpm_powergate_uvd(adev, !enable);
+			adev->pm.dpm.uvd_active = true;
+			adev->pm.dpm.state = POWER_STATE_TYPE_INTERNAL_UVD;
 			mutex_unlock(&adev->pm.mutex);
 		} else {
-			if (enable) {
-				mutex_lock(&adev->pm.mutex);
-				adev->pm.dpm.uvd_active = true;
-				adev->pm.dpm.state = POWER_STATE_TYPE_INTERNAL_UVD;
-				mutex_unlock(&adev->pm.mutex);
-			} else {
-				mutex_lock(&adev->pm.mutex);
-				adev->pm.dpm.uvd_active = false;
-				mutex_unlock(&adev->pm.mutex);
-			}
-			amdgpu_pm_compute_clocks(adev);
+			mutex_lock(&adev->pm.mutex);
+			adev->pm.dpm.uvd_active = false;
+			mutex_unlock(&adev->pm.mutex);
 		}
-
+		amdgpu_pm_compute_clocks(adev);
 	}
 }
 
 void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable)
 {
-	if (adev->pp_enabled)
+	if (adev->pp_enabled || adev->pm.funcs->powergate_vce) {
+		/* enable/disable VCE */
+		mutex_lock(&adev->pm.mutex);
 		amdgpu_dpm_powergate_vce(adev, !enable);
-	else {
-		if (adev->pm.funcs->powergate_vce) {
+		mutex_unlock(&adev->pm.mutex);
+	} else {
+		if (enable) {
 			mutex_lock(&adev->pm.mutex);
-			amdgpu_dpm_powergate_vce(adev, !enable);
+			adev->pm.dpm.vce_active = true;
+			/* XXX select vce level based on ring/task */
+			adev->pm.dpm.vce_level = AMDGPU_VCE_LEVEL_AC_ALL;
 			mutex_unlock(&adev->pm.mutex);
 		} else {
-			if (enable) {
-				mutex_lock(&adev->pm.mutex);
-				adev->pm.dpm.vce_active = true;
-				/* XXX select vce level based on ring/task */
-				adev->pm.dpm.vce_level = AMDGPU_VCE_LEVEL_AC_ALL;
-				mutex_unlock(&adev->pm.mutex);
-			} else {
-				mutex_lock(&adev->pm.mutex);
-				adev->pm.dpm.vce_active = false;
-				mutex_unlock(&adev->pm.mutex);
-			}
-			amdgpu_pm_compute_clocks(adev);
+			mutex_lock(&adev->pm.mutex);
+			adev->pm.dpm.vce_active = false;
+			mutex_unlock(&adev->pm.mutex);
 		}
+		amdgpu_pm_compute_clocks(adev);
 	}
 }
 
@@ -1333,6 +1322,64 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
  */
 #if defined(CONFIG_DEBUG_FS)
 
+static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *adev)
+{
+	int32_t value;
+
+	/* sanity check PP is enabled */
+	if (!(adev->powerplay.pp_funcs &&
+	      adev->powerplay.pp_funcs->read_sensor))
+	      return -EINVAL;
+
+	/* GPU Clocks */
+	seq_printf(m, "GFX Clocks and Power:\n");
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_MCLK, &value))
+		seq_printf(m, "\t%u MHz (MCLK)\n", value/100);
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_SCLK, &value))
+		seq_printf(m, "\t%u MHz (SCLK)\n", value/100);
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDGFX, &value))
+		seq_printf(m, "\t%u mV (VDDGFX)\n", value);
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDNB, &value))
+		seq_printf(m, "\t%u mV (VDDNB)\n", value);
+	seq_printf(m, "\n");
+
+	/* GPU Temp */
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_TEMP, &value))
+		seq_printf(m, "GPU Temperature: %u C\n", value/1000);
+
+	/* GPU Load */
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_LOAD, &value))
+		seq_printf(m, "GPU Load: %u %%\n", value);
+	seq_printf(m, "\n");
+
+	/* UVD clocks */
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_POWER, &value)) {
+		if (!value) {
+			seq_printf(m, "UVD: Disabled\n");
+		} else {
+			seq_printf(m, "UVD: Enabled\n");
+			if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_DCLK, &value))
+				seq_printf(m, "\t%u MHz (DCLK)\n", value/100);
+			if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_UVD_VCLK, &value))
+				seq_printf(m, "\t%u MHz (VCLK)\n", value/100);
+		}
+	}
+	seq_printf(m, "\n");
+
+	/* VCE clocks */
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCE_POWER, &value)) {
+		if (!value) {
+			seq_printf(m, "VCE: Disabled\n");
+		} else {
+			seq_printf(m, "VCE: Enabled\n");
+			if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VCE_ECCLK, &value))
+				seq_printf(m, "\t%u MHz (ECCLK)\n", value/100);
+		}
+	}
+
+	return 0;
+}
+
 static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -1348,11 +1395,11 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
 	     (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) {
 		seq_printf(m, "PX asic powered off\n");
 	} else if (adev->pp_enabled) {
-		amdgpu_dpm_debugfs_print_current_performance_level(adev, m);
+		return amdgpu_debugfs_pm_info_pp(m, adev);
 	} else {
 		mutex_lock(&adev->pm.mutex);
 		if (adev->pm.funcs->debugfs_print_current_performance_level)
-			amdgpu_dpm_debugfs_print_current_performance_level(adev, m);
+			adev->pm.funcs->debugfs_print_current_performance_level(adev, m);
 		else
 			seq_printf(m, "Debugfs support not implemented for this asic\n");
 		mutex_unlock(&adev->pm.mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
index c5738a22b690..7532ff822aa7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -30,6 +30,7 @@
 #include "amdgpu_pm.h"
 #include <drm/amdgpu_drm.h>
 #include "amdgpu_powerplay.h"
+#include "si_dpm.h"
 #include "cik_dpm.h"
 #include "vi_dpm.h"
 
@@ -41,7 +42,6 @@ static int amdgpu_powerplay_init(struct amdgpu_device *adev)
 	amd_pp = &(adev->powerplay);
 
 	if (adev->pp_enabled) {
-#ifdef CONFIG_DRM_AMD_POWERPLAY
 		struct amd_pp_init *pp_init;
 
 		pp_init = kzalloc(sizeof(struct amd_pp_init), GFP_KERNEL);
@@ -52,15 +52,21 @@ static int amdgpu_powerplay_init(struct amdgpu_device *adev)
 		pp_init->chip_family = adev->family;
 		pp_init->chip_id = adev->asic_type;
 		pp_init->device = amdgpu_cgs_create_device(adev);
-		pp_init->powercontainment_enabled = amdgpu_powercontainment;
-
 		ret = amd_powerplay_init(pp_init, amd_pp);
 		kfree(pp_init);
-#endif
 	} else {
 		amd_pp->pp_handle = (void *)adev;
 
 		switch (adev->asic_type) {
+#ifdef CONFIG_DRM_AMDGPU_SI
+		case CHIP_TAHITI:
+		case CHIP_PITCAIRN:
+		case CHIP_VERDE:
+		case CHIP_OLAND:
+		case CHIP_HAINAN:
+			amd_pp->ip_funcs = &si_dpm_ip_funcs;
+		break;
+#endif
 #ifdef CONFIG_DRM_AMDGPU_CIK
 		case CHIP_BONAIRE:
 		case CHIP_HAWAII:
@@ -72,15 +78,6 @@ static int amdgpu_powerplay_init(struct amdgpu_device *adev)
 			amd_pp->ip_funcs = &kv_dpm_ip_funcs;
 			break;
 #endif
-		case CHIP_TOPAZ:
-			amd_pp->ip_funcs = &iceland_dpm_ip_funcs;
-			break;
-		case CHIP_TONGA:
-			amd_pp->ip_funcs = &tonga_dpm_ip_funcs;
-			break;
-		case CHIP_FIJI:
-			amd_pp->ip_funcs = &fiji_dpm_ip_funcs;
-			break;
 		case CHIP_CARRIZO:
 		case CHIP_STONEY:
 			amd_pp->ip_funcs = &cz_dpm_ip_funcs;
@@ -98,19 +95,17 @@ static int amdgpu_pp_early_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int ret = 0;
 
-#ifdef CONFIG_DRM_AMD_POWERPLAY
 	switch (adev->asic_type) {
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
-		adev->pp_enabled = true;
-		break;
 	case CHIP_TONGA:
 	case CHIP_FIJI:
-		adev->pp_enabled = (amdgpu_powerplay == 0) ? false : true;
+	case CHIP_TOPAZ:
+		adev->pp_enabled = true;
 		break;
 	case CHIP_CARRIZO:
 	case CHIP_STONEY:
-		adev->pp_enabled = (amdgpu_powerplay > 0) ? true : false;
+		adev->pp_enabled = (amdgpu_powerplay == 0) ? false : true;
 		break;
 	/* These chips don't have powerplay implemenations */
 	case CHIP_BONAIRE:
@@ -118,14 +113,10 @@ static int amdgpu_pp_early_init(void *handle)
 	case CHIP_KABINI:
 	case CHIP_MULLINS:
 	case CHIP_KAVERI:
-	case CHIP_TOPAZ:
 	default:
 		adev->pp_enabled = false;
 		break;
 	}
-#else
-	adev->pp_enabled = false;
-#endif
 
 	ret = amdgpu_powerplay_init(adev);
 	if (ret)
@@ -147,12 +138,11 @@ static int amdgpu_pp_late_init(void *handle)
 		ret = adev->powerplay.ip_funcs->late_init(
 					adev->powerplay.pp_handle);
 
-#ifdef CONFIG_DRM_AMD_POWERPLAY
 	if (adev->pp_enabled && adev->pm.dpm_enabled) {
 		amdgpu_pm_sysfs_init(adev);
 		amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_COMPLETE_INIT, NULL, NULL);
 	}
-#endif
+
 	return ret;
 }
 
@@ -165,10 +155,8 @@ static int amdgpu_pp_sw_init(void *handle)
 		ret = adev->powerplay.ip_funcs->sw_init(
 					adev->powerplay.pp_handle);
 
-#ifdef CONFIG_DRM_AMD_POWERPLAY
 	if (adev->pp_enabled)
 		adev->pm.dpm_enabled = true;
-#endif
 
 	return ret;
 }
@@ -219,7 +207,6 @@ static int amdgpu_pp_hw_fini(void *handle)
 
 static void amdgpu_pp_late_fini(void *handle)
 {
-#ifdef CONFIG_DRM_AMD_POWERPLAY
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	if (adev->pp_enabled) {
@@ -230,7 +217,6 @@ static void amdgpu_pp_late_fini(void *handle)
 	if (adev->powerplay.ip_funcs->late_fini)
 		adev->powerplay.ip_funcs->late_fini(
 			  adev->powerplay.pp_handle);
-#endif
 }
 
 static int amdgpu_pp_suspend(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 85aeb0a804bb..e1fa8731d1e2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -222,33 +222,16 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 
 	/* Allocate ring buffer */
 	if (ring->ring_obj == NULL) {
-		r = amdgpu_bo_create(adev, ring->ring_size, PAGE_SIZE, true,
-				     AMDGPU_GEM_DOMAIN_GTT, 0,
-				     NULL, NULL, &ring->ring_obj);
+		r = amdgpu_bo_create_kernel(adev, ring->ring_size, PAGE_SIZE,
+					    AMDGPU_GEM_DOMAIN_GTT,
+					    &ring->ring_obj,
+					    &ring->gpu_addr,
+					    (void **)&ring->ring);
 		if (r) {
 			dev_err(adev->dev, "(%d) ring create failed\n", r);
 			return r;
 		}
-		r = amdgpu_bo_reserve(ring->ring_obj, false);
-		if (unlikely(r != 0))
-			return r;
-		r = amdgpu_bo_pin(ring->ring_obj, AMDGPU_GEM_DOMAIN_GTT,
-					&ring->gpu_addr);
-		if (r) {
-			amdgpu_bo_unreserve(ring->ring_obj);
-			dev_err(adev->dev, "(%d) ring pin failed\n", r);
-			return r;
-		}
-		r = amdgpu_bo_kmap(ring->ring_obj,
-				       (void **)&ring->ring);
-
 		memset((void *)ring->ring, 0, ring->ring_size);
-
-		amdgpu_bo_unreserve(ring->ring_obj);
-		if (r) {
-			dev_err(adev->dev, "(%d) ring map failed\n", r);
-			return r;
-		}
 	}
 	ring->ptr_mask = (ring->ring_size / 4) - 1;
 	ring->max_dw = max_dw;
@@ -269,29 +252,20 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
  */
 void amdgpu_ring_fini(struct amdgpu_ring *ring)
 {
-	int r;
-	struct amdgpu_bo *ring_obj;
-
-	ring_obj = ring->ring_obj;
 	ring->ready = false;
-	ring->ring = NULL;
-	ring->ring_obj = NULL;
 
 	amdgpu_wb_free(ring->adev, ring->cond_exe_offs);
 	amdgpu_wb_free(ring->adev, ring->fence_offs);
 	amdgpu_wb_free(ring->adev, ring->rptr_offs);
 	amdgpu_wb_free(ring->adev, ring->wptr_offs);
 
-	if (ring_obj) {
-		r = amdgpu_bo_reserve(ring_obj, false);
-		if (likely(r == 0)) {
-			amdgpu_bo_kunmap(ring_obj);
-			amdgpu_bo_unpin(ring_obj);
-			amdgpu_bo_unreserve(ring_obj);
-		}
-		amdgpu_bo_unref(&ring_obj);
-	}
+	amdgpu_bo_free_kernel(&ring->ring_obj,
+			      &ring->gpu_addr,
+			      (void **)&ring->ring);
+
 	amdgpu_debugfs_ring_fini(ring);
+
+	ring->adev->rings[ring->idx] = NULL;
 }
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
index 05a53f4fc334..b827c75e95de 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
@@ -111,7 +111,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 		amdgpu_bo_kunmap(gtt_obj[i]);
 
 		r = amdgpu_copy_buffer(ring, gtt_addr, vram_addr,
-				       size, NULL, &fence);
+				       size, NULL, &fence, false);
 
 		if (r) {
 			DRM_ERROR("Failed GTT->VRAM copy %d\n", i);
@@ -156,7 +156,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 		amdgpu_bo_kunmap(vram_obj);
 
 		r = amdgpu_copy_buffer(ring, vram_addr, gtt_addr,
-				       size, NULL, &fence);
+				       size, NULL, &fence, false);
 
 		if (r) {
 			DRM_ERROR("Failed VRAM->GTT copy %d\n", i);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 0d8d65eb46cd..067e5e683bb3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -247,7 +247,7 @@ DEFINE_EVENT(amdgpu_vm_mapping, amdgpu_vm_bo_mapping,
 	    TP_ARGS(mapping)
 );
 
-TRACE_EVENT(amdgpu_vm_set_page,
+TRACE_EVENT(amdgpu_vm_set_ptes,
 	    TP_PROTO(uint64_t pe, uint64_t addr, unsigned count,
 		     uint32_t incr, uint32_t flags),
 	    TP_ARGS(pe, addr, count, incr, flags),
@@ -271,6 +271,24 @@ TRACE_EVENT(amdgpu_vm_set_page,
 		      __entry->flags, __entry->count)
 );
 
+TRACE_EVENT(amdgpu_vm_copy_ptes,
+	    TP_PROTO(uint64_t pe, uint64_t src, unsigned count),
+	    TP_ARGS(pe, src, count),
+	    TP_STRUCT__entry(
+			     __field(u64, pe)
+			     __field(u64, src)
+			     __field(u32, count)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->pe = pe;
+			   __entry->src = src;
+			   __entry->count = count;
+			   ),
+	    TP_printk("pe=%010Lx, src=%010Lx, count=%u",
+		      __entry->pe, __entry->src, __entry->count)
+);
+
 TRACE_EVENT(amdgpu_vm_flush,
 	    TP_PROTO(uint64_t pd_addr, unsigned ring, unsigned id),
 	    TP_ARGS(pd_addr, ring, id),
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index b7742e62972a..887483b8b818 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -34,6 +34,7 @@
 #include <ttm/ttm_placement.h>
 #include <ttm/ttm_module.h>
 #include <ttm/ttm_page_alloc.h>
+#include <ttm/ttm_memory.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include <linux/seq_file.h>
@@ -74,7 +75,7 @@ static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref)
 	ttm_mem_global_release(ref->object);
 }
 
-static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
+int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 {
 	struct drm_global_reference *global_ref;
 	struct amdgpu_ring *ring;
@@ -88,10 +89,10 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 	global_ref->init = &amdgpu_ttm_mem_global_init;
 	global_ref->release = &amdgpu_ttm_mem_global_release;
 	r = drm_global_item_ref(global_ref);
-	if (r != 0) {
+	if (r) {
 		DRM_ERROR("Failed setting up TTM memory accounting "
 			  "subsystem.\n");
-		return r;
+		goto error_mem;
 	}
 
 	adev->mman.bo_global_ref.mem_glob =
@@ -102,26 +103,30 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 	global_ref->init = &ttm_bo_global_init;
 	global_ref->release = &ttm_bo_global_release;
 	r = drm_global_item_ref(global_ref);
-	if (r != 0) {
+	if (r) {
 		DRM_ERROR("Failed setting up TTM BO subsystem.\n");
-		drm_global_item_unref(&adev->mman.mem_global_ref);
-		return r;
+		goto error_bo;
 	}
 
 	ring = adev->mman.buffer_funcs_ring;
 	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
 	r = amd_sched_entity_init(&ring->sched, &adev->mman.entity,
 				  rq, amdgpu_sched_jobs);
-	if (r != 0) {
+	if (r) {
 		DRM_ERROR("Failed setting up TTM BO move run queue.\n");
-		drm_global_item_unref(&adev->mman.mem_global_ref);
-		drm_global_item_unref(&adev->mman.bo_global_ref.ref);
-		return r;
+		goto error_entity;
 	}
 
 	adev->mman.mem_global_referenced = true;
 
 	return 0;
+
+error_entity:
+	drm_global_item_unref(&adev->mman.bo_global_ref.ref);
+error_bo:
+	drm_global_item_unref(&adev->mman.mem_global_ref);
+error_mem:
+	return r;
 }
 
 static void amdgpu_ttm_global_fini(struct amdgpu_device *adev)
@@ -155,7 +160,7 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 		man->default_caching = TTM_PL_FLAG_CACHED;
 		break;
 	case TTM_PL_TT:
-		man->func = &ttm_bo_manager_func;
+		man->func = &amdgpu_gtt_mgr_func;
 		man->gpu_offset = adev->mc.gtt_start;
 		man->available_caching = TTM_PL_MASK_CACHING;
 		man->default_caching = TTM_PL_FLAG_CACHED;
@@ -190,12 +195,13 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 				struct ttm_placement *placement)
 {
-	struct amdgpu_bo *rbo;
+	struct amdgpu_bo *abo;
 	static struct ttm_place placements = {
 		.fpfn = 0,
 		.lpfn = 0,
 		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
 	};
+	unsigned i;
 
 	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) {
 		placement->placement = &placements;
@@ -204,28 +210,44 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 		placement->num_busy_placement = 1;
 		return;
 	}
-	rbo = container_of(bo, struct amdgpu_bo, tbo);
+	abo = container_of(bo, struct amdgpu_bo, tbo);
 	switch (bo->mem.mem_type) {
 	case TTM_PL_VRAM:
-		if (rbo->adev->mman.buffer_funcs_ring->ready == false)
-			amdgpu_ttm_placement_from_domain(rbo, AMDGPU_GEM_DOMAIN_CPU);
-		else
-			amdgpu_ttm_placement_from_domain(rbo, AMDGPU_GEM_DOMAIN_GTT);
+		if (abo->adev->mman.buffer_funcs_ring->ready == false) {
+			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
+		} else {
+			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
+			for (i = 0; i < abo->placement.num_placement; ++i) {
+				if (!(abo->placements[i].flags &
+				      TTM_PL_FLAG_TT))
+					continue;
+
+				if (abo->placements[i].lpfn)
+					continue;
+
+				/* set an upper limit to force directly
+				 * allocating address space for the BO.
+				 */
+				abo->placements[i].lpfn =
+					abo->adev->mc.gtt_size >> PAGE_SHIFT;
+			}
+		}
 		break;
 	case TTM_PL_TT:
 	default:
-		amdgpu_ttm_placement_from_domain(rbo, AMDGPU_GEM_DOMAIN_CPU);
+		amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 	}
-	*placement = rbo->placement;
+	*placement = abo->placement;
 }
 
 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 {
-	struct amdgpu_bo *rbo = container_of(bo, struct amdgpu_bo, tbo);
+	struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
 
 	if (amdgpu_ttm_tt_get_usermm(bo->ttm))
 		return -EPERM;
-	return drm_vma_node_verify_access(&rbo->gem_base.vma_node, filp);
+	return drm_vma_node_verify_access(&abo->gem_base.vma_node,
+					  filp->private_data);
 }
 
 static void amdgpu_move_null(struct ttm_buffer_object *bo,
@@ -251,26 +273,30 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 
 	adev = amdgpu_get_adev(bo->bdev);
 	ring = adev->mman.buffer_funcs_ring;
-	old_start = old_mem->start << PAGE_SHIFT;
-	new_start = new_mem->start << PAGE_SHIFT;
 
 	switch (old_mem->mem_type) {
-	case TTM_PL_VRAM:
-		old_start += adev->mc.vram_start;
-		break;
 	case TTM_PL_TT:
-		old_start += adev->mc.gtt_start;
+		r = amdgpu_ttm_bind(bo, old_mem);
+		if (r)
+			return r;
+
+	case TTM_PL_VRAM:
+		old_start = (u64)old_mem->start << PAGE_SHIFT;
+		old_start += bo->bdev->man[old_mem->mem_type].gpu_offset;
 		break;
 	default:
 		DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
 		return -EINVAL;
 	}
 	switch (new_mem->mem_type) {
-	case TTM_PL_VRAM:
-		new_start += adev->mc.vram_start;
-		break;
 	case TTM_PL_TT:
-		new_start += adev->mc.gtt_start;
+		r = amdgpu_ttm_bind(bo, new_mem);
+		if (r)
+			return r;
+
+	case TTM_PL_VRAM:
+		new_start = (u64)new_mem->start << PAGE_SHIFT;
+		new_start += bo->bdev->man[new_mem->mem_type].gpu_offset;
 		break;
 	default:
 		DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
@@ -285,7 +311,7 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 
 	r = amdgpu_copy_buffer(ring, old_start, new_start,
 			       new_mem->num_pages * PAGE_SIZE, /* bytes */
-			       bo->resv, &fence);
+			       bo->resv, &fence, false);
 	if (r)
 		return r;
 
@@ -314,7 +340,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = 0;
+	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);
@@ -335,7 +361,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
-	r = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem);
+	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem);
 out_cleanup:
 	ttm_bo_mem_put(bo, &tmp_mem);
 	return r;
@@ -361,14 +387,14 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = 0;
+	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);
 	if (unlikely(r)) {
 		return r;
 	}
-	r = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);
+	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
@@ -435,8 +461,7 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo,
 
 	if (r) {
 memcpy:
-		r = ttm_bo_move_memcpy(bo, evict, interruptible,
-				       no_wait_gpu, new_mem);
+		r = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, new_mem);
 		if (r) {
 			return r;
 		}
@@ -524,6 +549,7 @@ struct amdgpu_ttm_tt {
 	spinlock_t              guptasklock;
 	struct list_head        guptasks;
 	atomic_t		mmu_invalidations;
+	struct list_head        list;
 };
 
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
@@ -641,7 +667,6 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
 				   struct ttm_mem_reg *bo_mem)
 {
 	struct amdgpu_ttm_tt *gtt = (void*)ttm;
-	uint32_t flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);
 	int r;
 
 	if (gtt->userptr) {
@@ -651,7 +676,6 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
 			return r;
 		}
 	}
-	gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT);
 	if (!ttm->num_pages) {
 		WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
 		     ttm->num_pages, bo_mem, ttm);
@@ -662,14 +686,71 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
 	    bo_mem->mem_type == AMDGPU_PL_OA)
 		return -EINVAL;
 
+	return 0;
+}
+
+bool amdgpu_ttm_is_bound(struct ttm_tt *ttm)
+{
+	struct amdgpu_ttm_tt *gtt = (void *)ttm;
+
+	return gtt && !list_empty(&gtt->list);
+}
+
+int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
+{
+	struct ttm_tt *ttm = bo->ttm;
+	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
+	uint32_t flags;
+	int r;
+
+	if (!ttm || amdgpu_ttm_is_bound(ttm))
+		return 0;
+
+	r = amdgpu_gtt_mgr_alloc(&bo->bdev->man[TTM_PL_TT], bo,
+				 NULL, bo_mem);
+	if (r) {
+		DRM_ERROR("Failed to allocate GTT address space (%d)\n", r);
+		return r;
+	}
+
+	flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);
+	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
 	r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,
 		ttm->pages, gtt->ttm.dma_address, flags);
 
 	if (r) {
-		DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
-			  ttm->num_pages, (unsigned)gtt->offset);
+		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
+			  ttm->num_pages, gtt->offset);
 		return r;
 	}
+	spin_lock(&gtt->adev->gtt_list_lock);
+	list_add_tail(&gtt->list, &gtt->adev->gtt_list);
+	spin_unlock(&gtt->adev->gtt_list_lock);
+	return 0;
+}
+
+int amdgpu_ttm_recover_gart(struct amdgpu_device *adev)
+{
+	struct amdgpu_ttm_tt *gtt, *tmp;
+	struct ttm_mem_reg bo_mem;
+	uint32_t flags;
+	int r;
+
+	bo_mem.mem_type = TTM_PL_TT;
+	spin_lock(&adev->gtt_list_lock);
+	list_for_each_entry_safe(gtt, tmp, &adev->gtt_list, list) {
+		flags = amdgpu_ttm_tt_pte_flags(gtt->adev, &gtt->ttm.ttm, &bo_mem);
+		r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages,
+				     gtt->ttm.ttm.pages, gtt->ttm.dma_address,
+				     flags);
+		if (r) {
+			spin_unlock(&adev->gtt_list_lock);
+			DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
+				  gtt->ttm.ttm.num_pages, gtt->offset);
+			return r;
+		}
+	}
+	spin_unlock(&adev->gtt_list_lock);
 	return 0;
 }
 
@@ -677,12 +758,19 @@ static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
 {
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 
+	if (gtt->userptr)
+		amdgpu_ttm_tt_unpin_userptr(ttm);
+
+	if (!amdgpu_ttm_is_bound(ttm))
+		return 0;
+
 	/* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
 	if (gtt->adev->gart.ready)
 		amdgpu_gart_unbind(gtt->adev, gtt->offset, ttm->num_pages);
 
-	if (gtt->userptr)
-		amdgpu_ttm_tt_unpin_userptr(ttm);
+	spin_lock(&gtt->adev->gtt_list_lock);
+	list_del_init(&gtt->list);
+	spin_unlock(&gtt->adev->gtt_list_lock);
 
 	return 0;
 }
@@ -720,6 +808,7 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_bo_device *bdev,
 		kfree(gtt);
 		return NULL;
 	}
+	INIT_LIST_HEAD(&gtt->list);
 	return &gtt->ttm.ttm;
 }
 
@@ -950,6 +1039,8 @@ static struct list_head *amdgpu_ttm_lru_tail(struct ttm_buffer_object *tbo)
 	struct list_head *res = lru->lru[tbo->mem.mem_type];
 
 	lru->lru[tbo->mem.mem_type] = &tbo->lru;
+	while ((++lru)->lru[tbo->mem.mem_type] == res)
+		lru->lru[tbo->mem.mem_type] = &tbo->lru;
 
 	return res;
 }
@@ -960,6 +1051,8 @@ static struct list_head *amdgpu_ttm_swap_lru_tail(struct ttm_buffer_object *tbo)
 	struct list_head *res = lru->swap_lru;
 
 	lru->swap_lru = &tbo->swap;
+	while ((++lru)->swap_lru == res)
+		lru->swap_lru = &tbo->swap;
 
 	return res;
 }
@@ -987,10 +1080,6 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	unsigned i, j;
 	int r;
 
-	r = amdgpu_ttm_global_init(adev);
-	if (r) {
-		return r;
-	}
 	/* No others user of address space so set it to 0 */
 	r = ttm_bo_device_init(&adev->mman.bdev,
 			       adev->mman.bo_global_ref.ref.object,
@@ -1011,6 +1100,10 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 		lru->swap_lru = &adev->mman.bdev.glob->swap_lru;
 	}
 
+	for (j = 0; j < TTM_NUM_MEM_TYPES; ++j)
+		adev->mman.guard.lru[j] = NULL;
+	adev->mman.guard.swap_lru = NULL;
+
 	adev->mman.initialized = true;
 	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
 				adev->mc.real_vram_size >> PAGE_SHIFT);
@@ -1151,7 +1244,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
 		       uint64_t dst_offset,
 		       uint32_t byte_count,
 		       struct reservation_object *resv,
-		       struct fence **fence)
+		       struct fence **fence, bool direct_submit)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_job *job;
@@ -1195,8 +1288,79 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
 
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 	WARN_ON(job->ibs[0].length_dw > num_dw);
+	if (direct_submit) {
+		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs,
+				       NULL, NULL, fence);
+		job->fence = fence_get(*fence);
+		if (r)
+			DRM_ERROR("Error scheduling IBs (%d)\n", r);
+		amdgpu_job_free(job);
+	} else {
+		r = amdgpu_job_submit(job, ring, &adev->mman.entity,
+				      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
+		if (r)
+			goto error_free;
+	}
+
+	return r;
+
+error_free:
+	amdgpu_job_free(job);
+	return r;
+}
+
+int amdgpu_fill_buffer(struct amdgpu_bo *bo,
+		uint32_t src_data,
+		struct reservation_object *resv,
+		struct fence **fence)
+{
+	struct amdgpu_device *adev = bo->adev;
+	struct amdgpu_job *job;
+	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+
+	uint32_t max_bytes, byte_count;
+	uint64_t dst_offset;
+	unsigned int num_loops, num_dw;
+	unsigned int i;
+	int r;
+
+	byte_count = bo->tbo.num_pages << PAGE_SHIFT;
+	max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
+	num_loops = DIV_ROUND_UP(byte_count, max_bytes);
+	num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
+
+	/* for IB padding */
+	while (num_dw & 0x7)
+		num_dw++;
+
+	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
+	if (r)
+		return r;
+
+	if (resv) {
+		r = amdgpu_sync_resv(adev, &job->sync, resv,
+				AMDGPU_FENCE_OWNER_UNDEFINED);
+		if (r) {
+			DRM_ERROR("sync failed (%d).\n", r);
+			goto error_free;
+		}
+	}
+
+	dst_offset = bo->tbo.mem.start << PAGE_SHIFT;
+	for (i = 0; i < num_loops; i++) {
+		uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
+
+		amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
+				dst_offset, cur_size_in_bytes);
+
+		dst_offset += cur_size_in_bytes;
+		byte_count -= cur_size_in_bytes;
+	}
+
+	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+	WARN_ON(job->ibs[0].length_dw > num_dw);
 	r = amdgpu_job_submit(job, ring, &adev->mman.entity,
-			      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
+			AMDGPU_FENCE_OWNER_UNDEFINED, fence);
 	if (r)
 		goto error_free;
 
@@ -1387,3 +1551,8 @@ static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
 
 #endif
 }
+
+u64 amdgpu_ttm_get_gtt_mem_size(struct amdgpu_device *adev)
+{
+	return ttm_get_kernel_zone_memory_size(adev->mman.mem_global_ref.object);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
new file mode 100644
index 000000000000..9812c805326c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __AMDGPU_TTM_H__
+#define __AMDGPU_TTM_H__
+
+#include "gpu_scheduler.h"
+
+#define AMDGPU_PL_GDS		(TTM_PL_PRIV + 0)
+#define AMDGPU_PL_GWS		(TTM_PL_PRIV + 1)
+#define AMDGPU_PL_OA		(TTM_PL_PRIV + 2)
+
+#define AMDGPU_PL_FLAG_GDS		(TTM_PL_FLAG_PRIV << 0)
+#define AMDGPU_PL_FLAG_GWS		(TTM_PL_FLAG_PRIV << 1)
+#define AMDGPU_PL_FLAG_OA		(TTM_PL_FLAG_PRIV << 2)
+
+#define AMDGPU_TTM_LRU_SIZE	20
+
+struct amdgpu_mman_lru {
+	struct list_head		*lru[TTM_NUM_MEM_TYPES];
+	struct list_head		*swap_lru;
+};
+
+struct amdgpu_mman {
+	struct ttm_bo_global_ref        bo_global_ref;
+	struct drm_global_reference	mem_global_ref;
+	struct ttm_bo_device		bdev;
+	bool				mem_global_referenced;
+	bool				initialized;
+
+#if defined(CONFIG_DEBUG_FS)
+	struct dentry			*vram;
+	struct dentry			*gtt;
+#endif
+
+	/* buffer handling */
+	const struct amdgpu_buffer_funcs	*buffer_funcs;
+	struct amdgpu_ring			*buffer_funcs_ring;
+	/* Scheduler entity for buffer moves */
+	struct amd_sched_entity			entity;
+
+	/* custom LRU management */
+	struct amdgpu_mman_lru			log2_size[AMDGPU_TTM_LRU_SIZE];
+	/* guard for log2_size array, don't add anything in between */
+	struct amdgpu_mman_lru			guard;
+};
+
+extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func;
+
+int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
+			 struct ttm_buffer_object *tbo,
+			 const struct ttm_place *place,
+			 struct ttm_mem_reg *mem);
+
+int amdgpu_copy_buffer(struct amdgpu_ring *ring,
+		       uint64_t src_offset,
+		       uint64_t dst_offset,
+		       uint32_t byte_count,
+		       struct reservation_object *resv,
+		       struct fence **fence, bool direct_submit);
+int amdgpu_fill_buffer(struct amdgpu_bo *bo,
+			uint32_t src_data,
+			struct reservation_object *resv,
+			struct fence **fence);
+
+int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
+bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
+int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 5cc95f1a7dab..cb3d252f3c78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -247,40 +247,32 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 	const struct common_firmware_header *header = NULL;
 
 	err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
-			AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL, bo);
+			       AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL, bo);
 	if (err) {
 		dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err);
-		err = -ENOMEM;
 		goto failed;
 	}
 
 	err = amdgpu_bo_reserve(*bo, false);
 	if (err) {
-		amdgpu_bo_unref(bo);
 		dev_err(adev->dev, "(%d) Firmware buffer reserve failed\n", err);
-		goto failed;
+		goto failed_reserve;
 	}
 
 	err = amdgpu_bo_pin(*bo, AMDGPU_GEM_DOMAIN_GTT, &fw_mc_addr);
 	if (err) {
-		amdgpu_bo_unreserve(*bo);
-		amdgpu_bo_unref(bo);
 		dev_err(adev->dev, "(%d) Firmware buffer pin failed\n", err);
-		goto failed;
+		goto failed_pin;
 	}
 
 	err = amdgpu_bo_kmap(*bo, &fw_buf_ptr);
 	if (err) {
 		dev_err(adev->dev, "(%d) Firmware buffer kmap failed\n", err);
-		amdgpu_bo_unpin(*bo);
-		amdgpu_bo_unreserve(*bo);
-		amdgpu_bo_unref(bo);
-		goto failed;
+		goto failed_kmap;
 	}
 
 	amdgpu_bo_unreserve(*bo);
 
-	fw_offset = 0;
 	for (i = 0; i < AMDGPU_UCODE_ID_MAXIMUM; i++) {
 		ucode = &adev->firmware.ucode[i];
 		if (ucode->fw) {
@@ -290,10 +282,16 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 			fw_offset += ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 		}
 	}
+	return 0;
 
+failed_kmap:
+	amdgpu_bo_unpin(*bo);
+failed_pin:
+	amdgpu_bo_unreserve(*bo);
+failed_reserve:
+	amdgpu_bo_unref(bo);
 failed:
-	if (err)
-		adev->firmware.smu_load = false;
+	adev->firmware.smu_load = false;
 
 	return err;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index b11f4e8868d7..e3281cacc586 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -201,39 +201,14 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 	bo_size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8)
 		  +  AMDGPU_UVD_STACK_SIZE + AMDGPU_UVD_HEAP_SIZE
 		  +  AMDGPU_UVD_SESSION_SIZE * adev->uvd.max_handles;
-	r = amdgpu_bo_create(adev, bo_size, PAGE_SIZE, true,
-			     AMDGPU_GEM_DOMAIN_VRAM,
-			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-			     NULL, NULL, &adev->uvd.vcpu_bo);
+	r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE,
+				    AMDGPU_GEM_DOMAIN_VRAM, &adev->uvd.vcpu_bo,
+				    &adev->uvd.gpu_addr, &adev->uvd.cpu_addr);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r);
 		return r;
 	}
 
-	r = amdgpu_bo_reserve(adev->uvd.vcpu_bo, false);
-	if (r) {
-		amdgpu_bo_unref(&adev->uvd.vcpu_bo);
-		dev_err(adev->dev, "(%d) failed to reserve UVD bo\n", r);
-		return r;
-	}
-
-	r = amdgpu_bo_pin(adev->uvd.vcpu_bo, AMDGPU_GEM_DOMAIN_VRAM,
-			  &adev->uvd.gpu_addr);
-	if (r) {
-		amdgpu_bo_unreserve(adev->uvd.vcpu_bo);
-		amdgpu_bo_unref(&adev->uvd.vcpu_bo);
-		dev_err(adev->dev, "(%d) UVD bo pin failed\n", r);
-		return r;
-	}
-
-	r = amdgpu_bo_kmap(adev->uvd.vcpu_bo, &adev->uvd.cpu_addr);
-	if (r) {
-		dev_err(adev->dev, "(%d) UVD map failed\n", r);
-		return r;
-	}
-
-	amdgpu_bo_unreserve(adev->uvd.vcpu_bo);
-
 	ring = &adev->uvd.ring;
 	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
 	r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity,
@@ -274,22 +249,13 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 
 int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
 {
-	int r;
-
 	kfree(adev->uvd.saved_bo);
 
 	amd_sched_entity_fini(&adev->uvd.ring.sched, &adev->uvd.entity);
 
-	if (adev->uvd.vcpu_bo) {
-		r = amdgpu_bo_reserve(adev->uvd.vcpu_bo, false);
-		if (!r) {
-			amdgpu_bo_kunmap(adev->uvd.vcpu_bo);
-			amdgpu_bo_unpin(adev->uvd.vcpu_bo);
-			amdgpu_bo_unreserve(adev->uvd.vcpu_bo);
-		}
-
-		amdgpu_bo_unref(&adev->uvd.vcpu_bo);
-	}
+	amdgpu_bo_free_kernel(&adev->uvd.vcpu_bo,
+			      &adev->uvd.gpu_addr,
+			      (void **)&adev->uvd.cpu_addr);
 
 	amdgpu_ring_fini(&adev->uvd.ring);
 
@@ -323,7 +289,7 @@ int amdgpu_uvd_suspend(struct amdgpu_device *adev)
 	if (!adev->uvd.saved_bo)
 		return -ENOMEM;
 
-	memcpy(adev->uvd.saved_bo, ptr, size);
+	memcpy_fromio(adev->uvd.saved_bo, ptr, size);
 
 	return 0;
 }
@@ -340,7 +306,7 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 	ptr = adev->uvd.cpu_addr;
 
 	if (adev->uvd.saved_bo != NULL) {
-		memcpy(ptr, adev->uvd.saved_bo, size);
+		memcpy_toio(ptr, adev->uvd.saved_bo, size);
 		kfree(adev->uvd.saved_bo);
 		adev->uvd.saved_bo = NULL;
 	} else {
@@ -349,11 +315,11 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 
 		hdr = (const struct common_firmware_header *)adev->uvd.fw->data;
 		offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
-		memcpy(adev->uvd.cpu_addr, (adev->uvd.fw->data) + offset,
-			(adev->uvd.fw->size) - offset);
+		memcpy_toio(adev->uvd.cpu_addr, adev->uvd.fw->data + offset,
+			    le32_to_cpu(hdr->ucode_size_bytes));
 		size -= le32_to_cpu(hdr->ucode_size_bytes);
 		ptr += le32_to_cpu(hdr->ucode_size_bytes);
-		memset(ptr, 0, size);
+		memset_io(ptr, 0, size);
 	}
 
 	return 0;
@@ -385,12 +351,12 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
 	}
 }
 
-static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *rbo)
+static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo)
 {
 	int i;
-	for (i = 0; i < rbo->placement.num_placement; ++i) {
-		rbo->placements[i].fpfn = 0 >> PAGE_SHIFT;
-		rbo->placements[i].lpfn = (256 * 1024 * 1024) >> PAGE_SHIFT;
+	for (i = 0; i < abo->placement.num_placement; ++i) {
+		abo->placements[i].fpfn = 0 >> PAGE_SHIFT;
+		abo->placements[i].lpfn = (256 * 1024 * 1024) >> PAGE_SHIFT;
 	}
 }
 
@@ -843,6 +809,7 @@ static int amdgpu_uvd_cs_reg(struct amdgpu_uvd_cs_ctx *ctx,
 				return r;
 			break;
 		case mmUVD_ENGINE_CNTL:
+		case mmUVD_NO_OP:
 			break;
 		default:
 			DRM_ERROR("Invalid reg 0x%X!\n", reg);
@@ -915,6 +882,10 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
 		return -EINVAL;
 	}
 
+	r = amdgpu_cs_sysvm_access_required(parser);
+	if (r)
+		return r;
+
 	ctx.parser = parser;
 	ctx.buf_sizes = buf_sizes;
 	ctx.ib_idx = ib_idx;
@@ -981,8 +952,10 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 	ib->ptr[3] = addr >> 32;
 	ib->ptr[4] = PACKET0(mmUVD_GPCOM_VCPU_CMD, 0);
 	ib->ptr[5] = 0;
-	for (i = 6; i < 16; ++i)
-		ib->ptr[i] = PACKET2(0);
+	for (i = 6; i < 16; i += 2) {
+		ib->ptr[i] = PACKET0(mmUVD_NO_OP, 0);
+		ib->ptr[i+1] = 0;
+	}
 	ib->length_dw = 16;
 
 	if (direct) {
@@ -1114,15 +1087,9 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work)
 {
 	struct amdgpu_device *adev =
 		container_of(work, struct amdgpu_device, uvd.idle_work.work);
-	unsigned i, fences, handles = 0;
+	unsigned fences = amdgpu_fence_count_emitted(&adev->uvd.ring);
 
-	fences = amdgpu_fence_count_emitted(&adev->uvd.ring);
-
-	for (i = 0; i < adev->uvd.max_handles; ++i)
-		if (atomic_read(&adev->uvd.handles[i]))
-			++handles;
-
-	if (fences == 0 && handles == 0) {
+	if (fences == 0) {
 		if (adev->pm.dpm_enabled) {
 			amdgpu_dpm_enable_uvd(adev, false);
 		} else {
@@ -1187,7 +1154,8 @@ int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = 0;
 	}
 
-error:
 	fence_put(fence);
+
+error:
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 05865ce35351..7fe8fd884f06 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -210,6 +210,8 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
  */
 int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
 {
+	unsigned i;
+
 	if (adev->vce.vcpu_bo == NULL)
 		return 0;
 
@@ -217,8 +219,8 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
 
 	amdgpu_bo_unref(&adev->vce.vcpu_bo);
 
-	amdgpu_ring_fini(&adev->vce.ring[0]);
-	amdgpu_ring_fini(&adev->vce.ring[1]);
+	for (i = 0; i < adev->vce.num_rings; i++)
+		amdgpu_ring_fini(&adev->vce.ring[i]);
 
 	release_firmware(adev->vce.fw);
 	mutex_destroy(&adev->vce.idle_mutex);
@@ -282,8 +284,8 @@ int amdgpu_vce_resume(struct amdgpu_device *adev)
 
 	hdr = (const struct common_firmware_header *)adev->vce.fw->data;
 	offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
-	memcpy(cpu_addr, (adev->vce.fw->data) + offset,
-		(adev->vce.fw->size) - offset);
+	memcpy_toio(cpu_addr, adev->vce.fw->data + offset,
+		    adev->vce.fw->size - offset);
 
 	amdgpu_bo_kunmap(adev->vce.vcpu_bo);
 
@@ -303,9 +305,12 @@ static void amdgpu_vce_idle_work_handler(struct work_struct *work)
 {
 	struct amdgpu_device *adev =
 		container_of(work, struct amdgpu_device, vce.idle_work.work);
+	unsigned i, count = 0;
+
+	for (i = 0; i < adev->vce.num_rings; i++)
+		count += amdgpu_fence_count_emitted(&adev->vce.ring[i]);
 
-	if ((amdgpu_fence_count_emitted(&adev->vce.ring[0]) == 0) &&
-	    (amdgpu_fence_count_emitted(&adev->vce.ring[1]) == 0)) {
+	if (count == 0) {
 		if (adev->pm.dpm_enabled) {
 			amdgpu_dpm_enable_vce(adev, false);
 		} else {
@@ -634,7 +639,11 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
 	uint32_t allocated = 0;
 	uint32_t tmp, handle = 0;
 	uint32_t *size = &tmp;
-	int i, r = 0, idx = 0;
+	int i, r, idx = 0;
+
+	r = amdgpu_cs_sysvm_access_required(p);
+	if (r)
+		return r;
 
 	while (idx < ib->length_dw) {
 		uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
@@ -687,6 +696,21 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
 		case 0x04000008: /* rdo */
 		case 0x04000009: /* vui */
 		case 0x05000002: /* auxiliary buffer */
+		case 0x05000009: /* clock table */
+			break;
+
+		case 0x0500000c: /* hw config */
+			switch (p->adev->asic_type) {
+#ifdef CONFIG_DRM_AMDGPU_CIK
+			case CHIP_KAVERI:
+			case CHIP_MULLINS:
+#endif
+			case CHIP_CARRIZO:
+				break;
+			default:
+				r = -EINVAL;
+				goto out;
+			}
 			break;
 
 		case 0x03000001: /* encode */
@@ -799,6 +823,18 @@ void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 	amdgpu_ring_write(ring, VCE_CMD_END);
 }
 
+unsigned amdgpu_vce_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		4; /* amdgpu_vce_ring_emit_ib */
+}
+
+unsigned amdgpu_vce_ring_get_dma_frame_size(struct amdgpu_ring *ring)
+{
+	return
+		6; /* amdgpu_vce_ring_emit_fence  x1 no user fence */
+}
+
 /**
  * amdgpu_vce_ring_test_ring - test if VCE ring is working
  *
@@ -850,8 +886,8 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	struct fence *fence = NULL;
 	long r;
 
-	/* skip vce ring1 ib test for now, since it's not reliable */
-	if (ring == &ring->adev->vce.ring[1])
+	/* skip vce ring1/2 ib test for now, since it's not reliable */
+	if (ring != &ring->adev->vce.ring[0])
 		return 0;
 
 	r = amdgpu_vce_get_create_msg(ring, 1, NULL);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
index 63f83d0d985c..12729d2852df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
@@ -42,5 +42,7 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring);
 int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout);
 void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring);
 void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring);
+unsigned amdgpu_vce_ring_get_emit_ib_size(struct amdgpu_ring *ring);
+unsigned amdgpu_vce_ring_get_dma_frame_size(struct amdgpu_ring *ring);
 
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 88d68cb6e89d..2c37a374917f 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -19,22 +19,39 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  *
+ * Author: Monk.liu@amd.com
  */
+#ifndef AMDGPU_VIRT_H
+#define AMDGPU_VIRT_H
 
-#ifndef _POLARIS10_CLOCK_POWER_GATING_H_
-#define _POLARIS10_CLOCK_POWER_GATING_H_
+#define AMDGPU_SRIOV_CAPS_SRIOV_VBIOS  (1 << 0) /* vBIOS is sr-iov ready */
+#define AMDGPU_SRIOV_CAPS_ENABLE_IOV   (1 << 1) /* sr-iov is enabled on this GPU */
+#define AMDGPU_SRIOV_CAPS_IS_VF        (1 << 2) /* this GPU is a virtual function */
+#define AMDGPU_PASSTHROUGH_MODE        (1 << 3) /* thw whole GPU is pass through for VM */
+/* GPU virtualization */
+struct amdgpu_virtualization {
+	uint32_t virtual_caps;
+};
 
-#include "polaris10_hwmgr.h"
-#include "pp_asicblocks.h"
+#define amdgpu_sriov_enabled(adev) \
+((adev)->virtualization.virtual_caps & AMDGPU_SRIOV_CAPS_ENABLE_IOV)
 
-int polaris10_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate);
-int polaris10_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate);
-int polaris10_phm_powerdown_uvd(struct pp_hwmgr *hwmgr);
-int polaris10_phm_powergate_samu(struct pp_hwmgr *hwmgr, bool bgate);
-int polaris10_phm_powergate_acp(struct pp_hwmgr *hwmgr, bool bgate);
-int polaris10_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr);
-int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
-					const uint32_t *msg_id);
-int polaris10_phm_enable_per_cu_power_gating(struct pp_hwmgr *hwmgr, bool enable);
+#define amdgpu_sriov_vf(adev) \
+((adev)->virtualization.virtual_caps & AMDGPU_SRIOV_CAPS_IS_VF)
 
-#endif /* _POLARIS10_CLOCK_POWER_GATING_H_ */
+#define amdgpu_sriov_bios(adev) \
+((adev)->virtualization.virtual_caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)
+
+#define amdgpu_passthrough(adev) \
+((adev)->virtualization.virtual_caps & AMDGPU_PASSTHROUGH_MODE)
+
+static inline bool is_virtual_machine(void)
+{
+#ifdef CONFIG_X86
+	return boot_cpu_has(X86_FEATURE_HYPERVISOR);
+#else
+	return false;
+#endif
+}
+
+#endif
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 8e642fc48df4..06f24322e7c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -51,19 +51,22 @@
  * SI supports 16.
  */
 
-/* Special value that no flush is necessary */
-#define AMDGPU_VM_NO_FLUSH (~0ll)
-
 /* Local structure. Encapsulate some VM table update parameters to reduce
  * the number of function parameters
  */
-struct amdgpu_vm_update_params {
+struct amdgpu_pte_update_params {
+	/* amdgpu device we do this update for */
+	struct amdgpu_device *adev;
 	/* address where to copy page table entries from */
 	uint64_t src;
-	/* DMA addresses to use for mapping */
-	dma_addr_t *pages_addr;
 	/* indirect buffer to fill with commands */
 	struct amdgpu_ib *ib;
+	/* Function which actually does the update */
+	void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
+		     uint64_t addr, unsigned count, uint32_t incr,
+		     uint32_t flags);
+	/* indicate update pt or its shadow */
+	bool shadow;
 };
 
 /**
@@ -467,10 +470,9 @@ struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 }
 
 /**
- * amdgpu_vm_update_pages - helper to call the right asic function
+ * amdgpu_vm_do_set_ptes - helper to call the right asic function
  *
- * @adev: amdgpu_device pointer
- * @vm_update_params: see amdgpu_vm_update_params definition
+ * @params: see amdgpu_pte_update_params definition
  * @pe: addr of the page entry
  * @addr: dst addr to write into pe
  * @count: number of page entries to update
@@ -480,32 +482,46 @@ struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
  * Traces the parameters and calls the right asic functions
  * to setup the page table using the DMA.
  */
-static void amdgpu_vm_update_pages(struct amdgpu_device *adev,
-				   struct amdgpu_vm_update_params
-					*vm_update_params,
+static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
+				  uint64_t pe, uint64_t addr,
+				  unsigned count, uint32_t incr,
+				  uint32_t flags)
+{
+	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
+
+	if (count < 3) {
+		amdgpu_vm_write_pte(params->adev, params->ib, pe,
+				    addr | flags, count, incr);
+
+	} else {
+		amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
+				      count, incr, flags);
+	}
+}
+
+/**
+ * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
+ *
+ * @params: see amdgpu_pte_update_params definition
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: hw access flags
+ *
+ * Traces the parameters and calls the DMA function to copy the PTEs.
+ */
+static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
 				   uint64_t pe, uint64_t addr,
 				   unsigned count, uint32_t incr,
 				   uint32_t flags)
 {
-	trace_amdgpu_vm_set_page(pe, addr, count, incr, flags);
-
-	if (vm_update_params->src) {
-		amdgpu_vm_copy_pte(adev, vm_update_params->ib,
-			pe, (vm_update_params->src + (addr >> 12) * 8), count);
+	uint64_t src = (params->src + (addr >> 12) * 8);
 
-	} else if (vm_update_params->pages_addr) {
-		amdgpu_vm_write_pte(adev, vm_update_params->ib,
-			vm_update_params->pages_addr,
-			pe, addr, count, incr, flags);
 
-	} else if (count < 3) {
-		amdgpu_vm_write_pte(adev, vm_update_params->ib, NULL, pe, addr,
-				    count, incr, flags);
+	trace_amdgpu_vm_copy_ptes(pe, src, count);
 
-	} else {
-		amdgpu_vm_set_pte_pde(adev, vm_update_params->ib, pe, addr,
-				      count, incr, flags);
-	}
+	amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
 }
 
 /**
@@ -523,12 +539,11 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 	struct amdgpu_ring *ring;
 	struct fence *fence = NULL;
 	struct amdgpu_job *job;
-	struct amdgpu_vm_update_params vm_update_params;
+	struct amdgpu_pte_update_params params;
 	unsigned entries;
 	uint64_t addr;
 	int r;
 
-	memset(&vm_update_params, 0, sizeof(vm_update_params));
 	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 
 	r = reservation_object_reserve_shared(bo->tbo.resv);
@@ -539,6 +554,10 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 	if (r)
 		goto error;
 
+	r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
+	if (r)
+		goto error;
+
 	addr = amdgpu_bo_gpu_offset(bo);
 	entries = amdgpu_bo_size(bo) / 8;
 
@@ -546,9 +565,10 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 	if (r)
 		goto error;
 
-	vm_update_params.ib = &job->ibs[0];
-	amdgpu_vm_update_pages(adev, &vm_update_params, addr, 0, entries,
-			       0, 0);
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+	params.ib = &job->ibs[0];
+	amdgpu_vm_do_set_ptes(&params, addr, 0, entries, 0, 0);
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 
 	WARN_ON(job->ibs[0].length_dw > 64);
@@ -577,55 +597,46 @@ error:
  * Look up the physical address of the page that the pte resolves
  * to and return the pointer for the page table entry.
  */
-uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
+static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 {
 	uint64_t result;
 
-	if (pages_addr) {
-		/* page table offset */
-		result = pages_addr[addr >> PAGE_SHIFT];
-
-		/* in case cpu page size != gpu page size*/
-		result |= addr & (~PAGE_MASK);
+	/* page table offset */
+	result = pages_addr[addr >> PAGE_SHIFT];
 
-	} else {
-		/* No mapping required */
-		result = addr;
-	}
+	/* in case cpu page size != gpu page size*/
+	result |= addr & (~PAGE_MASK);
 
 	result &= 0xFFFFFFFFFFFFF000ULL;
 
 	return result;
 }
 
-/**
- * amdgpu_vm_update_pdes - make sure that page directory is valid
- *
- * @adev: amdgpu_device pointer
- * @vm: requested vm
- * @start: start of GPU address range
- * @end: end of GPU address range
- *
- * Allocates new page tables if necessary
- * and updates the page directory.
- * Returns 0 for success, error for failure.
- */
-int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
-				    struct amdgpu_vm *vm)
+static int amdgpu_vm_update_pd_or_shadow(struct amdgpu_device *adev,
+					 struct amdgpu_vm *vm,
+					 bool shadow)
 {
 	struct amdgpu_ring *ring;
-	struct amdgpu_bo *pd = vm->page_directory;
-	uint64_t pd_addr = amdgpu_bo_gpu_offset(pd);
+	struct amdgpu_bo *pd = shadow ? vm->page_directory->shadow :
+		vm->page_directory;
+	uint64_t pd_addr;
 	uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
 	uint64_t last_pde = ~0, last_pt = ~0;
 	unsigned count = 0, pt_idx, ndw;
 	struct amdgpu_job *job;
-	struct amdgpu_vm_update_params vm_update_params;
+	struct amdgpu_pte_update_params params;
 	struct fence *fence = NULL;
 
 	int r;
 
-	memset(&vm_update_params, 0, sizeof(vm_update_params));
+	if (!pd)
+		return 0;
+
+	r = amdgpu_ttm_bind(&pd->tbo, &pd->tbo.mem);
+	if (r)
+		return r;
+
+	pd_addr = amdgpu_bo_gpu_offset(pd);
 	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 
 	/* padding, etc. */
@@ -638,7 +649,9 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
 	if (r)
 		return r;
 
-	vm_update_params.ib = &job->ibs[0];
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+	params.ib = &job->ibs[0];
 
 	/* walk over the address space and update the page directory */
 	for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
@@ -648,20 +661,34 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
 		if (bo == NULL)
 			continue;
 
+		if (bo->shadow) {
+			struct amdgpu_bo *shadow = bo->shadow;
+
+			r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
+			if (r)
+				return r;
+		}
+
 		pt = amdgpu_bo_gpu_offset(bo);
-		if (vm->page_tables[pt_idx].addr == pt)
-			continue;
-		vm->page_tables[pt_idx].addr = pt;
+		if (!shadow) {
+			if (vm->page_tables[pt_idx].addr == pt)
+				continue;
+			vm->page_tables[pt_idx].addr = pt;
+		} else {
+			if (vm->page_tables[pt_idx].shadow_addr == pt)
+				continue;
+			vm->page_tables[pt_idx].shadow_addr = pt;
+		}
 
 		pde = pd_addr + pt_idx * 8;
 		if (((last_pde + 8 * count) != pde) ||
-		    ((last_pt + incr * count) != pt)) {
+		    ((last_pt + incr * count) != pt) ||
+		    (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
 
 			if (count) {
-				amdgpu_vm_update_pages(adev, &vm_update_params,
-						       last_pde, last_pt,
-						       count, incr,
-						       AMDGPU_PTE_VALID);
+				amdgpu_vm_do_set_ptes(&params, last_pde,
+						      last_pt, count, incr,
+						      AMDGPU_PTE_VALID);
 			}
 
 			count = 1;
@@ -673,15 +700,14 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
 	}
 
 	if (count)
-		amdgpu_vm_update_pages(adev, &vm_update_params,
-					last_pde, last_pt,
-					count, incr, AMDGPU_PTE_VALID);
+		amdgpu_vm_do_set_ptes(&params, last_pde, last_pt,
+				      count, incr, AMDGPU_PTE_VALID);
 
-	if (vm_update_params.ib->length_dw != 0) {
-		amdgpu_ring_pad_ib(ring, vm_update_params.ib);
+	if (params.ib->length_dw != 0) {
+		amdgpu_ring_pad_ib(ring, params.ib);
 		amdgpu_sync_resv(adev, &job->sync, pd->tbo.resv,
 				 AMDGPU_FENCE_OWNER_VM);
-		WARN_ON(vm_update_params.ib->length_dw > ndw);
+		WARN_ON(params.ib->length_dw > ndw);
 		r = amdgpu_job_submit(job, ring, &vm->entity,
 				      AMDGPU_FENCE_OWNER_VM, &fence);
 		if (r)
@@ -703,92 +729,33 @@ error_free:
 	return r;
 }
 
-/**
- * amdgpu_vm_frag_ptes - add fragment information to PTEs
+/*
+ * amdgpu_vm_update_pdes - make sure that page directory is valid
  *
  * @adev: amdgpu_device pointer
- * @vm_update_params: see amdgpu_vm_update_params definition
- * @pe_start: first PTE to handle
- * @pe_end: last PTE to handle
- * @addr: addr those PTEs should point to
- * @flags: hw mapping flags
+ * @vm: requested vm
+ * @start: start of GPU address range
+ * @end: end of GPU address range
+ *
+ * Allocates new page tables if necessary
+ * and updates the page directory.
+ * Returns 0 for success, error for failure.
  */
-static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
-				struct amdgpu_vm_update_params
-					*vm_update_params,
-				uint64_t pe_start, uint64_t pe_end,
-				uint64_t addr, uint32_t flags)
+int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
+                                   struct amdgpu_vm *vm)
 {
-	/**
-	 * The MC L1 TLB supports variable sized pages, based on a fragment
-	 * field in the PTE. When this field is set to a non-zero value, page
-	 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
-	 * flags are considered valid for all PTEs within the fragment range
-	 * and corresponding mappings are assumed to be physically contiguous.
-	 *
-	 * The L1 TLB can store a single PTE for the whole fragment,
-	 * significantly increasing the space available for translation
-	 * caching. This leads to large improvements in throughput when the
-	 * TLB is under pressure.
-	 *
-	 * The L2 TLB distributes small and large fragments into two
-	 * asymmetric partitions. The large fragment cache is significantly
-	 * larger. Thus, we try to use large fragments wherever possible.
-	 * Userspace can support this by aligning virtual base address and
-	 * allocation size to the fragment size.
-	 */
-
-	/* SI and newer are optimized for 64KB */
-	uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB;
-	uint64_t frag_align = 0x80;
-
-	uint64_t frag_start = ALIGN(pe_start, frag_align);
-	uint64_t frag_end = pe_end & ~(frag_align - 1);
-
-	unsigned count;
-
-	/* Abort early if there isn't anything to do */
-	if (pe_start == pe_end)
-		return;
-
-	/* system pages are non continuously */
-	if (vm_update_params->src || vm_update_params->pages_addr ||
-		!(flags & AMDGPU_PTE_VALID) || (frag_start >= frag_end)) {
-
-		count = (pe_end - pe_start) / 8;
-		amdgpu_vm_update_pages(adev, vm_update_params, pe_start,
-				       addr, count, AMDGPU_GPU_PAGE_SIZE,
-				       flags);
-		return;
-	}
-
-	/* handle the 4K area at the beginning */
-	if (pe_start != frag_start) {
-		count = (frag_start - pe_start) / 8;
-		amdgpu_vm_update_pages(adev, vm_update_params, pe_start, addr,
-				       count, AMDGPU_GPU_PAGE_SIZE, flags);
-		addr += AMDGPU_GPU_PAGE_SIZE * count;
-	}
-
-	/* handle the area in the middle */
-	count = (frag_end - frag_start) / 8;
-	amdgpu_vm_update_pages(adev, vm_update_params, frag_start, addr, count,
-			       AMDGPU_GPU_PAGE_SIZE, flags | frag_flags);
+	int r;
 
-	/* handle the 4K area at the end */
-	if (frag_end != pe_end) {
-		addr += AMDGPU_GPU_PAGE_SIZE * count;
-		count = (pe_end - frag_end) / 8;
-		amdgpu_vm_update_pages(adev, vm_update_params, frag_end, addr,
-				       count, AMDGPU_GPU_PAGE_SIZE, flags);
-	}
+	r = amdgpu_vm_update_pd_or_shadow(adev, vm, true);
+	if (r)
+		return r;
+	return amdgpu_vm_update_pd_or_shadow(adev, vm, false);
 }
 
 /**
  * amdgpu_vm_update_ptes - make sure that page tables are valid
  *
- * @adev: amdgpu_device pointer
- * @vm_update_params: see amdgpu_vm_update_params definition
+ * @params: see amdgpu_pte_update_params definition
  * @vm: requested vm
  * @start: start of GPU address range
  * @end: end of GPU address range
@@ -797,16 +764,14 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
  *
  * Update the page tables in the range @start - @end.
  */
-static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
-				  struct amdgpu_vm_update_params
-					*vm_update_params,
+static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 				  struct amdgpu_vm *vm,
 				  uint64_t start, uint64_t end,
 				  uint64_t dst, uint32_t flags)
 {
 	const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;
 
-	uint64_t cur_pe_start, cur_pe_end, cur_dst;
+	uint64_t cur_pe_start, cur_nptes, cur_dst;
 	uint64_t addr; /* next GPU address to be updated */
 	uint64_t pt_idx;
 	struct amdgpu_bo *pt;
@@ -817,7 +782,11 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
 	addr = start;
 	pt_idx = addr >> amdgpu_vm_block_size;
 	pt = vm->page_tables[pt_idx].entry.robj;
-
+	if (params->shadow) {
+		if (!pt->shadow)
+			return;
+		pt = vm->page_tables[pt_idx].entry.robj->shadow;
+	}
 	if ((addr & ~mask) == (end & ~mask))
 		nptes = end - addr;
 	else
@@ -825,7 +794,7 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
 
 	cur_pe_start = amdgpu_bo_gpu_offset(pt);
 	cur_pe_start += (addr & mask) * 8;
-	cur_pe_end = cur_pe_start + 8 * nptes;
+	cur_nptes = nptes;
 	cur_dst = dst;
 
 	/* for next ptb*/
@@ -836,6 +805,11 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
 	while (addr < end) {
 		pt_idx = addr >> amdgpu_vm_block_size;
 		pt = vm->page_tables[pt_idx].entry.robj;
+		if (params->shadow) {
+			if (!pt->shadow)
+				return;
+			pt = vm->page_tables[pt_idx].entry.robj->shadow;
+		}
 
 		if ((addr & ~mask) == (end & ~mask))
 			nptes = end - addr;
@@ -845,19 +819,19 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
 		next_pe_start = amdgpu_bo_gpu_offset(pt);
 		next_pe_start += (addr & mask) * 8;
 
-		if (cur_pe_end == next_pe_start) {
+		if ((cur_pe_start + 8 * cur_nptes) == next_pe_start &&
+		    ((cur_nptes + nptes) <= AMDGPU_VM_MAX_UPDATE_SIZE)) {
 			/* The next ptb is consecutive to current ptb.
-			 * Don't call amdgpu_vm_frag_ptes now.
+			 * Don't call the update function now.
 			 * Will update two ptbs together in future.
 			*/
-			cur_pe_end += 8 * nptes;
+			cur_nptes += nptes;
 		} else {
-			amdgpu_vm_frag_ptes(adev, vm_update_params,
-					    cur_pe_start, cur_pe_end,
-					    cur_dst, flags);
+			params->func(params, cur_pe_start, cur_dst, cur_nptes,
+				     AMDGPU_GPU_PAGE_SIZE, flags);
 
 			cur_pe_start = next_pe_start;
-			cur_pe_end = next_pe_start + 8 * nptes;
+			cur_nptes = nptes;
 			cur_dst = dst;
 		}
 
@@ -866,8 +840,75 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
 		dst += nptes * AMDGPU_GPU_PAGE_SIZE;
 	}
 
-	amdgpu_vm_frag_ptes(adev, vm_update_params, cur_pe_start,
-			    cur_pe_end, cur_dst, flags);
+	params->func(params, cur_pe_start, cur_dst, cur_nptes,
+		     AMDGPU_GPU_PAGE_SIZE, flags);
+}
+
+/*
+ * amdgpu_vm_frag_ptes - add fragment information to PTEs
+ *
+ * @params: see amdgpu_pte_update_params definition
+ * @vm: requested vm
+ * @start: first PTE to handle
+ * @end: last PTE to handle
+ * @dst: addr those PTEs should point to
+ * @flags: hw mapping flags
+ */
+static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
+				struct amdgpu_vm *vm,
+				uint64_t start, uint64_t end,
+				uint64_t dst, uint32_t flags)
+{
+	/**
+	 * The MC L1 TLB supports variable sized pages, based on a fragment
+	 * field in the PTE. When this field is set to a non-zero value, page
+	 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
+	 * flags are considered valid for all PTEs within the fragment range
+	 * and corresponding mappings are assumed to be physically contiguous.
+	 *
+	 * The L1 TLB can store a single PTE for the whole fragment,
+	 * significantly increasing the space available for translation
+	 * caching. This leads to large improvements in throughput when the
+	 * TLB is under pressure.
+	 *
+	 * The L2 TLB distributes small and large fragments into two
+	 * asymmetric partitions. The large fragment cache is significantly
+	 * larger. Thus, we try to use large fragments wherever possible.
+	 * Userspace can support this by aligning virtual base address and
+	 * allocation size to the fragment size.
+	 */
+
+	/* SI and newer are optimized for 64KB */
+	uint64_t frag_flags = AMDGPU_PTE_FRAG(AMDGPU_LOG2_PAGES_PER_FRAG);
+	uint64_t frag_align = 1 << AMDGPU_LOG2_PAGES_PER_FRAG;
+
+	uint64_t frag_start = ALIGN(start, frag_align);
+	uint64_t frag_end = end & ~(frag_align - 1);
+
+	/* system pages are non continuously */
+	if (params->src || !(flags & AMDGPU_PTE_VALID) ||
+	    (frag_start >= frag_end)) {
+
+		amdgpu_vm_update_ptes(params, vm, start, end, dst, flags);
+		return;
+	}
+
+	/* handle the 4K area at the beginning */
+	if (start != frag_start) {
+		amdgpu_vm_update_ptes(params, vm, start, frag_start,
+				      dst, flags);
+		dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
+	}
+
+	/* handle the area in the middle */
+	amdgpu_vm_update_ptes(params, vm, frag_start, frag_end, dst,
+			      flags | frag_flags);
+
+	/* handle the 4K area at the end */
+	if (frag_end != end) {
+		dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
+		amdgpu_vm_update_ptes(params, vm, frag_end, end, dst, flags);
+	}
 }
 
 /**
@@ -900,14 +941,19 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 	void *owner = AMDGPU_FENCE_OWNER_VM;
 	unsigned nptes, ncmds, ndw;
 	struct amdgpu_job *job;
-	struct amdgpu_vm_update_params vm_update_params;
+	struct amdgpu_pte_update_params params;
 	struct fence *f = NULL;
 	int r;
 
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+	params.src = src;
+
 	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
-	memset(&vm_update_params, 0, sizeof(vm_update_params));
-	vm_update_params.src = src;
-	vm_update_params.pages_addr = pages_addr;
+
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+	params.src = src;
 
 	/* sync to everything on unmapping */
 	if (!(flags & AMDGPU_PTE_VALID))
@@ -924,30 +970,53 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 	/* padding, etc. */
 	ndw = 64;
 
-	if (vm_update_params.src) {
+	if (src) {
 		/* only copy commands needed */
 		ndw += ncmds * 7;
 
-	} else if (vm_update_params.pages_addr) {
-		/* header for write data commands */
-		ndw += ncmds * 4;
+		params.func = amdgpu_vm_do_copy_ptes;
+
+	} else if (pages_addr) {
+		/* copy commands needed */
+		ndw += ncmds * 7;
 
-		/* body of write data command */
+		/* and also PTEs */
 		ndw += nptes * 2;
 
+		params.func = amdgpu_vm_do_copy_ptes;
+
 	} else {
 		/* set page commands needed */
 		ndw += ncmds * 10;
 
 		/* two extra commands for begin/end of fragment */
 		ndw += 2 * 10;
+
+		params.func = amdgpu_vm_do_set_ptes;
 	}
 
 	r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 	if (r)
 		return r;
 
-	vm_update_params.ib = &job->ibs[0];
+	params.ib = &job->ibs[0];
+
+	if (!src && pages_addr) {
+		uint64_t *pte;
+		unsigned i;
+
+		/* Put the PTEs at the end of the IB. */
+		i = ndw - nptes * 2;
+		pte= (uint64_t *)&(job->ibs->ptr[i]);
+		params.src = job->ibs->gpu_addr + i * 4;
+
+		for (i = 0; i < nptes; ++i) {
+			pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
+						    AMDGPU_GPU_PAGE_SIZE);
+			pte[i] |= flags;
+		}
+		addr = 0;
+	}
 
 	r = amdgpu_sync_fence(adev, &job->sync, exclusive);
 	if (r)
@@ -962,11 +1031,13 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 	if (r)
 		goto error_free;
 
-	amdgpu_vm_update_ptes(adev, &vm_update_params, vm, start,
-			      last + 1, addr, flags);
+	params.shadow = true;
+	amdgpu_vm_frag_ptes(&params, vm, start, last + 1, addr, flags);
+	params.shadow = false;
+	amdgpu_vm_frag_ptes(&params, vm, start, last + 1, addr, flags);
 
-	amdgpu_ring_pad_ib(ring, vm_update_params.ib);
-	WARN_ON(vm_update_params.ib->length_dw > ndw);
+	amdgpu_ring_pad_ib(ring, params.ib);
+	WARN_ON(params.ib->length_dw > ndw);
 	r = amdgpu_job_submit(job, ring, &vm->entity,
 			      AMDGPU_FENCE_OWNER_VM, &f);
 	if (r)
@@ -1062,28 +1133,32 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
  *
  * @adev: amdgpu_device pointer
  * @bo_va: requested BO and VM object
- * @mem: ttm mem
+ * @clear: if true clear the entries
  *
  * Fill in the page table entries for @bo_va.
  * Returns 0 for success, -EINVAL for failure.
- *
- * Object have to be reserved and mutex must be locked!
  */
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			struct amdgpu_bo_va *bo_va,
-			struct ttm_mem_reg *mem)
+			bool clear)
 {
 	struct amdgpu_vm *vm = bo_va->vm;
 	struct amdgpu_bo_va_mapping *mapping;
 	dma_addr_t *pages_addr = NULL;
 	uint32_t gtt_flags, flags;
+	struct ttm_mem_reg *mem;
 	struct fence *exclusive;
 	uint64_t addr;
 	int r;
 
-	if (mem) {
+	if (clear) {
+		mem = NULL;
+		addr = 0;
+		exclusive = NULL;
+	} else {
 		struct ttm_dma_tt *ttm;
 
+		mem = &bo_va->bo->tbo.mem;
 		addr = (u64)mem->start << PAGE_SHIFT;
 		switch (mem->mem_type) {
 		case TTM_PL_TT:
@@ -1101,13 +1176,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 		}
 
 		exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
-	} else {
-		addr = 0;
-		exclusive = NULL;
 	}
 
 	flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
-	gtt_flags = (adev == bo_va->bo->adev) ? flags : 0;
+	gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
+		adev == bo_va->bo->adev) ? flags : 0;
 
 	spin_lock(&vm->status_lock);
 	if (!list_empty(&bo_va->vm_status))
@@ -1134,7 +1207,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 	spin_lock(&vm->status_lock);
 	list_splice_init(&bo_va->invalids, &bo_va->valids);
 	list_del_init(&bo_va->vm_status);
-	if (!mem)
+	if (clear)
 		list_add(&bo_va->vm_status, &vm->cleared);
 	spin_unlock(&vm->status_lock);
 
@@ -1197,7 +1270,7 @@ int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
 			struct amdgpu_bo_va, vm_status);
 		spin_unlock(&vm->status_lock);
 
-		r = amdgpu_vm_bo_update(adev, bo_va, NULL);
+		r = amdgpu_vm_bo_update(adev, bo_va, true);
 		if (r)
 			return r;
 
@@ -1342,7 +1415,8 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 		r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
 				     AMDGPU_GPU_PAGE_SIZE, true,
 				     AMDGPU_GEM_DOMAIN_VRAM,
-				     AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
+				     AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
+				     AMDGPU_GEM_CREATE_SHADOW,
 				     NULL, resv, &pt);
 		if (r)
 			goto error_free;
@@ -1354,10 +1428,20 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 
 		r = amdgpu_vm_clear_bo(adev, vm, pt);
 		if (r) {
+			amdgpu_bo_unref(&pt->shadow);
 			amdgpu_bo_unref(&pt);
 			goto error_free;
 		}
 
+		if (pt->shadow) {
+			r = amdgpu_vm_clear_bo(adev, vm, pt->shadow);
+			if (r) {
+				amdgpu_bo_unref(&pt->shadow);
+				amdgpu_bo_unref(&pt);
+				goto error_free;
+			}
+		}
+
 		entry->robj = pt;
 		entry->priority = 0;
 		entry->tv.bo = &entry->robj->tbo;
@@ -1535,13 +1619,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	r = amd_sched_entity_init(&ring->sched, &vm->entity,
 				  rq, amdgpu_sched_jobs);
 	if (r)
-		return r;
+		goto err;
 
 	vm->page_directory_fence = NULL;
 
 	r = amdgpu_bo_create(adev, pd_size, align, true,
 			     AMDGPU_GEM_DOMAIN_VRAM,
-			     AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
+			     AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
+			     AMDGPU_GEM_CREATE_SHADOW,
 			     NULL, NULL, &vm->page_directory);
 	if (r)
 		goto error_free_sched_entity;
@@ -1551,20 +1636,34 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 		goto error_free_page_directory;
 
 	r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory);
-	amdgpu_bo_unreserve(vm->page_directory);
 	if (r)
-		goto error_free_page_directory;
+		goto error_unreserve;
+
+	if (vm->page_directory->shadow) {
+		r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory->shadow);
+		if (r)
+			goto error_unreserve;
+	}
+
 	vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
+	amdgpu_bo_unreserve(vm->page_directory);
 
 	return 0;
 
+error_unreserve:
+	amdgpu_bo_unreserve(vm->page_directory);
+
 error_free_page_directory:
+	amdgpu_bo_unref(&vm->page_directory->shadow);
 	amdgpu_bo_unref(&vm->page_directory);
 	vm->page_directory = NULL;
 
 error_free_sched_entity:
 	amd_sched_entity_fini(&ring->sched, &vm->entity);
 
+err:
+	drm_free_large(vm->page_tables);
+
 	return r;
 }
 
@@ -1597,10 +1696,18 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 		kfree(mapping);
 	}
 
-	for (i = 0; i < amdgpu_vm_num_pdes(adev); i++)
-		amdgpu_bo_unref(&vm->page_tables[i].entry.robj);
+	for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) {
+		struct amdgpu_bo *pt = vm->page_tables[i].entry.robj;
+
+		if (!pt)
+			continue;
+
+		amdgpu_bo_unref(&pt->shadow);
+		amdgpu_bo_unref(&pt);
+	}
 	drm_free_large(vm->page_tables);
 
+	amdgpu_bo_unref(&vm->page_directory->shadow);
 	amdgpu_bo_unref(&vm->page_directory);
 	fence_put(vm->page_directory_fence);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c
index 49a39b1a0a96..f7d236f95e74 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c
@@ -497,7 +497,13 @@ void amdgpu_atombios_crtc_set_disp_eng_pll(struct amdgpu_device *adev,
 			 * SetPixelClock provides the dividers
 			 */
 			args.v6.ulDispEngClkFreq = cpu_to_le32(dispclk);
-			args.v6.ucPpll = ATOM_EXT_PLL1;
+			if (adev->asic_type == CHIP_TAHITI ||
+			    adev->asic_type == CHIP_PITCAIRN ||
+			    adev->asic_type == CHIP_VERDE ||
+			    adev->asic_type == CHIP_OLAND)
+				args.v6.ucPpll = ATOM_PPLL0;
+			else
+				args.v6.ucPpll = ATOM_EXT_PLL1;
 			break;
 		default:
 			DRM_ERROR("Unknown table version %d %d\n", frev, crev);
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
index 7f85c2c1d681..f81068ba4cc6 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
@@ -88,7 +88,6 @@ static int amdgpu_atombios_dp_process_aux_ch(struct amdgpu_i2c_chan *chan,
 
 	/* timeout */
 	if (args.v2.ucReplyStatus == 1) {
-		DRM_DEBUG_KMS("dp_aux_ch timeout\n");
 		r = -ETIMEDOUT;
 		goto done;
 	}
@@ -339,22 +338,21 @@ int amdgpu_atombios_dp_get_dpcd(struct amdgpu_connector *amdgpu_connector)
 {
 	struct amdgpu_connector_atom_dig *dig_connector = amdgpu_connector->con_priv;
 	u8 msg[DP_DPCD_SIZE];
-	int ret, i;
+	int ret;
 
-	for (i = 0; i < 7; i++) {
-		ret = drm_dp_dpcd_read(&amdgpu_connector->ddc_bus->aux, DP_DPCD_REV, msg,
-				       DP_DPCD_SIZE);
-		if (ret == DP_DPCD_SIZE) {
-			memcpy(dig_connector->dpcd, msg, DP_DPCD_SIZE);
+	ret = drm_dp_dpcd_read(&amdgpu_connector->ddc_bus->aux, DP_DPCD_REV,
+			       msg, DP_DPCD_SIZE);
+	if (ret == DP_DPCD_SIZE) {
+		memcpy(dig_connector->dpcd, msg, DP_DPCD_SIZE);
 
-			DRM_DEBUG_KMS("DPCD: %*ph\n", (int)sizeof(dig_connector->dpcd),
-				      dig_connector->dpcd);
+		DRM_DEBUG_KMS("DPCD: %*ph\n", (int)sizeof(dig_connector->dpcd),
+			      dig_connector->dpcd);
 
-			amdgpu_atombios_dp_probe_oui(amdgpu_connector);
+		amdgpu_atombios_dp_probe_oui(amdgpu_connector);
 
-			return 0;
-		}
+		return 0;
 	}
+
 	dig_connector->dpcd[0] = 0;
 	return -EINVAL;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
index bc56c8a181e6..b374653bd6cf 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
@@ -27,6 +27,7 @@
 #include "amdgpu.h"
 #include "atom.h"
 #include "amdgpu_atombios.h"
+#include "atombios_i2c.h"
 
 #define TARGET_HW_I2C_CLOCK 50
 
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index e2f0e5d58d5c..1d8c375a3561 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -5396,7 +5396,7 @@ static void ci_dpm_disable(struct amdgpu_device *adev)
 	amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq,
 		       AMDGPU_THERMAL_IRQ_HIGH_TO_LOW);
 
-	ci_dpm_powergate_uvd(adev, false);
+	ci_dpm_powergate_uvd(adev, true);
 
 	if (!amdgpu_ci_is_smc_running(adev))
 		return;
@@ -5779,6 +5779,7 @@ static int ci_dpm_init_microcode(struct amdgpu_device *adev)
 		break;
 	case CHIP_KAVERI:
 	case CHIP_KABINI:
+	case CHIP_MULLINS:
 	default: BUG();
 	}
 
@@ -5873,7 +5874,10 @@ static int ci_dpm_init(struct amdgpu_device *adev)
 	pi->pcie_dpm_key_disabled = 0;
 	pi->thermal_sclk_dpm_enabled = 0;
 
-	pi->caps_sclk_ds = true;
+	if (amdgpu_sclk_deep_sleep_en)
+		pi->caps_sclk_ds = true;
+	else
+		pi->caps_sclk_ds = false;
 
 	pi->mclk_strobe_mode_threshold = 40000;
 	pi->mclk_stutter_mode_threshold = 40000;
@@ -6032,7 +6036,7 @@ static int ci_dpm_init(struct amdgpu_device *adev)
 
 	pi->caps_dynamic_ac_timing = true;
 
-	pi->uvd_power_gated = false;
+	pi->uvd_power_gated = true;
 
 	/* make sure dc limits are valid */
 	if ((adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
@@ -6175,8 +6179,6 @@ static int ci_dpm_late_init(void *handle)
 	if (ret)
 		return ret;
 
-	ci_dpm_powergate_uvd(adev, true);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 4efc901f658c..a845b6a93b79 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -67,6 +67,7 @@
 
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_powerplay.h"
+#include "dce_virtual.h"
 
 /*
  * Indirect registers accessor
@@ -962,12 +963,6 @@ static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 	return true;
 }
 
-static u32 cik_get_virtual_caps(struct amdgpu_device *adev)
-{
-	/* CIK does not support SR-IOV */
-	return 0;
-}
-
 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 	{mmGRBM_STATUS, false},
 	{mmGB_ADDR_CONFIG, false},
@@ -1640,6 +1635,12 @@ static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
 }
 
+static void cik_detect_hw_virtualization(struct amdgpu_device *adev)
+{
+	if (is_virtual_machine()) /* passthrough mode */
+		adev->virtualization.virtual_caps |= AMDGPU_PASSTHROUGH_MODE;
+}
+
 static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
 {
 	/* ORDER MATTERS! */
@@ -1708,6 +1709,74 @@ static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
 	},
 };
 
+static const struct amdgpu_ip_block_version bonaire_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gmc_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 8,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 7,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &gfx_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_sdma_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 4,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &uvd_v4_2_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vce_v2_0_ip_funcs,
+	},
+};
+
 static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
 {
 	/* ORDER MATTERS! */
@@ -1776,6 +1845,74 @@ static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
 	},
 };
 
+static const struct amdgpu_ip_block_version hawaii_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gmc_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 8,
+		.minor = 5,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 7,
+		.minor = 3,
+		.rev = 0,
+		.funcs = &gfx_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_sdma_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 4,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &uvd_v4_2_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vce_v2_0_ip_funcs,
+	},
+};
+
 static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
 {
 	/* ORDER MATTERS! */
@@ -1844,6 +1981,74 @@ static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
 	},
 };
 
+static const struct amdgpu_ip_block_version kabini_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gmc_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 8,
+		.minor = 3,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 7,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &gfx_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_sdma_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 4,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &uvd_v4_2_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vce_v2_0_ip_funcs,
+	},
+};
+
 static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
 {
 	/* ORDER MATTERS! */
@@ -1912,6 +2117,74 @@ static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
 	},
 };
 
+static const struct amdgpu_ip_block_version mullins_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gmc_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 8,
+		.minor = 3,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 7,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &gfx_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_sdma_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 4,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &uvd_v4_2_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vce_v2_0_ip_funcs,
+	},
+};
+
 static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
 {
 	/* ORDER MATTERS! */
@@ -1980,32 +2253,128 @@ static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
 	},
 };
 
+static const struct amdgpu_ip_block_version kaveri_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gmc_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 7,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 8,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 7,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &gfx_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cik_sdma_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 4,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &uvd_v4_2_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vce_v2_0_ip_funcs,
+	},
+};
+
 int cik_set_ip_blocks(struct amdgpu_device *adev)
 {
-	switch (adev->asic_type) {
-	case CHIP_BONAIRE:
-		adev->ip_blocks = bonaire_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
-		break;
-	case CHIP_HAWAII:
-		adev->ip_blocks = hawaii_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
-		break;
-	case CHIP_KAVERI:
-		adev->ip_blocks = kaveri_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
-		break;
-	case CHIP_KABINI:
-		adev->ip_blocks = kabini_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
-		break;
-	case CHIP_MULLINS:
-		adev->ip_blocks = mullins_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
-		break;
-	default:
-		/* FIXME: not supported yet */
-		return -EINVAL;
+	if (adev->enable_virtual_display) {
+		switch (adev->asic_type) {
+		case CHIP_BONAIRE:
+			adev->ip_blocks = bonaire_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks_vd);
+			break;
+		case CHIP_HAWAII:
+			adev->ip_blocks = hawaii_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks_vd);
+			break;
+		case CHIP_KAVERI:
+			adev->ip_blocks = kaveri_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks_vd);
+			break;
+		case CHIP_KABINI:
+			adev->ip_blocks = kabini_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks_vd);
+			break;
+		case CHIP_MULLINS:
+			adev->ip_blocks = mullins_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks_vd);
+			break;
+		default:
+			/* FIXME: not supported yet */
+			return -EINVAL;
+		}
+	} else {
+		switch (adev->asic_type) {
+		case CHIP_BONAIRE:
+			adev->ip_blocks = bonaire_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
+			break;
+		case CHIP_HAWAII:
+			adev->ip_blocks = hawaii_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
+			break;
+		case CHIP_KAVERI:
+			adev->ip_blocks = kaveri_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
+			break;
+		case CHIP_KABINI:
+			adev->ip_blocks = kabini_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
+			break;
+		case CHIP_MULLINS:
+			adev->ip_blocks = mullins_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
+			break;
+		default:
+			/* FIXME: not supported yet */
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -2015,13 +2384,13 @@ static const struct amdgpu_asic_funcs cik_asic_funcs =
 {
 	.read_disabled_bios = &cik_read_disabled_bios,
 	.read_bios_from_rom = &cik_read_bios_from_rom,
+	.detect_hw_virtualization = cik_detect_hw_virtualization,
 	.read_register = &cik_read_register,
 	.reset = &cik_asic_reset,
 	.set_vga_state = &cik_vga_set_state,
 	.get_xclk = &cik_get_xclk,
 	.set_uvd_clocks = &cik_set_uvd_clocks,
 	.set_vce_clocks = &cik_set_vce_clocks,
-	.get_virtual_caps = &cik_get_virtual_caps,
 };
 
 static int cik_common_early_init(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index ee6466912497..cb952acc7133 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -52,6 +52,7 @@ static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev);
 static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev);
 static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev);
 static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev);
+static int cik_sdma_soft_reset(void *handle);
 
 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
 MODULE_FIRMWARE("radeon/bonaire_sdma1.bin");
@@ -694,24 +695,16 @@ static void cik_sdma_vm_copy_pte(struct amdgpu_ib *ib,
 				 uint64_t pe, uint64_t src,
 				 unsigned count)
 {
-	while (count) {
-		unsigned bytes = count * 8;
-		if (bytes > 0x1FFFF8)
-			bytes = 0x1FFFF8;
-
-		ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY,
-			SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
-		ib->ptr[ib->length_dw++] = bytes;
-		ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
-		ib->ptr[ib->length_dw++] = lower_32_bits(src);
-		ib->ptr[ib->length_dw++] = upper_32_bits(src);
-		ib->ptr[ib->length_dw++] = lower_32_bits(pe);
-		ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-
-		pe += bytes;
-		src += bytes;
-		count -= bytes / 8;
-	}
+	unsigned bytes = count * 8;
+
+	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY,
+		SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
+	ib->ptr[ib->length_dw++] = bytes;
+	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+	ib->ptr[ib->length_dw++] = lower_32_bits(src);
+	ib->ptr[ib->length_dw++] = upper_32_bits(src);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 }
 
 /**
@@ -719,39 +712,27 @@ static void cik_sdma_vm_copy_pte(struct amdgpu_ib *ib,
  *
  * @ib: indirect buffer to fill with commands
  * @pe: addr of the page entry
- * @addr: dst addr to write into pe
+ * @value: dst addr to write into pe
  * @count: number of page entries to update
  * @incr: increase next addr by incr bytes
- * @flags: access flags
  *
  * Update PTEs by writing them manually using sDMA (CIK).
  */
-static void cik_sdma_vm_write_pte(struct amdgpu_ib *ib,
-				  const dma_addr_t *pages_addr, uint64_t pe,
-				  uint64_t addr, unsigned count,
-				  uint32_t incr, uint32_t flags)
+static void cik_sdma_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
+				  uint64_t value, unsigned count,
+				  uint32_t incr)
 {
-	uint64_t value;
-	unsigned ndw;
-
-	while (count) {
-		ndw = count * 2;
-		if (ndw > 0xFFFFE)
-			ndw = 0xFFFFE;
-
-		/* for non-physically contiguous pages (system) */
-		ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE,
-			SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
-		ib->ptr[ib->length_dw++] = pe;
-		ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-		ib->ptr[ib->length_dw++] = ndw;
-		for (; ndw > 0; ndw -= 2, --count, pe += 8) {
-			value = amdgpu_vm_map_gart(pages_addr, addr);
-			addr += incr;
-			value |= flags;
-			ib->ptr[ib->length_dw++] = value;
-			ib->ptr[ib->length_dw++] = upper_32_bits(value);
-		}
+	unsigned ndw = count * 2;
+
+	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE,
+		SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = ndw;
+	for (; ndw > 0; ndw -= 2) {
+		ib->ptr[ib->length_dw++] = lower_32_bits(value);
+		ib->ptr[ib->length_dw++] = upper_32_bits(value);
+		value += incr;
 	}
 }
 
@@ -767,40 +748,21 @@ static void cik_sdma_vm_write_pte(struct amdgpu_ib *ib,
  *
  * Update the page tables using sDMA (CIK).
  */
-static void cik_sdma_vm_set_pte_pde(struct amdgpu_ib *ib,
-				    uint64_t pe,
+static void cik_sdma_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe,
 				    uint64_t addr, unsigned count,
 				    uint32_t incr, uint32_t flags)
 {
-	uint64_t value;
-	unsigned ndw;
-
-	while (count) {
-		ndw = count;
-		if (ndw > 0x7FFFF)
-			ndw = 0x7FFFF;
-
-		if (flags & AMDGPU_PTE_VALID)
-			value = addr;
-		else
-			value = 0;
-
-		/* for physically contiguous pages (vram) */
-		ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
-		ib->ptr[ib->length_dw++] = pe; /* dst addr */
-		ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-		ib->ptr[ib->length_dw++] = flags; /* mask */
-		ib->ptr[ib->length_dw++] = 0;
-		ib->ptr[ib->length_dw++] = value; /* value */
-		ib->ptr[ib->length_dw++] = upper_32_bits(value);
-		ib->ptr[ib->length_dw++] = incr; /* increment size */
-		ib->ptr[ib->length_dw++] = 0;
-		ib->ptr[ib->length_dw++] = ndw; /* number of entries */
-
-		pe += ndw * 8;
-		addr += ndw * incr;
-		count -= ndw;
-	}
+	/* for physically contiguous pages (vram) */
+	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = flags; /* mask */
+	ib->ptr[ib->length_dw++] = 0;
+	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
+	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
+	ib->ptr[ib->length_dw++] = incr; /* increment size */
+	ib->ptr[ib->length_dw++] = 0;
+	ib->ptr[ib->length_dw++] = count; /* number of entries */
 }
 
 /**
@@ -886,6 +848,22 @@ static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */
 }
 
+static unsigned cik_sdma_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		7 + 4; /* cik_sdma_ring_emit_ib */
+}
+
+static unsigned cik_sdma_ring_get_dma_frame_size(struct amdgpu_ring *ring)
+{
+	return
+		6 + /* cik_sdma_ring_emit_hdp_flush */
+		3 + /* cik_sdma_ring_emit_hdp_invalidate */
+		6 + /* cik_sdma_ring_emit_pipeline_sync */
+		12 + /* cik_sdma_ring_emit_vm_flush */
+		9 + 9 + 9; /* cik_sdma_ring_emit_fence x3 for user fence, vm fence */
+}
+
 static void cik_enable_sdma_mgcg(struct amdgpu_device *adev,
 				 bool enable)
 {
@@ -1037,6 +1015,8 @@ static int cik_sdma_resume(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	cik_sdma_soft_reset(handle);
+
 	return cik_sdma_hw_init(adev);
 }
 
@@ -1259,6 +1239,8 @@ static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = {
 	.test_ib = cik_sdma_ring_test_ib,
 	.insert_nop = cik_sdma_ring_insert_nop,
 	.pad_ib = cik_sdma_ring_pad_ib,
+	.get_emit_ib_size = cik_sdma_ring_get_emit_ib_size,
+	.get_dma_frame_size = cik_sdma_ring_get_dma_frame_size,
 };
 
 static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/cikd.h b/drivers/gpu/drm/amd/amdgpu/cikd.h
index c4f6f00d62bc..8659852aea9e 100644
--- a/drivers/gpu/drm/amd/amdgpu/cikd.h
+++ b/drivers/gpu/drm/amd/amdgpu/cikd.h
@@ -562,4 +562,40 @@ enum {
 	MTYPE_NONCACHED = 3
 };
 
+/* mmPA_SC_RASTER_CONFIG mask */
+#define RB_MAP_PKR0(x)				((x) << 0)
+#define RB_MAP_PKR0_MASK			(0x3 << 0)
+#define RB_MAP_PKR1(x)				((x) << 2)
+#define RB_MAP_PKR1_MASK			(0x3 << 2)
+#define RB_XSEL2(x)				((x) << 4)
+#define RB_XSEL2_MASK				(0x3 << 4)
+#define RB_XSEL					(1 << 6)
+#define RB_YSEL					(1 << 7)
+#define PKR_MAP(x)				((x) << 8)
+#define PKR_MAP_MASK				(0x3 << 8)
+#define PKR_XSEL(x)				((x) << 10)
+#define PKR_XSEL_MASK				(0x3 << 10)
+#define PKR_YSEL(x)				((x) << 12)
+#define PKR_YSEL_MASK				(0x3 << 12)
+#define SC_MAP(x)				((x) << 16)
+#define SC_MAP_MASK				(0x3 << 16)
+#define SC_XSEL(x)				((x) << 18)
+#define SC_XSEL_MASK				(0x3 << 18)
+#define SC_YSEL(x)				((x) << 20)
+#define SC_YSEL_MASK				(0x3 << 20)
+#define SE_MAP(x)				((x) << 24)
+#define SE_MAP_MASK				(0x3 << 24)
+#define SE_XSEL(x)				((x) << 26)
+#define SE_XSEL_MASK				(0x3 << 26)
+#define SE_YSEL(x)				((x) << 28)
+#define SE_YSEL_MASK				(0x3 << 28)
+
+/* mmPA_SC_RASTER_CONFIG_1 mask */
+#define SE_PAIR_MAP(x)				((x) << 0)
+#define SE_PAIR_MAP_MASK			(0x3 << 0)
+#define SE_PAIR_XSEL(x)				((x) << 2)
+#define SE_PAIR_XSEL_MASK			(0x3 << 2)
+#define SE_PAIR_YSEL(x)				((x) << 4)
+#define SE_PAIR_YSEL_MASK			(0x3 << 4)
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
index 2a11413ed54a..f80a0834e889 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
@@ -44,6 +44,7 @@
 
 static void cz_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate);
 static void cz_dpm_powergate_vce(struct amdgpu_device *adev, bool gate);
+static void cz_dpm_fini(struct amdgpu_device *adev);
 
 static struct cz_ps *cz_get_ps(struct amdgpu_ps *rps)
 {
@@ -350,6 +351,8 @@ static int cz_parse_power_table(struct amdgpu_device *adev)
 
 		ps = kzalloc(sizeof(struct cz_ps), GFP_KERNEL);
 		if (ps == NULL) {
+			for (j = 0; j < i; j++)
+				kfree(adev->pm.dpm.ps[j].ps_priv);
 			kfree(adev->pm.dpm.ps);
 			return -ENOMEM;
 		}
@@ -409,11 +412,11 @@ static int cz_dpm_init(struct amdgpu_device *adev)
 
 	ret = amdgpu_get_platform_caps(adev);
 	if (ret)
-		return ret;
+		goto err;
 
 	ret = amdgpu_parse_extended_power_table(adev);
 	if (ret)
-		return ret;
+		goto err;
 
 	pi->sram_end = SMC_RAM_END;
 
@@ -435,7 +438,11 @@ static int cz_dpm_init(struct amdgpu_device *adev)
 		pi->caps_td_ramping = true;
 		pi->caps_tcp_ramping = true;
 	}
-	pi->caps_sclk_ds = true;
+	if (amdgpu_sclk_deep_sleep_en)
+		pi->caps_sclk_ds = true;
+	else
+		pi->caps_sclk_ds = false;
+
 	pi->voting_clients = 0x00c00033;
 	pi->auto_thermal_throttling_enabled = true;
 	pi->bapm_enabled = false;
@@ -463,23 +470,26 @@ static int cz_dpm_init(struct amdgpu_device *adev)
 
 	ret = cz_parse_sys_info_table(adev);
 	if (ret)
-		return ret;
+		goto err;
 
 	cz_patch_voltage_values(adev);
 	cz_construct_boot_state(adev);
 
 	ret = cz_parse_power_table(adev);
 	if (ret)
-		return ret;
+		goto err;
 
 	ret = cz_process_firmware_header(adev);
 	if (ret)
-		return ret;
+		goto err;
 
 	pi->dpm_enabled = true;
 	pi->uvd_dynamic_pg = false;
 
 	return 0;
+err:
+	cz_dpm_fini(adev);
+	return ret;
 }
 
 static void cz_dpm_fini(struct amdgpu_device *adev)
@@ -668,17 +678,12 @@ static void cz_reset_ap_mask(struct amdgpu_device *adev)
 	struct cz_power_info *pi = cz_get_pi(adev);
 
 	pi->active_process_mask = 0;
-
 }
 
 static int cz_dpm_download_pptable_from_smu(struct amdgpu_device *adev,
 							void **table)
 {
-	int ret = 0;
-
-	ret = cz_smu_download_pptable(adev, table);
-
-	return ret;
+	return cz_smu_download_pptable(adev, table);
 }
 
 static int cz_dpm_upload_pptable_to_smu(struct amdgpu_device *adev)
@@ -818,9 +823,9 @@ static void cz_init_sclk_limit(struct amdgpu_device *adev)
 	pi->sclk_dpm.hard_min_clk = 0;
 	cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxSclkLevel);
 	level = cz_get_argument(adev);
-	if (level < table->count)
+	if (level < table->count) {
 		clock = table->entries[level].clk;
-	else {
+	} else {
 		DRM_ERROR("Invalid SLCK Voltage Dependency table entry.\n");
 		clock = table->entries[table->count - 1].clk;
 	}
@@ -846,9 +851,9 @@ static void cz_init_uvd_limit(struct amdgpu_device *adev)
 	pi->uvd_dpm.hard_min_clk = 0;
 	cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxUvdLevel);
 	level = cz_get_argument(adev);
-	if (level < table->count)
+	if (level < table->count) {
 		clock = table->entries[level].vclk;
-	else {
+	} else {
 		DRM_ERROR("Invalid UVD Voltage Dependency table entry.\n");
 		clock = table->entries[table->count - 1].vclk;
 	}
@@ -874,9 +879,9 @@ static void cz_init_vce_limit(struct amdgpu_device *adev)
 	pi->vce_dpm.hard_min_clk = table->entries[0].ecclk;
 	cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxEclkLevel);
 	level = cz_get_argument(adev);
-	if (level < table->count)
+	if (level < table->count) {
 		clock = table->entries[level].ecclk;
-	else {
+	} else {
 		/* future BIOS would fix this error */
 		DRM_ERROR("Invalid VCE Voltage Dependency table entry.\n");
 		clock = table->entries[table->count - 1].ecclk;
@@ -903,9 +908,9 @@ static void cz_init_acp_limit(struct amdgpu_device *adev)
 	pi->acp_dpm.hard_min_clk = 0;
 	cz_send_msg_to_smc(adev, PPSMC_MSG_GetMaxAclkLevel);
 	level = cz_get_argument(adev);
-	if (level < table->count)
+	if (level < table->count) {
 		clock = table->entries[level].clk;
-	else {
+	} else {
 		DRM_ERROR("Invalid ACP Voltage Dependency table entry.\n");
 		clock = table->entries[table->count - 1].clk;
 	}
@@ -930,7 +935,6 @@ static void cz_init_sclk_threshold(struct amdgpu_device *adev)
 	struct cz_power_info *pi = cz_get_pi(adev);
 
 	pi->low_sclk_interrupt_threshold = 0;
-
 }
 
 static void cz_dpm_setup_asic(struct amdgpu_device *adev)
@@ -1203,7 +1207,7 @@ static int cz_enable_didt(struct amdgpu_device *adev, bool enable)
 	int ret;
 
 	if (pi->caps_sq_ramping || pi->caps_db_ramping ||
-			pi->caps_td_ramping || pi->caps_tcp_ramping) {
+	    pi->caps_td_ramping || pi->caps_tcp_ramping) {
 		if (adev->gfx.gfx_current_status != AMDGPU_GFX_SAFE_MODE) {
 			ret = cz_disable_cgpg(adev);
 			if (ret) {
@@ -1277,7 +1281,7 @@ static void cz_apply_state_adjust_rules(struct amdgpu_device *adev,
 	ps->force_high = false;
 	ps->need_dfs_bypass = true;
 	pi->video_start = new_rps->dclk || new_rps->vclk ||
-				new_rps->evclk || new_rps->ecclk;
+			  new_rps->evclk || new_rps->ecclk;
 
 	if ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
 			ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)
@@ -1335,7 +1339,6 @@ static int cz_dpm_enable(struct amdgpu_device *adev)
 	}
 
 	cz_reset_acp_boot_level(adev);
-
 	cz_update_current_ps(adev, adev->pm.dpm.boot_ps);
 
 	return 0;
@@ -1665,7 +1668,6 @@ static void cz_dpm_post_set_power_state(struct amdgpu_device *adev)
 	struct amdgpu_ps *ps = &pi->requested_rps;
 
 	cz_update_current_ps(adev, ps);
-
 }
 
 static int cz_dpm_force_highest(struct amdgpu_device *adev)
@@ -2108,29 +2110,58 @@ static void cz_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
 			/* disable clockgating so we can properly shut down the block */
 			ret = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
 							    AMD_CG_STATE_UNGATE);
+			if (ret) {
+				DRM_ERROR("UVD DPM Power Gating failed to set clockgating state\n");
+				return;
+			}
+
 			/* shutdown the UVD block */
 			ret = amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
 							    AMD_PG_STATE_GATE);
-			/* XXX: check for errors */
+
+			if (ret) {
+				DRM_ERROR("UVD DPM Power Gating failed to set powergating state\n");
+				return;
+			}
 		}
 		cz_update_uvd_dpm(adev, gate);
-		if (pi->caps_uvd_pg)
+		if (pi->caps_uvd_pg) {
 			/* power off the UVD block */
-			cz_send_msg_to_smc(adev, PPSMC_MSG_UVDPowerOFF);
+			ret = cz_send_msg_to_smc(adev, PPSMC_MSG_UVDPowerOFF);
+			if (ret) {
+				DRM_ERROR("UVD DPM Power Gating failed to send SMU PowerOFF message\n");
+				return;
+			}
+		}
 	} else {
 		if (pi->caps_uvd_pg) {
 			/* power on the UVD block */
 			if (pi->uvd_dynamic_pg)
-				cz_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_UVDPowerON, 1);
+				ret = cz_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_UVDPowerON, 1);
 			else
-				cz_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_UVDPowerON, 0);
+				ret = cz_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_UVDPowerON, 0);
+
+			if (ret) {
+				DRM_ERROR("UVD DPM Power Gating Failed to send SMU PowerON message\n");
+				return;
+			}
+
 			/* re-init the UVD block */
 			ret = amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
 							    AMD_PG_STATE_UNGATE);
+
+			if (ret) {
+				DRM_ERROR("UVD DPM Power Gating Failed to set powergating state\n");
+				return;
+			}
+
 			/* enable clockgating. hw will dynamically gate/ungate clocks on the fly */
 			ret = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
 							    AMD_CG_STATE_GATE);
-			/* XXX: check for errors */
+			if (ret) {
+				DRM_ERROR("UVD DPM Power Gating Failed to set clockgating state\n");
+				return;
+			}
 		}
 		cz_update_uvd_dpm(adev, gate);
 	}
@@ -2168,7 +2199,6 @@ static int cz_update_vce_dpm(struct amdgpu_device *adev)
 	/* Stable Pstate is enabled and we need to set the VCE DPM to highest level */
 	if (pi->caps_stable_power_state) {
 		pi->vce_dpm.hard_min_clk = table->entries[table->count-1].ecclk;
-
 	} else { /* non-stable p-state cases. without vce.Arbiter.EcclkHardMin */
 		/* leave it as set by user */
 		/*pi->vce_dpm.hard_min_clk = table->entries[0].ecclk;*/
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_smc.c b/drivers/gpu/drm/amd/amdgpu/cz_smc.c
index ac7fee7b7eca..aed7033c0973 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_smc.c
@@ -29,6 +29,8 @@
 #include "cz_smumgr.h"
 #include "smu_ucode_xfer_cz.h"
 #include "amdgpu_ucode.h"
+#include "cz_dpm.h"
+#include "vi_dpm.h"
 
 #include "smu/smu_8_0_d.h"
 #include "smu/smu_8_0_sh_mask.h"
@@ -48,7 +50,7 @@ static struct cz_smu_private_data *cz_smu_get_priv(struct amdgpu_device *adev)
 	return priv;
 }
 
-int cz_send_msg_to_smc_async(struct amdgpu_device *adev, u16 msg)
+static int cz_send_msg_to_smc_async(struct amdgpu_device *adev, u16 msg)
 {
 	int i;
 	u32 content = 0, tmp;
@@ -99,13 +101,6 @@ int cz_send_msg_to_smc(struct amdgpu_device *adev, u16 msg)
 	return 0;
 }
 
-int cz_send_msg_to_smc_with_parameter_async(struct amdgpu_device *adev,
-						u16 msg, u32 parameter)
-{
-	WREG32(mmSMU_MP1_SRBM2P_ARG_0, parameter);
-	return cz_send_msg_to_smc_async(adev, msg);
-}
-
 int cz_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
 						u16 msg, u32 parameter)
 {
@@ -140,7 +135,7 @@ int cz_read_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
 	return 0;
 }
 
-int cz_write_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
+static int cz_write_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
 						u32 value, u32 limit)
 {
 	int ret;
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index c1b04e9aab57..613ebb7ed50f 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -221,7 +221,7 @@ static bool dce_v10_0_is_counter_moving(struct amdgpu_device *adev, int crtc)
  */
 static void dce_v10_0_vblank_wait(struct amdgpu_device *adev, int crtc)
 {
-	unsigned i = 0;
+	unsigned i = 100;
 
 	if (crtc >= adev->mode_info.num_crtc)
 		return;
@@ -233,14 +233,16 @@ static void dce_v10_0_vblank_wait(struct amdgpu_device *adev, int crtc)
 	 * wait for another frame.
 	 */
 	while (dce_v10_0_is_in_vblank(adev, crtc)) {
-		if (i++ % 100 == 0) {
+		if (i++ == 100) {
+			i = 0;
 			if (!dce_v10_0_is_counter_moving(adev, crtc))
 				break;
 		}
 	}
 
 	while (!dce_v10_0_is_in_vblank(adev, crtc)) {
-		if (i++ % 100 == 0) {
+		if (i++ == 100) {
+			i = 0;
 			if (!dce_v10_0_is_counter_moving(adev, crtc))
 				break;
 		}
@@ -425,16 +427,6 @@ static void dce_v10_0_hpd_init(struct amdgpu_device *adev)
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
-		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
-			/* don't try to enable hpd on eDP or LVDS avoid breaking the
-			 * aux dp channel on imac and help (but not completely fix)
-			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
-			 * also avoid interrupt storms during dpms.
-			 */
-			continue;
-		}
-
 		switch (amdgpu_connector->hpd.hpd) {
 		case AMDGPU_HPD_1:
 			idx = 0;
@@ -458,6 +450,19 @@ static void dce_v10_0_hpd_init(struct amdgpu_device *adev)
 			continue;
 		}
 
+		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
+		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
+			/* don't try to enable hpd on eDP or LVDS avoid breaking the
+			 * aux dp channel on imac and help (but not completely fix)
+			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
+			 * also avoid interrupt storms during dpms.
+			 */
+			tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx]);
+			tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_EN, 0);
+			WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx], tmp);
+			continue;
+		}
+
 		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[idx]);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_CONTROL, DC_HPD_EN, 1);
 		WREG32(mmDC_HPD_CONTROL + hpd_offsets[idx], tmp);
@@ -646,8 +651,8 @@ static void dce_v10_0_resume_mc_access(struct amdgpu_device *adev,
 
 		if (save->crtc_enabled[i]) {
 			tmp = RREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i]);
-			if (REG_GET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE) != 3) {
-				tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE, 3);
+			if (REG_GET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE) != 0) {
+				tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE, 0);
 				WREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i], tmp);
 			}
 			tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
@@ -712,6 +717,45 @@ static void dce_v10_0_set_vga_render_state(struct amdgpu_device *adev,
 	WREG32(mmVGA_RENDER_CONTROL, tmp);
 }
 
+static int dce_v10_0_get_num_crtc(struct amdgpu_device *adev)
+{
+	int num_crtc = 0;
+
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+	case CHIP_TONGA:
+		num_crtc = 6;
+		break;
+	default:
+		num_crtc = 0;
+	}
+	return num_crtc;
+}
+
+void dce_v10_0_disable_dce(struct amdgpu_device *adev)
+{
+	/*Disable VGA render and enabled crtc, if has DCE engine*/
+	if (amdgpu_atombios_has_dce_engine_info(adev)) {
+		u32 tmp;
+		int crtc_enabled, i;
+
+		dce_v10_0_set_vga_render_state(adev, false);
+
+		/*Disable crtc*/
+		for (i = 0; i < dce_v10_0_get_num_crtc(adev); i++) {
+			crtc_enabled = REG_GET_FIELD(RREG32(mmCRTC_CONTROL + crtc_offsets[i]),
+									 CRTC_CONTROL, CRTC_MASTER_EN);
+			if (crtc_enabled) {
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
+				tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
+				tmp = REG_SET_FIELD(tmp, CRTC_CONTROL, CRTC_MASTER_EN, 0);
+				WREG32(mmCRTC_CONTROL + crtc_offsets[i], tmp);
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
+			}
+		}
+	}
+}
+
 static void dce_v10_0_program_fmt(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
@@ -2063,7 +2107,7 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	struct amdgpu_framebuffer *amdgpu_fb;
 	struct drm_framebuffer *target_fb;
 	struct drm_gem_object *obj;
-	struct amdgpu_bo *rbo;
+	struct amdgpu_bo *abo;
 	uint64_t fb_location, tiling_flags;
 	uint32_t fb_format, fb_pitch_pixels;
 	u32 fb_swap = REG_SET_FIELD(0, GRPH_SWAP_CNTL, GRPH_ENDIAN_SWAP, ENDIAN_NONE);
@@ -2071,6 +2115,7 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 tmp, viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
+	char *format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -2090,23 +2135,23 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	 * just update base pointers
 	 */
 	obj = amdgpu_fb->obj;
-	rbo = gem_to_amdgpu_bo(obj);
-	r = amdgpu_bo_reserve(rbo, false);
+	abo = gem_to_amdgpu_bo(obj);
+	r = amdgpu_bo_reserve(abo, false);
 	if (unlikely(r != 0))
 		return r;
 
 	if (atomic) {
-		fb_location = amdgpu_bo_gpu_offset(rbo);
+		fb_location = amdgpu_bo_gpu_offset(abo);
 	} else {
-		r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
+		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
 		if (unlikely(r != 0)) {
-			amdgpu_bo_unreserve(rbo);
+			amdgpu_bo_unreserve(abo);
 			return -EINVAL;
 		}
 	}
 
-	amdgpu_bo_get_tiling_flags(rbo, &tiling_flags);
-	amdgpu_bo_unreserve(rbo);
+	amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
+	amdgpu_bo_unreserve(abo);
 
 	pipe_config = AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
 
@@ -2182,8 +2227,9 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		DRM_ERROR("Unsupported screen format %s\n",
-			drm_get_format_name(target_fb->pixel_format));
+		format_name = drm_get_format_name(target_fb->pixel_format);
+		DRM_ERROR("Unsupported screen format %s\n", format_name);
+		kfree(format_name);
 		return -EINVAL;
 	}
 
@@ -2275,17 +2321,17 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(mmVIEWPORT_SIZE + amdgpu_crtc->crtc_offset,
 	       (viewport_w << 16) | viewport_h);
 
-	/* set pageflip to happen only at start of vblank interval (front porch) */
-	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 3);
+	/* set pageflip to happen anywhere in vblank interval */
+	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
 		amdgpu_fb = to_amdgpu_framebuffer(fb);
-		rbo = gem_to_amdgpu_bo(amdgpu_fb->obj);
-		r = amdgpu_bo_reserve(rbo, false);
+		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		r = amdgpu_bo_reserve(abo, false);
 		if (unlikely(r != 0))
 			return r;
-		amdgpu_bo_unpin(rbo);
-		amdgpu_bo_unreserve(rbo);
+		amdgpu_bo_unpin(abo);
+		amdgpu_bo_unreserve(abo);
 	}
 
 	/* Bytes per pixel may have changed */
@@ -2698,7 +2744,7 @@ static const struct drm_crtc_funcs dce_v10_0_crtc_funcs = {
 	.gamma_set = dce_v10_0_crtc_gamma_set,
 	.set_config = amdgpu_crtc_set_config,
 	.destroy = dce_v10_0_crtc_destroy,
-	.page_flip = amdgpu_crtc_page_flip,
+	.page_flip_target = amdgpu_crtc_page_flip_target,
 };
 
 static void dce_v10_0_crtc_dpms(struct drm_crtc *crtc, int mode)
@@ -2765,16 +2811,16 @@ static void dce_v10_0_crtc_disable(struct drm_crtc *crtc)
 	if (crtc->primary->fb) {
 		int r;
 		struct amdgpu_framebuffer *amdgpu_fb;
-		struct amdgpu_bo *rbo;
+		struct amdgpu_bo *abo;
 
 		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
-		rbo = gem_to_amdgpu_bo(amdgpu_fb->obj);
-		r = amdgpu_bo_reserve(rbo, false);
+		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		r = amdgpu_bo_reserve(abo, false);
 		if (unlikely(r))
-			DRM_ERROR("failed to reserve rbo before unpin\n");
+			DRM_ERROR("failed to reserve abo before unpin\n");
 		else {
-			amdgpu_bo_unpin(rbo);
-			amdgpu_bo_unreserve(rbo);
+			amdgpu_bo_unpin(abo);
+			amdgpu_bo_unreserve(abo);
 		}
 	}
 	/* disable the GRPH */
@@ -2962,10 +3008,11 @@ static int dce_v10_0_early_init(void *handle)
 	dce_v10_0_set_display_funcs(adev);
 	dce_v10_0_set_irq_funcs(adev);
 
+	adev->mode_info.num_crtc = dce_v10_0_get_num_crtc(adev);
+
 	switch (adev->asic_type) {
 	case CHIP_FIJI:
 	case CHIP_TONGA:
-		adev->mode_info.num_crtc = 6; /* XXX 7??? */
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 7;
 		break;
@@ -3141,11 +3188,26 @@ static int dce_v10_0_wait_for_idle(void *handle)
 	return 0;
 }
 
+static int dce_v10_0_check_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (dce_v10_0_is_display_hung(adev))
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_DCE].hang = true;
+	else
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_DCE].hang = false;
+
+	return 0;
+}
+
 static int dce_v10_0_soft_reset(void *handle)
 {
 	u32 srbm_soft_reset = 0, tmp;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_DCE].hang)
+		return 0;
+
 	if (dce_v10_0_is_display_hung(adev))
 		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK;
 
@@ -3512,6 +3574,7 @@ const struct amd_ip_funcs dce_v10_0_ip_funcs = {
 	.resume = dce_v10_0_resume,
 	.is_idle = dce_v10_0_is_idle,
 	.wait_for_idle = dce_v10_0_wait_for_idle,
+	.check_soft_reset = dce_v10_0_check_soft_reset,
 	.soft_reset = dce_v10_0_soft_reset,
 	.set_clockgating_state = dce_v10_0_set_clockgating_state,
 	.set_powergating_state = dce_v10_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.h b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.h
index 1bfa48ddd8a6..e3dc04d293e4 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.h
@@ -26,4 +26,6 @@
 
 extern const struct amd_ip_funcs dce_v10_0_ip_funcs;
 
+void dce_v10_0_disable_dce(struct amdgpu_device *adev);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index d4bf133908b1..f264b8f17ad1 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -443,16 +443,6 @@ static void dce_v11_0_hpd_init(struct amdgpu_device *adev)
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
-		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
-			/* don't try to enable hpd on eDP or LVDS avoid breaking the
-			 * aux dp channel on imac and help (but not completely fix)
-			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
-			 * also avoid interrupt storms during dpms.
-			 */
-			continue;
-		}
-
 		switch (amdgpu_connector->hpd.hpd) {
 		case AMDGPU_HPD_1:
 			idx = 0;
@@ -476,6 +466,19 @@ static void dce_v11_0_hpd_init(struct amdgpu_device *adev)
 			continue;
 		}
 
+		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
+		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
+			/* don't try to enable hpd on eDP or LVDS avoid breaking the
+			 * aux dp channel on imac and help (but not completely fix)
+			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
+			 * also avoid interrupt storms during dpms.
+			 */
+			tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx]);
+			tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_EN, 0);
+			WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx], tmp);
+			continue;
+		}
+
 		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[idx]);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_CONTROL, DC_HPD_EN, 1);
 		WREG32(mmDC_HPD_CONTROL + hpd_offsets[idx], tmp);
@@ -673,6 +676,53 @@ static void dce_v11_0_set_vga_render_state(struct amdgpu_device *adev,
 	WREG32(mmVGA_RENDER_CONTROL, tmp);
 }
 
+static int dce_v11_0_get_num_crtc (struct amdgpu_device *adev)
+{
+	int num_crtc = 0;
+
+	switch (adev->asic_type) {
+	case CHIP_CARRIZO:
+		num_crtc = 3;
+		break;
+	case CHIP_STONEY:
+		num_crtc = 2;
+		break;
+	case CHIP_POLARIS10:
+		num_crtc = 6;
+		break;
+	case CHIP_POLARIS11:
+		num_crtc = 5;
+		break;
+	default:
+		num_crtc = 0;
+	}
+	return num_crtc;
+}
+
+void dce_v11_0_disable_dce(struct amdgpu_device *adev)
+{
+	/*Disable VGA render and enabled crtc, if has DCE engine*/
+	if (amdgpu_atombios_has_dce_engine_info(adev)) {
+		u32 tmp;
+		int crtc_enabled, i;
+
+		dce_v11_0_set_vga_render_state(adev, false);
+
+		/*Disable crtc*/
+		for (i = 0; i < dce_v11_0_get_num_crtc(adev); i++) {
+			crtc_enabled = REG_GET_FIELD(RREG32(mmCRTC_CONTROL + crtc_offsets[i]),
+									 CRTC_CONTROL, CRTC_MASTER_EN);
+			if (crtc_enabled) {
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
+				tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
+				tmp = REG_SET_FIELD(tmp, CRTC_CONTROL, CRTC_MASTER_EN, 0);
+				WREG32(mmCRTC_CONTROL + crtc_offsets[i], tmp);
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
+			}
+		}
+	}
+}
+
 static void dce_v11_0_program_fmt(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
@@ -2038,7 +2088,7 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 	struct amdgpu_framebuffer *amdgpu_fb;
 	struct drm_framebuffer *target_fb;
 	struct drm_gem_object *obj;
-	struct amdgpu_bo *rbo;
+	struct amdgpu_bo *abo;
 	uint64_t fb_location, tiling_flags;
 	uint32_t fb_format, fb_pitch_pixels;
 	u32 fb_swap = REG_SET_FIELD(0, GRPH_SWAP_CNTL, GRPH_ENDIAN_SWAP, ENDIAN_NONE);
@@ -2046,6 +2096,7 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 tmp, viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
+	char *format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -2065,23 +2116,23 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 	 * just update base pointers
 	 */
 	obj = amdgpu_fb->obj;
-	rbo = gem_to_amdgpu_bo(obj);
-	r = amdgpu_bo_reserve(rbo, false);
+	abo = gem_to_amdgpu_bo(obj);
+	r = amdgpu_bo_reserve(abo, false);
 	if (unlikely(r != 0))
 		return r;
 
 	if (atomic) {
-		fb_location = amdgpu_bo_gpu_offset(rbo);
+		fb_location = amdgpu_bo_gpu_offset(abo);
 	} else {
-		r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
+		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
 		if (unlikely(r != 0)) {
-			amdgpu_bo_unreserve(rbo);
+			amdgpu_bo_unreserve(abo);
 			return -EINVAL;
 		}
 	}
 
-	amdgpu_bo_get_tiling_flags(rbo, &tiling_flags);
-	amdgpu_bo_unreserve(rbo);
+	amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
+	amdgpu_bo_unreserve(abo);
 
 	pipe_config = AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
 
@@ -2157,8 +2208,9 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		DRM_ERROR("Unsupported screen format %s\n",
-			drm_get_format_name(target_fb->pixel_format));
+		format_name = drm_get_format_name(target_fb->pixel_format);
+		DRM_ERROR("Unsupported screen format %s\n", format_name);
+		kfree(format_name);
 		return -EINVAL;
 	}
 
@@ -2250,17 +2302,17 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(mmVIEWPORT_SIZE + amdgpu_crtc->crtc_offset,
 	       (viewport_w << 16) | viewport_h);
 
-	/* set pageflip to happen only at start of vblank interval (front porch) */
-	WREG32(mmCRTC_MASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 3);
+	/* set pageflip to happen anywhere in vblank interval */
+	WREG32(mmCRTC_MASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
 		amdgpu_fb = to_amdgpu_framebuffer(fb);
-		rbo = gem_to_amdgpu_bo(amdgpu_fb->obj);
-		r = amdgpu_bo_reserve(rbo, false);
+		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		r = amdgpu_bo_reserve(abo, false);
 		if (unlikely(r != 0))
 			return r;
-		amdgpu_bo_unpin(rbo);
-		amdgpu_bo_unreserve(rbo);
+		amdgpu_bo_unpin(abo);
+		amdgpu_bo_unreserve(abo);
 	}
 
 	/* Bytes per pixel may have changed */
@@ -2708,7 +2760,7 @@ static const struct drm_crtc_funcs dce_v11_0_crtc_funcs = {
 	.gamma_set = dce_v11_0_crtc_gamma_set,
 	.set_config = amdgpu_crtc_set_config,
 	.destroy = dce_v11_0_crtc_destroy,
-	.page_flip = amdgpu_crtc_page_flip,
+	.page_flip_target = amdgpu_crtc_page_flip_target,
 };
 
 static void dce_v11_0_crtc_dpms(struct drm_crtc *crtc, int mode)
@@ -2775,16 +2827,16 @@ static void dce_v11_0_crtc_disable(struct drm_crtc *crtc)
 	if (crtc->primary->fb) {
 		int r;
 		struct amdgpu_framebuffer *amdgpu_fb;
-		struct amdgpu_bo *rbo;
+		struct amdgpu_bo *abo;
 
 		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
-		rbo = gem_to_amdgpu_bo(amdgpu_fb->obj);
-		r = amdgpu_bo_reserve(rbo, false);
+		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		r = amdgpu_bo_reserve(abo, false);
 		if (unlikely(r))
-			DRM_ERROR("failed to reserve rbo before unpin\n");
+			DRM_ERROR("failed to reserve abo before unpin\n");
 		else {
-			amdgpu_bo_unpin(rbo);
-			amdgpu_bo_unreserve(rbo);
+			amdgpu_bo_unpin(abo);
+			amdgpu_bo_unreserve(abo);
 		}
 	}
 	/* disable the GRPH */
@@ -2999,24 +3051,22 @@ static int dce_v11_0_early_init(void *handle)
 	dce_v11_0_set_display_funcs(adev);
 	dce_v11_0_set_irq_funcs(adev);
 
+	adev->mode_info.num_crtc = dce_v11_0_get_num_crtc(adev);
+
 	switch (adev->asic_type) {
 	case CHIP_CARRIZO:
-		adev->mode_info.num_crtc = 3;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 9;
 		break;
 	case CHIP_STONEY:
-		adev->mode_info.num_crtc = 2;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 9;
 		break;
 	case CHIP_POLARIS10:
-		adev->mode_info.num_crtc = 6;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
 		break;
 	case CHIP_POLARIS11:
-		adev->mode_info.num_crtc = 5;
 		adev->mode_info.num_hpd = 5;
 		adev->mode_info.num_dig = 5;
 		break;
@@ -3109,6 +3159,7 @@ static int dce_v11_0_sw_fini(void *handle)
 
 	dce_v11_0_afmt_fini(adev);
 
+	drm_mode_config_cleanup(adev->ddev);
 	adev->mode_info.mode_config_initialized = false;
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.h b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.h
index 84e4618f5253..1f58a65ba2ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.h
@@ -26,4 +26,6 @@
 
 extern const struct amd_ip_funcs dce_v11_0_ip_funcs;
 
+void dce_v11_0_disable_dce(struct amdgpu_device *adev);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
new file mode 100644
index 000000000000..b948d6cb1399
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -0,0 +1,3176 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "drmP.h"
+#include "amdgpu.h"
+#include "amdgpu_pm.h"
+#include "amdgpu_i2c.h"
+#include "atom.h"
+#include "amdgpu_atombios.h"
+#include "atombios_crtc.h"
+#include "atombios_encoders.h"
+#include "amdgpu_pll.h"
+#include "amdgpu_connectors.h"
+#include "si/si_reg.h"
+#include "si/sid.h"
+
+static void dce_v6_0_set_display_funcs(struct amdgpu_device *adev);
+static void dce_v6_0_set_irq_funcs(struct amdgpu_device *adev);
+
+static const u32 crtc_offsets[6] =
+{
+	SI_CRTC0_REGISTER_OFFSET,
+	SI_CRTC1_REGISTER_OFFSET,
+	SI_CRTC2_REGISTER_OFFSET,
+	SI_CRTC3_REGISTER_OFFSET,
+	SI_CRTC4_REGISTER_OFFSET,
+	SI_CRTC5_REGISTER_OFFSET
+};
+
+static const uint32_t dig_offsets[] = {
+	SI_CRTC0_REGISTER_OFFSET,
+	SI_CRTC1_REGISTER_OFFSET,
+	SI_CRTC2_REGISTER_OFFSET,
+	SI_CRTC3_REGISTER_OFFSET,
+	SI_CRTC4_REGISTER_OFFSET,
+	SI_CRTC5_REGISTER_OFFSET,
+	(0x13830 - 0x7030) >> 2,
+};
+
+static const struct {
+	uint32_t	reg;
+	uint32_t	vblank;
+	uint32_t	vline;
+	uint32_t	hpd;
+
+} interrupt_status_offsets[6] = { {
+	.reg = DISP_INTERRUPT_STATUS,
+	.vblank = DISP_INTERRUPT_STATUS__LB_D1_VBLANK_INTERRUPT_MASK,
+	.vline = DISP_INTERRUPT_STATUS__LB_D1_VLINE_INTERRUPT_MASK,
+	.hpd = DISP_INTERRUPT_STATUS__DC_HPD1_INTERRUPT_MASK
+}, {
+	.reg = DISP_INTERRUPT_STATUS_CONTINUE,
+	.vblank = DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VBLANK_INTERRUPT_MASK,
+	.vline = DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VLINE_INTERRUPT_MASK,
+	.hpd = DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_INTERRUPT_MASK
+}, {
+	.reg = DISP_INTERRUPT_STATUS_CONTINUE2,
+	.vblank = DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VBLANK_INTERRUPT_MASK,
+	.vline = DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VLINE_INTERRUPT_MASK,
+	.hpd = DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_INTERRUPT_MASK
+}, {
+	.reg = DISP_INTERRUPT_STATUS_CONTINUE3,
+	.vblank = DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VBLANK_INTERRUPT_MASK,
+	.vline = DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VLINE_INTERRUPT_MASK,
+	.hpd = DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_INTERRUPT_MASK
+}, {
+	.reg = DISP_INTERRUPT_STATUS_CONTINUE4,
+	.vblank = DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VBLANK_INTERRUPT_MASK,
+	.vline = DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VLINE_INTERRUPT_MASK,
+	.hpd = DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_INTERRUPT_MASK
+}, {
+	.reg = DISP_INTERRUPT_STATUS_CONTINUE5,
+	.vblank = DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VBLANK_INTERRUPT_MASK,
+	.vline = DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VLINE_INTERRUPT_MASK,
+	.hpd = DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_INTERRUPT_MASK
+} };
+
+static const uint32_t hpd_int_control_offsets[6] = {
+	DC_HPD1_INT_CONTROL,
+	DC_HPD2_INT_CONTROL,
+	DC_HPD3_INT_CONTROL,
+	DC_HPD4_INT_CONTROL,
+	DC_HPD5_INT_CONTROL,
+	DC_HPD6_INT_CONTROL,
+};
+
+static u32 dce_v6_0_audio_endpt_rreg(struct amdgpu_device *adev,
+				     u32 block_offset, u32 reg)
+{
+	DRM_INFO("xxxx: dce_v6_0_audio_endpt_rreg ----no impl!!!!\n");
+	return 0;
+}
+
+static void dce_v6_0_audio_endpt_wreg(struct amdgpu_device *adev,
+				      u32 block_offset, u32 reg, u32 v)
+{
+	DRM_INFO("xxxx: dce_v6_0_audio_endpt_wreg ----no impl!!!!\n");
+}
+
+static bool dce_v6_0_is_in_vblank(struct amdgpu_device *adev, int crtc)
+{
+	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
+		return true;
+	else
+		return false;
+}
+
+static bool dce_v6_0_is_counter_moving(struct amdgpu_device *adev, int crtc)
+{
+	u32 pos1, pos2;
+
+	pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
+	pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
+
+	if (pos1 != pos2)
+		return true;
+	else
+		return false;
+}
+
+/**
+ * dce_v6_0_wait_for_vblank - vblank wait asic callback.
+ *
+ * @crtc: crtc to wait for vblank on
+ *
+ * Wait for vblank on the requested crtc (evergreen+).
+ */
+static void dce_v6_0_vblank_wait(struct amdgpu_device *adev, int crtc)
+{
+	unsigned i = 100;
+
+	if (crtc >= adev->mode_info.num_crtc)
+		return;
+
+	if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
+		return;
+
+	/* depending on when we hit vblank, we may be close to active; if so,
+	 * wait for another frame.
+	 */
+	while (dce_v6_0_is_in_vblank(adev, crtc)) {
+		if (i++ == 100) {
+			i = 0;
+			if (!dce_v6_0_is_counter_moving(adev, crtc))
+				break;
+		}
+	}
+
+	while (!dce_v6_0_is_in_vblank(adev, crtc)) {
+		if (i++ == 100) {
+			i = 0;
+			if (!dce_v6_0_is_counter_moving(adev, crtc))
+				break;
+		}
+	}
+}
+
+static u32 dce_v6_0_vblank_get_counter(struct amdgpu_device *adev, int crtc)
+{
+	if (crtc >= adev->mode_info.num_crtc)
+		return 0;
+	else
+		return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
+}
+
+static void dce_v6_0_pageflip_interrupt_init(struct amdgpu_device *adev)
+{
+	unsigned i;
+
+	/* Enable pflip interrupts */
+	for (i = 0; i < adev->mode_info.num_crtc; i++)
+		amdgpu_irq_get(adev, &adev->pageflip_irq, i);
+}
+
+static void dce_v6_0_pageflip_interrupt_fini(struct amdgpu_device *adev)
+{
+	unsigned i;
+
+	/* Disable pflip interrupts */
+	for (i = 0; i < adev->mode_info.num_crtc; i++)
+		amdgpu_irq_put(adev, &adev->pageflip_irq, i);
+}
+
+/**
+ * dce_v6_0_page_flip - pageflip callback.
+ *
+ * @adev: amdgpu_device pointer
+ * @crtc_id: crtc to cleanup pageflip on
+ * @crtc_base: new address of the crtc (GPU MC address)
+ *
+ * Does the actual pageflip (evergreen+).
+ * During vblank we take the crtc lock and wait for the update_pending
+ * bit to go high, when it does, we release the lock, and allow the
+ * double buffered update to take place.
+ * Returns the current update pending status.
+ */
+static void dce_v6_0_page_flip(struct amdgpu_device *adev,
+			       int crtc_id, u64 crtc_base, bool async)
+{
+	struct amdgpu_crtc *amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
+
+	/* flip at hsync for async, default is vsync */
+	WREG32(EVERGREEN_GRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset, async ?
+	       EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
+	/* update the scanout addresses */
+	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+	       upper_32_bits(crtc_base));
+	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+	       (u32)crtc_base);
+
+	/* post the write */
+	RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset);
+}
+
+static int dce_v6_0_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
+					u32 *vbl, u32 *position)
+{
+	if ((crtc < 0) || (crtc >= adev->mode_info.num_crtc))
+		return -EINVAL;
+	*vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + crtc_offsets[crtc]);
+	*position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
+
+	return 0;
+
+}
+
+/**
+ * dce_v6_0_hpd_sense - hpd sense callback.
+ *
+ * @adev: amdgpu_device pointer
+ * @hpd: hpd (hotplug detect) pin
+ *
+ * Checks if a digital monitor is connected (evergreen+).
+ * Returns true if connected, false if not connected.
+ */
+static bool dce_v6_0_hpd_sense(struct amdgpu_device *adev,
+			       enum amdgpu_hpd_id hpd)
+{
+	bool connected = false;
+
+	switch (hpd) {
+	case AMDGPU_HPD_1:
+		if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
+			connected = true;
+		break;
+	case AMDGPU_HPD_2:
+		if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
+			connected = true;
+		break;
+	case AMDGPU_HPD_3:
+		if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
+			connected = true;
+		break;
+	case AMDGPU_HPD_4:
+		if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
+			connected = true;
+		break;
+	case AMDGPU_HPD_5:
+		if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
+			connected = true;
+		break;
+	case AMDGPU_HPD_6:
+		if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
+			connected = true;
+		break;
+	default:
+		break;
+	}
+
+	return connected;
+}
+
+/**
+ * dce_v6_0_hpd_set_polarity - hpd set polarity callback.
+ *
+ * @adev: amdgpu_device pointer
+ * @hpd: hpd (hotplug detect) pin
+ *
+ * Set the polarity of the hpd pin (evergreen+).
+ */
+static void dce_v6_0_hpd_set_polarity(struct amdgpu_device *adev,
+				      enum amdgpu_hpd_id hpd)
+{
+	u32 tmp;
+	bool connected = dce_v6_0_hpd_sense(adev, hpd);
+
+	switch (hpd) {
+	case AMDGPU_HPD_1:
+		tmp = RREG32(DC_HPD1_INT_CONTROL);
+		if (connected)
+			tmp &= ~DC_HPDx_INT_POLARITY;
+		else
+			tmp |= DC_HPDx_INT_POLARITY;
+		WREG32(DC_HPD1_INT_CONTROL, tmp);
+		break;
+	case AMDGPU_HPD_2:
+		tmp = RREG32(DC_HPD2_INT_CONTROL);
+		if (connected)
+			tmp &= ~DC_HPDx_INT_POLARITY;
+		else
+			tmp |= DC_HPDx_INT_POLARITY;
+		WREG32(DC_HPD2_INT_CONTROL, tmp);
+		break;
+	case AMDGPU_HPD_3:
+		tmp = RREG32(DC_HPD3_INT_CONTROL);
+		if (connected)
+			tmp &= ~DC_HPDx_INT_POLARITY;
+		else
+			tmp |= DC_HPDx_INT_POLARITY;
+		WREG32(DC_HPD3_INT_CONTROL, tmp);
+		break;
+	case AMDGPU_HPD_4:
+		tmp = RREG32(DC_HPD4_INT_CONTROL);
+		if (connected)
+			tmp &= ~DC_HPDx_INT_POLARITY;
+		else
+			tmp |= DC_HPDx_INT_POLARITY;
+		WREG32(DC_HPD4_INT_CONTROL, tmp);
+		break;
+	case AMDGPU_HPD_5:
+		tmp = RREG32(DC_HPD5_INT_CONTROL);
+		if (connected)
+			tmp &= ~DC_HPDx_INT_POLARITY;
+		else
+			tmp |= DC_HPDx_INT_POLARITY;
+		WREG32(DC_HPD5_INT_CONTROL, tmp);
+			break;
+	case AMDGPU_HPD_6:
+		tmp = RREG32(DC_HPD6_INT_CONTROL);
+		if (connected)
+			tmp &= ~DC_HPDx_INT_POLARITY;
+		else
+			tmp |= DC_HPDx_INT_POLARITY;
+		WREG32(DC_HPD6_INT_CONTROL, tmp);
+		break;
+	default:
+		break;
+	}
+}
+
+/**
+ * dce_v6_0_hpd_init - hpd setup callback.
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Setup the hpd pins used by the card (evergreen+).
+ * Enable the pin, set the polarity, and enable the hpd interrupts.
+ */
+static void dce_v6_0_hpd_init(struct amdgpu_device *adev)
+{
+	struct drm_device *dev = adev->ddev;
+	struct drm_connector *connector;
+	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
+		DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
+
+	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
+
+		switch (amdgpu_connector->hpd.hpd) {
+		case AMDGPU_HPD_1:
+			WREG32(DC_HPD1_CONTROL, tmp);
+			break;
+		case AMDGPU_HPD_2:
+			WREG32(DC_HPD2_CONTROL, tmp);
+			break;
+		case AMDGPU_HPD_3:
+			WREG32(DC_HPD3_CONTROL, tmp);
+			break;
+		case AMDGPU_HPD_4:
+			WREG32(DC_HPD4_CONTROL, tmp);
+			break;
+		case AMDGPU_HPD_5:
+			WREG32(DC_HPD5_CONTROL, tmp);
+			break;
+		case AMDGPU_HPD_6:
+			WREG32(DC_HPD6_CONTROL, tmp);
+			break;
+		default:
+			break;
+		}
+
+		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
+		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
+			/* don't try to enable hpd on eDP or LVDS avoid breaking the
+			 * aux dp channel on imac and help (but not completely fix)
+			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
+			 * also avoid interrupt storms during dpms.
+			 */
+			u32 dc_hpd_int_cntl_reg, dc_hpd_int_cntl;
+
+			switch (amdgpu_connector->hpd.hpd) {
+			case AMDGPU_HPD_1:
+				dc_hpd_int_cntl_reg = DC_HPD1_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_2:
+				dc_hpd_int_cntl_reg = DC_HPD2_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_3:
+				dc_hpd_int_cntl_reg = DC_HPD3_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_4:
+				dc_hpd_int_cntl_reg = DC_HPD4_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_5:
+				dc_hpd_int_cntl_reg = DC_HPD5_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_6:
+				dc_hpd_int_cntl_reg = DC_HPD6_INT_CONTROL;
+				break;
+			default:
+				continue;
+			}
+
+			dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
+			dc_hpd_int_cntl &= ~DC_HPDx_INT_EN;
+			WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+			continue;
+		}
+
+		dce_v6_0_hpd_set_polarity(adev, amdgpu_connector->hpd.hpd);
+		amdgpu_irq_get(adev, &adev->hpd_irq, amdgpu_connector->hpd.hpd);
+	}
+
+}
+
+/**
+ * dce_v6_0_hpd_fini - hpd tear down callback.
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Tear down the hpd pins used by the card (evergreen+).
+ * Disable the hpd interrupts.
+ */
+static void dce_v6_0_hpd_fini(struct amdgpu_device *adev)
+{
+	struct drm_device *dev = adev->ddev;
+	struct drm_connector *connector;
+
+	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
+
+		switch (amdgpu_connector->hpd.hpd) {
+		case AMDGPU_HPD_1:
+			WREG32(DC_HPD1_CONTROL, 0);
+			break;
+		case AMDGPU_HPD_2:
+			WREG32(DC_HPD2_CONTROL, 0);
+			break;
+		case AMDGPU_HPD_3:
+			WREG32(DC_HPD3_CONTROL, 0);
+			break;
+		case AMDGPU_HPD_4:
+			WREG32(DC_HPD4_CONTROL, 0);
+			break;
+		case AMDGPU_HPD_5:
+			WREG32(DC_HPD5_CONTROL, 0);
+			break;
+		case AMDGPU_HPD_6:
+			WREG32(DC_HPD6_CONTROL, 0);
+			break;
+		default:
+			break;
+		}
+		amdgpu_irq_put(adev, &adev->hpd_irq, amdgpu_connector->hpd.hpd);
+	}
+}
+
+static u32 dce_v6_0_hpd_get_gpio_reg(struct amdgpu_device *adev)
+{
+	return SI_DC_GPIO_HPD_A;
+}
+
+static bool dce_v6_0_is_display_hung(struct amdgpu_device *adev)
+{
+	DRM_INFO("xxxx: dce_v6_0_is_display_hung ----no imp!!!!!\n");
+
+	return true;
+}
+
+static u32 evergreen_get_vblank_counter(struct amdgpu_device* adev, int crtc)
+{
+	if (crtc >= adev->mode_info.num_crtc)
+		return 0;
+	else
+		return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
+}
+
+static void dce_v6_0_stop_mc_access(struct amdgpu_device *adev,
+				    struct amdgpu_mode_mc_save *save)
+{
+	u32 crtc_enabled, tmp, frame_count;
+	int i, j;
+
+	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
+	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
+
+	/* disable VGA render */
+	WREG32(VGA_RENDER_CONTROL, 0);
+
+	/* blank the display controllers */
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
+		if (crtc_enabled) {
+			save->crtc_enabled[i] = true;
+			tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
+
+			if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
+				dce_v6_0_vblank_wait(adev, i);
+				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
+				tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
+				WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
+				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
+			}
+			/* wait for the next frame */
+			frame_count = evergreen_get_vblank_counter(adev, i);
+			for (j = 0; j < adev->usec_timeout; j++) {
+				if (evergreen_get_vblank_counter(adev, i) != frame_count)
+					break;
+				udelay(1);
+			}
+
+			/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
+			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
+			tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
+			tmp &= ~EVERGREEN_CRTC_MASTER_EN;
+			WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
+			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
+			save->crtc_enabled[i] = false;
+			/* ***** */
+		} else {
+			save->crtc_enabled[i] = false;
+		}
+	}
+}
+
+static void dce_v6_0_resume_mc_access(struct amdgpu_device *adev,
+				      struct amdgpu_mode_mc_save *save)
+{
+	u32 tmp;
+	int i, j;
+
+	/* update crtc base addresses */
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
+		       upper_32_bits(adev->mc.vram_start));
+		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
+		       upper_32_bits(adev->mc.vram_start));
+		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
+		       (u32)adev->mc.vram_start);
+		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
+		       (u32)adev->mc.vram_start);
+	}
+
+	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(adev->mc.vram_start));
+	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)adev->mc.vram_start);
+
+	/* unlock regs and wait for update */
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		if (save->crtc_enabled[i]) {
+			tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
+			if ((tmp & 0x7) != 3) {
+				tmp &= ~0x7;
+				tmp |= 0x3;
+				WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
+			}
+			tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
+			if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
+				tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
+				WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
+			}
+			tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
+			if (tmp & 1) {
+				tmp &= ~1;
+				WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
+			}
+			for (j = 0; j < adev->usec_timeout; j++) {
+				tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
+				if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
+					break;
+				udelay(1);
+			}
+		}
+	}
+
+	/* Unlock vga access */
+	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
+	mdelay(1);
+	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
+
+}
+
+static void dce_v6_0_set_vga_render_state(struct amdgpu_device *adev,
+					  bool render)
+{
+	if (!render) 
+		WREG32(R_000300_VGA_RENDER_CONTROL,
+			RREG32(R_000300_VGA_RENDER_CONTROL) & C_000300_VGA_VSTATUS_CNTL);
+
+}
+
+static void dce_v6_0_program_fmt(struct drm_encoder *encoder)
+{
+
+	struct drm_device *dev = encoder->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+	struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder);
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(encoder->crtc);
+	int bpc = 0;
+	u32 tmp = 0;
+	enum amdgpu_connector_dither dither = AMDGPU_FMT_DITHER_DISABLE;
+
+	if (connector) {
+		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
+		bpc = amdgpu_connector_get_monitor_bpc(connector);
+		dither = amdgpu_connector->dither;
+	}
+
+	/* LVDS FMT is set up by atom */
+	if (amdgpu_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
+		return;
+
+	if (bpc == 0)
+		return;
+
+
+	switch (bpc) {
+	case 6:
+		if (dither == AMDGPU_FMT_DITHER_ENABLE)
+			/* XXX sort out optimal dither settings */
+			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
+				FMT_SPATIAL_DITHER_EN);
+		else
+			tmp |= FMT_TRUNCATE_EN;
+		break;
+	case 8:
+		if (dither == AMDGPU_FMT_DITHER_ENABLE)
+			/* XXX sort out optimal dither settings */
+			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
+				FMT_RGB_RANDOM_ENABLE |
+				FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
+		else
+			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
+		break;
+	case 10:
+	default:
+		/* not needed */
+		break;
+	}
+
+	WREG32(FMT_BIT_DEPTH_CONTROL + amdgpu_crtc->crtc_offset, tmp);
+}
+
+/**
+ * cik_get_number_of_dram_channels - get the number of dram channels
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Look up the number of video ram channels (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the number of dram channels
+ */
+static u32 si_get_number_of_dram_channels(struct amdgpu_device *adev)
+{
+	u32 tmp = RREG32(MC_SHARED_CHMAP);
+
+	switch ((tmp & MC_SHARED_CHMAP__NOOFCHAN_MASK) >> MC_SHARED_CHMAP__NOOFCHAN__SHIFT) {
+	case 0:
+	default:
+		return 1;
+	case 1:
+		return 2;
+	case 2:
+		return 4;
+	case 3:
+		return 8;
+	case 4:
+		return 3;
+	case 5:
+		return 6;
+	case 6:
+		return 10;
+	case 7:
+		return 12;
+	case 8:
+		return 16;
+	}
+}
+
+struct dce6_wm_params {
+	u32 dram_channels; /* number of dram channels */
+	u32 yclk;          /* bandwidth per dram data pin in kHz */
+	u32 sclk;          /* engine clock in kHz */
+	u32 disp_clk;      /* display clock in kHz */
+	u32 src_width;     /* viewport width */
+	u32 active_time;   /* active display time in ns */
+	u32 blank_time;    /* blank time in ns */
+	bool interlaced;    /* mode is interlaced */
+	fixed20_12 vsc;    /* vertical scale ratio */
+	u32 num_heads;     /* number of active crtcs */
+	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
+	u32 lb_size;       /* line buffer allocated to pipe */
+	u32 vtaps;         /* vertical scaler taps */
+};
+
+/**
+ * dce_v6_0_dram_bandwidth - get the dram bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the raw dram bandwidth (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the dram bandwidth in MBytes/s
+ */
+static u32 dce_v6_0_dram_bandwidth(struct dce6_wm_params *wm)
+{
+	/* Calculate raw DRAM Bandwidth */
+	fixed20_12 dram_efficiency; /* 0.7 */
+	fixed20_12 yclk, dram_channels, bandwidth;
+	fixed20_12 a;
+
+	a.full = dfixed_const(1000);
+	yclk.full = dfixed_const(wm->yclk);
+	yclk.full = dfixed_div(yclk, a);
+	dram_channels.full = dfixed_const(wm->dram_channels * 4);
+	a.full = dfixed_const(10);
+	dram_efficiency.full = dfixed_const(7);
+	dram_efficiency.full = dfixed_div(dram_efficiency, a);
+	bandwidth.full = dfixed_mul(dram_channels, yclk);
+	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
+
+	return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce_v6_0_dram_bandwidth_for_display - get the dram bandwidth for display
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the dram bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the dram bandwidth for display in MBytes/s
+ */
+static u32 dce_v6_0_dram_bandwidth_for_display(struct dce6_wm_params *wm)
+{
+	/* Calculate DRAM Bandwidth and the part allocated to display. */
+	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
+	fixed20_12 yclk, dram_channels, bandwidth;
+	fixed20_12 a;
+
+	a.full = dfixed_const(1000);
+	yclk.full = dfixed_const(wm->yclk);
+	yclk.full = dfixed_div(yclk, a);
+	dram_channels.full = dfixed_const(wm->dram_channels * 4);
+	a.full = dfixed_const(10);
+	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
+	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
+	bandwidth.full = dfixed_mul(dram_channels, yclk);
+	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
+
+	return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce_v6_0_data_return_bandwidth - get the data return bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the data return bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the data return bandwidth in MBytes/s
+ */
+static u32 dce_v6_0_data_return_bandwidth(struct dce6_wm_params *wm)
+{
+	/* Calculate the display Data return Bandwidth */
+	fixed20_12 return_efficiency; /* 0.8 */
+	fixed20_12 sclk, bandwidth;
+	fixed20_12 a;
+
+	a.full = dfixed_const(1000);
+	sclk.full = dfixed_const(wm->sclk);
+	sclk.full = dfixed_div(sclk, a);
+	a.full = dfixed_const(10);
+	return_efficiency.full = dfixed_const(8);
+	return_efficiency.full = dfixed_div(return_efficiency, a);
+	a.full = dfixed_const(32);
+	bandwidth.full = dfixed_mul(a, sclk);
+	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
+
+	return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce_v6_0_dmif_request_bandwidth - get the dmif bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the dmif bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the dmif bandwidth in MBytes/s
+ */
+static u32 dce_v6_0_dmif_request_bandwidth(struct dce6_wm_params *wm)
+{
+	/* Calculate the DMIF Request Bandwidth */
+	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
+	fixed20_12 disp_clk, bandwidth;
+	fixed20_12 a, b;
+
+	a.full = dfixed_const(1000);
+	disp_clk.full = dfixed_const(wm->disp_clk);
+	disp_clk.full = dfixed_div(disp_clk, a);
+	a.full = dfixed_const(32);
+	b.full = dfixed_mul(a, disp_clk);
+
+	a.full = dfixed_const(10);
+	disp_clk_request_efficiency.full = dfixed_const(8);
+	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
+
+	bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
+
+	return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce_v6_0_available_bandwidth - get the min available bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the min available bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the min available bandwidth in MBytes/s
+ */
+static u32 dce_v6_0_available_bandwidth(struct dce6_wm_params *wm)
+{
+	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
+	u32 dram_bandwidth = dce_v6_0_dram_bandwidth(wm);
+	u32 data_return_bandwidth = dce_v6_0_data_return_bandwidth(wm);
+	u32 dmif_req_bandwidth = dce_v6_0_dmif_request_bandwidth(wm);
+
+	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
+}
+
+/**
+ * dce_v6_0_average_bandwidth - get the average available bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the average available bandwidth used for display (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the average available bandwidth in MBytes/s
+ */
+static u32 dce_v6_0_average_bandwidth(struct dce6_wm_params *wm)
+{
+	/* Calculate the display mode Average Bandwidth
+	 * DisplayMode should contain the source and destination dimensions,
+	 * timing, etc.
+	 */
+	fixed20_12 bpp;
+	fixed20_12 line_time;
+	fixed20_12 src_width;
+	fixed20_12 bandwidth;
+	fixed20_12 a;
+
+	a.full = dfixed_const(1000);
+	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
+	line_time.full = dfixed_div(line_time, a);
+	bpp.full = dfixed_const(wm->bytes_per_pixel);
+	src_width.full = dfixed_const(wm->src_width);
+	bandwidth.full = dfixed_mul(src_width, bpp);
+	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
+	bandwidth.full = dfixed_div(bandwidth, line_time);
+
+	return dfixed_trunc(bandwidth);
+}
+
+/**
+ * dce_v6_0_latency_watermark - get the latency watermark
+ *
+ * @wm: watermark calculation data
+ *
+ * Calculate the latency watermark (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns the latency watermark in ns
+ */
+static u32 dce_v6_0_latency_watermark(struct dce6_wm_params *wm)
+{
+	/* First calculate the latency in ns */
+	u32 mc_latency = 2000; /* 2000 ns. */
+	u32 available_bandwidth = dce_v6_0_available_bandwidth(wm);
+	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
+	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
+	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
+	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
+		(wm->num_heads * cursor_line_pair_return_time);
+	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
+	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
+	u32 tmp, dmif_size = 12288;
+	fixed20_12 a, b, c;
+
+	if (wm->num_heads == 0)
+		return 0;
+
+	a.full = dfixed_const(2);
+	b.full = dfixed_const(1);
+	if ((wm->vsc.full > a.full) ||
+	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
+	    (wm->vtaps >= 5) ||
+	    ((wm->vsc.full >= a.full) && wm->interlaced))
+		max_src_lines_per_dst_line = 4;
+	else
+		max_src_lines_per_dst_line = 2;
+
+	a.full = dfixed_const(available_bandwidth);
+	b.full = dfixed_const(wm->num_heads);
+	a.full = dfixed_div(a, b);
+
+	b.full = dfixed_const(mc_latency + 512);
+	c.full = dfixed_const(wm->disp_clk);
+	b.full = dfixed_div(b, c);
+
+	c.full = dfixed_const(dmif_size);
+	b.full = dfixed_div(c, b);
+
+	tmp = min(dfixed_trunc(a), dfixed_trunc(b));
+
+	b.full = dfixed_const(1000);
+	c.full = dfixed_const(wm->disp_clk);
+	b.full = dfixed_div(c, b);
+	c.full = dfixed_const(wm->bytes_per_pixel);
+	b.full = dfixed_mul(b, c);
+
+	lb_fill_bw = min(tmp, dfixed_trunc(b));
+
+	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
+	b.full = dfixed_const(1000);
+	c.full = dfixed_const(lb_fill_bw);
+	b.full = dfixed_div(c, b);
+	a.full = dfixed_div(a, b);
+	line_fill_time = dfixed_trunc(a);
+
+	if (line_fill_time < wm->active_time)
+		return latency;
+	else
+		return latency + (line_fill_time - wm->active_time);
+
+}
+
+/**
+ * dce_v6_0_average_bandwidth_vs_dram_bandwidth_for_display - check
+ * average and available dram bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Check if the display average bandwidth fits in the display
+ * dram bandwidth (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns true if the display fits, false if not.
+ */
+static bool dce_v6_0_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
+{
+	if (dce_v6_0_average_bandwidth(wm) <=
+	    (dce_v6_0_dram_bandwidth_for_display(wm) / wm->num_heads))
+		return true;
+	else
+		return false;
+}
+
+/**
+ * dce_v6_0_average_bandwidth_vs_available_bandwidth - check
+ * average and available bandwidth
+ *
+ * @wm: watermark calculation data
+ *
+ * Check if the display average bandwidth fits in the display
+ * available bandwidth (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns true if the display fits, false if not.
+ */
+static bool dce_v6_0_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
+{
+	if (dce_v6_0_average_bandwidth(wm) <=
+	    (dce_v6_0_available_bandwidth(wm) / wm->num_heads))
+		return true;
+	else
+		return false;
+}
+
+/**
+ * dce_v6_0_check_latency_hiding - check latency hiding
+ *
+ * @wm: watermark calculation data
+ *
+ * Check latency hiding (CIK).
+ * Used for display watermark bandwidth calculations
+ * Returns true if the display fits, false if not.
+ */
+static bool dce_v6_0_check_latency_hiding(struct dce6_wm_params *wm)
+{
+	u32 lb_partitions = wm->lb_size / wm->src_width;
+	u32 line_time = wm->active_time + wm->blank_time;
+	u32 latency_tolerant_lines;
+	u32 latency_hiding;
+	fixed20_12 a;
+
+	a.full = dfixed_const(1);
+	if (wm->vsc.full > a.full)
+		latency_tolerant_lines = 1;
+	else {
+		if (lb_partitions <= (wm->vtaps + 1))
+			latency_tolerant_lines = 1;
+		else
+			latency_tolerant_lines = 2;
+	}
+
+	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
+
+	if (dce_v6_0_latency_watermark(wm) <= latency_hiding)
+		return true;
+	else
+		return false;
+}
+
+/**
+ * dce_v6_0_program_watermarks - program display watermarks
+ *
+ * @adev: amdgpu_device pointer
+ * @amdgpu_crtc: the selected display controller
+ * @lb_size: line buffer size
+ * @num_heads: number of display controllers in use
+ *
+ * Calculate and program the display watermarks for the
+ * selected display controller (CIK).
+ */
+static void dce_v6_0_program_watermarks(struct amdgpu_device *adev,
+					struct amdgpu_crtc *amdgpu_crtc,
+					u32 lb_size, u32 num_heads)
+{
+	struct drm_display_mode *mode = &amdgpu_crtc->base.mode;
+	struct dce6_wm_params wm_low, wm_high;
+	u32 dram_channels;
+	u32 pixel_period;
+	u32 line_time = 0;
+	u32 latency_watermark_a = 0, latency_watermark_b = 0;
+	u32 priority_a_mark = 0, priority_b_mark = 0;
+	u32 priority_a_cnt = PRIORITY_OFF;
+	u32 priority_b_cnt = PRIORITY_OFF;
+	u32 tmp, arb_control3;
+	fixed20_12 a, b, c;
+
+	if (amdgpu_crtc->base.enabled && num_heads && mode) {
+		pixel_period = 1000000 / (u32)mode->clock;
+		line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
+		priority_a_cnt = 0;
+		priority_b_cnt = 0;
+
+		dram_channels = si_get_number_of_dram_channels(adev);
+
+		/* watermark for high clocks */
+		if (adev->pm.dpm_enabled) {
+			wm_high.yclk =
+				amdgpu_dpm_get_mclk(adev, false) * 10;
+			wm_high.sclk =
+				amdgpu_dpm_get_sclk(adev, false) * 10;
+		} else {
+			wm_high.yclk = adev->pm.current_mclk * 10;
+			wm_high.sclk = adev->pm.current_sclk * 10;
+		}
+
+		wm_high.disp_clk = mode->clock;
+		wm_high.src_width = mode->crtc_hdisplay;
+		wm_high.active_time = mode->crtc_hdisplay * pixel_period;
+		wm_high.blank_time = line_time - wm_high.active_time;
+		wm_high.interlaced = false;
+		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+			wm_high.interlaced = true;
+		wm_high.vsc = amdgpu_crtc->vsc;
+		wm_high.vtaps = 1;
+		if (amdgpu_crtc->rmx_type != RMX_OFF)
+			wm_high.vtaps = 2;
+		wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
+		wm_high.lb_size = lb_size;
+		wm_high.dram_channels = dram_channels;
+		wm_high.num_heads = num_heads;
+
+		if (adev->pm.dpm_enabled) {
+		/* watermark for low clocks */
+			wm_low.yclk =
+				amdgpu_dpm_get_mclk(adev, true) * 10;
+			wm_low.sclk =
+				amdgpu_dpm_get_sclk(adev, true) * 10;
+		} else {
+			wm_low.yclk = adev->pm.current_mclk * 10;
+			wm_low.sclk = adev->pm.current_sclk * 10;
+		}
+
+		wm_low.disp_clk = mode->clock;
+		wm_low.src_width = mode->crtc_hdisplay;
+		wm_low.active_time = mode->crtc_hdisplay * pixel_period;
+		wm_low.blank_time = line_time - wm_low.active_time;
+		wm_low.interlaced = false;
+		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+			wm_low.interlaced = true;
+		wm_low.vsc = amdgpu_crtc->vsc;
+		wm_low.vtaps = 1;
+		if (amdgpu_crtc->rmx_type != RMX_OFF)
+			wm_low.vtaps = 2;
+		wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
+		wm_low.lb_size = lb_size;
+		wm_low.dram_channels = dram_channels;
+		wm_low.num_heads = num_heads;
+
+		/* set for high clocks */
+		latency_watermark_a = min(dce_v6_0_latency_watermark(&wm_high), (u32)65535);
+		/* set for low clocks */
+		latency_watermark_b = min(dce_v6_0_latency_watermark(&wm_low), (u32)65535);
+
+		/* possibly force display priority to high */
+		/* should really do this at mode validation time... */
+		if (!dce_v6_0_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
+		    !dce_v6_0_average_bandwidth_vs_available_bandwidth(&wm_high) ||
+		    !dce_v6_0_check_latency_hiding(&wm_high) ||
+		    (adev->mode_info.disp_priority == 2)) {
+			DRM_DEBUG_KMS("force priority to high\n");
+			priority_a_cnt |= PRIORITY_ALWAYS_ON;
+			priority_b_cnt |= PRIORITY_ALWAYS_ON;
+		}
+		if (!dce_v6_0_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
+		    !dce_v6_0_average_bandwidth_vs_available_bandwidth(&wm_low) ||
+		    !dce_v6_0_check_latency_hiding(&wm_low) ||
+		    (adev->mode_info.disp_priority == 2)) {
+			DRM_DEBUG_KMS("force priority to high\n");
+			priority_a_cnt |= PRIORITY_ALWAYS_ON;
+			priority_b_cnt |= PRIORITY_ALWAYS_ON;
+		}
+
+		a.full = dfixed_const(1000);
+		b.full = dfixed_const(mode->clock);
+		b.full = dfixed_div(b, a);
+		c.full = dfixed_const(latency_watermark_a);
+		c.full = dfixed_mul(c, b);
+		c.full = dfixed_mul(c, amdgpu_crtc->hsc);
+		c.full = dfixed_div(c, a);
+		a.full = dfixed_const(16);
+		c.full = dfixed_div(c, a);
+		priority_a_mark = dfixed_trunc(c);
+		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
+
+		a.full = dfixed_const(1000);
+		b.full = dfixed_const(mode->clock);
+		b.full = dfixed_div(b, a);
+		c.full = dfixed_const(latency_watermark_b);
+		c.full = dfixed_mul(c, b);
+		c.full = dfixed_mul(c, amdgpu_crtc->hsc);
+		c.full = dfixed_div(c, a);
+		a.full = dfixed_const(16);
+		c.full = dfixed_div(c, a);
+		priority_b_mark = dfixed_trunc(c);
+		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
+	}
+
+	/* select wm A */
+	arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset);
+	tmp = arb_control3;
+	tmp &= ~LATENCY_WATERMARK_MASK(3);
+	tmp |= LATENCY_WATERMARK_MASK(1);
+	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset, tmp);
+	WREG32(DPG_PIPE_LATENCY_CONTROL + amdgpu_crtc->crtc_offset,
+	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
+		LATENCY_HIGH_WATERMARK(line_time)));
+	/* select wm B */
+	tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset);
+	tmp &= ~LATENCY_WATERMARK_MASK(3);
+	tmp |= LATENCY_WATERMARK_MASK(2);
+	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset, tmp);
+	WREG32(DPG_PIPE_LATENCY_CONTROL + amdgpu_crtc->crtc_offset,
+	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
+		LATENCY_HIGH_WATERMARK(line_time)));
+	/* restore original selection */
+	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset, arb_control3);
+
+	/* write the priority marks */
+	WREG32(PRIORITY_A_CNT + amdgpu_crtc->crtc_offset, priority_a_cnt);
+	WREG32(PRIORITY_B_CNT + amdgpu_crtc->crtc_offset, priority_b_cnt);
+
+	/* save values for DPM */
+	amdgpu_crtc->line_time = line_time;
+	amdgpu_crtc->wm_high = latency_watermark_a;
+}
+
+/* watermark setup */
+static u32 dce_v6_0_line_buffer_adjust(struct amdgpu_device *adev,
+				   struct amdgpu_crtc *amdgpu_crtc,
+				   struct drm_display_mode *mode,
+				   struct drm_display_mode *other_mode)
+{
+	u32 tmp, buffer_alloc, i;
+	u32 pipe_offset = amdgpu_crtc->crtc_id * 0x8;
+	/*
+	 * Line Buffer Setup
+	 * There are 3 line buffers, each one shared by 2 display controllers.
+	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
+	 * the display controllers.  The paritioning is done via one of four
+	 * preset allocations specified in bits 21:20:
+	 *  0 - half lb
+	 *  2 - whole lb, other crtc must be disabled
+	 */
+	/* this can get tricky if we have two large displays on a paired group
+	 * of crtcs.  Ideally for multiple large displays we'd assign them to
+	 * non-linked crtcs for maximum line buffer allocation.
+	 */
+	if (amdgpu_crtc->base.enabled && mode) {
+		if (other_mode) {
+			tmp = 0; /* 1/2 */
+			buffer_alloc = 1;
+		} else {
+			tmp = 2; /* whole */
+			buffer_alloc = 2;
+		}
+	} else {
+		tmp = 0;
+		buffer_alloc = 0;
+	}
+
+	WREG32(DC_LB_MEMORY_SPLIT + amdgpu_crtc->crtc_offset,
+	       DC_LB_MEMORY_CONFIG(tmp));
+
+	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
+	       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
+		    DMIF_BUFFERS_ALLOCATED_COMPLETED)
+			break;
+		udelay(1);
+	}
+
+	if (amdgpu_crtc->base.enabled && mode) {
+		switch (tmp) {
+		case 0:
+		default:
+			return 4096 * 2;
+		case 2:
+			return 8192 * 2;
+		}
+	}
+
+	/* controller not enabled, so no lb used */
+	return 0;
+}
+
+
+/**
+ *
+ * dce_v6_0_bandwidth_update - program display watermarks
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Calculate and program the display watermarks and line
+ * buffer allocation (CIK).
+ */
+static void dce_v6_0_bandwidth_update(struct amdgpu_device *adev)
+{
+	struct drm_display_mode *mode0 = NULL;
+	struct drm_display_mode *mode1 = NULL;
+	u32 num_heads = 0, lb_size;
+	int i;
+
+	if (!adev->mode_info.mode_config_initialized)
+		return;
+
+	amdgpu_update_display_priority(adev);
+
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		if (adev->mode_info.crtcs[i]->base.enabled)
+			num_heads++;
+	}
+	for (i = 0; i < adev->mode_info.num_crtc; i += 2) {
+		mode0 = &adev->mode_info.crtcs[i]->base.mode;
+		mode1 = &adev->mode_info.crtcs[i+1]->base.mode;
+		lb_size = dce_v6_0_line_buffer_adjust(adev, adev->mode_info.crtcs[i], mode0, mode1);
+		dce_v6_0_program_watermarks(adev, adev->mode_info.crtcs[i], lb_size, num_heads);
+		lb_size = dce_v6_0_line_buffer_adjust(adev, adev->mode_info.crtcs[i+1], mode1, mode0);
+		dce_v6_0_program_watermarks(adev, adev->mode_info.crtcs[i+1], lb_size, num_heads);
+	}
+}
+/*
+static void dce_v6_0_audio_get_connected_pins(struct amdgpu_device *adev)
+{
+	int i;
+	u32 offset, tmp;
+
+	for (i = 0; i < adev->mode_info.audio.num_pins; i++) {
+		offset = adev->mode_info.audio.pin[i].offset;
+		tmp = RREG32_AUDIO_ENDPT(offset,
+				      AZ_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT);
+		if (((tmp & PORT_CONNECTIVITY_MASK) >> PORT_CONNECTIVITY_SHIFT) == 1)
+			adev->mode_info.audio.pin[i].connected = false;
+		else
+			adev->mode_info.audio.pin[i].connected = true;
+	}
+
+}
+
+static struct amdgpu_audio_pin *dce_v6_0_audio_get_pin(struct amdgpu_device *adev)
+{
+	int i;
+
+	dce_v6_0_audio_get_connected_pins(adev);
+
+	for (i = 0; i < adev->mode_info.audio.num_pins; i++) {
+		if (adev->mode_info.audio.pin[i].connected)
+			return &adev->mode_info.audio.pin[i];
+	}
+	DRM_ERROR("No connected audio pins found!\n");
+	return NULL;
+}
+
+static void dce_v6_0_afmt_audio_select_pin(struct drm_encoder *encoder)
+{
+	struct amdgpu_device *adev = encoder->dev->dev_private;
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+	struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv;
+	u32 offset;
+
+	if (!dig || !dig->afmt || !dig->afmt->pin)
+		return;
+
+	offset = dig->afmt->offset;
+
+	WREG32(AFMT_AUDIO_SRC_CONTROL + offset,
+	       AFMT_AUDIO_SRC_SELECT(dig->afmt->pin->id));
+
+}
+
+static void dce_v6_0_audio_write_latency_fields(struct drm_encoder *encoder,
+						struct drm_display_mode *mode)
+{
+	DRM_INFO("xxxx: dce_v6_0_audio_write_latency_fields---no imp!!!!!\n");
+}
+
+static void dce_v6_0_audio_write_speaker_allocation(struct drm_encoder *encoder)
+{
+	DRM_INFO("xxxx: dce_v6_0_audio_write_speaker_allocation---no imp!!!!!\n");
+}
+
+static void dce_v6_0_audio_write_sad_regs(struct drm_encoder *encoder)
+{
+	DRM_INFO("xxxx: dce_v6_0_audio_write_sad_regs---no imp!!!!!\n");
+
+}
+*/
+static void dce_v6_0_audio_enable(struct amdgpu_device *adev,
+				  struct amdgpu_audio_pin *pin,
+				  bool enable)
+{
+	DRM_INFO("xxxx: dce_v6_0_audio_enable---no imp!!!!!\n");
+}
+
+static const u32 pin_offsets[7] =
+{
+	(0x1780 - 0x1780),
+	(0x1786 - 0x1780),
+	(0x178c - 0x1780),
+	(0x1792 - 0x1780),
+	(0x1798 - 0x1780),
+	(0x179d - 0x1780),
+	(0x17a4 - 0x1780),
+};
+
+static int dce_v6_0_audio_init(struct amdgpu_device *adev)
+{
+	return 0;
+}
+
+static void dce_v6_0_audio_fini(struct amdgpu_device *adev)
+{
+
+}
+
+/*
+static void dce_v6_0_afmt_update_ACR(struct drm_encoder *encoder, uint32_t clock)
+{
+	DRM_INFO("xxxx: dce_v6_0_afmt_update_ACR---no imp!!!!!\n");
+}
+*/
+/*
+ * build a HDMI Video Info Frame
+ */
+/*
+static void dce_v6_0_afmt_update_avi_infoframe(struct drm_encoder *encoder,
+					       void *buffer, size_t size)
+{
+	DRM_INFO("xxxx: dce_v6_0_afmt_update_avi_infoframe---no imp!!!!!\n");
+}
+
+static void dce_v6_0_audio_set_dto(struct drm_encoder *encoder, u32 clock)
+{
+	DRM_INFO("xxxx: dce_v6_0_audio_set_dto---no imp!!!!!\n");
+}
+*/
+/*
+ * update the info frames with the data from the current display mode
+ */
+static void dce_v6_0_afmt_setmode(struct drm_encoder *encoder,
+				  struct drm_display_mode *mode)
+{
+	DRM_INFO("xxxx: dce_v6_0_afmt_setmode ----no impl !!!!!!!!\n");
+}
+
+static void dce_v6_0_afmt_enable(struct drm_encoder *encoder, bool enable)
+{
+	struct drm_device *dev = encoder->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+	struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv;
+
+	if (!dig || !dig->afmt)
+		return;
+
+	/* Silent, r600_hdmi_enable will raise WARN for us */
+	if (enable && dig->afmt->enabled)
+		return;
+	if (!enable && !dig->afmt->enabled)
+		return;
+
+	if (!enable && dig->afmt->pin) {
+		dce_v6_0_audio_enable(adev, dig->afmt->pin, false);
+		dig->afmt->pin = NULL;
+	}
+
+	dig->afmt->enabled = enable;
+
+	DRM_DEBUG("%sabling AFMT interface @ 0x%04X for encoder 0x%x\n",
+		  enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id);
+}
+
+static int dce_v6_0_afmt_init(struct amdgpu_device *adev)
+{
+	int i, j;
+
+	for (i = 0; i < adev->mode_info.num_dig; i++)
+		adev->mode_info.afmt[i] = NULL;
+
+	/* DCE6 has audio blocks tied to DIG encoders */
+	for (i = 0; i < adev->mode_info.num_dig; i++) {
+		adev->mode_info.afmt[i] = kzalloc(sizeof(struct amdgpu_afmt), GFP_KERNEL);
+		if (adev->mode_info.afmt[i]) {
+			adev->mode_info.afmt[i]->offset = dig_offsets[i];
+			adev->mode_info.afmt[i]->id = i;
+		} else {
+			for (j = 0; j < i; j++) {
+				kfree(adev->mode_info.afmt[j]);
+				adev->mode_info.afmt[j] = NULL;
+			}
+			DRM_ERROR("Out of memory allocating afmt table\n");
+			return -ENOMEM;
+		}
+	}
+	return 0;
+}
+
+static void dce_v6_0_afmt_fini(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < adev->mode_info.num_dig; i++) {
+		kfree(adev->mode_info.afmt[i]);
+		adev->mode_info.afmt[i] = NULL;
+	}
+}
+
+static const u32 vga_control_regs[6] =
+{
+	AVIVO_D1VGA_CONTROL,
+	AVIVO_D2VGA_CONTROL,
+	EVERGREEN_D3VGA_CONTROL,
+	EVERGREEN_D4VGA_CONTROL,
+	EVERGREEN_D5VGA_CONTROL,
+	EVERGREEN_D6VGA_CONTROL,
+};
+
+static void dce_v6_0_vga_enable(struct drm_crtc *crtc, bool enable)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	u32 vga_control;
+
+	vga_control = RREG32(vga_control_regs[amdgpu_crtc->crtc_id]) & ~1;
+	WREG32(vga_control_regs[amdgpu_crtc->crtc_id], vga_control | (enable ? 1 : 0));
+}
+
+static void dce_v6_0_grph_enable(struct drm_crtc *crtc, bool enable)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+
+	WREG32(EVERGREEN_GRPH_ENABLE + amdgpu_crtc->crtc_offset, enable ? 1 : 0);
+}
+
+static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
+				     struct drm_framebuffer *fb,
+				     int x, int y, int atomic)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	struct amdgpu_framebuffer *amdgpu_fb;
+	struct drm_framebuffer *target_fb;
+	struct drm_gem_object *obj;
+	struct amdgpu_bo *abo;
+	uint64_t fb_location, tiling_flags;
+	uint32_t fb_format, fb_pitch_pixels, pipe_config;
+	u32 fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_NONE);
+	u32 viewport_w, viewport_h;
+	int r;
+	bool bypass_lut = false;
+
+	/* no fb bound */
+	if (!atomic && !crtc->primary->fb) {
+		DRM_DEBUG_KMS("No FB bound\n");
+		return 0;
+	}
+
+	if (atomic) {
+		amdgpu_fb = to_amdgpu_framebuffer(fb);
+		target_fb = fb;
+	} else {
+		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
+		target_fb = crtc->primary->fb;
+	}
+
+	/* If atomic, assume fb object is pinned & idle & fenced and
+	 * just update base pointers
+	 */
+	obj = amdgpu_fb->obj;
+	abo = gem_to_amdgpu_bo(obj);
+	r = amdgpu_bo_reserve(abo, false);
+	if (unlikely(r != 0))
+		return r;
+
+	if (atomic) {
+		fb_location = amdgpu_bo_gpu_offset(abo);
+	} else {
+		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
+		if (unlikely(r != 0)) {
+			amdgpu_bo_unreserve(abo);
+			return -EINVAL;
+		}
+	}
+
+	amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
+	amdgpu_bo_unreserve(abo);
+
+	switch (target_fb->pixel_format) {
+	case DRM_FORMAT_C8:
+		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_8BPP) |
+			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_INDEXED));
+		break;
+	case DRM_FORMAT_XRGB4444:
+	case DRM_FORMAT_ARGB4444:
+		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
+			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB4444));
+#ifdef __BIG_ENDIAN
+		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+#endif
+		break;
+	case DRM_FORMAT_XRGB1555:
+	case DRM_FORMAT_ARGB1555:
+		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
+			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB1555));
+#ifdef __BIG_ENDIAN
+		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+#endif
+		break;
+	case DRM_FORMAT_BGRX5551:
+	case DRM_FORMAT_BGRA5551:
+		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
+			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_BGRA5551));
+#ifdef __BIG_ENDIAN
+		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+#endif
+		break;
+	case DRM_FORMAT_RGB565:
+		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
+			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB565));
+#ifdef __BIG_ENDIAN
+		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+#endif
+		break;
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ARGB8888:
+		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) |
+			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB8888));
+#ifdef __BIG_ENDIAN
+		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32);
+#endif
+		break;
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_ARGB2101010:
+		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) |
+			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB2101010));
+#ifdef __BIG_ENDIAN
+		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32);
+#endif
+		/* Greater 8 bpc fb needs to bypass hw-lut to retain precision */
+		bypass_lut = true;
+		break;
+	case DRM_FORMAT_BGRX1010102:
+	case DRM_FORMAT_BGRA1010102:
+		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) |
+			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_BGRA1010102));
+#ifdef __BIG_ENDIAN
+		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32);
+#endif
+		/* Greater 8 bpc fb needs to bypass hw-lut to retain precision */
+		bypass_lut = true;
+		break;
+	default:
+		DRM_ERROR("Unsupported screen format %s\n",
+			  drm_get_format_name(target_fb->pixel_format));
+		return -EINVAL;
+	}
+
+	if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == ARRAY_2D_TILED_THIN1) {
+		unsigned bankw, bankh, mtaspect, tile_split, num_banks;
+
+		bankw = AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH);
+		bankh = AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT);
+		mtaspect = AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT);
+		tile_split = AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT);
+		num_banks = AMDGPU_TILING_GET(tiling_flags, NUM_BANKS);
+
+		fb_format |= EVERGREEN_GRPH_NUM_BANKS(num_banks);
+		fb_format |= EVERGREEN_GRPH_ARRAY_MODE(EVERGREEN_GRPH_ARRAY_2D_TILED_THIN1);
+		fb_format |= EVERGREEN_GRPH_TILE_SPLIT(tile_split);
+		fb_format |= EVERGREEN_GRPH_BANK_WIDTH(bankw);
+		fb_format |= EVERGREEN_GRPH_BANK_HEIGHT(bankh);
+		fb_format |= EVERGREEN_GRPH_MACRO_TILE_ASPECT(mtaspect);
+	} else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == ARRAY_1D_TILED_THIN1) {
+		fb_format |= EVERGREEN_GRPH_ARRAY_MODE(EVERGREEN_GRPH_ARRAY_1D_TILED_THIN1);
+	}
+
+	pipe_config = AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
+	fb_format |= SI_GRPH_PIPE_CONFIG(pipe_config);
+
+	dce_v6_0_vga_enable(crtc, false);
+
+	/* Make sure surface address is updated at vertical blank rather than
+	 * horizontal blank
+	 */
+	WREG32(EVERGREEN_GRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset, 0);
+
+	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+	       upper_32_bits(fb_location));
+	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+	       upper_32_bits(fb_location));
+	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+	       (u32)fb_location & EVERGREEN_GRPH_SURFACE_ADDRESS_MASK);
+	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+	       (u32) fb_location & EVERGREEN_GRPH_SURFACE_ADDRESS_MASK);
+	WREG32(EVERGREEN_GRPH_CONTROL + amdgpu_crtc->crtc_offset, fb_format);
+	WREG32(EVERGREEN_GRPH_SWAP_CONTROL + amdgpu_crtc->crtc_offset, fb_swap);
+
+	/*
+	 * The LUT only has 256 slots for indexing by a 8 bpc fb. Bypass the LUT
+	 * for > 8 bpc scanout to avoid truncation of fb indices to 8 msb's, to
+	 * retain the full precision throughout the pipeline.
+	 */
+	WREG32_P(EVERGREEN_GRPH_LUT_10BIT_BYPASS_CONTROL + amdgpu_crtc->crtc_offset,
+		 (bypass_lut ? EVERGREEN_LUT_10BIT_BYPASS_EN : 0),
+		 ~EVERGREEN_LUT_10BIT_BYPASS_EN);
+
+	if (bypass_lut)
+		DRM_DEBUG_KMS("Bypassing hardware LUT due to 10 bit fb scanout.\n");
+
+	WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_X + amdgpu_crtc->crtc_offset, 0);
+	WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_Y + amdgpu_crtc->crtc_offset, 0);
+	WREG32(EVERGREEN_GRPH_X_START + amdgpu_crtc->crtc_offset, 0);
+	WREG32(EVERGREEN_GRPH_Y_START + amdgpu_crtc->crtc_offset, 0);
+	WREG32(EVERGREEN_GRPH_X_END + amdgpu_crtc->crtc_offset, target_fb->width);
+	WREG32(EVERGREEN_GRPH_Y_END + amdgpu_crtc->crtc_offset, target_fb->height);
+
+	fb_pitch_pixels = target_fb->pitches[0] / (target_fb->bits_per_pixel / 8);
+	WREG32(EVERGREEN_GRPH_PITCH + amdgpu_crtc->crtc_offset, fb_pitch_pixels);
+
+	dce_v6_0_grph_enable(crtc, true);
+
+	WREG32(EVERGREEN_DESKTOP_HEIGHT + amdgpu_crtc->crtc_offset,
+		       target_fb->height);
+	x &= ~3;
+	y &= ~1;
+	WREG32(EVERGREEN_VIEWPORT_START + amdgpu_crtc->crtc_offset,
+	       (x << 16) | y);
+	viewport_w = crtc->mode.hdisplay;
+	viewport_h = (crtc->mode.vdisplay + 1) & ~1;
+
+	WREG32(EVERGREEN_VIEWPORT_SIZE + amdgpu_crtc->crtc_offset,
+	       (viewport_w << 16) | viewport_h);
+
+	/* set pageflip to happen anywhere in vblank interval */
+	WREG32(EVERGREEN_MASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
+
+	if (!atomic && fb && fb != crtc->primary->fb) {
+		amdgpu_fb = to_amdgpu_framebuffer(fb);
+		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		r = amdgpu_bo_reserve(abo, false);
+		if (unlikely(r != 0))
+			return r;
+		amdgpu_bo_unpin(abo);
+		amdgpu_bo_unreserve(abo);
+	}
+
+	/* Bytes per pixel may have changed */
+	dce_v6_0_bandwidth_update(adev);
+
+	return 0;
+
+}
+
+static void dce_v6_0_set_interleave(struct drm_crtc *crtc,
+				    struct drm_display_mode *mode)
+{
+	struct drm_device *dev = crtc->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		WREG32(EVERGREEN_DATA_FORMAT + amdgpu_crtc->crtc_offset,
+		       EVERGREEN_INTERLEAVE_EN);
+	else
+		WREG32(EVERGREEN_DATA_FORMAT + amdgpu_crtc->crtc_offset, 0);
+}
+
+static void dce_v6_0_crtc_load_lut(struct drm_crtc *crtc)
+{
+
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	int i;
+
+	DRM_DEBUG_KMS("%d\n", amdgpu_crtc->crtc_id);
+
+	WREG32(NI_INPUT_CSC_CONTROL + amdgpu_crtc->crtc_offset,
+	       (NI_INPUT_CSC_GRPH_MODE(NI_INPUT_CSC_BYPASS) |
+		NI_INPUT_CSC_OVL_MODE(NI_INPUT_CSC_BYPASS)));
+	WREG32(NI_PRESCALE_GRPH_CONTROL + amdgpu_crtc->crtc_offset,
+	       NI_GRPH_PRESCALE_BYPASS);
+	WREG32(NI_PRESCALE_OVL_CONTROL + amdgpu_crtc->crtc_offset,
+	       NI_OVL_PRESCALE_BYPASS);
+	WREG32(NI_INPUT_GAMMA_CONTROL + amdgpu_crtc->crtc_offset,
+	       (NI_GRPH_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT) |
+		NI_OVL_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT)));
+
+
+
+	WREG32(EVERGREEN_DC_LUT_CONTROL + amdgpu_crtc->crtc_offset, 0);
+
+	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_BLUE + amdgpu_crtc->crtc_offset, 0);
+	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_GREEN + amdgpu_crtc->crtc_offset, 0);
+	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_RED + amdgpu_crtc->crtc_offset, 0);
+
+	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_BLUE + amdgpu_crtc->crtc_offset, 0xffff);
+	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_GREEN + amdgpu_crtc->crtc_offset, 0xffff);
+	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_RED + amdgpu_crtc->crtc_offset, 0xffff);
+
+	WREG32(EVERGREEN_DC_LUT_RW_MODE + amdgpu_crtc->crtc_offset, 0);
+	WREG32(EVERGREEN_DC_LUT_WRITE_EN_MASK + amdgpu_crtc->crtc_offset, 0x00000007);
+
+	WREG32(EVERGREEN_DC_LUT_RW_INDEX + amdgpu_crtc->crtc_offset, 0);
+	for (i = 0; i < 256; i++) {
+		WREG32(EVERGREEN_DC_LUT_30_COLOR + amdgpu_crtc->crtc_offset,
+		       (amdgpu_crtc->lut_r[i] << 20) |
+		       (amdgpu_crtc->lut_g[i] << 10) |
+		       (amdgpu_crtc->lut_b[i] << 0));
+	}
+
+	WREG32(NI_DEGAMMA_CONTROL + amdgpu_crtc->crtc_offset,
+	       (NI_GRPH_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
+		NI_OVL_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
+		NI_ICON_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
+		NI_CURSOR_DEGAMMA_MODE(NI_DEGAMMA_BYPASS)));
+	WREG32(NI_GAMUT_REMAP_CONTROL + amdgpu_crtc->crtc_offset,
+	       (NI_GRPH_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS) |
+		NI_OVL_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS)));
+	WREG32(NI_REGAMMA_CONTROL + amdgpu_crtc->crtc_offset,
+	       (NI_GRPH_REGAMMA_MODE(NI_REGAMMA_BYPASS) |
+		NI_OVL_REGAMMA_MODE(NI_REGAMMA_BYPASS)));
+	WREG32(NI_OUTPUT_CSC_CONTROL + amdgpu_crtc->crtc_offset,
+	       (NI_OUTPUT_CSC_GRPH_MODE(0) |
+		NI_OUTPUT_CSC_OVL_MODE(NI_OUTPUT_CSC_BYPASS)));
+	/* XXX match this to the depth of the crtc fmt block, move to modeset? */
+	WREG32(0x1a50 + amdgpu_crtc->crtc_offset, 0);
+
+
+}
+
+static int dce_v6_0_pick_dig_encoder(struct drm_encoder *encoder)
+{
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+	struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv;
+
+	switch (amdgpu_encoder->encoder_id) {
+	case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
+		return dig->linkb ? 1 : 0;
+	case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
+		return dig->linkb ? 3 : 2;
+	case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+		return dig->linkb ? 5 : 4;
+	case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
+		return 6;
+	default:
+		DRM_ERROR("invalid encoder_id: 0x%x\n", amdgpu_encoder->encoder_id);
+		return 0;
+	}
+}
+
+/**
+ * dce_v6_0_pick_pll - Allocate a PPLL for use by the crtc.
+ *
+ * @crtc: drm crtc
+ *
+ * Returns the PPLL (Pixel PLL) to be used by the crtc.  For DP monitors
+ * a single PPLL can be used for all DP crtcs/encoders.  For non-DP
+ * monitors a dedicated PPLL must be used.  If a particular board has
+ * an external DP PLL, return ATOM_PPLL_INVALID to skip PLL programming
+ * as there is no need to program the PLL itself.  If we are not able to
+ * allocate a PLL, return ATOM_PPLL_INVALID to skip PLL programming to
+ * avoid messing up an existing monitor.
+ *
+ *
+ */
+static u32 dce_v6_0_pick_pll(struct drm_crtc *crtc)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	u32 pll_in_use;
+	int pll;
+
+	if (ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(amdgpu_crtc->encoder))) {
+		if (adev->clock.dp_extclk)
+			/* skip PPLL programming if using ext clock */
+			return ATOM_PPLL_INVALID;
+		else
+			return ATOM_PPLL0;
+	} else {
+		/* use the same PPLL for all monitors with the same clock */
+		pll = amdgpu_pll_get_shared_nondp_ppll(crtc);
+		if (pll != ATOM_PPLL_INVALID)
+			return pll;
+	}
+
+	/*  PPLL1, and PPLL2 */
+	pll_in_use = amdgpu_pll_get_use_mask(crtc);
+	if (!(pll_in_use & (1 << ATOM_PPLL2)))
+		return ATOM_PPLL2;
+	if (!(pll_in_use & (1 << ATOM_PPLL1)))
+		return ATOM_PPLL1;
+	DRM_ERROR("unable to allocate a PPLL\n");
+	return ATOM_PPLL_INVALID;
+}
+
+static void dce_v6_0_lock_cursor(struct drm_crtc *crtc, bool lock)
+{
+	struct amdgpu_device *adev = crtc->dev->dev_private;
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	uint32_t cur_lock;
+
+	cur_lock = RREG32(EVERGREEN_CUR_UPDATE + amdgpu_crtc->crtc_offset);
+	if (lock)
+		cur_lock |= EVERGREEN_CURSOR_UPDATE_LOCK;
+	else
+		cur_lock &= ~EVERGREEN_CURSOR_UPDATE_LOCK;
+	WREG32(EVERGREEN_CUR_UPDATE + amdgpu_crtc->crtc_offset, cur_lock);
+}
+
+static void dce_v6_0_hide_cursor(struct drm_crtc *crtc)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
+
+	WREG32_IDX(EVERGREEN_CUR_CONTROL + amdgpu_crtc->crtc_offset,
+		   EVERGREEN_CURSOR_MODE(EVERGREEN_CURSOR_24_8_PRE_MULT) |
+		   EVERGREEN_CURSOR_URGENT_CONTROL(EVERGREEN_CURSOR_URGENT_1_2));
+
+
+}
+
+static void dce_v6_0_show_cursor(struct drm_crtc *crtc)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
+
+	WREG32(EVERGREEN_CUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+	       upper_32_bits(amdgpu_crtc->cursor_addr));
+	WREG32(EVERGREEN_CUR_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+	       lower_32_bits(amdgpu_crtc->cursor_addr));
+
+	WREG32_IDX(EVERGREEN_CUR_CONTROL + amdgpu_crtc->crtc_offset,
+		   EVERGREEN_CURSOR_EN |
+		   EVERGREEN_CURSOR_MODE(EVERGREEN_CURSOR_24_8_PRE_MULT) |
+		   EVERGREEN_CURSOR_URGENT_CONTROL(EVERGREEN_CURSOR_URGENT_1_2));
+
+}
+
+static int dce_v6_0_cursor_move_locked(struct drm_crtc *crtc,
+				       int x, int y)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
+	int xorigin = 0, yorigin = 0;
+
+	int w = amdgpu_crtc->cursor_width;
+
+	/* avivo cursor are offset into the total surface */
+	x += crtc->x;
+	y += crtc->y;
+	DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y);
+
+	if (x < 0) {
+		xorigin = min(-x, amdgpu_crtc->max_cursor_width - 1);
+		x = 0;
+	}
+	if (y < 0) {
+		yorigin = min(-y, amdgpu_crtc->max_cursor_height - 1);
+		y = 0;
+	}
+
+	WREG32(EVERGREEN_CUR_POSITION + amdgpu_crtc->crtc_offset, (x << 16) | y);
+	WREG32(EVERGREEN_CUR_HOT_SPOT + amdgpu_crtc->crtc_offset, (xorigin << 16) | yorigin);
+	WREG32(EVERGREEN_CUR_SIZE + amdgpu_crtc->crtc_offset,
+	       ((w - 1) << 16) | (amdgpu_crtc->cursor_height - 1));
+
+	amdgpu_crtc->cursor_x = x;
+	amdgpu_crtc->cursor_y = y;
+	return 0;
+}
+
+static int dce_v6_0_crtc_cursor_move(struct drm_crtc *crtc,
+				     int x, int y)
+{
+	int ret;
+
+	dce_v6_0_lock_cursor(crtc, true);
+	ret = dce_v6_0_cursor_move_locked(crtc, x, y);
+	dce_v6_0_lock_cursor(crtc, false);
+
+	return ret;
+}
+
+static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc,
+				     struct drm_file *file_priv,
+				     uint32_t handle,
+				     uint32_t width,
+				     uint32_t height,
+				     int32_t hot_x,
+				     int32_t hot_y)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct drm_gem_object *obj;
+	struct amdgpu_bo *aobj;
+	int ret;
+
+	if (!handle) {
+		/* turn off cursor */
+		dce_v6_0_hide_cursor(crtc);
+		obj = NULL;
+		goto unpin;
+	}
+
+	if ((width > amdgpu_crtc->max_cursor_width) ||
+	    (height > amdgpu_crtc->max_cursor_height)) {
+		DRM_ERROR("bad cursor width or height %d x %d\n", width, height);
+		return -EINVAL;
+	}
+
+	obj = drm_gem_object_lookup(file_priv, handle);
+	if (!obj) {
+		DRM_ERROR("Cannot find cursor object %x for crtc %d\n", handle, amdgpu_crtc->crtc_id);
+		return -ENOENT;
+	}
+
+	aobj = gem_to_amdgpu_bo(obj);
+	ret = amdgpu_bo_reserve(aobj, false);
+	if (ret != 0) {
+		drm_gem_object_unreference_unlocked(obj);
+		return ret;
+	}
+
+	ret = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM, &amdgpu_crtc->cursor_addr);
+	amdgpu_bo_unreserve(aobj);
+	if (ret) {
+		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
+		drm_gem_object_unreference_unlocked(obj);
+		return ret;
+	}
+
+	amdgpu_crtc->cursor_width = width;
+	amdgpu_crtc->cursor_height = height;
+
+	dce_v6_0_lock_cursor(crtc, true);
+
+	if (hot_x != amdgpu_crtc->cursor_hot_x ||
+	    hot_y != amdgpu_crtc->cursor_hot_y) {
+		int x, y;
+
+		x = amdgpu_crtc->cursor_x + amdgpu_crtc->cursor_hot_x - hot_x;
+		y = amdgpu_crtc->cursor_y + amdgpu_crtc->cursor_hot_y - hot_y;
+
+		dce_v6_0_cursor_move_locked(crtc, x, y);
+
+		amdgpu_crtc->cursor_hot_x = hot_x;
+		amdgpu_crtc->cursor_hot_y = hot_y;
+	}
+
+	dce_v6_0_show_cursor(crtc);
+	dce_v6_0_lock_cursor(crtc, false);
+
+unpin:
+	if (amdgpu_crtc->cursor_bo) {
+		struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
+		ret = amdgpu_bo_reserve(aobj, false);
+		if (likely(ret == 0)) {
+			amdgpu_bo_unpin(aobj);
+			amdgpu_bo_unreserve(aobj);
+		}
+		drm_gem_object_unreference_unlocked(amdgpu_crtc->cursor_bo);
+	}
+
+	amdgpu_crtc->cursor_bo = obj;
+	return 0;
+}
+
+static void dce_v6_0_cursor_reset(struct drm_crtc *crtc)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+	if (amdgpu_crtc->cursor_bo) {
+		dce_v6_0_lock_cursor(crtc, true);
+
+		dce_v6_0_cursor_move_locked(crtc, amdgpu_crtc->cursor_x,
+					    amdgpu_crtc->cursor_y);
+
+		dce_v6_0_show_cursor(crtc);
+		dce_v6_0_lock_cursor(crtc, false);
+	}
+}
+
+static int dce_v6_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+				   u16 *blue, uint32_t size)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	int i;
+
+	/* userspace palettes are always correct as is */
+	for (i = 0; i < size; i++) {
+		amdgpu_crtc->lut_r[i] = red[i] >> 6;
+		amdgpu_crtc->lut_g[i] = green[i] >> 6;
+		amdgpu_crtc->lut_b[i] = blue[i] >> 6;
+	}
+	dce_v6_0_crtc_load_lut(crtc);
+
+	return 0;
+}
+
+static void dce_v6_0_crtc_destroy(struct drm_crtc *crtc)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+	drm_crtc_cleanup(crtc);
+	kfree(amdgpu_crtc);
+}
+
+static const struct drm_crtc_funcs dce_v6_0_crtc_funcs = {
+	.cursor_set2 = dce_v6_0_crtc_cursor_set2,
+	.cursor_move = dce_v6_0_crtc_cursor_move,
+	.gamma_set = dce_v6_0_crtc_gamma_set,
+	.set_config = amdgpu_crtc_set_config,
+	.destroy = dce_v6_0_crtc_destroy,
+	.page_flip_target = amdgpu_crtc_page_flip_target,
+};
+
+static void dce_v6_0_crtc_dpms(struct drm_crtc *crtc, int mode)
+{
+	struct drm_device *dev = crtc->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	unsigned type;
+
+	switch (mode) {
+	case DRM_MODE_DPMS_ON:
+		amdgpu_crtc->enabled = true;
+		amdgpu_atombios_crtc_enable(crtc, ATOM_ENABLE);
+		amdgpu_atombios_crtc_blank(crtc, ATOM_DISABLE);
+		/* Make sure VBLANK and PFLIP interrupts are still enabled */
+		type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
+		amdgpu_irq_update(adev, &adev->crtc_irq, type);
+		amdgpu_irq_update(adev, &adev->pageflip_irq, type);
+		drm_vblank_post_modeset(dev, amdgpu_crtc->crtc_id);
+		dce_v6_0_crtc_load_lut(crtc);
+		break;
+	case DRM_MODE_DPMS_STANDBY:
+	case DRM_MODE_DPMS_SUSPEND:
+	case DRM_MODE_DPMS_OFF:
+		drm_vblank_pre_modeset(dev, amdgpu_crtc->crtc_id);
+		if (amdgpu_crtc->enabled)
+			amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE);
+		amdgpu_atombios_crtc_enable(crtc, ATOM_DISABLE);
+		amdgpu_crtc->enabled = false;
+		break;
+	}
+	/* adjust pm to dpms */
+	amdgpu_pm_compute_clocks(adev);
+}
+
+static void dce_v6_0_crtc_prepare(struct drm_crtc *crtc)
+{
+	/* disable crtc pair power gating before programming */
+	amdgpu_atombios_crtc_powergate(crtc, ATOM_DISABLE);
+	amdgpu_atombios_crtc_lock(crtc, ATOM_ENABLE);
+	dce_v6_0_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+}
+
+static void dce_v6_0_crtc_commit(struct drm_crtc *crtc)
+{
+	dce_v6_0_crtc_dpms(crtc, DRM_MODE_DPMS_ON);
+	amdgpu_atombios_crtc_lock(crtc, ATOM_DISABLE);
+}
+
+static void dce_v6_0_crtc_disable(struct drm_crtc *crtc)
+{
+
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	struct amdgpu_atom_ss ss;
+	int i;
+
+	dce_v6_0_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+	if (crtc->primary->fb) {
+		int r;
+		struct amdgpu_framebuffer *amdgpu_fb;
+		struct amdgpu_bo *abo;
+
+		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
+		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		r = amdgpu_bo_reserve(abo, false);
+		if (unlikely(r))
+			DRM_ERROR("failed to reserve abo before unpin\n");
+		else {
+			amdgpu_bo_unpin(abo);
+			amdgpu_bo_unreserve(abo);
+		}
+	}
+	/* disable the GRPH */
+	dce_v6_0_grph_enable(crtc, false);
+
+	amdgpu_atombios_crtc_powergate(crtc, ATOM_ENABLE);
+
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		if (adev->mode_info.crtcs[i] &&
+		    adev->mode_info.crtcs[i]->enabled &&
+		    i != amdgpu_crtc->crtc_id &&
+		    amdgpu_crtc->pll_id == adev->mode_info.crtcs[i]->pll_id) {
+			/* one other crtc is using this pll don't turn
+			 * off the pll
+			 */
+			goto done;
+		}
+	}
+
+	switch (amdgpu_crtc->pll_id) {
+	case ATOM_PPLL1:
+	case ATOM_PPLL2:
+		/* disable the ppll */
+		amdgpu_atombios_crtc_program_pll(crtc, amdgpu_crtc->crtc_id, amdgpu_crtc->pll_id,
+						 0, 0, ATOM_DISABLE, 0, 0, 0, 0, 0, false, &ss);
+		break;
+	default:
+		break;
+	}
+done:
+	amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
+	amdgpu_crtc->adjusted_clock = 0;
+	amdgpu_crtc->encoder = NULL;
+	amdgpu_crtc->connector = NULL;
+}
+
+static int dce_v6_0_crtc_mode_set(struct drm_crtc *crtc,
+				  struct drm_display_mode *mode,
+				  struct drm_display_mode *adjusted_mode,
+				  int x, int y, struct drm_framebuffer *old_fb)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+	if (!amdgpu_crtc->adjusted_clock)
+		return -EINVAL;
+
+	amdgpu_atombios_crtc_set_pll(crtc, adjusted_mode);
+	amdgpu_atombios_crtc_set_dtd_timing(crtc, adjusted_mode);
+	dce_v6_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
+	amdgpu_atombios_crtc_overscan_setup(crtc, mode, adjusted_mode);
+	amdgpu_atombios_crtc_scaler_setup(crtc);
+	dce_v6_0_cursor_reset(crtc);
+	/* update the hw version fpr dpm */
+	amdgpu_crtc->hw_mode = *adjusted_mode;
+
+	return 0;
+}
+
+static bool dce_v6_0_crtc_mode_fixup(struct drm_crtc *crtc,
+				     const struct drm_display_mode *mode,
+				     struct drm_display_mode *adjusted_mode)
+{
+
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct drm_encoder *encoder;
+
+	/* assign the encoder to the amdgpu crtc to avoid repeated lookups later */
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->crtc == crtc) {
+			amdgpu_crtc->encoder = encoder;
+			amdgpu_crtc->connector = amdgpu_get_connector_for_encoder(encoder);
+			break;
+		}
+	}
+	if ((amdgpu_crtc->encoder == NULL) || (amdgpu_crtc->connector == NULL)) {
+		amdgpu_crtc->encoder = NULL;
+		amdgpu_crtc->connector = NULL;
+		return false;
+	}
+	if (!amdgpu_crtc_scaling_mode_fixup(crtc, mode, adjusted_mode))
+		return false;
+	if (amdgpu_atombios_crtc_prepare_pll(crtc, adjusted_mode))
+		return false;
+	/* pick pll */
+	amdgpu_crtc->pll_id = dce_v6_0_pick_pll(crtc);
+	/* if we can't get a PPLL for a non-DP encoder, fail */
+	if ((amdgpu_crtc->pll_id == ATOM_PPLL_INVALID) &&
+	    !ENCODER_MODE_IS_DP(amdgpu_atombios_encoder_get_encoder_mode(amdgpu_crtc->encoder)))
+		return false;
+
+	return true;
+}
+
+static int dce_v6_0_crtc_set_base(struct drm_crtc *crtc, int x, int y,
+				  struct drm_framebuffer *old_fb)
+{
+	return dce_v6_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
+}
+
+static int dce_v6_0_crtc_set_base_atomic(struct drm_crtc *crtc,
+					 struct drm_framebuffer *fb,
+					 int x, int y, enum mode_set_atomic state)
+{
+       return dce_v6_0_crtc_do_set_base(crtc, fb, x, y, 1);
+}
+
+static const struct drm_crtc_helper_funcs dce_v6_0_crtc_helper_funcs = {
+	.dpms = dce_v6_0_crtc_dpms,
+	.mode_fixup = dce_v6_0_crtc_mode_fixup,
+	.mode_set = dce_v6_0_crtc_mode_set,
+	.mode_set_base = dce_v6_0_crtc_set_base,
+	.mode_set_base_atomic = dce_v6_0_crtc_set_base_atomic,
+	.prepare = dce_v6_0_crtc_prepare,
+	.commit = dce_v6_0_crtc_commit,
+	.load_lut = dce_v6_0_crtc_load_lut,
+	.disable = dce_v6_0_crtc_disable,
+};
+
+static int dce_v6_0_crtc_init(struct amdgpu_device *adev, int index)
+{
+	struct amdgpu_crtc *amdgpu_crtc;
+	int i;
+
+	amdgpu_crtc = kzalloc(sizeof(struct amdgpu_crtc) +
+			      (AMDGPUFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
+	if (amdgpu_crtc == NULL)
+		return -ENOMEM;
+
+	drm_crtc_init(adev->ddev, &amdgpu_crtc->base, &dce_v6_0_crtc_funcs);
+
+	drm_mode_crtc_set_gamma_size(&amdgpu_crtc->base, 256);
+	amdgpu_crtc->crtc_id = index;
+	adev->mode_info.crtcs[index] = amdgpu_crtc;
+
+	amdgpu_crtc->max_cursor_width = CURSOR_WIDTH;
+	amdgpu_crtc->max_cursor_height = CURSOR_HEIGHT;
+	adev->ddev->mode_config.cursor_width = amdgpu_crtc->max_cursor_width;
+	adev->ddev->mode_config.cursor_height = amdgpu_crtc->max_cursor_height;
+
+	for (i = 0; i < 256; i++) {
+		amdgpu_crtc->lut_r[i] = i << 2;
+		amdgpu_crtc->lut_g[i] = i << 2;
+		amdgpu_crtc->lut_b[i] = i << 2;
+	}
+
+	amdgpu_crtc->crtc_offset = crtc_offsets[amdgpu_crtc->crtc_id];
+
+	amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
+	amdgpu_crtc->adjusted_clock = 0;
+	amdgpu_crtc->encoder = NULL;
+	amdgpu_crtc->connector = NULL;
+	drm_crtc_helper_add(&amdgpu_crtc->base, &dce_v6_0_crtc_helper_funcs);
+
+	return 0;
+}
+
+static int dce_v6_0_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	adev->audio_endpt_rreg = &dce_v6_0_audio_endpt_rreg;
+	adev->audio_endpt_wreg = &dce_v6_0_audio_endpt_wreg;
+
+	dce_v6_0_set_display_funcs(adev);
+	dce_v6_0_set_irq_funcs(adev);
+
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+	case CHIP_VERDE:
+		adev->mode_info.num_crtc = 6;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 6;
+		break;
+	case CHIP_OLAND:
+		adev->mode_info.num_crtc = 2;
+		adev->mode_info.num_hpd = 2;
+		adev->mode_info.num_dig = 2;
+		break;
+	default:
+		/* FIXME: not supported yet */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int dce_v6_0_sw_init(void *handle)
+{
+	int r, i;
+	bool ret;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		r = amdgpu_irq_add_id(adev, i + 1, &adev->crtc_irq);
+		if (r)
+			return r;
+	}
+
+	for (i = 8; i < 20; i += 2) {
+		r = amdgpu_irq_add_id(adev, i, &adev->pageflip_irq);
+		if (r)
+			return r;
+	}
+
+	/* HPD hotplug */
+	r = amdgpu_irq_add_id(adev, 42, &adev->hpd_irq);
+	if (r)
+		return r;
+
+	adev->mode_info.mode_config_initialized = true;
+
+	adev->ddev->mode_config.funcs = &amdgpu_mode_funcs;
+	adev->ddev->mode_config.async_page_flip = true;
+	adev->ddev->mode_config.max_width = 16384;
+	adev->ddev->mode_config.max_height = 16384;
+	adev->ddev->mode_config.preferred_depth = 24;
+	adev->ddev->mode_config.prefer_shadow = 1;
+	adev->ddev->mode_config.fb_base = adev->mc.aper_base;
+
+	r = amdgpu_modeset_create_props(adev);
+	if (r)
+		return r;
+
+	adev->ddev->mode_config.max_width = 16384;
+	adev->ddev->mode_config.max_height = 16384;
+
+	/* allocate crtcs */
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		r = dce_v6_0_crtc_init(adev, i);
+		if (r)
+			return r;
+	}
+
+	ret = amdgpu_atombios_get_connector_info_from_object_table(adev);
+	if (ret)
+		amdgpu_print_display_setup(adev->ddev);
+	else
+		return -EINVAL;
+
+	/* setup afmt */
+	r = dce_v6_0_afmt_init(adev);
+	if (r)
+		return r;
+
+	r = dce_v6_0_audio_init(adev);
+	if (r)
+		return r;
+
+	drm_kms_helper_poll_init(adev->ddev);
+
+	return r;
+}
+
+static int dce_v6_0_sw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	kfree(adev->mode_info.bios_hardcoded_edid);
+
+	drm_kms_helper_poll_fini(adev->ddev);
+
+	dce_v6_0_audio_fini(adev);
+	dce_v6_0_afmt_fini(adev);
+
+	drm_mode_config_cleanup(adev->ddev);
+	adev->mode_info.mode_config_initialized = false;
+
+	return 0;
+}
+
+static int dce_v6_0_hw_init(void *handle)
+{
+	int i;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	/* init dig PHYs, disp eng pll */
+	amdgpu_atombios_encoder_init_dig(adev);
+	amdgpu_atombios_crtc_set_disp_eng_pll(adev, adev->clock.default_dispclk);
+
+	/* initialize hpd */
+	dce_v6_0_hpd_init(adev);
+
+	for (i = 0; i < adev->mode_info.audio.num_pins; i++) {
+		dce_v6_0_audio_enable(adev, &adev->mode_info.audio.pin[i], false);
+	}
+
+	dce_v6_0_pageflip_interrupt_init(adev);
+
+	return 0;
+}
+
+static int dce_v6_0_hw_fini(void *handle)
+{
+	int i;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	dce_v6_0_hpd_fini(adev);
+
+	for (i = 0; i < adev->mode_info.audio.num_pins; i++) {
+		dce_v6_0_audio_enable(adev, &adev->mode_info.audio.pin[i], false);
+	}
+
+	dce_v6_0_pageflip_interrupt_fini(adev);
+
+	return 0;
+}
+
+static int dce_v6_0_suspend(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	amdgpu_atombios_scratch_regs_save(adev);
+
+	return dce_v6_0_hw_fini(handle);
+}
+
+static int dce_v6_0_resume(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int ret;
+
+	ret = dce_v6_0_hw_init(handle);
+
+	amdgpu_atombios_scratch_regs_restore(adev);
+
+	/* turn on the BL */
+	if (adev->mode_info.bl_encoder) {
+		u8 bl_level = amdgpu_display_backlight_get_level(adev,
+								  adev->mode_info.bl_encoder);
+		amdgpu_display_backlight_set_level(adev, adev->mode_info.bl_encoder,
+						    bl_level);
+	}
+
+	return ret;
+}
+
+static bool dce_v6_0_is_idle(void *handle)
+{
+	return true;
+}
+
+static int dce_v6_0_wait_for_idle(void *handle)
+{
+	return 0;
+}
+
+static int dce_v6_0_soft_reset(void *handle)
+{
+	DRM_INFO("xxxx: dce_v6_0_soft_reset --- no impl!!\n");
+	return 0;
+}
+
+static void dce_v6_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
+						     int crtc,
+						     enum amdgpu_interrupt_state state)
+{
+	u32 reg_block, interrupt_mask;
+
+	if (crtc >= adev->mode_info.num_crtc) {
+		DRM_DEBUG("invalid crtc %d\n", crtc);
+		return;
+	}
+
+	switch (crtc) {
+	case 0:
+		reg_block = SI_CRTC0_REGISTER_OFFSET;
+		break;
+	case 1:
+		reg_block = SI_CRTC1_REGISTER_OFFSET;
+		break;
+	case 2:
+		reg_block = SI_CRTC2_REGISTER_OFFSET;
+		break;
+	case 3:
+		reg_block = SI_CRTC3_REGISTER_OFFSET;
+		break;
+	case 4:
+		reg_block = SI_CRTC4_REGISTER_OFFSET;
+		break;
+	case 5:
+		reg_block = SI_CRTC5_REGISTER_OFFSET;
+		break;
+	default:
+		DRM_DEBUG("invalid crtc %d\n", crtc);
+		return;
+	}
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		interrupt_mask = RREG32(INT_MASK + reg_block);
+		interrupt_mask &= ~VBLANK_INT_MASK;
+		WREG32(INT_MASK + reg_block, interrupt_mask);
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		interrupt_mask = RREG32(INT_MASK + reg_block);
+		interrupt_mask |= VBLANK_INT_MASK;
+		WREG32(INT_MASK + reg_block, interrupt_mask);
+		break;
+	default:
+		break;
+	}
+}
+
+static void dce_v6_0_set_crtc_vline_interrupt_state(struct amdgpu_device *adev,
+						    int crtc,
+						    enum amdgpu_interrupt_state state)
+{
+
+}
+
+static int dce_v6_0_set_hpd_interrupt_state(struct amdgpu_device *adev,
+					    struct amdgpu_irq_src *src,
+					    unsigned type,
+					    enum amdgpu_interrupt_state state)
+{
+	u32 dc_hpd_int_cntl_reg, dc_hpd_int_cntl;
+
+	switch (type) {
+	case AMDGPU_HPD_1:
+		dc_hpd_int_cntl_reg = DC_HPD1_INT_CONTROL;
+		break;
+	case AMDGPU_HPD_2:
+		dc_hpd_int_cntl_reg = DC_HPD2_INT_CONTROL;
+		break;
+	case AMDGPU_HPD_3:
+		dc_hpd_int_cntl_reg = DC_HPD3_INT_CONTROL;
+		break;
+	case AMDGPU_HPD_4:
+		dc_hpd_int_cntl_reg = DC_HPD4_INT_CONTROL;
+		break;
+	case AMDGPU_HPD_5:
+		dc_hpd_int_cntl_reg = DC_HPD5_INT_CONTROL;
+		break;
+	case AMDGPU_HPD_6:
+		dc_hpd_int_cntl_reg = DC_HPD6_INT_CONTROL;
+		break;
+	default:
+		DRM_DEBUG("invalid hdp %d\n", type);
+		return 0;
+	}
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
+		dc_hpd_int_cntl &= ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
+		WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
+		dc_hpd_int_cntl |= (DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
+		WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int dce_v6_0_set_crtc_interrupt_state(struct amdgpu_device *adev,
+					     struct amdgpu_irq_src *src,
+					     unsigned type,
+					     enum amdgpu_interrupt_state state)
+{
+	switch (type) {
+	case AMDGPU_CRTC_IRQ_VBLANK1:
+		dce_v6_0_set_crtc_vblank_interrupt_state(adev, 0, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VBLANK2:
+		dce_v6_0_set_crtc_vblank_interrupt_state(adev, 1, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VBLANK3:
+		dce_v6_0_set_crtc_vblank_interrupt_state(adev, 2, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VBLANK4:
+		dce_v6_0_set_crtc_vblank_interrupt_state(adev, 3, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VBLANK5:
+		dce_v6_0_set_crtc_vblank_interrupt_state(adev, 4, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VBLANK6:
+		dce_v6_0_set_crtc_vblank_interrupt_state(adev, 5, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VLINE1:
+		dce_v6_0_set_crtc_vline_interrupt_state(adev, 0, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VLINE2:
+		dce_v6_0_set_crtc_vline_interrupt_state(adev, 1, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VLINE3:
+		dce_v6_0_set_crtc_vline_interrupt_state(adev, 2, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VLINE4:
+		dce_v6_0_set_crtc_vline_interrupt_state(adev, 3, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VLINE5:
+		dce_v6_0_set_crtc_vline_interrupt_state(adev, 4, state);
+		break;
+	case AMDGPU_CRTC_IRQ_VLINE6:
+		dce_v6_0_set_crtc_vline_interrupt_state(adev, 5, state);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int dce_v6_0_crtc_irq(struct amdgpu_device *adev,
+			     struct amdgpu_irq_src *source,
+			     struct amdgpu_iv_entry *entry)
+{
+	unsigned crtc = entry->src_id - 1;
+	uint32_t disp_int = RREG32(interrupt_status_offsets[crtc].reg);
+	unsigned irq_type = amdgpu_crtc_idx_to_irq_type(adev, crtc);
+
+	switch (entry->src_data) {
+	case 0: /* vblank */
+		if (disp_int & interrupt_status_offsets[crtc].vblank)
+			WREG32(VBLANK_STATUS + crtc_offsets[crtc], VBLANK_ACK);
+		else
+			DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
+
+		if (amdgpu_irq_enabled(adev, source, irq_type)) {
+			drm_handle_vblank(adev->ddev, crtc);
+		}
+		DRM_DEBUG("IH: D%d vblank\n", crtc + 1);
+		break;
+	case 1: /* vline */
+		if (disp_int & interrupt_status_offsets[crtc].vline)
+			WREG32(VLINE_STATUS + crtc_offsets[crtc], VLINE_ACK);
+		else
+			DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
+
+		DRM_DEBUG("IH: D%d vline\n", crtc + 1);
+		break;
+	default:
+		DRM_DEBUG("Unhandled interrupt: %d %d\n", entry->src_id, entry->src_data);
+		break;
+	}
+
+	return 0;
+}
+
+static int dce_v6_0_set_pageflip_interrupt_state(struct amdgpu_device *adev,
+						 struct amdgpu_irq_src *src,
+						 unsigned type,
+						 enum amdgpu_interrupt_state state)
+{
+	u32 reg;
+
+	if (type >= adev->mode_info.num_crtc) {
+		DRM_ERROR("invalid pageflip crtc %d\n", type);
+		return -EINVAL;
+	}
+
+	reg = RREG32(GRPH_INT_CONTROL + crtc_offsets[type]);
+	if (state == AMDGPU_IRQ_STATE_DISABLE)
+		WREG32(GRPH_INT_CONTROL + crtc_offsets[type],
+		       reg & ~GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
+	else
+		WREG32(GRPH_INT_CONTROL + crtc_offsets[type],
+		       reg | GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
+
+	return 0;
+}
+
+static int dce_v6_0_pageflip_irq(struct amdgpu_device *adev,
+				 struct amdgpu_irq_src *source,
+				 struct amdgpu_iv_entry *entry)
+{
+		unsigned long flags;
+	unsigned crtc_id;
+	struct amdgpu_crtc *amdgpu_crtc;
+	struct amdgpu_flip_work *works;
+
+	crtc_id = (entry->src_id - 8) >> 1;
+	amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
+
+	if (crtc_id >= adev->mode_info.num_crtc) {
+		DRM_ERROR("invalid pageflip crtc %d\n", crtc_id);
+		return -EINVAL;
+	}
+
+	if (RREG32(GRPH_INT_STATUS + crtc_offsets[crtc_id]) &
+	    GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK)
+		WREG32(GRPH_INT_STATUS + crtc_offsets[crtc_id],
+		       GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK);
+
+	/* IRQ could occur when in initial stage */
+	if (amdgpu_crtc == NULL)
+		return 0;
+
+	spin_lock_irqsave(&adev->ddev->event_lock, flags);
+	works = amdgpu_crtc->pflip_works;
+	if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED){
+		DRM_DEBUG_DRIVER("amdgpu_crtc->pflip_status = %d != "
+						"AMDGPU_FLIP_SUBMITTED(%d)\n",
+						amdgpu_crtc->pflip_status,
+						AMDGPU_FLIP_SUBMITTED);
+		spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
+		return 0;
+	}
+
+	/* page flip completed. clean up */
+	amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE;
+	amdgpu_crtc->pflip_works = NULL;
+
+	/* wakeup usersapce */
+	if (works->event)
+		drm_crtc_send_vblank_event(&amdgpu_crtc->base, works->event);
+
+	spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
+
+	drm_crtc_vblank_put(&amdgpu_crtc->base);
+	schedule_work(&works->unpin_work);
+
+	return 0;
+}
+
+static int dce_v6_0_hpd_irq(struct amdgpu_device *adev,
+			    struct amdgpu_irq_src *source,
+			    struct amdgpu_iv_entry *entry)
+{
+	uint32_t disp_int, mask, int_control, tmp;
+	unsigned hpd;
+
+	if (entry->src_data >= adev->mode_info.num_hpd) {
+		DRM_DEBUG("Unhandled interrupt: %d %d\n", entry->src_id, entry->src_data);
+		return 0;
+	}
+
+	hpd = entry->src_data;
+	disp_int = RREG32(interrupt_status_offsets[hpd].reg);
+	mask = interrupt_status_offsets[hpd].hpd;
+	int_control = hpd_int_control_offsets[hpd];
+
+	if (disp_int & mask) {
+		tmp = RREG32(int_control);
+		tmp |= DC_HPD1_INT_CONTROL__DC_HPD1_INT_ACK_MASK;
+		WREG32(int_control, tmp);
+		schedule_work(&adev->hotplug_work);
+		DRM_INFO("IH: HPD%d\n", hpd + 1);
+	}
+
+	return 0;
+
+}
+
+static int dce_v6_0_set_clockgating_state(void *handle,
+					  enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int dce_v6_0_set_powergating_state(void *handle,
+					  enum amd_powergating_state state)
+{
+	return 0;
+}
+
+const struct amd_ip_funcs dce_v6_0_ip_funcs = {
+	.name = "dce_v6_0",
+	.early_init = dce_v6_0_early_init,
+	.late_init = NULL,
+	.sw_init = dce_v6_0_sw_init,
+	.sw_fini = dce_v6_0_sw_fini,
+	.hw_init = dce_v6_0_hw_init,
+	.hw_fini = dce_v6_0_hw_fini,
+	.suspend = dce_v6_0_suspend,
+	.resume = dce_v6_0_resume,
+	.is_idle = dce_v6_0_is_idle,
+	.wait_for_idle = dce_v6_0_wait_for_idle,
+	.soft_reset = dce_v6_0_soft_reset,
+	.set_clockgating_state = dce_v6_0_set_clockgating_state,
+	.set_powergating_state = dce_v6_0_set_powergating_state,
+};
+
+static void
+dce_v6_0_encoder_mode_set(struct drm_encoder *encoder,
+			  struct drm_display_mode *mode,
+			  struct drm_display_mode *adjusted_mode)
+{
+
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+
+	amdgpu_encoder->pixel_clock = adjusted_mode->clock;
+
+	/* need to call this here rather than in prepare() since we need some crtc info */
+	amdgpu_atombios_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
+
+	/* set scaler clears this on some chips */
+	dce_v6_0_set_interleave(encoder->crtc, mode);
+
+	if (amdgpu_atombios_encoder_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_HDMI) {
+		dce_v6_0_afmt_enable(encoder, true);
+		dce_v6_0_afmt_setmode(encoder, adjusted_mode);
+	}
+}
+
+static void dce_v6_0_encoder_prepare(struct drm_encoder *encoder)
+{
+
+	struct amdgpu_device *adev = encoder->dev->dev_private;
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+	struct drm_connector *connector = amdgpu_get_connector_for_encoder(encoder);
+
+	if ((amdgpu_encoder->active_device &
+	     (ATOM_DEVICE_DFP_SUPPORT | ATOM_DEVICE_LCD_SUPPORT)) ||
+	    (amdgpu_encoder_get_dp_bridge_encoder_id(encoder) !=
+	     ENCODER_OBJECT_ID_NONE)) {
+		struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv;
+		if (dig) {
+			dig->dig_encoder = dce_v6_0_pick_dig_encoder(encoder);
+			if (amdgpu_encoder->active_device & ATOM_DEVICE_DFP_SUPPORT)
+				dig->afmt = adev->mode_info.afmt[dig->dig_encoder];
+		}
+	}
+
+	amdgpu_atombios_scratch_regs_lock(adev, true);
+
+	if (connector) {
+		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
+
+		/* select the clock/data port if it uses a router */
+		if (amdgpu_connector->router.cd_valid)
+			amdgpu_i2c_router_select_cd_port(amdgpu_connector);
+
+		/* turn eDP panel on for mode set */
+		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+			amdgpu_atombios_encoder_set_edp_panel_power(connector,
+							     ATOM_TRANSMITTER_ACTION_POWER_ON);
+	}
+
+	/* this is needed for the pll/ss setup to work correctly in some cases */
+	amdgpu_atombios_encoder_set_crtc_source(encoder);
+	/* set up the FMT blocks */
+	dce_v6_0_program_fmt(encoder);
+}
+
+static void dce_v6_0_encoder_commit(struct drm_encoder *encoder)
+{
+
+	struct drm_device *dev = encoder->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+
+	/* need to call this here as we need the crtc set up */
+	amdgpu_atombios_encoder_dpms(encoder, DRM_MODE_DPMS_ON);
+	amdgpu_atombios_scratch_regs_lock(adev, false);
+}
+
+static void dce_v6_0_encoder_disable(struct drm_encoder *encoder)
+{
+
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+	struct amdgpu_encoder_atom_dig *dig;
+
+	amdgpu_atombios_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
+
+	if (amdgpu_atombios_encoder_is_digital(encoder)) {
+		if (amdgpu_atombios_encoder_get_encoder_mode(encoder) == ATOM_ENCODER_MODE_HDMI)
+			dce_v6_0_afmt_enable(encoder, false);
+		dig = amdgpu_encoder->enc_priv;
+		dig->dig_encoder = -1;
+	}
+	amdgpu_encoder->active_device = 0;
+}
+
+/* these are handled by the primary encoders */
+static void dce_v6_0_ext_prepare(struct drm_encoder *encoder)
+{
+
+}
+
+static void dce_v6_0_ext_commit(struct drm_encoder *encoder)
+{
+
+}
+
+static void
+dce_v6_0_ext_mode_set(struct drm_encoder *encoder,
+		      struct drm_display_mode *mode,
+		      struct drm_display_mode *adjusted_mode)
+{
+
+}
+
+static void dce_v6_0_ext_disable(struct drm_encoder *encoder)
+{
+
+}
+
+static void
+dce_v6_0_ext_dpms(struct drm_encoder *encoder, int mode)
+{
+
+}
+
+static bool dce_v6_0_ext_mode_fixup(struct drm_encoder *encoder,
+				    const struct drm_display_mode *mode,
+				    struct drm_display_mode *adjusted_mode)
+{
+	return true;
+}
+
+static const struct drm_encoder_helper_funcs dce_v6_0_ext_helper_funcs = {
+	.dpms = dce_v6_0_ext_dpms,
+	.mode_fixup = dce_v6_0_ext_mode_fixup,
+	.prepare = dce_v6_0_ext_prepare,
+	.mode_set = dce_v6_0_ext_mode_set,
+	.commit = dce_v6_0_ext_commit,
+	.disable = dce_v6_0_ext_disable,
+	/* no detect for TMDS/LVDS yet */
+};
+
+static const struct drm_encoder_helper_funcs dce_v6_0_dig_helper_funcs = {
+	.dpms = amdgpu_atombios_encoder_dpms,
+	.mode_fixup = amdgpu_atombios_encoder_mode_fixup,
+	.prepare = dce_v6_0_encoder_prepare,
+	.mode_set = dce_v6_0_encoder_mode_set,
+	.commit = dce_v6_0_encoder_commit,
+	.disable = dce_v6_0_encoder_disable,
+	.detect = amdgpu_atombios_encoder_dig_detect,
+};
+
+static const struct drm_encoder_helper_funcs dce_v6_0_dac_helper_funcs = {
+	.dpms = amdgpu_atombios_encoder_dpms,
+	.mode_fixup = amdgpu_atombios_encoder_mode_fixup,
+	.prepare = dce_v6_0_encoder_prepare,
+	.mode_set = dce_v6_0_encoder_mode_set,
+	.commit = dce_v6_0_encoder_commit,
+	.detect = amdgpu_atombios_encoder_dac_detect,
+};
+
+static void dce_v6_0_encoder_destroy(struct drm_encoder *encoder)
+{
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+	if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+		amdgpu_atombios_encoder_fini_backlight(amdgpu_encoder);
+	kfree(amdgpu_encoder->enc_priv);
+	drm_encoder_cleanup(encoder);
+	kfree(amdgpu_encoder);
+}
+
+static const struct drm_encoder_funcs dce_v6_0_encoder_funcs = {
+	.destroy = dce_v6_0_encoder_destroy,
+};
+
+static void dce_v6_0_encoder_add(struct amdgpu_device *adev,
+				 uint32_t encoder_enum,
+				 uint32_t supported_device,
+				 u16 caps)
+{
+	struct drm_device *dev = adev->ddev;
+	struct drm_encoder *encoder;
+	struct amdgpu_encoder *amdgpu_encoder;
+
+	/* see if we already added it */
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		amdgpu_encoder = to_amdgpu_encoder(encoder);
+		if (amdgpu_encoder->encoder_enum == encoder_enum) {
+			amdgpu_encoder->devices |= supported_device;
+			return;
+		}
+
+	}
+
+	/* add a new one */
+	amdgpu_encoder = kzalloc(sizeof(struct amdgpu_encoder), GFP_KERNEL);
+	if (!amdgpu_encoder)
+		return;
+
+	encoder = &amdgpu_encoder->base;
+	switch (adev->mode_info.num_crtc) {
+	case 1:
+		encoder->possible_crtcs = 0x1;
+		break;
+	case 2:
+	default:
+		encoder->possible_crtcs = 0x3;
+		break;
+	case 4:
+		encoder->possible_crtcs = 0xf;
+		break;
+	case 6:
+		encoder->possible_crtcs = 0x3f;
+		break;
+	}
+
+	amdgpu_encoder->enc_priv = NULL;
+	amdgpu_encoder->encoder_enum = encoder_enum;
+	amdgpu_encoder->encoder_id = (encoder_enum & OBJECT_ID_MASK) >> OBJECT_ID_SHIFT;
+	amdgpu_encoder->devices = supported_device;
+	amdgpu_encoder->rmx_type = RMX_OFF;
+	amdgpu_encoder->underscan_type = UNDERSCAN_OFF;
+	amdgpu_encoder->is_ext_encoder = false;
+	amdgpu_encoder->caps = caps;
+
+	switch (amdgpu_encoder->encoder_id) {
+	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
+	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2:
+		drm_encoder_init(dev, encoder, &dce_v6_0_encoder_funcs,
+				 DRM_MODE_ENCODER_DAC, NULL);
+		drm_encoder_helper_add(encoder, &dce_v6_0_dac_helper_funcs);
+		break;
+	case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
+	case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
+	case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
+	case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+	case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
+		if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
+			amdgpu_encoder->rmx_type = RMX_FULL;
+			drm_encoder_init(dev, encoder, &dce_v6_0_encoder_funcs,
+					 DRM_MODE_ENCODER_LVDS, NULL);
+			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_lcd_info(amdgpu_encoder);
+		} else if (amdgpu_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT)) {
+			drm_encoder_init(dev, encoder, &dce_v6_0_encoder_funcs,
+					 DRM_MODE_ENCODER_DAC, NULL);
+			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_dig_info(amdgpu_encoder);
+		} else {
+			drm_encoder_init(dev, encoder, &dce_v6_0_encoder_funcs,
+					 DRM_MODE_ENCODER_TMDS, NULL);
+			amdgpu_encoder->enc_priv = amdgpu_atombios_encoder_get_dig_info(amdgpu_encoder);
+		}
+		drm_encoder_helper_add(encoder, &dce_v6_0_dig_helper_funcs);
+		break;
+	case ENCODER_OBJECT_ID_SI170B:
+	case ENCODER_OBJECT_ID_CH7303:
+	case ENCODER_OBJECT_ID_EXTERNAL_SDVOA:
+	case ENCODER_OBJECT_ID_EXTERNAL_SDVOB:
+	case ENCODER_OBJECT_ID_TITFP513:
+	case ENCODER_OBJECT_ID_VT1623:
+	case ENCODER_OBJECT_ID_HDMI_SI1930:
+	case ENCODER_OBJECT_ID_TRAVIS:
+	case ENCODER_OBJECT_ID_NUTMEG:
+		/* these are handled by the primary encoders */
+		amdgpu_encoder->is_ext_encoder = true;
+		if (amdgpu_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+			drm_encoder_init(dev, encoder, &dce_v6_0_encoder_funcs,
+					 DRM_MODE_ENCODER_LVDS, NULL);
+		else if (amdgpu_encoder->devices & (ATOM_DEVICE_CRT_SUPPORT))
+			drm_encoder_init(dev, encoder, &dce_v6_0_encoder_funcs,
+					 DRM_MODE_ENCODER_DAC, NULL);
+		else
+			drm_encoder_init(dev, encoder, &dce_v6_0_encoder_funcs,
+					 DRM_MODE_ENCODER_TMDS, NULL);
+		drm_encoder_helper_add(encoder, &dce_v6_0_ext_helper_funcs);
+		break;
+	}
+}
+
+static const struct amdgpu_display_funcs dce_v6_0_display_funcs = {
+	.set_vga_render_state = &dce_v6_0_set_vga_render_state,
+	.bandwidth_update = &dce_v6_0_bandwidth_update,
+	.vblank_get_counter = &dce_v6_0_vblank_get_counter,
+	.vblank_wait = &dce_v6_0_vblank_wait,
+	.is_display_hung = &dce_v6_0_is_display_hung,
+	.backlight_set_level = &amdgpu_atombios_encoder_set_backlight_level,
+	.backlight_get_level = &amdgpu_atombios_encoder_get_backlight_level,
+	.hpd_sense = &dce_v6_0_hpd_sense,
+	.hpd_set_polarity = &dce_v6_0_hpd_set_polarity,
+	.hpd_get_gpio_reg = &dce_v6_0_hpd_get_gpio_reg,
+	.page_flip = &dce_v6_0_page_flip,
+	.page_flip_get_scanoutpos = &dce_v6_0_crtc_get_scanoutpos,
+	.add_encoder = &dce_v6_0_encoder_add,
+	.add_connector = &amdgpu_connector_add,
+	.stop_mc_access = &dce_v6_0_stop_mc_access,
+	.resume_mc_access = &dce_v6_0_resume_mc_access,
+};
+
+static void dce_v6_0_set_display_funcs(struct amdgpu_device *adev)
+{
+	if (adev->mode_info.funcs == NULL)
+		adev->mode_info.funcs = &dce_v6_0_display_funcs;
+}
+
+static const struct amdgpu_irq_src_funcs dce_v6_0_crtc_irq_funcs = {
+	.set = dce_v6_0_set_crtc_interrupt_state,
+	.process = dce_v6_0_crtc_irq,
+};
+
+static const struct amdgpu_irq_src_funcs dce_v6_0_pageflip_irq_funcs = {
+	.set = dce_v6_0_set_pageflip_interrupt_state,
+	.process = dce_v6_0_pageflip_irq,
+};
+
+static const struct amdgpu_irq_src_funcs dce_v6_0_hpd_irq_funcs = {
+	.set = dce_v6_0_set_hpd_interrupt_state,
+	.process = dce_v6_0_hpd_irq,
+};
+
+static void dce_v6_0_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST;
+	adev->crtc_irq.funcs = &dce_v6_0_crtc_irq_funcs;
+
+	adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST;
+	adev->pageflip_irq.funcs = &dce_v6_0_pageflip_irq_funcs;
+
+	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
+	adev->hpd_irq.funcs = &dce_v6_0_hpd_irq_funcs;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_smum.h b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.h
index c031ff99fe3e..6a5528105bb6 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_smum.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014 Advanced Micro Devices, Inc.
+ * Copyright 2015 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -21,22 +21,9 @@
  *
  */
 
-#ifndef TONGA_SMUMGR_H
-#define TONGA_SMUMGR_H
+#ifndef __DCE_V6_0_H__
+#define __DCE_V6_0_H__
 
-#include "tonga_ppsmc.h"
-
-int tonga_smu_init(struct amdgpu_device *adev);
-int tonga_smu_fini(struct amdgpu_device *adev);
-int tonga_smu_start(struct amdgpu_device *adev);
-
-struct tonga_smu_private_data
-{
-	uint8_t *header;
-	uint32_t smu_buffer_addr_high;
-	uint32_t smu_buffer_addr_low;
-	uint32_t header_addr_high;
-	uint32_t header_addr_low;
-};
+extern const struct amd_ip_funcs dce_v6_0_ip_funcs;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 4fdfab1e9200..5966166ec94c 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -170,7 +170,7 @@ static bool dce_v8_0_is_counter_moving(struct amdgpu_device *adev, int crtc)
  */
 static void dce_v8_0_vblank_wait(struct amdgpu_device *adev, int crtc)
 {
-	unsigned i = 0;
+	unsigned i = 100;
 
 	if (crtc >= adev->mode_info.num_crtc)
 		return;
@@ -182,14 +182,16 @@ static void dce_v8_0_vblank_wait(struct amdgpu_device *adev, int crtc)
 	 * wait for another frame.
 	 */
 	while (dce_v8_0_is_in_vblank(adev, crtc)) {
-		if (i++ % 100 == 0) {
+		if (i++ == 100) {
+			i = 0;
 			if (!dce_v8_0_is_counter_moving(adev, crtc))
 				break;
 		}
 	}
 
 	while (!dce_v8_0_is_in_vblank(adev, crtc)) {
-		if (i++ % 100 == 0) {
+		if (i++ == 100) {
+			i = 0;
 			if (!dce_v8_0_is_counter_moving(adev, crtc))
 				break;
 		}
@@ -395,15 +397,6 @@ static void dce_v8_0_hpd_init(struct amdgpu_device *adev)
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
-		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
-			/* don't try to enable hpd on eDP or LVDS avoid breaking the
-			 * aux dp channel on imac and help (but not completely fix)
-			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
-			 * also avoid interrupt storms during dpms.
-			 */
-			continue;
-		}
 		switch (amdgpu_connector->hpd.hpd) {
 		case AMDGPU_HPD_1:
 			WREG32(mmDC_HPD1_CONTROL, tmp);
@@ -426,6 +419,45 @@ static void dce_v8_0_hpd_init(struct amdgpu_device *adev)
 		default:
 			break;
 		}
+
+		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
+		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
+			/* don't try to enable hpd on eDP or LVDS avoid breaking the
+			 * aux dp channel on imac and help (but not completely fix)
+			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
+			 * also avoid interrupt storms during dpms.
+			 */
+			u32 dc_hpd_int_cntl_reg, dc_hpd_int_cntl;
+
+			switch (amdgpu_connector->hpd.hpd) {
+			case AMDGPU_HPD_1:
+				dc_hpd_int_cntl_reg = mmDC_HPD1_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_2:
+				dc_hpd_int_cntl_reg = mmDC_HPD2_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_3:
+				dc_hpd_int_cntl_reg = mmDC_HPD3_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_4:
+				dc_hpd_int_cntl_reg = mmDC_HPD4_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_5:
+				dc_hpd_int_cntl_reg = mmDC_HPD5_INT_CONTROL;
+				break;
+			case AMDGPU_HPD_6:
+				dc_hpd_int_cntl_reg = mmDC_HPD6_INT_CONTROL;
+				break;
+			default:
+				continue;
+			}
+
+			dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
+			dc_hpd_int_cntl &= ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK;
+			WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+			continue;
+		}
+
 		dce_v8_0_hpd_set_polarity(adev, amdgpu_connector->hpd.hpd);
 		amdgpu_irq_get(adev, &adev->hpd_irq, amdgpu_connector->hpd.hpd);
 	}
@@ -604,6 +636,52 @@ static void dce_v8_0_set_vga_render_state(struct amdgpu_device *adev,
 	WREG32(mmVGA_RENDER_CONTROL, tmp);
 }
 
+static int dce_v8_0_get_num_crtc(struct amdgpu_device *adev)
+{
+	int num_crtc = 0;
+
+	switch (adev->asic_type) {
+	case CHIP_BONAIRE:
+	case CHIP_HAWAII:
+		num_crtc = 6;
+		break;
+	case CHIP_KAVERI:
+		num_crtc = 4;
+		break;
+	case CHIP_KABINI:
+	case CHIP_MULLINS:
+		num_crtc = 2;
+		break;
+	default:
+		num_crtc = 0;
+	}
+	return num_crtc;
+}
+
+void dce_v8_0_disable_dce(struct amdgpu_device *adev)
+{
+	/*Disable VGA render and enabled crtc, if has DCE engine*/
+	if (amdgpu_atombios_has_dce_engine_info(adev)) {
+		u32 tmp;
+		int crtc_enabled, i;
+
+		dce_v8_0_set_vga_render_state(adev, false);
+
+		/*Disable crtc*/
+		for (i = 0; i < dce_v8_0_get_num_crtc(adev); i++) {
+			crtc_enabled = REG_GET_FIELD(RREG32(mmCRTC_CONTROL + crtc_offsets[i]),
+									 CRTC_CONTROL, CRTC_MASTER_EN);
+			if (crtc_enabled) {
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
+				tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
+				tmp = REG_SET_FIELD(tmp, CRTC_CONTROL, CRTC_MASTER_EN, 0);
+				WREG32(mmCRTC_CONTROL + crtc_offsets[i], tmp);
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
+			}
+		}
+	}
+}
+
 static void dce_v8_0_program_fmt(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
@@ -1501,13 +1579,13 @@ static void dce_v8_0_audio_write_sad_regs(struct drm_encoder *encoder)
 
 			if (sad->format == eld_reg_to_type[i][1]) {
 				if (sad->channels > max_channels) {
-				value = (sad->channels <<
-				 AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT) |
-				(sad->byte2 <<
-				 AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT) |
-				(sad->freq <<
-				 AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT);
-				max_channels = sad->channels;
+					value = (sad->channels <<
+						 AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT) |
+					        (sad->byte2 <<
+						 AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT) |
+					        (sad->freq <<
+						 AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT);
+					max_channels = sad->channels;
 				}
 
 				if (sad->format == HDMI_AUDIO_CODING_TYPE_PCM)
@@ -1613,7 +1691,7 @@ static void dce_v8_0_afmt_update_ACR(struct drm_encoder *encoder, uint32_t clock
 	struct amdgpu_encoder_atom_dig *dig = amdgpu_encoder->enc_priv;
 	uint32_t offset = dig->afmt->offset;
 
-	WREG32(mmHDMI_ACR_32_0 + offset, (acr.cts_32khz << HDMI_ACR_44_0__HDMI_ACR_CTS_44__SHIFT));
+	WREG32(mmHDMI_ACR_32_0 + offset, (acr.cts_32khz << HDMI_ACR_32_0__HDMI_ACR_CTS_32__SHIFT));
 	WREG32(mmHDMI_ACR_32_1 + offset, acr.n_32khz);
 
 	WREG32(mmHDMI_ACR_44_0 + offset, (acr.cts_44_1khz << HDMI_ACR_44_0__HDMI_ACR_CTS_44__SHIFT));
@@ -1693,6 +1771,7 @@ static void dce_v8_0_afmt_setmode(struct drm_encoder *encoder,
 	/* Silent, r600_hdmi_enable will raise WARN for us */
 	if (!dig->afmt->enabled)
 		return;
+
 	offset = dig->afmt->offset;
 
 	/* hdmi deep color mode general control packets setup, if bpc > 8 */
@@ -1817,7 +1896,7 @@ static void dce_v8_0_afmt_setmode(struct drm_encoder *encoder,
 
 	WREG32_OR(mmHDMI_INFOFRAME_CONTROL0 + offset,
 		  HDMI_INFOFRAME_CONTROL0__HDMI_AVI_INFO_SEND_MASK | /* enable AVI info frames */
-		  HDMI_INFOFRAME_CONTROL0__HDMI_AVI_INFO_SEND_MASK); /* required for audio info values to be updated */
+		  HDMI_INFOFRAME_CONTROL0__HDMI_AVI_INFO_CONT_MASK); /* required for audio info values to be updated */
 
 	WREG32_P(mmHDMI_INFOFRAME_CONTROL1 + offset,
 		 (2 << HDMI_INFOFRAME_CONTROL1__HDMI_AVI_INFO_LINE__SHIFT), /* anything other than 0 */
@@ -1826,13 +1905,12 @@ static void dce_v8_0_afmt_setmode(struct drm_encoder *encoder,
 	WREG32_OR(mmAFMT_AUDIO_PACKET_CONTROL + offset,
 		  AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_MASK); /* send audio packets */
 
-	/* it's unknown what these bits do excatly, but it's indeed quite useful for debugging */
 	WREG32(mmAFMT_RAMP_CONTROL0 + offset, 0x00FFFFFF);
 	WREG32(mmAFMT_RAMP_CONTROL1 + offset, 0x007FFFFF);
 	WREG32(mmAFMT_RAMP_CONTROL2 + offset, 0x00000001);
 	WREG32(mmAFMT_RAMP_CONTROL3 + offset, 0x00000001);
 
-	/* enable audio after to setting up hw */
+	/* enable audio after setting up hw */
 	dce_v8_0_audio_enable(adev, dig->afmt->pin, true);
 }
 
@@ -1944,7 +2022,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	struct amdgpu_framebuffer *amdgpu_fb;
 	struct drm_framebuffer *target_fb;
 	struct drm_gem_object *obj;
-	struct amdgpu_bo *rbo;
+	struct amdgpu_bo *abo;
 	uint64_t fb_location, tiling_flags;
 	uint32_t fb_format, fb_pitch_pixels;
 	u32 fb_swap = (GRPH_ENDIAN_NONE << GRPH_SWAP_CNTL__GRPH_ENDIAN_SWAP__SHIFT);
@@ -1952,6 +2030,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
+	char *format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -1971,23 +2050,23 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	 * just update base pointers
 	 */
 	obj = amdgpu_fb->obj;
-	rbo = gem_to_amdgpu_bo(obj);
-	r = amdgpu_bo_reserve(rbo, false);
+	abo = gem_to_amdgpu_bo(obj);
+	r = amdgpu_bo_reserve(abo, false);
 	if (unlikely(r != 0))
 		return r;
 
 	if (atomic) {
-		fb_location = amdgpu_bo_gpu_offset(rbo);
+		fb_location = amdgpu_bo_gpu_offset(abo);
 	} else {
-		r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
+		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location);
 		if (unlikely(r != 0)) {
-			amdgpu_bo_unreserve(rbo);
+			amdgpu_bo_unreserve(abo);
 			return -EINVAL;
 		}
 	}
 
-	amdgpu_bo_get_tiling_flags(rbo, &tiling_flags);
-	amdgpu_bo_unreserve(rbo);
+	amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
+	amdgpu_bo_unreserve(abo);
 
 	pipe_config = AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
 
@@ -1999,7 +2078,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	case DRM_FORMAT_XRGB4444:
 	case DRM_FORMAT_ARGB4444:
 		fb_format = ((GRPH_DEPTH_16BPP << GRPH_CONTROL__GRPH_DEPTH__SHIFT) |
-			     (GRPH_FORMAT_ARGB1555 << GRPH_CONTROL__GRPH_FORMAT__SHIFT));
+			     (GRPH_FORMAT_ARGB4444 << GRPH_CONTROL__GRPH_FORMAT__SHIFT));
 #ifdef __BIG_ENDIAN
 		fb_swap = (GRPH_ENDIAN_8IN16 << GRPH_SWAP_CNTL__GRPH_ENDIAN_SWAP__SHIFT);
 #endif
@@ -2056,8 +2135,9 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		DRM_ERROR("Unsupported screen format %s\n",
-			  drm_get_format_name(target_fb->pixel_format));
+		format_name = drm_get_format_name(target_fb->pixel_format);
+		DRM_ERROR("Unsupported screen format %s\n", format_name);
+		kfree(format_name);
 		return -EINVAL;
 	}
 
@@ -2137,17 +2217,17 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(mmVIEWPORT_SIZE + amdgpu_crtc->crtc_offset,
 	       (viewport_w << 16) | viewport_h);
 
-	/* set pageflip to happen only at start of vblank interval (front porch) */
-	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 3);
+	/* set pageflip to happen anywhere in vblank interval */
+	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
 		amdgpu_fb = to_amdgpu_framebuffer(fb);
-		rbo = gem_to_amdgpu_bo(amdgpu_fb->obj);
-		r = amdgpu_bo_reserve(rbo, false);
+		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		r = amdgpu_bo_reserve(abo, false);
 		if (unlikely(r != 0))
 			return r;
-		amdgpu_bo_unpin(rbo);
-		amdgpu_bo_unreserve(rbo);
+		amdgpu_bo_unpin(abo);
+		amdgpu_bo_unreserve(abo);
 	}
 
 	/* Bytes per pixel may have changed */
@@ -2552,7 +2632,7 @@ static const struct drm_crtc_funcs dce_v8_0_crtc_funcs = {
 	.gamma_set = dce_v8_0_crtc_gamma_set,
 	.set_config = amdgpu_crtc_set_config,
 	.destroy = dce_v8_0_crtc_destroy,
-	.page_flip = amdgpu_crtc_page_flip,
+	.page_flip_target = amdgpu_crtc_page_flip_target,
 };
 
 static void dce_v8_0_crtc_dpms(struct drm_crtc *crtc, int mode)
@@ -2619,16 +2699,16 @@ static void dce_v8_0_crtc_disable(struct drm_crtc *crtc)
 	if (crtc->primary->fb) {
 		int r;
 		struct amdgpu_framebuffer *amdgpu_fb;
-		struct amdgpu_bo *rbo;
+		struct amdgpu_bo *abo;
 
 		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
-		rbo = gem_to_amdgpu_bo(amdgpu_fb->obj);
-		r = amdgpu_bo_reserve(rbo, false);
+		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		r = amdgpu_bo_reserve(abo, false);
 		if (unlikely(r))
-			DRM_ERROR("failed to reserve rbo before unpin\n");
+			DRM_ERROR("failed to reserve abo before unpin\n");
 		else {
-			amdgpu_bo_unpin(rbo);
-			amdgpu_bo_unreserve(rbo);
+			amdgpu_bo_unpin(abo);
+			amdgpu_bo_unreserve(abo);
 		}
 	}
 	/* disable the GRPH */
@@ -2653,7 +2733,7 @@ static void dce_v8_0_crtc_disable(struct drm_crtc *crtc)
 	case ATOM_PPLL2:
 		/* disable the ppll */
 		amdgpu_atombios_crtc_program_pll(crtc, amdgpu_crtc->crtc_id, amdgpu_crtc->pll_id,
-					  0, 0, ATOM_DISABLE, 0, 0, 0, 0, 0, false, &ss);
+                                                 0, 0, ATOM_DISABLE, 0, 0, 0, 0, 0, false, &ss);
 		break;
 	case ATOM_PPLL0:
 		/* disable the ppll */
@@ -2803,21 +2883,20 @@ static int dce_v8_0_early_init(void *handle)
 	dce_v8_0_set_display_funcs(adev);
 	dce_v8_0_set_irq_funcs(adev);
 
+	adev->mode_info.num_crtc = dce_v8_0_get_num_crtc(adev);
+
 	switch (adev->asic_type) {
 	case CHIP_BONAIRE:
 	case CHIP_HAWAII:
-		adev->mode_info.num_crtc = 6;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
 		break;
 	case CHIP_KAVERI:
-		adev->mode_info.num_crtc = 4;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 7;
 		break;
 	case CHIP_KABINI:
 	case CHIP_MULLINS:
-		adev->mode_info.num_crtc = 2;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6; /* ? */
 		break;
@@ -3236,7 +3315,6 @@ static int dce_v8_0_crtc_irq(struct amdgpu_device *adev,
 			drm_handle_vblank(adev->ddev, crtc);
 		}
 		DRM_DEBUG("IH: D%d vblank\n", crtc + 1);
-
 		break;
 	case 1: /* vline */
 		if (disp_int & interrupt_status_offsets[crtc].vline)
@@ -3245,7 +3323,6 @@ static int dce_v8_0_crtc_irq(struct amdgpu_device *adev,
 			DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
 
 		DRM_DEBUG("IH: D%d vline\n", crtc + 1);
-
 		break;
 	default:
 		DRM_DEBUG("Unhandled interrupt: %d %d\n", entry->src_id, entry->src_data);
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.h b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.h
index 77016852b252..7d0770c3a49b 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.h
@@ -26,4 +26,6 @@
 
 extern const struct amd_ip_funcs dce_v8_0_ip_funcs;
 
+void dce_v8_0_disable_dce(struct amdgpu_device *adev);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
new file mode 100644
index 000000000000..c2bd9f045532
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -0,0 +1,802 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "drmP.h"
+#include "amdgpu.h"
+#include "amdgpu_pm.h"
+#include "amdgpu_i2c.h"
+#include "atom.h"
+#include "amdgpu_pll.h"
+#include "amdgpu_connectors.h"
+#ifdef CONFIG_DRM_AMDGPU_CIK
+#include "dce_v8_0.h"
+#endif
+#include "dce_v10_0.h"
+#include "dce_v11_0.h"
+#include "dce_virtual.h"
+
+static void dce_virtual_set_display_funcs(struct amdgpu_device *adev);
+static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev);
+static int dce_virtual_pageflip_irq(struct amdgpu_device *adev,
+				  struct amdgpu_irq_src *source,
+				  struct amdgpu_iv_entry *entry);
+
+/**
+ * dce_virtual_vblank_wait - vblank wait asic callback.
+ *
+ * @adev: amdgpu_device pointer
+ * @crtc: crtc to wait for vblank on
+ *
+ * Wait for vblank on the requested crtc (evergreen+).
+ */
+static void dce_virtual_vblank_wait(struct amdgpu_device *adev, int crtc)
+{
+	return;
+}
+
+static u32 dce_virtual_vblank_get_counter(struct amdgpu_device *adev, int crtc)
+{
+	return 0;
+}
+
+static void dce_virtual_page_flip(struct amdgpu_device *adev,
+			      int crtc_id, u64 crtc_base, bool async)
+{
+	return;
+}
+
+static int dce_virtual_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
+					u32 *vbl, u32 *position)
+{
+	*vbl = 0;
+	*position = 0;
+
+	return -EINVAL;
+}
+
+static bool dce_virtual_hpd_sense(struct amdgpu_device *adev,
+			       enum amdgpu_hpd_id hpd)
+{
+	return true;
+}
+
+static void dce_virtual_hpd_set_polarity(struct amdgpu_device *adev,
+				      enum amdgpu_hpd_id hpd)
+{
+	return;
+}
+
+static u32 dce_virtual_hpd_get_gpio_reg(struct amdgpu_device *adev)
+{
+	return 0;
+}
+
+static bool dce_virtual_is_display_hung(struct amdgpu_device *adev)
+{
+	return false;
+}
+
+static void dce_virtual_stop_mc_access(struct amdgpu_device *adev,
+			      struct amdgpu_mode_mc_save *save)
+{
+	switch (adev->asic_type) {
+#ifdef CONFIG_DRM_AMDGPU_CIK
+	case CHIP_BONAIRE:
+	case CHIP_HAWAII:
+	case CHIP_KAVERI:
+	case CHIP_KABINI:
+	case CHIP_MULLINS:
+		dce_v8_0_disable_dce(adev);
+		break;
+#endif
+	case CHIP_FIJI:
+	case CHIP_TONGA:
+		dce_v10_0_disable_dce(adev);
+		break;
+	case CHIP_CARRIZO:
+	case CHIP_STONEY:
+	case CHIP_POLARIS11:
+	case CHIP_POLARIS10:
+		dce_v11_0_disable_dce(adev);
+		break;
+	case CHIP_TOPAZ:
+		/* no DCE */
+		return;
+	default:
+		DRM_ERROR("Virtual display unsupported ASIC type: 0x%X\n", adev->asic_type);
+	}
+
+	return;
+}
+static void dce_virtual_resume_mc_access(struct amdgpu_device *adev,
+				struct amdgpu_mode_mc_save *save)
+{
+	return;
+}
+
+static void dce_virtual_set_vga_render_state(struct amdgpu_device *adev,
+				    bool render)
+{
+	return;
+}
+
+/**
+ * dce_virtual_bandwidth_update - program display watermarks
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Calculate and program the display watermarks and line
+ * buffer allocation (CIK).
+ */
+static void dce_virtual_bandwidth_update(struct amdgpu_device *adev)
+{
+	return;
+}
+
+static int dce_virtual_crtc_gamma_set(struct drm_crtc *crtc, u16 *red,
+				      u16 *green, u16 *blue, uint32_t size)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	int i;
+
+	/* userspace palettes are always correct as is */
+	for (i = 0; i < size; i++) {
+		amdgpu_crtc->lut_r[i] = red[i] >> 6;
+		amdgpu_crtc->lut_g[i] = green[i] >> 6;
+		amdgpu_crtc->lut_b[i] = blue[i] >> 6;
+	}
+
+	return 0;
+}
+
+static void dce_virtual_crtc_destroy(struct drm_crtc *crtc)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+	drm_crtc_cleanup(crtc);
+	kfree(amdgpu_crtc);
+}
+
+static const struct drm_crtc_funcs dce_virtual_crtc_funcs = {
+	.cursor_set2 = NULL,
+	.cursor_move = NULL,
+	.gamma_set = dce_virtual_crtc_gamma_set,
+	.set_config = amdgpu_crtc_set_config,
+	.destroy = dce_virtual_crtc_destroy,
+	.page_flip_target = amdgpu_crtc_page_flip_target,
+};
+
+static void dce_virtual_crtc_dpms(struct drm_crtc *crtc, int mode)
+{
+	struct drm_device *dev = crtc->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	unsigned type;
+
+	switch (mode) {
+	case DRM_MODE_DPMS_ON:
+		amdgpu_crtc->enabled = true;
+		/* Make sure VBLANK and PFLIP interrupts are still enabled */
+		type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
+		amdgpu_irq_update(adev, &adev->crtc_irq, type);
+		amdgpu_irq_update(adev, &adev->pageflip_irq, type);
+		drm_vblank_on(dev, amdgpu_crtc->crtc_id);
+		break;
+	case DRM_MODE_DPMS_STANDBY:
+	case DRM_MODE_DPMS_SUSPEND:
+	case DRM_MODE_DPMS_OFF:
+		drm_vblank_off(dev, amdgpu_crtc->crtc_id);
+		amdgpu_crtc->enabled = false;
+		break;
+	}
+}
+
+
+static void dce_virtual_crtc_prepare(struct drm_crtc *crtc)
+{
+	dce_virtual_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+}
+
+static void dce_virtual_crtc_commit(struct drm_crtc *crtc)
+{
+	dce_virtual_crtc_dpms(crtc, DRM_MODE_DPMS_ON);
+}
+
+static void dce_virtual_crtc_disable(struct drm_crtc *crtc)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+	dce_virtual_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+	if (crtc->primary->fb) {
+		int r;
+		struct amdgpu_framebuffer *amdgpu_fb;
+		struct amdgpu_bo *abo;
+
+		amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb);
+		abo = gem_to_amdgpu_bo(amdgpu_fb->obj);
+		r = amdgpu_bo_reserve(abo, false);
+		if (unlikely(r))
+			DRM_ERROR("failed to reserve abo before unpin\n");
+		else {
+			amdgpu_bo_unpin(abo);
+			amdgpu_bo_unreserve(abo);
+		}
+	}
+
+	amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
+	amdgpu_crtc->encoder = NULL;
+	amdgpu_crtc->connector = NULL;
+}
+
+static int dce_virtual_crtc_mode_set(struct drm_crtc *crtc,
+				  struct drm_display_mode *mode,
+				  struct drm_display_mode *adjusted_mode,
+				  int x, int y, struct drm_framebuffer *old_fb)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+	/* update the hw version fpr dpm */
+	amdgpu_crtc->hw_mode = *adjusted_mode;
+
+	return 0;
+}
+
+static bool dce_virtual_crtc_mode_fixup(struct drm_crtc *crtc,
+				     const struct drm_display_mode *mode,
+				     struct drm_display_mode *adjusted_mode)
+{
+	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct drm_encoder *encoder;
+
+	/* assign the encoder to the amdgpu crtc to avoid repeated lookups later */
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->crtc == crtc) {
+			amdgpu_crtc->encoder = encoder;
+			amdgpu_crtc->connector = amdgpu_get_connector_for_encoder(encoder);
+			break;
+		}
+	}
+	if ((amdgpu_crtc->encoder == NULL) || (amdgpu_crtc->connector == NULL)) {
+		amdgpu_crtc->encoder = NULL;
+		amdgpu_crtc->connector = NULL;
+		return false;
+	}
+
+	return true;
+}
+
+
+static int dce_virtual_crtc_set_base(struct drm_crtc *crtc, int x, int y,
+				  struct drm_framebuffer *old_fb)
+{
+	return 0;
+}
+
+static void dce_virtual_crtc_load_lut(struct drm_crtc *crtc)
+{
+	return;
+}
+
+static int dce_virtual_crtc_set_base_atomic(struct drm_crtc *crtc,
+					 struct drm_framebuffer *fb,
+					 int x, int y, enum mode_set_atomic state)
+{
+	return 0;
+}
+
+static const struct drm_crtc_helper_funcs dce_virtual_crtc_helper_funcs = {
+	.dpms = dce_virtual_crtc_dpms,
+	.mode_fixup = dce_virtual_crtc_mode_fixup,
+	.mode_set = dce_virtual_crtc_mode_set,
+	.mode_set_base = dce_virtual_crtc_set_base,
+	.mode_set_base_atomic = dce_virtual_crtc_set_base_atomic,
+	.prepare = dce_virtual_crtc_prepare,
+	.commit = dce_virtual_crtc_commit,
+	.load_lut = dce_virtual_crtc_load_lut,
+	.disable = dce_virtual_crtc_disable,
+};
+
+static int dce_virtual_crtc_init(struct amdgpu_device *adev, int index)
+{
+	struct amdgpu_crtc *amdgpu_crtc;
+	int i;
+
+	amdgpu_crtc = kzalloc(sizeof(struct amdgpu_crtc) +
+			      (AMDGPUFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
+	if (amdgpu_crtc == NULL)
+		return -ENOMEM;
+
+	drm_crtc_init(adev->ddev, &amdgpu_crtc->base, &dce_virtual_crtc_funcs);
+
+	drm_mode_crtc_set_gamma_size(&amdgpu_crtc->base, 256);
+	amdgpu_crtc->crtc_id = index;
+	adev->mode_info.crtcs[index] = amdgpu_crtc;
+
+	for (i = 0; i < 256; i++) {
+		amdgpu_crtc->lut_r[i] = i << 2;
+		amdgpu_crtc->lut_g[i] = i << 2;
+		amdgpu_crtc->lut_b[i] = i << 2;
+	}
+
+	amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
+	amdgpu_crtc->encoder = NULL;
+	amdgpu_crtc->connector = NULL;
+	drm_crtc_helper_add(&amdgpu_crtc->base, &dce_virtual_crtc_helper_funcs);
+
+	return 0;
+}
+
+static int dce_virtual_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	adev->mode_info.vsync_timer_enabled = AMDGPU_IRQ_STATE_DISABLE;
+	dce_virtual_set_display_funcs(adev);
+	dce_virtual_set_irq_funcs(adev);
+
+	adev->mode_info.num_crtc = 1;
+	adev->mode_info.num_hpd = 1;
+	adev->mode_info.num_dig = 1;
+	return 0;
+}
+
+static bool dce_virtual_get_connector_info(struct amdgpu_device *adev)
+{
+	struct amdgpu_i2c_bus_rec ddc_bus;
+	struct amdgpu_router router;
+	struct amdgpu_hpd hpd;
+
+	/* look up gpio for ddc, hpd */
+	ddc_bus.valid = false;
+	hpd.hpd = AMDGPU_HPD_NONE;
+	/* needed for aux chan transactions */
+	ddc_bus.hpd = hpd.hpd;
+
+	memset(&router, 0, sizeof(router));
+	router.ddc_valid = false;
+	router.cd_valid = false;
+	amdgpu_display_add_connector(adev,
+				      0,
+				      ATOM_DEVICE_CRT1_SUPPORT,
+				      DRM_MODE_CONNECTOR_VIRTUAL, &ddc_bus,
+				      CONNECTOR_OBJECT_ID_VIRTUAL,
+				      &hpd,
+				      &router);
+
+	amdgpu_display_add_encoder(adev, ENCODER_VIRTUAL_ENUM_VIRTUAL,
+							ATOM_DEVICE_CRT1_SUPPORT,
+							0);
+
+	amdgpu_link_encoder_connector(adev->ddev);
+
+	return true;
+}
+
+static int dce_virtual_sw_init(void *handle)
+{
+	int r, i;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	r = amdgpu_irq_add_id(adev, 229, &adev->crtc_irq);
+	if (r)
+		return r;
+
+	adev->ddev->max_vblank_count = 0;
+
+	adev->ddev->mode_config.funcs = &amdgpu_mode_funcs;
+
+	adev->ddev->mode_config.max_width = 16384;
+	adev->ddev->mode_config.max_height = 16384;
+
+	adev->ddev->mode_config.preferred_depth = 24;
+	adev->ddev->mode_config.prefer_shadow = 1;
+
+	adev->ddev->mode_config.fb_base = adev->mc.aper_base;
+
+	r = amdgpu_modeset_create_props(adev);
+	if (r)
+		return r;
+
+	adev->ddev->mode_config.max_width = 16384;
+	adev->ddev->mode_config.max_height = 16384;
+
+	/* allocate crtcs */
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		r = dce_virtual_crtc_init(adev, i);
+		if (r)
+			return r;
+	}
+
+	dce_virtual_get_connector_info(adev);
+	amdgpu_print_display_setup(adev->ddev);
+
+	drm_kms_helper_poll_init(adev->ddev);
+
+	adev->mode_info.mode_config_initialized = true;
+	return 0;
+}
+
+static int dce_virtual_sw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	kfree(adev->mode_info.bios_hardcoded_edid);
+
+	drm_kms_helper_poll_fini(adev->ddev);
+
+	drm_mode_config_cleanup(adev->ddev);
+	adev->mode_info.mode_config_initialized = false;
+	return 0;
+}
+
+static int dce_virtual_hw_init(void *handle)
+{
+	return 0;
+}
+
+static int dce_virtual_hw_fini(void *handle)
+{
+	return 0;
+}
+
+static int dce_virtual_suspend(void *handle)
+{
+	return dce_virtual_hw_fini(handle);
+}
+
+static int dce_virtual_resume(void *handle)
+{
+	return dce_virtual_hw_init(handle);
+}
+
+static bool dce_virtual_is_idle(void *handle)
+{
+	return true;
+}
+
+static int dce_virtual_wait_for_idle(void *handle)
+{
+	return 0;
+}
+
+static int dce_virtual_soft_reset(void *handle)
+{
+	return 0;
+}
+
+static int dce_virtual_set_clockgating_state(void *handle,
+					  enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int dce_virtual_set_powergating_state(void *handle,
+					  enum amd_powergating_state state)
+{
+	return 0;
+}
+
+const struct amd_ip_funcs dce_virtual_ip_funcs = {
+	.name = "dce_virtual",
+	.early_init = dce_virtual_early_init,
+	.late_init = NULL,
+	.sw_init = dce_virtual_sw_init,
+	.sw_fini = dce_virtual_sw_fini,
+	.hw_init = dce_virtual_hw_init,
+	.hw_fini = dce_virtual_hw_fini,
+	.suspend = dce_virtual_suspend,
+	.resume = dce_virtual_resume,
+	.is_idle = dce_virtual_is_idle,
+	.wait_for_idle = dce_virtual_wait_for_idle,
+	.soft_reset = dce_virtual_soft_reset,
+	.set_clockgating_state = dce_virtual_set_clockgating_state,
+	.set_powergating_state = dce_virtual_set_powergating_state,
+};
+
+/* these are handled by the primary encoders */
+static void dce_virtual_encoder_prepare(struct drm_encoder *encoder)
+{
+	return;
+}
+
+static void dce_virtual_encoder_commit(struct drm_encoder *encoder)
+{
+	return;
+}
+
+static void
+dce_virtual_encoder_mode_set(struct drm_encoder *encoder,
+		      struct drm_display_mode *mode,
+		      struct drm_display_mode *adjusted_mode)
+{
+	return;
+}
+
+static void dce_virtual_encoder_disable(struct drm_encoder *encoder)
+{
+	return;
+}
+
+static void
+dce_virtual_encoder_dpms(struct drm_encoder *encoder, int mode)
+{
+	return;
+}
+
+static bool dce_virtual_encoder_mode_fixup(struct drm_encoder *encoder,
+				    const struct drm_display_mode *mode,
+				    struct drm_display_mode *adjusted_mode)
+{
+
+	/* set the active encoder to connector routing */
+	amdgpu_encoder_set_active_device(encoder);
+
+	return true;
+}
+
+static const struct drm_encoder_helper_funcs dce_virtual_encoder_helper_funcs = {
+	.dpms = dce_virtual_encoder_dpms,
+	.mode_fixup = dce_virtual_encoder_mode_fixup,
+	.prepare = dce_virtual_encoder_prepare,
+	.mode_set = dce_virtual_encoder_mode_set,
+	.commit = dce_virtual_encoder_commit,
+	.disable = dce_virtual_encoder_disable,
+};
+
+static void dce_virtual_encoder_destroy(struct drm_encoder *encoder)
+{
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+
+	kfree(amdgpu_encoder->enc_priv);
+	drm_encoder_cleanup(encoder);
+	kfree(amdgpu_encoder);
+}
+
+static const struct drm_encoder_funcs dce_virtual_encoder_funcs = {
+	.destroy = dce_virtual_encoder_destroy,
+};
+
+static void dce_virtual_encoder_add(struct amdgpu_device *adev,
+				 uint32_t encoder_enum,
+				 uint32_t supported_device,
+				 u16 caps)
+{
+	struct drm_device *dev = adev->ddev;
+	struct drm_encoder *encoder;
+	struct amdgpu_encoder *amdgpu_encoder;
+
+	/* see if we already added it */
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		amdgpu_encoder = to_amdgpu_encoder(encoder);
+		if (amdgpu_encoder->encoder_enum == encoder_enum) {
+			amdgpu_encoder->devices |= supported_device;
+			return;
+		}
+
+	}
+
+	/* add a new one */
+	amdgpu_encoder = kzalloc(sizeof(struct amdgpu_encoder), GFP_KERNEL);
+	if (!amdgpu_encoder)
+		return;
+
+	encoder = &amdgpu_encoder->base;
+	encoder->possible_crtcs = 0x1;
+	amdgpu_encoder->enc_priv = NULL;
+	amdgpu_encoder->encoder_enum = encoder_enum;
+	amdgpu_encoder->encoder_id = (encoder_enum & OBJECT_ID_MASK) >> OBJECT_ID_SHIFT;
+	amdgpu_encoder->devices = supported_device;
+	amdgpu_encoder->rmx_type = RMX_OFF;
+	amdgpu_encoder->underscan_type = UNDERSCAN_OFF;
+	amdgpu_encoder->is_ext_encoder = false;
+	amdgpu_encoder->caps = caps;
+
+	drm_encoder_init(dev, encoder, &dce_virtual_encoder_funcs,
+					 DRM_MODE_ENCODER_VIRTUAL, NULL);
+	drm_encoder_helper_add(encoder, &dce_virtual_encoder_helper_funcs);
+	DRM_INFO("[FM]encoder: %d is VIRTUAL\n", amdgpu_encoder->encoder_id);
+}
+
+static const struct amdgpu_display_funcs dce_virtual_display_funcs = {
+	.set_vga_render_state = &dce_virtual_set_vga_render_state,
+	.bandwidth_update = &dce_virtual_bandwidth_update,
+	.vblank_get_counter = &dce_virtual_vblank_get_counter,
+	.vblank_wait = &dce_virtual_vblank_wait,
+	.is_display_hung = &dce_virtual_is_display_hung,
+	.backlight_set_level = NULL,
+	.backlight_get_level = NULL,
+	.hpd_sense = &dce_virtual_hpd_sense,
+	.hpd_set_polarity = &dce_virtual_hpd_set_polarity,
+	.hpd_get_gpio_reg = &dce_virtual_hpd_get_gpio_reg,
+	.page_flip = &dce_virtual_page_flip,
+	.page_flip_get_scanoutpos = &dce_virtual_crtc_get_scanoutpos,
+	.add_encoder = &dce_virtual_encoder_add,
+	.add_connector = &amdgpu_connector_add,
+	.stop_mc_access = &dce_virtual_stop_mc_access,
+	.resume_mc_access = &dce_virtual_resume_mc_access,
+};
+
+static void dce_virtual_set_display_funcs(struct amdgpu_device *adev)
+{
+	if (adev->mode_info.funcs == NULL)
+		adev->mode_info.funcs = &dce_virtual_display_funcs;
+}
+
+static enum hrtimer_restart dce_virtual_vblank_timer_handle(struct hrtimer *vblank_timer)
+{
+	struct amdgpu_mode_info *mode_info = container_of(vblank_timer, struct amdgpu_mode_info ,vblank_timer);
+	struct amdgpu_device *adev = container_of(mode_info, struct amdgpu_device ,mode_info);
+	unsigned crtc = 0;
+	drm_handle_vblank(adev->ddev, crtc);
+	dce_virtual_pageflip_irq(adev, NULL, NULL);
+	hrtimer_start(vblank_timer, ktime_set(0, DCE_VIRTUAL_VBLANK_PERIOD), HRTIMER_MODE_REL);
+	return HRTIMER_NORESTART;
+}
+
+static void dce_virtual_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
+						     int crtc,
+						     enum amdgpu_interrupt_state state)
+{
+	if (crtc >= adev->mode_info.num_crtc) {
+		DRM_DEBUG("invalid crtc %d\n", crtc);
+		return;
+	}
+
+	if (state && !adev->mode_info.vsync_timer_enabled) {
+		DRM_DEBUG("Enable software vsync timer\n");
+		hrtimer_init(&adev->mode_info.vblank_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+		hrtimer_set_expires(&adev->mode_info.vblank_timer, ktime_set(0, DCE_VIRTUAL_VBLANK_PERIOD));
+		adev->mode_info.vblank_timer.function = dce_virtual_vblank_timer_handle;
+		hrtimer_start(&adev->mode_info.vblank_timer, ktime_set(0, DCE_VIRTUAL_VBLANK_PERIOD), HRTIMER_MODE_REL);
+	} else if (!state && adev->mode_info.vsync_timer_enabled) {
+		DRM_DEBUG("Disable software vsync timer\n");
+		hrtimer_cancel(&adev->mode_info.vblank_timer);
+	}
+
+	adev->mode_info.vsync_timer_enabled = state;
+	DRM_DEBUG("[FM]set crtc %d vblank interrupt state %d\n", crtc, state);
+}
+
+
+static int dce_virtual_set_crtc_irq_state(struct amdgpu_device *adev,
+                                       struct amdgpu_irq_src *source,
+                                       unsigned type,
+                                       enum amdgpu_interrupt_state state)
+{
+	switch (type) {
+	case AMDGPU_CRTC_IRQ_VBLANK1:
+		dce_virtual_set_crtc_vblank_interrupt_state(adev, 0, state);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static void dce_virtual_crtc_vblank_int_ack(struct amdgpu_device *adev,
+					  int crtc)
+{
+	if (crtc >= adev->mode_info.num_crtc) {
+		DRM_DEBUG("invalid crtc %d\n", crtc);
+		return;
+	}
+}
+
+static int dce_virtual_crtc_irq(struct amdgpu_device *adev,
+			      struct amdgpu_irq_src *source,
+			      struct amdgpu_iv_entry *entry)
+{
+	unsigned crtc = 0;
+	unsigned irq_type = AMDGPU_CRTC_IRQ_VBLANK1;
+
+	dce_virtual_crtc_vblank_int_ack(adev, crtc);
+
+	if (amdgpu_irq_enabled(adev, source, irq_type)) {
+		drm_handle_vblank(adev->ddev, crtc);
+	}
+	dce_virtual_pageflip_irq(adev, NULL, NULL);
+	DRM_DEBUG("IH: D%d vblank\n", crtc + 1);
+	return 0;
+}
+
+static int dce_virtual_set_pageflip_irq_state(struct amdgpu_device *adev,
+					    struct amdgpu_irq_src *src,
+					    unsigned type,
+					    enum amdgpu_interrupt_state state)
+{
+	if (type >= adev->mode_info.num_crtc) {
+		DRM_ERROR("invalid pageflip crtc %d\n", type);
+		return -EINVAL;
+	}
+	DRM_DEBUG("[FM]set pageflip irq type %d state %d\n", type, state);
+
+	return 0;
+}
+
+static int dce_virtual_pageflip_irq(struct amdgpu_device *adev,
+				  struct amdgpu_irq_src *source,
+				  struct amdgpu_iv_entry *entry)
+{
+	unsigned long flags;
+	unsigned crtc_id = 0;
+	struct amdgpu_crtc *amdgpu_crtc;
+	struct amdgpu_flip_work *works;
+
+	crtc_id = 0;
+	amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
+
+	if (crtc_id >= adev->mode_info.num_crtc) {
+		DRM_ERROR("invalid pageflip crtc %d\n", crtc_id);
+		return -EINVAL;
+	}
+
+	/* IRQ could occur when in initial stage */
+	if (amdgpu_crtc == NULL)
+		return 0;
+
+	spin_lock_irqsave(&adev->ddev->event_lock, flags);
+	works = amdgpu_crtc->pflip_works;
+	if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) {
+		DRM_DEBUG_DRIVER("amdgpu_crtc->pflip_status = %d != "
+			"AMDGPU_FLIP_SUBMITTED(%d)\n",
+			amdgpu_crtc->pflip_status,
+			AMDGPU_FLIP_SUBMITTED);
+		spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
+		return 0;
+	}
+
+	/* page flip completed. clean up */
+	amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE;
+	amdgpu_crtc->pflip_works = NULL;
+
+	/* wakeup usersapce */
+	if (works->event)
+		drm_crtc_send_vblank_event(&amdgpu_crtc->base, works->event);
+
+	spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
+
+	drm_crtc_vblank_put(&amdgpu_crtc->base);
+	schedule_work(&works->unpin_work);
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs dce_virtual_crtc_irq_funcs = {
+	.set = dce_virtual_set_crtc_irq_state,
+	.process = dce_virtual_crtc_irq,
+};
+
+static const struct amdgpu_irq_src_funcs dce_virtual_pageflip_irq_funcs = {
+	.set = dce_virtual_set_pageflip_irq_state,
+	.process = dce_virtual_pageflip_irq,
+};
+
+static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST;
+	adev->crtc_irq.funcs = &dce_virtual_crtc_irq_funcs;
+
+	adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST;
+	adev->pageflip_irq.funcs = &dce_virtual_pageflip_irq_funcs;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_smum.h b/drivers/gpu/drm/amd/amdgpu/dce_virtual.h
index 5983e3150cc5..e239243f6ebc 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_smum.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.h
@@ -21,21 +21,11 @@
  *
  */
 
-#ifndef ICELAND_SMUM_H
-#define ICELAND_SMUM_H
+#ifndef __DCE_VIRTUAL_H__
+#define __DCE_VIRTUAL_H__
 
-#include "ppsmc.h"
-
-extern int iceland_smu_init(struct amdgpu_device *adev);
-extern int iceland_smu_fini(struct amdgpu_device *adev);
-extern int iceland_smu_start(struct amdgpu_device *adev);
-
-struct iceland_smu_private_data
-{
-	uint8_t *header;
-	uint8_t *mec_image;
-	uint32_t header_addr_high;
-	uint32_t header_addr_low;
-};
+extern const struct amd_ip_funcs dce_virtual_ip_funcs;
+#define DCE_VIRTUAL_VBLANK_PERIOD 16666666
 
 #endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c b/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c
deleted file mode 100644
index ed03b75175d4..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <linux/firmware.h>
-#include "drmP.h"
-#include "amdgpu.h"
-#include "fiji_smum.h"
-
-MODULE_FIRMWARE("amdgpu/fiji_smc.bin");
-
-static void fiji_dpm_set_funcs(struct amdgpu_device *adev);
-
-static int fiji_dpm_early_init(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	fiji_dpm_set_funcs(adev);
-
-	return 0;
-}
-
-static int fiji_dpm_init_microcode(struct amdgpu_device *adev)
-{
-	char fw_name[30] = "amdgpu/fiji_smc.bin";
-	int err;
-
-	err = request_firmware(&adev->pm.fw, fw_name, adev->dev);
-	if (err)
-		goto out;
-	err = amdgpu_ucode_validate(adev->pm.fw);
-
-out:
-	if (err) {
-		DRM_ERROR("Failed to load firmware \"%s\"", fw_name);
-		release_firmware(adev->pm.fw);
-		adev->pm.fw = NULL;
-	}
-	return err;
-}
-
-static int fiji_dpm_sw_init(void *handle)
-{
-	int ret;
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	ret = fiji_dpm_init_microcode(adev);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int fiji_dpm_sw_fini(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	release_firmware(adev->pm.fw);
-	adev->pm.fw = NULL;
-
-	return 0;
-}
-
-static int fiji_dpm_hw_init(void *handle)
-{
-	int ret;
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	mutex_lock(&adev->pm.mutex);
-
-	ret = fiji_smu_init(adev);
-	if (ret) {
-		DRM_ERROR("SMU initialization failed\n");
-		goto fail;
-	}
-
-	ret = fiji_smu_start(adev);
-	if (ret) {
-		DRM_ERROR("SMU start failed\n");
-		goto fail;
-	}
-
-	mutex_unlock(&adev->pm.mutex);
-	return 0;
-
-fail:
-	adev->firmware.smu_load = false;
-	mutex_unlock(&adev->pm.mutex);
-	return -EINVAL;
-}
-
-static int fiji_dpm_hw_fini(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	mutex_lock(&adev->pm.mutex);
-	fiji_smu_fini(adev);
-	mutex_unlock(&adev->pm.mutex);
-	return 0;
-}
-
-static int fiji_dpm_suspend(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	fiji_dpm_hw_fini(adev);
-
-	return 0;
-}
-
-static int fiji_dpm_resume(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	fiji_dpm_hw_init(adev);
-
-	return 0;
-}
-
-static int fiji_dpm_set_clockgating_state(void *handle,
-			enum amd_clockgating_state state)
-{
-	return 0;
-}
-
-static int fiji_dpm_set_powergating_state(void *handle,
-			enum amd_powergating_state state)
-{
-	return 0;
-}
-
-const struct amd_ip_funcs fiji_dpm_ip_funcs = {
-	.name = "fiji_dpm",
-	.early_init = fiji_dpm_early_init,
-	.late_init = NULL,
-	.sw_init = fiji_dpm_sw_init,
-	.sw_fini = fiji_dpm_sw_fini,
-	.hw_init = fiji_dpm_hw_init,
-	.hw_fini = fiji_dpm_hw_fini,
-	.suspend = fiji_dpm_suspend,
-	.resume = fiji_dpm_resume,
-	.is_idle = NULL,
-	.wait_for_idle = NULL,
-	.soft_reset = NULL,
-	.set_clockgating_state = fiji_dpm_set_clockgating_state,
-	.set_powergating_state = fiji_dpm_set_powergating_state,
-};
-
-static const struct amdgpu_dpm_funcs fiji_dpm_funcs = {
-	.get_temperature = NULL,
-	.pre_set_power_state = NULL,
-	.set_power_state = NULL,
-	.post_set_power_state = NULL,
-	.display_configuration_changed = NULL,
-	.get_sclk = NULL,
-	.get_mclk = NULL,
-	.print_power_state = NULL,
-	.debugfs_print_current_performance_level = NULL,
-	.force_performance_level = NULL,
-	.vblank_too_short = NULL,
-	.powergate_uvd = NULL,
-};
-
-static void fiji_dpm_set_funcs(struct amdgpu_device *adev)
-{
-	if (NULL == adev->pm.funcs)
-		adev->pm.funcs = &fiji_dpm_funcs;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
deleted file mode 100644
index b3e19ba4c57f..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
+++ /dev/null
@@ -1,863 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <linux/firmware.h>
-#include "drmP.h"
-#include "amdgpu.h"
-#include "fiji_ppsmc.h"
-#include "fiji_smum.h"
-#include "smu_ucode_xfer_vi.h"
-#include "amdgpu_ucode.h"
-
-#include "smu/smu_7_1_3_d.h"
-#include "smu/smu_7_1_3_sh_mask.h"
-
-#define FIJI_SMC_SIZE 0x20000
-
-static int fiji_set_smc_sram_address(struct amdgpu_device *adev, uint32_t smc_address, uint32_t limit)
-{
-	uint32_t val;
-
-	if (smc_address & 3)
-		return -EINVAL;
-
-	if ((smc_address + 3) > limit)
-		return -EINVAL;
-
-	WREG32(mmSMC_IND_INDEX_0, smc_address);
-
-	val = RREG32(mmSMC_IND_ACCESS_CNTL);
-	val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
-	WREG32(mmSMC_IND_ACCESS_CNTL, val);
-
-	return 0;
-}
-
-static int fiji_copy_bytes_to_smc(struct amdgpu_device *adev, uint32_t smc_start_address, const uint8_t *src, uint32_t byte_count, uint32_t limit)
-{
-	uint32_t addr;
-	uint32_t data, orig_data;
-	int result = 0;
-	uint32_t extra_shift;
-	unsigned long flags;
-
-	if (smc_start_address & 3)
-		return -EINVAL;
-
-	if ((smc_start_address + byte_count) > limit)
-		return -EINVAL;
-
-	addr = smc_start_address;
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	while (byte_count >= 4) {
-		/* Bytes are written into the SMC addres space with the MSB first */
-		data = (src[0] << 24) + (src[1] << 16) + (src[2] << 8) + src[3];
-
-		result = fiji_set_smc_sram_address(adev, addr, limit);
-
-		if (result)
-			goto out;
-
-		WREG32(mmSMC_IND_DATA_0, data);
-
-		src += 4;
-		byte_count -= 4;
-		addr += 4;
-	}
-
-	if (0 != byte_count) {
-		/* Now write odd bytes left, do a read modify write cycle */
-		data = 0;
-
-		result = fiji_set_smc_sram_address(adev, addr, limit);
-		if (result)
-			goto out;
-
-		orig_data = RREG32(mmSMC_IND_DATA_0);
-		extra_shift = 8 * (4 - byte_count);
-
-		while (byte_count > 0) {
-			data = (data << 8) + *src++;
-			byte_count--;
-		}
-
-		data <<= extra_shift;
-		data |= (orig_data & ~((~0UL) << extra_shift));
-
-		result = fiji_set_smc_sram_address(adev, addr, limit);
-		if (result)
-			goto out;
-
-		WREG32(mmSMC_IND_DATA_0, data);
-	}
-
-out:
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-	return result;
-}
-
-static int fiji_program_jump_on_start(struct amdgpu_device *adev)
-{
-	static unsigned char data[] = {0xE0, 0x00, 0x80, 0x40};
-	fiji_copy_bytes_to_smc(adev, 0x0, data, 4, sizeof(data)+1);
-
-	return 0;
-}
-
-static bool fiji_is_smc_ram_running(struct amdgpu_device *adev)
-{
-	uint32_t val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-	val = REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable);
-
-	return ((0 == val) && (0x20100 <= RREG32_SMC(ixSMC_PC_C)));
-}
-
-static int wait_smu_response(struct amdgpu_device *adev)
-{
-	int i;
-	uint32_t val;
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32(mmSMC_RESP_0);
-		if (REG_GET_FIELD(val, SMC_RESP_0, SMC_RESP))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout)
-		return -EINVAL;
-
-	return 0;
-}
-
-static int fiji_send_msg_to_smc_offset(struct amdgpu_device *adev)
-{
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MSG_ARG_0, 0x20000);
-	WREG32(mmSMC_MESSAGE_0, PPSMC_MSG_Test);
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send message\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int fiji_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg)
-{
-	if (!fiji_is_smc_ram_running(adev))
-	{
-		return -EINVAL;
-	}
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MESSAGE_0, msg);
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send message\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int fiji_send_msg_to_smc_without_waiting(struct amdgpu_device *adev,
-						PPSMC_Msg msg)
-{
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MESSAGE_0, msg);
-
-	return 0;
-}
-
-static int fiji_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
-						PPSMC_Msg msg,
-						uint32_t parameter)
-{
-	if (!fiji_is_smc_ram_running(adev))
-		return -EINVAL;
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MSG_ARG_0, parameter);
-
-	return fiji_send_msg_to_smc(adev, msg);
-}
-
-static int fiji_send_msg_to_smc_with_parameter_without_waiting(
-					struct amdgpu_device *adev,
-					PPSMC_Msg msg, uint32_t parameter)
-{
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MSG_ARG_0, parameter);
-
-	return fiji_send_msg_to_smc_without_waiting(adev, msg);
-}
-
-#if 0 /* not used yet */
-static int fiji_wait_for_smc_inactive(struct amdgpu_device *adev)
-{
-	int i;
-	uint32_t val;
-
-	if (!fiji_is_smc_ram_running(adev))
-		return -EINVAL;
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-		if (REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, cken) == 0)
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout)
-		return -EINVAL;
-
-	return 0;
-}
-#endif
-
-static int fiji_smu_upload_firmware_image(struct amdgpu_device *adev)
-{
-	const struct smc_firmware_header_v1_0 *hdr;
-	uint32_t ucode_size;
-	uint32_t ucode_start_address;
-	const uint8_t *src;
-	uint32_t val;
-	uint32_t byte_count;
-	uint32_t *data;
-	unsigned long flags;
-
-	if (!adev->pm.fw)
-		return -EINVAL;
-
-	/* Skip SMC ucode loading on SR-IOV capable boards.
-	 * vbios does this for us in asic_init in that case.
-	 */
-	if (adev->virtualization.supports_sr_iov)
-		return 0;
-
-	hdr = (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data;
-	amdgpu_ucode_print_smc_hdr(&hdr->header);
-
-	adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version);
-	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes);
-	ucode_start_address = le32_to_cpu(hdr->ucode_start_addr);
-	src = (const uint8_t *)
-		(adev->pm.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
-
-	if (ucode_size & 3) {
-		DRM_ERROR("SMC ucode is not 4 bytes aligned\n");
-		return -EINVAL;
-	}
-
-	if (ucode_size > FIJI_SMC_SIZE) {
-		DRM_ERROR("SMC address is beyond the SMC RAM area\n");
-		return -EINVAL;
-	}
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	WREG32(mmSMC_IND_INDEX_0, ucode_start_address);
-
-	val = RREG32(mmSMC_IND_ACCESS_CNTL);
-	val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1);
-	WREG32(mmSMC_IND_ACCESS_CNTL, val);
-
-	byte_count = ucode_size;
-	data = (uint32_t *)src;
-	for (; byte_count >= 4; data++, byte_count -= 4)
-		WREG32(mmSMC_IND_DATA_0, data[0]);
-
-	val = RREG32(mmSMC_IND_ACCESS_CNTL);
-	val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
-	WREG32(mmSMC_IND_ACCESS_CNTL, val);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-
-	return 0;
-}
-
-#if 0 /* not used yet */
-static int fiji_read_smc_sram_dword(struct amdgpu_device *adev,
-				uint32_t smc_address,
-				uint32_t *value,
-				uint32_t limit)
-{
-	int result;
-	unsigned long flags;
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	result = fiji_set_smc_sram_address(adev, smc_address, limit);
-	if (result == 0)
-		*value = RREG32(mmSMC_IND_DATA_0);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-	return result;
-}
-
-static int fiji_write_smc_sram_dword(struct amdgpu_device *adev,
-				uint32_t smc_address,
-				uint32_t value,
-				uint32_t limit)
-{
-	int result;
-	unsigned long flags;
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	result = fiji_set_smc_sram_address(adev, smc_address, limit);
-	if (result == 0)
-		WREG32(mmSMC_IND_DATA_0, value);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-	return result;
-}
-
-static int fiji_smu_stop_smc(struct amdgpu_device *adev)
-{
-	uint32_t val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-	val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 1);
-	WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
-
-	return 0;
-}
-#endif
-
-static enum AMDGPU_UCODE_ID fiji_convert_fw_type(uint32_t fw_type)
-{
-	switch (fw_type) {
-		case UCODE_ID_SDMA0:
-			return AMDGPU_UCODE_ID_SDMA0;
-		case UCODE_ID_SDMA1:
-			return AMDGPU_UCODE_ID_SDMA1;
-		case UCODE_ID_CP_CE:
-			return AMDGPU_UCODE_ID_CP_CE;
-		case UCODE_ID_CP_PFP:
-			return AMDGPU_UCODE_ID_CP_PFP;
-		case UCODE_ID_CP_ME:
-			return AMDGPU_UCODE_ID_CP_ME;
-		case UCODE_ID_CP_MEC:
-		case UCODE_ID_CP_MEC_JT1:
-		case UCODE_ID_CP_MEC_JT2:
-			return AMDGPU_UCODE_ID_CP_MEC1;
-		case UCODE_ID_RLC_G:
-			return AMDGPU_UCODE_ID_RLC_G;
-		default:
-			DRM_ERROR("ucode type is out of range!\n");
-			return AMDGPU_UCODE_ID_MAXIMUM;
-	}
-}
-
-static int fiji_smu_populate_single_firmware_entry(struct amdgpu_device *adev,
-						uint32_t fw_type,
-						struct SMU_Entry *entry)
-{
-	enum AMDGPU_UCODE_ID id = fiji_convert_fw_type(fw_type);
-	struct amdgpu_firmware_info *ucode = &adev->firmware.ucode[id];
-	const struct gfx_firmware_header_v1_0 *header = NULL;
-	uint64_t gpu_addr;
-	uint32_t data_size;
-
-	if (ucode->fw == NULL)
-		return -EINVAL;
-	gpu_addr  = ucode->mc_addr;
-	header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
-	data_size = le32_to_cpu(header->header.ucode_size_bytes);
-
-	if ((fw_type == UCODE_ID_CP_MEC_JT1) ||
-		(fw_type == UCODE_ID_CP_MEC_JT2)) {
-		gpu_addr += le32_to_cpu(header->jt_offset) << 2;
-		data_size = le32_to_cpu(header->jt_size) << 2;
-	}
-
-	entry->version = (uint16_t)le32_to_cpu(header->header.ucode_version);
-	entry->id = (uint16_t)fw_type;
-	entry->image_addr_high = upper_32_bits(gpu_addr);
-	entry->image_addr_low = lower_32_bits(gpu_addr);
-	entry->meta_data_addr_high = 0;
-	entry->meta_data_addr_low = 0;
-	entry->data_size_byte = data_size;
-	entry->num_register_entries = 0;
-
-	if (fw_type == UCODE_ID_RLC_G)
-		entry->flags = 1;
-	else
-		entry->flags = 0;
-
-	return 0;
-}
-
-static int fiji_smu_request_load_fw(struct amdgpu_device *adev)
-{
-	struct fiji_smu_private_data *private = (struct fiji_smu_private_data *)adev->smu.priv;
-	struct SMU_DRAMData_TOC *toc;
-	uint32_t fw_to_load;
-
-	WREG32_SMC(ixSOFT_REGISTERS_TABLE_28, 0);
-
-	fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SMU_DRAM_ADDR_HI, private->smu_buffer_addr_high);
-	fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SMU_DRAM_ADDR_LO, private->smu_buffer_addr_low);
-
-	toc = (struct SMU_DRAMData_TOC *)private->header;
-	toc->num_entries = 0;
-	toc->structure_version = 1;
-
-	if (!adev->firmware.smu_load)
-		return 0;
-
-	if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_RLC_G,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for RLC\n");
-		return -EINVAL;
-	}
-
-	if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_CE,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for CE\n");
-		return -EINVAL;
-	}
-
-	if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_PFP,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for PFP\n");
-		return -EINVAL;
-	}
-
-	if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_ME,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for ME\n");
-		return -EINVAL;
-	}
-
-	if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for MEC\n");
-		return -EINVAL;
-	}
-
-	if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC_JT1,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for MEC_JT1\n");
-		return -EINVAL;
-	}
-
-	if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC_JT2,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for MEC_JT2\n");
-		return -EINVAL;
-	}
-
-	if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA0,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for SDMA0\n");
-		return -EINVAL;
-	}
-
-	if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA1,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for SDMA1\n");
-		return -EINVAL;
-	}
-
-	fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_HI, private->header_addr_high);
-	fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_LO, private->header_addr_low);
-
-	fw_to_load = UCODE_ID_RLC_G_MASK |
-			UCODE_ID_SDMA0_MASK |
-			UCODE_ID_SDMA1_MASK |
-			UCODE_ID_CP_CE_MASK |
-			UCODE_ID_CP_ME_MASK |
-			UCODE_ID_CP_PFP_MASK |
-			UCODE_ID_CP_MEC_MASK;
-
-	if (fiji_send_msg_to_smc_with_parameter_without_waiting(adev, PPSMC_MSG_LoadUcodes, fw_to_load)) {
-		DRM_ERROR("Fail to request SMU load ucode\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static uint32_t fiji_smu_get_mask_for_fw_type(uint32_t fw_type)
-{
-	switch (fw_type) {
-		case AMDGPU_UCODE_ID_SDMA0:
-			return UCODE_ID_SDMA0_MASK;
-		case AMDGPU_UCODE_ID_SDMA1:
-			return UCODE_ID_SDMA1_MASK;
-		case AMDGPU_UCODE_ID_CP_CE:
-			return UCODE_ID_CP_CE_MASK;
-		case AMDGPU_UCODE_ID_CP_PFP:
-			return UCODE_ID_CP_PFP_MASK;
-		case AMDGPU_UCODE_ID_CP_ME:
-			return UCODE_ID_CP_ME_MASK;
-		case AMDGPU_UCODE_ID_CP_MEC1:
-			return UCODE_ID_CP_MEC_MASK;
-		case AMDGPU_UCODE_ID_CP_MEC2:
-			return UCODE_ID_CP_MEC_MASK;
-		case AMDGPU_UCODE_ID_RLC_G:
-			return UCODE_ID_RLC_G_MASK;
-		default:
-			DRM_ERROR("ucode type is out of range!\n");
-			return 0;
-	}
-}
-
-static int fiji_smu_check_fw_load_finish(struct amdgpu_device *adev,
-					uint32_t fw_type)
-{
-	uint32_t fw_mask = fiji_smu_get_mask_for_fw_type(fw_type);
-	int i;
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		if (fw_mask == (RREG32_SMC(ixSOFT_REGISTERS_TABLE_28) & fw_mask))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("check firmware loading failed\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int fiji_smu_start_in_protection_mode(struct amdgpu_device *adev)
-{
-	int result;
-	uint32_t val;
-	int i;
-
-	/* Assert reset */
-	val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	result = fiji_smu_upload_firmware_image(adev);
-	if (result)
-		return result;
-
-	/* Clear status */
-	WREG32_SMC(ixSMU_STATUS, 0);
-
-	/* Enable clock */
-	val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-	val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
-	WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
-
-	/* De-assert reset */
-	val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 0);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	/* Set SMU Auto Start */
-	val = RREG32_SMC(ixSMU_INPUT_DATA);
-	val = REG_SET_FIELD(val, SMU_INPUT_DATA, AUTO_START, 1);
-	WREG32_SMC(ixSMU_INPUT_DATA, val);
-
-	/* Clear firmware interrupt enable flag */
-	WREG32_SMC(ixFIRMWARE_FLAGS, 0);
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixRCU_UC_EVENTS);
-		if (REG_GET_FIELD(val, RCU_UC_EVENTS, INTERRUPTS_ENABLED))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("Interrupt is not enabled by firmware\n");
-		return -EINVAL;
-	}
-
-	/* Call Test SMU message with 0x20000 offset
-	 * to trigger SMU start
-	 */
-	fiji_send_msg_to_smc_offset(adev);
-	DRM_INFO("[FM]try triger smu start\n");
-	/* Wait for done bit to be set */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixSMU_STATUS);
-		if (REG_GET_FIELD(val, SMU_STATUS, SMU_DONE))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("Timeout for SMU start\n");
-		return -EINVAL;
-	}
-
-	/* Check pass/failed indicator */
-	val = RREG32_SMC(ixSMU_STATUS);
-	if (!REG_GET_FIELD(val, SMU_STATUS, SMU_PASS)) {
-		DRM_ERROR("SMU Firmware start failed\n");
-		return -EINVAL;
-	}
-	DRM_INFO("[FM]smu started\n");
-	/* Wait for firmware to initialize */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixFIRMWARE_FLAGS);
-		if(REG_GET_FIELD(val, FIRMWARE_FLAGS, INTERRUPTS_ENABLED))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("SMU firmware initialization failed\n");
-		return -EINVAL;
-	}
-	DRM_INFO("[FM]smu initialized\n");
-
-	return 0;
-}
-
-static int fiji_smu_start_in_non_protection_mode(struct amdgpu_device *adev)
-{
-	int i, result;
-	uint32_t val;
-
-	/* wait for smc boot up */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixRCU_UC_EVENTS);
-		val = REG_GET_FIELD(val, RCU_UC_EVENTS, boot_seq_done);
-		if (val)
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("SMC boot sequence is not completed\n");
-		return -EINVAL;
-	}
-
-	/* Clear firmware interrupt enable flag */
-	WREG32_SMC(ixFIRMWARE_FLAGS, 0);
-
-	/* Assert reset */
-	val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	result = fiji_smu_upload_firmware_image(adev);
-	if (result)
-		return result;
-
-	/* Set smc instruct start point at 0x0 */
-	fiji_program_jump_on_start(adev);
-
-	/* Enable clock */
-	val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-	val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
-	WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
-
-	/* De-assert reset */
-	val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 0);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	/* Wait for firmware to initialize */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixFIRMWARE_FLAGS);
-		if (REG_GET_FIELD(val, FIRMWARE_FLAGS, INTERRUPTS_ENABLED))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("Timeout for SMC firmware initialization\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int fiji_smu_start(struct amdgpu_device *adev)
-{
-	int result;
-	uint32_t val;
-
-	if (!fiji_is_smc_ram_running(adev)) {
-		val = RREG32_SMC(ixSMU_FIRMWARE);
-		if (!REG_GET_FIELD(val, SMU_FIRMWARE, SMU_MODE)) {
-			DRM_INFO("[FM]start smu in nonprotection mode\n");
-			result = fiji_smu_start_in_non_protection_mode(adev);
-			if (result)
-				return result;
-		} else {
-			DRM_INFO("[FM]start smu in protection mode\n");
-			result = fiji_smu_start_in_protection_mode(adev);
-			if (result)
-				return result;
-		}
-	}
-
-	return fiji_smu_request_load_fw(adev);
-}
-
-static const struct amdgpu_smumgr_funcs fiji_smumgr_funcs = {
-	.check_fw_load_finish = fiji_smu_check_fw_load_finish,
-	.request_smu_load_fw = NULL,
-	.request_smu_specific_fw = NULL,
-};
-
-int fiji_smu_init(struct amdgpu_device *adev)
-{
-	struct fiji_smu_private_data *private;
-	uint32_t image_size = ((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
-	uint32_t smu_internal_buffer_size = 200*4096;
-	struct amdgpu_bo **toc_buf = &adev->smu.toc_buf;
-	struct amdgpu_bo **smu_buf = &adev->smu.smu_buf;
-	uint64_t mc_addr;
-	void *toc_buf_ptr;
-	void *smu_buf_ptr;
-	int ret;
-
-	private = kzalloc(sizeof(struct fiji_smu_private_data), GFP_KERNEL);
-	if (NULL == private)
-		return -ENOMEM;
-
-	/* allocate firmware buffers */
-	if (adev->firmware.smu_load)
-		amdgpu_ucode_init_bo(adev);
-
-	adev->smu.priv = private;
-	adev->smu.fw_flags = 0;
-
-	/* Allocate FW image data structure and header buffer */
-	ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE,
-			       true, AMDGPU_GEM_DOMAIN_VRAM,
-			       AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-			       NULL, NULL, toc_buf);
-	if (ret) {
-		DRM_ERROR("Failed to allocate memory for TOC buffer\n");
-		return -ENOMEM;
-	}
-
-	/* Allocate buffer for SMU internal buffer */
-	ret = amdgpu_bo_create(adev, smu_internal_buffer_size, PAGE_SIZE,
-			       true, AMDGPU_GEM_DOMAIN_VRAM,
-			       AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-			       NULL, NULL, smu_buf);
-	if (ret) {
-		DRM_ERROR("Failed to allocate memory for SMU internal buffer\n");
-		return -ENOMEM;
-	}
-
-	/* Retrieve GPU address for header buffer and internal buffer */
-	ret = amdgpu_bo_reserve(adev->smu.toc_buf, false);
-	if (ret) {
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to reserve the TOC buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_pin(adev->smu.toc_buf, AMDGPU_GEM_DOMAIN_VRAM, &mc_addr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.toc_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to pin the TOC buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_kmap(*toc_buf, &toc_buf_ptr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.toc_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to map the TOC buffer\n");
-		return -EINVAL;
-	}
-
-	amdgpu_bo_unreserve(adev->smu.toc_buf);
-	private->header_addr_low = lower_32_bits(mc_addr);
-	private->header_addr_high = upper_32_bits(mc_addr);
-	private->header = toc_buf_ptr;
-
-	ret = amdgpu_bo_reserve(adev->smu.smu_buf, false);
-	if (ret) {
-		amdgpu_bo_unref(&adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to reserve the SMU internal buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_pin(adev->smu.smu_buf, AMDGPU_GEM_DOMAIN_VRAM, &mc_addr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to pin the SMU internal buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_kmap(*smu_buf, &smu_buf_ptr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to map the SMU internal buffer\n");
-		return -EINVAL;
-	}
-
-	amdgpu_bo_unreserve(adev->smu.smu_buf);
-	private->smu_buffer_addr_low = lower_32_bits(mc_addr);
-	private->smu_buffer_addr_high = upper_32_bits(mc_addr);
-
-	adev->smu.smumgr_funcs = &fiji_smumgr_funcs;
-
-	return 0;
-}
-
-int fiji_smu_fini(struct amdgpu_device *adev)
-{
-	amdgpu_bo_unref(&adev->smu.toc_buf);
-	amdgpu_bo_unref(&adev->smu.smu_buf);
-	kfree(adev->smu.priv);
-	adev->smu.priv = NULL;
-	if (adev->firmware.fw_buf)
-		amdgpu_ucode_fini_bo(adev);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
new file mode 100644
index 000000000000..40abb6b81c09
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -0,0 +1,3362 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/firmware.h>
+#include "amdgpu.h"
+#include "amdgpu_ih.h"
+#include "amdgpu_gfx.h"
+#include "amdgpu_ucode.h"
+#include "si/clearstate_si.h"
+#include "si/sid.h"
+
+#define GFX6_NUM_GFX_RINGS     1
+#define GFX6_NUM_COMPUTE_RINGS 2
+#define STATIC_PER_CU_PG_ENABLE                    (1 << 3)
+#define DYN_PER_CU_PG_ENABLE                       (1 << 2)
+#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
+#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
+
+
+static void gfx_v6_0_set_ring_funcs(struct amdgpu_device *adev);
+static void gfx_v6_0_set_irq_funcs(struct amdgpu_device *adev);
+static void gfx_v6_0_get_cu_info(struct amdgpu_device *adev);
+
+MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
+MODULE_FIRMWARE("radeon/tahiti_me.bin");
+MODULE_FIRMWARE("radeon/tahiti_ce.bin");
+MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
+
+MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
+MODULE_FIRMWARE("radeon/pitcairn_me.bin");
+MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
+MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
+
+MODULE_FIRMWARE("radeon/verde_pfp.bin");
+MODULE_FIRMWARE("radeon/verde_me.bin");
+MODULE_FIRMWARE("radeon/verde_ce.bin");
+MODULE_FIRMWARE("radeon/verde_rlc.bin");
+
+MODULE_FIRMWARE("radeon/oland_pfp.bin");
+MODULE_FIRMWARE("radeon/oland_me.bin");
+MODULE_FIRMWARE("radeon/oland_ce.bin");
+MODULE_FIRMWARE("radeon/oland_rlc.bin");
+
+MODULE_FIRMWARE("radeon/hainan_pfp.bin");
+MODULE_FIRMWARE("radeon/hainan_me.bin");
+MODULE_FIRMWARE("radeon/hainan_ce.bin");
+MODULE_FIRMWARE("radeon/hainan_rlc.bin");
+
+static u32 gfx_v6_0_get_csb_size(struct amdgpu_device *adev);
+static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev, volatile u32 *buffer);
+//static void gfx_v6_0_init_cp_pg_table(struct amdgpu_device *adev);
+static void gfx_v6_0_init_pg(struct amdgpu_device *adev);
+
+
+static const u32 verde_rlc_save_restore_register_list[] =
+{
+	(0x8000 << 16) | (0x98f4 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x98f4 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0xe80 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0xe80 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x89bc >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x89bc >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x8c1c >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x8c1c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x98f0 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0xe7c >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x9148 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x9148 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9150 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x897c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8d8c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0xac54 >> 2),
+	0X00000000,
+	0x3,
+	(0x9c00 << 16) | (0x98f8 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9910 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9914 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9918 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x991c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9920 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9924 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9928 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x992c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9930 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9934 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9938 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x993c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9940 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9944 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9948 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x994c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9950 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9954 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9958 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x995c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9960 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9964 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9968 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x996c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9970 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9974 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9978 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x997c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9980 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9984 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9988 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x998c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8c00 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8c14 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8c04 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8c08 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x9b7c >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x9b7c >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0xe84 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0xe84 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x89c0 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x89c0 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x914c >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x914c >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x8c20 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x8c20 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x9354 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x9354 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9060 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9364 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9100 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x913c >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x90e0 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x90e4 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x90e8 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x90e0 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x90e4 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x90e8 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8bcc >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8b24 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x88c4 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8e50 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8c0c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8e58 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8e5c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9508 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x950c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9494 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0xac0c >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0xac10 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0xac14 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0xae00 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0xac08 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x88d4 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x88c8 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x88cc >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x89b0 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8b10 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x8a14 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9830 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9834 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9838 >> 2),
+	0x00000000,
+	(0x9c00 << 16) | (0x9a10 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x9870 >> 2),
+	0x00000000,
+	(0x8000 << 16) | (0x9874 >> 2),
+	0x00000000,
+	(0x8001 << 16) | (0x9870 >> 2),
+	0x00000000,
+	(0x8001 << 16) | (0x9874 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x9870 >> 2),
+	0x00000000,
+	(0x8040 << 16) | (0x9874 >> 2),
+	0x00000000,
+	(0x8041 << 16) | (0x9870 >> 2),
+	0x00000000,
+	(0x8041 << 16) | (0x9874 >> 2),
+	0x00000000,
+	0x00000000
+};
+
+static int gfx_v6_0_init_microcode(struct amdgpu_device *adev)
+{
+	const char *chip_name;
+	char fw_name[30];
+	int err;
+	const struct gfx_firmware_header_v1_0 *cp_hdr;
+	const struct rlc_firmware_header_v1_0 *rlc_hdr;
+
+	DRM_DEBUG("\n");
+
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+		chip_name = "tahiti";
+		break;
+	case CHIP_PITCAIRN:
+		chip_name = "pitcairn";
+		break;
+	case CHIP_VERDE:
+		chip_name = "verde";
+		break;
+	case CHIP_OLAND:
+		chip_name = "oland";
+		break;
+	case CHIP_HAINAN:
+		chip_name = "hainan";
+		break;
+	default: BUG();
+	}
+
+	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
+	err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
+	if (err)
+		goto out;
+	err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
+	if (err)
+		goto out;
+	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
+	adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
+	adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+
+	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
+	err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
+	if (err)
+		goto out;
+	err = amdgpu_ucode_validate(adev->gfx.me_fw);
+	if (err)
+		goto out;
+	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
+	adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
+	adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+
+	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
+	err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
+	if (err)
+		goto out;
+	err = amdgpu_ucode_validate(adev->gfx.ce_fw);
+	if (err)
+		goto out;
+	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
+	adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
+	adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+
+	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
+	err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
+	if (err)
+		goto out;
+	err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
+	rlc_hdr = (const struct rlc_firmware_header_v1_0 *)adev->gfx.rlc_fw->data;
+	adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
+	adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
+
+out:
+	if (err) {
+		printk(KERN_ERR
+		       "gfx6: Failed to load firmware \"%s\"\n",
+		       fw_name);
+		release_firmware(adev->gfx.pfp_fw);
+		adev->gfx.pfp_fw = NULL;
+		release_firmware(adev->gfx.me_fw);
+		adev->gfx.me_fw = NULL;
+		release_firmware(adev->gfx.ce_fw);
+		adev->gfx.ce_fw = NULL;
+		release_firmware(adev->gfx.rlc_fw);
+		adev->gfx.rlc_fw = NULL;
+	}
+	return err;
+}
+
+static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
+{
+	const u32 num_tile_mode_states = 32;
+	u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
+
+	switch (adev->gfx.config.mem_row_size_in_kb) {
+	case 1:
+		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
+		break;
+	case 2:
+	default:
+		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
+		break;
+	case 4:
+		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
+		break;
+	}
+
+	if (adev->asic_type == CHIP_VERDE ||
+		adev->asic_type == CHIP_OLAND ||
+		adev->asic_type == CHIP_HAINAN) {
+		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
+			switch (reg_offset) {
+			case 0:
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
+				break;
+			case 1: 
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
+				break;
+			case 2:
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
+				break;
+			case 3:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
+				break;
+			case 4:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 5:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(split_equal_to_row_size) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 6:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(split_equal_to_row_size) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 7:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(split_equal_to_row_size) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
+				break;
+			case 8: 
+				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 9:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 10:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
+				break;
+			case 11:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 12:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 13:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 14:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 15:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 16:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 17:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+						 TILE_SPLIT(split_equal_to_row_size) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 21:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 22:  
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
+				break;
+			case 23: 
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 24: 
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 25: 
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
+						 NUM_BANKS(ADDR_SURF_8_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
+				break;
+			default:
+				gb_tile_moden = 0;
+				break;
+			}
+			adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
+			WREG32(GB_TILE_MODE0 + reg_offset, gb_tile_moden);
+		}
+	} else if ((adev->asic_type == CHIP_TAHITI) || (adev->asic_type == CHIP_PITCAIRN)) {
+		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
+			switch (reg_offset) {
+			case 0:  /* non-AA compressed depth or any compressed stencil */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 1:  /* 2xAA/4xAA compressed depth only */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 2:  /* 8xAA compressed depth only */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(split_equal_to_row_size) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(split_equal_to_row_size) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
+				break;
+			case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(split_equal_to_row_size) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 8:  /* 1D and 1D Array Surfaces */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 9:  /* Displayable maps. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 10:  /* Display 8bpp. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 11:  /* Display 16bpp. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 12:  /* Display 32bpp. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
+				break;
+			case 13:  /* Thin. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 14:  /* Thin 8 bpp. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
+				break;
+			case 15:  /* Thin 16 bpp. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
+				break;
+			case 16:  /* Thin 32 bpp. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
+				break;
+			case 17:  /* Thin 64 bpp. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(split_equal_to_row_size) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
+				break;
+			case 21:  /* 8 bpp PRT. */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 22:  /* 16 bpp PRT */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
+				break;
+			case 23:  /* 32 bpp PRT */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 24:  /* 64 bpp PRT */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
+						 NUM_BANKS(ADDR_SURF_16_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
+				break;
+			case 25:  /* 128 bpp PRT */
+				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
+						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
+						 NUM_BANKS(ADDR_SURF_8_BANK) |
+						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
+				break;
+			default:
+				gb_tile_moden = 0;
+				break;
+			}
+			adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
+			WREG32(GB_TILE_MODE0 + reg_offset, gb_tile_moden);
+		}
+	} else{
+
+		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
+	}
+
+}
+
+static void gfx_v6_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
+				  u32 sh_num, u32 instance)
+{
+	u32 data;
+
+	if (instance == 0xffffffff)
+		data = INSTANCE_BROADCAST_WRITES;
+	else
+		data = INSTANCE_INDEX(instance);
+
+	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
+		data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
+	else if (se_num == 0xffffffff)
+		data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
+	else if (sh_num == 0xffffffff)
+		data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
+	else
+		data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
+	WREG32(GRBM_GFX_INDEX, data);
+}
+
+static u32 gfx_v6_0_create_bitmask(u32 bit_width)
+{
+	return (u32)(((u64)1 << bit_width) - 1);
+}
+
+static u32 gfx_v6_0_get_rb_disabled(struct amdgpu_device *adev,
+				    u32 max_rb_num_per_se,
+				    u32 sh_per_se)
+{
+	u32 data, mask;
+
+	data = RREG32(CC_RB_BACKEND_DISABLE);
+	data &= BACKEND_DISABLE_MASK;
+	data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
+
+	data >>= BACKEND_DISABLE_SHIFT;
+
+	mask = gfx_v6_0_create_bitmask(max_rb_num_per_se / sh_per_se);
+
+	return data & mask;
+}
+
+static void gfx_v6_0_raster_config(struct amdgpu_device *adev, u32 *rconf)
+{
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+		*rconf |= RB_XSEL2(2) | RB_XSEL | PKR_MAP(2) | PKR_YSEL(1) |
+			  SE_MAP(2) | SE_XSEL(2) | SE_YSEL(2);
+		break;
+	case CHIP_VERDE:
+		*rconf |= RB_XSEL | PKR_MAP(2) | PKR_YSEL(1);
+		break;
+	case CHIP_OLAND:
+		*rconf |= RB_YSEL;
+		break;
+	case CHIP_HAINAN:
+		*rconf |= 0x0;
+		break;
+	default:
+		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
+		break;
+	}
+}
+
+static void gfx_v6_0_write_harvested_raster_configs(struct amdgpu_device *adev,
+						    u32 raster_config, unsigned rb_mask,
+						    unsigned num_rb)
+{
+	unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
+	unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
+	unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
+	unsigned rb_per_se = num_rb / num_se;
+	unsigned se_mask[4];
+	unsigned se;
+
+	se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
+	se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
+	se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
+	se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
+
+	WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
+	WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
+	WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
+
+	for (se = 0; se < num_se; se++) {
+		unsigned raster_config_se = raster_config;
+		unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
+		unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
+		int idx = (se / 2) * 2;
+
+		if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
+			raster_config_se &= ~SE_MAP_MASK;
+
+			if (!se_mask[idx]) {
+				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
+			} else {
+				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
+			}
+		}
+
+		pkr0_mask &= rb_mask;
+		pkr1_mask &= rb_mask;
+		if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
+			raster_config_se &= ~PKR_MAP_MASK;
+
+			if (!pkr0_mask) {
+				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
+			} else {
+				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
+			}
+		}
+
+		if (rb_per_se >= 2) {
+			unsigned rb0_mask = 1 << (se * rb_per_se);
+			unsigned rb1_mask = rb0_mask << 1;
+
+			rb0_mask &= rb_mask;
+			rb1_mask &= rb_mask;
+			if (!rb0_mask || !rb1_mask) {
+				raster_config_se &= ~RB_MAP_PKR0_MASK;
+
+				if (!rb0_mask) {
+					raster_config_se |=
+						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
+				} else {
+					raster_config_se |=
+						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
+				}
+			}
+
+			if (rb_per_se > 2) {
+				rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
+				rb1_mask = rb0_mask << 1;
+				rb0_mask &= rb_mask;
+				rb1_mask &= rb_mask;
+				if (!rb0_mask || !rb1_mask) {
+					raster_config_se &= ~RB_MAP_PKR1_MASK;
+
+					if (!rb0_mask) {
+						raster_config_se |=
+							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
+					} else {
+						raster_config_se |=
+							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
+					}
+				}
+			}
+		}
+
+		/* GRBM_GFX_INDEX has a different offset on SI */
+		gfx_v6_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
+		WREG32(PA_SC_RASTER_CONFIG, raster_config_se);
+	}
+
+	/* GRBM_GFX_INDEX has a different offset on SI */
+	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+}
+
+static void gfx_v6_0_setup_rb(struct amdgpu_device *adev,
+			      u32 se_num, u32 sh_per_se,
+			      u32 max_rb_num_per_se)
+{
+	int i, j;
+	u32 data, mask;
+	u32 disabled_rbs = 0;
+	u32 enabled_rbs = 0;
+	unsigned num_rb_pipes;
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	for (i = 0; i < se_num; i++) {
+		for (j = 0; j < sh_per_se; j++) {
+			gfx_v6_0_select_se_sh(adev, i, j, 0xffffffff);
+			data = gfx_v6_0_get_rb_disabled(adev, max_rb_num_per_se, sh_per_se);
+			disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
+		}
+	}
+	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+	mutex_unlock(&adev->grbm_idx_mutex);
+
+	mask = 1;
+	for (i = 0; i < max_rb_num_per_se * se_num; i++) {
+		if (!(disabled_rbs & mask))
+			enabled_rbs |= mask;
+		mask <<= 1;
+	}
+
+	adev->gfx.config.backend_enable_mask = enabled_rbs;
+	adev->gfx.config.num_rbs = hweight32(enabled_rbs);
+
+	num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
+			     adev->gfx.config.max_shader_engines, 16);
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	for (i = 0; i < se_num; i++) {
+		gfx_v6_0_select_se_sh(adev, i, 0xffffffff, 0xffffffff);
+		data = 0;
+		for (j = 0; j < sh_per_se; j++) {
+			switch (enabled_rbs & 3) {
+			case 1:
+				data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
+				break;
+			case 2:
+				data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
+				break;
+			case 3:
+			default:
+				data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
+				break;
+			}
+			enabled_rbs >>= 2;
+		}
+		gfx_v6_0_raster_config(adev, &data);
+
+		if (!adev->gfx.config.backend_enable_mask ||
+				adev->gfx.config.num_rbs >= num_rb_pipes)
+			WREG32(PA_SC_RASTER_CONFIG, data);
+		else
+			gfx_v6_0_write_harvested_raster_configs(adev, data,
+								adev->gfx.config.backend_enable_mask,
+								num_rb_pipes);
+	}
+	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+	mutex_unlock(&adev->grbm_idx_mutex);
+}
+/*
+static void gmc_v6_0_init_compute_vmid(struct amdgpu_device *adev)
+{
+}
+*/
+
+static u32 gfx_v6_0_get_cu_enabled(struct amdgpu_device *adev, u32 cu_per_sh)
+{
+	u32 data, mask;
+
+	data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
+	data &= INACTIVE_CUS_MASK;
+	data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
+
+	data >>= INACTIVE_CUS_SHIFT;
+
+	mask = gfx_v6_0_create_bitmask(cu_per_sh);
+
+	return ~data & mask;
+}
+
+
+static void gfx_v6_0_setup_spi(struct amdgpu_device *adev,
+			 u32 se_num, u32 sh_per_se,
+			 u32 cu_per_sh)
+{
+	int i, j, k;
+	u32 data, mask;
+	u32 active_cu = 0;
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	for (i = 0; i < se_num; i++) {
+		for (j = 0; j < sh_per_se; j++) {
+			gfx_v6_0_select_se_sh(adev, i, j, 0xffffffff);
+			data = RREG32(SPI_STATIC_THREAD_MGMT_3);
+			active_cu = gfx_v6_0_get_cu_enabled(adev, cu_per_sh);
+
+			mask = 1;
+			for (k = 0; k < 16; k++) {
+				mask <<= k;
+				if (active_cu & mask) {
+					data &= ~mask;
+					WREG32(SPI_STATIC_THREAD_MGMT_3, data);
+					break;
+				}
+			}
+		}
+	}
+	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+	mutex_unlock(&adev->grbm_idx_mutex);
+}
+
+static void gfx_v6_0_gpu_init(struct amdgpu_device *adev)
+{
+	u32 gb_addr_config = 0;
+	u32 mc_shared_chmap, mc_arb_ramcfg;
+	u32 sx_debug_1;
+	u32 hdp_host_path_cntl;
+	u32 tmp;
+
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+		adev->gfx.config.max_shader_engines = 2;
+		adev->gfx.config.max_tile_pipes = 12;
+		adev->gfx.config.max_cu_per_sh = 8;
+		adev->gfx.config.max_sh_per_se = 2;
+		adev->gfx.config.max_backends_per_se = 4;
+		adev->gfx.config.max_texture_channel_caches = 12;
+		adev->gfx.config.max_gprs = 256;
+		adev->gfx.config.max_gs_threads = 32;
+		adev->gfx.config.max_hw_contexts = 8;
+
+		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
+		break;
+	case CHIP_PITCAIRN:
+		adev->gfx.config.max_shader_engines = 2;
+		adev->gfx.config.max_tile_pipes = 8;
+		adev->gfx.config.max_cu_per_sh = 5;
+		adev->gfx.config.max_sh_per_se = 2;
+		adev->gfx.config.max_backends_per_se = 4;
+		adev->gfx.config.max_texture_channel_caches = 8;
+		adev->gfx.config.max_gprs = 256;
+		adev->gfx.config.max_gs_threads = 32;
+		adev->gfx.config.max_hw_contexts = 8;
+
+		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
+		break;
+
+	case CHIP_VERDE:
+		adev->gfx.config.max_shader_engines = 1;
+		adev->gfx.config.max_tile_pipes = 4;
+		adev->gfx.config.max_cu_per_sh = 5;
+		adev->gfx.config.max_sh_per_se = 2;
+		adev->gfx.config.max_backends_per_se = 4;
+		adev->gfx.config.max_texture_channel_caches = 4;
+		adev->gfx.config.max_gprs = 256;
+		adev->gfx.config.max_gs_threads = 32;
+		adev->gfx.config.max_hw_contexts = 8;
+
+		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+		adev->gfx.config.sc_prim_fifo_size_backend = 0x40;
+		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
+		break;
+	case CHIP_OLAND:
+		adev->gfx.config.max_shader_engines = 1;
+		adev->gfx.config.max_tile_pipes = 4;
+		adev->gfx.config.max_cu_per_sh = 6;
+		adev->gfx.config.max_sh_per_se = 1;
+		adev->gfx.config.max_backends_per_se = 2;
+		adev->gfx.config.max_texture_channel_caches = 4;
+		adev->gfx.config.max_gprs = 256;
+		adev->gfx.config.max_gs_threads = 16;
+		adev->gfx.config.max_hw_contexts = 8;
+
+		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+		adev->gfx.config.sc_prim_fifo_size_backend = 0x40;
+		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
+		break;
+	case CHIP_HAINAN:
+		adev->gfx.config.max_shader_engines = 1;
+		adev->gfx.config.max_tile_pipes = 4;
+		adev->gfx.config.max_cu_per_sh = 5;
+		adev->gfx.config.max_sh_per_se = 1;
+		adev->gfx.config.max_backends_per_se = 1;
+		adev->gfx.config.max_texture_channel_caches = 2;
+		adev->gfx.config.max_gprs = 256;
+		adev->gfx.config.max_gs_threads = 16;
+		adev->gfx.config.max_hw_contexts = 8;
+
+		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+		adev->gfx.config.sc_prim_fifo_size_backend = 0x40;
+		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+		gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
+		break;
+	default:
+		BUG();
+		break;
+	}
+
+	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
+	WREG32(SRBM_INT_CNTL, 1);
+	WREG32(SRBM_INT_ACK, 1);
+
+	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
+
+	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
+	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
+
+	adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
+	adev->gfx.config.mem_max_burst_length_bytes = 256;
+	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
+	adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
+	if (adev->gfx.config.mem_row_size_in_kb > 4)
+		adev->gfx.config.mem_row_size_in_kb = 4;
+	adev->gfx.config.shader_engine_tile_size = 32;
+	adev->gfx.config.num_gpus = 1;
+	adev->gfx.config.multi_gpu_tile_size = 64;
+
+	gb_addr_config &= ~ROW_SIZE_MASK;
+	switch (adev->gfx.config.mem_row_size_in_kb) {
+	case 1:
+	default:
+		gb_addr_config |= ROW_SIZE(0);
+		break;
+	case 2:
+		gb_addr_config |= ROW_SIZE(1);
+		break;
+	case 4:
+		gb_addr_config |= ROW_SIZE(2);
+		break;
+	}
+	adev->gfx.config.gb_addr_config = gb_addr_config;
+
+	WREG32(GB_ADDR_CONFIG, gb_addr_config);
+	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
+	WREG32(DMIF_ADDR_CALC, gb_addr_config);
+	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
+	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
+	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
+#if 0
+	if (adev->has_uvd) {
+		WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
+		WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
+		WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
+	}
+#endif
+	gfx_v6_0_tiling_mode_table_init(adev);
+
+	gfx_v6_0_setup_rb(adev, adev->gfx.config.max_shader_engines,
+		    adev->gfx.config.max_sh_per_se,
+		    adev->gfx.config.max_backends_per_se);
+
+	gfx_v6_0_setup_spi(adev, adev->gfx.config.max_shader_engines,
+		     adev->gfx.config.max_sh_per_se,
+		     adev->gfx.config.max_cu_per_sh);
+
+	gfx_v6_0_get_cu_info(adev);
+
+	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
+				     ROQ_IB2_START(0x2b)));
+	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
+
+	sx_debug_1 = RREG32(SX_DEBUG_1);
+	WREG32(SX_DEBUG_1, sx_debug_1);
+
+	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
+
+	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(adev->gfx.config.sc_prim_fifo_size_frontend) |
+				 SC_BACKEND_PRIM_FIFO_SIZE(adev->gfx.config.sc_prim_fifo_size_backend) |
+				 SC_HIZ_TILE_FIFO_SIZE(adev->gfx.config.sc_hiz_tile_fifo_size) |
+				 SC_EARLYZ_TILE_FIFO_SIZE(adev->gfx.config.sc_earlyz_tile_fifo_size)));
+
+	WREG32(VGT_NUM_INSTANCES, 1);
+	WREG32(CP_PERFMON_CNTL, 0);
+	WREG32(SQ_CONFIG, 0);
+	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
+					  FORCE_EOV_MAX_REZ_CNT(255)));
+
+	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
+	       AUTO_INVLD_EN(ES_AND_GS_AUTO));
+
+	WREG32(VGT_GS_VERTEX_REUSE, 16);
+	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
+
+	WREG32(CB_PERFCOUNTER0_SELECT0, 0);
+	WREG32(CB_PERFCOUNTER0_SELECT1, 0);
+	WREG32(CB_PERFCOUNTER1_SELECT0, 0);
+	WREG32(CB_PERFCOUNTER1_SELECT1, 0);
+	WREG32(CB_PERFCOUNTER2_SELECT0, 0);
+	WREG32(CB_PERFCOUNTER2_SELECT1, 0);
+	WREG32(CB_PERFCOUNTER3_SELECT0, 0);
+	WREG32(CB_PERFCOUNTER3_SELECT1, 0);
+
+	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
+	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
+
+	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
+
+	udelay(50);
+}
+
+
+static void gfx_v6_0_scratch_init(struct amdgpu_device *adev)
+{
+	int i;
+
+	adev->gfx.scratch.num_reg = 7;
+	adev->gfx.scratch.reg_base = SCRATCH_REG0;
+	for (i = 0; i < adev->gfx.scratch.num_reg; i++) {
+		adev->gfx.scratch.free[i] = true;
+		adev->gfx.scratch.reg[i] = adev->gfx.scratch.reg_base + i;
+	}
+}
+
+static int gfx_v6_0_ring_test_ring(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	uint32_t scratch;
+	uint32_t tmp = 0;
+	unsigned i;
+	int r;
+
+	r = amdgpu_gfx_scratch_get(adev, &scratch);
+	if (r) {
+		DRM_ERROR("amdgpu: cp failed to get scratch reg (%d).\n", r);
+		return r;
+	}
+	WREG32(scratch, 0xCAFEDEAD);
+
+	r = amdgpu_ring_alloc(ring, 3);
+	if (r) {
+		DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n", ring->idx, r);
+		amdgpu_gfx_scratch_free(adev, scratch);
+		return r;
+	}
+	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+	amdgpu_ring_write(ring, (scratch - PACKET3_SET_CONFIG_REG_START));
+	amdgpu_ring_write(ring, 0xDEADBEEF);
+	amdgpu_ring_commit(ring);
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = RREG32(scratch);
+		if (tmp == 0xDEADBEEF)
+			break;
+		DRM_UDELAY(1);
+	}
+	if (i < adev->usec_timeout) {
+		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+	} else {
+		DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
+			  ring->idx, scratch, tmp);
+		r = -EINVAL;
+	}
+	amdgpu_gfx_scratch_free(adev, scratch);
+	return r;
+}
+
+static void gfx_v6_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+	/* flush hdp cache */
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
+				 WRITE_DATA_DST_SEL(0)));
+	amdgpu_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, 0x1);
+}
+
+/**
+ * gfx_v6_0_ring_emit_hdp_invalidate - emit an hdp invalidate on the cp
+ *
+ * @adev: amdgpu_device pointer
+ * @ridx: amdgpu ring index
+ *
+ * Emits an hdp invalidate on the cp.
+ */
+static void gfx_v6_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
+{
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
+				 WRITE_DATA_DST_SEL(0)));
+	amdgpu_ring_write(ring, HDP_DEBUG0);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, 0x1);
+}
+
+static void gfx_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+				     u64 seq, unsigned flags)
+{
+	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
+	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
+	/* flush read cache over gart */
+	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+	amdgpu_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START));
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
+	amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
+			  PACKET3_TC_ACTION_ENA |
+			  PACKET3_SH_KCACHE_ACTION_ENA |
+			  PACKET3_SH_ICACHE_ACTION_ENA);
+	amdgpu_ring_write(ring, 0xFFFFFFFF);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, 10); /* poll interval */
+	/* EVENT_WRITE_EOP - flush caches, send int */
+	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
+	amdgpu_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
+	amdgpu_ring_write(ring, addr & 0xfffffffc);
+	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
+				DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
+	amdgpu_ring_write(ring, lower_32_bits(seq));
+	amdgpu_ring_write(ring, upper_32_bits(seq));
+}
+
+static void gfx_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
+				  struct amdgpu_ib *ib,
+				  unsigned vm_id, bool ctx_switch)
+{
+	u32 header, control = 0;
+
+	/* insert SWITCH_BUFFER packet before first IB in the ring frame */
+	if (ctx_switch) {
+		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+		amdgpu_ring_write(ring, 0);
+	}
+
+	if (ib->flags & AMDGPU_IB_FLAG_CE)
+		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
+	else
+		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
+
+	control |= ib->length_dw | (vm_id << 24);
+
+	amdgpu_ring_write(ring, header);
+	amdgpu_ring_write(ring,
+#ifdef __BIG_ENDIAN
+			  (2 << 0) |
+#endif
+			  (ib->gpu_addr & 0xFFFFFFFC));
+	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
+	amdgpu_ring_write(ring, control);
+}
+
+/**
+ * gfx_v6_0_ring_test_ib - basic ring IB test
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ *
+ * Allocate an IB and execute it on the gfx ring (SI).
+ * Provides a basic gfx ring test to verify that IBs are working.
+ * Returns 0 on success, error on failure.
+ */
+static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_ib ib;
+	struct fence *f = NULL;
+	uint32_t scratch;
+	uint32_t tmp = 0;
+	long r;
+
+	r = amdgpu_gfx_scratch_get(adev, &scratch);
+	if (r) {
+		DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
+		return r;
+	}
+	WREG32(scratch, 0xCAFEDEAD);
+	memset(&ib, 0, sizeof(ib));
+	r = amdgpu_ib_get(adev, NULL, 256, &ib);
+	if (r) {
+		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
+		goto err1;
+	}
+	ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
+	ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_START));
+	ib.ptr[2] = 0xDEADBEEF;
+	ib.length_dw = 3;
+
+	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+	if (r)
+		goto err2;
+
+	r = fence_wait_timeout(f, false, timeout);
+	if (r == 0) {
+		DRM_ERROR("amdgpu: IB test timed out\n");
+		r = -ETIMEDOUT;
+		goto err2;
+	} else if (r < 0) {
+		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+		goto err2;
+	}
+	tmp = RREG32(scratch);
+	if (tmp == 0xDEADBEEF) {
+		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		r = 0;
+	} else {
+		DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
+			  scratch, tmp);
+		r = -EINVAL;
+	}
+
+err2:
+	amdgpu_ib_free(adev, &ib, NULL);
+	fence_put(f);
+err1:
+	amdgpu_gfx_scratch_free(adev, scratch);
+	return r;
+}
+
+static void gfx_v6_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
+{
+	int i;
+	if (enable)
+		WREG32(CP_ME_CNTL, 0);
+	else {
+		WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
+		WREG32(SCRATCH_UMSK, 0);
+		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
+			adev->gfx.gfx_ring[i].ready = false;
+		for (i = 0; i < adev->gfx.num_compute_rings; i++)
+			adev->gfx.compute_ring[i].ready = false;
+	}
+	udelay(50);
+}
+
+static int gfx_v6_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
+{
+	unsigned i;
+	const struct gfx_firmware_header_v1_0 *pfp_hdr;
+	const struct gfx_firmware_header_v1_0 *ce_hdr;
+	const struct gfx_firmware_header_v1_0 *me_hdr;
+	const __le32 *fw_data;
+	u32 fw_size;
+
+	if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
+		return -EINVAL;
+
+	gfx_v6_0_cp_gfx_enable(adev, false);
+	pfp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
+	ce_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
+	me_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
+
+	amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
+	amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
+	amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
+
+	/* PFP */
+	fw_data = (const __le32 *)
+		(adev->gfx.pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
+	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
+	WREG32(CP_PFP_UCODE_ADDR, 0);
+	for (i = 0; i < fw_size; i++)
+		WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
+	WREG32(CP_PFP_UCODE_ADDR, 0);
+
+	/* CE */
+	fw_data = (const __le32 *)
+		(adev->gfx.ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
+	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
+	WREG32(CP_CE_UCODE_ADDR, 0);
+	for (i = 0; i < fw_size; i++)
+		WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
+	WREG32(CP_CE_UCODE_ADDR, 0);
+
+	/* ME */
+	fw_data = (const __be32 *)
+		(adev->gfx.me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
+	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
+	WREG32(CP_ME_RAM_WADDR, 0);
+	for (i = 0; i < fw_size; i++)
+		WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
+	WREG32(CP_ME_RAM_WADDR, 0);
+
+
+	WREG32(CP_PFP_UCODE_ADDR, 0);
+	WREG32(CP_CE_UCODE_ADDR, 0);
+	WREG32(CP_ME_RAM_WADDR, 0);
+	WREG32(CP_ME_RAM_RADDR, 0);
+	return 0;
+}
+
+static int gfx_v6_0_cp_gfx_start(struct amdgpu_device *adev)
+{
+	const struct cs_section_def *sect = NULL;
+	const struct cs_extent_def *ext = NULL;
+	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
+	int r, i;
+
+	r = amdgpu_ring_alloc(ring, 7 + 4);
+	if (r) {
+		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
+		return r;
+	}
+	amdgpu_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
+	amdgpu_ring_write(ring, 0x1);
+	amdgpu_ring_write(ring, 0x0);
+	amdgpu_ring_write(ring, adev->gfx.config.max_hw_contexts - 1);
+	amdgpu_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, 0);
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
+	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
+	amdgpu_ring_write(ring, 0xc000);
+	amdgpu_ring_write(ring, 0xe000);
+	amdgpu_ring_commit(ring);
+
+	gfx_v6_0_cp_gfx_enable(adev, true);
+
+	r = amdgpu_ring_alloc(ring, gfx_v6_0_get_csb_size(adev) + 10);
+	if (r) {
+		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
+		return r;
+	}
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
+
+	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
+		for (ext = sect->section; ext->extent != NULL; ++ext) {
+			if (sect->id == SECT_CONTEXT) {
+				amdgpu_ring_write(ring,
+						  PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
+				amdgpu_ring_write(ring, ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
+				for (i = 0; i < ext->reg_count; i++)
+					amdgpu_ring_write(ring, ext->extent[i]);
+			}
+		}
+	}
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
+	amdgpu_ring_write(ring, 0);
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
+	amdgpu_ring_write(ring, 0x00000316);
+	amdgpu_ring_write(ring, 0x0000000e);
+	amdgpu_ring_write(ring, 0x00000010);
+
+	amdgpu_ring_commit(ring);
+
+	return 0;
+}
+
+static int gfx_v6_0_cp_gfx_resume(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	u32 tmp;
+	u32 rb_bufsz;
+	int r;
+	u64 rptr_addr;
+
+	WREG32(CP_SEM_WAIT_TIMER, 0x0);
+	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
+
+	/* Set the write pointer delay */
+	WREG32(CP_RB_WPTR_DELAY, 0);
+
+	WREG32(CP_DEBUG, 0);
+	WREG32(SCRATCH_ADDR, 0);
+
+	/* ring 0 - compute and gfx */
+	/* Set ring buffer size */
+	ring = &adev->gfx.gfx_ring[0];
+	rb_bufsz = order_base_2(ring->ring_size / 8);
+	tmp = (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+
+#ifdef __BIG_ENDIAN
+	tmp |= BUF_SWAP_32BIT;
+#endif
+	WREG32(CP_RB0_CNTL, tmp);
+
+	/* Initialize the ring buffer's read and write pointers */
+	WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
+	ring->wptr = 0;
+	WREG32(CP_RB0_WPTR, ring->wptr);
+
+	/* set the wb address whether it's enabled or not */
+	rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
+	WREG32(CP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
+	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
+
+	WREG32(SCRATCH_UMSK, 0);
+
+	mdelay(1);
+	WREG32(CP_RB0_CNTL, tmp);
+
+	WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
+
+	/* start the rings */
+	gfx_v6_0_cp_gfx_start(adev);
+	ring->ready = true;
+	r = amdgpu_ring_test_ring(ring);
+	if (r) {
+		ring->ready = false;
+		return r;
+	}
+
+	return 0;
+}
+
+static u32 gfx_v6_0_ring_get_rptr(struct amdgpu_ring *ring)
+{
+	return ring->adev->wb.wb[ring->rptr_offs];
+}
+
+static u32 gfx_v6_0_ring_get_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	if (ring == &adev->gfx.gfx_ring[0])
+		return RREG32(CP_RB0_WPTR);
+	else if (ring == &adev->gfx.compute_ring[0])
+		return RREG32(CP_RB1_WPTR);
+	else if (ring == &adev->gfx.compute_ring[1])
+		return RREG32(CP_RB2_WPTR);
+	else
+		BUG();
+}
+
+static void gfx_v6_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	WREG32(CP_RB0_WPTR, ring->wptr);
+	(void)RREG32(CP_RB0_WPTR);
+}
+
+static void gfx_v6_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	if (ring == &adev->gfx.compute_ring[0]) {
+		WREG32(CP_RB1_WPTR, ring->wptr);
+		(void)RREG32(CP_RB1_WPTR);
+	} else if (ring == &adev->gfx.compute_ring[1]) {
+		WREG32(CP_RB2_WPTR, ring->wptr);
+		(void)RREG32(CP_RB2_WPTR);
+	} else {
+		BUG();
+	}
+
+}
+
+static int gfx_v6_0_cp_compute_resume(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	u32 tmp;
+	u32 rb_bufsz;
+	int r;
+	u64 rptr_addr;
+
+	/* ring1  - compute only */
+	/* Set ring buffer size */
+
+	ring = &adev->gfx.compute_ring[0];
+	rb_bufsz = order_base_2(ring->ring_size / 8);
+	tmp = (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+#ifdef __BIG_ENDIAN
+	tmp |= BUF_SWAP_32BIT;
+#endif
+	WREG32(CP_RB1_CNTL, tmp);
+
+	WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
+	ring->wptr = 0;
+	WREG32(CP_RB1_WPTR, ring->wptr);
+
+	rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
+	WREG32(CP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
+	WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
+
+	mdelay(1);
+	WREG32(CP_RB1_CNTL, tmp);
+	WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
+
+	ring = &adev->gfx.compute_ring[1];
+	rb_bufsz = order_base_2(ring->ring_size / 8);
+	tmp = (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
+#ifdef __BIG_ENDIAN
+	tmp |= BUF_SWAP_32BIT;
+#endif
+	WREG32(CP_RB2_CNTL, tmp);
+
+	WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
+	ring->wptr = 0;
+	WREG32(CP_RB2_WPTR, ring->wptr);
+	rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
+	WREG32(CP_RB2_RPTR_ADDR, lower_32_bits(rptr_addr));
+	WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
+
+	mdelay(1);
+	WREG32(CP_RB2_CNTL, tmp);
+	WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
+
+	adev->gfx.compute_ring[0].ready = true;
+	adev->gfx.compute_ring[1].ready = true;
+
+	r = amdgpu_ring_test_ring(&adev->gfx.compute_ring[0]);
+	if (r) {
+		adev->gfx.compute_ring[0].ready = false;
+		return r;
+	}
+
+	r = amdgpu_ring_test_ring(&adev->gfx.compute_ring[1]);
+	if (r) {
+		adev->gfx.compute_ring[1].ready = false;
+		return r;
+	}
+
+	return 0;
+}
+
+static void gfx_v6_0_cp_enable(struct amdgpu_device *adev, bool enable)
+{
+	gfx_v6_0_cp_gfx_enable(adev, enable);
+}
+
+static int gfx_v6_0_cp_load_microcode(struct amdgpu_device *adev)
+{
+	return gfx_v6_0_cp_gfx_load_microcode(adev);
+}
+
+static void gfx_v6_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
+					       bool enable)
+{	
+	u32 tmp = RREG32(CP_INT_CNTL_RING0);
+	u32 mask;
+	int i;
+
+	if (enable)
+		tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+	else
+		tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+	WREG32(CP_INT_CNTL_RING0, tmp);
+
+	if (!enable) {
+		/* read a gfx register */
+		tmp = RREG32(DB_DEPTH_INFO);
+
+		mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
+		for (i = 0; i < adev->usec_timeout; i++) {
+			if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
+				break;
+			udelay(1);
+		}
+	}
+}
+
+static int gfx_v6_0_cp_resume(struct amdgpu_device *adev)
+{
+	int r;
+
+	gfx_v6_0_enable_gui_idle_interrupt(adev, false);
+
+	r = gfx_v6_0_cp_load_microcode(adev);
+	if (r)
+		return r;
+
+	r = gfx_v6_0_cp_gfx_resume(adev);
+	if (r)
+		return r;
+	r = gfx_v6_0_cp_compute_resume(adev);
+	if (r)
+		return r;
+
+	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
+
+	return 0;
+}
+
+static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+	int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+	uint32_t seq = ring->fence_drv.sync_seq;
+	uint64_t addr = ring->fence_drv.gpu_addr;
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
+	amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
+				 WAIT_REG_MEM_FUNCTION(3) | /* equal */
+				 WAIT_REG_MEM_ENGINE(usepfp)));   /* pfp or me */
+	amdgpu_ring_write(ring, addr & 0xfffffffc);
+	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
+	amdgpu_ring_write(ring, seq);
+	amdgpu_ring_write(ring, 0xffffffff);
+	amdgpu_ring_write(ring, 4); /* poll interval */
+
+	if (usepfp) {
+		/* synce CE with ME to prevent CE fetch CEIB before context switch done */
+		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+		amdgpu_ring_write(ring, 0);
+		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+		amdgpu_ring_write(ring, 0);
+	}
+}
+
+static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
+					unsigned vm_id, uint64_t pd_addr)
+{
+	int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+
+	/* write new base address */
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
+				 WRITE_DATA_DST_SEL(0)));
+	if (vm_id < 8) {
+		amdgpu_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id ));
+	} else {
+		amdgpu_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vm_id - 8)));
+	}
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, pd_addr >> 12);
+
+	/* bits 0-15 are the VM contexts0-15 */
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
+				 WRITE_DATA_DST_SEL(0)));
+	amdgpu_ring_write(ring, VM_INVALIDATE_REQUEST);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, 1 << vm_id);
+
+	/* wait for the invalidate to complete */
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
+	amdgpu_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
+				 WAIT_REG_MEM_ENGINE(0))); /* me */
+	amdgpu_ring_write(ring, VM_INVALIDATE_REQUEST);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, 0); /* ref */
+	amdgpu_ring_write(ring, 0); /* mask */
+	amdgpu_ring_write(ring, 0x20); /* poll interval */
+
+	if (usepfp) {
+		/* sync PFP to ME, otherwise we might get invalid PFP reads */
+		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
+		amdgpu_ring_write(ring, 0x0);
+
+		/* synce CE with ME to prevent CE fetch CEIB before context switch done */
+		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+		amdgpu_ring_write(ring, 0);
+		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+		amdgpu_ring_write(ring, 0);
+	}
+}
+
+
+static void gfx_v6_0_rlc_fini(struct amdgpu_device *adev)
+{
+	int r;
+
+	if (adev->gfx.rlc.save_restore_obj) {
+		r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, false);
+		if (unlikely(r != 0))
+			dev_warn(adev->dev, "(%d) reserve RLC sr bo failed\n", r);
+		amdgpu_bo_unpin(adev->gfx.rlc.save_restore_obj);
+		amdgpu_bo_unreserve(adev->gfx.rlc.save_restore_obj);
+
+		amdgpu_bo_unref(&adev->gfx.rlc.save_restore_obj);
+		adev->gfx.rlc.save_restore_obj = NULL;
+	}
+
+	if (adev->gfx.rlc.clear_state_obj) {
+		r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
+		if (unlikely(r != 0))
+			dev_warn(adev->dev, "(%d) reserve RLC c bo failed\n", r);
+		amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
+		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
+
+		amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
+		adev->gfx.rlc.clear_state_obj = NULL;
+	}
+
+	if (adev->gfx.rlc.cp_table_obj) {
+		r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false);
+		if (unlikely(r != 0))
+			dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
+		amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
+		amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
+
+		amdgpu_bo_unref(&adev->gfx.rlc.cp_table_obj);
+		adev->gfx.rlc.cp_table_obj = NULL;
+	}
+}
+
+static int gfx_v6_0_rlc_init(struct amdgpu_device *adev)
+{
+	const u32 *src_ptr;
+	volatile u32 *dst_ptr;
+	u32 dws, i;
+	u64 reg_list_mc_addr;
+	const struct cs_section_def *cs_data;
+	int r;
+
+	adev->gfx.rlc.reg_list = verde_rlc_save_restore_register_list;
+	adev->gfx.rlc.reg_list_size =
+			(u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
+
+	adev->gfx.rlc.cs_data = si_cs_data;
+	src_ptr = adev->gfx.rlc.reg_list;
+	dws = adev->gfx.rlc.reg_list_size;
+	cs_data = adev->gfx.rlc.cs_data;
+
+	if (src_ptr) {
+		/* save restore block */
+		if (adev->gfx.rlc.save_restore_obj == NULL) {
+
+			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
+					     AMDGPU_GEM_DOMAIN_VRAM,
+					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+					     NULL, NULL,
+					     &adev->gfx.rlc.save_restore_obj);
+
+			if (r) {
+				dev_warn(adev->dev, "(%d) create RLC sr bo failed\n", r);
+				return r;
+			}
+		}
+
+		r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, false);
+		if (unlikely(r != 0)) {
+			gfx_v6_0_rlc_fini(adev);
+			return r;
+		}
+		r = amdgpu_bo_pin(adev->gfx.rlc.save_restore_obj, AMDGPU_GEM_DOMAIN_VRAM,
+				  &adev->gfx.rlc.save_restore_gpu_addr);
+		if (r) {
+			amdgpu_bo_unreserve(adev->gfx.rlc.save_restore_obj);
+			dev_warn(adev->dev, "(%d) pin RLC sr bo failed\n", r);
+			gfx_v6_0_rlc_fini(adev);
+			return r;
+		}
+
+		r = amdgpu_bo_kmap(adev->gfx.rlc.save_restore_obj, (void **)&adev->gfx.rlc.sr_ptr);
+		if (r) {
+			dev_warn(adev->dev, "(%d) map RLC sr bo failed\n", r);
+			gfx_v6_0_rlc_fini(adev);
+			return r;
+		}
+		/* write the sr buffer */
+		dst_ptr = adev->gfx.rlc.sr_ptr;
+		for (i = 0; i < adev->gfx.rlc.reg_list_size; i++)
+			dst_ptr[i] = cpu_to_le32(src_ptr[i]);
+		amdgpu_bo_kunmap(adev->gfx.rlc.save_restore_obj);
+		amdgpu_bo_unreserve(adev->gfx.rlc.save_restore_obj);
+	}
+
+	if (cs_data) {
+		/* clear state block */
+		adev->gfx.rlc.clear_state_size = gfx_v6_0_get_csb_size(adev);
+		dws = adev->gfx.rlc.clear_state_size + (256 / 4);
+
+		if (adev->gfx.rlc.clear_state_obj == NULL) {
+			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
+					     AMDGPU_GEM_DOMAIN_VRAM,
+					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+					     NULL, NULL,
+					     &adev->gfx.rlc.clear_state_obj);
+
+			if (r) {
+				dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
+				gfx_v6_0_rlc_fini(adev);
+				return r;
+			}
+		}
+		r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
+		if (unlikely(r != 0)) {
+			gfx_v6_0_rlc_fini(adev);
+			return r;
+		}
+		r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj, AMDGPU_GEM_DOMAIN_VRAM,
+				  &adev->gfx.rlc.clear_state_gpu_addr);
+		if (r) {
+			amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
+			dev_warn(adev->dev, "(%d) pin RLC c bo failed\n", r);
+			gfx_v6_0_rlc_fini(adev);
+			return r;
+		}
+
+		r = amdgpu_bo_kmap(adev->gfx.rlc.clear_state_obj, (void **)&adev->gfx.rlc.cs_ptr);
+		if (r) {
+			dev_warn(adev->dev, "(%d) map RLC c bo failed\n", r);
+			gfx_v6_0_rlc_fini(adev);
+			return r;
+		}
+		/* set up the cs buffer */
+		dst_ptr = adev->gfx.rlc.cs_ptr;
+		reg_list_mc_addr = adev->gfx.rlc.clear_state_gpu_addr + 256;
+		dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
+		dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
+		dst_ptr[2] = cpu_to_le32(adev->gfx.rlc.clear_state_size);
+		gfx_v6_0_get_csb_buffer(adev, &dst_ptr[(256/4)]);
+		amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj);
+		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
+	}
+
+	return 0;
+}
+
+static void gfx_v6_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
+{
+	u32 tmp;
+
+	tmp = RREG32(RLC_LB_CNTL);
+	if (enable)
+		tmp |= LOAD_BALANCE_ENABLE;
+	else
+		tmp &= ~LOAD_BALANCE_ENABLE;
+	WREG32(RLC_LB_CNTL, tmp);
+
+	if (!enable) {
+		gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+		WREG32(SPI_LB_CU_MASK, 0x00ff);
+	}
+
+}
+
+static void gfx_v6_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
+			break;
+		udelay(1);
+	}
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
+			break;
+		udelay(1);
+	}
+}
+
+static void gfx_v6_0_update_rlc(struct amdgpu_device *adev, u32 rlc)
+{
+	u32 tmp;
+
+	tmp = RREG32(RLC_CNTL);
+	if (tmp != rlc)
+		WREG32(RLC_CNTL, rlc);
+}
+
+static u32 gfx_v6_0_halt_rlc(struct amdgpu_device *adev)
+{
+	u32 data, orig;
+
+	orig = data = RREG32(RLC_CNTL);
+
+	if (data & RLC_ENABLE) {
+		data &= ~RLC_ENABLE;
+		WREG32(RLC_CNTL, data);
+
+		gfx_v6_0_wait_for_rlc_serdes(adev);
+	}
+
+	return orig;
+}
+
+static void gfx_v6_0_rlc_stop(struct amdgpu_device *adev)
+{
+	WREG32(RLC_CNTL, 0);
+
+	gfx_v6_0_enable_gui_idle_interrupt(adev, false);
+	gfx_v6_0_wait_for_rlc_serdes(adev);
+}
+
+static void gfx_v6_0_rlc_start(struct amdgpu_device *adev)
+{
+	WREG32(RLC_CNTL, RLC_ENABLE);
+
+	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
+
+	udelay(50);
+}
+
+static void gfx_v6_0_rlc_reset(struct amdgpu_device *adev)
+{
+	u32 tmp = RREG32(GRBM_SOFT_RESET);
+
+	tmp |= SOFT_RESET_RLC;
+	WREG32(GRBM_SOFT_RESET, tmp);
+	udelay(50);
+	tmp &= ~SOFT_RESET_RLC;
+	WREG32(GRBM_SOFT_RESET, tmp);
+	udelay(50);
+}
+
+static bool gfx_v6_0_lbpw_supported(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	/* Enable LBPW only for DDR3 */
+	tmp = RREG32(MC_SEQ_MISC0);
+	if ((tmp & 0xF0000000) == 0xB0000000)
+		return true;
+	return false;
+}
+static void gfx_v6_0_init_cg(struct amdgpu_device *adev)
+{
+}
+
+static int gfx_v6_0_rlc_resume(struct amdgpu_device *adev)
+{
+	u32 i;
+	const struct rlc_firmware_header_v1_0 *hdr;
+	const __le32 *fw_data;
+	u32 fw_size;
+
+
+	if (!adev->gfx.rlc_fw)
+		return -EINVAL;
+
+	gfx_v6_0_rlc_stop(adev);
+	gfx_v6_0_rlc_reset(adev);
+	gfx_v6_0_init_pg(adev);
+	gfx_v6_0_init_cg(adev);
+
+	WREG32(RLC_RL_BASE, 0);
+	WREG32(RLC_RL_SIZE, 0);
+	WREG32(RLC_LB_CNTL, 0);
+	WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
+	WREG32(RLC_LB_CNTR_INIT, 0);
+	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
+
+	WREG32(RLC_MC_CNTL, 0);
+	WREG32(RLC_UCODE_CNTL, 0);
+
+	hdr = (const struct rlc_firmware_header_v1_0 *)adev->gfx.rlc_fw->data;
+	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
+	fw_data = (const __le32 *)
+		(adev->gfx.rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+
+	amdgpu_ucode_print_rlc_hdr(&hdr->header);
+
+	for (i = 0; i < fw_size; i++) {
+		WREG32(RLC_UCODE_ADDR, i);
+		WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
+	}
+	WREG32(RLC_UCODE_ADDR, 0);
+
+	gfx_v6_0_enable_lbpw(adev, gfx_v6_0_lbpw_supported(adev));
+	gfx_v6_0_rlc_start(adev);
+
+	return 0;
+}
+
+static void gfx_v6_0_enable_cgcg(struct amdgpu_device *adev, bool enable)
+{
+	u32 data, orig, tmp;
+
+	orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
+
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
+		gfx_v6_0_enable_gui_idle_interrupt(adev, true);
+
+		WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
+
+		tmp = gfx_v6_0_halt_rlc(adev);
+
+		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
+		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
+		WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
+
+		gfx_v6_0_wait_for_rlc_serdes(adev);
+		gfx_v6_0_update_rlc(adev, tmp);
+
+		WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
+
+		data |= CGCG_EN | CGLS_EN;
+	} else {
+		gfx_v6_0_enable_gui_idle_interrupt(adev, false);
+
+		RREG32(CB_CGTT_SCLK_CTRL);
+		RREG32(CB_CGTT_SCLK_CTRL);
+		RREG32(CB_CGTT_SCLK_CTRL);
+		RREG32(CB_CGTT_SCLK_CTRL);
+
+		data &= ~(CGCG_EN | CGLS_EN);
+	}
+
+	if (orig != data)
+		WREG32(RLC_CGCG_CGLS_CTRL, data);
+
+}
+
+static void gfx_v6_0_enable_mgcg(struct amdgpu_device *adev, bool enable)
+{
+
+	u32 data, orig, tmp = 0;
+
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
+		orig = data = RREG32(CGTS_SM_CTRL_REG);
+		data = 0x96940200;
+		if (orig != data)
+			WREG32(CGTS_SM_CTRL_REG, data);
+
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
+			orig = data = RREG32(CP_MEM_SLP_CNTL);
+			data |= CP_MEM_LS_EN;
+			if (orig != data)
+				WREG32(CP_MEM_SLP_CNTL, data);
+		}
+
+		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
+		data &= 0xffffffc0;
+		if (orig != data)
+			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
+
+		tmp = gfx_v6_0_halt_rlc(adev);
+
+		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
+		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
+		WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
+
+		gfx_v6_0_update_rlc(adev, tmp);
+	} else {
+		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
+		data |= 0x00000003;
+		if (orig != data)
+			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
+
+		data = RREG32(CP_MEM_SLP_CNTL);
+		if (data & CP_MEM_LS_EN) {
+			data &= ~CP_MEM_LS_EN;
+			WREG32(CP_MEM_SLP_CNTL, data);
+		}
+		orig = data = RREG32(CGTS_SM_CTRL_REG);
+		data |= LS_OVERRIDE | OVERRIDE;
+		if (orig != data)
+			WREG32(CGTS_SM_CTRL_REG, data);
+
+		tmp = gfx_v6_0_halt_rlc(adev);
+
+		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
+		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
+		WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
+
+		gfx_v6_0_update_rlc(adev, tmp);
+	}
+}
+/*
+static void gfx_v6_0_update_cg(struct amdgpu_device *adev,
+			       bool enable)
+{
+	gfx_v6_0_enable_gui_idle_interrupt(adev, false);
+	if (enable) {
+		gfx_v6_0_enable_mgcg(adev, true);
+		gfx_v6_0_enable_cgcg(adev, true);
+	} else {
+		gfx_v6_0_enable_cgcg(adev, false);
+		gfx_v6_0_enable_mgcg(adev, false);
+	}
+	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
+}
+*/
+static void gfx_v6_0_enable_sclk_slowdown_on_pu(struct amdgpu_device *adev,
+						bool enable)
+{
+}
+
+static void gfx_v6_0_enable_sclk_slowdown_on_pd(struct amdgpu_device *adev,
+						bool enable)
+{
+}
+
+static void gfx_v6_0_enable_cp_pg(struct amdgpu_device *adev, bool enable)
+{
+	u32 data, orig;
+
+	orig = data = RREG32(RLC_PG_CNTL);
+	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_CP))
+		data &= ~0x8000;
+	else
+		data |= 0x8000;
+	if (orig != data)
+		WREG32(RLC_PG_CNTL, data);
+}
+
+static void gfx_v6_0_enable_gds_pg(struct amdgpu_device *adev, bool enable)
+{
+}
+/*
+static void gfx_v6_0_init_cp_pg_table(struct amdgpu_device *adev)
+{
+	const __le32 *fw_data;
+	volatile u32 *dst_ptr;
+	int me, i, max_me = 4;
+	u32 bo_offset = 0;
+	u32 table_offset, table_size;
+
+	if (adev->asic_type == CHIP_KAVERI)
+		max_me = 5;
+
+	if (adev->gfx.rlc.cp_table_ptr == NULL)
+		return;
+
+	dst_ptr = adev->gfx.rlc.cp_table_ptr;
+	for (me = 0; me < max_me; me++) {
+		if (me == 0) {
+			const struct gfx_firmware_header_v1_0 *hdr =
+				(const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
+			fw_data = (const __le32 *)
+				(adev->gfx.ce_fw->data +
+				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+			table_offset = le32_to_cpu(hdr->jt_offset);
+			table_size = le32_to_cpu(hdr->jt_size);
+		} else if (me == 1) {
+			const struct gfx_firmware_header_v1_0 *hdr =
+				(const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
+			fw_data = (const __le32 *)
+				(adev->gfx.pfp_fw->data +
+				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+			table_offset = le32_to_cpu(hdr->jt_offset);
+			table_size = le32_to_cpu(hdr->jt_size);
+		} else if (me == 2) {
+			const struct gfx_firmware_header_v1_0 *hdr =
+				(const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
+			fw_data = (const __le32 *)
+				(adev->gfx.me_fw->data +
+				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+			table_offset = le32_to_cpu(hdr->jt_offset);
+			table_size = le32_to_cpu(hdr->jt_size);
+		} else if (me == 3) {
+			const struct gfx_firmware_header_v1_0 *hdr =
+				(const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
+			fw_data = (const __le32 *)
+				(adev->gfx.mec_fw->data +
+				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+			table_offset = le32_to_cpu(hdr->jt_offset);
+			table_size = le32_to_cpu(hdr->jt_size);
+		} else {
+			const struct gfx_firmware_header_v1_0 *hdr =
+				(const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
+			fw_data = (const __le32 *)
+				(adev->gfx.mec2_fw->data +
+				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+			table_offset = le32_to_cpu(hdr->jt_offset);
+			table_size = le32_to_cpu(hdr->jt_size);
+		}
+
+		for (i = 0; i < table_size; i ++) {
+			dst_ptr[bo_offset + i] =
+				cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
+		}
+
+		bo_offset += table_size;
+	}
+}
+*/
+static void gfx_v6_0_enable_gfx_cgpg(struct amdgpu_device *adev,
+				     bool enable)
+{
+
+	u32 tmp;
+
+	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
+		tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
+		WREG32(RLC_TTOP_D, tmp);
+
+		tmp = RREG32(RLC_PG_CNTL);
+		tmp |= GFX_PG_ENABLE;
+		WREG32(RLC_PG_CNTL, tmp);
+
+		tmp = RREG32(RLC_AUTO_PG_CTRL);
+		tmp |= AUTO_PG_EN;
+		WREG32(RLC_AUTO_PG_CTRL, tmp);
+	} else {
+		tmp = RREG32(RLC_AUTO_PG_CTRL);
+		tmp &= ~AUTO_PG_EN;
+		WREG32(RLC_AUTO_PG_CTRL, tmp);
+
+		tmp = RREG32(DB_RENDER_CONTROL);
+	}
+}
+
+static u32 gfx_v6_0_get_cu_active_bitmap(struct amdgpu_device *adev,
+					 u32 se, u32 sh)
+{
+
+	u32 mask = 0, tmp, tmp1;
+	int i;
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	gfx_v6_0_select_se_sh(adev, se, sh, 0xffffffff);
+	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
+	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
+	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+	mutex_unlock(&adev->grbm_idx_mutex);
+
+	tmp &= 0xffff0000;
+
+	tmp |= tmp1;
+	tmp >>= 16;
+
+	for (i = 0; i < adev->gfx.config.max_cu_per_sh; i ++) {
+		mask <<= 1;
+		mask |= 1;
+	}
+
+	return (~tmp) & mask;
+}
+
+static void gfx_v6_0_init_ao_cu_mask(struct amdgpu_device *adev)
+{
+	u32 i, j, k, active_cu_number = 0;
+
+	u32 mask, counter, cu_bitmap;
+	u32 tmp = 0;
+
+	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
+		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
+			mask = 1;
+			cu_bitmap = 0;
+			counter  = 0;
+			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
+				if (gfx_v6_0_get_cu_active_bitmap(adev, i, j) & mask) {
+					if (counter < 2)
+						cu_bitmap |= mask;
+					counter++;
+				}
+				mask <<= 1;
+			}
+
+			active_cu_number += counter;
+			tmp |= (cu_bitmap << (i * 16 + j * 8));
+		}
+	}
+
+	WREG32(RLC_PG_AO_CU_MASK, tmp);
+
+	tmp = RREG32(RLC_MAX_PG_CU);
+	tmp &= ~MAX_PU_CU_MASK;
+	tmp |= MAX_PU_CU(active_cu_number);
+	WREG32(RLC_MAX_PG_CU, tmp);
+}
+
+static void gfx_v6_0_enable_gfx_static_mgpg(struct amdgpu_device *adev,
+					    bool enable)
+{
+	u32 data, orig;
+
+	orig = data = RREG32(RLC_PG_CNTL);
+	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG))
+		data |= STATIC_PER_CU_PG_ENABLE;
+	else
+		data &= ~STATIC_PER_CU_PG_ENABLE;
+	if (orig != data)
+		WREG32(RLC_PG_CNTL, data);
+}
+
+static void gfx_v6_0_enable_gfx_dynamic_mgpg(struct amdgpu_device *adev,
+					     bool enable)
+{
+	u32 data, orig;
+
+	orig = data = RREG32(RLC_PG_CNTL);
+	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG))
+		data |= DYN_PER_CU_PG_ENABLE;
+	else
+		data &= ~DYN_PER_CU_PG_ENABLE;
+	if (orig != data)
+		WREG32(RLC_PG_CNTL, data);
+}
+
+static void gfx_v6_0_init_gfx_cgpg(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	WREG32(RLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
+
+	tmp = RREG32(RLC_PG_CNTL);
+	tmp |= GFX_PG_SRC;
+	WREG32(RLC_PG_CNTL, tmp);
+
+	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
+
+	tmp = RREG32(RLC_AUTO_PG_CTRL);
+
+	tmp &= ~GRBM_REG_SGIT_MASK;
+	tmp |= GRBM_REG_SGIT(0x700);
+	tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
+	WREG32(RLC_AUTO_PG_CTRL, tmp);
+}
+
+static void gfx_v6_0_update_gfx_pg(struct amdgpu_device *adev, bool enable)
+{
+	gfx_v6_0_enable_gfx_cgpg(adev, enable);
+	gfx_v6_0_enable_gfx_static_mgpg(adev, enable);
+	gfx_v6_0_enable_gfx_dynamic_mgpg(adev, enable);
+}
+
+static u32 gfx_v6_0_get_csb_size(struct amdgpu_device *adev)
+{
+	u32 count = 0;
+	const struct cs_section_def *sect = NULL;
+	const struct cs_extent_def *ext = NULL;
+
+	if (adev->gfx.rlc.cs_data == NULL)
+		return 0;
+
+	/* begin clear state */
+	count += 2;
+	/* context control state */
+	count += 3;
+
+	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
+		for (ext = sect->section; ext->extent != NULL; ++ext) {
+			if (sect->id == SECT_CONTEXT)
+				count += 2 + ext->reg_count;
+			else
+				return 0;
+		}
+	}
+	/* pa_sc_raster_config */
+	count += 3;
+	/* end clear state */
+	count += 2;
+	/* clear state */
+	count += 2;
+
+	return count;
+}
+
+static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev,
+				    volatile u32 *buffer)
+{
+	u32 count = 0, i;
+	const struct cs_section_def *sect = NULL;
+	const struct cs_extent_def *ext = NULL;
+
+	if (adev->gfx.rlc.cs_data == NULL)
+		return;
+	if (buffer == NULL)
+		return;
+
+	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
+
+	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+	buffer[count++] = cpu_to_le32(0x80000000);
+	buffer[count++] = cpu_to_le32(0x80000000);
+
+	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
+		for (ext = sect->section; ext->extent != NULL; ++ext) {
+			if (sect->id == SECT_CONTEXT) {
+				buffer[count++] =
+					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
+				buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
+				for (i = 0; i < ext->reg_count; i++)
+					buffer[count++] = cpu_to_le32(ext->extent[i]);
+			} else {
+				return;
+			}
+		}
+	}
+
+	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
+	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
+
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+		buffer[count++] = cpu_to_le32(0x2a00126a);
+		break;
+	case CHIP_VERDE:
+		buffer[count++] = cpu_to_le32(0x0000124a);
+		break;
+	case CHIP_OLAND:
+		buffer[count++] = cpu_to_le32(0x00000082);
+		break;
+	case CHIP_HAINAN:
+		buffer[count++] = cpu_to_le32(0x00000000);
+		break;
+	default:
+		buffer[count++] = cpu_to_le32(0x00000000);
+		break;
+	}
+
+	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
+
+	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
+	buffer[count++] = cpu_to_le32(0);
+}
+
+static void gfx_v6_0_init_pg(struct amdgpu_device *adev)
+{
+	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
+			      AMD_PG_SUPPORT_GFX_SMG |
+			      AMD_PG_SUPPORT_GFX_DMG |
+			      AMD_PG_SUPPORT_CP |
+			      AMD_PG_SUPPORT_GDS |
+			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
+		gfx_v6_0_enable_sclk_slowdown_on_pu(adev, true);
+		gfx_v6_0_enable_sclk_slowdown_on_pd(adev, true);
+		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
+			gfx_v6_0_init_gfx_cgpg(adev);
+			gfx_v6_0_enable_cp_pg(adev, true);
+			gfx_v6_0_enable_gds_pg(adev, true);
+		} else {
+			WREG32(RLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
+			WREG32(RLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
+
+		}
+		gfx_v6_0_init_ao_cu_mask(adev);
+		gfx_v6_0_update_gfx_pg(adev, true);
+	} else {
+
+		WREG32(RLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
+		WREG32(RLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
+	}
+}
+
+static void gfx_v6_0_fini_pg(struct amdgpu_device *adev)
+{
+	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
+			      AMD_PG_SUPPORT_GFX_SMG |
+			      AMD_PG_SUPPORT_GFX_DMG |
+			      AMD_PG_SUPPORT_CP |
+			      AMD_PG_SUPPORT_GDS |
+			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
+		gfx_v6_0_update_gfx_pg(adev, false);
+		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
+			gfx_v6_0_enable_cp_pg(adev, false);
+			gfx_v6_0_enable_gds_pg(adev, false);
+		}
+	}
+}
+
+static uint64_t gfx_v6_0_get_gpu_clock_counter(struct amdgpu_device *adev)
+{
+	uint64_t clock;
+
+	mutex_lock(&adev->gfx.gpu_clock_mutex);
+	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
+	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
+	        ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
+	mutex_unlock(&adev->gfx.gpu_clock_mutex);
+	return clock;
+}
+
+static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
+{
+	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+	amdgpu_ring_write(ring, 0x80000000);
+	amdgpu_ring_write(ring, 0);
+}
+
+static unsigned gfx_v6_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		6; /* gfx_v6_0_ring_emit_ib */
+}
+
+static unsigned gfx_v6_0_ring_get_dma_frame_size_gfx(struct amdgpu_ring *ring)
+{
+	return
+		5 + /* gfx_v6_0_ring_emit_hdp_flush */
+		5 + /* gfx_v6_0_ring_emit_hdp_invalidate */
+		14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */
+		7 + 4 + /* gfx_v6_0_ring_emit_pipeline_sync */
+		17 + 6 + /* gfx_v6_0_ring_emit_vm_flush */
+		3; /* gfx_v6_ring_emit_cntxcntl */
+}
+
+static unsigned gfx_v6_0_ring_get_dma_frame_size_compute(struct amdgpu_ring *ring)
+{
+	return
+		5 + /* gfx_v6_0_ring_emit_hdp_flush */
+		5 + /* gfx_v6_0_ring_emit_hdp_invalidate */
+		7 + /* gfx_v6_0_ring_emit_pipeline_sync */
+		17 + /* gfx_v6_0_ring_emit_vm_flush */
+		14 + 14 + 14; /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */
+}
+
+static const struct amdgpu_gfx_funcs gfx_v6_0_gfx_funcs = {
+	.get_gpu_clock_counter = &gfx_v6_0_get_gpu_clock_counter,
+	.select_se_sh = &gfx_v6_0_select_se_sh,
+};
+
+static int gfx_v6_0_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	adev->gfx.num_gfx_rings = GFX6_NUM_GFX_RINGS;
+	adev->gfx.num_compute_rings = GFX6_NUM_COMPUTE_RINGS;
+	adev->gfx.funcs = &gfx_v6_0_gfx_funcs;
+	gfx_v6_0_set_ring_funcs(adev);
+	gfx_v6_0_set_irq_funcs(adev);
+
+	return 0;
+}
+
+static int gfx_v6_0_sw_init(void *handle)
+{
+	struct amdgpu_ring *ring;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int i, r;
+
+	r = amdgpu_irq_add_id(adev, 181, &adev->gfx.eop_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 184, &adev->gfx.priv_reg_irq);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 185, &adev->gfx.priv_inst_irq);
+	if (r)
+		return r;
+
+	gfx_v6_0_scratch_init(adev);
+
+	r = gfx_v6_0_init_microcode(adev);
+	if (r) {
+		DRM_ERROR("Failed to load gfx firmware!\n");
+		return r;
+	}
+
+	r = gfx_v6_0_rlc_init(adev);
+	if (r) {
+		DRM_ERROR("Failed to init rlc BOs!\n");
+		return r;
+	}
+
+	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+		ring = &adev->gfx.gfx_ring[i];
+		ring->ring_obj = NULL;
+		sprintf(ring->name, "gfx");
+		r = amdgpu_ring_init(adev, ring, 1024,
+				     0x80000000, 0xf,
+				     &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP,
+				     AMDGPU_RING_TYPE_GFX);
+		if (r)
+			return r;
+	}
+
+	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+		unsigned irq_type;
+
+		if ((i >= 32) || (i >= AMDGPU_MAX_COMPUTE_RINGS)) {
+			DRM_ERROR("Too many (%d) compute rings!\n", i);
+			break;
+		}
+		ring = &adev->gfx.compute_ring[i];
+		ring->ring_obj = NULL;
+		ring->use_doorbell = false;
+		ring->doorbell_index = 0;
+		ring->me = 1;
+		ring->pipe = i;
+		ring->queue = i;
+		sprintf(ring->name, "comp %d.%d.%d", ring->me, ring->pipe, ring->queue);
+		irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe;
+		r = amdgpu_ring_init(adev, ring, 1024,
+				     0x80000000, 0xf,
+				     &adev->gfx.eop_irq, irq_type,
+				     AMDGPU_RING_TYPE_COMPUTE);
+		if (r)
+			return r;
+	}
+
+	return r;
+}
+
+static int gfx_v6_0_sw_fini(void *handle)
+{
+	int i;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	amdgpu_bo_unref(&adev->gds.oa_gfx_bo);
+	amdgpu_bo_unref(&adev->gds.gws_gfx_bo);
+	amdgpu_bo_unref(&adev->gds.gds_gfx_bo);
+
+	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
+		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
+	for (i = 0; i < adev->gfx.num_compute_rings; i++)
+		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
+
+	gfx_v6_0_rlc_fini(adev);
+
+	return 0;
+}
+
+static int gfx_v6_0_hw_init(void *handle)
+{
+	int r;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	gfx_v6_0_gpu_init(adev);
+
+	r = gfx_v6_0_rlc_resume(adev);
+	if (r)
+		return r;
+
+	r = gfx_v6_0_cp_resume(adev);
+	if (r)
+		return r;
+
+	adev->gfx.ce_ram_size = 0x8000;
+
+	return r;
+}
+
+static int gfx_v6_0_hw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	gfx_v6_0_cp_enable(adev, false);
+	gfx_v6_0_rlc_stop(adev);
+	gfx_v6_0_fini_pg(adev);
+
+	return 0;
+}
+
+static int gfx_v6_0_suspend(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return gfx_v6_0_hw_fini(adev);
+}
+
+static int gfx_v6_0_resume(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return gfx_v6_0_hw_init(adev);
+}
+
+static bool gfx_v6_0_is_idle(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (RREG32(GRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK)
+		return false;
+	else
+		return true;
+}
+
+static int gfx_v6_0_wait_for_idle(void *handle)
+{
+	unsigned i;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (gfx_v6_0_is_idle(handle))
+			return 0;
+		udelay(1);
+	}
+	return -ETIMEDOUT;
+}
+
+static int gfx_v6_0_soft_reset(void *handle)
+{
+	return 0;
+}
+
+static void gfx_v6_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
+						 enum amdgpu_interrupt_state state)
+{
+	u32 cp_int_cntl;
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl &= ~CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
+		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl |= CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
+		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		break;
+	default:
+		break;
+	}
+}
+
+static void gfx_v6_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
+						     int ring,
+						     enum amdgpu_interrupt_state state)
+{
+	u32 cp_int_cntl;
+	switch (state){
+	case AMDGPU_IRQ_STATE_DISABLE:
+		if (ring == 0) {
+			cp_int_cntl = RREG32(CP_INT_CNTL_RING1);
+			cp_int_cntl &= ~CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
+			WREG32(CP_INT_CNTL_RING1, cp_int_cntl);
+			break;
+		} else {
+			cp_int_cntl = RREG32(CP_INT_CNTL_RING2);
+			cp_int_cntl &= ~CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
+			WREG32(CP_INT_CNTL_RING2, cp_int_cntl);
+			break;
+
+		}
+	case AMDGPU_IRQ_STATE_ENABLE:
+		if (ring == 0) {
+			cp_int_cntl = RREG32(CP_INT_CNTL_RING1);
+			cp_int_cntl |= CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
+			WREG32(CP_INT_CNTL_RING1, cp_int_cntl);
+			break;
+		} else {
+			cp_int_cntl = RREG32(CP_INT_CNTL_RING2);
+			cp_int_cntl |= CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
+			WREG32(CP_INT_CNTL_RING2, cp_int_cntl);
+			break;
+
+		}
+
+	default:
+		BUG();
+		break;
+
+	}
+}
+
+static int gfx_v6_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
+					     struct amdgpu_irq_src *src,
+					     unsigned type,
+					     enum amdgpu_interrupt_state state)
+{
+	u32 cp_int_cntl;
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
+		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
+		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int gfx_v6_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *src,
+					      unsigned type,
+					      enum amdgpu_interrupt_state state)
+{
+	u32 cp_int_cntl;
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
+		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
+		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int gfx_v6_0_set_eop_interrupt_state(struct amdgpu_device *adev,
+					    struct amdgpu_irq_src *src,
+					    unsigned type,
+					    enum amdgpu_interrupt_state state)
+{
+	switch (type) {
+	case AMDGPU_CP_IRQ_GFX_EOP:
+		gfx_v6_0_set_gfx_eop_interrupt_state(adev, state);
+		break;
+	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
+		gfx_v6_0_set_compute_eop_interrupt_state(adev, 0, state);
+		break;
+	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
+		gfx_v6_0_set_compute_eop_interrupt_state(adev, 1, state);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int gfx_v6_0_eop_irq(struct amdgpu_device *adev,
+			    struct amdgpu_irq_src *source,
+			    struct amdgpu_iv_entry *entry)
+{
+	switch (entry->ring_id) {
+	case 0:
+		amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
+		break;
+	case 1:
+	case 2:
+		amdgpu_fence_process(&adev->gfx.compute_ring[entry->ring_id -1]);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int gfx_v6_0_priv_reg_irq(struct amdgpu_device *adev,
+				 struct amdgpu_irq_src *source,
+				 struct amdgpu_iv_entry *entry)
+{
+	DRM_ERROR("Illegal register access in command stream\n");
+	schedule_work(&adev->reset_work);
+	return 0;
+}
+
+static int gfx_v6_0_priv_inst_irq(struct amdgpu_device *adev,
+				  struct amdgpu_irq_src *source,
+				  struct amdgpu_iv_entry *entry)
+{
+	DRM_ERROR("Illegal instruction in command stream\n");
+	schedule_work(&adev->reset_work);
+	return 0;
+}
+
+static int gfx_v6_0_set_clockgating_state(void *handle,
+					  enum amd_clockgating_state state)
+{
+	bool gate = false;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (state == AMD_CG_STATE_GATE)
+		gate = true;
+
+	gfx_v6_0_enable_gui_idle_interrupt(adev, false);
+	if (gate) {
+		gfx_v6_0_enable_mgcg(adev, true);
+		gfx_v6_0_enable_cgcg(adev, true);
+	} else {
+		gfx_v6_0_enable_cgcg(adev, false);
+		gfx_v6_0_enable_mgcg(adev, false);
+	}
+	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
+
+	return 0;
+}
+
+static int gfx_v6_0_set_powergating_state(void *handle,
+					  enum amd_powergating_state state)
+{
+	bool gate = false;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (state == AMD_PG_STATE_GATE)
+		gate = true;
+
+	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
+			      AMD_PG_SUPPORT_GFX_SMG |
+			      AMD_PG_SUPPORT_GFX_DMG |
+			      AMD_PG_SUPPORT_CP |
+			      AMD_PG_SUPPORT_GDS |
+			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
+		gfx_v6_0_update_gfx_pg(adev, gate);
+		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
+			gfx_v6_0_enable_cp_pg(adev, gate);
+			gfx_v6_0_enable_gds_pg(adev, gate);
+		}
+	}
+
+	return 0;
+}
+
+const struct amd_ip_funcs gfx_v6_0_ip_funcs = {
+	.name = "gfx_v6_0",
+	.early_init = gfx_v6_0_early_init,
+	.late_init = NULL,
+	.sw_init = gfx_v6_0_sw_init,
+	.sw_fini = gfx_v6_0_sw_fini,
+	.hw_init = gfx_v6_0_hw_init,
+	.hw_fini = gfx_v6_0_hw_fini,
+	.suspend = gfx_v6_0_suspend,
+	.resume = gfx_v6_0_resume,
+	.is_idle = gfx_v6_0_is_idle,
+	.wait_for_idle = gfx_v6_0_wait_for_idle,
+	.soft_reset = gfx_v6_0_soft_reset,
+	.set_clockgating_state = gfx_v6_0_set_clockgating_state,
+	.set_powergating_state = gfx_v6_0_set_powergating_state,
+};
+
+static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = {
+	.get_rptr = gfx_v6_0_ring_get_rptr,
+	.get_wptr = gfx_v6_0_ring_get_wptr,
+	.set_wptr = gfx_v6_0_ring_set_wptr_gfx,
+	.parse_cs = NULL,
+	.emit_ib = gfx_v6_0_ring_emit_ib,
+	.emit_fence = gfx_v6_0_ring_emit_fence,
+	.emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync,
+	.emit_vm_flush = gfx_v6_0_ring_emit_vm_flush,
+	.emit_hdp_flush = gfx_v6_0_ring_emit_hdp_flush,
+	.emit_hdp_invalidate = gfx_v6_0_ring_emit_hdp_invalidate,
+	.test_ring = gfx_v6_0_ring_test_ring,
+	.test_ib = gfx_v6_0_ring_test_ib,
+	.insert_nop = amdgpu_ring_insert_nop,
+	.emit_cntxcntl = gfx_v6_ring_emit_cntxcntl,
+	.get_emit_ib_size = gfx_v6_0_ring_get_emit_ib_size,
+	.get_dma_frame_size = gfx_v6_0_ring_get_dma_frame_size_gfx,
+};
+
+static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = {
+	.get_rptr = gfx_v6_0_ring_get_rptr,
+	.get_wptr = gfx_v6_0_ring_get_wptr,
+	.set_wptr = gfx_v6_0_ring_set_wptr_compute,
+	.parse_cs = NULL,
+	.emit_ib = gfx_v6_0_ring_emit_ib,
+	.emit_fence = gfx_v6_0_ring_emit_fence,
+	.emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync,
+	.emit_vm_flush = gfx_v6_0_ring_emit_vm_flush,
+	.emit_hdp_flush = gfx_v6_0_ring_emit_hdp_flush,
+	.emit_hdp_invalidate = gfx_v6_0_ring_emit_hdp_invalidate,
+	.test_ring = gfx_v6_0_ring_test_ring,
+	.test_ib = gfx_v6_0_ring_test_ib,
+	.insert_nop = amdgpu_ring_insert_nop,
+	.get_emit_ib_size = gfx_v6_0_ring_get_emit_ib_size,
+	.get_dma_frame_size = gfx_v6_0_ring_get_dma_frame_size_compute,
+};
+
+static void gfx_v6_0_set_ring_funcs(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
+		adev->gfx.gfx_ring[i].funcs = &gfx_v6_0_ring_funcs_gfx;
+	for (i = 0; i < adev->gfx.num_compute_rings; i++)
+		adev->gfx.compute_ring[i].funcs = &gfx_v6_0_ring_funcs_compute;
+}
+
+static const struct amdgpu_irq_src_funcs gfx_v6_0_eop_irq_funcs = {
+	.set = gfx_v6_0_set_eop_interrupt_state,
+	.process = gfx_v6_0_eop_irq,
+};
+
+static const struct amdgpu_irq_src_funcs gfx_v6_0_priv_reg_irq_funcs = {
+	.set = gfx_v6_0_set_priv_reg_fault_state,
+	.process = gfx_v6_0_priv_reg_irq,
+};
+
+static const struct amdgpu_irq_src_funcs gfx_v6_0_priv_inst_irq_funcs = {
+	.set = gfx_v6_0_set_priv_inst_fault_state,
+	.process = gfx_v6_0_priv_inst_irq,
+};
+
+static void gfx_v6_0_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
+	adev->gfx.eop_irq.funcs = &gfx_v6_0_eop_irq_funcs;
+
+	adev->gfx.priv_reg_irq.num_types = 1;
+	adev->gfx.priv_reg_irq.funcs = &gfx_v6_0_priv_reg_irq_funcs;
+
+	adev->gfx.priv_inst_irq.num_types = 1;
+	adev->gfx.priv_inst_irq.funcs = &gfx_v6_0_priv_inst_irq_funcs;
+}
+
+static void gfx_v6_0_get_cu_info(struct amdgpu_device *adev)
+{
+	int i, j, k, counter, active_cu_number = 0;
+	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
+	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
+
+	memset(cu_info, 0, sizeof(*cu_info));
+
+	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
+		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
+			mask = 1;
+			ao_bitmap = 0;
+			counter = 0;
+			bitmap = gfx_v6_0_get_cu_active_bitmap(adev, i, j);
+			cu_info->bitmap[i][j] = bitmap;
+
+			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
+				if (bitmap & mask) {
+					if (counter < 2)
+						ao_bitmap |= mask;
+					counter ++;
+				}
+				mask <<= 1;
+			}
+			active_cu_number += counter;
+			ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
+		}
+	}
+
+	cu_info->number = active_cu_number;
+	cu_info->ao_cu_mask = ao_cu_mask;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.h
new file mode 100644
index 000000000000..b9657e72b248
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __GFX_V6_0_H__
+#define __GFX_V6_0_H__
+
+extern const struct amd_ip_funcs gfx_v6_0_ip_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index d869d058ef24..71116da9e782 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -1645,6 +1645,147 @@ static u32 gfx_v7_0_get_rb_active_bitmap(struct amdgpu_device *adev)
 	return (~data) & mask;
 }
 
+static void
+gfx_v7_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
+{
+	switch (adev->asic_type) {
+	case CHIP_BONAIRE:
+		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
+			  SE_XSEL(1) | SE_YSEL(1);
+		*rconf1 |= 0x0;
+		break;
+	case CHIP_HAWAII:
+		*rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
+			  RB_XSEL2(1) | PKR_MAP(2) | PKR_XSEL(1) |
+			  PKR_YSEL(1) | SE_MAP(2) | SE_XSEL(2) |
+			  SE_YSEL(3);
+		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
+			   SE_PAIR_YSEL(2);
+		break;
+	case CHIP_KAVERI:
+		*rconf |= RB_MAP_PKR0(2);
+		*rconf1 |= 0x0;
+		break;
+	case CHIP_KABINI:
+	case CHIP_MULLINS:
+		*rconf |= 0x0;
+		*rconf1 |= 0x0;
+		break;
+	default:
+		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
+		break;
+	}
+}
+
+static void
+gfx_v7_0_write_harvested_raster_configs(struct amdgpu_device *adev,
+					u32 raster_config, u32 raster_config_1,
+					unsigned rb_mask, unsigned num_rb)
+{
+	unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
+	unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
+	unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
+	unsigned rb_per_se = num_rb / num_se;
+	unsigned se_mask[4];
+	unsigned se;
+
+	se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
+	se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
+	se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
+	se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
+
+	WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
+	WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
+	WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
+
+	if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
+			     (!se_mask[2] && !se_mask[3]))) {
+		raster_config_1 &= ~SE_PAIR_MAP_MASK;
+
+		if (!se_mask[0] && !se_mask[1]) {
+			raster_config_1 |=
+				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
+		} else {
+			raster_config_1 |=
+				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
+		}
+	}
+
+	for (se = 0; se < num_se; se++) {
+		unsigned raster_config_se = raster_config;
+		unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
+		unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
+		int idx = (se / 2) * 2;
+
+		if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
+			raster_config_se &= ~SE_MAP_MASK;
+
+			if (!se_mask[idx]) {
+				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
+			} else {
+				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
+			}
+		}
+
+		pkr0_mask &= rb_mask;
+		pkr1_mask &= rb_mask;
+		if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
+			raster_config_se &= ~PKR_MAP_MASK;
+
+			if (!pkr0_mask) {
+				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
+			} else {
+				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
+			}
+		}
+
+		if (rb_per_se >= 2) {
+			unsigned rb0_mask = 1 << (se * rb_per_se);
+			unsigned rb1_mask = rb0_mask << 1;
+
+			rb0_mask &= rb_mask;
+			rb1_mask &= rb_mask;
+			if (!rb0_mask || !rb1_mask) {
+				raster_config_se &= ~RB_MAP_PKR0_MASK;
+
+				if (!rb0_mask) {
+					raster_config_se |=
+						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
+				} else {
+					raster_config_se |=
+						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
+				}
+			}
+
+			if (rb_per_se > 2) {
+				rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
+				rb1_mask = rb0_mask << 1;
+				rb0_mask &= rb_mask;
+				rb1_mask &= rb_mask;
+				if (!rb0_mask || !rb1_mask) {
+					raster_config_se &= ~RB_MAP_PKR1_MASK;
+
+					if (!rb0_mask) {
+						raster_config_se |=
+							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
+					} else {
+						raster_config_se |=
+							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
+					}
+				}
+			}
+		}
+
+		/* GRBM_GFX_INDEX has a different offset on CI+ */
+		gfx_v7_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
+		WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
+		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
+	}
+
+	/* GRBM_GFX_INDEX has a different offset on CI+ */
+	gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+}
+
 /**
  * gfx_v7_0_setup_rb - setup the RBs on the asic
  *
@@ -1658,9 +1799,11 @@ static void gfx_v7_0_setup_rb(struct amdgpu_device *adev)
 {
 	int i, j;
 	u32 data;
+	u32 raster_config = 0, raster_config_1 = 0;
 	u32 active_rbs = 0;
 	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
 					adev->gfx.config.max_sh_per_se;
+	unsigned num_rb_pipes;
 
 	mutex_lock(&adev->grbm_idx_mutex);
 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
@@ -1672,10 +1815,25 @@ static void gfx_v7_0_setup_rb(struct amdgpu_device *adev)
 		}
 	}
 	gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
-	mutex_unlock(&adev->grbm_idx_mutex);
 
 	adev->gfx.config.backend_enable_mask = active_rbs;
 	adev->gfx.config.num_rbs = hweight32(active_rbs);
+
+	num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
+			     adev->gfx.config.max_shader_engines, 16);
+
+	gfx_v7_0_raster_config(adev, &raster_config, &raster_config_1);
+
+	if (!adev->gfx.config.backend_enable_mask ||
+			adev->gfx.config.num_rbs >= num_rb_pipes) {
+		WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
+		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
+	} else {
+		gfx_v7_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
+							adev->gfx.config.backend_enable_mask,
+							num_rb_pipes);
+	}
+	mutex_unlock(&adev->grbm_idx_mutex);
 }
 
 /**
@@ -2096,6 +2254,25 @@ static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, control);
 }
 
+static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
+{
+	uint32_t dw2 = 0;
+
+	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
+	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
+		/* set load_global_config & load_global_uconfig */
+		dw2 |= 0x8001;
+		/* set load_cs_sh_regs */
+		dw2 |= 0x01000000;
+		/* set load_per_context_state & load_gfx_sh_regs */
+		dw2 |= 0x10002;
+	}
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+	amdgpu_ring_write(ring, dw2);
+	amdgpu_ring_write(ring, 0);
+}
+
 /**
  * gfx_v7_0_ring_test_ib - basic ring IB test
  *
@@ -2443,7 +2620,7 @@ static int gfx_v7_0_cp_gfx_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-static u32 gfx_v7_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
+static u32 gfx_v7_0_ring_get_rptr(struct amdgpu_ring *ring)
 {
 	return ring->adev->wb.wb[ring->rptr_offs];
 }
@@ -2463,11 +2640,6 @@ static void gfx_v7_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
 	(void)RREG32(mmCP_RB0_WPTR);
 }
 
-static u32 gfx_v7_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
-{
-	return ring->adev->wb.wb[ring->rptr_offs];
-}
-
 static u32 gfx_v7_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
 {
 	/* XXX check if swapping is necessary on BE */
@@ -2755,8 +2927,7 @@ static int gfx_v7_0_cp_compute_resume(struct amdgpu_device *adev)
 	u64 wb_gpu_addr;
 	u32 *buf;
 	struct bonaire_mqd *mqd;
-
-	gfx_v7_0_cp_compute_enable(adev, true);
+	struct amdgpu_ring *ring;
 
 	/* fix up chicken bits */
 	tmp = RREG32(mmCP_CPF_DEBUG);
@@ -2791,7 +2962,7 @@ static int gfx_v7_0_cp_compute_resume(struct amdgpu_device *adev)
 
 	/* init the queues.  Just two for now. */
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
-		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
+		ring = &adev->gfx.compute_ring[i];
 
 		if (ring->mqd_obj == NULL) {
 			r = amdgpu_bo_create(adev,
@@ -2970,6 +3141,13 @@ static int gfx_v7_0_cp_compute_resume(struct amdgpu_device *adev)
 		amdgpu_bo_unreserve(ring->mqd_obj);
 
 		ring->ready = true;
+	}
+
+	gfx_v7_0_cp_compute_enable(adev, true);
+
+	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+		ring = &adev->gfx.compute_ring[i];
+
 		r = amdgpu_ring_test_ring(ring);
 		if (r)
 			ring->ready = false;
@@ -4176,6 +4354,41 @@ static void gfx_v7_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
 }
 
+static unsigned gfx_v7_0_ring_get_emit_ib_size_gfx(struct amdgpu_ring *ring)
+{
+	return
+		4; /* gfx_v7_0_ring_emit_ib_gfx */
+}
+
+static unsigned gfx_v7_0_ring_get_dma_frame_size_gfx(struct amdgpu_ring *ring)
+{
+	return
+		20 + /* gfx_v7_0_ring_emit_gds_switch */
+		7 + /* gfx_v7_0_ring_emit_hdp_flush */
+		5 + /* gfx_v7_0_ring_emit_hdp_invalidate */
+		12 + 12 + 12 + /* gfx_v7_0_ring_emit_fence_gfx x3 for user fence, vm fence */
+		7 + 4 + /* gfx_v7_0_ring_emit_pipeline_sync */
+		17 + 6 + /* gfx_v7_0_ring_emit_vm_flush */
+		3; /* gfx_v7_ring_emit_cntxcntl */
+}
+
+static unsigned gfx_v7_0_ring_get_emit_ib_size_compute(struct amdgpu_ring *ring)
+{
+	return
+		4; /* gfx_v7_0_ring_emit_ib_compute */
+}
+
+static unsigned gfx_v7_0_ring_get_dma_frame_size_compute(struct amdgpu_ring *ring)
+{
+	return
+		20 + /* gfx_v7_0_ring_emit_gds_switch */
+		7 + /* gfx_v7_0_ring_emit_hdp_flush */
+		5 + /* gfx_v7_0_ring_emit_hdp_invalidate */
+		7 + /* gfx_v7_0_ring_emit_pipeline_sync */
+		17 + /* gfx_v7_0_ring_emit_vm_flush */
+		7 + 7 + 7; /* gfx_v7_0_ring_emit_fence_compute x3 for user fence, vm fence */
+}
+
 static const struct amdgpu_gfx_funcs gfx_v7_0_gfx_funcs = {
 	.get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
 	.select_se_sh = &gfx_v7_0_select_se_sh,
@@ -4465,24 +4678,21 @@ static int gfx_v7_0_sw_init(void *handle)
 	}
 
 	/* reserve GDS, GWS and OA resource for gfx */
-	r = amdgpu_bo_create(adev, adev->gds.mem.gfx_partition_size,
-			PAGE_SIZE, true,
-			AMDGPU_GEM_DOMAIN_GDS, 0,
-			NULL, NULL, &adev->gds.gds_gfx_bo);
+	r = amdgpu_bo_create_kernel(adev, adev->gds.mem.gfx_partition_size,
+				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GDS,
+				    &adev->gds.gds_gfx_bo, NULL, NULL);
 	if (r)
 		return r;
 
-	r = amdgpu_bo_create(adev, adev->gds.gws.gfx_partition_size,
-		PAGE_SIZE, true,
-		AMDGPU_GEM_DOMAIN_GWS, 0,
-		NULL, NULL, &adev->gds.gws_gfx_bo);
+	r = amdgpu_bo_create_kernel(adev, adev->gds.gws.gfx_partition_size,
+				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GWS,
+				    &adev->gds.gws_gfx_bo, NULL, NULL);
 	if (r)
 		return r;
 
-	r = amdgpu_bo_create(adev, adev->gds.oa.gfx_partition_size,
-			PAGE_SIZE, true,
-			AMDGPU_GEM_DOMAIN_OA, 0,
-			NULL, NULL, &adev->gds.oa_gfx_bo);
+	r = amdgpu_bo_create_kernel(adev, adev->gds.oa.gfx_partition_size,
+				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_OA,
+				    &adev->gds.oa_gfx_bo, NULL, NULL);
 	if (r)
 		return r;
 
@@ -4498,9 +4708,9 @@ static int gfx_v7_0_sw_fini(void *handle)
 	int i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	amdgpu_bo_unref(&adev->gds.oa_gfx_bo);
-	amdgpu_bo_unref(&adev->gds.gws_gfx_bo);
-	amdgpu_bo_unref(&adev->gds.gds_gfx_bo);
+	amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
 
 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
 		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
@@ -4931,7 +5141,7 @@ const struct amd_ip_funcs gfx_v7_0_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
-	.get_rptr = gfx_v7_0_ring_get_rptr_gfx,
+	.get_rptr = gfx_v7_0_ring_get_rptr,
 	.get_wptr = gfx_v7_0_ring_get_wptr_gfx,
 	.set_wptr = gfx_v7_0_ring_set_wptr_gfx,
 	.parse_cs = NULL,
@@ -4946,10 +5156,13 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
 	.test_ib = gfx_v7_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.emit_cntxcntl = gfx_v7_ring_emit_cntxcntl,
+	.get_emit_ib_size = gfx_v7_0_ring_get_emit_ib_size_gfx,
+	.get_dma_frame_size = gfx_v7_0_ring_get_dma_frame_size_gfx,
 };
 
 static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
-	.get_rptr = gfx_v7_0_ring_get_rptr_compute,
+	.get_rptr = gfx_v7_0_ring_get_rptr,
 	.get_wptr = gfx_v7_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v7_0_ring_set_wptr_compute,
 	.parse_cs = NULL,
@@ -4964,6 +5177,8 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
 	.test_ib = gfx_v7_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.get_emit_ib_size = gfx_v7_0_ring_get_emit_ib_size_compute,
+	.get_dma_frame_size = gfx_v7_0_ring_get_dma_frame_size_compute,
 };
 
 static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index bff8668e9e6d..6c6ff57b1c95 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -270,7 +270,8 @@ static const u32 tonga_mgcg_cgcg_init[] =
 
 static const u32 golden_settings_polaris11_a11[] =
 {
-	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00006208,
+	mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208,
+	mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
@@ -279,7 +280,7 @@ static const u32 golden_settings_polaris11_a11[] =
 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
 	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
-	mmSQ_CONFIG, 0x07f80000, 0x07180000,
+	mmSQ_CONFIG, 0x07f80000, 0x01180000,
 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
@@ -301,8 +302,8 @@ static const u32 polaris11_golden_common_all[] =
 static const u32 golden_settings_polaris10_a11[] =
 {
 	mmATC_MISC_CG, 0x000c0fc0, 0x000c0200,
-	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
-	mmCB_HW_CONTROL_2, 0, 0x0f000000,
+	mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
+	mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
@@ -409,6 +410,7 @@ static const u32 golden_settings_iceland_a11[] =
 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
 	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002,
 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
@@ -505,8 +507,10 @@ static const u32 cz_golden_settings_a11[] =
 	mmGB_GPU_ID, 0x0000000f, 0x00000000,
 	mmPA_SC_ENHANCE, 0xffffffff, 0x00000001,
 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 	mmTA_CNTL_AUX, 0x000f000f, 0x00010000,
+	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
 	mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3,
 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302
@@ -699,7 +703,10 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 						 polaris10_golden_common_all,
 						 (const u32)ARRAY_SIZE(polaris10_golden_common_all));
 		WREG32_SMC(ixCG_ACLK_CNTL, 0x0000001C);
-		if (adev->pdev->revision == 0xc7) {
+		if (adev->pdev->revision == 0xc7 &&
+		    ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||
+		     (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) ||
+		     (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1682))) {
 			amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD);
 			amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0);
 		}
@@ -1229,10 +1236,9 @@ static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev)
 	if (adev->gfx.rlc.clear_state_obj) {
 		r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
 		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve RLC c bo failed\n", r);
+			dev_warn(adev->dev, "(%d) reserve RLC cbs bo failed\n", r);
 		amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
 		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-
 		amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
 		adev->gfx.rlc.clear_state_obj = NULL;
 	}
@@ -1244,7 +1250,6 @@ static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev)
 			dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
 		amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
 		amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
-
 		amdgpu_bo_unref(&adev->gfx.rlc.cp_table_obj);
 		adev->gfx.rlc.cp_table_obj = NULL;
 	}
@@ -1286,14 +1291,14 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 				  &adev->gfx.rlc.clear_state_gpu_addr);
 		if (r) {
 			amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-			dev_warn(adev->dev, "(%d) pin RLC c bo failed\n", r);
+			dev_warn(adev->dev, "(%d) pin RLC cbs bo failed\n", r);
 			gfx_v8_0_rlc_fini(adev);
 			return r;
 		}
 
 		r = amdgpu_bo_kmap(adev->gfx.rlc.clear_state_obj, (void **)&adev->gfx.rlc.cs_ptr);
 		if (r) {
-			dev_warn(adev->dev, "(%d) map RLC c bo failed\n", r);
+			dev_warn(adev->dev, "(%d) map RLC cbs bo failed\n", r);
 			gfx_v8_0_rlc_fini(adev);
 			return r;
 		}
@@ -1328,7 +1333,7 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 				  &adev->gfx.rlc.cp_table_gpu_addr);
 		if (r) {
 			amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
-			dev_warn(adev->dev, "(%d) pin RLC cp_table bo failed\n", r);
+			dev_warn(adev->dev, "(%d) pin RLC cp table bo failed\n", r);
 			return r;
 		}
 		r = amdgpu_bo_kmap(adev->gfx.rlc.cp_table_obj, (void **)&adev->gfx.rlc.cp_table_ptr);
@@ -1341,7 +1346,6 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 
 		amdgpu_bo_kunmap(adev->gfx.rlc.cp_table_obj);
 		amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
-
 	}
 
 	return 0;
@@ -1357,7 +1361,6 @@ static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
 			dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
 		amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
 		amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
-
 		amdgpu_bo_unref(&adev->gfx.mec.hpd_eop_obj);
 		adev->gfx.mec.hpd_eop_obj = NULL;
 	}
@@ -2078,24 +2081,21 @@ static int gfx_v8_0_sw_init(void *handle)
 	}
 
 	/* reserve GDS, GWS and OA resource for gfx */
-	r = amdgpu_bo_create(adev, adev->gds.mem.gfx_partition_size,
-			PAGE_SIZE, true,
-			AMDGPU_GEM_DOMAIN_GDS, 0, NULL,
-			NULL, &adev->gds.gds_gfx_bo);
+	r = amdgpu_bo_create_kernel(adev, adev->gds.mem.gfx_partition_size,
+				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GDS,
+				    &adev->gds.gds_gfx_bo, NULL, NULL);
 	if (r)
 		return r;
 
-	r = amdgpu_bo_create(adev, adev->gds.gws.gfx_partition_size,
-		PAGE_SIZE, true,
-		AMDGPU_GEM_DOMAIN_GWS, 0, NULL,
-		NULL, &adev->gds.gws_gfx_bo);
+	r = amdgpu_bo_create_kernel(adev, adev->gds.gws.gfx_partition_size,
+				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GWS,
+				    &adev->gds.gws_gfx_bo, NULL, NULL);
 	if (r)
 		return r;
 
-	r = amdgpu_bo_create(adev, adev->gds.oa.gfx_partition_size,
-			PAGE_SIZE, true,
-			AMDGPU_GEM_DOMAIN_OA, 0, NULL,
-			NULL, &adev->gds.oa_gfx_bo);
+	r = amdgpu_bo_create_kernel(adev, adev->gds.oa.gfx_partition_size,
+				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_OA,
+				    &adev->gds.oa_gfx_bo, NULL, NULL);
 	if (r)
 		return r;
 
@@ -2113,9 +2113,9 @@ static int gfx_v8_0_sw_fini(void *handle)
 	int i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	amdgpu_bo_unref(&adev->gds.oa_gfx_bo);
-	amdgpu_bo_unref(&adev->gds.gws_gfx_bo);
-	amdgpu_bo_unref(&adev->gds.gds_gfx_bo);
+	amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
 
 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
 		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
@@ -2123,9 +2123,7 @@ static int gfx_v8_0_sw_fini(void *handle)
 		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
 
 	gfx_v8_0_mec_fini(adev);
-
 	gfx_v8_0_rlc_fini(adev);
-
 	gfx_v8_0_free_microcode(adev);
 
 	return 0;
@@ -3461,19 +3459,16 @@ static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
 	else
 		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
 
-	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) {
-		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
+	if (se_num == 0xffffffff)
 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
-	} else if (se_num == 0xffffffff) {
-		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
-		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
-	} else if (sh_num == 0xffffffff) {
-		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
+	else
 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
-	} else {
+
+	if (sh_num == 0xffffffff)
+		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
+	else
 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
-		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
-	}
+
 	WREG32(mmGRBM_GFX_INDEX, data);
 }
 
@@ -3486,11 +3481,10 @@ static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
 {
 	u32 data, mask;
 
-	data = RREG32(mmCC_RB_BACKEND_DISABLE);
-	data |= RREG32(mmGC_USER_RB_BACKEND_DISABLE);
+	data =  RREG32(mmCC_RB_BACKEND_DISABLE) |
+		RREG32(mmGC_USER_RB_BACKEND_DISABLE);
 
-	data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
-	data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
+	data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE);
 
 	mask = gfx_v8_0_create_bitmask(adev->gfx.config.max_backends_per_se /
 				       adev->gfx.config.max_sh_per_se);
@@ -3498,13 +3492,163 @@ static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
 	return (~data) & mask;
 }
 
+static void
+gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
+{
+	switch (adev->asic_type) {
+	case CHIP_FIJI:
+		*rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
+			  RB_XSEL2(1) | PKR_MAP(2) |
+			  PKR_XSEL(1) | PKR_YSEL(1) |
+			  SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3);
+		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
+			   SE_PAIR_YSEL(2);
+		break;
+	case CHIP_TONGA:
+	case CHIP_POLARIS10:
+		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
+			  SE_XSEL(1) | SE_YSEL(1);
+		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) |
+			   SE_PAIR_YSEL(2);
+		break;
+	case CHIP_TOPAZ:
+	case CHIP_CARRIZO:
+		*rconf |= RB_MAP_PKR0(2);
+		*rconf1 |= 0x0;
+		break;
+	case CHIP_POLARIS11:
+		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
+			  SE_XSEL(1) | SE_YSEL(1);
+		*rconf1 |= 0x0;
+		break;
+	case CHIP_STONEY:
+		*rconf |= 0x0;
+		*rconf1 |= 0x0;
+		break;
+	default:
+		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
+		break;
+	}
+}
+
+static void
+gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev,
+					u32 raster_config, u32 raster_config_1,
+					unsigned rb_mask, unsigned num_rb)
+{
+	unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
+	unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
+	unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
+	unsigned rb_per_se = num_rb / num_se;
+	unsigned se_mask[4];
+	unsigned se;
+
+	se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
+	se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
+	se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
+	se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
+
+	WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
+	WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
+	WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
+
+	if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
+			     (!se_mask[2] && !se_mask[3]))) {
+		raster_config_1 &= ~SE_PAIR_MAP_MASK;
+
+		if (!se_mask[0] && !se_mask[1]) {
+			raster_config_1 |=
+				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
+		} else {
+			raster_config_1 |=
+				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
+		}
+	}
+
+	for (se = 0; se < num_se; se++) {
+		unsigned raster_config_se = raster_config;
+		unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
+		unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
+		int idx = (se / 2) * 2;
+
+		if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
+			raster_config_se &= ~SE_MAP_MASK;
+
+			if (!se_mask[idx]) {
+				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
+			} else {
+				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
+			}
+		}
+
+		pkr0_mask &= rb_mask;
+		pkr1_mask &= rb_mask;
+		if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
+			raster_config_se &= ~PKR_MAP_MASK;
+
+			if (!pkr0_mask) {
+				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
+			} else {
+				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
+			}
+		}
+
+		if (rb_per_se >= 2) {
+			unsigned rb0_mask = 1 << (se * rb_per_se);
+			unsigned rb1_mask = rb0_mask << 1;
+
+			rb0_mask &= rb_mask;
+			rb1_mask &= rb_mask;
+			if (!rb0_mask || !rb1_mask) {
+				raster_config_se &= ~RB_MAP_PKR0_MASK;
+
+				if (!rb0_mask) {
+					raster_config_se |=
+						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
+				} else {
+					raster_config_se |=
+						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
+				}
+			}
+
+			if (rb_per_se > 2) {
+				rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
+				rb1_mask = rb0_mask << 1;
+				rb0_mask &= rb_mask;
+				rb1_mask &= rb_mask;
+				if (!rb0_mask || !rb1_mask) {
+					raster_config_se &= ~RB_MAP_PKR1_MASK;
+
+					if (!rb0_mask) {
+						raster_config_se |=
+							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
+					} else {
+						raster_config_se |=
+							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
+					}
+				}
+			}
+		}
+
+		/* GRBM_GFX_INDEX has a different offset on VI */
+		gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
+		WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
+		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
+	}
+
+	/* GRBM_GFX_INDEX has a different offset on VI */
+	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+}
+
 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
 {
 	int i, j;
 	u32 data;
+	u32 raster_config = 0, raster_config_1 = 0;
 	u32 active_rbs = 0;
 	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
 					adev->gfx.config.max_sh_per_se;
+	unsigned num_rb_pipes;
 
 	mutex_lock(&adev->grbm_idx_mutex);
 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
@@ -3516,10 +3660,26 @@ static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
 		}
 	}
 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
-	mutex_unlock(&adev->grbm_idx_mutex);
 
 	adev->gfx.config.backend_enable_mask = active_rbs;
 	adev->gfx.config.num_rbs = hweight32(active_rbs);
+
+	num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
+			     adev->gfx.config.max_shader_engines, 16);
+
+	gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1);
+
+	if (!adev->gfx.config.backend_enable_mask ||
+			adev->gfx.config.num_rbs >= num_rb_pipes) {
+		WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
+		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
+	} else {
+		gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
+							adev->gfx.config.backend_enable_mask,
+							num_rb_pipes);
+	}
+
+	mutex_unlock(&adev->grbm_idx_mutex);
 }
 
 /**
@@ -3572,16 +3732,12 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
 	u32 tmp;
 	int i;
 
-	tmp = RREG32(mmGRBM_CNTL);
-	tmp = REG_SET_FIELD(tmp, GRBM_CNTL, READ_TIMEOUT, 0xff);
-	WREG32(mmGRBM_CNTL, tmp);
-
+	WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF);
 	WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 	WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 	WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
 
 	gfx_v8_0_tiling_mode_table_init(adev);
-
 	gfx_v8_0_setup_rb(adev);
 	gfx_v8_0_get_cu_info(adev);
 
@@ -3765,9 +3921,7 @@ static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
 				sizeof(indirect_start_offsets)/sizeof(int));
 
 	/* save and restore list */
-	temp = RREG32(mmRLC_SRM_CNTL);
-	temp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
-	WREG32(mmRLC_SRM_CNTL, temp);
+	WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);
 
 	WREG32(mmRLC_SRM_ARAM_ADDR, 0);
 	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
@@ -3804,11 +3958,7 @@ static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
 
 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)
 {
-	uint32_t data;
-
-	data = RREG32(mmRLC_SRM_CNTL);
-	data |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
-	WREG32(mmRLC_SRM_CNTL, data);
+	WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1);
 }
 
 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
@@ -3818,75 +3968,34 @@ static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
 			      AMD_PG_SUPPORT_GFX_SMG |
 			      AMD_PG_SUPPORT_GFX_DMG)) {
-		data = RREG32(mmCP_RB_WPTR_POLL_CNTL);
-		data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
-		data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
-		WREG32(mmCP_RB_WPTR_POLL_CNTL, data);
-
-		data = 0;
-		data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
-		data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
-		data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
-		data |= (0x10 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
-		WREG32(mmRLC_PG_DELAY, data);
+		WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60);
 
-		data = RREG32(mmRLC_PG_DELAY_2);
-		data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
-		data |= (0x3 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
-		WREG32(mmRLC_PG_DELAY_2, data);
+		data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10);
+		data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10);
+		data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10);
+		data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10);
+		WREG32(mmRLC_PG_DELAY, data);
 
-		data = RREG32(mmRLC_AUTO_PG_CTRL);
-		data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
-		data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
-		WREG32(mmRLC_AUTO_PG_CTRL, data);
+		WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3);
+		WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0);
 	}
 }
 
 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
 						bool enable)
 {
-	u32 data, orig;
-
-	orig = data = RREG32(mmRLC_PG_CNTL);
-
-	if (enable)
-		data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
-	else
-		data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
-
-	if (orig != data)
-		WREG32(mmRLC_PG_CNTL, data);
+	WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0);
 }
 
 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
 						  bool enable)
 {
-	u32 data, orig;
-
-	orig = data = RREG32(mmRLC_PG_CNTL);
-
-	if (enable)
-		data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
-	else
-		data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
-
-	if (orig != data)
-		WREG32(mmRLC_PG_CNTL, data);
+	WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0);
 }
 
 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)
 {
-	u32 data, orig;
-
-	orig = data = RREG32(mmRLC_PG_CNTL);
-
-	if (enable)
-		data &= ~RLC_PG_CNTL__CP_PG_DISABLE_MASK;
-	else
-		data |= RLC_PG_CNTL__CP_PG_DISABLE_MASK;
-
-	if (orig != data)
-		WREG32(mmRLC_PG_CNTL, data);
+	WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 1 : 0);
 }
 
 static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
@@ -3923,36 +4032,26 @@ static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
 	}
 }
 
-void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)
+static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)
 {
-	u32 tmp = RREG32(mmRLC_CNTL);
-
-	tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
-	WREG32(mmRLC_CNTL, tmp);
+	WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0);
 
 	gfx_v8_0_enable_gui_idle_interrupt(adev, false);
-
 	gfx_v8_0_wait_for_rlc_serdes(adev);
 }
 
 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev)
 {
-	u32 tmp = RREG32(mmGRBM_SOFT_RESET);
-
-	tmp = REG_SET_FIELD(tmp, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
-	WREG32(mmGRBM_SOFT_RESET, tmp);
+	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
 	udelay(50);
-	tmp = REG_SET_FIELD(tmp, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
-	WREG32(mmGRBM_SOFT_RESET, tmp);
+
+	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
 	udelay(50);
 }
 
 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev)
 {
-	u32 tmp = RREG32(mmRLC_CNTL);
-
-	tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 1);
-	WREG32(mmRLC_CNTL, tmp);
+	WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1);
 
 	/* carrizo do enable cp interrupt after cp inited */
 	if (!(adev->flags & AMD_IS_APU))
@@ -3994,14 +4093,13 @@ static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
 	/* disable CG */
 	WREG32(mmRLC_CGCG_CGLS_CTRL, 0);
 	if (adev->asic_type == CHIP_POLARIS11 ||
-		adev->asic_type == CHIP_POLARIS10)
+	    adev->asic_type == CHIP_POLARIS10)
 		WREG32(mmRLC_CGCG_CGLS_CTRL_3D, 0);
 
 	/* disable PG */
 	WREG32(mmRLC_PG_CNTL, 0);
 
 	gfx_v8_0_rlc_reset(adev);
-
 	gfx_v8_0_init_pg(adev);
 
 	if (!adev->pp_enabled) {
@@ -4296,12 +4394,10 @@ static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
 	gfx_v8_0_cp_gfx_start(adev);
 	ring->ready = true;
 	r = amdgpu_ring_test_ring(ring);
-	if (r) {
+	if (r)
 		ring->ready = false;
-		return r;
-	}
 
-	return 0;
+	return r;
 }
 
 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
@@ -4976,7 +5072,6 @@ static int gfx_v8_0_hw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	gfx_v8_0_init_golden_registers(adev);
-
 	gfx_v8_0_gpu_init(adev);
 
 	r = gfx_v8_0_rlc_resume(adev);
@@ -4984,8 +5079,6 @@ static int gfx_v8_0_hw_init(void *handle)
 		return r;
 
 	r = gfx_v8_0_cp_resume(adev);
-	if (r)
-		return r;
 
 	return r;
 }
@@ -5033,25 +5126,22 @@ static bool gfx_v8_0_is_idle(void *handle)
 static int gfx_v8_0_wait_for_idle(void *handle)
 {
 	unsigned i;
-	u32 tmp;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		/* read MC_STATUS */
-		tmp = RREG32(mmGRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK;
-
-		if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
+		if (gfx_v8_0_is_idle(handle))
 			return 0;
+
 		udelay(1);
 	}
 	return -ETIMEDOUT;
 }
 
-static int gfx_v8_0_soft_reset(void *handle)
+static int gfx_v8_0_check_soft_reset(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
 	u32 tmp;
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* GRBM_STATUS */
 	tmp = RREG32(mmGRBM_STATUS);
@@ -5060,16 +5150,12 @@ static int gfx_v8_0_soft_reset(void *handle)
 		   GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
 		   GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
 		   GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
-		   GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
+		   GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK |
+		   GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
 						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
 						GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
-	}
-
-	if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
-		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
-						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
 						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
 	}
@@ -5080,73 +5166,199 @@ static int gfx_v8_0_soft_reset(void *handle)
 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
 						GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
 
+	if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) ||
+	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) ||
+	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) {
+		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
+						SOFT_RESET_CPF, 1);
+		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
+						SOFT_RESET_CPC, 1);
+		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
+						SOFT_RESET_CPG, 1);
+		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
+						SOFT_RESET_GRBM, 1);
+	}
+
 	/* SRBM_STATUS */
 	tmp = RREG32(mmSRBM_STATUS);
 	if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING))
 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
 						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
+	if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY))
+		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
+						SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
 
 	if (grbm_soft_reset || srbm_soft_reset) {
-		/* stop the rlc */
-		gfx_v8_0_rlc_stop(adev);
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang = true;
+		adev->gfx.grbm_soft_reset = grbm_soft_reset;
+		adev->gfx.srbm_soft_reset = srbm_soft_reset;
+	} else {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang = false;
+		adev->gfx.grbm_soft_reset = 0;
+		adev->gfx.srbm_soft_reset = 0;
+	}
+
+	return 0;
+}
+
+static void gfx_v8_0_inactive_hqd(struct amdgpu_device *adev,
+				  struct amdgpu_ring *ring)
+{
+	int i;
+
+	vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+	if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {
+		u32 tmp;
+		tmp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_DEQUEUE_REQUEST,
+				    DEQUEUE_REQ, 2);
+		WREG32(mmCP_HQD_DEQUEUE_REQUEST, tmp);
+		for (i = 0; i < adev->usec_timeout; i++) {
+			if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))
+				break;
+			udelay(1);
+		}
+	}
+}
 
+static int gfx_v8_0_pre_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang)
+		return 0;
+
+	grbm_soft_reset = adev->gfx.grbm_soft_reset;
+	srbm_soft_reset = adev->gfx.srbm_soft_reset;
+
+	/* stop the rlc */
+	gfx_v8_0_rlc_stop(adev);
+
+	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
+	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
 		/* Disable GFX parsing/prefetching */
 		gfx_v8_0_cp_gfx_enable(adev, false);
 
+	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
+	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
+	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
+	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
+		int i;
+
+		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
+
+			gfx_v8_0_inactive_hqd(adev, ring);
+		}
 		/* Disable MEC parsing/prefetching */
 		gfx_v8_0_cp_compute_enable(adev, false);
+	}
 
-		if (grbm_soft_reset || srbm_soft_reset) {
-			tmp = RREG32(mmGMCON_DEBUG);
-			tmp = REG_SET_FIELD(tmp,
-					    GMCON_DEBUG, GFX_STALL, 1);
-			tmp = REG_SET_FIELD(tmp,
-					    GMCON_DEBUG, GFX_CLEAR, 1);
-			WREG32(mmGMCON_DEBUG, tmp);
+       return 0;
+}
 
-			udelay(50);
-		}
+static int gfx_v8_0_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
+	u32 tmp;
 
-		if (grbm_soft_reset) {
-			tmp = RREG32(mmGRBM_SOFT_RESET);
-			tmp |= grbm_soft_reset;
-			dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
-			WREG32(mmGRBM_SOFT_RESET, tmp);
-			tmp = RREG32(mmGRBM_SOFT_RESET);
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang)
+		return 0;
 
-			udelay(50);
+	grbm_soft_reset = adev->gfx.grbm_soft_reset;
+	srbm_soft_reset = adev->gfx.srbm_soft_reset;
 
-			tmp &= ~grbm_soft_reset;
-			WREG32(mmGRBM_SOFT_RESET, tmp);
-			tmp = RREG32(mmGRBM_SOFT_RESET);
-		}
+	if (grbm_soft_reset || srbm_soft_reset) {
+		tmp = RREG32(mmGMCON_DEBUG);
+		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1);
+		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1);
+		WREG32(mmGMCON_DEBUG, tmp);
+		udelay(50);
+	}
 
-		if (srbm_soft_reset) {
-			tmp = RREG32(mmSRBM_SOFT_RESET);
-			tmp |= srbm_soft_reset;
-			dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
-			WREG32(mmSRBM_SOFT_RESET, tmp);
-			tmp = RREG32(mmSRBM_SOFT_RESET);
+	if (grbm_soft_reset) {
+		tmp = RREG32(mmGRBM_SOFT_RESET);
+		tmp |= grbm_soft_reset;
+		dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
+		WREG32(mmGRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmGRBM_SOFT_RESET);
 
-			udelay(50);
+		udelay(50);
 
-			tmp &= ~srbm_soft_reset;
-			WREG32(mmSRBM_SOFT_RESET, tmp);
-			tmp = RREG32(mmSRBM_SOFT_RESET);
-		}
+		tmp &= ~grbm_soft_reset;
+		WREG32(mmGRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmGRBM_SOFT_RESET);
+	}
 
-		if (grbm_soft_reset || srbm_soft_reset) {
-			tmp = RREG32(mmGMCON_DEBUG);
-			tmp = REG_SET_FIELD(tmp,
-					    GMCON_DEBUG, GFX_STALL, 0);
-			tmp = REG_SET_FIELD(tmp,
-					    GMCON_DEBUG, GFX_CLEAR, 0);
-			WREG32(mmGMCON_DEBUG, tmp);
-		}
+	if (srbm_soft_reset) {
+		tmp = RREG32(mmSRBM_SOFT_RESET);
+		tmp |= srbm_soft_reset;
+		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
+		WREG32(mmSRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmSRBM_SOFT_RESET);
 
-		/* Wait a little for things to settle down */
 		udelay(50);
+
+		tmp &= ~srbm_soft_reset;
+		WREG32(mmSRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmSRBM_SOFT_RESET);
 	}
+
+	if (grbm_soft_reset || srbm_soft_reset) {
+		tmp = RREG32(mmGMCON_DEBUG);
+		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0);
+		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0);
+		WREG32(mmGMCON_DEBUG, tmp);
+	}
+
+	/* Wait a little for things to settle down */
+	udelay(50);
+
+	return 0;
+}
+
+static void gfx_v8_0_init_hqd(struct amdgpu_device *adev,
+			      struct amdgpu_ring *ring)
+{
+	vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+	WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
+	WREG32(mmCP_HQD_PQ_RPTR, 0);
+	WREG32(mmCP_HQD_PQ_WPTR, 0);
+	vi_srbm_select(adev, 0, 0, 0, 0);
+}
+
+static int gfx_v8_0_post_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GFX].hang)
+		return 0;
+
+	grbm_soft_reset = adev->gfx.grbm_soft_reset;
+	srbm_soft_reset = adev->gfx.srbm_soft_reset;
+
+	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
+	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
+		gfx_v8_0_cp_gfx_resume(adev);
+
+	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
+	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
+	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
+	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
+		int i;
+
+		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
+
+			gfx_v8_0_init_hqd(adev, ring);
+		}
+		gfx_v8_0_cp_compute_resume(adev);
+	}
+	gfx_v8_0_rlc_start(adev);
+
 	return 0;
 }
 
@@ -5265,8 +5477,6 @@ static int gfx_v8_0_late_init(void *handle)
 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
 						       bool enable)
 {
-	uint32_t data, temp;
-
 	if (adev->asic_type == CHIP_POLARIS11)
 		/* Send msg to SMU via Powerplay */
 		amdgpu_set_powergating_state(adev,
@@ -5274,83 +5484,35 @@ static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *ade
 					     enable ?
 					     AMD_PG_STATE_GATE : AMD_PG_STATE_UNGATE);
 
-	temp = data = RREG32(mmRLC_PG_CNTL);
-	/* Enable static MGPG */
-	if (enable)
-		data |= RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
-	else
-		data &= ~RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
-
-	if (temp != data)
-		WREG32(mmRLC_PG_CNTL, data);
+	WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);
 }
 
 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
 							bool enable)
 {
-	uint32_t data, temp;
-
-	temp = data = RREG32(mmRLC_PG_CNTL);
-	/* Enable dynamic MGPG */
-	if (enable)
-		data |= RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
-	else
-		data &= ~RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
-
-	if (temp != data)
-		WREG32(mmRLC_PG_CNTL, data);
+	WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0);
 }
 
 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,
 		bool enable)
 {
-	uint32_t data, temp;
-
-	temp = data = RREG32(mmRLC_PG_CNTL);
-	/* Enable quick PG */
-	if (enable)
-		data |= RLC_PG_CNTL__QUICK_PG_ENABLE_MASK;
-	else
-		data &= ~RLC_PG_CNTL__QUICK_PG_ENABLE_MASK;
-
-	if (temp != data)
-		WREG32(mmRLC_PG_CNTL, data);
+	WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0);
 }
 
 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
 					  bool enable)
 {
-	u32 data, orig;
-
-	orig = data = RREG32(mmRLC_PG_CNTL);
-
-	if (enable)
-		data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
-	else
-		data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
-
-	if (orig != data)
-		WREG32(mmRLC_PG_CNTL, data);
+	WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0);
 }
 
 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,
 						bool enable)
 {
-	u32 data, orig;
-
-	orig = data = RREG32(mmRLC_PG_CNTL);
-
-	if (enable)
-		data |= RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK;
-	else
-		data &= ~RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK;
-
-	if (orig != data)
-		WREG32(mmRLC_PG_CNTL, data);
+	WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0);
 
 	/* Read any GFX register to wake up GFX. */
 	if (!enable)
-		data = RREG32(mmDB_RENDER_CONTROL);
+		RREG32(mmDB_RENDER_CONTROL);
 }
 
 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,
@@ -5426,15 +5588,15 @@ static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
 
 	data = RREG32(mmRLC_SERDES_WR_CTRL);
 	if (adev->asic_type == CHIP_STONEY)
-			data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
-			RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
-			RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
-			RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
-			RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
-			RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
-			RLC_SERDES_WR_CTRL__POWER_UP_MASK |
-			RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
-			RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
+		data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
+			  RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
+			  RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
+			  RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
+			  RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
+			  RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
+			  RLC_SERDES_WR_CTRL__POWER_UP_MASK |
+			  RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
+			  RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
 	else
 		data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
 			  RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
@@ -5457,10 +5619,10 @@ static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
 
 #define MSG_ENTER_RLC_SAFE_MODE     1
 #define MSG_EXIT_RLC_SAFE_MODE      0
-
-#define RLC_GPR_REG2__REQ_MASK           0x00000001
-#define RLC_GPR_REG2__MESSAGE__SHIFT     0x00000001
-#define RLC_GPR_REG2__MESSAGE_MASK       0x0000001e
+#define RLC_GPR_REG2__REQ_MASK 0x00000001
+#define RLC_GPR_REG2__REQ__SHIFT 0
+#define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001
+#define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e
 
 static void cz_enter_rlc_safe_mode(struct amdgpu_device *adev)
 {
@@ -5490,7 +5652,7 @@ static void cz_enter_rlc_safe_mode(struct amdgpu_device *adev)
 		}
 
 		for (i = 0; i < adev->usec_timeout; i++) {
-			if ((RREG32(mmRLC_GPR_REG2) & RLC_GPR_REG2__REQ_MASK) == 0)
+			if (!REG_GET_FIELD(RREG32(mmRLC_GPR_REG2), RLC_GPR_REG2, REQ))
 				break;
 			udelay(1);
 		}
@@ -5518,7 +5680,7 @@ static void cz_exit_rlc_safe_mode(struct amdgpu_device *adev)
 	}
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		if ((RREG32(mmRLC_GPR_REG2) & RLC_GPR_REG2__REQ_MASK) == 0)
+		if (!REG_GET_FIELD(RREG32(mmRLC_GPR_REG2), RLC_GPR_REG2, REQ))
 			break;
 		udelay(1);
 	}
@@ -5550,7 +5712,7 @@ static void iceland_enter_rlc_safe_mode(struct amdgpu_device *adev)
 		}
 
 		for (i = 0; i < adev->usec_timeout; i++) {
-			if ((RREG32(mmRLC_SAFE_MODE) & RLC_SAFE_MODE__CMD_MASK) == 0)
+			if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
 				break;
 			udelay(1);
 		}
@@ -5577,7 +5739,7 @@ static void iceland_exit_rlc_safe_mode(struct amdgpu_device *adev)
 	}
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		if ((RREG32(mmRLC_SAFE_MODE) & RLC_SAFE_MODE__CMD_MASK) == 0)
+		if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
 			break;
 		udelay(1);
 	}
@@ -5618,21 +5780,12 @@ static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev
 	/* It is disabled by HW by default */
 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
-			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
+			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS)
 				/* 1 - RLC memory Light sleep */
-				temp = data = RREG32(mmRLC_MEM_SLP_CNTL);
-				data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
-				if (temp != data)
-					WREG32(mmRLC_MEM_SLP_CNTL, data);
-			}
+				WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1);
 
-			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
-				/* 2 - CP memory Light sleep */
-				temp = data = RREG32(mmCP_MEM_SLP_CNTL);
-				data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
-				if (temp != data)
-					WREG32(mmCP_MEM_SLP_CNTL, data);
-			}
+			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS)
+				WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1);
 		}
 
 		/* 3 - RLC_CGTT_MGCG_OVERRIDE */
@@ -5830,6 +5983,76 @@ static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,
 	return 0;
 }
 
+static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
+					  enum amd_clockgating_state state)
+{
+	uint32_t msg_id, pp_state;
+	void *pp_handle = adev->powerplay.pp_handle;
+
+	if (state == AMD_CG_STATE_UNGATE)
+		pp_state = 0;
+	else
+		pp_state = PP_STATE_CG | PP_STATE_LS;
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
+			PP_BLOCK_GFX_CG,
+			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+			pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
+			PP_BLOCK_GFX_MG,
+			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+			pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	return 0;
+}
+
+static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
+					  enum amd_clockgating_state state)
+{
+	uint32_t msg_id, pp_state;
+	void *pp_handle = adev->powerplay.pp_handle;
+
+	if (state == AMD_CG_STATE_UNGATE)
+		pp_state = 0;
+	else
+		pp_state = PP_STATE_CG | PP_STATE_LS;
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
+			PP_BLOCK_GFX_CG,
+			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+			pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
+			PP_BLOCK_GFX_3D,
+			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+			pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
+			PP_BLOCK_GFX_MG,
+			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+			pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
+			PP_BLOCK_GFX_RLC,
+			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+			pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
+			PP_BLOCK_GFX_CP,
+			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+			pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	return 0;
+}
+
 static int gfx_v8_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
@@ -5842,33 +6065,33 @@ static int gfx_v8_0_set_clockgating_state(void *handle,
 		gfx_v8_0_update_gfx_clock_gating(adev,
 						 state == AMD_CG_STATE_GATE ? true : false);
 		break;
+	case CHIP_TONGA:
+		gfx_v8_0_tonga_update_gfx_clock_gating(adev, state);
+		break;
+	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
+		gfx_v8_0_polaris_update_gfx_clock_gating(adev, state);
+		break;
 	default:
 		break;
 	}
 	return 0;
 }
 
-static u32 gfx_v8_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
+static u32 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring)
 {
-	u32 rptr;
-
-	rptr = ring->adev->wb.wb[ring->rptr_offs];
-
-	return rptr;
+	return ring->adev->wb.wb[ring->rptr_offs];
 }
 
 static u32 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
-	u32 wptr;
 
 	if (ring->use_doorbell)
 		/* XXX check if swapping is necessary on BE */
-		wptr = ring->adev->wb.wb[ring->wptr_offs];
+		return ring->adev->wb.wb[ring->wptr_offs];
 	else
-		wptr = RREG32(mmCP_RB0_WPTR);
-
-	return wptr;
+		return RREG32(mmCP_RB0_WPTR);
 }
 
 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
@@ -5935,12 +6158,6 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 {
 	u32 header, control = 0;
 
-	/* insert SWITCH_BUFFER packet before first IB in the ring frame */
-	if (ctx_switch) {
-		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
-		amdgpu_ring_write(ring, 0);
-	}
-
 	if (ib->flags & AMDGPU_IB_FLAG_CE)
 		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
 	else
@@ -5967,9 +6184,9 @@ static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
 	amdgpu_ring_write(ring,
 #ifdef __BIG_ENDIAN
-					  (2 << 0) |
+				(2 << 0) |
 #endif
-					  (ib->gpu_addr & 0xFFFFFFFC));
+				(ib->gpu_addr & 0xFFFFFFFC));
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
 	amdgpu_ring_write(ring, control);
 }
@@ -6010,14 +6227,6 @@ static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, seq);
 	amdgpu_ring_write(ring, 0xffffffff);
 	amdgpu_ring_write(ring, 4); /* poll interval */
-
-	if (usepfp) {
-		/* synce CE with ME to prevent CE fetch CEIB before context switch done */
-		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
-		amdgpu_ring_write(ring, 0);
-		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
-		amdgpu_ring_write(ring, 0);
-	}
 }
 
 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
@@ -6025,6 +6234,10 @@ static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 {
 	int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
 
+	/* GFX8 emits 128 dw nop to prevent DE do vm_flush before CE finish CEIB */
+	if (usepfp)
+		amdgpu_ring_insert_nop(ring, 128);
+
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
 				 WRITE_DATA_DST_SEL(0)) |
@@ -6064,18 +6277,11 @@ static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 		/* sync PFP to ME, otherwise we might get invalid PFP reads */
 		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
 		amdgpu_ring_write(ring, 0x0);
-		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
-		amdgpu_ring_write(ring, 0);
-		amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
-		amdgpu_ring_write(ring, 0);
+		/* GFX8 emits 128 dw nop to prevent CE access VM before vm_flush finish */
+		amdgpu_ring_insert_nop(ring, 128);
 	}
 }
 
-static u32 gfx_v8_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
-{
-	return ring->adev->wb.wb[ring->rptr_offs];
-}
-
 static u32 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
 {
 	return ring->adev->wb.wb[ring->wptr_offs];
@@ -6111,36 +6317,88 @@ static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, upper_32_bits(seq));
 }
 
-static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
-						 enum amdgpu_interrupt_state state)
+static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
 {
-	u32 cp_int_cntl;
+	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
+	amdgpu_ring_write(ring, 0);
+}
 
-	switch (state) {
-	case AMDGPU_IRQ_STATE_DISABLE:
-		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
-		cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
-					    TIME_STAMP_INT_ENABLE, 0);
-		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
-		break;
-	case AMDGPU_IRQ_STATE_ENABLE:
-		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
-		cp_int_cntl =
-			REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
-				      TIME_STAMP_INT_ENABLE, 1);
-		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
-		break;
-	default:
-		break;
+static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
+{
+	uint32_t dw2 = 0;
+
+	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
+	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
+		/* set load_global_config & load_global_uconfig */
+		dw2 |= 0x8001;
+		/* set load_cs_sh_regs */
+		dw2 |= 0x01000000;
+		/* set load_per_context_state & load_gfx_sh_regs for GFX */
+		dw2 |= 0x10002;
+
+		/* set load_ce_ram if preamble presented */
+		if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
+			dw2 |= 0x10000000;
+	} else {
+		/* still load_ce_ram if this is the first time preamble presented
+		 * although there is no context switch happens.
+		 */
+		if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
+			dw2 |= 0x10000000;
 	}
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+	amdgpu_ring_write(ring, dw2);
+	amdgpu_ring_write(ring, 0);
+}
+
+static unsigned gfx_v8_0_ring_get_emit_ib_size_gfx(struct amdgpu_ring *ring)
+{
+	return
+		4; /* gfx_v8_0_ring_emit_ib_gfx */
+}
+
+static unsigned gfx_v8_0_ring_get_dma_frame_size_gfx(struct amdgpu_ring *ring)
+{
+	return
+		20 + /* gfx_v8_0_ring_emit_gds_switch */
+		7 + /* gfx_v8_0_ring_emit_hdp_flush */
+		5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
+		6 + 6 + 6 +/* gfx_v8_0_ring_emit_fence_gfx x3 for user fence, vm fence */
+		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
+		256 + 19 + /* gfx_v8_0_ring_emit_vm_flush */
+		2 + /* gfx_v8_ring_emit_sb */
+		3; /* gfx_v8_ring_emit_cntxcntl */
+}
+
+static unsigned gfx_v8_0_ring_get_emit_ib_size_compute(struct amdgpu_ring *ring)
+{
+	return
+		4; /* gfx_v8_0_ring_emit_ib_compute */
+}
+
+static unsigned gfx_v8_0_ring_get_dma_frame_size_compute(struct amdgpu_ring *ring)
+{
+	return
+		20 + /* gfx_v8_0_ring_emit_gds_switch */
+		7 + /* gfx_v8_0_ring_emit_hdp_flush */
+		5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
+		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
+		17 + /* gfx_v8_0_ring_emit_vm_flush */
+		7 + 7 + 7; /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
+}
+
+static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
+						 enum amdgpu_interrupt_state state)
+{
+	WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE,
+		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
 }
 
 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
 						     int me, int pipe,
 						     enum amdgpu_interrupt_state state)
 {
-	u32 mec_int_cntl, mec_int_cntl_reg;
-
 	/*
 	 * amdgpu controls only pipe 0 of MEC1. That's why this function only
 	 * handles the setting of interrupts for this specific pipe. All other
@@ -6150,7 +6408,6 @@ static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
 	if (me == 1) {
 		switch (pipe) {
 		case 0:
-			mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL;
 			break;
 		default:
 			DRM_DEBUG("invalid pipe %d\n", pipe);
@@ -6161,22 +6418,8 @@ static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
 		return;
 	}
 
-	switch (state) {
-	case AMDGPU_IRQ_STATE_DISABLE:
-		mec_int_cntl = RREG32(mec_int_cntl_reg);
-		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
-					     TIME_STAMP_INT_ENABLE, 0);
-		WREG32(mec_int_cntl_reg, mec_int_cntl);
-		break;
-	case AMDGPU_IRQ_STATE_ENABLE:
-		mec_int_cntl = RREG32(mec_int_cntl_reg);
-		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
-					     TIME_STAMP_INT_ENABLE, 1);
-		WREG32(mec_int_cntl_reg, mec_int_cntl);
-		break;
-	default:
-		break;
-	}
+	WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, TIME_STAMP_INT_ENABLE,
+		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
 }
 
 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
@@ -6184,24 +6427,8 @@ static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
 					     unsigned type,
 					     enum amdgpu_interrupt_state state)
 {
-	u32 cp_int_cntl;
-
-	switch (state) {
-	case AMDGPU_IRQ_STATE_DISABLE:
-		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
-		cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
-					    PRIV_REG_INT_ENABLE, 0);
-		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
-		break;
-	case AMDGPU_IRQ_STATE_ENABLE:
-		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
-		cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
-					    PRIV_REG_INT_ENABLE, 1);
-		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
-		break;
-	default:
-		break;
-	}
+	WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE,
+		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
 
 	return 0;
 }
@@ -6211,24 +6438,8 @@ static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
 					      unsigned type,
 					      enum amdgpu_interrupt_state state)
 {
-	u32 cp_int_cntl;
-
-	switch (state) {
-	case AMDGPU_IRQ_STATE_DISABLE:
-		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
-		cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
-					    PRIV_INSTR_INT_ENABLE, 0);
-		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
-		break;
-	case AMDGPU_IRQ_STATE_ENABLE:
-		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
-		cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
-					    PRIV_INSTR_INT_ENABLE, 1);
-		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
-		break;
-	default:
-		break;
-	}
+	WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE,
+		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
 
 	return 0;
 }
@@ -6334,13 +6545,16 @@ const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
 	.resume = gfx_v8_0_resume,
 	.is_idle = gfx_v8_0_is_idle,
 	.wait_for_idle = gfx_v8_0_wait_for_idle,
+	.check_soft_reset = gfx_v8_0_check_soft_reset,
+	.pre_soft_reset = gfx_v8_0_pre_soft_reset,
 	.soft_reset = gfx_v8_0_soft_reset,
+	.post_soft_reset = gfx_v8_0_post_soft_reset,
 	.set_clockgating_state = gfx_v8_0_set_clockgating_state,
 	.set_powergating_state = gfx_v8_0_set_powergating_state,
 };
 
 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
-	.get_rptr = gfx_v8_0_ring_get_rptr_gfx,
+	.get_rptr = gfx_v8_0_ring_get_rptr,
 	.get_wptr = gfx_v8_0_ring_get_wptr_gfx,
 	.set_wptr = gfx_v8_0_ring_set_wptr_gfx,
 	.parse_cs = NULL,
@@ -6355,10 +6569,14 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
 	.test_ib = gfx_v8_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.emit_switch_buffer = gfx_v8_ring_emit_sb,
+	.emit_cntxcntl = gfx_v8_ring_emit_cntxcntl,
+	.get_emit_ib_size = gfx_v8_0_ring_get_emit_ib_size_gfx,
+	.get_dma_frame_size = gfx_v8_0_ring_get_dma_frame_size_gfx,
 };
 
 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
-	.get_rptr = gfx_v8_0_ring_get_rptr_compute,
+	.get_rptr = gfx_v8_0_ring_get_rptr,
 	.get_wptr = gfx_v8_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v8_0_ring_set_wptr_compute,
 	.parse_cs = NULL,
@@ -6373,6 +6591,8 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
 	.test_ib = gfx_v8_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.get_emit_ib_size = gfx_v8_0_ring_get_emit_ib_size_compute,
+	.get_dma_frame_size = gfx_v8_0_ring_get_dma_frame_size_compute,
 };
 
 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -6475,15 +6695,12 @@ static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)
 {
 	u32 data, mask;
 
-	data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG);
-	data |= RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
-
-	data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
-	data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
+	data =  RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) |
+		RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
 
 	mask = gfx_v8_0_create_bitmask(adev->gfx.config.max_cu_per_sh);
 
-	return (~data) & mask;
+	return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask;
 }
 
 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h
index bc82c794312c..ebed1f829297 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h
@@ -26,6 +26,4 @@
 
 extern const struct amd_ip_funcs gfx_v8_0_ip_funcs;
 
-void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num);
-
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
new file mode 100644
index 000000000000..b13c8aaec078
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -0,0 +1,1071 @@
+
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/firmware.h>
+#include "drmP.h"
+#include "amdgpu.h"
+#include "gmc_v6_0.h"
+#include "amdgpu_ucode.h"
+#include "si/sid.h"
+
+static void gmc_v6_0_set_gart_funcs(struct amdgpu_device *adev);
+static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev);
+static int gmc_v6_0_wait_for_idle(void *handle);
+
+MODULE_FIRMWARE("radeon/tahiti_mc.bin");
+MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
+MODULE_FIRMWARE("radeon/verde_mc.bin");
+MODULE_FIRMWARE("radeon/oland_mc.bin");
+
+static const u32 crtc_offsets[6] =
+{
+	SI_CRTC0_REGISTER_OFFSET,
+	SI_CRTC1_REGISTER_OFFSET,
+	SI_CRTC2_REGISTER_OFFSET,
+	SI_CRTC3_REGISTER_OFFSET,
+	SI_CRTC4_REGISTER_OFFSET,
+	SI_CRTC5_REGISTER_OFFSET
+};
+
+static void gmc_v6_0_mc_stop(struct amdgpu_device *adev,
+			     struct amdgpu_mode_mc_save *save)
+{
+	u32 blackout;
+
+	if (adev->mode_info.num_crtc)
+		amdgpu_display_stop_mc_access(adev, save);
+
+	gmc_v6_0_wait_for_idle((void *)adev);
+
+	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
+	if (REG_GET_FIELD(blackout, mmMC_SHARED_BLACKOUT_CNTL, xxBLACKOUT_MODE) != 1) {
+		/* Block CPU access */
+		WREG32(BIF_FB_EN, 0);
+		/* blackout the MC */
+		blackout = REG_SET_FIELD(blackout,
+					 mmMC_SHARED_BLACKOUT_CNTL, xxBLACKOUT_MODE, 0);
+		WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
+	}
+	/* wait for the MC to settle */
+	udelay(100);
+
+}
+
+static void gmc_v6_0_mc_resume(struct amdgpu_device *adev,
+			       struct amdgpu_mode_mc_save *save)
+{
+	u32 tmp;
+
+	/* unblackout the MC */
+	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
+	tmp = REG_SET_FIELD(tmp, mmMC_SHARED_BLACKOUT_CNTL, xxBLACKOUT_MODE, 0);
+	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
+	/* allow CPU access */
+	tmp = REG_SET_FIELD(0, mmBIF_FB_EN, xxFB_READ_EN, 1);
+	tmp = REG_SET_FIELD(tmp, mmBIF_FB_EN, xxFB_WRITE_EN, 1);
+	WREG32(BIF_FB_EN, tmp);
+
+	if (adev->mode_info.num_crtc)
+		amdgpu_display_resume_mc_access(adev, save);
+
+}
+
+static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)
+{
+	const char *chip_name;
+	char fw_name[30];
+	int err;
+
+	DRM_DEBUG("\n");
+
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+		chip_name = "tahiti";
+		break;
+	case CHIP_PITCAIRN:
+		chip_name = "pitcairn";
+		break;
+	case CHIP_VERDE:
+		chip_name = "verde";
+		break;
+	case CHIP_OLAND:
+		chip_name = "oland";
+		break;
+	case CHIP_HAINAN:
+		chip_name = "hainan";
+		break;
+	default: BUG();
+	}
+
+	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
+	err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
+	if (err)
+		goto out;
+
+	err = amdgpu_ucode_validate(adev->mc.fw);
+
+out:
+	if (err) {
+		dev_err(adev->dev,
+		       "si_mc: Failed to load firmware \"%s\"\n",
+		       fw_name);
+		release_firmware(adev->mc.fw);
+		adev->mc.fw = NULL;
+	}
+	return err;
+}
+
+static int gmc_v6_0_mc_load_microcode(struct amdgpu_device *adev)
+{
+	const __le32 *new_fw_data = NULL;
+	u32 running;
+	const __le32 *new_io_mc_regs = NULL;
+	int i, regs_size, ucode_size;
+	const struct mc_firmware_header_v1_0 *hdr;
+
+	if (!adev->mc.fw)
+		return -EINVAL;
+
+	hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
+
+	amdgpu_ucode_print_mc_hdr(&hdr->header);
+
+	adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
+	regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
+	new_io_mc_regs = (const __le32 *)
+		(adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
+	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
+	new_fw_data = (const __le32 *)
+		(adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+
+	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
+
+	if (running == 0) {
+
+		/* reset the engine and set to writable */
+		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
+		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
+
+		/* load mc io regs */
+		for (i = 0; i < regs_size; i++) {
+			WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
+			WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
+		}
+		/* load the MC ucode */
+		for (i = 0; i < ucode_size; i++) {
+			WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
+		}
+
+		/* put the engine back into the active state */
+		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
+		WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
+		WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
+
+		/* wait for training to complete */
+		for (i = 0; i < adev->usec_timeout; i++) {
+			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
+				break;
+			udelay(1);
+		}
+		for (i = 0; i < adev->usec_timeout; i++) {
+			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
+				break;
+			udelay(1);
+		}
+
+	}
+
+	return 0;
+}
+
+static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,
+				       struct amdgpu_mc *mc)
+{
+	if (mc->mc_vram_size > 0xFFC0000000ULL) {
+		dev_warn(adev->dev, "limiting VRAM\n");
+		mc->real_vram_size = 0xFFC0000000ULL;
+		mc->mc_vram_size = 0xFFC0000000ULL;
+	}
+	amdgpu_vram_location(adev, &adev->mc, 0);
+	adev->mc.gtt_base_align = 0;
+	amdgpu_gtt_location(adev, mc);
+}
+
+static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_mc_save save;
+	u32 tmp;
+	int i, j;
+
+	/* Initialize HDP */
+	for (i = 0, j = 0; i < 32; i++, j += 0x6) {
+		WREG32((0xb05 + j), 0x00000000);
+		WREG32((0xb06 + j), 0x00000000);
+		WREG32((0xb07 + j), 0x00000000);
+		WREG32((0xb08 + j), 0x00000000);
+		WREG32((0xb09 + j), 0x00000000);
+	}
+	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
+
+	gmc_v6_0_mc_stop(adev, &save);
+
+	if (gmc_v6_0_wait_for_idle((void *)adev)) {
+		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
+	}
+
+	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
+	/* Update configuration */
+	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
+	       adev->mc.vram_start >> 12);
+	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+	       adev->mc.vram_end >> 12);
+	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
+	       adev->vram_scratch.gpu_addr >> 12);
+	tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
+	tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
+	WREG32(MC_VM_FB_LOCATION, tmp);
+	/* XXX double check these! */
+	WREG32(HDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
+	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
+	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
+	WREG32(MC_VM_AGP_BASE, 0);
+	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
+	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
+
+	if (gmc_v6_0_wait_for_idle((void *)adev)) {
+		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
+	}
+	gmc_v6_0_mc_resume(adev, &save);
+	amdgpu_display_set_vga_render_state(adev, false);
+}
+
+static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
+{
+
+	u32 tmp;
+	int chansize, numchan;
+
+	tmp = RREG32(MC_ARB_RAMCFG);
+	if (tmp & CHANSIZE_OVERRIDE) {
+		chansize = 16;
+	} else if (tmp & CHANSIZE_MASK) {
+		chansize = 64;
+	} else {
+		chansize = 32;
+	}
+	tmp = RREG32(MC_SHARED_CHMAP);
+	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
+	case 0:
+	default:
+		numchan = 1;
+		break;
+	case 1:
+		numchan = 2;
+		break;
+	case 2:
+		numchan = 4;
+		break;
+	case 3:
+		numchan = 8;
+		break;
+	case 4:
+		numchan = 3;
+		break;
+	case 5:
+		numchan = 6;
+		break;
+	case 6:
+		numchan = 10;
+		break;
+	case 7:
+		numchan = 12;
+		break;
+	case 8:
+		numchan = 16;
+		break;
+	}
+	adev->mc.vram_width = numchan * chansize;
+	/* Could aper size report 0 ? */
+	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
+	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
+	/* size in MB on si */
+	adev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
+	adev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
+	adev->mc.visible_vram_size = adev->mc.aper_size;
+
+	/* unless the user had overridden it, set the gart
+	 * size equal to the 1024 or vram, whichever is larger.
+	 */
+	if (amdgpu_gart_size == -1)
+		adev->mc.gtt_size = amdgpu_ttm_get_gtt_mem_size(adev);
+	else
+		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+
+	gmc_v6_0_vram_gtt_location(adev, &adev->mc);
+
+	return 0;
+}
+
+static void gmc_v6_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
+					uint32_t vmid)
+{
+	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
+
+	WREG32(VM_INVALIDATE_REQUEST, 1 << vmid);
+}
+
+static int gmc_v6_0_gart_set_pte_pde(struct amdgpu_device *adev,
+				     void *cpu_pt_addr,
+				     uint32_t gpu_page_idx,
+				     uint64_t addr,
+				     uint32_t flags)
+{
+	void __iomem *ptr = (void *)cpu_pt_addr;
+	uint64_t value;
+
+	value = addr & 0xFFFFFFFFFFFFF000ULL;
+	value |= flags;
+	writeq(value, ptr + (gpu_page_idx * 8));
+
+	return 0;
+}
+
+static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev,
+					      bool value)
+{
+	u32 tmp;
+
+	tmp = RREG32(VM_CONTEXT1_CNTL);
+	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
+			    xxRANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
+			    xxDUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
+			    xxPDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
+			    xxVALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
+			    xxREAD_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
+			    xxWRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	WREG32(VM_CONTEXT1_CNTL, tmp);
+}
+
+static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
+{
+	int r, i;
+
+	if (adev->gart.robj == NULL) {
+		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
+		return -EINVAL;
+	}
+	r = amdgpu_gart_table_vram_pin(adev);
+	if (r)
+		return r;
+	/* Setup TLB control */
+	WREG32(MC_VM_MX_L1_TLB_CNTL,
+	       (0xA << 7) |
+	       ENABLE_L1_TLB |
+	       ENABLE_L1_FRAGMENT_PROCESSING |
+	       SYSTEM_ACCESS_MODE_NOT_IN_SYS |
+	       ENABLE_ADVANCED_DRIVER_MODEL |
+	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
+	/* Setup L2 cache */
+	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
+	       ENABLE_L2_FRAGMENT_PROCESSING |
+	       ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
+	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
+	       EFFECTIVE_L2_QUEUE_SIZE(7) |
+	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
+	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
+	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
+	       BANK_SELECT(4) |
+	       L2_CACHE_BIGK_FRAGMENT_SIZE(4));
+	/* setup context0 */
+	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
+	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
+	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
+	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
+			(u32)(adev->dummy_page.addr >> 12));
+	WREG32(VM_CONTEXT0_CNTL2, 0);
+	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
+				  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
+
+	WREG32(0x575, 0);
+	WREG32(0x576, 0);
+	WREG32(0x577, 0);
+
+	/* empty context1-15 */
+	/* set vm size, must be a multiple of 4 */
+	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
+	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
+	/* Assign the pt base to something valid for now; the pts used for
+	 * the VMs are determined by the application and setup and assigned
+	 * on the fly in the vm part of radeon_gart.c
+	 */
+	for (i = 1; i < 16; i++) {
+		if (i < 8)
+			WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
+			       adev->gart.table_addr >> 12);
+		else
+			WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
+			       adev->gart.table_addr >> 12);
+	}
+
+	/* enable context1-15 */
+	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
+	       (u32)(adev->dummy_page.addr >> 12));
+	WREG32(VM_CONTEXT1_CNTL2, 4);
+	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
+				PAGE_TABLE_BLOCK_SIZE(amdgpu_vm_block_size - 9) |
+				RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
+				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
+				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
+				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
+				PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
+				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
+				VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
+				VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
+				READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
+				READ_PROTECTION_FAULT_ENABLE_DEFAULT |
+				WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
+				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
+
+	gmc_v6_0_gart_flush_gpu_tlb(adev, 0);
+	dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
+		 (unsigned)(adev->mc.gtt_size >> 20),
+		 (unsigned long long)adev->gart.table_addr);
+	adev->gart.ready = true;
+	return 0;
+}
+
+static int gmc_v6_0_gart_init(struct amdgpu_device *adev)
+{
+	int r;
+
+	if (adev->gart.robj) {
+		dev_warn(adev->dev, "gmc_v6_0 PCIE GART already initialized\n");
+		return 0;
+	}
+	r = amdgpu_gart_init(adev);
+	if (r)
+		return r;
+	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
+	return amdgpu_gart_table_vram_alloc(adev);
+}
+
+static void gmc_v6_0_gart_disable(struct amdgpu_device *adev)
+{
+	/*unsigned i;
+
+	for (i = 1; i < 16; ++i) {
+		uint32_t reg;
+		if (i < 8)
+			reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i ;
+		else
+			reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (i - 8);
+		adev->vm_manager.saved_table_addr[i] = RREG32(reg);
+	}*/
+
+	/* Disable all tables */
+	WREG32(VM_CONTEXT0_CNTL, 0);
+	WREG32(VM_CONTEXT1_CNTL, 0);
+	/* Setup TLB control */
+	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
+	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
+	/* Setup L2 cache */
+	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
+	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
+	       EFFECTIVE_L2_QUEUE_SIZE(7) |
+	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
+	WREG32(VM_L2_CNTL2, 0);
+	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
+	       L2_CACHE_BIGK_FRAGMENT_SIZE(0));
+	amdgpu_gart_table_vram_unpin(adev);
+}
+
+static void gmc_v6_0_gart_fini(struct amdgpu_device *adev)
+{
+	amdgpu_gart_table_vram_free(adev);
+	amdgpu_gart_fini(adev);
+}
+
+static int gmc_v6_0_vm_init(struct amdgpu_device *adev)
+{
+	/*
+	 * number of VMs
+	 * VMID 0 is reserved for System
+	 * amdgpu graphics/compute will use VMIDs 1-7
+	 * amdkfd will use VMIDs 8-15
+	 */
+	adev->vm_manager.num_ids = AMDGPU_NUM_OF_VMIDS;
+	amdgpu_vm_manager_init(adev);
+
+	/* base offset of vram pages */
+	if (adev->flags & AMD_IS_APU) {
+		u64 tmp = RREG32(MC_VM_FB_OFFSET);
+		tmp <<= 22;
+		adev->vm_manager.vram_base_offset = tmp;
+	} else
+		adev->vm_manager.vram_base_offset = 0;
+
+	return 0;
+}
+
+static void gmc_v6_0_vm_fini(struct amdgpu_device *adev)
+{
+}
+
+static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,
+				     u32 status, u32 addr, u32 mc_client)
+{
+	u32 mc_id;
+	u32 vmid = REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS, xxVMID);
+	u32 protections = REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					xxPROTECTIONS);
+	char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
+		(mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
+
+	mc_id = REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
+			      xxMEMORY_CLIENT_ID);
+
+	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	       protections, vmid, addr,
+	       REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
+			     xxMEMORY_CLIENT_RW) ?
+	       "write" : "read", block, mc_client, mc_id);
+}
+
+/*
+static const u32 mc_cg_registers[] = {
+	MC_HUB_MISC_HUB_CG,
+	MC_HUB_MISC_SIP_CG,
+	MC_HUB_MISC_VM_CG,
+	MC_XPB_CLK_GAT,
+	ATC_MISC_CG,
+	MC_CITF_MISC_WR_CG,
+	MC_CITF_MISC_RD_CG,
+	MC_CITF_MISC_VM_CG,
+	VM_L2_CG,
+};
+
+static const u32 mc_cg_ls_en[] = {
+	MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK,
+	MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK,
+	MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK,
+	MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK,
+	ATC_MISC_CG__MEM_LS_ENABLE_MASK,
+	MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK,
+	MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK,
+	MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK,
+	VM_L2_CG__MEM_LS_ENABLE_MASK,
+};
+
+static const u32 mc_cg_en[] = {
+	MC_HUB_MISC_HUB_CG__ENABLE_MASK,
+	MC_HUB_MISC_SIP_CG__ENABLE_MASK,
+	MC_HUB_MISC_VM_CG__ENABLE_MASK,
+	MC_XPB_CLK_GAT__ENABLE_MASK,
+	ATC_MISC_CG__ENABLE_MASK,
+	MC_CITF_MISC_WR_CG__ENABLE_MASK,
+	MC_CITF_MISC_RD_CG__ENABLE_MASK,
+	MC_CITF_MISC_VM_CG__ENABLE_MASK,
+	VM_L2_CG__ENABLE_MASK,
+};
+
+static void gmc_v6_0_enable_mc_ls(struct amdgpu_device *adev,
+				  bool enable)
+{
+	int i;
+	u32 orig, data;
+
+	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
+		orig = data = RREG32(mc_cg_registers[i]);
+		if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_MC_LS))
+			data |= mc_cg_ls_en[i];
+		else
+			data &= ~mc_cg_ls_en[i];
+		if (data != orig)
+			WREG32(mc_cg_registers[i], data);
+	}
+}
+
+static void gmc_v6_0_enable_mc_mgcg(struct amdgpu_device *adev,
+				    bool enable)
+{
+	int i;
+	u32 orig, data;
+
+	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
+		orig = data = RREG32(mc_cg_registers[i]);
+		if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_MC_MGCG))
+			data |= mc_cg_en[i];
+		else
+			data &= ~mc_cg_en[i];
+		if (data != orig)
+			WREG32(mc_cg_registers[i], data);
+	}
+}
+
+static void gmc_v6_0_enable_bif_mgls(struct amdgpu_device *adev,
+				     bool enable)
+{
+	u32 orig, data;
+
+	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
+
+	if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_BIF_LS)) {
+		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 1);
+		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 1);
+		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 1);
+		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 1);
+	} else {
+		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 0);
+		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 0);
+		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 0);
+		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 0);
+	}
+
+	if (orig != data)
+		WREG32_PCIE(ixPCIE_CNTL2, data);
+}
+
+static void gmc_v6_0_enable_hdp_mgcg(struct amdgpu_device *adev,
+				     bool enable)
+{
+	u32 orig, data;
+
+	orig = data = RREG32(HDP_HOST_PATH_CNTL);
+
+	if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_HDP_MGCG))
+		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
+	else
+		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
+
+	if (orig != data)
+		WREG32(HDP_HOST_PATH_CNTL, data);
+}
+
+static void gmc_v6_0_enable_hdp_ls(struct amdgpu_device *adev,
+				   bool enable)
+{
+	u32 orig, data;
+
+	orig = data = RREG32(HDP_MEM_POWER_LS);
+
+	if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_HDP_LS))
+		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
+	else
+		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
+
+	if (orig != data)
+		WREG32(HDP_MEM_POWER_LS, data);
+}
+*/
+
+static int gmc_v6_0_convert_vram_type(int mc_seq_vram_type)
+{
+	switch (mc_seq_vram_type) {
+	case MC_SEQ_MISC0__MT__GDDR1:
+		return AMDGPU_VRAM_TYPE_GDDR1;
+	case MC_SEQ_MISC0__MT__DDR2:
+		return AMDGPU_VRAM_TYPE_DDR2;
+	case MC_SEQ_MISC0__MT__GDDR3:
+		return AMDGPU_VRAM_TYPE_GDDR3;
+	case MC_SEQ_MISC0__MT__GDDR4:
+		return AMDGPU_VRAM_TYPE_GDDR4;
+	case MC_SEQ_MISC0__MT__GDDR5:
+		return AMDGPU_VRAM_TYPE_GDDR5;
+	case MC_SEQ_MISC0__MT__DDR3:
+		return AMDGPU_VRAM_TYPE_DDR3;
+	default:
+		return AMDGPU_VRAM_TYPE_UNKNOWN;
+	}
+}
+
+static int gmc_v6_0_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	gmc_v6_0_set_gart_funcs(adev);
+	gmc_v6_0_set_irq_funcs(adev);
+
+	if (adev->flags & AMD_IS_APU) {
+		adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
+	} else {
+		u32 tmp = RREG32(MC_SEQ_MISC0);
+		tmp &= MC_SEQ_MISC0__MT__MASK;
+		adev->mc.vram_type = gmc_v6_0_convert_vram_type(tmp);
+	}
+
+	return 0;
+}
+
+static int gmc_v6_0_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
+}
+
+static int gmc_v6_0_sw_init(void *handle)
+{
+	int r;
+	int dma_bits;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	r = amdgpu_irq_add_id(adev, 146, &adev->mc.vm_fault);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_add_id(adev, 147, &adev->mc.vm_fault);
+	if (r)
+		return r;
+
+	adev->vm_manager.max_pfn = amdgpu_vm_size << 18;
+
+	adev->mc.mc_mask = 0xffffffffffULL;
+
+	adev->need_dma32 = false;
+	dma_bits = adev->need_dma32 ? 32 : 40;
+	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
+	if (r) {
+		adev->need_dma32 = true;
+		dma_bits = 32;
+		dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
+	}
+	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
+	if (r) {
+		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
+		dev_warn(adev->dev, "amdgpu: No coherent DMA available.\n");
+	}
+
+	r = gmc_v6_0_init_microcode(adev);
+	if (r) {
+		dev_err(adev->dev, "Failed to load mc firmware!\n");
+		return r;
+	}
+
+	r = amdgpu_ttm_global_init(adev);
+	if (r) {
+		return r;
+	}
+
+	r = gmc_v6_0_mc_init(adev);
+	if (r)
+		return r;
+
+	r = amdgpu_bo_init(adev);
+	if (r)
+		return r;
+
+	r = gmc_v6_0_gart_init(adev);
+	if (r)
+		return r;
+
+	if (!adev->vm_manager.enabled) {
+		r = gmc_v6_0_vm_init(adev);
+		if (r) {
+			dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
+			return r;
+		}
+		adev->vm_manager.enabled = true;
+	}
+
+	return r;
+}
+
+static int gmc_v6_0_sw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->vm_manager.enabled) {
+		gmc_v6_0_vm_fini(adev);
+		adev->vm_manager.enabled = false;
+	}
+	gmc_v6_0_gart_fini(adev);
+	amdgpu_gem_force_release(adev);
+	amdgpu_bo_fini(adev);
+
+	return 0;
+}
+
+static int gmc_v6_0_hw_init(void *handle)
+{
+	int r;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	gmc_v6_0_mc_program(adev);
+
+	if (!(adev->flags & AMD_IS_APU)) {
+		r = gmc_v6_0_mc_load_microcode(adev);
+		if (r) {
+			dev_err(adev->dev, "Failed to load MC firmware!\n");
+			return r;
+		}
+	}
+
+	r = gmc_v6_0_gart_enable(adev);
+	if (r)
+		return r;
+
+	return r;
+}
+
+static int gmc_v6_0_hw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
+	gmc_v6_0_gart_disable(adev);
+
+	return 0;
+}
+
+static int gmc_v6_0_suspend(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->vm_manager.enabled) {
+		gmc_v6_0_vm_fini(adev);
+		adev->vm_manager.enabled = false;
+	}
+	gmc_v6_0_hw_fini(adev);
+
+	return 0;
+}
+
+static int gmc_v6_0_resume(void *handle)
+{
+	int r;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	r = gmc_v6_0_hw_init(adev);
+	if (r)
+		return r;
+
+	if (!adev->vm_manager.enabled) {
+		r = gmc_v6_0_vm_init(adev);
+		if (r) {
+			dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
+			return r;
+		}
+		adev->vm_manager.enabled = true;
+	}
+
+	return r;
+}
+
+static bool gmc_v6_0_is_idle(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 tmp = RREG32(SRBM_STATUS);
+
+	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
+		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
+		return false;
+
+	return true;
+}
+
+static int gmc_v6_0_wait_for_idle(void *handle)
+{
+	unsigned i;
+	u32 tmp;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = RREG32(SRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
+					       SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
+					       SRBM_STATUS__MCC_BUSY_MASK |
+					       SRBM_STATUS__MCD_BUSY_MASK |
+					       SRBM_STATUS__VMC_BUSY_MASK);
+		if (!tmp)
+			return 0;
+		udelay(1);
+	}
+	return -ETIMEDOUT;
+
+}
+
+static int gmc_v6_0_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct amdgpu_mode_mc_save save;
+	u32 srbm_soft_reset = 0;
+	u32 tmp = RREG32(SRBM_STATUS);
+
+	if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
+		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
+						mmSRBM_SOFT_RESET, xxSOFT_RESET_VMC, 1);
+
+	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
+		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
+		if (!(adev->flags & AMD_IS_APU))
+			srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
+							mmSRBM_SOFT_RESET, xxSOFT_RESET_MC, 1);
+	}
+
+	if (srbm_soft_reset) {
+		gmc_v6_0_mc_stop(adev, &save);
+		if (gmc_v6_0_wait_for_idle(adev)) {
+			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
+		}
+
+
+		tmp = RREG32(SRBM_SOFT_RESET);
+		tmp |= srbm_soft_reset;
+		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
+		WREG32(SRBM_SOFT_RESET, tmp);
+		tmp = RREG32(SRBM_SOFT_RESET);
+
+		udelay(50);
+
+		tmp &= ~srbm_soft_reset;
+		WREG32(SRBM_SOFT_RESET, tmp);
+		tmp = RREG32(SRBM_SOFT_RESET);
+
+		udelay(50);
+
+		gmc_v6_0_mc_resume(adev, &save);
+		udelay(50);
+	}
+
+	return 0;
+}
+
+static int gmc_v6_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
+					     struct amdgpu_irq_src *src,
+					     unsigned type,
+					     enum amdgpu_interrupt_state state)
+{
+	u32 tmp;
+	u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		    VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		    VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		    VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		    VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		    VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		tmp = RREG32(VM_CONTEXT0_CNTL);
+		tmp &= ~bits;
+		WREG32(VM_CONTEXT0_CNTL, tmp);
+		tmp = RREG32(VM_CONTEXT1_CNTL);
+		tmp &= ~bits;
+		WREG32(VM_CONTEXT1_CNTL, tmp);
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		tmp = RREG32(VM_CONTEXT0_CNTL);
+		tmp |= bits;
+		WREG32(VM_CONTEXT0_CNTL, tmp);
+		tmp = RREG32(VM_CONTEXT1_CNTL);
+		tmp |= bits;
+		WREG32(VM_CONTEXT1_CNTL, tmp);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
+				      struct amdgpu_irq_src *source,
+				      struct amdgpu_iv_entry *entry)
+{
+	u32 addr, status;
+
+	addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
+	status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
+	WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
+
+	if (!addr && !status)
+		return 0;
+
+	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
+		gmc_v6_0_set_fault_enable_default(adev, false);
+
+	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
+		entry->src_id, entry->src_data);
+	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+		addr);
+	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+		status);
+	gmc_v6_0_vm_decode_fault(adev, status, addr, 0);
+
+	return 0;
+}
+
+static int gmc_v6_0_set_clockgating_state(void *handle,
+					  enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int gmc_v6_0_set_powergating_state(void *handle,
+					  enum amd_powergating_state state)
+{
+	return 0;
+}
+
+const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
+	.name = "gmc_v6_0",
+	.early_init = gmc_v6_0_early_init,
+	.late_init = gmc_v6_0_late_init,
+	.sw_init = gmc_v6_0_sw_init,
+	.sw_fini = gmc_v6_0_sw_fini,
+	.hw_init = gmc_v6_0_hw_init,
+	.hw_fini = gmc_v6_0_hw_fini,
+	.suspend = gmc_v6_0_suspend,
+	.resume = gmc_v6_0_resume,
+	.is_idle = gmc_v6_0_is_idle,
+	.wait_for_idle = gmc_v6_0_wait_for_idle,
+	.soft_reset = gmc_v6_0_soft_reset,
+	.set_clockgating_state = gmc_v6_0_set_clockgating_state,
+	.set_powergating_state = gmc_v6_0_set_powergating_state,
+};
+
+static const struct amdgpu_gart_funcs gmc_v6_0_gart_funcs = {
+	.flush_gpu_tlb = gmc_v6_0_gart_flush_gpu_tlb,
+	.set_pte_pde = gmc_v6_0_gart_set_pte_pde,
+};
+
+static const struct amdgpu_irq_src_funcs gmc_v6_0_irq_funcs = {
+	.set = gmc_v6_0_vm_fault_interrupt_state,
+	.process = gmc_v6_0_process_interrupt,
+};
+
+static void gmc_v6_0_set_gart_funcs(struct amdgpu_device *adev)
+{
+	if (adev->gart.gart_funcs == NULL)
+		adev->gart.gart_funcs = &gmc_v6_0_gart_funcs;
+}
+
+static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->mc.vm_fault.num_types = 1;
+	adev->mc.vm_fault.funcs = &gmc_v6_0_irq_funcs;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.h
new file mode 100644
index 000000000000..42c4fc676cd4
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __GMC_V6_0_H__
+#define __GMC_V6_0_H__
+
+extern const struct amd_ip_funcs gmc_v6_0_ip_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index d24a82bd0c7a..aa0c4b964621 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -144,6 +144,7 @@ static int gmc_v7_0_init_microcode(struct amdgpu_device *adev)
 		break;
 	case CHIP_KAVERI:
 	case CHIP_KABINI:
+	case CHIP_MULLINS:
 		return 0;
 	default: BUG();
 	}
@@ -182,7 +183,7 @@ static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)
 	const struct mc_firmware_header_v1_0 *hdr;
 	const __le32 *fw_data = NULL;
 	const __le32 *io_mc_regs = NULL;
-	u32 running, blackout = 0;
+	u32 running;
 	int i, ucode_size, regs_size;
 
 	if (!adev->mc.fw)
@@ -202,11 +203,6 @@ static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)
 	running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
 
 	if (running == 0) {
-		if (running) {
-			blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
-			WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
-		}
-
 		/* reset the engine and set to writable */
 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
@@ -238,9 +234,6 @@ static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)
 				break;
 			udelay(1);
 		}
-
-		if (running)
-			WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);
 	}
 
 	return 0;
@@ -392,7 +385,7 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
 	 * size equal to the 1024 or vram, whichever is larger.
 	 */
 	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size);
+		adev->mc.gtt_size = amdgpu_ttm_get_gtt_mem_size(adev);
 	else
 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
 
@@ -952,6 +945,11 @@ static int gmc_v7_0_sw_init(void *handle)
 		return r;
 	}
 
+	r = amdgpu_ttm_global_init(adev);
+	if (r) {
+		return r;
+	}
+
 	r = gmc_v7_0_mc_init(adev);
 	if (r)
 		return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 717359d3ba8c..1b319f5bc696 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -103,6 +103,11 @@ static const u32 stoney_mgcg_cgcg_init[] =
 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
 };
 
+static const u32 golden_settings_stoney_common[] =
+{
+	mmMC_HUB_RDREQ_UVD, MC_HUB_RDREQ_UVD__PRESCALE_MASK, 0x00000004,
+	mmMC_RD_GRP_OTH, MC_RD_GRP_OTH__UVD_MASK, 0x00600000
+};
 
 static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
 {
@@ -142,6 +147,9 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
 		amdgpu_program_register_sequence(adev,
 						 stoney_mgcg_cgcg_init,
 						 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+						 golden_settings_stoney_common,
+						 (const u32)ARRAY_SIZE(golden_settings_stoney_common));
 		break;
 	default:
 		break;
@@ -253,7 +261,7 @@ static int gmc_v8_0_mc_load_microcode(struct amdgpu_device *adev)
 	const struct mc_firmware_header_v1_0 *hdr;
 	const __le32 *fw_data = NULL;
 	const __le32 *io_mc_regs = NULL;
-	u32 running, blackout = 0;
+	u32 running;
 	int i, ucode_size, regs_size;
 
 	if (!adev->mc.fw)
@@ -261,8 +269,10 @@ static int gmc_v8_0_mc_load_microcode(struct amdgpu_device *adev)
 
 	/* Skip MC ucode loading on SR-IOV capable boards.
 	 * vbios does this for us in asic_init in that case.
+	 * Skip MC ucode loading on VF, because hypervisor will do that
+	 * for this adaptor.
 	 */
-	if (adev->virtualization.supports_sr_iov)
+	if (amdgpu_sriov_bios(adev))
 		return 0;
 
 	hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
@@ -279,11 +289,6 @@ static int gmc_v8_0_mc_load_microcode(struct amdgpu_device *adev)
 	running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
 
 	if (running == 0) {
-		if (running) {
-			blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
-			WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
-		}
-
 		/* reset the engine and set to writable */
 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
@@ -315,9 +320,6 @@ static int gmc_v8_0_mc_load_microcode(struct amdgpu_device *adev)
 				break;
 			udelay(1);
 		}
-
-		if (running)
-			WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);
 	}
 
 	return 0;
@@ -469,7 +471,7 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
 	 * size equal to the 1024 or vram, whichever is larger.
 	 */
 	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size);
+		adev->mc.gtt_size = amdgpu_ttm_get_gtt_mem_size(adev);
 	else
 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
 
@@ -949,6 +951,11 @@ static int gmc_v8_0_sw_init(void *handle)
 		return r;
 	}
 
+	r = amdgpu_ttm_global_init(adev);
+	if (r) {
+		return r;
+	}
+
 	r = gmc_v8_0_mc_init(adev);
 	if (r)
 		return r;
@@ -1092,9 +1099,8 @@ static int gmc_v8_0_wait_for_idle(void *handle)
 
 }
 
-static int gmc_v8_0_soft_reset(void *handle)
+static int gmc_v8_0_check_soft_reset(void *handle)
 {
-	struct amdgpu_mode_mc_save save;
 	u32 srbm_soft_reset = 0;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	u32 tmp = RREG32(mmSRBM_STATUS);
@@ -1109,13 +1115,42 @@ static int gmc_v8_0_soft_reset(void *handle)
 			srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
 							SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
 	}
-
 	if (srbm_soft_reset) {
-		gmc_v8_0_mc_stop(adev, &save);
-		if (gmc_v8_0_wait_for_idle((void *)adev)) {
-			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
-		}
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang = true;
+		adev->mc.srbm_soft_reset = srbm_soft_reset;
+	} else {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang = false;
+		adev->mc.srbm_soft_reset = 0;
+	}
+	return 0;
+}
+
+static int gmc_v8_0_pre_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang)
+		return 0;
 
+	gmc_v8_0_mc_stop(adev, &adev->mc.save);
+	if (gmc_v8_0_wait_for_idle(adev)) {
+		dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
+	}
+
+	return 0;
+}
+
+static int gmc_v8_0_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang)
+		return 0;
+	srbm_soft_reset = adev->mc.srbm_soft_reset;
+
+	if (srbm_soft_reset) {
+		u32 tmp;
 
 		tmp = RREG32(mmSRBM_SOFT_RESET);
 		tmp |= srbm_soft_reset;
@@ -1131,14 +1166,22 @@ static int gmc_v8_0_soft_reset(void *handle)
 
 		/* Wait a little for things to settle down */
 		udelay(50);
-
-		gmc_v8_0_mc_resume(adev, &save);
-		udelay(50);
 	}
 
 	return 0;
 }
 
+static int gmc_v8_0_post_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_GMC].hang)
+		return 0;
+
+	gmc_v8_0_mc_resume(adev, &adev->mc.save);
+	return 0;
+}
+
 static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
 					     struct amdgpu_irq_src *src,
 					     unsigned type,
@@ -1406,7 +1449,10 @@ const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
 	.resume = gmc_v8_0_resume,
 	.is_idle = gmc_v8_0_is_idle,
 	.wait_for_idle = gmc_v8_0_wait_for_idle,
+	.check_soft_reset = gmc_v8_0_check_soft_reset,
+	.pre_soft_reset = gmc_v8_0_pre_soft_reset,
 	.soft_reset = gmc_v8_0_soft_reset,
+	.post_soft_reset = gmc_v8_0_post_soft_reset,
 	.set_clockgating_state = gmc_v8_0_set_clockgating_state,
 	.set_powergating_state = gmc_v8_0_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c b/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c
deleted file mode 100644
index 2f078ad6095c..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <linux/firmware.h>
-#include "drmP.h"
-#include "amdgpu.h"
-#include "iceland_smum.h"
-
-MODULE_FIRMWARE("amdgpu/topaz_smc.bin");
-
-static void iceland_dpm_set_funcs(struct amdgpu_device *adev);
-
-static int iceland_dpm_early_init(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	iceland_dpm_set_funcs(adev);
-
-	return 0;
-}
-
-static int iceland_dpm_init_microcode(struct amdgpu_device *adev)
-{
-	char fw_name[30] = "amdgpu/topaz_smc.bin";
-	int err;
-
-	err = request_firmware(&adev->pm.fw, fw_name, adev->dev);
-	if (err)
-		goto out;
-	err = amdgpu_ucode_validate(adev->pm.fw);
-
-out:
-	if (err) {
-		DRM_ERROR("Failed to load firmware \"%s\"", fw_name);
-		release_firmware(adev->pm.fw);
-		adev->pm.fw = NULL;
-	}
-	return err;
-}
-
-static int iceland_dpm_sw_init(void *handle)
-{
-	int ret;
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	ret = iceland_dpm_init_microcode(adev);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int iceland_dpm_sw_fini(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	release_firmware(adev->pm.fw);
-	adev->pm.fw = NULL;
-
-	return 0;
-}
-
-static int iceland_dpm_hw_init(void *handle)
-{
-	int ret;
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	mutex_lock(&adev->pm.mutex);
-
-	/* smu init only needs to be called at startup, not resume.
-	 * It should be in sw_init, but requires the fw info gathered
-	 * in sw_init from other IP modules.
-	 */
-	ret = iceland_smu_init(adev);
-	if (ret) {
-		DRM_ERROR("SMU initialization failed\n");
-		goto fail;
-	}
-
-	ret = iceland_smu_start(adev);
-	if (ret) {
-		DRM_ERROR("SMU start failed\n");
-		goto fail;
-	}
-
-	mutex_unlock(&adev->pm.mutex);
-	return 0;
-
-fail:
-	adev->firmware.smu_load = false;
-	mutex_unlock(&adev->pm.mutex);
-	return -EINVAL;
-}
-
-static int iceland_dpm_hw_fini(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	mutex_lock(&adev->pm.mutex);
-	/* smu fini only needs to be called at teardown, not suspend.
-	 * It should be in sw_fini, but we put it here for symmetry
-	 * with smu init.
-	 */
-	iceland_smu_fini(adev);
-	mutex_unlock(&adev->pm.mutex);
-	return 0;
-}
-
-static int iceland_dpm_suspend(void *handle)
-{
-	return 0;
-}
-
-static int iceland_dpm_resume(void *handle)
-{
-	int ret;
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	mutex_lock(&adev->pm.mutex);
-
-	ret = iceland_smu_start(adev);
-	if (ret) {
-		DRM_ERROR("SMU start failed\n");
-		goto fail;
-	}
-
-fail:
-	mutex_unlock(&adev->pm.mutex);
-	return ret;
-}
-
-static int iceland_dpm_set_clockgating_state(void *handle,
-			enum amd_clockgating_state state)
-{
-	return 0;
-}
-
-static int iceland_dpm_set_powergating_state(void *handle,
-			enum amd_powergating_state state)
-{
-	return 0;
-}
-
-const struct amd_ip_funcs iceland_dpm_ip_funcs = {
-	.name = "iceland_dpm",
-	.early_init = iceland_dpm_early_init,
-	.late_init = NULL,
-	.sw_init = iceland_dpm_sw_init,
-	.sw_fini = iceland_dpm_sw_fini,
-	.hw_init = iceland_dpm_hw_init,
-	.hw_fini = iceland_dpm_hw_fini,
-	.suspend = iceland_dpm_suspend,
-	.resume = iceland_dpm_resume,
-	.is_idle = NULL,
-	.wait_for_idle = NULL,
-	.soft_reset = NULL,
-	.set_clockgating_state = iceland_dpm_set_clockgating_state,
-	.set_powergating_state = iceland_dpm_set_powergating_state,
-};
-
-static const struct amdgpu_dpm_funcs iceland_dpm_funcs = {
-	.get_temperature = NULL,
-	.pre_set_power_state = NULL,
-	.set_power_state = NULL,
-	.post_set_power_state = NULL,
-	.display_configuration_changed = NULL,
-	.get_sclk = NULL,
-	.get_mclk = NULL,
-	.print_power_state = NULL,
-	.debugfs_print_current_performance_level = NULL,
-	.force_performance_level = NULL,
-	.vblank_too_short = NULL,
-	.powergate_uvd = NULL,
-};
-
-static void iceland_dpm_set_funcs(struct amdgpu_device *adev)
-{
-	if (NULL == adev->pm.funcs)
-		adev->pm.funcs = &iceland_dpm_funcs;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c b/drivers/gpu/drm/amd/amdgpu/iceland_smc.c
deleted file mode 100644
index 211839913728..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c
+++ /dev/null
@@ -1,677 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <linux/firmware.h>
-#include "drmP.h"
-#include "amdgpu.h"
-#include "ppsmc.h"
-#include "iceland_smum.h"
-#include "smu_ucode_xfer_vi.h"
-#include "amdgpu_ucode.h"
-
-#include "smu/smu_7_1_1_d.h"
-#include "smu/smu_7_1_1_sh_mask.h"
-
-#define ICELAND_SMC_SIZE 0x20000
-
-static int iceland_set_smc_sram_address(struct amdgpu_device *adev,
-					uint32_t smc_address, uint32_t limit)
-{
-	uint32_t val;
-
-	if (smc_address & 3)
-		return -EINVAL;
-
-	if ((smc_address + 3) > limit)
-		return -EINVAL;
-
-	WREG32(mmSMC_IND_INDEX_0, smc_address);
-
-	val = RREG32(mmSMC_IND_ACCESS_CNTL);
-	val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
-	WREG32(mmSMC_IND_ACCESS_CNTL, val);
-
-	return 0;
-}
-
-static int iceland_copy_bytes_to_smc(struct amdgpu_device *adev,
-				     uint32_t smc_start_address,
-				     const uint8_t *src,
-				     uint32_t byte_count, uint32_t limit)
-{
-	uint32_t addr;
-	uint32_t data, orig_data;
-	int result = 0;
-	uint32_t extra_shift;
-	unsigned long flags;
-
-	if (smc_start_address & 3)
-		return -EINVAL;
-
-	if ((smc_start_address + byte_count) > limit)
-		return -EINVAL;
-
-	addr = smc_start_address;
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	while (byte_count >= 4) {
-		/* Bytes are written into the SMC addres space with the MSB first */
-		data = (src[0] << 24) + (src[1] << 16) + (src[2] << 8) + src[3];
-
-		result = iceland_set_smc_sram_address(adev, addr, limit);
-
-		if (result)
-			goto out;
-
-		WREG32(mmSMC_IND_DATA_0, data);
-
-		src += 4;
-		byte_count -= 4;
-		addr += 4;
-	}
-
-	if (0 != byte_count) {
-		/* Now write odd bytes left, do a read modify write cycle */
-		data = 0;
-
-		result = iceland_set_smc_sram_address(adev, addr, limit);
-		if (result)
-			goto out;
-
-		orig_data = RREG32(mmSMC_IND_DATA_0);
-		extra_shift = 8 * (4 - byte_count);
-
-		while (byte_count > 0) {
-			data = (data << 8) + *src++;
-			byte_count--;
-		}
-
-		data <<= extra_shift;
-		data |= (orig_data & ~((~0UL) << extra_shift));
-
-		result = iceland_set_smc_sram_address(adev, addr, limit);
-		if (result)
-			goto out;
-
-		WREG32(mmSMC_IND_DATA_0, data);
-	}
-
-out:
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-	return result;
-}
-
-void iceland_start_smc(struct amdgpu_device *adev)
-{
-	uint32_t val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 0);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-}
-
-void iceland_reset_smc(struct amdgpu_device *adev)
-{
-	uint32_t val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-}
-
-static int iceland_program_jump_on_start(struct amdgpu_device *adev)
-{
-	static unsigned char data[] = {0xE0, 0x00, 0x80, 0x40};
-	iceland_copy_bytes_to_smc(adev, 0x0, data, 4, sizeof(data)+1);
-
-	return 0;
-}
-
-void iceland_stop_smc_clock(struct amdgpu_device *adev)
-{
-	uint32_t val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-
-	val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 1);
-	WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
-}
-
-void iceland_start_smc_clock(struct amdgpu_device *adev)
-{
-	uint32_t val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-
-	val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
-	WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
-}
-
-static bool iceland_is_smc_ram_running(struct amdgpu_device *adev)
-{
-	uint32_t val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-	val = REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable);
-
-	return ((0 == val) && (0x20100 <= RREG32_SMC(ixSMC_PC_C)));
-}
-
-static int wait_smu_response(struct amdgpu_device *adev)
-{
-	int i;
-	uint32_t val;
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32(mmSMC_RESP_0);
-		if (REG_GET_FIELD(val, SMC_RESP_0, SMC_RESP))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout)
-		return -EINVAL;
-
-	return 0;
-}
-
-static int iceland_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg)
-{
-	if (!iceland_is_smc_ram_running(adev))
-		return -EINVAL;
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MESSAGE_0, msg);
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send message\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int iceland_send_msg_to_smc_without_waiting(struct amdgpu_device *adev,
-						   PPSMC_Msg msg)
-{
-	if (!iceland_is_smc_ram_running(adev))
-		return -EINVAL;
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MESSAGE_0, msg);
-
-	return 0;
-}
-
-static int iceland_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
-						  PPSMC_Msg msg,
-						  uint32_t parameter)
-{
-	WREG32(mmSMC_MSG_ARG_0, parameter);
-
-	return iceland_send_msg_to_smc(adev, msg);
-}
-
-static int iceland_send_msg_to_smc_with_parameter_without_waiting(
-					struct amdgpu_device *adev,
-					PPSMC_Msg msg, uint32_t parameter)
-{
-	WREG32(mmSMC_MSG_ARG_0, parameter);
-
-	return iceland_send_msg_to_smc_without_waiting(adev, msg);
-}
-
-#if 0 /* not used yet */
-static int iceland_wait_for_smc_inactive(struct amdgpu_device *adev)
-{
-	int i;
-	uint32_t val;
-
-	if (!iceland_is_smc_ram_running(adev))
-		return -EINVAL;
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-		if (REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, cken) == 0)
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout)
-		return -EINVAL;
-
-	return 0;
-}
-#endif
-
-static int iceland_smu_upload_firmware_image(struct amdgpu_device *adev)
-{
-	const struct smc_firmware_header_v1_0 *hdr;
-	uint32_t ucode_size;
-	uint32_t ucode_start_address;
-	const uint8_t *src;
-	uint32_t val;
-	uint32_t byte_count;
-	uint32_t data;
-	unsigned long flags;
-	int i;
-
-	if (!adev->pm.fw)
-		return -EINVAL;
-
-	/* Skip SMC ucode loading on SR-IOV capable boards.
-	 * vbios does this for us in asic_init in that case.
-	 */
-	if (adev->virtualization.supports_sr_iov)
-		return 0;
-
-	hdr = (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data;
-	amdgpu_ucode_print_smc_hdr(&hdr->header);
-
-	adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version);
-	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes);
-	ucode_start_address = le32_to_cpu(hdr->ucode_start_addr);
-	src = (const uint8_t *)
-		(adev->pm.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
-
-	if (ucode_size & 3) {
-		DRM_ERROR("SMC ucode is not 4 bytes aligned\n");
-		return -EINVAL;
-	}
-
-	if (ucode_size > ICELAND_SMC_SIZE) {
-		DRM_ERROR("SMC address is beyond the SMC RAM area\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixRCU_UC_EVENTS);
-		if (REG_GET_FIELD(val, RCU_UC_EVENTS, boot_seq_done) == 0)
-			break;
-		udelay(1);
-	}
-	val = RREG32_SMC(ixSMC_SYSCON_MISC_CNTL);
-	WREG32_SMC(ixSMC_SYSCON_MISC_CNTL, val | 1);
-
-	iceland_stop_smc_clock(adev);
-	iceland_reset_smc(adev);
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	WREG32(mmSMC_IND_INDEX_0, ucode_start_address);
-
-	val = RREG32(mmSMC_IND_ACCESS_CNTL);
-	val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1);
-	WREG32(mmSMC_IND_ACCESS_CNTL, val);
-
-	byte_count = ucode_size;
-	while (byte_count >= 4) {
-		data = (src[0] << 24) + (src[1] << 16) + (src[2] << 8) + src[3];
-		WREG32(mmSMC_IND_DATA_0, data);
-		src += 4;
-		byte_count -= 4;
-	}
-	val = RREG32(mmSMC_IND_ACCESS_CNTL);
-	val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
-	WREG32(mmSMC_IND_ACCESS_CNTL, val);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-
-	return 0;
-}
-
-#if 0 /* not used yet */
-static int iceland_read_smc_sram_dword(struct amdgpu_device *adev,
-				       uint32_t smc_address,
-				       uint32_t *value,
-				       uint32_t limit)
-{
-	int result;
-	unsigned long flags;
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	result = iceland_set_smc_sram_address(adev, smc_address, limit);
-	if (result == 0)
-		*value = RREG32(mmSMC_IND_DATA_0);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-	return result;
-}
-
-static int iceland_write_smc_sram_dword(struct amdgpu_device *adev,
-					uint32_t smc_address,
-					uint32_t value,
-					uint32_t limit)
-{
-	int result;
-	unsigned long flags;
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	result = iceland_set_smc_sram_address(adev, smc_address, limit);
-	if (result == 0)
-		WREG32(mmSMC_IND_DATA_0, value);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-	return result;
-}
-
-static int iceland_smu_stop_smc(struct amdgpu_device *adev)
-{
-	iceland_reset_smc(adev);
-	iceland_stop_smc_clock(adev);
-
-	return 0;
-}
-#endif
-
-static int iceland_smu_start_smc(struct amdgpu_device *adev)
-{
-	int i;
-	uint32_t val;
-
-	iceland_program_jump_on_start(adev);
-	iceland_start_smc_clock(adev);
-	iceland_start_smc(adev);
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixFIRMWARE_FLAGS);
-		if (REG_GET_FIELD(val, FIRMWARE_FLAGS, INTERRUPTS_ENABLED) == 1)
-			break;
-		udelay(1);
-	}
-	return 0;
-}
-
-static enum AMDGPU_UCODE_ID iceland_convert_fw_type(uint32_t fw_type)
-{
-	switch (fw_type) {
-		case UCODE_ID_SDMA0:
-			return AMDGPU_UCODE_ID_SDMA0;
-		case UCODE_ID_SDMA1:
-			return AMDGPU_UCODE_ID_SDMA1;
-		case UCODE_ID_CP_CE:
-			return AMDGPU_UCODE_ID_CP_CE;
-		case UCODE_ID_CP_PFP:
-			return AMDGPU_UCODE_ID_CP_PFP;
-		case UCODE_ID_CP_ME:
-			return AMDGPU_UCODE_ID_CP_ME;
-		case UCODE_ID_CP_MEC:
-		case UCODE_ID_CP_MEC_JT1:
-			return AMDGPU_UCODE_ID_CP_MEC1;
-		case UCODE_ID_CP_MEC_JT2:
-			return AMDGPU_UCODE_ID_CP_MEC2;
-		case UCODE_ID_RLC_G:
-			return AMDGPU_UCODE_ID_RLC_G;
-		default:
-			DRM_ERROR("ucode type is out of range!\n");
-			return AMDGPU_UCODE_ID_MAXIMUM;
-	}
-}
-
-static uint32_t iceland_smu_get_mask_for_fw_type(uint32_t fw_type)
-{
-	switch (fw_type) {
-		case AMDGPU_UCODE_ID_SDMA0:
-			return UCODE_ID_SDMA0_MASK;
-		case AMDGPU_UCODE_ID_SDMA1:
-			return UCODE_ID_SDMA1_MASK;
-		case AMDGPU_UCODE_ID_CP_CE:
-			return UCODE_ID_CP_CE_MASK;
-		case AMDGPU_UCODE_ID_CP_PFP:
-			return UCODE_ID_CP_PFP_MASK;
-		case AMDGPU_UCODE_ID_CP_ME:
-			return UCODE_ID_CP_ME_MASK;
-		case AMDGPU_UCODE_ID_CP_MEC1:
-			return UCODE_ID_CP_MEC_MASK | UCODE_ID_CP_MEC_JT1_MASK;
-		case AMDGPU_UCODE_ID_CP_MEC2:
-			return UCODE_ID_CP_MEC_MASK;
-		case AMDGPU_UCODE_ID_RLC_G:
-			return UCODE_ID_RLC_G_MASK;
-		default:
-			DRM_ERROR("ucode type is out of range!\n");
-			return 0;
-	}
-}
-
-static int iceland_smu_populate_single_firmware_entry(struct amdgpu_device *adev,
-						      uint32_t fw_type,
-						      struct SMU_Entry *entry)
-{
-	enum AMDGPU_UCODE_ID id = iceland_convert_fw_type(fw_type);
-	struct amdgpu_firmware_info *ucode = &adev->firmware.ucode[id];
-	const struct gfx_firmware_header_v1_0 *header = NULL;
-	uint64_t gpu_addr;
-	uint32_t data_size;
-
-	if (ucode->fw == NULL)
-		return -EINVAL;
-
-	gpu_addr  = ucode->mc_addr;
-	header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
-	data_size = le32_to_cpu(header->header.ucode_size_bytes);
-
-	entry->version = (uint16_t)le32_to_cpu(header->header.ucode_version);
-	entry->id = (uint16_t)fw_type;
-	entry->image_addr_high = upper_32_bits(gpu_addr);
-	entry->image_addr_low = lower_32_bits(gpu_addr);
-	entry->meta_data_addr_high = 0;
-	entry->meta_data_addr_low = 0;
-	entry->data_size_byte = data_size;
-	entry->num_register_entries = 0;
-	entry->flags = 0;
-
-	return 0;
-}
-
-static int iceland_smu_request_load_fw(struct amdgpu_device *adev)
-{
-	struct iceland_smu_private_data *private = (struct iceland_smu_private_data *)adev->smu.priv;
-	struct SMU_DRAMData_TOC *toc;
-	uint32_t fw_to_load;
-
-	toc = (struct SMU_DRAMData_TOC *)private->header;
-	toc->num_entries = 0;
-	toc->structure_version = 1;
-
-	if (!adev->firmware.smu_load)
-		return 0;
-
-	if (iceland_smu_populate_single_firmware_entry(adev, UCODE_ID_RLC_G,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for RLC\n");
-		return -EINVAL;
-	}
-
-	if (iceland_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_CE,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for CE\n");
-		return -EINVAL;
-	}
-
-	if (iceland_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_PFP,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for PFP\n");
-		return -EINVAL;
-	}
-
-	if (iceland_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_ME,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for ME\n");
-		return -EINVAL;
-	}
-
-	if (iceland_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for MEC\n");
-		return -EINVAL;
-	}
-
-	if (iceland_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC_JT1,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for MEC_JT1\n");
-		return -EINVAL;
-	}
-
-	if (iceland_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA0,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for SDMA0\n");
-		return -EINVAL;
-	}
-
-	if (iceland_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA1,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for SDMA1\n");
-		return -EINVAL;
-	}
-
-	iceland_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_HI, private->header_addr_high);
-	iceland_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_LO, private->header_addr_low);
-
-	fw_to_load = UCODE_ID_RLC_G_MASK |
-			UCODE_ID_SDMA0_MASK |
-			UCODE_ID_SDMA1_MASK |
-			UCODE_ID_CP_CE_MASK |
-			UCODE_ID_CP_ME_MASK |
-			UCODE_ID_CP_PFP_MASK |
-			UCODE_ID_CP_MEC_MASK |
-			UCODE_ID_CP_MEC_JT1_MASK;
-
-
-	if (iceland_send_msg_to_smc_with_parameter_without_waiting(adev, PPSMC_MSG_LoadUcodes, fw_to_load)) {
-		DRM_ERROR("Fail to request SMU load ucode\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int iceland_smu_check_fw_load_finish(struct amdgpu_device *adev,
-					    uint32_t fw_type)
-{
-	uint32_t fw_mask = iceland_smu_get_mask_for_fw_type(fw_type);
-	int i;
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		if (fw_mask == (RREG32_SMC(ixSOFT_REGISTERS_TABLE_27) & fw_mask))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("check firmware loading failed\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int iceland_smu_start(struct amdgpu_device *adev)
-{
-	int result;
-
-	result = iceland_smu_upload_firmware_image(adev);
-	if (result)
-		return result;
-	result = iceland_smu_start_smc(adev);
-	if (result)
-		return result;
-
-	return iceland_smu_request_load_fw(adev);
-}
-
-static const struct amdgpu_smumgr_funcs iceland_smumgr_funcs = {
-	.check_fw_load_finish = iceland_smu_check_fw_load_finish,
-	.request_smu_load_fw = NULL,
-	.request_smu_specific_fw = NULL,
-};
-
-int iceland_smu_init(struct amdgpu_device *adev)
-{
-	struct iceland_smu_private_data *private;
-	uint32_t image_size = ((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
-	struct amdgpu_bo **toc_buf = &adev->smu.toc_buf;
-	uint64_t mc_addr;
-	void *toc_buf_ptr;
-	int ret;
-
-	private = kzalloc(sizeof(struct iceland_smu_private_data), GFP_KERNEL);
-	if (NULL == private)
-		return -ENOMEM;
-
-	/* allocate firmware buffers */
-	if (adev->firmware.smu_load)
-		amdgpu_ucode_init_bo(adev);
-
-	adev->smu.priv = private;
-	adev->smu.fw_flags = 0;
-
-	/* Allocate FW image data structure and header buffer */
-	ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE,
-			       true, AMDGPU_GEM_DOMAIN_VRAM,
-			       AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-			       NULL, NULL, toc_buf);
-	if (ret) {
-		DRM_ERROR("Failed to allocate memory for TOC buffer\n");
-		return -ENOMEM;
-	}
-
-	/* Retrieve GPU address for header buffer and internal buffer */
-	ret = amdgpu_bo_reserve(adev->smu.toc_buf, false);
-	if (ret) {
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to reserve the TOC buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_pin(adev->smu.toc_buf, AMDGPU_GEM_DOMAIN_VRAM, &mc_addr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.toc_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to pin the TOC buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_kmap(*toc_buf, &toc_buf_ptr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.toc_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to map the TOC buffer\n");
-		return -EINVAL;
-	}
-
-	amdgpu_bo_unreserve(adev->smu.toc_buf);
-	private->header_addr_low = lower_32_bits(mc_addr);
-	private->header_addr_high = upper_32_bits(mc_addr);
-	private->header = toc_buf_ptr;
-
-	adev->smu.smumgr_funcs = &iceland_smumgr_funcs;
-
-	return 0;
-}
-
-int iceland_smu_fini(struct amdgpu_device *adev)
-{
-	amdgpu_bo_unref(&adev->smu.toc_buf);
-	kfree(adev->smu.priv);
-	adev->smu.priv = NULL;
-	if (adev->firmware.fw_buf)
-		amdgpu_ucode_fini_bo(adev);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index a845e883f5fa..f8618a3881a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -2845,7 +2845,11 @@ static int kv_dpm_init(struct amdgpu_device *adev)
 		pi->caps_tcp_ramping = true;
 	}
 
-	pi->caps_sclk_ds = true;
+	if (amdgpu_sclk_deep_sleep_en)
+		pi->caps_sclk_ds = true;
+	else
+		pi->caps_sclk_ds = false;
+
 	pi->enable_auto_thermal_throttling = true;
 	pi->disable_nb_ps3_in_battery = false;
 	if (amdgpu_bapm == 0)
diff --git a/drivers/gpu/drm/amd/amdgpu/r600_dpm.h b/drivers/gpu/drm/amd/amdgpu/r600_dpm.h
new file mode 100644
index 000000000000..055321f61ca7
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/r600_dpm.h
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __R600_DPM_H__
+#define __R600_DPM_H__
+
+#define R600_ASI_DFLT                                10000
+#define R600_BSP_DFLT                                0x41EB
+#define R600_BSU_DFLT                                0x2
+#define R600_AH_DFLT                                 5
+#define R600_RLP_DFLT                                25
+#define R600_RMP_DFLT                                65
+#define R600_LHP_DFLT                                40
+#define R600_LMP_DFLT                                15
+#define R600_TD_DFLT                                 0
+#define R600_UTC_DFLT_00                             0x24
+#define R600_UTC_DFLT_01                             0x22
+#define R600_UTC_DFLT_02                             0x22
+#define R600_UTC_DFLT_03                             0x22
+#define R600_UTC_DFLT_04                             0x22
+#define R600_UTC_DFLT_05                             0x22
+#define R600_UTC_DFLT_06                             0x22
+#define R600_UTC_DFLT_07                             0x22
+#define R600_UTC_DFLT_08                             0x22
+#define R600_UTC_DFLT_09                             0x22
+#define R600_UTC_DFLT_10                             0x22
+#define R600_UTC_DFLT_11                             0x22
+#define R600_UTC_DFLT_12                             0x22
+#define R600_UTC_DFLT_13                             0x22
+#define R600_UTC_DFLT_14                             0x22
+#define R600_DTC_DFLT_00                             0x24
+#define R600_DTC_DFLT_01                             0x22
+#define R600_DTC_DFLT_02                             0x22
+#define R600_DTC_DFLT_03                             0x22
+#define R600_DTC_DFLT_04                             0x22
+#define R600_DTC_DFLT_05                             0x22
+#define R600_DTC_DFLT_06                             0x22
+#define R600_DTC_DFLT_07                             0x22
+#define R600_DTC_DFLT_08                             0x22
+#define R600_DTC_DFLT_09                             0x22
+#define R600_DTC_DFLT_10                             0x22
+#define R600_DTC_DFLT_11                             0x22
+#define R600_DTC_DFLT_12                             0x22
+#define R600_DTC_DFLT_13                             0x22
+#define R600_DTC_DFLT_14                             0x22
+#define R600_VRC_DFLT                                0x0000C003
+#define R600_VOLTAGERESPONSETIME_DFLT                1000
+#define R600_BACKBIASRESPONSETIME_DFLT               1000
+#define R600_VRU_DFLT                                0x3
+#define R600_SPLLSTEPTIME_DFLT                       0x1000
+#define R600_SPLLSTEPUNIT_DFLT                       0x3
+#define R600_TPU_DFLT                                0
+#define R600_TPC_DFLT                                0x200
+#define R600_SSTU_DFLT                               0
+#define R600_SST_DFLT                                0x00C8
+#define R600_GICST_DFLT                              0x200
+#define R600_FCT_DFLT                                0x0400
+#define R600_FCTU_DFLT                               0
+#define R600_CTXCGTT3DRPHC_DFLT                      0x20
+#define R600_CTXCGTT3DRSDC_DFLT                      0x40
+#define R600_VDDC3DOORPHC_DFLT                       0x100
+#define R600_VDDC3DOORSDC_DFLT                       0x7
+#define R600_VDDC3DOORSU_DFLT                        0
+#define R600_MPLLLOCKTIME_DFLT                       100
+#define R600_MPLLRESETTIME_DFLT                      150
+#define R600_VCOSTEPPCT_DFLT                          20
+#define R600_ENDINGVCOSTEPPCT_DFLT                    5
+#define R600_REFERENCEDIVIDER_DFLT                    4
+
+#define R600_PM_NUMBER_OF_TC 15
+#define R600_PM_NUMBER_OF_SCLKS 20
+#define R600_PM_NUMBER_OF_MCLKS 4
+#define R600_PM_NUMBER_OF_VOLTAGE_LEVELS 4
+#define R600_PM_NUMBER_OF_ACTIVITY_LEVELS 3
+
+/* XXX are these ok? */
+#define R600_TEMP_RANGE_MIN (90 * 1000)
+#define R600_TEMP_RANGE_MAX (120 * 1000)
+
+#define FDO_PWM_MODE_STATIC  1
+#define FDO_PWM_MODE_STATIC_RPM 5
+
+enum r600_power_level {
+	R600_POWER_LEVEL_LOW = 0,
+	R600_POWER_LEVEL_MEDIUM = 1,
+	R600_POWER_LEVEL_HIGH = 2,
+	R600_POWER_LEVEL_CTXSW = 3,
+};
+
+enum r600_td {
+	R600_TD_AUTO,
+	R600_TD_UP,
+	R600_TD_DOWN,
+};
+
+enum r600_display_watermark {
+	R600_DISPLAY_WATERMARK_LOW = 0,
+	R600_DISPLAY_WATERMARK_HIGH = 1,
+};
+
+enum r600_display_gap
+{
+    R600_PM_DISPLAY_GAP_VBLANK_OR_WM = 0,
+    R600_PM_DISPLAY_GAP_VBLANK       = 1,
+    R600_PM_DISPLAY_GAP_WATERMARK    = 2,
+    R600_PM_DISPLAY_GAP_IGNORE       = 3,
+};
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 1351c7e834a2..565dab3c7218 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -190,12 +190,8 @@ out:
  */
 static uint32_t sdma_v2_4_ring_get_rptr(struct amdgpu_ring *ring)
 {
-	u32 rptr;
-
 	/* XXX check if swapping is necessary on BE */
-	rptr = ring->adev->wb.wb[ring->rptr_offs] >> 2;
-
-	return rptr;
+	return ring->adev->wb.wb[ring->rptr_offs] >> 2;
 }
 
 /**
@@ -714,7 +710,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		DRM_ERROR("amdgpu: IB test timed out\n");
 		r = -ETIMEDOUT;
 		goto err1;
-	} else if (r) {
+	} else if (r < 0) {
 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 		goto err1;
 	}
@@ -749,24 +745,16 @@ static void sdma_v2_4_vm_copy_pte(struct amdgpu_ib *ib,
 				  uint64_t pe, uint64_t src,
 				  unsigned count)
 {
-	while (count) {
-		unsigned bytes = count * 8;
-		if (bytes > 0x1FFFF8)
-			bytes = 0x1FFFF8;
-
-		ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
-			SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
-		ib->ptr[ib->length_dw++] = bytes;
-		ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
-		ib->ptr[ib->length_dw++] = lower_32_bits(src);
-		ib->ptr[ib->length_dw++] = upper_32_bits(src);
-		ib->ptr[ib->length_dw++] = lower_32_bits(pe);
-		ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-
-		pe += bytes;
-		src += bytes;
-		count -= bytes / 8;
-	}
+	unsigned bytes = count * 8;
+
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+	ib->ptr[ib->length_dw++] = bytes;
+	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+	ib->ptr[ib->length_dw++] = lower_32_bits(src);
+	ib->ptr[ib->length_dw++] = upper_32_bits(src);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 }
 
 /**
@@ -774,39 +762,27 @@ static void sdma_v2_4_vm_copy_pte(struct amdgpu_ib *ib,
  *
  * @ib: indirect buffer to fill with commands
  * @pe: addr of the page entry
- * @addr: dst addr to write into pe
+ * @value: dst addr to write into pe
  * @count: number of page entries to update
  * @incr: increase next addr by incr bytes
- * @flags: access flags
  *
  * Update PTEs by writing them manually using sDMA (CIK).
  */
-static void sdma_v2_4_vm_write_pte(struct amdgpu_ib *ib,
-				   const dma_addr_t *pages_addr, uint64_t pe,
-				   uint64_t addr, unsigned count,
-				   uint32_t incr, uint32_t flags)
+static void sdma_v2_4_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
+				   uint64_t value, unsigned count,
+				   uint32_t incr)
 {
-	uint64_t value;
-	unsigned ndw;
-
-	while (count) {
-		ndw = count * 2;
-		if (ndw > 0xFFFFE)
-			ndw = 0xFFFFE;
-
-		/* for non-physically contiguous pages (system) */
-		ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
-			SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
-		ib->ptr[ib->length_dw++] = pe;
-		ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-		ib->ptr[ib->length_dw++] = ndw;
-		for (; ndw > 0; ndw -= 2, --count, pe += 8) {
-			value = amdgpu_vm_map_gart(pages_addr, addr);
-			addr += incr;
-			value |= flags;
-			ib->ptr[ib->length_dw++] = value;
-			ib->ptr[ib->length_dw++] = upper_32_bits(value);
-		}
+	unsigned ndw = count * 2;
+
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+	ib->ptr[ib->length_dw++] = pe;
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = ndw;
+	for (; ndw > 0; ndw -= 2, --count, pe += 8) {
+		ib->ptr[ib->length_dw++] = lower_32_bits(value);
+		ib->ptr[ib->length_dw++] = upper_32_bits(value);
+		value += incr;
 	}
 }
 
@@ -822,40 +798,21 @@ static void sdma_v2_4_vm_write_pte(struct amdgpu_ib *ib,
  *
  * Update the page tables using sDMA (CIK).
  */
-static void sdma_v2_4_vm_set_pte_pde(struct amdgpu_ib *ib,
-				     uint64_t pe,
+static void sdma_v2_4_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe,
 				     uint64_t addr, unsigned count,
 				     uint32_t incr, uint32_t flags)
 {
-	uint64_t value;
-	unsigned ndw;
-
-	while (count) {
-		ndw = count;
-		if (ndw > 0x7FFFF)
-			ndw = 0x7FFFF;
-
-		if (flags & AMDGPU_PTE_VALID)
-			value = addr;
-		else
-			value = 0;
-
-		/* for physically contiguous pages (vram) */
-		ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_GEN_PTEPDE);
-		ib->ptr[ib->length_dw++] = pe; /* dst addr */
-		ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-		ib->ptr[ib->length_dw++] = flags; /* mask */
-		ib->ptr[ib->length_dw++] = 0;
-		ib->ptr[ib->length_dw++] = value; /* value */
-		ib->ptr[ib->length_dw++] = upper_32_bits(value);
-		ib->ptr[ib->length_dw++] = incr; /* increment size */
-		ib->ptr[ib->length_dw++] = 0;
-		ib->ptr[ib->length_dw++] = ndw; /* number of entries */
-
-		pe += ndw * 8;
-		addr += ndw * incr;
-		count -= ndw;
-	}
+	/* for physically contiguous pages (vram) */
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_GEN_PTEPDE);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = flags; /* mask */
+	ib->ptr[ib->length_dw++] = 0;
+	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
+	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
+	ib->ptr[ib->length_dw++] = incr; /* increment size */
+	ib->ptr[ib->length_dw++] = 0;
+	ib->ptr[ib->length_dw++] = count; /* number of entries */
 }
 
 /**
@@ -945,6 +902,22 @@ static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring,
 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
 }
 
+static unsigned sdma_v2_4_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		7 + 6; /* sdma_v2_4_ring_emit_ib */
+}
+
+static unsigned sdma_v2_4_ring_get_dma_frame_size(struct amdgpu_ring *ring)
+{
+	return
+		6 + /* sdma_v2_4_ring_emit_hdp_flush */
+		3 + /* sdma_v2_4_ring_emit_hdp_invalidate */
+		6 + /* sdma_v2_4_ring_emit_pipeline_sync */
+		12 + /* sdma_v2_4_ring_emit_vm_flush */
+		10 + 10 + 10; /* sdma_v2_4_ring_emit_fence x3 for user fence, vm fence */
+}
+
 static int sdma_v2_4_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -1263,6 +1236,8 @@ static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = {
 	.test_ib = sdma_v2_4_ring_test_ib,
 	.insert_nop = sdma_v2_4_ring_insert_nop,
 	.pad_ib = sdma_v2_4_ring_pad_ib,
+	.get_emit_ib_size = sdma_v2_4_ring_get_emit_ib_size,
+	.get_dma_frame_size = sdma_v2_4_ring_get_dma_frame_size,
 };
 
 static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 653ce5ed55ae..f325fd86430b 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -335,12 +335,8 @@ out:
  */
 static uint32_t sdma_v3_0_ring_get_rptr(struct amdgpu_ring *ring)
 {
-	u32 rptr;
-
 	/* XXX check if swapping is necessary on BE */
-	rptr = ring->adev->wb.wb[ring->rptr_offs] >> 2;
-
-	return rptr;
+	return ring->adev->wb.wb[ring->rptr_offs] >> 2;
 }
 
 /**
@@ -499,31 +495,6 @@ static void sdma_v3_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se
 	amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
 }
 
-unsigned init_cond_exec(struct amdgpu_ring *ring)
-{
-	unsigned ret;
-	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COND_EXE));
-	amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
-	amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
-	amdgpu_ring_write(ring, 1);
-	ret = ring->wptr;/* this is the offset we need patch later */
-	amdgpu_ring_write(ring, 0x55aa55aa);/* insert dummy here and patch it later */
-	return ret;
-}
-
-void patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
-{
-	unsigned cur;
-	BUG_ON(ring->ring[offset] != 0x55aa55aa);
-
-	cur = ring->wptr - 1;
-	if (likely(cur > offset))
-		ring->ring[offset] = cur - offset;
-	else
-		ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
-}
-
-
 /**
  * sdma_v3_0_gfx_stop - stop the gfx async dma engines
  *
@@ -976,24 +947,16 @@ static void sdma_v3_0_vm_copy_pte(struct amdgpu_ib *ib,
 				  uint64_t pe, uint64_t src,
 				  unsigned count)
 {
-	while (count) {
-		unsigned bytes = count * 8;
-		if (bytes > 0x1FFFF8)
-			bytes = 0x1FFFF8;
-
-		ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
-			SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
-		ib->ptr[ib->length_dw++] = bytes;
-		ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
-		ib->ptr[ib->length_dw++] = lower_32_bits(src);
-		ib->ptr[ib->length_dw++] = upper_32_bits(src);
-		ib->ptr[ib->length_dw++] = lower_32_bits(pe);
-		ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-
-		pe += bytes;
-		src += bytes;
-		count -= bytes / 8;
-	}
+	unsigned bytes = count * 8;
+
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+	ib->ptr[ib->length_dw++] = bytes;
+	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+	ib->ptr[ib->length_dw++] = lower_32_bits(src);
+	ib->ptr[ib->length_dw++] = upper_32_bits(src);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 }
 
 /**
@@ -1001,39 +964,27 @@ static void sdma_v3_0_vm_copy_pte(struct amdgpu_ib *ib,
  *
  * @ib: indirect buffer to fill with commands
  * @pe: addr of the page entry
- * @addr: dst addr to write into pe
+ * @value: dst addr to write into pe
  * @count: number of page entries to update
  * @incr: increase next addr by incr bytes
- * @flags: access flags
  *
  * Update PTEs by writing them manually using sDMA (CIK).
  */
-static void sdma_v3_0_vm_write_pte(struct amdgpu_ib *ib,
-				   const dma_addr_t *pages_addr, uint64_t pe,
-				   uint64_t addr, unsigned count,
-				   uint32_t incr, uint32_t flags)
-{
-	uint64_t value;
-	unsigned ndw;
-
-	while (count) {
-		ndw = count * 2;
-		if (ndw > 0xFFFFE)
-			ndw = 0xFFFFE;
-
-		/* for non-physically contiguous pages (system) */
-		ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
-			SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
-		ib->ptr[ib->length_dw++] = pe;
-		ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-		ib->ptr[ib->length_dw++] = ndw;
-		for (; ndw > 0; ndw -= 2, --count, pe += 8) {
-			value = amdgpu_vm_map_gart(pages_addr, addr);
-			addr += incr;
-			value |= flags;
-			ib->ptr[ib->length_dw++] = value;
-			ib->ptr[ib->length_dw++] = upper_32_bits(value);
-		}
+static void sdma_v3_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
+				   uint64_t value, unsigned count,
+				   uint32_t incr)
+{
+	unsigned ndw = count * 2;
+
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = ndw;
+	for (; ndw > 0; ndw -= 2, --count, pe += 8) {
+		ib->ptr[ib->length_dw++] = lower_32_bits(value);
+		ib->ptr[ib->length_dw++] = upper_32_bits(value);
+		value += incr;
 	}
 }
 
@@ -1049,40 +1000,21 @@ static void sdma_v3_0_vm_write_pte(struct amdgpu_ib *ib,
  *
  * Update the page tables using sDMA (CIK).
  */
-static void sdma_v3_0_vm_set_pte_pde(struct amdgpu_ib *ib,
-				     uint64_t pe,
+static void sdma_v3_0_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe,
 				     uint64_t addr, unsigned count,
 				     uint32_t incr, uint32_t flags)
 {
-	uint64_t value;
-	unsigned ndw;
-
-	while (count) {
-		ndw = count;
-		if (ndw > 0x7FFFF)
-			ndw = 0x7FFFF;
-
-		if (flags & AMDGPU_PTE_VALID)
-			value = addr;
-		else
-			value = 0;
-
-		/* for physically contiguous pages (vram) */
-		ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_GEN_PTEPDE);
-		ib->ptr[ib->length_dw++] = pe; /* dst addr */
-		ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-		ib->ptr[ib->length_dw++] = flags; /* mask */
-		ib->ptr[ib->length_dw++] = 0;
-		ib->ptr[ib->length_dw++] = value; /* value */
-		ib->ptr[ib->length_dw++] = upper_32_bits(value);
-		ib->ptr[ib->length_dw++] = incr; /* increment size */
-		ib->ptr[ib->length_dw++] = 0;
-		ib->ptr[ib->length_dw++] = ndw; /* number of entries */
-
-		pe += ndw * 8;
-		addr += ndw * incr;
-		count -= ndw;
-	}
+	/* for physically contiguous pages (vram) */
+	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_GEN_PTEPDE);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = flags; /* mask */
+	ib->ptr[ib->length_dw++] = 0;
+	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
+	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
+	ib->ptr[ib->length_dw++] = incr; /* increment size */
+	ib->ptr[ib->length_dw++] = 0;
+	ib->ptr[ib->length_dw++] = count; /* number of entries */
 }
 
 /**
@@ -1172,6 +1104,22 @@ static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
 }
 
+static unsigned sdma_v3_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		7 + 6; /* sdma_v3_0_ring_emit_ib */
+}
+
+static unsigned sdma_v3_0_ring_get_dma_frame_size(struct amdgpu_ring *ring)
+{
+	return
+		6 + /* sdma_v3_0_ring_emit_hdp_flush */
+		3 + /* sdma_v3_0_ring_emit_hdp_invalidate */
+		6 + /* sdma_v3_0_ring_emit_pipeline_sync */
+		12 + /* sdma_v3_0_ring_emit_vm_flush */
+		10 + 10 + 10; /* sdma_v3_0_ring_emit_fence x3 for user fence, vm fence */
+}
+
 static int sdma_v3_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -1320,28 +1268,79 @@ static int sdma_v3_0_wait_for_idle(void *handle)
 	return -ETIMEDOUT;
 }
 
-static int sdma_v3_0_soft_reset(void *handle)
+static int sdma_v3_0_check_soft_reset(void *handle)
 {
-	u32 srbm_soft_reset = 0;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset = 0;
 	u32 tmp = RREG32(mmSRBM_STATUS2);
 
-	if (tmp & SRBM_STATUS2__SDMA_BUSY_MASK) {
-		/* sdma0 */
-		tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);
-		tmp = REG_SET_FIELD(tmp, SDMA0_F32_CNTL, HALT, 0);
-		WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);
+	if ((tmp & SRBM_STATUS2__SDMA_BUSY_MASK) ||
+	    (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK)) {
 		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK;
-	}
-	if (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK) {
-		/* sdma1 */
-		tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);
-		tmp = REG_SET_FIELD(tmp, SDMA0_F32_CNTL, HALT, 0);
-		WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);
 		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK;
 	}
 
 	if (srbm_soft_reset) {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang = true;
+		adev->sdma.srbm_soft_reset = srbm_soft_reset;
+	} else {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang = false;
+		adev->sdma.srbm_soft_reset = 0;
+	}
+
+	return 0;
+}
+
+static int sdma_v3_0_pre_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset = 0;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang)
+		return 0;
+
+	srbm_soft_reset = adev->sdma.srbm_soft_reset;
+
+	if (REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA) ||
+	    REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA1)) {
+		sdma_v3_0_ctx_switch_enable(adev, false);
+		sdma_v3_0_enable(adev, false);
+	}
+
+	return 0;
+}
+
+static int sdma_v3_0_post_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset = 0;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang)
+		return 0;
+
+	srbm_soft_reset = adev->sdma.srbm_soft_reset;
+
+	if (REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA) ||
+	    REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA1)) {
+		sdma_v3_0_gfx_resume(adev);
+		sdma_v3_0_rlc_resume(adev);
+	}
+
+	return 0;
+}
+
+static int sdma_v3_0_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset = 0;
+	u32 tmp;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_SDMA].hang)
+		return 0;
+
+	srbm_soft_reset = adev->sdma.srbm_soft_reset;
+
+	if (srbm_soft_reset) {
 		tmp = RREG32(mmSRBM_SOFT_RESET);
 		tmp |= srbm_soft_reset;
 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
@@ -1559,6 +1558,9 @@ const struct amd_ip_funcs sdma_v3_0_ip_funcs = {
 	.resume = sdma_v3_0_resume,
 	.is_idle = sdma_v3_0_is_idle,
 	.wait_for_idle = sdma_v3_0_wait_for_idle,
+	.check_soft_reset = sdma_v3_0_check_soft_reset,
+	.pre_soft_reset = sdma_v3_0_pre_soft_reset,
+	.post_soft_reset = sdma_v3_0_post_soft_reset,
 	.soft_reset = sdma_v3_0_soft_reset,
 	.set_clockgating_state = sdma_v3_0_set_clockgating_state,
 	.set_powergating_state = sdma_v3_0_set_powergating_state,
@@ -1579,6 +1581,8 @@ static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = {
 	.test_ib = sdma_v3_0_ring_test_ib,
 	.insert_nop = sdma_v3_0_ring_insert_nop,
 	.pad_ib = sdma_v3_0_ring_pad_ib,
+	.get_emit_ib_size = sdma_v3_0_ring_get_emit_ib_size,
+	.get_dma_frame_size = sdma_v3_0_ring_get_dma_frame_size,
 };
 
 static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
new file mode 100644
index 000000000000..dc9511c5ecb8
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -0,0 +1,1965 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/firmware.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include "drmP.h"
+#include "amdgpu.h"
+#include "amdgpu_atombios.h"
+#include "amdgpu_ih.h"
+#include "amdgpu_uvd.h"
+#include "amdgpu_vce.h"
+#include "atom.h"
+#include "amdgpu_powerplay.h"
+#include "si/sid.h"
+#include "si_ih.h"
+#include "gfx_v6_0.h"
+#include "gmc_v6_0.h"
+#include "si_dma.h"
+#include "dce_v6_0.h"
+#include "si.h"
+
+static const u32 tahiti_golden_registers[] =
+{
+	0x2684, 0x00010000, 0x00018208,
+	0x260c, 0xffffffff, 0x00000000,
+	0x260d, 0xf00fffff, 0x00000400,
+	0x260e, 0x0002021c, 0x00020200,
+	0x031e, 0x00000080, 0x00000000,
+	0x340c, 0x000300c0, 0x00800040,
+	0x360c, 0x000300c0, 0x00800040,
+	0x16ec, 0x000000f0, 0x00000070,
+	0x16f0, 0x00200000, 0x50100000,
+	0x1c0c, 0x31000311, 0x00000011,
+	0x09df, 0x00000003, 0x000007ff,
+	0x0903, 0x000007ff, 0x00000000,
+	0x2285, 0xf000001f, 0x00000007,
+	0x22c9, 0xffffffff, 0x00ffffff,
+	0x22c4, 0x0000ff0f, 0x00000000,
+	0xa293, 0x07ffffff, 0x4e000000,
+	0xa0d4, 0x3f3f3fff, 0x2a00126a,
+	0x000c, 0x000000ff, 0x0040,
+	0x000d, 0x00000040, 0x00004040,
+	0x2440, 0x07ffffff, 0x03000000,
+	0x23a2, 0x01ff1f3f, 0x00000000,
+	0x23a1, 0x01ff1f3f, 0x00000000,
+	0x2418, 0x0000007f, 0x00000020,
+	0x2542, 0x00010000, 0x00010000,
+	0x2b05, 0x00000200, 0x000002fb,
+	0x2b04, 0xffffffff, 0x0000543b,
+	0x2b03, 0xffffffff, 0xa9210876,
+	0x2234, 0xffffffff, 0x000fff40,
+	0x2235, 0x0000001f, 0x00000010,
+	0x0504, 0x20000000, 0x20fffed8,
+	0x0570, 0x000c0fc0, 0x000c0400
+};
+
+static const u32 tahiti_golden_registers2[] =
+{
+	0x0319, 0x00000001, 0x00000001
+};
+
+static const u32 tahiti_golden_rlc_registers[] =
+{
+	0x3109, 0xffffffff, 0x00601005,
+	0x311f, 0xffffffff, 0x10104040,
+	0x3122, 0xffffffff, 0x0100000a,
+	0x30c5, 0xffffffff, 0x00000800,
+	0x30c3, 0xffffffff, 0x800000f4,
+	0x3d2a, 0xffffffff, 0x00000000
+};
+
+static const u32 pitcairn_golden_registers[] =
+{
+	0x2684, 0x00010000, 0x00018208,
+	0x260c, 0xffffffff, 0x00000000,
+	0x260d, 0xf00fffff, 0x00000400,
+	0x260e, 0x0002021c, 0x00020200,
+	0x031e, 0x00000080, 0x00000000,
+	0x340c, 0x000300c0, 0x00800040,
+	0x360c, 0x000300c0, 0x00800040,
+	0x16ec, 0x000000f0, 0x00000070,
+	0x16f0, 0x00200000, 0x50100000,
+	0x1c0c, 0x31000311, 0x00000011,
+	0x0ab9, 0x00073ffe, 0x000022a2,
+	0x0903, 0x000007ff, 0x00000000,
+	0x2285, 0xf000001f, 0x00000007,
+	0x22c9, 0xffffffff, 0x00ffffff,
+	0x22c4, 0x0000ff0f, 0x00000000,
+	0xa293, 0x07ffffff, 0x4e000000,
+	0xa0d4, 0x3f3f3fff, 0x2a00126a,
+	0x000c, 0x000000ff, 0x0040,
+	0x000d, 0x00000040, 0x00004040,
+	0x2440, 0x07ffffff, 0x03000000,
+	0x2418, 0x0000007f, 0x00000020,
+	0x2542, 0x00010000, 0x00010000,
+	0x2b05, 0x000003ff, 0x000000f7,
+	0x2b04, 0xffffffff, 0x00000000,
+	0x2b03, 0xffffffff, 0x32761054,
+	0x2235, 0x0000001f, 0x00000010,
+	0x0570, 0x000c0fc0, 0x000c0400
+};
+
+static const u32 pitcairn_golden_rlc_registers[] =
+{
+	0x3109, 0xffffffff, 0x00601004,
+	0x311f, 0xffffffff, 0x10102020,
+	0x3122, 0xffffffff, 0x01000020,
+	0x30c5, 0xffffffff, 0x00000800,
+	0x30c3, 0xffffffff, 0x800000a4
+};
+
+static const u32 verde_pg_init[] =
+{
+	0xd4f, 0xffffffff, 0x40000,
+	0xd4e, 0xffffffff, 0x200010ff,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x7007,
+	0xd4e, 0xffffffff, 0x300010ff,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x400000,
+	0xd4e, 0xffffffff, 0x100010ff,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x120200,
+	0xd4e, 0xffffffff, 0x500010ff,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x1e1e16,
+	0xd4e, 0xffffffff, 0x600010ff,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x171f1e,
+	0xd4e, 0xffffffff, 0x700010ff,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4f, 0xffffffff, 0x0,
+	0xd4e, 0xffffffff, 0x9ff,
+	0xd40, 0xffffffff, 0x0,
+	0xd41, 0xffffffff, 0x10000800,
+	0xd41, 0xffffffff, 0xf,
+	0xd41, 0xffffffff, 0xf,
+	0xd40, 0xffffffff, 0x4,
+	0xd41, 0xffffffff, 0x1000051e,
+	0xd41, 0xffffffff, 0xffff,
+	0xd41, 0xffffffff, 0xffff,
+	0xd40, 0xffffffff, 0x8,
+	0xd41, 0xffffffff, 0x80500,
+	0xd40, 0xffffffff, 0x12,
+	0xd41, 0xffffffff, 0x9050c,
+	0xd40, 0xffffffff, 0x1d,
+	0xd41, 0xffffffff, 0xb052c,
+	0xd40, 0xffffffff, 0x2a,
+	0xd41, 0xffffffff, 0x1053e,
+	0xd40, 0xffffffff, 0x2d,
+	0xd41, 0xffffffff, 0x10546,
+	0xd40, 0xffffffff, 0x30,
+	0xd41, 0xffffffff, 0xa054e,
+	0xd40, 0xffffffff, 0x3c,
+	0xd41, 0xffffffff, 0x1055f,
+	0xd40, 0xffffffff, 0x3f,
+	0xd41, 0xffffffff, 0x10567,
+	0xd40, 0xffffffff, 0x42,
+	0xd41, 0xffffffff, 0x1056f,
+	0xd40, 0xffffffff, 0x45,
+	0xd41, 0xffffffff, 0x10572,
+	0xd40, 0xffffffff, 0x48,
+	0xd41, 0xffffffff, 0x20575,
+	0xd40, 0xffffffff, 0x4c,
+	0xd41, 0xffffffff, 0x190801,
+	0xd40, 0xffffffff, 0x67,
+	0xd41, 0xffffffff, 0x1082a,
+	0xd40, 0xffffffff, 0x6a,
+	0xd41, 0xffffffff, 0x1b082d,
+	0xd40, 0xffffffff, 0x87,
+	0xd41, 0xffffffff, 0x310851,
+	0xd40, 0xffffffff, 0xba,
+	0xd41, 0xffffffff, 0x891,
+	0xd40, 0xffffffff, 0xbc,
+	0xd41, 0xffffffff, 0x893,
+	0xd40, 0xffffffff, 0xbe,
+	0xd41, 0xffffffff, 0x20895,
+	0xd40, 0xffffffff, 0xc2,
+	0xd41, 0xffffffff, 0x20899,
+	0xd40, 0xffffffff, 0xc6,
+	0xd41, 0xffffffff, 0x2089d,
+	0xd40, 0xffffffff, 0xca,
+	0xd41, 0xffffffff, 0x8a1,
+	0xd40, 0xffffffff, 0xcc,
+	0xd41, 0xffffffff, 0x8a3,
+	0xd40, 0xffffffff, 0xce,
+	0xd41, 0xffffffff, 0x308a5,
+	0xd40, 0xffffffff, 0xd3,
+	0xd41, 0xffffffff, 0x6d08cd,
+	0xd40, 0xffffffff, 0x142,
+	0xd41, 0xffffffff, 0x2000095a,
+	0xd41, 0xffffffff, 0x1,
+	0xd40, 0xffffffff, 0x144,
+	0xd41, 0xffffffff, 0x301f095b,
+	0xd40, 0xffffffff, 0x165,
+	0xd41, 0xffffffff, 0xc094d,
+	0xd40, 0xffffffff, 0x173,
+	0xd41, 0xffffffff, 0xf096d,
+	0xd40, 0xffffffff, 0x184,
+	0xd41, 0xffffffff, 0x15097f,
+	0xd40, 0xffffffff, 0x19b,
+	0xd41, 0xffffffff, 0xc0998,
+	0xd40, 0xffffffff, 0x1a9,
+	0xd41, 0xffffffff, 0x409a7,
+	0xd40, 0xffffffff, 0x1af,
+	0xd41, 0xffffffff, 0xcdc,
+	0xd40, 0xffffffff, 0x1b1,
+	0xd41, 0xffffffff, 0x800,
+	0xd42, 0xffffffff, 0x6c9b2000,
+	0xd44, 0xfc00, 0x2000,
+	0xd51, 0xffffffff, 0xfc0,
+	0xa35, 0x00000100, 0x100
+};
+
+static const u32 verde_golden_rlc_registers[] =
+{
+	0x3109, 0xffffffff, 0x033f1005,
+	0x311f, 0xffffffff, 0x10808020,
+	0x3122, 0xffffffff, 0x00800008,
+	0x30c5, 0xffffffff, 0x00001000,
+	0x30c3, 0xffffffff, 0x80010014
+};
+
+static const u32 verde_golden_registers[] =
+{
+	0x2684, 0x00010000, 0x00018208,
+	0x260c, 0xffffffff, 0x00000000,
+	0x260d, 0xf00fffff, 0x00000400,
+	0x260e, 0x0002021c, 0x00020200,
+	0x031e, 0x00000080, 0x00000000,
+	0x340c, 0x000300c0, 0x00800040,
+	0x340c, 0x000300c0, 0x00800040,
+	0x360c, 0x000300c0, 0x00800040,
+	0x360c, 0x000300c0, 0x00800040,
+	0x16ec, 0x000000f0, 0x00000070,
+	0x16f0, 0x00200000, 0x50100000,
+
+	0x1c0c, 0x31000311, 0x00000011,
+	0x0ab9, 0x00073ffe, 0x000022a2,
+	0x0ab9, 0x00073ffe, 0x000022a2,
+	0x0ab9, 0x00073ffe, 0x000022a2,
+	0x0903, 0x000007ff, 0x00000000,
+	0x0903, 0x000007ff, 0x00000000,
+	0x0903, 0x000007ff, 0x00000000,
+	0x2285, 0xf000001f, 0x00000007,
+	0x2285, 0xf000001f, 0x00000007,
+	0x2285, 0xf000001f, 0x00000007,
+	0x2285, 0xffffffff, 0x00ffffff,
+	0x22c4, 0x0000ff0f, 0x00000000,
+
+	0xa293, 0x07ffffff, 0x4e000000,
+	0xa0d4, 0x3f3f3fff, 0x0000124a,
+	0xa0d4, 0x3f3f3fff, 0x0000124a,
+	0xa0d4, 0x3f3f3fff, 0x0000124a,
+	0x000c, 0x000000ff, 0x0040,
+	0x000d, 0x00000040, 0x00004040,
+	0x2440, 0x07ffffff, 0x03000000,
+	0x2440, 0x07ffffff, 0x03000000,
+	0x23a2, 0x01ff1f3f, 0x00000000,
+	0x23a3, 0x01ff1f3f, 0x00000000,
+	0x23a2, 0x01ff1f3f, 0x00000000,
+	0x23a1, 0x01ff1f3f, 0x00000000,
+	0x23a1, 0x01ff1f3f, 0x00000000,
+
+	0x23a1, 0x01ff1f3f, 0x00000000,
+	0x2418, 0x0000007f, 0x00000020,
+	0x2542, 0x00010000, 0x00010000,
+	0x2b01, 0x000003ff, 0x00000003,
+	0x2b05, 0x000003ff, 0x00000003,
+	0x2b05, 0x000003ff, 0x00000003,
+	0x2b04, 0xffffffff, 0x00000000,
+	0x2b04, 0xffffffff, 0x00000000,
+	0x2b04, 0xffffffff, 0x00000000,
+	0x2b03, 0xffffffff, 0x00001032,
+	0x2b03, 0xffffffff, 0x00001032,
+	0x2b03, 0xffffffff, 0x00001032,
+	0x2235, 0x0000001f, 0x00000010,
+	0x2235, 0x0000001f, 0x00000010,
+	0x2235, 0x0000001f, 0x00000010,
+	0x0570, 0x000c0fc0, 0x000c0400
+};
+
+static const u32 oland_golden_registers[] =
+{
+	0x2684, 0x00010000, 0x00018208,
+	0x260c, 0xffffffff, 0x00000000,
+	0x260d, 0xf00fffff, 0x00000400,
+	0x260e, 0x0002021c, 0x00020200,
+	0x031e, 0x00000080, 0x00000000,
+	0x340c, 0x000300c0, 0x00800040,
+	0x360c, 0x000300c0, 0x00800040,
+	0x16ec, 0x000000f0, 0x00000070,
+	0x16f9, 0x00200000, 0x50100000,
+	0x1c0c, 0x31000311, 0x00000011,
+	0x0ab9, 0x00073ffe, 0x000022a2,
+	0x0903, 0x000007ff, 0x00000000,
+	0x2285, 0xf000001f, 0x00000007,
+	0x22c9, 0xffffffff, 0x00ffffff,
+	0x22c4, 0x0000ff0f, 0x00000000,
+	0xa293, 0x07ffffff, 0x4e000000,
+	0xa0d4, 0x3f3f3fff, 0x00000082,
+	0x000c, 0x000000ff, 0x0040,
+	0x000d, 0x00000040, 0x00004040,
+	0x2440, 0x07ffffff, 0x03000000,
+	0x2418, 0x0000007f, 0x00000020,
+	0x2542, 0x00010000, 0x00010000,
+	0x2b05, 0x000003ff, 0x000000f3,
+	0x2b04, 0xffffffff, 0x00000000,
+	0x2b03, 0xffffffff, 0x00003210,
+	0x2235, 0x0000001f, 0x00000010,
+	0x0570, 0x000c0fc0, 0x000c0400
+};
+
+static const u32 oland_golden_rlc_registers[] =
+{
+	0x3109, 0xffffffff, 0x00601005,
+	0x311f, 0xffffffff, 0x10104040,
+	0x3122, 0xffffffff, 0x0100000a,
+	0x30c5, 0xffffffff, 0x00000800,
+	0x30c3, 0xffffffff, 0x800000f4
+};
+
+static const u32 hainan_golden_registers[] =
+{
+	0x2684, 0x00010000, 0x00018208,
+	0x260c, 0xffffffff, 0x00000000,
+	0x260d, 0xf00fffff, 0x00000400,
+	0x260e, 0x0002021c, 0x00020200,
+	0x4595, 0xff000fff, 0x00000100,
+	0x340c, 0x000300c0, 0x00800040,
+	0x3630, 0xff000fff, 0x00000100,
+	0x360c, 0x000300c0, 0x00800040,
+	0x0ab9, 0x00073ffe, 0x000022a2,
+	0x0903, 0x000007ff, 0x00000000,
+	0x2285, 0xf000001f, 0x00000007,
+	0x22c9, 0xffffffff, 0x00ffffff,
+	0x22c4, 0x0000ff0f, 0x00000000,
+	0xa393, 0x07ffffff, 0x4e000000,
+	0xa0d4, 0x3f3f3fff, 0x00000000,
+	0x000c, 0x000000ff, 0x0040,
+	0x000d, 0x00000040, 0x00004040,
+	0x2440, 0x03e00000, 0x03600000,
+	0x2418, 0x0000007f, 0x00000020,
+	0x2542, 0x00010000, 0x00010000,
+	0x2b05, 0x000003ff, 0x000000f1,
+	0x2b04, 0xffffffff, 0x00000000,
+	0x2b03, 0xffffffff, 0x00003210,
+	0x2235, 0x0000001f, 0x00000010,
+	0x0570, 0x000c0fc0, 0x000c0400
+};
+
+static const u32 hainan_golden_registers2[] =
+{
+	0x263e, 0xffffffff, 0x02010001
+};
+
+static const u32 tahiti_mgcg_cgcg_init[] =
+{
+	0x3100, 0xffffffff, 0xfffffffc,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2698, 0xffffffff, 0x00000100,
+	0x24a9, 0xffffffff, 0x00000100,
+	0x3059, 0xffffffff, 0x00000100,
+	0x25dd, 0xffffffff, 0x00000100,
+	0x2261, 0xffffffff, 0x06000100,
+	0x2286, 0xffffffff, 0x00000100,
+	0x24a8, 0xffffffff, 0x00000100,
+	0x30e0, 0xffffffff, 0x00000100,
+	0x22ca, 0xffffffff, 0x00000100,
+	0x2451, 0xffffffff, 0x00000100,
+	0x2362, 0xffffffff, 0x00000100,
+	0x2363, 0xffffffff, 0x00000100,
+	0x240c, 0xffffffff, 0x00000100,
+	0x240d, 0xffffffff, 0x00000100,
+	0x240e, 0xffffffff, 0x00000100,
+	0x240f, 0xffffffff, 0x00000100,
+	0x2b60, 0xffffffff, 0x00000100,
+	0x2b15, 0xffffffff, 0x00000100,
+	0x225f, 0xffffffff, 0x06000100,
+	0x261a, 0xffffffff, 0x00000100,
+	0x2544, 0xffffffff, 0x00000100,
+	0x2bc1, 0xffffffff, 0x00000100,
+	0x2b81, 0xffffffff, 0x00000100,
+	0x2527, 0xffffffff, 0x00000100,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2458, 0xffffffff, 0x00010000,
+	0x2459, 0xffffffff, 0x00030002,
+	0x245a, 0xffffffff, 0x00040007,
+	0x245b, 0xffffffff, 0x00060005,
+	0x245c, 0xffffffff, 0x00090008,
+	0x245d, 0xffffffff, 0x00020001,
+	0x245e, 0xffffffff, 0x00040003,
+	0x245f, 0xffffffff, 0x00000007,
+	0x2460, 0xffffffff, 0x00060005,
+	0x2461, 0xffffffff, 0x00090008,
+	0x2462, 0xffffffff, 0x00030002,
+	0x2463, 0xffffffff, 0x00050004,
+	0x2464, 0xffffffff, 0x00000008,
+	0x2465, 0xffffffff, 0x00070006,
+	0x2466, 0xffffffff, 0x000a0009,
+	0x2467, 0xffffffff, 0x00040003,
+	0x2468, 0xffffffff, 0x00060005,
+	0x2469, 0xffffffff, 0x00000009,
+	0x246a, 0xffffffff, 0x00080007,
+	0x246b, 0xffffffff, 0x000b000a,
+	0x246c, 0xffffffff, 0x00050004,
+	0x246d, 0xffffffff, 0x00070006,
+	0x246e, 0xffffffff, 0x0008000b,
+	0x246f, 0xffffffff, 0x000a0009,
+	0x2470, 0xffffffff, 0x000d000c,
+	0x2471, 0xffffffff, 0x00060005,
+	0x2472, 0xffffffff, 0x00080007,
+	0x2473, 0xffffffff, 0x0000000b,
+	0x2474, 0xffffffff, 0x000a0009,
+	0x2475, 0xffffffff, 0x000d000c,
+	0x2476, 0xffffffff, 0x00070006,
+	0x2477, 0xffffffff, 0x00090008,
+	0x2478, 0xffffffff, 0x0000000c,
+	0x2479, 0xffffffff, 0x000b000a,
+	0x247a, 0xffffffff, 0x000e000d,
+	0x247b, 0xffffffff, 0x00080007,
+	0x247c, 0xffffffff, 0x000a0009,
+	0x247d, 0xffffffff, 0x0000000d,
+	0x247e, 0xffffffff, 0x000c000b,
+	0x247f, 0xffffffff, 0x000f000e,
+	0x2480, 0xffffffff, 0x00090008,
+	0x2481, 0xffffffff, 0x000b000a,
+	0x2482, 0xffffffff, 0x000c000f,
+	0x2483, 0xffffffff, 0x000e000d,
+	0x2484, 0xffffffff, 0x00110010,
+	0x2485, 0xffffffff, 0x000a0009,
+	0x2486, 0xffffffff, 0x000c000b,
+	0x2487, 0xffffffff, 0x0000000f,
+	0x2488, 0xffffffff, 0x000e000d,
+	0x2489, 0xffffffff, 0x00110010,
+	0x248a, 0xffffffff, 0x000b000a,
+	0x248b, 0xffffffff, 0x000d000c,
+	0x248c, 0xffffffff, 0x00000010,
+	0x248d, 0xffffffff, 0x000f000e,
+	0x248e, 0xffffffff, 0x00120011,
+	0x248f, 0xffffffff, 0x000c000b,
+	0x2490, 0xffffffff, 0x000e000d,
+	0x2491, 0xffffffff, 0x00000011,
+	0x2492, 0xffffffff, 0x0010000f,
+	0x2493, 0xffffffff, 0x00130012,
+	0x2494, 0xffffffff, 0x000d000c,
+	0x2495, 0xffffffff, 0x000f000e,
+	0x2496, 0xffffffff, 0x00100013,
+	0x2497, 0xffffffff, 0x00120011,
+	0x2498, 0xffffffff, 0x00150014,
+	0x2499, 0xffffffff, 0x000e000d,
+	0x249a, 0xffffffff, 0x0010000f,
+	0x249b, 0xffffffff, 0x00000013,
+	0x249c, 0xffffffff, 0x00120011,
+	0x249d, 0xffffffff, 0x00150014,
+	0x249e, 0xffffffff, 0x000f000e,
+	0x249f, 0xffffffff, 0x00110010,
+	0x24a0, 0xffffffff, 0x00000014,
+	0x24a1, 0xffffffff, 0x00130012,
+	0x24a2, 0xffffffff, 0x00160015,
+	0x24a3, 0xffffffff, 0x0010000f,
+	0x24a4, 0xffffffff, 0x00120011,
+	0x24a5, 0xffffffff, 0x00000015,
+	0x24a6, 0xffffffff, 0x00140013,
+	0x24a7, 0xffffffff, 0x00170016,
+	0x2454, 0xffffffff, 0x96940200,
+	0x21c2, 0xffffffff, 0x00900100,
+	0x311e, 0xffffffff, 0x00000080,
+	0x3101, 0xffffffff, 0x0020003f,
+	0xc, 0xffffffff, 0x0000001c,
+	0xd, 0x000f0000, 0x000f0000,
+	0x583, 0xffffffff, 0x00000100,
+	0x409, 0xffffffff, 0x00000100,
+	0x40b, 0x00000101, 0x00000000,
+	0x82a, 0xffffffff, 0x00000104,
+	0x993, 0x000c0000, 0x000c0000,
+	0x992, 0x000c0000, 0x000c0000,
+	0x1579, 0xff000fff, 0x00000100,
+	0x157a, 0x00000001, 0x00000001,
+	0xbd4, 0x00000001, 0x00000001,
+	0xc33, 0xc0000fff, 0x00000104,
+	0x3079, 0x00000001, 0x00000001,
+	0x3430, 0xfffffff0, 0x00000100,
+	0x3630, 0xfffffff0, 0x00000100
+};
+static const u32 pitcairn_mgcg_cgcg_init[] =
+{
+	0x3100, 0xffffffff, 0xfffffffc,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2698, 0xffffffff, 0x00000100,
+	0x24a9, 0xffffffff, 0x00000100,
+	0x3059, 0xffffffff, 0x00000100,
+	0x25dd, 0xffffffff, 0x00000100,
+	0x2261, 0xffffffff, 0x06000100,
+	0x2286, 0xffffffff, 0x00000100,
+	0x24a8, 0xffffffff, 0x00000100,
+	0x30e0, 0xffffffff, 0x00000100,
+	0x22ca, 0xffffffff, 0x00000100,
+	0x2451, 0xffffffff, 0x00000100,
+	0x2362, 0xffffffff, 0x00000100,
+	0x2363, 0xffffffff, 0x00000100,
+	0x240c, 0xffffffff, 0x00000100,
+	0x240d, 0xffffffff, 0x00000100,
+	0x240e, 0xffffffff, 0x00000100,
+	0x240f, 0xffffffff, 0x00000100,
+	0x2b60, 0xffffffff, 0x00000100,
+	0x2b15, 0xffffffff, 0x00000100,
+	0x225f, 0xffffffff, 0x06000100,
+	0x261a, 0xffffffff, 0x00000100,
+	0x2544, 0xffffffff, 0x00000100,
+	0x2bc1, 0xffffffff, 0x00000100,
+	0x2b81, 0xffffffff, 0x00000100,
+	0x2527, 0xffffffff, 0x00000100,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2458, 0xffffffff, 0x00010000,
+	0x2459, 0xffffffff, 0x00030002,
+	0x245a, 0xffffffff, 0x00040007,
+	0x245b, 0xffffffff, 0x00060005,
+	0x245c, 0xffffffff, 0x00090008,
+	0x245d, 0xffffffff, 0x00020001,
+	0x245e, 0xffffffff, 0x00040003,
+	0x245f, 0xffffffff, 0x00000007,
+	0x2460, 0xffffffff, 0x00060005,
+	0x2461, 0xffffffff, 0x00090008,
+	0x2462, 0xffffffff, 0x00030002,
+	0x2463, 0xffffffff, 0x00050004,
+	0x2464, 0xffffffff, 0x00000008,
+	0x2465, 0xffffffff, 0x00070006,
+	0x2466, 0xffffffff, 0x000a0009,
+	0x2467, 0xffffffff, 0x00040003,
+	0x2468, 0xffffffff, 0x00060005,
+	0x2469, 0xffffffff, 0x00000009,
+	0x246a, 0xffffffff, 0x00080007,
+	0x246b, 0xffffffff, 0x000b000a,
+	0x246c, 0xffffffff, 0x00050004,
+	0x246d, 0xffffffff, 0x00070006,
+	0x246e, 0xffffffff, 0x0008000b,
+	0x246f, 0xffffffff, 0x000a0009,
+	0x2470, 0xffffffff, 0x000d000c,
+	0x2480, 0xffffffff, 0x00090008,
+	0x2481, 0xffffffff, 0x000b000a,
+	0x2482, 0xffffffff, 0x000c000f,
+	0x2483, 0xffffffff, 0x000e000d,
+	0x2484, 0xffffffff, 0x00110010,
+	0x2485, 0xffffffff, 0x000a0009,
+	0x2486, 0xffffffff, 0x000c000b,
+	0x2487, 0xffffffff, 0x0000000f,
+	0x2488, 0xffffffff, 0x000e000d,
+	0x2489, 0xffffffff, 0x00110010,
+	0x248a, 0xffffffff, 0x000b000a,
+	0x248b, 0xffffffff, 0x000d000c,
+	0x248c, 0xffffffff, 0x00000010,
+	0x248d, 0xffffffff, 0x000f000e,
+	0x248e, 0xffffffff, 0x00120011,
+	0x248f, 0xffffffff, 0x000c000b,
+	0x2490, 0xffffffff, 0x000e000d,
+	0x2491, 0xffffffff, 0x00000011,
+	0x2492, 0xffffffff, 0x0010000f,
+	0x2493, 0xffffffff, 0x00130012,
+	0x2494, 0xffffffff, 0x000d000c,
+	0x2495, 0xffffffff, 0x000f000e,
+	0x2496, 0xffffffff, 0x00100013,
+	0x2497, 0xffffffff, 0x00120011,
+	0x2498, 0xffffffff, 0x00150014,
+	0x2454, 0xffffffff, 0x96940200,
+	0x21c2, 0xffffffff, 0x00900100,
+	0x311e, 0xffffffff, 0x00000080,
+	0x3101, 0xffffffff, 0x0020003f,
+	0xc, 0xffffffff, 0x0000001c,
+	0xd, 0x000f0000, 0x000f0000,
+	0x583, 0xffffffff, 0x00000100,
+	0x409, 0xffffffff, 0x00000100,
+	0x40b, 0x00000101, 0x00000000,
+	0x82a, 0xffffffff, 0x00000104,
+	0x1579, 0xff000fff, 0x00000100,
+	0x157a, 0x00000001, 0x00000001,
+	0xbd4, 0x00000001, 0x00000001,
+	0xc33, 0xc0000fff, 0x00000104,
+	0x3079, 0x00000001, 0x00000001,
+	0x3430, 0xfffffff0, 0x00000100,
+	0x3630, 0xfffffff0, 0x00000100
+};
+static const u32 verde_mgcg_cgcg_init[] =
+{
+	0x3100, 0xffffffff, 0xfffffffc,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2698, 0xffffffff, 0x00000100,
+	0x24a9, 0xffffffff, 0x00000100,
+	0x3059, 0xffffffff, 0x00000100,
+	0x25dd, 0xffffffff, 0x00000100,
+	0x2261, 0xffffffff, 0x06000100,
+	0x2286, 0xffffffff, 0x00000100,
+	0x24a8, 0xffffffff, 0x00000100,
+	0x30e0, 0xffffffff, 0x00000100,
+	0x22ca, 0xffffffff, 0x00000100,
+	0x2451, 0xffffffff, 0x00000100,
+	0x2362, 0xffffffff, 0x00000100,
+	0x2363, 0xffffffff, 0x00000100,
+	0x240c, 0xffffffff, 0x00000100,
+	0x240d, 0xffffffff, 0x00000100,
+	0x240e, 0xffffffff, 0x00000100,
+	0x240f, 0xffffffff, 0x00000100,
+	0x2b60, 0xffffffff, 0x00000100,
+	0x2b15, 0xffffffff, 0x00000100,
+	0x225f, 0xffffffff, 0x06000100,
+	0x261a, 0xffffffff, 0x00000100,
+	0x2544, 0xffffffff, 0x00000100,
+	0x2bc1, 0xffffffff, 0x00000100,
+	0x2b81, 0xffffffff, 0x00000100,
+	0x2527, 0xffffffff, 0x00000100,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2458, 0xffffffff, 0x00010000,
+	0x2459, 0xffffffff, 0x00030002,
+	0x245a, 0xffffffff, 0x00040007,
+	0x245b, 0xffffffff, 0x00060005,
+	0x245c, 0xffffffff, 0x00090008,
+	0x245d, 0xffffffff, 0x00020001,
+	0x245e, 0xffffffff, 0x00040003,
+	0x245f, 0xffffffff, 0x00000007,
+	0x2460, 0xffffffff, 0x00060005,
+	0x2461, 0xffffffff, 0x00090008,
+	0x2462, 0xffffffff, 0x00030002,
+	0x2463, 0xffffffff, 0x00050004,
+	0x2464, 0xffffffff, 0x00000008,
+	0x2465, 0xffffffff, 0x00070006,
+	0x2466, 0xffffffff, 0x000a0009,
+	0x2467, 0xffffffff, 0x00040003,
+	0x2468, 0xffffffff, 0x00060005,
+	0x2469, 0xffffffff, 0x00000009,
+	0x246a, 0xffffffff, 0x00080007,
+	0x246b, 0xffffffff, 0x000b000a,
+	0x246c, 0xffffffff, 0x00050004,
+	0x246d, 0xffffffff, 0x00070006,
+	0x246e, 0xffffffff, 0x0008000b,
+	0x246f, 0xffffffff, 0x000a0009,
+	0x2470, 0xffffffff, 0x000d000c,
+	0x2480, 0xffffffff, 0x00090008,
+	0x2481, 0xffffffff, 0x000b000a,
+	0x2482, 0xffffffff, 0x000c000f,
+	0x2483, 0xffffffff, 0x000e000d,
+	0x2484, 0xffffffff, 0x00110010,
+	0x2485, 0xffffffff, 0x000a0009,
+	0x2486, 0xffffffff, 0x000c000b,
+	0x2487, 0xffffffff, 0x0000000f,
+	0x2488, 0xffffffff, 0x000e000d,
+	0x2489, 0xffffffff, 0x00110010,
+	0x248a, 0xffffffff, 0x000b000a,
+	0x248b, 0xffffffff, 0x000d000c,
+	0x248c, 0xffffffff, 0x00000010,
+	0x248d, 0xffffffff, 0x000f000e,
+	0x248e, 0xffffffff, 0x00120011,
+	0x248f, 0xffffffff, 0x000c000b,
+	0x2490, 0xffffffff, 0x000e000d,
+	0x2491, 0xffffffff, 0x00000011,
+	0x2492, 0xffffffff, 0x0010000f,
+	0x2493, 0xffffffff, 0x00130012,
+	0x2494, 0xffffffff, 0x000d000c,
+	0x2495, 0xffffffff, 0x000f000e,
+	0x2496, 0xffffffff, 0x00100013,
+	0x2497, 0xffffffff, 0x00120011,
+	0x2498, 0xffffffff, 0x00150014,
+	0x2454, 0xffffffff, 0x96940200,
+	0x21c2, 0xffffffff, 0x00900100,
+	0x311e, 0xffffffff, 0x00000080,
+	0x3101, 0xffffffff, 0x0020003f,
+	0xc, 0xffffffff, 0x0000001c,
+	0xd, 0x000f0000, 0x000f0000,
+	0x583, 0xffffffff, 0x00000100,
+	0x409, 0xffffffff, 0x00000100,
+	0x40b, 0x00000101, 0x00000000,
+	0x82a, 0xffffffff, 0x00000104,
+	0x993, 0x000c0000, 0x000c0000,
+	0x992, 0x000c0000, 0x000c0000,
+	0x1579, 0xff000fff, 0x00000100,
+	0x157a, 0x00000001, 0x00000001,
+	0xbd4, 0x00000001, 0x00000001,
+	0xc33, 0xc0000fff, 0x00000104,
+	0x3079, 0x00000001, 0x00000001,
+	0x3430, 0xfffffff0, 0x00000100,
+	0x3630, 0xfffffff0, 0x00000100
+};
+static const u32 oland_mgcg_cgcg_init[] =
+{
+	0x3100, 0xffffffff, 0xfffffffc,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2698, 0xffffffff, 0x00000100,
+	0x24a9, 0xffffffff, 0x00000100,
+	0x3059, 0xffffffff, 0x00000100,
+	0x25dd, 0xffffffff, 0x00000100,
+	0x2261, 0xffffffff, 0x06000100,
+	0x2286, 0xffffffff, 0x00000100,
+	0x24a8, 0xffffffff, 0x00000100,
+	0x30e0, 0xffffffff, 0x00000100,
+	0x22ca, 0xffffffff, 0x00000100,
+	0x2451, 0xffffffff, 0x00000100,
+	0x2362, 0xffffffff, 0x00000100,
+	0x2363, 0xffffffff, 0x00000100,
+	0x240c, 0xffffffff, 0x00000100,
+	0x240d, 0xffffffff, 0x00000100,
+	0x240e, 0xffffffff, 0x00000100,
+	0x240f, 0xffffffff, 0x00000100,
+	0x2b60, 0xffffffff, 0x00000100,
+	0x2b15, 0xffffffff, 0x00000100,
+	0x225f, 0xffffffff, 0x06000100,
+	0x261a, 0xffffffff, 0x00000100,
+	0x2544, 0xffffffff, 0x00000100,
+	0x2bc1, 0xffffffff, 0x00000100,
+	0x2b81, 0xffffffff, 0x00000100,
+	0x2527, 0xffffffff, 0x00000100,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2458, 0xffffffff, 0x00010000,
+	0x2459, 0xffffffff, 0x00030002,
+	0x245a, 0xffffffff, 0x00040007,
+	0x245b, 0xffffffff, 0x00060005,
+	0x245c, 0xffffffff, 0x00090008,
+	0x245d, 0xffffffff, 0x00020001,
+	0x245e, 0xffffffff, 0x00040003,
+	0x245f, 0xffffffff, 0x00000007,
+	0x2460, 0xffffffff, 0x00060005,
+	0x2461, 0xffffffff, 0x00090008,
+	0x2462, 0xffffffff, 0x00030002,
+	0x2463, 0xffffffff, 0x00050004,
+	0x2464, 0xffffffff, 0x00000008,
+	0x2465, 0xffffffff, 0x00070006,
+	0x2466, 0xffffffff, 0x000a0009,
+	0x2467, 0xffffffff, 0x00040003,
+	0x2468, 0xffffffff, 0x00060005,
+	0x2469, 0xffffffff, 0x00000009,
+	0x246a, 0xffffffff, 0x00080007,
+	0x246b, 0xffffffff, 0x000b000a,
+	0x246c, 0xffffffff, 0x00050004,
+	0x246d, 0xffffffff, 0x00070006,
+	0x246e, 0xffffffff, 0x0008000b,
+	0x246f, 0xffffffff, 0x000a0009,
+	0x2470, 0xffffffff, 0x000d000c,
+	0x2471, 0xffffffff, 0x00060005,
+	0x2472, 0xffffffff, 0x00080007,
+	0x2473, 0xffffffff, 0x0000000b,
+	0x2474, 0xffffffff, 0x000a0009,
+	0x2475, 0xffffffff, 0x000d000c,
+	0x2454, 0xffffffff, 0x96940200,
+	0x21c2, 0xffffffff, 0x00900100,
+	0x311e, 0xffffffff, 0x00000080,
+	0x3101, 0xffffffff, 0x0020003f,
+	0xc, 0xffffffff, 0x0000001c,
+	0xd, 0x000f0000, 0x000f0000,
+	0x583, 0xffffffff, 0x00000100,
+	0x409, 0xffffffff, 0x00000100,
+	0x40b, 0x00000101, 0x00000000,
+	0x82a, 0xffffffff, 0x00000104,
+	0x993, 0x000c0000, 0x000c0000,
+	0x992, 0x000c0000, 0x000c0000,
+	0x1579, 0xff000fff, 0x00000100,
+	0x157a, 0x00000001, 0x00000001,
+	0xbd4, 0x00000001, 0x00000001,
+	0xc33, 0xc0000fff, 0x00000104,
+	0x3079, 0x00000001, 0x00000001,
+	0x3430, 0xfffffff0, 0x00000100,
+	0x3630, 0xfffffff0, 0x00000100
+};
+static const u32 hainan_mgcg_cgcg_init[] =
+{
+	0x3100, 0xffffffff, 0xfffffffc,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2698, 0xffffffff, 0x00000100,
+	0x24a9, 0xffffffff, 0x00000100,
+	0x3059, 0xffffffff, 0x00000100,
+	0x25dd, 0xffffffff, 0x00000100,
+	0x2261, 0xffffffff, 0x06000100,
+	0x2286, 0xffffffff, 0x00000100,
+	0x24a8, 0xffffffff, 0x00000100,
+	0x30e0, 0xffffffff, 0x00000100,
+	0x22ca, 0xffffffff, 0x00000100,
+	0x2451, 0xffffffff, 0x00000100,
+	0x2362, 0xffffffff, 0x00000100,
+	0x2363, 0xffffffff, 0x00000100,
+	0x240c, 0xffffffff, 0x00000100,
+	0x240d, 0xffffffff, 0x00000100,
+	0x240e, 0xffffffff, 0x00000100,
+	0x240f, 0xffffffff, 0x00000100,
+	0x2b60, 0xffffffff, 0x00000100,
+	0x2b15, 0xffffffff, 0x00000100,
+	0x225f, 0xffffffff, 0x06000100,
+	0x261a, 0xffffffff, 0x00000100,
+	0x2544, 0xffffffff, 0x00000100,
+	0x2bc1, 0xffffffff, 0x00000100,
+	0x2b81, 0xffffffff, 0x00000100,
+	0x2527, 0xffffffff, 0x00000100,
+	0x200b, 0xffffffff, 0xe0000000,
+	0x2458, 0xffffffff, 0x00010000,
+	0x2459, 0xffffffff, 0x00030002,
+	0x245a, 0xffffffff, 0x00040007,
+	0x245b, 0xffffffff, 0x00060005,
+	0x245c, 0xffffffff, 0x00090008,
+	0x245d, 0xffffffff, 0x00020001,
+	0x245e, 0xffffffff, 0x00040003,
+	0x245f, 0xffffffff, 0x00000007,
+	0x2460, 0xffffffff, 0x00060005,
+	0x2461, 0xffffffff, 0x00090008,
+	0x2462, 0xffffffff, 0x00030002,
+	0x2463, 0xffffffff, 0x00050004,
+	0x2464, 0xffffffff, 0x00000008,
+	0x2465, 0xffffffff, 0x00070006,
+	0x2466, 0xffffffff, 0x000a0009,
+	0x2467, 0xffffffff, 0x00040003,
+	0x2468, 0xffffffff, 0x00060005,
+	0x2469, 0xffffffff, 0x00000009,
+	0x246a, 0xffffffff, 0x00080007,
+	0x246b, 0xffffffff, 0x000b000a,
+	0x246c, 0xffffffff, 0x00050004,
+	0x246d, 0xffffffff, 0x00070006,
+	0x246e, 0xffffffff, 0x0008000b,
+	0x246f, 0xffffffff, 0x000a0009,
+	0x2470, 0xffffffff, 0x000d000c,
+	0x2471, 0xffffffff, 0x00060005,
+	0x2472, 0xffffffff, 0x00080007,
+	0x2473, 0xffffffff, 0x0000000b,
+	0x2474, 0xffffffff, 0x000a0009,
+	0x2475, 0xffffffff, 0x000d000c,
+	0x2454, 0xffffffff, 0x96940200,
+	0x21c2, 0xffffffff, 0x00900100,
+	0x311e, 0xffffffff, 0x00000080,
+	0x3101, 0xffffffff, 0x0020003f,
+	0xc, 0xffffffff, 0x0000001c,
+	0xd, 0x000f0000, 0x000f0000,
+	0x583, 0xffffffff, 0x00000100,
+	0x409, 0xffffffff, 0x00000100,
+	0x82a, 0xffffffff, 0x00000104,
+	0x993, 0x000c0000, 0x000c0000,
+	0x992, 0x000c0000, 0x000c0000,
+	0xbd4, 0x00000001, 0x00000001,
+	0xc33, 0xc0000fff, 0x00000104,
+	0x3079, 0x00000001, 0x00000001,
+	0x3430, 0xfffffff0, 0x00000100,
+	0x3630, 0xfffffff0, 0x00000100
+};
+
+static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
+{
+	unsigned long flags;
+	u32 r;
+
+	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	WREG32(AMDGPU_PCIE_INDEX, reg);
+	(void)RREG32(AMDGPU_PCIE_INDEX);
+	r = RREG32(AMDGPU_PCIE_DATA);
+	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	return r;
+}
+
+static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	WREG32(AMDGPU_PCIE_INDEX, reg);
+	(void)RREG32(AMDGPU_PCIE_INDEX);
+	WREG32(AMDGPU_PCIE_DATA, v);
+	(void)RREG32(AMDGPU_PCIE_DATA);
+	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+}
+
+u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
+{
+	unsigned long flags;
+	u32 r;
+
+	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
+	(void)RREG32(PCIE_PORT_INDEX);
+	r = RREG32(PCIE_PORT_DATA);
+	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	return r;
+}
+
+void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
+	(void)RREG32(PCIE_PORT_INDEX);
+	WREG32(PCIE_PORT_DATA, (v));
+	(void)RREG32(PCIE_PORT_DATA);
+	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+}
+
+static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
+{
+	unsigned long flags;
+	u32 r;
+
+	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	WREG32(SMC_IND_INDEX_0, (reg));
+	r = RREG32(SMC_IND_DATA_0);
+	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+	return r;
+}
+
+static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	WREG32(SMC_IND_INDEX_0, (reg));
+	WREG32(SMC_IND_DATA_0, (v));
+	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+}
+
+static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
+	{GRBM_STATUS, false},
+	{GB_ADDR_CONFIG, false},
+	{MC_ARB_RAMCFG, false},
+	{GB_TILE_MODE0, false},
+	{GB_TILE_MODE1, false},
+	{GB_TILE_MODE2, false},
+	{GB_TILE_MODE3, false},
+	{GB_TILE_MODE4, false},
+	{GB_TILE_MODE5, false},
+	{GB_TILE_MODE6, false},
+	{GB_TILE_MODE7, false},
+	{GB_TILE_MODE8, false},
+	{GB_TILE_MODE9, false},
+	{GB_TILE_MODE10, false},
+	{GB_TILE_MODE11, false},
+	{GB_TILE_MODE12, false},
+	{GB_TILE_MODE13, false},
+	{GB_TILE_MODE14, false},
+	{GB_TILE_MODE15, false},
+	{GB_TILE_MODE16, false},
+	{GB_TILE_MODE17, false},
+	{GB_TILE_MODE18, false},
+	{GB_TILE_MODE19, false},
+	{GB_TILE_MODE20, false},
+	{GB_TILE_MODE21, false},
+	{GB_TILE_MODE22, false},
+	{GB_TILE_MODE23, false},
+	{GB_TILE_MODE24, false},
+	{GB_TILE_MODE25, false},
+	{GB_TILE_MODE26, false},
+	{GB_TILE_MODE27, false},
+	{GB_TILE_MODE28, false},
+	{GB_TILE_MODE29, false},
+	{GB_TILE_MODE30, false},
+	{GB_TILE_MODE31, false},
+	{CC_RB_BACKEND_DISABLE, false, true},
+	{GC_USER_RB_BACKEND_DISABLE, false, true},
+	{PA_SC_RASTER_CONFIG, false, true},
+};
+
+static uint32_t si_read_indexed_register(struct amdgpu_device *adev,
+					  u32 se_num, u32 sh_num,
+					  u32 reg_offset)
+{
+	uint32_t val;
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	if (se_num != 0xffffffff || sh_num != 0xffffffff)
+		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
+
+	val = RREG32(reg_offset);
+
+	if (se_num != 0xffffffff || sh_num != 0xffffffff)
+		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+	mutex_unlock(&adev->grbm_idx_mutex);
+	return val;
+}
+
+static int si_read_register(struct amdgpu_device *adev, u32 se_num,
+			     u32 sh_num, u32 reg_offset, u32 *value)
+{
+	uint32_t i;
+
+	*value = 0;
+	for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
+		if (reg_offset != si_allowed_read_registers[i].reg_offset)
+			continue;
+
+		if (!si_allowed_read_registers[i].untouched)
+			*value = si_allowed_read_registers[i].grbm_indexed ?
+				 si_read_indexed_register(adev, se_num,
+							   sh_num, reg_offset) :
+				 RREG32(reg_offset);
+		return 0;
+	}
+	return -EINVAL;
+}
+
+static bool si_read_disabled_bios(struct amdgpu_device *adev)
+{
+	u32 bus_cntl;
+	u32 d1vga_control = 0;
+	u32 d2vga_control = 0;
+	u32 vga_render_control = 0;
+	u32 rom_cntl;
+	bool r;
+
+	bus_cntl = RREG32(R600_BUS_CNTL);
+	if (adev->mode_info.num_crtc) {
+		d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
+		d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
+		vga_render_control = RREG32(VGA_RENDER_CONTROL);
+	}
+	rom_cntl = RREG32(R600_ROM_CNTL);
+
+	/* enable the rom */
+	WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
+	if (adev->mode_info.num_crtc) {
+		/* Disable VGA mode */
+		WREG32(AVIVO_D1VGA_CONTROL,
+		       (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
+					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));
+		WREG32(AVIVO_D2VGA_CONTROL,
+		       (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
+					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));
+		WREG32(VGA_RENDER_CONTROL,
+		       (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
+	}
+	WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
+
+	r = amdgpu_read_bios(adev);
+
+	/* restore regs */
+	WREG32(R600_BUS_CNTL, bus_cntl);
+	if (adev->mode_info.num_crtc) {
+		WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
+		WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
+		WREG32(VGA_RENDER_CONTROL, vga_render_control);
+	}
+	WREG32(R600_ROM_CNTL, rom_cntl);
+	return r;
+}
+
+//xxx: not implemented
+static int si_asic_reset(struct amdgpu_device *adev)
+{
+	return 0;
+}
+
+static void si_vga_set_state(struct amdgpu_device *adev, bool state)
+{
+	uint32_t temp;
+
+	temp = RREG32(CONFIG_CNTL);
+	if (state == false) {
+		temp &= ~(1<<0);
+		temp |= (1<<1);
+	} else {
+		temp &= ~(1<<1);
+	}
+	WREG32(CONFIG_CNTL, temp);
+}
+
+static u32 si_get_xclk(struct amdgpu_device *adev)
+{
+        u32 reference_clock = adev->clock.spll.reference_freq;
+	u32 tmp;
+
+	tmp = RREG32(CG_CLKPIN_CNTL_2);
+	if (tmp & MUX_TCLK_TO_XCLK)
+		return TCLK;
+
+	tmp = RREG32(CG_CLKPIN_CNTL);
+	if (tmp & XTALIN_DIVIDE)
+		return reference_clock / 4;
+
+	return reference_clock;
+}
+
+//xxx:not implemented
+static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
+{
+	return 0;
+}
+
+static void si_detect_hw_virtualization(struct amdgpu_device *adev)
+{
+	if (is_virtual_machine()) /* passthrough mode */
+		adev->virtualization.virtual_caps |= AMDGPU_PASSTHROUGH_MODE;
+}
+
+static const struct amdgpu_asic_funcs si_asic_funcs =
+{
+	.read_disabled_bios = &si_read_disabled_bios,
+	.detect_hw_virtualization = si_detect_hw_virtualization,
+	.read_register = &si_read_register,
+	.reset = &si_asic_reset,
+	.set_vga_state = &si_vga_set_state,
+	.get_xclk = &si_get_xclk,
+	.set_uvd_clocks = &si_set_uvd_clocks,
+	.set_vce_clocks = NULL,
+};
+
+static uint32_t si_get_rev_id(struct amdgpu_device *adev)
+{
+	return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
+		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
+}
+
+static int si_common_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	adev->smc_rreg = &si_smc_rreg;
+	adev->smc_wreg = &si_smc_wreg;
+	adev->pcie_rreg = &si_pcie_rreg;
+	adev->pcie_wreg = &si_pcie_wreg;
+	adev->pciep_rreg = &si_pciep_rreg;
+	adev->pciep_wreg = &si_pciep_wreg;
+	adev->uvd_ctx_rreg = NULL;
+	adev->uvd_ctx_wreg = NULL;
+	adev->didt_rreg = NULL;
+	adev->didt_wreg = NULL;
+
+	adev->asic_funcs = &si_asic_funcs;
+
+	adev->rev_id = si_get_rev_id(adev);
+	adev->external_rev_id = 0xFF;
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+		adev->cg_flags =
+			AMD_CG_SUPPORT_GFX_MGCG |
+			AMD_CG_SUPPORT_GFX_MGLS |
+			/*AMD_CG_SUPPORT_GFX_CGCG |*/
+			AMD_CG_SUPPORT_GFX_CGLS |
+			AMD_CG_SUPPORT_GFX_CGTS |
+			AMD_CG_SUPPORT_GFX_CP_LS |
+			AMD_CG_SUPPORT_MC_MGCG |
+			AMD_CG_SUPPORT_SDMA_MGCG |
+			AMD_CG_SUPPORT_BIF_LS |
+			AMD_CG_SUPPORT_VCE_MGCG |
+			AMD_CG_SUPPORT_UVD_MGCG |
+			AMD_CG_SUPPORT_HDP_LS |
+			AMD_CG_SUPPORT_HDP_MGCG;
+			adev->pg_flags = 0;
+		break;
+	case CHIP_PITCAIRN:
+		adev->cg_flags =
+			AMD_CG_SUPPORT_GFX_MGCG |
+			AMD_CG_SUPPORT_GFX_MGLS |
+			/*AMD_CG_SUPPORT_GFX_CGCG |*/
+			AMD_CG_SUPPORT_GFX_CGLS |
+			AMD_CG_SUPPORT_GFX_CGTS |
+			AMD_CG_SUPPORT_GFX_CP_LS |
+			AMD_CG_SUPPORT_GFX_RLC_LS |
+			AMD_CG_SUPPORT_MC_LS |
+			AMD_CG_SUPPORT_MC_MGCG |
+			AMD_CG_SUPPORT_SDMA_MGCG |
+			AMD_CG_SUPPORT_BIF_LS |
+			AMD_CG_SUPPORT_VCE_MGCG |
+			AMD_CG_SUPPORT_UVD_MGCG |
+			AMD_CG_SUPPORT_HDP_LS |
+			AMD_CG_SUPPORT_HDP_MGCG;
+		adev->pg_flags = 0;
+		break;
+
+	case CHIP_VERDE:
+		adev->cg_flags =
+			AMD_CG_SUPPORT_GFX_MGCG |
+			AMD_CG_SUPPORT_GFX_MGLS |
+			AMD_CG_SUPPORT_GFX_CGLS |
+			AMD_CG_SUPPORT_GFX_CGTS |
+			AMD_CG_SUPPORT_GFX_CGTS_LS |
+			AMD_CG_SUPPORT_GFX_CP_LS |
+			AMD_CG_SUPPORT_MC_LS |
+			AMD_CG_SUPPORT_MC_MGCG |
+			AMD_CG_SUPPORT_SDMA_MGCG |
+			AMD_CG_SUPPORT_SDMA_LS |
+			AMD_CG_SUPPORT_BIF_LS |
+			AMD_CG_SUPPORT_VCE_MGCG |
+			AMD_CG_SUPPORT_UVD_MGCG |
+			AMD_CG_SUPPORT_HDP_LS |
+			AMD_CG_SUPPORT_HDP_MGCG;
+		adev->pg_flags = 0;
+		//???
+		adev->external_rev_id = adev->rev_id + 0x14;
+		break;
+	case CHIP_OLAND:
+		adev->cg_flags =
+			AMD_CG_SUPPORT_GFX_MGCG |
+			AMD_CG_SUPPORT_GFX_MGLS |
+			/*AMD_CG_SUPPORT_GFX_CGCG |*/
+			AMD_CG_SUPPORT_GFX_CGLS |
+			AMD_CG_SUPPORT_GFX_CGTS |
+			AMD_CG_SUPPORT_GFX_CP_LS |
+			AMD_CG_SUPPORT_GFX_RLC_LS |
+			AMD_CG_SUPPORT_MC_LS |
+			AMD_CG_SUPPORT_MC_MGCG |
+			AMD_CG_SUPPORT_SDMA_MGCG |
+			AMD_CG_SUPPORT_BIF_LS |
+			AMD_CG_SUPPORT_UVD_MGCG |
+			AMD_CG_SUPPORT_HDP_LS |
+			AMD_CG_SUPPORT_HDP_MGCG;
+		adev->pg_flags = 0;
+		break;
+	case CHIP_HAINAN:
+		adev->cg_flags =
+			AMD_CG_SUPPORT_GFX_MGCG |
+			AMD_CG_SUPPORT_GFX_MGLS |
+			/*AMD_CG_SUPPORT_GFX_CGCG |*/
+			AMD_CG_SUPPORT_GFX_CGLS |
+			AMD_CG_SUPPORT_GFX_CGTS |
+			AMD_CG_SUPPORT_GFX_CP_LS |
+			AMD_CG_SUPPORT_GFX_RLC_LS |
+			AMD_CG_SUPPORT_MC_LS |
+			AMD_CG_SUPPORT_MC_MGCG |
+			AMD_CG_SUPPORT_SDMA_MGCG |
+			AMD_CG_SUPPORT_BIF_LS |
+			AMD_CG_SUPPORT_HDP_LS |
+			AMD_CG_SUPPORT_HDP_MGCG;
+		adev->pg_flags = 0;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int si_common_sw_init(void *handle)
+{
+	return 0;
+}
+
+static int si_common_sw_fini(void *handle)
+{
+	return 0;
+}
+
+
+static void si_init_golden_registers(struct amdgpu_device *adev)
+{
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+		amdgpu_program_register_sequence(adev,
+						 tahiti_golden_registers,
+						 (const u32)ARRAY_SIZE(tahiti_golden_registers));
+		amdgpu_program_register_sequence(adev,
+						 tahiti_golden_rlc_registers,
+						 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
+		amdgpu_program_register_sequence(adev,
+						 tahiti_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+						 tahiti_golden_registers2,
+						 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
+		break;
+	case CHIP_PITCAIRN:
+		amdgpu_program_register_sequence(adev,
+						 pitcairn_golden_registers,
+						 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
+		amdgpu_program_register_sequence(adev,
+						 pitcairn_golden_rlc_registers,
+						 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
+		amdgpu_program_register_sequence(adev,
+						 pitcairn_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
+	case CHIP_VERDE:
+		amdgpu_program_register_sequence(adev,
+						 verde_golden_registers,
+						 (const u32)ARRAY_SIZE(verde_golden_registers));
+		amdgpu_program_register_sequence(adev,
+						 verde_golden_rlc_registers,
+						 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
+		amdgpu_program_register_sequence(adev,
+						 verde_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
+		amdgpu_program_register_sequence(adev,
+						 verde_pg_init,
+						 (const u32)ARRAY_SIZE(verde_pg_init));
+		break;
+	case CHIP_OLAND:
+		amdgpu_program_register_sequence(adev,
+						 oland_golden_registers,
+						 (const u32)ARRAY_SIZE(oland_golden_registers));
+		amdgpu_program_register_sequence(adev,
+						 oland_golden_rlc_registers,
+						 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
+		amdgpu_program_register_sequence(adev,
+						 oland_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
+	case CHIP_HAINAN:
+		amdgpu_program_register_sequence(adev,
+						 hainan_golden_registers,
+						 (const u32)ARRAY_SIZE(hainan_golden_registers));
+		amdgpu_program_register_sequence(adev,
+						 hainan_golden_registers2,
+						 (const u32)ARRAY_SIZE(hainan_golden_registers2));
+		amdgpu_program_register_sequence(adev,
+						 hainan_mgcg_cgcg_init,
+						 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
+		break;
+
+
+	default:
+		BUG();
+	}
+}
+
+static void si_pcie_gen3_enable(struct amdgpu_device *adev)
+{
+	struct pci_dev *root = adev->pdev->bus->self;
+	int bridge_pos, gpu_pos;
+	u32 speed_cntl, mask, current_data_rate;
+	int ret, i;
+	u16 tmp16;
+
+	if (pci_is_root_bus(adev->pdev->bus))
+		return;
+
+	if (amdgpu_pcie_gen2 == 0)
+		return;
+
+	if (adev->flags & AMD_IS_APU)
+		return;
+
+	ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
+	if (ret != 0)
+		return;
+
+	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
+		return;
+
+	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
+		LC_CURRENT_DATA_RATE_SHIFT;
+	if (mask & DRM_PCIE_SPEED_80) {
+		if (current_data_rate == 2) {
+			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
+			return;
+		}
+		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
+	} else if (mask & DRM_PCIE_SPEED_50) {
+		if (current_data_rate == 1) {
+			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
+			return;
+		}
+		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
+	}
+
+	bridge_pos = pci_pcie_cap(root);
+	if (!bridge_pos)
+		return;
+
+	gpu_pos = pci_pcie_cap(adev->pdev);
+	if (!gpu_pos)
+		return;
+
+	if (mask & DRM_PCIE_SPEED_80) {
+		if (current_data_rate != 2) {
+			u16 bridge_cfg, gpu_cfg;
+			u16 bridge_cfg2, gpu_cfg2;
+			u32 max_lw, current_lw, tmp;
+
+			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
+			pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
+
+			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
+			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
+
+			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
+			pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
+
+			tmp = RREG32_PCIE(PCIE_LC_STATUS1);
+			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
+			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
+
+			if (current_lw < max_lw) {
+				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
+				if (tmp & LC_RENEGOTIATION_SUPPORT) {
+					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
+					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
+					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
+					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
+				}
+			}
+
+			for (i = 0; i < 10; i++) {
+				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
+				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
+					break;
+
+				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
+				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
+
+				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
+				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
+
+				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
+				tmp |= LC_SET_QUIESCE;
+				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
+
+				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
+				tmp |= LC_REDO_EQ;
+				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
+
+				mdelay(100);
+
+				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
+				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
+				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
+				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
+
+				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
+				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
+				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
+				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
+
+				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
+				tmp16 &= ~((1 << 4) | (7 << 9));
+				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
+				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
+
+				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
+				tmp16 &= ~((1 << 4) | (7 << 9));
+				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
+				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
+
+				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
+				tmp &= ~LC_SET_QUIESCE;
+				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
+			}
+		}
+	}
+
+	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
+	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
+	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
+	tmp16 &= ~0xf;
+	if (mask & DRM_PCIE_SPEED_80)
+		tmp16 |= 3;
+	else if (mask & DRM_PCIE_SPEED_50)
+		tmp16 |= 2;
+	else
+		tmp16 |= 1;
+	pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
+
+	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
+	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
+		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
+			break;
+		udelay(1);
+	}
+}
+
+static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
+{
+	unsigned long flags;
+	u32 r;
+
+	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
+	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
+	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	return r;
+}
+
+static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
+	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
+	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+}
+
+static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
+{
+	unsigned long flags;
+	u32 r;
+
+	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
+	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
+	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+	return r;
+}
+
+static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
+	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
+	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
+	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
+}
+static void si_program_aspm(struct amdgpu_device *adev)
+{
+	u32 data, orig;
+	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
+	bool disable_clkreq = false;
+
+	if (amdgpu_aspm == 0)
+		return;
+
+	if (adev->flags & AMD_IS_APU)
+		return;
+	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
+	data &= ~LC_XMIT_N_FTS_MASK;
+	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
+	if (orig != data)
+		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
+
+	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
+	data |= LC_GO_TO_RECOVERY;
+	if (orig != data)
+		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
+
+	orig = data = RREG32_PCIE(PCIE_P_CNTL);
+	data |= P_IGNORE_EDB_ERR;
+	if (orig != data)
+		WREG32_PCIE(PCIE_P_CNTL, data);
+
+	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
+	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
+	data |= LC_PMI_TO_L1_DIS;
+	if (!disable_l0s)
+		data |= LC_L0S_INACTIVITY(7);
+
+	if (!disable_l1) {
+		data |= LC_L1_INACTIVITY(7);
+		data &= ~LC_PMI_TO_L1_DIS;
+		if (orig != data)
+			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
+
+		if (!disable_plloff_in_l1) {
+			bool clk_req_support;
+
+			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
+			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
+			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
+			if (orig != data)
+				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
+
+			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
+			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
+			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
+			if (orig != data)
+				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
+
+			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
+			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
+			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
+			if (orig != data)
+				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
+
+			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
+			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
+			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
+			if (orig != data)
+				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
+
+			if ((adev->family != CHIP_OLAND) && (adev->family != CHIP_HAINAN)) {
+				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
+				data &= ~PLL_RAMP_UP_TIME_0_MASK;
+				if (orig != data)
+					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
+
+				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
+				data &= ~PLL_RAMP_UP_TIME_1_MASK;
+				if (orig != data)
+					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
+
+				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
+				data &= ~PLL_RAMP_UP_TIME_2_MASK;
+				if (orig != data)
+					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
+
+				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
+				data &= ~PLL_RAMP_UP_TIME_3_MASK;
+				if (orig != data)
+					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
+
+				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
+				data &= ~PLL_RAMP_UP_TIME_0_MASK;
+				if (orig != data)
+					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
+
+				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
+				data &= ~PLL_RAMP_UP_TIME_1_MASK;
+				if (orig != data)
+					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
+
+				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
+				data &= ~PLL_RAMP_UP_TIME_2_MASK;
+				if (orig != data)
+					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
+
+				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
+				data &= ~PLL_RAMP_UP_TIME_3_MASK;
+				if (orig != data)
+					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
+			}
+			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
+			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
+			data |= LC_DYN_LANES_PWR_STATE(3);
+			if (orig != data)
+				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
+
+			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
+			data &= ~LS2_EXIT_TIME_MASK;
+			if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
+				data |= LS2_EXIT_TIME(5);
+			if (orig != data)
+				si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
+
+			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
+			data &= ~LS2_EXIT_TIME_MASK;
+			if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
+				data |= LS2_EXIT_TIME(5);
+			if (orig != data)
+				si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
+
+			if (!disable_clkreq &&
+			    !pci_is_root_bus(adev->pdev->bus)) {
+				struct pci_dev *root = adev->pdev->bus->self;
+				u32 lnkcap;
+
+				clk_req_support = false;
+				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
+				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
+					clk_req_support = true;
+			} else {
+				clk_req_support = false;
+			}
+
+			if (clk_req_support) {
+				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
+				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
+				if (orig != data)
+					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
+
+				orig = data = RREG32(THM_CLK_CNTL);
+				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
+				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
+				if (orig != data)
+					WREG32(THM_CLK_CNTL, data);
+
+				orig = data = RREG32(MISC_CLK_CNTL);
+				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
+				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
+				if (orig != data)
+					WREG32(MISC_CLK_CNTL, data);
+
+				orig = data = RREG32(CG_CLKPIN_CNTL);
+				data &= ~BCLK_AS_XCLK;
+				if (orig != data)
+					WREG32(CG_CLKPIN_CNTL, data);
+
+				orig = data = RREG32(CG_CLKPIN_CNTL_2);
+				data &= ~FORCE_BIF_REFCLK_EN;
+				if (orig != data)
+					WREG32(CG_CLKPIN_CNTL_2, data);
+
+				orig = data = RREG32(MPLL_BYPASSCLK_SEL);
+				data &= ~MPLL_CLKOUT_SEL_MASK;
+				data |= MPLL_CLKOUT_SEL(4);
+				if (orig != data)
+					WREG32(MPLL_BYPASSCLK_SEL, data);
+
+				orig = data = RREG32(SPLL_CNTL_MODE);
+				data &= ~SPLL_REFCLK_SEL_MASK;
+				if (orig != data)
+					WREG32(SPLL_CNTL_MODE, data);
+			}
+		}
+	} else {
+		if (orig != data)
+			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
+	}
+
+	orig = data = RREG32_PCIE(PCIE_CNTL2);
+	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
+	if (orig != data)
+		WREG32_PCIE(PCIE_CNTL2, data);
+
+	if (!disable_l0s) {
+		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
+		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
+			data = RREG32_PCIE(PCIE_LC_STATUS1);
+			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
+				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
+				data &= ~LC_L0S_INACTIVITY_MASK;
+				if (orig != data)
+					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
+			}
+		}
+	}
+}
+
+static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
+{
+	int readrq;
+	u16 v;
+
+	readrq = pcie_get_readrq(adev->pdev);
+	v = ffs(readrq) - 8;
+	if ((v == 0) || (v == 6) || (v == 7))
+		pcie_set_readrq(adev->pdev, 512);
+}
+
+static int si_common_hw_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	si_fix_pci_max_read_req_size(adev);
+	si_init_golden_registers(adev);
+	si_pcie_gen3_enable(adev);
+	si_program_aspm(adev);
+
+	return 0;
+}
+
+static int si_common_hw_fini(void *handle)
+{
+	return 0;
+}
+
+static int si_common_suspend(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return si_common_hw_fini(adev);
+}
+
+static int si_common_resume(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return si_common_hw_init(adev);
+}
+
+static bool si_common_is_idle(void *handle)
+{
+	return true;
+}
+
+static int si_common_wait_for_idle(void *handle)
+{
+	return 0;
+}
+
+static int si_common_soft_reset(void *handle)
+{
+	return 0;
+}
+
+static int si_common_set_clockgating_state(void *handle,
+					    enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int si_common_set_powergating_state(void *handle,
+					    enum amd_powergating_state state)
+{
+	return 0;
+}
+
+const struct amd_ip_funcs si_common_ip_funcs = {
+	.name = "si_common",
+	.early_init = si_common_early_init,
+	.late_init = NULL,
+	.sw_init = si_common_sw_init,
+	.sw_fini = si_common_sw_fini,
+	.hw_init = si_common_hw_init,
+	.hw_fini = si_common_hw_fini,
+	.suspend = si_common_suspend,
+	.resume = si_common_resume,
+	.is_idle = si_common_is_idle,
+	.wait_for_idle = si_common_wait_for_idle,
+	.soft_reset = si_common_soft_reset,
+	.set_clockgating_state = si_common_set_clockgating_state,
+	.set_powergating_state = si_common_set_powergating_state,
+};
+
+static const struct amdgpu_ip_block_version verde_ip_blocks[] =
+{
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &si_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gmc_v6_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &si_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &dce_v6_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gfx_v6_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &si_dma_ip_funcs,
+	},
+/*	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 3,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &si_null_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &si_null_ip_funcs,
+	},
+	*/
+};
+
+
+static const struct amdgpu_ip_block_version hainan_ip_blocks[] =
+{
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &si_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gmc_v6_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &si_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gfx_v6_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &si_dma_ip_funcs,
+	},
+};
+
+int si_set_ip_blocks(struct amdgpu_device *adev)
+{
+	switch (adev->asic_type) {
+	case CHIP_VERDE:
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+	case CHIP_OLAND:
+		adev->ip_blocks = verde_ip_blocks;
+		adev->num_ip_blocks = ARRAY_SIZE(verde_ip_blocks);
+		break;
+	case CHIP_HAINAN:
+		adev->ip_blocks = hainan_ip_blocks;
+		adev->num_ip_blocks = ARRAY_SIZE(hainan_ip_blocks);
+		break;
+	default:
+		BUG();
+	}
+	return 0;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_smum.h b/drivers/gpu/drm/amd/amdgpu/si.h
index 1cef03deeac3..959d7b63e0e5 100644
--- a/drivers/gpu/drm/amd/amdgpu/fiji_smum.h
+++ b/drivers/gpu/drm/amd/amdgpu/si.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2014 Advanced Micro Devices, Inc.
+ * Copyright 2015 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -21,22 +21,13 @@
  *
  */
 
-#ifndef FIJI_SMUMGR_H
-#define FIJI_SMUMGR_H
+#ifndef __SI_H__
+#define __SI_H__
 
-#include "fiji_ppsmc.h"
+extern const struct amd_ip_funcs si_common_ip_funcs;
 
-int fiji_smu_init(struct amdgpu_device *adev);
-int fiji_smu_fini(struct amdgpu_device *adev);
-int fiji_smu_start(struct amdgpu_device *adev);
-
-struct fiji_smu_private_data
-{
-	uint8_t *header;
-	uint32_t smu_buffer_addr_high;
-	uint32_t smu_buffer_addr_low;
-	uint32_t header_addr_high;
-	uint32_t header_addr_low;
-};
+void si_srbm_select(struct amdgpu_device *adev,
+		     u32 me, u32 pipe, u32 queue, u32 vmid);
+int si_set_ip_blocks(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
new file mode 100644
index 000000000000..de358193a8f9
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -0,0 +1,915 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_trace.h"
+#include "si/sid.h"
+
+const u32 sdma_offsets[SDMA_MAX_INSTANCE] =
+{
+	DMA0_REGISTER_OFFSET,
+	DMA1_REGISTER_OFFSET
+};
+
+static void si_dma_set_ring_funcs(struct amdgpu_device *adev);
+static void si_dma_set_buffer_funcs(struct amdgpu_device *adev);
+static void si_dma_set_vm_pte_funcs(struct amdgpu_device *adev);
+static void si_dma_set_irq_funcs(struct amdgpu_device *adev);
+
+static uint32_t si_dma_ring_get_rptr(struct amdgpu_ring *ring)
+{
+	return ring->adev->wb.wb[ring->rptr_offs>>2];
+}
+
+static uint32_t si_dma_ring_get_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
+
+	return (RREG32(DMA_RB_WPTR + sdma_offsets[me]) & 0x3fffc) >> 2;
+}
+
+static void si_dma_ring_set_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
+
+	WREG32(DMA_RB_WPTR + sdma_offsets[me], (ring->wptr << 2) & 0x3fffc);
+}
+
+static void si_dma_ring_emit_ib(struct amdgpu_ring *ring,
+				struct amdgpu_ib *ib,
+				unsigned vm_id, bool ctx_switch)
+{
+	/* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
+	 * Pad as necessary with NOPs.
+	 */
+	while ((ring->wptr & 7) != 5)
+		amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
+	amdgpu_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, vm_id, 0));
+	amdgpu_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
+	amdgpu_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
+
+}
+
+static void si_dma_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
+	amdgpu_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL));
+	amdgpu_ring_write(ring, 1);
+}
+
+static void si_dma_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
+{
+	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
+	amdgpu_ring_write(ring, (0xf << 16) | (HDP_DEBUG0));
+	amdgpu_ring_write(ring, 1);
+}
+
+/**
+ * si_dma_ring_emit_fence - emit a fence on the DMA ring
+ *
+ * @ring: amdgpu ring pointer
+ * @fence: amdgpu fence object
+ *
+ * Add a DMA fence packet to the ring to write
+ * the fence seq number and DMA trap packet to generate
+ * an interrupt if needed (VI).
+ */
+static void si_dma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
+				      unsigned flags)
+{
+
+	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
+	/* write the fence */
+	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0, 0));
+	amdgpu_ring_write(ring, addr & 0xfffffffc);
+	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xff));
+	amdgpu_ring_write(ring, seq);
+	/* optionally write high bits as well */
+	if (write64bit) {
+		addr += 4;
+		amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0, 0));
+		amdgpu_ring_write(ring, addr & 0xfffffffc);
+		amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xff));
+		amdgpu_ring_write(ring, upper_32_bits(seq));
+	}
+	/* generate an interrupt */
+	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0, 0));
+}
+
+static void si_dma_stop(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	u32 rb_cntl;
+	unsigned i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		ring = &adev->sdma.instance[i].ring;
+		/* dma0 */
+		rb_cntl = RREG32(DMA_RB_CNTL + sdma_offsets[i]);
+		rb_cntl &= ~DMA_RB_ENABLE;
+		WREG32(DMA_RB_CNTL + sdma_offsets[i], rb_cntl);
+
+		if (adev->mman.buffer_funcs_ring == ring)
+			amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
+		ring->ready = false;
+	}
+}
+
+static int si_dma_start(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	u32 rb_cntl, dma_cntl, ib_cntl, rb_bufsz;
+	int i, r;
+	uint64_t rptr_addr;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		ring = &adev->sdma.instance[i].ring;
+
+		WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + sdma_offsets[i], 0);
+		WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + sdma_offsets[i], 0);
+
+		/* Set ring buffer size in dwords */
+		rb_bufsz = order_base_2(ring->ring_size / 4);
+		rb_cntl = rb_bufsz << 1;
+#ifdef __BIG_ENDIAN
+		rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
+#endif
+		WREG32(DMA_RB_CNTL + sdma_offsets[i], rb_cntl);
+
+		/* Initialize the ring buffer's read and write pointers */
+		WREG32(DMA_RB_RPTR + sdma_offsets[i], 0);
+		WREG32(DMA_RB_WPTR + sdma_offsets[i], 0);
+
+		rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
+
+		WREG32(DMA_RB_RPTR_ADDR_LO + sdma_offsets[i], lower_32_bits(rptr_addr));
+		WREG32(DMA_RB_RPTR_ADDR_HI + sdma_offsets[i], upper_32_bits(rptr_addr) & 0xFF);
+
+		rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
+
+		WREG32(DMA_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8);
+
+		/* enable DMA IBs */
+		ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE;
+#ifdef __BIG_ENDIAN
+		ib_cntl |= DMA_IB_SWAP_ENABLE;
+#endif
+		WREG32(DMA_IB_CNTL + sdma_offsets[i], ib_cntl);
+
+		dma_cntl = RREG32(DMA_CNTL + sdma_offsets[i]);
+		dma_cntl &= ~CTXEMPTY_INT_ENABLE;
+		WREG32(DMA_CNTL + sdma_offsets[i], dma_cntl);
+
+		ring->wptr = 0;
+		WREG32(DMA_RB_WPTR + sdma_offsets[i], ring->wptr << 2);
+		WREG32(DMA_RB_CNTL + sdma_offsets[i], rb_cntl | DMA_RB_ENABLE);
+
+		ring->ready = true;
+
+		r = amdgpu_ring_test_ring(ring);
+		if (r) {
+			ring->ready = false;
+			return r;
+		}
+
+		if (adev->mman.buffer_funcs_ring == ring)
+			amdgpu_ttm_set_active_vram_size(adev, adev->mc.real_vram_size);
+	}
+
+	return 0;
+}
+
+/**
+ * si_dma_ring_test_ring - simple async dma engine test
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ *
+ * Test the DMA engine by writing using it to write an
+ * value to memory. (VI).
+ * Returns 0 for success, error for failure.
+ */
+static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	unsigned i;
+	unsigned index;
+	int r;
+	u32 tmp;
+	u64 gpu_addr;
+
+	r = amdgpu_wb_get(adev, &index);
+	if (r) {
+		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
+		return r;
+	}
+
+	gpu_addr = adev->wb.gpu_addr + (index * 4);
+	tmp = 0xCAFEDEAD;
+	adev->wb.wb[index] = cpu_to_le32(tmp);
+
+	r = amdgpu_ring_alloc(ring, 4);
+	if (r) {
+		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
+		amdgpu_wb_free(adev, index);
+		return r;
+	}
+
+	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1));
+	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
+	amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xff);
+	amdgpu_ring_write(ring, 0xDEADBEEF);
+	amdgpu_ring_commit(ring);
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = le32_to_cpu(adev->wb.wb[index]);
+		if (tmp == 0xDEADBEEF)
+			break;
+		DRM_UDELAY(1);
+	}
+
+	if (i < adev->usec_timeout) {
+		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+	} else {
+		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
+			  ring->idx, tmp);
+		r = -EINVAL;
+	}
+	amdgpu_wb_free(adev, index);
+
+	return r;
+}
+
+/**
+ * si_dma_ring_test_ib - test an IB on the DMA engine
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ *
+ * Test a simple IB in the DMA ring (VI).
+ * Returns 0 on success, error on failure.
+ */
+static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_ib ib;
+	struct fence *f = NULL;
+	unsigned index;
+	u32 tmp = 0;
+	u64 gpu_addr;
+	long r;
+
+	r = amdgpu_wb_get(adev, &index);
+	if (r) {
+		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
+		return r;
+	}
+
+	gpu_addr = adev->wb.gpu_addr + (index * 4);
+	tmp = 0xCAFEDEAD;
+	adev->wb.wb[index] = cpu_to_le32(tmp);
+	memset(&ib, 0, sizeof(ib));
+	r = amdgpu_ib_get(adev, NULL, 256, &ib);
+	if (r) {
+		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
+		goto err0;
+	}
+
+	ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1);
+	ib.ptr[1] = lower_32_bits(gpu_addr);
+	ib.ptr[2] = upper_32_bits(gpu_addr) & 0xff;
+	ib.ptr[3] = 0xDEADBEEF;
+	ib.length_dw = 4;
+	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, NULL, &f);
+	if (r)
+		goto err1;
+
+	r = fence_wait_timeout(f, false, timeout);
+	if (r == 0) {
+		DRM_ERROR("amdgpu: IB test timed out\n");
+		r = -ETIMEDOUT;
+		goto err1;
+	} else if (r < 0) {
+		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+		goto err1;
+	}
+	tmp = le32_to_cpu(adev->wb.wb[index]);
+	if (tmp == 0xDEADBEEF) {
+		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		r = 0;
+	} else {
+		DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
+		r = -EINVAL;
+	}
+
+err1:
+	amdgpu_ib_free(adev, &ib, NULL);
+	fence_put(f);
+err0:
+	amdgpu_wb_free(adev, index);
+	return r;
+}
+
+/**
+ * cik_dma_vm_copy_pte - update PTEs by copying them from the GART
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @src: src addr to copy from
+ * @count: number of page entries to update
+ *
+ * Update PTEs by copying them from the GART using DMA (SI).
+ */
+static void si_dma_vm_copy_pte(struct amdgpu_ib *ib,
+			       uint64_t pe, uint64_t src,
+			       unsigned count)
+{
+	unsigned bytes = count * 8;
+
+	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY,
+					      1, 0, 0, bytes);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = lower_32_bits(src);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
+	ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff;
+}
+
+/**
+ * si_dma_vm_write_pte - update PTEs by writing them manually
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @value: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ *
+ * Update PTEs by writing them manually using DMA (SI).
+ */
+static void si_dma_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
+				uint64_t value, unsigned count,
+				uint32_t incr)
+{
+	unsigned ndw = count * 2;
+
+	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	for (; ndw > 0; ndw -= 2) {
+		ib->ptr[ib->length_dw++] = lower_32_bits(value);
+		ib->ptr[ib->length_dw++] = upper_32_bits(value);
+		value += incr;
+	}
+}
+
+/**
+ * si_dma_vm_set_pte_pde - update the page tables using sDMA
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: access flags
+ *
+ * Update the page tables using sDMA (CIK).
+ */
+static void si_dma_vm_set_pte_pde(struct amdgpu_ib *ib,
+				     uint64_t pe,
+				     uint64_t addr, unsigned count,
+				     uint32_t incr, uint32_t flags)
+{
+	uint64_t value;
+	unsigned ndw;
+
+	while (count) {
+		ndw = count * 2;
+		if (ndw > 0xFFFFE)
+			ndw = 0xFFFFE;
+
+		if (flags & AMDGPU_PTE_VALID)
+			value = addr;
+		else
+			value = 0;
+
+		/* for physically contiguous pages (vram) */
+		ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
+		ib->ptr[ib->length_dw++] = pe; /* dst addr */
+		ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
+		ib->ptr[ib->length_dw++] = flags; /* mask */
+		ib->ptr[ib->length_dw++] = 0;
+		ib->ptr[ib->length_dw++] = value; /* value */
+		ib->ptr[ib->length_dw++] = upper_32_bits(value);
+		ib->ptr[ib->length_dw++] = incr; /* increment size */
+		ib->ptr[ib->length_dw++] = 0;
+		pe += ndw * 4;
+		addr += (ndw / 2) * incr;
+		count -= ndw / 2;
+	}
+}
+
+/**
+ * si_dma_pad_ib - pad the IB to the required number of dw
+ *
+ * @ib: indirect buffer to fill with padding
+ *
+ */
+static void si_dma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
+{
+	while (ib->length_dw & 0x7)
+		ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
+}
+
+/**
+ * cik_sdma_ring_emit_pipeline_sync - sync the pipeline
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Make sure all previous operations are completed (CIK).
+ */
+static void si_dma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+	uint32_t seq = ring->fence_drv.sync_seq;
+	uint64_t addr = ring->fence_drv.gpu_addr;
+
+	/* wait for idle */
+	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0) |
+			  (1 << 27)); /* Poll memory */
+	amdgpu_ring_write(ring, lower_32_bits(addr));
+	amdgpu_ring_write(ring, (0xff << 16) | upper_32_bits(addr)); /* retry, addr_hi */
+	amdgpu_ring_write(ring, 0xffffffff); /* mask */
+	amdgpu_ring_write(ring, seq); /* value */
+	amdgpu_ring_write(ring, (3 << 28) | 0x20); /* func(equal) | poll interval */
+}
+
+/**
+ * si_dma_ring_emit_vm_flush - cik vm flush using sDMA
+ *
+ * @ring: amdgpu_ring pointer
+ * @vm: amdgpu_vm pointer
+ *
+ * Update the page table base and flush the VM TLB
+ * using sDMA (VI).
+ */
+static void si_dma_ring_emit_vm_flush(struct amdgpu_ring *ring,
+				      unsigned vm_id, uint64_t pd_addr)
+{
+	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
+	if (vm_id < 8)
+		amdgpu_ring_write(ring, (0xf << 16) | (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
+	else
+		amdgpu_ring_write(ring, (0xf << 16) | (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vm_id - 8)));
+	amdgpu_ring_write(ring, pd_addr >> 12);
+
+	/* bits 0-7 are the VM contexts0-7 */
+	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
+	amdgpu_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST));
+	amdgpu_ring_write(ring, 1 << vm_id);
+
+	/* wait for invalidate to complete */
+	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0));
+	amdgpu_ring_write(ring, VM_INVALIDATE_REQUEST);
+	amdgpu_ring_write(ring, 0xff << 16); /* retry */
+	amdgpu_ring_write(ring, 1 << vm_id); /* mask */
+	amdgpu_ring_write(ring, 0); /* value */
+	amdgpu_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */
+}
+
+static unsigned si_dma_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		7 + 3; /* si_dma_ring_emit_ib */
+}
+
+static unsigned si_dma_ring_get_dma_frame_size(struct amdgpu_ring *ring)
+{
+	return
+		3 + /* si_dma_ring_emit_hdp_flush */
+		3 + /* si_dma_ring_emit_hdp_invalidate */
+		6 + /* si_dma_ring_emit_pipeline_sync */
+		12 + /* si_dma_ring_emit_vm_flush */
+		9 + 9 + 9; /* si_dma_ring_emit_fence x3 for user fence, vm fence */
+}
+
+static int si_dma_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	adev->sdma.num_instances = 2;
+
+	si_dma_set_ring_funcs(adev);
+	si_dma_set_buffer_funcs(adev);
+	si_dma_set_vm_pte_funcs(adev);
+	si_dma_set_irq_funcs(adev);
+
+	return 0;
+}
+
+static int si_dma_sw_init(void *handle)
+{
+	struct amdgpu_ring *ring;
+	int r, i;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	/* DMA0 trap event */
+	r = amdgpu_irq_add_id(adev, 224, &adev->sdma.trap_irq);
+	if (r)
+		return r;
+
+	/* DMA1 trap event */
+	r = amdgpu_irq_add_id(adev, 244, &adev->sdma.trap_irq_1);
+	if (r)
+		return r;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		ring = &adev->sdma.instance[i].ring;
+		ring->ring_obj = NULL;
+		ring->use_doorbell = false;
+		sprintf(ring->name, "sdma%d", i);
+		r = amdgpu_ring_init(adev, ring, 1024,
+				     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0), 0xf,
+				     &adev->sdma.trap_irq,
+				     (i == 0) ?
+				     AMDGPU_SDMA_IRQ_TRAP0 : AMDGPU_SDMA_IRQ_TRAP1,
+				     AMDGPU_RING_TYPE_SDMA);
+		if (r)
+			return r;
+	}
+
+	return r;
+}
+
+static int si_dma_sw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++)
+		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
+
+	return 0;
+}
+
+static int si_dma_hw_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return si_dma_start(adev);
+}
+
+static int si_dma_hw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	si_dma_stop(adev);
+
+	return 0;
+}
+
+static int si_dma_suspend(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return si_dma_hw_fini(adev);
+}
+
+static int si_dma_resume(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return si_dma_hw_init(adev);
+}
+
+static bool si_dma_is_idle(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 tmp = RREG32(SRBM_STATUS2);
+
+	if (tmp & (DMA_BUSY_MASK | DMA1_BUSY_MASK))
+	    return false;
+
+	return true;
+}
+
+static int si_dma_wait_for_idle(void *handle)
+{
+	unsigned i;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (si_dma_is_idle(handle))
+			return 0;
+		udelay(1);
+	}
+	return -ETIMEDOUT;
+}
+
+static int si_dma_soft_reset(void *handle)
+{
+	DRM_INFO("si_dma_soft_reset --- not implemented !!!!!!!\n");
+	return 0;
+}
+
+static int si_dma_set_trap_irq_state(struct amdgpu_device *adev,
+					struct amdgpu_irq_src *src,
+					unsigned type,
+					enum amdgpu_interrupt_state state)
+{
+	u32 sdma_cntl;
+
+	switch (type) {
+	case AMDGPU_SDMA_IRQ_TRAP0:
+		switch (state) {
+		case AMDGPU_IRQ_STATE_DISABLE:
+			sdma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET);
+			sdma_cntl &= ~TRAP_ENABLE;
+			WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, sdma_cntl);
+			break;
+		case AMDGPU_IRQ_STATE_ENABLE:
+			sdma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET);
+			sdma_cntl |= TRAP_ENABLE;
+			WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, sdma_cntl);
+			break;
+		default:
+			break;
+		}
+		break;
+	case AMDGPU_SDMA_IRQ_TRAP1:
+		switch (state) {
+		case AMDGPU_IRQ_STATE_DISABLE:
+			sdma_cntl = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET);
+			sdma_cntl &= ~TRAP_ENABLE;
+			WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, sdma_cntl);
+			break;
+		case AMDGPU_IRQ_STATE_ENABLE:
+			sdma_cntl = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET);
+			sdma_cntl |= TRAP_ENABLE;
+			WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, sdma_cntl);
+			break;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int si_dma_process_trap_irq(struct amdgpu_device *adev,
+				      struct amdgpu_irq_src *source,
+				      struct amdgpu_iv_entry *entry)
+{
+	amdgpu_fence_process(&adev->sdma.instance[0].ring);
+
+	return 0;
+}
+
+static int si_dma_process_trap_irq_1(struct amdgpu_device *adev,
+				      struct amdgpu_irq_src *source,
+				      struct amdgpu_iv_entry *entry)
+{
+	amdgpu_fence_process(&adev->sdma.instance[1].ring);
+
+	return 0;
+}
+
+static int si_dma_process_illegal_inst_irq(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *source,
+					      struct amdgpu_iv_entry *entry)
+{
+	DRM_ERROR("Illegal instruction in SDMA command stream\n");
+	schedule_work(&adev->reset_work);
+	return 0;
+}
+
+static int si_dma_set_clockgating_state(void *handle,
+					  enum amd_clockgating_state state)
+{
+	u32 orig, data, offset;
+	int i;
+	bool enable;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	enable = (state == AMD_CG_STATE_GATE) ? true : false;
+
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
+		for (i = 0; i < adev->sdma.num_instances; i++) {
+			if (i == 0)
+				offset = DMA0_REGISTER_OFFSET;
+			else
+				offset = DMA1_REGISTER_OFFSET;
+			orig = data = RREG32(DMA_POWER_CNTL + offset);
+			data &= ~MEM_POWER_OVERRIDE;
+			if (data != orig)
+				WREG32(DMA_POWER_CNTL + offset, data);
+			WREG32(DMA_CLK_CTRL + offset, 0x00000100);
+		}
+	} else {
+		for (i = 0; i < adev->sdma.num_instances; i++) {
+			if (i == 0)
+				offset = DMA0_REGISTER_OFFSET;
+			else
+				offset = DMA1_REGISTER_OFFSET;
+			orig = data = RREG32(DMA_POWER_CNTL + offset);
+			data |= MEM_POWER_OVERRIDE;
+			if (data != orig)
+				WREG32(DMA_POWER_CNTL + offset, data);
+
+			orig = data = RREG32(DMA_CLK_CTRL + offset);
+			data = 0xff000000;
+			if (data != orig)
+				WREG32(DMA_CLK_CTRL + offset, data);
+		}
+	}
+
+	return 0;
+}
+
+static int si_dma_set_powergating_state(void *handle,
+					  enum amd_powergating_state state)
+{
+	u32 tmp;
+
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	WREG32(DMA_PGFSM_WRITE,  0x00002000);
+	WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
+
+	for (tmp = 0; tmp < 5; tmp++)
+		WREG32(DMA_PGFSM_WRITE, 0);
+
+	return 0;
+}
+
+const struct amd_ip_funcs si_dma_ip_funcs = {
+	.name = "si_dma",
+	.early_init = si_dma_early_init,
+	.late_init = NULL,
+	.sw_init = si_dma_sw_init,
+	.sw_fini = si_dma_sw_fini,
+	.hw_init = si_dma_hw_init,
+	.hw_fini = si_dma_hw_fini,
+	.suspend = si_dma_suspend,
+	.resume = si_dma_resume,
+	.is_idle = si_dma_is_idle,
+	.wait_for_idle = si_dma_wait_for_idle,
+	.soft_reset = si_dma_soft_reset,
+	.set_clockgating_state = si_dma_set_clockgating_state,
+	.set_powergating_state = si_dma_set_powergating_state,
+};
+
+static const struct amdgpu_ring_funcs si_dma_ring_funcs = {
+	.get_rptr = si_dma_ring_get_rptr,
+	.get_wptr = si_dma_ring_get_wptr,
+	.set_wptr = si_dma_ring_set_wptr,
+	.parse_cs = NULL,
+	.emit_ib = si_dma_ring_emit_ib,
+	.emit_fence = si_dma_ring_emit_fence,
+	.emit_pipeline_sync = si_dma_ring_emit_pipeline_sync,
+	.emit_vm_flush = si_dma_ring_emit_vm_flush,
+	.emit_hdp_flush = si_dma_ring_emit_hdp_flush,
+	.emit_hdp_invalidate = si_dma_ring_emit_hdp_invalidate,
+	.test_ring = si_dma_ring_test_ring,
+	.test_ib = si_dma_ring_test_ib,
+	.insert_nop = amdgpu_ring_insert_nop,
+	.pad_ib = si_dma_ring_pad_ib,
+	.get_emit_ib_size = si_dma_ring_get_emit_ib_size,
+	.get_dma_frame_size = si_dma_ring_get_dma_frame_size,
+};
+
+static void si_dma_set_ring_funcs(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++)
+		adev->sdma.instance[i].ring.funcs = &si_dma_ring_funcs;
+}
+
+static const struct amdgpu_irq_src_funcs si_dma_trap_irq_funcs = {
+	.set = si_dma_set_trap_irq_state,
+	.process = si_dma_process_trap_irq,
+};
+
+static const struct amdgpu_irq_src_funcs si_dma_trap_irq_funcs_1 = {
+	.set = si_dma_set_trap_irq_state,
+	.process = si_dma_process_trap_irq_1,
+};
+
+static const struct amdgpu_irq_src_funcs si_dma_illegal_inst_irq_funcs = {
+	.process = si_dma_process_illegal_inst_irq,
+};
+
+static void si_dma_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
+	adev->sdma.trap_irq.funcs = &si_dma_trap_irq_funcs;
+	adev->sdma.trap_irq_1.funcs = &si_dma_trap_irq_funcs_1;
+	adev->sdma.illegal_inst_irq.funcs = &si_dma_illegal_inst_irq_funcs;
+}
+
+/**
+ * si_dma_emit_copy_buffer - copy buffer using the sDMA engine
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @src_offset: src GPU address
+ * @dst_offset: dst GPU address
+ * @byte_count: number of bytes to xfer
+ *
+ * Copy GPU buffers using the DMA engine (VI).
+ * Used by the amdgpu ttm implementation to move pages if
+ * registered as the asic copy callback.
+ */
+static void si_dma_emit_copy_buffer(struct amdgpu_ib *ib,
+				       uint64_t src_offset,
+				       uint64_t dst_offset,
+				       uint32_t byte_count)
+{
+	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY,
+					      1, 0, 0, byte_count);
+	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
+	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset) & 0xff;
+	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset) & 0xff;
+}
+
+/**
+ * si_dma_emit_fill_buffer - fill buffer using the sDMA engine
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @src_data: value to write to buffer
+ * @dst_offset: dst GPU address
+ * @byte_count: number of bytes to xfer
+ *
+ * Fill GPU buffers using the DMA engine (VI).
+ */
+static void si_dma_emit_fill_buffer(struct amdgpu_ib *ib,
+				       uint32_t src_data,
+				       uint64_t dst_offset,
+				       uint32_t byte_count)
+{
+	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_CONSTANT_FILL,
+					      0, 0, 0, byte_count / 4);
+	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+	ib->ptr[ib->length_dw++] = src_data;
+	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset) << 16;
+}
+
+
+static const struct amdgpu_buffer_funcs si_dma_buffer_funcs = {
+	.copy_max_bytes = 0xffff8,
+	.copy_num_dw = 5,
+	.emit_copy_buffer = si_dma_emit_copy_buffer,
+
+	.fill_max_bytes = 0xffff8,
+	.fill_num_dw = 4,
+	.emit_fill_buffer = si_dma_emit_fill_buffer,
+};
+
+static void si_dma_set_buffer_funcs(struct amdgpu_device *adev)
+{
+	if (adev->mman.buffer_funcs == NULL) {
+		adev->mman.buffer_funcs = &si_dma_buffer_funcs;
+		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+	}
+}
+
+static const struct amdgpu_vm_pte_funcs si_dma_vm_pte_funcs = {
+	.copy_pte = si_dma_vm_copy_pte,
+	.write_pte = si_dma_vm_write_pte,
+	.set_pte_pde = si_dma_vm_set_pte_pde,
+};
+
+static void si_dma_set_vm_pte_funcs(struct amdgpu_device *adev)
+{
+	unsigned i;
+
+	if (adev->vm_manager.vm_pte_funcs == NULL) {
+		adev->vm_manager.vm_pte_funcs = &si_dma_vm_pte_funcs;
+		for (i = 0; i < adev->sdma.num_instances; i++)
+			adev->vm_manager.vm_pte_rings[i] =
+				&adev->sdma.instance[i].ring;
+
+		adev->vm_manager.vm_pte_num_rings = adev->sdma.num_instances;
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.h b/drivers/gpu/drm/amd/amdgpu/si_dma.h
new file mode 100644
index 000000000000..3a3e0c78a54b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __SI_DMA_H__
+#define __SI_DMA_H__
+
+extern const struct amd_ip_funcs si_dma_ip_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
new file mode 100644
index 000000000000..8bd08925b370
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -0,0 +1,8006 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "drmP.h"
+#include "amdgpu.h"
+#include "amdgpu_pm.h"
+#include "amdgpu_dpm.h"
+#include "amdgpu_atombios.h"
+#include "si/sid.h"
+#include "r600_dpm.h"
+#include "si_dpm.h"
+#include "atom.h"
+#include "../include/pptable.h"
+#include <linux/math64.h>
+#include <linux/seq_file.h>
+#include <linux/firmware.h>
+
+#define MC_CG_ARB_FREQ_F0           0x0a
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+#define SMC_RAM_END                 0x20000
+
+#define SCLK_MIN_DEEPSLEEP_FREQ     1350
+
+
+/* sizeof(ATOM_PPLIB_EXTENDEDHEADER) */
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2 12
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3 14
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4 16
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V5 18
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6 20
+#define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7 22
+
+#define BIOS_SCRATCH_4                                    0x5cd
+
+MODULE_FIRMWARE("radeon/tahiti_smc.bin");
+MODULE_FIRMWARE("radeon/tahiti_k_smc.bin");
+MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
+MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin");
+MODULE_FIRMWARE("radeon/verde_smc.bin");
+MODULE_FIRMWARE("radeon/verde_k_smc.bin");
+MODULE_FIRMWARE("radeon/oland_smc.bin");
+MODULE_FIRMWARE("radeon/oland_k_smc.bin");
+MODULE_FIRMWARE("radeon/hainan_smc.bin");
+MODULE_FIRMWARE("radeon/hainan_k_smc.bin");
+
+union power_info {
+	struct _ATOM_POWERPLAY_INFO info;
+	struct _ATOM_POWERPLAY_INFO_V2 info_2;
+	struct _ATOM_POWERPLAY_INFO_V3 info_3;
+	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
+	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
+	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
+	struct _ATOM_PPLIB_POWERPLAYTABLE4 pplib4;
+	struct _ATOM_PPLIB_POWERPLAYTABLE5 pplib5;
+};
+
+union fan_info {
+	struct _ATOM_PPLIB_FANTABLE fan;
+	struct _ATOM_PPLIB_FANTABLE2 fan2;
+	struct _ATOM_PPLIB_FANTABLE3 fan3;
+};
+
+union pplib_clock_info {
+	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
+	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
+	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
+	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
+	struct _ATOM_PPLIB_SI_CLOCK_INFO si;
+};
+
+static const u32 r600_utc[R600_PM_NUMBER_OF_TC] =
+{
+	R600_UTC_DFLT_00,
+	R600_UTC_DFLT_01,
+	R600_UTC_DFLT_02,
+	R600_UTC_DFLT_03,
+	R600_UTC_DFLT_04,
+	R600_UTC_DFLT_05,
+	R600_UTC_DFLT_06,
+	R600_UTC_DFLT_07,
+	R600_UTC_DFLT_08,
+	R600_UTC_DFLT_09,
+	R600_UTC_DFLT_10,
+	R600_UTC_DFLT_11,
+	R600_UTC_DFLT_12,
+	R600_UTC_DFLT_13,
+	R600_UTC_DFLT_14,
+};
+
+static const u32 r600_dtc[R600_PM_NUMBER_OF_TC] =
+{
+	R600_DTC_DFLT_00,
+	R600_DTC_DFLT_01,
+	R600_DTC_DFLT_02,
+	R600_DTC_DFLT_03,
+	R600_DTC_DFLT_04,
+	R600_DTC_DFLT_05,
+	R600_DTC_DFLT_06,
+	R600_DTC_DFLT_07,
+	R600_DTC_DFLT_08,
+	R600_DTC_DFLT_09,
+	R600_DTC_DFLT_10,
+	R600_DTC_DFLT_11,
+	R600_DTC_DFLT_12,
+	R600_DTC_DFLT_13,
+	R600_DTC_DFLT_14,
+};
+
+static const struct si_cac_config_reg cac_weights_tahiti[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0xc, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0x101, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0xc, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x8fc, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x95, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x34e, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x1a1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0xda, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x46, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x208, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0xe7, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x948, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x167, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x31, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0x18e, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_tahiti[] =
+{
+	{ 0x143, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x146, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x146, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x149, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x149, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14c, 0x0001fffe, 1, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9e, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x101, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x101, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x107, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x107, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10a, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8f, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x92, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x92, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x95, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x95, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x152, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x152, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x155, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x155, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x158, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x158, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x113, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x113, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x116, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x116, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x119, 0x0001fffe, 1, 0x8, SISLANDS_CACCONFIG_CGIND },
+	{ 0x119, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x122, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x122, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x125, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x125, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x128, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x128, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+
+};
+
+static const struct si_cac_config_reg cac_override_tahiti[] =
+{
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_tahiti =
+{
+	((1 << 16) | 27027),
+	6,
+	0,
+	4,
+	95,
+	{
+		0UL,
+		0UL,
+		4521550UL,
+		309631529UL,
+		-1270850L,
+		4513710L,
+		40
+	},
+	595000000UL,
+	12,
+	{
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0
+	},
+	true
+};
+
+static const struct si_dte_data dte_data_tahiti =
+{
+	{ 1159409, 0, 0, 0, 0 },
+	{ 777, 0, 0, 0, 0 },
+	2,
+	54000,
+	127000,
+	25,
+	2,
+	10,
+	13,
+	{ 27, 31, 35, 39, 43, 47, 54, 61, 67, 74, 81, 88, 95, 0, 0, 0 },
+	{ 240888759, 221057860, 235370597, 162287531, 158510299, 131423027, 116673180, 103067515, 87941937, 76209048, 68209175, 64090048, 58301890, 0, 0, 0 },
+	{ 12024, 11189, 11451, 8411, 7939, 6666, 5681, 4905, 4241, 3720, 3354, 3122, 2890, 0, 0, 0 },
+	85,
+	false
+};
+
+#if 0
+static const struct si_dte_data dte_data_tahiti_le =
+{
+	{ 0x1E8480, 0x7A1200, 0x2160EC0, 0x3938700, 0 },
+	{ 0x7D, 0x7D, 0x4E4, 0xB00, 0 },
+	0x5,
+	0xAFC8,
+	0x64,
+	0x32,
+	1,
+	0,
+	0x10,
+	{ 0x78, 0x7C, 0x82, 0x88, 0x8E, 0x94, 0x9A, 0xA0, 0xA6, 0xAC, 0xB0, 0xB4, 0xB8, 0xBC, 0xC0, 0xC4 },
+	{ 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700, 0x3938700 },
+	{ 0x2AF8, 0x2AF8, 0x29BB, 0x27F9, 0x2637, 0x2475, 0x22B3, 0x20F1, 0x1F2F, 0x1D6D, 0x1734, 0x1414, 0x10F4, 0xDD4, 0xAB4, 0x794 },
+	85,
+	true
+};
+#endif
+
+static const struct si_dte_data dte_data_tahiti_pro =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x0, 0x0, 0x0, 0x0, 0x0 },
+	5,
+	45000,
+	100,
+	0xA,
+	1,
+	0,
+	0x10,
+	{ 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+	{ 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+	{ 0x7D0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_dte_data dte_data_new_zealand =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0 },
+	{ 0x29B, 0x3E9, 0x537, 0x7D2, 0 },
+	0x5,
+	0xAFC8,
+	0x69,
+	0x32,
+	1,
+	0,
+	0x10,
+	{ 0x82, 0xA0, 0xB4, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE },
+	{ 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+	{ 0xDAC, 0x1388, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685, 0x685 },
+	85,
+	true
+};
+
+static const struct si_dte_data dte_data_aruba_pro =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x0, 0x0, 0x0, 0x0, 0x0 },
+	5,
+	45000,
+	100,
+	0xA,
+	1,
+	0,
+	0x10,
+	{ 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+	{ 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+	{ 0x1000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_dte_data dte_data_malta =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x0, 0x0, 0x0, 0x0, 0x0 },
+	5,
+	45000,
+	100,
+	0xA,
+	1,
+	0,
+	0x10,
+	{ 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+	{ 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+	{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_cac_config_reg cac_weights_pitcairn[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x8a, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x24d, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x19, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0xc11, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0x7f3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x403, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x367, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x4c9, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x45d, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0x36d, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x534, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x5da, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x880, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0x201, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0x9f, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x1f, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x5de, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x7b, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x13, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0xf9, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x66, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x13, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0x186, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_pitcairn[] =
+{
+	{ 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x143, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x107, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x107, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x113, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x113, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x152, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x152, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8f, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x146, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x146, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9e, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10a, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x116, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x116, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x155, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x155, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x92, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x92, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x149, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x149, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x101, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x101, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x119, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x119, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x158, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x158, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x95, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x95, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x122, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x122, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x125, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x125, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x128, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x128, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_override_pitcairn[] =
+{
+    { 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_pitcairn =
+{
+	((1 << 16) | 27027),
+	5,
+	0,
+	6,
+	100,
+	{
+		51600000UL,
+		1800000UL,
+		7194395UL,
+		309631529UL,
+		-1270850L,
+		4513710L,
+		100
+	},
+	117830498UL,
+	12,
+	{
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0
+	},
+	true
+};
+
+static const struct si_dte_data dte_data_pitcairn =
+{
+	{ 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0 },
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	0,
+	false
+};
+
+static const struct si_dte_data dte_data_curacao_xt =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x0, 0x0, 0x0, 0x0, 0x0 },
+	5,
+	45000,
+	100,
+	0xA,
+	1,
+	0,
+	0x10,
+	{ 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+	{ 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+	{ 0x1D17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_dte_data dte_data_curacao_pro =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x0, 0x0, 0x0, 0x0, 0x0 },
+	5,
+	45000,
+	100,
+	0xA,
+	1,
+	0,
+	0x10,
+	{ 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+	{ 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+	{ 0x1D17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_dte_data dte_data_neptune_xt =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x0, 0x0, 0x0, 0x0, 0x0 },
+	5,
+	45000,
+	100,
+	0xA,
+	1,
+	0,
+	0x10,
+	{ 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+	{ 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+	{ 0x3A2F, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_cac_config_reg cac_weights_chelsea_pro[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x2BD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_chelsea_xt[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x30A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_heathrow[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x362, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_cape_verde_pro[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x315, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_cape_verde[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x3BA, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_cape_verde[] =
+{
+	{ 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x143, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x107, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x107, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x113, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x113, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x152, 0x0001fffe, 1, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x152, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x146, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x146, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_override_cape_verde[] =
+{
+    { 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_cape_verde =
+{
+	((1 << 16) | 0x6993),
+	5,
+	0,
+	7,
+	105,
+	{
+		0UL,
+		0UL,
+		7194395UL,
+		309631529UL,
+		-1270850L,
+		4513710L,
+		100
+	},
+	117830498UL,
+	12,
+	{
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0
+	},
+	true
+};
+
+static const struct si_dte_data dte_data_cape_verde =
+{
+	{ 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0 },
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	0,
+	false
+};
+
+static const struct si_dte_data dte_data_venus_xtx =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x71C, 0xAAB, 0xE39, 0x11C7, 0x0 },
+	5,
+	55000,
+	0x69,
+	0xA,
+	1,
+	0,
+	0x3,
+	{ 0x96, 0xB4, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	{ 0x895440, 0x3D0900, 0x989680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	{ 0xD6D8, 0x88B8, 0x1555, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_dte_data dte_data_venus_xt =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0xBDA, 0x11C7, 0x17B4, 0x1DA1, 0x0 },
+	5,
+	55000,
+	0x69,
+	0xA,
+	1,
+	0,
+	0x3,
+	{ 0x96, 0xB4, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	{ 0x895440, 0x3D0900, 0x989680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	{ 0xAFC8, 0x88B8, 0x238E, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_dte_data dte_data_venus_pro =
+{
+	{  0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x11C7, 0x1AAB, 0x238E, 0x2C72, 0x0 },
+	5,
+	55000,
+	0x69,
+	0xA,
+	1,
+	0,
+	0x3,
+	{ 0x96, 0xB4, 0xFF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	{ 0x895440, 0x3D0900, 0x989680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	{ 0x88B8, 0x88B8, 0x3555, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_cac_config_reg cac_weights_oland[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x82, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0x153, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x52, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x4F, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0x135, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0xAC, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x118, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0xBE, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x110, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x4CD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x37, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0x27, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0xC3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x35, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0x28, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x26C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3B2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x99D, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA3F, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0xA, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x34, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x3BA, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x30, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0x100, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_mars_pro[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_mars_xt[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x60, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_oland_pro[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x90, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_weights_oland_xt[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x43, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0xAF, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x2A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x29, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0xA0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0x59, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x1A5, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0x1D6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0x2A3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x8FD, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x76, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x8A, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0xA3, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0x71, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0x36, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0xA6, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x81, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0x3D2, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0x27C, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xA96, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0x5, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0xB, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x15, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x36, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x10, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x10, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x120, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x32, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x7E, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0x280, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0x7, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0x3C, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0x203, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0xB4, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_oland[] =
+{
+	{ 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x143, 0x0001fffe, 1, 0x4, SISLANDS_CACCONFIG_CGIND },
+	{ 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg lcac_mars_pro[] =
+{
+	{ 0x98, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x98, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x104, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x110, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x0001fffe, 1, 0x6, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x143, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x143, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x164, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x167, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16a, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15e, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x161, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15b, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x0001fffe, 1, 0x2, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16d, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x170, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x173, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x176, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x179, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17c, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x0001fffe, 1, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17f, 0x00000001, 0, 0x1, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_cac_config_reg cac_override_oland[] =
+{
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_oland =
+{
+	((1 << 16) | 0x6993),
+	5,
+	0,
+	7,
+	105,
+	{
+		0UL,
+		0UL,
+		7194395UL,
+		309631529UL,
+		-1270850L,
+		4513710L,
+		100
+	},
+	117830498UL,
+	12,
+	{
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0
+	},
+	true
+};
+
+static const struct si_powertune_data powertune_data_mars_pro =
+{
+	((1 << 16) | 0x6993),
+	5,
+	0,
+	7,
+	105,
+	{
+		0UL,
+		0UL,
+		7194395UL,
+		309631529UL,
+		-1270850L,
+		4513710L,
+		100
+	},
+	117830498UL,
+	12,
+	{
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0
+	},
+	true
+};
+
+static const struct si_dte_data dte_data_oland =
+{
+	{ 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0 },
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+	0,
+	false
+};
+
+static const struct si_dte_data dte_data_mars_pro =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x0, 0x0, 0x0, 0x0, 0x0 },
+	5,
+	55000,
+	105,
+	0xA,
+	1,
+	0,
+	0x10,
+	{ 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+	{ 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+	{ 0xF627, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+static const struct si_dte_data dte_data_sun_xt =
+{
+	{ 0x1E8480, 0x3D0900, 0x989680, 0x2625A00, 0x0 },
+	{ 0x0, 0x0, 0x0, 0x0, 0x0 },
+	5,
+	55000,
+	105,
+	0xA,
+	1,
+	0,
+	0x10,
+	{ 0x96, 0xB4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF },
+	{ 0x895440, 0x3D0900, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680, 0x989680 },
+	{ 0xD555, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
+	90,
+	true
+};
+
+
+static const struct si_cac_config_reg cac_weights_hainan[] =
+{
+	{ 0x0, 0x0000ffff, 0, 0x2d9, SISLANDS_CACCONFIG_CGIND },
+	{ 0x0, 0xffff0000, 16, 0x22b, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0x0000ffff, 0, 0x21c, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1, 0xffff0000, 16, 0x1dc, SISLANDS_CACCONFIG_CGIND },
+	{ 0x2, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0x0000ffff, 0, 0x24e, SISLANDS_CACCONFIG_CGIND },
+	{ 0x3, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0x0000ffff, 0, 0x35e, SISLANDS_CACCONFIG_CGIND },
+	{ 0x5, 0xffff0000, 16, 0x1143, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0x0000ffff, 0, 0xe17, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6, 0xffff0000, 16, 0x441, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18f, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0x0000ffff, 0, 0x28b, SISLANDS_CACCONFIG_CGIND },
+	{ 0x7, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x8, 0xffff0000, 16, 0xabe, SISLANDS_CACCONFIG_CGIND },
+	{ 0x9, 0x0000ffff, 0, 0xf11, SISLANDS_CACCONFIG_CGIND },
+	{ 0xa, 0x0000ffff, 0, 0x907, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0x0000ffff, 0, 0xb45, SISLANDS_CACCONFIG_CGIND },
+	{ 0xb, 0xffff0000, 16, 0xd1e, SISLANDS_CACCONFIG_CGIND },
+	{ 0xc, 0x0000ffff, 0, 0xa2c, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0x0000ffff, 0, 0x62, SISLANDS_CACCONFIG_CGIND },
+	{ 0xd, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0xe, 0x0000ffff, 0, 0x1f3, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0x0000ffff, 0, 0x42, SISLANDS_CACCONFIG_CGIND },
+	{ 0xf, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x10, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0x0000ffff, 0, 0x709, SISLANDS_CACCONFIG_CGIND },
+	{ 0x11, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x12, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x13, 0xffff0000, 16, 0x3a, SISLANDS_CACCONFIG_CGIND },
+	{ 0x14, 0x0000ffff, 0, 0x357, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0x0000ffff, 0, 0x9f, SISLANDS_CACCONFIG_CGIND },
+	{ 0x15, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x4e, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0x0000ffff, 0, 0x314, SISLANDS_CACCONFIG_CGIND },
+	{ 0x16, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x17, 0x0000ffff, 0, 0x6d, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x18, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0x0000ffff, 0, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x19, 0xffff0000, 16, 0x0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1a, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1b, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1c, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1d, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1e, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x1f, 0xffff0000, 16, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x20, 0x0000ffff, 0, 0, SISLANDS_CACCONFIG_CGIND },
+	{ 0x6d, 0x0000ffff, 0, 0x1b9, SISLANDS_CACCONFIG_CGIND },
+	{ 0xFFFFFFFF }
+};
+
+static const struct si_powertune_data powertune_data_hainan =
+{
+	((1 << 16) | 0x6993),
+	5,
+	0,
+	9,
+	105,
+	{
+		0UL,
+		0UL,
+		7194395UL,
+		309631529UL,
+		-1270850L,
+		4513710L,
+		100
+	},
+	117830498UL,
+	12,
+	{
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0
+	},
+	true
+};
+
+static struct rv7xx_power_info *rv770_get_pi(struct amdgpu_device *adev);
+static struct evergreen_power_info *evergreen_get_pi(struct amdgpu_device *adev);
+static struct ni_power_info *ni_get_pi(struct amdgpu_device *adev);
+static struct  si_ps *si_get_ps(struct amdgpu_ps *rps);
+
+static int si_populate_voltage_value(struct amdgpu_device *adev,
+				     const struct atom_voltage_table *table,
+				     u16 value, SISLANDS_SMC_VOLTAGE_VALUE *voltage);
+static int si_get_std_voltage_value(struct amdgpu_device *adev,
+				    SISLANDS_SMC_VOLTAGE_VALUE *voltage,
+				    u16 *std_voltage);
+static int si_write_smc_soft_register(struct amdgpu_device *adev,
+				      u16 reg_offset, u32 value);
+static int si_convert_power_level_to_smc(struct amdgpu_device *adev,
+					 struct rv7xx_pl *pl,
+					 SISLANDS_SMC_HW_PERFORMANCE_LEVEL *level);
+static int si_calculate_sclk_params(struct amdgpu_device *adev,
+				    u32 engine_clock,
+				    SISLANDS_SMC_SCLK_VALUE *sclk);
+
+static void si_thermal_start_smc_fan_control(struct amdgpu_device *adev);
+static void si_fan_ctrl_set_default_mode(struct amdgpu_device *adev);
+static void si_dpm_set_dpm_funcs(struct amdgpu_device *adev);
+static void si_dpm_set_irq_funcs(struct amdgpu_device *adev);
+
+static struct si_power_info *si_get_pi(struct amdgpu_device *adev)
+{
+	struct si_power_info *pi = adev->pm.dpm.priv;
+	return pi;
+}
+
+static void si_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
+						     u16 v, s32 t, u32 ileakage, u32 *leakage)
+{
+	s64 kt, kv, leakage_w, i_leakage, vddc;
+	s64 temperature, t_slope, t_intercept, av, bv, t_ref;
+	s64 tmp;
+
+	i_leakage = div64_s64(drm_int2fixp(ileakage), 100);
+	vddc = div64_s64(drm_int2fixp(v), 1000);
+	temperature = div64_s64(drm_int2fixp(t), 1000);
+
+	t_slope = div64_s64(drm_int2fixp(coeff->t_slope), 100000000);
+	t_intercept = div64_s64(drm_int2fixp(coeff->t_intercept), 100000000);
+	av = div64_s64(drm_int2fixp(coeff->av), 100000000);
+	bv = div64_s64(drm_int2fixp(coeff->bv), 100000000);
+	t_ref = drm_int2fixp(coeff->t_ref);
+
+	tmp = drm_fixp_mul(t_slope, vddc) + t_intercept;
+	kt = drm_fixp_exp(drm_fixp_mul(tmp, temperature));
+	kt = drm_fixp_div(kt, drm_fixp_exp(drm_fixp_mul(tmp, t_ref)));
+	kv = drm_fixp_mul(av, drm_fixp_exp(drm_fixp_mul(bv, vddc)));
+
+	leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
+
+	*leakage = drm_fixp2int(leakage_w * 1000);
+}
+
+static void si_calculate_leakage_for_v_and_t(struct amdgpu_device *adev,
+					     const struct ni_leakage_coeffients *coeff,
+					     u16 v,
+					     s32 t,
+					     u32 i_leakage,
+					     u32 *leakage)
+{
+	si_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
+}
+
+static void si_calculate_leakage_for_v_formula(const struct ni_leakage_coeffients *coeff,
+					       const u32 fixed_kt, u16 v,
+					       u32 ileakage, u32 *leakage)
+{
+	s64 kt, kv, leakage_w, i_leakage, vddc;
+
+	i_leakage = div64_s64(drm_int2fixp(ileakage), 100);
+	vddc = div64_s64(drm_int2fixp(v), 1000);
+
+	kt = div64_s64(drm_int2fixp(fixed_kt), 100000000);
+	kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 100000000),
+			  drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 100000000), vddc)));
+
+	leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
+
+	*leakage = drm_fixp2int(leakage_w * 1000);
+}
+
+static void si_calculate_leakage_for_v(struct amdgpu_device *adev,
+				       const struct ni_leakage_coeffients *coeff,
+				       const u32 fixed_kt,
+				       u16 v,
+				       u32 i_leakage,
+				       u32 *leakage)
+{
+	si_calculate_leakage_for_v_formula(coeff, fixed_kt, v, i_leakage, leakage);
+}
+
+
+static void si_update_dte_from_pl2(struct amdgpu_device *adev,
+				   struct si_dte_data *dte_data)
+{
+	u32 p_limit1 = adev->pm.dpm.tdp_limit;
+	u32 p_limit2 = adev->pm.dpm.near_tdp_limit;
+	u32 k = dte_data->k;
+	u32 t_max = dte_data->max_t;
+	u32 t_split[5] = { 10, 15, 20, 25, 30 };
+	u32 t_0 = dte_data->t0;
+	u32 i;
+
+	if (p_limit2 != 0 && p_limit2 <= p_limit1) {
+		dte_data->tdep_count = 3;
+
+		for (i = 0; i < k; i++) {
+			dte_data->r[i] =
+				(t_split[i] * (t_max - t_0/(u32)1000) * (1 << 14)) /
+				(p_limit2  * (u32)100);
+		}
+
+		dte_data->tdep_r[1] = dte_data->r[4] * 2;
+
+		for (i = 2; i < SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE; i++) {
+			dte_data->tdep_r[i] = dte_data->r[4];
+		}
+	} else {
+		DRM_ERROR("Invalid PL2! DTE will not be updated.\n");
+	}
+}
+
+static struct rv7xx_power_info *rv770_get_pi(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi = adev->pm.dpm.priv;
+
+	return pi;
+}
+
+static struct ni_power_info *ni_get_pi(struct amdgpu_device *adev)
+{
+	struct ni_power_info *pi = adev->pm.dpm.priv;
+
+	return pi;
+}
+
+static struct si_ps *si_get_ps(struct amdgpu_ps *aps)
+{
+	struct  si_ps *ps = aps->ps_priv;
+
+	return ps;
+}
+
+static void si_initialize_powertune_defaults(struct amdgpu_device *adev)
+{
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	bool update_dte_from_pl2 = false;
+
+	if (adev->asic_type == CHIP_TAHITI) {
+		si_pi->cac_weights = cac_weights_tahiti;
+		si_pi->lcac_config = lcac_tahiti;
+		si_pi->cac_override = cac_override_tahiti;
+		si_pi->powertune_data = &powertune_data_tahiti;
+		si_pi->dte_data = dte_data_tahiti;
+
+		switch (adev->pdev->device) {
+		case 0x6798:
+			si_pi->dte_data.enable_dte_by_default = true;
+			break;
+		case 0x6799:
+			si_pi->dte_data = dte_data_new_zealand;
+			break;
+		case 0x6790:
+		case 0x6791:
+		case 0x6792:
+		case 0x679E:
+			si_pi->dte_data = dte_data_aruba_pro;
+			update_dte_from_pl2 = true;
+			break;
+		case 0x679B:
+			si_pi->dte_data = dte_data_malta;
+			update_dte_from_pl2 = true;
+			break;
+		case 0x679A:
+			si_pi->dte_data = dte_data_tahiti_pro;
+			update_dte_from_pl2 = true;
+			break;
+		default:
+			if (si_pi->dte_data.enable_dte_by_default == true)
+				DRM_ERROR("DTE is not enabled!\n");
+			break;
+		}
+	} else if (adev->asic_type == CHIP_PITCAIRN) {
+		si_pi->cac_weights = cac_weights_pitcairn;
+		si_pi->lcac_config = lcac_pitcairn;
+		si_pi->cac_override = cac_override_pitcairn;
+		si_pi->powertune_data = &powertune_data_pitcairn;
+
+		switch (adev->pdev->device) {
+		case 0x6810:
+		case 0x6818:
+			si_pi->dte_data = dte_data_curacao_xt;
+			update_dte_from_pl2 = true;
+			break;
+		case 0x6819:
+		case 0x6811:
+			si_pi->dte_data = dte_data_curacao_pro;
+			update_dte_from_pl2 = true;
+			break;
+		case 0x6800:
+		case 0x6806:
+			si_pi->dte_data = dte_data_neptune_xt;
+			update_dte_from_pl2 = true;
+			break;
+		default:
+			si_pi->dte_data = dte_data_pitcairn;
+			break;
+		}
+	} else if (adev->asic_type == CHIP_VERDE) {
+		si_pi->lcac_config = lcac_cape_verde;
+		si_pi->cac_override = cac_override_cape_verde;
+		si_pi->powertune_data = &powertune_data_cape_verde;
+
+		switch (adev->pdev->device) {
+		case 0x683B:
+		case 0x683F:
+		case 0x6829:
+		case 0x6835:
+			si_pi->cac_weights = cac_weights_cape_verde_pro;
+			si_pi->dte_data = dte_data_cape_verde;
+			break;
+		case 0x682C:
+			si_pi->cac_weights = cac_weights_cape_verde_pro;
+			si_pi->dte_data = dte_data_sun_xt;
+			break;
+		case 0x6825:
+		case 0x6827:
+			si_pi->cac_weights = cac_weights_heathrow;
+			si_pi->dte_data = dte_data_cape_verde;
+			break;
+		case 0x6824:
+		case 0x682D:
+			si_pi->cac_weights = cac_weights_chelsea_xt;
+			si_pi->dte_data = dte_data_cape_verde;
+			break;
+		case 0x682F:
+			si_pi->cac_weights = cac_weights_chelsea_pro;
+			si_pi->dte_data = dte_data_cape_verde;
+			break;
+		case 0x6820:
+			si_pi->cac_weights = cac_weights_heathrow;
+			si_pi->dte_data = dte_data_venus_xtx;
+			break;
+		case 0x6821:
+			si_pi->cac_weights = cac_weights_heathrow;
+			si_pi->dte_data = dte_data_venus_xt;
+			break;
+		case 0x6823:
+		case 0x682B:
+		case 0x6822:
+		case 0x682A:
+			si_pi->cac_weights = cac_weights_chelsea_pro;
+			si_pi->dte_data = dte_data_venus_pro;
+			break;
+		default:
+			si_pi->cac_weights = cac_weights_cape_verde;
+			si_pi->dte_data = dte_data_cape_verde;
+			break;
+		}
+	} else if (adev->asic_type == CHIP_OLAND) {
+		si_pi->lcac_config = lcac_mars_pro;
+		si_pi->cac_override = cac_override_oland;
+		si_pi->powertune_data = &powertune_data_mars_pro;
+		si_pi->dte_data = dte_data_mars_pro;
+
+		switch (adev->pdev->device) {
+		case 0x6601:
+		case 0x6621:
+		case 0x6603:
+		case 0x6605:
+			si_pi->cac_weights = cac_weights_mars_pro;
+			update_dte_from_pl2 = true;
+			break;
+		case 0x6600:
+		case 0x6606:
+		case 0x6620:
+		case 0x6604:
+			si_pi->cac_weights = cac_weights_mars_xt;
+			update_dte_from_pl2 = true;
+			break;
+		case 0x6611:
+		case 0x6613:
+		case 0x6608:
+			si_pi->cac_weights = cac_weights_oland_pro;
+			update_dte_from_pl2 = true;
+			break;
+		case 0x6610:
+			si_pi->cac_weights = cac_weights_oland_xt;
+			update_dte_from_pl2 = true;
+			break;
+		default:
+			si_pi->cac_weights = cac_weights_oland;
+			si_pi->lcac_config = lcac_oland;
+			si_pi->cac_override = cac_override_oland;
+			si_pi->powertune_data = &powertune_data_oland;
+			si_pi->dte_data = dte_data_oland;
+			break;
+		}
+	} else if (adev->asic_type == CHIP_HAINAN) {
+		si_pi->cac_weights = cac_weights_hainan;
+		si_pi->lcac_config = lcac_oland;
+		si_pi->cac_override = cac_override_oland;
+		si_pi->powertune_data = &powertune_data_hainan;
+		si_pi->dte_data = dte_data_sun_xt;
+		update_dte_from_pl2 = true;
+	} else {
+		DRM_ERROR("Unknown SI asic revision, failed to initialize PowerTune!\n");
+		return;
+	}
+
+	ni_pi->enable_power_containment = false;
+	ni_pi->enable_cac = false;
+	ni_pi->enable_sq_ramping = false;
+	si_pi->enable_dte = false;
+
+	if (si_pi->powertune_data->enable_powertune_by_default) {
+		ni_pi->enable_power_containment = true;
+		ni_pi->enable_cac = true;
+		if (si_pi->dte_data.enable_dte_by_default) {
+			si_pi->enable_dte = true;
+			if (update_dte_from_pl2)
+				si_update_dte_from_pl2(adev, &si_pi->dte_data);
+
+		}
+		ni_pi->enable_sq_ramping = true;
+	}
+
+	ni_pi->driver_calculate_cac_leakage = true;
+	ni_pi->cac_configuration_required = true;
+
+	if (ni_pi->cac_configuration_required) {
+		ni_pi->support_cac_long_term_average = true;
+		si_pi->dyn_powertune_data.l2_lta_window_size =
+			si_pi->powertune_data->l2_lta_window_size_default;
+		si_pi->dyn_powertune_data.lts_truncate =
+			si_pi->powertune_data->lts_truncate_default;
+	} else {
+		ni_pi->support_cac_long_term_average = false;
+		si_pi->dyn_powertune_data.l2_lta_window_size = 0;
+		si_pi->dyn_powertune_data.lts_truncate = 0;
+	}
+
+	si_pi->dyn_powertune_data.disable_uvd_powertune = false;
+}
+
+static u32 si_get_smc_power_scaling_factor(struct amdgpu_device *adev)
+{
+	return 1;
+}
+
+static u32 si_calculate_cac_wintime(struct amdgpu_device *adev)
+{
+	u32 xclk;
+	u32 wintime;
+	u32 cac_window;
+	u32 cac_window_size;
+
+	xclk = amdgpu_asic_get_xclk(adev);
+
+	if (xclk == 0)
+		return 0;
+
+	cac_window = RREG32(CG_CAC_CTRL) & CAC_WINDOW_MASK;
+	cac_window_size = ((cac_window & 0xFFFF0000) >> 16) * (cac_window & 0x0000FFFF);
+
+	wintime = (cac_window_size * 100) / xclk;
+
+	return wintime;
+}
+
+static u32 si_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
+{
+	return power_in_watts;
+}
+
+static int si_calculate_adjusted_tdp_limits(struct amdgpu_device *adev,
+					    bool adjust_polarity,
+					    u32 tdp_adjustment,
+					    u32 *tdp_limit,
+					    u32 *near_tdp_limit)
+{
+	u32 adjustment_delta, max_tdp_limit;
+
+	if (tdp_adjustment > (u32)adev->pm.dpm.tdp_od_limit)
+		return -EINVAL;
+
+	max_tdp_limit = ((100 + 100) * adev->pm.dpm.tdp_limit) / 100;
+
+	if (adjust_polarity) {
+		*tdp_limit = ((100 + tdp_adjustment) * adev->pm.dpm.tdp_limit) / 100;
+		*near_tdp_limit = adev->pm.dpm.near_tdp_limit_adjusted + (*tdp_limit - adev->pm.dpm.tdp_limit);
+	} else {
+		*tdp_limit = ((100 - tdp_adjustment) * adev->pm.dpm.tdp_limit) / 100;
+		adjustment_delta  = adev->pm.dpm.tdp_limit - *tdp_limit;
+		if (adjustment_delta < adev->pm.dpm.near_tdp_limit_adjusted)
+			*near_tdp_limit = adev->pm.dpm.near_tdp_limit_adjusted - adjustment_delta;
+		else
+			*near_tdp_limit = 0;
+	}
+
+	if ((*tdp_limit <= 0) || (*tdp_limit > max_tdp_limit))
+		return -EINVAL;
+	if ((*near_tdp_limit <= 0) || (*near_tdp_limit > *tdp_limit))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int si_populate_smc_tdp_limits(struct amdgpu_device *adev,
+				      struct amdgpu_ps *amdgpu_state)
+{
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	if (ni_pi->enable_power_containment) {
+		SISLANDS_SMC_STATETABLE *smc_table = &si_pi->smc_statetable;
+		PP_SIslands_PAPMParameters *papm_parm;
+		struct amdgpu_ppm_table *ppm = adev->pm.dpm.dyn_state.ppm_table;
+		u32 scaling_factor = si_get_smc_power_scaling_factor(adev);
+		u32 tdp_limit;
+		u32 near_tdp_limit;
+		int ret;
+
+		if (scaling_factor == 0)
+			return -EINVAL;
+
+		memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE));
+
+		ret = si_calculate_adjusted_tdp_limits(adev,
+						       false, /* ??? */
+						       adev->pm.dpm.tdp_adjustment,
+						       &tdp_limit,
+						       &near_tdp_limit);
+		if (ret)
+			return ret;
+
+		smc_table->dpm2Params.TDPLimit =
+			cpu_to_be32(si_scale_power_for_smc(tdp_limit, scaling_factor) * 1000);
+		smc_table->dpm2Params.NearTDPLimit =
+			cpu_to_be32(si_scale_power_for_smc(near_tdp_limit, scaling_factor) * 1000);
+		smc_table->dpm2Params.SafePowerLimit =
+			cpu_to_be32(si_scale_power_for_smc((near_tdp_limit * SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100, scaling_factor) * 1000);
+
+		ret = amdgpu_si_copy_bytes_to_smc(adev,
+						  (si_pi->state_table_start + offsetof(SISLANDS_SMC_STATETABLE, dpm2Params) +
+						   offsetof(PP_SIslands_DPM2Parameters, TDPLimit)),
+						  (u8 *)(&(smc_table->dpm2Params.TDPLimit)),
+						  sizeof(u32) * 3,
+						  si_pi->sram_end);
+		if (ret)
+			return ret;
+
+		if (si_pi->enable_ppm) {
+			papm_parm = &si_pi->papm_parm;
+			memset(papm_parm, 0, sizeof(PP_SIslands_PAPMParameters));
+			papm_parm->NearTDPLimitTherm = cpu_to_be32(ppm->dgpu_tdp);
+			papm_parm->dGPU_T_Limit = cpu_to_be32(ppm->tj_max);
+			papm_parm->dGPU_T_Warning = cpu_to_be32(95);
+			papm_parm->dGPU_T_Hysteresis = cpu_to_be32(5);
+			papm_parm->PlatformPowerLimit = 0xffffffff;
+			papm_parm->NearTDPLimitPAPM = 0xffffffff;
+
+			ret = amdgpu_si_copy_bytes_to_smc(adev, si_pi->papm_cfg_table_start,
+							  (u8 *)papm_parm,
+							  sizeof(PP_SIslands_PAPMParameters),
+							  si_pi->sram_end);
+			if (ret)
+				return ret;
+		}
+	}
+	return 0;
+}
+
+static int si_populate_smc_tdp_limits_2(struct amdgpu_device *adev,
+					struct amdgpu_ps *amdgpu_state)
+{
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	if (ni_pi->enable_power_containment) {
+		SISLANDS_SMC_STATETABLE *smc_table = &si_pi->smc_statetable;
+		u32 scaling_factor = si_get_smc_power_scaling_factor(adev);
+		int ret;
+
+		memset(smc_table, 0, sizeof(SISLANDS_SMC_STATETABLE));
+
+		smc_table->dpm2Params.NearTDPLimit =
+			cpu_to_be32(si_scale_power_for_smc(adev->pm.dpm.near_tdp_limit_adjusted, scaling_factor) * 1000);
+		smc_table->dpm2Params.SafePowerLimit =
+			cpu_to_be32(si_scale_power_for_smc((adev->pm.dpm.near_tdp_limit_adjusted * SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100, scaling_factor) * 1000);
+
+		ret = amdgpu_si_copy_bytes_to_smc(adev,
+						  (si_pi->state_table_start +
+						   offsetof(SISLANDS_SMC_STATETABLE, dpm2Params) +
+						   offsetof(PP_SIslands_DPM2Parameters, NearTDPLimit)),
+						  (u8 *)(&(smc_table->dpm2Params.NearTDPLimit)),
+						  sizeof(u32) * 2,
+						  si_pi->sram_end);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static u16 si_calculate_power_efficiency_ratio(struct amdgpu_device *adev,
+					       const u16 prev_std_vddc,
+					       const u16 curr_std_vddc)
+{
+	u64 margin = (u64)SISLANDS_DPM2_PWREFFICIENCYRATIO_MARGIN;
+	u64 prev_vddc = (u64)prev_std_vddc;
+	u64 curr_vddc = (u64)curr_std_vddc;
+	u64 pwr_efficiency_ratio, n, d;
+
+	if ((prev_vddc == 0) || (curr_vddc == 0))
+		return 0;
+
+	n = div64_u64((u64)1024 * curr_vddc * curr_vddc * ((u64)1000 + margin), (u64)1000);
+	d = prev_vddc * prev_vddc;
+	pwr_efficiency_ratio = div64_u64(n, d);
+
+	if (pwr_efficiency_ratio > (u64)0xFFFF)
+		return 0;
+
+	return (u16)pwr_efficiency_ratio;
+}
+
+static bool si_should_disable_uvd_powertune(struct amdgpu_device *adev,
+					    struct amdgpu_ps *amdgpu_state)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	if (si_pi->dyn_powertune_data.disable_uvd_powertune &&
+	    amdgpu_state->vclk && amdgpu_state->dclk)
+		return true;
+
+	return false;
+}
+
+struct evergreen_power_info *evergreen_get_pi(struct amdgpu_device *adev)
+{
+	struct evergreen_power_info *pi = adev->pm.dpm.priv;
+
+	return pi;
+}
+
+static int si_populate_power_containment_values(struct amdgpu_device *adev,
+						struct amdgpu_ps *amdgpu_state,
+						SISLANDS_SMC_SWSTATE *smc_state)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct  si_ps *state = si_get_ps(amdgpu_state);
+	SISLANDS_SMC_VOLTAGE_VALUE vddc;
+	u32 prev_sclk;
+	u32 max_sclk;
+	u32 min_sclk;
+	u16 prev_std_vddc;
+	u16 curr_std_vddc;
+	int i;
+	u16 pwr_efficiency_ratio;
+	u8 max_ps_percent;
+	bool disable_uvd_power_tune;
+	int ret;
+
+	if (ni_pi->enable_power_containment == false)
+		return 0;
+
+	if (state->performance_level_count == 0)
+		return -EINVAL;
+
+	if (smc_state->levelCount != state->performance_level_count)
+		return -EINVAL;
+
+	disable_uvd_power_tune = si_should_disable_uvd_powertune(adev, amdgpu_state);
+
+	smc_state->levels[0].dpm2.MaxPS = 0;
+	smc_state->levels[0].dpm2.NearTDPDec = 0;
+	smc_state->levels[0].dpm2.AboveSafeInc = 0;
+	smc_state->levels[0].dpm2.BelowSafeInc = 0;
+	smc_state->levels[0].dpm2.PwrEfficiencyRatio = 0;
+
+	for (i = 1; i < state->performance_level_count; i++) {
+		prev_sclk = state->performance_levels[i-1].sclk;
+		max_sclk  = state->performance_levels[i].sclk;
+		if (i == 1)
+			max_ps_percent = SISLANDS_DPM2_MAXPS_PERCENT_M;
+		else
+			max_ps_percent = SISLANDS_DPM2_MAXPS_PERCENT_H;
+
+		if (prev_sclk > max_sclk)
+			return -EINVAL;
+
+		if ((max_ps_percent == 0) ||
+		    (prev_sclk == max_sclk) ||
+		    disable_uvd_power_tune)
+			min_sclk = max_sclk;
+		else if (i == 1)
+			min_sclk = prev_sclk;
+		else
+			min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
+
+		if (min_sclk < state->performance_levels[0].sclk)
+			min_sclk = state->performance_levels[0].sclk;
+
+		if (min_sclk == 0)
+			return -EINVAL;
+
+		ret = si_populate_voltage_value(adev, &eg_pi->vddc_voltage_table,
+						state->performance_levels[i-1].vddc, &vddc);
+		if (ret)
+			return ret;
+
+		ret = si_get_std_voltage_value(adev, &vddc, &prev_std_vddc);
+		if (ret)
+			return ret;
+
+		ret = si_populate_voltage_value(adev, &eg_pi->vddc_voltage_table,
+						state->performance_levels[i].vddc, &vddc);
+		if (ret)
+			return ret;
+
+		ret = si_get_std_voltage_value(adev, &vddc, &curr_std_vddc);
+		if (ret)
+			return ret;
+
+		pwr_efficiency_ratio = si_calculate_power_efficiency_ratio(adev,
+									   prev_std_vddc, curr_std_vddc);
+
+		smc_state->levels[i].dpm2.MaxPS = (u8)((SISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
+		smc_state->levels[i].dpm2.NearTDPDec = SISLANDS_DPM2_NEAR_TDP_DEC;
+		smc_state->levels[i].dpm2.AboveSafeInc = SISLANDS_DPM2_ABOVE_SAFE_INC;
+		smc_state->levels[i].dpm2.BelowSafeInc = SISLANDS_DPM2_BELOW_SAFE_INC;
+		smc_state->levels[i].dpm2.PwrEfficiencyRatio = cpu_to_be16(pwr_efficiency_ratio);
+	}
+
+	return 0;
+}
+
+static int si_populate_sq_ramping_values(struct amdgpu_device *adev,
+					 struct amdgpu_ps *amdgpu_state,
+					 SISLANDS_SMC_SWSTATE *smc_state)
+{
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct  si_ps *state = si_get_ps(amdgpu_state);
+	u32 sq_power_throttle, sq_power_throttle2;
+	bool enable_sq_ramping = ni_pi->enable_sq_ramping;
+	int i;
+
+	if (state->performance_level_count == 0)
+		return -EINVAL;
+
+	if (smc_state->levelCount != state->performance_level_count)
+		return -EINVAL;
+
+	if (adev->pm.dpm.sq_ramping_threshold == 0)
+		return -EINVAL;
+
+	if (SISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
+		enable_sq_ramping = false;
+
+	if (SISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
+		enable_sq_ramping = false;
+
+	if (SISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
+		enable_sq_ramping = false;
+
+	if (SISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
+		enable_sq_ramping = false;
+
+	if (SISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
+		enable_sq_ramping = false;
+
+	for (i = 0; i < state->performance_level_count; i++) {
+		sq_power_throttle = 0;
+		sq_power_throttle2 = 0;
+
+		if ((state->performance_levels[i].sclk >= adev->pm.dpm.sq_ramping_threshold) &&
+		    enable_sq_ramping) {
+			sq_power_throttle |= MAX_POWER(SISLANDS_DPM2_SQ_RAMP_MAX_POWER);
+			sq_power_throttle |= MIN_POWER(SISLANDS_DPM2_SQ_RAMP_MIN_POWER);
+			sq_power_throttle2 |= MAX_POWER_DELTA(SISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
+			sq_power_throttle2 |= STI_SIZE(SISLANDS_DPM2_SQ_RAMP_STI_SIZE);
+			sq_power_throttle2 |= LTI_RATIO(SISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
+		} else {
+			sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
+			sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
+		}
+
+		smc_state->levels[i].SQPowerThrottle = cpu_to_be32(sq_power_throttle);
+		smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
+	}
+
+	return 0;
+}
+
+static int si_enable_power_containment(struct amdgpu_device *adev,
+				       struct amdgpu_ps *amdgpu_new_state,
+				       bool enable)
+{
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	PPSMC_Result smc_result;
+	int ret = 0;
+
+	if (ni_pi->enable_power_containment) {
+		if (enable) {
+			if (!si_should_disable_uvd_powertune(adev, amdgpu_new_state)) {
+				smc_result = amdgpu_si_send_msg_to_smc(adev, PPSMC_TDPClampingActive);
+				if (smc_result != PPSMC_Result_OK) {
+					ret = -EINVAL;
+					ni_pi->pc_enabled = false;
+				} else {
+					ni_pi->pc_enabled = true;
+				}
+			}
+		} else {
+			smc_result = amdgpu_si_send_msg_to_smc(adev, PPSMC_TDPClampingInactive);
+			if (smc_result != PPSMC_Result_OK)
+				ret = -EINVAL;
+			ni_pi->pc_enabled = false;
+		}
+	}
+
+	return ret;
+}
+
+static int si_initialize_smc_dte_tables(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	int ret = 0;
+	struct si_dte_data *dte_data = &si_pi->dte_data;
+	Smc_SIslands_DTE_Configuration *dte_tables = NULL;
+	u32 table_size;
+	u8 tdep_count;
+	u32 i;
+
+	if (dte_data == NULL)
+		si_pi->enable_dte = false;
+
+	if (si_pi->enable_dte == false)
+		return 0;
+
+	if (dte_data->k <= 0)
+		return -EINVAL;
+
+	dte_tables = kzalloc(sizeof(Smc_SIslands_DTE_Configuration), GFP_KERNEL);
+	if (dte_tables == NULL) {
+		si_pi->enable_dte = false;
+		return -ENOMEM;
+	}
+
+	table_size = dte_data->k;
+
+	if (table_size > SMC_SISLANDS_DTE_MAX_FILTER_STAGES)
+		table_size = SMC_SISLANDS_DTE_MAX_FILTER_STAGES;
+
+	tdep_count = dte_data->tdep_count;
+	if (tdep_count > SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE)
+		tdep_count = SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE;
+
+	dte_tables->K = cpu_to_be32(table_size);
+	dte_tables->T0 = cpu_to_be32(dte_data->t0);
+	dte_tables->MaxT = cpu_to_be32(dte_data->max_t);
+	dte_tables->WindowSize = dte_data->window_size;
+	dte_tables->temp_select = dte_data->temp_select;
+	dte_tables->DTE_mode = dte_data->dte_mode;
+	dte_tables->Tthreshold = cpu_to_be32(dte_data->t_threshold);
+
+	if (tdep_count > 0)
+		table_size--;
+
+	for (i = 0; i < table_size; i++) {
+		dte_tables->tau[i] = cpu_to_be32(dte_data->tau[i]);
+		dte_tables->R[i]   = cpu_to_be32(dte_data->r[i]);
+	}
+
+	dte_tables->Tdep_count = tdep_count;
+
+	for (i = 0; i < (u32)tdep_count; i++) {
+		dte_tables->T_limits[i] = dte_data->t_limits[i];
+		dte_tables->Tdep_tau[i] = cpu_to_be32(dte_data->tdep_tau[i]);
+		dte_tables->Tdep_R[i] = cpu_to_be32(dte_data->tdep_r[i]);
+	}
+
+	ret = amdgpu_si_copy_bytes_to_smc(adev, si_pi->dte_table_start,
+					  (u8 *)dte_tables,
+					  sizeof(Smc_SIslands_DTE_Configuration),
+					  si_pi->sram_end);
+	kfree(dte_tables);
+
+	return ret;
+}
+
+static int si_get_cac_std_voltage_max_min(struct amdgpu_device *adev,
+					  u16 *max, u16 *min)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct amdgpu_cac_leakage_table *table =
+		&adev->pm.dpm.dyn_state.cac_leakage_table;
+	u32 i;
+	u32 v0_loadline;
+
+	if (table == NULL)
+		return -EINVAL;
+
+	*max = 0;
+	*min = 0xFFFF;
+
+	for (i = 0; i < table->count; i++) {
+		if (table->entries[i].vddc > *max)
+			*max = table->entries[i].vddc;
+		if (table->entries[i].vddc < *min)
+			*min = table->entries[i].vddc;
+	}
+
+	if (si_pi->powertune_data->lkge_lut_v0_percent > 100)
+		return -EINVAL;
+
+	v0_loadline = (*min) * (100 - si_pi->powertune_data->lkge_lut_v0_percent) / 100;
+
+	if (v0_loadline > 0xFFFFUL)
+		return -EINVAL;
+
+	*min = (u16)v0_loadline;
+
+	if ((*min > *max) || (*max == 0) || (*min == 0))
+		return -EINVAL;
+
+	return 0;
+}
+
+static u16 si_get_cac_std_voltage_step(u16 max, u16 min)
+{
+	return ((max - min) + (SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES - 1)) /
+		SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
+}
+
+static int si_init_dte_leakage_table(struct amdgpu_device *adev,
+				     PP_SIslands_CacConfig *cac_tables,
+				     u16 vddc_max, u16 vddc_min, u16 vddc_step,
+				     u16 t0, u16 t_step)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 leakage;
+	unsigned int i, j;
+	s32 t;
+	u32 smc_leakage;
+	u32 scaling_factor;
+	u16 voltage;
+
+	scaling_factor = si_get_smc_power_scaling_factor(adev);
+
+	for (i = 0; i < SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES ; i++) {
+		t = (1000 * (i * t_step + t0));
+
+		for (j = 0; j < SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
+			voltage = vddc_max - (vddc_step * j);
+
+			si_calculate_leakage_for_v_and_t(adev,
+							 &si_pi->powertune_data->leakage_coefficients,
+							 voltage,
+							 t,
+							 si_pi->dyn_powertune_data.cac_leakage,
+							 &leakage);
+
+			smc_leakage = si_scale_power_for_smc(leakage, scaling_factor) / 4;
+
+			if (smc_leakage > 0xFFFF)
+				smc_leakage = 0xFFFF;
+
+			cac_tables->cac_lkge_lut[i][SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES-1-j] =
+				cpu_to_be16((u16)smc_leakage);
+		}
+	}
+	return 0;
+}
+
+static int si_init_simplified_leakage_table(struct amdgpu_device *adev,
+					    PP_SIslands_CacConfig *cac_tables,
+					    u16 vddc_max, u16 vddc_min, u16 vddc_step)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 leakage;
+	unsigned int i, j;
+	u32 smc_leakage;
+	u32 scaling_factor;
+	u16 voltage;
+
+	scaling_factor = si_get_smc_power_scaling_factor(adev);
+
+	for (j = 0; j < SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
+		voltage = vddc_max - (vddc_step * j);
+
+		si_calculate_leakage_for_v(adev,
+					   &si_pi->powertune_data->leakage_coefficients,
+					   si_pi->powertune_data->fixed_kt,
+					   voltage,
+					   si_pi->dyn_powertune_data.cac_leakage,
+					   &leakage);
+
+		smc_leakage = si_scale_power_for_smc(leakage, scaling_factor) / 4;
+
+		if (smc_leakage > 0xFFFF)
+			smc_leakage = 0xFFFF;
+
+		for (i = 0; i < SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES ; i++)
+			cac_tables->cac_lkge_lut[i][SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES-1-j] =
+				cpu_to_be16((u16)smc_leakage);
+	}
+	return 0;
+}
+
+static int si_initialize_smc_cac_tables(struct amdgpu_device *adev)
+{
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	PP_SIslands_CacConfig *cac_tables = NULL;
+	u16 vddc_max, vddc_min, vddc_step;
+	u16 t0, t_step;
+	u32 load_line_slope, reg;
+	int ret = 0;
+	u32 ticks_per_us = amdgpu_asic_get_xclk(adev) / 100;
+
+	if (ni_pi->enable_cac == false)
+		return 0;
+
+	cac_tables = kzalloc(sizeof(PP_SIslands_CacConfig), GFP_KERNEL);
+	if (!cac_tables)
+		return -ENOMEM;
+
+	reg = RREG32(CG_CAC_CTRL) & ~CAC_WINDOW_MASK;
+	reg |= CAC_WINDOW(si_pi->powertune_data->cac_window);
+	WREG32(CG_CAC_CTRL, reg);
+
+	si_pi->dyn_powertune_data.cac_leakage = adev->pm.dpm.cac_leakage;
+	si_pi->dyn_powertune_data.dc_pwr_value =
+		si_pi->powertune_data->dc_cac[NISLANDS_DCCAC_LEVEL_0];
+	si_pi->dyn_powertune_data.wintime = si_calculate_cac_wintime(adev);
+	si_pi->dyn_powertune_data.shift_n = si_pi->powertune_data->shift_n_default;
+
+	si_pi->dyn_powertune_data.leakage_minimum_temperature = 80 * 1000;
+
+	ret = si_get_cac_std_voltage_max_min(adev, &vddc_max, &vddc_min);
+	if (ret)
+		goto done_free;
+
+	vddc_step = si_get_cac_std_voltage_step(vddc_max, vddc_min);
+	vddc_min = vddc_max - (vddc_step * (SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES - 1));
+	t_step = 4;
+	t0 = 60;
+
+	if (si_pi->enable_dte || ni_pi->driver_calculate_cac_leakage)
+		ret = si_init_dte_leakage_table(adev, cac_tables,
+						vddc_max, vddc_min, vddc_step,
+						t0, t_step);
+	else
+		ret = si_init_simplified_leakage_table(adev, cac_tables,
+						       vddc_max, vddc_min, vddc_step);
+	if (ret)
+		goto done_free;
+
+	load_line_slope = ((u32)adev->pm.dpm.load_line_slope << SMC_SISLANDS_SCALE_R) / 100;
+
+	cac_tables->l2numWin_TDP = cpu_to_be32(si_pi->dyn_powertune_data.l2_lta_window_size);
+	cac_tables->lts_truncate_n = si_pi->dyn_powertune_data.lts_truncate;
+	cac_tables->SHIFT_N = si_pi->dyn_powertune_data.shift_n;
+	cac_tables->lkge_lut_V0 = cpu_to_be32((u32)vddc_min);
+	cac_tables->lkge_lut_Vstep = cpu_to_be32((u32)vddc_step);
+	cac_tables->R_LL = cpu_to_be32(load_line_slope);
+	cac_tables->WinTime = cpu_to_be32(si_pi->dyn_powertune_data.wintime);
+	cac_tables->calculation_repeats = cpu_to_be32(2);
+	cac_tables->dc_cac = cpu_to_be32(0);
+	cac_tables->log2_PG_LKG_SCALE = 12;
+	cac_tables->cac_temp = si_pi->powertune_data->operating_temp;
+	cac_tables->lkge_lut_T0 = cpu_to_be32((u32)t0);
+	cac_tables->lkge_lut_Tstep = cpu_to_be32((u32)t_step);
+
+	ret = amdgpu_si_copy_bytes_to_smc(adev, si_pi->cac_table_start,
+					  (u8 *)cac_tables,
+					  sizeof(PP_SIslands_CacConfig),
+					  si_pi->sram_end);
+
+	if (ret)
+		goto done_free;
+
+	ret = si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_ticks_per_us, ticks_per_us);
+
+done_free:
+	if (ret) {
+		ni_pi->enable_cac = false;
+		ni_pi->enable_power_containment = false;
+	}
+
+	kfree(cac_tables);
+
+	return ret;
+}
+
+static int si_program_cac_config_registers(struct amdgpu_device *adev,
+					   const struct si_cac_config_reg *cac_config_regs)
+{
+	const struct si_cac_config_reg *config_regs = cac_config_regs;
+	u32 data = 0, offset;
+
+	if (!config_regs)
+		return -EINVAL;
+
+	while (config_regs->offset != 0xFFFFFFFF) {
+		switch (config_regs->type) {
+		case SISLANDS_CACCONFIG_CGIND:
+			offset = SMC_CG_IND_START + config_regs->offset;
+			if (offset < SMC_CG_IND_END)
+				data = RREG32_SMC(offset);
+			break;
+		default:
+			data = RREG32(config_regs->offset);
+			break;
+		}
+
+		data &= ~config_regs->mask;
+		data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
+
+		switch (config_regs->type) {
+		case SISLANDS_CACCONFIG_CGIND:
+			offset = SMC_CG_IND_START + config_regs->offset;
+			if (offset < SMC_CG_IND_END)
+				WREG32_SMC(offset, data);
+			break;
+		default:
+			WREG32(config_regs->offset, data);
+			break;
+		}
+		config_regs++;
+	}
+	return 0;
+}
+
+static int si_initialize_hardware_cac_manager(struct amdgpu_device *adev)
+{
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	int ret;
+
+	if ((ni_pi->enable_cac == false) ||
+	    (ni_pi->cac_configuration_required == false))
+		return 0;
+
+	ret = si_program_cac_config_registers(adev, si_pi->lcac_config);
+	if (ret)
+		return ret;
+	ret = si_program_cac_config_registers(adev, si_pi->cac_override);
+	if (ret)
+		return ret;
+	ret = si_program_cac_config_registers(adev, si_pi->cac_weights);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int si_enable_smc_cac(struct amdgpu_device *adev,
+			     struct amdgpu_ps *amdgpu_new_state,
+			     bool enable)
+{
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	PPSMC_Result smc_result;
+	int ret = 0;
+
+	if (ni_pi->enable_cac) {
+		if (enable) {
+			if (!si_should_disable_uvd_powertune(adev, amdgpu_new_state)) {
+				if (ni_pi->support_cac_long_term_average) {
+					smc_result = amdgpu_si_send_msg_to_smc(adev, PPSMC_CACLongTermAvgEnable);
+					if (smc_result != PPSMC_Result_OK)
+						ni_pi->support_cac_long_term_average = false;
+				}
+
+				smc_result = amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_EnableCac);
+				if (smc_result != PPSMC_Result_OK) {
+					ret = -EINVAL;
+					ni_pi->cac_enabled = false;
+				} else {
+					ni_pi->cac_enabled = true;
+				}
+
+				if (si_pi->enable_dte) {
+					smc_result = amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_EnableDTE);
+					if (smc_result != PPSMC_Result_OK)
+						ret = -EINVAL;
+				}
+			}
+		} else if (ni_pi->cac_enabled) {
+			if (si_pi->enable_dte)
+				smc_result = amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_DisableDTE);
+
+			smc_result = amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_DisableCac);
+
+			ni_pi->cac_enabled = false;
+
+			if (ni_pi->support_cac_long_term_average)
+				smc_result = amdgpu_si_send_msg_to_smc(adev, PPSMC_CACLongTermAvgDisable);
+		}
+	}
+	return ret;
+}
+
+static int si_init_smc_spll_table(struct amdgpu_device *adev)
+{
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	SMC_SISLANDS_SPLL_DIV_TABLE *spll_table;
+	SISLANDS_SMC_SCLK_VALUE sclk_params;
+	u32 fb_div, p_div;
+	u32 clk_s, clk_v;
+	u32 sclk = 0;
+	int ret = 0;
+	u32 tmp;
+	int i;
+
+	if (si_pi->spll_table_start == 0)
+		return -EINVAL;
+
+	spll_table = kzalloc(sizeof(SMC_SISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
+	if (spll_table == NULL)
+		return -ENOMEM;
+
+	for (i = 0; i < 256; i++) {
+		ret = si_calculate_sclk_params(adev, sclk, &sclk_params);
+		if (ret)
+			break;
+		p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
+		fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
+		clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
+		clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
+
+		fb_div &= ~0x00001FFF;
+		fb_div >>= 1;
+		clk_v >>= 6;
+
+		if (p_div & ~(SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
+			ret = -EINVAL;
+		if (fb_div & ~(SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT))
+			ret = -EINVAL;
+		if (clk_s & ~(SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
+			ret = -EINVAL;
+		if (clk_v & ~(SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
+			ret = -EINVAL;
+
+		if (ret)
+			break;
+
+		tmp = ((fb_div << SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
+			((p_div << SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
+		spll_table->freq[i] = cpu_to_be32(tmp);
+
+		tmp = ((clk_v << SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
+			((clk_s << SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
+		spll_table->ss[i] = cpu_to_be32(tmp);
+
+		sclk += 512;
+	}
+
+
+	if (!ret)
+		ret = amdgpu_si_copy_bytes_to_smc(adev, si_pi->spll_table_start,
+						  (u8 *)spll_table,
+						  sizeof(SMC_SISLANDS_SPLL_DIV_TABLE),
+						  si_pi->sram_end);
+
+	if (ret)
+		ni_pi->enable_power_containment = false;
+
+	kfree(spll_table);
+
+	return ret;
+}
+
+struct si_dpm_quirk {
+	u32 chip_vendor;
+	u32 chip_device;
+	u32 subsys_vendor;
+	u32 subsys_device;
+	u32 max_sclk;
+	u32 max_mclk;
+};
+
+/* cards with dpm stability problems */
+static struct si_dpm_quirk si_dpm_quirk_list[] = {
+	/* PITCAIRN - https://bugs.freedesktop.org/show_bug.cgi?id=76490 */
+	{ PCI_VENDOR_ID_ATI, 0x6810, 0x1462, 0x3036, 0, 120000 },
+	{ PCI_VENDOR_ID_ATI, 0x6811, 0x174b, 0xe271, 0, 120000 },
+	{ PCI_VENDOR_ID_ATI, 0x6811, 0x174b, 0x2015, 0, 120000 },
+	{ PCI_VENDOR_ID_ATI, 0x6810, 0x174b, 0xe271, 85000, 90000 },
+	{ PCI_VENDOR_ID_ATI, 0x6811, 0x1462, 0x2015, 0, 120000 },
+	{ PCI_VENDOR_ID_ATI, 0x6811, 0x1043, 0x2015, 0, 120000 },
+	{ PCI_VENDOR_ID_ATI, 0x6811, 0x148c, 0x2015, 0, 120000 },
+	{ PCI_VENDOR_ID_ATI, 0x6810, 0x1682, 0x9275, 0, 120000 },
+	{ 0, 0, 0, 0 },
+};
+
+static u16 si_get_lower_of_leakage_and_vce_voltage(struct amdgpu_device *adev,
+						   u16 vce_voltage)
+{
+	u16 highest_leakage = 0;
+	struct si_power_info *si_pi = si_get_pi(adev);
+	int i;
+
+	for (i = 0; i < si_pi->leakage_voltage.count; i++){
+		if (highest_leakage < si_pi->leakage_voltage.entries[i].voltage)
+			highest_leakage = si_pi->leakage_voltage.entries[i].voltage;
+	}
+
+	if (si_pi->leakage_voltage.count && (highest_leakage < vce_voltage))
+		return highest_leakage;
+
+	return vce_voltage;
+}
+
+static int si_get_vce_clock_voltage(struct amdgpu_device *adev,
+				    u32 evclk, u32 ecclk, u16 *voltage)
+{
+	u32 i;
+	int ret = -EINVAL;
+	struct amdgpu_vce_clock_voltage_dependency_table *table =
+		&adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
+
+	if (((evclk == 0) && (ecclk == 0)) ||
+	    (table && (table->count == 0))) {
+		*voltage = 0;
+		return 0;
+	}
+
+	for (i = 0; i < table->count; i++) {
+		if ((evclk <= table->entries[i].evclk) &&
+		    (ecclk <= table->entries[i].ecclk)) {
+			*voltage = table->entries[i].v;
+			ret = 0;
+			break;
+		}
+	}
+
+	/* if no match return the highest voltage */
+	if (ret)
+		*voltage = table->entries[table->count - 1].v;
+
+	*voltage = si_get_lower_of_leakage_and_vce_voltage(adev, *voltage);
+
+	return ret;
+}
+
+static bool si_dpm_vblank_too_short(struct amdgpu_device *adev)
+{
+
+	u32 vblank_time = amdgpu_dpm_get_vblank_time(adev);
+	/* we never hit the non-gddr5 limit so disable it */
+	u32 switch_limit = adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5 ? 450 : 0;
+
+	if (vblank_time < switch_limit)
+		return true;
+	else
+		return false;
+
+}
+
+static int ni_copy_and_switch_arb_sets(struct amdgpu_device *adev,
+				u32 arb_freq_src, u32 arb_freq_dest)
+{
+	u32 mc_arb_dram_timing;
+	u32 mc_arb_dram_timing2;
+	u32 burst_time;
+	u32 mc_cg_config;
+
+	switch (arb_freq_src) {
+	case MC_CG_ARB_FREQ_F0:
+		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
+		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
+		break;
+	case MC_CG_ARB_FREQ_F1:
+		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
+		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
+		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
+		break;
+	case MC_CG_ARB_FREQ_F2:
+		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
+		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
+		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
+		break;
+	case MC_CG_ARB_FREQ_F3:
+		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
+		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
+		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (arb_freq_dest) {
+	case MC_CG_ARB_FREQ_F0:
+		WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
+		WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
+		WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
+		break;
+	case MC_CG_ARB_FREQ_F1:
+		WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
+		WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
+		WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
+		break;
+	case MC_CG_ARB_FREQ_F2:
+		WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
+		WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
+		WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
+		break;
+	case MC_CG_ARB_FREQ_F3:
+		WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
+		WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
+		WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
+	WREG32(MC_CG_CONFIG, mc_cg_config);
+	WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
+
+	return 0;
+}
+
+static void ni_update_current_ps(struct amdgpu_device *adev,
+			  struct amdgpu_ps *rps)
+{
+	struct si_ps *new_ps = si_get_ps(rps);
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+
+	eg_pi->current_rps = *rps;
+	ni_pi->current_ps = *new_ps;
+	eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
+}
+
+static void ni_update_requested_ps(struct amdgpu_device *adev,
+			    struct amdgpu_ps *rps)
+{
+	struct si_ps *new_ps = si_get_ps(rps);
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+
+	eg_pi->requested_rps = *rps;
+	ni_pi->requested_ps = *new_ps;
+	eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
+}
+
+static void ni_set_uvd_clock_before_set_eng_clock(struct amdgpu_device *adev,
+					   struct amdgpu_ps *new_ps,
+					   struct amdgpu_ps *old_ps)
+{
+	struct si_ps *new_state = si_get_ps(new_ps);
+	struct si_ps *current_state = si_get_ps(old_ps);
+
+	if ((new_ps->vclk == old_ps->vclk) &&
+	    (new_ps->dclk == old_ps->dclk))
+		return;
+
+	if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
+	    current_state->performance_levels[current_state->performance_level_count - 1].sclk)
+		return;
+
+	amdgpu_asic_set_uvd_clocks(adev, new_ps->vclk, new_ps->dclk);
+}
+
+static void ni_set_uvd_clock_after_set_eng_clock(struct amdgpu_device *adev,
+					  struct amdgpu_ps *new_ps,
+					  struct amdgpu_ps *old_ps)
+{
+	struct si_ps *new_state = si_get_ps(new_ps);
+	struct si_ps *current_state = si_get_ps(old_ps);
+
+	if ((new_ps->vclk == old_ps->vclk) &&
+	    (new_ps->dclk == old_ps->dclk))
+		return;
+
+	if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
+	    current_state->performance_levels[current_state->performance_level_count - 1].sclk)
+		return;
+
+	amdgpu_asic_set_uvd_clocks(adev, new_ps->vclk, new_ps->dclk);
+}
+
+static u16 btc_find_voltage(struct atom_voltage_table *table, u16 voltage)
+{
+	unsigned int i;
+
+	for (i = 0; i < table->count; i++)
+		if (voltage <= table->entries[i].value)
+			return table->entries[i].value;
+
+	return table->entries[table->count - 1].value;
+}
+
+static u32 btc_find_valid_clock(struct amdgpu_clock_array *clocks,
+		                u32 max_clock, u32 requested_clock)
+{
+	unsigned int i;
+
+	if ((clocks == NULL) || (clocks->count == 0))
+		return (requested_clock < max_clock) ? requested_clock : max_clock;
+
+	for (i = 0; i < clocks->count; i++) {
+		if (clocks->values[i] >= requested_clock)
+			return (clocks->values[i] < max_clock) ? clocks->values[i] : max_clock;
+	}
+
+	return (clocks->values[clocks->count - 1] < max_clock) ?
+		clocks->values[clocks->count - 1] : max_clock;
+}
+
+static u32 btc_get_valid_mclk(struct amdgpu_device *adev,
+			      u32 max_mclk, u32 requested_mclk)
+{
+	return btc_find_valid_clock(&adev->pm.dpm.dyn_state.valid_mclk_values,
+				    max_mclk, requested_mclk);
+}
+
+static u32 btc_get_valid_sclk(struct amdgpu_device *adev,
+		              u32 max_sclk, u32 requested_sclk)
+{
+	return btc_find_valid_clock(&adev->pm.dpm.dyn_state.valid_sclk_values,
+				    max_sclk, requested_sclk);
+}
+
+static void btc_get_max_clock_from_voltage_dependency_table(struct amdgpu_clock_voltage_dependency_table *table,
+							    u32 *max_clock)
+{
+	u32 i, clock = 0;
+
+	if ((table == NULL) || (table->count == 0)) {
+		*max_clock = clock;
+		return;
+	}
+
+	for (i = 0; i < table->count; i++) {
+		if (clock < table->entries[i].clk)
+			clock = table->entries[i].clk;
+	}
+	*max_clock = clock;
+}
+
+static void btc_apply_voltage_dependency_rules(struct amdgpu_clock_voltage_dependency_table *table,
+					       u32 clock, u16 max_voltage, u16 *voltage)
+{
+	u32 i;
+
+	if ((table == NULL) || (table->count == 0))
+		return;
+
+	for (i= 0; i < table->count; i++) {
+		if (clock <= table->entries[i].clk) {
+			if (*voltage < table->entries[i].v)
+				*voltage = (u16)((table->entries[i].v < max_voltage) ?
+					   table->entries[i].v : max_voltage);
+			return;
+		}
+	}
+
+	*voltage = (*voltage > max_voltage) ? *voltage : max_voltage;
+}
+
+static void btc_adjust_clock_combinations(struct amdgpu_device *adev,
+					  const struct amdgpu_clock_and_voltage_limits *max_limits,
+					  struct rv7xx_pl *pl)
+{
+
+	if ((pl->mclk == 0) || (pl->sclk == 0))
+		return;
+
+	if (pl->mclk == pl->sclk)
+		return;
+
+	if (pl->mclk > pl->sclk) {
+		if (((pl->mclk + (pl->sclk - 1)) / pl->sclk) > adev->pm.dpm.dyn_state.mclk_sclk_ratio)
+			pl->sclk = btc_get_valid_sclk(adev,
+						      max_limits->sclk,
+						      (pl->mclk +
+						      (adev->pm.dpm.dyn_state.mclk_sclk_ratio - 1)) /
+						      adev->pm.dpm.dyn_state.mclk_sclk_ratio);
+	} else {
+		if ((pl->sclk - pl->mclk) > adev->pm.dpm.dyn_state.sclk_mclk_delta)
+			pl->mclk = btc_get_valid_mclk(adev,
+						      max_limits->mclk,
+						      pl->sclk -
+						      adev->pm.dpm.dyn_state.sclk_mclk_delta);
+	}
+}
+
+static void btc_apply_voltage_delta_rules(struct amdgpu_device *adev,
+					  u16 max_vddc, u16 max_vddci,
+					  u16 *vddc, u16 *vddci)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	u16 new_voltage;
+
+	if ((0 == *vddc) || (0 == *vddci))
+		return;
+
+	if (*vddc > *vddci) {
+		if ((*vddc - *vddci) > adev->pm.dpm.dyn_state.vddc_vddci_delta) {
+			new_voltage = btc_find_voltage(&eg_pi->vddci_voltage_table,
+						       (*vddc - adev->pm.dpm.dyn_state.vddc_vddci_delta));
+			*vddci = (new_voltage < max_vddci) ? new_voltage : max_vddci;
+		}
+	} else {
+		if ((*vddci - *vddc) > adev->pm.dpm.dyn_state.vddc_vddci_delta) {
+			new_voltage = btc_find_voltage(&eg_pi->vddc_voltage_table,
+						       (*vddci - adev->pm.dpm.dyn_state.vddc_vddci_delta));
+			*vddc = (new_voltage < max_vddc) ? new_voltage : max_vddc;
+		}
+	}
+}
+
+static enum amdgpu_pcie_gen r600_get_pcie_gen_support(struct amdgpu_device *adev,
+					       u32 sys_mask,
+					       enum amdgpu_pcie_gen asic_gen,
+					       enum amdgpu_pcie_gen default_gen)
+{
+	switch (asic_gen) {
+	case AMDGPU_PCIE_GEN1:
+		return AMDGPU_PCIE_GEN1;
+	case AMDGPU_PCIE_GEN2:
+		return AMDGPU_PCIE_GEN2;
+	case AMDGPU_PCIE_GEN3:
+		return AMDGPU_PCIE_GEN3;
+	default:
+		if ((sys_mask & DRM_PCIE_SPEED_80) && (default_gen == AMDGPU_PCIE_GEN3))
+			return AMDGPU_PCIE_GEN3;
+		else if ((sys_mask & DRM_PCIE_SPEED_50) && (default_gen == AMDGPU_PCIE_GEN2))
+			return AMDGPU_PCIE_GEN2;
+		else
+			return AMDGPU_PCIE_GEN1;
+	}
+	return AMDGPU_PCIE_GEN1;
+}
+
+static void r600_calculate_u_and_p(u32 i, u32 r_c, u32 p_b,
+			    u32 *p, u32 *u)
+{
+	u32 b_c = 0;
+	u32 i_c;
+	u32 tmp;
+
+	i_c = (i * r_c) / 100;
+	tmp = i_c >> p_b;
+
+	while (tmp) {
+		b_c++;
+		tmp >>= 1;
+	}
+
+	*u = (b_c + 1) / 2;
+	*p = i_c / (1 << (2 * (*u)));
+}
+
+static int r600_calculate_at(u32 t, u32 h, u32 fh, u32 fl, u32 *tl, u32 *th)
+{
+	u32 k, a, ah, al;
+	u32 t1;
+
+	if ((fl == 0) || (fh == 0) || (fl > fh))
+		return -EINVAL;
+
+	k = (100 * fh) / fl;
+	t1 = (t * (k - 100));
+	a = (1000 * (100 * h + t1)) / (10000 + (t1 / 100));
+	a = (a + 5) / 10;
+	ah = ((a * t) + 5000) / 10000;
+	al = a - ah;
+
+	*th = t - ah;
+	*tl = t + al;
+
+	return 0;
+}
+
+static bool r600_is_uvd_state(u32 class, u32 class2)
+{
+	if (class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+		return true;
+	if (class & ATOM_PPLIB_CLASSIFICATION_HD2STATE)
+		return true;
+	if (class & ATOM_PPLIB_CLASSIFICATION_HDSTATE)
+		return true;
+	if (class & ATOM_PPLIB_CLASSIFICATION_SDSTATE)
+		return true;
+	if (class2 & ATOM_PPLIB_CLASSIFICATION2_MVC)
+		return true;
+	return false;
+}
+
+static u8 rv770_get_memory_module_index(struct amdgpu_device *adev)
+{
+	return (u8) ((RREG32(BIOS_SCRATCH_4) >> 16) & 0xff);
+}
+
+static void rv770_get_max_vddc(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	u16 vddc;
+
+	if (amdgpu_atombios_get_max_vddc(adev, 0, 0, &vddc))
+		pi->max_vddc = 0;
+	else
+		pi->max_vddc = vddc;
+}
+
+static void rv770_get_engine_memory_ss(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct amdgpu_atom_ss ss;
+
+	pi->sclk_ss = amdgpu_atombios_get_asic_ss_info(adev, &ss,
+						       ASIC_INTERNAL_ENGINE_SS, 0);
+	pi->mclk_ss = amdgpu_atombios_get_asic_ss_info(adev, &ss,
+						       ASIC_INTERNAL_MEMORY_SS, 0);
+
+	if (pi->sclk_ss || pi->mclk_ss)
+		pi->dynamic_ss = true;
+	else
+		pi->dynamic_ss = false;
+}
+
+
+static void si_apply_state_adjust_rules(struct amdgpu_device *adev,
+					struct amdgpu_ps *rps)
+{
+	struct  si_ps *ps = si_get_ps(rps);
+	struct amdgpu_clock_and_voltage_limits *max_limits;
+	bool disable_mclk_switching = false;
+	bool disable_sclk_switching = false;
+	u32 mclk, sclk;
+	u16 vddc, vddci, min_vce_voltage = 0;
+	u32 max_sclk_vddc, max_mclk_vddci, max_mclk_vddc;
+	u32 max_sclk = 0, max_mclk = 0;
+	int i;
+	struct si_dpm_quirk *p = si_dpm_quirk_list;
+
+	/* Apply dpm quirks */
+	while (p && p->chip_device != 0) {
+		if (adev->pdev->vendor == p->chip_vendor &&
+		    adev->pdev->device == p->chip_device &&
+		    adev->pdev->subsystem_vendor == p->subsys_vendor &&
+		    adev->pdev->subsystem_device == p->subsys_device) {
+			max_sclk = p->max_sclk;
+			max_mclk = p->max_mclk;
+			break;
+		}
+		++p;
+	}
+	/* limit mclk on all R7 370 parts for stability */
+	if (adev->pdev->device == 0x6811 &&
+	    adev->pdev->revision == 0x81)
+		max_mclk = 120000;
+	/* limit sclk/mclk on Jet parts for stability */
+	if (adev->pdev->device == 0x6665 &&
+	    adev->pdev->revision == 0xc3) {
+		max_sclk = 75000;
+		max_mclk = 80000;
+	}
+
+	if (rps->vce_active) {
+		rps->evclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].evclk;
+		rps->ecclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].ecclk;
+		si_get_vce_clock_voltage(adev, rps->evclk, rps->ecclk,
+					 &min_vce_voltage);
+	} else {
+		rps->evclk = 0;
+		rps->ecclk = 0;
+	}
+
+	if ((adev->pm.dpm.new_active_crtc_count > 1) ||
+	    si_dpm_vblank_too_short(adev))
+		disable_mclk_switching = true;
+
+	if (rps->vclk || rps->dclk) {
+		disable_mclk_switching = true;
+		disable_sclk_switching = true;
+	}
+
+	if (adev->pm.dpm.ac_power)
+		max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
+	else
+		max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
+
+	for (i = ps->performance_level_count - 2; i >= 0; i--) {
+		if (ps->performance_levels[i].vddc > ps->performance_levels[i+1].vddc)
+			ps->performance_levels[i].vddc = ps->performance_levels[i+1].vddc;
+	}
+	if (adev->pm.dpm.ac_power == false) {
+		for (i = 0; i < ps->performance_level_count; i++) {
+			if (ps->performance_levels[i].mclk > max_limits->mclk)
+				ps->performance_levels[i].mclk = max_limits->mclk;
+			if (ps->performance_levels[i].sclk > max_limits->sclk)
+				ps->performance_levels[i].sclk = max_limits->sclk;
+			if (ps->performance_levels[i].vddc > max_limits->vddc)
+				ps->performance_levels[i].vddc = max_limits->vddc;
+			if (ps->performance_levels[i].vddci > max_limits->vddci)
+				ps->performance_levels[i].vddci = max_limits->vddci;
+		}
+	}
+
+	/* limit clocks to max supported clocks based on voltage dependency tables */
+	btc_get_max_clock_from_voltage_dependency_table(&adev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
+							&max_sclk_vddc);
+	btc_get_max_clock_from_voltage_dependency_table(&adev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
+							&max_mclk_vddci);
+	btc_get_max_clock_from_voltage_dependency_table(&adev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
+							&max_mclk_vddc);
+
+	for (i = 0; i < ps->performance_level_count; i++) {
+		if (max_sclk_vddc) {
+			if (ps->performance_levels[i].sclk > max_sclk_vddc)
+				ps->performance_levels[i].sclk = max_sclk_vddc;
+		}
+		if (max_mclk_vddci) {
+			if (ps->performance_levels[i].mclk > max_mclk_vddci)
+				ps->performance_levels[i].mclk = max_mclk_vddci;
+		}
+		if (max_mclk_vddc) {
+			if (ps->performance_levels[i].mclk > max_mclk_vddc)
+				ps->performance_levels[i].mclk = max_mclk_vddc;
+		}
+		if (max_mclk) {
+			if (ps->performance_levels[i].mclk > max_mclk)
+				ps->performance_levels[i].mclk = max_mclk;
+		}
+		if (max_sclk) {
+			if (ps->performance_levels[i].sclk > max_sclk)
+				ps->performance_levels[i].sclk = max_sclk;
+		}
+	}
+
+	/* XXX validate the min clocks required for display */
+
+	if (disable_mclk_switching) {
+		mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
+		vddci = ps->performance_levels[ps->performance_level_count - 1].vddci;
+	} else {
+		mclk = ps->performance_levels[0].mclk;
+		vddci = ps->performance_levels[0].vddci;
+	}
+
+	if (disable_sclk_switching) {
+		sclk = ps->performance_levels[ps->performance_level_count - 1].sclk;
+		vddc = ps->performance_levels[ps->performance_level_count - 1].vddc;
+	} else {
+		sclk = ps->performance_levels[0].sclk;
+		vddc = ps->performance_levels[0].vddc;
+	}
+
+	if (rps->vce_active) {
+		if (sclk < adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].sclk)
+			sclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].sclk;
+		if (mclk < adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].mclk)
+			mclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].mclk;
+	}
+
+	/* adjusted low state */
+	ps->performance_levels[0].sclk = sclk;
+	ps->performance_levels[0].mclk = mclk;
+	ps->performance_levels[0].vddc = vddc;
+	ps->performance_levels[0].vddci = vddci;
+
+	if (disable_sclk_switching) {
+		sclk = ps->performance_levels[0].sclk;
+		for (i = 1; i < ps->performance_level_count; i++) {
+			if (sclk < ps->performance_levels[i].sclk)
+				sclk = ps->performance_levels[i].sclk;
+		}
+		for (i = 0; i < ps->performance_level_count; i++) {
+			ps->performance_levels[i].sclk = sclk;
+			ps->performance_levels[i].vddc = vddc;
+		}
+	} else {
+		for (i = 1; i < ps->performance_level_count; i++) {
+			if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
+				ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
+			if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
+				ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
+		}
+	}
+
+	if (disable_mclk_switching) {
+		mclk = ps->performance_levels[0].mclk;
+		for (i = 1; i < ps->performance_level_count; i++) {
+			if (mclk < ps->performance_levels[i].mclk)
+				mclk = ps->performance_levels[i].mclk;
+		}
+		for (i = 0; i < ps->performance_level_count; i++) {
+			ps->performance_levels[i].mclk = mclk;
+			ps->performance_levels[i].vddci = vddci;
+		}
+	} else {
+		for (i = 1; i < ps->performance_level_count; i++) {
+			if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
+				ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
+			if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
+				ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
+		}
+	}
+
+	for (i = 0; i < ps->performance_level_count; i++)
+		btc_adjust_clock_combinations(adev, max_limits,
+					      &ps->performance_levels[i]);
+
+	for (i = 0; i < ps->performance_level_count; i++) {
+		if (ps->performance_levels[i].vddc < min_vce_voltage)
+			ps->performance_levels[i].vddc = min_vce_voltage;
+		btc_apply_voltage_dependency_rules(&adev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
+						   ps->performance_levels[i].sclk,
+						   max_limits->vddc,  &ps->performance_levels[i].vddc);
+		btc_apply_voltage_dependency_rules(&adev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
+						   ps->performance_levels[i].mclk,
+						   max_limits->vddci, &ps->performance_levels[i].vddci);
+		btc_apply_voltage_dependency_rules(&adev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
+						   ps->performance_levels[i].mclk,
+						   max_limits->vddc,  &ps->performance_levels[i].vddc);
+		btc_apply_voltage_dependency_rules(&adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
+						   adev->clock.current_dispclk,
+						   max_limits->vddc,  &ps->performance_levels[i].vddc);
+	}
+
+	for (i = 0; i < ps->performance_level_count; i++) {
+		btc_apply_voltage_delta_rules(adev,
+					      max_limits->vddc, max_limits->vddci,
+					      &ps->performance_levels[i].vddc,
+					      &ps->performance_levels[i].vddci);
+	}
+
+	ps->dc_compatible = true;
+	for (i = 0; i < ps->performance_level_count; i++) {
+		if (ps->performance_levels[i].vddc > adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
+			ps->dc_compatible = false;
+	}
+}
+
+#if 0
+static int si_read_smc_soft_register(struct amdgpu_device *adev,
+				     u16 reg_offset, u32 *value)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	return amdgpu_si_read_smc_sram_dword(adev,
+					     si_pi->soft_regs_start + reg_offset, value,
+					     si_pi->sram_end);
+}
+#endif
+
+static int si_write_smc_soft_register(struct amdgpu_device *adev,
+				      u16 reg_offset, u32 value)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	return amdgpu_si_write_smc_sram_dword(adev,
+					      si_pi->soft_regs_start + reg_offset,
+					      value, si_pi->sram_end);
+}
+
+static bool si_is_special_1gb_platform(struct amdgpu_device *adev)
+{
+	bool ret = false;
+	u32 tmp, width, row, column, bank, density;
+	bool is_memory_gddr5, is_special;
+
+	tmp = RREG32(MC_SEQ_MISC0);
+	is_memory_gddr5 = (MC_SEQ_MISC0_GDDR5_VALUE == ((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT));
+	is_special = (MC_SEQ_MISC0_REV_ID_VALUE == ((tmp & MC_SEQ_MISC0_REV_ID_MASK) >> MC_SEQ_MISC0_REV_ID_SHIFT))
+		& (MC_SEQ_MISC0_VEN_ID_VALUE == ((tmp & MC_SEQ_MISC0_VEN_ID_MASK) >> MC_SEQ_MISC0_VEN_ID_SHIFT));
+
+	WREG32(MC_SEQ_IO_DEBUG_INDEX, 0xb);
+	width = ((RREG32(MC_SEQ_IO_DEBUG_DATA) >> 1) & 1) ? 16 : 32;
+
+	tmp = RREG32(MC_ARB_RAMCFG);
+	row = ((tmp & NOOFROWS_MASK) >> NOOFROWS_SHIFT) + 10;
+	column = ((tmp & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) + 8;
+	bank = ((tmp & NOOFBANK_MASK) >> NOOFBANK_SHIFT) + 2;
+
+	density = (1 << (row + column - 20 + bank)) * width;
+
+	if ((adev->pdev->device == 0x6819) &&
+	    is_memory_gddr5 && is_special && (density == 0x400))
+		ret = true;
+
+	return ret;
+}
+
+static void si_get_leakage_vddc(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u16 vddc, count = 0;
+	int i, ret;
+
+	for (i = 0; i < SISLANDS_MAX_LEAKAGE_COUNT; i++) {
+		ret = amdgpu_atombios_get_leakage_vddc_based_on_leakage_idx(adev, &vddc, SISLANDS_LEAKAGE_INDEX0 + i);
+
+		if (!ret && (vddc > 0) && (vddc != (SISLANDS_LEAKAGE_INDEX0 + i))) {
+			si_pi->leakage_voltage.entries[count].voltage = vddc;
+			si_pi->leakage_voltage.entries[count].leakage_index =
+				SISLANDS_LEAKAGE_INDEX0 + i;
+			count++;
+		}
+	}
+	si_pi->leakage_voltage.count = count;
+}
+
+static int si_get_leakage_voltage_from_leakage_index(struct amdgpu_device *adev,
+						     u32 index, u16 *leakage_voltage)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	int i;
+
+	if (leakage_voltage == NULL)
+		return -EINVAL;
+
+	if ((index & 0xff00) != 0xff00)
+		return -EINVAL;
+
+	if ((index & 0xff) > SISLANDS_MAX_LEAKAGE_COUNT + 1)
+		return -EINVAL;
+
+	if (index < SISLANDS_LEAKAGE_INDEX0)
+		return -EINVAL;
+
+	for (i = 0; i < si_pi->leakage_voltage.count; i++) {
+		if (si_pi->leakage_voltage.entries[i].leakage_index == index) {
+			*leakage_voltage = si_pi->leakage_voltage.entries[i].voltage;
+			return 0;
+		}
+	}
+	return -EAGAIN;
+}
+
+static void si_set_dpm_event_sources(struct amdgpu_device *adev, u32 sources)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	bool want_thermal_protection;
+	enum amdgpu_dpm_event_src dpm_event_src;
+
+	switch (sources) {
+	case 0:
+	default:
+		want_thermal_protection = false;
+		break;
+	case (1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL):
+		want_thermal_protection = true;
+		dpm_event_src = AMDGPU_DPM_EVENT_SRC_DIGITAL;
+		break;
+	case (1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
+		want_thermal_protection = true;
+		dpm_event_src = AMDGPU_DPM_EVENT_SRC_EXTERNAL;
+		break;
+	case ((1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
+	      (1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL)):
+		want_thermal_protection = true;
+		dpm_event_src = AMDGPU_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
+		break;
+	}
+
+	if (want_thermal_protection) {
+		WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK);
+		if (pi->thermal_protection)
+			WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
+	} else {
+		WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
+	}
+}
+
+static void si_enable_auto_throttle_source(struct amdgpu_device *adev,
+					   enum amdgpu_dpm_auto_throttle_src source,
+					   bool enable)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+
+	if (enable) {
+		if (!(pi->active_auto_throttle_sources & (1 << source))) {
+			pi->active_auto_throttle_sources |= 1 << source;
+			si_set_dpm_event_sources(adev, pi->active_auto_throttle_sources);
+		}
+	} else {
+		if (pi->active_auto_throttle_sources & (1 << source)) {
+			pi->active_auto_throttle_sources &= ~(1 << source);
+			si_set_dpm_event_sources(adev, pi->active_auto_throttle_sources);
+		}
+	}
+}
+
+static void si_start_dpm(struct amdgpu_device *adev)
+{
+	WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
+}
+
+static void si_stop_dpm(struct amdgpu_device *adev)
+{
+	WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
+}
+
+static void si_enable_sclk_control(struct amdgpu_device *adev, bool enable)
+{
+	if (enable)
+		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF);
+	else
+		WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF);
+
+}
+
+#if 0
+static int si_notify_hardware_of_thermal_state(struct amdgpu_device *adev,
+					       u32 thermal_level)
+{
+	PPSMC_Result ret;
+
+	if (thermal_level == 0) {
+		ret = amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_EnableThermalInterrupt);
+		if (ret == PPSMC_Result_OK)
+			return 0;
+		else
+			return -EINVAL;
+	}
+	return 0;
+}
+
+static void si_notify_hardware_vpu_recovery_event(struct amdgpu_device *adev)
+{
+	si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_tdr_is_about_to_happen, true);
+}
+#endif
+
+#if 0
+static int si_notify_hw_of_powersource(struct amdgpu_device *adev, bool ac_power)
+{
+	if (ac_power)
+		return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
+			0 : -EINVAL;
+
+	return 0;
+}
+#endif
+
+static PPSMC_Result si_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
+						      PPSMC_Msg msg, u32 parameter)
+{
+	WREG32(SMC_SCRATCH0, parameter);
+	return amdgpu_si_send_msg_to_smc(adev, msg);
+}
+
+static int si_restrict_performance_levels_before_switch(struct amdgpu_device *adev)
+{
+	if (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
+		return -EINVAL;
+
+	return (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
+		0 : -EINVAL;
+}
+
+static int si_dpm_force_performance_level(struct amdgpu_device *adev,
+				   enum amdgpu_dpm_forced_level level)
+{
+	struct amdgpu_ps *rps = adev->pm.dpm.current_ps;
+	struct  si_ps *ps = si_get_ps(rps);
+	u32 levels = ps->performance_level_count;
+
+	if (level == AMDGPU_DPM_FORCED_LEVEL_HIGH) {
+		if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetEnabledLevels, levels) != PPSMC_Result_OK)
+			return -EINVAL;
+
+		if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
+			return -EINVAL;
+	} else if (level == AMDGPU_DPM_FORCED_LEVEL_LOW) {
+		if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
+			return -EINVAL;
+
+		if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
+			return -EINVAL;
+	} else if (level == AMDGPU_DPM_FORCED_LEVEL_AUTO) {
+		if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
+			return -EINVAL;
+
+		if (si_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SetEnabledLevels, levels) != PPSMC_Result_OK)
+			return -EINVAL;
+	}
+
+	adev->pm.dpm.forced_level = level;
+
+	return 0;
+}
+
+#if 0
+static int si_set_boot_state(struct amdgpu_device *adev)
+{
+	return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_SwitchToInitialState) == PPSMC_Result_OK) ?
+		0 : -EINVAL;
+}
+#endif
+
+static int si_set_sw_state(struct amdgpu_device *adev)
+{
+	return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_SwitchToSwState) == PPSMC_Result_OK) ?
+		0 : -EINVAL;
+}
+
+static int si_halt_smc(struct amdgpu_device *adev)
+{
+	if (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_Halt) != PPSMC_Result_OK)
+		return -EINVAL;
+
+	return (amdgpu_si_wait_for_smc_inactive(adev) == PPSMC_Result_OK) ?
+		0 : -EINVAL;
+}
+
+static int si_resume_smc(struct amdgpu_device *adev)
+{
+	if (amdgpu_si_send_msg_to_smc(adev, PPSMC_FlushDataCache) != PPSMC_Result_OK)
+		return -EINVAL;
+
+	return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_Resume) == PPSMC_Result_OK) ?
+		0 : -EINVAL;
+}
+
+static void si_dpm_start_smc(struct amdgpu_device *adev)
+{
+	amdgpu_si_program_jump_on_start(adev);
+	amdgpu_si_start_smc(adev);
+	amdgpu_si_smc_clock(adev, true);
+}
+
+static void si_dpm_stop_smc(struct amdgpu_device *adev)
+{
+	amdgpu_si_reset_smc(adev);
+	amdgpu_si_smc_clock(adev, false);
+}
+
+static int si_process_firmware_header(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 tmp;
+	int ret;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev,
+					    SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+					    SISLANDS_SMC_FIRMWARE_HEADER_stateTable,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	si_pi->state_table_start = tmp;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev,
+					    SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+					    SISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	si_pi->soft_regs_start = tmp;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev,
+					    SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+					    SISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	si_pi->mc_reg_table_start = tmp;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev,
+					    SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+					    SISLANDS_SMC_FIRMWARE_HEADER_fanTable,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	si_pi->fan_table_start = tmp;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev,
+					    SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+					    SISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	si_pi->arb_table_start = tmp;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev,
+					    SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+					    SISLANDS_SMC_FIRMWARE_HEADER_CacConfigTable,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	si_pi->cac_table_start = tmp;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev,
+					    SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+					    SISLANDS_SMC_FIRMWARE_HEADER_DteConfiguration,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	si_pi->dte_table_start = tmp;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev,
+					    SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+					    SISLANDS_SMC_FIRMWARE_HEADER_spllTable,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	si_pi->spll_table_start = tmp;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev,
+					    SISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
+					    SISLANDS_SMC_FIRMWARE_HEADER_PAPMParameters,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	si_pi->papm_cfg_table_start = tmp;
+
+	return ret;
+}
+
+static void si_read_clock_registers(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	si_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
+	si_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
+	si_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
+	si_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
+	si_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
+	si_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
+	si_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
+	si_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
+	si_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
+	si_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
+	si_pi->clock_registers.mpll_func_cntl = RREG32(MPLL_FUNC_CNTL);
+	si_pi->clock_registers.mpll_func_cntl_1 = RREG32(MPLL_FUNC_CNTL_1);
+	si_pi->clock_registers.mpll_func_cntl_2 = RREG32(MPLL_FUNC_CNTL_2);
+	si_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
+	si_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
+}
+
+static void si_enable_thermal_protection(struct amdgpu_device *adev,
+					  bool enable)
+{
+	if (enable)
+		WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
+	else
+		WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
+}
+
+static void si_enable_acpi_power_management(struct amdgpu_device *adev)
+{
+	WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN);
+}
+
+#if 0
+static int si_enter_ulp_state(struct amdgpu_device *adev)
+{
+	WREG32(SMC_MESSAGE_0, PPSMC_MSG_SwitchToMinimumPower);
+
+	udelay(25000);
+
+	return 0;
+}
+
+static int si_exit_ulp_state(struct amdgpu_device *adev)
+{
+	int i;
+
+	WREG32(SMC_MESSAGE_0, PPSMC_MSG_ResumeFromMinimumPower);
+
+	udelay(7000);
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (RREG32(SMC_RESP_0) == 1)
+			break;
+		udelay(1000);
+	}
+
+	return 0;
+}
+#endif
+
+static int si_notify_smc_display_change(struct amdgpu_device *adev,
+				     bool has_display)
+{
+	PPSMC_Msg msg = has_display ?
+		PPSMC_MSG_HasDisplay : PPSMC_MSG_NoDisplay;
+
+	return (amdgpu_si_send_msg_to_smc(adev, msg) == PPSMC_Result_OK) ?
+		0 : -EINVAL;
+}
+
+static void si_program_response_times(struct amdgpu_device *adev)
+{
+	u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
+	u32 vddc_dly, acpi_dly, vbi_dly;
+	u32 reference_clock;
+
+	si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
+
+	voltage_response_time = (u32)adev->pm.dpm.voltage_response_time;
+	backbias_response_time = (u32)adev->pm.dpm.backbias_response_time;
+
+	if (voltage_response_time == 0)
+		voltage_response_time = 1000;
+
+	acpi_delay_time = 15000;
+	vbi_time_out = 100000;
+
+	reference_clock = amdgpu_asic_get_xclk(adev);
+
+	vddc_dly = (voltage_response_time  * reference_clock) / 100;
+	acpi_dly = (acpi_delay_time * reference_clock) / 100;
+	vbi_dly  = (vbi_time_out * reference_clock) / 100;
+
+	si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
+	si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
+	si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
+	si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
+}
+
+static void si_program_ds_registers(struct amdgpu_device *adev)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	u32 tmp;
+
+	/* DEEP_SLEEP_CLK_SEL field should be 0x10 on tahiti A0 */
+	if (adev->asic_type == CHIP_TAHITI && adev->rev_id == 0x0)
+		tmp = 0x10;
+	else
+		tmp = 0x1;
+
+	if (eg_pi->sclk_deep_sleep) {
+		WREG32_P(MISC_CLK_CNTL, DEEP_SLEEP_CLK_SEL(tmp), ~DEEP_SLEEP_CLK_SEL_MASK);
+		WREG32_P(CG_SPLL_AUTOSCALE_CNTL, AUTOSCALE_ON_SS_CLEAR,
+			 ~AUTOSCALE_ON_SS_CLEAR);
+	}
+}
+
+static void si_program_display_gap(struct amdgpu_device *adev)
+{
+	u32 tmp, pipe;
+	int i;
+
+	tmp = RREG32(CG_DISPLAY_GAP_CNTL) & ~(DISP1_GAP_MASK | DISP2_GAP_MASK);
+	if (adev->pm.dpm.new_active_crtc_count > 0)
+		tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM);
+	else
+		tmp |= DISP1_GAP(R600_PM_DISPLAY_GAP_IGNORE);
+
+	if (adev->pm.dpm.new_active_crtc_count > 1)
+		tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM);
+	else
+		tmp |= DISP2_GAP(R600_PM_DISPLAY_GAP_IGNORE);
+
+	WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+
+	tmp = RREG32(DCCG_DISP_SLOW_SELECT_REG);
+	pipe = (tmp & DCCG_DISP1_SLOW_SELECT_MASK) >> DCCG_DISP1_SLOW_SELECT_SHIFT;
+
+	if ((adev->pm.dpm.new_active_crtc_count > 0) &&
+	    (!(adev->pm.dpm.new_active_crtcs & (1 << pipe)))) {
+		/* find the first active crtc */
+		for (i = 0; i < adev->mode_info.num_crtc; i++) {
+			if (adev->pm.dpm.new_active_crtcs & (1 << i))
+				break;
+		}
+		if (i == adev->mode_info.num_crtc)
+			pipe = 0;
+		else
+			pipe = i;
+
+		tmp &= ~DCCG_DISP1_SLOW_SELECT_MASK;
+		tmp |= DCCG_DISP1_SLOW_SELECT(pipe);
+		WREG32(DCCG_DISP_SLOW_SELECT_REG, tmp);
+	}
+
+	/* Setting this to false forces the performance state to low if the crtcs are disabled.
+	 * This can be a problem on PowerXpress systems or if you want to use the card
+	 * for offscreen rendering or compute if there are no crtcs enabled.
+	 */
+	si_notify_smc_display_change(adev, adev->pm.dpm.new_active_crtc_count > 0);
+}
+
+static void si_enable_spread_spectrum(struct amdgpu_device *adev, bool enable)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+
+	if (enable) {
+		if (pi->sclk_ss)
+			WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
+	} else {
+		WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN);
+		WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
+	}
+}
+
+static void si_setup_bsp(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	u32 xclk = amdgpu_asic_get_xclk(adev);
+
+	r600_calculate_u_and_p(pi->asi,
+			       xclk,
+			       16,
+			       &pi->bsp,
+			       &pi->bsu);
+
+	r600_calculate_u_and_p(pi->pasi,
+			       xclk,
+			       16,
+			       &pi->pbsp,
+			       &pi->pbsu);
+
+
+        pi->dsp = BSP(pi->bsp) | BSU(pi->bsu);
+	pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu);
+
+	WREG32(CG_BSP, pi->dsp);
+}
+
+static void si_program_git(struct amdgpu_device *adev)
+{
+	WREG32_P(CG_GIT, CG_GICST(R600_GICST_DFLT), ~CG_GICST_MASK);
+}
+
+static void si_program_tp(struct amdgpu_device *adev)
+{
+	int i;
+	enum r600_td td = R600_TD_DFLT;
+
+	for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
+		WREG32(CG_FFCT_0 + i, (UTC_0(r600_utc[i]) | DTC_0(r600_dtc[i])));
+
+	if (td == R600_TD_AUTO)
+		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL);
+	else
+		WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL);
+
+	if (td == R600_TD_UP)
+		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE);
+
+	if (td == R600_TD_DOWN)
+		WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE);
+}
+
+static void si_program_tpp(struct amdgpu_device *adev)
+{
+	WREG32(CG_TPC, R600_TPC_DFLT);
+}
+
+static void si_program_sstp(struct amdgpu_device *adev)
+{
+	WREG32(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT)));
+}
+
+static void si_enable_display_gap(struct amdgpu_device *adev)
+{
+	u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
+
+	tmp &= ~(DISP1_GAP_MASK | DISP2_GAP_MASK);
+	tmp |= (DISP1_GAP(R600_PM_DISPLAY_GAP_IGNORE) |
+		DISP2_GAP(R600_PM_DISPLAY_GAP_IGNORE));
+
+	tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
+	tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK) |
+		DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE));
+	WREG32(CG_DISPLAY_GAP_CNTL, tmp);
+}
+
+static void si_program_vc(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+
+	WREG32(CG_FTV, pi->vrc);
+}
+
+static void si_clear_vc(struct amdgpu_device *adev)
+{
+	WREG32(CG_FTV, 0);
+}
+
+static u8 si_get_ddr3_mclk_frequency_ratio(u32 memory_clock)
+{
+	u8 mc_para_index;
+
+	if (memory_clock < 10000)
+		mc_para_index = 0;
+	else if (memory_clock >= 80000)
+		mc_para_index = 0x0f;
+	else
+		mc_para_index = (u8)((memory_clock - 10000) / 5000 + 1);
+	return mc_para_index;
+}
+
+static u8 si_get_mclk_frequency_ratio(u32 memory_clock, bool strobe_mode)
+{
+	u8 mc_para_index;
+
+	if (strobe_mode) {
+		if (memory_clock < 12500)
+			mc_para_index = 0x00;
+		else if (memory_clock > 47500)
+			mc_para_index = 0x0f;
+		else
+			mc_para_index = (u8)((memory_clock - 10000) / 2500);
+	} else {
+		if (memory_clock < 65000)
+			mc_para_index = 0x00;
+		else if (memory_clock > 135000)
+			mc_para_index = 0x0f;
+		else
+			mc_para_index = (u8)((memory_clock - 60000) / 5000);
+	}
+	return mc_para_index;
+}
+
+static u8 si_get_strobe_mode_settings(struct amdgpu_device *adev, u32 mclk)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	bool strobe_mode = false;
+	u8 result = 0;
+
+	if (mclk <= pi->mclk_strobe_mode_threshold)
+		strobe_mode = true;
+
+	if (adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5)
+		result = si_get_mclk_frequency_ratio(mclk, strobe_mode);
+	else
+		result = si_get_ddr3_mclk_frequency_ratio(mclk);
+
+	if (strobe_mode)
+		result |= SISLANDS_SMC_STROBE_ENABLE;
+
+	return result;
+}
+
+static int si_upload_firmware(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	amdgpu_si_reset_smc(adev);
+	amdgpu_si_smc_clock(adev, false);
+
+	return amdgpu_si_load_smc_ucode(adev, si_pi->sram_end);
+}
+
+static bool si_validate_phase_shedding_tables(struct amdgpu_device *adev,
+					      const struct atom_voltage_table *table,
+					      const struct amdgpu_phase_shedding_limits_table *limits)
+{
+	u32 data, num_bits, num_levels;
+
+	if ((table == NULL) || (limits == NULL))
+		return false;
+
+	data = table->mask_low;
+
+	num_bits = hweight32(data);
+
+	if (num_bits == 0)
+		return false;
+
+	num_levels = (1 << num_bits);
+
+	if (table->count != num_levels)
+		return false;
+
+	if (limits->count != (num_levels - 1))
+		return false;
+
+	return true;
+}
+
+static void si_trim_voltage_table_to_fit_state_table(struct amdgpu_device *adev,
+					      u32 max_voltage_steps,
+					      struct atom_voltage_table *voltage_table)
+{
+	unsigned int i, diff;
+
+	if (voltage_table->count <= max_voltage_steps)
+		return;
+
+	diff = voltage_table->count - max_voltage_steps;
+
+	for (i= 0; i < max_voltage_steps; i++)
+		voltage_table->entries[i] = voltage_table->entries[i + diff];
+
+	voltage_table->count = max_voltage_steps;
+}
+
+static int si_get_svi2_voltage_table(struct amdgpu_device *adev,
+				     struct amdgpu_clock_voltage_dependency_table *voltage_dependency_table,
+				     struct atom_voltage_table *voltage_table)
+{
+	u32 i;
+
+	if (voltage_dependency_table == NULL)
+		return -EINVAL;
+
+	voltage_table->mask_low = 0;
+	voltage_table->phase_delay = 0;
+
+	voltage_table->count = voltage_dependency_table->count;
+	for (i = 0; i < voltage_table->count; i++) {
+		voltage_table->entries[i].value = voltage_dependency_table->entries[i].v;
+		voltage_table->entries[i].smio_low = 0;
+	}
+
+	return 0;
+}
+
+static int si_construct_voltage_tables(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	int ret;
+
+	if (pi->voltage_control) {
+		ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_VDDC,
+						    VOLTAGE_OBJ_GPIO_LUT, &eg_pi->vddc_voltage_table);
+		if (ret)
+			return ret;
+
+		if (eg_pi->vddc_voltage_table.count > SISLANDS_MAX_NO_VREG_STEPS)
+			si_trim_voltage_table_to_fit_state_table(adev,
+								 SISLANDS_MAX_NO_VREG_STEPS,
+								 &eg_pi->vddc_voltage_table);
+	} else if (si_pi->voltage_control_svi2) {
+		ret = si_get_svi2_voltage_table(adev,
+						&adev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
+						&eg_pi->vddc_voltage_table);
+		if (ret)
+			return ret;
+	} else {
+		return -EINVAL;
+	}
+
+	if (eg_pi->vddci_control) {
+		ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_VDDCI,
+						    VOLTAGE_OBJ_GPIO_LUT, &eg_pi->vddci_voltage_table);
+		if (ret)
+			return ret;
+
+		if (eg_pi->vddci_voltage_table.count > SISLANDS_MAX_NO_VREG_STEPS)
+			si_trim_voltage_table_to_fit_state_table(adev,
+								 SISLANDS_MAX_NO_VREG_STEPS,
+								 &eg_pi->vddci_voltage_table);
+	}
+	if (si_pi->vddci_control_svi2) {
+		ret = si_get_svi2_voltage_table(adev,
+						&adev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
+						&eg_pi->vddci_voltage_table);
+		if (ret)
+			return ret;
+	}
+
+	if (pi->mvdd_control) {
+		ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_MVDDC,
+						    VOLTAGE_OBJ_GPIO_LUT, &si_pi->mvdd_voltage_table);
+
+		if (ret) {
+			pi->mvdd_control = false;
+			return ret;
+		}
+
+		if (si_pi->mvdd_voltage_table.count == 0) {
+			pi->mvdd_control = false;
+			return -EINVAL;
+		}
+
+		if (si_pi->mvdd_voltage_table.count > SISLANDS_MAX_NO_VREG_STEPS)
+			si_trim_voltage_table_to_fit_state_table(adev,
+								 SISLANDS_MAX_NO_VREG_STEPS,
+								 &si_pi->mvdd_voltage_table);
+	}
+
+	if (si_pi->vddc_phase_shed_control) {
+		ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_VDDC,
+						    VOLTAGE_OBJ_PHASE_LUT, &si_pi->vddc_phase_shed_table);
+		if (ret)
+			si_pi->vddc_phase_shed_control = false;
+
+		if ((si_pi->vddc_phase_shed_table.count == 0) ||
+		    (si_pi->vddc_phase_shed_table.count > SISLANDS_MAX_NO_VREG_STEPS))
+			si_pi->vddc_phase_shed_control = false;
+	}
+
+	return 0;
+}
+
+static void si_populate_smc_voltage_table(struct amdgpu_device *adev,
+					  const struct atom_voltage_table *voltage_table,
+					  SISLANDS_SMC_STATETABLE *table)
+{
+	unsigned int i;
+
+	for (i = 0; i < voltage_table->count; i++)
+		table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
+}
+
+static int si_populate_smc_voltage_tables(struct amdgpu_device *adev,
+					  SISLANDS_SMC_STATETABLE *table)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u8 i;
+
+	if (si_pi->voltage_control_svi2) {
+		si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svc,
+			si_pi->svc_gpio_id);
+		si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svd,
+			si_pi->svd_gpio_id);
+		si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_svi_rework_plat_type,
+					   2);
+	} else {
+		if (eg_pi->vddc_voltage_table.count) {
+			si_populate_smc_voltage_table(adev, &eg_pi->vddc_voltage_table, table);
+			table->voltageMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDC] =
+				cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
+
+			for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
+				if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
+					table->maxVDDCIndexInPPTable = i;
+					break;
+				}
+			}
+		}
+
+		if (eg_pi->vddci_voltage_table.count) {
+			si_populate_smc_voltage_table(adev, &eg_pi->vddci_voltage_table, table);
+
+			table->voltageMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDCI] =
+				cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
+		}
+
+
+		if (si_pi->mvdd_voltage_table.count) {
+			si_populate_smc_voltage_table(adev, &si_pi->mvdd_voltage_table, table);
+
+			table->voltageMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_MVDD] =
+				cpu_to_be32(si_pi->mvdd_voltage_table.mask_low);
+		}
+
+		if (si_pi->vddc_phase_shed_control) {
+			if (si_validate_phase_shedding_tables(adev, &si_pi->vddc_phase_shed_table,
+							      &adev->pm.dpm.dyn_state.phase_shedding_limits_table)) {
+				si_populate_smc_voltage_table(adev, &si_pi->vddc_phase_shed_table, table);
+
+				table->phaseMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDC_PHASE_SHEDDING] =
+					cpu_to_be32(si_pi->vddc_phase_shed_table.mask_low);
+
+				si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_phase_shedding_delay,
+							   (u32)si_pi->vddc_phase_shed_table.phase_delay);
+			} else {
+				si_pi->vddc_phase_shed_control = false;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int si_populate_voltage_value(struct amdgpu_device *adev,
+				     const struct atom_voltage_table *table,
+				     u16 value, SISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+	unsigned int i;
+
+	for (i = 0; i < table->count; i++) {
+		if (value <= table->entries[i].value) {
+			voltage->index = (u8)i;
+			voltage->value = cpu_to_be16(table->entries[i].value);
+			break;
+		}
+	}
+
+	if (i >= table->count)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int si_populate_mvdd_value(struct amdgpu_device *adev, u32 mclk,
+				  SISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	if (pi->mvdd_control) {
+		if (mclk <= pi->mvdd_split_frequency)
+			voltage->index = 0;
+		else
+			voltage->index = (u8)(si_pi->mvdd_voltage_table.count) - 1;
+
+		voltage->value = cpu_to_be16(si_pi->mvdd_voltage_table.entries[voltage->index].value);
+	}
+	return 0;
+}
+
+static int si_get_std_voltage_value(struct amdgpu_device *adev,
+				    SISLANDS_SMC_VOLTAGE_VALUE *voltage,
+				    u16 *std_voltage)
+{
+	u16 v_index;
+	bool voltage_found = false;
+	*std_voltage = be16_to_cpu(voltage->value);
+
+	if (adev->pm.dpm.dyn_state.cac_leakage_table.entries) {
+		if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_NEW_CAC_VOLTAGE) {
+			if (adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries == NULL)
+				return -EINVAL;
+
+			for (v_index = 0; (u32)v_index < adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
+				if (be16_to_cpu(voltage->value) ==
+				    (u16)adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
+					voltage_found = true;
+					if ((u32)v_index < adev->pm.dpm.dyn_state.cac_leakage_table.count)
+						*std_voltage =
+							adev->pm.dpm.dyn_state.cac_leakage_table.entries[v_index].vddc;
+					else
+						*std_voltage =
+							adev->pm.dpm.dyn_state.cac_leakage_table.entries[adev->pm.dpm.dyn_state.cac_leakage_table.count-1].vddc;
+					break;
+				}
+			}
+
+			if (!voltage_found) {
+				for (v_index = 0; (u32)v_index < adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
+					if (be16_to_cpu(voltage->value) <=
+					    (u16)adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
+						voltage_found = true;
+						if ((u32)v_index < adev->pm.dpm.dyn_state.cac_leakage_table.count)
+							*std_voltage =
+								adev->pm.dpm.dyn_state.cac_leakage_table.entries[v_index].vddc;
+						else
+							*std_voltage =
+								adev->pm.dpm.dyn_state.cac_leakage_table.entries[adev->pm.dpm.dyn_state.cac_leakage_table.count-1].vddc;
+						break;
+					}
+				}
+			}
+		} else {
+			if ((u32)voltage->index < adev->pm.dpm.dyn_state.cac_leakage_table.count)
+				*std_voltage = adev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
+		}
+	}
+
+	return 0;
+}
+
+static int si_populate_std_voltage_value(struct amdgpu_device *adev,
+					 u16 value, u8 index,
+					 SISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+	voltage->index = index;
+	voltage->value = cpu_to_be16(value);
+
+	return 0;
+}
+
+static int si_populate_phase_shedding_value(struct amdgpu_device *adev,
+					    const struct amdgpu_phase_shedding_limits_table *limits,
+					    u16 voltage, u32 sclk, u32 mclk,
+					    SISLANDS_SMC_VOLTAGE_VALUE *smc_voltage)
+{
+	unsigned int i;
+
+	for (i = 0; i < limits->count; i++) {
+		if ((voltage <= limits->entries[i].voltage) &&
+		    (sclk <= limits->entries[i].sclk) &&
+		    (mclk <= limits->entries[i].mclk))
+			break;
+	}
+
+	smc_voltage->phase_settings = (u8)i;
+
+	return 0;
+}
+
+static int si_init_arb_table_index(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 tmp;
+	int ret;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev, si_pi->arb_table_start,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	tmp &= 0x00FFFFFF;
+	tmp |= MC_CG_ARB_FREQ_F1 << 24;
+
+	return amdgpu_si_write_smc_sram_dword(adev, si_pi->arb_table_start,
+					      tmp, si_pi->sram_end);
+}
+
+static int si_initial_switch_from_arb_f0_to_f1(struct amdgpu_device *adev)
+{
+	return ni_copy_and_switch_arb_sets(adev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
+}
+
+static int si_reset_to_default(struct amdgpu_device *adev)
+{
+	return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_ResetToDefaults) == PPSMC_Result_OK) ?
+		0 : -EINVAL;
+}
+
+static int si_force_switch_to_arb_f0(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 tmp;
+	int ret;
+
+	ret = amdgpu_si_read_smc_sram_dword(adev, si_pi->arb_table_start,
+					    &tmp, si_pi->sram_end);
+	if (ret)
+		return ret;
+
+	tmp = (tmp >> 24) & 0xff;
+
+	if (tmp == MC_CG_ARB_FREQ_F0)
+		return 0;
+
+	return ni_copy_and_switch_arb_sets(adev, tmp, MC_CG_ARB_FREQ_F0);
+}
+
+static u32 si_calculate_memory_refresh_rate(struct amdgpu_device *adev,
+					    u32 engine_clock)
+{
+	u32 dram_rows;
+	u32 dram_refresh_rate;
+	u32 mc_arb_rfsh_rate;
+	u32 tmp = (RREG32(MC_ARB_RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
+
+	if (tmp >= 4)
+		dram_rows = 16384;
+	else
+		dram_rows = 1 << (tmp + 10);
+
+	dram_refresh_rate = 1 << ((RREG32(MC_SEQ_MISC0) & 0x3) + 3);
+	mc_arb_rfsh_rate = ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64;
+
+	return mc_arb_rfsh_rate;
+}
+
+static int si_populate_memory_timing_parameters(struct amdgpu_device *adev,
+						struct rv7xx_pl *pl,
+						SMC_SIslands_MCArbDramTimingRegisterSet *arb_regs)
+{
+	u32 dram_timing;
+	u32 dram_timing2;
+	u32 burst_time;
+
+	arb_regs->mc_arb_rfsh_rate =
+		(u8)si_calculate_memory_refresh_rate(adev, pl->sclk);
+
+	amdgpu_atombios_set_engine_dram_timings(adev,
+					    pl->sclk,
+		                            pl->mclk);
+
+	dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
+	dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
+	burst_time = RREG32(MC_ARB_BURST_TIME) & STATE0_MASK;
+
+	arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
+	arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
+	arb_regs->mc_arb_burst_time = (u8)burst_time;
+
+	return 0;
+}
+
+static int si_do_program_memory_timing_parameters(struct amdgpu_device *adev,
+						  struct amdgpu_ps *amdgpu_state,
+						  unsigned int first_arb_set)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct  si_ps *state = si_get_ps(amdgpu_state);
+	SMC_SIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
+	int i, ret = 0;
+
+	for (i = 0; i < state->performance_level_count; i++) {
+		ret = si_populate_memory_timing_parameters(adev, &state->performance_levels[i], &arb_regs);
+		if (ret)
+			break;
+		ret = amdgpu_si_copy_bytes_to_smc(adev,
+						  si_pi->arb_table_start +
+						  offsetof(SMC_SIslands_MCArbDramTimingRegisters, data) +
+						  sizeof(SMC_SIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i),
+						  (u8 *)&arb_regs,
+						  sizeof(SMC_SIslands_MCArbDramTimingRegisterSet),
+						  si_pi->sram_end);
+		if (ret)
+			break;
+	}
+
+	return ret;
+}
+
+static int si_program_memory_timing_parameters(struct amdgpu_device *adev,
+					       struct amdgpu_ps *amdgpu_new_state)
+{
+	return si_do_program_memory_timing_parameters(adev, amdgpu_new_state,
+						      SISLANDS_DRIVER_STATE_ARB_INDEX);
+}
+
+static int si_populate_initial_mvdd_value(struct amdgpu_device *adev,
+					  struct SISLANDS_SMC_VOLTAGE_VALUE *voltage)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	if (pi->mvdd_control)
+		return si_populate_voltage_value(adev, &si_pi->mvdd_voltage_table,
+						 si_pi->mvdd_bootup_value, voltage);
+
+	return 0;
+}
+
+static int si_populate_smc_initial_state(struct amdgpu_device *adev,
+					 struct amdgpu_ps *amdgpu_initial_state,
+					 SISLANDS_SMC_STATETABLE *table)
+{
+	struct  si_ps *initial_state = si_get_ps(amdgpu_initial_state);
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 reg;
+	int ret;
+
+	table->initialState.levels[0].mclk.vDLL_CNTL =
+		cpu_to_be32(si_pi->clock_registers.dll_cntl);
+	table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
+		cpu_to_be32(si_pi->clock_registers.mclk_pwrmgt_cntl);
+	table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
+		cpu_to_be32(si_pi->clock_registers.mpll_ad_func_cntl);
+	table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
+		cpu_to_be32(si_pi->clock_registers.mpll_dq_func_cntl);
+	table->initialState.levels[0].mclk.vMPLL_FUNC_CNTL =
+		cpu_to_be32(si_pi->clock_registers.mpll_func_cntl);
+	table->initialState.levels[0].mclk.vMPLL_FUNC_CNTL_1 =
+		cpu_to_be32(si_pi->clock_registers.mpll_func_cntl_1);
+	table->initialState.levels[0].mclk.vMPLL_FUNC_CNTL_2 =
+		cpu_to_be32(si_pi->clock_registers.mpll_func_cntl_2);
+	table->initialState.levels[0].mclk.vMPLL_SS =
+		cpu_to_be32(si_pi->clock_registers.mpll_ss1);
+	table->initialState.levels[0].mclk.vMPLL_SS2 =
+		cpu_to_be32(si_pi->clock_registers.mpll_ss2);
+
+	table->initialState.levels[0].mclk.mclk_value =
+		cpu_to_be32(initial_state->performance_levels[0].mclk);
+
+	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
+		cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl);
+	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
+		cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl_2);
+	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
+		cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl_3);
+	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
+		cpu_to_be32(si_pi->clock_registers.cg_spll_func_cntl_4);
+	table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
+		cpu_to_be32(si_pi->clock_registers.cg_spll_spread_spectrum);
+	table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2  =
+		cpu_to_be32(si_pi->clock_registers.cg_spll_spread_spectrum_2);
+
+	table->initialState.levels[0].sclk.sclk_value =
+		cpu_to_be32(initial_state->performance_levels[0].sclk);
+
+	table->initialState.levels[0].arbRefreshState =
+		SISLANDS_INITIAL_STATE_ARB_INDEX;
+
+	table->initialState.levels[0].ACIndex = 0;
+
+	ret = si_populate_voltage_value(adev, &eg_pi->vddc_voltage_table,
+					initial_state->performance_levels[0].vddc,
+					&table->initialState.levels[0].vddc);
+
+	if (!ret) {
+		u16 std_vddc;
+
+		ret = si_get_std_voltage_value(adev,
+					       &table->initialState.levels[0].vddc,
+					       &std_vddc);
+		if (!ret)
+			si_populate_std_voltage_value(adev, std_vddc,
+						      table->initialState.levels[0].vddc.index,
+						      &table->initialState.levels[0].std_vddc);
+	}
+
+	if (eg_pi->vddci_control)
+		si_populate_voltage_value(adev,
+					  &eg_pi->vddci_voltage_table,
+					  initial_state->performance_levels[0].vddci,
+					  &table->initialState.levels[0].vddci);
+
+	if (si_pi->vddc_phase_shed_control)
+		si_populate_phase_shedding_value(adev,
+						 &adev->pm.dpm.dyn_state.phase_shedding_limits_table,
+						 initial_state->performance_levels[0].vddc,
+						 initial_state->performance_levels[0].sclk,
+						 initial_state->performance_levels[0].mclk,
+						 &table->initialState.levels[0].vddc);
+
+	si_populate_initial_mvdd_value(adev, &table->initialState.levels[0].mvdd);
+
+	reg = CG_R(0xffff) | CG_L(0);
+	table->initialState.levels[0].aT = cpu_to_be32(reg);
+	table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
+	table->initialState.levels[0].gen2PCIE = (u8)si_pi->boot_pcie_gen;
+
+	if (adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5) {
+		table->initialState.levels[0].strobeMode =
+			si_get_strobe_mode_settings(adev,
+						    initial_state->performance_levels[0].mclk);
+
+		if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
+			table->initialState.levels[0].mcFlags = SISLANDS_SMC_MC_EDC_RD_FLAG | SISLANDS_SMC_MC_EDC_WR_FLAG;
+		else
+			table->initialState.levels[0].mcFlags =  0;
+	}
+
+	table->initialState.levelCount = 1;
+
+	table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
+
+	table->initialState.levels[0].dpm2.MaxPS = 0;
+	table->initialState.levels[0].dpm2.NearTDPDec = 0;
+	table->initialState.levels[0].dpm2.AboveSafeInc = 0;
+	table->initialState.levels[0].dpm2.BelowSafeInc = 0;
+	table->initialState.levels[0].dpm2.PwrEfficiencyRatio = 0;
+
+	reg = MIN_POWER_MASK | MAX_POWER_MASK;
+	table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
+
+	reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
+	table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
+
+	return 0;
+}
+
+static int si_populate_smc_acpi_state(struct amdgpu_device *adev,
+				      SISLANDS_SMC_STATETABLE *table)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 spll_func_cntl = si_pi->clock_registers.cg_spll_func_cntl;
+	u32 spll_func_cntl_2 = si_pi->clock_registers.cg_spll_func_cntl_2;
+	u32 spll_func_cntl_3 = si_pi->clock_registers.cg_spll_func_cntl_3;
+	u32 spll_func_cntl_4 = si_pi->clock_registers.cg_spll_func_cntl_4;
+	u32 dll_cntl = si_pi->clock_registers.dll_cntl;
+	u32 mclk_pwrmgt_cntl = si_pi->clock_registers.mclk_pwrmgt_cntl;
+	u32 mpll_ad_func_cntl = si_pi->clock_registers.mpll_ad_func_cntl;
+	u32 mpll_dq_func_cntl = si_pi->clock_registers.mpll_dq_func_cntl;
+	u32 mpll_func_cntl = si_pi->clock_registers.mpll_func_cntl;
+	u32 mpll_func_cntl_1 = si_pi->clock_registers.mpll_func_cntl_1;
+	u32 mpll_func_cntl_2 = si_pi->clock_registers.mpll_func_cntl_2;
+	u32 reg;
+	int ret;
+
+	table->ACPIState = table->initialState;
+
+	table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+	if (pi->acpi_vddc) {
+		ret = si_populate_voltage_value(adev, &eg_pi->vddc_voltage_table,
+						pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
+		if (!ret) {
+			u16 std_vddc;
+
+			ret = si_get_std_voltage_value(adev,
+						       &table->ACPIState.levels[0].vddc, &std_vddc);
+			if (!ret)
+				si_populate_std_voltage_value(adev, std_vddc,
+							      table->ACPIState.levels[0].vddc.index,
+							      &table->ACPIState.levels[0].std_vddc);
+		}
+		table->ACPIState.levels[0].gen2PCIE = si_pi->acpi_pcie_gen;
+
+		if (si_pi->vddc_phase_shed_control) {
+			si_populate_phase_shedding_value(adev,
+							 &adev->pm.dpm.dyn_state.phase_shedding_limits_table,
+							 pi->acpi_vddc,
+							 0,
+							 0,
+							 &table->ACPIState.levels[0].vddc);
+		}
+	} else {
+		ret = si_populate_voltage_value(adev, &eg_pi->vddc_voltage_table,
+						pi->min_vddc_in_table, &table->ACPIState.levels[0].vddc);
+		if (!ret) {
+			u16 std_vddc;
+
+			ret = si_get_std_voltage_value(adev,
+						       &table->ACPIState.levels[0].vddc, &std_vddc);
+
+			if (!ret)
+				si_populate_std_voltage_value(adev, std_vddc,
+							      table->ACPIState.levels[0].vddc.index,
+							      &table->ACPIState.levels[0].std_vddc);
+		}
+		table->ACPIState.levels[0].gen2PCIE = (u8)r600_get_pcie_gen_support(adev,
+										    si_pi->sys_pcie_mask,
+										    si_pi->boot_pcie_gen,
+										    AMDGPU_PCIE_GEN1);
+
+		if (si_pi->vddc_phase_shed_control)
+			si_populate_phase_shedding_value(adev,
+							 &adev->pm.dpm.dyn_state.phase_shedding_limits_table,
+							 pi->min_vddc_in_table,
+							 0,
+							 0,
+							 &table->ACPIState.levels[0].vddc);
+	}
+
+	if (pi->acpi_vddc) {
+		if (eg_pi->acpi_vddci)
+			si_populate_voltage_value(adev, &eg_pi->vddci_voltage_table,
+						  eg_pi->acpi_vddci,
+						  &table->ACPIState.levels[0].vddci);
+	}
+
+	mclk_pwrmgt_cntl |= MRDCK0_RESET | MRDCK1_RESET;
+	mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB);
+
+	dll_cntl &= ~(MRDCK0_BYPASS | MRDCK1_BYPASS);
+
+	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+	spll_func_cntl_2 |= SCLK_MUX_SEL(4);
+
+	table->ACPIState.levels[0].mclk.vDLL_CNTL =
+		cpu_to_be32(dll_cntl);
+	table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
+		cpu_to_be32(mclk_pwrmgt_cntl);
+	table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
+		cpu_to_be32(mpll_ad_func_cntl);
+	table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
+		cpu_to_be32(mpll_dq_func_cntl);
+	table->ACPIState.levels[0].mclk.vMPLL_FUNC_CNTL =
+		cpu_to_be32(mpll_func_cntl);
+	table->ACPIState.levels[0].mclk.vMPLL_FUNC_CNTL_1 =
+		cpu_to_be32(mpll_func_cntl_1);
+	table->ACPIState.levels[0].mclk.vMPLL_FUNC_CNTL_2 =
+		cpu_to_be32(mpll_func_cntl_2);
+	table->ACPIState.levels[0].mclk.vMPLL_SS =
+		cpu_to_be32(si_pi->clock_registers.mpll_ss1);
+	table->ACPIState.levels[0].mclk.vMPLL_SS2 =
+		cpu_to_be32(si_pi->clock_registers.mpll_ss2);
+
+	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
+		cpu_to_be32(spll_func_cntl);
+	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
+		cpu_to_be32(spll_func_cntl_2);
+	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
+		cpu_to_be32(spll_func_cntl_3);
+	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
+		cpu_to_be32(spll_func_cntl_4);
+
+	table->ACPIState.levels[0].mclk.mclk_value = 0;
+	table->ACPIState.levels[0].sclk.sclk_value = 0;
+
+	si_populate_mvdd_value(adev, 0, &table->ACPIState.levels[0].mvdd);
+
+	if (eg_pi->dynamic_ac_timing)
+		table->ACPIState.levels[0].ACIndex = 0;
+
+	table->ACPIState.levels[0].dpm2.MaxPS = 0;
+	table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
+	table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
+	table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
+	table->ACPIState.levels[0].dpm2.PwrEfficiencyRatio = 0;
+
+	reg = MIN_POWER_MASK | MAX_POWER_MASK;
+	table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
+
+	reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
+	table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
+
+	return 0;
+}
+
+static int si_populate_ulv_state(struct amdgpu_device *adev,
+				 SISLANDS_SMC_SWSTATE *state)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct si_ulv_param *ulv = &si_pi->ulv;
+	u32 sclk_in_sr = 1350; /* ??? */
+	int ret;
+
+	ret = si_convert_power_level_to_smc(adev, &ulv->pl,
+					    &state->levels[0]);
+	if (!ret) {
+		if (eg_pi->sclk_deep_sleep) {
+			if (sclk_in_sr <= SCLK_MIN_DEEPSLEEP_FREQ)
+				state->levels[0].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_BYPASS;
+			else
+				state->levels[0].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE;
+		}
+		if (ulv->one_pcie_lane_in_ulv)
+			state->flags |= PPSMC_SWSTATE_FLAG_PCIE_X1;
+		state->levels[0].arbRefreshState = (u8)(SISLANDS_ULV_STATE_ARB_INDEX);
+		state->levels[0].ACIndex = 1;
+		state->levels[0].std_vddc = state->levels[0].vddc;
+		state->levelCount = 1;
+
+		state->flags |= PPSMC_SWSTATE_FLAG_DC;
+	}
+
+	return ret;
+}
+
+static int si_program_ulv_memory_timing_parameters(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct si_ulv_param *ulv = &si_pi->ulv;
+	SMC_SIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
+	int ret;
+
+	ret = si_populate_memory_timing_parameters(adev, &ulv->pl,
+						   &arb_regs);
+	if (ret)
+		return ret;
+
+	si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_ulv_volt_change_delay,
+				   ulv->volt_change_delay);
+
+	ret = amdgpu_si_copy_bytes_to_smc(adev,
+					  si_pi->arb_table_start +
+					  offsetof(SMC_SIslands_MCArbDramTimingRegisters, data) +
+					  sizeof(SMC_SIslands_MCArbDramTimingRegisterSet) * SISLANDS_ULV_STATE_ARB_INDEX,
+					  (u8 *)&arb_regs,
+					  sizeof(SMC_SIslands_MCArbDramTimingRegisterSet),
+					  si_pi->sram_end);
+
+	return ret;
+}
+
+static void si_get_mvdd_configuration(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+
+	pi->mvdd_split_frequency = 30000;
+}
+
+static int si_init_smc_table(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct amdgpu_ps *amdgpu_boot_state = adev->pm.dpm.boot_ps;
+	const struct si_ulv_param *ulv = &si_pi->ulv;
+	SISLANDS_SMC_STATETABLE  *table = &si_pi->smc_statetable;
+	int ret;
+	u32 lane_width;
+	u32 vr_hot_gpio;
+
+	si_populate_smc_voltage_tables(adev, table);
+
+	switch (adev->pm.int_thermal_type) {
+	case THERMAL_TYPE_SI:
+	case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
+		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
+		break;
+	case THERMAL_TYPE_NONE:
+		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
+		break;
+	default:
+		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
+		break;
+	}
+
+	if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
+		table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+	if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT) {
+		if ((adev->pdev->device != 0x6818) && (adev->pdev->device != 0x6819))
+			table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
+	}
+
+	if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
+		table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+	if (adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5)
+		table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+	if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REVERT_GPIO5_POLARITY)
+		table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_GPIO5_POLARITY_HIGH;
+
+	if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_VRHOT_GPIO_CONFIGURABLE) {
+		table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT_PROG_GPIO;
+		vr_hot_gpio = adev->pm.dpm.backbias_response_time;
+		si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_vr_hot_gpio,
+					   vr_hot_gpio);
+	}
+
+	ret = si_populate_smc_initial_state(adev, amdgpu_boot_state, table);
+	if (ret)
+		return ret;
+
+	ret = si_populate_smc_acpi_state(adev, table);
+	if (ret)
+		return ret;
+
+	table->driverState = table->initialState;
+
+	ret = si_do_program_memory_timing_parameters(adev, amdgpu_boot_state,
+						     SISLANDS_INITIAL_STATE_ARB_INDEX);
+	if (ret)
+		return ret;
+
+	if (ulv->supported && ulv->pl.vddc) {
+		ret = si_populate_ulv_state(adev, &table->ULVState);
+		if (ret)
+			return ret;
+
+		ret = si_program_ulv_memory_timing_parameters(adev);
+		if (ret)
+			return ret;
+
+		WREG32(CG_ULV_CONTROL, ulv->cg_ulv_control);
+		WREG32(CG_ULV_PARAMETER, ulv->cg_ulv_parameter);
+
+		lane_width = amdgpu_get_pcie_lanes(adev);
+		si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_non_ulv_pcie_link_width, lane_width);
+	} else {
+		table->ULVState = table->initialState;
+	}
+
+	return amdgpu_si_copy_bytes_to_smc(adev, si_pi->state_table_start,
+					   (u8 *)table, sizeof(SISLANDS_SMC_STATETABLE),
+					   si_pi->sram_end);
+}
+
+static int si_calculate_sclk_params(struct amdgpu_device *adev,
+				    u32 engine_clock,
+				    SISLANDS_SMC_SCLK_VALUE *sclk)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct atom_clock_dividers dividers;
+	u32 spll_func_cntl = si_pi->clock_registers.cg_spll_func_cntl;
+	u32 spll_func_cntl_2 = si_pi->clock_registers.cg_spll_func_cntl_2;
+	u32 spll_func_cntl_3 = si_pi->clock_registers.cg_spll_func_cntl_3;
+	u32 spll_func_cntl_4 = si_pi->clock_registers.cg_spll_func_cntl_4;
+	u32 cg_spll_spread_spectrum = si_pi->clock_registers.cg_spll_spread_spectrum;
+	u32 cg_spll_spread_spectrum_2 = si_pi->clock_registers.cg_spll_spread_spectrum_2;
+	u64 tmp;
+	u32 reference_clock = adev->clock.spll.reference_freq;
+	u32 reference_divider;
+	u32 fbdiv;
+	int ret;
+
+	ret = amdgpu_atombios_get_clock_dividers(adev, COMPUTE_ENGINE_PLL_PARAM,
+					     engine_clock, false, &dividers);
+	if (ret)
+		return ret;
+
+	reference_divider = 1 + dividers.ref_div;
+
+	tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16384;
+	do_div(tmp, reference_clock);
+	fbdiv = (u32) tmp;
+
+	spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
+	spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
+	spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
+
+	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
+	spll_func_cntl_2 |= SCLK_MUX_SEL(2);
+
+	spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
+	spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
+	spll_func_cntl_3 |= SPLL_DITHEN;
+
+	if (pi->sclk_ss) {
+		struct amdgpu_atom_ss ss;
+		u32 vco_freq = engine_clock * dividers.post_div;
+
+		if (amdgpu_atombios_get_asic_ss_info(adev, &ss,
+						     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
+			u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
+			u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
+
+			cg_spll_spread_spectrum &= ~CLK_S_MASK;
+			cg_spll_spread_spectrum |= CLK_S(clk_s);
+			cg_spll_spread_spectrum |= SSEN;
+
+			cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
+			cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
+		}
+	}
+
+	sclk->sclk_value = engine_clock;
+	sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
+	sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
+	sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
+	sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
+	sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
+	sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
+
+	return 0;
+}
+
+static int si_populate_sclk_value(struct amdgpu_device *adev,
+				  u32 engine_clock,
+				  SISLANDS_SMC_SCLK_VALUE *sclk)
+{
+	SISLANDS_SMC_SCLK_VALUE sclk_tmp;
+	int ret;
+
+	ret = si_calculate_sclk_params(adev, engine_clock, &sclk_tmp);
+	if (!ret) {
+		sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
+		sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
+		sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
+		sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
+		sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
+		sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
+		sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
+	}
+
+	return ret;
+}
+
+static int si_populate_mclk_value(struct amdgpu_device *adev,
+				  u32 engine_clock,
+				  u32 memory_clock,
+				  SISLANDS_SMC_MCLK_VALUE *mclk,
+				  bool strobe_mode,
+				  bool dll_state_on)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32  dll_cntl = si_pi->clock_registers.dll_cntl;
+	u32  mclk_pwrmgt_cntl = si_pi->clock_registers.mclk_pwrmgt_cntl;
+	u32  mpll_ad_func_cntl = si_pi->clock_registers.mpll_ad_func_cntl;
+	u32  mpll_dq_func_cntl = si_pi->clock_registers.mpll_dq_func_cntl;
+	u32  mpll_func_cntl = si_pi->clock_registers.mpll_func_cntl;
+	u32  mpll_func_cntl_1 = si_pi->clock_registers.mpll_func_cntl_1;
+	u32  mpll_func_cntl_2 = si_pi->clock_registers.mpll_func_cntl_2;
+	u32  mpll_ss1 = si_pi->clock_registers.mpll_ss1;
+	u32  mpll_ss2 = si_pi->clock_registers.mpll_ss2;
+	struct atom_mpll_param mpll_param;
+	int ret;
+
+	ret = amdgpu_atombios_get_memory_pll_dividers(adev, memory_clock, strobe_mode, &mpll_param);
+	if (ret)
+		return ret;
+
+	mpll_func_cntl &= ~BWCTRL_MASK;
+	mpll_func_cntl |= BWCTRL(mpll_param.bwcntl);
+
+	mpll_func_cntl_1 &= ~(CLKF_MASK | CLKFRAC_MASK | VCO_MODE_MASK);
+	mpll_func_cntl_1 |= CLKF(mpll_param.clkf) |
+		CLKFRAC(mpll_param.clkfrac) | VCO_MODE(mpll_param.vco_mode);
+
+	mpll_ad_func_cntl &= ~YCLK_POST_DIV_MASK;
+	mpll_ad_func_cntl |= YCLK_POST_DIV(mpll_param.post_div);
+
+	if (adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5) {
+		mpll_dq_func_cntl &= ~(YCLK_SEL_MASK | YCLK_POST_DIV_MASK);
+		mpll_dq_func_cntl |= YCLK_SEL(mpll_param.yclk_sel) |
+			YCLK_POST_DIV(mpll_param.post_div);
+	}
+
+	if (pi->mclk_ss) {
+		struct amdgpu_atom_ss ss;
+		u32 freq_nom;
+		u32 tmp;
+		u32 reference_clock = adev->clock.mpll.reference_freq;
+
+		if (adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5)
+			freq_nom = memory_clock * 4;
+		else
+			freq_nom = memory_clock * 2;
+
+		tmp = freq_nom / reference_clock;
+		tmp = tmp * tmp;
+		if (amdgpu_atombios_get_asic_ss_info(adev, &ss,
+		                                     ASIC_INTERNAL_MEMORY_SS, freq_nom)) {
+			u32 clks = reference_clock * 5 / ss.rate;
+			u32 clkv = (u32)((((131 * ss.percentage * ss.rate) / 100) * tmp) / freq_nom);
+
+		        mpll_ss1 &= ~CLKV_MASK;
+		        mpll_ss1 |= CLKV(clkv);
+
+		        mpll_ss2 &= ~CLKS_MASK;
+		        mpll_ss2 |= CLKS(clks);
+		}
+	}
+
+	mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
+	mclk_pwrmgt_cntl |= DLL_SPEED(mpll_param.dll_speed);
+
+	if (dll_state_on)
+		mclk_pwrmgt_cntl |= MRDCK0_PDNB | MRDCK1_PDNB;
+	else
+		mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB);
+
+	mclk->mclk_value = cpu_to_be32(memory_clock);
+	mclk->vMPLL_FUNC_CNTL = cpu_to_be32(mpll_func_cntl);
+	mclk->vMPLL_FUNC_CNTL_1 = cpu_to_be32(mpll_func_cntl_1);
+	mclk->vMPLL_FUNC_CNTL_2 = cpu_to_be32(mpll_func_cntl_2);
+	mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
+	mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
+	mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
+	mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
+	mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
+	mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
+
+	return 0;
+}
+
+static void si_populate_smc_sp(struct amdgpu_device *adev,
+			       struct amdgpu_ps *amdgpu_state,
+			       SISLANDS_SMC_SWSTATE *smc_state)
+{
+	struct  si_ps *ps = si_get_ps(amdgpu_state);
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	int i;
+
+	for (i = 0; i < ps->performance_level_count - 1; i++)
+		smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
+
+	smc_state->levels[ps->performance_level_count - 1].bSP =
+		cpu_to_be32(pi->psp);
+}
+
+static int si_convert_power_level_to_smc(struct amdgpu_device *adev,
+					 struct rv7xx_pl *pl,
+					 SISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	int ret;
+	bool dll_state_on;
+	u16 std_vddc;
+	bool gmc_pg = false;
+
+	if (eg_pi->pcie_performance_request &&
+	    (si_pi->force_pcie_gen != AMDGPU_PCIE_GEN_INVALID))
+		level->gen2PCIE = (u8)si_pi->force_pcie_gen;
+	else
+		level->gen2PCIE = (u8)pl->pcie_gen;
+
+	ret = si_populate_sclk_value(adev, pl->sclk, &level->sclk);
+	if (ret)
+		return ret;
+
+	level->mcFlags =  0;
+
+	if (pi->mclk_stutter_mode_threshold &&
+	    (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
+	    !eg_pi->uvd_enabled &&
+	    (RREG32(DPG_PIPE_STUTTER_CONTROL) & STUTTER_ENABLE) &&
+	    (adev->pm.dpm.new_active_crtc_count <= 2)) {
+		level->mcFlags |= SISLANDS_SMC_MC_STUTTER_EN;
+
+		if (gmc_pg)
+			level->mcFlags |= SISLANDS_SMC_MC_PG_EN;
+	}
+
+	if (adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5) {
+		if (pl->mclk > pi->mclk_edc_enable_threshold)
+			level->mcFlags |= SISLANDS_SMC_MC_EDC_RD_FLAG;
+
+		if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
+			level->mcFlags |= SISLANDS_SMC_MC_EDC_WR_FLAG;
+
+		level->strobeMode = si_get_strobe_mode_settings(adev, pl->mclk);
+
+		if (level->strobeMode & SISLANDS_SMC_STROBE_ENABLE) {
+			if (si_get_mclk_frequency_ratio(pl->mclk, true) >=
+			    ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
+				dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
+			else
+				dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
+		} else {
+			dll_state_on = false;
+		}
+	} else {
+		level->strobeMode = si_get_strobe_mode_settings(adev,
+								pl->mclk);
+
+		dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
+	}
+
+	ret = si_populate_mclk_value(adev,
+				     pl->sclk,
+				     pl->mclk,
+				     &level->mclk,
+				     (level->strobeMode & SISLANDS_SMC_STROBE_ENABLE) != 0, dll_state_on);
+	if (ret)
+		return ret;
+
+	ret = si_populate_voltage_value(adev,
+					&eg_pi->vddc_voltage_table,
+					pl->vddc, &level->vddc);
+	if (ret)
+		return ret;
+
+
+	ret = si_get_std_voltage_value(adev, &level->vddc, &std_vddc);
+	if (ret)
+		return ret;
+
+	ret = si_populate_std_voltage_value(adev, std_vddc,
+					    level->vddc.index, &level->std_vddc);
+	if (ret)
+		return ret;
+
+	if (eg_pi->vddci_control) {
+		ret = si_populate_voltage_value(adev, &eg_pi->vddci_voltage_table,
+						pl->vddci, &level->vddci);
+		if (ret)
+			return ret;
+	}
+
+	if (si_pi->vddc_phase_shed_control) {
+		ret = si_populate_phase_shedding_value(adev,
+						       &adev->pm.dpm.dyn_state.phase_shedding_limits_table,
+						       pl->vddc,
+						       pl->sclk,
+						       pl->mclk,
+						       &level->vddc);
+		if (ret)
+			return ret;
+	}
+
+	level->MaxPoweredUpCU = si_pi->max_cu;
+
+	ret = si_populate_mvdd_value(adev, pl->mclk, &level->mvdd);
+
+	return ret;
+}
+
+static int si_populate_smc_t(struct amdgpu_device *adev,
+			     struct amdgpu_ps *amdgpu_state,
+			     SISLANDS_SMC_SWSTATE *smc_state)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct  si_ps *state = si_get_ps(amdgpu_state);
+	u32 a_t;
+	u32 t_l, t_h;
+	u32 high_bsp;
+	int i, ret;
+
+	if (state->performance_level_count >= 9)
+		return -EINVAL;
+
+	if (state->performance_level_count < 2) {
+		a_t = CG_R(0xffff) | CG_L(0);
+		smc_state->levels[0].aT = cpu_to_be32(a_t);
+		return 0;
+	}
+
+	smc_state->levels[0].aT = cpu_to_be32(0);
+
+	for (i = 0; i <= state->performance_level_count - 2; i++) {
+		ret = r600_calculate_at(
+			(50 / SISLANDS_MAX_HARDWARE_POWERLEVELS) * 100 * (i + 1),
+			100 * R600_AH_DFLT,
+			state->performance_levels[i + 1].sclk,
+			state->performance_levels[i].sclk,
+			&t_l,
+			&t_h);
+
+		if (ret) {
+			t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
+			t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
+		}
+
+		a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
+		a_t |= CG_R(t_l * pi->bsp / 20000);
+		smc_state->levels[i].aT = cpu_to_be32(a_t);
+
+		high_bsp = (i == state->performance_level_count - 2) ?
+			pi->pbsp : pi->bsp;
+		a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
+		smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
+	}
+
+	return 0;
+}
+
+static int si_disable_ulv(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct si_ulv_param *ulv = &si_pi->ulv;
+
+	if (ulv->supported)
+		return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_DisableULV) == PPSMC_Result_OK) ?
+			0 : -EINVAL;
+
+	return 0;
+}
+
+static bool si_is_state_ulv_compatible(struct amdgpu_device *adev,
+				       struct amdgpu_ps *amdgpu_state)
+{
+	const struct si_power_info *si_pi = si_get_pi(adev);
+	const struct si_ulv_param *ulv = &si_pi->ulv;
+	const struct  si_ps *state = si_get_ps(amdgpu_state);
+	int i;
+
+	if (state->performance_levels[0].mclk != ulv->pl.mclk)
+		return false;
+
+	/* XXX validate against display requirements! */
+
+	for (i = 0; i < adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count; i++) {
+		if (adev->clock.current_dispclk <=
+		    adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[i].clk) {
+			if (ulv->pl.vddc <
+			    adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[i].v)
+				return false;
+		}
+	}
+
+	if ((amdgpu_state->vclk != 0) || (amdgpu_state->dclk != 0))
+		return false;
+
+	return true;
+}
+
+static int si_set_power_state_conditionally_enable_ulv(struct amdgpu_device *adev,
+						       struct amdgpu_ps *amdgpu_new_state)
+{
+	const struct si_power_info *si_pi = si_get_pi(adev);
+	const struct si_ulv_param *ulv = &si_pi->ulv;
+
+	if (ulv->supported) {
+		if (si_is_state_ulv_compatible(adev, amdgpu_new_state))
+			return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_EnableULV) == PPSMC_Result_OK) ?
+				0 : -EINVAL;
+	}
+	return 0;
+}
+
+static int si_convert_power_state_to_smc(struct amdgpu_device *adev,
+					 struct amdgpu_ps *amdgpu_state,
+					 SISLANDS_SMC_SWSTATE *smc_state)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct ni_power_info *ni_pi = ni_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct  si_ps *state = si_get_ps(amdgpu_state);
+	int i, ret;
+	u32 threshold;
+	u32 sclk_in_sr = 1350; /* ??? */
+
+	if (state->performance_level_count > SISLANDS_MAX_HARDWARE_POWERLEVELS)
+		return -EINVAL;
+
+	threshold = state->performance_levels[state->performance_level_count-1].sclk * 100 / 100;
+
+	if (amdgpu_state->vclk && amdgpu_state->dclk) {
+		eg_pi->uvd_enabled = true;
+		if (eg_pi->smu_uvd_hs)
+			smc_state->flags |= PPSMC_SWSTATE_FLAG_UVD;
+	} else {
+		eg_pi->uvd_enabled = false;
+	}
+
+	if (state->dc_compatible)
+		smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
+
+	smc_state->levelCount = 0;
+	for (i = 0; i < state->performance_level_count; i++) {
+		if (eg_pi->sclk_deep_sleep) {
+			if ((i == 0) || si_pi->sclk_deep_sleep_above_low) {
+				if (sclk_in_sr <= SCLK_MIN_DEEPSLEEP_FREQ)
+					smc_state->levels[i].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_BYPASS;
+				else
+					smc_state->levels[i].stateFlags |= PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE;
+			}
+		}
+
+		ret = si_convert_power_level_to_smc(adev, &state->performance_levels[i],
+						    &smc_state->levels[i]);
+		smc_state->levels[i].arbRefreshState =
+			(u8)(SISLANDS_DRIVER_STATE_ARB_INDEX + i);
+
+		if (ret)
+			return ret;
+
+		if (ni_pi->enable_power_containment)
+			smc_state->levels[i].displayWatermark =
+				(state->performance_levels[i].sclk < threshold) ?
+				PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
+		else
+			smc_state->levels[i].displayWatermark = (i < 2) ?
+				PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
+
+		if (eg_pi->dynamic_ac_timing)
+			smc_state->levels[i].ACIndex = SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
+		else
+			smc_state->levels[i].ACIndex = 0;
+
+		smc_state->levelCount++;
+	}
+
+	si_write_smc_soft_register(adev,
+				   SI_SMC_SOFT_REGISTER_watermark_threshold,
+				   threshold / 512);
+
+	si_populate_smc_sp(adev, amdgpu_state, smc_state);
+
+	ret = si_populate_power_containment_values(adev, amdgpu_state, smc_state);
+	if (ret)
+		ni_pi->enable_power_containment = false;
+
+	ret = si_populate_sq_ramping_values(adev, amdgpu_state, smc_state);
+	if (ret)
+		ni_pi->enable_sq_ramping = false;
+
+	return si_populate_smc_t(adev, amdgpu_state, smc_state);
+}
+
+static int si_upload_sw_state(struct amdgpu_device *adev,
+			      struct amdgpu_ps *amdgpu_new_state)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct  si_ps *new_state = si_get_ps(amdgpu_new_state);
+	int ret;
+	u32 address = si_pi->state_table_start +
+		offsetof(SISLANDS_SMC_STATETABLE, driverState);
+	u32 state_size = sizeof(SISLANDS_SMC_SWSTATE) +
+		((new_state->performance_level_count - 1) *
+		 sizeof(SISLANDS_SMC_HW_PERFORMANCE_LEVEL));
+	SISLANDS_SMC_SWSTATE *smc_state = &si_pi->smc_statetable.driverState;
+
+	memset(smc_state, 0, state_size);
+
+	ret = si_convert_power_state_to_smc(adev, amdgpu_new_state, smc_state);
+	if (ret)
+		return ret;
+
+	return amdgpu_si_copy_bytes_to_smc(adev, address, (u8 *)smc_state,
+					   state_size, si_pi->sram_end);
+}
+
+static int si_upload_ulv_state(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct si_ulv_param *ulv = &si_pi->ulv;
+	int ret = 0;
+
+	if (ulv->supported && ulv->pl.vddc) {
+		u32 address = si_pi->state_table_start +
+			offsetof(SISLANDS_SMC_STATETABLE, ULVState);
+		SISLANDS_SMC_SWSTATE *smc_state = &si_pi->smc_statetable.ULVState;
+		u32 state_size = sizeof(SISLANDS_SMC_SWSTATE);
+
+		memset(smc_state, 0, state_size);
+
+		ret = si_populate_ulv_state(adev, smc_state);
+		if (!ret)
+			ret = amdgpu_si_copy_bytes_to_smc(adev, address, (u8 *)smc_state,
+							  state_size, si_pi->sram_end);
+	}
+
+	return ret;
+}
+
+static int si_upload_smc_data(struct amdgpu_device *adev)
+{
+	struct amdgpu_crtc *amdgpu_crtc = NULL;
+	int i;
+
+	if (adev->pm.dpm.new_active_crtc_count == 0)
+		return 0;
+
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		if (adev->pm.dpm.new_active_crtcs & (1 << i)) {
+			amdgpu_crtc = adev->mode_info.crtcs[i];
+			break;
+		}
+	}
+
+	if (amdgpu_crtc == NULL)
+		return 0;
+
+	if (amdgpu_crtc->line_time <= 0)
+		return 0;
+
+	if (si_write_smc_soft_register(adev,
+				       SI_SMC_SOFT_REGISTER_crtc_index,
+				       amdgpu_crtc->crtc_id) != PPSMC_Result_OK)
+		return 0;
+
+	if (si_write_smc_soft_register(adev,
+				       SI_SMC_SOFT_REGISTER_mclk_change_block_cp_min,
+				       amdgpu_crtc->wm_high / amdgpu_crtc->line_time) != PPSMC_Result_OK)
+		return 0;
+
+	if (si_write_smc_soft_register(adev,
+				       SI_SMC_SOFT_REGISTER_mclk_change_block_cp_max,
+				       amdgpu_crtc->wm_low / amdgpu_crtc->line_time) != PPSMC_Result_OK)
+		return 0;
+
+	return 0;
+}
+
+static int si_set_mc_special_registers(struct amdgpu_device *adev,
+				       struct si_mc_reg_table *table)
+{
+	u8 i, j, k;
+	u32 temp_reg;
+
+	for (i = 0, j = table->last; i < table->last; i++) {
+		if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+			return -EINVAL;
+		switch (table->mc_reg_address[i].s1) {
+		case MC_SEQ_MISC1:
+			temp_reg = RREG32(MC_PMG_CMD_EMRS);
+			table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS;
+			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP;
+			for (k = 0; k < table->num_entries; k++)
+				table->mc_reg_table_entry[k].mc_data[j] =
+					((temp_reg & 0xffff0000)) |
+					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
+			j++;
+			if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+				return -EINVAL;
+
+			temp_reg = RREG32(MC_PMG_CMD_MRS);
+			table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS;
+			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					(temp_reg & 0xffff0000) |
+					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+				if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5)
+					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
+			}
+			j++;
+			if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+				return -EINVAL;
+
+			if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) {
+				table->mc_reg_address[j].s1 = MC_PMG_AUTO_CMD;
+				table->mc_reg_address[j].s0 = MC_PMG_AUTO_CMD;
+				for (k = 0; k < table->num_entries; k++)
+					table->mc_reg_table_entry[k].mc_data[j] =
+						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
+				j++;
+				if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+					return -EINVAL;
+			}
+			break;
+		case MC_SEQ_RESERVE_M:
+			temp_reg = RREG32(MC_PMG_CMD_MRS1);
+			table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1;
+			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP;
+			for(k = 0; k < table->num_entries; k++)
+				table->mc_reg_table_entry[k].mc_data[j] =
+					(temp_reg & 0xffff0000) |
+					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+			j++;
+			if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+				return -EINVAL;
+			break;
+		default:
+			break;
+		}
+	}
+
+	table->last = j;
+
+	return 0;
+}
+
+static bool si_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
+{
+	bool result = true;
+	switch (in_reg) {
+	case  MC_SEQ_RAS_TIMING:
+		*out_reg = MC_SEQ_RAS_TIMING_LP;
+		break;
+	case MC_SEQ_CAS_TIMING:
+		*out_reg = MC_SEQ_CAS_TIMING_LP;
+		break;
+	case MC_SEQ_MISC_TIMING:
+		*out_reg = MC_SEQ_MISC_TIMING_LP;
+		break;
+	case MC_SEQ_MISC_TIMING2:
+		*out_reg = MC_SEQ_MISC_TIMING2_LP;
+		break;
+	case MC_SEQ_RD_CTL_D0:
+		*out_reg = MC_SEQ_RD_CTL_D0_LP;
+		break;
+	case MC_SEQ_RD_CTL_D1:
+		*out_reg = MC_SEQ_RD_CTL_D1_LP;
+		break;
+	case MC_SEQ_WR_CTL_D0:
+		*out_reg = MC_SEQ_WR_CTL_D0_LP;
+		break;
+	case MC_SEQ_WR_CTL_D1:
+		*out_reg = MC_SEQ_WR_CTL_D1_LP;
+		break;
+	case MC_PMG_CMD_EMRS:
+		*out_reg = MC_SEQ_PMG_CMD_EMRS_LP;
+		break;
+	case MC_PMG_CMD_MRS:
+		*out_reg = MC_SEQ_PMG_CMD_MRS_LP;
+		break;
+	case MC_PMG_CMD_MRS1:
+		*out_reg = MC_SEQ_PMG_CMD_MRS1_LP;
+		break;
+	case MC_SEQ_PMG_TIMING:
+		*out_reg = MC_SEQ_PMG_TIMING_LP;
+		break;
+	case MC_PMG_CMD_MRS2:
+		*out_reg = MC_SEQ_PMG_CMD_MRS2_LP;
+		break;
+	case MC_SEQ_WR_CTL_2:
+		*out_reg = MC_SEQ_WR_CTL_2_LP;
+		break;
+	default:
+		result = false;
+		break;
+	}
+
+	return result;
+}
+
+static void si_set_valid_flag(struct si_mc_reg_table *table)
+{
+	u8 i, j;
+
+	for (i = 0; i < table->last; i++) {
+		for (j = 1; j < table->num_entries; j++) {
+			if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
+				table->valid_flag |= 1 << i;
+				break;
+			}
+		}
+	}
+}
+
+static void si_set_s0_mc_reg_index(struct si_mc_reg_table *table)
+{
+	u32 i;
+	u16 address;
+
+	for (i = 0; i < table->last; i++)
+		table->mc_reg_address[i].s0 = si_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
+			address : table->mc_reg_address[i].s1;
+
+}
+
+static int si_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
+				      struct si_mc_reg_table *si_table)
+{
+	u8 i, j;
+
+	if (table->last > SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+		return -EINVAL;
+	if (table->num_entries > MAX_AC_TIMING_ENTRIES)
+		return -EINVAL;
+
+	for (i = 0; i < table->last; i++)
+		si_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
+	si_table->last = table->last;
+
+	for (i = 0; i < table->num_entries; i++) {
+		si_table->mc_reg_table_entry[i].mclk_max =
+			table->mc_reg_table_entry[i].mclk_max;
+		for (j = 0; j < table->last; j++) {
+			si_table->mc_reg_table_entry[i].mc_data[j] =
+				table->mc_reg_table_entry[i].mc_data[j];
+		}
+	}
+	si_table->num_entries = table->num_entries;
+
+	return 0;
+}
+
+static int si_initialize_mc_reg_table(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct atom_mc_reg_table *table;
+	struct si_mc_reg_table *si_table = &si_pi->mc_reg_table;
+	u8 module_index = rv770_get_memory_module_index(adev);
+	int ret;
+
+	table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
+	if (!table)
+		return -ENOMEM;
+
+	WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
+	WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
+	WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
+	WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
+	WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
+	WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
+	WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
+	WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
+	WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
+	WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
+	WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
+	WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
+	WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
+	WREG32(MC_SEQ_WR_CTL_2_LP, RREG32(MC_SEQ_WR_CTL_2));
+
+	ret = amdgpu_atombios_init_mc_reg_table(adev, module_index, table);
+	if (ret)
+		goto init_mc_done;
+
+	ret = si_copy_vbios_mc_reg_table(table, si_table);
+	if (ret)
+		goto init_mc_done;
+
+	si_set_s0_mc_reg_index(si_table);
+
+	ret = si_set_mc_special_registers(adev, si_table);
+	if (ret)
+		goto init_mc_done;
+
+	si_set_valid_flag(si_table);
+
+init_mc_done:
+	kfree(table);
+
+	return ret;
+
+}
+
+static void si_populate_mc_reg_addresses(struct amdgpu_device *adev,
+					 SMC_SIslands_MCRegisters *mc_reg_table)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 i, j;
+
+	for (i = 0, j = 0; j < si_pi->mc_reg_table.last; j++) {
+		if (si_pi->mc_reg_table.valid_flag & (1 << j)) {
+			if (i >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+				break;
+			mc_reg_table->address[i].s0 =
+				cpu_to_be16(si_pi->mc_reg_table.mc_reg_address[j].s0);
+			mc_reg_table->address[i].s1 =
+				cpu_to_be16(si_pi->mc_reg_table.mc_reg_address[j].s1);
+			i++;
+		}
+	}
+	mc_reg_table->last = (u8)i;
+}
+
+static void si_convert_mc_registers(const struct si_mc_reg_entry *entry,
+				    SMC_SIslands_MCRegisterSet *data,
+				    u32 num_entries, u32 valid_flag)
+{
+	u32 i, j;
+
+	for(i = 0, j = 0; j < num_entries; j++) {
+		if (valid_flag & (1 << j)) {
+			data->value[i] = cpu_to_be32(entry->mc_data[j]);
+			i++;
+		}
+	}
+}
+
+static void si_convert_mc_reg_table_entry_to_smc(struct amdgpu_device *adev,
+						 struct rv7xx_pl *pl,
+						 SMC_SIslands_MCRegisterSet *mc_reg_table_data)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 i = 0;
+
+	for (i = 0; i < si_pi->mc_reg_table.num_entries; i++) {
+		if (pl->mclk <= si_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
+			break;
+	}
+
+	if ((i == si_pi->mc_reg_table.num_entries) && (i > 0))
+		--i;
+
+	si_convert_mc_registers(&si_pi->mc_reg_table.mc_reg_table_entry[i],
+				mc_reg_table_data, si_pi->mc_reg_table.last,
+				si_pi->mc_reg_table.valid_flag);
+}
+
+static void si_convert_mc_reg_table_to_smc(struct amdgpu_device *adev,
+					   struct amdgpu_ps *amdgpu_state,
+					   SMC_SIslands_MCRegisters *mc_reg_table)
+{
+	struct si_ps *state = si_get_ps(amdgpu_state);
+	int i;
+
+	for (i = 0; i < state->performance_level_count; i++) {
+		si_convert_mc_reg_table_entry_to_smc(adev,
+						     &state->performance_levels[i],
+						     &mc_reg_table->data[SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
+	}
+}
+
+static int si_populate_mc_reg_table(struct amdgpu_device *adev,
+				    struct amdgpu_ps *amdgpu_boot_state)
+{
+	struct  si_ps *boot_state = si_get_ps(amdgpu_boot_state);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct si_ulv_param *ulv = &si_pi->ulv;
+	SMC_SIslands_MCRegisters *smc_mc_reg_table = &si_pi->smc_mc_reg_table;
+
+	memset(smc_mc_reg_table, 0, sizeof(SMC_SIslands_MCRegisters));
+
+	si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_seq_index, 1);
+
+	si_populate_mc_reg_addresses(adev, smc_mc_reg_table);
+
+	si_convert_mc_reg_table_entry_to_smc(adev, &boot_state->performance_levels[0],
+					     &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_INITIAL_SLOT]);
+
+	si_convert_mc_registers(&si_pi->mc_reg_table.mc_reg_table_entry[0],
+				&smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_ACPI_SLOT],
+				si_pi->mc_reg_table.last,
+				si_pi->mc_reg_table.valid_flag);
+
+	if (ulv->supported && ulv->pl.vddc != 0)
+		si_convert_mc_reg_table_entry_to_smc(adev, &ulv->pl,
+						     &smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_ULV_SLOT]);
+	else
+		si_convert_mc_registers(&si_pi->mc_reg_table.mc_reg_table_entry[0],
+					&smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_ULV_SLOT],
+					si_pi->mc_reg_table.last,
+					si_pi->mc_reg_table.valid_flag);
+
+	si_convert_mc_reg_table_to_smc(adev, amdgpu_boot_state, smc_mc_reg_table);
+
+	return amdgpu_si_copy_bytes_to_smc(adev, si_pi->mc_reg_table_start,
+					   (u8 *)smc_mc_reg_table,
+					   sizeof(SMC_SIslands_MCRegisters), si_pi->sram_end);
+}
+
+static int si_upload_mc_reg_table(struct amdgpu_device *adev,
+				  struct amdgpu_ps *amdgpu_new_state)
+{
+	struct si_ps *new_state = si_get_ps(amdgpu_new_state);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 address = si_pi->mc_reg_table_start +
+		offsetof(SMC_SIslands_MCRegisters,
+			 data[SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
+	SMC_SIslands_MCRegisters *smc_mc_reg_table = &si_pi->smc_mc_reg_table;
+
+	memset(smc_mc_reg_table, 0, sizeof(SMC_SIslands_MCRegisters));
+
+	si_convert_mc_reg_table_to_smc(adev, amdgpu_new_state, smc_mc_reg_table);
+
+	return amdgpu_si_copy_bytes_to_smc(adev, address,
+					   (u8 *)&smc_mc_reg_table->data[SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
+					   sizeof(SMC_SIslands_MCRegisterSet) * new_state->performance_level_count,
+					   si_pi->sram_end);
+}
+
+static void si_enable_voltage_control(struct amdgpu_device *adev, bool enable)
+{
+	if (enable)
+		WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN);
+	else
+		WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN);
+}
+
+static enum amdgpu_pcie_gen si_get_maximum_link_speed(struct amdgpu_device *adev,
+						      struct amdgpu_ps *amdgpu_state)
+{
+	struct si_ps *state = si_get_ps(amdgpu_state);
+	int i;
+	u16 pcie_speed, max_speed = 0;
+
+	for (i = 0; i < state->performance_level_count; i++) {
+		pcie_speed = state->performance_levels[i].pcie_gen;
+		if (max_speed < pcie_speed)
+			max_speed = pcie_speed;
+	}
+	return max_speed;
+}
+
+static u16 si_get_current_pcie_speed(struct amdgpu_device *adev)
+{
+	u32 speed_cntl;
+
+	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE_MASK;
+	speed_cntl >>= LC_CURRENT_DATA_RATE_SHIFT;
+
+	return (u16)speed_cntl;
+}
+
+static void si_request_link_speed_change_before_state_change(struct amdgpu_device *adev,
+							     struct amdgpu_ps *amdgpu_new_state,
+							     struct amdgpu_ps *amdgpu_current_state)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	enum amdgpu_pcie_gen target_link_speed = si_get_maximum_link_speed(adev, amdgpu_new_state);
+	enum amdgpu_pcie_gen current_link_speed;
+
+	if (si_pi->force_pcie_gen == AMDGPU_PCIE_GEN_INVALID)
+		current_link_speed = si_get_maximum_link_speed(adev, amdgpu_current_state);
+	else
+		current_link_speed = si_pi->force_pcie_gen;
+
+	si_pi->force_pcie_gen = AMDGPU_PCIE_GEN_INVALID;
+	si_pi->pspp_notify_required = false;
+	if (target_link_speed > current_link_speed) {
+		switch (target_link_speed) {
+#if defined(CONFIG_ACPI)
+		case AMDGPU_PCIE_GEN3:
+			if (amdgpu_acpi_pcie_performance_request(adev, PCIE_PERF_REQ_PECI_GEN3, false) == 0)
+				break;
+			si_pi->force_pcie_gen = AMDGPU_PCIE_GEN2;
+			if (current_link_speed == AMDGPU_PCIE_GEN2)
+				break;
+		case AMDGPU_PCIE_GEN2:
+			if (amdgpu_acpi_pcie_performance_request(adev, PCIE_PERF_REQ_PECI_GEN2, false) == 0)
+				break;
+#endif
+		default:
+			si_pi->force_pcie_gen = si_get_current_pcie_speed(adev);
+			break;
+		}
+	} else {
+		if (target_link_speed < current_link_speed)
+			si_pi->pspp_notify_required = true;
+	}
+}
+
+static void si_notify_link_speed_change_after_state_change(struct amdgpu_device *adev,
+							   struct amdgpu_ps *amdgpu_new_state,
+							   struct amdgpu_ps *amdgpu_current_state)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	enum amdgpu_pcie_gen target_link_speed = si_get_maximum_link_speed(adev, amdgpu_new_state);
+	u8 request;
+
+	if (si_pi->pspp_notify_required) {
+		if (target_link_speed == AMDGPU_PCIE_GEN3)
+			request = PCIE_PERF_REQ_PECI_GEN3;
+		else if (target_link_speed == AMDGPU_PCIE_GEN2)
+			request = PCIE_PERF_REQ_PECI_GEN2;
+		else
+			request = PCIE_PERF_REQ_PECI_GEN1;
+
+		if ((request == PCIE_PERF_REQ_PECI_GEN1) &&
+		    (si_get_current_pcie_speed(adev) > 0))
+			return;
+
+#if defined(CONFIG_ACPI)
+		amdgpu_acpi_pcie_performance_request(adev, request, false);
+#endif
+	}
+}
+
+#if 0
+static int si_ds_request(struct amdgpu_device *adev,
+			 bool ds_status_on, u32 count_write)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+
+	if (eg_pi->sclk_deep_sleep) {
+		if (ds_status_on)
+			return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_CancelThrottleOVRDSCLKDS) ==
+				PPSMC_Result_OK) ?
+				0 : -EINVAL;
+		else
+			return (amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_ThrottleOVRDSCLKDS) ==
+				PPSMC_Result_OK) ? 0 : -EINVAL;
+	}
+	return 0;
+}
+#endif
+
+static void si_set_max_cu_value(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+
+	if (adev->asic_type == CHIP_VERDE) {
+		switch (adev->pdev->device) {
+		case 0x6820:
+		case 0x6825:
+		case 0x6821:
+		case 0x6823:
+		case 0x6827:
+			si_pi->max_cu = 10;
+			break;
+		case 0x682D:
+		case 0x6824:
+		case 0x682F:
+		case 0x6826:
+			si_pi->max_cu = 8;
+			break;
+		case 0x6828:
+		case 0x6830:
+		case 0x6831:
+		case 0x6838:
+		case 0x6839:
+		case 0x683D:
+			si_pi->max_cu = 10;
+			break;
+		case 0x683B:
+		case 0x683F:
+		case 0x6829:
+			si_pi->max_cu = 8;
+			break;
+		default:
+			si_pi->max_cu = 0;
+			break;
+		}
+	} else {
+		si_pi->max_cu = 0;
+	}
+}
+
+static int si_patch_single_dependency_table_based_on_leakage(struct amdgpu_device *adev,
+							     struct amdgpu_clock_voltage_dependency_table *table)
+{
+	u32 i;
+	int j;
+	u16 leakage_voltage;
+
+	if (table) {
+		for (i = 0; i < table->count; i++) {
+			switch (si_get_leakage_voltage_from_leakage_index(adev,
+									  table->entries[i].v,
+									  &leakage_voltage)) {
+			case 0:
+				table->entries[i].v = leakage_voltage;
+				break;
+			case -EAGAIN:
+				return -EINVAL;
+			case -EINVAL:
+			default:
+				break;
+			}
+		}
+
+		for (j = (table->count - 2); j >= 0; j--) {
+			table->entries[j].v = (table->entries[j].v <= table->entries[j + 1].v) ?
+				table->entries[j].v : table->entries[j + 1].v;
+		}
+	}
+	return 0;
+}
+
+static int si_patch_dependency_tables_based_on_leakage(struct amdgpu_device *adev)
+{
+	int ret = 0;
+
+	ret = si_patch_single_dependency_table_based_on_leakage(adev,
+								&adev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
+	if (ret)
+		DRM_ERROR("Could not patch vddc_on_sclk leakage table\n");
+	ret = si_patch_single_dependency_table_based_on_leakage(adev,
+								&adev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
+	if (ret)
+		DRM_ERROR("Could not patch vddc_on_mclk leakage table\n");
+	ret = si_patch_single_dependency_table_based_on_leakage(adev,
+								&adev->pm.dpm.dyn_state.vddci_dependency_on_mclk);
+	if (ret)
+		DRM_ERROR("Could not patch vddci_on_mclk leakage table\n");
+	return ret;
+}
+
+static void si_set_pcie_lane_width_in_smc(struct amdgpu_device *adev,
+					  struct amdgpu_ps *amdgpu_new_state,
+					  struct amdgpu_ps *amdgpu_current_state)
+{
+	u32 lane_width;
+	u32 new_lane_width =
+		(amdgpu_new_state->caps & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >> ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT;
+	u32 current_lane_width =
+		(amdgpu_current_state->caps & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >> ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT;
+
+	if (new_lane_width != current_lane_width) {
+		amdgpu_set_pcie_lanes(adev, new_lane_width);
+		lane_width = amdgpu_get_pcie_lanes(adev);
+		si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_non_ulv_pcie_link_width, lane_width);
+	}
+}
+
+static void si_dpm_setup_asic(struct amdgpu_device *adev)
+{
+	si_read_clock_registers(adev);
+	si_enable_acpi_power_management(adev);
+}
+
+static int si_thermal_enable_alert(struct amdgpu_device *adev,
+				   bool enable)
+{
+	u32 thermal_int = RREG32(CG_THERMAL_INT);
+
+	if (enable) {
+		PPSMC_Result result;
+
+		thermal_int &= ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
+		WREG32(CG_THERMAL_INT, thermal_int);
+		result = amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_EnableThermalInterrupt);
+		if (result != PPSMC_Result_OK) {
+			DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
+			return -EINVAL;
+		}
+	} else {
+		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
+		WREG32(CG_THERMAL_INT, thermal_int);
+	}
+
+	return 0;
+}
+
+static int si_thermal_set_temperature_range(struct amdgpu_device *adev,
+					    int min_temp, int max_temp)
+{
+	int low_temp = 0 * 1000;
+	int high_temp = 255 * 1000;
+
+	if (low_temp < min_temp)
+		low_temp = min_temp;
+	if (high_temp > max_temp)
+		high_temp = max_temp;
+	if (high_temp < low_temp) {
+		DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
+		return -EINVAL;
+	}
+
+	WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK);
+	WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK);
+	WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK);
+
+	adev->pm.dpm.thermal.min_temp = low_temp;
+	adev->pm.dpm.thermal.max_temp = high_temp;
+
+	return 0;
+}
+
+static void si_fan_ctrl_set_static_mode(struct amdgpu_device *adev, u32 mode)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 tmp;
+
+	if (si_pi->fan_ctrl_is_in_default_mode) {
+		tmp = (RREG32(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK) >> FDO_PWM_MODE_SHIFT;
+		si_pi->fan_ctrl_default_mode = tmp;
+		tmp = (RREG32(CG_FDO_CTRL2) & TMIN_MASK) >> TMIN_SHIFT;
+		si_pi->t_min = tmp;
+		si_pi->fan_ctrl_is_in_default_mode = false;
+	}
+
+	tmp = RREG32(CG_FDO_CTRL2) & ~TMIN_MASK;
+	tmp |= TMIN(0);
+	WREG32(CG_FDO_CTRL2, tmp);
+
+	tmp = RREG32(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK;
+	tmp |= FDO_PWM_MODE(mode);
+	WREG32(CG_FDO_CTRL2, tmp);
+}
+
+static int si_thermal_setup_fan_table(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	PP_SIslands_FanTable fan_table = { FDO_MODE_HARDWARE };
+	u32 duty100;
+	u32 t_diff1, t_diff2, pwm_diff1, pwm_diff2;
+	u16 fdo_min, slope1, slope2;
+	u32 reference_clock, tmp;
+	int ret;
+	u64 tmp64;
+
+	if (!si_pi->fan_table_start) {
+		adev->pm.dpm.fan.ucode_fan_control = false;
+		return 0;
+	}
+
+	duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT;
+
+	if (duty100 == 0) {
+		adev->pm.dpm.fan.ucode_fan_control = false;
+		return 0;
+	}
+
+	tmp64 = (u64)adev->pm.dpm.fan.pwm_min * duty100;
+	do_div(tmp64, 10000);
+	fdo_min = (u16)tmp64;
+
+	t_diff1 = adev->pm.dpm.fan.t_med - adev->pm.dpm.fan.t_min;
+	t_diff2 = adev->pm.dpm.fan.t_high - adev->pm.dpm.fan.t_med;
+
+	pwm_diff1 = adev->pm.dpm.fan.pwm_med - adev->pm.dpm.fan.pwm_min;
+	pwm_diff2 = adev->pm.dpm.fan.pwm_high - adev->pm.dpm.fan.pwm_med;
+
+	slope1 = (u16)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
+	slope2 = (u16)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
+
+	fan_table.temp_min = cpu_to_be16((50 + adev->pm.dpm.fan.t_min) / 100);
+	fan_table.temp_med = cpu_to_be16((50 + adev->pm.dpm.fan.t_med) / 100);
+	fan_table.temp_max = cpu_to_be16((50 + adev->pm.dpm.fan.t_max) / 100);
+	fan_table.slope1 = cpu_to_be16(slope1);
+	fan_table.slope2 = cpu_to_be16(slope2);
+	fan_table.fdo_min = cpu_to_be16(fdo_min);
+	fan_table.hys_down = cpu_to_be16(adev->pm.dpm.fan.t_hyst);
+	fan_table.hys_up = cpu_to_be16(1);
+	fan_table.hys_slope = cpu_to_be16(1);
+	fan_table.temp_resp_lim = cpu_to_be16(5);
+	reference_clock = amdgpu_asic_get_xclk(adev);
+
+	fan_table.refresh_period = cpu_to_be32((adev->pm.dpm.fan.cycle_delay *
+						reference_clock) / 1600);
+	fan_table.fdo_max = cpu_to_be16((u16)duty100);
+
+	tmp = (RREG32(CG_MULT_THERMAL_CTRL) & TEMP_SEL_MASK) >> TEMP_SEL_SHIFT;
+	fan_table.temp_src = (uint8_t)tmp;
+
+	ret = amdgpu_si_copy_bytes_to_smc(adev,
+					  si_pi->fan_table_start,
+					  (u8 *)(&fan_table),
+					  sizeof(fan_table),
+					  si_pi->sram_end);
+
+	if (ret) {
+		DRM_ERROR("Failed to load fan table to the SMC.");
+		adev->pm.dpm.fan.ucode_fan_control = false;
+	}
+
+	return ret;
+}
+
+static int si_fan_ctrl_start_smc_fan_control(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	PPSMC_Result ret;
+
+	ret = amdgpu_si_send_msg_to_smc(adev, PPSMC_StartFanControl);
+	if (ret == PPSMC_Result_OK) {
+		si_pi->fan_is_controlled_by_smc = true;
+		return 0;
+	} else {
+		return -EINVAL;
+	}
+}
+
+static int si_fan_ctrl_stop_smc_fan_control(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	PPSMC_Result ret;
+
+	ret = amdgpu_si_send_msg_to_smc(adev, PPSMC_StopFanControl);
+
+	if (ret == PPSMC_Result_OK) {
+		si_pi->fan_is_controlled_by_smc = false;
+		return 0;
+	} else {
+		return -EINVAL;
+	}
+}
+
+static int si_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
+				      u32 *speed)
+{
+	u32 duty, duty100;
+	u64 tmp64;
+
+	if (adev->pm.no_fan)
+		return -ENOENT;
+
+	duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT;
+	duty = (RREG32(CG_THERMAL_STATUS) & FDO_PWM_DUTY_MASK) >> FDO_PWM_DUTY_SHIFT;
+
+	if (duty100 == 0)
+		return -EINVAL;
+
+	tmp64 = (u64)duty * 100;
+	do_div(tmp64, duty100);
+	*speed = (u32)tmp64;
+
+	if (*speed > 100)
+		*speed = 100;
+
+	return 0;
+}
+
+static int si_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
+				      u32 speed)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 tmp;
+	u32 duty, duty100;
+	u64 tmp64;
+
+	if (adev->pm.no_fan)
+		return -ENOENT;
+
+	if (si_pi->fan_is_controlled_by_smc)
+		return -EINVAL;
+
+	if (speed > 100)
+		return -EINVAL;
+
+	duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT;
+
+	if (duty100 == 0)
+		return -EINVAL;
+
+	tmp64 = (u64)speed * duty100;
+	do_div(tmp64, 100);
+	duty = (u32)tmp64;
+
+	tmp = RREG32(CG_FDO_CTRL0) & ~FDO_STATIC_DUTY_MASK;
+	tmp |= FDO_STATIC_DUTY(duty);
+	WREG32(CG_FDO_CTRL0, tmp);
+
+	return 0;
+}
+
+static void si_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
+{
+	if (mode) {
+		/* stop auto-manage */
+		if (adev->pm.dpm.fan.ucode_fan_control)
+			si_fan_ctrl_stop_smc_fan_control(adev);
+		si_fan_ctrl_set_static_mode(adev, mode);
+	} else {
+		/* restart auto-manage */
+		if (adev->pm.dpm.fan.ucode_fan_control)
+			si_thermal_start_smc_fan_control(adev);
+		else
+			si_fan_ctrl_set_default_mode(adev);
+	}
+}
+
+static u32 si_dpm_get_fan_control_mode(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 tmp;
+
+	if (si_pi->fan_is_controlled_by_smc)
+		return 0;
+
+	tmp = RREG32(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK;
+	return (tmp >> FDO_PWM_MODE_SHIFT);
+}
+
+#if 0
+static int si_fan_ctrl_get_fan_speed_rpm(struct amdgpu_device *adev,
+					 u32 *speed)
+{
+	u32 tach_period;
+	u32 xclk = amdgpu_asic_get_xclk(adev);
+
+	if (adev->pm.no_fan)
+		return -ENOENT;
+
+	if (adev->pm.fan_pulses_per_revolution == 0)
+		return -ENOENT;
+
+	tach_period = (RREG32(CG_TACH_STATUS) & TACH_PERIOD_MASK) >> TACH_PERIOD_SHIFT;
+	if (tach_period == 0)
+		return -ENOENT;
+
+	*speed = 60 * xclk * 10000 / tach_period;
+
+	return 0;
+}
+
+static int si_fan_ctrl_set_fan_speed_rpm(struct amdgpu_device *adev,
+					 u32 speed)
+{
+	u32 tach_period, tmp;
+	u32 xclk = amdgpu_asic_get_xclk(adev);
+
+	if (adev->pm.no_fan)
+		return -ENOENT;
+
+	if (adev->pm.fan_pulses_per_revolution == 0)
+		return -ENOENT;
+
+	if ((speed < adev->pm.fan_min_rpm) ||
+	    (speed > adev->pm.fan_max_rpm))
+		return -EINVAL;
+
+	if (adev->pm.dpm.fan.ucode_fan_control)
+		si_fan_ctrl_stop_smc_fan_control(adev);
+
+	tach_period = 60 * xclk * 10000 / (8 * speed);
+	tmp = RREG32(CG_TACH_CTRL) & ~TARGET_PERIOD_MASK;
+	tmp |= TARGET_PERIOD(tach_period);
+	WREG32(CG_TACH_CTRL, tmp);
+
+	si_fan_ctrl_set_static_mode(adev, FDO_PWM_MODE_STATIC_RPM);
+
+	return 0;
+}
+#endif
+
+static void si_fan_ctrl_set_default_mode(struct amdgpu_device *adev)
+{
+	struct si_power_info *si_pi = si_get_pi(adev);
+	u32 tmp;
+
+	if (!si_pi->fan_ctrl_is_in_default_mode) {
+		tmp = RREG32(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK;
+		tmp |= FDO_PWM_MODE(si_pi->fan_ctrl_default_mode);
+		WREG32(CG_FDO_CTRL2, tmp);
+
+		tmp = RREG32(CG_FDO_CTRL2) & ~TMIN_MASK;
+		tmp |= TMIN(si_pi->t_min);
+		WREG32(CG_FDO_CTRL2, tmp);
+		si_pi->fan_ctrl_is_in_default_mode = true;
+	}
+}
+
+static void si_thermal_start_smc_fan_control(struct amdgpu_device *adev)
+{
+	if (adev->pm.dpm.fan.ucode_fan_control) {
+		si_fan_ctrl_start_smc_fan_control(adev);
+		si_fan_ctrl_set_static_mode(adev, FDO_PWM_MODE_STATIC);
+	}
+}
+
+static void si_thermal_initialize(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	if (adev->pm.fan_pulses_per_revolution) {
+		tmp = RREG32(CG_TACH_CTRL) & ~EDGE_PER_REV_MASK;
+		tmp |= EDGE_PER_REV(adev->pm.fan_pulses_per_revolution -1);
+		WREG32(CG_TACH_CTRL, tmp);
+	}
+
+	tmp = RREG32(CG_FDO_CTRL2) & ~TACH_PWM_RESP_RATE_MASK;
+	tmp |= TACH_PWM_RESP_RATE(0x28);
+	WREG32(CG_FDO_CTRL2, tmp);
+}
+
+static int si_thermal_start_thermal_controller(struct amdgpu_device *adev)
+{
+	int ret;
+
+	si_thermal_initialize(adev);
+	ret = si_thermal_set_temperature_range(adev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+	if (ret)
+		return ret;
+	ret = si_thermal_enable_alert(adev, true);
+	if (ret)
+		return ret;
+	if (adev->pm.dpm.fan.ucode_fan_control) {
+		ret = si_halt_smc(adev);
+		if (ret)
+			return ret;
+		ret = si_thermal_setup_fan_table(adev);
+		if (ret)
+			return ret;
+		ret = si_resume_smc(adev);
+		if (ret)
+			return ret;
+		si_thermal_start_smc_fan_control(adev);
+	}
+
+	return 0;
+}
+
+static void si_thermal_stop_thermal_controller(struct amdgpu_device *adev)
+{
+	if (!adev->pm.no_fan) {
+		si_fan_ctrl_set_default_mode(adev);
+		si_fan_ctrl_stop_smc_fan_control(adev);
+	}
+}
+
+static int si_dpm_enable(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct amdgpu_ps *boot_ps = adev->pm.dpm.boot_ps;
+	int ret;
+
+	if (amdgpu_si_is_smc_running(adev))
+		return -EINVAL;
+	if (pi->voltage_control || si_pi->voltage_control_svi2)
+		si_enable_voltage_control(adev, true);
+	if (pi->mvdd_control)
+		si_get_mvdd_configuration(adev);
+	if (pi->voltage_control || si_pi->voltage_control_svi2) {
+		ret = si_construct_voltage_tables(adev);
+		if (ret) {
+			DRM_ERROR("si_construct_voltage_tables failed\n");
+			return ret;
+		}
+	}
+	if (eg_pi->dynamic_ac_timing) {
+		ret = si_initialize_mc_reg_table(adev);
+		if (ret)
+			eg_pi->dynamic_ac_timing = false;
+	}
+	if (pi->dynamic_ss)
+		si_enable_spread_spectrum(adev, true);
+	if (pi->thermal_protection)
+		si_enable_thermal_protection(adev, true);
+	si_setup_bsp(adev);
+	si_program_git(adev);
+	si_program_tp(adev);
+	si_program_tpp(adev);
+	si_program_sstp(adev);
+	si_enable_display_gap(adev);
+	si_program_vc(adev);
+	ret = si_upload_firmware(adev);
+	if (ret) {
+		DRM_ERROR("si_upload_firmware failed\n");
+		return ret;
+	}
+	ret = si_process_firmware_header(adev);
+	if (ret) {
+		DRM_ERROR("si_process_firmware_header failed\n");
+		return ret;
+	}
+	ret = si_initial_switch_from_arb_f0_to_f1(adev);
+	if (ret) {
+		DRM_ERROR("si_initial_switch_from_arb_f0_to_f1 failed\n");
+		return ret;
+	}
+	ret = si_init_smc_table(adev);
+	if (ret) {
+		DRM_ERROR("si_init_smc_table failed\n");
+		return ret;
+	}
+	ret = si_init_smc_spll_table(adev);
+	if (ret) {
+		DRM_ERROR("si_init_smc_spll_table failed\n");
+		return ret;
+	}
+	ret = si_init_arb_table_index(adev);
+	if (ret) {
+		DRM_ERROR("si_init_arb_table_index failed\n");
+		return ret;
+	}
+	if (eg_pi->dynamic_ac_timing) {
+		ret = si_populate_mc_reg_table(adev, boot_ps);
+		if (ret) {
+			DRM_ERROR("si_populate_mc_reg_table failed\n");
+			return ret;
+		}
+	}
+	ret = si_initialize_smc_cac_tables(adev);
+	if (ret) {
+		DRM_ERROR("si_initialize_smc_cac_tables failed\n");
+		return ret;
+	}
+	ret = si_initialize_hardware_cac_manager(adev);
+	if (ret) {
+		DRM_ERROR("si_initialize_hardware_cac_manager failed\n");
+		return ret;
+	}
+	ret = si_initialize_smc_dte_tables(adev);
+	if (ret) {
+		DRM_ERROR("si_initialize_smc_dte_tables failed\n");
+		return ret;
+	}
+	ret = si_populate_smc_tdp_limits(adev, boot_ps);
+	if (ret) {
+		DRM_ERROR("si_populate_smc_tdp_limits failed\n");
+		return ret;
+	}
+	ret = si_populate_smc_tdp_limits_2(adev, boot_ps);
+	if (ret) {
+		DRM_ERROR("si_populate_smc_tdp_limits_2 failed\n");
+		return ret;
+	}
+	si_program_response_times(adev);
+	si_program_ds_registers(adev);
+	si_dpm_start_smc(adev);
+	ret = si_notify_smc_display_change(adev, false);
+	if (ret) {
+		DRM_ERROR("si_notify_smc_display_change failed\n");
+		return ret;
+	}
+	si_enable_sclk_control(adev, true);
+	si_start_dpm(adev);
+
+	si_enable_auto_throttle_source(adev, AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
+	si_thermal_start_thermal_controller(adev);
+	ni_update_current_ps(adev, boot_ps);
+
+	return 0;
+}
+
+static int si_set_temperature_range(struct amdgpu_device *adev)
+{
+	int ret;
+
+	ret = si_thermal_enable_alert(adev, false);
+	if (ret)
+		return ret;
+	ret = si_thermal_set_temperature_range(adev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
+	if (ret)
+		return ret;
+	ret = si_thermal_enable_alert(adev, true);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static void si_dpm_disable(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct amdgpu_ps *boot_ps = adev->pm.dpm.boot_ps;
+
+	if (!amdgpu_si_is_smc_running(adev))
+		return;
+	si_thermal_stop_thermal_controller(adev);
+	si_disable_ulv(adev);
+	si_clear_vc(adev);
+	if (pi->thermal_protection)
+		si_enable_thermal_protection(adev, false);
+	si_enable_power_containment(adev, boot_ps, false);
+	si_enable_smc_cac(adev, boot_ps, false);
+	si_enable_spread_spectrum(adev, false);
+	si_enable_auto_throttle_source(adev, AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
+	si_stop_dpm(adev);
+	si_reset_to_default(adev);
+	si_dpm_stop_smc(adev);
+	si_force_switch_to_arb_f0(adev);
+
+	ni_update_current_ps(adev, boot_ps);
+}
+
+static int si_dpm_pre_set_power_state(struct amdgpu_device *adev)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps;
+	struct amdgpu_ps *new_ps = &requested_ps;
+
+	ni_update_requested_ps(adev, new_ps);
+	si_apply_state_adjust_rules(adev, &eg_pi->requested_rps);
+
+	return 0;
+}
+
+static int si_power_control_set_level(struct amdgpu_device *adev)
+{
+	struct amdgpu_ps *new_ps = adev->pm.dpm.requested_ps;
+	int ret;
+
+	ret = si_restrict_performance_levels_before_switch(adev);
+	if (ret)
+		return ret;
+	ret = si_halt_smc(adev);
+	if (ret)
+		return ret;
+	ret = si_populate_smc_tdp_limits(adev, new_ps);
+	if (ret)
+		return ret;
+	ret = si_populate_smc_tdp_limits_2(adev, new_ps);
+	if (ret)
+		return ret;
+	ret = si_resume_smc(adev);
+	if (ret)
+		return ret;
+	ret = si_set_sw_state(adev);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static int si_dpm_set_power_state(struct amdgpu_device *adev)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct amdgpu_ps *new_ps = &eg_pi->requested_rps;
+	struct amdgpu_ps *old_ps = &eg_pi->current_rps;
+	int ret;
+
+	ret = si_disable_ulv(adev);
+	if (ret) {
+		DRM_ERROR("si_disable_ulv failed\n");
+		return ret;
+	}
+	ret = si_restrict_performance_levels_before_switch(adev);
+	if (ret) {
+		DRM_ERROR("si_restrict_performance_levels_before_switch failed\n");
+		return ret;
+	}
+	if (eg_pi->pcie_performance_request)
+		si_request_link_speed_change_before_state_change(adev, new_ps, old_ps);
+	ni_set_uvd_clock_before_set_eng_clock(adev, new_ps, old_ps);
+	ret = si_enable_power_containment(adev, new_ps, false);
+	if (ret) {
+		DRM_ERROR("si_enable_power_containment failed\n");
+		return ret;
+	}
+	ret = si_enable_smc_cac(adev, new_ps, false);
+	if (ret) {
+		DRM_ERROR("si_enable_smc_cac failed\n");
+		return ret;
+	}
+	ret = si_halt_smc(adev);
+	if (ret) {
+		DRM_ERROR("si_halt_smc failed\n");
+		return ret;
+	}
+	ret = si_upload_sw_state(adev, new_ps);
+	if (ret) {
+		DRM_ERROR("si_upload_sw_state failed\n");
+		return ret;
+	}
+	ret = si_upload_smc_data(adev);
+	if (ret) {
+		DRM_ERROR("si_upload_smc_data failed\n");
+		return ret;
+	}
+	ret = si_upload_ulv_state(adev);
+	if (ret) {
+		DRM_ERROR("si_upload_ulv_state failed\n");
+		return ret;
+	}
+	if (eg_pi->dynamic_ac_timing) {
+		ret = si_upload_mc_reg_table(adev, new_ps);
+		if (ret) {
+			DRM_ERROR("si_upload_mc_reg_table failed\n");
+			return ret;
+		}
+	}
+	ret = si_program_memory_timing_parameters(adev, new_ps);
+	if (ret) {
+		DRM_ERROR("si_program_memory_timing_parameters failed\n");
+		return ret;
+	}
+	si_set_pcie_lane_width_in_smc(adev, new_ps, old_ps);
+
+	ret = si_resume_smc(adev);
+	if (ret) {
+		DRM_ERROR("si_resume_smc failed\n");
+		return ret;
+	}
+	ret = si_set_sw_state(adev);
+	if (ret) {
+		DRM_ERROR("si_set_sw_state failed\n");
+		return ret;
+	}
+	ni_set_uvd_clock_after_set_eng_clock(adev, new_ps, old_ps);
+	if (eg_pi->pcie_performance_request)
+		si_notify_link_speed_change_after_state_change(adev, new_ps, old_ps);
+	ret = si_set_power_state_conditionally_enable_ulv(adev, new_ps);
+	if (ret) {
+		DRM_ERROR("si_set_power_state_conditionally_enable_ulv failed\n");
+		return ret;
+	}
+	ret = si_enable_smc_cac(adev, new_ps, true);
+	if (ret) {
+		DRM_ERROR("si_enable_smc_cac failed\n");
+		return ret;
+	}
+	ret = si_enable_power_containment(adev, new_ps, true);
+	if (ret) {
+		DRM_ERROR("si_enable_power_containment failed\n");
+		return ret;
+	}
+
+	ret = si_power_control_set_level(adev);
+	if (ret) {
+		DRM_ERROR("si_power_control_set_level failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void si_dpm_post_set_power_state(struct amdgpu_device *adev)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct amdgpu_ps *new_ps = &eg_pi->requested_rps;
+
+	ni_update_current_ps(adev, new_ps);
+}
+
+#if 0
+void si_dpm_reset_asic(struct amdgpu_device *adev)
+{
+	si_restrict_performance_levels_before_switch(adev);
+	si_disable_ulv(adev);
+	si_set_boot_state(adev);
+}
+#endif
+
+static void si_dpm_display_configuration_changed(struct amdgpu_device *adev)
+{
+	si_program_display_gap(adev);
+}
+
+
+static void si_parse_pplib_non_clock_info(struct amdgpu_device *adev,
+					  struct amdgpu_ps *rps,
+					  struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
+					  u8 table_rev)
+{
+	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
+	rps->class = le16_to_cpu(non_clock_info->usClassification);
+	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
+
+	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
+		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
+		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
+	} else if (r600_is_uvd_state(rps->class, rps->class2)) {
+		rps->vclk = RV770_DEFAULT_VCLK_FREQ;
+		rps->dclk = RV770_DEFAULT_DCLK_FREQ;
+	} else {
+		rps->vclk = 0;
+		rps->dclk = 0;
+	}
+
+	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
+		adev->pm.dpm.boot_ps = rps;
+	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
+		adev->pm.dpm.uvd_ps = rps;
+}
+
+static void si_parse_pplib_clock_info(struct amdgpu_device *adev,
+				      struct amdgpu_ps *rps, int index,
+				      union pplib_clock_info *clock_info)
+{
+	struct rv7xx_power_info *pi = rv770_get_pi(adev);
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct si_power_info *si_pi = si_get_pi(adev);
+	struct  si_ps *ps = si_get_ps(rps);
+	u16 leakage_voltage;
+	struct rv7xx_pl *pl = &ps->performance_levels[index];
+	int ret;
+
+	ps->performance_level_count = index + 1;
+
+	pl->sclk = le16_to_cpu(clock_info->si.usEngineClockLow);
+	pl->sclk |= clock_info->si.ucEngineClockHigh << 16;
+	pl->mclk = le16_to_cpu(clock_info->si.usMemoryClockLow);
+	pl->mclk |= clock_info->si.ucMemoryClockHigh << 16;
+
+	pl->vddc = le16_to_cpu(clock_info->si.usVDDC);
+	pl->vddci = le16_to_cpu(clock_info->si.usVDDCI);
+	pl->flags = le32_to_cpu(clock_info->si.ulFlags);
+	pl->pcie_gen = r600_get_pcie_gen_support(adev,
+						 si_pi->sys_pcie_mask,
+						 si_pi->boot_pcie_gen,
+						 clock_info->si.ucPCIEGen);
+
+	/* patch up vddc if necessary */
+	ret = si_get_leakage_voltage_from_leakage_index(adev, pl->vddc,
+							&leakage_voltage);
+	if (ret == 0)
+		pl->vddc = leakage_voltage;
+
+	if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
+		pi->acpi_vddc = pl->vddc;
+		eg_pi->acpi_vddci = pl->vddci;
+		si_pi->acpi_pcie_gen = pl->pcie_gen;
+	}
+
+	if ((rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) &&
+	    index == 0) {
+		/* XXX disable for A0 tahiti */
+		si_pi->ulv.supported = false;
+		si_pi->ulv.pl = *pl;
+		si_pi->ulv.one_pcie_lane_in_ulv = false;
+		si_pi->ulv.volt_change_delay = SISLANDS_ULVVOLTAGECHANGEDELAY_DFLT;
+		si_pi->ulv.cg_ulv_parameter = SISLANDS_CGULVPARAMETER_DFLT;
+		si_pi->ulv.cg_ulv_control = SISLANDS_CGULVCONTROL_DFLT;
+	}
+
+	if (pi->min_vddc_in_table > pl->vddc)
+		pi->min_vddc_in_table = pl->vddc;
+
+	if (pi->max_vddc_in_table < pl->vddc)
+		pi->max_vddc_in_table = pl->vddc;
+
+	/* patch up boot state */
+	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
+		u16 vddc, vddci, mvdd;
+		amdgpu_atombios_get_default_voltages(adev, &vddc, &vddci, &mvdd);
+		pl->mclk = adev->clock.default_mclk;
+		pl->sclk = adev->clock.default_sclk;
+		pl->vddc = vddc;
+		pl->vddci = vddci;
+		si_pi->mvdd_bootup_value = mvdd;
+	}
+
+	if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
+	    ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
+		adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
+		adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
+		adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
+		adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
+	}
+}
+
+union pplib_power_state {
+	struct _ATOM_PPLIB_STATE v1;
+	struct _ATOM_PPLIB_STATE_V2 v2;
+};
+
+static int si_parse_power_table(struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
+	union pplib_power_state *power_state;
+	int i, j, k, non_clock_array_index, clock_array_index;
+	union pplib_clock_info *clock_info;
+	struct _StateArray *state_array;
+	struct _ClockInfoArray *clock_info_array;
+	struct _NonClockInfoArray *non_clock_info_array;
+	union power_info *power_info;
+	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+	u16 data_offset;
+	u8 frev, crev;
+	u8 *power_state_offset;
+	struct  si_ps *ps;
+
+	if (!amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
+				   &frev, &crev, &data_offset))
+		return -EINVAL;
+	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+	amdgpu_add_thermal_controller(adev);
+
+	state_array = (struct _StateArray *)
+		(mode_info->atom_context->bios + data_offset +
+		 le16_to_cpu(power_info->pplib.usStateArrayOffset));
+	clock_info_array = (struct _ClockInfoArray *)
+		(mode_info->atom_context->bios + data_offset +
+		 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
+	non_clock_info_array = (struct _NonClockInfoArray *)
+		(mode_info->atom_context->bios + data_offset +
+		 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
+
+	adev->pm.dpm.ps = kzalloc(sizeof(struct amdgpu_ps) *
+				  state_array->ucNumEntries, GFP_KERNEL);
+	if (!adev->pm.dpm.ps)
+		return -ENOMEM;
+	power_state_offset = (u8 *)state_array->states;
+	for (i = 0; i < state_array->ucNumEntries; i++) {
+		u8 *idx;
+		power_state = (union pplib_power_state *)power_state_offset;
+		non_clock_array_index = power_state->v2.nonClockInfoIndex;
+		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
+			&non_clock_info_array->nonClockInfo[non_clock_array_index];
+		ps = kzalloc(sizeof(struct  si_ps), GFP_KERNEL);
+		if (ps == NULL) {
+			kfree(adev->pm.dpm.ps);
+			return -ENOMEM;
+		}
+		adev->pm.dpm.ps[i].ps_priv = ps;
+		si_parse_pplib_non_clock_info(adev, &adev->pm.dpm.ps[i],
+					      non_clock_info,
+					      non_clock_info_array->ucEntrySize);
+		k = 0;
+		idx = (u8 *)&power_state->v2.clockInfoIndex[0];
+		for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
+			clock_array_index = idx[j];
+			if (clock_array_index >= clock_info_array->ucNumEntries)
+				continue;
+			if (k >= SISLANDS_MAX_HARDWARE_POWERLEVELS)
+				break;
+			clock_info = (union pplib_clock_info *)
+				((u8 *)&clock_info_array->clockInfo[0] +
+				 (clock_array_index * clock_info_array->ucEntrySize));
+			si_parse_pplib_clock_info(adev,
+						  &adev->pm.dpm.ps[i], k,
+						  clock_info);
+			k++;
+		}
+		power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
+	}
+	adev->pm.dpm.num_ps = state_array->ucNumEntries;
+
+	/* fill in the vce power states */
+	for (i = 0; i < AMDGPU_MAX_VCE_LEVELS; i++) {
+		u32 sclk, mclk;
+		clock_array_index = adev->pm.dpm.vce_states[i].clk_idx;
+		clock_info = (union pplib_clock_info *)
+			&clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
+		sclk = le16_to_cpu(clock_info->si.usEngineClockLow);
+		sclk |= clock_info->si.ucEngineClockHigh << 16;
+		mclk = le16_to_cpu(clock_info->si.usMemoryClockLow);
+		mclk |= clock_info->si.ucMemoryClockHigh << 16;
+		adev->pm.dpm.vce_states[i].sclk = sclk;
+		adev->pm.dpm.vce_states[i].mclk = mclk;
+	}
+
+	return 0;
+}
+
+static int si_dpm_init(struct amdgpu_device *adev)
+{
+	struct rv7xx_power_info *pi;
+	struct evergreen_power_info *eg_pi;
+	struct ni_power_info *ni_pi;
+	struct si_power_info *si_pi;
+	struct atom_clock_dividers dividers;
+	int ret;
+	u32 mask;
+
+	si_pi = kzalloc(sizeof(struct si_power_info), GFP_KERNEL);
+	if (si_pi == NULL)
+		return -ENOMEM;
+	adev->pm.dpm.priv = si_pi;
+	ni_pi = &si_pi->ni;
+	eg_pi = &ni_pi->eg;
+	pi = &eg_pi->rv7xx;
+
+	ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
+	if (ret)
+		si_pi->sys_pcie_mask = 0;
+	else
+		si_pi->sys_pcie_mask = mask;
+	si_pi->force_pcie_gen = AMDGPU_PCIE_GEN_INVALID;
+	si_pi->boot_pcie_gen = si_get_current_pcie_speed(adev);
+
+	si_set_max_cu_value(adev);
+
+	rv770_get_max_vddc(adev);
+	si_get_leakage_vddc(adev);
+	si_patch_dependency_tables_based_on_leakage(adev);
+
+	pi->acpi_vddc = 0;
+	eg_pi->acpi_vddci = 0;
+	pi->min_vddc_in_table = 0;
+	pi->max_vddc_in_table = 0;
+
+	ret = amdgpu_get_platform_caps(adev);
+	if (ret)
+		return ret;
+
+	ret = amdgpu_parse_extended_power_table(adev);
+	if (ret)
+		return ret;
+
+	ret = si_parse_power_table(adev);
+	if (ret)
+		return ret;
+
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
+		kzalloc(4 * sizeof(struct amdgpu_clock_voltage_dependency_entry), GFP_KERNEL);
+	if (!adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
+		amdgpu_free_extended_power_table(adev);
+		return -ENOMEM;
+	}
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
+	adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
+
+	if (adev->pm.dpm.voltage_response_time == 0)
+		adev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
+	if (adev->pm.dpm.backbias_response_time == 0)
+		adev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
+
+	ret = amdgpu_atombios_get_clock_dividers(adev, COMPUTE_ENGINE_PLL_PARAM,
+					     0, false, &dividers);
+	if (ret)
+		pi->ref_div = dividers.ref_div + 1;
+	else
+		pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
+
+	eg_pi->smu_uvd_hs = false;
+
+	pi->mclk_strobe_mode_threshold = 40000;
+	if (si_is_special_1gb_platform(adev))
+		pi->mclk_stutter_mode_threshold = 0;
+	else
+		pi->mclk_stutter_mode_threshold = pi->mclk_strobe_mode_threshold;
+	pi->mclk_edc_enable_threshold = 40000;
+	eg_pi->mclk_edc_wr_enable_threshold = 40000;
+
+	ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
+
+	pi->voltage_control =
+		amdgpu_atombios_is_voltage_gpio(adev, SET_VOLTAGE_TYPE_ASIC_VDDC,
+					    VOLTAGE_OBJ_GPIO_LUT);
+	if (!pi->voltage_control) {
+		si_pi->voltage_control_svi2 =
+			amdgpu_atombios_is_voltage_gpio(adev, SET_VOLTAGE_TYPE_ASIC_VDDC,
+						    VOLTAGE_OBJ_SVID2);
+		if (si_pi->voltage_control_svi2)
+			amdgpu_atombios_get_svi2_info(adev, SET_VOLTAGE_TYPE_ASIC_VDDC,
+						  &si_pi->svd_gpio_id, &si_pi->svc_gpio_id);
+	}
+
+	pi->mvdd_control =
+		amdgpu_atombios_is_voltage_gpio(adev, SET_VOLTAGE_TYPE_ASIC_MVDDC,
+					    VOLTAGE_OBJ_GPIO_LUT);
+
+	eg_pi->vddci_control =
+		amdgpu_atombios_is_voltage_gpio(adev, SET_VOLTAGE_TYPE_ASIC_VDDCI,
+					    VOLTAGE_OBJ_GPIO_LUT);
+	if (!eg_pi->vddci_control)
+		si_pi->vddci_control_svi2 =
+			amdgpu_atombios_is_voltage_gpio(adev, SET_VOLTAGE_TYPE_ASIC_VDDCI,
+						    VOLTAGE_OBJ_SVID2);
+
+	si_pi->vddc_phase_shed_control =
+		amdgpu_atombios_is_voltage_gpio(adev, SET_VOLTAGE_TYPE_ASIC_VDDC,
+					    VOLTAGE_OBJ_PHASE_LUT);
+
+	rv770_get_engine_memory_ss(adev);
+
+	pi->asi = RV770_ASI_DFLT;
+	pi->pasi = CYPRESS_HASI_DFLT;
+	pi->vrc = SISLANDS_VRC_DFLT;
+
+	pi->gfx_clock_gating = true;
+
+	eg_pi->sclk_deep_sleep = true;
+	si_pi->sclk_deep_sleep_above_low = false;
+
+	if (adev->pm.int_thermal_type != THERMAL_TYPE_NONE)
+		pi->thermal_protection = true;
+	else
+		pi->thermal_protection = false;
+
+	eg_pi->dynamic_ac_timing = true;
+
+	eg_pi->light_sleep = true;
+#if defined(CONFIG_ACPI)
+	eg_pi->pcie_performance_request =
+		amdgpu_acpi_is_pcie_performance_request_supported(adev);
+#else
+	eg_pi->pcie_performance_request = false;
+#endif
+
+	si_pi->sram_end = SMC_RAM_END;
+
+	adev->pm.dpm.dyn_state.mclk_sclk_ratio = 4;
+	adev->pm.dpm.dyn_state.sclk_mclk_delta = 15000;
+	adev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
+	adev->pm.dpm.dyn_state.valid_sclk_values.count = 0;
+	adev->pm.dpm.dyn_state.valid_sclk_values.values = NULL;
+	adev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
+	adev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
+
+	si_initialize_powertune_defaults(adev);
+
+	/* make sure dc limits are valid */
+	if ((adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
+	    (adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
+		adev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
+			adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
+
+	si_pi->fan_ctrl_is_in_default_mode = true;
+
+	return 0;
+}
+
+static void si_dpm_fini(struct amdgpu_device *adev)
+{
+	int i;
+
+	if (adev->pm.dpm.ps)
+		for (i = 0; i < adev->pm.dpm.num_ps; i++)
+			kfree(adev->pm.dpm.ps[i].ps_priv);
+	kfree(adev->pm.dpm.ps);
+	kfree(adev->pm.dpm.priv);
+	kfree(adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
+	amdgpu_free_extended_power_table(adev);
+}
+
+static void si_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
+						    struct seq_file *m)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct amdgpu_ps *rps = &eg_pi->current_rps;
+	struct  si_ps *ps = si_get_ps(rps);
+	struct rv7xx_pl *pl;
+	u32 current_index =
+		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
+		CURRENT_STATE_INDEX_SHIFT;
+
+	if (current_index >= ps->performance_level_count) {
+		seq_printf(m, "invalid dpm profile %d\n", current_index);
+	} else {
+		pl = &ps->performance_levels[current_index];
+		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+		seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
+			   current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
+	}
+}
+
+static int si_dpm_set_interrupt_state(struct amdgpu_device *adev,
+				      struct amdgpu_irq_src *source,
+				      unsigned type,
+				      enum amdgpu_interrupt_state state)
+{
+	u32 cg_thermal_int;
+
+	switch (type) {
+	case AMDGPU_THERMAL_IRQ_LOW_TO_HIGH:
+		switch (state) {
+		case AMDGPU_IRQ_STATE_DISABLE:
+			cg_thermal_int = RREG32_SMC(CG_THERMAL_INT);
+			cg_thermal_int |= THERM_INT_MASK_HIGH;
+			WREG32_SMC(CG_THERMAL_INT, cg_thermal_int);
+			break;
+		case AMDGPU_IRQ_STATE_ENABLE:
+			cg_thermal_int = RREG32_SMC(CG_THERMAL_INT);
+			cg_thermal_int &= ~THERM_INT_MASK_HIGH;
+			WREG32_SMC(CG_THERMAL_INT, cg_thermal_int);
+			break;
+		default:
+			break;
+		}
+		break;
+
+	case AMDGPU_THERMAL_IRQ_HIGH_TO_LOW:
+		switch (state) {
+		case AMDGPU_IRQ_STATE_DISABLE:
+			cg_thermal_int = RREG32_SMC(CG_THERMAL_INT);
+			cg_thermal_int |= THERM_INT_MASK_LOW;
+			WREG32_SMC(CG_THERMAL_INT, cg_thermal_int);
+			break;
+		case AMDGPU_IRQ_STATE_ENABLE:
+			cg_thermal_int = RREG32_SMC(CG_THERMAL_INT);
+			cg_thermal_int &= ~THERM_INT_MASK_LOW;
+			WREG32_SMC(CG_THERMAL_INT, cg_thermal_int);
+			break;
+		default:
+			break;
+		}
+		break;
+
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int si_dpm_process_interrupt(struct amdgpu_device *adev,
+				    struct amdgpu_irq_src *source,
+				    struct amdgpu_iv_entry *entry)
+{
+	bool queue_thermal = false;
+
+	if (entry == NULL)
+		return -EINVAL;
+
+	switch (entry->src_id) {
+	case 230: /* thermal low to high */
+		DRM_DEBUG("IH: thermal low to high\n");
+		adev->pm.dpm.thermal.high_to_low = false;
+		queue_thermal = true;
+		break;
+	case 231: /* thermal high to low */
+		DRM_DEBUG("IH: thermal high to low\n");
+		adev->pm.dpm.thermal.high_to_low = true;
+		queue_thermal = true;
+		break;
+	default:
+		break;
+	}
+
+	if (queue_thermal)
+		schedule_work(&adev->pm.dpm.thermal.work);
+
+	return 0;
+}
+
+static int si_dpm_late_init(void *handle)
+{
+	int ret;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!amdgpu_dpm)
+		return 0;
+
+	/* init the sysfs and debugfs files late */
+	ret = amdgpu_pm_sysfs_init(adev);
+	if (ret)
+		return ret;
+
+	ret = si_set_temperature_range(adev);
+	if (ret)
+		return ret;
+#if 0 //TODO ?
+	si_dpm_powergate_uvd(adev, true);
+#endif
+	return 0;
+}
+
+/**
+ * si_dpm_init_microcode - load ucode images from disk
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Use the firmware interface to load the ucode images into
+ * the driver (not loaded into hw).
+ * Returns 0 on success, error on failure.
+ */
+static int si_dpm_init_microcode(struct amdgpu_device *adev)
+{
+	const char *chip_name;
+	char fw_name[30];
+	int err;
+
+	DRM_DEBUG("\n");
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+		chip_name = "tahiti";
+		break;
+	case CHIP_PITCAIRN:
+		if ((adev->pdev->revision == 0x81) ||
+		    (adev->pdev->device == 0x6810) ||
+		    (adev->pdev->device == 0x6811) ||
+		    (adev->pdev->device == 0x6816) ||
+		    (adev->pdev->device == 0x6817) ||
+		    (adev->pdev->device == 0x6806))
+			chip_name = "pitcairn_k";
+		else
+			chip_name = "pitcairn";
+		break;
+	case CHIP_VERDE:
+		if ((adev->pdev->revision == 0x81) ||
+		    (adev->pdev->revision == 0x83) ||
+		    (adev->pdev->revision == 0x87) ||
+		    (adev->pdev->device == 0x6820) ||
+		    (adev->pdev->device == 0x6821) ||
+		    (adev->pdev->device == 0x6822) ||
+		    (adev->pdev->device == 0x6823) ||
+		    (adev->pdev->device == 0x682A) ||
+		    (adev->pdev->device == 0x682B))
+			chip_name = "verde_k";
+		else
+			chip_name = "verde";
+		break;
+	case CHIP_OLAND:
+		if ((adev->pdev->revision == 0xC7) ||
+		    (adev->pdev->revision == 0x80) ||
+		    (adev->pdev->revision == 0x81) ||
+		    (adev->pdev->revision == 0x83) ||
+		    (adev->pdev->device == 0x6604) ||
+		    (adev->pdev->device == 0x6605))
+			chip_name = "oland_k";
+		else
+			chip_name = "oland";
+		break;
+	case CHIP_HAINAN:
+		if ((adev->pdev->revision == 0x81) ||
+		    (adev->pdev->revision == 0x83) ||
+		    (adev->pdev->revision == 0xC3) ||
+		    (adev->pdev->device == 0x6664) ||
+		    (adev->pdev->device == 0x6665) ||
+		    (adev->pdev->device == 0x6667))
+			chip_name = "hainan_k";
+		else
+			chip_name = "hainan";
+		break;
+	default: BUG();
+	}
+
+	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
+	err = request_firmware(&adev->pm.fw, fw_name, adev->dev);
+	if (err)
+		goto out;
+	err = amdgpu_ucode_validate(adev->pm.fw);
+
+out:
+	if (err) {
+		DRM_ERROR("si_smc: Failed to load firmware. err = %d\"%s\"\n",
+			  err, fw_name);
+		release_firmware(adev->pm.fw);
+		adev->pm.fw = NULL;
+	}
+	return err;
+
+}
+
+static int si_dpm_sw_init(void *handle)
+{
+	int ret;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	ret = amdgpu_irq_add_id(adev, 230, &adev->pm.dpm.thermal.irq);
+	if (ret)
+		return ret;
+
+	ret = amdgpu_irq_add_id(adev, 231, &adev->pm.dpm.thermal.irq);
+	if (ret)
+		return ret;
+
+	/* default to balanced state */
+	adev->pm.dpm.state = POWER_STATE_TYPE_BALANCED;
+	adev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED;
+	adev->pm.dpm.forced_level = AMDGPU_DPM_FORCED_LEVEL_AUTO;
+	adev->pm.default_sclk = adev->clock.default_sclk;
+	adev->pm.default_mclk = adev->clock.default_mclk;
+	adev->pm.current_sclk = adev->clock.default_sclk;
+	adev->pm.current_mclk = adev->clock.default_mclk;
+	adev->pm.int_thermal_type = THERMAL_TYPE_NONE;
+
+	if (amdgpu_dpm == 0)
+		return 0;
+
+	ret = si_dpm_init_microcode(adev);
+	if (ret)
+		return ret;
+
+	INIT_WORK(&adev->pm.dpm.thermal.work, amdgpu_dpm_thermal_work_handler);
+	mutex_lock(&adev->pm.mutex);
+	ret = si_dpm_init(adev);
+	if (ret)
+		goto dpm_failed;
+	adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps = adev->pm.dpm.boot_ps;
+	if (amdgpu_dpm == 1)
+		amdgpu_pm_print_power_states(adev);
+	mutex_unlock(&adev->pm.mutex);
+	DRM_INFO("amdgpu: dpm initialized\n");
+
+	return 0;
+
+dpm_failed:
+	si_dpm_fini(adev);
+	mutex_unlock(&adev->pm.mutex);
+	DRM_ERROR("amdgpu: dpm initialization failed\n");
+	return ret;
+}
+
+static int si_dpm_sw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	mutex_lock(&adev->pm.mutex);
+	amdgpu_pm_sysfs_fini(adev);
+	si_dpm_fini(adev);
+	mutex_unlock(&adev->pm.mutex);
+
+	return 0;
+}
+
+static int si_dpm_hw_init(void *handle)
+{
+	int ret;
+
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!amdgpu_dpm)
+		return 0;
+
+	mutex_lock(&adev->pm.mutex);
+	si_dpm_setup_asic(adev);
+	ret = si_dpm_enable(adev);
+	if (ret)
+		adev->pm.dpm_enabled = false;
+	else
+		adev->pm.dpm_enabled = true;
+	mutex_unlock(&adev->pm.mutex);
+
+	return ret;
+}
+
+static int si_dpm_hw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->pm.dpm_enabled) {
+		mutex_lock(&adev->pm.mutex);
+		si_dpm_disable(adev);
+		mutex_unlock(&adev->pm.mutex);
+	}
+
+	return 0;
+}
+
+static int si_dpm_suspend(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->pm.dpm_enabled) {
+		mutex_lock(&adev->pm.mutex);
+		/* disable dpm */
+		si_dpm_disable(adev);
+		/* reset the power state */
+		adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps = adev->pm.dpm.boot_ps;
+		mutex_unlock(&adev->pm.mutex);
+	}
+	return 0;
+}
+
+static int si_dpm_resume(void *handle)
+{
+	int ret;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (adev->pm.dpm_enabled) {
+		/* asic init will reset to the boot state */
+		mutex_lock(&adev->pm.mutex);
+		si_dpm_setup_asic(adev);
+		ret = si_dpm_enable(adev);
+		if (ret)
+			adev->pm.dpm_enabled = false;
+		else
+			adev->pm.dpm_enabled = true;
+		mutex_unlock(&adev->pm.mutex);
+		if (adev->pm.dpm_enabled)
+			amdgpu_pm_compute_clocks(adev);
+	}
+	return 0;
+}
+
+static bool si_dpm_is_idle(void *handle)
+{
+	/* XXX */
+	return true;
+}
+
+static int si_dpm_wait_for_idle(void *handle)
+{
+	/* XXX */
+	return 0;
+}
+
+static int si_dpm_soft_reset(void *handle)
+{
+	return 0;
+}
+
+static int si_dpm_set_clockgating_state(void *handle,
+					enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int si_dpm_set_powergating_state(void *handle,
+					enum amd_powergating_state state)
+{
+	return 0;
+}
+
+/* get temperature in millidegrees */
+static int si_dpm_get_temp(struct amdgpu_device *adev)
+{
+	u32 temp;
+	int actual_temp = 0;
+
+	temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
+		CTF_TEMP_SHIFT;
+
+	if (temp & 0x200)
+		actual_temp = 255;
+	else
+		actual_temp = temp & 0x1ff;
+
+	actual_temp = (actual_temp * 1000);
+
+	return actual_temp;
+}
+
+static u32 si_dpm_get_sclk(struct amdgpu_device *adev, bool low)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct  si_ps *requested_state = si_get_ps(&eg_pi->requested_rps);
+
+	if (low)
+		return requested_state->performance_levels[0].sclk;
+	else
+		return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
+}
+
+static u32 si_dpm_get_mclk(struct amdgpu_device *adev, bool low)
+{
+	struct evergreen_power_info *eg_pi = evergreen_get_pi(adev);
+	struct  si_ps *requested_state = si_get_ps(&eg_pi->requested_rps);
+
+	if (low)
+		return requested_state->performance_levels[0].mclk;
+	else
+		return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
+}
+
+static void si_dpm_print_power_state(struct amdgpu_device *adev,
+				     struct amdgpu_ps *rps)
+{
+	struct  si_ps *ps = si_get_ps(rps);
+	struct rv7xx_pl *pl;
+	int i;
+
+	amdgpu_dpm_print_class_info(rps->class, rps->class2);
+	amdgpu_dpm_print_cap_info(rps->caps);
+	DRM_INFO("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
+	for (i = 0; i < ps->performance_level_count; i++) {
+		pl = &ps->performance_levels[i];
+		if (adev->asic_type >= CHIP_TAHITI)
+			DRM_INFO("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
+				 i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
+		else
+			DRM_INFO("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
+				 i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
+	}
+	amdgpu_dpm_print_ps_status(adev, rps);
+}
+
+static int si_dpm_early_init(void *handle)
+{
+
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	si_dpm_set_dpm_funcs(adev);
+	si_dpm_set_irq_funcs(adev);
+	return 0;
+}
+
+
+const struct amd_ip_funcs si_dpm_ip_funcs = {
+	.name = "si_dpm",
+	.early_init = si_dpm_early_init,
+	.late_init = si_dpm_late_init,
+	.sw_init = si_dpm_sw_init,
+	.sw_fini = si_dpm_sw_fini,
+	.hw_init = si_dpm_hw_init,
+	.hw_fini = si_dpm_hw_fini,
+	.suspend = si_dpm_suspend,
+	.resume = si_dpm_resume,
+	.is_idle = si_dpm_is_idle,
+	.wait_for_idle = si_dpm_wait_for_idle,
+	.soft_reset = si_dpm_soft_reset,
+	.set_clockgating_state = si_dpm_set_clockgating_state,
+	.set_powergating_state = si_dpm_set_powergating_state,
+};
+
+static const struct amdgpu_dpm_funcs si_dpm_funcs = {
+	.get_temperature = &si_dpm_get_temp,
+	.pre_set_power_state = &si_dpm_pre_set_power_state,
+	.set_power_state = &si_dpm_set_power_state,
+	.post_set_power_state = &si_dpm_post_set_power_state,
+	.display_configuration_changed = &si_dpm_display_configuration_changed,
+	.get_sclk = &si_dpm_get_sclk,
+	.get_mclk = &si_dpm_get_mclk,
+	.print_power_state = &si_dpm_print_power_state,
+	.debugfs_print_current_performance_level = &si_dpm_debugfs_print_current_performance_level,
+	.force_performance_level = &si_dpm_force_performance_level,
+	.vblank_too_short = &si_dpm_vblank_too_short,
+	.set_fan_control_mode = &si_dpm_set_fan_control_mode,
+	.get_fan_control_mode = &si_dpm_get_fan_control_mode,
+	.set_fan_speed_percent = &si_dpm_set_fan_speed_percent,
+	.get_fan_speed_percent = &si_dpm_get_fan_speed_percent,
+};
+
+static void si_dpm_set_dpm_funcs(struct amdgpu_device *adev)
+{
+	if (adev->pm.funcs == NULL)
+		adev->pm.funcs = &si_dpm_funcs;
+}
+
+static const struct amdgpu_irq_src_funcs si_dpm_irq_funcs = {
+	.set = si_dpm_set_interrupt_state,
+	.process = si_dpm_process_interrupt,
+};
+
+static void si_dpm_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->pm.dpm.thermal.irq.num_types = AMDGPU_THERMAL_IRQ_LAST;
+	adev->pm.dpm.thermal.irq.funcs = &si_dpm_irq_funcs;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.h b/drivers/gpu/drm/amd/amdgpu/si_dpm.h
new file mode 100644
index 000000000000..51ce21c5f4fb
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.h
@@ -0,0 +1,1015 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SI_DPM_H__
+#define __SI_DPM_H__
+
+#include "amdgpu_atombios.h"
+#include "sislands_smc.h"
+
+#define MC_CG_CONFIG                                    0x96f
+#define MC_ARB_CG                                       0x9fa
+#define		CG_ARB_REQ(x)				((x) << 0)
+#define		CG_ARB_REQ_MASK				(0xff << 0)
+
+#define	MC_ARB_DRAM_TIMING_1				0x9fc
+#define	MC_ARB_DRAM_TIMING_2				0x9fd
+#define	MC_ARB_DRAM_TIMING_3				0x9fe
+#define	MC_ARB_DRAM_TIMING2_1				0x9ff
+#define	MC_ARB_DRAM_TIMING2_2				0xa00
+#define	MC_ARB_DRAM_TIMING2_3				0xa01
+
+#define MAX_NO_OF_MVDD_VALUES 2
+#define MAX_NO_VREG_STEPS 32
+#define NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE 16
+#define SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE 32
+#define SMC_NISLANDS_MC_REGISTER_ARRAY_SET_COUNT 20
+#define RV770_ASI_DFLT                                1000
+#define CYPRESS_HASI_DFLT                               400000
+#define PCIE_PERF_REQ_PECI_GEN1         2
+#define PCIE_PERF_REQ_PECI_GEN2         3
+#define PCIE_PERF_REQ_PECI_GEN3         4
+#define RV770_DEFAULT_VCLK_FREQ  53300 /* 10 khz */
+#define RV770_DEFAULT_DCLK_FREQ  40000 /* 10 khz */
+
+#define SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE 16
+
+#define RV770_SMC_TABLE_ADDRESS 0xB000
+#define RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE    3
+
+#define SMC_STROBE_RATIO    0x0F
+#define SMC_STROBE_ENABLE   0x10
+
+#define SMC_MC_EDC_RD_FLAG  0x01
+#define SMC_MC_EDC_WR_FLAG  0x02
+#define SMC_MC_RTT_ENABLE   0x04
+#define SMC_MC_STUTTER_EN   0x08
+
+#define RV770_SMC_VOLTAGEMASK_VDDC 0
+#define RV770_SMC_VOLTAGEMASK_MVDD 1
+#define RV770_SMC_VOLTAGEMASK_VDDCI 2
+#define RV770_SMC_VOLTAGEMASK_MAX  4
+
+#define NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE 16
+#define NISLANDS_SMC_STROBE_RATIO    0x0F
+#define NISLANDS_SMC_STROBE_ENABLE   0x10
+
+#define NISLANDS_SMC_MC_EDC_RD_FLAG  0x01
+#define NISLANDS_SMC_MC_EDC_WR_FLAG  0x02
+#define NISLANDS_SMC_MC_RTT_ENABLE   0x04
+#define NISLANDS_SMC_MC_STUTTER_EN   0x08
+
+#define MAX_NO_VREG_STEPS 32
+
+#define NISLANDS_SMC_VOLTAGEMASK_VDDC  0
+#define NISLANDS_SMC_VOLTAGEMASK_MVDD  1
+#define NISLANDS_SMC_VOLTAGEMASK_VDDCI 2
+#define NISLANDS_SMC_VOLTAGEMASK_MAX   4
+
+#define SISLANDS_MCREGISTERTABLE_INITIAL_SLOT               0
+#define SISLANDS_MCREGISTERTABLE_ACPI_SLOT                  1
+#define SISLANDS_MCREGISTERTABLE_ULV_SLOT                   2
+#define SISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT     3
+
+#define SISLANDS_LEAKAGE_INDEX0     0xff01
+#define SISLANDS_MAX_LEAKAGE_COUNT  4
+
+#define SISLANDS_MAX_HARDWARE_POWERLEVELS 5
+#define SISLANDS_INITIAL_STATE_ARB_INDEX    0
+#define SISLANDS_ACPI_STATE_ARB_INDEX       1
+#define SISLANDS_ULV_STATE_ARB_INDEX        2
+#define SISLANDS_DRIVER_STATE_ARB_INDEX     3
+
+#define SISLANDS_DPM2_MAX_PULSE_SKIP        256
+
+#define SISLANDS_DPM2_NEAR_TDP_DEC          10
+#define SISLANDS_DPM2_ABOVE_SAFE_INC        5
+#define SISLANDS_DPM2_BELOW_SAFE_INC        20
+
+#define SISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT            80
+
+#define SISLANDS_DPM2_MAXPS_PERCENT_H                   99
+#define SISLANDS_DPM2_MAXPS_PERCENT_M                   99
+
+#define SISLANDS_DPM2_SQ_RAMP_MAX_POWER                 0x3FFF
+#define SISLANDS_DPM2_SQ_RAMP_MIN_POWER                 0x12
+#define SISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA           0x15
+#define SISLANDS_DPM2_SQ_RAMP_STI_SIZE                  0x1E
+#define SISLANDS_DPM2_SQ_RAMP_LTI_RATIO                 0xF
+
+#define SISLANDS_DPM2_PWREFFICIENCYRATIO_MARGIN         10
+
+#define SISLANDS_VRC_DFLT                               0xC000B3
+#define SISLANDS_ULVVOLTAGECHANGEDELAY_DFLT             1687
+#define SISLANDS_CGULVPARAMETER_DFLT                    0x00040035
+#define SISLANDS_CGULVCONTROL_DFLT                      0x1f007550
+
+#define SI_ASI_DFLT                                10000
+#define SI_BSP_DFLT                                0x41EB
+#define SI_BSU_DFLT                                0x2
+#define SI_AH_DFLT                                 5
+#define SI_RLP_DFLT                                25
+#define SI_RMP_DFLT                                65
+#define SI_LHP_DFLT                                40
+#define SI_LMP_DFLT                                15
+#define SI_TD_DFLT                                 0
+#define SI_UTC_DFLT_00                             0x24
+#define SI_UTC_DFLT_01                             0x22
+#define SI_UTC_DFLT_02                             0x22
+#define SI_UTC_DFLT_03                             0x22
+#define SI_UTC_DFLT_04                             0x22
+#define SI_UTC_DFLT_05                             0x22
+#define SI_UTC_DFLT_06                             0x22
+#define SI_UTC_DFLT_07                             0x22
+#define SI_UTC_DFLT_08                             0x22
+#define SI_UTC_DFLT_09                             0x22
+#define SI_UTC_DFLT_10                             0x22
+#define SI_UTC_DFLT_11                             0x22
+#define SI_UTC_DFLT_12                             0x22
+#define SI_UTC_DFLT_13                             0x22
+#define SI_UTC_DFLT_14                             0x22
+#define SI_DTC_DFLT_00                             0x24
+#define SI_DTC_DFLT_01                             0x22
+#define SI_DTC_DFLT_02                             0x22
+#define SI_DTC_DFLT_03                             0x22
+#define SI_DTC_DFLT_04                             0x22
+#define SI_DTC_DFLT_05                             0x22
+#define SI_DTC_DFLT_06                             0x22
+#define SI_DTC_DFLT_07                             0x22
+#define SI_DTC_DFLT_08                             0x22
+#define SI_DTC_DFLT_09                             0x22
+#define SI_DTC_DFLT_10                             0x22
+#define SI_DTC_DFLT_11                             0x22
+#define SI_DTC_DFLT_12                             0x22
+#define SI_DTC_DFLT_13                             0x22
+#define SI_DTC_DFLT_14                             0x22
+#define SI_VRC_DFLT                                0x0000C003
+#define SI_VOLTAGERESPONSETIME_DFLT                1000
+#define SI_BACKBIASRESPONSETIME_DFLT               1000
+#define SI_VRU_DFLT                                0x3
+#define SI_SPLLSTEPTIME_DFLT                       0x1000
+#define SI_SPLLSTEPUNIT_DFLT                       0x3
+#define SI_TPU_DFLT                                0
+#define SI_TPC_DFLT                                0x200
+#define SI_SSTU_DFLT                               0
+#define SI_SST_DFLT                                0x00C8
+#define SI_GICST_DFLT                              0x200
+#define SI_FCT_DFLT                                0x0400
+#define SI_FCTU_DFLT                               0
+#define SI_CTXCGTT3DRPHC_DFLT                      0x20
+#define SI_CTXCGTT3DRSDC_DFLT                      0x40
+#define SI_VDDC3DOORPHC_DFLT                       0x100
+#define SI_VDDC3DOORSDC_DFLT                       0x7
+#define SI_VDDC3DOORSU_DFLT                        0
+#define SI_MPLLLOCKTIME_DFLT                       100
+#define SI_MPLLRESETTIME_DFLT                      150
+#define SI_VCOSTEPPCT_DFLT                          20
+#define SI_ENDINGVCOSTEPPCT_DFLT                    5
+#define SI_REFERENCEDIVIDER_DFLT                    4
+
+#define SI_PM_NUMBER_OF_TC 15
+#define SI_PM_NUMBER_OF_SCLKS 20
+#define SI_PM_NUMBER_OF_MCLKS 4
+#define SI_PM_NUMBER_OF_VOLTAGE_LEVELS 4
+#define SI_PM_NUMBER_OF_ACTIVITY_LEVELS 3
+
+/* XXX are these ok? */
+#define SI_TEMP_RANGE_MIN (90 * 1000)
+#define SI_TEMP_RANGE_MAX (120 * 1000)
+
+#define FDO_PWM_MODE_STATIC  1
+#define FDO_PWM_MODE_STATIC_RPM 5
+
+enum ni_dc_cac_level
+{
+	NISLANDS_DCCAC_LEVEL_0 = 0,
+	NISLANDS_DCCAC_LEVEL_1,
+	NISLANDS_DCCAC_LEVEL_2,
+	NISLANDS_DCCAC_LEVEL_3,
+	NISLANDS_DCCAC_LEVEL_4,
+	NISLANDS_DCCAC_LEVEL_5,
+	NISLANDS_DCCAC_LEVEL_6,
+	NISLANDS_DCCAC_LEVEL_7,
+	NISLANDS_DCCAC_MAX_LEVELS
+};
+
+enum si_cac_config_reg_type
+{
+	SISLANDS_CACCONFIG_MMR = 0,
+	SISLANDS_CACCONFIG_CGIND,
+	SISLANDS_CACCONFIG_MAX
+};
+
+enum si_power_level {
+	SI_POWER_LEVEL_LOW = 0,
+	SI_POWER_LEVEL_MEDIUM = 1,
+	SI_POWER_LEVEL_HIGH = 2,
+	SI_POWER_LEVEL_CTXSW = 3,
+};
+
+enum si_td {
+	SI_TD_AUTO,
+	SI_TD_UP,
+	SI_TD_DOWN,
+};
+
+enum si_display_watermark {
+	SI_DISPLAY_WATERMARK_LOW = 0,
+	SI_DISPLAY_WATERMARK_HIGH = 1,
+};
+
+enum si_display_gap
+{
+    SI_PM_DISPLAY_GAP_VBLANK_OR_WM = 0,
+    SI_PM_DISPLAY_GAP_VBLANK       = 1,
+    SI_PM_DISPLAY_GAP_WATERMARK    = 2,
+    SI_PM_DISPLAY_GAP_IGNORE       = 3,
+};
+
+extern const struct amd_ip_funcs si_dpm_ip_funcs;
+
+struct ni_leakage_coeffients
+{
+	u32 at;
+	u32 bt;
+	u32 av;
+	u32 bv;
+	s32 t_slope;
+	s32 t_intercept;
+	u32 t_ref;
+};
+
+struct SMC_Evergreen_MCRegisterAddress
+{
+    uint16_t s0;
+    uint16_t s1;
+};
+
+typedef struct SMC_Evergreen_MCRegisterAddress SMC_Evergreen_MCRegisterAddress;
+
+struct evergreen_mc_reg_entry {
+	u32 mclk_max;
+	u32 mc_data[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct evergreen_mc_reg_table {
+	u8 last;
+	u8 num_entries;
+	u16 valid_flag;
+	struct evergreen_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
+	SMC_Evergreen_MCRegisterAddress mc_reg_address[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct SMC_Evergreen_MCRegisterSet
+{
+    uint32_t value[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE];
+};
+
+typedef struct SMC_Evergreen_MCRegisterSet SMC_Evergreen_MCRegisterSet;
+
+struct SMC_Evergreen_MCRegisters
+{
+    uint8_t                             last;
+    uint8_t                             reserved[3];
+    SMC_Evergreen_MCRegisterAddress     address[SMC_EVERGREEN_MC_REGISTER_ARRAY_SIZE];
+    SMC_Evergreen_MCRegisterSet         data[5];
+};
+
+typedef struct SMC_Evergreen_MCRegisters SMC_Evergreen_MCRegisters;
+
+struct SMC_NIslands_MCRegisterSet
+{
+    uint32_t value[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+typedef struct SMC_NIslands_MCRegisterSet SMC_NIslands_MCRegisterSet;
+
+struct ni_mc_reg_entry {
+	u32 mclk_max;
+	u32 mc_data[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct SMC_NIslands_MCRegisterAddress
+{
+    uint16_t s0;
+    uint16_t s1;
+};
+
+typedef struct SMC_NIslands_MCRegisterAddress SMC_NIslands_MCRegisterAddress;
+
+struct SMC_NIslands_MCRegisters
+{
+    uint8_t                             last;
+    uint8_t                             reserved[3];
+    SMC_NIslands_MCRegisterAddress      address[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE];
+    SMC_NIslands_MCRegisterSet          data[SMC_NISLANDS_MC_REGISTER_ARRAY_SET_COUNT];
+};
+
+typedef struct SMC_NIslands_MCRegisters SMC_NIslands_MCRegisters;
+
+struct evergreen_ulv_param {
+	bool supported;
+	struct rv7xx_pl *pl;
+};
+
+struct evergreen_arb_registers {
+	u32 mc_arb_dram_timing;
+	u32 mc_arb_dram_timing2;
+	u32 mc_arb_rfsh_rate;
+	u32 mc_arb_burst_time;
+};
+
+struct at {
+	u32 rlp;
+	u32 rmp;
+	u32 lhp;
+	u32 lmp;
+};
+
+struct ni_clock_registers {
+	u32 cg_spll_func_cntl;
+	u32 cg_spll_func_cntl_2;
+	u32 cg_spll_func_cntl_3;
+	u32 cg_spll_func_cntl_4;
+	u32 cg_spll_spread_spectrum;
+	u32 cg_spll_spread_spectrum_2;
+	u32 mclk_pwrmgt_cntl;
+	u32 dll_cntl;
+	u32 mpll_ad_func_cntl;
+	u32 mpll_ad_func_cntl_2;
+	u32 mpll_dq_func_cntl;
+	u32 mpll_dq_func_cntl_2;
+	u32 mpll_ss1;
+	u32 mpll_ss2;
+};
+
+struct RV770_SMC_SCLK_VALUE
+{
+    uint32_t        vCG_SPLL_FUNC_CNTL;
+    uint32_t        vCG_SPLL_FUNC_CNTL_2;
+    uint32_t        vCG_SPLL_FUNC_CNTL_3;
+    uint32_t        vCG_SPLL_SPREAD_SPECTRUM;
+    uint32_t        vCG_SPLL_SPREAD_SPECTRUM_2;
+    uint32_t        sclk_value;
+};
+
+typedef struct RV770_SMC_SCLK_VALUE RV770_SMC_SCLK_VALUE;
+
+struct RV770_SMC_MCLK_VALUE
+{
+    uint32_t        vMPLL_AD_FUNC_CNTL;
+    uint32_t        vMPLL_AD_FUNC_CNTL_2;
+    uint32_t        vMPLL_DQ_FUNC_CNTL;
+    uint32_t        vMPLL_DQ_FUNC_CNTL_2;
+    uint32_t        vMCLK_PWRMGT_CNTL;
+    uint32_t        vDLL_CNTL;
+    uint32_t        vMPLL_SS;
+    uint32_t        vMPLL_SS2;
+    uint32_t        mclk_value;
+};
+
+typedef struct RV770_SMC_MCLK_VALUE RV770_SMC_MCLK_VALUE;
+
+
+struct RV730_SMC_MCLK_VALUE
+{
+    uint32_t        vMCLK_PWRMGT_CNTL;
+    uint32_t        vDLL_CNTL;
+    uint32_t        vMPLL_FUNC_CNTL;
+    uint32_t        vMPLL_FUNC_CNTL2;
+    uint32_t        vMPLL_FUNC_CNTL3;
+    uint32_t        vMPLL_SS;
+    uint32_t        vMPLL_SS2;
+    uint32_t        mclk_value;
+};
+
+typedef struct RV730_SMC_MCLK_VALUE RV730_SMC_MCLK_VALUE;
+
+struct RV770_SMC_VOLTAGE_VALUE
+{
+    uint16_t             value;
+    uint8_t              index;
+    uint8_t              padding;
+};
+
+typedef struct RV770_SMC_VOLTAGE_VALUE RV770_SMC_VOLTAGE_VALUE;
+
+union RV7XX_SMC_MCLK_VALUE
+{
+    RV770_SMC_MCLK_VALUE    mclk770;
+    RV730_SMC_MCLK_VALUE    mclk730;
+};
+
+typedef union RV7XX_SMC_MCLK_VALUE RV7XX_SMC_MCLK_VALUE, *LPRV7XX_SMC_MCLK_VALUE;
+
+struct RV770_SMC_HW_PERFORMANCE_LEVEL
+{
+    uint8_t                 arbValue;
+    union{
+        uint8_t             seqValue;
+        uint8_t             ACIndex;
+    };
+    uint8_t                 displayWatermark;
+    uint8_t                 gen2PCIE;
+    uint8_t                 gen2XSP;
+    uint8_t                 backbias;
+    uint8_t                 strobeMode;
+    uint8_t                 mcFlags;
+    uint32_t                aT;
+    uint32_t                bSP;
+    RV770_SMC_SCLK_VALUE    sclk;
+    RV7XX_SMC_MCLK_VALUE    mclk;
+    RV770_SMC_VOLTAGE_VALUE vddc;
+    RV770_SMC_VOLTAGE_VALUE mvdd;
+    RV770_SMC_VOLTAGE_VALUE vddci;
+    uint8_t                 reserved1;
+    uint8_t                 reserved2;
+    uint8_t                 stateFlags;
+    uint8_t                 padding;
+};
+
+typedef struct RV770_SMC_HW_PERFORMANCE_LEVEL RV770_SMC_HW_PERFORMANCE_LEVEL;
+
+struct RV770_SMC_SWSTATE
+{
+    uint8_t           flags;
+    uint8_t           padding1;
+    uint8_t           padding2;
+    uint8_t           padding3;
+    RV770_SMC_HW_PERFORMANCE_LEVEL levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
+};
+
+typedef struct RV770_SMC_SWSTATE RV770_SMC_SWSTATE;
+
+struct RV770_SMC_VOLTAGEMASKTABLE
+{
+    uint8_t  highMask[RV770_SMC_VOLTAGEMASK_MAX];
+    uint32_t lowMask[RV770_SMC_VOLTAGEMASK_MAX];
+};
+
+typedef struct RV770_SMC_VOLTAGEMASKTABLE RV770_SMC_VOLTAGEMASKTABLE;
+
+struct RV770_SMC_STATETABLE
+{
+    uint8_t             thermalProtectType;
+    uint8_t             systemFlags;
+    uint8_t             maxVDDCIndexInPPTable;
+    uint8_t             extraFlags;
+    uint8_t             highSMIO[MAX_NO_VREG_STEPS];
+    uint32_t            lowSMIO[MAX_NO_VREG_STEPS];
+    RV770_SMC_VOLTAGEMASKTABLE voltageMaskTable;
+    RV770_SMC_SWSTATE   initialState;
+    RV770_SMC_SWSTATE   ACPIState;
+    RV770_SMC_SWSTATE   driverState;
+    RV770_SMC_SWSTATE   ULVState;
+};
+
+typedef struct RV770_SMC_STATETABLE RV770_SMC_STATETABLE;
+
+struct vddc_table_entry {
+	u16 vddc;
+	u8 vddc_index;
+	u8 high_smio;
+	u32 low_smio;
+};
+
+struct rv770_clock_registers {
+	u32 cg_spll_func_cntl;
+	u32 cg_spll_func_cntl_2;
+	u32 cg_spll_func_cntl_3;
+	u32 cg_spll_spread_spectrum;
+	u32 cg_spll_spread_spectrum_2;
+	u32 mpll_ad_func_cntl;
+	u32 mpll_ad_func_cntl_2;
+	u32 mpll_dq_func_cntl;
+	u32 mpll_dq_func_cntl_2;
+	u32 mclk_pwrmgt_cntl;
+	u32 dll_cntl;
+	u32 mpll_ss1;
+	u32 mpll_ss2;
+};
+
+struct rv730_clock_registers {
+	u32 cg_spll_func_cntl;
+	u32 cg_spll_func_cntl_2;
+	u32 cg_spll_func_cntl_3;
+	u32 cg_spll_spread_spectrum;
+	u32 cg_spll_spread_spectrum_2;
+	u32 mclk_pwrmgt_cntl;
+	u32 dll_cntl;
+	u32 mpll_func_cntl;
+	u32 mpll_func_cntl2;
+	u32 mpll_func_cntl3;
+	u32 mpll_ss;
+	u32 mpll_ss2;
+};
+
+union r7xx_clock_registers {
+	struct rv770_clock_registers rv770;
+	struct rv730_clock_registers rv730;
+};
+
+struct rv7xx_power_info {
+	/* flags */
+	bool mem_gddr5;
+	bool pcie_gen2;
+	bool dynamic_pcie_gen2;
+	bool acpi_pcie_gen2;
+	bool boot_in_gen2;
+	bool voltage_control; /* vddc */
+	bool mvdd_control;
+	bool sclk_ss;
+	bool mclk_ss;
+	bool dynamic_ss;
+	bool gfx_clock_gating;
+	bool mg_clock_gating;
+	bool mgcgtssm;
+	bool power_gating;
+	bool thermal_protection;
+	bool display_gap;
+	bool dcodt;
+	bool ulps;
+	/* registers */
+	union r7xx_clock_registers clk_regs;
+	u32 s0_vid_lower_smio_cntl;
+	/* voltage */
+	u32 vddc_mask_low;
+	u32 mvdd_mask_low;
+	u32 mvdd_split_frequency;
+	u32 mvdd_low_smio[MAX_NO_OF_MVDD_VALUES];
+	u16 max_vddc;
+	u16 max_vddc_in_table;
+	u16 min_vddc_in_table;
+	struct vddc_table_entry vddc_table[MAX_NO_VREG_STEPS];
+	u8 valid_vddc_entries;
+	/* dc odt */
+	u32 mclk_odt_threshold;
+	u8 odt_value_0[2];
+	u8 odt_value_1[2];
+	/* stored values */
+	u32 boot_sclk;
+	u16 acpi_vddc;
+	u32 ref_div;
+	u32 active_auto_throttle_sources;
+	u32 mclk_stutter_mode_threshold;
+	u32 mclk_strobe_mode_threshold;
+	u32 mclk_edc_enable_threshold;
+	u32 bsp;
+	u32 bsu;
+	u32 pbsp;
+	u32 pbsu;
+	u32 dsp;
+	u32 psp;
+	u32 asi;
+	u32 pasi;
+	u32 vrc;
+	u32 restricted_levels;
+	u32 rlp;
+	u32 rmp;
+	u32 lhp;
+	u32 lmp;
+	/* smc offsets */
+	u16 state_table_start;
+	u16 soft_regs_start;
+	u16 sram_end;
+	/* scratch structs */
+	RV770_SMC_STATETABLE smc_statetable;
+};
+
+struct rv7xx_pl {
+	u32 sclk;
+	u32 mclk;
+	u16 vddc;
+	u16 vddci; /* eg+ only */
+	u32 flags;
+	enum amdgpu_pcie_gen pcie_gen; /* si+ only */
+};
+
+struct rv7xx_ps {
+	struct rv7xx_pl high;
+	struct rv7xx_pl medium;
+	struct rv7xx_pl low;
+	bool dc_compatible;
+};
+
+struct si_ps {
+	u16 performance_level_count;
+	bool dc_compatible;
+	struct rv7xx_pl performance_levels[NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
+};
+
+struct ni_mc_reg_table {
+	u8 last;
+	u8 num_entries;
+	u16 valid_flag;
+	struct ni_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
+	SMC_NIslands_MCRegisterAddress mc_reg_address[SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct ni_cac_data
+{
+	struct ni_leakage_coeffients leakage_coefficients;
+	u32 i_leakage;
+	s32 leakage_minimum_temperature;
+	u32 pwr_const;
+	u32 dc_cac_value;
+	u32 bif_cac_value;
+	u32 lkge_pwr;
+	u8 mc_wr_weight;
+	u8 mc_rd_weight;
+	u8 allow_ovrflw;
+	u8 num_win_tdp;
+	u8 l2num_win_tdp;
+	u8 lts_truncate_n;
+};
+
+struct evergreen_power_info {
+	/* must be first! */
+	struct rv7xx_power_info rv7xx;
+	/* flags */
+	bool vddci_control;
+	bool dynamic_ac_timing;
+	bool abm;
+	bool mcls;
+	bool light_sleep;
+	bool memory_transition;
+	bool pcie_performance_request;
+	bool pcie_performance_request_registered;
+	bool sclk_deep_sleep;
+	bool dll_default_on;
+	bool ls_clock_gating;
+	bool smu_uvd_hs;
+	bool uvd_enabled;
+	/* stored values */
+	u16 acpi_vddci;
+	u8 mvdd_high_index;
+	u8 mvdd_low_index;
+	u32 mclk_edc_wr_enable_threshold;
+	struct evergreen_mc_reg_table mc_reg_table;
+	struct atom_voltage_table vddc_voltage_table;
+	struct atom_voltage_table vddci_voltage_table;
+	struct evergreen_arb_registers bootup_arb_registers;
+	struct evergreen_ulv_param ulv;
+	struct at ats[2];
+	/* smc offsets */
+	u16 mc_reg_table_start;
+	struct amdgpu_ps current_rps;
+	struct rv7xx_ps current_ps;
+	struct amdgpu_ps requested_rps;
+	struct rv7xx_ps requested_ps;
+};
+
+struct PP_NIslands_Dpm2PerfLevel
+{
+    uint8_t     MaxPS;
+    uint8_t     TgtAct;
+    uint8_t     MaxPS_StepInc;
+    uint8_t     MaxPS_StepDec;
+    uint8_t     PSST;
+    uint8_t     NearTDPDec;
+    uint8_t     AboveSafeInc;
+    uint8_t     BelowSafeInc;
+    uint8_t     PSDeltaLimit;
+    uint8_t     PSDeltaWin;
+    uint8_t     Reserved[6];
+};
+
+typedef struct PP_NIslands_Dpm2PerfLevel PP_NIslands_Dpm2PerfLevel;
+
+struct PP_NIslands_DPM2Parameters
+{
+    uint32_t    TDPLimit;
+    uint32_t    NearTDPLimit;
+    uint32_t    SafePowerLimit;
+    uint32_t    PowerBoostLimit;
+};
+typedef struct PP_NIslands_DPM2Parameters PP_NIslands_DPM2Parameters;
+
+struct NISLANDS_SMC_SCLK_VALUE
+{
+    uint32_t        vCG_SPLL_FUNC_CNTL;
+    uint32_t        vCG_SPLL_FUNC_CNTL_2;
+    uint32_t        vCG_SPLL_FUNC_CNTL_3;
+    uint32_t        vCG_SPLL_FUNC_CNTL_4;
+    uint32_t        vCG_SPLL_SPREAD_SPECTRUM;
+    uint32_t        vCG_SPLL_SPREAD_SPECTRUM_2;
+    uint32_t        sclk_value;
+};
+
+typedef struct NISLANDS_SMC_SCLK_VALUE NISLANDS_SMC_SCLK_VALUE;
+
+struct NISLANDS_SMC_MCLK_VALUE
+{
+    uint32_t        vMPLL_FUNC_CNTL;
+    uint32_t        vMPLL_FUNC_CNTL_1;
+    uint32_t        vMPLL_FUNC_CNTL_2;
+    uint32_t        vMPLL_AD_FUNC_CNTL;
+    uint32_t        vMPLL_AD_FUNC_CNTL_2;
+    uint32_t        vMPLL_DQ_FUNC_CNTL;
+    uint32_t        vMPLL_DQ_FUNC_CNTL_2;
+    uint32_t        vMCLK_PWRMGT_CNTL;
+    uint32_t        vDLL_CNTL;
+    uint32_t        vMPLL_SS;
+    uint32_t        vMPLL_SS2;
+    uint32_t        mclk_value;
+};
+
+typedef struct NISLANDS_SMC_MCLK_VALUE NISLANDS_SMC_MCLK_VALUE;
+
+struct NISLANDS_SMC_VOLTAGE_VALUE
+{
+    uint16_t             value;
+    uint8_t              index;
+    uint8_t              padding;
+};
+
+typedef struct NISLANDS_SMC_VOLTAGE_VALUE NISLANDS_SMC_VOLTAGE_VALUE;
+
+struct NISLANDS_SMC_HW_PERFORMANCE_LEVEL
+{
+    uint8_t                     arbValue;
+    uint8_t                     ACIndex;
+    uint8_t                     displayWatermark;
+    uint8_t                     gen2PCIE;
+    uint8_t                     reserved1;
+    uint8_t                     reserved2;
+    uint8_t                     strobeMode;
+    uint8_t                     mcFlags;
+    uint32_t                    aT;
+    uint32_t                    bSP;
+    NISLANDS_SMC_SCLK_VALUE     sclk;
+    NISLANDS_SMC_MCLK_VALUE     mclk;
+    NISLANDS_SMC_VOLTAGE_VALUE  vddc;
+    NISLANDS_SMC_VOLTAGE_VALUE  mvdd;
+    NISLANDS_SMC_VOLTAGE_VALUE  vddci;
+    NISLANDS_SMC_VOLTAGE_VALUE  std_vddc;
+    uint32_t                    powergate_en;
+    uint8_t                     hUp;
+    uint8_t                     hDown;
+    uint8_t                     stateFlags;
+    uint8_t                     arbRefreshState;
+    uint32_t                    SQPowerThrottle;
+    uint32_t                    SQPowerThrottle_2;
+    uint32_t                    reserved[2];
+    PP_NIslands_Dpm2PerfLevel   dpm2;
+};
+
+typedef struct NISLANDS_SMC_HW_PERFORMANCE_LEVEL NISLANDS_SMC_HW_PERFORMANCE_LEVEL;
+
+struct NISLANDS_SMC_SWSTATE
+{
+    uint8_t                             flags;
+    uint8_t                             levelCount;
+    uint8_t                             padding2;
+    uint8_t                             padding3;
+    NISLANDS_SMC_HW_PERFORMANCE_LEVEL   levels[1];
+};
+
+typedef struct NISLANDS_SMC_SWSTATE NISLANDS_SMC_SWSTATE;
+
+struct NISLANDS_SMC_VOLTAGEMASKTABLE
+{
+    uint8_t  highMask[NISLANDS_SMC_VOLTAGEMASK_MAX];
+    uint32_t lowMask[NISLANDS_SMC_VOLTAGEMASK_MAX];
+};
+
+typedef struct NISLANDS_SMC_VOLTAGEMASKTABLE NISLANDS_SMC_VOLTAGEMASKTABLE;
+
+#define NISLANDS_MAX_NO_VREG_STEPS 32
+
+struct NISLANDS_SMC_STATETABLE
+{
+    uint8_t                             thermalProtectType;
+    uint8_t                             systemFlags;
+    uint8_t                             maxVDDCIndexInPPTable;
+    uint8_t                             extraFlags;
+    uint8_t                             highSMIO[NISLANDS_MAX_NO_VREG_STEPS];
+    uint32_t                            lowSMIO[NISLANDS_MAX_NO_VREG_STEPS];
+    NISLANDS_SMC_VOLTAGEMASKTABLE       voltageMaskTable;
+    PP_NIslands_DPM2Parameters          dpm2Params;
+    NISLANDS_SMC_SWSTATE                initialState;
+    NISLANDS_SMC_SWSTATE                ACPIState;
+    NISLANDS_SMC_SWSTATE                ULVState;
+    NISLANDS_SMC_SWSTATE                driverState;
+    NISLANDS_SMC_HW_PERFORMANCE_LEVEL   dpmLevels[NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1];
+};
+
+typedef struct NISLANDS_SMC_STATETABLE NISLANDS_SMC_STATETABLE;
+
+struct ni_power_info {
+	/* must be first! */
+	struct evergreen_power_info eg;
+	struct ni_clock_registers clock_registers;
+	struct ni_mc_reg_table mc_reg_table;
+	u32 mclk_rtt_mode_threshold;
+	/* flags */
+	bool use_power_boost_limit;
+	bool support_cac_long_term_average;
+	bool cac_enabled;
+	bool cac_configuration_required;
+	bool driver_calculate_cac_leakage;
+	bool pc_enabled;
+	bool enable_power_containment;
+	bool enable_cac;
+	bool enable_sq_ramping;
+	/* smc offsets */
+	u16 arb_table_start;
+	u16 fan_table_start;
+	u16 cac_table_start;
+	u16 spll_table_start;
+	/* CAC stuff */
+	struct ni_cac_data cac_data;
+	u32 dc_cac_table[NISLANDS_DCCAC_MAX_LEVELS];
+	const struct ni_cac_weights *cac_weights;
+	u8 lta_window_size;
+	u8 lts_truncate;
+	struct si_ps current_ps;
+	struct si_ps requested_ps;
+	/* scratch structs */
+	SMC_NIslands_MCRegisters smc_mc_reg_table;
+	NISLANDS_SMC_STATETABLE smc_statetable;
+};
+
+struct si_cac_config_reg
+{
+	u32 offset;
+	u32 mask;
+	u32 shift;
+	u32 value;
+	enum si_cac_config_reg_type type;
+};
+
+struct si_powertune_data
+{
+	u32 cac_window;
+	u32 l2_lta_window_size_default;
+	u8 lts_truncate_default;
+	u8 shift_n_default;
+	u8 operating_temp;
+	struct ni_leakage_coeffients leakage_coefficients;
+	u32 fixed_kt;
+	u32 lkge_lut_v0_percent;
+	u8 dc_cac[NISLANDS_DCCAC_MAX_LEVELS];
+	bool enable_powertune_by_default;
+};
+
+struct si_dyn_powertune_data
+{
+	u32 cac_leakage;
+	s32 leakage_minimum_temperature;
+	u32 wintime;
+	u32 l2_lta_window_size;
+	u8 lts_truncate;
+	u8 shift_n;
+	u8 dc_pwr_value;
+	bool disable_uvd_powertune;
+};
+
+struct si_dte_data
+{
+	u32 tau[SMC_SISLANDS_DTE_MAX_FILTER_STAGES];
+	u32 r[SMC_SISLANDS_DTE_MAX_FILTER_STAGES];
+	u32 k;
+	u32 t0;
+	u32 max_t;
+	u8 window_size;
+	u8 temp_select;
+	u8 dte_mode;
+	u8 tdep_count;
+	u8 t_limits[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+	u32 tdep_tau[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+	u32 tdep_r[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+	u32 t_threshold;
+	bool enable_dte_by_default;
+};
+
+struct si_clock_registers {
+	u32 cg_spll_func_cntl;
+	u32 cg_spll_func_cntl_2;
+	u32 cg_spll_func_cntl_3;
+	u32 cg_spll_func_cntl_4;
+	u32 cg_spll_spread_spectrum;
+	u32 cg_spll_spread_spectrum_2;
+	u32 dll_cntl;
+	u32 mclk_pwrmgt_cntl;
+	u32 mpll_ad_func_cntl;
+	u32 mpll_dq_func_cntl;
+	u32 mpll_func_cntl;
+	u32 mpll_func_cntl_1;
+	u32 mpll_func_cntl_2;
+	u32 mpll_ss1;
+	u32 mpll_ss2;
+};
+
+struct si_mc_reg_entry {
+	u32 mclk_max;
+	u32 mc_data[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct si_mc_reg_table {
+	u8 last;
+	u8 num_entries;
+	u16 valid_flag;
+	struct si_mc_reg_entry mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
+	SMC_NIslands_MCRegisterAddress mc_reg_address[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct si_leakage_voltage_entry
+{
+	u16 voltage;
+	u16 leakage_index;
+};
+
+struct si_leakage_voltage
+{
+	u16 count;
+	struct si_leakage_voltage_entry entries[SISLANDS_MAX_LEAKAGE_COUNT];
+};
+
+
+struct si_ulv_param {
+	bool supported;
+	u32 cg_ulv_control;
+	u32 cg_ulv_parameter;
+	u32 volt_change_delay;
+	struct rv7xx_pl pl;
+	bool one_pcie_lane_in_ulv;
+};
+
+struct si_power_info {
+	/* must be first! */
+	struct ni_power_info ni;
+	struct si_clock_registers clock_registers;
+	struct si_mc_reg_table mc_reg_table;
+	struct atom_voltage_table mvdd_voltage_table;
+	struct atom_voltage_table vddc_phase_shed_table;
+	struct si_leakage_voltage leakage_voltage;
+	u16 mvdd_bootup_value;
+	struct si_ulv_param ulv;
+	u32 max_cu;
+	/* pcie gen */
+	enum amdgpu_pcie_gen force_pcie_gen;
+	enum amdgpu_pcie_gen boot_pcie_gen;
+	enum amdgpu_pcie_gen acpi_pcie_gen;
+	u32 sys_pcie_mask;
+	/* flags */
+	bool enable_dte;
+	bool enable_ppm;
+	bool vddc_phase_shed_control;
+	bool pspp_notify_required;
+	bool sclk_deep_sleep_above_low;
+	bool voltage_control_svi2;
+	bool vddci_control_svi2;
+	/* smc offsets */
+	u32 sram_end;
+	u32 state_table_start;
+	u32 soft_regs_start;
+	u32 mc_reg_table_start;
+	u32 arb_table_start;
+	u32 cac_table_start;
+	u32 dte_table_start;
+	u32 spll_table_start;
+	u32 papm_cfg_table_start;
+	u32 fan_table_start;
+	/* CAC stuff */
+	const struct si_cac_config_reg *cac_weights;
+	const struct si_cac_config_reg *lcac_config;
+	const struct si_cac_config_reg *cac_override;
+	const struct si_powertune_data *powertune_data;
+	struct si_dyn_powertune_data dyn_powertune_data;
+	/* DTE stuff */
+	struct si_dte_data dte_data;
+	/* scratch structs */
+	SMC_SIslands_MCRegisters smc_mc_reg_table;
+	SISLANDS_SMC_STATETABLE smc_statetable;
+	PP_SIslands_PAPMParameters papm_parm;
+	/* SVI2 */
+	u8 svd_gpio_id;
+	u8 svc_gpio_id;
+	/* fan control */
+	bool fan_ctrl_is_in_default_mode;
+	u32 t_min;
+	u32 fan_ctrl_default_mode;
+	bool fan_is_controlled_by_smc;
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
new file mode 100644
index 000000000000..8fae3d4a2360
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -0,0 +1,299 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "drmP.h"
+#include "amdgpu.h"
+#include "amdgpu_ih.h"
+#include "si/sid.h"
+#include "si_ih.h"
+
+static void si_ih_set_interrupt_funcs(struct amdgpu_device *adev);
+
+static void si_ih_enable_interrupts(struct amdgpu_device *adev)
+{
+	u32 ih_cntl = RREG32(IH_CNTL);
+	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
+
+	ih_cntl |= ENABLE_INTR;
+	ih_rb_cntl |= IH_RB_ENABLE;
+	WREG32(IH_CNTL, ih_cntl);
+	WREG32(IH_RB_CNTL, ih_rb_cntl);
+	adev->irq.ih.enabled = true;
+}
+  
+static void si_ih_disable_interrupts(struct amdgpu_device *adev)
+{
+	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
+	u32 ih_cntl = RREG32(IH_CNTL);
+
+	ih_rb_cntl &= ~IH_RB_ENABLE;
+	ih_cntl &= ~ENABLE_INTR;
+	WREG32(IH_RB_CNTL, ih_rb_cntl);
+	WREG32(IH_CNTL, ih_cntl);
+	WREG32(IH_RB_RPTR, 0);
+	WREG32(IH_RB_WPTR, 0);
+	adev->irq.ih.enabled = false;
+	adev->irq.ih.rptr = 0;
+}
+
+static int si_ih_irq_init(struct amdgpu_device *adev)
+{
+	int rb_bufsz;
+	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
+	u64 wptr_off;
+
+	si_ih_disable_interrupts(adev);
+	WREG32(INTERRUPT_CNTL2, adev->irq.ih.gpu_addr >> 8);
+	interrupt_cntl = RREG32(INTERRUPT_CNTL);
+	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
+	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
+	WREG32(INTERRUPT_CNTL, interrupt_cntl);
+
+	WREG32(IH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
+	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
+
+	ih_rb_cntl = IH_WPTR_OVERFLOW_ENABLE |
+		     IH_WPTR_OVERFLOW_CLEAR |
+		     (rb_bufsz << 1) |
+		     IH_WPTR_WRITEBACK_ENABLE;
+
+	wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
+	WREG32(IH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
+	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF);
+	WREG32(IH_RB_CNTL, ih_rb_cntl);
+	WREG32(IH_RB_RPTR, 0);
+	WREG32(IH_RB_WPTR, 0);
+
+	ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
+	if (adev->irq.msi_enabled)
+		ih_cntl |= RPTR_REARM;
+	WREG32(IH_CNTL, ih_cntl);
+
+	pci_set_master(adev->pdev);
+	si_ih_enable_interrupts(adev);
+
+	return 0;
+}
+
+static void si_ih_irq_disable(struct amdgpu_device *adev)
+{
+	si_ih_disable_interrupts(adev);
+	mdelay(1);
+}
+
+static u32 si_ih_get_wptr(struct amdgpu_device *adev)
+{
+	u32 wptr, tmp;
+
+	wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
+
+	if (wptr & IH_RB_WPTR__RB_OVERFLOW_MASK) {
+		wptr &= ~IH_RB_WPTR__RB_OVERFLOW_MASK;
+		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
+			wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask);
+		adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask;
+		tmp = RREG32(IH_RB_CNTL);
+		tmp |= IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK;
+		WREG32(IH_RB_CNTL, tmp);
+	}
+	return (wptr & adev->irq.ih.ptr_mask);
+}
+
+static void si_ih_decode_iv(struct amdgpu_device *adev,
+			     struct amdgpu_iv_entry *entry)
+{
+	u32 ring_index = adev->irq.ih.rptr >> 2;
+	uint32_t dw[4];
+
+	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
+	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
+	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
+	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
+
+	entry->src_id = dw[0] & 0xff;
+	entry->src_data = dw[1] & 0xfffffff;
+	entry->ring_id = dw[2] & 0xff;
+	entry->vm_id = (dw[2] >> 8) & 0xff;
+
+	adev->irq.ih.rptr += 16;
+}
+
+static void si_ih_set_rptr(struct amdgpu_device *adev)
+{
+	WREG32(IH_RB_RPTR, adev->irq.ih.rptr);
+}
+
+static int si_ih_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	si_ih_set_interrupt_funcs(adev);
+
+	return 0;
+}
+
+static int si_ih_sw_init(void *handle)
+{
+	int r;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	r = amdgpu_ih_ring_init(adev, 64 * 1024, false);
+	if (r)
+		return r;
+
+	return amdgpu_irq_init(adev);
+}
+
+static int si_ih_sw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	amdgpu_irq_fini(adev);
+	amdgpu_ih_ring_fini(adev);
+
+	return 0;
+}
+
+static int si_ih_hw_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return si_ih_irq_init(adev);
+}
+
+static int si_ih_hw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	si_ih_irq_disable(adev);
+
+	return 0;
+}
+
+static int si_ih_suspend(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return si_ih_hw_fini(adev);
+}
+
+static int si_ih_resume(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	return si_ih_hw_init(adev);
+}
+
+static bool si_ih_is_idle(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 tmp = RREG32(SRBM_STATUS);
+
+	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
+		return false;
+
+	return true;
+}
+
+static int si_ih_wait_for_idle(void *handle)
+{
+	unsigned i;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (si_ih_is_idle(handle))
+			return 0;
+		udelay(1);
+	}
+	return -ETIMEDOUT;
+}
+
+static int si_ih_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	u32 srbm_soft_reset = 0;
+	u32 tmp = RREG32(SRBM_STATUS);
+
+	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
+		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK;
+
+	if (srbm_soft_reset) {
+		tmp = RREG32(SRBM_SOFT_RESET);
+		tmp |= srbm_soft_reset;
+		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
+		WREG32(SRBM_SOFT_RESET, tmp);
+		tmp = RREG32(SRBM_SOFT_RESET);
+
+		udelay(50);
+
+		tmp &= ~srbm_soft_reset;
+		WREG32(SRBM_SOFT_RESET, tmp);
+		tmp = RREG32(SRBM_SOFT_RESET);
+
+		udelay(50);
+	}
+
+	return 0;
+}
+
+static int si_ih_set_clockgating_state(void *handle,
+					  enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int si_ih_set_powergating_state(void *handle,
+					  enum amd_powergating_state state)
+{
+	return 0;
+}
+
+const struct amd_ip_funcs si_ih_ip_funcs = {
+	.name = "si_ih",
+	.early_init = si_ih_early_init,
+	.late_init = NULL,
+	.sw_init = si_ih_sw_init,
+	.sw_fini = si_ih_sw_fini,
+	.hw_init = si_ih_hw_init,
+	.hw_fini = si_ih_hw_fini,
+	.suspend = si_ih_suspend,
+	.resume = si_ih_resume,
+	.is_idle = si_ih_is_idle,
+	.wait_for_idle = si_ih_wait_for_idle,
+	.soft_reset = si_ih_soft_reset,
+	.set_clockgating_state = si_ih_set_clockgating_state,
+	.set_powergating_state = si_ih_set_powergating_state,
+};
+
+static const struct amdgpu_ih_funcs si_ih_funcs = {
+	.get_wptr = si_ih_get_wptr,
+	.decode_iv = si_ih_decode_iv,
+	.set_rptr = si_ih_set_rptr
+};
+
+static void si_ih_set_interrupt_funcs(struct amdgpu_device *adev)
+{
+	if (adev->irq.ih_funcs == NULL)
+		adev->irq.ih_funcs = &si_ih_funcs;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.h b/drivers/gpu/drm/amd/amdgpu/si_ih.h
new file mode 100644
index 000000000000..f3e3a954369c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __SI_IH_H__
+#define __SI_IH_H__
+
+extern const struct amd_ip_funcs si_ih_ip_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/si_smc.c b/drivers/gpu/drm/amd/amdgpu/si_smc.c
new file mode 100644
index 000000000000..668ba99d6c05
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/si_smc.c
@@ -0,0 +1,273 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+
+#include <linux/firmware.h>
+#include "drmP.h"
+#include "amdgpu.h"
+#include "si/sid.h"
+#include "ppsmc.h"
+#include "amdgpu_ucode.h"
+#include "sislands_smc.h"
+
+static int si_set_smc_sram_address(struct amdgpu_device *adev,
+				   u32 smc_address, u32 limit)
+{
+	if (smc_address & 3)
+		return -EINVAL;
+	if ((smc_address + 3) > limit)
+		return -EINVAL;
+
+	WREG32(SMC_IND_INDEX_0, smc_address);
+	WREG32_P(SMC_IND_ACCESS_CNTL, 0, ~AUTO_INCREMENT_IND_0);
+
+	return 0;
+}
+
+int amdgpu_si_copy_bytes_to_smc(struct amdgpu_device *adev,
+				u32 smc_start_address,
+				const u8 *src, u32 byte_count, u32 limit)
+{
+	unsigned long flags;
+	int ret = 0;
+	u32 data, original_data, addr, extra_shift;
+
+	if (smc_start_address & 3)
+		return -EINVAL;
+	if ((smc_start_address + byte_count) > limit)
+		return -EINVAL;
+
+	addr = smc_start_address;
+
+	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	while (byte_count >= 4) {
+		/* SMC address space is BE */
+		data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3];
+
+		ret = si_set_smc_sram_address(adev, addr, limit);
+		if (ret)
+			goto done;
+
+		WREG32(SMC_IND_DATA_0, data);
+
+		src += 4;
+		byte_count -= 4;
+		addr += 4;
+	}
+
+	/* RMW for the final bytes */
+	if (byte_count > 0) {
+		data = 0;
+
+		ret = si_set_smc_sram_address(adev, addr, limit);
+		if (ret)
+			goto done;
+
+		original_data = RREG32(SMC_IND_DATA_0);
+		extra_shift = 8 * (4 - byte_count);
+
+		while (byte_count > 0) {
+			/* SMC address space is BE */
+			data = (data << 8) + *src++;
+			byte_count--;
+		}
+
+		data <<= extra_shift;
+		data |= (original_data & ~((~0UL) << extra_shift));
+
+		ret = si_set_smc_sram_address(adev, addr, limit);
+		if (ret)
+			goto done;
+
+		WREG32(SMC_IND_DATA_0, data);
+	}
+
+done:
+	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+
+	return ret;
+}
+
+void amdgpu_si_start_smc(struct amdgpu_device *adev)
+{
+	u32 tmp = RREG32_SMC(SMC_SYSCON_RESET_CNTL);
+
+	tmp &= ~RST_REG;
+
+	WREG32_SMC(SMC_SYSCON_RESET_CNTL, tmp);
+}
+
+void amdgpu_si_reset_smc(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	RREG32(CB_CGTT_SCLK_CTRL);
+	RREG32(CB_CGTT_SCLK_CTRL);
+	RREG32(CB_CGTT_SCLK_CTRL);
+	RREG32(CB_CGTT_SCLK_CTRL);
+
+	tmp = RREG32_SMC(SMC_SYSCON_RESET_CNTL) |
+	      RST_REG;
+	WREG32_SMC(SMC_SYSCON_RESET_CNTL, tmp);
+}
+
+int amdgpu_si_program_jump_on_start(struct amdgpu_device *adev)
+{
+	static const u8 data[] = { 0x0E, 0x00, 0x40, 0x40 };
+
+	return amdgpu_si_copy_bytes_to_smc(adev, 0x0, data, 4, sizeof(data)+1);
+}
+
+void amdgpu_si_smc_clock(struct amdgpu_device *adev, bool enable)
+{
+	u32 tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0);
+
+	if (enable)
+		tmp &= ~CK_DISABLE;
+	else
+		tmp |= CK_DISABLE;
+
+	WREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0, tmp);
+}
+
+bool amdgpu_si_is_smc_running(struct amdgpu_device *adev)
+{
+	u32 rst = RREG32_SMC(SMC_SYSCON_RESET_CNTL);
+	u32 clk = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0);
+
+	if (!(rst & RST_REG) && !(clk & CK_DISABLE))
+		return true;
+
+	return false;
+}
+
+PPSMC_Result amdgpu_si_send_msg_to_smc(struct amdgpu_device *adev,
+				       PPSMC_Msg msg)
+{
+	u32 tmp;
+	int i;
+
+	if (!amdgpu_si_is_smc_running(adev))
+		return PPSMC_Result_Failed;
+
+	WREG32(SMC_MESSAGE_0, msg);
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = RREG32(SMC_RESP_0);
+		if (tmp != 0)
+			break;
+		udelay(1);
+	}
+
+	return (PPSMC_Result)RREG32(SMC_RESP_0);
+}
+
+PPSMC_Result amdgpu_si_wait_for_smc_inactive(struct amdgpu_device *adev)
+{
+	u32 tmp;
+	int i;
+
+	if (!amdgpu_si_is_smc_running(adev))
+		return PPSMC_Result_OK;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = RREG32_SMC(SMC_SYSCON_CLOCK_CNTL_0);
+		if ((tmp & CKEN) == 0)
+			break;
+		udelay(1);
+	}
+
+	return PPSMC_Result_OK;
+}
+
+int amdgpu_si_load_smc_ucode(struct amdgpu_device *adev, u32 limit)
+{
+	const struct smc_firmware_header_v1_0 *hdr;
+	unsigned long flags;
+	u32 ucode_start_address;
+	u32 ucode_size;
+	const u8 *src;
+	u32 data;
+
+	if (!adev->pm.fw)
+		return -EINVAL;
+
+	hdr = (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data;
+
+	amdgpu_ucode_print_smc_hdr(&hdr->header);
+
+	adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version);
+	ucode_start_address = le32_to_cpu(hdr->ucode_start_addr);
+	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes);
+	src = (const u8 *)
+		(adev->pm.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+	if (ucode_size & 3)
+		return -EINVAL;
+
+	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	WREG32(SMC_IND_INDEX_0, ucode_start_address);
+	WREG32_P(SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, ~AUTO_INCREMENT_IND_0);
+	while (ucode_size >= 4) {
+		/* SMC address space is BE */
+		data = (src[0] << 24) | (src[1] << 16) | (src[2] << 8) | src[3];
+
+		WREG32(SMC_IND_DATA_0, data);
+
+		src += 4;
+		ucode_size -= 4;
+	}
+	WREG32_P(SMC_IND_ACCESS_CNTL, 0, ~AUTO_INCREMENT_IND_0);
+	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+
+	return 0;
+}
+
+int amdgpu_si_read_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
+				  u32 *value, u32 limit)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	ret = si_set_smc_sram_address(adev, smc_address, limit);
+	if (ret == 0)
+		*value = RREG32(SMC_IND_DATA_0);
+	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+
+	return ret;
+}
+
+int amdgpu_si_write_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
+				   u32 value, u32 limit)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&adev->smc_idx_lock, flags);
+	ret = si_set_smc_sram_address(adev, smc_address, limit);
+	if (ret == 0)
+		WREG32(SMC_IND_DATA_0, value);
+	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/sislands_smc.h b/drivers/gpu/drm/amd/amdgpu/sislands_smc.h
new file mode 100644
index 000000000000..d2930eceaf3c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/sislands_smc.h
@@ -0,0 +1,423 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef PP_SISLANDS_SMC_H
+#define PP_SISLANDS_SMC_H
+
+#include "ppsmc.h"
+
+#pragma pack(push, 1)
+
+#define SISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE 16
+
+struct PP_SIslands_Dpm2PerfLevel
+{
+    uint8_t MaxPS;
+    uint8_t TgtAct;
+    uint8_t MaxPS_StepInc;
+    uint8_t MaxPS_StepDec;
+    uint8_t PSSamplingTime;
+    uint8_t NearTDPDec;
+    uint8_t AboveSafeInc;
+    uint8_t BelowSafeInc;
+    uint8_t PSDeltaLimit;
+    uint8_t PSDeltaWin;
+    uint16_t PwrEfficiencyRatio;
+    uint8_t Reserved[4];
+};
+
+typedef struct PP_SIslands_Dpm2PerfLevel PP_SIslands_Dpm2PerfLevel;
+
+struct PP_SIslands_DPM2Status
+{
+    uint32_t    dpm2Flags;
+    uint8_t     CurrPSkip;
+    uint8_t     CurrPSkipPowerShift;
+    uint8_t     CurrPSkipTDP;
+    uint8_t     CurrPSkipOCP;
+    uint8_t     MaxSPLLIndex;
+    uint8_t     MinSPLLIndex;
+    uint8_t     CurrSPLLIndex;
+    uint8_t     InfSweepMode;
+    uint8_t     InfSweepDir;
+    uint8_t     TDPexceeded;
+    uint8_t     reserved;
+    uint8_t     SwitchDownThreshold;
+    uint32_t    SwitchDownCounter;
+    uint32_t    SysScalingFactor;
+};
+
+typedef struct PP_SIslands_DPM2Status PP_SIslands_DPM2Status;
+
+struct PP_SIslands_DPM2Parameters
+{
+    uint32_t    TDPLimit;
+    uint32_t    NearTDPLimit;
+    uint32_t    SafePowerLimit;
+    uint32_t    PowerBoostLimit;
+    uint32_t    MinLimitDelta;
+};
+typedef struct PP_SIslands_DPM2Parameters PP_SIslands_DPM2Parameters;
+
+struct PP_SIslands_PAPMStatus
+{
+    uint32_t    EstimatedDGPU_T;
+    uint32_t    EstimatedDGPU_P;
+    uint32_t    EstimatedAPU_T;
+    uint32_t    EstimatedAPU_P;
+    uint8_t     dGPU_T_Limit_Exceeded;
+    uint8_t     reserved[3];
+};
+typedef struct PP_SIslands_PAPMStatus PP_SIslands_PAPMStatus;
+
+struct PP_SIslands_PAPMParameters
+{
+    uint32_t    NearTDPLimitTherm;
+    uint32_t    NearTDPLimitPAPM;
+    uint32_t    PlatformPowerLimit;
+    uint32_t    dGPU_T_Limit;
+    uint32_t    dGPU_T_Warning;
+    uint32_t    dGPU_T_Hysteresis;
+};
+typedef struct PP_SIslands_PAPMParameters PP_SIslands_PAPMParameters;
+
+struct SISLANDS_SMC_SCLK_VALUE
+{
+    uint32_t    vCG_SPLL_FUNC_CNTL;
+    uint32_t    vCG_SPLL_FUNC_CNTL_2;
+    uint32_t    vCG_SPLL_FUNC_CNTL_3;
+    uint32_t    vCG_SPLL_FUNC_CNTL_4;
+    uint32_t    vCG_SPLL_SPREAD_SPECTRUM;
+    uint32_t    vCG_SPLL_SPREAD_SPECTRUM_2;
+    uint32_t    sclk_value;
+};
+
+typedef struct SISLANDS_SMC_SCLK_VALUE SISLANDS_SMC_SCLK_VALUE;
+
+struct SISLANDS_SMC_MCLK_VALUE
+{
+    uint32_t    vMPLL_FUNC_CNTL;
+    uint32_t    vMPLL_FUNC_CNTL_1;
+    uint32_t    vMPLL_FUNC_CNTL_2;
+    uint32_t    vMPLL_AD_FUNC_CNTL;
+    uint32_t    vMPLL_DQ_FUNC_CNTL;
+    uint32_t    vMCLK_PWRMGT_CNTL;
+    uint32_t    vDLL_CNTL;
+    uint32_t    vMPLL_SS;
+    uint32_t    vMPLL_SS2;
+    uint32_t    mclk_value;
+};
+
+typedef struct SISLANDS_SMC_MCLK_VALUE SISLANDS_SMC_MCLK_VALUE;
+
+struct SISLANDS_SMC_VOLTAGE_VALUE
+{
+    uint16_t    value;
+    uint8_t     index;
+    uint8_t     phase_settings;
+};
+
+typedef struct SISLANDS_SMC_VOLTAGE_VALUE SISLANDS_SMC_VOLTAGE_VALUE;
+
+struct SISLANDS_SMC_HW_PERFORMANCE_LEVEL
+{
+    uint8_t                     ACIndex;
+    uint8_t                     displayWatermark;
+    uint8_t                     gen2PCIE;
+    uint8_t                     UVDWatermark;
+    uint8_t                     VCEWatermark;
+    uint8_t                     strobeMode;
+    uint8_t                     mcFlags;
+    uint8_t                     padding;
+    uint32_t                    aT;
+    uint32_t                    bSP;
+    SISLANDS_SMC_SCLK_VALUE     sclk;
+    SISLANDS_SMC_MCLK_VALUE     mclk;
+    SISLANDS_SMC_VOLTAGE_VALUE  vddc;
+    SISLANDS_SMC_VOLTAGE_VALUE  mvdd;
+    SISLANDS_SMC_VOLTAGE_VALUE  vddci;
+    SISLANDS_SMC_VOLTAGE_VALUE  std_vddc;
+    uint8_t                     hysteresisUp;
+    uint8_t                     hysteresisDown;
+    uint8_t                     stateFlags;
+    uint8_t                     arbRefreshState;
+    uint32_t                    SQPowerThrottle;
+    uint32_t                    SQPowerThrottle_2;
+    uint32_t                    MaxPoweredUpCU;
+    SISLANDS_SMC_VOLTAGE_VALUE  high_temp_vddc;
+    SISLANDS_SMC_VOLTAGE_VALUE  low_temp_vddc;
+    uint32_t                    reserved[2];
+    PP_SIslands_Dpm2PerfLevel   dpm2;
+};
+
+#define SISLANDS_SMC_STROBE_RATIO    0x0F
+#define SISLANDS_SMC_STROBE_ENABLE   0x10
+
+#define SISLANDS_SMC_MC_EDC_RD_FLAG  0x01
+#define SISLANDS_SMC_MC_EDC_WR_FLAG  0x02
+#define SISLANDS_SMC_MC_RTT_ENABLE   0x04
+#define SISLANDS_SMC_MC_STUTTER_EN   0x08
+#define SISLANDS_SMC_MC_PG_EN        0x10
+
+typedef struct SISLANDS_SMC_HW_PERFORMANCE_LEVEL SISLANDS_SMC_HW_PERFORMANCE_LEVEL;
+
+struct SISLANDS_SMC_SWSTATE
+{
+    uint8_t                             flags;
+    uint8_t                             levelCount;
+    uint8_t                             padding2;
+    uint8_t                             padding3;
+    SISLANDS_SMC_HW_PERFORMANCE_LEVEL   levels[1];
+};
+
+typedef struct SISLANDS_SMC_SWSTATE SISLANDS_SMC_SWSTATE;
+
+#define SISLANDS_SMC_VOLTAGEMASK_VDDC  0
+#define SISLANDS_SMC_VOLTAGEMASK_MVDD  1
+#define SISLANDS_SMC_VOLTAGEMASK_VDDCI 2
+#define SISLANDS_SMC_VOLTAGEMASK_VDDC_PHASE_SHEDDING 3
+#define SISLANDS_SMC_VOLTAGEMASK_MAX   4
+
+struct SISLANDS_SMC_VOLTAGEMASKTABLE
+{
+    uint32_t lowMask[SISLANDS_SMC_VOLTAGEMASK_MAX];
+};
+
+typedef struct SISLANDS_SMC_VOLTAGEMASKTABLE SISLANDS_SMC_VOLTAGEMASKTABLE;
+
+#define SISLANDS_MAX_NO_VREG_STEPS 32
+
+struct SISLANDS_SMC_STATETABLE
+{
+    uint8_t                             thermalProtectType;
+    uint8_t                             systemFlags;
+    uint8_t                             maxVDDCIndexInPPTable;
+    uint8_t                             extraFlags;
+    uint32_t                            lowSMIO[SISLANDS_MAX_NO_VREG_STEPS];
+    SISLANDS_SMC_VOLTAGEMASKTABLE       voltageMaskTable;
+    SISLANDS_SMC_VOLTAGEMASKTABLE       phaseMaskTable;
+    PP_SIslands_DPM2Parameters          dpm2Params;
+    SISLANDS_SMC_SWSTATE                initialState;
+    SISLANDS_SMC_SWSTATE                ACPIState;
+    SISLANDS_SMC_SWSTATE                ULVState;
+    SISLANDS_SMC_SWSTATE                driverState;
+    SISLANDS_SMC_HW_PERFORMANCE_LEVEL   dpmLevels[SISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1];
+};
+
+typedef struct SISLANDS_SMC_STATETABLE SISLANDS_SMC_STATETABLE;
+
+#define SI_SMC_SOFT_REGISTER_mclk_chg_timeout         0x0
+#define SI_SMC_SOFT_REGISTER_delay_vreg               0xC
+#define SI_SMC_SOFT_REGISTER_delay_acpi               0x28
+#define SI_SMC_SOFT_REGISTER_seq_index                0x5C
+#define SI_SMC_SOFT_REGISTER_mvdd_chg_time            0x60
+#define SI_SMC_SOFT_REGISTER_mclk_switch_lim          0x70
+#define SI_SMC_SOFT_REGISTER_watermark_threshold      0x78
+#define SI_SMC_SOFT_REGISTER_phase_shedding_delay     0x88
+#define SI_SMC_SOFT_REGISTER_ulv_volt_change_delay    0x8C
+#define SI_SMC_SOFT_REGISTER_mc_block_delay           0x98
+#define SI_SMC_SOFT_REGISTER_ticks_per_us             0xA8
+#define SI_SMC_SOFT_REGISTER_crtc_index               0xC4
+#define SI_SMC_SOFT_REGISTER_mclk_change_block_cp_min 0xC8
+#define SI_SMC_SOFT_REGISTER_mclk_change_block_cp_max 0xCC
+#define SI_SMC_SOFT_REGISTER_non_ulv_pcie_link_width  0xF4
+#define SI_SMC_SOFT_REGISTER_tdr_is_about_to_happen   0xFC
+#define SI_SMC_SOFT_REGISTER_vr_hot_gpio              0x100
+#define SI_SMC_SOFT_REGISTER_svi_rework_plat_type     0x118
+#define SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svd   0x11c
+#define SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svc   0x120
+
+struct PP_SIslands_FanTable
+{
+	uint8_t  fdo_mode;
+	uint8_t  padding;
+	int16_t  temp_min;
+	int16_t  temp_med;
+	int16_t  temp_max;
+	int16_t  slope1;
+	int16_t  slope2;
+	int16_t  fdo_min;
+	int16_t  hys_up;
+	int16_t  hys_down;
+	int16_t  hys_slope;
+	int16_t  temp_resp_lim;
+	int16_t  temp_curr;
+	int16_t  slope_curr;
+	int16_t  pwm_curr;
+	uint32_t refresh_period;
+	int16_t  fdo_max;
+	uint8_t  temp_src;
+	int8_t  padding2;
+};
+
+typedef struct PP_SIslands_FanTable PP_SIslands_FanTable;
+
+#define SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16
+#define SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES 32
+
+#define SMC_SISLANDS_SCALE_I  7
+#define SMC_SISLANDS_SCALE_R 12
+
+struct PP_SIslands_CacConfig
+{
+    uint16_t   cac_lkge_lut[SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES][SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES];
+    uint32_t   lkge_lut_V0;
+    uint32_t   lkge_lut_Vstep;
+    uint32_t   WinTime;
+    uint32_t   R_LL;
+    uint32_t   calculation_repeats;
+    uint32_t   l2numWin_TDP;
+    uint32_t   dc_cac;
+    uint8_t    lts_truncate_n;
+    uint8_t    SHIFT_N;
+    uint8_t    log2_PG_LKG_SCALE;
+    uint8_t    cac_temp;
+    uint32_t   lkge_lut_T0;
+    uint32_t   lkge_lut_Tstep;
+};
+
+typedef struct PP_SIslands_CacConfig PP_SIslands_CacConfig;
+
+#define SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE 16
+#define SMC_SISLANDS_MC_REGISTER_ARRAY_SET_COUNT 20
+
+struct SMC_SIslands_MCRegisterAddress
+{
+    uint16_t s0;
+    uint16_t s1;
+};
+
+typedef struct SMC_SIslands_MCRegisterAddress SMC_SIslands_MCRegisterAddress;
+
+struct SMC_SIslands_MCRegisterSet
+{
+    uint32_t value[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE];
+};
+
+typedef struct SMC_SIslands_MCRegisterSet SMC_SIslands_MCRegisterSet;
+
+struct SMC_SIslands_MCRegisters
+{
+    uint8_t                             last;
+    uint8_t                             reserved[3];
+    SMC_SIslands_MCRegisterAddress      address[SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE];
+    SMC_SIslands_MCRegisterSet          data[SMC_SISLANDS_MC_REGISTER_ARRAY_SET_COUNT];
+};
+
+typedef struct SMC_SIslands_MCRegisters SMC_SIslands_MCRegisters;
+
+struct SMC_SIslands_MCArbDramTimingRegisterSet
+{
+    uint32_t mc_arb_dram_timing;
+    uint32_t mc_arb_dram_timing2;
+    uint8_t  mc_arb_rfsh_rate;
+    uint8_t  mc_arb_burst_time;
+    uint8_t  padding[2];
+};
+
+typedef struct SMC_SIslands_MCArbDramTimingRegisterSet SMC_SIslands_MCArbDramTimingRegisterSet;
+
+struct SMC_SIslands_MCArbDramTimingRegisters
+{
+    uint8_t                                     arb_current;
+    uint8_t                                     reserved[3];
+    SMC_SIslands_MCArbDramTimingRegisterSet     data[16];
+};
+
+typedef struct SMC_SIslands_MCArbDramTimingRegisters SMC_SIslands_MCArbDramTimingRegisters;
+
+struct SMC_SISLANDS_SPLL_DIV_TABLE
+{
+    uint32_t    freq[256];
+    uint32_t    ss[256];
+};
+
+#define SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_MASK  0x01ffffff
+#define SMC_SISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT 0
+#define SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_MASK   0xfe000000
+#define SMC_SISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT  25
+#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_MASK   0x000fffff
+#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT  0
+#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_MASK   0xfff00000
+#define SMC_SISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT  20
+
+typedef struct SMC_SISLANDS_SPLL_DIV_TABLE SMC_SISLANDS_SPLL_DIV_TABLE;
+
+#define SMC_SISLANDS_DTE_MAX_FILTER_STAGES 5
+
+#define SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE 16
+
+struct Smc_SIslands_DTE_Configuration
+{
+    uint32_t tau[SMC_SISLANDS_DTE_MAX_FILTER_STAGES];
+    uint32_t R[SMC_SISLANDS_DTE_MAX_FILTER_STAGES];
+    uint32_t K;
+    uint32_t T0;
+    uint32_t MaxT;
+    uint8_t  WindowSize;
+    uint8_t  Tdep_count;
+    uint8_t  temp_select;
+    uint8_t  DTE_mode;
+    uint8_t  T_limits[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+    uint32_t Tdep_tau[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+    uint32_t Tdep_R[SMC_SISLANDS_DTE_MAX_TEMPERATURE_DEPENDENT_ARRAY_SIZE];
+    uint32_t Tthreshold;
+};
+
+typedef struct Smc_SIslands_DTE_Configuration Smc_SIslands_DTE_Configuration;
+
+#define SMC_SISLANDS_DTE_STATUS_FLAG_DTE_ON 1
+
+#define SISLANDS_SMC_FIRMWARE_HEADER_LOCATION 0x10000
+
+#define SISLANDS_SMC_FIRMWARE_HEADER_version                   0x0
+#define SISLANDS_SMC_FIRMWARE_HEADER_flags                     0x4
+#define SISLANDS_SMC_FIRMWARE_HEADER_softRegisters             0xC
+#define SISLANDS_SMC_FIRMWARE_HEADER_stateTable                0x10
+#define SISLANDS_SMC_FIRMWARE_HEADER_fanTable                  0x14
+#define SISLANDS_SMC_FIRMWARE_HEADER_CacConfigTable            0x18
+#define SISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable           0x24
+#define SISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable 0x30
+#define SISLANDS_SMC_FIRMWARE_HEADER_spllTable                 0x38
+#define SISLANDS_SMC_FIRMWARE_HEADER_DteConfiguration          0x40
+#define SISLANDS_SMC_FIRMWARE_HEADER_PAPMParameters            0x48
+
+#pragma pack(pop)
+
+int amdgpu_si_copy_bytes_to_smc(struct amdgpu_device *adev,
+				u32 smc_start_address,
+				const u8 *src, u32 byte_count, u32 limit);
+void amdgpu_si_start_smc(struct amdgpu_device *adev);
+void amdgpu_si_reset_smc(struct amdgpu_device *adev);
+int amdgpu_si_program_jump_on_start(struct amdgpu_device *adev);
+void amdgpu_si_smc_clock(struct amdgpu_device *adev, bool enable);
+bool amdgpu_si_is_smc_running(struct amdgpu_device *adev);
+PPSMC_Result amdgpu_si_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg);
+PPSMC_Result amdgpu_si_wait_for_smc_inactive(struct amdgpu_device *adev);
+int amdgpu_si_load_smc_ucode(struct amdgpu_device *adev, u32 limit);
+int amdgpu_si_read_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
+				  u32 *value, u32 limit);
+int amdgpu_si_write_smc_sram_dword(struct amdgpu_device *adev, u32 smc_address,
+				   u32 value, u32 limit);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_dpm.c b/drivers/gpu/drm/amd/amdgpu/tonga_dpm.c
deleted file mode 100644
index f06f6f4dc3a8..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/tonga_dpm.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <linux/firmware.h>
-#include "drmP.h"
-#include "amdgpu.h"
-#include "tonga_smum.h"
-
-MODULE_FIRMWARE("amdgpu/tonga_smc.bin");
-
-static void tonga_dpm_set_funcs(struct amdgpu_device *adev);
-
-static int tonga_dpm_early_init(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	tonga_dpm_set_funcs(adev);
-
-	return 0;
-}
-
-static int tonga_dpm_init_microcode(struct amdgpu_device *adev)
-{
-	char fw_name[30] = "amdgpu/tonga_smc.bin";
-	int err;
-	err = request_firmware(&adev->pm.fw, fw_name, adev->dev);
-	if (err)
-		goto out;
-	err = amdgpu_ucode_validate(adev->pm.fw);
-
-out:
-	if (err) {
-		DRM_ERROR("Failed to load firmware \"%s\"", fw_name);
-		release_firmware(adev->pm.fw);
-		adev->pm.fw = NULL;
-	}
-	return err;
-}
-
-static int tonga_dpm_sw_init(void *handle)
-{
-	int ret;
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	ret = tonga_dpm_init_microcode(adev);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int tonga_dpm_sw_fini(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	release_firmware(adev->pm.fw);
-	adev->pm.fw = NULL;
-
-	return 0;
-}
-
-static int tonga_dpm_hw_init(void *handle)
-{
-	int ret;
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	mutex_lock(&adev->pm.mutex);
-
-	/* smu init only needs to be called at startup, not resume.
-	 * It should be in sw_init, but requires the fw info gathered
-	 * in sw_init from other IP modules.
-	 */
-	ret = tonga_smu_init(adev);
-	if (ret) {
-		DRM_ERROR("SMU initialization failed\n");
-		goto fail;
-	}
-
-	ret = tonga_smu_start(adev);
-	if (ret) {
-		DRM_ERROR("SMU start failed\n");
-		goto fail;
-	}
-
-	mutex_unlock(&adev->pm.mutex);
-	return 0;
-
-fail:
-	adev->firmware.smu_load = false;
-	mutex_unlock(&adev->pm.mutex);
-	return -EINVAL;
-}
-
-static int tonga_dpm_hw_fini(void *handle)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	mutex_lock(&adev->pm.mutex);
-	/* smu fini only needs to be called at teardown, not suspend.
-	 * It should be in sw_fini, but we put it here for symmetry
-	 * with smu init.
-	 */
-	tonga_smu_fini(adev);
-	mutex_unlock(&adev->pm.mutex);
-	return 0;
-}
-
-static int tonga_dpm_suspend(void *handle)
-{
-	return tonga_dpm_hw_fini(handle);
-}
-
-static int tonga_dpm_resume(void *handle)
-{
-	return tonga_dpm_hw_init(handle);
-}
-
-static int tonga_dpm_set_clockgating_state(void *handle,
-			enum amd_clockgating_state state)
-{
-	return 0;
-}
-
-static int tonga_dpm_set_powergating_state(void *handle,
-			enum amd_powergating_state state)
-{
-	return 0;
-}
-
-const struct amd_ip_funcs tonga_dpm_ip_funcs = {
-	.name = "tonga_dpm",
-	.early_init = tonga_dpm_early_init,
-	.late_init = NULL,
-	.sw_init = tonga_dpm_sw_init,
-	.sw_fini = tonga_dpm_sw_fini,
-	.hw_init = tonga_dpm_hw_init,
-	.hw_fini = tonga_dpm_hw_fini,
-	.suspend = tonga_dpm_suspend,
-	.resume = tonga_dpm_resume,
-	.is_idle = NULL,
-	.wait_for_idle = NULL,
-	.soft_reset = NULL,
-	.set_clockgating_state = tonga_dpm_set_clockgating_state,
-	.set_powergating_state = tonga_dpm_set_powergating_state,
-};
-
-static const struct amdgpu_dpm_funcs tonga_dpm_funcs = {
-	.get_temperature = NULL,
-	.pre_set_power_state = NULL,
-	.set_power_state = NULL,
-	.post_set_power_state = NULL,
-	.display_configuration_changed = NULL,
-	.get_sclk = NULL,
-	.get_mclk = NULL,
-	.print_power_state = NULL,
-	.debugfs_print_current_performance_level = NULL,
-	.force_performance_level = NULL,
-	.vblank_too_short = NULL,
-	.powergate_uvd = NULL,
-};
-
-static void tonga_dpm_set_funcs(struct amdgpu_device *adev)
-{
-	if (NULL == adev->pm.funcs)
-		adev->pm.funcs = &tonga_dpm_funcs;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index c92055805a45..d127d59f953a 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -373,10 +373,10 @@ static int tonga_ih_wait_for_idle(void *handle)
 	return -ETIMEDOUT;
 }
 
-static int tonga_ih_soft_reset(void *handle)
+static int tonga_ih_check_soft_reset(void *handle)
 {
-	u32 srbm_soft_reset = 0;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset = 0;
 	u32 tmp = RREG32(mmSRBM_STATUS);
 
 	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
@@ -384,6 +384,48 @@ static int tonga_ih_soft_reset(void *handle)
 						SOFT_RESET_IH, 1);
 
 	if (srbm_soft_reset) {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang = true;
+		adev->irq.srbm_soft_reset = srbm_soft_reset;
+	} else {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang = false;
+		adev->irq.srbm_soft_reset = 0;
+	}
+
+	return 0;
+}
+
+static int tonga_ih_pre_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang)
+		return 0;
+
+	return tonga_ih_hw_fini(adev);
+}
+
+static int tonga_ih_post_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang)
+		return 0;
+
+	return tonga_ih_hw_init(adev);
+}
+
+static int tonga_ih_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_IH].hang)
+		return 0;
+	srbm_soft_reset = adev->irq.srbm_soft_reset;
+
+	if (srbm_soft_reset) {
+		u32 tmp;
+
 		tmp = RREG32(mmSRBM_SOFT_RESET);
 		tmp |= srbm_soft_reset;
 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
@@ -427,7 +469,10 @@ const struct amd_ip_funcs tonga_ih_ip_funcs = {
 	.resume = tonga_ih_resume,
 	.is_idle = tonga_ih_is_idle,
 	.wait_for_idle = tonga_ih_wait_for_idle,
+	.check_soft_reset = tonga_ih_check_soft_reset,
+	.pre_soft_reset = tonga_ih_pre_soft_reset,
 	.soft_reset = tonga_ih_soft_reset,
+	.post_soft_reset = tonga_ih_post_soft_reset,
 	.set_clockgating_state = tonga_ih_set_clockgating_state,
 	.set_powergating_state = tonga_ih_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
deleted file mode 100644
index 940de1836f8f..000000000000
--- a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
+++ /dev/null
@@ -1,862 +0,0 @@
-/*
- * Copyright 2014 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <linux/firmware.h>
-#include "drmP.h"
-#include "amdgpu.h"
-#include "tonga_ppsmc.h"
-#include "tonga_smum.h"
-#include "smu_ucode_xfer_vi.h"
-#include "amdgpu_ucode.h"
-
-#include "smu/smu_7_1_2_d.h"
-#include "smu/smu_7_1_2_sh_mask.h"
-
-#define TONGA_SMC_SIZE 0x20000
-
-static int tonga_set_smc_sram_address(struct amdgpu_device *adev, uint32_t smc_address, uint32_t limit)
-{
-	uint32_t val;
-
-	if (smc_address & 3)
-		return -EINVAL;
-
-	if ((smc_address + 3) > limit)
-		return -EINVAL;
-
-	WREG32(mmSMC_IND_INDEX_0, smc_address);
-
-	val = RREG32(mmSMC_IND_ACCESS_CNTL);
-	val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
-	WREG32(mmSMC_IND_ACCESS_CNTL, val);
-
-	return 0;
-}
-
-static int tonga_copy_bytes_to_smc(struct amdgpu_device *adev, uint32_t smc_start_address, const uint8_t *src, uint32_t byte_count, uint32_t limit)
-{
-	uint32_t addr;
-	uint32_t data, orig_data;
-	int result = 0;
-	uint32_t extra_shift;
-	unsigned long flags;
-
-	if (smc_start_address & 3)
-		return -EINVAL;
-
-	if ((smc_start_address + byte_count) > limit)
-		return -EINVAL;
-
-	addr = smc_start_address;
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	while (byte_count >= 4) {
-		/* Bytes are written into the SMC addres space with the MSB first */
-		data = (src[0] << 24) + (src[1] << 16) + (src[2] << 8) + src[3];
-
-		result = tonga_set_smc_sram_address(adev, addr, limit);
-
-		if (result)
-			goto out;
-
-		WREG32(mmSMC_IND_DATA_0, data);
-
-		src += 4;
-		byte_count -= 4;
-		addr += 4;
-	}
-
-	if (0 != byte_count) {
-		/* Now write odd bytes left, do a read modify write cycle */
-		data = 0;
-
-		result = tonga_set_smc_sram_address(adev, addr, limit);
-		if (result)
-			goto out;
-
-		orig_data = RREG32(mmSMC_IND_DATA_0);
-		extra_shift = 8 * (4 - byte_count);
-
-		while (byte_count > 0) {
-			data = (data << 8) + *src++;
-			byte_count--;
-		}
-
-		data <<= extra_shift;
-		data |= (orig_data & ~((~0UL) << extra_shift));
-
-		result = tonga_set_smc_sram_address(adev, addr, limit);
-		if (result)
-			goto out;
-
-		WREG32(mmSMC_IND_DATA_0, data);
-	}
-
-out:
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-	return result;
-}
-
-static int tonga_program_jump_on_start(struct amdgpu_device *adev)
-{
-	static unsigned char data[] = {0xE0, 0x00, 0x80, 0x40};
-	tonga_copy_bytes_to_smc(adev, 0x0, data, 4, sizeof(data)+1);
-
-	return 0;
-}
-
-static bool tonga_is_smc_ram_running(struct amdgpu_device *adev)
-{
-	uint32_t val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-	val = REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable);
-
-	return ((0 == val) && (0x20100 <= RREG32_SMC(ixSMC_PC_C)));
-}
-
-static int wait_smu_response(struct amdgpu_device *adev)
-{
-	int i;
-	uint32_t val;
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32(mmSMC_RESP_0);
-		if (REG_GET_FIELD(val, SMC_RESP_0, SMC_RESP))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout)
-		return -EINVAL;
-
-	return 0;
-}
-
-static int tonga_send_msg_to_smc_offset(struct amdgpu_device *adev)
-{
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MSG_ARG_0, 0x20000);
-	WREG32(mmSMC_MESSAGE_0, PPSMC_MSG_Test);
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send message\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int tonga_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg)
-{
-	if (!tonga_is_smc_ram_running(adev))
-	{
-		return -EINVAL;
-	}
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MESSAGE_0, msg);
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send message\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int tonga_send_msg_to_smc_without_waiting(struct amdgpu_device *adev,
-						PPSMC_Msg msg)
-{
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MESSAGE_0, msg);
-
-	return 0;
-}
-
-static int tonga_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
-						PPSMC_Msg msg,
-						uint32_t parameter)
-{
-	if (!tonga_is_smc_ram_running(adev))
-		return -EINVAL;
-
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MSG_ARG_0, parameter);
-
-	return tonga_send_msg_to_smc(adev, msg);
-}
-
-static int tonga_send_msg_to_smc_with_parameter_without_waiting(
-					struct amdgpu_device *adev,
-					PPSMC_Msg msg, uint32_t parameter)
-{
-	if (wait_smu_response(adev)) {
-		DRM_ERROR("Failed to send previous message\n");
-		return -EINVAL;
-	}
-
-	WREG32(mmSMC_MSG_ARG_0, parameter);
-
-	return tonga_send_msg_to_smc_without_waiting(adev, msg);
-}
-
-#if 0 /* not used yet */
-static int tonga_wait_for_smc_inactive(struct amdgpu_device *adev)
-{
-	int i;
-	uint32_t val;
-
-	if (!tonga_is_smc_ram_running(adev))
-		return -EINVAL;
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-		if (REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, cken) == 0)
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout)
-		return -EINVAL;
-
-	return 0;
-}
-#endif
-
-static int tonga_smu_upload_firmware_image(struct amdgpu_device *adev)
-{
-	const struct smc_firmware_header_v1_0 *hdr;
-	uint32_t ucode_size;
-	uint32_t ucode_start_address;
-	const uint8_t *src;
-	uint32_t val;
-	uint32_t byte_count;
-	uint32_t *data;
-	unsigned long flags;
-
-	if (!adev->pm.fw)
-		return -EINVAL;
-
-	/* Skip SMC ucode loading on SR-IOV capable boards.
-	 * vbios does this for us in asic_init in that case.
-	 */
-	if (adev->virtualization.supports_sr_iov)
-		return 0;
-
-	hdr = (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data;
-	amdgpu_ucode_print_smc_hdr(&hdr->header);
-
-	adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version);
-	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes);
-	ucode_start_address = le32_to_cpu(hdr->ucode_start_addr);
-	src = (const uint8_t *)
-		(adev->pm.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
-
-	if (ucode_size & 3) {
-		DRM_ERROR("SMC ucode is not 4 bytes aligned\n");
-		return -EINVAL;
-	}
-
-	if (ucode_size > TONGA_SMC_SIZE) {
-		DRM_ERROR("SMC address is beyond the SMC RAM area\n");
-		return -EINVAL;
-	}
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	WREG32(mmSMC_IND_INDEX_0, ucode_start_address);
-
-	val = RREG32(mmSMC_IND_ACCESS_CNTL);
-	val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1);
-	WREG32(mmSMC_IND_ACCESS_CNTL, val);
-
-	byte_count = ucode_size;
-	data = (uint32_t *)src;
-	for (; byte_count >= 4; data++, byte_count -= 4)
-		WREG32(mmSMC_IND_DATA_0, data[0]);
-
-	val = RREG32(mmSMC_IND_ACCESS_CNTL);
-	val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
-	WREG32(mmSMC_IND_ACCESS_CNTL, val);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-
-	return 0;
-}
-
-#if 0 /* not used yet */
-static int tonga_read_smc_sram_dword(struct amdgpu_device *adev,
-				uint32_t smc_address,
-				uint32_t *value,
-				uint32_t limit)
-{
-	int result;
-	unsigned long flags;
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	result = tonga_set_smc_sram_address(adev, smc_address, limit);
-	if (result == 0)
-		*value = RREG32(mmSMC_IND_DATA_0);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-	return result;
-}
-
-static int tonga_write_smc_sram_dword(struct amdgpu_device *adev,
-				uint32_t smc_address,
-				uint32_t value,
-				uint32_t limit)
-{
-	int result;
-	unsigned long flags;
-
-	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	result = tonga_set_smc_sram_address(adev, smc_address, limit);
-	if (result == 0)
-		WREG32(mmSMC_IND_DATA_0, value);
-	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
-	return result;
-}
-
-static int tonga_smu_stop_smc(struct amdgpu_device *adev)
-{
-	uint32_t val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-	val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 1);
-	WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
-
-	return 0;
-}
-#endif
-
-static enum AMDGPU_UCODE_ID tonga_convert_fw_type(uint32_t fw_type)
-{
-	switch (fw_type) {
-		case UCODE_ID_SDMA0:
-			return AMDGPU_UCODE_ID_SDMA0;
-		case UCODE_ID_SDMA1:
-			return AMDGPU_UCODE_ID_SDMA1;
-		case UCODE_ID_CP_CE:
-			return AMDGPU_UCODE_ID_CP_CE;
-		case UCODE_ID_CP_PFP:
-			return AMDGPU_UCODE_ID_CP_PFP;
-		case UCODE_ID_CP_ME:
-			return AMDGPU_UCODE_ID_CP_ME;
-		case UCODE_ID_CP_MEC:
-		case UCODE_ID_CP_MEC_JT1:
-			return AMDGPU_UCODE_ID_CP_MEC1;
-		case UCODE_ID_CP_MEC_JT2:
-			return AMDGPU_UCODE_ID_CP_MEC2;
-		case UCODE_ID_RLC_G:
-			return AMDGPU_UCODE_ID_RLC_G;
-		default:
-			DRM_ERROR("ucode type is out of range!\n");
-			return AMDGPU_UCODE_ID_MAXIMUM;
-	}
-}
-
-static int tonga_smu_populate_single_firmware_entry(struct amdgpu_device *adev,
-						uint32_t fw_type,
-						struct SMU_Entry *entry)
-{
-	enum AMDGPU_UCODE_ID id = tonga_convert_fw_type(fw_type);
-	struct amdgpu_firmware_info *ucode = &adev->firmware.ucode[id];
-	const struct gfx_firmware_header_v1_0 *header = NULL;
-	uint64_t gpu_addr;
-	uint32_t data_size;
-
-	if (ucode->fw == NULL)
-		return -EINVAL;
-
-	gpu_addr  = ucode->mc_addr;
-	header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
-	data_size = le32_to_cpu(header->header.ucode_size_bytes);
-
-	if ((fw_type == UCODE_ID_CP_MEC_JT1) ||
-		(fw_type == UCODE_ID_CP_MEC_JT2)) {
-		gpu_addr += le32_to_cpu(header->jt_offset) << 2;
-		data_size = le32_to_cpu(header->jt_size) << 2;
-	}
-
-	entry->version = (uint16_t)le32_to_cpu(header->header.ucode_version);
-	entry->id = (uint16_t)fw_type;
-	entry->image_addr_high = upper_32_bits(gpu_addr);
-	entry->image_addr_low = lower_32_bits(gpu_addr);
-	entry->meta_data_addr_high = 0;
-	entry->meta_data_addr_low = 0;
-	entry->data_size_byte = data_size;
-	entry->num_register_entries = 0;
-
-	if (fw_type == UCODE_ID_RLC_G)
-		entry->flags = 1;
-	else
-		entry->flags = 0;
-
-	return 0;
-}
-
-static int tonga_smu_request_load_fw(struct amdgpu_device *adev)
-{
-	struct tonga_smu_private_data *private = (struct tonga_smu_private_data *)adev->smu.priv;
-	struct SMU_DRAMData_TOC *toc;
-	uint32_t fw_to_load;
-
-	WREG32_SMC(ixSOFT_REGISTERS_TABLE_28, 0);
-
-	tonga_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SMU_DRAM_ADDR_HI, private->smu_buffer_addr_high);
-	tonga_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SMU_DRAM_ADDR_LO, private->smu_buffer_addr_low);
-
-	toc = (struct SMU_DRAMData_TOC *)private->header;
-	toc->num_entries = 0;
-	toc->structure_version = 1;
-
-	if (!adev->firmware.smu_load)
-		return 0;
-
-	if (tonga_smu_populate_single_firmware_entry(adev, UCODE_ID_RLC_G,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for RLC\n");
-		return -EINVAL;
-	}
-
-	if (tonga_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_CE,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for CE\n");
-		return -EINVAL;
-	}
-
-	if (tonga_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_PFP,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for PFP\n");
-		return -EINVAL;
-	}
-
-	if (tonga_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_ME,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for ME\n");
-		return -EINVAL;
-	}
-
-	if (tonga_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for MEC\n");
-		return -EINVAL;
-	}
-
-	if (tonga_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC_JT1,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for MEC_JT1\n");
-		return -EINVAL;
-	}
-
-	if (tonga_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC_JT2,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for MEC_JT2\n");
-		return -EINVAL;
-	}
-
-	if (tonga_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA0,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for SDMA0\n");
-		return -EINVAL;
-	}
-
-	if (tonga_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA1,
-			&toc->entry[toc->num_entries++])) {
-		DRM_ERROR("Failed to get firmware entry for SDMA1\n");
-		return -EINVAL;
-	}
-
-	tonga_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_HI, private->header_addr_high);
-	tonga_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_LO, private->header_addr_low);
-
-	fw_to_load = UCODE_ID_RLC_G_MASK |
-			UCODE_ID_SDMA0_MASK |
-			UCODE_ID_SDMA1_MASK |
-			UCODE_ID_CP_CE_MASK |
-			UCODE_ID_CP_ME_MASK |
-			UCODE_ID_CP_PFP_MASK |
-			UCODE_ID_CP_MEC_MASK;
-
-	if (tonga_send_msg_to_smc_with_parameter_without_waiting(adev, PPSMC_MSG_LoadUcodes, fw_to_load)) {
-		DRM_ERROR("Fail to request SMU load ucode\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static uint32_t tonga_smu_get_mask_for_fw_type(uint32_t fw_type)
-{
-	switch (fw_type) {
-		case AMDGPU_UCODE_ID_SDMA0:
-			return UCODE_ID_SDMA0_MASK;
-		case AMDGPU_UCODE_ID_SDMA1:
-			return UCODE_ID_SDMA1_MASK;
-		case AMDGPU_UCODE_ID_CP_CE:
-			return UCODE_ID_CP_CE_MASK;
-		case AMDGPU_UCODE_ID_CP_PFP:
-			return UCODE_ID_CP_PFP_MASK;
-		case AMDGPU_UCODE_ID_CP_ME:
-			return UCODE_ID_CP_ME_MASK;
-		case AMDGPU_UCODE_ID_CP_MEC1:
-			return UCODE_ID_CP_MEC_MASK;
-		case AMDGPU_UCODE_ID_CP_MEC2:
-			return UCODE_ID_CP_MEC_MASK;
-		case AMDGPU_UCODE_ID_RLC_G:
-			return UCODE_ID_RLC_G_MASK;
-		default:
-			DRM_ERROR("ucode type is out of range!\n");
-			return 0;
-	}
-}
-
-static int tonga_smu_check_fw_load_finish(struct amdgpu_device *adev,
-					uint32_t fw_type)
-{
-	uint32_t fw_mask = tonga_smu_get_mask_for_fw_type(fw_type);
-	int i;
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		if (fw_mask == (RREG32_SMC(ixSOFT_REGISTERS_TABLE_28) & fw_mask))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("check firmware loading failed\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int tonga_smu_start_in_protection_mode(struct amdgpu_device *adev)
-{
-	int result;
-	uint32_t val;
-	int i;
-
-	/* Assert reset */
-	val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	result = tonga_smu_upload_firmware_image(adev);
-	if (result)
-		return result;
-
-	/* Clear status */
-	WREG32_SMC(ixSMU_STATUS, 0);
-
-	/* Enable clock */
-	val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-	val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
-	WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
-
-	/* De-assert reset */
-	val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 0);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	/* Set SMU Auto Start */
-	val = RREG32_SMC(ixSMU_INPUT_DATA);
-	val = REG_SET_FIELD(val, SMU_INPUT_DATA, AUTO_START, 1);
-	WREG32_SMC(ixSMU_INPUT_DATA, val);
-
-	/* Clear firmware interrupt enable flag */
-	WREG32_SMC(ixFIRMWARE_FLAGS, 0);
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixRCU_UC_EVENTS);
-		if (REG_GET_FIELD(val, RCU_UC_EVENTS, INTERRUPTS_ENABLED))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("Interrupt is not enabled by firmware\n");
-		return -EINVAL;
-	}
-
-	/* Call Test SMU message with 0x20000 offset
-	 * to trigger SMU start
-	 */
-	tonga_send_msg_to_smc_offset(adev);
-
-	/* Wait for done bit to be set */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixSMU_STATUS);
-		if (REG_GET_FIELD(val, SMU_STATUS, SMU_DONE))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("Timeout for SMU start\n");
-		return -EINVAL;
-	}
-
-	/* Check pass/failed indicator */
-	val = RREG32_SMC(ixSMU_STATUS);
-	if (!REG_GET_FIELD(val, SMU_STATUS, SMU_PASS)) {
-		DRM_ERROR("SMU Firmware start failed\n");
-		return -EINVAL;
-	}
-
-	/* Wait for firmware to initialize */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixFIRMWARE_FLAGS);
-		if(REG_GET_FIELD(val, FIRMWARE_FLAGS, INTERRUPTS_ENABLED))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("SMU firmware initialization failed\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int tonga_smu_start_in_non_protection_mode(struct amdgpu_device *adev)
-{
-	int i, result;
-	uint32_t val;
-
-	/* wait for smc boot up */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixRCU_UC_EVENTS);
-		val = REG_GET_FIELD(val, RCU_UC_EVENTS, boot_seq_done);
-		if (val)
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("SMC boot sequence is not completed\n");
-		return -EINVAL;
-	}
-
-	/* Clear firmware interrupt enable flag */
-	WREG32_SMC(ixFIRMWARE_FLAGS, 0);
-
-	/* Assert reset */
-	val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	result = tonga_smu_upload_firmware_image(adev);
-	if (result)
-		return result;
-
-	/* Set smc instruct start point at 0x0 */
-	tonga_program_jump_on_start(adev);
-
-	/* Enable clock */
-	val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
-	val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
-	WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
-
-	/* De-assert reset */
-	val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
-	val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 0);
-	WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
-
-	/* Wait for firmware to initialize */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		val = RREG32_SMC(ixFIRMWARE_FLAGS);
-		if (REG_GET_FIELD(val, FIRMWARE_FLAGS, INTERRUPTS_ENABLED))
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout) {
-		DRM_ERROR("Timeout for SMC firmware initialization\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int tonga_smu_start(struct amdgpu_device *adev)
-{
-	int result;
-	uint32_t val;
-
-	if (!tonga_is_smc_ram_running(adev)) {
-		val = RREG32_SMC(ixSMU_FIRMWARE);
-		if (!REG_GET_FIELD(val, SMU_FIRMWARE, SMU_MODE)) {
-			result = tonga_smu_start_in_non_protection_mode(adev);
-			if (result)
-				return result;
-		} else {
-			result = tonga_smu_start_in_protection_mode(adev);
-			if (result)
-				return result;
-		}
-	}
-
-	return tonga_smu_request_load_fw(adev);
-}
-
-static const struct amdgpu_smumgr_funcs tonga_smumgr_funcs = {
-	.check_fw_load_finish = tonga_smu_check_fw_load_finish,
-	.request_smu_load_fw = NULL,
-	.request_smu_specific_fw = NULL,
-};
-
-int tonga_smu_init(struct amdgpu_device *adev)
-{
-	struct tonga_smu_private_data *private;
-	uint32_t image_size = ((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
-	uint32_t smu_internal_buffer_size = 200*4096;
-	struct amdgpu_bo **toc_buf = &adev->smu.toc_buf;
-	struct amdgpu_bo **smu_buf = &adev->smu.smu_buf;
-	uint64_t mc_addr;
-	void *toc_buf_ptr;
-	void *smu_buf_ptr;
-	int ret;
-
-	private = kzalloc(sizeof(struct tonga_smu_private_data), GFP_KERNEL);
-	if (NULL == private)
-		return -ENOMEM;
-
-	/* allocate firmware buffers */
-	if (adev->firmware.smu_load)
-		amdgpu_ucode_init_bo(adev);
-
-	adev->smu.priv = private;
-	adev->smu.fw_flags = 0;
-
-	/* Allocate FW image data structure and header buffer */
-	ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE,
-			       true, AMDGPU_GEM_DOMAIN_VRAM,
-			       AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-			       NULL, NULL, toc_buf);
-	if (ret) {
-		DRM_ERROR("Failed to allocate memory for TOC buffer\n");
-		return -ENOMEM;
-	}
-
-	/* Allocate buffer for SMU internal buffer */
-	ret = amdgpu_bo_create(adev, smu_internal_buffer_size, PAGE_SIZE,
-			       true, AMDGPU_GEM_DOMAIN_VRAM,
-			       AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-			       NULL, NULL, smu_buf);
-	if (ret) {
-		DRM_ERROR("Failed to allocate memory for SMU internal buffer\n");
-		return -ENOMEM;
-	}
-
-	/* Retrieve GPU address for header buffer and internal buffer */
-	ret = amdgpu_bo_reserve(adev->smu.toc_buf, false);
-	if (ret) {
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to reserve the TOC buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_pin(adev->smu.toc_buf, AMDGPU_GEM_DOMAIN_VRAM, &mc_addr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.toc_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to pin the TOC buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_kmap(*toc_buf, &toc_buf_ptr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.toc_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to map the TOC buffer\n");
-		return -EINVAL;
-	}
-
-	amdgpu_bo_unreserve(adev->smu.toc_buf);
-	private->header_addr_low = lower_32_bits(mc_addr);
-	private->header_addr_high = upper_32_bits(mc_addr);
-	private->header = toc_buf_ptr;
-
-	ret = amdgpu_bo_reserve(adev->smu.smu_buf, false);
-	if (ret) {
-		amdgpu_bo_unref(&adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to reserve the SMU internal buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_pin(adev->smu.smu_buf, AMDGPU_GEM_DOMAIN_VRAM, &mc_addr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to pin the SMU internal buffer\n");
-		return -EINVAL;
-	}
-
-	ret = amdgpu_bo_kmap(*smu_buf, &smu_buf_ptr);
-	if (ret) {
-		amdgpu_bo_unreserve(adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.smu_buf);
-		amdgpu_bo_unref(&adev->smu.toc_buf);
-		DRM_ERROR("Failed to map the SMU internal buffer\n");
-		return -EINVAL;
-	}
-
-	amdgpu_bo_unreserve(adev->smu.smu_buf);
-	private->smu_buffer_addr_low = lower_32_bits(mc_addr);
-	private->smu_buffer_addr_high = upper_32_bits(mc_addr);
-
-	adev->smu.smumgr_funcs = &tonga_smumgr_funcs;
-
-	return 0;
-}
-
-int tonga_smu_fini(struct amdgpu_device *adev)
-{
-	amdgpu_bo_unref(&adev->smu.toc_buf);
-	amdgpu_bo_unref(&adev->smu.smu_buf);
-	kfree(adev->smu.priv);
-	adev->smu.priv = NULL;
-	if (adev->firmware.fw_buf)
-		amdgpu_ucode_fini_bo(adev);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index 132e613ed674..f6c941550b8f 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -116,7 +116,7 @@ static int uvd_v4_2_sw_init(void *handle)
 
 	ring = &adev->uvd.ring;
 	sprintf(ring->name, "uvd");
-	r = amdgpu_ring_init(adev, ring, 512, CP_PACKET2, 0xf,
+	r = amdgpu_ring_init(adev, ring, 512, PACKET0(mmUVD_NO_OP, 0), 0xf,
 			     &adev->uvd.irq, 0, AMDGPU_RING_TYPE_UVD);
 
 	return r;
@@ -526,6 +526,20 @@ static void uvd_v4_2_ring_emit_ib(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
+static unsigned uvd_v4_2_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		4; /* uvd_v4_2_ring_emit_ib */
+}
+
+static unsigned uvd_v4_2_ring_get_dma_frame_size(struct amdgpu_ring *ring)
+{
+	return
+		2 + /* uvd_v4_2_ring_emit_hdp_flush */
+		2 + /* uvd_v4_2_ring_emit_hdp_invalidate */
+		14; /* uvd_v4_2_ring_emit_fence  x1 no user fence */
+}
+
 /**
  * uvd_v4_2_mc_resume - memory controller programming
  *
@@ -756,6 +770,8 @@ static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
+	.get_emit_ib_size = uvd_v4_2_ring_get_emit_ib_size,
+	.get_dma_frame_size = uvd_v4_2_ring_get_dma_frame_size,
 };
 
 static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index 101de136ba63..400c16fe579e 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -112,7 +112,7 @@ static int uvd_v5_0_sw_init(void *handle)
 
 	ring = &adev->uvd.ring;
 	sprintf(ring->name, "uvd");
-	r = amdgpu_ring_init(adev, ring, 512, CP_PACKET2, 0xf,
+	r = amdgpu_ring_init(adev, ring, 512, PACKET0(mmUVD_NO_OP, 0), 0xf,
 			     &adev->uvd.irq, 0, AMDGPU_RING_TYPE_UVD);
 
 	return r;
@@ -577,6 +577,20 @@ static void uvd_v5_0_ring_emit_ib(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
+static unsigned uvd_v5_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		6; /* uvd_v5_0_ring_emit_ib */
+}
+
+static unsigned uvd_v5_0_ring_get_dma_frame_size(struct amdgpu_ring *ring)
+{
+	return
+		2 + /* uvd_v5_0_ring_emit_hdp_flush */
+		2 + /* uvd_v5_0_ring_emit_hdp_invalidate */
+		14; /* uvd_v5_0_ring_emit_fence  x1 no user fence */
+}
+
 static bool uvd_v5_0_is_idle(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -807,6 +821,8 @@ static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
+	.get_emit_ib_size = uvd_v5_0_ring_get_emit_ib_size,
+	.get_dma_frame_size = uvd_v5_0_ring_get_dma_frame_size,
 };
 
 static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index 7f21102bfb99..e0fd9f21ed95 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -116,7 +116,7 @@ static int uvd_v6_0_sw_init(void *handle)
 
 	ring = &adev->uvd.ring;
 	sprintf(ring->name, "uvd");
-	r = amdgpu_ring_init(adev, ring, 512, CP_PACKET2, 0xf,
+	r = amdgpu_ring_init(adev, ring, 512, PACKET0(mmUVD_NO_OP, 0), 0xf,
 			     &adev->uvd.irq, 0, AMDGPU_RING_TYPE_UVD);
 
 	return r;
@@ -396,21 +396,14 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 
 	uvd_v6_0_mc_resume(adev);
 
-	/* Set dynamic clock gating in S/W control mode */
-	if (adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG) {
-		uvd_v6_0_set_sw_clock_gating(adev);
-	} else {
-		/* disable clock gating */
-		uint32_t data = RREG32(mmUVD_CGC_CTRL);
-		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
-		WREG32(mmUVD_CGC_CTRL, data);
-	}
+	/* disable clock gating */
+	WREG32_FIELD(UVD_CGC_CTRL, DYN_CLOCK_MODE, 0);
 
 	/* disable interupt */
-	WREG32_P(mmUVD_MASTINT_EN, 0, ~UVD_MASTINT_EN__VCPU_EN_MASK);
+	WREG32_FIELD(UVD_MASTINT_EN, VCPU_EN, 0);
 
 	/* stall UMC and register bus before resetting VCPU */
-	WREG32_P(mmUVD_LMI_CTRL2, UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+	WREG32_FIELD(UVD_LMI_CTRL2, STALL_ARB_UMC, 1);
 	mdelay(1);
 
 	/* put LMI, VCPU, RBC etc... into reset */
@@ -426,7 +419,7 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 	mdelay(5);
 
 	/* take UVD block out of reset */
-	WREG32_P(mmSRBM_SOFT_RESET, 0, ~SRBM_SOFT_RESET__SOFT_RESET_UVD_MASK);
+	WREG32_FIELD(SRBM_SOFT_RESET, SOFT_RESET_UVD, 0);
 	mdelay(5);
 
 	/* initialize UVD memory controller */
@@ -461,7 +454,7 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 	WREG32(mmUVD_VCPU_CNTL, UVD_VCPU_CNTL__CLK_EN_MASK);
 
 	/* enable UMC */
-	WREG32_P(mmUVD_LMI_CTRL2, 0, ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
+	WREG32_FIELD(UVD_LMI_CTRL2, STALL_ARB_UMC, 0);
 
 	/* boot up the VCPU */
 	WREG32(mmUVD_SOFT_RESET, 0);
@@ -481,11 +474,9 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 			break;
 
 		DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
-		WREG32_P(mmUVD_SOFT_RESET, UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
-				~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+		WREG32_FIELD(UVD_SOFT_RESET, VCPU_SOFT_RESET, 1);
 		mdelay(10);
-		WREG32_P(mmUVD_SOFT_RESET, 0,
-			 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
+		WREG32_FIELD(UVD_SOFT_RESET, VCPU_SOFT_RESET, 0);
 		mdelay(10);
 		r = -1;
 	}
@@ -502,15 +493,14 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 	/* clear the bit 4 of UVD_STATUS */
 	WREG32_P(mmUVD_STATUS, 0, ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
 
+	/* force RBC into idle state */
 	rb_bufsz = order_base_2(ring->ring_size);
-	tmp = 0;
-	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
+	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
-	/* force RBC into idle state */
 	WREG32(mmUVD_RBC_RB_CNTL, tmp);
 
 	/* set the write pointer delay */
@@ -531,7 +521,7 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 	ring->wptr = RREG32(mmUVD_RBC_RB_RPTR);
 	WREG32(mmUVD_RBC_RB_WPTR, ring->wptr);
 
-	WREG32_P(mmUVD_RBC_RB_CNTL, 0, ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
+	WREG32_FIELD(UVD_RBC_RB_CNTL, RB_NO_FETCH, 0);
 
 	return 0;
 }
@@ -735,6 +725,31 @@ static void uvd_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, 0xE);
 }
 
+static unsigned uvd_v6_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		8; /* uvd_v6_0_ring_emit_ib */
+}
+
+static unsigned uvd_v6_0_ring_get_dma_frame_size(struct amdgpu_ring *ring)
+{
+	return
+		2 + /* uvd_v6_0_ring_emit_hdp_flush */
+		2 + /* uvd_v6_0_ring_emit_hdp_invalidate */
+		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
+		14; /* uvd_v6_0_ring_emit_fence x1 no user fence */
+}
+
+static unsigned uvd_v6_0_ring_get_dma_frame_size_vm(struct amdgpu_ring *ring)
+{
+	return
+		2 + /* uvd_v6_0_ring_emit_hdp_flush */
+		2 + /* uvd_v6_0_ring_emit_hdp_invalidate */
+		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
+		20 + /* uvd_v6_0_ring_emit_vm_flush */
+		14 + 14; /* uvd_v6_0_ring_emit_fence x2 vm fence */
+}
+
 static bool uvd_v6_0_is_idle(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -748,20 +763,82 @@ static int uvd_v6_0_wait_for_idle(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		if (!(RREG32(mmSRBM_STATUS) & SRBM_STATUS__UVD_BUSY_MASK))
+		if (uvd_v6_0_is_idle(handle))
 			return 0;
 	}
 	return -ETIMEDOUT;
 }
 
-static int uvd_v6_0_soft_reset(void *handle)
+#define AMDGPU_UVD_STATUS_BUSY_MASK    0xfd
+static int uvd_v6_0_check_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset = 0;
+	u32 tmp = RREG32(mmSRBM_STATUS);
+
+	if (REG_GET_FIELD(tmp, SRBM_STATUS, UVD_RQ_PENDING) ||
+	    REG_GET_FIELD(tmp, SRBM_STATUS, UVD_BUSY) ||
+	    (RREG32(mmUVD_STATUS) & AMDGPU_UVD_STATUS_BUSY_MASK))
+		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_UVD, 1);
+
+	if (srbm_soft_reset) {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang = true;
+		adev->uvd.srbm_soft_reset = srbm_soft_reset;
+	} else {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang = false;
+		adev->uvd.srbm_soft_reset = 0;
+	}
+	return 0;
+}
+static int uvd_v6_0_pre_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang)
+		return 0;
 
 	uvd_v6_0_stop(adev);
+	return 0;
+}
+
+static int uvd_v6_0_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang)
+		return 0;
+	srbm_soft_reset = adev->uvd.srbm_soft_reset;
+
+	if (srbm_soft_reset) {
+		u32 tmp;
+
+		tmp = RREG32(mmSRBM_SOFT_RESET);
+		tmp |= srbm_soft_reset;
+		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
+		WREG32(mmSRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmSRBM_SOFT_RESET);
+
+		udelay(50);
+
+		tmp &= ~srbm_soft_reset;
+		WREG32(mmSRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmSRBM_SOFT_RESET);
+
+		/* Wait a little for things to settle down */
+		udelay(50);
+	}
+
+	return 0;
+}
+
+static int uvd_v6_0_post_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_UVD].hang)
+		return 0;
 
-	WREG32_P(mmSRBM_SOFT_RESET, SRBM_SOFT_RESET__SOFT_RESET_UVD_MASK,
-			~SRBM_SOFT_RESET__SOFT_RESET_UVD_MASK);
 	mdelay(5);
 
 	return uvd_v6_0_start(adev);
@@ -902,21 +979,15 @@ static int uvd_v6_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
-	static int curstate = -1;
 
 	if (adev->asic_type == CHIP_FIJI ||
-			adev->asic_type == CHIP_POLARIS10)
-		uvd_v6_set_bypass_mode(adev, enable);
+	    adev->asic_type == CHIP_POLARIS10)
+		uvd_v6_set_bypass_mode(adev, state == AMD_CG_STATE_GATE ? true : false);
 
 	if (!(adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG))
 		return 0;
 
-	if (curstate == state)
-		return 0;
-
-	curstate = state;
-	if (enable) {
+	if (state == AMD_CG_STATE_GATE) {
 		/* disable HW gating and enable Sw gating */
 		uvd_v6_0_set_sw_clock_gating(adev);
 	} else {
@@ -946,6 +1017,8 @@ static int uvd_v6_0_set_powergating_state(void *handle,
 	if (!(adev->pg_flags & AMD_PG_SUPPORT_UVD))
 		return 0;
 
+	WREG32(mmUVD_POWER_STATUS, UVD_POWER_STATUS__UVD_PG_EN_MASK);
+
 	if (state == AMD_PG_STATE_GATE) {
 		uvd_v6_0_stop(adev);
 		return 0;
@@ -966,7 +1039,10 @@ const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
 	.resume = uvd_v6_0_resume,
 	.is_idle = uvd_v6_0_is_idle,
 	.wait_for_idle = uvd_v6_0_wait_for_idle,
+	.check_soft_reset = uvd_v6_0_check_soft_reset,
+	.pre_soft_reset = uvd_v6_0_pre_soft_reset,
 	.soft_reset = uvd_v6_0_soft_reset,
+	.post_soft_reset = uvd_v6_0_post_soft_reset,
 	.set_clockgating_state = uvd_v6_0_set_clockgating_state,
 	.set_powergating_state = uvd_v6_0_set_powergating_state,
 };
@@ -986,6 +1062,8 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
+	.get_emit_ib_size = uvd_v6_0_ring_get_emit_ib_size,
+	.get_dma_frame_size = uvd_v6_0_ring_get_dma_frame_size,
 };
 
 static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
@@ -1005,6 +1083,8 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
+	.get_emit_ib_size = uvd_v6_0_ring_get_emit_ib_size,
+	.get_dma_frame_size = uvd_v6_0_ring_get_dma_frame_size_vm,
 };
 
 static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index 80a37a602181..76e64ad04a53 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -30,16 +30,17 @@
 #include "amdgpu.h"
 #include "amdgpu_vce.h"
 #include "cikd.h"
-
 #include "vce/vce_2_0_d.h"
 #include "vce/vce_2_0_sh_mask.h"
-
+#include "smu/smu_7_0_1_d.h"
+#include "smu/smu_7_0_1_sh_mask.h"
 #include "oss/oss_2_0_d.h"
 #include "oss/oss_2_0_sh_mask.h"
 
 #define VCE_V2_0_FW_SIZE	(256 * 1024)
 #define VCE_V2_0_STACK_SIZE	(64 * 1024)
 #define VCE_V2_0_DATA_SIZE	(23552 * AMDGPU_MAX_VCE_HANDLES)
+#define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK	0x02
 
 static void vce_v2_0_mc_resume(struct amdgpu_device *adev);
 static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev);
@@ -96,6 +97,49 @@ static void vce_v2_0_ring_set_wptr(struct amdgpu_ring *ring)
 		WREG32(mmVCE_RB_WPTR2, ring->wptr);
 }
 
+static int vce_v2_0_lmi_clean(struct amdgpu_device *adev)
+{
+	int i, j;
+
+	for (i = 0; i < 10; ++i) {
+		for (j = 0; j < 100; ++j) {
+			uint32_t status = RREG32(mmVCE_LMI_STATUS);
+
+			if (status & 0x337f)
+				return 0;
+			mdelay(10);
+		}
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int vce_v2_0_firmware_loaded(struct amdgpu_device *adev)
+{
+	int i, j;
+
+	for (i = 0; i < 10; ++i) {
+		for (j = 0; j < 100; ++j) {
+			uint32_t status = RREG32(mmVCE_STATUS);
+
+			if (status & VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK)
+				return 0;
+			mdelay(10);
+		}
+
+		DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
+		WREG32_P(mmVCE_SOFT_RESET,
+			VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
+			~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+		mdelay(10);
+		WREG32_P(mmVCE_SOFT_RESET, 0,
+			~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+		mdelay(10);
+	}
+
+	return -ETIMEDOUT;
+}
+
 /**
  * vce_v2_0_start - start VCE block
  *
@@ -106,7 +150,7 @@ static void vce_v2_0_ring_set_wptr(struct amdgpu_ring *ring)
 static int vce_v2_0_start(struct amdgpu_device *adev)
 {
 	struct amdgpu_ring *ring;
-	int i, j, r;
+	int r;
 
 	vce_v2_0_mc_resume(adev);
 
@@ -127,36 +171,12 @@ static int vce_v2_0_start(struct amdgpu_device *adev)
 	WREG32(mmVCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
 	WREG32(mmVCE_RB_SIZE2, ring->ring_size / 4);
 
-	WREG32_P(mmVCE_VCPU_CNTL, VCE_VCPU_CNTL__CLK_EN_MASK, ~VCE_VCPU_CNTL__CLK_EN_MASK);
-
-	WREG32_P(mmVCE_SOFT_RESET,
-		 VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
-		 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
-
+	WREG32_FIELD(VCE_VCPU_CNTL, CLK_EN, 1);
+	WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 1);
 	mdelay(100);
+	WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 0);
 
-	WREG32_P(mmVCE_SOFT_RESET, 0, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
-
-	for (i = 0; i < 10; ++i) {
-		uint32_t status;
-		for (j = 0; j < 100; ++j) {
-			status = RREG32(mmVCE_STATUS);
-			if (status & 2)
-				break;
-			mdelay(10);
-		}
-		r = 0;
-		if (status & 2)
-			break;
-
-		DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
-		WREG32_P(mmVCE_SOFT_RESET, VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
-				~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
-		mdelay(10);
-		WREG32_P(mmVCE_SOFT_RESET, 0, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
-		mdelay(10);
-		r = -1;
-	}
+	r = vce_v2_0_firmware_loaded(adev);
 
 	/* clear BUSY flag */
 	WREG32_P(mmVCE_STATUS, 0, ~1);
@@ -173,6 +193,8 @@ static int vce_v2_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	adev->vce.num_rings = 2;
+
 	vce_v2_0_set_ring_funcs(adev);
 	vce_v2_0_set_irq_funcs(adev);
 
@@ -182,7 +204,7 @@ static int vce_v2_0_early_init(void *handle)
 static int vce_v2_0_sw_init(void *handle)
 {
 	struct amdgpu_ring *ring;
-	int r;
+	int r, i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* VCE */
@@ -199,19 +221,14 @@ static int vce_v2_0_sw_init(void *handle)
 	if (r)
 		return r;
 
-	ring = &adev->vce.ring[0];
-	sprintf(ring->name, "vce0");
-	r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
-			     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
-	if (r)
-		return r;
-
-	ring = &adev->vce.ring[1];
-	sprintf(ring->name, "vce1");
-	r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
-			     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
-	if (r)
-		return r;
+	for (i = 0; i < adev->vce.num_rings; i++) {
+		ring = &adev->vce.ring[i];
+		sprintf(ring->name, "vce%d", i);
+		r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
+				     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
+		if (r)
+			return r;
+	}
 
 	return r;
 }
@@ -234,29 +251,23 @@ static int vce_v2_0_sw_fini(void *handle)
 
 static int vce_v2_0_hw_init(void *handle)
 {
-	struct amdgpu_ring *ring;
-	int r;
+	int r, i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	r = vce_v2_0_start(adev);
+	/* this error mean vcpu not in running state, so just skip ring test, not stop driver initialize */
 	if (r)
-/* this error mean vcpu not in running state, so just skip ring test, not stop driver initialize */
 		return 0;
 
-	ring = &adev->vce.ring[0];
-	ring->ready = true;
-	r = amdgpu_ring_test_ring(ring);
-	if (r) {
-		ring->ready = false;
-		return r;
-	}
+	for (i = 0; i < adev->vce.num_rings; i++)
+		adev->vce.ring[i].ready = false;
 
-	ring = &adev->vce.ring[1];
-	ring->ready = true;
-	r = amdgpu_ring_test_ring(ring);
-	if (r) {
-		ring->ready = false;
-		return r;
+	for (i = 0; i < adev->vce.num_rings; i++) {
+		r = amdgpu_ring_test_ring(&adev->vce.ring[i]);
+		if (r)
+			return r;
+		else
+			adev->vce.ring[i].ready = true;
 	}
 
 	DRM_INFO("VCE initialized successfully.\n");
@@ -338,47 +349,50 @@ static void vce_v2_0_set_sw_cg(struct amdgpu_device *adev, bool gated)
 
 static void vce_v2_0_set_dyn_cg(struct amdgpu_device *adev, bool gated)
 {
-	u32 orig, tmp;
+	if (vce_v2_0_wait_for_idle(adev)) {
+		DRM_INFO("VCE is busy, Can't set clock gateing");
+		return;
+	}
 
-	if (gated) {
-		if (vce_v2_0_wait_for_idle(adev)) {
-			DRM_INFO("VCE is busy, Can't set clock gateing");
-			return;
-		}
-		WREG32_P(mmVCE_VCPU_CNTL, 0, ~VCE_VCPU_CNTL__CLK_EN_MASK);
-		WREG32_P(mmVCE_SOFT_RESET, VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
-		mdelay(100);
-		WREG32(mmVCE_STATUS, 0);
-	} else {
-		WREG32_P(mmVCE_VCPU_CNTL, VCE_VCPU_CNTL__CLK_EN_MASK, ~VCE_VCPU_CNTL__CLK_EN_MASK);
-		WREG32_P(mmVCE_SOFT_RESET, VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
-		mdelay(100);
+	WREG32_P(mmVCE_LMI_CTRL2, 0x100, ~0x100);
+
+	if (vce_v2_0_lmi_clean(adev)) {
+		DRM_INFO("LMI is busy, Can't set clock gateing");
+		return;
 	}
 
-	tmp = RREG32(mmVCE_CLOCK_GATING_B);
-	tmp &= ~0x00060006;
+	WREG32_P(mmVCE_VCPU_CNTL, 0, ~VCE_VCPU_CNTL__CLK_EN_MASK);
+	WREG32_P(mmVCE_SOFT_RESET,
+		 VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
+		 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+	WREG32(mmVCE_STATUS, 0);
+
+	if (gated)
+		WREG32(mmVCE_CGTT_CLK_OVERRIDE, 0);
+	/* LMI_MC/LMI_UMC always set in dynamic, set {CGC_*_GATE_MODE, CGC_*_SW_GATE} = {0, 0} */
 	if (gated) {
-		tmp |= 0xe10000;
+		/* Force CLOCK OFF , set {CGC_*_GATE_MODE, CGC_*_SW_GATE} = {*, 1} */
+		WREG32(mmVCE_CLOCK_GATING_B, 0xe90010);
 	} else {
-		tmp |= 0xe1;
-		tmp &= ~0xe10000;
+		/* Force CLOCK ON, set {CGC_*_GATE_MODE, CGC_*_SW_GATE} = {1, 0} */
+		WREG32(mmVCE_CLOCK_GATING_B, 0x800f1);
 	}
-	WREG32(mmVCE_CLOCK_GATING_B, tmp);
 
-	orig = tmp = RREG32(mmVCE_UENC_CLOCK_GATING);
-	tmp &= ~0x1fe000;
-	tmp &= ~0xff000000;
-	if (tmp != orig)
-		WREG32(mmVCE_UENC_CLOCK_GATING, tmp);
+	/* Set VCE_UENC_CLOCK_GATING always in dynamic mode {*_FORCE_ON, *_FORCE_OFF} = {0, 0}*/;
+	WREG32(mmVCE_UENC_CLOCK_GATING, 0x40);
 
-	orig = tmp = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
-	tmp &= ~0x3fc;
-	if (tmp != orig)
-		WREG32(mmVCE_UENC_REG_CLOCK_GATING, tmp);
+	/* set VCE_UENC_REG_CLOCK_GATING always in dynamic mode */
+	WREG32(mmVCE_UENC_REG_CLOCK_GATING, 0x00);
 
-	if (gated)
-		WREG32(mmVCE_CGTT_CLK_OVERRIDE, 0);
-	WREG32_P(mmVCE_SOFT_RESET, 0, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+	WREG32_P(mmVCE_LMI_CTRL2, 0, ~0x100);
+	if(!gated) {
+		WREG32_P(mmVCE_VCPU_CNTL, VCE_VCPU_CNTL__CLK_EN_MASK, ~VCE_VCPU_CNTL__CLK_EN_MASK);
+		mdelay(100);
+		WREG32_P(mmVCE_SOFT_RESET, 0, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+
+		vce_v2_0_firmware_loaded(adev);
+		WREG32_P(mmVCE_STATUS, 0, ~VCE_STATUS__JOB_BUSY_MASK);
+	}
 }
 
 static void vce_v2_0_disable_cg(struct amdgpu_device *adev)
@@ -458,9 +472,7 @@ static void vce_v2_0_mc_resume(struct amdgpu_device *adev)
 	WREG32(mmVCE_VCPU_CACHE_SIZE2, size);
 
 	WREG32_P(mmVCE_LMI_CTRL2, 0x0, ~0x100);
-
-	WREG32_P(mmVCE_SYS_INT_EN, VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,
-		 ~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
+	WREG32_FIELD(VCE_SYS_INT_EN, VCE_SYS_INT_TRAP_INTERRUPT_EN, 1);
 
 	vce_v2_0_init_cg(adev);
 }
@@ -474,11 +486,11 @@ static bool vce_v2_0_is_idle(void *handle)
 
 static int vce_v2_0_wait_for_idle(void *handle)
 {
-	unsigned i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	unsigned i;
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		if (!(RREG32(mmSRBM_STATUS2) & SRBM_STATUS2__VCE_BUSY_MASK))
+		if (vce_v2_0_is_idle(handle))
 			return 0;
 	}
 	return -ETIMEDOUT;
@@ -488,8 +500,7 @@ static int vce_v2_0_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	WREG32_P(mmSRBM_SOFT_RESET, SRBM_SOFT_RESET__SOFT_RESET_VCE_MASK,
-			~SRBM_SOFT_RESET__SOFT_RESET_VCE_MASK);
+	WREG32_FIELD(SRBM_SOFT_RESET, SOFT_RESET_VCE, 1);
 	mdelay(5);
 
 	return vce_v2_0_start(adev);
@@ -516,10 +527,8 @@ static int vce_v2_0_process_interrupt(struct amdgpu_device *adev,
 	DRM_DEBUG("IH: VCE\n");
 	switch (entry->src_data) {
 	case 0:
-		amdgpu_fence_process(&adev->vce.ring[0]);
-		break;
 	case 1:
-		amdgpu_fence_process(&adev->vce.ring[1]);
+		amdgpu_fence_process(&adev->vce.ring[entry->src_data]);
 		break;
 	default:
 		DRM_ERROR("Unhandled interrupt: %d %d\n",
@@ -530,11 +539,28 @@ static int vce_v2_0_process_interrupt(struct amdgpu_device *adev,
 	return 0;
 }
 
+static void vce_v2_0_set_bypass_mode(struct amdgpu_device *adev, bool enable)
+{
+	u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);
+
+	if (enable)
+		tmp |= GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
+	else
+		tmp &= ~GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
+
+	WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp);
+}
+
+
 static int vce_v2_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
 	bool gate = false;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
+
+
+	vce_v2_0_set_bypass_mode(adev, enable);
 
 	if (state == AMD_CG_STATE_GATE)
 		gate = true;
@@ -596,12 +622,16 @@ static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_vce_ring_begin_use,
 	.end_use = amdgpu_vce_ring_end_use,
+	.get_emit_ib_size = amdgpu_vce_ring_get_emit_ib_size,
+	.get_dma_frame_size = amdgpu_vce_ring_get_dma_frame_size,
 };
 
 static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev)
 {
-	adev->vce.ring[0].funcs = &vce_v2_0_ring_funcs;
-	adev->vce.ring[1].funcs = &vce_v2_0_ring_funcs;
+	int i;
+
+	for (i = 0; i < adev->vce.num_rings; i++)
+		adev->vce.ring[i].funcs = &vce_v2_0_ring_funcs;
 }
 
 static const struct amdgpu_irq_src_funcs vce_v2_0_irq_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index c271abffd8dd..3f6db4ec0102 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -37,6 +37,9 @@
 #include "gca/gfx_8_0_d.h"
 #include "smu/smu_7_1_2_d.h"
 #include "smu/smu_7_1_2_sh_mask.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+
 
 #define GRBM_GFX_INDEX__VCE_INSTANCE__SHIFT	0x04
 #define GRBM_GFX_INDEX__VCE_INSTANCE_MASK	0x10
@@ -67,8 +70,10 @@ static uint32_t vce_v3_0_ring_get_rptr(struct amdgpu_ring *ring)
 
 	if (ring == &adev->vce.ring[0])
 		return RREG32(mmVCE_RB_RPTR);
-	else
+	else if (ring == &adev->vce.ring[1])
 		return RREG32(mmVCE_RB_RPTR2);
+	else
+		return RREG32(mmVCE_RB_RPTR3);
 }
 
 /**
@@ -84,8 +89,10 @@ static uint32_t vce_v3_0_ring_get_wptr(struct amdgpu_ring *ring)
 
 	if (ring == &adev->vce.ring[0])
 		return RREG32(mmVCE_RB_WPTR);
-	else
+	else if (ring == &adev->vce.ring[1])
 		return RREG32(mmVCE_RB_WPTR2);
+	else
+		return RREG32(mmVCE_RB_WPTR3);
 }
 
 /**
@@ -101,108 +108,80 @@ static void vce_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
 
 	if (ring == &adev->vce.ring[0])
 		WREG32(mmVCE_RB_WPTR, ring->wptr);
-	else
+	else if (ring == &adev->vce.ring[1])
 		WREG32(mmVCE_RB_WPTR2, ring->wptr);
+	else
+		WREG32(mmVCE_RB_WPTR3, ring->wptr);
 }
 
 static void vce_v3_0_override_vce_clock_gating(struct amdgpu_device *adev, bool override)
 {
-	u32 tmp, data;
-
-	tmp = data = RREG32(mmVCE_RB_ARB_CTRL);
-	if (override)
-		data |= VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;
-	else
-		data &= ~VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;
-
-	if (tmp != data)
-		WREG32(mmVCE_RB_ARB_CTRL, data);
+	WREG32_FIELD(VCE_RB_ARB_CTRL, VCE_CGTT_OVERRIDE, override ? 1 : 0);
 }
 
 static void vce_v3_0_set_vce_sw_clock_gating(struct amdgpu_device *adev,
 					     bool gated)
 {
-	u32 tmp, data;
+	u32 data;
+
 	/* Set Override to disable Clock Gating */
 	vce_v3_0_override_vce_clock_gating(adev, true);
 
-	if (!gated) {
-		/* Force CLOCK ON for VCE_CLOCK_GATING_B,
-		 * {*_FORCE_ON, *_FORCE_OFF} = {1, 0}
-		 * VREG can be FORCE ON or set to Dynamic, but can't be OFF
-		 */
-		tmp = data = RREG32(mmVCE_CLOCK_GATING_B);
+	/* This function enables MGCG which is controlled by firmware.
+	   With the clocks in the gated state the core is still
+	   accessible but the firmware will throttle the clocks on the
+	   fly as necessary.
+	*/
+	if (gated) {
+		data = RREG32(mmVCE_CLOCK_GATING_B);
 		data |= 0x1ff;
 		data &= ~0xef0000;
-		if (tmp != data)
-			WREG32(mmVCE_CLOCK_GATING_B, data);
+		WREG32(mmVCE_CLOCK_GATING_B, data);
 
-		/* Force CLOCK ON for VCE_UENC_CLOCK_GATING,
-		 * {*_FORCE_ON, *_FORCE_OFF} = {1, 0}
-		 */
-		tmp = data = RREG32(mmVCE_UENC_CLOCK_GATING);
+		data = RREG32(mmVCE_UENC_CLOCK_GATING);
 		data |= 0x3ff000;
 		data &= ~0xffc00000;
-		if (tmp != data)
-			WREG32(mmVCE_UENC_CLOCK_GATING, data);
+		WREG32(mmVCE_UENC_CLOCK_GATING, data);
 
-		/* set VCE_UENC_CLOCK_GATING_2 */
-		tmp = data = RREG32(mmVCE_UENC_CLOCK_GATING_2);
+		data = RREG32(mmVCE_UENC_CLOCK_GATING_2);
 		data |= 0x2;
-		data &= ~0x2;
-		if (tmp != data)
-			WREG32(mmVCE_UENC_CLOCK_GATING_2, data);
+		data &= ~0x00010000;
+		WREG32(mmVCE_UENC_CLOCK_GATING_2, data);
 
-		/* Force CLOCK ON for VCE_UENC_REG_CLOCK_GATING */
-		tmp = data = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
+		data = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
 		data |= 0x37f;
-		if (tmp != data)
-			WREG32(mmVCE_UENC_REG_CLOCK_GATING, data);
+		WREG32(mmVCE_UENC_REG_CLOCK_GATING, data);
 
-		/* Force VCE_UENC_DMA_DCLK_CTRL Clock ON */
-		tmp = data = RREG32(mmVCE_UENC_DMA_DCLK_CTRL);
+		data = RREG32(mmVCE_UENC_DMA_DCLK_CTRL);
 		data |= VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
-				VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
-				VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
-				0x8;
-		if (tmp != data)
-			WREG32(mmVCE_UENC_DMA_DCLK_CTRL, data);
+			VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
+			VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
+			0x8;
+		WREG32(mmVCE_UENC_DMA_DCLK_CTRL, data);
 	} else {
-		/* Force CLOCK OFF for VCE_CLOCK_GATING_B,
-		 * {*, *_FORCE_OFF} = {*, 1}
-		 * set VREG to Dynamic, as it can't be OFF
-		 */
-		tmp = data = RREG32(mmVCE_CLOCK_GATING_B);
+		data = RREG32(mmVCE_CLOCK_GATING_B);
 		data &= ~0x80010;
 		data |= 0xe70008;
-		if (tmp != data)
-			WREG32(mmVCE_CLOCK_GATING_B, data);
-		/* Force CLOCK OFF for VCE_UENC_CLOCK_GATING,
-		 * Force ClOCK OFF takes precedent over Force CLOCK ON setting.
-		 * {*_FORCE_ON, *_FORCE_OFF} = {*, 1}
-		 */
-		tmp = data = RREG32(mmVCE_UENC_CLOCK_GATING);
+		WREG32(mmVCE_CLOCK_GATING_B, data);
+
+		data = RREG32(mmVCE_UENC_CLOCK_GATING);
 		data |= 0xffc00000;
-		if (tmp != data)
-			WREG32(mmVCE_UENC_CLOCK_GATING, data);
-		/* Set VCE_UENC_CLOCK_GATING_2 */
-		tmp = data = RREG32(mmVCE_UENC_CLOCK_GATING_2);
+		WREG32(mmVCE_UENC_CLOCK_GATING, data);
+
+		data = RREG32(mmVCE_UENC_CLOCK_GATING_2);
 		data |= 0x10000;
-		if (tmp != data)
-			WREG32(mmVCE_UENC_CLOCK_GATING_2, data);
-		/* Set VCE_UENC_REG_CLOCK_GATING to dynamic */
-		tmp = data = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
+		WREG32(mmVCE_UENC_CLOCK_GATING_2, data);
+
+		data = RREG32(mmVCE_UENC_REG_CLOCK_GATING);
 		data &= ~0xffc00000;
-		if (tmp != data)
-			WREG32(mmVCE_UENC_REG_CLOCK_GATING, data);
-		/* Set VCE_UENC_DMA_DCLK_CTRL CG always in dynamic mode */
-		tmp = data = RREG32(mmVCE_UENC_DMA_DCLK_CTRL);
+		WREG32(mmVCE_UENC_REG_CLOCK_GATING, data);
+
+		data = RREG32(mmVCE_UENC_DMA_DCLK_CTRL);
 		data &= ~(VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
-				VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
-				VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
-				0x8);
-		if (tmp != data)
-			WREG32(mmVCE_UENC_DMA_DCLK_CTRL, data);
+			  VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
+			  VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK  |
+			  0x8);
+		WREG32(mmVCE_UENC_DMA_DCLK_CTRL, data);
 	}
 	vce_v3_0_override_vce_clock_gating(adev, false);
 }
@@ -221,12 +200,9 @@ static int vce_v3_0_firmware_loaded(struct amdgpu_device *adev)
 		}
 
 		DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
-		WREG32_P(mmVCE_SOFT_RESET,
-			VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
-			~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+		WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 1);
 		mdelay(10);
-		WREG32_P(mmVCE_SOFT_RESET, 0,
-			~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+		WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 0);
 		mdelay(10);
 	}
 
@@ -259,43 +235,34 @@ static int vce_v3_0_start(struct amdgpu_device *adev)
 	WREG32(mmVCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
 	WREG32(mmVCE_RB_SIZE2, ring->ring_size / 4);
 
+	ring = &adev->vce.ring[2];
+	WREG32(mmVCE_RB_RPTR3, ring->wptr);
+	WREG32(mmVCE_RB_WPTR3, ring->wptr);
+	WREG32(mmVCE_RB_BASE_LO3, ring->gpu_addr);
+	WREG32(mmVCE_RB_BASE_HI3, upper_32_bits(ring->gpu_addr));
+	WREG32(mmVCE_RB_SIZE3, ring->ring_size / 4);
+
 	mutex_lock(&adev->grbm_idx_mutex);
 	for (idx = 0; idx < 2; ++idx) {
 		if (adev->vce.harvest_config & (1 << idx))
 			continue;
 
-		if (idx == 0)
-			WREG32_P(mmGRBM_GFX_INDEX, 0,
-				~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
-		else
-			WREG32_P(mmGRBM_GFX_INDEX,
-				GRBM_GFX_INDEX__VCE_INSTANCE_MASK,
-				~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
-
+		WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, idx);
 		vce_v3_0_mc_resume(adev, idx);
-
-		WREG32_P(mmVCE_STATUS, VCE_STATUS__JOB_BUSY_MASK,
-		         ~VCE_STATUS__JOB_BUSY_MASK);
+		WREG32_FIELD(VCE_STATUS, JOB_BUSY, 1);
 
 		if (adev->asic_type >= CHIP_STONEY)
 			WREG32_P(mmVCE_VCPU_CNTL, 1, ~0x200001);
 		else
-			WREG32_P(mmVCE_VCPU_CNTL, VCE_VCPU_CNTL__CLK_EN_MASK,
-				~VCE_VCPU_CNTL__CLK_EN_MASK);
-
-		WREG32_P(mmVCE_SOFT_RESET, 0,
-			~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+			WREG32_FIELD(VCE_VCPU_CNTL, CLK_EN, 1);
 
+		WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 0);
 		mdelay(100);
 
 		r = vce_v3_0_firmware_loaded(adev);
 
 		/* clear BUSY flag */
-		WREG32_P(mmVCE_STATUS, 0, ~VCE_STATUS__JOB_BUSY_MASK);
-
-		/* Set Clock-Gating off */
-		if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)
-			vce_v3_0_set_vce_sw_clock_gating(adev, false);
+		WREG32_FIELD(VCE_STATUS, JOB_BUSY, 0);
 
 		if (r) {
 			DRM_ERROR("VCE not responding, giving up!!!\n");
@@ -304,7 +271,7 @@ static int vce_v3_0_start(struct amdgpu_device *adev)
 		}
 	}
 
-	WREG32_P(mmGRBM_GFX_INDEX, 0, ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+	WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, 0);
 	mutex_unlock(&adev->grbm_idx_mutex);
 
 	return 0;
@@ -319,33 +286,25 @@ static int vce_v3_0_stop(struct amdgpu_device *adev)
 		if (adev->vce.harvest_config & (1 << idx))
 			continue;
 
-		if (idx == 0)
-			WREG32_P(mmGRBM_GFX_INDEX, 0,
-				~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
-		else
-			WREG32_P(mmGRBM_GFX_INDEX,
-				GRBM_GFX_INDEX__VCE_INSTANCE_MASK,
-				~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+		WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, idx);
 
 		if (adev->asic_type >= CHIP_STONEY)
 			WREG32_P(mmVCE_VCPU_CNTL, 0, ~0x200001);
 		else
-			WREG32_P(mmVCE_VCPU_CNTL, 0,
-				~VCE_VCPU_CNTL__CLK_EN_MASK);
+			WREG32_FIELD(VCE_VCPU_CNTL, CLK_EN, 0);
+
 		/* hold on ECPU */
-		WREG32_P(mmVCE_SOFT_RESET,
-			 VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
-			 ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+		WREG32_FIELD(VCE_SOFT_RESET, ECPU_SOFT_RESET, 1);
 
 		/* clear BUSY flag */
-		WREG32_P(mmVCE_STATUS, 0, ~VCE_STATUS__JOB_BUSY_MASK);
+		WREG32_FIELD(VCE_STATUS, JOB_BUSY, 0);
 
 		/* Set Clock-Gating off */
 		if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)
 			vce_v3_0_set_vce_sw_clock_gating(adev, false);
 	}
 
-	WREG32_P(mmGRBM_GFX_INDEX, 0, ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+	WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, 0);
 	mutex_unlock(&adev->grbm_idx_mutex);
 
 	return 0;
@@ -399,6 +358,8 @@ static int vce_v3_0_early_init(void *handle)
 	    (AMDGPU_VCE_HARVEST_VCE0 | AMDGPU_VCE_HARVEST_VCE1))
 		return -ENOENT;
 
+	adev->vce.num_rings = 3;
+
 	vce_v3_0_set_ring_funcs(adev);
 	vce_v3_0_set_irq_funcs(adev);
 
@@ -409,7 +370,7 @@ static int vce_v3_0_sw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	struct amdgpu_ring *ring;
-	int r;
+	int r, i;
 
 	/* VCE */
 	r = amdgpu_irq_add_id(adev, 167, &adev->vce.irq);
@@ -425,19 +386,14 @@ static int vce_v3_0_sw_init(void *handle)
 	if (r)
 		return r;
 
-	ring = &adev->vce.ring[0];
-	sprintf(ring->name, "vce0");
-	r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
-			     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
-	if (r)
-		return r;
-
-	ring = &adev->vce.ring[1];
-	sprintf(ring->name, "vce1");
-	r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
-			     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
-	if (r)
-		return r;
+	for (i = 0; i < adev->vce.num_rings; i++) {
+		ring = &adev->vce.ring[i];
+		sprintf(ring->name, "vce%d", i);
+		r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
+				     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
+		if (r)
+			return r;
+	}
 
 	return r;
 }
@@ -467,10 +423,10 @@ static int vce_v3_0_hw_init(void *handle)
 	if (r)
 		return r;
 
-	adev->vce.ring[0].ready = false;
-	adev->vce.ring[1].ready = false;
+	for (i = 0; i < adev->vce.num_rings; i++)
+		adev->vce.ring[i].ready = false;
 
-	for (i = 0; i < 2; i++) {
+	for (i = 0; i < adev->vce.num_rings; i++) {
 		r = amdgpu_ring_test_ring(&adev->vce.ring[i]);
 		if (r)
 			return r;
@@ -534,7 +490,7 @@ static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx)
 	WREG32_P(mmVCE_CLOCK_GATING_A, 0, ~(1 << 16));
 	WREG32_P(mmVCE_UENC_CLOCK_GATING, 0x1FF000, ~0xFF9FF000);
 	WREG32_P(mmVCE_UENC_REG_CLOCK_GATING, 0x3F, ~0x3F);
-	WREG32(mmVCE_CLOCK_GATING_B, 0xf7);
+	WREG32(mmVCE_CLOCK_GATING_B, 0x1FF);
 
 	WREG32(mmVCE_LMI_CTRL, 0x00398000);
 	WREG32_P(mmVCE_LMI_CACHE_CTRL, 0x0, ~0x1);
@@ -573,9 +529,7 @@ static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx)
 	}
 
 	WREG32_P(mmVCE_LMI_CTRL2, 0x0, ~0x100);
-
-	WREG32_P(mmVCE_SYS_INT_EN, VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,
-		 ~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
+	WREG32_FIELD(VCE_SYS_INT_EN, VCE_SYS_INT_TRAP_INTERRUPT_EN, 1);
 }
 
 static bool vce_v3_0_is_idle(void *handle)
@@ -601,20 +555,108 @@ static int vce_v3_0_wait_for_idle(void *handle)
 	return -ETIMEDOUT;
 }
 
+#define  VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK  0x00000008L   /* AUTO_BUSY */
+#define  VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK   0x00000010L   /* RB0_BUSY */
+#define  VCE_STATUS_VCPU_REPORT_RB1_BUSY_MASK   0x00000020L   /* RB1_BUSY */
+#define  AMDGPU_VCE_STATUS_BUSY_MASK (VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK | \
+				      VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK)
+
+static int vce_v3_0_check_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	u32 srbm_soft_reset = 0;
+
+	/* According to VCE team , we should use VCE_STATUS instead
+	 * SRBM_STATUS.VCE_BUSY bit for busy status checking.
+	 * GRBM_GFX_INDEX.INSTANCE_INDEX is used to specify which VCE
+	 * instance's registers are accessed
+	 * (0 for 1st instance, 10 for 2nd instance).
+	 *
+	 *VCE_STATUS
+	 *|UENC|ACPI|AUTO ACTIVE|RB1 |RB0 |RB2 |          |FW_LOADED|JOB |
+	 *|----+----+-----------+----+----+----+----------+---------+----|
+	 *|bit8|bit7|    bit6   |bit5|bit4|bit3|   bit2   |  bit1   |bit0|
+	 *
+	 * VCE team suggest use bit 3--bit 6 for busy status check
+	 */
+	mutex_lock(&adev->grbm_idx_mutex);
+	WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0);
+	if (RREG32(mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
+		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
+		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
+	}
+	WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0x10);
+	if (RREG32(mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
+		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
+		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
+	}
+	WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0);
+
+	if (srbm_soft_reset) {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang = true;
+		adev->vce.srbm_soft_reset = srbm_soft_reset;
+	} else {
+		adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang = false;
+		adev->vce.srbm_soft_reset = 0;
+	}
+	mutex_unlock(&adev->grbm_idx_mutex);
+	return 0;
+}
+
 static int vce_v3_0_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	u32 mask = 0;
+	u32 srbm_soft_reset;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang)
+		return 0;
+	srbm_soft_reset = adev->vce.srbm_soft_reset;
+
+	if (srbm_soft_reset) {
+		u32 tmp;
 
-	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE0) ? 0 : SRBM_SOFT_RESET__SOFT_RESET_VCE0_MASK;
-	mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE1) ? 0 : SRBM_SOFT_RESET__SOFT_RESET_VCE1_MASK;
+		tmp = RREG32(mmSRBM_SOFT_RESET);
+		tmp |= srbm_soft_reset;
+		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
+		WREG32(mmSRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmSRBM_SOFT_RESET);
+
+		udelay(50);
+
+		tmp &= ~srbm_soft_reset;
+		WREG32(mmSRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmSRBM_SOFT_RESET);
+
+		/* Wait a little for things to settle down */
+		udelay(50);
+	}
+
+	return 0;
+}
+
+static int vce_v3_0_pre_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang)
+		return 0;
 
-	WREG32_P(mmSRBM_SOFT_RESET, mask,
-		 ~(SRBM_SOFT_RESET__SOFT_RESET_VCE0_MASK |
-		   SRBM_SOFT_RESET__SOFT_RESET_VCE1_MASK));
 	mdelay(5);
 
-	return vce_v3_0_start(adev);
+	return vce_v3_0_suspend(adev);
+}
+
+
+static int vce_v3_0_post_soft_reset(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!adev->ip_block_status[AMD_IP_BLOCK_TYPE_VCE].hang)
+		return 0;
+
+	mdelay(5);
+
+	return vce_v3_0_resume(adev);
 }
 
 static int vce_v3_0_set_interrupt_state(struct amdgpu_device *adev,
@@ -637,13 +679,12 @@ static int vce_v3_0_process_interrupt(struct amdgpu_device *adev,
 {
 	DRM_DEBUG("IH: VCE\n");
 
-	WREG32_P(mmVCE_SYS_INT_STATUS,
-		VCE_SYS_INT_STATUS__VCE_SYS_INT_TRAP_INTERRUPT_INT_MASK,
-		~VCE_SYS_INT_STATUS__VCE_SYS_INT_TRAP_INTERRUPT_INT_MASK);
+	WREG32_FIELD(VCE_SYS_INT_STATUS, VCE_SYS_INT_TRAP_INTERRUPT_INT, 1);
 
 	switch (entry->src_data) {
 	case 0:
 	case 1:
+	case 2:
 		amdgpu_fence_process(&adev->vce.ring[entry->src_data]);
 		break;
 	default:
@@ -655,7 +696,7 @@ static int vce_v3_0_process_interrupt(struct amdgpu_device *adev,
 	return 0;
 }
 
-static void vce_v3_set_bypass_mode(struct amdgpu_device *adev, bool enable)
+static void vce_v3_0_set_bypass_mode(struct amdgpu_device *adev, bool enable)
 {
 	u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);
 
@@ -674,8 +715,10 @@ static int vce_v3_0_set_clockgating_state(void *handle,
 	bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
 	int i;
 
-	if (adev->asic_type == CHIP_POLARIS10)
-		vce_v3_set_bypass_mode(adev, enable);
+	if ((adev->asic_type == CHIP_POLARIS10) ||
+		(adev->asic_type == CHIP_TONGA) ||
+		(adev->asic_type == CHIP_FIJI))
+		vce_v3_0_set_bypass_mode(adev, enable);
 
 	if (!(adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG))
 		return 0;
@@ -686,13 +729,7 @@ static int vce_v3_0_set_clockgating_state(void *handle,
 		if (adev->vce.harvest_config & (1 << i))
 			continue;
 
-		if (i == 0)
-			WREG32_P(mmGRBM_GFX_INDEX, 0,
-					~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
-		else
-			WREG32_P(mmGRBM_GFX_INDEX,
-					GRBM_GFX_INDEX__VCE_INSTANCE_MASK,
-					~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+		WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, i);
 
 		if (enable) {
 			/* initialize VCE_CLOCK_GATING_A: Clock ON/OFF delay */
@@ -711,7 +748,7 @@ static int vce_v3_0_set_clockgating_state(void *handle,
 		vce_v3_0_set_vce_sw_clock_gating(adev, enable);
 	}
 
-	WREG32_P(mmGRBM_GFX_INDEX, 0, ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+	WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, 0);
 	mutex_unlock(&adev->grbm_idx_mutex);
 
 	return 0;
@@ -739,6 +776,60 @@ static int vce_v3_0_set_powergating_state(void *handle,
 		return vce_v3_0_start(adev);
 }
 
+static void vce_v3_0_ring_emit_ib(struct amdgpu_ring *ring,
+		struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
+{
+	amdgpu_ring_write(ring, VCE_CMD_IB_VM);
+	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
+	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+	amdgpu_ring_write(ring, ib->length_dw);
+}
+
+static void vce_v3_0_emit_vm_flush(struct amdgpu_ring *ring,
+			 unsigned int vm_id, uint64_t pd_addr)
+{
+	amdgpu_ring_write(ring, VCE_CMD_UPDATE_PTB);
+	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, pd_addr >> 12);
+
+	amdgpu_ring_write(ring, VCE_CMD_FLUSH_TLB);
+	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, VCE_CMD_END);
+}
+
+static void vce_v3_0_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+	uint32_t seq = ring->fence_drv.sync_seq;
+	uint64_t addr = ring->fence_drv.gpu_addr;
+
+	amdgpu_ring_write(ring, VCE_CMD_WAIT_GE);
+	amdgpu_ring_write(ring, lower_32_bits(addr));
+	amdgpu_ring_write(ring, upper_32_bits(addr));
+	amdgpu_ring_write(ring, seq);
+}
+
+static unsigned vce_v3_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+{
+	return
+		5; /* vce_v3_0_ring_emit_ib */
+}
+
+static unsigned vce_v3_0_ring_get_dma_frame_size(struct amdgpu_ring *ring)
+{
+	return
+		4 + /* vce_v3_0_emit_pipeline_sync */
+		6; /* amdgpu_vce_ring_emit_fence x1 no user fence */
+}
+
+static unsigned vce_v3_0_ring_get_dma_frame_size_vm(struct amdgpu_ring *ring)
+{
+	return
+		6 + /* vce_v3_0_emit_vm_flush */
+		4 + /* vce_v3_0_emit_pipeline_sync */
+		6 + 6; /* amdgpu_vce_ring_emit_fence x2 vm fence */
+}
+
 const struct amd_ip_funcs vce_v3_0_ip_funcs = {
 	.name = "vce_v3_0",
 	.early_init = vce_v3_0_early_init,
@@ -751,12 +842,15 @@ const struct amd_ip_funcs vce_v3_0_ip_funcs = {
 	.resume = vce_v3_0_resume,
 	.is_idle = vce_v3_0_is_idle,
 	.wait_for_idle = vce_v3_0_wait_for_idle,
+	.check_soft_reset = vce_v3_0_check_soft_reset,
+	.pre_soft_reset = vce_v3_0_pre_soft_reset,
 	.soft_reset = vce_v3_0_soft_reset,
+	.post_soft_reset = vce_v3_0_post_soft_reset,
 	.set_clockgating_state = vce_v3_0_set_clockgating_state,
 	.set_powergating_state = vce_v3_0_set_powergating_state,
 };
 
-static const struct amdgpu_ring_funcs vce_v3_0_ring_funcs = {
+static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = {
 	.get_rptr = vce_v3_0_ring_get_rptr,
 	.get_wptr = vce_v3_0_ring_get_wptr,
 	.set_wptr = vce_v3_0_ring_set_wptr,
@@ -769,12 +863,42 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_vce_ring_begin_use,
 	.end_use = amdgpu_vce_ring_end_use,
+	.get_emit_ib_size = vce_v3_0_ring_get_emit_ib_size,
+	.get_dma_frame_size = vce_v3_0_ring_get_dma_frame_size,
+};
+
+static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = {
+	.get_rptr = vce_v3_0_ring_get_rptr,
+	.get_wptr = vce_v3_0_ring_get_wptr,
+	.set_wptr = vce_v3_0_ring_set_wptr,
+	.parse_cs = NULL,
+	.emit_ib = vce_v3_0_ring_emit_ib,
+	.emit_vm_flush = vce_v3_0_emit_vm_flush,
+	.emit_pipeline_sync = vce_v3_0_emit_pipeline_sync,
+	.emit_fence = amdgpu_vce_ring_emit_fence,
+	.test_ring = amdgpu_vce_ring_test_ring,
+	.test_ib = amdgpu_vce_ring_test_ib,
+	.insert_nop = amdgpu_ring_insert_nop,
+	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.begin_use = amdgpu_vce_ring_begin_use,
+	.end_use = amdgpu_vce_ring_end_use,
+	.get_emit_ib_size = vce_v3_0_ring_get_emit_ib_size,
+	.get_dma_frame_size = vce_v3_0_ring_get_dma_frame_size_vm,
 };
 
 static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev)
 {
-	adev->vce.ring[0].funcs = &vce_v3_0_ring_funcs;
-	adev->vce.ring[1].funcs = &vce_v3_0_ring_funcs;
+	int i;
+
+	if (adev->asic_type >= CHIP_STONEY) {
+		for (i = 0; i < adev->vce.num_rings; i++)
+			adev->vce.ring[i].funcs = &vce_v3_0_ring_vm_funcs;
+		DRM_INFO("VCE enabled in VM mode\n");
+	} else {
+		for (i = 0; i < adev->vce.num_rings; i++)
+			adev->vce.ring[i].funcs = &vce_v3_0_ring_phys_funcs;
+		DRM_INFO("VCE enabled in physical mode\n");
+	}
 }
 
 static const struct amdgpu_irq_src_funcs vce_v3_0_irq_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 03a31c53aec3..c0d9aad7126f 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -77,7 +77,11 @@
 #if defined(CONFIG_DRM_AMD_ACP)
 #include "amdgpu_acp.h"
 #endif
+#include "dce_virtual.h"
 
+MODULE_FIRMWARE("amdgpu/topaz_smc.bin");
+MODULE_FIRMWARE("amdgpu/tonga_smc.bin");
+MODULE_FIRMWARE("amdgpu/fiji_smc.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_smc.bin");
 MODULE_FIRMWARE("amdgpu/polaris10_smc_sk.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_smc.bin");
@@ -444,18 +448,21 @@ static bool vi_read_bios_from_rom(struct amdgpu_device *adev,
 	return true;
 }
 
-static u32 vi_get_virtual_caps(struct amdgpu_device *adev)
+static void vi_detect_hw_virtualization(struct amdgpu_device *adev)
 {
-	u32 caps = 0;
-	u32 reg = RREG32(mmBIF_IOV_FUNC_IDENTIFIER);
-
-	if (REG_GET_FIELD(reg, BIF_IOV_FUNC_IDENTIFIER, IOV_ENABLE))
-		caps |= AMDGPU_VIRT_CAPS_SRIOV_EN;
-
-	if (REG_GET_FIELD(reg, BIF_IOV_FUNC_IDENTIFIER, FUNC_IDENTIFIER))
-		caps |= AMDGPU_VIRT_CAPS_IS_VF;
-
-	return caps;
+	uint32_t reg = RREG32(mmBIF_IOV_FUNC_IDENTIFIER);
+	/* bit0: 0 means pf and 1 means vf */
+	/* bit31: 0 means disable IOV and 1 means enable */
+	if (reg & 1)
+		adev->virtualization.virtual_caps |= AMDGPU_SRIOV_CAPS_IS_VF;
+
+	if (reg & 0x80000000)
+		adev->virtualization.virtual_caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV;
+
+	if (reg == 0) {
+		if (is_virtual_machine()) /* passthrough mode exclus sr-iov mode */
+			adev->virtualization.virtual_caps |= AMDGPU_PASSTHROUGH_MODE;
+	}
 }
 
 static const struct amdgpu_allowed_register_entry tonga_allowed_read_registers[] = {
@@ -822,6 +829,60 @@ static const struct amdgpu_ip_block_version topaz_ip_blocks[] =
 	},
 };
 
+static const struct amdgpu_ip_block_version topaz_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vi_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 7,
+		.minor = 4,
+		.rev = 0,
+		.funcs = &gmc_v7_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 2,
+		.minor = 4,
+		.rev = 0,
+		.funcs = &iceland_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 7,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 1,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 8,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gfx_v8_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 2,
+		.minor = 4,
+		.rev = 0,
+		.funcs = &sdma_v2_4_ip_funcs,
+	},
+};
+
 static const struct amdgpu_ip_block_version tonga_ip_blocks[] =
 {
 	/* ORDER MATTERS! */
@@ -890,6 +951,74 @@ static const struct amdgpu_ip_block_version tonga_ip_blocks[] =
 	},
 };
 
+static const struct amdgpu_ip_block_version tonga_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vi_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 8,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gmc_v8_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 3,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &tonga_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 7,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 10,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 8,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gfx_v8_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 3,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &sdma_v3_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 5,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &uvd_v5_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 3,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vce_v3_0_ip_funcs,
+	},
+};
+
 static const struct amdgpu_ip_block_version fiji_ip_blocks[] =
 {
 	/* ORDER MATTERS! */
@@ -958,6 +1087,74 @@ static const struct amdgpu_ip_block_version fiji_ip_blocks[] =
 	},
 };
 
+static const struct amdgpu_ip_block_version fiji_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vi_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 8,
+		.minor = 5,
+		.rev = 0,
+		.funcs = &gmc_v8_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 3,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &tonga_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 7,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 10,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 8,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gfx_v8_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 3,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &sdma_v3_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &uvd_v6_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 3,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vce_v3_0_ip_funcs,
+	},
+};
+
 static const struct amdgpu_ip_block_version polaris11_ip_blocks[] =
 {
 	/* ORDER MATTERS! */
@@ -1026,6 +1223,74 @@ static const struct amdgpu_ip_block_version polaris11_ip_blocks[] =
 	},
 };
 
+static const struct amdgpu_ip_block_version polaris11_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vi_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 8,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &gmc_v8_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 3,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &tonga_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 7,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 11,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 8,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gfx_v8_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 3,
+		.minor = 1,
+		.rev = 0,
+		.funcs = &sdma_v3_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 6,
+		.minor = 3,
+		.rev = 0,
+		.funcs = &uvd_v6_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 3,
+		.minor = 4,
+		.rev = 0,
+		.funcs = &vce_v3_0_ip_funcs,
+	},
+};
+
 static const struct amdgpu_ip_block_version cz_ip_blocks[] =
 {
 	/* ORDER MATTERS! */
@@ -1103,34 +1368,142 @@ static const struct amdgpu_ip_block_version cz_ip_blocks[] =
 #endif
 };
 
+static const struct amdgpu_ip_block_version cz_ip_blocks_vd[] =
+{
+	/* ORDER MATTERS! */
+	{
+		.type = AMD_IP_BLOCK_TYPE_COMMON,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vi_common_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GMC,
+		.major = 8,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gmc_v8_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_IH,
+		.major = 3,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &cz_ih_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SMC,
+		.major = 8,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &amdgpu_pp_ip_funcs
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_DCE,
+		.major = 11,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &dce_virtual_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_GFX,
+		.major = 8,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &gfx_v8_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_SDMA,
+		.major = 3,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &sdma_v3_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &uvd_v6_0_ip_funcs,
+	},
+	{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 3,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vce_v3_0_ip_funcs,
+	},
+#if defined(CONFIG_DRM_AMD_ACP)
+	{
+		.type = AMD_IP_BLOCK_TYPE_ACP,
+		.major = 2,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &acp_ip_funcs,
+	},
+#endif
+};
+
 int vi_set_ip_blocks(struct amdgpu_device *adev)
 {
-	switch (adev->asic_type) {
-	case CHIP_TOPAZ:
-		adev->ip_blocks = topaz_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(topaz_ip_blocks);
-		break;
-	case CHIP_FIJI:
-		adev->ip_blocks = fiji_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(fiji_ip_blocks);
-		break;
-	case CHIP_TONGA:
-		adev->ip_blocks = tonga_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(tonga_ip_blocks);
-		break;
-	case CHIP_POLARIS11:
-	case CHIP_POLARIS10:
-		adev->ip_blocks = polaris11_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(polaris11_ip_blocks);
-		break;
-	case CHIP_CARRIZO:
-	case CHIP_STONEY:
-		adev->ip_blocks = cz_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(cz_ip_blocks);
-		break;
-	default:
-		/* FIXME: not supported yet */
-		return -EINVAL;
+	if (adev->enable_virtual_display) {
+		switch (adev->asic_type) {
+		case CHIP_TOPAZ:
+			adev->ip_blocks = topaz_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(topaz_ip_blocks_vd);
+			break;
+		case CHIP_FIJI:
+			adev->ip_blocks = fiji_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(fiji_ip_blocks_vd);
+			break;
+		case CHIP_TONGA:
+			adev->ip_blocks = tonga_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(tonga_ip_blocks_vd);
+			break;
+		case CHIP_POLARIS11:
+		case CHIP_POLARIS10:
+			adev->ip_blocks = polaris11_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(polaris11_ip_blocks_vd);
+			break;
+
+		case CHIP_CARRIZO:
+		case CHIP_STONEY:
+			adev->ip_blocks = cz_ip_blocks_vd;
+			adev->num_ip_blocks = ARRAY_SIZE(cz_ip_blocks_vd);
+			break;
+		default:
+			/* FIXME: not supported yet */
+			return -EINVAL;
+		}
+	} else {
+		switch (adev->asic_type) {
+		case CHIP_TOPAZ:
+			adev->ip_blocks = topaz_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(topaz_ip_blocks);
+			break;
+		case CHIP_FIJI:
+			adev->ip_blocks = fiji_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(fiji_ip_blocks);
+			break;
+		case CHIP_TONGA:
+			adev->ip_blocks = tonga_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(tonga_ip_blocks);
+			break;
+		case CHIP_POLARIS11:
+		case CHIP_POLARIS10:
+			adev->ip_blocks = polaris11_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(polaris11_ip_blocks);
+			break;
+		case CHIP_CARRIZO:
+		case CHIP_STONEY:
+			adev->ip_blocks = cz_ip_blocks;
+			adev->num_ip_blocks = ARRAY_SIZE(cz_ip_blocks);
+			break;
+		default:
+			/* FIXME: not supported yet */
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -1154,13 +1527,13 @@ static const struct amdgpu_asic_funcs vi_asic_funcs =
 {
 	.read_disabled_bios = &vi_read_disabled_bios,
 	.read_bios_from_rom = &vi_read_bios_from_rom,
+	.detect_hw_virtualization = vi_detect_hw_virtualization,
 	.read_register = &vi_read_register,
 	.reset = &vi_asic_reset,
 	.set_vga_state = &vi_vga_set_state,
 	.get_xclk = &vi_get_xclk,
 	.set_uvd_clocks = &vi_set_uvd_clocks,
 	.set_vce_clocks = &vi_set_vce_clocks,
-	.get_virtual_caps = &vi_get_virtual_caps,
 };
 
 static int vi_common_early_init(void *handle)
@@ -1248,8 +1621,17 @@ static int vi_common_early_init(void *handle)
 			AMD_CG_SUPPORT_HDP_MGCG |
 			AMD_CG_SUPPORT_HDP_LS |
 			AMD_CG_SUPPORT_SDMA_MGCG |
-			AMD_CG_SUPPORT_SDMA_LS;
+			AMD_CG_SUPPORT_SDMA_LS |
+			AMD_CG_SUPPORT_VCE_MGCG;
+		/* rev0 hardware requires workarounds to support PG */
 		adev->pg_flags = 0;
+		if (adev->rev_id != 0x00) {
+			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
+				AMD_PG_SUPPORT_GFX_SMG |
+				AMD_PG_SUPPORT_GFX_PIPELINE |
+				AMD_PG_SUPPORT_UVD |
+				AMD_PG_SUPPORT_VCE;
+		}
 		adev->external_rev_id = adev->rev_id + 0x1;
 		break;
 	case CHIP_STONEY:
@@ -1267,14 +1649,24 @@ static int vi_common_early_init(void *handle)
 			AMD_CG_SUPPORT_HDP_MGCG |
 			AMD_CG_SUPPORT_HDP_LS |
 			AMD_CG_SUPPORT_SDMA_MGCG |
-			AMD_CG_SUPPORT_SDMA_LS;
-		adev->external_rev_id = adev->rev_id + 0x1;
+			AMD_CG_SUPPORT_SDMA_LS |
+			AMD_CG_SUPPORT_VCE_MGCG;
+		adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
+			AMD_PG_SUPPORT_GFX_SMG |
+			AMD_PG_SUPPORT_GFX_PIPELINE |
+			AMD_PG_SUPPORT_UVD |
+			AMD_PG_SUPPORT_VCE;
+		adev->external_rev_id = adev->rev_id + 0x61;
 		break;
 	default:
 		/* FIXME: not supported yet */
 		return -EINVAL;
 	}
 
+	/* in early init stage, vbios code won't work */
+	if (adev->asic_funcs->detect_hw_virtualization)
+		amdgpu_asic_detect_hw_virtualization(adev);
+
 	if (amdgpu_smc_load_fw && smc_enabled)
 		adev->firmware.smu_load = true;
 
@@ -1418,6 +1810,63 @@ static void vi_update_rom_medium_grain_clock_gating(struct amdgpu_device *adev,
 		WREG32_SMC(ixCGTT_ROM_CLK_CTRL0, data);
 }
 
+static int vi_common_set_clockgating_state_by_smu(void *handle,
+					   enum amd_clockgating_state state)
+{
+	uint32_t msg_id, pp_state;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	void *pp_handle = adev->powerplay.pp_handle;
+
+	if (state == AMD_CG_STATE_UNGATE)
+		pp_state = 0;
+	else
+		pp_state = PP_STATE_CG | PP_STATE_LS;
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
+		       PP_BLOCK_SYS_MC,
+		       PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+		       pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
+		       PP_BLOCK_SYS_SDMA,
+		       PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+		       pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
+		       PP_BLOCK_SYS_HDP,
+		       PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
+		       pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
+		       PP_BLOCK_SYS_BIF,
+		       PP_STATE_SUPPORT_LS,
+		       pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
+		       PP_BLOCK_SYS_BIF,
+		       PP_STATE_SUPPORT_CG,
+		       pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
+		       PP_BLOCK_SYS_DRM,
+		       PP_STATE_SUPPORT_LS,
+		       pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
+		       PP_BLOCK_SYS_ROM,
+		       PP_STATE_SUPPORT_CG,
+		       pp_state);
+	amd_set_clockgating_by_smu(pp_handle, msg_id);
+
+	return 0;
+}
+
 static int vi_common_set_clockgating_state(void *handle,
 					   enum amd_clockgating_state state)
 {
@@ -1443,6 +1892,10 @@ static int vi_common_set_clockgating_state(void *handle,
 		vi_update_hdp_light_sleep(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
 		break;
+	case CHIP_TONGA:
+	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
+		vi_common_set_clockgating_state_by_smu(adev, state);
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h
index 062ee1676480..11746f22d0c5 100644
--- a/drivers/gpu/drm/amd/amdgpu/vid.h
+++ b/drivers/gpu/drm/amd/amdgpu/vid.h
@@ -369,4 +369,45 @@
 #define VCE_CMD_IB_AUTO	0x00000005
 #define VCE_CMD_SEMAPHORE	0x00000006
 
+#define VCE_CMD_IB_VM           0x00000102
+#define VCE_CMD_WAIT_GE         0x00000106
+#define VCE_CMD_UPDATE_PTB      0x00000107
+#define VCE_CMD_FLUSH_TLB       0x00000108
+
+/* mmPA_SC_RASTER_CONFIG mask */
+#define RB_MAP_PKR0(x)				((x) << 0)
+#define RB_MAP_PKR0_MASK			(0x3 << 0)
+#define RB_MAP_PKR1(x)				((x) << 2)
+#define RB_MAP_PKR1_MASK			(0x3 << 2)
+#define RB_XSEL2(x)				((x) << 4)
+#define RB_XSEL2_MASK				(0x3 << 4)
+#define RB_XSEL					(1 << 6)
+#define RB_YSEL					(1 << 7)
+#define PKR_MAP(x)				((x) << 8)
+#define PKR_MAP_MASK				(0x3 << 8)
+#define PKR_XSEL(x)				((x) << 10)
+#define PKR_XSEL_MASK				(0x3 << 10)
+#define PKR_YSEL(x)				((x) << 12)
+#define PKR_YSEL_MASK				(0x3 << 12)
+#define SC_MAP(x)				((x) << 16)
+#define SC_MAP_MASK				(0x3 << 16)
+#define SC_XSEL(x)				((x) << 18)
+#define SC_XSEL_MASK				(0x3 << 18)
+#define SC_YSEL(x)				((x) << 20)
+#define SC_YSEL_MASK				(0x3 << 20)
+#define SE_MAP(x)				((x) << 24)
+#define SE_MAP_MASK				(0x3 << 24)
+#define SE_XSEL(x)				((x) << 26)
+#define SE_XSEL_MASK				(0x3 << 26)
+#define SE_YSEL(x)				((x) << 28)
+#define SE_YSEL_MASK				(0x3 << 28)
+
+/* mmPA_SC_RASTER_CONFIG_1 mask */
+#define SE_PAIR_MAP(x)				((x) << 0)
+#define SE_PAIR_MAP_MASK			(0x3 << 0)
+#define SE_PAIR_XSEL(x)				((x) << 2)
+#define SE_PAIR_XSEL_MASK			(0x3 << 2)
+#define SE_PAIR_YSEL(x)				((x) << 4)
+#define SE_PAIR_YSEL_MASK			(0x3 << 4)
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
index e621eba63126..453c5d66e5c3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
@@ -142,13 +142,15 @@ int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
 
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
-	pr_debug("mapping doorbell page:\n");
-	pr_debug("     target user address == 0x%08llX\n",
-			(unsigned long long) vma->vm_start);
-	pr_debug("     physical address    == 0x%08llX\n", address);
-	pr_debug("     vm_flags            == 0x%04lX\n", vma->vm_flags);
-	pr_debug("     size                == 0x%04lX\n",
-			 doorbell_process_allocation());
+	pr_debug("kfd: mapping doorbell page in %s\n"
+		 "     target user address == 0x%08llX\n"
+		 "     physical address    == 0x%08llX\n"
+		 "     vm_flags            == 0x%04lX\n"
+		 "     size                == 0x%04lX\n",
+		 __func__,
+		 (unsigned long long) vma->vm_start, address, vma->vm_flags,
+		 doorbell_process_allocation());
+
 
 	return io_remap_pfn_range(vma,
 				vma->vm_start,
@@ -184,7 +186,7 @@ u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
 							sizeof(u32)) + inx;
 
 	pr_debug("kfd: get kernel queue doorbell\n"
-			 "     doorbell offset   == 0x%08d\n"
+			 "     doorbell offset   == 0x%08X\n"
 			 "     kernel address    == 0x%08lX\n",
 		*doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx));
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
index 9beae87aadd5..d135cd002a95 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
@@ -47,6 +47,9 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 	pr_debug("amdkfd: In func %s initializing queue type %d size %d\n",
 			__func__, KFD_QUEUE_TYPE_HIQ, queue_size);
 
+	memset(&prop, 0, sizeof(prop));
+	memset(&nop, 0, sizeof(nop));
+
 	nop.opcode = IT_NOP;
 	nop.type = PM4_TYPE_3;
 	nop.u32all |= PM4_COUNT_ZERO;
@@ -121,7 +124,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 	prop.eop_ring_buffer_address = kq->eop_gpu_addr;
 	prop.eop_ring_buffer_size = PAGE_SIZE;
 
-	if (init_queue(&kq->queue, prop) != 0)
+	if (init_queue(&kq->queue, &prop) != 0)
 		goto err_init_queue;
 
 	kq->queue->device = dev;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 80113c335966..4750cabe4252 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -619,7 +619,7 @@ int kfd_init_apertures(struct kfd_process *process);
 /* Queue Context Management */
 struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd);
 
-int init_queue(struct queue **q, struct queue_properties properties);
+int init_queue(struct queue **q, const struct queue_properties *properties);
 void uninit_queue(struct queue *q);
 void print_queue_properties(struct queue_properties *q);
 void print_queue(struct queue *q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 4f3849ac8c07..ef7c8de7060e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -404,58 +404,47 @@ void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
 {
 	struct kfd_process *p;
 	struct kfd_process_device *pdd;
-	int idx, i;
 
 	BUG_ON(dev == NULL);
 
-	idx = srcu_read_lock(&kfd_processes_srcu);
-
 	/*
 	 * Look for the process that matches the pasid. If there is no such
 	 * process, we either released it in amdkfd's own notifier, or there
 	 * is a bug. Unfortunately, there is no way to tell...
 	 */
-	hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
-		if (p->pasid == pasid) {
-
-			srcu_read_unlock(&kfd_processes_srcu, idx);
-
-			pr_debug("Unbinding process %d from IOMMU\n", pasid);
+	p = kfd_lookup_process_by_pasid(pasid);
+	if (!p)
+		return;
 
-			mutex_lock(&p->mutex);
-
-			if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
-				kfd_dbgmgr_destroy(dev->dbgmgr);
-
-			pqm_uninit(&p->pqm);
+	pr_debug("Unbinding process %d from IOMMU\n", pasid);
 
-			pdd = kfd_get_process_device_data(dev, p);
+	if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
+		kfd_dbgmgr_destroy(dev->dbgmgr);
 
-			if (!pdd) {
-				mutex_unlock(&p->mutex);
-				return;
-			}
+	pqm_uninit(&p->pqm);
 
-			if (pdd->reset_wavefronts) {
-				dbgdev_wave_reset_wavefronts(pdd->dev, p);
-				pdd->reset_wavefronts = false;
-			}
+	pdd = kfd_get_process_device_data(dev, p);
 
-			/*
-			 * Just mark pdd as unbound, because we still need it
-			 * to call amd_iommu_unbind_pasid() in when the
-			 * process exits.
-			 * We don't call amd_iommu_unbind_pasid() here
-			 * because the IOMMU called us.
-			 */
-			pdd->bound = false;
+	if (!pdd) {
+		mutex_unlock(&p->mutex);
+		return;
+	}
 
-			mutex_unlock(&p->mutex);
+	if (pdd->reset_wavefronts) {
+		dbgdev_wave_reset_wavefronts(pdd->dev, p);
+		pdd->reset_wavefronts = false;
+	}
 
-			return;
-		}
+	/*
+	 * Just mark pdd as unbound, because we still need it
+	 * to call amd_iommu_unbind_pasid() in when the
+	 * process exits.
+	 * We don't call amd_iommu_unbind_pasid() here
+	 * because the IOMMU called us.
+	 */
+	pdd->bound = false;
 
-	srcu_read_unlock(&kfd_processes_srcu, idx);
+	mutex_unlock(&p->mutex);
 }
 
 struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 7b69070f7ecc..e1fb40b84c72 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -129,7 +129,7 @@ static int create_cp_queue(struct process_queue_manager *pqm,
 	q_properties->vmid = 0;
 	q_properties->queue_id = qid;
 
-	retval = init_queue(q, *q_properties);
+	retval = init_queue(q, q_properties);
 	if (retval != 0)
 		goto err_init_queue;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
index 9a0c90b0702e..0ab197077f2d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
@@ -63,7 +63,7 @@ void print_queue(struct queue *q)
 	pr_debug("Queue Device Address: 0x%p\n", q->device);
 }
 
-int init_queue(struct queue **q, struct queue_properties properties)
+int init_queue(struct queue **q, const struct queue_properties *properties)
 {
 	struct queue *tmp;
 
@@ -73,7 +73,7 @@ int init_queue(struct queue **q, struct queue_properties properties)
 	if (!tmp)
 		return -ENOMEM;
 
-	memcpy(&tmp->properties, &properties, sizeof(struct queue_properties));
+	memcpy(&tmp->properties, properties, sizeof(struct queue_properties));
 
 	*q = tmp;
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 884c96f50c3d..1e5064749959 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1090,19 +1090,21 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
 {
 	uint32_t hashout;
 	uint32_t buf[7];
+	uint64_t local_mem_size;
 	int i;
 
 	if (!gpu)
 		return 0;
 
+	local_mem_size = gpu->kfd2kgd->get_vmem_size(gpu->kgd);
+
 	buf[0] = gpu->pdev->devfn;
 	buf[1] = gpu->pdev->subsystem_vendor;
 	buf[2] = gpu->pdev->subsystem_device;
 	buf[3] = gpu->pdev->device;
 	buf[4] = gpu->pdev->bus->number;
-	buf[5] = (uint32_t)(gpu->kfd2kgd->get_vmem_size(gpu->kgd)
-			& 0xffffffff);
-	buf[6] = (uint32_t)(gpu->kfd2kgd->get_vmem_size(gpu->kgd) >> 32);
+	buf[5] = lower_32_bits(local_mem_size);
+	buf[6] = upper_32_bits(local_mem_size);
 
 	for (i = 0, hashout = 0; i < 7; i++)
 		hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH);
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index a74a0d2ff1ca..c934b78c9e2f 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -29,7 +29,12 @@
  * Supported ASIC types
  */
 enum amd_asic_type {
-	CHIP_BONAIRE = 0,
+	CHIP_TAHITI = 0,
+	CHIP_PITCAIRN,
+	CHIP_VERDE,
+	CHIP_OLAND,
+	CHIP_HAINAN,
+	CHIP_BONAIRE,
 	CHIP_KAVERI,
 	CHIP_KABINI,
 	CHIP_HAWAII,
@@ -159,8 +164,14 @@ struct amd_ip_funcs {
 	bool (*is_idle)(void *handle);
 	/* poll for idle */
 	int (*wait_for_idle)(void *handle);
+	/* check soft reset the IP block */
+	int (*check_soft_reset)(void *handle);
+	/* pre soft reset the IP block */
+	int (*pre_soft_reset)(void *handle);
 	/* soft reset the IP block */
 	int (*soft_reset)(void *handle);
+	/* post soft reset the IP block */
+	int (*post_soft_reset)(void *handle);
 	/* enable/disable cg for the IP block */
 	int (*set_clockgating_state)(void *handle,
 				     enum amd_clockgating_state state);
diff --git a/drivers/gpu/drm/amd/include/asic_reg/si/clearstate_si.h b/drivers/gpu/drm/amd/include/asic_reg/si/clearstate_si.h
new file mode 100644
index 000000000000..66e39cdb5cb0
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/si/clearstate_si.h
@@ -0,0 +1,941 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+static const u32 si_SECT_CONTEXT_def_1[] =
+{
+    0x00000000, // DB_RENDER_CONTROL
+    0x00000000, // DB_COUNT_CONTROL
+    0x00000000, // DB_DEPTH_VIEW
+    0x00000000, // DB_RENDER_OVERRIDE
+    0x00000000, // DB_RENDER_OVERRIDE2
+    0x00000000, // DB_HTILE_DATA_BASE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // DB_DEPTH_BOUNDS_MIN
+    0x00000000, // DB_DEPTH_BOUNDS_MAX
+    0x00000000, // DB_STENCIL_CLEAR
+    0x00000000, // DB_DEPTH_CLEAR
+    0x00000000, // PA_SC_SCREEN_SCISSOR_TL
+    0x40004000, // PA_SC_SCREEN_SCISSOR_BR
+    0, // HOLE
+    0x00000000, // DB_DEPTH_INFO
+    0x00000000, // DB_Z_INFO
+    0x00000000, // DB_STENCIL_INFO
+    0x00000000, // DB_Z_READ_BASE
+    0x00000000, // DB_STENCIL_READ_BASE
+    0x00000000, // DB_Z_WRITE_BASE
+    0x00000000, // DB_STENCIL_WRITE_BASE
+    0x00000000, // DB_DEPTH_SIZE
+    0x00000000, // DB_DEPTH_SLICE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // TA_BC_BASE_ADDR
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // COHER_DEST_BASE_2
+    0x00000000, // COHER_DEST_BASE_3
+    0x00000000, // PA_SC_WINDOW_OFFSET
+    0x80000000, // PA_SC_WINDOW_SCISSOR_TL
+    0x40004000, // PA_SC_WINDOW_SCISSOR_BR
+    0x0000ffff, // PA_SC_CLIPRECT_RULE
+    0x00000000, // PA_SC_CLIPRECT_0_TL
+    0x40004000, // PA_SC_CLIPRECT_0_BR
+    0x00000000, // PA_SC_CLIPRECT_1_TL
+    0x40004000, // PA_SC_CLIPRECT_1_BR
+    0x00000000, // PA_SC_CLIPRECT_2_TL
+    0x40004000, // PA_SC_CLIPRECT_2_BR
+    0x00000000, // PA_SC_CLIPRECT_3_TL
+    0x40004000, // PA_SC_CLIPRECT_3_BR
+    0xaa99aaaa, // PA_SC_EDGERULE
+    0x00000000, // PA_SU_HARDWARE_SCREEN_OFFSET
+    0xffffffff, // CB_TARGET_MASK
+    0xffffffff, // CB_SHADER_MASK
+    0x80000000, // PA_SC_GENERIC_SCISSOR_TL
+    0x40004000, // PA_SC_GENERIC_SCISSOR_BR
+    0x00000000, // COHER_DEST_BASE_0
+    0x00000000, // COHER_DEST_BASE_1
+    0x80000000, // PA_SC_VPORT_SCISSOR_0_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_0_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_1_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_1_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_2_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_2_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_3_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_3_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_4_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_4_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_5_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_5_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_6_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_6_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_7_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_7_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_8_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_8_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_9_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_9_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_10_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_10_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_11_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_11_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_12_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_12_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_13_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_13_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_14_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_14_BR
+    0x80000000, // PA_SC_VPORT_SCISSOR_15_TL
+    0x40004000, // PA_SC_VPORT_SCISSOR_15_BR
+    0x00000000, // PA_SC_VPORT_ZMIN_0
+    0x3f800000, // PA_SC_VPORT_ZMAX_0
+    0x00000000, // PA_SC_VPORT_ZMIN_1
+    0x3f800000, // PA_SC_VPORT_ZMAX_1
+    0x00000000, // PA_SC_VPORT_ZMIN_2
+    0x3f800000, // PA_SC_VPORT_ZMAX_2
+    0x00000000, // PA_SC_VPORT_ZMIN_3
+    0x3f800000, // PA_SC_VPORT_ZMAX_3
+    0x00000000, // PA_SC_VPORT_ZMIN_4
+    0x3f800000, // PA_SC_VPORT_ZMAX_4
+    0x00000000, // PA_SC_VPORT_ZMIN_5
+    0x3f800000, // PA_SC_VPORT_ZMAX_5
+    0x00000000, // PA_SC_VPORT_ZMIN_6
+    0x3f800000, // PA_SC_VPORT_ZMAX_6
+    0x00000000, // PA_SC_VPORT_ZMIN_7
+    0x3f800000, // PA_SC_VPORT_ZMAX_7
+    0x00000000, // PA_SC_VPORT_ZMIN_8
+    0x3f800000, // PA_SC_VPORT_ZMAX_8
+    0x00000000, // PA_SC_VPORT_ZMIN_9
+    0x3f800000, // PA_SC_VPORT_ZMAX_9
+    0x00000000, // PA_SC_VPORT_ZMIN_10
+    0x3f800000, // PA_SC_VPORT_ZMAX_10
+    0x00000000, // PA_SC_VPORT_ZMIN_11
+    0x3f800000, // PA_SC_VPORT_ZMAX_11
+    0x00000000, // PA_SC_VPORT_ZMIN_12
+    0x3f800000, // PA_SC_VPORT_ZMAX_12
+    0x00000000, // PA_SC_VPORT_ZMIN_13
+    0x3f800000, // PA_SC_VPORT_ZMAX_13
+    0x00000000, // PA_SC_VPORT_ZMIN_14
+    0x3f800000, // PA_SC_VPORT_ZMAX_14
+    0x00000000, // PA_SC_VPORT_ZMIN_15
+    0x3f800000, // PA_SC_VPORT_ZMAX_15
+};
+static const u32 si_SECT_CONTEXT_def_2[] =
+{
+    0x00000000, // CP_PERFMON_CNTX_CNTL
+    0x00000000, // CP_RINGID
+    0x00000000, // CP_VMID
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0xffffffff, // VGT_MAX_VTX_INDX
+    0x00000000, // VGT_MIN_VTX_INDX
+    0x00000000, // VGT_INDX_OFFSET
+    0x00000000, // VGT_MULTI_PRIM_IB_RESET_INDX
+    0, // HOLE
+    0x00000000, // CB_BLEND_RED
+    0x00000000, // CB_BLEND_GREEN
+    0x00000000, // CB_BLEND_BLUE
+    0x00000000, // CB_BLEND_ALPHA
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // DB_STENCIL_CONTROL
+    0x00000000, // DB_STENCILREFMASK
+    0x00000000, // DB_STENCILREFMASK_BF
+    0, // HOLE
+    0x00000000, // PA_CL_VPORT_XSCALE
+    0x00000000, // PA_CL_VPORT_XOFFSET
+    0x00000000, // PA_CL_VPORT_YSCALE
+    0x00000000, // PA_CL_VPORT_YOFFSET
+    0x00000000, // PA_CL_VPORT_ZSCALE
+    0x00000000, // PA_CL_VPORT_ZOFFSET
+    0x00000000, // PA_CL_VPORT_XSCALE_1
+    0x00000000, // PA_CL_VPORT_XOFFSET_1
+    0x00000000, // PA_CL_VPORT_YSCALE_1
+    0x00000000, // PA_CL_VPORT_YOFFSET_1
+    0x00000000, // PA_CL_VPORT_ZSCALE_1
+    0x00000000, // PA_CL_VPORT_ZOFFSET_1
+    0x00000000, // PA_CL_VPORT_XSCALE_2
+    0x00000000, // PA_CL_VPORT_XOFFSET_2
+    0x00000000, // PA_CL_VPORT_YSCALE_2
+    0x00000000, // PA_CL_VPORT_YOFFSET_2
+    0x00000000, // PA_CL_VPORT_ZSCALE_2
+    0x00000000, // PA_CL_VPORT_ZOFFSET_2
+    0x00000000, // PA_CL_VPORT_XSCALE_3
+    0x00000000, // PA_CL_VPORT_XOFFSET_3
+    0x00000000, // PA_CL_VPORT_YSCALE_3
+    0x00000000, // PA_CL_VPORT_YOFFSET_3
+    0x00000000, // PA_CL_VPORT_ZSCALE_3
+    0x00000000, // PA_CL_VPORT_ZOFFSET_3
+    0x00000000, // PA_CL_VPORT_XSCALE_4
+    0x00000000, // PA_CL_VPORT_XOFFSET_4
+    0x00000000, // PA_CL_VPORT_YSCALE_4
+    0x00000000, // PA_CL_VPORT_YOFFSET_4
+    0x00000000, // PA_CL_VPORT_ZSCALE_4
+    0x00000000, // PA_CL_VPORT_ZOFFSET_4
+    0x00000000, // PA_CL_VPORT_XSCALE_5
+    0x00000000, // PA_CL_VPORT_XOFFSET_5
+    0x00000000, // PA_CL_VPORT_YSCALE_5
+    0x00000000, // PA_CL_VPORT_YOFFSET_5
+    0x00000000, // PA_CL_VPORT_ZSCALE_5
+    0x00000000, // PA_CL_VPORT_ZOFFSET_5
+    0x00000000, // PA_CL_VPORT_XSCALE_6
+    0x00000000, // PA_CL_VPORT_XOFFSET_6
+    0x00000000, // PA_CL_VPORT_YSCALE_6
+    0x00000000, // PA_CL_VPORT_YOFFSET_6
+    0x00000000, // PA_CL_VPORT_ZSCALE_6
+    0x00000000, // PA_CL_VPORT_ZOFFSET_6
+    0x00000000, // PA_CL_VPORT_XSCALE_7
+    0x00000000, // PA_CL_VPORT_XOFFSET_7
+    0x00000000, // PA_CL_VPORT_YSCALE_7
+    0x00000000, // PA_CL_VPORT_YOFFSET_7
+    0x00000000, // PA_CL_VPORT_ZSCALE_7
+    0x00000000, // PA_CL_VPORT_ZOFFSET_7
+    0x00000000, // PA_CL_VPORT_XSCALE_8
+    0x00000000, // PA_CL_VPORT_XOFFSET_8
+    0x00000000, // PA_CL_VPORT_YSCALE_8
+    0x00000000, // PA_CL_VPORT_YOFFSET_8
+    0x00000000, // PA_CL_VPORT_ZSCALE_8
+    0x00000000, // PA_CL_VPORT_ZOFFSET_8
+    0x00000000, // PA_CL_VPORT_XSCALE_9
+    0x00000000, // PA_CL_VPORT_XOFFSET_9
+    0x00000000, // PA_CL_VPORT_YSCALE_9
+    0x00000000, // PA_CL_VPORT_YOFFSET_9
+    0x00000000, // PA_CL_VPORT_ZSCALE_9
+    0x00000000, // PA_CL_VPORT_ZOFFSET_9
+    0x00000000, // PA_CL_VPORT_XSCALE_10
+    0x00000000, // PA_CL_VPORT_XOFFSET_10
+    0x00000000, // PA_CL_VPORT_YSCALE_10
+    0x00000000, // PA_CL_VPORT_YOFFSET_10
+    0x00000000, // PA_CL_VPORT_ZSCALE_10
+    0x00000000, // PA_CL_VPORT_ZOFFSET_10
+    0x00000000, // PA_CL_VPORT_XSCALE_11
+    0x00000000, // PA_CL_VPORT_XOFFSET_11
+    0x00000000, // PA_CL_VPORT_YSCALE_11
+    0x00000000, // PA_CL_VPORT_YOFFSET_11
+    0x00000000, // PA_CL_VPORT_ZSCALE_11
+    0x00000000, // PA_CL_VPORT_ZOFFSET_11
+    0x00000000, // PA_CL_VPORT_XSCALE_12
+    0x00000000, // PA_CL_VPORT_XOFFSET_12
+    0x00000000, // PA_CL_VPORT_YSCALE_12
+    0x00000000, // PA_CL_VPORT_YOFFSET_12
+    0x00000000, // PA_CL_VPORT_ZSCALE_12
+    0x00000000, // PA_CL_VPORT_ZOFFSET_12
+    0x00000000, // PA_CL_VPORT_XSCALE_13
+    0x00000000, // PA_CL_VPORT_XOFFSET_13
+    0x00000000, // PA_CL_VPORT_YSCALE_13
+    0x00000000, // PA_CL_VPORT_YOFFSET_13
+    0x00000000, // PA_CL_VPORT_ZSCALE_13
+    0x00000000, // PA_CL_VPORT_ZOFFSET_13
+    0x00000000, // PA_CL_VPORT_XSCALE_14
+    0x00000000, // PA_CL_VPORT_XOFFSET_14
+    0x00000000, // PA_CL_VPORT_YSCALE_14
+    0x00000000, // PA_CL_VPORT_YOFFSET_14
+    0x00000000, // PA_CL_VPORT_ZSCALE_14
+    0x00000000, // PA_CL_VPORT_ZOFFSET_14
+    0x00000000, // PA_CL_VPORT_XSCALE_15
+    0x00000000, // PA_CL_VPORT_XOFFSET_15
+    0x00000000, // PA_CL_VPORT_YSCALE_15
+    0x00000000, // PA_CL_VPORT_YOFFSET_15
+    0x00000000, // PA_CL_VPORT_ZSCALE_15
+    0x00000000, // PA_CL_VPORT_ZOFFSET_15
+    0x00000000, // PA_CL_UCP_0_X
+    0x00000000, // PA_CL_UCP_0_Y
+    0x00000000, // PA_CL_UCP_0_Z
+    0x00000000, // PA_CL_UCP_0_W
+    0x00000000, // PA_CL_UCP_1_X
+    0x00000000, // PA_CL_UCP_1_Y
+    0x00000000, // PA_CL_UCP_1_Z
+    0x00000000, // PA_CL_UCP_1_W
+    0x00000000, // PA_CL_UCP_2_X
+    0x00000000, // PA_CL_UCP_2_Y
+    0x00000000, // PA_CL_UCP_2_Z
+    0x00000000, // PA_CL_UCP_2_W
+    0x00000000, // PA_CL_UCP_3_X
+    0x00000000, // PA_CL_UCP_3_Y
+    0x00000000, // PA_CL_UCP_3_Z
+    0x00000000, // PA_CL_UCP_3_W
+    0x00000000, // PA_CL_UCP_4_X
+    0x00000000, // PA_CL_UCP_4_Y
+    0x00000000, // PA_CL_UCP_4_Z
+    0x00000000, // PA_CL_UCP_4_W
+    0x00000000, // PA_CL_UCP_5_X
+    0x00000000, // PA_CL_UCP_5_Y
+    0x00000000, // PA_CL_UCP_5_Z
+    0x00000000, // PA_CL_UCP_5_W
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SPI_PS_INPUT_CNTL_0
+    0x00000000, // SPI_PS_INPUT_CNTL_1
+    0x00000000, // SPI_PS_INPUT_CNTL_2
+    0x00000000, // SPI_PS_INPUT_CNTL_3
+    0x00000000, // SPI_PS_INPUT_CNTL_4
+    0x00000000, // SPI_PS_INPUT_CNTL_5
+    0x00000000, // SPI_PS_INPUT_CNTL_6
+    0x00000000, // SPI_PS_INPUT_CNTL_7
+    0x00000000, // SPI_PS_INPUT_CNTL_8
+    0x00000000, // SPI_PS_INPUT_CNTL_9
+    0x00000000, // SPI_PS_INPUT_CNTL_10
+    0x00000000, // SPI_PS_INPUT_CNTL_11
+    0x00000000, // SPI_PS_INPUT_CNTL_12
+    0x00000000, // SPI_PS_INPUT_CNTL_13
+    0x00000000, // SPI_PS_INPUT_CNTL_14
+    0x00000000, // SPI_PS_INPUT_CNTL_15
+    0x00000000, // SPI_PS_INPUT_CNTL_16
+    0x00000000, // SPI_PS_INPUT_CNTL_17
+    0x00000000, // SPI_PS_INPUT_CNTL_18
+    0x00000000, // SPI_PS_INPUT_CNTL_19
+    0x00000000, // SPI_PS_INPUT_CNTL_20
+    0x00000000, // SPI_PS_INPUT_CNTL_21
+    0x00000000, // SPI_PS_INPUT_CNTL_22
+    0x00000000, // SPI_PS_INPUT_CNTL_23
+    0x00000000, // SPI_PS_INPUT_CNTL_24
+    0x00000000, // SPI_PS_INPUT_CNTL_25
+    0x00000000, // SPI_PS_INPUT_CNTL_26
+    0x00000000, // SPI_PS_INPUT_CNTL_27
+    0x00000000, // SPI_PS_INPUT_CNTL_28
+    0x00000000, // SPI_PS_INPUT_CNTL_29
+    0x00000000, // SPI_PS_INPUT_CNTL_30
+    0x00000000, // SPI_PS_INPUT_CNTL_31
+    0x00000000, // SPI_VS_OUT_CONFIG
+    0, // HOLE
+    0x00000000, // SPI_PS_INPUT_ENA
+    0x00000000, // SPI_PS_INPUT_ADDR
+    0x00000000, // SPI_INTERP_CONTROL_0
+    0x00000002, // SPI_PS_IN_CONTROL
+    0, // HOLE
+    0x00000000, // SPI_BARYC_CNTL
+    0, // HOLE
+    0x00000000, // SPI_TMPRING_SIZE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // SPI_WAVE_MGMT_1
+    0x00000000, // SPI_WAVE_MGMT_2
+    0x00000000, // SPI_SHADER_POS_FORMAT
+    0x00000000, // SPI_SHADER_Z_FORMAT
+    0x00000000, // SPI_SHADER_COL_FORMAT
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_BLEND0_CONTROL
+    0x00000000, // CB_BLEND1_CONTROL
+    0x00000000, // CB_BLEND2_CONTROL
+    0x00000000, // CB_BLEND3_CONTROL
+    0x00000000, // CB_BLEND4_CONTROL
+    0x00000000, // CB_BLEND5_CONTROL
+    0x00000000, // CB_BLEND6_CONTROL
+    0x00000000, // CB_BLEND7_CONTROL
+};
+static const u32 si_SECT_CONTEXT_def_3[] =
+{
+    0x00000000, // PA_CL_POINT_X_RAD
+    0x00000000, // PA_CL_POINT_Y_RAD
+    0x00000000, // PA_CL_POINT_SIZE
+    0x00000000, // PA_CL_POINT_CULL_RAD
+    0x00000000, // VGT_DMA_BASE_HI
+    0x00000000, // VGT_DMA_BASE
+};
+static const u32 si_SECT_CONTEXT_def_4[] =
+{
+    0x00000000, // DB_DEPTH_CONTROL
+    0x00000000, // DB_EQAA
+    0x00000000, // CB_COLOR_CONTROL
+    0x00000000, // DB_SHADER_CONTROL
+    0x00090000, // PA_CL_CLIP_CNTL
+    0x00000004, // PA_SU_SC_MODE_CNTL
+    0x00000000, // PA_CL_VTE_CNTL
+    0x00000000, // PA_CL_VS_OUT_CNTL
+    0x00000000, // PA_CL_NANINF_CNTL
+    0x00000000, // PA_SU_LINE_STIPPLE_CNTL
+    0x00000000, // PA_SU_LINE_STIPPLE_SCALE
+    0x00000000, // PA_SU_PRIM_FILTER_CNTL
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // PA_SU_POINT_SIZE
+    0x00000000, // PA_SU_POINT_MINMAX
+    0x00000000, // PA_SU_LINE_CNTL
+    0x00000000, // PA_SC_LINE_STIPPLE
+    0x00000000, // VGT_OUTPUT_PATH_CNTL
+    0x00000000, // VGT_HOS_CNTL
+    0x00000000, // VGT_HOS_MAX_TESS_LEVEL
+    0x00000000, // VGT_HOS_MIN_TESS_LEVEL
+    0x00000000, // VGT_HOS_REUSE_DEPTH
+    0x00000000, // VGT_GROUP_PRIM_TYPE
+    0x00000000, // VGT_GROUP_FIRST_DECR
+    0x00000000, // VGT_GROUP_DECR
+    0x00000000, // VGT_GROUP_VECT_0_CNTL
+    0x00000000, // VGT_GROUP_VECT_1_CNTL
+    0x00000000, // VGT_GROUP_VECT_0_FMT_CNTL
+    0x00000000, // VGT_GROUP_VECT_1_FMT_CNTL
+    0x00000000, // VGT_GS_MODE
+    0, // HOLE
+    0x00000000, // PA_SC_MODE_CNTL_0
+    0x00000000, // PA_SC_MODE_CNTL_1
+    0x00000000, // VGT_ENHANCE
+    0x00000100, // VGT_GS_PER_ES
+    0x00000080, // VGT_ES_PER_GS
+    0x00000002, // VGT_GS_PER_VS
+    0x00000000, // VGT_GSVS_RING_OFFSET_1
+    0x00000000, // VGT_GSVS_RING_OFFSET_2
+    0x00000000, // VGT_GSVS_RING_OFFSET_3
+    0x00000000, // VGT_GS_OUT_PRIM_TYPE
+    0x00000000, // IA_ENHANCE
+};
+static const u32 si_SECT_CONTEXT_def_5[] =
+{
+    0x00000000, // VGT_PRIMITIVEID_EN
+};
+static const u32 si_SECT_CONTEXT_def_6[] =
+{
+    0x00000000, // VGT_PRIMITIVEID_RESET
+};
+static const u32 si_SECT_CONTEXT_def_7[] =
+{
+    0x00000000, // VGT_MULTI_PRIM_IB_RESET_EN
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_INSTANCE_STEP_RATE_0
+    0x00000000, // VGT_INSTANCE_STEP_RATE_1
+    0x000000ff, // IA_MULTI_VGT_PARAM
+    0x00000000, // VGT_ESGS_RING_ITEMSIZE
+    0x00000000, // VGT_GSVS_RING_ITEMSIZE
+    0x00000000, // VGT_REUSE_OFF
+    0x00000000, // VGT_VTX_CNT_EN
+    0x00000000, // DB_HTILE_SURFACE
+    0x00000000, // DB_SRESULTS_COMPARE_STATE0
+    0x00000000, // DB_SRESULTS_COMPARE_STATE1
+    0x00000000, // DB_PRELOAD_CONTROL
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_0
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_0
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_0
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_1
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_1
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_1
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_2
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_2
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_2
+    0x00000000, // VGT_STRMOUT_BUFFER_SIZE_3
+    0x00000000, // VGT_STRMOUT_VTX_STRIDE_3
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_BUFFER_OFFSET_3
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_OFFSET
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE
+    0x00000000, // VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE
+    0, // HOLE
+    0x00000000, // VGT_GS_MAX_VERT_OUT
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // VGT_SHADER_STAGES_EN
+    0x00000000, // VGT_LS_HS_CONFIG
+    0x00000000, // VGT_GS_VERT_ITEMSIZE
+    0x00000000, // VGT_GS_VERT_ITEMSIZE_1
+    0x00000000, // VGT_GS_VERT_ITEMSIZE_2
+    0x00000000, // VGT_GS_VERT_ITEMSIZE_3
+    0x00000000, // VGT_TF_PARAM
+    0x00000000, // DB_ALPHA_TO_MASK
+    0, // HOLE
+    0x00000000, // PA_SU_POLY_OFFSET_DB_FMT_CNTL
+    0x00000000, // PA_SU_POLY_OFFSET_CLAMP
+    0x00000000, // PA_SU_POLY_OFFSET_FRONT_SCALE
+    0x00000000, // PA_SU_POLY_OFFSET_FRONT_OFFSET
+    0x00000000, // PA_SU_POLY_OFFSET_BACK_SCALE
+    0x00000000, // PA_SU_POLY_OFFSET_BACK_OFFSET
+    0x00000000, // VGT_GS_INSTANCE_CNT
+    0x00000000, // VGT_STRMOUT_CONFIG
+    0x00000000, // VGT_STRMOUT_BUFFER_CONFIG
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // PA_SC_CENTROID_PRIORITY_0
+    0x00000000, // PA_SC_CENTROID_PRIORITY_1
+    0x00001000, // PA_SC_LINE_CNTL
+    0x00000000, // PA_SC_AA_CONFIG
+    0x00000005, // PA_SU_VTX_CNTL
+    0x3f800000, // PA_CL_GB_VERT_CLIP_ADJ
+    0x3f800000, // PA_CL_GB_VERT_DISC_ADJ
+    0x3f800000, // PA_CL_GB_HORZ_CLIP_ADJ
+    0x3f800000, // PA_CL_GB_HORZ_DISC_ADJ
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2
+    0x00000000, // PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3
+    0xffffffff, // PA_SC_AA_MASK_X0Y0_X1Y0
+    0xffffffff, // PA_SC_AA_MASK_X0Y1_X1Y1
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0, // HOLE
+    0x0000000e, // VGT_VERTEX_REUSE_BLOCK_CNTL
+    0x00000010, // VGT_OUT_DEALLOC_CNTL
+    0x00000000, // CB_COLOR0_BASE
+    0x00000000, // CB_COLOR0_PITCH
+    0x00000000, // CB_COLOR0_SLICE
+    0x00000000, // CB_COLOR0_VIEW
+    0x00000000, // CB_COLOR0_INFO
+    0x00000000, // CB_COLOR0_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR0_CMASK
+    0x00000000, // CB_COLOR0_CMASK_SLICE
+    0x00000000, // CB_COLOR0_FMASK
+    0x00000000, // CB_COLOR0_FMASK_SLICE
+    0x00000000, // CB_COLOR0_CLEAR_WORD0
+    0x00000000, // CB_COLOR0_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR1_BASE
+    0x00000000, // CB_COLOR1_PITCH
+    0x00000000, // CB_COLOR1_SLICE
+    0x00000000, // CB_COLOR1_VIEW
+    0x00000000, // CB_COLOR1_INFO
+    0x00000000, // CB_COLOR1_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR1_CMASK
+    0x00000000, // CB_COLOR1_CMASK_SLICE
+    0x00000000, // CB_COLOR1_FMASK
+    0x00000000, // CB_COLOR1_FMASK_SLICE
+    0x00000000, // CB_COLOR1_CLEAR_WORD0
+    0x00000000, // CB_COLOR1_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR2_BASE
+    0x00000000, // CB_COLOR2_PITCH
+    0x00000000, // CB_COLOR2_SLICE
+    0x00000000, // CB_COLOR2_VIEW
+    0x00000000, // CB_COLOR2_INFO
+    0x00000000, // CB_COLOR2_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR2_CMASK
+    0x00000000, // CB_COLOR2_CMASK_SLICE
+    0x00000000, // CB_COLOR2_FMASK
+    0x00000000, // CB_COLOR2_FMASK_SLICE
+    0x00000000, // CB_COLOR2_CLEAR_WORD0
+    0x00000000, // CB_COLOR2_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR3_BASE
+    0x00000000, // CB_COLOR3_PITCH
+    0x00000000, // CB_COLOR3_SLICE
+    0x00000000, // CB_COLOR3_VIEW
+    0x00000000, // CB_COLOR3_INFO
+    0x00000000, // CB_COLOR3_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR3_CMASK
+    0x00000000, // CB_COLOR3_CMASK_SLICE
+    0x00000000, // CB_COLOR3_FMASK
+    0x00000000, // CB_COLOR3_FMASK_SLICE
+    0x00000000, // CB_COLOR3_CLEAR_WORD0
+    0x00000000, // CB_COLOR3_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR4_BASE
+    0x00000000, // CB_COLOR4_PITCH
+    0x00000000, // CB_COLOR4_SLICE
+    0x00000000, // CB_COLOR4_VIEW
+    0x00000000, // CB_COLOR4_INFO
+    0x00000000, // CB_COLOR4_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR4_CMASK
+    0x00000000, // CB_COLOR4_CMASK_SLICE
+    0x00000000, // CB_COLOR4_FMASK
+    0x00000000, // CB_COLOR4_FMASK_SLICE
+    0x00000000, // CB_COLOR4_CLEAR_WORD0
+    0x00000000, // CB_COLOR4_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR5_BASE
+    0x00000000, // CB_COLOR5_PITCH
+    0x00000000, // CB_COLOR5_SLICE
+    0x00000000, // CB_COLOR5_VIEW
+    0x00000000, // CB_COLOR5_INFO
+    0x00000000, // CB_COLOR5_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR5_CMASK
+    0x00000000, // CB_COLOR5_CMASK_SLICE
+    0x00000000, // CB_COLOR5_FMASK
+    0x00000000, // CB_COLOR5_FMASK_SLICE
+    0x00000000, // CB_COLOR5_CLEAR_WORD0
+    0x00000000, // CB_COLOR5_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR6_BASE
+    0x00000000, // CB_COLOR6_PITCH
+    0x00000000, // CB_COLOR6_SLICE
+    0x00000000, // CB_COLOR6_VIEW
+    0x00000000, // CB_COLOR6_INFO
+    0x00000000, // CB_COLOR6_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR6_CMASK
+    0x00000000, // CB_COLOR6_CMASK_SLICE
+    0x00000000, // CB_COLOR6_FMASK
+    0x00000000, // CB_COLOR6_FMASK_SLICE
+    0x00000000, // CB_COLOR6_CLEAR_WORD0
+    0x00000000, // CB_COLOR6_CLEAR_WORD1
+    0, // HOLE
+    0, // HOLE
+    0x00000000, // CB_COLOR7_BASE
+    0x00000000, // CB_COLOR7_PITCH
+    0x00000000, // CB_COLOR7_SLICE
+    0x00000000, // CB_COLOR7_VIEW
+    0x00000000, // CB_COLOR7_INFO
+    0x00000000, // CB_COLOR7_ATTRIB
+    0, // HOLE
+    0x00000000, // CB_COLOR7_CMASK
+    0x00000000, // CB_COLOR7_CMASK_SLICE
+    0x00000000, // CB_COLOR7_FMASK
+    0x00000000, // CB_COLOR7_FMASK_SLICE
+    0x00000000, // CB_COLOR7_CLEAR_WORD0
+    0x00000000, // CB_COLOR7_CLEAR_WORD1
+};
+static const struct cs_extent_def si_SECT_CONTEXT_defs[] =
+{
+    {si_SECT_CONTEXT_def_1, 0x0000a000, 212 },
+    {si_SECT_CONTEXT_def_2, 0x0000a0d8, 272 },
+    {si_SECT_CONTEXT_def_3, 0x0000a1f5, 6 },
+    {si_SECT_CONTEXT_def_4, 0x0000a200, 157 },
+    {si_SECT_CONTEXT_def_5, 0x0000a2a1, 1 },
+    {si_SECT_CONTEXT_def_6, 0x0000a2a3, 1 },
+    {si_SECT_CONTEXT_def_7, 0x0000a2a5, 233 },
+    { NULL, 0, 0 }
+};
+static const struct cs_section_def si_cs_data[] = {
+    { si_SECT_CONTEXT_defs, SECT_CONTEXT },
+    { NULL, SECT_NONE }
+};
diff --git a/drivers/gpu/drm/amd/include/asic_reg/si/si_reg.h b/drivers/gpu/drm/amd/include/asic_reg/si/si_reg.h
new file mode 100644
index 000000000000..895c8e2353e3
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/si/si_reg.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2010 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef __SI_REG_H__
+#define __SI_REG_H__
+
+/* SI */
+#define SI_DC_GPIO_HPD_MASK                      0x196c
+#define SI_DC_GPIO_HPD_A                         0x196d
+#define SI_DC_GPIO_HPD_EN                        0x196e
+#define SI_DC_GPIO_HPD_Y                         0x196f
+
+#define SI_GRPH_CONTROL                          0x1a01
+#       define SI_GRPH_DEPTH(x)                  (((x) & 0x3) << 0)
+#       define SI_GRPH_DEPTH_8BPP                0
+#       define SI_GRPH_DEPTH_16BPP               1
+#       define SI_GRPH_DEPTH_32BPP               2
+#       define SI_GRPH_NUM_BANKS(x)              (((x) & 0x3) << 2)
+#       define SI_ADDR_SURF_2_BANK               0
+#       define SI_ADDR_SURF_4_BANK               1
+#       define SI_ADDR_SURF_8_BANK               2
+#       define SI_ADDR_SURF_16_BANK              3
+#       define SI_GRPH_Z(x)                      (((x) & 0x3) << 4)
+#       define SI_GRPH_BANK_WIDTH(x)             (((x) & 0x3) << 6)
+#       define SI_ADDR_SURF_BANK_WIDTH_1         0
+#       define SI_ADDR_SURF_BANK_WIDTH_2         1
+#       define SI_ADDR_SURF_BANK_WIDTH_4         2
+#       define SI_ADDR_SURF_BANK_WIDTH_8         3
+#       define SI_GRPH_FORMAT(x)                 (((x) & 0x7) << 8)
+/* 8 BPP */
+#       define SI_GRPH_FORMAT_INDEXED            0
+/* 16 BPP */
+#       define SI_GRPH_FORMAT_ARGB1555           0
+#       define SI_GRPH_FORMAT_ARGB565            1
+#       define SI_GRPH_FORMAT_ARGB4444           2
+#       define SI_GRPH_FORMAT_AI88               3
+#       define SI_GRPH_FORMAT_MONO16             4
+#       define SI_GRPH_FORMAT_BGRA5551           5
+/* 32 BPP */
+#       define SI_GRPH_FORMAT_ARGB8888           0
+#       define SI_GRPH_FORMAT_ARGB2101010        1
+#       define SI_GRPH_FORMAT_32BPP_DIG          2
+#       define SI_GRPH_FORMAT_8B_ARGB2101010     3
+#       define SI_GRPH_FORMAT_BGRA1010102        4
+#       define SI_GRPH_FORMAT_8B_BGRA1010102     5
+#       define SI_GRPH_FORMAT_RGB111110          6
+#       define SI_GRPH_FORMAT_BGR101111          7
+#       define SI_GRPH_BANK_HEIGHT(x)            (((x) & 0x3) << 11)
+#       define SI_ADDR_SURF_BANK_HEIGHT_1        0
+#       define SI_ADDR_SURF_BANK_HEIGHT_2        1
+#       define SI_ADDR_SURF_BANK_HEIGHT_4        2
+#       define SI_ADDR_SURF_BANK_HEIGHT_8        3
+#       define SI_GRPH_TILE_SPLIT(x)             (((x) & 0x7) << 13)
+#       define SI_ADDR_SURF_TILE_SPLIT_64B       0
+#       define SI_ADDR_SURF_TILE_SPLIT_128B      1
+#       define SI_ADDR_SURF_TILE_SPLIT_256B      2
+#       define SI_ADDR_SURF_TILE_SPLIT_512B      3
+#       define SI_ADDR_SURF_TILE_SPLIT_1KB       4
+#       define SI_ADDR_SURF_TILE_SPLIT_2KB       5
+#       define SI_ADDR_SURF_TILE_SPLIT_4KB       6
+#       define SI_GRPH_MACRO_TILE_ASPECT(x)      (((x) & 0x3) << 18)
+#       define SI_ADDR_SURF_MACRO_TILE_ASPECT_1  0
+#       define SI_ADDR_SURF_MACRO_TILE_ASPECT_2  1
+#       define SI_ADDR_SURF_MACRO_TILE_ASPECT_4  2
+#       define SI_ADDR_SURF_MACRO_TILE_ASPECT_8  3
+#       define SI_GRPH_ARRAY_MODE(x)             (((x) & 0x7) << 20)
+#       define SI_GRPH_ARRAY_LINEAR_GENERAL      0
+#       define SI_GRPH_ARRAY_LINEAR_ALIGNED      1
+#       define SI_GRPH_ARRAY_1D_TILED_THIN1      2
+#       define SI_GRPH_ARRAY_2D_TILED_THIN1      4
+#       define SI_GRPH_PIPE_CONFIG(x)		 (((x) & 0x1f) << 24)
+#       define SI_ADDR_SURF_P2			 0
+#       define SI_ADDR_SURF_P4_8x16		 4
+#       define SI_ADDR_SURF_P4_16x16		 5
+#       define SI_ADDR_SURF_P4_16x32		 6
+#       define SI_ADDR_SURF_P4_32x32		 7
+#       define SI_ADDR_SURF_P8_16x16_8x16	 8
+#       define SI_ADDR_SURF_P8_16x32_8x16	 9
+#       define SI_ADDR_SURF_P8_32x32_8x16	 10
+#       define SI_ADDR_SURF_P8_16x32_16x16	 11
+#       define SI_ADDR_SURF_P8_32x32_16x16	 12
+#       define SI_ADDR_SURF_P8_32x32_16x32	 13
+#       define SI_ADDR_SURF_P8_32x64_32x32	 14
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/si/sid.h b/drivers/gpu/drm/amd/include/asic_reg/si/sid.h
new file mode 100644
index 000000000000..c57eff159374
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/si/sid.h
@@ -0,0 +1,2461 @@
+/*
+ * Copyright 2011 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Alex Deucher
+ */
+#ifndef SI_H
+#define SI_H
+
+#define TAHITI_RB_BITMAP_WIDTH_PER_SH  2
+
+#define TAHITI_GB_ADDR_CONFIG_GOLDEN        0x12011003
+#define VERDE_GB_ADDR_CONFIG_GOLDEN         0x12010002
+#define HAINAN_GB_ADDR_CONFIG_GOLDEN        0x02010001
+
+#define SI_MAX_SH_GPRS		 	256
+#define SI_MAX_TEMP_GPRS         	16
+#define SI_MAX_SH_THREADS        	256
+#define SI_MAX_SH_STACK_ENTRIES  	4096
+#define SI_MAX_FRC_EOV_CNT       	16384
+#define SI_MAX_BACKENDS          	8
+#define SI_MAX_BACKENDS_MASK     	0xFF
+#define SI_MAX_BACKENDS_PER_SE_MASK     0x0F
+#define SI_MAX_SIMDS             	12
+#define SI_MAX_SIMDS_MASK        	0x0FFF
+#define SI_MAX_SIMDS_PER_SE_MASK        0x00FF
+#define SI_MAX_PIPES            	8
+#define SI_MAX_PIPES_MASK        	0xFF
+#define SI_MAX_PIPES_PER_SIMD_MASK      0x3F
+#define SI_MAX_LDS_NUM           	0xFFFF
+#define SI_MAX_TCC               	16
+#define SI_MAX_TCC_MASK          	0xFFFF
+
+#define AMDGPU_NUM_OF_VMIDS 		8
+
+/* SMC IND accessor regs */
+#define SMC_IND_INDEX_0                              0x80
+#define SMC_IND_DATA_0                               0x81
+
+#define SMC_IND_ACCESS_CNTL                          0x8A
+#       define AUTO_INCREMENT_IND_0                  (1 << 0)
+#define SMC_MESSAGE_0                                0x8B
+#define SMC_RESP_0                                   0x8C
+
+/* CG IND registers are accessed via SMC indirect space + SMC_CG_IND_START */
+#define SMC_CG_IND_START                    0xc0030000
+#define SMC_CG_IND_END                      0xc0040000
+
+#define	CG_CGTT_LOCAL_0				0x400
+#define	CG_CGTT_LOCAL_1				0x401
+
+/* SMC IND registers */
+#define	SMC_SYSCON_RESET_CNTL				0x80000000
+#       define RST_REG                                  (1 << 0)
+#define	SMC_SYSCON_CLOCK_CNTL_0				0x80000004
+#       define CK_DISABLE                               (1 << 0)
+#       define CKEN                                     (1 << 24)
+
+#define VGA_HDP_CONTROL  				0xCA
+#define		VGA_MEMORY_DISABLE				(1 << 4)
+
+#define DCCG_DISP_SLOW_SELECT_REG                       0x13F
+#define		DCCG_DISP1_SLOW_SELECT(x)		((x) << 0)
+#define		DCCG_DISP1_SLOW_SELECT_MASK		(7 << 0)
+#define		DCCG_DISP1_SLOW_SELECT_SHIFT		0
+#define		DCCG_DISP2_SLOW_SELECT(x)		((x) << 4)
+#define		DCCG_DISP2_SLOW_SELECT_MASK		(7 << 4)
+#define		DCCG_DISP2_SLOW_SELECT_SHIFT		4
+
+#define	CG_SPLL_FUNC_CNTL				0x180
+#define		SPLL_RESET				(1 << 0)
+#define		SPLL_SLEEP				(1 << 1)
+#define		SPLL_BYPASS_EN				(1 << 3)
+#define		SPLL_REF_DIV(x)				((x) << 4)
+#define		SPLL_REF_DIV_MASK			(0x3f << 4)
+#define		SPLL_PDIV_A(x)				((x) << 20)
+#define		SPLL_PDIV_A_MASK			(0x7f << 20)
+#define		SPLL_PDIV_A_SHIFT			20
+#define	CG_SPLL_FUNC_CNTL_2				0x181
+#define		SCLK_MUX_SEL(x)				((x) << 0)
+#define		SCLK_MUX_SEL_MASK			(0x1ff << 0)
+#define		SPLL_CTLREQ_CHG				(1 << 23)
+#define		SCLK_MUX_UPDATE				(1 << 26)
+#define	CG_SPLL_FUNC_CNTL_3				0x182
+#define		SPLL_FB_DIV(x)				((x) << 0)
+#define		SPLL_FB_DIV_MASK			(0x3ffffff << 0)
+#define		SPLL_FB_DIV_SHIFT			0
+#define		SPLL_DITHEN				(1 << 28)
+#define	CG_SPLL_FUNC_CNTL_4				0x183
+
+#define	SPLL_STATUS					0x185
+#define		SPLL_CHG_STATUS				(1 << 1)
+#define	SPLL_CNTL_MODE					0x186
+#define		SPLL_SW_DIR_CONTROL			(1 << 0)
+#	define SPLL_REFCLK_SEL(x)			((x) << 26)
+#	define SPLL_REFCLK_SEL_MASK			(3 << 26)
+
+#define	CG_SPLL_SPREAD_SPECTRUM				0x188
+#define		SSEN					(1 << 0)
+#define		CLK_S(x)				((x) << 4)
+#define		CLK_S_MASK				(0xfff << 4)
+#define		CLK_S_SHIFT				4
+#define	CG_SPLL_SPREAD_SPECTRUM_2			0x189
+#define		CLK_V(x)				((x) << 0)
+#define		CLK_V_MASK				(0x3ffffff << 0)
+#define		CLK_V_SHIFT				0
+
+#define	CG_SPLL_AUTOSCALE_CNTL				0x18b
+#       define AUTOSCALE_ON_SS_CLEAR                    (1 << 9)
+
+/* discrete uvd clocks */
+#define	CG_UPLL_FUNC_CNTL				0x18d
+#	define UPLL_RESET_MASK				0x00000001
+#	define UPLL_SLEEP_MASK				0x00000002
+#	define UPLL_BYPASS_EN_MASK			0x00000004
+#	define UPLL_CTLREQ_MASK				0x00000008
+#	define UPLL_VCO_MODE_MASK			0x00000600
+#	define UPLL_REF_DIV_MASK			0x003F0000
+#	define UPLL_CTLACK_MASK				0x40000000
+#	define UPLL_CTLACK2_MASK			0x80000000
+#define	CG_UPLL_FUNC_CNTL_2				0x18e
+#	define UPLL_PDIV_A(x)				((x) << 0)
+#	define UPLL_PDIV_A_MASK				0x0000007F
+#	define UPLL_PDIV_B(x)				((x) << 8)
+#	define UPLL_PDIV_B_MASK				0x00007F00
+#	define VCLK_SRC_SEL(x)				((x) << 20)
+#	define VCLK_SRC_SEL_MASK			0x01F00000
+#	define DCLK_SRC_SEL(x)				((x) << 25)
+#	define DCLK_SRC_SEL_MASK			0x3E000000
+#define	CG_UPLL_FUNC_CNTL_3				0x18f
+#	define UPLL_FB_DIV(x)				((x) << 0)
+#	define UPLL_FB_DIV_MASK				0x01FFFFFF
+#define	CG_UPLL_FUNC_CNTL_4                             0x191
+#	define UPLL_SPARE_ISPARE9			0x00020000
+#define	CG_UPLL_FUNC_CNTL_5				0x192
+#	define RESET_ANTI_MUX_MASK			0x00000200
+#define	CG_UPLL_SPREAD_SPECTRUM				0x194
+#	define SSEN_MASK				0x00000001
+
+#define	MPLL_BYPASSCLK_SEL				0x197
+#	define MPLL_CLKOUT_SEL(x)			((x) << 8)
+#	define MPLL_CLKOUT_SEL_MASK			0xFF00
+
+#define CG_CLKPIN_CNTL                                    0x198
+#       define XTALIN_DIVIDE                              (1 << 1)
+#       define BCLK_AS_XCLK                               (1 << 2)
+#define CG_CLKPIN_CNTL_2                                  0x199
+#       define FORCE_BIF_REFCLK_EN                        (1 << 3)
+#       define MUX_TCLK_TO_XCLK                           (1 << 8)
+
+#define	THM_CLK_CNTL					0x19b
+#	define CMON_CLK_SEL(x)				((x) << 0)
+#	define CMON_CLK_SEL_MASK			0xFF
+#	define TMON_CLK_SEL(x)				((x) << 8)
+#	define TMON_CLK_SEL_MASK			0xFF00
+#define	MISC_CLK_CNTL					0x19c
+#	define DEEP_SLEEP_CLK_SEL(x)			((x) << 0)
+#	define DEEP_SLEEP_CLK_SEL_MASK			0xFF
+#	define ZCLK_SEL(x)				((x) << 8)
+#	define ZCLK_SEL_MASK				0xFF00
+
+#define	CG_THERMAL_CTRL					0x1c0
+#define 	DPM_EVENT_SRC(x)			((x) << 0)
+#define 	DPM_EVENT_SRC_MASK			(7 << 0)
+#define		DIG_THERM_DPM(x)			((x) << 14)
+#define		DIG_THERM_DPM_MASK			0x003FC000
+#define		DIG_THERM_DPM_SHIFT			14
+#define	CG_THERMAL_STATUS				0x1c1
+#define		FDO_PWM_DUTY(x)				((x) << 9)
+#define		FDO_PWM_DUTY_MASK			(0xff << 9)
+#define		FDO_PWM_DUTY_SHIFT			9
+#define	CG_THERMAL_INT					0x1c2
+#define		DIG_THERM_INTH(x)			((x) << 8)
+#define		DIG_THERM_INTH_MASK			0x0000FF00
+#define		DIG_THERM_INTH_SHIFT			8
+#define		DIG_THERM_INTL(x)			((x) << 16)
+#define		DIG_THERM_INTL_MASK			0x00FF0000
+#define		DIG_THERM_INTL_SHIFT			16
+#define 	THERM_INT_MASK_HIGH			(1 << 24)
+#define 	THERM_INT_MASK_LOW			(1 << 25)
+
+#define	CG_MULT_THERMAL_CTRL					0x1c4
+#define		TEMP_SEL(x)					((x) << 20)
+#define		TEMP_SEL_MASK					(0xff << 20)
+#define		TEMP_SEL_SHIFT					20
+#define	CG_MULT_THERMAL_STATUS					0x1c5
+#define		ASIC_MAX_TEMP(x)				((x) << 0)
+#define		ASIC_MAX_TEMP_MASK				0x000001ff
+#define		ASIC_MAX_TEMP_SHIFT				0
+#define		CTF_TEMP(x)					((x) << 9)
+#define		CTF_TEMP_MASK					0x0003fe00
+#define		CTF_TEMP_SHIFT					9
+
+#define	CG_FDO_CTRL0					0x1d5
+#define		FDO_STATIC_DUTY(x)			((x) << 0)
+#define		FDO_STATIC_DUTY_MASK			0x000000FF
+#define		FDO_STATIC_DUTY_SHIFT			0
+#define	CG_FDO_CTRL1					0x1d6
+#define		FMAX_DUTY100(x)				((x) << 0)
+#define		FMAX_DUTY100_MASK			0x000000FF
+#define		FMAX_DUTY100_SHIFT			0
+#define	CG_FDO_CTRL2					0x1d7
+#define		TMIN(x)					((x) << 0)
+#define		TMIN_MASK				0x000000FF
+#define		TMIN_SHIFT				0
+#define		FDO_PWM_MODE(x)				((x) << 11)
+#define		FDO_PWM_MODE_MASK			(7 << 11)
+#define		FDO_PWM_MODE_SHIFT			11
+#define		TACH_PWM_RESP_RATE(x)			((x) << 25)
+#define		TACH_PWM_RESP_RATE_MASK			(0x7f << 25)
+#define		TACH_PWM_RESP_RATE_SHIFT		25
+
+#define CG_TACH_CTRL                                    0x1dc
+#       define EDGE_PER_REV(x)                          ((x) << 0)
+#       define EDGE_PER_REV_MASK                        (0x7 << 0)
+#       define EDGE_PER_REV_SHIFT                       0
+#       define TARGET_PERIOD(x)                         ((x) << 3)
+#       define TARGET_PERIOD_MASK                       0xfffffff8
+#       define TARGET_PERIOD_SHIFT                      3
+#define CG_TACH_STATUS                                  0x1dd
+#       define TACH_PERIOD(x)                           ((x) << 0)
+#       define TACH_PERIOD_MASK                         0xffffffff
+#       define TACH_PERIOD_SHIFT                        0
+
+#define GENERAL_PWRMGT                                  0x1e0
+#       define GLOBAL_PWRMGT_EN                         (1 << 0)
+#       define STATIC_PM_EN                             (1 << 1)
+#       define THERMAL_PROTECTION_DIS                   (1 << 2)
+#       define THERMAL_PROTECTION_TYPE                  (1 << 3)
+#       define SW_SMIO_INDEX(x)                         ((x) << 6)
+#       define SW_SMIO_INDEX_MASK                       (1 << 6)
+#       define SW_SMIO_INDEX_SHIFT                      6
+#       define VOLT_PWRMGT_EN                           (1 << 10)
+#       define DYN_SPREAD_SPECTRUM_EN                   (1 << 23)
+#define CG_TPC                                            0x1e1
+#define SCLK_PWRMGT_CNTL                                  0x1e2
+#       define SCLK_PWRMGT_OFF                            (1 << 0)
+#       define SCLK_LOW_D1                                (1 << 1)
+#       define FIR_RESET                                  (1 << 4)
+#       define FIR_FORCE_TREND_SEL                        (1 << 5)
+#       define FIR_TREND_MODE                             (1 << 6)
+#       define DYN_GFX_CLK_OFF_EN                         (1 << 7)
+#       define GFX_CLK_FORCE_ON                           (1 << 8)
+#       define GFX_CLK_REQUEST_OFF                        (1 << 9)
+#       define GFX_CLK_FORCE_OFF                          (1 << 10)
+#       define GFX_CLK_OFF_ACPI_D1                        (1 << 11)
+#       define GFX_CLK_OFF_ACPI_D2                        (1 << 12)
+#       define GFX_CLK_OFF_ACPI_D3                        (1 << 13)
+#       define DYN_LIGHT_SLEEP_EN                         (1 << 14)
+
+#define TARGET_AND_CURRENT_PROFILE_INDEX                  0x1e6
+#       define CURRENT_STATE_INDEX_MASK                   (0xf << 4)
+#       define CURRENT_STATE_INDEX_SHIFT                  4
+
+#define CG_FTV                                            0x1ef
+
+#define CG_FFCT_0                                         0x1f0
+#       define UTC_0(x)                                   ((x) << 0)
+#       define UTC_0_MASK                                 (0x3ff << 0)
+#       define DTC_0(x)                                   ((x) << 10)
+#       define DTC_0_MASK                                 (0x3ff << 10)
+
+#define CG_BSP                                          0x1ff
+#       define BSP(x)					((x) << 0)
+#       define BSP_MASK					(0xffff << 0)
+#       define BSU(x)					((x) << 16)
+#       define BSU_MASK					(0xf << 16)
+#define CG_AT                                           0x200
+#       define CG_R(x)					((x) << 0)
+#       define CG_R_MASK				(0xffff << 0)
+#       define CG_L(x)					((x) << 16)
+#       define CG_L_MASK				(0xffff << 16)
+
+#define CG_GIT                                          0x201
+#       define CG_GICST(x)                              ((x) << 0)
+#       define CG_GICST_MASK                            (0xffff << 0)
+#       define CG_GIPOT(x)                              ((x) << 16)
+#       define CG_GIPOT_MASK                            (0xffff << 16)
+
+#define CG_SSP                                            0x203
+#       define SST(x)                                     ((x) << 0)
+#       define SST_MASK                                   (0xffff << 0)
+#       define SSTU(x)                                    ((x) << 16)
+#       define SSTU_MASK                                  (0xf << 16)
+
+#define CG_DISPLAY_GAP_CNTL                               0x20a
+#       define DISP1_GAP(x)                               ((x) << 0)
+#       define DISP1_GAP_MASK                             (3 << 0)
+#       define DISP2_GAP(x)                               ((x) << 2)
+#       define DISP2_GAP_MASK                             (3 << 2)
+#       define VBI_TIMER_COUNT(x)                         ((x) << 4)
+#       define VBI_TIMER_COUNT_MASK                       (0x3fff << 4)
+#       define VBI_TIMER_UNIT(x)                          ((x) << 20)
+#       define VBI_TIMER_UNIT_MASK                        (7 << 20)
+#       define DISP1_GAP_MCHG(x)                          ((x) << 24)
+#       define DISP1_GAP_MCHG_MASK                        (3 << 24)
+#       define DISP2_GAP_MCHG(x)                          ((x) << 26)
+#       define DISP2_GAP_MCHG_MASK                        (3 << 26)
+
+#define	CG_ULV_CONTROL					0x21e
+#define	CG_ULV_PARAMETER				0x21f
+
+#define	SMC_SCRATCH0					0x221
+
+#define	CG_CAC_CTRL					0x22e
+#	define CAC_WINDOW(x)				((x) << 0)
+#	define CAC_WINDOW_MASK				0x00ffffff
+
+#define DMIF_ADDR_CONFIG  				0x2F5
+
+#define DMIF_ADDR_CALC  				0x300
+
+#define	PIPE0_DMIF_BUFFER_CONTROL			  0x0328
+#       define DMIF_BUFFERS_ALLOCATED(x)                  ((x) << 0)
+#       define DMIF_BUFFERS_ALLOCATED_COMPLETED           (1 << 4)
+
+#define	SRBM_STATUS				        0x394
+#define		GRBM_RQ_PENDING 			(1 << 5)
+#define		VMC_BUSY 				(1 << 8)
+#define		MCB_BUSY 				(1 << 9)
+#define		MCB_NON_DISPLAY_BUSY 			(1 << 10)
+#define		MCC_BUSY 				(1 << 11)
+#define		MCD_BUSY 				(1 << 12)
+#define		SEM_BUSY 				(1 << 14)
+#define		IH_BUSY 				(1 << 17)
+
+#define	SRBM_SOFT_RESET				        0x398
+#define		SOFT_RESET_BIF				(1 << 1)
+#define		SOFT_RESET_DC				(1 << 5)
+#define		SOFT_RESET_DMA1				(1 << 6)
+#define		SOFT_RESET_GRBM				(1 << 8)
+#define		SOFT_RESET_HDP				(1 << 9)
+#define		SOFT_RESET_IH				(1 << 10)
+#define		SOFT_RESET_MC				(1 << 11)
+#define		SOFT_RESET_ROM				(1 << 14)
+#define		SOFT_RESET_SEM				(1 << 15)
+#define		SOFT_RESET_VMC				(1 << 17)
+#define		SOFT_RESET_DMA				(1 << 20)
+#define		SOFT_RESET_TST				(1 << 21)
+#define		SOFT_RESET_REGBB			(1 << 22)
+#define		SOFT_RESET_ORB				(1 << 23)
+
+#define	CC_SYS_RB_BACKEND_DISABLE			0x3A0
+#define	GC_USER_SYS_RB_BACKEND_DISABLE			0x3A1
+
+#define SRBM_READ_ERROR					0x3A6
+#define SRBM_INT_CNTL					0x3A8
+#define SRBM_INT_ACK					0x3AA
+
+#define	SRBM_STATUS2				        0x3B1
+#define		DMA_BUSY 				(1 << 5)
+#define		DMA1_BUSY 				(1 << 6)
+
+#define VM_L2_CNTL					0x500
+#define		ENABLE_L2_CACHE					(1 << 0)
+#define		ENABLE_L2_FRAGMENT_PROCESSING			(1 << 1)
+#define		L2_CACHE_PTE_ENDIAN_SWAP_MODE(x)		((x) << 2)
+#define		L2_CACHE_PDE_ENDIAN_SWAP_MODE(x)		((x) << 4)
+#define		ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE		(1 << 9)
+#define		ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE	(1 << 10)
+#define		EFFECTIVE_L2_QUEUE_SIZE(x)			(((x) & 7) << 15)
+#define		CONTEXT1_IDENTITY_ACCESS_MODE(x)		(((x) & 3) << 19)
+#define VM_L2_CNTL2					0x501
+#define		INVALIDATE_ALL_L1_TLBS				(1 << 0)
+#define		INVALIDATE_L2_CACHE				(1 << 1)
+#define		INVALIDATE_CACHE_MODE(x)			((x) << 26)
+#define			INVALIDATE_PTE_AND_PDE_CACHES		0
+#define			INVALIDATE_ONLY_PTE_CACHES		1
+#define			INVALIDATE_ONLY_PDE_CACHES		2
+#define VM_L2_CNTL3					0x502
+#define		BANK_SELECT(x)					((x) << 0)
+#define		L2_CACHE_UPDATE_MODE(x)				((x) << 6)
+#define		L2_CACHE_BIGK_FRAGMENT_SIZE(x)			((x) << 15)
+#define		L2_CACHE_BIGK_ASSOCIATIVITY			(1 << 20)
+#define	VM_L2_STATUS					0x503
+#define		L2_BUSY						(1 << 0)
+#define VM_CONTEXT0_CNTL				0x504
+#define		ENABLE_CONTEXT					(1 << 0)
+#define		PAGE_TABLE_DEPTH(x)				(((x) & 3) << 1)
+#define		RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT		(1 << 3)
+#define		RANGE_PROTECTION_FAULT_ENABLE_DEFAULT		(1 << 4)
+#define		DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT	(1 << 6)
+#define		DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT	(1 << 7)
+#define		PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT		(1 << 9)
+#define		PDE0_PROTECTION_FAULT_ENABLE_DEFAULT		(1 << 10)
+#define		VALID_PROTECTION_FAULT_ENABLE_INTERRUPT		(1 << 12)
+#define		VALID_PROTECTION_FAULT_ENABLE_DEFAULT		(1 << 13)
+#define		READ_PROTECTION_FAULT_ENABLE_INTERRUPT		(1 << 15)
+#define		READ_PROTECTION_FAULT_ENABLE_DEFAULT		(1 << 16)
+#define		WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT		(1 << 18)
+#define		WRITE_PROTECTION_FAULT_ENABLE_DEFAULT		(1 << 19)
+#define		PAGE_TABLE_BLOCK_SIZE(x)			(((x) & 0xF) << 24)
+#define VM_CONTEXT1_CNTL				0x505
+#define VM_CONTEXT0_CNTL2				0x50C
+#define VM_CONTEXT1_CNTL2				0x50D
+#define	VM_CONTEXT8_PAGE_TABLE_BASE_ADDR		0x50E
+#define	VM_CONTEXT9_PAGE_TABLE_BASE_ADDR		0x50F
+#define	VM_CONTEXT10_PAGE_TABLE_BASE_ADDR		0x510
+#define	VM_CONTEXT11_PAGE_TABLE_BASE_ADDR		0x511
+#define	VM_CONTEXT12_PAGE_TABLE_BASE_ADDR		0x512
+#define	VM_CONTEXT13_PAGE_TABLE_BASE_ADDR		0x513
+#define	VM_CONTEXT14_PAGE_TABLE_BASE_ADDR		0x514
+#define	VM_CONTEXT15_PAGE_TABLE_BASE_ADDR		0x515
+
+#define	VM_CONTEXT1_PROTECTION_FAULT_ADDR		0x53f
+#define	VM_CONTEXT1_PROTECTION_FAULT_STATUS		0x537
+#define		PROTECTIONS_MASK			(0xf << 0)
+#define		PROTECTIONS_SHIFT			0
+		/* bit 0: range
+		 * bit 1: pde0
+		 * bit 2: valid
+		 * bit 3: read
+		 * bit 4: write
+		 */
+#define		MEMORY_CLIENT_ID_MASK			(0xff << 12)
+#define		MEMORY_CLIENT_ID_SHIFT			12
+#define		MEMORY_CLIENT_RW_MASK			(1 << 24)
+#define		MEMORY_CLIENT_RW_SHIFT			24
+#define		FAULT_VMID_MASK				(0xf << 25)
+#define		FAULT_VMID_SHIFT			25
+
+#define VM_INVALIDATE_REQUEST				0x51E
+#define VM_INVALIDATE_RESPONSE				0x51F
+
+#define VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR	0x546
+#define VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR	0x547
+
+#define	VM_CONTEXT0_PAGE_TABLE_BASE_ADDR		0x54F
+#define	VM_CONTEXT1_PAGE_TABLE_BASE_ADDR		0x550
+#define	VM_CONTEXT2_PAGE_TABLE_BASE_ADDR		0x551
+#define	VM_CONTEXT3_PAGE_TABLE_BASE_ADDR		0x552
+#define	VM_CONTEXT4_PAGE_TABLE_BASE_ADDR		0x553
+#define	VM_CONTEXT5_PAGE_TABLE_BASE_ADDR		0x554
+#define	VM_CONTEXT6_PAGE_TABLE_BASE_ADDR		0x555
+#define	VM_CONTEXT7_PAGE_TABLE_BASE_ADDR		0x556
+#define	VM_CONTEXT0_PAGE_TABLE_START_ADDR		0x557
+#define	VM_CONTEXT1_PAGE_TABLE_START_ADDR		0x558
+
+#define	VM_CONTEXT0_PAGE_TABLE_END_ADDR			0x55F
+#define	VM_CONTEXT1_PAGE_TABLE_END_ADDR			0x560
+
+#define VM_L2_CG           				0x570
+#define		MC_CG_ENABLE				(1 << 18)
+#define		MC_LS_ENABLE				(1 << 19)
+
+#define MC_SHARED_CHMAP						0x801
+#define		NOOFCHAN_SHIFT					12
+#define		NOOFCHAN_MASK					0x0000f000
+#define MC_SHARED_CHREMAP					0x802
+
+#define	MC_VM_FB_LOCATION				0x809
+#define	MC_VM_AGP_TOP					0x80A
+#define	MC_VM_AGP_BOT					0x80B
+#define	MC_VM_AGP_BASE					0x80C
+#define	MC_VM_SYSTEM_APERTURE_LOW_ADDR			0x80D
+#define	MC_VM_SYSTEM_APERTURE_HIGH_ADDR			0x80E
+#define	MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR		0x80F
+
+#define	MC_VM_MX_L1_TLB_CNTL				0x819
+#define		ENABLE_L1_TLB					(1 << 0)
+#define		ENABLE_L1_FRAGMENT_PROCESSING			(1 << 1)
+#define		SYSTEM_ACCESS_MODE_PA_ONLY			(0 << 3)
+#define		SYSTEM_ACCESS_MODE_USE_SYS_MAP			(1 << 3)
+#define		SYSTEM_ACCESS_MODE_IN_SYS			(2 << 3)
+#define		SYSTEM_ACCESS_MODE_NOT_IN_SYS			(3 << 3)
+#define		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU	(0 << 5)
+#define		ENABLE_ADVANCED_DRIVER_MODEL			(1 << 6)
+
+#define MC_SHARED_BLACKOUT_CNTL           		0x82B
+
+#define MC_HUB_MISC_HUB_CG           			0x82E
+#define MC_HUB_MISC_VM_CG           			0x82F
+
+#define MC_HUB_MISC_SIP_CG           			0x830
+
+#define MC_XPB_CLK_GAT           			0x91E
+
+#define MC_CITF_MISC_RD_CG           			0x992
+#define MC_CITF_MISC_WR_CG           			0x993
+#define MC_CITF_MISC_VM_CG           			0x994
+
+#define	MC_ARB_RAMCFG					0x9D8
+#define		NOOFBANK_SHIFT					0
+#define		NOOFBANK_MASK					0x00000003
+#define		NOOFRANK_SHIFT					2
+#define		NOOFRANK_MASK					0x00000004
+#define		NOOFROWS_SHIFT					3
+#define		NOOFROWS_MASK					0x00000038
+#define		NOOFCOLS_SHIFT					6
+#define		NOOFCOLS_MASK					0x000000C0
+#define		CHANSIZE_SHIFT					8
+#define		CHANSIZE_MASK					0x00000100
+#define		CHANSIZE_OVERRIDE				(1 << 11)
+#define		NOOFGROUPS_SHIFT				12
+#define		NOOFGROUPS_MASK					0x00001000
+
+#define	MC_ARB_DRAM_TIMING				0x9DD
+#define	MC_ARB_DRAM_TIMING2				0x9DE
+
+#define MC_ARB_BURST_TIME                               0xA02
+#define		STATE0(x)				((x) << 0)
+#define		STATE0_MASK				(0x1f << 0)
+#define		STATE0_SHIFT				0
+#define		STATE1(x)				((x) << 5)
+#define		STATE1_MASK				(0x1f << 5)
+#define		STATE1_SHIFT				5
+#define		STATE2(x)				((x) << 10)
+#define		STATE2_MASK				(0x1f << 10)
+#define		STATE2_SHIFT				10
+#define		STATE3(x)				((x) << 15)
+#define		STATE3_MASK				(0x1f << 15)
+#define		STATE3_SHIFT				15
+
+#define	MC_SEQ_TRAIN_WAKEUP_CNTL			0xA3A
+#define		TRAIN_DONE_D0      			(1 << 30)
+#define		TRAIN_DONE_D1      			(1 << 31)
+
+#define MC_SEQ_SUP_CNTL           			0xA32
+#define		RUN_MASK      				(1 << 0)
+#define MC_SEQ_SUP_PGM           			0xA33
+#define MC_PMG_AUTO_CMD           			0xA34
+
+#define MC_IO_PAD_CNTL_D0           			0xA74
+#define		MEM_FALL_OUT_CMD      			(1 << 8)
+
+#define MC_SEQ_RAS_TIMING                               0xA28
+#define MC_SEQ_CAS_TIMING                               0xA29
+#define MC_SEQ_MISC_TIMING                              0xA2A
+#define MC_SEQ_MISC_TIMING2                             0xA2B
+#define MC_SEQ_PMG_TIMING                               0xA2C
+#define MC_SEQ_RD_CTL_D0                                0xA2D
+#define MC_SEQ_RD_CTL_D1                                0xA2E
+#define MC_SEQ_WR_CTL_D0                                0xA2F
+#define MC_SEQ_WR_CTL_D1                                0xA30
+
+#define MC_SEQ_MISC0           				0xA80
+#define 	MC_SEQ_MISC0_VEN_ID_SHIFT               8
+#define 	MC_SEQ_MISC0_VEN_ID_MASK                0x00000f00
+#define 	MC_SEQ_MISC0_VEN_ID_VALUE               3
+#define 	MC_SEQ_MISC0_REV_ID_SHIFT               12
+#define 	MC_SEQ_MISC0_REV_ID_MASK                0x0000f000
+#define 	MC_SEQ_MISC0_REV_ID_VALUE               1
+#define 	MC_SEQ_MISC0_GDDR5_SHIFT                28
+#define 	MC_SEQ_MISC0_GDDR5_MASK                 0xf0000000
+#define 	MC_SEQ_MISC0_GDDR5_VALUE                5
+#define MC_SEQ_MISC1                                    0xA81
+#define MC_SEQ_RESERVE_M                                0xA82
+#define MC_PMG_CMD_EMRS                                 0xA83
+
+#define MC_SEQ_IO_DEBUG_INDEX           		0xA91
+#define MC_SEQ_IO_DEBUG_DATA           			0xA92
+
+#define MC_SEQ_MISC5                                    0xA95
+#define MC_SEQ_MISC6                                    0xA96
+
+#define MC_SEQ_MISC7                                    0xA99
+
+#define MC_SEQ_RAS_TIMING_LP                            0xA9B
+#define MC_SEQ_CAS_TIMING_LP                            0xA9C
+#define MC_SEQ_MISC_TIMING_LP                           0xA9D
+#define MC_SEQ_MISC_TIMING2_LP                          0xA9E
+#define MC_SEQ_WR_CTL_D0_LP                             0xA9F
+#define MC_SEQ_WR_CTL_D1_LP                             0xAA0
+#define MC_SEQ_PMG_CMD_EMRS_LP                          0xAA1
+#define MC_SEQ_PMG_CMD_MRS_LP                           0xAA2
+
+#define MC_PMG_CMD_MRS                                  0xAAB
+
+#define MC_SEQ_RD_CTL_D0_LP                             0xAC7
+#define MC_SEQ_RD_CTL_D1_LP                             0xAC8
+
+#define MC_PMG_CMD_MRS1                                 0xAD1
+#define MC_SEQ_PMG_CMD_MRS1_LP                          0xAD2
+#define MC_SEQ_PMG_TIMING_LP                            0xAD3
+
+#define MC_SEQ_WR_CTL_2                                 0xAD5
+#define MC_SEQ_WR_CTL_2_LP                              0xAD6
+#define MC_PMG_CMD_MRS2                                 0xAD7
+#define MC_SEQ_PMG_CMD_MRS2_LP                          0xAD8
+
+#define	MCLK_PWRMGT_CNTL				0xAE8
+#       define DLL_SPEED(x)				((x) << 0)
+#       define DLL_SPEED_MASK				(0x1f << 0)
+#       define DLL_READY                                (1 << 6)
+#       define MC_INT_CNTL                              (1 << 7)
+#       define MRDCK0_PDNB                              (1 << 8)
+#       define MRDCK1_PDNB                              (1 << 9)
+#       define MRDCK0_RESET                             (1 << 16)
+#       define MRDCK1_RESET                             (1 << 17)
+#       define DLL_READY_READ                           (1 << 24)
+#define	DLL_CNTL					0xAE9
+#       define MRDCK0_BYPASS                            (1 << 24)
+#       define MRDCK1_BYPASS                            (1 << 25)
+
+#define	MPLL_CNTL_MODE					0xAEC
+#       define MPLL_MCLK_SEL                            (1 << 11)
+#define	MPLL_FUNC_CNTL					0xAED
+#define		BWCTRL(x)				((x) << 20)
+#define		BWCTRL_MASK				(0xff << 20)
+#define	MPLL_FUNC_CNTL_1				0xAEE
+#define		VCO_MODE(x)				((x) << 0)
+#define		VCO_MODE_MASK				(3 << 0)
+#define		CLKFRAC(x)				((x) << 4)
+#define		CLKFRAC_MASK				(0xfff << 4)
+#define		CLKF(x)					((x) << 16)
+#define		CLKF_MASK				(0xfff << 16)
+#define	MPLL_FUNC_CNTL_2				0xAEF
+#define	MPLL_AD_FUNC_CNTL				0xAF0
+#define		YCLK_POST_DIV(x)			((x) << 0)
+#define		YCLK_POST_DIV_MASK			(7 << 0)
+#define	MPLL_DQ_FUNC_CNTL				0xAF1
+#define		YCLK_SEL(x)				((x) << 4)
+#define		YCLK_SEL_MASK				(1 << 4)
+
+#define	MPLL_SS1					0xAF3
+#define		CLKV(x)					((x) << 0)
+#define		CLKV_MASK				(0x3ffffff << 0)
+#define	MPLL_SS2					0xAF4
+#define		CLKS(x)					((x) << 0)
+#define		CLKS_MASK				(0xfff << 0)
+
+#define	HDP_HOST_PATH_CNTL				0xB00
+#define 	CLOCK_GATING_DIS			(1 << 23)
+#define	HDP_NONSURFACE_BASE				0xB01
+#define	HDP_NONSURFACE_INFO				0xB02
+#define	HDP_NONSURFACE_SIZE				0xB03
+
+#define HDP_DEBUG0  					0xBCC
+
+#define HDP_ADDR_CONFIG  				0xBD2
+#define HDP_MISC_CNTL					0xBD3
+#define 	HDP_FLUSH_INVALIDATE_CACHE			(1 << 0)
+#define HDP_MEM_POWER_LS				0xBD4
+#define 	HDP_LS_ENABLE				(1 << 0)
+
+#define ATC_MISC_CG           				0xCD4
+
+#define IH_RB_CNTL                                        0xF80
+#       define IH_RB_ENABLE                               (1 << 0)
+#       define IH_IB_SIZE(x)                              ((x) << 1) /* log2 */
+#       define IH_RB_FULL_DRAIN_ENABLE                    (1 << 6)
+#       define IH_WPTR_WRITEBACK_ENABLE                   (1 << 8)
+#       define IH_WPTR_WRITEBACK_TIMER(x)                 ((x) << 9) /* log2 */
+#       define IH_WPTR_OVERFLOW_ENABLE                    (1 << 16)
+#       define IH_WPTR_OVERFLOW_CLEAR                     (1 << 31)
+#define IH_RB_BASE                                        0xF81
+#define IH_RB_RPTR                                        0xF82
+#define IH_RB_WPTR                                        0xF83
+#       define RB_OVERFLOW                                (1 << 0)
+#       define WPTR_OFFSET_MASK                           0x3fffc
+#define IH_RB_WPTR_ADDR_HI                                0xF84
+#define IH_RB_WPTR_ADDR_LO                                0xF85
+#define IH_CNTL                                           0xF86
+#       define ENABLE_INTR                                (1 << 0)
+#       define IH_MC_SWAP(x)                              ((x) << 1)
+#       define IH_MC_SWAP_NONE                            0
+#       define IH_MC_SWAP_16BIT                           1
+#       define IH_MC_SWAP_32BIT                           2
+#       define IH_MC_SWAP_64BIT                           3
+#       define RPTR_REARM                                 (1 << 4)
+#       define MC_WRREQ_CREDIT(x)                         ((x) << 15)
+#       define MC_WR_CLEAN_CNT(x)                         ((x) << 20)
+#       define MC_VMID(x)                                 ((x) << 25)
+
+#define	CONFIG_MEMSIZE					0x150A
+
+#define INTERRUPT_CNTL                                    0x151A
+#       define IH_DUMMY_RD_OVERRIDE                       (1 << 0)
+#       define IH_DUMMY_RD_EN                             (1 << 1)
+#       define IH_REQ_NONSNOOP_EN                         (1 << 3)
+#       define GEN_IH_INT_EN                              (1 << 8)
+#define INTERRUPT_CNTL2                                   0x151B
+
+#define HDP_MEM_COHERENCY_FLUSH_CNTL			0x1520
+
+#define	BIF_FB_EN						0x1524
+#define		FB_READ_EN					(1 << 0)
+#define		FB_WRITE_EN					(1 << 1)
+
+#define HDP_REG_COHERENCY_FLUSH_CNTL			0x1528
+
+/* DCE6 ELD audio interface */
+#define AZ_F0_CODEC_ENDPOINT_INDEX                       0x1780
+#       define AZ_ENDPOINT_REG_INDEX(x)                  (((x) & 0xff) << 0)
+#       define AZ_ENDPOINT_REG_WRITE_EN                  (1 << 8)
+#define AZ_F0_CODEC_ENDPOINT_DATA                        0x1781
+
+#define AZ_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER          0x25
+#define		SPEAKER_ALLOCATION(x)			(((x) & 0x7f) << 0)
+#define		SPEAKER_ALLOCATION_MASK			(0x7f << 0)
+#define		SPEAKER_ALLOCATION_SHIFT		0
+#define		HDMI_CONNECTION				(1 << 16)
+#define		DP_CONNECTION				(1 << 17)
+
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0        0x28 /* LPCM */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1        0x29 /* AC3 */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2        0x2A /* MPEG1 */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3        0x2B /* MP3 */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4        0x2C /* MPEG2 */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5        0x2D /* AAC */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6        0x2E /* DTS */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7        0x2F /* ATRAC */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8        0x30 /* one bit audio - leave at 0 (default) */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9        0x31 /* Dolby Digital */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10       0x32 /* DTS-HD */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11       0x33 /* MAT-MLP */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12       0x34 /* DTS */
+#define AZ_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13       0x35 /* WMA Pro */
+#       define MAX_CHANNELS(x)                            (((x) & 0x7) << 0)
+/* max channels minus one.  7 = 8 channels */
+#       define SUPPORTED_FREQUENCIES(x)                   (((x) & 0xff) << 8)
+#       define DESCRIPTOR_BYTE_2(x)                       (((x) & 0xff) << 16)
+#       define SUPPORTED_FREQUENCIES_STEREO(x)            (((x) & 0xff) << 24) /* LPCM only */
+/* SUPPORTED_FREQUENCIES, SUPPORTED_FREQUENCIES_STEREO
+ * bit0 = 32 kHz
+ * bit1 = 44.1 kHz
+ * bit2 = 48 kHz
+ * bit3 = 88.2 kHz
+ * bit4 = 96 kHz
+ * bit5 = 176.4 kHz
+ * bit6 = 192 kHz
+ */
+
+#define AZ_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC         0x37
+#       define VIDEO_LIPSYNC(x)                           (((x) & 0xff) << 0)
+#       define AUDIO_LIPSYNC(x)                           (((x) & 0xff) << 8)
+/* VIDEO_LIPSYNC, AUDIO_LIPSYNC
+ * 0   = invalid
+ * x   = legal delay value
+ * 255 = sync not supported
+ */
+#define AZ_F0_CODEC_PIN_CONTROL_RESPONSE_HBR             0x38
+#       define HBR_CAPABLE                                (1 << 0) /* enabled by default */
+
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO0               0x3a
+#       define MANUFACTURER_ID(x)                        (((x) & 0xffff) << 0)
+#       define PRODUCT_ID(x)                             (((x) & 0xffff) << 16)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO1               0x3b
+#       define SINK_DESCRIPTION_LEN(x)                   (((x) & 0xff) << 0)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO2               0x3c
+#       define PORT_ID0(x)                               (((x) & 0xffffffff) << 0)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO3               0x3d
+#       define PORT_ID1(x)                               (((x) & 0xffffffff) << 0)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO4               0x3e
+#       define DESCRIPTION0(x)                           (((x) & 0xff) << 0)
+#       define DESCRIPTION1(x)                           (((x) & 0xff) << 8)
+#       define DESCRIPTION2(x)                           (((x) & 0xff) << 16)
+#       define DESCRIPTION3(x)                           (((x) & 0xff) << 24)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO5               0x3f
+#       define DESCRIPTION4(x)                           (((x) & 0xff) << 0)
+#       define DESCRIPTION5(x)                           (((x) & 0xff) << 8)
+#       define DESCRIPTION6(x)                           (((x) & 0xff) << 16)
+#       define DESCRIPTION7(x)                           (((x) & 0xff) << 24)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO6               0x40
+#       define DESCRIPTION8(x)                           (((x) & 0xff) << 0)
+#       define DESCRIPTION9(x)                           (((x) & 0xff) << 8)
+#       define DESCRIPTION10(x)                          (((x) & 0xff) << 16)
+#       define DESCRIPTION11(x)                          (((x) & 0xff) << 24)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO7               0x41
+#       define DESCRIPTION12(x)                          (((x) & 0xff) << 0)
+#       define DESCRIPTION13(x)                          (((x) & 0xff) << 8)
+#       define DESCRIPTION14(x)                          (((x) & 0xff) << 16)
+#       define DESCRIPTION15(x)                          (((x) & 0xff) << 24)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO8               0x42
+#       define DESCRIPTION16(x)                          (((x) & 0xff) << 0)
+#       define DESCRIPTION17(x)                          (((x) & 0xff) << 8)
+
+#define AZ_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL         0x54
+#       define AUDIO_ENABLED                             (1 << 31)
+
+#define AZ_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT  0x56
+#define		PORT_CONNECTIVITY_MASK				(3 << 30)
+#define		PORT_CONNECTIVITY_SHIFT				30
+
+#define	DC_LB_MEMORY_SPLIT					0x1AC3
+#define		DC_LB_MEMORY_CONFIG(x)				((x) << 20)
+
+#define	PRIORITY_A_CNT						0x1AC6
+#define		PRIORITY_MARK_MASK				0x7fff
+#define		PRIORITY_OFF					(1 << 16)
+#define		PRIORITY_ALWAYS_ON				(1 << 20)
+#define	PRIORITY_B_CNT						0x1AC7
+
+#define	DPG_PIPE_ARBITRATION_CONTROL3				0x1B32
+#       define LATENCY_WATERMARK_MASK(x)			((x) << 16)
+#define	DPG_PIPE_LATENCY_CONTROL				0x1B33
+#       define LATENCY_LOW_WATERMARK(x)				((x) << 0)
+#       define LATENCY_HIGH_WATERMARK(x)			((x) << 16)
+
+/* 0x6bb8, 0x77b8, 0x103b8, 0x10fb8, 0x11bb8, 0x127b8 */
+#define VLINE_STATUS                                    0x1AEE
+#       define VLINE_OCCURRED                           (1 << 0)
+#       define VLINE_ACK                                (1 << 4)
+#       define VLINE_STAT                               (1 << 12)
+#       define VLINE_INTERRUPT                          (1 << 16)
+#       define VLINE_INTERRUPT_TYPE                     (1 << 17)
+/* 0x6bbc, 0x77bc, 0x103bc, 0x10fbc, 0x11bbc, 0x127bc */
+#define VBLANK_STATUS                                   0x1AEF
+#       define VBLANK_OCCURRED                          (1 << 0)
+#       define VBLANK_ACK                               (1 << 4)
+#       define VBLANK_STAT                              (1 << 12)
+#       define VBLANK_INTERRUPT                         (1 << 16)
+#       define VBLANK_INTERRUPT_TYPE                    (1 << 17)
+
+/* 0x6b40, 0x7740, 0x10340, 0x10f40, 0x11b40, 0x12740 */
+#define INT_MASK                                        0x1AD0
+#       define VBLANK_INT_MASK                          (1 << 0)
+#       define VLINE_INT_MASK                           (1 << 4)
+
+#define DISP_INTERRUPT_STATUS                           0x183D
+#       define LB_D1_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D1_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD1_INTERRUPT                        (1 << 17)
+#       define DC_HPD1_RX_INTERRUPT                     (1 << 18)
+#       define DACA_AUTODETECT_INTERRUPT                (1 << 22)
+#       define DACB_AUTODETECT_INTERRUPT                (1 << 23)
+#       define DC_I2C_SW_DONE_INTERRUPT                 (1 << 24)
+#       define DC_I2C_HW_DONE_INTERRUPT                 (1 << 25)
+#define DISP_INTERRUPT_STATUS_CONTINUE                  0x183E
+#       define LB_D2_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D2_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD2_INTERRUPT                        (1 << 17)
+#       define DC_HPD2_RX_INTERRUPT                     (1 << 18)
+#       define DISP_TIMER_INTERRUPT                     (1 << 24)
+#define DISP_INTERRUPT_STATUS_CONTINUE2                 0x183F
+#       define LB_D3_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D3_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD3_INTERRUPT                        (1 << 17)
+#       define DC_HPD3_RX_INTERRUPT                     (1 << 18)
+#define DISP_INTERRUPT_STATUS_CONTINUE3                 0x1840
+#       define LB_D4_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D4_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD4_INTERRUPT                        (1 << 17)
+#       define DC_HPD4_RX_INTERRUPT                     (1 << 18)
+#define DISP_INTERRUPT_STATUS_CONTINUE4                 0x1853
+#       define LB_D5_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D5_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD5_INTERRUPT                        (1 << 17)
+#       define DC_HPD5_RX_INTERRUPT                     (1 << 18)
+#define DISP_INTERRUPT_STATUS_CONTINUE5                 0x1854
+#       define LB_D6_VLINE_INTERRUPT                    (1 << 2)
+#       define LB_D6_VBLANK_INTERRUPT                   (1 << 3)
+#       define DC_HPD6_INTERRUPT                        (1 << 17)
+#       define DC_HPD6_RX_INTERRUPT                     (1 << 18)
+
+/* 0x6858, 0x7458, 0x10058, 0x10c58, 0x11858, 0x12458 */
+#define GRPH_INT_STATUS                                 0x1A16
+#       define GRPH_PFLIP_INT_OCCURRED                  (1 << 0)
+#       define GRPH_PFLIP_INT_CLEAR                     (1 << 8)
+/* 0x685c, 0x745c, 0x1005c, 0x10c5c, 0x1185c, 0x1245c */
+#define	GRPH_INT_CONTROL			        0x1A17
+#       define GRPH_PFLIP_INT_MASK                      (1 << 0)
+#       define GRPH_PFLIP_INT_TYPE                      (1 << 8)
+
+#define	DAC_AUTODETECT_INT_CONTROL			0x19F2
+
+#define DC_HPD1_INT_STATUS                              0x1807
+#define DC_HPD2_INT_STATUS                              0x180A
+#define DC_HPD3_INT_STATUS                              0x180D
+#define DC_HPD4_INT_STATUS                              0x1810
+#define DC_HPD5_INT_STATUS                              0x1813
+#define DC_HPD6_INT_STATUS                              0x1816
+#       define DC_HPDx_INT_STATUS                       (1 << 0)
+#       define DC_HPDx_SENSE                            (1 << 1)
+#       define DC_HPDx_RX_INT_STATUS                    (1 << 8)
+
+#define DC_HPD1_INT_CONTROL                             0x1808
+#define DC_HPD2_INT_CONTROL                             0x180B
+#define DC_HPD3_INT_CONTROL                             0x180E
+#define DC_HPD4_INT_CONTROL                             0x1811
+#define DC_HPD5_INT_CONTROL                             0x1814
+#define DC_HPD6_INT_CONTROL                             0x1817
+#       define DC_HPDx_INT_ACK                          (1 << 0)
+#       define DC_HPDx_INT_POLARITY                     (1 << 8)
+#       define DC_HPDx_INT_EN                           (1 << 16)
+#       define DC_HPDx_RX_INT_ACK                       (1 << 20)
+#       define DC_HPDx_RX_INT_EN                        (1 << 24)
+
+#define DC_HPD1_CONTROL                                   0x1809
+#define DC_HPD2_CONTROL                                   0x180C
+#define DC_HPD3_CONTROL                                   0x180F
+#define DC_HPD4_CONTROL                                   0x1812
+#define DC_HPD5_CONTROL                                   0x1815
+#define DC_HPD6_CONTROL                                   0x1818
+#       define DC_HPDx_CONNECTION_TIMER(x)                ((x) << 0)
+#       define DC_HPDx_RX_INT_TIMER(x)                    ((x) << 16)
+#       define DC_HPDx_EN                                 (1 << 28)
+
+#define DPG_PIPE_STUTTER_CONTROL                          0x1B35
+#       define STUTTER_ENABLE                             (1 << 0)
+
+/* 0x6e98, 0x7a98, 0x10698, 0x11298, 0x11e98, 0x12a98 */
+#define CRTC_STATUS_FRAME_COUNT                         0x1BA6
+
+/* Audio clocks */
+#define DCCG_AUDIO_DTO_SOURCE                           0x05ac
+#       define DCCG_AUDIO_DTO0_SOURCE_SEL(x) ((x) << 0) /* crtc0 - crtc5 */
+#       define DCCG_AUDIO_DTO_SEL            (1 << 4)   /* 0=dto0 1=dto1 */
+
+#define DCCG_AUDIO_DTO0_PHASE                           0x05b0
+#define DCCG_AUDIO_DTO0_MODULE                          0x05b4
+#define DCCG_AUDIO_DTO1_PHASE                           0x05c0
+#define DCCG_AUDIO_DTO1_MODULE                          0x05c4
+
+#define AFMT_AUDIO_SRC_CONTROL                          0x1c4f
+#define		AFMT_AUDIO_SRC_SELECT(x)		(((x) & 7) << 0)
+/* AFMT_AUDIO_SRC_SELECT
+ * 0 = stream0
+ * 1 = stream1
+ * 2 = stream2
+ * 3 = stream3
+ * 4 = stream4
+ * 5 = stream5
+ */
+
+#define	GRBM_CNTL					0x2000
+#define		GRBM_READ_TIMEOUT(x)				((x) << 0)
+
+#define	GRBM_STATUS2					0x2002
+#define		RLC_RQ_PENDING 					(1 << 0)
+#define		RLC_BUSY 					(1 << 8)
+#define		TC_BUSY 					(1 << 9)
+
+#define	GRBM_STATUS					0x2004
+#define		CMDFIFO_AVAIL_MASK				0x0000000F
+#define		RING2_RQ_PENDING				(1 << 4)
+#define		SRBM_RQ_PENDING					(1 << 5)
+#define		RING1_RQ_PENDING				(1 << 6)
+#define		CF_RQ_PENDING					(1 << 7)
+#define		PF_RQ_PENDING					(1 << 8)
+#define		GDS_DMA_RQ_PENDING				(1 << 9)
+#define		GRBM_EE_BUSY					(1 << 10)
+#define		DB_CLEAN					(1 << 12)
+#define		CB_CLEAN					(1 << 13)
+#define		TA_BUSY 					(1 << 14)
+#define		GDS_BUSY 					(1 << 15)
+#define		VGT_BUSY					(1 << 17)
+#define		IA_BUSY_NO_DMA					(1 << 18)
+#define		IA_BUSY						(1 << 19)
+#define		SX_BUSY 					(1 << 20)
+#define		SPI_BUSY					(1 << 22)
+#define		BCI_BUSY					(1 << 23)
+#define		SC_BUSY 					(1 << 24)
+#define		PA_BUSY 					(1 << 25)
+#define		DB_BUSY 					(1 << 26)
+#define		CP_COHERENCY_BUSY      				(1 << 28)
+#define		CP_BUSY 					(1 << 29)
+#define		CB_BUSY 					(1 << 30)
+#define		GUI_ACTIVE					(1 << 31)
+#define	GRBM_STATUS_SE0					0x2005
+#define	GRBM_STATUS_SE1					0x2006
+#define		SE_DB_CLEAN					(1 << 1)
+#define		SE_CB_CLEAN					(1 << 2)
+#define		SE_BCI_BUSY					(1 << 22)
+#define		SE_VGT_BUSY					(1 << 23)
+#define		SE_PA_BUSY					(1 << 24)
+#define		SE_TA_BUSY					(1 << 25)
+#define		SE_SX_BUSY					(1 << 26)
+#define		SE_SPI_BUSY					(1 << 27)
+#define		SE_SC_BUSY					(1 << 29)
+#define		SE_DB_BUSY					(1 << 30)
+#define		SE_CB_BUSY					(1 << 31)
+
+#define	GRBM_SOFT_RESET					0x2008
+#define		SOFT_RESET_CP					(1 << 0)
+#define		SOFT_RESET_CB					(1 << 1)
+#define		SOFT_RESET_RLC					(1 << 2)
+#define		SOFT_RESET_DB					(1 << 3)
+#define		SOFT_RESET_GDS					(1 << 4)
+#define		SOFT_RESET_PA					(1 << 5)
+#define		SOFT_RESET_SC					(1 << 6)
+#define		SOFT_RESET_BCI					(1 << 7)
+#define		SOFT_RESET_SPI					(1 << 8)
+#define		SOFT_RESET_SX					(1 << 10)
+#define		SOFT_RESET_TC					(1 << 11)
+#define		SOFT_RESET_TA					(1 << 12)
+#define		SOFT_RESET_VGT					(1 << 14)
+#define		SOFT_RESET_IA					(1 << 15)
+
+#define GRBM_GFX_INDEX          			0x200B
+#define		INSTANCE_INDEX(x)			((x) << 0)
+#define		SH_INDEX(x)     			((x) << 8)
+#define		SE_INDEX(x)     			((x) << 16)
+#define		SH_BROADCAST_WRITES      		(1 << 29)
+#define		INSTANCE_BROADCAST_WRITES      		(1 << 30)
+#define		SE_BROADCAST_WRITES      		(1 << 31)
+
+#define GRBM_INT_CNTL                                   0x2018
+#       define RDERR_INT_ENABLE                         (1 << 0)
+#       define GUI_IDLE_INT_ENABLE                      (1 << 19)
+
+#define	CP_STRMOUT_CNTL					0x213F
+#define	SCRATCH_REG0					0x2140
+#define	SCRATCH_REG1					0x2141
+#define	SCRATCH_REG2					0x2142
+#define	SCRATCH_REG3					0x2143
+#define	SCRATCH_REG4					0x2144
+#define	SCRATCH_REG5					0x2145
+#define	SCRATCH_REG6					0x2146
+#define	SCRATCH_REG7					0x2147
+
+#define	SCRATCH_UMSK					0x2150
+#define	SCRATCH_ADDR					0x2151
+
+#define	CP_SEM_WAIT_TIMER				0x216F
+
+#define	CP_SEM_INCOMPLETE_TIMER_CNTL			0x2172
+
+#define CP_ME_CNTL					0x21B6
+#define		CP_CE_HALT					(1 << 24)
+#define		CP_PFP_HALT					(1 << 26)
+#define		CP_ME_HALT					(1 << 28)
+
+#define	CP_COHER_CNTL2					0x217A
+
+#define	CP_RB2_RPTR					0x21BE
+#define	CP_RB1_RPTR					0x21BF
+#define	CP_RB0_RPTR					0x21C0
+#define	CP_RB_WPTR_DELAY				0x21C1
+
+#define	CP_QUEUE_THRESHOLDS				0x21D8
+#define		ROQ_IB1_START(x)				((x) << 0)
+#define		ROQ_IB2_START(x)				((x) << 8)
+#define CP_MEQ_THRESHOLDS				0x21D9
+#define		MEQ1_START(x)				((x) << 0)
+#define		MEQ2_START(x)				((x) << 8)
+
+#define	CP_PERFMON_CNTL					0x21FF
+
+#define	VGT_VTX_VECT_EJECT_REG				0x222C
+
+#define	VGT_CACHE_INVALIDATION				0x2231
+#define		CACHE_INVALIDATION(x)				((x) << 0)
+#define			VC_ONLY						0
+#define			TC_ONLY						1
+#define			VC_AND_TC					2
+#define		AUTO_INVLD_EN(x)				((x) << 6)
+#define			NO_AUTO						0
+#define			ES_AUTO						1
+#define			GS_AUTO						2
+#define			ES_AND_GS_AUTO					3
+#define	VGT_ESGS_RING_SIZE				0x2232
+#define	VGT_GSVS_RING_SIZE				0x2233
+
+#define	VGT_GS_VERTEX_REUSE				0x2235
+
+#define	VGT_PRIMITIVE_TYPE				0x2256
+#define	VGT_INDEX_TYPE					0x2257
+
+#define	VGT_NUM_INDICES					0x225C
+#define	VGT_NUM_INSTANCES				0x225D
+
+#define	VGT_TF_RING_SIZE				0x2262
+
+#define	VGT_HS_OFFCHIP_PARAM				0x226C
+
+#define	VGT_TF_MEMORY_BASE				0x226E
+
+#define CC_GC_SHADER_ARRAY_CONFIG			0x226F
+#define		INACTIVE_CUS_MASK			0xFFFF0000
+#define		INACTIVE_CUS_SHIFT			16
+#define GC_USER_SHADER_ARRAY_CONFIG			0x2270
+
+#define	PA_CL_ENHANCE					0x2285
+#define		CLIP_VTX_REORDER_ENA				(1 << 0)
+#define		NUM_CLIP_SEQ(x)					((x) << 1)
+
+#define	PA_SU_LINE_STIPPLE_VALUE			0x2298
+
+#define	PA_SC_LINE_STIPPLE_STATE			0x22C4
+
+#define	PA_SC_FORCE_EOV_MAX_CNTS			0x22C9
+#define		FORCE_EOV_MAX_CLK_CNT(x)			((x) << 0)
+#define		FORCE_EOV_MAX_REZ_CNT(x)			((x) << 16)
+
+#define	PA_SC_FIFO_SIZE					0x22F3
+#define		SC_FRONTEND_PRIM_FIFO_SIZE(x)			((x) << 0)
+#define		SC_BACKEND_PRIM_FIFO_SIZE(x)			((x) << 6)
+#define		SC_HIZ_TILE_FIFO_SIZE(x)			((x) << 15)
+#define		SC_EARLYZ_TILE_FIFO_SIZE(x)			((x) << 23)
+
+#define	PA_SC_ENHANCE					0x22FC
+
+#define	SQ_CONFIG					0x2300
+
+#define	SQC_CACHES					0x2302
+
+#define SQ_POWER_THROTTLE                               0x2396
+#define		MIN_POWER(x)				((x) << 0)
+#define		MIN_POWER_MASK				(0x3fff << 0)
+#define		MIN_POWER_SHIFT				0
+#define		MAX_POWER(x)				((x) << 16)
+#define		MAX_POWER_MASK				(0x3fff << 16)
+#define		MAX_POWER_SHIFT				0
+#define SQ_POWER_THROTTLE2                              0x2397
+#define		MAX_POWER_DELTA(x)			((x) << 0)
+#define		MAX_POWER_DELTA_MASK			(0x3fff << 0)
+#define		MAX_POWER_DELTA_SHIFT			0
+#define		STI_SIZE(x)				((x) << 16)
+#define		STI_SIZE_MASK				(0x3ff << 16)
+#define		STI_SIZE_SHIFT				16
+#define		LTI_RATIO(x)				((x) << 27)
+#define		LTI_RATIO_MASK				(0xf << 27)
+#define		LTI_RATIO_SHIFT				27
+
+#define	SX_DEBUG_1					0x2418
+
+#define	SPI_STATIC_THREAD_MGMT_1			0x2438
+#define	SPI_STATIC_THREAD_MGMT_2			0x2439
+#define	SPI_STATIC_THREAD_MGMT_3			0x243A
+#define	SPI_PS_MAX_WAVE_ID				0x243B
+
+#define	SPI_CONFIG_CNTL					0x2440
+
+#define	SPI_CONFIG_CNTL_1				0x244F
+#define		VTX_DONE_DELAY(x)				((x) << 0)
+#define		INTERP_ONE_PRIM_PER_ROW				(1 << 4)
+
+#define	CGTS_TCC_DISABLE				0x2452
+#define	CGTS_USER_TCC_DISABLE				0x2453
+#define		TCC_DISABLE_MASK				0xFFFF0000
+#define		TCC_DISABLE_SHIFT				16
+#define	CGTS_SM_CTRL_REG				0x2454
+#define		OVERRIDE				(1 << 21)
+#define		LS_OVERRIDE				(1 << 22)
+
+#define	SPI_LB_CU_MASK					0x24D5
+
+#define	TA_CNTL_AUX					0x2542
+
+#define CC_RB_BACKEND_DISABLE				0x263D
+#define		BACKEND_DISABLE(x)     			((x) << 16)
+#define GB_ADDR_CONFIG  				0x263E
+#define		NUM_PIPES(x)				((x) << 0)
+#define		NUM_PIPES_MASK				0x00000007
+#define		NUM_PIPES_SHIFT				0
+#define		PIPE_INTERLEAVE_SIZE(x)			((x) << 4)
+#define		PIPE_INTERLEAVE_SIZE_MASK		0x00000070
+#define		PIPE_INTERLEAVE_SIZE_SHIFT		4
+#define		NUM_SHADER_ENGINES(x)			((x) << 12)
+#define		NUM_SHADER_ENGINES_MASK			0x00003000
+#define		NUM_SHADER_ENGINES_SHIFT		12
+#define		SHADER_ENGINE_TILE_SIZE(x)     		((x) << 16)
+#define		SHADER_ENGINE_TILE_SIZE_MASK		0x00070000
+#define		SHADER_ENGINE_TILE_SIZE_SHIFT		16
+#define		NUM_GPUS(x)     			((x) << 20)
+#define		NUM_GPUS_MASK				0x00700000
+#define		NUM_GPUS_SHIFT				20
+#define		MULTI_GPU_TILE_SIZE(x)     		((x) << 24)
+#define		MULTI_GPU_TILE_SIZE_MASK		0x03000000
+#define		MULTI_GPU_TILE_SIZE_SHIFT		24
+#define		ROW_SIZE(x)             		((x) << 28)
+#define		ROW_SIZE_MASK				0x30000000
+#define		ROW_SIZE_SHIFT				28
+
+#define	GB_TILE_MODE0					0x2644
+#       define MICRO_TILE_MODE(x)				((x) << 0)
+#              define	ADDR_SURF_DISPLAY_MICRO_TILING		0
+#              define	ADDR_SURF_THIN_MICRO_TILING		1
+#              define	ADDR_SURF_DEPTH_MICRO_TILING		2
+#       define ARRAY_MODE(x)					((x) << 2)
+#              define	ARRAY_LINEAR_GENERAL			0
+#              define	ARRAY_LINEAR_ALIGNED			1
+#              define	ARRAY_1D_TILED_THIN1			2
+#              define	ARRAY_2D_TILED_THIN1			4
+#       define PIPE_CONFIG(x)					((x) << 6)
+#              define	ADDR_SURF_P2				0
+#              define	ADDR_SURF_P4_8x16			4
+#              define	ADDR_SURF_P4_16x16			5
+#              define	ADDR_SURF_P4_16x32			6
+#              define	ADDR_SURF_P4_32x32			7
+#              define	ADDR_SURF_P8_16x16_8x16			8
+#              define	ADDR_SURF_P8_16x32_8x16			9
+#              define	ADDR_SURF_P8_32x32_8x16			10
+#              define	ADDR_SURF_P8_16x32_16x16		11
+#              define	ADDR_SURF_P8_32x32_16x16		12
+#              define	ADDR_SURF_P8_32x32_16x32		13
+#              define	ADDR_SURF_P8_32x64_32x32		14
+#       define TILE_SPLIT(x)					((x) << 11)
+#              define	ADDR_SURF_TILE_SPLIT_64B		0
+#              define	ADDR_SURF_TILE_SPLIT_128B		1
+#              define	ADDR_SURF_TILE_SPLIT_256B		2
+#              define	ADDR_SURF_TILE_SPLIT_512B		3
+#              define	ADDR_SURF_TILE_SPLIT_1KB		4
+#              define	ADDR_SURF_TILE_SPLIT_2KB		5
+#              define	ADDR_SURF_TILE_SPLIT_4KB		6
+#       define BANK_WIDTH(x)					((x) << 14)
+#              define	ADDR_SURF_BANK_WIDTH_1			0
+#              define	ADDR_SURF_BANK_WIDTH_2			1
+#              define	ADDR_SURF_BANK_WIDTH_4			2
+#              define	ADDR_SURF_BANK_WIDTH_8			3
+#       define BANK_HEIGHT(x)					((x) << 16)
+#              define	ADDR_SURF_BANK_HEIGHT_1			0
+#              define	ADDR_SURF_BANK_HEIGHT_2			1
+#              define	ADDR_SURF_BANK_HEIGHT_4			2
+#              define	ADDR_SURF_BANK_HEIGHT_8			3
+#       define MACRO_TILE_ASPECT(x)				((x) << 18)
+#              define	ADDR_SURF_MACRO_ASPECT_1		0
+#              define	ADDR_SURF_MACRO_ASPECT_2		1
+#              define	ADDR_SURF_MACRO_ASPECT_4		2
+#              define	ADDR_SURF_MACRO_ASPECT_8		3
+#       define NUM_BANKS(x)					((x) << 20)
+#              define	ADDR_SURF_2_BANK			0
+#              define	ADDR_SURF_4_BANK			1
+#              define	ADDR_SURF_8_BANK			2
+#              define	ADDR_SURF_16_BANK			3
+#define	GB_TILE_MODE1					0x2645
+#define	GB_TILE_MODE2					0x2646
+#define	GB_TILE_MODE3					0x2647
+#define	GB_TILE_MODE4					0x2648
+#define	GB_TILE_MODE5					0x2649
+#define	GB_TILE_MODE6					0x264a
+#define	GB_TILE_MODE7					0x264b
+#define	GB_TILE_MODE8					0x264c
+#define	GB_TILE_MODE9					0x264d
+#define	GB_TILE_MODE10					0x264e
+#define	GB_TILE_MODE11					0x264f
+#define	GB_TILE_MODE12					0x2650
+#define	GB_TILE_MODE13					0x2651
+#define	GB_TILE_MODE14					0x2652
+#define	GB_TILE_MODE15					0x2653
+#define	GB_TILE_MODE16					0x2654
+#define	GB_TILE_MODE17					0x2655
+#define	GB_TILE_MODE18					0x2656
+#define	GB_TILE_MODE19					0x2657
+#define	GB_TILE_MODE20					0x2658
+#define	GB_TILE_MODE21					0x2659
+#define	GB_TILE_MODE22					0x265a
+#define	GB_TILE_MODE23					0x265b
+#define	GB_TILE_MODE24					0x265c
+#define	GB_TILE_MODE25					0x265d
+#define	GB_TILE_MODE26					0x265e
+#define	GB_TILE_MODE27					0x265f
+#define	GB_TILE_MODE28					0x2660
+#define	GB_TILE_MODE29					0x2661
+#define	GB_TILE_MODE30					0x2662
+#define	GB_TILE_MODE31					0x2663
+
+#define	CB_PERFCOUNTER0_SELECT0				0x2688
+#define	CB_PERFCOUNTER0_SELECT1				0x2689
+#define	CB_PERFCOUNTER1_SELECT0				0x268A
+#define	CB_PERFCOUNTER1_SELECT1				0x268B
+#define	CB_PERFCOUNTER2_SELECT0				0x268C
+#define	CB_PERFCOUNTER2_SELECT1				0x268D
+#define	CB_PERFCOUNTER3_SELECT0				0x268E
+#define	CB_PERFCOUNTER3_SELECT1				0x268F
+
+#define	CB_CGTT_SCLK_CTRL				0x2698
+
+#define	GC_USER_RB_BACKEND_DISABLE			0x26DF
+#define		BACKEND_DISABLE_MASK			0x00FF0000
+#define		BACKEND_DISABLE_SHIFT			16
+
+#define	TCP_CHAN_STEER_LO				0x2B03
+#define	TCP_CHAN_STEER_HI				0x2B94
+
+#define	CP_RB0_BASE					0x3040
+#define	CP_RB0_CNTL					0x3041
+#define		RB_BUFSZ(x)					((x) << 0)
+#define		RB_BLKSZ(x)					((x) << 8)
+#define		BUF_SWAP_32BIT					(2 << 16)
+#define		RB_NO_UPDATE					(1 << 27)
+#define		RB_RPTR_WR_ENA					(1 << 31)
+
+#define	CP_RB0_RPTR_ADDR				0x3043
+#define	CP_RB0_RPTR_ADDR_HI				0x3044
+#define	CP_RB0_WPTR					0x3045
+
+#define	CP_PFP_UCODE_ADDR				0x3054
+#define	CP_PFP_UCODE_DATA				0x3055
+#define	CP_ME_RAM_RADDR					0x3056
+#define	CP_ME_RAM_WADDR					0x3057
+#define	CP_ME_RAM_DATA					0x3058
+
+#define	CP_CE_UCODE_ADDR				0x305A
+#define	CP_CE_UCODE_DATA				0x305B
+
+#define	CP_RB1_BASE					0x3060
+#define	CP_RB1_CNTL					0x3061
+#define	CP_RB1_RPTR_ADDR				0x3062
+#define	CP_RB1_RPTR_ADDR_HI				0x3063
+#define	CP_RB1_WPTR					0x3064
+#define	CP_RB2_BASE					0x3065
+#define	CP_RB2_CNTL					0x3066
+#define	CP_RB2_RPTR_ADDR				0x3067
+#define	CP_RB2_RPTR_ADDR_HI				0x3068
+#define	CP_RB2_WPTR					0x3069
+#define CP_INT_CNTL_RING0                               0x306A
+#define CP_INT_CNTL_RING1                               0x306B
+#define CP_INT_CNTL_RING2                               0x306C
+#       define CNTX_BUSY_INT_ENABLE                     (1 << 19)
+#       define CNTX_EMPTY_INT_ENABLE                    (1 << 20)
+#       define WAIT_MEM_SEM_INT_ENABLE                  (1 << 21)
+#       define TIME_STAMP_INT_ENABLE                    (1 << 26)
+#       define CP_RINGID2_INT_ENABLE                    (1 << 29)
+#       define CP_RINGID1_INT_ENABLE                    (1 << 30)
+#       define CP_RINGID0_INT_ENABLE                    (1 << 31)
+#define CP_INT_STATUS_RING0                             0x306D
+#define CP_INT_STATUS_RING1                             0x306E
+#define CP_INT_STATUS_RING2                             0x306F
+#       define WAIT_MEM_SEM_INT_STAT                    (1 << 21)
+#       define TIME_STAMP_INT_STAT                      (1 << 26)
+#       define CP_RINGID2_INT_STAT                      (1 << 29)
+#       define CP_RINGID1_INT_STAT                      (1 << 30)
+#       define CP_RINGID0_INT_STAT                      (1 << 31)
+
+#define	CP_MEM_SLP_CNTL					0x3079
+#       define CP_MEM_LS_EN                             (1 << 0)
+
+#define	CP_DEBUG					0x307F
+
+#define RLC_CNTL                                          0x30C0
+#       define RLC_ENABLE                                 (1 << 0)
+#define RLC_RL_BASE                                       0x30C1
+#define RLC_RL_SIZE                                       0x30C2
+#define RLC_LB_CNTL                                       0x30C3
+#       define LOAD_BALANCE_ENABLE                        (1 << 0)
+#define RLC_SAVE_AND_RESTORE_BASE                         0x30C4
+#define RLC_LB_CNTR_MAX                                   0x30C5
+#define RLC_LB_CNTR_INIT                                  0x30C6
+
+#define RLC_CLEAR_STATE_RESTORE_BASE                      0x30C8
+
+#define RLC_UCODE_ADDR                                    0x30CB
+#define RLC_UCODE_DATA                                    0x30CC
+
+#define RLC_GPU_CLOCK_COUNT_LSB                           0x30CE
+#define RLC_GPU_CLOCK_COUNT_MSB                           0x30CF
+#define RLC_CAPTURE_GPU_CLOCK_COUNT                       0x30D0
+#define RLC_MC_CNTL                                       0x30D1
+#define RLC_UCODE_CNTL                                    0x30D2
+#define RLC_STAT                                          0x30D3
+#       define RLC_BUSY_STATUS                            (1 << 0)
+#       define GFX_POWER_STATUS                           (1 << 1)
+#       define GFX_CLOCK_STATUS                           (1 << 2)
+#       define GFX_LS_STATUS                              (1 << 3)
+
+#define	RLC_PG_CNTL					0x30D7
+#	define GFX_PG_ENABLE				(1 << 0)
+#	define GFX_PG_SRC				(1 << 1)
+
+#define	RLC_CGTT_MGCG_OVERRIDE				0x3100
+#define	RLC_CGCG_CGLS_CTRL				0x3101
+#	define CGCG_EN					(1 << 0)
+#	define CGLS_EN					(1 << 1)
+
+#define	RLC_TTOP_D					0x3105
+#	define RLC_PUD(x)				((x) << 0)
+#	define RLC_PUD_MASK				(0xff << 0)
+#	define RLC_PDD(x)				((x) << 8)
+#	define RLC_PDD_MASK				(0xff << 8)
+#	define RLC_TTPD(x)				((x) << 16)
+#	define RLC_TTPD_MASK				(0xff << 16)
+#	define RLC_MSD(x)				((x) << 24)
+#	define RLC_MSD_MASK				(0xff << 24)
+
+#define RLC_LB_INIT_CU_MASK                               0x3107
+
+#define	RLC_PG_AO_CU_MASK				0x310B
+#define	RLC_MAX_PG_CU					0x310C
+#	define MAX_PU_CU(x)				((x) << 0)
+#	define MAX_PU_CU_MASK				(0xff << 0)
+#define	RLC_AUTO_PG_CTRL				0x310C
+#	define AUTO_PG_EN				(1 << 0)
+#	define GRBM_REG_SGIT(x)				((x) << 3)
+#	define GRBM_REG_SGIT_MASK			(0xffff << 3)
+#	define PG_AFTER_GRBM_REG_ST(x)			((x) << 19)
+#	define PG_AFTER_GRBM_REG_ST_MASK		(0x1fff << 19)
+
+#define RLC_SERDES_WR_MASTER_MASK_0                       0x3115
+#define RLC_SERDES_WR_MASTER_MASK_1                       0x3116
+#define RLC_SERDES_WR_CTRL                                0x3117
+
+#define RLC_SERDES_MASTER_BUSY_0                          0x3119
+#define RLC_SERDES_MASTER_BUSY_1                          0x311A
+
+#define RLC_GCPM_GENERAL_3                                0x311E
+
+#define	DB_RENDER_CONTROL				0xA000
+
+#define DB_DEPTH_INFO                                   0xA00F
+
+#define PA_SC_RASTER_CONFIG                             0xA0D4
+#	define RB_MAP_PKR0(x)				((x) << 0)
+#	define RB_MAP_PKR0_MASK				(0x3 << 0)
+#	define RB_MAP_PKR1(x)				((x) << 2)
+#	define RB_MAP_PKR1_MASK				(0x3 << 2)
+#       define RASTER_CONFIG_RB_MAP_0                   0
+#       define RASTER_CONFIG_RB_MAP_1                   1
+#       define RASTER_CONFIG_RB_MAP_2                   2
+#       define RASTER_CONFIG_RB_MAP_3                   3
+#	define RB_XSEL2(x)				((x) << 4)
+#	define RB_XSEL2_MASK				(0x3 << 4)
+#	define RB_XSEL					(1 << 6)
+#	define RB_YSEL					(1 << 7)
+#	define PKR_MAP(x)				((x) << 8)
+#	define PKR_MAP_MASK				(0x3 << 8)
+#       define RASTER_CONFIG_PKR_MAP_0			0
+#       define RASTER_CONFIG_PKR_MAP_1			1
+#       define RASTER_CONFIG_PKR_MAP_2			2
+#       define RASTER_CONFIG_PKR_MAP_3			3
+#	define PKR_XSEL(x)				((x) << 10)
+#	define PKR_XSEL_MASK				(0x3 << 10)
+#	define PKR_YSEL(x)				((x) << 12)
+#	define PKR_YSEL_MASK				(0x3 << 12)
+#	define SC_MAP(x)				((x) << 16)
+#	define SC_MAP_MASK				(0x3 << 16)
+#	define SC_XSEL(x)				((x) << 18)
+#	define SC_XSEL_MASK				(0x3 << 18)
+#	define SC_YSEL(x)				((x) << 20)
+#	define SC_YSEL_MASK				(0x3 << 20)
+#	define SE_MAP(x)				((x) << 24)
+#	define SE_MAP_MASK				(0x3 << 24)
+#       define RASTER_CONFIG_SE_MAP_0			0
+#       define RASTER_CONFIG_SE_MAP_1			1
+#       define RASTER_CONFIG_SE_MAP_2			2
+#       define RASTER_CONFIG_SE_MAP_3			3
+#	define SE_XSEL(x)				((x) << 26)
+#	define SE_XSEL_MASK				(0x3 << 26)
+#	define SE_YSEL(x)				((x) << 28)
+#	define SE_YSEL_MASK				(0x3 << 28)
+
+
+#define VGT_EVENT_INITIATOR                             0xA2A4
+#       define SAMPLE_STREAMOUTSTATS1                   (1 << 0)
+#       define SAMPLE_STREAMOUTSTATS2                   (2 << 0)
+#       define SAMPLE_STREAMOUTSTATS3                   (3 << 0)
+#       define CACHE_FLUSH_TS                           (4 << 0)
+#       define CACHE_FLUSH                              (6 << 0)
+#       define CS_PARTIAL_FLUSH                         (7 << 0)
+#       define VGT_STREAMOUT_RESET                      (10 << 0)
+#       define END_OF_PIPE_INCR_DE                      (11 << 0)
+#       define END_OF_PIPE_IB_END                       (12 << 0)
+#       define RST_PIX_CNT                              (13 << 0)
+#       define VS_PARTIAL_FLUSH                         (15 << 0)
+#       define PS_PARTIAL_FLUSH                         (16 << 0)
+#       define CACHE_FLUSH_AND_INV_TS_EVENT             (20 << 0)
+#       define ZPASS_DONE                               (21 << 0)
+#       define CACHE_FLUSH_AND_INV_EVENT                (22 << 0)
+#       define PERFCOUNTER_START                        (23 << 0)
+#       define PERFCOUNTER_STOP                         (24 << 0)
+#       define PIPELINESTAT_START                       (25 << 0)
+#       define PIPELINESTAT_STOP                        (26 << 0)
+#       define PERFCOUNTER_SAMPLE                       (27 << 0)
+#       define SAMPLE_PIPELINESTAT                      (30 << 0)
+#       define SAMPLE_STREAMOUTSTATS                    (32 << 0)
+#       define RESET_VTX_CNT                            (33 << 0)
+#       define VGT_FLUSH                                (36 << 0)
+#       define BOTTOM_OF_PIPE_TS                        (40 << 0)
+#       define DB_CACHE_FLUSH_AND_INV                   (42 << 0)
+#       define FLUSH_AND_INV_DB_DATA_TS                 (43 << 0)
+#       define FLUSH_AND_INV_DB_META                    (44 << 0)
+#       define FLUSH_AND_INV_CB_DATA_TS                 (45 << 0)
+#       define FLUSH_AND_INV_CB_META                    (46 << 0)
+#       define CS_DONE                                  (47 << 0)
+#       define PS_DONE                                  (48 << 0)
+#       define FLUSH_AND_INV_CB_PIXEL_DATA              (49 << 0)
+#       define THREAD_TRACE_START                       (51 << 0)
+#       define THREAD_TRACE_STOP                        (52 << 0)
+#       define THREAD_TRACE_FLUSH                       (54 << 0)
+#       define THREAD_TRACE_FINISH                      (55 << 0)
+
+/* PIF PHY0 registers idx/data 0x8/0xc */
+#define PB0_PIF_CNTL                                      0x10
+#       define LS2_EXIT_TIME(x)                           ((x) << 17)
+#       define LS2_EXIT_TIME_MASK                         (0x7 << 17)
+#       define LS2_EXIT_TIME_SHIFT                        17
+#define PB0_PIF_PAIRING                                   0x11
+#       define MULTI_PIF                                  (1 << 25)
+#define PB0_PIF_PWRDOWN_0                                 0x12
+#       define PLL_POWER_STATE_IN_TXS2_0(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_0_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_0_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_0(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_0_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_0_SHIFT             10
+#       define PLL_RAMP_UP_TIME_0(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_0_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_0_SHIFT                   24
+#define PB0_PIF_PWRDOWN_1                                 0x13
+#       define PLL_POWER_STATE_IN_TXS2_1(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_1_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_1_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_1(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_1_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_1_SHIFT             10
+#       define PLL_RAMP_UP_TIME_1(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_1_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_1_SHIFT                   24
+
+#define PB0_PIF_PWRDOWN_2                                 0x17
+#       define PLL_POWER_STATE_IN_TXS2_2(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_2_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_2_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_2(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_2_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_2_SHIFT             10
+#       define PLL_RAMP_UP_TIME_2(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_2_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_2_SHIFT                   24
+#define PB0_PIF_PWRDOWN_3                                 0x18
+#       define PLL_POWER_STATE_IN_TXS2_3(x)               ((x) << 7)
+#       define PLL_POWER_STATE_IN_TXS2_3_MASK             (0x7 << 7)
+#       define PLL_POWER_STATE_IN_TXS2_3_SHIFT            7
+#       define PLL_POWER_STATE_IN_OFF_3(x)                ((x) << 10)
+#       define PLL_POWER_STATE_IN_OFF_3_MASK              (0x7 << 10)
+#       define PLL_POWER_STATE_IN_OFF_3_SHIFT             10
+#       define PLL_RAMP_UP_TIME_3(x)                      ((x) << 24)
+#       define PLL_RAMP_UP_TIME_3_MASK                    (0x7 << 24)
+#       define PLL_RAMP_UP_TIME_3_SHIFT                   24
+/* PIF PHY1 registers idx/data 0x10/0x14 */
+#define PB1_PIF_CNTL                                      0x10
+#define PB1_PIF_PAIRING                                   0x11
+#define PB1_PIF_PWRDOWN_0                                 0x12
+#define PB1_PIF_PWRDOWN_1                                 0x13
+
+#define PB1_PIF_PWRDOWN_2                                 0x17
+#define PB1_PIF_PWRDOWN_3                                 0x18
+/* PCIE registers idx/data 0x30/0x34 */
+#define PCIE_CNTL2                                        0x1c /* PCIE */
+#       define SLV_MEM_LS_EN                              (1 << 16)
+#       define SLV_MEM_AGGRESSIVE_LS_EN                   (1 << 17)
+#       define MST_MEM_LS_EN                              (1 << 18)
+#       define REPLAY_MEM_LS_EN                           (1 << 19)
+#define PCIE_LC_STATUS1                                   0x28 /* PCIE */
+#       define LC_REVERSE_RCVR                            (1 << 0)
+#       define LC_REVERSE_XMIT                            (1 << 1)
+#       define LC_OPERATING_LINK_WIDTH_MASK               (0x7 << 2)
+#       define LC_OPERATING_LINK_WIDTH_SHIFT              2
+#       define LC_DETECTED_LINK_WIDTH_MASK                (0x7 << 5)
+#       define LC_DETECTED_LINK_WIDTH_SHIFT               5
+
+#define PCIE_P_CNTL                                       0x40 /* PCIE */
+#       define P_IGNORE_EDB_ERR                           (1 << 6)
+
+/* PCIE PORT registers idx/data 0x38/0x3c */
+#define PCIE_LC_CNTL                                      0xa0
+#       define LC_L0S_INACTIVITY(x)                       ((x) << 8)
+#       define LC_L0S_INACTIVITY_MASK                     (0xf << 8)
+#       define LC_L0S_INACTIVITY_SHIFT                    8
+#       define LC_L1_INACTIVITY(x)                        ((x) << 12)
+#       define LC_L1_INACTIVITY_MASK                      (0xf << 12)
+#       define LC_L1_INACTIVITY_SHIFT                     12
+#       define LC_PMI_TO_L1_DIS                           (1 << 16)
+#       define LC_ASPM_TO_L1_DIS                          (1 << 24)
+#define PCIE_LC_LINK_WIDTH_CNTL                           0xa2 /* PCIE_P */
+#       define LC_LINK_WIDTH_SHIFT                        0
+#       define LC_LINK_WIDTH_MASK                         0x7
+#       define LC_LINK_WIDTH_X0                           0
+#       define LC_LINK_WIDTH_X1                           1
+#       define LC_LINK_WIDTH_X2                           2
+#       define LC_LINK_WIDTH_X4                           3
+#       define LC_LINK_WIDTH_X8                           4
+#       define LC_LINK_WIDTH_X16                          6
+#       define LC_LINK_WIDTH_RD_SHIFT                     4
+#       define LC_LINK_WIDTH_RD_MASK                      0x70
+#       define LC_RECONFIG_ARC_MISSING_ESCAPE             (1 << 7)
+#       define LC_RECONFIG_NOW                            (1 << 8)
+#       define LC_RENEGOTIATION_SUPPORT                   (1 << 9)
+#       define LC_RENEGOTIATE_EN                          (1 << 10)
+#       define LC_SHORT_RECONFIG_EN                       (1 << 11)
+#       define LC_UPCONFIGURE_SUPPORT                     (1 << 12)
+#       define LC_UPCONFIGURE_DIS                         (1 << 13)
+#       define LC_DYN_LANES_PWR_STATE(x)                  ((x) << 21)
+#       define LC_DYN_LANES_PWR_STATE_MASK                (0x3 << 21)
+#       define LC_DYN_LANES_PWR_STATE_SHIFT               21
+#define PCIE_LC_N_FTS_CNTL                                0xa3 /* PCIE_P */
+#       define LC_XMIT_N_FTS(x)                           ((x) << 0)
+#       define LC_XMIT_N_FTS_MASK                         (0xff << 0)
+#       define LC_XMIT_N_FTS_SHIFT                        0
+#       define LC_XMIT_N_FTS_OVERRIDE_EN                  (1 << 8)
+#       define LC_N_FTS_MASK                              (0xff << 24)
+#define PCIE_LC_SPEED_CNTL                                0xa4 /* PCIE_P */
+#       define LC_GEN2_EN_STRAP                           (1 << 0)
+#       define LC_GEN3_EN_STRAP                           (1 << 1)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_EN           (1 << 2)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_MASK         (0x3 << 3)
+#       define LC_TARGET_LINK_SPEED_OVERRIDE_SHIFT        3
+#       define LC_FORCE_EN_SW_SPEED_CHANGE                (1 << 5)
+#       define LC_FORCE_DIS_SW_SPEED_CHANGE               (1 << 6)
+#       define LC_FORCE_EN_HW_SPEED_CHANGE                (1 << 7)
+#       define LC_FORCE_DIS_HW_SPEED_CHANGE               (1 << 8)
+#       define LC_INITIATE_LINK_SPEED_CHANGE              (1 << 9)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK      (0x3 << 10)
+#       define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT     10
+#       define LC_CURRENT_DATA_RATE_MASK                  (0x3 << 13) /* 0/1/2 = gen1/2/3 */
+#       define LC_CURRENT_DATA_RATE_SHIFT                 13
+#       define LC_CLR_FAILED_SPD_CHANGE_CNT               (1 << 16)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN2               (1 << 18)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN2                (1 << 19)
+#       define LC_OTHER_SIDE_EVER_SENT_GEN3               (1 << 20)
+#       define LC_OTHER_SIDE_SUPPORTS_GEN3                (1 << 21)
+
+#define PCIE_LC_CNTL2                                     0xb1
+#       define LC_ALLOW_PDWN_IN_L1                        (1 << 17)
+#       define LC_ALLOW_PDWN_IN_L23                       (1 << 18)
+
+#define PCIE_LC_CNTL3                                     0xb5 /* PCIE_P */
+#       define LC_GO_TO_RECOVERY                          (1 << 30)
+#define PCIE_LC_CNTL4                                     0xb6 /* PCIE_P */
+#       define LC_REDO_EQ                                 (1 << 5)
+#       define LC_SET_QUIESCE                             (1 << 13)
+
+/*
+ * UVD
+ */
+#define UVD_UDEC_ADDR_CONFIG				0x3bd3
+#define UVD_UDEC_DB_ADDR_CONFIG				0x3bd4
+#define UVD_UDEC_DBW_ADDR_CONFIG			0x3bd5
+#define UVD_RBC_RB_RPTR					0x3da4
+#define UVD_RBC_RB_WPTR					0x3da5
+#define UVD_STATUS					0x3daf
+
+#define	UVD_CGC_CTRL					0x3dc2
+#	define DCM					(1 << 0)
+#	define CG_DT(x)					((x) << 2)
+#	define CG_DT_MASK				(0xf << 2)
+#	define CLK_OD(x)				((x) << 6)
+#	define CLK_OD_MASK				(0x1f << 6)
+
+ /* UVD CTX indirect */
+#define	UVD_CGC_MEM_CTRL				0xC0
+#define	UVD_CGC_CTRL2					0xC1
+#	define DYN_OR_EN				(1 << 0)
+#	define DYN_RR_EN				(1 << 1)
+#	define G_DIV_ID(x)				((x) << 2)
+#	define G_DIV_ID_MASK				(0x7 << 2)
+
+/*
+ * PM4
+ */
+#define PACKET0(reg, n)	((RADEON_PACKET_TYPE0 << 30) |			\
+			 (((reg) >> 2) & 0xFFFF) |			\
+			 ((n) & 0x3FFF) << 16)
+#define CP_PACKET2			0x80000000
+#define		PACKET2_PAD_SHIFT		0
+#define		PACKET2_PAD_MASK		(0x3fffffff << 0)
+
+#define PACKET2(v)	(CP_PACKET2 | REG_SET(PACKET2_PAD, (v)))
+#define RADEON_PACKET_TYPE3 3
+#define PACKET3(op, n)	((RADEON_PACKET_TYPE3 << 30) |			\
+			 (((op) & 0xFF) << 8) |				\
+			 ((n) & 0x3FFF) << 16)
+
+#define PACKET3_COMPUTE(op, n) (PACKET3(op, n) | 1 << 1)
+
+/* Packet 3 types */
+#define	PACKET3_NOP					0x10
+#define	PACKET3_SET_BASE				0x11
+#define		PACKET3_BASE_INDEX(x)                  ((x) << 0)
+#define			GDS_PARTITION_BASE		2
+#define			CE_PARTITION_BASE		3
+#define	PACKET3_CLEAR_STATE				0x12
+#define	PACKET3_INDEX_BUFFER_SIZE			0x13
+#define	PACKET3_DISPATCH_DIRECT				0x15
+#define	PACKET3_DISPATCH_INDIRECT			0x16
+#define	PACKET3_ALLOC_GDS				0x1B
+#define	PACKET3_WRITE_GDS_RAM				0x1C
+#define	PACKET3_ATOMIC_GDS				0x1D
+#define	PACKET3_ATOMIC					0x1E
+#define	PACKET3_OCCLUSION_QUERY				0x1F
+#define	PACKET3_SET_PREDICATION				0x20
+#define	PACKET3_REG_RMW					0x21
+#define	PACKET3_COND_EXEC				0x22
+#define	PACKET3_PRED_EXEC				0x23
+#define	PACKET3_DRAW_INDIRECT				0x24
+#define	PACKET3_DRAW_INDEX_INDIRECT			0x25
+#define	PACKET3_INDEX_BASE				0x26
+#define	PACKET3_DRAW_INDEX_2				0x27
+#define	PACKET3_CONTEXT_CONTROL				0x28
+#define	PACKET3_INDEX_TYPE				0x2A
+#define	PACKET3_DRAW_INDIRECT_MULTI			0x2C
+#define	PACKET3_DRAW_INDEX_AUTO				0x2D
+#define	PACKET3_DRAW_INDEX_IMMD				0x2E
+#define	PACKET3_NUM_INSTANCES				0x2F
+#define	PACKET3_DRAW_INDEX_MULTI_AUTO			0x30
+#define	PACKET3_INDIRECT_BUFFER_CONST			0x31
+#define	PACKET3_INDIRECT_BUFFER				0x3F
+#define	PACKET3_STRMOUT_BUFFER_UPDATE			0x34
+#define	PACKET3_DRAW_INDEX_OFFSET_2			0x35
+#define	PACKET3_DRAW_INDEX_MULTI_ELEMENT		0x36
+#define	PACKET3_WRITE_DATA				0x37
+#define		WRITE_DATA_DST_SEL(x)                   ((x) << 8)
+                /* 0 - register
+		 * 1 - memory (sync - via GRBM)
+		 * 2 - tc/l2
+		 * 3 - gds
+		 * 4 - reserved
+		 * 5 - memory (async - direct)
+		 */
+#define		WR_ONE_ADDR                             (1 << 16)
+#define		WR_CONFIRM                              (1 << 20)
+#define		WRITE_DATA_ENGINE_SEL(x)                ((x) << 30)
+                /* 0 - me
+		 * 1 - pfp
+		 * 2 - ce
+		 */
+#define	PACKET3_DRAW_INDEX_INDIRECT_MULTI		0x38
+#define	PACKET3_MEM_SEMAPHORE				0x39
+#define	PACKET3_MPEG_INDEX				0x3A
+#define	PACKET3_COPY_DW					0x3B
+#define	PACKET3_WAIT_REG_MEM				0x3C
+#define		WAIT_REG_MEM_FUNCTION(x)                ((x) << 0)
+                /* 0 - always
+		 * 1 - <
+		 * 2 - <=
+		 * 3 - ==
+		 * 4 - !=
+		 * 5 - >=
+		 * 6 - >
+		 */
+#define		WAIT_REG_MEM_MEM_SPACE(x)               ((x) << 4)
+                /* 0 - reg
+		 * 1 - mem
+		 */
+#define		WAIT_REG_MEM_ENGINE(x)                  ((x) << 8)
+                /* 0 - me
+		 * 1 - pfp
+		 */
+#define	PACKET3_MEM_WRITE				0x3D
+#define	PACKET3_COPY_DATA				0x40
+#define	PACKET3_CP_DMA					0x41
+/* 1. header
+ * 2. SRC_ADDR_LO or DATA [31:0]
+ * 3. CP_SYNC [31] | SRC_SEL [30:29] | ENGINE [27] | DST_SEL [21:20] |
+ *    SRC_ADDR_HI [7:0]
+ * 4. DST_ADDR_LO [31:0]
+ * 5. DST_ADDR_HI [7:0]
+ * 6. COMMAND [30:21] | BYTE_COUNT [20:0]
+ */
+#              define PACKET3_CP_DMA_DST_SEL(x)    ((x) << 20)
+                /* 0 - DST_ADDR
+		 * 1 - GDS
+		 */
+#              define PACKET3_CP_DMA_ENGINE(x)     ((x) << 27)
+                /* 0 - ME
+		 * 1 - PFP
+		 */
+#              define PACKET3_CP_DMA_SRC_SEL(x)    ((x) << 29)
+                /* 0 - SRC_ADDR
+		 * 1 - GDS
+		 * 2 - DATA
+		 */
+#              define PACKET3_CP_DMA_CP_SYNC       (1 << 31)
+/* COMMAND */
+#              define PACKET3_CP_DMA_DIS_WC        (1 << 21)
+#              define PACKET3_CP_DMA_CMD_SRC_SWAP(x) ((x) << 22)
+                /* 0 - none
+		 * 1 - 8 in 16
+		 * 2 - 8 in 32
+		 * 3 - 8 in 64
+		 */
+#              define PACKET3_CP_DMA_CMD_DST_SWAP(x) ((x) << 24)
+                /* 0 - none
+		 * 1 - 8 in 16
+		 * 2 - 8 in 32
+		 * 3 - 8 in 64
+		 */
+#              define PACKET3_CP_DMA_CMD_SAS       (1 << 26)
+                /* 0 - memory
+		 * 1 - register
+		 */
+#              define PACKET3_CP_DMA_CMD_DAS       (1 << 27)
+                /* 0 - memory
+		 * 1 - register
+		 */
+#              define PACKET3_CP_DMA_CMD_SAIC      (1 << 28)
+#              define PACKET3_CP_DMA_CMD_DAIC      (1 << 29)
+#              define PACKET3_CP_DMA_CMD_RAW_WAIT  (1 << 30)
+#define	PACKET3_PFP_SYNC_ME				0x42
+#define	PACKET3_SURFACE_SYNC				0x43
+#              define PACKET3_DEST_BASE_0_ENA      (1 << 0)
+#              define PACKET3_DEST_BASE_1_ENA      (1 << 1)
+#              define PACKET3_CB0_DEST_BASE_ENA    (1 << 6)
+#              define PACKET3_CB1_DEST_BASE_ENA    (1 << 7)
+#              define PACKET3_CB2_DEST_BASE_ENA    (1 << 8)
+#              define PACKET3_CB3_DEST_BASE_ENA    (1 << 9)
+#              define PACKET3_CB4_DEST_BASE_ENA    (1 << 10)
+#              define PACKET3_CB5_DEST_BASE_ENA    (1 << 11)
+#              define PACKET3_CB6_DEST_BASE_ENA    (1 << 12)
+#              define PACKET3_CB7_DEST_BASE_ENA    (1 << 13)
+#              define PACKET3_DB_DEST_BASE_ENA     (1 << 14)
+#              define PACKET3_DEST_BASE_2_ENA      (1 << 19)
+#              define PACKET3_DEST_BASE_3_ENA      (1 << 21)
+#              define PACKET3_TCL1_ACTION_ENA      (1 << 22)
+#              define PACKET3_TC_ACTION_ENA        (1 << 23)
+#              define PACKET3_CB_ACTION_ENA        (1 << 25)
+#              define PACKET3_DB_ACTION_ENA        (1 << 26)
+#              define PACKET3_SH_KCACHE_ACTION_ENA (1 << 27)
+#              define PACKET3_SH_ICACHE_ACTION_ENA (1 << 29)
+#define	PACKET3_ME_INITIALIZE				0x44
+#define		PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16)
+#define	PACKET3_COND_WRITE				0x45
+#define	PACKET3_EVENT_WRITE				0x46
+#define		EVENT_TYPE(x)                           ((x) << 0)
+#define		EVENT_INDEX(x)                          ((x) << 8)
+                /* 0 - any non-TS event
+		 * 1 - ZPASS_DONE
+		 * 2 - SAMPLE_PIPELINESTAT
+		 * 3 - SAMPLE_STREAMOUTSTAT*
+		 * 4 - *S_PARTIAL_FLUSH
+		 * 5 - EOP events
+		 * 6 - EOS events
+		 * 7 - CACHE_FLUSH, CACHE_FLUSH_AND_INV_EVENT
+		 */
+#define		INV_L2                                  (1 << 20)
+                /* INV TC L2 cache when EVENT_INDEX = 7 */
+#define	PACKET3_EVENT_WRITE_EOP				0x47
+#define		DATA_SEL(x)                             ((x) << 29)
+                /* 0 - discard
+		 * 1 - send low 32bit data
+		 * 2 - send 64bit data
+		 * 3 - send 64bit counter value
+		 */
+#define		INT_SEL(x)                              ((x) << 24)
+                /* 0 - none
+		 * 1 - interrupt only (DATA_SEL = 0)
+		 * 2 - interrupt when data write is confirmed
+		 */
+#define	PACKET3_EVENT_WRITE_EOS				0x48
+#define	PACKET3_PREAMBLE_CNTL				0x4A
+#              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
+#              define PACKET3_PREAMBLE_END_CLEAR_STATE       (3 << 28)
+#define	PACKET3_ONE_REG_WRITE				0x57
+#define	PACKET3_LOAD_CONFIG_REG				0x5F
+#define	PACKET3_LOAD_CONTEXT_REG			0x60
+#define	PACKET3_LOAD_SH_REG				0x61
+#define	PACKET3_SET_CONFIG_REG				0x68
+#define		PACKET3_SET_CONFIG_REG_START			0x00002000
+#define		PACKET3_SET_CONFIG_REG_END			0x00002c00
+#define	PACKET3_SET_CONTEXT_REG				0x69
+#define		PACKET3_SET_CONTEXT_REG_START			0x000a000
+#define		PACKET3_SET_CONTEXT_REG_END			0x000a400
+#define	PACKET3_SET_CONTEXT_REG_INDIRECT		0x73
+#define	PACKET3_SET_RESOURCE_INDIRECT			0x74
+#define	PACKET3_SET_SH_REG				0x76
+#define		PACKET3_SET_SH_REG_START			0x00002c00
+#define		PACKET3_SET_SH_REG_END				0x00003000
+#define	PACKET3_SET_SH_REG_OFFSET			0x77
+#define	PACKET3_ME_WRITE				0x7A
+#define	PACKET3_SCRATCH_RAM_WRITE			0x7D
+#define	PACKET3_SCRATCH_RAM_READ			0x7E
+#define	PACKET3_CE_WRITE				0x7F
+#define	PACKET3_LOAD_CONST_RAM				0x80
+#define	PACKET3_WRITE_CONST_RAM				0x81
+#define	PACKET3_WRITE_CONST_RAM_OFFSET			0x82
+#define	PACKET3_DUMP_CONST_RAM				0x83
+#define	PACKET3_INCREMENT_CE_COUNTER			0x84
+#define	PACKET3_INCREMENT_DE_COUNTER			0x85
+#define	PACKET3_WAIT_ON_CE_COUNTER			0x86
+#define	PACKET3_WAIT_ON_DE_COUNTER			0x87
+#define	PACKET3_WAIT_ON_DE_COUNTER_DIFF			0x88
+#define	PACKET3_SET_CE_DE_COUNTERS			0x89
+#define	PACKET3_WAIT_ON_AVAIL_BUFFER			0x8A
+#define	PACKET3_SWITCH_BUFFER				0x8B
+
+/* ASYNC DMA - first instance at 0xd000, second at 0xd800 */
+#define DMA0_REGISTER_OFFSET                              0x0 /* not a register */
+#define DMA1_REGISTER_OFFSET                              0x200 /* not a register */
+
+#define DMA_RB_CNTL                                       0x3400
+#       define DMA_RB_ENABLE                              (1 << 0)
+#       define DMA_RB_SIZE(x)                             ((x) << 1) /* log2 */
+#       define DMA_RB_SWAP_ENABLE                         (1 << 9) /* 8IN32 */
+#       define DMA_RPTR_WRITEBACK_ENABLE                  (1 << 12)
+#       define DMA_RPTR_WRITEBACK_SWAP_ENABLE             (1 << 13)  /* 8IN32 */
+#       define DMA_RPTR_WRITEBACK_TIMER(x)                ((x) << 16) /* log2 */
+#define DMA_RB_BASE                                       0x3401
+#define DMA_RB_RPTR                                       0x3402
+#define DMA_RB_WPTR                                       0x3403
+
+#define DMA_RB_RPTR_ADDR_HI                               0x3407
+#define DMA_RB_RPTR_ADDR_LO                               0x3408
+
+#define DMA_IB_CNTL                                       0x3409
+#       define DMA_IB_ENABLE                              (1 << 0)
+#       define DMA_IB_SWAP_ENABLE                         (1 << 4)
+#       define CMD_VMID_FORCE                             (1 << 31)
+#define DMA_IB_RPTR                                       0x340a
+#define DMA_CNTL                                          0x340b
+#       define TRAP_ENABLE                                (1 << 0)
+#       define SEM_INCOMPLETE_INT_ENABLE                  (1 << 1)
+#       define SEM_WAIT_INT_ENABLE                        (1 << 2)
+#       define DATA_SWAP_ENABLE                           (1 << 3)
+#       define FENCE_SWAP_ENABLE                          (1 << 4)
+#       define CTXEMPTY_INT_ENABLE                        (1 << 28)
+#define DMA_STATUS_REG                                    0x340d
+#       define DMA_IDLE                                   (1 << 0)
+#define DMA_TILING_CONFIG  				  0x342e
+
+#define	DMA_POWER_CNTL					0x342f
+#       define MEM_POWER_OVERRIDE                       (1 << 8)
+#define	DMA_CLK_CTRL					0x3430
+
+#define	DMA_PG						0x3435
+#	define PG_CNTL_ENABLE				(1 << 0)
+#define	DMA_PGFSM_CONFIG				0x3436
+#define	DMA_PGFSM_WRITE					0x3437
+
+#define DMA_PACKET(cmd, b, t, s, n)	((((cmd) & 0xF) << 28) |	\
+					 (((b) & 0x1) << 26) |		\
+					 (((t) & 0x1) << 23) |		\
+					 (((s) & 0x1) << 22) |		\
+					 (((n) & 0xFFFFF) << 0))
+
+#define DMA_IB_PACKET(cmd, vmid, n)	((((cmd) & 0xF) << 28) |	\
+					 (((vmid) & 0xF) << 20) |	\
+					 (((n) & 0xFFFFF) << 0))
+
+#define DMA_PTE_PDE_PACKET(n)		((2 << 28) |			\
+					 (1 << 26) |			\
+					 (1 << 21) |			\
+					 (((n) & 0xFFFFF) << 0))
+
+/* async DMA Packet types */
+#define	DMA_PACKET_WRITE				  0x2
+#define	DMA_PACKET_COPY					  0x3
+#define	DMA_PACKET_INDIRECT_BUFFER			  0x4
+#define	DMA_PACKET_SEMAPHORE				  0x5
+#define	DMA_PACKET_FENCE				  0x6
+#define	DMA_PACKET_TRAP					  0x7
+#define	DMA_PACKET_SRBM_WRITE				  0x9
+#define	DMA_PACKET_CONSTANT_FILL			  0xd
+#define	DMA_PACKET_POLL_REG_MEM				  0xe
+#define	DMA_PACKET_NOP					  0xf
+
+#define VCE_STATUS					0x20004
+#define VCE_VCPU_CNTL					0x20014
+#define		VCE_CLK_EN				(1 << 0)
+#define VCE_VCPU_CACHE_OFFSET0				0x20024
+#define VCE_VCPU_CACHE_SIZE0				0x20028
+#define VCE_VCPU_CACHE_OFFSET1				0x2002c
+#define VCE_VCPU_CACHE_SIZE1				0x20030
+#define VCE_VCPU_CACHE_OFFSET2				0x20034
+#define VCE_VCPU_CACHE_SIZE2				0x20038
+#define VCE_SOFT_RESET					0x20120
+#define 	VCE_ECPU_SOFT_RESET			(1 << 0)
+#define 	VCE_FME_SOFT_RESET			(1 << 2)
+#define VCE_RB_BASE_LO2					0x2016c
+#define VCE_RB_BASE_HI2					0x20170
+#define VCE_RB_SIZE2					0x20174
+#define VCE_RB_RPTR2					0x20178
+#define VCE_RB_WPTR2					0x2017c
+#define VCE_RB_BASE_LO					0x20180
+#define VCE_RB_BASE_HI					0x20184
+#define VCE_RB_SIZE					0x20188
+#define VCE_RB_RPTR					0x2018c
+#define VCE_RB_WPTR					0x20190
+#define VCE_CLOCK_GATING_A				0x202f8
+#define VCE_CLOCK_GATING_B				0x202fc
+#define VCE_UENC_CLOCK_GATING				0x205bc
+#define VCE_UENC_REG_CLOCK_GATING			0x205c0
+#define VCE_FW_REG_STATUS				0x20e10
+#	define VCE_FW_REG_STATUS_BUSY			(1 << 0)
+#	define VCE_FW_REG_STATUS_PASS			(1 << 3)
+#	define VCE_FW_REG_STATUS_DONE			(1 << 11)
+#define VCE_LMI_FW_START_KEYSEL				0x20e18
+#define VCE_LMI_FW_PERIODIC_CTRL			0x20e20
+#define VCE_LMI_CTRL2					0x20e74
+#define VCE_LMI_CTRL					0x20e98
+#define VCE_LMI_VM_CTRL					0x20ea0
+#define VCE_LMI_SWAP_CNTL				0x20eb4
+#define VCE_LMI_SWAP_CNTL1				0x20eb8
+#define VCE_LMI_CACHE_CTRL				0x20ef4
+
+#define VCE_CMD_NO_OP					0x00000000
+#define VCE_CMD_END					0x00000001
+#define VCE_CMD_IB					0x00000002
+#define VCE_CMD_FENCE					0x00000003
+#define VCE_CMD_TRAP					0x00000004
+#define VCE_CMD_IB_AUTO					0x00000005
+#define VCE_CMD_SEMAPHORE				0x00000006
+
+
+//#dce stupp
+/* display controller offsets used for crtc/cur/lut/grph/viewport/etc. */
+#define SI_CRTC0_REGISTER_OFFSET                0 //(0x6df0 - 0x6df0)/4
+#define SI_CRTC1_REGISTER_OFFSET                0x300 //(0x79f0 - 0x6df0)/4
+#define SI_CRTC2_REGISTER_OFFSET                0x2600 //(0x105f0 - 0x6df0)/4
+#define SI_CRTC3_REGISTER_OFFSET                0x2900 //(0x111f0 - 0x6df0)/4
+#define SI_CRTC4_REGISTER_OFFSET                0x2c00 //(0x11df0 - 0x6df0)/4
+#define SI_CRTC5_REGISTER_OFFSET                0x2f00 //(0x129f0 - 0x6df0)/4
+
+#define CURSOR_WIDTH 64
+#define CURSOR_HEIGHT 64
+#define AMDGPU_MM_INDEX		        0x0000
+#define AMDGPU_MM_DATA		        0x0001
+
+#define VERDE_NUM_CRTC 6
+#define	BLACKOUT_MODE_MASK			0x00000007
+#define	VGA_RENDER_CONTROL			0xC0
+#define R_000300_VGA_RENDER_CONTROL             0xC0
+#define C_000300_VGA_VSTATUS_CNTL               0xFFFCFFFF
+#define EVERGREEN_CRTC_STATUS                   0x1BA3
+#define EVERGREEN_CRTC_V_BLANK                  (1 << 0)
+#define EVERGREEN_CRTC_STATUS_POSITION          0x1BA4
+/* CRTC blocks at 0x6df0, 0x79f0, 0x105f0, 0x111f0, 0x11df0, 0x129f0 */
+#define EVERGREEN_CRTC_V_BLANK_START_END                0x1b8d
+#define EVERGREEN_CRTC_CONTROL                          0x1b9c
+#define EVERGREEN_CRTC_MASTER_EN                 (1 << 0)
+#define EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE (1 << 24)
+#define EVERGREEN_CRTC_BLANK_CONTROL                    0x1b9d
+#define EVERGREEN_CRTC_BLANK_DATA_EN             (1 << 8)
+#define EVERGREEN_CRTC_V_BLANK                   (1 << 0)
+#define EVERGREEN_CRTC_STATUS_HV_COUNT                  0x1ba8
+#define EVERGREEN_CRTC_UPDATE_LOCK                      0x1bb5
+#define EVERGREEN_MASTER_UPDATE_LOCK                    0x1bbd
+#define EVERGREEN_MASTER_UPDATE_MODE                    0x1bbe
+#define EVERGREEN_GRPH_UPDATE_LOCK               (1 << 16)
+#define EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH     0x1a07
+#define EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH   0x1a08
+#define EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS          0x1a04
+#define EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS        0x1a05
+#define EVERGREEN_GRPH_UPDATE                           0x1a11
+#define EVERGREEN_VGA_MEMORY_BASE_ADDRESS               0xc4
+#define EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH          0xc9
+#define EVERGREEN_GRPH_SURFACE_UPDATE_PENDING    (1 << 2)
+
+#define EVERGREEN_DATA_FORMAT                           0x1ac0
+#       define EVERGREEN_INTERLEAVE_EN                  (1 << 0)
+
+#define MC_SHARED_CHMAP__NOOFCHAN_MASK 0xf000
+#define MC_SHARED_CHMAP__NOOFCHAN__SHIFT 0xc
+
+#define R600_D1GRPH_ARRAY_MODE_LINEAR_GENERAL            (0 << 20)
+#define R600_D1GRPH_ARRAY_MODE_LINEAR_ALIGNED            (1 << 20)
+#define R600_D1GRPH_ARRAY_MODE_1D_TILED_THIN1            (2 << 20)
+#define R600_D1GRPH_ARRAY_MODE_2D_TILED_THIN1            (4 << 20)
+
+#define R700_D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH                0x1a45
+#define R700_D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH                0x1845
+
+#define R700_D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH              0x1847
+#define R700_D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH              0x1a47
+
+#define DISP_INTERRUPT_STATUS__LB_D1_VBLANK_INTERRUPT_MASK 0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VBLANK_INTERRUPT_MASK 0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VBLANK_INTERRUPT_MASK 0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VBLANK_INTERRUPT_MASK 0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VBLANK_INTERRUPT_MASK 0x8
+#define DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VBLANK_INTERRUPT_MASK 0x8
+
+#define DISP_INTERRUPT_STATUS__LB_D1_VLINE_INTERRUPT_MASK 0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VLINE_INTERRUPT_MASK 0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VLINE_INTERRUPT_MASK 0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VLINE_INTERRUPT_MASK 0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VLINE_INTERRUPT_MASK 0x4
+#define DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VLINE_INTERRUPT_MASK 0x4
+
+#define DISP_INTERRUPT_STATUS__DC_HPD1_INTERRUPT_MASK 0x20000
+#define DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_INTERRUPT_MASK 0x20000
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_INTERRUPT_MASK 0x20000
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_INTERRUPT_MASK 0x20000
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_INTERRUPT_MASK 0x20000
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_INTERRUPT_MASK 0x20000
+
+#define GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK 0x1
+#define GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK 0x100
+
+#define DC_HPD1_INT_CONTROL__DC_HPD1_INT_ACK_MASK 0x1
+
+#define R600_D1GRPH_SWAP_CONTROL                               0x1843
+#define R600_D1GRPH_SWAP_ENDIAN_NONE                    (0 << 0)
+#define R600_D1GRPH_SWAP_ENDIAN_16BIT                   (1 << 0)
+#define R600_D1GRPH_SWAP_ENDIAN_32BIT                   (2 << 0)
+#define R600_D1GRPH_SWAP_ENDIAN_64BIT                   (3 << 0)
+
+#define AVIVO_D1VGA_CONTROL					0x00cc
+#       define AVIVO_DVGA_CONTROL_MODE_ENABLE            (1 << 0)
+#       define AVIVO_DVGA_CONTROL_TIMING_SELECT          (1 << 8)
+#       define AVIVO_DVGA_CONTROL_SYNC_POLARITY_SELECT   (1 << 9)
+#       define AVIVO_DVGA_CONTROL_OVERSCAN_TIMING_SELECT (1 << 10)
+#       define AVIVO_DVGA_CONTROL_OVERSCAN_COLOR_EN      (1 << 16)
+#       define AVIVO_DVGA_CONTROL_ROTATE                 (1 << 24)
+#define AVIVO_D2VGA_CONTROL					0x00ce
+
+#define R600_BUS_CNTL                                           0x1508
+#       define R600_BIOS_ROM_DIS                                (1 << 1)
+
+#define R600_ROM_CNTL                              0x580
+#       define R600_SCK_OVERWRITE                  (1 << 1)
+#       define R600_SCK_PRESCALE_CRYSTAL_CLK_SHIFT 28
+#       define R600_SCK_PRESCALE_CRYSTAL_CLK_MASK  (0xf << 28)
+
+#define GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK 0x1
+
+#define FMT_BIT_DEPTH_CONTROL                0x1bf2
+#define FMT_TRUNCATE_EN               (1 << 0)
+#define FMT_TRUNCATE_DEPTH            (1 << 4)
+#define FMT_SPATIAL_DITHER_EN         (1 << 8)
+#define FMT_SPATIAL_DITHER_MODE(x)    ((x) << 9)
+#define FMT_SPATIAL_DITHER_DEPTH      (1 << 12)
+#define FMT_FRAME_RANDOM_ENABLE       (1 << 13)
+#define FMT_RGB_RANDOM_ENABLE         (1 << 14)
+#define FMT_HIGHPASS_RANDOM_ENABLE    (1 << 15)
+#define FMT_TEMPORAL_DITHER_EN        (1 << 16)
+#define FMT_TEMPORAL_DITHER_DEPTH     (1 << 20)
+#define FMT_TEMPORAL_DITHER_OFFSET(x) ((x) << 21)
+#define FMT_TEMPORAL_LEVEL            (1 << 24)
+#define FMT_TEMPORAL_DITHER_RESET     (1 << 25)
+#define FMT_25FRC_SEL(x)              ((x) << 26)
+#define FMT_50FRC_SEL(x)              ((x) << 28)
+#define FMT_75FRC_SEL(x)              ((x) << 30)
+
+#define EVERGREEN_DC_LUT_CONTROL                        0x1a80
+#define EVERGREEN_DC_LUT_BLACK_OFFSET_BLUE              0x1a81
+#define EVERGREEN_DC_LUT_BLACK_OFFSET_GREEN             0x1a82
+#define EVERGREEN_DC_LUT_BLACK_OFFSET_RED               0x1a83
+#define EVERGREEN_DC_LUT_WHITE_OFFSET_BLUE              0x1a84
+#define EVERGREEN_DC_LUT_WHITE_OFFSET_GREEN             0x1a85
+#define EVERGREEN_DC_LUT_WHITE_OFFSET_RED               0x1a86
+#define EVERGREEN_DC_LUT_30_COLOR                       0x1a7c
+#define EVERGREEN_DC_LUT_RW_INDEX                       0x1a79
+#define EVERGREEN_DC_LUT_WRITE_EN_MASK                  0x1a7e
+#define EVERGREEN_DC_LUT_RW_MODE                        0x1a78
+
+#define EVERGREEN_GRPH_ENABLE                           0x1a00
+#define EVERGREEN_GRPH_CONTROL                          0x1a01
+#define EVERGREEN_GRPH_DEPTH(x)                  (((x) & 0x3) << 0)
+#define EVERGREEN_GRPH_DEPTH_8BPP                0
+#define EVERGREEN_GRPH_DEPTH_16BPP               1
+#define EVERGREEN_GRPH_DEPTH_32BPP               2
+#define EVERGREEN_GRPH_NUM_BANKS(x)              (((x) & 0x3) << 2)
+#define EVERGREEN_ADDR_SURF_2_BANK               0
+#define EVERGREEN_ADDR_SURF_4_BANK               1
+#define EVERGREEN_ADDR_SURF_8_BANK               2
+#define EVERGREEN_ADDR_SURF_16_BANK              3
+#define EVERGREEN_GRPH_Z(x)                      (((x) & 0x3) << 4)
+#define EVERGREEN_GRPH_BANK_WIDTH(x)             (((x) & 0x3) << 6)
+#define EVERGREEN_ADDR_SURF_BANK_WIDTH_1         0
+#define EVERGREEN_ADDR_SURF_BANK_WIDTH_2         1
+#define EVERGREEN_ADDR_SURF_BANK_WIDTH_4         2
+#define EVERGREEN_ADDR_SURF_BANK_WIDTH_8         3
+#define EVERGREEN_GRPH_FORMAT(x)                 (((x) & 0x7) << 8)
+
+#define EVERGREEN_GRPH_FORMAT_INDEXED            0
+#define EVERGREEN_GRPH_FORMAT_ARGB1555           0
+#define EVERGREEN_GRPH_FORMAT_ARGB565            1
+#define EVERGREEN_GRPH_FORMAT_ARGB4444           2
+#define EVERGREEN_GRPH_FORMAT_AI88               3
+#define EVERGREEN_GRPH_FORMAT_MONO16             4
+#define EVERGREEN_GRPH_FORMAT_BGRA5551           5
+
+/* 32 BPP */
+#define EVERGREEN_GRPH_FORMAT_ARGB8888           0
+#define EVERGREEN_GRPH_FORMAT_ARGB2101010        1
+#define EVERGREEN_GRPH_FORMAT_32BPP_DIG          2
+#define EVERGREEN_GRPH_FORMAT_8B_ARGB2101010     3
+#define EVERGREEN_GRPH_FORMAT_BGRA1010102        4
+#define EVERGREEN_GRPH_FORMAT_8B_BGRA1010102     5
+#define EVERGREEN_GRPH_FORMAT_RGB111110          6
+#define EVERGREEN_GRPH_FORMAT_BGR101111          7
+#define EVERGREEN_GRPH_BANK_HEIGHT(x)            (((x) & 0x3) << 11)
+#define EVERGREEN_ADDR_SURF_BANK_HEIGHT_1        0
+#define EVERGREEN_ADDR_SURF_BANK_HEIGHT_2        1
+#define EVERGREEN_ADDR_SURF_BANK_HEIGHT_4        2
+#define EVERGREEN_ADDR_SURF_BANK_HEIGHT_8        3
+#define EVERGREEN_GRPH_TILE_SPLIT(x)             (((x) & 0x7) << 13)
+#define EVERGREEN_ADDR_SURF_TILE_SPLIT_64B       0
+#define EVERGREEN_ADDR_SURF_TILE_SPLIT_128B      1
+#define EVERGREEN_ADDR_SURF_TILE_SPLIT_256B      2
+#define EVERGREEN_ADDR_SURF_TILE_SPLIT_512B      3
+#define EVERGREEN_ADDR_SURF_TILE_SPLIT_1KB       4
+#define EVERGREEN_ADDR_SURF_TILE_SPLIT_2KB       5
+#define EVERGREEN_ADDR_SURF_TILE_SPLIT_4KB       6
+#define EVERGREEN_GRPH_MACRO_TILE_ASPECT(x)      (((x) & 0x3) << 18)
+#define EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1  0
+#define EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2  1
+#define EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4  2
+#define EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8  3
+#define EVERGREEN_GRPH_ARRAY_MODE(x)             (((x) & 0x7) << 20)
+#define EVERGREEN_GRPH_ARRAY_LINEAR_GENERAL      0
+#define EVERGREEN_GRPH_ARRAY_LINEAR_ALIGNED      1
+#define EVERGREEN_GRPH_ARRAY_1D_TILED_THIN1      2
+#define EVERGREEN_GRPH_ARRAY_2D_TILED_THIN1      4
+#define EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1  0
+#define EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2  1
+#define EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4  2
+#define EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8  3
+
+#define EVERGREEN_GRPH_SWAP_CONTROL                     0x1a03
+#define EVERGREEN_GRPH_ENDIAN_SWAP(x)            (((x) & 0x3) << 0)
+#       define EVERGREEN_GRPH_ENDIAN_NONE               0
+#       define EVERGREEN_GRPH_ENDIAN_8IN16              1
+#       define EVERGREEN_GRPH_ENDIAN_8IN32              2
+#       define EVERGREEN_GRPH_ENDIAN_8IN64              3
+
+#define EVERGREEN_D3VGA_CONTROL                         0xf8
+#define EVERGREEN_D4VGA_CONTROL                         0xf9
+#define EVERGREEN_D5VGA_CONTROL                         0xfa
+#define EVERGREEN_D6VGA_CONTROL                         0xfb
+
+#define EVERGREEN_GRPH_SURFACE_ADDRESS_MASK      0xffffff00
+
+#define EVERGREEN_GRPH_LUT_10BIT_BYPASS_CONTROL         0x1a02
+#define EVERGREEN_LUT_10BIT_BYPASS_EN            (1 << 8)
+
+#define EVERGREEN_GRPH_PITCH                            0x1a06
+#define EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH     0x1a07
+#define EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH   0x1a08
+#define EVERGREEN_GRPH_SURFACE_OFFSET_X                 0x1a09
+#define EVERGREEN_GRPH_SURFACE_OFFSET_Y                 0x1a0a
+#define EVERGREEN_GRPH_X_START                          0x1a0b
+#define EVERGREEN_GRPH_Y_START                          0x1a0c
+#define EVERGREEN_GRPH_X_END                            0x1a0d
+#define EVERGREEN_GRPH_Y_END                            0x1a0e
+#define EVERGREEN_GRPH_UPDATE                           0x1a11
+#define EVERGREEN_GRPH_SURFACE_UPDATE_PENDING    (1 << 2)
+#define EVERGREEN_GRPH_UPDATE_LOCK               (1 << 16)
+#define EVERGREEN_GRPH_FLIP_CONTROL                     0x1a12
+#define EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN (1 << 0)
+
+#define EVERGREEN_VIEWPORT_START                        0x1b5c
+#define EVERGREEN_VIEWPORT_SIZE                         0x1b5d
+#define EVERGREEN_DESKTOP_HEIGHT                        0x1ac1
+
+/* CUR blocks at 0x6998, 0x7598, 0x10198, 0x10d98, 0x11998, 0x12598 */
+#define EVERGREEN_CUR_CONTROL                           0x1a66
+#       define EVERGREEN_CURSOR_EN                      (1 << 0)
+#       define EVERGREEN_CURSOR_MODE(x)                 (((x) & 0x3) << 8)
+#       define EVERGREEN_CURSOR_MONO                    0
+#       define EVERGREEN_CURSOR_24_1                    1
+#       define EVERGREEN_CURSOR_24_8_PRE_MULT           2
+#       define EVERGREEN_CURSOR_24_8_UNPRE_MULT         3
+#       define EVERGREEN_CURSOR_2X_MAGNIFY              (1 << 16)
+#       define EVERGREEN_CURSOR_FORCE_MC_ON             (1 << 20)
+#       define EVERGREEN_CURSOR_URGENT_CONTROL(x)       (((x) & 0x7) << 24)
+#       define EVERGREEN_CURSOR_URGENT_ALWAYS           0
+#       define EVERGREEN_CURSOR_URGENT_1_8              1
+#       define EVERGREEN_CURSOR_URGENT_1_4              2
+#       define EVERGREEN_CURSOR_URGENT_3_8              3
+#       define EVERGREEN_CURSOR_URGENT_1_2              4
+#define EVERGREEN_CUR_SURFACE_ADDRESS                   0x1a67
+#       define EVERGREEN_CUR_SURFACE_ADDRESS_MASK       0xfffff000
+#define EVERGREEN_CUR_SIZE                              0x1a68
+#define EVERGREEN_CUR_SURFACE_ADDRESS_HIGH              0x1a69
+#define EVERGREEN_CUR_POSITION                          0x1a6a
+#define EVERGREEN_CUR_HOT_SPOT                          0x1a6b
+#define EVERGREEN_CUR_COLOR1                            0x1a6c
+#define EVERGREEN_CUR_COLOR2                            0x1a6d
+#define EVERGREEN_CUR_UPDATE                            0x1a6e
+#       define EVERGREEN_CURSOR_UPDATE_PENDING          (1 << 0)
+#       define EVERGREEN_CURSOR_UPDATE_TAKEN            (1 << 1)
+#       define EVERGREEN_CURSOR_UPDATE_LOCK             (1 << 16)
+#       define EVERGREEN_CURSOR_DISABLE_MULTIPLE_UPDATE (1 << 24)
+
+
+#define NI_INPUT_CSC_CONTROL                           0x1a35
+#       define NI_INPUT_CSC_GRPH_MODE(x)               (((x) & 0x3) << 0)
+#       define NI_INPUT_CSC_BYPASS                     0
+#       define NI_INPUT_CSC_PROG_COEFF                 1
+#       define NI_INPUT_CSC_PROG_SHARED_MATRIXA        2
+#       define NI_INPUT_CSC_OVL_MODE(x)                (((x) & 0x3) << 4)
+
+#define NI_OUTPUT_CSC_CONTROL                          0x1a3c
+#       define NI_OUTPUT_CSC_GRPH_MODE(x)              (((x) & 0x7) << 0)
+#       define NI_OUTPUT_CSC_BYPASS                    0
+#       define NI_OUTPUT_CSC_TV_RGB                    1
+#       define NI_OUTPUT_CSC_YCBCR_601                 2
+#       define NI_OUTPUT_CSC_YCBCR_709                 3
+#       define NI_OUTPUT_CSC_PROG_COEFF                4
+#       define NI_OUTPUT_CSC_PROG_SHARED_MATRIXB       5
+#       define NI_OUTPUT_CSC_OVL_MODE(x)               (((x) & 0x7) << 4)
+
+#define NI_DEGAMMA_CONTROL                             0x1a58
+#       define NI_GRPH_DEGAMMA_MODE(x)                 (((x) & 0x3) << 0)
+#       define NI_DEGAMMA_BYPASS                       0
+#       define NI_DEGAMMA_SRGB_24                      1
+#       define NI_DEGAMMA_XVYCC_222                    2
+#       define NI_OVL_DEGAMMA_MODE(x)                  (((x) & 0x3) << 4)
+#       define NI_ICON_DEGAMMA_MODE(x)                 (((x) & 0x3) << 8)
+#       define NI_CURSOR_DEGAMMA_MODE(x)               (((x) & 0x3) << 12)
+
+#define NI_GAMUT_REMAP_CONTROL                         0x1a59
+#       define NI_GRPH_GAMUT_REMAP_MODE(x)             (((x) & 0x3) << 0)
+#       define NI_GAMUT_REMAP_BYPASS                   0
+#       define NI_GAMUT_REMAP_PROG_COEFF               1
+#       define NI_GAMUT_REMAP_PROG_SHARED_MATRIXA      2
+#       define NI_GAMUT_REMAP_PROG_SHARED_MATRIXB      3
+#       define NI_OVL_GAMUT_REMAP_MODE(x)              (((x) & 0x3) << 4)
+
+#define NI_REGAMMA_CONTROL                             0x1aa0
+#       define NI_GRPH_REGAMMA_MODE(x)                 (((x) & 0x7) << 0)
+#       define NI_REGAMMA_BYPASS                       0
+#       define NI_REGAMMA_SRGB_24                      1
+#       define NI_REGAMMA_XVYCC_222                    2
+#       define NI_REGAMMA_PROG_A                       3
+#       define NI_REGAMMA_PROG_B                       4
+#       define NI_OVL_REGAMMA_MODE(x)                  (((x) & 0x7) << 4)
+
+
+#define NI_PRESCALE_GRPH_CONTROL                       0x1a2d
+#       define NI_GRPH_PRESCALE_BYPASS                 (1 << 4)
+
+#define NI_PRESCALE_OVL_CONTROL                        0x1a31
+#       define NI_OVL_PRESCALE_BYPASS                  (1 << 4)
+
+#define NI_INPUT_GAMMA_CONTROL                         0x1a10
+#       define NI_GRPH_INPUT_GAMMA_MODE(x)             (((x) & 0x3) << 0)
+#       define NI_INPUT_GAMMA_USE_LUT                  0
+#       define NI_INPUT_GAMMA_BYPASS                   1
+#       define NI_INPUT_GAMMA_SRGB_24                  2
+#       define NI_INPUT_GAMMA_XVYCC_222                3
+#       define NI_OVL_INPUT_GAMMA_MODE(x)              (((x) & 0x3) << 4)
+
+#define IH_RB_WPTR__RB_OVERFLOW_MASK	0x1
+#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK 0x80000000
+#define SRBM_STATUS__IH_BUSY_MASK	0x20000
+#define SRBM_SOFT_RESET__SOFT_RESET_IH_MASK	0x400
+
+#define	BLACKOUT_MODE_MASK			0x00000007
+#define	VGA_RENDER_CONTROL			0xC0
+#define R_000300_VGA_RENDER_CONTROL             0xC0
+#define C_000300_VGA_VSTATUS_CNTL               0xFFFCFFFF
+#define EVERGREEN_CRTC_STATUS                   0x1BA3
+#define EVERGREEN_CRTC_V_BLANK                  (1 << 0)
+#define EVERGREEN_CRTC_STATUS_POSITION          0x1BA4
+/* CRTC blocks at 0x6df0, 0x79f0, 0x105f0, 0x111f0, 0x11df0, 0x129f0 */
+#define EVERGREEN_CRTC_V_BLANK_START_END                0x1b8d
+#define EVERGREEN_CRTC_CONTROL                          0x1b9c
+#       define EVERGREEN_CRTC_MASTER_EN                 (1 << 0)
+#       define EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE (1 << 24)
+#define EVERGREEN_CRTC_BLANK_CONTROL                    0x1b9d
+#       define EVERGREEN_CRTC_BLANK_DATA_EN             (1 << 8)
+#       define EVERGREEN_CRTC_V_BLANK                   (1 << 0)
+#define EVERGREEN_CRTC_STATUS_HV_COUNT                  0x1ba8
+#define EVERGREEN_CRTC_UPDATE_LOCK                      0x1bb5
+#define EVERGREEN_MASTER_UPDATE_LOCK                    0x1bbd
+#define EVERGREEN_MASTER_UPDATE_MODE                    0x1bbe
+#define EVERGREEN_GRPH_UPDATE_LOCK               (1 << 16)
+#define EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH     0x1a07
+#define EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH   0x1a08
+#define EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS          0x1a04
+#define EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS        0x1a05
+#define EVERGREEN_GRPH_UPDATE                           0x1a11
+#define EVERGREEN_VGA_MEMORY_BASE_ADDRESS               0xc4
+#define EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH          0xc9
+#define EVERGREEN_GRPH_SURFACE_UPDATE_PENDING    (1 << 2)
+
+#define mmVM_CONTEXT1_CNTL__xxRANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x10
+#define mmVM_CONTEXT1_CNTL__xxRANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x4
+#define mmVM_CONTEXT1_CNTL__xxDUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x80
+#define mmVM_CONTEXT1_CNTL__xxDUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x7
+#define mmVM_CONTEXT1_CNTL__xxPDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x400
+#define mmVM_CONTEXT1_CNTL__xxPDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0xa
+#define mmVM_CONTEXT1_CNTL__xxVALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x2000
+#define mmVM_CONTEXT1_CNTL__xxVALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0xd
+#define mmVM_CONTEXT1_CNTL__xxREAD_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x10000
+#define mmVM_CONTEXT1_CNTL__xxREAD_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x10
+#define mmVM_CONTEXT1_CNTL__xxWRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x80000
+#define mmVM_CONTEXT1_CNTL__xxWRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x13
+
+#define mmVM_CONTEXT1_PROTECTION_FAULT_STATUS__xxVMID_MASK 0x1e000000
+#define mmVM_CONTEXT1_PROTECTION_FAULT_STATUS__xxVMID__SHIFT 0x19
+#define mmVM_CONTEXT1_PROTECTION_FAULT_STATUS__xxPROTECTIONS_MASK 0xff
+#define mmVM_CONTEXT1_PROTECTION_FAULT_STATUS__xxPROTECTIONS__SHIFT 0x0
+#define mmVM_CONTEXT1_PROTECTION_FAULT_STATUS__xxMEMORY_CLIENT_ID_MASK 0xff000
+#define mmVM_CONTEXT1_PROTECTION_FAULT_STATUS__xxMEMORY_CLIENT_ID__SHIFT 0xc
+#define mmVM_CONTEXT1_PROTECTION_FAULT_STATUS__xxMEMORY_CLIENT_RW_MASK 0x1000000
+#define mmVM_CONTEXT1_PROTECTION_FAULT_STATUS__xxMEMORY_CLIENT_RW__SHIFT 0x18
+
+#define mmMC_SHARED_BLACKOUT_CNTL__xxBLACKOUT_MODE_MASK 0x7
+#define mmMC_SHARED_BLACKOUT_CNTL__xxBLACKOUT_MODE__SHIFT 0x0
+
+#define mmBIF_FB_EN__xxFB_READ_EN_MASK 0x1
+#define mmBIF_FB_EN__xxFB_READ_EN__SHIFT 0x0
+#define mmBIF_FB_EN__xxFB_WRITE_EN_MASK 0x2
+#define mmBIF_FB_EN__xxFB_WRITE_EN__SHIFT 0x1
+
+#define mmSRBM_SOFT_RESET__xxSOFT_RESET_VMC_MASK 0x20000
+#define mmSRBM_SOFT_RESET__xxSOFT_RESET_VMC__SHIFT 0x11
+#define mmSRBM_SOFT_RESET__xxSOFT_RESET_MC_MASK 0x800
+#define mmSRBM_SOFT_RESET__xxSOFT_RESET_MC__SHIFT 0xb
+
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x8
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x3
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x40
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x6
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x200
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x9
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x1000
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0xc
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x8000
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0xf
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x40000
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x12
+
+#define MC_SEQ_MISC0__MT__MASK	0xf0000000
+#define MC_SEQ_MISC0__MT__GDDR1  0x10000000
+#define MC_SEQ_MISC0__MT__DDR2   0x20000000
+#define MC_SEQ_MISC0__MT__GDDR3  0x30000000
+#define MC_SEQ_MISC0__MT__GDDR4  0x40000000
+#define MC_SEQ_MISC0__MT__GDDR5  0x50000000
+#define MC_SEQ_MISC0__MT__HBM    0x60000000
+#define MC_SEQ_MISC0__MT__DDR3   0xB0000000
+
+#define SRBM_STATUS__MCB_BUSY_MASK 0x200
+#define SRBM_STATUS__MCB_BUSY__SHIFT 0x9
+#define SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK 0x400
+#define SRBM_STATUS__MCB_NON_DISPLAY_BUSY__SHIFT 0xa
+#define SRBM_STATUS__MCC_BUSY_MASK 0x800
+#define SRBM_STATUS__MCC_BUSY__SHIFT 0xb
+#define SRBM_STATUS__MCD_BUSY_MASK 0x1000
+#define SRBM_STATUS__MCD_BUSY__SHIFT 0xc
+#define SRBM_STATUS__VMC_BUSY_MASK 0x100
+#define SRBM_STATUS__VMC_BUSY__SHIFT 0x8
+
+
+#define GRBM_STATUS__GUI_ACTIVE_MASK 0x80000000
+#define CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK 0x4000000
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK 0x800000
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK 0x400000
+#define PACKET3_SEM_WAIT_ON_SIGNAL    (0x1 << 12)
+#define PACKET3_SEM_SEL_SIGNAL	    (0x6 << 29)
+#define PACKET3_SEM_SEL_WAIT	    (0x7 << 29)
+
+#define CONFIG_CNTL	0x1509
+#define CC_DRM_ID_STRAPS	0X1559
+#define AMDGPU_PCIE_INDEX	0xc
+#define AMDGPU_PCIE_DATA	0xd
+
+#define DMA_SEM_INCOMPLETE_TIMER_CNTL                     0x3411
+#define DMA_SEM_WAIT_FAIL_TIMER_CNTL                      0x3412
+#define DMA_MODE                                          0x342f
+#define DMA_RB_RPTR_ADDR_HI                               0x3407
+#define DMA_RB_RPTR_ADDR_LO                               0x3408
+#define DMA_BUSY_MASK 0x20
+#define DMA1_BUSY_MASK 0X40
+#define SDMA_MAX_INSTANCE 2
+
+#define PCIE_BUS_CLK    10000
+#define TCLK            (PCIE_BUS_CLK / 10)
+#define CC_DRM_ID_STRAPS__ATI_REV_ID_MASK		0xf0000000
+#define CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT 0x1c
+#define	PCIE_PORT_INDEX					0xe
+#define	PCIE_PORT_DATA					0xf
+#define EVERGREEN_PIF_PHY0_INDEX                        0x8
+#define EVERGREEN_PIF_PHY0_DATA                         0xc
+#define EVERGREEN_PIF_PHY1_INDEX                        0x10
+#define EVERGREEN_PIF_PHY1_DATA				0x14
+
+#define	MC_VM_FB_OFFSET					0x81a
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_2_d.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_2_d.h
index f3e53b118361..19802e96417e 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_2_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_2_d.h
@@ -34,6 +34,7 @@
 #define mmUVD_UDEC_ADDR_CONFIG                                                  0x3bd3
 #define mmUVD_UDEC_DB_ADDR_CONFIG                                               0x3bd4
 #define mmUVD_UDEC_DBW_ADDR_CONFIG                                              0x3bd5
+#define mmUVD_NO_OP                                                             0x3bff
 #define mmUVD_SEMA_CNTL                                                         0x3d00
 #define mmUVD_LMI_EXT40_ADDR                                                    0x3d26
 #define mmUVD_CTX_INDEX                                                         0x3d28
diff --git a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_5_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_5_0_d.h
index eb4cf53427da..cc972d237a7e 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_5_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_5_0_d.h
@@ -34,6 +34,7 @@
 #define mmUVD_UDEC_ADDR_CONFIG                                                  0x3bd3
 #define mmUVD_UDEC_DB_ADDR_CONFIG                                               0x3bd4
 #define mmUVD_UDEC_DBW_ADDR_CONFIG                                              0x3bd5
+#define mmUVD_NO_OP                                                             0x3bff
 #define mmUVD_LMI_RBC_RB_64BIT_BAR_LOW                                          0x3c69
 #define mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH                                         0x3c68
 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW                                          0x3c67
diff --git a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h
index ec69869c55ff..378f4b6b43da 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h
@@ -35,6 +35,7 @@
 #define mmUVD_UDEC_DB_ADDR_CONFIG                                               0x3bd4
 #define mmUVD_UDEC_DBW_ADDR_CONFIG                                              0x3bd5
 #define mmUVD_POWER_STATUS_U                                                    0x3bfd
+#define mmUVD_NO_OP                                                             0x3bff
 #define mmUVD_LMI_RBC_RB_64BIT_BAR_LOW                                          0x3c69
 #define mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH                                         0x3c68
 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW                                          0x3c67
diff --git a/drivers/gpu/drm/amd/include/atombios.h b/drivers/gpu/drm/amd/include/atombios.h
index 3493da5c8f0e..4a4d3797a6d3 100644
--- a/drivers/gpu/drm/amd/include/atombios.h
+++ b/drivers/gpu/drm/amd/include/atombios.h
@@ -494,6 +494,7 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V3
   union
   {
     ATOM_COMPUTE_CLOCK_FREQ  ulClock;         //Input Parameter
+    ULONG ulClockParams;                      //ULONG access for BE
     ATOM_S_MPLL_FB_DIVIDER   ulFbDiv;         //Output Parameter
   };
   UCHAR   ucRefDiv;                           //Output Parameter
@@ -526,6 +527,7 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5
   union
   {
     ATOM_COMPUTE_CLOCK_FREQ  ulClock;         //Input Parameter
+    ULONG ulClockParams;                      //ULONG access for BE
     ATOM_S_MPLL_FB_DIVIDER   ulFbDiv;         //Output Parameter
   };
   UCHAR   ucRefDiv;                           //Output Parameter
diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h
index b86aba9d019f..df7c18b6a02a 100644..100755
--- a/drivers/gpu/drm/amd/include/cgs_common.h
+++ b/drivers/gpu/drm/amd/include/cgs_common.h
@@ -119,6 +119,8 @@ enum cgs_system_info_id {
 	CGS_SYSTEM_INFO_PG_FLAGS,
 	CGS_SYSTEM_INFO_GFX_CU_INFO,
 	CGS_SYSTEM_INFO_GFX_SE_INFO,
+	CGS_SYSTEM_INFO_PCIE_SUB_SYS_ID,
+	CGS_SYSTEM_INFO_PCIE_SUB_SYS_VENDOR_ID,
 	CGS_SYSTEM_INFO_ID_MAXIMUM,
 };
 
@@ -159,6 +161,7 @@ struct cgs_clock_limits {
  */
 struct cgs_firmware_info {
 	uint16_t		version;
+	uint16_t		fw_version;
 	uint16_t		feature_version;
 	uint32_t		image_size;
 	uint64_t		mc_addr;
diff --git a/drivers/gpu/drm/amd/powerplay/Kconfig b/drivers/gpu/drm/amd/powerplay/Kconfig
deleted file mode 100644
index af380335b425..000000000000
--- a/drivers/gpu/drm/amd/powerplay/Kconfig
+++ /dev/null
@@ -1,6 +0,0 @@
-config DRM_AMD_POWERPLAY
-	bool  "Enable AMD powerplay component"
-	depends on DRM_AMDGPU
-	default n
-	help
-	  select this option will enable AMD powerplay component.
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index abbb658bdc1e..7174f7a68266 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -31,6 +31,7 @@
 #include "eventmanager.h"
 #include "pp_debug.h"
 
+
 #define PP_CHECK(handle)						\
 	do {								\
 		if ((handle) == NULL || (handle)->pp_valid != PP_VALID)	\
@@ -162,12 +163,12 @@ static int pp_hw_fini(void *handle)
 	pp_handle = (struct pp_instance *)handle;
 	eventmgr = pp_handle->eventmgr;
 
-	if (eventmgr != NULL || eventmgr->pp_eventmgr_fini != NULL)
+	if (eventmgr != NULL && eventmgr->pp_eventmgr_fini != NULL)
 		eventmgr->pp_eventmgr_fini(eventmgr);
 
 	smumgr = pp_handle->smu_mgr;
 
-	if (smumgr != NULL || smumgr->smumgr_funcs != NULL ||
+	if (smumgr != NULL && smumgr->smumgr_funcs != NULL &&
 		smumgr->smumgr_funcs->smu_fini != NULL)
 		smumgr->smumgr_funcs->smu_fini(smumgr);
 
@@ -190,11 +191,9 @@ static int pp_sw_reset(void *handle)
 }
 
 
-static int pp_set_clockgating_state(void *handle,
-				    enum amd_clockgating_state state)
+int amd_set_clockgating_by_smu(void *handle, uint32_t msg_id)
 {
 	struct pp_hwmgr  *hwmgr;
-	uint32_t msg_id, pp_state;
 
 	if (handle == NULL)
 		return -EINVAL;
@@ -208,76 +207,7 @@ static int pp_set_clockgating_state(void *handle,
 		return 0;
 	}
 
-	if (state == AMD_CG_STATE_UNGATE)
-		pp_state = 0;
-	else
-		pp_state = PP_STATE_CG | PP_STATE_LS;
-
-	/* Enable/disable GFX blocks clock gating through SMU */
-	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
-			PP_BLOCK_GFX_CG,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
-			PP_BLOCK_GFX_3D,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
-			PP_BLOCK_GFX_RLC,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
-			PP_BLOCK_GFX_CP,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
-			PP_BLOCK_GFX_MG,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-
-	/* Enable/disable System blocks clock gating through SMU */
-	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
-			PP_BLOCK_SYS_BIF,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
-			PP_BLOCK_SYS_BIF,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
-			PP_BLOCK_SYS_MC,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
-			PP_BLOCK_SYS_ROM,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
-			PP_BLOCK_SYS_DRM,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
-			PP_BLOCK_SYS_HDP,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-	msg_id = PP_CG_MSG_ID(PP_GROUP_SYS,
-			PP_BLOCK_SYS_SDMA,
-			PP_STATE_SUPPORT_CG | PP_STATE_SUPPORT_LS,
-			pp_state);
-	hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
-
-	return 0;
+	return hwmgr->hwmgr_func->update_clock_gatings(hwmgr, &msg_id);
 }
 
 static int pp_set_powergating_state(void *handle,
@@ -361,7 +291,7 @@ const struct amd_ip_funcs pp_ip_funcs = {
 	.is_idle = pp_is_idle,
 	.wait_for_idle = pp_wait_for_idle,
 	.soft_reset = pp_sw_reset,
-	.set_clockgating_state = pp_set_clockgating_state,
+	.set_clockgating_state = NULL,
 	.set_powergating_state = pp_set_powergating_state,
 };
 
@@ -537,7 +467,6 @@ int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id, void *input,
 		ret = pem_handle_event(pp_handle->eventmgr, event_id, &data);
 		break;
 	case AMD_PP_EVENT_READJUST_POWER_STATE:
-		pp_handle->hwmgr->current_ps = pp_handle->hwmgr->boot_ps;
 		ret = pem_handle_event(pp_handle->eventmgr, event_id, &data);
 		break;
 	default:
@@ -576,28 +505,6 @@ enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
 	}
 }
 
-static void
-pp_debugfs_print_current_performance_level(void *handle,
-					       struct seq_file *m)
-{
-	struct pp_hwmgr  *hwmgr;
-
-	if (handle == NULL)
-		return;
-
-	hwmgr = ((struct pp_instance *)handle)->hwmgr;
-
-	if (hwmgr == NULL || hwmgr->hwmgr_func == NULL)
-		return;
-
-	if (hwmgr->hwmgr_func->print_current_perforce_level == NULL) {
-		printk(KERN_INFO "%s was not implemented.\n", __func__);
-		return;
-	}
-
-	hwmgr->hwmgr_func->print_current_perforce_level(hwmgr, m);
-}
-
 static int pp_dpm_set_fan_control_mode(void *handle, uint32_t mode)
 {
 	struct pp_hwmgr  *hwmgr;
@@ -764,15 +671,12 @@ static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size)
 	PP_CHECK_HW(hwmgr);
 
 	if (!hwmgr->hardcode_pp_table) {
-		hwmgr->hardcode_pp_table =
-				kzalloc(hwmgr->soft_pp_table_size, GFP_KERNEL);
+		hwmgr->hardcode_pp_table = kmemdup(hwmgr->soft_pp_table,
+						   hwmgr->soft_pp_table_size,
+						   GFP_KERNEL);
 
 		if (!hwmgr->hardcode_pp_table)
 			return -ENOMEM;
-
-		/* to avoid powerplay crash when hardcode pptable is empty */
-		memcpy(hwmgr->hardcode_pp_table, hwmgr->soft_pp_table,
-				hwmgr->soft_pp_table_size);
 	}
 
 	memcpy(hwmgr->hardcode_pp_table, buf, size);
@@ -897,6 +801,25 @@ static int pp_dpm_set_mclk_od(void *handle, uint32_t value)
 	return hwmgr->hwmgr_func->set_mclk_od(hwmgr, value);
 }
 
+static int pp_dpm_read_sensor(void *handle, int idx, int32_t *value)
+{
+	struct pp_hwmgr *hwmgr;
+
+	if (!handle)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	PP_CHECK_HW(hwmgr);
+
+	if (hwmgr->hwmgr_func->read_sensor == NULL) {
+		printk(KERN_INFO "%s was not implemented.\n", __func__);
+		return 0;
+	}
+
+	return hwmgr->hwmgr_func->read_sensor(hwmgr, idx, value);
+}
+
 const struct amd_powerplay_funcs pp_dpm_funcs = {
 	.get_temperature = pp_dpm_get_temperature,
 	.load_firmware = pp_dpm_load_fw,
@@ -909,7 +832,6 @@ const struct amd_powerplay_funcs pp_dpm_funcs = {
 	.powergate_vce = pp_dpm_powergate_vce,
 	.powergate_uvd = pp_dpm_powergate_uvd,
 	.dispatch_tasks = pp_dpm_dispatch_tasks,
-	.print_current_performance_level = pp_debugfs_print_current_performance_level,
 	.set_fan_control_mode = pp_dpm_set_fan_control_mode,
 	.get_fan_control_mode = pp_dpm_get_fan_control_mode,
 	.set_fan_speed_percent = pp_dpm_set_fan_speed_percent,
@@ -923,6 +845,7 @@ const struct amd_powerplay_funcs pp_dpm_funcs = {
 	.set_sclk_od = pp_dpm_set_sclk_od,
 	.get_mclk_od = pp_dpm_get_mclk_od,
 	.set_mclk_od = pp_dpm_set_mclk_od,
+	.read_sensor = pp_dpm_read_sensor,
 };
 
 static int amd_pp_instance_init(struct amd_pp_init *pp_init,
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
index 635fc4b48184..92b117843875 100644
--- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
@@ -262,6 +262,8 @@ static const pem_event_action * const display_config_change_event[] = {
 	unblock_adjust_power_state_tasks,
 	set_cpu_power_state,
 	notify_hw_power_source_tasks,
+	get_2d_performance_state_tasks,
+	set_performance_state_tasks,
 	/* updateDALConfigurationTasks,
 	variBrightDisplayConfigurationChangeTasks, */
 	adjust_power_state_tasks,
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/psm.c b/drivers/gpu/drm/amd/powerplay/eventmgr/psm.c
index a46225c0fc01..489908887e9c 100644
--- a/drivers/gpu/drm/amd/powerplay/eventmgr/psm.c
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/psm.c
@@ -70,11 +70,12 @@ int psm_set_states(struct pp_eventmgr *eventmgr, unsigned long *state_id)
 	int i;
 
 	table_entries = hwmgr->num_ps;
+
 	state = hwmgr->ps;
 
 	for (i = 0; i < table_entries; i++) {
 		if (state->id == *state_id) {
-			hwmgr->request_ps = state;
+			memcpy(hwmgr->request_ps, state, hwmgr->ps_size);
 			return 0;
 		}
 		state = (struct pp_power_state *)((unsigned long)state + hwmgr->ps_size);
@@ -100,13 +101,14 @@ int psm_adjust_power_state_dynamic(struct pp_eventmgr *eventmgr, bool skip)
 	if (requested == NULL)
 		return 0;
 
+	phm_apply_state_adjust_rules(hwmgr, requested, pcurrent);
+
 	if (pcurrent == NULL || (0 != phm_check_states_equal(hwmgr, &pcurrent->hardware, &requested->hardware, &equal)))
 		equal = false;
 
 	if (!equal || phm_check_smc_update_required_for_display_configuration(hwmgr)) {
-		phm_apply_state_adjust_rules(hwmgr, requested, pcurrent);
 		phm_set_power_state(hwmgr, &pcurrent->hardware, &requested->hardware);
-		hwmgr->current_ps = requested;
+		memcpy(hwmgr->current_ps, hwmgr->request_ps, hwmgr->ps_size);
 	}
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile b/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile
index f7ce4cb71346..5fff1d636ab7 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile
@@ -3,14 +3,12 @@
 # It provides the hardware management services for the driver.
 
 HARDWARE_MGR = hwmgr.o processpptables.o functiontables.o \
-	       hardwaremanager.o pp_acpi.o cz_hwmgr.o \
-               cz_clockpowergating.o \
-	       tonga_processpptables.o ppatomctrl.o \
-               tonga_hwmgr.o pppcielanes.o  tonga_thermal.o\
-               fiji_powertune.o fiji_hwmgr.o tonga_clockpowergating.o \
-               fiji_clockpowergating.o fiji_thermal.o \
-	       polaris10_hwmgr.o polaris10_powertune.o polaris10_thermal.o \
-	       polaris10_clockpowergating.o
+		hardwaremanager.o pp_acpi.o cz_hwmgr.o \
+		cz_clockpowergating.o pppcielanes.o\
+		process_pptables_v1_0.o ppatomctrl.o \
+		smu7_hwmgr.o smu7_powertune.o smu7_thermal.o \
+		smu7_clockpowergating.o
+
 
 AMD_PP_HWMGR = $(addprefix $(AMD_PP_PATH)/hwmgr/,$(HARDWARE_MGR))
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
index 8cc0df9b534a..7e4fcbbbe086 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -178,7 +178,6 @@ static int cz_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
 	int result;
 
 	cz_hwmgr->gfx_ramp_step = 256*25/100;
-
 	cz_hwmgr->gfx_ramp_delay = 1; /* by default, we delay 1us */
 
 	for (i = 0; i < CZ_MAX_HARDWARE_POWERLEVELS; i++)
@@ -186,33 +185,19 @@ static int cz_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
 
 	cz_hwmgr->mgcg_cgtt_local0 = 0x00000000;
 	cz_hwmgr->mgcg_cgtt_local1 = 0x00000000;
-
 	cz_hwmgr->clock_slow_down_freq = 25000;
-
 	cz_hwmgr->skip_clock_slow_down = 1;
-
 	cz_hwmgr->enable_nb_ps_policy = 1; /* disable until UNB is ready, Enabled */
-
 	cz_hwmgr->voltage_drop_in_dce_power_gating = 0; /* disable until fully verified */
-
 	cz_hwmgr->voting_rights_clients = 0x00C00033;
-
 	cz_hwmgr->static_screen_threshold = 8;
-
 	cz_hwmgr->ddi_power_gating_disabled = 0;
-
 	cz_hwmgr->bapm_enabled = 1;
-
 	cz_hwmgr->voltage_drop_threshold = 0;
-
 	cz_hwmgr->gfx_power_gating_threshold = 500;
-
 	cz_hwmgr->vce_slow_sclk_threshold = 20000;
-
 	cz_hwmgr->dce_slow_sclk_threshold = 30000;
-
 	cz_hwmgr->disable_driver_thermal_policy = 1;
-
 	cz_hwmgr->disable_nb_ps3_in_battery = 0;
 
 	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
@@ -221,9 +206,6 @@ static int cz_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
 	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 				    PHM_PlatformCaps_NonABMSupportInPPLib);
 
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-					   PHM_PlatformCaps_SclkDeepSleep);
-
 	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 					PHM_PlatformCaps_DynamicM3Arbiter);
 
@@ -233,9 +215,7 @@ static int cz_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
 				  PHM_PlatformCaps_DynamicPatchPowerState);
 
 	cz_hwmgr->thermal_auto_throttling_treshold = 0;
-
 	cz_hwmgr->tdr_clock = 0;
-
 	cz_hwmgr->disable_gfx_power_gating_in_uvd = 0;
 
 	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
@@ -450,19 +430,12 @@ static int cz_construct_boot_state(struct pp_hwmgr *hwmgr)
 			(uint8_t)cz_hwmgr->sys_info.bootup_nb_voltage_index;
 
 	cz_hwmgr->boot_power_level.dsDividerIndex = 0;
-
 	cz_hwmgr->boot_power_level.ssDividerIndex = 0;
-
 	cz_hwmgr->boot_power_level.allowGnbSlow = 1;
-
 	cz_hwmgr->boot_power_level.forceNBPstate = 0;
-
 	cz_hwmgr->boot_power_level.hysteresis_up = 0;
-
 	cz_hwmgr->boot_power_level.numSIMDToPowerDown = 0;
-
 	cz_hwmgr->boot_power_level.display_wm = 0;
-
 	cz_hwmgr->boot_power_level.vce_wm = 0;
 
 	return 0;
@@ -749,7 +722,6 @@ static int cz_tf_update_sclk_limit(struct pp_hwmgr *hwmgr,
 		cz_hwmgr->sclk_dpm.soft_max_clk  = table->entries[table->count - 1].clk;
 
 	clock = hwmgr->display_config.min_core_set_clock;
-;
 	if (clock == 0)
 		printk(KERN_INFO "[ powerplay ] min_core_set_clock not set\n");
 
@@ -832,7 +804,7 @@ static int cz_tf_set_watermark_threshold(struct pp_hwmgr *hwmgr,
 
 	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
 					PPSMC_MSG_SetWatermarkFrequency,
-				      cz_hwmgr->sclk_dpm.soft_max_clk);
+					cz_hwmgr->sclk_dpm.soft_max_clk);
 
 	return 0;
 }
@@ -858,9 +830,9 @@ static int cz_tf_enable_nb_dpm(struct pp_hwmgr *hwmgr,
 		PP_DBG_LOG("enabling ALL SMU features.\n");
 		dpm_features |= NB_DPM_MASK;
 		ret = smum_send_msg_to_smc_with_parameter(
-							     hwmgr->smumgr,
-					 PPSMC_MSG_EnableAllSmuFeatures,
-							     dpm_features);
+							  hwmgr->smumgr,
+							  PPSMC_MSG_EnableAllSmuFeatures,
+							  dpm_features);
 		if (ret == 0)
 			cz_hwmgr->is_nb_dpm_enabled = true;
 	}
@@ -1246,7 +1218,7 @@ static int cz_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 
 static int cz_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
 {
-	if (hwmgr != NULL || hwmgr->backend != NULL) {
+	if (hwmgr != NULL && hwmgr->backend != NULL) {
 		kfree(hwmgr->backend);
 		kfree(hwmgr);
 	}
@@ -1402,10 +1374,12 @@ int cz_dpm_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
 						   PPSMC_MSG_SetUvdHardMin));
 
 			cz_enable_disable_uvd_dpm(hwmgr, true);
-		} else
+		} else {
 			cz_enable_disable_uvd_dpm(hwmgr, true);
-	} else
+		}
+	} else {
 		cz_enable_disable_uvd_dpm(hwmgr, false);
+	}
 
 	return 0;
 }
@@ -1564,78 +1538,6 @@ int cz_get_power_state_size(struct pp_hwmgr *hwmgr)
 	return sizeof(struct cz_power_state);
 }
 
-static void
-cz_print_current_perforce_level(struct pp_hwmgr *hwmgr, struct seq_file *m)
-{
-	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
-
-	struct phm_clock_voltage_dependency_table *table =
-				hwmgr->dyn_state.vddc_dependency_on_sclk;
-
-	struct phm_vce_clock_voltage_dependency_table *vce_table =
-		hwmgr->dyn_state.vce_clock_voltage_dependency_table;
-
-	struct phm_uvd_clock_voltage_dependency_table *uvd_table =
-		hwmgr->dyn_state.uvd_clock_voltage_dependency_table;
-
-	uint32_t sclk_index = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixTARGET_AND_CURRENT_PROFILE_INDEX),
-					TARGET_AND_CURRENT_PROFILE_INDEX, CURR_SCLK_INDEX);
-	uint32_t uvd_index = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixTARGET_AND_CURRENT_PROFILE_INDEX_2),
-					TARGET_AND_CURRENT_PROFILE_INDEX_2, CURR_UVD_INDEX);
-	uint32_t vce_index = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixTARGET_AND_CURRENT_PROFILE_INDEX_2),
-					TARGET_AND_CURRENT_PROFILE_INDEX_2, CURR_VCE_INDEX);
-
-	uint32_t sclk, vclk, dclk, ecclk, tmp, activity_percent;
-	uint16_t vddnb, vddgfx;
-	int result;
-
-	if (sclk_index >= NUM_SCLK_LEVELS) {
-		seq_printf(m, "\n invalid sclk dpm profile %d\n", sclk_index);
-	} else {
-		sclk = table->entries[sclk_index].clk;
-		seq_printf(m, "\n index: %u sclk: %u MHz\n", sclk_index, sclk/100);
-	}
-
-	tmp = (cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixSMUSVI_NB_CURRENTVID) &
-		CURRENT_NB_VID_MASK) >> CURRENT_NB_VID__SHIFT;
-	vddnb = cz_convert_8Bit_index_to_voltage(hwmgr, tmp);
-	tmp = (cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixSMUSVI_GFX_CURRENTVID) &
-		CURRENT_GFX_VID_MASK) >> CURRENT_GFX_VID__SHIFT;
-	vddgfx = cz_convert_8Bit_index_to_voltage(hwmgr, (u16)tmp);
-	seq_printf(m, "\n vddnb: %u vddgfx: %u\n", vddnb, vddgfx);
-
-	seq_printf(m, "\n uvd    %sabled\n", cz_hwmgr->uvd_power_gated ? "dis" : "en");
-	if (!cz_hwmgr->uvd_power_gated) {
-		if (uvd_index >= CZ_MAX_HARDWARE_POWERLEVELS) {
-			seq_printf(m, "\n invalid uvd dpm level %d\n", uvd_index);
-		} else {
-			vclk = uvd_table->entries[uvd_index].vclk;
-			dclk = uvd_table->entries[uvd_index].dclk;
-			seq_printf(m, "\n index: %u uvd vclk: %u MHz dclk: %u MHz\n", uvd_index, vclk/100, dclk/100);
-		}
-	}
-
-	seq_printf(m, "\n vce    %sabled\n", cz_hwmgr->vce_power_gated ? "dis" : "en");
-	if (!cz_hwmgr->vce_power_gated) {
-		if (vce_index >= CZ_MAX_HARDWARE_POWERLEVELS) {
-			seq_printf(m, "\n invalid vce dpm level %d\n", vce_index);
-		} else {
-			ecclk = vce_table->entries[vce_index].ecclk;
-			seq_printf(m, "\n index: %u vce ecclk: %u MHz\n", vce_index, ecclk/100);
-		}
-	}
-
-	result = smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_GetAverageGraphicsActivity);
-	if (0 == result) {
-		activity_percent = cgs_read_register(hwmgr->device, mmSMU_MP1_SRBM2P_ARG_0);
-		activity_percent = activity_percent > 100 ? 100 : activity_percent;
-	} else {
-		activity_percent = 50;
-	}
-
-	seq_printf(m, "\n [GPU load]: %u %%\n\n", activity_percent);
-}
-
 static void cz_hw_print_display_cfg(
 	const struct cc6_settings *cc6_settings)
 {
@@ -1690,13 +1592,10 @@ static int cz_store_cc6_data(struct pp_hwmgr *hwmgr, uint32_t separation_time,
 	struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend);
 
 	if (separation_time !=
-		hw_data->cc6_settings.cpu_pstate_separation_time
-		|| cc6_disable !=
-		hw_data->cc6_settings.cpu_cc6_disable
-		|| pstate_disable !=
-		hw_data->cc6_settings.cpu_pstate_disable
-		|| pstate_switch_disable !=
-		hw_data->cc6_settings.nb_pstate_switch_disable) {
+	    hw_data->cc6_settings.cpu_pstate_separation_time ||
+	    cc6_disable != hw_data->cc6_settings.cpu_cc6_disable ||
+	    pstate_disable != hw_data->cc6_settings.cpu_pstate_disable ||
+	    pstate_switch_disable != hw_data->cc6_settings.nb_pstate_switch_disable) {
 
 		hw_data->cc6_settings.cc6_setting_changed = true;
 
@@ -1799,8 +1698,7 @@ static int cz_get_performance_level(struct pp_hwmgr *hwmgr, const struct pp_hw_p
 	ps = cast_const_PhwCzPowerState(state);
 
 	level_index = index > ps->level - 1 ? ps->level - 1 : index;
-
-	level->coreClock  = ps->levels[level_index].engineClock;
+	level->coreClock = ps->levels[level_index].engineClock;
 
 	if (designation == PHM_PerformanceLevelDesignation_PowerContainment) {
 		for (i = 1; i < ps->level; i++) {
@@ -1887,6 +1785,107 @@ static int cz_get_max_high_clocks(struct pp_hwmgr *hwmgr, struct amd_pp_simple_c
 	return 0;
 }
 
+static int cz_read_sensor(struct pp_hwmgr *hwmgr, int idx, int32_t *value)
+{
+	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
+
+	struct phm_clock_voltage_dependency_table *table =
+				hwmgr->dyn_state.vddc_dependency_on_sclk;
+
+	struct phm_vce_clock_voltage_dependency_table *vce_table =
+		hwmgr->dyn_state.vce_clock_voltage_dependency_table;
+
+	struct phm_uvd_clock_voltage_dependency_table *uvd_table =
+		hwmgr->dyn_state.uvd_clock_voltage_dependency_table;
+
+	uint32_t sclk_index = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixTARGET_AND_CURRENT_PROFILE_INDEX),
+					TARGET_AND_CURRENT_PROFILE_INDEX, CURR_SCLK_INDEX);
+	uint32_t uvd_index = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixTARGET_AND_CURRENT_PROFILE_INDEX_2),
+					TARGET_AND_CURRENT_PROFILE_INDEX_2, CURR_UVD_INDEX);
+	uint32_t vce_index = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixTARGET_AND_CURRENT_PROFILE_INDEX_2),
+					TARGET_AND_CURRENT_PROFILE_INDEX_2, CURR_VCE_INDEX);
+
+	uint32_t sclk, vclk, dclk, ecclk, tmp, activity_percent;
+	uint16_t vddnb, vddgfx;
+	int result;
+
+	switch (idx) {
+	case AMDGPU_PP_SENSOR_GFX_SCLK:
+		if (sclk_index < NUM_SCLK_LEVELS) {
+			sclk = table->entries[sclk_index].clk;
+			*value = sclk;
+			return 0;
+		}
+		return -EINVAL;
+	case AMDGPU_PP_SENSOR_VDDNB:
+		tmp = (cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixSMUSVI_NB_CURRENTVID) &
+			CURRENT_NB_VID_MASK) >> CURRENT_NB_VID__SHIFT;
+		vddnb = cz_convert_8Bit_index_to_voltage(hwmgr, tmp);
+		*value = vddnb;
+		return 0;
+	case AMDGPU_PP_SENSOR_VDDGFX:
+		tmp = (cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixSMUSVI_GFX_CURRENTVID) &
+			CURRENT_GFX_VID_MASK) >> CURRENT_GFX_VID__SHIFT;
+		vddgfx = cz_convert_8Bit_index_to_voltage(hwmgr, (u16)tmp);
+		*value = vddgfx;
+		return 0;
+	case AMDGPU_PP_SENSOR_UVD_VCLK:
+		if (!cz_hwmgr->uvd_power_gated) {
+			if (uvd_index >= CZ_MAX_HARDWARE_POWERLEVELS) {
+				return -EINVAL;
+			} else {
+				vclk = uvd_table->entries[uvd_index].vclk;
+				*value = vclk;
+				return 0;
+			}
+		}
+		*value = 0;
+		return 0;
+	case AMDGPU_PP_SENSOR_UVD_DCLK:
+		if (!cz_hwmgr->uvd_power_gated) {
+			if (uvd_index >= CZ_MAX_HARDWARE_POWERLEVELS) {
+				return -EINVAL;
+			} else {
+				dclk = uvd_table->entries[uvd_index].dclk;
+				*value = dclk;
+				return 0;
+			}
+		}
+		*value = 0;
+		return 0;
+	case AMDGPU_PP_SENSOR_VCE_ECCLK:
+		if (!cz_hwmgr->vce_power_gated) {
+			if (vce_index >= CZ_MAX_HARDWARE_POWERLEVELS) {
+				return -EINVAL;
+			} else {
+				ecclk = vce_table->entries[vce_index].ecclk;
+				*value = ecclk;
+				return 0;
+			}
+		}
+		*value = 0;
+		return 0;
+	case AMDGPU_PP_SENSOR_GPU_LOAD:
+		result = smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_GetAverageGraphicsActivity);
+		if (0 == result) {
+			activity_percent = cgs_read_register(hwmgr->device, mmSMU_MP1_SRBM2P_ARG_0);
+			activity_percent = activity_percent > 100 ? 100 : activity_percent;
+		} else {
+			activity_percent = 50;
+		}
+		*value = activity_percent;
+		return 0;
+	case AMDGPU_PP_SENSOR_UVD_POWER:
+		*value = cz_hwmgr->uvd_power_gated ? 0 : 1;
+		return 0;
+	case AMDGPU_PP_SENSOR_VCE_POWER:
+		*value = cz_hwmgr->vce_power_gated ? 0 : 1;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
 static const struct pp_hwmgr_func cz_hwmgr_funcs = {
 	.backend_init = cz_hwmgr_backend_init,
 	.backend_fini = cz_hwmgr_backend_fini,
@@ -1902,7 +1901,6 @@ static const struct pp_hwmgr_func cz_hwmgr_funcs = {
 	.patch_boot_state = cz_dpm_patch_boot_state,
 	.get_pp_table_entry = cz_dpm_get_pp_table_entry,
 	.get_num_of_pp_table_entries = cz_dpm_get_num_of_pp_table_entries,
-	.print_current_perforce_level = cz_print_current_perforce_level,
 	.set_cpu_power_state = cz_set_cpu_power_state,
 	.store_cc6_data = cz_store_cc6_data,
 	.force_clock_level = cz_force_clock_level,
@@ -1912,6 +1910,7 @@ static const struct pp_hwmgr_func cz_hwmgr_funcs = {
 	.get_current_shallow_sleep_clocks = cz_get_current_shallow_sleep_clocks,
 	.get_clock_by_type = cz_get_clock_by_type,
 	.get_max_high_clocks = cz_get_max_high_clocks,
+	.read_sensor = cz_read_sensor,
 };
 
 int cz_hwmgr_init(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c
deleted file mode 100644
index 5afe82068b29..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "hwmgr.h"
-#include "fiji_clockpowergating.h"
-#include "fiji_ppsmc.h"
-#include "fiji_hwmgr.h"
-
-int fiji_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	data->uvd_power_gated = false;
-	data->vce_power_gated = false;
-	data->samu_power_gated = false;
-	data->acp_power_gated = false;
-
-	return 0;
-}
-
-int fiji_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (data->uvd_power_gated == bgate)
-		return 0;
-
-	data->uvd_power_gated = bgate;
-
-	if (bgate) {
-		cgs_set_clockgating_state(hwmgr->device,
-					  AMD_IP_BLOCK_TYPE_UVD,
-					  AMD_CG_STATE_GATE);
-		fiji_update_uvd_dpm(hwmgr, true);
-	} else {
-		fiji_update_uvd_dpm(hwmgr, false);
-		cgs_set_clockgating_state(hwmgr->device,
-					  AMD_IP_BLOCK_TYPE_UVD,
-					  AMD_CG_STATE_UNGATE);
-	}
-
-	return 0;
-}
-
-int fiji_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_set_power_state_input states;
-	const struct pp_power_state  *pcurrent;
-	struct pp_power_state  *requested;
-
-	if (data->vce_power_gated == bgate)
-		return 0;
-
-	data->vce_power_gated = bgate;
-
-	pcurrent = hwmgr->current_ps;
-	requested = hwmgr->request_ps;
-
-	states.pcurrent_state = &(pcurrent->hardware);
-	states.pnew_state = &(requested->hardware);
-
-	fiji_update_vce_dpm(hwmgr, &states);
-	fiji_enable_disable_vce_dpm(hwmgr, !bgate);
-
-	return 0;
-}
-
-int fiji_phm_powergate_samu(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (data->samu_power_gated == bgate)
-		return 0;
-
-	data->samu_power_gated = bgate;
-
-	if (bgate)
-		fiji_update_samu_dpm(hwmgr, true);
-	else
-		fiji_update_samu_dpm(hwmgr, false);
-
-	return 0;
-}
-
-int fiji_phm_powergate_acp(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (data->acp_power_gated == bgate)
-		return 0;
-
-	data->acp_power_gated = bgate;
-
-	if (bgate)
-		fiji_update_acp_dpm(hwmgr, true);
-	else
-		fiji_update_acp_dpm(hwmgr, false);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_dyn_defaults.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_dyn_defaults.h
deleted file mode 100644
index 32d43e8fecb2..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_dyn_defaults.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef FIJI_DYN_DEFAULTS_H
-#define FIJI_DYN_DEFAULTS_H
-
-/** \file
-* Volcanic Islands Dynamic default parameters.
-*/
-
-enum FIJIdpm_TrendDetection
-{
-    FIJIAdpm_TrendDetection_AUTO,
-    FIJIAdpm_TrendDetection_UP,
-    FIJIAdpm_TrendDetection_DOWN
-};
-typedef enum FIJIdpm_TrendDetection FIJIdpm_TrendDetection;
-
-/* We need to fill in the default values!!!!!!!!!!!!!!!!!!!!!!! */
-
-/* Bit vector representing same fields as hardware register. */
-#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT0    0x3FFFC102  /* CP_Gfx_busy ????
-                                                         * HDP_busy
-                                                         * IH_busy
-                                                         * UVD_busy
-                                                         * VCE_busy
-                                                         * ACP_busy
-                                                         * SAMU_busy
-                                                         * SDMA enabled */
-#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT1    0x000400  /* FE_Gfx_busy  - Intended for primary usage.   Rest are for flexibility. ????
-                                                       * SH_Gfx_busy
-                                                       * RB_Gfx_busy
-                                                       * VCE_busy */
-
-#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT2    0xC00080  /* SH_Gfx_busy - Intended for primary usage.   Rest are for flexibility.
-                                                       * FE_Gfx_busy
-                                                       * RB_Gfx_busy
-                                                       * ACP_busy */
-
-#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT3    0xC00200  /* RB_Gfx_busy - Intended for primary usage.   Rest are for flexibility.
-                                                       * FE_Gfx_busy
-                                                       * SH_Gfx_busy
-                                                       * UVD_busy */
-
-#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT4    0xC01680  /* UVD_busy
-                                                       * VCE_busy
-                                                       * ACP_busy
-                                                       * SAMU_busy */
-
-#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT5    0xC00033  /* GFX, HDP */
-#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT6    0xC00033  /* GFX, HDP */
-#define PPFIJI_VOTINGRIGHTSCLIENTS_DFLT7    0x3FFFC000  /* GFX, HDP */
-
-
-/* thermal protection counter (units). */
-#define PPFIJI_THERMALPROTECTCOUNTER_DFLT            0x200 /* ~19us */
-
-/* static screen threshold unit */
-#define PPFIJI_STATICSCREENTHRESHOLDUNIT_DFLT    0
-
-/* static screen threshold */
-#define PPFIJI_STATICSCREENTHRESHOLD_DFLT        0x00C8
-
-/* gfx idle clock stop threshold */
-#define PPFIJI_GFXIDLECLOCKSTOPTHRESHOLD_DFLT        0x200 /* ~19us with static screen threshold unit of 0 */
-
-/* Fixed reference divider to use when building baby stepping tables. */
-#define PPFIJI_REFERENCEDIVIDER_DFLT                  4
-
-/* ULV voltage change delay time
- * Used to be delay_vreg in N.I. split for S.I.
- * Using N.I. delay_vreg value as default
- * ReferenceClock = 2700
- * VoltageResponseTime = 1000
- * VDDCDelayTime = (VoltageResponseTime * ReferenceClock) / 1600 = 1687
- */
-#define PPFIJI_ULVVOLTAGECHANGEDELAY_DFLT             1687
-
-#define PPFIJI_CGULVPARAMETER_DFLT			0x00040035
-#define PPFIJI_CGULVCONTROL_DFLT			0x00007450
-#define PPFIJI_TARGETACTIVITY_DFLT			30 /* 30%*/
-#define PPFIJI_MCLK_TARGETACTIVITY_DFLT		10 /* 10% */
-
-#endif
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c
deleted file mode 100644
index 120a9e2c3152..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c
+++ /dev/null
@@ -1,5599 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/fb.h>
-#include "linux/delay.h"
-
-#include "hwmgr.h"
-#include "fiji_smumgr.h"
-#include "atombios.h"
-#include "hardwaremanager.h"
-#include "ppatomctrl.h"
-#include "atombios.h"
-#include "cgs_common.h"
-#include "fiji_dyn_defaults.h"
-#include "fiji_powertune.h"
-#include "smu73.h"
-#include "smu/smu_7_1_3_d.h"
-#include "smu/smu_7_1_3_sh_mask.h"
-#include "gmc/gmc_8_1_d.h"
-#include "gmc/gmc_8_1_sh_mask.h"
-#include "bif/bif_5_0_d.h"
-#include "bif/bif_5_0_sh_mask.h"
-#include "dce/dce_10_0_d.h"
-#include "dce/dce_10_0_sh_mask.h"
-#include "pppcielanes.h"
-#include "fiji_hwmgr.h"
-#include "tonga_processpptables.h"
-#include "tonga_pptable.h"
-#include "pp_debug.h"
-#include "pp_acpi.h"
-#include "amd_pcie_helpers.h"
-#include "cgs_linux.h"
-#include "ppinterrupt.h"
-
-#include "fiji_clockpowergating.h"
-#include "fiji_thermal.h"
-
-#define VOLTAGE_SCALE	4
-#define SMC_RAM_END		0x40000
-#define VDDC_VDDCI_DELTA	300
-
-#define MC_SEQ_MISC0_GDDR5_SHIFT 28
-#define MC_SEQ_MISC0_GDDR5_MASK  0xf0000000
-#define MC_SEQ_MISC0_GDDR5_VALUE 5
-
-#define MC_CG_ARB_FREQ_F0           0x0a /* boot-up default */
-#define MC_CG_ARB_FREQ_F1           0x0b
-#define MC_CG_ARB_FREQ_F2           0x0c
-#define MC_CG_ARB_FREQ_F3           0x0d
-
-/* From smc_reg.h */
-#define SMC_CG_IND_START            0xc0030000
-#define SMC_CG_IND_END              0xc0040000  /* First byte after SMC_CG_IND */
-
-#define VOLTAGE_SCALE               4
-#define VOLTAGE_VID_OFFSET_SCALE1   625
-#define VOLTAGE_VID_OFFSET_SCALE2   100
-
-#define VDDC_VDDCI_DELTA            300
-
-#define ixSWRST_COMMAND_1           0x1400103
-#define MC_SEQ_CNTL__CAC_EN_MASK    0x40000000
-
-/** Values for the CG_THERMAL_CTRL::DPM_EVENT_SRC field. */
-enum DPM_EVENT_SRC {
-    DPM_EVENT_SRC_ANALOG = 0,               /* Internal analog trip point */
-    DPM_EVENT_SRC_EXTERNAL = 1,             /* External (GPIO 17) signal */
-    DPM_EVENT_SRC_DIGITAL = 2,              /* Internal digital trip point (DIG_THERM_DPM) */
-    DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,   /* Internal analog or external */
-    DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL = 4   /* Internal digital or external */
-};
-
-
-/* [2.5%,~2.5%] Clock stretched is multiple of 2.5% vs
- * not and [Fmin, Fmax, LDO_REFSEL, USE_FOR_LOW_FREQ]
- */
-static const uint16_t fiji_clock_stretcher_lookup_table[2][4] =
-{ {600, 1050, 3, 0}, {600, 1050, 6, 1} };
-
-/* [FF, SS] type, [] 4 voltage ranges, and
- * [Floor Freq, Boundary Freq, VID min , VID max]
- */
-static const uint32_t fiji_clock_stretcher_ddt_table[2][4][4] =
-{ { {265, 529, 120, 128}, {325, 650, 96, 119}, {430, 860, 32, 95}, {0, 0, 0, 31} },
-  { {275, 550, 104, 112}, {319, 638, 96, 103}, {360, 720, 64, 95}, {384, 768, 32, 63} } };
-
-/* [Use_For_Low_freq] value, [0%, 5%, 10%, 7.14%, 14.28%, 20%]
- * (coming from PWR_CKS_CNTL.stretch_amount reg spec)
- */
-static const uint8_t fiji_clock_stretch_amount_conversion[2][6] =
-{ {0, 1, 3, 2, 4, 5}, {0, 2, 4, 5, 6, 5} };
-
-static const unsigned long PhwFiji_Magic = (unsigned long)(PHM_VIslands_Magic);
-
-struct fiji_power_state *cast_phw_fiji_power_state(
-				  struct pp_hw_power_state *hw_ps)
-{
-	PP_ASSERT_WITH_CODE((PhwFiji_Magic == hw_ps->magic),
-				"Invalid Powerstate Type!",
-				 return NULL;);
-
-	return (struct fiji_power_state *)hw_ps;
-}
-
-const struct fiji_power_state *cast_const_phw_fiji_power_state(
-				 const struct pp_hw_power_state *hw_ps)
-{
-	PP_ASSERT_WITH_CODE((PhwFiji_Magic == hw_ps->magic),
-				"Invalid Powerstate Type!",
-				 return NULL;);
-
-	return (const struct fiji_power_state *)hw_ps;
-}
-
-static bool fiji_is_dpm_running(struct pp_hwmgr *hwmgr)
-{
-	return (1 == PHM_READ_INDIRECT_FIELD(hwmgr->device,
-			CGS_IND_REG__SMC, FEATURE_STATUS, VOLTAGE_CONTROLLER_ON))
-			? true : false;
-}
-
-static void fiji_init_dpm_defaults(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_ulv_parm *ulv = &data->ulv;
-
-	ulv->cg_ulv_parameter = PPFIJI_CGULVPARAMETER_DFLT;
-	data->voting_rights_clients0 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT0;
-	data->voting_rights_clients1 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT1;
-	data->voting_rights_clients2 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT2;
-	data->voting_rights_clients3 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT3;
-	data->voting_rights_clients4 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT4;
-	data->voting_rights_clients5 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT5;
-	data->voting_rights_clients6 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT6;
-	data->voting_rights_clients7 = PPFIJI_VOTINGRIGHTSCLIENTS_DFLT7;
-
-	data->static_screen_threshold_unit =
-			PPFIJI_STATICSCREENTHRESHOLDUNIT_DFLT;
-	data->static_screen_threshold =
-			PPFIJI_STATICSCREENTHRESHOLD_DFLT;
-
-	/* Unset ABM cap as it moved to DAL.
-	 * Add PHM_PlatformCaps_NonABMSupportInPPLib
-	 * for re-direct ABM related request to DAL
-	 */
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ABM);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_NonABMSupportInPPLib);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_DynamicACTiming);
-
-	fiji_initialize_power_tune_defaults(hwmgr);
-
-	data->mclk_stutter_mode_threshold = 60000;
-	data->pcie_gen_performance.max = PP_PCIEGen1;
-	data->pcie_gen_performance.min = PP_PCIEGen3;
-	data->pcie_gen_power_saving.max = PP_PCIEGen1;
-	data->pcie_gen_power_saving.min = PP_PCIEGen3;
-	data->pcie_lane_performance.max = 0;
-	data->pcie_lane_performance.min = 16;
-	data->pcie_lane_power_saving.max = 0;
-	data->pcie_lane_power_saving.min = 16;
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_DynamicUVDState);
-}
-
-static int fiji_get_sclk_for_voltage_evv(struct pp_hwmgr *hwmgr,
-	phm_ppt_v1_voltage_lookup_table *lookup_table,
-	uint16_t virtual_voltage_id, int32_t *sclk)
-{
-	uint8_t entryId;
-	uint8_t voltageId;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	PP_ASSERT_WITH_CODE(lookup_table->count != 0, "Lookup table is empty", return -EINVAL);
-
-	/* search for leakage voltage ID 0xff01 ~ 0xff08 and sckl */
-	for (entryId = 0; entryId < table_info->vdd_dep_on_sclk->count; entryId++) {
-		voltageId = table_info->vdd_dep_on_sclk->entries[entryId].vddInd;
-		if (lookup_table->entries[voltageId].us_vdd == virtual_voltage_id)
-			break;
-	}
-
-	PP_ASSERT_WITH_CODE(entryId < table_info->vdd_dep_on_sclk->count,
-			"Can't find requested voltage id in vdd_dep_on_sclk table!",
-			return -EINVAL;
-			);
-
-	*sclk = table_info->vdd_dep_on_sclk->entries[entryId].clk;
-
-	return 0;
-}
-
-/**
-* Get Leakage VDDC based on leakage ID.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int fiji_get_evv_voltages(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint16_t    vv_id;
-	uint16_t    vddc = 0;
-	uint16_t    evv_default = 1150;
-	uint16_t    i, j;
-	uint32_t  sclk = 0;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)hwmgr->pptable;
-	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
-			table_info->vdd_dep_on_sclk;
-	int result;
-
-	for (i = 0; i < FIJI_MAX_LEAKAGE_COUNT; i++) {
-		vv_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
-		if (!fiji_get_sclk_for_voltage_evv(hwmgr,
-				table_info->vddc_lookup_table, vv_id, &sclk)) {
-			if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_ClockStretcher)) {
-				for (j = 1; j < sclk_table->count; j++) {
-					if (sclk_table->entries[j].clk == sclk &&
-							sclk_table->entries[j].cks_enable == 0) {
-						sclk += 5000;
-						break;
-					}
-				}
-			}
-
-			if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_EnableDriverEVV))
-				result = atomctrl_calculate_voltage_evv_on_sclk(hwmgr,
-						VOLTAGE_TYPE_VDDC, sclk, vv_id, &vddc, i, true);
-			else
-				result = -EINVAL;
-
-			if (result)
-				result = atomctrl_get_voltage_evv_on_sclk(hwmgr,
-						VOLTAGE_TYPE_VDDC, sclk,vv_id, &vddc);
-
-			/* need to make sure vddc is less than 2v or else, it could burn the ASIC. */
-			PP_ASSERT_WITH_CODE((vddc < 2000),
-					"Invalid VDDC value, greater than 2v!", result = -EINVAL;);
-
-			if (result)
-				/* 1.15V is the default safe value for Fiji */
-				vddc = evv_default;
-
-			/* the voltage should not be zero nor equal to leakage ID */
-			if (vddc != 0 && vddc != vv_id) {
-				data->vddc_leakage.actual_voltage
-				[data->vddc_leakage.count] = vddc;
-				data->vddc_leakage.leakage_id
-				[data->vddc_leakage.count] = vv_id;
-				data->vddc_leakage.count++;
-			}
-		}
-	}
-	return 0;
-}
-
-/**
- * Change virtual leakage voltage to actual value.
- *
- * @param     hwmgr  the address of the powerplay hardware manager.
- * @param     pointer to changing voltage
- * @param     pointer to leakage table
- */
-static void fiji_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr,
-		uint16_t *voltage, struct fiji_leakage_voltage *leakage_table)
-{
-	uint32_t index;
-
-	/* search for leakage voltage ID 0xff01 ~ 0xff08 */
-	for (index = 0; index < leakage_table->count; index++) {
-		/* if this voltage matches a leakage voltage ID */
-		/* patch with actual leakage voltage */
-		if (leakage_table->leakage_id[index] == *voltage) {
-			*voltage = leakage_table->actual_voltage[index];
-			break;
-		}
-	}
-
-	if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
-		printk(KERN_ERR "Voltage value looks like a Leakage ID but it's not patched \n");
-}
-
-/**
-* Patch voltage lookup table by EVV leakages.
-*
-* @param     hwmgr  the address of the powerplay hardware manager.
-* @param     pointer to voltage lookup table
-* @param     pointer to leakage table
-* @return     always 0
-*/
-static int fiji_patch_lookup_table_with_leakage(struct pp_hwmgr *hwmgr,
-		phm_ppt_v1_voltage_lookup_table *lookup_table,
-		struct fiji_leakage_voltage *leakage_table)
-{
-	uint32_t i;
-
-	for (i = 0; i < lookup_table->count; i++)
-		fiji_patch_with_vdd_leakage(hwmgr,
-				&lookup_table->entries[i].us_vdd, leakage_table);
-
-	return 0;
-}
-
-static int fiji_patch_clock_voltage_limits_with_vddc_leakage(
-		struct pp_hwmgr *hwmgr, struct fiji_leakage_voltage *leakage_table,
-		uint16_t *vddc)
-{
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	fiji_patch_with_vdd_leakage(hwmgr, (uint16_t *)vddc, leakage_table);
-	hwmgr->dyn_state.max_clock_voltage_on_dc.vddc =
-			table_info->max_clock_voltage_on_dc.vddc;
-	return 0;
-}
-
-static int fiji_patch_voltage_dependency_tables_with_lookup_table(
-		struct pp_hwmgr *hwmgr)
-{
-	uint8_t entryId;
-	uint8_t voltageId;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
-			table_info->vdd_dep_on_sclk;
-	struct phm_ppt_v1_clock_voltage_dependency_table *mclk_table =
-			table_info->vdd_dep_on_mclk;
-	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
-			table_info->mm_dep_table;
-
-	for (entryId = 0; entryId < sclk_table->count; ++entryId) {
-		voltageId = sclk_table->entries[entryId].vddInd;
-		sclk_table->entries[entryId].vddc =
-				table_info->vddc_lookup_table->entries[voltageId].us_vdd;
-	}
-
-	for (entryId = 0; entryId < mclk_table->count; ++entryId) {
-		voltageId = mclk_table->entries[entryId].vddInd;
-		mclk_table->entries[entryId].vddc =
-			table_info->vddc_lookup_table->entries[voltageId].us_vdd;
-	}
-
-	for (entryId = 0; entryId < mm_table->count; ++entryId) {
-		voltageId = mm_table->entries[entryId].vddcInd;
-		mm_table->entries[entryId].vddc =
-			table_info->vddc_lookup_table->entries[voltageId].us_vdd;
-	}
-
-	return 0;
-
-}
-
-static int fiji_calc_voltage_dependency_tables(struct pp_hwmgr *hwmgr)
-{
-	/* Need to determine if we need calculated voltage. */
-	return 0;
-}
-
-static int fiji_calc_mm_voltage_dependency_table(struct pp_hwmgr *hwmgr)
-{
-	/* Need to determine if we need calculated voltage from mm table. */
-	return 0;
-}
-
-static int fiji_sort_lookup_table(struct pp_hwmgr *hwmgr,
-		struct phm_ppt_v1_voltage_lookup_table *lookup_table)
-{
-	uint32_t table_size, i, j;
-	struct phm_ppt_v1_voltage_lookup_record tmp_voltage_lookup_record;
-	table_size = lookup_table->count;
-
-	PP_ASSERT_WITH_CODE(0 != lookup_table->count,
-		"Lookup table is empty", return -EINVAL);
-
-	/* Sorting voltages */
-	for (i = 0; i < table_size - 1; i++) {
-		for (j = i + 1; j > 0; j--) {
-			if (lookup_table->entries[j].us_vdd <
-					lookup_table->entries[j - 1].us_vdd) {
-				tmp_voltage_lookup_record = lookup_table->entries[j - 1];
-				lookup_table->entries[j - 1] = lookup_table->entries[j];
-				lookup_table->entries[j] = tmp_voltage_lookup_record;
-			}
-		}
-	}
-
-	return 0;
-}
-
-static int fiji_complete_dependency_tables(struct pp_hwmgr *hwmgr)
-{
-	int result = 0;
-	int tmp_result;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	tmp_result = fiji_patch_lookup_table_with_leakage(hwmgr,
-			table_info->vddc_lookup_table, &(data->vddc_leakage));
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = fiji_patch_clock_voltage_limits_with_vddc_leakage(hwmgr,
-			&(data->vddc_leakage), &table_info->max_clock_voltage_on_dc.vddc);
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = fiji_patch_voltage_dependency_tables_with_lookup_table(hwmgr);
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = fiji_calc_voltage_dependency_tables(hwmgr);
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = fiji_calc_mm_voltage_dependency_table(hwmgr);
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = fiji_sort_lookup_table(hwmgr, table_info->vddc_lookup_table);
-	if(tmp_result)
-		result = tmp_result;
-
-	return result;
-}
-
-static int fiji_set_private_data_based_on_pptable(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	struct phm_ppt_v1_clock_voltage_dependency_table *allowed_sclk_vdd_table =
-			table_info->vdd_dep_on_sclk;
-	struct phm_ppt_v1_clock_voltage_dependency_table *allowed_mclk_vdd_table =
-			table_info->vdd_dep_on_mclk;
-
-	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table != NULL,
-		"VDD dependency on SCLK table is missing.	\
-		This table is mandatory", return -EINVAL);
-	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table->count >= 1,
-		"VDD dependency on SCLK table has to have is missing.	\
-		This table is mandatory", return -EINVAL);
-
-	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table != NULL,
-		"VDD dependency on MCLK table is missing.	\
-		This table is mandatory", return -EINVAL);
-	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table->count >= 1,
-		"VDD dependency on MCLK table has to have is missing.	 \
-		This table is mandatory", return -EINVAL);
-
-	data->min_vddc_in_pptable = (uint16_t)allowed_sclk_vdd_table->entries[0].vddc;
-	data->max_vddc_in_pptable =	(uint16_t)allowed_sclk_vdd_table->
-			entries[allowed_sclk_vdd_table->count - 1].vddc;
-
-	table_info->max_clock_voltage_on_ac.sclk =
-		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].clk;
-	table_info->max_clock_voltage_on_ac.mclk =
-		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].clk;
-	table_info->max_clock_voltage_on_ac.vddc =
-		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].vddc;
-	table_info->max_clock_voltage_on_ac.vddci =
-		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].vddci;
-
-	hwmgr->dyn_state.max_clock_voltage_on_ac.sclk =
-		table_info->max_clock_voltage_on_ac.sclk;
-	hwmgr->dyn_state.max_clock_voltage_on_ac.mclk =
-		table_info->max_clock_voltage_on_ac.mclk;
-	hwmgr->dyn_state.max_clock_voltage_on_ac.vddc =
-		table_info->max_clock_voltage_on_ac.vddc;
-	hwmgr->dyn_state.max_clock_voltage_on_ac.vddci =
-		table_info->max_clock_voltage_on_ac.vddci;
-
-	return 0;
-}
-
-static uint16_t fiji_get_current_pcie_speed(struct pp_hwmgr *hwmgr)
-{
-	uint32_t speedCntl = 0;
-
-	/* mmPCIE_PORT_INDEX rename as mmPCIE_INDEX */
-	speedCntl = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__PCIE,
-			ixPCIE_LC_SPEED_CNTL);
-	return((uint16_t)PHM_GET_FIELD(speedCntl,
-			PCIE_LC_SPEED_CNTL, LC_CURRENT_DATA_RATE));
-}
-
-static int fiji_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr)
-{
-	uint32_t link_width;
-
-	/* mmPCIE_PORT_INDEX rename as mmPCIE_INDEX */
-	link_width = PHM_READ_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE,
-			PCIE_LC_LINK_WIDTH_CNTL, LC_LINK_WIDTH_RD);
-
-	PP_ASSERT_WITH_CODE((7 >= link_width),
-			"Invalid PCIe lane width!", return 0);
-
-	return decode_pcie_lane_width(link_width);
-}
-
-/** Patch the Boot State to match VBIOS boot clocks and voltage.
-*
-* @param hwmgr Pointer to the hardware manager.
-* @param pPowerState The address of the PowerState instance being created.
-*
-*/
-static int fiji_patch_boot_state(struct pp_hwmgr *hwmgr,
-		struct pp_hw_power_state *hw_ps)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_power_state *ps = (struct fiji_power_state *)hw_ps;
-	ATOM_FIRMWARE_INFO_V2_2 *fw_info;
-	uint16_t size;
-	uint8_t frev, crev;
-	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
-
-	/* First retrieve the Boot clocks and VDDC from the firmware info table.
-	 * We assume here that fw_info is unchanged if this call fails.
-	 */
-	fw_info = (ATOM_FIRMWARE_INFO_V2_2 *)cgs_atom_get_data_table(
-			hwmgr->device, index,
-			&size, &frev, &crev);
-	if (!fw_info)
-		/* During a test, there is no firmware info table. */
-		return 0;
-
-	/* Patch the state. */
-	data->vbios_boot_state.sclk_bootup_value =
-			le32_to_cpu(fw_info->ulDefaultEngineClock);
-	data->vbios_boot_state.mclk_bootup_value =
-			le32_to_cpu(fw_info->ulDefaultMemoryClock);
-	data->vbios_boot_state.mvdd_bootup_value =
-			le16_to_cpu(fw_info->usBootUpMVDDCVoltage);
-	data->vbios_boot_state.vddc_bootup_value =
-			le16_to_cpu(fw_info->usBootUpVDDCVoltage);
-	data->vbios_boot_state.vddci_bootup_value =
-			le16_to_cpu(fw_info->usBootUpVDDCIVoltage);
-	data->vbios_boot_state.pcie_gen_bootup_value =
-			fiji_get_current_pcie_speed(hwmgr);
-	data->vbios_boot_state.pcie_lane_bootup_value =
-			(uint16_t)fiji_get_current_pcie_lane_number(hwmgr);
-
-	/* set boot power state */
-	ps->performance_levels[0].memory_clock = data->vbios_boot_state.mclk_bootup_value;
-	ps->performance_levels[0].engine_clock = data->vbios_boot_state.sclk_bootup_value;
-	ps->performance_levels[0].pcie_gen = data->vbios_boot_state.pcie_gen_bootup_value;
-	ps->performance_levels[0].pcie_lane = data->vbios_boot_state.pcie_lane_bootup_value;
-
-	return 0;
-}
-
-static int fiji_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
-{
-	return phm_hwmgr_backend_fini(hwmgr);
-}
-
-static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data;
-	uint32_t i;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	bool stay_in_boot;
-	int result;
-
-	data = kzalloc(sizeof(struct fiji_hwmgr), GFP_KERNEL);
-	if (data == NULL)
-		return -ENOMEM;
-
-	hwmgr->backend = data;
-
-	data->dll_default_on = false;
-	data->sram_end = SMC_RAM_END;
-
-	for (i = 0; i < SMU73_MAX_LEVELS_GRAPHICS; i++)
-		data->activity_target[i] = FIJI_AT_DFLT;
-
-	data->vddc_vddci_delta = VDDC_VDDCI_DELTA;
-
-	data->mclk_activity_target = PPFIJI_MCLK_TARGETACTIVITY_DFLT;
-	data->mclk_dpm0_activity_target = 0xa;
-
-	data->sclk_dpm_key_disabled = 0;
-	data->mclk_dpm_key_disabled = 0;
-	data->pcie_dpm_key_disabled = 0;
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_UnTabledHardwareInterface);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_TablelessHardwareInterface);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkDeepSleep);
-
-	data->gpio_debug = 0;
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_DynamicPatchPowerState);
-
-	/* need to set voltage control types before EVV patching */
-	data->voltage_control = FIJI_VOLTAGE_CONTROL_NONE;
-	data->vddci_control = FIJI_VOLTAGE_CONTROL_NONE;
-	data->mvdd_control = FIJI_VOLTAGE_CONTROL_NONE;
-
-	data->force_pcie_gen = PP_PCIEGenInvalid;
-
-	if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-			VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2))
-		data->voltage_control = FIJI_VOLTAGE_CONTROL_BY_SVID2;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_EnableMVDDControl))
-		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-				VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT))
-			data->mvdd_control = FIJI_VOLTAGE_CONTROL_BY_GPIO;
-
-	if (data->mvdd_control == FIJI_VOLTAGE_CONTROL_NONE)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_EnableMVDDControl);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ControlVDDCI)) {
-		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT))
-			data->vddci_control = FIJI_VOLTAGE_CONTROL_BY_GPIO;
-		else if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2))
-			data->vddci_control = FIJI_VOLTAGE_CONTROL_BY_SVID2;
-	}
-
-	if (data->vddci_control == FIJI_VOLTAGE_CONTROL_NONE)
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_ControlVDDCI);
-
-	if (table_info && table_info->cac_dtp_table->usClockStretchAmount)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_ClockStretcher);
-
-	fiji_init_dpm_defaults(hwmgr);
-
-	/* Get leakage voltage based on leakage ID. */
-	fiji_get_evv_voltages(hwmgr);
-
-	/* Patch our voltage dependency table with actual leakage voltage
-	 * We need to perform leakage translation before it's used by other functions
-	 */
-	fiji_complete_dependency_tables(hwmgr);
-
-	/* Parse pptable data read from VBIOS */
-	fiji_set_private_data_based_on_pptable(hwmgr);
-
-	/* ULV Support */
-	data->ulv.ulv_supported = true; /* ULV feature is enabled by default */
-
-	/* Initalize Dynamic State Adjustment Rule Settings */
-	result = tonga_initializa_dynamic_state_adjustment_rule_settings(hwmgr);
-
-	if (!result) {
-		data->uvd_enabled = false;
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_EnableSMU7ThermalManagement);
-		data->vddc_phase_shed_control = false;
-	}
-
-	stay_in_boot = phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_StayInBootState);
-
-	if (0 == result) {
-		struct cgs_system_info sys_info = {0};
-
-		data->is_tlu_enabled = false;
-		hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =
-				FIJI_MAX_HARDWARE_POWERLEVELS;
-		hwmgr->platform_descriptor.hardwarePerformanceLevels = 2;
-		hwmgr->platform_descriptor.minimumClocksReductionPercentage  = 50;
-
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_FanSpeedInTableIsRPM);
-
-		if (table_info->cac_dtp_table->usDefaultTargetOperatingTemp &&
-				hwmgr->thermal_controller.
-				advanceFanControlParameters.ucFanControlMode) {
-			hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanPWM =
-					hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanPWM;
-			hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanRPM =
-					hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanRPM;
-			hwmgr->dyn_state.cac_dtp_table->usOperatingTempMinLimit =
-					table_info->cac_dtp_table->usOperatingTempMinLimit;
-			hwmgr->dyn_state.cac_dtp_table->usOperatingTempMaxLimit =
-					table_info->cac_dtp_table->usOperatingTempMaxLimit;
-			hwmgr->dyn_state.cac_dtp_table->usDefaultTargetOperatingTemp =
-					table_info->cac_dtp_table->usDefaultTargetOperatingTemp;
-			hwmgr->dyn_state.cac_dtp_table->usOperatingTempStep =
-					table_info->cac_dtp_table->usOperatingTempStep;
-			hwmgr->dyn_state.cac_dtp_table->usTargetOperatingTemp =
-					table_info->cac_dtp_table->usTargetOperatingTemp;
-
-			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_ODFuzzyFanControlSupport);
-		}
-
-		sys_info.size = sizeof(struct cgs_system_info);
-		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO;
-		result = cgs_query_system_info(hwmgr->device, &sys_info);
-		if (result)
-			data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK;
-		else
-			data->pcie_gen_cap = (uint32_t)sys_info.value;
-		if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
-			data->pcie_spc_cap = 20;
-		sys_info.size = sizeof(struct cgs_system_info);
-		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW;
-		result = cgs_query_system_info(hwmgr->device, &sys_info);
-		if (result)
-			data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK;
-		else
-			data->pcie_lane_cap = (uint32_t)sys_info.value;
-	} else {
-		/* Ignore return value in here, we are cleaning up a mess. */
-		fiji_hwmgr_backend_fini(hwmgr);
-	}
-
-	return 0;
-}
-
-/**
- * Read clock related registers.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-static int fiji_read_clock_registers(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	data->clock_registers.vCG_SPLL_FUNC_CNTL =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixCG_SPLL_FUNC_CNTL);
-	data->clock_registers.vCG_SPLL_FUNC_CNTL_2 =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixCG_SPLL_FUNC_CNTL_2);
-	data->clock_registers.vCG_SPLL_FUNC_CNTL_3 =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixCG_SPLL_FUNC_CNTL_3);
-	data->clock_registers.vCG_SPLL_FUNC_CNTL_4 =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixCG_SPLL_FUNC_CNTL_4);
-	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixCG_SPLL_SPREAD_SPECTRUM);
-	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2 =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixCG_SPLL_SPREAD_SPECTRUM_2);
-
-	return 0;
-}
-
-/**
- * Find out if memory is GDDR5.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-static int fiji_get_memory_type(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t temp;
-
-	temp = cgs_read_register(hwmgr->device, mmMC_SEQ_MISC0);
-
-	data->is_memory_gddr5 = (MC_SEQ_MISC0_GDDR5_VALUE ==
-			((temp & MC_SEQ_MISC0_GDDR5_MASK) >>
-			 MC_SEQ_MISC0_GDDR5_SHIFT));
-
-	return 0;
-}
-
-/**
- * Enables Dynamic Power Management by SMC
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-static int fiji_enable_acpi_power_management(struct pp_hwmgr *hwmgr)
-{
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			GENERAL_PWRMGT, STATIC_PM_EN, 1);
-
-	return 0;
-}
-
-/**
- * Initialize PowerGating States for different engines
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-static int fiji_init_power_gate_state(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	data->uvd_power_gated = false;
-	data->vce_power_gated = false;
-	data->samu_power_gated = false;
-	data->acp_power_gated = false;
-	data->pg_acp_init = true;
-
-	return 0;
-}
-
-static int fiji_init_sclk_threshold(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	data->low_sclk_interrupt_threshold = 0;
-
-	return 0;
-}
-
-static int fiji_setup_asic_task(struct pp_hwmgr *hwmgr)
-{
-	int tmp_result, result = 0;
-
-	tmp_result = fiji_read_clock_registers(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to read clock registers!", result = tmp_result);
-
-	tmp_result = fiji_get_memory_type(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to get memory type!", result = tmp_result);
-
-	tmp_result = fiji_enable_acpi_power_management(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable ACPI power management!", result = tmp_result);
-
-	tmp_result = fiji_init_power_gate_state(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to init power gate state!", result = tmp_result);
-
-	tmp_result = tonga_get_mc_microcode_version(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to get MC microcode version!", result = tmp_result);
-
-	tmp_result = fiji_init_sclk_threshold(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to init sclk threshold!", result = tmp_result);
-
-	return result;
-}
-
-/**
-* Checks if we want to support voltage control
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-*/
-static bool fiji_voltage_control(const struct pp_hwmgr *hwmgr)
-{
-	const struct fiji_hwmgr *data =
-			(const struct fiji_hwmgr *)(hwmgr->backend);
-
-	return (FIJI_VOLTAGE_CONTROL_NONE != data->voltage_control);
-}
-
-/**
-* Enable voltage control
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int fiji_enable_voltage_control(struct pp_hwmgr *hwmgr)
-{
-	/* enable voltage control */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			GENERAL_PWRMGT, VOLT_PWRMGT_EN, 1);
-
-	return 0;
-}
-
-/**
-* Remove repeated voltage values and create table with unique values.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    vol_table  the pointer to changing voltage table
-* @return    0 in success
-*/
-
-static int fiji_trim_voltage_table(struct pp_hwmgr *hwmgr,
-		struct pp_atomctrl_voltage_table *vol_table)
-{
-	uint32_t i, j;
-	uint16_t vvalue;
-	bool found = false;
-	struct pp_atomctrl_voltage_table *table;
-
-	PP_ASSERT_WITH_CODE((NULL != vol_table),
-			"Voltage Table empty.", return -EINVAL);
-	table = kzalloc(sizeof(struct pp_atomctrl_voltage_table),
-			GFP_KERNEL);
-
-	if (NULL == table)
-		return -ENOMEM;
-
-	table->mask_low = vol_table->mask_low;
-	table->phase_delay = vol_table->phase_delay;
-
-	for (i = 0; i < vol_table->count; i++) {
-		vvalue = vol_table->entries[i].value;
-		found = false;
-
-		for (j = 0; j < table->count; j++) {
-			if (vvalue == table->entries[j].value) {
-				found = true;
-				break;
-			}
-		}
-
-		if (!found) {
-			table->entries[table->count].value = vvalue;
-			table->entries[table->count].smio_low =
-					vol_table->entries[i].smio_low;
-			table->count++;
-		}
-	}
-
-	memcpy(vol_table, table, sizeof(struct pp_atomctrl_voltage_table));
-	kfree(table);
-
-	return 0;
-}
-
-static int fiji_get_svi2_mvdd_voltage_table(struct pp_hwmgr *hwmgr,
-		phm_ppt_v1_clock_voltage_dependency_table *dep_table)
-{
-	uint32_t i;
-	int result;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct pp_atomctrl_voltage_table *vol_table = &(data->mvdd_voltage_table);
-
-	PP_ASSERT_WITH_CODE((0 != dep_table->count),
-			"Voltage Dependency Table empty.", return -EINVAL);
-
-	vol_table->mask_low = 0;
-	vol_table->phase_delay = 0;
-	vol_table->count = dep_table->count;
-
-	for (i = 0; i < dep_table->count; i++) {
-		vol_table->entries[i].value = dep_table->entries[i].mvdd;
-		vol_table->entries[i].smio_low = 0;
-	}
-
-	result = fiji_trim_voltage_table(hwmgr, vol_table);
-	PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to trim MVDD table.", return result);
-
-	return 0;
-}
-
-static int fiji_get_svi2_vddci_voltage_table(struct pp_hwmgr *hwmgr,
-		phm_ppt_v1_clock_voltage_dependency_table *dep_table)
-{
-	uint32_t i;
-	int result;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct pp_atomctrl_voltage_table *vol_table = &(data->vddci_voltage_table);
-
-	PP_ASSERT_WITH_CODE((0 != dep_table->count),
-			"Voltage Dependency Table empty.", return -EINVAL);
-
-	vol_table->mask_low = 0;
-	vol_table->phase_delay = 0;
-	vol_table->count = dep_table->count;
-
-	for (i = 0; i < dep_table->count; i++) {
-		vol_table->entries[i].value = dep_table->entries[i].vddci;
-		vol_table->entries[i].smio_low = 0;
-	}
-
-	result = fiji_trim_voltage_table(hwmgr, vol_table);
-	PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to trim VDDCI table.", return result);
-
-	return 0;
-}
-
-static int fiji_get_svi2_vdd_voltage_table(struct pp_hwmgr *hwmgr,
-		phm_ppt_v1_voltage_lookup_table *lookup_table)
-{
-	int i = 0;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct pp_atomctrl_voltage_table *vol_table = &(data->vddc_voltage_table);
-
-	PP_ASSERT_WITH_CODE((0 != lookup_table->count),
-			"Voltage Lookup Table empty.", return -EINVAL);
-
-	vol_table->mask_low = 0;
-	vol_table->phase_delay = 0;
-
-	vol_table->count = lookup_table->count;
-
-	for (i = 0; i < vol_table->count; i++) {
-		vol_table->entries[i].value = lookup_table->entries[i].us_vdd;
-		vol_table->entries[i].smio_low = 0;
-	}
-
-	return 0;
-}
-
-/* ---- Voltage Tables ----
- * If the voltage table would be bigger than
- * what will fit into the state table on
- * the SMC keep only the higher entries.
- */
-static void fiji_trim_voltage_table_to_fit_state_table(struct pp_hwmgr *hwmgr,
-		uint32_t max_vol_steps, struct pp_atomctrl_voltage_table *vol_table)
-{
-	unsigned int i, diff;
-
-	if (vol_table->count <= max_vol_steps)
-		return;
-
-	diff = vol_table->count - max_vol_steps;
-
-	for (i = 0; i < max_vol_steps; i++)
-		vol_table->entries[i] = vol_table->entries[i + diff];
-
-	vol_table->count = max_vol_steps;
-
-	return;
-}
-
-/**
-* Create Voltage Tables.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int fiji_construct_voltage_tables(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)hwmgr->pptable;
-	int result;
-
-	if (FIJI_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
-		result = atomctrl_get_voltage_table_v3(hwmgr,
-				VOLTAGE_TYPE_MVDDC,	VOLTAGE_OBJ_GPIO_LUT,
-				&(data->mvdd_voltage_table));
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve MVDD table.",
-				return result);
-	} else if (FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
-		result = fiji_get_svi2_mvdd_voltage_table(hwmgr,
-				table_info->vdd_dep_on_mclk);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve SVI2 MVDD table from dependancy table.",
-				return result;);
-	}
-
-	if (FIJI_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
-		result = atomctrl_get_voltage_table_v3(hwmgr,
-				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT,
-				&(data->vddci_voltage_table));
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve VDDCI table.",
-				return result);
-	} else if (FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
-		result = fiji_get_svi2_vddci_voltage_table(hwmgr,
-				table_info->vdd_dep_on_mclk);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve SVI2 VDDCI table from dependancy table.",
-				return result);
-	}
-
-	if(FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
-		result = fiji_get_svi2_vdd_voltage_table(hwmgr,
-				table_info->vddc_lookup_table);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve SVI2 VDDC table from lookup table.",
-				return result);
-	}
-
-	PP_ASSERT_WITH_CODE(
-			(data->vddc_voltage_table.count <= (SMU73_MAX_LEVELS_VDDC)),
-			"Too many voltage values for VDDC. Trimming to fit state table.",
-			fiji_trim_voltage_table_to_fit_state_table(hwmgr,
-					SMU73_MAX_LEVELS_VDDC, &(data->vddc_voltage_table)));
-
-	PP_ASSERT_WITH_CODE(
-			(data->vddci_voltage_table.count <= (SMU73_MAX_LEVELS_VDDCI)),
-			"Too many voltage values for VDDCI. Trimming to fit state table.",
-			fiji_trim_voltage_table_to_fit_state_table(hwmgr,
-					SMU73_MAX_LEVELS_VDDCI, &(data->vddci_voltage_table)));
-
-	PP_ASSERT_WITH_CODE(
-			(data->mvdd_voltage_table.count <= (SMU73_MAX_LEVELS_MVDD)),
-			"Too many voltage values for MVDD. Trimming to fit state table.",
-			fiji_trim_voltage_table_to_fit_state_table(hwmgr,
-					SMU73_MAX_LEVELS_MVDD, &(data->mvdd_voltage_table)));
-
-	return 0;
-}
-
-static int fiji_initialize_mc_reg_table(struct pp_hwmgr *hwmgr)
-{
-	/* Program additional LP registers
-	 * that are no longer programmed by VBIOS
-	 */
-	cgs_write_register(hwmgr->device, mmMC_SEQ_RAS_TIMING_LP,
-			cgs_read_register(hwmgr->device, mmMC_SEQ_RAS_TIMING));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_CAS_TIMING_LP,
-			cgs_read_register(hwmgr->device, mmMC_SEQ_CAS_TIMING));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2_LP,
-			cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1_LP,
-			cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0_LP,
-			cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1_LP,
-			cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_TIMING_LP,
-			cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_TIMING));
-
-	return 0;
-}
-
-/**
-* Programs static screed detection parameters
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int fiji_program_static_screen_threshold_parameters(
-		struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	/* Set static screen threshold unit */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD_UNIT,
-			data->static_screen_threshold_unit);
-	/* Set static screen threshold */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD,
-			data->static_screen_threshold);
-
-	return 0;
-}
-
-/**
-* Setup display gap for glitch free memory clock switching.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always  0
-*/
-static int fiji_enable_display_gap(struct pp_hwmgr *hwmgr)
-{
-	uint32_t displayGap =
-			cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixCG_DISPLAY_GAP_CNTL);
-
-	displayGap = PHM_SET_FIELD(displayGap, CG_DISPLAY_GAP_CNTL,
-			DISP_GAP, DISPLAY_GAP_IGNORE);
-
-	displayGap = PHM_SET_FIELD(displayGap, CG_DISPLAY_GAP_CNTL,
-			DISP_GAP_MCHG, DISPLAY_GAP_VBLANK);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_DISPLAY_GAP_CNTL, displayGap);
-
-	return 0;
-}
-
-/**
-* Programs activity state transition voting clients
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always  0
-*/
-static int fiji_program_voting_clients(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	/* Clear reset for voting clients before enabling DPM */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 0);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 0);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_0, data->voting_rights_clients0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_1, data->voting_rights_clients1);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_2, data->voting_rights_clients2);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_3, data->voting_rights_clients3);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_4, data->voting_rights_clients4);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_5, data->voting_rights_clients5);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_6, data->voting_rights_clients6);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_7, data->voting_rights_clients7);
-
-	return 0;
-}
-
-static int fiji_clear_voting_clients(struct pp_hwmgr *hwmgr)
-{
-	/* Reset voting clients before disabling DPM */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 1);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 1);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_0, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_1, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_2, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_3, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_4, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_5, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_6, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_7, 0);
-
-	return 0;
-}
-
-/**
-* Get the location of various tables inside the FW image.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always  0
-*/
-static int fiji_process_firmware_header(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
-	uint32_t tmp;
-	int result;
-	bool error = false;
-
-	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU73_Firmware_Header, DpmTable),
-			&tmp, data->sram_end);
-
-	if (0 == result)
-		data->dpm_table_start = tmp;
-
-	error |= (0 != result);
-
-	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU73_Firmware_Header, SoftRegisters),
-			&tmp, data->sram_end);
-
-	if (!result) {
-		data->soft_regs_start = tmp;
-		smu_data->soft_regs_start = tmp;
-	}
-
-	error |= (0 != result);
-
-	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU73_Firmware_Header, mcRegisterTable),
-			&tmp, data->sram_end);
-
-	if (!result)
-		data->mc_reg_table_start = tmp;
-
-	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU73_Firmware_Header, FanTable),
-			&tmp, data->sram_end);
-
-	if (!result)
-		data->fan_table_start = tmp;
-
-	error |= (0 != result);
-
-	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU73_Firmware_Header, mcArbDramTimingTable),
-			&tmp, data->sram_end);
-
-	if (!result)
-		data->arb_table_start = tmp;
-
-	error |= (0 != result);
-
-	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU73_Firmware_Header, Version),
-			&tmp, data->sram_end);
-
-	if (!result)
-		hwmgr->microcode_version_info.SMC = tmp;
-
-	error |= (0 != result);
-
-	return error ? -1 : 0;
-}
-
-/* Copy one arb setting to another and then switch the active set.
- * arb_src and arb_dest is one of the MC_CG_ARB_FREQ_Fx constants.
- */
-static int fiji_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr,
-		uint32_t arb_src, uint32_t arb_dest)
-{
-	uint32_t mc_arb_dram_timing;
-	uint32_t mc_arb_dram_timing2;
-	uint32_t burst_time;
-	uint32_t mc_cg_config;
-
-	switch (arb_src) {
-	case MC_CG_ARB_FREQ_F0:
-		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
-		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
-		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
-		break;
-	case MC_CG_ARB_FREQ_F1:
-		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1);
-		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1);
-		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	switch (arb_dest) {
-	case MC_CG_ARB_FREQ_F0:
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING, mc_arb_dram_timing);
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
-		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0, burst_time);
-		break;
-	case MC_CG_ARB_FREQ_F1:
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
-		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1, burst_time);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	mc_cg_config = cgs_read_register(hwmgr->device, mmMC_CG_CONFIG);
-	mc_cg_config |= 0x0000000F;
-	cgs_write_register(hwmgr->device, mmMC_CG_CONFIG, mc_cg_config);
-	PHM_WRITE_FIELD(hwmgr->device, MC_ARB_CG, CG_ARB_REQ, arb_dest);
-
-	return 0;
-}
-
-/**
-* Call SMC to reset S0/S1 to S1 and Reset SMIO to initial value
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   if success then 0;
-*/
-static int fiji_reset_to_default(struct pp_hwmgr *hwmgr)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_ResetToDefaults);
-}
-
-/**
-* Initial switch from ARB F0->F1
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-* This function is to be called from the SetPowerState table.
-*/
-static int fiji_initial_switch_from_arbf0_to_f1(struct pp_hwmgr *hwmgr)
-{
-	return fiji_copy_and_switch_arb_sets(hwmgr,
-			MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
-}
-
-static int fiji_force_switch_to_arbf0(struct pp_hwmgr *hwmgr)
-{
-	uint32_t tmp;
-
-	tmp = (cgs_read_ind_register(hwmgr->device,
-			CGS_IND_REG__SMC, ixSMC_SCRATCH9) &
-			0x0000ff00) >> 8;
-
-	if (tmp == MC_CG_ARB_FREQ_F0)
-		return 0;
-
-	return fiji_copy_and_switch_arb_sets(hwmgr,
-			tmp, MC_CG_ARB_FREQ_F0);
-}
-
-static int fiji_reset_single_dpm_table(struct pp_hwmgr *hwmgr,
-		struct fiji_single_dpm_table *dpm_table, uint32_t count)
-{
-	int i;
-	PP_ASSERT_WITH_CODE(count <= MAX_REGULAR_DPM_NUMBER,
-			"Fatal error, can not set up single DPM table entries "
-			"to exceed max number!",);
-
-	dpm_table->count = count;
-	for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++)
-		dpm_table->dpm_levels[i].enabled = false;
-
-	return 0;
-}
-
-static void fiji_setup_pcie_table_entry(
-	struct fiji_single_dpm_table *dpm_table,
-	uint32_t index, uint32_t pcie_gen,
-	uint32_t pcie_lanes)
-{
-	dpm_table->dpm_levels[index].value = pcie_gen;
-	dpm_table->dpm_levels[index].param1 = pcie_lanes;
-	dpm_table->dpm_levels[index].enabled = true;
-}
-
-static int fiji_setup_default_pcie_table(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
-	uint32_t i, max_entry;
-
-	PP_ASSERT_WITH_CODE((data->use_pcie_performance_levels ||
-			data->use_pcie_power_saving_levels), "No pcie performance levels!",
-			return -EINVAL);
-
-	if (data->use_pcie_performance_levels &&
-			!data->use_pcie_power_saving_levels) {
-		data->pcie_gen_power_saving = data->pcie_gen_performance;
-		data->pcie_lane_power_saving = data->pcie_lane_performance;
-	} else if (!data->use_pcie_performance_levels &&
-			data->use_pcie_power_saving_levels) {
-		data->pcie_gen_performance = data->pcie_gen_power_saving;
-		data->pcie_lane_performance = data->pcie_lane_power_saving;
-	}
-
-	fiji_reset_single_dpm_table(hwmgr,
-			&data->dpm_table.pcie_speed_table, SMU73_MAX_LEVELS_LINK);
-
-	if (pcie_table != NULL) {
-		/* max_entry is used to make sure we reserve one PCIE level
-		 * for boot level (fix for A+A PSPP issue).
-		 * If PCIE table from PPTable have ULV entry + 8 entries,
-		 * then ignore the last entry.*/
-		max_entry = (SMU73_MAX_LEVELS_LINK < pcie_table->count) ?
-				SMU73_MAX_LEVELS_LINK : pcie_table->count;
-		for (i = 1; i < max_entry; i++) {
-			fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, i - 1,
-					get_pcie_gen_support(data->pcie_gen_cap,
-							pcie_table->entries[i].gen_speed),
-					get_pcie_lane_support(data->pcie_lane_cap,
-							pcie_table->entries[i].lane_width));
-		}
-		data->dpm_table.pcie_speed_table.count = max_entry - 1;
-	} else {
-		/* Hardcode Pcie Table */
-		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 0,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Min_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 1,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Min_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 2,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Max_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 3,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Max_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 4,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Max_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 5,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Max_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-
-		data->dpm_table.pcie_speed_table.count = 6;
-	}
-	/* Populate last level for boot PCIE level, but do not increment count. */
-	fiji_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table,
-			data->dpm_table.pcie_speed_table.count,
-			get_pcie_gen_support(data->pcie_gen_cap,
-					PP_Min_PCIEGen),
-			get_pcie_lane_support(data->pcie_lane_cap,
-					PP_Max_PCIELane));
-
-	return 0;
-}
-
-/*
- * This function is to initalize all DPM state tables
- * for SMU7 based on the dependency table.
- * Dynamic state patching function will then trim these
- * state tables to the allowed range based
- * on the power policy or external client requests,
- * such as UVD request, etc.
- */
-static int fiji_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint32_t i;
-
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_sclk_table =
-			table_info->vdd_dep_on_sclk;
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
-			table_info->vdd_dep_on_mclk;
-
-	PP_ASSERT_WITH_CODE(dep_sclk_table != NULL,
-			"SCLK dependency table is missing. This table is mandatory",
-			return -EINVAL);
-	PP_ASSERT_WITH_CODE(dep_sclk_table->count >= 1,
-			"SCLK dependency table has to have is missing. "
-			"This table is mandatory",
-			return -EINVAL);
-
-	PP_ASSERT_WITH_CODE(dep_mclk_table != NULL,
-			"MCLK dependency table is missing. This table is mandatory",
-			return -EINVAL);
-	PP_ASSERT_WITH_CODE(dep_mclk_table->count >= 1,
-			"MCLK dependency table has to have is missing. "
-			"This table is mandatory",
-			return -EINVAL);
-
-	/* clear the state table to reset everything to default */
-	fiji_reset_single_dpm_table(hwmgr,
-			&data->dpm_table.sclk_table, SMU73_MAX_LEVELS_GRAPHICS);
-	fiji_reset_single_dpm_table(hwmgr,
-			&data->dpm_table.mclk_table, SMU73_MAX_LEVELS_MEMORY);
-
-	/* Initialize Sclk DPM table based on allow Sclk values */
-	data->dpm_table.sclk_table.count = 0;
-	for (i = 0; i < dep_sclk_table->count; i++) {
-		if (i == 0 || data->dpm_table.sclk_table.dpm_levels
-				[data->dpm_table.sclk_table.count - 1].value !=
-						dep_sclk_table->entries[i].clk) {
-			data->dpm_table.sclk_table.dpm_levels
-			[data->dpm_table.sclk_table.count].value =
-					dep_sclk_table->entries[i].clk;
-			data->dpm_table.sclk_table.dpm_levels
-			[data->dpm_table.sclk_table.count].enabled =
-					(i == 0) ? true : false;
-			data->dpm_table.sclk_table.count++;
-		}
-	}
-
-	/* Initialize Mclk DPM table based on allow Mclk values */
-	data->dpm_table.mclk_table.count = 0;
-	for (i=0; i<dep_mclk_table->count; i++) {
-		if ( i==0 || data->dpm_table.mclk_table.dpm_levels
-				[data->dpm_table.mclk_table.count - 1].value !=
-						dep_mclk_table->entries[i].clk) {
-			data->dpm_table.mclk_table.dpm_levels
-			[data->dpm_table.mclk_table.count].value =
-					dep_mclk_table->entries[i].clk;
-			data->dpm_table.mclk_table.dpm_levels
-			[data->dpm_table.mclk_table.count].enabled =
-					(i == 0) ? true : false;
-			data->dpm_table.mclk_table.count++;
-		}
-	}
-
-	/* setup PCIE gen speed levels */
-	fiji_setup_default_pcie_table(hwmgr);
-
-	/* save a copy of the default DPM table */
-	memcpy(&(data->golden_dpm_table), &(data->dpm_table),
-			sizeof(struct fiji_dpm_table));
-
-	return 0;
-}
-
-/**
- * @brief PhwFiji_GetVoltageOrder
- *  Returns index of requested voltage record in lookup(table)
- * @param lookup_table - lookup list to search in
- * @param voltage - voltage to look for
- * @return 0 on success
- */
-uint8_t fiji_get_voltage_index(
-		struct phm_ppt_v1_voltage_lookup_table *lookup_table, uint16_t voltage)
-{
-	uint8_t count = (uint8_t) (lookup_table->count);
-	uint8_t i;
-
-	PP_ASSERT_WITH_CODE((NULL != lookup_table),
-			"Lookup Table empty.", return 0);
-	PP_ASSERT_WITH_CODE((0 != count),
-			"Lookup Table empty.", return 0);
-
-	for (i = 0; i < lookup_table->count; i++) {
-		/* find first voltage equal or bigger than requested */
-		if (lookup_table->entries[i].us_vdd >= voltage)
-			return i;
-	}
-	/* voltage is bigger than max voltage in the table */
-	return i - 1;
-}
-
-/**
-* Preparation of vddc and vddgfx CAC tables for SMC.
-*
-* @param    hwmgr  the address of the hardware manager
-* @param    table  the SMC DPM table structure to be populated
-* @return   always 0
-*/
-static int fiji_populate_cac_table(struct pp_hwmgr *hwmgr,
-		struct SMU73_Discrete_DpmTable *table)
-{
-	uint32_t count;
-	uint8_t index;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_voltage_lookup_table *lookup_table =
-			table_info->vddc_lookup_table;
-	/* tables is already swapped, so in order to use the value from it,
-	 * we need to swap it back.
-	 * We are populating vddc CAC data to BapmVddc table
-	 * in split and merged mode
-	 */
-	for( count = 0; count<lookup_table->count; count++) {
-		index = fiji_get_voltage_index(lookup_table,
-				data->vddc_voltage_table.entries[count].value);
-		table->BapmVddcVidLoSidd[count] = (uint8_t) ((6200 -
-				(lookup_table->entries[index].us_cac_low *
-						VOLTAGE_SCALE)) / 25);
-		table->BapmVddcVidHiSidd[count] = (uint8_t) ((6200 -
-				(lookup_table->entries[index].us_cac_high *
-						VOLTAGE_SCALE)) / 25);
-	}
-
-	return 0;
-}
-
-/**
-* Preparation of voltage tables for SMC.
-*
-* @param    hwmgr   the address of the hardware manager
-* @param    table   the SMC DPM table structure to be populated
-* @return   always  0
-*/
-
-int fiji_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
-		struct SMU73_Discrete_DpmTable *table)
-{
-	int result;
-
-	result = fiji_populate_cac_table(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"can not populate CAC voltage tables to SMC",
-			return -EINVAL);
-
-	return 0;
-}
-
-static int fiji_populate_ulv_level(struct pp_hwmgr *hwmgr,
-		struct SMU73_Discrete_Ulv *state)
-{
-	int result = 0;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	state->CcPwrDynRm = 0;
-	state->CcPwrDynRm1 = 0;
-
-	state->VddcOffset = (uint16_t) table_info->us_ulv_voltage_offset;
-	state->VddcOffsetVid = (uint8_t)( table_info->us_ulv_voltage_offset *
-			VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1 );
-
-	state->VddcPhase = (data->vddc_phase_shed_control) ? 0 : 1;
-
-	if (!result) {
-		CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm);
-		CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm1);
-		CONVERT_FROM_HOST_TO_SMC_US(state->VddcOffset);
-	}
-	return result;
-}
-
-static int fiji_populate_ulv_state(struct pp_hwmgr *hwmgr,
-		struct SMU73_Discrete_DpmTable *table)
-{
-	return fiji_populate_ulv_level(hwmgr, &table->Ulv);
-}
-
-static int32_t fiji_get_dpm_level_enable_mask_value(
-		struct fiji_single_dpm_table* dpm_table)
-{
-	int32_t i;
-	int32_t mask = 0;
-
-	for (i = dpm_table->count; i > 0; i--) {
-		mask = mask << 1;
-		if (dpm_table->dpm_levels[i - 1].enabled)
-			mask |= 0x1;
-		else
-			mask &= 0xFFFFFFFE;
-	}
-	return mask;
-}
-
-static int fiji_populate_smc_link_level(struct pp_hwmgr *hwmgr,
-		struct SMU73_Discrete_DpmTable *table)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_dpm_table *dpm_table = &data->dpm_table;
-	int i;
-
-	/* Index (dpm_table->pcie_speed_table.count)
-	 * is reserved for PCIE boot level. */
-	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
-		table->LinkLevel[i].PcieGenSpeed  =
-				(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
-		table->LinkLevel[i].PcieLaneCount = (uint8_t)encode_pcie_lane_width(
-				dpm_table->pcie_speed_table.dpm_levels[i].param1);
-		table->LinkLevel[i].EnabledForActivity = 1;
-		table->LinkLevel[i].SPC = (uint8_t)(data->pcie_spc_cap & 0xff);
-		table->LinkLevel[i].DownThreshold = PP_HOST_TO_SMC_UL(5);
-		table->LinkLevel[i].UpThreshold = PP_HOST_TO_SMC_UL(30);
-	}
-
-	data->smc_state_table.LinkLevelCount =
-			(uint8_t)dpm_table->pcie_speed_table.count;
-	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
-			fiji_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
-
-	return 0;
-}
-
-/**
-* Calculates the SCLK dividers using the provided engine clock
-*
-* @param    hwmgr  the address of the hardware manager
-* @param    clock  the engine clock to use to populate the structure
-* @param    sclk   the SMC SCLK structure to be populated
-*/
-static int fiji_calculate_sclk_params(struct pp_hwmgr *hwmgr,
-		uint32_t clock, struct SMU73_Discrete_GraphicsLevel *sclk)
-{
-	const struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct pp_atomctrl_clock_dividers_vi dividers;
-	uint32_t spll_func_cntl            = data->clock_registers.vCG_SPLL_FUNC_CNTL;
-	uint32_t spll_func_cntl_3          = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
-	uint32_t spll_func_cntl_4          = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
-	uint32_t cg_spll_spread_spectrum   = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
-	uint32_t cg_spll_spread_spectrum_2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
-	uint32_t ref_clock;
-	uint32_t ref_divider;
-	uint32_t fbdiv;
-	int result;
-
-	/* get the engine clock dividers for this clock value */
-	result = atomctrl_get_engine_pll_dividers_vi(hwmgr, clock,  &dividers);
-
-	PP_ASSERT_WITH_CODE(result == 0,
-			"Error retrieving Engine Clock dividers from VBIOS.",
-			return result);
-
-	/* To get FBDIV we need to multiply this by 16384 and divide it by Fref. */
-	ref_clock = atomctrl_get_reference_clock(hwmgr);
-	ref_divider = 1 + dividers.uc_pll_ref_div;
-
-	/* low 14 bits is fraction and high 12 bits is divider */
-	fbdiv = dividers.ul_fb_div.ul_fb_divider & 0x3FFFFFF;
-
-	/* SPLL_FUNC_CNTL setup */
-	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
-			SPLL_REF_DIV, dividers.uc_pll_ref_div);
-	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
-			SPLL_PDIV_A,  dividers.uc_pll_post_div);
-
-	/* SPLL_FUNC_CNTL_3 setup*/
-	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3, CG_SPLL_FUNC_CNTL_3,
-			SPLL_FB_DIV, fbdiv);
-
-	/* set to use fractional accumulation*/
-	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3, CG_SPLL_FUNC_CNTL_3,
-			SPLL_DITHEN, 1);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_EngineSpreadSpectrumSupport)) {
-		struct pp_atomctrl_internal_ss_info ssInfo;
-
-		uint32_t vco_freq = clock * dividers.uc_pll_post_div;
-		if (!atomctrl_get_engine_clock_spread_spectrum(hwmgr,
-				vco_freq, &ssInfo)) {
-			/*
-			 * ss_info.speed_spectrum_percentage -- in unit of 0.01%
-			 * ss_info.speed_spectrum_rate -- in unit of khz
-			 *
-			 * clks = reference_clock * 10 / (REFDIV + 1) / speed_spectrum_rate / 2
-			 */
-			uint32_t clk_s = ref_clock * 5 /
-					(ref_divider * ssInfo.speed_spectrum_rate);
-			/* clkv = 2 * D * fbdiv / NS */
-			uint32_t clk_v = 4 * ssInfo.speed_spectrum_percentage *
-					fbdiv / (clk_s * 10000);
-
-			cg_spll_spread_spectrum = PHM_SET_FIELD(cg_spll_spread_spectrum,
-					CG_SPLL_SPREAD_SPECTRUM, CLKS, clk_s);
-			cg_spll_spread_spectrum = PHM_SET_FIELD(cg_spll_spread_spectrum,
-					CG_SPLL_SPREAD_SPECTRUM, SSEN, 1);
-			cg_spll_spread_spectrum_2 = PHM_SET_FIELD(cg_spll_spread_spectrum_2,
-					CG_SPLL_SPREAD_SPECTRUM_2, CLKV, clk_v);
-		}
-	}
-
-	sclk->SclkFrequency        = clock;
-	sclk->CgSpllFuncCntl3      = spll_func_cntl_3;
-	sclk->CgSpllFuncCntl4      = spll_func_cntl_4;
-	sclk->SpllSpreadSpectrum   = cg_spll_spread_spectrum;
-	sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
-	sclk->SclkDid              = (uint8_t)dividers.pll_post_divider;
-
-	return 0;
-}
-
-static uint16_t fiji_find_closest_vddci(struct pp_hwmgr *hwmgr, uint16_t vddci)
-{
-	uint32_t  i;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct pp_atomctrl_voltage_table *vddci_table =
-			&(data->vddci_voltage_table);
-
-	for (i = 0; i < vddci_table->count; i++) {
-		if (vddci_table->entries[i].value >= vddci)
-			return vddci_table->entries[i].value;
-	}
-
-	PP_ASSERT_WITH_CODE(false,
-			"VDDCI is larger than max VDDCI in VDDCI Voltage Table!",
-			return vddci_table->entries[i-1].value);
-}
-
-static int fiji_get_dependency_volt_by_clk(struct pp_hwmgr *hwmgr,
-		struct phm_ppt_v1_clock_voltage_dependency_table* dep_table,
-		uint32_t clock, SMU_VoltageLevel *voltage, uint32_t *mvdd)
-{
-	uint32_t i;
-	uint16_t vddci;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	*voltage = *mvdd = 0;
-
-	/* clock - voltage dependency table is empty table */
-	if (dep_table->count == 0)
-		return -EINVAL;
-
-	for (i = 0; i < dep_table->count; i++) {
-		/* find first sclk bigger than request */
-		if (dep_table->entries[i].clk >= clock) {
-			*voltage |= (dep_table->entries[i].vddc *
-					VOLTAGE_SCALE) << VDDC_SHIFT;
-			if (FIJI_VOLTAGE_CONTROL_NONE == data->vddci_control)
-				*voltage |= (data->vbios_boot_state.vddci_bootup_value *
-						VOLTAGE_SCALE) << VDDCI_SHIFT;
-			else if (dep_table->entries[i].vddci)
-				*voltage |= (dep_table->entries[i].vddci *
-						VOLTAGE_SCALE) << VDDCI_SHIFT;
-			else {
-				vddci = fiji_find_closest_vddci(hwmgr,
-						(dep_table->entries[i].vddc -
-								(uint16_t)data->vddc_vddci_delta));
-				*voltage |= (vddci * VOLTAGE_SCALE) <<	VDDCI_SHIFT;
-			}
-
-			if (FIJI_VOLTAGE_CONTROL_NONE == data->mvdd_control)
-				*mvdd = data->vbios_boot_state.mvdd_bootup_value *
-					VOLTAGE_SCALE;
-			else if (dep_table->entries[i].mvdd)
-				*mvdd = (uint32_t) dep_table->entries[i].mvdd *
-					VOLTAGE_SCALE;
-
-			*voltage |= 1 << PHASES_SHIFT;
-			return 0;
-		}
-	}
-
-	/* sclk is bigger than max sclk in the dependence table */
-	*voltage |= (dep_table->entries[i - 1].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
-
-	if (FIJI_VOLTAGE_CONTROL_NONE == data->vddci_control)
-		*voltage |= (data->vbios_boot_state.vddci_bootup_value *
-				VOLTAGE_SCALE) << VDDCI_SHIFT;
-	else if (dep_table->entries[i-1].vddci) {
-		vddci = fiji_find_closest_vddci(hwmgr,
-				(dep_table->entries[i].vddc -
-						(uint16_t)data->vddc_vddci_delta));
-		*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
-	}
-
-	if (FIJI_VOLTAGE_CONTROL_NONE == data->mvdd_control)
-		*mvdd = data->vbios_boot_state.mvdd_bootup_value * VOLTAGE_SCALE;
-	else if (dep_table->entries[i].mvdd)
-		*mvdd = (uint32_t) dep_table->entries[i - 1].mvdd * VOLTAGE_SCALE;
-
-	return 0;
-}
-
-static uint8_t fiji_get_sleep_divider_id_from_clock(uint32_t clock,
-		uint32_t clock_insr)
-{
-	uint8_t i;
-	uint32_t temp;
-	uint32_t min = max(clock_insr, (uint32_t)FIJI_MINIMUM_ENGINE_CLOCK);
-
-	PP_ASSERT_WITH_CODE((clock >= min), "Engine clock can't satisfy stutter requirement!", return 0);
-	for (i = FIJI_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
-		temp = clock >> i;
-
-		if (temp >= min || i == 0)
-			break;
-	}
-	return i;
-}
-/**
-* Populates single SMC SCLK structure using the provided engine clock
-*
-* @param    hwmgr      the address of the hardware manager
-* @param    clock the engine clock to use to populate the structure
-* @param    sclk        the SMC SCLK structure to be populated
-*/
-
-static int fiji_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
-		uint32_t clock, uint16_t sclk_al_threshold,
-		struct SMU73_Discrete_GraphicsLevel *level)
-{
-	int result;
-	/* PP_Clocks minClocks; */
-	uint32_t threshold, mvdd;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	result = fiji_calculate_sclk_params(hwmgr, clock, level);
-
-	/* populate graphics levels */
-	result = fiji_get_dependency_volt_by_clk(hwmgr,
-			table_info->vdd_dep_on_sclk, clock,
-			&level->MinVoltage, &mvdd);
-	PP_ASSERT_WITH_CODE((0 == result),
-			"can not find VDDC voltage value for "
-			"VDDC engine clock dependency table",
-			return result);
-
-	level->SclkFrequency = clock;
-	level->ActivityLevel = sclk_al_threshold;
-	level->CcPwrDynRm = 0;
-	level->CcPwrDynRm1 = 0;
-	level->EnabledForActivity = 0;
-	level->EnabledForThrottle = 1;
-	level->UpHyst = 10;
-	level->DownHyst = 0;
-	level->VoltageDownHyst = 0;
-	level->PowerThrottle = 0;
-
-	threshold = clock * data->fast_watermark_threshold / 100;
-
-
-	data->display_timing.min_clock_in_sr = hwmgr->display_config.min_core_set_clock_in_sr;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep))
-		level->DeepSleepDivId = fiji_get_sleep_divider_id_from_clock(clock,
-								hwmgr->display_config.min_core_set_clock_in_sr);
-
-
-	/* Default to slow, highest DPM level will be
-	 * set to PPSMC_DISPLAY_WATERMARK_LOW later.
-	 */
-	level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
-
-	CONVERT_FROM_HOST_TO_SMC_UL(level->MinVoltage);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->SclkFrequency);
-	CONVERT_FROM_HOST_TO_SMC_US(level->ActivityLevel);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->CgSpllFuncCntl3);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->CgSpllFuncCntl4);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->SpllSpreadSpectrum);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->SpllSpreadSpectrum2);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm1);
-
-	return 0;
-}
-/**
-* Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states
-*
-* @param    hwmgr      the address of the hardware manager
-*/
-static int fiji_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_dpm_table *dpm_table = &data->dpm_table;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
-	uint8_t pcie_entry_cnt = (uint8_t) data->dpm_table.pcie_speed_table.count;
-	int result = 0;
-	uint32_t array = data->dpm_table_start +
-			offsetof(SMU73_Discrete_DpmTable, GraphicsLevel);
-	uint32_t array_size = sizeof(struct SMU73_Discrete_GraphicsLevel) *
-			SMU73_MAX_LEVELS_GRAPHICS;
-	struct SMU73_Discrete_GraphicsLevel *levels =
-			data->smc_state_table.GraphicsLevel;
-	uint32_t i, max_entry;
-	uint8_t hightest_pcie_level_enabled = 0,
-			lowest_pcie_level_enabled = 0,
-			mid_pcie_level_enabled = 0,
-			count = 0;
-
-	for (i = 0; i < dpm_table->sclk_table.count; i++) {
-		result = fiji_populate_single_graphic_level(hwmgr,
-				dpm_table->sclk_table.dpm_levels[i].value,
-				(uint16_t)data->activity_target[i],
-				&levels[i]);
-		if (result)
-			return result;
-
-		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
-		if (i > 1)
-			levels[i].DeepSleepDivId = 0;
-	}
-
-	/* Only enable level 0 for now.*/
-	levels[0].EnabledForActivity = 1;
-
-	/* set highest level watermark to high */
-	levels[dpm_table->sclk_table.count - 1].DisplayWatermark =
-			PPSMC_DISPLAY_WATERMARK_HIGH;
-
-	data->smc_state_table.GraphicsDpmLevelCount =
-			(uint8_t)dpm_table->sclk_table.count;
-	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
-			fiji_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
-
-	if (pcie_table != NULL) {
-		PP_ASSERT_WITH_CODE((1 <= pcie_entry_cnt),
-				"There must be 1 or more PCIE levels defined in PPTable.",
-				return -EINVAL);
-		max_entry = pcie_entry_cnt - 1;
-		for (i = 0; i < dpm_table->sclk_table.count; i++)
-			levels[i].pcieDpmLevel =
-					(uint8_t) ((i < max_entry)? i : max_entry);
-	} else {
-		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
-				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
-						(1 << (hightest_pcie_level_enabled + 1))) != 0 ))
-			hightest_pcie_level_enabled++;
-
-		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
-				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
-						(1 << lowest_pcie_level_enabled)) == 0 ))
-			lowest_pcie_level_enabled++;
-
-		while ((count < hightest_pcie_level_enabled) &&
-				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
-						(1 << (lowest_pcie_level_enabled + 1 + count))) == 0 ))
-			count++;
-
-		mid_pcie_level_enabled = (lowest_pcie_level_enabled + 1+ count) <
-				hightest_pcie_level_enabled?
-						(lowest_pcie_level_enabled + 1 + count) :
-						hightest_pcie_level_enabled;
-
-		/* set pcieDpmLevel to hightest_pcie_level_enabled */
-		for(i = 2; i < dpm_table->sclk_table.count; i++)
-			levels[i].pcieDpmLevel = hightest_pcie_level_enabled;
-
-		/* set pcieDpmLevel to lowest_pcie_level_enabled */
-		levels[0].pcieDpmLevel = lowest_pcie_level_enabled;
-
-		/* set pcieDpmLevel to mid_pcie_level_enabled */
-		levels[1].pcieDpmLevel = mid_pcie_level_enabled;
-	}
-	/* level count will send to smc once at init smc table and never change */
-	result = fiji_copy_bytes_to_smc(hwmgr->smumgr, array, (uint8_t *)levels,
-			(uint32_t)array_size, data->sram_end);
-
-	return result;
-}
-
-/**
- * MCLK Frequency Ratio
- * SEQ_CG_RESP  Bit[31:24] - 0x0
- * Bit[27:24] \96 DDR3 Frequency ratio
- * 0x0 <= 100MHz,       450 < 0x8 <= 500MHz
- * 100 < 0x1 <= 150MHz,       500 < 0x9 <= 550MHz
- * 150 < 0x2 <= 200MHz,       550 < 0xA <= 600MHz
- * 200 < 0x3 <= 250MHz,       600 < 0xB <= 650MHz
- * 250 < 0x4 <= 300MHz,       650 < 0xC <= 700MHz
- * 300 < 0x5 <= 350MHz,       700 < 0xD <= 750MHz
- * 350 < 0x6 <= 400MHz,       750 < 0xE <= 800MHz
- * 400 < 0x7 <= 450MHz,       800 < 0xF
- */
-static uint8_t fiji_get_mclk_frequency_ratio(uint32_t mem_clock)
-{
-	if (mem_clock <= 10000) return 0x0;
-	if (mem_clock <= 15000) return 0x1;
-	if (mem_clock <= 20000) return 0x2;
-	if (mem_clock <= 25000) return 0x3;
-	if (mem_clock <= 30000) return 0x4;
-	if (mem_clock <= 35000) return 0x5;
-	if (mem_clock <= 40000) return 0x6;
-	if (mem_clock <= 45000) return 0x7;
-	if (mem_clock <= 50000) return 0x8;
-	if (mem_clock <= 55000) return 0x9;
-	if (mem_clock <= 60000) return 0xa;
-	if (mem_clock <= 65000) return 0xb;
-	if (mem_clock <= 70000) return 0xc;
-	if (mem_clock <= 75000) return 0xd;
-	if (mem_clock <= 80000) return 0xe;
-	/* mem_clock > 800MHz */
-	return 0xf;
-}
-
-/**
-* Populates the SMC MCLK structure using the provided memory clock
-*
-* @param    hwmgr   the address of the hardware manager
-* @param    clock   the memory clock to use to populate the structure
-* @param    sclk    the SMC SCLK structure to be populated
-*/
-static int fiji_calculate_mclk_params(struct pp_hwmgr *hwmgr,
-    uint32_t clock, struct SMU73_Discrete_MemoryLevel *mclk)
-{
-	struct pp_atomctrl_memory_clock_param mem_param;
-	int result;
-
-	result = atomctrl_get_memory_pll_dividers_vi(hwmgr, clock, &mem_param);
-	PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to get Memory PLL Dividers.",);
-
-	/* Save the result data to outpupt memory level structure */
-	mclk->MclkFrequency   = clock;
-	mclk->MclkDivider     = (uint8_t)mem_param.mpll_post_divider;
-	mclk->FreqRange       = fiji_get_mclk_frequency_ratio(clock);
-
-	return result;
-}
-
-static int fiji_populate_single_memory_level(struct pp_hwmgr *hwmgr,
-		uint32_t clock, struct SMU73_Discrete_MemoryLevel *mem_level)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	int result = 0;
-
-	if (table_info->vdd_dep_on_mclk) {
-		result = fiji_get_dependency_volt_by_clk(hwmgr,
-				table_info->vdd_dep_on_mclk, clock,
-				&mem_level->MinVoltage, &mem_level->MinMvdd);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find MinVddc voltage value from memory "
-				"VDDC voltage dependency table", return result);
-	}
-
-	mem_level->EnabledForThrottle = 1;
-	mem_level->EnabledForActivity = 0;
-	mem_level->UpHyst = 0;
-	mem_level->DownHyst = 100;
-	mem_level->VoltageDownHyst = 0;
-	mem_level->ActivityLevel = (uint16_t)data->mclk_activity_target;
-	mem_level->StutterEnable = false;
-
-	mem_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
-
-	/* enable stutter mode if all the follow condition applied
-	 * PECI_GetNumberOfActiveDisplays(hwmgr->pPECI,
-	 * &(data->DisplayTiming.numExistingDisplays));
-	 */
-	data->display_timing.num_existing_displays = 1;
-
-	if ((data->mclk_stutter_mode_threshold) &&
-		(clock <= data->mclk_stutter_mode_threshold) &&
-		(!data->is_uvd_enabled) &&
-		(PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL,
-				STUTTER_ENABLE) & 0x1))
-		mem_level->StutterEnable = true;
-
-	result = fiji_calculate_mclk_params(hwmgr, clock, mem_level);
-	if (!result) {
-		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinMvdd);
-		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MclkFrequency);
-		CONVERT_FROM_HOST_TO_SMC_US(mem_level->ActivityLevel);
-		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinVoltage);
-	}
-	return result;
-}
-
-/**
-* Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states
-*
-* @param    hwmgr      the address of the hardware manager
-*/
-static int fiji_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_dpm_table *dpm_table = &data->dpm_table;
-	int result;
-	/* populate MCLK dpm table to SMU7 */
-	uint32_t array = data->dpm_table_start +
-			offsetof(SMU73_Discrete_DpmTable, MemoryLevel);
-	uint32_t array_size = sizeof(SMU73_Discrete_MemoryLevel) *
-			SMU73_MAX_LEVELS_MEMORY;
-	struct SMU73_Discrete_MemoryLevel *levels =
-			data->smc_state_table.MemoryLevel;
-	uint32_t i;
-
-	for (i = 0; i < dpm_table->mclk_table.count; i++) {
-		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
-				"can not populate memory level as memory clock is zero",
-				return -EINVAL);
-		result = fiji_populate_single_memory_level(hwmgr,
-				dpm_table->mclk_table.dpm_levels[i].value,
-				&levels[i]);
-		if (result)
-			return result;
-	}
-
-	/* Only enable level 0 for now. */
-	levels[0].EnabledForActivity = 1;
-
-	/* in order to prevent MC activity from stutter mode to push DPM up.
-	 * the UVD change complements this by putting the MCLK in
-	 * a higher state by default such that we are not effected by
-	 * up threshold or and MCLK DPM latency.
-	 */
-	levels[0].ActivityLevel = (uint16_t)data->mclk_dpm0_activity_target;
-	CONVERT_FROM_HOST_TO_SMC_US(levels[0].ActivityLevel);
-
-	data->smc_state_table.MemoryDpmLevelCount =
-			(uint8_t)dpm_table->mclk_table.count;
-	data->dpm_level_enable_mask.mclk_dpm_enable_mask =
-			fiji_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
-	/* set highest level watermark to high */
-	levels[dpm_table->mclk_table.count - 1].DisplayWatermark =
-			PPSMC_DISPLAY_WATERMARK_HIGH;
-
-	/* level count will send to smc once at init smc table and never change */
-	result = fiji_copy_bytes_to_smc(hwmgr->smumgr, array, (uint8_t *)levels,
-			(uint32_t)array_size, data->sram_end);
-
-	return result;
-}
-
-/**
-* Populates the SMC MVDD structure using the provided memory clock.
-*
-* @param    hwmgr      the address of the hardware manager
-* @param    mclk        the MCLK value to be used in the decision if MVDD should be high or low.
-* @param    voltage     the SMC VOLTAGE structure to be populated
-*/
-int fiji_populate_mvdd_value(struct pp_hwmgr *hwmgr,
-		uint32_t mclk, SMIO_Pattern *smio_pat)
-{
-	const struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint32_t i = 0;
-
-	if (FIJI_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
-		/* find mvdd value which clock is more than request */
-		for (i = 0; i < table_info->vdd_dep_on_mclk->count; i++) {
-			if (mclk <= table_info->vdd_dep_on_mclk->entries[i].clk) {
-				smio_pat->Voltage = data->mvdd_voltage_table.entries[i].value;
-				break;
-			}
-		}
-		PP_ASSERT_WITH_CODE(i < table_info->vdd_dep_on_mclk->count,
-				"MVDD Voltage is outside the supported range.",
-				return -EINVAL);
-	} else
-		return -EINVAL;
-
-	return 0;
-}
-
-static int fiji_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
-		SMU73_Discrete_DpmTable *table)
-{
-	int result = 0;
-	const struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct pp_atomctrl_clock_dividers_vi dividers;
-	SMIO_Pattern vol_level;
-	uint32_t mvdd;
-	uint16_t us_mvdd;
-	uint32_t spll_func_cntl    = data->clock_registers.vCG_SPLL_FUNC_CNTL;
-	uint32_t spll_func_cntl_2  = data->clock_registers.vCG_SPLL_FUNC_CNTL_2;
-
-	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
-
-	if (!data->sclk_dpm_key_disabled) {
-		/* Get MinVoltage and Frequency from DPM0,
-		 * already converted to SMC_UL */
-		table->ACPILevel.SclkFrequency =
-				data->dpm_table.sclk_table.dpm_levels[0].value;
-		result = fiji_get_dependency_volt_by_clk(hwmgr,
-				table_info->vdd_dep_on_sclk,
-				table->ACPILevel.SclkFrequency,
-				&table->ACPILevel.MinVoltage, &mvdd);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Cannot find ACPI VDDC voltage value "
-				"in Clock Dependency Table",);
-	} else {
-		table->ACPILevel.SclkFrequency =
-				data->vbios_boot_state.sclk_bootup_value;
-		table->ACPILevel.MinVoltage =
-				data->vbios_boot_state.vddc_bootup_value * VOLTAGE_SCALE;
-	}
-
-	/* get the engine clock dividers for this clock value */
-	result = atomctrl_get_engine_pll_dividers_vi(hwmgr,
-			table->ACPILevel.SclkFrequency,  &dividers);
-	PP_ASSERT_WITH_CODE(result == 0,
-			"Error retrieving Engine Clock dividers from VBIOS.",
-			return result);
-
-	table->ACPILevel.SclkDid = (uint8_t)dividers.pll_post_divider;
-	table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
-	table->ACPILevel.DeepSleepDivId = 0;
-
-	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
-			SPLL_PWRON, 0);
-	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
-			SPLL_RESET, 1);
-	spll_func_cntl_2 = PHM_SET_FIELD(spll_func_cntl_2, CG_SPLL_FUNC_CNTL_2,
-			SCLK_MUX_SEL, 4);
-
-	table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
-	table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
-	table->ACPILevel.CgSpllFuncCntl3 = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
-	table->ACPILevel.CgSpllFuncCntl4 = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
-	table->ACPILevel.SpllSpreadSpectrum = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
-	table->ACPILevel.SpllSpreadSpectrum2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
-	table->ACPILevel.CcPwrDynRm = 0;
-	table->ACPILevel.CcPwrDynRm1 = 0;
-
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkFrequency);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.MinVoltage);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl2);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl3);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl4);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum2);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
-
-	if (!data->mclk_dpm_key_disabled) {
-		/* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */
-		table->MemoryACPILevel.MclkFrequency =
-				data->dpm_table.mclk_table.dpm_levels[0].value;
-		result = fiji_get_dependency_volt_by_clk(hwmgr,
-				table_info->vdd_dep_on_mclk,
-				table->MemoryACPILevel.MclkFrequency,
-				&table->MemoryACPILevel.MinVoltage, &mvdd);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Cannot find ACPI VDDCI voltage value "
-				"in Clock Dependency Table",);
-	} else {
-		table->MemoryACPILevel.MclkFrequency =
-				data->vbios_boot_state.mclk_bootup_value;
-		table->MemoryACPILevel.MinVoltage =
-				data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE;
-	}
-
-	us_mvdd = 0;
-	if ((FIJI_VOLTAGE_CONTROL_NONE == data->mvdd_control) ||
-			(data->mclk_dpm_key_disabled))
-		us_mvdd = data->vbios_boot_state.mvdd_bootup_value;
-	else {
-		if (!fiji_populate_mvdd_value(hwmgr,
-				data->dpm_table.mclk_table.dpm_levels[0].value,
-				&vol_level))
-			us_mvdd = vol_level.Voltage;
-	}
-
-	table->MemoryACPILevel.MinMvdd =
-			PP_HOST_TO_SMC_UL(us_mvdd * VOLTAGE_SCALE);
-
-	table->MemoryACPILevel.EnabledForThrottle = 0;
-	table->MemoryACPILevel.EnabledForActivity = 0;
-	table->MemoryACPILevel.UpHyst = 0;
-	table->MemoryACPILevel.DownHyst = 100;
-	table->MemoryACPILevel.VoltageDownHyst = 0;
-	table->MemoryACPILevel.ActivityLevel =
-			PP_HOST_TO_SMC_US((uint16_t)data->mclk_activity_target);
-
-	table->MemoryACPILevel.StutterEnable = false;
-	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MclkFrequency);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MinVoltage);
-
-	return result;
-}
-
-static int fiji_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
-		SMU73_Discrete_DpmTable *table)
-{
-	int result = -EINVAL;
-	uint8_t count;
-	struct pp_atomctrl_clock_dividers_vi dividers;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
-			table_info->mm_dep_table;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	table->VceLevelCount = (uint8_t)(mm_table->count);
-	table->VceBootLevel = 0;
-
-	for(count = 0; count < table->VceLevelCount; count++) {
-		table->VceLevel[count].Frequency = mm_table->entries[count].eclk;
-		table->VceLevel[count].MinVoltage = 0;
-		table->VceLevel[count].MinVoltage |=
-				(mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
-		table->VceLevel[count].MinVoltage |=
-				((mm_table->entries[count].vddc - data->vddc_vddci_delta) *
-						VOLTAGE_SCALE) << VDDCI_SHIFT;
-		table->VceLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
-
-		/*retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-				table->VceLevel[count].Frequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for VCE engine clock",
-				return result);
-
-		table->VceLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].Frequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].MinVoltage);
-	}
-	return result;
-}
-
-static int fiji_populate_smc_acp_level(struct pp_hwmgr *hwmgr,
-		SMU73_Discrete_DpmTable *table)
-{
-	int result = -EINVAL;
-	uint8_t count;
-	struct pp_atomctrl_clock_dividers_vi dividers;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
-			table_info->mm_dep_table;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	table->AcpLevelCount = (uint8_t)(mm_table->count);
-	table->AcpBootLevel = 0;
-
-	for (count = 0; count < table->AcpLevelCount; count++) {
-		table->AcpLevel[count].Frequency = mm_table->entries[count].aclk;
-		table->AcpLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
-				VOLTAGE_SCALE) << VDDC_SHIFT;
-		table->AcpLevel[count].MinVoltage |= ((mm_table->entries[count].vddc -
-				data->vddc_vddci_delta) * VOLTAGE_SCALE) << VDDCI_SHIFT;
-		table->AcpLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
-
-		/* retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-				table->AcpLevel[count].Frequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for engine clock", return result);
-
-		table->AcpLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->AcpLevel[count].Frequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->AcpLevel[count].MinVoltage);
-	}
-	return result;
-}
-
-static int fiji_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
-		SMU73_Discrete_DpmTable *table)
-{
-	int result = -EINVAL;
-	uint8_t count;
-	struct pp_atomctrl_clock_dividers_vi dividers;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
-			table_info->mm_dep_table;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	table->SamuBootLevel = 0;
-	table->SamuLevelCount = (uint8_t)(mm_table->count);
-
-	for (count = 0; count < table->SamuLevelCount; count++) {
-		/* not sure whether we need evclk or not */
-		table->SamuLevel[count].MinVoltage = 0;
-		table->SamuLevel[count].Frequency = mm_table->entries[count].samclock;
-		table->SamuLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
-				VOLTAGE_SCALE) << VDDC_SHIFT;
-		table->SamuLevel[count].MinVoltage |= ((mm_table->entries[count].vddc -
-				data->vddc_vddci_delta) * VOLTAGE_SCALE) << VDDCI_SHIFT;
-		table->SamuLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
-
-		/* retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-				table->SamuLevel[count].Frequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for samu clock", return result);
-
-		table->SamuLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].Frequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].MinVoltage);
-	}
-	return result;
-}
-
-static int fiji_populate_memory_timing_parameters(struct pp_hwmgr *hwmgr,
-		int32_t eng_clock, int32_t mem_clock,
-		struct SMU73_Discrete_MCArbDramTimingTableEntry *arb_regs)
-{
-	uint32_t dram_timing;
-	uint32_t dram_timing2;
-	uint32_t burstTime;
-	ULONG state, trrds, trrdl;
-	int result;
-
-	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
-			eng_clock, mem_clock);
-	PP_ASSERT_WITH_CODE(result == 0,
-			"Error calling VBIOS to set DRAM_TIMING.", return result);
-
-	dram_timing = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
-	dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
-	burstTime = cgs_read_register(hwmgr->device, mmMC_ARB_BURST_TIME);
-
-	state = PHM_GET_FIELD(burstTime, MC_ARB_BURST_TIME, STATE0);
-	trrds = PHM_GET_FIELD(burstTime, MC_ARB_BURST_TIME, TRRDS0);
-	trrdl = PHM_GET_FIELD(burstTime, MC_ARB_BURST_TIME, TRRDL0);
-
-	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dram_timing);
-	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dram_timing2);
-	arb_regs->McArbBurstTime   = (uint8_t)burstTime;
-	arb_regs->TRRDS            = (uint8_t)trrds;
-	arb_regs->TRRDL            = (uint8_t)trrdl;
-
-	return 0;
-}
-
-static int fiji_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct SMU73_Discrete_MCArbDramTimingTable arb_regs;
-	uint32_t i, j;
-	int result = 0;
-
-	for (i = 0; i < data->dpm_table.sclk_table.count; i++) {
-		for (j = 0; j < data->dpm_table.mclk_table.count; j++) {
-			result = fiji_populate_memory_timing_parameters(hwmgr,
-					data->dpm_table.sclk_table.dpm_levels[i].value,
-					data->dpm_table.mclk_table.dpm_levels[j].value,
-					&arb_regs.entries[i][j]);
-			if (result)
-				break;
-		}
-	}
-
-	if (!result)
-		result = fiji_copy_bytes_to_smc(
-				hwmgr->smumgr,
-				data->arb_table_start,
-				(uint8_t *)&arb_regs,
-				sizeof(SMU73_Discrete_MCArbDramTimingTable),
-				data->sram_end);
-	return result;
-}
-
-static int fiji_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
-		struct SMU73_Discrete_DpmTable *table)
-{
-	int result = -EINVAL;
-	uint8_t count;
-	struct pp_atomctrl_clock_dividers_vi dividers;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
-			table_info->mm_dep_table;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	table->UvdLevelCount = (uint8_t)(mm_table->count);
-	table->UvdBootLevel = 0;
-
-	for (count = 0; count < table->UvdLevelCount; count++) {
-		table->UvdLevel[count].MinVoltage = 0;
-		table->UvdLevel[count].VclkFrequency = mm_table->entries[count].vclk;
-		table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk;
-		table->UvdLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
-				VOLTAGE_SCALE) << VDDC_SHIFT;
-		table->UvdLevel[count].MinVoltage |= ((mm_table->entries[count].vddc -
-				data->vddc_vddci_delta) * VOLTAGE_SCALE) << VDDCI_SHIFT;
-		table->UvdLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
-
-		/* retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-				table->UvdLevel[count].VclkFrequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for Vclk clock", return result);
-
-		table->UvdLevel[count].VclkDivider = (uint8_t)dividers.pll_post_divider;
-
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-				table->UvdLevel[count].DclkFrequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for Dclk clock", return result);
-
-		table->UvdLevel[count].DclkDivider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].MinVoltage);
-
-	}
-	return result;
-}
-
-static int fiji_find_boot_level(struct fiji_single_dpm_table *table,
-		uint32_t value, uint32_t *boot_level)
-{
-	int result = -EINVAL;
-	uint32_t i;
-
-	for (i = 0; i < table->count; i++) {
-		if (value == table->dpm_levels[i].value) {
-			*boot_level = i;
-			result = 0;
-		}
-	}
-	return result;
-}
-
-static int fiji_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
-		struct SMU73_Discrete_DpmTable *table)
-{
-	int result = 0;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	table->GraphicsBootLevel = 0;
-	table->MemoryBootLevel = 0;
-
-	/* find boot level from dpm table */
-	result = fiji_find_boot_level(&(data->dpm_table.sclk_table),
-			data->vbios_boot_state.sclk_bootup_value,
-			(uint32_t *)&(table->GraphicsBootLevel));
-
-	result = fiji_find_boot_level(&(data->dpm_table.mclk_table),
-			data->vbios_boot_state.mclk_bootup_value,
-			(uint32_t *)&(table->MemoryBootLevel));
-
-	table->BootVddc  = data->vbios_boot_state.vddc_bootup_value *
-			VOLTAGE_SCALE;
-	table->BootVddci = data->vbios_boot_state.vddci_bootup_value *
-			VOLTAGE_SCALE;
-	table->BootMVdd  = data->vbios_boot_state.mvdd_bootup_value *
-			VOLTAGE_SCALE;
-
-	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddc);
-	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddci);
-	CONVERT_FROM_HOST_TO_SMC_US(table->BootMVdd);
-
-	return 0;
-}
-
-static int fiji_populate_smc_initailial_state(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint8_t count, level;
-
-	count = (uint8_t)(table_info->vdd_dep_on_sclk->count);
-	for (level = 0; level < count; level++) {
-		if(table_info->vdd_dep_on_sclk->entries[level].clk >=
-				data->vbios_boot_state.sclk_bootup_value) {
-			data->smc_state_table.GraphicsBootLevel = level;
-			break;
-		}
-	}
-
-	count = (uint8_t)(table_info->vdd_dep_on_mclk->count);
-	for (level = 0; level < count; level++) {
-		if(table_info->vdd_dep_on_mclk->entries[level].clk >=
-				data->vbios_boot_state.mclk_bootup_value) {
-			data->smc_state_table.MemoryBootLevel = level;
-			break;
-		}
-	}
-
-	return 0;
-}
-
-static int fiji_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
-{
-	uint32_t ro, efuse, efuse2, clock_freq, volt_without_cks,
-			volt_with_cks, value;
-	uint16_t clock_freq_u16;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint8_t type, i, j, cks_setting, stretch_amount, stretch_amount2,
-			volt_offset = 0;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
-			table_info->vdd_dep_on_sclk;
-
-	stretch_amount = (uint8_t)table_info->cac_dtp_table->usClockStretchAmount;
-
-	/* Read SMU_Eefuse to read and calculate RO and determine
-	 * if the part is SS or FF. if RO >= 1660MHz, part is FF.
-	 */
-	efuse = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixSMU_EFUSE_0 + (146 * 4));
-	efuse2 = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixSMU_EFUSE_0 + (148 * 4));
-	efuse &= 0xFF000000;
-	efuse = efuse >> 24;
-	efuse2 &= 0xF;
-
-	if (efuse2 == 1)
-		ro = (2300 - 1350) * efuse / 255 + 1350;
-	else
-		ro = (2500 - 1000) * efuse / 255 + 1000;
-
-	if (ro >= 1660)
-		type = 0;
-	else
-		type = 1;
-
-	/* Populate Stretch amount */
-	data->smc_state_table.ClockStretcherAmount = stretch_amount;
-
-	/* Populate Sclk_CKS_masterEn0_7 and Sclk_voltageOffset */
-	for (i = 0; i < sclk_table->count; i++) {
-		data->smc_state_table.Sclk_CKS_masterEn0_7 |=
-				sclk_table->entries[i].cks_enable << i;
-		volt_without_cks = (uint32_t)((14041 *
-			(sclk_table->entries[i].clk/100) / 10000 + 3571 + 75 - ro) * 1000 /
-			(4026 - (13924 * (sclk_table->entries[i].clk/100) / 10000)));
-		volt_with_cks = (uint32_t)((13946 *
-			(sclk_table->entries[i].clk/100) / 10000 + 3320 + 45 - ro) * 1000 /
-			(3664 - (11454 * (sclk_table->entries[i].clk/100) / 10000)));
-		if (volt_without_cks >= volt_with_cks)
-			volt_offset = (uint8_t)(((volt_without_cks - volt_with_cks +
-					sclk_table->entries[i].cks_voffset) * 100 / 625) + 1);
-		data->smc_state_table.Sclk_voltageOffset[i] = volt_offset;
-	}
-
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
-			STRETCH_ENABLE, 0x0);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
-			masterReset, 0x1);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
-			staticEnable, 0x1);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
-			masterReset, 0x0);
-
-	/* Populate CKS Lookup Table */
-	if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5)
-		stretch_amount2 = 0;
-	else if (stretch_amount == 3 || stretch_amount == 4)
-		stretch_amount2 = 1;
-	else {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_ClockStretcher);
-		PP_ASSERT_WITH_CODE(false,
-				"Stretch Amount in PPTable not supported\n",
-				return -EINVAL);
-	}
-
-	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixPWR_CKS_CNTL);
-	value &= 0xFFC2FF87;
-	data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].minFreq =
-			fiji_clock_stretcher_lookup_table[stretch_amount2][0];
-	data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].maxFreq =
-			fiji_clock_stretcher_lookup_table[stretch_amount2][1];
-	clock_freq_u16 = (uint16_t)(PP_SMC_TO_HOST_UL(data->smc_state_table.
-			GraphicsLevel[data->smc_state_table.GraphicsDpmLevelCount - 1].
-			SclkFrequency) / 100);
-	if (fiji_clock_stretcher_lookup_table[stretch_amount2][0] <
-			clock_freq_u16 &&
-	    fiji_clock_stretcher_lookup_table[stretch_amount2][1] >
-			clock_freq_u16) {
-		/* Program PWR_CKS_CNTL. CKS_USE_FOR_LOW_FREQ */
-		value |= (fiji_clock_stretcher_lookup_table[stretch_amount2][3]) << 16;
-		/* Program PWR_CKS_CNTL. CKS_LDO_REFSEL */
-		value |= (fiji_clock_stretcher_lookup_table[stretch_amount2][2]) << 18;
-		/* Program PWR_CKS_CNTL. CKS_STRETCH_AMOUNT */
-		value |= (fiji_clock_stretch_amount_conversion
-				[fiji_clock_stretcher_lookup_table[stretch_amount2][3]]
-				 [stretch_amount]) << 3;
-	}
-	CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.CKS_LOOKUPTable.
-			CKS_LOOKUPTableEntry[0].minFreq);
-	CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.CKS_LOOKUPTable.
-			CKS_LOOKUPTableEntry[0].maxFreq);
-	data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting =
-			fiji_clock_stretcher_lookup_table[stretch_amount2][2] & 0x7F;
-	data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting |=
-			(fiji_clock_stretcher_lookup_table[stretch_amount2][3]) << 7;
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixPWR_CKS_CNTL, value);
-
-	/* Populate DDT Lookup Table */
-	for (i = 0; i < 4; i++) {
-		/* Assign the minimum and maximum VID stored
-		 * in the last row of Clock Stretcher Voltage Table.
-		 */
-		data->smc_state_table.ClockStretcherDataTable.
-		ClockStretcherDataTableEntry[i].minVID =
-				(uint8_t) fiji_clock_stretcher_ddt_table[type][i][2];
-		data->smc_state_table.ClockStretcherDataTable.
-		ClockStretcherDataTableEntry[i].maxVID =
-				(uint8_t) fiji_clock_stretcher_ddt_table[type][i][3];
-		/* Loop through each SCLK and check the frequency
-		 * to see if it lies within the frequency for clock stretcher.
-		 */
-		for (j = 0; j < data->smc_state_table.GraphicsDpmLevelCount; j++) {
-			cks_setting = 0;
-			clock_freq = PP_SMC_TO_HOST_UL(
-					data->smc_state_table.GraphicsLevel[j].SclkFrequency);
-			/* Check the allowed frequency against the sclk level[j].
-			 *  Sclk's endianness has already been converted,
-			 *  and it's in 10Khz unit,
-			 *  as opposed to Data table, which is in Mhz unit.
-			 */
-			if (clock_freq >=
-					(fiji_clock_stretcher_ddt_table[type][i][0]) * 100) {
-				cks_setting |= 0x2;
-				if (clock_freq <
-						(fiji_clock_stretcher_ddt_table[type][i][1]) * 100)
-					cks_setting |= 0x1;
-			}
-			data->smc_state_table.ClockStretcherDataTable.
-			ClockStretcherDataTableEntry[i].setting |= cks_setting << (j * 2);
-		}
-		CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.
-				ClockStretcherDataTable.
-				ClockStretcherDataTableEntry[i].setting);
-	}
-
-	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL);
-	value &= 0xFFFFFFFE;
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL, value);
-
-	return 0;
-}
-
-/**
-* Populates the SMC VRConfig field in DPM table.
-*
-* @param    hwmgr   the address of the hardware manager
-* @param    table   the SMC DPM table structure to be populated
-* @return   always 0
-*/
-static int fiji_populate_vr_config(struct pp_hwmgr *hwmgr,
-		struct SMU73_Discrete_DpmTable *table)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint16_t config;
-
-	config = VR_MERGED_WITH_VDDC;
-	table->VRConfig |= (config << VRCONF_VDDGFX_SHIFT);
-
-	/* Set Vddc Voltage Controller */
-	if(FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
-		config = VR_SVI2_PLANE_1;
-		table->VRConfig |= config;
-	} else {
-		PP_ASSERT_WITH_CODE(false,
-				"VDDC should be on SVI2 control in merged mode!",);
-	}
-	/* Set Vddci Voltage Controller */
-	if(FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
-		config = VR_SVI2_PLANE_2;  /* only in merged mode */
-		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
-	} else if (FIJI_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
-		config = VR_SMIO_PATTERN_1;
-		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
-	} else {
-		config = VR_STATIC_VOLTAGE;
-		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
-	}
-	/* Set Mvdd Voltage Controller */
-	if(FIJI_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
-		config = VR_SVI2_PLANE_2;
-		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
-	} else if(FIJI_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
-		config = VR_SMIO_PATTERN_2;
-		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
-	} else {
-		config = VR_STATIC_VOLTAGE;
-		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
-	}
-
-	return 0;
-}
-
-/**
-* Initializes the SMC table and uploads it
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput  the pointer to input data (PowerState)
-* @return   always 0
-*/
-static int fiji_init_smc_table(struct pp_hwmgr *hwmgr)
-{
-	int result;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct SMU73_Discrete_DpmTable *table = &(data->smc_state_table);
-	const struct fiji_ulv_parm *ulv = &(data->ulv);
-	uint8_t i;
-	struct pp_atomctrl_gpio_pin_assignment gpio_pin;
-
-	result = fiji_setup_default_dpm_tables(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to setup default DPM tables!", return result);
-
-	if(FIJI_VOLTAGE_CONTROL_NONE != data->voltage_control)
-		fiji_populate_smc_voltage_tables(hwmgr, table);
-
-	table->SystemFlags = 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_AutomaticDCTransition))
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_StepVddc))
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
-
-	if (data->is_memory_gddr5)
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
-
-	if (ulv->ulv_supported && table_info->us_ulv_voltage_offset) {
-		result = fiji_populate_ulv_state(hwmgr, table);
-		PP_ASSERT_WITH_CODE(0 == result,
-				"Failed to initialize ULV state!", return result);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixCG_ULV_PARAMETER, ulv->cg_ulv_parameter);
-	}
-
-	result = fiji_populate_smc_link_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Link Level!", return result);
-
-	result = fiji_populate_all_graphic_levels(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Graphics Level!", return result);
-
-	result = fiji_populate_all_memory_levels(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Memory Level!", return result);
-
-	result = fiji_populate_smc_acpi_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize ACPI Level!", return result);
-
-	result = fiji_populate_smc_vce_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize VCE Level!", return result);
-
-	result = fiji_populate_smc_acp_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize ACP Level!", return result);
-
-	result = fiji_populate_smc_samu_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize SAMU Level!", return result);
-
-	/* Since only the initial state is completely set up at this point
-	 * (the other states are just copies of the boot state) we only
-	 * need to populate the  ARB settings for the initial state.
-	 */
-	result = fiji_program_memory_timing_parameters(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to Write ARB settings for the initial state.", return result);
-
-	result = fiji_populate_smc_uvd_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize UVD Level!", return result);
-
-	result = fiji_populate_smc_boot_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Boot Level!", return result);
-
-	result = fiji_populate_smc_initailial_state(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Boot State!", return result);
-
-	result = fiji_populate_bapm_parameters_in_dpm_table(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to populate BAPM Parameters!", return result);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ClockStretcher)) {
-		result = fiji_populate_clock_stretcher_data_table(hwmgr);
-		PP_ASSERT_WITH_CODE(0 == result,
-				"Failed to populate Clock Stretcher Data Table!",
-				return result);
-	}
-
-	table->GraphicsVoltageChangeEnable  = 1;
-	table->GraphicsThermThrottleEnable  = 1;
-	table->GraphicsInterval = 1;
-	table->VoltageInterval  = 1;
-	table->ThermalInterval  = 1;
-	table->TemperatureLimitHigh =
-			table_info->cac_dtp_table->usTargetOperatingTemp *
-			FIJI_Q88_FORMAT_CONVERSION_UNIT;
-	table->TemperatureLimitLow  =
-			(table_info->cac_dtp_table->usTargetOperatingTemp - 1) *
-			FIJI_Q88_FORMAT_CONVERSION_UNIT;
-	table->MemoryVoltageChangeEnable = 1;
-	table->MemoryInterval = 1;
-	table->VoltageResponseTime = 0;
-	table->PhaseResponseTime = 0;
-	table->MemoryThermThrottleEnable = 1;
-	table->PCIeBootLinkLevel = 0;      /* 0:Gen1 1:Gen2 2:Gen3*/
-	table->PCIeGenInterval = 1;
-	table->VRConfig = 0;
-
-	result = fiji_populate_vr_config(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to populate VRConfig setting!", return result);
-
-	table->ThermGpio = 17;
-	table->SclkStepSize = 0x4000;
-
-	if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_VRHOT_GPIO_PINID, &gpio_pin)) {
-		table->VRHotGpio = gpio_pin.uc_gpio_pin_bit_shift;
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_RegulatorHot);
-	} else {
-		table->VRHotGpio = FIJI_UNUSED_GPIO_PIN;
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_RegulatorHot);
-	}
-
-	if (atomctrl_get_pp_assign_pin(hwmgr, PP_AC_DC_SWITCH_GPIO_PINID,
-			&gpio_pin)) {
-		table->AcDcGpio = gpio_pin.uc_gpio_pin_bit_shift;
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_AutomaticDCTransition);
-	} else {
-		table->AcDcGpio = FIJI_UNUSED_GPIO_PIN;
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_AutomaticDCTransition);
-	}
-
-	/* Thermal Output GPIO */
-	if (atomctrl_get_pp_assign_pin(hwmgr, THERMAL_INT_OUTPUT_GPIO_PINID,
-			&gpio_pin)) {
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_ThermalOutGPIO);
-
-		table->ThermOutGpio = gpio_pin.uc_gpio_pin_bit_shift;
-
-		/* For porlarity read GPIOPAD_A with assigned Gpio pin
-		 * since VBIOS will program this register to set 'inactive state',
-		 * driver can then determine 'active state' from this and
-		 * program SMU with correct polarity
-		 */
-		table->ThermOutPolarity = (0 == (cgs_read_register(hwmgr->device, mmGPIOPAD_A) &
-				(1 << gpio_pin.uc_gpio_pin_bit_shift))) ? 1:0;
-		table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_ONLY;
-
-		/* if required, combine VRHot/PCC with thermal out GPIO */
-		if(phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_RegulatorHot) &&
-			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_CombinePCCWithThermalSignal))
-			table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_VRHOT;
-	} else {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_ThermalOutGPIO);
-		table->ThermOutGpio = 17;
-		table->ThermOutPolarity = 1;
-		table->ThermOutMode = SMU7_THERM_OUT_MODE_DISABLE;
-	}
-
-	for (i = 0; i < SMU73_MAX_ENTRIES_SMIO; i++)
-		table->Smio[i] = PP_HOST_TO_SMC_UL(table->Smio[i]);
-
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->VRConfig);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask1);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask2);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
-	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
-	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
-	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
-	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
-
-	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
-	result = fiji_copy_bytes_to_smc(hwmgr->smumgr,
-			data->dpm_table_start +
-			offsetof(SMU73_Discrete_DpmTable, SystemFlags),
-			(uint8_t *)&(table->SystemFlags),
-			sizeof(SMU73_Discrete_DpmTable) - 3 * sizeof(SMU73_PIDController),
-			data->sram_end);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to upload dpm data to SMC memory!", return result);
-
-	return 0;
-}
-
-/**
-* Initialize the ARB DRAM timing table's index field.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int fiji_init_arb_table_index(struct pp_hwmgr *hwmgr)
-{
-	const struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t tmp;
-	int result;
-
-	/* This is a read-modify-write on the first byte of the ARB table.
-	 * The first byte in the SMU73_Discrete_MCArbDramTimingTable structure
-	 * is the field 'current'.
-	 * This solution is ugly, but we never write the whole table only
-	 * individual fields in it.
-	 * In reality this field should not be in that structure
-	 * but in a soft register.
-	 */
-	result = fiji_read_smc_sram_dword(hwmgr->smumgr,
-			data->arb_table_start, &tmp, data->sram_end);
-
-	if (result)
-		return result;
-
-	tmp &= 0x00FFFFFF;
-	tmp |= ((uint32_t)MC_CG_ARB_FREQ_F1) << 24;
-
-	return fiji_write_smc_sram_dword(hwmgr->smumgr,
-			data->arb_table_start,  tmp, data->sram_end);
-}
-
-static int fiji_enable_vrhot_gpio_interrupt(struct pp_hwmgr *hwmgr)
-{
-	if(phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_RegulatorHot))
-		return smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_EnableVRHotGPIOInterrupt);
-
-	return 0;
-}
-
-static int fiji_enable_sclk_control(struct pp_hwmgr *hwmgr)
-{
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
-			SCLK_PWRMGT_OFF, 0);
-	return 0;
-}
-
-static int fiji_enable_ulv(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_ulv_parm *ulv = &(data->ulv);
-
-	if (ulv->ulv_supported)
-		return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_EnableULV);
-
-	return 0;
-}
-
-static int fiji_disable_ulv(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_ulv_parm *ulv = &(data->ulv);
-
-	if (ulv->ulv_supported)
-		return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DisableULV);
-
-	return 0;
-}
-
-static int fiji_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
-{
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkDeepSleep)) {
-		if (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_MASTER_DeepSleep_ON))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to enable Master Deep Sleep switch failed!",
-					return -1);
-	} else {
-		if (smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_MASTER_DeepSleep_OFF)) {
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to disable Master Deep Sleep switch failed!",
-					return -1);
-		}
-	}
-
-	return 0;
-}
-
-static int fiji_disable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
-{
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkDeepSleep)) {
-		if (smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_MASTER_DeepSleep_OFF)) {
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to disable Master Deep Sleep switch failed!",
-					return -1);
-		}
-	}
-
-	return 0;
-}
-
-static int fiji_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t   val, val0, val2;
-	uint32_t   i, cpl_cntl, cpl_threshold, mc_threshold;
-
-	/* enable SCLK dpm */
-	if(!data->sclk_dpm_key_disabled)
-		PP_ASSERT_WITH_CODE(
-		(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DPM_Enable)),
-		"Failed to enable SCLK DPM during DPM Start Function!",
-		return -1);
-
-	/* enable MCLK dpm */
-	if(0 == data->mclk_dpm_key_disabled) {
-		cpl_threshold = 0;
-		mc_threshold = 0;
-
-		/* Read per MCD tile (0 - 7) */
-		for (i = 0; i < 8; i++) {
-			PHM_WRITE_FIELD(hwmgr->device, MC_CONFIG_MCD, MC_RD_ENABLE, i);
-			val = cgs_read_register(hwmgr->device, mmMC_SEQ_RESERVE_0_S) & 0xf0000000;
-			if (0xf0000000 != val) {
-				/* count number of MCQ that has channel(s) enabled */
-				cpl_threshold++;
-				/* only harvest 3 or full 4 supported */
-				mc_threshold = val ? 3 : 4;
-			}
-		}
-		PP_ASSERT_WITH_CODE(0 != cpl_threshold,
-				"Number of MCQ is zero!", return -EINVAL;);
-
-		mc_threshold = ((mc_threshold & LCAC_MC0_CNTL__MC0_THRESHOLD_MASK) <<
-				LCAC_MC0_CNTL__MC0_THRESHOLD__SHIFT) |
-						LCAC_MC0_CNTL__MC0_ENABLE_MASK;
-		cpl_cntl = ((cpl_threshold & LCAC_CPL_CNTL__CPL_THRESHOLD_MASK) <<
-				LCAC_CPL_CNTL__CPL_THRESHOLD__SHIFT) |
-						LCAC_CPL_CNTL__CPL_ENABLE_MASK;
-		cpl_cntl = (cpl_cntl | (8 << LCAC_CPL_CNTL__CPL_BLOCK_ID__SHIFT));
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixLCAC_MC0_CNTL, mc_threshold);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixLCAC_MC1_CNTL, mc_threshold);
-		if (8 == cpl_threshold) {
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC2_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC3_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC4_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC5_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC6_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC7_CNTL, mc_threshold);
-		}
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixLCAC_CPL_CNTL, cpl_cntl);
-
-		udelay(5);
-
-		mc_threshold = mc_threshold |
-				(1 << LCAC_MC0_CNTL__MC0_SIGNAL_ID__SHIFT);
-		cpl_cntl = cpl_cntl | (1 << LCAC_CPL_CNTL__CPL_SIGNAL_ID__SHIFT);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixLCAC_MC0_CNTL, mc_threshold);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixLCAC_MC1_CNTL, mc_threshold);
-		if (8 == cpl_threshold) {
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC2_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC3_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC4_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC5_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC6_CNTL, mc_threshold);
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixLCAC_MC7_CNTL, mc_threshold);
-		}
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixLCAC_CPL_CNTL, cpl_cntl);
-
-		/* Program CAC_EN per MCD (0-7) Tile */
-		val0 = val = cgs_read_register(hwmgr->device, mmMC_CONFIG_MCD);
-		val &= ~(MC_CONFIG_MCD__MCD0_WR_ENABLE_MASK |
-				MC_CONFIG_MCD__MCD1_WR_ENABLE_MASK |
-				MC_CONFIG_MCD__MCD2_WR_ENABLE_MASK |
-				MC_CONFIG_MCD__MCD3_WR_ENABLE_MASK |
-				MC_CONFIG_MCD__MCD4_WR_ENABLE_MASK |
-				MC_CONFIG_MCD__MCD5_WR_ENABLE_MASK |
-				MC_CONFIG_MCD__MCD6_WR_ENABLE_MASK |
-				MC_CONFIG_MCD__MCD7_WR_ENABLE_MASK |
-				MC_CONFIG_MCD__MC_RD_ENABLE_MASK);
-
-		for (i = 0; i < 8; i++) {
-			/* Enable MCD i Tile read & write */
-			val2  = (val | (i << MC_CONFIG_MCD__MC_RD_ENABLE__SHIFT) |
-					(1 << i));
-			cgs_write_register(hwmgr->device, mmMC_CONFIG_MCD, val2);
-			/* Enbale CAC_ON MCD i Tile */
-			val2 = cgs_read_register(hwmgr->device, mmMC_SEQ_CNTL);
-			val2 |= MC_SEQ_CNTL__CAC_EN_MASK;
-			cgs_write_register(hwmgr->device, mmMC_SEQ_CNTL, val2);
-		}
-		/* Set MC_CONFIG_MCD back to its default setting val0 */
-		cgs_write_register(hwmgr->device, mmMC_CONFIG_MCD, val0);
-
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_MCLKDPM_Enable)),
-				"Failed to enable MCLK DPM during DPM Start Function!",
-				return -1);
-	}
-	return 0;
-}
-
-static int fiji_start_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	/*enable general power management */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
-			GLOBAL_PWRMGT_EN, 1);
-	/* enable sclk deep sleep */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
-			DYNAMIC_PM_EN, 1);
-	/* prepare for PCIE DPM */
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			data->soft_regs_start + offsetof(SMU73_SoftRegisters,
-					VoltageChangeTimeout), 0x1000);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE,
-			SWRST_COMMAND_1, RESETLC, 0x0);
-
-	PP_ASSERT_WITH_CODE(
-			(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					PPSMC_MSG_Voltage_Cntl_Enable)),
-			"Failed to enable voltage DPM during DPM Start Function!",
-			return -1);
-
-	if (fiji_enable_sclk_mclk_dpm(hwmgr)) {
-		printk(KERN_ERR "Failed to enable Sclk DPM and Mclk DPM!");
-		return -1;
-	}
-
-	/* enable PCIE dpm */
-	if(!data->pcie_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_PCIeDPM_Enable)),
-				"Failed to enable pcie DPM during DPM Start Function!",
-				return -1);
-	}
-
-	return 0;
-}
-
-static int fiji_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	/* disable SCLK dpm */
-	if (!data->sclk_dpm_key_disabled)
-		PP_ASSERT_WITH_CODE(
-				(smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_DPM_Disable) == 0),
-				"Failed to disable SCLK DPM!",
-				return -1);
-
-	/* disable MCLK dpm */
-	if (!data->mclk_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE(
-				(smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-				PPSMC_MSG_MCLKDPM_SetEnabledMask, 1) == 0),
-				"Failed to force MCLK DPM0!",
-				return -1);
-
-		PP_ASSERT_WITH_CODE(
-				(smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_MCLKDPM_Disable) == 0),
-				"Failed to disable MCLK DPM!",
-				return -1);
-	}
-
-	return 0;
-}
-
-static int fiji_stop_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	/* disable general power management */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
-			GLOBAL_PWRMGT_EN, 0);
-	/* disable sclk deep sleep */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
-			DYNAMIC_PM_EN, 0);
-
-	/* disable PCIE dpm */
-	if (!data->pcie_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE(
-				(smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_PCIeDPM_Disable) == 0),
-				"Failed to disable pcie DPM during DPM Stop Function!",
-				return -1);
-	}
-
-	if (fiji_disable_sclk_mclk_dpm(hwmgr)) {
-		printk(KERN_ERR "Failed to disable Sclk DPM and Mclk DPM!");
-		return -1;
-	}
-
-	PP_ASSERT_WITH_CODE(
-			(smum_send_msg_to_smc(hwmgr->smumgr,
-					PPSMC_MSG_Voltage_Cntl_Disable) == 0),
-			"Failed to disable voltage DPM during DPM Stop Function!",
-			return -1);
-
-	return 0;
-}
-
-static void fiji_set_dpm_event_sources(struct pp_hwmgr *hwmgr,
-		uint32_t sources)
-{
-	bool protection;
-	enum DPM_EVENT_SRC src;
-
-	switch (sources) {
-	default:
-		printk(KERN_ERR "Unknown throttling event sources.");
-		/* fall through */
-	case 0:
-		protection = false;
-		/* src is unused */
-		break;
-	case (1 << PHM_AutoThrottleSource_Thermal):
-		protection = true;
-		src = DPM_EVENT_SRC_DIGITAL;
-		break;
-	case (1 << PHM_AutoThrottleSource_External):
-		protection = true;
-		src = DPM_EVENT_SRC_EXTERNAL;
-		break;
-	case (1 << PHM_AutoThrottleSource_External) |
-			(1 << PHM_AutoThrottleSource_Thermal):
-		protection = true;
-		src = DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL;
-		break;
-	}
-	/* Order matters - don't enable thermal protection for the wrong source. */
-	if (protection) {
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_CTRL,
-				DPM_EVENT_SRC, src);
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
-				THERMAL_PROTECTION_DIS,
-				!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_ThermalController));
-	} else
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
-				THERMAL_PROTECTION_DIS, 1);
-}
-
-static int fiji_enable_auto_throttle_source(struct pp_hwmgr *hwmgr,
-		PHM_AutoThrottleSource source)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (!(data->active_auto_throttle_sources & (1 << source))) {
-		data->active_auto_throttle_sources |= 1 << source;
-		fiji_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources);
-	}
-	return 0;
-}
-
-static int fiji_enable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
-{
-	return fiji_enable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
-}
-
-static int fiji_disable_auto_throttle_source(struct pp_hwmgr *hwmgr,
-		PHM_AutoThrottleSource source)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (data->active_auto_throttle_sources & (1 << source)) {
-		data->active_auto_throttle_sources &= ~(1 << source);
-		fiji_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources);
-	}
-	return 0;
-}
-
-static int fiji_disable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
-{
-	return fiji_disable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
-}
-
-static int fiji_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
-{
-	int tmp_result, result = 0;
-
-	tmp_result = (!fiji_is_dpm_running(hwmgr))? 0 : -1;
-	PP_ASSERT_WITH_CODE(result == 0,
-			"DPM is already running right now, no need to enable DPM!",
-			return 0);
-
-	if (fiji_voltage_control(hwmgr)) {
-		tmp_result = fiji_enable_voltage_control(hwmgr);
-		PP_ASSERT_WITH_CODE(tmp_result == 0,
-				"Failed to enable voltage control!",
-				result = tmp_result);
-	}
-
-	if (fiji_voltage_control(hwmgr)) {
-		tmp_result = fiji_construct_voltage_tables(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == tmp_result),
-				"Failed to contruct voltage tables!",
-				result = tmp_result);
-	}
-
-	tmp_result = fiji_initialize_mc_reg_table(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to initialize MC reg table!", result = tmp_result);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_EngineSpreadSpectrumSupport))
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 1);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ThermalController))
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 0);
-
-	tmp_result = fiji_program_static_screen_threshold_parameters(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to program static screen threshold parameters!",
-			result = tmp_result);
-
-	tmp_result = fiji_enable_display_gap(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable display gap!", result = tmp_result);
-
-	tmp_result = fiji_program_voting_clients(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to program voting clients!", result = tmp_result);
-
-	tmp_result = fiji_process_firmware_header(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to process firmware header!", result = tmp_result);
-
-	tmp_result = fiji_initial_switch_from_arbf0_to_f1(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to initialize switch from ArbF0 to F1!",
-			result = tmp_result);
-
-	tmp_result = fiji_init_smc_table(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to initialize SMC table!", result = tmp_result);
-
-	tmp_result = fiji_init_arb_table_index(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to initialize ARB table index!", result = tmp_result);
-
-	tmp_result = fiji_populate_pm_fuses(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to populate PM fuses!", result = tmp_result);
-
-	tmp_result = fiji_enable_vrhot_gpio_interrupt(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable VR hot GPIO interrupt!", result = tmp_result);
-
-	tmp_result = tonga_notify_smc_display_change(hwmgr, false);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to notify no display!", result = tmp_result);
-
-	tmp_result = fiji_enable_sclk_control(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable SCLK control!", result = tmp_result);
-
-	tmp_result = fiji_enable_ulv(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable ULV!", result = tmp_result);
-
-	tmp_result = fiji_enable_deep_sleep_master_switch(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable deep sleep master switch!", result = tmp_result);
-
-	tmp_result = fiji_start_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to start DPM!", result = tmp_result);
-
-	tmp_result = fiji_enable_smc_cac(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable SMC CAC!", result = tmp_result);
-
-	tmp_result = fiji_enable_power_containment(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable power containment!", result = tmp_result);
-
-	tmp_result = fiji_power_control_set_level(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to power control set level!", result = tmp_result);
-
-	tmp_result = fiji_enable_thermal_auto_throttle(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable thermal auto throttle!", result = tmp_result);
-
-	return result;
-}
-
-static int fiji_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
-{
-	int tmp_result, result = 0;
-
-	tmp_result = (fiji_is_dpm_running(hwmgr)) ? 0 : -1;
-	PP_ASSERT_WITH_CODE(tmp_result == 0,
-			"DPM is not running right now, no need to disable DPM!",
-			return 0);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ThermalController))
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 1);
-
-	tmp_result = fiji_disable_power_containment(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable power containment!", result = tmp_result);
-
-	tmp_result = fiji_disable_smc_cac(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable SMC CAC!", result = tmp_result);
-
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_SPLL_SPREAD_SPECTRUM, SSEN, 0);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 0);
-
-	tmp_result = fiji_disable_thermal_auto_throttle(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable thermal auto throttle!", result = tmp_result);
-
-	tmp_result = fiji_stop_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to stop DPM!", result = tmp_result);
-
-	tmp_result = fiji_disable_deep_sleep_master_switch(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable deep sleep master switch!", result = tmp_result);
-
-	tmp_result = fiji_disable_ulv(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable ULV!", result = tmp_result);
-
-	tmp_result = fiji_clear_voting_clients(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to clear voting clients!", result = tmp_result);
-
-	tmp_result = fiji_reset_to_default(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to reset to default!", result = tmp_result);
-
-	tmp_result = fiji_force_switch_to_arbf0(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to force to switch arbf0!", result = tmp_result);
-
-	return result;
-}
-
-static int fiji_force_dpm_highest(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t level, tmp;
-
-	if (!data->sclk_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
-			level = 0;
-			tmp = data->dpm_level_enable_mask.sclk_dpm_enable_mask;
-			while (tmp >>= 1)
-				level++;
-			if (level)
-				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-						PPSMC_MSG_SCLKDPM_SetEnabledMask,
-						(1 << level));
-		}
-	}
-
-	if (!data->mclk_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
-			level = 0;
-			tmp = data->dpm_level_enable_mask.mclk_dpm_enable_mask;
-			while (tmp >>= 1)
-				level++;
-			if (level)
-				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-						PPSMC_MSG_MCLKDPM_SetEnabledMask,
-						(1 << level));
-		}
-	}
-
-	if (!data->pcie_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask) {
-			level = 0;
-			tmp = data->dpm_level_enable_mask.pcie_dpm_enable_mask;
-			while (tmp >>= 1)
-				level++;
-			if (level)
-				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-						PPSMC_MSG_PCIeDPM_ForceLevel,
-						(1 << level));
-		}
-	}
-	return 0;
-}
-
-static int fiji_upload_dpmlevel_enable_mask(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	phm_apply_dal_min_voltage_request(hwmgr);
-
-	if (!data->sclk_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_SCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.sclk_dpm_enable_mask);
-	}
-	return 0;
-}
-
-static int fiji_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (!fiji_is_dpm_running(hwmgr))
-		return -EINVAL;
-
-	if (!data->pcie_dpm_key_disabled) {
-		smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_PCIeDPM_UnForceLevel);
-	}
-
-	return fiji_upload_dpmlevel_enable_mask(hwmgr);
-}
-
-static uint32_t fiji_get_lowest_enabled_level(
-		struct pp_hwmgr *hwmgr, uint32_t mask)
-{
-	uint32_t level = 0;
-
-	while(0 == (mask & (1 << level)))
-		level++;
-
-	return level;
-}
-
-static int fiji_force_dpm_lowest(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data =
-			(struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t level;
-
-	if (!data->sclk_dpm_key_disabled)
-		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
-			level = fiji_get_lowest_enabled_level(hwmgr,
-							      data->dpm_level_enable_mask.sclk_dpm_enable_mask);
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-							    PPSMC_MSG_SCLKDPM_SetEnabledMask,
-							    (1 << level));
-
-	}
-
-	if (!data->mclk_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
-			level = fiji_get_lowest_enabled_level(hwmgr,
-							      data->dpm_level_enable_mask.mclk_dpm_enable_mask);
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-							    PPSMC_MSG_MCLKDPM_SetEnabledMask,
-							    (1 << level));
-		}
-	}
-
-	if (!data->pcie_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask) {
-			level = fiji_get_lowest_enabled_level(hwmgr,
-							      data->dpm_level_enable_mask.pcie_dpm_enable_mask);
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-							    PPSMC_MSG_PCIeDPM_ForceLevel,
-							    (1 << level));
-		}
-	}
-
-	return 0;
-
-}
-static int fiji_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
-				enum amd_dpm_forced_level level)
-{
-	int ret = 0;
-
-	switch (level) {
-	case AMD_DPM_FORCED_LEVEL_HIGH:
-		ret = fiji_force_dpm_highest(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	case AMD_DPM_FORCED_LEVEL_LOW:
-		ret = fiji_force_dpm_lowest(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	case AMD_DPM_FORCED_LEVEL_AUTO:
-		ret = fiji_unforce_dpm_levels(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	default:
-		break;
-	}
-
-	hwmgr->dpm_level = level;
-
-	return ret;
-}
-
-static int fiji_get_power_state_size(struct pp_hwmgr *hwmgr)
-{
-	return sizeof(struct fiji_power_state);
-}
-
-static int fiji_get_pp_table_entry_callback_func(struct pp_hwmgr *hwmgr,
-		void *state, struct pp_power_state *power_state,
-		void *pp_table, uint32_t classification_flag)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_power_state  *fiji_power_state =
-			(struct fiji_power_state *)(&(power_state->hardware));
-	struct fiji_performance_level *performance_level;
-	ATOM_Tonga_State *state_entry = (ATOM_Tonga_State *)state;
-	ATOM_Tonga_POWERPLAYTABLE *powerplay_table =
-			(ATOM_Tonga_POWERPLAYTABLE *)pp_table;
-	ATOM_Tonga_SCLK_Dependency_Table *sclk_dep_table =
-			(ATOM_Tonga_SCLK_Dependency_Table *)
-			(((unsigned long)powerplay_table) +
-				le16_to_cpu(powerplay_table->usSclkDependencyTableOffset));
-	ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table =
-			(ATOM_Tonga_MCLK_Dependency_Table *)
-			(((unsigned long)powerplay_table) +
-				le16_to_cpu(powerplay_table->usMclkDependencyTableOffset));
-
-	/* The following fields are not initialized here: id orderedList allStatesList */
-	power_state->classification.ui_label =
-			(le16_to_cpu(state_entry->usClassification) &
-			ATOM_PPLIB_CLASSIFICATION_UI_MASK) >>
-			ATOM_PPLIB_CLASSIFICATION_UI_SHIFT;
-	power_state->classification.flags = classification_flag;
-	/* NOTE: There is a classification2 flag in BIOS that is not being used right now */
-
-	power_state->classification.temporary_state = false;
-	power_state->classification.to_be_deleted = false;
-
-	power_state->validation.disallowOnDC =
-			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) &
-					ATOM_Tonga_DISALLOW_ON_DC));
-
-	power_state->pcie.lanes = 0;
-
-	power_state->display.disableFrameModulation = false;
-	power_state->display.limitRefreshrate = false;
-	power_state->display.enableVariBright =
-			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) &
-					ATOM_Tonga_ENABLE_VARIBRIGHT));
-
-	power_state->validation.supportedPowerLevels = 0;
-	power_state->uvd_clocks.VCLK = 0;
-	power_state->uvd_clocks.DCLK = 0;
-	power_state->temperatures.min = 0;
-	power_state->temperatures.max = 0;
-
-	performance_level = &(fiji_power_state->performance_levels
-			[fiji_power_state->performance_level_count++]);
-
-	PP_ASSERT_WITH_CODE(
-			(fiji_power_state->performance_level_count < SMU73_MAX_LEVELS_GRAPHICS),
-			"Performance levels exceeds SMC limit!",
-			return -1);
-
-	PP_ASSERT_WITH_CODE(
-			(fiji_power_state->performance_level_count <=
-					hwmgr->platform_descriptor.hardwareActivityPerformanceLevels),
-			"Performance levels exceeds Driver limit!",
-			return -1);
-
-	/* Performance levels are arranged from low to high. */
-	performance_level->memory_clock = mclk_dep_table->entries
-			[state_entry->ucMemoryClockIndexLow].ulMclk;
-	performance_level->engine_clock = sclk_dep_table->entries
-			[state_entry->ucEngineClockIndexLow].ulSclk;
-	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap,
-			state_entry->ucPCIEGenLow);
-	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap,
-			state_entry->ucPCIELaneHigh);
-
-	performance_level = &(fiji_power_state->performance_levels
-			[fiji_power_state->performance_level_count++]);
-	performance_level->memory_clock = mclk_dep_table->entries
-			[state_entry->ucMemoryClockIndexHigh].ulMclk;
-	performance_level->engine_clock = sclk_dep_table->entries
-			[state_entry->ucEngineClockIndexHigh].ulSclk;
-	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap,
-			state_entry->ucPCIEGenHigh);
-	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap,
-			state_entry->ucPCIELaneHigh);
-
-	return 0;
-}
-
-static int fiji_get_pp_table_entry(struct pp_hwmgr *hwmgr,
-		unsigned long entry_index, struct pp_power_state *state)
-{
-	int result;
-	struct fiji_power_state *ps;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
-			table_info->vdd_dep_on_mclk;
-
-	state->hardware.magic = PHM_VIslands_Magic;
-
-	ps = (struct fiji_power_state *)(&state->hardware);
-
-	result = tonga_get_powerplay_table_entry(hwmgr, entry_index, state,
-			fiji_get_pp_table_entry_callback_func);
-
-	/* This is the earliest time we have all the dependency table and the VBIOS boot state
-	 * as PP_Tables_GetPowerPlayTableEntry retrieves the VBIOS boot state
-	 * if there is only one VDDCI/MCLK level, check if it's the same as VBIOS boot state
-	 */
-	if (dep_mclk_table != NULL && dep_mclk_table->count == 1) {
-		if (dep_mclk_table->entries[0].clk !=
-				data->vbios_boot_state.mclk_bootup_value)
-			printk(KERN_ERR "Single MCLK entry VDDCI/MCLK dependency table "
-					"does not match VBIOS boot MCLK level");
-		if (dep_mclk_table->entries[0].vddci !=
-				data->vbios_boot_state.vddci_bootup_value)
-			printk(KERN_ERR "Single VDDCI entry VDDCI/MCLK dependency table "
-					"does not match VBIOS boot VDDCI level");
-	}
-
-	/* set DC compatible flag if this state supports DC */
-	if (!state->validation.disallowOnDC)
-		ps->dc_compatible = true;
-
-	if (state->classification.flags & PP_StateClassificationFlag_ACPI)
-		data->acpi_pcie_gen = ps->performance_levels[0].pcie_gen;
-
-	ps->uvd_clks.vclk = state->uvd_clocks.VCLK;
-	ps->uvd_clks.dclk = state->uvd_clocks.DCLK;
-
-	if (!result) {
-		uint32_t i;
-
-		switch (state->classification.ui_label) {
-		case PP_StateUILabel_Performance:
-			data->use_pcie_performance_levels = true;
-
-			for (i = 0; i < ps->performance_level_count; i++) {
-				if (data->pcie_gen_performance.max <
-						ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_performance.max =
-							ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_gen_performance.min >
-						ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_performance.min =
-							ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_lane_performance.max <
-						ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_performance.max =
-							ps->performance_levels[i].pcie_lane;
-
-				if (data->pcie_lane_performance.min >
-						ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_performance.min =
-							ps->performance_levels[i].pcie_lane;
-			}
-			break;
-		case PP_StateUILabel_Battery:
-			data->use_pcie_power_saving_levels = true;
-
-			for (i = 0; i < ps->performance_level_count; i++) {
-				if (data->pcie_gen_power_saving.max <
-						ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_power_saving.max =
-							ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_gen_power_saving.min >
-						ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_power_saving.min =
-							ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_lane_power_saving.max <
-						ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_power_saving.max =
-							ps->performance_levels[i].pcie_lane;
-
-				if (data->pcie_lane_power_saving.min >
-						ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_power_saving.min =
-							ps->performance_levels[i].pcie_lane;
-			}
-			break;
-		default:
-			break;
-		}
-	}
-	return 0;
-}
-
-static int fiji_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
-				struct pp_power_state  *request_ps,
-			const struct pp_power_state *current_ps)
-{
-	struct fiji_power_state *fiji_ps =
-				cast_phw_fiji_power_state(&request_ps->hardware);
-	uint32_t sclk;
-	uint32_t mclk;
-	struct PP_Clocks minimum_clocks = {0};
-	bool disable_mclk_switching;
-	bool disable_mclk_switching_for_frame_lock;
-	struct cgs_display_info info = {0};
-	const struct phm_clock_and_voltage_limits *max_limits;
-	uint32_t i;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	int32_t count;
-	int32_t stable_pstate_sclk = 0, stable_pstate_mclk = 0;
-
-	data->battery_state = (PP_StateUILabel_Battery ==
-			request_ps->classification.ui_label);
-
-	PP_ASSERT_WITH_CODE(fiji_ps->performance_level_count == 2,
-				 "VI should always have 2 performance levels",);
-
-	max_limits = (PP_PowerSource_AC == hwmgr->power_source) ?
-			&(hwmgr->dyn_state.max_clock_voltage_on_ac) :
-			&(hwmgr->dyn_state.max_clock_voltage_on_dc);
-
-	/* Cap clock DPM tables at DC MAX if it is in DC. */
-	if (PP_PowerSource_DC == hwmgr->power_source) {
-		for (i = 0; i < fiji_ps->performance_level_count; i++) {
-			if (fiji_ps->performance_levels[i].memory_clock > max_limits->mclk)
-				fiji_ps->performance_levels[i].memory_clock = max_limits->mclk;
-			if (fiji_ps->performance_levels[i].engine_clock > max_limits->sclk)
-				fiji_ps->performance_levels[i].engine_clock = max_limits->sclk;
-		}
-	}
-
-	fiji_ps->vce_clks.evclk = hwmgr->vce_arbiter.evclk;
-	fiji_ps->vce_clks.ecclk = hwmgr->vce_arbiter.ecclk;
-
-	fiji_ps->acp_clk = hwmgr->acp_arbiter.acpclk;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	/*TO DO result = PHM_CheckVBlankTime(hwmgr, &vblankTooShort);*/
-
-	/* TO DO GetMinClockSettings(hwmgr->pPECI, &minimum_clocks); */
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_StablePState)) {
-		max_limits = &(hwmgr->dyn_state.max_clock_voltage_on_ac);
-		stable_pstate_sclk = (max_limits->sclk * 75) / 100;
-
-		for (count = table_info->vdd_dep_on_sclk->count - 1;
-				count >= 0; count--) {
-			if (stable_pstate_sclk >=
-					table_info->vdd_dep_on_sclk->entries[count].clk) {
-				stable_pstate_sclk =
-						table_info->vdd_dep_on_sclk->entries[count].clk;
-				break;
-			}
-		}
-
-		if (count < 0)
-			stable_pstate_sclk = table_info->vdd_dep_on_sclk->entries[0].clk;
-
-		stable_pstate_mclk = max_limits->mclk;
-
-		minimum_clocks.engineClock = stable_pstate_sclk;
-		minimum_clocks.memoryClock = stable_pstate_mclk;
-	}
-
-	if (minimum_clocks.engineClock < hwmgr->gfx_arbiter.sclk)
-		minimum_clocks.engineClock = hwmgr->gfx_arbiter.sclk;
-
-	if (minimum_clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
-		minimum_clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
-
-	fiji_ps->sclk_threshold = hwmgr->gfx_arbiter.sclk_threshold;
-
-	if (0 != hwmgr->gfx_arbiter.sclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.sclk_over_drive <=
-				hwmgr->platform_descriptor.overdriveLimit.engineClock),
-				"Overdrive sclk exceeds limit",
-				hwmgr->gfx_arbiter.sclk_over_drive =
-						hwmgr->platform_descriptor.overdriveLimit.engineClock);
-
-		if (hwmgr->gfx_arbiter.sclk_over_drive >= hwmgr->gfx_arbiter.sclk)
-			fiji_ps->performance_levels[1].engine_clock =
-					hwmgr->gfx_arbiter.sclk_over_drive;
-	}
-
-	if (0 != hwmgr->gfx_arbiter.mclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.mclk_over_drive <=
-				hwmgr->platform_descriptor.overdriveLimit.memoryClock),
-				"Overdrive mclk exceeds limit",
-				hwmgr->gfx_arbiter.mclk_over_drive =
-						hwmgr->platform_descriptor.overdriveLimit.memoryClock);
-
-		if (hwmgr->gfx_arbiter.mclk_over_drive >= hwmgr->gfx_arbiter.mclk)
-			fiji_ps->performance_levels[1].memory_clock =
-					hwmgr->gfx_arbiter.mclk_over_drive;
-	}
-
-	disable_mclk_switching_for_frame_lock = phm_cap_enabled(
-				    hwmgr->platform_descriptor.platformCaps,
-				    PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
-
-	disable_mclk_switching = (1 < info.display_count) ||
-				    disable_mclk_switching_for_frame_lock;
-
-	sclk = fiji_ps->performance_levels[0].engine_clock;
-	mclk = fiji_ps->performance_levels[0].memory_clock;
-
-	if (disable_mclk_switching)
-		mclk = fiji_ps->performance_levels
-		[fiji_ps->performance_level_count - 1].memory_clock;
-
-	if (sclk < minimum_clocks.engineClock)
-		sclk = (minimum_clocks.engineClock > max_limits->sclk) ?
-				max_limits->sclk : minimum_clocks.engineClock;
-
-	if (mclk < minimum_clocks.memoryClock)
-		mclk = (minimum_clocks.memoryClock > max_limits->mclk) ?
-				max_limits->mclk : minimum_clocks.memoryClock;
-
-	fiji_ps->performance_levels[0].engine_clock = sclk;
-	fiji_ps->performance_levels[0].memory_clock = mclk;
-
-	fiji_ps->performance_levels[1].engine_clock =
-		(fiji_ps->performance_levels[1].engine_clock >=
-				fiji_ps->performance_levels[0].engine_clock) ?
-						fiji_ps->performance_levels[1].engine_clock :
-						fiji_ps->performance_levels[0].engine_clock;
-
-	if (disable_mclk_switching) {
-		if (mclk < fiji_ps->performance_levels[1].memory_clock)
-			mclk = fiji_ps->performance_levels[1].memory_clock;
-
-		fiji_ps->performance_levels[0].memory_clock = mclk;
-		fiji_ps->performance_levels[1].memory_clock = mclk;
-	} else {
-		if (fiji_ps->performance_levels[1].memory_clock <
-				fiji_ps->performance_levels[0].memory_clock)
-			fiji_ps->performance_levels[1].memory_clock =
-					fiji_ps->performance_levels[0].memory_clock;
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_StablePState)) {
-		for (i = 0; i < fiji_ps->performance_level_count; i++) {
-			fiji_ps->performance_levels[i].engine_clock = stable_pstate_sclk;
-			fiji_ps->performance_levels[i].memory_clock = stable_pstate_mclk;
-			fiji_ps->performance_levels[i].pcie_gen = data->pcie_gen_performance.max;
-			fiji_ps->performance_levels[i].pcie_lane = data->pcie_gen_performance.max;
-		}
-	}
-
-	return 0;
-}
-
-static int fiji_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	const struct fiji_power_state *fiji_ps =
-			cast_const_phw_fiji_power_state(states->pnew_state);
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
-	uint32_t sclk = fiji_ps->performance_levels
-			[fiji_ps->performance_level_count - 1].engine_clock;
-	struct fiji_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
-	uint32_t mclk = fiji_ps->performance_levels
-			[fiji_ps->performance_level_count - 1].memory_clock;
-	uint32_t i;
-	struct cgs_display_info info = {0};
-
-	data->need_update_smu7_dpm_table = 0;
-
-	for (i = 0; i < sclk_table->count; i++) {
-		if (sclk == sclk_table->dpm_levels[i].value)
-			break;
-	}
-
-	if (i >= sclk_table->count)
-		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
-	else {
-		if(data->display_timing.min_clock_in_sr !=
-			hwmgr->display_config.min_core_set_clock_in_sr)
-			data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
-	}
-
-	for (i = 0; i < mclk_table->count; i++) {
-		if (mclk == mclk_table->dpm_levels[i].value)
-			break;
-	}
-
-	if (i >= mclk_table->count)
-		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	if (data->display_timing.num_existing_displays != info.display_count)
-		data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
-
-	return 0;
-}
-
-static uint16_t fiji_get_maximum_link_speed(struct pp_hwmgr *hwmgr,
-		const struct fiji_power_state *fiji_ps)
-{
-	uint32_t i;
-	uint32_t sclk, max_sclk = 0;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_dpm_table *dpm_table = &data->dpm_table;
-
-	for (i = 0; i < fiji_ps->performance_level_count; i++) {
-		sclk = fiji_ps->performance_levels[i].engine_clock;
-		if (max_sclk < sclk)
-			max_sclk = sclk;
-	}
-
-	for (i = 0; i < dpm_table->sclk_table.count; i++) {
-		if (dpm_table->sclk_table.dpm_levels[i].value == max_sclk)
-			return (uint16_t) ((i >= dpm_table->pcie_speed_table.count) ?
-					dpm_table->pcie_speed_table.dpm_levels
-					[dpm_table->pcie_speed_table.count - 1].value :
-					dpm_table->pcie_speed_table.dpm_levels[i].value);
-	}
-
-	return 0;
-}
-
-static int fiji_request_link_speed_change_before_state_change(
-		struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	const struct fiji_power_state *fiji_nps =
-			cast_const_phw_fiji_power_state(states->pnew_state);
-	const struct fiji_power_state *fiji_cps =
-			cast_const_phw_fiji_power_state(states->pcurrent_state);
-
-	uint16_t target_link_speed = fiji_get_maximum_link_speed(hwmgr, fiji_nps);
-	uint16_t current_link_speed;
-
-	if (data->force_pcie_gen == PP_PCIEGenInvalid)
-		current_link_speed = fiji_get_maximum_link_speed(hwmgr, fiji_cps);
-	else
-		current_link_speed = data->force_pcie_gen;
-
-	data->force_pcie_gen = PP_PCIEGenInvalid;
-	data->pspp_notify_required = false;
-	if (target_link_speed > current_link_speed) {
-		switch(target_link_speed) {
-		case PP_PCIEGen3:
-			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN3, false))
-				break;
-			data->force_pcie_gen = PP_PCIEGen2;
-			if (current_link_speed == PP_PCIEGen2)
-				break;
-		case PP_PCIEGen2:
-			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN2, false))
-				break;
-		default:
-			data->force_pcie_gen = fiji_get_current_pcie_speed(hwmgr);
-			break;
-		}
-	} else {
-		if (target_link_speed < current_link_speed)
-			data->pspp_notify_required = true;
-	}
-
-	return 0;
-}
-
-static int fiji_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (0 == data->need_update_smu7_dpm_table)
-		return 0;
-
-	if ((0 == data->sclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table &
-			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
-		PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr),
-				    "Trying to freeze SCLK DPM when DPM is disabled",
-				    );
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_SCLKDPM_FreezeLevel),
-				"Failed to freeze SCLK DPM during FreezeSclkMclkDPM Function!",
-				return -1);
-	}
-
-	if ((0 == data->mclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table &
-		 DPMTABLE_OD_UPDATE_MCLK)) {
-		PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr),
-				    "Trying to freeze MCLK DPM when DPM is disabled",
-				    );
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_MCLKDPM_FreezeLevel),
-				"Failed to freeze MCLK DPM during FreezeSclkMclkDPM Function!",
-				return -1);
-	}
-
-	return 0;
-}
-
-static int fiji_populate_and_upload_sclk_mclk_dpm_levels(
-		struct pp_hwmgr *hwmgr, const void *input)
-{
-	int result = 0;
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	const struct fiji_power_state *fiji_ps =
-			cast_const_phw_fiji_power_state(states->pnew_state);
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t sclk = fiji_ps->performance_levels
-			[fiji_ps->performance_level_count - 1].engine_clock;
-	uint32_t mclk = fiji_ps->performance_levels
-			[fiji_ps->performance_level_count - 1].memory_clock;
-	struct fiji_dpm_table *dpm_table = &data->dpm_table;
-
-	struct fiji_dpm_table *golden_dpm_table = &data->golden_dpm_table;
-	uint32_t dpm_count, clock_percent;
-	uint32_t i;
-
-	if (0 == data->need_update_smu7_dpm_table)
-		return 0;
-
-	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK) {
-		dpm_table->sclk_table.dpm_levels
-		[dpm_table->sclk_table.count - 1].value = sclk;
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_OD6PlusinACSupport) ||
-			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_OD6PlusinDCSupport)) {
-		/* Need to do calculation based on the golden DPM table
-		 * as the Heatmap GPU Clock axis is also based on the default values
-		 */
-			PP_ASSERT_WITH_CODE(
-				(golden_dpm_table->sclk_table.dpm_levels
-						[golden_dpm_table->sclk_table.count - 1].value != 0),
-				"Divide by 0!",
-				return -1);
-			dpm_count = dpm_table->sclk_table.count < 2 ?
-					0 : dpm_table->sclk_table.count - 2;
-			for (i = dpm_count; i > 1; i--) {
-				if (sclk > golden_dpm_table->sclk_table.dpm_levels
-						[golden_dpm_table->sclk_table.count-1].value) {
-					clock_percent =
-						((sclk - golden_dpm_table->sclk_table.dpm_levels
-							[golden_dpm_table->sclk_table.count-1].value) * 100) /
-						golden_dpm_table->sclk_table.dpm_levels
-							[golden_dpm_table->sclk_table.count-1].value;
-
-					dpm_table->sclk_table.dpm_levels[i].value =
-							golden_dpm_table->sclk_table.dpm_levels[i].value +
-							(golden_dpm_table->sclk_table.dpm_levels[i].value *
-								clock_percent)/100;
-
-				} else if (golden_dpm_table->sclk_table.dpm_levels
-						[dpm_table->sclk_table.count-1].value > sclk) {
-					clock_percent =
-						((golden_dpm_table->sclk_table.dpm_levels
-						[golden_dpm_table->sclk_table.count - 1].value - sclk) *
-								100) /
-						golden_dpm_table->sclk_table.dpm_levels
-							[golden_dpm_table->sclk_table.count-1].value;
-
-					dpm_table->sclk_table.dpm_levels[i].value =
-							golden_dpm_table->sclk_table.dpm_levels[i].value -
-							(golden_dpm_table->sclk_table.dpm_levels[i].value *
-									clock_percent) / 100;
-				} else
-					dpm_table->sclk_table.dpm_levels[i].value =
-							golden_dpm_table->sclk_table.dpm_levels[i].value;
-			}
-		}
-	}
-
-	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK) {
-		dpm_table->mclk_table.dpm_levels
-			[dpm_table->mclk_table.count - 1].value = mclk;
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_OD6PlusinACSupport) ||
-			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_OD6PlusinDCSupport)) {
-
-			PP_ASSERT_WITH_CODE(
-					(golden_dpm_table->mclk_table.dpm_levels
-						[golden_dpm_table->mclk_table.count-1].value != 0),
-					"Divide by 0!",
-					return -1);
-			dpm_count = dpm_table->mclk_table.count < 2 ?
-					0 : dpm_table->mclk_table.count - 2;
-			for (i = dpm_count; i > 1; i--) {
-				if (mclk > golden_dpm_table->mclk_table.dpm_levels
-						[golden_dpm_table->mclk_table.count-1].value) {
-					clock_percent = ((mclk -
-							golden_dpm_table->mclk_table.dpm_levels
-							[golden_dpm_table->mclk_table.count-1].value) * 100) /
-							golden_dpm_table->mclk_table.dpm_levels
-							[golden_dpm_table->mclk_table.count-1].value;
-
-					dpm_table->mclk_table.dpm_levels[i].value =
-							golden_dpm_table->mclk_table.dpm_levels[i].value +
-							(golden_dpm_table->mclk_table.dpm_levels[i].value *
-									clock_percent) / 100;
-
-				} else if (golden_dpm_table->mclk_table.dpm_levels
-						[dpm_table->mclk_table.count-1].value > mclk) {
-					clock_percent = ((golden_dpm_table->mclk_table.dpm_levels
-							[golden_dpm_table->mclk_table.count-1].value - mclk) * 100) /
-									golden_dpm_table->mclk_table.dpm_levels
-									[golden_dpm_table->mclk_table.count-1].value;
-
-					dpm_table->mclk_table.dpm_levels[i].value =
-							golden_dpm_table->mclk_table.dpm_levels[i].value -
-							(golden_dpm_table->mclk_table.dpm_levels[i].value *
-									clock_percent) / 100;
-				} else
-					dpm_table->mclk_table.dpm_levels[i].value =
-							golden_dpm_table->mclk_table.dpm_levels[i].value;
-			}
-		}
-	}
-
-	if (data->need_update_smu7_dpm_table &
-			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK)) {
-		result = fiji_populate_all_graphic_levels(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
-				return result);
-	}
-
-	if (data->need_update_smu7_dpm_table &
-			(DPMTABLE_OD_UPDATE_MCLK + DPMTABLE_UPDATE_MCLK)) {
-		/*populate MCLK dpm table to SMU7 */
-		result = fiji_populate_all_memory_levels(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
-				return result);
-	}
-
-	return result;
-}
-
-static int fiji_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
-			  struct fiji_single_dpm_table * dpm_table,
-			     uint32_t low_limit, uint32_t high_limit)
-{
-	uint32_t i;
-
-	for (i = 0; i < dpm_table->count; i++) {
-		if ((dpm_table->dpm_levels[i].value < low_limit) ||
-		    (dpm_table->dpm_levels[i].value > high_limit))
-			dpm_table->dpm_levels[i].enabled = false;
-		else
-			dpm_table->dpm_levels[i].enabled = true;
-	}
-	return 0;
-}
-
-static int fiji_trim_dpm_states(struct pp_hwmgr *hwmgr,
-		const struct fiji_power_state *fiji_ps)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t high_limit_count;
-
-	PP_ASSERT_WITH_CODE((fiji_ps->performance_level_count >= 1),
-			"power state did not have any performance level",
-			return -1);
-
-	high_limit_count = (1 == fiji_ps->performance_level_count) ? 0 : 1;
-
-	fiji_trim_single_dpm_states(hwmgr,
-			&(data->dpm_table.sclk_table),
-			fiji_ps->performance_levels[0].engine_clock,
-			fiji_ps->performance_levels[high_limit_count].engine_clock);
-
-	fiji_trim_single_dpm_states(hwmgr,
-			&(data->dpm_table.mclk_table),
-			fiji_ps->performance_levels[0].memory_clock,
-			fiji_ps->performance_levels[high_limit_count].memory_clock);
-
-	return 0;
-}
-
-static int fiji_generate_dpm_level_enable_mask(
-		struct pp_hwmgr *hwmgr, const void *input)
-{
-	int result;
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	const struct fiji_power_state *fiji_ps =
-			cast_const_phw_fiji_power_state(states->pnew_state);
-
-	result = fiji_trim_dpm_states(hwmgr, fiji_ps);
-	if (result)
-		return result;
-
-	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
-			fiji_get_dpm_level_enable_mask_value(&data->dpm_table.sclk_table);
-	data->dpm_level_enable_mask.mclk_dpm_enable_mask =
-			fiji_get_dpm_level_enable_mask_value(&data->dpm_table.mclk_table);
-	data->last_mclk_dpm_enable_mask =
-			data->dpm_level_enable_mask.mclk_dpm_enable_mask;
-
-	if (data->uvd_enabled) {
-		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask & 1)
-			data->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
-	}
-
-	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
-			fiji_get_dpm_level_enable_mask_value(&data->dpm_table.pcie_speed_table);
-
-	return 0;
-}
-
-int fiji_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
-				  (PPSMC_Msg)PPSMC_MSG_UVDDPM_Enable :
-				  (PPSMC_Msg)PPSMC_MSG_UVDDPM_Disable);
-}
-
-int fiji_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, enable?
-			PPSMC_MSG_VCEDPM_Enable :
-			PPSMC_MSG_VCEDPM_Disable);
-}
-
-int fiji_enable_disable_samu_dpm(struct pp_hwmgr *hwmgr, bool enable)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, enable?
-			PPSMC_MSG_SAMUDPM_Enable :
-			PPSMC_MSG_SAMUDPM_Disable);
-}
-
-int fiji_enable_disable_acp_dpm(struct pp_hwmgr *hwmgr, bool enable)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, enable?
-			PPSMC_MSG_ACPDPM_Enable :
-			PPSMC_MSG_ACPDPM_Disable);
-}
-
-int fiji_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t mm_boot_level_offset, mm_boot_level_value;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (!bgate) {
-		data->smc_state_table.UvdBootLevel = 0;
-		if (table_info->mm_dep_table->count > 0)
-			data->smc_state_table.UvdBootLevel =
-					(uint8_t) (table_info->mm_dep_table->count - 1);
-		mm_boot_level_offset = data->dpm_table_start +
-				offsetof(SMU73_Discrete_DpmTable, UvdBootLevel);
-		mm_boot_level_offset /= 4;
-		mm_boot_level_offset *= 4;
-		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset);
-		mm_boot_level_value &= 0x00FFFFFF;
-		mm_boot_level_value |= data->smc_state_table.UvdBootLevel << 24;
-		cgs_write_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
-
-		if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_UVDDPM) ||
-			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_StablePState))
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_UVDDPM_SetEnabledMask,
-					(uint32_t)(1 << data->smc_state_table.UvdBootLevel));
-	}
-
-	return fiji_enable_disable_uvd_dpm(hwmgr, !bgate);
-}
-
-int fiji_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	const struct fiji_power_state *fiji_nps =
-			cast_const_phw_fiji_power_state(states->pnew_state);
-	const struct fiji_power_state *fiji_cps =
-			cast_const_phw_fiji_power_state(states->pcurrent_state);
-
-	uint32_t mm_boot_level_offset, mm_boot_level_value;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (fiji_nps->vce_clks.evclk >0 &&
-	(fiji_cps == NULL || fiji_cps->vce_clks.evclk == 0)) {
-		data->smc_state_table.VceBootLevel =
-				(uint8_t) (table_info->mm_dep_table->count - 1);
-
-		mm_boot_level_offset = data->dpm_table_start +
-				offsetof(SMU73_Discrete_DpmTable, VceBootLevel);
-		mm_boot_level_offset /= 4;
-		mm_boot_level_offset *= 4;
-		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset);
-		mm_boot_level_value &= 0xFF00FFFF;
-		mm_boot_level_value |= data->smc_state_table.VceBootLevel << 16;
-		cgs_write_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_StablePState)) {
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_VCEDPM_SetEnabledMask,
-					(uint32_t)1 << data->smc_state_table.VceBootLevel);
-
-			fiji_enable_disable_vce_dpm(hwmgr, true);
-		} else if (fiji_nps->vce_clks.evclk == 0 &&
-				fiji_cps != NULL &&
-				fiji_cps->vce_clks.evclk > 0)
-			fiji_enable_disable_vce_dpm(hwmgr, false);
-	}
-
-	return 0;
-}
-
-int fiji_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t mm_boot_level_offset, mm_boot_level_value;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (!bgate) {
-		data->smc_state_table.SamuBootLevel =
-				(uint8_t) (table_info->mm_dep_table->count - 1);
-		mm_boot_level_offset = data->dpm_table_start +
-				offsetof(SMU73_Discrete_DpmTable, SamuBootLevel);
-		mm_boot_level_offset /= 4;
-		mm_boot_level_offset *= 4;
-		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset);
-		mm_boot_level_value &= 0xFFFFFF00;
-		mm_boot_level_value |= data->smc_state_table.SamuBootLevel << 0;
-		cgs_write_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_StablePState))
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_SAMUDPM_SetEnabledMask,
-					(uint32_t)(1 << data->smc_state_table.SamuBootLevel));
-	}
-
-	return fiji_enable_disable_samu_dpm(hwmgr, !bgate);
-}
-
-int fiji_update_acp_dpm(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t mm_boot_level_offset, mm_boot_level_value;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (!bgate) {
-		data->smc_state_table.AcpBootLevel =
-				(uint8_t) (table_info->mm_dep_table->count - 1);
-		mm_boot_level_offset = data->dpm_table_start +
-				offsetof(SMU73_Discrete_DpmTable, AcpBootLevel);
-		mm_boot_level_offset /= 4;
-		mm_boot_level_offset *= 4;
-		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset);
-		mm_boot_level_value &= 0xFFFF00FF;
-		mm_boot_level_value |= data->smc_state_table.AcpBootLevel << 8;
-		cgs_write_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_StablePState))
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-						PPSMC_MSG_ACPDPM_SetEnabledMask,
-						(uint32_t)(1 << data->smc_state_table.AcpBootLevel));
-	}
-
-	return fiji_enable_disable_acp_dpm(hwmgr, !bgate);
-}
-
-static int fiji_update_sclk_threshold(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	int result = 0;
-	uint32_t low_sclk_interrupt_threshold = 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkThrottleLowNotification)
-		&& (hwmgr->gfx_arbiter.sclk_threshold !=
-				data->low_sclk_interrupt_threshold)) {
-		data->low_sclk_interrupt_threshold =
-				hwmgr->gfx_arbiter.sclk_threshold;
-		low_sclk_interrupt_threshold =
-				data->low_sclk_interrupt_threshold;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
-
-		result = fiji_copy_bytes_to_smc(
-				hwmgr->smumgr,
-				data->dpm_table_start +
-				offsetof(SMU73_Discrete_DpmTable,
-					LowSclkInterruptThreshold),
-				(uint8_t *)&low_sclk_interrupt_threshold,
-				sizeof(uint32_t),
-				data->sram_end);
-	}
-
-	return result;
-}
-
-static int fiji_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (data->need_update_smu7_dpm_table &
-		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_OD_UPDATE_MCLK))
-		return fiji_program_memory_timing_parameters(hwmgr);
-
-	return 0;
-}
-
-static int fiji_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (0 == data->need_update_smu7_dpm_table)
-		return 0;
-
-	if ((0 == data->sclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table &
-		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
-
-		PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr),
-				    "Trying to Unfreeze SCLK DPM when DPM is disabled",
-				    );
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_SCLKDPM_UnfreezeLevel),
-			"Failed to unfreeze SCLK DPM during UnFreezeSclkMclkDPM Function!",
-			return -1);
-	}
-
-	if ((0 == data->mclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
-
-		PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr),
-				    "Trying to Unfreeze MCLK DPM when DPM is disabled",
-				    );
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_SCLKDPM_UnfreezeLevel),
-		    "Failed to unfreeze MCLK DPM during UnFreezeSclkMclkDPM Function!",
-		    return -1);
-	}
-
-	data->need_update_smu7_dpm_table = 0;
-
-	return 0;
-}
-
-/* Look up the voltaged based on DAL's requested level.
- * and then send the requested VDDC voltage to SMC
- */
-static void fiji_apply_dal_minimum_voltage_request(struct pp_hwmgr *hwmgr)
-{
-	return;
-}
-
-int fiji_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr)
-{
-	int result;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	/* Apply minimum voltage based on DAL's request level */
-	fiji_apply_dal_minimum_voltage_request(hwmgr);
-
-	if (0 == data->sclk_dpm_key_disabled) {
-		/* Checking if DPM is running.  If we discover hang because of this,
-		 *  we should skip this message.
-		 */
-		if (!fiji_is_dpm_running(hwmgr))
-			printk(KERN_ERR "[ powerplay ] "
-					"Trying to set Enable Mask when DPM is disabled \n");
-
-		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
-			result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_SCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.sclk_dpm_enable_mask);
-			PP_ASSERT_WITH_CODE((0 == result),
-				"Set Sclk Dpm enable Mask failed", return -1);
-		}
-	}
-
-	if (0 == data->mclk_dpm_key_disabled) {
-		/* Checking if DPM is running.  If we discover hang because of this,
-		 *  we should skip this message.
-		 */
-		if (!fiji_is_dpm_running(hwmgr))
-			printk(KERN_ERR "[ powerplay ]"
-					" Trying to set Enable Mask when DPM is disabled \n");
-
-		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
-			result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_MCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.mclk_dpm_enable_mask);
-			PP_ASSERT_WITH_CODE((0 == result),
-				"Set Mclk Dpm enable Mask failed", return -1);
-		}
-	}
-
-	return 0;
-}
-
-static int fiji_notify_link_speed_change_after_state_change(
-		struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	const struct fiji_power_state *fiji_ps =
-			cast_const_phw_fiji_power_state(states->pnew_state);
-	uint16_t target_link_speed = fiji_get_maximum_link_speed(hwmgr, fiji_ps);
-	uint8_t  request;
-
-	if (data->pspp_notify_required) {
-		if (target_link_speed == PP_PCIEGen3)
-			request = PCIE_PERF_REQ_GEN3;
-		else if (target_link_speed == PP_PCIEGen2)
-			request = PCIE_PERF_REQ_GEN2;
-		else
-			request = PCIE_PERF_REQ_GEN1;
-
-		if(request == PCIE_PERF_REQ_GEN1 &&
-				fiji_get_current_pcie_speed(hwmgr) > 0)
-			return 0;
-
-		if (acpi_pcie_perf_request(hwmgr->device, request, false)) {
-			if (PP_PCIEGen2 == target_link_speed)
-				printk("PSPP request to switch to Gen2 from Gen3 Failed!");
-			else
-				printk("PSPP request to switch to Gen1 from Gen2 Failed!");
-		}
-	}
-
-	return 0;
-}
-
-static int fiji_set_power_state_tasks(struct pp_hwmgr *hwmgr,
-		const void *input)
-{
-	int tmp_result, result = 0;
-
-	tmp_result = fiji_find_dpm_states_clocks_in_dpm_table(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to find DPM states clocks in DPM table!",
-			result = tmp_result);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PCIEPerformanceRequest)) {
-		tmp_result =
-			fiji_request_link_speed_change_before_state_change(hwmgr, input);
-		PP_ASSERT_WITH_CODE((0 == tmp_result),
-				"Failed to request link speed change before state change!",
-				result = tmp_result);
-	}
-
-	tmp_result = fiji_freeze_sclk_mclk_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to freeze SCLK MCLK DPM!", result = tmp_result);
-
-	tmp_result = fiji_populate_and_upload_sclk_mclk_dpm_levels(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to populate and upload SCLK MCLK DPM levels!",
-			result = tmp_result);
-
-	tmp_result = fiji_generate_dpm_level_enable_mask(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to generate DPM level enabled mask!",
-			result = tmp_result);
-
-	tmp_result = fiji_update_vce_dpm(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to update VCE DPM!",
-			result = tmp_result);
-
-	tmp_result = fiji_update_sclk_threshold(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to update SCLK threshold!",
-			result = tmp_result);
-
-	tmp_result = fiji_program_mem_timing_parameters(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to program memory timing parameters!",
-			result = tmp_result);
-
-	tmp_result = fiji_unfreeze_sclk_mclk_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to unfreeze SCLK MCLK DPM!",
-			result = tmp_result);
-
-	tmp_result = fiji_upload_dpm_level_enable_mask(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to upload DPM level enabled mask!",
-			result = tmp_result);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PCIEPerformanceRequest)) {
-		tmp_result =
-			fiji_notify_link_speed_change_after_state_change(hwmgr, input);
-		PP_ASSERT_WITH_CODE((0 == tmp_result),
-				"Failed to notify link speed change after state change!",
-				result = tmp_result);
-	}
-
-	return result;
-}
-
-static int fiji_dpm_get_sclk(struct pp_hwmgr *hwmgr, bool low)
-{
-	struct pp_power_state  *ps;
-	struct fiji_power_state  *fiji_ps;
-
-	if (hwmgr == NULL)
-		return -EINVAL;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	fiji_ps = cast_phw_fiji_power_state(&ps->hardware);
-
-	if (low)
-		return fiji_ps->performance_levels[0].engine_clock;
-	else
-		return fiji_ps->performance_levels
-				[fiji_ps->performance_level_count-1].engine_clock;
-}
-
-static int fiji_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
-{
-	struct pp_power_state  *ps;
-	struct fiji_power_state  *fiji_ps;
-
-	if (hwmgr == NULL)
-		return -EINVAL;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	fiji_ps = cast_phw_fiji_power_state(&ps->hardware);
-
-	if (low)
-		return fiji_ps->performance_levels[0].memory_clock;
-	else
-		return fiji_ps->performance_levels
-				[fiji_ps->performance_level_count-1].memory_clock;
-}
-
-static void fiji_print_current_perforce_level(
-		struct pp_hwmgr *hwmgr, struct seq_file *m)
-{
-	uint32_t sclk, mclk, activity_percent = 0;
-	uint32_t offset;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetSclkFrequency);
-
-	sclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-
-	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetMclkFrequency);
-
-	mclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-	seq_printf(m, "\n [  mclk  ]: %u MHz\n\n [  sclk  ]: %u MHz\n",
-			mclk / 100, sclk / 100);
-
-	offset = data->soft_regs_start + offsetof(SMU73_SoftRegisters, AverageGraphicsActivity);
-	activity_percent = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, offset);
-	activity_percent += 0x80;
-	activity_percent >>= 8;
-
-	seq_printf(m, "\n [GPU load]: %u%%\n\n", activity_percent > 100 ? 100 : activity_percent);
-
-	seq_printf(m, "uvd    %sabled\n", data->uvd_power_gated ? "dis" : "en");
-
-	seq_printf(m, "vce    %sabled\n", data->vce_power_gated ? "dis" : "en");
-}
-
-static int fiji_program_display_gap(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t num_active_displays = 0;
-	uint32_t display_gap = cgs_read_ind_register(hwmgr->device,
-			CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL);
-	uint32_t display_gap2;
-	uint32_t pre_vbi_time_in_us;
-	uint32_t frame_time_in_us;
-	uint32_t ref_clock;
-	uint32_t refresh_rate = 0;
-	struct cgs_display_info info = {0};
-	struct cgs_mode_info mode_info;
-
-	info.mode_info = &mode_info;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	num_active_displays = info.display_count;
-
-	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL,
-			DISP_GAP, (num_active_displays > 0)?
-			DISPLAY_GAP_VBLANK_OR_WM : DISPLAY_GAP_IGNORE);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_DISPLAY_GAP_CNTL, display_gap);
-
-	ref_clock = mode_info.ref_clock;
-	refresh_rate = mode_info.refresh_rate;
-
-	if (refresh_rate == 0)
-		refresh_rate = 60;
-
-	frame_time_in_us = 1000000 / refresh_rate;
-
-	pre_vbi_time_in_us = frame_time_in_us - 200 - mode_info.vblank_time_us;
-	display_gap2 = pre_vbi_time_in_us * (ref_clock / 100);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_DISPLAY_GAP_CNTL2, display_gap2);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			data->soft_regs_start +
-			offsetof(SMU73_SoftRegisters, PreVBlankGap), 0x64);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			data->soft_regs_start +
-			offsetof(SMU73_SoftRegisters, VBlankTimeout),
-			(frame_time_in_us - pre_vbi_time_in_us));
-
-	if (num_active_displays == 1)
-		tonga_notify_smc_display_change(hwmgr, true);
-
-	return 0;
-}
-
-int fiji_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
-{
-	return fiji_program_display_gap(hwmgr);
-}
-
-static int fiji_set_max_fan_pwm_output(struct pp_hwmgr *hwmgr,
-		uint16_t us_max_fan_pwm)
-{
-	hwmgr->thermal_controller.
-	advanceFanControlParameters.usMaxFanPWM = us_max_fan_pwm;
-
-	if (phm_is_hw_access_blocked(hwmgr))
-		return 0;
-
-	return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-			PPSMC_MSG_SetFanPwmMax, us_max_fan_pwm);
-}
-
-static int fiji_set_max_fan_rpm_output(struct pp_hwmgr *hwmgr,
-		uint16_t us_max_fan_rpm)
-{
-	hwmgr->thermal_controller.
-	advanceFanControlParameters.usMaxFanRPM = us_max_fan_rpm;
-
-	if (phm_is_hw_access_blocked(hwmgr))
-		return 0;
-
-	return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-			PPSMC_MSG_SetFanRpmMax, us_max_fan_rpm);
-}
-
-int fiji_dpm_set_interrupt_state(void *private_data,
-					 unsigned src_id, unsigned type,
-					 int enabled)
-{
-	uint32_t cg_thermal_int;
-	struct pp_hwmgr *hwmgr = ((struct pp_eventmgr *)private_data)->hwmgr;
-
-	if (hwmgr == NULL)
-		return -EINVAL;
-
-	switch (type) {
-	case AMD_THERMAL_IRQ_LOW_TO_HIGH:
-		if (enabled) {
-			cg_thermal_int = cgs_read_ind_register(hwmgr->device,
-					CGS_IND_REG__SMC, ixCG_THERMAL_INT);
-			cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
-			cgs_write_ind_register(hwmgr->device,
-					CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
-		} else {
-			cg_thermal_int = cgs_read_ind_register(hwmgr->device,
-					CGS_IND_REG__SMC, ixCG_THERMAL_INT);
-			cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
-			cgs_write_ind_register(hwmgr->device,
-					CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
-		}
-		break;
-
-	case AMD_THERMAL_IRQ_HIGH_TO_LOW:
-		if (enabled) {
-			cg_thermal_int = cgs_read_ind_register(hwmgr->device,
-					CGS_IND_REG__SMC, ixCG_THERMAL_INT);
-			cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
-			cgs_write_ind_register(hwmgr->device,
-					CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
-		} else {
-			cg_thermal_int = cgs_read_ind_register(hwmgr->device,
-					CGS_IND_REG__SMC, ixCG_THERMAL_INT);
-			cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
-			cgs_write_ind_register(hwmgr->device,
-					CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
-		}
-		break;
-	default:
-		break;
-	}
-	return 0;
-}
-
-int fiji_register_internal_thermal_interrupt(struct pp_hwmgr *hwmgr,
-					const void *thermal_interrupt_info)
-{
-	int result;
-	const struct pp_interrupt_registration_info *info =
-			(const struct pp_interrupt_registration_info *)
-			thermal_interrupt_info;
-
-	if (info == NULL)
-		return -EINVAL;
-
-	result = cgs_add_irq_source(hwmgr->device, 230, AMD_THERMAL_IRQ_LAST,
-				fiji_dpm_set_interrupt_state,
-				info->call_back, info->context);
-
-	if (result)
-		return -EINVAL;
-
-	result = cgs_add_irq_source(hwmgr->device, 231, AMD_THERMAL_IRQ_LAST,
-				fiji_dpm_set_interrupt_state,
-				info->call_back, info->context);
-
-	if (result)
-		return -EINVAL;
-
-	return 0;
-}
-
-static int fiji_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
-{
-	if (mode) {
-		/* stop auto-manage */
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_MicrocodeFanControl))
-			fiji_fan_ctrl_stop_smc_fan_control(hwmgr);
-		fiji_fan_ctrl_set_static_mode(hwmgr, mode);
-	} else
-		/* restart auto-manage */
-		fiji_fan_ctrl_reset_fan_speed_to_default(hwmgr);
-
-	return 0;
-}
-
-static int fiji_get_fan_control_mode(struct pp_hwmgr *hwmgr)
-{
-	if (hwmgr->fan_ctrl_is_in_default_mode)
-		return hwmgr->fan_ctrl_default_mode;
-	else
-		return PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				CG_FDO_CTRL2, FDO_PWM_MODE);
-}
-
-static int fiji_force_clock_level(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, uint32_t mask)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if (hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL)
-		return -EINVAL;
-
-	switch (type) {
-	case PP_SCLK:
-		if (!data->sclk_dpm_key_disabled)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_SCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.sclk_dpm_enable_mask & mask);
-		break;
-
-	case PP_MCLK:
-		if (!data->mclk_dpm_key_disabled)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_MCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.mclk_dpm_enable_mask & mask);
-		break;
-
-	case PP_PCIE:
-	{
-		uint32_t tmp = mask & data->dpm_level_enable_mask.pcie_dpm_enable_mask;
-		uint32_t level = 0;
-
-		while (tmp >>= 1)
-			level++;
-
-		if (!data->pcie_dpm_key_disabled)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_PCIeDPM_ForceLevel,
-					level);
-		break;
-	}
-	default:
-		break;
-	}
-
-	return 0;
-}
-
-static int fiji_print_clock_levels(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, char *buf)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
-	struct fiji_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
-	struct fiji_single_dpm_table *pcie_table = &(data->dpm_table.pcie_speed_table);
-	int i, now, size = 0;
-	uint32_t clock, pcie_speed;
-
-	switch (type) {
-	case PP_SCLK:
-		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetSclkFrequency);
-		clock = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-
-		for (i = 0; i < sclk_table->count; i++) {
-			if (clock > sclk_table->dpm_levels[i].value)
-				continue;
-			break;
-		}
-		now = i;
-
-		for (i = 0; i < sclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, sclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
-		break;
-	case PP_MCLK:
-		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetMclkFrequency);
-		clock = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-
-		for (i = 0; i < mclk_table->count; i++) {
-			if (clock > mclk_table->dpm_levels[i].value)
-				continue;
-			break;
-		}
-		now = i;
-
-		for (i = 0; i < mclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, mclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
-		break;
-	case PP_PCIE:
-		pcie_speed = fiji_get_current_pcie_speed(hwmgr);
-		for (i = 0; i < pcie_table->count; i++) {
-			if (pcie_speed != pcie_table->dpm_levels[i].value)
-				continue;
-			break;
-		}
-		now = i;
-
-		for (i = 0; i < pcie_table->count; i++)
-			size += sprintf(buf + size, "%d: %s %s\n", i,
-					(pcie_table->dpm_levels[i].value == 0) ? "2.5GB, x1" :
-					(pcie_table->dpm_levels[i].value == 1) ? "5.0GB, x16" :
-					(pcie_table->dpm_levels[i].value == 2) ? "8.0GB, x16" : "",
-					(i == now) ? "*" : "");
-		break;
-	default:
-		break;
-	}
-	return size;
-}
-
-static inline bool fiji_are_power_levels_equal(const struct fiji_performance_level *pl1,
-							   const struct fiji_performance_level *pl2)
-{
-	return ((pl1->memory_clock == pl2->memory_clock) &&
-		  (pl1->engine_clock == pl2->engine_clock) &&
-		  (pl1->pcie_gen == pl2->pcie_gen) &&
-		  (pl1->pcie_lane == pl2->pcie_lane));
-}
-
-int fiji_check_states_equal(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *pstate1, const struct pp_hw_power_state *pstate2, bool *equal)
-{
-	const struct fiji_power_state *psa = cast_const_phw_fiji_power_state(pstate1);
-	const struct fiji_power_state *psb = cast_const_phw_fiji_power_state(pstate2);
-	int i;
-
-	if (equal == NULL || psa == NULL || psb == NULL)
-		return -EINVAL;
-
-	/* If the two states don't even have the same number of performance levels they cannot be the same state. */
-	if (psa->performance_level_count != psb->performance_level_count) {
-		*equal = false;
-		return 0;
-	}
-
-	for (i = 0; i < psa->performance_level_count; i++) {
-		if (!fiji_are_power_levels_equal(&(psa->performance_levels[i]), &(psb->performance_levels[i]))) {
-			/* If we have found even one performance level pair that is different the states are different. */
-			*equal = false;
-			return 0;
-		}
-	}
-
-	/* If all performance levels are the same try to use the UVD clocks to break the tie.*/
-	*equal = ((psa->uvd_clks.vclk == psb->uvd_clks.vclk) && (psa->uvd_clks.dclk == psb->uvd_clks.dclk));
-	*equal &= ((psa->vce_clks.evclk == psb->vce_clks.evclk) && (psa->vce_clks.ecclk == psb->vce_clks.ecclk));
-	*equal &= (psa->sclk_threshold == psb->sclk_threshold);
-	*equal &= (psa->acp_clk == psb->acp_clk);
-
-	return 0;
-}
-
-bool fiji_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	bool is_update_required = false;
-	struct cgs_display_info info = {0,0,NULL};
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	if (data->display_timing.num_existing_displays != info.display_count)
-		is_update_required = true;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep)) {
-		if(hwmgr->display_config.min_core_set_clock_in_sr != data->display_timing.min_clock_in_sr)
-			is_update_required = true;
-	}
-
-	return is_update_required;
-}
-
-static int fiji_get_sclk_od(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
-	struct fiji_single_dpm_table *golden_sclk_table =
-			&(data->golden_dpm_table.sclk_table);
-	int value;
-
-	value = (sclk_table->dpm_levels[sclk_table->count - 1].value -
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) *
-			100 /
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
-
-	return value;
-}
-
-static int fiji_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_single_dpm_table *golden_sclk_table =
-			&(data->golden_dpm_table.sclk_table);
-	struct pp_power_state  *ps;
-	struct fiji_power_state  *fiji_ps;
-
-	if (value > 20)
-		value = 20;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	fiji_ps = cast_phw_fiji_power_state(&ps->hardware);
-
-	fiji_ps->performance_levels[fiji_ps->performance_level_count - 1].engine_clock =
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value *
-			value / 100 +
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
-
-	return 0;
-}
-
-static int fiji_get_mclk_od(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
-	struct fiji_single_dpm_table *golden_mclk_table =
-			&(data->golden_dpm_table.mclk_table);
-	int value;
-
-	value = (mclk_table->dpm_levels[mclk_table->count - 1].value -
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) *
-			100 /
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
-
-	return value;
-}
-
-static int fiji_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct fiji_single_dpm_table *golden_mclk_table =
-			&(data->golden_dpm_table.mclk_table);
-	struct pp_power_state  *ps;
-	struct fiji_power_state  *fiji_ps;
-
-	if (value > 20)
-		value = 20;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	fiji_ps = cast_phw_fiji_power_state(&ps->hardware);
-
-	fiji_ps->performance_levels[fiji_ps->performance_level_count - 1].memory_clock =
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value *
-			value / 100 +
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
-
-	return 0;
-}
-
-static const struct pp_hwmgr_func fiji_hwmgr_funcs = {
-	.backend_init = &fiji_hwmgr_backend_init,
-	.backend_fini = &fiji_hwmgr_backend_fini,
-	.asic_setup = &fiji_setup_asic_task,
-	.dynamic_state_management_enable = &fiji_enable_dpm_tasks,
-	.dynamic_state_management_disable = &fiji_disable_dpm_tasks,
-	.force_dpm_level = &fiji_dpm_force_dpm_level,
-	.get_num_of_pp_table_entries = &tonga_get_number_of_powerplay_table_entries,
-	.get_power_state_size = &fiji_get_power_state_size,
-	.get_pp_table_entry = &fiji_get_pp_table_entry,
-	.patch_boot_state = &fiji_patch_boot_state,
-	.apply_state_adjust_rules = &fiji_apply_state_adjust_rules,
-	.power_state_set = &fiji_set_power_state_tasks,
-	.get_sclk = &fiji_dpm_get_sclk,
-	.get_mclk = &fiji_dpm_get_mclk,
-	.print_current_perforce_level = &fiji_print_current_perforce_level,
-	.powergate_uvd = &fiji_phm_powergate_uvd,
-	.powergate_vce = &fiji_phm_powergate_vce,
-	.disable_clock_power_gating = &fiji_phm_disable_clock_power_gating,
-	.notify_smc_display_config_after_ps_adjustment =
-			&tonga_notify_smc_display_config_after_ps_adjustment,
-	.display_config_changed = &fiji_display_configuration_changed_task,
-	.set_max_fan_pwm_output = fiji_set_max_fan_pwm_output,
-	.set_max_fan_rpm_output = fiji_set_max_fan_rpm_output,
-	.get_temperature = fiji_thermal_get_temperature,
-	.stop_thermal_controller = fiji_thermal_stop_thermal_controller,
-	.get_fan_speed_info = fiji_fan_ctrl_get_fan_speed_info,
-	.get_fan_speed_percent = fiji_fan_ctrl_get_fan_speed_percent,
-	.set_fan_speed_percent = fiji_fan_ctrl_set_fan_speed_percent,
-	.reset_fan_speed_to_default = fiji_fan_ctrl_reset_fan_speed_to_default,
-	.get_fan_speed_rpm = fiji_fan_ctrl_get_fan_speed_rpm,
-	.set_fan_speed_rpm = fiji_fan_ctrl_set_fan_speed_rpm,
-	.uninitialize_thermal_controller = fiji_thermal_ctrl_uninitialize_thermal_controller,
-	.register_internal_thermal_interrupt = fiji_register_internal_thermal_interrupt,
-	.set_fan_control_mode = fiji_set_fan_control_mode,
-	.get_fan_control_mode = fiji_get_fan_control_mode,
-	.check_states_equal = fiji_check_states_equal,
-	.check_smc_update_required_for_display_configuration = fiji_check_smc_update_required_for_display_configuration,
-	.force_clock_level = fiji_force_clock_level,
-	.print_clock_levels = fiji_print_clock_levels,
-	.get_sclk_od = fiji_get_sclk_od,
-	.set_sclk_od = fiji_set_sclk_od,
-	.get_mclk_od = fiji_get_mclk_od,
-	.set_mclk_od = fiji_set_mclk_od,
-};
-
-int fiji_hwmgr_init(struct pp_hwmgr *hwmgr)
-{
-	hwmgr->hwmgr_func = &fiji_hwmgr_funcs;
-	hwmgr->pptable_func = &tonga_pptable_funcs;
-	pp_fiji_thermal_initialize(hwmgr);
-	return 0;
-}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h
deleted file mode 100644
index bf67c2a92c68..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef _FIJI_HWMGR_H_
-#define _FIJI_HWMGR_H_
-
-#include "hwmgr.h"
-#include "smu73.h"
-#include "smu73_discrete.h"
-#include "ppatomctrl.h"
-#include "fiji_ppsmc.h"
-#include "pp_endian.h"
-
-#define FIJI_MAX_HARDWARE_POWERLEVELS	2
-#define FIJI_AT_DFLT	30
-
-#define FIJI_VOLTAGE_CONTROL_NONE                   0x0
-#define FIJI_VOLTAGE_CONTROL_BY_GPIO                0x1
-#define FIJI_VOLTAGE_CONTROL_BY_SVID2               0x2
-#define FIJI_VOLTAGE_CONTROL_MERGED                 0x3
-
-#define DPMTABLE_OD_UPDATE_SCLK     0x00000001
-#define DPMTABLE_OD_UPDATE_MCLK     0x00000002
-#define DPMTABLE_UPDATE_SCLK        0x00000004
-#define DPMTABLE_UPDATE_MCLK        0x00000008
-
-struct fiji_performance_level {
-	uint32_t  memory_clock;
-	uint32_t  engine_clock;
-	uint16_t  pcie_gen;
-	uint16_t  pcie_lane;
-};
-
-struct fiji_uvd_clocks {
-	uint32_t  vclk;
-	uint32_t  dclk;
-};
-
-struct fiji_vce_clocks {
-	uint32_t  evclk;
-	uint32_t  ecclk;
-};
-
-struct fiji_power_state {
-    uint32_t                  magic;
-    struct fiji_uvd_clocks    uvd_clks;
-    struct fiji_vce_clocks    vce_clks;
-    uint32_t                  sam_clk;
-    uint32_t                  acp_clk;
-    uint16_t                  performance_level_count;
-    bool                      dc_compatible;
-    uint32_t                  sclk_threshold;
-    struct fiji_performance_level  performance_levels[FIJI_MAX_HARDWARE_POWERLEVELS];
-};
-
-struct fiji_dpm_level {
-	bool	enabled;
-    uint32_t	value;
-    uint32_t	param1;
-};
-
-#define FIJI_MAX_DEEPSLEEP_DIVIDER_ID 5
-#define MAX_REGULAR_DPM_NUMBER 8
-#define FIJI_MINIMUM_ENGINE_CLOCK 2500
-
-struct fiji_single_dpm_table {
-	uint32_t		count;
-	struct fiji_dpm_level	dpm_levels[MAX_REGULAR_DPM_NUMBER];
-};
-
-struct fiji_dpm_table {
-	struct fiji_single_dpm_table  sclk_table;
-	struct fiji_single_dpm_table  mclk_table;
-	struct fiji_single_dpm_table  pcie_speed_table;
-	struct fiji_single_dpm_table  vddc_table;
-	struct fiji_single_dpm_table  vddci_table;
-	struct fiji_single_dpm_table  mvdd_table;
-};
-
-struct fiji_clock_registers {
-	uint32_t  vCG_SPLL_FUNC_CNTL;
-	uint32_t  vCG_SPLL_FUNC_CNTL_2;
-	uint32_t  vCG_SPLL_FUNC_CNTL_3;
-	uint32_t  vCG_SPLL_FUNC_CNTL_4;
-	uint32_t  vCG_SPLL_SPREAD_SPECTRUM;
-	uint32_t  vCG_SPLL_SPREAD_SPECTRUM_2;
-	uint32_t  vDLL_CNTL;
-	uint32_t  vMCLK_PWRMGT_CNTL;
-	uint32_t  vMPLL_AD_FUNC_CNTL;
-	uint32_t  vMPLL_DQ_FUNC_CNTL;
-	uint32_t  vMPLL_FUNC_CNTL;
-	uint32_t  vMPLL_FUNC_CNTL_1;
-	uint32_t  vMPLL_FUNC_CNTL_2;
-	uint32_t  vMPLL_SS1;
-	uint32_t  vMPLL_SS2;
-};
-
-struct fiji_voltage_smio_registers {
-	uint32_t vS0_VID_LOWER_SMIO_CNTL;
-};
-
-#define FIJI_MAX_LEAKAGE_COUNT  8
-struct fiji_leakage_voltage {
-	uint16_t  count;
-	uint16_t  leakage_id[FIJI_MAX_LEAKAGE_COUNT];
-	uint16_t  actual_voltage[FIJI_MAX_LEAKAGE_COUNT];
-};
-
-struct fiji_vbios_boot_state {
-	uint16_t    mvdd_bootup_value;
-	uint16_t    vddc_bootup_value;
-	uint16_t    vddci_bootup_value;
-	uint32_t    sclk_bootup_value;
-	uint32_t    mclk_bootup_value;
-	uint16_t    pcie_gen_bootup_value;
-	uint16_t    pcie_lane_bootup_value;
-};
-
-struct fiji_bacos {
-	uint32_t                       best_match;
-	uint32_t                       baco_flags;
-	struct fiji_performance_level  performance_level;
-};
-
-/* Ultra Low Voltage parameter structure */
-struct fiji_ulv_parm {
-	bool                           ulv_supported;
-	uint32_t                       cg_ulv_parameter;
-	uint32_t                       ulv_volt_change_delay;
-	struct fiji_performance_level  ulv_power_level;
-};
-
-struct fiji_display_timing {
-	uint32_t  min_clock_in_sr;
-	uint32_t  num_existing_displays;
-};
-
-struct fiji_dpmlevel_enable_mask {
-	uint32_t  uvd_dpm_enable_mask;
-	uint32_t  vce_dpm_enable_mask;
-	uint32_t  acp_dpm_enable_mask;
-	uint32_t  samu_dpm_enable_mask;
-	uint32_t  sclk_dpm_enable_mask;
-	uint32_t  mclk_dpm_enable_mask;
-	uint32_t  pcie_dpm_enable_mask;
-};
-
-struct fiji_pcie_perf_range {
-	uint16_t  max;
-	uint16_t  min;
-};
-
-struct fiji_hwmgr {
-	struct fiji_dpm_table			dpm_table;
-	struct fiji_dpm_table			golden_dpm_table;
-
-	uint32_t						voting_rights_clients0;
-	uint32_t						voting_rights_clients1;
-	uint32_t						voting_rights_clients2;
-	uint32_t						voting_rights_clients3;
-	uint32_t						voting_rights_clients4;
-	uint32_t						voting_rights_clients5;
-	uint32_t						voting_rights_clients6;
-	uint32_t						voting_rights_clients7;
-	uint32_t						static_screen_threshold_unit;
-	uint32_t						static_screen_threshold;
-	uint32_t						voltage_control;
-	uint32_t						vddc_vddci_delta;
-
-	uint32_t						active_auto_throttle_sources;
-
-	struct fiji_clock_registers            clock_registers;
-	struct fiji_voltage_smio_registers      voltage_smio_registers;
-
-	bool                           is_memory_gddr5;
-	uint16_t                       acpi_vddc;
-	bool                           pspp_notify_required;
-	uint16_t                       force_pcie_gen;
-	uint16_t                       acpi_pcie_gen;
-	uint32_t                       pcie_gen_cap;
-	uint32_t                       pcie_lane_cap;
-	uint32_t                       pcie_spc_cap;
-	struct fiji_leakage_voltage          vddc_leakage;
-	struct fiji_leakage_voltage          Vddci_leakage;
-
-	uint32_t                             mvdd_control;
-	uint32_t                             vddc_mask_low;
-	uint32_t                             mvdd_mask_low;
-	uint16_t                            max_vddc_in_pptable;
-	uint16_t                            min_vddc_in_pptable;
-	uint16_t                            max_vddci_in_pptable;
-	uint16_t                            min_vddci_in_pptable;
-	uint32_t                             mclk_strobe_mode_threshold;
-	uint32_t                             mclk_stutter_mode_threshold;
-	uint32_t                             mclk_edc_enable_threshold;
-	uint32_t                             mclk_edcwr_enable_threshold;
-	bool                                is_uvd_enabled;
-	struct fiji_vbios_boot_state        vbios_boot_state;
-
-	bool                           battery_state;
-	bool                           is_tlu_enabled;
-
-	/* ---- SMC SRAM Address of firmware header tables ---- */
-	uint32_t                             sram_end;
-	uint32_t                             dpm_table_start;
-	uint32_t                             soft_regs_start;
-	uint32_t                             mc_reg_table_start;
-	uint32_t                             fan_table_start;
-	uint32_t                             arb_table_start;
-	struct SMU73_Discrete_DpmTable       smc_state_table;
-	struct SMU73_Discrete_Ulv            ulv_setting;
-
-	/* ---- Stuff originally coming from Evergreen ---- */
-	uint32_t                             vddci_control;
-	struct pp_atomctrl_voltage_table     vddc_voltage_table;
-	struct pp_atomctrl_voltage_table     vddci_voltage_table;
-	struct pp_atomctrl_voltage_table     mvdd_voltage_table;
-
-	uint32_t                             mgcg_cgtt_local2;
-	uint32_t                             mgcg_cgtt_local3;
-	uint32_t                             gpio_debug;
-	uint32_t                             mc_micro_code_feature;
-	uint32_t                             highest_mclk;
-	uint16_t                             acpi_vddci;
-	uint8_t                              mvdd_high_index;
-	uint8_t                              mvdd_low_index;
-	bool                                 dll_default_on;
-	bool                                 performance_request_registered;
-
-	/* ---- Low Power Features ---- */
-	struct fiji_bacos                    bacos;
-	struct fiji_ulv_parm                 ulv;
-
-	/* ---- CAC Stuff ---- */
-	uint32_t                       cac_table_start;
-	bool                           cac_configuration_required;
-	bool                           driver_calculate_cac_leakage;
-	bool                           cac_enabled;
-
-	/* ---- DPM2 Parameters ---- */
-	uint32_t                       power_containment_features;
-	bool                           enable_dte_feature;
-	bool                           enable_tdc_limit_feature;
-	bool                           enable_pkg_pwr_tracking_feature;
-	bool                           disable_uvd_power_tune_feature;
-	const struct fiji_pt_defaults  *power_tune_defaults;
-	struct SMU73_Discrete_PmFuses  power_tune_table;
-	uint32_t                       dte_tj_offset;
-	uint32_t                       fast_watermark_threshold;
-
-	/* ---- Phase Shedding ---- */
-	bool                           vddc_phase_shed_control;
-
-	/* ---- DI/DT ---- */
-	struct fiji_display_timing        display_timing;
-
-	/* ---- Thermal Temperature Setting ---- */
-	struct fiji_dpmlevel_enable_mask     dpm_level_enable_mask;
-	uint32_t                             need_update_smu7_dpm_table;
-	uint32_t                             sclk_dpm_key_disabled;
-	uint32_t                             mclk_dpm_key_disabled;
-	uint32_t                             pcie_dpm_key_disabled;
-	uint32_t                             min_engine_clocks;
-	struct fiji_pcie_perf_range          pcie_gen_performance;
-	struct fiji_pcie_perf_range          pcie_lane_performance;
-	struct fiji_pcie_perf_range          pcie_gen_power_saving;
-	struct fiji_pcie_perf_range          pcie_lane_power_saving;
-	bool                                 use_pcie_performance_levels;
-	bool                                 use_pcie_power_saving_levels;
-	uint32_t                             activity_target[SMU73_MAX_LEVELS_GRAPHICS];
-	uint32_t                             mclk_activity_target;
-	uint32_t                             mclk_dpm0_activity_target;
-	uint32_t                             low_sclk_interrupt_threshold;
-	uint32_t                             last_mclk_dpm_enable_mask;
-	bool                                 uvd_enabled;
-
-	/* ---- Power Gating States ---- */
-	bool                           uvd_power_gated;
-	bool                           vce_power_gated;
-	bool                           samu_power_gated;
-	bool                           acp_power_gated;
-	bool                           pg_acp_init;
-	bool                           frtc_enabled;
-	bool                           frtc_status_changed;
-};
-
-/* To convert to Q8.8 format for firmware */
-#define FIJI_Q88_FORMAT_CONVERSION_UNIT             256
-
-enum Fiji_I2CLineID {
-    Fiji_I2CLineID_DDC1 = 0x90,
-    Fiji_I2CLineID_DDC2 = 0x91,
-    Fiji_I2CLineID_DDC3 = 0x92,
-    Fiji_I2CLineID_DDC4 = 0x93,
-    Fiji_I2CLineID_DDC5 = 0x94,
-    Fiji_I2CLineID_DDC6 = 0x95,
-    Fiji_I2CLineID_SCLSDA = 0x96,
-    Fiji_I2CLineID_DDCVGA = 0x97
-};
-
-#define Fiji_I2C_DDC1DATA          0
-#define Fiji_I2C_DDC1CLK           1
-#define Fiji_I2C_DDC2DATA          2
-#define Fiji_I2C_DDC2CLK           3
-#define Fiji_I2C_DDC3DATA          4
-#define Fiji_I2C_DDC3CLK           5
-#define Fiji_I2C_SDA               40
-#define Fiji_I2C_SCL               41
-#define Fiji_I2C_DDC4DATA          65
-#define Fiji_I2C_DDC4CLK           66
-#define Fiji_I2C_DDC5DATA          0x48
-#define Fiji_I2C_DDC5CLK           0x49
-#define Fiji_I2C_DDC6DATA          0x4a
-#define Fiji_I2C_DDC6CLK           0x4b
-#define Fiji_I2C_DDCVGADATA        0x4c
-#define Fiji_I2C_DDCVGACLK         0x4d
-
-#define FIJI_UNUSED_GPIO_PIN       0x7F
-
-extern int tonga_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr);
-extern int tonga_get_mc_microcode_version (struct pp_hwmgr *hwmgr);
-extern int tonga_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr);
-extern int tonga_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display);
-int fiji_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input);
-int fiji_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate);
-int fiji_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate);
-int fiji_update_acp_dpm(struct pp_hwmgr *hwmgr, bool bgate);
-int fiji_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable);
-
-#endif /* _FIJI_HWMGR_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c
deleted file mode 100644
index 44658451a8d2..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c
+++ /dev/null
@@ -1,613 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "hwmgr.h"
-#include "smumgr.h"
-#include "fiji_hwmgr.h"
-#include "fiji_powertune.h"
-#include "fiji_smumgr.h"
-#include "smu73_discrete.h"
-#include "pp_debug.h"
-
-#define VOLTAGE_SCALE  4
-#define POWERTUNE_DEFAULT_SET_MAX    1
-
-const struct fiji_pt_defaults fiji_power_tune_data_set_array[POWERTUNE_DEFAULT_SET_MAX] = {
-		/*sviLoadLIneEn,  SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc */
-		{1,               0xF,             0xFD,
-		/* TDC_MAWt, TdcWaterfallCtl, DTEAmbientTempBase */
-		0x19,        5,               45}
-};
-
-void fiji_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *fiji_hwmgr = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct  phm_ppt_v1_information *table_info =
-			(struct  phm_ppt_v1_information *)(hwmgr->pptable);
-	uint32_t tmp = 0;
-
-	if(table_info &&
-			table_info->cac_dtp_table->usPowerTuneDataSetID <= POWERTUNE_DEFAULT_SET_MAX &&
-			table_info->cac_dtp_table->usPowerTuneDataSetID)
-		fiji_hwmgr->power_tune_defaults =
-				&fiji_power_tune_data_set_array
-				[table_info->cac_dtp_table->usPowerTuneDataSetID - 1];
-	else
-		fiji_hwmgr->power_tune_defaults = &fiji_power_tune_data_set_array[0];
-
-	/* Assume disabled */
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PowerContainment);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_CAC);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SQRamping);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_DBRamping);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_TDRamping);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_TCPRamping);
-
-	fiji_hwmgr->dte_tj_offset = tmp;
-
-	if (!tmp) {
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_CAC);
-
-		fiji_hwmgr->fast_watermark_threshold = 100;
-
-		if (hwmgr->powercontainment_enabled) {
-			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				    PHM_PlatformCaps_PowerContainment);
-			tmp = 1;
-			fiji_hwmgr->enable_dte_feature = tmp ? false : true;
-			fiji_hwmgr->enable_tdc_limit_feature = tmp ? true : false;
-			fiji_hwmgr->enable_pkg_pwr_tracking_feature = tmp ? true : false;
-		}
-	}
-}
-
-/* PPGen has the gain setting generated in x * 100 unit
- * This function is to convert the unit to x * 4096(0x1000) unit.
- *  This is the unit expected by SMC firmware
- */
-static uint16_t scale_fan_gain_settings(uint16_t raw_setting)
-{
-	uint32_t tmp;
-	tmp = raw_setting * 4096 / 100;
-	return (uint16_t)tmp;
-}
-
-static void get_scl_sda_value(uint8_t line, uint8_t *scl, uint8_t* sda)
-{
-	switch (line) {
-	case Fiji_I2CLineID_DDC1 :
-		*scl = Fiji_I2C_DDC1CLK;
-		*sda = Fiji_I2C_DDC1DATA;
-		break;
-	case Fiji_I2CLineID_DDC2 :
-		*scl = Fiji_I2C_DDC2CLK;
-		*sda = Fiji_I2C_DDC2DATA;
-		break;
-	case Fiji_I2CLineID_DDC3 :
-		*scl = Fiji_I2C_DDC3CLK;
-		*sda = Fiji_I2C_DDC3DATA;
-		break;
-	case Fiji_I2CLineID_DDC4 :
-		*scl = Fiji_I2C_DDC4CLK;
-		*sda = Fiji_I2C_DDC4DATA;
-		break;
-	case Fiji_I2CLineID_DDC5 :
-		*scl = Fiji_I2C_DDC5CLK;
-		*sda = Fiji_I2C_DDC5DATA;
-		break;
-	case Fiji_I2CLineID_DDC6 :
-		*scl = Fiji_I2C_DDC6CLK;
-		*sda = Fiji_I2C_DDC6DATA;
-		break;
-	case Fiji_I2CLineID_SCLSDA :
-		*scl = Fiji_I2C_SCL;
-		*sda = Fiji_I2C_SDA;
-		break;
-	case Fiji_I2CLineID_DDCVGA :
-		*scl = Fiji_I2C_DDCVGACLK;
-		*sda = Fiji_I2C_DDCVGADATA;
-		break;
-	default:
-		*scl = 0;
-		*sda = 0;
-		break;
-	}
-}
-
-int fiji_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	const struct fiji_pt_defaults *defaults = data->power_tune_defaults;
-	SMU73_Discrete_DpmTable  *dpm_table = &(data->smc_state_table);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_cac_tdp_table *cac_dtp_table = table_info->cac_dtp_table;
-	struct pp_advance_fan_control_parameters *fan_table=
-			&hwmgr->thermal_controller.advanceFanControlParameters;
-	uint8_t uc_scl, uc_sda;
-
-	/* TDP number of fraction bits are changed from 8 to 7 for Fiji
-	 * as requested by SMC team
-	 */
-	dpm_table->DefaultTdp = PP_HOST_TO_SMC_US(
-			(uint16_t)(cac_dtp_table->usTDP * 128));
-	dpm_table->TargetTdp = PP_HOST_TO_SMC_US(
-			(uint16_t)(cac_dtp_table->usTDP * 128));
-
-	PP_ASSERT_WITH_CODE(cac_dtp_table->usTargetOperatingTemp <= 255,
-			"Target Operating Temp is out of Range!",);
-
-	dpm_table->GpuTjMax = (uint8_t)(cac_dtp_table->usTargetOperatingTemp);
-	dpm_table->GpuTjHyst = 8;
-
-	dpm_table->DTEAmbientTempBase = defaults->DTEAmbientTempBase;
-
-	/* The following are for new Fiji Multi-input fan/thermal control */
-	dpm_table->TemperatureLimitEdge = PP_HOST_TO_SMC_US(
-			cac_dtp_table->usTargetOperatingTemp * 256);
-	dpm_table->TemperatureLimitHotspot = PP_HOST_TO_SMC_US(
-			cac_dtp_table->usTemperatureLimitHotspot * 256);
-	dpm_table->TemperatureLimitLiquid1 = PP_HOST_TO_SMC_US(
-			cac_dtp_table->usTemperatureLimitLiquid1 * 256);
-	dpm_table->TemperatureLimitLiquid2 = PP_HOST_TO_SMC_US(
-			cac_dtp_table->usTemperatureLimitLiquid2 * 256);
-	dpm_table->TemperatureLimitVrVddc = PP_HOST_TO_SMC_US(
-			cac_dtp_table->usTemperatureLimitVrVddc * 256);
-	dpm_table->TemperatureLimitVrMvdd = PP_HOST_TO_SMC_US(
-			cac_dtp_table->usTemperatureLimitVrMvdd * 256);
-	dpm_table->TemperatureLimitPlx = PP_HOST_TO_SMC_US(
-			cac_dtp_table->usTemperatureLimitPlx * 256);
-
-	dpm_table->FanGainEdge = PP_HOST_TO_SMC_US(
-			scale_fan_gain_settings(fan_table->usFanGainEdge));
-	dpm_table->FanGainHotspot = PP_HOST_TO_SMC_US(
-			scale_fan_gain_settings(fan_table->usFanGainHotspot));
-	dpm_table->FanGainLiquid = PP_HOST_TO_SMC_US(
-			scale_fan_gain_settings(fan_table->usFanGainLiquid));
-	dpm_table->FanGainVrVddc = PP_HOST_TO_SMC_US(
-			scale_fan_gain_settings(fan_table->usFanGainVrVddc));
-	dpm_table->FanGainVrMvdd = PP_HOST_TO_SMC_US(
-			scale_fan_gain_settings(fan_table->usFanGainVrMvdd));
-	dpm_table->FanGainPlx = PP_HOST_TO_SMC_US(
-			scale_fan_gain_settings(fan_table->usFanGainPlx));
-	dpm_table->FanGainHbm = PP_HOST_TO_SMC_US(
-			scale_fan_gain_settings(fan_table->usFanGainHbm));
-
-	dpm_table->Liquid1_I2C_address = cac_dtp_table->ucLiquid1_I2C_address;
-	dpm_table->Liquid2_I2C_address = cac_dtp_table->ucLiquid2_I2C_address;
-	dpm_table->Vr_I2C_address = cac_dtp_table->ucVr_I2C_address;
-	dpm_table->Plx_I2C_address = cac_dtp_table->ucPlx_I2C_address;
-
-	get_scl_sda_value(cac_dtp_table->ucLiquid_I2C_Line, &uc_scl, &uc_sda);
-	dpm_table->Liquid_I2C_LineSCL = uc_scl;
-	dpm_table->Liquid_I2C_LineSDA = uc_sda;
-
-	get_scl_sda_value(cac_dtp_table->ucVr_I2C_Line, &uc_scl, &uc_sda);
-	dpm_table->Vr_I2C_LineSCL = uc_scl;
-	dpm_table->Vr_I2C_LineSDA = uc_sda;
-
-	get_scl_sda_value(cac_dtp_table->ucPlx_I2C_Line, &uc_scl, &uc_sda);
-	dpm_table->Plx_I2C_LineSCL = uc_scl;
-	dpm_table->Plx_I2C_LineSDA = uc_sda;
-
-	return 0;
-}
-
-static int fiji_populate_svi_load_line(struct pp_hwmgr *hwmgr)
-{
-    struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-    const struct fiji_pt_defaults *defaults = data->power_tune_defaults;
-
-    data->power_tune_table.SviLoadLineEn = defaults->SviLoadLineEn;
-    data->power_tune_table.SviLoadLineVddC = defaults->SviLoadLineVddC;
-    data->power_tune_table.SviLoadLineTrimVddC = 3;
-    data->power_tune_table.SviLoadLineOffsetVddC = 0;
-
-    return 0;
-}
-
-static int fiji_populate_tdc_limit(struct pp_hwmgr *hwmgr)
-{
-	uint16_t tdc_limit;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	const struct fiji_pt_defaults *defaults = data->power_tune_defaults;
-
-	/* TDC number of fraction bits are changed from 8 to 7
-	 * for Fiji as requested by SMC team
-	 */
-	tdc_limit = (uint16_t)(table_info->cac_dtp_table->usTDC * 128);
-	data->power_tune_table.TDC_VDDC_PkgLimit =
-			CONVERT_FROM_HOST_TO_SMC_US(tdc_limit);
-	data->power_tune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
-			defaults->TDC_VDDC_ThrottleReleaseLimitPerc;
-	data->power_tune_table.TDC_MAWt = defaults->TDC_MAWt;
-
-	return 0;
-}
-
-static int fiji_populate_dw8(struct pp_hwmgr *hwmgr, uint32_t fuse_table_offset)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	const struct fiji_pt_defaults *defaults = data->power_tune_defaults;
-	uint32_t temp;
-
-	if (fiji_read_smc_sram_dword(hwmgr->smumgr,
-			fuse_table_offset +
-			offsetof(SMU73_Discrete_PmFuses, TdcWaterfallCtl),
-			(uint32_t *)&temp, data->sram_end))
-		PP_ASSERT_WITH_CODE(false,
-				"Attempt to read PmFuses.DW6 (SviLoadLineEn) from SMC Failed!",
-				return -EINVAL);
-	else {
-		data->power_tune_table.TdcWaterfallCtl = defaults->TdcWaterfallCtl;
-		data->power_tune_table.LPMLTemperatureMin =
-				(uint8_t)((temp >> 16) & 0xff);
-		data->power_tune_table.LPMLTemperatureMax =
-				(uint8_t)((temp >> 8) & 0xff);
-		data->power_tune_table.Reserved = (uint8_t)(temp & 0xff);
-	}
-	return 0;
-}
-
-static int fiji_populate_temperature_scaler(struct pp_hwmgr *hwmgr)
-{
-	int i;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	/* Currently not used. Set all to zero. */
-	for (i = 0; i < 16; i++)
-		data->power_tune_table.LPMLTemperatureScaler[i] = 0;
-
-	return 0;
-}
-
-static int fiji_populate_fuzzy_fan(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if( (hwmgr->thermal_controller.advanceFanControlParameters.
-			usFanOutputSensitivity & (1 << 15)) ||
-			0 == hwmgr->thermal_controller.advanceFanControlParameters.
-			usFanOutputSensitivity )
-		hwmgr->thermal_controller.advanceFanControlParameters.
-		usFanOutputSensitivity = hwmgr->thermal_controller.
-			advanceFanControlParameters.usDefaultFanOutputSensitivity;
-
-	data->power_tune_table.FuzzyFan_PwmSetDelta =
-			PP_HOST_TO_SMC_US(hwmgr->thermal_controller.
-					advanceFanControlParameters.usFanOutputSensitivity);
-	return 0;
-}
-
-static int fiji_populate_gnb_lpml(struct pp_hwmgr *hwmgr)
-{
-	int i;
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	/* Currently not used. Set all to zero. */
-	for (i = 0; i < 16; i++)
-		data->power_tune_table.GnbLPML[i] = 0;
-
-	return 0;
-}
-
-static int fiji_min_max_vgnb_lpml_id_from_bapm_vddc(struct pp_hwmgr *hwmgr)
-{
- /*   int  i, min, max;
-    struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-    uint8_t * pHiVID = data->power_tune_table.BapmVddCVidHiSidd;
-    uint8_t * pLoVID = data->power_tune_table.BapmVddCVidLoSidd;
-
-    min = max = pHiVID[0];
-    for (i = 0; i < 8; i++) {
-        if (0 != pHiVID[i]) {
-            if (min > pHiVID[i])
-                min = pHiVID[i];
-            if (max < pHiVID[i])
-                max = pHiVID[i];
-        }
-
-        if (0 != pLoVID[i]) {
-            if (min > pLoVID[i])
-                min = pLoVID[i];
-            if (max < pLoVID[i])
-                max = pLoVID[i];
-        }
-    }
-
-    PP_ASSERT_WITH_CODE((0 != min) && (0 != max), "BapmVddcVidSidd table does not exist!", return int_Failed);
-    data->power_tune_table.GnbLPMLMaxVid = (uint8_t)max;
-    data->power_tune_table.GnbLPMLMinVid = (uint8_t)min;
-*/
-    return 0;
-}
-
-static int fiji_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint16_t HiSidd = data->power_tune_table.BapmVddCBaseLeakageHiSidd;
-	uint16_t LoSidd = data->power_tune_table.BapmVddCBaseLeakageLoSidd;
-	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
-
-	HiSidd = (uint16_t)(cac_table->usHighCACLeakage / 100 * 256);
-	LoSidd = (uint16_t)(cac_table->usLowCACLeakage / 100 * 256);
-
-	data->power_tune_table.BapmVddCBaseLeakageHiSidd =
-			CONVERT_FROM_HOST_TO_SMC_US(HiSidd);
-	data->power_tune_table.BapmVddCBaseLeakageLoSidd =
-			CONVERT_FROM_HOST_TO_SMC_US(LoSidd);
-
-	return 0;
-}
-
-int fiji_populate_pm_fuses(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	uint32_t pm_fuse_table_offset;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PowerContainment)) {
-		if (fiji_read_smc_sram_dword(hwmgr->smumgr,
-				SMU7_FIRMWARE_HEADER_LOCATION +
-				offsetof(SMU73_Firmware_Header, PmFuseTable),
-				&pm_fuse_table_offset, data->sram_end))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to get pm_fuse_table_offset Failed!",
-					return -EINVAL);
-
-		/* DW6 */
-		if (fiji_populate_svi_load_line(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate SviLoadLine Failed!",
-					return -EINVAL);
-		/* DW7 */
-		if (fiji_populate_tdc_limit(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate TDCLimit Failed!", return -EINVAL);
-		/* DW8 */
-		if (fiji_populate_dw8(hwmgr, pm_fuse_table_offset))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate TdcWaterfallCtl, "
-					"LPMLTemperature Min and Max Failed!",
-					return -EINVAL);
-
-		/* DW9-DW12 */
-		if (0 != fiji_populate_temperature_scaler(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate LPMLTemperatureScaler Failed!",
-					return -EINVAL);
-
-		/* DW13-DW14 */
-		if(fiji_populate_fuzzy_fan(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate Fuzzy Fan Control parameters Failed!",
-					return -EINVAL);
-
-		/* DW15-DW18 */
-		if (fiji_populate_gnb_lpml(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate GnbLPML Failed!",
-					return -EINVAL);
-
-		/* DW19 */
-		if (fiji_min_max_vgnb_lpml_id_from_bapm_vddc(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate GnbLPML Min and Max Vid Failed!",
-					return -EINVAL);
-
-		/* DW20 */
-		if (fiji_populate_bapm_vddc_base_leakage_sidd(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate BapmVddCBaseLeakage Hi and Lo "
-					"Sidd Failed!", return -EINVAL);
-
-		if (fiji_copy_bytes_to_smc(hwmgr->smumgr, pm_fuse_table_offset,
-				(uint8_t *)&data->power_tune_table,
-				sizeof(struct SMU73_Discrete_PmFuses), data->sram_end))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to download PmFuseTable Failed!",
-					return -EINVAL);
-	}
-	return 0;
-}
-
-int fiji_enable_smc_cac(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	int result = 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_CAC)) {
-		int smc_result;
-		smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
-				(uint16_t)(PPSMC_MSG_EnableCac));
-		PP_ASSERT_WITH_CODE((0 == smc_result),
-				"Failed to enable CAC in SMC.", result = -1);
-
-        data->cac_enabled = (0 == smc_result) ? true : false;
-	}
-	return result;
-}
-
-int fiji_disable_smc_cac(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	int result = 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_CAC) && data->cac_enabled) {
-		int smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
-				(uint16_t)(PPSMC_MSG_DisableCac));
-		PP_ASSERT_WITH_CODE((smc_result == 0),
-				"Failed to disable CAC in SMC.", result = -1);
-
-		data->cac_enabled = false;
-	}
-	return result;
-}
-
-int fiji_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-	if(data->power_containment_features &
-			POWERCONTAINMENT_FEATURE_PkgPwrLimit)
-		return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-				PPSMC_MSG_PkgPwrSetLimit, n);
-	return 0;
-}
-
-static int fiji_set_overdriver_target_tdp(struct pp_hwmgr *pHwMgr, uint32_t target_tdp)
-{
-	return smum_send_msg_to_smc_with_parameter(pHwMgr->smumgr,
-			PPSMC_MSG_OverDriveSetTargetTdp, target_tdp);
-}
-
-int fiji_enable_power_containment(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	int smc_result;
-	int result = 0;
-
-	data->power_containment_features = 0;
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PowerContainment)) {
-		if (data->enable_dte_feature) {
-			smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
-					(uint16_t)(PPSMC_MSG_EnableDTE));
-			PP_ASSERT_WITH_CODE((0 == smc_result),
-					"Failed to enable DTE in SMC.", result = -1;);
-			if (0 == smc_result)
-				data->power_containment_features |= POWERCONTAINMENT_FEATURE_DTE;
-		}
-
-		if (data->enable_tdc_limit_feature) {
-			smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
-					(uint16_t)(PPSMC_MSG_TDCLimitEnable));
-			PP_ASSERT_WITH_CODE((0 == smc_result),
-					"Failed to enable TDCLimit in SMC.", result = -1;);
-			if (0 == smc_result)
-				data->power_containment_features |=
-						POWERCONTAINMENT_FEATURE_TDCLimit;
-		}
-
-		if (data->enable_pkg_pwr_tracking_feature) {
-			smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
-					(uint16_t)(PPSMC_MSG_PkgPwrLimitEnable));
-			PP_ASSERT_WITH_CODE((0 == smc_result),
-					"Failed to enable PkgPwrTracking in SMC.", result = -1;);
-			if (0 == smc_result) {
-				struct phm_cac_tdp_table *cac_table =
-						table_info->cac_dtp_table;
-				uint32_t default_limit =
-					(uint32_t)(cac_table->usMaximumPowerDeliveryLimit * 256);
-
-				data->power_containment_features |=
-						POWERCONTAINMENT_FEATURE_PkgPwrLimit;
-
-				if (fiji_set_power_limit(hwmgr, default_limit))
-					printk(KERN_ERR "Failed to set Default Power Limit in SMC!");
-			}
-		}
-	}
-	return result;
-}
-
-int fiji_disable_power_containment(struct pp_hwmgr *hwmgr)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	int result = 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PowerContainment) &&
-			data->power_containment_features) {
-		int smc_result;
-
-		if (data->power_containment_features &
-				POWERCONTAINMENT_FEATURE_TDCLimit) {
-			smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
-					(uint16_t)(PPSMC_MSG_TDCLimitDisable));
-			PP_ASSERT_WITH_CODE((smc_result == 0),
-					"Failed to disable TDCLimit in SMC.",
-					result = smc_result);
-		}
-
-		if (data->power_containment_features &
-				POWERCONTAINMENT_FEATURE_DTE) {
-			smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
-					(uint16_t)(PPSMC_MSG_DisableDTE));
-			PP_ASSERT_WITH_CODE((smc_result == 0),
-					"Failed to disable DTE in SMC.",
-					result = smc_result);
-		}
-
-		if (data->power_containment_features &
-				POWERCONTAINMENT_FEATURE_PkgPwrLimit) {
-			smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
-					(uint16_t)(PPSMC_MSG_PkgPwrLimitDisable));
-			PP_ASSERT_WITH_CODE((smc_result == 0),
-					"Failed to disable PkgPwrTracking in SMC.",
-					result = smc_result);
-		}
-		data->power_containment_features = 0;
-	}
-
-	return result;
-}
-
-int fiji_power_control_set_level(struct pp_hwmgr *hwmgr)
-{
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
-	int adjust_percent, target_tdp;
-	int result = 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PowerContainment)) {
-		/* adjustment percentage has already been validated */
-		adjust_percent = hwmgr->platform_descriptor.TDPAdjustmentPolarity ?
-				hwmgr->platform_descriptor.TDPAdjustment :
-				(-1 * hwmgr->platform_descriptor.TDPAdjustment);
-		/* SMC requested that target_tdp to be 7 bit fraction in DPM table
-		 * but message to be 8 bit fraction for messages
-		 */
-		target_tdp = ((100 + adjust_percent) * (int)(cac_table->usTDP * 256)) / 100;
-		result = fiji_set_overdriver_target_tdp(hwmgr, (uint32_t)target_tdp);
-	}
-
-	return result;
-}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h
deleted file mode 100644
index fec772421733..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#ifndef FIJI_POWERTUNE_H
-#define FIJI_POWERTUNE_H
-
-enum fiji_pt_config_reg_type {
-	FIJI_CONFIGREG_MMR = 0,
-	FIJI_CONFIGREG_SMC_IND,
-	FIJI_CONFIGREG_DIDT_IND,
-	FIJI_CONFIGREG_CACHE,
-	FIJI_CONFIGREG_MAX
-};
-
-/* PowerContainment Features */
-#define POWERCONTAINMENT_FEATURE_DTE             0x00000001
-#define POWERCONTAINMENT_FEATURE_TDCLimit        0x00000002
-#define POWERCONTAINMENT_FEATURE_PkgPwrLimit     0x00000004
-
-#define DIDT_SQ_CTRL0__UNUSED_0_MASK             0xffffffc0
-#define DIDT_SQ_CTRL0__UNUSED_0__SHIFT           0x6
-#define DIDT_TD_CTRL0__UNUSED_0_MASK             0xffffffc0
-#define DIDT_TD_CTRL0__UNUSED_0__SHIFT           0x6
-#define DIDT_TCP_CTRL0__UNUSED_0_MASK            0xffffffc0
-#define DIDT_TCP_CTRL0__UNUSED_0__SHIFT          0x6
-#define DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK                 0xe0000000
-#define DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT               0x0000001d
-#define DIDT_TD_TUNING_CTRL__UNUSED_0_MASK                 0xe0000000
-#define DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT               0x0000001d
-#define DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK                0xe0000000
-#define DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT              0x0000001d
-
-struct fiji_pt_config_reg {
-	uint32_t                           offset;
-	uint32_t                           mask;
-	uint32_t                           shift;
-	uint32_t                           value;
-	enum fiji_pt_config_reg_type       type;
-};
-
-struct fiji_pt_defaults
-{
-    uint8_t   SviLoadLineEn;
-    uint8_t   SviLoadLineVddC;
-    uint8_t   TDC_VDDC_ThrottleReleaseLimitPerc;
-    uint8_t   TDC_MAWt;
-    uint8_t   TdcWaterfallCtl;
-    uint8_t   DTEAmbientTempBase;
-};
-
-void fiji_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr);
-int fiji_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr);
-int fiji_populate_pm_fuses(struct pp_hwmgr *hwmgr);
-int fiji_enable_smc_cac(struct pp_hwmgr *hwmgr);
-int fiji_disable_smc_cac(struct pp_hwmgr *hwmgr);
-int fiji_enable_power_containment(struct pp_hwmgr *hwmgr);
-int fiji_disable_power_containment(struct pp_hwmgr *hwmgr);
-int fiji_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n);
-int fiji_power_control_set_level(struct pp_hwmgr *hwmgr);
-
-#endif  /* FIJI_POWERTUNE_H */
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.h
deleted file mode 100644
index 8621493b8574..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef FIJI_THERMAL_H
-#define FIJI_THERMAL_H
-
-#include "hwmgr.h"
-
-#define FIJI_THERMAL_HIGH_ALERT_MASK         0x1
-#define FIJI_THERMAL_LOW_ALERT_MASK          0x2
-
-#define FIJI_THERMAL_MINIMUM_TEMP_READING    -256
-#define FIJI_THERMAL_MAXIMUM_TEMP_READING    255
-
-#define FIJI_THERMAL_MINIMUM_ALERT_TEMP      0
-#define FIJI_THERMAL_MAXIMUM_ALERT_TEMP      255
-
-#define FDO_PWM_MODE_STATIC  1
-#define FDO_PWM_MODE_STATIC_RPM 5
-
-
-extern int tf_fiji_thermal_initialize(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
-extern int tf_fiji_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
-extern int tf_fiji_thermal_enable_alert(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
-
-extern int fiji_thermal_get_temperature(struct pp_hwmgr *hwmgr);
-extern int fiji_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr);
-extern int fiji_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info);
-extern int fiji_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t *speed);
-extern int fiji_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr);
-extern int fiji_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode);
-extern int fiji_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed);
-extern int fiji_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr);
-extern int pp_fiji_thermal_initialize(struct pp_hwmgr *hwmgr);
-extern int fiji_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr);
-extern int fiji_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed);
-extern int fiji_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed);
-extern int fiji_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr);
-extern uint32_t tonga_get_xclk(struct pp_hwmgr *hwmgr);
-
-#endif
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
index 789f98ad2615..14f8c1f4da3d 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
@@ -24,8 +24,6 @@
 #include "hwmgr.h"
 #include "hardwaremanager.h"
 #include "power_state.h"
-#include "pp_acpi.h"
-#include "amd_acpi.h"
 #include "pp_debug.h"
 
 #define PHM_FUNC_CHECK(hw) \
@@ -34,38 +32,6 @@
 			return -EINVAL;				\
 	} while (0)
 
-void phm_init_dynamic_caps(struct pp_hwmgr *hwmgr)
-{
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableVoltageTransition);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableEngineTransition);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMemoryTransition);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMGClockGating);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMGCGTSSM);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableLSClockGating);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_Force3DClockSupport);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableLightSleep);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMCLS);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisablePowerGating);
-
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableDPM);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableSMUUVDHandshake);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_ThermalAutoThrottling);
-
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest);
-
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_NoOD5Support);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_UserMaxClockForMultiDisplays);
-
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_VpuRecoveryInProgress);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_UVDDPM);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_VCEDPM);
-
-	if (acpi_atcs_functions_supported(hwmgr->device, ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST) &&
-		acpi_atcs_functions_supported(hwmgr->device, ATCS_FUNCTION_PCIE_DEVICE_READY_NOTIFICATION))
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest);
-}
-
 bool phm_is_hw_access_blocked(struct pp_hwmgr *hwmgr)
 {
 	return hwmgr->block_hw_access;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
index 27e07624ac28..1167205057b3 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -32,13 +32,22 @@
 #include "pp_debug.h"
 #include "ppatomctrl.h"
 #include "ppsmc.h"
-
-#define VOLTAGE_SCALE               4
+#include "pp_acpi.h"
+#include "amd_acpi.h"
 
 extern int cz_hwmgr_init(struct pp_hwmgr *hwmgr);
-extern int tonga_hwmgr_init(struct pp_hwmgr *hwmgr);
-extern int fiji_hwmgr_init(struct pp_hwmgr *hwmgr);
-extern int polaris10_hwmgr_init(struct pp_hwmgr *hwmgr);
+
+static int polaris_set_asic_special_caps(struct pp_hwmgr *hwmgr);
+static void hwmgr_init_default_caps(struct pp_hwmgr *hwmgr);
+static int hwmgr_set_user_specify_caps(struct pp_hwmgr *hwmgr);
+static int fiji_set_asic_special_caps(struct pp_hwmgr *hwmgr);
+static int tonga_set_asic_special_caps(struct pp_hwmgr *hwmgr);
+static int topaz_set_asic_special_caps(struct pp_hwmgr *hwmgr);
+
+uint8_t convert_to_vid(uint16_t vddc)
+{
+	return (uint8_t) ((6200 - (vddc * VOLTAGE_SCALE)) / 25);
+}
 
 int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
 {
@@ -56,10 +65,12 @@ int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
 	hwmgr->device = pp_init->device;
 	hwmgr->chip_family = pp_init->chip_family;
 	hwmgr->chip_id = pp_init->chip_id;
-	hwmgr->hw_revision = pp_init->rev_id;
 	hwmgr->usec_timeout = AMD_MAX_USEC_TIMEOUT;
 	hwmgr->power_source = PP_PowerSource_AC;
-	hwmgr->powercontainment_enabled = pp_init->powercontainment_enabled;
+	hwmgr->pp_table_version = PP_TABLE_V1;
+
+	hwmgr_init_default_caps(hwmgr);
+	hwmgr_set_user_specify_caps(hwmgr);
 
 	switch (hwmgr->chip_family) {
 	case AMDGPU_FAMILY_CZ:
@@ -67,26 +78,38 @@ int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
 		break;
 	case AMDGPU_FAMILY_VI:
 		switch (hwmgr->chip_id) {
+		case CHIP_TOPAZ:
+			topaz_set_asic_special_caps(hwmgr);
+			hwmgr->feature_mask &= ~(PP_SMC_VOLTAGE_CONTROL_MASK |
+						PP_VBI_TIME_SUPPORT_MASK |
+						PP_ENABLE_GFX_CG_THRU_SMU);
+			hwmgr->pp_table_version = PP_TABLE_V0;
+			break;
 		case CHIP_TONGA:
-			tonga_hwmgr_init(hwmgr);
+			tonga_set_asic_special_caps(hwmgr);
+			hwmgr->feature_mask &= ~(PP_SMC_VOLTAGE_CONTROL_MASK |
+						PP_VBI_TIME_SUPPORT_MASK);
 			break;
 		case CHIP_FIJI:
-			fiji_hwmgr_init(hwmgr);
+			fiji_set_asic_special_caps(hwmgr);
+			hwmgr->feature_mask &= ~(PP_SMC_VOLTAGE_CONTROL_MASK |
+						PP_VBI_TIME_SUPPORT_MASK |
+						PP_ENABLE_GFX_CG_THRU_SMU);
 			break;
 		case CHIP_POLARIS11:
 		case CHIP_POLARIS10:
-			polaris10_hwmgr_init(hwmgr);
+			polaris_set_asic_special_caps(hwmgr);
+			hwmgr->feature_mask &= ~(PP_UVD_HANDSHAKE_MASK);
 			break;
 		default:
 			return -EINVAL;
 		}
+		smu7_hwmgr_init(hwmgr);
 		break;
 	default:
 		return -EINVAL;
 	}
 
-	phm_init_dynamic_caps(hwmgr);
-
 	return 0;
 }
 
@@ -105,6 +128,8 @@ int hwmgr_fini(struct pp_hwmgr *hwmgr)
 	kfree(hwmgr->set_temperature_range.function_list);
 
 	kfree(hwmgr->ps);
+	kfree(hwmgr->current_ps);
+	kfree(hwmgr->request_ps);
 	kfree(hwmgr);
 	return 0;
 }
@@ -129,10 +154,17 @@ int hw_init_power_state_table(struct pp_hwmgr *hwmgr)
 					  sizeof(struct pp_power_state);
 
 	hwmgr->ps = kzalloc(size * table_entries, GFP_KERNEL);
-
 	if (hwmgr->ps == NULL)
 		return -ENOMEM;
 
+	hwmgr->request_ps = kzalloc(size, GFP_KERNEL);
+	if (hwmgr->request_ps == NULL)
+		return -ENOMEM;
+
+	hwmgr->current_ps = kzalloc(size, GFP_KERNEL);
+	if (hwmgr->current_ps == NULL)
+		return -ENOMEM;
+
 	state = hwmgr->ps;
 
 	for (i = 0; i < table_entries; i++) {
@@ -140,7 +172,8 @@ int hw_init_power_state_table(struct pp_hwmgr *hwmgr)
 
 		if (state->classification.flags & PP_StateClassificationFlag_Boot) {
 			hwmgr->boot_ps = state;
-			hwmgr->current_ps = hwmgr->request_ps = state;
+			memcpy(hwmgr->current_ps, state, size);
+			memcpy(hwmgr->request_ps, state, size);
 		}
 
 		state->id = i + 1; /* assigned unique num for every power state id */
@@ -150,6 +183,7 @@ int hw_init_power_state_table(struct pp_hwmgr *hwmgr)
 		state = (struct pp_power_state *)((unsigned long)state + size);
 	}
 
+
 	return 0;
 }
 
@@ -182,30 +216,6 @@ int phm_wait_on_register(struct pp_hwmgr *hwmgr, uint32_t index,
 	return 0;
 }
 
-int phm_wait_for_register_unequal(struct pp_hwmgr *hwmgr,
-				uint32_t index, uint32_t value, uint32_t mask)
-{
-	uint32_t i;
-	uint32_t cur_value;
-
-	if (hwmgr == NULL || hwmgr->device == NULL) {
-		printk(KERN_ERR "[ powerplay ] Invalid Hardware Manager!");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < hwmgr->usec_timeout; i++) {
-		cur_value = cgs_read_register(hwmgr->device, index);
-		if ((cur_value & mask) != (value & mask))
-			break;
-		udelay(1);
-	}
-
-	/* timeout means wrong logic*/
-	if (i == hwmgr->usec_timeout)
-		return -1;
-	return 0;
-}
-
 
 /**
  * Returns once the part of the register indicated by the mask has
@@ -227,21 +237,7 @@ void phm_wait_on_indirect_register(struct pp_hwmgr *hwmgr,
 	phm_wait_on_register(hwmgr, indirect_port + 1, mask, value);
 }
 
-void phm_wait_for_indirect_register_unequal(struct pp_hwmgr *hwmgr,
-					uint32_t indirect_port,
-					uint32_t index,
-					uint32_t value,
-					uint32_t mask)
-{
-	if (hwmgr == NULL || hwmgr->device == NULL) {
-		printk(KERN_ERR "[ powerplay ] Invalid Hardware Manager!");
-		return;
-	}
 
-	cgs_write_register(hwmgr->device, indirect_port, index);
-	phm_wait_for_register_unequal(hwmgr, indirect_port + 1,
-				      value, mask);
-}
 
 bool phm_cf_want_uvd_power_gating(struct pp_hwmgr *hwmgr)
 {
@@ -403,12 +399,9 @@ int phm_reset_single_dpm_table(void *table,
 
 	struct vi_dpm_table *dpm_table = (struct vi_dpm_table *)table;
 
-	PP_ASSERT_WITH_CODE(count <= max,
-			"Fatal error, can not set up single DPM table entries to exceed max number!",
-			   );
+	dpm_table->count = count > max ? max : count;
 
-	dpm_table->count = count;
-	for (i = 0; i < max; i++)
+	for (i = 0; i < dpm_table->count; i++)
 		dpm_table->dpm_level[i].enabled = false;
 
 	return 0;
@@ -462,6 +455,27 @@ uint8_t phm_get_voltage_index(
 	return i - 1;
 }
 
+uint8_t phm_get_voltage_id(pp_atomctrl_voltage_table *voltage_table,
+		uint32_t voltage)
+{
+	uint8_t count = (uint8_t) (voltage_table->count);
+	uint8_t i = 0;
+
+	PP_ASSERT_WITH_CODE((NULL != voltage_table),
+		"Voltage Table empty.", return 0;);
+	PP_ASSERT_WITH_CODE((0 != count),
+		"Voltage Table empty.", return 0;);
+
+	for (i = 0; i < count; i++) {
+		/* find first voltage bigger than requested */
+		if (voltage_table->entries[i].value >= voltage)
+			return i;
+	}
+
+	/* voltage is bigger than max voltage in the table */
+	return i - 1;
+}
+
 uint16_t phm_find_closest_vddci(struct pp_atomctrl_voltage_table *vddci_table, uint16_t vddci)
 {
 	uint32_t  i;
@@ -549,7 +563,8 @@ int phm_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr
 		table_clk_vlt->entries[2].v = 810;
 		table_clk_vlt->entries[3].clk = PP_DAL_POWERLEVEL_PERFORMANCE;
 		table_clk_vlt->entries[3].v = 900;
-		pptable_info->vddc_dep_on_dal_pwrl = table_clk_vlt;
+		if (pptable_info != NULL)
+			pptable_info->vddc_dep_on_dal_pwrl = table_clk_vlt;
 		hwmgr->dyn_state.vddc_dep_on_dal_pwrl = table_clk_vlt;
 	}
 
@@ -615,3 +630,186 @@ void phm_apply_dal_min_voltage_request(struct pp_hwmgr *hwmgr)
 	printk(KERN_ERR "DAL requested level can not"
 			" found a available voltage in VDDC DPM Table \n");
 }
+
+void hwmgr_init_default_caps(struct pp_hwmgr *hwmgr)
+{
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableVoltageTransition);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableEngineTransition);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMemoryTransition);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMGClockGating);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMGCGTSSM);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableLSClockGating);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_Force3DClockSupport);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableLightSleep);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMCLS);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisablePowerGating);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableDPM);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableSMUUVDHandshake);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_ThermalAutoThrottling);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_NoOD5Support);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_UserMaxClockForMultiDisplays);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_VpuRecoveryInProgress);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_UVDDPM);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_VCEDPM);
+
+	if (acpi_atcs_functions_supported(hwmgr->device, ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST) &&
+		acpi_atcs_functions_supported(hwmgr->device, ATCS_FUNCTION_PCIE_DEVICE_READY_NOTIFICATION))
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_DynamicPatchPowerState);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_EnableSMU7ThermalManagement);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DynamicPowerManagement);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_SMC);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_DynamicUVDState);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_FanSpeedInTableIsRPM);
+
+	return;
+}
+
+int hwmgr_set_user_specify_caps(struct pp_hwmgr *hwmgr)
+{
+	if (amdgpu_sclk_deep_sleep_en)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkDeepSleep);
+	else
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkDeepSleep);
+
+	if (amdgpu_powercontainment)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			    PHM_PlatformCaps_PowerContainment);
+	else
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			    PHM_PlatformCaps_PowerContainment);
+
+	hwmgr->feature_mask = amdgpu_pp_feature_mask;
+
+	return 0;
+}
+
+int phm_get_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type,
+				uint32_t sclk, uint16_t id, uint16_t *voltage)
+{
+	uint32_t vol;
+	int ret = 0;
+
+	if (hwmgr->chip_id < CHIP_POLARIS10) {
+		atomctrl_get_voltage_evv_on_sclk(hwmgr, voltage_type, sclk, id, voltage);
+		if (*voltage >= 2000 || *voltage == 0)
+			*voltage = 1150;
+	} else {
+		ret = atomctrl_get_voltage_evv_on_sclk_ai(hwmgr, voltage_type, sclk, id, &vol);
+		*voltage = (uint16_t)vol/100;
+	}
+	return ret;
+}
+
+int polaris_set_asic_special_caps(struct pp_hwmgr *hwmgr)
+{
+	/* power tune caps Assume disabled */
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_SQRamping);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_DBRamping);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_TDRamping);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_TCPRamping);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+							PHM_PlatformCaps_CAC);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_RegulatorHot);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_AutomaticDCTransition);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_TablelessHardwareInterface);
+
+	if (hwmgr->chip_id == CHIP_POLARIS11)
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_SPLLShutdownSupport);
+	return 0;
+}
+
+int fiji_set_asic_special_caps(struct pp_hwmgr *hwmgr)
+{
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SQRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DBRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TDRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TCPRamping);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TablelessHardwareInterface);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_CAC);
+	return 0;
+}
+
+int tonga_set_asic_special_caps(struct pp_hwmgr *hwmgr)
+{
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SQRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DBRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TDRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TCPRamping);
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+		      PHM_PlatformCaps_UVDPowerGating);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+		      PHM_PlatformCaps_VCEPowerGating);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			 PHM_PlatformCaps_TablelessHardwareInterface);
+
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_CAC);
+
+	return 0;
+}
+
+int topaz_set_asic_special_caps(struct pp_hwmgr *hwmgr)
+{
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SQRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DBRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TDRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_TCPRamping);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			 PHM_PlatformCaps_TablelessHardwareInterface);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_CAC);
+	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+		    PHM_PlatformCaps_EVV);
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_dyn_defaults.h b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_dyn_defaults.h
deleted file mode 100644
index f78ffd935cee..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_dyn_defaults.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef POLARIS10_DYN_DEFAULTS_H
-#define POLARIS10_DYN_DEFAULTS_H
-
-
-enum Polaris10dpm_TrendDetection {
-	Polaris10Adpm_TrendDetection_AUTO,
-	Polaris10Adpm_TrendDetection_UP,
-	Polaris10Adpm_TrendDetection_DOWN
-};
-typedef enum Polaris10dpm_TrendDetection Polaris10dpm_TrendDetection;
-
-/*  We need to fill in the default values */
-
-
-#define PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT0              0x3FFFC102
-#define PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT1              0x000400
-#define PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT2              0xC00080
-#define PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT3              0xC00200
-#define PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT4              0xC01680
-#define PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT5              0xC00033
-#define PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT6              0xC00033
-#define PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT7              0x3FFFC000
-
-
-#define PPPOLARIS10_THERMALPROTECTCOUNTER_DFLT            0x200
-#define PPPOLARIS10_STATICSCREENTHRESHOLDUNIT_DFLT        0
-#define PPPOLARIS10_STATICSCREENTHRESHOLD_DFLT            0x00C8
-#define PPPOLARIS10_GFXIDLECLOCKSTOPTHRESHOLD_DFLT        0x200
-#define PPPOLARIS10_REFERENCEDIVIDER_DFLT                  4
-
-#define PPPOLARIS10_ULVVOLTAGECHANGEDELAY_DFLT             1687
-
-#define PPPOLARIS10_CGULVPARAMETER_DFLT                    0x00040035
-#define PPPOLARIS10_CGULVCONTROL_DFLT                      0x00007450
-#define PPPOLARIS10_TARGETACTIVITY_DFLT                     50
-#define PPPOLARIS10_MCLK_TARGETACTIVITY_DFLT                10
-
-#endif
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c
deleted file mode 100644
index 769636a0c5b5..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c
+++ /dev/null
@@ -1,5290 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/fb.h>
-#include <asm/div64.h>
-#include "linux/delay.h"
-#include "pp_acpi.h"
-#include "hwmgr.h"
-#include "polaris10_hwmgr.h"
-#include "polaris10_powertune.h"
-#include "polaris10_dyn_defaults.h"
-#include "polaris10_smumgr.h"
-#include "pp_debug.h"
-#include "ppatomctrl.h"
-#include "atombios.h"
-#include "tonga_pptable.h"
-#include "pppcielanes.h"
-#include "amd_pcie_helpers.h"
-#include "hardwaremanager.h"
-#include "tonga_processpptables.h"
-#include "cgs_common.h"
-#include "smu74.h"
-#include "smu_ucode_xfer_vi.h"
-#include "smu74_discrete.h"
-#include "smu/smu_7_1_3_d.h"
-#include "smu/smu_7_1_3_sh_mask.h"
-#include "gmc/gmc_8_1_d.h"
-#include "gmc/gmc_8_1_sh_mask.h"
-#include "oss/oss_3_0_d.h"
-#include "gca/gfx_8_0_d.h"
-#include "bif/bif_5_0_d.h"
-#include "bif/bif_5_0_sh_mask.h"
-#include "gmc/gmc_8_1_d.h"
-#include "gmc/gmc_8_1_sh_mask.h"
-#include "bif/bif_5_0_d.h"
-#include "bif/bif_5_0_sh_mask.h"
-#include "dce/dce_10_0_d.h"
-#include "dce/dce_10_0_sh_mask.h"
-
-#include "polaris10_thermal.h"
-#include "polaris10_clockpowergating.h"
-
-#define MC_CG_ARB_FREQ_F0           0x0a
-#define MC_CG_ARB_FREQ_F1           0x0b
-#define MC_CG_ARB_FREQ_F2           0x0c
-#define MC_CG_ARB_FREQ_F3           0x0d
-
-#define MC_CG_SEQ_DRAMCONF_S0       0x05
-#define MC_CG_SEQ_DRAMCONF_S1       0x06
-#define MC_CG_SEQ_YCLK_SUSPEND      0x04
-#define MC_CG_SEQ_YCLK_RESUME       0x0a
-
-
-#define SMC_RAM_END 0x40000
-
-#define SMC_CG_IND_START            0xc0030000
-#define SMC_CG_IND_END              0xc0040000
-
-#define VOLTAGE_SCALE               4
-#define VOLTAGE_VID_OFFSET_SCALE1   625
-#define VOLTAGE_VID_OFFSET_SCALE2   100
-
-#define VDDC_VDDCI_DELTA            200
-
-#define MEM_FREQ_LOW_LATENCY        25000
-#define MEM_FREQ_HIGH_LATENCY       80000
-
-#define MEM_LATENCY_HIGH            45
-#define MEM_LATENCY_LOW             35
-#define MEM_LATENCY_ERR             0xFFFF
-
-#define MC_SEQ_MISC0_GDDR5_SHIFT 28
-#define MC_SEQ_MISC0_GDDR5_MASK  0xf0000000
-#define MC_SEQ_MISC0_GDDR5_VALUE 5
-
-
-#define PCIE_BUS_CLK                10000
-#define TCLK                        (PCIE_BUS_CLK / 10)
-
-
-static const uint16_t polaris10_clock_stretcher_lookup_table[2][4] =
-{ {600, 1050, 3, 0}, {600, 1050, 6, 1} };
-
-/*  [FF, SS] type, [] 4 voltage ranges, and [Floor Freq, Boundary Freq, VID min , VID max] */
-static const uint32_t polaris10_clock_stretcher_ddt_table[2][4][4] =
-{ { {265, 529, 120, 128}, {325, 650, 96, 119}, {430, 860, 32, 95}, {0, 0, 0, 31} },
-  { {275, 550, 104, 112}, {319, 638, 96, 103}, {360, 720, 64, 95}, {384, 768, 32, 63} } };
-
-/*  [Use_For_Low_freq] value, [0%, 5%, 10%, 7.14%, 14.28%, 20%] (coming from PWR_CKS_CNTL.stretch_amount reg spec) */
-static const uint8_t polaris10_clock_stretch_amount_conversion[2][6] =
-{ {0, 1, 3, 2, 4, 5}, {0, 2, 4, 5, 6, 5} };
-
-/** Values for the CG_THERMAL_CTRL::DPM_EVENT_SRC field. */
-enum DPM_EVENT_SRC {
-	DPM_EVENT_SRC_ANALOG = 0,
-	DPM_EVENT_SRC_EXTERNAL = 1,
-	DPM_EVENT_SRC_DIGITAL = 2,
-	DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,
-	DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL = 4
-};
-
-static const unsigned long PhwPolaris10_Magic = (unsigned long)(PHM_VIslands_Magic);
-
-struct polaris10_power_state *cast_phw_polaris10_power_state(
-				  struct pp_hw_power_state *hw_ps)
-{
-	PP_ASSERT_WITH_CODE((PhwPolaris10_Magic == hw_ps->magic),
-				"Invalid Powerstate Type!",
-				 return NULL);
-
-	return (struct polaris10_power_state *)hw_ps;
-}
-
-const struct polaris10_power_state *cast_const_phw_polaris10_power_state(
-				 const struct pp_hw_power_state *hw_ps)
-{
-	PP_ASSERT_WITH_CODE((PhwPolaris10_Magic == hw_ps->magic),
-				"Invalid Powerstate Type!",
-				 return NULL);
-
-	return (const struct polaris10_power_state *)hw_ps;
-}
-
-static bool polaris10_is_dpm_running(struct pp_hwmgr *hwmgr)
-{
-	return (1 == PHM_READ_INDIRECT_FIELD(hwmgr->device,
-			CGS_IND_REG__SMC, FEATURE_STATUS, VOLTAGE_CONTROLLER_ON))
-			? true : false;
-}
-
-/**
- * Find the MC microcode version and store it in the HwMgr struct
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int phm_get_mc_microcode_version (struct pp_hwmgr *hwmgr)
-{
-	cgs_write_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_INDEX, 0x9F);
-
-	hwmgr->microcode_version_info.MC = cgs_read_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_DATA);
-
-	return 0;
-}
-
-uint16_t phm_get_current_pcie_speed(struct pp_hwmgr *hwmgr)
-{
-	uint32_t speedCntl = 0;
-
-	/* mmPCIE_PORT_INDEX rename as mmPCIE_INDEX */
-	speedCntl = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__PCIE,
-			ixPCIE_LC_SPEED_CNTL);
-	return((uint16_t)PHM_GET_FIELD(speedCntl,
-			PCIE_LC_SPEED_CNTL, LC_CURRENT_DATA_RATE));
-}
-
-int phm_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr)
-{
-	uint32_t link_width;
-
-	/* mmPCIE_PORT_INDEX rename as mmPCIE_INDEX */
-	link_width = PHM_READ_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE,
-			PCIE_LC_LINK_WIDTH_CNTL, LC_LINK_WIDTH_RD);
-
-	PP_ASSERT_WITH_CODE((7 >= link_width),
-			"Invalid PCIe lane width!", return 0);
-
-	return decode_pcie_lane_width(link_width);
-}
-
-/**
-* Enable voltage control
-*
-* @param    pHwMgr  the address of the powerplay hardware manager.
-* @return   always PP_Result_OK
-*/
-int polaris10_enable_smc_voltage_controller(struct pp_hwmgr *hwmgr)
-{
-	PP_ASSERT_WITH_CODE(
-		(hwmgr->smumgr->smumgr_funcs->send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Voltage_Cntl_Enable) == 0),
-		"Failed to enable voltage DPM during DPM Start Function!",
-		return 1;
-	);
-
-	return 0;
-}
-
-/**
-* Checks if we want to support voltage control
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-*/
-static bool polaris10_voltage_control(const struct pp_hwmgr *hwmgr)
-{
-	const struct polaris10_hwmgr *data =
-			(const struct polaris10_hwmgr *)(hwmgr->backend);
-
-	return (POLARIS10_VOLTAGE_CONTROL_NONE != data->voltage_control);
-}
-
-/**
-* Enable voltage control
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int polaris10_enable_voltage_control(struct pp_hwmgr *hwmgr)
-{
-	/* enable voltage control */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			GENERAL_PWRMGT, VOLT_PWRMGT_EN, 1);
-
-	return 0;
-}
-
-/**
-* Create Voltage Tables.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int polaris10_construct_voltage_tables(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)hwmgr->pptable;
-	int result;
-
-	if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
-		result = atomctrl_get_voltage_table_v3(hwmgr,
-				VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT,
-				&(data->mvdd_voltage_table));
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve MVDD table.",
-				return result);
-	} else if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
-		result = phm_get_svi2_mvdd_voltage_table(&(data->mvdd_voltage_table),
-				table_info->vdd_dep_on_mclk);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve SVI2 MVDD table from dependancy table.",
-				return result;);
-	}
-
-	if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
-		result = atomctrl_get_voltage_table_v3(hwmgr,
-				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT,
-				&(data->vddci_voltage_table));
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve VDDCI table.",
-				return result);
-	} else if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
-		result = phm_get_svi2_vddci_voltage_table(&(data->vddci_voltage_table),
-				table_info->vdd_dep_on_mclk);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve SVI2 VDDCI table from dependancy table.",
-				return result);
-	}
-
-	if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
-		result = phm_get_svi2_vdd_voltage_table(&(data->vddc_voltage_table),
-				table_info->vddc_lookup_table);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to retrieve SVI2 VDDC table from lookup table.",
-				return result);
-	}
-
-	PP_ASSERT_WITH_CODE(
-			(data->vddc_voltage_table.count <= (SMU74_MAX_LEVELS_VDDC)),
-			"Too many voltage values for VDDC. Trimming to fit state table.",
-			phm_trim_voltage_table_to_fit_state_table(SMU74_MAX_LEVELS_VDDC,
-								&(data->vddc_voltage_table)));
-
-	PP_ASSERT_WITH_CODE(
-			(data->vddci_voltage_table.count <= (SMU74_MAX_LEVELS_VDDCI)),
-			"Too many voltage values for VDDCI. Trimming to fit state table.",
-			phm_trim_voltage_table_to_fit_state_table(SMU74_MAX_LEVELS_VDDCI,
-					&(data->vddci_voltage_table)));
-
-	PP_ASSERT_WITH_CODE(
-			(data->mvdd_voltage_table.count <= (SMU74_MAX_LEVELS_MVDD)),
-			"Too many voltage values for MVDD. Trimming to fit state table.",
-			phm_trim_voltage_table_to_fit_state_table(SMU74_MAX_LEVELS_MVDD,
-							   &(data->mvdd_voltage_table)));
-
-	return 0;
-}
-
-/**
-* Programs static screed detection parameters
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int polaris10_program_static_screen_threshold_parameters(
-							struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	/* Set static screen threshold unit */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD_UNIT,
-			data->static_screen_threshold_unit);
-	/* Set static screen threshold */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD,
-			data->static_screen_threshold);
-
-	return 0;
-}
-
-/**
-* Setup display gap for glitch free memory clock switching.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always  0
-*/
-static int polaris10_enable_display_gap(struct pp_hwmgr *hwmgr)
-{
-	uint32_t display_gap =
-			cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-					ixCG_DISPLAY_GAP_CNTL);
-
-	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL,
-			DISP_GAP, DISPLAY_GAP_IGNORE);
-
-	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL,
-			DISP_GAP_MCHG, DISPLAY_GAP_VBLANK);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_DISPLAY_GAP_CNTL, display_gap);
-
-	return 0;
-}
-
-/**
-* Programs activity state transition voting clients
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always  0
-*/
-static int polaris10_program_voting_clients(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	/* Clear reset for voting clients before enabling DPM */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 0);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 0);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_0, data->voting_rights_clients0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_1, data->voting_rights_clients1);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_2, data->voting_rights_clients2);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_3, data->voting_rights_clients3);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_4, data->voting_rights_clients4);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_5, data->voting_rights_clients5);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_6, data->voting_rights_clients6);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_7, data->voting_rights_clients7);
-
-	return 0;
-}
-
-static int polaris10_clear_voting_clients(struct pp_hwmgr *hwmgr)
-{
-	/* Reset voting clients before disabling DPM */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 1);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 1);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_0, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_1, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_2, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_3, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_4, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_5, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_6, 0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_FREQ_TRAN_VOTING_7, 0);
-
-	return 0;
-}
-
-/**
-* Get the location of various tables inside the FW image.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always  0
-*/
-static int polaris10_process_firmware_header(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
-	uint32_t tmp;
-	int result;
-	bool error = false;
-
-	result = polaris10_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU74_Firmware_Header, DpmTable),
-			&tmp, data->sram_end);
-
-	if (0 == result)
-		data->dpm_table_start = tmp;
-
-	error |= (0 != result);
-
-	result = polaris10_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU74_Firmware_Header, SoftRegisters),
-			&tmp, data->sram_end);
-
-	if (!result) {
-		data->soft_regs_start = tmp;
-		smu_data->soft_regs_start = tmp;
-	}
-
-	error |= (0 != result);
-
-	result = polaris10_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU74_Firmware_Header, mcRegisterTable),
-			&tmp, data->sram_end);
-
-	if (!result)
-		data->mc_reg_table_start = tmp;
-
-	result = polaris10_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU74_Firmware_Header, FanTable),
-			&tmp, data->sram_end);
-
-	if (!result)
-		data->fan_table_start = tmp;
-
-	error |= (0 != result);
-
-	result = polaris10_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU74_Firmware_Header, mcArbDramTimingTable),
-			&tmp, data->sram_end);
-
-	if (!result)
-		data->arb_table_start = tmp;
-
-	error |= (0 != result);
-
-	result = polaris10_read_smc_sram_dword(hwmgr->smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION +
-			offsetof(SMU74_Firmware_Header, Version),
-			&tmp, data->sram_end);
-
-	if (!result)
-		hwmgr->microcode_version_info.SMC = tmp;
-
-	error |= (0 != result);
-
-	return error ? -1 : 0;
-}
-
-/* Copy one arb setting to another and then switch the active set.
- * arb_src and arb_dest is one of the MC_CG_ARB_FREQ_Fx constants.
- */
-static int polaris10_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr,
-		uint32_t arb_src, uint32_t arb_dest)
-{
-	uint32_t mc_arb_dram_timing;
-	uint32_t mc_arb_dram_timing2;
-	uint32_t burst_time;
-	uint32_t mc_cg_config;
-
-	switch (arb_src) {
-	case MC_CG_ARB_FREQ_F0:
-		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
-		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
-		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
-		break;
-	case MC_CG_ARB_FREQ_F1:
-		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1);
-		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1);
-		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	switch (arb_dest) {
-	case MC_CG_ARB_FREQ_F0:
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING, mc_arb_dram_timing);
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
-		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0, burst_time);
-		break;
-	case MC_CG_ARB_FREQ_F1:
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
-		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1, burst_time);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	mc_cg_config = cgs_read_register(hwmgr->device, mmMC_CG_CONFIG);
-	mc_cg_config |= 0x0000000F;
-	cgs_write_register(hwmgr->device, mmMC_CG_CONFIG, mc_cg_config);
-	PHM_WRITE_FIELD(hwmgr->device, MC_ARB_CG, CG_ARB_REQ, arb_dest);
-
-	return 0;
-}
-
-static int polaris10_reset_to_default(struct pp_hwmgr *hwmgr)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_ResetToDefaults);
-}
-
-/**
-* Initial switch from ARB F0->F1
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-* This function is to be called from the SetPowerState table.
-*/
-static int polaris10_initial_switch_from_arbf0_to_f1(struct pp_hwmgr *hwmgr)
-{
-	return polaris10_copy_and_switch_arb_sets(hwmgr,
-			MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
-}
-
-static int polaris10_force_switch_to_arbf0(struct pp_hwmgr *hwmgr)
-{
-	uint32_t tmp;
-
-	tmp = (cgs_read_ind_register(hwmgr->device,
-			CGS_IND_REG__SMC, ixSMC_SCRATCH9) &
-			0x0000ff00) >> 8;
-
-	if (tmp == MC_CG_ARB_FREQ_F0)
-		return 0;
-
-	return polaris10_copy_and_switch_arb_sets(hwmgr,
-			tmp, MC_CG_ARB_FREQ_F0);
-}
-
-static int polaris10_setup_default_pcie_table(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
-	uint32_t i, max_entry;
-
-	PP_ASSERT_WITH_CODE((data->use_pcie_performance_levels ||
-			data->use_pcie_power_saving_levels), "No pcie performance levels!",
-			return -EINVAL);
-
-	if (data->use_pcie_performance_levels &&
-			!data->use_pcie_power_saving_levels) {
-		data->pcie_gen_power_saving = data->pcie_gen_performance;
-		data->pcie_lane_power_saving = data->pcie_lane_performance;
-	} else if (!data->use_pcie_performance_levels &&
-			data->use_pcie_power_saving_levels) {
-		data->pcie_gen_performance = data->pcie_gen_power_saving;
-		data->pcie_lane_performance = data->pcie_lane_power_saving;
-	}
-
-	phm_reset_single_dpm_table(&data->dpm_table.pcie_speed_table,
-					SMU74_MAX_LEVELS_LINK,
-					MAX_REGULAR_DPM_NUMBER);
-
-	if (pcie_table != NULL) {
-		/* max_entry is used to make sure we reserve one PCIE level
-		 * for boot level (fix for A+A PSPP issue).
-		 * If PCIE table from PPTable have ULV entry + 8 entries,
-		 * then ignore the last entry.*/
-		max_entry = (SMU74_MAX_LEVELS_LINK < pcie_table->count) ?
-				SMU74_MAX_LEVELS_LINK : pcie_table->count;
-		for (i = 1; i < max_entry; i++) {
-			phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, i - 1,
-					get_pcie_gen_support(data->pcie_gen_cap,
-							pcie_table->entries[i].gen_speed),
-					get_pcie_lane_support(data->pcie_lane_cap,
-							pcie_table->entries[i].lane_width));
-		}
-		data->dpm_table.pcie_speed_table.count = max_entry - 1;
-
-		/* Setup BIF_SCLK levels */
-		for (i = 0; i < max_entry; i++)
-			data->bif_sclk_table[i] = pcie_table->entries[i].pcie_sclk;
-	} else {
-		/* Hardcode Pcie Table */
-		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 0,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Min_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 1,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Min_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 2,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Max_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 3,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Max_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 4,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Max_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 5,
-				get_pcie_gen_support(data->pcie_gen_cap,
-						PP_Max_PCIEGen),
-				get_pcie_lane_support(data->pcie_lane_cap,
-						PP_Max_PCIELane));
-
-		data->dpm_table.pcie_speed_table.count = 6;
-	}
-	/* Populate last level for boot PCIE level, but do not increment count. */
-	phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table,
-			data->dpm_table.pcie_speed_table.count,
-			get_pcie_gen_support(data->pcie_gen_cap,
-					PP_Min_PCIEGen),
-			get_pcie_lane_support(data->pcie_lane_cap,
-					PP_Max_PCIELane));
-
-	return 0;
-}
-
-/*
- * This function is to initalize all DPM state tables
- * for SMU7 based on the dependency table.
- * Dynamic state patching function will then trim these
- * state tables to the allowed range based
- * on the power policy or external client requests,
- * such as UVD request, etc.
- */
-int polaris10_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint32_t i;
-
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_sclk_table =
-			table_info->vdd_dep_on_sclk;
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
-			table_info->vdd_dep_on_mclk;
-
-	PP_ASSERT_WITH_CODE(dep_sclk_table != NULL,
-			"SCLK dependency table is missing. This table is mandatory",
-			return -EINVAL);
-	PP_ASSERT_WITH_CODE(dep_sclk_table->count >= 1,
-			"SCLK dependency table has to have is missing."
-			"This table is mandatory",
-			return -EINVAL);
-
-	PP_ASSERT_WITH_CODE(dep_mclk_table != NULL,
-			"MCLK dependency table is missing. This table is mandatory",
-			return -EINVAL);
-	PP_ASSERT_WITH_CODE(dep_mclk_table->count >= 1,
-			"MCLK dependency table has to have is missing."
-			"This table is mandatory",
-			return -EINVAL);
-
-	/* clear the state table to reset everything to default */
-	phm_reset_single_dpm_table(
-			&data->dpm_table.sclk_table, SMU74_MAX_LEVELS_GRAPHICS, MAX_REGULAR_DPM_NUMBER);
-	phm_reset_single_dpm_table(
-			&data->dpm_table.mclk_table, SMU74_MAX_LEVELS_MEMORY, MAX_REGULAR_DPM_NUMBER);
-
-
-	/* Initialize Sclk DPM table based on allow Sclk values */
-	data->dpm_table.sclk_table.count = 0;
-	for (i = 0; i < dep_sclk_table->count; i++) {
-		if (i == 0 || data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count - 1].value !=
-						dep_sclk_table->entries[i].clk) {
-
-			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].value =
-					dep_sclk_table->entries[i].clk;
-
-			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled =
-					(i == 0) ? true : false;
-			data->dpm_table.sclk_table.count++;
-		}
-	}
-
-	/* Initialize Mclk DPM table based on allow Mclk values */
-	data->dpm_table.mclk_table.count = 0;
-	for (i = 0; i < dep_mclk_table->count; i++) {
-		if (i == 0 || data->dpm_table.mclk_table.dpm_levels
-				[data->dpm_table.mclk_table.count - 1].value !=
-						dep_mclk_table->entries[i].clk) {
-			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].value =
-							dep_mclk_table->entries[i].clk;
-			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].enabled =
-							(i == 0) ? true : false;
-			data->dpm_table.mclk_table.count++;
-		}
-	}
-
-	/* setup PCIE gen speed levels */
-	polaris10_setup_default_pcie_table(hwmgr);
-
-	/* save a copy of the default DPM table */
-	memcpy(&(data->golden_dpm_table), &(data->dpm_table),
-			sizeof(struct polaris10_dpm_table));
-
-	return 0;
-}
-
-uint8_t convert_to_vid(uint16_t vddc)
-{
-	return (uint8_t) ((6200 - (vddc * VOLTAGE_SCALE)) / 25);
-}
-
-/**
- * Mvdd table preparation for SMC.
- *
- * @param    *hwmgr The address of the hardware manager.
- * @param    *table The SMC DPM table structure to be populated.
- * @return   0
- */
-static int polaris10_populate_smc_mvdd_table(struct pp_hwmgr *hwmgr,
-			SMU74_Discrete_DpmTable *table)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t count, level;
-
-	if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
-		count = data->mvdd_voltage_table.count;
-		if (count > SMU_MAX_SMIO_LEVELS)
-			count = SMU_MAX_SMIO_LEVELS;
-		for (level = 0; level < count; level++) {
-			table->SmioTable2.Pattern[level].Voltage =
-				PP_HOST_TO_SMC_US(data->mvdd_voltage_table.entries[count].value * VOLTAGE_SCALE);
-			/* Index into DpmTable.Smio. Drive bits from Smio entry to get this voltage level.*/
-			table->SmioTable2.Pattern[level].Smio =
-				(uint8_t) level;
-			table->Smio[level] |=
-				data->mvdd_voltage_table.entries[level].smio_low;
-		}
-		table->SmioMask2 = data->mvdd_voltage_table.mask_low;
-
-		table->MvddLevelCount = (uint32_t) PP_HOST_TO_SMC_UL(count);
-	}
-
-	return 0;
-}
-
-static int polaris10_populate_smc_vddci_table(struct pp_hwmgr *hwmgr,
-					struct SMU74_Discrete_DpmTable *table)
-{
-	uint32_t count, level;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	count = data->vddci_voltage_table.count;
-
-	if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
-		if (count > SMU_MAX_SMIO_LEVELS)
-			count = SMU_MAX_SMIO_LEVELS;
-		for (level = 0; level < count; ++level) {
-			table->SmioTable1.Pattern[level].Voltage =
-				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[level].value * VOLTAGE_SCALE);
-			table->SmioTable1.Pattern[level].Smio = (uint8_t) level;
-
-			table->Smio[level] |= data->vddci_voltage_table.entries[level].smio_low;
-		}
-	}
-
-	table->SmioMask1 = data->vddci_voltage_table.mask_low;
-
-	return 0;
-}
-
-/**
-* Preparation of vddc and vddgfx CAC tables for SMC.
-*
-* @param    hwmgr  the address of the hardware manager
-* @param    table  the SMC DPM table structure to be populated
-* @return   always 0
-*/
-static int polaris10_populate_cac_table(struct pp_hwmgr *hwmgr,
-		struct SMU74_Discrete_DpmTable *table)
-{
-	uint32_t count;
-	uint8_t index;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_voltage_lookup_table *lookup_table =
-			table_info->vddc_lookup_table;
-	/* tables is already swapped, so in order to use the value from it,
-	 * we need to swap it back.
-	 * We are populating vddc CAC data to BapmVddc table
-	 * in split and merged mode
-	 */
-	for (count = 0; count < lookup_table->count; count++) {
-		index = phm_get_voltage_index(lookup_table,
-				data->vddc_voltage_table.entries[count].value);
-		table->BapmVddcVidLoSidd[count] = convert_to_vid(lookup_table->entries[index].us_cac_low);
-		table->BapmVddcVidHiSidd[count] = convert_to_vid(lookup_table->entries[index].us_cac_mid);
-		table->BapmVddcVidHiSidd2[count] = convert_to_vid(lookup_table->entries[index].us_cac_high);
-	}
-
-	return 0;
-}
-
-/**
-* Preparation of voltage tables for SMC.
-*
-* @param    hwmgr   the address of the hardware manager
-* @param    table   the SMC DPM table structure to be populated
-* @return   always  0
-*/
-
-int polaris10_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
-		struct SMU74_Discrete_DpmTable *table)
-{
-	polaris10_populate_smc_vddci_table(hwmgr, table);
-	polaris10_populate_smc_mvdd_table(hwmgr, table);
-	polaris10_populate_cac_table(hwmgr, table);
-
-	return 0;
-}
-
-static int polaris10_populate_ulv_level(struct pp_hwmgr *hwmgr,
-		struct SMU74_Discrete_Ulv *state)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	state->CcPwrDynRm = 0;
-	state->CcPwrDynRm1 = 0;
-
-	state->VddcOffset = (uint16_t) table_info->us_ulv_voltage_offset;
-	state->VddcOffsetVid = (uint8_t)(table_info->us_ulv_voltage_offset *
-			VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1);
-
-	state->VddcPhase = (data->vddc_phase_shed_control) ? 0 : 1;
-
-	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm);
-	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm1);
-	CONVERT_FROM_HOST_TO_SMC_US(state->VddcOffset);
-
-	return 0;
-}
-
-static int polaris10_populate_ulv_state(struct pp_hwmgr *hwmgr,
-		struct SMU74_Discrete_DpmTable *table)
-{
-	return polaris10_populate_ulv_level(hwmgr, &table->Ulv);
-}
-
-static int polaris10_populate_smc_link_level(struct pp_hwmgr *hwmgr,
-		struct SMU74_Discrete_DpmTable *table)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_dpm_table *dpm_table = &data->dpm_table;
-	int i;
-
-	/* Index (dpm_table->pcie_speed_table.count)
-	 * is reserved for PCIE boot level. */
-	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
-		table->LinkLevel[i].PcieGenSpeed  =
-				(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
-		table->LinkLevel[i].PcieLaneCount = (uint8_t)encode_pcie_lane_width(
-				dpm_table->pcie_speed_table.dpm_levels[i].param1);
-		table->LinkLevel[i].EnabledForActivity = 1;
-		table->LinkLevel[i].SPC = (uint8_t)(data->pcie_spc_cap & 0xff);
-		table->LinkLevel[i].DownThreshold = PP_HOST_TO_SMC_UL(5);
-		table->LinkLevel[i].UpThreshold = PP_HOST_TO_SMC_UL(30);
-	}
-
-	data->smc_state_table.LinkLevelCount =
-			(uint8_t)dpm_table->pcie_speed_table.count;
-	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
-			phm_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
-
-	return 0;
-}
-
-static uint32_t polaris10_get_xclk(struct pp_hwmgr *hwmgr)
-{
-	uint32_t reference_clock, tmp;
-	struct cgs_display_info info = {0};
-	struct cgs_mode_info mode_info;
-
-	info.mode_info = &mode_info;
-
-	tmp = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_CLKPIN_CNTL_2, MUX_TCLK_TO_XCLK);
-
-	if (tmp)
-		return TCLK;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	reference_clock = mode_info.ref_clock;
-
-	tmp = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_CLKPIN_CNTL, XTALIN_DIVIDE);
-
-	if (0 != tmp)
-		return reference_clock / 4;
-
-	return reference_clock;
-}
-
-/**
-* Calculates the SCLK dividers using the provided engine clock
-*
-* @param    hwmgr  the address of the hardware manager
-* @param    clock  the engine clock to use to populate the structure
-* @param    sclk   the SMC SCLK structure to be populated
-*/
-static int polaris10_calculate_sclk_params(struct pp_hwmgr *hwmgr,
-		uint32_t clock, SMU_SclkSetting *sclk_setting)
-{
-	const struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	const SMU74_Discrete_DpmTable *table = &(data->smc_state_table);
-	struct pp_atomctrl_clock_dividers_ai dividers;
-
-	uint32_t ref_clock;
-	uint32_t pcc_target_percent, pcc_target_freq, ss_target_percent, ss_target_freq;
-	uint8_t i;
-	int result;
-	uint64_t temp;
-
-	sclk_setting->SclkFrequency = clock;
-	/* get the engine clock dividers for this clock value */
-	result = atomctrl_get_engine_pll_dividers_ai(hwmgr, clock,  &dividers);
-	if (result == 0) {
-		sclk_setting->Fcw_int = dividers.usSclk_fcw_int;
-		sclk_setting->Fcw_frac = dividers.usSclk_fcw_frac;
-		sclk_setting->Pcc_fcw_int = dividers.usPcc_fcw_int;
-		sclk_setting->PllRange = dividers.ucSclkPllRange;
-		sclk_setting->Sclk_slew_rate = 0x400;
-		sclk_setting->Pcc_up_slew_rate = dividers.usPcc_fcw_slew_frac;
-		sclk_setting->Pcc_down_slew_rate = 0xffff;
-		sclk_setting->SSc_En = dividers.ucSscEnable;
-		sclk_setting->Fcw1_int = dividers.usSsc_fcw1_int;
-		sclk_setting->Fcw1_frac = dividers.usSsc_fcw1_frac;
-		sclk_setting->Sclk_ss_slew_rate = dividers.usSsc_fcw_slew_frac;
-		return result;
-	}
-
-	ref_clock = polaris10_get_xclk(hwmgr);
-
-	for (i = 0; i < NUM_SCLK_RANGE; i++) {
-		if (clock > data->range_table[i].trans_lower_frequency
-		&& clock <= data->range_table[i].trans_upper_frequency) {
-			sclk_setting->PllRange = i;
-			break;
-		}
-	}
-
-	sclk_setting->Fcw_int = (uint16_t)((clock << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv) / ref_clock);
-	temp = clock << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv;
-	temp <<= 0x10;
-	do_div(temp, ref_clock);
-	sclk_setting->Fcw_frac = temp & 0xffff;
-
-	pcc_target_percent = 10; /*  Hardcode 10% for now. */
-	pcc_target_freq = clock - (clock * pcc_target_percent / 100);
-	sclk_setting->Pcc_fcw_int = (uint16_t)((pcc_target_freq << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv) / ref_clock);
-
-	ss_target_percent = 2; /*  Hardcode 2% for now. */
-	sclk_setting->SSc_En = 0;
-	if (ss_target_percent) {
-		sclk_setting->SSc_En = 1;
-		ss_target_freq = clock - (clock * ss_target_percent / 100);
-		sclk_setting->Fcw1_int = (uint16_t)((ss_target_freq << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv) / ref_clock);
-		temp = ss_target_freq << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv;
-		temp <<= 0x10;
-		do_div(temp, ref_clock);
-		sclk_setting->Fcw1_frac = temp & 0xffff;
-	}
-
-	return 0;
-}
-
-static int polaris10_get_dependency_volt_by_clk(struct pp_hwmgr *hwmgr,
-		struct phm_ppt_v1_clock_voltage_dependency_table *dep_table,
-		uint32_t clock, SMU_VoltageLevel *voltage, uint32_t *mvdd)
-{
-	uint32_t i;
-	uint16_t vddci;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	*voltage = *mvdd = 0;
-
-	/* clock - voltage dependency table is empty table */
-	if (dep_table->count == 0)
-		return -EINVAL;
-
-	for (i = 0; i < dep_table->count; i++) {
-		/* find first sclk bigger than request */
-		if (dep_table->entries[i].clk >= clock) {
-			*voltage |= (dep_table->entries[i].vddc *
-					VOLTAGE_SCALE) << VDDC_SHIFT;
-			if (POLARIS10_VOLTAGE_CONTROL_NONE == data->vddci_control)
-				*voltage |= (data->vbios_boot_state.vddci_bootup_value *
-						VOLTAGE_SCALE) << VDDCI_SHIFT;
-			else if (dep_table->entries[i].vddci)
-				*voltage |= (dep_table->entries[i].vddci *
-						VOLTAGE_SCALE) << VDDCI_SHIFT;
-			else {
-				vddci = phm_find_closest_vddci(&(data->vddci_voltage_table),
-						(dep_table->entries[i].vddc -
-								(uint16_t)data->vddc_vddci_delta));
-				*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
-			}
-
-			if (POLARIS10_VOLTAGE_CONTROL_NONE == data->mvdd_control)
-				*mvdd = data->vbios_boot_state.mvdd_bootup_value *
-					VOLTAGE_SCALE;
-			else if (dep_table->entries[i].mvdd)
-				*mvdd = (uint32_t) dep_table->entries[i].mvdd *
-					VOLTAGE_SCALE;
-
-			*voltage |= 1 << PHASES_SHIFT;
-			return 0;
-		}
-	}
-
-	/* sclk is bigger than max sclk in the dependence table */
-	*voltage |= (dep_table->entries[i - 1].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
-
-	if (POLARIS10_VOLTAGE_CONTROL_NONE == data->vddci_control)
-		*voltage |= (data->vbios_boot_state.vddci_bootup_value *
-				VOLTAGE_SCALE) << VDDCI_SHIFT;
-	else if (dep_table->entries[i-1].vddci) {
-		vddci = phm_find_closest_vddci(&(data->vddci_voltage_table),
-				(dep_table->entries[i].vddc -
-						(uint16_t)data->vddc_vddci_delta));
-		*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
-	}
-
-	if (POLARIS10_VOLTAGE_CONTROL_NONE == data->mvdd_control)
-		*mvdd = data->vbios_boot_state.mvdd_bootup_value * VOLTAGE_SCALE;
-	else if (dep_table->entries[i].mvdd)
-		*mvdd = (uint32_t) dep_table->entries[i - 1].mvdd * VOLTAGE_SCALE;
-
-	return 0;
-}
-
-static const sclkFcwRange_t Range_Table[NUM_SCLK_RANGE] =
-{ {VCO_2_4, POSTDIV_DIV_BY_16,  75, 160, 112},
-  {VCO_3_6, POSTDIV_DIV_BY_16, 112, 224, 160},
-  {VCO_2_4, POSTDIV_DIV_BY_8,   75, 160, 112},
-  {VCO_3_6, POSTDIV_DIV_BY_8,  112, 224, 160},
-  {VCO_2_4, POSTDIV_DIV_BY_4,   75, 160, 112},
-  {VCO_3_6, POSTDIV_DIV_BY_4,  112, 216, 160},
-  {VCO_2_4, POSTDIV_DIV_BY_2,   75, 160, 108},
-  {VCO_3_6, POSTDIV_DIV_BY_2,  112, 216, 160} };
-
-static void polaris10_get_sclk_range_table(struct pp_hwmgr *hwmgr)
-{
-	uint32_t i, ref_clk;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	SMU74_Discrete_DpmTable  *table = &(data->smc_state_table);
-	struct pp_atom_ctrl_sclk_range_table range_table_from_vbios = { { {0} } };
-
-	ref_clk = polaris10_get_xclk(hwmgr);
-
-	if (0 == atomctrl_get_smc_sclk_range_table(hwmgr, &range_table_from_vbios)) {
-		for (i = 0; i < NUM_SCLK_RANGE; i++) {
-			table->SclkFcwRangeTable[i].vco_setting = range_table_from_vbios.entry[i].ucVco_setting;
-			table->SclkFcwRangeTable[i].postdiv = range_table_from_vbios.entry[i].ucPostdiv;
-			table->SclkFcwRangeTable[i].fcw_pcc = range_table_from_vbios.entry[i].usFcw_pcc;
-
-			table->SclkFcwRangeTable[i].fcw_trans_upper = range_table_from_vbios.entry[i].usFcw_trans_upper;
-			table->SclkFcwRangeTable[i].fcw_trans_lower = range_table_from_vbios.entry[i].usRcw_trans_lower;
-
-			CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_pcc);
-			CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_upper);
-			CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_lower);
-		}
-		return;
-	}
-
-	for (i = 0; i < NUM_SCLK_RANGE; i++) {
-
-		data->range_table[i].trans_lower_frequency = (ref_clk * Range_Table[i].fcw_trans_lower) >> Range_Table[i].postdiv;
-		data->range_table[i].trans_upper_frequency = (ref_clk * Range_Table[i].fcw_trans_upper) >> Range_Table[i].postdiv;
-
-		table->SclkFcwRangeTable[i].vco_setting = Range_Table[i].vco_setting;
-		table->SclkFcwRangeTable[i].postdiv = Range_Table[i].postdiv;
-		table->SclkFcwRangeTable[i].fcw_pcc = Range_Table[i].fcw_pcc;
-
-		table->SclkFcwRangeTable[i].fcw_trans_upper = Range_Table[i].fcw_trans_upper;
-		table->SclkFcwRangeTable[i].fcw_trans_lower = Range_Table[i].fcw_trans_lower;
-
-		CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_pcc);
-		CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_upper);
-		CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_lower);
-	}
-}
-
-/**
-* Populates single SMC SCLK structure using the provided engine clock
-*
-* @param    hwmgr      the address of the hardware manager
-* @param    clock the engine clock to use to populate the structure
-* @param    sclk        the SMC SCLK structure to be populated
-*/
-
-static int polaris10_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
-		uint32_t clock, uint16_t sclk_al_threshold,
-		struct SMU74_Discrete_GraphicsLevel *level)
-{
-	int result, i, temp;
-	/* PP_Clocks minClocks; */
-	uint32_t mvdd;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	SMU_SclkSetting curr_sclk_setting = { 0 };
-
-	result = polaris10_calculate_sclk_params(hwmgr, clock, &curr_sclk_setting);
-
-	/* populate graphics levels */
-	result = polaris10_get_dependency_volt_by_clk(hwmgr,
-			table_info->vdd_dep_on_sclk, clock,
-			&level->MinVoltage, &mvdd);
-
-	PP_ASSERT_WITH_CODE((0 == result),
-			"can not find VDDC voltage value for "
-			"VDDC engine clock dependency table",
-			return result);
-	level->ActivityLevel = sclk_al_threshold;
-
-	level->CcPwrDynRm = 0;
-	level->CcPwrDynRm1 = 0;
-	level->EnabledForActivity = 0;
-	level->EnabledForThrottle = 1;
-	level->UpHyst = 10;
-	level->DownHyst = 0;
-	level->VoltageDownHyst = 0;
-	level->PowerThrottle = 0;
-
-	/*
-	* TODO: get minimum clocks from dal configaration
-	* PECI_GetMinClockSettings(hwmgr->pPECI, &minClocks);
-	*/
-	/* data->DisplayTiming.minClockInSR = minClocks.engineClockInSR; */
-
-	/* get level->DeepSleepDivId
-	if (phm_cap_enabled(hwmgr->platformDescriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep))
-		level->DeepSleepDivId = PhwFiji_GetSleepDividerIdFromClock(hwmgr, clock, minClocks.engineClockInSR);
-	*/
-	PP_ASSERT_WITH_CODE((clock >= POLARIS10_MINIMUM_ENGINE_CLOCK), "Engine clock can't satisfy stutter requirement!", return 0);
-	for (i = POLARIS10_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
-		temp = clock >> i;
-
-		if (temp >= POLARIS10_MINIMUM_ENGINE_CLOCK || i == 0)
-			break;
-	}
-
-	level->DeepSleepDivId = i;
-
-	/* Default to slow, highest DPM level will be
-	 * set to PPSMC_DISPLAY_WATERMARK_LOW later.
-	 */
-	if (data->update_up_hyst)
-		level->UpHyst = (uint8_t)data->up_hyst;
-	if (data->update_down_hyst)
-		level->DownHyst = (uint8_t)data->down_hyst;
-
-	level->SclkSetting = curr_sclk_setting;
-
-	CONVERT_FROM_HOST_TO_SMC_UL(level->MinVoltage);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm1);
-	CONVERT_FROM_HOST_TO_SMC_US(level->ActivityLevel);
-	CONVERT_FROM_HOST_TO_SMC_UL(level->SclkSetting.SclkFrequency);
-	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw_int);
-	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw_frac);
-	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Pcc_fcw_int);
-	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Sclk_slew_rate);
-	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Pcc_up_slew_rate);
-	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Pcc_down_slew_rate);
-	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw1_int);
-	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw1_frac);
-	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Sclk_ss_slew_rate);
-	return 0;
-}
-
-/**
-* Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states
-*
-* @param    hwmgr      the address of the hardware manager
-*/
-static int polaris10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_dpm_table *dpm_table = &data->dpm_table;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
-	uint8_t pcie_entry_cnt = (uint8_t) data->dpm_table.pcie_speed_table.count;
-	int result = 0;
-	uint32_t array = data->dpm_table_start +
-			offsetof(SMU74_Discrete_DpmTable, GraphicsLevel);
-	uint32_t array_size = sizeof(struct SMU74_Discrete_GraphicsLevel) *
-			SMU74_MAX_LEVELS_GRAPHICS;
-	struct SMU74_Discrete_GraphicsLevel *levels =
-			data->smc_state_table.GraphicsLevel;
-	uint32_t i, max_entry;
-	uint8_t hightest_pcie_level_enabled = 0,
-		lowest_pcie_level_enabled = 0,
-		mid_pcie_level_enabled = 0,
-		count = 0;
-
-	polaris10_get_sclk_range_table(hwmgr);
-
-	for (i = 0; i < dpm_table->sclk_table.count; i++) {
-
-		result = polaris10_populate_single_graphic_level(hwmgr,
-				dpm_table->sclk_table.dpm_levels[i].value,
-				(uint16_t)data->activity_target[i],
-				&(data->smc_state_table.GraphicsLevel[i]));
-		if (result)
-			return result;
-
-		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
-		if (i > 1)
-			levels[i].DeepSleepDivId = 0;
-	}
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_SPLLShutdownSupport))
-		data->smc_state_table.GraphicsLevel[0].SclkSetting.SSc_En = 0;
-
-	data->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1;
-	data->smc_state_table.GraphicsDpmLevelCount =
-			(uint8_t)dpm_table->sclk_table.count;
-	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
-			phm_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
-
-
-	if (pcie_table != NULL) {
-		PP_ASSERT_WITH_CODE((1 <= pcie_entry_cnt),
-				"There must be 1 or more PCIE levels defined in PPTable.",
-				return -EINVAL);
-		max_entry = pcie_entry_cnt - 1;
-		for (i = 0; i < dpm_table->sclk_table.count; i++)
-			levels[i].pcieDpmLevel =
-					(uint8_t) ((i < max_entry) ? i : max_entry);
-	} else {
-		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
-				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
-						(1 << (hightest_pcie_level_enabled + 1))) != 0))
-			hightest_pcie_level_enabled++;
-
-		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
-				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
-						(1 << lowest_pcie_level_enabled)) == 0))
-			lowest_pcie_level_enabled++;
-
-		while ((count < hightest_pcie_level_enabled) &&
-				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
-						(1 << (lowest_pcie_level_enabled + 1 + count))) == 0))
-			count++;
-
-		mid_pcie_level_enabled = (lowest_pcie_level_enabled + 1 + count) <
-				hightest_pcie_level_enabled ?
-						(lowest_pcie_level_enabled + 1 + count) :
-						hightest_pcie_level_enabled;
-
-		/* set pcieDpmLevel to hightest_pcie_level_enabled */
-		for (i = 2; i < dpm_table->sclk_table.count; i++)
-			levels[i].pcieDpmLevel = hightest_pcie_level_enabled;
-
-		/* set pcieDpmLevel to lowest_pcie_level_enabled */
-		levels[0].pcieDpmLevel = lowest_pcie_level_enabled;
-
-		/* set pcieDpmLevel to mid_pcie_level_enabled */
-		levels[1].pcieDpmLevel = mid_pcie_level_enabled;
-	}
-	/* level count will send to smc once at init smc table and never change */
-	result = polaris10_copy_bytes_to_smc(hwmgr->smumgr, array, (uint8_t *)levels,
-			(uint32_t)array_size, data->sram_end);
-
-	return result;
-}
-
-static int polaris10_populate_single_memory_level(struct pp_hwmgr *hwmgr,
-		uint32_t clock, struct SMU74_Discrete_MemoryLevel *mem_level)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	int result = 0;
-	struct cgs_display_info info = {0, 0, NULL};
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	if (table_info->vdd_dep_on_mclk) {
-		result = polaris10_get_dependency_volt_by_clk(hwmgr,
-				table_info->vdd_dep_on_mclk, clock,
-				&mem_level->MinVoltage, &mem_level->MinMvdd);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find MinVddc voltage value from memory "
-				"VDDC voltage dependency table", return result);
-	}
-
-	mem_level->MclkFrequency = clock;
-	mem_level->EnabledForThrottle = 1;
-	mem_level->EnabledForActivity = 0;
-	mem_level->UpHyst = 0;
-	mem_level->DownHyst = 100;
-	mem_level->VoltageDownHyst = 0;
-	mem_level->ActivityLevel = (uint16_t)data->mclk_activity_target;
-	mem_level->StutterEnable = false;
-	mem_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
-
-	data->display_timing.num_existing_displays = info.display_count;
-
-	if ((data->mclk_stutter_mode_threshold) &&
-		(clock <= data->mclk_stutter_mode_threshold) &&
-		(PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL,
-				STUTTER_ENABLE) & 0x1))
-		mem_level->StutterEnable = true;
-
-	if (!result) {
-		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinMvdd);
-		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MclkFrequency);
-		CONVERT_FROM_HOST_TO_SMC_US(mem_level->ActivityLevel);
-		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinVoltage);
-	}
-	return result;
-}
-
-/**
-* Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states
-*
-* @param    hwmgr      the address of the hardware manager
-*/
-static int polaris10_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_dpm_table *dpm_table = &data->dpm_table;
-	int result;
-	/* populate MCLK dpm table to SMU7 */
-	uint32_t array = data->dpm_table_start +
-			offsetof(SMU74_Discrete_DpmTable, MemoryLevel);
-	uint32_t array_size = sizeof(SMU74_Discrete_MemoryLevel) *
-			SMU74_MAX_LEVELS_MEMORY;
-	struct SMU74_Discrete_MemoryLevel *levels =
-			data->smc_state_table.MemoryLevel;
-	uint32_t i;
-
-	for (i = 0; i < dpm_table->mclk_table.count; i++) {
-		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
-				"can not populate memory level as memory clock is zero",
-				return -EINVAL);
-		result = polaris10_populate_single_memory_level(hwmgr,
-				dpm_table->mclk_table.dpm_levels[i].value,
-				&levels[i]);
-		if (i == dpm_table->mclk_table.count - 1) {
-			levels[i].DisplayWatermark = PPSMC_DISPLAY_WATERMARK_HIGH;
-			levels[i].EnabledForActivity = 1;
-		}
-		if (result)
-			return result;
-	}
-
-	/* In order to prevent MC activity from stutter mode to push DPM up,
-	 * the UVD change complements this by putting the MCLK in
-	 * a higher state by default such that we are not affected by
-	 * up threshold or and MCLK DPM latency.
-	 */
-	levels[0].ActivityLevel = 0x1f;
-	CONVERT_FROM_HOST_TO_SMC_US(levels[0].ActivityLevel);
-
-	data->smc_state_table.MemoryDpmLevelCount =
-			(uint8_t)dpm_table->mclk_table.count;
-	data->dpm_level_enable_mask.mclk_dpm_enable_mask =
-			phm_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
-
-	/* level count will send to smc once at init smc table and never change */
-	result = polaris10_copy_bytes_to_smc(hwmgr->smumgr, array, (uint8_t *)levels,
-			(uint32_t)array_size, data->sram_end);
-
-	return result;
-}
-
-/**
-* Populates the SMC MVDD structure using the provided memory clock.
-*
-* @param    hwmgr      the address of the hardware manager
-* @param    mclk        the MCLK value to be used in the decision if MVDD should be high or low.
-* @param    voltage     the SMC VOLTAGE structure to be populated
-*/
-int polaris10_populate_mvdd_value(struct pp_hwmgr *hwmgr,
-		uint32_t mclk, SMIO_Pattern *smio_pat)
-{
-	const struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint32_t i = 0;
-
-	if (POLARIS10_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
-		/* find mvdd value which clock is more than request */
-		for (i = 0; i < table_info->vdd_dep_on_mclk->count; i++) {
-			if (mclk <= table_info->vdd_dep_on_mclk->entries[i].clk) {
-				smio_pat->Voltage = data->mvdd_voltage_table.entries[i].value;
-				break;
-			}
-		}
-		PP_ASSERT_WITH_CODE(i < table_info->vdd_dep_on_mclk->count,
-				"MVDD Voltage is outside the supported range.",
-				return -EINVAL);
-	} else
-		return -EINVAL;
-
-	return 0;
-}
-
-static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
-		SMU74_Discrete_DpmTable *table)
-{
-	int result = 0;
-	uint32_t sclk_frequency;
-	const struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	SMIO_Pattern vol_level;
-	uint32_t mvdd;
-	uint16_t us_mvdd;
-
-	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
-
-
-	/* Get MinVoltage and Frequency from DPM0,
-	 * already converted to SMC_UL */
-	sclk_frequency = data->vbios_boot_state.sclk_bootup_value;
-	result = polaris10_get_dependency_volt_by_clk(hwmgr,
-			table_info->vdd_dep_on_sclk,
-			sclk_frequency,
-			&table->ACPILevel.MinVoltage, &mvdd);
-	PP_ASSERT_WITH_CODE((0 == result),
-			"Cannot find ACPI VDDC voltage value "
-			"in Clock Dependency Table",
-			);
-
-
-	result = polaris10_calculate_sclk_params(hwmgr, sclk_frequency,  &(table->ACPILevel.SclkSetting));
-	PP_ASSERT_WITH_CODE(result == 0, "Error retrieving Engine Clock dividers from VBIOS.", return result);
-
-	table->ACPILevel.DeepSleepDivId = 0;
-	table->ACPILevel.CcPwrDynRm = 0;
-	table->ACPILevel.CcPwrDynRm1 = 0;
-
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.MinVoltage);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
-
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkSetting.SclkFrequency);
-	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw_int);
-	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw_frac);
-	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Pcc_fcw_int);
-	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Sclk_slew_rate);
-	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Pcc_up_slew_rate);
-	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Pcc_down_slew_rate);
-	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw1_int);
-	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw1_frac);
-	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Sclk_ss_slew_rate);
-
-
-	/* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */
-	table->MemoryACPILevel.MclkFrequency = data->vbios_boot_state.mclk_bootup_value;
-	result = polaris10_get_dependency_volt_by_clk(hwmgr,
-			table_info->vdd_dep_on_mclk,
-			table->MemoryACPILevel.MclkFrequency,
-			&table->MemoryACPILevel.MinVoltage, &mvdd);
-	PP_ASSERT_WITH_CODE((0 == result),
-			"Cannot find ACPI VDDCI voltage value "
-			"in Clock Dependency Table",
-			);
-
-	us_mvdd = 0;
-	if ((POLARIS10_VOLTAGE_CONTROL_NONE == data->mvdd_control) ||
-			(data->mclk_dpm_key_disabled))
-		us_mvdd = data->vbios_boot_state.mvdd_bootup_value;
-	else {
-		if (!polaris10_populate_mvdd_value(hwmgr,
-				data->dpm_table.mclk_table.dpm_levels[0].value,
-				&vol_level))
-			us_mvdd = vol_level.Voltage;
-	}
-
-	if (0 == polaris10_populate_mvdd_value(hwmgr, 0, &vol_level))
-		table->MemoryACPILevel.MinMvdd = PP_HOST_TO_SMC_UL(vol_level.Voltage);
-	else
-		table->MemoryACPILevel.MinMvdd = 0;
-
-	table->MemoryACPILevel.StutterEnable = false;
-
-	table->MemoryACPILevel.EnabledForThrottle = 0;
-	table->MemoryACPILevel.EnabledForActivity = 0;
-	table->MemoryACPILevel.UpHyst = 0;
-	table->MemoryACPILevel.DownHyst = 100;
-	table->MemoryACPILevel.VoltageDownHyst = 0;
-	table->MemoryACPILevel.ActivityLevel =
-			PP_HOST_TO_SMC_US((uint16_t)data->mclk_activity_target);
-
-	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MclkFrequency);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MinVoltage);
-
-	return result;
-}
-
-static int polaris10_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
-		SMU74_Discrete_DpmTable *table)
-{
-	int result = -EINVAL;
-	uint8_t count;
-	struct pp_atomctrl_clock_dividers_vi dividers;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
-			table_info->mm_dep_table;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t vddci;
-
-	table->VceLevelCount = (uint8_t)(mm_table->count);
-	table->VceBootLevel = 0;
-
-	for (count = 0; count < table->VceLevelCount; count++) {
-		table->VceLevel[count].Frequency = mm_table->entries[count].eclk;
-		table->VceLevel[count].MinVoltage = 0;
-		table->VceLevel[count].MinVoltage |=
-				(mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
-
-		if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
-			vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table),
-						mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
-		else if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
-			vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA;
-		else
-			vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT;
-
-
-		table->VceLevel[count].MinVoltage |=
-				(vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
-		table->VceLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
-
-		/*retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-				table->VceLevel[count].Frequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for VCE engine clock",
-				return result);
-
-		table->VceLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].Frequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].MinVoltage);
-	}
-	return result;
-}
-
-static int polaris10_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
-		SMU74_Discrete_DpmTable *table)
-{
-	int result = -EINVAL;
-	uint8_t count;
-	struct pp_atomctrl_clock_dividers_vi dividers;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
-			table_info->mm_dep_table;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t vddci;
-
-	table->SamuBootLevel = 0;
-	table->SamuLevelCount = (uint8_t)(mm_table->count);
-
-	for (count = 0; count < table->SamuLevelCount; count++) {
-		/* not sure whether we need evclk or not */
-		table->SamuLevel[count].MinVoltage = 0;
-		table->SamuLevel[count].Frequency = mm_table->entries[count].samclock;
-		table->SamuLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
-				VOLTAGE_SCALE) << VDDC_SHIFT;
-
-		if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
-			vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table),
-						mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
-		else if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
-			vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA;
-		else
-			vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT;
-
-		table->SamuLevel[count].MinVoltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
-		table->SamuLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
-
-		/* retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-				table->SamuLevel[count].Frequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for samu clock", return result);
-
-		table->SamuLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].Frequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].MinVoltage);
-	}
-	return result;
-}
-
-static int polaris10_populate_memory_timing_parameters(struct pp_hwmgr *hwmgr,
-		int32_t eng_clock, int32_t mem_clock,
-		SMU74_Discrete_MCArbDramTimingTableEntry *arb_regs)
-{
-	uint32_t dram_timing;
-	uint32_t dram_timing2;
-	uint32_t burst_time;
-	int result;
-
-	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
-			eng_clock, mem_clock);
-	PP_ASSERT_WITH_CODE(result == 0,
-			"Error calling VBIOS to set DRAM_TIMING.", return result);
-
-	dram_timing = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
-	dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
-	burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
-
-
-	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dram_timing);
-	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dram_timing2);
-	arb_regs->McArbBurstTime   = (uint8_t)burst_time;
-
-	return 0;
-}
-
-static int polaris10_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct SMU74_Discrete_MCArbDramTimingTable arb_regs;
-	uint32_t i, j;
-	int result = 0;
-
-	for (i = 0; i < data->dpm_table.sclk_table.count; i++) {
-		for (j = 0; j < data->dpm_table.mclk_table.count; j++) {
-			result = polaris10_populate_memory_timing_parameters(hwmgr,
-					data->dpm_table.sclk_table.dpm_levels[i].value,
-					data->dpm_table.mclk_table.dpm_levels[j].value,
-					&arb_regs.entries[i][j]);
-			if (result == 0)
-				result = atomctrl_set_ac_timing_ai(hwmgr, data->dpm_table.mclk_table.dpm_levels[j].value, j);
-			if (result != 0)
-				return result;
-		}
-	}
-
-	result = polaris10_copy_bytes_to_smc(
-			hwmgr->smumgr,
-			data->arb_table_start,
-			(uint8_t *)&arb_regs,
-			sizeof(SMU74_Discrete_MCArbDramTimingTable),
-			data->sram_end);
-	return result;
-}
-
-static int polaris10_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
-		struct SMU74_Discrete_DpmTable *table)
-{
-	int result = -EINVAL;
-	uint8_t count;
-	struct pp_atomctrl_clock_dividers_vi dividers;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
-			table_info->mm_dep_table;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t vddci;
-
-	table->UvdLevelCount = (uint8_t)(mm_table->count);
-	table->UvdBootLevel = 0;
-
-	for (count = 0; count < table->UvdLevelCount; count++) {
-		table->UvdLevel[count].MinVoltage = 0;
-		table->UvdLevel[count].VclkFrequency = mm_table->entries[count].vclk;
-		table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk;
-		table->UvdLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
-				VOLTAGE_SCALE) << VDDC_SHIFT;
-
-		if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
-			vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table),
-						mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
-		else if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
-			vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA;
-		else
-			vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT;
-
-		table->UvdLevel[count].MinVoltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
-		table->UvdLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
-
-		/* retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-				table->UvdLevel[count].VclkFrequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for Vclk clock", return result);
-
-		table->UvdLevel[count].VclkDivider = (uint8_t)dividers.pll_post_divider;
-
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-				table->UvdLevel[count].DclkFrequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for Dclk clock", return result);
-
-		table->UvdLevel[count].DclkDivider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].MinVoltage);
-	}
-
-	return result;
-}
-
-static int polaris10_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
-		struct SMU74_Discrete_DpmTable *table)
-{
-	int result = 0;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	table->GraphicsBootLevel = 0;
-	table->MemoryBootLevel = 0;
-
-	/* find boot level from dpm table */
-	result = phm_find_boot_level(&(data->dpm_table.sclk_table),
-			data->vbios_boot_state.sclk_bootup_value,
-			(uint32_t *)&(table->GraphicsBootLevel));
-
-	result = phm_find_boot_level(&(data->dpm_table.mclk_table),
-			data->vbios_boot_state.mclk_bootup_value,
-			(uint32_t *)&(table->MemoryBootLevel));
-
-	table->BootVddc  = data->vbios_boot_state.vddc_bootup_value *
-			VOLTAGE_SCALE;
-	table->BootVddci = data->vbios_boot_state.vddci_bootup_value *
-			VOLTAGE_SCALE;
-	table->BootMVdd  = data->vbios_boot_state.mvdd_bootup_value *
-			VOLTAGE_SCALE;
-
-	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddc);
-	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddci);
-	CONVERT_FROM_HOST_TO_SMC_US(table->BootMVdd);
-
-	return 0;
-}
-
-
-static int polaris10_populate_smc_initailial_state(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint8_t count, level;
-
-	count = (uint8_t)(table_info->vdd_dep_on_sclk->count);
-
-	for (level = 0; level < count; level++) {
-		if (table_info->vdd_dep_on_sclk->entries[level].clk >=
-				data->vbios_boot_state.sclk_bootup_value) {
-			data->smc_state_table.GraphicsBootLevel = level;
-			break;
-		}
-	}
-
-	count = (uint8_t)(table_info->vdd_dep_on_mclk->count);
-	for (level = 0; level < count; level++) {
-		if (table_info->vdd_dep_on_mclk->entries[level].clk >=
-				data->vbios_boot_state.mclk_bootup_value) {
-			data->smc_state_table.MemoryBootLevel = level;
-			break;
-		}
-	}
-
-	return 0;
-}
-
-static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
-{
-	uint32_t ro, efuse, volt_without_cks, volt_with_cks, value, max, min;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint8_t i, stretch_amount, volt_offset = 0;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
-			table_info->vdd_dep_on_sclk;
-
-	stretch_amount = (uint8_t)table_info->cac_dtp_table->usClockStretchAmount;
-
-	/* Read SMU_Eefuse to read and calculate RO and determine
-	 * if the part is SS or FF. if RO >= 1660MHz, part is FF.
-	 */
-	efuse = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixSMU_EFUSE_0 + (67 * 4));
-	efuse &= 0xFF000000;
-	efuse = efuse >> 24;
-
-	if (hwmgr->chip_id == CHIP_POLARIS10) {
-		min = 1000;
-		max = 2300;
-	} else {
-		min = 1100;
-		max = 2100;
-	}
-
-	ro = efuse * (max -min)/255 + min;
-
-	/* Populate Sclk_CKS_masterEn0_7 and Sclk_voltageOffset */
-	for (i = 0; i < sclk_table->count; i++) {
-		data->smc_state_table.Sclk_CKS_masterEn0_7 |=
-				sclk_table->entries[i].cks_enable << i;
-		if (hwmgr->chip_id == CHIP_POLARIS10) {
-			volt_without_cks = (uint32_t)((2753594000U + (sclk_table->entries[i].clk/100) * 136418 -(ro - 70) * 1000000) / \
-						(2424180 - (sclk_table->entries[i].clk/100) * 1132925/1000));
-			volt_with_cks = (uint32_t)((2797202000U + sclk_table->entries[i].clk/100 * 3232 - (ro - 65) * 1000000) / \
-					(2522480 - sclk_table->entries[i].clk/100 * 115764/100));
-		} else {
-			volt_without_cks = (uint32_t)((2416794800U + (sclk_table->entries[i].clk/100) * 1476925/10 -(ro - 50) * 1000000) / \
-						(2625416 - (sclk_table->entries[i].clk/100) * (12586807/10000)));
-			volt_with_cks = (uint32_t)((2999656000U - sclk_table->entries[i].clk/100 * 392803 - (ro - 44) * 1000000) / \
-					(3422454 - sclk_table->entries[i].clk/100 * (18886376/10000)));
-		}
-
-		if (volt_without_cks >= volt_with_cks)
-			volt_offset = (uint8_t)(((volt_without_cks - volt_with_cks +
-					sclk_table->entries[i].cks_voffset) * 100 + 624) / 625);
-
-		data->smc_state_table.Sclk_voltageOffset[i] = volt_offset;
-	}
-
-	data->smc_state_table.LdoRefSel = (table_info->cac_dtp_table->ucCKS_LDO_REFSEL != 0) ? table_info->cac_dtp_table->ucCKS_LDO_REFSEL : 6;
-	/* Populate CKS Lookup Table */
-	if (stretch_amount != 1 && stretch_amount != 2 && stretch_amount != 3 &&
-			stretch_amount != 4 && stretch_amount != 5) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_ClockStretcher);
-		PP_ASSERT_WITH_CODE(false,
-				"Stretch Amount in PPTable not supported\n",
-				return -EINVAL);
-	}
-
-	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL);
-	value &= 0xFFFFFFFE;
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL, value);
-
-	return 0;
-}
-
-/**
-* Populates the SMC VRConfig field in DPM table.
-*
-* @param    hwmgr   the address of the hardware manager
-* @param    table   the SMC DPM table structure to be populated
-* @return   always 0
-*/
-static int polaris10_populate_vr_config(struct pp_hwmgr *hwmgr,
-		struct SMU74_Discrete_DpmTable *table)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint16_t config;
-
-	config = VR_MERGED_WITH_VDDC;
-	table->VRConfig |= (config << VRCONF_VDDGFX_SHIFT);
-
-	/* Set Vddc Voltage Controller */
-	if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
-		config = VR_SVI2_PLANE_1;
-		table->VRConfig |= config;
-	} else {
-		PP_ASSERT_WITH_CODE(false,
-				"VDDC should be on SVI2 control in merged mode!",
-				);
-	}
-	/* Set Vddci Voltage Controller */
-	if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
-		config = VR_SVI2_PLANE_2;  /* only in merged mode */
-		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
-	} else if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
-		config = VR_SMIO_PATTERN_1;
-		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
-	} else {
-		config = VR_STATIC_VOLTAGE;
-		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
-	}
-	/* Set Mvdd Voltage Controller */
-	if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
-		config = VR_SVI2_PLANE_2;
-		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start +
-			offsetof(SMU74_SoftRegisters, AllowMvddSwitch), 0x1);
-	} else {
-		config = VR_STATIC_VOLTAGE;
-		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
-	}
-
-	return 0;
-}
-
-
-int polaris10_populate_avfs_parameters(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	SMU74_Discrete_DpmTable  *table = &(data->smc_state_table);
-	int result = 0;
-	struct pp_atom_ctrl__avfs_parameters avfs_params = {0};
-	AVFS_meanNsigma_t AVFS_meanNsigma = { {0} };
-	AVFS_Sclk_Offset_t AVFS_SclkOffset = { {0} };
-	uint32_t tmp, i;
-	struct pp_smumgr *smumgr = hwmgr->smumgr;
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
-
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)hwmgr->pptable;
-	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
-			table_info->vdd_dep_on_sclk;
-
-
-	if (smu_data->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED)
-		return result;
-
-	result = atomctrl_get_avfs_information(hwmgr, &avfs_params);
-
-	if (0 == result) {
-		table->BTCGB_VDROOP_TABLE[0].a0  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a0);
-		table->BTCGB_VDROOP_TABLE[0].a1  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a1);
-		table->BTCGB_VDROOP_TABLE[0].a2  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a2);
-		table->BTCGB_VDROOP_TABLE[1].a0  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a0);
-		table->BTCGB_VDROOP_TABLE[1].a1  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a1);
-		table->BTCGB_VDROOP_TABLE[1].a2  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a2);
-		table->AVFSGB_VDROOP_TABLE[0].m1 = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSON_m1);
-		table->AVFSGB_VDROOP_TABLE[0].m2 = PP_HOST_TO_SMC_US(avfs_params.usAVFSGB_FUSE_TABLE_CKSON_m2);
-		table->AVFSGB_VDROOP_TABLE[0].b  = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSON_b);
-		table->AVFSGB_VDROOP_TABLE[0].m1_shift = 24;
-		table->AVFSGB_VDROOP_TABLE[0].m2_shift  = 12;
-		table->AVFSGB_VDROOP_TABLE[1].m1 = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSOFF_m1);
-		table->AVFSGB_VDROOP_TABLE[1].m2 = PP_HOST_TO_SMC_US(avfs_params.usAVFSGB_FUSE_TABLE_CKSOFF_m2);
-		table->AVFSGB_VDROOP_TABLE[1].b  = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSOFF_b);
-		table->AVFSGB_VDROOP_TABLE[1].m1_shift = 24;
-		table->AVFSGB_VDROOP_TABLE[1].m2_shift  = 12;
-		table->MaxVoltage                = PP_HOST_TO_SMC_US(avfs_params.usMaxVoltage_0_25mv);
-		AVFS_meanNsigma.Aconstant[0]      = PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant0);
-		AVFS_meanNsigma.Aconstant[1]      = PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant1);
-		AVFS_meanNsigma.Aconstant[2]      = PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant2);
-		AVFS_meanNsigma.DC_tol_sigma      = PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_DC_tol_sigma);
-		AVFS_meanNsigma.Platform_mean     = PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_Platform_mean);
-		AVFS_meanNsigma.PSM_Age_CompFactor = PP_HOST_TO_SMC_US(avfs_params.usPSM_Age_ComFactor);
-		AVFS_meanNsigma.Platform_sigma     = PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_Platform_sigma);
-
-		for (i = 0; i < NUM_VFT_COLUMNS; i++) {
-			AVFS_meanNsigma.Static_Voltage_Offset[i] = (uint8_t)(sclk_table->entries[i].cks_voffset * 100 / 625);
-			AVFS_SclkOffset.Sclk_Offset[i] = PP_HOST_TO_SMC_US((uint16_t)(sclk_table->entries[i].sclk_offset) / 100);
-		}
-
-		result = polaris10_read_smc_sram_dword(smumgr,
-				SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU74_Firmware_Header, AvfsMeanNSigma),
-				&tmp, data->sram_end);
-
-		polaris10_copy_bytes_to_smc(smumgr,
-					tmp,
-					(uint8_t *)&AVFS_meanNsigma,
-					sizeof(AVFS_meanNsigma_t),
-					data->sram_end);
-
-		result = polaris10_read_smc_sram_dword(smumgr,
-				SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU74_Firmware_Header, AvfsSclkOffsetTable),
-				&tmp, data->sram_end);
-		polaris10_copy_bytes_to_smc(smumgr,
-					tmp,
-					(uint8_t *)&AVFS_SclkOffset,
-					sizeof(AVFS_Sclk_Offset_t),
-					data->sram_end);
-
-		data->avfs_vdroop_override_setting = (avfs_params.ucEnableGB_VDROOP_TABLE_CKSON << BTCGB0_Vdroop_Enable_SHIFT) |
-						(avfs_params.ucEnableGB_VDROOP_TABLE_CKSOFF << BTCGB1_Vdroop_Enable_SHIFT) |
-						(avfs_params.ucEnableGB_FUSE_TABLE_CKSON << AVFSGB0_Vdroop_Enable_SHIFT) |
-						(avfs_params.ucEnableGB_FUSE_TABLE_CKSOFF << AVFSGB1_Vdroop_Enable_SHIFT);
-		data->apply_avfs_cks_off_voltage = (avfs_params.ucEnableApplyAVFS_CKS_OFF_Voltage == 1) ? true : false;
-	}
-	return result;
-}
-
-
-/**
-* Initializes the SMC table and uploads it
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int polaris10_init_smc_table(struct pp_hwmgr *hwmgr)
-{
-	int result;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct SMU74_Discrete_DpmTable *table = &(data->smc_state_table);
-	const struct polaris10_ulv_parm *ulv = &(data->ulv);
-	uint8_t i;
-	struct pp_atomctrl_gpio_pin_assignment gpio_pin;
-	pp_atomctrl_clock_dividers_vi dividers;
-
-	result = polaris10_setup_default_dpm_tables(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to setup default DPM tables!", return result);
-
-	if (POLARIS10_VOLTAGE_CONTROL_NONE != data->voltage_control)
-		polaris10_populate_smc_voltage_tables(hwmgr, table);
-
-	table->SystemFlags = 0;
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_AutomaticDCTransition))
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_StepVddc))
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
-
-	if (data->is_memory_gddr5)
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
-
-	if (ulv->ulv_supported && table_info->us_ulv_voltage_offset) {
-		result = polaris10_populate_ulv_state(hwmgr, table);
-		PP_ASSERT_WITH_CODE(0 == result,
-				"Failed to initialize ULV state!", return result);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				ixCG_ULV_PARAMETER, PPPOLARIS10_CGULVPARAMETER_DFLT);
-	}
-
-	result = polaris10_populate_smc_link_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Link Level!", return result);
-
-	result = polaris10_populate_all_graphic_levels(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Graphics Level!", return result);
-
-	result = polaris10_populate_all_memory_levels(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Memory Level!", return result);
-
-	result = polaris10_populate_smc_acpi_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize ACPI Level!", return result);
-
-	result = polaris10_populate_smc_vce_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize VCE Level!", return result);
-
-	result = polaris10_populate_smc_samu_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize SAMU Level!", return result);
-
-	/* Since only the initial state is completely set up at this point
-	 * (the other states are just copies of the boot state) we only
-	 * need to populate the  ARB settings for the initial state.
-	 */
-	result = polaris10_program_memory_timing_parameters(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to Write ARB settings for the initial state.", return result);
-
-	result = polaris10_populate_smc_uvd_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize UVD Level!", return result);
-
-	result = polaris10_populate_smc_boot_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Boot Level!", return result);
-
-	result = polaris10_populate_smc_initailial_state(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize Boot State!", return result);
-
-	result = polaris10_populate_bapm_parameters_in_dpm_table(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to populate BAPM Parameters!", return result);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ClockStretcher)) {
-		result = polaris10_populate_clock_stretcher_data_table(hwmgr);
-		PP_ASSERT_WITH_CODE(0 == result,
-				"Failed to populate Clock Stretcher Data Table!",
-				return result);
-	}
-
-	result = polaris10_populate_avfs_parameters(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result, "Failed to populate AVFS Parameters!", return result;);
-
-	table->CurrSclkPllRange = 0xff;
-	table->GraphicsVoltageChangeEnable  = 1;
-	table->GraphicsThermThrottleEnable  = 1;
-	table->GraphicsInterval = 1;
-	table->VoltageInterval  = 1;
-	table->ThermalInterval  = 1;
-	table->TemperatureLimitHigh =
-			table_info->cac_dtp_table->usTargetOperatingTemp *
-			POLARIS10_Q88_FORMAT_CONVERSION_UNIT;
-	table->TemperatureLimitLow  =
-			(table_info->cac_dtp_table->usTargetOperatingTemp - 1) *
-			POLARIS10_Q88_FORMAT_CONVERSION_UNIT;
-	table->MemoryVoltageChangeEnable = 1;
-	table->MemoryInterval = 1;
-	table->VoltageResponseTime = 0;
-	table->PhaseResponseTime = 0;
-	table->MemoryThermThrottleEnable = 1;
-	table->PCIeBootLinkLevel = 0;
-	table->PCIeGenInterval = 1;
-	table->VRConfig = 0;
-
-	result = polaris10_populate_vr_config(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to populate VRConfig setting!", return result);
-
-	table->ThermGpio = 17;
-	table->SclkStepSize = 0x4000;
-
-	if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_VRHOT_GPIO_PINID, &gpio_pin)) {
-		table->VRHotGpio = gpio_pin.uc_gpio_pin_bit_shift;
-	} else {
-		table->VRHotGpio = POLARIS10_UNUSED_GPIO_PIN;
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_RegulatorHot);
-	}
-
-	if (atomctrl_get_pp_assign_pin(hwmgr, PP_AC_DC_SWITCH_GPIO_PINID,
-			&gpio_pin)) {
-		table->AcDcGpio = gpio_pin.uc_gpio_pin_bit_shift;
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_AutomaticDCTransition);
-	} else {
-		table->AcDcGpio = POLARIS10_UNUSED_GPIO_PIN;
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_AutomaticDCTransition);
-	}
-
-	/* Thermal Output GPIO */
-	if (atomctrl_get_pp_assign_pin(hwmgr, THERMAL_INT_OUTPUT_GPIO_PINID,
-			&gpio_pin)) {
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_ThermalOutGPIO);
-
-		table->ThermOutGpio = gpio_pin.uc_gpio_pin_bit_shift;
-
-		/* For porlarity read GPIOPAD_A with assigned Gpio pin
-		 * since VBIOS will program this register to set 'inactive state',
-		 * driver can then determine 'active state' from this and
-		 * program SMU with correct polarity
-		 */
-		table->ThermOutPolarity = (0 == (cgs_read_register(hwmgr->device, mmGPIOPAD_A)
-					& (1 << gpio_pin.uc_gpio_pin_bit_shift))) ? 1:0;
-		table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_ONLY;
-
-		/* if required, combine VRHot/PCC with thermal out GPIO */
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_RegulatorHot)
-		&& phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_CombinePCCWithThermalSignal))
-			table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_VRHOT;
-	} else {
-		table->ThermOutGpio = 17;
-		table->ThermOutPolarity = 1;
-		table->ThermOutMode = SMU7_THERM_OUT_MODE_DISABLE;
-	}
-
-	/* Populate BIF_SCLK levels into SMC DPM table */
-	for (i = 0; i <= data->dpm_table.pcie_speed_table.count; i++) {
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr, data->bif_sclk_table[i], &dividers);
-		PP_ASSERT_WITH_CODE((result == 0), "Can not find DFS divide id for Sclk", return result);
-
-		if (i == 0)
-			table->Ulv.BifSclkDfs = PP_HOST_TO_SMC_US((USHORT)(dividers.pll_post_divider));
-		else
-			table->LinkLevel[i-1].BifSclkDfs = PP_HOST_TO_SMC_US((USHORT)(dividers.pll_post_divider));
-	}
-
-	for (i = 0; i < SMU74_MAX_ENTRIES_SMIO; i++)
-		table->Smio[i] = PP_HOST_TO_SMC_UL(table->Smio[i]);
-
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->VRConfig);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask1);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask2);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->CurrSclkPllRange);
-	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
-	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
-	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
-	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
-
-	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
-	result = polaris10_copy_bytes_to_smc(hwmgr->smumgr,
-			data->dpm_table_start +
-			offsetof(SMU74_Discrete_DpmTable, SystemFlags),
-			(uint8_t *)&(table->SystemFlags),
-			sizeof(SMU74_Discrete_DpmTable) - 3 * sizeof(SMU74_PIDController),
-			data->sram_end);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to upload dpm data to SMC memory!", return result);
-
-	return 0;
-}
-
-/**
-* Initialize the ARB DRAM timing table's index field.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int polaris10_init_arb_table_index(struct pp_hwmgr *hwmgr)
-{
-	const struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t tmp;
-	int result;
-
-	/* This is a read-modify-write on the first byte of the ARB table.
-	 * The first byte in the SMU73_Discrete_MCArbDramTimingTable structure
-	 * is the field 'current'.
-	 * This solution is ugly, but we never write the whole table only
-	 * individual fields in it.
-	 * In reality this field should not be in that structure
-	 * but in a soft register.
-	 */
-	result = polaris10_read_smc_sram_dword(hwmgr->smumgr,
-			data->arb_table_start, &tmp, data->sram_end);
-
-	if (result)
-		return result;
-
-	tmp &= 0x00FFFFFF;
-	tmp |= ((uint32_t)MC_CG_ARB_FREQ_F1) << 24;
-
-	return polaris10_write_smc_sram_dword(hwmgr->smumgr,
-			data->arb_table_start, tmp, data->sram_end);
-}
-
-static int polaris10_enable_vrhot_gpio_interrupt(struct pp_hwmgr *hwmgr)
-{
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_RegulatorHot))
-		return smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_EnableVRHotGPIOInterrupt);
-
-	return 0;
-}
-
-static int polaris10_enable_sclk_control(struct pp_hwmgr *hwmgr)
-{
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
-			SCLK_PWRMGT_OFF, 0);
-	return 0;
-}
-
-static int polaris10_enable_ulv(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_ulv_parm *ulv = &(data->ulv);
-
-	if (ulv->ulv_supported)
-		return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_EnableULV);
-
-	return 0;
-}
-
-static int polaris10_disable_ulv(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_ulv_parm *ulv = &(data->ulv);
-
-	if (ulv->ulv_supported)
-		return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DisableULV);
-
-	return 0;
-}
-
-static int polaris10_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
-{
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkDeepSleep)) {
-		if (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_MASTER_DeepSleep_ON))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to enable Master Deep Sleep switch failed!",
-					return -1);
-	} else {
-		if (smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_MASTER_DeepSleep_OFF)) {
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to disable Master Deep Sleep switch failed!",
-					return -1);
-		}
-	}
-
-	return 0;
-}
-
-static int polaris10_disable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
-{
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkDeepSleep)) {
-		if (smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_MASTER_DeepSleep_OFF)) {
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to disable Master Deep Sleep switch failed!",
-					return -1);
-		}
-	}
-
-	return 0;
-}
-
-static int polaris10_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t soft_register_value = 0;
-	uint32_t handshake_disables_offset = data->soft_regs_start
-				+ offsetof(SMU74_SoftRegisters, HandshakeDisables);
-
-	/* enable SCLK dpm */
-	if (!data->sclk_dpm_key_disabled)
-		PP_ASSERT_WITH_CODE(
-		(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DPM_Enable)),
-		"Failed to enable SCLK DPM during DPM Start Function!",
-		return -1);
-
-	/* enable MCLK dpm */
-	if (0 == data->mclk_dpm_key_disabled) {
-/* Disable UVD - SMU handshake for MCLK. */
-		soft_register_value = cgs_read_ind_register(hwmgr->device,
-					CGS_IND_REG__SMC, handshake_disables_offset);
-		soft_register_value |= SMU7_UVD_MCLK_HANDSHAKE_DISABLE;
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-				handshake_disables_offset, soft_register_value);
-
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_MCLKDPM_Enable)),
-				"Failed to enable MCLK DPM during DPM Start Function!",
-				return -1);
-
-		PHM_WRITE_FIELD(hwmgr->device, MC_SEQ_CNTL_3, CAC_EN, 0x1);
-
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC0_CNTL, 0x5);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC1_CNTL, 0x5);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_CPL_CNTL, 0x100005);
-		udelay(10);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC0_CNTL, 0x400005);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC1_CNTL, 0x400005);
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_CPL_CNTL, 0x500005);
-	}
-
-	return 0;
-}
-
-static int polaris10_start_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	/*enable general power management */
-
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
-			GLOBAL_PWRMGT_EN, 1);
-
-	/* enable sclk deep sleep */
-
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
-			DYNAMIC_PM_EN, 1);
-
-	/* prepare for PCIE DPM */
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			data->soft_regs_start + offsetof(SMU74_SoftRegisters,
-					VoltageChangeTimeout), 0x1000);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE,
-			SWRST_COMMAND_1, RESETLC, 0x0);
-/*
-	PP_ASSERT_WITH_CODE(
-			(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					PPSMC_MSG_Voltage_Cntl_Enable)),
-			"Failed to enable voltage DPM during DPM Start Function!",
-			return -1);
-*/
-
-	if (polaris10_enable_sclk_mclk_dpm(hwmgr)) {
-		printk(KERN_ERR "Failed to enable Sclk DPM and Mclk DPM!");
-		return -1;
-	}
-
-	/* enable PCIE dpm */
-	if (0 == data->pcie_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_PCIeDPM_Enable)),
-				"Failed to enable pcie DPM during DPM Start Function!",
-				return -1);
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_Falcon_QuickTransition)) {
-		PP_ASSERT_WITH_CODE((0 == smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_EnableACDCGPIOInterrupt)),
-				"Failed to enable AC DC GPIO Interrupt!",
-				);
-	}
-
-	return 0;
-}
-
-static int polaris10_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	/* disable SCLK dpm */
-	if (!data->sclk_dpm_key_disabled)
-		PP_ASSERT_WITH_CODE(
-				(smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_DPM_Disable) == 0),
-				"Failed to disable SCLK DPM!",
-				return -1);
-
-	/* disable MCLK dpm */
-	if (!data->mclk_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE(
-				(smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_MCLKDPM_Disable) == 0),
-				"Failed to disable MCLK DPM!",
-				return -1);
-	}
-
-	return 0;
-}
-
-static int polaris10_stop_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	/* disable general power management */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
-			GLOBAL_PWRMGT_EN, 0);
-	/* disable sclk deep sleep */
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
-			DYNAMIC_PM_EN, 0);
-
-	/* disable PCIE dpm */
-	if (!data->pcie_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE(
-				(smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_PCIeDPM_Disable) == 0),
-				"Failed to disable pcie DPM during DPM Stop Function!",
-				return -1);
-	}
-
-	if (polaris10_disable_sclk_mclk_dpm(hwmgr)) {
-		printk(KERN_ERR "Failed to disable Sclk DPM and Mclk DPM!");
-		return -1;
-	}
-
-	return 0;
-}
-
-static void polaris10_set_dpm_event_sources(struct pp_hwmgr *hwmgr, uint32_t sources)
-{
-	bool protection;
-	enum DPM_EVENT_SRC src;
-
-	switch (sources) {
-	default:
-		printk(KERN_ERR "Unknown throttling event sources.");
-		/* fall through */
-	case 0:
-		protection = false;
-		/* src is unused */
-		break;
-	case (1 << PHM_AutoThrottleSource_Thermal):
-		protection = true;
-		src = DPM_EVENT_SRC_DIGITAL;
-		break;
-	case (1 << PHM_AutoThrottleSource_External):
-		protection = true;
-		src = DPM_EVENT_SRC_EXTERNAL;
-		break;
-	case (1 << PHM_AutoThrottleSource_External) |
-			(1 << PHM_AutoThrottleSource_Thermal):
-		protection = true;
-		src = DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL;
-		break;
-	}
-	/* Order matters - don't enable thermal protection for the wrong source. */
-	if (protection) {
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_CTRL,
-				DPM_EVENT_SRC, src);
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
-				THERMAL_PROTECTION_DIS,
-				!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_ThermalController));
-	} else
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
-				THERMAL_PROTECTION_DIS, 1);
-}
-
-static int polaris10_enable_auto_throttle_source(struct pp_hwmgr *hwmgr,
-		PHM_AutoThrottleSource source)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if (!(data->active_auto_throttle_sources & (1 << source))) {
-		data->active_auto_throttle_sources |= 1 << source;
-		polaris10_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources);
-	}
-	return 0;
-}
-
-static int polaris10_enable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
-{
-	return polaris10_enable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
-}
-
-static int polaris10_disable_auto_throttle_source(struct pp_hwmgr *hwmgr,
-		PHM_AutoThrottleSource source)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if (data->active_auto_throttle_sources & (1 << source)) {
-		data->active_auto_throttle_sources &= ~(1 << source);
-		polaris10_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources);
-	}
-	return 0;
-}
-
-static int polaris10_disable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
-{
-	return polaris10_disable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
-}
-
-int polaris10_pcie_performance_request(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	data->pcie_performance_request = true;
-
-	return 0;
-}
-
-int polaris10_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
-{
-	int tmp_result, result = 0;
-	tmp_result = (!polaris10_is_dpm_running(hwmgr)) ? 0 : -1;
-	PP_ASSERT_WITH_CODE(result == 0,
-			"DPM is already running right now, no need to enable DPM!",
-			return 0);
-
-	if (polaris10_voltage_control(hwmgr)) {
-		tmp_result = polaris10_enable_voltage_control(hwmgr);
-		PP_ASSERT_WITH_CODE(tmp_result == 0,
-				"Failed to enable voltage control!",
-				result = tmp_result);
-
-		tmp_result = polaris10_construct_voltage_tables(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == tmp_result),
-				"Failed to contruct voltage tables!",
-				result = tmp_result);
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_EngineSpreadSpectrumSupport))
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 1);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ThermalController))
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 0);
-
-	tmp_result = polaris10_program_static_screen_threshold_parameters(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to program static screen threshold parameters!",
-			result = tmp_result);
-
-	tmp_result = polaris10_enable_display_gap(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable display gap!", result = tmp_result);
-
-	tmp_result = polaris10_program_voting_clients(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to program voting clients!", result = tmp_result);
-
-	tmp_result = polaris10_process_firmware_header(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to process firmware header!", result = tmp_result);
-
-	tmp_result = polaris10_initial_switch_from_arbf0_to_f1(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to initialize switch from ArbF0 to F1!",
-			result = tmp_result);
-
-	tmp_result = polaris10_init_smc_table(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to initialize SMC table!", result = tmp_result);
-
-	tmp_result = polaris10_init_arb_table_index(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to initialize ARB table index!", result = tmp_result);
-
-	tmp_result = polaris10_populate_pm_fuses(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to populate PM fuses!", result = tmp_result);
-
-	tmp_result = polaris10_enable_vrhot_gpio_interrupt(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable VR hot GPIO interrupt!", result = tmp_result);
-
-	smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_HasDisplay);
-
-	tmp_result = polaris10_enable_sclk_control(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable SCLK control!", result = tmp_result);
-
-	tmp_result = polaris10_enable_smc_voltage_controller(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable voltage control!", result = tmp_result);
-
-	tmp_result = polaris10_enable_ulv(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable ULV!", result = tmp_result);
-
-	tmp_result = polaris10_enable_deep_sleep_master_switch(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable deep sleep master switch!", result = tmp_result);
-
-	tmp_result = polaris10_enable_didt_config(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to enable deep sleep master switch!", result = tmp_result);
-
-	tmp_result = polaris10_start_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to start DPM!", result = tmp_result);
-
-	tmp_result = polaris10_enable_smc_cac(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable SMC CAC!", result = tmp_result);
-
-	tmp_result = polaris10_enable_power_containment(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable power containment!", result = tmp_result);
-
-	tmp_result = polaris10_power_control_set_level(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to power control set level!", result = tmp_result);
-
-	tmp_result = polaris10_enable_thermal_auto_throttle(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable thermal auto throttle!", result = tmp_result);
-
-	tmp_result = polaris10_pcie_performance_request(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"pcie performance request failed!", result = tmp_result);
-
-	return result;
-}
-
-int polaris10_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
-{
-	int tmp_result, result = 0;
-
-	tmp_result = (polaris10_is_dpm_running(hwmgr)) ? 0 : -1;
-	PP_ASSERT_WITH_CODE(tmp_result == 0,
-			"DPM is not running right now, no need to disable DPM!",
-			return 0);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ThermalController))
-		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 1);
-
-	tmp_result = polaris10_disable_power_containment(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable power containment!", result = tmp_result);
-
-	tmp_result = polaris10_disable_smc_cac(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable SMC CAC!", result = tmp_result);
-
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_SPLL_SPREAD_SPECTRUM, SSEN, 0);
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 0);
-
-	tmp_result = polaris10_disable_thermal_auto_throttle(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable thermal auto throttle!", result = tmp_result);
-
-	tmp_result = polaris10_stop_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to stop DPM!", result = tmp_result);
-
-	tmp_result = polaris10_disable_deep_sleep_master_switch(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable deep sleep master switch!", result = tmp_result);
-
-	tmp_result = polaris10_disable_ulv(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to disable ULV!", result = tmp_result);
-
-	tmp_result = polaris10_clear_voting_clients(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to clear voting clients!", result = tmp_result);
-
-	tmp_result = polaris10_reset_to_default(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to reset to default!", result = tmp_result);
-
-	tmp_result = polaris10_force_switch_to_arbf0(hwmgr);
-	PP_ASSERT_WITH_CODE((tmp_result == 0),
-			"Failed to force to switch arbf0!", result = tmp_result);
-
-	return result;
-}
-
-int polaris10_reset_asic_tasks(struct pp_hwmgr *hwmgr)
-{
-
-	return 0;
-}
-
-int polaris10_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
-{
-	return phm_hwmgr_backend_fini(hwmgr);
-}
-
-int polaris10_set_features_platform_caps(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkDeepSleep);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_DynamicPatchPowerState);
-
-	if (data->mvdd_control == POLARIS10_VOLTAGE_CONTROL_NONE)
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_EnableMVDDControl);
-
-	if (data->vddci_control == POLARIS10_VOLTAGE_CONTROL_NONE)
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_ControlVDDCI);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			 PHM_PlatformCaps_TablelessHardwareInterface);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_EnableSMU7ThermalManagement);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_DynamicPowerManagement);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_UnTabledHardwareInterface);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_TablelessHardwareInterface);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_SMC);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_NonABMSupportInPPLib);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_DynamicUVDState);
-
-	/* power tune caps Assume disabled */
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_SQRamping);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_DBRamping);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_TDRamping);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_TCPRamping);
-
-	if (hwmgr->powercontainment_enabled)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			    PHM_PlatformCaps_PowerContainment);
-	else
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			    PHM_PlatformCaps_PowerContainment);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-							PHM_PlatformCaps_CAC);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_RegulatorHot);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_AutomaticDCTransition);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_ODFuzzyFanControlSupport);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_FanSpeedInTableIsRPM);
-
-	if (hwmgr->chip_id == CHIP_POLARIS11)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_SPLLShutdownSupport);
-	return 0;
-}
-
-static void polaris10_init_dpm_defaults(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	polaris10_initialize_power_tune_defaults(hwmgr);
-
-	data->pcie_gen_performance.max = PP_PCIEGen1;
-	data->pcie_gen_performance.min = PP_PCIEGen3;
-	data->pcie_gen_power_saving.max = PP_PCIEGen1;
-	data->pcie_gen_power_saving.min = PP_PCIEGen3;
-	data->pcie_lane_performance.max = 0;
-	data->pcie_lane_performance.min = 16;
-	data->pcie_lane_power_saving.max = 0;
-	data->pcie_lane_power_saving.min = 16;
-}
-
-/**
-* Get Leakage VDDC based on leakage ID.
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always 0
-*/
-static int polaris10_get_evv_voltages(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint16_t vv_id;
-	uint32_t vddc = 0;
-	uint16_t i, j;
-	uint32_t sclk = 0;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)hwmgr->pptable;
-	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
-			table_info->vdd_dep_on_sclk;
-	int result;
-
-	for (i = 0; i < POLARIS10_MAX_LEAKAGE_COUNT; i++) {
-		vv_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
-		if (!phm_get_sclk_for_voltage_evv(hwmgr,
-				table_info->vddc_lookup_table, vv_id, &sclk)) {
-			if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_ClockStretcher)) {
-				for (j = 1; j < sclk_table->count; j++) {
-					if (sclk_table->entries[j].clk == sclk &&
-							sclk_table->entries[j].cks_enable == 0) {
-						sclk += 5000;
-						break;
-					}
-				}
-			}
-
-			if (atomctrl_get_voltage_evv_on_sclk_ai(hwmgr,
-						VOLTAGE_TYPE_VDDC,
-						sclk, vv_id, &vddc) != 0) {
-				printk(KERN_WARNING "failed to retrieving EVV voltage!\n");
-				continue;
-			}
-
-			/* need to make sure vddc is less than 2v or else, it could burn the ASIC.
-			 * real voltage level in unit of 0.01mv */
-			PP_ASSERT_WITH_CODE((vddc < 200000 && vddc != 0),
-					"Invalid VDDC value", result = -EINVAL;);
-
-			/* the voltage should not be zero nor equal to leakage ID */
-			if (vddc != 0 && vddc != vv_id) {
-				data->vddc_leakage.actual_voltage[data->vddc_leakage.count] = (uint16_t)(vddc/100);
-				data->vddc_leakage.leakage_id[data->vddc_leakage.count] = vv_id;
-				data->vddc_leakage.count++;
-			}
-		}
-	}
-
-	return 0;
-}
-
-/**
- * Change virtual leakage voltage to actual value.
- *
- * @param     hwmgr  the address of the powerplay hardware manager.
- * @param     pointer to changing voltage
- * @param     pointer to leakage table
- */
-static void polaris10_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr,
-		uint16_t *voltage, struct polaris10_leakage_voltage *leakage_table)
-{
-	uint32_t index;
-
-	/* search for leakage voltage ID 0xff01 ~ 0xff08 */
-	for (index = 0; index < leakage_table->count; index++) {
-		/* if this voltage matches a leakage voltage ID */
-		/* patch with actual leakage voltage */
-		if (leakage_table->leakage_id[index] == *voltage) {
-			*voltage = leakage_table->actual_voltage[index];
-			break;
-		}
-	}
-
-	if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
-		printk(KERN_ERR "Voltage value looks like a Leakage ID but it's not patched \n");
-}
-
-/**
-* Patch voltage lookup table by EVV leakages.
-*
-* @param     hwmgr  the address of the powerplay hardware manager.
-* @param     pointer to voltage lookup table
-* @param     pointer to leakage table
-* @return     always 0
-*/
-static int polaris10_patch_lookup_table_with_leakage(struct pp_hwmgr *hwmgr,
-		phm_ppt_v1_voltage_lookup_table *lookup_table,
-		struct polaris10_leakage_voltage *leakage_table)
-{
-	uint32_t i;
-
-	for (i = 0; i < lookup_table->count; i++)
-		polaris10_patch_with_vdd_leakage(hwmgr,
-				&lookup_table->entries[i].us_vdd, leakage_table);
-
-	return 0;
-}
-
-static int polaris10_patch_clock_voltage_limits_with_vddc_leakage(
-		struct pp_hwmgr *hwmgr, struct polaris10_leakage_voltage *leakage_table,
-		uint16_t *vddc)
-{
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	polaris10_patch_with_vdd_leakage(hwmgr, (uint16_t *)vddc, leakage_table);
-	hwmgr->dyn_state.max_clock_voltage_on_dc.vddc =
-			table_info->max_clock_voltage_on_dc.vddc;
-	return 0;
-}
-
-static int polaris10_patch_voltage_dependency_tables_with_lookup_table(
-		struct pp_hwmgr *hwmgr)
-{
-	uint8_t entryId;
-	uint8_t voltageId;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
-			table_info->vdd_dep_on_sclk;
-	struct phm_ppt_v1_clock_voltage_dependency_table *mclk_table =
-			table_info->vdd_dep_on_mclk;
-	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
-			table_info->mm_dep_table;
-
-	for (entryId = 0; entryId < sclk_table->count; ++entryId) {
-		voltageId = sclk_table->entries[entryId].vddInd;
-		sclk_table->entries[entryId].vddc =
-				table_info->vddc_lookup_table->entries[voltageId].us_vdd;
-	}
-
-	for (entryId = 0; entryId < mclk_table->count; ++entryId) {
-		voltageId = mclk_table->entries[entryId].vddInd;
-		mclk_table->entries[entryId].vddc =
-			table_info->vddc_lookup_table->entries[voltageId].us_vdd;
-	}
-
-	for (entryId = 0; entryId < mm_table->count; ++entryId) {
-		voltageId = mm_table->entries[entryId].vddcInd;
-		mm_table->entries[entryId].vddc =
-			table_info->vddc_lookup_table->entries[voltageId].us_vdd;
-	}
-
-	return 0;
-
-}
-
-static int polaris10_calc_voltage_dependency_tables(struct pp_hwmgr *hwmgr)
-{
-	/* Need to determine if we need calculated voltage. */
-	return 0;
-}
-
-static int polaris10_calc_mm_voltage_dependency_table(struct pp_hwmgr *hwmgr)
-{
-	/* Need to determine if we need calculated voltage from mm table. */
-	return 0;
-}
-
-static int polaris10_sort_lookup_table(struct pp_hwmgr *hwmgr,
-		struct phm_ppt_v1_voltage_lookup_table *lookup_table)
-{
-	uint32_t table_size, i, j;
-	struct phm_ppt_v1_voltage_lookup_record tmp_voltage_lookup_record;
-	table_size = lookup_table->count;
-
-	PP_ASSERT_WITH_CODE(0 != lookup_table->count,
-		"Lookup table is empty", return -EINVAL);
-
-	/* Sorting voltages */
-	for (i = 0; i < table_size - 1; i++) {
-		for (j = i + 1; j > 0; j--) {
-			if (lookup_table->entries[j].us_vdd <
-					lookup_table->entries[j - 1].us_vdd) {
-				tmp_voltage_lookup_record = lookup_table->entries[j - 1];
-				lookup_table->entries[j - 1] = lookup_table->entries[j];
-				lookup_table->entries[j] = tmp_voltage_lookup_record;
-			}
-		}
-	}
-
-	return 0;
-}
-
-static int polaris10_complete_dependency_tables(struct pp_hwmgr *hwmgr)
-{
-	int result = 0;
-	int tmp_result;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	tmp_result = polaris10_patch_lookup_table_with_leakage(hwmgr,
-			table_info->vddc_lookup_table, &(data->vddc_leakage));
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = polaris10_patch_clock_voltage_limits_with_vddc_leakage(hwmgr,
-			&(data->vddc_leakage), &table_info->max_clock_voltage_on_dc.vddc);
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = polaris10_patch_voltage_dependency_tables_with_lookup_table(hwmgr);
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = polaris10_calc_voltage_dependency_tables(hwmgr);
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = polaris10_calc_mm_voltage_dependency_table(hwmgr);
-	if (tmp_result)
-		result = tmp_result;
-
-	tmp_result = polaris10_sort_lookup_table(hwmgr, table_info->vddc_lookup_table);
-	if (tmp_result)
-		result = tmp_result;
-
-	return result;
-}
-
-static int polaris10_set_private_data_based_on_pptable(struct pp_hwmgr *hwmgr)
-{
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	struct phm_ppt_v1_clock_voltage_dependency_table *allowed_sclk_vdd_table =
-						table_info->vdd_dep_on_sclk;
-	struct phm_ppt_v1_clock_voltage_dependency_table *allowed_mclk_vdd_table =
-						table_info->vdd_dep_on_mclk;
-
-	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table != NULL,
-		"VDD dependency on SCLK table is missing.	\
-		This table is mandatory", return -EINVAL);
-	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table->count >= 1,
-		"VDD dependency on SCLK table has to have is missing.	\
-		This table is mandatory", return -EINVAL);
-
-	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table != NULL,
-		"VDD dependency on MCLK table is missing.	\
-		This table is mandatory", return -EINVAL);
-	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table->count >= 1,
-		"VDD dependency on MCLK table has to have is missing.	 \
-		This table is mandatory", return -EINVAL);
-
-	table_info->max_clock_voltage_on_ac.sclk =
-		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].clk;
-	table_info->max_clock_voltage_on_ac.mclk =
-		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].clk;
-	table_info->max_clock_voltage_on_ac.vddc =
-		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].vddc;
-	table_info->max_clock_voltage_on_ac.vddci =
-		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].vddci;
-
-	hwmgr->dyn_state.max_clock_voltage_on_ac.sclk = table_info->max_clock_voltage_on_ac.sclk;
-	hwmgr->dyn_state.max_clock_voltage_on_ac.mclk = table_info->max_clock_voltage_on_ac.mclk;
-	hwmgr->dyn_state.max_clock_voltage_on_ac.vddc = table_info->max_clock_voltage_on_ac.vddc;
-	hwmgr->dyn_state.max_clock_voltage_on_ac.vddci =table_info->max_clock_voltage_on_ac.vddci;
-
-	return 0;
-}
-
-int polaris10_patch_voltage_workaround(struct pp_hwmgr *hwmgr)
-{
-	struct phm_ppt_v1_information *table_info =
-		       (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
-			table_info->vdd_dep_on_mclk;
-	struct phm_ppt_v1_voltage_lookup_table *lookup_table =
-			table_info->vddc_lookup_table;
-	uint32_t i;
-
-	if (hwmgr->chip_id == CHIP_POLARIS10 && hwmgr->hw_revision == 0xC7) {
-		if (lookup_table->entries[dep_mclk_table->entries[dep_mclk_table->count-1].vddInd].us_vdd >= 1000)
-			return 0;
-
-		for (i = 0; i < lookup_table->count; i++) {
-			if (lookup_table->entries[i].us_vdd < 0xff01 && lookup_table->entries[i].us_vdd >= 1000) {
-				dep_mclk_table->entries[dep_mclk_table->count-1].vddInd = (uint8_t) i;
-				return 0;
-			}
-		}
-	}
-	return 0;
-}
-
-
-int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data;
-	struct pp_atomctrl_gpio_pin_assignment gpio_pin_assignment;
-	uint32_t temp_reg;
-	int result;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	data = kzalloc(sizeof(struct polaris10_hwmgr), GFP_KERNEL);
-	if (data == NULL)
-		return -ENOMEM;
-
-	hwmgr->backend = data;
-
-	data->dll_default_on = false;
-	data->sram_end = SMC_RAM_END;
-	data->mclk_dpm0_activity_target = 0xa;
-	data->disable_dpm_mask = 0xFF;
-	data->static_screen_threshold = PPPOLARIS10_STATICSCREENTHRESHOLD_DFLT;
-	data->static_screen_threshold_unit = PPPOLARIS10_STATICSCREENTHRESHOLD_DFLT;
-	data->activity_target[0] = PPPOLARIS10_TARGETACTIVITY_DFLT;
-	data->activity_target[1] = PPPOLARIS10_TARGETACTIVITY_DFLT;
-	data->activity_target[2] = PPPOLARIS10_TARGETACTIVITY_DFLT;
-	data->activity_target[3] = PPPOLARIS10_TARGETACTIVITY_DFLT;
-	data->activity_target[4] = PPPOLARIS10_TARGETACTIVITY_DFLT;
-	data->activity_target[5] = PPPOLARIS10_TARGETACTIVITY_DFLT;
-	data->activity_target[6] = PPPOLARIS10_TARGETACTIVITY_DFLT;
-	data->activity_target[7] = PPPOLARIS10_TARGETACTIVITY_DFLT;
-
-	data->voting_rights_clients0 = PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT0;
-	data->voting_rights_clients1 = PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT1;
-	data->voting_rights_clients2 = PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT2;
-	data->voting_rights_clients3 = PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT3;
-	data->voting_rights_clients4 = PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT4;
-	data->voting_rights_clients5 = PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT5;
-	data->voting_rights_clients6 = PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT6;
-	data->voting_rights_clients7 = PPPOLARIS10_VOTINGRIGHTSCLIENTS_DFLT7;
-
-	data->vddc_vddci_delta = VDDC_VDDCI_DELTA;
-
-	data->mclk_activity_target = PPPOLARIS10_MCLK_TARGETACTIVITY_DFLT;
-
-	/* need to set voltage control types before EVV patching */
-	data->voltage_control = POLARIS10_VOLTAGE_CONTROL_NONE;
-	data->vddci_control = POLARIS10_VOLTAGE_CONTROL_NONE;
-	data->mvdd_control = POLARIS10_VOLTAGE_CONTROL_NONE;
-
-	data->enable_tdc_limit_feature = true;
-	data->enable_pkg_pwr_tracking_feature = true;
-	data->force_pcie_gen = PP_PCIEGenInvalid;
-	data->mclk_stutter_mode_threshold = 40000;
-
-	if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-			VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2))
-		data->voltage_control = POLARIS10_VOLTAGE_CONTROL_BY_SVID2;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_EnableMVDDControl)) {
-		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-				VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT))
-			data->mvdd_control = POLARIS10_VOLTAGE_CONTROL_BY_GPIO;
-		else if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-				VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_SVID2))
-			data->mvdd_control = POLARIS10_VOLTAGE_CONTROL_BY_SVID2;
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ControlVDDCI)) {
-		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT))
-			data->vddci_control = POLARIS10_VOLTAGE_CONTROL_BY_GPIO;
-		else if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2))
-			data->vddci_control = POLARIS10_VOLTAGE_CONTROL_BY_SVID2;
-	}
-
-	if (table_info->cac_dtp_table->usClockStretchAmount != 0)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-					PHM_PlatformCaps_ClockStretcher);
-
-	polaris10_set_features_platform_caps(hwmgr);
-
-	polaris10_patch_voltage_workaround(hwmgr);
-	polaris10_init_dpm_defaults(hwmgr);
-
-	/* Get leakage voltage based on leakage ID. */
-	result = polaris10_get_evv_voltages(hwmgr);
-
-	if (result) {
-		printk("Get EVV Voltage Failed.  Abort Driver loading!\n");
-		return -1;
-	}
-
-	polaris10_complete_dependency_tables(hwmgr);
-	polaris10_set_private_data_based_on_pptable(hwmgr);
-
-	/* Initalize Dynamic State Adjustment Rule Settings */
-	result = phm_initializa_dynamic_state_adjustment_rule_settings(hwmgr);
-
-	if (0 == result) {
-		struct cgs_system_info sys_info = {0};
-
-		data->is_tlu_enabled = false;
-
-		hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =
-							POLARIS10_MAX_HARDWARE_POWERLEVELS;
-		hwmgr->platform_descriptor.hardwarePerformanceLevels = 2;
-		hwmgr->platform_descriptor.minimumClocksReductionPercentage = 50;
-
-
-		if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_PCC_GPIO_PINID, &gpio_pin_assignment)) {
-			temp_reg = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCNB_PWRMGT_CNTL);
-			switch (gpio_pin_assignment.uc_gpio_pin_bit_shift) {
-			case 0:
-				temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, GNB_SLOW_MODE, 0x1);
-				break;
-			case 1:
-				temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, GNB_SLOW_MODE, 0x2);
-				break;
-			case 2:
-				temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, GNB_SLOW, 0x1);
-				break;
-			case 3:
-				temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, FORCE_NB_PS1, 0x1);
-				break;
-			case 4:
-				temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, DPM_ENABLED, 0x1);
-				break;
-			default:
-				PP_ASSERT_WITH_CODE(0,
-				"Failed to setup PCC HW register! Wrong GPIO assigned for VDDC_PCC_GPIO_PINID!",
-				);
-				break;
-			}
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCNB_PWRMGT_CNTL, temp_reg);
-		}
-
-		if (table_info->cac_dtp_table->usDefaultTargetOperatingTemp != 0 &&
-			hwmgr->thermal_controller.advanceFanControlParameters.ucFanControlMode) {
-			hwmgr->thermal_controller.advanceFanControlParameters.usFanPWMMinLimit =
-				(uint16_t)hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit;
-
-			hwmgr->thermal_controller.advanceFanControlParameters.usFanPWMMaxLimit =
-				(uint16_t)hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanPWM;
-
-			hwmgr->thermal_controller.advanceFanControlParameters.usFanPWMStep = 1;
-
-			hwmgr->thermal_controller.advanceFanControlParameters.usFanRPMMaxLimit = 100;
-
-			hwmgr->thermal_controller.advanceFanControlParameters.usFanRPMMinLimit =
-				(uint16_t)hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit;
-
-			hwmgr->thermal_controller.advanceFanControlParameters.usFanRPMStep = 1;
-
-			table_info->cac_dtp_table->usDefaultTargetOperatingTemp = (table_info->cac_dtp_table->usDefaultTargetOperatingTemp >= 50) ?
-									(table_info->cac_dtp_table->usDefaultTargetOperatingTemp -50) : 0;
-
-			table_info->cac_dtp_table->usOperatingTempMaxLimit = table_info->cac_dtp_table->usDefaultTargetOperatingTemp;
-			table_info->cac_dtp_table->usOperatingTempStep = 1;
-			table_info->cac_dtp_table->usOperatingTempHyst = 1;
-
-			hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanPWM =
-				       hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanPWM;
-
-			hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanRPM =
-				       hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanRPM;
-
-			hwmgr->dyn_state.cac_dtp_table->usOperatingTempMinLimit =
-				       table_info->cac_dtp_table->usOperatingTempMinLimit;
-
-			hwmgr->dyn_state.cac_dtp_table->usOperatingTempMaxLimit =
-				       table_info->cac_dtp_table->usOperatingTempMaxLimit;
-
-			hwmgr->dyn_state.cac_dtp_table->usDefaultTargetOperatingTemp =
-				       table_info->cac_dtp_table->usDefaultTargetOperatingTemp;
-
-			hwmgr->dyn_state.cac_dtp_table->usOperatingTempStep =
-				       table_info->cac_dtp_table->usOperatingTempStep;
-
-			hwmgr->dyn_state.cac_dtp_table->usTargetOperatingTemp =
-				       table_info->cac_dtp_table->usTargetOperatingTemp;
-		}
-
-		sys_info.size = sizeof(struct cgs_system_info);
-		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO;
-		result = cgs_query_system_info(hwmgr->device, &sys_info);
-		if (result)
-			data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK;
-		else
-			data->pcie_gen_cap = (uint32_t)sys_info.value;
-		if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
-			data->pcie_spc_cap = 20;
-		sys_info.size = sizeof(struct cgs_system_info);
-		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW;
-		result = cgs_query_system_info(hwmgr->device, &sys_info);
-		if (result)
-			data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK;
-		else
-			data->pcie_lane_cap = (uint32_t)sys_info.value;
-
-		hwmgr->platform_descriptor.vbiosInterruptId = 0x20000400; /* IRQ_SOURCE1_SW_INT */
-/* The true clock step depends on the frequency, typically 4.5 or 9 MHz. Here we use 5. */
-		hwmgr->platform_descriptor.clockStep.engineClock = 500;
-		hwmgr->platform_descriptor.clockStep.memoryClock = 500;
-	} else {
-		/* Ignore return value in here, we are cleaning up a mess. */
-		polaris10_hwmgr_backend_fini(hwmgr);
-	}
-
-	return 0;
-}
-
-static int polaris10_force_dpm_highest(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t level, tmp;
-
-	if (!data->pcie_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask) {
-			level = 0;
-			tmp = data->dpm_level_enable_mask.pcie_dpm_enable_mask;
-			while (tmp >>= 1)
-				level++;
-
-			if (level)
-				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-						PPSMC_MSG_PCIeDPM_ForceLevel, level);
-		}
-	}
-
-	if (!data->sclk_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
-			level = 0;
-			tmp = data->dpm_level_enable_mask.sclk_dpm_enable_mask;
-			while (tmp >>= 1)
-				level++;
-
-			if (level)
-				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-						PPSMC_MSG_SCLKDPM_SetEnabledMask,
-						(1 << level));
-		}
-	}
-
-	if (!data->mclk_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
-			level = 0;
-			tmp = data->dpm_level_enable_mask.mclk_dpm_enable_mask;
-			while (tmp >>= 1)
-				level++;
-
-			if (level)
-				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-						PPSMC_MSG_MCLKDPM_SetEnabledMask,
-						(1 << level));
-		}
-	}
-
-	return 0;
-}
-
-static int polaris10_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	phm_apply_dal_min_voltage_request(hwmgr);
-
-	if (!data->sclk_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_SCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.sclk_dpm_enable_mask);
-	}
-
-	if (!data->mclk_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_MCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.mclk_dpm_enable_mask);
-	}
-
-	return 0;
-}
-
-static int polaris10_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if (!polaris10_is_dpm_running(hwmgr))
-		return -EINVAL;
-
-	if (!data->pcie_dpm_key_disabled) {
-		smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_PCIeDPM_UnForceLevel);
-	}
-
-	return polaris10_upload_dpm_level_enable_mask(hwmgr);
-}
-
-static int polaris10_force_dpm_lowest(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data =
-			(struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t level;
-
-	if (!data->sclk_dpm_key_disabled)
-		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
-			level = phm_get_lowest_enabled_level(hwmgr,
-							      data->dpm_level_enable_mask.sclk_dpm_enable_mask);
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-							    PPSMC_MSG_SCLKDPM_SetEnabledMask,
-							    (1 << level));
-
-	}
-
-	if (!data->mclk_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
-			level = phm_get_lowest_enabled_level(hwmgr,
-							      data->dpm_level_enable_mask.mclk_dpm_enable_mask);
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-							    PPSMC_MSG_MCLKDPM_SetEnabledMask,
-							    (1 << level));
-		}
-	}
-
-	if (!data->pcie_dpm_key_disabled) {
-		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask) {
-			level = phm_get_lowest_enabled_level(hwmgr,
-							      data->dpm_level_enable_mask.pcie_dpm_enable_mask);
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-							    PPSMC_MSG_PCIeDPM_ForceLevel,
-							    (level));
-		}
-	}
-
-	return 0;
-
-}
-static int polaris10_force_dpm_level(struct pp_hwmgr *hwmgr,
-				enum amd_dpm_forced_level level)
-{
-	int ret = 0;
-
-	switch (level) {
-	case AMD_DPM_FORCED_LEVEL_HIGH:
-		ret = polaris10_force_dpm_highest(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	case AMD_DPM_FORCED_LEVEL_LOW:
-		ret = polaris10_force_dpm_lowest(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	case AMD_DPM_FORCED_LEVEL_AUTO:
-		ret = polaris10_unforce_dpm_levels(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	default:
-		break;
-	}
-
-	hwmgr->dpm_level = level;
-
-	return ret;
-}
-
-static int polaris10_get_power_state_size(struct pp_hwmgr *hwmgr)
-{
-	return sizeof(struct polaris10_power_state);
-}
-
-
-static int polaris10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
-				struct pp_power_state *request_ps,
-			const struct pp_power_state *current_ps)
-{
-
-	struct polaris10_power_state *polaris10_ps =
-				cast_phw_polaris10_power_state(&request_ps->hardware);
-	uint32_t sclk;
-	uint32_t mclk;
-	struct PP_Clocks minimum_clocks = {0};
-	bool disable_mclk_switching;
-	bool disable_mclk_switching_for_frame_lock;
-	struct cgs_display_info info = {0};
-	const struct phm_clock_and_voltage_limits *max_limits;
-	uint32_t i;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	int32_t count;
-	int32_t stable_pstate_sclk = 0, stable_pstate_mclk = 0;
-
-	data->battery_state = (PP_StateUILabel_Battery ==
-			request_ps->classification.ui_label);
-
-	PP_ASSERT_WITH_CODE(polaris10_ps->performance_level_count == 2,
-				 "VI should always have 2 performance levels",
-				);
-
-	max_limits = (PP_PowerSource_AC == hwmgr->power_source) ?
-			&(hwmgr->dyn_state.max_clock_voltage_on_ac) :
-			&(hwmgr->dyn_state.max_clock_voltage_on_dc);
-
-	/* Cap clock DPM tables at DC MAX if it is in DC. */
-	if (PP_PowerSource_DC == hwmgr->power_source) {
-		for (i = 0; i < polaris10_ps->performance_level_count; i++) {
-			if (polaris10_ps->performance_levels[i].memory_clock > max_limits->mclk)
-				polaris10_ps->performance_levels[i].memory_clock = max_limits->mclk;
-			if (polaris10_ps->performance_levels[i].engine_clock > max_limits->sclk)
-				polaris10_ps->performance_levels[i].engine_clock = max_limits->sclk;
-		}
-	}
-
-	polaris10_ps->vce_clks.evclk = hwmgr->vce_arbiter.evclk;
-	polaris10_ps->vce_clks.ecclk = hwmgr->vce_arbiter.ecclk;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	/*TO DO result = PHM_CheckVBlankTime(hwmgr, &vblankTooShort);*/
-
-	/* TO DO GetMinClockSettings(hwmgr->pPECI, &minimum_clocks); */
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_StablePState)) {
-		max_limits = &(hwmgr->dyn_state.max_clock_voltage_on_ac);
-		stable_pstate_sclk = (max_limits->sclk * 75) / 100;
-
-		for (count = table_info->vdd_dep_on_sclk->count - 1;
-				count >= 0; count--) {
-			if (stable_pstate_sclk >=
-					table_info->vdd_dep_on_sclk->entries[count].clk) {
-				stable_pstate_sclk =
-						table_info->vdd_dep_on_sclk->entries[count].clk;
-				break;
-			}
-		}
-
-		if (count < 0)
-			stable_pstate_sclk = table_info->vdd_dep_on_sclk->entries[0].clk;
-
-		stable_pstate_mclk = max_limits->mclk;
-
-		minimum_clocks.engineClock = stable_pstate_sclk;
-		minimum_clocks.memoryClock = stable_pstate_mclk;
-	}
-
-	if (minimum_clocks.engineClock < hwmgr->gfx_arbiter.sclk)
-		minimum_clocks.engineClock = hwmgr->gfx_arbiter.sclk;
-
-	if (minimum_clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
-		minimum_clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
-
-	polaris10_ps->sclk_threshold = hwmgr->gfx_arbiter.sclk_threshold;
-
-	if (0 != hwmgr->gfx_arbiter.sclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.sclk_over_drive <=
-				hwmgr->platform_descriptor.overdriveLimit.engineClock),
-				"Overdrive sclk exceeds limit",
-				hwmgr->gfx_arbiter.sclk_over_drive =
-						hwmgr->platform_descriptor.overdriveLimit.engineClock);
-
-		if (hwmgr->gfx_arbiter.sclk_over_drive >= hwmgr->gfx_arbiter.sclk)
-			polaris10_ps->performance_levels[1].engine_clock =
-					hwmgr->gfx_arbiter.sclk_over_drive;
-	}
-
-	if (0 != hwmgr->gfx_arbiter.mclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.mclk_over_drive <=
-				hwmgr->platform_descriptor.overdriveLimit.memoryClock),
-				"Overdrive mclk exceeds limit",
-				hwmgr->gfx_arbiter.mclk_over_drive =
-						hwmgr->platform_descriptor.overdriveLimit.memoryClock);
-
-		if (hwmgr->gfx_arbiter.mclk_over_drive >= hwmgr->gfx_arbiter.mclk)
-			polaris10_ps->performance_levels[1].memory_clock =
-					hwmgr->gfx_arbiter.mclk_over_drive;
-	}
-
-	disable_mclk_switching_for_frame_lock = phm_cap_enabled(
-				    hwmgr->platform_descriptor.platformCaps,
-				    PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
-
-
-	disable_mclk_switching = (1 < info.display_count) ||
-				    disable_mclk_switching_for_frame_lock;
-
-	sclk = polaris10_ps->performance_levels[0].engine_clock;
-	mclk = polaris10_ps->performance_levels[0].memory_clock;
-
-	if (disable_mclk_switching)
-		mclk = polaris10_ps->performance_levels
-		[polaris10_ps->performance_level_count - 1].memory_clock;
-
-	if (sclk < minimum_clocks.engineClock)
-		sclk = (minimum_clocks.engineClock > max_limits->sclk) ?
-				max_limits->sclk : minimum_clocks.engineClock;
-
-	if (mclk < minimum_clocks.memoryClock)
-		mclk = (minimum_clocks.memoryClock > max_limits->mclk) ?
-				max_limits->mclk : minimum_clocks.memoryClock;
-
-	polaris10_ps->performance_levels[0].engine_clock = sclk;
-	polaris10_ps->performance_levels[0].memory_clock = mclk;
-
-	polaris10_ps->performance_levels[1].engine_clock =
-		(polaris10_ps->performance_levels[1].engine_clock >=
-				polaris10_ps->performance_levels[0].engine_clock) ?
-						polaris10_ps->performance_levels[1].engine_clock :
-						polaris10_ps->performance_levels[0].engine_clock;
-
-	if (disable_mclk_switching) {
-		if (mclk < polaris10_ps->performance_levels[1].memory_clock)
-			mclk = polaris10_ps->performance_levels[1].memory_clock;
-
-		polaris10_ps->performance_levels[0].memory_clock = mclk;
-		polaris10_ps->performance_levels[1].memory_clock = mclk;
-	} else {
-		if (polaris10_ps->performance_levels[1].memory_clock <
-				polaris10_ps->performance_levels[0].memory_clock)
-			polaris10_ps->performance_levels[1].memory_clock =
-					polaris10_ps->performance_levels[0].memory_clock;
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_StablePState)) {
-		for (i = 0; i < polaris10_ps->performance_level_count; i++) {
-			polaris10_ps->performance_levels[i].engine_clock = stable_pstate_sclk;
-			polaris10_ps->performance_levels[i].memory_clock = stable_pstate_mclk;
-			polaris10_ps->performance_levels[i].pcie_gen = data->pcie_gen_performance.max;
-			polaris10_ps->performance_levels[i].pcie_lane = data->pcie_gen_performance.max;
-		}
-	}
-	return 0;
-}
-
-
-static int polaris10_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
-{
-	struct pp_power_state  *ps;
-	struct polaris10_power_state  *polaris10_ps;
-
-	if (hwmgr == NULL)
-		return -EINVAL;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	polaris10_ps = cast_phw_polaris10_power_state(&ps->hardware);
-
-	if (low)
-		return polaris10_ps->performance_levels[0].memory_clock;
-	else
-		return polaris10_ps->performance_levels
-				[polaris10_ps->performance_level_count-1].memory_clock;
-}
-
-static int polaris10_dpm_get_sclk(struct pp_hwmgr *hwmgr, bool low)
-{
-	struct pp_power_state  *ps;
-	struct polaris10_power_state  *polaris10_ps;
-
-	if (hwmgr == NULL)
-		return -EINVAL;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	polaris10_ps = cast_phw_polaris10_power_state(&ps->hardware);
-
-	if (low)
-		return polaris10_ps->performance_levels[0].engine_clock;
-	else
-		return polaris10_ps->performance_levels
-				[polaris10_ps->performance_level_count-1].engine_clock;
-}
-
-static int polaris10_dpm_patch_boot_state(struct pp_hwmgr *hwmgr,
-					struct pp_hw_power_state *hw_ps)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_power_state *ps = (struct polaris10_power_state *)hw_ps;
-	ATOM_FIRMWARE_INFO_V2_2 *fw_info;
-	uint16_t size;
-	uint8_t frev, crev;
-	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
-
-	/* First retrieve the Boot clocks and VDDC from the firmware info table.
-	 * We assume here that fw_info is unchanged if this call fails.
-	 */
-	fw_info = (ATOM_FIRMWARE_INFO_V2_2 *)cgs_atom_get_data_table(
-			hwmgr->device, index,
-			&size, &frev, &crev);
-	if (!fw_info)
-		/* During a test, there is no firmware info table. */
-		return 0;
-
-	/* Patch the state. */
-	data->vbios_boot_state.sclk_bootup_value =
-			le32_to_cpu(fw_info->ulDefaultEngineClock);
-	data->vbios_boot_state.mclk_bootup_value =
-			le32_to_cpu(fw_info->ulDefaultMemoryClock);
-	data->vbios_boot_state.mvdd_bootup_value =
-			le16_to_cpu(fw_info->usBootUpMVDDCVoltage);
-	data->vbios_boot_state.vddc_bootup_value =
-			le16_to_cpu(fw_info->usBootUpVDDCVoltage);
-	data->vbios_boot_state.vddci_bootup_value =
-			le16_to_cpu(fw_info->usBootUpVDDCIVoltage);
-	data->vbios_boot_state.pcie_gen_bootup_value =
-			phm_get_current_pcie_speed(hwmgr);
-
-	data->vbios_boot_state.pcie_lane_bootup_value =
-			(uint16_t)phm_get_current_pcie_lane_number(hwmgr);
-
-	/* set boot power state */
-	ps->performance_levels[0].memory_clock = data->vbios_boot_state.mclk_bootup_value;
-	ps->performance_levels[0].engine_clock = data->vbios_boot_state.sclk_bootup_value;
-	ps->performance_levels[0].pcie_gen = data->vbios_boot_state.pcie_gen_bootup_value;
-	ps->performance_levels[0].pcie_lane = data->vbios_boot_state.pcie_lane_bootup_value;
-
-	return 0;
-}
-
-static int polaris10_get_pp_table_entry_callback_func(struct pp_hwmgr *hwmgr,
-		void *state, struct pp_power_state *power_state,
-		void *pp_table, uint32_t classification_flag)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_power_state  *polaris10_power_state =
-			(struct polaris10_power_state *)(&(power_state->hardware));
-	struct polaris10_performance_level *performance_level;
-	ATOM_Tonga_State *state_entry = (ATOM_Tonga_State *)state;
-	ATOM_Tonga_POWERPLAYTABLE *powerplay_table =
-			(ATOM_Tonga_POWERPLAYTABLE *)pp_table;
-	PPTable_Generic_SubTable_Header *sclk_dep_table =
-			(PPTable_Generic_SubTable_Header *)
-			(((unsigned long)powerplay_table) +
-				le16_to_cpu(powerplay_table->usSclkDependencyTableOffset));
-
-	ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table =
-			(ATOM_Tonga_MCLK_Dependency_Table *)
-			(((unsigned long)powerplay_table) +
-				le16_to_cpu(powerplay_table->usMclkDependencyTableOffset));
-
-	/* The following fields are not initialized here: id orderedList allStatesList */
-	power_state->classification.ui_label =
-			(le16_to_cpu(state_entry->usClassification) &
-			ATOM_PPLIB_CLASSIFICATION_UI_MASK) >>
-			ATOM_PPLIB_CLASSIFICATION_UI_SHIFT;
-	power_state->classification.flags = classification_flag;
-	/* NOTE: There is a classification2 flag in BIOS that is not being used right now */
-
-	power_state->classification.temporary_state = false;
-	power_state->classification.to_be_deleted = false;
-
-	power_state->validation.disallowOnDC =
-			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) &
-					ATOM_Tonga_DISALLOW_ON_DC));
-
-	power_state->pcie.lanes = 0;
-
-	power_state->display.disableFrameModulation = false;
-	power_state->display.limitRefreshrate = false;
-	power_state->display.enableVariBright =
-			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) &
-					ATOM_Tonga_ENABLE_VARIBRIGHT));
-
-	power_state->validation.supportedPowerLevels = 0;
-	power_state->uvd_clocks.VCLK = 0;
-	power_state->uvd_clocks.DCLK = 0;
-	power_state->temperatures.min = 0;
-	power_state->temperatures.max = 0;
-
-	performance_level = &(polaris10_power_state->performance_levels
-			[polaris10_power_state->performance_level_count++]);
-
-	PP_ASSERT_WITH_CODE(
-			(polaris10_power_state->performance_level_count < SMU74_MAX_LEVELS_GRAPHICS),
-			"Performance levels exceeds SMC limit!",
-			return -1);
-
-	PP_ASSERT_WITH_CODE(
-			(polaris10_power_state->performance_level_count <=
-					hwmgr->platform_descriptor.hardwareActivityPerformanceLevels),
-			"Performance levels exceeds Driver limit!",
-			return -1);
-
-	/* Performance levels are arranged from low to high. */
-	performance_level->memory_clock = mclk_dep_table->entries
-			[state_entry->ucMemoryClockIndexLow].ulMclk;
-	if (sclk_dep_table->ucRevId == 0)
-		performance_level->engine_clock = ((ATOM_Tonga_SCLK_Dependency_Table *)sclk_dep_table)->entries
-			[state_entry->ucEngineClockIndexLow].ulSclk;
-	else if (sclk_dep_table->ucRevId == 1)
-		performance_level->engine_clock = ((ATOM_Polaris_SCLK_Dependency_Table *)sclk_dep_table)->entries
-			[state_entry->ucEngineClockIndexLow].ulSclk;
-	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap,
-			state_entry->ucPCIEGenLow);
-	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap,
-			state_entry->ucPCIELaneHigh);
-
-	performance_level = &(polaris10_power_state->performance_levels
-			[polaris10_power_state->performance_level_count++]);
-	performance_level->memory_clock = mclk_dep_table->entries
-			[state_entry->ucMemoryClockIndexHigh].ulMclk;
-
-	if (sclk_dep_table->ucRevId == 0)
-		performance_level->engine_clock = ((ATOM_Tonga_SCLK_Dependency_Table *)sclk_dep_table)->entries
-			[state_entry->ucEngineClockIndexHigh].ulSclk;
-	else if (sclk_dep_table->ucRevId == 1)
-		performance_level->engine_clock = ((ATOM_Polaris_SCLK_Dependency_Table *)sclk_dep_table)->entries
-			[state_entry->ucEngineClockIndexHigh].ulSclk;
-
-	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap,
-			state_entry->ucPCIEGenHigh);
-	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap,
-			state_entry->ucPCIELaneHigh);
-
-	return 0;
-}
-
-static int polaris10_get_pp_table_entry(struct pp_hwmgr *hwmgr,
-		unsigned long entry_index, struct pp_power_state *state)
-{
-	int result;
-	struct polaris10_power_state *ps;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
-			table_info->vdd_dep_on_mclk;
-
-	state->hardware.magic = PHM_VIslands_Magic;
-
-	ps = (struct polaris10_power_state *)(&state->hardware);
-
-	result = tonga_get_powerplay_table_entry(hwmgr, entry_index, state,
-			polaris10_get_pp_table_entry_callback_func);
-
-	/* This is the earliest time we have all the dependency table and the VBIOS boot state
-	 * as PP_Tables_GetPowerPlayTableEntry retrieves the VBIOS boot state
-	 * if there is only one VDDCI/MCLK level, check if it's the same as VBIOS boot state
-	 */
-	if (dep_mclk_table != NULL && dep_mclk_table->count == 1) {
-		if (dep_mclk_table->entries[0].clk !=
-				data->vbios_boot_state.mclk_bootup_value)
-			printk(KERN_ERR "Single MCLK entry VDDCI/MCLK dependency table "
-					"does not match VBIOS boot MCLK level");
-		if (dep_mclk_table->entries[0].vddci !=
-				data->vbios_boot_state.vddci_bootup_value)
-			printk(KERN_ERR "Single VDDCI entry VDDCI/MCLK dependency table "
-					"does not match VBIOS boot VDDCI level");
-	}
-
-	/* set DC compatible flag if this state supports DC */
-	if (!state->validation.disallowOnDC)
-		ps->dc_compatible = true;
-
-	if (state->classification.flags & PP_StateClassificationFlag_ACPI)
-		data->acpi_pcie_gen = ps->performance_levels[0].pcie_gen;
-
-	ps->uvd_clks.vclk = state->uvd_clocks.VCLK;
-	ps->uvd_clks.dclk = state->uvd_clocks.DCLK;
-
-	if (!result) {
-		uint32_t i;
-
-		switch (state->classification.ui_label) {
-		case PP_StateUILabel_Performance:
-			data->use_pcie_performance_levels = true;
-			for (i = 0; i < ps->performance_level_count; i++) {
-				if (data->pcie_gen_performance.max <
-						ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_performance.max =
-							ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_gen_performance.min >
-						ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_performance.min =
-							ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_lane_performance.max <
-						ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_performance.max =
-							ps->performance_levels[i].pcie_lane;
-				if (data->pcie_lane_performance.min >
-						ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_performance.min =
-							ps->performance_levels[i].pcie_lane;
-			}
-			break;
-		case PP_StateUILabel_Battery:
-			data->use_pcie_power_saving_levels = true;
-
-			for (i = 0; i < ps->performance_level_count; i++) {
-				if (data->pcie_gen_power_saving.max <
-						ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_power_saving.max =
-							ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_gen_power_saving.min >
-						ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_power_saving.min =
-							ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_lane_power_saving.max <
-						ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_power_saving.max =
-							ps->performance_levels[i].pcie_lane;
-
-				if (data->pcie_lane_power_saving.min >
-						ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_power_saving.min =
-							ps->performance_levels[i].pcie_lane;
-			}
-			break;
-		default:
-			break;
-		}
-	}
-	return 0;
-}
-
-static void
-polaris10_print_current_perforce_level(struct pp_hwmgr *hwmgr, struct seq_file *m)
-{
-	uint32_t sclk, mclk, activity_percent;
-	uint32_t offset;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetSclkFrequency);
-
-	sclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-
-	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetMclkFrequency);
-
-	mclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-	seq_printf(m, "\n [  mclk  ]: %u MHz\n\n [  sclk  ]: %u MHz\n",
-			mclk / 100, sclk / 100);
-
-	offset = data->soft_regs_start + offsetof(SMU74_SoftRegisters, AverageGraphicsActivity);
-	activity_percent = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, offset);
-	activity_percent += 0x80;
-	activity_percent >>= 8;
-
-	seq_printf(m, "\n [GPU load]: %u%%\n\n", activity_percent > 100 ? 100 : activity_percent);
-
-	seq_printf(m, "uvd    %sabled\n", data->uvd_power_gated ? "dis" : "en");
-
-	seq_printf(m, "vce    %sabled\n", data->vce_power_gated ? "dis" : "en");
-}
-
-static int polaris10_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	const struct polaris10_power_state *polaris10_ps =
-			cast_const_phw_polaris10_power_state(states->pnew_state);
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
-	uint32_t sclk = polaris10_ps->performance_levels
-			[polaris10_ps->performance_level_count - 1].engine_clock;
-	struct polaris10_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
-	uint32_t mclk = polaris10_ps->performance_levels
-			[polaris10_ps->performance_level_count - 1].memory_clock;
-	struct PP_Clocks min_clocks = {0};
-	uint32_t i;
-	struct cgs_display_info info = {0};
-
-	data->need_update_smu7_dpm_table = 0;
-
-	for (i = 0; i < sclk_table->count; i++) {
-		if (sclk == sclk_table->dpm_levels[i].value)
-			break;
-	}
-
-	if (i >= sclk_table->count)
-		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
-	else {
-	/* TODO: Check SCLK in DAL's minimum clocks
-	 * in case DeepSleep divider update is required.
-	 */
-		if (data->display_timing.min_clock_in_sr != min_clocks.engineClockInSR &&
-			(min_clocks.engineClockInSR >= POLARIS10_MINIMUM_ENGINE_CLOCK ||
-				data->display_timing.min_clock_in_sr >= POLARIS10_MINIMUM_ENGINE_CLOCK))
-			data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
-	}
-
-	for (i = 0; i < mclk_table->count; i++) {
-		if (mclk == mclk_table->dpm_levels[i].value)
-			break;
-	}
-
-	if (i >= mclk_table->count)
-		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	if (data->display_timing.num_existing_displays != info.display_count)
-		data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
-
-	return 0;
-}
-
-static uint16_t polaris10_get_maximum_link_speed(struct pp_hwmgr *hwmgr,
-		const struct polaris10_power_state *polaris10_ps)
-{
-	uint32_t i;
-	uint32_t sclk, max_sclk = 0;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_dpm_table *dpm_table = &data->dpm_table;
-
-	for (i = 0; i < polaris10_ps->performance_level_count; i++) {
-		sclk = polaris10_ps->performance_levels[i].engine_clock;
-		if (max_sclk < sclk)
-			max_sclk = sclk;
-	}
-
-	for (i = 0; i < dpm_table->sclk_table.count; i++) {
-		if (dpm_table->sclk_table.dpm_levels[i].value == max_sclk)
-			return (uint16_t) ((i >= dpm_table->pcie_speed_table.count) ?
-					dpm_table->pcie_speed_table.dpm_levels
-					[dpm_table->pcie_speed_table.count - 1].value :
-					dpm_table->pcie_speed_table.dpm_levels[i].value);
-	}
-
-	return 0;
-}
-
-static int polaris10_request_link_speed_change_before_state_change(
-		struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	const struct polaris10_power_state *polaris10_nps =
-			cast_const_phw_polaris10_power_state(states->pnew_state);
-	const struct polaris10_power_state *polaris10_cps =
-			cast_const_phw_polaris10_power_state(states->pcurrent_state);
-
-	uint16_t target_link_speed = polaris10_get_maximum_link_speed(hwmgr, polaris10_nps);
-	uint16_t current_link_speed;
-
-	if (data->force_pcie_gen == PP_PCIEGenInvalid)
-		current_link_speed = polaris10_get_maximum_link_speed(hwmgr, polaris10_cps);
-	else
-		current_link_speed = data->force_pcie_gen;
-
-	data->force_pcie_gen = PP_PCIEGenInvalid;
-	data->pspp_notify_required = false;
-
-	if (target_link_speed > current_link_speed) {
-		switch (target_link_speed) {
-		case PP_PCIEGen3:
-			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN3, false))
-				break;
-			data->force_pcie_gen = PP_PCIEGen2;
-			if (current_link_speed == PP_PCIEGen2)
-				break;
-		case PP_PCIEGen2:
-			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN2, false))
-				break;
-		default:
-			data->force_pcie_gen = phm_get_current_pcie_speed(hwmgr);
-			break;
-		}
-	} else {
-		if (target_link_speed < current_link_speed)
-			data->pspp_notify_required = true;
-	}
-
-	return 0;
-}
-
-static int polaris10_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if (0 == data->need_update_smu7_dpm_table)
-		return 0;
-
-	if ((0 == data->sclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table &
-			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
-		PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr),
-				    "Trying to freeze SCLK DPM when DPM is disabled",
-				);
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_SCLKDPM_FreezeLevel),
-				"Failed to freeze SCLK DPM during FreezeSclkMclkDPM Function!",
-				return -1);
-	}
-
-	if ((0 == data->mclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table &
-		 DPMTABLE_OD_UPDATE_MCLK)) {
-		PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr),
-				    "Trying to freeze MCLK DPM when DPM is disabled",
-				);
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_MCLKDPM_FreezeLevel),
-				"Failed to freeze MCLK DPM during FreezeSclkMclkDPM Function!",
-				return -1);
-	}
-
-	return 0;
-}
-
-static int polaris10_populate_and_upload_sclk_mclk_dpm_levels(
-		struct pp_hwmgr *hwmgr, const void *input)
-{
-	int result = 0;
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	const struct polaris10_power_state *polaris10_ps =
-			cast_const_phw_polaris10_power_state(states->pnew_state);
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t sclk = polaris10_ps->performance_levels
-			[polaris10_ps->performance_level_count - 1].engine_clock;
-	uint32_t mclk = polaris10_ps->performance_levels
-			[polaris10_ps->performance_level_count - 1].memory_clock;
-	struct polaris10_dpm_table *dpm_table = &data->dpm_table;
-
-	struct polaris10_dpm_table *golden_dpm_table = &data->golden_dpm_table;
-	uint32_t dpm_count, clock_percent;
-	uint32_t i;
-
-	if (0 == data->need_update_smu7_dpm_table)
-		return 0;
-
-	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK) {
-		dpm_table->sclk_table.dpm_levels
-		[dpm_table->sclk_table.count - 1].value = sclk;
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinACSupport) ||
-		    phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinDCSupport)) {
-		/* Need to do calculation based on the golden DPM table
-		 * as the Heatmap GPU Clock axis is also based on the default values
-		 */
-			PP_ASSERT_WITH_CODE(
-				(golden_dpm_table->sclk_table.dpm_levels
-						[golden_dpm_table->sclk_table.count - 1].value != 0),
-				"Divide by 0!",
-				return -1);
-			dpm_count = dpm_table->sclk_table.count < 2 ? 0 : dpm_table->sclk_table.count - 2;
-
-			for (i = dpm_count; i > 1; i--) {
-				if (sclk > golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count-1].value) {
-					clock_percent =
-					      ((sclk
-						- golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count-1].value
-						) * 100)
-						/ golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count-1].value;
-
-					dpm_table->sclk_table.dpm_levels[i].value =
-							golden_dpm_table->sclk_table.dpm_levels[i].value +
-							(golden_dpm_table->sclk_table.dpm_levels[i].value *
-								clock_percent)/100;
-
-				} else if (golden_dpm_table->sclk_table.dpm_levels[dpm_table->sclk_table.count-1].value > sclk) {
-					clock_percent =
-						((golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count - 1].value
-						- sclk) * 100)
-						/ golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count-1].value;
-
-					dpm_table->sclk_table.dpm_levels[i].value =
-							golden_dpm_table->sclk_table.dpm_levels[i].value -
-							(golden_dpm_table->sclk_table.dpm_levels[i].value *
-									clock_percent) / 100;
-				} else
-					dpm_table->sclk_table.dpm_levels[i].value =
-							golden_dpm_table->sclk_table.dpm_levels[i].value;
-			}
-		}
-	}
-
-	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK) {
-		dpm_table->mclk_table.dpm_levels
-			[dpm_table->mclk_table.count - 1].value = mclk;
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinACSupport) ||
-		    phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinDCSupport)) {
-
-			PP_ASSERT_WITH_CODE(
-					(golden_dpm_table->mclk_table.dpm_levels
-						[golden_dpm_table->mclk_table.count-1].value != 0),
-					"Divide by 0!",
-					return -1);
-			dpm_count = dpm_table->mclk_table.count < 2 ? 0 : dpm_table->mclk_table.count - 2;
-			for (i = dpm_count; i > 1; i--) {
-				if (golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value < mclk) {
-					clock_percent = ((mclk -
-					golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value) * 100)
-					/ golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value;
-
-					dpm_table->mclk_table.dpm_levels[i].value =
-							golden_dpm_table->mclk_table.dpm_levels[i].value +
-							(golden_dpm_table->mclk_table.dpm_levels[i].value *
-							clock_percent) / 100;
-
-				} else if (golden_dpm_table->mclk_table.dpm_levels[dpm_table->mclk_table.count-1].value > mclk) {
-					clock_percent = (
-					 (golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value - mclk)
-					* 100)
-					/ golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value;
-
-					dpm_table->mclk_table.dpm_levels[i].value =
-							golden_dpm_table->mclk_table.dpm_levels[i].value -
-							(golden_dpm_table->mclk_table.dpm_levels[i].value *
-									clock_percent) / 100;
-				} else
-					dpm_table->mclk_table.dpm_levels[i].value =
-							golden_dpm_table->mclk_table.dpm_levels[i].value;
-			}
-		}
-	}
-
-	if (data->need_update_smu7_dpm_table &
-			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK)) {
-		result = polaris10_populate_all_graphic_levels(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
-				return result);
-	}
-
-	if (data->need_update_smu7_dpm_table &
-			(DPMTABLE_OD_UPDATE_MCLK + DPMTABLE_UPDATE_MCLK)) {
-		/*populate MCLK dpm table to SMU7 */
-		result = polaris10_populate_all_memory_levels(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
-				return result);
-	}
-
-	return result;
-}
-
-static int polaris10_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
-			  struct polaris10_single_dpm_table *dpm_table,
-			uint32_t low_limit, uint32_t high_limit)
-{
-	uint32_t i;
-
-	for (i = 0; i < dpm_table->count; i++) {
-		if ((dpm_table->dpm_levels[i].value < low_limit)
-		|| (dpm_table->dpm_levels[i].value > high_limit))
-			dpm_table->dpm_levels[i].enabled = false;
-		else
-			dpm_table->dpm_levels[i].enabled = true;
-	}
-
-	return 0;
-}
-
-static int polaris10_trim_dpm_states(struct pp_hwmgr *hwmgr,
-		const struct polaris10_power_state *polaris10_ps)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t high_limit_count;
-
-	PP_ASSERT_WITH_CODE((polaris10_ps->performance_level_count >= 1),
-			"power state did not have any performance level",
-			return -1);
-
-	high_limit_count = (1 == polaris10_ps->performance_level_count) ? 0 : 1;
-
-	polaris10_trim_single_dpm_states(hwmgr,
-			&(data->dpm_table.sclk_table),
-			polaris10_ps->performance_levels[0].engine_clock,
-			polaris10_ps->performance_levels[high_limit_count].engine_clock);
-
-	polaris10_trim_single_dpm_states(hwmgr,
-			&(data->dpm_table.mclk_table),
-			polaris10_ps->performance_levels[0].memory_clock,
-			polaris10_ps->performance_levels[high_limit_count].memory_clock);
-
-	return 0;
-}
-
-static int polaris10_generate_dpm_level_enable_mask(
-		struct pp_hwmgr *hwmgr, const void *input)
-{
-	int result;
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	const struct polaris10_power_state *polaris10_ps =
-			cast_const_phw_polaris10_power_state(states->pnew_state);
-
-	result = polaris10_trim_dpm_states(hwmgr, polaris10_ps);
-	if (result)
-		return result;
-
-	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
-			phm_get_dpm_level_enable_mask_value(&data->dpm_table.sclk_table);
-	data->dpm_level_enable_mask.mclk_dpm_enable_mask =
-			phm_get_dpm_level_enable_mask_value(&data->dpm_table.mclk_table);
-	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
-			phm_get_dpm_level_enable_mask_value(&data->dpm_table.pcie_speed_table);
-
-	return 0;
-}
-
-int polaris10_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
-			PPSMC_MSG_UVDDPM_Enable :
-			PPSMC_MSG_UVDDPM_Disable);
-}
-
-int polaris10_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, enable?
-			PPSMC_MSG_VCEDPM_Enable :
-			PPSMC_MSG_VCEDPM_Disable);
-}
-
-int polaris10_enable_disable_samu_dpm(struct pp_hwmgr *hwmgr, bool enable)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, enable?
-			PPSMC_MSG_SAMUDPM_Enable :
-			PPSMC_MSG_SAMUDPM_Disable);
-}
-
-int polaris10_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t mm_boot_level_offset, mm_boot_level_value;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (!bgate) {
-		data->smc_state_table.UvdBootLevel = 0;
-		if (table_info->mm_dep_table->count > 0)
-			data->smc_state_table.UvdBootLevel =
-					(uint8_t) (table_info->mm_dep_table->count - 1);
-		mm_boot_level_offset = data->dpm_table_start +
-				offsetof(SMU74_Discrete_DpmTable, UvdBootLevel);
-		mm_boot_level_offset /= 4;
-		mm_boot_level_offset *= 4;
-		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset);
-		mm_boot_level_value &= 0x00FFFFFF;
-		mm_boot_level_value |= data->smc_state_table.UvdBootLevel << 24;
-		cgs_write_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
-
-		if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_UVDDPM) ||
-			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_StablePState))
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_UVDDPM_SetEnabledMask,
-					(uint32_t)(1 << data->smc_state_table.UvdBootLevel));
-	}
-
-	return polaris10_enable_disable_uvd_dpm(hwmgr, !bgate);
-}
-
-int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t mm_boot_level_offset, mm_boot_level_value;
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (!bgate) {
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_StablePState))
-			data->smc_state_table.VceBootLevel =
-				(uint8_t) (table_info->mm_dep_table->count - 1);
-		else
-			data->smc_state_table.VceBootLevel = 0;
-
-		mm_boot_level_offset = data->dpm_table_start +
-				offsetof(SMU74_Discrete_DpmTable, VceBootLevel);
-		mm_boot_level_offset /= 4;
-		mm_boot_level_offset *= 4;
-		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset);
-		mm_boot_level_value &= 0xFF00FFFF;
-		mm_boot_level_value |= data->smc_state_table.VceBootLevel << 16;
-		cgs_write_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_VCEDPM_SetEnabledMask,
-					(uint32_t)1 << data->smc_state_table.VceBootLevel);
-	}
-
-	polaris10_enable_disable_vce_dpm(hwmgr, !bgate);
-
-	return 0;
-}
-
-int polaris10_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t mm_boot_level_offset, mm_boot_level_value;
-
-	if (!bgate) {
-		data->smc_state_table.SamuBootLevel = 0;
-		mm_boot_level_offset = data->dpm_table_start +
-				offsetof(SMU74_Discrete_DpmTable, SamuBootLevel);
-		mm_boot_level_offset /= 4;
-		mm_boot_level_offset *= 4;
-		mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset);
-		mm_boot_level_value &= 0xFFFFFF00;
-		mm_boot_level_value |= data->smc_state_table.SamuBootLevel << 0;
-		cgs_write_ind_register(hwmgr->device,
-				CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_StablePState))
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_SAMUDPM_SetEnabledMask,
-					(uint32_t)(1 << data->smc_state_table.SamuBootLevel));
-	}
-
-	return polaris10_enable_disable_samu_dpm(hwmgr, !bgate);
-}
-
-static int polaris10_update_sclk_threshold(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	int result = 0;
-	uint32_t low_sclk_interrupt_threshold = 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkThrottleLowNotification)
-		&& (hwmgr->gfx_arbiter.sclk_threshold !=
-				data->low_sclk_interrupt_threshold)) {
-		data->low_sclk_interrupt_threshold =
-				hwmgr->gfx_arbiter.sclk_threshold;
-		low_sclk_interrupt_threshold =
-				data->low_sclk_interrupt_threshold;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
-
-		result = polaris10_copy_bytes_to_smc(
-				hwmgr->smumgr,
-				data->dpm_table_start +
-				offsetof(SMU74_Discrete_DpmTable,
-					LowSclkInterruptThreshold),
-				(uint8_t *)&low_sclk_interrupt_threshold,
-				sizeof(uint32_t),
-				data->sram_end);
-	}
-
-	return result;
-}
-
-static int polaris10_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if (data->need_update_smu7_dpm_table &
-		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_OD_UPDATE_MCLK))
-		return polaris10_program_memory_timing_parameters(hwmgr);
-
-	return 0;
-}
-
-static int polaris10_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if (0 == data->need_update_smu7_dpm_table)
-		return 0;
-
-	if ((0 == data->sclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table &
-		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
-
-		PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr),
-				    "Trying to Unfreeze SCLK DPM when DPM is disabled",
-				);
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_SCLKDPM_UnfreezeLevel),
-			"Failed to unfreeze SCLK DPM during UnFreezeSclkMclkDPM Function!",
-			return -1);
-	}
-
-	if ((0 == data->mclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
-
-		PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr),
-				    "Trying to Unfreeze MCLK DPM when DPM is disabled",
-				);
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-				PPSMC_MSG_SCLKDPM_UnfreezeLevel),
-		    "Failed to unfreeze MCLK DPM during UnFreezeSclkMclkDPM Function!",
-		    return -1);
-	}
-
-	data->need_update_smu7_dpm_table = 0;
-
-	return 0;
-}
-
-static int polaris10_notify_link_speed_change_after_state_change(
-		struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states =
-			(const struct phm_set_power_state_input *)input;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	const struct polaris10_power_state *polaris10_ps =
-			cast_const_phw_polaris10_power_state(states->pnew_state);
-	uint16_t target_link_speed = polaris10_get_maximum_link_speed(hwmgr, polaris10_ps);
-	uint8_t  request;
-
-	if (data->pspp_notify_required) {
-		if (target_link_speed == PP_PCIEGen3)
-			request = PCIE_PERF_REQ_GEN3;
-		else if (target_link_speed == PP_PCIEGen2)
-			request = PCIE_PERF_REQ_GEN2;
-		else
-			request = PCIE_PERF_REQ_GEN1;
-
-		if (request == PCIE_PERF_REQ_GEN1 &&
-				phm_get_current_pcie_speed(hwmgr) > 0)
-			return 0;
-
-		if (acpi_pcie_perf_request(hwmgr->device, request, false)) {
-			if (PP_PCIEGen2 == target_link_speed)
-				printk("PSPP request to switch to Gen2 from Gen3 Failed!");
-			else
-				printk("PSPP request to switch to Gen1 from Gen2 Failed!");
-		}
-	}
-
-	return 0;
-}
-
-static int polaris10_notify_smc_display(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-		(PPSMC_Msg)PPSMC_MSG_SetVBITimeout, data->frame_time_x2);
-	return (smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_HasDisplay) == 0) ?  0 : -EINVAL;
-}
-
-
-
-static int polaris10_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *input)
-{
-	int tmp_result, result = 0;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	tmp_result = polaris10_find_dpm_states_clocks_in_dpm_table(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to find DPM states clocks in DPM table!",
-			result = tmp_result);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PCIEPerformanceRequest)) {
-		tmp_result =
-			polaris10_request_link_speed_change_before_state_change(hwmgr, input);
-		PP_ASSERT_WITH_CODE((0 == tmp_result),
-				"Failed to request link speed change before state change!",
-				result = tmp_result);
-	}
-
-	tmp_result = polaris10_freeze_sclk_mclk_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to freeze SCLK MCLK DPM!", result = tmp_result);
-
-	tmp_result = polaris10_populate_and_upload_sclk_mclk_dpm_levels(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to populate and upload SCLK MCLK DPM levels!",
-			result = tmp_result);
-
-	tmp_result = polaris10_generate_dpm_level_enable_mask(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to generate DPM level enabled mask!",
-			result = tmp_result);
-
-	tmp_result = polaris10_update_sclk_threshold(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to update SCLK threshold!",
-			result = tmp_result);
-
-	tmp_result = polaris10_program_mem_timing_parameters(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to program memory timing parameters!",
-			result = tmp_result);
-
-	tmp_result = polaris10_notify_smc_display(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to notify smc display settings!",
-			result = tmp_result);
-
-	tmp_result = polaris10_unfreeze_sclk_mclk_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to unfreeze SCLK MCLK DPM!",
-			result = tmp_result);
-
-	tmp_result = polaris10_upload_dpm_level_enable_mask(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to upload DPM level enabled mask!",
-			result = tmp_result);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PCIEPerformanceRequest)) {
-		tmp_result =
-			polaris10_notify_link_speed_change_after_state_change(hwmgr, input);
-		PP_ASSERT_WITH_CODE((0 == tmp_result),
-				"Failed to notify link speed change after state change!",
-				result = tmp_result);
-	}
-	data->apply_optimized_settings = false;
-	return result;
-}
-
-static int polaris10_set_max_fan_pwm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_pwm)
-{
-	hwmgr->thermal_controller.
-	advanceFanControlParameters.usMaxFanPWM = us_max_fan_pwm;
-
-	if (phm_is_hw_access_blocked(hwmgr))
-		return 0;
-
-	return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-			PPSMC_MSG_SetFanPwmMax, us_max_fan_pwm);
-}
-
-
-int polaris10_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display)
-{
-	PPSMC_Msg msg = has_display ? (PPSMC_Msg)PPSMC_HasDisplay : (PPSMC_Msg)PPSMC_NoDisplay;
-
-	return (smum_send_msg_to_smc(hwmgr->smumgr, msg) == 0) ?  0 : -1;
-}
-
-int polaris10_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
-{
-	uint32_t num_active_displays = 0;
-	struct cgs_display_info info = {0};
-	info.mode_info = NULL;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	num_active_displays = info.display_count;
-
-	if (num_active_displays > 1)  /* to do && (pHwMgr->pPECI->displayConfiguration.bMultiMonitorInSync != TRUE)) */
-		polaris10_notify_smc_display_change(hwmgr, false);
-
-
-	return 0;
-}
-
-/**
-* Programs the display gap
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always OK
-*/
-int polaris10_program_display_gap(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t num_active_displays = 0;
-	uint32_t display_gap = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL);
-	uint32_t display_gap2;
-	uint32_t pre_vbi_time_in_us;
-	uint32_t frame_time_in_us;
-	uint32_t ref_clock;
-	uint32_t refresh_rate = 0;
-	struct cgs_display_info info = {0};
-	struct cgs_mode_info mode_info;
-
-	info.mode_info = &mode_info;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	num_active_displays = info.display_count;
-
-	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL, DISP_GAP, (num_active_displays > 0) ? DISPLAY_GAP_VBLANK_OR_WM : DISPLAY_GAP_IGNORE);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL, display_gap);
-
-	ref_clock = mode_info.ref_clock;
-	refresh_rate = mode_info.refresh_rate;
-
-	if (0 == refresh_rate)
-		refresh_rate = 60;
-
-	frame_time_in_us = 1000000 / refresh_rate;
-
-	pre_vbi_time_in_us = frame_time_in_us - 200 - mode_info.vblank_time_us;
-	data->frame_time_x2 = frame_time_in_us * 2 / 100;
-
-	display_gap2 = pre_vbi_time_in_us * (ref_clock / 100);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL2, display_gap2);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + offsetof(SMU74_SoftRegisters, PreVBlankGap), 0x64);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + offsetof(SMU74_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us));
-
-
-	return 0;
-}
-
-
-int polaris10_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
-{
-	return polaris10_program_display_gap(hwmgr);
-}
-
-/**
-*  Set maximum target operating fan output RPM
-*
-* @param    hwmgr:  the address of the powerplay hardware manager.
-* @param    usMaxFanRpm:  max operating fan RPM value.
-* @return   The response that came from the SMC.
-*/
-static int polaris10_set_max_fan_rpm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_rpm)
-{
-	hwmgr->thermal_controller.
-	advanceFanControlParameters.usMaxFanRPM = us_max_fan_rpm;
-
-	if (phm_is_hw_access_blocked(hwmgr))
-		return 0;
-
-	return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-			PPSMC_MSG_SetFanRpmMax, us_max_fan_rpm);
-}
-
-int polaris10_register_internal_thermal_interrupt(struct pp_hwmgr *hwmgr,
-					const void *thermal_interrupt_info)
-{
-	return 0;
-}
-
-bool polaris10_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	bool is_update_required = false;
-	struct cgs_display_info info = {0, 0, NULL};
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	if (data->display_timing.num_existing_displays != info.display_count)
-		is_update_required = true;
-/* TO DO NEED TO GET DEEP SLEEP CLOCK FROM DAL
-	if (phm_cap_enabled(hwmgr->hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep)) {
-		cgs_get_min_clock_settings(hwmgr->device, &min_clocks);
-		if (min_clocks.engineClockInSR != data->display_timing.minClockInSR &&
-			(min_clocks.engineClockInSR >= POLARIS10_MINIMUM_ENGINE_CLOCK ||
-				data->display_timing.minClockInSR >= POLARIS10_MINIMUM_ENGINE_CLOCK))
-			is_update_required = true;
-*/
-	return is_update_required;
-}
-
-static inline bool polaris10_are_power_levels_equal(const struct polaris10_performance_level *pl1,
-							   const struct polaris10_performance_level *pl2)
-{
-	return ((pl1->memory_clock == pl2->memory_clock) &&
-		  (pl1->engine_clock == pl2->engine_clock) &&
-		  (pl1->pcie_gen == pl2->pcie_gen) &&
-		  (pl1->pcie_lane == pl2->pcie_lane));
-}
-
-int polaris10_check_states_equal(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *pstate1, const struct pp_hw_power_state *pstate2, bool *equal)
-{
-	const struct polaris10_power_state *psa = cast_const_phw_polaris10_power_state(pstate1);
-	const struct polaris10_power_state *psb = cast_const_phw_polaris10_power_state(pstate2);
-	int i;
-
-	if (pstate1 == NULL || pstate2 == NULL || equal == NULL)
-		return -EINVAL;
-
-	/* If the two states don't even have the same number of performance levels they cannot be the same state. */
-	if (psa->performance_level_count != psb->performance_level_count) {
-		*equal = false;
-		return 0;
-	}
-
-	for (i = 0; i < psa->performance_level_count; i++) {
-		if (!polaris10_are_power_levels_equal(&(psa->performance_levels[i]), &(psb->performance_levels[i]))) {
-			/* If we have found even one performance level pair that is different the states are different. */
-			*equal = false;
-			return 0;
-		}
-	}
-
-	/* If all performance levels are the same try to use the UVD clocks to break the tie.*/
-	*equal = ((psa->uvd_clks.vclk == psb->uvd_clks.vclk) && (psa->uvd_clks.dclk == psb->uvd_clks.dclk));
-	*equal &= ((psa->vce_clks.evclk == psb->vce_clks.evclk) && (psa->vce_clks.ecclk == psb->vce_clks.ecclk));
-	*equal &= (psa->sclk_threshold == psb->sclk_threshold);
-
-	return 0;
-}
-
-int polaris10_upload_mc_firmware(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	uint32_t vbios_version;
-
-	/*  Read MC indirect register offset 0x9F bits [3:0] to see if VBIOS has already loaded a full version of MC ucode or not.*/
-
-	phm_get_mc_microcode_version(hwmgr);
-	vbios_version = hwmgr->microcode_version_info.MC & 0xf;
-	/*  Full version of MC ucode has already been loaded. */
-	if (vbios_version == 0) {
-		data->need_long_memory_training = false;
-		return 0;
-	}
-
-	data->need_long_memory_training = false;
-
-/*
- *	PPMCME_FirmwareDescriptorEntry *pfd = NULL;
-	pfd = &tonga_mcmeFirmware;
-	if (0 == PHM_READ_FIELD(hwmgr->device, MC_SEQ_SUP_CNTL, RUN))
-		polaris10_load_mc_microcode(hwmgr, pfd->dpmThreshold,
-					pfd->cfgArray, pfd->cfgSize, pfd->ioDebugArray,
-					pfd->ioDebugSize, pfd->ucodeArray, pfd->ucodeSize);
-*/
-	return 0;
-}
-
-/**
- * Read clock related registers.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-static int polaris10_read_clock_registers(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	data->clock_registers.vCG_SPLL_FUNC_CNTL = cgs_read_ind_register(hwmgr->device,
-						CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL)
-						& CG_SPLL_FUNC_CNTL__SPLL_BYPASS_EN_MASK;
-
-	data->clock_registers.vCG_SPLL_FUNC_CNTL_2 = cgs_read_ind_register(hwmgr->device,
-						CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_2)
-						& CG_SPLL_FUNC_CNTL_2__SCLK_MUX_SEL_MASK;
-
-	data->clock_registers.vCG_SPLL_FUNC_CNTL_4 = cgs_read_ind_register(hwmgr->device,
-						CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_4)
-						& CG_SPLL_FUNC_CNTL_4__SPLL_SPARE_MASK;
-
-	return 0;
-}
-
-/**
- * Find out if memory is GDDR5.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-static int polaris10_get_memory_type(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t temp;
-
-	temp = cgs_read_register(hwmgr->device, mmMC_SEQ_MISC0);
-
-	data->is_memory_gddr5 = (MC_SEQ_MISC0_GDDR5_VALUE ==
-			((temp & MC_SEQ_MISC0_GDDR5_MASK) >>
-			 MC_SEQ_MISC0_GDDR5_SHIFT));
-
-	return 0;
-}
-
-/**
- * Enables Dynamic Power Management by SMC
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-static int polaris10_enable_acpi_power_management(struct pp_hwmgr *hwmgr)
-{
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			GENERAL_PWRMGT, STATIC_PM_EN, 1);
-
-	return 0;
-}
-
-/**
- * Initialize PowerGating States for different engines
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-static int polaris10_init_power_gate_state(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	data->uvd_power_gated = false;
-	data->vce_power_gated = false;
-	data->samu_power_gated = false;
-
-	return 0;
-}
-
-static int polaris10_init_sclk_threshold(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	data->low_sclk_interrupt_threshold = 0;
-
-	return 0;
-}
-
-int polaris10_setup_asic_task(struct pp_hwmgr *hwmgr)
-{
-	int tmp_result, result = 0;
-
-	polaris10_upload_mc_firmware(hwmgr);
-
-	tmp_result = polaris10_read_clock_registers(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to read clock registers!", result = tmp_result);
-
-	tmp_result = polaris10_get_memory_type(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to get memory type!", result = tmp_result);
-
-	tmp_result = polaris10_enable_acpi_power_management(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable ACPI power management!", result = tmp_result);
-
-	tmp_result = polaris10_init_power_gate_state(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to init power gate state!", result = tmp_result);
-
-	tmp_result = phm_get_mc_microcode_version(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to get MC microcode version!", result = tmp_result);
-
-	tmp_result = polaris10_init_sclk_threshold(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to init sclk threshold!", result = tmp_result);
-
-	return result;
-}
-
-static int polaris10_force_clock_level(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, uint32_t mask)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if (hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL)
-		return -EINVAL;
-
-	switch (type) {
-	case PP_SCLK:
-		if (!data->sclk_dpm_key_disabled)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_SCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.sclk_dpm_enable_mask & mask);
-		break;
-	case PP_MCLK:
-		if (!data->mclk_dpm_key_disabled)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_MCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.mclk_dpm_enable_mask & mask);
-		break;
-	case PP_PCIE:
-	{
-		uint32_t tmp = mask & data->dpm_level_enable_mask.pcie_dpm_enable_mask;
-		uint32_t level = 0;
-
-		while (tmp >>= 1)
-			level++;
-
-		if (!data->pcie_dpm_key_disabled)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_PCIeDPM_ForceLevel,
-					level);
-		break;
-	}
-	default:
-		break;
-	}
-
-	return 0;
-}
-
-static uint16_t polaris10_get_current_pcie_speed(struct pp_hwmgr *hwmgr)
-{
-	uint32_t speedCntl = 0;
-
-	/* mmPCIE_PORT_INDEX rename as mmPCIE_INDEX */
-	speedCntl = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__PCIE,
-			ixPCIE_LC_SPEED_CNTL);
-	return((uint16_t)PHM_GET_FIELD(speedCntl,
-			PCIE_LC_SPEED_CNTL, LC_CURRENT_DATA_RATE));
-}
-
-static int polaris10_print_clock_levels(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, char *buf)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
-	struct polaris10_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
-	struct polaris10_single_dpm_table *pcie_table = &(data->dpm_table.pcie_speed_table);
-	int i, now, size = 0;
-	uint32_t clock, pcie_speed;
-
-	switch (type) {
-	case PP_SCLK:
-		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetSclkFrequency);
-		clock = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-
-		for (i = 0; i < sclk_table->count; i++) {
-			if (clock > sclk_table->dpm_levels[i].value)
-				continue;
-			break;
-		}
-		now = i;
-
-		for (i = 0; i < sclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, sclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
-		break;
-	case PP_MCLK:
-		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetMclkFrequency);
-		clock = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-
-		for (i = 0; i < mclk_table->count; i++) {
-			if (clock > mclk_table->dpm_levels[i].value)
-				continue;
-			break;
-		}
-		now = i;
-
-		for (i = 0; i < mclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, mclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
-		break;
-	case PP_PCIE:
-		pcie_speed = polaris10_get_current_pcie_speed(hwmgr);
-		for (i = 0; i < pcie_table->count; i++) {
-			if (pcie_speed != pcie_table->dpm_levels[i].value)
-				continue;
-			break;
-		}
-		now = i;
-
-		for (i = 0; i < pcie_table->count; i++)
-			size += sprintf(buf + size, "%d: %s %s\n", i,
-					(pcie_table->dpm_levels[i].value == 0) ? "2.5GB, x8" :
-					(pcie_table->dpm_levels[i].value == 1) ? "5.0GB, x16" :
-					(pcie_table->dpm_levels[i].value == 2) ? "8.0GB, x16" : "",
-					(i == now) ? "*" : "");
-		break;
-	default:
-		break;
-	}
-	return size;
-}
-
-static int polaris10_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
-{
-	if (mode) {
-		/* stop auto-manage */
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_MicrocodeFanControl))
-			polaris10_fan_ctrl_stop_smc_fan_control(hwmgr);
-		polaris10_fan_ctrl_set_static_mode(hwmgr, mode);
-	} else
-		/* restart auto-manage */
-		polaris10_fan_ctrl_reset_fan_speed_to_default(hwmgr);
-
-	return 0;
-}
-
-static int polaris10_get_fan_control_mode(struct pp_hwmgr *hwmgr)
-{
-	if (hwmgr->fan_ctrl_is_in_default_mode)
-		return hwmgr->fan_ctrl_default_mode;
-	else
-		return PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				CG_FDO_CTRL2, FDO_PWM_MODE);
-}
-
-static int polaris10_get_sclk_od(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
-	struct polaris10_single_dpm_table *golden_sclk_table =
-			&(data->golden_dpm_table.sclk_table);
-	int value;
-
-	value = (sclk_table->dpm_levels[sclk_table->count - 1].value -
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) *
-			100 /
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
-
-	return value;
-}
-
-static int polaris10_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_single_dpm_table *golden_sclk_table =
-			&(data->golden_dpm_table.sclk_table);
-	struct pp_power_state  *ps;
-	struct polaris10_power_state  *polaris10_ps;
-
-	if (value > 20)
-		value = 20;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	polaris10_ps = cast_phw_polaris10_power_state(&ps->hardware);
-
-	polaris10_ps->performance_levels[polaris10_ps->performance_level_count - 1].engine_clock =
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value *
-			value / 100 +
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
-
-	return 0;
-}
-
-static int polaris10_get_mclk_od(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
-	struct polaris10_single_dpm_table *golden_mclk_table =
-			&(data->golden_dpm_table.mclk_table);
-	int value;
-
-	value = (mclk_table->dpm_levels[mclk_table->count - 1].value -
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) *
-			100 /
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
-
-	return value;
-}
-
-static int polaris10_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct polaris10_single_dpm_table *golden_mclk_table =
-			&(data->golden_dpm_table.mclk_table);
-	struct pp_power_state  *ps;
-	struct polaris10_power_state  *polaris10_ps;
-
-	if (value > 20)
-		value = 20;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	polaris10_ps = cast_phw_polaris10_power_state(&ps->hardware);
-
-	polaris10_ps->performance_levels[polaris10_ps->performance_level_count - 1].memory_clock =
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value *
-			value / 100 +
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
-
-	return 0;
-}
-static const struct pp_hwmgr_func polaris10_hwmgr_funcs = {
-	.backend_init = &polaris10_hwmgr_backend_init,
-	.backend_fini = &polaris10_hwmgr_backend_fini,
-	.asic_setup = &polaris10_setup_asic_task,
-	.dynamic_state_management_enable = &polaris10_enable_dpm_tasks,
-	.apply_state_adjust_rules = polaris10_apply_state_adjust_rules,
-	.force_dpm_level = &polaris10_force_dpm_level,
-	.power_state_set = polaris10_set_power_state_tasks,
-	.get_power_state_size = polaris10_get_power_state_size,
-	.get_mclk = polaris10_dpm_get_mclk,
-	.get_sclk = polaris10_dpm_get_sclk,
-	.patch_boot_state = polaris10_dpm_patch_boot_state,
-	.get_pp_table_entry = polaris10_get_pp_table_entry,
-	.get_num_of_pp_table_entries = tonga_get_number_of_powerplay_table_entries,
-	.print_current_perforce_level = polaris10_print_current_perforce_level,
-	.powerdown_uvd = polaris10_phm_powerdown_uvd,
-	.powergate_uvd = polaris10_phm_powergate_uvd,
-	.powergate_vce = polaris10_phm_powergate_vce,
-	.disable_clock_power_gating = polaris10_phm_disable_clock_power_gating,
-	.update_clock_gatings = polaris10_phm_update_clock_gatings,
-	.notify_smc_display_config_after_ps_adjustment = polaris10_notify_smc_display_config_after_ps_adjustment,
-	.display_config_changed = polaris10_display_configuration_changed_task,
-	.set_max_fan_pwm_output = polaris10_set_max_fan_pwm_output,
-	.set_max_fan_rpm_output = polaris10_set_max_fan_rpm_output,
-	.get_temperature = polaris10_thermal_get_temperature,
-	.stop_thermal_controller = polaris10_thermal_stop_thermal_controller,
-	.get_fan_speed_info = polaris10_fan_ctrl_get_fan_speed_info,
-	.get_fan_speed_percent = polaris10_fan_ctrl_get_fan_speed_percent,
-	.set_fan_speed_percent = polaris10_fan_ctrl_set_fan_speed_percent,
-	.reset_fan_speed_to_default = polaris10_fan_ctrl_reset_fan_speed_to_default,
-	.get_fan_speed_rpm = polaris10_fan_ctrl_get_fan_speed_rpm,
-	.set_fan_speed_rpm = polaris10_fan_ctrl_set_fan_speed_rpm,
-	.uninitialize_thermal_controller = polaris10_thermal_ctrl_uninitialize_thermal_controller,
-	.register_internal_thermal_interrupt = polaris10_register_internal_thermal_interrupt,
-	.check_smc_update_required_for_display_configuration = polaris10_check_smc_update_required_for_display_configuration,
-	.check_states_equal = polaris10_check_states_equal,
-	.set_fan_control_mode = polaris10_set_fan_control_mode,
-	.get_fan_control_mode = polaris10_get_fan_control_mode,
-	.force_clock_level = polaris10_force_clock_level,
-	.print_clock_levels = polaris10_print_clock_levels,
-	.enable_per_cu_power_gating = polaris10_phm_enable_per_cu_power_gating,
-	.get_sclk_od = polaris10_get_sclk_od,
-	.set_sclk_od = polaris10_set_sclk_od,
-	.get_mclk_od = polaris10_get_mclk_od,
-	.set_mclk_od = polaris10_set_mclk_od,
-};
-
-int polaris10_hwmgr_init(struct pp_hwmgr *hwmgr)
-{
-	hwmgr->hwmgr_func = &polaris10_hwmgr_funcs;
-	hwmgr->pptable_func = &tonga_pptable_funcs;
-	pp_polaris10_thermal_initialize(hwmgr);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.c
deleted file mode 100644
index b206632d4650..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.c
+++ /dev/null
@@ -1,716 +0,0 @@
-/*
- * Copyright 2016 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <asm/div64.h>
-#include "polaris10_thermal.h"
-#include "polaris10_hwmgr.h"
-#include "polaris10_smumgr.h"
-#include "polaris10_ppsmc.h"
-#include "smu/smu_7_1_3_d.h"
-#include "smu/smu_7_1_3_sh_mask.h"
-
-int polaris10_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr,
-		struct phm_fan_speed_info *fan_speed_info)
-{
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		return 0;
-
-	fan_speed_info->supports_percent_read = true;
-	fan_speed_info->supports_percent_write = true;
-	fan_speed_info->min_percent = 0;
-	fan_speed_info->max_percent = 100;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_FanSpeedInTableIsRPM) &&
-		hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution) {
-		fan_speed_info->supports_rpm_read = true;
-		fan_speed_info->supports_rpm_write = true;
-		fan_speed_info->min_rpm = hwmgr->thermal_controller.fanInfo.ulMinRPM;
-		fan_speed_info->max_rpm = hwmgr->thermal_controller.fanInfo.ulMaxRPM;
-	} else {
-		fan_speed_info->min_rpm = 0;
-		fan_speed_info->max_rpm = 0;
-	}
-
-	return 0;
-}
-
-int polaris10_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr,
-		uint32_t *speed)
-{
-	uint32_t duty100;
-	uint32_t duty;
-	uint64_t tmp64;
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		return 0;
-
-	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_FDO_CTRL1, FMAX_DUTY100);
-	duty = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_THERMAL_STATUS, FDO_PWM_DUTY);
-
-	if (duty100 == 0)
-		return -EINVAL;
-
-
-	tmp64 = (uint64_t)duty * 100;
-	do_div(tmp64, duty100);
-	*speed = (uint32_t)tmp64;
-
-	if (*speed > 100)
-		*speed = 100;
-
-	return 0;
-}
-
-int polaris10_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
-{
-	uint32_t tach_period;
-	uint32_t crystal_clock_freq;
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan ||
-			(hwmgr->thermal_controller.fanInfo.
-				ucTachometerPulsesPerRevolution == 0))
-		return 0;
-
-	tach_period = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_TACH_STATUS, TACH_PERIOD);
-
-	if (tach_period == 0)
-		return -EINVAL;
-
-	crystal_clock_freq = tonga_get_xclk(hwmgr);
-
-	*speed = 60 * crystal_clock_freq * 10000 / tach_period;
-
-	return 0;
-}
-
-/**
-* Set Fan Speed Control to static mode, so that the user can decide what speed to use.
-* @param    hwmgr  the address of the powerplay hardware manager.
-*           mode    the fan control mode, 0 default, 1 by percent, 5, by RPM
-* @exception Should always succeed.
-*/
-int polaris10_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
-{
-
-	if (hwmgr->fan_ctrl_is_in_default_mode) {
-		hwmgr->fan_ctrl_default_mode =
-				PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,	CGS_IND_REG__SMC,
-						CG_FDO_CTRL2, FDO_PWM_MODE);
-		hwmgr->tmin =
-				PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-						CG_FDO_CTRL2, TMIN);
-		hwmgr->fan_ctrl_is_in_default_mode = false;
-	}
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_FDO_CTRL2, TMIN, 0);
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_FDO_CTRL2, FDO_PWM_MODE, mode);
-
-	return 0;
-}
-
-/**
-* Reset Fan Speed Control to default mode.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @exception Should always succeed.
-*/
-int polaris10_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr)
-{
-	if (!hwmgr->fan_ctrl_is_in_default_mode) {
-		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				CG_FDO_CTRL2, FDO_PWM_MODE, hwmgr->fan_ctrl_default_mode);
-		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				CG_FDO_CTRL2, TMIN, hwmgr->tmin);
-		hwmgr->fan_ctrl_is_in_default_mode = true;
-	}
-
-	return 0;
-}
-
-int polaris10_fan_ctrl_start_smc_fan_control(struct pp_hwmgr *hwmgr)
-{
-	int result;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ODFuzzyFanControlSupport)) {
-		cgs_write_register(hwmgr->device, mmSMC_MSG_ARG_0, FAN_CONTROL_FUZZY);
-		result = smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StartFanControl);
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_FanSpeedInTableIsRPM))
-			hwmgr->hwmgr_func->set_max_fan_rpm_output(hwmgr,
-					hwmgr->thermal_controller.
-					advanceFanControlParameters.usMaxFanRPM);
-		else
-			hwmgr->hwmgr_func->set_max_fan_pwm_output(hwmgr,
-					hwmgr->thermal_controller.
-					advanceFanControlParameters.usMaxFanPWM);
-
-	} else {
-		cgs_write_register(hwmgr->device, mmSMC_MSG_ARG_0, FAN_CONTROL_TABLE);
-		result = smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StartFanControl);
-	}
-
-	if (!result && hwmgr->thermal_controller.
-			advanceFanControlParameters.ucTargetTemperature)
-		result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-				PPSMC_MSG_SetFanTemperatureTarget,
-				hwmgr->thermal_controller.
-				advanceFanControlParameters.ucTargetTemperature);
-
-	return result;
-}
-
-
-int polaris10_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StopFanControl);
-}
-
-/**
-* Set Fan Speed in percent.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    speed is the percentage value (0% - 100%) to be set.
-* @exception Fails is the 100% setting appears to be 0.
-*/
-int polaris10_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
-		uint32_t speed)
-{
-	uint32_t duty100;
-	uint32_t duty;
-	uint64_t tmp64;
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		return 0;
-
-	if (speed > 100)
-		speed = 100;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_MicrocodeFanControl))
-		polaris10_fan_ctrl_stop_smc_fan_control(hwmgr);
-
-	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_FDO_CTRL1, FMAX_DUTY100);
-
-	if (duty100 == 0)
-		return -EINVAL;
-
-	tmp64 = (uint64_t)speed * duty100;
-	do_div(tmp64, 100);
-	duty = (uint32_t)tmp64;
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_FDO_CTRL0, FDO_STATIC_DUTY, duty);
-
-	return polaris10_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
-}
-
-/**
-* Reset Fan Speed to default.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @exception Always succeeds.
-*/
-int polaris10_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
-{
-	int result;
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		return 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_MicrocodeFanControl)) {
-		result = polaris10_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
-		if (!result)
-			result = polaris10_fan_ctrl_start_smc_fan_control(hwmgr);
-	} else
-		result = polaris10_fan_ctrl_set_default_mode(hwmgr);
-
-	return result;
-}
-
-/**
-* Set Fan Speed in RPM.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    speed is the percentage value (min - max) to be set.
-* @exception Fails is the speed not lie between min and max.
-*/
-int polaris10_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
-{
-	uint32_t tach_period;
-	uint32_t crystal_clock_freq;
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan ||
-			(hwmgr->thermal_controller.fanInfo.
-			ucTachometerPulsesPerRevolution == 0) ||
-			(speed < hwmgr->thermal_controller.fanInfo.ulMinRPM) ||
-			(speed > hwmgr->thermal_controller.fanInfo.ulMaxRPM))
-		return 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_MicrocodeFanControl))
-		polaris10_fan_ctrl_stop_smc_fan_control(hwmgr);
-
-	crystal_clock_freq = tonga_get_xclk(hwmgr);
-
-	tach_period = 60 * crystal_clock_freq * 10000 / (8 * speed);
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				CG_TACH_STATUS, TACH_PERIOD, tach_period);
-
-	return polaris10_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
-}
-
-/**
-* Reads the remote temperature from the SIslands thermal controller.
-*
-* @param    hwmgr The address of the hardware manager.
-*/
-int polaris10_thermal_get_temperature(struct pp_hwmgr *hwmgr)
-{
-	int temp;
-
-	temp = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_MULT_THERMAL_STATUS, CTF_TEMP);
-
-	/* Bit 9 means the reading is lower than the lowest usable value. */
-	if (temp & 0x200)
-		temp = POLARIS10_THERMAL_MAXIMUM_TEMP_READING;
-	else
-		temp = temp & 0x1ff;
-
-	temp *= PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
-
-	return temp;
-}
-
-/**
-* Set the requested temperature range for high and low alert signals
-*
-* @param    hwmgr The address of the hardware manager.
-* @param    range Temperature range to be programmed for high and low alert signals
-* @exception PP_Result_BadInput if the input data is not valid.
-*/
-static int polaris10_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
-		uint32_t low_temp, uint32_t high_temp)
-{
-	uint32_t low = POLARIS10_THERMAL_MINIMUM_ALERT_TEMP *
-			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
-	uint32_t high = POLARIS10_THERMAL_MAXIMUM_ALERT_TEMP *
-			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
-
-	if (low < low_temp)
-		low = low_temp;
-	if (high > high_temp)
-		high = high_temp;
-
-	if (low > high)
-		return -EINVAL;
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_THERMAL_INT, DIG_THERM_INTH,
-			(high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_THERMAL_INT, DIG_THERM_INTL,
-			(low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_THERMAL_CTRL, DIG_THERM_DPM,
-			(high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
-
-	return 0;
-}
-
-/**
-* Programs thermal controller one-time setting registers
-*
-* @param    hwmgr The address of the hardware manager.
-*/
-static int polaris10_thermal_initialize(struct pp_hwmgr *hwmgr)
-{
-	if (hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution)
-		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				CG_TACH_CTRL, EDGE_PER_REV,
-				hwmgr->thermal_controller.fanInfo.
-				ucTachometerPulsesPerRevolution - 1);
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_FDO_CTRL2, TACH_PWM_RESP_RATE, 0x28);
-
-	return 0;
-}
-
-/**
-* Enable thermal alerts on the RV770 thermal controller.
-*
-* @param    hwmgr The address of the hardware manager.
-*/
-static int polaris10_thermal_enable_alert(struct pp_hwmgr *hwmgr)
-{
-	uint32_t alert;
-
-	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_THERMAL_INT, THERM_INT_MASK);
-	alert &= ~(POLARIS10_THERMAL_HIGH_ALERT_MASK | POLARIS10_THERMAL_LOW_ALERT_MASK);
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_THERMAL_INT, THERM_INT_MASK, alert);
-
-	/* send message to SMU to enable internal thermal interrupts */
-	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Thermal_Cntl_Enable);
-}
-
-/**
-* Disable thermal alerts on the RV770 thermal controller.
-* @param    hwmgr The address of the hardware manager.
-*/
-static int polaris10_thermal_disable_alert(struct pp_hwmgr *hwmgr)
-{
-	uint32_t alert;
-
-	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_THERMAL_INT, THERM_INT_MASK);
-	alert |= (POLARIS10_THERMAL_HIGH_ALERT_MASK | POLARIS10_THERMAL_LOW_ALERT_MASK);
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_THERMAL_INT, THERM_INT_MASK, alert);
-
-	/* send message to SMU to disable internal thermal interrupts */
-	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Thermal_Cntl_Disable);
-}
-
-/**
-* Uninitialize the thermal controller.
-* Currently just disables alerts.
-* @param    hwmgr The address of the hardware manager.
-*/
-int polaris10_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr)
-{
-	int result = polaris10_thermal_disable_alert(hwmgr);
-
-	if (!hwmgr->thermal_controller.fanInfo.bNoFan)
-		polaris10_fan_ctrl_set_default_mode(hwmgr);
-
-	return result;
-}
-
-/**
-* Set up the fan table to control the fan using the SMC.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from set temperature range routine
-*/
-int tf_polaris10_thermal_setup_fan_table(struct pp_hwmgr *hwmgr,
-		void *input, void *output, void *storage, int result)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	SMU74_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
-	uint32_t duty100;
-	uint32_t t_diff1, t_diff2, pwm_diff1, pwm_diff2;
-	uint16_t fdo_min, slope1, slope2;
-	uint32_t reference_clock;
-	int res;
-	uint64_t tmp64;
-
-	if (data->fan_table_start == 0) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_MicrocodeFanControl);
-		return 0;
-	}
-
-	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_FDO_CTRL1, FMAX_DUTY100);
-
-	if (duty100 == 0) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_MicrocodeFanControl);
-		return 0;
-	}
-
-	tmp64 = hwmgr->thermal_controller.advanceFanControlParameters.
-			usPWMMin * duty100;
-	do_div(tmp64, 10000);
-	fdo_min = (uint16_t)tmp64;
-
-	t_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usTMed -
-			hwmgr->thermal_controller.advanceFanControlParameters.usTMin;
-	t_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usTHigh -
-			hwmgr->thermal_controller.advanceFanControlParameters.usTMed;
-
-	pwm_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed -
-			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin;
-	pwm_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh -
-			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed;
-
-	slope1 = (uint16_t)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
-	slope2 = (uint16_t)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
-
-	fan_table.TempMin = cpu_to_be16((50 + hwmgr->
-			thermal_controller.advanceFanControlParameters.usTMin) / 100);
-	fan_table.TempMed = cpu_to_be16((50 + hwmgr->
-			thermal_controller.advanceFanControlParameters.usTMed) / 100);
-	fan_table.TempMax = cpu_to_be16((50 + hwmgr->
-			thermal_controller.advanceFanControlParameters.usTMax) / 100);
-
-	fan_table.Slope1 = cpu_to_be16(slope1);
-	fan_table.Slope2 = cpu_to_be16(slope2);
-
-	fan_table.FdoMin = cpu_to_be16(fdo_min);
-
-	fan_table.HystDown = cpu_to_be16(hwmgr->
-			thermal_controller.advanceFanControlParameters.ucTHyst);
-
-	fan_table.HystUp = cpu_to_be16(1);
-
-	fan_table.HystSlope = cpu_to_be16(1);
-
-	fan_table.TempRespLim = cpu_to_be16(5);
-
-	reference_clock = tonga_get_xclk(hwmgr);
-
-	fan_table.RefreshPeriod = cpu_to_be32((hwmgr->
-			thermal_controller.advanceFanControlParameters.ulCycleDelay *
-			reference_clock) / 1600);
-
-	fan_table.FdoMax = cpu_to_be16((uint16_t)duty100);
-
-	fan_table.TempSrc = (uint8_t)PHM_READ_VFPF_INDIRECT_FIELD(
-			hwmgr->device, CGS_IND_REG__SMC,
-			CG_MULT_THERMAL_CTRL, TEMP_SEL);
-
-	res = polaris10_copy_bytes_to_smc(hwmgr->smumgr, data->fan_table_start,
-			(uint8_t *)&fan_table, (uint32_t)sizeof(fan_table),
-			data->sram_end);
-
-	if (!res && hwmgr->thermal_controller.
-			advanceFanControlParameters.ucMinimumPWMLimit)
-		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-				PPSMC_MSG_SetFanMinPwm,
-				hwmgr->thermal_controller.
-				advanceFanControlParameters.ucMinimumPWMLimit);
-
-	if (!res && hwmgr->thermal_controller.
-			advanceFanControlParameters.ulMinFanSCLKAcousticLimit)
-		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-				PPSMC_MSG_SetFanSclkTarget,
-				hwmgr->thermal_controller.
-				advanceFanControlParameters.ulMinFanSCLKAcousticLimit);
-
-	if (res)
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_MicrocodeFanControl);
-
-	return 0;
-}
-
-/**
-* Start the fan control on the SMC.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from set temperature range routine
-*/
-int tf_polaris10_thermal_start_smc_fan_control(struct pp_hwmgr *hwmgr,
-		void *input, void *output, void *storage, int result)
-{
-/* If the fantable setup has failed we could have disabled
- * PHM_PlatformCaps_MicrocodeFanControl even after
- * this function was included in the table.
- * Make sure that we still think controlling the fan is OK.
-*/
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_MicrocodeFanControl)) {
-		polaris10_fan_ctrl_start_smc_fan_control(hwmgr);
-		polaris10_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
-	}
-
-	return 0;
-}
-
-/**
-* Set temperature range for high and low alerts
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from set temperature range routine
-*/
-int tf_polaris10_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
-		void *input, void *output, void *storage, int result)
-{
-	struct PP_TemperatureRange *range = (struct PP_TemperatureRange *)input;
-
-	if (range == NULL)
-		return -EINVAL;
-
-	return polaris10_thermal_set_temperature_range(hwmgr, range->min, range->max);
-}
-
-/**
-* Programs one-time setting registers
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from initialize thermal controller routine
-*/
-int tf_polaris10_thermal_initialize(struct pp_hwmgr *hwmgr,
-		void *input, void *output, void *storage, int result)
-{
-    return polaris10_thermal_initialize(hwmgr);
-}
-
-/**
-* Enable high and low alerts
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from enable alert routine
-*/
-int tf_polaris10_thermal_enable_alert(struct pp_hwmgr *hwmgr,
-		void *input, void *output, void *storage, int result)
-{
-	return polaris10_thermal_enable_alert(hwmgr);
-}
-
-/**
-* Disable high and low alerts
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from disable alert routine
-*/
-static int tf_polaris10_thermal_disable_alert(struct pp_hwmgr *hwmgr,
-		void *input, void *output, void *storage, int result)
-{
-	return polaris10_thermal_disable_alert(hwmgr);
-}
-
-static int tf_polaris10_thermal_avfs_enable(struct pp_hwmgr *hwmgr,
-		void *input, void *output, void *storage, int result)
-{
-	int ret;
-	struct pp_smumgr *smumgr = (struct pp_smumgr *)(hwmgr->smumgr);
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if (smu_data->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED)
-		return 0;
-
-	ret = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-			PPSMC_MSG_SetGBDroopSettings, data->avfs_vdroop_override_setting);
-
-	ret = (smum_send_msg_to_smc(smumgr, PPSMC_MSG_EnableAvfs) == 0) ?
-			0 : -1;
-
-	if (!ret)
-		/* If this param is not changed, this function could fire unnecessarily */
-		smu_data->avfs.avfs_btc_status = AVFS_BTC_COMPLETED_PREVIOUSLY;
-
-	return ret;
-}
-
-static const struct phm_master_table_item
-polaris10_thermal_start_thermal_controller_master_list[] = {
-	{NULL, tf_polaris10_thermal_initialize},
-	{NULL, tf_polaris10_thermal_set_temperature_range},
-	{NULL, tf_polaris10_thermal_enable_alert},
-	{NULL, tf_polaris10_thermal_avfs_enable},
-/* We should restrict performance levels to low before we halt the SMC.
- * On the other hand we are still in boot state when we do this
- * so it would be pointless.
- * If this assumption changes we have to revisit this table.
- */
-	{NULL, tf_polaris10_thermal_setup_fan_table},
-	{NULL, tf_polaris10_thermal_start_smc_fan_control},
-	{NULL, NULL}
-};
-
-static const struct phm_master_table_header
-polaris10_thermal_start_thermal_controller_master = {
-	0,
-	PHM_MasterTableFlag_None,
-	polaris10_thermal_start_thermal_controller_master_list
-};
-
-static const struct phm_master_table_item
-polaris10_thermal_set_temperature_range_master_list[] = {
-	{NULL, tf_polaris10_thermal_disable_alert},
-	{NULL, tf_polaris10_thermal_set_temperature_range},
-	{NULL, tf_polaris10_thermal_enable_alert},
-	{NULL, NULL}
-};
-
-static const struct phm_master_table_header
-polaris10_thermal_set_temperature_range_master = {
-	0,
-	PHM_MasterTableFlag_None,
-	polaris10_thermal_set_temperature_range_master_list
-};
-
-int polaris10_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr)
-{
-	if (!hwmgr->thermal_controller.fanInfo.bNoFan)
-		polaris10_fan_ctrl_set_default_mode(hwmgr);
-	return 0;
-}
-
-/**
-* Initializes the thermal controller related functions in the Hardware Manager structure.
-* @param    hwmgr The address of the hardware manager.
-* @exception Any error code from the low-level communication.
-*/
-int pp_polaris10_thermal_initialize(struct pp_hwmgr *hwmgr)
-{
-	int result;
-
-	result = phm_construct_table(hwmgr,
-			&polaris10_thermal_set_temperature_range_master,
-			&(hwmgr->set_temperature_range));
-
-	if (!result) {
-		result = phm_construct_table(hwmgr,
-				&polaris10_thermal_start_thermal_controller_master,
-				&(hwmgr->start_thermal_controller));
-		if (result)
-			phm_destroy_table(hwmgr, &(hwmgr->set_temperature_range));
-	}
-
-	if (!result)
-		hwmgr->fan_ctrl_is_in_default_mode = true;
-	return result;
-}
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.h b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.h
deleted file mode 100644
index 62f8cbc2d590..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_thermal.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2016 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef _POLARIS10_THERMAL_H_
-#define _POLARIS10_THERMAL_H_
-
-#include "hwmgr.h"
-
-#define POLARIS10_THERMAL_HIGH_ALERT_MASK         0x1
-#define POLARIS10_THERMAL_LOW_ALERT_MASK          0x2
-
-#define POLARIS10_THERMAL_MINIMUM_TEMP_READING    -256
-#define POLARIS10_THERMAL_MAXIMUM_TEMP_READING    255
-
-#define POLARIS10_THERMAL_MINIMUM_ALERT_TEMP      0
-#define POLARIS10_THERMAL_MAXIMUM_ALERT_TEMP      255
-
-#define FDO_PWM_MODE_STATIC  1
-#define FDO_PWM_MODE_STATIC_RPM 5
-
-
-extern int tf_polaris10_thermal_initialize(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
-extern int tf_polaris10_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
-extern int tf_polaris10_thermal_enable_alert(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
-
-extern int polaris10_thermal_get_temperature(struct pp_hwmgr *hwmgr);
-extern int polaris10_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr);
-extern int polaris10_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info);
-extern int polaris10_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t *speed);
-extern int polaris10_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr);
-extern int polaris10_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode);
-extern int polaris10_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed);
-extern int polaris10_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr);
-extern int pp_polaris10_thermal_initialize(struct pp_hwmgr *hwmgr);
-extern int polaris10_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr);
-extern int polaris10_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed);
-extern int polaris10_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed);
-extern int polaris10_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr);
-extern uint32_t tonga_get_xclk(struct pp_hwmgr *hwmgr);
-
-#endif
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
index 26f3e30d0fef..1126bd4f74dc 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
@@ -22,7 +22,6 @@
  */
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/fb.h>
 
 #include "ppatomctrl.h"
 #include "atombios.h"
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_pptable.h b/drivers/gpu/drm/amd/powerplay/hwmgr/pptable_v1_0.h
index f127198aafc4..1e870f58dd12 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_pptable.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pptable_v1_0.h
@@ -164,7 +164,7 @@ typedef struct _ATOM_Tonga_State {
 typedef struct _ATOM_Tonga_State_Array {
 	UCHAR ucRevId;
 	UCHAR ucNumEntries;		/* Number of entries. */
-	ATOM_Tonga_State states[1];	/* Dynamically allocate entries. */
+	ATOM_Tonga_State entries[1];	/* Dynamically allocate entries. */
 } ATOM_Tonga_State_Array;
 
 typedef struct _ATOM_Tonga_MCLK_Dependency_Record {
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
index cfb647f76cbe..7de701d8a450 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
@@ -22,15 +22,14 @@
  */
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/fb.h>
 
-#include "tonga_processpptables.h"
+#include "process_pptables_v1_0.h"
 #include "ppatomctrl.h"
 #include "atombios.h"
 #include "pp_debug.h"
 #include "hwmgr.h"
 #include "cgs_common.h"
-#include "tonga_pptable.h"
+#include "pptable_v1_0.h"
 
 /**
  * Private Function used during initialization.
@@ -154,12 +153,14 @@ const void *get_powerplay_table(struct pp_hwmgr *hwmgr)
 static int get_vddc_lookup_table(
 		struct pp_hwmgr	*hwmgr,
 		phm_ppt_v1_voltage_lookup_table	**lookup_table,
-		const ATOM_Tonga_Voltage_Lookup_Table	*vddc_lookup_pp_tables,
-		uint32_t	max_levels
+		const ATOM_Tonga_Voltage_Lookup_Table *vddc_lookup_pp_tables,
+		uint32_t max_levels
 		)
 {
 	uint32_t table_size, i;
 	phm_ppt_v1_voltage_lookup_table *table;
+	phm_ppt_v1_voltage_lookup_record *record;
+	ATOM_Tonga_Voltage_Lookup_Record *atom_record;
 
 	PP_ASSERT_WITH_CODE((0 != vddc_lookup_pp_tables->ucNumEntries),
 		"Invalid CAC Leakage PowerPlay Table!", return 1);
@@ -177,15 +178,17 @@ static int get_vddc_lookup_table(
 	table->count = vddc_lookup_pp_tables->ucNumEntries;
 
 	for (i = 0; i < vddc_lookup_pp_tables->ucNumEntries; i++) {
-		table->entries[i].us_calculated = 0;
-		table->entries[i].us_vdd =
-			vddc_lookup_pp_tables->entries[i].usVdd;
-		table->entries[i].us_cac_low =
-			vddc_lookup_pp_tables->entries[i].usCACLow;
-		table->entries[i].us_cac_mid =
-			vddc_lookup_pp_tables->entries[i].usCACMid;
-		table->entries[i].us_cac_high =
-			vddc_lookup_pp_tables->entries[i].usCACHigh;
+		record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					phm_ppt_v1_voltage_lookup_record,
+					entries, table, i);
+		atom_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					ATOM_Tonga_Voltage_Lookup_Record,
+					entries, vddc_lookup_pp_tables, i);
+		record->us_calculated = 0;
+		record->us_vdd = atom_record->usVdd;
+		record->us_cac_low = atom_record->usCACLow;
+		record->us_cac_mid = atom_record->usCACMid;
+		record->us_cac_high = atom_record->usCACHigh;
 	}
 
 	*lookup_table = table;
@@ -314,11 +317,12 @@ static int init_dpm_2_parameters(
 static int get_valid_clk(
 		struct pp_hwmgr *hwmgr,
 		struct phm_clock_array **clk_table,
-		const phm_ppt_v1_clock_voltage_dependency_table  * clk_volt_pp_table
+		phm_ppt_v1_clock_voltage_dependency_table const *clk_volt_pp_table
 		)
 {
 	uint32_t table_size, i;
 	struct phm_clock_array *table;
+	phm_ppt_v1_clock_voltage_dependency_record *dep_record;
 
 	PP_ASSERT_WITH_CODE((0 != clk_volt_pp_table->count),
 		"Invalid PowerPlay Table!", return -1);
@@ -335,9 +339,12 @@ static int get_valid_clk(
 
 	table->count = (uint32_t)clk_volt_pp_table->count;
 
-	for (i = 0; i < table->count; i++)
-		table->values[i] = (uint32_t)clk_volt_pp_table->entries[i].clk;
-
+	for (i = 0; i < table->count; i++) {
+		dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+				phm_ppt_v1_clock_voltage_dependency_record,
+				entries, clk_volt_pp_table, i);
+		table->values[i] = (uint32_t)dep_record->clk;
+	}
 	*clk_table = table;
 
 	return 0;
@@ -346,7 +353,7 @@ static int get_valid_clk(
 static int get_hard_limits(
 		struct pp_hwmgr *hwmgr,
 		struct phm_clock_and_voltage_limits *limits,
-		const ATOM_Tonga_Hard_Limit_Table * limitable
+		ATOM_Tonga_Hard_Limit_Table const *limitable
 		)
 {
 	PP_ASSERT_WITH_CODE((0 != limitable->ucNumEntries), "Invalid PowerPlay Table!", return -1);
@@ -364,11 +371,13 @@ static int get_hard_limits(
 static int get_mclk_voltage_dependency_table(
 		struct pp_hwmgr *hwmgr,
 		phm_ppt_v1_clock_voltage_dependency_table **pp_tonga_mclk_dep_table,
-		const ATOM_Tonga_MCLK_Dependency_Table * mclk_dep_table
+		ATOM_Tonga_MCLK_Dependency_Table const *mclk_dep_table
 		)
 {
 	uint32_t table_size, i;
 	phm_ppt_v1_clock_voltage_dependency_table *mclk_table;
+	phm_ppt_v1_clock_voltage_dependency_record *mclk_table_record;
+	ATOM_Tonga_MCLK_Dependency_Record *mclk_dep_record;
 
 	PP_ASSERT_WITH_CODE((0 != mclk_dep_table->ucNumEntries),
 		"Invalid PowerPlay Table!", return -1);
@@ -386,16 +395,17 @@ static int get_mclk_voltage_dependency_table(
 	mclk_table->count = (uint32_t)mclk_dep_table->ucNumEntries;
 
 	for (i = 0; i < mclk_dep_table->ucNumEntries; i++) {
-		mclk_table->entries[i].vddInd =
-			mclk_dep_table->entries[i].ucVddcInd;
-		mclk_table->entries[i].vdd_offset =
-			mclk_dep_table->entries[i].usVddgfxOffset;
-		mclk_table->entries[i].vddci =
-			mclk_dep_table->entries[i].usVddci;
-		mclk_table->entries[i].mvdd =
-			mclk_dep_table->entries[i].usMvdd;
-		mclk_table->entries[i].clk =
-			mclk_dep_table->entries[i].ulMclk;
+		mclk_table_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					phm_ppt_v1_clock_voltage_dependency_record,
+						entries, mclk_table, i);
+		mclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					ATOM_Tonga_MCLK_Dependency_Record,
+						entries, mclk_dep_table, i);
+		mclk_table_record->vddInd = mclk_dep_record->ucVddcInd;
+		mclk_table_record->vdd_offset = mclk_dep_record->usVddgfxOffset;
+		mclk_table_record->vddci = mclk_dep_record->usVddci;
+		mclk_table_record->mvdd = mclk_dep_record->usMvdd;
+		mclk_table_record->clk = mclk_dep_record->ulMclk;
 	}
 
 	*pp_tonga_mclk_dep_table = mclk_table;
@@ -406,15 +416,17 @@ static int get_mclk_voltage_dependency_table(
 static int get_sclk_voltage_dependency_table(
 		struct pp_hwmgr *hwmgr,
 		phm_ppt_v1_clock_voltage_dependency_table **pp_tonga_sclk_dep_table,
-		const PPTable_Generic_SubTable_Header *sclk_dep_table
+		PPTable_Generic_SubTable_Header const  *sclk_dep_table
 		)
 {
 	uint32_t table_size, i;
 	phm_ppt_v1_clock_voltage_dependency_table *sclk_table;
+	phm_ppt_v1_clock_voltage_dependency_record *sclk_table_record;
 
 	if (sclk_dep_table->ucRevId < 1) {
 		const ATOM_Tonga_SCLK_Dependency_Table *tonga_table =
 			    (ATOM_Tonga_SCLK_Dependency_Table *)sclk_dep_table;
+		ATOM_Tonga_SCLK_Dependency_Record *sclk_dep_record;
 
 		PP_ASSERT_WITH_CODE((0 != tonga_table->ucNumEntries),
 			"Invalid PowerPlay Table!", return -1);
@@ -432,20 +444,23 @@ static int get_sclk_voltage_dependency_table(
 		sclk_table->count = (uint32_t)tonga_table->ucNumEntries;
 
 		for (i = 0; i < tonga_table->ucNumEntries; i++) {
-			sclk_table->entries[i].vddInd =
-				tonga_table->entries[i].ucVddInd;
-			sclk_table->entries[i].vdd_offset =
-				tonga_table->entries[i].usVddcOffset;
-			sclk_table->entries[i].clk =
-				tonga_table->entries[i].ulSclk;
-			sclk_table->entries[i].cks_enable =
-				(((tonga_table->entries[i].ucCKSVOffsetandDisable & 0x80) >> 7) == 0) ? 1 : 0;
-			sclk_table->entries[i].cks_voffset =
-				(tonga_table->entries[i].ucCKSVOffsetandDisable & 0x7F);
+			sclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						ATOM_Tonga_SCLK_Dependency_Record,
+						entries, tonga_table, i);
+			sclk_table_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						phm_ppt_v1_clock_voltage_dependency_record,
+						entries, sclk_table, i);
+			sclk_table_record->vddInd = sclk_dep_record->ucVddInd;
+			sclk_table_record->vdd_offset = sclk_dep_record->usVddcOffset;
+			sclk_table_record->clk = sclk_dep_record->ulSclk;
+			sclk_table_record->cks_enable =
+				(((sclk_dep_record->ucCKSVOffsetandDisable & 0x80) >> 7) == 0) ? 1 : 0;
+			sclk_table_record->cks_voffset = (sclk_dep_record->ucCKSVOffsetandDisable & 0x7F);
 		}
 	} else {
 		const ATOM_Polaris_SCLK_Dependency_Table *polaris_table =
 			    (ATOM_Polaris_SCLK_Dependency_Table *)sclk_dep_table;
+		ATOM_Polaris_SCLK_Dependency_Record *sclk_dep_record;
 
 		PP_ASSERT_WITH_CODE((0 != polaris_table->ucNumEntries),
 			"Invalid PowerPlay Table!", return -1);
@@ -463,17 +478,19 @@ static int get_sclk_voltage_dependency_table(
 		sclk_table->count = (uint32_t)polaris_table->ucNumEntries;
 
 		for (i = 0; i < polaris_table->ucNumEntries; i++) {
-			sclk_table->entries[i].vddInd =
-				polaris_table->entries[i].ucVddInd;
-			sclk_table->entries[i].vdd_offset =
-				polaris_table->entries[i].usVddcOffset;
-			sclk_table->entries[i].clk =
-				polaris_table->entries[i].ulSclk;
-			sclk_table->entries[i].cks_enable =
-				(((polaris_table->entries[i].ucCKSVOffsetandDisable & 0x80) >> 7) == 0) ? 1 : 0;
-			sclk_table->entries[i].cks_voffset =
-				(polaris_table->entries[i].ucCKSVOffsetandDisable & 0x7F);
-			sclk_table->entries[i].sclk_offset = polaris_table->entries[i].ulSclkOffset;
+			sclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						ATOM_Polaris_SCLK_Dependency_Record,
+						entries, polaris_table, i);
+			sclk_table_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						phm_ppt_v1_clock_voltage_dependency_record,
+						entries, sclk_table, i);
+			sclk_table_record->vddInd = sclk_dep_record->ucVddInd;
+			sclk_table_record->vdd_offset = sclk_dep_record->usVddcOffset;
+			sclk_table_record->clk = sclk_dep_record->ulSclk;
+			sclk_table_record->cks_enable =
+				(((sclk_dep_record->ucCKSVOffsetandDisable & 0x80) >> 7) == 0) ? 1 : 0;
+			sclk_table_record->cks_voffset = (sclk_dep_record->ucCKSVOffsetandDisable & 0x7F);
+			sclk_table_record->sclk_offset = sclk_dep_record->ulSclkOffset;
 		}
 	}
 	*pp_tonga_sclk_dep_table = sclk_table;
@@ -484,16 +501,19 @@ static int get_sclk_voltage_dependency_table(
 static int get_pcie_table(
 		struct pp_hwmgr *hwmgr,
 		phm_ppt_v1_pcie_table **pp_tonga_pcie_table,
-		const PPTable_Generic_SubTable_Header * pTable
+		PPTable_Generic_SubTable_Header const *ptable
 		)
 {
 	uint32_t table_size, i, pcie_count;
 	phm_ppt_v1_pcie_table *pcie_table;
 	struct phm_ppt_v1_information *pp_table_information =
 		(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_pcie_record *pcie_record;
+
+	if (ptable->ucRevId < 1) {
+		const ATOM_Tonga_PCIE_Table *atom_pcie_table = (ATOM_Tonga_PCIE_Table *)ptable;
+		ATOM_Tonga_PCIE_Record *atom_pcie_record;
 
-	if (pTable->ucRevId < 1) {
-		const ATOM_Tonga_PCIE_Table *atom_pcie_table = (ATOM_Tonga_PCIE_Table *)pTable;
 		PP_ASSERT_WITH_CODE((atom_pcie_table->ucNumEntries != 0),
 			"Invalid PowerPlay Table!", return -1);
 
@@ -519,18 +539,23 @@ static int get_pcie_table(
 			Disregarding the excess entries... \n");
 
 		pcie_table->count = pcie_count;
-
 		for (i = 0; i < pcie_count; i++) {
-			pcie_table->entries[i].gen_speed =
-				atom_pcie_table->entries[i].ucPCIEGenSpeed;
-			pcie_table->entries[i].lane_width =
-				atom_pcie_table->entries[i].usPCIELaneWidth;
+			pcie_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						phm_ppt_v1_pcie_record,
+						entries, pcie_table, i);
+			atom_pcie_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						ATOM_Tonga_PCIE_Record,
+						entries, atom_pcie_table, i);
+			pcie_record->gen_speed = atom_pcie_record->ucPCIEGenSpeed;
+			pcie_record->lane_width = atom_pcie_record->usPCIELaneWidth;
 		}
 
 		*pp_tonga_pcie_table = pcie_table;
 	} else {
 		/* Polaris10/Polaris11 and newer. */
-		const ATOM_Polaris10_PCIE_Table *atom_pcie_table = (ATOM_Polaris10_PCIE_Table *)pTable;
+		const ATOM_Polaris10_PCIE_Table *atom_pcie_table = (ATOM_Polaris10_PCIE_Table *)ptable;
+		ATOM_Polaris10_PCIE_Record *atom_pcie_record;
+
 		PP_ASSERT_WITH_CODE((atom_pcie_table->ucNumEntries != 0),
 			"Invalid PowerPlay Table!", return -1);
 
@@ -558,12 +583,15 @@ static int get_pcie_table(
 		pcie_table->count = pcie_count;
 
 		for (i = 0; i < pcie_count; i++) {
-			pcie_table->entries[i].gen_speed =
-				atom_pcie_table->entries[i].ucPCIEGenSpeed;
-			pcie_table->entries[i].lane_width =
-				atom_pcie_table->entries[i].usPCIELaneWidth;
-			pcie_table->entries[i].pcie_sclk =
-				atom_pcie_table->entries[i].ulPCIE_Sclk;
+			pcie_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						phm_ppt_v1_pcie_record,
+						entries, pcie_table, i);
+			atom_pcie_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						ATOM_Polaris10_PCIE_Record,
+						entries, atom_pcie_table, i);
+			pcie_record->gen_speed = atom_pcie_record->ucPCIEGenSpeed;
+			pcie_record->lane_width = atom_pcie_record->usPCIELaneWidth;
+			pcie_record->pcie_sclk = atom_pcie_record->ulPCIE_Sclk;
 		}
 
 		*pp_tonga_pcie_table = pcie_table;
@@ -685,6 +713,7 @@ static int get_mm_clock_voltage_table(
 	uint32_t table_size, i;
 	const ATOM_Tonga_MM_Dependency_Record *mm_dependency_record;
 	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table;
+	phm_ppt_v1_mm_clock_voltage_dependency_record *mm_table_record;
 
 	PP_ASSERT_WITH_CODE((0 != mm_dependency_table->ucNumEntries),
 		"Invalid PowerPlay Table!", return -1);
@@ -701,14 +730,19 @@ static int get_mm_clock_voltage_table(
 	mm_table->count = mm_dependency_table->ucNumEntries;
 
 	for (i = 0; i < mm_dependency_table->ucNumEntries; i++) {
-		mm_dependency_record = &mm_dependency_table->entries[i];
-		mm_table->entries[i].vddcInd = mm_dependency_record->ucVddcInd;
-		mm_table->entries[i].vddgfx_offset = mm_dependency_record->usVddgfxOffset;
-		mm_table->entries[i].aclk = mm_dependency_record->ulAClk;
-		mm_table->entries[i].samclock = mm_dependency_record->ulSAMUClk;
-		mm_table->entries[i].eclk = mm_dependency_record->ulEClk;
-		mm_table->entries[i].vclk = mm_dependency_record->ulVClk;
-		mm_table->entries[i].dclk = mm_dependency_record->ulDClk;
+		mm_dependency_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						ATOM_Tonga_MM_Dependency_Record,
+						entries, mm_dependency_table, i);
+		mm_table_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					phm_ppt_v1_mm_clock_voltage_dependency_record,
+					entries, mm_table, i);
+		mm_table_record->vddcInd = mm_dependency_record->ucVddcInd;
+		mm_table_record->vddgfx_offset = mm_dependency_record->usVddgfxOffset;
+		mm_table_record->aclk = mm_dependency_record->ulAClk;
+		mm_table_record->samclock = mm_dependency_record->ulSAMUClk;
+		mm_table_record->eclk = mm_dependency_record->ulEClk;
+		mm_table_record->vclk = mm_dependency_record->ulVClk;
+		mm_table_record->dclk = mm_dependency_record->ulDClk;
 	}
 
 	*tonga_mm_table = mm_table;
@@ -1015,7 +1049,7 @@ static int check_powerplay_tables(
 	return 0;
 }
 
-int tonga_pp_tables_initialize(struct pp_hwmgr *hwmgr)
+int pp_tables_v1_0_initialize(struct pp_hwmgr *hwmgr)
 {
 	int result = 0;
 	const ATOM_Tonga_POWERPLAYTABLE *powerplay_table;
@@ -1066,7 +1100,7 @@ int tonga_pp_tables_initialize(struct pp_hwmgr *hwmgr)
 	return result;
 }
 
-int tonga_pp_tables_uninitialize(struct pp_hwmgr *hwmgr)
+int pp_tables_v1_0_uninitialize(struct pp_hwmgr *hwmgr)
 {
 	struct phm_ppt_v1_information *pp_table_information =
 		(struct phm_ppt_v1_information *)(hwmgr->pptable);
@@ -1110,14 +1144,14 @@ int tonga_pp_tables_uninitialize(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-const struct pp_table_func tonga_pptable_funcs = {
-	.pptable_init = tonga_pp_tables_initialize,
-	.pptable_fini = tonga_pp_tables_uninitialize,
+const struct pp_table_func pptable_v1_0_funcs = {
+	.pptable_init = pp_tables_v1_0_initialize,
+	.pptable_fini = pp_tables_v1_0_uninitialize,
 };
 
-int tonga_get_number_of_powerplay_table_entries(struct pp_hwmgr *hwmgr)
+int get_number_of_powerplay_table_entries_v1_0(struct pp_hwmgr *hwmgr)
 {
-	const ATOM_Tonga_State_Array * state_arrays;
+	ATOM_Tonga_State_Array const *state_arrays;
 	const ATOM_Tonga_POWERPLAYTABLE *pp_table = get_powerplay_table(hwmgr);
 
 	PP_ASSERT_WITH_CODE((NULL != pp_table),
@@ -1164,6 +1198,71 @@ static uint32_t make_classification_flags(struct pp_hwmgr *hwmgr,
 	return result;
 }
 
+static int ppt_get_num_of_vce_state_table_entries_v1_0(struct pp_hwmgr *hwmgr)
+{
+	const ATOM_Tonga_POWERPLAYTABLE *pp_table = get_powerplay_table(hwmgr);
+	const ATOM_Tonga_VCE_State_Table *vce_state_table =
+				(ATOM_Tonga_VCE_State_Table *)(((unsigned long)pp_table) + le16_to_cpu(pp_table->usVCEStateTableOffset));
+
+	if (vce_state_table == NULL)
+		return 0;
+
+	return vce_state_table->ucNumEntries;
+}
+
+static int ppt_get_vce_state_table_entry_v1_0(struct pp_hwmgr *hwmgr, uint32_t i,
+		struct pp_vce_state *vce_state, void **clock_info, uint32_t *flag)
+{
+	const ATOM_Tonga_VCE_State_Record *vce_state_record;
+	ATOM_Tonga_SCLK_Dependency_Record *sclk_dep_record;
+	ATOM_Tonga_MCLK_Dependency_Record *mclk_dep_record;
+	ATOM_Tonga_MM_Dependency_Record *mm_dep_record;
+	const ATOM_Tonga_POWERPLAYTABLE *pptable = get_powerplay_table(hwmgr);
+	const ATOM_Tonga_VCE_State_Table *vce_state_table = (ATOM_Tonga_VCE_State_Table *)(((unsigned long)pptable)
+							  + le16_to_cpu(pptable->usVCEStateTableOffset));
+	const ATOM_Tonga_SCLK_Dependency_Table *sclk_dep_table = (ATOM_Tonga_SCLK_Dependency_Table *)(((unsigned long)pptable)
+							  + le16_to_cpu(pptable->usSclkDependencyTableOffset));
+	const ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table = (ATOM_Tonga_MCLK_Dependency_Table *)(((unsigned long)pptable)
+							  + le16_to_cpu(pptable->usMclkDependencyTableOffset));
+	const ATOM_Tonga_MM_Dependency_Table *mm_dep_table = (ATOM_Tonga_MM_Dependency_Table *)(((unsigned long)pptable)
+							  + le16_to_cpu(pptable->usMMDependencyTableOffset));
+
+	PP_ASSERT_WITH_CODE((i < vce_state_table->ucNumEntries),
+			 "Requested state entry ID is out of range!",
+			 return -EINVAL);
+
+	vce_state_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					ATOM_Tonga_VCE_State_Record,
+					entries, vce_state_table, i);
+	sclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					ATOM_Tonga_SCLK_Dependency_Record,
+					entries, sclk_dep_table,
+					vce_state_record->ucSCLKIndex);
+	mm_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					ATOM_Tonga_MM_Dependency_Record,
+					entries, mm_dep_table,
+					vce_state_record->ucVCEClockIndex);
+	*flag = vce_state_record->ucFlag;
+
+	vce_state->evclk = mm_dep_record->ulEClk;
+	vce_state->ecclk = mm_dep_record->ulEClk;
+	vce_state->sclk = sclk_dep_record->ulSclk;
+
+	if (vce_state_record->ucMCLKIndex >= mclk_dep_table->ucNumEntries)
+		mclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					ATOM_Tonga_MCLK_Dependency_Record,
+					entries, mclk_dep_table,
+					mclk_dep_table->ucNumEntries - 1);
+	else
+		mclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+					ATOM_Tonga_MCLK_Dependency_Record,
+					entries, mclk_dep_table,
+					vce_state_record->ucMCLKIndex);
+
+	vce_state->mclk = mclk_dep_record->ulMclk;
+	return 0;
+}
+
 /**
 * Create a Power State out of an entry in the PowerPlay table.
 * This function is called by the hardware back-end.
@@ -1172,15 +1271,17 @@ static uint32_t make_classification_flags(struct pp_hwmgr *hwmgr,
 * @param power_state The address of the PowerState instance being created.
 * @return -1 if the entry cannot be retrieved.
 */
-int tonga_get_powerplay_table_entry(struct pp_hwmgr *hwmgr,
+int get_powerplay_table_entry_v1_0(struct pp_hwmgr *hwmgr,
 		uint32_t entry_index, struct pp_power_state *power_state,
 		int (*call_back_func)(struct pp_hwmgr *, void *,
 				struct pp_power_state *, void *, uint32_t))
 {
 	int result = 0;
-	const ATOM_Tonga_State_Array * state_arrays;
+	const ATOM_Tonga_State_Array *state_arrays;
 	const ATOM_Tonga_State *state_entry;
 	const ATOM_Tonga_POWERPLAYTABLE *pp_table = get_powerplay_table(hwmgr);
+	int i, j;
+	uint32_t flags = 0;
 
 	PP_ASSERT_WITH_CODE((NULL != pp_table), "Missing PowerPlay Table!", return -1;);
 	power_state->classification.bios_index = entry_index;
@@ -1197,7 +1298,9 @@ int tonga_get_powerplay_table_entry(struct pp_hwmgr *hwmgr,
 		PP_ASSERT_WITH_CODE((entry_index <= state_arrays->ucNumEntries),
 				"Invalid PowerPlay Table State Array Entry.", return -1);
 
-		state_entry = &(state_arrays->states[entry_index]);
+		state_entry = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						ATOM_Tonga_State, entries,
+						state_arrays, entry_index);
 
 		result = call_back_func(hwmgr, (void *)state_entry, power_state,
 				(void *)pp_table,
@@ -1210,5 +1313,13 @@ int tonga_get_powerplay_table_entry(struct pp_hwmgr *hwmgr,
 			PP_StateClassificationFlag_Boot))
 		result = hwmgr->hwmgr_func->patch_boot_state(hwmgr, &(power_state->hardware));
 
+	hwmgr->num_vce_state_tables = i = ppt_get_num_of_vce_state_table_entries_v1_0(hwmgr);
+
+	if ((i != 0) && (i <= PP_MAX_VCE_LEVELS)) {
+		for (j = 0; j < i; j++)
+			ppt_get_vce_state_table_entry_v1_0(hwmgr, j, &(hwmgr->vce_states[j]), NULL, &flags);
+	}
+
 	return result;
 }
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.h b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.h
index d24b8887f466..b9710abdff01 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.h
@@ -20,14 +20,14 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
-#ifndef TONGA_PROCESSPPTABLES_H
-#define TONGA_PROCESSPPTABLES_H
+#ifndef _PROCESSPPTABLES_V1_0_H
+#define _PROCESSPPTABLES_V1_0_H
 
 #include "hwmgr.h"
 
-extern const struct pp_table_func tonga_pptable_funcs;
-extern int tonga_get_number_of_powerplay_table_entries(struct pp_hwmgr *hwmgr);
-extern int tonga_get_powerplay_table_entry(struct pp_hwmgr *hwmgr, uint32_t entry_index,
+extern const struct pp_table_func pptable_v1_0_funcs;
+extern int get_number_of_powerplay_table_entries_v1_0(struct pp_hwmgr *hwmgr);
+extern int get_powerplay_table_entry_v1_0(struct pp_hwmgr *hwmgr, uint32_t entry_index,
 		struct pp_power_state *power_state, int (*call_back_func)(struct pp_hwmgr *, void *,
 				struct pp_power_state *, void *, uint32_t));
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
index 6c321b0d8a1e..ccf7ebeaf892 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
@@ -1523,7 +1523,7 @@ int get_number_of_vce_state_table_entries(
 
 int get_vce_state_table_entry(struct pp_hwmgr *hwmgr,
 							unsigned long i,
-							struct PP_VCEState *vce_state,
+							struct pp_vce_state *vce_state,
 							void **clock_info,
 							unsigned long *flag)
 {
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
index b5edb5105986..6eb6db199250 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
@@ -21,9 +21,53 @@
  *
  */
 
-#include "polaris10_clockpowergating.h"
+#include "smu7_hwmgr.h"
+#include "smu7_clockpowergating.h"
+#include "smu7_common.h"
 
-int polaris10_phm_powerdown_uvd(struct pp_hwmgr *hwmgr)
+static int smu7_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
+			PPSMC_MSG_UVDDPM_Enable :
+			PPSMC_MSG_UVDDPM_Disable);
+}
+
+static int smu7_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
+			PPSMC_MSG_VCEDPM_Enable :
+			PPSMC_MSG_VCEDPM_Disable);
+}
+
+static int smu7_enable_disable_samu_dpm(struct pp_hwmgr *hwmgr, bool enable)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
+			PPSMC_MSG_SAMUDPM_Enable :
+			PPSMC_MSG_SAMUDPM_Disable);
+}
+
+static int smu7_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	if (!bgate)
+		smum_update_smc_table(hwmgr, SMU_UVD_TABLE);
+	return smu7_enable_disable_uvd_dpm(hwmgr, !bgate);
+}
+
+static int smu7_update_vce_dpm(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	if (!bgate)
+		smum_update_smc_table(hwmgr, SMU_VCE_TABLE);
+	return smu7_enable_disable_vce_dpm(hwmgr, !bgate);
+}
+
+static int smu7_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate)
+{
+	if (!bgate)
+		smum_update_smc_table(hwmgr, SMU_SAMU_TABLE);
+	return smu7_enable_disable_samu_dpm(hwmgr, !bgate);
+}
+
+int smu7_powerdown_uvd(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cf_want_uvd_power_gating(hwmgr))
 		return smum_send_msg_to_smc(hwmgr->smumgr,
@@ -31,7 +75,7 @@ int polaris10_phm_powerdown_uvd(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int polaris10_phm_powerup_uvd(struct pp_hwmgr *hwmgr)
+int smu7_powerup_uvd(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cf_want_uvd_power_gating(hwmgr)) {
 		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
@@ -47,7 +91,7 @@ int polaris10_phm_powerup_uvd(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int polaris10_phm_powerdown_vce(struct pp_hwmgr *hwmgr)
+int smu7_powerdown_vce(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cf_want_vce_power_gating(hwmgr))
 		return smum_send_msg_to_smc(hwmgr->smumgr,
@@ -55,7 +99,7 @@ int polaris10_phm_powerdown_vce(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int polaris10_phm_powerup_vce(struct pp_hwmgr *hwmgr)
+int smu7_powerup_vce(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cf_want_vce_power_gating(hwmgr))
 		return smum_send_msg_to_smc(hwmgr->smumgr,
@@ -63,7 +107,7 @@ int polaris10_phm_powerup_vce(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int polaris10_phm_powerdown_samu(struct pp_hwmgr *hwmgr)
+int smu7_powerdown_samu(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SamuPowerGating))
@@ -72,7 +116,7 @@ int polaris10_phm_powerdown_samu(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int polaris10_phm_powerup_samu(struct pp_hwmgr *hwmgr)
+int smu7_powerup_samu(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SamuPowerGating))
@@ -81,27 +125,24 @@ int polaris10_phm_powerup_samu(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int polaris10_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr)
+int smu7_disable_clock_power_gating(struct pp_hwmgr *hwmgr)
 {
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
 	data->uvd_power_gated = false;
 	data->vce_power_gated = false;
 	data->samu_power_gated = false;
 
-	polaris10_phm_powerup_uvd(hwmgr);
-	polaris10_phm_powerup_vce(hwmgr);
-	polaris10_phm_powerup_samu(hwmgr);
+	smu7_powerup_uvd(hwmgr);
+	smu7_powerup_vce(hwmgr);
+	smu7_powerup_samu(hwmgr);
 
 	return 0;
 }
 
-int polaris10_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
+int smu7_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
 {
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if (data->uvd_power_gated == bgate)
-		return 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
 	data->uvd_power_gated = bgate;
 
@@ -109,11 +150,11 @@ int polaris10_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
 		cgs_set_clockgating_state(hwmgr->device,
 				AMD_IP_BLOCK_TYPE_UVD,
 				AMD_CG_STATE_GATE);
-		polaris10_update_uvd_dpm(hwmgr, true);
-		polaris10_phm_powerdown_uvd(hwmgr);
+		smu7_update_uvd_dpm(hwmgr, true);
+		smu7_powerdown_uvd(hwmgr);
 	} else {
-		polaris10_phm_powerup_uvd(hwmgr);
-		polaris10_update_uvd_dpm(hwmgr, false);
+		smu7_powerup_uvd(hwmgr);
+		smu7_update_uvd_dpm(hwmgr, false);
 		cgs_set_clockgating_state(hwmgr->device,
 				AMD_IP_BLOCK_TYPE_UVD,
 				AMD_CG_STATE_UNGATE);
@@ -122,9 +163,9 @@ int polaris10_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
 	return 0;
 }
 
-int polaris10_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
+int smu7_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 {
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
 	if (data->vce_power_gated == bgate)
 		return 0;
@@ -135,11 +176,11 @@ int polaris10_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 		cgs_set_clockgating_state(hwmgr->device,
 				AMD_IP_BLOCK_TYPE_VCE,
 				AMD_CG_STATE_GATE);
-		polaris10_update_vce_dpm(hwmgr, true);
-		polaris10_phm_powerdown_vce(hwmgr);
+		smu7_update_vce_dpm(hwmgr, true);
+		smu7_powerdown_vce(hwmgr);
 	} else {
-		polaris10_phm_powerup_vce(hwmgr);
-		polaris10_update_vce_dpm(hwmgr, false);
+		smu7_powerup_vce(hwmgr);
+		smu7_update_vce_dpm(hwmgr, false);
 		cgs_set_clockgating_state(hwmgr->device,
 				AMD_IP_BLOCK_TYPE_VCE,
 				AMD_CG_STATE_UNGATE);
@@ -147,9 +188,9 @@ int polaris10_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 	return 0;
 }
 
-int polaris10_phm_powergate_samu(struct pp_hwmgr *hwmgr, bool bgate)
+int smu7_powergate_samu(struct pp_hwmgr *hwmgr, bool bgate)
 {
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
 	if (data->samu_power_gated == bgate)
 		return 0;
@@ -157,22 +198,25 @@ int polaris10_phm_powergate_samu(struct pp_hwmgr *hwmgr, bool bgate)
 	data->samu_power_gated = bgate;
 
 	if (bgate) {
-		polaris10_update_samu_dpm(hwmgr, true);
-		polaris10_phm_powerdown_samu(hwmgr);
+		smu7_update_samu_dpm(hwmgr, true);
+		smu7_powerdown_samu(hwmgr);
 	} else {
-		polaris10_phm_powerup_samu(hwmgr);
-		polaris10_update_samu_dpm(hwmgr, false);
+		smu7_powerup_samu(hwmgr);
+		smu7_update_samu_dpm(hwmgr, false);
 	}
 
 	return 0;
 }
 
-int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
+int smu7_update_clock_gatings(struct pp_hwmgr *hwmgr,
 					const uint32_t *msg_id)
 {
 	PPSMC_Msg msg;
 	uint32_t value;
 
+	if (!(hwmgr->feature_mask & PP_ENABLE_GFX_CG_THRU_SMU))
+		return 0;
+
 	switch ((*msg_id & PP_GROUP_MASK) >> PP_GROUP_SHIFT) {
 	case PP_GROUP_GFX:
 		switch ((*msg_id & PP_BLOCK_MASK) >> PP_BLOCK_SHIFT) {
@@ -185,7 +229,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			if (PP_STATE_SUPPORT_LS & *msg_id) {
 				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
@@ -195,7 +239,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
@@ -208,7 +252,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 
 			if  (PP_STATE_SUPPORT_LS & *msg_id) {
@@ -219,7 +263,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
@@ -232,7 +276,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
@@ -245,7 +289,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
@@ -259,12 +303,12 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
 		default:
-			return -1;
+			return -EINVAL;
 		}
 		break;
 
@@ -279,7 +323,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			if  (PP_STATE_SUPPORT_LS & *msg_id) {
 				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS ?
@@ -289,7 +333,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
@@ -302,7 +346,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 
 			if (PP_STATE_SUPPORT_LS & *msg_id) {
@@ -313,7 +357,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
@@ -326,7 +370,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			if (PP_STATE_SUPPORT_LS & *msg_id) {
 				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS ?
@@ -336,7 +380,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
@@ -349,7 +393,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 
 			if (PP_STATE_SUPPORT_LS & *msg_id) {
@@ -360,7 +404,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
@@ -373,7 +417,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 
 			if (PP_STATE_SUPPORT_LS & *msg_id) {
@@ -384,7 +428,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
@@ -397,18 +441,18 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
 
 				if (smum_send_msg_to_smc_with_parameter(
 						hwmgr->smumgr, msg, value))
-					return -1;
+					return -EINVAL;
 			}
 			break;
 
 		default:
-			return -1;
+			return -EINVAL;
 
 		}
 		break;
 
 	default:
-		return -1;
+		return -EINVAL;
 
 	}
 
@@ -419,7 +463,7 @@ int polaris10_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
  * Powerplay will only control the static per CU Power Gating.
  * Dynamic per CU Power Gating will be done in gfx.
  */
-int polaris10_phm_enable_per_cu_power_gating(struct pp_hwmgr *hwmgr, bool enable)
+int smu7_enable_per_cu_power_gating(struct pp_hwmgr *hwmgr, bool enable)
 {
 	struct cgs_system_info sys_info = {0};
 	uint32_t active_cus;
@@ -432,8 +476,8 @@ int polaris10_phm_enable_per_cu_power_gating(struct pp_hwmgr *hwmgr, bool enable
 
 	if (result)
 		return -EINVAL;
-	else
-		active_cus = sys_info.value;
+
+	active_cus = sys_info.value;
 
 	if (enable)
 		return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h
index 33af5f511ab8..d52a28c343e3 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015 Advanced Micro Devices, Inc.
+ * Copyright 2016 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -21,15 +21,20 @@
  *
  */
 
-#ifndef _FIJI_CLOCK_POWER_GATING_H_
-#define _FIJI_CLOCK_POWER_GATING_H_
+#ifndef _SMU7_CLOCK_POWER_GATING_H_
+#define _SMU7_CLOCK__POWER_GATING_H_
 
-#include "fiji_hwmgr.h"
+#include "smu7_hwmgr.h"
 #include "pp_asicblocks.h"
 
-extern int fiji_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate);
-extern int fiji_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate);
-extern int fiji_phm_powergate_samu(struct pp_hwmgr *hwmgr, bool bgate);
-extern int fiji_phm_powergate_acp(struct pp_hwmgr *hwmgr, bool bgate);
-extern int fiji_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr);
-#endif /* _TONGA_CLOCK_POWER_GATING_H_ */
+int smu7_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate);
+int smu7_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate);
+int smu7_powerdown_uvd(struct pp_hwmgr *hwmgr);
+int smu7_powergate_samu(struct pp_hwmgr *hwmgr, bool bgate);
+int smu7_powergate_acp(struct pp_hwmgr *hwmgr, bool bgate);
+int smu7_disable_clock_power_gating(struct pp_hwmgr *hwmgr);
+int smu7_update_clock_gatings(struct pp_hwmgr *hwmgr,
+					const uint32_t *msg_id);
+int smu7_enable_per_cu_power_gating(struct pp_hwmgr *hwmgr, bool enable);
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_dyn_defaults.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_dyn_defaults.h
new file mode 100644
index 000000000000..f967613191cf
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_dyn_defaults.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _SMU7_DYN_DEFAULTS_H
+#define _SMU7_DYN_DEFAULTS_H
+
+
+/*  We need to fill in the default values */
+
+
+#define SMU7_VOTINGRIGHTSCLIENTS_DFLT0              0x3FFFC102
+#define SMU7_VOTINGRIGHTSCLIENTS_DFLT1              0x000400
+#define SMU7_VOTINGRIGHTSCLIENTS_DFLT2              0xC00080
+#define SMU7_VOTINGRIGHTSCLIENTS_DFLT3              0xC00200
+#define SMU7_VOTINGRIGHTSCLIENTS_DFLT4              0xC01680
+#define SMU7_VOTINGRIGHTSCLIENTS_DFLT5              0xC00033
+#define SMU7_VOTINGRIGHTSCLIENTS_DFLT6              0xC00033
+#define SMU7_VOTINGRIGHTSCLIENTS_DFLT7              0x3FFFC000
+
+
+#define SMU7_THERMALPROTECTCOUNTER_DFLT            0x200
+#define SMU7_STATICSCREENTHRESHOLDUNIT_DFLT        0
+#define SMU7_STATICSCREENTHRESHOLD_DFLT            0x00C8
+#define SMU7_GFXIDLECLOCKSTOPTHRESHOLD_DFLT        0x200
+#define SMU7_REFERENCEDIVIDER_DFLT                  4
+
+#define SMU7_ULVVOLTAGECHANGEDELAY_DFLT             1687
+
+#define SMU7_CGULVPARAMETER_DFLT                    0x00040035
+#define SMU7_CGULVCONTROL_DFLT                      0x00007450
+#define SMU7_TARGETACTIVITY_DFLT                     50
+#define SMU7_MCLK_TARGETACTIVITY_DFLT                10
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
new file mode 100644
index 000000000000..508245d49d33
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
@@ -0,0 +1,4359 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/fb.h>
+#include <asm/div64.h>
+#include "linux/delay.h"
+#include "pp_acpi.h"
+#include "pp_debug.h"
+#include "ppatomctrl.h"
+#include "atombios.h"
+#include "pptable_v1_0.h"
+#include "pppcielanes.h"
+#include "amd_pcie_helpers.h"
+#include "hardwaremanager.h"
+#include "process_pptables_v1_0.h"
+#include "cgs_common.h"
+
+#include "smu7_common.h"
+
+#include "hwmgr.h"
+#include "smu7_hwmgr.h"
+#include "smu7_powertune.h"
+#include "smu7_dyn_defaults.h"
+#include "smu7_thermal.h"
+#include "smu7_clockpowergating.h"
+#include "processpptables.h"
+
+#define MC_CG_ARB_FREQ_F0           0x0a
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define MC_CG_ARB_FREQ_F2           0x0c
+#define MC_CG_ARB_FREQ_F3           0x0d
+
+#define MC_CG_SEQ_DRAMCONF_S0       0x05
+#define MC_CG_SEQ_DRAMCONF_S1       0x06
+#define MC_CG_SEQ_YCLK_SUSPEND      0x04
+#define MC_CG_SEQ_YCLK_RESUME       0x0a
+
+#define SMC_CG_IND_START            0xc0030000
+#define SMC_CG_IND_END              0xc0040000
+
+#define VOLTAGE_SCALE               4
+#define VOLTAGE_VID_OFFSET_SCALE1   625
+#define VOLTAGE_VID_OFFSET_SCALE2   100
+
+#define MEM_FREQ_LOW_LATENCY        25000
+#define MEM_FREQ_HIGH_LATENCY       80000
+
+#define MEM_LATENCY_HIGH            45
+#define MEM_LATENCY_LOW             35
+#define MEM_LATENCY_ERR             0xFFFF
+
+#define MC_SEQ_MISC0_GDDR5_SHIFT 28
+#define MC_SEQ_MISC0_GDDR5_MASK  0xf0000000
+#define MC_SEQ_MISC0_GDDR5_VALUE 5
+
+#define PCIE_BUS_CLK                10000
+#define TCLK                        (PCIE_BUS_CLK / 10)
+
+
+/** Values for the CG_THERMAL_CTRL::DPM_EVENT_SRC field. */
+enum DPM_EVENT_SRC {
+	DPM_EVENT_SRC_ANALOG = 0,
+	DPM_EVENT_SRC_EXTERNAL = 1,
+	DPM_EVENT_SRC_DIGITAL = 2,
+	DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,
+	DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL = 4
+};
+
+static const unsigned long PhwVIslands_Magic = (unsigned long)(PHM_VIslands_Magic);
+
+struct smu7_power_state *cast_phw_smu7_power_state(
+				  struct pp_hw_power_state *hw_ps)
+{
+	PP_ASSERT_WITH_CODE((PhwVIslands_Magic == hw_ps->magic),
+				"Invalid Powerstate Type!",
+				 return NULL);
+
+	return (struct smu7_power_state *)hw_ps;
+}
+
+const struct smu7_power_state *cast_const_phw_smu7_power_state(
+				 const struct pp_hw_power_state *hw_ps)
+{
+	PP_ASSERT_WITH_CODE((PhwVIslands_Magic == hw_ps->magic),
+				"Invalid Powerstate Type!",
+				 return NULL);
+
+	return (const struct smu7_power_state *)hw_ps;
+}
+
+/**
+ * Find the MC microcode version and store it in the HwMgr struct
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int smu7_get_mc_microcode_version (struct pp_hwmgr *hwmgr)
+{
+	cgs_write_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_INDEX, 0x9F);
+
+	hwmgr->microcode_version_info.MC = cgs_read_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_DATA);
+
+	return 0;
+}
+
+uint16_t smu7_get_current_pcie_speed(struct pp_hwmgr *hwmgr)
+{
+	uint32_t speedCntl = 0;
+
+	/* mmPCIE_PORT_INDEX rename as mmPCIE_INDEX */
+	speedCntl = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__PCIE,
+			ixPCIE_LC_SPEED_CNTL);
+	return((uint16_t)PHM_GET_FIELD(speedCntl,
+			PCIE_LC_SPEED_CNTL, LC_CURRENT_DATA_RATE));
+}
+
+int smu7_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr)
+{
+	uint32_t link_width;
+
+	/* mmPCIE_PORT_INDEX rename as mmPCIE_INDEX */
+	link_width = PHM_READ_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE,
+			PCIE_LC_LINK_WIDTH_CNTL, LC_LINK_WIDTH_RD);
+
+	PP_ASSERT_WITH_CODE((7 >= link_width),
+			"Invalid PCIe lane width!", return 0);
+
+	return decode_pcie_lane_width(link_width);
+}
+
+/**
+* Enable voltage control
+*
+* @param    pHwMgr  the address of the powerplay hardware manager.
+* @return   always PP_Result_OK
+*/
+int smu7_enable_smc_voltage_controller(struct pp_hwmgr *hwmgr)
+{
+	if (hwmgr->feature_mask & PP_SMC_VOLTAGE_CONTROL_MASK)
+		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Voltage_Cntl_Enable);
+
+	return 0;
+}
+
+/**
+* Checks if we want to support voltage control
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+*/
+static bool smu7_voltage_control(const struct pp_hwmgr *hwmgr)
+{
+	const struct smu7_hwmgr *data =
+			(const struct smu7_hwmgr *)(hwmgr->backend);
+
+	return (SMU7_VOLTAGE_CONTROL_NONE != data->voltage_control);
+}
+
+/**
+* Enable voltage control
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int smu7_enable_voltage_control(struct pp_hwmgr *hwmgr)
+{
+	/* enable voltage control */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			GENERAL_PWRMGT, VOLT_PWRMGT_EN, 1);
+
+	return 0;
+}
+
+static int phm_get_svi2_voltage_table_v0(pp_atomctrl_voltage_table *voltage_table,
+		struct phm_clock_voltage_dependency_table *voltage_dependency_table
+		)
+{
+	uint32_t i;
+
+	PP_ASSERT_WITH_CODE((NULL != voltage_table),
+			"Voltage Dependency Table empty.", return -EINVAL;);
+
+	voltage_table->mask_low = 0;
+	voltage_table->phase_delay = 0;
+	voltage_table->count = voltage_dependency_table->count;
+
+	for (i = 0; i < voltage_dependency_table->count; i++) {
+		voltage_table->entries[i].value =
+			voltage_dependency_table->entries[i].v;
+		voltage_table->entries[i].smio_low = 0;
+	}
+
+	return 0;
+}
+
+
+/**
+* Create Voltage Tables.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int smu7_construct_voltage_tables(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)hwmgr->pptable;
+	int result = 0;
+	uint32_t tmp;
+
+	if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		result = atomctrl_get_voltage_table_v3(hwmgr,
+				VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT,
+				&(data->mvdd_voltage_table));
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to retrieve MVDD table.",
+				return result);
+	} else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
+		if (hwmgr->pp_table_version == PP_TABLE_V1)
+			result = phm_get_svi2_mvdd_voltage_table(&(data->mvdd_voltage_table),
+					table_info->vdd_dep_on_mclk);
+		else if (hwmgr->pp_table_version == PP_TABLE_V0)
+			result = phm_get_svi2_voltage_table_v0(&(data->mvdd_voltage_table),
+					hwmgr->dyn_state.mvdd_dependency_on_mclk);
+
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to retrieve SVI2 MVDD table from dependancy table.",
+				return result;);
+	}
+
+	if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+		result = atomctrl_get_voltage_table_v3(hwmgr,
+				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT,
+				&(data->vddci_voltage_table));
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to retrieve VDDCI table.",
+				return result);
+	} else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
+		if (hwmgr->pp_table_version == PP_TABLE_V1)
+			result = phm_get_svi2_vddci_voltage_table(&(data->vddci_voltage_table),
+					table_info->vdd_dep_on_mclk);
+		else if (hwmgr->pp_table_version == PP_TABLE_V0)
+			result = phm_get_svi2_voltage_table_v0(&(data->vddci_voltage_table),
+					hwmgr->dyn_state.vddci_dependency_on_mclk);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to retrieve SVI2 VDDCI table from dependancy table.",
+				return result);
+	}
+
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_gfx_control) {
+		/* VDDGFX has only SVI2 voltage control */
+		result = phm_get_svi2_vdd_voltage_table(&(data->vddgfx_voltage_table),
+					table_info->vddgfx_lookup_table);
+		PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to retrieve SVI2 VDDGFX table from lookup table.", return result;);
+	}
+
+
+	if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->voltage_control) {
+		result = atomctrl_get_voltage_table_v3(hwmgr,
+					VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_GPIO_LUT,
+					&data->vddc_voltage_table);
+		PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to retrieve VDDC table.", return result;);
+	} else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+
+		if (hwmgr->pp_table_version == PP_TABLE_V0)
+			result = phm_get_svi2_voltage_table_v0(&data->vddc_voltage_table,
+					hwmgr->dyn_state.vddc_dependency_on_mclk);
+		else if (hwmgr->pp_table_version == PP_TABLE_V1)
+			result = phm_get_svi2_vdd_voltage_table(&(data->vddc_voltage_table),
+				table_info->vddc_lookup_table);
+
+		PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to retrieve SVI2 VDDC table from dependancy table.", return result;);
+	}
+
+	tmp = smum_get_mac_definition(hwmgr->smumgr, SMU_MAX_LEVELS_VDDC);
+	PP_ASSERT_WITH_CODE(
+			(data->vddc_voltage_table.count <= tmp),
+		"Too many voltage values for VDDC. Trimming to fit state table.",
+			phm_trim_voltage_table_to_fit_state_table(tmp,
+						&(data->vddc_voltage_table)));
+
+	tmp = smum_get_mac_definition(hwmgr->smumgr, SMU_MAX_LEVELS_VDDGFX);
+	PP_ASSERT_WITH_CODE(
+			(data->vddgfx_voltage_table.count <= tmp),
+		"Too many voltage values for VDDC. Trimming to fit state table.",
+			phm_trim_voltage_table_to_fit_state_table(tmp,
+						&(data->vddgfx_voltage_table)));
+
+	tmp = smum_get_mac_definition(hwmgr->smumgr, SMU_MAX_LEVELS_VDDCI);
+	PP_ASSERT_WITH_CODE(
+			(data->vddci_voltage_table.count <= tmp),
+		"Too many voltage values for VDDCI. Trimming to fit state table.",
+			phm_trim_voltage_table_to_fit_state_table(tmp,
+					&(data->vddci_voltage_table)));
+
+	tmp = smum_get_mac_definition(hwmgr->smumgr, SMU_MAX_LEVELS_MVDD);
+	PP_ASSERT_WITH_CODE(
+			(data->mvdd_voltage_table.count <= tmp),
+		"Too many voltage values for MVDD. Trimming to fit state table.",
+			phm_trim_voltage_table_to_fit_state_table(tmp,
+						&(data->mvdd_voltage_table)));
+
+	return 0;
+}
+
+/**
+* Programs static screed detection parameters
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int smu7_program_static_screen_threshold_parameters(
+							struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	/* Set static screen threshold unit */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD_UNIT,
+			data->static_screen_threshold_unit);
+	/* Set static screen threshold */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD,
+			data->static_screen_threshold);
+
+	return 0;
+}
+
+/**
+* Setup display gap for glitch free memory clock switching.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always  0
+*/
+static int smu7_enable_display_gap(struct pp_hwmgr *hwmgr)
+{
+	uint32_t display_gap =
+			cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixCG_DISPLAY_GAP_CNTL);
+
+	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL,
+			DISP_GAP, DISPLAY_GAP_IGNORE);
+
+	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL,
+			DISP_GAP_MCHG, DISPLAY_GAP_VBLANK);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_DISPLAY_GAP_CNTL, display_gap);
+
+	return 0;
+}
+
+/**
+* Programs activity state transition voting clients
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always  0
+*/
+static int smu7_program_voting_clients(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	/* Clear reset for voting clients before enabling DPM */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 0);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 0);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_0, data->voting_rights_clients0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_1, data->voting_rights_clients1);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_2, data->voting_rights_clients2);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_3, data->voting_rights_clients3);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_4, data->voting_rights_clients4);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_5, data->voting_rights_clients5);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_6, data->voting_rights_clients6);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_7, data->voting_rights_clients7);
+
+	return 0;
+}
+
+static int smu7_clear_voting_clients(struct pp_hwmgr *hwmgr)
+{
+	/* Reset voting clients before disabling DPM */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 1);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 1);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_0, 0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_1, 0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_2, 0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_3, 0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_4, 0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_5, 0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_6, 0);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_FREQ_TRAN_VOTING_7, 0);
+
+	return 0;
+}
+
+/* Copy one arb setting to another and then switch the active set.
+ * arb_src and arb_dest is one of the MC_CG_ARB_FREQ_Fx constants.
+ */
+static int smu7_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr,
+		uint32_t arb_src, uint32_t arb_dest)
+{
+	uint32_t mc_arb_dram_timing;
+	uint32_t mc_arb_dram_timing2;
+	uint32_t burst_time;
+	uint32_t mc_cg_config;
+
+	switch (arb_src) {
+	case MC_CG_ARB_FREQ_F0:
+		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
+		break;
+	case MC_CG_ARB_FREQ_F1:
+		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1);
+		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1);
+		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (arb_dest) {
+	case MC_CG_ARB_FREQ_F0:
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING, mc_arb_dram_timing);
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
+		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0, burst_time);
+		break;
+	case MC_CG_ARB_FREQ_F1:
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
+		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
+		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1, burst_time);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	mc_cg_config = cgs_read_register(hwmgr->device, mmMC_CG_CONFIG);
+	mc_cg_config |= 0x0000000F;
+	cgs_write_register(hwmgr->device, mmMC_CG_CONFIG, mc_cg_config);
+	PHM_WRITE_FIELD(hwmgr->device, MC_ARB_CG, CG_ARB_REQ, arb_dest);
+
+	return 0;
+}
+
+static int smu7_reset_to_default(struct pp_hwmgr *hwmgr)
+{
+	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_ResetToDefaults);
+}
+
+/**
+* Initial switch from ARB F0->F1
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+* This function is to be called from the SetPowerState table.
+*/
+static int smu7_initial_switch_from_arbf0_to_f1(struct pp_hwmgr *hwmgr)
+{
+	return smu7_copy_and_switch_arb_sets(hwmgr,
+			MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
+}
+
+static int smu7_force_switch_to_arbf0(struct pp_hwmgr *hwmgr)
+{
+	uint32_t tmp;
+
+	tmp = (cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, ixSMC_SCRATCH9) &
+			0x0000ff00) >> 8;
+
+	if (tmp == MC_CG_ARB_FREQ_F0)
+		return 0;
+
+	return smu7_copy_and_switch_arb_sets(hwmgr,
+			tmp, MC_CG_ARB_FREQ_F0);
+}
+
+static int smu7_setup_default_pcie_table(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_pcie_table *pcie_table = NULL;
+
+	uint32_t i, max_entry;
+	uint32_t tmp;
+
+	PP_ASSERT_WITH_CODE((data->use_pcie_performance_levels ||
+			data->use_pcie_power_saving_levels), "No pcie performance levels!",
+			return -EINVAL);
+
+	if (table_info != NULL)
+		pcie_table = table_info->pcie_table;
+
+	if (data->use_pcie_performance_levels &&
+			!data->use_pcie_power_saving_levels) {
+		data->pcie_gen_power_saving = data->pcie_gen_performance;
+		data->pcie_lane_power_saving = data->pcie_lane_performance;
+	} else if (!data->use_pcie_performance_levels &&
+			data->use_pcie_power_saving_levels) {
+		data->pcie_gen_performance = data->pcie_gen_power_saving;
+		data->pcie_lane_performance = data->pcie_lane_power_saving;
+	}
+	tmp = smum_get_mac_definition(hwmgr->smumgr, SMU_MAX_LEVELS_LINK);
+	phm_reset_single_dpm_table(&data->dpm_table.pcie_speed_table,
+					tmp,
+					MAX_REGULAR_DPM_NUMBER);
+
+	if (pcie_table != NULL) {
+		/* max_entry is used to make sure we reserve one PCIE level
+		 * for boot level (fix for A+A PSPP issue).
+		 * If PCIE table from PPTable have ULV entry + 8 entries,
+		 * then ignore the last entry.*/
+		max_entry = (tmp < pcie_table->count) ? tmp : pcie_table->count;
+		for (i = 1; i < max_entry; i++) {
+			phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, i - 1,
+					get_pcie_gen_support(data->pcie_gen_cap,
+							pcie_table->entries[i].gen_speed),
+					get_pcie_lane_support(data->pcie_lane_cap,
+							pcie_table->entries[i].lane_width));
+		}
+		data->dpm_table.pcie_speed_table.count = max_entry - 1;
+		smum_update_smc_table(hwmgr, SMU_BIF_TABLE);
+	} else {
+		/* Hardcode Pcie Table */
+		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 0,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Min_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 1,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Min_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 2,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Max_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 3,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Max_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 4,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Max_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+		phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 5,
+				get_pcie_gen_support(data->pcie_gen_cap,
+						PP_Max_PCIEGen),
+				get_pcie_lane_support(data->pcie_lane_cap,
+						PP_Max_PCIELane));
+
+		data->dpm_table.pcie_speed_table.count = 6;
+	}
+	/* Populate last level for boot PCIE level, but do not increment count. */
+	phm_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table,
+			data->dpm_table.pcie_speed_table.count,
+			get_pcie_gen_support(data->pcie_gen_cap,
+					PP_Min_PCIEGen),
+			get_pcie_lane_support(data->pcie_lane_cap,
+					PP_Max_PCIELane));
+
+	return 0;
+}
+
+static int smu7_reset_dpm_tables(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	memset(&(data->dpm_table), 0x00, sizeof(data->dpm_table));
+
+	phm_reset_single_dpm_table(
+			&data->dpm_table.sclk_table,
+				smum_get_mac_definition(hwmgr->smumgr,
+					SMU_MAX_LEVELS_GRAPHICS),
+					MAX_REGULAR_DPM_NUMBER);
+	phm_reset_single_dpm_table(
+			&data->dpm_table.mclk_table,
+			smum_get_mac_definition(hwmgr->smumgr,
+				SMU_MAX_LEVELS_MEMORY), MAX_REGULAR_DPM_NUMBER);
+
+	phm_reset_single_dpm_table(
+			&data->dpm_table.vddc_table,
+				smum_get_mac_definition(hwmgr->smumgr,
+					SMU_MAX_LEVELS_VDDC),
+					MAX_REGULAR_DPM_NUMBER);
+	phm_reset_single_dpm_table(
+			&data->dpm_table.vddci_table,
+			smum_get_mac_definition(hwmgr->smumgr,
+				SMU_MAX_LEVELS_VDDCI), MAX_REGULAR_DPM_NUMBER);
+
+	phm_reset_single_dpm_table(
+			&data->dpm_table.mvdd_table,
+				smum_get_mac_definition(hwmgr->smumgr,
+					SMU_MAX_LEVELS_MVDD),
+					MAX_REGULAR_DPM_NUMBER);
+	return 0;
+}
+/*
+ * This function is to initialize all DPM state tables
+ * for SMU7 based on the dependency table.
+ * Dynamic state patching function will then trim these
+ * state tables to the allowed range based
+ * on the power policy or external client requests,
+ * such as UVD request, etc.
+ */
+
+static int smu7_setup_dpm_tables_v0(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_clock_voltage_dependency_table *allowed_vdd_sclk_table =
+		hwmgr->dyn_state.vddc_dependency_on_sclk;
+	struct phm_clock_voltage_dependency_table *allowed_vdd_mclk_table =
+		hwmgr->dyn_state.vddc_dependency_on_mclk;
+	struct phm_cac_leakage_table *std_voltage_table =
+		hwmgr->dyn_state.cac_leakage_table;
+	uint32_t i;
+
+	PP_ASSERT_WITH_CODE(allowed_vdd_sclk_table != NULL,
+		"SCLK dependency table is missing. This table is mandatory", return -EINVAL);
+	PP_ASSERT_WITH_CODE(allowed_vdd_sclk_table->count >= 1,
+		"SCLK dependency table has to have is missing. This table is mandatory", return -EINVAL);
+
+	PP_ASSERT_WITH_CODE(allowed_vdd_mclk_table != NULL,
+		"MCLK dependency table is missing. This table is mandatory", return -EINVAL);
+	PP_ASSERT_WITH_CODE(allowed_vdd_mclk_table->count >= 1,
+		"VMCLK dependency table has to have is missing. This table is mandatory", return -EINVAL);
+
+
+	/* Initialize Sclk DPM table based on allow Sclk values*/
+	data->dpm_table.sclk_table.count = 0;
+
+	for (i = 0; i < allowed_vdd_sclk_table->count; i++) {
+		if (i == 0 || data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count-1].value !=
+				allowed_vdd_sclk_table->entries[i].clk) {
+			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].value =
+				allowed_vdd_sclk_table->entries[i].clk;
+			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled = 1; /*(i==0) ? 1 : 0; to do */
+			data->dpm_table.sclk_table.count++;
+		}
+	}
+
+	PP_ASSERT_WITH_CODE(allowed_vdd_mclk_table != NULL,
+		"MCLK dependency table is missing. This table is mandatory", return -EINVAL);
+	/* Initialize Mclk DPM table based on allow Mclk values */
+	data->dpm_table.mclk_table.count = 0;
+	for (i = 0; i < allowed_vdd_mclk_table->count; i++) {
+		if (i == 0 || data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count-1].value !=
+			allowed_vdd_mclk_table->entries[i].clk) {
+			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].value =
+				allowed_vdd_mclk_table->entries[i].clk;
+			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].enabled = 1; /*(i==0) ? 1 : 0; */
+			data->dpm_table.mclk_table.count++;
+		}
+	}
+
+	/* Initialize Vddc DPM table based on allow Vddc values.  And populate corresponding std values. */
+	for (i = 0; i < allowed_vdd_sclk_table->count; i++) {
+		data->dpm_table.vddc_table.dpm_levels[i].value = allowed_vdd_mclk_table->entries[i].v;
+		data->dpm_table.vddc_table.dpm_levels[i].param1 = std_voltage_table->entries[i].Leakage;
+		/* param1 is for corresponding std voltage */
+		data->dpm_table.vddc_table.dpm_levels[i].enabled = 1;
+	}
+
+	data->dpm_table.vddc_table.count = allowed_vdd_sclk_table->count;
+	allowed_vdd_mclk_table = hwmgr->dyn_state.vddci_dependency_on_mclk;
+
+	if (NULL != allowed_vdd_mclk_table) {
+		/* Initialize Vddci DPM table based on allow Mclk values */
+		for (i = 0; i < allowed_vdd_mclk_table->count; i++) {
+			data->dpm_table.vddci_table.dpm_levels[i].value = allowed_vdd_mclk_table->entries[i].v;
+			data->dpm_table.vddci_table.dpm_levels[i].enabled = 1;
+		}
+		data->dpm_table.vddci_table.count = allowed_vdd_mclk_table->count;
+	}
+
+	allowed_vdd_mclk_table = hwmgr->dyn_state.mvdd_dependency_on_mclk;
+
+	if (NULL != allowed_vdd_mclk_table) {
+		/*
+		 * Initialize MVDD DPM table based on allow Mclk
+		 * values
+		 */
+		for (i = 0; i < allowed_vdd_mclk_table->count; i++) {
+			data->dpm_table.mvdd_table.dpm_levels[i].value = allowed_vdd_mclk_table->entries[i].v;
+			data->dpm_table.mvdd_table.dpm_levels[i].enabled = 1;
+		}
+		data->dpm_table.mvdd_table.count = allowed_vdd_mclk_table->count;
+	}
+
+	return 0;
+}
+
+static int smu7_setup_dpm_tables_v1(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t i;
+
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_sclk_table;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table;
+
+	if (table_info == NULL)
+		return -EINVAL;
+
+	dep_sclk_table = table_info->vdd_dep_on_sclk;
+	dep_mclk_table = table_info->vdd_dep_on_mclk;
+
+	PP_ASSERT_WITH_CODE(dep_sclk_table != NULL,
+			"SCLK dependency table is missing.",
+			return -EINVAL);
+	PP_ASSERT_WITH_CODE(dep_sclk_table->count >= 1,
+			"SCLK dependency table count is 0.",
+			return -EINVAL);
+
+	PP_ASSERT_WITH_CODE(dep_mclk_table != NULL,
+			"MCLK dependency table is missing.",
+			return -EINVAL);
+	PP_ASSERT_WITH_CODE(dep_mclk_table->count >= 1,
+			"MCLK dependency table count is 0",
+			return -EINVAL);
+
+	/* Initialize Sclk DPM table based on allow Sclk values */
+	data->dpm_table.sclk_table.count = 0;
+	for (i = 0; i < dep_sclk_table->count; i++) {
+		if (i == 0 || data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count - 1].value !=
+						dep_sclk_table->entries[i].clk) {
+
+			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].value =
+					dep_sclk_table->entries[i].clk;
+
+			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled =
+					(i == 0) ? true : false;
+			data->dpm_table.sclk_table.count++;
+		}
+	}
+
+	/* Initialize Mclk DPM table based on allow Mclk values */
+	data->dpm_table.mclk_table.count = 0;
+	for (i = 0; i < dep_mclk_table->count; i++) {
+		if (i == 0 || data->dpm_table.mclk_table.dpm_levels
+				[data->dpm_table.mclk_table.count - 1].value !=
+						dep_mclk_table->entries[i].clk) {
+			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].value =
+							dep_mclk_table->entries[i].clk;
+			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].enabled =
+							(i == 0) ? true : false;
+			data->dpm_table.mclk_table.count++;
+		}
+	}
+
+	return 0;
+}
+
+int smu7_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	smu7_reset_dpm_tables(hwmgr);
+
+	if (hwmgr->pp_table_version == PP_TABLE_V1)
+		smu7_setup_dpm_tables_v1(hwmgr);
+	else if (hwmgr->pp_table_version == PP_TABLE_V0)
+		smu7_setup_dpm_tables_v0(hwmgr);
+
+	smu7_setup_default_pcie_table(hwmgr);
+
+	/* save a copy of the default DPM table */
+	memcpy(&(data->golden_dpm_table), &(data->dpm_table),
+			sizeof(struct smu7_dpm_table));
+	return 0;
+}
+
+uint32_t smu7_get_xclk(struct pp_hwmgr *hwmgr)
+{
+	uint32_t reference_clock, tmp;
+	struct cgs_display_info info = {0};
+	struct cgs_mode_info mode_info;
+
+	info.mode_info = &mode_info;
+
+	tmp = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_CLKPIN_CNTL_2, MUX_TCLK_TO_XCLK);
+
+	if (tmp)
+		return TCLK;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+	reference_clock = mode_info.ref_clock;
+
+	tmp = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_CLKPIN_CNTL, XTALIN_DIVIDE);
+
+	if (0 != tmp)
+		return reference_clock / 4;
+
+	return reference_clock;
+}
+
+static int smu7_enable_vrhot_gpio_interrupt(struct pp_hwmgr *hwmgr)
+{
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_RegulatorHot))
+		return smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_EnableVRHotGPIOInterrupt);
+
+	return 0;
+}
+
+static int smu7_enable_sclk_control(struct pp_hwmgr *hwmgr)
+{
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
+			SCLK_PWRMGT_OFF, 0);
+	return 0;
+}
+
+static int smu7_enable_ulv(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (data->ulv_supported)
+		return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_EnableULV);
+
+	return 0;
+}
+
+static int smu7_disable_ulv(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (data->ulv_supported)
+		return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DisableULV);
+
+	return 0;
+}
+
+static int smu7_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkDeepSleep)) {
+		if (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_MASTER_DeepSleep_ON))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to enable Master Deep Sleep switch failed!",
+					return -EINVAL);
+	} else {
+		if (smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_MASTER_DeepSleep_OFF)) {
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to disable Master Deep Sleep switch failed!",
+					return -EINVAL);
+		}
+	}
+
+	return 0;
+}
+
+static int smu7_disable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
+{
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkDeepSleep)) {
+		if (smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_MASTER_DeepSleep_OFF)) {
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to disable Master Deep Sleep switch failed!",
+					return -EINVAL);
+		}
+	}
+
+	return 0;
+}
+
+static int smu7_disable_handshake_uvd(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t soft_register_value = 0;
+	uint32_t handshake_disables_offset = data->soft_regs_start
+				+ smum_get_offsetof(hwmgr->smumgr,
+					SMU_SoftRegisters, HandshakeDisables);
+
+	soft_register_value = cgs_read_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC, handshake_disables_offset);
+	soft_register_value |= smum_get_mac_definition(hwmgr->smumgr,
+					SMU_UVD_MCLK_HANDSHAKE_DISABLE);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			handshake_disables_offset, soft_register_value);
+	return 0;
+}
+
+static int smu7_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	/* enable SCLK dpm */
+	if (!data->sclk_dpm_key_disabled)
+		PP_ASSERT_WITH_CODE(
+		(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DPM_Enable)),
+		"Failed to enable SCLK DPM during DPM Start Function!",
+		return -EINVAL);
+
+	/* enable MCLK dpm */
+	if (0 == data->mclk_dpm_key_disabled) {
+		if (!(hwmgr->feature_mask & PP_UVD_HANDSHAKE_MASK))
+			smu7_disable_handshake_uvd(hwmgr);
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+						PPSMC_MSG_MCLKDPM_Enable)),
+				"Failed to enable MCLK DPM during DPM Start Function!",
+				return -EINVAL);
+
+		PHM_WRITE_FIELD(hwmgr->device, MC_SEQ_CNTL_3, CAC_EN, 0x1);
+
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC0_CNTL, 0x5);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC1_CNTL, 0x5);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_CPL_CNTL, 0x100005);
+		udelay(10);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC0_CNTL, 0x400005);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_MC1_CNTL, 0x400005);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixLCAC_CPL_CNTL, 0x500005);
+	}
+
+	return 0;
+}
+
+static int smu7_start_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	/*enable general power management */
+
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
+			GLOBAL_PWRMGT_EN, 1);
+
+	/* enable sclk deep sleep */
+
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
+			DYNAMIC_PM_EN, 1);
+
+	/* prepare for PCIE DPM */
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			data->soft_regs_start +
+			smum_get_offsetof(hwmgr->smumgr, SMU_SoftRegisters,
+						VoltageChangeTimeout), 0x1000);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE,
+			SWRST_COMMAND_1, RESETLC, 0x0);
+
+	PP_ASSERT_WITH_CODE(
+			(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+					PPSMC_MSG_Voltage_Cntl_Enable)),
+			"Failed to enable voltage DPM during DPM Start Function!",
+			return -EINVAL);
+
+
+	if (smu7_enable_sclk_mclk_dpm(hwmgr)) {
+		printk(KERN_ERR "Failed to enable Sclk DPM and Mclk DPM!");
+		return -EINVAL;
+	}
+
+	/* enable PCIE dpm */
+	if (0 == data->pcie_dpm_key_disabled) {
+		PP_ASSERT_WITH_CODE(
+				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+						PPSMC_MSG_PCIeDPM_Enable)),
+				"Failed to enable pcie DPM during DPM Start Function!",
+				return -EINVAL);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_Falcon_QuickTransition)) {
+		PP_ASSERT_WITH_CODE((0 == smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_EnableACDCGPIOInterrupt)),
+				"Failed to enable AC DC GPIO Interrupt!",
+				);
+	}
+
+	return 0;
+}
+
+static int smu7_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	/* disable SCLK dpm */
+	if (!data->sclk_dpm_key_disabled)
+		PP_ASSERT_WITH_CODE(
+				(smum_send_msg_to_smc(hwmgr->smumgr,
+						PPSMC_MSG_DPM_Disable) == 0),
+				"Failed to disable SCLK DPM!",
+				return -EINVAL);
+
+	/* disable MCLK dpm */
+	if (!data->mclk_dpm_key_disabled) {
+		PP_ASSERT_WITH_CODE(
+				(smum_send_msg_to_smc(hwmgr->smumgr,
+						PPSMC_MSG_MCLKDPM_Disable) == 0),
+				"Failed to disable MCLK DPM!",
+				return -EINVAL);
+	}
+
+	return 0;
+}
+
+static int smu7_stop_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	/* disable general power management */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
+			GLOBAL_PWRMGT_EN, 0);
+	/* disable sclk deep sleep */
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
+			DYNAMIC_PM_EN, 0);
+
+	/* disable PCIE dpm */
+	if (!data->pcie_dpm_key_disabled) {
+		PP_ASSERT_WITH_CODE(
+				(smum_send_msg_to_smc(hwmgr->smumgr,
+						PPSMC_MSG_PCIeDPM_Disable) == 0),
+				"Failed to disable pcie DPM during DPM Stop Function!",
+				return -EINVAL);
+	}
+
+	if (smu7_disable_sclk_mclk_dpm(hwmgr)) {
+		printk(KERN_ERR "Failed to disable Sclk DPM and Mclk DPM!");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void smu7_set_dpm_event_sources(struct pp_hwmgr *hwmgr, uint32_t sources)
+{
+	bool protection;
+	enum DPM_EVENT_SRC src;
+
+	switch (sources) {
+	default:
+		printk(KERN_ERR "Unknown throttling event sources.");
+		/* fall through */
+	case 0:
+		protection = false;
+		/* src is unused */
+		break;
+	case (1 << PHM_AutoThrottleSource_Thermal):
+		protection = true;
+		src = DPM_EVENT_SRC_DIGITAL;
+		break;
+	case (1 << PHM_AutoThrottleSource_External):
+		protection = true;
+		src = DPM_EVENT_SRC_EXTERNAL;
+		break;
+	case (1 << PHM_AutoThrottleSource_External) |
+			(1 << PHM_AutoThrottleSource_Thermal):
+		protection = true;
+		src = DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL;
+		break;
+	}
+	/* Order matters - don't enable thermal protection for the wrong source. */
+	if (protection) {
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_CTRL,
+				DPM_EVENT_SRC, src);
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
+				THERMAL_PROTECTION_DIS,
+				!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_ThermalController));
+	} else
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
+				THERMAL_PROTECTION_DIS, 1);
+}
+
+static int smu7_enable_auto_throttle_source(struct pp_hwmgr *hwmgr,
+		PHM_AutoThrottleSource source)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (!(data->active_auto_throttle_sources & (1 << source))) {
+		data->active_auto_throttle_sources |= 1 << source;
+		smu7_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources);
+	}
+	return 0;
+}
+
+static int smu7_enable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
+{
+	return smu7_enable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
+}
+
+static int smu7_disable_auto_throttle_source(struct pp_hwmgr *hwmgr,
+		PHM_AutoThrottleSource source)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (data->active_auto_throttle_sources & (1 << source)) {
+		data->active_auto_throttle_sources &= ~(1 << source);
+		smu7_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources);
+	}
+	return 0;
+}
+
+static int smu7_disable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
+{
+	return smu7_disable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
+}
+
+int smu7_pcie_performance_request(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	data->pcie_performance_request = true;
+
+	return 0;
+}
+
+int smu7_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
+{
+	int tmp_result = 0;
+	int result = 0;
+
+	tmp_result = (!smum_is_dpm_running(hwmgr)) ? 0 : -1;
+	PP_ASSERT_WITH_CODE(tmp_result == 0,
+			"DPM is already running right now, no need to enable DPM!",
+			return 0);
+
+	if (smu7_voltage_control(hwmgr)) {
+		tmp_result = smu7_enable_voltage_control(hwmgr);
+		PP_ASSERT_WITH_CODE(tmp_result == 0,
+				"Failed to enable voltage control!",
+				result = tmp_result);
+
+		tmp_result = smu7_construct_voltage_tables(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == tmp_result),
+				"Failed to contruct voltage tables!",
+				result = tmp_result);
+	}
+	smum_initialize_mc_reg_table(hwmgr);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EngineSpreadSpectrumSupport))
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 1);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ThermalController))
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 0);
+
+	tmp_result = smu7_program_static_screen_threshold_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to program static screen threshold parameters!",
+			result = tmp_result);
+
+	tmp_result = smu7_enable_display_gap(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable display gap!", result = tmp_result);
+
+	tmp_result = smu7_program_voting_clients(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to program voting clients!", result = tmp_result);
+
+	tmp_result = smum_process_firmware_header(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to process firmware header!", result = tmp_result);
+
+	tmp_result = smu7_initial_switch_from_arbf0_to_f1(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to initialize switch from ArbF0 to F1!",
+			result = tmp_result);
+
+	result = smu7_setup_default_dpm_tables(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to setup default DPM tables!", return result);
+
+	tmp_result = smum_init_smc_table(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to initialize SMC table!", result = tmp_result);
+
+	tmp_result = smu7_enable_vrhot_gpio_interrupt(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable VR hot GPIO interrupt!", result = tmp_result);
+
+	smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_HasDisplay);
+
+	tmp_result = smu7_enable_sclk_control(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable SCLK control!", result = tmp_result);
+
+	tmp_result = smu7_enable_smc_voltage_controller(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable voltage control!", result = tmp_result);
+
+	tmp_result = smu7_enable_ulv(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable ULV!", result = tmp_result);
+
+	tmp_result = smu7_enable_deep_sleep_master_switch(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable deep sleep master switch!", result = tmp_result);
+
+	tmp_result = smu7_enable_didt_config(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to enable deep sleep master switch!", result = tmp_result);
+
+	tmp_result = smu7_start_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to start DPM!", result = tmp_result);
+
+	tmp_result = smu7_enable_smc_cac(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable SMC CAC!", result = tmp_result);
+
+	tmp_result = smu7_enable_power_containment(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable power containment!", result = tmp_result);
+
+	tmp_result = smu7_power_control_set_level(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to power control set level!", result = tmp_result);
+
+	tmp_result = smu7_enable_thermal_auto_throttle(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable thermal auto throttle!", result = tmp_result);
+
+	tmp_result = smu7_pcie_performance_request(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"pcie performance request failed!", result = tmp_result);
+
+	return 0;
+}
+
+int smu7_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
+{
+	int tmp_result, result = 0;
+
+	tmp_result = (smum_is_dpm_running(hwmgr)) ? 0 : -1;
+	PP_ASSERT_WITH_CODE(tmp_result == 0,
+			"DPM is not running right now, no need to disable DPM!",
+			return 0);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ThermalController))
+		PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 1);
+
+	tmp_result = smu7_disable_power_containment(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to disable power containment!", result = tmp_result);
+
+	tmp_result = smu7_disable_smc_cac(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to disable SMC CAC!", result = tmp_result);
+
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_SPLL_SPREAD_SPECTRUM, SSEN, 0);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 0);
+
+	tmp_result = smu7_disable_thermal_auto_throttle(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to disable thermal auto throttle!", result = tmp_result);
+
+	tmp_result = smu7_stop_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to stop DPM!", result = tmp_result);
+
+	tmp_result = smu7_disable_deep_sleep_master_switch(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to disable deep sleep master switch!", result = tmp_result);
+
+	tmp_result = smu7_disable_ulv(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to disable ULV!", result = tmp_result);
+
+	tmp_result = smu7_clear_voting_clients(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to clear voting clients!", result = tmp_result);
+
+	tmp_result = smu7_reset_to_default(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to reset to default!", result = tmp_result);
+
+	tmp_result = smu7_force_switch_to_arbf0(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to force to switch arbf0!", result = tmp_result);
+
+	return result;
+}
+
+int smu7_reset_asic_tasks(struct pp_hwmgr *hwmgr)
+{
+
+	return 0;
+}
+
+static void smu7_init_dpm_defaults(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	data->dll_default_on = false;
+	data->mclk_dpm0_activity_target = 0xa;
+	data->mclk_activity_target = SMU7_MCLK_TARGETACTIVITY_DFLT;
+	data->vddc_vddgfx_delta = 300;
+	data->static_screen_threshold = SMU7_STATICSCREENTHRESHOLD_DFLT;
+	data->static_screen_threshold_unit = SMU7_STATICSCREENTHRESHOLDUNIT_DFLT;
+	data->voting_rights_clients0 = SMU7_VOTINGRIGHTSCLIENTS_DFLT0;
+	data->voting_rights_clients1 = SMU7_VOTINGRIGHTSCLIENTS_DFLT1;
+	data->voting_rights_clients2 = SMU7_VOTINGRIGHTSCLIENTS_DFLT2;
+	data->voting_rights_clients3 = SMU7_VOTINGRIGHTSCLIENTS_DFLT3;
+	data->voting_rights_clients4 = SMU7_VOTINGRIGHTSCLIENTS_DFLT4;
+	data->voting_rights_clients5 = SMU7_VOTINGRIGHTSCLIENTS_DFLT5;
+	data->voting_rights_clients6 = SMU7_VOTINGRIGHTSCLIENTS_DFLT6;
+	data->voting_rights_clients7 = SMU7_VOTINGRIGHTSCLIENTS_DFLT7;
+
+	data->mclk_dpm_key_disabled = hwmgr->feature_mask & PP_MCLK_DPM_MASK ? false : true;
+	data->sclk_dpm_key_disabled = hwmgr->feature_mask & PP_SCLK_DPM_MASK ? false : true;
+	data->pcie_dpm_key_disabled = hwmgr->feature_mask & PP_PCIE_DPM_MASK ? false : true;
+	/* need to set voltage control types before EVV patching */
+	data->voltage_control = SMU7_VOLTAGE_CONTROL_NONE;
+	data->vddci_control = SMU7_VOLTAGE_CONTROL_NONE;
+	data->mvdd_control = SMU7_VOLTAGE_CONTROL_NONE;
+	data->enable_tdc_limit_feature = true;
+	data->enable_pkg_pwr_tracking_feature = true;
+	data->force_pcie_gen = PP_PCIEGenInvalid;
+	data->ulv_supported = hwmgr->feature_mask & PP_ULV_MASK ? true : false;
+
+	data->fast_watermark_threshold = 100;
+	if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+			VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2))
+		data->voltage_control = SMU7_VOLTAGE_CONTROL_BY_SVID2;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ControlVDDGFX)) {
+		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+			VOLTAGE_TYPE_VDDGFX, VOLTAGE_OBJ_SVID2)) {
+			data->vdd_gfx_control = SMU7_VOLTAGE_CONTROL_BY_SVID2;
+		}
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EnableMVDDControl)) {
+		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+				VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT))
+			data->mvdd_control = SMU7_VOLTAGE_CONTROL_BY_GPIO;
+		else if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+				VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_SVID2))
+			data->mvdd_control = SMU7_VOLTAGE_CONTROL_BY_SVID2;
+	}
+
+	if (SMU7_VOLTAGE_CONTROL_NONE == data->vdd_gfx_control) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ControlVDDGFX);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ControlVDDCI)) {
+		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT))
+			data->vddci_control = SMU7_VOLTAGE_CONTROL_BY_GPIO;
+		else if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
+				VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2))
+			data->vddci_control = SMU7_VOLTAGE_CONTROL_BY_SVID2;
+	}
+
+	if (data->mvdd_control == SMU7_VOLTAGE_CONTROL_NONE)
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_EnableMVDDControl);
+
+	if (data->vddci_control == SMU7_VOLTAGE_CONTROL_NONE)
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ControlVDDCI);
+
+	if ((hwmgr->pp_table_version != PP_TABLE_V0)
+		&& (table_info->cac_dtp_table->usClockStretchAmount != 0))
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_ClockStretcher);
+
+	data->pcie_gen_performance.max = PP_PCIEGen1;
+	data->pcie_gen_performance.min = PP_PCIEGen3;
+	data->pcie_gen_power_saving.max = PP_PCIEGen1;
+	data->pcie_gen_power_saving.min = PP_PCIEGen3;
+	data->pcie_lane_performance.max = 0;
+	data->pcie_lane_performance.min = 16;
+	data->pcie_lane_power_saving.max = 0;
+	data->pcie_lane_power_saving.min = 16;
+}
+
+/**
+* Get Leakage VDDC based on leakage ID.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int smu7_get_evv_voltages(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint16_t vv_id;
+	uint16_t vddc = 0;
+	uint16_t vddgfx = 0;
+	uint16_t i, j;
+	uint32_t sclk = 0;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)hwmgr->pptable;
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table = NULL;
+
+
+	if (table_info != NULL)
+		sclk_table = table_info->vdd_dep_on_sclk;
+
+	for (i = 0; i < SMU7_MAX_LEAKAGE_COUNT; i++) {
+		vv_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
+
+		if (data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) {
+			if (0 == phm_get_sclk_for_voltage_evv(hwmgr,
+						table_info->vddgfx_lookup_table, vv_id, &sclk)) {
+				if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+							PHM_PlatformCaps_ClockStretcher)) {
+					for (j = 1; j < sclk_table->count; j++) {
+						if (sclk_table->entries[j].clk == sclk &&
+								sclk_table->entries[j].cks_enable == 0) {
+							sclk += 5000;
+							break;
+						}
+					}
+				}
+				if (0 == atomctrl_get_voltage_evv_on_sclk
+				    (hwmgr, VOLTAGE_TYPE_VDDGFX, sclk,
+				     vv_id, &vddgfx)) {
+					/* need to make sure vddgfx is less than 2v or else, it could burn the ASIC. */
+					PP_ASSERT_WITH_CODE((vddgfx < 2000 && vddgfx != 0), "Invalid VDDGFX value!", return -EINVAL);
+
+					/* the voltage should not be zero nor equal to leakage ID */
+					if (vddgfx != 0 && vddgfx != vv_id) {
+						data->vddcgfx_leakage.actual_voltage[data->vddcgfx_leakage.count] = vddgfx;
+						data->vddcgfx_leakage.leakage_id[data->vddcgfx_leakage.count] = vv_id;
+						data->vddcgfx_leakage.count++;
+					}
+				} else {
+					printk("Error retrieving EVV voltage value!\n");
+				}
+			}
+		} else {
+
+			if ((hwmgr->pp_table_version == PP_TABLE_V0)
+				|| !phm_get_sclk_for_voltage_evv(hwmgr,
+					table_info->vddc_lookup_table, vv_id, &sclk)) {
+				if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_ClockStretcher)) {
+					for (j = 1; j < sclk_table->count; j++) {
+						if (sclk_table->entries[j].clk == sclk &&
+								sclk_table->entries[j].cks_enable == 0) {
+							sclk += 5000;
+							break;
+						}
+					}
+				}
+
+				if (phm_get_voltage_evv_on_sclk(hwmgr,
+							VOLTAGE_TYPE_VDDC,
+							sclk, vv_id, &vddc) == 0) {
+					if (vddc >= 2000 || vddc == 0)
+						return -EINVAL;
+				} else {
+					printk(KERN_WARNING "failed to retrieving EVV voltage!\n");
+					continue;
+				}
+
+				/* the voltage should not be zero nor equal to leakage ID */
+				if (vddc != 0 && vddc != vv_id) {
+					data->vddc_leakage.actual_voltage[data->vddc_leakage.count] = (uint16_t)(vddc);
+					data->vddc_leakage.leakage_id[data->vddc_leakage.count] = vv_id;
+					data->vddc_leakage.count++;
+				}
+			}
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Change virtual leakage voltage to actual value.
+ *
+ * @param     hwmgr  the address of the powerplay hardware manager.
+ * @param     pointer to changing voltage
+ * @param     pointer to leakage table
+ */
+static void smu7_patch_ppt_v1_with_vdd_leakage(struct pp_hwmgr *hwmgr,
+		uint16_t *voltage, struct smu7_leakage_voltage *leakage_table)
+{
+	uint32_t index;
+
+	/* search for leakage voltage ID 0xff01 ~ 0xff08 */
+	for (index = 0; index < leakage_table->count; index++) {
+		/* if this voltage matches a leakage voltage ID */
+		/* patch with actual leakage voltage */
+		if (leakage_table->leakage_id[index] == *voltage) {
+			*voltage = leakage_table->actual_voltage[index];
+			break;
+		}
+	}
+
+	if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
+		printk(KERN_ERR "Voltage value looks like a Leakage ID but it's not patched \n");
+}
+
+/**
+* Patch voltage lookup table by EVV leakages.
+*
+* @param     hwmgr  the address of the powerplay hardware manager.
+* @param     pointer to voltage lookup table
+* @param     pointer to leakage table
+* @return     always 0
+*/
+static int smu7_patch_lookup_table_with_leakage(struct pp_hwmgr *hwmgr,
+		phm_ppt_v1_voltage_lookup_table *lookup_table,
+		struct smu7_leakage_voltage *leakage_table)
+{
+	uint32_t i;
+
+	for (i = 0; i < lookup_table->count; i++)
+		smu7_patch_ppt_v1_with_vdd_leakage(hwmgr,
+				&lookup_table->entries[i].us_vdd, leakage_table);
+
+	return 0;
+}
+
+static int smu7_patch_clock_voltage_limits_with_vddc_leakage(
+		struct pp_hwmgr *hwmgr, struct smu7_leakage_voltage *leakage_table,
+		uint16_t *vddc)
+{
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	smu7_patch_ppt_v1_with_vdd_leakage(hwmgr, (uint16_t *)vddc, leakage_table);
+	hwmgr->dyn_state.max_clock_voltage_on_dc.vddc =
+			table_info->max_clock_voltage_on_dc.vddc;
+	return 0;
+}
+
+static int smu7_patch_voltage_dependency_tables_with_lookup_table(
+		struct pp_hwmgr *hwmgr)
+{
+	uint8_t entry_id;
+	uint8_t voltage_id;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+	struct phm_ppt_v1_clock_voltage_dependency_table *mclk_table =
+			table_info->vdd_dep_on_mclk;
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+
+	if (data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) {
+		for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) {
+			voltage_id = sclk_table->entries[entry_id].vddInd;
+			sclk_table->entries[entry_id].vddgfx =
+				table_info->vddgfx_lookup_table->entries[voltage_id].us_vdd;
+		}
+	} else {
+		for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) {
+			voltage_id = sclk_table->entries[entry_id].vddInd;
+			sclk_table->entries[entry_id].vddc =
+				table_info->vddc_lookup_table->entries[voltage_id].us_vdd;
+		}
+	}
+
+	for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) {
+		voltage_id = mclk_table->entries[entry_id].vddInd;
+		mclk_table->entries[entry_id].vddc =
+			table_info->vddc_lookup_table->entries[voltage_id].us_vdd;
+	}
+
+	for (entry_id = 0; entry_id < mm_table->count; ++entry_id) {
+		voltage_id = mm_table->entries[entry_id].vddcInd;
+		mm_table->entries[entry_id].vddc =
+			table_info->vddc_lookup_table->entries[voltage_id].us_vdd;
+	}
+
+	return 0;
+
+}
+
+static int phm_add_voltage(struct pp_hwmgr *hwmgr,
+			phm_ppt_v1_voltage_lookup_table *look_up_table,
+			phm_ppt_v1_voltage_lookup_record *record)
+{
+	uint32_t i;
+
+	PP_ASSERT_WITH_CODE((NULL != look_up_table),
+		"Lookup Table empty.", return -EINVAL);
+	PP_ASSERT_WITH_CODE((0 != look_up_table->count),
+		"Lookup Table empty.", return -EINVAL);
+
+	i = smum_get_mac_definition(hwmgr->smumgr, SMU_MAX_LEVELS_VDDGFX);
+	PP_ASSERT_WITH_CODE((i >= look_up_table->count),
+		"Lookup Table is full.", return -EINVAL);
+
+	/* This is to avoid entering duplicate calculated records. */
+	for (i = 0; i < look_up_table->count; i++) {
+		if (look_up_table->entries[i].us_vdd == record->us_vdd) {
+			if (look_up_table->entries[i].us_calculated == 1)
+				return 0;
+			break;
+		}
+	}
+
+	look_up_table->entries[i].us_calculated = 1;
+	look_up_table->entries[i].us_vdd = record->us_vdd;
+	look_up_table->entries[i].us_cac_low = record->us_cac_low;
+	look_up_table->entries[i].us_cac_mid = record->us_cac_mid;
+	look_up_table->entries[i].us_cac_high = record->us_cac_high;
+	/* Only increment the count when we're appending, not replacing duplicate entry. */
+	if (i == look_up_table->count)
+		look_up_table->count++;
+
+	return 0;
+}
+
+
+static int smu7_calc_voltage_dependency_tables(struct pp_hwmgr *hwmgr)
+{
+	uint8_t entry_id;
+	struct phm_ppt_v1_voltage_lookup_record v_record;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	phm_ppt_v1_clock_voltage_dependency_table *sclk_table = pptable_info->vdd_dep_on_sclk;
+	phm_ppt_v1_clock_voltage_dependency_table *mclk_table = pptable_info->vdd_dep_on_mclk;
+
+	if (data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) {
+		for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) {
+			if (sclk_table->entries[entry_id].vdd_offset & (1 << 15))
+				v_record.us_vdd = sclk_table->entries[entry_id].vddgfx +
+					sclk_table->entries[entry_id].vdd_offset - 0xFFFF;
+			else
+				v_record.us_vdd = sclk_table->entries[entry_id].vddgfx +
+					sclk_table->entries[entry_id].vdd_offset;
+
+			sclk_table->entries[entry_id].vddc =
+				v_record.us_cac_low = v_record.us_cac_mid =
+				v_record.us_cac_high = v_record.us_vdd;
+
+			phm_add_voltage(hwmgr, pptable_info->vddc_lookup_table, &v_record);
+		}
+
+		for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) {
+			if (mclk_table->entries[entry_id].vdd_offset & (1 << 15))
+				v_record.us_vdd = mclk_table->entries[entry_id].vddc +
+					mclk_table->entries[entry_id].vdd_offset - 0xFFFF;
+			else
+				v_record.us_vdd = mclk_table->entries[entry_id].vddc +
+					mclk_table->entries[entry_id].vdd_offset;
+
+			mclk_table->entries[entry_id].vddgfx = v_record.us_cac_low =
+				v_record.us_cac_mid = v_record.us_cac_high = v_record.us_vdd;
+			phm_add_voltage(hwmgr, pptable_info->vddgfx_lookup_table, &v_record);
+		}
+	}
+	return 0;
+}
+
+static int smu7_calc_mm_voltage_dependency_table(struct pp_hwmgr *hwmgr)
+{
+	uint8_t entry_id;
+	struct phm_ppt_v1_voltage_lookup_record v_record;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
+
+	if (data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) {
+		for (entry_id = 0; entry_id < mm_table->count; entry_id++) {
+			if (mm_table->entries[entry_id].vddgfx_offset & (1 << 15))
+				v_record.us_vdd = mm_table->entries[entry_id].vddc +
+					mm_table->entries[entry_id].vddgfx_offset - 0xFFFF;
+			else
+				v_record.us_vdd = mm_table->entries[entry_id].vddc +
+					mm_table->entries[entry_id].vddgfx_offset;
+
+			/* Add the calculated VDDGFX to the VDDGFX lookup table */
+			mm_table->entries[entry_id].vddgfx = v_record.us_cac_low =
+				v_record.us_cac_mid = v_record.us_cac_high = v_record.us_vdd;
+			phm_add_voltage(hwmgr, pptable_info->vddgfx_lookup_table, &v_record);
+		}
+	}
+	return 0;
+}
+
+static int smu7_sort_lookup_table(struct pp_hwmgr *hwmgr,
+		struct phm_ppt_v1_voltage_lookup_table *lookup_table)
+{
+	uint32_t table_size, i, j;
+	struct phm_ppt_v1_voltage_lookup_record tmp_voltage_lookup_record;
+	table_size = lookup_table->count;
+
+	PP_ASSERT_WITH_CODE(0 != lookup_table->count,
+		"Lookup table is empty", return -EINVAL);
+
+	/* Sorting voltages */
+	for (i = 0; i < table_size - 1; i++) {
+		for (j = i + 1; j > 0; j--) {
+			if (lookup_table->entries[j].us_vdd <
+					lookup_table->entries[j - 1].us_vdd) {
+				tmp_voltage_lookup_record = lookup_table->entries[j - 1];
+				lookup_table->entries[j - 1] = lookup_table->entries[j];
+				lookup_table->entries[j] = tmp_voltage_lookup_record;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int smu7_complete_dependency_tables(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+	int tmp_result;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) {
+		tmp_result = smu7_patch_lookup_table_with_leakage(hwmgr,
+			table_info->vddgfx_lookup_table, &(data->vddcgfx_leakage));
+		if (tmp_result != 0)
+			result = tmp_result;
+
+		smu7_patch_ppt_v1_with_vdd_leakage(hwmgr,
+			&table_info->max_clock_voltage_on_dc.vddgfx, &(data->vddcgfx_leakage));
+	} else {
+
+		tmp_result = smu7_patch_lookup_table_with_leakage(hwmgr,
+				table_info->vddc_lookup_table, &(data->vddc_leakage));
+		if (tmp_result)
+			result = tmp_result;
+
+		tmp_result = smu7_patch_clock_voltage_limits_with_vddc_leakage(hwmgr,
+				&(data->vddc_leakage), &table_info->max_clock_voltage_on_dc.vddc);
+		if (tmp_result)
+			result = tmp_result;
+	}
+
+	tmp_result = smu7_patch_voltage_dependency_tables_with_lookup_table(hwmgr);
+	if (tmp_result)
+		result = tmp_result;
+
+	tmp_result = smu7_calc_voltage_dependency_tables(hwmgr);
+	if (tmp_result)
+		result = tmp_result;
+
+	tmp_result = smu7_calc_mm_voltage_dependency_table(hwmgr);
+	if (tmp_result)
+		result = tmp_result;
+
+	tmp_result = smu7_sort_lookup_table(hwmgr, table_info->vddgfx_lookup_table);
+	if (tmp_result)
+		result = tmp_result;
+
+	tmp_result = smu7_sort_lookup_table(hwmgr, table_info->vddc_lookup_table);
+	if (tmp_result)
+		result = tmp_result;
+
+	return result;
+}
+
+static int smu7_set_private_data_based_on_pptable_v1(struct pp_hwmgr *hwmgr)
+{
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	struct phm_ppt_v1_clock_voltage_dependency_table *allowed_sclk_vdd_table =
+						table_info->vdd_dep_on_sclk;
+	struct phm_ppt_v1_clock_voltage_dependency_table *allowed_mclk_vdd_table =
+						table_info->vdd_dep_on_mclk;
+
+	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table != NULL,
+		"VDD dependency on SCLK table is missing.",
+		return -EINVAL);
+	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table->count >= 1,
+		"VDD dependency on SCLK table has to have is missing.",
+		return -EINVAL);
+
+	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table != NULL,
+		"VDD dependency on MCLK table is missing",
+		return -EINVAL);
+	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table->count >= 1,
+		"VDD dependency on MCLK table has to have is missing.",
+		return -EINVAL);
+
+	table_info->max_clock_voltage_on_ac.sclk =
+		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].clk;
+	table_info->max_clock_voltage_on_ac.mclk =
+		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].clk;
+	table_info->max_clock_voltage_on_ac.vddc =
+		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].vddc;
+	table_info->max_clock_voltage_on_ac.vddci =
+		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].vddci;
+
+	hwmgr->dyn_state.max_clock_voltage_on_ac.sclk = table_info->max_clock_voltage_on_ac.sclk;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.mclk = table_info->max_clock_voltage_on_ac.mclk;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.vddc = table_info->max_clock_voltage_on_ac.vddc;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.vddci = table_info->max_clock_voltage_on_ac.vddci;
+
+	return 0;
+}
+
+int smu7_patch_voltage_workaround(struct pp_hwmgr *hwmgr)
+{
+	struct phm_ppt_v1_information *table_info =
+		       (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table;
+	struct phm_ppt_v1_voltage_lookup_table *lookup_table;
+	uint32_t i;
+	uint32_t hw_revision, sub_vendor_id, sub_sys_id;
+	struct cgs_system_info sys_info = {0};
+
+	if (table_info != NULL) {
+		dep_mclk_table = table_info->vdd_dep_on_mclk;
+		lookup_table = table_info->vddc_lookup_table;
+	} else
+		return 0;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_REV;
+	cgs_query_system_info(hwmgr->device, &sys_info);
+	hw_revision = (uint32_t)sys_info.value;
+
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_SUB_SYS_ID;
+	cgs_query_system_info(hwmgr->device, &sys_info);
+	sub_sys_id = (uint32_t)sys_info.value;
+
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_SUB_SYS_VENDOR_ID;
+	cgs_query_system_info(hwmgr->device, &sys_info);
+	sub_vendor_id = (uint32_t)sys_info.value;
+
+	if (hwmgr->chip_id == CHIP_POLARIS10 && hw_revision == 0xC7 &&
+			((sub_sys_id == 0xb37 && sub_vendor_id == 0x1002) ||
+		    (sub_sys_id == 0x4a8 && sub_vendor_id == 0x1043) ||
+		    (sub_sys_id == 0x9480 && sub_vendor_id == 0x1682))) {
+		if (lookup_table->entries[dep_mclk_table->entries[dep_mclk_table->count-1].vddInd].us_vdd >= 1000)
+			return 0;
+
+		for (i = 0; i < lookup_table->count; i++) {
+			if (lookup_table->entries[i].us_vdd < 0xff01 && lookup_table->entries[i].us_vdd >= 1000) {
+				dep_mclk_table->entries[dep_mclk_table->count-1].vddInd = (uint8_t) i;
+				return 0;
+			}
+		}
+	}
+	return 0;
+}
+
+static int smu7_thermal_parameter_init(struct pp_hwmgr *hwmgr)
+{
+	struct pp_atomctrl_gpio_pin_assignment gpio_pin_assignment;
+	uint32_t temp_reg;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+
+	if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_PCC_GPIO_PINID, &gpio_pin_assignment)) {
+		temp_reg = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCNB_PWRMGT_CNTL);
+		switch (gpio_pin_assignment.uc_gpio_pin_bit_shift) {
+		case 0:
+			temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, GNB_SLOW_MODE, 0x1);
+			break;
+		case 1:
+			temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, GNB_SLOW_MODE, 0x2);
+			break;
+		case 2:
+			temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, GNB_SLOW, 0x1);
+			break;
+		case 3:
+			temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, FORCE_NB_PS1, 0x1);
+			break;
+		case 4:
+			temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, DPM_ENABLED, 0x1);
+			break;
+		default:
+			PP_ASSERT_WITH_CODE(0,
+			"Failed to setup PCC HW register! Wrong GPIO assigned for VDDC_PCC_GPIO_PINID!",
+			);
+			break;
+		}
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCNB_PWRMGT_CNTL, temp_reg);
+	}
+
+	if (table_info == NULL)
+		return 0;
+
+	if (table_info->cac_dtp_table->usDefaultTargetOperatingTemp != 0 &&
+		hwmgr->thermal_controller.advanceFanControlParameters.ucFanControlMode) {
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanPWMMinLimit =
+			(uint16_t)hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit;
+
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanPWMMaxLimit =
+			(uint16_t)hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanPWM;
+
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanPWMStep = 1;
+
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanRPMMaxLimit = 100;
+
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanRPMMinLimit =
+			(uint16_t)hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit;
+
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanRPMStep = 1;
+
+		table_info->cac_dtp_table->usDefaultTargetOperatingTemp = (table_info->cac_dtp_table->usDefaultTargetOperatingTemp >= 50) ?
+								(table_info->cac_dtp_table->usDefaultTargetOperatingTemp - 50) : 0;
+
+		table_info->cac_dtp_table->usOperatingTempMaxLimit = table_info->cac_dtp_table->usDefaultTargetOperatingTemp;
+		table_info->cac_dtp_table->usOperatingTempStep = 1;
+		table_info->cac_dtp_table->usOperatingTempHyst = 1;
+
+		hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanPWM =
+			       hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanPWM;
+
+		hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanRPM =
+			       hwmgr->thermal_controller.advanceFanControlParameters.usDefaultMaxFanRPM;
+
+		hwmgr->dyn_state.cac_dtp_table->usOperatingTempMinLimit =
+			       table_info->cac_dtp_table->usOperatingTempMinLimit;
+
+		hwmgr->dyn_state.cac_dtp_table->usOperatingTempMaxLimit =
+			       table_info->cac_dtp_table->usOperatingTempMaxLimit;
+
+		hwmgr->dyn_state.cac_dtp_table->usDefaultTargetOperatingTemp =
+			       table_info->cac_dtp_table->usDefaultTargetOperatingTemp;
+
+		hwmgr->dyn_state.cac_dtp_table->usOperatingTempStep =
+			       table_info->cac_dtp_table->usOperatingTempStep;
+
+		hwmgr->dyn_state.cac_dtp_table->usTargetOperatingTemp =
+			       table_info->cac_dtp_table->usTargetOperatingTemp;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+						PHM_PlatformCaps_ODFuzzyFanControlSupport);
+	}
+
+	return 0;
+}
+
+/**
+ * Change virtual leakage voltage to actual value.
+ *
+ * @param     hwmgr  the address of the powerplay hardware manager.
+ * @param     pointer to changing voltage
+ * @param     pointer to leakage table
+ */
+static void smu7_patch_ppt_v0_with_vdd_leakage(struct pp_hwmgr *hwmgr,
+		uint32_t *voltage, struct smu7_leakage_voltage *leakage_table)
+{
+	uint32_t index;
+
+	/* search for leakage voltage ID 0xff01 ~ 0xff08 */
+	for (index = 0; index < leakage_table->count; index++) {
+		/* if this voltage matches a leakage voltage ID */
+		/* patch with actual leakage voltage */
+		if (leakage_table->leakage_id[index] == *voltage) {
+			*voltage = leakage_table->actual_voltage[index];
+			break;
+		}
+	}
+
+	if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
+		printk(KERN_ERR "Voltage value looks like a Leakage ID but it's not patched \n");
+}
+
+
+static int smu7_patch_vddc(struct pp_hwmgr *hwmgr,
+			      struct phm_clock_voltage_dependency_table *tab)
+{
+	uint16_t i;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (tab)
+		for (i = 0; i < tab->count; i++)
+			smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &tab->entries[i].v,
+						&data->vddc_leakage);
+
+	return 0;
+}
+
+static int smu7_patch_vddci(struct pp_hwmgr *hwmgr,
+			       struct phm_clock_voltage_dependency_table *tab)
+{
+	uint16_t i;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (tab)
+		for (i = 0; i < tab->count; i++)
+			smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &tab->entries[i].v,
+							&data->vddci_leakage);
+
+	return 0;
+}
+
+static int smu7_patch_vce_vddc(struct pp_hwmgr *hwmgr,
+				  struct phm_vce_clock_voltage_dependency_table *tab)
+{
+	uint16_t i;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (tab)
+		for (i = 0; i < tab->count; i++)
+			smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &tab->entries[i].v,
+							&data->vddc_leakage);
+
+	return 0;
+}
+
+
+static int smu7_patch_uvd_vddc(struct pp_hwmgr *hwmgr,
+				  struct phm_uvd_clock_voltage_dependency_table *tab)
+{
+	uint16_t i;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (tab)
+		for (i = 0; i < tab->count; i++)
+			smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &tab->entries[i].v,
+							&data->vddc_leakage);
+
+	return 0;
+}
+
+static int smu7_patch_vddc_shed_limit(struct pp_hwmgr *hwmgr,
+					 struct phm_phase_shedding_limits_table *tab)
+{
+	uint16_t i;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (tab)
+		for (i = 0; i < tab->count; i++)
+			smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &tab->entries[i].Voltage,
+							&data->vddc_leakage);
+
+	return 0;
+}
+
+static int smu7_patch_samu_vddc(struct pp_hwmgr *hwmgr,
+				   struct phm_samu_clock_voltage_dependency_table *tab)
+{
+	uint16_t i;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (tab)
+		for (i = 0; i < tab->count; i++)
+			smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &tab->entries[i].v,
+							&data->vddc_leakage);
+
+	return 0;
+}
+
+static int smu7_patch_acp_vddc(struct pp_hwmgr *hwmgr,
+				  struct phm_acp_clock_voltage_dependency_table *tab)
+{
+	uint16_t i;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (tab)
+		for (i = 0; i < tab->count; i++)
+			smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &tab->entries[i].v,
+					&data->vddc_leakage);
+
+	return 0;
+}
+
+static int smu7_patch_limits_vddc(struct pp_hwmgr *hwmgr,
+				     struct phm_clock_and_voltage_limits *tab)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (tab) {
+		smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, (uint32_t *)&tab->vddc,
+							&data->vddc_leakage);
+		smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, (uint32_t *)&tab->vddci,
+							&data->vddci_leakage);
+	}
+
+	return 0;
+}
+
+static int smu7_patch_cac_vddc(struct pp_hwmgr *hwmgr, struct phm_cac_leakage_table *tab)
+{
+	uint32_t i;
+	uint32_t vddc;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (tab) {
+		for (i = 0; i < tab->count; i++) {
+			vddc = (uint32_t)(tab->entries[i].Vddc);
+			smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &vddc, &data->vddc_leakage);
+			tab->entries[i].Vddc = (uint16_t)vddc;
+		}
+	}
+
+	return 0;
+}
+
+static int smu7_patch_dependency_tables_with_leakage(struct pp_hwmgr *hwmgr)
+{
+	int tmp;
+
+	tmp = smu7_patch_vddc(hwmgr, hwmgr->dyn_state.vddc_dependency_on_sclk);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_vddc(hwmgr, hwmgr->dyn_state.vddc_dependency_on_mclk);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_vddc(hwmgr, hwmgr->dyn_state.vddc_dep_on_dal_pwrl);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_vddci(hwmgr, hwmgr->dyn_state.vddci_dependency_on_mclk);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_vce_vddc(hwmgr, hwmgr->dyn_state.vce_clock_voltage_dependency_table);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_uvd_vddc(hwmgr, hwmgr->dyn_state.uvd_clock_voltage_dependency_table);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_samu_vddc(hwmgr, hwmgr->dyn_state.samu_clock_voltage_dependency_table);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_acp_vddc(hwmgr, hwmgr->dyn_state.acp_clock_voltage_dependency_table);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_vddc_shed_limit(hwmgr, hwmgr->dyn_state.vddc_phase_shed_limits_table);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_limits_vddc(hwmgr, &hwmgr->dyn_state.max_clock_voltage_on_ac);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_limits_vddc(hwmgr, &hwmgr->dyn_state.max_clock_voltage_on_dc);
+	if (tmp)
+		return -EINVAL;
+
+	tmp = smu7_patch_cac_vddc(hwmgr, hwmgr->dyn_state.cac_leakage_table);
+	if (tmp)
+		return -EINVAL;
+
+	return 0;
+}
+
+
+static int smu7_set_private_data_based_on_pptable_v0(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	struct phm_clock_voltage_dependency_table *allowed_sclk_vddc_table = hwmgr->dyn_state.vddc_dependency_on_sclk;
+	struct phm_clock_voltage_dependency_table *allowed_mclk_vddc_table = hwmgr->dyn_state.vddc_dependency_on_mclk;
+	struct phm_clock_voltage_dependency_table *allowed_mclk_vddci_table = hwmgr->dyn_state.vddci_dependency_on_mclk;
+
+	PP_ASSERT_WITH_CODE(allowed_sclk_vddc_table != NULL,
+		"VDDC dependency on SCLK table is missing. This table is mandatory\n", return -EINVAL);
+	PP_ASSERT_WITH_CODE(allowed_sclk_vddc_table->count >= 1,
+		"VDDC dependency on SCLK table has to have is missing. This table is mandatory\n", return -EINVAL);
+
+	PP_ASSERT_WITH_CODE(allowed_mclk_vddc_table != NULL,
+		"VDDC dependency on MCLK table is missing. This table is mandatory\n", return -EINVAL);
+	PP_ASSERT_WITH_CODE(allowed_mclk_vddc_table->count >= 1,
+		"VDD dependency on MCLK table has to have is missing. This table is mandatory\n", return -EINVAL);
+
+	data->min_vddc_in_pptable = (uint16_t)allowed_sclk_vddc_table->entries[0].v;
+	data->max_vddc_in_pptable = (uint16_t)allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v;
+
+	hwmgr->dyn_state.max_clock_voltage_on_ac.sclk =
+		allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.mclk =
+		allowed_mclk_vddc_table->entries[allowed_mclk_vddc_table->count - 1].clk;
+	hwmgr->dyn_state.max_clock_voltage_on_ac.vddc =
+		allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v;
+
+	if (allowed_mclk_vddci_table != NULL && allowed_mclk_vddci_table->count >= 1) {
+		data->min_vddci_in_pptable = (uint16_t)allowed_mclk_vddci_table->entries[0].v;
+		data->max_vddci_in_pptable = (uint16_t)allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v;
+	}
+
+	if (hwmgr->dyn_state.vddci_dependency_on_mclk != NULL && hwmgr->dyn_state.vddci_dependency_on_mclk->count > 1)
+		hwmgr->dyn_state.max_clock_voltage_on_ac.vddci = hwmgr->dyn_state.vddci_dependency_on_mclk->entries[hwmgr->dyn_state.vddci_dependency_on_mclk->count - 1].v;
+
+	return 0;
+}
+
+int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data;
+	int result;
+
+	data = kzalloc(sizeof(struct smu7_hwmgr), GFP_KERNEL);
+	if (data == NULL)
+		return -ENOMEM;
+
+	hwmgr->backend = data;
+
+	smu7_patch_voltage_workaround(hwmgr);
+	smu7_init_dpm_defaults(hwmgr);
+
+	/* Get leakage voltage based on leakage ID. */
+	result = smu7_get_evv_voltages(hwmgr);
+
+	if (result) {
+		printk("Get EVV Voltage Failed.  Abort Driver loading!\n");
+		return -EINVAL;
+	}
+
+	if (hwmgr->pp_table_version == PP_TABLE_V1) {
+		smu7_complete_dependency_tables(hwmgr);
+		smu7_set_private_data_based_on_pptable_v1(hwmgr);
+	} else if (hwmgr->pp_table_version == PP_TABLE_V0) {
+		smu7_patch_dependency_tables_with_leakage(hwmgr);
+		smu7_set_private_data_based_on_pptable_v0(hwmgr);
+	}
+
+	/* Initalize Dynamic State Adjustment Rule Settings */
+	result = phm_initializa_dynamic_state_adjustment_rule_settings(hwmgr);
+
+	if (0 == result) {
+		struct cgs_system_info sys_info = {0};
+
+		data->is_tlu_enabled = false;
+
+		hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =
+							SMU7_MAX_HARDWARE_POWERLEVELS;
+		hwmgr->platform_descriptor.hardwarePerformanceLevels = 2;
+		hwmgr->platform_descriptor.minimumClocksReductionPercentage = 50;
+
+		sys_info.size = sizeof(struct cgs_system_info);
+		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO;
+		result = cgs_query_system_info(hwmgr->device, &sys_info);
+		if (result)
+			data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK;
+		else
+			data->pcie_gen_cap = (uint32_t)sys_info.value;
+		if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
+			data->pcie_spc_cap = 20;
+		sys_info.size = sizeof(struct cgs_system_info);
+		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW;
+		result = cgs_query_system_info(hwmgr->device, &sys_info);
+		if (result)
+			data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK;
+		else
+			data->pcie_lane_cap = (uint32_t)sys_info.value;
+
+		hwmgr->platform_descriptor.vbiosInterruptId = 0x20000400; /* IRQ_SOURCE1_SW_INT */
+/* The true clock step depends on the frequency, typically 4.5 or 9 MHz. Here we use 5. */
+		hwmgr->platform_descriptor.clockStep.engineClock = 500;
+		hwmgr->platform_descriptor.clockStep.memoryClock = 500;
+		smu7_thermal_parameter_init(hwmgr);
+	} else {
+		/* Ignore return value in here, we are cleaning up a mess. */
+		phm_hwmgr_backend_fini(hwmgr);
+	}
+
+	return 0;
+}
+
+static int smu7_force_dpm_highest(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t level, tmp;
+
+	if (!data->pcie_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask) {
+			level = 0;
+			tmp = data->dpm_level_enable_mask.pcie_dpm_enable_mask;
+			while (tmp >>= 1)
+				level++;
+
+			if (level)
+				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_PCIeDPM_ForceLevel, level);
+		}
+	}
+
+	if (!data->sclk_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
+			level = 0;
+			tmp = data->dpm_level_enable_mask.sclk_dpm_enable_mask;
+			while (tmp >>= 1)
+				level++;
+
+			if (level)
+				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_SCLKDPM_SetEnabledMask,
+						(1 << level));
+		}
+	}
+
+	if (!data->mclk_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
+			level = 0;
+			tmp = data->dpm_level_enable_mask.mclk_dpm_enable_mask;
+			while (tmp >>= 1)
+				level++;
+
+			if (level)
+				smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+						PPSMC_MSG_MCLKDPM_SetEnabledMask,
+						(1 << level));
+		}
+	}
+
+	return 0;
+}
+
+static int smu7_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (hwmgr->pp_table_version == PP_TABLE_V1)
+		phm_apply_dal_min_voltage_request(hwmgr);
+/* TO DO  for v0 iceland and Ci*/
+
+	if (!data->sclk_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask)
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_SCLKDPM_SetEnabledMask,
+					data->dpm_level_enable_mask.sclk_dpm_enable_mask);
+	}
+
+	if (!data->mclk_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask)
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_MCLKDPM_SetEnabledMask,
+					data->dpm_level_enable_mask.mclk_dpm_enable_mask);
+	}
+
+	return 0;
+}
+
+static int smu7_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (!smum_is_dpm_running(hwmgr))
+		return -EINVAL;
+
+	if (!data->pcie_dpm_key_disabled) {
+		smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_PCIeDPM_UnForceLevel);
+	}
+
+	return smu7_upload_dpm_level_enable_mask(hwmgr);
+}
+
+static int smu7_force_dpm_lowest(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data =
+			(struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t level;
+
+	if (!data->sclk_dpm_key_disabled)
+		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
+			level = phm_get_lowest_enabled_level(hwmgr,
+							      data->dpm_level_enable_mask.sclk_dpm_enable_mask);
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+							    PPSMC_MSG_SCLKDPM_SetEnabledMask,
+							    (1 << level));
+
+	}
+
+	if (!data->mclk_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
+			level = phm_get_lowest_enabled_level(hwmgr,
+							      data->dpm_level_enable_mask.mclk_dpm_enable_mask);
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+							    PPSMC_MSG_MCLKDPM_SetEnabledMask,
+							    (1 << level));
+		}
+	}
+
+	if (!data->pcie_dpm_key_disabled) {
+		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask) {
+			level = phm_get_lowest_enabled_level(hwmgr,
+							      data->dpm_level_enable_mask.pcie_dpm_enable_mask);
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+							    PPSMC_MSG_PCIeDPM_ForceLevel,
+							    (level));
+		}
+	}
+
+	return 0;
+
+}
+static int smu7_force_dpm_level(struct pp_hwmgr *hwmgr,
+				enum amd_dpm_forced_level level)
+{
+	int ret = 0;
+
+	switch (level) {
+	case AMD_DPM_FORCED_LEVEL_HIGH:
+		ret = smu7_force_dpm_highest(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	case AMD_DPM_FORCED_LEVEL_LOW:
+		ret = smu7_force_dpm_lowest(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	case AMD_DPM_FORCED_LEVEL_AUTO:
+		ret = smu7_unforce_dpm_levels(hwmgr);
+		if (ret)
+			return ret;
+		break;
+	default:
+		break;
+	}
+
+	hwmgr->dpm_level = level;
+
+	return ret;
+}
+
+static int smu7_get_power_state_size(struct pp_hwmgr *hwmgr)
+{
+	return sizeof(struct smu7_power_state);
+}
+
+
+static int smu7_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
+				struct pp_power_state *request_ps,
+			const struct pp_power_state *current_ps)
+{
+
+	struct smu7_power_state *smu7_ps =
+				cast_phw_smu7_power_state(&request_ps->hardware);
+	uint32_t sclk;
+	uint32_t mclk;
+	struct PP_Clocks minimum_clocks = {0};
+	bool disable_mclk_switching;
+	bool disable_mclk_switching_for_frame_lock;
+	struct cgs_display_info info = {0};
+	const struct phm_clock_and_voltage_limits *max_limits;
+	uint32_t i;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int32_t count;
+	int32_t stable_pstate_sclk = 0, stable_pstate_mclk = 0;
+
+	data->battery_state = (PP_StateUILabel_Battery ==
+			request_ps->classification.ui_label);
+
+	PP_ASSERT_WITH_CODE(smu7_ps->performance_level_count == 2,
+				 "VI should always have 2 performance levels",
+				);
+
+	max_limits = (PP_PowerSource_AC == hwmgr->power_source) ?
+			&(hwmgr->dyn_state.max_clock_voltage_on_ac) :
+			&(hwmgr->dyn_state.max_clock_voltage_on_dc);
+
+	/* Cap clock DPM tables at DC MAX if it is in DC. */
+	if (PP_PowerSource_DC == hwmgr->power_source) {
+		for (i = 0; i < smu7_ps->performance_level_count; i++) {
+			if (smu7_ps->performance_levels[i].memory_clock > max_limits->mclk)
+				smu7_ps->performance_levels[i].memory_clock = max_limits->mclk;
+			if (smu7_ps->performance_levels[i].engine_clock > max_limits->sclk)
+				smu7_ps->performance_levels[i].engine_clock = max_limits->sclk;
+		}
+	}
+
+	smu7_ps->vce_clks.evclk = hwmgr->vce_arbiter.evclk;
+	smu7_ps->vce_clks.ecclk = hwmgr->vce_arbiter.ecclk;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	/*TO DO result = PHM_CheckVBlankTime(hwmgr, &vblankTooShort);*/
+
+	minimum_clocks.engineClock = hwmgr->display_config.min_core_set_clock;
+	minimum_clocks.memoryClock = hwmgr->display_config.min_mem_set_clock;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState)) {
+		max_limits = &(hwmgr->dyn_state.max_clock_voltage_on_ac);
+		stable_pstate_sclk = (max_limits->sclk * 75) / 100;
+
+		for (count = table_info->vdd_dep_on_sclk->count - 1;
+				count >= 0; count--) {
+			if (stable_pstate_sclk >=
+					table_info->vdd_dep_on_sclk->entries[count].clk) {
+				stable_pstate_sclk =
+						table_info->vdd_dep_on_sclk->entries[count].clk;
+				break;
+			}
+		}
+
+		if (count < 0)
+			stable_pstate_sclk = table_info->vdd_dep_on_sclk->entries[0].clk;
+
+		stable_pstate_mclk = max_limits->mclk;
+
+		minimum_clocks.engineClock = stable_pstate_sclk;
+		minimum_clocks.memoryClock = stable_pstate_mclk;
+	}
+
+	if (minimum_clocks.engineClock < hwmgr->gfx_arbiter.sclk)
+		minimum_clocks.engineClock = hwmgr->gfx_arbiter.sclk;
+
+	if (minimum_clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
+		minimum_clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
+
+	smu7_ps->sclk_threshold = hwmgr->gfx_arbiter.sclk_threshold;
+
+	if (0 != hwmgr->gfx_arbiter.sclk_over_drive) {
+		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.sclk_over_drive <=
+				hwmgr->platform_descriptor.overdriveLimit.engineClock),
+				"Overdrive sclk exceeds limit",
+				hwmgr->gfx_arbiter.sclk_over_drive =
+						hwmgr->platform_descriptor.overdriveLimit.engineClock);
+
+		if (hwmgr->gfx_arbiter.sclk_over_drive >= hwmgr->gfx_arbiter.sclk)
+			smu7_ps->performance_levels[1].engine_clock =
+					hwmgr->gfx_arbiter.sclk_over_drive;
+	}
+
+	if (0 != hwmgr->gfx_arbiter.mclk_over_drive) {
+		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.mclk_over_drive <=
+				hwmgr->platform_descriptor.overdriveLimit.memoryClock),
+				"Overdrive mclk exceeds limit",
+				hwmgr->gfx_arbiter.mclk_over_drive =
+						hwmgr->platform_descriptor.overdriveLimit.memoryClock);
+
+		if (hwmgr->gfx_arbiter.mclk_over_drive >= hwmgr->gfx_arbiter.mclk)
+			smu7_ps->performance_levels[1].memory_clock =
+					hwmgr->gfx_arbiter.mclk_over_drive;
+	}
+
+	disable_mclk_switching_for_frame_lock = phm_cap_enabled(
+				    hwmgr->platform_descriptor.platformCaps,
+				    PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
+
+
+	disable_mclk_switching = (1 < info.display_count) ||
+				    disable_mclk_switching_for_frame_lock;
+
+	sclk = smu7_ps->performance_levels[0].engine_clock;
+	mclk = smu7_ps->performance_levels[0].memory_clock;
+
+	if (disable_mclk_switching)
+		mclk = smu7_ps->performance_levels
+		[smu7_ps->performance_level_count - 1].memory_clock;
+
+	if (sclk < minimum_clocks.engineClock)
+		sclk = (minimum_clocks.engineClock > max_limits->sclk) ?
+				max_limits->sclk : minimum_clocks.engineClock;
+
+	if (mclk < minimum_clocks.memoryClock)
+		mclk = (minimum_clocks.memoryClock > max_limits->mclk) ?
+				max_limits->mclk : minimum_clocks.memoryClock;
+
+	smu7_ps->performance_levels[0].engine_clock = sclk;
+	smu7_ps->performance_levels[0].memory_clock = mclk;
+
+	smu7_ps->performance_levels[1].engine_clock =
+		(smu7_ps->performance_levels[1].engine_clock >=
+				smu7_ps->performance_levels[0].engine_clock) ?
+						smu7_ps->performance_levels[1].engine_clock :
+						smu7_ps->performance_levels[0].engine_clock;
+
+	if (disable_mclk_switching) {
+		if (mclk < smu7_ps->performance_levels[1].memory_clock)
+			mclk = smu7_ps->performance_levels[1].memory_clock;
+
+		smu7_ps->performance_levels[0].memory_clock = mclk;
+		smu7_ps->performance_levels[1].memory_clock = mclk;
+	} else {
+		if (smu7_ps->performance_levels[1].memory_clock <
+				smu7_ps->performance_levels[0].memory_clock)
+			smu7_ps->performance_levels[1].memory_clock =
+					smu7_ps->performance_levels[0].memory_clock;
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState)) {
+		for (i = 0; i < smu7_ps->performance_level_count; i++) {
+			smu7_ps->performance_levels[i].engine_clock = stable_pstate_sclk;
+			smu7_ps->performance_levels[i].memory_clock = stable_pstate_mclk;
+			smu7_ps->performance_levels[i].pcie_gen = data->pcie_gen_performance.max;
+			smu7_ps->performance_levels[i].pcie_lane = data->pcie_gen_performance.max;
+		}
+	}
+	return 0;
+}
+
+
+static int smu7_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
+{
+	struct pp_power_state  *ps;
+	struct smu7_power_state  *smu7_ps;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	ps = hwmgr->request_ps;
+
+	if (ps == NULL)
+		return -EINVAL;
+
+	smu7_ps = cast_phw_smu7_power_state(&ps->hardware);
+
+	if (low)
+		return smu7_ps->performance_levels[0].memory_clock;
+	else
+		return smu7_ps->performance_levels
+				[smu7_ps->performance_level_count-1].memory_clock;
+}
+
+static int smu7_dpm_get_sclk(struct pp_hwmgr *hwmgr, bool low)
+{
+	struct pp_power_state  *ps;
+	struct smu7_power_state  *smu7_ps;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	ps = hwmgr->request_ps;
+
+	if (ps == NULL)
+		return -EINVAL;
+
+	smu7_ps = cast_phw_smu7_power_state(&ps->hardware);
+
+	if (low)
+		return smu7_ps->performance_levels[0].engine_clock;
+	else
+		return smu7_ps->performance_levels
+				[smu7_ps->performance_level_count-1].engine_clock;
+}
+
+static int smu7_dpm_patch_boot_state(struct pp_hwmgr *hwmgr,
+					struct pp_hw_power_state *hw_ps)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_power_state *ps = (struct smu7_power_state *)hw_ps;
+	ATOM_FIRMWARE_INFO_V2_2 *fw_info;
+	uint16_t size;
+	uint8_t frev, crev;
+	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
+
+	/* First retrieve the Boot clocks and VDDC from the firmware info table.
+	 * We assume here that fw_info is unchanged if this call fails.
+	 */
+	fw_info = (ATOM_FIRMWARE_INFO_V2_2 *)cgs_atom_get_data_table(
+			hwmgr->device, index,
+			&size, &frev, &crev);
+	if (!fw_info)
+		/* During a test, there is no firmware info table. */
+		return 0;
+
+	/* Patch the state. */
+	data->vbios_boot_state.sclk_bootup_value =
+			le32_to_cpu(fw_info->ulDefaultEngineClock);
+	data->vbios_boot_state.mclk_bootup_value =
+			le32_to_cpu(fw_info->ulDefaultMemoryClock);
+	data->vbios_boot_state.mvdd_bootup_value =
+			le16_to_cpu(fw_info->usBootUpMVDDCVoltage);
+	data->vbios_boot_state.vddc_bootup_value =
+			le16_to_cpu(fw_info->usBootUpVDDCVoltage);
+	data->vbios_boot_state.vddci_bootup_value =
+			le16_to_cpu(fw_info->usBootUpVDDCIVoltage);
+	data->vbios_boot_state.pcie_gen_bootup_value =
+			smu7_get_current_pcie_speed(hwmgr);
+
+	data->vbios_boot_state.pcie_lane_bootup_value =
+			(uint16_t)smu7_get_current_pcie_lane_number(hwmgr);
+
+	/* set boot power state */
+	ps->performance_levels[0].memory_clock = data->vbios_boot_state.mclk_bootup_value;
+	ps->performance_levels[0].engine_clock = data->vbios_boot_state.sclk_bootup_value;
+	ps->performance_levels[0].pcie_gen = data->vbios_boot_state.pcie_gen_bootup_value;
+	ps->performance_levels[0].pcie_lane = data->vbios_boot_state.pcie_lane_bootup_value;
+
+	return 0;
+}
+
+static int smu7_get_number_of_powerplay_table_entries(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	unsigned long ret = 0;
+
+	if (hwmgr->pp_table_version == PP_TABLE_V0) {
+		result = pp_tables_get_num_of_entries(hwmgr, &ret);
+		return result ? 0 : ret;
+	} else if (hwmgr->pp_table_version == PP_TABLE_V1) {
+		result = get_number_of_powerplay_table_entries_v1_0(hwmgr);
+		return result;
+	}
+	return 0;
+}
+
+static int smu7_get_pp_table_entry_callback_func_v1(struct pp_hwmgr *hwmgr,
+		void *state, struct pp_power_state *power_state,
+		void *pp_table, uint32_t classification_flag)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_power_state  *smu7_power_state =
+			(struct smu7_power_state *)(&(power_state->hardware));
+	struct smu7_performance_level *performance_level;
+	ATOM_Tonga_State *state_entry = (ATOM_Tonga_State *)state;
+	ATOM_Tonga_POWERPLAYTABLE *powerplay_table =
+			(ATOM_Tonga_POWERPLAYTABLE *)pp_table;
+	PPTable_Generic_SubTable_Header *sclk_dep_table =
+			(PPTable_Generic_SubTable_Header *)
+			(((unsigned long)powerplay_table) +
+				le16_to_cpu(powerplay_table->usSclkDependencyTableOffset));
+
+	ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table =
+			(ATOM_Tonga_MCLK_Dependency_Table *)
+			(((unsigned long)powerplay_table) +
+				le16_to_cpu(powerplay_table->usMclkDependencyTableOffset));
+
+	/* The following fields are not initialized here: id orderedList allStatesList */
+	power_state->classification.ui_label =
+			(le16_to_cpu(state_entry->usClassification) &
+			ATOM_PPLIB_CLASSIFICATION_UI_MASK) >>
+			ATOM_PPLIB_CLASSIFICATION_UI_SHIFT;
+	power_state->classification.flags = classification_flag;
+	/* NOTE: There is a classification2 flag in BIOS that is not being used right now */
+
+	power_state->classification.temporary_state = false;
+	power_state->classification.to_be_deleted = false;
+
+	power_state->validation.disallowOnDC =
+			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) &
+					ATOM_Tonga_DISALLOW_ON_DC));
+
+	power_state->pcie.lanes = 0;
+
+	power_state->display.disableFrameModulation = false;
+	power_state->display.limitRefreshrate = false;
+	power_state->display.enableVariBright =
+			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) &
+					ATOM_Tonga_ENABLE_VARIBRIGHT));
+
+	power_state->validation.supportedPowerLevels = 0;
+	power_state->uvd_clocks.VCLK = 0;
+	power_state->uvd_clocks.DCLK = 0;
+	power_state->temperatures.min = 0;
+	power_state->temperatures.max = 0;
+
+	performance_level = &(smu7_power_state->performance_levels
+			[smu7_power_state->performance_level_count++]);
+
+	PP_ASSERT_WITH_CODE(
+			(smu7_power_state->performance_level_count < smum_get_mac_definition(hwmgr->smumgr, SMU_MAX_LEVELS_GRAPHICS)),
+			"Performance levels exceeds SMC limit!",
+			return -EINVAL);
+
+	PP_ASSERT_WITH_CODE(
+			(smu7_power_state->performance_level_count <=
+					hwmgr->platform_descriptor.hardwareActivityPerformanceLevels),
+			"Performance levels exceeds Driver limit!",
+			return -EINVAL);
+
+	/* Performance levels are arranged from low to high. */
+	performance_level->memory_clock = mclk_dep_table->entries
+			[state_entry->ucMemoryClockIndexLow].ulMclk;
+	if (sclk_dep_table->ucRevId == 0)
+		performance_level->engine_clock = ((ATOM_Tonga_SCLK_Dependency_Table *)sclk_dep_table)->entries
+			[state_entry->ucEngineClockIndexLow].ulSclk;
+	else if (sclk_dep_table->ucRevId == 1)
+		performance_level->engine_clock = ((ATOM_Polaris_SCLK_Dependency_Table *)sclk_dep_table)->entries
+			[state_entry->ucEngineClockIndexLow].ulSclk;
+	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap,
+			state_entry->ucPCIEGenLow);
+	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap,
+			state_entry->ucPCIELaneHigh);
+
+	performance_level = &(smu7_power_state->performance_levels
+			[smu7_power_state->performance_level_count++]);
+	performance_level->memory_clock = mclk_dep_table->entries
+			[state_entry->ucMemoryClockIndexHigh].ulMclk;
+
+	if (sclk_dep_table->ucRevId == 0)
+		performance_level->engine_clock = ((ATOM_Tonga_SCLK_Dependency_Table *)sclk_dep_table)->entries
+			[state_entry->ucEngineClockIndexHigh].ulSclk;
+	else if (sclk_dep_table->ucRevId == 1)
+		performance_level->engine_clock = ((ATOM_Polaris_SCLK_Dependency_Table *)sclk_dep_table)->entries
+			[state_entry->ucEngineClockIndexHigh].ulSclk;
+
+	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap,
+			state_entry->ucPCIEGenHigh);
+	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap,
+			state_entry->ucPCIELaneHigh);
+
+	return 0;
+}
+
+static int smu7_get_pp_table_entry_v1(struct pp_hwmgr *hwmgr,
+		unsigned long entry_index, struct pp_power_state *state)
+{
+	int result;
+	struct smu7_power_state *ps;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
+			table_info->vdd_dep_on_mclk;
+
+	state->hardware.magic = PHM_VIslands_Magic;
+
+	ps = (struct smu7_power_state *)(&state->hardware);
+
+	result = get_powerplay_table_entry_v1_0(hwmgr, entry_index, state,
+			smu7_get_pp_table_entry_callback_func_v1);
+
+	/* This is the earliest time we have all the dependency table and the VBIOS boot state
+	 * as PP_Tables_GetPowerPlayTableEntry retrieves the VBIOS boot state
+	 * if there is only one VDDCI/MCLK level, check if it's the same as VBIOS boot state
+	 */
+	if (dep_mclk_table != NULL && dep_mclk_table->count == 1) {
+		if (dep_mclk_table->entries[0].clk !=
+				data->vbios_boot_state.mclk_bootup_value)
+			printk(KERN_ERR "Single MCLK entry VDDCI/MCLK dependency table "
+					"does not match VBIOS boot MCLK level");
+		if (dep_mclk_table->entries[0].vddci !=
+				data->vbios_boot_state.vddci_bootup_value)
+			printk(KERN_ERR "Single VDDCI entry VDDCI/MCLK dependency table "
+					"does not match VBIOS boot VDDCI level");
+	}
+
+	/* set DC compatible flag if this state supports DC */
+	if (!state->validation.disallowOnDC)
+		ps->dc_compatible = true;
+
+	if (state->classification.flags & PP_StateClassificationFlag_ACPI)
+		data->acpi_pcie_gen = ps->performance_levels[0].pcie_gen;
+
+	ps->uvd_clks.vclk = state->uvd_clocks.VCLK;
+	ps->uvd_clks.dclk = state->uvd_clocks.DCLK;
+
+	if (!result) {
+		uint32_t i;
+
+		switch (state->classification.ui_label) {
+		case PP_StateUILabel_Performance:
+			data->use_pcie_performance_levels = true;
+			for (i = 0; i < ps->performance_level_count; i++) {
+				if (data->pcie_gen_performance.max <
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_performance.max =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_gen_performance.min >
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_performance.min =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_lane_performance.max <
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_performance.max =
+							ps->performance_levels[i].pcie_lane;
+				if (data->pcie_lane_performance.min >
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_performance.min =
+							ps->performance_levels[i].pcie_lane;
+			}
+			break;
+		case PP_StateUILabel_Battery:
+			data->use_pcie_power_saving_levels = true;
+
+			for (i = 0; i < ps->performance_level_count; i++) {
+				if (data->pcie_gen_power_saving.max <
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_power_saving.max =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_gen_power_saving.min >
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_power_saving.min =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_lane_power_saving.max <
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_power_saving.max =
+							ps->performance_levels[i].pcie_lane;
+
+				if (data->pcie_lane_power_saving.min >
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_power_saving.min =
+							ps->performance_levels[i].pcie_lane;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+	return 0;
+}
+
+static int smu7_get_pp_table_entry_callback_func_v0(struct pp_hwmgr *hwmgr,
+					struct pp_hw_power_state *power_state,
+					unsigned int index, const void *clock_info)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_power_state  *ps = cast_phw_smu7_power_state(power_state);
+	const ATOM_PPLIB_CI_CLOCK_INFO *visland_clk_info = clock_info;
+	struct smu7_performance_level *performance_level;
+	uint32_t engine_clock, memory_clock;
+	uint16_t pcie_gen_from_bios;
+
+	engine_clock = visland_clk_info->ucEngineClockHigh << 16 | visland_clk_info->usEngineClockLow;
+	memory_clock = visland_clk_info->ucMemoryClockHigh << 16 | visland_clk_info->usMemoryClockLow;
+
+	if (!(data->mc_micro_code_feature & DISABLE_MC_LOADMICROCODE) && memory_clock > data->highest_mclk)
+		data->highest_mclk = memory_clock;
+
+	performance_level = &(ps->performance_levels
+			[ps->performance_level_count++]);
+
+	PP_ASSERT_WITH_CODE(
+			(ps->performance_level_count < smum_get_mac_definition(hwmgr->smumgr, SMU_MAX_LEVELS_GRAPHICS)),
+			"Performance levels exceeds SMC limit!",
+			return -EINVAL);
+
+	PP_ASSERT_WITH_CODE(
+			(ps->performance_level_count <=
+					hwmgr->platform_descriptor.hardwareActivityPerformanceLevels),
+			"Performance levels exceeds Driver limit!",
+			return -EINVAL);
+
+	/* Performance levels are arranged from low to high. */
+	performance_level->memory_clock = memory_clock;
+	performance_level->engine_clock = engine_clock;
+
+	pcie_gen_from_bios = visland_clk_info->ucPCIEGen;
+
+	performance_level->pcie_gen = get_pcie_gen_support(data->pcie_gen_cap, pcie_gen_from_bios);
+	performance_level->pcie_lane = get_pcie_lane_support(data->pcie_lane_cap, visland_clk_info->usPCIELane);
+
+	return 0;
+}
+
+static int smu7_get_pp_table_entry_v0(struct pp_hwmgr *hwmgr,
+		unsigned long entry_index, struct pp_power_state *state)
+{
+	int result;
+	struct smu7_power_state *ps;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_clock_voltage_dependency_table *dep_mclk_table =
+			hwmgr->dyn_state.vddci_dependency_on_mclk;
+
+	memset(&state->hardware, 0x00, sizeof(struct pp_hw_power_state));
+
+	state->hardware.magic = PHM_VIslands_Magic;
+
+	ps = (struct smu7_power_state *)(&state->hardware);
+
+	result = pp_tables_get_entry(hwmgr, entry_index, state,
+			smu7_get_pp_table_entry_callback_func_v0);
+
+	/*
+	 * This is the earliest time we have all the dependency table
+	 * and the VBIOS boot state as
+	 * PP_Tables_GetPowerPlayTableEntry retrieves the VBIOS boot
+	 * state if there is only one VDDCI/MCLK level, check if it's
+	 * the same as VBIOS boot state
+	 */
+	if (dep_mclk_table != NULL && dep_mclk_table->count == 1) {
+		if (dep_mclk_table->entries[0].clk !=
+				data->vbios_boot_state.mclk_bootup_value)
+			printk(KERN_ERR "Single MCLK entry VDDCI/MCLK dependency table "
+					"does not match VBIOS boot MCLK level");
+		if (dep_mclk_table->entries[0].v !=
+				data->vbios_boot_state.vddci_bootup_value)
+			printk(KERN_ERR "Single VDDCI entry VDDCI/MCLK dependency table "
+					"does not match VBIOS boot VDDCI level");
+	}
+
+	/* set DC compatible flag if this state supports DC */
+	if (!state->validation.disallowOnDC)
+		ps->dc_compatible = true;
+
+	if (state->classification.flags & PP_StateClassificationFlag_ACPI)
+		data->acpi_pcie_gen = ps->performance_levels[0].pcie_gen;
+
+	ps->uvd_clks.vclk = state->uvd_clocks.VCLK;
+	ps->uvd_clks.dclk = state->uvd_clocks.DCLK;
+
+	if (!result) {
+		uint32_t i;
+
+		switch (state->classification.ui_label) {
+		case PP_StateUILabel_Performance:
+			data->use_pcie_performance_levels = true;
+
+			for (i = 0; i < ps->performance_level_count; i++) {
+				if (data->pcie_gen_performance.max <
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_performance.max =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_gen_performance.min >
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_performance.min =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_lane_performance.max <
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_performance.max =
+							ps->performance_levels[i].pcie_lane;
+
+				if (data->pcie_lane_performance.min >
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_performance.min =
+							ps->performance_levels[i].pcie_lane;
+			}
+			break;
+		case PP_StateUILabel_Battery:
+			data->use_pcie_power_saving_levels = true;
+
+			for (i = 0; i < ps->performance_level_count; i++) {
+				if (data->pcie_gen_power_saving.max <
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_power_saving.max =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_gen_power_saving.min >
+						ps->performance_levels[i].pcie_gen)
+					data->pcie_gen_power_saving.min =
+							ps->performance_levels[i].pcie_gen;
+
+				if (data->pcie_lane_power_saving.max <
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_power_saving.max =
+							ps->performance_levels[i].pcie_lane;
+
+				if (data->pcie_lane_power_saving.min >
+						ps->performance_levels[i].pcie_lane)
+					data->pcie_lane_power_saving.min =
+							ps->performance_levels[i].pcie_lane;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+	return 0;
+}
+
+static int smu7_get_pp_table_entry(struct pp_hwmgr *hwmgr,
+		unsigned long entry_index, struct pp_power_state *state)
+{
+	if (hwmgr->pp_table_version == PP_TABLE_V0)
+		return smu7_get_pp_table_entry_v0(hwmgr, entry_index, state);
+	else if (hwmgr->pp_table_version == PP_TABLE_V1)
+		return smu7_get_pp_table_entry_v1(hwmgr, entry_index, state);
+
+	return 0;
+}
+
+static int smu7_read_sensor(struct pp_hwmgr *hwmgr, int idx, int32_t *value)
+{
+	uint32_t sclk, mclk, activity_percent;
+	uint32_t offset;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	switch (idx) {
+	case AMDGPU_PP_SENSOR_GFX_SCLK:
+		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetSclkFrequency);
+		sclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+		*value = sclk;
+		return 0;
+	case AMDGPU_PP_SENSOR_GFX_MCLK:
+		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetMclkFrequency);
+		mclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+		*value = mclk;
+		return 0;
+	case AMDGPU_PP_SENSOR_GPU_LOAD:
+		offset = data->soft_regs_start + smum_get_offsetof(hwmgr->smumgr,
+								SMU_SoftRegisters,
+								AverageGraphicsActivity);
+
+		activity_percent = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, offset);
+		activity_percent += 0x80;
+		activity_percent >>= 8;
+		*value = activity_percent > 100 ? 100 : activity_percent;
+		return 0;
+	case AMDGPU_PP_SENSOR_GPU_TEMP:
+		*value = smu7_thermal_get_temperature(hwmgr);
+		return 0;
+	case AMDGPU_PP_SENSOR_UVD_POWER:
+		*value = data->uvd_power_gated ? 0 : 1;
+		return 0;
+	case AMDGPU_PP_SENSOR_VCE_POWER:
+		*value = data->vce_power_gated ? 0 : 1;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int smu7_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	const struct smu7_power_state *smu7_ps =
+			cast_const_phw_smu7_power_state(states->pnew_state);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
+	uint32_t sclk = smu7_ps->performance_levels
+			[smu7_ps->performance_level_count - 1].engine_clock;
+	struct smu7_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
+	uint32_t mclk = smu7_ps->performance_levels
+			[smu7_ps->performance_level_count - 1].memory_clock;
+	struct PP_Clocks min_clocks = {0};
+	uint32_t i;
+	struct cgs_display_info info = {0};
+
+	data->need_update_smu7_dpm_table = 0;
+
+	for (i = 0; i < sclk_table->count; i++) {
+		if (sclk == sclk_table->dpm_levels[i].value)
+			break;
+	}
+
+	if (i >= sclk_table->count)
+		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
+	else {
+	/* TODO: Check SCLK in DAL's minimum clocks
+	 * in case DeepSleep divider update is required.
+	 */
+		if (data->display_timing.min_clock_in_sr != min_clocks.engineClockInSR &&
+			(min_clocks.engineClockInSR >= SMU7_MINIMUM_ENGINE_CLOCK ||
+				data->display_timing.min_clock_in_sr >= SMU7_MINIMUM_ENGINE_CLOCK))
+			data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
+	}
+
+	for (i = 0; i < mclk_table->count; i++) {
+		if (mclk == mclk_table->dpm_levels[i].value)
+			break;
+	}
+
+	if (i >= mclk_table->count)
+		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	if (data->display_timing.num_existing_displays != info.display_count)
+		data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
+
+	return 0;
+}
+
+static uint16_t smu7_get_maximum_link_speed(struct pp_hwmgr *hwmgr,
+		const struct smu7_power_state *smu7_ps)
+{
+	uint32_t i;
+	uint32_t sclk, max_sclk = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+
+	for (i = 0; i < smu7_ps->performance_level_count; i++) {
+		sclk = smu7_ps->performance_levels[i].engine_clock;
+		if (max_sclk < sclk)
+			max_sclk = sclk;
+	}
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++) {
+		if (dpm_table->sclk_table.dpm_levels[i].value == max_sclk)
+			return (uint16_t) ((i >= dpm_table->pcie_speed_table.count) ?
+					dpm_table->pcie_speed_table.dpm_levels
+					[dpm_table->pcie_speed_table.count - 1].value :
+					dpm_table->pcie_speed_table.dpm_levels[i].value);
+	}
+
+	return 0;
+}
+
+static int smu7_request_link_speed_change_before_state_change(
+		struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	const struct smu7_power_state *smu7_nps =
+			cast_const_phw_smu7_power_state(states->pnew_state);
+	const struct smu7_power_state *polaris10_cps =
+			cast_const_phw_smu7_power_state(states->pcurrent_state);
+
+	uint16_t target_link_speed = smu7_get_maximum_link_speed(hwmgr, smu7_nps);
+	uint16_t current_link_speed;
+
+	if (data->force_pcie_gen == PP_PCIEGenInvalid)
+		current_link_speed = smu7_get_maximum_link_speed(hwmgr, polaris10_cps);
+	else
+		current_link_speed = data->force_pcie_gen;
+
+	data->force_pcie_gen = PP_PCIEGenInvalid;
+	data->pspp_notify_required = false;
+
+	if (target_link_speed > current_link_speed) {
+		switch (target_link_speed) {
+		case PP_PCIEGen3:
+			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN3, false))
+				break;
+			data->force_pcie_gen = PP_PCIEGen2;
+			if (current_link_speed == PP_PCIEGen2)
+				break;
+		case PP_PCIEGen2:
+			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN2, false))
+				break;
+		default:
+			data->force_pcie_gen = smu7_get_current_pcie_speed(hwmgr);
+			break;
+		}
+	} else {
+		if (target_link_speed < current_link_speed)
+			data->pspp_notify_required = true;
+	}
+
+	return 0;
+}
+
+static int smu7_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (0 == data->need_update_smu7_dpm_table)
+		return 0;
+
+	if ((0 == data->sclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table &
+			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
+		PP_ASSERT_WITH_CODE(true == smum_is_dpm_running(hwmgr),
+				"Trying to freeze SCLK DPM when DPM is disabled",
+				);
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_SCLKDPM_FreezeLevel),
+				"Failed to freeze SCLK DPM during FreezeSclkMclkDPM Function!",
+				return -EINVAL);
+	}
+
+	if ((0 == data->mclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table &
+		 DPMTABLE_OD_UPDATE_MCLK)) {
+		PP_ASSERT_WITH_CODE(true == smum_is_dpm_running(hwmgr),
+				"Trying to freeze MCLK DPM when DPM is disabled",
+				);
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_MCLKDPM_FreezeLevel),
+				"Failed to freeze MCLK DPM during FreezeSclkMclkDPM Function!",
+				return -EINVAL);
+	}
+
+	return 0;
+}
+
+static int smu7_populate_and_upload_sclk_mclk_dpm_levels(
+		struct pp_hwmgr *hwmgr, const void *input)
+{
+	int result = 0;
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	const struct smu7_power_state *smu7_ps =
+			cast_const_phw_smu7_power_state(states->pnew_state);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t sclk = smu7_ps->performance_levels
+			[smu7_ps->performance_level_count - 1].engine_clock;
+	uint32_t mclk = smu7_ps->performance_levels
+			[smu7_ps->performance_level_count - 1].memory_clock;
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+
+	struct smu7_dpm_table *golden_dpm_table = &data->golden_dpm_table;
+	uint32_t dpm_count, clock_percent;
+	uint32_t i;
+
+	if (0 == data->need_update_smu7_dpm_table)
+		return 0;
+
+	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK) {
+		dpm_table->sclk_table.dpm_levels
+		[dpm_table->sclk_table.count - 1].value = sclk;
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinACSupport) ||
+		    phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinDCSupport)) {
+		/* Need to do calculation based on the golden DPM table
+		 * as the Heatmap GPU Clock axis is also based on the default values
+		 */
+			PP_ASSERT_WITH_CODE(
+				(golden_dpm_table->sclk_table.dpm_levels
+						[golden_dpm_table->sclk_table.count - 1].value != 0),
+				"Divide by 0!",
+				return -EINVAL);
+			dpm_count = dpm_table->sclk_table.count < 2 ? 0 : dpm_table->sclk_table.count - 2;
+
+			for (i = dpm_count; i > 1; i--) {
+				if (sclk > golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count-1].value) {
+					clock_percent =
+					      ((sclk
+						- golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count-1].value
+						) * 100)
+						/ golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count-1].value;
+
+					dpm_table->sclk_table.dpm_levels[i].value =
+							golden_dpm_table->sclk_table.dpm_levels[i].value +
+							(golden_dpm_table->sclk_table.dpm_levels[i].value *
+								clock_percent)/100;
+
+				} else if (golden_dpm_table->sclk_table.dpm_levels[dpm_table->sclk_table.count-1].value > sclk) {
+					clock_percent =
+						((golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count - 1].value
+						- sclk) * 100)
+						/ golden_dpm_table->sclk_table.dpm_levels[golden_dpm_table->sclk_table.count-1].value;
+
+					dpm_table->sclk_table.dpm_levels[i].value =
+							golden_dpm_table->sclk_table.dpm_levels[i].value -
+							(golden_dpm_table->sclk_table.dpm_levels[i].value *
+									clock_percent) / 100;
+				} else
+					dpm_table->sclk_table.dpm_levels[i].value =
+							golden_dpm_table->sclk_table.dpm_levels[i].value;
+			}
+		}
+	}
+
+	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK) {
+		dpm_table->mclk_table.dpm_levels
+			[dpm_table->mclk_table.count - 1].value = mclk;
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinACSupport) ||
+		    phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinDCSupport)) {
+
+			PP_ASSERT_WITH_CODE(
+					(golden_dpm_table->mclk_table.dpm_levels
+						[golden_dpm_table->mclk_table.count-1].value != 0),
+					"Divide by 0!",
+					return -EINVAL);
+			dpm_count = dpm_table->mclk_table.count < 2 ? 0 : dpm_table->mclk_table.count - 2;
+			for (i = dpm_count; i > 1; i--) {
+				if (golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value < mclk) {
+					clock_percent = ((mclk -
+					golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value) * 100)
+					/ golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value;
+
+					dpm_table->mclk_table.dpm_levels[i].value =
+							golden_dpm_table->mclk_table.dpm_levels[i].value +
+							(golden_dpm_table->mclk_table.dpm_levels[i].value *
+							clock_percent) / 100;
+
+				} else if (golden_dpm_table->mclk_table.dpm_levels[dpm_table->mclk_table.count-1].value > mclk) {
+					clock_percent = (
+					 (golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value - mclk)
+					* 100)
+					/ golden_dpm_table->mclk_table.dpm_levels[golden_dpm_table->mclk_table.count-1].value;
+
+					dpm_table->mclk_table.dpm_levels[i].value =
+							golden_dpm_table->mclk_table.dpm_levels[i].value -
+							(golden_dpm_table->mclk_table.dpm_levels[i].value *
+									clock_percent) / 100;
+				} else
+					dpm_table->mclk_table.dpm_levels[i].value =
+							golden_dpm_table->mclk_table.dpm_levels[i].value;
+			}
+		}
+	}
+
+	if (data->need_update_smu7_dpm_table &
+			(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK)) {
+		result = smum_populate_all_graphic_levels(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
+				return result);
+	}
+
+	if (data->need_update_smu7_dpm_table &
+			(DPMTABLE_OD_UPDATE_MCLK + DPMTABLE_UPDATE_MCLK)) {
+		/*populate MCLK dpm table to SMU7 */
+		result = smum_populate_all_memory_levels(hwmgr);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
+				return result);
+	}
+
+	return result;
+}
+
+static int smu7_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
+			  struct smu7_single_dpm_table *dpm_table,
+			uint32_t low_limit, uint32_t high_limit)
+{
+	uint32_t i;
+
+	for (i = 0; i < dpm_table->count; i++) {
+		if ((dpm_table->dpm_levels[i].value < low_limit)
+		|| (dpm_table->dpm_levels[i].value > high_limit))
+			dpm_table->dpm_levels[i].enabled = false;
+		else
+			dpm_table->dpm_levels[i].enabled = true;
+	}
+
+	return 0;
+}
+
+static int smu7_trim_dpm_states(struct pp_hwmgr *hwmgr,
+		const struct smu7_power_state *smu7_ps)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t high_limit_count;
+
+	PP_ASSERT_WITH_CODE((smu7_ps->performance_level_count >= 1),
+			"power state did not have any performance level",
+			return -EINVAL);
+
+	high_limit_count = (1 == smu7_ps->performance_level_count) ? 0 : 1;
+
+	smu7_trim_single_dpm_states(hwmgr,
+			&(data->dpm_table.sclk_table),
+			smu7_ps->performance_levels[0].engine_clock,
+			smu7_ps->performance_levels[high_limit_count].engine_clock);
+
+	smu7_trim_single_dpm_states(hwmgr,
+			&(data->dpm_table.mclk_table),
+			smu7_ps->performance_levels[0].memory_clock,
+			smu7_ps->performance_levels[high_limit_count].memory_clock);
+
+	return 0;
+}
+
+static int smu7_generate_dpm_level_enable_mask(
+		struct pp_hwmgr *hwmgr, const void *input)
+{
+	int result;
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	const struct smu7_power_state *smu7_ps =
+			cast_const_phw_smu7_power_state(states->pnew_state);
+
+	result = smu7_trim_dpm_states(hwmgr, smu7_ps);
+	if (result)
+		return result;
+
+	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&data->dpm_table.sclk_table);
+	data->dpm_level_enable_mask.mclk_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&data->dpm_table.mclk_table);
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&data->dpm_table.pcie_speed_table);
+
+	return 0;
+}
+
+static int smu7_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (0 == data->need_update_smu7_dpm_table)
+		return 0;
+
+	if ((0 == data->sclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
+
+		PP_ASSERT_WITH_CODE(true == smum_is_dpm_running(hwmgr),
+				"Trying to Unfreeze SCLK DPM when DPM is disabled",
+				);
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_SCLKDPM_UnfreezeLevel),
+			"Failed to unfreeze SCLK DPM during UnFreezeSclkMclkDPM Function!",
+			return -EINVAL);
+	}
+
+	if ((0 == data->mclk_dpm_key_disabled) &&
+		(data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
+
+		PP_ASSERT_WITH_CODE(true == smum_is_dpm_running(hwmgr),
+				"Trying to Unfreeze MCLK DPM when DPM is disabled",
+				);
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
+				PPSMC_MSG_SCLKDPM_UnfreezeLevel),
+		    "Failed to unfreeze MCLK DPM during UnFreezeSclkMclkDPM Function!",
+		    return -EINVAL);
+	}
+
+	data->need_update_smu7_dpm_table = 0;
+
+	return 0;
+}
+
+static int smu7_notify_link_speed_change_after_state_change(
+		struct pp_hwmgr *hwmgr, const void *input)
+{
+	const struct phm_set_power_state_input *states =
+			(const struct phm_set_power_state_input *)input;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	const struct smu7_power_state *smu7_ps =
+			cast_const_phw_smu7_power_state(states->pnew_state);
+	uint16_t target_link_speed = smu7_get_maximum_link_speed(hwmgr, smu7_ps);
+	uint8_t  request;
+
+	if (data->pspp_notify_required) {
+		if (target_link_speed == PP_PCIEGen3)
+			request = PCIE_PERF_REQ_GEN3;
+		else if (target_link_speed == PP_PCIEGen2)
+			request = PCIE_PERF_REQ_GEN2;
+		else
+			request = PCIE_PERF_REQ_GEN1;
+
+		if (request == PCIE_PERF_REQ_GEN1 &&
+				smu7_get_current_pcie_speed(hwmgr) > 0)
+			return 0;
+
+		if (acpi_pcie_perf_request(hwmgr->device, request, false)) {
+			if (PP_PCIEGen2 == target_link_speed)
+				printk("PSPP request to switch to Gen2 from Gen3 Failed!");
+			else
+				printk("PSPP request to switch to Gen1 from Gen2 Failed!");
+		}
+	}
+
+	return 0;
+}
+
+static int smu7_notify_smc_display(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (hwmgr->feature_mask & PP_VBI_TIME_SUPPORT_MASK)
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			(PPSMC_Msg)PPSMC_MSG_SetVBITimeout, data->frame_time_x2);
+	return (smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_HasDisplay) == 0) ?  0 : -EINVAL;
+}
+
+static int smu7_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *input)
+{
+	int tmp_result, result = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	tmp_result = smu7_find_dpm_states_clocks_in_dpm_table(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to find DPM states clocks in DPM table!",
+			result = tmp_result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PCIEPerformanceRequest)) {
+		tmp_result =
+			smu7_request_link_speed_change_before_state_change(hwmgr, input);
+		PP_ASSERT_WITH_CODE((0 == tmp_result),
+				"Failed to request link speed change before state change!",
+				result = tmp_result);
+	}
+
+	tmp_result = smu7_freeze_sclk_mclk_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to freeze SCLK MCLK DPM!", result = tmp_result);
+
+	tmp_result = smu7_populate_and_upload_sclk_mclk_dpm_levels(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to populate and upload SCLK MCLK DPM levels!",
+			result = tmp_result);
+
+	tmp_result = smu7_generate_dpm_level_enable_mask(hwmgr, input);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to generate DPM level enabled mask!",
+			result = tmp_result);
+
+	tmp_result = smum_update_sclk_threshold(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to update SCLK threshold!",
+			result = tmp_result);
+
+	tmp_result = smu7_notify_smc_display(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to notify smc display settings!",
+			result = tmp_result);
+
+	tmp_result = smu7_unfreeze_sclk_mclk_dpm(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to unfreeze SCLK MCLK DPM!",
+			result = tmp_result);
+
+	tmp_result = smu7_upload_dpm_level_enable_mask(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to upload DPM level enabled mask!",
+			result = tmp_result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PCIEPerformanceRequest)) {
+		tmp_result =
+			smu7_notify_link_speed_change_after_state_change(hwmgr, input);
+		PP_ASSERT_WITH_CODE((0 == tmp_result),
+				"Failed to notify link speed change after state change!",
+				result = tmp_result);
+	}
+	data->apply_optimized_settings = false;
+	return result;
+}
+
+static int smu7_set_max_fan_pwm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_pwm)
+{
+	hwmgr->thermal_controller.
+	advanceFanControlParameters.usMaxFanPWM = us_max_fan_pwm;
+
+	if (phm_is_hw_access_blocked(hwmgr))
+		return 0;
+
+	return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			PPSMC_MSG_SetFanPwmMax, us_max_fan_pwm);
+}
+
+int smu7_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display)
+{
+	PPSMC_Msg msg = has_display ? (PPSMC_Msg)PPSMC_HasDisplay : (PPSMC_Msg)PPSMC_NoDisplay;
+
+	return (smum_send_msg_to_smc(hwmgr->smumgr, msg) == 0) ?  0 : -1;
+}
+
+int smu7_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
+{
+	uint32_t num_active_displays = 0;
+	struct cgs_display_info info = {0};
+
+	info.mode_info = NULL;
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	num_active_displays = info.display_count;
+
+	if (num_active_displays > 1 && hwmgr->display_config.multi_monitor_in_sync != true)
+		smu7_notify_smc_display_change(hwmgr, false);
+
+	return 0;
+}
+
+/**
+* Programs the display gap
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always OK
+*/
+int smu7_program_display_gap(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t num_active_displays = 0;
+	uint32_t display_gap = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL);
+	uint32_t display_gap2;
+	uint32_t pre_vbi_time_in_us;
+	uint32_t frame_time_in_us;
+	uint32_t ref_clock;
+	uint32_t refresh_rate = 0;
+	struct cgs_display_info info = {0};
+	struct cgs_mode_info mode_info;
+
+	info.mode_info = &mode_info;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+	num_active_displays = info.display_count;
+
+	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL, DISP_GAP, (num_active_displays > 0) ? DISPLAY_GAP_VBLANK_OR_WM : DISPLAY_GAP_IGNORE);
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL, display_gap);
+
+	ref_clock = mode_info.ref_clock;
+	refresh_rate = mode_info.refresh_rate;
+
+	if (0 == refresh_rate)
+		refresh_rate = 60;
+
+	frame_time_in_us = 1000000 / refresh_rate;
+
+	pre_vbi_time_in_us = frame_time_in_us - 200 - mode_info.vblank_time_us;
+	data->frame_time_x2 = frame_time_in_us * 2 / 100;
+
+	display_gap2 = pre_vbi_time_in_us * (ref_clock / 100);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL2, display_gap2);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			data->soft_regs_start + smum_get_offsetof(hwmgr->smumgr,
+							SMU_SoftRegisters,
+							PreVBlankGap), 0x64);
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			data->soft_regs_start + smum_get_offsetof(hwmgr->smumgr,
+							SMU_SoftRegisters,
+							VBlankTimeout),
+					(frame_time_in_us - pre_vbi_time_in_us));
+
+	return 0;
+}
+
+int smu7_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
+{
+	return smu7_program_display_gap(hwmgr);
+}
+
+/**
+*  Set maximum target operating fan output RPM
+*
+* @param    hwmgr:  the address of the powerplay hardware manager.
+* @param    usMaxFanRpm:  max operating fan RPM value.
+* @return   The response that came from the SMC.
+*/
+static int smu7_set_max_fan_rpm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_rpm)
+{
+	hwmgr->thermal_controller.
+	advanceFanControlParameters.usMaxFanRPM = us_max_fan_rpm;
+
+	if (phm_is_hw_access_blocked(hwmgr))
+		return 0;
+
+	return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			PPSMC_MSG_SetFanRpmMax, us_max_fan_rpm);
+}
+
+int smu7_register_internal_thermal_interrupt(struct pp_hwmgr *hwmgr,
+					const void *thermal_interrupt_info)
+{
+	return 0;
+}
+
+bool smu7_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	bool is_update_required = false;
+	struct cgs_display_info info = {0, 0, NULL};
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	if (data->display_timing.num_existing_displays != info.display_count)
+		is_update_required = true;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep)) {
+		if (data->display_timing.min_clock_in_sr != hwmgr->display_config.min_core_set_clock_in_sr &&
+			(data->display_timing.min_clock_in_sr >= SMU7_MINIMUM_ENGINE_CLOCK ||
+			hwmgr->display_config.min_core_set_clock_in_sr >= SMU7_MINIMUM_ENGINE_CLOCK))
+			is_update_required = true;
+	}
+	return is_update_required;
+}
+
+static inline bool smu7_are_power_levels_equal(const struct smu7_performance_level *pl1,
+							   const struct smu7_performance_level *pl2)
+{
+	return ((pl1->memory_clock == pl2->memory_clock) &&
+		  (pl1->engine_clock == pl2->engine_clock) &&
+		  (pl1->pcie_gen == pl2->pcie_gen) &&
+		  (pl1->pcie_lane == pl2->pcie_lane));
+}
+
+int smu7_check_states_equal(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *pstate1, const struct pp_hw_power_state *pstate2, bool *equal)
+{
+	const struct smu7_power_state *psa = cast_const_phw_smu7_power_state(pstate1);
+	const struct smu7_power_state *psb = cast_const_phw_smu7_power_state(pstate2);
+	int i;
+
+	if (pstate1 == NULL || pstate2 == NULL || equal == NULL)
+		return -EINVAL;
+
+	/* If the two states don't even have the same number of performance levels they cannot be the same state. */
+	if (psa->performance_level_count != psb->performance_level_count) {
+		*equal = false;
+		return 0;
+	}
+
+	for (i = 0; i < psa->performance_level_count; i++) {
+		if (!smu7_are_power_levels_equal(&(psa->performance_levels[i]), &(psb->performance_levels[i]))) {
+			/* If we have found even one performance level pair that is different the states are different. */
+			*equal = false;
+			return 0;
+		}
+	}
+
+	/* If all performance levels are the same try to use the UVD clocks to break the tie.*/
+	*equal = ((psa->uvd_clks.vclk == psb->uvd_clks.vclk) && (psa->uvd_clks.dclk == psb->uvd_clks.dclk));
+	*equal &= ((psa->vce_clks.evclk == psb->vce_clks.evclk) && (psa->vce_clks.ecclk == psb->vce_clks.ecclk));
+	*equal &= (psa->sclk_threshold == psb->sclk_threshold);
+
+	return 0;
+}
+
+int smu7_upload_mc_firmware(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	uint32_t vbios_version;
+	uint32_t tmp;
+
+	/* Read MC indirect register offset 0x9F bits [3:0] to see
+	 * if VBIOS has already loaded a full version of MC ucode
+	 * or not.
+	 */
+
+	smu7_get_mc_microcode_version(hwmgr);
+	vbios_version = hwmgr->microcode_version_info.MC & 0xf;
+
+	data->need_long_memory_training = false;
+
+	cgs_write_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_INDEX,
+							ixMC_IO_DEBUG_UP_13);
+	tmp = cgs_read_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_DATA);
+
+	if (tmp & (1 << 23)) {
+		data->mem_latency_high = MEM_LATENCY_HIGH;
+		data->mem_latency_low = MEM_LATENCY_LOW;
+	} else {
+		data->mem_latency_high = 330;
+		data->mem_latency_low = 330;
+	}
+
+	return 0;
+}
+
+static int smu7_read_clock_registers(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	data->clock_registers.vCG_SPLL_FUNC_CNTL         =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL);
+	data->clock_registers.vCG_SPLL_FUNC_CNTL_2       =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_2);
+	data->clock_registers.vCG_SPLL_FUNC_CNTL_3       =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_3);
+	data->clock_registers.vCG_SPLL_FUNC_CNTL_4       =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_4);
+	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM   =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_SPREAD_SPECTRUM);
+	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2 =
+		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_SPREAD_SPECTRUM_2);
+	data->clock_registers.vDLL_CNTL                  =
+		cgs_read_register(hwmgr->device, mmDLL_CNTL);
+	data->clock_registers.vMCLK_PWRMGT_CNTL          =
+		cgs_read_register(hwmgr->device, mmMCLK_PWRMGT_CNTL);
+	data->clock_registers.vMPLL_AD_FUNC_CNTL         =
+		cgs_read_register(hwmgr->device, mmMPLL_AD_FUNC_CNTL);
+	data->clock_registers.vMPLL_DQ_FUNC_CNTL         =
+		cgs_read_register(hwmgr->device, mmMPLL_DQ_FUNC_CNTL);
+	data->clock_registers.vMPLL_FUNC_CNTL            =
+		cgs_read_register(hwmgr->device, mmMPLL_FUNC_CNTL);
+	data->clock_registers.vMPLL_FUNC_CNTL_1          =
+		cgs_read_register(hwmgr->device, mmMPLL_FUNC_CNTL_1);
+	data->clock_registers.vMPLL_FUNC_CNTL_2          =
+		cgs_read_register(hwmgr->device, mmMPLL_FUNC_CNTL_2);
+	data->clock_registers.vMPLL_SS1                  =
+		cgs_read_register(hwmgr->device, mmMPLL_SS1);
+	data->clock_registers.vMPLL_SS2                  =
+		cgs_read_register(hwmgr->device, mmMPLL_SS2);
+	return 0;
+
+}
+
+/**
+ * Find out if memory is GDDR5.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+static int smu7_get_memory_type(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t temp;
+
+	temp = cgs_read_register(hwmgr->device, mmMC_SEQ_MISC0);
+
+	data->is_memory_gddr5 = (MC_SEQ_MISC0_GDDR5_VALUE ==
+			((temp & MC_SEQ_MISC0_GDDR5_MASK) >>
+			 MC_SEQ_MISC0_GDDR5_SHIFT));
+
+	return 0;
+}
+
+/**
+ * Enables Dynamic Power Management by SMC
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+static int smu7_enable_acpi_power_management(struct pp_hwmgr *hwmgr)
+{
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			GENERAL_PWRMGT, STATIC_PM_EN, 1);
+
+	return 0;
+}
+
+/**
+ * Initialize PowerGating States for different engines
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+static int smu7_init_power_gate_state(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	data->uvd_power_gated = false;
+	data->vce_power_gated = false;
+	data->samu_power_gated = false;
+
+	return 0;
+}
+
+static int smu7_init_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	data->low_sclk_interrupt_threshold = 0;
+	return 0;
+}
+
+int smu7_setup_asic_task(struct pp_hwmgr *hwmgr)
+{
+	int tmp_result, result = 0;
+
+	smu7_upload_mc_firmware(hwmgr);
+
+	tmp_result = smu7_read_clock_registers(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to read clock registers!", result = tmp_result);
+
+	tmp_result = smu7_get_memory_type(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to get memory type!", result = tmp_result);
+
+	tmp_result = smu7_enable_acpi_power_management(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to enable ACPI power management!", result = tmp_result);
+
+	tmp_result = smu7_init_power_gate_state(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to init power gate state!", result = tmp_result);
+
+	tmp_result = smu7_get_mc_microcode_version(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to get MC microcode version!", result = tmp_result);
+
+	tmp_result = smu7_init_sclk_threshold(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == tmp_result),
+			"Failed to init sclk threshold!", result = tmp_result);
+
+	return result;
+}
+
+static int smu7_force_clock_level(struct pp_hwmgr *hwmgr,
+		enum pp_clock_type type, uint32_t mask)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL)
+		return -EINVAL;
+
+	switch (type) {
+	case PP_SCLK:
+		if (!data->sclk_dpm_key_disabled)
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_SCLKDPM_SetEnabledMask,
+					data->dpm_level_enable_mask.sclk_dpm_enable_mask & mask);
+		break;
+	case PP_MCLK:
+		if (!data->mclk_dpm_key_disabled)
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_MCLKDPM_SetEnabledMask,
+					data->dpm_level_enable_mask.mclk_dpm_enable_mask & mask);
+		break;
+	case PP_PCIE:
+	{
+		uint32_t tmp = mask & data->dpm_level_enable_mask.pcie_dpm_enable_mask;
+		uint32_t level = 0;
+
+		while (tmp >>= 1)
+			level++;
+
+		if (!data->pcie_dpm_key_disabled)
+			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_PCIeDPM_ForceLevel,
+					level);
+		break;
+	}
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
+		enum pp_clock_type type, char *buf)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
+	struct smu7_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
+	struct smu7_single_dpm_table *pcie_table = &(data->dpm_table.pcie_speed_table);
+	int i, now, size = 0;
+	uint32_t clock, pcie_speed;
+
+	switch (type) {
+	case PP_SCLK:
+		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetSclkFrequency);
+		clock = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+
+		for (i = 0; i < sclk_table->count; i++) {
+			if (clock > sclk_table->dpm_levels[i].value)
+				continue;
+			break;
+		}
+		now = i;
+
+		for (i = 0; i < sclk_table->count; i++)
+			size += sprintf(buf + size, "%d: %uMhz %s\n",
+					i, sclk_table->dpm_levels[i].value / 100,
+					(i == now) ? "*" : "");
+		break;
+	case PP_MCLK:
+		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetMclkFrequency);
+		clock = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
+
+		for (i = 0; i < mclk_table->count; i++) {
+			if (clock > mclk_table->dpm_levels[i].value)
+				continue;
+			break;
+		}
+		now = i;
+
+		for (i = 0; i < mclk_table->count; i++)
+			size += sprintf(buf + size, "%d: %uMhz %s\n",
+					i, mclk_table->dpm_levels[i].value / 100,
+					(i == now) ? "*" : "");
+		break;
+	case PP_PCIE:
+		pcie_speed = smu7_get_current_pcie_speed(hwmgr);
+		for (i = 0; i < pcie_table->count; i++) {
+			if (pcie_speed != pcie_table->dpm_levels[i].value)
+				continue;
+			break;
+		}
+		now = i;
+
+		for (i = 0; i < pcie_table->count; i++)
+			size += sprintf(buf + size, "%d: %s %s\n", i,
+					(pcie_table->dpm_levels[i].value == 0) ? "2.5GB, x8" :
+					(pcie_table->dpm_levels[i].value == 1) ? "5.0GB, x16" :
+					(pcie_table->dpm_levels[i].value == 2) ? "8.0GB, x16" : "",
+					(i == now) ? "*" : "");
+		break;
+	default:
+		break;
+	}
+	return size;
+}
+
+static int smu7_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
+{
+	if (mode) {
+		/* stop auto-manage */
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl))
+			smu7_fan_ctrl_stop_smc_fan_control(hwmgr);
+		smu7_fan_ctrl_set_static_mode(hwmgr, mode);
+	} else
+		/* restart auto-manage */
+		smu7_fan_ctrl_reset_fan_speed_to_default(hwmgr);
+
+	return 0;
+}
+
+static int smu7_get_fan_control_mode(struct pp_hwmgr *hwmgr)
+{
+	if (hwmgr->fan_ctrl_is_in_default_mode)
+		return hwmgr->fan_ctrl_default_mode;
+	else
+		return PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+				CG_FDO_CTRL2, FDO_PWM_MODE);
+}
+
+static int smu7_get_sclk_od(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
+	struct smu7_single_dpm_table *golden_sclk_table =
+			&(data->golden_dpm_table.sclk_table);
+	int value;
+
+	value = (sclk_table->dpm_levels[sclk_table->count - 1].value -
+			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) *
+			100 /
+			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+	return value;
+}
+
+static int smu7_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_single_dpm_table *golden_sclk_table =
+			&(data->golden_dpm_table.sclk_table);
+	struct pp_power_state  *ps;
+	struct smu7_power_state  *smu7_ps;
+
+	if (value > 20)
+		value = 20;
+
+	ps = hwmgr->request_ps;
+
+	if (ps == NULL)
+		return -EINVAL;
+
+	smu7_ps = cast_phw_smu7_power_state(&ps->hardware);
+
+	smu7_ps->performance_levels[smu7_ps->performance_level_count - 1].engine_clock =
+			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value *
+			value / 100 +
+			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+	return 0;
+}
+
+static int smu7_get_mclk_od(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
+	struct smu7_single_dpm_table *golden_mclk_table =
+			&(data->golden_dpm_table.mclk_table);
+	int value;
+
+	value = (mclk_table->dpm_levels[mclk_table->count - 1].value -
+			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) *
+			100 /
+			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+	return value;
+}
+
+static int smu7_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_single_dpm_table *golden_mclk_table =
+			&(data->golden_dpm_table.mclk_table);
+	struct pp_power_state  *ps;
+	struct smu7_power_state  *smu7_ps;
+
+	if (value > 20)
+		value = 20;
+
+	ps = hwmgr->request_ps;
+
+	if (ps == NULL)
+		return -EINVAL;
+
+	smu7_ps = cast_phw_smu7_power_state(&ps->hardware);
+
+	smu7_ps->performance_levels[smu7_ps->performance_level_count - 1].memory_clock =
+			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value *
+			value / 100 +
+			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+	return 0;
+}
+
+
+static int smu7_get_sclks(struct pp_hwmgr *hwmgr, struct amd_pp_clocks *clocks)
+{
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)hwmgr->pptable;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_sclk_table;
+	int i;
+
+	if (table_info == NULL)
+		return -EINVAL;
+
+	dep_sclk_table = table_info->vdd_dep_on_sclk;
+
+	for (i = 0; i < dep_sclk_table->count; i++) {
+		clocks->clock[i] = dep_sclk_table->entries[i].clk;
+		clocks->count++;
+	}
+	return 0;
+}
+
+static uint32_t smu7_get_mem_latency(struct pp_hwmgr *hwmgr, uint32_t clk)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (clk >= MEM_FREQ_LOW_LATENCY && clk < MEM_FREQ_HIGH_LATENCY)
+		return data->mem_latency_high;
+	else if (clk >= MEM_FREQ_HIGH_LATENCY)
+		return data->mem_latency_low;
+	else
+		return MEM_LATENCY_ERR;
+}
+
+static int smu7_get_mclks(struct pp_hwmgr *hwmgr, struct amd_pp_clocks *clocks)
+{
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)hwmgr->pptable;
+	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table;
+	int i;
+
+	if (table_info == NULL)
+		return -EINVAL;
+
+	dep_mclk_table = table_info->vdd_dep_on_mclk;
+
+	for (i = 0; i < dep_mclk_table->count; i++) {
+		clocks->clock[i] = dep_mclk_table->entries[i].clk;
+		clocks->latency[i] = smu7_get_mem_latency(hwmgr,
+						dep_mclk_table->entries[i].clk);
+		clocks->count++;
+	}
+	return 0;
+}
+
+static int smu7_get_clock_by_type(struct pp_hwmgr *hwmgr, enum amd_pp_clock_type type,
+						struct amd_pp_clocks *clocks)
+{
+	switch (type) {
+	case amd_pp_sys_clock:
+		smu7_get_sclks(hwmgr, clocks);
+		break;
+	case amd_pp_mem_clock:
+		smu7_get_mclks(hwmgr, clocks);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
+	.backend_init = &smu7_hwmgr_backend_init,
+	.backend_fini = &phm_hwmgr_backend_fini,
+	.asic_setup = &smu7_setup_asic_task,
+	.dynamic_state_management_enable = &smu7_enable_dpm_tasks,
+	.apply_state_adjust_rules = smu7_apply_state_adjust_rules,
+	.force_dpm_level = &smu7_force_dpm_level,
+	.power_state_set = smu7_set_power_state_tasks,
+	.get_power_state_size = smu7_get_power_state_size,
+	.get_mclk = smu7_dpm_get_mclk,
+	.get_sclk = smu7_dpm_get_sclk,
+	.patch_boot_state = smu7_dpm_patch_boot_state,
+	.get_pp_table_entry = smu7_get_pp_table_entry,
+	.get_num_of_pp_table_entries = smu7_get_number_of_powerplay_table_entries,
+	.powerdown_uvd = smu7_powerdown_uvd,
+	.powergate_uvd = smu7_powergate_uvd,
+	.powergate_vce = smu7_powergate_vce,
+	.disable_clock_power_gating = smu7_disable_clock_power_gating,
+	.update_clock_gatings = smu7_update_clock_gatings,
+	.notify_smc_display_config_after_ps_adjustment = smu7_notify_smc_display_config_after_ps_adjustment,
+	.display_config_changed = smu7_display_configuration_changed_task,
+	.set_max_fan_pwm_output = smu7_set_max_fan_pwm_output,
+	.set_max_fan_rpm_output = smu7_set_max_fan_rpm_output,
+	.get_temperature = smu7_thermal_get_temperature,
+	.stop_thermal_controller = smu7_thermal_stop_thermal_controller,
+	.get_fan_speed_info = smu7_fan_ctrl_get_fan_speed_info,
+	.get_fan_speed_percent = smu7_fan_ctrl_get_fan_speed_percent,
+	.set_fan_speed_percent = smu7_fan_ctrl_set_fan_speed_percent,
+	.reset_fan_speed_to_default = smu7_fan_ctrl_reset_fan_speed_to_default,
+	.get_fan_speed_rpm = smu7_fan_ctrl_get_fan_speed_rpm,
+	.set_fan_speed_rpm = smu7_fan_ctrl_set_fan_speed_rpm,
+	.uninitialize_thermal_controller = smu7_thermal_ctrl_uninitialize_thermal_controller,
+	.register_internal_thermal_interrupt = smu7_register_internal_thermal_interrupt,
+	.check_smc_update_required_for_display_configuration = smu7_check_smc_update_required_for_display_configuration,
+	.check_states_equal = smu7_check_states_equal,
+	.set_fan_control_mode = smu7_set_fan_control_mode,
+	.get_fan_control_mode = smu7_get_fan_control_mode,
+	.force_clock_level = smu7_force_clock_level,
+	.print_clock_levels = smu7_print_clock_levels,
+	.enable_per_cu_power_gating = smu7_enable_per_cu_power_gating,
+	.get_sclk_od = smu7_get_sclk_od,
+	.set_sclk_od = smu7_set_sclk_od,
+	.get_mclk_od = smu7_get_mclk_od,
+	.set_mclk_od = smu7_set_mclk_od,
+	.get_clock_by_type = smu7_get_clock_by_type,
+	.read_sensor = smu7_read_sensor,
+};
+
+uint8_t smu7_get_sleep_divider_id_from_clock(uint32_t clock,
+		uint32_t clock_insr)
+{
+	uint8_t i;
+	uint32_t temp;
+	uint32_t min = max(clock_insr, (uint32_t)SMU7_MINIMUM_ENGINE_CLOCK);
+
+	PP_ASSERT_WITH_CODE((clock >= min), "Engine clock can't satisfy stutter requirement!", return 0);
+	for (i = SMU7_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
+		temp = clock >> i;
+
+		if (temp >= min || i == 0)
+			break;
+	}
+	return i;
+}
+
+int smu7_hwmgr_init(struct pp_hwmgr *hwmgr)
+{
+	int ret = 0;
+
+	hwmgr->hwmgr_func = &smu7_hwmgr_funcs;
+	if (hwmgr->pp_table_version == PP_TABLE_V0)
+		hwmgr->pptable_func = &pptable_funcs;
+	else if (hwmgr->pp_table_version == PP_TABLE_V1)
+		hwmgr->pptable_func = &pptable_v1_0_funcs;
+
+	pp_smu7_thermal_initialize(hwmgr);
+	return ret;
+}
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.h
index 33c33947e827..27e7f76ad8a6 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.h
@@ -21,81 +21,100 @@
  *
  */
 
-#ifndef POLARIS10_HWMGR_H
-#define POLARIS10_HWMGR_H
+#ifndef _SMU7_HWMGR_H
+#define _SMU7_HWMGR_H
 
 #include "hwmgr.h"
-#include "smu74.h"
-#include "smu74_discrete.h"
 #include "ppatomctrl.h"
-#include "polaris10_ppsmc.h"
-#include "polaris10_powertune.h"
 
-#define POLARIS10_MAX_HARDWARE_POWERLEVELS	2
+#define SMU7_MAX_HARDWARE_POWERLEVELS   2
 
-#define POLARIS10_VOLTAGE_CONTROL_NONE                   0x0
-#define POLARIS10_VOLTAGE_CONTROL_BY_GPIO                0x1
-#define POLARIS10_VOLTAGE_CONTROL_BY_SVID2               0x2
-#define POLARIS10_VOLTAGE_CONTROL_MERGED                 0x3
+#define SMU7_VOLTAGE_CONTROL_NONE                   0x0
+#define SMU7_VOLTAGE_CONTROL_BY_GPIO                0x1
+#define SMU7_VOLTAGE_CONTROL_BY_SVID2               0x2
+#define SMU7_VOLTAGE_CONTROL_MERGED                 0x3
 
 #define DPMTABLE_OD_UPDATE_SCLK     0x00000001
 #define DPMTABLE_OD_UPDATE_MCLK     0x00000002
 #define DPMTABLE_UPDATE_SCLK        0x00000004
 #define DPMTABLE_UPDATE_MCLK        0x00000008
 
-struct polaris10_performance_level {
+enum gpu_pt_config_reg_type {
+	GPU_CONFIGREG_MMR = 0,
+	GPU_CONFIGREG_SMC_IND,
+	GPU_CONFIGREG_DIDT_IND,
+	GPU_CONFIGREG_GC_CAC_IND,
+	GPU_CONFIGREG_CACHE,
+	GPU_CONFIGREG_MAX
+};
+
+struct gpu_pt_config_reg {
+	uint32_t                           offset;
+	uint32_t                           mask;
+	uint32_t                           shift;
+	uint32_t                           value;
+	enum gpu_pt_config_reg_type       type;
+};
+
+struct smu7_performance_level {
 	uint32_t  memory_clock;
 	uint32_t  engine_clock;
 	uint16_t  pcie_gen;
 	uint16_t  pcie_lane;
 };
 
-struct polaris10_uvd_clocks {
+struct smu7_thermal_temperature_setting {
+	long temperature_low;
+	long temperature_high;
+	long temperature_shutdown;
+};
+
+struct smu7_uvd_clocks {
 	uint32_t  vclk;
 	uint32_t  dclk;
 };
 
-struct polaris10_vce_clocks {
+struct smu7_vce_clocks {
 	uint32_t  evclk;
 	uint32_t  ecclk;
 };
 
-struct polaris10_power_state {
+struct smu7_power_state {
 	uint32_t                  magic;
-	struct polaris10_uvd_clocks    uvd_clks;
-	struct polaris10_vce_clocks    vce_clks;
+	struct smu7_uvd_clocks    uvd_clks;
+	struct smu7_vce_clocks    vce_clks;
 	uint32_t                  sam_clk;
 	uint16_t                  performance_level_count;
 	bool                      dc_compatible;
 	uint32_t                  sclk_threshold;
-	struct polaris10_performance_level  performance_levels[POLARIS10_MAX_HARDWARE_POWERLEVELS];
+	struct smu7_performance_level  performance_levels[SMU7_MAX_HARDWARE_POWERLEVELS];
 };
 
-struct polaris10_dpm_level {
+struct smu7_dpm_level {
 	bool	enabled;
 	uint32_t	value;
 	uint32_t	param1;
 };
 
-#define POLARIS10_MAX_DEEPSLEEP_DIVIDER_ID 5
+#define SMU7_MAX_DEEPSLEEP_DIVIDER_ID 5
 #define MAX_REGULAR_DPM_NUMBER 8
-#define POLARIS10_MINIMUM_ENGINE_CLOCK 2500
+#define SMU7_MINIMUM_ENGINE_CLOCK 2500
 
-struct polaris10_single_dpm_table {
+struct smu7_single_dpm_table {
 	uint32_t		count;
-	struct polaris10_dpm_level	dpm_levels[MAX_REGULAR_DPM_NUMBER];
+	struct smu7_dpm_level	dpm_levels[MAX_REGULAR_DPM_NUMBER];
 };
 
-struct polaris10_dpm_table {
-	struct polaris10_single_dpm_table  sclk_table;
-	struct polaris10_single_dpm_table  mclk_table;
-	struct polaris10_single_dpm_table  pcie_speed_table;
-	struct polaris10_single_dpm_table  vddc_table;
-	struct polaris10_single_dpm_table  vddci_table;
-	struct polaris10_single_dpm_table  mvdd_table;
+struct smu7_dpm_table {
+	struct smu7_single_dpm_table  sclk_table;
+	struct smu7_single_dpm_table  mclk_table;
+	struct smu7_single_dpm_table  pcie_speed_table;
+	struct smu7_single_dpm_table  vddc_table;
+	struct smu7_single_dpm_table  vddci_table;
+	struct smu7_single_dpm_table  mvdd_table;
 };
 
-struct polaris10_clock_registers {
+struct smu7_clock_registers {
 	uint32_t  vCG_SPLL_FUNC_CNTL;
 	uint32_t  vCG_SPLL_FUNC_CNTL_2;
 	uint32_t  vCG_SPLL_FUNC_CNTL_3;
@@ -116,42 +135,35 @@ struct polaris10_clock_registers {
 #define DISABLE_MC_LOADMICROCODE   1
 #define DISABLE_MC_CFGPROGRAMMING  2
 
-struct polaris10_voltage_smio_registers {
+struct smu7_voltage_smio_registers {
 	uint32_t vS0_VID_LOWER_SMIO_CNTL;
 };
 
-#define POLARIS10_MAX_LEAKAGE_COUNT  8
+#define SMU7_MAX_LEAKAGE_COUNT  8
 
-struct polaris10_leakage_voltage {
+struct smu7_leakage_voltage {
 	uint16_t  count;
-	uint16_t  leakage_id[POLARIS10_MAX_LEAKAGE_COUNT];
-	uint16_t  actual_voltage[POLARIS10_MAX_LEAKAGE_COUNT];
+	uint16_t  leakage_id[SMU7_MAX_LEAKAGE_COUNT];
+	uint16_t  actual_voltage[SMU7_MAX_LEAKAGE_COUNT];
 };
 
-struct polaris10_vbios_boot_state {
+struct smu7_vbios_boot_state {
 	uint16_t    mvdd_bootup_value;
 	uint16_t    vddc_bootup_value;
 	uint16_t    vddci_bootup_value;
+	uint16_t    vddgfx_bootup_value;
 	uint32_t    sclk_bootup_value;
 	uint32_t    mclk_bootup_value;
 	uint16_t    pcie_gen_bootup_value;
 	uint16_t    pcie_lane_bootup_value;
 };
 
-/* Ultra Low Voltage parameter structure */
-struct polaris10_ulv_parm {
-	bool                           ulv_supported;
-	uint32_t                       cg_ulv_parameter;
-	uint32_t                       ulv_volt_change_delay;
-	struct polaris10_performance_level  ulv_power_level;
-};
-
-struct polaris10_display_timing {
+struct smu7_display_timing {
 	uint32_t  min_clock_in_sr;
 	uint32_t  num_existing_displays;
 };
 
-struct polaris10_dpmlevel_enable_mask {
+struct smu7_dpmlevel_enable_mask {
 	uint32_t  uvd_dpm_enable_mask;
 	uint32_t  vce_dpm_enable_mask;
 	uint32_t  acp_dpm_enable_mask;
@@ -161,22 +173,15 @@ struct polaris10_dpmlevel_enable_mask {
 	uint32_t  pcie_dpm_enable_mask;
 };
 
-struct polaris10_pcie_perf_range {
+struct smu7_pcie_perf_range {
 	uint16_t  max;
 	uint16_t  min;
 };
-struct polaris10_range_table {
-	uint32_t trans_lower_frequency; /* in 10khz */
-	uint32_t trans_upper_frequency;
-};
 
-struct polaris10_hwmgr {
-	struct polaris10_dpm_table			dpm_table;
-	struct polaris10_dpm_table			golden_dpm_table;
-	SMU74_Discrete_DpmTable				smc_state_table;
-	struct SMU74_Discrete_Ulv            ulv_setting;
+struct smu7_hwmgr {
+	struct smu7_dpm_table			dpm_table;
+	struct smu7_dpm_table			golden_dpm_table;
 
-	struct polaris10_range_table                range_table[NUM_SCLK_RANGE];
 	uint32_t						voting_rights_clients0;
 	uint32_t						voting_rights_clients1;
 	uint32_t						voting_rights_clients2;
@@ -188,12 +193,11 @@ struct polaris10_hwmgr {
 	uint32_t						static_screen_threshold_unit;
 	uint32_t						static_screen_threshold;
 	uint32_t						voltage_control;
-	uint32_t						vddc_vddci_delta;
-
+	uint32_t						vdd_gfx_control;
+	uint32_t						vddc_vddgfx_delta;
 	uint32_t						active_auto_throttle_sources;
 
-	struct polaris10_clock_registers            clock_registers;
-	struct polaris10_voltage_smio_registers      voltage_smio_registers;
+	struct smu7_clock_registers            clock_registers;
 
 	bool                           is_memory_gddr5;
 	uint16_t                       acpi_vddc;
@@ -203,8 +207,9 @@ struct polaris10_hwmgr {
 	uint32_t                       pcie_gen_cap;
 	uint32_t                       pcie_lane_cap;
 	uint32_t                       pcie_spc_cap;
-	struct polaris10_leakage_voltage          vddc_leakage;
-	struct polaris10_leakage_voltage          Vddci_leakage;
+	struct smu7_leakage_voltage          vddc_leakage;
+	struct smu7_leakage_voltage          vddci_leakage;
+	struct smu7_leakage_voltage          vddcgfx_leakage;
 
 	uint32_t                             mvdd_control;
 	uint32_t                             vddc_mask_low;
@@ -213,30 +218,23 @@ struct polaris10_hwmgr {
 	uint16_t                            min_vddc_in_pptable;
 	uint16_t                            max_vddci_in_pptable;
 	uint16_t                            min_vddci_in_pptable;
-	uint32_t                             mclk_strobe_mode_threshold;
-	uint32_t                             mclk_stutter_mode_threshold;
-	uint32_t                             mclk_edc_enable_threshold;
-	uint32_t                             mclk_edcwr_enable_threshold;
 	bool                                is_uvd_enabled;
-	struct polaris10_vbios_boot_state        vbios_boot_state;
+	struct smu7_vbios_boot_state        vbios_boot_state;
 
 	bool                           pcie_performance_request;
 	bool                           battery_state;
 	bool                           is_tlu_enabled;
+	bool                           disable_handshake;
+	bool                           smc_voltage_control_enabled;
+	bool                           vbi_time_out_support;
 
-	/* ---- SMC SRAM Address of firmware header tables ---- */
-	uint32_t                             sram_end;
-	uint32_t                             dpm_table_start;
-	uint32_t                             soft_regs_start;
-	uint32_t                             mc_reg_table_start;
-	uint32_t                             fan_table_start;
-	uint32_t                             arb_table_start;
-
+	uint32_t                       soft_regs_start;
 	/* ---- Stuff originally coming from Evergreen ---- */
 	uint32_t                             vddci_control;
 	struct pp_atomctrl_voltage_table     vddc_voltage_table;
 	struct pp_atomctrl_voltage_table     vddci_voltage_table;
 	struct pp_atomctrl_voltage_table     mvdd_voltage_table;
+	struct pp_atomctrl_voltage_table     vddgfx_voltage_table;
 
 	uint32_t                             mgcg_cgtt_local2;
 	uint32_t                             mgcg_cgtt_local3;
@@ -250,7 +248,7 @@ struct polaris10_hwmgr {
 	bool                                 performance_request_registered;
 
 	/* ---- Low Power Features ---- */
-	struct polaris10_ulv_parm                 ulv;
+	bool                           ulv_supported;
 
 	/* ---- CAC Stuff ---- */
 	uint32_t                       cac_table_start;
@@ -264,8 +262,8 @@ struct polaris10_hwmgr {
 	bool                           enable_tdc_limit_feature;
 	bool                           enable_pkg_pwr_tracking_feature;
 	bool                           disable_uvd_power_tune_feature;
-	const struct polaris10_pt_defaults       *power_tune_defaults;
-	struct SMU74_Discrete_PmFuses  power_tune_table;
+
+
 	uint32_t                       dte_tj_offset;
 	uint32_t                       fast_watermark_threshold;
 
@@ -273,23 +271,22 @@ struct polaris10_hwmgr {
 	bool                           vddc_phase_shed_control;
 
 	/* ---- DI/DT ---- */
-	struct polaris10_display_timing        display_timing;
-	uint32_t                      bif_sclk_table[SMU74_MAX_LEVELS_LINK];
+	struct smu7_display_timing        display_timing;
 
 	/* ---- Thermal Temperature Setting ---- */
-	struct polaris10_dpmlevel_enable_mask     dpm_level_enable_mask;
+	struct smu7_thermal_temperature_setting  thermal_temp_setting;
+	struct smu7_dpmlevel_enable_mask     dpm_level_enable_mask;
 	uint32_t                                  need_update_smu7_dpm_table;
 	uint32_t                                  sclk_dpm_key_disabled;
 	uint32_t                                  mclk_dpm_key_disabled;
 	uint32_t                                  pcie_dpm_key_disabled;
 	uint32_t                                  min_engine_clocks;
-	struct polaris10_pcie_perf_range          pcie_gen_performance;
-	struct polaris10_pcie_perf_range          pcie_lane_performance;
-	struct polaris10_pcie_perf_range          pcie_gen_power_saving;
-	struct polaris10_pcie_perf_range          pcie_lane_power_saving;
+	struct smu7_pcie_perf_range          pcie_gen_performance;
+	struct smu7_pcie_perf_range          pcie_lane_performance;
+	struct smu7_pcie_perf_range          pcie_gen_power_saving;
+	struct smu7_pcie_perf_range          pcie_lane_power_saving;
 	bool                                      use_pcie_performance_levels;
 	bool                                      use_pcie_power_saving_levels;
-	uint32_t                                  activity_target[SMU74_MAX_LEVELS_GRAPHICS];
 	uint32_t                                  mclk_activity_target;
 	uint32_t                                  mclk_dpm0_activity_target;
 	uint32_t                                  low_sclk_interrupt_threshold;
@@ -309,49 +306,48 @@ struct polaris10_hwmgr {
 	uint32_t                           up_hyst;
 	uint32_t disable_dpm_mask;
 	bool apply_optimized_settings;
+
 	uint32_t                              avfs_vdroop_override_setting;
 	bool                                  apply_avfs_cks_off_voltage;
 	uint32_t                              frame_time_x2;
+	uint16_t                              mem_latency_high;
+	uint16_t                              mem_latency_low;
 };
 
 /* To convert to Q8.8 format for firmware */
-#define POLARIS10_Q88_FORMAT_CONVERSION_UNIT             256
-
-enum Polaris10_I2CLineID {
-	Polaris10_I2CLineID_DDC1 = 0x90,
-	Polaris10_I2CLineID_DDC2 = 0x91,
-	Polaris10_I2CLineID_DDC3 = 0x92,
-	Polaris10_I2CLineID_DDC4 = 0x93,
-	Polaris10_I2CLineID_DDC5 = 0x94,
-	Polaris10_I2CLineID_DDC6 = 0x95,
-	Polaris10_I2CLineID_SCLSDA = 0x96,
-	Polaris10_I2CLineID_DDCVGA = 0x97
+#define SMU7_Q88_FORMAT_CONVERSION_UNIT             256
+
+enum SMU7_I2CLineID {
+	SMU7_I2CLineID_DDC1 = 0x90,
+	SMU7_I2CLineID_DDC2 = 0x91,
+	SMU7_I2CLineID_DDC3 = 0x92,
+	SMU7_I2CLineID_DDC4 = 0x93,
+	SMU7_I2CLineID_DDC5 = 0x94,
+	SMU7_I2CLineID_DDC6 = 0x95,
+	SMU7_I2CLineID_SCLSDA = 0x96,
+	SMU7_I2CLineID_DDCVGA = 0x97
 };
 
-#define POLARIS10_I2C_DDC1DATA          0
-#define POLARIS10_I2C_DDC1CLK           1
-#define POLARIS10_I2C_DDC2DATA          2
-#define POLARIS10_I2C_DDC2CLK           3
-#define POLARIS10_I2C_DDC3DATA          4
-#define POLARIS10_I2C_DDC3CLK           5
-#define POLARIS10_I2C_SDA               40
-#define POLARIS10_I2C_SCL               41
-#define POLARIS10_I2C_DDC4DATA          65
-#define POLARIS10_I2C_DDC4CLK           66
-#define POLARIS10_I2C_DDC5DATA          0x48
-#define POLARIS10_I2C_DDC5CLK           0x49
-#define POLARIS10_I2C_DDC6DATA          0x4a
-#define POLARIS10_I2C_DDC6CLK           0x4b
-#define POLARIS10_I2C_DDCVGADATA        0x4c
-#define POLARIS10_I2C_DDCVGACLK         0x4d
-
-#define POLARIS10_UNUSED_GPIO_PIN       0x7F
-
-int polaris10_hwmgr_init(struct pp_hwmgr *hwmgr);
-
-int polaris10_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate);
-int polaris10_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate);
-int polaris10_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable);
-int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, bool bgate);
+#define SMU7_I2C_DDC1DATA          0
+#define SMU7_I2C_DDC1CLK           1
+#define SMU7_I2C_DDC2DATA          2
+#define SMU7_I2C_DDC2CLK           3
+#define SMU7_I2C_DDC3DATA          4
+#define SMU7_I2C_DDC3CLK           5
+#define SMU7_I2C_SDA               40
+#define SMU7_I2C_SCL               41
+#define SMU7_I2C_DDC4DATA          65
+#define SMU7_I2C_DDC4CLK           66
+#define SMU7_I2C_DDC5DATA          0x48
+#define SMU7_I2C_DDC5CLK           0x49
+#define SMU7_I2C_DDC6DATA          0x4a
+#define SMU7_I2C_DDC6CLK           0x4b
+#define SMU7_I2C_DDCVGADATA        0x4c
+#define SMU7_I2C_DDCVGACLK         0x4d
+
+#define SMU7_UNUSED_GPIO_PIN       0x7F
+uint32_t smu7_get_xclk(struct pp_hwmgr *hwmgr);
+uint8_t smu7_get_sleep_divider_id_from_clock(uint32_t clock,
+		uint32_t clock_insr);
 #endif
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
index b9cb240a135d..41b634ffa5b0 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
@@ -20,546 +20,364 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
-
 #include "hwmgr.h"
 #include "smumgr.h"
-#include "polaris10_hwmgr.h"
-#include "polaris10_powertune.h"
-#include "polaris10_smumgr.h"
-#include "smu74_discrete.h"
+#include "smu7_hwmgr.h"
+#include "smu7_powertune.h"
 #include "pp_debug.h"
-#include "gca/gfx_8_0_d.h"
-#include "gca/gfx_8_0_sh_mask.h"
-#include "oss/oss_3_0_sh_mask.h"
+#include "smu7_common.h"
 
 #define VOLTAGE_SCALE  4
-#define POWERTUNE_DEFAULT_SET_MAX    1
 
-uint32_t DIDTBlock_Info = SQ_IR_MASK | TCP_IR_MASK | TD_PCC_MASK;
+static uint32_t DIDTBlock_Info = SQ_IR_MASK | TCP_IR_MASK | TD_PCC_MASK;
 
-struct polaris10_pt_config_reg GCCACConfig_Polaris10[] = {
+static const struct gpu_pt_config_reg GCCACConfig_Polaris10[] = {
 /* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  *      Offset                             Mask                                                Shift                                               Value       Type
  * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  */
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00060013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00860013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01060013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01860013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02060013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02860013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03060013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03860013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x04060013, POLARIS10_CONFIGREG_GC_CAC_IND },
-
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x000E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x008E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x010E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x018E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x020E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
-
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00100013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00900013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01100013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01900013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02100013, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02900013, POLARIS10_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00060013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00860013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01060013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01860013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02060013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02860013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03060013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03860013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x04060013, GPU_CONFIGREG_GC_CAC_IND },
+
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x000E0013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x008E0013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x010E0013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x018E0013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x020E0013, GPU_CONFIGREG_GC_CAC_IND },
+
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00100013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00900013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01100013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01900013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02100013, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02900013, GPU_CONFIGREG_GC_CAC_IND },
 
 	{   0xFFFFFFFF  }
 };
 
-struct polaris10_pt_config_reg GCCACConfig_Polaris11[] = {
+static const struct gpu_pt_config_reg GCCACConfig_Polaris11[] = {
 /* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  *      Offset                             Mask                                                Shift                                               Value       Type
  * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  */
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00060011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00860011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01060011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01860011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02060011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02860011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03060011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03860011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x04060011, POLARIS10_CONFIGREG_GC_CAC_IND },
-
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x000E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x008E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x010E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x018E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x020E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
-
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00100011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00900011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01100011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01900011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02100011, POLARIS10_CONFIGREG_GC_CAC_IND },
-	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02900011, POLARIS10_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00060011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00860011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01060011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01860011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02060011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02860011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03060011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03860011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x04060011, GPU_CONFIGREG_GC_CAC_IND },
+
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x000E0011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x008E0011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x010E0011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x018E0011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x020E0011, GPU_CONFIGREG_GC_CAC_IND },
+
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00100011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00900011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01100011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01900011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02100011, GPU_CONFIGREG_GC_CAC_IND },
+	{   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02900011, GPU_CONFIGREG_GC_CAC_IND },
 
 	{   0xFFFFFFFF  }
 };
 
-struct polaris10_pt_config_reg DIDTConfig_Polaris10[] = {
+static const struct gpu_pt_config_reg DIDTConfig_Polaris10[] = {
 /* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  *      Offset                             Mask                                                Shift                                               Value       Type
  * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  */
-	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT,                  0x0073,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT,                  0x00ab,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0084,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT,                  0x005a,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0067,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT,                  0x0084,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0027,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0046,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT,                 0x00aa,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MIN_POWER_MASK,                      DIDT_SQ_CTRL1__MIN_POWER__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                      DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                    0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3853,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_0_MASK,                       DIDT_SQ_CTRL2__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x005a,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_1_MASK,                       DIDT_SQ_CTRL2__UNUSED_1__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_2_MASK,                       DIDT_SQ_CTRL2__UNUSED_2__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__UNUSED_0_MASK,                  DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x3853,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x3153,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,                   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__UNUSED_0_MASK,                       DIDT_SQ_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT,                  0x000a,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT,                  0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0017,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT,                  0x002f,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0046,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT,                  0x005d,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MIN_POWER_MASK,                      DIDT_TD_CTRL1__MIN_POWER__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MAX_POWER_MASK,                      DIDT_TD_CTRL1__MAX_POWER__SHIFT,                    0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__UNUSED_0_MASK,                    DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0x00ff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_0_MASK,                       DIDT_TD_CTRL2__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x000f,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_1_MASK,                       DIDT_TD_CTRL2__UNUSED_1__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_2_MASK,                       DIDT_TD_CTRL2__UNUSED_2__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__UNUSED_0_MASK,                  DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x0dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x0dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0009,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0009,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__UNUSED_0_MASK,                       DIDT_TD_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT,                 0x0004,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT,                 0x0037,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT,                 0x00ff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT,                 0x0054,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MIN_POWER_MASK,                     DIDT_TCP_CTRL1__MIN_POWER__SHIFT,                   0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MAX_POWER_MASK,                     DIDT_TCP_CTRL1__MAX_POWER__SHIFT,                   0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__UNUSED_0_MASK,                   DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK,              DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT,            0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK,               DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT,             0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_0_MASK,                      DIDT_TCP_CTRL2__UNUSED_0__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,      DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,    0x0032,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_1_MASK,                      DIDT_TCP_CTRL2__UNUSED_1__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,      DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_2_MASK,                      DIDT_TCP_CTRL2__UNUSED_2__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,  DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__UNUSED_0_MASK,                 DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,      DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,    0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,    0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK,                DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT,              0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__UNUSED_0_MASK,                       DIDT_TCP_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT,                  0x0073,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT,                  0x00ab,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0084,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT,                  0x005a,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0067,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT,                  0x0084,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0027,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0046,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT,                 0x00aa,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MIN_POWER_MASK,                      DIDT_SQ_CTRL1__MIN_POWER__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                      DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                    0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3853,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_0_MASK,                       DIDT_SQ_CTRL2__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x005a,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_1_MASK,                       DIDT_SQ_CTRL2__UNUSED_1__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_2_MASK,                       DIDT_SQ_CTRL2__UNUSED_2__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__UNUSED_0_MASK,                  DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x3853,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x3153,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,                   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__UNUSED_0_MASK,                       DIDT_SQ_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT,                  0x000a,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT,                  0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0017,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT,                  0x002f,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0046,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT,                  0x005d,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MIN_POWER_MASK,                      DIDT_TD_CTRL1__MIN_POWER__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MAX_POWER_MASK,                      DIDT_TD_CTRL1__MAX_POWER__SHIFT,                    0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__UNUSED_0_MASK,                    DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0x00ff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_0_MASK,                       DIDT_TD_CTRL2__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x000f,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_1_MASK,                       DIDT_TD_CTRL2__UNUSED_1__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_2_MASK,                       DIDT_TD_CTRL2__UNUSED_2__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__UNUSED_0_MASK,                  DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0009,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0009,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__UNUSED_0_MASK,                       DIDT_TD_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT,                 0x0004,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT,                 0x0037,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT,                 0x00ff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT,                 0x0054,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MIN_POWER_MASK,                     DIDT_TCP_CTRL1__MIN_POWER__SHIFT,                   0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MAX_POWER_MASK,                     DIDT_TCP_CTRL1__MAX_POWER__SHIFT,                   0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__UNUSED_0_MASK,                   DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK,              DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT,            0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK,               DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT,             0x3dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_0_MASK,                      DIDT_TCP_CTRL2__UNUSED_0__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,      DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,    0x0032,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_1_MASK,                      DIDT_TCP_CTRL2__UNUSED_1__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,      DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_2_MASK,                      DIDT_TCP_CTRL2__UNUSED_2__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,  DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__UNUSED_0_MASK,                 DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,      DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,    0x3dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,    0x3dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK,                DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT,              0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__UNUSED_0_MASK,                       DIDT_TCP_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
 
 	{   0xFFFFFFFF  }
 };
 
-struct polaris10_pt_config_reg DIDTConfig_Polaris11[] = {
+static const struct gpu_pt_config_reg DIDTConfig_Polaris11[] = {
 /* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  *      Offset                             Mask                                                Shift                                               Value       Type
  * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  */
-	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT,                  0x0073,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT,                  0x00ab,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0084,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT,                  0x005a,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0067,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT,                  0x0084,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0027,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0046,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT,                 0x00aa,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MIN_POWER_MASK,                      DIDT_SQ_CTRL1__MIN_POWER__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                      DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                    0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3853,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_0_MASK,                       DIDT_SQ_CTRL2__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x005a,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_1_MASK,                       DIDT_SQ_CTRL2__UNUSED_1__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_2_MASK,                       DIDT_SQ_CTRL2__UNUSED_2__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__UNUSED_0_MASK,                  DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x3853,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x3153,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,                   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__UNUSED_0_MASK,                       DIDT_SQ_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT,                  0x000a,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT,                  0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0017,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT,                  0x002f,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0046,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT,                  0x005d,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MIN_POWER_MASK,                      DIDT_TD_CTRL1__MIN_POWER__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MAX_POWER_MASK,                      DIDT_TD_CTRL1__MAX_POWER__SHIFT,                    0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__UNUSED_0_MASK,                    DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0x00ff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_0_MASK,                       DIDT_TD_CTRL2__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x000f,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_1_MASK,                       DIDT_TD_CTRL2__UNUSED_1__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_2_MASK,                       DIDT_TD_CTRL2__UNUSED_2__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__UNUSED_0_MASK,                  DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x0dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x0dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0008,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0008,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__UNUSED_0_MASK,                       DIDT_TD_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT,                 0x0004,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT,                 0x0037,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT,                 0x00ff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT,                 0x0054,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MIN_POWER_MASK,                     DIDT_TCP_CTRL1__MIN_POWER__SHIFT,                   0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MAX_POWER_MASK,                     DIDT_TCP_CTRL1__MAX_POWER__SHIFT,                   0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__UNUSED_0_MASK,                   DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK,              DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT,            0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK,               DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT,             0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_0_MASK,                      DIDT_TCP_CTRL2__UNUSED_0__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,      DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,    0x0032,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_1_MASK,                      DIDT_TCP_CTRL2__UNUSED_1__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,      DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_2_MASK,                      DIDT_TCP_CTRL2__UNUSED_2__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,  DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__UNUSED_0_MASK,                 DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,      DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,    0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,    0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK,                DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT,              0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
-	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__UNUSED_0_MASK,                       DIDT_TCP_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT,                  0x0073,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT,                  0x00ab,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0084,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT,                  0x005a,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0067,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT,                  0x0084,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0027,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0046,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT,                 0x00aa,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MIN_POWER_MASK,                      DIDT_SQ_CTRL1__MIN_POWER__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                      DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                    0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3853,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_0_MASK,                       DIDT_SQ_CTRL2__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x005a,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_1_MASK,                       DIDT_SQ_CTRL2__UNUSED_1__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_2_MASK,                       DIDT_SQ_CTRL2__UNUSED_2__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__UNUSED_0_MASK,                  DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x3853,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x3153,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,                   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__UNUSED_0_MASK,                       DIDT_SQ_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT,                  0x000a,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT,                  0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0017,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT,                  0x002f,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0046,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT,                  0x005d,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MIN_POWER_MASK,                      DIDT_TD_CTRL1__MIN_POWER__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MAX_POWER_MASK,                      DIDT_TD_CTRL1__MAX_POWER__SHIFT,                    0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__UNUSED_0_MASK,                    DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0x00ff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_0_MASK,                       DIDT_TD_CTRL2__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x000f,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_1_MASK,                       DIDT_TD_CTRL2__UNUSED_1__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_2_MASK,                       DIDT_TD_CTRL2__UNUSED_2__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__UNUSED_0_MASK,                  DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x0dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0008,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0008,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__UNUSED_0_MASK,                       DIDT_TD_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT,                 0x0004,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT,                 0x0037,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT,                 0x00ff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT,                 0x0054,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MIN_POWER_MASK,                     DIDT_TCP_CTRL1__MIN_POWER__SHIFT,                   0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MAX_POWER_MASK,                     DIDT_TCP_CTRL1__MAX_POWER__SHIFT,                   0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__UNUSED_0_MASK,                   DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK,              DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT,            0xffff,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK,               DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT,             0x3dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_0_MASK,                      DIDT_TCP_CTRL2__UNUSED_0__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,      DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,    0x0032,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_1_MASK,                      DIDT_TCP_CTRL2__UNUSED_1__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,      DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,    0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_2_MASK,                      DIDT_TCP_CTRL2__UNUSED_2__SHIFT,                    0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,  DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__UNUSED_0_MASK,                 DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT,               0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,      DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,    0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,    0x3dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,    0x3dde,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK,                DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT,              0x0000,     GPU_CONFIGREG_DIDT_IND },
+
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     GPU_CONFIGREG_DIDT_IND },
+	{   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__UNUSED_0_MASK,                       DIDT_TCP_CTRL0__UNUSED_0__SHIFT,                     0x0000,     GPU_CONFIGREG_DIDT_IND },
 	{   0xFFFFFFFF  }
 };
 
-static const struct polaris10_pt_defaults polaris10_power_tune_data_set_array[POWERTUNE_DEFAULT_SET_MAX] = {
-	/* sviLoadLIneEn, SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc, TDC_MAWt,
-	 * TdcWaterfallCtl, DTEAmbientTempBase, DisplayCac, BAPM_TEMP_GRADIENT */
-	{ 1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000,
-	{ 0x79, 0x253, 0x25D, 0xAE, 0x72, 0x80, 0x83, 0x86, 0x6F, 0xC8, 0xC9, 0xC9, 0x2F, 0x4D, 0x61},
-	{ 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 } },
-};
-
-void polaris10_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *polaris10_hwmgr = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct  phm_ppt_v1_information *table_info =
-			(struct  phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (table_info &&
-			table_info->cac_dtp_table->usPowerTuneDataSetID <= POWERTUNE_DEFAULT_SET_MAX &&
-			table_info->cac_dtp_table->usPowerTuneDataSetID)
-		polaris10_hwmgr->power_tune_defaults =
-				&polaris10_power_tune_data_set_array
-				[table_info->cac_dtp_table->usPowerTuneDataSetID - 1];
-	else
-		polaris10_hwmgr->power_tune_defaults = &polaris10_power_tune_data_set_array[0];
-
-}
-
-static uint16_t scale_fan_gain_settings(uint16_t raw_setting)
-{
-	uint32_t tmp;
-	tmp = raw_setting * 4096 / 100;
-	return (uint16_t)tmp;
-}
-
-int polaris10_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	const struct polaris10_pt_defaults *defaults = data->power_tune_defaults;
-	SMU74_Discrete_DpmTable  *dpm_table = &(data->smc_state_table);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_cac_tdp_table *cac_dtp_table = table_info->cac_dtp_table;
-	struct pp_advance_fan_control_parameters *fan_table=
-			&hwmgr->thermal_controller.advanceFanControlParameters;
-	int i, j, k;
-	const uint16_t *pdef1;
-	const uint16_t *pdef2;
-
-	dpm_table->DefaultTdp = PP_HOST_TO_SMC_US((uint16_t)(cac_dtp_table->usTDP * 128));
-	dpm_table->TargetTdp  = PP_HOST_TO_SMC_US((uint16_t)(cac_dtp_table->usTDP * 128));
-
-	PP_ASSERT_WITH_CODE(cac_dtp_table->usTargetOperatingTemp <= 255,
-				"Target Operating Temp is out of Range!",
-				);
-
-	dpm_table->TemperatureLimitEdge = PP_HOST_TO_SMC_US(
-			cac_dtp_table->usTargetOperatingTemp * 256);
-	dpm_table->TemperatureLimitHotspot = PP_HOST_TO_SMC_US(
-			cac_dtp_table->usTemperatureLimitHotspot * 256);
-	dpm_table->FanGainEdge = PP_HOST_TO_SMC_US(
-			scale_fan_gain_settings(fan_table->usFanGainEdge));
-	dpm_table->FanGainHotspot = PP_HOST_TO_SMC_US(
-			scale_fan_gain_settings(fan_table->usFanGainHotspot));
-
-	pdef1 = defaults->BAPMTI_R;
-	pdef2 = defaults->BAPMTI_RC;
-
-	for (i = 0; i < SMU74_DTE_ITERATIONS; i++) {
-		for (j = 0; j < SMU74_DTE_SOURCES; j++) {
-			for (k = 0; k < SMU74_DTE_SINKS; k++) {
-				dpm_table->BAPMTI_R[i][j][k] = PP_HOST_TO_SMC_US(*pdef1);
-				dpm_table->BAPMTI_RC[i][j][k] = PP_HOST_TO_SMC_US(*pdef2);
-				pdef1++;
-				pdef2++;
-			}
-		}
-	}
-
-	return 0;
-}
-
-static int polaris10_populate_svi_load_line(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	const struct polaris10_pt_defaults *defaults = data->power_tune_defaults;
-
-	data->power_tune_table.SviLoadLineEn = defaults->SviLoadLineEn;
-	data->power_tune_table.SviLoadLineVddC = defaults->SviLoadLineVddC;
-	data->power_tune_table.SviLoadLineTrimVddC = 3;
-	data->power_tune_table.SviLoadLineOffsetVddC = 0;
-
-	return 0;
-}
 
-static int polaris10_populate_tdc_limit(struct pp_hwmgr *hwmgr)
-{
-	uint16_t tdc_limit;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	const struct polaris10_pt_defaults *defaults = data->power_tune_defaults;
-
-	tdc_limit = (uint16_t)(table_info->cac_dtp_table->usTDC * 128);
-	data->power_tune_table.TDC_VDDC_PkgLimit =
-			CONVERT_FROM_HOST_TO_SMC_US(tdc_limit);
-	data->power_tune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
-			defaults->TDC_VDDC_ThrottleReleaseLimitPerc;
-	data->power_tune_table.TDC_MAWt = defaults->TDC_MAWt;
-
-	return 0;
-}
-
-static int polaris10_populate_dw8(struct pp_hwmgr *hwmgr, uint32_t fuse_table_offset)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	const struct polaris10_pt_defaults *defaults = data->power_tune_defaults;
-	uint32_t temp;
-
-	if (polaris10_read_smc_sram_dword(hwmgr->smumgr,
-			fuse_table_offset +
-			offsetof(SMU74_Discrete_PmFuses, TdcWaterfallCtl),
-			(uint32_t *)&temp, data->sram_end))
-		PP_ASSERT_WITH_CODE(false,
-				"Attempt to read PmFuses.DW6 (SviLoadLineEn) from SMC Failed!",
-				return -EINVAL);
-	else {
-		data->power_tune_table.TdcWaterfallCtl = defaults->TdcWaterfallCtl;
-		data->power_tune_table.LPMLTemperatureMin =
-				(uint8_t)((temp >> 16) & 0xff);
-		data->power_tune_table.LPMLTemperatureMax =
-				(uint8_t)((temp >> 8) & 0xff);
-		data->power_tune_table.Reserved = (uint8_t)(temp & 0xff);
-	}
-	return 0;
-}
-
-static int polaris10_populate_temperature_scaler(struct pp_hwmgr *hwmgr)
-{
-	int i;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	/* Currently not used. Set all to zero. */
-	for (i = 0; i < 16; i++)
-		data->power_tune_table.LPMLTemperatureScaler[i] = 0;
-
-	return 0;
-}
-
-static int polaris10_populate_fuzzy_fan(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	if ((hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity & (1 << 15))
-		|| 0 == hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity)
-		hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity =
-			hwmgr->thermal_controller.advanceFanControlParameters.usDefaultFanOutputSensitivity;
-
-	data->power_tune_table.FuzzyFan_PwmSetDelta = PP_HOST_TO_SMC_US(
-				hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity);
-	return 0;
-}
-
-static int polaris10_populate_gnb_lpml(struct pp_hwmgr *hwmgr)
-{
-	int i;
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-	/* Currently not used. Set all to zero. */
-	for (i = 0; i < 16; i++)
-		data->power_tune_table.GnbLPML[i] = 0;
-
-	return 0;
-}
-
-static int polaris10_min_max_vgnb_lpml_id_from_bapm_vddc(struct pp_hwmgr *hwmgr)
-{
-	return 0;
-}
-
-static int polaris10_enable_didt(struct pp_hwmgr *hwmgr, const bool enable)
+static int smu7_enable_didt(struct pp_hwmgr *hwmgr, const bool enable)
 {
 
 	uint32_t en = enable ? 1 : 0;
@@ -608,29 +426,29 @@ static int polaris10_enable_didt(struct pp_hwmgr *hwmgr, const bool enable)
 	return result;
 }
 
-static int polaris10_program_pt_config_registers(struct pp_hwmgr *hwmgr,
-				struct polaris10_pt_config_reg *cac_config_regs)
+static int smu7_program_pt_config_registers(struct pp_hwmgr *hwmgr,
+				const struct gpu_pt_config_reg *cac_config_regs)
 {
-	struct polaris10_pt_config_reg *config_regs = cac_config_regs;
+	const struct gpu_pt_config_reg *config_regs = cac_config_regs;
 	uint32_t cache = 0;
 	uint32_t data = 0;
 
 	PP_ASSERT_WITH_CODE((config_regs != NULL), "Invalid config register table.", return -EINVAL);
 
 	while (config_regs->offset != 0xFFFFFFFF) {
-		if (config_regs->type == POLARIS10_CONFIGREG_CACHE)
+		if (config_regs->type == GPU_CONFIGREG_CACHE)
 			cache |= ((config_regs->value << config_regs->shift) & config_regs->mask);
 		else {
 			switch (config_regs->type) {
-			case POLARIS10_CONFIGREG_SMC_IND:
+			case GPU_CONFIGREG_SMC_IND:
 				data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, config_regs->offset);
 				break;
 
-			case POLARIS10_CONFIGREG_DIDT_IND:
+			case GPU_CONFIGREG_DIDT_IND:
 				data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, config_regs->offset);
 				break;
 
-			case POLARIS10_CONFIGREG_GC_CAC_IND:
+			case GPU_CONFIGREG_GC_CAC_IND:
 				data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG_GC_CAC, config_regs->offset);
 				break;
 
@@ -644,15 +462,15 @@ static int polaris10_program_pt_config_registers(struct pp_hwmgr *hwmgr,
 			data |= cache;
 
 			switch (config_regs->type) {
-			case POLARIS10_CONFIGREG_SMC_IND:
+			case GPU_CONFIGREG_SMC_IND:
 				cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, config_regs->offset, data);
 				break;
 
-			case POLARIS10_CONFIGREG_DIDT_IND:
+			case GPU_CONFIGREG_DIDT_IND:
 				cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, config_regs->offset, data);
 				break;
 
-			case POLARIS10_CONFIGREG_GC_CAC_IND:
+			case GPU_CONFIGREG_GC_CAC_IND:
 				cgs_write_ind_register(hwmgr->device, CGS_IND_REG_GC_CAC, config_regs->offset, data);
 				break;
 
@@ -669,7 +487,7 @@ static int polaris10_program_pt_config_registers(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-int polaris10_enable_didt_config(struct pp_hwmgr *hwmgr)
+int smu7_enable_didt_config(struct pp_hwmgr *hwmgr)
 {
 	int result;
 	uint32_t num_se = 0;
@@ -699,20 +517,20 @@ int polaris10_enable_didt_config(struct pp_hwmgr *hwmgr)
 			cgs_write_register(hwmgr->device, mmGRBM_GFX_INDEX, value);
 
 			if (hwmgr->chip_id == CHIP_POLARIS10) {
-				result = polaris10_program_pt_config_registers(hwmgr, GCCACConfig_Polaris10);
+				result = smu7_program_pt_config_registers(hwmgr, GCCACConfig_Polaris10);
 				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
-				result = polaris10_program_pt_config_registers(hwmgr, DIDTConfig_Polaris10);
+				result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_Polaris10);
 				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
 			} else if (hwmgr->chip_id == CHIP_POLARIS11) {
-				result = polaris10_program_pt_config_registers(hwmgr, GCCACConfig_Polaris11);
+				result = smu7_program_pt_config_registers(hwmgr, GCCACConfig_Polaris11);
 				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
-				result = polaris10_program_pt_config_registers(hwmgr, DIDTConfig_Polaris11);
+				result = smu7_program_pt_config_registers(hwmgr, DIDTConfig_Polaris11);
 				PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
 			}
 		}
 		cgs_write_register(hwmgr->device, mmGRBM_GFX_INDEX, value2);
 
-		result = polaris10_enable_didt(hwmgr, true);
+		result = smu7_enable_didt(hwmgr, true);
 		PP_ASSERT_WITH_CODE((result == 0), "EnableDiDt failed.", return result);
 
 		/* TO DO Post DIDT enable clock gating */
@@ -721,7 +539,7 @@ int polaris10_enable_didt_config(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int polaris10_disable_didt_config(struct pp_hwmgr *hwmgr)
+int smu7_disable_didt_config(struct pp_hwmgr *hwmgr)
 {
 	int result;
 
@@ -731,7 +549,7 @@ int polaris10_disable_didt_config(struct pp_hwmgr *hwmgr)
 		phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) {
 		/* TO DO Pre DIDT disable clock gating */
 
-		result = polaris10_enable_didt(hwmgr, false);
+		result = smu7_enable_didt(hwmgr, false);
 		PP_ASSERT_WITH_CODE((result == 0), "Post DIDT enable clock gating failed.", return result);
 		/* TO DO Post DIDT enable clock gating */
 	}
@@ -739,95 +557,9 @@ int polaris10_disable_didt_config(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-
-static int polaris10_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr)
+int smu7_enable_smc_cac(struct pp_hwmgr *hwmgr)
 {
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint16_t hi_sidd = data->power_tune_table.BapmVddCBaseLeakageHiSidd;
-	uint16_t lo_sidd = data->power_tune_table.BapmVddCBaseLeakageLoSidd;
-	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
-
-	hi_sidd = (uint16_t)(cac_table->usHighCACLeakage / 100 * 256);
-	lo_sidd = (uint16_t)(cac_table->usLowCACLeakage / 100 * 256);
-
-	data->power_tune_table.BapmVddCBaseLeakageHiSidd =
-			CONVERT_FROM_HOST_TO_SMC_US(hi_sidd);
-	data->power_tune_table.BapmVddCBaseLeakageLoSidd =
-			CONVERT_FROM_HOST_TO_SMC_US(lo_sidd);
-
-	return 0;
-}
-
-int polaris10_populate_pm_fuses(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-	uint32_t pm_fuse_table_offset;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_PowerContainment)) {
-		if (polaris10_read_smc_sram_dword(hwmgr->smumgr,
-				SMU7_FIRMWARE_HEADER_LOCATION +
-				offsetof(SMU74_Firmware_Header, PmFuseTable),
-				&pm_fuse_table_offset, data->sram_end))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to get pm_fuse_table_offset Failed!",
-					return -EINVAL);
-
-		if (polaris10_populate_svi_load_line(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate SviLoadLine Failed!",
-					return -EINVAL);
-
-		if (polaris10_populate_tdc_limit(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate TDCLimit Failed!", return -EINVAL);
-
-		if (polaris10_populate_dw8(hwmgr, pm_fuse_table_offset))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate TdcWaterfallCtl, "
-					"LPMLTemperature Min and Max Failed!",
-					return -EINVAL);
-
-		if (0 != polaris10_populate_temperature_scaler(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate LPMLTemperatureScaler Failed!",
-					return -EINVAL);
-
-		if (polaris10_populate_fuzzy_fan(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate Fuzzy Fan Control parameters Failed!",
-					return -EINVAL);
-
-		if (polaris10_populate_gnb_lpml(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate GnbLPML Failed!",
-					return -EINVAL);
-
-		if (polaris10_min_max_vgnb_lpml_id_from_bapm_vddc(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate GnbLPML Min and Max Vid Failed!",
-					return -EINVAL);
-
-		if (polaris10_populate_bapm_vddc_base_leakage_sidd(hwmgr))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to populate BapmVddCBaseLeakage Hi and Lo "
-					"Sidd Failed!", return -EINVAL);
-
-		if (polaris10_copy_bytes_to_smc(hwmgr->smumgr, pm_fuse_table_offset,
-				(uint8_t *)&data->power_tune_table,
-				(sizeof(struct SMU74_Discrete_PmFuses) - 92), data->sram_end))
-			PP_ASSERT_WITH_CODE(false,
-					"Attempt to download PmFuseTable Failed!",
-					return -EINVAL);
-	}
-	return 0;
-}
-
-int polaris10_enable_smc_cac(struct pp_hwmgr *hwmgr)
-{
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	int result = 0;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
@@ -843,9 +575,9 @@ int polaris10_enable_smc_cac(struct pp_hwmgr *hwmgr)
 	return result;
 }
 
-int polaris10_disable_smc_cac(struct pp_hwmgr *hwmgr)
+int smu7_disable_smc_cac(struct pp_hwmgr *hwmgr)
 {
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	int result = 0;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
@@ -860,9 +592,9 @@ int polaris10_disable_smc_cac(struct pp_hwmgr *hwmgr)
 	return result;
 }
 
-int polaris10_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n)
+int smu7_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n)
 {
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
 	if (data->power_containment_features &
 			POWERCONTAINMENT_FEATURE_PkgPwrLimit)
@@ -871,21 +603,27 @@ int polaris10_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n)
 	return 0;
 }
 
-static int polaris10_set_overdriver_target_tdp(struct pp_hwmgr *pHwMgr, uint32_t target_tdp)
+static int smu7_set_overdriver_target_tdp(struct pp_hwmgr *pHwMgr, uint32_t target_tdp)
 {
 	return smum_send_msg_to_smc_with_parameter(pHwMgr->smumgr,
 			PPSMC_MSG_OverDriveSetTargetTdp, target_tdp);
 }
 
-int polaris10_enable_power_containment(struct pp_hwmgr *hwmgr)
+int smu7_enable_power_containment(struct pp_hwmgr *hwmgr)
 {
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	struct phm_ppt_v1_information *table_info =
 			(struct phm_ppt_v1_information *)(hwmgr->pptable);
 	int smc_result;
 	int result = 0;
+	struct phm_cac_tdp_table *cac_table;
 
 	data->power_containment_features = 0;
+	if (hwmgr->pp_table_version == PP_TABLE_V1)
+		cac_table = table_info->cac_dtp_table;
+	else
+		cac_table = hwmgr->dyn_state.cac_dtp_table;
+
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_PowerContainment)) {
 
@@ -905,15 +643,13 @@ int polaris10_enable_power_containment(struct pp_hwmgr *hwmgr)
 			PP_ASSERT_WITH_CODE((0 == smc_result),
 					"Failed to enable PkgPwrTracking in SMC.", result = -1;);
 			if (0 == smc_result) {
-				struct phm_cac_tdp_table *cac_table =
-						table_info->cac_dtp_table;
 				uint32_t default_limit =
 					(uint32_t)(cac_table->usMaximumPowerDeliveryLimit * 256);
 
 				data->power_containment_features |=
 						POWERCONTAINMENT_FEATURE_PkgPwrLimit;
 
-				if (polaris10_set_power_limit(hwmgr, default_limit))
+				if (smu7_set_power_limit(hwmgr, default_limit))
 					printk(KERN_ERR "Failed to set Default Power Limit in SMC!");
 			}
 		}
@@ -921,9 +657,9 @@ int polaris10_enable_power_containment(struct pp_hwmgr *hwmgr)
 	return result;
 }
 
-int polaris10_disable_power_containment(struct pp_hwmgr *hwmgr)
+int smu7_disable_power_containment(struct pp_hwmgr *hwmgr)
 {
-	struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	int result = 0;
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
@@ -963,14 +699,19 @@ int polaris10_disable_power_containment(struct pp_hwmgr *hwmgr)
 	return result;
 }
 
-int polaris10_power_control_set_level(struct pp_hwmgr *hwmgr)
+int smu7_power_control_set_level(struct pp_hwmgr *hwmgr)
 {
 	struct phm_ppt_v1_information *table_info =
 			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
+	struct phm_cac_tdp_table *cac_table;
+
 	int adjust_percent, target_tdp;
 	int result = 0;
 
+	if (hwmgr->pp_table_version == PP_TABLE_V1)
+		cac_table = table_info->cac_dtp_table;
+	else
+		cac_table = hwmgr->dyn_state.cac_dtp_table;
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_PowerContainment)) {
 		/* adjustment percentage has already been validated */
@@ -981,7 +722,7 @@ int polaris10_power_control_set_level(struct pp_hwmgr *hwmgr)
 		 * but message to be 8 bit fraction for messages
 		 */
 		target_tdp = ((100 + adjust_percent) * (int)(cac_table->usTDP * 256)) / 100;
-		result = polaris10_set_overdriver_target_tdp(hwmgr, (uint32_t)target_tdp);
+		result = smu7_set_overdriver_target_tdp(hwmgr, (uint32_t)target_tdp);
 	}
 
 	return result;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.h
index bc78e28f010d..22f86b6bf1be 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.h
@@ -20,17 +20,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
-#ifndef POLARIS10_POWERTUNE_H
-#define POLARIS10_POWERTUNE_H
-
-enum polaris10_pt_config_reg_type {
-	POLARIS10_CONFIGREG_MMR = 0,
-	POLARIS10_CONFIGREG_SMC_IND,
-	POLARIS10_CONFIGREG_DIDT_IND,
-	POLARIS10_CONFIGREG_GC_CAC_IND,
-	POLARIS10_CONFIGREG_CACHE,
-	POLARIS10_CONFIGREG_MAX
-};
+#ifndef _SMU7_POWERTUNE_H
+#define _SMU7_POWERTUNE_H
 
 #define DIDT_SQ_CTRL0__UNUSED_0_MASK    0xfffc0000
 #define DIDT_SQ_CTRL0__UNUSED_0__SHIFT  0x12
@@ -52,43 +43,20 @@ enum polaris10_pt_config_reg_type {
 
 #define ixGC_CAC_CNTL 0x0000
 #define ixDIDT_SQ_STALL_CTRL 0x0004
-#define  ixDIDT_SQ_TUNING_CTRL 0x0005
+#define ixDIDT_SQ_TUNING_CTRL 0x0005
 #define ixDIDT_TD_STALL_CTRL 0x0044
 #define ixDIDT_TD_TUNING_CTRL 0x0045
 #define ixDIDT_TCP_STALL_CTRL 0x0064
 #define ixDIDT_TCP_TUNING_CTRL 0x0065
 
-struct polaris10_pt_config_reg {
-	uint32_t                           offset;
-	uint32_t                           mask;
-	uint32_t                           shift;
-	uint32_t                           value;
-	enum polaris10_pt_config_reg_type       type;
-};
-
-struct polaris10_pt_defaults {
-	uint8_t   SviLoadLineEn;
-	uint8_t   SviLoadLineVddC;
-	uint8_t   TDC_VDDC_ThrottleReleaseLimitPerc;
-	uint8_t   TDC_MAWt;
-	uint8_t   TdcWaterfallCtl;
-	uint8_t   DTEAmbientTempBase;
-
-	uint32_t  DisplayCac;
-	uint32_t  BAPM_TEMP_GRADIENT;
-	uint16_t  BAPMTI_R[SMU74_DTE_ITERATIONS * SMU74_DTE_SOURCES * SMU74_DTE_SINKS];
-	uint16_t  BAPMTI_RC[SMU74_DTE_ITERATIONS * SMU74_DTE_SOURCES * SMU74_DTE_SINKS];
-};
 
-void polaris10_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr);
-int polaris10_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr);
-int polaris10_populate_pm_fuses(struct pp_hwmgr *hwmgr);
-int polaris10_enable_smc_cac(struct pp_hwmgr *hwmgr);
-int polaris10_disable_smc_cac(struct pp_hwmgr *hwmgr);
-int polaris10_enable_power_containment(struct pp_hwmgr *hwmgr);
-int polaris10_disable_power_containment(struct pp_hwmgr *hwmgr);
-int polaris10_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n);
-int polaris10_power_control_set_level(struct pp_hwmgr *hwmgr);
-int polaris10_enable_didt_config(struct pp_hwmgr *hwmgr);
-#endif  /* POLARIS10_POWERTUNE_H */
+int smu7_enable_smc_cac(struct pp_hwmgr *hwmgr);
+int smu7_disable_smc_cac(struct pp_hwmgr *hwmgr);
+int smu7_enable_power_containment(struct pp_hwmgr *hwmgr);
+int smu7_disable_power_containment(struct pp_hwmgr *hwmgr);
+int smu7_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n);
+int smu7_power_control_set_level(struct pp_hwmgr *hwmgr);
+int smu7_enable_didt_config(struct pp_hwmgr *hwmgr);
+int smu7_disable_didt_config(struct pp_hwmgr *hwmgr);
+#endif  /* DGPU_POWERTUNE_H */
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.c
index 92976b68d6fd..fb6c6f6106d5 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_thermal.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015 Advanced Micro Devices, Inc.
+ * Copyright 2016 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -20,18 +20,15 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+
 #include <asm/div64.h>
-#include "fiji_thermal.h"
-#include "fiji_hwmgr.h"
-#include "fiji_smumgr.h"
-#include "fiji_ppsmc.h"
-#include "smu/smu_7_1_3_d.h"
-#include "smu/smu_7_1_3_sh_mask.h"
-
-int fiji_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr,
+#include "smu7_thermal.h"
+#include "smu7_hwmgr.h"
+#include "smu7_common.h"
+
+int smu7_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr,
 		struct phm_fan_speed_info *fan_speed_info)
 {
-
 	if (hwmgr->thermal_controller.fanInfo.bNoFan)
 		return 0;
 
@@ -55,7 +52,7 @@ int fiji_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-int fiji_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr,
+int smu7_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr,
 		uint32_t *speed)
 {
 	uint32_t duty100;
@@ -84,7 +81,7 @@ int fiji_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-int fiji_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
+int smu7_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
 {
 	uint32_t tach_period;
 	uint32_t crystal_clock_freq;
@@ -100,9 +97,9 @@ int fiji_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
 	if (tach_period == 0)
 		return -EINVAL;
 
-	crystal_clock_freq = tonga_get_xclk(hwmgr);
+	crystal_clock_freq = smu7_get_xclk(hwmgr);
 
-	*speed = 60 * crystal_clock_freq * 10000/ tach_period;
+	*speed = 60 * crystal_clock_freq * 10000 / tach_period;
 
 	return 0;
 }
@@ -113,7 +110,7 @@ int fiji_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
 *           mode    the fan control mode, 0 default, 1 by percent, 5, by RPM
 * @exception Should always succeed.
 */
-int fiji_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
+int smu7_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
 {
 
 	if (hwmgr->fan_ctrl_is_in_default_mode) {
@@ -139,7 +136,7 @@ int fiji_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
 * @param    hwmgr  the address of the powerplay hardware manager.
 * @exception Should always succeed.
 */
-int fiji_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr)
+int smu7_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr)
 {
 	if (!hwmgr->fan_ctrl_is_in_default_mode) {
 		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
@@ -152,7 +149,7 @@ int fiji_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int fiji_fan_ctrl_start_smc_fan_control(struct pp_hwmgr *hwmgr)
+static int smu7_fan_ctrl_start_smc_fan_control(struct pp_hwmgr *hwmgr)
 {
 	int result;
 
@@ -187,7 +184,7 @@ int fiji_fan_ctrl_start_smc_fan_control(struct pp_hwmgr *hwmgr)
 }
 
 
-int fiji_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr)
+int smu7_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr)
 {
 	return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StopFanControl);
 }
@@ -198,7 +195,7 @@ int fiji_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr)
 * @param    speed is the percentage value (0% - 100%) to be set.
 * @exception Fails is the 100% setting appears to be 0.
 */
-int fiji_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
+int smu7_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
 		uint32_t speed)
 {
 	uint32_t duty100;
@@ -213,7 +210,7 @@ int fiji_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_MicrocodeFanControl))
-		fiji_fan_ctrl_stop_smc_fan_control(hwmgr);
+		smu7_fan_ctrl_stop_smc_fan_control(hwmgr);
 
 	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
 			CG_FDO_CTRL1, FMAX_DUTY100);
@@ -228,7 +225,7 @@ int fiji_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
 	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
 			CG_FDO_CTRL0, FDO_STATIC_DUTY, duty);
 
-	return fiji_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+	return smu7_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
 }
 
 /**
@@ -236,7 +233,7 @@ int fiji_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr,
 * @param    hwmgr  the address of the powerplay hardware manager.
 * @exception Always succeeds.
 */
-int fiji_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
+int smu7_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
 {
 	int result;
 
@@ -245,11 +242,11 @@ int fiji_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_MicrocodeFanControl)) {
-		result = fiji_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+		result = smu7_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
 		if (!result)
-			result = fiji_fan_ctrl_start_smc_fan_control(hwmgr);
+			result = smu7_fan_ctrl_start_smc_fan_control(hwmgr);
 	} else
-		result = fiji_fan_ctrl_set_default_mode(hwmgr);
+		result = smu7_fan_ctrl_set_default_mode(hwmgr);
 
 	return result;
 }
@@ -260,7 +257,7 @@ int fiji_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
 * @param    speed is the percentage value (min - max) to be set.
 * @exception Fails is the speed not lie between min and max.
 */
-int fiji_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
+int smu7_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
 {
 	uint32_t tach_period;
 	uint32_t crystal_clock_freq;
@@ -272,14 +269,18 @@ int fiji_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
 			(speed > hwmgr->thermal_controller.fanInfo.ulMaxRPM))
 		return 0;
 
-	crystal_clock_freq = tonga_get_xclk(hwmgr);
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MicrocodeFanControl))
+		smu7_fan_ctrl_stop_smc_fan_control(hwmgr);
+
+	crystal_clock_freq = smu7_get_xclk(hwmgr);
 
 	tach_period = 60 * crystal_clock_freq * 10000 / (8 * speed);
 
 	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
 				CG_TACH_STATUS, TACH_PERIOD, tach_period);
 
-	return fiji_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+	return smu7_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
 }
 
 /**
@@ -287,7 +288,7 @@ int fiji_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
 *
 * @param    hwmgr The address of the hardware manager.
 */
-int fiji_thermal_get_temperature(struct pp_hwmgr *hwmgr)
+int smu7_thermal_get_temperature(struct pp_hwmgr *hwmgr)
 {
 	int temp;
 
@@ -296,7 +297,7 @@ int fiji_thermal_get_temperature(struct pp_hwmgr *hwmgr)
 
 	/* Bit 9 means the reading is lower than the lowest usable value. */
 	if (temp & 0x200)
-		temp = FIJI_THERMAL_MAXIMUM_TEMP_READING;
+		temp = SMU7_THERMAL_MAXIMUM_TEMP_READING;
 	else
 		temp = temp & 0x1ff;
 
@@ -312,12 +313,12 @@ int fiji_thermal_get_temperature(struct pp_hwmgr *hwmgr)
 * @param    range Temperature range to be programmed for high and low alert signals
 * @exception PP_Result_BadInput if the input data is not valid.
 */
-static int fiji_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
+static int smu7_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 		uint32_t low_temp, uint32_t high_temp)
 {
-	uint32_t low = FIJI_THERMAL_MINIMUM_ALERT_TEMP *
+	uint32_t low = SMU7_THERMAL_MINIMUM_ALERT_TEMP *
 			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
-	uint32_t high = FIJI_THERMAL_MAXIMUM_ALERT_TEMP *
+	uint32_t high = SMU7_THERMAL_MAXIMUM_ALERT_TEMP *
 			PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
 
 	if (low < low_temp)
@@ -346,7 +347,7 @@ static int fiji_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 *
 * @param    hwmgr The address of the hardware manager.
 */
-static int fiji_thermal_initialize(struct pp_hwmgr *hwmgr)
+static int smu7_thermal_initialize(struct pp_hwmgr *hwmgr)
 {
 	if (hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution)
 		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
@@ -365,13 +366,13 @@ static int fiji_thermal_initialize(struct pp_hwmgr *hwmgr)
 *
 * @param    hwmgr The address of the hardware manager.
 */
-static int fiji_thermal_enable_alert(struct pp_hwmgr *hwmgr)
+int smu7_thermal_enable_alert(struct pp_hwmgr *hwmgr)
 {
 	uint32_t alert;
 
 	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
 			CG_THERMAL_INT, THERM_INT_MASK);
-	alert &= ~(FIJI_THERMAL_HIGH_ALERT_MASK | FIJI_THERMAL_LOW_ALERT_MASK);
+	alert &= ~(SMU7_THERMAL_HIGH_ALERT_MASK | SMU7_THERMAL_LOW_ALERT_MASK);
 	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
 			CG_THERMAL_INT, THERM_INT_MASK, alert);
 
@@ -383,13 +384,13 @@ static int fiji_thermal_enable_alert(struct pp_hwmgr *hwmgr)
 * Disable thermal alerts on the RV770 thermal controller.
 * @param    hwmgr The address of the hardware manager.
 */
-static int fiji_thermal_disable_alert(struct pp_hwmgr *hwmgr)
+int smu7_thermal_disable_alert(struct pp_hwmgr *hwmgr)
 {
 	uint32_t alert;
 
 	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
 			CG_THERMAL_INT, THERM_INT_MASK);
-	alert |= (FIJI_THERMAL_HIGH_ALERT_MASK | FIJI_THERMAL_LOW_ALERT_MASK);
+	alert |= (SMU7_THERMAL_HIGH_ALERT_MASK | SMU7_THERMAL_LOW_ALERT_MASK);
 	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
 			CG_THERMAL_INT, THERM_INT_MASK, alert);
 
@@ -402,129 +403,17 @@ static int fiji_thermal_disable_alert(struct pp_hwmgr *hwmgr)
 * Currently just disables alerts.
 * @param    hwmgr The address of the hardware manager.
 */
-int fiji_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr)
+int smu7_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr)
 {
-	int result = fiji_thermal_disable_alert(hwmgr);
+	int result = smu7_thermal_disable_alert(hwmgr);
 
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		fiji_fan_ctrl_set_default_mode(hwmgr);
+	if (!hwmgr->thermal_controller.fanInfo.bNoFan)
+		smu7_fan_ctrl_set_default_mode(hwmgr);
 
 	return result;
 }
 
 /**
-* Set up the fan table to control the fan using the SMC.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from set temperature range routine
-*/
-int tf_fiji_thermal_setup_fan_table(struct pp_hwmgr *hwmgr,
-		void *input, void *output, void *storage, int result)
-{
-	struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-	SMU73_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
-	uint32_t duty100;
-	uint32_t t_diff1, t_diff2, pwm_diff1, pwm_diff2;
-	uint16_t fdo_min, slope1, slope2;
-	uint32_t reference_clock;
-	int res;
-	uint64_t tmp64;
-
-	if (data->fan_table_start == 0) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_MicrocodeFanControl);
-		return 0;
-	}
-
-	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-			CG_FDO_CTRL1, FMAX_DUTY100);
-
-	if (duty100 == 0) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_MicrocodeFanControl);
-		return 0;
-	}
-
-	tmp64 = hwmgr->thermal_controller.advanceFanControlParameters.
-			usPWMMin * duty100;
-	do_div(tmp64, 10000);
-	fdo_min = (uint16_t)tmp64;
-
-	t_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usTMed -
-			hwmgr->thermal_controller.advanceFanControlParameters.usTMin;
-	t_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usTHigh -
-			hwmgr->thermal_controller.advanceFanControlParameters.usTMed;
-
-	pwm_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed -
-			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin;
-	pwm_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh -
-			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed;
-
-	slope1 = (uint16_t)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
-	slope2 = (uint16_t)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
-
-	fan_table.TempMin = cpu_to_be16((50 + hwmgr->
-			thermal_controller.advanceFanControlParameters.usTMin) / 100);
-	fan_table.TempMed = cpu_to_be16((50 + hwmgr->
-			thermal_controller.advanceFanControlParameters.usTMed) / 100);
-	fan_table.TempMax = cpu_to_be16((50 + hwmgr->
-			thermal_controller.advanceFanControlParameters.usTMax) / 100);
-
-	fan_table.Slope1 = cpu_to_be16(slope1);
-	fan_table.Slope2 = cpu_to_be16(slope2);
-
-	fan_table.FdoMin = cpu_to_be16(fdo_min);
-
-	fan_table.HystDown = cpu_to_be16(hwmgr->
-			thermal_controller.advanceFanControlParameters.ucTHyst);
-
-	fan_table.HystUp = cpu_to_be16(1);
-
-	fan_table.HystSlope = cpu_to_be16(1);
-
-	fan_table.TempRespLim = cpu_to_be16(5);
-
-	reference_clock = tonga_get_xclk(hwmgr);
-
-	fan_table.RefreshPeriod = cpu_to_be32((hwmgr->
-			thermal_controller.advanceFanControlParameters.ulCycleDelay *
-			reference_clock) / 1600);
-
-	fan_table.FdoMax = cpu_to_be16((uint16_t)duty100);
-
-	fan_table.TempSrc = (uint8_t)PHM_READ_VFPF_INDIRECT_FIELD(
-			hwmgr->device, CGS_IND_REG__SMC,
-			CG_MULT_THERMAL_CTRL, TEMP_SEL);
-
-	res = fiji_copy_bytes_to_smc(hwmgr->smumgr, data->fan_table_start,
-			(uint8_t *)&fan_table, (uint32_t)sizeof(fan_table),
-			data->sram_end);
-
-	if (!res && hwmgr->thermal_controller.
-			advanceFanControlParameters.ucMinimumPWMLimit)
-		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-				PPSMC_MSG_SetFanMinPwm,
-				hwmgr->thermal_controller.
-				advanceFanControlParameters.ucMinimumPWMLimit);
-
-	if (!res && hwmgr->thermal_controller.
-			advanceFanControlParameters.ulMinFanSCLKAcousticLimit)
-		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-				PPSMC_MSG_SetFanSclkTarget,
-				hwmgr->thermal_controller.
-				advanceFanControlParameters.ulMinFanSCLKAcousticLimit);
-
-	if (res)
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_MicrocodeFanControl);
-
-	return 0;
-}
-
-/**
 * Start the fan control on the SMC.
 * @param    hwmgr  the address of the powerplay hardware manager.
 * @param    pInput the pointer to input data
@@ -533,7 +422,7 @@ int tf_fiji_thermal_setup_fan_table(struct pp_hwmgr *hwmgr,
 * @param    Result the last failure code
 * @return   result from set temperature range routine
 */
-int tf_fiji_thermal_start_smc_fan_control(struct pp_hwmgr *hwmgr,
+static int tf_smu7_thermal_start_smc_fan_control(struct pp_hwmgr *hwmgr,
 		void *input, void *output, void *storage, int result)
 {
 /* If the fantable setup has failed we could have disabled
@@ -543,8 +432,8 @@ int tf_fiji_thermal_start_smc_fan_control(struct pp_hwmgr *hwmgr,
 */
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_MicrocodeFanControl)) {
-		fiji_fan_ctrl_start_smc_fan_control(hwmgr);
-		fiji_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
+		smu7_fan_ctrl_start_smc_fan_control(hwmgr);
+		smu7_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
 	}
 
 	return 0;
@@ -559,7 +448,7 @@ int tf_fiji_thermal_start_smc_fan_control(struct pp_hwmgr *hwmgr,
 * @param    Result the last failure code
 * @return   result from set temperature range routine
 */
-int tf_fiji_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
+static int tf_smu7_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 		void *input, void *output, void *storage, int result)
 {
 	struct PP_TemperatureRange *range = (struct PP_TemperatureRange *)input;
@@ -567,7 +456,7 @@ int tf_fiji_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 	if (range == NULL)
 		return -EINVAL;
 
-	return fiji_thermal_set_temperature_range(hwmgr, range->min, range->max);
+	return smu7_thermal_set_temperature_range(hwmgr, range->min, range->max);
 }
 
 /**
@@ -579,10 +468,10 @@ int tf_fiji_thermal_set_temperature_range(struct pp_hwmgr *hwmgr,
 * @param    Result the last failure code
 * @return   result from initialize thermal controller routine
 */
-int tf_fiji_thermal_initialize(struct pp_hwmgr *hwmgr,
+static int tf_smu7_thermal_initialize(struct pp_hwmgr *hwmgr,
 		void *input, void *output, void *storage, int result)
 {
-    return fiji_thermal_initialize(hwmgr);
+	return smu7_thermal_initialize(hwmgr);
 }
 
 /**
@@ -594,10 +483,10 @@ int tf_fiji_thermal_initialize(struct pp_hwmgr *hwmgr,
 * @param    Result the last failure code
 * @return   result from enable alert routine
 */
-int tf_fiji_thermal_enable_alert(struct pp_hwmgr *hwmgr,
+static int tf_smu7_thermal_enable_alert(struct pp_hwmgr *hwmgr,
 		void *input, void *output, void *storage, int result)
 {
-	return fiji_thermal_enable_alert(hwmgr);
+	return smu7_thermal_enable_alert(hwmgr);
 }
 
 /**
@@ -609,53 +498,54 @@ int tf_fiji_thermal_enable_alert(struct pp_hwmgr *hwmgr,
 * @param    Result the last failure code
 * @return   result from disable alert routine
 */
-static int tf_fiji_thermal_disable_alert(struct pp_hwmgr *hwmgr,
+static int tf_smu7_thermal_disable_alert(struct pp_hwmgr *hwmgr,
 		void *input, void *output, void *storage, int result)
 {
-	return fiji_thermal_disable_alert(hwmgr);
+	return smu7_thermal_disable_alert(hwmgr);
 }
 
 static const struct phm_master_table_item
-fiji_thermal_start_thermal_controller_master_list[] = {
-	{NULL, tf_fiji_thermal_initialize},
-	{NULL, tf_fiji_thermal_set_temperature_range},
-	{NULL, tf_fiji_thermal_enable_alert},
+phm_thermal_start_thermal_controller_master_list[] = {
+	{NULL, tf_smu7_thermal_initialize},
+	{NULL, tf_smu7_thermal_set_temperature_range},
+	{NULL, tf_smu7_thermal_enable_alert},
+	{NULL, smum_thermal_avfs_enable},
 /* We should restrict performance levels to low before we halt the SMC.
  * On the other hand we are still in boot state when we do this
  * so it would be pointless.
  * If this assumption changes we have to revisit this table.
  */
-	{NULL, tf_fiji_thermal_setup_fan_table},
-	{NULL, tf_fiji_thermal_start_smc_fan_control},
+	{NULL, smum_thermal_setup_fan_table},
+	{NULL, tf_smu7_thermal_start_smc_fan_control},
 	{NULL, NULL}
 };
 
 static const struct phm_master_table_header
-fiji_thermal_start_thermal_controller_master = {
+phm_thermal_start_thermal_controller_master = {
 	0,
 	PHM_MasterTableFlag_None,
-	fiji_thermal_start_thermal_controller_master_list
+	phm_thermal_start_thermal_controller_master_list
 };
 
 static const struct phm_master_table_item
-fiji_thermal_set_temperature_range_master_list[] = {
-	{NULL, tf_fiji_thermal_disable_alert},
-	{NULL, tf_fiji_thermal_set_temperature_range},
-	{NULL, tf_fiji_thermal_enable_alert},
+phm_thermal_set_temperature_range_master_list[] = {
+	{NULL, tf_smu7_thermal_disable_alert},
+	{NULL, tf_smu7_thermal_set_temperature_range},
+	{NULL, tf_smu7_thermal_enable_alert},
 	{NULL, NULL}
 };
 
 static const struct phm_master_table_header
-fiji_thermal_set_temperature_range_master = {
+phm_thermal_set_temperature_range_master = {
 	0,
 	PHM_MasterTableFlag_None,
-	fiji_thermal_set_temperature_range_master_list
+	phm_thermal_set_temperature_range_master_list
 };
 
-int fiji_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr)
+int smu7_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr)
 {
 	if (!hwmgr->thermal_controller.fanInfo.bNoFan)
-		fiji_fan_ctrl_set_default_mode(hwmgr);
+		smu7_fan_ctrl_set_default_mode(hwmgr);
 	return 0;
 }
 
@@ -664,17 +554,17 @@ int fiji_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr)
 * @param    hwmgr The address of the hardware manager.
 * @exception Any error code from the low-level communication.
 */
-int pp_fiji_thermal_initialize(struct pp_hwmgr *hwmgr)
+int pp_smu7_thermal_initialize(struct pp_hwmgr *hwmgr)
 {
 	int result;
 
 	result = phm_construct_table(hwmgr,
-			&fiji_thermal_set_temperature_range_master,
+			&phm_thermal_set_temperature_range_master,
 			&(hwmgr->set_temperature_range));
 
 	if (!result) {
 		result = phm_construct_table(hwmgr,
-				&fiji_thermal_start_thermal_controller_master,
+				&phm_thermal_start_thermal_controller_master,
 				&(hwmgr->start_thermal_controller));
 		if (result)
 			phm_destroy_table(hwmgr, &(hwmgr->set_temperature_range));
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.h
new file mode 100644
index 000000000000..6face973be43
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_thermal.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _SMU7_THERMAL_H_
+#define _SMU7_THERMAL_H_
+
+#include "hwmgr.h"
+
+#define SMU7_THERMAL_HIGH_ALERT_MASK         0x1
+#define SMU7_THERMAL_LOW_ALERT_MASK          0x2
+
+#define SMU7_THERMAL_MINIMUM_TEMP_READING    -256
+#define SMU7_THERMAL_MAXIMUM_TEMP_READING    255
+
+#define SMU7_THERMAL_MINIMUM_ALERT_TEMP      0
+#define SMU7_THERMAL_MAXIMUM_ALERT_TEMP      255
+
+#define FDO_PWM_MODE_STATIC  1
+#define FDO_PWM_MODE_STATIC_RPM 5
+
+extern int smu7_thermal_get_temperature(struct pp_hwmgr *hwmgr);
+extern int smu7_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr);
+extern int smu7_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info);
+extern int smu7_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t *speed);
+extern int smu7_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr);
+extern int smu7_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode);
+extern int smu7_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed);
+extern int smu7_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr);
+extern int pp_smu7_thermal_initialize(struct pp_hwmgr *hwmgr);
+extern int smu7_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr);
+extern int smu7_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed);
+extern int smu7_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed);
+extern int smu7_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr);
+extern int smu7_thermal_enable_alert(struct pp_hwmgr *hwmgr);
+extern int smu7_thermal_disable_alert(struct pp_hwmgr *hwmgr);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.c
deleted file mode 100644
index e58d038a997b..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.c
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "hwmgr.h"
-#include "tonga_clockpowergating.h"
-#include "tonga_ppsmc.h"
-#include "tonga_hwmgr.h"
-
-int tonga_phm_powerdown_uvd(struct pp_hwmgr *hwmgr)
-{
-	if (phm_cf_want_uvd_power_gating(hwmgr))
-		return smum_send_msg_to_smc(hwmgr->smumgr,
-						     PPSMC_MSG_UVDPowerOFF);
-	return 0;
-}
-
-int tonga_phm_powerup_uvd(struct pp_hwmgr *hwmgr)
-{
-	if (phm_cf_want_uvd_power_gating(hwmgr)) {
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				  PHM_PlatformCaps_UVDDynamicPowerGating)) {
-			return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-								PPSMC_MSG_UVDPowerON, 1);
-		} else {
-			return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-								PPSMC_MSG_UVDPowerON, 0);
-		}
-	}
-
-	return 0;
-}
-
-int tonga_phm_powerdown_vce(struct pp_hwmgr *hwmgr)
-{
-	if (phm_cf_want_vce_power_gating(hwmgr))
-		return smum_send_msg_to_smc(hwmgr->smumgr,
-						  PPSMC_MSG_VCEPowerOFF);
-	return 0;
-}
-
-int tonga_phm_powerup_vce(struct pp_hwmgr *hwmgr)
-{
-	if (phm_cf_want_vce_power_gating(hwmgr))
-		return smum_send_msg_to_smc(hwmgr->smumgr,
-						  PPSMC_MSG_VCEPowerON);
-	return 0;
-}
-
-int tonga_phm_set_asic_block_gating(struct pp_hwmgr *hwmgr, enum PHM_AsicBlock block, enum PHM_ClockGateSetting gating)
-{
-	int ret = 0;
-
-	switch (block) {
-	case PHM_AsicBlock_UVD_MVC:
-	case PHM_AsicBlock_UVD:
-	case PHM_AsicBlock_UVD_HD:
-	case PHM_AsicBlock_UVD_SD:
-		if (gating == PHM_ClockGateSetting_StaticOff)
-			ret = tonga_phm_powerdown_uvd(hwmgr);
-		else
-			ret = tonga_phm_powerup_uvd(hwmgr);
-		break;
-	case PHM_AsicBlock_GFX:
-	default:
-		break;
-	}
-
-	return ret;
-}
-
-int tonga_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	data->uvd_power_gated = false;
-	data->vce_power_gated = false;
-
-	tonga_phm_powerup_uvd(hwmgr);
-	tonga_phm_powerup_vce(hwmgr);
-
-	return 0;
-}
-
-int tonga_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	if (data->uvd_power_gated == bgate)
-		return 0;
-
-	data->uvd_power_gated = bgate;
-
-	if (bgate) {
-		cgs_set_clockgating_state(hwmgr->device,
-						AMD_IP_BLOCK_TYPE_UVD,
-						AMD_CG_STATE_UNGATE);
-		cgs_set_powergating_state(hwmgr->device,
-						AMD_IP_BLOCK_TYPE_UVD,
-						AMD_PG_STATE_GATE);
-		tonga_update_uvd_dpm(hwmgr, true);
-		tonga_phm_powerdown_uvd(hwmgr);
-	} else {
-		tonga_phm_powerup_uvd(hwmgr);
-		cgs_set_powergating_state(hwmgr->device,
-						AMD_IP_BLOCK_TYPE_UVD,
-						AMD_PG_STATE_UNGATE);
-		cgs_set_clockgating_state(hwmgr->device,
-						AMD_IP_BLOCK_TYPE_UVD,
-						AMD_PG_STATE_GATE);
-
-		tonga_update_uvd_dpm(hwmgr, false);
-	}
-
-	return 0;
-}
-
-int tonga_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct phm_set_power_state_input states;
-	const struct pp_power_state  *pcurrent;
-	struct pp_power_state  *requested;
-
-	pcurrent = hwmgr->current_ps;
-	requested = hwmgr->request_ps;
-
-	states.pcurrent_state = &(pcurrent->hardware);
-	states.pnew_state = &(requested->hardware);
-
-	if (phm_cf_want_vce_power_gating(hwmgr)) {
-		if (data->vce_power_gated != bgate) {
-			if (bgate) {
-				cgs_set_clockgating_state(
-							hwmgr->device,
-							AMD_IP_BLOCK_TYPE_VCE,
-							AMD_CG_STATE_UNGATE);
-				cgs_set_powergating_state(
-							hwmgr->device,
-							AMD_IP_BLOCK_TYPE_VCE,
-							AMD_PG_STATE_GATE);
-				tonga_enable_disable_vce_dpm(hwmgr, false);
-				data->vce_power_gated = true;
-			} else {
-				tonga_phm_powerup_vce(hwmgr);
-				data->vce_power_gated = false;
-				cgs_set_powergating_state(
-							hwmgr->device,
-							AMD_IP_BLOCK_TYPE_VCE,
-							AMD_PG_STATE_UNGATE);
-				cgs_set_clockgating_state(
-							hwmgr->device,
-							AMD_IP_BLOCK_TYPE_VCE,
-							AMD_PG_STATE_GATE);
-
-				tonga_update_vce_dpm(hwmgr, &states);
-				tonga_enable_disable_vce_dpm(hwmgr, true);
-				return 0;
-			}
-		}
-	} else {
-		tonga_update_vce_dpm(hwmgr, &states);
-		tonga_enable_disable_vce_dpm(hwmgr, true);
-		return 0;
-	}
-
-	if (!data->vce_power_gated)
-		tonga_update_vce_dpm(hwmgr, &states);
-
-	return 0;
-}
-
-int tonga_phm_update_clock_gatings(struct pp_hwmgr *hwmgr,
-					const uint32_t *msg_id)
-{
-	PPSMC_Msg msg;
-	uint32_t value;
-
-	switch ((*msg_id & PP_GROUP_MASK) >> PP_GROUP_SHIFT) {
-	case PP_GROUP_GFX:
-		switch ((*msg_id & PP_BLOCK_MASK) >> PP_BLOCK_SHIFT) {
-		case PP_BLOCK_GFX_CG:
-			if (PP_STATE_SUPPORT_CG & *msg_id) {
-				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
-				  ? PPSMC_MSG_EnableClockGatingFeature
-				  : PPSMC_MSG_DisableClockGatingFeature;
-				value = CG_GFX_CGCG_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-			if (PP_STATE_SUPPORT_LS & *msg_id) {
-				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
-					? PPSMC_MSG_EnableClockGatingFeature
-					: PPSMC_MSG_DisableClockGatingFeature;
-				value = CG_GFX_CGLS_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-			break;
-
-		case PP_BLOCK_GFX_MG:
-			/* For GFX MGCG, there are three different ones;
-			 * CPF, RLC, and all others.  CPF MGCG will not be used for Tonga.
-			 * For GFX MGLS, Tonga will not support it.
-			 * */
-			if (PP_STATE_SUPPORT_CG & *msg_id) {
-				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
-				? PPSMC_MSG_EnableClockGatingFeature
-				: PPSMC_MSG_DisableClockGatingFeature;
-				value = (CG_RLC_MGCG_MASK | CG_GFX_OTHERS_MGCG_MASK);
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-			break;
-
-		default:
-			return -1;
-		}
-		break;
-
-	case PP_GROUP_SYS:
-		switch ((*msg_id & PP_BLOCK_MASK) >> PP_BLOCK_SHIFT) {
-		case PP_BLOCK_SYS_BIF:
-			if (PP_STATE_SUPPORT_LS & *msg_id) {
-				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
-				? PPSMC_MSG_EnableClockGatingFeature
-				: PPSMC_MSG_DisableClockGatingFeature;
-				value = CG_SYS_BIF_MGLS_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-			break;
-
-		case PP_BLOCK_SYS_MC:
-			if (PP_STATE_SUPPORT_CG & *msg_id) {
-				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
-				? PPSMC_MSG_EnableClockGatingFeature
-				: PPSMC_MSG_DisableClockGatingFeature;
-				value = CG_SYS_MC_MGCG_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-
-			if (PP_STATE_SUPPORT_LS & *msg_id) {
-				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
-				? PPSMC_MSG_EnableClockGatingFeature
-				: PPSMC_MSG_DisableClockGatingFeature;
-				value = CG_SYS_MC_MGLS_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-
-			}
-			break;
-
-		case PP_BLOCK_SYS_HDP:
-			if (PP_STATE_SUPPORT_CG & *msg_id) {
-				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
-					? PPSMC_MSG_EnableClockGatingFeature
-					: PPSMC_MSG_DisableClockGatingFeature;
-				value = CG_SYS_HDP_MGCG_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-
-			if (PP_STATE_SUPPORT_LS & *msg_id) {
-				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
-					? PPSMC_MSG_EnableClockGatingFeature
-					: PPSMC_MSG_DisableClockGatingFeature;
-
-				value = CG_SYS_HDP_MGLS_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-			break;
-
-		case PP_BLOCK_SYS_SDMA:
-			if (PP_STATE_SUPPORT_CG & *msg_id) {
-				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
-				? PPSMC_MSG_EnableClockGatingFeature
-				: PPSMC_MSG_DisableClockGatingFeature;
-				value = CG_SYS_SDMA_MGCG_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-
-			if (PP_STATE_SUPPORT_LS & *msg_id) {
-				msg = (*msg_id & PP_STATE_MASK) & PP_STATE_LS
-				? PPSMC_MSG_EnableClockGatingFeature
-				: PPSMC_MSG_DisableClockGatingFeature;
-
-				value = CG_SYS_SDMA_MGLS_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-			break;
-
-		case PP_BLOCK_SYS_ROM:
-			if (PP_STATE_SUPPORT_CG & *msg_id) {
-				msg = ((*msg_id & PP_STATE_MASK) & PP_STATE_CG)
-				? PPSMC_MSG_EnableClockGatingFeature
-				: PPSMC_MSG_DisableClockGatingFeature;
-				value = CG_SYS_ROM_MASK;
-
-				if (0 != smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, msg, value))
-					return -1;
-			}
-			break;
-
-		default:
-			return -1;
-
-		}
-		break;
-
-	default:
-		return -1;
-
-	}
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_dyn_defaults.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_dyn_defaults.h
deleted file mode 100644
index 080d69d77f04..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_dyn_defaults.h
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#ifndef TONGA_DYN_DEFAULTS_H
-#define TONGA_DYN_DEFAULTS_H
-
-
-/** \file
- * Volcanic Islands Dynamic default parameters.
- */
-
-enum TONGAdpm_TrendDetection {
-	TONGAdpm_TrendDetection_AUTO,
-	TONGAdpm_TrendDetection_UP,
-	TONGAdpm_TrendDetection_DOWN
-};
-typedef enum TONGAdpm_TrendDetection TONGAdpm_TrendDetection;
-
-/* Bit vector representing same fields as hardware register. */
-#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT0              0x3FFFC102  /* CP_Gfx_busy */
-/* HDP_busy */
-/* IH_busy */
-/* DRM_busy */
-/* DRMDMA_busy */
-/* UVD_busy */
-/* VCE_busy */
-/* ACP_busy */
-/* SAMU_busy */
-/* AVP_busy  */
-/* SDMA enabled */
-#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT1              0x000400  /* FE_Gfx_busy  - Intended for primary usage.   Rest are for flexibility. */
-/* SH_Gfx_busy */
-/* RB_Gfx_busy */
-/* VCE_busy */
-
-#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT2              0xC00080  /* SH_Gfx_busy - Intended for primary usage.   Rest are for flexibility. */
-/* FE_Gfx_busy */
-/* RB_Gfx_busy */
-/* ACP_busy */
-
-#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT3              0xC00200  /* RB_Gfx_busy - Intended for primary usage.   Rest are for flexibility. */
-/* FE_Gfx_busy */
-/* SH_Gfx_busy */
-/* UVD_busy */
-
-#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT4              0xC01680  /* UVD_busy */
-/* VCE_busy */
-/* ACP_busy */
-/* SAMU_busy */
-
-#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT5              0xC00033  /* GFX, HDP, DRMDMA */
-#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT6              0xC00033  /* GFX, HDP, DRMDMA */
-#define PPTONGA_VOTINGRIGHTSCLIENTS_DFLT7              0x3FFFC000  /* GFX, HDP, DRMDMA */
-
-
-/* thermal protection counter (units).*/
-#define PPTONGA_THERMALPROTECTCOUNTER_DFLT            0x200 /* ~19us */
-
-/* static screen threshold unit */
-#define PPTONGA_STATICSCREENTHRESHOLDUNIT_DFLT        0
-
-/* static screen threshold */
-#define PPTONGA_STATICSCREENTHRESHOLD_DFLT            0x00C8
-
-/* gfx idle clock stop threshold */
-#define PPTONGA_GFXIDLECLOCKSTOPTHRESHOLD_DFLT        0x200 /* ~19us with static screen threshold unit of 0 */
-
-/* Fixed reference divider to use when building baby stepping tables. */
-#define PPTONGA_REFERENCEDIVIDER_DFLT                  4
-
-/*
- * ULV voltage change delay time
- * Used to be delay_vreg in N.I. split for S.I.
- * Using N.I. delay_vreg value as default
- * ReferenceClock = 2700
- * VoltageResponseTime = 1000
- * VDDCDelayTime = (VoltageResponseTime * ReferenceClock) / 1600 = 1687
- */
-
-#define PPTONGA_ULVVOLTAGECHANGEDELAY_DFLT             1687
-
-#define PPTONGA_CGULVPARAMETER_DFLT                    0x00040035
-#define PPTONGA_CGULVCONTROL_DFLT                      0x00007450
-#define PPTONGA_TARGETACTIVITY_DFLT                     30  /*30%  */
-#define PPTONGA_MCLK_TARGETACTIVITY_DFLT                10  /*10%  */
-
-#endif
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c
deleted file mode 100644
index c7dc111221c2..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c
+++ /dev/null
@@ -1,6276 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/fb.h>
-#include "linux/delay.h"
-#include "pp_acpi.h"
-#include "hwmgr.h"
-#include <atombios.h>
-#include "tonga_hwmgr.h"
-#include "pptable.h"
-#include "processpptables.h"
-#include "tonga_processpptables.h"
-#include "tonga_pptable.h"
-#include "pp_debug.h"
-#include "tonga_ppsmc.h"
-#include "cgs_common.h"
-#include "pppcielanes.h"
-#include "tonga_dyn_defaults.h"
-#include "smumgr.h"
-#include "tonga_smumgr.h"
-#include "tonga_clockpowergating.h"
-#include "tonga_thermal.h"
-
-#include "smu/smu_7_1_2_d.h"
-#include "smu/smu_7_1_2_sh_mask.h"
-
-#include "gmc/gmc_8_1_d.h"
-#include "gmc/gmc_8_1_sh_mask.h"
-
-#include "bif/bif_5_0_d.h"
-#include "bif/bif_5_0_sh_mask.h"
-
-#include "dce/dce_10_0_d.h"
-#include "dce/dce_10_0_sh_mask.h"
-
-#include "cgs_linux.h"
-#include "eventmgr.h"
-#include "amd_pcie_helpers.h"
-
-#define MC_CG_ARB_FREQ_F0           0x0a
-#define MC_CG_ARB_FREQ_F1           0x0b
-#define MC_CG_ARB_FREQ_F2           0x0c
-#define MC_CG_ARB_FREQ_F3           0x0d
-
-#define MC_CG_SEQ_DRAMCONF_S0       0x05
-#define MC_CG_SEQ_DRAMCONF_S1       0x06
-#define MC_CG_SEQ_YCLK_SUSPEND      0x04
-#define MC_CG_SEQ_YCLK_RESUME       0x0a
-
-#define PCIE_BUS_CLK                10000
-#define TCLK                        (PCIE_BUS_CLK / 10)
-
-#define SMC_RAM_END 0x40000
-#define SMC_CG_IND_START            0xc0030000
-#define SMC_CG_IND_END              0xc0040000  /* First byte after SMC_CG_IND*/
-
-#define VOLTAGE_SCALE               4
-#define VOLTAGE_VID_OFFSET_SCALE1   625
-#define VOLTAGE_VID_OFFSET_SCALE2   100
-
-#define VDDC_VDDCI_DELTA            200
-#define VDDC_VDDGFX_DELTA           300
-
-#define MC_SEQ_MISC0_GDDR5_SHIFT 28
-#define MC_SEQ_MISC0_GDDR5_MASK  0xf0000000
-#define MC_SEQ_MISC0_GDDR5_VALUE 5
-
-typedef uint32_t PECI_RegistryValue;
-
-/* [2.5%,~2.5%] Clock stretched is multiple of 2.5% vs not and [Fmin, Fmax, LDO_REFSEL, USE_FOR_LOW_FREQ] */
-static const uint16_t PP_ClockStretcherLookupTable[2][4] = {
-	{600, 1050, 3, 0},
-	{600, 1050, 6, 1} };
-
-/* [FF, SS] type, [] 4 voltage ranges, and [Floor Freq, Boundary Freq, VID min , VID max] */
-static const uint32_t PP_ClockStretcherDDTTable[2][4][4] = {
-	{ {265, 529, 120, 128}, {325, 650, 96, 119}, {430, 860, 32, 95}, {0, 0, 0, 31} },
-	{ {275, 550, 104, 112}, {319, 638, 96, 103}, {360, 720, 64, 95}, {384, 768, 32, 63} } };
-
-/* [Use_For_Low_freq] value, [0%, 5%, 10%, 7.14%, 14.28%, 20%] (coming from PWR_CKS_CNTL.stretch_amount reg spec) */
-static const uint8_t PP_ClockStretchAmountConversion[2][6] = {
-	{0, 1, 3, 2, 4, 5},
-	{0, 2, 4, 5, 6, 5} };
-
-/* Values for the CG_THERMAL_CTRL::DPM_EVENT_SRC field. */
-enum DPM_EVENT_SRC {
-	DPM_EVENT_SRC_ANALOG = 0,               /* Internal analog trip point */
-	DPM_EVENT_SRC_EXTERNAL = 1,             /* External (GPIO 17) signal */
-	DPM_EVENT_SRC_DIGITAL = 2,              /* Internal digital trip point (DIG_THERM_DPM) */
-	DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,   /* Internal analog or external */
-	DPM_EVENT_SRC_DIGITAL_OR_EXTERNAL = 4   /* Internal digital or external */
-};
-typedef enum DPM_EVENT_SRC DPM_EVENT_SRC;
-
-static const unsigned long PhwTonga_Magic = (unsigned long)(PHM_VIslands_Magic);
-
-struct tonga_power_state *cast_phw_tonga_power_state(
-				  struct pp_hw_power_state *hw_ps)
-{
-	if (hw_ps == NULL)
-		return NULL;
-
-	PP_ASSERT_WITH_CODE((PhwTonga_Magic == hw_ps->magic),
-				"Invalid Powerstate Type!",
-				 return NULL);
-
-	return (struct tonga_power_state *)hw_ps;
-}
-
-const struct tonga_power_state *cast_const_phw_tonga_power_state(
-				 const struct pp_hw_power_state *hw_ps)
-{
-	if (hw_ps == NULL)
-		return NULL;
-
-	PP_ASSERT_WITH_CODE((PhwTonga_Magic == hw_ps->magic),
-				"Invalid Powerstate Type!",
-				 return NULL);
-
-	return (const struct tonga_power_state *)hw_ps;
-}
-
-int tonga_add_voltage(struct pp_hwmgr *hwmgr,
-	phm_ppt_v1_voltage_lookup_table *look_up_table,
-	phm_ppt_v1_voltage_lookup_record *record)
-{
-	uint32_t i;
-	PP_ASSERT_WITH_CODE((NULL != look_up_table),
-		"Lookup Table empty.", return -1;);
-	PP_ASSERT_WITH_CODE((0 != look_up_table->count),
-		"Lookup Table empty.", return -1;);
-	PP_ASSERT_WITH_CODE((SMU72_MAX_LEVELS_VDDGFX >= look_up_table->count),
-		"Lookup Table is full.", return -1;);
-
-	/* This is to avoid entering duplicate calculated records. */
-	for (i = 0; i < look_up_table->count; i++) {
-		if (look_up_table->entries[i].us_vdd == record->us_vdd) {
-			if (look_up_table->entries[i].us_calculated == 1)
-				return 0;
-			else
-				break;
-		}
-	}
-
-	look_up_table->entries[i].us_calculated = 1;
-	look_up_table->entries[i].us_vdd = record->us_vdd;
-	look_up_table->entries[i].us_cac_low = record->us_cac_low;
-	look_up_table->entries[i].us_cac_mid = record->us_cac_mid;
-	look_up_table->entries[i].us_cac_high = record->us_cac_high;
-	/* Only increment the count when we're appending, not replacing duplicate entry. */
-	if (i == look_up_table->count)
-		look_up_table->count++;
-
-	return 0;
-}
-
-int tonga_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display)
-{
-	PPSMC_Msg msg = has_display? (PPSMC_Msg)PPSMC_HasDisplay : (PPSMC_Msg)PPSMC_NoDisplay;
-
-	return (smum_send_msg_to_smc(hwmgr->smumgr, msg) == 0) ?  0 : -1;
-}
-
-uint8_t tonga_get_voltage_id(pp_atomctrl_voltage_table *voltage_table,
-		uint32_t voltage)
-{
-	uint8_t count = (uint8_t) (voltage_table->count);
-	uint8_t i = 0;
-
-	PP_ASSERT_WITH_CODE((NULL != voltage_table),
-		"Voltage Table empty.", return 0;);
-	PP_ASSERT_WITH_CODE((0 != count),
-		"Voltage Table empty.", return 0;);
-
-	for (i = 0; i < count; i++) {
-		/* find first voltage bigger than requested */
-		if (voltage_table->entries[i].value >= voltage)
-			return i;
-	}
-
-	/* voltage is bigger than max voltage in the table */
-	return i - 1;
-}
-
-/**
- * @brief PhwTonga_GetVoltageOrder
- *  Returns index of requested voltage record in lookup(table)
- * @param hwmgr - pointer to hardware manager
- * @param lookupTable - lookup list to search in
- * @param voltage - voltage to look for
- * @return 0 on success
- */
-uint8_t tonga_get_voltage_index(phm_ppt_v1_voltage_lookup_table *look_up_table,
-		uint16_t voltage)
-{
-	uint8_t count = (uint8_t) (look_up_table->count);
-	uint8_t i;
-
-	PP_ASSERT_WITH_CODE((NULL != look_up_table), "Lookup Table empty.", return 0;);
-	PP_ASSERT_WITH_CODE((0 != count), "Lookup Table empty.", return 0;);
-
-	for (i = 0; i < count; i++) {
-		/* find first voltage equal or bigger than requested */
-		if (look_up_table->entries[i].us_vdd >= voltage)
-			return i;
-	}
-
-	/* voltage is bigger than max voltage in the table */
-	return i-1;
-}
-
-bool tonga_is_dpm_running(struct pp_hwmgr *hwmgr)
-{
-	/*
-	 * We return the status of Voltage Control instead of checking SCLK/MCLK DPM
-	 * because we may have test scenarios that need us intentionly disable SCLK/MCLK DPM,
-	 * whereas voltage control is a fundemental change that will not be disabled
-	 */
-
-	return (0 == PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-					FEATURE_STATUS, VOLTAGE_CONTROLLER_ON) ? 1 : 0);
-}
-
-/**
- * Re-generate the DPM level mask value
- * @param    hwmgr      the address of the hardware manager
- */
-static uint32_t tonga_get_dpm_level_enable_mask_value(
-			struct tonga_single_dpm_table * dpm_table)
-{
-	uint32_t i;
-	uint32_t mask_value = 0;
-
-	for (i = dpm_table->count; i > 0; i--) {
-		mask_value = mask_value << 1;
-
-		if (dpm_table->dpm_levels[i-1].enabled)
-			mask_value |= 0x1;
-		else
-			mask_value &= 0xFFFFFFFE;
-	}
-	return mask_value;
-}
-
-/**
- * Retrieve DPM default values from registry (if available)
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- */
-void tonga_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	phw_tonga_ulv_parm *ulv = &(data->ulv);
-	uint32_t tmp;
-
-	ulv->ch_ulv_parameter = PPTONGA_CGULVPARAMETER_DFLT;
-	data->voting_rights_clients0 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT0;
-	data->voting_rights_clients1 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT1;
-	data->voting_rights_clients2 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT2;
-	data->voting_rights_clients3 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT3;
-	data->voting_rights_clients4 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT4;
-	data->voting_rights_clients5 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT5;
-	data->voting_rights_clients6 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT6;
-	data->voting_rights_clients7 = PPTONGA_VOTINGRIGHTSCLIENTS_DFLT7;
-
-	data->static_screen_threshold_unit = PPTONGA_STATICSCREENTHRESHOLDUNIT_DFLT;
-	data->static_screen_threshold = PPTONGA_STATICSCREENTHRESHOLD_DFLT;
-
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_ABM);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_NonABMSupportInPPLib);
-
-	tmp = 0;
-	if (tmp == 0)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_DynamicACTiming);
-
-	tmp = 0;
-	if (0 != tmp)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_DisableMemoryTransition);
-
-	data->mclk_strobe_mode_threshold = 40000;
-	data->mclk_stutter_mode_threshold = 30000;
-	data->mclk_edc_enable_threshold = 40000;
-	data->mclk_edc_wr_enable_threshold = 40000;
-
-	tmp = 0;
-	if (tmp != 0)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_DisableMCLS);
-
-	data->pcie_gen_performance.max = PP_PCIEGen1;
-	data->pcie_gen_performance.min = PP_PCIEGen3;
-	data->pcie_gen_power_saving.max = PP_PCIEGen1;
-	data->pcie_gen_power_saving.min = PP_PCIEGen3;
-
-	data->pcie_lane_performance.max = 0;
-	data->pcie_lane_performance.min = 16;
-	data->pcie_lane_power_saving.max = 0;
-	data->pcie_lane_power_saving.min = 16;
-
-	tmp = 0;
-
-	if (tmp)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkThrottleLowNotification);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_DynamicUVDState);
-
-}
-
-int tonga_update_sclk_threshold(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	int result = 0;
-	uint32_t low_sclk_interrupt_threshold = 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkThrottleLowNotification)
-		&& (hwmgr->gfx_arbiter.sclk_threshold != data->low_sclk_interrupt_threshold)) {
-		data->low_sclk_interrupt_threshold = hwmgr->gfx_arbiter.sclk_threshold;
-		low_sclk_interrupt_threshold = data->low_sclk_interrupt_threshold;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
-
-		result = tonga_copy_bytes_to_smc(
-				hwmgr->smumgr,
-				data->dpm_table_start + offsetof(SMU72_Discrete_DpmTable,
-				LowSclkInterruptThreshold),
-				(uint8_t *)&low_sclk_interrupt_threshold,
-				sizeof(uint32_t),
-				data->sram_end
-				);
-	}
-
-	return result;
-}
-
-/**
- * Find SCLK value that is associated with specified virtual_voltage_Id.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @param    virtual_voltage_Id  voltageId to look for.
- * @param    sclk output value .
- * @return   always 0 if success and 2 if association not found
- */
-static int tonga_get_sclk_for_voltage_evv(struct pp_hwmgr *hwmgr,
-	phm_ppt_v1_voltage_lookup_table *lookup_table,
-	uint16_t virtual_voltage_id, uint32_t *sclk)
-{
-	uint8_t entryId;
-	uint8_t voltageId;
-	struct phm_ppt_v1_information *pptable_info =
-					(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	PP_ASSERT_WITH_CODE(lookup_table->count != 0, "Lookup table is empty", return -1);
-
-	/* search for leakage voltage ID 0xff01 ~ 0xff08 and sckl */
-	for (entryId = 0; entryId < pptable_info->vdd_dep_on_sclk->count; entryId++) {
-		voltageId = pptable_info->vdd_dep_on_sclk->entries[entryId].vddInd;
-		if (lookup_table->entries[voltageId].us_vdd == virtual_voltage_id)
-			break;
-	}
-
-	PP_ASSERT_WITH_CODE(entryId < pptable_info->vdd_dep_on_sclk->count,
-			"Can't find requested voltage id in vdd_dep_on_sclk table!",
-			return -1;
-			);
-
-	*sclk = pptable_info->vdd_dep_on_sclk->entries[entryId].clk;
-
-	return 0;
-}
-
-/**
- * Get Leakage VDDC based on leakage ID.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   2 if vddgfx returned is greater than 2V or if BIOS
- */
-int tonga_get_evv_voltage(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	phm_ppt_v1_clock_voltage_dependency_table *sclk_table = pptable_info->vdd_dep_on_sclk;
-	uint16_t    virtual_voltage_id;
-	uint16_t    vddc = 0;
-	uint16_t    vddgfx = 0;
-	uint16_t    i, j;
-	uint32_t  sclk = 0;
-
-	/* retrieve voltage for leakage ID (0xff01 + i) */
-	for (i = 0; i < TONGA_MAX_LEAKAGE_COUNT; i++) {
-		virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
-
-		/* in split mode we should have only vddgfx EVV leakages */
-		if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
-			if (0 == tonga_get_sclk_for_voltage_evv(hwmgr,
-						pptable_info->vddgfx_lookup_table, virtual_voltage_id, &sclk)) {
-				if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-							PHM_PlatformCaps_ClockStretcher)) {
-					for (j = 1; j < sclk_table->count; j++) {
-						if (sclk_table->entries[j].clk == sclk &&
-								sclk_table->entries[j].cks_enable == 0) {
-							sclk += 5000;
-							break;
-						}
-					}
-				}
-				if (0 == atomctrl_get_voltage_evv_on_sclk
-				    (hwmgr, VOLTAGE_TYPE_VDDGFX, sclk,
-				     virtual_voltage_id, &vddgfx)) {
-					/* need to make sure vddgfx is less than 2v or else, it could burn the ASIC. */
-					PP_ASSERT_WITH_CODE((vddgfx < 2000 && vddgfx != 0), "Invalid VDDGFX value!", return -1);
-
-					/* the voltage should not be zero nor equal to leakage ID */
-					if (vddgfx != 0 && vddgfx != virtual_voltage_id) {
-						data->vddcgfx_leakage.actual_voltage[data->vddcgfx_leakage.count] = vddgfx;
-						data->vddcgfx_leakage.leakage_id[data->vddcgfx_leakage.count] = virtual_voltage_id;
-						data->vddcgfx_leakage.count++;
-					}
-				} else {
-					printk("Error retrieving EVV voltage value!\n");
-				}
-			}
-		} else {
-			/*  in merged mode we have only vddc EVV leakages */
-			if (0 == tonga_get_sclk_for_voltage_evv(hwmgr,
-						pptable_info->vddc_lookup_table,
-						virtual_voltage_id, &sclk)) {
-				if (0 == atomctrl_get_voltage_evv_on_sclk
-				    (hwmgr, VOLTAGE_TYPE_VDDC, sclk,
-				     virtual_voltage_id, &vddc)) {
-					/* need to make sure vddc is less than 2v or else, it could burn the ASIC. */
-					PP_ASSERT_WITH_CODE(vddc < 2000, "Invalid VDDC value!", return -1);
-
-					/* the voltage should not be zero nor equal to leakage ID */
-					if (vddc != 0 && vddc != virtual_voltage_id) {
-						data->vddc_leakage.actual_voltage[data->vddc_leakage.count] = vddc;
-						data->vddc_leakage.leakage_id[data->vddc_leakage.count] = virtual_voltage_id;
-						data->vddc_leakage.count++;
-					}
-				} else {
-					printk("Error retrieving EVV voltage value!\n");
-				}
-			}
-		}
-	}
-
-	return 0;
-}
-
-int tonga_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	/* enable SCLK dpm */
-	if (0 == data->sclk_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-						   PPSMC_MSG_DPM_Enable)),
-				"Failed to enable SCLK DPM during DPM Start Function!",
-				return -1);
-	}
-
-	/* enable MCLK dpm */
-	if (0 == data->mclk_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					     PPSMC_MSG_MCLKDPM_Enable)),
-				"Failed to enable MCLK DPM during DPM Start Function!",
-				return -1);
-
-		PHM_WRITE_FIELD(hwmgr->device, MC_SEQ_CNTL_3, CAC_EN, 0x1);
-
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixLCAC_MC0_CNTL, 0x05);/* CH0,1 read */
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixLCAC_MC1_CNTL, 0x05);/* CH2,3 read */
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixLCAC_CPL_CNTL, 0x100005);/*Read */
-
-		udelay(10);
-
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixLCAC_MC0_CNTL, 0x400005);/* CH0,1 write */
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixLCAC_MC1_CNTL, 0x400005);/* CH2,3 write */
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixLCAC_CPL_CNTL, 0x500005);/* write */
-
-	}
-
-	return 0;
-}
-
-int tonga_start_dpm(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	/* enable general power management */
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, 1);
-	/* enable sclk deep sleep */
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL, DYNAMIC_PM_EN, 1);
-
-	/* prepare for PCIE DPM */
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start +
-			offsetof(SMU72_SoftRegisters, VoltageChangeTimeout), 0x1000);
-
-	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE, SWRST_COMMAND_1, RESETLC, 0x0);
-
-	PP_ASSERT_WITH_CODE(
-			(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					PPSMC_MSG_Voltage_Cntl_Enable)),
-			"Failed to enable voltage DPM during DPM Start Function!",
-			return -1);
-
-	if (0 != tonga_enable_sclk_mclk_dpm(hwmgr)) {
-		PP_ASSERT_WITH_CODE(0, "Failed to enable Sclk DPM and Mclk DPM!", return -1);
-	}
-
-	/* enable PCIE dpm */
-	if (0 == data->pcie_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_PCIeDPM_Enable)),
-				"Failed to enable pcie DPM during DPM Start Function!",
-				return -1
-				);
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_Falcon_QuickTransition)) {
-				   smum_send_msg_to_smc(hwmgr->smumgr,
-				    PPSMC_MSG_EnableACDCGPIOInterrupt);
-	}
-
-	return 0;
-}
-
-int tonga_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	/* disable SCLK dpm */
-	if (0 == data->sclk_dpm_key_disabled) {
-		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
-		PP_ASSERT_WITH_CODE(
-				!tonga_is_dpm_running(hwmgr),
-				"Trying to Disable SCLK DPM when DPM is disabled",
-				return -1
-				);
-
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-						  PPSMC_MSG_DPM_Disable)),
-				"Failed to disable SCLK DPM during DPM stop Function!",
-				return -1);
-	}
-
-	/* disable MCLK dpm */
-	if (0 == data->mclk_dpm_key_disabled) {
-		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message. */
-		PP_ASSERT_WITH_CODE(
-				!tonga_is_dpm_running(hwmgr),
-				"Trying to Disable MCLK DPM when DPM is disabled",
-				return -1
-				);
-
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					    PPSMC_MSG_MCLKDPM_Disable)),
-				"Failed to Disable MCLK DPM during DPM stop Function!",
-				return -1);
-	}
-
-	return 0;
-}
-
-int tonga_stop_dpm(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, 0);
-	/* disable sclk deep sleep*/
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL, DYNAMIC_PM_EN, 0);
-
-	/* disable PCIE dpm */
-	if (0 == data->pcie_dpm_key_disabled) {
-		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
-		PP_ASSERT_WITH_CODE(
-				!tonga_is_dpm_running(hwmgr),
-				"Trying to Disable PCIE DPM when DPM is disabled",
-				return -1
-				);
-		PP_ASSERT_WITH_CODE(
-				(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-						PPSMC_MSG_PCIeDPM_Disable)),
-				"Failed to disable pcie DPM during DPM stop Function!",
-				return -1);
-	}
-
-	if (0 != tonga_disable_sclk_mclk_dpm(hwmgr))
-		PP_ASSERT_WITH_CODE(0, "Failed to disable Sclk DPM and Mclk DPM!", return -1);
-
-	/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
-	PP_ASSERT_WITH_CODE(
-			!tonga_is_dpm_running(hwmgr),
-			"Trying to Disable Voltage CNTL when DPM is disabled",
-			return -1
-			);
-
-	PP_ASSERT_WITH_CODE(
-			(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					PPSMC_MSG_Voltage_Cntl_Disable)),
-			"Failed to disable voltage DPM during DPM stop Function!",
-			return -1);
-
-	return 0;
-}
-
-int tonga_enable_sclk_control(struct pp_hwmgr *hwmgr)
-{
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, 0);
-
-	return 0;
-}
-
-/**
- * Send a message to the SMC and return a parameter
- *
- * @param    hwmgr:  the address of the powerplay hardware manager.
- * @param    msg: the message to send.
- * @param    parameter: pointer to the received parameter
- * @return   The response that came from the SMC.
- */
-PPSMC_Result tonga_send_msg_to_smc_return_parameter(
-		struct pp_hwmgr *hwmgr,
-		PPSMC_Msg msg,
-		uint32_t *parameter)
-{
-	int result;
-
-	result = smum_send_msg_to_smc(hwmgr->smumgr, msg);
-
-	if ((0 == result) && parameter) {
-		*parameter = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-	}
-
-	return result;
-}
-
-/**
- * force DPM power State
- *
- * @param    hwmgr:  the address of the powerplay hardware manager.
- * @param    n     :  DPM level
- * @return   The response that came from the SMC.
- */
-int tonga_dpm_force_state(struct pp_hwmgr *hwmgr, uint32_t n)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	uint32_t level_mask = 1 << n;
-
-	/* Checking if DPM is running.  If we discover hang because of this, we should skip this message. */
-	PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
-			    "Trying to force SCLK when DPM is disabled",
-			    return -1;);
-	if (0 == data->sclk_dpm_key_disabled)
-		return (0 == smum_send_msg_to_smc_with_parameter(
-							     hwmgr->smumgr,
-		     (PPSMC_Msg)(PPSMC_MSG_SCLKDPM_SetEnabledMask),
-						    level_mask) ? 0 : 1);
-
-	return 0;
-}
-
-/**
- * force DPM power State
- *
- * @param    hwmgr:  the address of the powerplay hardware manager.
- * @param    n     :  DPM level
- * @return   The response that came from the SMC.
- */
-int tonga_dpm_force_state_mclk(struct pp_hwmgr *hwmgr, uint32_t n)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	uint32_t level_mask = 1 << n;
-
-	/* Checking if DPM is running.  If we discover hang because of this, we should skip this message. */
-	PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
-			    "Trying to Force MCLK when DPM is disabled",
-			    return -1;);
-	if (0 == data->mclk_dpm_key_disabled)
-		return (0 == smum_send_msg_to_smc_with_parameter(
-								hwmgr->smumgr,
-								(PPSMC_Msg)(PPSMC_MSG_MCLKDPM_SetEnabledMask),
-								level_mask) ? 0 : 1);
-
-	return 0;
-}
-
-/**
- * force DPM power State
- *
- * @param    hwmgr:  the address of the powerplay hardware manager.
- * @param    n     :  DPM level
- * @return   The response that came from the SMC.
- */
-int tonga_dpm_force_state_pcie(struct pp_hwmgr *hwmgr, uint32_t n)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
-	PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
-			    "Trying to Force PCIE level when DPM is disabled",
-			    return -1;);
-	if (0 == data->pcie_dpm_key_disabled)
-		return (0 == smum_send_msg_to_smc_with_parameter(
-							     hwmgr->smumgr,
-			   (PPSMC_Msg)(PPSMC_MSG_PCIeDPM_ForceLevel),
-								n) ? 0 : 1);
-
-	return 0;
-}
-
-/**
- * Set the initial state by calling SMC to switch to this state directly
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_set_boot_state(struct pp_hwmgr *hwmgr)
-{
-	/*
-	* SMC only stores one state that SW will ask to switch too,
-	* so we switch the the just uploaded one
-	*/
-	return (0 == tonga_disable_sclk_mclk_dpm(hwmgr)) ? 0 : 1;
-}
-
-/**
- * Get the location of various tables inside the FW image.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_process_firmware_header(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_smumgr *tonga_smu = (struct tonga_smumgr *)(hwmgr->smumgr->backend);
-
-	uint32_t tmp;
-	int result;
-	bool error = false;
-
-	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
-				SMU72_FIRMWARE_HEADER_LOCATION +
-				offsetof(SMU72_Firmware_Header, DpmTable),
-				&tmp, data->sram_end);
-
-	if (0 == result) {
-		data->dpm_table_start = tmp;
-	}
-
-	error |= (0 != result);
-
-	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
-				SMU72_FIRMWARE_HEADER_LOCATION +
-				offsetof(SMU72_Firmware_Header, SoftRegisters),
-				&tmp, data->sram_end);
-
-	if (0 == result) {
-		data->soft_regs_start = tmp;
-		tonga_smu->ulSoftRegsStart = tmp;
-	}
-
-	error |= (0 != result);
-
-
-	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
-				SMU72_FIRMWARE_HEADER_LOCATION +
-				offsetof(SMU72_Firmware_Header, mcRegisterTable),
-				&tmp, data->sram_end);
-
-	if (0 == result) {
-		data->mc_reg_table_start = tmp;
-	}
-
-	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
-				SMU72_FIRMWARE_HEADER_LOCATION +
-				offsetof(SMU72_Firmware_Header, FanTable),
-				&tmp, data->sram_end);
-
-	if (0 == result) {
-		data->fan_table_start = tmp;
-	}
-
-	error |= (0 != result);
-
-	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
-				SMU72_FIRMWARE_HEADER_LOCATION +
-				offsetof(SMU72_Firmware_Header, mcArbDramTimingTable),
-				&tmp, data->sram_end);
-
-	if (0 == result) {
-		data->arb_table_start = tmp;
-	}
-
-	error |= (0 != result);
-
-
-	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
-				SMU72_FIRMWARE_HEADER_LOCATION +
-				offsetof(SMU72_Firmware_Header, Version),
-				&tmp, data->sram_end);
-
-	if (0 == result) {
-		hwmgr->microcode_version_info.SMC = tmp;
-	}
-
-	error |= (0 != result);
-
-	return error ? 1 : 0;
-}
-
-/**
- * Read clock related registers.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_read_clock_registers(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	data->clock_registers.vCG_SPLL_FUNC_CNTL         =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL);
-	data->clock_registers.vCG_SPLL_FUNC_CNTL_2       =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_2);
-	data->clock_registers.vCG_SPLL_FUNC_CNTL_3       =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_3);
-	data->clock_registers.vCG_SPLL_FUNC_CNTL_4       =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_FUNC_CNTL_4);
-	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM   =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_SPREAD_SPECTRUM);
-	data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2 =
-		cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_SPLL_SPREAD_SPECTRUM_2);
-	data->clock_registers.vDLL_CNTL                  =
-		cgs_read_register(hwmgr->device, mmDLL_CNTL);
-	data->clock_registers.vMCLK_PWRMGT_CNTL          =
-		cgs_read_register(hwmgr->device, mmMCLK_PWRMGT_CNTL);
-	data->clock_registers.vMPLL_AD_FUNC_CNTL         =
-		cgs_read_register(hwmgr->device, mmMPLL_AD_FUNC_CNTL);
-	data->clock_registers.vMPLL_DQ_FUNC_CNTL         =
-		cgs_read_register(hwmgr->device, mmMPLL_DQ_FUNC_CNTL);
-	data->clock_registers.vMPLL_FUNC_CNTL            =
-		cgs_read_register(hwmgr->device, mmMPLL_FUNC_CNTL);
-	data->clock_registers.vMPLL_FUNC_CNTL_1          =
-		cgs_read_register(hwmgr->device, mmMPLL_FUNC_CNTL_1);
-	data->clock_registers.vMPLL_FUNC_CNTL_2          =
-		cgs_read_register(hwmgr->device, mmMPLL_FUNC_CNTL_2);
-	data->clock_registers.vMPLL_SS1                  =
-		cgs_read_register(hwmgr->device, mmMPLL_SS1);
-	data->clock_registers.vMPLL_SS2                  =
-		cgs_read_register(hwmgr->device, mmMPLL_SS2);
-
-	return 0;
-}
-
-/**
- * Find out if memory is GDDR5.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_get_memory_type(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	uint32_t temp;
-
-	temp = cgs_read_register(hwmgr->device, mmMC_SEQ_MISC0);
-
-	data->is_memory_GDDR5 = (MC_SEQ_MISC0_GDDR5_VALUE ==
-			((temp & MC_SEQ_MISC0_GDDR5_MASK) >>
-			 MC_SEQ_MISC0_GDDR5_SHIFT));
-
-	return 0;
-}
-
-/**
- * Enables Dynamic Power Management by SMC
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_enable_acpi_power_management(struct pp_hwmgr *hwmgr)
-{
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, STATIC_PM_EN, 1);
-
-	return 0;
-}
-
-/**
- * Initialize PowerGating States for different engines
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_init_power_gate_state(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	data->uvd_power_gated = false;
-	data->vce_power_gated = false;
-	data->samu_power_gated = false;
-	data->acp_power_gated = false;
-	data->pg_acp_init = true;
-
-	return 0;
-}
-
-/**
- * Checks if DPM is enabled
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_check_for_dpm_running(struct pp_hwmgr *hwmgr)
-{
-	/*
-	 * We return the status of Voltage Control instead of checking SCLK/MCLK DPM
-	 * because we may have test scenarios that need us intentionly disable SCLK/MCLK DPM,
-	 * whereas voltage control is a fundemental change that will not be disabled
-	 */
-	return (!tonga_is_dpm_running(hwmgr) ? 0 : 1);
-}
-
-/**
- * Checks if DPM is stopped
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_check_for_dpm_stopped(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	if (tonga_is_dpm_running(hwmgr)) {
-		/* If HW Virtualization is enabled, dpm_table_start will not have a valid value */
-		if (!data->dpm_table_start) {
-			return 1;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * Remove repeated voltage values and create table with unique values.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @param    voltage_table  the pointer to changing voltage table
- * @return    1 in success
- */
-
-static int tonga_trim_voltage_table(struct pp_hwmgr *hwmgr,
-			pp_atomctrl_voltage_table *voltage_table)
-{
-	uint32_t table_size, i, j;
-	uint16_t vvalue;
-	bool bVoltageFound = false;
-	pp_atomctrl_voltage_table *table;
-
-	PP_ASSERT_WITH_CODE((NULL != voltage_table), "Voltage Table empty.", return -1;);
-	table_size = sizeof(pp_atomctrl_voltage_table);
-	table = kzalloc(table_size, GFP_KERNEL);
-
-	if (NULL == table)
-		return -ENOMEM;
-
-	memset(table, 0x00, table_size);
-	table->mask_low = voltage_table->mask_low;
-	table->phase_delay = voltage_table->phase_delay;
-
-	for (i = 0; i < voltage_table->count; i++) {
-		vvalue = voltage_table->entries[i].value;
-		bVoltageFound = false;
-
-		for (j = 0; j < table->count; j++) {
-			if (vvalue == table->entries[j].value) {
-				bVoltageFound = true;
-				break;
-			}
-		}
-
-		if (!bVoltageFound) {
-			table->entries[table->count].value = vvalue;
-			table->entries[table->count].smio_low =
-				voltage_table->entries[i].smio_low;
-			table->count++;
-		}
-	}
-
-	memcpy(table, voltage_table, sizeof(pp_atomctrl_voltage_table));
-
-	kfree(table);
-
-	return 0;
-}
-
-static int tonga_get_svi2_vdd_ci_voltage_table(
-		struct pp_hwmgr *hwmgr,
-		phm_ppt_v1_clock_voltage_dependency_table *voltage_dependency_table)
-{
-	uint32_t i;
-	int result;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	pp_atomctrl_voltage_table *vddci_voltage_table = &(data->vddci_voltage_table);
-
-	PP_ASSERT_WITH_CODE((0 != voltage_dependency_table->count),
-			"Voltage Dependency Table empty.", return -1;);
-
-	vddci_voltage_table->mask_low = 0;
-	vddci_voltage_table->phase_delay = 0;
-	vddci_voltage_table->count = voltage_dependency_table->count;
-
-	for (i = 0; i < voltage_dependency_table->count; i++) {
-		vddci_voltage_table->entries[i].value =
-			voltage_dependency_table->entries[i].vddci;
-		vddci_voltage_table->entries[i].smio_low = 0;
-	}
-
-	result = tonga_trim_voltage_table(hwmgr, vddci_voltage_table);
-	PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to trim VDDCI table.", return result;);
-
-	return 0;
-}
-
-
-
-static int tonga_get_svi2_vdd_voltage_table(
-		struct pp_hwmgr *hwmgr,
-		phm_ppt_v1_voltage_lookup_table *look_up_table,
-		pp_atomctrl_voltage_table *voltage_table)
-{
-	uint8_t i = 0;
-
-	PP_ASSERT_WITH_CODE((0 != look_up_table->count),
-			"Voltage Lookup Table empty.", return -1;);
-
-	voltage_table->mask_low = 0;
-	voltage_table->phase_delay = 0;
-
-	voltage_table->count = look_up_table->count;
-
-	for (i = 0; i < voltage_table->count; i++) {
-		voltage_table->entries[i].value = look_up_table->entries[i].us_vdd;
-		voltage_table->entries[i].smio_low = 0;
-	}
-
-	return 0;
-}
-
-/*
- * -------------------------------------------------------- Voltage Tables --------------------------------------------------------------------------
- * If the voltage table would be bigger than what will fit into the state table on the SMC keep only the higher entries.
- */
-
-static void tonga_trim_voltage_table_to_fit_state_table(
-		struct pp_hwmgr *hwmgr,
-		uint32_t max_voltage_steps,
-		pp_atomctrl_voltage_table *voltage_table)
-{
-	unsigned int i, diff;
-
-	if (voltage_table->count <= max_voltage_steps) {
-		return;
-	}
-
-	diff = voltage_table->count - max_voltage_steps;
-
-	for (i = 0; i < max_voltage_steps; i++) {
-		voltage_table->entries[i] = voltage_table->entries[i + diff];
-	}
-
-	voltage_table->count = max_voltage_steps;
-
-	return;
-}
-
-/**
- * Create Voltage Tables.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_construct_voltage_tables(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	int result;
-
-	/* MVDD has only GPIO voltage control */
-	if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
-		result = atomctrl_get_voltage_table_v3(hwmgr,
-					VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT, &(data->mvdd_voltage_table));
-		PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to retrieve MVDD table.", return result;);
-	}
-
-	if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->vdd_ci_control) {
-		/* GPIO voltage */
-		result = atomctrl_get_voltage_table_v3(hwmgr,
-					VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT, &(data->vddci_voltage_table));
-		PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to retrieve VDDCI table.", return result;);
-	} else if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_ci_control) {
-		/* SVI2 voltage */
-		result = tonga_get_svi2_vdd_ci_voltage_table(hwmgr,
-					pptable_info->vdd_dep_on_mclk);
-		PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to retrieve SVI2 VDDCI table from dependancy table.", return result;);
-	}
-
-	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_gfx_control) {
-		/* VDDGFX has only SVI2 voltage control */
-		result = tonga_get_svi2_vdd_voltage_table(hwmgr,
-					pptable_info->vddgfx_lookup_table, &(data->vddgfx_voltage_table));
-		PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to retrieve SVI2 VDDGFX table from lookup table.", return result;);
-	}
-
-	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
-		/* VDDC has only SVI2 voltage control */
-		result = tonga_get_svi2_vdd_voltage_table(hwmgr,
-					pptable_info->vddc_lookup_table, &(data->vddc_voltage_table));
-		PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to retrieve SVI2 VDDC table from lookup table.", return result;);
-	}
-
-	PP_ASSERT_WITH_CODE(
-			(data->vddc_voltage_table.count <= (SMU72_MAX_LEVELS_VDDC)),
-			"Too many voltage values for VDDC. Trimming to fit state table.",
-			tonga_trim_voltage_table_to_fit_state_table(hwmgr,
-			SMU72_MAX_LEVELS_VDDC, &(data->vddc_voltage_table));
-			);
-
-	PP_ASSERT_WITH_CODE(
-			(data->vddgfx_voltage_table.count <= (SMU72_MAX_LEVELS_VDDGFX)),
-			"Too many voltage values for VDDGFX. Trimming to fit state table.",
-			tonga_trim_voltage_table_to_fit_state_table(hwmgr,
-			SMU72_MAX_LEVELS_VDDGFX, &(data->vddgfx_voltage_table));
-			);
-
-	PP_ASSERT_WITH_CODE(
-			(data->vddci_voltage_table.count <= (SMU72_MAX_LEVELS_VDDCI)),
-			"Too many voltage values for VDDCI. Trimming to fit state table.",
-			tonga_trim_voltage_table_to_fit_state_table(hwmgr,
-			SMU72_MAX_LEVELS_VDDCI, &(data->vddci_voltage_table));
-			);
-
-	PP_ASSERT_WITH_CODE(
-			(data->mvdd_voltage_table.count <= (SMU72_MAX_LEVELS_MVDD)),
-			"Too many voltage values for MVDD. Trimming to fit state table.",
-			tonga_trim_voltage_table_to_fit_state_table(hwmgr,
-			SMU72_MAX_LEVELS_MVDD, &(data->mvdd_voltage_table));
-			);
-
-	return 0;
-}
-
-/**
- * Vddc table preparation for SMC.
- *
- * @param    hwmgr      the address of the hardware manager
- * @param    table     the SMC DPM table structure to be populated
- * @return   always 0
- */
-static int tonga_populate_smc_vddc_table(struct pp_hwmgr *hwmgr,
-			SMU72_Discrete_DpmTable *table)
-{
-	unsigned int count;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
-		table->VddcLevelCount = data->vddc_voltage_table.count;
-		for (count = 0; count < table->VddcLevelCount; count++) {
-			table->VddcTable[count] =
-				PP_HOST_TO_SMC_US(data->vddc_voltage_table.entries[count].value * VOLTAGE_SCALE);
-		}
-		CONVERT_FROM_HOST_TO_SMC_UL(table->VddcLevelCount);
-	}
-	return 0;
-}
-
-/**
- * VddGfx table preparation for SMC.
- *
- * @param    hwmgr      the address of the hardware manager
- * @param    table     the SMC DPM table structure to be populated
- * @return   always 0
- */
-static int tonga_populate_smc_vdd_gfx_table(struct pp_hwmgr *hwmgr,
-			SMU72_Discrete_DpmTable *table)
-{
-	unsigned int count;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_gfx_control) {
-		table->VddGfxLevelCount = data->vddgfx_voltage_table.count;
-		for (count = 0; count < data->vddgfx_voltage_table.count; count++) {
-			table->VddGfxTable[count] =
-				PP_HOST_TO_SMC_US(data->vddgfx_voltage_table.entries[count].value * VOLTAGE_SCALE);
-		}
-		CONVERT_FROM_HOST_TO_SMC_UL(table->VddGfxLevelCount);
-	}
-	return 0;
-}
-
-/**
- * Vddci table preparation for SMC.
- *
- * @param    *hwmgr The address of the hardware manager.
- * @param    *table The SMC DPM table structure to be populated.
- * @return   0
- */
-static int tonga_populate_smc_vdd_ci_table(struct pp_hwmgr *hwmgr,
-			SMU72_Discrete_DpmTable *table)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	uint32_t count;
-
-	table->VddciLevelCount = data->vddci_voltage_table.count;
-	for (count = 0; count < table->VddciLevelCount; count++) {
-		if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_ci_control) {
-			table->VddciTable[count] =
-				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[count].value * VOLTAGE_SCALE);
-		} else if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->vdd_ci_control) {
-			table->SmioTable1.Pattern[count].Voltage =
-				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[count].value * VOLTAGE_SCALE);
-			/* Index into DpmTable.Smio. Drive bits from Smio entry to get this voltage level. */
-			table->SmioTable1.Pattern[count].Smio =
-				(uint8_t) count;
-			table->Smio[count] |=
-				data->vddci_voltage_table.entries[count].smio_low;
-			table->VddciTable[count] =
-				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[count].value * VOLTAGE_SCALE);
-		}
-	}
-
-	table->SmioMask1 = data->vddci_voltage_table.mask_low;
-	CONVERT_FROM_HOST_TO_SMC_UL(table->VddciLevelCount);
-
-	return 0;
-}
-
-/**
- * Mvdd table preparation for SMC.
- *
- * @param    *hwmgr The address of the hardware manager.
- * @param    *table The SMC DPM table structure to be populated.
- * @return   0
- */
-static int tonga_populate_smc_mvdd_table(struct pp_hwmgr *hwmgr,
-			SMU72_Discrete_DpmTable *table)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	uint32_t count;
-
-	if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
-		table->MvddLevelCount = data->mvdd_voltage_table.count;
-		for (count = 0; count < table->MvddLevelCount; count++) {
-			table->SmioTable2.Pattern[count].Voltage =
-				PP_HOST_TO_SMC_US(data->mvdd_voltage_table.entries[count].value * VOLTAGE_SCALE);
-			/* Index into DpmTable.Smio. Drive bits from Smio entry to get this voltage level.*/
-			table->SmioTable2.Pattern[count].Smio =
-				(uint8_t) count;
-			table->Smio[count] |=
-				data->mvdd_voltage_table.entries[count].smio_low;
-		}
-		table->SmioMask2 = data->mvdd_voltage_table.mask_low;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->MvddLevelCount);
-	}
-
-	return 0;
-}
-
-/**
- * Convert a voltage value in mv unit to VID number required by SMU firmware
- */
-static uint8_t convert_to_vid(uint16_t vddc)
-{
-	return (uint8_t) ((6200 - (vddc * VOLTAGE_SCALE)) / 25);
-}
-
-
-/**
- * Preparation of vddc and vddgfx CAC tables for SMC.
- *
- * @param    hwmgr      the address of the hardware manager
- * @param    table     the SMC DPM table structure to be populated
- * @return   always 0
- */
-static int tonga_populate_cac_tables(struct pp_hwmgr *hwmgr,
-			SMU72_Discrete_DpmTable *table)
-{
-	uint32_t count;
-	uint8_t index;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct phm_ppt_v1_voltage_lookup_table *vddgfx_lookup_table = pptable_info->vddgfx_lookup_table;
-	struct phm_ppt_v1_voltage_lookup_table *vddc_lookup_table = pptable_info->vddc_lookup_table;
-
-	/* pTables is already swapped, so in order to use the value from it, we need to swap it back. */
-	uint32_t vddcLevelCount = PP_SMC_TO_HOST_UL(table->VddcLevelCount);
-	uint32_t vddgfxLevelCount = PP_SMC_TO_HOST_UL(table->VddGfxLevelCount);
-
-	for (count = 0; count < vddcLevelCount; count++) {
-		/* We are populating vddc CAC data to BapmVddc table in split and merged mode */
-		index = tonga_get_voltage_index(vddc_lookup_table,
-			data->vddc_voltage_table.entries[count].value);
-		table->BapmVddcVidLoSidd[count] =
-			convert_to_vid(vddc_lookup_table->entries[index].us_cac_low);
-		table->BapmVddcVidHiSidd[count] =
-			convert_to_vid(vddc_lookup_table->entries[index].us_cac_mid);
-		table->BapmVddcVidHiSidd2[count] =
-			convert_to_vid(vddc_lookup_table->entries[index].us_cac_high);
-	}
-
-	if ((data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2)) {
-		/* We are populating vddgfx CAC data to BapmVddgfx table in split mode */
-		for (count = 0; count < vddgfxLevelCount; count++) {
-			index = tonga_get_voltage_index(vddgfx_lookup_table,
-				data->vddgfx_voltage_table.entries[count].value);
-			table->BapmVddGfxVidLoSidd[count] =
-				convert_to_vid(vddgfx_lookup_table->entries[index].us_cac_low);
-			table->BapmVddGfxVidHiSidd[count] =
-				convert_to_vid(vddgfx_lookup_table->entries[index].us_cac_mid);
-			table->BapmVddGfxVidHiSidd2[count] =
-				convert_to_vid(vddgfx_lookup_table->entries[index].us_cac_high);
-		}
-	} else {
-		for (count = 0; count < vddcLevelCount; count++) {
-			index = tonga_get_voltage_index(vddc_lookup_table,
-				data->vddc_voltage_table.entries[count].value);
-			table->BapmVddGfxVidLoSidd[count] =
-				convert_to_vid(vddc_lookup_table->entries[index].us_cac_low);
-			table->BapmVddGfxVidHiSidd[count] =
-				convert_to_vid(vddc_lookup_table->entries[index].us_cac_mid);
-			table->BapmVddGfxVidHiSidd2[count] =
-				convert_to_vid(vddc_lookup_table->entries[index].us_cac_high);
-		}
-	}
-
-	return 0;
-}
-
-
-/**
- * Preparation of voltage tables for SMC.
- *
- * @param    hwmgr      the address of the hardware manager
- * @param    table     the SMC DPM table structure to be populated
- * @return   always 0
- */
-
-int tonga_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
-	SMU72_Discrete_DpmTable *table)
-{
-	int result;
-
-	result = tonga_populate_smc_vddc_table(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"can not populate VDDC voltage table to SMC", return -1);
-
-	result = tonga_populate_smc_vdd_ci_table(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"can not populate VDDCI voltage table to SMC", return -1);
-
-	result = tonga_populate_smc_vdd_gfx_table(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"can not populate VDDGFX voltage table to SMC", return -1);
-
-	result = tonga_populate_smc_mvdd_table(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"can not populate MVDD voltage table to SMC", return -1);
-
-	result = tonga_populate_cac_tables(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"can not populate CAC voltage tables to SMC", return -1);
-
-	return 0;
-}
-
-/**
- * Populates the SMC VRConfig field in DPM table.
- *
- * @param    hwmgr      the address of the hardware manager
- * @param    table     the SMC DPM table structure to be populated
- * @return   always 0
- */
-static int tonga_populate_vr_config(struct pp_hwmgr *hwmgr,
-			SMU72_Discrete_DpmTable *table)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	uint16_t config;
-
-	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_gfx_control) {
-		/*  Splitted mode */
-		config = VR_SVI2_PLANE_1;
-		table->VRConfig |= (config<<VRCONF_VDDGFX_SHIFT);
-
-		if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
-			config = VR_SVI2_PLANE_2;
-			table->VRConfig |= config;
-		} else {
-			printk(KERN_ERR "[ powerplay ] VDDC and VDDGFX should be both on SVI2 control in splitted mode! \n");
-		}
-	} else {
-		/* Merged mode  */
-		config = VR_MERGED_WITH_VDDC;
-		table->VRConfig |= (config<<VRCONF_VDDGFX_SHIFT);
-
-		/* Set Vddc Voltage Controller  */
-		if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
-			config = VR_SVI2_PLANE_1;
-			table->VRConfig |= config;
-		} else {
-			printk(KERN_ERR "[ powerplay ] VDDC should be on SVI2 control in merged mode! \n");
-		}
-	}
-
-	/* Set Vddci Voltage Controller  */
-	if (TONGA_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_ci_control) {
-		config = VR_SVI2_PLANE_2;  /* only in merged mode */
-		table->VRConfig |= (config<<VRCONF_VDDCI_SHIFT);
-	} else if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->vdd_ci_control) {
-		config = VR_SMIO_PATTERN_1;
-		table->VRConfig |= (config<<VRCONF_VDDCI_SHIFT);
-	}
-
-	/* Set Mvdd Voltage Controller */
-	if (TONGA_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
-		config = VR_SMIO_PATTERN_2;
-		table->VRConfig |= (config<<VRCONF_MVDD_SHIFT);
-	}
-
-	return 0;
-}
-
-static int tonga_get_dependecy_volt_by_clk(struct pp_hwmgr *hwmgr,
-	phm_ppt_v1_clock_voltage_dependency_table *allowed_clock_voltage_table,
-	uint32_t clock, SMU_VoltageLevel *voltage, uint32_t *mvdd)
-{
-	uint32_t i = 0;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	/* clock - voltage dependency table is empty table */
-	if (allowed_clock_voltage_table->count == 0)
-		return -1;
-
-	for (i = 0; i < allowed_clock_voltage_table->count; i++) {
-		/* find first sclk bigger than request */
-		if (allowed_clock_voltage_table->entries[i].clk >= clock) {
-			voltage->VddGfx = tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
-								allowed_clock_voltage_table->entries[i].vddgfx);
-
-			voltage->Vddc = tonga_get_voltage_index(pptable_info->vddc_lookup_table,
-								allowed_clock_voltage_table->entries[i].vddc);
-
-			if (allowed_clock_voltage_table->entries[i].vddci) {
-				voltage->Vddci = tonga_get_voltage_id(&data->vddci_voltage_table,
-									allowed_clock_voltage_table->entries[i].vddci);
-			} else {
-				voltage->Vddci = tonga_get_voltage_id(&data->vddci_voltage_table,
-									allowed_clock_voltage_table->entries[i].vddc - data->vddc_vddci_delta);
-			}
-
-			if (allowed_clock_voltage_table->entries[i].mvdd) {
-				*mvdd = (uint32_t) allowed_clock_voltage_table->entries[i].mvdd;
-			}
-
-			voltage->Phases = 1;
-			return 0;
-		}
-	}
-
-	/* sclk is bigger than max sclk in the dependence table */
-	voltage->VddGfx = tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
-		allowed_clock_voltage_table->entries[i-1].vddgfx);
-	voltage->Vddc = tonga_get_voltage_index(pptable_info->vddc_lookup_table,
-		allowed_clock_voltage_table->entries[i-1].vddc);
-
-	if (allowed_clock_voltage_table->entries[i-1].vddci) {
-		voltage->Vddci = tonga_get_voltage_id(&data->vddci_voltage_table,
-			allowed_clock_voltage_table->entries[i-1].vddci);
-	}
-	if (allowed_clock_voltage_table->entries[i-1].mvdd) {
-		*mvdd = (uint32_t) allowed_clock_voltage_table->entries[i-1].mvdd;
-	}
-
-	return 0;
-}
-
-/**
- * Call SMC to reset S0/S1 to S1 and Reset SMIO to initial value
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_reset_to_default(struct pp_hwmgr *hwmgr)
-{
-	return (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_ResetToDefaults) == 0) ? 0 : 1;
-}
-
-int tonga_populate_memory_timing_parameters(
-		struct pp_hwmgr *hwmgr,
-		uint32_t engine_clock,
-		uint32_t memory_clock,
-		struct SMU72_Discrete_MCArbDramTimingTableEntry *arb_regs
-		)
-{
-	uint32_t dramTiming;
-	uint32_t dramTiming2;
-	uint32_t burstTime;
-	int result;
-
-	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
-				engine_clock, memory_clock);
-
-	PP_ASSERT_WITH_CODE(result == 0,
-		"Error calling VBIOS to set DRAM_TIMING.", return result);
-
-	dramTiming  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
-	dramTiming2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
-	burstTime = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
-
-	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dramTiming);
-	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dramTiming2);
-	arb_regs->McArbBurstTime = (uint8_t)burstTime;
-
-	return 0;
-}
-
-/**
- * Setup parameters for the MC ARB.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- * This function is to be called from the SetPowerState table.
- */
-int tonga_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	int result = 0;
-	SMU72_Discrete_MCArbDramTimingTable  arb_regs;
-	uint32_t i, j;
-
-	memset(&arb_regs, 0x00, sizeof(SMU72_Discrete_MCArbDramTimingTable));
-
-	for (i = 0; i < data->dpm_table.sclk_table.count; i++) {
-		for (j = 0; j < data->dpm_table.mclk_table.count; j++) {
-			result = tonga_populate_memory_timing_parameters
-				(hwmgr, data->dpm_table.sclk_table.dpm_levels[i].value,
-				 data->dpm_table.mclk_table.dpm_levels[j].value,
-				 &arb_regs.entries[i][j]);
-
-			if (0 != result) {
-				break;
-			}
-		}
-	}
-
-	if (0 == result) {
-		result = tonga_copy_bytes_to_smc(
-				hwmgr->smumgr,
-				data->arb_table_start,
-				(uint8_t *)&arb_regs,
-				sizeof(SMU72_Discrete_MCArbDramTimingTable),
-				data->sram_end
-				);
-	}
-
-	return result;
-}
-
-static int tonga_populate_smc_link_level(struct pp_hwmgr *hwmgr, SMU72_Discrete_DpmTable *table)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_dpm_table *dpm_table = &data->dpm_table;
-	uint32_t i;
-
-	/* Index (dpm_table->pcie_speed_table.count) is reserved for PCIE boot level. */
-	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
-		table->LinkLevel[i].PcieGenSpeed  =
-			(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
-		table->LinkLevel[i].PcieLaneCount =
-			(uint8_t)encode_pcie_lane_width(dpm_table->pcie_speed_table.dpm_levels[i].param1);
-		table->LinkLevel[i].EnabledForActivity =
-			1;
-		table->LinkLevel[i].SPC =
-			(uint8_t)(data->pcie_spc_cap & 0xff);
-		table->LinkLevel[i].DownThreshold =
-			PP_HOST_TO_SMC_UL(5);
-		table->LinkLevel[i].UpThreshold =
-			PP_HOST_TO_SMC_UL(30);
-	}
-
-	data->smc_state_table.LinkLevelCount =
-		(uint8_t)dpm_table->pcie_speed_table.count;
-	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
-		tonga_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
-
-	return 0;
-}
-
-static int tonga_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
-					SMU72_Discrete_DpmTable *table)
-{
-	int result = 0;
-
-	uint8_t count;
-	pp_atomctrl_clock_dividers_vi dividers;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
-
-	table->UvdLevelCount = (uint8_t) (mm_table->count);
-	table->UvdBootLevel = 0;
-
-	for (count = 0; count < table->UvdLevelCount; count++) {
-		table->UvdLevel[count].VclkFrequency = mm_table->entries[count].vclk;
-		table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk;
-		table->UvdLevel[count].MinVoltage.Vddc =
-			tonga_get_voltage_index(pptable_info->vddc_lookup_table,
-						mm_table->entries[count].vddc);
-		table->UvdLevel[count].MinVoltage.VddGfx =
-			(data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) ?
-			tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
-						mm_table->entries[count].vddgfx) : 0;
-		table->UvdLevel[count].MinVoltage.Vddci =
-			tonga_get_voltage_id(&data->vddci_voltage_table,
-					     mm_table->entries[count].vddc - data->vddc_vddci_delta);
-		table->UvdLevel[count].MinVoltage.Phases = 1;
-
-		/* retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-							  table->UvdLevel[count].VclkFrequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				    "can not find divide id for Vclk clock", return result);
-
-		table->UvdLevel[count].VclkDivider = (uint8_t)dividers.pll_post_divider;
-
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-							  table->UvdLevel[count].DclkFrequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				    "can not find divide id for Dclk clock", return result);
-
-		table->UvdLevel[count].DclkDivider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency);
-		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency);
-		//CONVERT_FROM_HOST_TO_SMC_UL((uint32_t)table->UvdLevel[count].MinVoltage);
-	}
-
-	return result;
-
-}
-
-static int tonga_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
-		SMU72_Discrete_DpmTable *table)
-{
-	int result = 0;
-
-	uint8_t count;
-	pp_atomctrl_clock_dividers_vi dividers;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
-
-	table->VceLevelCount = (uint8_t) (mm_table->count);
-	table->VceBootLevel = 0;
-
-	for (count = 0; count < table->VceLevelCount; count++) {
-		table->VceLevel[count].Frequency =
-			mm_table->entries[count].eclk;
-		table->VceLevel[count].MinVoltage.Vddc =
-			tonga_get_voltage_index(pptable_info->vddc_lookup_table,
-				mm_table->entries[count].vddc);
-		table->VceLevel[count].MinVoltage.VddGfx =
-			(data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) ?
-			tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
-				mm_table->entries[count].vddgfx) : 0;
-		table->VceLevel[count].MinVoltage.Vddci =
-			tonga_get_voltage_id(&data->vddci_voltage_table,
-				mm_table->entries[count].vddc - data->vddc_vddci_delta);
-		table->VceLevel[count].MinVoltage.Phases = 1;
-
-		/* retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-					table->VceLevel[count].Frequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"can not find divide id for VCE engine clock", return result);
-
-		table->VceLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].Frequency);
-	}
-
-	return result;
-}
-
-static int tonga_populate_smc_acp_level(struct pp_hwmgr *hwmgr,
-		SMU72_Discrete_DpmTable *table)
-{
-	int result = 0;
-	uint8_t count;
-	pp_atomctrl_clock_dividers_vi dividers;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
-
-	table->AcpLevelCount = (uint8_t) (mm_table->count);
-	table->AcpBootLevel = 0;
-
-	for (count = 0; count < table->AcpLevelCount; count++) {
-		table->AcpLevel[count].Frequency =
-			pptable_info->mm_dep_table->entries[count].aclk;
-		table->AcpLevel[count].MinVoltage.Vddc =
-			tonga_get_voltage_index(pptable_info->vddc_lookup_table,
-			mm_table->entries[count].vddc);
-		table->AcpLevel[count].MinVoltage.VddGfx =
-			(data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) ?
-			tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
-				mm_table->entries[count].vddgfx) : 0;
-		table->AcpLevel[count].MinVoltage.Vddci =
-			tonga_get_voltage_id(&data->vddci_voltage_table,
-				mm_table->entries[count].vddc - data->vddc_vddci_delta);
-		table->AcpLevel[count].MinVoltage.Phases = 1;
-
-		/* retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-			table->AcpLevel[count].Frequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-			"can not find divide id for engine clock", return result);
-
-		table->AcpLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->AcpLevel[count].Frequency);
-	}
-
-	return result;
-}
-
-static int tonga_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
-	SMU72_Discrete_DpmTable *table)
-{
-	int result = 0;
-	uint8_t count;
-	pp_atomctrl_clock_dividers_vi dividers;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
-
-	table->SamuBootLevel = 0;
-	table->SamuLevelCount = (uint8_t) (mm_table->count);
-
-	for (count = 0; count < table->SamuLevelCount; count++) {
-		/* not sure whether we need evclk or not */
-		table->SamuLevel[count].Frequency =
-			pptable_info->mm_dep_table->entries[count].samclock;
-		table->SamuLevel[count].MinVoltage.Vddc =
-			tonga_get_voltage_index(pptable_info->vddc_lookup_table,
-				mm_table->entries[count].vddc);
-		table->SamuLevel[count].MinVoltage.VddGfx =
-			(data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) ?
-			tonga_get_voltage_index(pptable_info->vddgfx_lookup_table,
-				mm_table->entries[count].vddgfx) : 0;
-		table->SamuLevel[count].MinVoltage.Vddci =
-			tonga_get_voltage_id(&data->vddci_voltage_table,
-				mm_table->entries[count].vddc - data->vddc_vddci_delta);
-		table->SamuLevel[count].MinVoltage.Phases = 1;
-
-		/* retrieve divider value for VBIOS */
-		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
-					table->SamuLevel[count].Frequency, &dividers);
-		PP_ASSERT_WITH_CODE((0 == result),
-			"can not find divide id for samu clock", return result);
-
-		table->SamuLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
-
-		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].Frequency);
-	}
-
-	return result;
-}
-
-/**
- * Populates the SMC MCLK structure using the provided memory clock
- *
- * @param    hwmgr      the address of the hardware manager
- * @param    memory_clock the memory clock to use to populate the structure
- * @param    sclk        the SMC SCLK structure to be populated
- */
-static int tonga_calculate_mclk_params(
-		struct pp_hwmgr *hwmgr,
-		uint32_t memory_clock,
-		SMU72_Discrete_MemoryLevel *mclk,
-		bool strobe_mode,
-		bool dllStateOn
-		)
-{
-	const tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	uint32_t  dll_cntl = data->clock_registers.vDLL_CNTL;
-	uint32_t  mclk_pwrmgt_cntl = data->clock_registers.vMCLK_PWRMGT_CNTL;
-	uint32_t  mpll_ad_func_cntl = data->clock_registers.vMPLL_AD_FUNC_CNTL;
-	uint32_t  mpll_dq_func_cntl = data->clock_registers.vMPLL_DQ_FUNC_CNTL;
-	uint32_t  mpll_func_cntl = data->clock_registers.vMPLL_FUNC_CNTL;
-	uint32_t  mpll_func_cntl_1 = data->clock_registers.vMPLL_FUNC_CNTL_1;
-	uint32_t  mpll_func_cntl_2 = data->clock_registers.vMPLL_FUNC_CNTL_2;
-	uint32_t  mpll_ss1 = data->clock_registers.vMPLL_SS1;
-	uint32_t  mpll_ss2 = data->clock_registers.vMPLL_SS2;
-
-	pp_atomctrl_memory_clock_param mpll_param;
-	int result;
-
-	result = atomctrl_get_memory_pll_dividers_si(hwmgr,
-				memory_clock, &mpll_param, strobe_mode);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Error retrieving Memory Clock Parameters from VBIOS.", return result);
-
-	/* MPLL_FUNC_CNTL setup*/
-	mpll_func_cntl = PHM_SET_FIELD(mpll_func_cntl, MPLL_FUNC_CNTL, BWCTRL, mpll_param.bw_ctrl);
-
-	/* MPLL_FUNC_CNTL_1 setup*/
-	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
-							MPLL_FUNC_CNTL_1, CLKF, mpll_param.mpll_fb_divider.cl_kf);
-	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
-							MPLL_FUNC_CNTL_1, CLKFRAC, mpll_param.mpll_fb_divider.clk_frac);
-	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
-							MPLL_FUNC_CNTL_1, VCO_MODE, mpll_param.vco_mode);
-
-	/* MPLL_AD_FUNC_CNTL setup*/
-	mpll_ad_func_cntl = PHM_SET_FIELD(mpll_ad_func_cntl,
-							MPLL_AD_FUNC_CNTL, YCLK_POST_DIV, mpll_param.mpll_post_divider);
-
-	if (data->is_memory_GDDR5) {
-		/* MPLL_DQ_FUNC_CNTL setup*/
-		mpll_dq_func_cntl  = PHM_SET_FIELD(mpll_dq_func_cntl,
-								MPLL_DQ_FUNC_CNTL, YCLK_SEL, mpll_param.yclk_sel);
-		mpll_dq_func_cntl  = PHM_SET_FIELD(mpll_dq_func_cntl,
-								MPLL_DQ_FUNC_CNTL, YCLK_POST_DIV, mpll_param.mpll_post_divider);
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_MemorySpreadSpectrumSupport)) {
-		/*
-		 ************************************
-		 Fref = Reference Frequency
-		 NF = Feedback divider ratio
-		 NR = Reference divider ratio
-		 Fnom = Nominal VCO output frequency = Fref * NF / NR
-		 Fs = Spreading Rate
-		 D = Percentage down-spread / 2
-		 Fint = Reference input frequency to PFD = Fref / NR
-		 NS = Spreading rate divider ratio = int(Fint / (2 * Fs))
-		 CLKS = NS - 1 = ISS_STEP_NUM[11:0]
-		 NV = D * Fs / Fnom * 4 * ((Fnom/Fref * NR) ^ 2)
-		 CLKV = 65536 * NV = ISS_STEP_SIZE[25:0]
-		 *************************************
-		 */
-		pp_atomctrl_internal_ss_info ss_info;
-		uint32_t freq_nom;
-		uint32_t tmp;
-		uint32_t reference_clock = atomctrl_get_mpll_reference_clock(hwmgr);
-
-		/* for GDDR5 for all modes and DDR3 */
-		if (1 == mpll_param.qdr)
-			freq_nom = memory_clock * 4 * (1 << mpll_param.mpll_post_divider);
-		else
-			freq_nom = memory_clock * 2 * (1 << mpll_param.mpll_post_divider);
-
-		/* tmp = (freq_nom / reference_clock * reference_divider) ^ 2  Note: S.I. reference_divider = 1*/
-		tmp = (freq_nom / reference_clock);
-		tmp = tmp * tmp;
-
-		if (0 == atomctrl_get_memory_clock_spread_spectrum(hwmgr, freq_nom, &ss_info)) {
-			/* ss_info.speed_spectrum_percentage -- in unit of 0.01% */
-			/* ss.Info.speed_spectrum_rate -- in unit of khz */
-			/* CLKS = reference_clock / (2 * speed_spectrum_rate * reference_divider) * 10 */
-			/*     = reference_clock * 5 / speed_spectrum_rate */
-			uint32_t clks = reference_clock * 5 / ss_info.speed_spectrum_rate;
-
-			/* CLKV = 65536 * speed_spectrum_percentage / 2 * spreadSpecrumRate / freq_nom * 4 / 100000 * ((freq_nom / reference_clock) ^ 2) */
-			/*     = 131 * speed_spectrum_percentage * speed_spectrum_rate / 100 * ((freq_nom / reference_clock) ^ 2) / freq_nom */
-			uint32_t clkv =
-				(uint32_t)((((131 * ss_info.speed_spectrum_percentage *
-							ss_info.speed_spectrum_rate) / 100) * tmp) / freq_nom);
-
-			mpll_ss1 = PHM_SET_FIELD(mpll_ss1, MPLL_SS1, CLKV, clkv);
-			mpll_ss2 = PHM_SET_FIELD(mpll_ss2, MPLL_SS2, CLKS, clks);
-		}
-	}
-
-	/* MCLK_PWRMGT_CNTL setup */
-	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
-		MCLK_PWRMGT_CNTL, DLL_SPEED, mpll_param.dll_speed);
-	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
-		MCLK_PWRMGT_CNTL, MRDCK0_PDNB, dllStateOn);
-	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
-		MCLK_PWRMGT_CNTL, MRDCK1_PDNB, dllStateOn);
-
-
-	/* Save the result data to outpupt memory level structure */
-	mclk->MclkFrequency   = memory_clock;
-	mclk->MpllFuncCntl    = mpll_func_cntl;
-	mclk->MpllFuncCntl_1  = mpll_func_cntl_1;
-	mclk->MpllFuncCntl_2  = mpll_func_cntl_2;
-	mclk->MpllAdFuncCntl  = mpll_ad_func_cntl;
-	mclk->MpllDqFuncCntl  = mpll_dq_func_cntl;
-	mclk->MclkPwrmgtCntl  = mclk_pwrmgt_cntl;
-	mclk->DllCntl         = dll_cntl;
-	mclk->MpllSs1         = mpll_ss1;
-	mclk->MpllSs2         = mpll_ss2;
-
-	return 0;
-}
-
-static uint8_t tonga_get_mclk_frequency_ratio(uint32_t memory_clock,
-		bool strobe_mode)
-{
-	uint8_t mc_para_index;
-
-	if (strobe_mode) {
-		if (memory_clock < 12500) {
-			mc_para_index = 0x00;
-		} else if (memory_clock > 47500) {
-			mc_para_index = 0x0f;
-		} else {
-			mc_para_index = (uint8_t)((memory_clock - 10000) / 2500);
-		}
-	} else {
-		if (memory_clock < 65000) {
-			mc_para_index = 0x00;
-		} else if (memory_clock > 135000) {
-			mc_para_index = 0x0f;
-		} else {
-			mc_para_index = (uint8_t)((memory_clock - 60000) / 5000);
-		}
-	}
-
-	return mc_para_index;
-}
-
-static uint8_t tonga_get_ddr3_mclk_frequency_ratio(uint32_t memory_clock)
-{
-	uint8_t mc_para_index;
-
-	if (memory_clock < 10000) {
-		mc_para_index = 0;
-	} else if (memory_clock >= 80000) {
-		mc_para_index = 0x0f;
-	} else {
-		mc_para_index = (uint8_t)((memory_clock - 10000) / 5000 + 1);
-	}
-
-	return mc_para_index;
-}
-
-static int tonga_populate_single_memory_level(
-		struct pp_hwmgr *hwmgr,
-		uint32_t memory_clock,
-		SMU72_Discrete_MemoryLevel *memory_level
-		)
-{
-	uint32_t minMvdd = 0;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	int result = 0;
-	bool dllStateOn;
-	struct cgs_display_info info = {0};
-
-
-	if (NULL != pptable_info->vdd_dep_on_mclk) {
-		result = tonga_get_dependecy_volt_by_clk(hwmgr,
-			pptable_info->vdd_dep_on_mclk, memory_clock, &memory_level->MinVoltage, &minMvdd);
-		PP_ASSERT_WITH_CODE((0 == result),
-			"can not find MinVddc voltage value from memory VDDC voltage dependency table", return result);
-	}
-
-	if (data->mvdd_control == TONGA_VOLTAGE_CONTROL_NONE) {
-		memory_level->MinMvdd = data->vbios_boot_state.mvdd_bootup_value;
-	} else {
-		memory_level->MinMvdd = minMvdd;
-	}
-	memory_level->EnabledForThrottle = 1;
-	memory_level->EnabledForActivity = 0;
-	memory_level->UpHyst = 0;
-	memory_level->DownHyst = 100;
-	memory_level->VoltageDownHyst = 0;
-
-	/* Indicates maximum activity level for this performance level.*/
-	memory_level->ActivityLevel = (uint16_t)data->mclk_activity_target;
-	memory_level->StutterEnable = 0;
-	memory_level->StrobeEnable = 0;
-	memory_level->EdcReadEnable = 0;
-	memory_level->EdcWriteEnable = 0;
-	memory_level->RttEnable = 0;
-
-	/* default set to low watermark. Highest level will be set to high later.*/
-	memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	data->display_timing.num_existing_displays = info.display_count;
-
-	if ((data->mclk_stutter_mode_threshold != 0) &&
-	    (memory_clock <= data->mclk_stutter_mode_threshold) &&
-	    (!data->is_uvd_enabled)
-	    && (PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL, STUTTER_ENABLE) & 0x1)
-	    && (data->display_timing.num_existing_displays <= 2)
-	    && (data->display_timing.num_existing_displays != 0))
-		memory_level->StutterEnable = 1;
-
-	/* decide strobe mode*/
-	memory_level->StrobeEnable = (data->mclk_strobe_mode_threshold != 0) &&
-		(memory_clock <= data->mclk_strobe_mode_threshold);
-
-	/* decide EDC mode and memory clock ratio*/
-	if (data->is_memory_GDDR5) {
-		memory_level->StrobeRatio = tonga_get_mclk_frequency_ratio(memory_clock,
-					memory_level->StrobeEnable);
-
-		if ((data->mclk_edc_enable_threshold != 0) &&
-				(memory_clock > data->mclk_edc_enable_threshold)) {
-			memory_level->EdcReadEnable = 1;
-		}
-
-		if ((data->mclk_edc_wr_enable_threshold != 0) &&
-				(memory_clock > data->mclk_edc_wr_enable_threshold)) {
-			memory_level->EdcWriteEnable = 1;
-		}
-
-		if (memory_level->StrobeEnable) {
-			if (tonga_get_mclk_frequency_ratio(memory_clock, 1) >=
-					((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC7) >> 16) & 0xf)) {
-				dllStateOn = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC5) >> 1) & 0x1) ? 1 : 0;
-			} else {
-				dllStateOn = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC6) >> 1) & 0x1) ? 1 : 0;
-			}
-
-		} else {
-			dllStateOn = data->dll_defaule_on;
-		}
-	} else {
-		memory_level->StrobeRatio =
-			tonga_get_ddr3_mclk_frequency_ratio(memory_clock);
-		dllStateOn = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC5) >> 1) & 0x1) ? 1 : 0;
-	}
-
-	result = tonga_calculate_mclk_params(hwmgr,
-		memory_clock, memory_level, memory_level->StrobeEnable, dllStateOn);
-
-	if (0 == result) {
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MinMvdd);
-		/* MCLK frequency in units of 10KHz*/
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MclkFrequency);
-		/* Indicates maximum activity level for this performance level.*/
-		CONVERT_FROM_HOST_TO_SMC_US(memory_level->ActivityLevel);
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl);
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl_1);
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl_2);
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllAdFuncCntl);
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllDqFuncCntl);
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MclkPwrmgtCntl);
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->DllCntl);
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllSs1);
-		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllSs2);
-	}
-
-	return result;
-}
-
-/**
- * Populates the SMC MVDD structure using the provided memory clock.
- *
- * @param    hwmgr      the address of the hardware manager
- * @param    mclk        the MCLK value to be used in the decision if MVDD should be high or low.
- * @param    voltage     the SMC VOLTAGE structure to be populated
- */
-int tonga_populate_mvdd_value(struct pp_hwmgr *hwmgr, uint32_t mclk, SMIO_Pattern *smio_pattern)
-{
-	const tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint32_t i = 0;
-
-	if (TONGA_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
-		/* find mvdd value which clock is more than request */
-		for (i = 0; i < pptable_info->vdd_dep_on_mclk->count; i++) {
-			if (mclk <= pptable_info->vdd_dep_on_mclk->entries[i].clk) {
-				/* Always round to higher voltage. */
-				smio_pattern->Voltage = data->mvdd_voltage_table.entries[i].value;
-				break;
-			}
-		}
-
-		PP_ASSERT_WITH_CODE(i < pptable_info->vdd_dep_on_mclk->count,
-			"MVDD Voltage is outside the supported range.", return -1);
-
-	} else {
-		return -1;
-	}
-
-	return 0;
-}
-
-
-static int tonga_populate_smv_acpi_level(struct pp_hwmgr *hwmgr,
-	SMU72_Discrete_DpmTable *table)
-{
-	int result = 0;
-	const tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	pp_atomctrl_clock_dividers_vi dividers;
-	SMIO_Pattern voltage_level;
-	uint32_t spll_func_cntl    = data->clock_registers.vCG_SPLL_FUNC_CNTL;
-	uint32_t spll_func_cntl_2  = data->clock_registers.vCG_SPLL_FUNC_CNTL_2;
-	uint32_t dll_cntl          = data->clock_registers.vDLL_CNTL;
-	uint32_t mclk_pwrmgt_cntl  = data->clock_registers.vMCLK_PWRMGT_CNTL;
-
-	/* The ACPI state should not do DPM on DC (or ever).*/
-	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
-
-	table->ACPILevel.MinVoltage = data->smc_state_table.GraphicsLevel[0].MinVoltage;
-
-	/* assign zero for now*/
-	table->ACPILevel.SclkFrequency = atomctrl_get_reference_clock(hwmgr);
-
-	/* get the engine clock dividers for this clock value*/
-	result = atomctrl_get_engine_pll_dividers_vi(hwmgr,
-		table->ACPILevel.SclkFrequency,  &dividers);
-
-	PP_ASSERT_WITH_CODE(result == 0,
-		"Error retrieving Engine Clock dividers from VBIOS.", return result);
-
-	/* divider ID for required SCLK*/
-	table->ACPILevel.SclkDid = (uint8_t)dividers.pll_post_divider;
-	table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
-	table->ACPILevel.DeepSleepDivId = 0;
-
-	spll_func_cntl      = PHM_SET_FIELD(spll_func_cntl,
-							CG_SPLL_FUNC_CNTL,   SPLL_PWRON,     0);
-	spll_func_cntl      = PHM_SET_FIELD(spll_func_cntl,
-							CG_SPLL_FUNC_CNTL,   SPLL_RESET,     1);
-	spll_func_cntl_2    = PHM_SET_FIELD(spll_func_cntl_2,
-							CG_SPLL_FUNC_CNTL_2, SCLK_MUX_SEL,   4);
-
-	table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
-	table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
-	table->ACPILevel.CgSpllFuncCntl3 = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
-	table->ACPILevel.CgSpllFuncCntl4 = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
-	table->ACPILevel.SpllSpreadSpectrum = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
-	table->ACPILevel.SpllSpreadSpectrum2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
-	table->ACPILevel.CcPwrDynRm = 0;
-	table->ACPILevel.CcPwrDynRm1 = 0;
-
-
-	/* For various features to be enabled/disabled while this level is active.*/
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
-	/* SCLK frequency in units of 10KHz*/
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkFrequency);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl2);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl3);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl4);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum2);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
-
-	/* table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases;*/
-	table->MemoryACPILevel.MinVoltage = data->smc_state_table.MemoryLevel[0].MinVoltage;
-
-	/*  CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MinVoltage);*/
-
-	if (0 == tonga_populate_mvdd_value(hwmgr, 0, &voltage_level))
-		table->MemoryACPILevel.MinMvdd =
-			PP_HOST_TO_SMC_UL(voltage_level.Voltage * VOLTAGE_SCALE);
-	else
-		table->MemoryACPILevel.MinMvdd = 0;
-
-	/* Force reset on DLL*/
-	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
-		MCLK_PWRMGT_CNTL, MRDCK0_RESET, 0x1);
-	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
-		MCLK_PWRMGT_CNTL, MRDCK1_RESET, 0x1);
-
-	/* Disable DLL in ACPIState*/
-	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
-		MCLK_PWRMGT_CNTL, MRDCK0_PDNB, 0);
-	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
-		MCLK_PWRMGT_CNTL, MRDCK1_PDNB, 0);
-
-	/* Enable DLL bypass signal*/
-	dll_cntl            = PHM_SET_FIELD(dll_cntl,
-		DLL_CNTL, MRDCK0_BYPASS, 0);
-	dll_cntl            = PHM_SET_FIELD(dll_cntl,
-		DLL_CNTL, MRDCK1_BYPASS, 0);
-
-	table->MemoryACPILevel.DllCntl            =
-		PP_HOST_TO_SMC_UL(dll_cntl);
-	table->MemoryACPILevel.MclkPwrmgtCntl     =
-		PP_HOST_TO_SMC_UL(mclk_pwrmgt_cntl);
-	table->MemoryACPILevel.MpllAdFuncCntl     =
-		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_AD_FUNC_CNTL);
-	table->MemoryACPILevel.MpllDqFuncCntl     =
-		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_DQ_FUNC_CNTL);
-	table->MemoryACPILevel.MpllFuncCntl       =
-		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL);
-	table->MemoryACPILevel.MpllFuncCntl_1     =
-		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL_1);
-	table->MemoryACPILevel.MpllFuncCntl_2     =
-		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL_2);
-	table->MemoryACPILevel.MpllSs1            =
-		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_SS1);
-	table->MemoryACPILevel.MpllSs2            =
-		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_SS2);
-
-	table->MemoryACPILevel.EnabledForThrottle = 0;
-	table->MemoryACPILevel.EnabledForActivity = 0;
-	table->MemoryACPILevel.UpHyst = 0;
-	table->MemoryACPILevel.DownHyst = 100;
-	table->MemoryACPILevel.VoltageDownHyst = 0;
-	/* Indicates maximum activity level for this performance level.*/
-	table->MemoryACPILevel.ActivityLevel = PP_HOST_TO_SMC_US((uint16_t)data->mclk_activity_target);
-
-	table->MemoryACPILevel.StutterEnable = 0;
-	table->MemoryACPILevel.StrobeEnable = 0;
-	table->MemoryACPILevel.EdcReadEnable = 0;
-	table->MemoryACPILevel.EdcWriteEnable = 0;
-	table->MemoryACPILevel.RttEnable = 0;
-
-	return result;
-}
-
-static int tonga_find_boot_level(struct tonga_single_dpm_table *table, uint32_t value, uint32_t *boot_level)
-{
-	int result = 0;
-	uint32_t i;
-
-	for (i = 0; i < table->count; i++) {
-		if (value == table->dpm_levels[i].value) {
-			*boot_level = i;
-			result = 0;
-		}
-	}
-	return result;
-}
-
-static int tonga_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
-			SMU72_Discrete_DpmTable *table)
-{
-	int result = 0;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	table->GraphicsBootLevel  = 0;        /* 0 == DPM[0] (low), etc. */
-	table->MemoryBootLevel    = 0;        /* 0 == DPM[0] (low), etc. */
-
-	/* find boot level from dpm table*/
-	result = tonga_find_boot_level(&(data->dpm_table.sclk_table),
-	data->vbios_boot_state.sclk_bootup_value,
-	(uint32_t *)&(data->smc_state_table.GraphicsBootLevel));
-
-	if (0 != result) {
-		data->smc_state_table.GraphicsBootLevel = 0;
-		printk(KERN_ERR "[ powerplay ] VBIOS did not find boot engine clock value \
-			in dependency table. Using Graphics DPM level 0!");
-		result = 0;
-	}
-
-	result = tonga_find_boot_level(&(data->dpm_table.mclk_table),
-		data->vbios_boot_state.mclk_bootup_value,
-		(uint32_t *)&(data->smc_state_table.MemoryBootLevel));
-
-	if (0 != result) {
-		data->smc_state_table.MemoryBootLevel = 0;
-		printk(KERN_ERR "[ powerplay ] VBIOS did not find boot engine clock value \
-			in dependency table. Using Memory DPM level 0!");
-		result = 0;
-	}
-
-	table->BootVoltage.Vddc =
-		tonga_get_voltage_id(&(data->vddc_voltage_table),
-			data->vbios_boot_state.vddc_bootup_value);
-	table->BootVoltage.VddGfx =
-		tonga_get_voltage_id(&(data->vddgfx_voltage_table),
-			data->vbios_boot_state.vddgfx_bootup_value);
-	table->BootVoltage.Vddci =
-		tonga_get_voltage_id(&(data->vddci_voltage_table),
-			data->vbios_boot_state.vddci_bootup_value);
-	table->BootMVdd = data->vbios_boot_state.mvdd_bootup_value;
-
-	CONVERT_FROM_HOST_TO_SMC_US(table->BootMVdd);
-
-	return result;
-}
-
-
-/**
- * Calculates the SCLK dividers using the provided engine clock
- *
- * @param    hwmgr      the address of the hardware manager
- * @param    engine_clock the engine clock to use to populate the structure
- * @param    sclk        the SMC SCLK structure to be populated
- */
-int tonga_calculate_sclk_params(struct pp_hwmgr *hwmgr,
-		uint32_t engine_clock, SMU72_Discrete_GraphicsLevel *sclk)
-{
-	const tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	pp_atomctrl_clock_dividers_vi dividers;
-	uint32_t spll_func_cntl            = data->clock_registers.vCG_SPLL_FUNC_CNTL;
-	uint32_t spll_func_cntl_3          = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
-	uint32_t spll_func_cntl_4          = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
-	uint32_t cg_spll_spread_spectrum   = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
-	uint32_t cg_spll_spread_spectrum_2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
-	uint32_t    reference_clock;
-	uint32_t reference_divider;
-	uint32_t fbdiv;
-	int result;
-
-	/* get the engine clock dividers for this clock value*/
-	result = atomctrl_get_engine_pll_dividers_vi(hwmgr, engine_clock,  &dividers);
-
-	PP_ASSERT_WITH_CODE(result == 0,
-		"Error retrieving Engine Clock dividers from VBIOS.", return result);
-
-	/* To get FBDIV we need to multiply this by 16384 and divide it by Fref.*/
-	reference_clock = atomctrl_get_reference_clock(hwmgr);
-
-	reference_divider = 1 + dividers.uc_pll_ref_div;
-
-	/* low 14 bits is fraction and high 12 bits is divider*/
-	fbdiv = dividers.ul_fb_div.ul_fb_divider & 0x3FFFFFF;
-
-	/* SPLL_FUNC_CNTL setup*/
-	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl,
-		CG_SPLL_FUNC_CNTL, SPLL_REF_DIV, dividers.uc_pll_ref_div);
-	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl,
-		CG_SPLL_FUNC_CNTL, SPLL_PDIV_A,  dividers.uc_pll_post_div);
-
-	/* SPLL_FUNC_CNTL_3 setup*/
-	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3,
-		CG_SPLL_FUNC_CNTL_3, SPLL_FB_DIV, fbdiv);
-
-	/* set to use fractional accumulation*/
-	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3,
-		CG_SPLL_FUNC_CNTL_3, SPLL_DITHEN, 1);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_EngineSpreadSpectrumSupport)) {
-		pp_atomctrl_internal_ss_info ss_info;
-
-		uint32_t vcoFreq = engine_clock * dividers.uc_pll_post_div;
-		if (0 == atomctrl_get_engine_clock_spread_spectrum(hwmgr, vcoFreq, &ss_info)) {
-			/*
-			* ss_info.speed_spectrum_percentage -- in unit of 0.01%
-			* ss_info.speed_spectrum_rate -- in unit of khz
-			*/
-			/* clks = reference_clock * 10 / (REFDIV + 1) / speed_spectrum_rate / 2 */
-			uint32_t clkS = reference_clock * 5 / (reference_divider * ss_info.speed_spectrum_rate);
-
-			/* clkv = 2 * D * fbdiv / NS */
-			uint32_t clkV = 4 * ss_info.speed_spectrum_percentage * fbdiv / (clkS * 10000);
-
-			cg_spll_spread_spectrum =
-				PHM_SET_FIELD(cg_spll_spread_spectrum, CG_SPLL_SPREAD_SPECTRUM, CLKS, clkS);
-			cg_spll_spread_spectrum =
-				PHM_SET_FIELD(cg_spll_spread_spectrum, CG_SPLL_SPREAD_SPECTRUM, SSEN, 1);
-			cg_spll_spread_spectrum_2 =
-				PHM_SET_FIELD(cg_spll_spread_spectrum_2, CG_SPLL_SPREAD_SPECTRUM_2, CLKV, clkV);
-		}
-	}
-
-	sclk->SclkFrequency        = engine_clock;
-	sclk->CgSpllFuncCntl3      = spll_func_cntl_3;
-	sclk->CgSpllFuncCntl4      = spll_func_cntl_4;
-	sclk->SpllSpreadSpectrum   = cg_spll_spread_spectrum;
-	sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
-	sclk->SclkDid              = (uint8_t)dividers.pll_post_divider;
-
-	return 0;
-}
-
-static uint8_t tonga_get_sleep_divider_id_from_clock(uint32_t engine_clock,
-		uint32_t min_engine_clock_in_sr)
-{
-	uint32_t i, temp;
-	uint32_t min = max(min_engine_clock_in_sr, (uint32_t)TONGA_MINIMUM_ENGINE_CLOCK);
-
-	PP_ASSERT_WITH_CODE((engine_clock >= min),
-			"Engine clock can't satisfy stutter requirement!", return 0);
-
-	for (i = TONGA_MAX_DEEPSLEEP_DIVIDER_ID;; i--) {
-		temp = engine_clock >> i;
-
-		if(temp >= min || i == 0)
-			break;
-	}
-	return (uint8_t)i;
-}
-
-/**
- * Populates single SMC SCLK structure using the provided engine clock
- *
- * @param    hwmgr      the address of the hardware manager
- * @param    engine_clock the engine clock to use to populate the structure
- * @param    sclk        the SMC SCLK structure to be populated
- */
-static int tonga_populate_single_graphic_level(struct pp_hwmgr *hwmgr, uint32_t engine_clock, uint16_t sclk_activity_level_threshold, SMU72_Discrete_GraphicsLevel *graphic_level)
-{
-	int result;
-	uint32_t threshold;
-	uint32_t mvdd;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	result = tonga_calculate_sclk_params(hwmgr, engine_clock, graphic_level);
-
-
-	/* populate graphics levels*/
-	result = tonga_get_dependecy_volt_by_clk(hwmgr,
-		pptable_info->vdd_dep_on_sclk, engine_clock,
-		&graphic_level->MinVoltage, &mvdd);
-	PP_ASSERT_WITH_CODE((0 == result),
-		"can not find VDDC voltage value for VDDC	\
-		engine clock dependency table", return result);
-
-	/* SCLK frequency in units of 10KHz*/
-	graphic_level->SclkFrequency = engine_clock;
-
-	/* Indicates maximum activity level for this performance level. 50% for now*/
-	graphic_level->ActivityLevel = sclk_activity_level_threshold;
-
-	graphic_level->CcPwrDynRm = 0;
-	graphic_level->CcPwrDynRm1 = 0;
-	/* this level can be used if activity is high enough.*/
-	graphic_level->EnabledForActivity = 0;
-	/* this level can be used for throttling.*/
-	graphic_level->EnabledForThrottle = 1;
-	graphic_level->UpHyst = 0;
-	graphic_level->DownHyst = 0;
-	graphic_level->VoltageDownHyst = 0;
-	graphic_level->PowerThrottle = 0;
-
-	threshold = engine_clock * data->fast_watemark_threshold / 100;
-/*
-	*get the DAL clock. do it in funture.
-	PECI_GetMinClockSettings(hwmgr->peci, &minClocks);
-	data->display_timing.min_clock_insr = minClocks.engineClockInSR;
-*/
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_SclkDeepSleep))
-		graphic_level->DeepSleepDivId =
-				tonga_get_sleep_divider_id_from_clock(engine_clock,
-						data->display_timing.min_clock_insr);
-
-	/* Default to slow, highest DPM level will be set to PPSMC_DISPLAY_WATERMARK_LOW later.*/
-	graphic_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
-
-	if (0 == result) {
-		/* CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->MinVoltage);*/
-		/* CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->MinVddcPhases);*/
-		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SclkFrequency);
-		CONVERT_FROM_HOST_TO_SMC_US(graphic_level->ActivityLevel);
-		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CgSpllFuncCntl3);
-		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CgSpllFuncCntl4);
-		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SpllSpreadSpectrum);
-		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SpllSpreadSpectrum2);
-		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CcPwrDynRm);
-		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CcPwrDynRm1);
-	}
-
-	return result;
-}
-
-/**
- * Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states
- *
- * @param    hwmgr      the address of the hardware manager
- */
-static int tonga_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	struct tonga_dpm_table *dpm_table = &data->dpm_table;
-	phm_ppt_v1_pcie_table *pcie_table = pptable_info->pcie_table;
-	uint8_t pcie_entry_count = (uint8_t) data->dpm_table.pcie_speed_table.count;
-	int result = 0;
-	uint32_t level_array_adress = data->dpm_table_start +
-		offsetof(SMU72_Discrete_DpmTable, GraphicsLevel);
-	uint32_t level_array_size = sizeof(SMU72_Discrete_GraphicsLevel) *
-		SMU72_MAX_LEVELS_GRAPHICS;   /* 64 -> long; 32 -> int*/
-	SMU72_Discrete_GraphicsLevel *levels = data->smc_state_table.GraphicsLevel;
-	uint32_t i, maxEntry;
-	uint8_t highest_pcie_level_enabled = 0, lowest_pcie_level_enabled = 0, mid_pcie_level_enabled = 0, count = 0;
-	PECI_RegistryValue reg_value;
-	memset(levels, 0x00, level_array_size);
-
-	for (i = 0; i < dpm_table->sclk_table.count; i++) {
-		result = tonga_populate_single_graphic_level(hwmgr,
-					dpm_table->sclk_table.dpm_levels[i].value,
-					(uint16_t)data->activity_target[i],
-					&(data->smc_state_table.GraphicsLevel[i]));
-
-		if (0 != result)
-			return result;
-
-		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
-		if (i > 1)
-			data->smc_state_table.GraphicsLevel[i].DeepSleepDivId = 0;
-
-		if (0 == i) {
-			reg_value = 0;
-			if (reg_value != 0)
-				data->smc_state_table.GraphicsLevel[0].UpHyst = (uint8_t)reg_value;
-		}
-
-		if (1 == i) {
-			reg_value = 0;
-			if (reg_value != 0)
-				data->smc_state_table.GraphicsLevel[1].UpHyst = (uint8_t)reg_value;
-		}
-	}
-
-	/* Only enable level 0 for now. */
-	data->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1;
-
-	/* set highest level watermark to high */
-	if (dpm_table->sclk_table.count > 1)
-		data->smc_state_table.GraphicsLevel[dpm_table->sclk_table.count-1].DisplayWatermark =
-			PPSMC_DISPLAY_WATERMARK_HIGH;
-
-	data->smc_state_table.GraphicsDpmLevelCount =
-		(uint8_t)dpm_table->sclk_table.count;
-	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
-		tonga_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
-
-	if (pcie_table != NULL) {
-		PP_ASSERT_WITH_CODE((pcie_entry_count >= 1),
-			"There must be 1 or more PCIE levels defined in PPTable.", return -1);
-		maxEntry = pcie_entry_count - 1; /* for indexing, we need to decrement by 1.*/
-		for (i = 0; i < dpm_table->sclk_table.count; i++) {
-			data->smc_state_table.GraphicsLevel[i].pcieDpmLevel =
-				(uint8_t) ((i < maxEntry) ? i : maxEntry);
-		}
-	} else {
-		if (0 == data->dpm_level_enable_mask.pcie_dpm_enable_mask)
-			printk(KERN_ERR "[ powerplay ] Pcie Dpm Enablemask is 0!");
-
-		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
-				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
-					(1<<(highest_pcie_level_enabled+1))) != 0)) {
-			highest_pcie_level_enabled++;
-		}
-
-		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
-				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
-					(1<<lowest_pcie_level_enabled)) == 0)) {
-			lowest_pcie_level_enabled++;
-		}
-
-		while ((count < highest_pcie_level_enabled) &&
-				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
-					(1<<(lowest_pcie_level_enabled+1+count))) == 0)) {
-			count++;
-		}
-		mid_pcie_level_enabled = (lowest_pcie_level_enabled+1+count) < highest_pcie_level_enabled ?
-			(lowest_pcie_level_enabled+1+count) : highest_pcie_level_enabled;
-
-
-		/* set pcieDpmLevel to highest_pcie_level_enabled*/
-		for (i = 2; i < dpm_table->sclk_table.count; i++) {
-			data->smc_state_table.GraphicsLevel[i].pcieDpmLevel = highest_pcie_level_enabled;
-		}
-
-		/* set pcieDpmLevel to lowest_pcie_level_enabled*/
-		data->smc_state_table.GraphicsLevel[0].pcieDpmLevel = lowest_pcie_level_enabled;
-
-		/* set pcieDpmLevel to mid_pcie_level_enabled*/
-		data->smc_state_table.GraphicsLevel[1].pcieDpmLevel = mid_pcie_level_enabled;
-	}
-	/* level count will send to smc once at init smc table and never change*/
-	result = tonga_copy_bytes_to_smc(hwmgr->smumgr, level_array_adress, (uint8_t *)levels, (uint32_t)level_array_size, data->sram_end);
-
-	if (0 != result)
-		return result;
-
-	return 0;
-}
-
-/**
- * Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states
- *
- * @param    hwmgr      the address of the hardware manager
- */
-
-static int tonga_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_dpm_table *dpm_table = &data->dpm_table;
-	int result;
-	/* populate MCLK dpm table to SMU7 */
-	uint32_t level_array_adress = data->dpm_table_start + offsetof(SMU72_Discrete_DpmTable, MemoryLevel);
-	uint32_t level_array_size = sizeof(SMU72_Discrete_MemoryLevel) * SMU72_MAX_LEVELS_MEMORY;
-	SMU72_Discrete_MemoryLevel *levels = data->smc_state_table.MemoryLevel;
-	uint32_t i;
-
-	memset(levels, 0x00, level_array_size);
-
-	for (i = 0; i < dpm_table->mclk_table.count; i++) {
-		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
-			"can not populate memory level as memory clock is zero", return -1);
-		result = tonga_populate_single_memory_level(hwmgr, dpm_table->mclk_table.dpm_levels[i].value,
-			&(data->smc_state_table.MemoryLevel[i]));
-		if (0 != result) {
-			return result;
-		}
-	}
-
-	/* Only enable level 0 for now.*/
-	data->smc_state_table.MemoryLevel[0].EnabledForActivity = 1;
-
-	/*
-	* in order to prevent MC activity from stutter mode to push DPM up.
-	* the UVD change complements this by putting the MCLK in a higher state
-	* by default such that we are not effected by up threshold or and MCLK DPM latency.
-	*/
-	data->smc_state_table.MemoryLevel[0].ActivityLevel = 0x1F;
-	CONVERT_FROM_HOST_TO_SMC_US(data->smc_state_table.MemoryLevel[0].ActivityLevel);
-
-	data->smc_state_table.MemoryDpmLevelCount = (uint8_t)dpm_table->mclk_table.count;
-	data->dpm_level_enable_mask.mclk_dpm_enable_mask = tonga_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
-	/* set highest level watermark to high*/
-	data->smc_state_table.MemoryLevel[dpm_table->mclk_table.count-1].DisplayWatermark = PPSMC_DISPLAY_WATERMARK_HIGH;
-
-	/* level count will send to smc once at init smc table and never change*/
-	result = tonga_copy_bytes_to_smc(hwmgr->smumgr,
-		level_array_adress, (uint8_t *)levels, (uint32_t)level_array_size, data->sram_end);
-
-	if (0 != result) {
-		return result;
-	}
-
-	return 0;
-}
-
-struct TONGA_DLL_SPEED_SETTING {
-	uint16_t            Min;                          /* Minimum Data Rate*/
-	uint16_t            Max;                          /* Maximum Data Rate*/
-	uint32_t			dll_speed;                     /* The desired DLL_SPEED setting*/
-};
-
-static int tonga_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
-{
-	return 0;
-}
-
-/* ---------------------------------------- ULV related functions ----------------------------------------------------*/
-
-
-static int tonga_reset_single_dpm_table(
-	struct pp_hwmgr *hwmgr,
-	struct tonga_single_dpm_table *dpm_table,
-	uint32_t count)
-{
-	uint32_t i;
-	if (!(count <= MAX_REGULAR_DPM_NUMBER))
-		printk(KERN_ERR "[ powerplay ] Fatal error, can not set up single DPM \
-			table entries to exceed max number! \n");
-
-	dpm_table->count = count;
-	for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++) {
-		dpm_table->dpm_levels[i].enabled = false;
-	}
-
-	return 0;
-}
-
-static void tonga_setup_pcie_table_entry(
-	struct tonga_single_dpm_table *dpm_table,
-	uint32_t index, uint32_t pcie_gen,
-	uint32_t pcie_lanes)
-{
-	dpm_table->dpm_levels[index].value = pcie_gen;
-	dpm_table->dpm_levels[index].param1 = pcie_lanes;
-	dpm_table->dpm_levels[index].enabled = true;
-}
-
-static int tonga_setup_default_pcie_tables(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	phm_ppt_v1_pcie_table *pcie_table = pptable_info->pcie_table;
-	uint32_t i, maxEntry;
-
-	if (data->use_pcie_performance_levels && !data->use_pcie_power_saving_levels) {
-		data->pcie_gen_power_saving = data->pcie_gen_performance;
-		data->pcie_lane_power_saving = data->pcie_lane_performance;
-	} else if (!data->use_pcie_performance_levels && data->use_pcie_power_saving_levels) {
-		data->pcie_gen_performance = data->pcie_gen_power_saving;
-		data->pcie_lane_performance = data->pcie_lane_power_saving;
-	}
-
-	tonga_reset_single_dpm_table(hwmgr, &data->dpm_table.pcie_speed_table, SMU72_MAX_LEVELS_LINK);
-
-	if (pcie_table != NULL) {
-		/*
-		* maxEntry is used to make sure we reserve one PCIE level for boot level (fix for A+A PSPP issue).
-		* If PCIE table from PPTable have ULV entry + 8 entries, then ignore the last entry.
-		*/
-		maxEntry = (SMU72_MAX_LEVELS_LINK < pcie_table->count) ?
-						SMU72_MAX_LEVELS_LINK : pcie_table->count;
-		for (i = 1; i < maxEntry; i++) {
-			tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, i-1,
-				get_pcie_gen_support(data->pcie_gen_cap, pcie_table->entries[i].gen_speed),
-				get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
-		}
-		data->dpm_table.pcie_speed_table.count = maxEntry - 1;
-	} else {
-		/* Hardcode Pcie Table */
-		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 0,
-			get_pcie_gen_support(data->pcie_gen_cap, PP_Min_PCIEGen),
-			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
-		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 1,
-			get_pcie_gen_support(data->pcie_gen_cap, PP_Min_PCIEGen),
-			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
-		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 2,
-			get_pcie_gen_support(data->pcie_gen_cap, PP_Max_PCIEGen),
-			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
-		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 3,
-			get_pcie_gen_support(data->pcie_gen_cap, PP_Max_PCIEGen),
-			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
-		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 4,
-			get_pcie_gen_support(data->pcie_gen_cap, PP_Max_PCIEGen),
-			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
-		tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table, 5,
-			get_pcie_gen_support(data->pcie_gen_cap, PP_Max_PCIEGen),
-			get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
-		data->dpm_table.pcie_speed_table.count = 6;
-	}
-	/* Populate last level for boot PCIE level, but do not increment count. */
-	tonga_setup_pcie_table_entry(&data->dpm_table.pcie_speed_table,
-		data->dpm_table.pcie_speed_table.count,
-		get_pcie_gen_support(data->pcie_gen_cap, PP_Min_PCIEGen),
-		get_pcie_lane_support(data->pcie_lane_cap, PP_Max_PCIELane));
-
-	return 0;
-
-}
-
-/*
- * This function is to initalize all DPM state tables for SMU7 based on the dependency table.
- * Dynamic state patching function will then trim these state tables to the allowed range based
- * on the power policy or external client requests, such as UVD request, etc.
- */
-static int tonga_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint32_t i;
-
-	phm_ppt_v1_clock_voltage_dependency_table *allowed_vdd_sclk_table =
-		pptable_info->vdd_dep_on_sclk;
-	phm_ppt_v1_clock_voltage_dependency_table *allowed_vdd_mclk_table =
-		pptable_info->vdd_dep_on_mclk;
-
-	PP_ASSERT_WITH_CODE(allowed_vdd_sclk_table != NULL,
-		"SCLK dependency table is missing. This table is mandatory", return -1);
-	PP_ASSERT_WITH_CODE(allowed_vdd_sclk_table->count >= 1,
-		"SCLK dependency table has to have is missing. This table is mandatory", return -1);
-
-	PP_ASSERT_WITH_CODE(allowed_vdd_mclk_table != NULL,
-		"MCLK dependency table is missing. This table is mandatory", return -1);
-	PP_ASSERT_WITH_CODE(allowed_vdd_mclk_table->count >= 1,
-		"VMCLK dependency table has to have is missing. This table is mandatory", return -1);
-
-	/* clear the state table to reset everything to default */
-	memset(&(data->dpm_table), 0x00, sizeof(data->dpm_table));
-	tonga_reset_single_dpm_table(hwmgr, &data->dpm_table.sclk_table, SMU72_MAX_LEVELS_GRAPHICS);
-	tonga_reset_single_dpm_table(hwmgr, &data->dpm_table.mclk_table, SMU72_MAX_LEVELS_MEMORY);
-	/* tonga_reset_single_dpm_table(hwmgr, &tonga_hwmgr->dpm_table.VddcTable, SMU72_MAX_LEVELS_VDDC); */
-	/* tonga_reset_single_dpm_table(hwmgr, &tonga_hwmgr->dpm_table.vdd_gfx_table, SMU72_MAX_LEVELS_VDDGFX);*/
-	/* tonga_reset_single_dpm_table(hwmgr, &tonga_hwmgr->dpm_table.vdd_ci_table, SMU72_MAX_LEVELS_VDDCI);*/
-	/* tonga_reset_single_dpm_table(hwmgr, &tonga_hwmgr->dpm_table.mvdd_table, SMU72_MAX_LEVELS_MVDD);*/
-
-	PP_ASSERT_WITH_CODE(allowed_vdd_sclk_table != NULL,
-		"SCLK dependency table is missing. This table is mandatory", return -1);
-	/* Initialize Sclk DPM table based on allow Sclk values*/
-	data->dpm_table.sclk_table.count = 0;
-
-	for (i = 0; i < allowed_vdd_sclk_table->count; i++) {
-		if (i == 0 || data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count-1].value !=
-				allowed_vdd_sclk_table->entries[i].clk) {
-			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].value =
-				allowed_vdd_sclk_table->entries[i].clk;
-			data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled = true; /*(i==0) ? 1 : 0; to do */
-			data->dpm_table.sclk_table.count++;
-		}
-	}
-
-	PP_ASSERT_WITH_CODE(allowed_vdd_mclk_table != NULL,
-		"MCLK dependency table is missing. This table is mandatory", return -1);
-	/* Initialize Mclk DPM table based on allow Mclk values */
-	data->dpm_table.mclk_table.count = 0;
-	for (i = 0; i < allowed_vdd_mclk_table->count; i++) {
-		if (i == 0 || data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count-1].value !=
-			allowed_vdd_mclk_table->entries[i].clk) {
-			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].value =
-				allowed_vdd_mclk_table->entries[i].clk;
-			data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].enabled = true; /*(i==0) ? 1 : 0; */
-			data->dpm_table.mclk_table.count++;
-		}
-	}
-
-	/* setup PCIE gen speed levels*/
-	tonga_setup_default_pcie_tables(hwmgr);
-
-	/* save a copy of the default DPM table*/
-	memcpy(&(data->golden_dpm_table), &(data->dpm_table), sizeof(struct tonga_dpm_table));
-
-	return 0;
-}
-
-int tonga_populate_smc_initial_state(struct pp_hwmgr *hwmgr,
-		const struct tonga_power_state *bootState)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	uint8_t count, level;
-
-	count = (uint8_t) (pptable_info->vdd_dep_on_sclk->count);
-	for (level = 0; level < count; level++) {
-		if (pptable_info->vdd_dep_on_sclk->entries[level].clk >=
-			bootState->performance_levels[0].engine_clock) {
-			data->smc_state_table.GraphicsBootLevel = level;
-			break;
-		}
-	}
-
-	count = (uint8_t) (pptable_info->vdd_dep_on_mclk->count);
-	for (level = 0; level < count; level++) {
-		if (pptable_info->vdd_dep_on_mclk->entries[level].clk >=
-			bootState->performance_levels[0].memory_clock) {
-			data->smc_state_table.MemoryBootLevel = level;
-			break;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * Initializes the SMC table and uploads it
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @param    pInput  the pointer to input data (PowerState)
- * @return   always 0
- */
-int tonga_init_smc_table(struct pp_hwmgr *hwmgr)
-{
-	int result;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	SMU72_Discrete_DpmTable  *table = &(data->smc_state_table);
-	const phw_tonga_ulv_parm *ulv = &(data->ulv);
-	uint8_t i;
-	PECI_RegistryValue reg_value;
-	pp_atomctrl_gpio_pin_assignment gpio_pin_assignment;
-
-	result = tonga_setup_default_dpm_tables(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to setup default DPM tables!", return result;);
-	memset(&(data->smc_state_table), 0x00, sizeof(data->smc_state_table));
-	if (TONGA_VOLTAGE_CONTROL_NONE != data->voltage_control) {
-		tonga_populate_smc_voltage_tables(hwmgr, table);
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_AutomaticDCTransition)) {
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_StepVddc)) {
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
-	}
-
-	if (data->is_memory_GDDR5) {
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
-	}
-
-	i = PHM_READ_FIELD(hwmgr->device, CC_MC_MAX_CHANNEL, NOOFCHAN);
-
-	if (i == 1 || i == 0) {
-		table->SystemFlags |= PPSMC_SYSTEMFLAG_12CHANNEL;
-	}
-
-	if (ulv->ulv_supported && pptable_info->us_ulv_voltage_offset) {
-		PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to initialize ULV state!", return result;);
-
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCG_ULV_PARAMETER, ulv->ch_ulv_parameter);
-	}
-
-	result = tonga_populate_smc_link_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize Link Level!", return result;);
-
-	result = tonga_populate_all_graphic_levels(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize Graphics Level!", return result;);
-
-	result = tonga_populate_all_memory_levels(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize Memory Level!", return result;);
-
-	result = tonga_populate_smv_acpi_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize ACPI Level!", return result;);
-
-	result = tonga_populate_smc_vce_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize VCE Level!", return result;);
-
-	result = tonga_populate_smc_acp_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize ACP Level!", return result;);
-
-	result = tonga_populate_smc_samu_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize SAMU Level!", return result;);
-
-	/* Since only the initial state is completely set up at this point (the other states are just copies of the boot state) we only */
-	/* need to populate the  ARB settings for the initial state. */
-	result = tonga_program_memory_timing_parameters(hwmgr);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to Write ARB settings for the initial state.", return result;);
-
-	result = tonga_populate_smc_uvd_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize UVD Level!", return result;);
-
-	result = tonga_populate_smc_boot_level(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize Boot Level!", return result;);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ClockStretcher)) {
-		result = tonga_populate_clock_stretcher_data_table(hwmgr);
-		PP_ASSERT_WITH_CODE(0 == result,
-			"Failed to populate Clock Stretcher Data Table!", return result;);
-	}
-	table->GraphicsVoltageChangeEnable  = 1;
-	table->GraphicsThermThrottleEnable  = 1;
-	table->GraphicsInterval = 1;
-	table->VoltageInterval  = 1;
-	table->ThermalInterval  = 1;
-	table->TemperatureLimitHigh =
-		pptable_info->cac_dtp_table->usTargetOperatingTemp *
-		TONGA_Q88_FORMAT_CONVERSION_UNIT;
-	table->TemperatureLimitLow =
-		(pptable_info->cac_dtp_table->usTargetOperatingTemp - 1) *
-		TONGA_Q88_FORMAT_CONVERSION_UNIT;
-	table->MemoryVoltageChangeEnable  = 1;
-	table->MemoryInterval  = 1;
-	table->VoltageResponseTime  = 0;
-	table->PhaseResponseTime  = 0;
-	table->MemoryThermThrottleEnable  = 1;
-
-	/*
-	* Cail reads current link status and reports it as cap (we cannot change this due to some previous issues we had)
-	* SMC drops the link status to lowest level after enabling DPM by PowerPlay. After pnp or toggling CF, driver gets reloaded again
-	* but this time Cail reads current link status which was set to low by SMC and reports it as cap to powerplay
-	* To avoid it, we set PCIeBootLinkLevel to highest dpm level
-	*/
-	PP_ASSERT_WITH_CODE((1 <= data->dpm_table.pcie_speed_table.count),
-			"There must be 1 or more PCIE levels defined in PPTable.",
-			return -1);
-
-	table->PCIeBootLinkLevel = (uint8_t) (data->dpm_table.pcie_speed_table.count);
-
-	table->PCIeGenInterval  = 1;
-
-	result = tonga_populate_vr_config(hwmgr, table);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to populate VRConfig setting!", return result);
-
-	table->ThermGpio  = 17;
-	table->SclkStepSize = 0x4000;
-
-	reg_value = 0;
-	if ((0 == reg_value) &&
-		(atomctrl_get_pp_assign_pin(hwmgr, VDDC_VRHOT_GPIO_PINID,
-						&gpio_pin_assignment))) {
-		table->VRHotGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_RegulatorHot);
-	} else {
-		table->VRHotGpio = TONGA_UNUSED_GPIO_PIN;
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_RegulatorHot);
-	}
-
-	/* ACDC Switch GPIO */
-	reg_value = 0;
-	if ((0 == reg_value) &&
-		(atomctrl_get_pp_assign_pin(hwmgr, PP_AC_DC_SWITCH_GPIO_PINID,
-						&gpio_pin_assignment))) {
-		table->AcDcGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_AutomaticDCTransition);
-	} else {
-		table->AcDcGpio = TONGA_UNUSED_GPIO_PIN;
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_AutomaticDCTransition);
-	}
-
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_Falcon_QuickTransition);
-
-	reg_value = 0;
-	if (1 == reg_value) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_AutomaticDCTransition);
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_Falcon_QuickTransition);
-	}
-
-	reg_value = 0;
-	if ((0 == reg_value) && (atomctrl_get_pp_assign_pin(hwmgr,
-			THERMAL_INT_OUTPUT_GPIO_PINID, &gpio_pin_assignment))) {
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ThermalOutGPIO);
-
-		table->ThermOutGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
-
-		table->ThermOutPolarity =
-			(0 == (cgs_read_register(hwmgr->device, mmGPIOPAD_A) &
-			(1 << gpio_pin_assignment.uc_gpio_pin_bit_shift))) ? 1:0;
-
-		table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_ONLY;
-
-		/* if required, combine VRHot/PCC with thermal out GPIO*/
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_RegulatorHot) &&
-			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_CombinePCCWithThermalSignal)){
-			table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_VRHOT;
-		}
-	} else {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ThermalOutGPIO);
-
-		table->ThermOutGpio = 17;
-		table->ThermOutPolarity = 1;
-		table->ThermOutMode = SMU7_THERM_OUT_MODE_DISABLE;
-	}
-
-	for (i = 0; i < SMU72_MAX_ENTRIES_SMIO; i++) {
-		table->Smio[i] = PP_HOST_TO_SMC_UL(table->Smio[i]);
-	}
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->VRConfig);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask1);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask2);
-	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
-	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
-	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
-	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
-	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
-
-	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
-	result = tonga_copy_bytes_to_smc(hwmgr->smumgr, data->dpm_table_start +
-										offsetof(SMU72_Discrete_DpmTable, SystemFlags),
-										(uint8_t *)&(table->SystemFlags),
-										sizeof(SMU72_Discrete_DpmTable)-3 * sizeof(SMU72_PIDController),
-										data->sram_end);
-
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to upload dpm data to SMC memory!", return result;);
-
-	return result;
-}
-
-/* Look up the voltaged based on DAL's requested level. and then send the requested VDDC voltage to SMC*/
-static void tonga_apply_dal_minimum_voltage_request(struct pp_hwmgr *hwmgr)
-{
-	return;
-}
-
-int tonga_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr)
-{
-	PPSMC_Result result;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	/* Apply minimum voltage based on DAL's request level */
-	tonga_apply_dal_minimum_voltage_request(hwmgr);
-
-	if (0 == data->sclk_dpm_key_disabled) {
-		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
-		if (tonga_is_dpm_running(hwmgr))
-			printk(KERN_ERR "[ powerplay ] Trying to set Enable Mask when DPM is disabled \n");
-
-		if (0 != data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
-			result = smum_send_msg_to_smc_with_parameter(
-								hwmgr->smumgr,
-				(PPSMC_Msg)PPSMC_MSG_SCLKDPM_SetEnabledMask,
-				data->dpm_level_enable_mask.sclk_dpm_enable_mask);
-			PP_ASSERT_WITH_CODE((0 == result),
-				"Set Sclk Dpm enable Mask failed", return -1);
-		}
-	}
-
-	if (0 == data->mclk_dpm_key_disabled) {
-		/* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
-		if (tonga_is_dpm_running(hwmgr))
-			printk(KERN_ERR "[ powerplay ] Trying to set Enable Mask when DPM is disabled \n");
-
-		if (0 != data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
-			result = smum_send_msg_to_smc_with_parameter(
-								hwmgr->smumgr,
-				(PPSMC_Msg)PPSMC_MSG_MCLKDPM_SetEnabledMask,
-				data->dpm_level_enable_mask.mclk_dpm_enable_mask);
-			PP_ASSERT_WITH_CODE((0 == result),
-				"Set Mclk Dpm enable Mask failed", return -1);
-		}
-	}
-
-	return 0;
-}
-
-
-int tonga_force_dpm_highest(struct pp_hwmgr *hwmgr)
-{
-	uint32_t level, tmp;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	if (0 == data->pcie_dpm_key_disabled) {
-		/* PCIE */
-		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask != 0) {
-			level = 0;
-			tmp = data->dpm_level_enable_mask.pcie_dpm_enable_mask;
-			while (tmp >>= 1)
-				level++ ;
-
-			if (0 != level) {
-				PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state_pcie(hwmgr, level)),
-					"force highest pcie dpm state failed!", return -1);
-			}
-		}
-	}
-
-	if (0 == data->sclk_dpm_key_disabled) {
-		/* SCLK */
-		if (data->dpm_level_enable_mask.sclk_dpm_enable_mask != 0) {
-			level = 0;
-			tmp = data->dpm_level_enable_mask.sclk_dpm_enable_mask;
-			while (tmp >>= 1)
-				level++ ;
-
-			if (0 != level) {
-				PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state(hwmgr, level)),
-					"force highest sclk dpm state failed!", return -1);
-				if (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,
-					CGS_IND_REG__SMC, TARGET_AND_CURRENT_PROFILE_INDEX, CURR_SCLK_INDEX) != level)
-					printk(KERN_ERR "[ powerplay ] Target_and_current_Profile_Index. \
-						Curr_Sclk_Index does not match the level \n");
-
-			}
-		}
-	}
-
-	if (0 == data->mclk_dpm_key_disabled) {
-		/* MCLK */
-		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask != 0) {
-			level = 0;
-			tmp = data->dpm_level_enable_mask.mclk_dpm_enable_mask;
-			while (tmp >>= 1)
-				level++ ;
-
-			if (0 != level) {
-				PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state_mclk(hwmgr, level)),
-					"force highest mclk dpm state failed!", return -1);
-				if (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-					TARGET_AND_CURRENT_PROFILE_INDEX, CURR_MCLK_INDEX) != level)
-					printk(KERN_ERR "[ powerplay ] Target_and_current_Profile_Index. \
-						Curr_Mclk_Index does not match the level \n");
-			}
-		}
-	}
-
-	return 0;
-}
-
-/**
- * Find the MC microcode version and store it in the HwMgr struct
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_get_mc_microcode_version (struct pp_hwmgr *hwmgr)
-{
-	cgs_write_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_INDEX, 0x9F);
-
-	hwmgr->microcode_version_info.MC = cgs_read_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_DATA);
-
-	return 0;
-}
-
-/**
- * Initialize Dynamic State Adjustment Rule Settings
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- */
-int tonga_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr)
-{
-	uint32_t table_size;
-	struct phm_clock_voltage_dependency_table *table_clk_vlt;
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	hwmgr->dyn_state.mclk_sclk_ratio = 4;
-	hwmgr->dyn_state.sclk_mclk_delta = 15000;      /* 150 MHz */
-	hwmgr->dyn_state.vddc_vddci_delta = 200;       /* 200mV */
-
-	/* initialize vddc_dep_on_dal_pwrl table */
-	table_size = sizeof(uint32_t) + 4 * sizeof(struct phm_clock_voltage_dependency_record);
-	table_clk_vlt = kzalloc(table_size, GFP_KERNEL);
-
-	if (NULL == table_clk_vlt) {
-		printk(KERN_ERR "[ powerplay ] Can not allocate space for vddc_dep_on_dal_pwrl! \n");
-		return -ENOMEM;
-	} else {
-		table_clk_vlt->count = 4;
-		table_clk_vlt->entries[0].clk = PP_DAL_POWERLEVEL_ULTRALOW;
-		table_clk_vlt->entries[0].v = 0;
-		table_clk_vlt->entries[1].clk = PP_DAL_POWERLEVEL_LOW;
-		table_clk_vlt->entries[1].v = 720;
-		table_clk_vlt->entries[2].clk = PP_DAL_POWERLEVEL_NOMINAL;
-		table_clk_vlt->entries[2].v = 810;
-		table_clk_vlt->entries[3].clk = PP_DAL_POWERLEVEL_PERFORMANCE;
-		table_clk_vlt->entries[3].v = 900;
-		pptable_info->vddc_dep_on_dal_pwrl = table_clk_vlt;
-		hwmgr->dyn_state.vddc_dep_on_dal_pwrl = table_clk_vlt;
-	}
-
-	return 0;
-}
-
-static int tonga_set_private_var_based_on_pptale(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	phm_ppt_v1_clock_voltage_dependency_table *allowed_sclk_vdd_table =
-		pptable_info->vdd_dep_on_sclk;
-	phm_ppt_v1_clock_voltage_dependency_table *allowed_mclk_vdd_table =
-		pptable_info->vdd_dep_on_mclk;
-
-	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table != NULL,
-		"VDD dependency on SCLK table is missing.	\
-		This table is mandatory", return -1);
-	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table->count >= 1,
-		"VDD dependency on SCLK table has to have is missing.	\
-		This table is mandatory", return -1);
-
-	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table != NULL,
-		"VDD dependency on MCLK table is missing.	\
-		This table is mandatory", return -1);
-	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table->count >= 1,
-		"VDD dependency on MCLK table has to have is missing.	 \
-		This table is mandatory", return -1);
-
-	data->min_vddc_in_pp_table = (uint16_t)allowed_sclk_vdd_table->entries[0].vddc;
-	data->max_vddc_in_pp_table = (uint16_t)allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].vddc;
-
-	pptable_info->max_clock_voltage_on_ac.sclk =
-		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].clk;
-	pptable_info->max_clock_voltage_on_ac.mclk =
-		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].clk;
-	pptable_info->max_clock_voltage_on_ac.vddc =
-		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].vddc;
-	pptable_info->max_clock_voltage_on_ac.vddci =
-		allowed_mclk_vdd_table->entries[allowed_mclk_vdd_table->count - 1].vddci;
-
-	hwmgr->dyn_state.max_clock_voltage_on_ac.sclk =
-		pptable_info->max_clock_voltage_on_ac.sclk;
-	hwmgr->dyn_state.max_clock_voltage_on_ac.mclk =
-		pptable_info->max_clock_voltage_on_ac.mclk;
-	hwmgr->dyn_state.max_clock_voltage_on_ac.vddc =
-		pptable_info->max_clock_voltage_on_ac.vddc;
-	hwmgr->dyn_state.max_clock_voltage_on_ac.vddci =
-		pptable_info->max_clock_voltage_on_ac.vddci;
-
-	return 0;
-}
-
-int tonga_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	int result = 1;
-
-	PP_ASSERT_WITH_CODE (!tonga_is_dpm_running(hwmgr),
-			     "Trying to Unforce DPM when DPM is disabled. Returning without sending SMC message.",
-			     return result);
-
-	if (0 == data->pcie_dpm_key_disabled) {
-		PP_ASSERT_WITH_CODE((0 == smum_send_msg_to_smc(
-							     hwmgr->smumgr,
-					PPSMC_MSG_PCIeDPM_UnForceLevel)),
-					   "unforce pcie level failed!",
-								return -1);
-	}
-
-	result = tonga_upload_dpm_level_enable_mask(hwmgr);
-
-	return result;
-}
-
-static uint32_t tonga_get_lowest_enable_level(
-				struct pp_hwmgr *hwmgr, uint32_t level_mask)
-{
-	uint32_t level = 0;
-
-	while (0 == (level_mask & (1 << level)))
-		level++;
-
-	return level;
-}
-
-static int tonga_force_dpm_lowest(struct pp_hwmgr *hwmgr)
-{
-	uint32_t level;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	if (0 == data->pcie_dpm_key_disabled) {
-		/* PCIE */
-		if (data->dpm_level_enable_mask.pcie_dpm_enable_mask != 0) {
-			level = tonga_get_lowest_enable_level(hwmgr,
-							      data->dpm_level_enable_mask.pcie_dpm_enable_mask);
-			PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state_pcie(hwmgr, level)),
-					    "force lowest pcie dpm state failed!", return -1);
-		}
-	}
-
-	if (0 == data->sclk_dpm_key_disabled) {
-		/* SCLK */
-		if (0 != data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
-			level = tonga_get_lowest_enable_level(hwmgr,
-							      data->dpm_level_enable_mask.sclk_dpm_enable_mask);
-
-			PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state(hwmgr, level)),
-					    "force sclk dpm state failed!", return -1);
-
-			if (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,
-							 CGS_IND_REG__SMC, TARGET_AND_CURRENT_PROFILE_INDEX, CURR_SCLK_INDEX) != level)
-				printk(KERN_ERR "[ powerplay ] Target_and_current_Profile_Index.	\
-				Curr_Sclk_Index does not match the level \n");
-		}
-	}
-
-	if (0 == data->mclk_dpm_key_disabled) {
-		/* MCLK */
-		if (data->dpm_level_enable_mask.mclk_dpm_enable_mask != 0) {
-			level = tonga_get_lowest_enable_level(hwmgr,
-							      data->dpm_level_enable_mask.mclk_dpm_enable_mask);
-			PP_ASSERT_WITH_CODE((0 == tonga_dpm_force_state_mclk(hwmgr, level)),
-					    "force lowest mclk dpm state failed!", return -1);
-			if (PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-							 TARGET_AND_CURRENT_PROFILE_INDEX, CURR_MCLK_INDEX) != level)
-				printk(KERN_ERR "[ powerplay ] Target_and_current_Profile_Index. \
-						Curr_Mclk_Index does not match the level \n");
-		}
-	}
-
-	return 0;
-}
-
-static int tonga_patch_voltage_dependency_tables_with_lookup_table(struct pp_hwmgr *hwmgr)
-{
-	uint8_t entryId;
-	uint8_t voltageId;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	phm_ppt_v1_clock_voltage_dependency_table *sclk_table = pptable_info->vdd_dep_on_sclk;
-	phm_ppt_v1_clock_voltage_dependency_table *mclk_table = pptable_info->vdd_dep_on_mclk;
-	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
-
-	if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
-		for (entryId = 0; entryId < sclk_table->count; ++entryId) {
-			voltageId = sclk_table->entries[entryId].vddInd;
-			sclk_table->entries[entryId].vddgfx =
-				pptable_info->vddgfx_lookup_table->entries[voltageId].us_vdd;
-		}
-	} else {
-		for (entryId = 0; entryId < sclk_table->count; ++entryId) {
-			voltageId = sclk_table->entries[entryId].vddInd;
-			sclk_table->entries[entryId].vddc =
-				pptable_info->vddc_lookup_table->entries[voltageId].us_vdd;
-		}
-	}
-
-	for (entryId = 0; entryId < mclk_table->count; ++entryId) {
-		voltageId = mclk_table->entries[entryId].vddInd;
-		mclk_table->entries[entryId].vddc =
-			pptable_info->vddc_lookup_table->entries[voltageId].us_vdd;
-	}
-
-	for (entryId = 0; entryId < mm_table->count; ++entryId) {
-		voltageId = mm_table->entries[entryId].vddcInd;
-		mm_table->entries[entryId].vddc =
-			pptable_info->vddc_lookup_table->entries[voltageId].us_vdd;
-	}
-
-	return 0;
-
-}
-
-static int tonga_calc_voltage_dependency_tables(struct pp_hwmgr *hwmgr)
-{
-	uint8_t entryId;
-	phm_ppt_v1_voltage_lookup_record v_record;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	phm_ppt_v1_clock_voltage_dependency_table *sclk_table = pptable_info->vdd_dep_on_sclk;
-	phm_ppt_v1_clock_voltage_dependency_table *mclk_table = pptable_info->vdd_dep_on_mclk;
-
-	if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
-		for (entryId = 0; entryId < sclk_table->count; ++entryId) {
-			if (sclk_table->entries[entryId].vdd_offset & (1 << 15))
-				v_record.us_vdd = sclk_table->entries[entryId].vddgfx +
-					sclk_table->entries[entryId].vdd_offset - 0xFFFF;
-			else
-				v_record.us_vdd = sclk_table->entries[entryId].vddgfx +
-					sclk_table->entries[entryId].vdd_offset;
-
-			sclk_table->entries[entryId].vddc =
-				v_record.us_cac_low = v_record.us_cac_mid =
-				v_record.us_cac_high = v_record.us_vdd;
-
-			tonga_add_voltage(hwmgr, pptable_info->vddc_lookup_table, &v_record);
-		}
-
-		for (entryId = 0; entryId < mclk_table->count; ++entryId) {
-			if (mclk_table->entries[entryId].vdd_offset & (1 << 15))
-				v_record.us_vdd = mclk_table->entries[entryId].vddc +
-					mclk_table->entries[entryId].vdd_offset - 0xFFFF;
-			else
-				v_record.us_vdd = mclk_table->entries[entryId].vddc +
-					mclk_table->entries[entryId].vdd_offset;
-
-			mclk_table->entries[entryId].vddgfx = v_record.us_cac_low =
-				v_record.us_cac_mid = v_record.us_cac_high = v_record.us_vdd;
-			tonga_add_voltage(hwmgr, pptable_info->vddgfx_lookup_table, &v_record);
-		}
-	}
-
-	return 0;
-
-}
-
-static int tonga_calc_mm_voltage_dependency_table(struct pp_hwmgr *hwmgr)
-{
-	uint32_t entryId;
-	phm_ppt_v1_voltage_lookup_record v_record;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table = pptable_info->mm_dep_table;
-
-	if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
-		for (entryId = 0; entryId < mm_table->count; entryId++) {
-			if (mm_table->entries[entryId].vddgfx_offset & (1 << 15))
-				v_record.us_vdd = mm_table->entries[entryId].vddc +
-					mm_table->entries[entryId].vddgfx_offset - 0xFFFF;
-			else
-				v_record.us_vdd = mm_table->entries[entryId].vddc +
-					mm_table->entries[entryId].vddgfx_offset;
-
-			/* Add the calculated VDDGFX to the VDDGFX lookup table */
-			mm_table->entries[entryId].vddgfx = v_record.us_cac_low =
-				v_record.us_cac_mid = v_record.us_cac_high = v_record.us_vdd;
-			tonga_add_voltage(hwmgr, pptable_info->vddgfx_lookup_table, &v_record);
-		}
-	}
-	return 0;
-}
-
-
-/**
- * Change virtual leakage voltage to actual value.
- *
- * @param     hwmgr  the address of the powerplay hardware manager.
- * @param     pointer to changing voltage
- * @param     pointer to leakage table
- */
-static void tonga_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr,
-		uint16_t *voltage, phw_tonga_leakage_voltage *pLeakageTable)
-{
-	uint32_t leakage_index;
-
-	/* search for leakage voltage ID 0xff01 ~ 0xff08 */
-	for (leakage_index = 0; leakage_index < pLeakageTable->count; leakage_index++) {
-		/* if this voltage matches a leakage voltage ID */
-		/* patch with actual leakage voltage */
-		if (pLeakageTable->leakage_id[leakage_index] == *voltage) {
-			*voltage = pLeakageTable->actual_voltage[leakage_index];
-			break;
-		}
-	}
-
-	if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
-		printk(KERN_ERR "[ powerplay ] Voltage value looks like a Leakage ID but it's not patched \n");
-}
-
-/**
- * Patch voltage lookup table by EVV leakages.
- *
- * @param     hwmgr  the address of the powerplay hardware manager.
- * @param     pointer to voltage lookup table
- * @param     pointer to leakage table
- * @return     always 0
- */
-static int tonga_patch_lookup_table_with_leakage(struct pp_hwmgr *hwmgr,
-		phm_ppt_v1_voltage_lookup_table *lookup_table,
-		phw_tonga_leakage_voltage *pLeakageTable)
-{
-	uint32_t i;
-
-	for (i = 0; i < lookup_table->count; i++) {
-		tonga_patch_with_vdd_leakage(hwmgr,
-			&lookup_table->entries[i].us_vdd, pLeakageTable);
-	}
-
-	return 0;
-}
-
-static int tonga_patch_clock_voltage_lomits_with_vddc_leakage(struct pp_hwmgr *hwmgr,
-		phw_tonga_leakage_voltage *pLeakageTable, uint16_t *Vddc)
-{
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	tonga_patch_with_vdd_leakage(hwmgr, (uint16_t *)Vddc, pLeakageTable);
-	hwmgr->dyn_state.max_clock_voltage_on_dc.vddc =
-		pptable_info->max_clock_voltage_on_dc.vddc;
-
-	return 0;
-}
-
-static int tonga_patch_clock_voltage_limits_with_vddgfx_leakage(
-		struct pp_hwmgr *hwmgr, phw_tonga_leakage_voltage *pLeakageTable,
-		uint16_t *Vddgfx)
-{
-	tonga_patch_with_vdd_leakage(hwmgr, (uint16_t *)Vddgfx, pLeakageTable);
-	return 0;
-}
-
-int tonga_sort_lookup_table(struct pp_hwmgr *hwmgr,
-		phm_ppt_v1_voltage_lookup_table *lookup_table)
-{
-	uint32_t table_size, i, j;
-	phm_ppt_v1_voltage_lookup_record tmp_voltage_lookup_record;
-	table_size = lookup_table->count;
-
-	PP_ASSERT_WITH_CODE(0 != lookup_table->count,
-		"Lookup table is empty", return -1);
-
-	/* Sorting voltages */
-	for (i = 0; i < table_size - 1; i++) {
-		for (j = i + 1; j > 0; j--) {
-			if (lookup_table->entries[j].us_vdd < lookup_table->entries[j-1].us_vdd) {
-				tmp_voltage_lookup_record = lookup_table->entries[j-1];
-				lookup_table->entries[j-1] = lookup_table->entries[j];
-				lookup_table->entries[j] = tmp_voltage_lookup_record;
-			}
-		}
-	}
-
-	return 0;
-}
-
-static int tonga_complete_dependency_tables(struct pp_hwmgr *hwmgr)
-{
-	int result = 0;
-	int tmp_result;
-	tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (data->vdd_gfx_control == TONGA_VOLTAGE_CONTROL_BY_SVID2) {
-		tmp_result = tonga_patch_lookup_table_with_leakage(hwmgr,
-			pptable_info->vddgfx_lookup_table, &(data->vddcgfx_leakage));
-		if (tmp_result != 0)
-			result = tmp_result;
-
-		tmp_result = tonga_patch_clock_voltage_limits_with_vddgfx_leakage(hwmgr,
-			&(data->vddcgfx_leakage), &pptable_info->max_clock_voltage_on_dc.vddgfx);
-		if (tmp_result != 0)
-			result = tmp_result;
-	} else {
-		tmp_result = tonga_patch_lookup_table_with_leakage(hwmgr,
-			pptable_info->vddc_lookup_table, &(data->vddc_leakage));
-		if (tmp_result != 0)
-			result = tmp_result;
-
-		tmp_result = tonga_patch_clock_voltage_lomits_with_vddc_leakage(hwmgr,
-			&(data->vddc_leakage), &pptable_info->max_clock_voltage_on_dc.vddc);
-		if (tmp_result != 0)
-			result = tmp_result;
-	}
-
-	tmp_result = tonga_patch_voltage_dependency_tables_with_lookup_table(hwmgr);
-	if (tmp_result != 0)
-		result = tmp_result;
-
-	tmp_result = tonga_calc_voltage_dependency_tables(hwmgr);
-	if (tmp_result != 0)
-		result = tmp_result;
-
-	tmp_result = tonga_calc_mm_voltage_dependency_table(hwmgr);
-	if (tmp_result != 0)
-		result = tmp_result;
-
-	tmp_result = tonga_sort_lookup_table(hwmgr, pptable_info->vddgfx_lookup_table);
-	if (tmp_result != 0)
-		result = tmp_result;
-
-	tmp_result = tonga_sort_lookup_table(hwmgr, pptable_info->vddc_lookup_table);
-	if (tmp_result != 0)
-		result = tmp_result;
-
-	return result;
-}
-
-int tonga_init_sclk_threshold(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	data->low_sclk_interrupt_threshold = 0;
-
-	return 0;
-}
-
-int tonga_setup_asic_task(struct pp_hwmgr *hwmgr)
-{
-	int tmp_result, result = 0;
-
-	tmp_result = tonga_read_clock_registers(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to read clock registers!", result = tmp_result);
-
-	tmp_result = tonga_get_memory_type(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to get memory type!", result = tmp_result);
-
-	tmp_result = tonga_enable_acpi_power_management(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to enable ACPI power management!", result = tmp_result);
-
-	tmp_result = tonga_init_power_gate_state(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to init power gate state!", result = tmp_result);
-
-	tmp_result = tonga_get_mc_microcode_version(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to get MC microcode version!", result = tmp_result);
-
-	tmp_result = tonga_init_sclk_threshold(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to init sclk threshold!", result = tmp_result);
-
-	return result;
-}
-
-/**
- * Enable voltage control
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_enable_voltage_control(struct pp_hwmgr *hwmgr)
-{
-	/* enable voltage control */
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, VOLT_PWRMGT_EN, 1);
-
-	return 0;
-}
-
-/**
- * Checks if we want to support voltage control
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- */
-bool cf_tonga_voltage_control(const struct pp_hwmgr *hwmgr)
-{
-	const struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	return(TONGA_VOLTAGE_CONTROL_NONE != data->voltage_control);
-}
-
-/*---------------------------MC----------------------------*/
-
-uint8_t tonga_get_memory_modile_index(struct pp_hwmgr *hwmgr)
-{
-	return (uint8_t) (0xFF & (cgs_read_register(hwmgr->device, mmBIOS_SCRATCH_4) >> 16));
-}
-
-bool tonga_check_s0_mc_reg_index(uint16_t inReg, uint16_t *outReg)
-{
-	bool result = true;
-
-	switch (inReg) {
-	case  mmMC_SEQ_RAS_TIMING:
-		*outReg = mmMC_SEQ_RAS_TIMING_LP;
-		break;
-
-	case  mmMC_SEQ_DLL_STBY:
-		*outReg = mmMC_SEQ_DLL_STBY_LP;
-		break;
-
-	case  mmMC_SEQ_G5PDX_CMD0:
-		*outReg = mmMC_SEQ_G5PDX_CMD0_LP;
-		break;
-
-	case  mmMC_SEQ_G5PDX_CMD1:
-		*outReg = mmMC_SEQ_G5PDX_CMD1_LP;
-		break;
-
-	case  mmMC_SEQ_G5PDX_CTRL:
-		*outReg = mmMC_SEQ_G5PDX_CTRL_LP;
-		break;
-
-	case mmMC_SEQ_CAS_TIMING:
-		*outReg = mmMC_SEQ_CAS_TIMING_LP;
-		break;
-
-	case mmMC_SEQ_MISC_TIMING:
-		*outReg = mmMC_SEQ_MISC_TIMING_LP;
-		break;
-
-	case mmMC_SEQ_MISC_TIMING2:
-		*outReg = mmMC_SEQ_MISC_TIMING2_LP;
-		break;
-
-	case mmMC_SEQ_PMG_DVS_CMD:
-		*outReg = mmMC_SEQ_PMG_DVS_CMD_LP;
-		break;
-
-	case mmMC_SEQ_PMG_DVS_CTL:
-		*outReg = mmMC_SEQ_PMG_DVS_CTL_LP;
-		break;
-
-	case mmMC_SEQ_RD_CTL_D0:
-		*outReg = mmMC_SEQ_RD_CTL_D0_LP;
-		break;
-
-	case mmMC_SEQ_RD_CTL_D1:
-		*outReg = mmMC_SEQ_RD_CTL_D1_LP;
-		break;
-
-	case mmMC_SEQ_WR_CTL_D0:
-		*outReg = mmMC_SEQ_WR_CTL_D0_LP;
-		break;
-
-	case mmMC_SEQ_WR_CTL_D1:
-		*outReg = mmMC_SEQ_WR_CTL_D1_LP;
-		break;
-
-	case mmMC_PMG_CMD_EMRS:
-		*outReg = mmMC_SEQ_PMG_CMD_EMRS_LP;
-		break;
-
-	case mmMC_PMG_CMD_MRS:
-		*outReg = mmMC_SEQ_PMG_CMD_MRS_LP;
-		break;
-
-	case mmMC_PMG_CMD_MRS1:
-		*outReg = mmMC_SEQ_PMG_CMD_MRS1_LP;
-		break;
-
-	case mmMC_SEQ_PMG_TIMING:
-		*outReg = mmMC_SEQ_PMG_TIMING_LP;
-		break;
-
-	case mmMC_PMG_CMD_MRS2:
-		*outReg = mmMC_SEQ_PMG_CMD_MRS2_LP;
-		break;
-
-	case mmMC_SEQ_WR_CTL_2:
-		*outReg = mmMC_SEQ_WR_CTL_2_LP;
-		break;
-
-	default:
-		result = false;
-		break;
-	}
-
-	return result;
-}
-
-int tonga_set_s0_mc_reg_index(phw_tonga_mc_reg_table *table)
-{
-	uint32_t i;
-	uint16_t address;
-
-	for (i = 0; i < table->last; i++) {
-		table->mc_reg_address[i].s0 =
-			tonga_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address)
-			? address : table->mc_reg_address[i].s1;
-	}
-	return 0;
-}
-
-int tonga_copy_vbios_smc_reg_table(const pp_atomctrl_mc_reg_table *table, phw_tonga_mc_reg_table *ni_table)
-{
-	uint8_t i, j;
-
-	PP_ASSERT_WITH_CODE((table->last <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-		"Invalid VramInfo table.", return -1);
-	PP_ASSERT_WITH_CODE((table->num_entries <= MAX_AC_TIMING_ENTRIES),
-		"Invalid VramInfo table.", return -1);
-
-	for (i = 0; i < table->last; i++) {
-		ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
-	}
-	ni_table->last = table->last;
-
-	for (i = 0; i < table->num_entries; i++) {
-		ni_table->mc_reg_table_entry[i].mclk_max =
-			table->mc_reg_table_entry[i].mclk_max;
-		for (j = 0; j < table->last; j++) {
-			ni_table->mc_reg_table_entry[i].mc_data[j] =
-				table->mc_reg_table_entry[i].mc_data[j];
-		}
-	}
-
-	ni_table->num_entries = table->num_entries;
-
-	return 0;
-}
-
-/**
- * VBIOS omits some information to reduce size, we need to recover them here.
- * 1.   when we see mmMC_SEQ_MISC1, bit[31:16] EMRS1, need to be write to  mmMC_PMG_CMD_EMRS /_LP[15:0].
- *      Bit[15:0] MRS, need to be update mmMC_PMG_CMD_MRS/_LP[15:0]
- * 2.   when we see mmMC_SEQ_RESERVE_M, bit[15:0] EMRS2, need to be write to mmMC_PMG_CMD_MRS1/_LP[15:0].
- * 3.   need to set these data for each clock range
- *
- * @param    hwmgr the address of the powerplay hardware manager.
- * @param    table the address of MCRegTable
- * @return   always 0
- */
-int tonga_set_mc_special_registers(struct pp_hwmgr *hwmgr, phw_tonga_mc_reg_table *table)
-{
-	uint8_t i, j, k;
-	uint32_t temp_reg;
-	const tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	for (i = 0, j = table->last; i < table->last; i++) {
-		PP_ASSERT_WITH_CODE((j < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-			"Invalid VramInfo table.", return -1);
-		switch (table->mc_reg_address[i].s1) {
-		/*
-		* mmMC_SEQ_MISC1, bit[31:16] EMRS1, need to be write to  mmMC_PMG_CMD_EMRS /_LP[15:0].
-		* Bit[15:0] MRS, need to be update mmMC_PMG_CMD_MRS/_LP[15:0]
-		*/
-		case mmMC_SEQ_MISC1:
-			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_EMRS);
-			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_EMRS;
-			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_EMRS_LP;
-			for (k = 0; k < table->num_entries; k++) {
-				table->mc_reg_table_entry[k].mc_data[j] =
-					((temp_reg & 0xffff0000)) |
-					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
-			}
-			j++;
-			PP_ASSERT_WITH_CODE((j < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-				"Invalid VramInfo table.", return -1);
-
-			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS);
-			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
-			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
-			for (k = 0; k < table->num_entries; k++) {
-				table->mc_reg_table_entry[k].mc_data[j] =
-					(temp_reg & 0xffff0000) |
-					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
-
-				if (!data->is_memory_GDDR5) {
-					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
-				}
-			}
-			j++;
-			PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-				"Invalid VramInfo table.", return -1);
-
-			if (!data->is_memory_GDDR5) {
-				table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
-				table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
-				for (k = 0; k < table->num_entries; k++) {
-					table->mc_reg_table_entry[k].mc_data[j] =
-						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
-				}
-				j++;
-				PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-					"Invalid VramInfo table.", return -1);
-			}
-
-			break;
-
-		case mmMC_SEQ_RESERVE_M:
-			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS1);
-			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS1;
-			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS1_LP;
-			for (k = 0; k < table->num_entries; k++) {
-				table->mc_reg_table_entry[k].mc_data[j] =
-					(temp_reg & 0xffff0000) |
-					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
-			}
-			j++;
-			PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-				"Invalid VramInfo table.", return -1);
-			break;
-
-		default:
-			break;
-		}
-
-	}
-
-	table->last = j;
-
-	return 0;
-}
-
-int tonga_set_valid_flag(phw_tonga_mc_reg_table *table)
-{
-	uint8_t i, j;
-	for (i = 0; i < table->last; i++) {
-		for (j = 1; j < table->num_entries; j++) {
-			if (table->mc_reg_table_entry[j-1].mc_data[i] !=
-				table->mc_reg_table_entry[j].mc_data[i]) {
-				table->validflag |= (1<<i);
-				break;
-			}
-		}
-	}
-
-	return 0;
-}
-
-int tonga_initialize_mc_reg_table(struct pp_hwmgr *hwmgr)
-{
-	int result;
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-	pp_atomctrl_mc_reg_table *table;
-	phw_tonga_mc_reg_table *ni_table = &data->tonga_mc_reg_table;
-	uint8_t module_index = tonga_get_memory_modile_index(hwmgr);
-
-	table = kzalloc(sizeof(pp_atomctrl_mc_reg_table), GFP_KERNEL);
-
-	if (NULL == table)
-		return -ENOMEM;
-
-	/* Program additional LP registers that are no longer programmed by VBIOS */
-	cgs_write_register(hwmgr->device, mmMC_SEQ_RAS_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_RAS_TIMING));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_CAS_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_CAS_TIMING));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_DLL_STBY_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_DLL_STBY));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD0_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD0));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD1_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD1));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CTRL_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CTRL));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CMD_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CMD));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CTL_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CTL));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_EMRS_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_EMRS));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS1_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS1));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D0_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D0));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_TIMING));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS2_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS2));
-	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_2_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_2));
-
-	memset(table, 0x00, sizeof(pp_atomctrl_mc_reg_table));
-
-	result = atomctrl_initialize_mc_reg_table(hwmgr, module_index, table);
-
-	if (0 == result)
-		result = tonga_copy_vbios_smc_reg_table(table, ni_table);
-
-	if (0 == result) {
-		tonga_set_s0_mc_reg_index(ni_table);
-		result = tonga_set_mc_special_registers(hwmgr, ni_table);
-	}
-
-	if (0 == result)
-		tonga_set_valid_flag(ni_table);
-
-	kfree(table);
-	return result;
-}
-
-/*
-* Copy one arb setting to another and then switch the active set.
-* arbFreqSrc and arbFreqDest is one of the MC_CG_ARB_FREQ_Fx constants.
-*/
-int tonga_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr,
-		uint32_t arbFreqSrc, uint32_t arbFreqDest)
-{
-	uint32_t mc_arb_dram_timing;
-	uint32_t mc_arb_dram_timing2;
-	uint32_t burst_time;
-	uint32_t mc_cg_config;
-
-	switch (arbFreqSrc) {
-	case MC_CG_ARB_FREQ_F0:
-		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
-		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
-		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
-		break;
-
-	case MC_CG_ARB_FREQ_F1:
-		mc_arb_dram_timing  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1);
-		mc_arb_dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1);
-		burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1);
-		break;
-
-	default:
-		return -1;
-	}
-
-	switch (arbFreqDest) {
-	case MC_CG_ARB_FREQ_F0:
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING, mc_arb_dram_timing);
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
-		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0, burst_time);
-		break;
-
-	case MC_CG_ARB_FREQ_F1:
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
-		cgs_write_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
-		PHM_WRITE_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE1, burst_time);
-		break;
-
-	default:
-		return -1;
-	}
-
-	mc_cg_config = cgs_read_register(hwmgr->device, mmMC_CG_CONFIG);
-	mc_cg_config |= 0x0000000F;
-	cgs_write_register(hwmgr->device, mmMC_CG_CONFIG, mc_cg_config);
-	PHM_WRITE_FIELD(hwmgr->device, MC_ARB_CG, CG_ARB_REQ, arbFreqDest);
-
-	return 0;
-}
-
-/**
- * Initial switch from ARB F0->F1
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- * This function is to be called from the SetPowerState table.
- */
-int tonga_initial_switch_from_arb_f0_to_f1(struct pp_hwmgr *hwmgr)
-{
-	return tonga_copy_and_switch_arb_sets(hwmgr, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
-}
-
-/**
- * Initialize the ARB DRAM timing table's index field.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_init_arb_table_index(struct pp_hwmgr *hwmgr)
-{
-	const tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	uint32_t tmp;
-	int result;
-
-	/*
-	* This is a read-modify-write on the first byte of the ARB table.
-	* The first byte in the SMU72_Discrete_MCArbDramTimingTable structure is the field 'current'.
-	* This solution is ugly, but we never write the whole table only individual fields in it.
-	* In reality this field should not be in that structure but in a soft register.
-	*/
-	result = tonga_read_smc_sram_dword(hwmgr->smumgr,
-				data->arb_table_start, &tmp, data->sram_end);
-
-	if (0 != result)
-		return result;
-
-	tmp &= 0x00FFFFFF;
-	tmp |= ((uint32_t)MC_CG_ARB_FREQ_F1) << 24;
-
-	return tonga_write_smc_sram_dword(hwmgr->smumgr,
-			data->arb_table_start,  tmp, data->sram_end);
-}
-
-int tonga_populate_mc_reg_address(struct pp_hwmgr *hwmgr, SMU72_Discrete_MCRegisters *mc_reg_table)
-{
-	const struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	uint32_t i, j;
-
-	for (i = 0, j = 0; j < data->tonga_mc_reg_table.last; j++) {
-		if (data->tonga_mc_reg_table.validflag & 1<<j) {
-			PP_ASSERT_WITH_CODE(i < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE,
-				"Index of mc_reg_table->address[] array out of boundary", return -1);
-			mc_reg_table->address[i].s0 =
-				PP_HOST_TO_SMC_US(data->tonga_mc_reg_table.mc_reg_address[j].s0);
-			mc_reg_table->address[i].s1 =
-				PP_HOST_TO_SMC_US(data->tonga_mc_reg_table.mc_reg_address[j].s1);
-			i++;
-		}
-	}
-
-	mc_reg_table->last = (uint8_t)i;
-
-	return 0;
-}
-
-/*convert register values from driver to SMC format */
-void tonga_convert_mc_registers(
-	const phw_tonga_mc_reg_entry * pEntry,
-	SMU72_Discrete_MCRegisterSet *pData,
-	uint32_t numEntries, uint32_t validflag)
-{
-	uint32_t i, j;
-
-	for (i = 0, j = 0; j < numEntries; j++) {
-		if (validflag & 1<<j) {
-			pData->value[i] = PP_HOST_TO_SMC_UL(pEntry->mc_data[j]);
-			i++;
-		}
-	}
-}
-
-/* find the entry in the memory range table, then populate the value to SMC's tonga_mc_reg_table */
-int tonga_convert_mc_reg_table_entry_to_smc(
-		struct pp_hwmgr *hwmgr,
-		const uint32_t memory_clock,
-		SMU72_Discrete_MCRegisterSet *mc_reg_table_data
-		)
-{
-	const tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	uint32_t i = 0;
-
-	for (i = 0; i < data->tonga_mc_reg_table.num_entries; i++) {
-		if (memory_clock <=
-			data->tonga_mc_reg_table.mc_reg_table_entry[i].mclk_max) {
-			break;
-		}
-	}
-
-	if ((i == data->tonga_mc_reg_table.num_entries) && (i > 0))
-		--i;
-
-	tonga_convert_mc_registers(&data->tonga_mc_reg_table.mc_reg_table_entry[i],
-		mc_reg_table_data, data->tonga_mc_reg_table.last, data->tonga_mc_reg_table.validflag);
-
-	return 0;
-}
-
-int tonga_convert_mc_reg_table_to_smc(struct pp_hwmgr *hwmgr,
-		SMU72_Discrete_MCRegisters *mc_reg_table)
-{
-	int result = 0;
-	tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	int res;
-	uint32_t i;
-
-	for (i = 0; i < data->dpm_table.mclk_table.count; i++) {
-		res = tonga_convert_mc_reg_table_entry_to_smc(
-				hwmgr,
-				data->dpm_table.mclk_table.dpm_levels[i].value,
-				&mc_reg_table->data[i]
-				);
-
-		if (0 != res)
-			result = res;
-	}
-
-	return result;
-}
-
-int tonga_populate_initial_mc_reg_table(struct pp_hwmgr *hwmgr)
-{
-	int result;
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	memset(&data->mc_reg_table, 0x00, sizeof(SMU72_Discrete_MCRegisters));
-	result = tonga_populate_mc_reg_address(hwmgr, &(data->mc_reg_table));
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize MCRegTable for the MC register addresses!", return result;);
-
-	result = tonga_convert_mc_reg_table_to_smc(hwmgr, &data->mc_reg_table);
-	PP_ASSERT_WITH_CODE(0 == result,
-		"Failed to initialize MCRegTable for driver state!", return result;);
-
-	return tonga_copy_bytes_to_smc(hwmgr->smumgr, data->mc_reg_table_start,
-			(uint8_t *)&data->mc_reg_table, sizeof(SMU72_Discrete_MCRegisters), data->sram_end);
-}
-
-/**
- * Programs static screed detection parameters
- *
- * @param   hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_program_static_screen_threshold_parameters(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	/* Set static screen threshold unit*/
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device,
-		CGS_IND_REG__SMC, CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD_UNIT,
-		data->static_screen_threshold_unit);
-	/* Set static screen threshold*/
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device,
-		CGS_IND_REG__SMC, CG_STATIC_SCREEN_PARAMETER, STATIC_SCREEN_THRESHOLD,
-		data->static_screen_threshold);
-
-	return 0;
-}
-
-/**
- * Setup display gap for glitch free memory clock switching.
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_enable_display_gap(struct pp_hwmgr *hwmgr)
-{
-	uint32_t display_gap = cgs_read_ind_register(hwmgr->device,
-							CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL);
-
-	display_gap = PHM_SET_FIELD(display_gap,
-					CG_DISPLAY_GAP_CNTL, DISP_GAP, DISPLAY_GAP_IGNORE);
-
-	display_gap = PHM_SET_FIELD(display_gap,
-					CG_DISPLAY_GAP_CNTL, DISP_GAP_MCHG, DISPLAY_GAP_VBLANK);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-		ixCG_DISPLAY_GAP_CNTL, display_gap);
-
-	return 0;
-}
-
-/**
- * Programs activity state transition voting clients
- *
- * @param    hwmgr  the address of the powerplay hardware manager.
- * @return   always 0
- */
-int tonga_program_voting_clients(struct pp_hwmgr *hwmgr)
-{
-	tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
-
-	/* Clear reset for voting clients before enabling DPM */
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-		SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 0);
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-		SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 0);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-		ixCG_FREQ_TRAN_VOTING_0, data->voting_rights_clients0);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-		ixCG_FREQ_TRAN_VOTING_1, data->voting_rights_clients1);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-		ixCG_FREQ_TRAN_VOTING_2, data->voting_rights_clients2);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-		ixCG_FREQ_TRAN_VOTING_3, data->voting_rights_clients3);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-		ixCG_FREQ_TRAN_VOTING_4, data->voting_rights_clients4);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-		ixCG_FREQ_TRAN_VOTING_5, data->voting_rights_clients5);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-		ixCG_FREQ_TRAN_VOTING_6, data->voting_rights_clients6);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-		ixCG_FREQ_TRAN_VOTING_7, data->voting_rights_clients7);
-
-	return 0;
-}
-
-
-int tonga_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
-{
-	int tmp_result, result = 0;
-
-	tmp_result = tonga_check_for_dpm_stopped(hwmgr);
-
-	if (cf_tonga_voltage_control(hwmgr)) {
-		tmp_result = tonga_enable_voltage_control(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to enable voltage control!", result = tmp_result);
-
-		tmp_result = tonga_construct_voltage_tables(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == tmp_result),
-			"Failed to contruct voltage tables!", result = tmp_result);
-	}
-
-	tmp_result = tonga_initialize_mc_reg_table(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to initialize MC reg table!", result = tmp_result);
-
-	tmp_result = tonga_program_static_screen_threshold_parameters(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to program static screen threshold parameters!", result = tmp_result);
-
-	tmp_result = tonga_enable_display_gap(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to enable display gap!", result = tmp_result);
-
-	tmp_result = tonga_program_voting_clients(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to program voting clients!", result = tmp_result);
-
-	tmp_result = tonga_process_firmware_header(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to process firmware header!", result = tmp_result);
-
-	tmp_result = tonga_initial_switch_from_arb_f0_to_f1(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to initialize switch from ArbF0 to F1!", result = tmp_result);
-
-	tmp_result = tonga_init_smc_table(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to initialize SMC table!", result = tmp_result);
-
-	tmp_result = tonga_init_arb_table_index(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to initialize ARB table index!", result = tmp_result);
-
-	tmp_result = tonga_populate_initial_mc_reg_table(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to populate initialize MC Reg table!", result = tmp_result);
-
-	tmp_result = tonga_notify_smc_display_change(hwmgr, false);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to notify no display!", result = tmp_result);
-
-	/* enable SCLK control */
-	tmp_result = tonga_enable_sclk_control(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to enable SCLK control!", result = tmp_result);
-
-	/* enable DPM */
-	tmp_result = tonga_start_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to start DPM!", result = tmp_result);
-
-	return result;
-}
-
-int tonga_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
-{
-	int tmp_result, result = 0;
-
-	tmp_result = tonga_check_for_dpm_running(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"SMC is still running!", return 0);
-
-	tmp_result = tonga_stop_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to stop DPM!", result = tmp_result);
-
-	tmp_result = tonga_reset_to_default(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result),
-		"Failed to reset to default!", result = tmp_result);
-
-	return result;
-}
-
-int tonga_reset_asic_tasks(struct pp_hwmgr *hwmgr)
-{
-	int result;
-
-	result = tonga_set_boot_state(hwmgr);
-	if (0 != result)
-		printk(KERN_ERR "[ powerplay ] Failed to reset asic via set boot state! \n");
-
-	return result;
-}
-
-int tonga_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
-{
-	return phm_hwmgr_backend_fini(hwmgr);
-}
-
-/**
- * Initializes the Volcanic Islands Hardware Manager
- *
- * @param   hwmgr the address of the powerplay hardware manager.
- * @return   1 if success; otherwise appropriate error code.
- */
-int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
-{
-	int result = 0;
-	SMU72_Discrete_DpmTable  *table = NULL;
-	tonga_hwmgr *data;
-	pp_atomctrl_gpio_pin_assignment gpio_pin_assignment;
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	phw_tonga_ulv_parm *ulv;
-	struct cgs_system_info sys_info = {0};
-
-	PP_ASSERT_WITH_CODE((NULL != hwmgr),
-		"Invalid Parameter!", return -1;);
-
-	data = kzalloc(sizeof(struct tonga_hwmgr), GFP_KERNEL);
-	if (data == NULL)
-		return -ENOMEM;
-
-	hwmgr->backend = data;
-
-	data->dll_defaule_on = false;
-	data->sram_end = SMC_RAM_END;
-
-	data->activity_target[0] = PPTONGA_TARGETACTIVITY_DFLT;
-	data->activity_target[1] = PPTONGA_TARGETACTIVITY_DFLT;
-	data->activity_target[2] = PPTONGA_TARGETACTIVITY_DFLT;
-	data->activity_target[3] = PPTONGA_TARGETACTIVITY_DFLT;
-	data->activity_target[4] = PPTONGA_TARGETACTIVITY_DFLT;
-	data->activity_target[5] = PPTONGA_TARGETACTIVITY_DFLT;
-	data->activity_target[6] = PPTONGA_TARGETACTIVITY_DFLT;
-	data->activity_target[7] = PPTONGA_TARGETACTIVITY_DFLT;
-
-	data->vddc_vddci_delta = VDDC_VDDCI_DELTA;
-	data->vddc_vddgfx_delta = VDDC_VDDGFX_DELTA;
-	data->mclk_activity_target = PPTONGA_MCLK_TARGETACTIVITY_DFLT;
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_DisableVoltageIsland);
-
-	data->sclk_dpm_key_disabled = 0;
-	data->mclk_dpm_key_disabled = 0;
-	data->pcie_dpm_key_disabled = 0;
-	data->pcc_monitor_enabled = 0;
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_UnTabledHardwareInterface);
-
-	data->gpio_debug = 0;
-	data->engine_clock_data = 0;
-	data->memory_clock_data = 0;
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_DynamicPatchPowerState);
-
-	/* need to set voltage control types before EVV patching*/
-	data->voltage_control = TONGA_VOLTAGE_CONTROL_NONE;
-	data->vdd_ci_control = TONGA_VOLTAGE_CONTROL_NONE;
-	data->vdd_gfx_control = TONGA_VOLTAGE_CONTROL_NONE;
-	data->mvdd_control = TONGA_VOLTAGE_CONTROL_NONE;
-	data->force_pcie_gen = PP_PCIEGenInvalid;
-
-	if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-				VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2)) {
-		data->voltage_control = TONGA_VOLTAGE_CONTROL_BY_SVID2;
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ControlVDDGFX)) {
-		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-			VOLTAGE_TYPE_VDDGFX, VOLTAGE_OBJ_SVID2)) {
-			data->vdd_gfx_control = TONGA_VOLTAGE_CONTROL_BY_SVID2;
-		}
-	}
-
-	if (TONGA_VOLTAGE_CONTROL_NONE == data->vdd_gfx_control) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ControlVDDGFX);
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_EnableMVDDControl)) {
-		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-					VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT)) {
-			data->mvdd_control = TONGA_VOLTAGE_CONTROL_BY_GPIO;
-		}
-	}
-
-	if (TONGA_VOLTAGE_CONTROL_NONE == data->mvdd_control) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_EnableMVDDControl);
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ControlVDDCI)) {
-		if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-					VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT))
-			data->vdd_ci_control = TONGA_VOLTAGE_CONTROL_BY_GPIO;
-		else if (atomctrl_is_voltage_controled_by_gpio_v3(hwmgr,
-						VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2))
-			data->vdd_ci_control = TONGA_VOLTAGE_CONTROL_BY_SVID2;
-	}
-
-	if (TONGA_VOLTAGE_CONTROL_NONE == data->vdd_ci_control)
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_ControlVDDCI);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_TablelessHardwareInterface);
-
-	if (pptable_info->cac_dtp_table->usClockStretchAmount != 0)
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_ClockStretcher);
-
-	/* Initializes DPM default values*/
-	tonga_initialize_dpm_defaults(hwmgr);
-
-	/* Get leakage voltage based on leakage ID.*/
-	PP_ASSERT_WITH_CODE((0 == tonga_get_evv_voltage(hwmgr)),
-		"Get EVV Voltage Failed.  Abort Driver loading!", return -1);
-
-	tonga_complete_dependency_tables(hwmgr);
-
-	/* Parse pptable data read from VBIOS*/
-	tonga_set_private_var_based_on_pptale(hwmgr);
-
-	/* ULV Support*/
-	ulv = &(data->ulv);
-	ulv->ulv_supported = false;
-
-	/* Initalize Dynamic State Adjustment Rule Settings*/
-	result = tonga_initializa_dynamic_state_adjustment_rule_settings(hwmgr);
-	if (result)
-		printk(KERN_ERR "[ powerplay ] tonga_initializa_dynamic_state_adjustment_rule_settings failed!\n");
-	data->uvd_enabled = false;
-
-	table = &(data->smc_state_table);
-
-	/*
-	* if ucGPIO_ID=VDDC_PCC_GPIO_PINID in GPIO_LUTable,
-	* Peak Current Control feature is enabled and we should program PCC HW register
-	*/
-	if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_PCC_GPIO_PINID, &gpio_pin_assignment)) {
-		uint32_t temp_reg = cgs_read_ind_register(hwmgr->device,
-										CGS_IND_REG__SMC, ixCNB_PWRMGT_CNTL);
-
-		switch (gpio_pin_assignment.uc_gpio_pin_bit_shift) {
-		case 0:
-			temp_reg = PHM_SET_FIELD(temp_reg,
-				CNB_PWRMGT_CNTL, GNB_SLOW_MODE, 0x1);
-			break;
-		case 1:
-			temp_reg = PHM_SET_FIELD(temp_reg,
-				CNB_PWRMGT_CNTL, GNB_SLOW_MODE, 0x2);
-			break;
-		case 2:
-			temp_reg = PHM_SET_FIELD(temp_reg,
-				CNB_PWRMGT_CNTL, GNB_SLOW, 0x1);
-			break;
-		case 3:
-			temp_reg = PHM_SET_FIELD(temp_reg,
-				CNB_PWRMGT_CNTL, FORCE_NB_PS1, 0x1);
-			break;
-		case 4:
-			temp_reg = PHM_SET_FIELD(temp_reg,
-				CNB_PWRMGT_CNTL, DPM_ENABLED, 0x1);
-			break;
-		default:
-			printk(KERN_ERR "[ powerplay ] Failed to setup PCC HW register!	\
-				Wrong GPIO assigned for VDDC_PCC_GPIO_PINID! \n");
-			break;
-		}
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
-			ixCNB_PWRMGT_CNTL, temp_reg);
-	}
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_EnableSMU7ThermalManagement);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-		PHM_PlatformCaps_SMU7);
-
-	data->vddc_phase_shed_control = false;
-
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-		      PHM_PlatformCaps_UVDPowerGating);
-	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
-		      PHM_PlatformCaps_VCEPowerGating);
-	sys_info.size = sizeof(struct cgs_system_info);
-	sys_info.info_id = CGS_SYSTEM_INFO_PG_FLAGS;
-	result = cgs_query_system_info(hwmgr->device, &sys_info);
-	if (!result) {
-		if (sys_info.value & AMD_PG_SUPPORT_UVD)
-			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				      PHM_PlatformCaps_UVDPowerGating);
-		if (sys_info.value & AMD_PG_SUPPORT_VCE)
-			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-				      PHM_PlatformCaps_VCEPowerGating);
-	}
-
-	if (0 == result) {
-		data->is_tlu_enabled = false;
-		hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =
-			TONGA_MAX_HARDWARE_POWERLEVELS;
-		hwmgr->platform_descriptor.hardwarePerformanceLevels = 2;
-		hwmgr->platform_descriptor.minimumClocksReductionPercentage  = 50;
-
-		sys_info.size = sizeof(struct cgs_system_info);
-		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO;
-		result = cgs_query_system_info(hwmgr->device, &sys_info);
-		if (result)
-			data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK;
-		else
-			data->pcie_gen_cap = (uint32_t)sys_info.value;
-		if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
-			data->pcie_spc_cap = 20;
-		sys_info.size = sizeof(struct cgs_system_info);
-		sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW;
-		result = cgs_query_system_info(hwmgr->device, &sys_info);
-		if (result)
-			data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK;
-		else
-			data->pcie_lane_cap = (uint32_t)sys_info.value;
-	} else {
-		/* Ignore return value in here, we are cleaning up a mess. */
-		tonga_hwmgr_backend_fini(hwmgr);
-	}
-
-	return result;
-}
-
-static int tonga_force_dpm_level(struct pp_hwmgr *hwmgr,
-		enum amd_dpm_forced_level level)
-{
-	int ret = 0;
-
-	switch (level) {
-	case AMD_DPM_FORCED_LEVEL_HIGH:
-		ret = tonga_force_dpm_highest(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	case AMD_DPM_FORCED_LEVEL_LOW:
-		ret = tonga_force_dpm_lowest(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	case AMD_DPM_FORCED_LEVEL_AUTO:
-		ret = tonga_unforce_dpm_levels(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	default:
-		break;
-	}
-
-	hwmgr->dpm_level = level;
-	return ret;
-}
-
-static int tonga_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
-				struct pp_power_state  *prequest_ps,
-			const struct pp_power_state *pcurrent_ps)
-{
-	struct tonga_power_state *tonga_ps =
-				cast_phw_tonga_power_state(&prequest_ps->hardware);
-
-	uint32_t sclk;
-	uint32_t mclk;
-	struct PP_Clocks minimum_clocks = {0};
-	bool disable_mclk_switching;
-	bool disable_mclk_switching_for_frame_lock;
-	struct cgs_display_info info = {0};
-	const struct phm_clock_and_voltage_limits *max_limits;
-	uint32_t i;
-	tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	int32_t count;
-	int32_t stable_pstate_sclk = 0, stable_pstate_mclk = 0;
-
-	data->battery_state = (PP_StateUILabel_Battery == prequest_ps->classification.ui_label);
-
-	PP_ASSERT_WITH_CODE(tonga_ps->performance_level_count == 2,
-				 "VI should always have 2 performance levels",
-				 );
-
-	max_limits = (PP_PowerSource_AC == hwmgr->power_source) ?
-			&(hwmgr->dyn_state.max_clock_voltage_on_ac) :
-			&(hwmgr->dyn_state.max_clock_voltage_on_dc);
-
-	if (PP_PowerSource_DC == hwmgr->power_source) {
-		for (i = 0; i < tonga_ps->performance_level_count; i++) {
-			if (tonga_ps->performance_levels[i].memory_clock > max_limits->mclk)
-				tonga_ps->performance_levels[i].memory_clock = max_limits->mclk;
-			if (tonga_ps->performance_levels[i].engine_clock > max_limits->sclk)
-				tonga_ps->performance_levels[i].engine_clock = max_limits->sclk;
-		}
-	}
-
-	tonga_ps->vce_clocks.EVCLK = hwmgr->vce_arbiter.evclk;
-	tonga_ps->vce_clocks.ECCLK = hwmgr->vce_arbiter.ecclk;
-
-	tonga_ps->acp_clk = hwmgr->acp_arbiter.acpclk;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	/*TO DO result = PHM_CheckVBlankTime(hwmgr, &vblankTooShort);*/
-
-	/* TO DO GetMinClockSettings(hwmgr->pPECI, &minimum_clocks); */
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState)) {
-
-		max_limits = &(hwmgr->dyn_state.max_clock_voltage_on_ac);
-		stable_pstate_sclk = (max_limits->sclk * 75) / 100;
-
-		for (count = pptable_info->vdd_dep_on_sclk->count-1; count >= 0; count--) {
-			if (stable_pstate_sclk >= pptable_info->vdd_dep_on_sclk->entries[count].clk) {
-				stable_pstate_sclk = pptable_info->vdd_dep_on_sclk->entries[count].clk;
-				break;
-			}
-		}
-
-		if (count < 0)
-			stable_pstate_sclk = pptable_info->vdd_dep_on_sclk->entries[0].clk;
-
-		stable_pstate_mclk = max_limits->mclk;
-
-		minimum_clocks.engineClock = stable_pstate_sclk;
-		minimum_clocks.memoryClock = stable_pstate_mclk;
-	}
-
-	if (minimum_clocks.engineClock < hwmgr->gfx_arbiter.sclk)
-		minimum_clocks.engineClock = hwmgr->gfx_arbiter.sclk;
-
-	if (minimum_clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
-		minimum_clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
-
-	tonga_ps->sclk_threshold = hwmgr->gfx_arbiter.sclk_threshold;
-
-	if (0 != hwmgr->gfx_arbiter.sclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.sclk_over_drive <= hwmgr->platform_descriptor.overdriveLimit.engineClock),
-					"Overdrive sclk exceeds limit",
-					hwmgr->gfx_arbiter.sclk_over_drive = hwmgr->platform_descriptor.overdriveLimit.engineClock);
-
-		if (hwmgr->gfx_arbiter.sclk_over_drive >= hwmgr->gfx_arbiter.sclk)
-			tonga_ps->performance_levels[1].engine_clock = hwmgr->gfx_arbiter.sclk_over_drive;
-	}
-
-	if (0 != hwmgr->gfx_arbiter.mclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.mclk_over_drive <= hwmgr->platform_descriptor.overdriveLimit.memoryClock),
-			"Overdrive mclk exceeds limit",
-			hwmgr->gfx_arbiter.mclk_over_drive = hwmgr->platform_descriptor.overdriveLimit.memoryClock);
-
-		if (hwmgr->gfx_arbiter.mclk_over_drive >= hwmgr->gfx_arbiter.mclk)
-			tonga_ps->performance_levels[1].memory_clock = hwmgr->gfx_arbiter.mclk_over_drive;
-	}
-
-	disable_mclk_switching_for_frame_lock = phm_cap_enabled(
-				    hwmgr->platform_descriptor.platformCaps,
-				    PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
-
-	disable_mclk_switching = (1 < info.display_count) ||
-				    disable_mclk_switching_for_frame_lock;
-
-	sclk  = tonga_ps->performance_levels[0].engine_clock;
-	mclk  = tonga_ps->performance_levels[0].memory_clock;
-
-	if (disable_mclk_switching)
-		mclk  = tonga_ps->performance_levels[tonga_ps->performance_level_count - 1].memory_clock;
-
-	if (sclk < minimum_clocks.engineClock)
-		sclk = (minimum_clocks.engineClock > max_limits->sclk) ? max_limits->sclk : minimum_clocks.engineClock;
-
-	if (mclk < minimum_clocks.memoryClock)
-		mclk = (minimum_clocks.memoryClock > max_limits->mclk) ? max_limits->mclk : minimum_clocks.memoryClock;
-
-	tonga_ps->performance_levels[0].engine_clock = sclk;
-	tonga_ps->performance_levels[0].memory_clock = mclk;
-
-	tonga_ps->performance_levels[1].engine_clock =
-		(tonga_ps->performance_levels[1].engine_clock >= tonga_ps->performance_levels[0].engine_clock) ?
-			      tonga_ps->performance_levels[1].engine_clock :
-			      tonga_ps->performance_levels[0].engine_clock;
-
-	if (disable_mclk_switching) {
-		if (mclk < tonga_ps->performance_levels[1].memory_clock)
-			mclk = tonga_ps->performance_levels[1].memory_clock;
-
-		tonga_ps->performance_levels[0].memory_clock = mclk;
-		tonga_ps->performance_levels[1].memory_clock = mclk;
-	} else {
-		if (tonga_ps->performance_levels[1].memory_clock < tonga_ps->performance_levels[0].memory_clock)
-			tonga_ps->performance_levels[1].memory_clock = tonga_ps->performance_levels[0].memory_clock;
-	}
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState)) {
-		for (i=0; i < tonga_ps->performance_level_count; i++) {
-			tonga_ps->performance_levels[i].engine_clock = stable_pstate_sclk;
-			tonga_ps->performance_levels[i].memory_clock = stable_pstate_mclk;
-			tonga_ps->performance_levels[i].pcie_gen = data->pcie_gen_performance.max;
-			tonga_ps->performance_levels[i].pcie_lane = data->pcie_gen_performance.max;
-		}
-	}
-
-	return 0;
-}
-
-int tonga_get_power_state_size(struct pp_hwmgr *hwmgr)
-{
-	return sizeof(struct tonga_power_state);
-}
-
-static int tonga_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
-{
-	struct pp_power_state  *ps;
-	struct tonga_power_state  *tonga_ps;
-
-	if (hwmgr == NULL)
-		return -EINVAL;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	tonga_ps = cast_phw_tonga_power_state(&ps->hardware);
-
-	if (low)
-		return tonga_ps->performance_levels[0].memory_clock;
-	else
-		return tonga_ps->performance_levels[tonga_ps->performance_level_count-1].memory_clock;
-}
-
-static int tonga_dpm_get_sclk(struct pp_hwmgr *hwmgr, bool low)
-{
-	struct pp_power_state  *ps;
-	struct tonga_power_state  *tonga_ps;
-
-	if (hwmgr == NULL)
-		return -EINVAL;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	tonga_ps = cast_phw_tonga_power_state(&ps->hardware);
-
-	if (low)
-		return tonga_ps->performance_levels[0].engine_clock;
-	else
-		return tonga_ps->performance_levels[tonga_ps->performance_level_count-1].engine_clock;
-}
-
-static uint16_t tonga_get_current_pcie_speed(
-						   struct pp_hwmgr *hwmgr)
-{
-	uint32_t speed_cntl = 0;
-
-	speed_cntl = cgs_read_ind_register(hwmgr->device,
-						   CGS_IND_REG__PCIE,
-						   ixPCIE_LC_SPEED_CNTL);
-	return((uint16_t)PHM_GET_FIELD(speed_cntl,
-			PCIE_LC_SPEED_CNTL, LC_CURRENT_DATA_RATE));
-}
-
-static int tonga_get_current_pcie_lane_number(
-						   struct pp_hwmgr *hwmgr)
-{
-	uint32_t link_width;
-
-	link_width = PHM_READ_INDIRECT_FIELD(hwmgr->device,
-							CGS_IND_REG__PCIE,
-						  PCIE_LC_LINK_WIDTH_CNTL,
-							LC_LINK_WIDTH_RD);
-
-	PP_ASSERT_WITH_CODE((7 >= link_width),
-			"Invalid PCIe lane width!", return 0);
-
-	return decode_pcie_lane_width(link_width);
-}
-
-static int tonga_dpm_patch_boot_state(struct pp_hwmgr *hwmgr,
-					struct pp_hw_power_state *hw_ps)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_power_state *ps = (struct tonga_power_state *)hw_ps;
-	ATOM_FIRMWARE_INFO_V2_2 *fw_info;
-	uint16_t size;
-	uint8_t frev, crev;
-	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
-
-	/* First retrieve the Boot clocks and VDDC from the firmware info table.
-	 * We assume here that fw_info is unchanged if this call fails.
-	 */
-	fw_info = (ATOM_FIRMWARE_INFO_V2_2 *)cgs_atom_get_data_table(
-			hwmgr->device, index,
-			&size, &frev, &crev);
-	if (!fw_info)
-		/* During a test, there is no firmware info table. */
-		return 0;
-
-	/* Patch the state. */
-	data->vbios_boot_state.sclk_bootup_value  = le32_to_cpu(fw_info->ulDefaultEngineClock);
-	data->vbios_boot_state.mclk_bootup_value  = le32_to_cpu(fw_info->ulDefaultMemoryClock);
-	data->vbios_boot_state.mvdd_bootup_value  = le16_to_cpu(fw_info->usBootUpMVDDCVoltage);
-	data->vbios_boot_state.vddc_bootup_value  = le16_to_cpu(fw_info->usBootUpVDDCVoltage);
-	data->vbios_boot_state.vddci_bootup_value = le16_to_cpu(fw_info->usBootUpVDDCIVoltage);
-	data->vbios_boot_state.pcie_gen_bootup_value = tonga_get_current_pcie_speed(hwmgr);
-	data->vbios_boot_state.pcie_lane_bootup_value =
-			(uint16_t)tonga_get_current_pcie_lane_number(hwmgr);
-
-	/* set boot power state */
-	ps->performance_levels[0].memory_clock = data->vbios_boot_state.mclk_bootup_value;
-	ps->performance_levels[0].engine_clock = data->vbios_boot_state.sclk_bootup_value;
-	ps->performance_levels[0].pcie_gen = data->vbios_boot_state.pcie_gen_bootup_value;
-	ps->performance_levels[0].pcie_lane = data->vbios_boot_state.pcie_lane_bootup_value;
-
-	return 0;
-}
-
-static int tonga_get_pp_table_entry_callback_func(struct pp_hwmgr *hwmgr,
-		void *state, struct pp_power_state *power_state,
-		void *pp_table, uint32_t classification_flag)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	struct tonga_power_state  *tonga_ps =
-			(struct tonga_power_state *)(&(power_state->hardware));
-
-	struct tonga_performance_level *performance_level;
-
-	ATOM_Tonga_State *state_entry = (ATOM_Tonga_State *)state;
-
-	ATOM_Tonga_POWERPLAYTABLE *powerplay_table =
-			(ATOM_Tonga_POWERPLAYTABLE *)pp_table;
-
-	ATOM_Tonga_SCLK_Dependency_Table *sclk_dep_table =
-			(ATOM_Tonga_SCLK_Dependency_Table *)
-			(((unsigned long)powerplay_table) +
-			le16_to_cpu(powerplay_table->usSclkDependencyTableOffset));
-
-	ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table =
-			(ATOM_Tonga_MCLK_Dependency_Table *)
-			(((unsigned long)powerplay_table) +
-			le16_to_cpu(powerplay_table->usMclkDependencyTableOffset));
-
-	/* The following fields are not initialized here: id orderedList allStatesList */
-	power_state->classification.ui_label =
-			(le16_to_cpu(state_entry->usClassification) &
-			ATOM_PPLIB_CLASSIFICATION_UI_MASK) >>
-			ATOM_PPLIB_CLASSIFICATION_UI_SHIFT;
-	power_state->classification.flags = classification_flag;
-	/* NOTE: There is a classification2 flag in BIOS that is not being used right now */
-
-	power_state->classification.temporary_state = false;
-	power_state->classification.to_be_deleted = false;
-
-	power_state->validation.disallowOnDC =
-			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) & ATOM_Tonga_DISALLOW_ON_DC));
-
-	power_state->pcie.lanes = 0;
-
-	power_state->display.disableFrameModulation = false;
-	power_state->display.limitRefreshrate = false;
-	power_state->display.enableVariBright =
-			(0 != (le32_to_cpu(state_entry->ulCapsAndSettings) & ATOM_Tonga_ENABLE_VARIBRIGHT));
-
-	power_state->validation.supportedPowerLevels = 0;
-	power_state->uvd_clocks.VCLK = 0;
-	power_state->uvd_clocks.DCLK = 0;
-	power_state->temperatures.min = 0;
-	power_state->temperatures.max = 0;
-
-	performance_level = &(tonga_ps->performance_levels
-			[tonga_ps->performance_level_count++]);
-
-	PP_ASSERT_WITH_CODE(
-			(tonga_ps->performance_level_count < SMU72_MAX_LEVELS_GRAPHICS),
-			"Performance levels exceeds SMC limit!",
-			return -1);
-
-	PP_ASSERT_WITH_CODE(
-			(tonga_ps->performance_level_count <=
-					hwmgr->platform_descriptor.hardwareActivityPerformanceLevels),
-			"Performance levels exceeds Driver limit!",
-			return -1);
-
-	/* Performance levels are arranged from low to high. */
-	performance_level->memory_clock =
-				le32_to_cpu(mclk_dep_table->entries[state_entry->ucMemoryClockIndexLow].ulMclk);
-
-	performance_level->engine_clock =
-				le32_to_cpu(sclk_dep_table->entries[state_entry->ucEngineClockIndexLow].ulSclk);
-
-	performance_level->pcie_gen = get_pcie_gen_support(
-							data->pcie_gen_cap,
-					     state_entry->ucPCIEGenLow);
-
-	performance_level->pcie_lane = get_pcie_lane_support(
-						    data->pcie_lane_cap,
-					   state_entry->ucPCIELaneHigh);
-
-	performance_level =
-			&(tonga_ps->performance_levels[tonga_ps->performance_level_count++]);
-
-	performance_level->memory_clock =
-				le32_to_cpu(mclk_dep_table->entries[state_entry->ucMemoryClockIndexHigh].ulMclk);
-
-	performance_level->engine_clock =
-				le32_to_cpu(sclk_dep_table->entries[state_entry->ucEngineClockIndexHigh].ulSclk);
-
-	performance_level->pcie_gen = get_pcie_gen_support(
-							data->pcie_gen_cap,
-					    state_entry->ucPCIEGenHigh);
-
-	performance_level->pcie_lane = get_pcie_lane_support(
-						    data->pcie_lane_cap,
-					   state_entry->ucPCIELaneHigh);
-
-	return 0;
-}
-
-static int tonga_get_pp_table_entry(struct pp_hwmgr *hwmgr,
-		    unsigned long entry_index, struct pp_power_state *ps)
-{
-	int result;
-	struct tonga_power_state *tonga_ps;
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	struct phm_ppt_v1_information *table_info =
-			(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	struct phm_ppt_v1_clock_voltage_dependency_table *dep_mclk_table =
-					   table_info->vdd_dep_on_mclk;
-
-	ps->hardware.magic = PhwTonga_Magic;
-
-	tonga_ps = cast_phw_tonga_power_state(&(ps->hardware));
-
-	result = tonga_get_powerplay_table_entry(hwmgr, entry_index, ps,
-			tonga_get_pp_table_entry_callback_func);
-
-	/* This is the earliest time we have all the dependency table and the VBIOS boot state
-	 * as PP_Tables_GetPowerPlayTableEntry retrieves the VBIOS boot state
-	 * if there is only one VDDCI/MCLK level, check if it's the same as VBIOS boot state
-	 */
-	if (dep_mclk_table != NULL && dep_mclk_table->count == 1) {
-		if (dep_mclk_table->entries[0].clk !=
-				data->vbios_boot_state.mclk_bootup_value)
-			printk(KERN_ERR "Single MCLK entry VDDCI/MCLK dependency table "
-					"does not match VBIOS boot MCLK level");
-		if (dep_mclk_table->entries[0].vddci !=
-				data->vbios_boot_state.vddci_bootup_value)
-			printk(KERN_ERR "Single VDDCI entry VDDCI/MCLK dependency table "
-					"does not match VBIOS boot VDDCI level");
-	}
-
-	/* set DC compatible flag if this state supports DC */
-	if (!ps->validation.disallowOnDC)
-		tonga_ps->dc_compatible = true;
-
-	if (ps->classification.flags & PP_StateClassificationFlag_ACPI)
-		data->acpi_pcie_gen = tonga_ps->performance_levels[0].pcie_gen;
-	else if (ps->classification.flags & PP_StateClassificationFlag_Boot) {
-		if (data->bacos.best_match == 0xffff) {
-			/* For V.I. use boot state as base BACO state */
-			data->bacos.best_match = PP_StateClassificationFlag_Boot;
-			data->bacos.performance_level = tonga_ps->performance_levels[0];
-		}
-	}
-
-	tonga_ps->uvd_clocks.VCLK = ps->uvd_clocks.VCLK;
-	tonga_ps->uvd_clocks.DCLK = ps->uvd_clocks.DCLK;
-
-	if (!result) {
-		uint32_t i;
-
-		switch (ps->classification.ui_label) {
-		case PP_StateUILabel_Performance:
-			data->use_pcie_performance_levels = true;
-
-			for (i = 0; i < tonga_ps->performance_level_count; i++) {
-				if (data->pcie_gen_performance.max <
-						tonga_ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_performance.max =
-							tonga_ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_gen_performance.min >
-						tonga_ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_performance.min =
-							tonga_ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_lane_performance.max <
-						tonga_ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_performance.max =
-							tonga_ps->performance_levels[i].pcie_lane;
-
-				if (data->pcie_lane_performance.min >
-						tonga_ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_performance.min =
-							tonga_ps->performance_levels[i].pcie_lane;
-			}
-			break;
-		case PP_StateUILabel_Battery:
-			data->use_pcie_power_saving_levels = true;
-
-			for (i = 0; i < tonga_ps->performance_level_count; i++) {
-				if (data->pcie_gen_power_saving.max <
-						tonga_ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_power_saving.max =
-							tonga_ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_gen_power_saving.min >
-						tonga_ps->performance_levels[i].pcie_gen)
-					data->pcie_gen_power_saving.min =
-							tonga_ps->performance_levels[i].pcie_gen;
-
-				if (data->pcie_lane_power_saving.max <
-						tonga_ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_power_saving.max =
-							tonga_ps->performance_levels[i].pcie_lane;
-
-				if (data->pcie_lane_power_saving.min >
-						tonga_ps->performance_levels[i].pcie_lane)
-					data->pcie_lane_power_saving.min =
-							tonga_ps->performance_levels[i].pcie_lane;
-			}
-			break;
-		default:
-			break;
-		}
-	}
-	return 0;
-}
-
-static void
-tonga_print_current_perforce_level(struct pp_hwmgr *hwmgr, struct seq_file *m)
-{
-	uint32_t sclk, mclk, activity_percent;
-	uint32_t offset;
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)(PPSMC_MSG_API_GetSclkFrequency));
-
-	sclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-
-	smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)(PPSMC_MSG_API_GetMclkFrequency));
-
-	mclk = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-	seq_printf(m, "\n [  mclk  ]: %u MHz\n\n [  sclk  ]: %u MHz\n", mclk/100, sclk/100);
-
-	offset = data->soft_regs_start + offsetof(SMU72_SoftRegisters, AverageGraphicsActivity);
-	activity_percent = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, offset);
-	activity_percent += 0x80;
-	activity_percent >>= 8;
-
-	seq_printf(m, "\n [GPU load]: %u%%\n\n", activity_percent > 100 ? 100 : activity_percent);
-
-	seq_printf(m, "uvd    %sabled\n", data->uvd_power_gated ? "dis" : "en");
-
-	seq_printf(m, "vce    %sabled\n", data->vce_power_gated ? "dis" : "en");
-}
-
-static int tonga_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
-	const struct tonga_power_state *tonga_ps = cast_const_phw_tonga_power_state(states->pnew_state);
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_single_dpm_table *psclk_table = &(data->dpm_table.sclk_table);
-	uint32_t sclk = tonga_ps->performance_levels[tonga_ps->performance_level_count-1].engine_clock;
-	struct tonga_single_dpm_table *pmclk_table = &(data->dpm_table.mclk_table);
-	uint32_t mclk = tonga_ps->performance_levels[tonga_ps->performance_level_count-1].memory_clock;
-	struct PP_Clocks min_clocks = {0};
-	uint32_t i;
-	struct cgs_display_info info = {0};
-
-	data->need_update_smu7_dpm_table = 0;
-
-	for (i = 0; i < psclk_table->count; i++) {
-		if (sclk == psclk_table->dpm_levels[i].value)
-			break;
-	}
-
-	if (i >= psclk_table->count)
-		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
-	else {
-	/* TODO: Check SCLK in DAL's minimum clocks in case DeepSleep divider update is required.*/
-		if(data->display_timing.min_clock_insr != min_clocks.engineClockInSR)
-			data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
-	}
-
-	for (i=0; i < pmclk_table->count; i++) {
-		if (mclk == pmclk_table->dpm_levels[i].value)
-			break;
-	}
-
-	if (i >= pmclk_table->count)
-		data->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	if (data->display_timing.num_existing_displays != info.display_count)
-		data->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
-
-	return 0;
-}
-
-static uint16_t tonga_get_maximum_link_speed(struct pp_hwmgr *hwmgr, const struct tonga_power_state *hw_ps)
-{
-	uint32_t i;
-	uint32_t sclk, max_sclk = 0;
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_dpm_table *pdpm_table = &data->dpm_table;
-
-	for (i = 0; i < hw_ps->performance_level_count; i++) {
-		sclk = hw_ps->performance_levels[i].engine_clock;
-		if (max_sclk < sclk)
-			max_sclk = sclk;
-	}
-
-	for (i = 0; i < pdpm_table->sclk_table.count; i++) {
-		if (pdpm_table->sclk_table.dpm_levels[i].value == max_sclk)
-			return (uint16_t) ((i >= pdpm_table->pcie_speed_table.count) ?
-					pdpm_table->pcie_speed_table.dpm_levels[pdpm_table->pcie_speed_table.count-1].value :
-					pdpm_table->pcie_speed_table.dpm_levels[i].value);
-	}
-
-	return 0;
-}
-
-static int tonga_request_link_speed_change_before_state_change(struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	const struct tonga_power_state *tonga_nps = cast_const_phw_tonga_power_state(states->pnew_state);
-	const struct tonga_power_state *tonga_cps = cast_const_phw_tonga_power_state(states->pcurrent_state);
-
-	uint16_t target_link_speed = tonga_get_maximum_link_speed(hwmgr, tonga_nps);
-	uint16_t current_link_speed;
-
-	if (data->force_pcie_gen == PP_PCIEGenInvalid)
-		current_link_speed = tonga_get_maximum_link_speed(hwmgr, tonga_cps);
-	else
-		current_link_speed = data->force_pcie_gen;
-
-	data->force_pcie_gen = PP_PCIEGenInvalid;
-	data->pspp_notify_required = false;
-	if (target_link_speed > current_link_speed) {
-		switch(target_link_speed) {
-		case PP_PCIEGen3:
-			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN3, false))
-				break;
-			data->force_pcie_gen = PP_PCIEGen2;
-			if (current_link_speed == PP_PCIEGen2)
-				break;
-		case PP_PCIEGen2:
-			if (0 == acpi_pcie_perf_request(hwmgr->device, PCIE_PERF_REQ_GEN2, false))
-				break;
-		default:
-			data->force_pcie_gen = tonga_get_current_pcie_speed(hwmgr);
-			break;
-		}
-	} else {
-		if (target_link_speed < current_link_speed)
-			data->pspp_notify_required = true;
-	}
-
-	return 0;
-}
-
-static int tonga_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	if (0 == data->need_update_smu7_dpm_table)
-		return 0;
-
-	if ((0 == data->sclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table &
-		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
-		PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
-				    "Trying to freeze SCLK DPM when DPM is disabled",
-			);
-		PP_ASSERT_WITH_CODE(
-			0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					  PPSMC_MSG_SCLKDPM_FreezeLevel),
-			"Failed to freeze SCLK DPM during FreezeSclkMclkDPM Function!",
-			return -1);
-	}
-
-	if ((0 == data->mclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table &
-		 DPMTABLE_OD_UPDATE_MCLK)) {
-		PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
-				    "Trying to freeze MCLK DPM when DPM is disabled",
-			);
-		PP_ASSERT_WITH_CODE(
-			0 == smum_send_msg_to_smc(hwmgr->smumgr,
-							PPSMC_MSG_MCLKDPM_FreezeLevel),
-			"Failed to freeze MCLK DPM during FreezeSclkMclkDPM Function!",
-			return -1);
-	}
-
-	return 0;
-}
-
-static int tonga_populate_and_upload_sclk_mclk_dpm_levels(struct pp_hwmgr *hwmgr, const void *input)
-{
-	int result = 0;
-
-	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
-	const struct tonga_power_state *tonga_ps = cast_const_phw_tonga_power_state(states->pnew_state);
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	uint32_t sclk = tonga_ps->performance_levels[tonga_ps->performance_level_count-1].engine_clock;
-	uint32_t mclk = tonga_ps->performance_levels[tonga_ps->performance_level_count-1].memory_clock;
-	struct tonga_dpm_table *pdpm_table = &data->dpm_table;
-
-	struct tonga_dpm_table *pgolden_dpm_table = &data->golden_dpm_table;
-	uint32_t dpm_count, clock_percent;
-	uint32_t i;
-
-	if (0 == data->need_update_smu7_dpm_table)
-		return 0;
-
-	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK) {
-		pdpm_table->sclk_table.dpm_levels[pdpm_table->sclk_table.count-1].value = sclk;
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinACSupport) ||
-		    phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinDCSupport)) {
-		/* Need to do calculation based on the golden DPM table
-		 * as the Heatmap GPU Clock axis is also based on the default values
-		 */
-			PP_ASSERT_WITH_CODE(
-				(pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value != 0),
-				"Divide by 0!",
-				return -1);
-			dpm_count = pdpm_table->sclk_table.count < 2 ? 0 : pdpm_table->sclk_table.count-2;
-			for (i = dpm_count; i > 1; i--) {
-				if (sclk > pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value) {
-					clock_percent = ((sclk - pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value)*100) /
-							pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value;
-
-					pdpm_table->sclk_table.dpm_levels[i].value =
-							pgolden_dpm_table->sclk_table.dpm_levels[i].value +
-							(pgolden_dpm_table->sclk_table.dpm_levels[i].value * clock_percent)/100;
-
-				} else if (pgolden_dpm_table->sclk_table.dpm_levels[pdpm_table->sclk_table.count-1].value > sclk) {
-					clock_percent = ((pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value - sclk)*100) /
-								pgolden_dpm_table->sclk_table.dpm_levels[pgolden_dpm_table->sclk_table.count-1].value;
-
-					pdpm_table->sclk_table.dpm_levels[i].value =
-							pgolden_dpm_table->sclk_table.dpm_levels[i].value -
-							(pgolden_dpm_table->sclk_table.dpm_levels[i].value * clock_percent)/100;
-				} else
-					pdpm_table->sclk_table.dpm_levels[i].value =
-							pgolden_dpm_table->sclk_table.dpm_levels[i].value;
-			}
-		}
-	}
-
-	if (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK) {
-		pdpm_table->mclk_table.dpm_levels[pdpm_table->mclk_table.count-1].value = mclk;
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinACSupport) ||
-			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_OD6PlusinDCSupport)) {
-
-			PP_ASSERT_WITH_CODE(
-					(pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value != 0),
-					"Divide by 0!",
-					return -1);
-			dpm_count = pdpm_table->mclk_table.count < 2? 0 : pdpm_table->mclk_table.count-2;
-			for (i = dpm_count; i > 1; i--) {
-				if (mclk > pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value) {
-						clock_percent = ((mclk - pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value)*100) /
-								    pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value;
-
-						pdpm_table->mclk_table.dpm_levels[i].value =
-										pgolden_dpm_table->mclk_table.dpm_levels[i].value +
-										(pgolden_dpm_table->mclk_table.dpm_levels[i].value * clock_percent)/100;
-
-				} else if (pgolden_dpm_table->mclk_table.dpm_levels[pdpm_table->mclk_table.count-1].value > mclk) {
-						clock_percent = ((pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value - mclk)*100) /
-								    pgolden_dpm_table->mclk_table.dpm_levels[pgolden_dpm_table->mclk_table.count-1].value;
-
-						pdpm_table->mclk_table.dpm_levels[i].value =
-									pgolden_dpm_table->mclk_table.dpm_levels[i].value -
-									(pgolden_dpm_table->mclk_table.dpm_levels[i].value * clock_percent)/100;
-				} else
-					pdpm_table->mclk_table.dpm_levels[i].value = pgolden_dpm_table->mclk_table.dpm_levels[i].value;
-			}
-		}
-	}
-
-	if (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK)) {
-		result = tonga_populate_all_graphic_levels(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == result),
-			"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
-			return result);
-	}
-
-	if (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_MCLK + DPMTABLE_UPDATE_MCLK)) {
-		/*populate MCLK dpm table to SMU7 */
-		result = tonga_populate_all_memory_levels(hwmgr);
-		PP_ASSERT_WITH_CODE((0 == result),
-				"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
-				return result);
-	}
-
-	return result;
-}
-
-static	int tonga_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
-			  struct tonga_single_dpm_table * pdpm_table,
-			     uint32_t low_limit, uint32_t high_limit)
-{
-	uint32_t i;
-
-	for (i = 0; i < pdpm_table->count; i++) {
-		if ((pdpm_table->dpm_levels[i].value < low_limit) ||
-		    (pdpm_table->dpm_levels[i].value > high_limit))
-			pdpm_table->dpm_levels[i].enabled = false;
-		else
-			pdpm_table->dpm_levels[i].enabled = true;
-	}
-	return 0;
-}
-
-static int tonga_trim_dpm_states(struct pp_hwmgr *hwmgr, const struct tonga_power_state *hw_state)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	uint32_t high_limit_count;
-
-	PP_ASSERT_WITH_CODE((hw_state->performance_level_count >= 1),
-				"power state did not have any performance level",
-				 return -1);
-
-	high_limit_count = (1 == hw_state->performance_level_count) ? 0: 1;
-
-	tonga_trim_single_dpm_states(hwmgr,
-					&(data->dpm_table.sclk_table),
-					hw_state->performance_levels[0].engine_clock,
-					hw_state->performance_levels[high_limit_count].engine_clock);
-
-	tonga_trim_single_dpm_states(hwmgr,
-						&(data->dpm_table.mclk_table),
-						hw_state->performance_levels[0].memory_clock,
-						hw_state->performance_levels[high_limit_count].memory_clock);
-
-	return 0;
-}
-
-static int tonga_generate_dpm_level_enable_mask(struct pp_hwmgr *hwmgr, const void *input)
-{
-	int result;
-	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	const struct tonga_power_state *tonga_ps = cast_const_phw_tonga_power_state(states->pnew_state);
-
-	result = tonga_trim_dpm_states(hwmgr, tonga_ps);
-	if (0 != result)
-		return result;
-
-	data->dpm_level_enable_mask.sclk_dpm_enable_mask = tonga_get_dpm_level_enable_mask_value(&data->dpm_table.sclk_table);
-	data->dpm_level_enable_mask.mclk_dpm_enable_mask = tonga_get_dpm_level_enable_mask_value(&data->dpm_table.mclk_table);
-	data->last_mclk_dpm_enable_mask = data->dpm_level_enable_mask.mclk_dpm_enable_mask;
-	if (data->uvd_enabled)
-		data->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
-
-	data->dpm_level_enable_mask.pcie_dpm_enable_mask = tonga_get_dpm_level_enable_mask_value(&data->dpm_table.pcie_speed_table);
-
-	return 0;
-}
-
-int tonga_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
-				  (PPSMC_Msg)PPSMC_MSG_VCEDPM_Enable :
-				  (PPSMC_Msg)PPSMC_MSG_VCEDPM_Disable);
-}
-
-int tonga_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable)
-{
-	return smum_send_msg_to_smc(hwmgr->smumgr, enable ?
-				  (PPSMC_Msg)PPSMC_MSG_UVDDPM_Enable :
-				  (PPSMC_Msg)PPSMC_MSG_UVDDPM_Disable);
-}
-
-int tonga_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	uint32_t mm_boot_level_offset, mm_boot_level_value;
-	struct phm_ppt_v1_information *ptable_information = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (!bgate) {
-		data->smc_state_table.UvdBootLevel = (uint8_t) (ptable_information->mm_dep_table->count - 1);
-		mm_boot_level_offset = data->dpm_table_start + offsetof(SMU72_Discrete_DpmTable, UvdBootLevel);
-		mm_boot_level_offset /= 4;
-		mm_boot_level_offset *= 4;
-		mm_boot_level_value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, mm_boot_level_offset);
-		mm_boot_level_value &= 0x00FFFFFF;
-		mm_boot_level_value |= data->smc_state_table.UvdBootLevel << 24;
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
-
-		if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_UVDDPM) ||
-		    phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-						PPSMC_MSG_UVDDPM_SetEnabledMask,
-						(uint32_t)(1 << data->smc_state_table.UvdBootLevel));
-	}
-
-	return tonga_enable_disable_uvd_dpm(hwmgr, !bgate);
-}
-
-int tonga_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	const struct tonga_power_state *tonga_nps = cast_const_phw_tonga_power_state(states->pnew_state);
-	const struct tonga_power_state *tonga_cps = cast_const_phw_tonga_power_state(states->pcurrent_state);
-
-	uint32_t mm_boot_level_offset, mm_boot_level_value;
-	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	if (tonga_nps->vce_clocks.EVCLK > 0 && (tonga_cps == NULL || tonga_cps->vce_clocks.EVCLK == 0)) {
-		data->smc_state_table.VceBootLevel = (uint8_t) (pptable_info->mm_dep_table->count - 1);
-
-		mm_boot_level_offset = data->dpm_table_start + offsetof(SMU72_Discrete_DpmTable, VceBootLevel);
-		mm_boot_level_offset /= 4;
-		mm_boot_level_offset *= 4;
-		mm_boot_level_value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, mm_boot_level_offset);
-		mm_boot_level_value &= 0xFF00FFFF;
-		mm_boot_level_value |= data->smc_state_table.VceBootLevel << 16;
-		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
-
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_VCEDPM_SetEnabledMask,
-				(uint32_t)(1 << data->smc_state_table.VceBootLevel));
-
-		tonga_enable_disable_vce_dpm(hwmgr, true);
-	} else if (tonga_nps->vce_clocks.EVCLK == 0 && tonga_cps != NULL && tonga_cps->vce_clocks.EVCLK > 0)
-		tonga_enable_disable_vce_dpm(hwmgr, false);
-
-	return 0;
-}
-
-static int tonga_update_and_upload_mc_reg_table(struct pp_hwmgr *hwmgr)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	uint32_t address;
-	int32_t result;
-
-	if (0 == (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK))
-		return 0;
-
-
-	memset(&data->mc_reg_table, 0, sizeof(SMU72_Discrete_MCRegisters));
-
-	result = tonga_convert_mc_reg_table_to_smc(hwmgr, &(data->mc_reg_table));
-
-	if(result != 0)
-		return result;
-
-
-	address = data->mc_reg_table_start + (uint32_t)offsetof(SMU72_Discrete_MCRegisters, data[0]);
-
-	return  tonga_copy_bytes_to_smc(hwmgr->smumgr, address,
-				 (uint8_t *)&data->mc_reg_table.data[0],
-				sizeof(SMU72_Discrete_MCRegisterSet) * data->dpm_table.mclk_table.count,
-				data->sram_end);
-}
-
-static int tonga_program_memory_timing_parameters_conditionally(struct pp_hwmgr *hwmgr)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	if (data->need_update_smu7_dpm_table &
-		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_OD_UPDATE_MCLK))
-		return tonga_program_memory_timing_parameters(hwmgr);
-
-	return 0;
-}
-
-static int tonga_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	if (0 == data->need_update_smu7_dpm_table)
-		return 0;
-
-	if ((0 == data->sclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table &
-		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
-
-		PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
-				    "Trying to Unfreeze SCLK DPM when DPM is disabled",
-			);
-		PP_ASSERT_WITH_CODE(
-			 0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					 PPSMC_MSG_SCLKDPM_UnfreezeLevel),
-			"Failed to unfreeze SCLK DPM during UnFreezeSclkMclkDPM Function!",
-			return -1);
-	}
-
-	if ((0 == data->mclk_dpm_key_disabled) &&
-		(data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
-
-		PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
-				    "Trying to Unfreeze MCLK DPM when DPM is disabled",
-				);
-		PP_ASSERT_WITH_CODE(
-			 0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					 PPSMC_MSG_SCLKDPM_UnfreezeLevel),
-		    "Failed to unfreeze MCLK DPM during UnFreezeSclkMclkDPM Function!",
-		    return -1);
-	}
-
-	data->need_update_smu7_dpm_table = 0;
-
-	return 0;
-}
-
-static int tonga_notify_link_speed_change_after_state_change(struct pp_hwmgr *hwmgr, const void *input)
-{
-	const struct phm_set_power_state_input *states = (const struct phm_set_power_state_input *)input;
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	const struct tonga_power_state *tonga_ps = cast_const_phw_tonga_power_state(states->pnew_state);
-	uint16_t target_link_speed = tonga_get_maximum_link_speed(hwmgr, tonga_ps);
-	uint8_t  request;
-
-	if (data->pspp_notify_required  ||
-	    data->pcie_performance_request) {
-		if (target_link_speed == PP_PCIEGen3)
-			request = PCIE_PERF_REQ_GEN3;
-		else if (target_link_speed == PP_PCIEGen2)
-			request = PCIE_PERF_REQ_GEN2;
-		else
-			request = PCIE_PERF_REQ_GEN1;
-
-		if(request == PCIE_PERF_REQ_GEN1 && tonga_get_current_pcie_speed(hwmgr) > 0) {
-			data->pcie_performance_request = false;
-			return 0;
-		}
-
-		if (0 != acpi_pcie_perf_request(hwmgr->device, request, false)) {
-			if (PP_PCIEGen2 == target_link_speed)
-				printk("PSPP request to switch to Gen2 from Gen3 Failed!");
-			else
-				printk("PSPP request to switch to Gen1 from Gen2 Failed!");
-		}
-	}
-
-	data->pcie_performance_request = false;
-	return 0;
-}
-
-static int tonga_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *input)
-{
-	int tmp_result, result = 0;
-
-	tmp_result = tonga_find_dpm_states_clocks_in_dpm_table(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to find DPM states clocks in DPM table!", result = tmp_result);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest)) {
-		tmp_result = tonga_request_link_speed_change_before_state_change(hwmgr, input);
-		PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to request link speed change before state change!", result = tmp_result);
-	}
-
-	tmp_result = tonga_freeze_sclk_mclk_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to freeze SCLK MCLK DPM!", result = tmp_result);
-
-	tmp_result = tonga_populate_and_upload_sclk_mclk_dpm_levels(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to populate and upload SCLK MCLK DPM levels!", result = tmp_result);
-
-	tmp_result = tonga_generate_dpm_level_enable_mask(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to generate DPM level enabled mask!", result = tmp_result);
-
-	tmp_result = tonga_update_vce_dpm(hwmgr, input);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to update VCE DPM!", result = tmp_result);
-
-	tmp_result = tonga_update_sclk_threshold(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to update SCLK threshold!", result = tmp_result);
-
-	tmp_result = tonga_update_and_upload_mc_reg_table(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to upload MC reg table!", result = tmp_result);
-
-	tmp_result = tonga_program_memory_timing_parameters_conditionally(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to program memory timing parameters!", result = tmp_result);
-
-	tmp_result = tonga_unfreeze_sclk_mclk_dpm(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to unfreeze SCLK MCLK DPM!", result = tmp_result);
-
-	tmp_result = tonga_upload_dpm_level_enable_mask(hwmgr);
-	PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to upload DPM level enabled mask!", result = tmp_result);
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PCIEPerformanceRequest)) {
-		tmp_result = tonga_notify_link_speed_change_after_state_change(hwmgr, input);
-		PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to notify link speed change after state change!", result = tmp_result);
-	}
-
-	return result;
-}
-
-/**
-*  Set maximum target operating fan output PWM
-*
-* @param    pHwMgr:  the address of the powerplay hardware manager.
-* @param    usMaxFanPwm:  max operating fan PWM in percents
-* @return   The response that came from the SMC.
-*/
-static int tonga_set_max_fan_pwm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_pwm)
-{
-	hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanPWM = us_max_fan_pwm;
-
-	if (phm_is_hw_access_blocked(hwmgr))
-		return 0;
-
-	return (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanPwmMax, us_max_fan_pwm) ? 0 : -1);
-}
-
-int tonga_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
-{
-	uint32_t num_active_displays = 0;
-	struct cgs_display_info info = {0};
-	info.mode_info = NULL;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	num_active_displays = info.display_count;
-
-	if (num_active_displays > 1)  /* to do && (pHwMgr->pPECI->displayConfiguration.bMultiMonitorInSync != TRUE)) */
-		tonga_notify_smc_display_change(hwmgr, false);
-	else
-		tonga_notify_smc_display_change(hwmgr, true);
-
-	return 0;
-}
-
-/**
-* Programs the display gap
-*
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @return   always OK
-*/
-int tonga_program_display_gap(struct pp_hwmgr *hwmgr)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	uint32_t num_active_displays = 0;
-	uint32_t display_gap = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL);
-	uint32_t display_gap2;
-	uint32_t pre_vbi_time_in_us;
-	uint32_t frame_time_in_us;
-	uint32_t ref_clock;
-	uint32_t refresh_rate = 0;
-	struct cgs_display_info info = {0};
-	struct cgs_mode_info mode_info;
-
-	info.mode_info = &mode_info;
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-	num_active_displays = info.display_count;
-
-	display_gap = PHM_SET_FIELD(display_gap, CG_DISPLAY_GAP_CNTL, DISP_GAP, (num_active_displays > 0)? DISPLAY_GAP_VBLANK_OR_WM : DISPLAY_GAP_IGNORE);
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL, display_gap);
-
-	ref_clock = mode_info.ref_clock;
-	refresh_rate = mode_info.refresh_rate;
-
-	if(0 == refresh_rate)
-		refresh_rate = 60;
-
-	frame_time_in_us = 1000000 / refresh_rate;
-
-	pre_vbi_time_in_us = frame_time_in_us - 200 - mode_info.vblank_time_us;
-	display_gap2 = pre_vbi_time_in_us * (ref_clock / 100);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_DISPLAY_GAP_CNTL2, display_gap2);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + offsetof(SMU72_SoftRegisters, PreVBlankGap), 0x64);
-
-	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + offsetof(SMU72_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us));
-
-	if (num_active_displays == 1)
-		tonga_notify_smc_display_change(hwmgr, true);
-
-	return 0;
-}
-
-int tonga_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
-{
-
-	tonga_program_display_gap(hwmgr);
-
-	/* to do PhwTonga_CacUpdateDisplayConfiguration(pHwMgr); */
-	return 0;
-}
-
-/**
-*  Set maximum target operating fan output RPM
-*
-* @param    pHwMgr:  the address of the powerplay hardware manager.
-* @param    usMaxFanRpm:  max operating fan RPM value.
-* @return   The response that came from the SMC.
-*/
-static int tonga_set_max_fan_rpm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_pwm)
-{
-	hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanRPM = us_max_fan_pwm;
-
-	if (phm_is_hw_access_blocked(hwmgr))
-		return 0;
-
-	return (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanRpmMax, us_max_fan_pwm) ? 0 : -1);
-}
-
-uint32_t tonga_get_xclk(struct pp_hwmgr *hwmgr)
-{
-	uint32_t reference_clock;
-	uint32_t tc;
-	uint32_t divide;
-
-	ATOM_FIRMWARE_INFO *fw_info;
-	uint16_t size;
-	uint8_t frev, crev;
-	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
-
-	tc = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_CLKPIN_CNTL_2, MUX_TCLK_TO_XCLK);
-
-	if (tc)
-		return TCLK;
-
-	fw_info = (ATOM_FIRMWARE_INFO *)cgs_atom_get_data_table(hwmgr->device, index,
-						  &size, &frev, &crev);
-
-	if (!fw_info)
-		return 0;
-
-	reference_clock = le16_to_cpu(fw_info->usReferenceClock);
-
-	divide = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_CLKPIN_CNTL, XTALIN_DIVIDE);
-
-	if (0 != divide)
-		return reference_clock / 4;
-
-	return reference_clock;
-}
-
-int tonga_dpm_set_interrupt_state(void *private_data,
-					 unsigned src_id, unsigned type,
-					 int enabled)
-{
-	uint32_t cg_thermal_int;
-	struct pp_hwmgr *hwmgr = ((struct pp_eventmgr *)private_data)->hwmgr;
-
-	if (hwmgr == NULL)
-		return -EINVAL;
-
-	switch (type) {
-	case AMD_THERMAL_IRQ_LOW_TO_HIGH:
-		if (enabled) {
-			cg_thermal_int = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT);
-			cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
-		} else {
-			cg_thermal_int = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT);
-			cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
-		}
-		break;
-
-	case AMD_THERMAL_IRQ_HIGH_TO_LOW:
-		if (enabled) {
-			cg_thermal_int = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT);
-			cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
-		} else {
-			cg_thermal_int = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT);
-			cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
-			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCG_THERMAL_INT, cg_thermal_int);
-		}
-		break;
-	default:
-		break;
-	}
-	return 0;
-}
-
-int tonga_register_internal_thermal_interrupt(struct pp_hwmgr *hwmgr,
-					const void *thermal_interrupt_info)
-{
-	int result;
-	const struct pp_interrupt_registration_info *info =
-			(const struct pp_interrupt_registration_info *)thermal_interrupt_info;
-
-	if (info == NULL)
-		return -EINVAL;
-
-	result = cgs_add_irq_source(hwmgr->device, 230, AMD_THERMAL_IRQ_LAST,
-				tonga_dpm_set_interrupt_state,
-				info->call_back, info->context);
-
-	if (result)
-		return -EINVAL;
-
-	result = cgs_add_irq_source(hwmgr->device, 231, AMD_THERMAL_IRQ_LAST,
-				tonga_dpm_set_interrupt_state,
-				info->call_back, info->context);
-
-	if (result)
-		return -EINVAL;
-
-	return 0;
-}
-
-bool tonga_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	bool is_update_required = false;
-	struct cgs_display_info info = {0,0,NULL};
-
-	cgs_get_active_displays_info(hwmgr->device, &info);
-
-	if (data->display_timing.num_existing_displays != info.display_count)
-		is_update_required = true;
-/* TO DO NEED TO GET DEEP SLEEP CLOCK FROM DAL
-	if (phm_cap_enabled(hwmgr->hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep)) {
-		cgs_get_min_clock_settings(hwmgr->device, &min_clocks);
-		if(min_clocks.engineClockInSR != data->display_timing.minClockInSR)
-			is_update_required = true;
-*/
-	return is_update_required;
-}
-
-static inline bool tonga_are_power_levels_equal(const struct tonga_performance_level *pl1,
-							   const struct tonga_performance_level *pl2)
-{
-	return ((pl1->memory_clock == pl2->memory_clock) &&
-		  (pl1->engine_clock == pl2->engine_clock) &&
-		  (pl1->pcie_gen == pl2->pcie_gen) &&
-		  (pl1->pcie_lane == pl2->pcie_lane));
-}
-
-int tonga_check_states_equal(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *pstate1, const struct pp_hw_power_state *pstate2, bool *equal)
-{
-	const struct tonga_power_state *psa = cast_const_phw_tonga_power_state(pstate1);
-	const struct tonga_power_state *psb = cast_const_phw_tonga_power_state(pstate2);
-	int i;
-
-	if (equal == NULL || psa == NULL || psb == NULL)
-		return -EINVAL;
-
-	/* If the two states don't even have the same number of performance levels they cannot be the same state. */
-	if (psa->performance_level_count != psb->performance_level_count) {
-		*equal = false;
-		return 0;
-	}
-
-	for (i = 0; i < psa->performance_level_count; i++) {
-		if (!tonga_are_power_levels_equal(&(psa->performance_levels[i]), &(psb->performance_levels[i]))) {
-			/* If we have found even one performance level pair that is different the states are different. */
-			*equal = false;
-			return 0;
-		}
-	}
-
-	/* If all performance levels are the same try to use the UVD clocks to break the tie.*/
-	*equal = ((psa->uvd_clocks.VCLK == psb->uvd_clocks.VCLK) && (psa->uvd_clocks.DCLK == psb->uvd_clocks.DCLK));
-	*equal &= ((psa->vce_clocks.EVCLK == psb->vce_clocks.EVCLK) && (psa->vce_clocks.ECCLK == psb->vce_clocks.ECCLK));
-	*equal &= (psa->sclk_threshold == psb->sclk_threshold);
-	*equal &= (psa->acp_clk == psb->acp_clk);
-
-	return 0;
-}
-
-static int tonga_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
-{
-	if (mode) {
-		/* stop auto-manage */
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
-				PHM_PlatformCaps_MicrocodeFanControl))
-			tonga_fan_ctrl_stop_smc_fan_control(hwmgr);
-		tonga_fan_ctrl_set_static_mode(hwmgr, mode);
-	} else
-		/* restart auto-manage */
-		tonga_fan_ctrl_reset_fan_speed_to_default(hwmgr);
-
-	return 0;
-}
-
-static int tonga_get_fan_control_mode(struct pp_hwmgr *hwmgr)
-{
-	if (hwmgr->fan_ctrl_is_in_default_mode)
-		return hwmgr->fan_ctrl_default_mode;
-	else
-		return PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-				CG_FDO_CTRL2, FDO_PWM_MODE);
-}
-
-static int tonga_force_clock_level(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, uint32_t mask)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-	if (hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL)
-		return -EINVAL;
-
-	switch (type) {
-	case PP_SCLK:
-		if (!data->sclk_dpm_key_disabled)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_SCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.sclk_dpm_enable_mask & mask);
-		break;
-	case PP_MCLK:
-		if (!data->mclk_dpm_key_disabled)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_MCLKDPM_SetEnabledMask,
-					data->dpm_level_enable_mask.mclk_dpm_enable_mask & mask);
-		break;
-	case PP_PCIE:
-	{
-		uint32_t tmp = mask & data->dpm_level_enable_mask.pcie_dpm_enable_mask;
-		uint32_t level = 0;
-
-		while (tmp >>= 1)
-			level++;
-
-		if (!data->pcie_dpm_key_disabled)
-			smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-					PPSMC_MSG_PCIeDPM_ForceLevel,
-					level);
-		break;
-	}
-	default:
-		break;
-	}
-
-	return 0;
-}
-
-static int tonga_print_clock_levels(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, char *buf)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
-	struct tonga_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
-	struct tonga_single_dpm_table *pcie_table = &(data->dpm_table.pcie_speed_table);
-	int i, now, size = 0;
-	uint32_t clock, pcie_speed;
-
-	switch (type) {
-	case PP_SCLK:
-		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetSclkFrequency);
-		clock = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-
-		for (i = 0; i < sclk_table->count; i++) {
-			if (clock > sclk_table->dpm_levels[i].value)
-				continue;
-			break;
-		}
-		now = i;
-
-		for (i = 0; i < sclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, sclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
-		break;
-	case PP_MCLK:
-		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_API_GetMclkFrequency);
-		clock = cgs_read_register(hwmgr->device, mmSMC_MSG_ARG_0);
-
-		for (i = 0; i < mclk_table->count; i++) {
-			if (clock > mclk_table->dpm_levels[i].value)
-				continue;
-			break;
-		}
-		now = i;
-
-		for (i = 0; i < mclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, mclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
-		break;
-	case PP_PCIE:
-		pcie_speed = tonga_get_current_pcie_speed(hwmgr);
-		for (i = 0; i < pcie_table->count; i++) {
-			if (pcie_speed != pcie_table->dpm_levels[i].value)
-				continue;
-			break;
-		}
-		now = i;
-
-		for (i = 0; i < pcie_table->count; i++)
-			size += sprintf(buf + size, "%d: %s %s\n", i,
-					(pcie_table->dpm_levels[i].value == 0) ? "2.5GB, x8" :
-					(pcie_table->dpm_levels[i].value == 1) ? "5.0GB, x16" :
-					(pcie_table->dpm_levels[i].value == 2) ? "8.0GB, x16" : "",
-					(i == now) ? "*" : "");
-		break;
-	default:
-		break;
-	}
-	return size;
-}
-
-static int tonga_get_sclk_od(struct pp_hwmgr *hwmgr)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
-	struct tonga_single_dpm_table *golden_sclk_table =
-			&(data->golden_dpm_table.sclk_table);
-	int value;
-
-	value = (sclk_table->dpm_levels[sclk_table->count - 1].value -
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) *
-			100 /
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
-
-	return value;
-}
-
-static int tonga_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_single_dpm_table *golden_sclk_table =
-			&(data->golden_dpm_table.sclk_table);
-	struct pp_power_state  *ps;
-	struct tonga_power_state  *tonga_ps;
-
-	if (value > 20)
-		value = 20;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	tonga_ps = cast_phw_tonga_power_state(&ps->hardware);
-
-	tonga_ps->performance_levels[tonga_ps->performance_level_count - 1].engine_clock =
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value *
-			value / 100 +
-			golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
-
-	return 0;
-}
-
-static int tonga_get_mclk_od(struct pp_hwmgr *hwmgr)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
-	struct tonga_single_dpm_table *golden_mclk_table =
-			&(data->golden_dpm_table.mclk_table);
-	int value;
-
-	value = (mclk_table->dpm_levels[mclk_table->count - 1].value -
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) *
-			100 /
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
-
-	return value;
-}
-
-static int tonga_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	struct tonga_single_dpm_table *golden_mclk_table =
-			&(data->golden_dpm_table.mclk_table);
-	struct pp_power_state  *ps;
-	struct tonga_power_state  *tonga_ps;
-
-	if (value > 20)
-		value = 20;
-
-	ps = hwmgr->request_ps;
-
-	if (ps == NULL)
-		return -EINVAL;
-
-	tonga_ps = cast_phw_tonga_power_state(&ps->hardware);
-
-	tonga_ps->performance_levels[tonga_ps->performance_level_count - 1].memory_clock =
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value *
-			value / 100 +
-			golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
-
-	return 0;
-}
-
-static const struct pp_hwmgr_func tonga_hwmgr_funcs = {
-	.backend_init = &tonga_hwmgr_backend_init,
-	.backend_fini = &tonga_hwmgr_backend_fini,
-	.asic_setup = &tonga_setup_asic_task,
-	.dynamic_state_management_enable = &tonga_enable_dpm_tasks,
-	.dynamic_state_management_disable = &tonga_disable_dpm_tasks,
-	.apply_state_adjust_rules = tonga_apply_state_adjust_rules,
-	.force_dpm_level = &tonga_force_dpm_level,
-	.power_state_set = tonga_set_power_state_tasks,
-	.get_power_state_size = tonga_get_power_state_size,
-	.get_mclk = tonga_dpm_get_mclk,
-	.get_sclk = tonga_dpm_get_sclk,
-	.patch_boot_state = tonga_dpm_patch_boot_state,
-	.get_pp_table_entry = tonga_get_pp_table_entry,
-	.get_num_of_pp_table_entries = tonga_get_number_of_powerplay_table_entries,
-	.print_current_perforce_level = tonga_print_current_perforce_level,
-	.powerdown_uvd = tonga_phm_powerdown_uvd,
-	.powergate_uvd = tonga_phm_powergate_uvd,
-	.powergate_vce = tonga_phm_powergate_vce,
-	.disable_clock_power_gating = tonga_phm_disable_clock_power_gating,
-	.update_clock_gatings = tonga_phm_update_clock_gatings,
-	.notify_smc_display_config_after_ps_adjustment = tonga_notify_smc_display_config_after_ps_adjustment,
-	.display_config_changed = tonga_display_configuration_changed_task,
-	.set_max_fan_pwm_output = tonga_set_max_fan_pwm_output,
-	.set_max_fan_rpm_output = tonga_set_max_fan_rpm_output,
-	.get_temperature = tonga_thermal_get_temperature,
-	.stop_thermal_controller = tonga_thermal_stop_thermal_controller,
-	.get_fan_speed_info = tonga_fan_ctrl_get_fan_speed_info,
-	.get_fan_speed_percent = tonga_fan_ctrl_get_fan_speed_percent,
-	.set_fan_speed_percent = tonga_fan_ctrl_set_fan_speed_percent,
-	.reset_fan_speed_to_default = tonga_fan_ctrl_reset_fan_speed_to_default,
-	.get_fan_speed_rpm = tonga_fan_ctrl_get_fan_speed_rpm,
-	.set_fan_speed_rpm = tonga_fan_ctrl_set_fan_speed_rpm,
-	.uninitialize_thermal_controller = tonga_thermal_ctrl_uninitialize_thermal_controller,
-	.register_internal_thermal_interrupt = tonga_register_internal_thermal_interrupt,
-	.check_smc_update_required_for_display_configuration = tonga_check_smc_update_required_for_display_configuration,
-	.check_states_equal = tonga_check_states_equal,
-	.set_fan_control_mode = tonga_set_fan_control_mode,
-	.get_fan_control_mode = tonga_get_fan_control_mode,
-	.force_clock_level = tonga_force_clock_level,
-	.print_clock_levels = tonga_print_clock_levels,
-	.get_sclk_od = tonga_get_sclk_od,
-	.set_sclk_od = tonga_set_sclk_od,
-	.get_mclk_od = tonga_get_mclk_od,
-	.set_mclk_od = tonga_set_mclk_od,
-};
-
-int tonga_hwmgr_init(struct pp_hwmgr *hwmgr)
-{
-	hwmgr->hwmgr_func = &tonga_hwmgr_funcs;
-	hwmgr->pptable_func = &tonga_pptable_funcs;
-	pp_tonga_thermal_initialize(hwmgr);
-	return 0;
-}
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h
deleted file mode 100644
index 3961884bfa9b..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#ifndef TONGA_HWMGR_H
-#define TONGA_HWMGR_H
-
-#include "hwmgr.h"
-#include "smu72_discrete.h"
-#include "ppatomctrl.h"
-#include "ppinterrupt.h"
-#include "tonga_powertune.h"
-#include "pp_endian.h"
-
-#define TONGA_MAX_HARDWARE_POWERLEVELS 2
-#define TONGA_DYNCLK_NUMBER_OF_TREND_COEFFICIENTS 15
-
-struct tonga_performance_level {
-	uint32_t	memory_clock;
-	uint32_t	engine_clock;
-	uint16_t    pcie_gen;
-	uint16_t    pcie_lane;
-};
-
-struct _phw_tonga_bacos {
-	uint32_t                          best_match;
-	uint32_t                          baco_flags;
-	struct tonga_performance_level		  performance_level;
-};
-typedef struct _phw_tonga_bacos phw_tonga_bacos;
-
-struct _phw_tonga_uvd_clocks {
-	uint32_t   VCLK;
-	uint32_t   DCLK;
-};
-
-typedef struct _phw_tonga_uvd_clocks phw_tonga_uvd_clocks;
-
-struct _phw_tonga_vce_clocks {
-	uint32_t   EVCLK;
-	uint32_t   ECCLK;
-};
-
-typedef struct _phw_tonga_vce_clocks phw_tonga_vce_clocks;
-
-struct tonga_power_state {
-	uint32_t                    magic;
-	phw_tonga_uvd_clocks        uvd_clocks;
-	phw_tonga_vce_clocks        vce_clocks;
-	uint32_t                    sam_clk;
-	uint32_t                    acp_clk;
-	uint16_t                    performance_level_count;
-	bool                        dc_compatible;
-	uint32_t                    sclk_threshold;
-	struct tonga_performance_level performance_levels[TONGA_MAX_HARDWARE_POWERLEVELS];
-};
-
-struct _phw_tonga_dpm_level {
-	bool		enabled;
-	uint32_t    value;
-	uint32_t    param1;
-};
-typedef struct _phw_tonga_dpm_level phw_tonga_dpm_level;
-
-#define TONGA_MAX_DEEPSLEEP_DIVIDER_ID 5
-#define MAX_REGULAR_DPM_NUMBER 8
-#define TONGA_MINIMUM_ENGINE_CLOCK 2500
-
-struct tonga_single_dpm_table {
-	uint32_t count;
-	phw_tonga_dpm_level dpm_levels[MAX_REGULAR_DPM_NUMBER];
-};
-
-struct tonga_dpm_table {
-	struct tonga_single_dpm_table  sclk_table;
-	struct tonga_single_dpm_table  mclk_table;
-	struct tonga_single_dpm_table  pcie_speed_table;
-	struct tonga_single_dpm_table  vddc_table;
-	struct tonga_single_dpm_table  vdd_gfx_table;
-	struct tonga_single_dpm_table  vdd_ci_table;
-	struct tonga_single_dpm_table  mvdd_table;
-};
-typedef struct _phw_tonga_dpm_table phw_tonga_dpm_table;
-
-
-struct _phw_tonga_clock_regisiters {
-	uint32_t  vCG_SPLL_FUNC_CNTL;
-	uint32_t  vCG_SPLL_FUNC_CNTL_2;
-	uint32_t  vCG_SPLL_FUNC_CNTL_3;
-	uint32_t  vCG_SPLL_FUNC_CNTL_4;
-	uint32_t  vCG_SPLL_SPREAD_SPECTRUM;
-	uint32_t  vCG_SPLL_SPREAD_SPECTRUM_2;
-	uint32_t  vDLL_CNTL;
-	uint32_t  vMCLK_PWRMGT_CNTL;
-	uint32_t  vMPLL_AD_FUNC_CNTL;
-	uint32_t  vMPLL_DQ_FUNC_CNTL;
-	uint32_t  vMPLL_FUNC_CNTL;
-	uint32_t  vMPLL_FUNC_CNTL_1;
-	uint32_t  vMPLL_FUNC_CNTL_2;
-	uint32_t  vMPLL_SS1;
-	uint32_t  vMPLL_SS2;
-};
-typedef struct _phw_tonga_clock_regisiters phw_tonga_clock_registers;
-
-struct _phw_tonga_voltage_smio_registers {
-	uint32_t vs0_vid_lower_smio_cntl;
-};
-typedef struct _phw_tonga_voltage_smio_registers phw_tonga_voltage_smio_registers;
-
-
-struct _phw_tonga_mc_reg_entry {
-	uint32_t mclk_max;
-	uint32_t mc_data[SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE];
-};
-typedef struct _phw_tonga_mc_reg_entry phw_tonga_mc_reg_entry;
-
-struct _phw_tonga_mc_reg_table {
-	uint8_t   last;               /* number of registers*/
-	uint8_t   num_entries;        /* number of entries in mc_reg_table_entry used*/
-	uint16_t  validflag;          /* indicate the corresponding register is valid or not. 1: valid, 0: invalid. bit0->address[0], bit1->address[1], etc.*/
-	phw_tonga_mc_reg_entry    mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
-	SMU72_Discrete_MCRegisterAddress mc_reg_address[SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE];
-};
-typedef struct _phw_tonga_mc_reg_table phw_tonga_mc_reg_table;
-
-#define DISABLE_MC_LOADMICROCODE   1
-#define DISABLE_MC_CFGPROGRAMMING  2
-
-/*Ultra Low Voltage parameter structure */
-struct _phw_tonga_ulv_parm{
-	bool	ulv_supported;
-	uint32_t   ch_ulv_parameter;
-	uint32_t   ulv_volt_change_delay;
-	struct tonga_performance_level   ulv_power_level;
-};
-typedef struct _phw_tonga_ulv_parm phw_tonga_ulv_parm;
-
-#define TONGA_MAX_LEAKAGE_COUNT  8
-
-struct _phw_tonga_leakage_voltage {
-	uint16_t  count;
-	uint16_t  leakage_id[TONGA_MAX_LEAKAGE_COUNT];
-	uint16_t  actual_voltage[TONGA_MAX_LEAKAGE_COUNT];
-};
-typedef struct _phw_tonga_leakage_voltage phw_tonga_leakage_voltage;
-
-struct _phw_tonga_display_timing {
-	uint32_t min_clock_insr;
-	uint32_t num_existing_displays;
-};
-typedef struct _phw_tonga_display_timing phw_tonga_display_timing;
-
-struct _phw_tonga_dpmlevel_enable_mask {
-	uint32_t uvd_dpm_enable_mask;
-	uint32_t vce_dpm_enable_mask;
-	uint32_t acp_dpm_enable_mask;
-	uint32_t samu_dpm_enable_mask;
-	uint32_t sclk_dpm_enable_mask;
-	uint32_t mclk_dpm_enable_mask;
-	uint32_t pcie_dpm_enable_mask;
-};
-typedef struct _phw_tonga_dpmlevel_enable_mask phw_tonga_dpmlevel_enable_mask;
-
-struct _phw_tonga_pcie_perf_range {
-	uint16_t max;
-	uint16_t min;
-};
-typedef struct _phw_tonga_pcie_perf_range phw_tonga_pcie_perf_range;
-
-struct _phw_tonga_vbios_boot_state {
-	uint16_t					mvdd_bootup_value;
-	uint16_t					vddc_bootup_value;
-	uint16_t					vddci_bootup_value;
-	uint16_t					vddgfx_bootup_value;
-	uint32_t					sclk_bootup_value;
-	uint32_t					mclk_bootup_value;
-	uint16_t					pcie_gen_bootup_value;
-	uint16_t					pcie_lane_bootup_value;
-};
-typedef struct _phw_tonga_vbios_boot_state phw_tonga_vbios_boot_state;
-
-#define DPMTABLE_OD_UPDATE_SCLK     0x00000001
-#define DPMTABLE_OD_UPDATE_MCLK     0x00000002
-#define DPMTABLE_UPDATE_SCLK        0x00000004
-#define DPMTABLE_UPDATE_MCLK        0x00000008
-
-/* We need to review which fields are needed. */
-/* This is mostly a copy of the RV7xx/Evergreen structure which is close, but not identical to the N.Islands one. */
-struct tonga_hwmgr {
-	struct tonga_dpm_table               dpm_table;
-	struct tonga_dpm_table               golden_dpm_table;
-
-	uint32_t                           voting_rights_clients0;
-	uint32_t                           voting_rights_clients1;
-	uint32_t                           voting_rights_clients2;
-	uint32_t                           voting_rights_clients3;
-	uint32_t                           voting_rights_clients4;
-	uint32_t                           voting_rights_clients5;
-	uint32_t                           voting_rights_clients6;
-	uint32_t                           voting_rights_clients7;
-	uint32_t                           static_screen_threshold_unit;
-	uint32_t                           static_screen_threshold;
-	uint32_t                           voltage_control;
-	uint32_t                           vdd_gfx_control;
-
-	uint32_t                           vddc_vddci_delta;
-	uint32_t                           vddc_vddgfx_delta;
-
-	struct pp_interrupt_registration_info    internal_high_thermal_interrupt_info;
-	struct pp_interrupt_registration_info    internal_low_thermal_interrupt_info;
-	struct pp_interrupt_registration_info    smc_to_host_interrupt_info;
-	uint32_t                          active_auto_throttle_sources;
-
-	struct pp_interrupt_registration_info    external_throttle_interrupt;
-	irq_handler_func_t             external_throttle_callback;
-	void                             *external_throttle_context;
-
-	struct pp_interrupt_registration_info    ctf_interrupt_info;
-	irq_handler_func_t             ctf_callback;
-	void                             *ctf_context;
-
-	phw_tonga_clock_registers	  clock_registers;
-	phw_tonga_voltage_smio_registers  voltage_smio_registers;
-
-	bool	is_memory_GDDR5;
-	uint16_t                          acpi_vddc;
-	bool	pspp_notify_required;        /* Flag to indicate if PSPP notification to SBIOS is required */
-	uint16_t                          force_pcie_gen;            /* The forced PCI-E speed if not 0xffff */
-	uint16_t                          acpi_pcie_gen;             /* The PCI-E speed at ACPI time */
-	uint32_t                           pcie_gen_cap;             /* The PCI-E speed capabilities bitmap from CAIL */
-	uint32_t                           pcie_lane_cap;            /* The PCI-E lane capabilities bitmap from CAIL */
-	uint32_t                           pcie_spc_cap;             /* Symbol Per Clock Capabilities from registry */
-	phw_tonga_leakage_voltage	vddc_leakage;            /* The Leakage VDDC supported (based on leakage ID).*/
-	phw_tonga_leakage_voltage	vddcgfx_leakage;         /* The Leakage VDDC supported (based on leakage ID). */
-	phw_tonga_leakage_voltage	vddci_leakage;           /* The Leakage VDDCI supported (based on leakage ID). */
-
-	uint32_t                           mvdd_control;
-	uint32_t                           vddc_mask_low;
-	uint32_t                           mvdd_mask_low;
-	uint16_t                          max_vddc_in_pp_table;        /* the maximum VDDC value in the powerplay table*/
-	uint16_t                          min_vddc_in_pp_table;
-	uint16_t                          max_vddci_in_pp_table;       /* the maximum VDDCI value in the powerplay table */
-	uint16_t                          min_vddci_in_pp_table;
-	uint32_t                           mclk_strobe_mode_threshold;
-	uint32_t                           mclk_stutter_mode_threshold;
-	uint32_t                           mclk_edc_enable_threshold;
-	uint32_t                           mclk_edc_wr_enable_threshold;
-	bool	is_uvd_enabled;
-	bool	is_xdma_enabled;
-	phw_tonga_vbios_boot_state      vbios_boot_state;
-
-	bool                         battery_state;
-	bool                         is_tlu_enabled;
-	bool                         pcie_performance_request;
-
-	/* -------------- SMC SRAM Address of firmware header tables ----------------*/
-	uint32_t                           sram_end;                 /* The first address after the SMC SRAM. */
-	uint32_t                           dpm_table_start;          /* The start of the dpm table in the SMC SRAM. */
-	uint32_t                           soft_regs_start;          /* The start of the soft registers in the SMC SRAM. */
-	uint32_t                           mc_reg_table_start;       /* The start of the mc register table in the SMC SRAM. */
-	uint32_t                           fan_table_start;          /* The start of the fan table in the SMC SRAM. */
-	uint32_t                           arb_table_start;          /* The start of the ARB setting table in the SMC SRAM. */
-	SMU72_Discrete_DpmTable         smc_state_table;             /* The carbon copy of the SMC state table. */
-	SMU72_Discrete_MCRegisters      mc_reg_table;
-	SMU72_Discrete_Ulv              ulv_setting;                 /* The carbon copy of ULV setting. */
-	/* -------------- Stuff originally coming from Evergreen --------------------*/
-	phw_tonga_mc_reg_table			tonga_mc_reg_table;
-	uint32_t                         vdd_ci_control;
-	pp_atomctrl_voltage_table        vddc_voltage_table;
-	pp_atomctrl_voltage_table        vddci_voltage_table;
-	pp_atomctrl_voltage_table        vddgfx_voltage_table;
-	pp_atomctrl_voltage_table        mvdd_voltage_table;
-
-	uint32_t                           mgcg_cgtt_local2;
-	uint32_t                           mgcg_cgtt_local3;
-	uint32_t                           gpio_debug;
-	uint32_t							mc_micro_code_feature;
-	uint32_t							highest_mclk;
-	uint16_t                          acpi_vdd_ci;
-	uint8_t                           mvdd_high_index;
-	uint8_t                           mvdd_low_index;
-	bool                         dll_defaule_on;
-	bool                         performance_request_registered;
-
-	/* ----------------- Low Power Features ---------------------*/
-	phw_tonga_bacos					bacos;
-	phw_tonga_ulv_parm              ulv;
-	/* ----------------- CAC Stuff ---------------------*/
-	uint32_t					cac_table_start;
-	bool                         cac_configuration_required;    /* TRUE if PP_CACConfigurationRequired == 1 */
-	bool                         driver_calculate_cac_leakage;  /* TRUE if PP_DriverCalculateCACLeakage == 1 */
-	bool                         cac_enabled;
-	/* ----------------- DPM2 Parameters ---------------------*/
-	uint32_t					power_containment_features;
-	bool                         enable_bapm_feature;
-	bool                         enable_tdc_limit_feature;
-	bool                         enable_pkg_pwr_tracking_feature;
-	bool                         disable_uvd_power_tune_feature;
-	phw_tonga_pt_defaults           *power_tune_defaults;
-	SMU72_Discrete_PmFuses           power_tune_table;
-	uint32_t                           ul_dte_tj_offset;             /* Fudge factor in DPM table to correct HW DTE errors */
-	uint32_t                           fast_watemark_threshold;      /* use fast watermark if clock is equal or above this. In percentage of the target high sclk. */
-
-	/* ----------------- Phase Shedding ---------------------*/
-	bool                         vddc_phase_shed_control;
-	/* --------------------- DI/DT --------------------------*/
-	phw_tonga_display_timing       display_timing;
-	/* --------- ReadRegistry data for memory and engine clock margins ---- */
-	uint32_t                           engine_clock_data;
-	uint32_t                           memory_clock_data;
-	/* -------- Thermal Temperature Setting --------------*/
-	phw_tonga_dpmlevel_enable_mask     dpm_level_enable_mask;
-	uint32_t                           need_update_smu7_dpm_table;
-	uint32_t                           sclk_dpm_key_disabled;
-	uint32_t                           mclk_dpm_key_disabled;
-	uint32_t                           pcie_dpm_key_disabled;
-	uint32_t                           min_engine_clocks; /* used to store the previous dal min sclock */
-	phw_tonga_pcie_perf_range       pcie_gen_performance;
-	phw_tonga_pcie_perf_range       pcie_lane_performance;
-	phw_tonga_pcie_perf_range       pcie_gen_power_saving;
-	phw_tonga_pcie_perf_range       pcie_lane_power_saving;
-	bool                            use_pcie_performance_levels;
-	bool                            use_pcie_power_saving_levels;
-	uint32_t                           activity_target[SMU72_MAX_LEVELS_GRAPHICS]; /* percentage value from 0-100, default 50 */
-	uint32_t                           mclk_activity_target;
-	uint32_t                           low_sclk_interrupt_threshold;
-	uint32_t                           last_mclk_dpm_enable_mask;
-	bool								uvd_enabled;
-	uint32_t                           pcc_monitor_enabled;
-
-	/* --------- Power Gating States ------------*/
-	bool                           uvd_power_gated;  /* 1: gated, 0:not gated */
-	bool                           vce_power_gated;  /* 1: gated, 0:not gated */
-	bool                           samu_power_gated; /* 1: gated, 0:not gated */
-	bool                           acp_power_gated;  /* 1: gated, 0:not gated */
-	bool                           pg_acp_init;
-};
-
-typedef struct tonga_hwmgr tonga_hwmgr;
-
-#define TONGA_DPM2_NEAR_TDP_DEC          10
-#define TONGA_DPM2_ABOVE_SAFE_INC        5
-#define TONGA_DPM2_BELOW_SAFE_INC        20
-
-#define TONGA_DPM2_LTA_WINDOW_SIZE       7  /* Log2 of the LTA window size (l2numWin_TDP). Eg. If LTA windows size is 128, then this value should be Log2(128) = 7. */
-
-#define TONGA_DPM2_LTS_TRUNCATE          0
-
-#define TONGA_DPM2_TDP_SAFE_LIMIT_PERCENT            80  /* Maximum 100 */
-
-#define TONGA_DPM2_MAXPS_PERCENT_H                   90  /* Maximum 0xFF */
-#define TONGA_DPM2_MAXPS_PERCENT_M                   90  /* Maximum 0xFF */
-
-#define TONGA_DPM2_PWREFFICIENCYRATIO_MARGIN         50
-
-#define TONGA_DPM2_SQ_RAMP_MAX_POWER                 0x3FFF
-#define TONGA_DPM2_SQ_RAMP_MIN_POWER                 0x12
-#define TONGA_DPM2_SQ_RAMP_MAX_POWER_DELTA           0x15
-#define TONGA_DPM2_SQ_RAMP_SHORT_TERM_INTERVAL_SIZE  0x1E
-#define TONGA_DPM2_SQ_RAMP_LONG_TERM_INTERVAL_RATIO  0xF
-
-#define TONGA_VOLTAGE_CONTROL_NONE                   0x0
-#define TONGA_VOLTAGE_CONTROL_BY_GPIO                0x1
-#define TONGA_VOLTAGE_CONTROL_BY_SVID2               0x2
-#define TONGA_VOLTAGE_CONTROL_MERGED                 0x3
-
-#define TONGA_Q88_FORMAT_CONVERSION_UNIT             256 /*To convert to Q8.8 format for firmware */
-
-#define TONGA_UNUSED_GPIO_PIN                        0x7F
-
-int tonga_hwmgr_init(struct pp_hwmgr *hwmgr);
-int tonga_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input);
-int tonga_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate);
-int tonga_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable);
-int tonga_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable);
-uint32_t tonga_get_xclk(struct pp_hwmgr *hwmgr);
-
-#endif
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.c
deleted file mode 100644
index 47ef1ca2d78b..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.c
+++ /dev/null
@@ -1,590 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#include <asm/div64.h>
-#include "tonga_thermal.h"
-#include "tonga_hwmgr.h"
-#include "tonga_smumgr.h"
-#include "tonga_ppsmc.h"
-#include "smu/smu_7_1_2_d.h"
-#include "smu/smu_7_1_2_sh_mask.h"
-
-/**
-* Get Fan Speed Control Parameters.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pSpeed is the address of the structure where the result is to be placed.
-* @exception Always succeeds except if we cannot zero out the output structure.
-*/
-int tonga_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info)
-{
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		return 0;
-
-	fan_speed_info->supports_percent_read = true;
-	fan_speed_info->supports_percent_write = true;
-	fan_speed_info->min_percent = 0;
-	fan_speed_info->max_percent = 100;
-
-	if (0 != hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution) {
-		fan_speed_info->supports_rpm_read = true;
-		fan_speed_info->supports_rpm_write = true;
-		fan_speed_info->min_rpm = hwmgr->thermal_controller.fanInfo.ulMinRPM;
-		fan_speed_info->max_rpm = hwmgr->thermal_controller.fanInfo.ulMaxRPM;
-	} else {
-		fan_speed_info->min_rpm = 0;
-		fan_speed_info->max_rpm = 0;
-	}
-
-	return 0;
-}
-
-/**
-* Get Fan Speed in percent.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pSpeed is the address of the structure where the result is to be placed.
-* @exception Fails is the 100% setting appears to be 0.
-*/
-int tonga_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t *speed)
-{
-	uint32_t duty100;
-	uint32_t duty;
-	uint64_t tmp64;
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		return 0;
-
-	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL1, FMAX_DUTY100);
-	duty = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_STATUS, FDO_PWM_DUTY);
-
-	if (0 == duty100)
-		return -EINVAL;
-
-
-	tmp64 = (uint64_t)duty * 100;
-	do_div(tmp64, duty100);
-	*speed = (uint32_t)tmp64;
-
-	if (*speed > 100)
-		*speed = 100;
-
-	return 0;
-}
-
-/**
-* Get Fan Speed in RPM.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    speed is the address of the structure where the result is to be placed.
-* @exception Returns not supported if no fan is found or if pulses per revolution are not set
-*/
-int tonga_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed)
-{
-	return 0;
-}
-
-/**
-* Set Fan Speed Control to static mode, so that the user can decide what speed to use.
-* @param    hwmgr  the address of the powerplay hardware manager.
-*           mode    the fan control mode, 0 default, 1 by percent, 5, by RPM
-* @exception Should always succeed.
-*/
-int tonga_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
-{
-
-	if (hwmgr->fan_ctrl_is_in_default_mode) {
-		hwmgr->fan_ctrl_default_mode = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, FDO_PWM_MODE);
-		hwmgr->tmin = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, TMIN);
-		hwmgr->fan_ctrl_is_in_default_mode = false;
-	}
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, TMIN, 0);
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, FDO_PWM_MODE, mode);
-
-	return 0;
-}
-
-/**
-* Reset Fan Speed Control to default mode.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @exception Should always succeed.
-*/
-int tonga_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr)
-{
-	if (!hwmgr->fan_ctrl_is_in_default_mode) {
-		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, FDO_PWM_MODE, hwmgr->fan_ctrl_default_mode);
-		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, TMIN, hwmgr->tmin);
-		hwmgr->fan_ctrl_is_in_default_mode = true;
-	}
-
-	return 0;
-}
-
-int tonga_fan_ctrl_start_smc_fan_control(struct pp_hwmgr *hwmgr)
-{
-	int result;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_ODFuzzyFanControlSupport)) {
-		cgs_write_register(hwmgr->device, mmSMC_MSG_ARG_0, FAN_CONTROL_FUZZY);
-		result = (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StartFanControl) == 0) ?  0 : -EINVAL;
-/*
-		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_FanSpeedInTableIsRPM))
-			hwmgr->set_max_fan_rpm_output(hwmgr, hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanRPM);
-		else
-			hwmgr->set_max_fan_pwm_output(hwmgr, hwmgr->thermal_controller.advanceFanControlParameters.usMaxFanPWM);
-*/
-	} else {
-		cgs_write_register(hwmgr->device, mmSMC_MSG_ARG_0, FAN_CONTROL_TABLE);
-		result = (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StartFanControl) == 0) ?  0 : -EINVAL;
-	}
-/* TO DO FOR SOME DEVICE ID 0X692b, send this msg return invalid command.
-	if (result == 0 && hwmgr->thermal_controller.advanceFanControlParameters.ucTargetTemperature != 0)
-		result = (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanTemperatureTarget, \
-								hwmgr->thermal_controller.advanceFanControlParameters.ucTargetTemperature) ? 0 : -EINVAL);
-*/
-	return result;
-}
-
-
-int tonga_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr)
-{
-	return (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_StopFanControl) == 0) ?  0 : -EINVAL;
-}
-
-/**
-* Set Fan Speed in percent.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    speed is the percentage value (0% - 100%) to be set.
-* @exception Fails is the 100% setting appears to be 0.
-*/
-int tonga_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed)
-{
-	uint32_t duty100;
-	uint32_t duty;
-	uint64_t tmp64;
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		return -EINVAL;
-
-	if (speed > 100)
-		speed = 100;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl))
-		tonga_fan_ctrl_stop_smc_fan_control(hwmgr);
-
-	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL1, FMAX_DUTY100);
-
-	if (0 == duty100)
-		return -EINVAL;
-
-	tmp64 = (uint64_t)speed * duty100;
-	do_div(tmp64, 100);
-	duty = (uint32_t)tmp64;
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL0, FDO_STATIC_DUTY, duty);
-
-	return tonga_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
-}
-
-/**
-* Reset Fan Speed to default.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @exception Always succeeds.
-*/
-int tonga_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
-{
-	int result;
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		return 0;
-
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl)) {
-		result = tonga_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
-		if (0 == result)
-			result = tonga_fan_ctrl_start_smc_fan_control(hwmgr);
-	} else
-		result = tonga_fan_ctrl_set_default_mode(hwmgr);
-
-	return result;
-}
-
-/**
-* Set Fan Speed in RPM.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    speed is the percentage value (min - max) to be set.
-* @exception Fails is the speed not lie between min and max.
-*/
-int tonga_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed)
-{
-	return 0;
-}
-
-/**
-* Reads the remote temperature from the SIslands thermal controller.
-*
-* @param    hwmgr The address of the hardware manager.
-*/
-int tonga_thermal_get_temperature(struct pp_hwmgr *hwmgr)
-{
-	int temp;
-
-	temp = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_MULT_THERMAL_STATUS, CTF_TEMP);
-
-/* Bit 9 means the reading is lower than the lowest usable value. */
-	if (0 != (0x200 & temp))
-		temp = TONGA_THERMAL_MAXIMUM_TEMP_READING;
-	else
-		temp = (temp & 0x1ff);
-
-	temp = temp * PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
-
-	return temp;
-}
-
-/**
-* Set the requested temperature range for high and low alert signals
-*
-* @param    hwmgr The address of the hardware manager.
-* @param    range Temperature range to be programmed for high and low alert signals
-* @exception PP_Result_BadInput if the input data is not valid.
-*/
-static int tonga_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, uint32_t low_temp, uint32_t high_temp)
-{
-	uint32_t low = TONGA_THERMAL_MINIMUM_ALERT_TEMP * PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
-	uint32_t high = TONGA_THERMAL_MAXIMUM_ALERT_TEMP * PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
-
-	if (low < low_temp)
-		low = low_temp;
-	if (high > high_temp)
-		high = high_temp;
-
-	if (low > high)
-		return -EINVAL;
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_CTRL, DIG_THERM_DPM, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
-
-	return 0;
-}
-
-/**
-* Programs thermal controller one-time setting registers
-*
-* @param    hwmgr The address of the hardware manager.
-*/
-static int tonga_thermal_initialize(struct pp_hwmgr *hwmgr)
-{
-	if (0 != hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution)
-		PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
-						CG_TACH_CTRL, EDGE_PER_REV,
-						hwmgr->thermal_controller.fanInfo.ucTachometerPulsesPerRevolution - 1);
-
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL2, TACH_PWM_RESP_RATE, 0x28);
-
-	return 0;
-}
-
-/**
-* Enable thermal alerts on the RV770 thermal controller.
-*
-* @param    hwmgr The address of the hardware manager.
-*/
-static int tonga_thermal_enable_alert(struct pp_hwmgr *hwmgr)
-{
-	uint32_t alert;
-
-	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, THERM_INT_MASK);
-	alert &= ~(TONGA_THERMAL_HIGH_ALERT_MASK | TONGA_THERMAL_LOW_ALERT_MASK);
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, THERM_INT_MASK, alert);
-
-	/* send message to SMU to enable internal thermal interrupts */
-	return (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Thermal_Cntl_Enable) == 0) ? 0 : -1;
-}
-
-/**
-* Disable thermal alerts on the RV770 thermal controller.
-* @param    hwmgr The address of the hardware manager.
-*/
-static int tonga_thermal_disable_alert(struct pp_hwmgr *hwmgr)
-{
-	uint32_t alert;
-
-	alert = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, THERM_INT_MASK);
-	alert |= (TONGA_THERMAL_HIGH_ALERT_MASK | TONGA_THERMAL_LOW_ALERT_MASK);
-	PHM_WRITE_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_THERMAL_INT, THERM_INT_MASK, alert);
-
-	/* send message to SMU to disable internal thermal interrupts */
-	return (smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Thermal_Cntl_Disable) == 0) ? 0 : -1;
-}
-
-/**
-* Uninitialize the thermal controller.
-* Currently just disables alerts.
-* @param    hwmgr The address of the hardware manager.
-*/
-int tonga_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr)
-{
-	int result = tonga_thermal_disable_alert(hwmgr);
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		tonga_fan_ctrl_set_default_mode(hwmgr);
-
-	return result;
-}
-
-/**
-* Set up the fan table to control the fan using the SMC.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from set temperature range routine
-*/
-int tf_tonga_thermal_setup_fan_table(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
-{
-	struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-	SMU72_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
-	uint32_t duty100;
-	uint32_t t_diff1, t_diff2, pwm_diff1, pwm_diff2;
-	uint16_t fdo_min, slope1, slope2;
-	uint32_t reference_clock;
-	int res;
-	uint64_t tmp64;
-
-	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl))
-		return 0;
-
-	if (0 == data->fan_table_start) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl);
-		return 0;
-	}
-
-	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL1, FMAX_DUTY100);
-
-	if (0 == duty100) {
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl);
-		return 0;
-	}
-
-	tmp64 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin * duty100;
-	do_div(tmp64, 10000);
-	fdo_min = (uint16_t)tmp64;
-
-	t_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usTMed - hwmgr->thermal_controller.advanceFanControlParameters.usTMin;
-	t_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usTHigh - hwmgr->thermal_controller.advanceFanControlParameters.usTMed;
-
-	pwm_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed - hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin;
-	pwm_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh - hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed;
-
-	slope1 = (uint16_t)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
-	slope2 = (uint16_t)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
-
-	fan_table.TempMin = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMin) / 100);
-	fan_table.TempMed = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMed) / 100);
-	fan_table.TempMax = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMax) / 100);
-
-	fan_table.Slope1 = cpu_to_be16(slope1);
-	fan_table.Slope2 = cpu_to_be16(slope2);
-
-	fan_table.FdoMin = cpu_to_be16(fdo_min);
-
-	fan_table.HystDown = cpu_to_be16(hwmgr->thermal_controller.advanceFanControlParameters.ucTHyst);
-
-	fan_table.HystUp = cpu_to_be16(1);
-
-	fan_table.HystSlope = cpu_to_be16(1);
-
-	fan_table.TempRespLim = cpu_to_be16(5);
-
-	reference_clock = tonga_get_xclk(hwmgr);
-
-	fan_table.RefreshPeriod = cpu_to_be32((hwmgr->thermal_controller.advanceFanControlParameters.ulCycleDelay * reference_clock) / 1600);
-
-	fan_table.FdoMax = cpu_to_be16((uint16_t)duty100);
-
-	fan_table.TempSrc = (uint8_t)PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_MULT_THERMAL_CTRL, TEMP_SEL);
-
-	fan_table.FanControl_GL_Flag = 1;
-
-	res = tonga_copy_bytes_to_smc(hwmgr->smumgr, data->fan_table_start, (uint8_t *)&fan_table, (uint32_t)sizeof(fan_table), data->sram_end);
-/* TO DO FOR SOME DEVICE ID 0X692b, send this msg return invalid command.
-	if (res == 0 && hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit != 0)
-		res = (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanMinPwm, \
-						hwmgr->thermal_controller.advanceFanControlParameters.ucMinimumPWMLimit) ? 0 : -1);
-
-	if (res == 0 && hwmgr->thermal_controller.advanceFanControlParameters.ulMinFanSCLKAcousticLimit != 0)
-		res = (0 == smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_SetFanSclkTarget, \
-					hwmgr->thermal_controller.advanceFanControlParameters.ulMinFanSCLKAcousticLimit) ? 0 : -1);
-
-	if (0 != res)
-		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl);
-*/
-	return 0;
-}
-
-/**
-* Start the fan control on the SMC.
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from set temperature range routine
-*/
-int tf_tonga_thermal_start_smc_fan_control(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
-{
-/* If the fantable setup has failed we could have disabled PHM_PlatformCaps_MicrocodeFanControl even after this function was included in the table.
- * Make sure that we still think controlling the fan is OK.
-*/
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl)) {
-		tonga_fan_ctrl_start_smc_fan_control(hwmgr);
-		tonga_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
-	}
-
-	return 0;
-}
-
-/**
-* Set temperature range for high and low alerts
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from set temperature range routine
-*/
-int tf_tonga_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
-{
-	struct PP_TemperatureRange *range = (struct PP_TemperatureRange *)input;
-
-	if (range == NULL)
-		return -EINVAL;
-
-	return tonga_thermal_set_temperature_range(hwmgr, range->min, range->max);
-}
-
-/**
-* Programs one-time setting registers
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from initialize thermal controller routine
-*/
-int tf_tonga_thermal_initialize(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
-{
-    return tonga_thermal_initialize(hwmgr);
-}
-
-/**
-* Enable high and low alerts
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from enable alert routine
-*/
-int tf_tonga_thermal_enable_alert(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
-{
-	return tonga_thermal_enable_alert(hwmgr);
-}
-
-/**
-* Disable high and low alerts
-* @param    hwmgr  the address of the powerplay hardware manager.
-* @param    pInput the pointer to input data
-* @param    pOutput the pointer to output data
-* @param    pStorage the pointer to temporary storage
-* @param    Result the last failure code
-* @return   result from disable alert routine
-*/
-static int tf_tonga_thermal_disable_alert(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result)
-{
-	return tonga_thermal_disable_alert(hwmgr);
-}
-
-static const struct phm_master_table_item tonga_thermal_start_thermal_controller_master_list[] = {
-	{ NULL, tf_tonga_thermal_initialize },
-	{ NULL, tf_tonga_thermal_set_temperature_range },
-	{ NULL, tf_tonga_thermal_enable_alert },
-/* We should restrict performance levels to low before we halt the SMC.
- * On the other hand we are still in boot state when we do this so it would be pointless.
- * If this assumption changes we have to revisit this table.
- */
-	{ NULL, tf_tonga_thermal_setup_fan_table},
-	{ NULL, tf_tonga_thermal_start_smc_fan_control},
-	{ NULL, NULL }
-};
-
-static const struct phm_master_table_header tonga_thermal_start_thermal_controller_master = {
-	0,
-	PHM_MasterTableFlag_None,
-	tonga_thermal_start_thermal_controller_master_list
-};
-
-static const struct phm_master_table_item tonga_thermal_set_temperature_range_master_list[] = {
-	{ NULL, tf_tonga_thermal_disable_alert},
-	{ NULL, tf_tonga_thermal_set_temperature_range},
-	{ NULL, tf_tonga_thermal_enable_alert},
-	{ NULL, NULL }
-};
-
-static const struct phm_master_table_header tonga_thermal_set_temperature_range_master = {
-	0,
-	PHM_MasterTableFlag_None,
-	tonga_thermal_set_temperature_range_master_list
-};
-
-int tonga_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr)
-{
-	if (!hwmgr->thermal_controller.fanInfo.bNoFan)
-		tonga_fan_ctrl_set_default_mode(hwmgr);
-	return 0;
-}
-
-/**
-* Initializes the thermal controller related functions in the Hardware Manager structure.
-* @param    hwmgr The address of the hardware manager.
-* @exception Any error code from the low-level communication.
-*/
-int pp_tonga_thermal_initialize(struct pp_hwmgr *hwmgr)
-{
-	int result;
-
-	result = phm_construct_table(hwmgr, &tonga_thermal_set_temperature_range_master, &(hwmgr->set_temperature_range));
-
-	if (0 == result) {
-		result = phm_construct_table(hwmgr,
-						&tonga_thermal_start_thermal_controller_master,
-						&(hwmgr->start_thermal_controller));
-		if (0 != result)
-			phm_destroy_table(hwmgr, &(hwmgr->set_temperature_range));
-	}
-
-	if (0 == result)
-		hwmgr->fan_ctrl_is_in_default_mode = true;
-	return result;
-}
-
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.h
deleted file mode 100644
index aa335f267e25..000000000000
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_thermal.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef TONGA_THERMAL_H
-#define TONGA_THERMAL_H
-
-#include "hwmgr.h"
-
-#define TONGA_THERMAL_HIGH_ALERT_MASK         0x1
-#define TONGA_THERMAL_LOW_ALERT_MASK          0x2
-
-#define TONGA_THERMAL_MINIMUM_TEMP_READING    -256
-#define TONGA_THERMAL_MAXIMUM_TEMP_READING    255
-
-#define TONGA_THERMAL_MINIMUM_ALERT_TEMP      0
-#define TONGA_THERMAL_MAXIMUM_ALERT_TEMP      255
-
-#define FDO_PWM_MODE_STATIC  1
-#define FDO_PWM_MODE_STATIC_RPM 5
-
-
-extern int tf_tonga_thermal_initialize(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
-extern int tf_tonga_thermal_set_temperature_range(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
-extern int tf_tonga_thermal_enable_alert(struct pp_hwmgr *hwmgr, void *input, void *output, void *storage, int result);
-
-extern int tonga_thermal_get_temperature(struct pp_hwmgr *hwmgr);
-extern int tonga_thermal_stop_thermal_controller(struct pp_hwmgr *hwmgr);
-extern int tonga_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, struct phm_fan_speed_info *fan_speed_info);
-extern int tonga_fan_ctrl_get_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t *speed);
-extern int tonga_fan_ctrl_set_default_mode(struct pp_hwmgr *hwmgr);
-extern int tonga_fan_ctrl_set_static_mode(struct pp_hwmgr *hwmgr, uint32_t mode);
-extern int tonga_fan_ctrl_set_fan_speed_percent(struct pp_hwmgr *hwmgr, uint32_t speed);
-extern int tonga_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr);
-extern int pp_tonga_thermal_initialize(struct pp_hwmgr *hwmgr);
-extern int tonga_thermal_ctrl_uninitialize_thermal_controller(struct pp_hwmgr *hwmgr);
-extern int tonga_fan_ctrl_set_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t speed);
-extern int tonga_fan_ctrl_get_fan_speed_rpm(struct pp_hwmgr *hwmgr, uint32_t *speed);
-extern int tonga_fan_ctrl_stop_smc_fan_control(struct pp_hwmgr *hwmgr);
-
-#endif
-
diff --git a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
index b764c8c05ec8..3fb5e57a378b 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
@@ -29,6 +29,19 @@
 #include "amd_shared.h"
 #include "cgs_common.h"
 
+enum amd_pp_sensors {
+	AMDGPU_PP_SENSOR_GFX_SCLK = 0,
+	AMDGPU_PP_SENSOR_VDDNB,
+	AMDGPU_PP_SENSOR_VDDGFX,
+	AMDGPU_PP_SENSOR_UVD_VCLK,
+	AMDGPU_PP_SENSOR_UVD_DCLK,
+	AMDGPU_PP_SENSOR_VCE_ECCLK,
+	AMDGPU_PP_SENSOR_GPU_LOAD,
+	AMDGPU_PP_SENSOR_GFX_MCLK,
+	AMDGPU_PP_SENSOR_GPU_TEMP,
+	AMDGPU_PP_SENSOR_VCE_POWER,
+	AMDGPU_PP_SENSOR_UVD_POWER,
+};
 
 enum amd_pp_event {
 	AMD_PP_EVENT_INITIALIZE = 0,
@@ -131,9 +144,8 @@ struct amd_pp_init {
 	struct cgs_device *device;
 	uint32_t chip_family;
 	uint32_t chip_id;
-	uint32_t rev_id;
-	bool powercontainment_enabled;
 };
+
 enum amd_pp_display_config_type{
 	AMD_PP_DisplayConfigType_None = 0,
 	AMD_PP_DisplayConfigType_DP54 ,
@@ -261,6 +273,7 @@ enum amd_pp_clock_type {
 struct amd_pp_clocks {
 	uint32_t count;
 	uint32_t clock[MAX_NUM_CLOCKS];
+	uint32_t latency[MAX_NUM_CLOCKS];
 };
 
 
@@ -332,8 +345,6 @@ struct amd_powerplay_funcs {
 	int (*powergate_uvd)(void *handle, bool gate);
 	int (*dispatch_tasks)(void *handle, enum amd_pp_event event_id,
 				   void *input, void *output);
-	void (*print_current_performance_level)(void *handle,
-						      struct seq_file *m);
 	int (*set_fan_control_mode)(void *handle, uint32_t mode);
 	int (*get_fan_control_mode)(void *handle);
 	int (*set_fan_speed_percent)(void *handle, uint32_t percent);
@@ -347,6 +358,7 @@ struct amd_powerplay_funcs {
 	int (*set_sclk_od)(void *handle, uint32_t value);
 	int (*get_mclk_od)(void *handle);
 	int (*set_mclk_od)(void *handle, uint32_t value);
+	int (*read_sensor)(void *handle, int idx, int32_t *value);
 };
 
 struct amd_powerplay {
@@ -378,4 +390,6 @@ int amd_powerplay_get_clock_by_type(void *handle,
 int amd_powerplay_get_display_mode_validation_clocks(void *handle,
 		struct amd_pp_simple_clock_info *output);
 
+int amd_set_clockgating_by_smu(void *handle, uint32_t msg_id);
+
 #endif /* _AMD_POWERPLAY_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
index 962cb5385951..d4495839c64c 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
@@ -341,7 +341,6 @@ extern int phm_powerdown_uvd(struct pp_hwmgr *hwmgr);
 extern int phm_setup_asic(struct pp_hwmgr *hwmgr);
 extern int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr);
 extern int phm_disable_dynamic_state_management(struct pp_hwmgr *hwmgr);
-extern void phm_init_dynamic_caps(struct pp_hwmgr *hwmgr);
 extern bool phm_is_hw_access_blocked(struct pp_hwmgr *hwmgr);
 extern int phm_block_hw_access(struct pp_hwmgr *hwmgr, bool block);
 extern int phm_set_power_state(struct pp_hwmgr *hwmgr,
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
index bf0d2accf7bf..4f0fedd1e9d3 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
@@ -31,15 +31,20 @@
 #include "hwmgr_ppt.h"
 #include "ppatomctrl.h"
 #include "hwmgr_ppt.h"
+#include "power_state.h"
 
 struct pp_instance;
 struct pp_hwmgr;
-struct pp_hw_power_state;
-struct pp_power_state;
-struct PP_VCEState;
 struct phm_fan_speed_info;
 struct pp_atomctrl_voltage_table;
 
+extern int amdgpu_powercontainment;
+extern int amdgpu_sclk_deep_sleep_en;
+extern unsigned amdgpu_pp_feature_mask;
+
+#define VOLTAGE_SCALE 4
+
+uint8_t convert_to_vid(uint16_t vddc);
 
 enum DISPLAY_GAP {
 	DISPLAY_GAP_VBLANK_OR_WM = 0,   /* Wait for vblank or MCHG watermark. */
@@ -49,7 +54,6 @@ enum DISPLAY_GAP {
 };
 typedef enum DISPLAY_GAP DISPLAY_GAP;
 
-
 struct vi_dpm_level {
 	bool enabled;
 	uint32_t value;
@@ -71,6 +75,19 @@ enum PP_Result {
 #define PCIE_PERF_REQ_GEN2         3
 #define PCIE_PERF_REQ_GEN3         4
 
+enum PP_FEATURE_MASK {
+	PP_SCLK_DPM_MASK = 0x1,
+	PP_MCLK_DPM_MASK = 0x2,
+	PP_PCIE_DPM_MASK = 0x4,
+	PP_SCLK_DEEP_SLEEP_MASK = 0x8,
+	PP_POWER_CONTAINMENT_MASK = 0x10,
+	PP_UVD_HANDSHAKE_MASK = 0x20,
+	PP_SMC_VOLTAGE_CONTROL_MASK = 0x40,
+	PP_VBI_TIME_SUPPORT_MASK = 0x80,
+	PP_ULV_MASK = 0x100,
+	PP_ENABLE_GFX_CG_THRU_SMU = 0x200
+};
+
 enum PHM_BackEnd_Magic {
 	PHM_Dummy_Magic       = 0xAA5555AA,
 	PHM_RV770_Magic       = 0xDCBAABCD,
@@ -294,8 +311,6 @@ struct pp_hwmgr_func {
 	int (*get_sclk)(struct pp_hwmgr *hwmgr, bool low);
 	int (*power_state_set)(struct pp_hwmgr *hwmgr,
 						const void *state);
-	void (*print_current_perforce_level)(struct pp_hwmgr *hwmgr,
-							struct seq_file *m);
 	int (*enable_clock_power_gating)(struct pp_hwmgr *hwmgr);
 	int (*notify_smc_display_config_after_ps_adjustment)(struct pp_hwmgr *hwmgr);
 	int (*display_config_changed)(struct pp_hwmgr *hwmgr);
@@ -342,6 +357,7 @@ struct pp_hwmgr_func {
 	int (*set_sclk_od)(struct pp_hwmgr *hwmgr, uint32_t value);
 	int (*get_mclk_od)(struct pp_hwmgr *hwmgr);
 	int (*set_mclk_od)(struct pp_hwmgr *hwmgr, uint32_t value);
+	int (*read_sensor)(struct pp_hwmgr *hwmgr, int idx, int32_t *value);
 };
 
 struct pp_table_func {
@@ -351,7 +367,7 @@ struct pp_table_func {
 	int (*pptable_get_vce_state_table_entry)(
 						struct pp_hwmgr *hwmgr,
 						unsigned long i,
-						struct PP_VCEState *vce_state,
+						struct pp_vce_state *vce_state,
 						void **clock_info,
 						unsigned long *flag);
 };
@@ -570,22 +586,43 @@ struct phm_microcode_version_info {
 	uint32_t NB;
 };
 
+#define PP_MAX_VCE_LEVELS 6
+
+enum PP_VCE_LEVEL {
+	PP_VCE_LEVEL_AC_ALL = 0,     /* AC, All cases */
+	PP_VCE_LEVEL_DC_EE = 1,      /* DC, entropy encoding */
+	PP_VCE_LEVEL_DC_LL_LOW = 2,  /* DC, low latency queue, res <= 720 */
+	PP_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */
+	PP_VCE_LEVEL_DC_GP_LOW = 4,  /* DC, general purpose queue, res <= 720 */
+	PP_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */
+};
+
+
+enum PP_TABLE_VERSION {
+	PP_TABLE_V0 = 0,
+	PP_TABLE_V1,
+	PP_TABLE_V2,
+	PP_TABLE_MAX
+};
+
 /**
  * The main hardware manager structure.
  */
 struct pp_hwmgr {
 	uint32_t chip_family;
 	uint32_t chip_id;
-	uint32_t hw_revision;
-	uint32_t sub_sys_id;
-	uint32_t sub_vendor_id;
 
+	uint32_t pp_table_version;
 	void *device;
 	struct pp_smumgr *smumgr;
 	const void *soft_pp_table;
 	uint32_t soft_pp_table_size;
 	void *hardcode_pp_table;
 	bool need_pp_table_upload;
+
+	struct pp_vce_state vce_states[PP_MAX_VCE_LEVELS];
+	uint32_t num_vce_state_tables;
+
 	enum amd_dpm_forced_level dpm_level;
 	bool block_hw_access;
 	struct phm_gfx_arbiter gfx_arbiter;
@@ -614,7 +651,6 @@ struct pp_hwmgr {
 	uint32_t num_ps;
 	struct pp_thermal_controller_info thermal_controller;
 	bool fan_ctrl_is_in_default_mode;
-	bool powercontainment_enabled;
 	uint32_t fan_ctrl_default_mode;
 	uint32_t tmin;
 	struct phm_microcode_version_info microcode_version_info;
@@ -624,6 +660,7 @@ struct pp_hwmgr {
 	struct pp_power_state    *boot_ps;
 	struct pp_power_state    *uvd_ps;
 	struct amd_pp_display_configuration display_config;
+	uint32_t feature_mask;
 };
 
 
@@ -637,16 +674,7 @@ extern int hw_init_power_state_table(struct pp_hwmgr *hwmgr);
 extern int phm_wait_on_register(struct pp_hwmgr *hwmgr, uint32_t index,
 				uint32_t value, uint32_t mask);
 
-extern int phm_wait_for_register_unequal(struct pp_hwmgr *hwmgr,
-				uint32_t index, uint32_t value, uint32_t mask);
 
-extern uint32_t phm_read_indirect_register(struct pp_hwmgr *hwmgr,
-		uint32_t indirect_port, uint32_t index);
-
-extern void phm_write_indirect_register(struct pp_hwmgr *hwmgr,
-		uint32_t indirect_port,
-		uint32_t index,
-		uint32_t value);
 
 extern void phm_wait_on_indirect_register(struct pp_hwmgr *hwmgr,
 				uint32_t indirect_port,
@@ -654,12 +682,7 @@ extern void phm_wait_on_indirect_register(struct pp_hwmgr *hwmgr,
 				uint32_t value,
 				uint32_t mask);
 
-extern void phm_wait_for_indirect_register_unequal(
-				struct pp_hwmgr *hwmgr,
-				uint32_t indirect_port,
-				uint32_t index,
-				uint32_t value,
-				uint32_t mask);
+
 
 extern bool phm_cf_want_uvd_power_gating(struct pp_hwmgr *hwmgr);
 extern bool phm_cf_want_vce_power_gating(struct pp_hwmgr *hwmgr);
@@ -673,6 +696,8 @@ extern void phm_trim_voltage_table_to_fit_state_table(uint32_t max_vol_steps, st
 extern int phm_reset_single_dpm_table(void *table, uint32_t count, int max);
 extern void phm_setup_pcie_table_entry(void *table, uint32_t index, uint32_t pcie_gen, uint32_t pcie_lanes);
 extern int32_t phm_get_dpm_level_enable_mask_value(void *table);
+extern uint8_t phm_get_voltage_id(struct pp_atomctrl_voltage_table *voltage_table,
+		uint32_t voltage);
 extern uint8_t phm_get_voltage_index(struct phm_ppt_v1_voltage_lookup_table *lookup_table, uint16_t voltage);
 extern uint16_t phm_find_closest_vddci(struct pp_atomctrl_voltage_table *vddci_table, uint16_t vddci);
 extern int phm_find_boot_level(void *table, uint32_t value, uint32_t *boot_level);
@@ -683,6 +708,10 @@ extern int phm_hwmgr_backend_fini(struct pp_hwmgr *hwmgr);
 extern uint32_t phm_get_lowest_enabled_level(struct pp_hwmgr *hwmgr, uint32_t mask);
 extern void phm_apply_dal_min_voltage_request(struct pp_hwmgr *hwmgr);
 
+extern int smu7_hwmgr_init(struct pp_hwmgr *hwmgr);
+extern int phm_get_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type,
+				uint32_t sclk, uint16_t id, uint16_t *voltage);
+
 #define PHM_ENTIRE_REGISTER_MASK 0xFFFFFFFFU
 
 #define PHM_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
@@ -697,44 +726,6 @@ extern void phm_apply_dal_min_voltage_request(struct pp_hwmgr *hwmgr);
 	 PHM_FIELD_SHIFT(reg, field))
 
 
-#define PHM_WAIT_REGISTER_GIVEN_INDEX(hwmgr, index, value, mask)	\
-	phm_wait_on_register(hwmgr, index, value, mask)
-
-#define PHM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, index, value, mask)	\
-	phm_wait_for_register_unequal(hwmgr, index, value, mask)
-
-#define PHM_WAIT_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, index, value, mask)	\
-	phm_wait_on_indirect_register(hwmgr, mm##port##_INDEX, index, value, mask)
-
-#define PHM_WAIT_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, port, index, value, mask)	\
-	phm_wait_for_indirect_register_unequal(hwmgr, mm##port##_INDEX, index, value, mask)
-
-#define PHM_WAIT_VFPF_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, index, value, mask)	\
-	phm_wait_on_indirect_register(hwmgr, mm##port##_INDEX_0, index, value, mask)
-
-#define PHM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, port, index, value, mask)	\
-	phm_wait_for_indirect_register_unequal(hwmgr, mm##port##_INDEX_0, index, value, mask)
-
-/* Operations on named registers. */
-
-#define PHM_WAIT_REGISTER(hwmgr, reg, value, mask)	\
-	PHM_WAIT_REGISTER_GIVEN_INDEX(hwmgr, mm##reg, value, mask)
-
-#define PHM_WAIT_REGISTER_UNEQUAL(hwmgr, reg, value, mask)	\
-	PHM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, mm##reg, value, mask)
-
-#define PHM_WAIT_INDIRECT_REGISTER(hwmgr, port, reg, value, mask)	\
-	PHM_WAIT_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, ix##reg, value, mask)
-
-#define PHM_WAIT_INDIRECT_REGISTER_UNEQUAL(hwmgr, port, reg, value, mask)	\
-	PHM_WAIT_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, port, ix##reg, value, mask)
-
-#define PHM_WAIT_VFPF_INDIRECT_REGISTER(hwmgr, port, reg, value, mask)	\
-	PHM_WAIT_VFPF_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, ix##reg, value, mask)
-
-#define PHM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL(hwmgr, port, reg, value, mask)	\
-	PHM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, port, ix##reg, value, mask)
-
 /* Operations on named fields. */
 
 #define PHM_READ_FIELD(device, reg, field)	\
@@ -762,60 +753,16 @@ extern void phm_apply_dal_min_voltage_request(struct pp_hwmgr *hwmgr);
 			PHM_SET_FIELD(cgs_read_ind_register(device, port, ix##reg),	\
 				reg, field, fieldval))
 
-#define PHM_WAIT_FIELD(hwmgr, reg, field, fieldval)	\
-	PHM_WAIT_REGISTER(hwmgr, reg, (fieldval)	\
-			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
-
-#define PHM_WAIT_INDIRECT_FIELD(hwmgr, port, reg, field, fieldval)	\
-	PHM_WAIT_INDIRECT_REGISTER(hwmgr, port, reg, (fieldval)	\
-			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
-
-#define PHM_WAIT_VFPF_INDIRECT_FIELD(hwmgr, port, reg, field, fieldval)	\
-	PHM_WAIT_VFPF_INDIRECT_REGISTER(hwmgr, port, reg, (fieldval)	\
-			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
+#define PHM_WAIT_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, index, value, mask)        \
+       phm_wait_on_indirect_register(hwmgr, mm##port##_INDEX, index, value, mask)
 
-#define PHM_WAIT_FIELD_UNEQUAL(hwmgr, reg, field, fieldval)	\
-	PHM_WAIT_REGISTER_UNEQUAL(hwmgr, reg, (fieldval)	\
-			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
 
-#define PHM_WAIT_INDIRECT_FIELD_UNEQUAL(hwmgr, port, reg, field, fieldval)	\
-	PHM_WAIT_INDIRECT_REGISTER_UNEQUAL(hwmgr, port, reg, (fieldval)	\
-			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
+#define PHM_WAIT_INDIRECT_REGISTER(hwmgr, port, reg, value, mask)      \
+       PHM_WAIT_INDIRECT_REGISTER_GIVEN_INDEX(hwmgr, port, ix##reg, value, mask)
 
-#define PHM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(hwmgr, port, reg, field, fieldval)	\
-	PHM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL(hwmgr, port, reg, (fieldval)	\
+#define PHM_WAIT_INDIRECT_FIELD(hwmgr, port, reg, field, fieldval)	\
+	PHM_WAIT_INDIRECT_REGISTER(hwmgr, port, reg, (fieldval)	\
 			<< PHM_FIELD_SHIFT(reg, field), PHM_FIELD_MASK(reg, field))
 
-/* Operations on arrays of registers & fields. */
-
-#define PHM_READ_ARRAY_REGISTER(device, reg, offset)	\
-	cgs_read_register(device, mm##reg + (offset))
-
-#define PHM_WRITE_ARRAY_REGISTER(device, reg, offset, value)	\
-	cgs_write_register(device, mm##reg + (offset), value)
-
-#define PHM_WAIT_ARRAY_REGISTER(hwmgr, reg, offset, value, mask)	\
-	PHM_WAIT_REGISTER_GIVEN_INDEX(hwmgr, mm##reg + (offset), value, mask)
-
-#define PHM_WAIT_ARRAY_REGISTER_UNEQUAL(hwmgr, reg, offset, value, mask)	\
-	PHM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, mm##reg + (offset), value, mask)
-
-#define PHM_READ_ARRAY_FIELD(hwmgr, reg, offset, field) \
-	PHM_GET_FIELD(PHM_READ_ARRAY_REGISTER(hwmgr->device, reg, offset), reg, field)
-
-#define PHM_WRITE_ARRAY_FIELD(hwmgr, reg, offset, field, fieldvalue)	\
-	PHM_WRITE_ARRAY_REGISTER(hwmgr->device, reg, offset,	\
-			PHM_SET_FIELD(PHM_READ_ARRAY_REGISTER(hwmgr->device, reg, offset),	\
-				reg, field, fieldvalue))
-
-#define PHM_WAIT_ARRAY_FIELD(hwmgr, reg, offset, field, fieldvalue)	\
-	PHM_WAIT_REGISTER_GIVEN_INDEX(hwmgr, mm##reg + (offset),	\
-			(fieldvalue) << PHM_FIELD_SHIFT(reg, field),	\
-			PHM_FIELD_MASK(reg, field))
-
-#define PHM_WAIT_ARRAY_FIELD_UNEQUAL(hwmgr, reg, offset, field, fieldvalue)	\
-	PHM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(hwmgr, mm##reg + (offset),	\
-			(fieldvalue) << PHM_FIELD_SHIFT(reg, field),	\
-			PHM_FIELD_MASK(reg, field))
 
 #endif /* _HWMGR_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/polaris10_pwrvirus.h b/drivers/gpu/drm/amd/powerplay/inc/polaris10_pwrvirus.h
index f497e7d98e6d..0de443612312 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/polaris10_pwrvirus.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/polaris10_pwrvirus.h
@@ -23,8 +23,7 @@
 #ifndef _POLARIS10_PWRVIRUS_H
 #define _POLARIS10_PWRVIRUS_H
 
-#define mmSMC_IND_INDEX_11                              0x01AC
-#define mmSMC_IND_DATA_11                               0x01AD
+
 #define mmCP_HYP_MEC1_UCODE_ADDR	0xf81a
 #define mmCP_HYP_MEC1_UCODE_DATA	0xf81b
 #define mmCP_HYP_MEC2_UCODE_ADDR	0xf81c
diff --git a/drivers/gpu/drm/amd/powerplay/inc/power_state.h b/drivers/gpu/drm/amd/powerplay/inc/power_state.h
index a3f0ce4d5835..9ceaed9ac52a 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/power_state.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/power_state.h
@@ -158,7 +158,7 @@ struct pp_power_state {
 
 
 /*Structure to hold a VCE state entry*/
-struct PP_VCEState {
+struct pp_vce_state {
 	uint32_t evclk;
 	uint32_t ecclk;
 	uint32_t sclk;
@@ -171,30 +171,28 @@ enum PP_MMProfilingState {
 	PP_MMProfilingState_Stopped
 };
 
-struct PP_Clock_Engine_Request {
-	unsigned long clientType;
-	unsigned long ctxid;
+struct pp_clock_engine_request {
+	unsigned long client_type;
+	unsigned long ctx_id;
 	uint64_t  context_handle;
 	unsigned long sclk;
-	unsigned long sclkHardMin;
+	unsigned long sclk_hard_min;
 	unsigned long mclk;
 	unsigned long iclk;
 	unsigned long evclk;
 	unsigned long ecclk;
-	unsigned long ecclkHardMin;
+	unsigned long ecclk_hard_min;
 	unsigned long vclk;
 	unsigned long dclk;
-	unsigned long samclk;
-	unsigned long acpclk;
-	unsigned long sclkOverdrive;
-	unsigned long mclkOverdrive;
+	unsigned long sclk_over_drive;
+	unsigned long mclk_over_drive;
 	unsigned long sclk_threshold;
 	unsigned long flag;
 	unsigned long vclk_ceiling;
 	unsigned long dclk_ceiling;
 	unsigned long num_cus;
-	unsigned long pmflag;
-	enum PP_MMProfilingState MMProfilingState;
+	unsigned long pm_flag;
+	enum PP_MMProfilingState mm_profiling_state;
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h b/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
index d7d83b7c7f95..bfdbec10cdd5 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
@@ -43,5 +43,8 @@
 	} while (0)
 
 
+#define GET_FLEXIBLE_ARRAY_MEMBER_ADDR(type, member, ptr, n)	\
+	(type *)((char *)&(ptr)->member + (sizeof(type) * (n)))
+
 #endif /* PP_DEBUG_H */
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu71.h b/drivers/gpu/drm/amd/powerplay/inc/smu71.h
new file mode 100644
index 000000000000..71c9b2d28640
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu71.h
@@ -0,0 +1,510 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef SMU71_H
+#define SMU71_H
+
+#if !defined(SMC_MICROCODE)
+#pragma pack(push, 1)
+#endif
+
+#define SMU__NUM_PCIE_DPM_LEVELS 8
+#define SMU__NUM_SCLK_DPM_STATE 8
+#define SMU__NUM_MCLK_DPM_LEVELS 4
+#define SMU__VARIANT__ICELAND 1
+#define SMU__DGPU_ONLY 1
+#define SMU__DYNAMIC_MCARB_SETTINGS 1
+
+enum SID_OPTION {
+  SID_OPTION_HI,
+  SID_OPTION_LO,
+  SID_OPTION_COUNT
+};
+
+typedef struct {
+  uint32_t high;
+  uint32_t low;
+} data_64_t;
+
+typedef struct {
+  data_64_t high;
+  data_64_t low;
+} data_128_t;
+
+#define SMU7_CONTEXT_ID_SMC        1
+#define SMU7_CONTEXT_ID_VBIOS      2
+
+#define SMU71_MAX_LEVELS_VDDC            8
+#define SMU71_MAX_LEVELS_VDDCI           4
+#define SMU71_MAX_LEVELS_MVDD            4
+#define SMU71_MAX_LEVELS_VDDNB           8
+
+#define SMU71_MAX_LEVELS_GRAPHICS        SMU__NUM_SCLK_DPM_STATE
+#define SMU71_MAX_LEVELS_MEMORY          SMU__NUM_MCLK_DPM_LEVELS
+#define SMU71_MAX_LEVELS_GIO             SMU__NUM_LCLK_DPM_LEVELS
+#define SMU71_MAX_LEVELS_LINK            SMU__NUM_PCIE_DPM_LEVELS
+#define SMU71_MAX_ENTRIES_SMIO           32
+
+#define DPM_NO_LIMIT 0
+#define DPM_NO_UP 1
+#define DPM_GO_DOWN 2
+#define DPM_GO_UP 3
+
+#define SMU7_FIRST_DPM_GRAPHICS_LEVEL    0
+#define SMU7_FIRST_DPM_MEMORY_LEVEL      0
+
+#define GPIO_CLAMP_MODE_VRHOT      1
+#define GPIO_CLAMP_MODE_THERM      2
+#define GPIO_CLAMP_MODE_DC         4
+
+#define SCRATCH_B_TARG_PCIE_INDEX_SHIFT 0
+#define SCRATCH_B_TARG_PCIE_INDEX_MASK  (0x7<<SCRATCH_B_TARG_PCIE_INDEX_SHIFT)
+#define SCRATCH_B_CURR_PCIE_INDEX_SHIFT 3
+#define SCRATCH_B_CURR_PCIE_INDEX_MASK  (0x7<<SCRATCH_B_CURR_PCIE_INDEX_SHIFT)
+#define SCRATCH_B_TARG_UVD_INDEX_SHIFT  6
+#define SCRATCH_B_TARG_UVD_INDEX_MASK   (0x7<<SCRATCH_B_TARG_UVD_INDEX_SHIFT)
+#define SCRATCH_B_CURR_UVD_INDEX_SHIFT  9
+#define SCRATCH_B_CURR_UVD_INDEX_MASK   (0x7<<SCRATCH_B_CURR_UVD_INDEX_SHIFT)
+#define SCRATCH_B_TARG_VCE_INDEX_SHIFT  12
+#define SCRATCH_B_TARG_VCE_INDEX_MASK   (0x7<<SCRATCH_B_TARG_VCE_INDEX_SHIFT)
+#define SCRATCH_B_CURR_VCE_INDEX_SHIFT  15
+#define SCRATCH_B_CURR_VCE_INDEX_MASK   (0x7<<SCRATCH_B_CURR_VCE_INDEX_SHIFT)
+#define SCRATCH_B_TARG_ACP_INDEX_SHIFT  18
+#define SCRATCH_B_TARG_ACP_INDEX_MASK   (0x7<<SCRATCH_B_TARG_ACP_INDEX_SHIFT)
+#define SCRATCH_B_CURR_ACP_INDEX_SHIFT  21
+#define SCRATCH_B_CURR_ACP_INDEX_MASK   (0x7<<SCRATCH_B_CURR_ACP_INDEX_SHIFT)
+#define SCRATCH_B_TARG_SAMU_INDEX_SHIFT 24
+#define SCRATCH_B_TARG_SAMU_INDEX_MASK  (0x7<<SCRATCH_B_TARG_SAMU_INDEX_SHIFT)
+#define SCRATCH_B_CURR_SAMU_INDEX_SHIFT 27
+#define SCRATCH_B_CURR_SAMU_INDEX_MASK  (0x7<<SCRATCH_B_CURR_SAMU_INDEX_SHIFT)
+
+
+#if defined SMU__DGPU_ONLY
+#define SMU71_DTE_ITERATIONS 5
+#define SMU71_DTE_SOURCES 3
+#define SMU71_DTE_SINKS 1
+#define SMU71_NUM_CPU_TES 0
+#define SMU71_NUM_GPU_TES 1
+#define SMU71_NUM_NON_TES 2
+
+#endif
+
+#if defined SMU__FUSION_ONLY
+#define SMU7_DTE_ITERATIONS 5
+#define SMU7_DTE_SOURCES 5
+#define SMU7_DTE_SINKS 3
+#define SMU7_NUM_CPU_TES 2
+#define SMU7_NUM_GPU_TES 1
+#define SMU7_NUM_NON_TES 2
+
+#endif
+
+struct SMU71_PIDController
+{
+    uint32_t Ki;
+    int32_t LFWindupUpperLim;
+    int32_t LFWindupLowerLim;
+    uint32_t StatePrecision;
+    uint32_t LfPrecision;
+    uint32_t LfOffset;
+    uint32_t MaxState;
+    uint32_t MaxLfFraction;
+    uint32_t StateShift;
+};
+
+typedef struct SMU71_PIDController SMU71_PIDController;
+
+struct SMU7_LocalDpmScoreboard
+{
+    uint32_t PercentageBusy;
+
+    int32_t  PIDError;
+    int32_t  PIDIntegral;
+    int32_t  PIDOutput;
+
+    uint32_t SigmaDeltaAccum;
+    uint32_t SigmaDeltaOutput;
+    uint32_t SigmaDeltaLevel;
+
+    uint32_t UtilizationSetpoint;
+
+    uint8_t  TdpClampMode;
+    uint8_t  TdcClampMode;
+    uint8_t  ThermClampMode;
+    uint8_t  VoltageBusy;
+
+    int8_t   CurrLevel;
+    int8_t   TargLevel;
+    uint8_t  LevelChangeInProgress;
+    uint8_t  UpHyst;
+
+    uint8_t  DownHyst;
+    uint8_t  VoltageDownHyst;
+    uint8_t  DpmEnable;
+    uint8_t  DpmRunning;
+
+    uint8_t  DpmForce;
+    uint8_t  DpmForceLevel;
+    uint8_t  DisplayWatermark;
+    uint8_t  McArbIndex;
+
+    uint32_t MinimumPerfSclk;
+
+    uint8_t  AcpiReq;
+    uint8_t  AcpiAck;
+    uint8_t  GfxClkSlow;
+    uint8_t  GpioClampMode;
+
+    uint8_t  FpsFilterWeight;
+    uint8_t  EnabledLevelsChange;
+    uint8_t  DteClampMode;
+    uint8_t  FpsClampMode;
+
+    uint16_t LevelResidencyCounters [SMU71_MAX_LEVELS_GRAPHICS];
+    uint16_t LevelSwitchCounters [SMU71_MAX_LEVELS_GRAPHICS];
+
+    void     (*TargetStateCalculator)(uint8_t);
+    void     (*SavedTargetStateCalculator)(uint8_t);
+
+    uint16_t AutoDpmInterval;
+    uint16_t AutoDpmRange;
+
+    uint8_t  FpsEnabled;
+    uint8_t  MaxPerfLevel;
+    uint8_t  AllowLowClkInterruptToHost;
+    uint8_t  FpsRunning;
+
+    uint32_t MaxAllowedFrequency;
+};
+
+typedef struct SMU7_LocalDpmScoreboard SMU7_LocalDpmScoreboard;
+
+#define SMU7_MAX_VOLTAGE_CLIENTS 12
+
+struct SMU7_VoltageScoreboard
+{
+    uint16_t CurrentVoltage;
+    uint16_t HighestVoltage;
+    uint16_t MaxVid;
+    uint8_t  HighestVidOffset;
+    uint8_t  CurrentVidOffset;
+#if defined (SMU__DGPU_ONLY)
+    uint8_t  CurrentPhases;
+    uint8_t  HighestPhases;
+#else
+    uint8_t  AvsOffset;
+    uint8_t  AvsOffsetApplied;
+#endif
+    uint8_t  ControllerBusy;
+    uint8_t  CurrentVid;
+    uint16_t RequestedVoltage[SMU7_MAX_VOLTAGE_CLIENTS];
+#if defined (SMU__DGPU_ONLY)
+    uint8_t  RequestedPhases[SMU7_MAX_VOLTAGE_CLIENTS];
+#endif
+    uint8_t  EnabledRequest[SMU7_MAX_VOLTAGE_CLIENTS];
+    uint8_t  TargetIndex;
+    uint8_t  Delay;
+    uint8_t  ControllerEnable;
+    uint8_t  ControllerRunning;
+    uint16_t CurrentStdVoltageHiSidd;
+    uint16_t CurrentStdVoltageLoSidd;
+#if defined (SMU__DGPU_ONLY)
+    uint16_t RequestedVddci;
+    uint16_t CurrentVddci;
+    uint16_t HighestVddci;
+    uint8_t  CurrentVddciVid;
+    uint8_t  TargetVddciIndex;
+#endif
+};
+
+typedef struct SMU7_VoltageScoreboard SMU7_VoltageScoreboard;
+
+// -------------------------------------------------------------------------------------------------------------------------
+#define SMU7_MAX_PCIE_LINK_SPEEDS 3 /* 0:Gen1 1:Gen2 2:Gen3 */
+
+struct SMU7_PCIeLinkSpeedScoreboard
+{
+    uint8_t     DpmEnable;
+    uint8_t     DpmRunning;
+    uint8_t     DpmForce;
+    uint8_t     DpmForceLevel;
+
+    uint8_t     CurrentLinkSpeed;
+    uint8_t     EnabledLevelsChange;
+    uint16_t    AutoDpmInterval;
+
+    uint16_t    AutoDpmRange;
+    uint16_t    AutoDpmCount;
+
+    uint8_t     DpmMode;
+    uint8_t     AcpiReq;
+    uint8_t     AcpiAck;
+    uint8_t     CurrentLinkLevel;
+
+};
+
+typedef struct SMU7_PCIeLinkSpeedScoreboard SMU7_PCIeLinkSpeedScoreboard;
+
+// -------------------------------------------------------- CAC table ------------------------------------------------------
+#define SMU7_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16
+#define SMU7_LKGE_LUT_NUM_OF_VOLT_ENTRIES 16
+
+#define SMU7_SCALE_I  7
+#define SMU7_SCALE_R 12
+
+struct SMU7_PowerScoreboard
+{
+    uint16_t   MinVoltage;
+    uint16_t   MaxVoltage;
+
+    uint32_t   AvgGpuPower;
+
+    uint16_t   VddcLeakagePower[SID_OPTION_COUNT];
+    uint16_t   VddcSclkConstantPower[SID_OPTION_COUNT];
+    uint16_t   VddcSclkDynamicPower[SID_OPTION_COUNT];
+    uint16_t   VddcNonSclkDynamicPower[SID_OPTION_COUNT];
+    uint16_t   VddcTotalPower[SID_OPTION_COUNT];
+    uint16_t   VddcTotalCurrent[SID_OPTION_COUNT];
+    uint16_t   VddcLoadVoltage[SID_OPTION_COUNT];
+    uint16_t   VddcNoLoadVoltage[SID_OPTION_COUNT];
+
+    uint16_t   DisplayPhyPower;
+    uint16_t   PciePhyPower;
+
+    uint16_t   VddciTotalPower;
+    uint16_t   Vddr1TotalPower;
+
+    uint32_t   RocPower;
+
+    uint32_t   last_power;
+    uint32_t   enableWinAvg;
+
+    uint32_t   lkg_acc;
+    uint16_t   VoltLkgeScaler;
+    uint16_t   TempLkgeScaler;
+
+    uint32_t   uvd_cac_dclk;
+    uint32_t   uvd_cac_vclk;
+    uint32_t   vce_cac_eclk;
+    uint32_t   samu_cac_samclk;
+    uint32_t   display_cac_dispclk;
+    uint32_t   acp_cac_aclk;
+    uint32_t   unb_cac;
+
+    uint32_t   WinTime;
+
+    uint16_t  GpuPwr_MAWt;
+    uint16_t  FilteredVddcTotalPower;
+
+    uint8_t   CalculationRepeats;
+    uint8_t   WaterfallUp;
+    uint8_t   WaterfallDown;
+    uint8_t   WaterfallLimit;
+};
+
+typedef struct SMU7_PowerScoreboard SMU7_PowerScoreboard;
+
+// --------------------------------------------------------------------------------------------------
+
+struct SMU7_ThermalScoreboard
+{
+   int16_t  GpuLimit;
+   int16_t  GpuHyst;
+   uint16_t CurrGnbTemp;
+   uint16_t FilteredGnbTemp;
+   uint8_t  ControllerEnable;
+   uint8_t  ControllerRunning;
+   uint8_t  WaterfallUp;
+   uint8_t  WaterfallDown;
+   uint8_t  WaterfallLimit;
+   uint8_t  padding[3];
+};
+
+typedef struct SMU7_ThermalScoreboard SMU7_ThermalScoreboard;
+
+// For FeatureEnables:
+#define SMU7_SCLK_DPM_CONFIG_MASK                        0x01
+#define SMU7_VOLTAGE_CONTROLLER_CONFIG_MASK              0x02
+#define SMU7_THERMAL_CONTROLLER_CONFIG_MASK              0x04
+#define SMU7_MCLK_DPM_CONFIG_MASK                        0x08
+#define SMU7_UVD_DPM_CONFIG_MASK                         0x10
+#define SMU7_VCE_DPM_CONFIG_MASK                         0x20
+#define SMU7_ACP_DPM_CONFIG_MASK                         0x40
+#define SMU7_SAMU_DPM_CONFIG_MASK                        0x80
+#define SMU7_PCIEGEN_DPM_CONFIG_MASK                    0x100
+
+#define SMU7_ACP_MCLK_HANDSHAKE_DISABLE                  0x00000001
+#define SMU7_ACP_SCLK_HANDSHAKE_DISABLE                  0x00000002
+#define SMU7_UVD_MCLK_HANDSHAKE_DISABLE                  0x00000100
+#define SMU7_UVD_SCLK_HANDSHAKE_DISABLE                  0x00000200
+#define SMU7_VCE_MCLK_HANDSHAKE_DISABLE                  0x00010000
+#define SMU7_VCE_SCLK_HANDSHAKE_DISABLE                  0x00020000
+
+// All 'soft registers' should be uint32_t.
+struct SMU71_SoftRegisters
+{
+    uint32_t        RefClockFrequency;
+    uint32_t        PmTimerPeriod;
+    uint32_t        FeatureEnables;
+#if defined (SMU__DGPU_ONLY)
+    uint32_t        PreVBlankGap;
+    uint32_t        VBlankTimeout;
+    uint32_t        TrainTimeGap;
+    uint32_t        MvddSwitchTime;
+    uint32_t        LongestAcpiTrainTime;
+    uint32_t        AcpiDelay;
+    uint32_t        G5TrainTime;
+    uint32_t        DelayMpllPwron;
+    uint32_t        VoltageChangeTimeout;
+#endif
+    uint32_t        HandshakeDisables;
+
+    uint8_t         DisplayPhy1Config;
+    uint8_t         DisplayPhy2Config;
+    uint8_t         DisplayPhy3Config;
+    uint8_t         DisplayPhy4Config;
+
+    uint8_t         DisplayPhy5Config;
+    uint8_t         DisplayPhy6Config;
+    uint8_t         DisplayPhy7Config;
+    uint8_t         DisplayPhy8Config;
+
+    uint32_t        AverageGraphicsActivity;
+    uint32_t        AverageMemoryActivity;
+    uint32_t        AverageGioActivity;
+
+    uint8_t         SClkDpmEnabledLevels;
+    uint8_t         MClkDpmEnabledLevels;
+    uint8_t         LClkDpmEnabledLevels;
+    uint8_t         PCIeDpmEnabledLevels;
+
+    uint32_t        DRAM_LOG_ADDR_H;
+    uint32_t        DRAM_LOG_ADDR_L;
+    uint32_t        DRAM_LOG_PHY_ADDR_H;
+    uint32_t        DRAM_LOG_PHY_ADDR_L;
+    uint32_t        DRAM_LOG_BUFF_SIZE;
+    uint32_t        UlvEnterCount;
+    uint32_t        UlvTime;
+    uint32_t        UcodeLoadStatus;
+    uint8_t         DPMFreezeAndForced;
+    uint8_t         Activity_Weight;
+    uint8_t         Reserved8[2];
+    uint32_t        Reserved;
+};
+
+typedef struct SMU71_SoftRegisters SMU71_SoftRegisters;
+
+struct SMU71_Firmware_Header
+{
+    uint32_t Digest[5];
+    uint32_t Version;
+    uint32_t HeaderSize;
+    uint32_t Flags;
+    uint32_t EntryPoint;
+    uint32_t CodeSize;
+    uint32_t ImageSize;
+
+    uint32_t Rtos;
+    uint32_t SoftRegisters;
+    uint32_t DpmTable;
+    uint32_t FanTable;
+    uint32_t CacConfigTable;
+    uint32_t CacStatusTable;
+
+    uint32_t mcRegisterTable;
+
+    uint32_t mcArbDramTimingTable;
+
+    uint32_t PmFuseTable;
+    uint32_t Globals;
+    uint32_t UvdDpmTable;
+    uint32_t AcpDpmTable;
+    uint32_t VceDpmTable;
+    uint32_t SamuDpmTable;
+    uint32_t UlvSettings;
+    uint32_t Reserved[37];
+    uint32_t Signature;
+};
+
+typedef struct SMU71_Firmware_Header SMU71_Firmware_Header;
+
+struct SMU7_HystController_Data
+{
+    uint8_t waterfall_up;
+    uint8_t waterfall_down;
+    uint8_t pstate;
+    uint8_t clamp_mode;
+};
+
+typedef struct SMU7_HystController_Data SMU7_HystController_Data;
+
+#define SMU71_FIRMWARE_HEADER_LOCATION 0x20000
+
+enum  DisplayConfig {
+    PowerDown = 1,
+    DP54x4,
+    DP54x2,
+    DP54x1,
+    DP27x4,
+    DP27x2,
+    DP27x1,
+    HDMI297,
+    HDMI162,
+    LVDS,
+    DP324x4,
+    DP324x2,
+    DP324x1
+};
+
+//#define SX_BLOCK_COUNT 8
+//#define MC_BLOCK_COUNT 1
+//#define CPL_BLOCK_COUNT 27
+
+#if defined SMU__VARIANT__ICELAND
+  #define SX_BLOCK_COUNT 8
+  #define MC_BLOCK_COUNT 1
+  #define CPL_BLOCK_COUNT 29
+#endif
+
+struct SMU7_Local_Cac {
+  uint8_t BlockId;
+  uint8_t SignalId;
+  uint8_t Threshold;
+  uint8_t Padding;
+};
+
+typedef struct SMU7_Local_Cac SMU7_Local_Cac;
+
+struct SMU7_Local_Cac_Table {
+  SMU7_Local_Cac SxLocalCac[SX_BLOCK_COUNT];
+  SMU7_Local_Cac CplLocalCac[CPL_BLOCK_COUNT];
+  SMU7_Local_Cac McLocalCac[MC_BLOCK_COUNT];
+};
+
+typedef struct SMU7_Local_Cac_Table SMU7_Local_Cac_Table;
+
+#if !defined(SMC_MICROCODE)
+#pragma pack(pop)
+#endif
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu71_discrete.h b/drivers/gpu/drm/amd/powerplay/inc/smu71_discrete.h
new file mode 100644
index 000000000000..c0e3936d5c2e
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu71_discrete.h
@@ -0,0 +1,631 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef SMU71_DISCRETE_H
+#define SMU71_DISCRETE_H
+
+#include "smu71.h"
+
+#if !defined(SMC_MICROCODE)
+#pragma pack(push, 1)
+#endif
+
+#define VDDC_ON_SVI2  0x1
+#define VDDCI_ON_SVI2 0x2
+#define MVDD_ON_SVI2  0x4
+
+struct SMU71_Discrete_VoltageLevel
+{
+    uint16_t    Voltage;
+    uint16_t    StdVoltageHiSidd;
+    uint16_t    StdVoltageLoSidd;
+    uint8_t     Smio;
+    uint8_t     padding;
+};
+
+typedef struct SMU71_Discrete_VoltageLevel SMU71_Discrete_VoltageLevel;
+
+struct SMU71_Discrete_GraphicsLevel
+{
+    uint32_t    MinVddc;
+    uint32_t    MinVddcPhases;
+
+    uint32_t    SclkFrequency;
+
+    uint8_t     pcieDpmLevel;
+    uint8_t     DeepSleepDivId;
+    uint16_t    ActivityLevel;
+
+    uint32_t    CgSpllFuncCntl3;
+    uint32_t    CgSpllFuncCntl4;
+    uint32_t    SpllSpreadSpectrum;
+    uint32_t    SpllSpreadSpectrum2;
+    uint32_t    CcPwrDynRm;
+    uint32_t    CcPwrDynRm1;
+    uint8_t     SclkDid;
+    uint8_t     DisplayWatermark;
+    uint8_t     EnabledForActivity;
+    uint8_t     EnabledForThrottle;
+    uint8_t     UpHyst;
+    uint8_t     DownHyst;
+    uint8_t     VoltageDownHyst;
+    uint8_t     PowerThrottle;
+};
+
+typedef struct SMU71_Discrete_GraphicsLevel SMU71_Discrete_GraphicsLevel;
+
+struct SMU71_Discrete_ACPILevel
+{
+    uint32_t    Flags;
+    uint32_t    MinVddc;
+    uint32_t    MinVddcPhases;
+    uint32_t    SclkFrequency;
+    uint8_t     SclkDid;
+    uint8_t     DisplayWatermark;
+    uint8_t     DeepSleepDivId;
+    uint8_t     padding;
+    uint32_t    CgSpllFuncCntl;
+    uint32_t    CgSpllFuncCntl2;
+    uint32_t    CgSpllFuncCntl3;
+    uint32_t    CgSpllFuncCntl4;
+    uint32_t    SpllSpreadSpectrum;
+    uint32_t    SpllSpreadSpectrum2;
+    uint32_t    CcPwrDynRm;
+    uint32_t    CcPwrDynRm1;
+};
+
+typedef struct SMU71_Discrete_ACPILevel SMU71_Discrete_ACPILevel;
+
+struct SMU71_Discrete_Ulv
+{
+    uint32_t    CcPwrDynRm;
+    uint32_t    CcPwrDynRm1;
+    uint16_t    VddcOffset;
+    uint8_t     VddcOffsetVid;
+    uint8_t     VddcPhase;
+    uint32_t    Reserved;
+};
+
+typedef struct SMU71_Discrete_Ulv SMU71_Discrete_Ulv;
+
+struct SMU71_Discrete_MemoryLevel
+{
+    uint32_t    MinVddc;
+    uint32_t    MinVddcPhases;
+    uint32_t    MinVddci;
+    uint32_t    MinMvdd;
+
+    uint32_t    MclkFrequency;
+
+    uint8_t     EdcReadEnable;
+    uint8_t     EdcWriteEnable;
+    uint8_t     RttEnable;
+    uint8_t     StutterEnable;
+
+    uint8_t     StrobeEnable;
+    uint8_t     StrobeRatio;
+    uint8_t     EnabledForThrottle;
+    uint8_t     EnabledForActivity;
+
+    uint8_t     UpHyst;
+    uint8_t     DownHyst;
+    uint8_t     VoltageDownHyst;
+    uint8_t     padding;
+
+    uint16_t    ActivityLevel;
+    uint8_t     DisplayWatermark;
+    uint8_t     padding1;
+
+    uint32_t    MpllFuncCntl;
+    uint32_t    MpllFuncCntl_1;
+    uint32_t    MpllFuncCntl_2;
+    uint32_t    MpllAdFuncCntl;
+    uint32_t    MpllDqFuncCntl;
+    uint32_t    MclkPwrmgtCntl;
+    uint32_t    DllCntl;
+    uint32_t    MpllSs1;
+    uint32_t    MpllSs2;
+};
+
+typedef struct SMU71_Discrete_MemoryLevel SMU71_Discrete_MemoryLevel;
+
+struct SMU71_Discrete_LinkLevel
+{
+    uint8_t     PcieGenSpeed;           ///< 0:PciE-gen1 1:PciE-gen2 2:PciE-gen3
+    uint8_t     PcieLaneCount;          ///< 1=x1, 2=x2, 3=x4, 4=x8, 5=x12, 6=x16
+    uint8_t     EnabledForActivity;
+    uint8_t     SPC;
+    uint32_t    DownThreshold;
+    uint32_t    UpThreshold;
+    uint32_t    Reserved;
+};
+
+typedef struct SMU71_Discrete_LinkLevel SMU71_Discrete_LinkLevel;
+
+
+#ifdef SMU__DYNAMIC_MCARB_SETTINGS
+// MC ARB DRAM Timing registers.
+struct SMU71_Discrete_MCArbDramTimingTableEntry
+{
+    uint32_t McArbDramTiming;
+    uint32_t McArbDramTiming2;
+    uint8_t  McArbBurstTime;
+    uint8_t  padding[3];
+};
+
+typedef struct SMU71_Discrete_MCArbDramTimingTableEntry SMU71_Discrete_MCArbDramTimingTableEntry;
+
+struct SMU71_Discrete_MCArbDramTimingTable
+{
+    SMU71_Discrete_MCArbDramTimingTableEntry entries[SMU__NUM_SCLK_DPM_STATE][SMU__NUM_MCLK_DPM_LEVELS];
+};
+
+typedef struct SMU71_Discrete_MCArbDramTimingTable SMU71_Discrete_MCArbDramTimingTable;
+#endif
+
+// UVD VCLK/DCLK state (level) definition.
+struct SMU71_Discrete_UvdLevel
+{
+    uint32_t VclkFrequency;
+    uint32_t DclkFrequency;
+    uint16_t MinVddc;
+    uint8_t  MinVddcPhases;
+    uint8_t  VclkDivider;
+    uint8_t  DclkDivider;
+    uint8_t  padding[3];
+};
+
+typedef struct SMU71_Discrete_UvdLevel SMU71_Discrete_UvdLevel;
+
+// Clocks for other external blocks (VCE, ACP, SAMU).
+struct SMU71_Discrete_ExtClkLevel
+{
+    uint32_t Frequency;
+    uint16_t MinVoltage;
+    uint8_t  MinPhases;
+    uint8_t  Divider;
+};
+
+typedef struct SMU71_Discrete_ExtClkLevel SMU71_Discrete_ExtClkLevel;
+
+// Everything that we need to keep track of about the current state.
+// Use this instead of copies of the GraphicsLevel and MemoryLevel structures to keep track of state parameters
+// that need to be checked later.
+// We don't need to cache everything about a state, just a few parameters.
+struct SMU71_Discrete_StateInfo
+{
+    uint32_t SclkFrequency;
+    uint32_t MclkFrequency;
+    uint32_t VclkFrequency;
+    uint32_t DclkFrequency;
+    uint32_t SamclkFrequency;
+    uint32_t AclkFrequency;
+    uint32_t EclkFrequency;
+    uint16_t MvddVoltage;
+    uint16_t padding16;
+    uint8_t  DisplayWatermark;
+    uint8_t  McArbIndex;
+    uint8_t  McRegIndex;
+    uint8_t  SeqIndex;
+    uint8_t  SclkDid;
+    int8_t   SclkIndex;
+    int8_t   MclkIndex;
+    uint8_t  PCIeGen;
+
+};
+
+typedef struct SMU71_Discrete_StateInfo SMU71_Discrete_StateInfo;
+
+
+struct SMU71_Discrete_DpmTable
+{
+    // Multi-DPM controller settings
+    SMU71_PIDController                  GraphicsPIDController;
+    SMU71_PIDController                  MemoryPIDController;
+    SMU71_PIDController                  LinkPIDController;
+
+    uint32_t                            SystemFlags;
+
+    // SMIO masks for voltage and phase controls
+    uint32_t                            SmioMaskVddcVid;
+    uint32_t                            SmioMaskVddcPhase;
+    uint32_t                            SmioMaskVddciVid;
+    uint32_t                            SmioMaskMvddVid;
+
+    uint32_t                            VddcLevelCount;
+    uint32_t                            VddciLevelCount;
+    uint32_t                            MvddLevelCount;
+
+    SMU71_Discrete_VoltageLevel          VddcLevel               [SMU71_MAX_LEVELS_VDDC];
+    SMU71_Discrete_VoltageLevel          VddciLevel              [SMU71_MAX_LEVELS_VDDCI];
+    SMU71_Discrete_VoltageLevel          MvddLevel               [SMU71_MAX_LEVELS_MVDD];
+
+    uint8_t                             GraphicsDpmLevelCount;
+    uint8_t                             MemoryDpmLevelCount;
+    uint8_t                             LinkLevelCount;
+    uint8_t                             MasterDeepSleepControl;
+
+    uint32_t                            Reserved[5];
+
+    // State table entries for each DPM state
+    SMU71_Discrete_GraphicsLevel         GraphicsLevel           [SMU71_MAX_LEVELS_GRAPHICS];
+    SMU71_Discrete_MemoryLevel           MemoryACPILevel;
+    SMU71_Discrete_MemoryLevel           MemoryLevel             [SMU71_MAX_LEVELS_MEMORY];
+    SMU71_Discrete_LinkLevel             LinkLevel               [SMU71_MAX_LEVELS_LINK];
+    SMU71_Discrete_ACPILevel             ACPILevel;
+
+    uint32_t                            SclkStepSize;
+    uint32_t                            Smio                    [SMU71_MAX_ENTRIES_SMIO];
+
+    uint8_t                             GraphicsBootLevel;
+    uint8_t                             GraphicsVoltageChangeEnable;
+    uint8_t                             GraphicsThermThrottleEnable;
+    uint8_t                             GraphicsInterval;
+
+    uint8_t                             VoltageInterval;
+    uint8_t                             ThermalInterval;
+    uint16_t                            TemperatureLimitHigh;
+
+    uint16_t                            TemperatureLimitLow;
+    uint8_t                             MemoryBootLevel;
+    uint8_t                             MemoryVoltageChangeEnable;
+
+    uint8_t                             MemoryInterval;
+    uint8_t                             MemoryThermThrottleEnable;
+    uint8_t                             MergedVddci;
+    uint8_t                             padding2;
+
+    uint16_t                            VoltageResponseTime;
+    uint16_t                            PhaseResponseTime;
+
+    uint8_t                             PCIeBootLinkLevel;
+    uint8_t                             PCIeGenInterval;
+    uint8_t                             DTEInterval;
+    uint8_t                             DTEMode;
+
+    uint8_t                             SVI2Enable;
+    uint8_t                             VRHotGpio;
+    uint8_t                             AcDcGpio;
+    uint8_t                             ThermGpio;
+
+    uint32_t                            DisplayCac;
+
+    uint16_t                            MaxPwr;
+    uint16_t                            NomPwr;
+
+    uint16_t                            FpsHighThreshold;
+    uint16_t                            FpsLowThreshold;
+
+    uint16_t                            BAPMTI_R  [SMU71_DTE_ITERATIONS][SMU71_DTE_SOURCES][SMU71_DTE_SINKS];
+    uint16_t                            BAPMTI_RC [SMU71_DTE_ITERATIONS][SMU71_DTE_SOURCES][SMU71_DTE_SINKS];
+
+    uint8_t                             DTEAmbientTempBase;
+    uint8_t                             DTETjOffset;
+    uint8_t                             GpuTjMax;
+    uint8_t                             GpuTjHyst;
+
+    uint16_t                            BootVddc;
+    uint16_t                            BootVddci;
+
+    uint16_t                            BootMVdd;
+    uint16_t                            padding;
+
+    uint32_t                            BAPM_TEMP_GRADIENT;
+
+    uint32_t                            LowSclkInterruptThreshold;
+    uint32_t                            VddGfxReChkWait;
+
+    uint16_t                            PPM_PkgPwrLimit;
+    uint16_t                            PPM_TemperatureLimit;
+
+    uint16_t                            DefaultTdp;
+    uint16_t                            TargetTdp;
+};
+
+typedef struct SMU71_Discrete_DpmTable SMU71_Discrete_DpmTable;
+
+// --------------------------------------------------- AC Timing Parameters ------------------------------------------------
+#define SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE 16
+#define SMU71_DISCRETE_MC_REGISTER_ARRAY_SET_COUNT SMU71_MAX_LEVELS_MEMORY
+
+struct SMU71_Discrete_MCRegisterAddress
+{
+    uint16_t s0;
+    uint16_t s1;
+};
+
+typedef struct SMU71_Discrete_MCRegisterAddress SMU71_Discrete_MCRegisterAddress;
+
+struct SMU71_Discrete_MCRegisterSet
+{
+    uint32_t value[SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE];
+};
+
+typedef struct SMU71_Discrete_MCRegisterSet SMU71_Discrete_MCRegisterSet;
+
+struct SMU71_Discrete_MCRegisters
+{
+    uint8_t                             last;
+    uint8_t                             reserved[3];
+    SMU71_Discrete_MCRegisterAddress     address[SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE];
+    SMU71_Discrete_MCRegisterSet         data[SMU71_DISCRETE_MC_REGISTER_ARRAY_SET_COUNT];
+};
+
+typedef struct SMU71_Discrete_MCRegisters SMU71_Discrete_MCRegisters;
+
+
+// --------------------------------------------------- Fan Table -----------------------------------------------------------
+struct SMU71_Discrete_FanTable
+{
+    uint16_t FdoMode;
+    int16_t  TempMin;
+    int16_t  TempMed;
+    int16_t  TempMax;
+    int16_t  Slope1;
+    int16_t  Slope2;
+    int16_t  FdoMin;
+    int16_t  HystUp;
+    int16_t  HystDown;
+    int16_t  HystSlope;
+    int16_t  TempRespLim;
+    int16_t  TempCurr;
+    int16_t  SlopeCurr;
+    int16_t  PwmCurr;
+    uint32_t RefreshPeriod;
+    int16_t  FdoMax;
+    uint8_t  TempSrc;
+    int8_t   Padding;
+};
+
+typedef struct SMU71_Discrete_FanTable SMU71_Discrete_FanTable;
+
+#define SMU7_DISCRETE_GPIO_SCLK_DEBUG             4
+#define SMU7_DISCRETE_GPIO_SCLK_DEBUG_BIT         (0x1 << SMU7_DISCRETE_GPIO_SCLK_DEBUG)
+
+struct SMU71_MclkDpmScoreboard
+{
+
+    uint32_t PercentageBusy;
+
+    int32_t  PIDError;
+    int32_t  PIDIntegral;
+    int32_t  PIDOutput;
+
+    uint32_t SigmaDeltaAccum;
+    uint32_t SigmaDeltaOutput;
+    uint32_t SigmaDeltaLevel;
+
+    uint32_t UtilizationSetpoint;
+
+    uint8_t  TdpClampMode;
+    uint8_t  TdcClampMode;
+    uint8_t  ThermClampMode;
+    uint8_t  VoltageBusy;
+
+    int8_t   CurrLevel;
+    int8_t   TargLevel;
+    uint8_t  LevelChangeInProgress;
+    uint8_t  UpHyst;
+
+    uint8_t  DownHyst;
+    uint8_t  VoltageDownHyst;
+    uint8_t  DpmEnable;
+    uint8_t  DpmRunning;
+
+    uint8_t  DpmForce;
+    uint8_t  DpmForceLevel;
+    uint8_t  DisplayWatermark;
+    uint8_t  McArbIndex;
+
+    uint32_t MinimumPerfMclk;
+
+    uint8_t  AcpiReq;
+    uint8_t  AcpiAck;
+    uint8_t  MclkSwitchInProgress;
+    uint8_t  MclkSwitchCritical;
+
+    uint8_t  TargetMclkIndex;
+    uint8_t  TargetMvddIndex;
+    uint8_t  MclkSwitchResult;
+
+    uint8_t  EnabledLevelsChange;
+
+    uint16_t LevelResidencyCounters [SMU71_MAX_LEVELS_MEMORY];
+    uint16_t LevelSwitchCounters [SMU71_MAX_LEVELS_MEMORY];
+
+    void     (*TargetStateCalculator)(uint8_t);
+    void     (*SavedTargetStateCalculator)(uint8_t);
+
+    uint16_t AutoDpmInterval;
+    uint16_t AutoDpmRange;
+
+    uint16_t  MclkSwitchingTime;
+    uint8_t padding[2];
+};
+
+typedef struct SMU71_MclkDpmScoreboard SMU71_MclkDpmScoreboard;
+
+struct SMU71_UlvScoreboard
+{
+    uint8_t     EnterUlv;
+    uint8_t     ExitUlv;
+    uint8_t     UlvActive;
+    uint8_t     WaitingForUlv;
+    uint8_t     UlvEnable;
+    uint8_t     UlvRunning;
+    uint8_t     UlvMasterEnable;
+    uint8_t     padding;
+    uint32_t    UlvAbortedCount;
+    uint32_t    UlvTimeStamp;
+};
+
+typedef struct SMU71_UlvScoreboard SMU71_UlvScoreboard;
+
+struct SMU71_VddGfxScoreboard
+{
+    uint8_t     VddGfxEnable;
+    uint8_t     VddGfxActive;
+    uint8_t     padding[2];
+
+    uint32_t    VddGfxEnteredCount;
+    uint32_t    VddGfxAbortedCount;
+};
+
+typedef struct SMU71_VddGfxScoreboard SMU71_VddGfxScoreboard;
+
+struct SMU71_AcpiScoreboard {
+  uint32_t SavedInterruptMask[2];
+  uint8_t LastACPIRequest;
+  uint8_t CgBifResp;
+  uint8_t RequestType;
+  uint8_t Padding;
+  SMU71_Discrete_ACPILevel D0Level;
+};
+
+typedef struct SMU71_AcpiScoreboard SMU71_AcpiScoreboard;
+
+
+struct SMU71_Discrete_PmFuses {
+  // dw0-dw1
+  uint8_t BapmVddCVidHiSidd[8];
+
+  // dw2-dw3
+  uint8_t BapmVddCVidLoSidd[8];
+
+  // dw4-dw5
+  uint8_t VddCVid[8];
+
+  // dw6
+  uint8_t SviLoadLineEn;
+  uint8_t SviLoadLineVddC;
+  uint8_t SviLoadLineTrimVddC;
+  uint8_t SviLoadLineOffsetVddC;
+
+  // dw7
+  uint16_t TDC_VDDC_PkgLimit;
+  uint8_t TDC_VDDC_ThrottleReleaseLimitPerc;
+  uint8_t TDC_MAWt;
+
+  // dw8
+  uint8_t TdcWaterfallCtl;
+  uint8_t LPMLTemperatureMin;
+  uint8_t LPMLTemperatureMax;
+  uint8_t Reserved;
+
+  // dw9-dw12
+  uint8_t LPMLTemperatureScaler[16];
+
+  // dw13-dw14
+  int16_t FuzzyFan_ErrorSetDelta;
+  int16_t FuzzyFan_ErrorRateSetDelta;
+  int16_t FuzzyFan_PwmSetDelta;
+  uint16_t Reserved6;
+
+  // dw15
+  uint8_t GnbLPML[16];
+
+  // dw15
+  uint8_t GnbLPMLMaxVid;
+  uint8_t GnbLPMLMinVid;
+  uint8_t Reserved1[2];
+
+  // dw16
+  uint16_t BapmVddCBaseLeakageHiSidd;
+  uint16_t BapmVddCBaseLeakageLoSidd;
+};
+
+typedef struct SMU71_Discrete_PmFuses SMU71_Discrete_PmFuses;
+
+struct SMU71_Discrete_Log_Header_Table {
+  uint32_t    version;
+  uint32_t    asic_id;
+  uint16_t    flags;
+  uint16_t    entry_size;
+  uint32_t    total_size;
+  uint32_t    num_of_entries;
+  uint8_t     type;
+  uint8_t     mode;
+  uint8_t     filler_0[2];
+  uint32_t    filler_1[2];
+};
+
+typedef struct SMU71_Discrete_Log_Header_Table SMU71_Discrete_Log_Header_Table;
+
+struct SMU71_Discrete_Log_Cntl {
+    uint8_t             Enabled;
+    uint8_t             Type;
+    uint8_t             padding[2];
+    uint32_t            BufferSize;
+    uint32_t            SamplesLogged;
+    uint32_t            SampleSize;
+    uint32_t            AddrL;
+    uint32_t            AddrH;
+};
+
+typedef struct SMU71_Discrete_Log_Cntl SMU71_Discrete_Log_Cntl;
+
+#if defined SMU__DGPU_ONLY
+  #define CAC_ACC_NW_NUM_OF_SIGNALS 83
+#endif
+
+
+struct SMU71_Discrete_Cac_Collection_Table {
+  uint32_t temperature;
+  uint32_t cac_acc_nw[CAC_ACC_NW_NUM_OF_SIGNALS];
+  uint32_t filler[4];
+};
+
+typedef struct SMU71_Discrete_Cac_Collection_Table SMU71_Discrete_Cac_Collection_Table;
+
+struct SMU71_Discrete_Cac_Verification_Table {
+  uint32_t VddcTotalPower;
+  uint32_t VddcLeakagePower;
+  uint32_t VddcConstantPower;
+  uint32_t VddcGfxDynamicPower;
+  uint32_t VddcUvdDynamicPower;
+  uint32_t VddcVceDynamicPower;
+  uint32_t VddcAcpDynamicPower;
+  uint32_t VddcPcieDynamicPower;
+  uint32_t VddcDceDynamicPower;
+  uint32_t VddcCurrent;
+  uint32_t VddcVoltage;
+  uint32_t VddciTotalPower;
+  uint32_t VddciLeakagePower;
+  uint32_t VddciConstantPower;
+  uint32_t VddciDynamicPower;
+  uint32_t Vddr1TotalPower;
+  uint32_t Vddr1LeakagePower;
+  uint32_t Vddr1ConstantPower;
+  uint32_t Vddr1DynamicPower;
+  uint32_t spare[8];
+  uint32_t temperature;
+};
+
+typedef struct SMU71_Discrete_Cac_Verification_Table SMU71_Discrete_Cac_Verification_Table;
+
+#if !defined(SMC_MICROCODE)
+#pragma pack(pop)
+#endif
+
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu7_common.h b/drivers/gpu/drm/amd/powerplay/inc/smu7_common.h
new file mode 100644
index 000000000000..65eb630bfea3
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu7_common.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _PP_COMMON_H
+#define _PP_COMMON_H
+
+#include "smu7_ppsmc.h"
+#include "cgs_common.h"
+
+#include "smu/smu_7_1_3_d.h"
+#include "smu/smu_7_1_3_sh_mask.h"
+
+
+#include "smu74.h"
+#include "smu74_discrete.h"
+
+#include "gmc/gmc_8_1_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+
+
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
+
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu7_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/smu7_ppsmc.h
new file mode 100644
index 000000000000..bce00096d80d
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu7_ppsmc.h
@@ -0,0 +1,412 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef DGPU_VI_PP_SMC_H
+#define DGPU_VI_PP_SMC_H
+
+
+#pragma pack(push, 1)
+
+#define PPSMC_MSG_SetGBDroopSettings          ((uint16_t) 0x305)
+
+#define PPSMC_SWSTATE_FLAG_DC                           0x01
+#define PPSMC_SWSTATE_FLAG_UVD                          0x02
+#define PPSMC_SWSTATE_FLAG_VCE                          0x04
+
+#define PPSMC_THERMAL_PROTECT_TYPE_INTERNAL             0x00
+#define PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL             0x01
+#define PPSMC_THERMAL_PROTECT_TYPE_NONE                 0xff
+
+#define PPSMC_SYSTEMFLAG_GPIO_DC                        0x01
+#define PPSMC_SYSTEMFLAG_STEPVDDC                       0x02
+#define PPSMC_SYSTEMFLAG_GDDR5                          0x04
+
+#define PPSMC_SYSTEMFLAG_DISABLE_BABYSTEP               0x08
+
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT                  0x10
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_ANALOG           0x20
+
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_MASK              0x07
+#define PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK     0x08
+
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTODPMLOWSTATE   0x00
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE  0x01
+
+
+#define PPSMC_DPM2FLAGS_TDPCLMP                         0x01
+#define PPSMC_DPM2FLAGS_PWRSHFT                         0x02
+#define PPSMC_DPM2FLAGS_OCP                             0x04
+
+
+#define PPSMC_DISPLAY_WATERMARK_LOW                     0
+#define PPSMC_DISPLAY_WATERMARK_HIGH                    1
+
+
+#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP    0x01
+#define PPSMC_STATEFLAG_POWERBOOST         0x02
+#define PPSMC_STATEFLAG_PSKIP_ON_TDP_FAULT 0x04
+#define PPSMC_STATEFLAG_POWERSHIFT         0x08
+#define PPSMC_STATEFLAG_SLOW_READ_MARGIN   0x10
+#define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 0x20
+#define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS   0x40
+
+
+#define FDO_MODE_HARDWARE 0
+#define FDO_MODE_PIECE_WISE_LINEAR 1
+
+enum FAN_CONTROL {
+	FAN_CONTROL_FUZZY,
+	FAN_CONTROL_TABLE
+};
+
+
+#define PPSMC_Result_OK             ((uint16_t)0x01)
+#define PPSMC_Result_NoMore         ((uint16_t)0x02)
+
+#define PPSMC_Result_NotNow         ((uint16_t)0x03)
+#define PPSMC_Result_Failed         ((uint16_t)0xFF)
+#define PPSMC_Result_UnknownCmd     ((uint16_t)0xFE)
+#define PPSMC_Result_UnknownVT      ((uint16_t)0xFD)
+
+typedef uint16_t PPSMC_Result;
+
+#define PPSMC_isERROR(x) ((uint16_t)0x80 & (x))
+
+
+#define PPSMC_MSG_Halt                      ((uint16_t)0x10)
+#define PPSMC_MSG_Resume                    ((uint16_t)0x11)
+#define PPSMC_MSG_EnableDPMLevel            ((uint16_t)0x12)
+#define PPSMC_MSG_ZeroLevelsDisabled        ((uint16_t)0x13)
+#define PPSMC_MSG_OneLevelsDisabled         ((uint16_t)0x14)
+#define PPSMC_MSG_TwoLevelsDisabled         ((uint16_t)0x15)
+#define PPSMC_MSG_EnableThermalInterrupt    ((uint16_t)0x16)
+#define PPSMC_MSG_RunningOnAC               ((uint16_t)0x17)
+#define PPSMC_MSG_LevelUp                   ((uint16_t)0x18)
+#define PPSMC_MSG_LevelDown                 ((uint16_t)0x19)
+#define PPSMC_MSG_ResetDPMCounters          ((uint16_t)0x1a)
+#define PPSMC_MSG_SwitchToSwState           ((uint16_t)0x20)
+#define PPSMC_MSG_SwitchToSwStateLast       ((uint16_t)0x3f)
+#define PPSMC_MSG_SwitchToInitialState      ((uint16_t)0x40)
+#define PPSMC_MSG_NoForcedLevel             ((uint16_t)0x41)
+#define PPSMC_MSG_ForceHigh                 ((uint16_t)0x42)
+#define PPSMC_MSG_ForceMediumOrHigh         ((uint16_t)0x43)
+#define PPSMC_MSG_SwitchToMinimumPower      ((uint16_t)0x51)
+#define PPSMC_MSG_ResumeFromMinimumPower    ((uint16_t)0x52)
+#define PPSMC_MSG_EnableCac                 ((uint16_t)0x53)
+#define PPSMC_MSG_DisableCac                ((uint16_t)0x54)
+#define PPSMC_DPMStateHistoryStart          ((uint16_t)0x55)
+#define PPSMC_DPMStateHistoryStop           ((uint16_t)0x56)
+#define PPSMC_CACHistoryStart               ((uint16_t)0x57)
+#define PPSMC_CACHistoryStop                ((uint16_t)0x58)
+#define PPSMC_TDPClampingActive             ((uint16_t)0x59)
+#define PPSMC_TDPClampingInactive           ((uint16_t)0x5A)
+#define PPSMC_StartFanControl               ((uint16_t)0x5B)
+#define PPSMC_StopFanControl                ((uint16_t)0x5C)
+#define PPSMC_NoDisplay                     ((uint16_t)0x5D)
+#define PPSMC_HasDisplay                    ((uint16_t)0x5E)
+#define PPSMC_MSG_UVDPowerOFF               ((uint16_t)0x60)
+#define PPSMC_MSG_UVDPowerON                ((uint16_t)0x61)
+#define PPSMC_MSG_EnableULV                 ((uint16_t)0x62)
+#define PPSMC_MSG_DisableULV                ((uint16_t)0x63)
+#define PPSMC_MSG_EnterULV                  ((uint16_t)0x64)
+#define PPSMC_MSG_ExitULV                   ((uint16_t)0x65)
+#define PPSMC_PowerShiftActive              ((uint16_t)0x6A)
+#define PPSMC_PowerShiftInactive            ((uint16_t)0x6B)
+#define PPSMC_OCPActive                     ((uint16_t)0x6C)
+#define PPSMC_OCPInactive                   ((uint16_t)0x6D)
+#define PPSMC_CACLongTermAvgEnable          ((uint16_t)0x6E)
+#define PPSMC_CACLongTermAvgDisable         ((uint16_t)0x6F)
+#define PPSMC_MSG_InferredStateSweep_Start  ((uint16_t)0x70)
+#define PPSMC_MSG_InferredStateSweep_Stop   ((uint16_t)0x71)
+#define PPSMC_MSG_SwitchToLowestInfState    ((uint16_t)0x72)
+#define PPSMC_MSG_SwitchToNonInfState       ((uint16_t)0x73)
+#define PPSMC_MSG_AllStateSweep_Start       ((uint16_t)0x74)
+#define PPSMC_MSG_AllStateSweep_Stop        ((uint16_t)0x75)
+#define PPSMC_MSG_SwitchNextLowerInfState   ((uint16_t)0x76)
+#define PPSMC_MSG_SwitchNextHigherInfState  ((uint16_t)0x77)
+#define PPSMC_MSG_MclkRetrainingTest        ((uint16_t)0x78)
+#define PPSMC_MSG_ForceTDPClamping          ((uint16_t)0x79)
+#define PPSMC_MSG_CollectCAC_PowerCorreln   ((uint16_t)0x7A)
+#define PPSMC_MSG_CollectCAC_WeightCalib    ((uint16_t)0x7B)
+#define PPSMC_MSG_CollectCAC_SQonly         ((uint16_t)0x7C)
+#define PPSMC_MSG_CollectCAC_TemperaturePwr ((uint16_t)0x7D)
+
+#define PPSMC_MSG_ExtremitiesTest_Start     ((uint16_t)0x7E)
+#define PPSMC_MSG_ExtremitiesTest_Stop      ((uint16_t)0x7F)
+#define PPSMC_FlushDataCache                ((uint16_t)0x80)
+#define PPSMC_FlushInstrCache               ((uint16_t)0x81)
+
+#define PPSMC_MSG_SetEnabledLevels          ((uint16_t)0x82)
+#define PPSMC_MSG_SetForcedLevels           ((uint16_t)0x83)
+
+#define PPSMC_MSG_ResetToDefaults           ((uint16_t)0x84)
+
+#define PPSMC_MSG_SetForcedLevelsAndJump      ((uint16_t)0x85)
+#define PPSMC_MSG_SetCACHistoryMode           ((uint16_t)0x86)
+#define PPSMC_MSG_EnableDTE                   ((uint16_t)0x87)
+#define PPSMC_MSG_DisableDTE                  ((uint16_t)0x88)
+
+#define PPSMC_MSG_SmcSpaceSetAddress          ((uint16_t)0x89)
+#define PPSM_MSG_SmcSpaceWriteDWordInc        ((uint16_t)0x8A)
+#define PPSM_MSG_SmcSpaceWriteWordInc         ((uint16_t)0x8B)
+#define PPSM_MSG_SmcSpaceWriteByteInc         ((uint16_t)0x8C)
+
+#define PPSMC_MSG_BREAK                       ((uint16_t)0xF8)
+
+#define PPSMC_MSG_Test                        ((uint16_t) 0x100)
+#define PPSMC_MSG_DPM_Voltage_Pwrmgt          ((uint16_t) 0x101)
+#define PPSMC_MSG_DPM_Config                  ((uint16_t) 0x102)
+#define PPSMC_MSG_PM_Controller_Start         ((uint16_t) 0x103)
+#define PPSMC_MSG_DPM_ForceState              ((uint16_t) 0x104)
+#define PPSMC_MSG_PG_PowerDownSIMD            ((uint16_t) 0x105)
+#define PPSMC_MSG_PG_PowerUpSIMD              ((uint16_t) 0x106)
+#define PPSMC_MSG_PM_Controller_Stop          ((uint16_t) 0x107)
+#define PPSMC_MSG_PG_SIMD_Config              ((uint16_t) 0x108)
+#define PPSMC_MSG_Voltage_Cntl_Enable         ((uint16_t) 0x109)
+#define PPSMC_MSG_Thermal_Cntl_Enable         ((uint16_t) 0x10a)
+#define PPSMC_MSG_Reset_Service               ((uint16_t) 0x10b)
+#define PPSMC_MSG_VCEPowerOFF                 ((uint16_t) 0x10e)
+#define PPSMC_MSG_VCEPowerON                  ((uint16_t) 0x10f)
+#define PPSMC_MSG_DPM_Disable_VCE_HS          ((uint16_t) 0x110)
+#define PPSMC_MSG_DPM_Enable_VCE_HS           ((uint16_t) 0x111)
+#define PPSMC_MSG_DPM_N_LevelsDisabled        ((uint16_t) 0x112)
+#define PPSMC_MSG_DCEPowerOFF                 ((uint16_t) 0x113)
+#define PPSMC_MSG_DCEPowerON                  ((uint16_t) 0x114)
+#define PPSMC_MSG_PCIE_DDIPowerDown           ((uint16_t) 0x117)
+#define PPSMC_MSG_PCIE_DDIPowerUp             ((uint16_t) 0x118)
+#define PPSMC_MSG_PCIE_CascadePLLPowerDown    ((uint16_t) 0x119)
+#define PPSMC_MSG_PCIE_CascadePLLPowerUp      ((uint16_t) 0x11a)
+#define PPSMC_MSG_SYSPLLPowerOff              ((uint16_t) 0x11b)
+#define PPSMC_MSG_SYSPLLPowerOn               ((uint16_t) 0x11c)
+#define PPSMC_MSG_DCE_RemoveVoltageAdjustment ((uint16_t) 0x11d)
+#define PPSMC_MSG_DCE_AllowVoltageAdjustment  ((uint16_t) 0x11e)
+#define PPSMC_MSG_DISPLAYPHYStatusNotify      ((uint16_t) 0x11f)
+#define PPSMC_MSG_EnableBAPM                  ((uint16_t) 0x120)
+#define PPSMC_MSG_DisableBAPM                 ((uint16_t) 0x121)
+#define PPSMC_MSG_Spmi_Enable                 ((uint16_t) 0x122)
+#define PPSMC_MSG_Spmi_Timer                  ((uint16_t) 0x123)
+#define PPSMC_MSG_LCLK_DPM_Config             ((uint16_t) 0x124)
+#define PPSMC_MSG_VddNB_Request               ((uint16_t) 0x125)
+#define PPSMC_MSG_PCIE_DDIPhyPowerDown        ((uint32_t) 0x126)
+#define PPSMC_MSG_PCIE_DDIPhyPowerUp          ((uint32_t) 0x127)
+#define PPSMC_MSG_MCLKDPM_Config              ((uint16_t) 0x128)
+
+#define PPSMC_MSG_UVDDPM_Config               ((uint16_t) 0x129)
+#define PPSMC_MSG_VCEDPM_Config               ((uint16_t) 0x12A)
+#define PPSMC_MSG_ACPDPM_Config               ((uint16_t) 0x12B)
+#define PPSMC_MSG_SAMUDPM_Config              ((uint16_t) 0x12C)
+#define PPSMC_MSG_UVDDPM_SetEnabledMask       ((uint16_t) 0x12D)
+#define PPSMC_MSG_VCEDPM_SetEnabledMask       ((uint16_t) 0x12E)
+#define PPSMC_MSG_ACPDPM_SetEnabledMask       ((uint16_t) 0x12F)
+#define PPSMC_MSG_SAMUDPM_SetEnabledMask      ((uint16_t) 0x130)
+#define PPSMC_MSG_MCLKDPM_ForceState          ((uint16_t) 0x131)
+#define PPSMC_MSG_MCLKDPM_NoForcedLevel       ((uint16_t) 0x132)
+#define PPSMC_MSG_Thermal_Cntl_Disable        ((uint16_t) 0x133)
+#define PPSMC_MSG_SetTDPLimit                 ((uint16_t) 0x134)
+#define PPSMC_MSG_Voltage_Cntl_Disable        ((uint16_t) 0x135)
+#define PPSMC_MSG_PCIeDPM_Enable              ((uint16_t) 0x136)
+#define PPSMC_MSG_ACPPowerOFF                 ((uint16_t) 0x137)
+#define PPSMC_MSG_ACPPowerON                  ((uint16_t) 0x138)
+#define PPSMC_MSG_SAMPowerOFF                 ((uint16_t) 0x139)
+#define PPSMC_MSG_SAMPowerON                  ((uint16_t) 0x13a)
+#define PPSMC_MSG_SDMAPowerOFF                ((uint16_t) 0x13b)
+#define PPSMC_MSG_SDMAPowerON                 ((uint16_t) 0x13c)
+#define PPSMC_MSG_PCIeDPM_Disable             ((uint16_t) 0x13d)
+#define PPSMC_MSG_IOMMUPowerOFF               ((uint16_t) 0x13e)
+#define PPSMC_MSG_IOMMUPowerON                ((uint16_t) 0x13f)
+#define PPSMC_MSG_NBDPM_Enable                ((uint16_t) 0x140)
+#define PPSMC_MSG_NBDPM_Disable               ((uint16_t) 0x141)
+#define PPSMC_MSG_NBDPM_ForceNominal          ((uint16_t) 0x142)
+#define PPSMC_MSG_NBDPM_ForcePerformance      ((uint16_t) 0x143)
+#define PPSMC_MSG_NBDPM_UnForce               ((uint16_t) 0x144)
+#define PPSMC_MSG_SCLKDPM_SetEnabledMask      ((uint16_t) 0x145)
+#define PPSMC_MSG_MCLKDPM_SetEnabledMask      ((uint16_t) 0x146)
+#define PPSMC_MSG_PCIeDPM_ForceLevel          ((uint16_t) 0x147)
+#define PPSMC_MSG_PCIeDPM_UnForceLevel        ((uint16_t) 0x148)
+#define PPSMC_MSG_EnableACDCGPIOInterrupt     ((uint16_t) 0x149)
+#define PPSMC_MSG_EnableVRHotGPIOInterrupt    ((uint16_t) 0x14a)
+#define PPSMC_MSG_SwitchToAC                  ((uint16_t) 0x14b)
+#define PPSMC_MSG_XDMAPowerOFF                ((uint16_t) 0x14c)
+#define PPSMC_MSG_XDMAPowerON                 ((uint16_t) 0x14d)
+
+#define PPSMC_MSG_DPM_Enable                  ((uint16_t) 0x14e)
+#define PPSMC_MSG_DPM_Disable                 ((uint16_t) 0x14f)
+#define PPSMC_MSG_MCLKDPM_Enable              ((uint16_t) 0x150)
+#define PPSMC_MSG_MCLKDPM_Disable             ((uint16_t) 0x151)
+#define PPSMC_MSG_LCLKDPM_Enable              ((uint16_t) 0x152)
+#define PPSMC_MSG_LCLKDPM_Disable             ((uint16_t) 0x153)
+#define PPSMC_MSG_UVDDPM_Enable               ((uint16_t) 0x154)
+#define PPSMC_MSG_UVDDPM_Disable              ((uint16_t) 0x155)
+#define PPSMC_MSG_SAMUDPM_Enable              ((uint16_t) 0x156)
+#define PPSMC_MSG_SAMUDPM_Disable             ((uint16_t) 0x157)
+#define PPSMC_MSG_ACPDPM_Enable               ((uint16_t) 0x158)
+#define PPSMC_MSG_ACPDPM_Disable              ((uint16_t) 0x159)
+#define PPSMC_MSG_VCEDPM_Enable               ((uint16_t) 0x15a)
+#define PPSMC_MSG_VCEDPM_Disable              ((uint16_t) 0x15b)
+#define PPSMC_MSG_LCLKDPM_SetEnabledMask      ((uint16_t) 0x15c)
+#define PPSMC_MSG_DPM_FPS_Mode                ((uint16_t) 0x15d)
+#define PPSMC_MSG_DPM_Activity_Mode           ((uint16_t) 0x15e)
+#define PPSMC_MSG_VddC_Request                ((uint16_t) 0x15f)
+#define PPSMC_MSG_MCLKDPM_GetEnabledMask      ((uint16_t) 0x160)
+#define PPSMC_MSG_LCLKDPM_GetEnabledMask      ((uint16_t) 0x161)
+#define PPSMC_MSG_SCLKDPM_GetEnabledMask      ((uint16_t) 0x162)
+#define PPSMC_MSG_UVDDPM_GetEnabledMask       ((uint16_t) 0x163)
+#define PPSMC_MSG_SAMUDPM_GetEnabledMask      ((uint16_t) 0x164)
+#define PPSMC_MSG_ACPDPM_GetEnabledMask       ((uint16_t) 0x165)
+#define PPSMC_MSG_VCEDPM_GetEnabledMask       ((uint16_t) 0x166)
+#define PPSMC_MSG_PCIeDPM_SetEnabledMask      ((uint16_t) 0x167)
+#define PPSMC_MSG_PCIeDPM_GetEnabledMask      ((uint16_t) 0x168)
+#define PPSMC_MSG_TDCLimitEnable              ((uint16_t) 0x169)
+#define PPSMC_MSG_TDCLimitDisable             ((uint16_t) 0x16a)
+#define PPSMC_MSG_DPM_AutoRotate_Mode         ((uint16_t) 0x16b)
+#define PPSMC_MSG_DISPCLK_FROM_FCH            ((uint16_t) 0x16c)
+#define PPSMC_MSG_DISPCLK_FROM_DFS            ((uint16_t) 0x16d)
+#define PPSMC_MSG_DPREFCLK_FROM_FCH           ((uint16_t) 0x16e)
+#define PPSMC_MSG_DPREFCLK_FROM_DFS           ((uint16_t) 0x16f)
+#define PPSMC_MSG_PmStatusLogStart            ((uint16_t) 0x170)
+#define PPSMC_MSG_PmStatusLogSample           ((uint16_t) 0x171)
+#define PPSMC_MSG_SCLK_AutoDPM_ON             ((uint16_t) 0x172)
+#define PPSMC_MSG_MCLK_AutoDPM_ON             ((uint16_t) 0x173)
+#define PPSMC_MSG_LCLK_AutoDPM_ON             ((uint16_t) 0x174)
+#define PPSMC_MSG_UVD_AutoDPM_ON              ((uint16_t) 0x175)
+#define PPSMC_MSG_SAMU_AutoDPM_ON             ((uint16_t) 0x176)
+#define PPSMC_MSG_ACP_AutoDPM_ON              ((uint16_t) 0x177)
+#define PPSMC_MSG_VCE_AutoDPM_ON              ((uint16_t) 0x178)
+#define PPSMC_MSG_PCIe_AutoDPM_ON             ((uint16_t) 0x179)
+#define PPSMC_MSG_MASTER_AutoDPM_ON           ((uint16_t) 0x17a)
+#define PPSMC_MSG_MASTER_AutoDPM_OFF          ((uint16_t) 0x17b)
+#define PPSMC_MSG_DYNAMICDISPPHYPOWER         ((uint16_t) 0x17c)
+#define PPSMC_MSG_CAC_COLLECTION_ON           ((uint16_t) 0x17d)
+#define PPSMC_MSG_CAC_COLLECTION_OFF          ((uint16_t) 0x17e)
+#define PPSMC_MSG_CAC_CORRELATION_ON          ((uint16_t) 0x17f)
+#define PPSMC_MSG_CAC_CORRELATION_OFF         ((uint16_t) 0x180)
+#define PPSMC_MSG_PM_STATUS_TO_DRAM_ON        ((uint16_t) 0x181)
+#define PPSMC_MSG_PM_STATUS_TO_DRAM_OFF       ((uint16_t) 0x182)
+#define PPSMC_MSG_ALLOW_LOWSCLK_INTERRUPT     ((uint16_t) 0x184)
+#define PPSMC_MSG_PkgPwrLimitEnable           ((uint16_t) 0x185)
+#define PPSMC_MSG_PkgPwrLimitDisable          ((uint16_t) 0x186)
+#define PPSMC_MSG_PkgPwrSetLimit              ((uint16_t) 0x187)
+#define PPSMC_MSG_OverDriveSetTargetTdp       ((uint16_t) 0x188)
+#define PPSMC_MSG_SCLKDPM_FreezeLevel         ((uint16_t) 0x189)
+#define PPSMC_MSG_SCLKDPM_UnfreezeLevel       ((uint16_t) 0x18A)
+#define PPSMC_MSG_MCLKDPM_FreezeLevel         ((uint16_t) 0x18B)
+#define PPSMC_MSG_MCLKDPM_UnfreezeLevel       ((uint16_t) 0x18C)
+#define PPSMC_MSG_START_DRAM_LOGGING          ((uint16_t) 0x18D)
+#define PPSMC_MSG_STOP_DRAM_LOGGING           ((uint16_t) 0x18E)
+#define PPSMC_MSG_MASTER_DeepSleep_ON         ((uint16_t) 0x18F)
+#define PPSMC_MSG_MASTER_DeepSleep_OFF        ((uint16_t) 0x190)
+#define PPSMC_MSG_Remove_DC_Clamp             ((uint16_t) 0x191)
+#define PPSMC_MSG_DisableACDCGPIOInterrupt    ((uint16_t) 0x192)
+#define PPSMC_MSG_OverrideVoltageControl_SetVddc       ((uint16_t) 0x193)
+#define PPSMC_MSG_OverrideVoltageControl_SetVddci      ((uint16_t) 0x194)
+#define PPSMC_MSG_SetVidOffset_1              ((uint16_t) 0x195)
+#define PPSMC_MSG_SetVidOffset_2              ((uint16_t) 0x207)
+#define PPSMC_MSG_GetVidOffset_1              ((uint16_t) 0x196)
+#define PPSMC_MSG_GetVidOffset_2              ((uint16_t) 0x208)
+#define PPSMC_MSG_THERMAL_OVERDRIVE_Enable    ((uint16_t) 0x197)
+#define PPSMC_MSG_THERMAL_OVERDRIVE_Disable   ((uint16_t) 0x198)
+#define PPSMC_MSG_SetTjMax                    ((uint16_t) 0x199)
+#define PPSMC_MSG_SetFanPwmMax                ((uint16_t) 0x19A)
+#define PPSMC_MSG_WaitForMclkSwitchFinish     ((uint16_t) 0x19B)
+#define PPSMC_MSG_ENABLE_THERMAL_DPM          ((uint16_t) 0x19C)
+#define PPSMC_MSG_DISABLE_THERMAL_DPM         ((uint16_t) 0x19D)
+
+#define PPSMC_MSG_API_GetSclkFrequency        ((uint16_t) 0x200)
+#define PPSMC_MSG_API_GetMclkFrequency        ((uint16_t) 0x201)
+#define PPSMC_MSG_API_GetSclkBusy             ((uint16_t) 0x202)
+#define PPSMC_MSG_API_GetMclkBusy             ((uint16_t) 0x203)
+#define PPSMC_MSG_API_GetAsicPower            ((uint16_t) 0x204)
+#define PPSMC_MSG_SetFanRpmMax                ((uint16_t) 0x205)
+#define PPSMC_MSG_SetFanSclkTarget            ((uint16_t) 0x206)
+#define PPSMC_MSG_SetFanMinPwm                ((uint16_t) 0x209)
+#define PPSMC_MSG_SetFanTemperatureTarget     ((uint16_t) 0x20A)
+
+#define PPSMC_MSG_BACO_StartMonitor           ((uint16_t) 0x240)
+#define PPSMC_MSG_BACO_Cancel                 ((uint16_t) 0x241)
+#define PPSMC_MSG_EnableVddGfx                ((uint16_t) 0x242)
+#define PPSMC_MSG_DisableVddGfx               ((uint16_t) 0x243)
+#define PPSMC_MSG_UcodeAddressLow             ((uint16_t) 0x244)
+#define PPSMC_MSG_UcodeAddressHigh            ((uint16_t) 0x245)
+#define PPSMC_MSG_UcodeLoadStatus             ((uint16_t) 0x246)
+
+#define PPSMC_MSG_DRV_DRAM_ADDR_HI            ((uint16_t) 0x250)
+#define PPSMC_MSG_DRV_DRAM_ADDR_LO            ((uint16_t) 0x251)
+#define PPSMC_MSG_SMU_DRAM_ADDR_HI            ((uint16_t) 0x252)
+#define PPSMC_MSG_SMU_DRAM_ADDR_LO            ((uint16_t) 0x253)
+#define PPSMC_MSG_LoadUcodes                  ((uint16_t) 0x254)
+#define PPSMC_MSG_PowerStateNotify            ((uint16_t) 0x255)
+#define PPSMC_MSG_COND_EXEC_DRAM_ADDR_HI      ((uint16_t) 0x256)
+#define PPSMC_MSG_COND_EXEC_DRAM_ADDR_LO      ((uint16_t) 0x257)
+#define PPSMC_MSG_VBIOS_DRAM_ADDR_HI          ((uint16_t) 0x258)
+#define PPSMC_MSG_VBIOS_DRAM_ADDR_LO          ((uint16_t) 0x259)
+#define PPSMC_MSG_LoadVBios                   ((uint16_t) 0x25A)
+#define PPSMC_MSG_GetUcodeVersion             ((uint16_t) 0x25B)
+#define DMCUSMC_MSG_PSREntry                  ((uint16_t) 0x25C)
+#define DMCUSMC_MSG_PSRExit                   ((uint16_t) 0x25D)
+#define PPSMC_MSG_EnableClockGatingFeature    ((uint16_t) 0x260)
+#define PPSMC_MSG_DisableClockGatingFeature   ((uint16_t) 0x261)
+#define PPSMC_MSG_IsDeviceRunning             ((uint16_t) 0x262)
+#define PPSMC_MSG_LoadMetaData                ((uint16_t) 0x263)
+#define PPSMC_MSG_TMON_AutoCaliberate_Enable  ((uint16_t) 0x264)
+#define PPSMC_MSG_TMON_AutoCaliberate_Disable ((uint16_t) 0x265)
+#define PPSMC_MSG_GetTelemetry1Slope          ((uint16_t) 0x266)
+#define PPSMC_MSG_GetTelemetry1Offset         ((uint16_t) 0x267)
+#define PPSMC_MSG_GetTelemetry2Slope          ((uint16_t) 0x268)
+#define PPSMC_MSG_GetTelemetry2Offset         ((uint16_t) 0x269)
+#define PPSMC_MSG_EnableAvfs                  ((uint16_t) 0x26A)
+#define PPSMC_MSG_DisableAvfs                 ((uint16_t) 0x26B)
+
+#define PPSMC_MSG_PerformBtc                  ((uint16_t) 0x26C)
+#define PPSMC_MSG_VftTableIsValid             ((uint16_t) 0x275)
+#define PPSMC_MSG_UseNewGPIOScheme            ((uint16_t) 0x277)
+#define PPSMC_MSG_GetEnabledPsm               ((uint16_t) 0x400)
+#define PPSMC_MSG_AgmStartPsm                 ((uint16_t) 0x401)
+#define PPSMC_MSG_AgmReadPsm                  ((uint16_t) 0x402)
+#define PPSMC_MSG_AgmResetPsm                 ((uint16_t) 0x403)
+#define PPSMC_MSG_ReadVftCell                 ((uint16_t) 0x404)
+
+#define PPSMC_MSG_GFX_CU_PG_ENABLE            ((uint16_t) 0x280)
+#define PPSMC_MSG_GFX_CU_PG_DISABLE           ((uint16_t) 0x281)
+#define PPSMC_MSG_GetCurrPkgPwr               ((uint16_t) 0x282)
+
+#define PPSMC_MSG_SetGpuPllDfsForSclk         ((uint16_t) 0x300)
+#define PPSMC_MSG_Didt_Block_Function		  ((uint16_t) 0x301)
+
+#define PPSMC_MSG_SetVBITimeout               ((uint16_t) 0x306)
+
+#define PPSMC_MSG_SecureSRBMWrite             ((uint16_t) 0x600)
+#define PPSMC_MSG_SecureSRBMRead              ((uint16_t) 0x601)
+#define PPSMC_MSG_SetAddress                  ((uint16_t) 0x800)
+#define PPSMC_MSG_GetData                     ((uint16_t) 0x801)
+#define PPSMC_MSG_SetData                     ((uint16_t) 0x802)
+
+typedef uint16_t PPSMC_Msg;
+
+#define PPSMC_EVENT_STATUS_THERMAL          0x00000001
+#define PPSMC_EVENT_STATUS_REGULATORHOT     0x00000002
+#define PPSMC_EVENT_STATUS_DC               0x00000004
+
+#pragma pack(pop)
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
index 3c235f0177cd..2139072065cc 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
@@ -28,6 +28,7 @@
 
 struct pp_smumgr;
 struct pp_instance;
+struct pp_hwmgr;
 
 #define smu_lower_32_bits(n) ((uint32_t)(n))
 #define smu_upper_32_bits(n) ((uint32_t)(((n)>>16)>>16))
@@ -53,6 +54,45 @@ enum AVFS_BTC_STATUS {
 	AVFS_BTC_SMUMSG_ERROR
 };
 
+enum SMU_TABLE {
+	SMU_UVD_TABLE = 0,
+	SMU_VCE_TABLE,
+	SMU_SAMU_TABLE,
+	SMU_BIF_TABLE,
+};
+
+enum SMU_TYPE {
+	SMU_SoftRegisters = 0,
+	SMU_Discrete_DpmTable,
+};
+
+enum SMU_MEMBER {
+	HandshakeDisables = 0,
+	VoltageChangeTimeout,
+	AverageGraphicsActivity,
+	PreVBlankGap,
+	VBlankTimeout,
+	UcodeLoadStatus,
+	UvdBootLevel,
+	VceBootLevel,
+	SamuBootLevel,
+	LowSclkInterruptThreshold,
+};
+
+
+enum SMU_MAC_DEFINITION {
+	SMU_MAX_LEVELS_GRAPHICS = 0,
+	SMU_MAX_LEVELS_MEMORY,
+	SMU_MAX_LEVELS_LINK,
+	SMU_MAX_ENTRIES_SMIO,
+	SMU_MAX_LEVELS_VDDC,
+	SMU_MAX_LEVELS_VDDGFX,
+	SMU_MAX_LEVELS_VDDCI,
+	SMU_MAX_LEVELS_MVDD,
+	SMU_UVD_MCLK_HANDSHAKE_DISABLE,
+};
+
+
 struct pp_smumgr_func {
 	int (*smu_init)(struct pp_smumgr *smumgr);
 	int (*smu_fini)(struct pp_smumgr *smumgr);
@@ -69,12 +109,23 @@ struct pp_smumgr_func {
 	int (*download_pptable_settings)(struct pp_smumgr *smumgr,
 					 void **table);
 	int (*upload_pptable_settings)(struct pp_smumgr *smumgr);
+	int (*update_smc_table)(struct pp_hwmgr *hwmgr, uint32_t type);
+	int (*process_firmware_header)(struct pp_hwmgr *hwmgr);
+	int (*update_sclk_threshold)(struct pp_hwmgr *hwmgr);
+	int (*thermal_setup_fan_table)(struct pp_hwmgr *hwmgr);
+	int (*thermal_avfs_enable)(struct pp_hwmgr *hwmgr);
+	int (*init_smc_table)(struct pp_hwmgr *hwmgr);
+	int (*populate_all_graphic_levels)(struct pp_hwmgr *hwmgr);
+	int (*populate_all_memory_levels)(struct pp_hwmgr *hwmgr);
+	int (*initialize_mc_reg_table)(struct pp_hwmgr *hwmgr);
+	uint32_t (*get_offsetof)(uint32_t type, uint32_t member);
+	uint32_t (*get_mac_definition)(uint32_t value);
+	bool (*is_dpm_running)(struct pp_hwmgr *hwmgr);
 };
 
 struct pp_smumgr {
 	uint32_t chip_family;
 	uint32_t chip_id;
-	uint32_t hw_revision;
 	void *device;
 	void *backend;
 	uint32_t usec_timeout;
@@ -122,6 +173,30 @@ extern int smu_allocate_memory(void *device, uint32_t size,
 
 extern int smu_free_memory(void *device, void *handle);
 
+extern int cz_smum_init(struct pp_smumgr *smumgr);
+extern int iceland_smum_init(struct pp_smumgr *smumgr);
+extern int tonga_smum_init(struct pp_smumgr *smumgr);
+extern int fiji_smum_init(struct pp_smumgr *smumgr);
+extern int polaris10_smum_init(struct pp_smumgr *smumgr);
+
+extern int smum_update_sclk_threshold(struct pp_hwmgr *hwmgr);
+
+extern int smum_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type);
+extern int smum_process_firmware_header(struct pp_hwmgr *hwmgr);
+extern int smum_thermal_avfs_enable(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result);
+extern int smum_thermal_setup_fan_table(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result);
+extern int smum_init_smc_table(struct pp_hwmgr *hwmgr);
+extern int smum_populate_all_graphic_levels(struct pp_hwmgr *hwmgr);
+extern int smum_populate_all_memory_levels(struct pp_hwmgr *hwmgr);
+extern int smum_initialize_mc_reg_table(struct pp_hwmgr *hwmgr);
+extern uint32_t smum_get_offsetof(struct pp_smumgr *smumgr,
+				uint32_t type, uint32_t member);
+extern uint32_t smum_get_mac_definition(struct pp_smumgr *smumgr, uint32_t value);
+
+extern bool smum_is_dpm_running(struct pp_hwmgr *hwmgr);
+
 #define SMUM_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
 
 #define SMUM_FIELD_MASK(reg, field) reg##__##field##_MASK
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/Makefile b/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
index f10fb64ef981..51ff08301651 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/Makefile
@@ -2,7 +2,9 @@
 # Makefile for the 'smu manager' sub-component of powerplay.
 # It provides the smu management services for the driver.
 
-SMU_MGR = smumgr.o cz_smumgr.o tonga_smumgr.o fiji_smumgr.o polaris10_smumgr.o
+SMU_MGR = smumgr.o cz_smumgr.o tonga_smumgr.o fiji_smumgr.o fiji_smc.o \
+	  polaris10_smumgr.o iceland_smumgr.o polaris10_smc.o tonga_smc.o \
+	  smu7_smumgr.o iceland_smc.o
 
 AMD_PP_SMUMGR = $(addprefix $(AMD_PP_PATH)/smumgr/,$(SMU_MGR))
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
index 87c023e518ab..5a44485526d2 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
@@ -89,13 +89,8 @@ static int cz_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
 	if (result != 0)
 		return result;
 
-	result = SMUM_WAIT_FIELD_UNEQUAL(smumgr,
+	return SMUM_WAIT_FIELD_UNEQUAL(smumgr,
 					SMU_MP1_SRBM2P_RESP_0, CONTENT, 0);
-
-	if (result != 0)
-		return result;
-
-	return 0;
 }
 
 static int cz_set_smc_sram_address(struct pp_smumgr *smumgr,
@@ -106,12 +101,12 @@ static int cz_set_smc_sram_address(struct pp_smumgr *smumgr,
 
 	if (0 != (3 & smc_address)) {
 		printk(KERN_ERR "[ powerplay ] SMC address must be 4 byte aligned\n");
-		return -1;
+		return -EINVAL;
 	}
 
 	if (limit <= (smc_address + 3)) {
 		printk(KERN_ERR "[ powerplay ] SMC address beyond the SMC RAM area\n");
-		return -1;
+		return -EINVAL;
 	}
 
 	cgs_write_register(smumgr->device, mmMP0PUB_IND_INDEX_0,
@@ -129,9 +124,10 @@ static int cz_write_smc_sram_dword(struct pp_smumgr *smumgr,
 		return -EINVAL;
 
 	result = cz_set_smc_sram_address(smumgr, smc_address, limit);
-	cgs_write_register(smumgr->device, mmMP0PUB_IND_DATA_0, value);
+	if (!result)
+		cgs_write_register(smumgr->device, mmMP0PUB_IND_DATA_0, value);
 
-	return 0;
+	return result;
 }
 
 static int cz_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
@@ -148,7 +144,6 @@ static int cz_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
 static int cz_request_smu_load_fw(struct pp_smumgr *smumgr)
 {
 	struct cz_smumgr *cz_smu = (struct cz_smumgr *)(smumgr->backend);
-	int result = 0;
 	uint32_t smc_address;
 
 	if (!smumgr->reload_fw) {
@@ -177,11 +172,9 @@ static int cz_request_smu_load_fw(struct pp_smumgr *smumgr)
 	cz_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_ExecuteJob,
 				cz_smu->toc_entry_power_profiling_index);
 
-	result = cz_send_msg_to_smc_with_parameter(smumgr,
+	return cz_send_msg_to_smc_with_parameter(smumgr,
 					PPSMC_MSG_ExecuteJob,
 					cz_smu->toc_entry_initialize_index);
-
-	return result;
 }
 
 static int cz_check_fw_load_finish(struct pp_smumgr *smumgr,
@@ -195,9 +188,6 @@ static int cz_check_fw_load_finish(struct pp_smumgr *smumgr,
 	if (smumgr == NULL || smumgr->device == NULL)
 		return -EINVAL;
 
-	return cgs_read_register(smumgr->device,
-					mmSMU_MP1_SRBM2P_ARG_0);
-
 	cgs_write_register(smumgr->device, mmMP0PUB_IND_INDEX, index);
 
 	for (i = 0; i < smumgr->usec_timeout; i++) {
@@ -275,7 +265,10 @@ static int cz_start_smu(struct pp_smumgr *smumgr)
 	if (smumgr->chip_id == CHIP_STONEY)
 		fw_to_check &= ~(UCODE_ID_SDMA1_MASK | UCODE_ID_CP_MEC_JT2_MASK);
 
-	cz_request_smu_load_fw(smumgr);
+	ret = cz_request_smu_load_fw(smumgr);
+	if (ret)
+		printk(KERN_ERR "[ powerplay] SMU firmware load failed\n");
+
 	cz_check_fw_load_finish(smumgr, fw_to_check);
 
 	ret = cz_load_mec_firmware(smumgr);
@@ -566,10 +559,7 @@ static int cz_smu_construct_toc_for_bootup(struct pp_smumgr *smumgr)
 
 	cz_smu_populate_single_ucode_load_task(smumgr,
 				CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0, false);
-	if (smumgr->chip_id == CHIP_STONEY)
-		cz_smu_populate_single_ucode_load_task(smumgr,
-				CZ_SCRATCH_ENTRY_UCODE_ID_SDMA0, false);
-	else
+	if (smumgr->chip_id != CHIP_STONEY)
 		cz_smu_populate_single_ucode_load_task(smumgr,
 				CZ_SCRATCH_ENTRY_UCODE_ID_SDMA1, false);
 	cz_smu_populate_single_ucode_load_task(smumgr,
@@ -580,10 +570,7 @@ static int cz_smu_construct_toc_for_bootup(struct pp_smumgr *smumgr)
 				CZ_SCRATCH_ENTRY_UCODE_ID_CP_ME, false);
 	cz_smu_populate_single_ucode_load_task(smumgr,
 				CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
-	if (smumgr->chip_id == CHIP_STONEY)
-		cz_smu_populate_single_ucode_load_task(smumgr,
-				CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT1, false);
-	else
+	if (smumgr->chip_id != CHIP_STONEY)
 		cz_smu_populate_single_ucode_load_task(smumgr,
 				CZ_SCRATCH_ENTRY_UCODE_ID_CP_MEC_JT2, false);
 	cz_smu_populate_single_ucode_load_task(smumgr,
@@ -610,19 +597,12 @@ static int cz_smu_construct_toc(struct pp_smumgr *smumgr)
 	struct cz_smumgr *cz_smu = (struct cz_smumgr *)smumgr->backend;
 
 	cz_smu->toc_entry_used_count = 0;
-
 	cz_smu_initialize_toc_empty_job_list(smumgr);
-
 	cz_smu_construct_toc_for_rlc_aram_save(smumgr);
-
 	cz_smu_construct_toc_for_vddgfx_enter(smumgr);
-
 	cz_smu_construct_toc_for_vddgfx_exit(smumgr);
-
 	cz_smu_construct_toc_for_power_profiling(smumgr);
-
 	cz_smu_construct_toc_for_bootup(smumgr);
-
 	cz_smu_construct_toc_for_clock_table(smumgr);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
new file mode 100644
index 000000000000..76310ac7ef0d
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
@@ -0,0 +1,2374 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "fiji_smc.h"
+#include "smu7_dyn_defaults.h"
+
+#include "smu7_hwmgr.h"
+#include "hardwaremanager.h"
+#include "ppatomctrl.h"
+#include "pp_debug.h"
+#include "cgs_common.h"
+#include "atombios.h"
+#include "fiji_smumgr.h"
+#include "pppcielanes.h"
+#include "smu7_ppsmc.h"
+#include "smu73.h"
+#include "smu/smu_7_1_3_d.h"
+#include "smu/smu_7_1_3_sh_mask.h"
+#include "gmc/gmc_8_1_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "smu7_smumgr.h"
+
+#define VOLTAGE_SCALE 4
+#define POWERTUNE_DEFAULT_SET_MAX    1
+#define VOLTAGE_VID_OFFSET_SCALE1   625
+#define VOLTAGE_VID_OFFSET_SCALE2   100
+#define VDDC_VDDCI_DELTA            300
+#define MC_CG_ARB_FREQ_F1           0x0b
+
+/* [2.5%,~2.5%] Clock stretched is multiple of 2.5% vs
+ * not and [Fmin, Fmax, LDO_REFSEL, USE_FOR_LOW_FREQ]
+ */
+static const uint16_t fiji_clock_stretcher_lookup_table[2][4] = {
+				{600, 1050, 3, 0}, {600, 1050, 6, 1} };
+
+/* [FF, SS] type, [] 4 voltage ranges, and
+ * [Floor Freq, Boundary Freq, VID min , VID max]
+ */
+static const uint32_t fiji_clock_stretcher_ddt_table[2][4][4] = {
+	{ {265, 529, 120, 128}, {325, 650, 96, 119}, {430, 860, 32, 95}, {0, 0, 0, 31} },
+	{ {275, 550, 104, 112}, {319, 638, 96, 103}, {360, 720, 64, 95}, {384, 768, 32, 63} } };
+
+/* [Use_For_Low_freq] value, [0%, 5%, 10%, 7.14%, 14.28%, 20%]
+ * (coming from PWR_CKS_CNTL.stretch_amount reg spec)
+ */
+static const uint8_t fiji_clock_stretch_amount_conversion[2][6] = {
+				{0, 1, 3, 2, 4, 5}, {0, 2, 4, 5, 6, 5} };
+
+static const struct fiji_pt_defaults fiji_power_tune_data_set_array[POWERTUNE_DEFAULT_SET_MAX] = {
+		/*sviLoadLIneEn,  SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc */
+		{1,               0xF,             0xFD,
+		/* TDC_MAWt, TdcWaterfallCtl, DTEAmbientTempBase */
+		0x19,        5,               45}
+};
+
+/* PPGen has the gain setting generated in x * 100 unit
+ * This function is to convert the unit to x * 4096(0x1000) unit.
+ *  This is the unit expected by SMC firmware
+ */
+static int fiji_get_dependency_volt_by_clk(struct pp_hwmgr *hwmgr,
+		struct phm_ppt_v1_clock_voltage_dependency_table *dep_table,
+		uint32_t clock, uint32_t *voltage, uint32_t *mvdd)
+{
+	uint32_t i;
+	uint16_t vddci;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	*voltage = *mvdd = 0;
+
+
+	/* clock - voltage dependency table is empty table */
+	if (dep_table->count == 0)
+		return -EINVAL;
+
+	for (i = 0; i < dep_table->count; i++) {
+		/* find first sclk bigger than request */
+		if (dep_table->entries[i].clk >= clock) {
+			*voltage |= (dep_table->entries[i].vddc *
+					VOLTAGE_SCALE) << VDDC_SHIFT;
+			if (SMU7_VOLTAGE_CONTROL_NONE == data->vddci_control)
+				*voltage |= (data->vbios_boot_state.vddci_bootup_value *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+			else if (dep_table->entries[i].vddci)
+				*voltage |= (dep_table->entries[i].vddci *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+			else {
+				vddci = phm_find_closest_vddci(&(data->vddci_voltage_table),
+						(dep_table->entries[i].vddc -
+								VDDC_VDDCI_DELTA));
+				*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+			}
+
+			if (SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control)
+				*mvdd = data->vbios_boot_state.mvdd_bootup_value *
+					VOLTAGE_SCALE;
+			else if (dep_table->entries[i].mvdd)
+				*mvdd = (uint32_t) dep_table->entries[i].mvdd *
+					VOLTAGE_SCALE;
+
+			*voltage |= 1 << PHASES_SHIFT;
+			return 0;
+		}
+	}
+
+	/* sclk is bigger than max sclk in the dependence table */
+	*voltage |= (dep_table->entries[i - 1].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
+
+	if (SMU7_VOLTAGE_CONTROL_NONE == data->vddci_control)
+		*voltage |= (data->vbios_boot_state.vddci_bootup_value *
+				VOLTAGE_SCALE) << VDDCI_SHIFT;
+	else if (dep_table->entries[i-1].vddci) {
+		vddci = phm_find_closest_vddci(&(data->vddci_voltage_table),
+				(dep_table->entries[i].vddc -
+						VDDC_VDDCI_DELTA));
+		*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+	}
+
+	if (SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control)
+		*mvdd = data->vbios_boot_state.mvdd_bootup_value * VOLTAGE_SCALE;
+	else if (dep_table->entries[i].mvdd)
+		*mvdd = (uint32_t) dep_table->entries[i - 1].mvdd * VOLTAGE_SCALE;
+
+	return 0;
+}
+
+
+static uint16_t scale_fan_gain_settings(uint16_t raw_setting)
+{
+	uint32_t tmp;
+	tmp = raw_setting * 4096 / 100;
+	return (uint16_t)tmp;
+}
+
+static void get_scl_sda_value(uint8_t line, uint8_t *scl, uint8_t *sda)
+{
+	switch (line) {
+	case SMU7_I2CLineID_DDC1:
+		*scl = SMU7_I2C_DDC1CLK;
+		*sda = SMU7_I2C_DDC1DATA;
+		break;
+	case SMU7_I2CLineID_DDC2:
+		*scl = SMU7_I2C_DDC2CLK;
+		*sda = SMU7_I2C_DDC2DATA;
+		break;
+	case SMU7_I2CLineID_DDC3:
+		*scl = SMU7_I2C_DDC3CLK;
+		*sda = SMU7_I2C_DDC3DATA;
+		break;
+	case SMU7_I2CLineID_DDC4:
+		*scl = SMU7_I2C_DDC4CLK;
+		*sda = SMU7_I2C_DDC4DATA;
+		break;
+	case SMU7_I2CLineID_DDC5:
+		*scl = SMU7_I2C_DDC5CLK;
+		*sda = SMU7_I2C_DDC5DATA;
+		break;
+	case SMU7_I2CLineID_DDC6:
+		*scl = SMU7_I2C_DDC6CLK;
+		*sda = SMU7_I2C_DDC6DATA;
+		break;
+	case SMU7_I2CLineID_SCLSDA:
+		*scl = SMU7_I2C_SCL;
+		*sda = SMU7_I2C_SDA;
+		break;
+	case SMU7_I2CLineID_DDCVGA:
+		*scl = SMU7_I2C_DDCVGACLK;
+		*sda = SMU7_I2C_DDCVGADATA;
+		break;
+	default:
+		*scl = 0;
+		*sda = 0;
+		break;
+	}
+}
+
+static void fiji_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct  phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (table_info &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID <= POWERTUNE_DEFAULT_SET_MAX &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID)
+		smu_data->power_tune_defaults =
+				&fiji_power_tune_data_set_array
+				[table_info->cac_dtp_table->usPowerTuneDataSetID - 1];
+	else
+		smu_data->power_tune_defaults = &fiji_power_tune_data_set_array[0];
+
+}
+
+static int fiji_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr)
+{
+
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	const struct fiji_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	SMU73_Discrete_DpmTable  *dpm_table = &(smu_data->smc_state_table);
+
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_cac_tdp_table *cac_dtp_table = table_info->cac_dtp_table;
+	struct pp_advance_fan_control_parameters *fan_table =
+			&hwmgr->thermal_controller.advanceFanControlParameters;
+	uint8_t uc_scl, uc_sda;
+
+	/* TDP number of fraction bits are changed from 8 to 7 for Fiji
+	 * as requested by SMC team
+	 */
+	dpm_table->DefaultTdp = PP_HOST_TO_SMC_US(
+			(uint16_t)(cac_dtp_table->usTDP * 128));
+	dpm_table->TargetTdp = PP_HOST_TO_SMC_US(
+			(uint16_t)(cac_dtp_table->usTDP * 128));
+
+	PP_ASSERT_WITH_CODE(cac_dtp_table->usTargetOperatingTemp <= 255,
+			"Target Operating Temp is out of Range!",
+			);
+
+	dpm_table->GpuTjMax = (uint8_t)(cac_dtp_table->usTargetOperatingTemp);
+	dpm_table->GpuTjHyst = 8;
+
+	dpm_table->DTEAmbientTempBase = defaults->DTEAmbientTempBase;
+
+	/* The following are for new Fiji Multi-input fan/thermal control */
+	dpm_table->TemperatureLimitEdge = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTargetOperatingTemp * 256);
+	dpm_table->TemperatureLimitHotspot = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitHotspot * 256);
+	dpm_table->TemperatureLimitLiquid1 = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitLiquid1 * 256);
+	dpm_table->TemperatureLimitLiquid2 = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitLiquid2 * 256);
+	dpm_table->TemperatureLimitVrVddc = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitVrVddc * 256);
+	dpm_table->TemperatureLimitVrMvdd = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitVrMvdd * 256);
+	dpm_table->TemperatureLimitPlx = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitPlx * 256);
+
+	dpm_table->FanGainEdge = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainEdge));
+	dpm_table->FanGainHotspot = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainHotspot));
+	dpm_table->FanGainLiquid = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainLiquid));
+	dpm_table->FanGainVrVddc = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainVrVddc));
+	dpm_table->FanGainVrMvdd = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainVrMvdd));
+	dpm_table->FanGainPlx = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainPlx));
+	dpm_table->FanGainHbm = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainHbm));
+
+	dpm_table->Liquid1_I2C_address = cac_dtp_table->ucLiquid1_I2C_address;
+	dpm_table->Liquid2_I2C_address = cac_dtp_table->ucLiquid2_I2C_address;
+	dpm_table->Vr_I2C_address = cac_dtp_table->ucVr_I2C_address;
+	dpm_table->Plx_I2C_address = cac_dtp_table->ucPlx_I2C_address;
+
+	get_scl_sda_value(cac_dtp_table->ucLiquid_I2C_Line, &uc_scl, &uc_sda);
+	dpm_table->Liquid_I2C_LineSCL = uc_scl;
+	dpm_table->Liquid_I2C_LineSDA = uc_sda;
+
+	get_scl_sda_value(cac_dtp_table->ucVr_I2C_Line, &uc_scl, &uc_sda);
+	dpm_table->Vr_I2C_LineSCL = uc_scl;
+	dpm_table->Vr_I2C_LineSDA = uc_sda;
+
+	get_scl_sda_value(cac_dtp_table->ucPlx_I2C_Line, &uc_scl, &uc_sda);
+	dpm_table->Plx_I2C_LineSCL = uc_scl;
+	dpm_table->Plx_I2C_LineSDA = uc_sda;
+
+	return 0;
+}
+
+
+static int fiji_populate_svi_load_line(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	const struct fiji_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	smu_data->power_tune_table.SviLoadLineEn = defaults->SviLoadLineEn;
+	smu_data->power_tune_table.SviLoadLineVddC = defaults->SviLoadLineVddC;
+	smu_data->power_tune_table.SviLoadLineTrimVddC = 3;
+	smu_data->power_tune_table.SviLoadLineOffsetVddC = 0;
+
+	return 0;
+}
+
+
+static int fiji_populate_tdc_limit(struct pp_hwmgr *hwmgr)
+{
+	uint16_t tdc_limit;
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	const struct fiji_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	/* TDC number of fraction bits are changed from 8 to 7
+	 * for Fiji as requested by SMC team
+	 */
+	tdc_limit = (uint16_t)(table_info->cac_dtp_table->usTDC * 128);
+	smu_data->power_tune_table.TDC_VDDC_PkgLimit =
+			CONVERT_FROM_HOST_TO_SMC_US(tdc_limit);
+	smu_data->power_tune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
+			defaults->TDC_VDDC_ThrottleReleaseLimitPerc;
+	smu_data->power_tune_table.TDC_MAWt = defaults->TDC_MAWt;
+
+	return 0;
+}
+
+static int fiji_populate_dw8(struct pp_hwmgr *hwmgr, uint32_t fuse_table_offset)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	const struct fiji_pt_defaults *defaults = smu_data->power_tune_defaults;
+	uint32_t temp;
+
+	if (smu7_read_smc_sram_dword(hwmgr->smumgr,
+			fuse_table_offset +
+			offsetof(SMU73_Discrete_PmFuses, TdcWaterfallCtl),
+			(uint32_t *)&temp, SMC_RAM_END))
+		PP_ASSERT_WITH_CODE(false,
+				"Attempt to read PmFuses.DW6 (SviLoadLineEn) from SMC Failed!",
+				return -EINVAL);
+	else {
+		smu_data->power_tune_table.TdcWaterfallCtl = defaults->TdcWaterfallCtl;
+		smu_data->power_tune_table.LPMLTemperatureMin =
+				(uint8_t)((temp >> 16) & 0xff);
+		smu_data->power_tune_table.LPMLTemperatureMax =
+				(uint8_t)((temp >> 8) & 0xff);
+		smu_data->power_tune_table.Reserved = (uint8_t)(temp & 0xff);
+	}
+	return 0;
+}
+
+static int fiji_populate_temperature_scaler(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		smu_data->power_tune_table.LPMLTemperatureScaler[i] = 0;
+
+	return 0;
+}
+
+static int fiji_populate_fuzzy_fan(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+
+	if ((hwmgr->thermal_controller.advanceFanControlParameters.
+			usFanOutputSensitivity & (1 << 15)) ||
+			0 == hwmgr->thermal_controller.advanceFanControlParameters.
+			usFanOutputSensitivity)
+		hwmgr->thermal_controller.advanceFanControlParameters.
+		usFanOutputSensitivity = hwmgr->thermal_controller.
+			advanceFanControlParameters.usDefaultFanOutputSensitivity;
+
+	smu_data->power_tune_table.FuzzyFan_PwmSetDelta =
+			PP_HOST_TO_SMC_US(hwmgr->thermal_controller.
+					advanceFanControlParameters.usFanOutputSensitivity);
+	return 0;
+}
+
+static int fiji_populate_gnb_lpml(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		smu_data->power_tune_table.GnbLPML[i] = 0;
+
+	return 0;
+}
+
+static int fiji_min_max_vgnb_lpml_id_from_bapm_vddc(struct pp_hwmgr *hwmgr)
+{
+	return 0;
+}
+
+static int fiji_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint16_t HiSidd = smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd;
+	uint16_t LoSidd = smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd;
+	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
+
+	HiSidd = (uint16_t)(cac_table->usHighCACLeakage / 100 * 256);
+	LoSidd = (uint16_t)(cac_table->usLowCACLeakage / 100 * 256);
+
+	smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(HiSidd);
+	smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(LoSidd);
+
+	return 0;
+}
+
+static int fiji_populate_pm_fuses(struct pp_hwmgr *hwmgr)
+{
+	uint32_t pm_fuse_table_offset;
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerContainment)) {
+		if (smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU7_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU73_Firmware_Header, PmFuseTable),
+				&pm_fuse_table_offset, SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to get pm_fuse_table_offset Failed!",
+					return -EINVAL);
+
+		/* DW6 */
+		if (fiji_populate_svi_load_line(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate SviLoadLine Failed!",
+					return -EINVAL);
+		/* DW7 */
+		if (fiji_populate_tdc_limit(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TDCLimit Failed!", return -EINVAL);
+		/* DW8 */
+		if (fiji_populate_dw8(hwmgr, pm_fuse_table_offset))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TdcWaterfallCtl, "
+					"LPMLTemperature Min and Max Failed!",
+					return -EINVAL);
+
+		/* DW9-DW12 */
+		if (0 != fiji_populate_temperature_scaler(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate LPMLTemperatureScaler Failed!",
+					return -EINVAL);
+
+		/* DW13-DW14 */
+		if (fiji_populate_fuzzy_fan(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate Fuzzy Fan Control parameters Failed!",
+					return -EINVAL);
+
+		/* DW15-DW18 */
+		if (fiji_populate_gnb_lpml(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate GnbLPML Failed!",
+					return -EINVAL);
+
+		/* DW19 */
+		if (fiji_min_max_vgnb_lpml_id_from_bapm_vddc(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate GnbLPML Min and Max Vid Failed!",
+					return -EINVAL);
+
+		/* DW20 */
+		if (fiji_populate_bapm_vddc_base_leakage_sidd(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate BapmVddCBaseLeakage Hi and Lo "
+					"Sidd Failed!", return -EINVAL);
+
+		if (smu7_copy_bytes_to_smc(hwmgr->smumgr, pm_fuse_table_offset,
+				(uint8_t *)&smu_data->power_tune_table,
+				sizeof(struct SMU73_Discrete_PmFuses), SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to download PmFuseTable Failed!",
+					return -EINVAL);
+	}
+	return 0;
+}
+
+/**
+* Preparation of vddc and vddgfx CAC tables for SMC.
+*
+* @param    hwmgr  the address of the hardware manager
+* @param    table  the SMC DPM table structure to be populated
+* @return   always 0
+*/
+static int fiji_populate_cac_table(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	uint32_t count;
+	uint8_t index;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_voltage_lookup_table *lookup_table =
+			table_info->vddc_lookup_table;
+	/* tables is already swapped, so in order to use the value from it,
+	 * we need to swap it back.
+	 * We are populating vddc CAC data to BapmVddc table
+	 * in split and merged mode
+	 */
+
+	for (count = 0; count < lookup_table->count; count++) {
+		index = phm_get_voltage_index(lookup_table,
+				data->vddc_voltage_table.entries[count].value);
+		table->BapmVddcVidLoSidd[count] =
+			convert_to_vid(lookup_table->entries[index].us_cac_low);
+		table->BapmVddcVidHiSidd[count] =
+			convert_to_vid(lookup_table->entries[index].us_cac_high);
+	}
+
+	return 0;
+}
+
+/**
+* Preparation of voltage tables for SMC.
+*
+* @param    hwmgr   the address of the hardware manager
+* @param    table   the SMC DPM table structure to be populated
+* @return   always  0
+*/
+
+static int fiji_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	int result;
+
+	result = fiji_populate_cac_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate CAC voltage tables to SMC",
+			return -EINVAL);
+
+	return 0;
+}
+
+static int fiji_populate_ulv_level(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_Ulv *state)
+{
+	int result = 0;
+
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	state->CcPwrDynRm = 0;
+	state->CcPwrDynRm1 = 0;
+
+	state->VddcOffset = (uint16_t) table_info->us_ulv_voltage_offset;
+	state->VddcOffsetVid = (uint8_t)(table_info->us_ulv_voltage_offset *
+			VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1);
+
+	state->VddcPhase = 1;
+
+	if (!result) {
+		CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm);
+		CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm1);
+		CONVERT_FROM_HOST_TO_SMC_US(state->VddcOffset);
+	}
+	return result;
+}
+
+static int fiji_populate_ulv_state(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	return fiji_populate_ulv_level(hwmgr, &table->Ulv);
+}
+
+static int fiji_populate_smc_link_level(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	int i;
+
+	/* Index (dpm_table->pcie_speed_table.count)
+	 * is reserved for PCIE boot level. */
+	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
+		table->LinkLevel[i].PcieGenSpeed  =
+				(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
+		table->LinkLevel[i].PcieLaneCount = (uint8_t)encode_pcie_lane_width(
+				dpm_table->pcie_speed_table.dpm_levels[i].param1);
+		table->LinkLevel[i].EnabledForActivity = 1;
+		table->LinkLevel[i].SPC = (uint8_t)(data->pcie_spc_cap & 0xff);
+		table->LinkLevel[i].DownThreshold = PP_HOST_TO_SMC_UL(5);
+		table->LinkLevel[i].UpThreshold = PP_HOST_TO_SMC_UL(30);
+	}
+
+	smu_data->smc_state_table.LinkLevelCount =
+			(uint8_t)dpm_table->pcie_speed_table.count;
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
+
+	return 0;
+}
+
+
+/**
+* Calculates the SCLK dividers using the provided engine clock
+*
+* @param    hwmgr  the address of the hardware manager
+* @param    clock  the engine clock to use to populate the structure
+* @param    sclk   the SMC SCLK structure to be populated
+*/
+static int fiji_calculate_sclk_params(struct pp_hwmgr *hwmgr,
+		uint32_t clock, struct SMU73_Discrete_GraphicsLevel *sclk)
+{
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	uint32_t spll_func_cntl            = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_3          = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	uint32_t spll_func_cntl_4          = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	uint32_t cg_spll_spread_spectrum   = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	uint32_t cg_spll_spread_spectrum_2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	uint32_t ref_clock;
+	uint32_t ref_divider;
+	uint32_t fbdiv;
+	int result;
+
+	/* get the engine clock dividers for this clock value */
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr, clock,  &dividers);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+			"Error retrieving Engine Clock dividers from VBIOS.",
+			return result);
+
+	/* To get FBDIV we need to multiply this by 16384 and divide it by Fref. */
+	ref_clock = atomctrl_get_reference_clock(hwmgr);
+	ref_divider = 1 + dividers.uc_pll_ref_div;
+
+	/* low 14 bits is fraction and high 12 bits is divider */
+	fbdiv = dividers.ul_fb_div.ul_fb_divider & 0x3FFFFFF;
+
+	/* SPLL_FUNC_CNTL setup */
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+			SPLL_REF_DIV, dividers.uc_pll_ref_div);
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+			SPLL_PDIV_A,  dividers.uc_pll_post_div);
+
+	/* SPLL_FUNC_CNTL_3 setup*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3, CG_SPLL_FUNC_CNTL_3,
+			SPLL_FB_DIV, fbdiv);
+
+	/* set to use fractional accumulation*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3, CG_SPLL_FUNC_CNTL_3,
+			SPLL_DITHEN, 1);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_EngineSpreadSpectrumSupport)) {
+		struct pp_atomctrl_internal_ss_info ssInfo;
+
+		uint32_t vco_freq = clock * dividers.uc_pll_post_div;
+		if (!atomctrl_get_engine_clock_spread_spectrum(hwmgr,
+				vco_freq, &ssInfo)) {
+			/*
+			 * ss_info.speed_spectrum_percentage -- in unit of 0.01%
+			 * ss_info.speed_spectrum_rate -- in unit of khz
+			 *
+			 * clks = reference_clock * 10 / (REFDIV + 1) / speed_spectrum_rate / 2
+			 */
+			uint32_t clk_s = ref_clock * 5 /
+					(ref_divider * ssInfo.speed_spectrum_rate);
+			/* clkv = 2 * D * fbdiv / NS */
+			uint32_t clk_v = 4 * ssInfo.speed_spectrum_percentage *
+					fbdiv / (clk_s * 10000);
+
+			cg_spll_spread_spectrum = PHM_SET_FIELD(cg_spll_spread_spectrum,
+					CG_SPLL_SPREAD_SPECTRUM, CLKS, clk_s);
+			cg_spll_spread_spectrum = PHM_SET_FIELD(cg_spll_spread_spectrum,
+					CG_SPLL_SPREAD_SPECTRUM, SSEN, 1);
+			cg_spll_spread_spectrum_2 = PHM_SET_FIELD(cg_spll_spread_spectrum_2,
+					CG_SPLL_SPREAD_SPECTRUM_2, CLKV, clk_v);
+		}
+	}
+
+	sclk->SclkFrequency        = clock;
+	sclk->CgSpllFuncCntl3      = spll_func_cntl_3;
+	sclk->CgSpllFuncCntl4      = spll_func_cntl_4;
+	sclk->SpllSpreadSpectrum   = cg_spll_spread_spectrum;
+	sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
+	sclk->SclkDid              = (uint8_t)dividers.pll_post_divider;
+
+	return 0;
+}
+
+/**
+* Populates single SMC SCLK structure using the provided engine clock
+*
+* @param    hwmgr      the address of the hardware manager
+* @param    clock the engine clock to use to populate the structure
+* @param    sclk        the SMC SCLK structure to be populated
+*/
+
+static int fiji_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
+		uint32_t clock, uint16_t sclk_al_threshold,
+		struct SMU73_Discrete_GraphicsLevel *level)
+{
+	int result;
+	/* PP_Clocks minClocks; */
+	uint32_t threshold, mvdd;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	result = fiji_calculate_sclk_params(hwmgr, clock, level);
+
+	/* populate graphics levels */
+	result = fiji_get_dependency_volt_by_clk(hwmgr,
+			table_info->vdd_dep_on_sclk, clock,
+			(uint32_t *)(&level->MinVoltage), &mvdd);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"can not find VDDC voltage value for "
+			"VDDC engine clock dependency table",
+			return result);
+
+	level->SclkFrequency = clock;
+	level->ActivityLevel = sclk_al_threshold;
+	level->CcPwrDynRm = 0;
+	level->CcPwrDynRm1 = 0;
+	level->EnabledForActivity = 0;
+	level->EnabledForThrottle = 1;
+	level->UpHyst = 10;
+	level->DownHyst = 0;
+	level->VoltageDownHyst = 0;
+	level->PowerThrottle = 0;
+
+	threshold = clock * data->fast_watermark_threshold / 100;
+
+	data->display_timing.min_clock_in_sr = hwmgr->display_config.min_core_set_clock_in_sr;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep))
+		level->DeepSleepDivId = smu7_get_sleep_divider_id_from_clock(clock,
+								hwmgr->display_config.min_core_set_clock_in_sr);
+
+
+	/* Default to slow, highest DPM level will be
+	 * set to PPSMC_DISPLAY_WATERMARK_LOW later.
+	 */
+	level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(level->MinVoltage);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_US(level->ActivityLevel);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CgSpllFuncCntl3);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CgSpllFuncCntl4);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->SpllSpreadSpectrum);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->SpllSpreadSpectrum2);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm1);
+
+	return 0;
+}
+/**
+* Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states
+*
+* @param    hwmgr      the address of the hardware manager
+*/
+int fiji_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
+	uint8_t pcie_entry_cnt = (uint8_t) data->dpm_table.pcie_speed_table.count;
+	int result = 0;
+	uint32_t array = smu_data->smu7_data.dpm_table_start +
+			offsetof(SMU73_Discrete_DpmTable, GraphicsLevel);
+	uint32_t array_size = sizeof(struct SMU73_Discrete_GraphicsLevel) *
+			SMU73_MAX_LEVELS_GRAPHICS;
+	struct SMU73_Discrete_GraphicsLevel *levels =
+			smu_data->smc_state_table.GraphicsLevel;
+	uint32_t i, max_entry;
+	uint8_t hightest_pcie_level_enabled = 0,
+			lowest_pcie_level_enabled = 0,
+			mid_pcie_level_enabled = 0,
+			count = 0;
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++) {
+		result = fiji_populate_single_graphic_level(hwmgr,
+				dpm_table->sclk_table.dpm_levels[i].value,
+				(uint16_t)smu_data->activity_target[i],
+				&levels[i]);
+		if (result)
+			return result;
+
+		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
+		if (i > 1)
+			levels[i].DeepSleepDivId = 0;
+	}
+
+	/* Only enable level 0 for now.*/
+	levels[0].EnabledForActivity = 1;
+
+	/* set highest level watermark to high */
+	levels[dpm_table->sclk_table.count - 1].DisplayWatermark =
+			PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	smu_data->smc_state_table.GraphicsDpmLevelCount =
+			(uint8_t)dpm_table->sclk_table.count;
+	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
+
+	if (pcie_table != NULL) {
+		PP_ASSERT_WITH_CODE((1 <= pcie_entry_cnt),
+				"There must be 1 or more PCIE levels defined in PPTable.",
+				return -EINVAL);
+		max_entry = pcie_entry_cnt - 1;
+		for (i = 0; i < dpm_table->sclk_table.count; i++)
+			levels[i].pcieDpmLevel =
+					(uint8_t) ((i < max_entry) ? i : max_entry);
+	} else {
+		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << (hightest_pcie_level_enabled + 1))) != 0))
+			hightest_pcie_level_enabled++;
+
+		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << lowest_pcie_level_enabled)) == 0))
+			lowest_pcie_level_enabled++;
+
+		while ((count < hightest_pcie_level_enabled) &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << (lowest_pcie_level_enabled + 1 + count))) == 0))
+			count++;
+
+		mid_pcie_level_enabled = (lowest_pcie_level_enabled + 1 + count) <
+				hightest_pcie_level_enabled ?
+						(lowest_pcie_level_enabled + 1 + count) :
+						hightest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to hightest_pcie_level_enabled */
+		for (i = 2; i < dpm_table->sclk_table.count; i++)
+			levels[i].pcieDpmLevel = hightest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to lowest_pcie_level_enabled */
+		levels[0].pcieDpmLevel = lowest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to mid_pcie_level_enabled */
+		levels[1].pcieDpmLevel = mid_pcie_level_enabled;
+	}
+	/* level count will send to smc once at init smc table and never change */
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr, array, (uint8_t *)levels,
+			(uint32_t)array_size, SMC_RAM_END);
+
+	return result;
+}
+
+
+/**
+ * MCLK Frequency Ratio
+ * SEQ_CG_RESP  Bit[31:24] - 0x0
+ * Bit[27:24] \96 DDR3 Frequency ratio
+ * 0x0 <= 100MHz,       450 < 0x8 <= 500MHz
+ * 100 < 0x1 <= 150MHz,       500 < 0x9 <= 550MHz
+ * 150 < 0x2 <= 200MHz,       550 < 0xA <= 600MHz
+ * 200 < 0x3 <= 250MHz,       600 < 0xB <= 650MHz
+ * 250 < 0x4 <= 300MHz,       650 < 0xC <= 700MHz
+ * 300 < 0x5 <= 350MHz,       700 < 0xD <= 750MHz
+ * 350 < 0x6 <= 400MHz,       750 < 0xE <= 800MHz
+ * 400 < 0x7 <= 450MHz,       800 < 0xF
+ */
+static uint8_t fiji_get_mclk_frequency_ratio(uint32_t mem_clock)
+{
+	if (mem_clock <= 10000)
+		return 0x0;
+	if (mem_clock <= 15000)
+		return 0x1;
+	if (mem_clock <= 20000)
+		return 0x2;
+	if (mem_clock <= 25000)
+		return 0x3;
+	if (mem_clock <= 30000)
+		return 0x4;
+	if (mem_clock <= 35000)
+		return 0x5;
+	if (mem_clock <= 40000)
+		return 0x6;
+	if (mem_clock <= 45000)
+		return 0x7;
+	if (mem_clock <= 50000)
+		return 0x8;
+	if (mem_clock <= 55000)
+		return 0x9;
+	if (mem_clock <= 60000)
+		return 0xa;
+	if (mem_clock <= 65000)
+		return 0xb;
+	if (mem_clock <= 70000)
+		return 0xc;
+	if (mem_clock <= 75000)
+		return 0xd;
+	if (mem_clock <= 80000)
+		return 0xe;
+	/* mem_clock > 800MHz */
+	return 0xf;
+}
+
+/**
+* Populates the SMC MCLK structure using the provided memory clock
+*
+* @param    hwmgr   the address of the hardware manager
+* @param    clock   the memory clock to use to populate the structure
+* @param    sclk    the SMC SCLK structure to be populated
+*/
+static int fiji_calculate_mclk_params(struct pp_hwmgr *hwmgr,
+    uint32_t clock, struct SMU73_Discrete_MemoryLevel *mclk)
+{
+	struct pp_atomctrl_memory_clock_param mem_param;
+	int result;
+
+	result = atomctrl_get_memory_pll_dividers_vi(hwmgr, clock, &mem_param);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Failed to get Memory PLL Dividers.",
+			);
+
+	/* Save the result data to outpupt memory level structure */
+	mclk->MclkFrequency   = clock;
+	mclk->MclkDivider     = (uint8_t)mem_param.mpll_post_divider;
+	mclk->FreqRange       = fiji_get_mclk_frequency_ratio(clock);
+
+	return result;
+}
+
+static int fiji_populate_single_memory_level(struct pp_hwmgr *hwmgr,
+		uint32_t clock, struct SMU73_Discrete_MemoryLevel *mem_level)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int result = 0;
+	uint32_t mclk_stutter_mode_threshold = 60000;
+
+	if (table_info->vdd_dep_on_mclk) {
+		result = fiji_get_dependency_volt_by_clk(hwmgr,
+				table_info->vdd_dep_on_mclk, clock,
+				(uint32_t *)(&mem_level->MinVoltage), &mem_level->MinMvdd);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find MinVddc voltage value from memory "
+				"VDDC voltage dependency table", return result);
+	}
+
+	mem_level->EnabledForThrottle = 1;
+	mem_level->EnabledForActivity = 0;
+	mem_level->UpHyst = 0;
+	mem_level->DownHyst = 100;
+	mem_level->VoltageDownHyst = 0;
+	mem_level->ActivityLevel = (uint16_t)data->mclk_activity_target;
+	mem_level->StutterEnable = false;
+
+	mem_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	/* enable stutter mode if all the follow condition applied
+	 * PECI_GetNumberOfActiveDisplays(hwmgr->pPECI,
+	 * &(data->DisplayTiming.numExistingDisplays));
+	 */
+	data->display_timing.num_existing_displays = 1;
+
+	if (mclk_stutter_mode_threshold &&
+		(clock <= mclk_stutter_mode_threshold) &&
+		(!data->is_uvd_enabled) &&
+		(PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL,
+				STUTTER_ENABLE) & 0x1))
+		mem_level->StutterEnable = true;
+
+	result = fiji_calculate_mclk_params(hwmgr, clock, mem_level);
+	if (!result) {
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinMvdd);
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_US(mem_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinVoltage);
+	}
+	return result;
+}
+
+/**
+* Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states
+*
+* @param    hwmgr      the address of the hardware manager
+*/
+int fiji_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	int result;
+	/* populate MCLK dpm table to SMU7 */
+	uint32_t array = smu_data->smu7_data.dpm_table_start +
+			offsetof(SMU73_Discrete_DpmTable, MemoryLevel);
+	uint32_t array_size = sizeof(SMU73_Discrete_MemoryLevel) *
+			SMU73_MAX_LEVELS_MEMORY;
+	struct SMU73_Discrete_MemoryLevel *levels =
+			smu_data->smc_state_table.MemoryLevel;
+	uint32_t i;
+
+	for (i = 0; i < dpm_table->mclk_table.count; i++) {
+		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
+				"can not populate memory level as memory clock is zero",
+				return -EINVAL);
+		result = fiji_populate_single_memory_level(hwmgr,
+				dpm_table->mclk_table.dpm_levels[i].value,
+				&levels[i]);
+		if (result)
+			return result;
+	}
+
+	/* Only enable level 0 for now. */
+	levels[0].EnabledForActivity = 1;
+
+	/* in order to prevent MC activity from stutter mode to push DPM up.
+	 * the UVD change complements this by putting the MCLK in
+	 * a higher state by default such that we are not effected by
+	 * up threshold or and MCLK DPM latency.
+	 */
+	levels[0].ActivityLevel = (uint16_t)data->mclk_dpm0_activity_target;
+	CONVERT_FROM_HOST_TO_SMC_US(levels[0].ActivityLevel);
+
+	smu_data->smc_state_table.MemoryDpmLevelCount =
+			(uint8_t)dpm_table->mclk_table.count;
+	data->dpm_level_enable_mask.mclk_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
+	/* set highest level watermark to high */
+	levels[dpm_table->mclk_table.count - 1].DisplayWatermark =
+			PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	/* level count will send to smc once at init smc table and never change */
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr, array, (uint8_t *)levels,
+			(uint32_t)array_size, SMC_RAM_END);
+
+	return result;
+}
+
+
+/**
+* Populates the SMC MVDD structure using the provided memory clock.
+*
+* @param    hwmgr      the address of the hardware manager
+* @param    mclk        the MCLK value to be used in the decision if MVDD should be high or low.
+* @param    voltage     the SMC VOLTAGE structure to be populated
+*/
+static int fiji_populate_mvdd_value(struct pp_hwmgr *hwmgr,
+		uint32_t mclk, SMIO_Pattern *smio_pat)
+{
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t i = 0;
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
+		/* find mvdd value which clock is more than request */
+		for (i = 0; i < table_info->vdd_dep_on_mclk->count; i++) {
+			if (mclk <= table_info->vdd_dep_on_mclk->entries[i].clk) {
+				smio_pat->Voltage = data->mvdd_voltage_table.entries[i].value;
+				break;
+			}
+		}
+		PP_ASSERT_WITH_CODE(i < table_info->vdd_dep_on_mclk->count,
+				"MVDD Voltage is outside the supported range.",
+				return -EINVAL);
+	} else
+		return -EINVAL;
+
+	return 0;
+}
+
+static int fiji_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
+		SMU73_Discrete_DpmTable *table)
+{
+	int result = 0;
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	SMIO_Pattern vol_level;
+	uint32_t mvdd;
+	uint16_t us_mvdd;
+	uint32_t spll_func_cntl    = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_2  = data->clock_registers.vCG_SPLL_FUNC_CNTL_2;
+
+	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+	if (!data->sclk_dpm_key_disabled) {
+		/* Get MinVoltage and Frequency from DPM0,
+		 * already converted to SMC_UL */
+		table->ACPILevel.SclkFrequency =
+				data->dpm_table.sclk_table.dpm_levels[0].value;
+		result = fiji_get_dependency_volt_by_clk(hwmgr,
+				table_info->vdd_dep_on_sclk,
+				table->ACPILevel.SclkFrequency,
+				(uint32_t *)(&table->ACPILevel.MinVoltage), &mvdd);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Cannot find ACPI VDDC voltage value " \
+				"in Clock Dependency Table",
+				);
+	} else {
+		table->ACPILevel.SclkFrequency =
+				data->vbios_boot_state.sclk_bootup_value;
+		table->ACPILevel.MinVoltage =
+				data->vbios_boot_state.vddc_bootup_value * VOLTAGE_SCALE;
+	}
+
+	/* get the engine clock dividers for this clock value */
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr,
+			table->ACPILevel.SclkFrequency,  &dividers);
+	PP_ASSERT_WITH_CODE(result == 0,
+			"Error retrieving Engine Clock dividers from VBIOS.",
+			return result);
+
+	table->ACPILevel.SclkDid = (uint8_t)dividers.pll_post_divider;
+	table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+	table->ACPILevel.DeepSleepDivId = 0;
+
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+			SPLL_PWRON, 0);
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+			SPLL_RESET, 1);
+	spll_func_cntl_2 = PHM_SET_FIELD(spll_func_cntl_2, CG_SPLL_FUNC_CNTL_2,
+			SCLK_MUX_SEL, 4);
+
+	table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
+	table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
+	table->ACPILevel.CgSpllFuncCntl3 = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	table->ACPILevel.CgSpllFuncCntl4 = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	table->ACPILevel.SpllSpreadSpectrum = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	table->ACPILevel.SpllSpreadSpectrum2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	table->ACPILevel.CcPwrDynRm = 0;
+	table->ACPILevel.CcPwrDynRm1 = 0;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.MinVoltage);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl3);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl4);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
+
+	if (!data->mclk_dpm_key_disabled) {
+		/* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */
+		table->MemoryACPILevel.MclkFrequency =
+				data->dpm_table.mclk_table.dpm_levels[0].value;
+		result = fiji_get_dependency_volt_by_clk(hwmgr,
+				table_info->vdd_dep_on_mclk,
+				table->MemoryACPILevel.MclkFrequency,
+			(uint32_t *)(&table->MemoryACPILevel.MinVoltage), &mvdd);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"Cannot find ACPI VDDCI voltage value in Clock Dependency Table",
+				);
+	} else {
+		table->MemoryACPILevel.MclkFrequency =
+				data->vbios_boot_state.mclk_bootup_value;
+		table->MemoryACPILevel.MinVoltage =
+				data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE;
+	}
+
+	us_mvdd = 0;
+	if ((SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control) ||
+			(data->mclk_dpm_key_disabled))
+		us_mvdd = data->vbios_boot_state.mvdd_bootup_value;
+	else {
+		if (!fiji_populate_mvdd_value(hwmgr,
+				data->dpm_table.mclk_table.dpm_levels[0].value,
+				&vol_level))
+			us_mvdd = vol_level.Voltage;
+	}
+
+	table->MemoryACPILevel.MinMvdd =
+			PP_HOST_TO_SMC_UL(us_mvdd * VOLTAGE_SCALE);
+
+	table->MemoryACPILevel.EnabledForThrottle = 0;
+	table->MemoryACPILevel.EnabledForActivity = 0;
+	table->MemoryACPILevel.UpHyst = 0;
+	table->MemoryACPILevel.DownHyst = 100;
+	table->MemoryACPILevel.VoltageDownHyst = 0;
+	table->MemoryACPILevel.ActivityLevel =
+			PP_HOST_TO_SMC_US((uint16_t)data->mclk_activity_target);
+
+	table->MemoryACPILevel.StutterEnable = false;
+	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MinVoltage);
+
+	return result;
+}
+
+static int fiji_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
+		SMU73_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+
+	table->VceLevelCount = (uint8_t)(mm_table->count);
+	table->VceBootLevel = 0;
+
+	for (count = 0; count < table->VceLevelCount; count++) {
+		table->VceLevel[count].Frequency = mm_table->entries[count].eclk;
+		table->VceLevel[count].MinVoltage = 0;
+		table->VceLevel[count].MinVoltage |=
+				(mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
+		table->VceLevel[count].MinVoltage |=
+				((mm_table->entries[count].vddc - VDDC_VDDCI_DELTA) *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->VceLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/*retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->VceLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for VCE engine clock",
+				return result);
+
+		table->VceLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+static int fiji_populate_smc_acp_level(struct pp_hwmgr *hwmgr,
+		SMU73_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+
+	table->AcpLevelCount = (uint8_t)(mm_table->count);
+	table->AcpBootLevel = 0;
+
+	for (count = 0; count < table->AcpLevelCount; count++) {
+		table->AcpLevel[count].Frequency = mm_table->entries[count].aclk;
+		table->AcpLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+		table->AcpLevel[count].MinVoltage |= ((mm_table->entries[count].vddc -
+				VDDC_VDDCI_DELTA) * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->AcpLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->AcpLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for engine clock", return result);
+
+		table->AcpLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->AcpLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->AcpLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+static int fiji_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
+		SMU73_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+
+	table->SamuBootLevel = 0;
+	table->SamuLevelCount = (uint8_t)(mm_table->count);
+
+	for (count = 0; count < table->SamuLevelCount; count++) {
+		/* not sure whether we need evclk or not */
+		table->SamuLevel[count].MinVoltage = 0;
+		table->SamuLevel[count].Frequency = mm_table->entries[count].samclock;
+		table->SamuLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+		table->SamuLevel[count].MinVoltage |= ((mm_table->entries[count].vddc -
+				VDDC_VDDCI_DELTA) * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->SamuLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->SamuLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for samu clock", return result);
+
+		table->SamuLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+static int fiji_populate_memory_timing_parameters(struct pp_hwmgr *hwmgr,
+		int32_t eng_clock, int32_t mem_clock,
+		struct SMU73_Discrete_MCArbDramTimingTableEntry *arb_regs)
+{
+	uint32_t dram_timing;
+	uint32_t dram_timing2;
+	uint32_t burstTime;
+	ULONG state, trrds, trrdl;
+	int result;
+
+	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
+			eng_clock, mem_clock);
+	PP_ASSERT_WITH_CODE(result == 0,
+			"Error calling VBIOS to set DRAM_TIMING.", return result);
+
+	dram_timing = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+	dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+	burstTime = cgs_read_register(hwmgr->device, mmMC_ARB_BURST_TIME);
+
+	state = PHM_GET_FIELD(burstTime, MC_ARB_BURST_TIME, STATE0);
+	trrds = PHM_GET_FIELD(burstTime, MC_ARB_BURST_TIME, TRRDS0);
+	trrdl = PHM_GET_FIELD(burstTime, MC_ARB_BURST_TIME, TRRDL0);
+
+	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dram_timing);
+	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dram_timing2);
+	arb_regs->McArbBurstTime   = (uint8_t)burstTime;
+	arb_regs->TRRDS            = (uint8_t)trrds;
+	arb_regs->TRRDL            = (uint8_t)trrdl;
+
+	return 0;
+}
+
+static int fiji_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	struct SMU73_Discrete_MCArbDramTimingTable arb_regs;
+	uint32_t i, j;
+	int result = 0;
+
+	for (i = 0; i < data->dpm_table.sclk_table.count; i++) {
+		for (j = 0; j < data->dpm_table.mclk_table.count; j++) {
+			result = fiji_populate_memory_timing_parameters(hwmgr,
+					data->dpm_table.sclk_table.dpm_levels[i].value,
+					data->dpm_table.mclk_table.dpm_levels[j].value,
+					&arb_regs.entries[i][j]);
+			if (result)
+				break;
+		}
+	}
+
+	if (!result)
+		result = smu7_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				smu_data->smu7_data.arb_table_start,
+				(uint8_t *)&arb_regs,
+				sizeof(SMU73_Discrete_MCArbDramTimingTable),
+				SMC_RAM_END);
+	return result;
+}
+
+static int fiji_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+
+	table->UvdLevelCount = (uint8_t)(mm_table->count);
+	table->UvdBootLevel = 0;
+
+	for (count = 0; count < table->UvdLevelCount; count++) {
+		table->UvdLevel[count].MinVoltage = 0;
+		table->UvdLevel[count].VclkFrequency = mm_table->entries[count].vclk;
+		table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk;
+		table->UvdLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+		table->UvdLevel[count].MinVoltage |= ((mm_table->entries[count].vddc -
+				VDDC_VDDCI_DELTA) * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->UvdLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->UvdLevel[count].VclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for Vclk clock", return result);
+
+		table->UvdLevel[count].VclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->UvdLevel[count].DclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for Dclk clock", return result);
+
+		table->UvdLevel[count].DclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].MinVoltage);
+
+	}
+	return result;
+}
+
+static int fiji_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	int result = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	table->GraphicsBootLevel = 0;
+	table->MemoryBootLevel = 0;
+
+	/* find boot level from dpm table */
+	result = phm_find_boot_level(&(data->dpm_table.sclk_table),
+			data->vbios_boot_state.sclk_bootup_value,
+			(uint32_t *)&(table->GraphicsBootLevel));
+
+	result = phm_find_boot_level(&(data->dpm_table.mclk_table),
+			data->vbios_boot_state.mclk_bootup_value,
+			(uint32_t *)&(table->MemoryBootLevel));
+
+	table->BootVddc  = data->vbios_boot_state.vddc_bootup_value *
+			VOLTAGE_SCALE;
+	table->BootVddci = data->vbios_boot_state.vddci_bootup_value *
+			VOLTAGE_SCALE;
+	table->BootMVdd  = data->vbios_boot_state.mvdd_bootup_value *
+			VOLTAGE_SCALE;
+
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddc);
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddci);
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootMVdd);
+
+	return 0;
+}
+
+static int fiji_populate_smc_initailial_state(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint8_t count, level;
+
+	count = (uint8_t)(table_info->vdd_dep_on_sclk->count);
+	for (level = 0; level < count; level++) {
+		if (table_info->vdd_dep_on_sclk->entries[level].clk >=
+				data->vbios_boot_state.sclk_bootup_value) {
+			smu_data->smc_state_table.GraphicsBootLevel = level;
+			break;
+		}
+	}
+
+	count = (uint8_t)(table_info->vdd_dep_on_mclk->count);
+	for (level = 0; level < count; level++) {
+		if (table_info->vdd_dep_on_mclk->entries[level].clk >=
+				data->vbios_boot_state.mclk_bootup_value) {
+			smu_data->smc_state_table.MemoryBootLevel = level;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int fiji_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
+{
+	uint32_t ro, efuse, efuse2, clock_freq, volt_without_cks,
+			volt_with_cks, value;
+	uint16_t clock_freq_u16;
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	uint8_t type, i, j, cks_setting, stretch_amount, stretch_amount2,
+			volt_offset = 0;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+
+	stretch_amount = (uint8_t)table_info->cac_dtp_table->usClockStretchAmount;
+
+	/* Read SMU_Eefuse to read and calculate RO and determine
+	 * if the part is SS or FF. if RO >= 1660MHz, part is FF.
+	 */
+	efuse = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixSMU_EFUSE_0 + (146 * 4));
+	efuse2 = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixSMU_EFUSE_0 + (148 * 4));
+	efuse &= 0xFF000000;
+	efuse = efuse >> 24;
+	efuse2 &= 0xF;
+
+	if (efuse2 == 1)
+		ro = (2300 - 1350) * efuse / 255 + 1350;
+	else
+		ro = (2500 - 1000) * efuse / 255 + 1000;
+
+	if (ro >= 1660)
+		type = 0;
+	else
+		type = 1;
+
+	/* Populate Stretch amount */
+	smu_data->smc_state_table.ClockStretcherAmount = stretch_amount;
+
+	/* Populate Sclk_CKS_masterEn0_7 and Sclk_voltageOffset */
+	for (i = 0; i < sclk_table->count; i++) {
+		smu_data->smc_state_table.Sclk_CKS_masterEn0_7 |=
+				sclk_table->entries[i].cks_enable << i;
+		volt_without_cks = (uint32_t)((14041 *
+			(sclk_table->entries[i].clk/100) / 10000 + 3571 + 75 - ro) * 1000 /
+			(4026 - (13924 * (sclk_table->entries[i].clk/100) / 10000)));
+		volt_with_cks = (uint32_t)((13946 *
+			(sclk_table->entries[i].clk/100) / 10000 + 3320 + 45 - ro) * 1000 /
+			(3664 - (11454 * (sclk_table->entries[i].clk/100) / 10000)));
+		if (volt_without_cks >= volt_with_cks)
+			volt_offset = (uint8_t)(((volt_without_cks - volt_with_cks +
+					sclk_table->entries[i].cks_voffset) * 100 / 625) + 1);
+		smu_data->smc_state_table.Sclk_voltageOffset[i] = volt_offset;
+	}
+
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			STRETCH_ENABLE, 0x0);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			masterReset, 0x1);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			staticEnable, 0x1);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			masterReset, 0x0);
+
+	/* Populate CKS Lookup Table */
+	if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5)
+		stretch_amount2 = 0;
+	else if (stretch_amount == 3 || stretch_amount == 4)
+		stretch_amount2 = 1;
+	else {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ClockStretcher);
+		PP_ASSERT_WITH_CODE(false,
+				"Stretch Amount in PPTable not supported\n",
+				return -EINVAL);
+	}
+
+	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixPWR_CKS_CNTL);
+	value &= 0xFFC2FF87;
+	smu_data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].minFreq =
+			fiji_clock_stretcher_lookup_table[stretch_amount2][0];
+	smu_data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].maxFreq =
+			fiji_clock_stretcher_lookup_table[stretch_amount2][1];
+	clock_freq_u16 = (uint16_t)(PP_SMC_TO_HOST_UL(smu_data->smc_state_table.
+			GraphicsLevel[smu_data->smc_state_table.GraphicsDpmLevelCount - 1].
+			SclkFrequency) / 100);
+	if (fiji_clock_stretcher_lookup_table[stretch_amount2][0] <
+			clock_freq_u16 &&
+	    fiji_clock_stretcher_lookup_table[stretch_amount2][1] >
+			clock_freq_u16) {
+		/* Program PWR_CKS_CNTL. CKS_USE_FOR_LOW_FREQ */
+		value |= (fiji_clock_stretcher_lookup_table[stretch_amount2][3]) << 16;
+		/* Program PWR_CKS_CNTL. CKS_LDO_REFSEL */
+		value |= (fiji_clock_stretcher_lookup_table[stretch_amount2][2]) << 18;
+		/* Program PWR_CKS_CNTL. CKS_STRETCH_AMOUNT */
+		value |= (fiji_clock_stretch_amount_conversion
+				[fiji_clock_stretcher_lookup_table[stretch_amount2][3]]
+				 [stretch_amount]) << 3;
+	}
+	CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.CKS_LOOKUPTable.
+			CKS_LOOKUPTableEntry[0].minFreq);
+	CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.CKS_LOOKUPTable.
+			CKS_LOOKUPTableEntry[0].maxFreq);
+	smu_data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting =
+			fiji_clock_stretcher_lookup_table[stretch_amount2][2] & 0x7F;
+	smu_data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting |=
+			(fiji_clock_stretcher_lookup_table[stretch_amount2][3]) << 7;
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixPWR_CKS_CNTL, value);
+
+	/* Populate DDT Lookup Table */
+	for (i = 0; i < 4; i++) {
+		/* Assign the minimum and maximum VID stored
+		 * in the last row of Clock Stretcher Voltage Table.
+		 */
+		smu_data->smc_state_table.ClockStretcherDataTable.
+		ClockStretcherDataTableEntry[i].minVID =
+				(uint8_t) fiji_clock_stretcher_ddt_table[type][i][2];
+		smu_data->smc_state_table.ClockStretcherDataTable.
+		ClockStretcherDataTableEntry[i].maxVID =
+				(uint8_t) fiji_clock_stretcher_ddt_table[type][i][3];
+		/* Loop through each SCLK and check the frequency
+		 * to see if it lies within the frequency for clock stretcher.
+		 */
+		for (j = 0; j < smu_data->smc_state_table.GraphicsDpmLevelCount; j++) {
+			cks_setting = 0;
+			clock_freq = PP_SMC_TO_HOST_UL(
+					smu_data->smc_state_table.GraphicsLevel[j].SclkFrequency);
+			/* Check the allowed frequency against the sclk level[j].
+			 *  Sclk's endianness has already been converted,
+			 *  and it's in 10Khz unit,
+			 *  as opposed to Data table, which is in Mhz unit.
+			 */
+			if (clock_freq >=
+					(fiji_clock_stretcher_ddt_table[type][i][0]) * 100) {
+				cks_setting |= 0x2;
+				if (clock_freq <
+						(fiji_clock_stretcher_ddt_table[type][i][1]) * 100)
+					cks_setting |= 0x1;
+			}
+			smu_data->smc_state_table.ClockStretcherDataTable.
+			ClockStretcherDataTableEntry[i].setting |= cks_setting << (j * 2);
+		}
+		CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.
+				ClockStretcherDataTable.
+				ClockStretcherDataTableEntry[i].setting);
+	}
+
+	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL);
+	value &= 0xFFFFFFFE;
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL, value);
+
+	return 0;
+}
+
+/**
+* Populates the SMC VRConfig field in DPM table.
+*
+* @param    hwmgr   the address of the hardware manager
+* @param    table   the SMC DPM table structure to be populated
+* @return   always 0
+*/
+static int fiji_populate_vr_config(struct pp_hwmgr *hwmgr,
+		struct SMU73_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint16_t config;
+
+	config = VR_MERGED_WITH_VDDC;
+	table->VRConfig |= (config << VRCONF_VDDGFX_SHIFT);
+
+	/* Set Vddc Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+		config = VR_SVI2_PLANE_1;
+		table->VRConfig |= config;
+	} else {
+		PP_ASSERT_WITH_CODE(false,
+				"VDDC should be on SVI2 control in merged mode!",
+				);
+	}
+	/* Set Vddci Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
+		config = VR_SVI2_PLANE_2;  /* only in merged mode */
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	} else if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+		config = VR_SMIO_PATTERN_1;
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	} else {
+		config = VR_STATIC_VOLTAGE;
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	}
+	/* Set Mvdd Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
+		config = VR_SVI2_PLANE_2;
+		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+	} else if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		config = VR_SMIO_PATTERN_2;
+		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+	} else {
+		config = VR_STATIC_VOLTAGE;
+		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+	}
+
+	return 0;
+}
+
+static int fiji_init_arb_table_index(struct pp_smumgr *smumgr)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(smumgr->backend);
+	uint32_t tmp;
+	int result;
+
+	/* This is a read-modify-write on the first byte of the ARB table.
+	 * The first byte in the SMU73_Discrete_MCArbDramTimingTable structure
+	 * is the field 'current'.
+	 * This solution is ugly, but we never write the whole table only
+	 * individual fields in it.
+	 * In reality this field should not be in that structure
+	 * but in a soft register.
+	 */
+	result = smu7_read_smc_sram_dword(smumgr,
+			smu_data->smu7_data.arb_table_start, &tmp, SMC_RAM_END);
+
+	if (result)
+		return result;
+
+	tmp &= 0x00FFFFFF;
+	tmp |= ((uint32_t)MC_CG_ARB_FREQ_F1) << 24;
+
+	return smu7_write_smc_sram_dword(smumgr,
+			smu_data->smu7_data.arb_table_start,  tmp, SMC_RAM_END);
+}
+
+/**
+* Initializes the SMC table and uploads it
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput  the pointer to input data (PowerState)
+* @return   always 0
+*/
+int fiji_init_smc_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct SMU73_Discrete_DpmTable *table = &(smu_data->smc_state_table);
+	uint8_t i;
+	struct pp_atomctrl_gpio_pin_assignment gpio_pin;
+
+	fiji_initialize_power_tune_defaults(hwmgr);
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != data->voltage_control)
+		fiji_populate_smc_voltage_tables(hwmgr, table);
+
+	table->SystemFlags = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StepVddc))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+	if (data->is_memory_gddr5)
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+	if (data->ulv_supported && table_info->us_ulv_voltage_offset) {
+		result = fiji_populate_ulv_state(hwmgr, table);
+		PP_ASSERT_WITH_CODE(0 == result,
+				"Failed to initialize ULV state!", return result);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_ULV_PARAMETER, 0x40035);
+	}
+
+	result = fiji_populate_smc_link_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Link Level!", return result);
+
+	result = fiji_populate_all_graphic_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Graphics Level!", return result);
+
+	result = fiji_populate_all_memory_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Memory Level!", return result);
+
+	result = fiji_populate_smc_acpi_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize ACPI Level!", return result);
+
+	result = fiji_populate_smc_vce_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize VCE Level!", return result);
+
+	result = fiji_populate_smc_acp_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize ACP Level!", return result);
+
+	result = fiji_populate_smc_samu_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize SAMU Level!", return result);
+
+	/* Since only the initial state is completely set up at this point
+	 * (the other states are just copies of the boot state) we only
+	 * need to populate the  ARB settings for the initial state.
+	 */
+	result = fiji_program_memory_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to Write ARB settings for the initial state.", return result);
+
+	result = fiji_populate_smc_uvd_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize UVD Level!", return result);
+
+	result = fiji_populate_smc_boot_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Boot Level!", return result);
+
+	result = fiji_populate_smc_initailial_state(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Boot State!", return result);
+
+	result = fiji_populate_bapm_parameters_in_dpm_table(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to populate BAPM Parameters!", return result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ClockStretcher)) {
+		result = fiji_populate_clock_stretcher_data_table(hwmgr);
+		PP_ASSERT_WITH_CODE(0 == result,
+				"Failed to populate Clock Stretcher Data Table!",
+				return result);
+	}
+
+	table->GraphicsVoltageChangeEnable  = 1;
+	table->GraphicsThermThrottleEnable  = 1;
+	table->GraphicsInterval = 1;
+	table->VoltageInterval  = 1;
+	table->ThermalInterval  = 1;
+	table->TemperatureLimitHigh =
+			table_info->cac_dtp_table->usTargetOperatingTemp *
+			SMU7_Q88_FORMAT_CONVERSION_UNIT;
+	table->TemperatureLimitLow  =
+			(table_info->cac_dtp_table->usTargetOperatingTemp - 1) *
+			SMU7_Q88_FORMAT_CONVERSION_UNIT;
+	table->MemoryVoltageChangeEnable = 1;
+	table->MemoryInterval = 1;
+	table->VoltageResponseTime = 0;
+	table->PhaseResponseTime = 0;
+	table->MemoryThermThrottleEnable = 1;
+	table->PCIeBootLinkLevel = 0;      /* 0:Gen1 1:Gen2 2:Gen3*/
+	table->PCIeGenInterval = 1;
+	table->VRConfig = 0;
+
+	result = fiji_populate_vr_config(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to populate VRConfig setting!", return result);
+
+	table->ThermGpio = 17;
+	table->SclkStepSize = 0x4000;
+
+	if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_VRHOT_GPIO_PINID, &gpio_pin)) {
+		table->VRHotGpio = gpio_pin.uc_gpio_pin_bit_shift;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_RegulatorHot);
+	} else {
+		table->VRHotGpio = SMU7_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_RegulatorHot);
+	}
+
+	if (atomctrl_get_pp_assign_pin(hwmgr, PP_AC_DC_SWITCH_GPIO_PINID,
+			&gpio_pin)) {
+		table->AcDcGpio = gpio_pin.uc_gpio_pin_bit_shift;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_AutomaticDCTransition);
+	} else {
+		table->AcDcGpio = SMU7_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_AutomaticDCTransition);
+	}
+
+	/* Thermal Output GPIO */
+	if (atomctrl_get_pp_assign_pin(hwmgr, THERMAL_INT_OUTPUT_GPIO_PINID,
+			&gpio_pin)) {
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ThermalOutGPIO);
+
+		table->ThermOutGpio = gpio_pin.uc_gpio_pin_bit_shift;
+
+		/* For porlarity read GPIOPAD_A with assigned Gpio pin
+		 * since VBIOS will program this register to set 'inactive state',
+		 * driver can then determine 'active state' from this and
+		 * program SMU with correct polarity
+		 */
+		table->ThermOutPolarity = (0 == (cgs_read_register(hwmgr->device, mmGPIOPAD_A) &
+				(1 << gpio_pin.uc_gpio_pin_bit_shift))) ? 1:0;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_ONLY;
+
+		/* if required, combine VRHot/PCC with thermal out GPIO */
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_RegulatorHot) &&
+			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_CombinePCCWithThermalSignal))
+			table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_VRHOT;
+	} else {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ThermalOutGPIO);
+		table->ThermOutGpio = 17;
+		table->ThermOutPolarity = 1;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_DISABLE;
+	}
+
+	for (i = 0; i < SMU73_MAX_ENTRIES_SMIO; i++)
+		table->Smio[i] = PP_HOST_TO_SMC_UL(table->Smio[i]);
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VRConfig);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask1);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
+	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
+	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
+
+	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr,
+			smu_data->smu7_data.dpm_table_start +
+			offsetof(SMU73_Discrete_DpmTable, SystemFlags),
+			(uint8_t *)&(table->SystemFlags),
+			sizeof(SMU73_Discrete_DpmTable) - 3 * sizeof(SMU73_PIDController),
+			SMC_RAM_END);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to upload dpm data to SMC memory!", return result);
+
+	result = fiji_init_arb_table_index(hwmgr->smumgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to upload arb data to SMC memory!", return result);
+
+	result = fiji_populate_pm_fuses(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to  populate PM fuses to SMC memory!", return result);
+	return 0;
+}
+
+/**
+* Set up the fan table to control the fan using the SMC.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int fiji_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+
+	SMU73_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
+	uint32_t duty100;
+	uint32_t t_diff1, t_diff2, pwm_diff1, pwm_diff2;
+	uint16_t fdo_min, slope1, slope2;
+	uint32_t reference_clock;
+	int res;
+	uint64_t tmp64;
+
+	if (smu_data->smu7_data.fan_table_start == 0) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_FDO_CTRL1, FMAX_DUTY100);
+
+	if (duty100 == 0) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	tmp64 = hwmgr->thermal_controller.advanceFanControlParameters.
+			usPWMMin * duty100;
+	do_div(tmp64, 10000);
+	fdo_min = (uint16_t)tmp64;
+
+	t_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usTMed -
+			hwmgr->thermal_controller.advanceFanControlParameters.usTMin;
+	t_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usTHigh -
+			hwmgr->thermal_controller.advanceFanControlParameters.usTMed;
+
+	pwm_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed -
+			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin;
+	pwm_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh -
+			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed;
+
+	slope1 = (uint16_t)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
+	slope2 = (uint16_t)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
+
+	fan_table.TempMin = cpu_to_be16((50 + hwmgr->
+			thermal_controller.advanceFanControlParameters.usTMin) / 100);
+	fan_table.TempMed = cpu_to_be16((50 + hwmgr->
+			thermal_controller.advanceFanControlParameters.usTMed) / 100);
+	fan_table.TempMax = cpu_to_be16((50 + hwmgr->
+			thermal_controller.advanceFanControlParameters.usTMax) / 100);
+
+	fan_table.Slope1 = cpu_to_be16(slope1);
+	fan_table.Slope2 = cpu_to_be16(slope2);
+
+	fan_table.FdoMin = cpu_to_be16(fdo_min);
+
+	fan_table.HystDown = cpu_to_be16(hwmgr->
+			thermal_controller.advanceFanControlParameters.ucTHyst);
+
+	fan_table.HystUp = cpu_to_be16(1);
+
+	fan_table.HystSlope = cpu_to_be16(1);
+
+	fan_table.TempRespLim = cpu_to_be16(5);
+
+	reference_clock = smu7_get_xclk(hwmgr);
+
+	fan_table.RefreshPeriod = cpu_to_be32((hwmgr->
+			thermal_controller.advanceFanControlParameters.ulCycleDelay *
+			reference_clock) / 1600);
+
+	fan_table.FdoMax = cpu_to_be16((uint16_t)duty100);
+
+	fan_table.TempSrc = (uint8_t)PHM_READ_VFPF_INDIRECT_FIELD(
+			hwmgr->device, CGS_IND_REG__SMC,
+			CG_MULT_THERMAL_CTRL, TEMP_SEL);
+
+	res = smu7_copy_bytes_to_smc(hwmgr->smumgr, smu_data->smu7_data.fan_table_start,
+			(uint8_t *)&fan_table, (uint32_t)sizeof(fan_table),
+			SMC_RAM_END);
+
+	if (!res && hwmgr->thermal_controller.
+			advanceFanControlParameters.ucMinimumPWMLimit)
+		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetFanMinPwm,
+				hwmgr->thermal_controller.
+				advanceFanControlParameters.ucMinimumPWMLimit);
+
+	if (!res && hwmgr->thermal_controller.
+			advanceFanControlParameters.ulMinFanSCLKAcousticLimit)
+		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetFanSclkTarget,
+				hwmgr->thermal_controller.
+				advanceFanControlParameters.ulMinFanSCLKAcousticLimit);
+
+	if (res)
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+
+	return 0;
+}
+
+int fiji_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_OD_UPDATE_MCLK))
+		return fiji_program_memory_timing_parameters(hwmgr);
+
+	return 0;
+}
+
+int fiji_update_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+
+	int result = 0;
+	uint32_t low_sclk_interrupt_threshold = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkThrottleLowNotification)
+		&& (hwmgr->gfx_arbiter.sclk_threshold !=
+				data->low_sclk_interrupt_threshold)) {
+		data->low_sclk_interrupt_threshold =
+				hwmgr->gfx_arbiter.sclk_threshold;
+		low_sclk_interrupt_threshold =
+				data->low_sclk_interrupt_threshold;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
+
+		result = smu7_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU73_Discrete_DpmTable,
+					LowSclkInterruptThreshold),
+				(uint8_t *)&low_sclk_interrupt_threshold,
+				sizeof(uint32_t),
+				SMC_RAM_END);
+	}
+	result = fiji_program_mem_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE((result == 0),
+			"Failed to program memory timing parameters!",
+			);
+	return result;
+}
+
+uint32_t fiji_get_offsetof(uint32_t type, uint32_t member)
+{
+	switch (type) {
+	case SMU_SoftRegisters:
+		switch (member) {
+		case HandshakeDisables:
+			return offsetof(SMU73_SoftRegisters, HandshakeDisables);
+		case VoltageChangeTimeout:
+			return offsetof(SMU73_SoftRegisters, VoltageChangeTimeout);
+		case AverageGraphicsActivity:
+			return offsetof(SMU73_SoftRegisters, AverageGraphicsActivity);
+		case PreVBlankGap:
+			return offsetof(SMU73_SoftRegisters, PreVBlankGap);
+		case VBlankTimeout:
+			return offsetof(SMU73_SoftRegisters, VBlankTimeout);
+		case UcodeLoadStatus:
+			return offsetof(SMU73_SoftRegisters, UcodeLoadStatus);
+		}
+	case SMU_Discrete_DpmTable:
+		switch (member) {
+		case UvdBootLevel:
+			return offsetof(SMU73_Discrete_DpmTable, UvdBootLevel);
+		case VceBootLevel:
+			return offsetof(SMU73_Discrete_DpmTable, VceBootLevel);
+		case SamuBootLevel:
+			return offsetof(SMU73_Discrete_DpmTable, SamuBootLevel);
+		case LowSclkInterruptThreshold:
+			return offsetof(SMU73_Discrete_DpmTable, LowSclkInterruptThreshold);
+		}
+	}
+	printk("cant't get the offset of type %x member %x \n", type, member);
+	return 0;
+}
+
+uint32_t fiji_get_mac_definition(uint32_t value)
+{
+	switch (value) {
+	case SMU_MAX_LEVELS_GRAPHICS:
+		return SMU73_MAX_LEVELS_GRAPHICS;
+	case SMU_MAX_LEVELS_MEMORY:
+		return SMU73_MAX_LEVELS_MEMORY;
+	case SMU_MAX_LEVELS_LINK:
+		return SMU73_MAX_LEVELS_LINK;
+	case SMU_MAX_ENTRIES_SMIO:
+		return SMU73_MAX_ENTRIES_SMIO;
+	case SMU_MAX_LEVELS_VDDC:
+		return SMU73_MAX_LEVELS_VDDC;
+	case SMU_MAX_LEVELS_VDDGFX:
+		return SMU73_MAX_LEVELS_VDDGFX;
+	case SMU_MAX_LEVELS_VDDCI:
+		return SMU73_MAX_LEVELS_VDDCI;
+	case SMU_MAX_LEVELS_MVDD:
+		return SMU73_MAX_LEVELS_MVDD;
+	}
+
+	printk("cant't get the mac of %x \n", value);
+	return 0;
+}
+
+
+static int fiji_update_uvd_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	smu_data->smc_state_table.UvdBootLevel = 0;
+	if (table_info->mm_dep_table->count > 0)
+		smu_data->smc_state_table.UvdBootLevel =
+				(uint8_t) (table_info->mm_dep_table->count - 1);
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start + offsetof(SMU73_Discrete_DpmTable,
+						UvdBootLevel);
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0x00FFFFFF;
+	mm_boot_level_value |= smu_data->smc_state_table.UvdBootLevel << 24;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_UVDDPM) ||
+		phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_UVDDPM_SetEnabledMask,
+				(uint32_t)(1 << smu_data->smc_state_table.UvdBootLevel));
+	return 0;
+}
+
+static int fiji_update_vce_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_StablePState))
+		smu_data->smc_state_table.VceBootLevel =
+			(uint8_t) (table_info->mm_dep_table->count - 1);
+	else
+		smu_data->smc_state_table.VceBootLevel = 0;
+
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start +
+					offsetof(SMU73_Discrete_DpmTable, VceBootLevel);
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0xFF00FFFF;
+	mm_boot_level_value |= smu_data->smc_state_table.VceBootLevel << 16;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_VCEDPM_SetEnabledMask,
+				(uint32_t)1 << smu_data->smc_state_table.VceBootLevel);
+	return 0;
+}
+
+static int fiji_update_samu_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+
+
+	smu_data->smc_state_table.SamuBootLevel = 0;
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU73_Discrete_DpmTable, SamuBootLevel);
+
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0xFFFFFF00;
+	mm_boot_level_value |= smu_data->smc_state_table.SamuBootLevel << 0;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SAMUDPM_SetEnabledMask,
+				(uint32_t)(1 << smu_data->smc_state_table.SamuBootLevel));
+	return 0;
+}
+
+int fiji_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type)
+{
+	switch (type) {
+	case SMU_UVD_TABLE:
+		fiji_update_uvd_smc_table(hwmgr);
+		break;
+	case SMU_VCE_TABLE:
+		fiji_update_vce_smc_table(hwmgr);
+		break;
+	case SMU_SAMU_TABLE:
+		fiji_update_samu_smc_table(hwmgr);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+
+/**
+* Get the location of various tables inside the FW image.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always  0
+*/
+int fiji_process_firmware_header(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct fiji_smumgr *smu_data = (struct fiji_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t tmp;
+	int result;
+	bool error = false;
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, DpmTable),
+			&tmp, SMC_RAM_END);
+
+	if (0 == result)
+		smu_data->smu7_data.dpm_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, SoftRegisters),
+			&tmp, SMC_RAM_END);
+
+	if (!result) {
+		data->soft_regs_start = tmp;
+		smu_data->smu7_data.soft_regs_start = tmp;
+	}
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, mcRegisterTable),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.mc_reg_table_start = tmp;
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, FanTable),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.fan_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, mcArbDramTimingTable),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.arb_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU73_Firmware_Header, Version),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		hwmgr->microcode_version_info.SMC = tmp;
+
+	error |= (0 != result);
+
+	return error ? -1 : 0;
+}
+
+int fiji_initialize_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+
+	/* Program additional LP registers
+	 * that are no longer programmed by VBIOS
+	 */
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RAS_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_RAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_CAS_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_CAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_TIMING));
+
+	return 0;
+}
+
+bool fiji_is_dpm_running(struct pp_hwmgr *hwmgr)
+{
+	return (1 == PHM_READ_INDIRECT_FIELD(hwmgr->device,
+			CGS_IND_REG__SMC, FEATURE_STATUS, VOLTAGE_CONTROLLER_ON))
+			? true : false;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.h b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.h
new file mode 100644
index 000000000000..d30d150f9ca6
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef FIJI_SMC_H
+#define FIJI_SMC_H
+
+#include "smumgr.h"
+#include "smu73.h"
+
+struct fiji_pt_defaults {
+	uint8_t   SviLoadLineEn;
+	uint8_t   SviLoadLineVddC;
+	uint8_t   TDC_VDDC_ThrottleReleaseLimitPerc;
+	uint8_t   TDC_MAWt;
+	uint8_t   TdcWaterfallCtl;
+	uint8_t   DTEAmbientTempBase;
+};
+
+int fiji_populate_all_graphic_levels(struct pp_hwmgr *hwmgr);
+int fiji_populate_all_memory_levels(struct pp_hwmgr *hwmgr);
+int fiji_init_smc_table(struct pp_hwmgr *hwmgr);
+int fiji_thermal_setup_fan_table(struct pp_hwmgr *hwmgr);
+int fiji_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type);
+int fiji_update_sclk_threshold(struct pp_hwmgr *hwmgr);
+uint32_t fiji_get_offsetof(uint32_t type, uint32_t member);
+uint32_t fiji_get_mac_definition(uint32_t value);
+int fiji_process_firmware_header(struct pp_hwmgr *hwmgr);
+int fiji_initialize_mc_reg_table(struct pp_hwmgr *hwmgr);
+bool fiji_is_dpm_running(struct pp_hwmgr *hwmgr);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
index 8e52a2e82db5..02fe1df855a9 100644..100755
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
@@ -38,6 +38,7 @@
 #include "bif/bif_5_0_sh_mask.h"
 #include "pp_debug.h"
 #include "fiji_pwrvirus.h"
+#include "fiji_smc.h"
 
 #define AVFS_EN_MSB                                        1568
 #define AVFS_EN_LSB                                        1568
@@ -57,509 +58,6 @@ static const struct SMU73_Discrete_GraphicsLevel avfs_graphics_level[8] = {
 		{ 0xf811d047, 0x80380100,   0x01,     0x00,   0x1e00, 0x00000610, 0x87020000, 0x21680000, 0x12000000,   0,      0,   0x0c,   0x01,       0x01,        0x01,      0x00,   0x00,      0x00,     0x00 }
 };
 
-static enum cgs_ucode_id fiji_convert_fw_type_to_cgs(uint32_t fw_type)
-{
-	enum cgs_ucode_id result = CGS_UCODE_ID_MAXIMUM;
-
-	switch (fw_type) {
-	case UCODE_ID_SMU:
-		result = CGS_UCODE_ID_SMU;
-		break;
-	case UCODE_ID_SDMA0:
-		result = CGS_UCODE_ID_SDMA0;
-		break;
-	case UCODE_ID_SDMA1:
-		result = CGS_UCODE_ID_SDMA1;
-		break;
-	case UCODE_ID_CP_CE:
-		result = CGS_UCODE_ID_CP_CE;
-		break;
-	case UCODE_ID_CP_PFP:
-		result = CGS_UCODE_ID_CP_PFP;
-		break;
-	case UCODE_ID_CP_ME:
-		result = CGS_UCODE_ID_CP_ME;
-		break;
-	case UCODE_ID_CP_MEC:
-		result = CGS_UCODE_ID_CP_MEC;
-		break;
-	case UCODE_ID_CP_MEC_JT1:
-		result = CGS_UCODE_ID_CP_MEC_JT1;
-		break;
-	case UCODE_ID_CP_MEC_JT2:
-		result = CGS_UCODE_ID_CP_MEC_JT2;
-		break;
-	case UCODE_ID_RLC_G:
-		result = CGS_UCODE_ID_RLC_G;
-		break;
-	default:
-		break;
-	}
-
-	return result;
-}
-/**
-* Set the address for reading/writing the SMC SRAM space.
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    smc_addr the address in the SMC RAM to access.
-*/
-static int fiji_set_smc_sram_address(struct pp_smumgr *smumgr,
-		uint32_t smc_addr, uint32_t limit)
-{
-	PP_ASSERT_WITH_CODE((0 == (3 & smc_addr)),
-			"SMC address must be 4 byte aligned.", return -EINVAL;);
-	PP_ASSERT_WITH_CODE((limit > (smc_addr + 3)),
-			"SMC address is beyond the SMC RAM area.", return -EINVAL;);
-
-	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_0, smc_addr);
-	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
-
-	return 0;
-}
-
-/**
-* Copy bytes from an array into the SMC RAM space.
-*
-* @param    smumgr  the address of the powerplay SMU manager.
-* @param    smcStartAddress the start address in the SMC RAM to copy bytes to.
-* @param    src the byte array to copy the bytes from.
-* @param    byteCount the number of bytes to copy.
-*/
-int fiji_copy_bytes_to_smc(struct pp_smumgr *smumgr,
-		uint32_t smcStartAddress, const uint8_t *src,
-		uint32_t byteCount, uint32_t limit)
-{
-	int result;
-	uint32_t data, originalData;
-	uint32_t addr, extraShift;
-
-	PP_ASSERT_WITH_CODE((0 == (3 & smcStartAddress)),
-			"SMC address must be 4 byte aligned.", return -EINVAL;);
-	PP_ASSERT_WITH_CODE((limit > (smcStartAddress + byteCount)),
-			"SMC address is beyond the SMC RAM area.", return -EINVAL;);
-
-	addr = smcStartAddress;
-
-	while (byteCount >= 4) {
-		/* Bytes are written into the SMC addres space with the MSB first. */
-		data = src[0] * 0x1000000 + src[1] * 0x10000 + src[2] * 0x100 + src[3];
-
-		result = fiji_set_smc_sram_address(smumgr, addr, limit);
-		if (result)
-			return result;
-
-		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data);
-
-		src += 4;
-		byteCount -= 4;
-		addr += 4;
-	}
-
-	if (byteCount) {
-		/* Now write the odd bytes left.
-		 * Do a read modify write cycle.
-		 */
-		data = 0;
-
-		result = fiji_set_smc_sram_address(smumgr, addr, limit);
-		if (result)
-			return result;
-
-		originalData = cgs_read_register(smumgr->device, mmSMC_IND_DATA_0);
-		extraShift = 8 * (4 - byteCount);
-
-		while (byteCount > 0) {
-			/* Bytes are written into the SMC addres
-			 * space with the MSB first.
-			 */
-			data = (0x100 * data) + *src++;
-			byteCount--;
-		}
-		data <<= extraShift;
-		data |= (originalData & ~((~0UL) << extraShift));
-
-		result = fiji_set_smc_sram_address(smumgr, addr, limit);
-		if (!result)
-			return result;
-
-		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data);
-	}
-	return 0;
-}
-
-int fiji_program_jump_on_start(struct pp_smumgr *smumgr)
-{
-	static const unsigned char data[] = { 0xE0, 0x00, 0x80, 0x40 };
-
-	fiji_copy_bytes_to_smc(smumgr, 0x0, data, 4, sizeof(data) + 1);
-
-	return 0;
-}
-
-/**
-* Return if the SMC is currently running.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-*/
-bool fiji_is_smc_ram_running(struct pp_smumgr *smumgr)
-{
-	return ((0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device,
-			CGS_IND_REG__SMC,
-			SMC_SYSCON_CLOCK_CNTL_0, ck_disable))
-			&& (0x20100 <= cgs_read_ind_register(smumgr->device,
-					CGS_IND_REG__SMC, ixSMC_PC_C)));
-}
-
-/**
-* Send a message to the SMC, and wait for its response.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    msg the message to send.
-* @return   The response that came from the SMC.
-*/
-int fiji_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
-{
-	if (!fiji_is_smc_ram_running(smumgr))
-		return -1;
-
-	if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP)) {
-		printk(KERN_ERR "Failed to send Previous Message.");
-		SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-	}
-
-	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-
-	return 0;
-}
-
-/**
- * Send a message to the SMC with parameter
- * @param    smumgr:  the address of the powerplay hardware manager.
- * @param    msg: the message to send.
- * @param    parameter: the parameter to send
- * @return   The response that came from the SMC.
- */
-int fiji_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
-		uint16_t msg, uint32_t parameter)
-{
-	if (!fiji_is_smc_ram_running(smumgr))
-		return -1;
-
-	if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP)) {
-		printk(KERN_ERR "Failed to send Previous Message.");
-		SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-	}
-
-	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
-	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-
-	return 0;
-}
-
-
-/**
-* Send a message to the SMC with parameter, do not wait for response
-*
-* @param    smumgr:  the address of the powerplay hardware manager.
-* @param    msg: the message to send.
-* @param    parameter: the parameter to send
-* @return   The response that came from the SMC.
-*/
-int fiji_send_msg_to_smc_with_parameter_without_waiting(
-		struct pp_smumgr *smumgr, uint16_t msg, uint32_t parameter)
-{
-	if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP)) {
-		printk(KERN_ERR "Failed to send Previous Message.");
-		SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-	}
-	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
-	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
-
-	return 0;
-}
-
-/**
-* Uploads the SMU firmware from .hex file
-*
-* @param    smumgr  the address of the powerplay SMU manager.
-* @return   0 or -1.
-*/
-
-static int fiji_upload_smu_firmware_image(struct pp_smumgr *smumgr)
-{
-	const uint8_t *src;
-	uint32_t byte_count;
-	uint32_t *data;
-	struct cgs_firmware_info info = {0};
-
-	cgs_get_firmware_info(smumgr->device,
-			fiji_convert_fw_type_to_cgs(UCODE_ID_SMU), &info);
-
-	if (info.image_size & 3) {
-		printk(KERN_ERR "SMC ucode is not 4 bytes aligned\n");
-		return -EINVAL;
-	}
-
-	if (info.image_size > FIJI_SMC_SIZE) {
-		printk(KERN_ERR "SMC address is beyond the SMC RAM area\n");
-		return -EINVAL;
-	}
-
-	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_0, 0x20000);
-	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1);
-
-	byte_count = info.image_size;
-	src = (const uint8_t *)info.kptr;
-
-	data = (uint32_t *)src;
-	for (; byte_count >= 4; data++, byte_count -= 4)
-		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data[0]);
-
-	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
-	return 0;
-}
-
-/**
-* Read a 32bit value from the SMC SRAM space.
-* ALL PARAMETERS ARE IN HOST BYTE ORDER.
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    smc_addr the address in the SMC RAM to access.
-* @param    value and output parameter for the data read from the SMC SRAM.
-*/
-int fiji_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr,
-		uint32_t *value, uint32_t limit)
-{
-	int	result = fiji_set_smc_sram_address(smumgr, smc_addr, limit);
-
-	if (result)
-		return result;
-
-	*value = cgs_read_register(smumgr->device, mmSMC_IND_DATA_0);
-	return 0;
-}
-
-/**
-* Write a 32bit value to the SMC SRAM space.
-* ALL PARAMETERS ARE IN HOST BYTE ORDER.
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    smc_addr the address in the SMC RAM to access.
-* @param    value to write to the SMC SRAM.
-*/
-int fiji_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr,
-		uint32_t value, uint32_t limit)
-{
-	int result;
-
-	result = fiji_set_smc_sram_address(smumgr, smc_addr, limit);
-
-	if (result)
-		return result;
-
-	cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, value);
-	return 0;
-}
-
-static uint32_t fiji_get_mask_for_firmware_type(uint32_t fw_type)
-{
-	uint32_t result = 0;
-
-	switch (fw_type) {
-	case UCODE_ID_SDMA0:
-		result = UCODE_ID_SDMA0_MASK;
-		break;
-	case UCODE_ID_SDMA1:
-		result = UCODE_ID_SDMA1_MASK;
-		break;
-	case UCODE_ID_CP_CE:
-		result = UCODE_ID_CP_CE_MASK;
-		break;
-	case UCODE_ID_CP_PFP:
-		result = UCODE_ID_CP_PFP_MASK;
-		break;
-	case UCODE_ID_CP_ME:
-		result = UCODE_ID_CP_ME_MASK;
-		break;
-	case UCODE_ID_CP_MEC_JT1:
-		result = UCODE_ID_CP_MEC_MASK | UCODE_ID_CP_MEC_JT1_MASK;
-		break;
-	case UCODE_ID_CP_MEC_JT2:
-		result = UCODE_ID_CP_MEC_MASK | UCODE_ID_CP_MEC_JT2_MASK;
-		break;
-	case UCODE_ID_RLC_G:
-		result = UCODE_ID_RLC_G_MASK;
-		break;
-	default:
-		printk(KERN_ERR "UCode type is out of range!");
-		result = 0;
-	}
-
-	return result;
-}
-
-/* Populate one firmware image to the data structure */
-static int fiji_populate_single_firmware_entry(struct pp_smumgr *smumgr,
-		uint32_t fw_type, struct SMU_Entry *entry)
-{
-	int result;
-	struct cgs_firmware_info info = {0};
-
-	result = cgs_get_firmware_info(
-			smumgr->device,
-			fiji_convert_fw_type_to_cgs(fw_type),
-			&info);
-
-	if (!result) {
-		entry->version = 0;
-		entry->id = (uint16_t)fw_type;
-		entry->image_addr_high = smu_upper_32_bits(info.mc_addr);
-		entry->image_addr_low = smu_lower_32_bits(info.mc_addr);
-		entry->meta_data_addr_high = 0;
-		entry->meta_data_addr_low = 0;
-		entry->data_size_byte = info.image_size;
-		entry->num_register_entries = 0;
-
-		if (fw_type == UCODE_ID_RLC_G)
-			entry->flags = 1;
-		else
-			entry->flags = 0;
-	}
-
-	return result;
-}
-
-static int fiji_request_smu_load_fw(struct pp_smumgr *smumgr)
-{
-	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
-	uint32_t fw_to_load;
-	struct SMU_DRAMData_TOC *toc;
-
-	if (priv->soft_regs_start)
-		cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
-				priv->soft_regs_start +
-				offsetof(SMU73_SoftRegisters, UcodeLoadStatus),
-				0x0);
-
-	toc = (struct SMU_DRAMData_TOC *)priv->header;
-	toc->num_entries = 0;
-	toc->structure_version = 1;
-
-	PP_ASSERT_WITH_CODE(
-			0 == fiji_populate_single_firmware_entry(smumgr,
-					UCODE_ID_RLC_G, &toc->entry[toc->num_entries++]),
-			"Failed to Get Firmware Entry.\n" , return -1 );
-	PP_ASSERT_WITH_CODE(
-			0 == fiji_populate_single_firmware_entry(smumgr,
-					UCODE_ID_CP_CE, &toc->entry[toc->num_entries++]),
-			"Failed to Get Firmware Entry.\n" , return -1 );
-	PP_ASSERT_WITH_CODE(
-			0 == fiji_populate_single_firmware_entry(smumgr,
-					UCODE_ID_CP_PFP, &toc->entry[toc->num_entries++]),
-			"Failed to Get Firmware Entry.\n" , return -1 );
-	PP_ASSERT_WITH_CODE(
-			0 == fiji_populate_single_firmware_entry(smumgr,
-					UCODE_ID_CP_ME, &toc->entry[toc->num_entries++]),
-			"Failed to Get Firmware Entry.\n" , return -1 );
-	PP_ASSERT_WITH_CODE(
-			0 == fiji_populate_single_firmware_entry(smumgr,
-					UCODE_ID_CP_MEC, &toc->entry[toc->num_entries++]),
-			"Failed to Get Firmware Entry.\n" , return -1 );
-	PP_ASSERT_WITH_CODE(
-			0 == fiji_populate_single_firmware_entry(smumgr,
-					UCODE_ID_CP_MEC_JT1, &toc->entry[toc->num_entries++]),
-					"Failed to Get Firmware Entry.\n" , return -1 );
-	PP_ASSERT_WITH_CODE(
-			0 == fiji_populate_single_firmware_entry(smumgr,
-					UCODE_ID_CP_MEC_JT2, &toc->entry[toc->num_entries++]),
-					"Failed to Get Firmware Entry.\n" , return -1 );
-	PP_ASSERT_WITH_CODE(
-			0 == fiji_populate_single_firmware_entry(smumgr,
-					UCODE_ID_SDMA0, &toc->entry[toc->num_entries++]),
-					"Failed to Get Firmware Entry.\n" , return -1 );
-	PP_ASSERT_WITH_CODE(
-			0 == fiji_populate_single_firmware_entry(smumgr,
-					UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]),
-					"Failed to Get Firmware Entry.\n" , return -1 );
-
-	fiji_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_DRV_DRAM_ADDR_HI,
-			priv->header_buffer.mc_addr_high);
-	fiji_send_msg_to_smc_with_parameter(smumgr,PPSMC_MSG_DRV_DRAM_ADDR_LO,
-			priv->header_buffer.mc_addr_low);
-
-	fw_to_load = UCODE_ID_RLC_G_MASK
-			+ UCODE_ID_SDMA0_MASK
-			+ UCODE_ID_SDMA1_MASK
-			+ UCODE_ID_CP_CE_MASK
-			+ UCODE_ID_CP_ME_MASK
-			+ UCODE_ID_CP_PFP_MASK
-			+ UCODE_ID_CP_MEC_MASK
-			+ UCODE_ID_CP_MEC_JT1_MASK
-			+ UCODE_ID_CP_MEC_JT2_MASK;
-
-	if (fiji_send_msg_to_smc_with_parameter(smumgr,
-			PPSMC_MSG_LoadUcodes, fw_to_load))
-		printk(KERN_ERR "Fail to Request SMU Load uCode");
-
-	return 0;
-}
-
-
-/* Check if the FW has been loaded, SMU will not return
- * if loading has not finished.
- */
-static int fiji_check_fw_load_finish(struct pp_smumgr *smumgr,
-		uint32_t fw_type)
-{
-	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
-	uint32_t mask = fiji_get_mask_for_firmware_type(fw_type);
-
-	/* Check SOFT_REGISTERS_TABLE_28.UcodeLoadStatus */
-	if (smum_wait_on_indirect_register(smumgr, mmSMC_IND_INDEX,
-			priv->soft_regs_start +
-			offsetof(SMU73_SoftRegisters, UcodeLoadStatus),
-			mask, mask)) {
-		printk(KERN_ERR "check firmware loading failed\n");
-		return -EINVAL;
-	}
-	return 0;
-}
-
-
-static int fiji_reload_firmware(struct pp_smumgr *smumgr)
-{
-	return smumgr->smumgr_funcs->start_smu(smumgr);
-}
-
-static bool fiji_is_hw_virtualization_enabled(struct pp_smumgr *smumgr)
-{
-	uint32_t value;
-
-	value = cgs_read_register(smumgr->device, mmBIF_IOV_FUNC_IDENTIFIER);
-	if (value & BIF_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK) {
-		/* driver reads on SR-IOV enabled PF: 0x80000000
-		 * driver reads on SR-IOV enabled VF: 0x80000001
-		 * driver reads on SR-IOV disabled:   0x00000000
-		 */
-		return true;
-	}
-	return false;
-}
-
-static int fiji_request_smu_specific_fw_load(struct pp_smumgr *smumgr, uint32_t fw_type)
-{
-	if (fiji_is_hw_virtualization_enabled(smumgr)) {
-		uint32_t masks = fiji_get_mask_for_firmware_type(fw_type);
-		if (fiji_send_msg_to_smc_with_parameter_without_waiting(smumgr,
-				PPSMC_MSG_LoadUcodes, masks))
-			printk(KERN_ERR "Fail to Request SMU Load uCode");
-	}
-	/* For non-virtualization cases,
-	 * SMU loads all FWs at once in fiji_request_smu_load_fw.
-	 */
-	return 0;
-}
-
 static int fiji_start_smu_in_protection_mode(struct pp_smumgr *smumgr)
 {
 	int result = 0;
@@ -571,7 +69,7 @@ static int fiji_start_smu_in_protection_mode(struct pp_smumgr *smumgr)
 	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
 			SMC_SYSCON_RESET_CNTL, rst_reg, 1);
 
-	result = fiji_upload_smu_firmware_image(smumgr);
+	result = smu7_upload_smu_firmware_image(smumgr);
 	if (result)
 		return result;
 
@@ -610,8 +108,8 @@ static int fiji_start_smu_in_protection_mode(struct pp_smumgr *smumgr)
 			SMU_STATUS, SMU_DONE, 0);
 
 	/* Check pass/failed indicator */
-	if (1 != SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
-			SMU_STATUS, SMU_PASS)) {
+	if (SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+			SMU_STATUS, SMU_PASS) != 1) {
 		PP_ASSERT_WITH_CODE(false,
 				"SMU Firmware start failed!", return -1);
 	}
@@ -639,12 +137,12 @@ static int fiji_start_smu_in_non_protection_mode(struct pp_smumgr *smumgr)
 	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
 			SMC_SYSCON_RESET_CNTL, rst_reg, 1);
 
-	result = fiji_upload_smu_firmware_image(smumgr);
+	result = smu7_upload_smu_firmware_image(smumgr);
 	if (result)
 		return result;
 
 	/* Set smc instruct start point at 0x0 */
-	fiji_program_jump_on_start(smumgr);
+	smu7_program_jump_on_start(smumgr);
 
 	/* Enable clock */
 	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
@@ -698,15 +196,15 @@ static int fiji_start_avfs_btc(struct pp_smumgr *smumgr)
 
 	priv->avfs.AvfsBtcStatus = AVFS_BTC_STARTED;
 	if (priv->avfs.AvfsBtcParam) {
-		if (!fiji_send_msg_to_smc_with_parameter(smumgr,
+		if (!smum_send_msg_to_smc_with_parameter(smumgr,
 				PPSMC_MSG_PerformBtc, priv->avfs.AvfsBtcParam)) {
-			if (!fiji_send_msg_to_smc(smumgr, PPSMC_MSG_EnableAvfs)) {
+			if (!smum_send_msg_to_smc(smumgr, PPSMC_MSG_EnableAvfs)) {
 				priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_UNSAVED;
 				result = 0;
 			} else {
 				printk(KERN_ERR "[AVFS][fiji_start_avfs_btc] Attempt"
 						" to Enable AVFS Failed!");
-				fiji_send_msg_to_smc(smumgr, PPSMC_MSG_DisableAvfs);
+				smum_send_msg_to_smc(smumgr, PPSMC_MSG_DisableAvfs);
 				result = -1;
 			}
 		} else {
@@ -736,7 +234,7 @@ int fiji_setup_pm_fuse_for_avfs(struct pp_smumgr *smumgr)
 	charz_freq = 0x30750000; /* In 10KHz units 0x00007530 Actual value */
 	inversion_voltage = 0x1A04; /* mV Q14.2 0x41A Actual value */
 
-	PP_ASSERT_WITH_CODE(0 == fiji_read_smc_sram_dword(smumgr,
+	PP_ASSERT_WITH_CODE(0 == smu7_read_smc_sram_dword(smumgr,
 			SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU73_Firmware_Header,
 					PmFuseTable), &table_start, 0x40000),
 			"[AVFS][Fiji_SetupGfxLvlStruct] SMU could not communicate "
@@ -748,13 +246,13 @@ int fiji_setup_pm_fuse_for_avfs(struct pp_smumgr *smumgr)
 	inversion_voltage_addr = table_start +
 			offsetof(struct SMU73_Discrete_PmFuses, InversionVoltage);
 
-	result = fiji_copy_bytes_to_smc(smumgr, charz_freq_addr,
+	result = smu7_copy_bytes_to_smc(smumgr, charz_freq_addr,
 			(uint8_t *)(&charz_freq), sizeof(charz_freq), 0x40000);
 	PP_ASSERT_WITH_CODE(0 == result,
 			"[AVFS][fiji_setup_pm_fuse_for_avfs] charz_freq could not "
 			"be populated.", return -1;);
 
-	result = fiji_copy_bytes_to_smc(smumgr, inversion_voltage_addr,
+	result = smu7_copy_bytes_to_smc(smumgr, inversion_voltage_addr,
 			(uint8_t *)(&inversion_voltage), sizeof(inversion_voltage), 0x40000);
 	PP_ASSERT_WITH_CODE(0 == result, "[AVFS][fiji_setup_pm_fuse_for_avfs] "
 			"charz_freq could not be populated.", return -1;);
@@ -769,7 +267,7 @@ int fiji_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 	uint32_t level_addr, vr_config_addr;
 	uint32_t level_size = sizeof(avfs_graphics_level);
 
-	PP_ASSERT_WITH_CODE(0 == fiji_read_smc_sram_dword(smumgr,
+	PP_ASSERT_WITH_CODE(0 == smu7_read_smc_sram_dword(smumgr,
 			SMU7_FIRMWARE_HEADER_LOCATION +
 			offsetof(SMU73_Firmware_Header, DpmTable),
 			&table_start, 0x40000),
@@ -784,7 +282,7 @@ int fiji_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 	vr_config_addr = table_start +
 			offsetof(SMU73_Discrete_DpmTable, VRConfig);
 
-	PP_ASSERT_WITH_CODE(0 == fiji_copy_bytes_to_smc(smumgr, vr_config_addr,
+	PP_ASSERT_WITH_CODE(0 == smu7_copy_bytes_to_smc(smumgr, vr_config_addr,
 			(uint8_t *)&vr_config, sizeof(int32_t), 0x40000),
 			"[AVFS][Fiji_SetupGfxLvlStruct] Problems copying "
 			"vr_config value over to SMC",
@@ -792,7 +290,7 @@ int fiji_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 
 	level_addr = table_start + offsetof(SMU73_Discrete_DpmTable, GraphicsLevel);
 
-	PP_ASSERT_WITH_CODE(0 == fiji_copy_bytes_to_smc(smumgr, level_addr,
+	PP_ASSERT_WITH_CODE(0 == smu7_copy_bytes_to_smc(smumgr, level_addr,
 			(uint8_t *)(&avfs_graphics_level), level_size, 0x40000),
 			"[AVFS][Fiji_SetupGfxLvlStruct] Copying of DPM table failed!",
 			return -1;);
@@ -839,13 +337,13 @@ int fiji_avfs_event_mgr(struct pp_smumgr *smumgr, bool smu_started)
 		break;
 	case AVFS_BTC_COMPLETED_RESTORED: /*S3 State - Post SMU Start*/
 		priv->avfs.AvfsBtcStatus = AVFS_BTC_SMUMSG_ERROR;
-		PP_ASSERT_WITH_CODE(0 == fiji_send_msg_to_smc(smumgr,
-				PPSMC_MSG_VftTableIsValid),
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(smumgr,
+				0x666),
 				"[AVFS][fiji_avfs_event_mgr] SMU did not respond "
 				"correctly to VftTableIsValid Msg",
 				return -1;);
 		priv->avfs.AvfsBtcStatus = AVFS_BTC_SMUMSG_ERROR;
-		PP_ASSERT_WITH_CODE(0 == fiji_send_msg_to_smc(smumgr,
+		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(smumgr,
 				PPSMC_MSG_EnableAvfs),
 				"[AVFS][fiji_avfs_event_mgr] SMU did not respond "
 				"correctly to EnableAvfs Message Msg",
@@ -898,7 +396,7 @@ static int fiji_start_smu(struct pp_smumgr *smumgr)
 	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
 
 	/* Only start SMC if SMC RAM is not running */
-	if (!fiji_is_smc_ram_running(smumgr)) {
+	if (!smu7_is_smc_ram_running(smumgr)) {
 		fiji_avfs_event_mgr(smumgr, false);
 
 		/* Check if SMU is running in protected mode */
@@ -929,12 +427,12 @@ static int fiji_start_smu(struct pp_smumgr *smumgr)
 	/* Setup SoftRegsStart here for register lookup in case
 	 * DummyBackEnd is used and ProcessFirmwareHeader is not executed
 	 */
-	fiji_read_smc_sram_dword(smumgr,
+	smu7_read_smc_sram_dword(smumgr,
 			SMU7_FIRMWARE_HEADER_LOCATION +
 			offsetof(SMU73_Firmware_Header, SoftRegisters),
-			&(priv->soft_regs_start), 0x40000);
+			&(priv->smu7_data.soft_regs_start), 0x40000);
 
-	result = fiji_request_smu_load_fw(smumgr);
+	result = smu7_request_smu_load_fw(smumgr);
 
 	return result;
 }
@@ -963,28 +461,10 @@ static bool fiji_is_hw_avfs_present(struct pp_smumgr *smumgr)
 static int fiji_smu_init(struct pp_smumgr *smumgr)
 {
 	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
-	uint64_t mc_addr;
-
-	priv->header_buffer.data_size =
-			((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
-	smu_allocate_memory(smumgr->device,
-			priv->header_buffer.data_size,
-			CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
-			PAGE_SIZE,
-			&mc_addr,
-			&priv->header_buffer.kaddr,
-			&priv->header_buffer.handle);
-
-	priv->header = priv->header_buffer.kaddr;
-	priv->header_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
-	priv->header_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
-
-	PP_ASSERT_WITH_CODE((NULL != priv->header),
-			"Out of memory.",
-			kfree(smumgr->backend);
-			cgs_free_gpu_mem(smumgr->device,
-			(cgs_handle_t)priv->header_buffer.handle);
-			return -1);
+	int i;
+
+	if (smu7_init(smumgr))
+		return -EINVAL;
 
 	priv->avfs.AvfsBtcStatus = AVFS_BTC_BOOT;
 	if (fiji_is_hw_avfs_present(smumgr))
@@ -999,37 +479,35 @@ static int fiji_smu_init(struct pp_smumgr *smumgr)
 	else
 		priv->avfs.AvfsBtcStatus = AVFS_BTC_NOTSUPPORTED;
 
-	priv->acpi_optimization = 1;
+	for (i = 0; i < SMU73_MAX_LEVELS_GRAPHICS; i++)
+		priv->activity_target[i] = 30;
 
 	return 0;
 }
 
-static int fiji_smu_fini(struct pp_smumgr *smumgr)
-{
-	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
-
-	smu_free_memory(smumgr->device, (void *)priv->header_buffer.handle);
-
-	if (smumgr->backend) {
-		kfree(smumgr->backend);
-		smumgr->backend = NULL;
-	}
-
-	cgs_rel_firmware(smumgr->device, CGS_UCODE_ID_SMU);
-	return 0;
-}
 
 static const struct pp_smumgr_func fiji_smu_funcs = {
 	.smu_init = &fiji_smu_init,
-	.smu_fini = &fiji_smu_fini,
+	.smu_fini = &smu7_smu_fini,
 	.start_smu = &fiji_start_smu,
-	.check_fw_load_finish = &fiji_check_fw_load_finish,
-	.request_smu_load_fw = &fiji_reload_firmware,
-	.request_smu_load_specific_fw = &fiji_request_smu_specific_fw_load,
-	.send_msg_to_smc = &fiji_send_msg_to_smc,
-	.send_msg_to_smc_with_parameter = &fiji_send_msg_to_smc_with_parameter,
+	.check_fw_load_finish = &smu7_check_fw_load_finish,
+	.request_smu_load_fw = &smu7_reload_firmware,
+	.request_smu_load_specific_fw = NULL,
+	.send_msg_to_smc = &smu7_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = &smu7_send_msg_to_smc_with_parameter,
 	.download_pptable_settings = NULL,
 	.upload_pptable_settings = NULL,
+	.update_smc_table = fiji_update_smc_table,
+	.get_offsetof = fiji_get_offsetof,
+	.process_firmware_header = fiji_process_firmware_header,
+	.init_smc_table = fiji_init_smc_table,
+	.update_sclk_threshold = fiji_update_sclk_threshold,
+	.thermal_setup_fan_table = fiji_thermal_setup_fan_table,
+	.populate_all_graphic_levels = fiji_populate_all_graphic_levels,
+	.populate_all_memory_levels = fiji_populate_all_memory_levels,
+	.get_mac_definition = fiji_get_mac_definition,
+	.initialize_mc_reg_table = fiji_initialize_mc_reg_table,
+	.is_dpm_running = fiji_is_dpm_running,
 };
 
 int fiji_smum_init(struct pp_smumgr *smumgr)
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h
index b4eb483215b1..adcbdfb209be 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h
@@ -23,37 +23,31 @@
 #ifndef _FIJI_SMUMANAGER_H_
 #define _FIJI_SMUMANAGER_H_
 
+#include "smu73_discrete.h"
+#include <pp_endian.h>
+#include "smu7_smumgr.h"
+
+
 
 struct fiji_smu_avfs {
 	enum AVFS_BTC_STATUS AvfsBtcStatus;
 	uint32_t           AvfsBtcParam;
 };
 
-struct fiji_buffer_entry {
-	uint32_t data_size;
-	uint32_t mc_addr_low;
-	uint32_t mc_addr_high;
-	void *kaddr;
-	unsigned long  handle;
-};
 
 struct fiji_smumgr {
-	uint8_t        *header;
-	uint8_t        *mec_image;
-	uint32_t        soft_regs_start;
+	struct smu7_smumgr                   smu7_data;
+
 	struct fiji_smu_avfs avfs;
-	uint32_t        acpi_optimization;
+	struct SMU73_Discrete_DpmTable       smc_state_table;
+	struct SMU73_Discrete_Ulv            ulv_setting;
+	struct SMU73_Discrete_PmFuses  power_tune_table;
+	const struct fiji_pt_defaults  *power_tune_defaults;
+	uint32_t        activity_target[SMU73_MAX_LEVELS_GRAPHICS];
 
-	struct fiji_buffer_entry header_buffer;
 };
 
-int fiji_smum_init(struct pp_smumgr *smumgr);
-int fiji_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smcAddress,
-		uint32_t *value, uint32_t limit);
-int fiji_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr,
-		uint32_t value, uint32_t limit);
-int fiji_copy_bytes_to_smc(struct pp_smumgr *smumgr, uint32_t smcStartAddress,
-		const uint8_t *src,	uint32_t byteCount, uint32_t limit);
+
 
 #endif
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
new file mode 100644
index 000000000000..eda802bc63c8
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
@@ -0,0 +1,2576 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ *
+ */
+
+#include "iceland_smc.h"
+#include "smu7_dyn_defaults.h"
+
+#include "smu7_hwmgr.h"
+#include "hardwaremanager.h"
+#include "ppatomctrl.h"
+#include "pp_debug.h"
+#include "cgs_common.h"
+#include "atombios.h"
+#include "pppcielanes.h"
+#include "pp_endian.h"
+#include "smu7_ppsmc.h"
+
+#include "smu71_discrete.h"
+
+#include "smu/smu_7_1_1_d.h"
+#include "smu/smu_7_1_1_sh_mask.h"
+
+#include "gmc/gmc_8_1_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "processpptables.h"
+
+#include "iceland_smumgr.h"
+
+#define VOLTAGE_SCALE 4
+#define POWERTUNE_DEFAULT_SET_MAX    1
+#define VOLTAGE_VID_OFFSET_SCALE1   625
+#define VOLTAGE_VID_OFFSET_SCALE2   100
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define VDDC_VDDCI_DELTA            200
+
+#define DEVICE_ID_VI_ICELAND_M_6900	0x6900
+#define DEVICE_ID_VI_ICELAND_M_6901	0x6901
+#define DEVICE_ID_VI_ICELAND_M_6902	0x6902
+#define DEVICE_ID_VI_ICELAND_M_6903	0x6903
+
+static const struct iceland_pt_defaults defaults_iceland = {
+	/*
+	 * sviLoadLIneEn, SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc,
+	 * TDC_MAWt, TdcWaterfallCtl, DTEAmbientTempBase, DisplayCac, BAPM_TEMP_GRADIENT
+	 */
+	1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000,
+	{ 0x79,  0x253, 0x25D, 0xAE,  0x72,  0x80,  0x83,  0x86,  0x6F,  0xC8,  0xC9,  0xC9,  0x2F,  0x4D,  0x61  },
+	{ 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 }
+};
+
+/* 35W - XT, XTL */
+static const struct iceland_pt_defaults defaults_icelandxt = {
+	/*
+	 * sviLoadLIneEn, SviLoadLineVddC,
+	 * TDC_VDDC_ThrottleReleaseLimitPerc, TDC_MAWt,
+	 * TdcWaterfallCtl, DTEAmbientTempBase, DisplayCac,
+	 * BAPM_TEMP_GRADIENT
+	 */
+	1, 0xF, 0xFD, 0x19, 5, 45, 0, 0x0,
+	{ 0xA7,  0x0, 0x0, 0xB5,  0x0, 0x0, 0x9F,  0x0, 0x0, 0xD6,  0x0, 0x0, 0xD7,  0x0, 0x0},
+	{ 0x1EA, 0x0, 0x0, 0x224, 0x0, 0x0, 0x25E, 0x0, 0x0, 0x28E, 0x0, 0x0, 0x2AB, 0x0, 0x0}
+};
+
+/* 25W - PRO, LE */
+static const struct iceland_pt_defaults defaults_icelandpro = {
+	/*
+	 * sviLoadLIneEn, SviLoadLineVddC,
+	 * TDC_VDDC_ThrottleReleaseLimitPerc, TDC_MAWt,
+	 * TdcWaterfallCtl, DTEAmbientTempBase, DisplayCac,
+	 * BAPM_TEMP_GRADIENT
+	 */
+	1, 0xF, 0xFD, 0x19, 5, 45, 0, 0x0,
+	{ 0xB7,  0x0, 0x0, 0xC3,  0x0, 0x0, 0xB5,  0x0, 0x0, 0xEA,  0x0, 0x0, 0xE6,  0x0, 0x0},
+	{ 0x1EA, 0x0, 0x0, 0x224, 0x0, 0x0, 0x25E, 0x0, 0x0, 0x28E, 0x0, 0x0, 0x2AB, 0x0, 0x0}
+};
+
+static void iceland_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
+{
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	struct cgs_system_info sys_info = {0};
+	uint32_t dev_id;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_DEV;
+	cgs_query_system_info(hwmgr->device, &sys_info);
+	dev_id = (uint32_t)sys_info.value;
+
+	switch (dev_id) {
+	case DEVICE_ID_VI_ICELAND_M_6900:
+	case DEVICE_ID_VI_ICELAND_M_6903:
+		smu_data->power_tune_defaults = &defaults_icelandxt;
+		break;
+
+	case DEVICE_ID_VI_ICELAND_M_6901:
+	case DEVICE_ID_VI_ICELAND_M_6902:
+		smu_data->power_tune_defaults = &defaults_icelandpro;
+		break;
+	default:
+		smu_data->power_tune_defaults = &defaults_iceland;
+		pr_warning("Unknown V.I. Device ID.\n");
+		break;
+	}
+	return;
+}
+
+static int iceland_populate_svi_load_line(struct pp_hwmgr *hwmgr)
+{
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	const struct iceland_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	smu_data->power_tune_table.SviLoadLineEn = defaults->svi_load_line_en;
+	smu_data->power_tune_table.SviLoadLineVddC = defaults->svi_load_line_vddc;
+	smu_data->power_tune_table.SviLoadLineTrimVddC = 3;
+	smu_data->power_tune_table.SviLoadLineOffsetVddC = 0;
+
+	return 0;
+}
+
+static int iceland_populate_tdc_limit(struct pp_hwmgr *hwmgr)
+{
+	uint16_t tdc_limit;
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	const struct iceland_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	tdc_limit = (uint16_t)(hwmgr->dyn_state.cac_dtp_table->usTDC * 256);
+	smu_data->power_tune_table.TDC_VDDC_PkgLimit =
+			CONVERT_FROM_HOST_TO_SMC_US(tdc_limit);
+	smu_data->power_tune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
+			defaults->tdc_vddc_throttle_release_limit_perc;
+	smu_data->power_tune_table.TDC_MAWt = defaults->tdc_mawt;
+
+	return 0;
+}
+
+static int iceland_populate_dw8(struct pp_hwmgr *hwmgr, uint32_t fuse_table_offset)
+{
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	const struct iceland_pt_defaults *defaults = smu_data->power_tune_defaults;
+	uint32_t temp;
+
+	if (smu7_read_smc_sram_dword(hwmgr->smumgr,
+			fuse_table_offset +
+			offsetof(SMU71_Discrete_PmFuses, TdcWaterfallCtl),
+			(uint32_t *)&temp, SMC_RAM_END))
+		PP_ASSERT_WITH_CODE(false,
+				"Attempt to read PmFuses.DW6 (SviLoadLineEn) from SMC Failed!",
+				return -EINVAL);
+	else
+		smu_data->power_tune_table.TdcWaterfallCtl = defaults->tdc_waterfall_ctl;
+
+	return 0;
+}
+
+static int iceland_populate_temperature_scaler(struct pp_hwmgr *hwmgr)
+{
+	return 0;
+}
+
+static int iceland_populate_gnb_lpml(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 8; i++)
+		smu_data->power_tune_table.GnbLPML[i] = 0;
+
+	return 0;
+}
+
+static int iceland_min_max_vgnb_lpml_id_from_bapm_vddc(struct pp_hwmgr *hwmgr)
+{
+	return 0;
+}
+
+static int iceland_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr)
+{
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	uint16_t HiSidd = smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd;
+	uint16_t LoSidd = smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd;
+	struct phm_cac_tdp_table *cac_table = hwmgr->dyn_state.cac_dtp_table;
+
+	HiSidd = (uint16_t)(cac_table->usHighCACLeakage / 100 * 256);
+	LoSidd = (uint16_t)(cac_table->usLowCACLeakage / 100 * 256);
+
+	smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(HiSidd);
+	smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(LoSidd);
+
+	return 0;
+}
+
+static int iceland_populate_bapm_vddc_vid_sidd(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	uint8_t *hi_vid = smu_data->power_tune_table.BapmVddCVidHiSidd;
+	uint8_t *lo_vid = smu_data->power_tune_table.BapmVddCVidLoSidd;
+
+	PP_ASSERT_WITH_CODE(NULL != hwmgr->dyn_state.cac_leakage_table,
+			    "The CAC Leakage table does not exist!", return -EINVAL);
+	PP_ASSERT_WITH_CODE(hwmgr->dyn_state.cac_leakage_table->count <= 8,
+			    "There should never be more than 8 entries for BapmVddcVid!!!", return -EINVAL);
+	PP_ASSERT_WITH_CODE(hwmgr->dyn_state.cac_leakage_table->count == hwmgr->dyn_state.vddc_dependency_on_sclk->count,
+			    "CACLeakageTable->count and VddcDependencyOnSCLk->count not equal", return -EINVAL);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_EVV)) {
+		for (i = 0; (uint32_t) i < hwmgr->dyn_state.cac_leakage_table->count; i++) {
+			lo_vid[i] = convert_to_vid(hwmgr->dyn_state.cac_leakage_table->entries[i].Vddc1);
+			hi_vid[i] = convert_to_vid(hwmgr->dyn_state.cac_leakage_table->entries[i].Vddc2);
+		}
+	} else {
+		PP_ASSERT_WITH_CODE(false, "Iceland should always support EVV", return -EINVAL);
+	}
+
+	return 0;
+}
+
+static int iceland_populate_vddc_vid(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	uint8_t *vid = smu_data->power_tune_table.VddCVid;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	PP_ASSERT_WITH_CODE(data->vddc_voltage_table.count <= 8,
+		"There should never be more than 8 entries for VddcVid!!!",
+		return -EINVAL);
+
+	for (i = 0; i < (int)data->vddc_voltage_table.count; i++) {
+		vid[i] = convert_to_vid(data->vddc_voltage_table.entries[i].value);
+	}
+
+	return 0;
+}
+
+
+
+static int iceland_populate_pm_fuses(struct pp_hwmgr *hwmgr)
+{
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t pm_fuse_table_offset;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerContainment)) {
+		if (smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU71_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU71_Firmware_Header, PmFuseTable),
+				&pm_fuse_table_offset, SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to get pm_fuse_table_offset Failed!",
+					return -EINVAL);
+
+		/* DW0 - DW3 */
+		if (iceland_populate_bapm_vddc_vid_sidd(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate bapm vddc vid Failed!",
+					return -EINVAL);
+
+		/* DW4 - DW5 */
+		if (iceland_populate_vddc_vid(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate vddc vid Failed!",
+					return -EINVAL);
+
+		/* DW6 */
+		if (iceland_populate_svi_load_line(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate SviLoadLine Failed!",
+					return -EINVAL);
+		/* DW7 */
+		if (iceland_populate_tdc_limit(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TDCLimit Failed!", return -EINVAL);
+		/* DW8 */
+		if (iceland_populate_dw8(hwmgr, pm_fuse_table_offset))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TdcWaterfallCtl, "
+					"LPMLTemperature Min and Max Failed!",
+					return -EINVAL);
+
+		/* DW9-DW12 */
+		if (0 != iceland_populate_temperature_scaler(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate LPMLTemperatureScaler Failed!",
+					return -EINVAL);
+
+		/* DW13-DW16 */
+		if (iceland_populate_gnb_lpml(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate GnbLPML Failed!",
+					return -EINVAL);
+
+		/* DW17 */
+		if (iceland_min_max_vgnb_lpml_id_from_bapm_vddc(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate GnbLPML Min and Max Vid Failed!",
+					return -EINVAL);
+
+		/* DW18 */
+		if (iceland_populate_bapm_vddc_base_leakage_sidd(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate BapmVddCBaseLeakage Hi and Lo Sidd Failed!",
+					return -EINVAL);
+
+		if (smu7_copy_bytes_to_smc(hwmgr->smumgr, pm_fuse_table_offset,
+				(uint8_t *)&smu_data->power_tune_table,
+				sizeof(struct SMU71_Discrete_PmFuses), SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to download PmFuseTable Failed!",
+					return -EINVAL);
+	}
+	return 0;
+}
+
+static int iceland_get_dependecy_volt_by_clk(struct pp_hwmgr *hwmgr,
+	struct phm_clock_voltage_dependency_table *allowed_clock_voltage_table,
+	uint32_t clock, uint32_t *vol)
+{
+	uint32_t i = 0;
+
+	/* clock - voltage dependency table is empty table */
+	if (allowed_clock_voltage_table->count == 0)
+		return -EINVAL;
+
+	for (i = 0; i < allowed_clock_voltage_table->count; i++) {
+		/* find first sclk bigger than request */
+		if (allowed_clock_voltage_table->entries[i].clk >= clock) {
+			*vol = allowed_clock_voltage_table->entries[i].v;
+			return 0;
+		}
+	}
+
+	/* sclk is bigger than max sclk in the dependence table */
+	*vol = allowed_clock_voltage_table->entries[i - 1].v;
+
+	return 0;
+}
+
+static int iceland_get_std_voltage_value_sidd(struct pp_hwmgr *hwmgr,
+		pp_atomctrl_voltage_table_entry *tab, uint16_t *hi,
+		uint16_t *lo)
+{
+	uint16_t v_index;
+	bool vol_found = false;
+	*hi = tab->value * VOLTAGE_SCALE;
+	*lo = tab->value * VOLTAGE_SCALE;
+
+	/* SCLK/VDDC Dependency Table has to exist. */
+	PP_ASSERT_WITH_CODE(NULL != hwmgr->dyn_state.vddc_dependency_on_sclk,
+			"The SCLK/VDDC Dependency Table does not exist.\n",
+			return -EINVAL);
+
+	if (NULL == hwmgr->dyn_state.cac_leakage_table) {
+		pr_warning("CAC Leakage Table does not exist, using vddc.\n");
+		return 0;
+	}
+
+	/*
+	 * Since voltage in the sclk/vddc dependency table is not
+	 * necessarily in ascending order because of ELB voltage
+	 * patching, loop through entire list to find exact voltage.
+	 */
+	for (v_index = 0; (uint32_t)v_index < hwmgr->dyn_state.vddc_dependency_on_sclk->count; v_index++) {
+		if (tab->value == hwmgr->dyn_state.vddc_dependency_on_sclk->entries[v_index].v) {
+			vol_found = true;
+			if ((uint32_t)v_index < hwmgr->dyn_state.cac_leakage_table->count) {
+				*lo = hwmgr->dyn_state.cac_leakage_table->entries[v_index].Vddc * VOLTAGE_SCALE;
+				*hi = (uint16_t)(hwmgr->dyn_state.cac_leakage_table->entries[v_index].Leakage * VOLTAGE_SCALE);
+			} else {
+				pr_warning("Index from SCLK/VDDC Dependency Table exceeds the CAC Leakage Table index, using maximum index from CAC table.\n");
+				*lo = hwmgr->dyn_state.cac_leakage_table->entries[hwmgr->dyn_state.cac_leakage_table->count - 1].Vddc * VOLTAGE_SCALE;
+				*hi = (uint16_t)(hwmgr->dyn_state.cac_leakage_table->entries[hwmgr->dyn_state.cac_leakage_table->count - 1].Leakage * VOLTAGE_SCALE);
+			}
+			break;
+		}
+	}
+
+	/*
+	 * If voltage is not found in the first pass, loop again to
+	 * find the best match, equal or higher value.
+	 */
+	if (!vol_found) {
+		for (v_index = 0; (uint32_t)v_index < hwmgr->dyn_state.vddc_dependency_on_sclk->count; v_index++) {
+			if (tab->value <= hwmgr->dyn_state.vddc_dependency_on_sclk->entries[v_index].v) {
+				vol_found = true;
+				if ((uint32_t)v_index < hwmgr->dyn_state.cac_leakage_table->count) {
+					*lo = hwmgr->dyn_state.cac_leakage_table->entries[v_index].Vddc * VOLTAGE_SCALE;
+					*hi = (uint16_t)(hwmgr->dyn_state.cac_leakage_table->entries[v_index].Leakage) * VOLTAGE_SCALE;
+				} else {
+					pr_warning("Index from SCLK/VDDC Dependency Table exceeds the CAC Leakage Table index in second look up, using maximum index from CAC table.");
+					*lo = hwmgr->dyn_state.cac_leakage_table->entries[hwmgr->dyn_state.cac_leakage_table->count - 1].Vddc * VOLTAGE_SCALE;
+					*hi = (uint16_t)(hwmgr->dyn_state.cac_leakage_table->entries[hwmgr->dyn_state.cac_leakage_table->count - 1].Leakage * VOLTAGE_SCALE);
+				}
+				break;
+			}
+		}
+
+		if (!vol_found)
+			pr_warning("Unable to get std_vddc from SCLK/VDDC Dependency Table, using vddc.\n");
+	}
+
+	return 0;
+}
+
+static int iceland_populate_smc_voltage_table(struct pp_hwmgr *hwmgr,
+		pp_atomctrl_voltage_table_entry *tab,
+		SMU71_Discrete_VoltageLevel *smc_voltage_tab)
+{
+	int result;
+
+	result = iceland_get_std_voltage_value_sidd(hwmgr, tab,
+			&smc_voltage_tab->StdVoltageHiSidd,
+			&smc_voltage_tab->StdVoltageLoSidd);
+	if (0 != result) {
+		smc_voltage_tab->StdVoltageHiSidd = tab->value * VOLTAGE_SCALE;
+		smc_voltage_tab->StdVoltageLoSidd = tab->value * VOLTAGE_SCALE;
+	}
+
+	smc_voltage_tab->Voltage = PP_HOST_TO_SMC_US(tab->value * VOLTAGE_SCALE);
+	CONVERT_FROM_HOST_TO_SMC_US(smc_voltage_tab->StdVoltageHiSidd);
+	CONVERT_FROM_HOST_TO_SMC_US(smc_voltage_tab->StdVoltageHiSidd);
+
+	return 0;
+}
+
+static int iceland_populate_smc_vddc_table(struct pp_hwmgr *hwmgr,
+			SMU71_Discrete_DpmTable *table)
+{
+	unsigned int count;
+	int result;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	table->VddcLevelCount = data->vddc_voltage_table.count;
+	for (count = 0; count < table->VddcLevelCount; count++) {
+		result = iceland_populate_smc_voltage_table(hwmgr,
+				&(data->vddc_voltage_table.entries[count]),
+				&(table->VddcLevel[count]));
+		PP_ASSERT_WITH_CODE(0 == result, "do not populate SMC VDDC voltage table", return -EINVAL);
+
+		/* GPIO voltage control */
+		if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->voltage_control)
+			table->VddcLevel[count].Smio |= data->vddc_voltage_table.entries[count].smio_low;
+		else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control)
+			table->VddcLevel[count].Smio = 0;
+	}
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VddcLevelCount);
+
+	return 0;
+}
+
+static int iceland_populate_smc_vdd_ci_table(struct pp_hwmgr *hwmgr,
+			SMU71_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t count;
+	int result;
+
+	table->VddciLevelCount = data->vddci_voltage_table.count;
+
+	for (count = 0; count < table->VddciLevelCount; count++) {
+		result = iceland_populate_smc_voltage_table(hwmgr,
+				&(data->vddci_voltage_table.entries[count]),
+				&(table->VddciLevel[count]));
+		PP_ASSERT_WITH_CODE(result == 0, "do not populate SMC VDDCI voltage table", return -EINVAL);
+		if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
+			table->VddciLevel[count].Smio |= data->vddci_voltage_table.entries[count].smio_low;
+		else
+			table->VddciLevel[count].Smio |= 0;
+	}
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VddciLevelCount);
+
+	return 0;
+}
+
+static int iceland_populate_smc_mvdd_table(struct pp_hwmgr *hwmgr,
+			SMU71_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t count;
+	int result;
+
+	table->MvddLevelCount = data->mvdd_voltage_table.count;
+
+	for (count = 0; count < table->VddciLevelCount; count++) {
+		result = iceland_populate_smc_voltage_table(hwmgr,
+				&(data->mvdd_voltage_table.entries[count]),
+				&table->MvddLevel[count]);
+		PP_ASSERT_WITH_CODE(result == 0, "do not populate SMC mvdd voltage table", return -EINVAL);
+		if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control)
+			table->MvddLevel[count].Smio |= data->mvdd_voltage_table.entries[count].smio_low;
+		else
+			table->MvddLevel[count].Smio |= 0;
+	}
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->MvddLevelCount);
+
+	return 0;
+}
+
+
+static int iceland_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
+	SMU71_Discrete_DpmTable *table)
+{
+	int result;
+
+	result = iceland_populate_smc_vddc_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate VDDC voltage table to SMC", return -EINVAL);
+
+	result = iceland_populate_smc_vdd_ci_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate VDDCI voltage table to SMC", return -EINVAL);
+
+	result = iceland_populate_smc_mvdd_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"can not populate MVDD voltage table to SMC", return -EINVAL);
+
+	return 0;
+}
+
+static int iceland_populate_ulv_level(struct pp_hwmgr *hwmgr,
+		struct SMU71_Discrete_Ulv *state)
+{
+	uint32_t voltage_response_time, ulv_voltage;
+	int result;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	state->CcPwrDynRm = 0;
+	state->CcPwrDynRm1 = 0;
+
+	result = pp_tables_get_response_times(hwmgr, &voltage_response_time, &ulv_voltage);
+	PP_ASSERT_WITH_CODE((0 == result), "can not get ULV voltage value", return result;);
+
+	if (ulv_voltage == 0) {
+		data->ulv_supported = false;
+		return 0;
+	}
+
+	if (data->voltage_control != SMU7_VOLTAGE_CONTROL_BY_SVID2) {
+		/* use minimum voltage if ulv voltage in pptable is bigger than minimum voltage */
+		if (ulv_voltage > hwmgr->dyn_state.vddc_dependency_on_sclk->entries[0].v)
+			state->VddcOffset = 0;
+		else
+			/* used in SMIO Mode. not implemented for now. this is backup only for CI. */
+			state->VddcOffset = (uint16_t)(hwmgr->dyn_state.vddc_dependency_on_sclk->entries[0].v - ulv_voltage);
+	} else {
+		/* use minimum voltage if ulv voltage in pptable is bigger than minimum voltage */
+		if (ulv_voltage > hwmgr->dyn_state.vddc_dependency_on_sclk->entries[0].v)
+			state->VddcOffsetVid = 0;
+		else  /* used in SVI2 Mode */
+			state->VddcOffsetVid = (uint8_t)(
+					(hwmgr->dyn_state.vddc_dependency_on_sclk->entries[0].v - ulv_voltage)
+						* VOLTAGE_VID_OFFSET_SCALE2
+						/ VOLTAGE_VID_OFFSET_SCALE1);
+	}
+	state->VddcPhase = 1;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm1);
+	CONVERT_FROM_HOST_TO_SMC_US(state->VddcOffset);
+
+	return 0;
+}
+
+static int iceland_populate_ulv_state(struct pp_hwmgr *hwmgr,
+		 SMU71_Discrete_Ulv *ulv_level)
+{
+	return iceland_populate_ulv_level(hwmgr, ulv_level);
+}
+
+static int iceland_populate_smc_link_level(struct pp_hwmgr *hwmgr, SMU71_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t i;
+
+	/* Index (dpm_table->pcie_speed_table.count) is reserved for PCIE boot level. */
+	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
+		table->LinkLevel[i].PcieGenSpeed  =
+			(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
+		table->LinkLevel[i].PcieLaneCount =
+			(uint8_t)encode_pcie_lane_width(dpm_table->pcie_speed_table.dpm_levels[i].param1);
+		table->LinkLevel[i].EnabledForActivity =
+			1;
+		table->LinkLevel[i].SPC =
+			(uint8_t)(data->pcie_spc_cap & 0xff);
+		table->LinkLevel[i].DownThreshold =
+			PP_HOST_TO_SMC_UL(5);
+		table->LinkLevel[i].UpThreshold =
+			PP_HOST_TO_SMC_UL(30);
+	}
+
+	smu_data->smc_state_table.LinkLevelCount =
+		(uint8_t)dpm_table->pcie_speed_table.count;
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
+		phm_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
+
+	return 0;
+}
+
+/**
+ * Calculates the SCLK dividers using the provided engine clock
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    engine_clock the engine clock to use to populate the structure
+ * @param    sclk        the SMC SCLK structure to be populated
+ */
+static int iceland_calculate_sclk_params(struct pp_hwmgr *hwmgr,
+		uint32_t engine_clock, SMU71_Discrete_GraphicsLevel *sclk)
+{
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	pp_atomctrl_clock_dividers_vi dividers;
+	uint32_t spll_func_cntl            = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_3          = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	uint32_t spll_func_cntl_4          = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	uint32_t cg_spll_spread_spectrum   = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	uint32_t cg_spll_spread_spectrum_2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	uint32_t    reference_clock;
+	uint32_t reference_divider;
+	uint32_t fbdiv;
+	int result;
+
+	/* get the engine clock dividers for this clock value*/
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr, engine_clock,  &dividers);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+		"Error retrieving Engine Clock dividers from VBIOS.", return result);
+
+	/* To get FBDIV we need to multiply this by 16384 and divide it by Fref.*/
+	reference_clock = atomctrl_get_reference_clock(hwmgr);
+
+	reference_divider = 1 + dividers.uc_pll_ref_div;
+
+	/* low 14 bits is fraction and high 12 bits is divider*/
+	fbdiv = dividers.ul_fb_div.ul_fb_divider & 0x3FFFFFF;
+
+	/* SPLL_FUNC_CNTL setup*/
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl,
+		CG_SPLL_FUNC_CNTL, SPLL_REF_DIV, dividers.uc_pll_ref_div);
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl,
+		CG_SPLL_FUNC_CNTL, SPLL_PDIV_A,  dividers.uc_pll_post_div);
+
+	/* SPLL_FUNC_CNTL_3 setup*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3,
+		CG_SPLL_FUNC_CNTL_3, SPLL_FB_DIV, fbdiv);
+
+	/* set to use fractional accumulation*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3,
+		CG_SPLL_FUNC_CNTL_3, SPLL_DITHEN, 1);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EngineSpreadSpectrumSupport)) {
+		pp_atomctrl_internal_ss_info ss_info;
+
+		uint32_t vcoFreq = engine_clock * dividers.uc_pll_post_div;
+		if (0 == atomctrl_get_engine_clock_spread_spectrum(hwmgr, vcoFreq, &ss_info)) {
+			/*
+			* ss_info.speed_spectrum_percentage -- in unit of 0.01%
+			* ss_info.speed_spectrum_rate -- in unit of khz
+			*/
+			/* clks = reference_clock * 10 / (REFDIV + 1) / speed_spectrum_rate / 2 */
+			uint32_t clkS = reference_clock * 5 / (reference_divider * ss_info.speed_spectrum_rate);
+
+			/* clkv = 2 * D * fbdiv / NS */
+			uint32_t clkV = 4 * ss_info.speed_spectrum_percentage * fbdiv / (clkS * 10000);
+
+			cg_spll_spread_spectrum =
+				PHM_SET_FIELD(cg_spll_spread_spectrum, CG_SPLL_SPREAD_SPECTRUM, CLKS, clkS);
+			cg_spll_spread_spectrum =
+				PHM_SET_FIELD(cg_spll_spread_spectrum, CG_SPLL_SPREAD_SPECTRUM, SSEN, 1);
+			cg_spll_spread_spectrum_2 =
+				PHM_SET_FIELD(cg_spll_spread_spectrum_2, CG_SPLL_SPREAD_SPECTRUM_2, CLKV, clkV);
+		}
+	}
+
+	sclk->SclkFrequency        = engine_clock;
+	sclk->CgSpllFuncCntl3      = spll_func_cntl_3;
+	sclk->CgSpllFuncCntl4      = spll_func_cntl_4;
+	sclk->SpllSpreadSpectrum   = cg_spll_spread_spectrum;
+	sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
+	sclk->SclkDid              = (uint8_t)dividers.pll_post_divider;
+
+	return 0;
+}
+
+static int iceland_populate_phase_value_based_on_sclk(struct pp_hwmgr *hwmgr,
+				const struct phm_phase_shedding_limits_table *pl,
+					uint32_t sclk, uint32_t *p_shed)
+{
+	unsigned int i;
+
+	/* use the minimum phase shedding */
+	*p_shed = 1;
+
+	for (i = 0; i < pl->count; i++) {
+		if (sclk < pl->entries[i].Sclk) {
+			*p_shed = i;
+			break;
+		}
+	}
+	return 0;
+}
+
+/**
+ * Populates single SMC SCLK structure using the provided engine clock
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    engine_clock the engine clock to use to populate the structure
+ * @param    sclk        the SMC SCLK structure to be populated
+ */
+static int iceland_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
+						uint32_t engine_clock,
+				uint16_t sclk_activity_level_threshold,
+				SMU71_Discrete_GraphicsLevel *graphic_level)
+{
+	int result;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	result = iceland_calculate_sclk_params(hwmgr, engine_clock, graphic_level);
+
+	/* populate graphics levels*/
+	result = iceland_get_dependecy_volt_by_clk(hwmgr,
+		hwmgr->dyn_state.vddc_dependency_on_sclk, engine_clock,
+		&graphic_level->MinVddc);
+	PP_ASSERT_WITH_CODE((0 == result),
+		"can not find VDDC voltage value for VDDC	\
+		engine clock dependency table", return result);
+
+	/* SCLK frequency in units of 10KHz*/
+	graphic_level->SclkFrequency = engine_clock;
+	graphic_level->MinVddcPhases = 1;
+
+	if (data->vddc_phase_shed_control)
+		iceland_populate_phase_value_based_on_sclk(hwmgr,
+				hwmgr->dyn_state.vddc_phase_shed_limits_table,
+				engine_clock,
+				&graphic_level->MinVddcPhases);
+
+	/* Indicates maximum activity level for this performance level. 50% for now*/
+	graphic_level->ActivityLevel = sclk_activity_level_threshold;
+
+	graphic_level->CcPwrDynRm = 0;
+	graphic_level->CcPwrDynRm1 = 0;
+	/* this level can be used if activity is high enough.*/
+	graphic_level->EnabledForActivity = 0;
+	/* this level can be used for throttling.*/
+	graphic_level->EnabledForThrottle = 1;
+	graphic_level->UpHyst = 0;
+	graphic_level->DownHyst = 100;
+	graphic_level->VoltageDownHyst = 0;
+	graphic_level->PowerThrottle = 0;
+
+	data->display_timing.min_clock_in_sr =
+			hwmgr->display_config.min_core_set_clock_in_sr;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkDeepSleep))
+		graphic_level->DeepSleepDivId =
+				smu7_get_sleep_divider_id_from_clock(engine_clock,
+						data->display_timing.min_clock_in_sr);
+
+	/* Default to slow, highest DPM level will be set to PPSMC_DISPLAY_WATERMARK_LOW later.*/
+	graphic_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	if (0 == result) {
+		graphic_level->MinVddc = PP_HOST_TO_SMC_UL(graphic_level->MinVddc * VOLTAGE_SCALE);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->MinVddcPhases);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_US(graphic_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CgSpllFuncCntl3);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CgSpllFuncCntl4);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SpllSpreadSpectrum);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SpllSpreadSpectrum2);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CcPwrDynRm);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CcPwrDynRm1);
+	}
+
+	return result;
+}
+
+/**
+ * Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states
+ *
+ * @param    hwmgr      the address of the hardware manager
+ */
+int iceland_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	uint32_t level_array_adress = smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU71_Discrete_DpmTable, GraphicsLevel);
+
+	uint32_t level_array_size = sizeof(SMU71_Discrete_GraphicsLevel) *
+						SMU71_MAX_LEVELS_GRAPHICS;
+
+	SMU71_Discrete_GraphicsLevel *levels = smu_data->smc_state_table.GraphicsLevel;
+
+	uint32_t i;
+	uint8_t highest_pcie_level_enabled = 0;
+	uint8_t lowest_pcie_level_enabled = 0, mid_pcie_level_enabled = 0;
+	uint8_t count = 0;
+	int result = 0;
+
+	memset(levels, 0x00, level_array_size);
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++) {
+		result = iceland_populate_single_graphic_level(hwmgr,
+					dpm_table->sclk_table.dpm_levels[i].value,
+					(uint16_t)smu_data->activity_target[i],
+					&(smu_data->smc_state_table.GraphicsLevel[i]));
+		if (result != 0)
+			return result;
+
+		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
+		if (i > 1)
+			smu_data->smc_state_table.GraphicsLevel[i].DeepSleepDivId = 0;
+	}
+
+	/* Only enable level 0 for now. */
+	smu_data->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1;
+
+	/* set highest level watermark to high */
+	if (dpm_table->sclk_table.count > 1)
+		smu_data->smc_state_table.GraphicsLevel[dpm_table->sclk_table.count-1].DisplayWatermark =
+			PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	smu_data->smc_state_table.GraphicsDpmLevelCount =
+		(uint8_t)dpm_table->sclk_table.count;
+	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
+		phm_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
+
+	while ((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+				(1 << (highest_pcie_level_enabled + 1))) != 0) {
+		highest_pcie_level_enabled++;
+	}
+
+	while ((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+		(1 << lowest_pcie_level_enabled)) == 0) {
+		lowest_pcie_level_enabled++;
+	}
+
+	while ((count < highest_pcie_level_enabled) &&
+			((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+				(1 << (lowest_pcie_level_enabled + 1 + count))) == 0)) {
+		count++;
+	}
+
+	mid_pcie_level_enabled = (lowest_pcie_level_enabled+1+count) < highest_pcie_level_enabled ?
+		(lowest_pcie_level_enabled+1+count) : highest_pcie_level_enabled;
+
+
+	/* set pcieDpmLevel to highest_pcie_level_enabled*/
+	for (i = 2; i < dpm_table->sclk_table.count; i++) {
+		smu_data->smc_state_table.GraphicsLevel[i].pcieDpmLevel = highest_pcie_level_enabled;
+	}
+
+	/* set pcieDpmLevel to lowest_pcie_level_enabled*/
+	smu_data->smc_state_table.GraphicsLevel[0].pcieDpmLevel = lowest_pcie_level_enabled;
+
+	/* set pcieDpmLevel to mid_pcie_level_enabled*/
+	smu_data->smc_state_table.GraphicsLevel[1].pcieDpmLevel = mid_pcie_level_enabled;
+
+	/* level count will send to smc once at init smc table and never change*/
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr, level_array_adress,
+				(uint8_t *)levels, (uint32_t)level_array_size,
+								SMC_RAM_END);
+
+	return result;
+}
+
+/**
+ * Populates the SMC MCLK structure using the provided memory clock
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    memory_clock the memory clock to use to populate the structure
+ * @param    sclk        the SMC SCLK structure to be populated
+ */
+static int iceland_calculate_mclk_params(
+		struct pp_hwmgr *hwmgr,
+		uint32_t memory_clock,
+		SMU71_Discrete_MemoryLevel *mclk,
+		bool strobe_mode,
+		bool dllStateOn
+		)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	uint32_t  dll_cntl = data->clock_registers.vDLL_CNTL;
+	uint32_t  mclk_pwrmgt_cntl = data->clock_registers.vMCLK_PWRMGT_CNTL;
+	uint32_t  mpll_ad_func_cntl = data->clock_registers.vMPLL_AD_FUNC_CNTL;
+	uint32_t  mpll_dq_func_cntl = data->clock_registers.vMPLL_DQ_FUNC_CNTL;
+	uint32_t  mpll_func_cntl = data->clock_registers.vMPLL_FUNC_CNTL;
+	uint32_t  mpll_func_cntl_1 = data->clock_registers.vMPLL_FUNC_CNTL_1;
+	uint32_t  mpll_func_cntl_2 = data->clock_registers.vMPLL_FUNC_CNTL_2;
+	uint32_t  mpll_ss1 = data->clock_registers.vMPLL_SS1;
+	uint32_t  mpll_ss2 = data->clock_registers.vMPLL_SS2;
+
+	pp_atomctrl_memory_clock_param mpll_param;
+	int result;
+
+	result = atomctrl_get_memory_pll_dividers_si(hwmgr,
+				memory_clock, &mpll_param, strobe_mode);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Error retrieving Memory Clock Parameters from VBIOS.", return result);
+
+	/* MPLL_FUNC_CNTL setup*/
+	mpll_func_cntl = PHM_SET_FIELD(mpll_func_cntl, MPLL_FUNC_CNTL, BWCTRL, mpll_param.bw_ctrl);
+
+	/* MPLL_FUNC_CNTL_1 setup*/
+	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
+							MPLL_FUNC_CNTL_1, CLKF, mpll_param.mpll_fb_divider.cl_kf);
+	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
+							MPLL_FUNC_CNTL_1, CLKFRAC, mpll_param.mpll_fb_divider.clk_frac);
+	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
+							MPLL_FUNC_CNTL_1, VCO_MODE, mpll_param.vco_mode);
+
+	/* MPLL_AD_FUNC_CNTL setup*/
+	mpll_ad_func_cntl = PHM_SET_FIELD(mpll_ad_func_cntl,
+							MPLL_AD_FUNC_CNTL, YCLK_POST_DIV, mpll_param.mpll_post_divider);
+
+	if (data->is_memory_gddr5) {
+		/* MPLL_DQ_FUNC_CNTL setup*/
+		mpll_dq_func_cntl  = PHM_SET_FIELD(mpll_dq_func_cntl,
+								MPLL_DQ_FUNC_CNTL, YCLK_SEL, mpll_param.yclk_sel);
+		mpll_dq_func_cntl  = PHM_SET_FIELD(mpll_dq_func_cntl,
+								MPLL_DQ_FUNC_CNTL, YCLK_POST_DIV, mpll_param.mpll_post_divider);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MemorySpreadSpectrumSupport)) {
+		/*
+		 ************************************
+		 Fref = Reference Frequency
+		 NF = Feedback divider ratio
+		 NR = Reference divider ratio
+		 Fnom = Nominal VCO output frequency = Fref * NF / NR
+		 Fs = Spreading Rate
+		 D = Percentage down-spread / 2
+		 Fint = Reference input frequency to PFD = Fref / NR
+		 NS = Spreading rate divider ratio = int(Fint / (2 * Fs))
+		 CLKS = NS - 1 = ISS_STEP_NUM[11:0]
+		 NV = D * Fs / Fnom * 4 * ((Fnom/Fref * NR) ^ 2)
+		 CLKV = 65536 * NV = ISS_STEP_SIZE[25:0]
+		 *************************************
+		 */
+		pp_atomctrl_internal_ss_info ss_info;
+		uint32_t freq_nom;
+		uint32_t tmp;
+		uint32_t reference_clock = atomctrl_get_mpll_reference_clock(hwmgr);
+
+		/* for GDDR5 for all modes and DDR3 */
+		if (1 == mpll_param.qdr)
+			freq_nom = memory_clock * 4 * (1 << mpll_param.mpll_post_divider);
+		else
+			freq_nom = memory_clock * 2 * (1 << mpll_param.mpll_post_divider);
+
+		/* tmp = (freq_nom / reference_clock * reference_divider) ^ 2  Note: S.I. reference_divider = 1*/
+		tmp = (freq_nom / reference_clock);
+		tmp = tmp * tmp;
+
+		if (0 == atomctrl_get_memory_clock_spread_spectrum(hwmgr, freq_nom, &ss_info)) {
+			/* ss_info.speed_spectrum_percentage -- in unit of 0.01% */
+			/* ss.Info.speed_spectrum_rate -- in unit of khz */
+			/* CLKS = reference_clock / (2 * speed_spectrum_rate * reference_divider) * 10 */
+			/*     = reference_clock * 5 / speed_spectrum_rate */
+			uint32_t clks = reference_clock * 5 / ss_info.speed_spectrum_rate;
+
+			/* CLKV = 65536 * speed_spectrum_percentage / 2 * spreadSpecrumRate / freq_nom * 4 / 100000 * ((freq_nom / reference_clock) ^ 2) */
+			/*     = 131 * speed_spectrum_percentage * speed_spectrum_rate / 100 * ((freq_nom / reference_clock) ^ 2) / freq_nom */
+			uint32_t clkv =
+				(uint32_t)((((131 * ss_info.speed_spectrum_percentage *
+							ss_info.speed_spectrum_rate) / 100) * tmp) / freq_nom);
+
+			mpll_ss1 = PHM_SET_FIELD(mpll_ss1, MPLL_SS1, CLKV, clkv);
+			mpll_ss2 = PHM_SET_FIELD(mpll_ss2, MPLL_SS2, CLKS, clks);
+		}
+	}
+
+	/* MCLK_PWRMGT_CNTL setup */
+	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, DLL_SPEED, mpll_param.dll_speed);
+	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK0_PDNB, dllStateOn);
+	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK1_PDNB, dllStateOn);
+
+
+	/* Save the result data to outpupt memory level structure */
+	mclk->MclkFrequency   = memory_clock;
+	mclk->MpllFuncCntl    = mpll_func_cntl;
+	mclk->MpllFuncCntl_1  = mpll_func_cntl_1;
+	mclk->MpllFuncCntl_2  = mpll_func_cntl_2;
+	mclk->MpllAdFuncCntl  = mpll_ad_func_cntl;
+	mclk->MpllDqFuncCntl  = mpll_dq_func_cntl;
+	mclk->MclkPwrmgtCntl  = mclk_pwrmgt_cntl;
+	mclk->DllCntl         = dll_cntl;
+	mclk->MpllSs1         = mpll_ss1;
+	mclk->MpllSs2         = mpll_ss2;
+
+	return 0;
+}
+
+static uint8_t iceland_get_mclk_frequency_ratio(uint32_t memory_clock,
+		bool strobe_mode)
+{
+	uint8_t mc_para_index;
+
+	if (strobe_mode) {
+		if (memory_clock < 12500) {
+			mc_para_index = 0x00;
+		} else if (memory_clock > 47500) {
+			mc_para_index = 0x0f;
+		} else {
+			mc_para_index = (uint8_t)((memory_clock - 10000) / 2500);
+		}
+	} else {
+		if (memory_clock < 65000) {
+			mc_para_index = 0x00;
+		} else if (memory_clock > 135000) {
+			mc_para_index = 0x0f;
+		} else {
+			mc_para_index = (uint8_t)((memory_clock - 60000) / 5000);
+		}
+	}
+
+	return mc_para_index;
+}
+
+static uint8_t iceland_get_ddr3_mclk_frequency_ratio(uint32_t memory_clock)
+{
+	uint8_t mc_para_index;
+
+	if (memory_clock < 10000) {
+		mc_para_index = 0;
+	} else if (memory_clock >= 80000) {
+		mc_para_index = 0x0f;
+	} else {
+		mc_para_index = (uint8_t)((memory_clock - 10000) / 5000 + 1);
+	}
+
+	return mc_para_index;
+}
+
+static int iceland_populate_phase_value_based_on_mclk(struct pp_hwmgr *hwmgr, const struct phm_phase_shedding_limits_table *pl,
+					uint32_t memory_clock, uint32_t *p_shed)
+{
+	unsigned int i;
+
+	*p_shed = 1;
+
+	for (i = 0; i < pl->count; i++) {
+		if (memory_clock < pl->entries[i].Mclk) {
+			*p_shed = i;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int iceland_populate_single_memory_level(
+		struct pp_hwmgr *hwmgr,
+		uint32_t memory_clock,
+		SMU71_Discrete_MemoryLevel *memory_level
+		)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	int result = 0;
+	bool dll_state_on;
+	struct cgs_display_info info = {0};
+	uint32_t mclk_edc_wr_enable_threshold = 40000;
+	uint32_t mclk_edc_enable_threshold = 40000;
+	uint32_t mclk_strobe_mode_threshold = 40000;
+
+	if (hwmgr->dyn_state.vddc_dependency_on_mclk != NULL) {
+		result = iceland_get_dependecy_volt_by_clk(hwmgr,
+			hwmgr->dyn_state.vddc_dependency_on_mclk, memory_clock, &memory_level->MinVddc);
+		PP_ASSERT_WITH_CODE((0 == result),
+			"can not find MinVddc voltage value from memory VDDC voltage dependency table", return result);
+	}
+
+	if (data->vddci_control == SMU7_VOLTAGE_CONTROL_NONE) {
+		memory_level->MinVddci = memory_level->MinVddc;
+	} else if (NULL != hwmgr->dyn_state.vddci_dependency_on_mclk) {
+		result = iceland_get_dependecy_volt_by_clk(hwmgr,
+				hwmgr->dyn_state.vddci_dependency_on_mclk,
+				memory_clock,
+				&memory_level->MinVddci);
+		PP_ASSERT_WITH_CODE((0 == result),
+			"can not find MinVddci voltage value from memory VDDCI voltage dependency table", return result);
+	}
+
+	memory_level->MinVddcPhases = 1;
+
+	if (data->vddc_phase_shed_control) {
+		iceland_populate_phase_value_based_on_mclk(hwmgr, hwmgr->dyn_state.vddc_phase_shed_limits_table,
+				memory_clock, &memory_level->MinVddcPhases);
+	}
+
+	memory_level->EnabledForThrottle = 1;
+	memory_level->EnabledForActivity = 0;
+	memory_level->UpHyst = 0;
+	memory_level->DownHyst = 100;
+	memory_level->VoltageDownHyst = 0;
+
+	/* Indicates maximum activity level for this performance level.*/
+	memory_level->ActivityLevel = (uint16_t)data->mclk_activity_target;
+	memory_level->StutterEnable = 0;
+	memory_level->StrobeEnable = 0;
+	memory_level->EdcReadEnable = 0;
+	memory_level->EdcWriteEnable = 0;
+	memory_level->RttEnable = 0;
+
+	/* default set to low watermark. Highest level will be set to high later.*/
+	memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+	data->display_timing.num_existing_displays = info.display_count;
+
+	/* stutter mode not support on iceland */
+
+	/* decide strobe mode*/
+	memory_level->StrobeEnable = (mclk_strobe_mode_threshold != 0) &&
+		(memory_clock <= mclk_strobe_mode_threshold);
+
+	/* decide EDC mode and memory clock ratio*/
+	if (data->is_memory_gddr5) {
+		memory_level->StrobeRatio = iceland_get_mclk_frequency_ratio(memory_clock,
+					memory_level->StrobeEnable);
+
+		if ((mclk_edc_enable_threshold != 0) &&
+				(memory_clock > mclk_edc_enable_threshold)) {
+			memory_level->EdcReadEnable = 1;
+		}
+
+		if ((mclk_edc_wr_enable_threshold != 0) &&
+				(memory_clock > mclk_edc_wr_enable_threshold)) {
+			memory_level->EdcWriteEnable = 1;
+		}
+
+		if (memory_level->StrobeEnable) {
+			if (iceland_get_mclk_frequency_ratio(memory_clock, 1) >=
+					((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC7) >> 16) & 0xf))
+				dll_state_on = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC5) >> 1) & 0x1) ? 1 : 0;
+			else
+				dll_state_on = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC6) >> 1) & 0x1) ? 1 : 0;
+		} else
+			dll_state_on = data->dll_default_on;
+	} else {
+		memory_level->StrobeRatio =
+			iceland_get_ddr3_mclk_frequency_ratio(memory_clock);
+		dll_state_on = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC5) >> 1) & 0x1) ? 1 : 0;
+	}
+
+	result = iceland_calculate_mclk_params(hwmgr,
+		memory_clock, memory_level, memory_level->StrobeEnable, dll_state_on);
+
+	if (0 == result) {
+		memory_level->MinVddc = PP_HOST_TO_SMC_UL(memory_level->MinVddc * VOLTAGE_SCALE);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MinVddcPhases);
+		memory_level->MinVddci = PP_HOST_TO_SMC_UL(memory_level->MinVddci * VOLTAGE_SCALE);
+		memory_level->MinMvdd = PP_HOST_TO_SMC_UL(memory_level->MinMvdd * VOLTAGE_SCALE);
+		/* MCLK frequency in units of 10KHz*/
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MclkFrequency);
+		/* Indicates maximum activity level for this performance level.*/
+		CONVERT_FROM_HOST_TO_SMC_US(memory_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl_1);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl_2);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllAdFuncCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllDqFuncCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MclkPwrmgtCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->DllCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllSs1);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllSs2);
+	}
+
+	return result;
+}
+
+/**
+ * Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states
+ *
+ * @param    hwmgr      the address of the hardware manager
+ */
+
+int iceland_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	int result;
+
+	/* populate MCLK dpm table to SMU7 */
+	uint32_t level_array_adress = smu_data->smu7_data.dpm_table_start + offsetof(SMU71_Discrete_DpmTable, MemoryLevel);
+	uint32_t level_array_size = sizeof(SMU71_Discrete_MemoryLevel) * SMU71_MAX_LEVELS_MEMORY;
+	SMU71_Discrete_MemoryLevel *levels = smu_data->smc_state_table.MemoryLevel;
+	uint32_t i;
+
+	memset(levels, 0x00, level_array_size);
+
+	for (i = 0; i < dpm_table->mclk_table.count; i++) {
+		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
+			"can not populate memory level as memory clock is zero", return -EINVAL);
+		result = iceland_populate_single_memory_level(hwmgr, dpm_table->mclk_table.dpm_levels[i].value,
+			&(smu_data->smc_state_table.MemoryLevel[i]));
+		if (0 != result) {
+			return result;
+		}
+	}
+
+	/* Only enable level 0 for now.*/
+	smu_data->smc_state_table.MemoryLevel[0].EnabledForActivity = 1;
+
+	/*
+	* in order to prevent MC activity from stutter mode to push DPM up.
+	* the UVD change complements this by putting the MCLK in a higher state
+	* by default such that we are not effected by up threshold or and MCLK DPM latency.
+	*/
+	smu_data->smc_state_table.MemoryLevel[0].ActivityLevel = 0x1F;
+	CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.MemoryLevel[0].ActivityLevel);
+
+	smu_data->smc_state_table.MemoryDpmLevelCount = (uint8_t)dpm_table->mclk_table.count;
+	data->dpm_level_enable_mask.mclk_dpm_enable_mask = phm_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
+	/* set highest level watermark to high*/
+	smu_data->smc_state_table.MemoryLevel[dpm_table->mclk_table.count-1].DisplayWatermark = PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	/* level count will send to smc once at init smc table and never change*/
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr,
+		level_array_adress, (uint8_t *)levels, (uint32_t)level_array_size,
+		SMC_RAM_END);
+
+	return result;
+}
+
+static int iceland_populate_mvdd_value(struct pp_hwmgr *hwmgr, uint32_t mclk,
+					SMU71_Discrete_VoltageLevel *voltage)
+{
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	uint32_t i = 0;
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
+		/* find mvdd value which clock is more than request */
+		for (i = 0; i < hwmgr->dyn_state.mvdd_dependency_on_mclk->count; i++) {
+			if (mclk <= hwmgr->dyn_state.mvdd_dependency_on_mclk->entries[i].clk) {
+				/* Always round to higher voltage. */
+				voltage->Voltage = data->mvdd_voltage_table.entries[i].value;
+				break;
+			}
+		}
+
+		PP_ASSERT_WITH_CODE(i < hwmgr->dyn_state.mvdd_dependency_on_mclk->count,
+			"MVDD Voltage is outside the supported range.", return -EINVAL);
+
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int iceland_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
+	SMU71_Discrete_DpmTable *table)
+{
+	int result = 0;
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	uint32_t vddc_phase_shed_control = 0;
+
+	SMU71_Discrete_VoltageLevel voltage_level;
+	uint32_t spll_func_cntl    = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_2  = data->clock_registers.vCG_SPLL_FUNC_CNTL_2;
+	uint32_t dll_cntl          = data->clock_registers.vDLL_CNTL;
+	uint32_t mclk_pwrmgt_cntl  = data->clock_registers.vMCLK_PWRMGT_CNTL;
+
+
+	/* The ACPI state should not do DPM on DC (or ever).*/
+	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+	if (data->acpi_vddc)
+		table->ACPILevel.MinVddc = PP_HOST_TO_SMC_UL(data->acpi_vddc * VOLTAGE_SCALE);
+	else
+		table->ACPILevel.MinVddc = PP_HOST_TO_SMC_UL(data->min_vddc_in_pptable * VOLTAGE_SCALE);
+
+	table->ACPILevel.MinVddcPhases = vddc_phase_shed_control ? 0 : 1;
+	/* assign zero for now*/
+	table->ACPILevel.SclkFrequency = atomctrl_get_reference_clock(hwmgr);
+
+	/* get the engine clock dividers for this clock value*/
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr,
+		table->ACPILevel.SclkFrequency,  &dividers);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+		"Error retrieving Engine Clock dividers from VBIOS.", return result);
+
+	/* divider ID for required SCLK*/
+	table->ACPILevel.SclkDid = (uint8_t)dividers.pll_post_divider;
+	table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+	table->ACPILevel.DeepSleepDivId = 0;
+
+	spll_func_cntl      = PHM_SET_FIELD(spll_func_cntl,
+							CG_SPLL_FUNC_CNTL,   SPLL_PWRON,     0);
+	spll_func_cntl      = PHM_SET_FIELD(spll_func_cntl,
+							CG_SPLL_FUNC_CNTL,   SPLL_RESET,     1);
+	spll_func_cntl_2    = PHM_SET_FIELD(spll_func_cntl_2,
+							CG_SPLL_FUNC_CNTL_2, SCLK_MUX_SEL,   4);
+
+	table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
+	table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
+	table->ACPILevel.CgSpllFuncCntl3 = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	table->ACPILevel.CgSpllFuncCntl4 = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	table->ACPILevel.SpllSpreadSpectrum = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	table->ACPILevel.SpllSpreadSpectrum2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	table->ACPILevel.CcPwrDynRm = 0;
+	table->ACPILevel.CcPwrDynRm1 = 0;
+
+
+	/* For various features to be enabled/disabled while this level is active.*/
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
+	/* SCLK frequency in units of 10KHz*/
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl3);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl4);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
+
+	/* table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases;*/
+	table->MemoryACPILevel.MinVddc = table->ACPILevel.MinVddc;
+	table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases;
+
+	if (SMU7_VOLTAGE_CONTROL_NONE == data->vddci_control)
+		table->MemoryACPILevel.MinVddci = table->MemoryACPILevel.MinVddc;
+	else {
+		if (data->acpi_vddci != 0)
+			table->MemoryACPILevel.MinVddci = PP_HOST_TO_SMC_UL(data->acpi_vddci * VOLTAGE_SCALE);
+		else
+			table->MemoryACPILevel.MinVddci = PP_HOST_TO_SMC_UL(data->min_vddci_in_pptable * VOLTAGE_SCALE);
+	}
+
+	if (0 == iceland_populate_mvdd_value(hwmgr, 0, &voltage_level))
+		table->MemoryACPILevel.MinMvdd =
+			PP_HOST_TO_SMC_UL(voltage_level.Voltage * VOLTAGE_SCALE);
+	else
+		table->MemoryACPILevel.MinMvdd = 0;
+
+	/* Force reset on DLL*/
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK0_RESET, 0x1);
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK1_RESET, 0x1);
+
+	/* Disable DLL in ACPIState*/
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK0_PDNB, 0);
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK1_PDNB, 0);
+
+	/* Enable DLL bypass signal*/
+	dll_cntl            = PHM_SET_FIELD(dll_cntl,
+		DLL_CNTL, MRDCK0_BYPASS, 0);
+	dll_cntl            = PHM_SET_FIELD(dll_cntl,
+		DLL_CNTL, MRDCK1_BYPASS, 0);
+
+	table->MemoryACPILevel.DllCntl            =
+		PP_HOST_TO_SMC_UL(dll_cntl);
+	table->MemoryACPILevel.MclkPwrmgtCntl     =
+		PP_HOST_TO_SMC_UL(mclk_pwrmgt_cntl);
+	table->MemoryACPILevel.MpllAdFuncCntl     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_AD_FUNC_CNTL);
+	table->MemoryACPILevel.MpllDqFuncCntl     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_DQ_FUNC_CNTL);
+	table->MemoryACPILevel.MpllFuncCntl       =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL);
+	table->MemoryACPILevel.MpllFuncCntl_1     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL_1);
+	table->MemoryACPILevel.MpllFuncCntl_2     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL_2);
+	table->MemoryACPILevel.MpllSs1            =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_SS1);
+	table->MemoryACPILevel.MpllSs2            =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_SS2);
+
+	table->MemoryACPILevel.EnabledForThrottle = 0;
+	table->MemoryACPILevel.EnabledForActivity = 0;
+	table->MemoryACPILevel.UpHyst = 0;
+	table->MemoryACPILevel.DownHyst = 100;
+	table->MemoryACPILevel.VoltageDownHyst = 0;
+	/* Indicates maximum activity level for this performance level.*/
+	table->MemoryACPILevel.ActivityLevel = PP_HOST_TO_SMC_US((uint16_t)data->mclk_activity_target);
+
+	table->MemoryACPILevel.StutterEnable = 0;
+	table->MemoryACPILevel.StrobeEnable = 0;
+	table->MemoryACPILevel.EdcReadEnable = 0;
+	table->MemoryACPILevel.EdcWriteEnable = 0;
+	table->MemoryACPILevel.RttEnable = 0;
+
+	return result;
+}
+
+static int iceland_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
+					SMU71_Discrete_DpmTable *table)
+{
+	return 0;
+}
+
+static int iceland_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
+		SMU71_Discrete_DpmTable *table)
+{
+	return 0;
+}
+
+static int iceland_populate_smc_acp_level(struct pp_hwmgr *hwmgr,
+		SMU71_Discrete_DpmTable *table)
+{
+	return 0;
+}
+
+static int iceland_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
+	SMU71_Discrete_DpmTable *table)
+{
+	return 0;
+}
+
+static int iceland_populate_memory_timing_parameters(
+		struct pp_hwmgr *hwmgr,
+		uint32_t engine_clock,
+		uint32_t memory_clock,
+		struct SMU71_Discrete_MCArbDramTimingTableEntry *arb_regs
+		)
+{
+	uint32_t dramTiming;
+	uint32_t dramTiming2;
+	uint32_t burstTime;
+	int result;
+
+	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
+				engine_clock, memory_clock);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+		"Error calling VBIOS to set DRAM_TIMING.", return result);
+
+	dramTiming  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+	dramTiming2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+	burstTime = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
+
+	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dramTiming);
+	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dramTiming2);
+	arb_regs->McArbBurstTime = (uint8_t)burstTime;
+
+	return 0;
+}
+
+/**
+ * Setup parameters for the MC ARB.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ * This function is to be called from the SetPowerState table.
+ */
+static int iceland_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	int result = 0;
+	SMU71_Discrete_MCArbDramTimingTable  arb_regs;
+	uint32_t i, j;
+
+	memset(&arb_regs, 0x00, sizeof(SMU71_Discrete_MCArbDramTimingTable));
+
+	for (i = 0; i < data->dpm_table.sclk_table.count; i++) {
+		for (j = 0; j < data->dpm_table.mclk_table.count; j++) {
+			result = iceland_populate_memory_timing_parameters
+				(hwmgr, data->dpm_table.sclk_table.dpm_levels[i].value,
+				 data->dpm_table.mclk_table.dpm_levels[j].value,
+				 &arb_regs.entries[i][j]);
+
+			if (0 != result) {
+				break;
+			}
+		}
+	}
+
+	if (0 == result) {
+		result = smu7_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				smu_data->smu7_data.arb_table_start,
+				(uint8_t *)&arb_regs,
+				sizeof(SMU71_Discrete_MCArbDramTimingTable),
+				SMC_RAM_END
+				);
+	}
+
+	return result;
+}
+
+static int iceland_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
+			SMU71_Discrete_DpmTable *table)
+{
+	int result = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	table->GraphicsBootLevel = 0;
+	table->MemoryBootLevel = 0;
+
+	/* find boot level from dpm table*/
+	result = phm_find_boot_level(&(data->dpm_table.sclk_table),
+			data->vbios_boot_state.sclk_bootup_value,
+			(uint32_t *)&(smu_data->smc_state_table.GraphicsBootLevel));
+
+	if (0 != result) {
+		smu_data->smc_state_table.GraphicsBootLevel = 0;
+		printk(KERN_ERR "[ powerplay ] VBIOS did not find boot engine clock value \
+			in dependency table. Using Graphics DPM level 0!");
+		result = 0;
+	}
+
+	result = phm_find_boot_level(&(data->dpm_table.mclk_table),
+		data->vbios_boot_state.mclk_bootup_value,
+		(uint32_t *)&(smu_data->smc_state_table.MemoryBootLevel));
+
+	if (0 != result) {
+		smu_data->smc_state_table.MemoryBootLevel = 0;
+		printk(KERN_ERR "[ powerplay ] VBIOS did not find boot engine clock value \
+			in dependency table. Using Memory DPM level 0!");
+		result = 0;
+	}
+
+	table->BootVddc = data->vbios_boot_state.vddc_bootup_value;
+	if (SMU7_VOLTAGE_CONTROL_NONE == data->vddci_control)
+		table->BootVddci = table->BootVddc;
+	else
+		table->BootVddci = data->vbios_boot_state.vddci_bootup_value;
+
+	table->BootMVdd = data->vbios_boot_state.mvdd_bootup_value;
+
+	return result;
+}
+
+static int iceland_populate_mc_reg_address(struct pp_smumgr *smumgr,
+				 SMU71_Discrete_MCRegisters *mc_reg_table)
+{
+	const struct iceland_smumgr *smu_data = (struct iceland_smumgr *)smumgr->backend;
+
+	uint32_t i, j;
+
+	for (i = 0, j = 0; j < smu_data->mc_reg_table.last; j++) {
+		if (smu_data->mc_reg_table.validflag & 1<<j) {
+			PP_ASSERT_WITH_CODE(i < SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE,
+				"Index of mc_reg_table->address[] array out of boundary", return -EINVAL);
+			mc_reg_table->address[i].s0 =
+				PP_HOST_TO_SMC_US(smu_data->mc_reg_table.mc_reg_address[j].s0);
+			mc_reg_table->address[i].s1 =
+				PP_HOST_TO_SMC_US(smu_data->mc_reg_table.mc_reg_address[j].s1);
+			i++;
+		}
+	}
+
+	mc_reg_table->last = (uint8_t)i;
+
+	return 0;
+}
+
+/*convert register values from driver to SMC format */
+static void iceland_convert_mc_registers(
+	const struct iceland_mc_reg_entry *entry,
+	SMU71_Discrete_MCRegisterSet *data,
+	uint32_t num_entries, uint32_t valid_flag)
+{
+	uint32_t i, j;
+
+	for (i = 0, j = 0; j < num_entries; j++) {
+		if (valid_flag & 1<<j) {
+			data->value[i] = PP_HOST_TO_SMC_UL(entry->mc_data[j]);
+			i++;
+		}
+	}
+}
+
+static int iceland_convert_mc_reg_table_entry_to_smc(
+		struct pp_smumgr *smumgr,
+		const uint32_t memory_clock,
+		SMU71_Discrete_MCRegisterSet *mc_reg_table_data
+		)
+{
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(smumgr->backend);
+	uint32_t i = 0;
+
+	for (i = 0; i < smu_data->mc_reg_table.num_entries; i++) {
+		if (memory_clock <=
+			smu_data->mc_reg_table.mc_reg_table_entry[i].mclk_max) {
+			break;
+		}
+	}
+
+	if ((i == smu_data->mc_reg_table.num_entries) && (i > 0))
+		--i;
+
+	iceland_convert_mc_registers(&smu_data->mc_reg_table.mc_reg_table_entry[i],
+				mc_reg_table_data, smu_data->mc_reg_table.last,
+				smu_data->mc_reg_table.validflag);
+
+	return 0;
+}
+
+static int iceland_convert_mc_reg_table_to_smc(struct pp_hwmgr *hwmgr,
+		SMU71_Discrete_MCRegisters *mc_regs)
+{
+	int result = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	int res;
+	uint32_t i;
+
+	for (i = 0; i < data->dpm_table.mclk_table.count; i++) {
+		res = iceland_convert_mc_reg_table_entry_to_smc(
+				hwmgr->smumgr,
+				data->dpm_table.mclk_table.dpm_levels[i].value,
+				&mc_regs->data[i]
+				);
+
+		if (0 != res)
+			result = res;
+	}
+
+	return result;
+}
+
+static int iceland_update_and_upload_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(smumgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t address;
+	int32_t result;
+
+	if (0 == (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK))
+		return 0;
+
+
+	memset(&smu_data->mc_regs, 0, sizeof(SMU71_Discrete_MCRegisters));
+
+	result = iceland_convert_mc_reg_table_to_smc(hwmgr, &(smu_data->mc_regs));
+
+	if (result != 0)
+		return result;
+
+
+	address = smu_data->smu7_data.mc_reg_table_start + (uint32_t)offsetof(SMU71_Discrete_MCRegisters, data[0]);
+
+	return  smu7_copy_bytes_to_smc(hwmgr->smumgr, address,
+				 (uint8_t *)&smu_data->mc_regs.data[0],
+				sizeof(SMU71_Discrete_MCRegisterSet) * data->dpm_table.mclk_table.count,
+				SMC_RAM_END);
+}
+
+static int iceland_populate_initial_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(smumgr->backend);
+
+	memset(&smu_data->mc_regs, 0x00, sizeof(SMU71_Discrete_MCRegisters));
+	result = iceland_populate_mc_reg_address(smumgr, &(smu_data->mc_regs));
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize MCRegTable for the MC register addresses!", return result;);
+
+	result = iceland_convert_mc_reg_table_to_smc(hwmgr, &smu_data->mc_regs);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize MCRegTable for driver state!", return result;);
+
+	return smu7_copy_bytes_to_smc(smumgr, smu_data->smu7_data.mc_reg_table_start,
+			(uint8_t *)&smu_data->mc_regs, sizeof(SMU71_Discrete_MCRegisters), SMC_RAM_END);
+}
+
+static int iceland_populate_smc_initial_state(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	uint8_t count, level;
+
+	count = (uint8_t)(hwmgr->dyn_state.vddc_dependency_on_sclk->count);
+
+	for (level = 0; level < count; level++) {
+		if (hwmgr->dyn_state.vddc_dependency_on_sclk->entries[level].clk
+			 >= data->vbios_boot_state.sclk_bootup_value) {
+			smu_data->smc_state_table.GraphicsBootLevel = level;
+			break;
+		}
+	}
+
+	count = (uint8_t)(hwmgr->dyn_state.vddc_dependency_on_mclk->count);
+
+	for (level = 0; level < count; level++) {
+		if (hwmgr->dyn_state.vddc_dependency_on_mclk->entries[level].clk
+			>= data->vbios_boot_state.mclk_bootup_value) {
+			smu_data->smc_state_table.MemoryBootLevel = level;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int iceland_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	const struct iceland_pt_defaults *defaults = smu_data->power_tune_defaults;
+	SMU71_Discrete_DpmTable  *dpm_table = &(smu_data->smc_state_table);
+	struct phm_cac_tdp_table *cac_dtp_table = hwmgr->dyn_state.cac_dtp_table;
+	struct phm_ppm_table *ppm = hwmgr->dyn_state.ppm_parameter_table;
+	const uint16_t *def1, *def2;
+	int i, j, k;
+
+
+	/*
+	 * TDP number of fraction bits are changed from 8 to 7 for Iceland
+	 * as requested by SMC team
+	 */
+
+	dpm_table->DefaultTdp = PP_HOST_TO_SMC_US((uint16_t)(cac_dtp_table->usTDP * 256));
+	dpm_table->TargetTdp = PP_HOST_TO_SMC_US((uint16_t)(cac_dtp_table->usConfigurableTDP * 256));
+
+
+	dpm_table->DTETjOffset = 0;
+
+	dpm_table->GpuTjMax = (uint8_t)(data->thermal_temp_setting.temperature_high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES);
+	dpm_table->GpuTjHyst = 8;
+
+	dpm_table->DTEAmbientTempBase = defaults->dte_ambient_temp_base;
+
+	/* The following are for new Iceland Multi-input fan/thermal control */
+	if (NULL != ppm) {
+		dpm_table->PPM_PkgPwrLimit = (uint16_t)ppm->dgpu_tdp * 256 / 1000;
+		dpm_table->PPM_TemperatureLimit = (uint16_t)ppm->tj_max * 256;
+	} else {
+		dpm_table->PPM_PkgPwrLimit = 0;
+		dpm_table->PPM_TemperatureLimit = 0;
+	}
+
+	CONVERT_FROM_HOST_TO_SMC_US(dpm_table->PPM_PkgPwrLimit);
+	CONVERT_FROM_HOST_TO_SMC_US(dpm_table->PPM_TemperatureLimit);
+
+	dpm_table->BAPM_TEMP_GRADIENT = PP_HOST_TO_SMC_UL(defaults->bamp_temp_gradient);
+	def1 = defaults->bapmti_r;
+	def2 = defaults->bapmti_rc;
+
+	for (i = 0; i < SMU71_DTE_ITERATIONS; i++) {
+		for (j = 0; j < SMU71_DTE_SOURCES; j++) {
+			for (k = 0; k < SMU71_DTE_SINKS; k++) {
+				dpm_table->BAPMTI_R[i][j][k] = PP_HOST_TO_SMC_US(*def1);
+				dpm_table->BAPMTI_RC[i][j][k] = PP_HOST_TO_SMC_US(*def2);
+				def1++;
+				def2++;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int iceland_populate_smc_svi2_config(struct pp_hwmgr *hwmgr,
+					    SMU71_Discrete_DpmTable *tab)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control)
+		tab->SVI2Enable |= VDDC_ON_SVI2;
+
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
+		tab->SVI2Enable |= VDDCI_ON_SVI2;
+	else
+		tab->MergedVddci = 1;
+
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control)
+		tab->SVI2Enable |= MVDD_ON_SVI2;
+
+	PP_ASSERT_WITH_CODE(tab->SVI2Enable != (VDDC_ON_SVI2 | VDDCI_ON_SVI2 | MVDD_ON_SVI2) &&
+		(tab->SVI2Enable & VDDC_ON_SVI2), "SVI2 domain configuration is incorrect!", return -EINVAL);
+
+	return 0;
+}
+
+/**
+ * Initializes the SMC table and uploads it
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @param    pInput  the pointer to input data (PowerState)
+ * @return   always 0
+ */
+int iceland_init_smc_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	SMU71_Discrete_DpmTable  *table = &(smu_data->smc_state_table);
+
+
+	iceland_initialize_power_tune_defaults(hwmgr);
+	memset(&(smu_data->smc_state_table), 0x00, sizeof(smu_data->smc_state_table));
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != data->voltage_control) {
+		iceland_populate_smc_voltage_tables(hwmgr, table);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StepVddc))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+	if (data->is_memory_gddr5)
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+
+	if (data->ulv_supported) {
+		result = iceland_populate_ulv_state(hwmgr, &(smu_data->ulv_setting));
+		PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize ULV state!", return result;);
+
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_ULV_PARAMETER, 0x40035);
+	}
+
+	result = iceland_populate_smc_link_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize Link Level!", return result;);
+
+	result = iceland_populate_all_graphic_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize Graphics Level!", return result;);
+
+	result = iceland_populate_all_memory_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize Memory Level!", return result;);
+
+	result = iceland_populate_smc_acpi_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize ACPI Level!", return result;);
+
+	result = iceland_populate_smc_vce_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize VCE Level!", return result;);
+
+	result = iceland_populate_smc_acp_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize ACP Level!", return result;);
+
+	result = iceland_populate_smc_samu_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize SAMU Level!", return result;);
+
+	/* Since only the initial state is completely set up at this point (the other states are just copies of the boot state) we only */
+	/* need to populate the  ARB settings for the initial state. */
+	result = iceland_program_memory_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to Write ARB settings for the initial state.", return result;);
+
+	result = iceland_populate_smc_uvd_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize UVD Level!", return result;);
+
+	table->GraphicsBootLevel = 0;
+	table->MemoryBootLevel = 0;
+
+	result = iceland_populate_smc_boot_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to initialize Boot Level!", return result;);
+
+	result = iceland_populate_smc_initial_state(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result, "Failed to initialize Boot State!", return result);
+
+	result = iceland_populate_bapm_parameters_in_dpm_table(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result, "Failed to populate BAPM Parameters!", return result);
+
+	table->GraphicsVoltageChangeEnable  = 1;
+	table->GraphicsThermThrottleEnable  = 1;
+	table->GraphicsInterval = 1;
+	table->VoltageInterval  = 1;
+	table->ThermalInterval  = 1;
+
+	table->TemperatureLimitHigh =
+		(data->thermal_temp_setting.temperature_high *
+		 SMU7_Q88_FORMAT_CONVERSION_UNIT) / PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+	table->TemperatureLimitLow =
+		(data->thermal_temp_setting.temperature_low *
+		SMU7_Q88_FORMAT_CONVERSION_UNIT) / PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+
+	table->MemoryVoltageChangeEnable  = 1;
+	table->MemoryInterval  = 1;
+	table->VoltageResponseTime  = 0;
+	table->PhaseResponseTime  = 0;
+	table->MemoryThermThrottleEnable  = 1;
+	table->PCIeBootLinkLevel = 0;
+	table->PCIeGenInterval = 1;
+
+	result = iceland_populate_smc_svi2_config(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to populate SVI2 setting!", return result);
+
+	table->ThermGpio  = 17;
+	table->SclkStepSize = 0x4000;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMaskVddcVid);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMaskVddcPhase);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMaskVddciVid);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMaskMvddVid);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
+	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
+	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
+
+	table->BootVddc = PP_HOST_TO_SMC_US(table->BootVddc * VOLTAGE_SCALE);
+	table->BootVddci = PP_HOST_TO_SMC_US(table->BootVddci * VOLTAGE_SCALE);
+	table->BootMVdd = PP_HOST_TO_SMC_US(table->BootMVdd * VOLTAGE_SCALE);
+
+	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr, smu_data->smu7_data.dpm_table_start +
+										offsetof(SMU71_Discrete_DpmTable, SystemFlags),
+										(uint8_t *)&(table->SystemFlags),
+										sizeof(SMU71_Discrete_DpmTable)-3 * sizeof(SMU71_PIDController),
+										SMC_RAM_END);
+
+	PP_ASSERT_WITH_CODE(0 == result,
+		"Failed to upload dpm data to SMC memory!", return result;);
+
+	/* Upload all ulv setting to SMC memory.(dpm level, dpm level count etc) */
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr,
+			smu_data->smu7_data.ulv_setting_starts,
+			(uint8_t *)&(smu_data->ulv_setting),
+			sizeof(SMU71_Discrete_Ulv),
+			SMC_RAM_END);
+
+
+	result = iceland_populate_initial_mc_reg_table(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == result),
+		"Failed to populate initialize MC Reg table!", return result);
+
+	result = iceland_populate_pm_fuses(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to  populate PM fuses to SMC memory!", return result);
+
+	return 0;
+}
+
+/**
+* Set up the fan table to control the fan using the SMC.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int iceland_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_smumgr *smu7_data = (struct smu7_smumgr *)(hwmgr->smumgr->backend);
+	SMU71_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
+	uint32_t duty100;
+	uint32_t t_diff1, t_diff2, pwm_diff1, pwm_diff2;
+	uint16_t fdo_min, slope1, slope2;
+	uint32_t reference_clock;
+	int res;
+	uint64_t tmp64;
+
+	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl))
+		return 0;
+
+	if (0 == smu7_data->fan_table_start) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_FDO_CTRL1, FMAX_DUTY100);
+
+	if (0 == duty100) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	tmp64 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin * duty100;
+	do_div(tmp64, 10000);
+	fdo_min = (uint16_t)tmp64;
+
+	t_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usTMed - hwmgr->thermal_controller.advanceFanControlParameters.usTMin;
+	t_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usTHigh - hwmgr->thermal_controller.advanceFanControlParameters.usTMed;
+
+	pwm_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed - hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin;
+	pwm_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh - hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed;
+
+	slope1 = (uint16_t)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
+	slope2 = (uint16_t)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
+
+	fan_table.TempMin = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMin) / 100);
+	fan_table.TempMed = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMed) / 100);
+	fan_table.TempMax = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMax) / 100);
+
+	fan_table.Slope1 = cpu_to_be16(slope1);
+	fan_table.Slope2 = cpu_to_be16(slope2);
+
+	fan_table.FdoMin = cpu_to_be16(fdo_min);
+
+	fan_table.HystDown = cpu_to_be16(hwmgr->thermal_controller.advanceFanControlParameters.ucTHyst);
+
+	fan_table.HystUp = cpu_to_be16(1);
+
+	fan_table.HystSlope = cpu_to_be16(1);
+
+	fan_table.TempRespLim = cpu_to_be16(5);
+
+	reference_clock = smu7_get_xclk(hwmgr);
+
+	fan_table.RefreshPeriod = cpu_to_be32((hwmgr->thermal_controller.advanceFanControlParameters.ulCycleDelay * reference_clock) / 1600);
+
+	fan_table.FdoMax = cpu_to_be16((uint16_t)duty100);
+
+	fan_table.TempSrc = (uint8_t)PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_MULT_THERMAL_CTRL, TEMP_SEL);
+
+	/* fan_table.FanControl_GL_Flag = 1; */
+
+	res = smu7_copy_bytes_to_smc(hwmgr->smumgr, smu7_data->fan_table_start, (uint8_t *)&fan_table, (uint32_t)sizeof(fan_table), SMC_RAM_END);
+
+	return 0;
+}
+
+
+static int iceland_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_OD_UPDATE_MCLK))
+		return iceland_program_memory_timing_parameters(hwmgr);
+
+	return 0;
+}
+
+int iceland_update_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+
+	int result = 0;
+	uint32_t low_sclk_interrupt_threshold = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkThrottleLowNotification)
+		&& (hwmgr->gfx_arbiter.sclk_threshold !=
+				data->low_sclk_interrupt_threshold)) {
+		data->low_sclk_interrupt_threshold =
+				hwmgr->gfx_arbiter.sclk_threshold;
+		low_sclk_interrupt_threshold =
+				data->low_sclk_interrupt_threshold;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
+
+		result = smu7_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU71_Discrete_DpmTable,
+					LowSclkInterruptThreshold),
+				(uint8_t *)&low_sclk_interrupt_threshold,
+				sizeof(uint32_t),
+				SMC_RAM_END);
+	}
+
+	result = iceland_update_and_upload_mc_reg_table(hwmgr);
+
+	PP_ASSERT_WITH_CODE((0 == result), "Failed to upload MC reg table!", return result);
+
+	result = iceland_program_mem_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE((result == 0),
+			"Failed to program memory timing parameters!",
+			);
+
+	return result;
+}
+
+uint32_t iceland_get_offsetof(uint32_t type, uint32_t member)
+{
+	switch (type) {
+	case SMU_SoftRegisters:
+		switch (member) {
+		case HandshakeDisables:
+			return offsetof(SMU71_SoftRegisters, HandshakeDisables);
+		case VoltageChangeTimeout:
+			return offsetof(SMU71_SoftRegisters, VoltageChangeTimeout);
+		case AverageGraphicsActivity:
+			return offsetof(SMU71_SoftRegisters, AverageGraphicsActivity);
+		case PreVBlankGap:
+			return offsetof(SMU71_SoftRegisters, PreVBlankGap);
+		case VBlankTimeout:
+			return offsetof(SMU71_SoftRegisters, VBlankTimeout);
+		case UcodeLoadStatus:
+			return offsetof(SMU71_SoftRegisters, UcodeLoadStatus);
+		}
+	case SMU_Discrete_DpmTable:
+		switch (member) {
+		case LowSclkInterruptThreshold:
+			return offsetof(SMU71_Discrete_DpmTable, LowSclkInterruptThreshold);
+		}
+	}
+	printk("cant't get the offset of type %x member %x \n", type, member);
+	return 0;
+}
+
+uint32_t iceland_get_mac_definition(uint32_t value)
+{
+	switch (value) {
+	case SMU_MAX_LEVELS_GRAPHICS:
+		return SMU71_MAX_LEVELS_GRAPHICS;
+	case SMU_MAX_LEVELS_MEMORY:
+		return SMU71_MAX_LEVELS_MEMORY;
+	case SMU_MAX_LEVELS_LINK:
+		return SMU71_MAX_LEVELS_LINK;
+	case SMU_MAX_ENTRIES_SMIO:
+		return SMU71_MAX_ENTRIES_SMIO;
+	case SMU_MAX_LEVELS_VDDC:
+		return SMU71_MAX_LEVELS_VDDC;
+	case SMU_MAX_LEVELS_VDDCI:
+		return SMU71_MAX_LEVELS_VDDCI;
+	case SMU_MAX_LEVELS_MVDD:
+		return SMU71_MAX_LEVELS_MVDD;
+	}
+
+	printk("cant't get the mac of %x \n", value);
+	return 0;
+}
+
+/**
+ * Get the location of various tables inside the FW image.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int iceland_process_firmware_header(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_smumgr *smu7_data = (struct smu7_smumgr *)(hwmgr->smumgr->backend);
+
+	uint32_t tmp;
+	int result;
+	bool error = false;
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU71_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU71_Firmware_Header, DpmTable),
+				&tmp, SMC_RAM_END);
+
+	if (0 == result) {
+		smu7_data->dpm_table_start = tmp;
+	}
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU71_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU71_Firmware_Header, SoftRegisters),
+				&tmp, SMC_RAM_END);
+
+	if (0 == result) {
+		data->soft_regs_start = tmp;
+		smu7_data->soft_regs_start = tmp;
+	}
+
+	error |= (0 != result);
+
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU71_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU71_Firmware_Header, mcRegisterTable),
+				&tmp, SMC_RAM_END);
+
+	if (0 == result) {
+		smu7_data->mc_reg_table_start = tmp;
+	}
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU71_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU71_Firmware_Header, FanTable),
+				&tmp, SMC_RAM_END);
+
+	if (0 == result) {
+		smu7_data->fan_table_start = tmp;
+	}
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU71_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU71_Firmware_Header, mcArbDramTimingTable),
+				&tmp, SMC_RAM_END);
+
+	if (0 == result) {
+		smu7_data->arb_table_start = tmp;
+	}
+
+	error |= (0 != result);
+
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU71_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU71_Firmware_Header, Version),
+				&tmp, SMC_RAM_END);
+
+	if (0 == result) {
+		hwmgr->microcode_version_info.SMC = tmp;
+	}
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU71_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU71_Firmware_Header, UlvSettings),
+				&tmp, SMC_RAM_END);
+
+	if (0 == result) {
+		smu7_data->ulv_setting_starts = tmp;
+	}
+
+	error |= (0 != result);
+
+	return error ? 1 : 0;
+}
+
+/*---------------------------MC----------------------------*/
+
+static uint8_t iceland_get_memory_modile_index(struct pp_hwmgr *hwmgr)
+{
+	return (uint8_t) (0xFF & (cgs_read_register(hwmgr->device, mmBIOS_SCRATCH_4) >> 16));
+}
+
+static bool iceland_check_s0_mc_reg_index(uint16_t in_reg, uint16_t *out_reg)
+{
+	bool result = true;
+
+	switch (in_reg) {
+	case  mmMC_SEQ_RAS_TIMING:
+		*out_reg = mmMC_SEQ_RAS_TIMING_LP;
+		break;
+
+	case  mmMC_SEQ_DLL_STBY:
+		*out_reg = mmMC_SEQ_DLL_STBY_LP;
+		break;
+
+	case  mmMC_SEQ_G5PDX_CMD0:
+		*out_reg = mmMC_SEQ_G5PDX_CMD0_LP;
+		break;
+
+	case  mmMC_SEQ_G5PDX_CMD1:
+		*out_reg = mmMC_SEQ_G5PDX_CMD1_LP;
+		break;
+
+	case  mmMC_SEQ_G5PDX_CTRL:
+		*out_reg = mmMC_SEQ_G5PDX_CTRL_LP;
+		break;
+
+	case mmMC_SEQ_CAS_TIMING:
+		*out_reg = mmMC_SEQ_CAS_TIMING_LP;
+		break;
+
+	case mmMC_SEQ_MISC_TIMING:
+		*out_reg = mmMC_SEQ_MISC_TIMING_LP;
+		break;
+
+	case mmMC_SEQ_MISC_TIMING2:
+		*out_reg = mmMC_SEQ_MISC_TIMING2_LP;
+		break;
+
+	case mmMC_SEQ_PMG_DVS_CMD:
+		*out_reg = mmMC_SEQ_PMG_DVS_CMD_LP;
+		break;
+
+	case mmMC_SEQ_PMG_DVS_CTL:
+		*out_reg = mmMC_SEQ_PMG_DVS_CTL_LP;
+		break;
+
+	case mmMC_SEQ_RD_CTL_D0:
+		*out_reg = mmMC_SEQ_RD_CTL_D0_LP;
+		break;
+
+	case mmMC_SEQ_RD_CTL_D1:
+		*out_reg = mmMC_SEQ_RD_CTL_D1_LP;
+		break;
+
+	case mmMC_SEQ_WR_CTL_D0:
+		*out_reg = mmMC_SEQ_WR_CTL_D0_LP;
+		break;
+
+	case mmMC_SEQ_WR_CTL_D1:
+		*out_reg = mmMC_SEQ_WR_CTL_D1_LP;
+		break;
+
+	case mmMC_PMG_CMD_EMRS:
+		*out_reg = mmMC_SEQ_PMG_CMD_EMRS_LP;
+		break;
+
+	case mmMC_PMG_CMD_MRS:
+		*out_reg = mmMC_SEQ_PMG_CMD_MRS_LP;
+		break;
+
+	case mmMC_PMG_CMD_MRS1:
+		*out_reg = mmMC_SEQ_PMG_CMD_MRS1_LP;
+		break;
+
+	case mmMC_SEQ_PMG_TIMING:
+		*out_reg = mmMC_SEQ_PMG_TIMING_LP;
+		break;
+
+	case mmMC_PMG_CMD_MRS2:
+		*out_reg = mmMC_SEQ_PMG_CMD_MRS2_LP;
+		break;
+
+	case mmMC_SEQ_WR_CTL_2:
+		*out_reg = mmMC_SEQ_WR_CTL_2_LP;
+		break;
+
+	default:
+		result = false;
+		break;
+	}
+
+	return result;
+}
+
+static int iceland_set_s0_mc_reg_index(struct iceland_mc_reg_table *table)
+{
+	uint32_t i;
+	uint16_t address;
+
+	for (i = 0; i < table->last; i++) {
+		table->mc_reg_address[i].s0 =
+			iceland_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address)
+			? address : table->mc_reg_address[i].s1;
+	}
+	return 0;
+}
+
+static int iceland_copy_vbios_smc_reg_table(const pp_atomctrl_mc_reg_table *table,
+					struct iceland_mc_reg_table *ni_table)
+{
+	uint8_t i, j;
+
+	PP_ASSERT_WITH_CODE((table->last <= SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+		"Invalid VramInfo table.", return -EINVAL);
+	PP_ASSERT_WITH_CODE((table->num_entries <= MAX_AC_TIMING_ENTRIES),
+		"Invalid VramInfo table.", return -EINVAL);
+
+	for (i = 0; i < table->last; i++) {
+		ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
+	}
+	ni_table->last = table->last;
+
+	for (i = 0; i < table->num_entries; i++) {
+		ni_table->mc_reg_table_entry[i].mclk_max =
+			table->mc_reg_table_entry[i].mclk_max;
+		for (j = 0; j < table->last; j++) {
+			ni_table->mc_reg_table_entry[i].mc_data[j] =
+				table->mc_reg_table_entry[i].mc_data[j];
+		}
+	}
+
+	ni_table->num_entries = table->num_entries;
+
+	return 0;
+}
+
+/**
+ * VBIOS omits some information to reduce size, we need to recover them here.
+ * 1.   when we see mmMC_SEQ_MISC1, bit[31:16] EMRS1, need to be write to  mmMC_PMG_CMD_EMRS /_LP[15:0].
+ *      Bit[15:0] MRS, need to be update mmMC_PMG_CMD_MRS/_LP[15:0]
+ * 2.   when we see mmMC_SEQ_RESERVE_M, bit[15:0] EMRS2, need to be write to mmMC_PMG_CMD_MRS1/_LP[15:0].
+ * 3.   need to set these data for each clock range
+ *
+ * @param    hwmgr the address of the powerplay hardware manager.
+ * @param    table the address of MCRegTable
+ * @return   always 0
+ */
+static int iceland_set_mc_special_registers(struct pp_hwmgr *hwmgr,
+					struct iceland_mc_reg_table *table)
+{
+	uint8_t i, j, k;
+	uint32_t temp_reg;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	for (i = 0, j = table->last; i < table->last; i++) {
+		PP_ASSERT_WITH_CODE((j < SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+			"Invalid VramInfo table.", return -EINVAL);
+
+		switch (table->mc_reg_address[i].s1) {
+
+		case mmMC_SEQ_MISC1:
+			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_EMRS);
+			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_EMRS;
+			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_EMRS_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					((temp_reg & 0xffff0000)) |
+					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
+			}
+			j++;
+			PP_ASSERT_WITH_CODE((j < SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+				"Invalid VramInfo table.", return -EINVAL);
+
+			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS);
+			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
+			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					(temp_reg & 0xffff0000) |
+					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+
+				if (!data->is_memory_gddr5) {
+					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
+				}
+			}
+			j++;
+			PP_ASSERT_WITH_CODE((j <= SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+				"Invalid VramInfo table.", return -EINVAL);
+
+			if (!data->is_memory_gddr5) {
+				table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
+				table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
+				for (k = 0; k < table->num_entries; k++) {
+					table->mc_reg_table_entry[k].mc_data[j] =
+						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
+				}
+				j++;
+				PP_ASSERT_WITH_CODE((j <= SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+					"Invalid VramInfo table.", return -EINVAL);
+			}
+
+			break;
+
+		case mmMC_SEQ_RESERVE_M:
+			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS1);
+			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS1;
+			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS1_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					(temp_reg & 0xffff0000) |
+					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+			}
+			j++;
+			PP_ASSERT_WITH_CODE((j <= SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+				"Invalid VramInfo table.", return -EINVAL);
+			break;
+
+		default:
+			break;
+		}
+
+	}
+
+	table->last = j;
+
+	return 0;
+}
+
+static int iceland_set_valid_flag(struct iceland_mc_reg_table *table)
+{
+	uint8_t i, j;
+	for (i = 0; i < table->last; i++) {
+		for (j = 1; j < table->num_entries; j++) {
+			if (table->mc_reg_table_entry[j-1].mc_data[i] !=
+				table->mc_reg_table_entry[j].mc_data[i]) {
+				table->validflag |= (1<<i);
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
+
+int iceland_initialize_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(hwmgr->smumgr->backend);
+	pp_atomctrl_mc_reg_table *table;
+	struct iceland_mc_reg_table *ni_table = &smu_data->mc_reg_table;
+	uint8_t module_index = iceland_get_memory_modile_index(hwmgr);
+
+	table = kzalloc(sizeof(pp_atomctrl_mc_reg_table), GFP_KERNEL);
+
+	if (NULL == table)
+		return -ENOMEM;
+
+	/* Program additional LP registers that are no longer programmed by VBIOS */
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RAS_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_RAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_CAS_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_CAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_DLL_STBY_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_DLL_STBY));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD0_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD1_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CTRL_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CTRL));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CMD_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CMD));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CTL_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CTL));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_EMRS_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_EMRS));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS1_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D0_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_TIMING_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS2_LP, cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS2));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_2_LP, cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_2));
+
+	memset(table, 0x00, sizeof(pp_atomctrl_mc_reg_table));
+
+	result = atomctrl_initialize_mc_reg_table(hwmgr, module_index, table);
+
+	if (0 == result)
+		result = iceland_copy_vbios_smc_reg_table(table, ni_table);
+
+	if (0 == result) {
+		iceland_set_s0_mc_reg_index(ni_table);
+		result = iceland_set_mc_special_registers(hwmgr, ni_table);
+	}
+
+	if (0 == result)
+		iceland_set_valid_flag(ni_table);
+
+	kfree(table);
+
+	return result;
+}
+
+bool iceland_is_dpm_running(struct pp_hwmgr *hwmgr)
+{
+	return (1 == PHM_READ_INDIRECT_FIELD(hwmgr->device,
+			CGS_IND_REG__SMC, FEATURE_STATUS, VOLTAGE_CONTROLLER_ON))
+			? true : false;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.h b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.h
index 8bc38cb17b7f..13c8dbbccaf2 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_clockpowergating.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.h
@@ -20,17 +20,21 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#ifndef _ICELAND_SMC_H
+#define _ICELAND_SMC_H
 
-#ifndef _TONGA_CLOCK_POWER_GATING_H_
-#define _TONGA_CLOCK_POWER_GATING_H_
+#include "smumgr.h"
 
-#include "tonga_hwmgr.h"
-#include "pp_asicblocks.h"
 
-extern int tonga_phm_set_asic_block_gating(struct pp_hwmgr *hwmgr, enum PHM_AsicBlock block, enum PHM_ClockGateSetting gating);
-extern int tonga_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate);
-extern int tonga_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate);
-extern int tonga_phm_powerdown_uvd(struct pp_hwmgr *hwmgr);
-extern int tonga_phm_disable_clock_power_gating(struct pp_hwmgr *hwmgr);
-extern int tonga_phm_update_clock_gatings(struct pp_hwmgr *hwmgr, const uint32_t *msg_id);
-#endif /* _TONGA_CLOCK_POWER_GATING_H_ */
+int iceland_populate_all_graphic_levels(struct pp_hwmgr *hwmgr);
+int iceland_populate_all_memory_levels(struct pp_hwmgr *hwmgr);
+int iceland_init_smc_table(struct pp_hwmgr *hwmgr);
+int iceland_thermal_setup_fan_table(struct pp_hwmgr *hwmgr);
+int iceland_update_sclk_threshold(struct pp_hwmgr *hwmgr);
+uint32_t iceland_get_offsetof(uint32_t type, uint32_t member);
+uint32_t iceland_get_mac_definition(uint32_t value);
+int iceland_process_firmware_header(struct pp_hwmgr *hwmgr);
+int iceland_initialize_mc_reg_table(struct pp_hwmgr *hwmgr);
+bool iceland_is_dpm_running(struct pp_hwmgr *hwmgr);
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
new file mode 100644
index 000000000000..eeafefc4acba
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
@@ -0,0 +1,250 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Huang Rui <ray.huang@amd.com>
+ *
+ */
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/gfp.h>
+
+#include "smumgr.h"
+#include "iceland_smumgr.h"
+#include "pp_debug.h"
+#include "smu_ucode_xfer_vi.h"
+#include "ppsmc.h"
+#include "smu/smu_7_1_1_d.h"
+#include "smu/smu_7_1_1_sh_mask.h"
+#include "cgs_common.h"
+#include "iceland_smc.h"
+
+#define ICELAND_SMC_SIZE               0x20000
+
+static int iceland_start_smc(struct pp_smumgr *smumgr)
+{
+	SMUM_WRITE_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+				  SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+
+	return 0;
+}
+
+static void iceland_reset_smc(struct pp_smumgr *smumgr)
+{
+	SMUM_WRITE_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+				  SMC_SYSCON_RESET_CNTL,
+				  rst_reg, 1);
+}
+
+
+static void iceland_stop_smc_clock(struct pp_smumgr *smumgr)
+{
+	SMUM_WRITE_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+				  SMC_SYSCON_CLOCK_CNTL_0,
+				  ck_disable, 1);
+}
+
+static void iceland_start_smc_clock(struct pp_smumgr *smumgr)
+{
+	SMUM_WRITE_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
+				  SMC_SYSCON_CLOCK_CNTL_0,
+				  ck_disable, 0);
+}
+
+static int iceland_smu_start_smc(struct pp_smumgr *smumgr)
+{
+	/* set smc instruct start point at 0x0 */
+	smu7_program_jump_on_start(smumgr);
+
+	/* enable smc clock */
+	iceland_start_smc_clock(smumgr);
+
+	/* de-assert reset */
+	iceland_start_smc(smumgr);
+
+	SMUM_WAIT_INDIRECT_FIELD(smumgr, SMC_IND, FIRMWARE_FLAGS,
+				 INTERRUPTS_ENABLED, 1);
+
+	return 0;
+}
+
+
+static int iceland_upload_smc_firmware_data(struct pp_smumgr *smumgr,
+					uint32_t length, const uint8_t *src,
+					uint32_t limit, uint32_t start_addr)
+{
+	uint32_t byte_count = length;
+	uint32_t data;
+
+	PP_ASSERT_WITH_CODE((limit >= byte_count), "SMC address is beyond the SMC RAM area.", return -EINVAL);
+
+	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_0, start_addr);
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1);
+
+	while (byte_count >= 4) {
+		data = src[0] * 0x1000000 + src[1] * 0x10000 + src[2] * 0x100 + src[3];
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data);
+		src += 4;
+		byte_count -= 4;
+	}
+
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
+
+	PP_ASSERT_WITH_CODE((0 == byte_count), "SMC size must be dividable by 4.", return -EINVAL);
+
+	return 0;
+}
+
+
+static int iceland_smu_upload_firmware_image(struct pp_smumgr *smumgr)
+{
+	uint32_t val;
+	struct cgs_firmware_info info = {0};
+
+	if (smumgr == NULL || smumgr->device == NULL)
+		return -EINVAL;
+
+	/* load SMC firmware */
+	cgs_get_firmware_info(smumgr->device,
+		smu7_convert_fw_type_to_cgs(UCODE_ID_SMU), &info);
+
+	if (info.image_size & 3) {
+		pr_err("[ powerplay ] SMC ucode is not 4 bytes aligned\n");
+		return -EINVAL;
+	}
+
+	if (info.image_size > ICELAND_SMC_SIZE) {
+		pr_err("[ powerplay ] SMC address is beyond the SMC RAM area\n");
+		return -EINVAL;
+	}
+
+	/* wait for smc boot up */
+	SMUM_WAIT_INDIRECT_FIELD_UNEQUAL(smumgr, SMC_IND,
+					 RCU_UC_EVENTS, boot_seq_done, 0);
+
+	/* clear firmware interrupt enable flag */
+	val = cgs_read_ind_register(smumgr->device, CGS_IND_REG__SMC,
+				    ixSMC_SYSCON_MISC_CNTL);
+	cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
+			       ixSMC_SYSCON_MISC_CNTL, val | 1);
+
+	/* stop smc clock */
+	iceland_stop_smc_clock(smumgr);
+
+	/* reset smc */
+	iceland_reset_smc(smumgr);
+	iceland_upload_smc_firmware_data(smumgr, info.image_size,
+				(uint8_t *)info.kptr, ICELAND_SMC_SIZE,
+				info.ucode_start_address);
+
+	return 0;
+}
+
+static int iceland_request_smu_load_specific_fw(struct pp_smumgr *smumgr,
+						uint32_t firmwareType)
+{
+	return 0;
+}
+
+static int iceland_start_smu(struct pp_smumgr *smumgr)
+{
+	int result;
+
+	result = iceland_smu_upload_firmware_image(smumgr);
+	if (result)
+		return result;
+	result = iceland_smu_start_smc(smumgr);
+	if (result)
+		return result;
+
+	if (!smu7_is_smc_ram_running(smumgr)) {
+		printk("smu not running, upload firmware again \n");
+		result = iceland_smu_upload_firmware_image(smumgr);
+		if (result)
+			return result;
+
+		result = iceland_smu_start_smc(smumgr);
+		if (result)
+			return result;
+	}
+
+	result = smu7_request_smu_load_fw(smumgr);
+
+	return result;
+}
+
+/**
+ * Write a 32bit value to the SMC SRAM space.
+ * ALL PARAMETERS ARE IN HOST BYTE ORDER.
+ * @param    smumgr  the address of the powerplay hardware manager.
+ * @param    smcAddress the address in the SMC RAM to access.
+ * @param    value to write to the SMC SRAM.
+ */
+static int iceland_smu_init(struct pp_smumgr *smumgr)
+{
+	int i;
+	struct iceland_smumgr *smu_data = (struct iceland_smumgr *)(smumgr->backend);
+	if (smu7_init(smumgr))
+		return -EINVAL;
+
+	for (i = 0; i < SMU71_MAX_LEVELS_GRAPHICS; i++)
+		smu_data->activity_target[i] = 30;
+
+	return 0;
+}
+
+static const struct pp_smumgr_func iceland_smu_funcs = {
+	.smu_init = &iceland_smu_init,
+	.smu_fini = &smu7_smu_fini,
+	.start_smu = &iceland_start_smu,
+	.check_fw_load_finish = &smu7_check_fw_load_finish,
+	.request_smu_load_fw = &smu7_reload_firmware,
+	.request_smu_load_specific_fw = &iceland_request_smu_load_specific_fw,
+	.send_msg_to_smc = &smu7_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = &smu7_send_msg_to_smc_with_parameter,
+	.download_pptable_settings = NULL,
+	.upload_pptable_settings = NULL,
+	.get_offsetof = iceland_get_offsetof,
+	.process_firmware_header = iceland_process_firmware_header,
+	.init_smc_table = iceland_init_smc_table,
+	.update_sclk_threshold = iceland_update_sclk_threshold,
+	.thermal_setup_fan_table = iceland_thermal_setup_fan_table,
+	.populate_all_graphic_levels = iceland_populate_all_graphic_levels,
+	.populate_all_memory_levels = iceland_populate_all_memory_levels,
+	.get_mac_definition = iceland_get_mac_definition,
+	.initialize_mc_reg_table = iceland_initialize_mc_reg_table,
+	.is_dpm_running = iceland_is_dpm_running,
+};
+
+int iceland_smum_init(struct pp_smumgr *smumgr)
+{
+	struct iceland_smumgr *iceland_smu = NULL;
+
+	iceland_smu = kzalloc(sizeof(struct iceland_smumgr), GFP_KERNEL);
+
+	if (iceland_smu == NULL)
+		return -ENOMEM;
+
+	smumgr->backend = iceland_smu;
+	smumgr->smumgr_funcs = &iceland_smu_funcs;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.h
new file mode 100644
index 000000000000..8eae01b37c40
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Huang Rui <ray.huang@amd.com>
+ *
+ */
+
+#ifndef _ICELAND_SMUMGR_H_
+#define _ICELAND_SMUMGR_H_
+
+
+#include "smu7_smumgr.h"
+#include "pp_endian.h"
+#include "smu71_discrete.h"
+
+struct iceland_pt_defaults {
+	uint8_t   svi_load_line_en;
+	uint8_t   svi_load_line_vddc;
+	uint8_t   tdc_vddc_throttle_release_limit_perc;
+	uint8_t   tdc_mawt;
+	uint8_t   tdc_waterfall_ctl;
+	uint8_t   dte_ambient_temp_base;
+	uint32_t  display_cac;
+	uint32_t  bamp_temp_gradient;
+	uint16_t  bapmti_r[SMU71_DTE_ITERATIONS * SMU71_DTE_SOURCES * SMU71_DTE_SINKS];
+	uint16_t  bapmti_rc[SMU71_DTE_ITERATIONS * SMU71_DTE_SOURCES * SMU71_DTE_SINKS];
+};
+
+struct iceland_mc_reg_entry {
+	uint32_t mclk_max;
+	uint32_t mc_data[SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct iceland_mc_reg_table {
+	uint8_t   last;               /* number of registers*/
+	uint8_t   num_entries;        /* number of entries in mc_reg_table_entry used*/
+	uint16_t  validflag;          /* indicate the corresponding register is valid or not. 1: valid, 0: invalid. bit0->address[0], bit1->address[1], etc.*/
+	struct iceland_mc_reg_entry    mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
+	SMU71_Discrete_MCRegisterAddress mc_reg_address[SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct iceland_smumgr {
+	struct smu7_smumgr smu7_data;
+	struct SMU71_Discrete_DpmTable       smc_state_table;
+	struct SMU71_Discrete_PmFuses  power_tune_table;
+	struct SMU71_Discrete_Ulv            ulv_setting;
+	const struct iceland_pt_defaults  *power_tune_defaults;
+	SMU71_Discrete_MCRegisters      mc_regs;
+	struct iceland_mc_reg_table mc_reg_table;
+	uint32_t        activity_target[SMU71_MAX_LEVELS_GRAPHICS];
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
new file mode 100644
index 000000000000..4ccc0b72324d
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
@@ -0,0 +1,2287 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "polaris10_smc.h"
+#include "smu7_dyn_defaults.h"
+
+#include "smu7_hwmgr.h"
+#include "hardwaremanager.h"
+#include "ppatomctrl.h"
+#include "pp_debug.h"
+#include "cgs_common.h"
+#include "atombios.h"
+#include "polaris10_smumgr.h"
+#include "pppcielanes.h"
+
+#include "smu_ucode_xfer_vi.h"
+#include "smu74_discrete.h"
+#include "smu/smu_7_1_3_d.h"
+#include "smu/smu_7_1_3_sh_mask.h"
+#include "gmc/gmc_8_1_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "gca/gfx_8_0_d.h"
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+#include "polaris10_pwrvirus.h"
+#include "smu7_ppsmc.h"
+#include "smu7_smumgr.h"
+
+#define POLARIS10_SMC_SIZE 0x20000
+#define VOLTAGE_VID_OFFSET_SCALE1   625
+#define VOLTAGE_VID_OFFSET_SCALE2   100
+#define POWERTUNE_DEFAULT_SET_MAX    1
+#define VDDC_VDDCI_DELTA            200
+#define MC_CG_ARB_FREQ_F1           0x0b
+
+static const struct polaris10_pt_defaults polaris10_power_tune_data_set_array[POWERTUNE_DEFAULT_SET_MAX] = {
+	/* sviLoadLIneEn, SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc, TDC_MAWt,
+	 * TdcWaterfallCtl, DTEAmbientTempBase, DisplayCac, BAPM_TEMP_GRADIENT */
+	{ 1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000,
+	{ 0x79, 0x253, 0x25D, 0xAE, 0x72, 0x80, 0x83, 0x86, 0x6F, 0xC8, 0xC9, 0xC9, 0x2F, 0x4D, 0x61},
+	{ 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 } },
+};
+
+static const sclkFcwRange_t Range_Table[NUM_SCLK_RANGE] = {
+			{VCO_2_4, POSTDIV_DIV_BY_16,  75, 160, 112},
+			{VCO_3_6, POSTDIV_DIV_BY_16, 112, 224, 160},
+			{VCO_2_4, POSTDIV_DIV_BY_8,   75, 160, 112},
+			{VCO_3_6, POSTDIV_DIV_BY_8,  112, 224, 160},
+			{VCO_2_4, POSTDIV_DIV_BY_4,   75, 160, 112},
+			{VCO_3_6, POSTDIV_DIV_BY_4,  112, 216, 160},
+			{VCO_2_4, POSTDIV_DIV_BY_2,   75, 160, 108},
+			{VCO_3_6, POSTDIV_DIV_BY_2,  112, 216, 160} };
+
+static int polaris10_get_dependency_volt_by_clk(struct pp_hwmgr *hwmgr,
+		struct phm_ppt_v1_clock_voltage_dependency_table *dep_table,
+		uint32_t clock, SMU_VoltageLevel *voltage, uint32_t *mvdd)
+{
+	uint32_t i;
+	uint16_t vddci;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	*voltage = *mvdd = 0;
+
+	/* clock - voltage dependency table is empty table */
+	if (dep_table->count == 0)
+		return -EINVAL;
+
+	for (i = 0; i < dep_table->count; i++) {
+		/* find first sclk bigger than request */
+		if (dep_table->entries[i].clk >= clock) {
+			*voltage |= (dep_table->entries[i].vddc *
+					VOLTAGE_SCALE) << VDDC_SHIFT;
+			if (SMU7_VOLTAGE_CONTROL_NONE == data->vddci_control)
+				*voltage |= (data->vbios_boot_state.vddci_bootup_value *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+			else if (dep_table->entries[i].vddci)
+				*voltage |= (dep_table->entries[i].vddci *
+						VOLTAGE_SCALE) << VDDCI_SHIFT;
+			else {
+				vddci = phm_find_closest_vddci(&(data->vddci_voltage_table),
+						(dep_table->entries[i].vddc -
+								(uint16_t)VDDC_VDDCI_DELTA));
+				*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+			}
+
+			if (SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control)
+				*mvdd = data->vbios_boot_state.mvdd_bootup_value *
+					VOLTAGE_SCALE;
+			else if (dep_table->entries[i].mvdd)
+				*mvdd = (uint32_t) dep_table->entries[i].mvdd *
+					VOLTAGE_SCALE;
+
+			*voltage |= 1 << PHASES_SHIFT;
+			return 0;
+		}
+	}
+
+	/* sclk is bigger than max sclk in the dependence table */
+	*voltage |= (dep_table->entries[i - 1].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
+
+	if (SMU7_VOLTAGE_CONTROL_NONE == data->vddci_control)
+		*voltage |= (data->vbios_boot_state.vddci_bootup_value *
+				VOLTAGE_SCALE) << VDDCI_SHIFT;
+	else if (dep_table->entries[i-1].vddci) {
+		vddci = phm_find_closest_vddci(&(data->vddci_voltage_table),
+				(dep_table->entries[i].vddc -
+						(uint16_t)VDDC_VDDCI_DELTA));
+		*voltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+	}
+
+	if (SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control)
+		*mvdd = data->vbios_boot_state.mvdd_bootup_value * VOLTAGE_SCALE;
+	else if (dep_table->entries[i].mvdd)
+		*mvdd = (uint32_t) dep_table->entries[i - 1].mvdd * VOLTAGE_SCALE;
+
+	return 0;
+}
+
+static uint16_t scale_fan_gain_settings(uint16_t raw_setting)
+{
+	uint32_t tmp;
+	tmp = raw_setting * 4096 / 100;
+	return (uint16_t)tmp;
+}
+
+static int polaris10_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+
+	const struct polaris10_pt_defaults *defaults = smu_data->power_tune_defaults;
+	SMU74_Discrete_DpmTable  *table = &(smu_data->smc_state_table);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_cac_tdp_table *cac_dtp_table = table_info->cac_dtp_table;
+	struct pp_advance_fan_control_parameters *fan_table =
+			&hwmgr->thermal_controller.advanceFanControlParameters;
+	int i, j, k;
+	const uint16_t *pdef1;
+	const uint16_t *pdef2;
+
+	table->DefaultTdp = PP_HOST_TO_SMC_US((uint16_t)(cac_dtp_table->usTDP * 128));
+	table->TargetTdp  = PP_HOST_TO_SMC_US((uint16_t)(cac_dtp_table->usTDP * 128));
+
+	PP_ASSERT_WITH_CODE(cac_dtp_table->usTargetOperatingTemp <= 255,
+				"Target Operating Temp is out of Range!",
+				);
+
+	table->TemperatureLimitEdge = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTargetOperatingTemp * 256);
+	table->TemperatureLimitHotspot = PP_HOST_TO_SMC_US(
+			cac_dtp_table->usTemperatureLimitHotspot * 256);
+	table->FanGainEdge = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainEdge));
+	table->FanGainHotspot = PP_HOST_TO_SMC_US(
+			scale_fan_gain_settings(fan_table->usFanGainHotspot));
+
+	pdef1 = defaults->BAPMTI_R;
+	pdef2 = defaults->BAPMTI_RC;
+
+	for (i = 0; i < SMU74_DTE_ITERATIONS; i++) {
+		for (j = 0; j < SMU74_DTE_SOURCES; j++) {
+			for (k = 0; k < SMU74_DTE_SINKS; k++) {
+				table->BAPMTI_R[i][j][k] = PP_HOST_TO_SMC_US(*pdef1);
+				table->BAPMTI_RC[i][j][k] = PP_HOST_TO_SMC_US(*pdef2);
+				pdef1++;
+				pdef2++;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int polaris10_populate_svi_load_line(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	const struct polaris10_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	smu_data->power_tune_table.SviLoadLineEn = defaults->SviLoadLineEn;
+	smu_data->power_tune_table.SviLoadLineVddC = defaults->SviLoadLineVddC;
+	smu_data->power_tune_table.SviLoadLineTrimVddC = 3;
+	smu_data->power_tune_table.SviLoadLineOffsetVddC = 0;
+
+	return 0;
+}
+
+static int polaris10_populate_tdc_limit(struct pp_hwmgr *hwmgr)
+{
+	uint16_t tdc_limit;
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	const struct polaris10_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	tdc_limit = (uint16_t)(table_info->cac_dtp_table->usTDC * 128);
+	smu_data->power_tune_table.TDC_VDDC_PkgLimit =
+			CONVERT_FROM_HOST_TO_SMC_US(tdc_limit);
+	smu_data->power_tune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
+			defaults->TDC_VDDC_ThrottleReleaseLimitPerc;
+	smu_data->power_tune_table.TDC_MAWt = defaults->TDC_MAWt;
+
+	return 0;
+}
+
+static int polaris10_populate_dw8(struct pp_hwmgr *hwmgr, uint32_t fuse_table_offset)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	const struct polaris10_pt_defaults *defaults = smu_data->power_tune_defaults;
+	uint32_t temp;
+
+	if (smu7_read_smc_sram_dword(hwmgr->smumgr,
+			fuse_table_offset +
+			offsetof(SMU74_Discrete_PmFuses, TdcWaterfallCtl),
+			(uint32_t *)&temp, SMC_RAM_END))
+		PP_ASSERT_WITH_CODE(false,
+				"Attempt to read PmFuses.DW6 (SviLoadLineEn) from SMC Failed!",
+				return -EINVAL);
+	else {
+		smu_data->power_tune_table.TdcWaterfallCtl = defaults->TdcWaterfallCtl;
+		smu_data->power_tune_table.LPMLTemperatureMin =
+				(uint8_t)((temp >> 16) & 0xff);
+		smu_data->power_tune_table.LPMLTemperatureMax =
+				(uint8_t)((temp >> 8) & 0xff);
+		smu_data->power_tune_table.Reserved = (uint8_t)(temp & 0xff);
+	}
+	return 0;
+}
+
+static int polaris10_populate_temperature_scaler(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		smu_data->power_tune_table.LPMLTemperatureScaler[i] = 0;
+
+	return 0;
+}
+
+static int polaris10_populate_fuzzy_fan(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+
+/* TO DO move to hwmgr */
+	if ((hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity & (1 << 15))
+		|| 0 == hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity)
+		hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity =
+			hwmgr->thermal_controller.advanceFanControlParameters.usDefaultFanOutputSensitivity;
+
+	smu_data->power_tune_table.FuzzyFan_PwmSetDelta = PP_HOST_TO_SMC_US(
+				hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity);
+	return 0;
+}
+
+static int polaris10_populate_gnb_lpml(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		smu_data->power_tune_table.GnbLPML[i] = 0;
+
+	return 0;
+}
+
+static int polaris10_min_max_vgnb_lpml_id_from_bapm_vddc(struct pp_hwmgr *hwmgr)
+{
+	return 0;
+}
+
+static int polaris10_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint16_t hi_sidd = smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd;
+	uint16_t lo_sidd = smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd;
+	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
+
+	hi_sidd = (uint16_t)(cac_table->usHighCACLeakage / 100 * 256);
+	lo_sidd = (uint16_t)(cac_table->usLowCACLeakage / 100 * 256);
+
+	smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(hi_sidd);
+	smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(lo_sidd);
+
+	return 0;
+}
+
+static int polaris10_populate_pm_fuses(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t pm_fuse_table_offset;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerContainment)) {
+		if (smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU7_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU74_Firmware_Header, PmFuseTable),
+				&pm_fuse_table_offset, SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to get pm_fuse_table_offset Failed!",
+					return -EINVAL);
+
+		if (polaris10_populate_svi_load_line(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate SviLoadLine Failed!",
+					return -EINVAL);
+
+		if (polaris10_populate_tdc_limit(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TDCLimit Failed!", return -EINVAL);
+
+		if (polaris10_populate_dw8(hwmgr, pm_fuse_table_offset))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TdcWaterfallCtl, "
+					"LPMLTemperature Min and Max Failed!",
+					return -EINVAL);
+
+		if (0 != polaris10_populate_temperature_scaler(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate LPMLTemperatureScaler Failed!",
+					return -EINVAL);
+
+		if (polaris10_populate_fuzzy_fan(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate Fuzzy Fan Control parameters Failed!",
+					return -EINVAL);
+
+		if (polaris10_populate_gnb_lpml(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate GnbLPML Failed!",
+					return -EINVAL);
+
+		if (polaris10_min_max_vgnb_lpml_id_from_bapm_vddc(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate GnbLPML Min and Max Vid Failed!",
+					return -EINVAL);
+
+		if (polaris10_populate_bapm_vddc_base_leakage_sidd(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate BapmVddCBaseLeakage Hi and Lo "
+					"Sidd Failed!", return -EINVAL);
+
+		if (smu7_copy_bytes_to_smc(hwmgr->smumgr, pm_fuse_table_offset,
+				(uint8_t *)&smu_data->power_tune_table,
+				(sizeof(struct SMU74_Discrete_PmFuses) - 92), SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to download PmFuseTable Failed!",
+					return -EINVAL);
+	}
+	return 0;
+}
+
+/**
+ * Mvdd table preparation for SMC.
+ *
+ * @param    *hwmgr The address of the hardware manager.
+ * @param    *table The SMC DPM table structure to be populated.
+ * @return   0
+ */
+static int polaris10_populate_smc_mvdd_table(struct pp_hwmgr *hwmgr,
+			SMU74_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t count, level;
+
+	if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		count = data->mvdd_voltage_table.count;
+		if (count > SMU_MAX_SMIO_LEVELS)
+			count = SMU_MAX_SMIO_LEVELS;
+		for (level = 0; level < count; level++) {
+			table->SmioTable2.Pattern[level].Voltage =
+				PP_HOST_TO_SMC_US(data->mvdd_voltage_table.entries[count].value * VOLTAGE_SCALE);
+			/* Index into DpmTable.Smio. Drive bits from Smio entry to get this voltage level.*/
+			table->SmioTable2.Pattern[level].Smio =
+				(uint8_t) level;
+			table->Smio[level] |=
+				data->mvdd_voltage_table.entries[level].smio_low;
+		}
+		table->SmioMask2 = data->mvdd_voltage_table.mask_low;
+
+		table->MvddLevelCount = (uint32_t) PP_HOST_TO_SMC_UL(count);
+	}
+
+	return 0;
+}
+
+static int polaris10_populate_smc_vddci_table(struct pp_hwmgr *hwmgr,
+					struct SMU74_Discrete_DpmTable *table)
+{
+	uint32_t count, level;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	count = data->vddci_voltage_table.count;
+
+	if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+		if (count > SMU_MAX_SMIO_LEVELS)
+			count = SMU_MAX_SMIO_LEVELS;
+		for (level = 0; level < count; ++level) {
+			table->SmioTable1.Pattern[level].Voltage =
+				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[level].value * VOLTAGE_SCALE);
+			table->SmioTable1.Pattern[level].Smio = (uint8_t) level;
+
+			table->Smio[level] |= data->vddci_voltage_table.entries[level].smio_low;
+		}
+	}
+
+	table->SmioMask1 = data->vddci_voltage_table.mask_low;
+
+	return 0;
+}
+
+/**
+* Preparation of vddc and vddgfx CAC tables for SMC.
+*
+* @param    hwmgr  the address of the hardware manager
+* @param    table  the SMC DPM table structure to be populated
+* @return   always 0
+*/
+static int polaris10_populate_cac_table(struct pp_hwmgr *hwmgr,
+		struct SMU74_Discrete_DpmTable *table)
+{
+	uint32_t count;
+	uint8_t index;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_voltage_lookup_table *lookup_table =
+			table_info->vddc_lookup_table;
+	/* tables is already swapped, so in order to use the value from it,
+	 * we need to swap it back.
+	 * We are populating vddc CAC data to BapmVddc table
+	 * in split and merged mode
+	 */
+	for (count = 0; count < lookup_table->count; count++) {
+		index = phm_get_voltage_index(lookup_table,
+				data->vddc_voltage_table.entries[count].value);
+		table->BapmVddcVidLoSidd[count] = convert_to_vid(lookup_table->entries[index].us_cac_low);
+		table->BapmVddcVidHiSidd[count] = convert_to_vid(lookup_table->entries[index].us_cac_mid);
+		table->BapmVddcVidHiSidd2[count] = convert_to_vid(lookup_table->entries[index].us_cac_high);
+	}
+
+	return 0;
+}
+
+/**
+* Preparation of voltage tables for SMC.
+*
+* @param    hwmgr   the address of the hardware manager
+* @param    table   the SMC DPM table structure to be populated
+* @return   always  0
+*/
+
+static int polaris10_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
+		struct SMU74_Discrete_DpmTable *table)
+{
+	polaris10_populate_smc_vddci_table(hwmgr, table);
+	polaris10_populate_smc_mvdd_table(hwmgr, table);
+	polaris10_populate_cac_table(hwmgr, table);
+
+	return 0;
+}
+
+static int polaris10_populate_ulv_level(struct pp_hwmgr *hwmgr,
+		struct SMU74_Discrete_Ulv *state)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	state->CcPwrDynRm = 0;
+	state->CcPwrDynRm1 = 0;
+
+	state->VddcOffset = (uint16_t) table_info->us_ulv_voltage_offset;
+	state->VddcOffsetVid = (uint8_t)(table_info->us_ulv_voltage_offset *
+			VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1);
+
+	state->VddcPhase = (data->vddc_phase_shed_control) ? 0 : 1;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm1);
+	CONVERT_FROM_HOST_TO_SMC_US(state->VddcOffset);
+
+	return 0;
+}
+
+static int polaris10_populate_ulv_state(struct pp_hwmgr *hwmgr,
+		struct SMU74_Discrete_DpmTable *table)
+{
+	return polaris10_populate_ulv_level(hwmgr, &table->Ulv);
+}
+
+static int polaris10_populate_smc_link_level(struct pp_hwmgr *hwmgr,
+		struct SMU74_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	int i;
+
+	/* Index (dpm_table->pcie_speed_table.count)
+	 * is reserved for PCIE boot level. */
+	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
+		table->LinkLevel[i].PcieGenSpeed  =
+				(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
+		table->LinkLevel[i].PcieLaneCount = (uint8_t)encode_pcie_lane_width(
+				dpm_table->pcie_speed_table.dpm_levels[i].param1);
+		table->LinkLevel[i].EnabledForActivity = 1;
+		table->LinkLevel[i].SPC = (uint8_t)(data->pcie_spc_cap & 0xff);
+		table->LinkLevel[i].DownThreshold = PP_HOST_TO_SMC_UL(5);
+		table->LinkLevel[i].UpThreshold = PP_HOST_TO_SMC_UL(30);
+	}
+
+	smu_data->smc_state_table.LinkLevelCount =
+			(uint8_t)dpm_table->pcie_speed_table.count;
+
+/* To Do move to hwmgr */
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
+
+	return 0;
+}
+
+
+static void polaris10_get_sclk_range_table(struct pp_hwmgr *hwmgr,
+				   SMU74_Discrete_DpmTable  *table)
+{
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	uint32_t i, ref_clk;
+
+	struct pp_atom_ctrl_sclk_range_table range_table_from_vbios = { { {0} } };
+
+	ref_clk = smu7_get_xclk(hwmgr);
+
+	if (0 == atomctrl_get_smc_sclk_range_table(hwmgr, &range_table_from_vbios)) {
+		for (i = 0; i < NUM_SCLK_RANGE; i++) {
+			table->SclkFcwRangeTable[i].vco_setting = range_table_from_vbios.entry[i].ucVco_setting;
+			table->SclkFcwRangeTable[i].postdiv = range_table_from_vbios.entry[i].ucPostdiv;
+			table->SclkFcwRangeTable[i].fcw_pcc = range_table_from_vbios.entry[i].usFcw_pcc;
+
+			table->SclkFcwRangeTable[i].fcw_trans_upper = range_table_from_vbios.entry[i].usFcw_trans_upper;
+			table->SclkFcwRangeTable[i].fcw_trans_lower = range_table_from_vbios.entry[i].usRcw_trans_lower;
+
+			CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_pcc);
+			CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_upper);
+			CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_lower);
+		}
+		return;
+	}
+
+	for (i = 0; i < NUM_SCLK_RANGE; i++) {
+		smu_data->range_table[i].trans_lower_frequency = (ref_clk * Range_Table[i].fcw_trans_lower) >> Range_Table[i].postdiv;
+		smu_data->range_table[i].trans_upper_frequency = (ref_clk * Range_Table[i].fcw_trans_upper) >> Range_Table[i].postdiv;
+
+		table->SclkFcwRangeTable[i].vco_setting = Range_Table[i].vco_setting;
+		table->SclkFcwRangeTable[i].postdiv = Range_Table[i].postdiv;
+		table->SclkFcwRangeTable[i].fcw_pcc = Range_Table[i].fcw_pcc;
+
+		table->SclkFcwRangeTable[i].fcw_trans_upper = Range_Table[i].fcw_trans_upper;
+		table->SclkFcwRangeTable[i].fcw_trans_lower = Range_Table[i].fcw_trans_lower;
+
+		CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_pcc);
+		CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_upper);
+		CONVERT_FROM_HOST_TO_SMC_US(table->SclkFcwRangeTable[i].fcw_trans_lower);
+	}
+}
+
+/**
+* Calculates the SCLK dividers using the provided engine clock
+*
+* @param    hwmgr  the address of the hardware manager
+* @param    clock  the engine clock to use to populate the structure
+* @param    sclk   the SMC SCLK structure to be populated
+*/
+static int polaris10_calculate_sclk_params(struct pp_hwmgr *hwmgr,
+		uint32_t clock, SMU_SclkSetting *sclk_setting)
+{
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	const SMU74_Discrete_DpmTable *table = &(smu_data->smc_state_table);
+	struct pp_atomctrl_clock_dividers_ai dividers;
+	uint32_t ref_clock;
+	uint32_t pcc_target_percent, pcc_target_freq, ss_target_percent, ss_target_freq;
+	uint8_t i;
+	int result;
+	uint64_t temp;
+
+	sclk_setting->SclkFrequency = clock;
+	/* get the engine clock dividers for this clock value */
+	result = atomctrl_get_engine_pll_dividers_ai(hwmgr, clock,  &dividers);
+	if (result == 0) {
+		sclk_setting->Fcw_int = dividers.usSclk_fcw_int;
+		sclk_setting->Fcw_frac = dividers.usSclk_fcw_frac;
+		sclk_setting->Pcc_fcw_int = dividers.usPcc_fcw_int;
+		sclk_setting->PllRange = dividers.ucSclkPllRange;
+		sclk_setting->Sclk_slew_rate = 0x400;
+		sclk_setting->Pcc_up_slew_rate = dividers.usPcc_fcw_slew_frac;
+		sclk_setting->Pcc_down_slew_rate = 0xffff;
+		sclk_setting->SSc_En = dividers.ucSscEnable;
+		sclk_setting->Fcw1_int = dividers.usSsc_fcw1_int;
+		sclk_setting->Fcw1_frac = dividers.usSsc_fcw1_frac;
+		sclk_setting->Sclk_ss_slew_rate = dividers.usSsc_fcw_slew_frac;
+		return result;
+	}
+
+	ref_clock = smu7_get_xclk(hwmgr);
+
+	for (i = 0; i < NUM_SCLK_RANGE; i++) {
+		if (clock > smu_data->range_table[i].trans_lower_frequency
+		&& clock <= smu_data->range_table[i].trans_upper_frequency) {
+			sclk_setting->PllRange = i;
+			break;
+		}
+	}
+
+	sclk_setting->Fcw_int = (uint16_t)((clock << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv) / ref_clock);
+	temp = clock << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv;
+	temp <<= 0x10;
+	do_div(temp, ref_clock);
+	sclk_setting->Fcw_frac = temp & 0xffff;
+
+	pcc_target_percent = 10; /*  Hardcode 10% for now. */
+	pcc_target_freq = clock - (clock * pcc_target_percent / 100);
+	sclk_setting->Pcc_fcw_int = (uint16_t)((pcc_target_freq << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv) / ref_clock);
+
+	ss_target_percent = 2; /*  Hardcode 2% for now. */
+	sclk_setting->SSc_En = 0;
+	if (ss_target_percent) {
+		sclk_setting->SSc_En = 1;
+		ss_target_freq = clock - (clock * ss_target_percent / 100);
+		sclk_setting->Fcw1_int = (uint16_t)((ss_target_freq << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv) / ref_clock);
+		temp = ss_target_freq << table->SclkFcwRangeTable[sclk_setting->PllRange].postdiv;
+		temp <<= 0x10;
+		do_div(temp, ref_clock);
+		sclk_setting->Fcw1_frac = temp & 0xffff;
+	}
+
+	return 0;
+}
+
+/**
+* Populates single SMC SCLK structure using the provided engine clock
+*
+* @param    hwmgr      the address of the hardware manager
+* @param    clock the engine clock to use to populate the structure
+* @param    sclk        the SMC SCLK structure to be populated
+*/
+
+static int polaris10_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
+		uint32_t clock, uint16_t sclk_al_threshold,
+		struct SMU74_Discrete_GraphicsLevel *level)
+{
+	int result;
+	/* PP_Clocks minClocks; */
+	uint32_t mvdd;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	SMU_SclkSetting curr_sclk_setting = { 0 };
+
+	result = polaris10_calculate_sclk_params(hwmgr, clock, &curr_sclk_setting);
+
+	/* populate graphics levels */
+	result = polaris10_get_dependency_volt_by_clk(hwmgr,
+			table_info->vdd_dep_on_sclk, clock,
+			&level->MinVoltage, &mvdd);
+
+	PP_ASSERT_WITH_CODE((0 == result),
+			"can not find VDDC voltage value for "
+			"VDDC engine clock dependency table",
+			return result);
+	level->ActivityLevel = sclk_al_threshold;
+
+	level->CcPwrDynRm = 0;
+	level->CcPwrDynRm1 = 0;
+	level->EnabledForActivity = 0;
+	level->EnabledForThrottle = 1;
+	level->UpHyst = 10;
+	level->DownHyst = 0;
+	level->VoltageDownHyst = 0;
+	level->PowerThrottle = 0;
+	data->display_timing.min_clock_in_sr = hwmgr->display_config.min_core_set_clock_in_sr;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SclkDeepSleep))
+		level->DeepSleepDivId = smu7_get_sleep_divider_id_from_clock(clock,
+								hwmgr->display_config.min_core_set_clock_in_sr);
+
+	/* Default to slow, highest DPM level will be
+	 * set to PPSMC_DISPLAY_WATERMARK_LOW later.
+	 */
+	if (data->update_up_hyst)
+		level->UpHyst = (uint8_t)data->up_hyst;
+	if (data->update_down_hyst)
+		level->DownHyst = (uint8_t)data->down_hyst;
+
+	level->SclkSetting = curr_sclk_setting;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(level->MinVoltage);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->CcPwrDynRm1);
+	CONVERT_FROM_HOST_TO_SMC_US(level->ActivityLevel);
+	CONVERT_FROM_HOST_TO_SMC_UL(level->SclkSetting.SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw_int);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw_frac);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Pcc_fcw_int);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Sclk_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Pcc_up_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Pcc_down_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw1_int);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Fcw1_frac);
+	CONVERT_FROM_HOST_TO_SMC_US(level->SclkSetting.Sclk_ss_slew_rate);
+	return 0;
+}
+
+/**
+* Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states
+*
+* @param    hwmgr      the address of the hardware manager
+*/
+int polaris10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
+{
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	struct smu7_dpm_table *dpm_table = &hw_data->dpm_table;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
+	uint8_t pcie_entry_cnt = (uint8_t) hw_data->dpm_table.pcie_speed_table.count;
+	int result = 0;
+	uint32_t array = smu_data->smu7_data.dpm_table_start +
+			offsetof(SMU74_Discrete_DpmTable, GraphicsLevel);
+	uint32_t array_size = sizeof(struct SMU74_Discrete_GraphicsLevel) *
+			SMU74_MAX_LEVELS_GRAPHICS;
+	struct SMU74_Discrete_GraphicsLevel *levels =
+			smu_data->smc_state_table.GraphicsLevel;
+	uint32_t i, max_entry;
+	uint8_t hightest_pcie_level_enabled = 0,
+		lowest_pcie_level_enabled = 0,
+		mid_pcie_level_enabled = 0,
+		count = 0;
+
+	polaris10_get_sclk_range_table(hwmgr, &(smu_data->smc_state_table));
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++) {
+
+		result = polaris10_populate_single_graphic_level(hwmgr,
+				dpm_table->sclk_table.dpm_levels[i].value,
+				(uint16_t)smu_data->activity_target[i],
+				&(smu_data->smc_state_table.GraphicsLevel[i]));
+		if (result)
+			return result;
+
+		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
+		if (i > 1)
+			levels[i].DeepSleepDivId = 0;
+	}
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_SPLLShutdownSupport))
+		smu_data->smc_state_table.GraphicsLevel[0].SclkSetting.SSc_En = 0;
+
+	smu_data->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1;
+	smu_data->smc_state_table.GraphicsDpmLevelCount =
+			(uint8_t)dpm_table->sclk_table.count;
+	hw_data->dpm_level_enable_mask.sclk_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
+
+
+	if (pcie_table != NULL) {
+		PP_ASSERT_WITH_CODE((1 <= pcie_entry_cnt),
+				"There must be 1 or more PCIE levels defined in PPTable.",
+				return -EINVAL);
+		max_entry = pcie_entry_cnt - 1;
+		for (i = 0; i < dpm_table->sclk_table.count; i++)
+			levels[i].pcieDpmLevel =
+					(uint8_t) ((i < max_entry) ? i : max_entry);
+	} else {
+		while (hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << (hightest_pcie_level_enabled + 1))) != 0))
+			hightest_pcie_level_enabled++;
+
+		while (hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << lowest_pcie_level_enabled)) == 0))
+			lowest_pcie_level_enabled++;
+
+		while ((count < hightest_pcie_level_enabled) &&
+				((hw_data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+						(1 << (lowest_pcie_level_enabled + 1 + count))) == 0))
+			count++;
+
+		mid_pcie_level_enabled = (lowest_pcie_level_enabled + 1 + count) <
+				hightest_pcie_level_enabled ?
+						(lowest_pcie_level_enabled + 1 + count) :
+						hightest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to hightest_pcie_level_enabled */
+		for (i = 2; i < dpm_table->sclk_table.count; i++)
+			levels[i].pcieDpmLevel = hightest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to lowest_pcie_level_enabled */
+		levels[0].pcieDpmLevel = lowest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to mid_pcie_level_enabled */
+		levels[1].pcieDpmLevel = mid_pcie_level_enabled;
+	}
+	/* level count will send to smc once at init smc table and never change */
+	result = smu7_copy_bytes_to_smc(smumgr, array, (uint8_t *)levels,
+			(uint32_t)array_size, SMC_RAM_END);
+
+	return result;
+}
+
+
+static int polaris10_populate_single_memory_level(struct pp_hwmgr *hwmgr,
+		uint32_t clock, struct SMU74_Discrete_MemoryLevel *mem_level)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int result = 0;
+	struct cgs_display_info info = {0, 0, NULL};
+	uint32_t mclk_stutter_mode_threshold = 40000;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+
+	if (table_info->vdd_dep_on_mclk) {
+		result = polaris10_get_dependency_volt_by_clk(hwmgr,
+				table_info->vdd_dep_on_mclk, clock,
+				&mem_level->MinVoltage, &mem_level->MinMvdd);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find MinVddc voltage value from memory "
+				"VDDC voltage dependency table", return result);
+	}
+
+	mem_level->MclkFrequency = clock;
+	mem_level->EnabledForThrottle = 1;
+	mem_level->EnabledForActivity = 0;
+	mem_level->UpHyst = 0;
+	mem_level->DownHyst = 100;
+	mem_level->VoltageDownHyst = 0;
+	mem_level->ActivityLevel = (uint16_t)data->mclk_activity_target;
+	mem_level->StutterEnable = false;
+	mem_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	data->display_timing.num_existing_displays = info.display_count;
+
+	if (mclk_stutter_mode_threshold &&
+		(clock <= mclk_stutter_mode_threshold) &&
+		(SMUM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL,
+				STUTTER_ENABLE) & 0x1))
+		mem_level->StutterEnable = true;
+
+	if (!result) {
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinMvdd);
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_US(mem_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(mem_level->MinVoltage);
+	}
+	return result;
+}
+
+/**
+* Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states
+*
+* @param    hwmgr      the address of the hardware manager
+*/
+int polaris10_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
+{
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	struct smu7_dpm_table *dpm_table = &hw_data->dpm_table;
+	int result;
+	/* populate MCLK dpm table to SMU7 */
+	uint32_t array = smu_data->smu7_data.dpm_table_start +
+			offsetof(SMU74_Discrete_DpmTable, MemoryLevel);
+	uint32_t array_size = sizeof(SMU74_Discrete_MemoryLevel) *
+			SMU74_MAX_LEVELS_MEMORY;
+	struct SMU74_Discrete_MemoryLevel *levels =
+			smu_data->smc_state_table.MemoryLevel;
+	uint32_t i;
+
+	for (i = 0; i < dpm_table->mclk_table.count; i++) {
+		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
+				"can not populate memory level as memory clock is zero",
+				return -EINVAL);
+		result = polaris10_populate_single_memory_level(hwmgr,
+				dpm_table->mclk_table.dpm_levels[i].value,
+				&levels[i]);
+		if (i == dpm_table->mclk_table.count - 1) {
+			levels[i].DisplayWatermark = PPSMC_DISPLAY_WATERMARK_HIGH;
+			levels[i].EnabledForActivity = 1;
+		}
+		if (result)
+			return result;
+	}
+
+	/* In order to prevent MC activity from stutter mode to push DPM up,
+	 * the UVD change complements this by putting the MCLK in
+	 * a higher state by default such that we are not affected by
+	 * up threshold or and MCLK DPM latency.
+	 */
+	levels[0].ActivityLevel = 0x1f;
+	CONVERT_FROM_HOST_TO_SMC_US(levels[0].ActivityLevel);
+
+	smu_data->smc_state_table.MemoryDpmLevelCount =
+			(uint8_t)dpm_table->mclk_table.count;
+	hw_data->dpm_level_enable_mask.mclk_dpm_enable_mask =
+			phm_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
+
+	/* level count will send to smc once at init smc table and never change */
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr, array, (uint8_t *)levels,
+			(uint32_t)array_size, SMC_RAM_END);
+
+	return result;
+}
+
+/**
+* Populates the SMC MVDD structure using the provided memory clock.
+*
+* @param    hwmgr      the address of the hardware manager
+* @param    mclk        the MCLK value to be used in the decision if MVDD should be high or low.
+* @param    voltage     the SMC VOLTAGE structure to be populated
+*/
+static int polaris10_populate_mvdd_value(struct pp_hwmgr *hwmgr,
+		uint32_t mclk, SMIO_Pattern *smio_pat)
+{
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t i = 0;
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
+		/* find mvdd value which clock is more than request */
+		for (i = 0; i < table_info->vdd_dep_on_mclk->count; i++) {
+			if (mclk <= table_info->vdd_dep_on_mclk->entries[i].clk) {
+				smio_pat->Voltage = data->mvdd_voltage_table.entries[i].value;
+				break;
+			}
+		}
+		PP_ASSERT_WITH_CODE(i < table_info->vdd_dep_on_mclk->count,
+				"MVDD Voltage is outside the supported range.",
+				return -EINVAL);
+	} else
+		return -EINVAL;
+
+	return 0;
+}
+
+static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
+		SMU74_Discrete_DpmTable *table)
+{
+	int result = 0;
+	uint32_t sclk_frequency;
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	SMIO_Pattern vol_level;
+	uint32_t mvdd;
+	uint16_t us_mvdd;
+
+	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+	/* Get MinVoltage and Frequency from DPM0,
+	 * already converted to SMC_UL */
+	sclk_frequency = data->vbios_boot_state.sclk_bootup_value;
+	result = polaris10_get_dependency_volt_by_clk(hwmgr,
+			table_info->vdd_dep_on_sclk,
+			sclk_frequency,
+			&table->ACPILevel.MinVoltage, &mvdd);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Cannot find ACPI VDDC voltage value "
+			"in Clock Dependency Table",
+			);
+
+	result = polaris10_calculate_sclk_params(hwmgr, sclk_frequency,  &(table->ACPILevel.SclkSetting));
+	PP_ASSERT_WITH_CODE(result == 0, "Error retrieving Engine Clock dividers from VBIOS.", return result);
+
+	table->ACPILevel.DeepSleepDivId = 0;
+	table->ACPILevel.CcPwrDynRm = 0;
+	table->ACPILevel.CcPwrDynRm1 = 0;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.MinVoltage);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkSetting.SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw_int);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw_frac);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Pcc_fcw_int);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Sclk_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Pcc_up_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Pcc_down_slew_rate);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw1_int);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Fcw1_frac);
+	CONVERT_FROM_HOST_TO_SMC_US(table->ACPILevel.SclkSetting.Sclk_ss_slew_rate);
+
+
+	/* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */
+	table->MemoryACPILevel.MclkFrequency = data->vbios_boot_state.mclk_bootup_value;
+	result = polaris10_get_dependency_volt_by_clk(hwmgr,
+			table_info->vdd_dep_on_mclk,
+			table->MemoryACPILevel.MclkFrequency,
+			&table->MemoryACPILevel.MinVoltage, &mvdd);
+	PP_ASSERT_WITH_CODE((0 == result),
+			"Cannot find ACPI VDDCI voltage value "
+			"in Clock Dependency Table",
+			);
+
+	us_mvdd = 0;
+	if ((SMU7_VOLTAGE_CONTROL_NONE == data->mvdd_control) ||
+			(data->mclk_dpm_key_disabled))
+		us_mvdd = data->vbios_boot_state.mvdd_bootup_value;
+	else {
+		if (!polaris10_populate_mvdd_value(hwmgr,
+				data->dpm_table.mclk_table.dpm_levels[0].value,
+				&vol_level))
+			us_mvdd = vol_level.Voltage;
+	}
+
+	if (0 == polaris10_populate_mvdd_value(hwmgr, 0, &vol_level))
+		table->MemoryACPILevel.MinMvdd = PP_HOST_TO_SMC_UL(vol_level.Voltage);
+	else
+		table->MemoryACPILevel.MinMvdd = 0;
+
+	table->MemoryACPILevel.StutterEnable = false;
+
+	table->MemoryACPILevel.EnabledForThrottle = 0;
+	table->MemoryACPILevel.EnabledForActivity = 0;
+	table->MemoryACPILevel.UpHyst = 0;
+	table->MemoryACPILevel.DownHyst = 100;
+	table->MemoryACPILevel.VoltageDownHyst = 0;
+	table->MemoryACPILevel.ActivityLevel =
+			PP_HOST_TO_SMC_US((uint16_t)data->mclk_activity_target);
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MinVoltage);
+
+	return result;
+}
+
+static int polaris10_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
+		SMU74_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t vddci;
+
+	table->VceLevelCount = (uint8_t)(mm_table->count);
+	table->VceBootLevel = 0;
+
+	for (count = 0; count < table->VceLevelCount; count++) {
+		table->VceLevel[count].Frequency = mm_table->entries[count].eclk;
+		table->VceLevel[count].MinVoltage = 0;
+		table->VceLevel[count].MinVoltage |=
+				(mm_table->entries[count].vddc * VOLTAGE_SCALE) << VDDC_SHIFT;
+
+		if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
+			vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table),
+						mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
+			vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA;
+		else
+			vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT;
+
+
+		table->VceLevel[count].MinVoltage |=
+				(vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->VceLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/*retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->VceLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for VCE engine clock",
+				return result);
+
+		table->VceLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+
+static int polaris10_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
+		SMU74_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t vddci;
+
+	table->SamuBootLevel = 0;
+	table->SamuLevelCount = (uint8_t)(mm_table->count);
+
+	for (count = 0; count < table->SamuLevelCount; count++) {
+		/* not sure whether we need evclk or not */
+		table->SamuLevel[count].MinVoltage = 0;
+		table->SamuLevel[count].Frequency = mm_table->entries[count].samclock;
+		table->SamuLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+
+		if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
+			vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table),
+						mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
+			vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA;
+		else
+			vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT;
+
+		table->SamuLevel[count].MinVoltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->SamuLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->SamuLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for samu clock", return result);
+
+		table->SamuLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].Frequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].MinVoltage);
+	}
+	return result;
+}
+
+static int polaris10_populate_memory_timing_parameters(struct pp_hwmgr *hwmgr,
+		int32_t eng_clock, int32_t mem_clock,
+		SMU74_Discrete_MCArbDramTimingTableEntry *arb_regs)
+{
+	uint32_t dram_timing;
+	uint32_t dram_timing2;
+	uint32_t burst_time;
+	int result;
+
+	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
+			eng_clock, mem_clock);
+	PP_ASSERT_WITH_CODE(result == 0,
+			"Error calling VBIOS to set DRAM_TIMING.", return result);
+
+	dram_timing = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+	dram_timing2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+	burst_time = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
+
+
+	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dram_timing);
+	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dram_timing2);
+	arb_regs->McArbBurstTime   = (uint8_t)burst_time;
+
+	return 0;
+}
+
+static int polaris10_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	struct SMU74_Discrete_MCArbDramTimingTable arb_regs;
+	uint32_t i, j;
+	int result = 0;
+
+	for (i = 0; i < hw_data->dpm_table.sclk_table.count; i++) {
+		for (j = 0; j < hw_data->dpm_table.mclk_table.count; j++) {
+			result = polaris10_populate_memory_timing_parameters(hwmgr,
+					hw_data->dpm_table.sclk_table.dpm_levels[i].value,
+					hw_data->dpm_table.mclk_table.dpm_levels[j].value,
+					&arb_regs.entries[i][j]);
+			if (result == 0)
+				result = atomctrl_set_ac_timing_ai(hwmgr, hw_data->dpm_table.mclk_table.dpm_levels[j].value, j);
+			if (result != 0)
+				return result;
+		}
+	}
+
+	result = smu7_copy_bytes_to_smc(
+			hwmgr->smumgr,
+			smu_data->smu7_data.arb_table_start,
+			(uint8_t *)&arb_regs,
+			sizeof(SMU74_Discrete_MCArbDramTimingTable),
+			SMC_RAM_END);
+	return result;
+}
+
+static int polaris10_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
+		struct SMU74_Discrete_DpmTable *table)
+{
+	int result = -EINVAL;
+	uint8_t count;
+	struct pp_atomctrl_clock_dividers_vi dividers;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+			table_info->mm_dep_table;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t vddci;
+
+	table->UvdLevelCount = (uint8_t)(mm_table->count);
+	table->UvdBootLevel = 0;
+
+	for (count = 0; count < table->UvdLevelCount; count++) {
+		table->UvdLevel[count].MinVoltage = 0;
+		table->UvdLevel[count].VclkFrequency = mm_table->entries[count].vclk;
+		table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk;
+		table->UvdLevel[count].MinVoltage |= (mm_table->entries[count].vddc *
+				VOLTAGE_SCALE) << VDDC_SHIFT;
+
+		if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control)
+			vddci = (uint32_t)phm_find_closest_vddci(&(data->vddci_voltage_table),
+						mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		else if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control)
+			vddci = mm_table->entries[count].vddc - VDDC_VDDCI_DELTA;
+		else
+			vddci = (data->vbios_boot_state.vddci_bootup_value * VOLTAGE_SCALE) << VDDCI_SHIFT;
+
+		table->UvdLevel[count].MinVoltage |= (vddci * VOLTAGE_SCALE) << VDDCI_SHIFT;
+		table->UvdLevel[count].MinVoltage |= 1 << PHASES_SHIFT;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->UvdLevel[count].VclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for Vclk clock", return result);
+
+		table->UvdLevel[count].VclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+				table->UvdLevel[count].DclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((0 == result),
+				"can not find divide id for Dclk clock", return result);
+
+		table->UvdLevel[count].DclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].MinVoltage);
+	}
+
+	return result;
+}
+
+static int polaris10_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
+		struct SMU74_Discrete_DpmTable *table)
+{
+	int result = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	table->GraphicsBootLevel = 0;
+	table->MemoryBootLevel = 0;
+
+	/* find boot level from dpm table */
+	result = phm_find_boot_level(&(data->dpm_table.sclk_table),
+			data->vbios_boot_state.sclk_bootup_value,
+			(uint32_t *)&(table->GraphicsBootLevel));
+
+	result = phm_find_boot_level(&(data->dpm_table.mclk_table),
+			data->vbios_boot_state.mclk_bootup_value,
+			(uint32_t *)&(table->MemoryBootLevel));
+
+	table->BootVddc  = data->vbios_boot_state.vddc_bootup_value *
+			VOLTAGE_SCALE;
+	table->BootVddci = data->vbios_boot_state.vddci_bootup_value *
+			VOLTAGE_SCALE;
+	table->BootMVdd  = data->vbios_boot_state.mvdd_bootup_value *
+			VOLTAGE_SCALE;
+
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddc);
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootVddci);
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootMVdd);
+
+	return 0;
+}
+
+static int polaris10_populate_smc_initailial_state(struct pp_hwmgr *hwmgr)
+{
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint8_t count, level;
+
+	count = (uint8_t)(table_info->vdd_dep_on_sclk->count);
+
+	for (level = 0; level < count; level++) {
+		if (table_info->vdd_dep_on_sclk->entries[level].clk >=
+				hw_data->vbios_boot_state.sclk_bootup_value) {
+			smu_data->smc_state_table.GraphicsBootLevel = level;
+			break;
+		}
+	}
+
+	count = (uint8_t)(table_info->vdd_dep_on_mclk->count);
+	for (level = 0; level < count; level++) {
+		if (table_info->vdd_dep_on_mclk->entries[level].clk >=
+				hw_data->vbios_boot_state.mclk_bootup_value) {
+			smu_data->smc_state_table.MemoryBootLevel = level;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+
+static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
+{
+	uint32_t ro, efuse, volt_without_cks, volt_with_cks, value, max, min;
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+
+	uint8_t i, stretch_amount, stretch_amount2, volt_offset = 0;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+
+	stretch_amount = (uint8_t)table_info->cac_dtp_table->usClockStretchAmount;
+
+	/* Read SMU_Eefuse to read and calculate RO and determine
+	 * if the part is SS or FF. if RO >= 1660MHz, part is FF.
+	 */
+	efuse = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixSMU_EFUSE_0 + (67 * 4));
+	efuse &= 0xFF000000;
+	efuse = efuse >> 24;
+
+	if (hwmgr->chip_id == CHIP_POLARIS10) {
+		min = 1000;
+		max = 2300;
+	} else {
+		min = 1100;
+		max = 2100;
+	}
+
+	ro = efuse * (max - min) / 255 + min;
+
+	/* Populate Sclk_CKS_masterEn0_7 and Sclk_voltageOffset */
+	for (i = 0; i < sclk_table->count; i++) {
+		smu_data->smc_state_table.Sclk_CKS_masterEn0_7 |=
+				sclk_table->entries[i].cks_enable << i;
+		if (hwmgr->chip_id == CHIP_POLARIS10) {
+			volt_without_cks = (uint32_t)((2753594000U + (sclk_table->entries[i].clk/100) * 136418 - (ro - 70) * 1000000) / \
+						(2424180 - (sclk_table->entries[i].clk/100) * 1132925/1000));
+			volt_with_cks = (uint32_t)((2797202000U + sclk_table->entries[i].clk/100 * 3232 - (ro - 65) * 1000000) / \
+					(2522480 - sclk_table->entries[i].clk/100 * 115764/100));
+		} else {
+			volt_without_cks = (uint32_t)((2416794800U + (sclk_table->entries[i].clk/100) * 1476925/10 - (ro - 50) * 1000000) / \
+						(2625416 - (sclk_table->entries[i].clk/100) * (12586807/10000)));
+			volt_with_cks = (uint32_t)((2999656000U - sclk_table->entries[i].clk/100 * 392803 - (ro - 44) * 1000000) / \
+					(3422454 - sclk_table->entries[i].clk/100 * (18886376/10000)));
+		}
+
+		if (volt_without_cks >= volt_with_cks)
+			volt_offset = (uint8_t)(((volt_without_cks - volt_with_cks +
+					sclk_table->entries[i].cks_voffset) * 100 + 624) / 625);
+
+		smu_data->smc_state_table.Sclk_voltageOffset[i] = volt_offset;
+	}
+
+	smu_data->smc_state_table.LdoRefSel = (table_info->cac_dtp_table->ucCKS_LDO_REFSEL != 0) ? table_info->cac_dtp_table->ucCKS_LDO_REFSEL : 6;
+	/* Populate CKS Lookup Table */
+	if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5)
+		stretch_amount2 = 0;
+	else if (stretch_amount == 3 || stretch_amount == 4)
+		stretch_amount2 = 1;
+	else {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ClockStretcher);
+		PP_ASSERT_WITH_CODE(false,
+				"Stretch Amount in PPTable not supported\n",
+				return -EINVAL);
+	}
+
+	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL);
+	value &= 0xFFFFFFFE;
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixPWR_CKS_CNTL, value);
+
+	return 0;
+}
+
+/**
+* Populates the SMC VRConfig field in DPM table.
+*
+* @param    hwmgr   the address of the hardware manager
+* @param    table   the SMC DPM table structure to be populated
+* @return   always 0
+*/
+static int polaris10_populate_vr_config(struct pp_hwmgr *hwmgr,
+		struct SMU74_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	uint16_t config;
+
+	config = VR_MERGED_WITH_VDDC;
+	table->VRConfig |= (config << VRCONF_VDDGFX_SHIFT);
+
+	/* Set Vddc Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+		config = VR_SVI2_PLANE_1;
+		table->VRConfig |= config;
+	} else {
+		PP_ASSERT_WITH_CODE(false,
+				"VDDC should be on SVI2 control in merged mode!",
+				);
+	}
+	/* Set Vddci Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
+		config = VR_SVI2_PLANE_2;  /* only in merged mode */
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	} else if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+		config = VR_SMIO_PATTERN_1;
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	} else {
+		config = VR_STATIC_VOLTAGE;
+		table->VRConfig |= (config << VRCONF_VDDCI_SHIFT);
+	}
+	/* Set Mvdd Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
+		config = VR_SVI2_PLANE_2;
+		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, smu_data->smu7_data.soft_regs_start +
+			offsetof(SMU74_SoftRegisters, AllowMvddSwitch), 0x1);
+	} else {
+		config = VR_STATIC_VOLTAGE;
+		table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+	}
+
+	return 0;
+}
+
+
+static int polaris10_populate_avfs_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+
+	SMU74_Discrete_DpmTable  *table = &(smu_data->smc_state_table);
+	int result = 0;
+	struct pp_atom_ctrl__avfs_parameters avfs_params = {0};
+	AVFS_meanNsigma_t AVFS_meanNsigma = { {0} };
+	AVFS_Sclk_Offset_t AVFS_SclkOffset = { {0} };
+	uint32_t tmp, i;
+
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)hwmgr->pptable;
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+
+
+	if (smu_data->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED)
+		return result;
+
+	result = atomctrl_get_avfs_information(hwmgr, &avfs_params);
+
+	if (0 == result) {
+		table->BTCGB_VDROOP_TABLE[0].a0  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a0);
+		table->BTCGB_VDROOP_TABLE[0].a1  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a1);
+		table->BTCGB_VDROOP_TABLE[0].a2  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSON_a2);
+		table->BTCGB_VDROOP_TABLE[1].a0  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a0);
+		table->BTCGB_VDROOP_TABLE[1].a1  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a1);
+		table->BTCGB_VDROOP_TABLE[1].a2  = PP_HOST_TO_SMC_UL(avfs_params.ulGB_VDROOP_TABLE_CKSOFF_a2);
+		table->AVFSGB_VDROOP_TABLE[0].m1 = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSON_m1);
+		table->AVFSGB_VDROOP_TABLE[0].m2 = PP_HOST_TO_SMC_US(avfs_params.usAVFSGB_FUSE_TABLE_CKSON_m2);
+		table->AVFSGB_VDROOP_TABLE[0].b  = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSON_b);
+		table->AVFSGB_VDROOP_TABLE[0].m1_shift = 24;
+		table->AVFSGB_VDROOP_TABLE[0].m2_shift  = 12;
+		table->AVFSGB_VDROOP_TABLE[1].m1 = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSOFF_m1);
+		table->AVFSGB_VDROOP_TABLE[1].m2 = PP_HOST_TO_SMC_US(avfs_params.usAVFSGB_FUSE_TABLE_CKSOFF_m2);
+		table->AVFSGB_VDROOP_TABLE[1].b  = PP_HOST_TO_SMC_UL(avfs_params.ulAVFSGB_FUSE_TABLE_CKSOFF_b);
+		table->AVFSGB_VDROOP_TABLE[1].m1_shift = 24;
+		table->AVFSGB_VDROOP_TABLE[1].m2_shift  = 12;
+		table->MaxVoltage                = PP_HOST_TO_SMC_US(avfs_params.usMaxVoltage_0_25mv);
+		AVFS_meanNsigma.Aconstant[0]      = PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant0);
+		AVFS_meanNsigma.Aconstant[1]      = PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant1);
+		AVFS_meanNsigma.Aconstant[2]      = PP_HOST_TO_SMC_UL(avfs_params.ulAVFS_meanNsigma_Acontant2);
+		AVFS_meanNsigma.DC_tol_sigma      = PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_DC_tol_sigma);
+		AVFS_meanNsigma.Platform_mean     = PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_Platform_mean);
+		AVFS_meanNsigma.PSM_Age_CompFactor = PP_HOST_TO_SMC_US(avfs_params.usPSM_Age_ComFactor);
+		AVFS_meanNsigma.Platform_sigma     = PP_HOST_TO_SMC_US(avfs_params.usAVFS_meanNsigma_Platform_sigma);
+
+		for (i = 0; i < NUM_VFT_COLUMNS; i++) {
+			AVFS_meanNsigma.Static_Voltage_Offset[i] = (uint8_t)(sclk_table->entries[i].cks_voffset * 100 / 625);
+			AVFS_SclkOffset.Sclk_Offset[i] = PP_HOST_TO_SMC_US((uint16_t)(sclk_table->entries[i].sclk_offset) / 100);
+		}
+
+		result = smu7_read_smc_sram_dword(smumgr,
+				SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU74_Firmware_Header, AvfsMeanNSigma),
+				&tmp, SMC_RAM_END);
+
+		smu7_copy_bytes_to_smc(smumgr,
+					tmp,
+					(uint8_t *)&AVFS_meanNsigma,
+					sizeof(AVFS_meanNsigma_t),
+					SMC_RAM_END);
+
+		result = smu7_read_smc_sram_dword(smumgr,
+				SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU74_Firmware_Header, AvfsSclkOffsetTable),
+				&tmp, SMC_RAM_END);
+		smu7_copy_bytes_to_smc(smumgr,
+					tmp,
+					(uint8_t *)&AVFS_SclkOffset,
+					sizeof(AVFS_Sclk_Offset_t),
+					SMC_RAM_END);
+
+		data->avfs_vdroop_override_setting = (avfs_params.ucEnableGB_VDROOP_TABLE_CKSON << BTCGB0_Vdroop_Enable_SHIFT) |
+						(avfs_params.ucEnableGB_VDROOP_TABLE_CKSOFF << BTCGB1_Vdroop_Enable_SHIFT) |
+						(avfs_params.ucEnableGB_FUSE_TABLE_CKSON << AVFSGB0_Vdroop_Enable_SHIFT) |
+						(avfs_params.ucEnableGB_FUSE_TABLE_CKSOFF << AVFSGB1_Vdroop_Enable_SHIFT);
+		data->apply_avfs_cks_off_voltage = (avfs_params.ucEnableApplyAVFS_CKS_OFF_Voltage == 1) ? true : false;
+	}
+	return result;
+}
+
+
+/**
+* Initialize the ARB DRAM timing table's index field.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+static int polaris10_init_arb_table_index(struct pp_smumgr *smumgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	uint32_t tmp;
+	int result;
+
+	/* This is a read-modify-write on the first byte of the ARB table.
+	 * The first byte in the SMU73_Discrete_MCArbDramTimingTable structure
+	 * is the field 'current'.
+	 * This solution is ugly, but we never write the whole table only
+	 * individual fields in it.
+	 * In reality this field should not be in that structure
+	 * but in a soft register.
+	 */
+	result = smu7_read_smc_sram_dword(smumgr,
+			smu_data->smu7_data.arb_table_start, &tmp, SMC_RAM_END);
+
+	if (result)
+		return result;
+
+	tmp &= 0x00FFFFFF;
+	tmp |= ((uint32_t)MC_CG_ARB_FREQ_F1) << 24;
+
+	return smu7_write_smc_sram_dword(smumgr,
+			smu_data->smu7_data.arb_table_start, tmp, SMC_RAM_END);
+}
+
+static void polaris10_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	struct  phm_ppt_v1_information *table_info =
+			(struct  phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (table_info &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID <= POWERTUNE_DEFAULT_SET_MAX &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID)
+		smu_data->power_tune_defaults =
+				&polaris10_power_tune_data_set_array
+				[table_info->cac_dtp_table->usPowerTuneDataSetID - 1];
+	else
+		smu_data->power_tune_defaults = &polaris10_power_tune_data_set_array[0];
+
+}
+
+/**
+* Initializes the SMC table and uploads it
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always 0
+*/
+int polaris10_init_smc_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct smu7_hwmgr *hw_data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct SMU74_Discrete_DpmTable *table = &(smu_data->smc_state_table);
+	uint8_t i;
+	struct pp_atomctrl_gpio_pin_assignment gpio_pin;
+	pp_atomctrl_clock_dividers_vi dividers;
+
+	polaris10_initialize_power_tune_defaults(hwmgr);
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != hw_data->voltage_control)
+		polaris10_populate_smc_voltage_tables(hwmgr, table);
+
+	table->SystemFlags = 0;
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StepVddc))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+	if (hw_data->is_memory_gddr5)
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+	if (hw_data->ulv_supported && table_info->us_ulv_voltage_offset) {
+		result = polaris10_populate_ulv_state(hwmgr, table);
+		PP_ASSERT_WITH_CODE(0 == result,
+				"Failed to initialize ULV state!", return result);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+				ixCG_ULV_PARAMETER, SMU7_CGULVPARAMETER_DFLT);
+	}
+
+	result = polaris10_populate_smc_link_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Link Level!", return result);
+
+	result = polaris10_populate_all_graphic_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Graphics Level!", return result);
+
+	result = polaris10_populate_all_memory_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Memory Level!", return result);
+
+	result = polaris10_populate_smc_acpi_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize ACPI Level!", return result);
+
+	result = polaris10_populate_smc_vce_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize VCE Level!", return result);
+
+	result = polaris10_populate_smc_samu_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize SAMU Level!", return result);
+
+	/* Since only the initial state is completely set up at this point
+	 * (the other states are just copies of the boot state) we only
+	 * need to populate the  ARB settings for the initial state.
+	 */
+	result = polaris10_program_memory_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to Write ARB settings for the initial state.", return result);
+
+	result = polaris10_populate_smc_uvd_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize UVD Level!", return result);
+
+	result = polaris10_populate_smc_boot_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Boot Level!", return result);
+
+	result = polaris10_populate_smc_initailial_state(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to initialize Boot State!", return result);
+
+	result = polaris10_populate_bapm_parameters_in_dpm_table(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to populate BAPM Parameters!", return result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ClockStretcher)) {
+		result = polaris10_populate_clock_stretcher_data_table(hwmgr);
+		PP_ASSERT_WITH_CODE(0 == result,
+				"Failed to populate Clock Stretcher Data Table!",
+				return result);
+	}
+
+	result = polaris10_populate_avfs_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result, "Failed to populate AVFS Parameters!", return result;);
+
+	table->CurrSclkPllRange = 0xff;
+	table->GraphicsVoltageChangeEnable  = 1;
+	table->GraphicsThermThrottleEnable  = 1;
+	table->GraphicsInterval = 1;
+	table->VoltageInterval  = 1;
+	table->ThermalInterval  = 1;
+	table->TemperatureLimitHigh =
+			table_info->cac_dtp_table->usTargetOperatingTemp *
+			SMU7_Q88_FORMAT_CONVERSION_UNIT;
+	table->TemperatureLimitLow  =
+			(table_info->cac_dtp_table->usTargetOperatingTemp - 1) *
+			SMU7_Q88_FORMAT_CONVERSION_UNIT;
+	table->MemoryVoltageChangeEnable = 1;
+	table->MemoryInterval = 1;
+	table->VoltageResponseTime = 0;
+	table->PhaseResponseTime = 0;
+	table->MemoryThermThrottleEnable = 1;
+	table->PCIeBootLinkLevel = 0;
+	table->PCIeGenInterval = 1;
+	table->VRConfig = 0;
+
+	result = polaris10_populate_vr_config(hwmgr, table);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to populate VRConfig setting!", return result);
+
+	table->ThermGpio = 17;
+	table->SclkStepSize = 0x4000;
+
+	if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_VRHOT_GPIO_PINID, &gpio_pin)) {
+		table->VRHotGpio = gpio_pin.uc_gpio_pin_bit_shift;
+	} else {
+		table->VRHotGpio = SMU7_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_RegulatorHot);
+	}
+
+	if (atomctrl_get_pp_assign_pin(hwmgr, PP_AC_DC_SWITCH_GPIO_PINID,
+			&gpio_pin)) {
+		table->AcDcGpio = gpio_pin.uc_gpio_pin_bit_shift;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_AutomaticDCTransition);
+	} else {
+		table->AcDcGpio = SMU7_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_AutomaticDCTransition);
+	}
+
+	/* Thermal Output GPIO */
+	if (atomctrl_get_pp_assign_pin(hwmgr, THERMAL_INT_OUTPUT_GPIO_PINID,
+			&gpio_pin)) {
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ThermalOutGPIO);
+
+		table->ThermOutGpio = gpio_pin.uc_gpio_pin_bit_shift;
+
+		/* For porlarity read GPIOPAD_A with assigned Gpio pin
+		 * since VBIOS will program this register to set 'inactive state',
+		 * driver can then determine 'active state' from this and
+		 * program SMU with correct polarity
+		 */
+		table->ThermOutPolarity = (0 == (cgs_read_register(hwmgr->device, mmGPIOPAD_A)
+					& (1 << gpio_pin.uc_gpio_pin_bit_shift))) ? 1:0;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_ONLY;
+
+		/* if required, combine VRHot/PCC with thermal out GPIO */
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_RegulatorHot)
+		&& phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_CombinePCCWithThermalSignal))
+			table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_VRHOT;
+	} else {
+		table->ThermOutGpio = 17;
+		table->ThermOutPolarity = 1;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_DISABLE;
+	}
+
+	/* Populate BIF_SCLK levels into SMC DPM table */
+	for (i = 0; i <= hw_data->dpm_table.pcie_speed_table.count; i++) {
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr, smu_data->bif_sclk_table[i], &dividers);
+		PP_ASSERT_WITH_CODE((result == 0), "Can not find DFS divide id for Sclk", return result);
+
+		if (i == 0)
+			table->Ulv.BifSclkDfs = PP_HOST_TO_SMC_US((USHORT)(dividers.pll_post_divider));
+		else
+			table->LinkLevel[i-1].BifSclkDfs = PP_HOST_TO_SMC_US((USHORT)(dividers.pll_post_divider));
+	}
+
+	for (i = 0; i < SMU74_MAX_ENTRIES_SMIO; i++)
+		table->Smio[i] = PP_HOST_TO_SMC_UL(table->Smio[i]);
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VRConfig);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask1);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->CurrSclkPllRange);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
+	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
+	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
+
+	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr,
+			smu_data->smu7_data.dpm_table_start +
+			offsetof(SMU74_Discrete_DpmTable, SystemFlags),
+			(uint8_t *)&(table->SystemFlags),
+			sizeof(SMU74_Discrete_DpmTable) - 3 * sizeof(SMU74_PIDController),
+			SMC_RAM_END);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to upload dpm data to SMC memory!", return result);
+
+	result = polaris10_init_arb_table_index(hwmgr->smumgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to upload arb data to SMC memory!", return result);
+
+	result = polaris10_populate_pm_fuses(hwmgr);
+	PP_ASSERT_WITH_CODE(0 == result,
+			"Failed to  populate PM fuses to SMC memory!", return result);
+	return 0;
+}
+
+static int polaris10_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_OD_UPDATE_MCLK))
+		return polaris10_program_memory_timing_parameters(hwmgr);
+
+	return 0;
+}
+
+int polaris10_thermal_avfs_enable(struct pp_hwmgr *hwmgr)
+{
+	int ret;
+	struct pp_smumgr *smumgr = (struct pp_smumgr *)(hwmgr->smumgr);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (smu_data->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED)
+		return 0;
+
+	ret = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			PPSMC_MSG_SetGBDroopSettings, data->avfs_vdroop_override_setting);
+
+	ret = (smum_send_msg_to_smc(smumgr, PPSMC_MSG_EnableAvfs) == 0) ?
+			0 : -1;
+
+	if (!ret)
+		/* If this param is not changed, this function could fire unnecessarily */
+		smu_data->avfs.avfs_btc_status = AVFS_BTC_COMPLETED_PREVIOUSLY;
+
+	return ret;
+}
+
+/**
+* Set up the fan table to control the fan using the SMC.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int polaris10_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	SMU74_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
+	uint32_t duty100;
+	uint32_t t_diff1, t_diff2, pwm_diff1, pwm_diff2;
+	uint16_t fdo_min, slope1, slope2;
+	uint32_t reference_clock;
+	int res;
+	uint64_t tmp64;
+
+	if (smu_data->smu7_data.fan_table_start == 0) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+			CG_FDO_CTRL1, FMAX_DUTY100);
+
+	if (duty100 == 0) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	tmp64 = hwmgr->thermal_controller.advanceFanControlParameters.
+			usPWMMin * duty100;
+	do_div(tmp64, 10000);
+	fdo_min = (uint16_t)tmp64;
+
+	t_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usTMed -
+			hwmgr->thermal_controller.advanceFanControlParameters.usTMin;
+	t_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usTHigh -
+			hwmgr->thermal_controller.advanceFanControlParameters.usTMed;
+
+	pwm_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed -
+			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin;
+	pwm_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh -
+			hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed;
+
+	slope1 = (uint16_t)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
+	slope2 = (uint16_t)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
+
+	fan_table.TempMin = cpu_to_be16((50 + hwmgr->
+			thermal_controller.advanceFanControlParameters.usTMin) / 100);
+	fan_table.TempMed = cpu_to_be16((50 + hwmgr->
+			thermal_controller.advanceFanControlParameters.usTMed) / 100);
+	fan_table.TempMax = cpu_to_be16((50 + hwmgr->
+			thermal_controller.advanceFanControlParameters.usTMax) / 100);
+
+	fan_table.Slope1 = cpu_to_be16(slope1);
+	fan_table.Slope2 = cpu_to_be16(slope2);
+
+	fan_table.FdoMin = cpu_to_be16(fdo_min);
+
+	fan_table.HystDown = cpu_to_be16(hwmgr->
+			thermal_controller.advanceFanControlParameters.ucTHyst);
+
+	fan_table.HystUp = cpu_to_be16(1);
+
+	fan_table.HystSlope = cpu_to_be16(1);
+
+	fan_table.TempRespLim = cpu_to_be16(5);
+
+	reference_clock = smu7_get_xclk(hwmgr);
+
+	fan_table.RefreshPeriod = cpu_to_be32((hwmgr->
+			thermal_controller.advanceFanControlParameters.ulCycleDelay *
+			reference_clock) / 1600);
+
+	fan_table.FdoMax = cpu_to_be16((uint16_t)duty100);
+
+	fan_table.TempSrc = (uint8_t)PHM_READ_VFPF_INDIRECT_FIELD(
+			hwmgr->device, CGS_IND_REG__SMC,
+			CG_MULT_THERMAL_CTRL, TEMP_SEL);
+
+	res = smu7_copy_bytes_to_smc(hwmgr->smumgr, smu_data->smu7_data.fan_table_start,
+			(uint8_t *)&fan_table, (uint32_t)sizeof(fan_table),
+			SMC_RAM_END);
+
+	if (!res && hwmgr->thermal_controller.
+			advanceFanControlParameters.ucMinimumPWMLimit)
+		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetFanMinPwm,
+				hwmgr->thermal_controller.
+				advanceFanControlParameters.ucMinimumPWMLimit);
+
+	if (!res && hwmgr->thermal_controller.
+			advanceFanControlParameters.ulMinFanSCLKAcousticLimit)
+		res = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetFanSclkTarget,
+				hwmgr->thermal_controller.
+				advanceFanControlParameters.ulMinFanSCLKAcousticLimit);
+
+	if (res)
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+
+	return 0;
+}
+
+static int polaris10_update_uvd_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	smu_data->smc_state_table.UvdBootLevel = 0;
+	if (table_info->mm_dep_table->count > 0)
+		smu_data->smc_state_table.UvdBootLevel =
+				(uint8_t) (table_info->mm_dep_table->count - 1);
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start + offsetof(SMU74_Discrete_DpmTable,
+						UvdBootLevel);
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0x00FFFFFF;
+	mm_boot_level_value |= smu_data->smc_state_table.UvdBootLevel << 24;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_UVDDPM) ||
+		phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_UVDDPM_SetEnabledMask,
+				(uint32_t)(1 << smu_data->smc_state_table.UvdBootLevel));
+	return 0;
+}
+
+static int polaris10_update_vce_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_StablePState))
+		smu_data->smc_state_table.VceBootLevel =
+			(uint8_t) (table_info->mm_dep_table->count - 1);
+	else
+		smu_data->smc_state_table.VceBootLevel = 0;
+
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start +
+					offsetof(SMU74_Discrete_DpmTable, VceBootLevel);
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0xFF00FFFF;
+	mm_boot_level_value |= smu_data->smc_state_table.VceBootLevel << 16;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_VCEDPM_SetEnabledMask,
+				(uint32_t)1 << smu_data->smc_state_table.VceBootLevel);
+	return 0;
+}
+
+static int polaris10_update_samu_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+
+
+	smu_data->smc_state_table.SamuBootLevel = 0;
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU74_Discrete_DpmTable, SamuBootLevel);
+
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0xFFFFFF00;
+	mm_boot_level_value |= smu_data->smc_state_table.SamuBootLevel << 0;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SAMUDPM_SetEnabledMask,
+				(uint32_t)(1 << smu_data->smc_state_table.SamuBootLevel));
+	return 0;
+}
+
+
+static int polaris10_update_bif_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_pcie_table *pcie_table = table_info->pcie_table;
+	int max_entry, i;
+
+	max_entry = (SMU74_MAX_LEVELS_LINK < pcie_table->count) ?
+						SMU74_MAX_LEVELS_LINK :
+						pcie_table->count;
+	/* Setup BIF_SCLK levels */
+	for (i = 0; i < max_entry; i++)
+		smu_data->bif_sclk_table[i] = pcie_table->entries[i].pcie_sclk;
+	return 0;
+}
+
+int polaris10_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type)
+{
+	switch (type) {
+	case SMU_UVD_TABLE:
+		polaris10_update_uvd_smc_table(hwmgr);
+		break;
+	case SMU_VCE_TABLE:
+		polaris10_update_vce_smc_table(hwmgr);
+		break;
+	case SMU_SAMU_TABLE:
+		polaris10_update_samu_smc_table(hwmgr);
+		break;
+	case SMU_BIF_TABLE:
+		polaris10_update_bif_smc_table(hwmgr);
+	default:
+		break;
+	}
+	return 0;
+}
+
+int polaris10_update_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+
+	int result = 0;
+	uint32_t low_sclk_interrupt_threshold = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkThrottleLowNotification)
+		&& (hwmgr->gfx_arbiter.sclk_threshold !=
+				data->low_sclk_interrupt_threshold)) {
+		data->low_sclk_interrupt_threshold =
+				hwmgr->gfx_arbiter.sclk_threshold;
+		low_sclk_interrupt_threshold =
+				data->low_sclk_interrupt_threshold;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
+
+		result = smu7_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU74_Discrete_DpmTable,
+					LowSclkInterruptThreshold),
+				(uint8_t *)&low_sclk_interrupt_threshold,
+				sizeof(uint32_t),
+				SMC_RAM_END);
+	}
+	PP_ASSERT_WITH_CODE((result == 0),
+			"Failed to update SCLK threshold!", return result);
+
+	result = polaris10_program_mem_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE((result == 0),
+			"Failed to program memory timing parameters!",
+			);
+
+	return result;
+}
+
+uint32_t polaris10_get_offsetof(uint32_t type, uint32_t member)
+{
+	switch (type) {
+	case SMU_SoftRegisters:
+		switch (member) {
+		case HandshakeDisables:
+			return offsetof(SMU74_SoftRegisters, HandshakeDisables);
+		case VoltageChangeTimeout:
+			return offsetof(SMU74_SoftRegisters, VoltageChangeTimeout);
+		case AverageGraphicsActivity:
+			return offsetof(SMU74_SoftRegisters, AverageGraphicsActivity);
+		case PreVBlankGap:
+			return offsetof(SMU74_SoftRegisters, PreVBlankGap);
+		case VBlankTimeout:
+			return offsetof(SMU74_SoftRegisters, VBlankTimeout);
+		case UcodeLoadStatus:
+			return offsetof(SMU74_SoftRegisters, UcodeLoadStatus);
+		}
+	case SMU_Discrete_DpmTable:
+		switch (member) {
+		case UvdBootLevel:
+			return offsetof(SMU74_Discrete_DpmTable, UvdBootLevel);
+		case VceBootLevel:
+			return offsetof(SMU74_Discrete_DpmTable, VceBootLevel);
+		case SamuBootLevel:
+			return offsetof(SMU74_Discrete_DpmTable, SamuBootLevel);
+		case LowSclkInterruptThreshold:
+			return offsetof(SMU74_Discrete_DpmTable, LowSclkInterruptThreshold);
+		}
+	}
+	printk("cant't get the offset of type %x member %x \n", type, member);
+	return 0;
+}
+
+uint32_t polaris10_get_mac_definition(uint32_t value)
+{
+	switch (value) {
+	case SMU_MAX_LEVELS_GRAPHICS:
+		return SMU74_MAX_LEVELS_GRAPHICS;
+	case SMU_MAX_LEVELS_MEMORY:
+		return SMU74_MAX_LEVELS_MEMORY;
+	case SMU_MAX_LEVELS_LINK:
+		return SMU74_MAX_LEVELS_LINK;
+	case SMU_MAX_ENTRIES_SMIO:
+		return SMU74_MAX_ENTRIES_SMIO;
+	case SMU_MAX_LEVELS_VDDC:
+		return SMU74_MAX_LEVELS_VDDC;
+	case SMU_MAX_LEVELS_VDDGFX:
+		return SMU74_MAX_LEVELS_VDDGFX;
+	case SMU_MAX_LEVELS_VDDCI:
+		return SMU74_MAX_LEVELS_VDDCI;
+	case SMU_MAX_LEVELS_MVDD:
+		return SMU74_MAX_LEVELS_MVDD;
+	case SMU_UVD_MCLK_HANDSHAKE_DISABLE:
+		return SMU7_UVD_MCLK_HANDSHAKE_DISABLE;
+	}
+
+	printk("cant't get the mac of %x \n", value);
+	return 0;
+}
+
+/**
+* Get the location of various tables inside the FW image.
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   always  0
+*/
+int polaris10_process_firmware_header(struct pp_hwmgr *hwmgr)
+{
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t tmp;
+	int result;
+	bool error = false;
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU74_Firmware_Header, DpmTable),
+			&tmp, SMC_RAM_END);
+
+	if (0 == result)
+		smu_data->smu7_data.dpm_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU74_Firmware_Header, SoftRegisters),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.soft_regs_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU74_Firmware_Header, mcRegisterTable),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.mc_reg_table_start = tmp;
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU74_Firmware_Header, FanTable),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.fan_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU74_Firmware_Header, mcArbDramTimingTable),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.arb_table_start = tmp;
+
+	error |= (0 != result);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+			SMU7_FIRMWARE_HEADER_LOCATION +
+			offsetof(SMU74_Firmware_Header, Version),
+			&tmp, SMC_RAM_END);
+
+	if (!result)
+		hwmgr->microcode_version_info.SMC = tmp;
+
+	error |= (0 != result);
+
+	return error ? -1 : 0;
+}
+
+bool polaris10_is_dpm_running(struct pp_hwmgr *hwmgr)
+{
+	return (1 == PHM_READ_INDIRECT_FIELD(hwmgr->device,
+			CGS_IND_REG__SMC, FEATURE_STATUS, VOLTAGE_CONTROLLER_ON))
+			? true : false;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.h b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.h
new file mode 100644
index 000000000000..5ade3cea8bb7
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef POLARIS10_SMC_H
+#define POLARIS10_SMC_H
+
+#include "smumgr.h"
+
+
+int polaris10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr);
+int polaris10_populate_all_memory_levels(struct pp_hwmgr *hwmgr);
+int polaris10_init_smc_table(struct pp_hwmgr *hwmgr);
+int polaris10_thermal_setup_fan_table(struct pp_hwmgr *hwmgr);
+int polaris10_thermal_avfs_enable(struct pp_hwmgr *hwmgr);
+int polaris10_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type);
+int polaris10_update_sclk_threshold(struct pp_hwmgr *hwmgr);
+uint32_t polaris10_get_offsetof(uint32_t type, uint32_t member);
+uint32_t polaris10_get_mac_definition(uint32_t value);
+int polaris10_process_firmware_header(struct pp_hwmgr *hwmgr);
+bool polaris10_is_dpm_running(struct pp_hwmgr *hwmgr);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
index 5dba7c509710..5c3598ab7dae 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
@@ -38,16 +38,11 @@
 #include "ppatomctrl.h"
 #include "pp_debug.h"
 #include "cgs_common.h"
+#include "polaris10_smc.h"
+#include "smu7_ppsmc.h"
+#include "smu7_smumgr.h"
 
-#define POLARIS10_SMC_SIZE 0x20000
-#define VOLTAGE_SCALE 4
-
-/* Microcode file is stored in this buffer */
-#define BUFFER_SIZE                 80000
-#define MAX_STRING_SIZE             15
-#define BUFFER_SIZETWO              131072  /* 128 *1024 */
-
-#define SMC_RAM_END 0x40000
+#define PPPOLARIS10_TARGETACTIVITY_DFLT                     50
 
 static const SMU74_Discrete_GraphicsLevel avfs_graphics_level_polaris10[8] = {
 	/*  Min      pcie   DeepSleep Activity  CgSpll      CgSpll    CcPwr  CcPwr  Sclk         Enabled      Enabled                       Voltage    Power */
@@ -62,572 +57,9 @@ static const SMU74_Discrete_GraphicsLevel avfs_graphics_level_polaris10[8] = {
 	{ 0xa00fa446, 0x01, 0x00, 0x3200, 0, 0, 0, 0, 0, 0, 0x01, 0x01, 0x0a, 0x00, 0x00, 0x00, { 0xa0860100, 0x2800, 0, 0x2000, 2, 1, 0x0004, 0x0c02, 0xffff, 0x2700, 0x6433, 0x2100 } }
 };
 
-static const SMU74_Discrete_MemoryLevel avfs_memory_level_polaris10 =
-	{0x100ea446, 0, 0x30750000, 0x01, 0x01, 0x01, 0x00, 0x00, 0x64, 0x00, 0x00, 0x1f00, 0x00, 0x00};
-
-/**
-* Set the address for reading/writing the SMC SRAM space.
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    smcAddress the address in the SMC RAM to access.
-*/
-static int polaris10_set_smc_sram_address(struct pp_smumgr *smumgr, uint32_t smc_addr, uint32_t limit)
-{
-	PP_ASSERT_WITH_CODE((0 == (3 & smc_addr)), "SMC address must be 4 byte aligned.", return -EINVAL);
-	PP_ASSERT_WITH_CODE((limit > (smc_addr + 3)), "SMC addr is beyond the SMC RAM area.", return -EINVAL);
-
-	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_11, smc_addr);
-	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_11, 0);
-
-	return 0;
-}
-
-/**
-* Copy bytes from SMC RAM space into driver memory.
-*
-* @param    smumgr  the address of the powerplay SMU manager.
-* @param    smc_start_address the start address in the SMC RAM to copy bytes from
-* @param    src the byte array to copy the bytes to.
-* @param    byte_count the number of bytes to copy.
-*/
-int polaris10_copy_bytes_from_smc(struct pp_smumgr *smumgr, uint32_t smc_start_address, uint32_t *dest, uint32_t byte_count, uint32_t limit)
-{
-	uint32_t data;
-	uint32_t addr;
-	uint8_t *dest_byte;
-	uint8_t i, data_byte[4] = {0};
-	uint32_t *pdata = (uint32_t *)&data_byte;
-
-	PP_ASSERT_WITH_CODE((0 == (3 & smc_start_address)), "SMC address must be 4 byte aligned.", return -1;);
-	PP_ASSERT_WITH_CODE((limit > (smc_start_address + byte_count)), "SMC address is beyond the SMC RAM area.", return -1);
-
-	addr = smc_start_address;
-
-	while (byte_count >= 4) {
-		polaris10_read_smc_sram_dword(smumgr, addr, &data, limit);
-
-		*dest = PP_SMC_TO_HOST_UL(data);
-
-		dest += 1;
-		byte_count -= 4;
-		addr += 4;
-	}
-
-	if (byte_count) {
-		polaris10_read_smc_sram_dword(smumgr, addr, &data, limit);
-		*pdata = PP_SMC_TO_HOST_UL(data);
-	/* Cast dest into byte type in dest_byte.  This way, we don't overflow if the allocated memory is not 4-byte aligned. */
-		dest_byte = (uint8_t *)dest;
-		for (i = 0; i < byte_count; i++)
-			dest_byte[i] = data_byte[i];
-	}
-
-	return 0;
-}
-
-/**
-* Copy bytes from an array into the SMC RAM space.
-*
-* @param    pSmuMgr  the address of the powerplay SMU manager.
-* @param    smc_start_address the start address in the SMC RAM to copy bytes to.
-* @param    src the byte array to copy the bytes from.
-* @param    byte_count the number of bytes to copy.
-*/
-int polaris10_copy_bytes_to_smc(struct pp_smumgr *smumgr, uint32_t smc_start_address,
-				const uint8_t *src, uint32_t byte_count, uint32_t limit)
-{
-	int result;
-	uint32_t data = 0;
-	uint32_t original_data;
-	uint32_t addr = 0;
-	uint32_t extra_shift;
-
-	PP_ASSERT_WITH_CODE((0 == (3 & smc_start_address)), "SMC address must be 4 byte aligned.", return -1);
-	PP_ASSERT_WITH_CODE((limit > (smc_start_address + byte_count)), "SMC address is beyond the SMC RAM area.", return -1);
-
-	addr = smc_start_address;
-
-	while (byte_count >= 4) {
-	/* Bytes are written into the SMC addres space with the MSB first. */
-		data = src[0] * 0x1000000 + src[1] * 0x10000 + src[2] * 0x100 + src[3];
-
-		result = polaris10_set_smc_sram_address(smumgr, addr, limit);
-
-		if (0 != result)
-			return result;
-
-		cgs_write_register(smumgr->device, mmSMC_IND_DATA_11, data);
-
-		src += 4;
-		byte_count -= 4;
-		addr += 4;
-	}
-
-	if (0 != byte_count) {
-
-		data = 0;
-
-		result = polaris10_set_smc_sram_address(smumgr, addr, limit);
-
-		if (0 != result)
-			return result;
-
-
-		original_data = cgs_read_register(smumgr->device, mmSMC_IND_DATA_11);
-
-		extra_shift = 8 * (4 - byte_count);
-
-		while (byte_count > 0) {
-			/* Bytes are written into the SMC addres space with the MSB first. */
-			data = (0x100 * data) + *src++;
-			byte_count--;
-		}
-
-		data <<= extra_shift;
-
-		data |= (original_data & ~((~0UL) << extra_shift));
-
-		result = polaris10_set_smc_sram_address(smumgr, addr, limit);
-
-		if (0 != result)
-			return result;
-
-		cgs_write_register(smumgr->device, mmSMC_IND_DATA_11, data);
-	}
-
-	return 0;
-}
-
-
-static int polaris10_program_jump_on_start(struct pp_smumgr *smumgr)
-{
-	static const unsigned char data[4] = { 0xE0, 0x00, 0x80, 0x40 };
-
-	polaris10_copy_bytes_to_smc(smumgr, 0x0, data, 4, sizeof(data)+1);
-
-	return 0;
-}
-
-/**
-* Return if the SMC is currently running.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-*/
-bool polaris10_is_smc_ram_running(struct pp_smumgr *smumgr)
-{
-	return ((0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC, SMC_SYSCON_CLOCK_CNTL_0, ck_disable))
-	&& (0x20100 <= cgs_read_ind_register(smumgr->device, CGS_IND_REG__SMC, ixSMC_PC_C)));
-}
-
-static bool polaris10_is_hw_avfs_present(struct pp_smumgr *smumgr)
-{
-	uint32_t efuse;
-
-	efuse = cgs_read_ind_register(smumgr->device, CGS_IND_REG__SMC, ixSMU_EFUSE_0 + (49*4));
-	efuse &= 0x00000001;
-	if (efuse)
-		return true;
-
-	return false;
-}
-
-/**
-* Send a message to the SMC, and wait for its response.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    msg the message to send.
-* @return   The response that came from the SMC.
-*/
-int polaris10_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
-{
-	int ret;
-
-	if (!polaris10_is_smc_ram_running(smumgr))
-		return -1;
-
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-
-	ret = SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP);
-
-	if (ret != 1)
-		printk("\n failed to send pre message %x ret is %d \n",  msg, ret);
-
-	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-
-	ret = SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP);
-
-	if (ret != 1)
-		printk("\n failed to send message %x ret is %d \n",  msg, ret);
-
-	return 0;
-}
-
-
-/**
-* Send a message to the SMC, and do not wait for its response.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    msg the message to send.
-* @return   Always return 0.
-*/
-int polaris10_send_msg_to_smc_without_waiting(struct pp_smumgr *smumgr, uint16_t msg)
-{
-	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
-
-	return 0;
-}
-
-/**
-* Send a message to the SMC with parameter
-*
-* @param    smumgr:  the address of the powerplay hardware manager.
-* @param    msg: the message to send.
-* @param    parameter: the parameter to send
-* @return   The response that came from the SMC.
-*/
-int polaris10_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr, uint16_t msg, uint32_t parameter)
-{
-	if (!polaris10_is_smc_ram_running(smumgr)) {
-		return -1;
-	}
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-
-	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
-
-	return polaris10_send_msg_to_smc(smumgr, msg);
-}
-
-
-/**
-* Send a message to the SMC with parameter, do not wait for response
-*
-* @param    smumgr:  the address of the powerplay hardware manager.
-* @param    msg: the message to send.
-* @param    parameter: the parameter to send
-* @return   The response that came from the SMC.
-*/
-int polaris10_send_msg_to_smc_with_parameter_without_waiting(struct pp_smumgr *smumgr, uint16_t msg, uint32_t parameter)
-{
-	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
-
-	return polaris10_send_msg_to_smc_without_waiting(smumgr, msg);
-}
-
-int polaris10_send_msg_to_smc_offset(struct pp_smumgr *smumgr)
-{
-	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, 0x20000);
-
-	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, PPSMC_MSG_Test);
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-
-	if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP))
-		printk("Failed to send Message.\n");
-
-	return 0;
-}
-
-/**
-* Wait until the SMC is doing nithing. Doing nothing means that the SMC is either turned off or it is sitting on the STOP instruction.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    msg the message to send.
-* @return   The response that came from the SMC.
-*/
-int polaris10_wait_for_smc_inactive(struct pp_smumgr *smumgr)
-{
-	/* If the SMC is not even on it qualifies as inactive. */
-	if (!polaris10_is_smc_ram_running(smumgr))
-		return -1;
-
-	SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, SMC_IND, SMC_SYSCON_CLOCK_CNTL_0, cken, 0);
-	return 0;
-}
-
-
-/**
-* Upload the SMC firmware to the SMC microcontroller.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    pFirmware the data structure containing the various sections of the firmware.
-*/
-static int polaris10_upload_smc_firmware_data(struct pp_smumgr *smumgr, uint32_t length, uint32_t *src, uint32_t limit)
-{
-	uint32_t byte_count = length;
-
-	PP_ASSERT_WITH_CODE((limit >= byte_count), "SMC address is beyond the SMC RAM area.", return -1);
+static const SMU74_Discrete_MemoryLevel avfs_memory_level_polaris10 = {
+	0x100ea446, 0, 0x30750000, 0x01, 0x01, 0x01, 0x00, 0x00, 0x64, 0x00, 0x00, 0x1f00, 0x00, 0x00};
 
-	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_11, 0x20000);
-	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_11, 1);
-
-	for (; byte_count >= 4; byte_count -= 4)
-		cgs_write_register(smumgr->device, mmSMC_IND_DATA_11, *src++);
-
-	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_11, 0);
-
-	PP_ASSERT_WITH_CODE((0 == byte_count), "SMC size must be dividable by 4.", return -1);
-
-	return 0;
-}
-
-static enum cgs_ucode_id polaris10_convert_fw_type_to_cgs(uint32_t fw_type)
-{
-	enum cgs_ucode_id result = CGS_UCODE_ID_MAXIMUM;
-
-	switch (fw_type) {
-	case UCODE_ID_SMU:
-		result = CGS_UCODE_ID_SMU;
-		break;
-	case UCODE_ID_SMU_SK:
-		result = CGS_UCODE_ID_SMU_SK;
-		break;
-	case UCODE_ID_SDMA0:
-		result = CGS_UCODE_ID_SDMA0;
-		break;
-	case UCODE_ID_SDMA1:
-		result = CGS_UCODE_ID_SDMA1;
-		break;
-	case UCODE_ID_CP_CE:
-		result = CGS_UCODE_ID_CP_CE;
-		break;
-	case UCODE_ID_CP_PFP:
-		result = CGS_UCODE_ID_CP_PFP;
-		break;
-	case UCODE_ID_CP_ME:
-		result = CGS_UCODE_ID_CP_ME;
-		break;
-	case UCODE_ID_CP_MEC:
-		result = CGS_UCODE_ID_CP_MEC;
-		break;
-	case UCODE_ID_CP_MEC_JT1:
-		result = CGS_UCODE_ID_CP_MEC_JT1;
-		break;
-	case UCODE_ID_CP_MEC_JT2:
-		result = CGS_UCODE_ID_CP_MEC_JT2;
-		break;
-	case UCODE_ID_RLC_G:
-		result = CGS_UCODE_ID_RLC_G;
-		break;
-	default:
-		break;
-	}
-
-	return result;
-}
-
-static int polaris10_upload_smu_firmware_image(struct pp_smumgr *smumgr)
-{
-	int result = 0;
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
-
-	struct cgs_firmware_info info = {0};
-
-	if (smu_data->security_hard_key == 1)
-		cgs_get_firmware_info(smumgr->device,
-			polaris10_convert_fw_type_to_cgs(UCODE_ID_SMU), &info);
-	else
-		cgs_get_firmware_info(smumgr->device,
-			polaris10_convert_fw_type_to_cgs(UCODE_ID_SMU_SK), &info);
-
-	/* TO DO cgs_init_samu_load_smu(smumgr->device, (uint32_t *)info.kptr, info.image_size, smu_data->post_initial_boot);*/
-	result = polaris10_upload_smc_firmware_data(smumgr, info.image_size, (uint32_t *)info.kptr, POLARIS10_SMC_SIZE);
-
-	return result;
-}
-
-/**
-* Read a 32bit value from the SMC SRAM space.
-* ALL PARAMETERS ARE IN HOST BYTE ORDER.
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    smcAddress the address in the SMC RAM to access.
-* @param    value and output parameter for the data read from the SMC SRAM.
-*/
-int polaris10_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr, uint32_t *value, uint32_t limit)
-{
-	int result;
-
-	result = polaris10_set_smc_sram_address(smumgr, smc_addr, limit);
-
-	if (result)
-		return result;
-
-	*value = cgs_read_register(smumgr->device, mmSMC_IND_DATA_11);
-	return 0;
-}
-
-/**
-* Write a 32bit value to the SMC SRAM space.
-* ALL PARAMETERS ARE IN HOST BYTE ORDER.
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    smc_addr the address in the SMC RAM to access.
-* @param    value to write to the SMC SRAM.
-*/
-int polaris10_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr, uint32_t value, uint32_t limit)
-{
-	int result;
-
-	result = polaris10_set_smc_sram_address(smumgr, smc_addr, limit);
-
-	if (result)
-		return result;
-
-	cgs_write_register(smumgr->device, mmSMC_IND_DATA_11, value);
-
-	return 0;
-}
-
-
-int polaris10_smu_fini(struct pp_smumgr *smumgr)
-{
-	if (smumgr->backend) {
-		kfree(smumgr->backend);
-		smumgr->backend = NULL;
-	}
-	cgs_rel_firmware(smumgr->device, CGS_UCODE_ID_SMU);
-	return 0;
-}
-
-/* Convert the firmware type to SMU type mask. For MEC, we need to check all MEC related type */
-static uint32_t polaris10_get_mask_for_firmware_type(uint32_t fw_type)
-{
-	uint32_t result = 0;
-
-	switch (fw_type) {
-	case UCODE_ID_SDMA0:
-		result = UCODE_ID_SDMA0_MASK;
-		break;
-	case UCODE_ID_SDMA1:
-		result = UCODE_ID_SDMA1_MASK;
-		break;
-	case UCODE_ID_CP_CE:
-		result = UCODE_ID_CP_CE_MASK;
-		break;
-	case UCODE_ID_CP_PFP:
-		result = UCODE_ID_CP_PFP_MASK;
-		break;
-	case UCODE_ID_CP_ME:
-		result = UCODE_ID_CP_ME_MASK;
-		break;
-	case UCODE_ID_CP_MEC_JT1:
-	case UCODE_ID_CP_MEC_JT2:
-		result = UCODE_ID_CP_MEC_MASK;
-		break;
-	case UCODE_ID_RLC_G:
-		result = UCODE_ID_RLC_G_MASK;
-		break;
-	default:
-		printk("UCode type is out of range! \n");
-		result = 0;
-	}
-
-	return result;
-}
-
-/* Populate one firmware image to the data structure */
-
-static int polaris10_populate_single_firmware_entry(struct pp_smumgr *smumgr,
-						uint32_t fw_type,
-						struct SMU_Entry *entry)
-{
-	int result = 0;
-	struct cgs_firmware_info info = {0};
-
-	result = cgs_get_firmware_info(smumgr->device,
-				polaris10_convert_fw_type_to_cgs(fw_type),
-				&info);
-
-	if (!result) {
-		entry->version = info.version;
-		entry->id = (uint16_t)fw_type;
-		entry->image_addr_high = smu_upper_32_bits(info.mc_addr);
-		entry->image_addr_low = smu_lower_32_bits(info.mc_addr);
-		entry->meta_data_addr_high = 0;
-		entry->meta_data_addr_low = 0;
-		entry->data_size_byte = info.image_size;
-		entry->num_register_entries = 0;
-	}
-
-	if (fw_type == UCODE_ID_RLC_G)
-		entry->flags = 1;
-	else
-		entry->flags = 0;
-
-	return 0;
-}
-
-static int polaris10_request_smu_load_fw(struct pp_smumgr *smumgr)
-{
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
-	uint32_t fw_to_load;
-
-	int result = 0;
-	struct SMU_DRAMData_TOC *toc;
-
-	if (!smumgr->reload_fw) {
-		printk(KERN_INFO "[ powerplay ] skip reloading...\n");
-		return 0;
-	}
-
-	if (smu_data->soft_regs_start)
-		cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
-					smu_data->soft_regs_start + offsetof(SMU74_SoftRegisters, UcodeLoadStatus),
-					0x0);
-
-	polaris10_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_SMU_DRAM_ADDR_HI, smu_data->smu_buffer.mc_addr_high);
-	polaris10_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_SMU_DRAM_ADDR_LO, smu_data->smu_buffer.mc_addr_low);
-
-	toc = (struct SMU_DRAMData_TOC *)smu_data->header;
-	toc->num_entries = 0;
-	toc->structure_version = 1;
-
-	PP_ASSERT_WITH_CODE(0 == polaris10_populate_single_firmware_entry(smumgr, UCODE_ID_RLC_G, &toc->entry[toc->num_entries++]), "Failed to Get Firmware Entry.", return -1);
-	PP_ASSERT_WITH_CODE(0 == polaris10_populate_single_firmware_entry(smumgr, UCODE_ID_CP_CE, &toc->entry[toc->num_entries++]), "Failed to Get Firmware Entry.", return -1);
-	PP_ASSERT_WITH_CODE(0 == polaris10_populate_single_firmware_entry(smumgr, UCODE_ID_CP_PFP, &toc->entry[toc->num_entries++]), "Failed to Get Firmware Entry.", return -1);
-	PP_ASSERT_WITH_CODE(0 == polaris10_populate_single_firmware_entry(smumgr, UCODE_ID_CP_ME, &toc->entry[toc->num_entries++]), "Failed to Get Firmware Entry.", return -1);
-	PP_ASSERT_WITH_CODE(0 == polaris10_populate_single_firmware_entry(smumgr, UCODE_ID_CP_MEC, &toc->entry[toc->num_entries++]), "Failed to Get Firmware Entry.", return -1);
-	PP_ASSERT_WITH_CODE(0 == polaris10_populate_single_firmware_entry(smumgr, UCODE_ID_CP_MEC_JT1, &toc->entry[toc->num_entries++]), "Failed to Get Firmware Entry.", return -1);
-	PP_ASSERT_WITH_CODE(0 == polaris10_populate_single_firmware_entry(smumgr, UCODE_ID_CP_MEC_JT2, &toc->entry[toc->num_entries++]), "Failed to Get Firmware Entry.", return -1);
-	PP_ASSERT_WITH_CODE(0 == polaris10_populate_single_firmware_entry(smumgr, UCODE_ID_SDMA0, &toc->entry[toc->num_entries++]), "Failed to Get Firmware Entry.", return -1);
-	PP_ASSERT_WITH_CODE(0 == polaris10_populate_single_firmware_entry(smumgr, UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]), "Failed to Get Firmware Entry.", return -1);
-
-	polaris10_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_DRV_DRAM_ADDR_HI, smu_data->header_buffer.mc_addr_high);
-	polaris10_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_DRV_DRAM_ADDR_LO, smu_data->header_buffer.mc_addr_low);
-
-	fw_to_load = UCODE_ID_RLC_G_MASK
-		   + UCODE_ID_SDMA0_MASK
-		   + UCODE_ID_SDMA1_MASK
-		   + UCODE_ID_CP_CE_MASK
-		   + UCODE_ID_CP_ME_MASK
-		   + UCODE_ID_CP_PFP_MASK
-		   + UCODE_ID_CP_MEC_MASK;
-
-	if (polaris10_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_LoadUcodes, fw_to_load))
-		printk(KERN_ERR "Fail to Request SMU Load uCode");
-
-	return result;
-}
-
-/* Check if the FW has been loaded, SMU will not return if loading has not finished. */
-static int polaris10_check_fw_load_finish(struct pp_smumgr *smumgr, uint32_t fw_type)
-{
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
-	uint32_t fw_mask = polaris10_get_mask_for_firmware_type(fw_type);
-	uint32_t ret;
-	/* Check SOFT_REGISTERS_TABLE_28.UcodeLoadStatus */
-	ret = smum_wait_on_indirect_register(smumgr, mmSMC_IND_INDEX_11,
-					smu_data->soft_regs_start + offsetof(SMU74_SoftRegisters, UcodeLoadStatus),
-					fw_mask, fw_mask);
-
-	return ret;
-}
-
-static int polaris10_reload_firmware(struct pp_smumgr *smumgr)
-{
-	return smumgr->smumgr_funcs->start_smu(smumgr);
-}
 
 static int polaris10_setup_pwr_virus(struct pp_smumgr *smumgr)
 {
@@ -669,7 +101,7 @@ static int polaris10_perform_btc(struct pp_smumgr *smumgr)
 	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
 
 	if (0 != smu_data->avfs.avfs_btc_param) {
-		if (0 != polaris10_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_PerformBtc, smu_data->avfs.avfs_btc_param)) {
+		if (0 != smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_PerformBtc, smu_data->avfs.avfs_btc_param)) {
 			printk("[AVFS][SmuPolaris10_PerformBtc] PerformBTC SMU msg failed");
 			result = -1;
 		}
@@ -697,7 +129,7 @@ int polaris10_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 	graphics_level_size = sizeof(avfs_graphics_level_polaris10);
 	u16_boot_mvdd = PP_HOST_TO_SMC_US(1300 * VOLTAGE_SCALE);
 
-	PP_ASSERT_WITH_CODE(0 == polaris10_read_smc_sram_dword(smumgr,
+	PP_ASSERT_WITH_CODE(0 == smu7_read_smc_sram_dword(smumgr,
 				SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU74_Firmware_Header, DpmTable),
 				&dpm_table_start, 0x40000),
 			"[AVFS][Polaris10_SetupGfxLvlStruct] SMU could not communicate starting address of DPM table",
@@ -708,14 +140,14 @@ int polaris10_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 
 	vr_config_address = dpm_table_start + offsetof(SMU74_Discrete_DpmTable, VRConfig);
 
-	PP_ASSERT_WITH_CODE(0 == polaris10_copy_bytes_to_smc(smumgr, vr_config_address,
+	PP_ASSERT_WITH_CODE(0 == smu7_copy_bytes_to_smc(smumgr, vr_config_address,
 				(uint8_t *)&vr_config, sizeof(uint32_t), 0x40000),
 			"[AVFS][Polaris10_SetupGfxLvlStruct] Problems copying VRConfig value over to SMC",
 			return -1);
 
 	graphics_level_address = dpm_table_start + offsetof(SMU74_Discrete_DpmTable, GraphicsLevel);
 
-	PP_ASSERT_WITH_CODE(0 == polaris10_copy_bytes_to_smc(smumgr, graphics_level_address,
+	PP_ASSERT_WITH_CODE(0 == smu7_copy_bytes_to_smc(smumgr, graphics_level_address,
 				(uint8_t *)(&avfs_graphics_level_polaris10),
 				graphics_level_size, 0x40000),
 			"[AVFS][Polaris10_SetupGfxLvlStruct] Copying of SCLK DPM table failed!",
@@ -723,7 +155,7 @@ int polaris10_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 
 	graphics_level_address = dpm_table_start + offsetof(SMU74_Discrete_DpmTable, MemoryLevel);
 
-	PP_ASSERT_WITH_CODE(0 == polaris10_copy_bytes_to_smc(smumgr, graphics_level_address,
+	PP_ASSERT_WITH_CODE(0 == smu7_copy_bytes_to_smc(smumgr, graphics_level_address,
 				(uint8_t *)(&avfs_memory_level_polaris10), sizeof(avfs_memory_level_polaris10), 0x40000),
 				"[AVFS][Polaris10_SetupGfxLvlStruct] Copying of MCLK DPM table failed!",
 			return -1);
@@ -732,7 +164,7 @@ int polaris10_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 
 	graphics_level_address = dpm_table_start + offsetof(SMU74_Discrete_DpmTable, BootMVdd);
 
-	PP_ASSERT_WITH_CODE(0 == polaris10_copy_bytes_to_smc(smumgr, graphics_level_address,
+	PP_ASSERT_WITH_CODE(0 == smu7_copy_bytes_to_smc(smumgr, graphics_level_address,
 			(uint8_t *)(&u16_boot_mvdd), sizeof(u16_boot_mvdd), 0x40000),
 			"[AVFS][Polaris10_SetupGfxLvlStruct] Copying of DPM table failed!",
 			return -1);
@@ -793,7 +225,7 @@ static int polaris10_start_smu_in_protection_mode(struct pp_smumgr *smumgr)
 	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
 					SMC_SYSCON_RESET_CNTL, rst_reg, 1);
 
-	result = polaris10_upload_smu_firmware_image(smumgr);
+	result = smu7_upload_smu_firmware_image(smumgr);
 	if (result != 0)
 		return result;
 
@@ -812,7 +244,7 @@ static int polaris10_start_smu_in_protection_mode(struct pp_smumgr *smumgr)
 
 
 	/* Call Test SMU message with 0x20000 offset to trigger SMU start */
-	polaris10_send_msg_to_smc_offset(smumgr);
+	smu7_send_msg_to_smc_offset(smumgr);
 
 	/* Wait done bit to be set */
 	/* Check pass/failed indicator */
@@ -853,12 +285,12 @@ static int polaris10_start_smu_in_non_protection_mode(struct pp_smumgr *smumgr)
 					SMC_SYSCON_RESET_CNTL,
 					rst_reg, 1);
 
-	result = polaris10_upload_smu_firmware_image(smumgr);
+	result = smu7_upload_smu_firmware_image(smumgr);
 	if (result != 0)
 		return result;
 
 	/* Set smc instruct start point at 0x0 */
-	polaris10_program_jump_on_start(smumgr);
+	smu7_program_jump_on_start(smumgr);
 
 	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
 					SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
@@ -881,10 +313,10 @@ static int polaris10_start_smu(struct pp_smumgr *smumgr)
 	bool SMU_VFT_INTACT;
 
 	/* Only start SMC if SMC RAM is not running */
-	if (!polaris10_is_smc_ram_running(smumgr)) {
+	if (!smu7_is_smc_ram_running(smumgr)) {
 		SMU_VFT_INTACT = false;
 		smu_data->protected_mode = (uint8_t) (SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC, SMU_FIRMWARE, SMU_MODE));
-		smu_data->security_hard_key = (uint8_t) (SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC, SMU_FIRMWARE, SMU_SEL));
+		smu_data->smu7_data.security_hard_key = (uint8_t) (SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC, SMU_FIRMWARE, SMU_SEL));
 
 		/* Check if SMU is running in protected mode */
 		if (smu_data->protected_mode == 0) {
@@ -894,7 +326,7 @@ static int polaris10_start_smu(struct pp_smumgr *smumgr)
 
 			/* If failed, try with different security Key. */
 			if (result != 0) {
-				smu_data->security_hard_key ^= 1;
+				smu_data->smu7_data.security_hard_key ^= 1;
 				result = polaris10_start_smu_in_protection_mode(smumgr);
 			}
 		}
@@ -906,89 +338,69 @@ static int polaris10_start_smu(struct pp_smumgr *smumgr)
 	} else
 		SMU_VFT_INTACT = true; /*Driver went offline but SMU was still alive and contains the VFT table */
 
-	smu_data->post_initial_boot = true;
 	polaris10_avfs_event_mgr(smumgr, SMU_VFT_INTACT);
 	/* Setup SoftRegsStart here for register lookup in case DummyBackEnd is used and ProcessFirmwareHeader is not executed */
-	polaris10_read_smc_sram_dword(smumgr, SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU74_Firmware_Header, SoftRegisters),
-					&(smu_data->soft_regs_start), 0x40000);
+	smu7_read_smc_sram_dword(smumgr, SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU74_Firmware_Header, SoftRegisters),
+					&(smu_data->smu7_data.soft_regs_start), 0x40000);
 
-	result = polaris10_request_smu_load_fw(smumgr);
+	result = smu7_request_smu_load_fw(smumgr);
 
 	return result;
 }
 
+static bool polaris10_is_hw_avfs_present(struct pp_smumgr *smumgr)
+{
+	uint32_t efuse;
+
+	efuse = cgs_read_ind_register(smumgr->device, CGS_IND_REG__SMC, ixSMU_EFUSE_0 + (49*4));
+	efuse &= 0x00000001;
+	if (efuse)
+		return true;
+
+	return false;
+}
+
 static int polaris10_smu_init(struct pp_smumgr *smumgr)
 {
-	struct polaris10_smumgr *smu_data;
-	uint8_t *internal_buf;
-	uint64_t mc_addr = 0;
-	/* Allocate memory for backend private data */
-	smu_data = (struct polaris10_smumgr *)(smumgr->backend);
-	smu_data->header_buffer.data_size =
-		((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
-	smu_data->smu_buffer.data_size = 200*4096;
-	smu_data->avfs.avfs_btc_status = AVFS_BTC_NOTSUPPORTED;
-/* Allocate FW image data structure and header buffer and
- * send the header buffer address to SMU */
-	smu_allocate_memory(smumgr->device,
-		smu_data->header_buffer.data_size,
-		CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
-		PAGE_SIZE,
-		&mc_addr,
-		&smu_data->header_buffer.kaddr,
-		&smu_data->header_buffer.handle);
-
-	smu_data->header = smu_data->header_buffer.kaddr;
-	smu_data->header_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
-	smu_data->header_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
-
-	PP_ASSERT_WITH_CODE((NULL != smu_data->header),
-		"Out of memory.",
-		kfree(smumgr->backend);
-		cgs_free_gpu_mem(smumgr->device,
-		(cgs_handle_t)smu_data->header_buffer.handle);
-		return -1);
+	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	int i;
 
-/* Allocate buffer for SMU internal buffer and send the address to SMU.
- * Iceland SMU does not need internal buffer.*/
-	smu_allocate_memory(smumgr->device,
-		smu_data->smu_buffer.data_size,
-		CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
-		PAGE_SIZE,
-		&mc_addr,
-		&smu_data->smu_buffer.kaddr,
-		&smu_data->smu_buffer.handle);
-
-	internal_buf = smu_data->smu_buffer.kaddr;
-	smu_data->smu_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
-	smu_data->smu_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
-
-	PP_ASSERT_WITH_CODE((NULL != internal_buf),
-		"Out of memory.",
-		kfree(smumgr->backend);
-		cgs_free_gpu_mem(smumgr->device,
-		(cgs_handle_t)smu_data->smu_buffer.handle);
-		return -1;);
+	if (smu7_init(smumgr))
+		return -EINVAL;
 
 	if (polaris10_is_hw_avfs_present(smumgr))
 		smu_data->avfs.avfs_btc_status = AVFS_BTC_BOOT;
 	else
 		smu_data->avfs.avfs_btc_status = AVFS_BTC_NOTSUPPORTED;
 
+	for (i = 0; i < SMU74_MAX_LEVELS_GRAPHICS; i++)
+		smu_data->activity_target[i] = PPPOLARIS10_TARGETACTIVITY_DFLT;
+
 	return 0;
 }
 
-static const struct pp_smumgr_func ellsemere_smu_funcs = {
+static const struct pp_smumgr_func polaris10_smu_funcs = {
 	.smu_init = polaris10_smu_init,
-	.smu_fini = polaris10_smu_fini,
+	.smu_fini = smu7_smu_fini,
 	.start_smu = polaris10_start_smu,
-	.check_fw_load_finish = polaris10_check_fw_load_finish,
-	.request_smu_load_fw = polaris10_reload_firmware,
+	.check_fw_load_finish = smu7_check_fw_load_finish,
+	.request_smu_load_fw = smu7_reload_firmware,
 	.request_smu_load_specific_fw = NULL,
-	.send_msg_to_smc = polaris10_send_msg_to_smc,
-	.send_msg_to_smc_with_parameter = polaris10_send_msg_to_smc_with_parameter,
+	.send_msg_to_smc = smu7_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = smu7_send_msg_to_smc_with_parameter,
 	.download_pptable_settings = NULL,
 	.upload_pptable_settings = NULL,
+	.update_smc_table = polaris10_update_smc_table,
+	.get_offsetof = polaris10_get_offsetof,
+	.process_firmware_header = polaris10_process_firmware_header,
+	.init_smc_table = polaris10_init_smc_table,
+	.update_sclk_threshold = polaris10_update_sclk_threshold,
+	.thermal_avfs_enable = polaris10_thermal_avfs_enable,
+	.thermal_setup_fan_table = polaris10_thermal_setup_fan_table,
+	.populate_all_graphic_levels = polaris10_populate_all_graphic_levels,
+	.populate_all_memory_levels = polaris10_populate_all_memory_levels,
+	.get_mac_definition = polaris10_get_mac_definition,
+	.is_dpm_running = polaris10_is_dpm_running,
 };
 
 int polaris10_smum_init(struct pp_smumgr *smumgr)
@@ -998,10 +410,10 @@ int polaris10_smum_init(struct pp_smumgr *smumgr)
 	polaris10_smu = kzalloc(sizeof(struct polaris10_smumgr), GFP_KERNEL);
 
 	if (polaris10_smu == NULL)
-		return -1;
+		return -EINVAL;
 
 	smumgr->backend = polaris10_smu;
-	smumgr->smumgr_funcs = &ellsemere_smu_funcs;
+	smumgr->smumgr_funcs = &polaris10_smu_funcs;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h
index e5377aec057f..49ebf1d5a53c 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h
@@ -24,45 +24,52 @@
 #ifndef _POLARIS10_SMUMANAGER_H
 #define _POLARIS10_SMUMANAGER_H
 
-#include <polaris10_ppsmc.h>
+
 #include <pp_endian.h>
+#include "smu74.h"
+#include "smu74_discrete.h"
+#include "smu7_smumgr.h"
+
+#define SMC_RAM_END 0x40000
 
 struct polaris10_avfs {
 	enum AVFS_BTC_STATUS avfs_btc_status;
 	uint32_t           avfs_btc_param;
 };
 
-struct polaris10_buffer_entry {
-	uint32_t data_size;
-	uint32_t mc_addr_low;
-	uint32_t mc_addr_high;
-	void *kaddr;
-	unsigned long  handle;
+struct polaris10_pt_defaults {
+	uint8_t   SviLoadLineEn;
+	uint8_t   SviLoadLineVddC;
+	uint8_t   TDC_VDDC_ThrottleReleaseLimitPerc;
+	uint8_t   TDC_MAWt;
+	uint8_t   TdcWaterfallCtl;
+	uint8_t   DTEAmbientTempBase;
+
+	uint32_t  DisplayCac;
+	uint32_t  BAPM_TEMP_GRADIENT;
+	uint16_t  BAPMTI_R[SMU74_DTE_ITERATIONS * SMU74_DTE_SOURCES * SMU74_DTE_SINKS];
+	uint16_t  BAPMTI_RC[SMU74_DTE_ITERATIONS * SMU74_DTE_SOURCES * SMU74_DTE_SINKS];
+};
+
+
+
+struct polaris10_range_table {
+	uint32_t trans_lower_frequency; /* in 10khz */
+	uint32_t trans_upper_frequency;
 };
 
 struct polaris10_smumgr {
-	uint8_t *header;
-	uint8_t *mec_image;
-	struct polaris10_buffer_entry smu_buffer;
-	struct polaris10_buffer_entry header_buffer;
-	uint32_t soft_regs_start;
-	uint8_t *read_rrm_straps;
-	uint32_t read_drm_straps_mc_address_high;
-	uint32_t read_drm_straps_mc_address_low;
-	uint32_t acpi_optimization;
-	bool post_initial_boot;
+	struct smu7_smumgr smu7_data;
 	uint8_t protected_mode;
-	uint8_t security_hard_key;
 	struct polaris10_avfs  avfs;
+	SMU74_Discrete_DpmTable              smc_state_table;
+	struct SMU74_Discrete_Ulv            ulv_setting;
+	struct SMU74_Discrete_PmFuses  power_tune_table;
+	struct polaris10_range_table                range_table[NUM_SCLK_RANGE];
+	const struct polaris10_pt_defaults       *power_tune_defaults;
+	uint32_t                   activity_target[SMU74_MAX_LEVELS_GRAPHICS];
+	uint32_t                   bif_sclk_table[SMU74_MAX_LEVELS_LINK];
 };
 
 
-int polaris10_smum_init(struct pp_smumgr *smumgr);
-
-int polaris10_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr, uint32_t *value, uint32_t limit);
-int polaris10_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr, uint32_t value, uint32_t limit);
-int polaris10_copy_bytes_to_smc(struct pp_smumgr *smumgr, uint32_t smc_start_address,
-				const uint8_t *src, uint32_t byte_count, uint32_t limit);
-
 #endif
-
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
new file mode 100644
index 000000000000..6af744f42ec9
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
@@ -0,0 +1,589 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+
+#include "smumgr.h"
+#include "smu_ucode_xfer_vi.h"
+#include "smu/smu_7_1_3_d.h"
+#include "smu/smu_7_1_3_sh_mask.h"
+#include "ppatomctrl.h"
+#include "pp_debug.h"
+#include "cgs_common.h"
+#include "smu7_ppsmc.h"
+#include "smu7_smumgr.h"
+
+#define SMU7_SMC_SIZE 0x20000
+
+static int smu7_set_smc_sram_address(struct pp_smumgr *smumgr, uint32_t smc_addr, uint32_t limit)
+{
+	PP_ASSERT_WITH_CODE((0 == (3 & smc_addr)), "SMC address must be 4 byte aligned.", return -EINVAL);
+	PP_ASSERT_WITH_CODE((limit > (smc_addr + 3)), "SMC addr is beyond the SMC RAM area.", return -EINVAL);
+
+	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_11, smc_addr);
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_11, 0); /* on ci, SMC_IND_ACCESS_CNTL is different */
+	return 0;
+}
+
+
+int smu7_copy_bytes_from_smc(struct pp_smumgr *smumgr, uint32_t smc_start_address, uint32_t *dest, uint32_t byte_count, uint32_t limit)
+{
+	uint32_t data;
+	uint32_t addr;
+	uint8_t *dest_byte;
+	uint8_t i, data_byte[4] = {0};
+	uint32_t *pdata = (uint32_t *)&data_byte;
+
+	PP_ASSERT_WITH_CODE((0 == (3 & smc_start_address)), "SMC address must be 4 byte aligned.", return -EINVAL);
+	PP_ASSERT_WITH_CODE((limit > (smc_start_address + byte_count)), "SMC address is beyond the SMC RAM area.", return -EINVAL);
+
+	addr = smc_start_address;
+
+	while (byte_count >= 4) {
+		smu7_read_smc_sram_dword(smumgr, addr, &data, limit);
+
+		*dest = PP_SMC_TO_HOST_UL(data);
+
+		dest += 1;
+		byte_count -= 4;
+		addr += 4;
+	}
+
+	if (byte_count) {
+		smu7_read_smc_sram_dword(smumgr, addr, &data, limit);
+		*pdata = PP_SMC_TO_HOST_UL(data);
+	/* Cast dest into byte type in dest_byte.  This way, we don't overflow if the allocated memory is not 4-byte aligned. */
+		dest_byte = (uint8_t *)dest;
+		for (i = 0; i < byte_count; i++)
+			dest_byte[i] = data_byte[i];
+	}
+
+	return 0;
+}
+
+
+int smu7_copy_bytes_to_smc(struct pp_smumgr *smumgr, uint32_t smc_start_address,
+				const uint8_t *src, uint32_t byte_count, uint32_t limit)
+{
+	int result;
+	uint32_t data = 0;
+	uint32_t original_data;
+	uint32_t addr = 0;
+	uint32_t extra_shift;
+
+	PP_ASSERT_WITH_CODE((0 == (3 & smc_start_address)), "SMC address must be 4 byte aligned.", return -EINVAL);
+	PP_ASSERT_WITH_CODE((limit > (smc_start_address + byte_count)), "SMC address is beyond the SMC RAM area.", return -EINVAL);
+
+	addr = smc_start_address;
+
+	while (byte_count >= 4) {
+	/* Bytes are written into the SMC addres space with the MSB first. */
+		data = src[0] * 0x1000000 + src[1] * 0x10000 + src[2] * 0x100 + src[3];
+
+		result = smu7_set_smc_sram_address(smumgr, addr, limit);
+
+		if (0 != result)
+			return result;
+
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_11, data);
+
+		src += 4;
+		byte_count -= 4;
+		addr += 4;
+	}
+
+	if (0 != byte_count) {
+
+		data = 0;
+
+		result = smu7_set_smc_sram_address(smumgr, addr, limit);
+
+		if (0 != result)
+			return result;
+
+
+		original_data = cgs_read_register(smumgr->device, mmSMC_IND_DATA_11);
+
+		extra_shift = 8 * (4 - byte_count);
+
+		while (byte_count > 0) {
+			/* Bytes are written into the SMC addres space with the MSB first. */
+			data = (0x100 * data) + *src++;
+			byte_count--;
+		}
+
+		data <<= extra_shift;
+
+		data |= (original_data & ~((~0UL) << extra_shift));
+
+		result = smu7_set_smc_sram_address(smumgr, addr, limit);
+
+		if (0 != result)
+			return result;
+
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_11, data);
+	}
+
+	return 0;
+}
+
+
+int smu7_program_jump_on_start(struct pp_smumgr *smumgr)
+{
+	static const unsigned char data[4] = { 0xE0, 0x00, 0x80, 0x40 };
+
+	smu7_copy_bytes_to_smc(smumgr, 0x0, data, 4, sizeof(data)+1);
+
+	return 0;
+}
+
+bool smu7_is_smc_ram_running(struct pp_smumgr *smumgr)
+{
+	return ((0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC, SMC_SYSCON_CLOCK_CNTL_0, ck_disable))
+	&& (0x20100 <= cgs_read_ind_register(smumgr->device, CGS_IND_REG__SMC, ixSMC_PC_C)));
+}
+
+int smu7_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
+{
+	int ret;
+
+	if (!smu7_is_smc_ram_running(smumgr))
+		return -EINVAL;
+
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	ret = SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP);
+
+	if (ret != 1)
+		printk("\n failed to send pre message %x ret is %d \n",  msg, ret);
+
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	ret = SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP);
+
+	if (ret != 1)
+		printk("\n failed to send message %x ret is %d \n",  msg, ret);
+
+	return 0;
+}
+
+int smu7_send_msg_to_smc_without_waiting(struct pp_smumgr *smumgr, uint16_t msg)
+{
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
+
+	return 0;
+}
+
+int smu7_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr, uint16_t msg, uint32_t parameter)
+{
+	if (!smu7_is_smc_ram_running(smumgr)) {
+		return -EINVAL;
+	}
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
+
+	return smu7_send_msg_to_smc(smumgr, msg);
+}
+
+int smu7_send_msg_to_smc_with_parameter_without_waiting(struct pp_smumgr *smumgr, uint16_t msg, uint32_t parameter)
+{
+	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
+
+	return smu7_send_msg_to_smc_without_waiting(smumgr, msg);
+}
+
+int smu7_send_msg_to_smc_offset(struct pp_smumgr *smumgr)
+{
+	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, 0x20000);
+
+	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, PPSMC_MSG_Test);
+
+	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
+
+	if (1 != SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP))
+		printk("Failed to send Message.\n");
+
+	return 0;
+}
+
+int smu7_wait_for_smc_inactive(struct pp_smumgr *smumgr)
+{
+	if (!smu7_is_smc_ram_running(smumgr))
+		return -EINVAL;
+
+	SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, SMC_IND, SMC_SYSCON_CLOCK_CNTL_0, cken, 0);
+	return 0;
+}
+
+
+enum cgs_ucode_id smu7_convert_fw_type_to_cgs(uint32_t fw_type)
+{
+	enum cgs_ucode_id result = CGS_UCODE_ID_MAXIMUM;
+
+	switch (fw_type) {
+	case UCODE_ID_SMU:
+		result = CGS_UCODE_ID_SMU;
+		break;
+	case UCODE_ID_SMU_SK:
+		result = CGS_UCODE_ID_SMU_SK;
+		break;
+	case UCODE_ID_SDMA0:
+		result = CGS_UCODE_ID_SDMA0;
+		break;
+	case UCODE_ID_SDMA1:
+		result = CGS_UCODE_ID_SDMA1;
+		break;
+	case UCODE_ID_CP_CE:
+		result = CGS_UCODE_ID_CP_CE;
+		break;
+	case UCODE_ID_CP_PFP:
+		result = CGS_UCODE_ID_CP_PFP;
+		break;
+	case UCODE_ID_CP_ME:
+		result = CGS_UCODE_ID_CP_ME;
+		break;
+	case UCODE_ID_CP_MEC:
+		result = CGS_UCODE_ID_CP_MEC;
+		break;
+	case UCODE_ID_CP_MEC_JT1:
+		result = CGS_UCODE_ID_CP_MEC_JT1;
+		break;
+	case UCODE_ID_CP_MEC_JT2:
+		result = CGS_UCODE_ID_CP_MEC_JT2;
+		break;
+	case UCODE_ID_RLC_G:
+		result = CGS_UCODE_ID_RLC_G;
+		break;
+	default:
+		break;
+	}
+
+	return result;
+}
+
+
+int smu7_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr, uint32_t *value, uint32_t limit)
+{
+	int result;
+
+	result = smu7_set_smc_sram_address(smumgr, smc_addr, limit);
+
+	if (result)
+		return result;
+
+	*value = cgs_read_register(smumgr->device, mmSMC_IND_DATA_11);
+	return 0;
+}
+
+int smu7_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr, uint32_t value, uint32_t limit)
+{
+	int result;
+
+	result = smu7_set_smc_sram_address(smumgr, smc_addr, limit);
+
+	if (result)
+		return result;
+
+	cgs_write_register(smumgr->device, mmSMC_IND_DATA_11, value);
+
+	return 0;
+}
+
+/* Convert the firmware type to SMU type mask. For MEC, we need to check all MEC related type */
+
+static uint32_t smu7_get_mask_for_firmware_type(uint32_t fw_type)
+{
+	uint32_t result = 0;
+
+	switch (fw_type) {
+	case UCODE_ID_SDMA0:
+		result = UCODE_ID_SDMA0_MASK;
+		break;
+	case UCODE_ID_SDMA1:
+		result = UCODE_ID_SDMA1_MASK;
+		break;
+	case UCODE_ID_CP_CE:
+		result = UCODE_ID_CP_CE_MASK;
+		break;
+	case UCODE_ID_CP_PFP:
+		result = UCODE_ID_CP_PFP_MASK;
+		break;
+	case UCODE_ID_CP_ME:
+		result = UCODE_ID_CP_ME_MASK;
+		break;
+	case UCODE_ID_CP_MEC:
+	case UCODE_ID_CP_MEC_JT1:
+	case UCODE_ID_CP_MEC_JT2:
+		result = UCODE_ID_CP_MEC_MASK;
+		break;
+	case UCODE_ID_RLC_G:
+		result = UCODE_ID_RLC_G_MASK;
+		break;
+	default:
+		printk("UCode type is out of range! \n");
+		result = 0;
+	}
+
+	return result;
+}
+
+static int smu7_populate_single_firmware_entry(struct pp_smumgr *smumgr,
+						uint32_t fw_type,
+						struct SMU_Entry *entry)
+{
+	int result = 0;
+	struct cgs_firmware_info info = {0};
+
+	result = cgs_get_firmware_info(smumgr->device,
+				smu7_convert_fw_type_to_cgs(fw_type),
+				&info);
+
+	if (!result) {
+		entry->version = info.version;
+		entry->id = (uint16_t)fw_type;
+		entry->image_addr_high = smu_upper_32_bits(info.mc_addr);
+		entry->image_addr_low = smu_lower_32_bits(info.mc_addr);
+		entry->meta_data_addr_high = 0;
+		entry->meta_data_addr_low = 0;
+		entry->data_size_byte = info.image_size;
+		entry->num_register_entries = 0;
+	}
+
+	if (fw_type == UCODE_ID_RLC_G)
+		entry->flags = 1;
+	else
+		entry->flags = 0;
+
+	return 0;
+}
+
+int smu7_request_smu_load_fw(struct pp_smumgr *smumgr)
+{
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
+	uint32_t fw_to_load;
+	int result = 0;
+	struct SMU_DRAMData_TOC *toc;
+
+	if (!smumgr->reload_fw) {
+		printk(KERN_INFO "[ powerplay ] skip reloading...\n");
+		return 0;
+	}
+
+	if (smu_data->soft_regs_start)
+		cgs_write_ind_register(smumgr->device, CGS_IND_REG__SMC,
+					smu_data->soft_regs_start + smum_get_offsetof(smumgr,
+					SMU_SoftRegisters, UcodeLoadStatus),
+					0x0);
+
+	if (smumgr->chip_id > CHIP_TOPAZ) { /* add support for Topaz */
+		smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_SMU_DRAM_ADDR_HI, smu_data->smu_buffer.mc_addr_high);
+		smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_SMU_DRAM_ADDR_LO, smu_data->smu_buffer.mc_addr_low);
+		fw_to_load = UCODE_ID_RLC_G_MASK
+			   + UCODE_ID_SDMA0_MASK
+			   + UCODE_ID_SDMA1_MASK
+			   + UCODE_ID_CP_CE_MASK
+			   + UCODE_ID_CP_ME_MASK
+			   + UCODE_ID_CP_PFP_MASK
+			   + UCODE_ID_CP_MEC_MASK;
+	} else {
+		fw_to_load = UCODE_ID_RLC_G_MASK
+			   + UCODE_ID_SDMA0_MASK
+			   + UCODE_ID_SDMA1_MASK
+			   + UCODE_ID_CP_CE_MASK
+			   + UCODE_ID_CP_ME_MASK
+			   + UCODE_ID_CP_PFP_MASK
+			   + UCODE_ID_CP_MEC_MASK
+			   + UCODE_ID_CP_MEC_JT1_MASK
+			   + UCODE_ID_CP_MEC_JT2_MASK;
+	}
+
+	toc = (struct SMU_DRAMData_TOC *)smu_data->header;
+	toc->num_entries = 0;
+	toc->structure_version = 1;
+
+	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_RLC_G, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
+	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_CP_CE, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
+	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_CP_PFP, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
+	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_CP_ME, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
+	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_CP_MEC, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
+	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_CP_MEC_JT1, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
+	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_CP_MEC_JT2, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
+	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_SDMA0, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
+	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
+
+	smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_DRV_DRAM_ADDR_HI, smu_data->header_buffer.mc_addr_high);
+	smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_DRV_DRAM_ADDR_LO, smu_data->header_buffer.mc_addr_low);
+
+	if (smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_LoadUcodes, fw_to_load))
+		printk(KERN_ERR "Fail to Request SMU Load uCode");
+
+	return result;
+}
+
+/* Check if the FW has been loaded, SMU will not return if loading has not finished. */
+int smu7_check_fw_load_finish(struct pp_smumgr *smumgr, uint32_t fw_type)
+{
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
+	uint32_t fw_mask = smu7_get_mask_for_firmware_type(fw_type);
+	uint32_t ret;
+
+	ret = smum_wait_on_indirect_register(smumgr, mmSMC_IND_INDEX_11,
+					smu_data->soft_regs_start + smum_get_offsetof(smumgr,
+					SMU_SoftRegisters, UcodeLoadStatus),
+					fw_mask, fw_mask);
+
+	return ret;
+}
+
+int smu7_reload_firmware(struct pp_smumgr *smumgr)
+{
+	return smumgr->smumgr_funcs->start_smu(smumgr);
+}
+
+static int smu7_upload_smc_firmware_data(struct pp_smumgr *smumgr, uint32_t length, uint32_t *src, uint32_t limit)
+{
+	uint32_t byte_count = length;
+
+	PP_ASSERT_WITH_CODE((limit >= byte_count), "SMC address is beyond the SMC RAM area.", return -EINVAL);
+
+	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_11, 0x20000);
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_11, 1);
+
+	for (; byte_count >= 4; byte_count -= 4)
+		cgs_write_register(smumgr->device, mmSMC_IND_DATA_11, *src++);
+
+	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_11, 0);
+
+	PP_ASSERT_WITH_CODE((0 == byte_count), "SMC size must be dividable by 4.", return -EINVAL);
+
+	return 0;
+}
+
+
+int smu7_upload_smu_firmware_image(struct pp_smumgr *smumgr)
+{
+	int result = 0;
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
+
+	struct cgs_firmware_info info = {0};
+
+	if (smu_data->security_hard_key == 1)
+		cgs_get_firmware_info(smumgr->device,
+			smu7_convert_fw_type_to_cgs(UCODE_ID_SMU), &info);
+	else
+		cgs_get_firmware_info(smumgr->device,
+			smu7_convert_fw_type_to_cgs(UCODE_ID_SMU_SK), &info);
+
+	result = smu7_upload_smc_firmware_data(smumgr, info.image_size, (uint32_t *)info.kptr, SMU7_SMC_SIZE);
+
+	return result;
+}
+
+
+int smu7_init(struct pp_smumgr *smumgr)
+{
+	struct smu7_smumgr *smu_data;
+	uint8_t *internal_buf;
+	uint64_t mc_addr = 0;
+
+	/* Allocate memory for backend private data */
+	smu_data = (struct smu7_smumgr *)(smumgr->backend);
+	smu_data->header_buffer.data_size =
+			((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
+	smu_data->smu_buffer.data_size = 200*4096;
+
+/* Allocate FW image data structure and header buffer and
+ * send the header buffer address to SMU */
+	smu_allocate_memory(smumgr->device,
+		smu_data->header_buffer.data_size,
+		CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
+		PAGE_SIZE,
+		&mc_addr,
+		&smu_data->header_buffer.kaddr,
+		&smu_data->header_buffer.handle);
+
+	smu_data->header = smu_data->header_buffer.kaddr;
+	smu_data->header_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
+	smu_data->header_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
+
+	PP_ASSERT_WITH_CODE((NULL != smu_data->header),
+		"Out of memory.",
+		kfree(smumgr->backend);
+		cgs_free_gpu_mem(smumgr->device,
+		(cgs_handle_t)smu_data->header_buffer.handle);
+		return -EINVAL);
+
+	smu_allocate_memory(smumgr->device,
+		smu_data->smu_buffer.data_size,
+		CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
+		PAGE_SIZE,
+		&mc_addr,
+		&smu_data->smu_buffer.kaddr,
+		&smu_data->smu_buffer.handle);
+
+	internal_buf = smu_data->smu_buffer.kaddr;
+	smu_data->smu_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
+	smu_data->smu_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
+
+	PP_ASSERT_WITH_CODE((NULL != internal_buf),
+		"Out of memory.",
+		kfree(smumgr->backend);
+		cgs_free_gpu_mem(smumgr->device,
+		(cgs_handle_t)smu_data->smu_buffer.handle);
+		return -EINVAL);
+
+	return 0;
+}
+
+
+int smu7_smu_fini(struct pp_smumgr *smumgr)
+{
+	if (smumgr->backend) {
+		kfree(smumgr->backend);
+		smumgr->backend = NULL;
+	}
+	cgs_rel_firmware(smumgr->device, CGS_UCODE_ID_SMU);
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
new file mode 100644
index 000000000000..76352f2423ae
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _SMU7_SMUMANAGER_H
+#define _SMU7_SMUMANAGER_H
+
+
+#include <pp_endian.h>
+
+#define SMC_RAM_END 0x40000
+#define mmSMC_IND_INDEX_11                              0x01AC
+#define mmSMC_IND_DATA_11                               0x01AD
+
+struct smu7_buffer_entry {
+	uint32_t data_size;
+	uint32_t mc_addr_low;
+	uint32_t mc_addr_high;
+	void *kaddr;
+	unsigned long  handle;
+};
+
+struct smu7_smumgr {
+	uint8_t *header;
+	uint8_t *mec_image;
+	struct smu7_buffer_entry smu_buffer;
+	struct smu7_buffer_entry header_buffer;
+
+	uint32_t                             soft_regs_start;
+	uint32_t                             dpm_table_start;
+	uint32_t                             mc_reg_table_start;
+	uint32_t                             fan_table_start;
+	uint32_t                             arb_table_start;
+	uint32_t                             ulv_setting_starts;
+	uint8_t                              security_hard_key;
+	uint32_t acpi_optimization;
+};
+
+
+int smu7_copy_bytes_from_smc(struct pp_smumgr *smumgr, uint32_t smc_start_address,
+				uint32_t *dest, uint32_t byte_count, uint32_t limit);
+int smu7_copy_bytes_to_smc(struct pp_smumgr *smumgr, uint32_t smc_start_address,
+			const uint8_t *src, uint32_t byte_count, uint32_t limit);
+int smu7_program_jump_on_start(struct pp_smumgr *smumgr);
+bool smu7_is_smc_ram_running(struct pp_smumgr *smumgr);
+int smu7_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg);
+int smu7_send_msg_to_smc_without_waiting(struct pp_smumgr *smumgr, uint16_t msg);
+int smu7_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr, uint16_t msg,
+						uint32_t parameter);
+int smu7_send_msg_to_smc_with_parameter_without_waiting(struct pp_smumgr *smumgr,
+						uint16_t msg, uint32_t parameter);
+int smu7_send_msg_to_smc_offset(struct pp_smumgr *smumgr);
+int smu7_wait_for_smc_inactive(struct pp_smumgr *smumgr);
+
+enum cgs_ucode_id smu7_convert_fw_type_to_cgs(uint32_t fw_type);
+int smu7_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr,
+						uint32_t *value, uint32_t limit);
+int smu7_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smc_addr,
+						uint32_t value, uint32_t limit);
+
+int smu7_request_smu_load_fw(struct pp_smumgr *smumgr);
+int smu7_check_fw_load_finish(struct pp_smumgr *smumgr, uint32_t fw_type);
+int smu7_reload_firmware(struct pp_smumgr *smumgr);
+int smu7_upload_smu_firmware_image(struct pp_smumgr *smumgr);
+int smu7_init(struct pp_smumgr *smumgr);
+int smu7_smu_fini(struct pp_smumgr *smumgr);
+
+#endif
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
index 7723473e51a0..e5812aa456f3 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
@@ -28,10 +28,7 @@
 #include "smumgr.h"
 #include "cgs_common.h"
 #include "linux/delay.h"
-#include "cz_smumgr.h"
-#include "tonga_smumgr.h"
-#include "fiji_smumgr.h"
-#include "polaris10_smumgr.h"
+
 
 int smum_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
 {
@@ -47,7 +44,6 @@ int smum_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
 	smumgr->device = pp_init->device;
 	smumgr->chip_family = pp_init->chip_family;
 	smumgr->chip_id = pp_init->chip_id;
-	smumgr->hw_revision = pp_init->rev_id;
 	smumgr->usec_timeout = AMD_MAX_USEC_TIMEOUT;
 	smumgr->reload_fw = 1;
 	handle->smu_mgr = smumgr;
@@ -58,6 +54,9 @@ int smum_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
 		break;
 	case AMDGPU_FAMILY_VI:
 		switch (smumgr->chip_id) {
+		case CHIP_TOPAZ:
+			iceland_smum_init(smumgr);
+			break;
 		case CHIP_TONGA:
 			tonga_smum_init(smumgr);
 			break;
@@ -87,6 +86,57 @@ int smum_fini(struct pp_smumgr *smumgr)
 	return 0;
 }
 
+int smum_thermal_avfs_enable(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result)
+{
+	if (NULL != hwmgr->smumgr->smumgr_funcs->thermal_avfs_enable)
+		return hwmgr->smumgr->smumgr_funcs->thermal_avfs_enable(hwmgr);
+
+	return 0;
+}
+
+int smum_thermal_setup_fan_table(struct pp_hwmgr *hwmgr,
+		void *input, void *output, void *storage, int result)
+{
+	if (NULL != hwmgr->smumgr->smumgr_funcs->thermal_setup_fan_table)
+		return hwmgr->smumgr->smumgr_funcs->thermal_setup_fan_table(hwmgr);
+
+	return 0;
+}
+
+int smum_update_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+
+	if (NULL != hwmgr->smumgr->smumgr_funcs->update_sclk_threshold)
+		return hwmgr->smumgr->smumgr_funcs->update_sclk_threshold(hwmgr);
+
+	return 0;
+}
+
+int smum_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type)
+{
+
+	if (NULL != hwmgr->smumgr->smumgr_funcs->update_smc_table)
+		return hwmgr->smumgr->smumgr_funcs->update_smc_table(hwmgr, type);
+
+	return 0;
+}
+
+uint32_t smum_get_offsetof(struct pp_smumgr *smumgr, uint32_t type, uint32_t member)
+{
+	if (NULL != smumgr->smumgr_funcs->get_offsetof)
+		return smumgr->smumgr_funcs->get_offsetof(type, member);
+
+	return 0;
+}
+
+int smum_process_firmware_header(struct pp_hwmgr *hwmgr)
+{
+	if (NULL != hwmgr->smumgr->smumgr_funcs->process_firmware_header)
+		return hwmgr->smumgr->smumgr_funcs->process_firmware_header(hwmgr);
+	return 0;
+}
+
 int smum_get_argument(struct pp_smumgr *smumgr)
 {
 	if (NULL != smumgr->smumgr_funcs->get_argument)
@@ -95,13 +145,20 @@ int smum_get_argument(struct pp_smumgr *smumgr)
 	return 0;
 }
 
+uint32_t smum_get_mac_definition(struct pp_smumgr *smumgr, uint32_t value)
+{
+	if (NULL != smumgr->smumgr_funcs->get_mac_definition)
+		return smumgr->smumgr_funcs->get_mac_definition(value);
+
+	return 0;
+}
+
 int smum_download_powerplay_table(struct pp_smumgr *smumgr,
 								void **table)
 {
 	if (NULL != smumgr->smumgr_funcs->download_pptable_settings)
 		return smumgr->smumgr_funcs->download_pptable_settings(smumgr,
 									table);
-
 	return 0;
 }
 
@@ -268,3 +325,44 @@ int smu_free_memory(void *device, void *handle)
 
 	return 0;
 }
+
+int smum_init_smc_table(struct pp_hwmgr *hwmgr)
+{
+	if (NULL != hwmgr->smumgr->smumgr_funcs->init_smc_table)
+		return hwmgr->smumgr->smumgr_funcs->init_smc_table(hwmgr);
+
+	return 0;
+}
+
+int smum_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
+{
+	if (NULL != hwmgr->smumgr->smumgr_funcs->populate_all_graphic_levels)
+		return hwmgr->smumgr->smumgr_funcs->populate_all_graphic_levels(hwmgr);
+
+	return 0;
+}
+
+int smum_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
+{
+	if (NULL != hwmgr->smumgr->smumgr_funcs->populate_all_memory_levels)
+		return hwmgr->smumgr->smumgr_funcs->populate_all_memory_levels(hwmgr);
+
+	return 0;
+}
+
+/*this interface is needed by island ci/vi */
+int smum_initialize_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	if (NULL != hwmgr->smumgr->smumgr_funcs->initialize_mc_reg_table)
+		return hwmgr->smumgr->smumgr_funcs->initialize_mc_reg_table(hwmgr);
+
+	return 0;
+}
+
+bool smum_is_dpm_running(struct pp_hwmgr *hwmgr)
+{
+	if (NULL != hwmgr->smumgr->smumgr_funcs->is_dpm_running)
+		return hwmgr->smumgr->smumgr_funcs->is_dpm_running(hwmgr);
+
+	return true;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c
new file mode 100644
index 000000000000..de2a24d85f48
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c
@@ -0,0 +1,3206 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ *
+ */
+
+#include "tonga_smc.h"
+#include "smu7_dyn_defaults.h"
+
+#include "smu7_hwmgr.h"
+#include "hardwaremanager.h"
+#include "ppatomctrl.h"
+#include "pp_debug.h"
+#include "cgs_common.h"
+#include "atombios.h"
+#include "tonga_smumgr.h"
+#include "pppcielanes.h"
+#include "pp_endian.h"
+#include "smu7_ppsmc.h"
+
+#include "smu72_discrete.h"
+
+#include "smu/smu_7_1_2_d.h"
+#include "smu/smu_7_1_2_sh_mask.h"
+
+#include "gmc/gmc_8_1_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+
+#include "bif/bif_5_0_d.h"
+#include "bif/bif_5_0_sh_mask.h"
+
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+
+
+#define VOLTAGE_SCALE 4
+#define POWERTUNE_DEFAULT_SET_MAX    1
+#define VOLTAGE_VID_OFFSET_SCALE1   625
+#define VOLTAGE_VID_OFFSET_SCALE2   100
+#define MC_CG_ARB_FREQ_F1           0x0b
+#define VDDC_VDDCI_DELTA            200
+
+
+static const struct tonga_pt_defaults tonga_power_tune_data_set_array[POWERTUNE_DEFAULT_SET_MAX] = {
+/* sviLoadLIneEn, SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc,  TDC_MAWt,
+ * TdcWaterfallCtl, DTEAmbientTempBase, DisplayCac,        BAPM_TEMP_GRADIENT
+ */
+	{1,               0xF,             0xFD,                0x19,
+	 5,               45,                 0,              0xB0000,
+	 {0x79, 0x253, 0x25D, 0xAE, 0x72, 0x80, 0x83, 0x86, 0x6F, 0xC8,
+		0xC9, 0xC9, 0x2F, 0x4D, 0x61},
+	 {0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203,
+		0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4}
+	},
+};
+
+/* [Fmin, Fmax, LDO_REFSEL, USE_FOR_LOW_FREQ] */
+static const uint16_t tonga_clock_stretcher_lookup_table[2][4] = {
+	{600, 1050, 3, 0},
+	{600, 1050, 6, 1}
+};
+
+/* [FF, SS] type, [] 4 voltage ranges,
+ * and [Floor Freq, Boundary Freq, VID min , VID max]
+ */
+static const uint32_t tonga_clock_stretcher_ddt_table[2][4][4] = {
+	{ {265, 529, 120, 128}, {325, 650, 96, 119}, {430, 860, 32, 95}, {0, 0, 0, 31} },
+	{ {275, 550, 104, 112}, {319, 638, 96, 103}, {360, 720, 64, 95}, {384, 768, 32, 63} }
+};
+
+/* [Use_For_Low_freq] value, [0%, 5%, 10%, 7.14%, 14.28%, 20%] */
+static const uint8_t tonga_clock_stretch_amount_conversion[2][6] = {
+	{0, 1, 3, 2, 4, 5},
+	{0, 2, 4, 5, 6, 5}
+};
+
+/* PPGen has the gain setting generated in x * 100 unit
+ * This function is to convert the unit to x * 4096(0x1000) unit.
+ *  This is the unit expected by SMC firmware
+ */
+
+
+static int tonga_get_dependecy_volt_by_clk(struct pp_hwmgr *hwmgr,
+	phm_ppt_v1_clock_voltage_dependency_table *allowed_clock_voltage_table,
+	uint32_t clock, SMU_VoltageLevel *voltage, uint32_t *mvdd)
+{
+	uint32_t i = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info =
+			   (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	/* clock - voltage dependency table is empty table */
+	if (allowed_clock_voltage_table->count == 0)
+		return -EINVAL;
+
+	for (i = 0; i < allowed_clock_voltage_table->count; i++) {
+		/* find first sclk bigger than request */
+		if (allowed_clock_voltage_table->entries[i].clk >= clock) {
+			voltage->VddGfx = phm_get_voltage_index(
+					pptable_info->vddgfx_lookup_table,
+				allowed_clock_voltage_table->entries[i].vddgfx);
+			voltage->Vddc = phm_get_voltage_index(
+						pptable_info->vddc_lookup_table,
+				  allowed_clock_voltage_table->entries[i].vddc);
+
+			if (allowed_clock_voltage_table->entries[i].vddci)
+				voltage->Vddci =
+					phm_get_voltage_id(&data->vddci_voltage_table, allowed_clock_voltage_table->entries[i].vddci);
+			else
+				voltage->Vddci =
+					phm_get_voltage_id(&data->vddci_voltage_table,
+						allowed_clock_voltage_table->entries[i].vddc - VDDC_VDDCI_DELTA);
+
+
+			if (allowed_clock_voltage_table->entries[i].mvdd)
+				*mvdd = (uint32_t) allowed_clock_voltage_table->entries[i].mvdd;
+
+			voltage->Phases = 1;
+			return 0;
+		}
+	}
+
+	/* sclk is bigger than max sclk in the dependence table */
+	voltage->VddGfx = phm_get_voltage_index(pptable_info->vddgfx_lookup_table,
+		allowed_clock_voltage_table->entries[i-1].vddgfx);
+	voltage->Vddc = phm_get_voltage_index(pptable_info->vddc_lookup_table,
+		allowed_clock_voltage_table->entries[i-1].vddc);
+
+	if (allowed_clock_voltage_table->entries[i-1].vddci)
+		voltage->Vddci = phm_get_voltage_id(&data->vddci_voltage_table,
+			allowed_clock_voltage_table->entries[i-1].vddci);
+
+	if (allowed_clock_voltage_table->entries[i-1].mvdd)
+		*mvdd = (uint32_t) allowed_clock_voltage_table->entries[i-1].mvdd;
+
+	return 0;
+}
+
+
+/**
+ * Vddc table preparation for SMC.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+static int tonga_populate_smc_vddc_table(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	unsigned int count;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+		table->VddcLevelCount = data->vddc_voltage_table.count;
+		for (count = 0; count < table->VddcLevelCount; count++) {
+			table->VddcTable[count] =
+				PP_HOST_TO_SMC_US(data->vddc_voltage_table.entries[count].value * VOLTAGE_SCALE);
+		}
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VddcLevelCount);
+	}
+	return 0;
+}
+
+/**
+ * VddGfx table preparation for SMC.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+static int tonga_populate_smc_vdd_gfx_table(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	unsigned int count;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_gfx_control) {
+		table->VddGfxLevelCount = data->vddgfx_voltage_table.count;
+		for (count = 0; count < data->vddgfx_voltage_table.count; count++) {
+			table->VddGfxTable[count] =
+				PP_HOST_TO_SMC_US(data->vddgfx_voltage_table.entries[count].value * VOLTAGE_SCALE);
+		}
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VddGfxLevelCount);
+	}
+	return 0;
+}
+
+/**
+ * Vddci table preparation for SMC.
+ *
+ * @param    *hwmgr The address of the hardware manager.
+ * @param    *table The SMC DPM table structure to be populated.
+ * @return   0
+ */
+static int tonga_populate_smc_vdd_ci_table(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t count;
+
+	table->VddciLevelCount = data->vddci_voltage_table.count;
+	for (count = 0; count < table->VddciLevelCount; count++) {
+		if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
+			table->VddciTable[count] =
+				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[count].value * VOLTAGE_SCALE);
+		} else if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+			table->SmioTable1.Pattern[count].Voltage =
+				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[count].value * VOLTAGE_SCALE);
+			/* Index into DpmTable.Smio. Drive bits from Smio entry to get this voltage level. */
+			table->SmioTable1.Pattern[count].Smio =
+				(uint8_t) count;
+			table->Smio[count] |=
+				data->vddci_voltage_table.entries[count].smio_low;
+			table->VddciTable[count] =
+				PP_HOST_TO_SMC_US(data->vddci_voltage_table.entries[count].value * VOLTAGE_SCALE);
+		}
+	}
+
+	table->SmioMask1 = data->vddci_voltage_table.mask_low;
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VddciLevelCount);
+
+	return 0;
+}
+
+/**
+ * Mvdd table preparation for SMC.
+ *
+ * @param    *hwmgr The address of the hardware manager.
+ * @param    *table The SMC DPM table structure to be populated.
+ * @return   0
+ */
+static int tonga_populate_smc_mvdd_table(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t count;
+
+	if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		table->MvddLevelCount = data->mvdd_voltage_table.count;
+		for (count = 0; count < table->MvddLevelCount; count++) {
+			table->SmioTable2.Pattern[count].Voltage =
+				PP_HOST_TO_SMC_US(data->mvdd_voltage_table.entries[count].value * VOLTAGE_SCALE);
+			/* Index into DpmTable.Smio. Drive bits from Smio entry to get this voltage level.*/
+			table->SmioTable2.Pattern[count].Smio =
+				(uint8_t) count;
+			table->Smio[count] |=
+				data->mvdd_voltage_table.entries[count].smio_low;
+		}
+		table->SmioMask2 = data->mvdd_voltage_table.mask_low;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->MvddLevelCount);
+	}
+
+	return 0;
+}
+
+/**
+ * Preparation of vddc and vddgfx CAC tables for SMC.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+static int tonga_populate_cac_tables(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	uint32_t count;
+	uint8_t index = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_voltage_lookup_table *vddgfx_lookup_table =
+					   pptable_info->vddgfx_lookup_table;
+	struct phm_ppt_v1_voltage_lookup_table *vddc_lookup_table =
+						pptable_info->vddc_lookup_table;
+
+	/* table is already swapped, so in order to use the value from it
+	 * we need to swap it back.
+	 */
+	uint32_t vddc_level_count = PP_SMC_TO_HOST_UL(table->VddcLevelCount);
+	uint32_t vddgfx_level_count = PP_SMC_TO_HOST_UL(table->VddGfxLevelCount);
+
+	for (count = 0; count < vddc_level_count; count++) {
+		/* We are populating vddc CAC data to BapmVddc table in split and merged mode */
+		index = phm_get_voltage_index(vddc_lookup_table,
+			data->vddc_voltage_table.entries[count].value);
+		table->BapmVddcVidLoSidd[count] =
+			convert_to_vid(vddc_lookup_table->entries[index].us_cac_low);
+		table->BapmVddcVidHiSidd[count] =
+			convert_to_vid(vddc_lookup_table->entries[index].us_cac_mid);
+		table->BapmVddcVidHiSidd2[count] =
+			convert_to_vid(vddc_lookup_table->entries[index].us_cac_high);
+	}
+
+	if ((data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2)) {
+		/* We are populating vddgfx CAC data to BapmVddgfx table in split mode */
+		for (count = 0; count < vddgfx_level_count; count++) {
+			index = phm_get_voltage_index(vddgfx_lookup_table,
+				convert_to_vid(vddgfx_lookup_table->entries[index].us_cac_mid));
+			table->BapmVddGfxVidHiSidd2[count] =
+				convert_to_vid(vddgfx_lookup_table->entries[index].us_cac_high);
+		}
+	} else {
+		for (count = 0; count < vddc_level_count; count++) {
+			index = phm_get_voltage_index(vddc_lookup_table,
+				data->vddc_voltage_table.entries[count].value);
+			table->BapmVddGfxVidLoSidd[count] =
+				convert_to_vid(vddc_lookup_table->entries[index].us_cac_low);
+			table->BapmVddGfxVidHiSidd[count] =
+				convert_to_vid(vddc_lookup_table->entries[index].us_cac_mid);
+			table->BapmVddGfxVidHiSidd2[count] =
+				convert_to_vid(vddc_lookup_table->entries[index].us_cac_high);
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Preparation of voltage tables for SMC.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+
+static int tonga_populate_smc_voltage_tables(struct pp_hwmgr *hwmgr,
+	SMU72_Discrete_DpmTable *table)
+{
+	int result;
+
+	result = tonga_populate_smc_vddc_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"can not populate VDDC voltage table to SMC",
+			return -EINVAL);
+
+	result = tonga_populate_smc_vdd_ci_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"can not populate VDDCI voltage table to SMC",
+			return -EINVAL);
+
+	result = tonga_populate_smc_vdd_gfx_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"can not populate VDDGFX voltage table to SMC",
+			return -EINVAL);
+
+	result = tonga_populate_smc_mvdd_table(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"can not populate MVDD voltage table to SMC",
+			return -EINVAL);
+
+	result = tonga_populate_cac_tables(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+			"can not populate CAC voltage tables to SMC",
+			return -EINVAL);
+
+	return 0;
+}
+
+static int tonga_populate_ulv_level(struct pp_hwmgr *hwmgr,
+		struct SMU72_Discrete_Ulv *state)
+{
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	state->CcPwrDynRm = 0;
+	state->CcPwrDynRm1 = 0;
+
+	state->VddcOffset = (uint16_t) table_info->us_ulv_voltage_offset;
+	state->VddcOffsetVid = (uint8_t)(table_info->us_ulv_voltage_offset *
+			VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1);
+
+	state->VddcPhase = 1;
+
+	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(state->CcPwrDynRm1);
+	CONVERT_FROM_HOST_TO_SMC_US(state->VddcOffset);
+
+	return 0;
+}
+
+static int tonga_populate_ulv_state(struct pp_hwmgr *hwmgr,
+		struct SMU72_Discrete_DpmTable *table)
+{
+	return tonga_populate_ulv_level(hwmgr, &table->Ulv);
+}
+
+static int tonga_populate_smc_link_level(struct pp_hwmgr *hwmgr, SMU72_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t i;
+
+	/* Index (dpm_table->pcie_speed_table.count) is reserved for PCIE boot level. */
+	for (i = 0; i <= dpm_table->pcie_speed_table.count; i++) {
+		table->LinkLevel[i].PcieGenSpeed  =
+			(uint8_t)dpm_table->pcie_speed_table.dpm_levels[i].value;
+		table->LinkLevel[i].PcieLaneCount =
+			(uint8_t)encode_pcie_lane_width(dpm_table->pcie_speed_table.dpm_levels[i].param1);
+		table->LinkLevel[i].EnabledForActivity =
+			1;
+		table->LinkLevel[i].SPC =
+			(uint8_t)(data->pcie_spc_cap & 0xff);
+		table->LinkLevel[i].DownThreshold =
+			PP_HOST_TO_SMC_UL(5);
+		table->LinkLevel[i].UpThreshold =
+			PP_HOST_TO_SMC_UL(30);
+	}
+
+	smu_data->smc_state_table.LinkLevelCount =
+		(uint8_t)dpm_table->pcie_speed_table.count;
+	data->dpm_level_enable_mask.pcie_dpm_enable_mask =
+		phm_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
+
+	return 0;
+}
+
+/**
+ * Calculates the SCLK dividers using the provided engine clock
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    engine_clock the engine clock to use to populate the structure
+ * @param    sclk        the SMC SCLK structure to be populated
+ */
+static int tonga_calculate_sclk_params(struct pp_hwmgr *hwmgr,
+		uint32_t engine_clock, SMU72_Discrete_GraphicsLevel *sclk)
+{
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	pp_atomctrl_clock_dividers_vi dividers;
+	uint32_t spll_func_cntl            = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_3          = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	uint32_t spll_func_cntl_4          = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	uint32_t cg_spll_spread_spectrum   = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	uint32_t cg_spll_spread_spectrum_2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	uint32_t    reference_clock;
+	uint32_t reference_divider;
+	uint32_t fbdiv;
+	int result;
+
+	/* get the engine clock dividers for this clock value*/
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr, engine_clock,  &dividers);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+		"Error retrieving Engine Clock dividers from VBIOS.", return result);
+
+	/* To get FBDIV we need to multiply this by 16384 and divide it by Fref.*/
+	reference_clock = atomctrl_get_reference_clock(hwmgr);
+
+	reference_divider = 1 + dividers.uc_pll_ref_div;
+
+	/* low 14 bits is fraction and high 12 bits is divider*/
+	fbdiv = dividers.ul_fb_div.ul_fb_divider & 0x3FFFFFF;
+
+	/* SPLL_FUNC_CNTL setup*/
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl,
+		CG_SPLL_FUNC_CNTL, SPLL_REF_DIV, dividers.uc_pll_ref_div);
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl,
+		CG_SPLL_FUNC_CNTL, SPLL_PDIV_A,  dividers.uc_pll_post_div);
+
+	/* SPLL_FUNC_CNTL_3 setup*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3,
+		CG_SPLL_FUNC_CNTL_3, SPLL_FB_DIV, fbdiv);
+
+	/* set to use fractional accumulation*/
+	spll_func_cntl_3 = PHM_SET_FIELD(spll_func_cntl_3,
+		CG_SPLL_FUNC_CNTL_3, SPLL_DITHEN, 1);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_EngineSpreadSpectrumSupport)) {
+		pp_atomctrl_internal_ss_info ss_info;
+
+		uint32_t vcoFreq = engine_clock * dividers.uc_pll_post_div;
+		if (0 == atomctrl_get_engine_clock_spread_spectrum(hwmgr, vcoFreq, &ss_info)) {
+			/*
+			* ss_info.speed_spectrum_percentage -- in unit of 0.01%
+			* ss_info.speed_spectrum_rate -- in unit of khz
+			*/
+			/* clks = reference_clock * 10 / (REFDIV + 1) / speed_spectrum_rate / 2 */
+			uint32_t clkS = reference_clock * 5 / (reference_divider * ss_info.speed_spectrum_rate);
+
+			/* clkv = 2 * D * fbdiv / NS */
+			uint32_t clkV = 4 * ss_info.speed_spectrum_percentage * fbdiv / (clkS * 10000);
+
+			cg_spll_spread_spectrum =
+				PHM_SET_FIELD(cg_spll_spread_spectrum, CG_SPLL_SPREAD_SPECTRUM, CLKS, clkS);
+			cg_spll_spread_spectrum =
+				PHM_SET_FIELD(cg_spll_spread_spectrum, CG_SPLL_SPREAD_SPECTRUM, SSEN, 1);
+			cg_spll_spread_spectrum_2 =
+				PHM_SET_FIELD(cg_spll_spread_spectrum_2, CG_SPLL_SPREAD_SPECTRUM_2, CLKV, clkV);
+		}
+	}
+
+	sclk->SclkFrequency        = engine_clock;
+	sclk->CgSpllFuncCntl3      = spll_func_cntl_3;
+	sclk->CgSpllFuncCntl4      = spll_func_cntl_4;
+	sclk->SpllSpreadSpectrum   = cg_spll_spread_spectrum;
+	sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
+	sclk->SclkDid              = (uint8_t)dividers.pll_post_divider;
+
+	return 0;
+}
+
+/**
+ * Populates single SMC SCLK structure using the provided engine clock
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    engine_clock the engine clock to use to populate the structure
+ * @param    sclk        the SMC SCLK structure to be populated
+ */
+static int tonga_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
+						uint32_t engine_clock,
+				uint16_t sclk_activity_level_threshold,
+				SMU72_Discrete_GraphicsLevel *graphic_level)
+{
+	int result;
+	uint32_t mvdd;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info =
+			    (struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	result = tonga_calculate_sclk_params(hwmgr, engine_clock, graphic_level);
+
+	/* populate graphics levels*/
+	result = tonga_get_dependecy_volt_by_clk(hwmgr,
+		pptable_info->vdd_dep_on_sclk, engine_clock,
+		&graphic_level->MinVoltage, &mvdd);
+	PP_ASSERT_WITH_CODE((!result),
+		"can not find VDDC voltage value for VDDC "
+		"engine clock dependency table", return result);
+
+	/* SCLK frequency in units of 10KHz*/
+	graphic_level->SclkFrequency = engine_clock;
+	/* Indicates maximum activity level for this performance level. 50% for now*/
+	graphic_level->ActivityLevel = sclk_activity_level_threshold;
+
+	graphic_level->CcPwrDynRm = 0;
+	graphic_level->CcPwrDynRm1 = 0;
+	/* this level can be used if activity is high enough.*/
+	graphic_level->EnabledForActivity = 0;
+	/* this level can be used for throttling.*/
+	graphic_level->EnabledForThrottle = 1;
+	graphic_level->UpHyst = 0;
+	graphic_level->DownHyst = 0;
+	graphic_level->VoltageDownHyst = 0;
+	graphic_level->PowerThrottle = 0;
+
+	data->display_timing.min_clock_in_sr =
+			hwmgr->display_config.min_core_set_clock_in_sr;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkDeepSleep))
+		graphic_level->DeepSleepDivId =
+				smu7_get_sleep_divider_id_from_clock(engine_clock,
+						data->display_timing.min_clock_in_sr);
+
+	/* Default to slow, highest DPM level will be set to PPSMC_DISPLAY_WATERMARK_LOW later.*/
+	graphic_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	if (!result) {
+		/* CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->MinVoltage);*/
+		/* CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->MinVddcPhases);*/
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_US(graphic_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CgSpllFuncCntl3);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CgSpllFuncCntl4);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SpllSpreadSpectrum);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->SpllSpreadSpectrum2);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CcPwrDynRm);
+		CONVERT_FROM_HOST_TO_SMC_UL(graphic_level->CcPwrDynRm1);
+	}
+
+	return result;
+}
+
+/**
+ * Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states
+ *
+ * @param    hwmgr      the address of the hardware manager
+ */
+int tonga_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	struct phm_ppt_v1_pcie_table *pcie_table = pptable_info->pcie_table;
+	uint8_t pcie_entry_count = (uint8_t) data->dpm_table.pcie_speed_table.count;
+	uint32_t level_array_address = smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU72_Discrete_DpmTable, GraphicsLevel);
+
+	uint32_t level_array_size = sizeof(SMU72_Discrete_GraphicsLevel) *
+						SMU72_MAX_LEVELS_GRAPHICS;
+
+	SMU72_Discrete_GraphicsLevel *levels = smu_data->smc_state_table.GraphicsLevel;
+
+	uint32_t i, max_entry;
+	uint8_t highest_pcie_level_enabled = 0;
+	uint8_t lowest_pcie_level_enabled = 0, mid_pcie_level_enabled = 0;
+	uint8_t count = 0;
+	int result = 0;
+
+	memset(levels, 0x00, level_array_size);
+
+	for (i = 0; i < dpm_table->sclk_table.count; i++) {
+		result = tonga_populate_single_graphic_level(hwmgr,
+					dpm_table->sclk_table.dpm_levels[i].value,
+					(uint16_t)smu_data->activity_target[i],
+					&(smu_data->smc_state_table.GraphicsLevel[i]));
+		if (result != 0)
+			return result;
+
+		/* Making sure only DPM level 0-1 have Deep Sleep Div ID populated. */
+		if (i > 1)
+			smu_data->smc_state_table.GraphicsLevel[i].DeepSleepDivId = 0;
+	}
+
+	/* Only enable level 0 for now. */
+	smu_data->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1;
+
+	/* set highest level watermark to high */
+	if (dpm_table->sclk_table.count > 1)
+		smu_data->smc_state_table.GraphicsLevel[dpm_table->sclk_table.count-1].DisplayWatermark =
+			PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	smu_data->smc_state_table.GraphicsDpmLevelCount =
+		(uint8_t)dpm_table->sclk_table.count;
+	data->dpm_level_enable_mask.sclk_dpm_enable_mask =
+		phm_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
+
+	if (pcie_table != NULL) {
+		PP_ASSERT_WITH_CODE((pcie_entry_count >= 1),
+			"There must be 1 or more PCIE levels defined in PPTable.",
+			return -EINVAL);
+		max_entry = pcie_entry_count - 1; /* for indexing, we need to decrement by 1.*/
+		for (i = 0; i < dpm_table->sclk_table.count; i++) {
+			smu_data->smc_state_table.GraphicsLevel[i].pcieDpmLevel =
+				(uint8_t) ((i < max_entry) ? i : max_entry);
+		}
+	} else {
+		if (0 == data->dpm_level_enable_mask.pcie_dpm_enable_mask)
+			printk(KERN_ERR "[ powerplay ] Pcie Dpm Enablemask is 0 !");
+
+		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+					(1<<(highest_pcie_level_enabled+1))) != 0)) {
+			highest_pcie_level_enabled++;
+		}
+
+		while (data->dpm_level_enable_mask.pcie_dpm_enable_mask &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+					(1<<lowest_pcie_level_enabled)) == 0)) {
+			lowest_pcie_level_enabled++;
+		}
+
+		while ((count < highest_pcie_level_enabled) &&
+				((data->dpm_level_enable_mask.pcie_dpm_enable_mask &
+					(1<<(lowest_pcie_level_enabled+1+count))) == 0)) {
+			count++;
+		}
+		mid_pcie_level_enabled = (lowest_pcie_level_enabled+1+count) < highest_pcie_level_enabled ?
+			(lowest_pcie_level_enabled+1+count) : highest_pcie_level_enabled;
+
+
+		/* set pcieDpmLevel to highest_pcie_level_enabled*/
+		for (i = 2; i < dpm_table->sclk_table.count; i++)
+			smu_data->smc_state_table.GraphicsLevel[i].pcieDpmLevel = highest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to lowest_pcie_level_enabled*/
+		smu_data->smc_state_table.GraphicsLevel[0].pcieDpmLevel = lowest_pcie_level_enabled;
+
+		/* set pcieDpmLevel to mid_pcie_level_enabled*/
+		smu_data->smc_state_table.GraphicsLevel[1].pcieDpmLevel = mid_pcie_level_enabled;
+	}
+	/* level count will send to smc once at init smc table and never change*/
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr, level_array_address,
+				(uint8_t *)levels, (uint32_t)level_array_size,
+								SMC_RAM_END);
+
+	return result;
+}
+
+/**
+ * Populates the SMC MCLK structure using the provided memory clock
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    memory_clock the memory clock to use to populate the structure
+ * @param    sclk        the SMC SCLK structure to be populated
+ */
+static int tonga_calculate_mclk_params(
+		struct pp_hwmgr *hwmgr,
+		uint32_t memory_clock,
+		SMU72_Discrete_MemoryLevel *mclk,
+		bool strobe_mode,
+		bool dllStateOn
+		)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	uint32_t dll_cntl = data->clock_registers.vDLL_CNTL;
+	uint32_t mclk_pwrmgt_cntl = data->clock_registers.vMCLK_PWRMGT_CNTL;
+	uint32_t mpll_ad_func_cntl = data->clock_registers.vMPLL_AD_FUNC_CNTL;
+	uint32_t mpll_dq_func_cntl = data->clock_registers.vMPLL_DQ_FUNC_CNTL;
+	uint32_t mpll_func_cntl = data->clock_registers.vMPLL_FUNC_CNTL;
+	uint32_t mpll_func_cntl_1 = data->clock_registers.vMPLL_FUNC_CNTL_1;
+	uint32_t mpll_func_cntl_2 = data->clock_registers.vMPLL_FUNC_CNTL_2;
+	uint32_t mpll_ss1 = data->clock_registers.vMPLL_SS1;
+	uint32_t mpll_ss2 = data->clock_registers.vMPLL_SS2;
+
+	pp_atomctrl_memory_clock_param mpll_param;
+	int result;
+
+	result = atomctrl_get_memory_pll_dividers_si(hwmgr,
+				memory_clock, &mpll_param, strobe_mode);
+	PP_ASSERT_WITH_CODE(
+			!result,
+			"Error retrieving Memory Clock Parameters from VBIOS.",
+			return result);
+
+	/* MPLL_FUNC_CNTL setup*/
+	mpll_func_cntl = PHM_SET_FIELD(mpll_func_cntl, MPLL_FUNC_CNTL, BWCTRL,
+					mpll_param.bw_ctrl);
+
+	/* MPLL_FUNC_CNTL_1 setup*/
+	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
+					MPLL_FUNC_CNTL_1, CLKF,
+					mpll_param.mpll_fb_divider.cl_kf);
+	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
+					MPLL_FUNC_CNTL_1, CLKFRAC,
+					mpll_param.mpll_fb_divider.clk_frac);
+	mpll_func_cntl_1  = PHM_SET_FIELD(mpll_func_cntl_1,
+						MPLL_FUNC_CNTL_1, VCO_MODE,
+						mpll_param.vco_mode);
+
+	/* MPLL_AD_FUNC_CNTL setup*/
+	mpll_ad_func_cntl = PHM_SET_FIELD(mpll_ad_func_cntl,
+					MPLL_AD_FUNC_CNTL, YCLK_POST_DIV,
+					mpll_param.mpll_post_divider);
+
+	if (data->is_memory_gddr5) {
+		/* MPLL_DQ_FUNC_CNTL setup*/
+		mpll_dq_func_cntl  = PHM_SET_FIELD(mpll_dq_func_cntl,
+						MPLL_DQ_FUNC_CNTL, YCLK_SEL,
+						mpll_param.yclk_sel);
+		mpll_dq_func_cntl  = PHM_SET_FIELD(mpll_dq_func_cntl,
+						MPLL_DQ_FUNC_CNTL, YCLK_POST_DIV,
+						mpll_param.mpll_post_divider);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MemorySpreadSpectrumSupport)) {
+		/*
+		 ************************************
+		 Fref = Reference Frequency
+		 NF = Feedback divider ratio
+		 NR = Reference divider ratio
+		 Fnom = Nominal VCO output frequency = Fref * NF / NR
+		 Fs = Spreading Rate
+		 D = Percentage down-spread / 2
+		 Fint = Reference input frequency to PFD = Fref / NR
+		 NS = Spreading rate divider ratio = int(Fint / (2 * Fs))
+		 CLKS = NS - 1 = ISS_STEP_NUM[11:0]
+		 NV = D * Fs / Fnom * 4 * ((Fnom/Fref * NR) ^ 2)
+		 CLKV = 65536 * NV = ISS_STEP_SIZE[25:0]
+		 *************************************
+		 */
+		pp_atomctrl_internal_ss_info ss_info;
+		uint32_t freq_nom;
+		uint32_t tmp;
+		uint32_t reference_clock = atomctrl_get_mpll_reference_clock(hwmgr);
+
+		/* for GDDR5 for all modes and DDR3 */
+		if (1 == mpll_param.qdr)
+			freq_nom = memory_clock * 4 * (1 << mpll_param.mpll_post_divider);
+		else
+			freq_nom = memory_clock * 2 * (1 << mpll_param.mpll_post_divider);
+
+		/* tmp = (freq_nom / reference_clock * reference_divider) ^ 2  Note: S.I. reference_divider = 1*/
+		tmp = (freq_nom / reference_clock);
+		tmp = tmp * tmp;
+
+		if (0 == atomctrl_get_memory_clock_spread_spectrum(hwmgr, freq_nom, &ss_info)) {
+			/* ss_info.speed_spectrum_percentage -- in unit of 0.01% */
+			/* ss.Info.speed_spectrum_rate -- in unit of khz */
+			/* CLKS = reference_clock / (2 * speed_spectrum_rate * reference_divider) * 10 */
+			/*     = reference_clock * 5 / speed_spectrum_rate */
+			uint32_t clks = reference_clock * 5 / ss_info.speed_spectrum_rate;
+
+			/* CLKV = 65536 * speed_spectrum_percentage / 2 * spreadSpecrumRate / freq_nom * 4 / 100000 * ((freq_nom / reference_clock) ^ 2) */
+			/*     = 131 * speed_spectrum_percentage * speed_spectrum_rate / 100 * ((freq_nom / reference_clock) ^ 2) / freq_nom */
+			uint32_t clkv =
+				(uint32_t)((((131 * ss_info.speed_spectrum_percentage *
+							ss_info.speed_spectrum_rate) / 100) * tmp) / freq_nom);
+
+			mpll_ss1 = PHM_SET_FIELD(mpll_ss1, MPLL_SS1, CLKV, clkv);
+			mpll_ss2 = PHM_SET_FIELD(mpll_ss2, MPLL_SS2, CLKS, clks);
+		}
+	}
+
+	/* MCLK_PWRMGT_CNTL setup */
+	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, DLL_SPEED, mpll_param.dll_speed);
+	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK0_PDNB, dllStateOn);
+	mclk_pwrmgt_cntl = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK1_PDNB, dllStateOn);
+
+	/* Save the result data to outpupt memory level structure */
+	mclk->MclkFrequency   = memory_clock;
+	mclk->MpllFuncCntl    = mpll_func_cntl;
+	mclk->MpllFuncCntl_1  = mpll_func_cntl_1;
+	mclk->MpllFuncCntl_2  = mpll_func_cntl_2;
+	mclk->MpllAdFuncCntl  = mpll_ad_func_cntl;
+	mclk->MpllDqFuncCntl  = mpll_dq_func_cntl;
+	mclk->MclkPwrmgtCntl  = mclk_pwrmgt_cntl;
+	mclk->DllCntl         = dll_cntl;
+	mclk->MpllSs1         = mpll_ss1;
+	mclk->MpllSs2         = mpll_ss2;
+
+	return 0;
+}
+
+static uint8_t tonga_get_mclk_frequency_ratio(uint32_t memory_clock,
+		bool strobe_mode)
+{
+	uint8_t mc_para_index;
+
+	if (strobe_mode) {
+		if (memory_clock < 12500)
+			mc_para_index = 0x00;
+		else if (memory_clock > 47500)
+			mc_para_index = 0x0f;
+		else
+			mc_para_index = (uint8_t)((memory_clock - 10000) / 2500);
+	} else {
+		if (memory_clock < 65000)
+			mc_para_index = 0x00;
+		else if (memory_clock > 135000)
+			mc_para_index = 0x0f;
+		else
+			mc_para_index = (uint8_t)((memory_clock - 60000) / 5000);
+	}
+
+	return mc_para_index;
+}
+
+static uint8_t tonga_get_ddr3_mclk_frequency_ratio(uint32_t memory_clock)
+{
+	uint8_t mc_para_index;
+
+	if (memory_clock < 10000)
+		mc_para_index = 0;
+	else if (memory_clock >= 80000)
+		mc_para_index = 0x0f;
+	else
+		mc_para_index = (uint8_t)((memory_clock - 10000) / 5000 + 1);
+
+	return mc_para_index;
+}
+
+
+static int tonga_populate_single_memory_level(
+		struct pp_hwmgr *hwmgr,
+		uint32_t memory_clock,
+		SMU72_Discrete_MemoryLevel *memory_level
+		)
+{
+	uint32_t mvdd = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info =
+			  (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	int result = 0;
+	bool dll_state_on;
+	struct cgs_display_info info = {0};
+	uint32_t mclk_edc_wr_enable_threshold = 40000;
+	uint32_t mclk_stutter_mode_threshold = 30000;
+	uint32_t mclk_edc_enable_threshold = 40000;
+	uint32_t mclk_strobe_mode_threshold = 40000;
+
+	if (NULL != pptable_info->vdd_dep_on_mclk) {
+		result = tonga_get_dependecy_volt_by_clk(hwmgr,
+				pptable_info->vdd_dep_on_mclk,
+				memory_clock,
+				&memory_level->MinVoltage, &mvdd);
+		PP_ASSERT_WITH_CODE(
+			!result,
+			"can not find MinVddc voltage value from memory VDDC "
+			"voltage dependency table",
+			return result);
+	}
+
+	if (data->mvdd_control == SMU7_VOLTAGE_CONTROL_NONE)
+		memory_level->MinMvdd = data->vbios_boot_state.mvdd_bootup_value;
+	else
+		memory_level->MinMvdd = mvdd;
+
+	memory_level->EnabledForThrottle = 1;
+	memory_level->EnabledForActivity = 0;
+	memory_level->UpHyst = 0;
+	memory_level->DownHyst = 100;
+	memory_level->VoltageDownHyst = 0;
+
+	/* Indicates maximum activity level for this performance level.*/
+	memory_level->ActivityLevel = (uint16_t)data->mclk_activity_target;
+	memory_level->StutterEnable = 0;
+	memory_level->StrobeEnable = 0;
+	memory_level->EdcReadEnable = 0;
+	memory_level->EdcWriteEnable = 0;
+	memory_level->RttEnable = 0;
+
+	/* default set to low watermark. Highest level will be set to high later.*/
+	memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+
+	cgs_get_active_displays_info(hwmgr->device, &info);
+	data->display_timing.num_existing_displays = info.display_count;
+
+	if ((mclk_stutter_mode_threshold != 0) &&
+	    (memory_clock <= mclk_stutter_mode_threshold) &&
+	    (!data->is_uvd_enabled)
+	    && (PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL, STUTTER_ENABLE) & 0x1)
+	    && (data->display_timing.num_existing_displays <= 2)
+	    && (data->display_timing.num_existing_displays != 0))
+		memory_level->StutterEnable = 1;
+
+	/* decide strobe mode*/
+	memory_level->StrobeEnable = (mclk_strobe_mode_threshold != 0) &&
+		(memory_clock <= mclk_strobe_mode_threshold);
+
+	/* decide EDC mode and memory clock ratio*/
+	if (data->is_memory_gddr5) {
+		memory_level->StrobeRatio = tonga_get_mclk_frequency_ratio(memory_clock,
+					memory_level->StrobeEnable);
+
+		if ((mclk_edc_enable_threshold != 0) &&
+				(memory_clock > mclk_edc_enable_threshold)) {
+			memory_level->EdcReadEnable = 1;
+		}
+
+		if ((mclk_edc_wr_enable_threshold != 0) &&
+				(memory_clock > mclk_edc_wr_enable_threshold)) {
+			memory_level->EdcWriteEnable = 1;
+		}
+
+		if (memory_level->StrobeEnable) {
+			if (tonga_get_mclk_frequency_ratio(memory_clock, 1) >=
+					((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC7) >> 16) & 0xf)) {
+				dll_state_on = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC5) >> 1) & 0x1) ? 1 : 0;
+			} else {
+				dll_state_on = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC6) >> 1) & 0x1) ? 1 : 0;
+			}
+
+		} else {
+			dll_state_on = data->dll_default_on;
+		}
+	} else {
+		memory_level->StrobeRatio =
+			tonga_get_ddr3_mclk_frequency_ratio(memory_clock);
+		dll_state_on = ((cgs_read_register(hwmgr->device, mmMC_SEQ_MISC5) >> 1) & 0x1) ? 1 : 0;
+	}
+
+	result = tonga_calculate_mclk_params(hwmgr,
+		memory_clock, memory_level, memory_level->StrobeEnable, dll_state_on);
+
+	if (!result) {
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MinMvdd);
+		/* MCLK frequency in units of 10KHz*/
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MclkFrequency);
+		/* Indicates maximum activity level for this performance level.*/
+		CONVERT_FROM_HOST_TO_SMC_US(memory_level->ActivityLevel);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl_1);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllFuncCntl_2);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllAdFuncCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllDqFuncCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MclkPwrmgtCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->DllCntl);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllSs1);
+		CONVERT_FROM_HOST_TO_SMC_UL(memory_level->MpllSs2);
+	}
+
+	return result;
+}
+
+int tonga_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct tonga_smumgr *smu_data =
+			(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	struct smu7_dpm_table *dpm_table = &data->dpm_table;
+	int result;
+
+	/* populate MCLK dpm table to SMU7 */
+	uint32_t level_array_address =
+				smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU72_Discrete_DpmTable, MemoryLevel);
+	uint32_t level_array_size =
+				sizeof(SMU72_Discrete_MemoryLevel) *
+				SMU72_MAX_LEVELS_MEMORY;
+	SMU72_Discrete_MemoryLevel *levels =
+				smu_data->smc_state_table.MemoryLevel;
+	uint32_t i;
+
+	memset(levels, 0x00, level_array_size);
+
+	for (i = 0; i < dpm_table->mclk_table.count; i++) {
+		PP_ASSERT_WITH_CODE((0 != dpm_table->mclk_table.dpm_levels[i].value),
+			"can not populate memory level as memory clock is zero",
+			return -EINVAL);
+		result = tonga_populate_single_memory_level(
+				hwmgr,
+				dpm_table->mclk_table.dpm_levels[i].value,
+				&(smu_data->smc_state_table.MemoryLevel[i]));
+		if (result)
+			return result;
+	}
+
+	/* Only enable level 0 for now.*/
+	smu_data->smc_state_table.MemoryLevel[0].EnabledForActivity = 1;
+
+	/*
+	* in order to prevent MC activity from stutter mode to push DPM up.
+	* the UVD change complements this by putting the MCLK in a higher state
+	* by default such that we are not effected by up threshold or and MCLK DPM latency.
+	*/
+	smu_data->smc_state_table.MemoryLevel[0].ActivityLevel = 0x1F;
+	CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.MemoryLevel[0].ActivityLevel);
+
+	smu_data->smc_state_table.MemoryDpmLevelCount = (uint8_t)dpm_table->mclk_table.count;
+	data->dpm_level_enable_mask.mclk_dpm_enable_mask = phm_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
+	/* set highest level watermark to high*/
+	smu_data->smc_state_table.MemoryLevel[dpm_table->mclk_table.count-1].DisplayWatermark = PPSMC_DISPLAY_WATERMARK_HIGH;
+
+	/* level count will send to smc once at init smc table and never change*/
+	result = smu7_copy_bytes_to_smc(hwmgr->smumgr,
+		level_array_address, (uint8_t *)levels, (uint32_t)level_array_size,
+		SMC_RAM_END);
+
+	return result;
+}
+
+static int tonga_populate_mvdd_value(struct pp_hwmgr *hwmgr,
+				uint32_t mclk, SMIO_Pattern *smio_pattern)
+{
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint32_t i = 0;
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != data->mvdd_control) {
+		/* find mvdd value which clock is more than request */
+		for (i = 0; i < table_info->vdd_dep_on_mclk->count; i++) {
+			if (mclk <= table_info->vdd_dep_on_mclk->entries[i].clk) {
+				/* Always round to higher voltage. */
+				smio_pattern->Voltage =
+				      data->mvdd_voltage_table.entries[i].value;
+				break;
+			}
+		}
+
+		PP_ASSERT_WITH_CODE(i < table_info->vdd_dep_on_mclk->count,
+			"MVDD Voltage is outside the supported range.",
+			return -EINVAL);
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+
+static int tonga_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
+	SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	const struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct pp_atomctrl_clock_dividers_vi dividers;
+
+	SMIO_Pattern voltage_level;
+	uint32_t spll_func_cntl    = data->clock_registers.vCG_SPLL_FUNC_CNTL;
+	uint32_t spll_func_cntl_2  = data->clock_registers.vCG_SPLL_FUNC_CNTL_2;
+	uint32_t dll_cntl          = data->clock_registers.vDLL_CNTL;
+	uint32_t mclk_pwrmgt_cntl  = data->clock_registers.vMCLK_PWRMGT_CNTL;
+
+	/* The ACPI state should not do DPM on DC (or ever).*/
+	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
+
+	table->ACPILevel.MinVoltage =
+			smu_data->smc_state_table.GraphicsLevel[0].MinVoltage;
+
+	/* assign zero for now*/
+	table->ACPILevel.SclkFrequency = atomctrl_get_reference_clock(hwmgr);
+
+	/* get the engine clock dividers for this clock value*/
+	result = atomctrl_get_engine_pll_dividers_vi(hwmgr,
+		table->ACPILevel.SclkFrequency,  &dividers);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+		"Error retrieving Engine Clock dividers from VBIOS.",
+		return result);
+
+	/* divider ID for required SCLK*/
+	table->ACPILevel.SclkDid = (uint8_t)dividers.pll_post_divider;
+	table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
+	table->ACPILevel.DeepSleepDivId = 0;
+
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+					SPLL_PWRON, 0);
+	spll_func_cntl = PHM_SET_FIELD(spll_func_cntl, CG_SPLL_FUNC_CNTL,
+						SPLL_RESET, 1);
+	spll_func_cntl_2 = PHM_SET_FIELD(spll_func_cntl_2, CG_SPLL_FUNC_CNTL_2,
+						SCLK_MUX_SEL, 4);
+
+	table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
+	table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
+	table->ACPILevel.CgSpllFuncCntl3 = data->clock_registers.vCG_SPLL_FUNC_CNTL_3;
+	table->ACPILevel.CgSpllFuncCntl4 = data->clock_registers.vCG_SPLL_FUNC_CNTL_4;
+	table->ACPILevel.SpllSpreadSpectrum = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM;
+	table->ACPILevel.SpllSpreadSpectrum2 = data->clock_registers.vCG_SPLL_SPREAD_SPECTRUM_2;
+	table->ACPILevel.CcPwrDynRm = 0;
+	table->ACPILevel.CcPwrDynRm1 = 0;
+
+
+	/* For various features to be enabled/disabled while this level is active.*/
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.Flags);
+	/* SCLK frequency in units of 10KHz*/
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SclkFrequency);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl3);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CgSpllFuncCntl4);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.SpllSpreadSpectrum2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->ACPILevel.CcPwrDynRm1);
+
+	/* table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases;*/
+	table->MemoryACPILevel.MinVoltage =
+			    smu_data->smc_state_table.MemoryLevel[0].MinVoltage;
+
+	/*  CONVERT_FROM_HOST_TO_SMC_UL(table->MemoryACPILevel.MinVoltage);*/
+
+	if (0 == tonga_populate_mvdd_value(hwmgr, 0, &voltage_level))
+		table->MemoryACPILevel.MinMvdd =
+			PP_HOST_TO_SMC_UL(voltage_level.Voltage * VOLTAGE_SCALE);
+	else
+		table->MemoryACPILevel.MinMvdd = 0;
+
+	/* Force reset on DLL*/
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK0_RESET, 0x1);
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK1_RESET, 0x1);
+
+	/* Disable DLL in ACPIState*/
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK0_PDNB, 0);
+	mclk_pwrmgt_cntl    = PHM_SET_FIELD(mclk_pwrmgt_cntl,
+		MCLK_PWRMGT_CNTL, MRDCK1_PDNB, 0);
+
+	/* Enable DLL bypass signal*/
+	dll_cntl            = PHM_SET_FIELD(dll_cntl,
+		DLL_CNTL, MRDCK0_BYPASS, 0);
+	dll_cntl            = PHM_SET_FIELD(dll_cntl,
+		DLL_CNTL, MRDCK1_BYPASS, 0);
+
+	table->MemoryACPILevel.DllCntl            =
+		PP_HOST_TO_SMC_UL(dll_cntl);
+	table->MemoryACPILevel.MclkPwrmgtCntl     =
+		PP_HOST_TO_SMC_UL(mclk_pwrmgt_cntl);
+	table->MemoryACPILevel.MpllAdFuncCntl     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_AD_FUNC_CNTL);
+	table->MemoryACPILevel.MpllDqFuncCntl     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_DQ_FUNC_CNTL);
+	table->MemoryACPILevel.MpllFuncCntl       =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL);
+	table->MemoryACPILevel.MpllFuncCntl_1     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL_1);
+	table->MemoryACPILevel.MpllFuncCntl_2     =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_FUNC_CNTL_2);
+	table->MemoryACPILevel.MpllSs1            =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_SS1);
+	table->MemoryACPILevel.MpllSs2            =
+		PP_HOST_TO_SMC_UL(data->clock_registers.vMPLL_SS2);
+
+	table->MemoryACPILevel.EnabledForThrottle = 0;
+	table->MemoryACPILevel.EnabledForActivity = 0;
+	table->MemoryACPILevel.UpHyst = 0;
+	table->MemoryACPILevel.DownHyst = 100;
+	table->MemoryACPILevel.VoltageDownHyst = 0;
+	/* Indicates maximum activity level for this performance level.*/
+	table->MemoryACPILevel.ActivityLevel =
+			PP_HOST_TO_SMC_US((uint16_t)data->mclk_activity_target);
+
+	table->MemoryACPILevel.StutterEnable = 0;
+	table->MemoryACPILevel.StrobeEnable = 0;
+	table->MemoryACPILevel.EdcReadEnable = 0;
+	table->MemoryACPILevel.EdcWriteEnable = 0;
+	table->MemoryACPILevel.RttEnable = 0;
+
+	return result;
+}
+
+static int tonga_populate_smc_uvd_level(struct pp_hwmgr *hwmgr,
+					SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+
+	uint8_t count;
+	pp_atomctrl_clock_dividers_vi dividers;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info =
+				(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+						pptable_info->mm_dep_table;
+
+	table->UvdLevelCount = (uint8_t) (mm_table->count);
+	table->UvdBootLevel = 0;
+
+	for (count = 0; count < table->UvdLevelCount; count++) {
+		table->UvdLevel[count].VclkFrequency = mm_table->entries[count].vclk;
+		table->UvdLevel[count].DclkFrequency = mm_table->entries[count].dclk;
+		table->UvdLevel[count].MinVoltage.Vddc =
+			phm_get_voltage_index(pptable_info->vddc_lookup_table,
+						mm_table->entries[count].vddc);
+		table->UvdLevel[count].MinVoltage.VddGfx =
+			(data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) ?
+			phm_get_voltage_index(pptable_info->vddgfx_lookup_table,
+						mm_table->entries[count].vddgfx) : 0;
+		table->UvdLevel[count].MinVoltage.Vddci =
+			phm_get_voltage_id(&data->vddci_voltage_table,
+					     mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		table->UvdLevel[count].MinVoltage.Phases = 1;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(
+					hwmgr,
+					table->UvdLevel[count].VclkFrequency,
+					&dividers);
+
+		PP_ASSERT_WITH_CODE((!result),
+				    "can not find divide id for Vclk clock",
+					return result);
+
+		table->UvdLevel[count].VclkDivider = (uint8_t)dividers.pll_post_divider;
+
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+							  table->UvdLevel[count].DclkFrequency, &dividers);
+		PP_ASSERT_WITH_CODE((!result),
+				    "can not find divide id for Dclk clock",
+					return result);
+
+		table->UvdLevel[count].DclkDivider =
+					(uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].VclkFrequency);
+		CONVERT_FROM_HOST_TO_SMC_UL(table->UvdLevel[count].DclkFrequency);
+	}
+
+	return result;
+
+}
+
+static int tonga_populate_smc_vce_level(struct pp_hwmgr *hwmgr,
+		SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+
+	uint8_t count;
+	pp_atomctrl_clock_dividers_vi dividers;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info =
+			      (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+						     pptable_info->mm_dep_table;
+
+	table->VceLevelCount = (uint8_t) (mm_table->count);
+	table->VceBootLevel = 0;
+
+	for (count = 0; count < table->VceLevelCount; count++) {
+		table->VceLevel[count].Frequency =
+			mm_table->entries[count].eclk;
+		table->VceLevel[count].MinVoltage.Vddc =
+			phm_get_voltage_index(pptable_info->vddc_lookup_table,
+				mm_table->entries[count].vddc);
+		table->VceLevel[count].MinVoltage.VddGfx =
+			(data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) ?
+			phm_get_voltage_index(pptable_info->vddgfx_lookup_table,
+				mm_table->entries[count].vddgfx) : 0;
+		table->VceLevel[count].MinVoltage.Vddci =
+			phm_get_voltage_id(&data->vddci_voltage_table,
+				mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		table->VceLevel[count].MinVoltage.Phases = 1;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+					table->VceLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((!result),
+				"can not find divide id for VCE engine clock",
+				return result);
+
+		table->VceLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->VceLevel[count].Frequency);
+	}
+
+	return result;
+}
+
+static int tonga_populate_smc_acp_level(struct pp_hwmgr *hwmgr,
+		SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+	uint8_t count;
+	pp_atomctrl_clock_dividers_vi dividers;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info =
+			     (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+						    pptable_info->mm_dep_table;
+
+	table->AcpLevelCount = (uint8_t) (mm_table->count);
+	table->AcpBootLevel = 0;
+
+	for (count = 0; count < table->AcpLevelCount; count++) {
+		table->AcpLevel[count].Frequency =
+			pptable_info->mm_dep_table->entries[count].aclk;
+		table->AcpLevel[count].MinVoltage.Vddc =
+			phm_get_voltage_index(pptable_info->vddc_lookup_table,
+			mm_table->entries[count].vddc);
+		table->AcpLevel[count].MinVoltage.VddGfx =
+			(data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) ?
+			phm_get_voltage_index(pptable_info->vddgfx_lookup_table,
+				mm_table->entries[count].vddgfx) : 0;
+		table->AcpLevel[count].MinVoltage.Vddci =
+			phm_get_voltage_id(&data->vddci_voltage_table,
+				mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		table->AcpLevel[count].MinVoltage.Phases = 1;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+			table->AcpLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((!result),
+			"can not find divide id for engine clock", return result);
+
+		table->AcpLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->AcpLevel[count].Frequency);
+	}
+
+	return result;
+}
+
+static int tonga_populate_smc_samu_level(struct pp_hwmgr *hwmgr,
+		SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+	uint8_t count;
+	pp_atomctrl_clock_dividers_vi dividers;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct phm_ppt_v1_information *pptable_info =
+			     (struct phm_ppt_v1_information *)(hwmgr->pptable);
+	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table =
+						    pptable_info->mm_dep_table;
+
+	table->SamuBootLevel = 0;
+	table->SamuLevelCount = (uint8_t) (mm_table->count);
+
+	for (count = 0; count < table->SamuLevelCount; count++) {
+		/* not sure whether we need evclk or not */
+		table->SamuLevel[count].Frequency =
+			pptable_info->mm_dep_table->entries[count].samclock;
+		table->SamuLevel[count].MinVoltage.Vddc =
+			phm_get_voltage_index(pptable_info->vddc_lookup_table,
+				mm_table->entries[count].vddc);
+		table->SamuLevel[count].MinVoltage.VddGfx =
+			(data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) ?
+			phm_get_voltage_index(pptable_info->vddgfx_lookup_table,
+				mm_table->entries[count].vddgfx) : 0;
+		table->SamuLevel[count].MinVoltage.Vddci =
+			phm_get_voltage_id(&data->vddci_voltage_table,
+				mm_table->entries[count].vddc - VDDC_VDDCI_DELTA);
+		table->SamuLevel[count].MinVoltage.Phases = 1;
+
+		/* retrieve divider value for VBIOS */
+		result = atomctrl_get_dfs_pll_dividers_vi(hwmgr,
+					table->SamuLevel[count].Frequency, &dividers);
+		PP_ASSERT_WITH_CODE((!result),
+			"can not find divide id for samu clock", return result);
+
+		table->SamuLevel[count].Divider = (uint8_t)dividers.pll_post_divider;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(table->SamuLevel[count].Frequency);
+	}
+
+	return result;
+}
+
+static int tonga_populate_memory_timing_parameters(
+		struct pp_hwmgr *hwmgr,
+		uint32_t engine_clock,
+		uint32_t memory_clock,
+		struct SMU72_Discrete_MCArbDramTimingTableEntry *arb_regs
+		)
+{
+	uint32_t dramTiming;
+	uint32_t dramTiming2;
+	uint32_t burstTime;
+	int result;
+
+	result = atomctrl_set_engine_dram_timings_rv770(hwmgr,
+				engine_clock, memory_clock);
+
+	PP_ASSERT_WITH_CODE(result == 0,
+		"Error calling VBIOS to set DRAM_TIMING.", return result);
+
+	dramTiming  = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING);
+	dramTiming2 = cgs_read_register(hwmgr->device, mmMC_ARB_DRAM_TIMING2);
+	burstTime = PHM_READ_FIELD(hwmgr->device, MC_ARB_BURST_TIME, STATE0);
+
+	arb_regs->McArbDramTiming  = PP_HOST_TO_SMC_UL(dramTiming);
+	arb_regs->McArbDramTiming2 = PP_HOST_TO_SMC_UL(dramTiming2);
+	arb_regs->McArbBurstTime = (uint8_t)burstTime;
+
+	return 0;
+}
+
+/**
+ * Setup parameters for the MC ARB.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ * This function is to be called from the SetPowerState table.
+ */
+static int tonga_program_memory_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	int result = 0;
+	SMU72_Discrete_MCArbDramTimingTable  arb_regs;
+	uint32_t i, j;
+
+	memset(&arb_regs, 0x00, sizeof(SMU72_Discrete_MCArbDramTimingTable));
+
+	for (i = 0; i < data->dpm_table.sclk_table.count; i++) {
+		for (j = 0; j < data->dpm_table.mclk_table.count; j++) {
+			result = tonga_populate_memory_timing_parameters
+				(hwmgr, data->dpm_table.sclk_table.dpm_levels[i].value,
+				 data->dpm_table.mclk_table.dpm_levels[j].value,
+				 &arb_regs.entries[i][j]);
+
+			if (result)
+				break;
+		}
+	}
+
+	if (!result) {
+		result = smu7_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				smu_data->smu7_data.arb_table_start,
+				(uint8_t *)&arb_regs,
+				sizeof(SMU72_Discrete_MCArbDramTimingTable),
+				SMC_RAM_END
+				);
+	}
+
+	return result;
+}
+
+static int tonga_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	int result = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	table->GraphicsBootLevel = 0;
+	table->MemoryBootLevel = 0;
+
+	/* find boot level from dpm table*/
+	result = phm_find_boot_level(&(data->dpm_table.sclk_table),
+	data->vbios_boot_state.sclk_bootup_value,
+	(uint32_t *)&(smu_data->smc_state_table.GraphicsBootLevel));
+
+	if (result != 0) {
+		smu_data->smc_state_table.GraphicsBootLevel = 0;
+		printk(KERN_ERR "[powerplay] VBIOS did not find boot engine "
+				"clock value in dependency table. "
+				"Using Graphics DPM level 0 !");
+		result = 0;
+	}
+
+	result = phm_find_boot_level(&(data->dpm_table.mclk_table),
+		data->vbios_boot_state.mclk_bootup_value,
+		(uint32_t *)&(smu_data->smc_state_table.MemoryBootLevel));
+
+	if (result != 0) {
+		smu_data->smc_state_table.MemoryBootLevel = 0;
+		printk(KERN_ERR "[powerplay] VBIOS did not find boot "
+				"engine clock value in dependency table."
+				"Using Memory DPM level 0 !");
+		result = 0;
+	}
+
+	table->BootVoltage.Vddc =
+		phm_get_voltage_id(&(data->vddc_voltage_table),
+			data->vbios_boot_state.vddc_bootup_value);
+	table->BootVoltage.VddGfx =
+		phm_get_voltage_id(&(data->vddgfx_voltage_table),
+			data->vbios_boot_state.vddgfx_bootup_value);
+	table->BootVoltage.Vddci =
+		phm_get_voltage_id(&(data->vddci_voltage_table),
+			data->vbios_boot_state.vddci_bootup_value);
+	table->BootMVdd = data->vbios_boot_state.mvdd_bootup_value;
+
+	CONVERT_FROM_HOST_TO_SMC_US(table->BootMVdd);
+
+	return result;
+}
+
+static int tonga_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
+{
+	uint32_t ro, efuse, efuse2, clock_freq, volt_without_cks,
+			volt_with_cks, value;
+	uint16_t clock_freq_u16;
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	uint8_t type, i, j, cks_setting, stretch_amount, stretch_amount2,
+			volt_offset = 0;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
+			table_info->vdd_dep_on_sclk;
+	uint32_t hw_revision, dev_id;
+	struct cgs_system_info sys_info = {0};
+
+	stretch_amount = (uint8_t)table_info->cac_dtp_table->usClockStretchAmount;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_REV;
+	cgs_query_system_info(hwmgr->device, &sys_info);
+	hw_revision = (uint32_t)sys_info.value;
+
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_DEV;
+	cgs_query_system_info(hwmgr->device, &sys_info);
+	dev_id = (uint32_t)sys_info.value;
+
+	/* Read SMU_Eefuse to read and calculate RO and determine
+	 * if the part is SS or FF. if RO >= 1660MHz, part is FF.
+	 */
+	efuse = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixSMU_EFUSE_0 + (146 * 4));
+	efuse2 = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixSMU_EFUSE_0 + (148 * 4));
+	efuse &= 0xFF000000;
+	efuse = efuse >> 24;
+	efuse2 &= 0xF;
+
+	if (efuse2 == 1)
+		ro = (2300 - 1350) * efuse / 255 + 1350;
+	else
+		ro = (2500 - 1000) * efuse / 255 + 1000;
+
+	if (ro >= 1660)
+		type = 0;
+	else
+		type = 1;
+
+	/* Populate Stretch amount */
+	smu_data->smc_state_table.ClockStretcherAmount = stretch_amount;
+
+
+	/* Populate Sclk_CKS_masterEn0_7 and Sclk_voltageOffset */
+	for (i = 0; i < sclk_table->count; i++) {
+		smu_data->smc_state_table.Sclk_CKS_masterEn0_7 |=
+				sclk_table->entries[i].cks_enable << i;
+		if (ASICID_IS_TONGA_P(dev_id, hw_revision)) {
+			volt_without_cks = (uint32_t)((7732 + 60 - ro - 20838 *
+				(sclk_table->entries[i].clk/100) / 10000) * 1000 /
+				(8730 - (5301 * (sclk_table->entries[i].clk/100) / 1000)));
+			volt_with_cks = (uint32_t)((5250 + 51 - ro - 2404 *
+				(sclk_table->entries[i].clk/100) / 100000) * 1000 /
+				(6146 - (3193 * (sclk_table->entries[i].clk/100) / 1000)));
+		} else {
+			volt_without_cks = (uint32_t)((14041 *
+				(sclk_table->entries[i].clk/100) / 10000 + 3571 + 75 - ro) * 1000 /
+				(4026 - (13924 * (sclk_table->entries[i].clk/100) / 10000)));
+			volt_with_cks = (uint32_t)((13946 *
+				(sclk_table->entries[i].clk/100) / 10000 + 3320 + 45 - ro) * 1000 /
+				(3664 - (11454 * (sclk_table->entries[i].clk/100) / 10000)));
+		}
+		if (volt_without_cks >= volt_with_cks)
+			volt_offset = (uint8_t)(((volt_without_cks - volt_with_cks +
+					sclk_table->entries[i].cks_voffset) * 100 / 625) + 1);
+		smu_data->smc_state_table.Sclk_voltageOffset[i] = volt_offset;
+	}
+
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			STRETCH_ENABLE, 0x0);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			masterReset, 0x1);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			staticEnable, 0x1);
+	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, PWR_CKS_ENABLE,
+			masterReset, 0x0);
+
+	/* Populate CKS Lookup Table */
+	if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5)
+		stretch_amount2 = 0;
+	else if (stretch_amount == 3 || stretch_amount == 4)
+		stretch_amount2 = 1;
+	else {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_ClockStretcher);
+		PP_ASSERT_WITH_CODE(false,
+				"Stretch Amount in PPTable not supported\n",
+				return -EINVAL);
+	}
+
+	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixPWR_CKS_CNTL);
+	value &= 0xFFC2FF87;
+	smu_data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].minFreq =
+			tonga_clock_stretcher_lookup_table[stretch_amount2][0];
+	smu_data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].maxFreq =
+			tonga_clock_stretcher_lookup_table[stretch_amount2][1];
+	clock_freq_u16 = (uint16_t)(PP_SMC_TO_HOST_UL(smu_data->smc_state_table.
+			GraphicsLevel[smu_data->smc_state_table.GraphicsDpmLevelCount - 1].
+			SclkFrequency) / 100);
+	if (tonga_clock_stretcher_lookup_table[stretch_amount2][0] <
+			clock_freq_u16 &&
+	    tonga_clock_stretcher_lookup_table[stretch_amount2][1] >
+			clock_freq_u16) {
+		/* Program PWR_CKS_CNTL. CKS_USE_FOR_LOW_FREQ */
+		value |= (tonga_clock_stretcher_lookup_table[stretch_amount2][3]) << 16;
+		/* Program PWR_CKS_CNTL. CKS_LDO_REFSEL */
+		value |= (tonga_clock_stretcher_lookup_table[stretch_amount2][2]) << 18;
+		/* Program PWR_CKS_CNTL. CKS_STRETCH_AMOUNT */
+		value |= (tonga_clock_stretch_amount_conversion
+				[tonga_clock_stretcher_lookup_table[stretch_amount2][3]]
+				 [stretch_amount]) << 3;
+	}
+	CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.CKS_LOOKUPTable.
+			CKS_LOOKUPTableEntry[0].minFreq);
+	CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.CKS_LOOKUPTable.
+			CKS_LOOKUPTableEntry[0].maxFreq);
+	smu_data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting =
+			tonga_clock_stretcher_lookup_table[stretch_amount2][2] & 0x7F;
+	smu_data->smc_state_table.CKS_LOOKUPTable.CKS_LOOKUPTableEntry[0].setting |=
+			(tonga_clock_stretcher_lookup_table[stretch_amount2][3]) << 7;
+
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixPWR_CKS_CNTL, value);
+
+	/* Populate DDT Lookup Table */
+	for (i = 0; i < 4; i++) {
+		/* Assign the minimum and maximum VID stored
+		 * in the last row of Clock Stretcher Voltage Table.
+		 */
+		smu_data->smc_state_table.ClockStretcherDataTable.
+		ClockStretcherDataTableEntry[i].minVID =
+				(uint8_t) tonga_clock_stretcher_ddt_table[type][i][2];
+		smu_data->smc_state_table.ClockStretcherDataTable.
+		ClockStretcherDataTableEntry[i].maxVID =
+				(uint8_t) tonga_clock_stretcher_ddt_table[type][i][3];
+		/* Loop through each SCLK and check the frequency
+		 * to see if it lies within the frequency for clock stretcher.
+		 */
+		for (j = 0; j < smu_data->smc_state_table.GraphicsDpmLevelCount; j++) {
+			cks_setting = 0;
+			clock_freq = PP_SMC_TO_HOST_UL(
+					smu_data->smc_state_table.GraphicsLevel[j].SclkFrequency);
+			/* Check the allowed frequency against the sclk level[j].
+			 *  Sclk's endianness has already been converted,
+			 *  and it's in 10Khz unit,
+			 *  as opposed to Data table, which is in Mhz unit.
+			 */
+			if (clock_freq >= tonga_clock_stretcher_ddt_table[type][i][0] * 100) {
+				cks_setting |= 0x2;
+				if (clock_freq < tonga_clock_stretcher_ddt_table[type][i][1] * 100)
+					cks_setting |= 0x1;
+			}
+			smu_data->smc_state_table.ClockStretcherDataTable.
+			ClockStretcherDataTableEntry[i].setting |= cks_setting << (j * 2);
+		}
+		CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.
+				ClockStretcherDataTable.
+				ClockStretcherDataTableEntry[i].setting);
+	}
+
+	value = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixPWR_CKS_CNTL);
+	value &= 0xFFFFFFFE;
+	cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+					ixPWR_CKS_CNTL, value);
+
+	return 0;
+}
+
+/**
+ * Populates the SMC VRConfig field in DPM table.
+ *
+ * @param    hwmgr      the address of the hardware manager
+ * @param    table     the SMC DPM table structure to be populated
+ * @return   always 0
+ */
+static int tonga_populate_vr_config(struct pp_hwmgr *hwmgr,
+			SMU72_Discrete_DpmTable *table)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint16_t config;
+
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vdd_gfx_control) {
+		/*  Splitted mode */
+		config = VR_SVI2_PLANE_1;
+		table->VRConfig |= (config<<VRCONF_VDDGFX_SHIFT);
+
+		if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+			config = VR_SVI2_PLANE_2;
+			table->VRConfig |= config;
+		} else {
+			printk(KERN_ERR "[ powerplay ] VDDC and VDDGFX should "
+				"be both on SVI2 control in splitted mode !\n");
+		}
+	} else {
+		/* Merged mode  */
+		config = VR_MERGED_WITH_VDDC;
+		table->VRConfig |= (config<<VRCONF_VDDGFX_SHIFT);
+
+		/* Set Vddc Voltage Controller  */
+		if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->voltage_control) {
+			config = VR_SVI2_PLANE_1;
+			table->VRConfig |= config;
+		} else {
+			printk(KERN_ERR "[ powerplay ] VDDC should be on "
+					"SVI2 control in merged mode !\n");
+		}
+	}
+
+	/* Set Vddci Voltage Controller  */
+	if (SMU7_VOLTAGE_CONTROL_BY_SVID2 == data->vddci_control) {
+		config = VR_SVI2_PLANE_2;  /* only in merged mode */
+		table->VRConfig |= (config<<VRCONF_VDDCI_SHIFT);
+	} else if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->vddci_control) {
+		config = VR_SMIO_PATTERN_1;
+		table->VRConfig |= (config<<VRCONF_VDDCI_SHIFT);
+	}
+
+	/* Set Mvdd Voltage Controller */
+	if (SMU7_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
+		config = VR_SMIO_PATTERN_2;
+		table->VRConfig |= (config<<VRCONF_MVDD_SHIFT);
+	}
+
+	return 0;
+}
+
+
+/**
+ * Initialize the ARB DRAM timing table's index field.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+static int tonga_init_arb_table_index(struct pp_smumgr *smumgr)
+{
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(smumgr->backend);
+	uint32_t tmp;
+	int result;
+
+	/*
+	* This is a read-modify-write on the first byte of the ARB table.
+	* The first byte in the SMU72_Discrete_MCArbDramTimingTable structure
+	* is the field 'current'.
+	* This solution is ugly, but we never write the whole table only
+	* individual fields in it.
+	* In reality this field should not be in that structure
+	* but in a soft register.
+	*/
+	result = smu7_read_smc_sram_dword(smumgr,
+				smu_data->smu7_data.arb_table_start, &tmp, SMC_RAM_END);
+
+	if (result != 0)
+		return result;
+
+	tmp &= 0x00FFFFFF;
+	tmp |= ((uint32_t)MC_CG_ARB_FREQ_F1) << 24;
+
+	return smu7_write_smc_sram_dword(smumgr,
+			smu_data->smu7_data.arb_table_start, tmp, SMC_RAM_END);
+}
+
+
+static int tonga_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	const struct tonga_pt_defaults *defaults = smu_data->power_tune_defaults;
+	SMU72_Discrete_DpmTable  *dpm_table = &(smu_data->smc_state_table);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct phm_cac_tdp_table *cac_dtp_table = table_info->cac_dtp_table;
+	int  i, j, k;
+	const uint16_t *pdef1, *pdef2;
+
+	dpm_table->DefaultTdp = PP_HOST_TO_SMC_US(
+			(uint16_t)(cac_dtp_table->usTDP * 256));
+	dpm_table->TargetTdp = PP_HOST_TO_SMC_US(
+			(uint16_t)(cac_dtp_table->usConfigurableTDP * 256));
+
+	PP_ASSERT_WITH_CODE(cac_dtp_table->usTargetOperatingTemp <= 255,
+			"Target Operating Temp is out of Range !",
+			);
+
+	dpm_table->GpuTjMax = (uint8_t)(cac_dtp_table->usTargetOperatingTemp);
+	dpm_table->GpuTjHyst = 8;
+
+	dpm_table->DTEAmbientTempBase = defaults->dte_ambient_temp_base;
+
+	dpm_table->BAPM_TEMP_GRADIENT =
+				PP_HOST_TO_SMC_UL(defaults->bamp_temp_gradient);
+	pdef1 = defaults->bapmti_r;
+	pdef2 = defaults->bapmti_rc;
+
+	for (i = 0; i < SMU72_DTE_ITERATIONS; i++) {
+		for (j = 0; j < SMU72_DTE_SOURCES; j++) {
+			for (k = 0; k < SMU72_DTE_SINKS; k++) {
+				dpm_table->BAPMTI_R[i][j][k] =
+						PP_HOST_TO_SMC_US(*pdef1);
+				dpm_table->BAPMTI_RC[i][j][k] =
+						PP_HOST_TO_SMC_US(*pdef2);
+				pdef1++;
+				pdef2++;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int tonga_populate_svi_load_line(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	const struct tonga_pt_defaults *defaults = smu_data->power_tune_defaults;
+
+	smu_data->power_tune_table.SviLoadLineEn = defaults->svi_load_line_en;
+	smu_data->power_tune_table.SviLoadLineVddC = defaults->svi_load_line_vddC;
+	smu_data->power_tune_table.SviLoadLineTrimVddC = 3;
+	smu_data->power_tune_table.SviLoadLineOffsetVddC = 0;
+
+	return 0;
+}
+
+static int tonga_populate_tdc_limit(struct pp_hwmgr *hwmgr)
+{
+	uint16_t tdc_limit;
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	const struct tonga_pt_defaults *defaults = smu_data->power_tune_defaults;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	/* TDC number of fraction bits are changed from 8 to 7
+	 * for Fiji as requested by SMC team
+	 */
+	tdc_limit = (uint16_t)(table_info->cac_dtp_table->usTDC * 256);
+	smu_data->power_tune_table.TDC_VDDC_PkgLimit =
+			CONVERT_FROM_HOST_TO_SMC_US(tdc_limit);
+	smu_data->power_tune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
+			defaults->tdc_vddc_throttle_release_limit_perc;
+	smu_data->power_tune_table.TDC_MAWt = defaults->tdc_mawt;
+
+	return 0;
+}
+
+static int tonga_populate_dw8(struct pp_hwmgr *hwmgr, uint32_t fuse_table_offset)
+{
+	struct tonga_smumgr *smu_data =
+			(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	const struct tonga_pt_defaults *defaults = smu_data->power_tune_defaults;
+	uint32_t temp;
+
+	if (smu7_read_smc_sram_dword(hwmgr->smumgr,
+			fuse_table_offset +
+			offsetof(SMU72_Discrete_PmFuses, TdcWaterfallCtl),
+			(uint32_t *)&temp, SMC_RAM_END))
+		PP_ASSERT_WITH_CODE(false,
+				"Attempt to read PmFuses.DW6 "
+				"(SviLoadLineEn) from SMC Failed !",
+				return -EINVAL);
+	else
+		smu_data->power_tune_table.TdcWaterfallCtl = defaults->tdc_waterfall_ctl;
+
+	return 0;
+}
+
+static int tonga_populate_temperature_scaler(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		smu_data->power_tune_table.LPMLTemperatureScaler[i] = 0;
+
+	return 0;
+}
+
+static int tonga_populate_fuzzy_fan(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(hwmgr->smumgr->backend);
+
+	if ((hwmgr->thermal_controller.advanceFanControlParameters.
+			usFanOutputSensitivity & (1 << 15)) ||
+		(hwmgr->thermal_controller.advanceFanControlParameters.usFanOutputSensitivity == 0))
+		hwmgr->thermal_controller.advanceFanControlParameters.
+		usFanOutputSensitivity = hwmgr->thermal_controller.
+			advanceFanControlParameters.usDefaultFanOutputSensitivity;
+
+	smu_data->power_tune_table.FuzzyFan_PwmSetDelta =
+			PP_HOST_TO_SMC_US(hwmgr->thermal_controller.
+					advanceFanControlParameters.usFanOutputSensitivity);
+	return 0;
+}
+
+static int tonga_populate_gnb_lpml(struct pp_hwmgr *hwmgr)
+{
+	int i;
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+
+	/* Currently not used. Set all to zero. */
+	for (i = 0; i < 16; i++)
+		smu_data->power_tune_table.GnbLPML[i] = 0;
+
+	return 0;
+}
+
+static int tonga_min_max_vgnb_lpml_id_from_bapm_vddc(struct pp_hwmgr *hwmgr)
+{
+	return 0;
+}
+
+static int tonga_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	uint16_t hi_sidd = smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd;
+	uint16_t lo_sidd = smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd;
+	struct phm_cac_tdp_table *cac_table = table_info->cac_dtp_table;
+
+	hi_sidd = (uint16_t)(cac_table->usHighCACLeakage / 100 * 256);
+	lo_sidd = (uint16_t)(cac_table->usLowCACLeakage / 100 * 256);
+
+	smu_data->power_tune_table.BapmVddCBaseLeakageHiSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(hi_sidd);
+	smu_data->power_tune_table.BapmVddCBaseLeakageLoSidd =
+			CONVERT_FROM_HOST_TO_SMC_US(lo_sidd);
+
+	return 0;
+}
+
+static int tonga_populate_pm_fuses(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t pm_fuse_table_offset;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PowerContainment)) {
+		if (smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, PmFuseTable),
+				&pm_fuse_table_offset, SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+				"Attempt to get pm_fuse_table_offset Failed !",
+				return -EINVAL);
+
+		/* DW6 */
+		if (tonga_populate_svi_load_line(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+				"Attempt to populate SviLoadLine Failed !",
+				return -EINVAL);
+		/* DW7 */
+		if (tonga_populate_tdc_limit(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to populate TDCLimit Failed !",
+					return -EINVAL);
+		/* DW8 */
+		if (tonga_populate_dw8(hwmgr, pm_fuse_table_offset))
+			PP_ASSERT_WITH_CODE(false,
+				"Attempt to populate TdcWaterfallCtl Failed !",
+				return -EINVAL);
+
+		/* DW9-DW12 */
+		if (tonga_populate_temperature_scaler(hwmgr) != 0)
+			PP_ASSERT_WITH_CODE(false,
+				"Attempt to populate LPMLTemperatureScaler Failed !",
+				return -EINVAL);
+
+		/* DW13-DW14 */
+		if (tonga_populate_fuzzy_fan(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+				"Attempt to populate Fuzzy Fan "
+				"Control parameters Failed !",
+				return -EINVAL);
+
+		/* DW15-DW18 */
+		if (tonga_populate_gnb_lpml(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+				"Attempt to populate GnbLPML Failed !",
+				return -EINVAL);
+
+		/* DW19 */
+		if (tonga_min_max_vgnb_lpml_id_from_bapm_vddc(hwmgr))
+			PP_ASSERT_WITH_CODE(false,
+				"Attempt to populate GnbLPML "
+				"Min and Max Vid Failed !",
+				return -EINVAL);
+
+		/* DW20 */
+		if (tonga_populate_bapm_vddc_base_leakage_sidd(hwmgr))
+			PP_ASSERT_WITH_CODE(
+				false,
+				"Attempt to populate BapmVddCBaseLeakage "
+				"Hi and Lo Sidd Failed !",
+				return -EINVAL);
+
+		if (smu7_copy_bytes_to_smc(hwmgr->smumgr, pm_fuse_table_offset,
+				(uint8_t *)&smu_data->power_tune_table,
+				sizeof(struct SMU72_Discrete_PmFuses), SMC_RAM_END))
+			PP_ASSERT_WITH_CODE(false,
+					"Attempt to download PmFuseTable Failed !",
+					return -EINVAL);
+	}
+	return 0;
+}
+
+static int tonga_populate_mc_reg_address(struct pp_smumgr *smumgr,
+				 SMU72_Discrete_MCRegisters *mc_reg_table)
+{
+	const struct tonga_smumgr *smu_data = (struct tonga_smumgr *)smumgr->backend;
+
+	uint32_t i, j;
+
+	for (i = 0, j = 0; j < smu_data->mc_reg_table.last; j++) {
+		if (smu_data->mc_reg_table.validflag & 1<<j) {
+			PP_ASSERT_WITH_CODE(
+				i < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE,
+				"Index of mc_reg_table->address[] array "
+				"out of boundary",
+				return -EINVAL);
+			mc_reg_table->address[i].s0 =
+				PP_HOST_TO_SMC_US(smu_data->mc_reg_table.mc_reg_address[j].s0);
+			mc_reg_table->address[i].s1 =
+				PP_HOST_TO_SMC_US(smu_data->mc_reg_table.mc_reg_address[j].s1);
+			i++;
+		}
+	}
+
+	mc_reg_table->last = (uint8_t)i;
+
+	return 0;
+}
+
+/*convert register values from driver to SMC format */
+static void tonga_convert_mc_registers(
+	const struct tonga_mc_reg_entry *entry,
+	SMU72_Discrete_MCRegisterSet *data,
+	uint32_t num_entries, uint32_t valid_flag)
+{
+	uint32_t i, j;
+
+	for (i = 0, j = 0; j < num_entries; j++) {
+		if (valid_flag & 1<<j) {
+			data->value[i] = PP_HOST_TO_SMC_UL(entry->mc_data[j]);
+			i++;
+		}
+	}
+}
+
+static int tonga_convert_mc_reg_table_entry_to_smc(
+		struct pp_smumgr *smumgr,
+		const uint32_t memory_clock,
+		SMU72_Discrete_MCRegisterSet *mc_reg_table_data
+		)
+{
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(smumgr->backend);
+	uint32_t i = 0;
+
+	for (i = 0; i < smu_data->mc_reg_table.num_entries; i++) {
+		if (memory_clock <=
+			smu_data->mc_reg_table.mc_reg_table_entry[i].mclk_max) {
+			break;
+		}
+	}
+
+	if ((i == smu_data->mc_reg_table.num_entries) && (i > 0))
+		--i;
+
+	tonga_convert_mc_registers(&smu_data->mc_reg_table.mc_reg_table_entry[i],
+				mc_reg_table_data, smu_data->mc_reg_table.last,
+				smu_data->mc_reg_table.validflag);
+
+	return 0;
+}
+
+static int tonga_convert_mc_reg_table_to_smc(struct pp_hwmgr *hwmgr,
+		SMU72_Discrete_MCRegisters *mc_regs)
+{
+	int result = 0;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	int res;
+	uint32_t i;
+
+	for (i = 0; i < data->dpm_table.mclk_table.count; i++) {
+		res = tonga_convert_mc_reg_table_entry_to_smc(
+				hwmgr->smumgr,
+				data->dpm_table.mclk_table.dpm_levels[i].value,
+				&mc_regs->data[i]
+				);
+
+		if (0 != res)
+			result = res;
+	}
+
+	return result;
+}
+
+static int tonga_update_and_upload_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(smumgr->backend);
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	uint32_t address;
+	int32_t result;
+
+	if (0 == (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK))
+		return 0;
+
+
+	memset(&smu_data->mc_regs, 0, sizeof(SMU72_Discrete_MCRegisters));
+
+	result = tonga_convert_mc_reg_table_to_smc(hwmgr, &(smu_data->mc_regs));
+
+	if (result != 0)
+		return result;
+
+
+	address = smu_data->smu7_data.mc_reg_table_start +
+			(uint32_t)offsetof(SMU72_Discrete_MCRegisters, data[0]);
+
+	return  smu7_copy_bytes_to_smc(
+			hwmgr->smumgr, address,
+			(uint8_t *)&smu_data->mc_regs.data[0],
+			sizeof(SMU72_Discrete_MCRegisterSet) *
+			data->dpm_table.mclk_table.count,
+			SMC_RAM_END);
+}
+
+static int tonga_populate_initial_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct pp_smumgr *smumgr = hwmgr->smumgr;
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(smumgr->backend);
+
+	memset(&smu_data->mc_regs, 0x00, sizeof(SMU72_Discrete_MCRegisters));
+	result = tonga_populate_mc_reg_address(smumgr, &(smu_data->mc_regs));
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize MCRegTable for the MC register addresses !",
+		return result;);
+
+	result = tonga_convert_mc_reg_table_to_smc(hwmgr, &smu_data->mc_regs);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize MCRegTable for driver state !",
+		return result;);
+
+	return smu7_copy_bytes_to_smc(smumgr, smu_data->smu7_data.mc_reg_table_start,
+			(uint8_t *)&smu_data->mc_regs, sizeof(SMU72_Discrete_MCRegisters), SMC_RAM_END);
+}
+
+static void tonga_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	struct  phm_ppt_v1_information *table_info =
+			(struct  phm_ppt_v1_information *)(hwmgr->pptable);
+
+	if (table_info &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID <= POWERTUNE_DEFAULT_SET_MAX &&
+			table_info->cac_dtp_table->usPowerTuneDataSetID)
+		smu_data->power_tune_defaults =
+				&tonga_power_tune_data_set_array
+				[table_info->cac_dtp_table->usPowerTuneDataSetID - 1];
+	else
+		smu_data->power_tune_defaults = &tonga_power_tune_data_set_array[0];
+}
+
+/**
+ * Initializes the SMC table and uploads it
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @param    pInput  the pointer to input data (PowerState)
+ * @return   always 0
+ */
+int tonga_init_smc_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct tonga_smumgr *smu_data =
+			(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	SMU72_Discrete_DpmTable *table = &(smu_data->smc_state_table);
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	uint8_t i;
+	pp_atomctrl_gpio_pin_assignment gpio_pin_assignment;
+
+
+	memset(&(smu_data->smc_state_table), 0x00, sizeof(smu_data->smc_state_table));
+
+	tonga_initialize_power_tune_defaults(hwmgr);
+
+	if (SMU7_VOLTAGE_CONTROL_NONE != data->voltage_control)
+		tonga_populate_smc_voltage_tables(hwmgr, table);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
+
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StepVddc))
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
+
+	if (data->is_memory_gddr5)
+		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
+
+	i = PHM_READ_FIELD(hwmgr->device, CC_MC_MAX_CHANNEL, NOOFCHAN);
+
+	if (i == 1 || i == 0)
+		table->SystemFlags |= 0x40;
+
+	if (data->ulv_supported && table_info->us_ulv_voltage_offset) {
+		result = tonga_populate_ulv_state(hwmgr, table);
+		PP_ASSERT_WITH_CODE(!result,
+			"Failed to initialize ULV state !",
+			return result;);
+
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+			ixCG_ULV_PARAMETER, 0x40035);
+	}
+
+	result = tonga_populate_smc_link_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize Link Level !", return result);
+
+	result = tonga_populate_all_graphic_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize Graphics Level !", return result);
+
+	result = tonga_populate_all_memory_levels(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize Memory Level !", return result);
+
+	result = tonga_populate_smc_acpi_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize ACPI Level !", return result);
+
+	result = tonga_populate_smc_vce_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize VCE Level !", return result);
+
+	result = tonga_populate_smc_acp_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize ACP Level !", return result);
+
+	result = tonga_populate_smc_samu_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize SAMU Level !", return result);
+
+	/* Since only the initial state is completely set up at this
+	* point (the other states are just copies of the boot state) we only
+	* need to populate the  ARB settings for the initial state.
+	*/
+	result = tonga_program_memory_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to Write ARB settings for the initial state.",
+		return result;);
+
+	result = tonga_populate_smc_uvd_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize UVD Level !", return result);
+
+	result = tonga_populate_smc_boot_level(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to initialize Boot Level !", return result);
+
+	tonga_populate_bapm_parameters_in_dpm_table(hwmgr);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to populate BAPM Parameters !", return result);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ClockStretcher)) {
+		result = tonga_populate_clock_stretcher_data_table(hwmgr);
+		PP_ASSERT_WITH_CODE(!result,
+			"Failed to populate Clock Stretcher Data Table !",
+			return result;);
+	}
+	table->GraphicsVoltageChangeEnable  = 1;
+	table->GraphicsThermThrottleEnable  = 1;
+	table->GraphicsInterval = 1;
+	table->VoltageInterval  = 1;
+	table->ThermalInterval  = 1;
+	table->TemperatureLimitHigh =
+		table_info->cac_dtp_table->usTargetOperatingTemp *
+		SMU7_Q88_FORMAT_CONVERSION_UNIT;
+	table->TemperatureLimitLow =
+		(table_info->cac_dtp_table->usTargetOperatingTemp - 1) *
+		SMU7_Q88_FORMAT_CONVERSION_UNIT;
+	table->MemoryVoltageChangeEnable  = 1;
+	table->MemoryInterval  = 1;
+	table->VoltageResponseTime  = 0;
+	table->PhaseResponseTime  = 0;
+	table->MemoryThermThrottleEnable  = 1;
+
+	/*
+	* Cail reads current link status and reports it as cap (we cannot
+	* change this due to some previous issues we had)
+	* SMC drops the link status to lowest level after enabling
+	* DPM by PowerPlay. After pnp or toggling CF, driver gets reloaded again
+	* but this time Cail reads current link status which was set to low by
+	* SMC and reports it as cap to powerplay
+	* To avoid it, we set PCIeBootLinkLevel to highest dpm level
+	*/
+	PP_ASSERT_WITH_CODE((1 <= data->dpm_table.pcie_speed_table.count),
+			"There must be 1 or more PCIE levels defined in PPTable.",
+			return -EINVAL);
+
+	table->PCIeBootLinkLevel = (uint8_t) (data->dpm_table.pcie_speed_table.count);
+
+	table->PCIeGenInterval  = 1;
+
+	result = tonga_populate_vr_config(hwmgr, table);
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to populate VRConfig setting !", return result);
+
+	table->ThermGpio  = 17;
+	table->SclkStepSize = 0x4000;
+
+	if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_VRHOT_GPIO_PINID,
+						&gpio_pin_assignment)) {
+		table->VRHotGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_RegulatorHot);
+	} else {
+		table->VRHotGpio = SMU7_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_RegulatorHot);
+	}
+
+	if (atomctrl_get_pp_assign_pin(hwmgr, PP_AC_DC_SWITCH_GPIO_PINID,
+						&gpio_pin_assignment)) {
+		table->AcDcGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition);
+	} else {
+		table->AcDcGpio = SMU7_UNUSED_GPIO_PIN;
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition);
+	}
+
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_Falcon_QuickTransition);
+
+	if (0) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_AutomaticDCTransition);
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_Falcon_QuickTransition);
+	}
+
+	if (atomctrl_get_pp_assign_pin(hwmgr,
+			THERMAL_INT_OUTPUT_GPIO_PINID, &gpio_pin_assignment)) {
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ThermalOutGPIO);
+
+		table->ThermOutGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
+
+		table->ThermOutPolarity =
+			(0 == (cgs_read_register(hwmgr->device, mmGPIOPAD_A) &
+			(1 << gpio_pin_assignment.uc_gpio_pin_bit_shift))) ? 1 : 0;
+
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_ONLY;
+
+		/* if required, combine VRHot/PCC with thermal out GPIO*/
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_RegulatorHot) &&
+			phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_CombinePCCWithThermalSignal)){
+			table->ThermOutMode = SMU7_THERM_OUT_MODE_THERM_VRHOT;
+		}
+	} else {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_ThermalOutGPIO);
+
+		table->ThermOutGpio = 17;
+		table->ThermOutPolarity = 1;
+		table->ThermOutMode = SMU7_THERM_OUT_MODE_DISABLE;
+	}
+
+	for (i = 0; i < SMU72_MAX_ENTRIES_SMIO; i++)
+		table->Smio[i] = PP_HOST_TO_SMC_UL(table->Smio[i]);
+
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SystemFlags);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->VRConfig);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask1);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SmioMask2);
+	CONVERT_FROM_HOST_TO_SMC_UL(table->SclkStepSize);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitHigh);
+	CONVERT_FROM_HOST_TO_SMC_US(table->TemperatureLimitLow);
+	CONVERT_FROM_HOST_TO_SMC_US(table->VoltageResponseTime);
+	CONVERT_FROM_HOST_TO_SMC_US(table->PhaseResponseTime);
+
+	/* Upload all dpm data to SMC memory.(dpm level, dpm level count etc) */
+	result = smu7_copy_bytes_to_smc(
+			hwmgr->smumgr,
+			smu_data->smu7_data.dpm_table_start + offsetof(SMU72_Discrete_DpmTable, SystemFlags),
+			(uint8_t *)&(table->SystemFlags),
+			sizeof(SMU72_Discrete_DpmTable) - 3 * sizeof(SMU72_PIDController),
+			SMC_RAM_END);
+
+	PP_ASSERT_WITH_CODE(!result,
+		"Failed to upload dpm data to SMC memory !", return result;);
+
+	result = tonga_init_arb_table_index(hwmgr->smumgr);
+	PP_ASSERT_WITH_CODE(!result,
+			"Failed to upload arb data to SMC memory !", return result);
+
+	tonga_populate_pm_fuses(hwmgr);
+	PP_ASSERT_WITH_CODE((!result),
+		"Failed to populate initialize pm fuses !", return result);
+
+	result = tonga_populate_initial_mc_reg_table(hwmgr);
+	PP_ASSERT_WITH_CODE((!result),
+		"Failed to populate initialize MC Reg table !", return result);
+
+	return 0;
+}
+
+/**
+* Set up the fan table to control the fan using the SMC.
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @param    pInput the pointer to input data
+* @param    pOutput the pointer to output data
+* @param    pStorage the pointer to temporary storage
+* @param    Result the last failure code
+* @return   result from set temperature range routine
+*/
+int tonga_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data =
+			(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	SMU72_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
+	uint32_t duty100;
+	uint32_t t_diff1, t_diff2, pwm_diff1, pwm_diff2;
+	uint16_t fdo_min, slope1, slope2;
+	uint32_t reference_clock;
+	int res;
+	uint64_t tmp64;
+
+	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_MicrocodeFanControl))
+		return 0;
+
+	if (0 == smu_data->smu7_data.fan_table_start) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	duty100 = PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,
+						CGS_IND_REG__SMC,
+						CG_FDO_CTRL1, FMAX_DUTY100);
+
+	if (0 == duty100) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+				PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
+	tmp64 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin * duty100;
+	do_div(tmp64, 10000);
+	fdo_min = (uint16_t)tmp64;
+
+	t_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usTMed -
+		   hwmgr->thermal_controller.advanceFanControlParameters.usTMin;
+	t_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usTHigh -
+		  hwmgr->thermal_controller.advanceFanControlParameters.usTMed;
+
+	pwm_diff1 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed -
+		    hwmgr->thermal_controller.advanceFanControlParameters.usPWMMin;
+	pwm_diff2 = hwmgr->thermal_controller.advanceFanControlParameters.usPWMHigh -
+		    hwmgr->thermal_controller.advanceFanControlParameters.usPWMMed;
+
+	slope1 = (uint16_t)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
+	slope2 = (uint16_t)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
+
+	fan_table.TempMin = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMin) / 100);
+	fan_table.TempMed = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMed) / 100);
+	fan_table.TempMax = cpu_to_be16((50 + hwmgr->thermal_controller.advanceFanControlParameters.usTMax) / 100);
+
+	fan_table.Slope1 = cpu_to_be16(slope1);
+	fan_table.Slope2 = cpu_to_be16(slope2);
+
+	fan_table.FdoMin = cpu_to_be16(fdo_min);
+
+	fan_table.HystDown = cpu_to_be16(hwmgr->thermal_controller.advanceFanControlParameters.ucTHyst);
+
+	fan_table.HystUp = cpu_to_be16(1);
+
+	fan_table.HystSlope = cpu_to_be16(1);
+
+	fan_table.TempRespLim = cpu_to_be16(5);
+
+	reference_clock = smu7_get_xclk(hwmgr);
+
+	fan_table.RefreshPeriod = cpu_to_be32((hwmgr->thermal_controller.advanceFanControlParameters.ulCycleDelay * reference_clock) / 1600);
+
+	fan_table.FdoMax = cpu_to_be16((uint16_t)duty100);
+
+	fan_table.TempSrc = (uint8_t)PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, CG_MULT_THERMAL_CTRL, TEMP_SEL);
+
+	fan_table.FanControl_GL_Flag = 1;
+
+	res = smu7_copy_bytes_to_smc(hwmgr->smumgr,
+					smu_data->smu7_data.fan_table_start,
+					(uint8_t *)&fan_table,
+					(uint32_t)sizeof(fan_table),
+					SMC_RAM_END);
+
+	return 0;
+}
+
+
+static int tonga_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	if (data->need_update_smu7_dpm_table &
+		(DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_OD_UPDATE_MCLK))
+		return tonga_program_memory_timing_parameters(hwmgr);
+
+	return 0;
+}
+
+int tonga_update_sclk_threshold(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct tonga_smumgr *smu_data =
+			(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+
+	int result = 0;
+	uint32_t low_sclk_interrupt_threshold = 0;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_SclkThrottleLowNotification)
+		&& (hwmgr->gfx_arbiter.sclk_threshold !=
+				data->low_sclk_interrupt_threshold)) {
+		data->low_sclk_interrupt_threshold =
+				hwmgr->gfx_arbiter.sclk_threshold;
+		low_sclk_interrupt_threshold =
+				data->low_sclk_interrupt_threshold;
+
+		CONVERT_FROM_HOST_TO_SMC_UL(low_sclk_interrupt_threshold);
+
+		result = smu7_copy_bytes_to_smc(
+				hwmgr->smumgr,
+				smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU72_Discrete_DpmTable,
+					LowSclkInterruptThreshold),
+				(uint8_t *)&low_sclk_interrupt_threshold,
+				sizeof(uint32_t),
+				SMC_RAM_END);
+	}
+
+	result = tonga_update_and_upload_mc_reg_table(hwmgr);
+
+	PP_ASSERT_WITH_CODE((!result),
+				"Failed to upload MC reg table !",
+				return result);
+
+	result = tonga_program_mem_timing_parameters(hwmgr);
+	PP_ASSERT_WITH_CODE((result == 0),
+			"Failed to program memory timing parameters !",
+			);
+
+	return result;
+}
+
+uint32_t tonga_get_offsetof(uint32_t type, uint32_t member)
+{
+	switch (type) {
+	case SMU_SoftRegisters:
+		switch (member) {
+		case HandshakeDisables:
+			return offsetof(SMU72_SoftRegisters, HandshakeDisables);
+		case VoltageChangeTimeout:
+			return offsetof(SMU72_SoftRegisters, VoltageChangeTimeout);
+		case AverageGraphicsActivity:
+			return offsetof(SMU72_SoftRegisters, AverageGraphicsActivity);
+		case PreVBlankGap:
+			return offsetof(SMU72_SoftRegisters, PreVBlankGap);
+		case VBlankTimeout:
+			return offsetof(SMU72_SoftRegisters, VBlankTimeout);
+		case UcodeLoadStatus:
+			return offsetof(SMU72_SoftRegisters, UcodeLoadStatus);
+		}
+	case SMU_Discrete_DpmTable:
+		switch (member) {
+		case UvdBootLevel:
+			return offsetof(SMU72_Discrete_DpmTable, UvdBootLevel);
+		case VceBootLevel:
+			return offsetof(SMU72_Discrete_DpmTable, VceBootLevel);
+		case SamuBootLevel:
+			return offsetof(SMU72_Discrete_DpmTable, SamuBootLevel);
+		case LowSclkInterruptThreshold:
+			return offsetof(SMU72_Discrete_DpmTable, LowSclkInterruptThreshold);
+		}
+	}
+	printk("cant't get the offset of type %x member %x\n", type, member);
+	return 0;
+}
+
+uint32_t tonga_get_mac_definition(uint32_t value)
+{
+	switch (value) {
+	case SMU_MAX_LEVELS_GRAPHICS:
+		return SMU72_MAX_LEVELS_GRAPHICS;
+	case SMU_MAX_LEVELS_MEMORY:
+		return SMU72_MAX_LEVELS_MEMORY;
+	case SMU_MAX_LEVELS_LINK:
+		return SMU72_MAX_LEVELS_LINK;
+	case SMU_MAX_ENTRIES_SMIO:
+		return SMU72_MAX_ENTRIES_SMIO;
+	case SMU_MAX_LEVELS_VDDC:
+		return SMU72_MAX_LEVELS_VDDC;
+	case SMU_MAX_LEVELS_VDDGFX:
+		return SMU72_MAX_LEVELS_VDDGFX;
+	case SMU_MAX_LEVELS_VDDCI:
+		return SMU72_MAX_LEVELS_VDDCI;
+	case SMU_MAX_LEVELS_MVDD:
+		return SMU72_MAX_LEVELS_MVDD;
+	}
+	printk("cant't get the mac value %x\n", value);
+
+	return 0;
+}
+
+
+static int tonga_update_uvd_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+	smu_data->smc_state_table.UvdBootLevel = 0;
+	if (table_info->mm_dep_table->count > 0)
+		smu_data->smc_state_table.UvdBootLevel =
+				(uint8_t) (table_info->mm_dep_table->count - 1);
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU72_Discrete_DpmTable, UvdBootLevel);
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0x00FFFFFF;
+	mm_boot_level_value |= smu_data->smc_state_table.UvdBootLevel << 24;
+	cgs_write_ind_register(hwmgr->device,
+				CGS_IND_REG__SMC,
+				mm_boot_level_offset, mm_boot_level_value);
+
+	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_UVDDPM) ||
+		phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_UVDDPM_SetEnabledMask,
+				(uint32_t)(1 << smu_data->smc_state_table.UvdBootLevel));
+	return 0;
+}
+
+static int tonga_update_vce_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data =
+				(struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+	struct phm_ppt_v1_information *table_info =
+			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+
+
+	smu_data->smc_state_table.VceBootLevel =
+		(uint8_t) (table_info->mm_dep_table->count - 1);
+
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start +
+					offsetof(SMU72_Discrete_DpmTable, VceBootLevel);
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0xFF00FFFF;
+	mm_boot_level_value |= smu_data->smc_state_table.VceBootLevel << 16;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_VCEDPM_SetEnabledMask,
+				(uint32_t)1 << smu_data->smc_state_table.VceBootLevel);
+	return 0;
+}
+
+static int tonga_update_samu_smc_table(struct pp_hwmgr *hwmgr)
+{
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	uint32_t mm_boot_level_offset, mm_boot_level_value;
+
+	smu_data->smc_state_table.SamuBootLevel = 0;
+	mm_boot_level_offset = smu_data->smu7_data.dpm_table_start +
+				offsetof(SMU72_Discrete_DpmTable, SamuBootLevel);
+
+	mm_boot_level_offset /= 4;
+	mm_boot_level_offset *= 4;
+	mm_boot_level_value = cgs_read_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset);
+	mm_boot_level_value &= 0xFFFFFF00;
+	mm_boot_level_value |= smu_data->smc_state_table.SamuBootLevel << 0;
+	cgs_write_ind_register(hwmgr->device,
+			CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_StablePState))
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SAMUDPM_SetEnabledMask,
+				(uint32_t)(1 << smu_data->smc_state_table.SamuBootLevel));
+	return 0;
+}
+
+int tonga_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type)
+{
+	switch (type) {
+	case SMU_UVD_TABLE:
+		tonga_update_uvd_smc_table(hwmgr);
+		break;
+	case SMU_VCE_TABLE:
+		tonga_update_vce_smc_table(hwmgr);
+		break;
+	case SMU_SAMU_TABLE:
+		tonga_update_samu_smc_table(hwmgr);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+
+/**
+ * Get the location of various tables inside the FW image.
+ *
+ * @param    hwmgr  the address of the powerplay hardware manager.
+ * @return   always 0
+ */
+int tonga_process_firmware_header(struct pp_hwmgr *hwmgr)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(hwmgr->smumgr->backend);
+
+	uint32_t tmp;
+	int result;
+	bool error = false;
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, DpmTable),
+				&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.dpm_table_start = tmp;
+
+	error |= (result != 0);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, SoftRegisters),
+				&tmp, SMC_RAM_END);
+
+	if (!result) {
+		data->soft_regs_start = tmp;
+		smu_data->smu7_data.soft_regs_start = tmp;
+	}
+
+	error |= (result != 0);
+
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, mcRegisterTable),
+				&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.mc_reg_table_start = tmp;
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, FanTable),
+				&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.fan_table_start = tmp;
+
+	error |= (result != 0);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, mcArbDramTimingTable),
+				&tmp, SMC_RAM_END);
+
+	if (!result)
+		smu_data->smu7_data.arb_table_start = tmp;
+
+	error |= (result != 0);
+
+	result = smu7_read_smc_sram_dword(hwmgr->smumgr,
+				SMU72_FIRMWARE_HEADER_LOCATION +
+				offsetof(SMU72_Firmware_Header, Version),
+				&tmp, SMC_RAM_END);
+
+	if (!result)
+		hwmgr->microcode_version_info.SMC = tmp;
+
+	error |= (result != 0);
+
+	return error ? 1 : 0;
+}
+
+/*---------------------------MC----------------------------*/
+
+static uint8_t tonga_get_memory_modile_index(struct pp_hwmgr *hwmgr)
+{
+	return (uint8_t) (0xFF & (cgs_read_register(hwmgr->device, mmBIOS_SCRATCH_4) >> 16));
+}
+
+static bool tonga_check_s0_mc_reg_index(uint16_t in_reg, uint16_t *out_reg)
+{
+	bool result = true;
+
+	switch (in_reg) {
+	case  mmMC_SEQ_RAS_TIMING:
+		*out_reg = mmMC_SEQ_RAS_TIMING_LP;
+		break;
+
+	case  mmMC_SEQ_DLL_STBY:
+		*out_reg = mmMC_SEQ_DLL_STBY_LP;
+		break;
+
+	case  mmMC_SEQ_G5PDX_CMD0:
+		*out_reg = mmMC_SEQ_G5PDX_CMD0_LP;
+		break;
+
+	case  mmMC_SEQ_G5PDX_CMD1:
+		*out_reg = mmMC_SEQ_G5PDX_CMD1_LP;
+		break;
+
+	case  mmMC_SEQ_G5PDX_CTRL:
+		*out_reg = mmMC_SEQ_G5PDX_CTRL_LP;
+		break;
+
+	case mmMC_SEQ_CAS_TIMING:
+		*out_reg = mmMC_SEQ_CAS_TIMING_LP;
+		break;
+
+	case mmMC_SEQ_MISC_TIMING:
+		*out_reg = mmMC_SEQ_MISC_TIMING_LP;
+		break;
+
+	case mmMC_SEQ_MISC_TIMING2:
+		*out_reg = mmMC_SEQ_MISC_TIMING2_LP;
+		break;
+
+	case mmMC_SEQ_PMG_DVS_CMD:
+		*out_reg = mmMC_SEQ_PMG_DVS_CMD_LP;
+		break;
+
+	case mmMC_SEQ_PMG_DVS_CTL:
+		*out_reg = mmMC_SEQ_PMG_DVS_CTL_LP;
+		break;
+
+	case mmMC_SEQ_RD_CTL_D0:
+		*out_reg = mmMC_SEQ_RD_CTL_D0_LP;
+		break;
+
+	case mmMC_SEQ_RD_CTL_D1:
+		*out_reg = mmMC_SEQ_RD_CTL_D1_LP;
+		break;
+
+	case mmMC_SEQ_WR_CTL_D0:
+		*out_reg = mmMC_SEQ_WR_CTL_D0_LP;
+		break;
+
+	case mmMC_SEQ_WR_CTL_D1:
+		*out_reg = mmMC_SEQ_WR_CTL_D1_LP;
+		break;
+
+	case mmMC_PMG_CMD_EMRS:
+		*out_reg = mmMC_SEQ_PMG_CMD_EMRS_LP;
+		break;
+
+	case mmMC_PMG_CMD_MRS:
+		*out_reg = mmMC_SEQ_PMG_CMD_MRS_LP;
+		break;
+
+	case mmMC_PMG_CMD_MRS1:
+		*out_reg = mmMC_SEQ_PMG_CMD_MRS1_LP;
+		break;
+
+	case mmMC_SEQ_PMG_TIMING:
+		*out_reg = mmMC_SEQ_PMG_TIMING_LP;
+		break;
+
+	case mmMC_PMG_CMD_MRS2:
+		*out_reg = mmMC_SEQ_PMG_CMD_MRS2_LP;
+		break;
+
+	case mmMC_SEQ_WR_CTL_2:
+		*out_reg = mmMC_SEQ_WR_CTL_2_LP;
+		break;
+
+	default:
+		result = false;
+		break;
+	}
+
+	return result;
+}
+
+static int tonga_set_s0_mc_reg_index(struct tonga_mc_reg_table *table)
+{
+	uint32_t i;
+	uint16_t address;
+
+	for (i = 0; i < table->last; i++) {
+		table->mc_reg_address[i].s0 =
+			tonga_check_s0_mc_reg_index(table->mc_reg_address[i].s1,
+							&address) ?
+							address :
+						 table->mc_reg_address[i].s1;
+	}
+	return 0;
+}
+
+static int tonga_copy_vbios_smc_reg_table(const pp_atomctrl_mc_reg_table *table,
+					struct tonga_mc_reg_table *ni_table)
+{
+	uint8_t i, j;
+
+	PP_ASSERT_WITH_CODE((table->last <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+		"Invalid VramInfo table.", return -EINVAL);
+	PP_ASSERT_WITH_CODE((table->num_entries <= MAX_AC_TIMING_ENTRIES),
+		"Invalid VramInfo table.", return -EINVAL);
+
+	for (i = 0; i < table->last; i++)
+		ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
+
+	ni_table->last = table->last;
+
+	for (i = 0; i < table->num_entries; i++) {
+		ni_table->mc_reg_table_entry[i].mclk_max =
+			table->mc_reg_table_entry[i].mclk_max;
+		for (j = 0; j < table->last; j++) {
+			ni_table->mc_reg_table_entry[i].mc_data[j] =
+				table->mc_reg_table_entry[i].mc_data[j];
+		}
+	}
+
+	ni_table->num_entries = table->num_entries;
+
+	return 0;
+}
+
+/**
+ * VBIOS omits some information to reduce size, we need to recover them here.
+ * 1.   when we see mmMC_SEQ_MISC1, bit[31:16] EMRS1, need to be write to
+ *      mmMC_PMG_CMD_EMRS /_LP[15:0]. Bit[15:0] MRS, need to be update
+ *      mmMC_PMG_CMD_MRS/_LP[15:0]
+ * 2.   when we see mmMC_SEQ_RESERVE_M, bit[15:0] EMRS2, need to be write to
+ *      mmMC_PMG_CMD_MRS1/_LP[15:0].
+ * 3.   need to set these data for each clock range
+ * @param    hwmgr the address of the powerplay hardware manager.
+ * @param    table the address of MCRegTable
+ * @return   always 0
+ */
+static int tonga_set_mc_special_registers(struct pp_hwmgr *hwmgr,
+					struct tonga_mc_reg_table *table)
+{
+	uint8_t i, j, k;
+	uint32_t temp_reg;
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+
+	for (i = 0, j = table->last; i < table->last; i++) {
+		PP_ASSERT_WITH_CODE((j < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+			"Invalid VramInfo table.", return -EINVAL);
+
+		switch (table->mc_reg_address[i].s1) {
+
+		case mmMC_SEQ_MISC1:
+			temp_reg = cgs_read_register(hwmgr->device,
+							mmMC_PMG_CMD_EMRS);
+			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_EMRS;
+			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_EMRS_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					((temp_reg & 0xffff0000)) |
+					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
+			}
+			j++;
+			PP_ASSERT_WITH_CODE((j < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+				"Invalid VramInfo table.", return -EINVAL);
+
+			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS);
+			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
+			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					(temp_reg & 0xffff0000) |
+					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+
+				if (!data->is_memory_gddr5)
+					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
+			}
+			j++;
+			PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+				"Invalid VramInfo table.", return -EINVAL);
+
+			if (!data->is_memory_gddr5) {
+				table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
+				table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
+				for (k = 0; k < table->num_entries; k++)
+					table->mc_reg_table_entry[k].mc_data[j] =
+						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
+				j++;
+				PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+					"Invalid VramInfo table.", return -EINVAL);
+			}
+
+			break;
+
+		case mmMC_SEQ_RESERVE_M:
+			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS1);
+			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS1;
+			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS1_LP;
+			for (k = 0; k < table->num_entries; k++) {
+				table->mc_reg_table_entry[k].mc_data[j] =
+					(temp_reg & 0xffff0000) |
+					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
+			}
+			j++;
+			PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+				"Invalid VramInfo table.", return -EINVAL);
+			break;
+
+		default:
+			break;
+		}
+
+	}
+
+	table->last = j;
+
+	return 0;
+}
+
+static int tonga_set_valid_flag(struct tonga_mc_reg_table *table)
+{
+	uint8_t i, j;
+
+	for (i = 0; i < table->last; i++) {
+		for (j = 1; j < table->num_entries; j++) {
+			if (table->mc_reg_table_entry[j-1].mc_data[i] !=
+				table->mc_reg_table_entry[j].mc_data[i]) {
+				table->validflag |= (1<<i);
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
+
+int tonga_initialize_mc_reg_table(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(hwmgr->smumgr->backend);
+	pp_atomctrl_mc_reg_table *table;
+	struct tonga_mc_reg_table *ni_table = &smu_data->mc_reg_table;
+	uint8_t module_index = tonga_get_memory_modile_index(hwmgr);
+
+	table = kzalloc(sizeof(pp_atomctrl_mc_reg_table), GFP_KERNEL);
+
+	if (table == NULL)
+		return -ENOMEM;
+
+	/* Program additional LP registers that are no longer programmed by VBIOS */
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RAS_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_RAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_CAS_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_CAS_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_DLL_STBY_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_DLL_STBY));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD0_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD1_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CMD1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_G5PDX_CTRL_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_G5PDX_CTRL));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CMD_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CMD));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CTL_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_DVS_CTL));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_MISC_TIMING2));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_EMRS_LP,
+			cgs_read_register(hwmgr->device, mmMC_PMG_CMD_EMRS));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS_LP,
+			cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS1_LP,
+			cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D0_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D0));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_RD_CTL_D1));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_TIMING_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_PMG_TIMING));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_PMG_CMD_MRS2_LP,
+			cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS2));
+	cgs_write_register(hwmgr->device, mmMC_SEQ_WR_CTL_2_LP,
+			cgs_read_register(hwmgr->device, mmMC_SEQ_WR_CTL_2));
+
+	memset(table, 0x00, sizeof(pp_atomctrl_mc_reg_table));
+
+	result = atomctrl_initialize_mc_reg_table(hwmgr, module_index, table);
+
+	if (!result)
+		result = tonga_copy_vbios_smc_reg_table(table, ni_table);
+
+	if (!result) {
+		tonga_set_s0_mc_reg_index(ni_table);
+		result = tonga_set_mc_special_registers(hwmgr, ni_table);
+	}
+
+	if (!result)
+		tonga_set_valid_flag(ni_table);
+
+	kfree(table);
+
+	return result;
+}
+
+bool tonga_is_dpm_running(struct pp_hwmgr *hwmgr)
+{
+	return (1 == PHM_READ_INDIRECT_FIELD(hwmgr->device,
+			CGS_IND_REG__SMC, FEATURE_STATUS, VOLTAGE_CONTROLLER_ON))
+			? true : false;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_powertune.h b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.h
index 8e6670b3cb67..8ae169ff541d 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_powertune.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.h
@@ -20,35 +20,19 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+#ifndef _TONGA_SMC_H
+#define _TONGA_SMC_H
 
-#ifndef TONGA_POWERTUNE_H
-#define TONGA_POWERTUNE_H
+#include "smumgr.h"
+#include "smu72.h"
 
-enum _phw_tonga_ptc_config_reg_type {
-	TONGA_CONFIGREG_MMR = 0,
-	TONGA_CONFIGREG_SMC_IND,
-	TONGA_CONFIGREG_DIDT_IND,
-	TONGA_CONFIGREG_CACHE,
 
-	TONGA_CONFIGREG_MAX
-};
-typedef enum _phw_tonga_ptc_config_reg_type phw_tonga_ptc_config_reg_type;
-
-/* PowerContainment Features */
-#define POWERCONTAINMENT_FEATURE_BAPM            0x00000001
-#define POWERCONTAINMENT_FEATURE_TDCLimit        0x00000002
-#define POWERCONTAINMENT_FEATURE_PkgPwrLimit     0x00000004
+#define ASICID_IS_TONGA_P(wDID, bRID)	 \
+	(((wDID == 0x6930) && ((bRID == 0xF0) || (bRID == 0xF1) || (bRID == 0xFF))) \
+	|| ((wDID == 0x6920) && ((bRID == 0) || (bRID == 1))))
 
-struct _phw_tonga_pt_config_reg {
-	uint32_t                           Offset;
-	uint32_t                           Mask;
-	uint32_t                           Shift;
-	uint32_t                           Value;
-	phw_tonga_ptc_config_reg_type     Type;
-};
-typedef struct _phw_tonga_pt_config_reg phw_tonga_pt_config_reg;
 
-struct _phw_tonga_pt_defaults {
+struct tonga_pt_defaults {
 	uint8_t   svi_load_line_en;
 	uint8_t   svi_load_line_vddC;
 	uint8_t   tdc_vddc_throttle_release_limit_perc;
@@ -60,7 +44,17 @@ struct _phw_tonga_pt_defaults {
 	uint16_t  bapmti_r[SMU72_DTE_ITERATIONS * SMU72_DTE_SOURCES * SMU72_DTE_SINKS];
 	uint16_t  bapmti_rc[SMU72_DTE_ITERATIONS * SMU72_DTE_SOURCES * SMU72_DTE_SINKS];
 };
-typedef struct _phw_tonga_pt_defaults phw_tonga_pt_defaults;
 
+int tonga_populate_all_graphic_levels(struct pp_hwmgr *hwmgr);
+int tonga_populate_all_memory_levels(struct pp_hwmgr *hwmgr);
+int tonga_init_smc_table(struct pp_hwmgr *hwmgr);
+int tonga_thermal_setup_fan_table(struct pp_hwmgr *hwmgr);
+int tonga_update_smc_table(struct pp_hwmgr *hwmgr, uint32_t type);
+int tonga_update_sclk_threshold(struct pp_hwmgr *hwmgr);
+uint32_t tonga_get_offsetof(uint32_t type, uint32_t member);
+uint32_t tonga_get_mac_definition(uint32_t value);
+int tonga_process_firmware_header(struct pp_hwmgr *hwmgr);
+int tonga_initialize_mc_reg_table(struct pp_hwmgr *hwmgr);
+bool tonga_is_dpm_running(struct pp_hwmgr *hwmgr);
 #endif
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
index f42c536b3af1..5f9124046b9b 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
@@ -33,587 +33,9 @@
 #include "smu/smu_7_1_2_d.h"
 #include "smu/smu_7_1_2_sh_mask.h"
 #include "cgs_common.h"
+#include "tonga_smc.h"
+#include "smu7_smumgr.h"
 
-#define TONGA_SMC_SIZE			0x20000
-#define BUFFER_SIZE			80000
-#define MAX_STRING_SIZE			15
-#define BUFFER_SIZETWO              131072 /*128 *1024*/
-
-/**
-* Set the address for reading/writing the SMC SRAM space.
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    smcAddress the address in the SMC RAM to access.
-*/
-static int tonga_set_smc_sram_address(struct pp_smumgr *smumgr,
-				uint32_t smcAddress, uint32_t limit)
-{
-	if (smumgr == NULL || smumgr->device == NULL)
-		return -EINVAL;
-	PP_ASSERT_WITH_CODE((0 == (3 & smcAddress)),
-		"SMC address must be 4 byte aligned.",
-		return -1;);
-
-	PP_ASSERT_WITH_CODE((limit > (smcAddress + 3)),
-		"SMC address is beyond the SMC RAM area.",
-		return -1;);
-
-	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_0, smcAddress);
-	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_11, 0);
-
-	return 0;
-}
-
-/**
-* Copy bytes from an array into the SMC RAM space.
-*
-* @param    smumgr  the address of the powerplay SMU manager.
-* @param    smcStartAddress the start address in the SMC RAM to copy bytes to.
-* @param    src the byte array to copy the bytes from.
-* @param    byteCount the number of bytes to copy.
-*/
-int tonga_copy_bytes_to_smc(struct pp_smumgr *smumgr,
-		uint32_t smcStartAddress, const uint8_t *src,
-		uint32_t byteCount, uint32_t limit)
-{
-	uint32_t addr;
-	uint32_t data, orig_data;
-	int result = 0;
-	uint32_t extra_shift;
-
-	if (smumgr == NULL || smumgr->device == NULL)
-		return -EINVAL;
-	PP_ASSERT_WITH_CODE((0 == (3 & smcStartAddress)),
-		"SMC address must be 4 byte aligned.",
-		return 0;);
-
-	PP_ASSERT_WITH_CODE((limit > (smcStartAddress + byteCount)),
-		"SMC address is beyond the SMC RAM area.",
-		return 0;);
-
-	addr = smcStartAddress;
-
-	while (byteCount >= 4) {
-		/*
-		 * Bytes are written into the
-		 * SMC address space with the MSB first
-		 */
-		data = (src[0] << 24) + (src[1] << 16) + (src[2] << 8) + src[3];
-
-		result = tonga_set_smc_sram_address(smumgr, addr, limit);
-
-		if (result)
-			goto out;
-
-		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data);
-
-		src += 4;
-		byteCount -= 4;
-		addr += 4;
-	}
-
-	if (0 != byteCount) {
-		/* Now write odd bytes left, do a read modify write cycle */
-		data = 0;
-
-		result = tonga_set_smc_sram_address(smumgr, addr, limit);
-		if (result)
-			goto out;
-
-		orig_data = cgs_read_register(smumgr->device,
-							mmSMC_IND_DATA_0);
-		extra_shift = 8 * (4 - byteCount);
-
-		while (byteCount > 0) {
-			data = (data << 8) + *src++;
-			byteCount--;
-		}
-
-		data <<= extra_shift;
-		data |= (orig_data & ~((~0UL) << extra_shift));
-
-		result = tonga_set_smc_sram_address(smumgr, addr, limit);
-		if (result)
-			goto out;
-
-		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data);
-	}
-
-out:
-	return result;
-}
-
-
-int tonga_program_jump_on_start(struct pp_smumgr *smumgr)
-{
-	static const unsigned char pData[] = { 0xE0, 0x00, 0x80, 0x40 };
-
-	tonga_copy_bytes_to_smc(smumgr, 0x0, pData, 4, sizeof(pData)+1);
-
-	return 0;
-}
-
-/**
-* Return if the SMC is currently running.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-*/
-static int tonga_is_smc_ram_running(struct pp_smumgr *smumgr)
-{
-	return ((0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
-					SMC_SYSCON_CLOCK_CNTL_0, ck_disable))
-			&& (0x20100 <= cgs_read_ind_register(smumgr->device,
-					CGS_IND_REG__SMC, ixSMC_PC_C)));
-}
-
-static int tonga_send_msg_to_smc_offset(struct pp_smumgr *smumgr)
-{
-	if (smumgr == NULL || smumgr->device == NULL)
-		return -EINVAL;
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-
-	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, 0x20000);
-	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, PPSMC_MSG_Test);
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-
-	return 0;
-}
-
-/**
-* Send a message to the SMC, and wait for its response.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    msg the message to send.
-* @return   The response that came from the SMC.
-*/
-static int tonga_send_msg_to_smc(struct pp_smumgr *smumgr, uint16_t msg)
-{
-	if (smumgr == NULL || smumgr->device == NULL)
-		return -EINVAL;
-
-	if (!tonga_is_smc_ram_running(smumgr))
-		return -1;
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-	PP_ASSERT_WITH_CODE(
-		1 == SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP),
-		"Failed to send Previous Message.",
-		);
-
-	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-	PP_ASSERT_WITH_CODE(
-		1 == SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP),
-		"Failed to send Message.",
-		);
-
-	return 0;
-}
-
-/*
-* Send a message to the SMC, and do not wait for its response.
-*
-* @param    smumgr  the address of the powerplay hardware manager.
-* @param    msg the message to send.
-* @return   The response that came from the SMC.
-*/
-static int tonga_send_msg_to_smc_without_waiting
-				(struct pp_smumgr *smumgr, uint16_t msg)
-{
-	if (smumgr == NULL || smumgr->device == NULL)
-		return -EINVAL;
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-	PP_ASSERT_WITH_CODE(
-		1 == SMUM_READ_FIELD(smumgr->device, SMC_RESP_0, SMC_RESP),
-		"Failed to send Previous Message.",
-		);
-	cgs_write_register(smumgr->device, mmSMC_MESSAGE_0, msg);
-
-	return 0;
-}
-
-/*
-* Send a message to the SMC with parameter
-*
-* @param    smumgr:  the address of the powerplay hardware manager.
-* @param    msg: the message to send.
-* @param    parameter: the parameter to send
-* @return   The response that came from the SMC.
-*/
-static int tonga_send_msg_to_smc_with_parameter(struct pp_smumgr *smumgr,
-				uint16_t msg, uint32_t parameter)
-{
-	if (smumgr == NULL || smumgr->device == NULL)
-		return -EINVAL;
-
-	if (!tonga_is_smc_ram_running(smumgr))
-		return PPSMC_Result_Failed;
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
-
-	return tonga_send_msg_to_smc(smumgr, msg);
-}
-
-/*
-* Send a message to the SMC with parameter, do not wait for response
-*
-* @param    smumgr:  the address of the powerplay hardware manager.
-* @param    msg: the message to send.
-* @param    parameter: the parameter to send
-* @return   The response that came from the SMC.
-*/
-static int tonga_send_msg_to_smc_with_parameter_without_waiting(
-			struct pp_smumgr *smumgr,
-			uint16_t msg, uint32_t parameter)
-{
-	if (smumgr == NULL || smumgr->device == NULL)
-		return -EINVAL;
-
-	SMUM_WAIT_FIELD_UNEQUAL(smumgr, SMC_RESP_0, SMC_RESP, 0);
-
-	cgs_write_register(smumgr->device, mmSMC_MSG_ARG_0, parameter);
-
-	return tonga_send_msg_to_smc_without_waiting(smumgr, msg);
-}
-
-/*
- * Read a 32bit value from the SMC SRAM space.
- * ALL PARAMETERS ARE IN HOST BYTE ORDER.
- * @param    smumgr  the address of the powerplay hardware manager.
- * @param    smcAddress the address in the SMC RAM to access.
- * @param    value and output parameter for the data read from the SMC SRAM.
- */
-int tonga_read_smc_sram_dword(struct pp_smumgr *smumgr,
-					uint32_t smcAddress, uint32_t *value,
-					uint32_t limit)
-{
-	int result;
-
-	result = tonga_set_smc_sram_address(smumgr, smcAddress, limit);
-
-	if (0 != result)
-		return result;
-
-	*value = cgs_read_register(smumgr->device, mmSMC_IND_DATA_0);
-
-	return 0;
-}
-
-/*
- * Write a 32bit value to the SMC SRAM space.
- * ALL PARAMETERS ARE IN HOST BYTE ORDER.
- * @param    smumgr  the address of the powerplay hardware manager.
- * @param    smcAddress the address in the SMC RAM to access.
- * @param    value to write to the SMC SRAM.
- */
-int tonga_write_smc_sram_dword(struct pp_smumgr *smumgr,
-					uint32_t smcAddress, uint32_t value,
-					uint32_t limit)
-{
-	int result;
-
-	result = tonga_set_smc_sram_address(smumgr, smcAddress, limit);
-
-	if (0 != result)
-		return result;
-
-	cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, value);
-
-	return 0;
-}
-
-static int tonga_smu_fini(struct pp_smumgr *smumgr)
-{
-	struct tonga_smumgr *priv = (struct tonga_smumgr *)(smumgr->backend);
-
-	smu_free_memory(smumgr->device, (void *)priv->smu_buffer.handle);
-	smu_free_memory(smumgr->device, (void *)priv->header_buffer.handle);
-
-	if (smumgr->backend != NULL) {
-		kfree(smumgr->backend);
-		smumgr->backend = NULL;
-	}
-
-	cgs_rel_firmware(smumgr->device, CGS_UCODE_ID_SMU);
-	return 0;
-}
-
-static enum cgs_ucode_id tonga_convert_fw_type_to_cgs(uint32_t fw_type)
-{
-	enum cgs_ucode_id result = CGS_UCODE_ID_MAXIMUM;
-
-	switch (fw_type) {
-	case UCODE_ID_SMU:
-		result = CGS_UCODE_ID_SMU;
-		break;
-	case UCODE_ID_SDMA0:
-		result = CGS_UCODE_ID_SDMA0;
-		break;
-	case UCODE_ID_SDMA1:
-		result = CGS_UCODE_ID_SDMA1;
-		break;
-	case UCODE_ID_CP_CE:
-		result = CGS_UCODE_ID_CP_CE;
-		break;
-	case UCODE_ID_CP_PFP:
-		result = CGS_UCODE_ID_CP_PFP;
-		break;
-	case UCODE_ID_CP_ME:
-		result = CGS_UCODE_ID_CP_ME;
-		break;
-	case UCODE_ID_CP_MEC:
-		result = CGS_UCODE_ID_CP_MEC;
-		break;
-	case UCODE_ID_CP_MEC_JT1:
-		result = CGS_UCODE_ID_CP_MEC_JT1;
-		break;
-	case UCODE_ID_CP_MEC_JT2:
-		result = CGS_UCODE_ID_CP_MEC_JT2;
-		break;
-	case UCODE_ID_RLC_G:
-		result = CGS_UCODE_ID_RLC_G;
-		break;
-	default:
-		break;
-	}
-
-	return result;
-}
-
-/**
- * Convert the PPIRI firmware type to SMU type mask.
- * For MEC, we need to check all MEC related type
-*/
-static uint16_t tonga_get_mask_for_firmware_type(uint16_t firmwareType)
-{
-	uint16_t result = 0;
-
-	switch (firmwareType) {
-	case UCODE_ID_SDMA0:
-		result = UCODE_ID_SDMA0_MASK;
-		break;
-	case UCODE_ID_SDMA1:
-		result = UCODE_ID_SDMA1_MASK;
-		break;
-	case UCODE_ID_CP_CE:
-		result = UCODE_ID_CP_CE_MASK;
-		break;
-	case UCODE_ID_CP_PFP:
-		result = UCODE_ID_CP_PFP_MASK;
-		break;
-	case UCODE_ID_CP_ME:
-		result = UCODE_ID_CP_ME_MASK;
-		break;
-	case UCODE_ID_CP_MEC:
-	case UCODE_ID_CP_MEC_JT1:
-	case UCODE_ID_CP_MEC_JT2:
-		result = UCODE_ID_CP_MEC_MASK;
-		break;
-	case UCODE_ID_RLC_G:
-		result = UCODE_ID_RLC_G_MASK;
-		break;
-	default:
-		break;
-	}
-
-	return result;
-}
-
-/**
- * Check if the FW has been loaded,
- * SMU will not return if loading has not finished.
-*/
-static int tonga_check_fw_load_finish(struct pp_smumgr *smumgr, uint32_t fwType)
-{
-	uint16_t fwMask = tonga_get_mask_for_firmware_type(fwType);
-
-	if (0 != SMUM_WAIT_VFPF_INDIRECT_REGISTER(smumgr, SMC_IND,
-				SOFT_REGISTERS_TABLE_28, fwMask, fwMask)) {
-		printk(KERN_ERR "[ powerplay ] check firmware loading failed\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* Populate one firmware image to the data structure */
-static int tonga_populate_single_firmware_entry(struct pp_smumgr *smumgr,
-				uint16_t firmware_type,
-				struct SMU_Entry *pentry)
-{
-	int result;
-	struct cgs_firmware_info info = {0};
-
-	result = cgs_get_firmware_info(
-				smumgr->device,
-				tonga_convert_fw_type_to_cgs(firmware_type),
-				&info);
-
-	if (result == 0) {
-		pentry->version = 0;
-		pentry->id = (uint16_t)firmware_type;
-		pentry->image_addr_high = smu_upper_32_bits(info.mc_addr);
-		pentry->image_addr_low = smu_lower_32_bits(info.mc_addr);
-		pentry->meta_data_addr_high = 0;
-		pentry->meta_data_addr_low = 0;
-		pentry->data_size_byte = info.image_size;
-		pentry->num_register_entries = 0;
-
-		if (firmware_type == UCODE_ID_RLC_G)
-			pentry->flags = 1;
-		else
-			pentry->flags = 0;
-	} else {
-		return result;
-	}
-
-	return result;
-}
-
-static int tonga_request_smu_reload_fw(struct pp_smumgr *smumgr)
-{
-	struct tonga_smumgr *tonga_smu =
-		(struct tonga_smumgr *)(smumgr->backend);
-	uint16_t fw_to_load;
-	struct SMU_DRAMData_TOC *toc;
-	/**
-	 * First time this gets called during SmuMgr init,
-	 * we haven't processed SMU header file yet,
-	 * so Soft Register Start offset is unknown.
-	 * However, for this case, UcodeLoadStatus is already 0,
-	 * so we can skip this if the Soft Registers Start offset is 0.
-	 */
-	cgs_write_ind_register(smumgr->device,
-		CGS_IND_REG__SMC, ixSOFT_REGISTERS_TABLE_28, 0);
-
-	tonga_send_msg_to_smc_with_parameter(smumgr,
-		PPSMC_MSG_SMU_DRAM_ADDR_HI,
-		tonga_smu->smu_buffer.mc_addr_high);
-	tonga_send_msg_to_smc_with_parameter(smumgr,
-		PPSMC_MSG_SMU_DRAM_ADDR_LO,
-		tonga_smu->smu_buffer.mc_addr_low);
-
-	toc = (struct SMU_DRAMData_TOC *)tonga_smu->pHeader;
-	toc->num_entries = 0;
-	toc->structure_version = 1;
-
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_populate_single_firmware_entry(smumgr,
-		UCODE_ID_RLC_G,
-		&toc->entry[toc->num_entries++]),
-		"Failed to Get Firmware Entry.\n",
-		return -1);
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_populate_single_firmware_entry(smumgr,
-		UCODE_ID_CP_CE,
-		&toc->entry[toc->num_entries++]),
-		"Failed to Get Firmware Entry.\n",
-		return -1);
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_populate_single_firmware_entry
-		(smumgr, UCODE_ID_CP_PFP, &toc->entry[toc->num_entries++]),
-		"Failed to Get Firmware Entry.\n", return -1);
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_populate_single_firmware_entry
-		(smumgr, UCODE_ID_CP_ME, &toc->entry[toc->num_entries++]),
-		"Failed to Get Firmware Entry.\n", return -1);
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_populate_single_firmware_entry
-		(smumgr, UCODE_ID_CP_MEC, &toc->entry[toc->num_entries++]),
-		"Failed to Get Firmware Entry.\n", return -1);
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_populate_single_firmware_entry
-		(smumgr, UCODE_ID_CP_MEC_JT1, &toc->entry[toc->num_entries++]),
-		"Failed to Get Firmware Entry.\n", return -1);
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_populate_single_firmware_entry
-		(smumgr, UCODE_ID_CP_MEC_JT2, &toc->entry[toc->num_entries++]),
-		"Failed to Get Firmware Entry.\n", return -1);
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_populate_single_firmware_entry
-		(smumgr, UCODE_ID_SDMA0, &toc->entry[toc->num_entries++]),
-		"Failed to Get Firmware Entry.\n", return -1);
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_populate_single_firmware_entry
-		(smumgr, UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]),
-		"Failed to Get Firmware Entry.\n", return -1);
-
-	tonga_send_msg_to_smc_with_parameter(smumgr,
-		PPSMC_MSG_DRV_DRAM_ADDR_HI,
-		tonga_smu->header_buffer.mc_addr_high);
-	tonga_send_msg_to_smc_with_parameter(smumgr,
-		PPSMC_MSG_DRV_DRAM_ADDR_LO,
-		tonga_smu->header_buffer.mc_addr_low);
-
-	fw_to_load = UCODE_ID_RLC_G_MASK
-			+ UCODE_ID_SDMA0_MASK
-			+ UCODE_ID_SDMA1_MASK
-			+ UCODE_ID_CP_CE_MASK
-			+ UCODE_ID_CP_ME_MASK
-			+ UCODE_ID_CP_PFP_MASK
-			+ UCODE_ID_CP_MEC_MASK;
-
-	PP_ASSERT_WITH_CODE(
-		0 == tonga_send_msg_to_smc_with_parameter_without_waiting(
-		smumgr, PPSMC_MSG_LoadUcodes, fw_to_load),
-		"Fail to Request SMU Load uCode", return 0);
-
-	return 0;
-}
-
-static int tonga_request_smu_load_specific_fw(struct pp_smumgr *smumgr,
-				uint32_t firmwareType)
-{
-	return 0;
-}
-
-/**
- * Upload the SMC firmware to the SMC microcontroller.
- *
- * @param    smumgr  the address of the powerplay hardware manager.
- * @param    pFirmware the data structure containing the various sections of the firmware.
- */
-static int tonga_smu_upload_firmware_image(struct pp_smumgr *smumgr)
-{
-	const uint8_t *src;
-	uint32_t byte_count;
-	uint32_t *data;
-	struct cgs_firmware_info info = {0};
-
-	if (smumgr == NULL || smumgr->device == NULL)
-		return -EINVAL;
-
-	cgs_get_firmware_info(smumgr->device,
-		tonga_convert_fw_type_to_cgs(UCODE_ID_SMU), &info);
-
-	if (info.image_size & 3) {
-		printk(KERN_ERR "[ powerplay ] SMC ucode is not 4 bytes aligned\n");
-		return -EINVAL;
-	}
-
-	if (info.image_size > TONGA_SMC_SIZE) {
-		printk(KERN_ERR "[ powerplay ] SMC address is beyond the SMC RAM area\n");
-		return -EINVAL;
-	}
-
-	cgs_write_register(smumgr->device, mmSMC_IND_INDEX_0, 0x20000);
-	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1);
-
-	byte_count = info.image_size;
-	src = (const uint8_t *)info.kptr;
-
-	data = (uint32_t *)src;
-	for (; byte_count >= 4; data++, byte_count -= 4)
-		cgs_write_register(smumgr->device, mmSMC_IND_DATA_0, data[0]);
-
-	SMUM_WRITE_FIELD(smumgr->device, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
-
-	return 0;
-}
 
 static int tonga_start_in_protection_mode(struct pp_smumgr *smumgr)
 {
@@ -623,7 +45,7 @@ static int tonga_start_in_protection_mode(struct pp_smumgr *smumgr)
 	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
 		SMC_SYSCON_RESET_CNTL, rst_reg, 1);
 
-	result = tonga_smu_upload_firmware_image(smumgr);
+	result = smu7_upload_smu_firmware_image(smumgr);
 	if (result)
 		return result;
 
@@ -653,7 +75,7 @@ static int tonga_start_in_protection_mode(struct pp_smumgr *smumgr)
 	/**
 	 * Call Test SMU message with 0x20000 offset to trigger SMU start
 	 */
-	tonga_send_msg_to_smc_offset(smumgr);
+	smu7_send_msg_to_smc_offset(smumgr);
 
 	/* Wait for done bit to be set */
 	SMUM_WAIT_VFPF_INDIRECT_FIELD_UNEQUAL(smumgr, SMC_IND,
@@ -690,13 +112,13 @@ static int tonga_start_in_non_protection_mode(struct pp_smumgr *smumgr)
 	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
 		SMC_SYSCON_RESET_CNTL, rst_reg, 1);
 
-	result = tonga_smu_upload_firmware_image(smumgr);
+	result = smu7_upload_smu_firmware_image(smumgr);
 
 	if (result != 0)
 		return result;
 
 	/* Set smc instruct start point at 0x0 */
-	tonga_program_jump_on_start(smumgr);
+	smu7_program_jump_on_start(smumgr);
 
 
 	SMUM_WRITE_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
@@ -718,7 +140,7 @@ static int tonga_start_smu(struct pp_smumgr *smumgr)
 	int result;
 
 	/* Only start SMC if SMC RAM is not running */
-	if (!tonga_is_smc_ram_running(smumgr)) {
+	if (!smu7_is_smc_ram_running(smumgr)) {
 		/*Check if SMU is running in protected mode*/
 		if (0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
 					SMU_FIRMWARE, SMU_MODE)) {
@@ -732,7 +154,7 @@ static int tonga_start_smu(struct pp_smumgr *smumgr)
 		}
 	}
 
-	result = tonga_request_smu_reload_fw(smumgr);
+	result = smu7_request_smu_load_fw(smumgr);
 
 	return result;
 }
@@ -746,67 +168,41 @@ static int tonga_start_smu(struct pp_smumgr *smumgr)
  */
 static int tonga_smu_init(struct pp_smumgr *smumgr)
 {
-	struct tonga_smumgr *tonga_smu;
-	uint8_t *internal_buf;
-	uint64_t mc_addr = 0;
-	/* Allocate memory for backend private data */
-	tonga_smu = (struct tonga_smumgr *)(smumgr->backend);
-	tonga_smu->header_buffer.data_size =
-		((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
-	tonga_smu->smu_buffer.data_size = 200*4096;
-
-	smu_allocate_memory(smumgr->device,
-		tonga_smu->header_buffer.data_size,
-		CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
-		PAGE_SIZE,
-		&mc_addr,
-		&tonga_smu->header_buffer.kaddr,
-		&tonga_smu->header_buffer.handle);
-
-	tonga_smu->pHeader = tonga_smu->header_buffer.kaddr;
-	tonga_smu->header_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
-	tonga_smu->header_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
-
-	PP_ASSERT_WITH_CODE((NULL != tonga_smu->pHeader),
-		"Out of memory.",
-		kfree(smumgr->backend);
-		cgs_free_gpu_mem(smumgr->device,
-		(cgs_handle_t)tonga_smu->header_buffer.handle);
-		return -1);
-
-	smu_allocate_memory(smumgr->device,
-		tonga_smu->smu_buffer.data_size,
-		CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
-		PAGE_SIZE,
-		&mc_addr,
-		&tonga_smu->smu_buffer.kaddr,
-		&tonga_smu->smu_buffer.handle);
-
-	internal_buf = tonga_smu->smu_buffer.kaddr;
-	tonga_smu->smu_buffer.mc_addr_high = smu_upper_32_bits(mc_addr);
-	tonga_smu->smu_buffer.mc_addr_low = smu_lower_32_bits(mc_addr);
-
-	PP_ASSERT_WITH_CODE((NULL != internal_buf),
-		"Out of memory.",
-		kfree(smumgr->backend);
-		cgs_free_gpu_mem(smumgr->device,
-		(cgs_handle_t)tonga_smu->smu_buffer.handle);
-		return -1;);
+	struct tonga_smumgr *smu_data = (struct tonga_smumgr *)(smumgr->backend);
+
+	int i;
+
+	if (smu7_init(smumgr))
+		return -EINVAL;
+
+	for (i = 0; i < SMU72_MAX_LEVELS_GRAPHICS; i++)
+		smu_data->activity_target[i] = 30;
 
 	return 0;
 }
 
 static const struct pp_smumgr_func tonga_smu_funcs = {
 	.smu_init = &tonga_smu_init,
-	.smu_fini = &tonga_smu_fini,
+	.smu_fini = &smu7_smu_fini,
 	.start_smu = &tonga_start_smu,
-	.check_fw_load_finish = &tonga_check_fw_load_finish,
-	.request_smu_load_fw = &tonga_request_smu_reload_fw,
-	.request_smu_load_specific_fw = &tonga_request_smu_load_specific_fw,
-	.send_msg_to_smc = &tonga_send_msg_to_smc,
-	.send_msg_to_smc_with_parameter = &tonga_send_msg_to_smc_with_parameter,
+	.check_fw_load_finish = &smu7_check_fw_load_finish,
+	.request_smu_load_fw = &smu7_request_smu_load_fw,
+	.request_smu_load_specific_fw = NULL,
+	.send_msg_to_smc = &smu7_send_msg_to_smc,
+	.send_msg_to_smc_with_parameter = &smu7_send_msg_to_smc_with_parameter,
 	.download_pptable_settings = NULL,
 	.upload_pptable_settings = NULL,
+	.update_smc_table = tonga_update_smc_table,
+	.get_offsetof = tonga_get_offsetof,
+	.process_firmware_header = tonga_process_firmware_header,
+	.init_smc_table = tonga_init_smc_table,
+	.update_sclk_threshold = tonga_update_sclk_threshold,
+	.thermal_setup_fan_table = tonga_thermal_setup_fan_table,
+	.populate_all_graphic_levels = tonga_populate_all_graphic_levels,
+	.populate_all_memory_levels = tonga_populate_all_memory_levels,
+	.get_mac_definition = tonga_get_mac_definition,
+	.initialize_mc_reg_table = tonga_initialize_mc_reg_table,
+	.is_dpm_running = tonga_is_dpm_running,
 };
 
 int tonga_smum_init(struct pp_smumgr *smumgr)
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h
index 33c788d7f05c..8c4f761d5bc8 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.h
@@ -24,30 +24,36 @@
 #ifndef _TONGA_SMUMGR_H_
 #define _TONGA_SMUMGR_H_
 
-struct tonga_buffer_entry {
-	uint32_t data_size;
-	uint32_t mc_addr_low;
-	uint32_t mc_addr_high;
-	void *kaddr;
-	unsigned long  handle;
+#include "smu72_discrete.h"
+
+#include "smu7_smumgr.h"
+
+struct tonga_mc_reg_entry {
+	uint32_t mclk_max;
+	uint32_t mc_data[SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE];
+};
+
+struct tonga_mc_reg_table {
+	uint8_t   last;               /* number of registers*/
+	uint8_t   num_entries;        /* number of entries in mc_reg_table_entry used*/
+	uint16_t  validflag;          /* indicate the corresponding register is valid or not. 1: valid, 0: invalid. bit0->address[0], bit1->address[1], etc.*/
+	struct tonga_mc_reg_entry    mc_reg_table_entry[MAX_AC_TIMING_ENTRIES];
+	SMU72_Discrete_MCRegisterAddress mc_reg_address[SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE];
 };
 
+
 struct tonga_smumgr {
-	uint8_t *pHeader;
-	uint8_t *pMecImage;
-	uint32_t ulSoftRegsStart;
 
-	struct tonga_buffer_entry header_buffer;
-	struct tonga_buffer_entry smu_buffer;
-};
+	struct smu7_smumgr                   smu7_data;
+	struct SMU72_Discrete_DpmTable       smc_state_table;
+	struct SMU72_Discrete_Ulv            ulv_setting;
+	struct SMU72_Discrete_PmFuses  power_tune_table;
+	const struct tonga_pt_defaults  *power_tune_defaults;
+	SMU72_Discrete_MCRegisters      mc_regs;
+	struct tonga_mc_reg_table mc_reg_table;
 
-extern int tonga_smum_init(struct pp_smumgr *smumgr);
-extern int tonga_copy_bytes_to_smc(struct pp_smumgr *smumgr,
-		uint32_t smcStartAddress, const uint8_t *src,
-		uint32_t byteCount, uint32_t limit);
-extern int tonga_read_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smcAddress,
-		uint32_t *value, uint32_t limit);
-extern int tonga_write_smc_sram_dword(struct pp_smumgr *smumgr, uint32_t smcAddress,
-		uint32_t value, uint32_t limit);
+	uint32_t        activity_target[SMU72_MAX_LEVELS_GRAPHICS];
+
+};
 
 #endif
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
index ef312bb75fda..963a24d46a93 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
@@ -405,7 +405,7 @@ void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
 	spin_lock(&sched->job_list_lock);
 	s_job = list_first_entry_or_null(&sched->ring_mirror_list,
 					 struct amd_sched_job, node);
-	if (s_job)
+	if (s_job && sched->timeout != MAX_SCHEDULE_TIMEOUT)
 		schedule_delayed_work(&s_job->work_tdr, sched->timeout);
 
 	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
diff --git a/drivers/gpu/drm/arc/arcpgu_crtc.c b/drivers/gpu/drm/arc/arcpgu_crtc.c
index ee0a61c2861b..7130b044b004 100644
--- a/drivers/gpu/drm/arc/arcpgu_crtc.c
+++ b/drivers/gpu/drm/arc/arcpgu_crtc.c
@@ -183,8 +183,6 @@ static void arc_pgu_plane_atomic_update(struct drm_plane *plane,
 }
 
 static const struct drm_plane_helper_funcs arc_pgu_plane_helper_funcs = {
-	.prepare_fb = NULL,
-	.cleanup_fb = NULL,
 	.atomic_update = arc_pgu_plane_atomic_update,
 };
 
diff --git a/drivers/gpu/drm/arc/arcpgu_drv.c b/drivers/gpu/drm/arc/arcpgu_drv.c
index 6d4ff34737cb..28e6471257d0 100644
--- a/drivers/gpu/drm/arc/arcpgu_drv.c
+++ b/drivers/gpu/drm/arc/arcpgu_drv.c
@@ -198,8 +198,8 @@ static int arcpgu_probe(struct platform_device *pdev)
 	int ret;
 
 	drm = drm_dev_alloc(&arcpgu_drm_driver, &pdev->dev);
-	if (!drm)
-		return -ENOMEM;
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
 
 	ret = arcpgu_load(drm);
 	if (ret)
diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c
index d83b46a30327..fb6a418ce6be 100644
--- a/drivers/gpu/drm/arm/hdlcd_drv.c
+++ b/drivers/gpu/drm/arm/hdlcd_drv.c
@@ -326,8 +326,8 @@ static int hdlcd_drm_bind(struct device *dev)
 		return -ENOMEM;
 
 	drm = drm_dev_alloc(&hdlcd_driver, dev);
-	if (!drm)
-		return -ENOMEM;
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
 
 	drm->dev_private = hdlcd;
 	dev_set_drvdata(dev, drm);
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
index 82171d223f2d..9280358b8f15 100644
--- a/drivers/gpu/drm/arm/malidp_drv.c
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -91,7 +91,8 @@ static void malidp_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_modeset_disables(drm, state);
 	drm_atomic_helper_commit_modeset_enables(drm, state);
-	drm_atomic_helper_commit_planes(drm, state, true);
+	drm_atomic_helper_commit_planes(drm, state,
+					DRM_PLANE_COMMIT_ACTIVE_ONLY);
 
 	malidp_atomic_commit_hw_done(state);
 
@@ -310,8 +311,8 @@ static int malidp_bind(struct device *dev)
 		return ret;
 
 	drm = drm_dev_alloc(&malidp_driver, dev);
-	if (!drm) {
-		ret = -ENOMEM;
+	if (IS_ERR(drm)) {
+		ret = PTR_ERR(drm);
 		goto alloc_fail;
 	}
 
diff --git a/drivers/gpu/drm/arm/malidp_drv.h b/drivers/gpu/drm/arm/malidp_drv.h
index 95558fde214b..271d2fb9711c 100644
--- a/drivers/gpu/drm/arm/malidp_drv.h
+++ b/drivers/gpu/drm/arm/malidp_drv.h
@@ -49,6 +49,6 @@ void malidp_de_planes_destroy(struct drm_device *drm);
 int malidp_crtc_init(struct drm_device *drm);
 
 /* often used combination of rotational bits */
-#define MALIDP_ROTATED_MASK	(BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270))
+#define MALIDP_ROTATED_MASK	(DRM_ROTATE_90 | DRM_ROTATE_270)
 
 #endif  /* __MALIDP_DRV_H__ */
diff --git a/drivers/gpu/drm/arm/malidp_planes.c b/drivers/gpu/drm/arm/malidp_planes.c
index 725098d6179a..82c193e5e0d6 100644
--- a/drivers/gpu/drm/arm/malidp_planes.c
+++ b/drivers/gpu/drm/arm/malidp_planes.c
@@ -108,7 +108,7 @@ static int malidp_de_plane_check(struct drm_plane *plane,
 		return -EINVAL;
 
 	/* packed RGB888 / BGR888 can't be rotated or flipped */
-	if (state->rotation != BIT(DRM_ROTATE_0) &&
+	if (state->rotation != DRM_ROTATE_0 &&
 	    (state->fb->pixel_format == DRM_FORMAT_RGB888 ||
 	     state->fb->pixel_format == DRM_FORMAT_BGR888))
 		return -EINVAL;
@@ -188,9 +188,9 @@ static void malidp_de_plane_update(struct drm_plane *plane,
 	/* setup the rotation and axis flip bits */
 	if (plane->state->rotation & DRM_ROTATE_MASK)
 		val = ilog2(plane->state->rotation & DRM_ROTATE_MASK) << LAYER_ROT_OFFSET;
-	if (plane->state->rotation & BIT(DRM_REFLECT_X))
+	if (plane->state->rotation & DRM_REFLECT_X)
 		val |= LAYER_V_FLIP;
-	if (plane->state->rotation & BIT(DRM_REFLECT_Y))
+	if (plane->state->rotation & DRM_REFLECT_Y)
 		val |= LAYER_H_FLIP;
 
 	/* set the 'enable layer' bit */
@@ -255,12 +255,12 @@ int malidp_de_planes_init(struct drm_device *drm)
 			goto cleanup;
 
 		if (!drm->mode_config.rotation_property) {
-			unsigned long flags = BIT(DRM_ROTATE_0) |
-					      BIT(DRM_ROTATE_90) |
-					      BIT(DRM_ROTATE_180) |
-					      BIT(DRM_ROTATE_270) |
-					      BIT(DRM_REFLECT_X) |
-					      BIT(DRM_REFLECT_Y);
+			unsigned long flags = DRM_ROTATE_0 |
+					      DRM_ROTATE_90 |
+					      DRM_ROTATE_180 |
+					      DRM_ROTATE_270 |
+					      DRM_REFLECT_X |
+					      DRM_REFLECT_Y;
 			drm->mode_config.rotation_property =
 				drm_mode_create_rotation_property(drm, flags);
 		}
@@ -268,7 +268,7 @@ int malidp_de_planes_init(struct drm_device *drm)
 		if (drm->mode_config.rotation_property && (id != DE_SMART))
 			drm_object_attach_property(&plane->base.base,
 						   drm->mode_config.rotation_property,
-						   BIT(DRM_ROTATE_0));
+						   DRM_ROTATE_0);
 
 		drm_plane_helper_add(&plane->base,
 				     &malidp_de_plane_helper_funcs);
diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
index f5ebdd681445..1e0e68f608e4 100644
--- a/drivers/gpu/drm/armada/armada_drv.c
+++ b/drivers/gpu/drm/armada/armada_drv.c
@@ -211,7 +211,7 @@ static struct drm_driver armada_drm_driver = {
 	.desc			= "Armada SoC DRM",
 	.date			= "20120730",
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET |
-				  DRIVER_HAVE_IRQ | DRIVER_PRIME,
+				  DRIVER_PRIME,
 	.ioctls			= armada_ioctls,
 	.fops			= &armada_drm_fops,
 };
diff --git a/drivers/gpu/drm/armada/armada_fbdev.c b/drivers/gpu/drm/armada/armada_fbdev.c
index 7d03c51abcb9..ca73ad8614fe 100644
--- a/drivers/gpu/drm/armada/armada_fbdev.c
+++ b/drivers/gpu/drm/armada/armada_fbdev.c
@@ -7,7 +7,6 @@
  * published by the Free Software Foundation.
  */
 #include <linux/errno.h>
-#include <linux/fb.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 
diff --git a/drivers/gpu/drm/armada/armada_gem.c b/drivers/gpu/drm/armada/armada_gem.c
index cb8f0347b934..806791897304 100644
--- a/drivers/gpu/drm/armada/armada_gem.c
+++ b/drivers/gpu/drm/armada/armada_gem.c
@@ -387,7 +387,7 @@ int armada_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 	if (!access_ok(VERIFY_READ, ptr, args->size))
 		return -EFAULT;
 
-	ret = fault_in_multipages_readable(ptr, args->size);
+	ret = fault_in_pages_readable(ptr, args->size);
 	if (ret)
 		return ret;
 
@@ -547,7 +547,7 @@ armada_gem_prime_export(struct drm_device *dev, struct drm_gem_object *obj,
 	exp_info.flags = O_RDWR;
 	exp_info.priv = obj;
 
-	return dma_buf_export(&exp_info);
+	return drm_gem_dmabuf_export(dev, &exp_info);
 }
 
 struct drm_gem_object *
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index 1ee707ef6b8d..152b4e716269 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -121,7 +121,7 @@ armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 	int ret;
 
 	ret = drm_plane_helper_check_update(plane, crtc, fb, &src, &dest, &clip,
-					    BIT(DRM_ROTATE_0),
+					    DRM_ROTATE_0,
 					    0, INT_MAX, true, false, &visible);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/ast/ast_fb.c b/drivers/gpu/drm/ast/ast_fb.c
index c017a9330a18..7a86e24e2687 100644
--- a/drivers/gpu/drm/ast/ast_fb.c
+++ b/drivers/gpu/drm/ast/ast_fb.c
@@ -33,7 +33,6 @@
 #include <linux/tty.h>
 #include <linux/sysrq.h>
 #include <linux/delay.h>
-#include <linux/fb.h>
 #include <linux/init.h>
 
 
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index b29a41218fc9..608df4c90520 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -150,7 +150,8 @@ static int ast_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 {
 	struct ast_bo *astbo = ast_bo(bo);
 
-	return drm_vma_node_verify_access(&astbo->gem.vma_node, filp);
+	return drm_vma_node_verify_access(&astbo->gem.vma_node,
+					  filp->private_data);
 }
 
 static int ast_ttm_io_mem_reserve(struct ttm_bo_device *bdev,
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
index a978381ef95b..9b17a66cf0e1 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
@@ -387,7 +387,7 @@ void atmel_hlcdc_crtc_irq(struct drm_crtc *c)
 	atmel_hlcdc_crtc_finish_page_flip(drm_crtc_to_atmel_hlcdc_crtc(c));
 }
 
-void atmel_hlcdc_crtc_reset(struct drm_crtc *crtc)
+static void atmel_hlcdc_crtc_reset(struct drm_crtc *crtc)
 {
 	struct atmel_hlcdc_crtc_state *state;
 
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index d4a3d61b7b06..5f484310bee9 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -457,7 +457,7 @@ atmel_hlcdc_dc_atomic_complete(struct atmel_hlcdc_dc_commit *commit)
 
 	/* Apply the atomic update. */
 	drm_atomic_helper_commit_modeset_disables(dev, old_state);
-	drm_atomic_helper_commit_planes(dev, old_state, false);
+	drm_atomic_helper_commit_planes(dev, old_state, 0);
 	drm_atomic_helper_commit_modeset_enables(dev, old_state);
 
 	drm_atomic_helper_wait_for_vblanks(dev, old_state);
@@ -797,8 +797,8 @@ static int atmel_hlcdc_dc_drm_probe(struct platform_device *pdev)
 	int ret;
 
 	ddev = drm_dev_alloc(&atmel_hlcdc_dc_driver, &pdev->dev);
-	if (!ddev)
-		return -ENOMEM;
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
 
 	ret = atmel_hlcdc_dc_load(ddev);
 	if (ret)
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
index 016c191221f3..9d4c030672f0 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
@@ -320,19 +320,19 @@ atmel_hlcdc_plane_update_pos_and_size(struct atmel_hlcdc_plane *plane,
 			u32 *coeff_tab = heo_upscaling_ycoef;
 			u32 max_memsize;
 
-			if (state->crtc_w < state->src_w)
+			if (state->crtc_h < state->src_h)
 				coeff_tab = heo_downscaling_ycoef;
 			for (i = 0; i < ARRAY_SIZE(heo_upscaling_ycoef); i++)
 				atmel_hlcdc_layer_update_cfg(&plane->layer,
 							     33 + i,
 							     0xffffffff,
 							     coeff_tab[i]);
-			factor = ((8 * 256 * state->src_w) - (256 * 4)) /
-				 state->crtc_w;
+			factor = ((8 * 256 * state->src_h) - (256 * 4)) /
+				 state->crtc_h;
 			factor++;
-			max_memsize = ((factor * state->crtc_w) + (256 * 4)) /
+			max_memsize = ((factor * state->crtc_h) + (256 * 4)) /
 				      2048;
-			if (max_memsize > state->src_w)
+			if (max_memsize > state->src_h)
 				factor--;
 			factor_reg |= (factor << 16) | 0x80000000;
 		}
@@ -393,7 +393,7 @@ static void atmel_hlcdc_plane_update_format(struct atmel_hlcdc_plane *plane,
 
 	if ((state->base.fb->pixel_format == DRM_FORMAT_YUV422 ||
 	     state->base.fb->pixel_format == DRM_FORMAT_NV61) &&
-	    (state->base.rotation & (BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270))))
+	    (state->base.rotation & (DRM_ROTATE_90 | DRM_ROTATE_270)))
 		cfg |= ATMEL_HLCDC_YUV422ROT;
 
 	atmel_hlcdc_layer_update_cfg(&plane->layer,
@@ -628,7 +628,7 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p,
 	/*
 	 * Swap width and size in case of 90 or 270 degrees rotation
 	 */
-	if (state->base.rotation & (BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270))) {
+	if (state->base.rotation & (DRM_ROTATE_90 | DRM_ROTATE_270)) {
 		tmp = state->crtc_w;
 		state->crtc_w = state->crtc_h;
 		state->crtc_h = tmp;
@@ -677,7 +677,7 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p,
 			return -EINVAL;
 
 		switch (state->base.rotation & DRM_ROTATE_MASK) {
-		case BIT(DRM_ROTATE_90):
+		case DRM_ROTATE_90:
 			offset = ((y_offset + state->src_y + patched_src_w - 1) /
 				  ydiv) * fb->pitches[i];
 			offset += ((x_offset + state->src_x) / xdiv) *
@@ -686,7 +686,7 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p,
 					  fb->pitches[i];
 			state->pstride[i] = -fb->pitches[i] - state->bpp[i];
 			break;
-		case BIT(DRM_ROTATE_180):
+		case DRM_ROTATE_180:
 			offset = ((y_offset + state->src_y + patched_src_h - 1) /
 				  ydiv) * fb->pitches[i];
 			offset += ((x_offset + state->src_x + patched_src_w - 1) /
@@ -695,7 +695,7 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p,
 					   state->bpp[i]) - fb->pitches[i];
 			state->pstride[i] = -2 * state->bpp[i];
 			break;
-		case BIT(DRM_ROTATE_270):
+		case DRM_ROTATE_270:
 			offset = ((y_offset + state->src_y) / ydiv) *
 				 fb->pitches[i];
 			offset += ((x_offset + state->src_x + patched_src_h - 1) /
@@ -705,7 +705,7 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p,
 					  (2 * state->bpp[i]);
 			state->pstride[i] = fb->pitches[i] - state->bpp[i];
 			break;
-		case BIT(DRM_ROTATE_0):
+		case DRM_ROTATE_0:
 		default:
 			offset = ((y_offset + state->src_y) / ydiv) *
 				 fb->pitches[i];
@@ -755,7 +755,7 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p,
 }
 
 static int atmel_hlcdc_plane_prepare_fb(struct drm_plane *p,
-					const struct drm_plane_state *new_state)
+					struct drm_plane_state *new_state)
 {
 	/*
 	 * FIXME: we should avoid this const -> non-const cast but it's
@@ -780,7 +780,7 @@ static int atmel_hlcdc_plane_prepare_fb(struct drm_plane *p,
 }
 
 static void atmel_hlcdc_plane_cleanup_fb(struct drm_plane *p,
-				const struct drm_plane_state *old_state)
+					 struct drm_plane_state *old_state)
 {
 	/*
 	 * FIXME: we should avoid this const -> non-const cast but it's
@@ -905,7 +905,7 @@ static void atmel_hlcdc_plane_init_properties(struct atmel_hlcdc_plane *plane,
 	if (desc->layout.xstride && desc->layout.pstride)
 		drm_object_attach_property(&plane->base.base,
 				plane->base.dev->mode_config.rotation_property,
-				BIT(DRM_ROTATE_0));
+				DRM_ROTATE_0);
 
 	if (desc->layout.csc) {
 		/*
@@ -1056,10 +1056,10 @@ atmel_hlcdc_plane_create_properties(struct drm_device *dev)
 
 	dev->mode_config.rotation_property =
 			drm_mode_create_rotation_property(dev,
-							  BIT(DRM_ROTATE_0) |
-							  BIT(DRM_ROTATE_90) |
-							  BIT(DRM_ROTATE_180) |
-							  BIT(DRM_ROTATE_270));
+							  DRM_ROTATE_0 |
+							  DRM_ROTATE_90 |
+							  DRM_ROTATE_180 |
+							  DRM_ROTATE_270);
 	if (!dev->mode_config.rotation_property)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/gpu/drm/bochs/bochs.h b/drivers/gpu/drm/bochs/bochs.h
index 19b5adaebe24..32dfe418cc98 100644
--- a/drivers/gpu/drm/bochs/bochs.h
+++ b/drivers/gpu/drm/bochs/bochs.h
@@ -1,5 +1,4 @@
 #include <linux/io.h>
-#include <linux/fb.h>
 #include <linux/console.h>
 
 #include <drm/drmP.h>
diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c
index abace82de6ea..534227df23f3 100644
--- a/drivers/gpu/drm/bochs/bochs_drv.c
+++ b/drivers/gpu/drm/bochs/bochs_drv.c
@@ -8,6 +8,7 @@
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <drm/drm_fb_helper.h>
 
 #include "bochs.h"
 
@@ -153,7 +154,7 @@ static int bochs_kick_out_firmware_fb(struct pci_dev *pdev)
 
 	ap->ranges[0].base = pci_resource_start(pdev, 0);
 	ap->ranges[0].size = pci_resource_len(pdev, 0);
-	remove_conflicting_framebuffers(ap, "bochsdrmfb", false);
+	drm_fb_helper_remove_conflicting_framebuffers(ap, "bochsdrmfb", false);
 	kfree(ap);
 
 	return 0;
@@ -162,8 +163,15 @@ static int bochs_kick_out_firmware_fb(struct pci_dev *pdev)
 static int bochs_pci_probe(struct pci_dev *pdev,
 			   const struct pci_device_id *ent)
 {
+	unsigned long fbsize;
 	int ret;
 
+	fbsize = pci_resource_len(pdev, 0);
+	if (fbsize < 4 * 1024 * 1024) {
+		DRM_ERROR("less than 4 MB video memory, ignoring device\n");
+		return -ENOMEM;
+	}
+
 	ret = bochs_kick_out_firmware_fb(pdev);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/bochs/bochs_kms.c b/drivers/gpu/drm/bochs/bochs_kms.c
index 207a2cbcc113..0b4e5d117043 100644
--- a/drivers/gpu/drm/bochs/bochs_kms.c
+++ b/drivers/gpu/drm/bochs/bochs_kms.c
@@ -178,7 +178,7 @@ static void bochs_encoder_init(struct drm_device *dev)
 }
 
 
-int bochs_connector_get_modes(struct drm_connector *connector)
+static int bochs_connector_get_modes(struct drm_connector *connector)
 {
 	int count;
 
diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index 5c5638a777a1..269cfca9ca06 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -128,7 +128,8 @@ static int bochs_bo_verify_access(struct ttm_buffer_object *bo,
 {
 	struct bochs_bo *bochsbo = bochs_bo(bo);
 
-	return drm_vma_node_verify_access(&bochsbo->gem.vma_node, filp);
+	return drm_vma_node_verify_access(&bochsbo->gem.vma_node,
+					  filp->private_data);
 }
 
 static int bochs_ttm_io_mem_reserve(struct ttm_bo_device *bdev,
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index b590e678052d..10e12e74fc9f 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -17,6 +17,13 @@ config DRM_ANALOGIX_ANX78XX
 	  the HDMI output of an application processor to MyDP
 	  or DisplayPort.
 
+config DRM_DUMB_VGA_DAC
+	tristate "Dumb VGA DAC Bridge support"
+	depends on OF
+	select DRM_KMS_HELPER
+	help
+	  Support for RGB to VGA DAC based bridges
+
 config DRM_DW_HDMI
 	tristate
 	select DRM_KMS_HELPER
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
index efdb07e878f5..cdf3a3cf765d 100644
--- a/drivers/gpu/drm/bridge/Makefile
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -1,6 +1,7 @@
 ccflags-y := -Iinclude/drm
 
 obj-$(CONFIG_DRM_ANALOGIX_ANX78XX) += analogix-anx78xx.o
+obj-$(CONFIG_DRM_DUMB_VGA_DAC) += dumb-vga-dac.o
 obj-$(CONFIG_DRM_DW_HDMI) += dw-hdmi.o
 obj-$(CONFIG_DRM_DW_HDMI_AHB_AUDIO) += dw-hdmi-ahb-audio.o
 obj-$(CONFIG_DRM_NXP_PTN3460) += nxp-ptn3460.o
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index ec8fb2ed3275..8ed3906dd411 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -922,15 +922,13 @@ static int adv7511_parse_dt(struct device_node *np,
 	return 0;
 }
 
-static const int edid_i2c_addr = 0x7e;
-static const int packet_i2c_addr = 0x70;
-static const int cec_i2c_addr = 0x78;
-
 static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 {
 	struct adv7511_link_config link_config;
 	struct adv7511 *adv7511;
 	struct device *dev = &i2c->dev;
+	unsigned int main_i2c_addr = i2c->addr << 1;
+	unsigned int edid_i2c_addr = main_i2c_addr + 4;
 	unsigned int val;
 	int ret;
 
@@ -991,8 +989,10 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 
 	regmap_write(adv7511->regmap, ADV7511_REG_EDID_I2C_ADDR, edid_i2c_addr);
 	regmap_write(adv7511->regmap, ADV7511_REG_PACKET_I2C_ADDR,
-		     packet_i2c_addr);
-	regmap_write(adv7511->regmap, ADV7511_REG_CEC_I2C_ADDR, cec_i2c_addr);
+		     main_i2c_addr - 0xa);
+	regmap_write(adv7511->regmap, ADV7511_REG_CEC_I2C_ADDR,
+		     main_i2c_addr - 2);
+
 	adv7511_packet_disable(adv7511, 0xffff);
 
 	adv7511->i2c_main = i2c;
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7533.c b/drivers/gpu/drm/bridge/adv7511/adv7533.c
index 5eebd15899b1..d7f7b7ce8ebe 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7533.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7533.c
@@ -149,13 +149,12 @@ void adv7533_uninit_cec(struct adv7511 *adv)
 	i2c_unregister_device(adv->i2c_cec);
 }
 
-static const int cec_i2c_addr = 0x78;
-
 int adv7533_init_cec(struct adv7511 *adv)
 {
 	int ret;
 
-	adv->i2c_cec = i2c_new_dummy(adv->i2c_main->adapter, cec_i2c_addr >> 1);
+	adv->i2c_cec = i2c_new_dummy(adv->i2c_main->adapter,
+				     adv->i2c_main->addr - 1);
 	if (!adv->i2c_cec)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/bridge/analogix-anx78xx.c b/drivers/gpu/drm/bridge/analogix-anx78xx.c
index f9f03bcba0af..a2a82366a771 100644
--- a/drivers/gpu/drm/bridge/analogix-anx78xx.c
+++ b/drivers/gpu/drm/bridge/analogix-anx78xx.c
@@ -1001,16 +1001,11 @@ static enum drm_connector_status anx78xx_detect(struct drm_connector *connector,
 	return connector_status_connected;
 }
 
-static void anx78xx_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_cleanup(connector);
-}
-
 static const struct drm_connector_funcs anx78xx_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = anx78xx_detect,
-	.destroy = anx78xx_connector_destroy,
+	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index 32715daf73cb..6e0447f329a2 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -31,6 +31,7 @@
 #include <drm/bridge/analogix_dp.h>
 
 #include "analogix_dp_core.h"
+#include "analogix_dp_reg.h"
 
 #define to_dp(nm)	container_of(nm, struct analogix_dp_device, nm)
 
@@ -97,133 +98,89 @@ static int analogix_dp_detect_hpd(struct analogix_dp_device *dp)
 	return 0;
 }
 
-static unsigned char analogix_dp_calc_edid_check_sum(unsigned char *edid_data)
+int analogix_dp_psr_supported(struct device *dev)
 {
-	int i;
-	unsigned char sum = 0;
-
-	for (i = 0; i < EDID_BLOCK_LENGTH; i++)
-		sum = sum + edid_data[i];
+	struct analogix_dp_device *dp = dev_get_drvdata(dev);
 
-	return sum;
+	return dp->psr_support;
 }
+EXPORT_SYMBOL_GPL(analogix_dp_psr_supported);
 
-static int analogix_dp_read_edid(struct analogix_dp_device *dp)
+int analogix_dp_enable_psr(struct device *dev)
 {
-	unsigned char *edid = dp->edid;
-	unsigned int extend_block = 0;
-	unsigned char sum;
-	unsigned char test_vector;
-	int retval;
+	struct analogix_dp_device *dp = dev_get_drvdata(dev);
+	struct edp_vsc_psr psr_vsc;
 
-	/*
-	 * EDID device address is 0x50.
-	 * However, if necessary, you must have set upper address
-	 * into E-EDID in I2C device, 0x30.
-	 */
+	if (!dp->psr_support)
+		return -EINVAL;
 
-	/* Read Extension Flag, Number of 128-byte EDID extension blocks */
-	retval = analogix_dp_read_byte_from_i2c(dp, I2C_EDID_DEVICE_ADDR,
-						EDID_EXTENSION_FLAG,
-						&extend_block);
-	if (retval)
-		return retval;
+	/* Prepare VSC packet as per EDP 1.4 spec, Table 6.9 */
+	memset(&psr_vsc, 0, sizeof(psr_vsc));
+	psr_vsc.sdp_header.HB0 = 0;
+	psr_vsc.sdp_header.HB1 = 0x7;
+	psr_vsc.sdp_header.HB2 = 0x2;
+	psr_vsc.sdp_header.HB3 = 0x8;
 
-	if (extend_block > 0) {
-		dev_dbg(dp->dev, "EDID data includes a single extension!\n");
-
-		/* Read EDID data */
-		retval = analogix_dp_read_bytes_from_i2c(dp,
-						I2C_EDID_DEVICE_ADDR,
-						EDID_HEADER_PATTERN,
-						EDID_BLOCK_LENGTH,
-						&edid[EDID_HEADER_PATTERN]);
-		if (retval != 0) {
-			dev_err(dp->dev, "EDID Read failed!\n");
-			return -EIO;
-		}
-		sum = analogix_dp_calc_edid_check_sum(edid);
-		if (sum != 0) {
-			dev_err(dp->dev, "EDID bad checksum!\n");
-			return -EIO;
-		}
+	psr_vsc.DB0 = 0;
+	psr_vsc.DB1 = EDP_VSC_PSR_STATE_ACTIVE | EDP_VSC_PSR_CRC_VALUES_VALID;
 
-		/* Read additional EDID data */
-		retval = analogix_dp_read_bytes_from_i2c(dp,
-				I2C_EDID_DEVICE_ADDR,
-				EDID_BLOCK_LENGTH,
-				EDID_BLOCK_LENGTH,
-				&edid[EDID_BLOCK_LENGTH]);
-		if (retval != 0) {
-			dev_err(dp->dev, "EDID Read failed!\n");
-			return -EIO;
-		}
-		sum = analogix_dp_calc_edid_check_sum(&edid[EDID_BLOCK_LENGTH]);
-		if (sum != 0) {
-			dev_err(dp->dev, "EDID bad checksum!\n");
-			return -EIO;
-		}
+	analogix_dp_send_psr_spd(dp, &psr_vsc);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(analogix_dp_enable_psr);
 
-		analogix_dp_read_byte_from_dpcd(dp, DP_TEST_REQUEST,
-						&test_vector);
-		if (test_vector & DP_TEST_LINK_EDID_READ) {
-			analogix_dp_write_byte_to_dpcd(dp,
-				DP_TEST_EDID_CHECKSUM,
-				edid[EDID_BLOCK_LENGTH + EDID_CHECKSUM]);
-			analogix_dp_write_byte_to_dpcd(dp,
-				DP_TEST_RESPONSE,
-				DP_TEST_EDID_CHECKSUM_WRITE);
-		}
-	} else {
-		dev_info(dp->dev, "EDID data does not include any extensions.\n");
-
-		/* Read EDID data */
-		retval = analogix_dp_read_bytes_from_i2c(dp,
-				I2C_EDID_DEVICE_ADDR, EDID_HEADER_PATTERN,
-				EDID_BLOCK_LENGTH, &edid[EDID_HEADER_PATTERN]);
-		if (retval != 0) {
-			dev_err(dp->dev, "EDID Read failed!\n");
-			return -EIO;
-		}
-		sum = analogix_dp_calc_edid_check_sum(edid);
-		if (sum != 0) {
-			dev_err(dp->dev, "EDID bad checksum!\n");
-			return -EIO;
-		}
+int analogix_dp_disable_psr(struct device *dev)
+{
+	struct analogix_dp_device *dp = dev_get_drvdata(dev);
+	struct edp_vsc_psr psr_vsc;
 
-		analogix_dp_read_byte_from_dpcd(dp, DP_TEST_REQUEST,
-						&test_vector);
-		if (test_vector & DP_TEST_LINK_EDID_READ) {
-			analogix_dp_write_byte_to_dpcd(dp,
-				DP_TEST_EDID_CHECKSUM, edid[EDID_CHECKSUM]);
-			analogix_dp_write_byte_to_dpcd(dp,
-				DP_TEST_RESPONSE, DP_TEST_EDID_CHECKSUM_WRITE);
-		}
-	}
+	if (!dp->psr_support)
+		return -EINVAL;
+
+	/* Prepare VSC packet as per EDP 1.4 spec, Table 6.9 */
+	memset(&psr_vsc, 0, sizeof(psr_vsc));
+	psr_vsc.sdp_header.HB0 = 0;
+	psr_vsc.sdp_header.HB1 = 0x7;
+	psr_vsc.sdp_header.HB2 = 0x2;
+	psr_vsc.sdp_header.HB3 = 0x8;
+
+	psr_vsc.DB0 = 0;
+	psr_vsc.DB1 = 0;
 
-	dev_dbg(dp->dev, "EDID Read success!\n");
+	analogix_dp_send_psr_spd(dp, &psr_vsc);
 	return 0;
 }
+EXPORT_SYMBOL_GPL(analogix_dp_disable_psr);
 
-static int analogix_dp_handle_edid(struct analogix_dp_device *dp)
+static bool analogix_dp_detect_sink_psr(struct analogix_dp_device *dp)
 {
-	u8 buf[12];
-	int i;
-	int retval;
+	unsigned char psr_version;
 
-	/* Read DPCD DP_DPCD_REV~RECEIVE_PORT1_CAP_1 */
-	retval = analogix_dp_read_bytes_from_dpcd(dp, DP_DPCD_REV, 12, buf);
-	if (retval)
-		return retval;
+	drm_dp_dpcd_readb(&dp->aux, DP_PSR_SUPPORT, &psr_version);
+	dev_dbg(dp->dev, "Panel PSR version : %x\n", psr_version);
 
-	/* Read EDID */
-	for (i = 0; i < 3; i++) {
-		retval = analogix_dp_read_edid(dp);
-		if (!retval)
-			break;
-	}
+	return (psr_version & DP_PSR_IS_SUPPORTED) ? true : false;
+}
 
-	return retval;
+static void analogix_dp_enable_sink_psr(struct analogix_dp_device *dp)
+{
+	unsigned char psr_en;
+
+	/* Disable psr function */
+	drm_dp_dpcd_readb(&dp->aux, DP_PSR_EN_CFG, &psr_en);
+	psr_en &= ~DP_PSR_ENABLE;
+	drm_dp_dpcd_writeb(&dp->aux, DP_PSR_EN_CFG, psr_en);
+
+	/* Main-Link transmitter remains active during PSR active states */
+	psr_en = DP_PSR_MAIN_LINK_ACTIVE | DP_PSR_CRC_VERIFICATION;
+	drm_dp_dpcd_writeb(&dp->aux, DP_PSR_EN_CFG, psr_en);
+
+	/* Enable psr function */
+	psr_en = DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE |
+		 DP_PSR_CRC_VERIFICATION;
+	drm_dp_dpcd_writeb(&dp->aux, DP_PSR_EN_CFG, psr_en);
+
+	analogix_dp_enable_psr_crc(dp);
 }
 
 static void
@@ -232,15 +189,15 @@ analogix_dp_enable_rx_to_enhanced_mode(struct analogix_dp_device *dp,
 {
 	u8 data;
 
-	analogix_dp_read_byte_from_dpcd(dp, DP_LANE_COUNT_SET, &data);
+	drm_dp_dpcd_readb(&dp->aux, DP_LANE_COUNT_SET, &data);
 
 	if (enable)
-		analogix_dp_write_byte_to_dpcd(dp, DP_LANE_COUNT_SET,
-					       DP_LANE_COUNT_ENHANCED_FRAME_EN |
-					       DPCD_LANE_COUNT_SET(data));
+		drm_dp_dpcd_writeb(&dp->aux, DP_LANE_COUNT_SET,
+				   DP_LANE_COUNT_ENHANCED_FRAME_EN |
+					DPCD_LANE_COUNT_SET(data));
 	else
-		analogix_dp_write_byte_to_dpcd(dp, DP_LANE_COUNT_SET,
-					       DPCD_LANE_COUNT_SET(data));
+		drm_dp_dpcd_writeb(&dp->aux, DP_LANE_COUNT_SET,
+				   DPCD_LANE_COUNT_SET(data));
 }
 
 static int analogix_dp_is_enhanced_mode_available(struct analogix_dp_device *dp)
@@ -248,7 +205,7 @@ static int analogix_dp_is_enhanced_mode_available(struct analogix_dp_device *dp)
 	u8 data;
 	int retval;
 
-	analogix_dp_read_byte_from_dpcd(dp, DP_MAX_LANE_COUNT, &data);
+	drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &data);
 	retval = DPCD_ENHANCED_FRAME_CAP(data);
 
 	return retval;
@@ -267,8 +224,8 @@ static void analogix_dp_training_pattern_dis(struct analogix_dp_device *dp)
 {
 	analogix_dp_set_training_pattern(dp, DP_NONE);
 
-	analogix_dp_write_byte_to_dpcd(dp, DP_TRAINING_PATTERN_SET,
-				       DP_TRAINING_PATTERN_DISABLE);
+	drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
+			   DP_TRAINING_PATTERN_DISABLE);
 }
 
 static void
@@ -313,8 +270,8 @@ static int analogix_dp_link_start(struct analogix_dp_device *dp)
 	/* Setup RX configuration */
 	buf[0] = dp->link_train.link_rate;
 	buf[1] = dp->link_train.lane_count;
-	retval = analogix_dp_write_bytes_to_dpcd(dp, DP_LINK_BW_SET, 2, buf);
-	if (retval)
+	retval = drm_dp_dpcd_write(&dp->aux, DP_LINK_BW_SET, buf, 2);
+	if (retval < 0)
 		return retval;
 
 	/* Set TX pre-emphasis to minimum */
@@ -338,20 +295,22 @@ static int analogix_dp_link_start(struct analogix_dp_device *dp)
 	analogix_dp_set_training_pattern(dp, TRAINING_PTN1);
 
 	/* Set RX training pattern */
-	retval = analogix_dp_write_byte_to_dpcd(dp,
-			DP_TRAINING_PATTERN_SET,
-			DP_LINK_SCRAMBLING_DISABLE | DP_TRAINING_PATTERN_1);
-	if (retval)
+	retval = drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
+				    DP_LINK_SCRAMBLING_DISABLE |
+					DP_TRAINING_PATTERN_1);
+	if (retval < 0)
 		return retval;
 
 	for (lane = 0; lane < lane_count; lane++)
 		buf[lane] = DP_TRAIN_PRE_EMPH_LEVEL_0 |
 			    DP_TRAIN_VOLTAGE_SWING_LEVEL_0;
 
-	retval = analogix_dp_write_bytes_to_dpcd(dp, DP_TRAINING_LANE0_SET,
-						 lane_count, buf);
+	retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, buf,
+				   lane_count);
+	if (retval < 0)
+		return retval;
 
-	return retval;
+	return 0;
 }
 
 static unsigned char analogix_dp_get_lane_status(u8 link_status[2], int lane)
@@ -503,25 +462,23 @@ static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp)
 
 	lane_count = dp->link_train.lane_count;
 
-	retval =  analogix_dp_read_bytes_from_dpcd(dp,
-			DP_LANE0_1_STATUS, 2, link_status);
-	if (retval)
+	retval = drm_dp_dpcd_read(&dp->aux, DP_LANE0_1_STATUS, link_status, 2);
+	if (retval < 0)
 		return retval;
 
-	retval =  analogix_dp_read_bytes_from_dpcd(dp,
-			DP_ADJUST_REQUEST_LANE0_1, 2, adjust_request);
-	if (retval)
+	retval = drm_dp_dpcd_read(&dp->aux, DP_ADJUST_REQUEST_LANE0_1,
+				  adjust_request, 2);
+	if (retval < 0)
 		return retval;
 
 	if (analogix_dp_clock_recovery_ok(link_status, lane_count) == 0) {
 		/* set training pattern 2 for EQ */
 		analogix_dp_set_training_pattern(dp, TRAINING_PTN2);
 
-		retval = analogix_dp_write_byte_to_dpcd(dp,
-				DP_TRAINING_PATTERN_SET,
-				DP_LINK_SCRAMBLING_DISABLE |
-				DP_TRAINING_PATTERN_2);
-		if (retval)
+		retval = drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
+					    DP_LINK_SCRAMBLING_DISABLE |
+						DP_TRAINING_PATTERN_2);
+		if (retval < 0)
 			return retval;
 
 		dev_info(dp->dev, "Link Training Clock Recovery success\n");
@@ -559,13 +516,12 @@ static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp)
 		analogix_dp_set_lane_link_training(dp,
 			dp->link_train.training_lane[lane], lane);
 
-	retval = analogix_dp_write_bytes_to_dpcd(dp,
-			DP_TRAINING_LANE0_SET, lane_count,
-			dp->link_train.training_lane);
-	if (retval)
+	retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET,
+				   dp->link_train.training_lane, lane_count);
+	if (retval < 0)
 		return retval;
 
-	return retval;
+	return 0;
 }
 
 static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp)
@@ -578,9 +534,8 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp)
 
 	lane_count = dp->link_train.lane_count;
 
-	retval = analogix_dp_read_bytes_from_dpcd(dp,
-			DP_LANE0_1_STATUS, 2, link_status);
-	if (retval)
+	retval = drm_dp_dpcd_read(&dp->aux, DP_LANE0_1_STATUS, link_status, 2);
+	if (retval < 0)
 		return retval;
 
 	if (analogix_dp_clock_recovery_ok(link_status, lane_count)) {
@@ -588,14 +543,14 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp)
 		return -EIO;
 	}
 
-	retval = analogix_dp_read_bytes_from_dpcd(dp,
-			DP_ADJUST_REQUEST_LANE0_1, 2, adjust_request);
-	if (retval)
+	retval = drm_dp_dpcd_read(&dp->aux, DP_ADJUST_REQUEST_LANE0_1,
+				  adjust_request, 2);
+	if (retval < 0)
 		return retval;
 
-	retval = analogix_dp_read_byte_from_dpcd(dp,
-			DP_LANE_ALIGN_STATUS_UPDATED, &link_align);
-	if (retval)
+	retval = drm_dp_dpcd_readb(&dp->aux, DP_LANE_ALIGN_STATUS_UPDATED,
+				   &link_align);
+	if (retval < 0)
 		return retval;
 
 	analogix_dp_get_adjust_training_lane(dp, adjust_request);
@@ -636,10 +591,12 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp)
 		analogix_dp_set_lane_link_training(dp,
 			dp->link_train.training_lane[lane], lane);
 
-	retval = analogix_dp_write_bytes_to_dpcd(dp, DP_TRAINING_LANE0_SET,
-			lane_count, dp->link_train.training_lane);
+	retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET,
+				   dp->link_train.training_lane, lane_count);
+	if (retval < 0)
+		return retval;
 
-	return retval;
+	return 0;
 }
 
 static void analogix_dp_get_max_rx_bandwidth(struct analogix_dp_device *dp,
@@ -653,7 +610,7 @@ static void analogix_dp_get_max_rx_bandwidth(struct analogix_dp_device *dp,
 	 * For DP rev.1.2, Maximum link rate of Main Link lanes
 	 * 0x06 = 1.62 Gbps, 0x0a = 2.7 Gbps, 0x14 = 5.4Gbps
 	 */
-	analogix_dp_read_byte_from_dpcd(dp, DP_MAX_LINK_RATE, &data);
+	drm_dp_dpcd_readb(&dp->aux, DP_MAX_LINK_RATE, &data);
 	*bandwidth = data;
 }
 
@@ -666,7 +623,7 @@ static void analogix_dp_get_max_rx_lane_count(struct analogix_dp_device *dp,
 	 * For DP rev.1.1, Maximum number of Main Link lanes
 	 * 0x01 = 1 lane, 0x02 = 2 lanes, 0x04 = 4 lanes
 	 */
-	analogix_dp_read_byte_from_dpcd(dp, DP_MAX_LANE_COUNT, &data);
+	drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &data);
 	*lane_count = DPCD_MAX_LANE_COUNT(data);
 }
 
@@ -835,19 +792,15 @@ static void analogix_dp_enable_scramble(struct analogix_dp_device *dp,
 	if (enable) {
 		analogix_dp_enable_scrambling(dp);
 
-		analogix_dp_read_byte_from_dpcd(dp, DP_TRAINING_PATTERN_SET,
-						&data);
-		analogix_dp_write_byte_to_dpcd(dp,
-			DP_TRAINING_PATTERN_SET,
-			(u8)(data & ~DP_LINK_SCRAMBLING_DISABLE));
+		drm_dp_dpcd_readb(&dp->aux, DP_TRAINING_PATTERN_SET, &data);
+		drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
+				   (u8)(data & ~DP_LINK_SCRAMBLING_DISABLE));
 	} else {
 		analogix_dp_disable_scrambling(dp);
 
-		analogix_dp_read_byte_from_dpcd(dp, DP_TRAINING_PATTERN_SET,
-						&data);
-		analogix_dp_write_byte_to_dpcd(dp,
-			DP_TRAINING_PATTERN_SET,
-			(u8)(data | DP_LINK_SCRAMBLING_DISABLE));
+		drm_dp_dpcd_readb(&dp->aux, DP_TRAINING_PATTERN_SET, &data);
+		drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET,
+				   (u8)(data | DP_LINK_SCRAMBLING_DISABLE));
 	}
 }
 
@@ -921,21 +874,85 @@ static void analogix_dp_commit(struct analogix_dp_device *dp)
 
 	/* Enable video */
 	analogix_dp_start_video(dp);
+
+	dp->psr_support = analogix_dp_detect_sink_psr(dp);
+	if (dp->psr_support)
+		analogix_dp_enable_sink_psr(dp);
 }
 
-int analogix_dp_get_modes(struct drm_connector *connector)
+/*
+ * This function is a bit of a catch-all for panel preparation, hopefully
+ * simplifying the logic of functions that need to prepare/unprepare the panel
+ * below.
+ *
+ * If @prepare is true, this function will prepare the panel. Conversely, if it
+ * is false, the panel will be unprepared.
+ *
+ * If @is_modeset_prepare is true, the function will disregard the current state
+ * of the panel and either prepare/unprepare the panel based on @prepare. Once
+ * it finishes, it will update dp->panel_is_modeset to reflect the current state
+ * of the panel.
+ */
+static int analogix_dp_prepare_panel(struct analogix_dp_device *dp,
+				     bool prepare, bool is_modeset_prepare)
 {
-	struct analogix_dp_device *dp = to_dp(connector);
-	struct edid *edid = (struct edid *)dp->edid;
-	int num_modes = 0;
+	int ret = 0;
 
-	if (analogix_dp_handle_edid(dp) == 0) {
-		drm_mode_connector_update_edid_property(&dp->connector, edid);
-		num_modes += drm_add_edid_modes(&dp->connector, edid);
-	}
+	if (!dp->plat_data->panel)
+		return 0;
 
-	if (dp->plat_data->panel)
+	mutex_lock(&dp->panel_lock);
+
+	/*
+	 * Exit early if this is a temporary prepare/unprepare and we're already
+	 * modeset (since we neither want to prepare twice or unprepare early).
+	 */
+	if (dp->panel_is_modeset && !is_modeset_prepare)
+		goto out;
+
+	if (prepare)
+		ret = drm_panel_prepare(dp->plat_data->panel);
+	else
+		ret = drm_panel_unprepare(dp->plat_data->panel);
+
+	if (ret)
+		goto out;
+
+	if (is_modeset_prepare)
+		dp->panel_is_modeset = prepare;
+
+out:
+	mutex_unlock(&dp->panel_lock);
+	return ret;
+}
+
+static int analogix_dp_get_modes(struct drm_connector *connector)
+{
+	struct analogix_dp_device *dp = to_dp(connector);
+	struct edid *edid;
+	int ret, num_modes = 0;
+
+	if (dp->plat_data->panel) {
 		num_modes += drm_panel_get_modes(dp->plat_data->panel);
+	} else {
+		ret = analogix_dp_prepare_panel(dp, true, false);
+		if (ret) {
+			DRM_ERROR("Failed to prepare panel (%d)\n", ret);
+			return 0;
+		}
+
+		edid = drm_get_edid(connector, &dp->aux.ddc);
+		if (edid) {
+			drm_mode_connector_update_edid_property(&dp->connector,
+								edid);
+			num_modes += drm_add_edid_modes(&dp->connector, edid);
+			kfree(edid);
+		}
+
+		ret = analogix_dp_prepare_panel(dp, false, false);
+		if (ret)
+			DRM_ERROR("Failed to unprepare panel (%d)\n", ret);
+	}
 
 	if (dp->plat_data->get_modes)
 		num_modes += dp->plat_data->get_modes(dp->plat_data, connector);
@@ -956,29 +973,37 @@ static const struct drm_connector_helper_funcs analogix_dp_connector_helper_func
 	.best_encoder = analogix_dp_best_encoder,
 };
 
-enum drm_connector_status
+static enum drm_connector_status
 analogix_dp_detect(struct drm_connector *connector, bool force)
 {
 	struct analogix_dp_device *dp = to_dp(connector);
+	enum drm_connector_status status = connector_status_disconnected;
+	int ret;
 
-	if (analogix_dp_detect_hpd(dp))
+	if (dp->plat_data->panel)
+		return connector_status_connected;
+
+	ret = analogix_dp_prepare_panel(dp, true, false);
+	if (ret) {
+		DRM_ERROR("Failed to prepare panel (%d)\n", ret);
 		return connector_status_disconnected;
+	}
 
-	return connector_status_connected;
-}
+	if (!analogix_dp_detect_hpd(dp))
+		status = connector_status_connected;
 
-static void analogix_dp_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
+	ret = analogix_dp_prepare_panel(dp, false, false);
+	if (ret)
+		DRM_ERROR("Failed to unprepare panel (%d)\n", ret);
 
+	return status;
 }
 
 static const struct drm_connector_funcs analogix_dp_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = analogix_dp_detect,
-	.destroy = analogix_dp_connector_destroy,
+	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
@@ -1035,6 +1060,16 @@ static int analogix_dp_bridge_attach(struct drm_bridge *bridge)
 	return 0;
 }
 
+static void analogix_dp_bridge_pre_enable(struct drm_bridge *bridge)
+{
+	struct analogix_dp_device *dp = bridge->driver_private;
+	int ret;
+
+	ret = analogix_dp_prepare_panel(dp, true, true);
+	if (ret)
+		DRM_ERROR("failed to setup the panel ret = %d\n", ret);
+}
+
 static void analogix_dp_bridge_enable(struct drm_bridge *bridge)
 {
 	struct analogix_dp_device *dp = bridge->driver_private;
@@ -1058,6 +1093,7 @@ static void analogix_dp_bridge_enable(struct drm_bridge *bridge)
 static void analogix_dp_bridge_disable(struct drm_bridge *bridge)
 {
 	struct analogix_dp_device *dp = bridge->driver_private;
+	int ret;
 
 	if (dp->dpms_mode != DRM_MODE_DPMS_ON)
 		return;
@@ -1077,6 +1113,10 @@ static void analogix_dp_bridge_disable(struct drm_bridge *bridge)
 
 	pm_runtime_put_sync(dp->dev);
 
+	ret = analogix_dp_prepare_panel(dp, false, true);
+	if (ret)
+		DRM_ERROR("failed to setup the panel ret = %d\n", ret);
+
 	dp->dpms_mode = DRM_MODE_DPMS_OFF;
 }
 
@@ -1165,9 +1205,9 @@ static void analogix_dp_bridge_nop(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs analogix_dp_bridge_funcs = {
+	.pre_enable = analogix_dp_bridge_pre_enable,
 	.enable = analogix_dp_bridge_enable,
 	.disable = analogix_dp_bridge_disable,
-	.pre_enable = analogix_dp_bridge_nop,
 	.post_disable = analogix_dp_bridge_nop,
 	.mode_set = analogix_dp_bridge_mode_set,
 	.attach = analogix_dp_bridge_attach,
@@ -1231,6 +1271,14 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp)
 	return 0;
 }
 
+static ssize_t analogix_dpaux_transfer(struct drm_dp_aux *aux,
+				       struct drm_dp_aux_msg *msg)
+{
+	struct analogix_dp_device *dp = to_dp(aux);
+
+	return analogix_dp_transfer(dp, msg);
+}
+
 int analogix_dp_bind(struct device *dev, struct drm_device *drm_dev,
 		     struct analogix_dp_plat_data *plat_data)
 {
@@ -1254,6 +1302,9 @@ int analogix_dp_bind(struct device *dev, struct drm_device *drm_dev,
 	dp->dev = &pdev->dev;
 	dp->dpms_mode = DRM_MODE_DPMS_OFF;
 
+	mutex_init(&dp->panel_lock);
+	dp->panel_is_modeset = false;
+
 	/*
 	 * platform dp driver need containor_of the plat_data to get
 	 * the driver private data, so we need to store the point of
@@ -1333,13 +1384,6 @@ int analogix_dp_bind(struct device *dev, struct drm_device *drm_dev,
 
 	phy_power_on(dp->phy);
 
-	if (dp->plat_data->panel) {
-		if (drm_panel_prepare(dp->plat_data->panel)) {
-			DRM_ERROR("failed to setup the panel\n");
-			return -EBUSY;
-		}
-	}
-
 	analogix_dp_init_dp(dp);
 
 	ret = devm_request_threaded_irq(&pdev->dev, dp->irq,
@@ -1355,6 +1399,14 @@ int analogix_dp_bind(struct device *dev, struct drm_device *drm_dev,
 	dp->drm_dev = drm_dev;
 	dp->encoder = dp->plat_data->encoder;
 
+	dp->aux.name = "DP-AUX";
+	dp->aux.transfer = analogix_dpaux_transfer;
+	dp->aux.dev = &pdev->dev;
+
+	ret = drm_dp_aux_register(&dp->aux);
+	if (ret)
+		goto err_disable_pm_runtime;
+
 	ret = analogix_dp_create_bridge(drm_dev, dp);
 	if (ret) {
 		DRM_ERROR("failed to create bridge (%d)\n", ret);
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
index b45638043ec4..5c6a28806129 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
@@ -20,15 +20,6 @@
 #define MAX_CR_LOOP 5
 #define MAX_EQ_LOOP 5
 
-/* I2C EDID Chip ID, Slave Address */
-#define I2C_EDID_DEVICE_ADDR			0x50
-#define I2C_E_EDID_DEVICE_ADDR			0x30
-
-#define EDID_BLOCK_LENGTH			0x80
-#define EDID_HEADER_PATTERN			0x00
-#define EDID_EXTENSION_FLAG			0x7e
-#define EDID_CHECKSUM				0x7f
-
 /* DP_MAX_LANE_COUNT */
 #define DPCD_ENHANCED_FRAME_CAP(x)		(((x) >> 7) & 0x1)
 #define DPCD_MAX_LANE_COUNT(x)			((x) & 0x1f)
@@ -166,6 +157,7 @@ struct analogix_dp_device {
 	struct drm_device	*drm_dev;
 	struct drm_connector	connector;
 	struct drm_bridge	*bridge;
+	struct drm_dp_aux       aux;
 	struct clk		*clock;
 	unsigned int		irq;
 	void __iomem		*reg_base;
@@ -176,7 +168,10 @@ struct analogix_dp_device {
 	int			dpms_mode;
 	int			hpd_gpio;
 	bool                    force_hpd;
-	unsigned char           edid[EDID_BLOCK_LENGTH * 2];
+	bool			psr_support;
+
+	struct mutex		panel_lock;
+	bool			panel_is_modeset;
 
 	struct analogix_dp_plat_data *plat_data;
 };
@@ -206,33 +201,6 @@ void analogix_dp_reset_aux(struct analogix_dp_device *dp);
 void analogix_dp_init_aux(struct analogix_dp_device *dp);
 int analogix_dp_get_plug_in_status(struct analogix_dp_device *dp);
 void analogix_dp_enable_sw_function(struct analogix_dp_device *dp);
-int analogix_dp_start_aux_transaction(struct analogix_dp_device *dp);
-int analogix_dp_write_byte_to_dpcd(struct analogix_dp_device *dp,
-				   unsigned int reg_addr,
-				   unsigned char data);
-int analogix_dp_read_byte_from_dpcd(struct analogix_dp_device *dp,
-				    unsigned int reg_addr,
-				    unsigned char *data);
-int analogix_dp_write_bytes_to_dpcd(struct analogix_dp_device *dp,
-				    unsigned int reg_addr,
-				    unsigned int count,
-				    unsigned char data[]);
-int analogix_dp_read_bytes_from_dpcd(struct analogix_dp_device *dp,
-				     unsigned int reg_addr,
-				     unsigned int count,
-				     unsigned char data[]);
-int analogix_dp_select_i2c_device(struct analogix_dp_device *dp,
-				  unsigned int device_addr,
-				  unsigned int reg_addr);
-int analogix_dp_read_byte_from_i2c(struct analogix_dp_device *dp,
-				   unsigned int device_addr,
-				   unsigned int reg_addr,
-				   unsigned int *data);
-int analogix_dp_read_bytes_from_i2c(struct analogix_dp_device *dp,
-				    unsigned int device_addr,
-				    unsigned int reg_addr,
-				    unsigned int count,
-				    unsigned char edid[]);
 void analogix_dp_set_link_bandwidth(struct analogix_dp_device *dp, u32 bwtype);
 void analogix_dp_get_link_bandwidth(struct analogix_dp_device *dp, u32 *bwtype);
 void analogix_dp_set_lane_count(struct analogix_dp_device *dp, u32 count);
@@ -278,4 +246,10 @@ int analogix_dp_is_video_stream_on(struct analogix_dp_device *dp);
 void analogix_dp_config_video_slave_mode(struct analogix_dp_device *dp);
 void analogix_dp_enable_scrambling(struct analogix_dp_device *dp);
 void analogix_dp_disable_scrambling(struct analogix_dp_device *dp);
+void analogix_dp_enable_psr_crc(struct analogix_dp_device *dp);
+void analogix_dp_send_psr_spd(struct analogix_dp_device *dp,
+			      struct edp_vsc_psr *vsc);
+ssize_t analogix_dp_transfer(struct analogix_dp_device *dp,
+			     struct drm_dp_aux_msg *msg);
+
 #endif /* _ANALOGIX_DP_CORE_H */
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
index 48030f0cf497..cd37ac058675 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
@@ -585,330 +585,6 @@ int analogix_dp_write_byte_to_dpcd(struct analogix_dp_device *dp,
 	return retval;
 }
 
-int analogix_dp_read_byte_from_dpcd(struct analogix_dp_device *dp,
-				    unsigned int reg_addr,
-				    unsigned char *data)
-{
-	u32 reg;
-	int i;
-	int retval;
-
-	for (i = 0; i < 3; i++) {
-		/* Clear AUX CH data buffer */
-		reg = BUF_CLR;
-		writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL);
-
-		/* Select DPCD device address */
-		reg = AUX_ADDR_7_0(reg_addr);
-		writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0);
-		reg = AUX_ADDR_15_8(reg_addr);
-		writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8);
-		reg = AUX_ADDR_19_16(reg_addr);
-		writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16);
-
-		/*
-		 * Set DisplayPort transaction and read 1 byte
-		 * If bit 3 is 1, DisplayPort transaction.
-		 * If Bit 3 is 0, I2C transaction.
-		 */
-		reg = AUX_TX_COMM_DP_TRANSACTION | AUX_TX_COMM_READ;
-		writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1);
-
-		/* Start AUX transaction */
-		retval = analogix_dp_start_aux_transaction(dp);
-		if (retval == 0)
-			break;
-
-		dev_dbg(dp->dev, "%s: Aux Transaction fail!\n", __func__);
-	}
-
-	/* Read data buffer */
-	reg = readl(dp->reg_base + ANALOGIX_DP_BUF_DATA_0);
-	*data = (unsigned char)(reg & 0xff);
-
-	return retval;
-}
-
-int analogix_dp_write_bytes_to_dpcd(struct analogix_dp_device *dp,
-				    unsigned int reg_addr,
-				    unsigned int count,
-				    unsigned char data[])
-{
-	u32 reg;
-	unsigned int start_offset;
-	unsigned int cur_data_count;
-	unsigned int cur_data_idx;
-	int i;
-	int retval = 0;
-
-	/* Clear AUX CH data buffer */
-	reg = BUF_CLR;
-	writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL);
-
-	start_offset = 0;
-	while (start_offset < count) {
-		/* Buffer size of AUX CH is 16 * 4bytes */
-		if ((count - start_offset) > 16)
-			cur_data_count = 16;
-		else
-			cur_data_count = count - start_offset;
-
-		for (i = 0; i < 3; i++) {
-			/* Select DPCD device address */
-			reg = AUX_ADDR_7_0(reg_addr + start_offset);
-			writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0);
-			reg = AUX_ADDR_15_8(reg_addr + start_offset);
-			writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8);
-			reg = AUX_ADDR_19_16(reg_addr + start_offset);
-			writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16);
-
-			for (cur_data_idx = 0; cur_data_idx < cur_data_count;
-			     cur_data_idx++) {
-				reg = data[start_offset + cur_data_idx];
-				writel(reg, dp->reg_base +
-				       ANALOGIX_DP_BUF_DATA_0 +
-				       4 * cur_data_idx);
-			}
-
-			/*
-			 * Set DisplayPort transaction and write
-			 * If bit 3 is 1, DisplayPort transaction.
-			 * If Bit 3 is 0, I2C transaction.
-			 */
-			reg = AUX_LENGTH(cur_data_count) |
-				AUX_TX_COMM_DP_TRANSACTION | AUX_TX_COMM_WRITE;
-			writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1);
-
-			/* Start AUX transaction */
-			retval = analogix_dp_start_aux_transaction(dp);
-			if (retval == 0)
-				break;
-
-			dev_dbg(dp->dev, "%s: Aux Transaction fail!\n",
-				__func__);
-		}
-
-		start_offset += cur_data_count;
-	}
-
-	return retval;
-}
-
-int analogix_dp_read_bytes_from_dpcd(struct analogix_dp_device *dp,
-				     unsigned int reg_addr,
-				     unsigned int count,
-				     unsigned char data[])
-{
-	u32 reg;
-	unsigned int start_offset;
-	unsigned int cur_data_count;
-	unsigned int cur_data_idx;
-	int i;
-	int retval = 0;
-
-	/* Clear AUX CH data buffer */
-	reg = BUF_CLR;
-	writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL);
-
-	start_offset = 0;
-	while (start_offset < count) {
-		/* Buffer size of AUX CH is 16 * 4bytes */
-		if ((count - start_offset) > 16)
-			cur_data_count = 16;
-		else
-			cur_data_count = count - start_offset;
-
-		/* AUX CH Request Transaction process */
-		for (i = 0; i < 3; i++) {
-			/* Select DPCD device address */
-			reg = AUX_ADDR_7_0(reg_addr + start_offset);
-			writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0);
-			reg = AUX_ADDR_15_8(reg_addr + start_offset);
-			writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8);
-			reg = AUX_ADDR_19_16(reg_addr + start_offset);
-			writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16);
-
-			/*
-			 * Set DisplayPort transaction and read
-			 * If bit 3 is 1, DisplayPort transaction.
-			 * If Bit 3 is 0, I2C transaction.
-			 */
-			reg = AUX_LENGTH(cur_data_count) |
-				AUX_TX_COMM_DP_TRANSACTION | AUX_TX_COMM_READ;
-			writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1);
-
-			/* Start AUX transaction */
-			retval = analogix_dp_start_aux_transaction(dp);
-			if (retval == 0)
-				break;
-
-			dev_dbg(dp->dev, "%s: Aux Transaction fail!\n",
-				__func__);
-		}
-
-		for (cur_data_idx = 0; cur_data_idx < cur_data_count;
-		    cur_data_idx++) {
-			reg = readl(dp->reg_base + ANALOGIX_DP_BUF_DATA_0
-						 + 4 * cur_data_idx);
-			data[start_offset + cur_data_idx] =
-				(unsigned char)reg;
-		}
-
-		start_offset += cur_data_count;
-	}
-
-	return retval;
-}
-
-int analogix_dp_select_i2c_device(struct analogix_dp_device *dp,
-				  unsigned int device_addr,
-				  unsigned int reg_addr)
-{
-	u32 reg;
-	int retval;
-
-	/* Set EDID device address */
-	reg = device_addr;
-	writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0);
-	writel(0x0, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8);
-	writel(0x0, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16);
-
-	/* Set offset from base address of EDID device */
-	writel(reg_addr, dp->reg_base + ANALOGIX_DP_BUF_DATA_0);
-
-	/*
-	 * Set I2C transaction and write address
-	 * If bit 3 is 1, DisplayPort transaction.
-	 * If Bit 3 is 0, I2C transaction.
-	 */
-	reg = AUX_TX_COMM_I2C_TRANSACTION | AUX_TX_COMM_MOT |
-		AUX_TX_COMM_WRITE;
-	writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1);
-
-	/* Start AUX transaction */
-	retval = analogix_dp_start_aux_transaction(dp);
-	if (retval != 0)
-		dev_dbg(dp->dev, "%s: Aux Transaction fail!\n", __func__);
-
-	return retval;
-}
-
-int analogix_dp_read_byte_from_i2c(struct analogix_dp_device *dp,
-				   unsigned int device_addr,
-				   unsigned int reg_addr,
-				   unsigned int *data)
-{
-	u32 reg;
-	int i;
-	int retval;
-
-	for (i = 0; i < 3; i++) {
-		/* Clear AUX CH data buffer */
-		reg = BUF_CLR;
-		writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL);
-
-		/* Select EDID device */
-		retval = analogix_dp_select_i2c_device(dp, device_addr,
-						       reg_addr);
-		if (retval != 0)
-			continue;
-
-		/*
-		 * Set I2C transaction and read data
-		 * If bit 3 is 1, DisplayPort transaction.
-		 * If Bit 3 is 0, I2C transaction.
-		 */
-		reg = AUX_TX_COMM_I2C_TRANSACTION |
-			AUX_TX_COMM_READ;
-		writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1);
-
-		/* Start AUX transaction */
-		retval = analogix_dp_start_aux_transaction(dp);
-		if (retval == 0)
-			break;
-
-		dev_dbg(dp->dev, "%s: Aux Transaction fail!\n", __func__);
-	}
-
-	/* Read data */
-	if (retval == 0)
-		*data = readl(dp->reg_base + ANALOGIX_DP_BUF_DATA_0);
-
-	return retval;
-}
-
-int analogix_dp_read_bytes_from_i2c(struct analogix_dp_device *dp,
-				    unsigned int device_addr,
-				    unsigned int reg_addr,
-				    unsigned int count,
-				    unsigned char edid[])
-{
-	u32 reg;
-	unsigned int i, j;
-	unsigned int cur_data_idx;
-	unsigned int defer = 0;
-	int retval = 0;
-
-	for (i = 0; i < count; i += 16) {
-		for (j = 0; j < 3; j++) {
-			/* Clear AUX CH data buffer */
-			reg = BUF_CLR;
-			writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL);
-
-			/* Set normal AUX CH command */
-			reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2);
-			reg &= ~ADDR_ONLY;
-			writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2);
-
-			/*
-			 * If Rx sends defer, Tx sends only reads
-			 * request without sending address
-			 */
-			if (!defer)
-				retval = analogix_dp_select_i2c_device(dp,
-						device_addr, reg_addr + i);
-			else
-				defer = 0;
-
-			if (retval == 0) {
-				/*
-				 * Set I2C transaction and write data
-				 * If bit 3 is 1, DisplayPort transaction.
-				 * If Bit 3 is 0, I2C transaction.
-				 */
-				reg = AUX_LENGTH(16) |
-					AUX_TX_COMM_I2C_TRANSACTION |
-					AUX_TX_COMM_READ;
-				writel(reg, dp->reg_base +
-					ANALOGIX_DP_AUX_CH_CTL_1);
-
-				/* Start AUX transaction */
-				retval = analogix_dp_start_aux_transaction(dp);
-				if (retval == 0)
-					break;
-
-				dev_dbg(dp->dev, "%s: Aux Transaction fail!\n",
-					__func__);
-			}
-			/* Check if Rx sends defer */
-			reg = readl(dp->reg_base + ANALOGIX_DP_AUX_RX_COMM);
-			if (reg == AUX_RX_COMM_AUX_DEFER ||
-			    reg == AUX_RX_COMM_I2C_DEFER) {
-				dev_err(dp->dev, "Defer: %d\n\n", reg);
-				defer = 1;
-			}
-		}
-
-		for (cur_data_idx = 0; cur_data_idx < 16; cur_data_idx++) {
-			reg = readl(dp->reg_base + ANALOGIX_DP_BUF_DATA_0
-						 + 4 * cur_data_idx);
-			edid[i + cur_data_idx] = (unsigned char)reg;
-		}
-	}
-
-	return retval;
-}
-
 void analogix_dp_set_link_bandwidth(struct analogix_dp_device *dp, u32 bwtype)
 {
 	u32 reg;
@@ -1073,34 +749,22 @@ void analogix_dp_set_lane3_link_training(struct analogix_dp_device *dp,
 
 u32 analogix_dp_get_lane0_link_training(struct analogix_dp_device *dp)
 {
-	u32 reg;
-
-	reg = readl(dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL);
-	return reg;
+	return readl(dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL);
 }
 
 u32 analogix_dp_get_lane1_link_training(struct analogix_dp_device *dp)
 {
-	u32 reg;
-
-	reg = readl(dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL);
-	return reg;
+	return readl(dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL);
 }
 
 u32 analogix_dp_get_lane2_link_training(struct analogix_dp_device *dp)
 {
-	u32 reg;
-
-	reg = readl(dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL);
-	return reg;
+	return readl(dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL);
 }
 
 u32 analogix_dp_get_lane3_link_training(struct analogix_dp_device *dp)
 {
-	u32 reg;
-
-	reg = readl(dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL);
-	return reg;
+	return readl(dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL);
 }
 
 void analogix_dp_reset_macro(struct analogix_dp_device *dp)
@@ -1322,3 +986,181 @@ void analogix_dp_disable_scrambling(struct analogix_dp_device *dp)
 	reg |= SCRAMBLING_DISABLE;
 	writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET);
 }
+
+void analogix_dp_enable_psr_crc(struct analogix_dp_device *dp)
+{
+	writel(PSR_VID_CRC_ENABLE, dp->reg_base + ANALOGIX_DP_CRC_CON);
+}
+
+void analogix_dp_send_psr_spd(struct analogix_dp_device *dp,
+			      struct edp_vsc_psr *vsc)
+{
+	unsigned int val;
+
+	/* don't send info frame */
+	val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL);
+	val &= ~IF_EN;
+	writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL);
+
+	/* configure single frame update mode */
+	writel(PSR_FRAME_UP_TYPE_BURST | PSR_CRC_SEL_HARDWARE,
+	       dp->reg_base + ANALOGIX_DP_PSR_FRAME_UPDATE_CTRL);
+
+	/* configure VSC HB0~HB3 */
+	writel(vsc->sdp_header.HB0, dp->reg_base + ANALOGIX_DP_SPD_HB0);
+	writel(vsc->sdp_header.HB1, dp->reg_base + ANALOGIX_DP_SPD_HB1);
+	writel(vsc->sdp_header.HB2, dp->reg_base + ANALOGIX_DP_SPD_HB2);
+	writel(vsc->sdp_header.HB3, dp->reg_base + ANALOGIX_DP_SPD_HB3);
+
+	/* configure reused VSC PB0~PB3, magic number from vendor */
+	writel(0x00, dp->reg_base + ANALOGIX_DP_SPD_PB0);
+	writel(0x16, dp->reg_base + ANALOGIX_DP_SPD_PB1);
+	writel(0xCE, dp->reg_base + ANALOGIX_DP_SPD_PB2);
+	writel(0x5D, dp->reg_base + ANALOGIX_DP_SPD_PB3);
+
+	/* configure DB0 / DB1 values */
+	writel(vsc->DB0, dp->reg_base + ANALOGIX_DP_VSC_SHADOW_DB0);
+	writel(vsc->DB1, dp->reg_base + ANALOGIX_DP_VSC_SHADOW_DB1);
+
+	/* set reuse spd inforframe */
+	val = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3);
+	val |= REUSE_SPD_EN;
+	writel(val, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3);
+
+	/* mark info frame update */
+	val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL);
+	val = (val | IF_UP) & ~IF_EN;
+	writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL);
+
+	/* send info frame */
+	val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL);
+	val |= IF_EN;
+	writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL);
+}
+
+ssize_t analogix_dp_transfer(struct analogix_dp_device *dp,
+			     struct drm_dp_aux_msg *msg)
+{
+	u32 reg;
+	u8 *buffer = msg->buffer;
+	int timeout_loop = 0;
+	unsigned int i;
+	int num_transferred = 0;
+
+	/* Buffer size of AUX CH is 16 bytes */
+	if (WARN_ON(msg->size > 16))
+		return -E2BIG;
+
+	/* Clear AUX CH data buffer */
+	reg = BUF_CLR;
+	writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL);
+
+	switch (msg->request & ~DP_AUX_I2C_MOT) {
+	case DP_AUX_I2C_WRITE:
+		reg = AUX_TX_COMM_WRITE | AUX_TX_COMM_I2C_TRANSACTION;
+		if (msg->request & DP_AUX_I2C_MOT)
+			reg |= AUX_TX_COMM_MOT;
+		break;
+
+	case DP_AUX_I2C_READ:
+		reg = AUX_TX_COMM_READ | AUX_TX_COMM_I2C_TRANSACTION;
+		if (msg->request & DP_AUX_I2C_MOT)
+			reg |= AUX_TX_COMM_MOT;
+		break;
+
+	case DP_AUX_NATIVE_WRITE:
+		reg = AUX_TX_COMM_WRITE | AUX_TX_COMM_DP_TRANSACTION;
+		break;
+
+	case DP_AUX_NATIVE_READ:
+		reg = AUX_TX_COMM_READ | AUX_TX_COMM_DP_TRANSACTION;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	reg |= AUX_LENGTH(msg->size);
+	writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1);
+
+	/* Select DPCD device address */
+	reg = AUX_ADDR_7_0(msg->address);
+	writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0);
+	reg = AUX_ADDR_15_8(msg->address);
+	writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8);
+	reg = AUX_ADDR_19_16(msg->address);
+	writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16);
+
+	if (!(msg->request & DP_AUX_I2C_READ)) {
+		for (i = 0; i < msg->size; i++) {
+			reg = buffer[i];
+			writel(reg, dp->reg_base + ANALOGIX_DP_BUF_DATA_0 +
+			       4 * i);
+			num_transferred++;
+		}
+	}
+
+	/* Enable AUX CH operation */
+	reg = AUX_EN;
+
+	/* Zero-sized messages specify address-only transactions. */
+	if (msg->size < 1)
+		reg |= ADDR_ONLY;
+
+	writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2);
+
+	/* Is AUX CH command reply received? */
+	/* TODO: Wait for an interrupt instead of looping? */
+	reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA);
+	while (!(reg & RPLY_RECEIV)) {
+		timeout_loop++;
+		if (timeout_loop > DP_TIMEOUT_LOOP_COUNT) {
+			dev_err(dp->dev, "AUX CH command reply failed!\n");
+			return -ETIMEDOUT;
+		}
+		reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA);
+		usleep_range(10, 11);
+	}
+
+	/* Clear interrupt source for AUX CH command reply */
+	writel(RPLY_RECEIV, dp->reg_base + ANALOGIX_DP_INT_STA);
+
+	/* Clear interrupt source for AUX CH access error */
+	reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA);
+	if (reg & AUX_ERR) {
+		writel(AUX_ERR, dp->reg_base + ANALOGIX_DP_INT_STA);
+		return -EREMOTEIO;
+	}
+
+	/* Check AUX CH error access status */
+	reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_STA);
+	if ((reg & AUX_STATUS_MASK)) {
+		dev_err(dp->dev, "AUX CH error happened: %d\n\n",
+			reg & AUX_STATUS_MASK);
+		return -EREMOTEIO;
+	}
+
+	if (msg->request & DP_AUX_I2C_READ) {
+		for (i = 0; i < msg->size; i++) {
+			reg = readl(dp->reg_base + ANALOGIX_DP_BUF_DATA_0 +
+				    4 * i);
+			buffer[i] = (unsigned char)reg;
+			num_transferred++;
+		}
+	}
+
+	/* Check if Rx sends defer */
+	reg = readl(dp->reg_base + ANALOGIX_DP_AUX_RX_COMM);
+	if (reg == AUX_RX_COMM_AUX_DEFER)
+		msg->reply = DP_AUX_NATIVE_REPLY_DEFER;
+	else if (reg == AUX_RX_COMM_I2C_DEFER)
+		msg->reply = DP_AUX_I2C_REPLY_DEFER;
+	else if ((msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_I2C_WRITE ||
+		 (msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_I2C_READ)
+		msg->reply = DP_AUX_I2C_REPLY_ACK;
+	else if ((msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_NATIVE_WRITE ||
+		 (msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_NATIVE_READ)
+		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+
+	return num_transferred;
+}
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
index cdcc6c5add5e..40200c652533 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
@@ -22,6 +22,8 @@
 #define ANALOGIX_DP_VIDEO_CTL_8			0x3C
 #define ANALOGIX_DP_VIDEO_CTL_10		0x44
 
+#define ANALOGIX_DP_SPDIF_AUDIO_CTL_0		0xD8
+
 #define ANALOGIX_DP_PLL_REG_1			0xfc
 #define ANALOGIX_DP_PLL_REG_2			0x9e4
 #define ANALOGIX_DP_PLL_REG_3			0x9e8
@@ -30,6 +32,21 @@
 
 #define ANALOGIX_DP_PD				0x12c
 
+#define ANALOGIX_DP_IF_TYPE			0x244
+#define ANALOGIX_DP_IF_PKT_DB1			0x254
+#define ANALOGIX_DP_IF_PKT_DB2			0x258
+#define ANALOGIX_DP_SPD_HB0			0x2F8
+#define ANALOGIX_DP_SPD_HB1			0x2FC
+#define ANALOGIX_DP_SPD_HB2			0x300
+#define ANALOGIX_DP_SPD_HB3			0x304
+#define ANALOGIX_DP_SPD_PB0			0x308
+#define ANALOGIX_DP_SPD_PB1			0x30C
+#define ANALOGIX_DP_SPD_PB2			0x310
+#define ANALOGIX_DP_SPD_PB3			0x314
+#define ANALOGIX_DP_PSR_FRAME_UPDATE_CTRL	0x318
+#define ANALOGIX_DP_VSC_SHADOW_DB0		0x31C
+#define ANALOGIX_DP_VSC_SHADOW_DB1		0x320
+
 #define ANALOGIX_DP_LANE_MAP			0x35C
 
 #define ANALOGIX_DP_ANALOG_CTL_1		0x370
@@ -103,6 +120,8 @@
 
 #define ANALOGIX_DP_SOC_GENERAL_CTL		0x800
 
+#define ANALOGIX_DP_CRC_CON			0x890
+
 /* ANALOGIX_DP_TX_SW_RESET */
 #define RESET_DP_TX				(0x1 << 0)
 
@@ -151,6 +170,7 @@
 #define VID_CHK_UPDATE_TYPE_SHIFT		(4)
 #define VID_CHK_UPDATE_TYPE_1			(0x1 << 4)
 #define VID_CHK_UPDATE_TYPE_0			(0x0 << 4)
+#define REUSE_SPD_EN				(0x1 << 3)
 
 /* ANALOGIX_DP_VIDEO_CTL_8 */
 #define VID_HRES_TH(x)				(((x) & 0xf) << 4)
@@ -167,6 +187,12 @@
 #define REF_CLK_27M				(0x0 << 0)
 #define REF_CLK_MASK				(0x1 << 0)
 
+/* ANALOGIX_DP_PSR_FRAME_UPDATE_CTRL */
+#define PSR_FRAME_UP_TYPE_BURST			(0x1 << 0)
+#define PSR_FRAME_UP_TYPE_SINGLE		(0x0 << 0)
+#define PSR_CRC_SEL_HARDWARE			(0x1 << 1)
+#define PSR_CRC_SEL_MANUALLY			(0x0 << 1)
+
 /* ANALOGIX_DP_LANE_MAP */
 #define LANE3_MAP_LOGIC_LANE_0			(0x0 << 6)
 #define LANE3_MAP_LOGIC_LANE_1			(0x1 << 6)
@@ -376,4 +402,12 @@
 #define VIDEO_MODE_SLAVE_MODE			(0x1 << 0)
 #define VIDEO_MODE_MASTER_MODE			(0x0 << 0)
 
+/* ANALOGIX_DP_PKT_SEND_CTL */
+#define IF_UP					(0x1 << 4)
+#define IF_EN					(0x1 << 0)
+
+/* ANALOGIX_DP_CRC_CON */
+#define PSR_VID_CRC_FLUSH			(0x1 << 2)
+#define PSR_VID_CRC_ENABLE			(0x1 << 0)
+
 #endif /* _ANALOGIX_DP_REG_H */
diff --git a/drivers/gpu/drm/bridge/dumb-vga-dac.c b/drivers/gpu/drm/bridge/dumb-vga-dac.c
new file mode 100644
index 000000000000..afec232185a7
--- /dev/null
+++ b/drivers/gpu/drm/bridge/dumb-vga-dac.c
@@ -0,0 +1,223 @@
+/*
+ * Copyright (C) 2015-2016 Free Electrons
+ * Copyright (C) 2015-2016 NextThing Co
+ *
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/of_graph.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+
+struct dumb_vga {
+	struct drm_bridge	bridge;
+	struct drm_connector	connector;
+
+	struct i2c_adapter	*ddc;
+};
+
+static inline struct dumb_vga *
+drm_bridge_to_dumb_vga(struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct dumb_vga, bridge);
+}
+
+static inline struct dumb_vga *
+drm_connector_to_dumb_vga(struct drm_connector *connector)
+{
+	return container_of(connector, struct dumb_vga, connector);
+}
+
+static int dumb_vga_get_modes(struct drm_connector *connector)
+{
+	struct dumb_vga *vga = drm_connector_to_dumb_vga(connector);
+	struct edid *edid;
+	int ret;
+
+	if (IS_ERR(vga->ddc))
+		goto fallback;
+
+	edid = drm_get_edid(connector, vga->ddc);
+	if (!edid) {
+		DRM_INFO("EDID readout failed, falling back to standard modes\n");
+		goto fallback;
+	}
+
+	drm_mode_connector_update_edid_property(connector, edid);
+	return drm_add_edid_modes(connector, edid);
+
+fallback:
+	/*
+	 * In case we cannot retrieve the EDIDs (broken or missing i2c
+	 * bus), fallback on the XGA standards
+	 */
+	ret = drm_add_modes_noedid(connector, 1920, 1200);
+
+	/* And prefer a mode pretty much anyone can handle */
+	drm_set_preferred_mode(connector, 1024, 768);
+
+	return ret;
+}
+
+static const struct drm_connector_helper_funcs dumb_vga_con_helper_funcs = {
+	.get_modes	= dumb_vga_get_modes,
+};
+
+static enum drm_connector_status
+dumb_vga_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct dumb_vga *vga = drm_connector_to_dumb_vga(connector);
+
+	/*
+	 * Even if we have an I2C bus, we can't assume that the cable
+	 * is disconnected if drm_probe_ddc fails. Some cables don't
+	 * wire the DDC pins, or the I2C bus might not be working at
+	 * all.
+	 */
+	if (!IS_ERR(vga->ddc) && drm_probe_ddc(vga->ddc))
+		return connector_status_connected;
+
+	return connector_status_unknown;
+}
+
+static const struct drm_connector_funcs dumb_vga_con_funcs = {
+	.dpms			= drm_atomic_helper_connector_dpms,
+	.detect			= dumb_vga_connector_detect,
+	.fill_modes		= drm_helper_probe_single_connector_modes,
+	.destroy		= drm_connector_cleanup,
+	.reset			= drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
+};
+
+static int dumb_vga_attach(struct drm_bridge *bridge)
+{
+	struct dumb_vga *vga = drm_bridge_to_dumb_vga(bridge);
+	int ret;
+
+	if (!bridge->encoder) {
+		DRM_ERROR("Missing encoder\n");
+		return -ENODEV;
+	}
+
+	drm_connector_helper_add(&vga->connector,
+				 &dumb_vga_con_helper_funcs);
+	ret = drm_connector_init(bridge->dev, &vga->connector,
+				 &dumb_vga_con_funcs, DRM_MODE_CONNECTOR_VGA);
+	if (ret) {
+		DRM_ERROR("Failed to initialize connector\n");
+		return ret;
+	}
+
+	drm_mode_connector_attach_encoder(&vga->connector,
+					  bridge->encoder);
+
+	return 0;
+}
+
+static const struct drm_bridge_funcs dumb_vga_bridge_funcs = {
+	.attach		= dumb_vga_attach,
+};
+
+static struct i2c_adapter *dumb_vga_retrieve_ddc(struct device *dev)
+{
+	struct device_node *end_node, *phandle, *remote;
+	struct i2c_adapter *ddc;
+
+	end_node = of_graph_get_endpoint_by_regs(dev->of_node, 1, -1);
+	if (!end_node) {
+		dev_err(dev, "Missing connector endpoint\n");
+		return ERR_PTR(-ENODEV);
+	}
+
+	remote = of_graph_get_remote_port_parent(end_node);
+	of_node_put(end_node);
+	if (!remote) {
+		dev_err(dev, "Enable to parse remote node\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	phandle = of_parse_phandle(remote, "ddc-i2c-bus", 0);
+	of_node_put(remote);
+	if (!phandle)
+		return ERR_PTR(-ENODEV);
+
+	ddc = of_get_i2c_adapter_by_node(phandle);
+	of_node_put(phandle);
+	if (!ddc)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	return ddc;
+}
+
+static int dumb_vga_probe(struct platform_device *pdev)
+{
+	struct dumb_vga *vga;
+	int ret;
+
+	vga = devm_kzalloc(&pdev->dev, sizeof(*vga), GFP_KERNEL);
+	if (!vga)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, vga);
+
+	vga->ddc = dumb_vga_retrieve_ddc(&pdev->dev);
+	if (IS_ERR(vga->ddc)) {
+		if (PTR_ERR(vga->ddc) == -ENODEV) {
+			dev_dbg(&pdev->dev,
+				"No i2c bus specified. Disabling EDID readout\n");
+		} else {
+			dev_err(&pdev->dev, "Couldn't retrieve i2c bus\n");
+			return PTR_ERR(vga->ddc);
+		}
+	}
+
+	vga->bridge.funcs = &dumb_vga_bridge_funcs;
+	vga->bridge.of_node = pdev->dev.of_node;
+
+	ret = drm_bridge_add(&vga->bridge);
+	if (ret && !IS_ERR(vga->ddc))
+		i2c_put_adapter(vga->ddc);
+
+	return ret;
+}
+
+static int dumb_vga_remove(struct platform_device *pdev)
+{
+	struct dumb_vga *vga = platform_get_drvdata(pdev);
+
+	drm_bridge_remove(&vga->bridge);
+
+	if (!IS_ERR(vga->ddc))
+		i2c_put_adapter(vga->ddc);
+
+	return 0;
+}
+
+static const struct of_device_id dumb_vga_match[] = {
+	{ .compatible = "dumb-vga-dac" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, dumb_vga_match);
+
+static struct platform_driver dumb_vga_driver = {
+	.probe	= dumb_vga_probe,
+	.remove	= dumb_vga_remove,
+	.driver		= {
+		.name		= "dumb-vga-dac",
+		.of_match_table	= dumb_vga_match,
+	},
+};
+module_platform_driver(dumb_vga_driver);
+
+MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
+MODULE_DESCRIPTION("Dumb VGA DAC bridge driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/bridge/dw-hdmi-ahb-audio.c b/drivers/gpu/drm/bridge/dw-hdmi-ahb-audio.c
index 122bb015f4a9..8f2d1379c880 100644
--- a/drivers/gpu/drm/bridge/dw-hdmi-ahb-audio.c
+++ b/drivers/gpu/drm/bridge/dw-hdmi-ahb-audio.c
@@ -640,7 +640,6 @@ static struct platform_driver snd_dw_hdmi_driver = {
 	.remove	= snd_dw_hdmi_remove,
 	.driver	= {
 		.name = DRIVER_NAME,
-		.owner = THIS_MODULE,
 		.pm = PM_OPS,
 	},
 };
diff --git a/drivers/gpu/drm/bridge/dw-hdmi.c b/drivers/gpu/drm/bridge/dw-hdmi.c
index 77ab47341658..ab7023e5dfde 100644
--- a/drivers/gpu/drm/bridge/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/dw-hdmi.c
@@ -940,10 +940,11 @@ static void hdmi_config_AVI(struct dw_hdmi *hdmi, struct drm_display_mode *mode)
 	 */
 
 	/*
-	 * AVI data byte 1 differences: Colorspace in bits 4,5 rather than 5,6,
-	 * active aspect present in bit 6 rather than 4.
+	 * AVI data byte 1 differences: Colorspace in bits 0,1 rather than 5,6,
+	 * scan info in bits 4,5 rather than 0,1 and active aspect present in
+	 * bit 6 rather than 4.
 	 */
-	val = (frame.colorspace & 3) << 4 | (frame.scan_mode & 0x3);
+	val = (frame.scan_mode & 3) << 4 | (frame.colorspace & 3);
 	if (frame.active_aspect & 15)
 		val |= HDMI_FC_AVICONF0_ACTIVE_FMT_INFO_PRESENT;
 	if (frame.top_bar || frame.bottom_bar)
@@ -1476,12 +1477,6 @@ dw_hdmi_connector_mode_valid(struct drm_connector *connector,
 	return mode_status;
 }
 
-static void dw_hdmi_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
-}
-
 static void dw_hdmi_connector_force(struct drm_connector *connector)
 {
 	struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
@@ -1498,7 +1493,7 @@ static const struct drm_connector_funcs dw_hdmi_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = dw_hdmi_connector_detect,
-	.destroy = dw_hdmi_connector_destroy,
+	.destroy = drm_connector_cleanup,
 	.force = dw_hdmi_connector_force,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -1812,9 +1807,6 @@ void dw_hdmi_unbind(struct device *dev, struct device *master, void *data)
 	/* Disable all interrupts */
 	hdmi_writeb(hdmi, ~0, HDMI_IH_MUTE_PHY_STAT0);
 
-	hdmi->connector.funcs->destroy(&hdmi->connector);
-	hdmi->encoder->funcs->destroy(hdmi->encoder);
-
 	clk_disable_unprepare(hdmi->iahb_clk);
 	clk_disable_unprepare(hdmi->isfr_clk);
 	i2c_put_adapter(hdmi->ddc);
diff --git a/drivers/gpu/drm/bridge/nxp-ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c
index 93f3dacf9e27..f1a99938e924 100644
--- a/drivers/gpu/drm/bridge/nxp-ptn3460.c
+++ b/drivers/gpu/drm/bridge/nxp-ptn3460.c
@@ -245,16 +245,11 @@ static enum drm_connector_status ptn3460_detect(struct drm_connector *connector,
 	return connector_status_connected;
 }
 
-static void ptn3460_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_cleanup(connector);
-}
-
 static const struct drm_connector_funcs ptn3460_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = ptn3460_detect,
-	.destroy = ptn3460_connector_destroy,
+	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
diff --git a/drivers/gpu/drm/bridge/parade-ps8622.c b/drivers/gpu/drm/bridge/parade-ps8622.c
index 583b8ce614e3..6f7c2f9860d2 100644
--- a/drivers/gpu/drm/bridge/parade-ps8622.c
+++ b/drivers/gpu/drm/bridge/parade-ps8622.c
@@ -16,7 +16,6 @@
 #include <linux/backlight.h>
 #include <linux/delay.h>
 #include <linux/err.h>
-#include <linux/fb.h>
 #include <linux/gpio.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
@@ -484,16 +483,11 @@ static enum drm_connector_status ps8622_detect(struct drm_connector *connector,
 	return connector_status_connected;
 }
 
-static void ps8622_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_cleanup(connector);
-}
-
 static const struct drm_connector_funcs ps8622_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = ps8622_detect,
-	.destroy = ps8622_connector_destroy,
+	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index a09825d8c94a..44d476ea6d2e 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -1165,17 +1165,11 @@ static const struct drm_connector_helper_funcs tc_connector_helper_funcs = {
 	.best_encoder = tc_connector_best_encoder,
 };
 
-static void tc_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
-}
-
 static const struct drm_connector_funcs tc_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = tc_connector_detect,
-	.destroy = tc_connector_destroy,
+	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.c b/drivers/gpu/drm/cirrus/cirrus_drv.c
index b05f7eae32ce..6c76d125995b 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.c
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.c
@@ -57,7 +57,7 @@ static int cirrus_kick_out_firmware_fb(struct pci_dev *pdev)
 #ifdef CONFIG_X86
 	primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
 #endif
-	remove_conflicting_framebuffers(ap, "cirrusdrmfb", primary);
+	drm_fb_helper_remove_conflicting_framebuffers(ap, "cirrusdrmfb", primary);
 	kfree(ap);
 
 	return 0;
diff --git a/drivers/gpu/drm/cirrus/cirrus_fbdev.c b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
index 3b5be7272357..daecf1ad76a4 100644
--- a/drivers/gpu/drm/cirrus/cirrus_fbdev.c
+++ b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
@@ -13,8 +13,6 @@
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_crtc_helper.h>
 
-#include <linux/fb.h>
-
 #include "cirrus_drv.h"
 
 static void cirrus_dirty_update(struct cirrus_fbdev *afbdev,
diff --git a/drivers/gpu/drm/cirrus/cirrus_main.c b/drivers/gpu/drm/cirrus/cirrus_main.c
index 80446e2d3ab6..76bcb43e7c06 100644
--- a/drivers/gpu/drm/cirrus/cirrus_main.c
+++ b/drivers/gpu/drm/cirrus/cirrus_main.c
@@ -185,14 +185,23 @@ int cirrus_driver_load(struct drm_device *dev, unsigned long flags)
 		goto out;
 	}
 
+	/*
+	 * cirrus_modeset_init() is initializing/registering the emulated fbdev
+	 * and DRM internals can access/test some of the fields in
+	 * mode_config->funcs as part of the fbdev registration process.
+	 * Make sure dev->mode_config.funcs is properly set to avoid
+	 * dereferencing a NULL pointer.
+	 * FIXME: mode_config.funcs assignment should probably be done in
+	 * cirrus_modeset_init() (that's a common pattern seen in other DRM
+	 * drivers).
+	 */
+	dev->mode_config.funcs = &cirrus_mode_funcs;
 	r = cirrus_modeset_init(cdev);
 	if (r) {
 		dev_err(&dev->pdev->dev, "Fatal error during modeset init: %d\n", r);
 		goto out;
 	}
 
-	dev->mode_config.funcs = (void *)&cirrus_mode_funcs;
-
 	return 0;
 out:
 	cirrus_driver_unload(dev);
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index 1cc9ee607128..bb2438dd8733 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -150,7 +150,8 @@ static int cirrus_bo_verify_access(struct ttm_buffer_object *bo, struct file *fi
 {
 	struct cirrus_bo *cirrusbo = cirrus_bo(bo);
 
-	return drm_vma_node_verify_access(&cirrusbo->gem.vma_node, filp);
+	return drm_vma_node_verify_access(&cirrusbo->gem.vma_node,
+					  filp->private_data);
 }
 
 static int cirrus_ttm_io_mem_reserve(struct ttm_bo_device *bdev,
diff --git a/drivers/gpu/drm/drm_agpsupport.c b/drivers/gpu/drm/drm_agpsupport.c
index 605bd243fb36..d621c8a4cf00 100644
--- a/drivers/gpu/drm/drm_agpsupport.c
+++ b/drivers/gpu/drm/drm_agpsupport.c
@@ -430,9 +430,7 @@ struct drm_agp_head *drm_agp_init(struct drm_device *dev)
  * intact so it can still be used. It is safe to call this if AGP is disabled or
  * was already removed.
  *
- * If DRIVER_MODESET is active, nothing is done to protect the modesetting
- * resources from getting destroyed. Drivers are responsible of cleaning them up
- * during device shutdown.
+ * Cleanup is only done for drivers who have DRIVER_LEGACY set.
  */
 void drm_legacy_agp_clear(struct drm_device *dev)
 {
@@ -440,7 +438,7 @@ void drm_legacy_agp_clear(struct drm_device *dev)
 
 	if (!dev->agp)
 		return;
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return;
 
 	list_for_each_entry_safe(entry, tempe, &dev->agp->memory, head) {
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index fa3930757972..23739609427d 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -475,7 +475,7 @@ int drm_atomic_crtc_set_property(struct drm_crtc *crtc,
 					val,
 					-1,
 					&replaced);
-		state->color_mgmt_changed = replaced;
+		state->color_mgmt_changed |= replaced;
 		return ret;
 	} else if (property == config->ctm_property) {
 		ret = drm_atomic_replace_property_blob_from_id(crtc,
@@ -483,7 +483,7 @@ int drm_atomic_crtc_set_property(struct drm_crtc *crtc,
 					val,
 					sizeof(struct drm_color_ctm),
 					&replaced);
-		state->color_mgmt_changed = replaced;
+		state->color_mgmt_changed |= replaced;
 		return ret;
 	} else if (property == config->gamma_lut_property) {
 		ret = drm_atomic_replace_property_blob_from_id(crtc,
@@ -491,7 +491,7 @@ int drm_atomic_crtc_set_property(struct drm_crtc *crtc,
 					val,
 					-1,
 					&replaced);
-		state->color_mgmt_changed = replaced;
+		state->color_mgmt_changed |= replaced;
 		return ret;
 	} else if (crtc->funcs->atomic_set_property)
 		return crtc->funcs->atomic_set_property(crtc, state, property, val);
@@ -837,8 +837,9 @@ static int drm_atomic_plane_check(struct drm_plane *plane,
 	/* Check whether this plane supports the fb pixel format. */
 	ret = drm_plane_check_pixel_format(plane, state->fb->pixel_format);
 	if (ret) {
-		DRM_DEBUG_ATOMIC("Invalid pixel format %s\n",
-				 drm_get_format_name(state->fb->pixel_format));
+		char *format_name = drm_get_format_name(state->fb->pixel_format);
+		DRM_DEBUG_ATOMIC("Invalid pixel format %s\n", format_name);
+		kfree(format_name);
 		return ret;
 	}
 
@@ -1690,7 +1691,7 @@ retry:
 				goto out;
 			}
 
-			prop = drm_property_find(dev, prop_id);
+			prop = drm_mode_obj_find_prop_id(obj, prop_id);
 			if (!prop) {
 				drm_mode_object_unreference(obj);
 				ret = -ENOENT;
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index 20be86d89a20..c3f83476f996 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -594,7 +594,7 @@ drm_atomic_helper_check_planes(struct drm_device *dev,
 	struct drm_plane_state *plane_state;
 	int i, ret = 0;
 
-	ret = drm_atomic_helper_normalize_zpos(dev, state);
+	ret = drm_atomic_normalize_zpos(dev, state);
 	if (ret)
 		return ret;
 
@@ -749,6 +749,8 @@ disable_outputs(struct drm_device *dev, struct drm_atomic_state *old_state)
 		/* Right function depends upon target state. */
 		if (crtc->state->enable && funcs->prepare)
 			funcs->prepare(crtc);
+		else if (funcs->atomic_disable)
+			funcs->atomic_disable(crtc, old_crtc_state);
 		else if (funcs->disable)
 			funcs->disable(crtc);
 		else
@@ -886,8 +888,12 @@ crtc_set_mode(struct drm_device *dev, struct drm_atomic_state *old_state)
 		 * Each encoder has at most one connector (since we always steal
 		 * it away), so we won't call mode_set hooks twice.
 		 */
-		if (funcs && funcs->mode_set)
+		if (funcs && funcs->atomic_mode_set) {
+			funcs->atomic_mode_set(encoder, new_crtc_state,
+					       connector->state);
+		} else if (funcs && funcs->mode_set) {
 			funcs->mode_set(encoder, mode, adjusted_mode);
+		}
 
 		drm_bridge_mode_set(encoder->bridge, mode, adjusted_mode);
 	}
@@ -1003,29 +1009,46 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_modeset_enables);
  * drm_atomic_helper_wait_for_fences - wait for fences stashed in plane state
  * @dev: DRM device
  * @state: atomic state object with old state structures
+ * @pre_swap: if true, do an interruptible wait
  *
  * For implicit sync, driver should fish the exclusive fence out from the
  * incoming fb's and stash it in the drm_plane_state.  This is called after
  * drm_atomic_helper_swap_state() so it uses the current plane state (and
  * just uses the atomic state to find the changed planes)
+ *
+ * Returns zero if success or < 0 if fence_wait() fails.
  */
-void drm_atomic_helper_wait_for_fences(struct drm_device *dev,
-			    struct drm_atomic_state *state)
+int drm_atomic_helper_wait_for_fences(struct drm_device *dev,
+				      struct drm_atomic_state *state,
+				      bool pre_swap)
 {
 	struct drm_plane *plane;
 	struct drm_plane_state *plane_state;
-	int i;
+	int i, ret;
 
 	for_each_plane_in_state(state, plane, plane_state, i) {
-		if (!plane->state->fence)
+		if (!pre_swap)
+			plane_state = plane->state;
+
+		if (!plane_state->fence)
 			continue;
 
-		WARN_ON(!plane->state->fb);
+		WARN_ON(!plane_state->fb);
+
+		/*
+		 * If waiting for fences pre-swap (ie: nonblock), userspace can
+		 * still interrupt the operation. Instead of blocking until the
+		 * timer expires, make the wait interruptible.
+		 */
+		ret = fence_wait(plane_state->fence, pre_swap);
+		if (ret)
+			return ret;
 
-		fence_wait(plane->state->fence, false);
-		fence_put(plane->state->fence);
-		plane->state->fence = NULL;
+		fence_put(plane_state->fence);
+		plane_state->fence = NULL;
 	}
+
+	return 0;
 }
 EXPORT_SYMBOL(drm_atomic_helper_wait_for_fences);
 
@@ -1142,7 +1165,8 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_vblanks);
  *
  *     drm_atomic_helper_commit_modeset_enables(dev, state);
  *
- *     drm_atomic_helper_commit_planes(dev, state, true);
+ *     drm_atomic_helper_commit_planes(dev, state,
+ *                                     DRM_PLANE_COMMIT_ACTIVE_ONLY);
  *
  * for committing the atomic update to hardware.  See the kerneldoc entries for
  * these three functions for more details.
@@ -1153,7 +1177,7 @@ void drm_atomic_helper_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_modeset_disables(dev, state);
 
-	drm_atomic_helper_commit_planes(dev, state, false);
+	drm_atomic_helper_commit_planes(dev, state, 0);
 
 	drm_atomic_helper_commit_modeset_enables(dev, state);
 
@@ -1172,7 +1196,7 @@ static void commit_tail(struct drm_atomic_state *state)
 
 	funcs = dev->mode_config.helper_private;
 
-	drm_atomic_helper_wait_for_fences(dev, state);
+	drm_atomic_helper_wait_for_fences(dev, state, false);
 
 	drm_atomic_helper_wait_for_dependencies(state);
 
@@ -1231,6 +1255,12 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 	if (ret)
 		return ret;
 
+	if (!nonblock) {
+		ret = drm_atomic_helper_wait_for_fences(dev, state, true);
+		if (ret)
+			return ret;
+	}
+
 	/*
 	 * This is the point of no return - everything below never fails except
 	 * when the hw goes bonghits. Which means we can commit the new state on
@@ -1631,6 +1661,9 @@ int drm_atomic_helper_prepare_planes(struct drm_device *dev,
 
 		funcs = plane->helper_private;
 
+		if (!drm_atomic_helper_framebuffer_changed(dev, state, plane_state->crtc))
+			continue;
+
 		if (funcs->prepare_fb) {
 			ret = funcs->prepare_fb(plane, plane_state);
 			if (ret)
@@ -1647,18 +1680,20 @@ fail:
 		if (j >= i)
 			continue;
 
+		if (!drm_atomic_helper_framebuffer_changed(dev, state, plane_state->crtc))
+			continue;
+
 		funcs = plane->helper_private;
 
 		if (funcs->cleanup_fb)
 			funcs->cleanup_fb(plane, plane_state);
-
 	}
 
 	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_prepare_planes);
 
-bool plane_crtc_active(struct drm_plane_state *state)
+static bool plane_crtc_active(const struct drm_plane_state *state)
 {
 	return state->crtc && state->crtc->state->active;
 }
@@ -1667,7 +1702,7 @@ bool plane_crtc_active(struct drm_plane_state *state)
  * drm_atomic_helper_commit_planes - commit plane state
  * @dev: DRM device
  * @old_state: atomic state object with old state structures
- * @active_only: Only commit on active CRTC if set
+ * @flags: flags for committing plane state
  *
  * This function commits the new plane state using the plane and atomic helper
  * functions for planes and crtcs. It assumes that the atomic state has already
@@ -1687,25 +1722,34 @@ bool plane_crtc_active(struct drm_plane_state *state)
  * most drivers don't need to be immediately notified of plane updates for a
  * disabled CRTC.
  *
- * Unless otherwise needed, drivers are advised to set the @active_only
- * parameters to true in order not to receive plane update notifications related
- * to a disabled CRTC. This avoids the need to manually ignore plane updates in
+ * Unless otherwise needed, drivers are advised to set the ACTIVE_ONLY flag in
+ * @flags in order not to receive plane update notifications related to a
+ * disabled CRTC. This avoids the need to manually ignore plane updates in
  * driver code when the driver and/or hardware can't or just don't need to deal
  * with updates on disabled CRTCs, for example when supporting runtime PM.
  *
- * The drm_atomic_helper_commit() default implementation only sets @active_only
- * to false to most closely match the behaviour of the legacy helpers. This should
- * not be copied blindly by drivers.
+ * Drivers may set the NO_DISABLE_AFTER_MODESET flag in @flags if the relevant
+ * display controllers require to disable a CRTC's planes when the CRTC is
+ * disabled. This function would skip the ->atomic_disable call for a plane if
+ * the CRTC of the old plane state needs a modesetting operation. Of course,
+ * the drivers need to disable the planes in their CRTC disable callbacks
+ * since no one else would do that.
+ *
+ * The drm_atomic_helper_commit() default implementation doesn't set the
+ * ACTIVE_ONLY flag to most closely match the behaviour of the legacy helpers.
+ * This should not be copied blindly by drivers.
  */
 void drm_atomic_helper_commit_planes(struct drm_device *dev,
 				     struct drm_atomic_state *old_state,
-				     bool active_only)
+				     uint32_t flags)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state;
 	struct drm_plane *plane;
 	struct drm_plane_state *old_plane_state;
 	int i;
+	bool active_only = flags & DRM_PLANE_COMMIT_ACTIVE_ONLY;
+	bool no_disable = flags & DRM_PLANE_COMMIT_NO_DISABLE_AFTER_MODESET;
 
 	for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) {
 		const struct drm_crtc_helper_funcs *funcs;
@@ -1749,10 +1793,19 @@ void drm_atomic_helper_commit_planes(struct drm_device *dev,
 		/*
 		 * Special-case disabling the plane if drivers support it.
 		 */
-		if (disabling && funcs->atomic_disable)
+		if (disabling && funcs->atomic_disable) {
+			struct drm_crtc_state *crtc_state;
+
+			crtc_state = old_plane_state->crtc->state;
+
+			if (drm_atomic_crtc_needs_modeset(crtc_state) &&
+			    no_disable)
+				continue;
+
 			funcs->atomic_disable(plane, old_plane_state);
-		else if (plane->state->crtc || disabling)
+		} else if (plane->state->crtc || disabling) {
 			funcs->atomic_update(plane, old_plane_state);
+		}
 	}
 
 	for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) {
@@ -1831,12 +1884,12 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_planes_on_crtc);
 
 /**
  * drm_atomic_helper_disable_planes_on_crtc - helper to disable CRTC's planes
- * @crtc: CRTC
+ * @old_crtc_state: atomic state object with the old CRTC state
  * @atomic: if set, synchronize with CRTC's atomic_begin/flush hooks
  *
  * Disables all planes associated with the given CRTC. This can be
- * used for instance in the CRTC helper disable callback to disable
- * all planes before shutting down the display pipeline.
+ * used for instance in the CRTC helper atomic_disable callback to disable
+ * all planes.
  *
  * If the atomic-parameter is set the function calls the CRTC's
  * atomic_begin hook before and atomic_flush hook after disabling the
@@ -1845,9 +1898,11 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_planes_on_crtc);
  * It is a bug to call this function without having implemented the
  * ->atomic_disable() plane hook.
  */
-void drm_atomic_helper_disable_planes_on_crtc(struct drm_crtc *crtc,
-					      bool atomic)
+void
+drm_atomic_helper_disable_planes_on_crtc(struct drm_crtc_state *old_crtc_state,
+					 bool atomic)
 {
+	struct drm_crtc *crtc = old_crtc_state->crtc;
 	const struct drm_crtc_helper_funcs *crtc_funcs =
 		crtc->helper_private;
 	struct drm_plane *plane;
@@ -1855,11 +1910,11 @@ void drm_atomic_helper_disable_planes_on_crtc(struct drm_crtc *crtc,
 	if (atomic && crtc_funcs && crtc_funcs->atomic_begin)
 		crtc_funcs->atomic_begin(crtc, NULL);
 
-	drm_for_each_plane(plane, crtc->dev) {
+	drm_atomic_crtc_state_for_each_plane(plane, old_crtc_state) {
 		const struct drm_plane_helper_funcs *plane_funcs =
 			plane->helper_private;
 
-		if (plane->state->crtc != crtc || !plane_funcs)
+		if (!plane_funcs)
 			continue;
 
 		WARN_ON(!plane_funcs->atomic_disable);
@@ -1894,6 +1949,9 @@ void drm_atomic_helper_cleanup_planes(struct drm_device *dev,
 	for_each_plane_in_state(old_state, plane, plane_state, i) {
 		const struct drm_plane_helper_funcs *funcs;
 
+		if (!drm_atomic_helper_framebuffer_changed(dev, old_state, plane_state->crtc))
+			continue;
+
 		funcs = plane->helper_private;
 
 		if (funcs->cleanup_fb)
@@ -2354,7 +2412,7 @@ int __drm_atomic_helper_set_config(struct drm_mode_set *set,
 	primary_state->crtc_h = vdisplay;
 	primary_state->src_x = set->x << 16;
 	primary_state->src_y = set->y << 16;
-	if (primary_state->rotation & (BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270))) {
+	if (primary_state->rotation & (DRM_ROTATE_90 | DRM_ROTATE_270)) {
 		primary_state->src_w = vdisplay << 16;
 		primary_state->src_h = hdisplay << 16;
 	} else {
@@ -3039,7 +3097,7 @@ void drm_atomic_helper_plane_reset(struct drm_plane *plane)
 
 	if (plane->state) {
 		plane->state->plane = plane;
-		plane->state->rotation = BIT(DRM_ROTATE_0);
+		plane->state->rotation = DRM_ROTATE_0;
 	}
 }
 EXPORT_SYMBOL(drm_atomic_helper_plane_reset);
diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c
index 4153e8a193af..6b143514a566 100644
--- a/drivers/gpu/drm/drm_auth.c
+++ b/drivers/gpu/drm/drm_auth.c
@@ -251,7 +251,7 @@ void drm_master_release(struct drm_file *file_priv)
 	if (!drm_is_current_master(file_priv))
 		goto out;
 
-	if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
+	if (drm_core_check_feature(dev, DRIVER_LEGACY)) {
 		/*
 		 * Since the master is disappearing, so is the
 		 * possibility to lock.
diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c
index f3c0942bd756..85172a977bf3 100644
--- a/drivers/gpu/drm/drm_blend.c
+++ b/drivers/gpu/drm/drm_blend.c
@@ -25,12 +25,173 @@
  */
 #include <drm/drmP.h>
 #include <drm/drm_atomic.h>
-#include <drm/drm_crtc.h>
+#include <drm/drm_blend.h>
 #include <linux/export.h>
 #include <linux/slab.h>
 #include <linux/sort.h>
 
-#include "drm_internal.h"
+#include "drm_crtc_internal.h"
+
+/**
+ * DOC: overview
+ *
+ * The basic plane composition model supported by standard plane properties only
+ * has a source rectangle (in logical pixels within the &drm_framebuffer), with
+ * sub-pixel accuracy, which is scaled up to a pixel-aligned destination
+ * rectangle in the visible area of a &drm_crtc. The visible area of a CRTC is
+ * defined by the horizontal and vertical visible pixels (stored in @hdisplay
+ * and @vdisplay) of the requested mode (stored in @mode in the
+ * &drm_crtc_state). These two rectangles are both stored in the
+ * &drm_plane_state.
+ *
+ * For the atomic ioctl the following standard (atomic) properties on the plane object
+ * encode the basic plane composition model:
+ *
+ * SRC_X:
+ * 	X coordinate offset for the source rectangle within the
+ * 	&drm_framebuffer, in 16.16 fixed point. Must be positive.
+ * SRC_Y:
+ * 	Y coordinate offset for the source rectangle within the
+ * 	&drm_framebuffer, in 16.16 fixed point. Must be positive.
+ * SRC_W:
+ * 	Width for the source rectangle within the &drm_framebuffer, in 16.16
+ * 	fixed point. SRC_X plus SRC_W must be within the width of the source
+ * 	framebuffer. Must be positive.
+ * SRC_H:
+ * 	Height for the source rectangle within the &drm_framebuffer, in 16.16
+ * 	fixed point. SRC_Y plus SRC_H must be within the height of the source
+ * 	framebuffer. Must be positive.
+ * CRTC_X:
+ * 	X coordinate offset for the destination rectangle. Can be negative.
+ * CRTC_Y:
+ * 	Y coordinate offset for the destination rectangle. Can be negative.
+ * CRTC_W:
+ * 	Width for the destination rectangle. CRTC_X plus CRTC_W can extend past
+ * 	the currently visible horizontal area of the &drm_crtc.
+ * CRTC_H:
+ * 	Height for the destination rectangle. CRTC_Y plus CRTC_H can extend past
+ * 	the currently visible vertical area of the &drm_crtc.
+ * FB_ID:
+ * 	Mode object ID of the &drm_framebuffer this plane should scan out.
+ * CRTC_ID:
+ * 	Mode object ID of the &drm_crtc this plane should be connected to.
+ *
+ * Note that the source rectangle must fully lie within the bounds of the
+ * &drm_framebuffer. The destination rectangle can lie outside of the visible
+ * area of the current mode of the CRTC. It must be apprpriately clipped by the
+ * driver, which can be done by calling drm_plane_helper_check_update(). Drivers
+ * are also allowed to round the subpixel sampling positions appropriately, but
+ * only to the next full pixel. No pixel outside of the source rectangle may
+ * ever be sampled, which is important when applying more sophisticated
+ * filtering than just a bilinear one when scaling. The filtering mode when
+ * scaling is unspecified.
+ *
+ * On top of this basic transformation additional properties can be exposed by
+ * the driver:
+ *
+ * - Rotation is set up with drm_mode_create_rotation_property(). It adds a
+ *   rotation and reflection step between the source and destination rectangles.
+ *   Without this property the rectangle is only scaled, but not rotated or
+ *   reflected.
+ *
+ * - Z position is set up with drm_plane_create_zpos_immutable_property() and
+ *   drm_plane_create_zpos_property(). It controls the visibility of overlapping
+ *   planes. Without this property the primary plane is always below the cursor
+ *   plane, and ordering between all other planes is undefined.
+ *
+ * Note that all the property extensions described here apply either to the
+ * plane or the CRTC (e.g. for the background color, which currently is not
+ * exposed and assumed to be black).
+ */
+
+/**
+ * drm_mode_create_rotation_property - create a new rotation property
+ * @dev: DRM device
+ * @supported_rotations: bitmask of supported rotations and reflections
+ *
+ * This creates a new property with the selected support for transformations.
+ * The resulting property should be stored in @rotation_property in
+ * &drm_mode_config. It then must be attached to each plane which supports
+ * rotations using drm_object_attach_property().
+ *
+ * FIXME: Probably better if the rotation property is created on each plane,
+ * like the zpos property. Otherwise it's not possible to allow different
+ * rotation modes on different planes.
+ *
+ * Since a rotation by 180° degress is the same as reflecting both along the x
+ * and the y axis the rotation property is somewhat redundant. Drivers can use
+ * drm_rotation_simplify() to normalize values of this property.
+ *
+ * The property exposed to userspace is a bitmask property (see
+ * drm_property_create_bitmask()) called "rotation" and has the following
+ * bitmask enumaration values:
+ *
+ * DRM_ROTATE_0:
+ * 	"rotate-0"
+ * DRM_ROTATE_90:
+ * 	"rotate-90"
+ * DRM_ROTATE_180:
+ * 	"rotate-180"
+ * DRM_ROTATE_270:
+ * 	"rotate-270"
+ * DRM_REFLECT_X:
+ * 	"reflect-x"
+ * DRM_REFELCT_Y:
+ * 	"reflect-y"
+ *
+ * Rotation is the specified amount in degrees in counter clockwise direction,
+ * the X and Y axis are within the source rectangle, i.e.  the X/Y axis before
+ * rotation. After reflection, the rotation is applied to the image sampled from
+ * the source rectangle, before scaling it to fit the destination rectangle.
+ */
+struct drm_property *drm_mode_create_rotation_property(struct drm_device *dev,
+						       unsigned int supported_rotations)
+{
+	static const struct drm_prop_enum_list props[] = {
+		{ __builtin_ffs(DRM_ROTATE_0) - 1,   "rotate-0" },
+		{ __builtin_ffs(DRM_ROTATE_90) - 1,  "rotate-90" },
+		{ __builtin_ffs(DRM_ROTATE_180) - 1, "rotate-180" },
+		{ __builtin_ffs(DRM_ROTATE_270) - 1, "rotate-270" },
+		{ __builtin_ffs(DRM_REFLECT_X) - 1,  "reflect-x" },
+		{ __builtin_ffs(DRM_REFLECT_Y) - 1,  "reflect-y" },
+	};
+
+	return drm_property_create_bitmask(dev, 0, "rotation",
+					   props, ARRAY_SIZE(props),
+					   supported_rotations);
+}
+EXPORT_SYMBOL(drm_mode_create_rotation_property);
+
+/**
+ * drm_rotation_simplify() - Try to simplify the rotation
+ * @rotation: Rotation to be simplified
+ * @supported_rotations: Supported rotations
+ *
+ * Attempt to simplify the rotation to a form that is supported.
+ * Eg. if the hardware supports everything except DRM_REFLECT_X
+ * one could call this function like this:
+ *
+ * drm_rotation_simplify(rotation, DRM_ROTATE_0 |
+ *                       DRM_ROTATE_90 | DRM_ROTATE_180 |
+ *                       DRM_ROTATE_270 | DRM_REFLECT_Y);
+ *
+ * to eliminate the DRM_ROTATE_X flag. Depending on what kind of
+ * transforms the hardware supports, this function may not
+ * be able to produce a supported transform, so the caller should
+ * check the result afterwards.
+ */
+unsigned int drm_rotation_simplify(unsigned int rotation,
+				   unsigned int supported_rotations)
+{
+	if (rotation & ~supported_rotations) {
+		rotation ^= DRM_REFLECT_X | DRM_REFLECT_Y;
+		rotation = (rotation & DRM_REFLECT_MASK) |
+		           BIT((ffs(rotation & DRM_ROTATE_MASK) + 1) % 4);
+	}
+
+	return rotation;
+}
+EXPORT_SYMBOL(drm_rotation_simplify);
 
 /**
  * drm_plane_create_zpos_property - create mutable zpos property
@@ -49,10 +210,14 @@
  * If zpos of some planes cannot be changed (like fixed background or
  * cursor/topmost planes), driver should adjust min/max values and assign those
  * planes immutable zpos property with lower or higher values (for more
- * information, see drm_mode_create_zpos_immutable_property() function). In such
+ * information, see drm_plane_create_zpos_immutable_property() function). In such
  * case driver should also assign proper initial zpos values for all planes in
  * its plane_reset() callback, so the planes will be always sorted properly.
  *
+ * See also drm_atomic_normalize_zpos().
+ *
+ * The property exposed to userspace is called "zpos".
+ *
  * Returns:
  * Zero on success, negative errno on failure.
  */
@@ -88,7 +253,9 @@ EXPORT_SYMBOL(drm_plane_create_zpos_property);
  * support for it in drm core. Using this property driver lets userspace
  * to get the arrangement of the planes for blending operation and notifies
  * it that the hardware (or driver) doesn't support changing of the planes'
- * order.
+ * order. For mutable zpos see drm_plane_create_zpos_property().
+ *
+ * The property exposed to userspace is called "zpos".
  *
  * Returns:
  * Zero on success, negative errno on failure.
@@ -127,20 +294,6 @@ static int drm_atomic_state_zpos_cmp(const void *a, const void *b)
 		return sa->plane->base.id - sb->plane->base.id;
 }
 
-/**
- * drm_atomic_helper_crtc_normalize_zpos - calculate normalized zpos values
- * @crtc: crtc with planes, which have to be considered for normalization
- * @crtc_state: new atomic state to apply
- *
- * This function checks new states of all planes assigned to given crtc and
- * calculates normalized zpos value for them. Planes are compared first by their
- * zpos values, then by plane id (if zpos equals). Plane with lowest zpos value
- * is at the bottom. The plane_state->normalized_zpos is then filled with unique
- * values from 0 to number of active planes in crtc minus one.
- *
- * RETURNS
- * Zero for success or -errno
- */
 static int drm_atomic_helper_crtc_normalize_zpos(struct drm_crtc *crtc,
 					  struct drm_crtc_state *crtc_state)
 {
@@ -193,20 +346,25 @@ done:
 }
 
 /**
- * drm_atomic_helper_normalize_zpos - calculate normalized zpos values for all
- *				      crtcs
+ * drm_atomic_normalize_zpos - calculate normalized zpos values for all crtcs
  * @dev: DRM device
  * @state: atomic state of DRM device
  *
  * This function calculates normalized zpos value for all modified planes in
- * the provided atomic state of DRM device. For more information, see
- * drm_atomic_helper_crtc_normalize_zpos() function.
+ * the provided atomic state of DRM device.
+ *
+ * For every CRTC this function checks new states of all planes assigned to
+ * it and calculates normalized zpos value for these planes. Planes are compared
+ * first by their zpos values, then by plane id (if zpos is equal). The plane
+ * with lowest zpos value is at the bottom. The plane_state->normalized_zpos is
+ * then filled with unique values from 0 to number of active planes in crtc
+ * minus one.
  *
  * RETURNS
  * Zero for success or -errno
  */
-int drm_atomic_helper_normalize_zpos(struct drm_device *dev,
-				     struct drm_atomic_state *state)
+int drm_atomic_normalize_zpos(struct drm_device *dev,
+			      struct drm_atomic_state *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
@@ -236,3 +394,4 @@ int drm_atomic_helper_normalize_zpos(struct drm_device *dev,
 	}
 	return 0;
 }
+EXPORT_SYMBOL(drm_atomic_normalize_zpos);
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index 255543086590..0ee052b7c21a 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -23,10 +23,9 @@
 
 #include <linux/err.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 
-#include <drm/drm_crtc.h>
-
-#include "drm/drmP.h"
+#include <drm/drm_bridge.h>
 
 /**
  * DOC: overview
@@ -98,11 +97,11 @@ EXPORT_SYMBOL(drm_bridge_remove);
  * @dev: DRM device
  * @bridge: bridge control structure
  *
- * called by a kms driver to link one of our encoder/bridge to the given
+ * Called by a kms driver to link one of our encoder/bridge to the given
  * bridge.
  *
  * Note that setting up links between the bridge and our encoder/bridge
- * objects needs to be handled by the kms driver itself
+ * objects needs to be handled by the kms driver itself.
  *
  * RETURNS:
  * Zero on success, error code on failure
@@ -125,6 +124,31 @@ int drm_bridge_attach(struct drm_device *dev, struct drm_bridge *bridge)
 EXPORT_SYMBOL(drm_bridge_attach);
 
 /**
+ * drm_bridge_detach - deassociate given bridge from its DRM device
+ *
+ * @bridge: bridge control structure
+ *
+ * Called by a kms driver to unlink the given bridge from its DRM device.
+ *
+ * Note that tearing down links between the bridge and our encoder/bridge
+ * objects needs to be handled by the kms driver itself.
+ */
+void drm_bridge_detach(struct drm_bridge *bridge)
+{
+	if (WARN_ON(!bridge))
+		return;
+
+	if (WARN_ON(!bridge->dev))
+		return;
+
+	if (bridge->funcs->detach)
+		bridge->funcs->detach(bridge);
+
+	bridge->dev = NULL;
+}
+EXPORT_SYMBOL(drm_bridge_detach);
+
+/**
  * DOC: bridge callbacks
  *
  * The &drm_bridge_funcs ops are populated by the bridge driver. The DRM
diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c
index c3a12cd8bd0d..adb1dd7fde5f 100644
--- a/drivers/gpu/drm/drm_bufs.c
+++ b/drivers/gpu/drm/drm_bufs.c
@@ -397,7 +397,7 @@ int drm_legacy_addmap_ioctl(struct drm_device *dev, void *data,
 		return -EPERM;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	err = drm_addmap_core(dev, map->offset, map->size, map->type,
@@ -443,7 +443,7 @@ int drm_legacy_getmap_ioctl(struct drm_device *dev, void *data,
 	int i;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	idx = map->offset;
@@ -545,7 +545,7 @@ EXPORT_SYMBOL(drm_legacy_rmmap_locked);
 void drm_legacy_rmmap(struct drm_device *dev, struct drm_local_map *map)
 {
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return;
 
 	mutex_lock(&dev->struct_mutex);
@@ -558,7 +558,7 @@ void drm_legacy_master_rmmaps(struct drm_device *dev, struct drm_master *master)
 {
 	struct drm_map_list *r_list, *list_temp;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return;
 
 	mutex_lock(&dev->struct_mutex);
@@ -595,7 +595,7 @@ int drm_legacy_rmmap_ioctl(struct drm_device *dev, void *data,
 	int ret;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	mutex_lock(&dev->struct_mutex);
@@ -755,7 +755,7 @@ int drm_legacy_addbufs_agp(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	entry->buflist = kzalloc(count * sizeof(*entry->buflist), GFP_KERNEL);
+	entry->buflist = kcalloc(count, sizeof(*entry->buflist), GFP_KERNEL);
 	if (!entry->buflist) {
 		mutex_unlock(&dev->struct_mutex);
 		atomic_dec(&dev->buf_alloc);
@@ -905,14 +905,14 @@ int drm_legacy_addbufs_pci(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	entry->buflist = kzalloc(count * sizeof(*entry->buflist), GFP_KERNEL);
+	entry->buflist = kcalloc(count, sizeof(*entry->buflist), GFP_KERNEL);
 	if (!entry->buflist) {
 		mutex_unlock(&dev->struct_mutex);
 		atomic_dec(&dev->buf_alloc);
 		return -ENOMEM;
 	}
 
-	entry->seglist = kzalloc(count * sizeof(*entry->seglist), GFP_KERNEL);
+	entry->seglist = kcalloc(count, sizeof(*entry->seglist), GFP_KERNEL);
 	if (!entry->seglist) {
 		kfree(entry->buflist);
 		mutex_unlock(&dev->struct_mutex);
@@ -923,8 +923,9 @@ int drm_legacy_addbufs_pci(struct drm_device *dev,
 	/* Keep the original pagelist until we know all the allocations
 	 * have succeeded
 	 */
-	temp_pagelist = kmalloc((dma->page_count + (count << page_order)) *
-			       sizeof(*dma->pagelist), GFP_KERNEL);
+	temp_pagelist = kmalloc_array(dma->page_count + (count << page_order),
+				      sizeof(*dma->pagelist),
+				      GFP_KERNEL);
 	if (!temp_pagelist) {
 		kfree(entry->buflist);
 		kfree(entry->seglist);
@@ -1116,8 +1117,7 @@ static int drm_legacy_addbufs_sg(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	entry->buflist = kzalloc(count * sizeof(*entry->buflist),
-				GFP_KERNEL);
+	entry->buflist = kcalloc(count, sizeof(*entry->buflist), GFP_KERNEL);
 	if (!entry->buflist) {
 		mutex_unlock(&dev->struct_mutex);
 		atomic_dec(&dev->buf_alloc);
@@ -1220,7 +1220,7 @@ int drm_legacy_addbufs(struct drm_device *dev, void *data,
 	struct drm_buf_desc *request = data;
 	int ret;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (!drm_core_check_feature(dev, DRIVER_HAVE_DMA))
@@ -1266,7 +1266,7 @@ int drm_legacy_infobufs(struct drm_device *dev, void *data,
 	int i;
 	int count;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (!drm_core_check_feature(dev, DRIVER_HAVE_DMA))
@@ -1347,7 +1347,7 @@ int drm_legacy_markbufs(struct drm_device *dev, void *data,
 	int order;
 	struct drm_buf_entry *entry;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (!drm_core_check_feature(dev, DRIVER_HAVE_DMA))
@@ -1395,7 +1395,7 @@ int drm_legacy_freebufs(struct drm_device *dev, void *data,
 	int idx;
 	struct drm_buf *buf;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (!drm_core_check_feature(dev, DRIVER_HAVE_DMA))
@@ -1450,7 +1450,7 @@ int drm_legacy_mapbufs(struct drm_device *dev, void *data,
 	struct drm_buf_map *request = data;
 	int i;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (!drm_core_check_feature(dev, DRIVER_HAVE_DMA))
@@ -1530,7 +1530,7 @@ int drm_legacy_mapbufs(struct drm_device *dev, void *data,
 int drm_legacy_dma_ioctl(struct drm_device *dev, void *data,
 		  struct drm_file *file_priv)
 {
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (dev->driver->dma_ioctl)
diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c
new file mode 100644
index 000000000000..d28ffdd2b929
--- /dev/null
+++ b/drivers/gpu/drm/drm_color_mgmt.c
@@ -0,0 +1,296 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_color_mgmt.h>
+
+#include "drm_crtc_internal.h"
+
+/**
+ * DOC: overview
+ *
+ * Color management or color space adjustments is supported through a set of 5
+ * properties on the &drm_crtc object. They are set up by calling
+ * drm_crtc_enable_color_mgmt().
+ *
+ * "DEGAMMA_LUT�:
+ *	Blob property to set the degamma lookup table (LUT) mapping pixel data
+ *	from the framebuffer before it is given to the transformation matrix.
+ *	The data is interpreted as an array of struct &drm_color_lut elements.
+ *	Hardware might choose not to use the full precision of the LUT elements
+ *	nor use all the elements of the LUT (for example the hardware might
+ *	choose to interpolate between LUT[0] and LUT[4]).
+ *
+ * “DEGAMMA_LUT_SIZE�:
+ *	Unsinged range property to give the size of the lookup table to be set
+ *	on the DEGAMMA_LUT property (the size depends on the underlying
+ *	hardware). If drivers support multiple LUT sizes then they should
+ *	publish the largest size, and sub-sample smaller sized LUTs (e.g. for
+ *	split-gamma modes) appropriately.
+ *
+ * “CTM�:
+ *	Blob property to set the current transformation matrix (CTM) apply to
+ *	pixel data after the lookup through the degamma LUT and before the
+ *	lookup through the gamma LUT. The data is interpreted as a struct
+ *	&drm_color_ctm.
+ *
+ * “GAMMA_LUT�:
+ *	Blob property to set the gamma lookup table (LUT) mapping pixel data
+ *	after the transformation matrix to data sent to the connector. The
+ *	data is interpreted as an array of struct &drm_color_lut elements.
+ *	Hardware might choose not to use the full precision of the LUT elements
+ *	nor use all the elements of the LUT (for example the hardware might
+ *	choose to interpolate between LUT[0] and LUT[4]).
+ *
+ * “GAMMA_LUT_SIZE�:
+ *	Unsigned range property to give the size of the lookup table to be set
+ *	on the GAMMA_LUT property (the size depends on the underlying hardware).
+ *	If drivers support multiple LUT sizes then they should publish the
+ *	largest size, and sub-sample smaller sized LUTs (e.g. for split-gamma
+ *	modes) appropriately.
+ *
+ * There is also support for a legacy gamma table, which is set up by calling
+ * drm_mode_crtc_set_gamma_size(). Drivers which support both should use
+ * drm_atomic_helper_legacy_gamma_set() to alias the legacy gamma ramp with the
+ * "GAMMA_LUT" property above.
+ */
+
+/**
+ * drm_crtc_enable_color_mgmt - enable color management properties
+ * @crtc: DRM CRTC
+ * @degamma_lut_size: the size of the degamma lut (before CSC)
+ * @has_ctm: whether to attach ctm_property for CSC matrix
+ * @gamma_lut_size: the size of the gamma lut (after CSC)
+ *
+ * This function lets the driver enable the color correction
+ * properties on a CRTC. This includes 3 degamma, csc and gamma
+ * properties that userspace can set and 2 size properties to inform
+ * the userspace of the lut sizes. Each of the properties are
+ * optional. The gamma and degamma properties are only attached if
+ * their size is not 0 and ctm_property is only attached if has_ctm is
+ * true.
+ */
+void drm_crtc_enable_color_mgmt(struct drm_crtc *crtc,
+				uint degamma_lut_size,
+				bool has_ctm,
+				uint gamma_lut_size)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_mode_config *config = &dev->mode_config;
+
+	if (degamma_lut_size) {
+		drm_object_attach_property(&crtc->base,
+					   config->degamma_lut_property, 0);
+		drm_object_attach_property(&crtc->base,
+					   config->degamma_lut_size_property,
+					   degamma_lut_size);
+	}
+
+	if (has_ctm)
+		drm_object_attach_property(&crtc->base,
+					   config->ctm_property, 0);
+
+	if (gamma_lut_size) {
+		drm_object_attach_property(&crtc->base,
+					   config->gamma_lut_property, 0);
+		drm_object_attach_property(&crtc->base,
+					   config->gamma_lut_size_property,
+					   gamma_lut_size);
+	}
+}
+EXPORT_SYMBOL(drm_crtc_enable_color_mgmt);
+
+/**
+ * drm_mode_crtc_set_gamma_size - set the gamma table size
+ * @crtc: CRTC to set the gamma table size for
+ * @gamma_size: size of the gamma table
+ *
+ * Drivers which support gamma tables should set this to the supported gamma
+ * table size when initializing the CRTC. Currently the drm core only supports a
+ * fixed gamma table size.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_crtc_set_gamma_size(struct drm_crtc *crtc,
+				 int gamma_size)
+{
+	uint16_t *r_base, *g_base, *b_base;
+	int i;
+
+	crtc->gamma_size = gamma_size;
+
+	crtc->gamma_store = kcalloc(gamma_size, sizeof(uint16_t) * 3,
+				    GFP_KERNEL);
+	if (!crtc->gamma_store) {
+		crtc->gamma_size = 0;
+		return -ENOMEM;
+	}
+
+	r_base = crtc->gamma_store;
+	g_base = r_base + gamma_size;
+	b_base = g_base + gamma_size;
+	for (i = 0; i < gamma_size; i++) {
+		r_base[i] = i << 8;
+		g_base[i] = i << 8;
+		b_base[i] = i << 8;
+	}
+
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_mode_crtc_set_gamma_size);
+
+/**
+ * drm_mode_gamma_set_ioctl - set the gamma table
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Set the gamma table of a CRTC to the one passed in by the user. Userspace can
+ * inquire the required gamma table size through drm_mode_gamma_get_ioctl.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_gamma_set_ioctl(struct drm_device *dev,
+			     void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_crtc_lut *crtc_lut = data;
+	struct drm_crtc *crtc;
+	void *r_base, *g_base, *b_base;
+	int size;
+	int ret = 0;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	drm_modeset_lock_all(dev);
+	crtc = drm_crtc_find(dev, crtc_lut->crtc_id);
+	if (!crtc) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	if (crtc->funcs->gamma_set == NULL) {
+		ret = -ENOSYS;
+		goto out;
+	}
+
+	/* memcpy into gamma store */
+	if (crtc_lut->gamma_size != crtc->gamma_size) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	size = crtc_lut->gamma_size * (sizeof(uint16_t));
+	r_base = crtc->gamma_store;
+	if (copy_from_user(r_base, (void __user *)(unsigned long)crtc_lut->red, size)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	g_base = r_base + size;
+	if (copy_from_user(g_base, (void __user *)(unsigned long)crtc_lut->green, size)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	b_base = g_base + size;
+	if (copy_from_user(b_base, (void __user *)(unsigned long)crtc_lut->blue, size)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	ret = crtc->funcs->gamma_set(crtc, r_base, g_base, b_base, crtc->gamma_size);
+
+out:
+	drm_modeset_unlock_all(dev);
+	return ret;
+
+}
+
+/**
+ * drm_mode_gamma_get_ioctl - get the gamma table
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Copy the current gamma table into the storage provided. This also provides
+ * the gamma table size the driver expects, which can be used to size the
+ * allocated storage.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_gamma_get_ioctl(struct drm_device *dev,
+			     void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_crtc_lut *crtc_lut = data;
+	struct drm_crtc *crtc;
+	void *r_base, *g_base, *b_base;
+	int size;
+	int ret = 0;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	drm_modeset_lock_all(dev);
+	crtc = drm_crtc_find(dev, crtc_lut->crtc_id);
+	if (!crtc) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	/* memcpy into gamma store */
+	if (crtc_lut->gamma_size != crtc->gamma_size) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	size = crtc_lut->gamma_size * (sizeof(uint16_t));
+	r_base = crtc->gamma_store;
+	if (copy_to_user((void __user *)(unsigned long)crtc_lut->red, r_base, size)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	g_base = r_base + size;
+	if (copy_to_user((void __user *)(unsigned long)crtc_lut->green, g_base, size)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	b_base = g_base + size;
+	if (copy_to_user((void __user *)(unsigned long)crtc_lut->blue, b_base, size)) {
+		ret = -EFAULT;
+		goto out;
+	}
+out:
+	drm_modeset_unlock_all(dev);
+	return ret;
+}
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
new file mode 100644
index 000000000000..2db7fb510b6c
--- /dev/null
+++ b/drivers/gpu/drm/drm_connector.c
@@ -0,0 +1,1123 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_edid.h>
+
+#include "drm_crtc_internal.h"
+#include "drm_internal.h"
+
+/**
+ * DOC: overview
+ *
+ * In DRM connectors are the general abstraction for display sinks, and include
+ * als fixed panels or anything else that can display pixels in some form. As
+ * opposed to all other KMS objects representing hardware (like CRTC, encoder or
+ * plane abstractions) connectors can be hotplugged and unplugged at runtime.
+ * Hence they are reference-counted using drm_connector_reference() and
+ * drm_connector_unreference().
+ *
+ * KMS driver must create, initialize, register and attach at a struct
+ * &drm_connector for each such sink. The instance is created as other KMS
+ * objects and initialized by setting the following fields.
+ *
+ * The connector is then registered with a call to drm_connector_init() with a
+ * pointer to the connector functions and a connector type, and exposed through
+ * sysfs with a call to drm_connector_register().
+ *
+ * Connectors must be attached to an encoder to be used. For devices that map
+ * connectors to encoders 1:1, the connector should be attached at
+ * initialization time with a call to drm_mode_connector_attach_encoder(). The
+ * driver must also set the struct &drm_connector encoder field to point to the
+ * attached encoder.
+ *
+ * For connectors which are not fixed (like built-in panels) the driver needs to
+ * support hotplug notifications. The simplest way to do that is by using the
+ * probe helpers, see drm_kms_helper_poll_init() for connectors which don't have
+ * hardware support for hotplug interrupts. Connectors with hardware hotplug
+ * support can instead use e.g. drm_helper_hpd_irq_event().
+ */
+
+struct drm_conn_prop_enum_list {
+	int type;
+	const char *name;
+	struct ida ida;
+};
+
+/*
+ * Connector and encoder types.
+ */
+static struct drm_conn_prop_enum_list drm_connector_enum_list[] = {
+	{ DRM_MODE_CONNECTOR_Unknown, "Unknown" },
+	{ DRM_MODE_CONNECTOR_VGA, "VGA" },
+	{ DRM_MODE_CONNECTOR_DVII, "DVI-I" },
+	{ DRM_MODE_CONNECTOR_DVID, "DVI-D" },
+	{ DRM_MODE_CONNECTOR_DVIA, "DVI-A" },
+	{ DRM_MODE_CONNECTOR_Composite, "Composite" },
+	{ DRM_MODE_CONNECTOR_SVIDEO, "SVIDEO" },
+	{ DRM_MODE_CONNECTOR_LVDS, "LVDS" },
+	{ DRM_MODE_CONNECTOR_Component, "Component" },
+	{ DRM_MODE_CONNECTOR_9PinDIN, "DIN" },
+	{ DRM_MODE_CONNECTOR_DisplayPort, "DP" },
+	{ DRM_MODE_CONNECTOR_HDMIA, "HDMI-A" },
+	{ DRM_MODE_CONNECTOR_HDMIB, "HDMI-B" },
+	{ DRM_MODE_CONNECTOR_TV, "TV" },
+	{ DRM_MODE_CONNECTOR_eDP, "eDP" },
+	{ DRM_MODE_CONNECTOR_VIRTUAL, "Virtual" },
+	{ DRM_MODE_CONNECTOR_DSI, "DSI" },
+	{ DRM_MODE_CONNECTOR_DPI, "DPI" },
+};
+
+void drm_connector_ida_init(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(drm_connector_enum_list); i++)
+		ida_init(&drm_connector_enum_list[i].ida);
+}
+
+void drm_connector_ida_destroy(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(drm_connector_enum_list); i++)
+		ida_destroy(&drm_connector_enum_list[i].ida);
+}
+
+/**
+ * drm_connector_get_cmdline_mode - reads the user's cmdline mode
+ * @connector: connector to quwery
+ *
+ * The kernel supports per-connector configuration of its consoles through
+ * use of the video= parameter. This function parses that option and
+ * extracts the user's specified mode (or enable/disable status) for a
+ * particular connector. This is typically only used during the early fbdev
+ * setup.
+ */
+static void drm_connector_get_cmdline_mode(struct drm_connector *connector)
+{
+	struct drm_cmdline_mode *mode = &connector->cmdline_mode;
+	char *option = NULL;
+
+	if (fb_get_options(connector->name, &option))
+		return;
+
+	if (!drm_mode_parse_command_line_for_connector(option,
+						       connector,
+						       mode))
+		return;
+
+	if (mode->force) {
+		const char *s;
+
+		switch (mode->force) {
+		case DRM_FORCE_OFF:
+			s = "OFF";
+			break;
+		case DRM_FORCE_ON_DIGITAL:
+			s = "ON - dig";
+			break;
+		default:
+		case DRM_FORCE_ON:
+			s = "ON";
+			break;
+		}
+
+		DRM_INFO("forcing %s connector %s\n", connector->name, s);
+		connector->force = mode->force;
+	}
+
+	DRM_DEBUG_KMS("cmdline mode for connector %s %dx%d@%dHz%s%s%s\n",
+		      connector->name,
+		      mode->xres, mode->yres,
+		      mode->refresh_specified ? mode->refresh : 60,
+		      mode->rb ? " reduced blanking" : "",
+		      mode->margins ? " with margins" : "",
+		      mode->interlace ?  " interlaced" : "");
+}
+
+static void drm_connector_free(struct kref *kref)
+{
+	struct drm_connector *connector =
+		container_of(kref, struct drm_connector, base.refcount);
+	struct drm_device *dev = connector->dev;
+
+	drm_mode_object_unregister(dev, &connector->base);
+	connector->funcs->destroy(connector);
+}
+
+/**
+ * drm_connector_init - Init a preallocated connector
+ * @dev: DRM device
+ * @connector: the connector to init
+ * @funcs: callbacks for this connector
+ * @connector_type: user visible type of the connector
+ *
+ * Initialises a preallocated connector. Connectors should be
+ * subclassed as part of driver connector objects.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_connector_init(struct drm_device *dev,
+		       struct drm_connector *connector,
+		       const struct drm_connector_funcs *funcs,
+		       int connector_type)
+{
+	struct drm_mode_config *config = &dev->mode_config;
+	int ret;
+	struct ida *connector_ida =
+		&drm_connector_enum_list[connector_type].ida;
+
+	drm_modeset_lock_all(dev);
+
+	ret = drm_mode_object_get_reg(dev, &connector->base,
+				      DRM_MODE_OBJECT_CONNECTOR,
+				      false, drm_connector_free);
+	if (ret)
+		goto out_unlock;
+
+	connector->base.properties = &connector->properties;
+	connector->dev = dev;
+	connector->funcs = funcs;
+
+	ret = ida_simple_get(&config->connector_ida, 0, 0, GFP_KERNEL);
+	if (ret < 0)
+		goto out_put;
+	connector->index = ret;
+	ret = 0;
+
+	connector->connector_type = connector_type;
+	connector->connector_type_id =
+		ida_simple_get(connector_ida, 1, 0, GFP_KERNEL);
+	if (connector->connector_type_id < 0) {
+		ret = connector->connector_type_id;
+		goto out_put_id;
+	}
+	connector->name =
+		kasprintf(GFP_KERNEL, "%s-%d",
+			  drm_connector_enum_list[connector_type].name,
+			  connector->connector_type_id);
+	if (!connector->name) {
+		ret = -ENOMEM;
+		goto out_put_type_id;
+	}
+
+	INIT_LIST_HEAD(&connector->probed_modes);
+	INIT_LIST_HEAD(&connector->modes);
+	connector->edid_blob_ptr = NULL;
+	connector->status = connector_status_unknown;
+
+	drm_connector_get_cmdline_mode(connector);
+
+	/* We should add connectors at the end to avoid upsetting the connector
+	 * index too much. */
+	list_add_tail(&connector->head, &config->connector_list);
+	config->num_connector++;
+
+	if (connector_type != DRM_MODE_CONNECTOR_VIRTUAL)
+		drm_object_attach_property(&connector->base,
+					      config->edid_property,
+					      0);
+
+	drm_object_attach_property(&connector->base,
+				      config->dpms_property, 0);
+
+	if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
+		drm_object_attach_property(&connector->base, config->prop_crtc_id, 0);
+	}
+
+	connector->debugfs_entry = NULL;
+out_put_type_id:
+	if (ret)
+		ida_simple_remove(connector_ida, connector->connector_type_id);
+out_put_id:
+	if (ret)
+		ida_simple_remove(&config->connector_ida, connector->index);
+out_put:
+	if (ret)
+		drm_mode_object_unregister(dev, &connector->base);
+
+out_unlock:
+	drm_modeset_unlock_all(dev);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_connector_init);
+
+/**
+ * drm_mode_connector_attach_encoder - attach a connector to an encoder
+ * @connector: connector to attach
+ * @encoder: encoder to attach @connector to
+ *
+ * This function links up a connector to an encoder. Note that the routing
+ * restrictions between encoders and crtcs are exposed to userspace through the
+ * possible_clones and possible_crtcs bitmasks.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_connector_attach_encoder(struct drm_connector *connector,
+				      struct drm_encoder *encoder)
+{
+	int i;
+
+	/*
+	 * In the past, drivers have attempted to model the static association
+	 * of connector to encoder in simple connector/encoder devices using a
+	 * direct assignment of connector->encoder = encoder. This connection
+	 * is a logical one and the responsibility of the core, so drivers are
+	 * expected not to mess with this.
+	 *
+	 * Note that the error return should've been enough here, but a large
+	 * majority of drivers ignores the return value, so add in a big WARN
+	 * to get people's attention.
+	 */
+	if (WARN_ON(connector->encoder))
+		return -EINVAL;
+
+	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
+		if (connector->encoder_ids[i] == 0) {
+			connector->encoder_ids[i] = encoder->base.id;
+			return 0;
+		}
+	}
+	return -ENOMEM;
+}
+EXPORT_SYMBOL(drm_mode_connector_attach_encoder);
+
+static void drm_mode_remove(struct drm_connector *connector,
+			    struct drm_display_mode *mode)
+{
+	list_del(&mode->head);
+	drm_mode_destroy(connector->dev, mode);
+}
+
+/**
+ * drm_connector_cleanup - cleans up an initialised connector
+ * @connector: connector to cleanup
+ *
+ * Cleans up the connector but doesn't free the object.
+ */
+void drm_connector_cleanup(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_display_mode *mode, *t;
+
+	/* The connector should have been removed from userspace long before
+	 * it is finally destroyed.
+	 */
+	if (WARN_ON(connector->registered))
+		drm_connector_unregister(connector);
+
+	if (connector->tile_group) {
+		drm_mode_put_tile_group(dev, connector->tile_group);
+		connector->tile_group = NULL;
+	}
+
+	list_for_each_entry_safe(mode, t, &connector->probed_modes, head)
+		drm_mode_remove(connector, mode);
+
+	list_for_each_entry_safe(mode, t, &connector->modes, head)
+		drm_mode_remove(connector, mode);
+
+	ida_simple_remove(&drm_connector_enum_list[connector->connector_type].ida,
+			  connector->connector_type_id);
+
+	ida_simple_remove(&dev->mode_config.connector_ida,
+			  connector->index);
+
+	kfree(connector->display_info.bus_formats);
+	drm_mode_object_unregister(dev, &connector->base);
+	kfree(connector->name);
+	connector->name = NULL;
+	list_del(&connector->head);
+	dev->mode_config.num_connector--;
+
+	WARN_ON(connector->state && !connector->funcs->atomic_destroy_state);
+	if (connector->state && connector->funcs->atomic_destroy_state)
+		connector->funcs->atomic_destroy_state(connector,
+						       connector->state);
+
+	memset(connector, 0, sizeof(*connector));
+}
+EXPORT_SYMBOL(drm_connector_cleanup);
+
+/**
+ * drm_connector_register - register a connector
+ * @connector: the connector to register
+ *
+ * Register userspace interfaces for a connector
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_connector_register(struct drm_connector *connector)
+{
+	int ret;
+
+	if (connector->registered)
+		return 0;
+
+	ret = drm_sysfs_connector_add(connector);
+	if (ret)
+		return ret;
+
+	ret = drm_debugfs_connector_add(connector);
+	if (ret) {
+		goto err_sysfs;
+	}
+
+	if (connector->funcs->late_register) {
+		ret = connector->funcs->late_register(connector);
+		if (ret)
+			goto err_debugfs;
+	}
+
+	drm_mode_object_register(connector->dev, &connector->base);
+
+	connector->registered = true;
+	return 0;
+
+err_debugfs:
+	drm_debugfs_connector_remove(connector);
+err_sysfs:
+	drm_sysfs_connector_remove(connector);
+	return ret;
+}
+EXPORT_SYMBOL(drm_connector_register);
+
+/**
+ * drm_connector_unregister - unregister a connector
+ * @connector: the connector to unregister
+ *
+ * Unregister userspace interfaces for a connector
+ */
+void drm_connector_unregister(struct drm_connector *connector)
+{
+	if (!connector->registered)
+		return;
+
+	if (connector->funcs->early_unregister)
+		connector->funcs->early_unregister(connector);
+
+	drm_sysfs_connector_remove(connector);
+	drm_debugfs_connector_remove(connector);
+
+	connector->registered = false;
+}
+EXPORT_SYMBOL(drm_connector_unregister);
+
+void drm_connector_unregister_all(struct drm_device *dev)
+{
+	struct drm_connector *connector;
+
+	/* FIXME: taking the mode config mutex ends up in a clash with sysfs */
+	list_for_each_entry(connector, &dev->mode_config.connector_list, head)
+		drm_connector_unregister(connector);
+}
+
+int drm_connector_register_all(struct drm_device *dev)
+{
+	struct drm_connector *connector;
+	int ret;
+
+	/* FIXME: taking the mode config mutex ends up in a clash with
+	 * fbcon/backlight registration */
+	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+		ret = drm_connector_register(connector);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+
+err:
+	mutex_unlock(&dev->mode_config.mutex);
+	drm_connector_unregister_all(dev);
+	return ret;
+}
+
+/**
+ * drm_get_connector_status_name - return a string for connector status
+ * @status: connector status to compute name of
+ *
+ * In contrast to the other drm_get_*_name functions this one here returns a
+ * const pointer and hence is threadsafe.
+ */
+const char *drm_get_connector_status_name(enum drm_connector_status status)
+{
+	if (status == connector_status_connected)
+		return "connected";
+	else if (status == connector_status_disconnected)
+		return "disconnected";
+	else
+		return "unknown";
+}
+EXPORT_SYMBOL(drm_get_connector_status_name);
+
+static const struct drm_prop_enum_list drm_subpixel_enum_list[] = {
+	{ SubPixelUnknown, "Unknown" },
+	{ SubPixelHorizontalRGB, "Horizontal RGB" },
+	{ SubPixelHorizontalBGR, "Horizontal BGR" },
+	{ SubPixelVerticalRGB, "Vertical RGB" },
+	{ SubPixelVerticalBGR, "Vertical BGR" },
+	{ SubPixelNone, "None" },
+};
+
+/**
+ * drm_get_subpixel_order_name - return a string for a given subpixel enum
+ * @order: enum of subpixel_order
+ *
+ * Note you could abuse this and return something out of bounds, but that
+ * would be a caller error.  No unscrubbed user data should make it here.
+ */
+const char *drm_get_subpixel_order_name(enum subpixel_order order)
+{
+	return drm_subpixel_enum_list[order].name;
+}
+EXPORT_SYMBOL(drm_get_subpixel_order_name);
+
+static const struct drm_prop_enum_list drm_dpms_enum_list[] = {
+	{ DRM_MODE_DPMS_ON, "On" },
+	{ DRM_MODE_DPMS_STANDBY, "Standby" },
+	{ DRM_MODE_DPMS_SUSPEND, "Suspend" },
+	{ DRM_MODE_DPMS_OFF, "Off" }
+};
+DRM_ENUM_NAME_FN(drm_get_dpms_name, drm_dpms_enum_list)
+
+/**
+ * drm_display_info_set_bus_formats - set the supported bus formats
+ * @info: display info to store bus formats in
+ * @formats: array containing the supported bus formats
+ * @num_formats: the number of entries in the fmts array
+ *
+ * Store the supported bus formats in display info structure.
+ * See MEDIA_BUS_FMT_* definitions in include/uapi/linux/media-bus-format.h for
+ * a full list of available formats.
+ */
+int drm_display_info_set_bus_formats(struct drm_display_info *info,
+				     const u32 *formats,
+				     unsigned int num_formats)
+{
+	u32 *fmts = NULL;
+
+	if (!formats && num_formats)
+		return -EINVAL;
+
+	if (formats && num_formats) {
+		fmts = kmemdup(formats, sizeof(*formats) * num_formats,
+			       GFP_KERNEL);
+		if (!fmts)
+			return -ENOMEM;
+	}
+
+	kfree(info->bus_formats);
+	info->bus_formats = fmts;
+	info->num_bus_formats = num_formats;
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_display_info_set_bus_formats);
+
+/* Optional connector properties. */
+static const struct drm_prop_enum_list drm_scaling_mode_enum_list[] = {
+	{ DRM_MODE_SCALE_NONE, "None" },
+	{ DRM_MODE_SCALE_FULLSCREEN, "Full" },
+	{ DRM_MODE_SCALE_CENTER, "Center" },
+	{ DRM_MODE_SCALE_ASPECT, "Full aspect" },
+};
+
+static const struct drm_prop_enum_list drm_aspect_ratio_enum_list[] = {
+	{ DRM_MODE_PICTURE_ASPECT_NONE, "Automatic" },
+	{ DRM_MODE_PICTURE_ASPECT_4_3, "4:3" },
+	{ DRM_MODE_PICTURE_ASPECT_16_9, "16:9" },
+};
+
+static const struct drm_prop_enum_list drm_dvi_i_select_enum_list[] = {
+	{ DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
+	{ DRM_MODE_SUBCONNECTOR_DVID,      "DVI-D"     }, /* DVI-I  */
+	{ DRM_MODE_SUBCONNECTOR_DVIA,      "DVI-A"     }, /* DVI-I  */
+};
+DRM_ENUM_NAME_FN(drm_get_dvi_i_select_name, drm_dvi_i_select_enum_list)
+
+static const struct drm_prop_enum_list drm_dvi_i_subconnector_enum_list[] = {
+	{ DRM_MODE_SUBCONNECTOR_Unknown,   "Unknown"   }, /* DVI-I and TV-out */
+	{ DRM_MODE_SUBCONNECTOR_DVID,      "DVI-D"     }, /* DVI-I  */
+	{ DRM_MODE_SUBCONNECTOR_DVIA,      "DVI-A"     }, /* DVI-I  */
+};
+DRM_ENUM_NAME_FN(drm_get_dvi_i_subconnector_name,
+		 drm_dvi_i_subconnector_enum_list)
+
+static const struct drm_prop_enum_list drm_tv_select_enum_list[] = {
+	{ DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
+	{ DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
+	{ DRM_MODE_SUBCONNECTOR_SVIDEO,    "SVIDEO"    }, /* TV-out */
+	{ DRM_MODE_SUBCONNECTOR_Component, "Component" }, /* TV-out */
+	{ DRM_MODE_SUBCONNECTOR_SCART,     "SCART"     }, /* TV-out */
+};
+DRM_ENUM_NAME_FN(drm_get_tv_select_name, drm_tv_select_enum_list)
+
+static const struct drm_prop_enum_list drm_tv_subconnector_enum_list[] = {
+	{ DRM_MODE_SUBCONNECTOR_Unknown,   "Unknown"   }, /* DVI-I and TV-out */
+	{ DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
+	{ DRM_MODE_SUBCONNECTOR_SVIDEO,    "SVIDEO"    }, /* TV-out */
+	{ DRM_MODE_SUBCONNECTOR_Component, "Component" }, /* TV-out */
+	{ DRM_MODE_SUBCONNECTOR_SCART,     "SCART"     }, /* TV-out */
+};
+DRM_ENUM_NAME_FN(drm_get_tv_subconnector_name,
+		 drm_tv_subconnector_enum_list)
+
+int drm_connector_create_standard_properties(struct drm_device *dev)
+{
+	struct drm_property *prop;
+
+	prop = drm_property_create(dev, DRM_MODE_PROP_BLOB |
+				   DRM_MODE_PROP_IMMUTABLE,
+				   "EDID", 0);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.edid_property = prop;
+
+	prop = drm_property_create_enum(dev, 0,
+				   "DPMS", drm_dpms_enum_list,
+				   ARRAY_SIZE(drm_dpms_enum_list));
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.dpms_property = prop;
+
+	prop = drm_property_create(dev,
+				   DRM_MODE_PROP_BLOB |
+				   DRM_MODE_PROP_IMMUTABLE,
+				   "PATH", 0);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.path_property = prop;
+
+	prop = drm_property_create(dev,
+				   DRM_MODE_PROP_BLOB |
+				   DRM_MODE_PROP_IMMUTABLE,
+				   "TILE", 0);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.tile_property = prop;
+
+	return 0;
+}
+
+/**
+ * drm_mode_create_dvi_i_properties - create DVI-I specific connector properties
+ * @dev: DRM device
+ *
+ * Called by a driver the first time a DVI-I connector is made.
+ */
+int drm_mode_create_dvi_i_properties(struct drm_device *dev)
+{
+	struct drm_property *dvi_i_selector;
+	struct drm_property *dvi_i_subconnector;
+
+	if (dev->mode_config.dvi_i_select_subconnector_property)
+		return 0;
+
+	dvi_i_selector =
+		drm_property_create_enum(dev, 0,
+				    "select subconnector",
+				    drm_dvi_i_select_enum_list,
+				    ARRAY_SIZE(drm_dvi_i_select_enum_list));
+	dev->mode_config.dvi_i_select_subconnector_property = dvi_i_selector;
+
+	dvi_i_subconnector = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
+				    "subconnector",
+				    drm_dvi_i_subconnector_enum_list,
+				    ARRAY_SIZE(drm_dvi_i_subconnector_enum_list));
+	dev->mode_config.dvi_i_subconnector_property = dvi_i_subconnector;
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_mode_create_dvi_i_properties);
+
+/**
+ * drm_create_tv_properties - create TV specific connector properties
+ * @dev: DRM device
+ * @num_modes: number of different TV formats (modes) supported
+ * @modes: array of pointers to strings containing name of each format
+ *
+ * Called by a driver's TV initialization routine, this function creates
+ * the TV specific connector properties for a given device.  Caller is
+ * responsible for allocating a list of format names and passing them to
+ * this routine.
+ */
+int drm_mode_create_tv_properties(struct drm_device *dev,
+				  unsigned int num_modes,
+				  const char * const modes[])
+{
+	struct drm_property *tv_selector;
+	struct drm_property *tv_subconnector;
+	unsigned int i;
+
+	if (dev->mode_config.tv_select_subconnector_property)
+		return 0;
+
+	/*
+	 * Basic connector properties
+	 */
+	tv_selector = drm_property_create_enum(dev, 0,
+					  "select subconnector",
+					  drm_tv_select_enum_list,
+					  ARRAY_SIZE(drm_tv_select_enum_list));
+	if (!tv_selector)
+		goto nomem;
+
+	dev->mode_config.tv_select_subconnector_property = tv_selector;
+
+	tv_subconnector =
+		drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
+				    "subconnector",
+				    drm_tv_subconnector_enum_list,
+				    ARRAY_SIZE(drm_tv_subconnector_enum_list));
+	if (!tv_subconnector)
+		goto nomem;
+	dev->mode_config.tv_subconnector_property = tv_subconnector;
+
+	/*
+	 * Other, TV specific properties: margins & TV modes.
+	 */
+	dev->mode_config.tv_left_margin_property =
+		drm_property_create_range(dev, 0, "left margin", 0, 100);
+	if (!dev->mode_config.tv_left_margin_property)
+		goto nomem;
+
+	dev->mode_config.tv_right_margin_property =
+		drm_property_create_range(dev, 0, "right margin", 0, 100);
+	if (!dev->mode_config.tv_right_margin_property)
+		goto nomem;
+
+	dev->mode_config.tv_top_margin_property =
+		drm_property_create_range(dev, 0, "top margin", 0, 100);
+	if (!dev->mode_config.tv_top_margin_property)
+		goto nomem;
+
+	dev->mode_config.tv_bottom_margin_property =
+		drm_property_create_range(dev, 0, "bottom margin", 0, 100);
+	if (!dev->mode_config.tv_bottom_margin_property)
+		goto nomem;
+
+	dev->mode_config.tv_mode_property =
+		drm_property_create(dev, DRM_MODE_PROP_ENUM,
+				    "mode", num_modes);
+	if (!dev->mode_config.tv_mode_property)
+		goto nomem;
+
+	for (i = 0; i < num_modes; i++)
+		drm_property_add_enum(dev->mode_config.tv_mode_property, i,
+				      i, modes[i]);
+
+	dev->mode_config.tv_brightness_property =
+		drm_property_create_range(dev, 0, "brightness", 0, 100);
+	if (!dev->mode_config.tv_brightness_property)
+		goto nomem;
+
+	dev->mode_config.tv_contrast_property =
+		drm_property_create_range(dev, 0, "contrast", 0, 100);
+	if (!dev->mode_config.tv_contrast_property)
+		goto nomem;
+
+	dev->mode_config.tv_flicker_reduction_property =
+		drm_property_create_range(dev, 0, "flicker reduction", 0, 100);
+	if (!dev->mode_config.tv_flicker_reduction_property)
+		goto nomem;
+
+	dev->mode_config.tv_overscan_property =
+		drm_property_create_range(dev, 0, "overscan", 0, 100);
+	if (!dev->mode_config.tv_overscan_property)
+		goto nomem;
+
+	dev->mode_config.tv_saturation_property =
+		drm_property_create_range(dev, 0, "saturation", 0, 100);
+	if (!dev->mode_config.tv_saturation_property)
+		goto nomem;
+
+	dev->mode_config.tv_hue_property =
+		drm_property_create_range(dev, 0, "hue", 0, 100);
+	if (!dev->mode_config.tv_hue_property)
+		goto nomem;
+
+	return 0;
+nomem:
+	return -ENOMEM;
+}
+EXPORT_SYMBOL(drm_mode_create_tv_properties);
+
+/**
+ * drm_mode_create_scaling_mode_property - create scaling mode property
+ * @dev: DRM device
+ *
+ * Called by a driver the first time it's needed, must be attached to desired
+ * connectors.
+ */
+int drm_mode_create_scaling_mode_property(struct drm_device *dev)
+{
+	struct drm_property *scaling_mode;
+
+	if (dev->mode_config.scaling_mode_property)
+		return 0;
+
+	scaling_mode =
+		drm_property_create_enum(dev, 0, "scaling mode",
+				drm_scaling_mode_enum_list,
+				    ARRAY_SIZE(drm_scaling_mode_enum_list));
+
+	dev->mode_config.scaling_mode_property = scaling_mode;
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_mode_create_scaling_mode_property);
+
+/**
+ * drm_mode_create_aspect_ratio_property - create aspect ratio property
+ * @dev: DRM device
+ *
+ * Called by a driver the first time it's needed, must be attached to desired
+ * connectors.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_create_aspect_ratio_property(struct drm_device *dev)
+{
+	if (dev->mode_config.aspect_ratio_property)
+		return 0;
+
+	dev->mode_config.aspect_ratio_property =
+		drm_property_create_enum(dev, 0, "aspect ratio",
+				drm_aspect_ratio_enum_list,
+				ARRAY_SIZE(drm_aspect_ratio_enum_list));
+
+	if (dev->mode_config.aspect_ratio_property == NULL)
+		return -ENOMEM;
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_mode_create_aspect_ratio_property);
+
+/**
+ * drm_mode_create_suggested_offset_properties - create suggests offset properties
+ * @dev: DRM device
+ *
+ * Create the the suggested x/y offset property for connectors.
+ */
+int drm_mode_create_suggested_offset_properties(struct drm_device *dev)
+{
+	if (dev->mode_config.suggested_x_property && dev->mode_config.suggested_y_property)
+		return 0;
+
+	dev->mode_config.suggested_x_property =
+		drm_property_create_range(dev, DRM_MODE_PROP_IMMUTABLE, "suggested X", 0, 0xffffffff);
+
+	dev->mode_config.suggested_y_property =
+		drm_property_create_range(dev, DRM_MODE_PROP_IMMUTABLE, "suggested Y", 0, 0xffffffff);
+
+	if (dev->mode_config.suggested_x_property == NULL ||
+	    dev->mode_config.suggested_y_property == NULL)
+		return -ENOMEM;
+	return 0;
+}
+EXPORT_SYMBOL(drm_mode_create_suggested_offset_properties);
+
+/**
+ * drm_mode_connector_set_path_property - set tile property on connector
+ * @connector: connector to set property on.
+ * @path: path to use for property; must not be NULL.
+ *
+ * This creates a property to expose to userspace to specify a
+ * connector path. This is mainly used for DisplayPort MST where
+ * connectors have a topology and we want to allow userspace to give
+ * them more meaningful names.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_connector_set_path_property(struct drm_connector *connector,
+					 const char *path)
+{
+	struct drm_device *dev = connector->dev;
+	int ret;
+
+	ret = drm_property_replace_global_blob(dev,
+	                                       &connector->path_blob_ptr,
+	                                       strlen(path) + 1,
+	                                       path,
+	                                       &connector->base,
+	                                       dev->mode_config.path_property);
+	return ret;
+}
+EXPORT_SYMBOL(drm_mode_connector_set_path_property);
+
+/**
+ * drm_mode_connector_set_tile_property - set tile property on connector
+ * @connector: connector to set property on.
+ *
+ * This looks up the tile information for a connector, and creates a
+ * property for userspace to parse if it exists. The property is of
+ * the form of 8 integers using ':' as a separator.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
+int drm_mode_connector_set_tile_property(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	char tile[256];
+	int ret;
+
+	if (!connector->has_tile) {
+		ret  = drm_property_replace_global_blob(dev,
+		                                        &connector->tile_blob_ptr,
+		                                        0,
+		                                        NULL,
+		                                        &connector->base,
+		                                        dev->mode_config.tile_property);
+		return ret;
+	}
+
+	snprintf(tile, 256, "%d:%d:%d:%d:%d:%d:%d:%d",
+		 connector->tile_group->id, connector->tile_is_single_monitor,
+		 connector->num_h_tile, connector->num_v_tile,
+		 connector->tile_h_loc, connector->tile_v_loc,
+		 connector->tile_h_size, connector->tile_v_size);
+
+	ret = drm_property_replace_global_blob(dev,
+	                                       &connector->tile_blob_ptr,
+	                                       strlen(tile) + 1,
+	                                       tile,
+	                                       &connector->base,
+	                                       dev->mode_config.tile_property);
+	return ret;
+}
+EXPORT_SYMBOL(drm_mode_connector_set_tile_property);
+
+/**
+ * drm_mode_connector_update_edid_property - update the edid property of a connector
+ * @connector: drm connector
+ * @edid: new value of the edid property
+ *
+ * This function creates a new blob modeset object and assigns its id to the
+ * connector's edid property.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_connector_update_edid_property(struct drm_connector *connector,
+					    const struct edid *edid)
+{
+	struct drm_device *dev = connector->dev;
+	size_t size = 0;
+	int ret;
+
+	/* ignore requests to set edid when overridden */
+	if (connector->override_edid)
+		return 0;
+
+	if (edid)
+		size = EDID_LENGTH * (1 + edid->extensions);
+
+	ret = drm_property_replace_global_blob(dev,
+					       &connector->edid_blob_ptr,
+	                                       size,
+	                                       edid,
+	                                       &connector->base,
+	                                       dev->mode_config.edid_property);
+	return ret;
+}
+EXPORT_SYMBOL(drm_mode_connector_update_edid_property);
+
+int drm_mode_connector_set_obj_prop(struct drm_mode_object *obj,
+				    struct drm_property *property,
+				    uint64_t value)
+{
+	int ret = -EINVAL;
+	struct drm_connector *connector = obj_to_connector(obj);
+
+	/* Do DPMS ourselves */
+	if (property == connector->dev->mode_config.dpms_property) {
+		ret = (*connector->funcs->dpms)(connector, (int)value);
+	} else if (connector->funcs->set_property)
+		ret = connector->funcs->set_property(connector, property, value);
+
+	/* store the property value if successful */
+	if (!ret)
+		drm_object_property_set_value(&connector->base, property, value);
+	return ret;
+}
+
+int drm_mode_connector_property_set_ioctl(struct drm_device *dev,
+				       void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_connector_set_property *conn_set_prop = data;
+	struct drm_mode_obj_set_property obj_set_prop = {
+		.value = conn_set_prop->value,
+		.prop_id = conn_set_prop->prop_id,
+		.obj_id = conn_set_prop->connector_id,
+		.obj_type = DRM_MODE_OBJECT_CONNECTOR
+	};
+
+	/* It does all the locking and checking we need */
+	return drm_mode_obj_set_property_ioctl(dev, &obj_set_prop, file_priv);
+}
+
+static struct drm_encoder *drm_connector_get_encoder(struct drm_connector *connector)
+{
+	/* For atomic drivers only state objects are synchronously updated and
+	 * protected by modeset locks, so check those first. */
+	if (connector->state)
+		return connector->state->best_encoder;
+	return connector->encoder;
+}
+
+static bool drm_mode_expose_to_userspace(const struct drm_display_mode *mode,
+					 const struct drm_file *file_priv)
+{
+	/*
+	 * If user-space hasn't configured the driver to expose the stereo 3D
+	 * modes, don't expose them.
+	 */
+	if (!file_priv->stereo_allowed && drm_mode_is_stereo(mode))
+		return false;
+
+	return true;
+}
+
+int drm_mode_getconnector(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv)
+{
+	struct drm_mode_get_connector *out_resp = data;
+	struct drm_connector *connector;
+	struct drm_encoder *encoder;
+	struct drm_display_mode *mode;
+	int mode_count = 0;
+	int encoders_count = 0;
+	int ret = 0;
+	int copied = 0;
+	int i;
+	struct drm_mode_modeinfo u_mode;
+	struct drm_mode_modeinfo __user *mode_ptr;
+	uint32_t __user *encoder_ptr;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	memset(&u_mode, 0, sizeof(struct drm_mode_modeinfo));
+
+	mutex_lock(&dev->mode_config.mutex);
+
+	connector = drm_connector_lookup(dev, out_resp->connector_id);
+	if (!connector) {
+		ret = -ENOENT;
+		goto out_unlock;
+	}
+
+	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++)
+		if (connector->encoder_ids[i] != 0)
+			encoders_count++;
+
+	if (out_resp->count_modes == 0) {
+		connector->funcs->fill_modes(connector,
+					     dev->mode_config.max_width,
+					     dev->mode_config.max_height);
+	}
+
+	/* delayed so we get modes regardless of pre-fill_modes state */
+	list_for_each_entry(mode, &connector->modes, head)
+		if (drm_mode_expose_to_userspace(mode, file_priv))
+			mode_count++;
+
+	out_resp->connector_id = connector->base.id;
+	out_resp->connector_type = connector->connector_type;
+	out_resp->connector_type_id = connector->connector_type_id;
+	out_resp->mm_width = connector->display_info.width_mm;
+	out_resp->mm_height = connector->display_info.height_mm;
+	out_resp->subpixel = connector->display_info.subpixel_order;
+	out_resp->connection = connector->status;
+
+	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
+	encoder = drm_connector_get_encoder(connector);
+	if (encoder)
+		out_resp->encoder_id = encoder->base.id;
+	else
+		out_resp->encoder_id = 0;
+
+	/*
+	 * This ioctl is called twice, once to determine how much space is
+	 * needed, and the 2nd time to fill it.
+	 */
+	if ((out_resp->count_modes >= mode_count) && mode_count) {
+		copied = 0;
+		mode_ptr = (struct drm_mode_modeinfo __user *)(unsigned long)out_resp->modes_ptr;
+		list_for_each_entry(mode, &connector->modes, head) {
+			if (!drm_mode_expose_to_userspace(mode, file_priv))
+				continue;
+
+			drm_mode_convert_to_umode(&u_mode, mode);
+			if (copy_to_user(mode_ptr + copied,
+					 &u_mode, sizeof(u_mode))) {
+				ret = -EFAULT;
+				goto out;
+			}
+			copied++;
+		}
+	}
+	out_resp->count_modes = mode_count;
+
+	ret = drm_mode_object_get_properties(&connector->base, file_priv->atomic,
+			(uint32_t __user *)(unsigned long)(out_resp->props_ptr),
+			(uint64_t __user *)(unsigned long)(out_resp->prop_values_ptr),
+			&out_resp->count_props);
+	if (ret)
+		goto out;
+
+	if ((out_resp->count_encoders >= encoders_count) && encoders_count) {
+		copied = 0;
+		encoder_ptr = (uint32_t __user *)(unsigned long)(out_resp->encoders_ptr);
+		for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
+			if (connector->encoder_ids[i] != 0) {
+				if (put_user(connector->encoder_ids[i],
+					     encoder_ptr + copied)) {
+					ret = -EFAULT;
+					goto out;
+				}
+				copied++;
+			}
+		}
+	}
+	out_resp->count_encoders = encoders_count;
+
+out:
+	drm_modeset_unlock(&dev->mode_config.connection_mutex);
+
+	drm_connector_unreference(connector);
+out_unlock:
+	mutex_unlock(&dev->mode_config.mutex);
+
+	return ret;
+}
+
diff --git a/drivers/gpu/drm/drm_context.c b/drivers/gpu/drm/drm_context.c
index 192a5f9eeb74..3c4000facb36 100644
--- a/drivers/gpu/drm/drm_context.c
+++ b/drivers/gpu/drm/drm_context.c
@@ -54,7 +54,7 @@ struct drm_ctx_list {
 void drm_legacy_ctxbitmap_free(struct drm_device * dev, int ctx_handle)
 {
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return;
 
 	mutex_lock(&dev->struct_mutex);
@@ -92,7 +92,7 @@ static int drm_legacy_ctxbitmap_next(struct drm_device * dev)
 void drm_legacy_ctxbitmap_init(struct drm_device * dev)
 {
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return;
 
 	idr_init(&dev->ctx_idr);
@@ -109,7 +109,7 @@ void drm_legacy_ctxbitmap_init(struct drm_device * dev)
 void drm_legacy_ctxbitmap_cleanup(struct drm_device * dev)
 {
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return;
 
 	mutex_lock(&dev->struct_mutex);
@@ -131,7 +131,7 @@ void drm_legacy_ctxbitmap_flush(struct drm_device *dev, struct drm_file *file)
 	struct drm_ctx_list *pos, *tmp;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return;
 
 	mutex_lock(&dev->ctxlist_mutex);
@@ -177,7 +177,7 @@ int drm_legacy_getsareactx(struct drm_device *dev, void *data,
 	struct drm_map_list *_entry;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	mutex_lock(&dev->struct_mutex);
@@ -225,7 +225,7 @@ int drm_legacy_setsareactx(struct drm_device *dev, void *data,
 	struct drm_map_list *r_list = NULL;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	mutex_lock(&dev->struct_mutex);
@@ -329,7 +329,7 @@ int drm_legacy_resctx(struct drm_device *dev, void *data,
 	int i;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (res->count >= DRM_RESERVED_CONTEXTS) {
@@ -363,7 +363,7 @@ int drm_legacy_addctx(struct drm_device *dev, void *data,
 	struct drm_ctx *ctx = data;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	ctx->handle = drm_legacy_ctxbitmap_next(dev);
@@ -410,7 +410,7 @@ int drm_legacy_getctx(struct drm_device *dev, void *data,
 	struct drm_ctx *ctx = data;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	/* This is 0, because we don't handle any context flags */
@@ -436,7 +436,7 @@ int drm_legacy_switchctx(struct drm_device *dev, void *data,
 	struct drm_ctx *ctx = data;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	DRM_DEBUG("%d\n", ctx->handle);
@@ -460,7 +460,7 @@ int drm_legacy_newctx(struct drm_device *dev, void *data,
 	struct drm_ctx *ctx = data;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	DRM_DEBUG("%d\n", ctx->handle);
@@ -486,7 +486,7 @@ int drm_legacy_rmctx(struct drm_device *dev, void *data,
 	struct drm_ctx *ctx = data;
 
 	if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) &&
-	    drm_core_check_feature(dev, DRIVER_MODESET))
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	DRM_DEBUG("%d\n", ctx->handle);
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index f1d9f0569d7f..2d7bedf28647 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -40,39 +40,14 @@
 #include <drm/drm_modeset_lock.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_auth.h>
+#include <drm/drm_framebuffer.h>
 
 #include "drm_crtc_internal.h"
 #include "drm_internal.h"
 
-static struct drm_framebuffer *
-internal_framebuffer_create(struct drm_device *dev,
-			    const struct drm_mode_fb_cmd2 *r,
-			    struct drm_file *file_priv);
-
-/* Avoid boilerplate.  I'm tired of typing. */
-#define DRM_ENUM_NAME_FN(fnname, list)				\
-	const char *fnname(int val)				\
-	{							\
-		int i;						\
-		for (i = 0; i < ARRAY_SIZE(list); i++) {	\
-			if (list[i].type == val)		\
-				return list[i].name;		\
-		}						\
-		return "(unknown)";				\
-	}
-
 /*
  * Global properties
  */
-static const struct drm_prop_enum_list drm_dpms_enum_list[] = {
-	{ DRM_MODE_DPMS_ON, "On" },
-	{ DRM_MODE_DPMS_STANDBY, "Standby" },
-	{ DRM_MODE_DPMS_SUSPEND, "Suspend" },
-	{ DRM_MODE_DPMS_OFF, "Off" }
-};
-
-DRM_ENUM_NAME_FN(drm_get_dpms_name, drm_dpms_enum_list)
-
 static const struct drm_prop_enum_list drm_plane_type_enum_list[] = {
 	{ DRM_PLANE_TYPE_OVERLAY, "Overlay" },
 	{ DRM_PLANE_TYPE_PRIMARY, "Primary" },
@@ -82,320 +57,6 @@ static const struct drm_prop_enum_list drm_plane_type_enum_list[] = {
 /*
  * Optional properties
  */
-static const struct drm_prop_enum_list drm_scaling_mode_enum_list[] = {
-	{ DRM_MODE_SCALE_NONE, "None" },
-	{ DRM_MODE_SCALE_FULLSCREEN, "Full" },
-	{ DRM_MODE_SCALE_CENTER, "Center" },
-	{ DRM_MODE_SCALE_ASPECT, "Full aspect" },
-};
-
-static const struct drm_prop_enum_list drm_aspect_ratio_enum_list[] = {
-	{ DRM_MODE_PICTURE_ASPECT_NONE, "Automatic" },
-	{ DRM_MODE_PICTURE_ASPECT_4_3, "4:3" },
-	{ DRM_MODE_PICTURE_ASPECT_16_9, "16:9" },
-};
-
-/*
- * Non-global properties, but "required" for certain connectors.
- */
-static const struct drm_prop_enum_list drm_dvi_i_select_enum_list[] = {
-	{ DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
-	{ DRM_MODE_SUBCONNECTOR_DVID,      "DVI-D"     }, /* DVI-I  */
-	{ DRM_MODE_SUBCONNECTOR_DVIA,      "DVI-A"     }, /* DVI-I  */
-};
-
-DRM_ENUM_NAME_FN(drm_get_dvi_i_select_name, drm_dvi_i_select_enum_list)
-
-static const struct drm_prop_enum_list drm_dvi_i_subconnector_enum_list[] = {
-	{ DRM_MODE_SUBCONNECTOR_Unknown,   "Unknown"   }, /* DVI-I and TV-out */
-	{ DRM_MODE_SUBCONNECTOR_DVID,      "DVI-D"     }, /* DVI-I  */
-	{ DRM_MODE_SUBCONNECTOR_DVIA,      "DVI-A"     }, /* DVI-I  */
-};
-
-DRM_ENUM_NAME_FN(drm_get_dvi_i_subconnector_name,
-		 drm_dvi_i_subconnector_enum_list)
-
-static const struct drm_prop_enum_list drm_tv_select_enum_list[] = {
-	{ DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
-	{ DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
-	{ DRM_MODE_SUBCONNECTOR_SVIDEO,    "SVIDEO"    }, /* TV-out */
-	{ DRM_MODE_SUBCONNECTOR_Component, "Component" }, /* TV-out */
-	{ DRM_MODE_SUBCONNECTOR_SCART,     "SCART"     }, /* TV-out */
-};
-
-DRM_ENUM_NAME_FN(drm_get_tv_select_name, drm_tv_select_enum_list)
-
-static const struct drm_prop_enum_list drm_tv_subconnector_enum_list[] = {
-	{ DRM_MODE_SUBCONNECTOR_Unknown,   "Unknown"   }, /* DVI-I and TV-out */
-	{ DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
-	{ DRM_MODE_SUBCONNECTOR_SVIDEO,    "SVIDEO"    }, /* TV-out */
-	{ DRM_MODE_SUBCONNECTOR_Component, "Component" }, /* TV-out */
-	{ DRM_MODE_SUBCONNECTOR_SCART,     "SCART"     }, /* TV-out */
-};
-
-DRM_ENUM_NAME_FN(drm_get_tv_subconnector_name,
-		 drm_tv_subconnector_enum_list)
-
-static const struct drm_prop_enum_list drm_dirty_info_enum_list[] = {
-	{ DRM_MODE_DIRTY_OFF,      "Off"      },
-	{ DRM_MODE_DIRTY_ON,       "On"       },
-	{ DRM_MODE_DIRTY_ANNOTATE, "Annotate" },
-};
-
-struct drm_conn_prop_enum_list {
-	int type;
-	const char *name;
-	struct ida ida;
-};
-
-/*
- * Connector and encoder types.
- */
-static struct drm_conn_prop_enum_list drm_connector_enum_list[] = {
-	{ DRM_MODE_CONNECTOR_Unknown, "Unknown" },
-	{ DRM_MODE_CONNECTOR_VGA, "VGA" },
-	{ DRM_MODE_CONNECTOR_DVII, "DVI-I" },
-	{ DRM_MODE_CONNECTOR_DVID, "DVI-D" },
-	{ DRM_MODE_CONNECTOR_DVIA, "DVI-A" },
-	{ DRM_MODE_CONNECTOR_Composite, "Composite" },
-	{ DRM_MODE_CONNECTOR_SVIDEO, "SVIDEO" },
-	{ DRM_MODE_CONNECTOR_LVDS, "LVDS" },
-	{ DRM_MODE_CONNECTOR_Component, "Component" },
-	{ DRM_MODE_CONNECTOR_9PinDIN, "DIN" },
-	{ DRM_MODE_CONNECTOR_DisplayPort, "DP" },
-	{ DRM_MODE_CONNECTOR_HDMIA, "HDMI-A" },
-	{ DRM_MODE_CONNECTOR_HDMIB, "HDMI-B" },
-	{ DRM_MODE_CONNECTOR_TV, "TV" },
-	{ DRM_MODE_CONNECTOR_eDP, "eDP" },
-	{ DRM_MODE_CONNECTOR_VIRTUAL, "Virtual" },
-	{ DRM_MODE_CONNECTOR_DSI, "DSI" },
-	{ DRM_MODE_CONNECTOR_DPI, "DPI" },
-};
-
-static const struct drm_prop_enum_list drm_encoder_enum_list[] = {
-	{ DRM_MODE_ENCODER_NONE, "None" },
-	{ DRM_MODE_ENCODER_DAC, "DAC" },
-	{ DRM_MODE_ENCODER_TMDS, "TMDS" },
-	{ DRM_MODE_ENCODER_LVDS, "LVDS" },
-	{ DRM_MODE_ENCODER_TVDAC, "TV" },
-	{ DRM_MODE_ENCODER_VIRTUAL, "Virtual" },
-	{ DRM_MODE_ENCODER_DSI, "DSI" },
-	{ DRM_MODE_ENCODER_DPMST, "DP MST" },
-	{ DRM_MODE_ENCODER_DPI, "DPI" },
-};
-
-static const struct drm_prop_enum_list drm_subpixel_enum_list[] = {
-	{ SubPixelUnknown, "Unknown" },
-	{ SubPixelHorizontalRGB, "Horizontal RGB" },
-	{ SubPixelHorizontalBGR, "Horizontal BGR" },
-	{ SubPixelVerticalRGB, "Vertical RGB" },
-	{ SubPixelVerticalBGR, "Vertical BGR" },
-	{ SubPixelNone, "None" },
-};
-
-void drm_connector_ida_init(void)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(drm_connector_enum_list); i++)
-		ida_init(&drm_connector_enum_list[i].ida);
-}
-
-void drm_connector_ida_destroy(void)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(drm_connector_enum_list); i++)
-		ida_destroy(&drm_connector_enum_list[i].ida);
-}
-
-/**
- * drm_get_connector_status_name - return a string for connector status
- * @status: connector status to compute name of
- *
- * In contrast to the other drm_get_*_name functions this one here returns a
- * const pointer and hence is threadsafe.
- */
-const char *drm_get_connector_status_name(enum drm_connector_status status)
-{
-	if (status == connector_status_connected)
-		return "connected";
-	else if (status == connector_status_disconnected)
-		return "disconnected";
-	else
-		return "unknown";
-}
-EXPORT_SYMBOL(drm_get_connector_status_name);
-
-/**
- * drm_get_subpixel_order_name - return a string for a given subpixel enum
- * @order: enum of subpixel_order
- *
- * Note you could abuse this and return something out of bounds, but that
- * would be a caller error.  No unscrubbed user data should make it here.
- */
-const char *drm_get_subpixel_order_name(enum subpixel_order order)
-{
-	return drm_subpixel_enum_list[order].name;
-}
-EXPORT_SYMBOL(drm_get_subpixel_order_name);
-
-/*
- * Internal function to assign a slot in the object idr and optionally
- * register the object into the idr.
- */
-static int drm_mode_object_get_reg(struct drm_device *dev,
-				   struct drm_mode_object *obj,
-				   uint32_t obj_type,
-				   bool register_obj,
-				   void (*obj_free_cb)(struct kref *kref))
-{
-	int ret;
-
-	mutex_lock(&dev->mode_config.idr_mutex);
-	ret = idr_alloc(&dev->mode_config.crtc_idr, register_obj ? obj : NULL, 1, 0, GFP_KERNEL);
-	if (ret >= 0) {
-		/*
-		 * Set up the object linking under the protection of the idr
-		 * lock so that other users can't see inconsistent state.
-		 */
-		obj->id = ret;
-		obj->type = obj_type;
-		if (obj_free_cb) {
-			obj->free_cb = obj_free_cb;
-			kref_init(&obj->refcount);
-		}
-	}
-	mutex_unlock(&dev->mode_config.idr_mutex);
-
-	return ret < 0 ? ret : 0;
-}
-
-/**
- * drm_mode_object_get - allocate a new modeset identifier
- * @dev: DRM device
- * @obj: object pointer, used to generate unique ID
- * @obj_type: object type
- *
- * Create a unique identifier based on @ptr in @dev's identifier space.  Used
- * for tracking modes, CRTCs and connectors. Note that despite the _get postfix
- * modeset identifiers are _not_ reference counted. Hence don't use this for
- * reference counted modeset objects like framebuffers.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_mode_object_get(struct drm_device *dev,
-			struct drm_mode_object *obj, uint32_t obj_type)
-{
-	return drm_mode_object_get_reg(dev, obj, obj_type, true, NULL);
-}
-
-static void drm_mode_object_register(struct drm_device *dev,
-				     struct drm_mode_object *obj)
-{
-	mutex_lock(&dev->mode_config.idr_mutex);
-	idr_replace(&dev->mode_config.crtc_idr, obj, obj->id);
-	mutex_unlock(&dev->mode_config.idr_mutex);
-}
-
-/**
- * drm_mode_object_unregister - free a modeset identifer
- * @dev: DRM device
- * @object: object to free
- *
- * Free @id from @dev's unique identifier pool.
- * This function can be called multiple times, and guards against
- * multiple removals.
- * These modeset identifiers are _not_ reference counted. Hence don't use this
- * for reference counted modeset objects like framebuffers.
- */
-void drm_mode_object_unregister(struct drm_device *dev,
-			 struct drm_mode_object *object)
-{
-	mutex_lock(&dev->mode_config.idr_mutex);
-	if (object->id) {
-		idr_remove(&dev->mode_config.crtc_idr, object->id);
-		object->id = 0;
-	}
-	mutex_unlock(&dev->mode_config.idr_mutex);
-}
-
-static struct drm_mode_object *_object_find(struct drm_device *dev,
-		uint32_t id, uint32_t type)
-{
-	struct drm_mode_object *obj = NULL;
-
-	mutex_lock(&dev->mode_config.idr_mutex);
-	obj = idr_find(&dev->mode_config.crtc_idr, id);
-	if (obj && type != DRM_MODE_OBJECT_ANY && obj->type != type)
-		obj = NULL;
-	if (obj && obj->id != id)
-		obj = NULL;
-
-	if (obj && obj->free_cb) {
-		if (!kref_get_unless_zero(&obj->refcount))
-			obj = NULL;
-	}
-	mutex_unlock(&dev->mode_config.idr_mutex);
-
-	return obj;
-}
-
-/**
- * drm_mode_object_find - look up a drm object with static lifetime
- * @dev: drm device
- * @id: id of the mode object
- * @type: type of the mode object
- *
- * This function is used to look up a modeset object. It will acquire a
- * reference for reference counted objects. This reference must be dropped again
- * by callind drm_mode_object_unreference().
- */
-struct drm_mode_object *drm_mode_object_find(struct drm_device *dev,
-		uint32_t id, uint32_t type)
-{
-	struct drm_mode_object *obj = NULL;
-
-	obj = _object_find(dev, id, type);
-	return obj;
-}
-EXPORT_SYMBOL(drm_mode_object_find);
-
-/**
- * drm_mode_object_unreference - decr the object refcnt
- * @obj: mode_object
- *
- * This functions decrements the object's refcount if it is a refcounted modeset
- * object. It is a no-op on any other object. This is used to drop references
- * acquired with drm_mode_object_reference().
- */
-void drm_mode_object_unreference(struct drm_mode_object *obj)
-{
-	if (obj->free_cb) {
-		DRM_DEBUG("OBJ ID: %d (%d)\n", obj->id, atomic_read(&obj->refcount.refcount));
-		kref_put(&obj->refcount, obj->free_cb);
-	}
-}
-EXPORT_SYMBOL(drm_mode_object_unreference);
-
-/**
- * drm_mode_object_reference - incr the object refcnt
- * @obj: mode_object
- *
- * This functions increments the object's refcount if it is a refcounted modeset
- * object. It is a no-op on any other object. References should be dropped again
- * by calling drm_mode_object_unreference().
- */
-void drm_mode_object_reference(struct drm_mode_object *obj)
-{
-	if (obj->free_cb) {
-		DRM_DEBUG("OBJ ID: %d (%d)\n", obj->id, atomic_read(&obj->refcount.refcount));
-		kref_get(&obj->refcount);
-	}
-}
-EXPORT_SYMBOL(drm_mode_object_reference);
-
 /**
  * drm_crtc_force_disable - Forcibly turn off a CRTC
  * @crtc: CRTC to turn off
@@ -441,199 +102,6 @@ out:
 }
 EXPORT_SYMBOL(drm_crtc_force_disable_all);
 
-static void drm_framebuffer_free(struct kref *kref)
-{
-	struct drm_framebuffer *fb =
-			container_of(kref, struct drm_framebuffer, base.refcount);
-	struct drm_device *dev = fb->dev;
-
-	/*
-	 * The lookup idr holds a weak reference, which has not necessarily been
-	 * removed at this point. Check for that.
-	 */
-	drm_mode_object_unregister(dev, &fb->base);
-
-	fb->funcs->destroy(fb);
-}
-
-/**
- * drm_framebuffer_init - initialize a framebuffer
- * @dev: DRM device
- * @fb: framebuffer to be initialized
- * @funcs: ... with these functions
- *
- * Allocates an ID for the framebuffer's parent mode object, sets its mode
- * functions & device file and adds it to the master fd list.
- *
- * IMPORTANT:
- * This functions publishes the fb and makes it available for concurrent access
- * by other users. Which means by this point the fb _must_ be fully set up -
- * since all the fb attributes are invariant over its lifetime, no further
- * locking but only correct reference counting is required.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_framebuffer_init(struct drm_device *dev, struct drm_framebuffer *fb,
-			 const struct drm_framebuffer_funcs *funcs)
-{
-	int ret;
-
-	INIT_LIST_HEAD(&fb->filp_head);
-	fb->dev = dev;
-	fb->funcs = funcs;
-
-	ret = drm_mode_object_get_reg(dev, &fb->base, DRM_MODE_OBJECT_FB,
-				      false, drm_framebuffer_free);
-	if (ret)
-		goto out;
-
-	mutex_lock(&dev->mode_config.fb_lock);
-	dev->mode_config.num_fb++;
-	list_add(&fb->head, &dev->mode_config.fb_list);
-	mutex_unlock(&dev->mode_config.fb_lock);
-
-	drm_mode_object_register(dev, &fb->base);
-out:
-	return ret;
-}
-EXPORT_SYMBOL(drm_framebuffer_init);
-
-/**
- * drm_framebuffer_lookup - look up a drm framebuffer and grab a reference
- * @dev: drm device
- * @id: id of the fb object
- *
- * If successful, this grabs an additional reference to the framebuffer -
- * callers need to make sure to eventually unreference the returned framebuffer
- * again, using @drm_framebuffer_unreference.
- */
-struct drm_framebuffer *drm_framebuffer_lookup(struct drm_device *dev,
-					       uint32_t id)
-{
-	struct drm_mode_object *obj;
-	struct drm_framebuffer *fb = NULL;
-
-	obj = _object_find(dev, id, DRM_MODE_OBJECT_FB);
-	if (obj)
-		fb = obj_to_fb(obj);
-	return fb;
-}
-EXPORT_SYMBOL(drm_framebuffer_lookup);
-
-/**
- * drm_framebuffer_unregister_private - unregister a private fb from the lookup idr
- * @fb: fb to unregister
- *
- * Drivers need to call this when cleaning up driver-private framebuffers, e.g.
- * those used for fbdev. Note that the caller must hold a reference of it's own,
- * i.e. the object may not be destroyed through this call (since it'll lead to a
- * locking inversion).
- */
-void drm_framebuffer_unregister_private(struct drm_framebuffer *fb)
-{
-	struct drm_device *dev;
-
-	if (!fb)
-		return;
-
-	dev = fb->dev;
-
-	/* Mark fb as reaped and drop idr ref. */
-	drm_mode_object_unregister(dev, &fb->base);
-}
-EXPORT_SYMBOL(drm_framebuffer_unregister_private);
-
-/**
- * drm_framebuffer_cleanup - remove a framebuffer object
- * @fb: framebuffer to remove
- *
- * Cleanup framebuffer. This function is intended to be used from the drivers
- * ->destroy callback. It can also be used to clean up driver private
- * framebuffers embedded into a larger structure.
- *
- * Note that this function does not remove the fb from active usuage - if it is
- * still used anywhere, hilarity can ensue since userspace could call getfb on
- * the id and get back -EINVAL. Obviously no concern at driver unload time.
- *
- * Also, the framebuffer will not be removed from the lookup idr - for
- * user-created framebuffers this will happen in in the rmfb ioctl. For
- * driver-private objects (e.g. for fbdev) drivers need to explicitly call
- * drm_framebuffer_unregister_private.
- */
-void drm_framebuffer_cleanup(struct drm_framebuffer *fb)
-{
-	struct drm_device *dev = fb->dev;
-
-	mutex_lock(&dev->mode_config.fb_lock);
-	list_del(&fb->head);
-	dev->mode_config.num_fb--;
-	mutex_unlock(&dev->mode_config.fb_lock);
-}
-EXPORT_SYMBOL(drm_framebuffer_cleanup);
-
-/**
- * drm_framebuffer_remove - remove and unreference a framebuffer object
- * @fb: framebuffer to remove
- *
- * Scans all the CRTCs and planes in @dev's mode_config.  If they're
- * using @fb, removes it, setting it to NULL. Then drops the reference to the
- * passed-in framebuffer. Might take the modeset locks.
- *
- * Note that this function optimizes the cleanup away if the caller holds the
- * last reference to the framebuffer. It is also guaranteed to not take the
- * modeset locks in this case.
- */
-void drm_framebuffer_remove(struct drm_framebuffer *fb)
-{
-	struct drm_device *dev;
-	struct drm_crtc *crtc;
-	struct drm_plane *plane;
-
-	if (!fb)
-		return;
-
-	dev = fb->dev;
-
-	WARN_ON(!list_empty(&fb->filp_head));
-
-	/*
-	 * drm ABI mandates that we remove any deleted framebuffers from active
-	 * useage. But since most sane clients only remove framebuffers they no
-	 * longer need, try to optimize this away.
-	 *
-	 * Since we're holding a reference ourselves, observing a refcount of 1
-	 * means that we're the last holder and can skip it. Also, the refcount
-	 * can never increase from 1 again, so we don't need any barriers or
-	 * locks.
-	 *
-	 * Note that userspace could try to race with use and instate a new
-	 * usage _after_ we've cleared all current ones. End result will be an
-	 * in-use fb with fb-id == 0. Userspace is allowed to shoot its own foot
-	 * in this manner.
-	 */
-	if (drm_framebuffer_read_refcount(fb) > 1) {
-		drm_modeset_lock_all(dev);
-		/* remove from any CRTC */
-		drm_for_each_crtc(crtc, dev) {
-			if (crtc->primary->fb == fb) {
-				/* should turn off the crtc */
-				if (drm_crtc_force_disable(crtc))
-					DRM_ERROR("failed to reset crtc %p when fb was deleted\n", crtc);
-			}
-		}
-
-		drm_for_each_plane(plane, dev) {
-			if (plane->fb == fb)
-				drm_plane_force_disable(plane);
-		}
-		drm_modeset_unlock_all(dev);
-	}
-
-	drm_framebuffer_unreference(fb);
-}
-EXPORT_SYMBOL(drm_framebuffer_remove);
-
 DEFINE_WW_CLASS(crtc_ww_class);
 
 static unsigned int drm_num_crtcs(struct drm_device *dev)
@@ -683,7 +151,11 @@ static void drm_crtc_unregister_all(struct drm_device *dev)
  * @funcs: callbacks for the new CRTC
  * @name: printf style format string for the CRTC name, or NULL for default name
  *
- * Inits a new object created as base part of a driver crtc object.
+ * Inits a new object created as base part of a driver crtc object. Drivers
+ * should use this function instead of drm_crtc_init(), which is only provided
+ * for backwards compatibility with drivers which do not yet support universal
+ * planes). For really simple hardware which has only 1 plane look at
+ * drm_simple_display_pipe_init() instead.
  *
  * Returns:
  * Zero on success, error code on failure.
@@ -783,722 +255,6 @@ void drm_crtc_cleanup(struct drm_crtc *crtc)
 }
 EXPORT_SYMBOL(drm_crtc_cleanup);
 
-/*
- * drm_mode_remove - remove and free a mode
- * @connector: connector list to modify
- * @mode: mode to remove
- *
- * Remove @mode from @connector's mode list, then free it.
- */
-static void drm_mode_remove(struct drm_connector *connector,
-			    struct drm_display_mode *mode)
-{
-	list_del(&mode->head);
-	drm_mode_destroy(connector->dev, mode);
-}
-
-/**
- * drm_display_info_set_bus_formats - set the supported bus formats
- * @info: display info to store bus formats in
- * @formats: array containing the supported bus formats
- * @num_formats: the number of entries in the fmts array
- *
- * Store the supported bus formats in display info structure.
- * See MEDIA_BUS_FMT_* definitions in include/uapi/linux/media-bus-format.h for
- * a full list of available formats.
- */
-int drm_display_info_set_bus_formats(struct drm_display_info *info,
-				     const u32 *formats,
-				     unsigned int num_formats)
-{
-	u32 *fmts = NULL;
-
-	if (!formats && num_formats)
-		return -EINVAL;
-
-	if (formats && num_formats) {
-		fmts = kmemdup(formats, sizeof(*formats) * num_formats,
-			       GFP_KERNEL);
-		if (!fmts)
-			return -ENOMEM;
-	}
-
-	kfree(info->bus_formats);
-	info->bus_formats = fmts;
-	info->num_bus_formats = num_formats;
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_display_info_set_bus_formats);
-
-/**
- * drm_connector_get_cmdline_mode - reads the user's cmdline mode
- * @connector: connector to quwery
- *
- * The kernel supports per-connector configration of its consoles through
- * use of the video= parameter. This function parses that option and
- * extracts the user's specified mode (or enable/disable status) for a
- * particular connector. This is typically only used during the early fbdev
- * setup.
- */
-static void drm_connector_get_cmdline_mode(struct drm_connector *connector)
-{
-	struct drm_cmdline_mode *mode = &connector->cmdline_mode;
-	char *option = NULL;
-
-	if (fb_get_options(connector->name, &option))
-		return;
-
-	if (!drm_mode_parse_command_line_for_connector(option,
-						       connector,
-						       mode))
-		return;
-
-	if (mode->force) {
-		const char *s;
-
-		switch (mode->force) {
-		case DRM_FORCE_OFF:
-			s = "OFF";
-			break;
-		case DRM_FORCE_ON_DIGITAL:
-			s = "ON - dig";
-			break;
-		default:
-		case DRM_FORCE_ON:
-			s = "ON";
-			break;
-		}
-
-		DRM_INFO("forcing %s connector %s\n", connector->name, s);
-		connector->force = mode->force;
-	}
-
-	DRM_DEBUG_KMS("cmdline mode for connector %s %dx%d@%dHz%s%s%s\n",
-		      connector->name,
-		      mode->xres, mode->yres,
-		      mode->refresh_specified ? mode->refresh : 60,
-		      mode->rb ? " reduced blanking" : "",
-		      mode->margins ? " with margins" : "",
-		      mode->interlace ?  " interlaced" : "");
-}
-
-static void drm_connector_free(struct kref *kref)
-{
-	struct drm_connector *connector =
-		container_of(kref, struct drm_connector, base.refcount);
-	struct drm_device *dev = connector->dev;
-
-	drm_mode_object_unregister(dev, &connector->base);
-	connector->funcs->destroy(connector);
-}
-
-/**
- * drm_connector_init - Init a preallocated connector
- * @dev: DRM device
- * @connector: the connector to init
- * @funcs: callbacks for this connector
- * @connector_type: user visible type of the connector
- *
- * Initialises a preallocated connector. Connectors should be
- * subclassed as part of driver connector objects.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_connector_init(struct drm_device *dev,
-		       struct drm_connector *connector,
-		       const struct drm_connector_funcs *funcs,
-		       int connector_type)
-{
-	struct drm_mode_config *config = &dev->mode_config;
-	int ret;
-	struct ida *connector_ida =
-		&drm_connector_enum_list[connector_type].ida;
-
-	drm_modeset_lock_all(dev);
-
-	ret = drm_mode_object_get_reg(dev, &connector->base,
-				      DRM_MODE_OBJECT_CONNECTOR,
-				      false, drm_connector_free);
-	if (ret)
-		goto out_unlock;
-
-	connector->base.properties = &connector->properties;
-	connector->dev = dev;
-	connector->funcs = funcs;
-
-	ret = ida_simple_get(&config->connector_ida, 0, 0, GFP_KERNEL);
-	if (ret < 0)
-		goto out_put;
-	connector->index = ret;
-	ret = 0;
-
-	connector->connector_type = connector_type;
-	connector->connector_type_id =
-		ida_simple_get(connector_ida, 1, 0, GFP_KERNEL);
-	if (connector->connector_type_id < 0) {
-		ret = connector->connector_type_id;
-		goto out_put_id;
-	}
-	connector->name =
-		kasprintf(GFP_KERNEL, "%s-%d",
-			  drm_connector_enum_list[connector_type].name,
-			  connector->connector_type_id);
-	if (!connector->name) {
-		ret = -ENOMEM;
-		goto out_put_type_id;
-	}
-
-	INIT_LIST_HEAD(&connector->probed_modes);
-	INIT_LIST_HEAD(&connector->modes);
-	connector->edid_blob_ptr = NULL;
-	connector->status = connector_status_unknown;
-
-	drm_connector_get_cmdline_mode(connector);
-
-	/* We should add connectors at the end to avoid upsetting the connector
-	 * index too much. */
-	list_add_tail(&connector->head, &config->connector_list);
-	config->num_connector++;
-
-	if (connector_type != DRM_MODE_CONNECTOR_VIRTUAL)
-		drm_object_attach_property(&connector->base,
-					      config->edid_property,
-					      0);
-
-	drm_object_attach_property(&connector->base,
-				      config->dpms_property, 0);
-
-	if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
-		drm_object_attach_property(&connector->base, config->prop_crtc_id, 0);
-	}
-
-	connector->debugfs_entry = NULL;
-out_put_type_id:
-	if (ret)
-		ida_remove(connector_ida, connector->connector_type_id);
-out_put_id:
-	if (ret)
-		ida_remove(&config->connector_ida, connector->index);
-out_put:
-	if (ret)
-		drm_mode_object_unregister(dev, &connector->base);
-
-out_unlock:
-	drm_modeset_unlock_all(dev);
-
-	return ret;
-}
-EXPORT_SYMBOL(drm_connector_init);
-
-/**
- * drm_connector_cleanup - cleans up an initialised connector
- * @connector: connector to cleanup
- *
- * Cleans up the connector but doesn't free the object.
- */
-void drm_connector_cleanup(struct drm_connector *connector)
-{
-	struct drm_device *dev = connector->dev;
-	struct drm_display_mode *mode, *t;
-
-	/* The connector should have been removed from userspace long before
-	 * it is finally destroyed.
-	 */
-	if (WARN_ON(connector->registered))
-		drm_connector_unregister(connector);
-
-	if (connector->tile_group) {
-		drm_mode_put_tile_group(dev, connector->tile_group);
-		connector->tile_group = NULL;
-	}
-
-	list_for_each_entry_safe(mode, t, &connector->probed_modes, head)
-		drm_mode_remove(connector, mode);
-
-	list_for_each_entry_safe(mode, t, &connector->modes, head)
-		drm_mode_remove(connector, mode);
-
-	ida_remove(&drm_connector_enum_list[connector->connector_type].ida,
-		   connector->connector_type_id);
-
-	ida_remove(&dev->mode_config.connector_ida,
-		   connector->index);
-
-	kfree(connector->display_info.bus_formats);
-	drm_mode_object_unregister(dev, &connector->base);
-	kfree(connector->name);
-	connector->name = NULL;
-	list_del(&connector->head);
-	dev->mode_config.num_connector--;
-
-	WARN_ON(connector->state && !connector->funcs->atomic_destroy_state);
-	if (connector->state && connector->funcs->atomic_destroy_state)
-		connector->funcs->atomic_destroy_state(connector,
-						       connector->state);
-
-	memset(connector, 0, sizeof(*connector));
-}
-EXPORT_SYMBOL(drm_connector_cleanup);
-
-/**
- * drm_connector_register - register a connector
- * @connector: the connector to register
- *
- * Register userspace interfaces for a connector
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_connector_register(struct drm_connector *connector)
-{
-	int ret;
-
-	if (connector->registered)
-		return 0;
-
-	ret = drm_sysfs_connector_add(connector);
-	if (ret)
-		return ret;
-
-	ret = drm_debugfs_connector_add(connector);
-	if (ret) {
-		goto err_sysfs;
-	}
-
-	if (connector->funcs->late_register) {
-		ret = connector->funcs->late_register(connector);
-		if (ret)
-			goto err_debugfs;
-	}
-
-	drm_mode_object_register(connector->dev, &connector->base);
-
-	connector->registered = true;
-	return 0;
-
-err_debugfs:
-	drm_debugfs_connector_remove(connector);
-err_sysfs:
-	drm_sysfs_connector_remove(connector);
-	return ret;
-}
-EXPORT_SYMBOL(drm_connector_register);
-
-/**
- * drm_connector_unregister - unregister a connector
- * @connector: the connector to unregister
- *
- * Unregister userspace interfaces for a connector
- */
-void drm_connector_unregister(struct drm_connector *connector)
-{
-	if (!connector->registered)
-		return;
-
-	if (connector->funcs->early_unregister)
-		connector->funcs->early_unregister(connector);
-
-	drm_sysfs_connector_remove(connector);
-	drm_debugfs_connector_remove(connector);
-
-	connector->registered = false;
-}
-EXPORT_SYMBOL(drm_connector_unregister);
-
-static void drm_connector_unregister_all(struct drm_device *dev)
-{
-	struct drm_connector *connector;
-
-	/* FIXME: taking the mode config mutex ends up in a clash with sysfs */
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head)
-		drm_connector_unregister(connector);
-}
-
-static int drm_connector_register_all(struct drm_device *dev)
-{
-	struct drm_connector *connector;
-	int ret;
-
-	mutex_lock(&dev->mode_config.mutex);
-
-	drm_for_each_connector(connector, dev) {
-		ret = drm_connector_register(connector);
-		if (ret)
-			goto err;
-	}
-
-	mutex_unlock(&dev->mode_config.mutex);
-
-	return 0;
-
-err:
-	mutex_unlock(&dev->mode_config.mutex);
-	drm_connector_unregister_all(dev);
-	return ret;
-}
-
-static int drm_encoder_register_all(struct drm_device *dev)
-{
-	struct drm_encoder *encoder;
-	int ret = 0;
-
-	drm_for_each_encoder(encoder, dev) {
-		if (encoder->funcs->late_register)
-			ret = encoder->funcs->late_register(encoder);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static void drm_encoder_unregister_all(struct drm_device *dev)
-{
-	struct drm_encoder *encoder;
-
-	drm_for_each_encoder(encoder, dev) {
-		if (encoder->funcs->early_unregister)
-			encoder->funcs->early_unregister(encoder);
-	}
-}
-
-/**
- * drm_encoder_init - Init a preallocated encoder
- * @dev: drm device
- * @encoder: the encoder to init
- * @funcs: callbacks for this encoder
- * @encoder_type: user visible type of the encoder
- * @name: printf style format string for the encoder name, or NULL for default name
- *
- * Initialises a preallocated encoder. Encoder should be
- * subclassed as part of driver encoder objects.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_encoder_init(struct drm_device *dev,
-		      struct drm_encoder *encoder,
-		      const struct drm_encoder_funcs *funcs,
-		      int encoder_type, const char *name, ...)
-{
-	int ret;
-
-	drm_modeset_lock_all(dev);
-
-	ret = drm_mode_object_get(dev, &encoder->base, DRM_MODE_OBJECT_ENCODER);
-	if (ret)
-		goto out_unlock;
-
-	encoder->dev = dev;
-	encoder->encoder_type = encoder_type;
-	encoder->funcs = funcs;
-	if (name) {
-		va_list ap;
-
-		va_start(ap, name);
-		encoder->name = kvasprintf(GFP_KERNEL, name, ap);
-		va_end(ap);
-	} else {
-		encoder->name = kasprintf(GFP_KERNEL, "%s-%d",
-					  drm_encoder_enum_list[encoder_type].name,
-					  encoder->base.id);
-	}
-	if (!encoder->name) {
-		ret = -ENOMEM;
-		goto out_put;
-	}
-
-	list_add_tail(&encoder->head, &dev->mode_config.encoder_list);
-	encoder->index = dev->mode_config.num_encoder++;
-
-out_put:
-	if (ret)
-		drm_mode_object_unregister(dev, &encoder->base);
-
-out_unlock:
-	drm_modeset_unlock_all(dev);
-
-	return ret;
-}
-EXPORT_SYMBOL(drm_encoder_init);
-
-/**
- * drm_encoder_cleanup - cleans up an initialised encoder
- * @encoder: encoder to cleanup
- *
- * Cleans up the encoder but doesn't free the object.
- */
-void drm_encoder_cleanup(struct drm_encoder *encoder)
-{
-	struct drm_device *dev = encoder->dev;
-
-	/* Note that the encoder_list is considered to be static; should we
-	 * remove the drm_encoder at runtime we would have to decrement all
-	 * the indices on the drm_encoder after us in the encoder_list.
-	 */
-
-	drm_modeset_lock_all(dev);
-	drm_mode_object_unregister(dev, &encoder->base);
-	kfree(encoder->name);
-	list_del(&encoder->head);
-	dev->mode_config.num_encoder--;
-	drm_modeset_unlock_all(dev);
-
-	memset(encoder, 0, sizeof(*encoder));
-}
-EXPORT_SYMBOL(drm_encoder_cleanup);
-
-static unsigned int drm_num_planes(struct drm_device *dev)
-{
-	unsigned int num = 0;
-	struct drm_plane *tmp;
-
-	drm_for_each_plane(tmp, dev) {
-		num++;
-	}
-
-	return num;
-}
-
-/**
- * drm_universal_plane_init - Initialize a new universal plane object
- * @dev: DRM device
- * @plane: plane object to init
- * @possible_crtcs: bitmask of possible CRTCs
- * @funcs: callbacks for the new plane
- * @formats: array of supported formats (%DRM_FORMAT_*)
- * @format_count: number of elements in @formats
- * @type: type of plane (overlay, primary, cursor)
- * @name: printf style format string for the plane name, or NULL for default name
- *
- * Initializes a plane object of type @type.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
-			     unsigned long possible_crtcs,
-			     const struct drm_plane_funcs *funcs,
-			     const uint32_t *formats, unsigned int format_count,
-			     enum drm_plane_type type,
-			     const char *name, ...)
-{
-	struct drm_mode_config *config = &dev->mode_config;
-	int ret;
-
-	ret = drm_mode_object_get(dev, &plane->base, DRM_MODE_OBJECT_PLANE);
-	if (ret)
-		return ret;
-
-	drm_modeset_lock_init(&plane->mutex);
-
-	plane->base.properties = &plane->properties;
-	plane->dev = dev;
-	plane->funcs = funcs;
-	plane->format_types = kmalloc_array(format_count, sizeof(uint32_t),
-					    GFP_KERNEL);
-	if (!plane->format_types) {
-		DRM_DEBUG_KMS("out of memory when allocating plane\n");
-		drm_mode_object_unregister(dev, &plane->base);
-		return -ENOMEM;
-	}
-
-	if (name) {
-		va_list ap;
-
-		va_start(ap, name);
-		plane->name = kvasprintf(GFP_KERNEL, name, ap);
-		va_end(ap);
-	} else {
-		plane->name = kasprintf(GFP_KERNEL, "plane-%d",
-					drm_num_planes(dev));
-	}
-	if (!plane->name) {
-		kfree(plane->format_types);
-		drm_mode_object_unregister(dev, &plane->base);
-		return -ENOMEM;
-	}
-
-	memcpy(plane->format_types, formats, format_count * sizeof(uint32_t));
-	plane->format_count = format_count;
-	plane->possible_crtcs = possible_crtcs;
-	plane->type = type;
-
-	list_add_tail(&plane->head, &config->plane_list);
-	plane->index = config->num_total_plane++;
-	if (plane->type == DRM_PLANE_TYPE_OVERLAY)
-		config->num_overlay_plane++;
-
-	drm_object_attach_property(&plane->base,
-				   config->plane_type_property,
-				   plane->type);
-
-	if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
-		drm_object_attach_property(&plane->base, config->prop_fb_id, 0);
-		drm_object_attach_property(&plane->base, config->prop_crtc_id, 0);
-		drm_object_attach_property(&plane->base, config->prop_crtc_x, 0);
-		drm_object_attach_property(&plane->base, config->prop_crtc_y, 0);
-		drm_object_attach_property(&plane->base, config->prop_crtc_w, 0);
-		drm_object_attach_property(&plane->base, config->prop_crtc_h, 0);
-		drm_object_attach_property(&plane->base, config->prop_src_x, 0);
-		drm_object_attach_property(&plane->base, config->prop_src_y, 0);
-		drm_object_attach_property(&plane->base, config->prop_src_w, 0);
-		drm_object_attach_property(&plane->base, config->prop_src_h, 0);
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_universal_plane_init);
-
-static int drm_plane_register_all(struct drm_device *dev)
-{
-	struct drm_plane *plane;
-	int ret = 0;
-
-	drm_for_each_plane(plane, dev) {
-		if (plane->funcs->late_register)
-			ret = plane->funcs->late_register(plane);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static void drm_plane_unregister_all(struct drm_device *dev)
-{
-	struct drm_plane *plane;
-
-	drm_for_each_plane(plane, dev) {
-		if (plane->funcs->early_unregister)
-			plane->funcs->early_unregister(plane);
-	}
-}
-
-/**
- * drm_plane_init - Initialize a legacy plane
- * @dev: DRM device
- * @plane: plane object to init
- * @possible_crtcs: bitmask of possible CRTCs
- * @funcs: callbacks for the new plane
- * @formats: array of supported formats (%DRM_FORMAT_*)
- * @format_count: number of elements in @formats
- * @is_primary: plane type (primary vs overlay)
- *
- * Legacy API to initialize a DRM plane.
- *
- * New drivers should call drm_universal_plane_init() instead.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
-		   unsigned long possible_crtcs,
-		   const struct drm_plane_funcs *funcs,
-		   const uint32_t *formats, unsigned int format_count,
-		   bool is_primary)
-{
-	enum drm_plane_type type;
-
-	type = is_primary ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
-	return drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
-					formats, format_count, type, NULL);
-}
-EXPORT_SYMBOL(drm_plane_init);
-
-/**
- * drm_plane_cleanup - Clean up the core plane usage
- * @plane: plane to cleanup
- *
- * This function cleans up @plane and removes it from the DRM mode setting
- * core. Note that the function does *not* free the plane structure itself,
- * this is the responsibility of the caller.
- */
-void drm_plane_cleanup(struct drm_plane *plane)
-{
-	struct drm_device *dev = plane->dev;
-
-	drm_modeset_lock_all(dev);
-	kfree(plane->format_types);
-	drm_mode_object_unregister(dev, &plane->base);
-
-	BUG_ON(list_empty(&plane->head));
-
-	/* Note that the plane_list is considered to be static; should we
-	 * remove the drm_plane at runtime we would have to decrement all
-	 * the indices on the drm_plane after us in the plane_list.
-	 */
-
-	list_del(&plane->head);
-	dev->mode_config.num_total_plane--;
-	if (plane->type == DRM_PLANE_TYPE_OVERLAY)
-		dev->mode_config.num_overlay_plane--;
-	drm_modeset_unlock_all(dev);
-
-	WARN_ON(plane->state && !plane->funcs->atomic_destroy_state);
-	if (plane->state && plane->funcs->atomic_destroy_state)
-		plane->funcs->atomic_destroy_state(plane, plane->state);
-
-	kfree(plane->name);
-
-	memset(plane, 0, sizeof(*plane));
-}
-EXPORT_SYMBOL(drm_plane_cleanup);
-
-/**
- * drm_plane_from_index - find the registered plane at an index
- * @dev: DRM device
- * @idx: index of registered plane to find for
- *
- * Given a plane index, return the registered plane from DRM device's
- * list of planes with matching index.
- */
-struct drm_plane *
-drm_plane_from_index(struct drm_device *dev, int idx)
-{
-	struct drm_plane *plane;
-
-	drm_for_each_plane(plane, dev)
-		if (idx == plane->index)
-			return plane;
-
-	return NULL;
-}
-EXPORT_SYMBOL(drm_plane_from_index);
-
-/**
- * drm_plane_force_disable - Forcibly disable a plane
- * @plane: plane to disable
- *
- * Forces the plane to be disabled.
- *
- * Used when the plane's current framebuffer is destroyed,
- * and when restoring fbdev mode.
- */
-void drm_plane_force_disable(struct drm_plane *plane)
-{
-	int ret;
-
-	if (!plane->fb)
-		return;
-
-	plane->old_fb = plane->fb;
-	ret = plane->funcs->disable_plane(plane);
-	if (ret) {
-		DRM_ERROR("failed to disable plane with busy fb\n");
-		plane->old_fb = NULL;
-		return;
-	}
-	/* disconnect the plane from the fb and crtc: */
-	drm_framebuffer_unreference(plane->old_fb);
-	plane->old_fb = NULL;
-	plane->fb = NULL;
-	plane->crtc = NULL;
-}
-EXPORT_SYMBOL(drm_plane_force_disable);
-
 int drm_modeset_register_all(struct drm_device *dev)
 {
 	int ret;
@@ -1542,39 +298,11 @@ void drm_modeset_unregister_all(struct drm_device *dev)
 static int drm_mode_create_standard_properties(struct drm_device *dev)
 {
 	struct drm_property *prop;
+	int ret;
 
-	/*
-	 * Standard properties (apply to all connectors)
-	 */
-	prop = drm_property_create(dev, DRM_MODE_PROP_BLOB |
-				   DRM_MODE_PROP_IMMUTABLE,
-				   "EDID", 0);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.edid_property = prop;
-
-	prop = drm_property_create_enum(dev, 0,
-				   "DPMS", drm_dpms_enum_list,
-				   ARRAY_SIZE(drm_dpms_enum_list));
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.dpms_property = prop;
-
-	prop = drm_property_create(dev,
-				   DRM_MODE_PROP_BLOB |
-				   DRM_MODE_PROP_IMMUTABLE,
-				   "PATH", 0);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.path_property = prop;
-
-	prop = drm_property_create(dev,
-				   DRM_MODE_PROP_BLOB |
-				   DRM_MODE_PROP_IMMUTABLE,
-				   "TILE", 0);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.tile_property = prop;
+	ret = drm_connector_create_standard_properties(dev);
+	if (ret)
+		return ret;
 
 	prop = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
 					"type", drm_plane_type_enum_list,
@@ -1695,250 +423,6 @@ static int drm_mode_create_standard_properties(struct drm_device *dev)
 }
 
 /**
- * drm_mode_create_dvi_i_properties - create DVI-I specific connector properties
- * @dev: DRM device
- *
- * Called by a driver the first time a DVI-I connector is made.
- */
-int drm_mode_create_dvi_i_properties(struct drm_device *dev)
-{
-	struct drm_property *dvi_i_selector;
-	struct drm_property *dvi_i_subconnector;
-
-	if (dev->mode_config.dvi_i_select_subconnector_property)
-		return 0;
-
-	dvi_i_selector =
-		drm_property_create_enum(dev, 0,
-				    "select subconnector",
-				    drm_dvi_i_select_enum_list,
-				    ARRAY_SIZE(drm_dvi_i_select_enum_list));
-	dev->mode_config.dvi_i_select_subconnector_property = dvi_i_selector;
-
-	dvi_i_subconnector = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
-				    "subconnector",
-				    drm_dvi_i_subconnector_enum_list,
-				    ARRAY_SIZE(drm_dvi_i_subconnector_enum_list));
-	dev->mode_config.dvi_i_subconnector_property = dvi_i_subconnector;
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_mode_create_dvi_i_properties);
-
-/**
- * drm_create_tv_properties - create TV specific connector properties
- * @dev: DRM device
- * @num_modes: number of different TV formats (modes) supported
- * @modes: array of pointers to strings containing name of each format
- *
- * Called by a driver's TV initialization routine, this function creates
- * the TV specific connector properties for a given device.  Caller is
- * responsible for allocating a list of format names and passing them to
- * this routine.
- */
-int drm_mode_create_tv_properties(struct drm_device *dev,
-				  unsigned int num_modes,
-				  const char * const modes[])
-{
-	struct drm_property *tv_selector;
-	struct drm_property *tv_subconnector;
-	unsigned int i;
-
-	if (dev->mode_config.tv_select_subconnector_property)
-		return 0;
-
-	/*
-	 * Basic connector properties
-	 */
-	tv_selector = drm_property_create_enum(dev, 0,
-					  "select subconnector",
-					  drm_tv_select_enum_list,
-					  ARRAY_SIZE(drm_tv_select_enum_list));
-	if (!tv_selector)
-		goto nomem;
-
-	dev->mode_config.tv_select_subconnector_property = tv_selector;
-
-	tv_subconnector =
-		drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
-				    "subconnector",
-				    drm_tv_subconnector_enum_list,
-				    ARRAY_SIZE(drm_tv_subconnector_enum_list));
-	if (!tv_subconnector)
-		goto nomem;
-	dev->mode_config.tv_subconnector_property = tv_subconnector;
-
-	/*
-	 * Other, TV specific properties: margins & TV modes.
-	 */
-	dev->mode_config.tv_left_margin_property =
-		drm_property_create_range(dev, 0, "left margin", 0, 100);
-	if (!dev->mode_config.tv_left_margin_property)
-		goto nomem;
-
-	dev->mode_config.tv_right_margin_property =
-		drm_property_create_range(dev, 0, "right margin", 0, 100);
-	if (!dev->mode_config.tv_right_margin_property)
-		goto nomem;
-
-	dev->mode_config.tv_top_margin_property =
-		drm_property_create_range(dev, 0, "top margin", 0, 100);
-	if (!dev->mode_config.tv_top_margin_property)
-		goto nomem;
-
-	dev->mode_config.tv_bottom_margin_property =
-		drm_property_create_range(dev, 0, "bottom margin", 0, 100);
-	if (!dev->mode_config.tv_bottom_margin_property)
-		goto nomem;
-
-	dev->mode_config.tv_mode_property =
-		drm_property_create(dev, DRM_MODE_PROP_ENUM,
-				    "mode", num_modes);
-	if (!dev->mode_config.tv_mode_property)
-		goto nomem;
-
-	for (i = 0; i < num_modes; i++)
-		drm_property_add_enum(dev->mode_config.tv_mode_property, i,
-				      i, modes[i]);
-
-	dev->mode_config.tv_brightness_property =
-		drm_property_create_range(dev, 0, "brightness", 0, 100);
-	if (!dev->mode_config.tv_brightness_property)
-		goto nomem;
-
-	dev->mode_config.tv_contrast_property =
-		drm_property_create_range(dev, 0, "contrast", 0, 100);
-	if (!dev->mode_config.tv_contrast_property)
-		goto nomem;
-
-	dev->mode_config.tv_flicker_reduction_property =
-		drm_property_create_range(dev, 0, "flicker reduction", 0, 100);
-	if (!dev->mode_config.tv_flicker_reduction_property)
-		goto nomem;
-
-	dev->mode_config.tv_overscan_property =
-		drm_property_create_range(dev, 0, "overscan", 0, 100);
-	if (!dev->mode_config.tv_overscan_property)
-		goto nomem;
-
-	dev->mode_config.tv_saturation_property =
-		drm_property_create_range(dev, 0, "saturation", 0, 100);
-	if (!dev->mode_config.tv_saturation_property)
-		goto nomem;
-
-	dev->mode_config.tv_hue_property =
-		drm_property_create_range(dev, 0, "hue", 0, 100);
-	if (!dev->mode_config.tv_hue_property)
-		goto nomem;
-
-	return 0;
-nomem:
-	return -ENOMEM;
-}
-EXPORT_SYMBOL(drm_mode_create_tv_properties);
-
-/**
- * drm_mode_create_scaling_mode_property - create scaling mode property
- * @dev: DRM device
- *
- * Called by a driver the first time it's needed, must be attached to desired
- * connectors.
- */
-int drm_mode_create_scaling_mode_property(struct drm_device *dev)
-{
-	struct drm_property *scaling_mode;
-
-	if (dev->mode_config.scaling_mode_property)
-		return 0;
-
-	scaling_mode =
-		drm_property_create_enum(dev, 0, "scaling mode",
-				drm_scaling_mode_enum_list,
-				    ARRAY_SIZE(drm_scaling_mode_enum_list));
-
-	dev->mode_config.scaling_mode_property = scaling_mode;
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_mode_create_scaling_mode_property);
-
-/**
- * drm_mode_create_aspect_ratio_property - create aspect ratio property
- * @dev: DRM device
- *
- * Called by a driver the first time it's needed, must be attached to desired
- * connectors.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_create_aspect_ratio_property(struct drm_device *dev)
-{
-	if (dev->mode_config.aspect_ratio_property)
-		return 0;
-
-	dev->mode_config.aspect_ratio_property =
-		drm_property_create_enum(dev, 0, "aspect ratio",
-				drm_aspect_ratio_enum_list,
-				ARRAY_SIZE(drm_aspect_ratio_enum_list));
-
-	if (dev->mode_config.aspect_ratio_property == NULL)
-		return -ENOMEM;
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_mode_create_aspect_ratio_property);
-
-/**
- * drm_mode_create_dirty_property - create dirty property
- * @dev: DRM device
- *
- * Called by a driver the first time it's needed, must be attached to desired
- * connectors.
- */
-int drm_mode_create_dirty_info_property(struct drm_device *dev)
-{
-	struct drm_property *dirty_info;
-
-	if (dev->mode_config.dirty_info_property)
-		return 0;
-
-	dirty_info =
-		drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
-				    "dirty",
-				    drm_dirty_info_enum_list,
-				    ARRAY_SIZE(drm_dirty_info_enum_list));
-	dev->mode_config.dirty_info_property = dirty_info;
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_mode_create_dirty_info_property);
-
-/**
- * drm_mode_create_suggested_offset_properties - create suggests offset properties
- * @dev: DRM device
- *
- * Create the the suggested x/y offset property for connectors.
- */
-int drm_mode_create_suggested_offset_properties(struct drm_device *dev)
-{
-	if (dev->mode_config.suggested_x_property && dev->mode_config.suggested_y_property)
-		return 0;
-
-	dev->mode_config.suggested_x_property =
-		drm_property_create_range(dev, DRM_MODE_PROP_IMMUTABLE, "suggested X", 0, 0xffffffff);
-
-	dev->mode_config.suggested_y_property =
-		drm_property_create_range(dev, DRM_MODE_PROP_IMMUTABLE, "suggested Y", 0, 0xffffffff);
-
-	if (dev->mode_config.suggested_x_property == NULL ||
-	    dev->mode_config.suggested_y_property == NULL)
-		return -ENOMEM;
-	return 0;
-}
-EXPORT_SYMBOL(drm_mode_create_suggested_offset_properties);
-
-/**
  * drm_mode_getresources - get graphics configuration
  * @dev: drm device for the ioctl
  * @data: data pointer for the ioctl
@@ -2125,599 +609,6 @@ int drm_mode_getcrtc(struct drm_device *dev,
 	return 0;
 }
 
-static bool drm_mode_expose_to_userspace(const struct drm_display_mode *mode,
-					 const struct drm_file *file_priv)
-{
-	/*
-	 * If user-space hasn't configured the driver to expose the stereo 3D
-	 * modes, don't expose them.
-	 */
-	if (!file_priv->stereo_allowed && drm_mode_is_stereo(mode))
-		return false;
-
-	return true;
-}
-
-static struct drm_encoder *drm_connector_get_encoder(struct drm_connector *connector)
-{
-	/* For atomic drivers only state objects are synchronously updated and
-	 * protected by modeset locks, so check those first. */
-	if (connector->state)
-		return connector->state->best_encoder;
-	return connector->encoder;
-}
-
-/* helper for getconnector and getproperties ioctls */
-static int get_properties(struct drm_mode_object *obj, bool atomic,
-		uint32_t __user *prop_ptr, uint64_t __user *prop_values,
-		uint32_t *arg_count_props)
-{
-	int props_count;
-	int i, ret, copied;
-
-	props_count = obj->properties->count;
-	if (!atomic)
-		props_count -= obj->properties->atomic_count;
-
-	if ((*arg_count_props >= props_count) && props_count) {
-		for (i = 0, copied = 0; copied < props_count; i++) {
-			struct drm_property *prop = obj->properties->properties[i];
-			uint64_t val;
-
-			if ((prop->flags & DRM_MODE_PROP_ATOMIC) && !atomic)
-				continue;
-
-			ret = drm_object_property_get_value(obj, prop, &val);
-			if (ret)
-				return ret;
-
-			if (put_user(prop->base.id, prop_ptr + copied))
-				return -EFAULT;
-
-			if (put_user(val, prop_values + copied))
-				return -EFAULT;
-
-			copied++;
-		}
-	}
-	*arg_count_props = props_count;
-
-	return 0;
-}
-
-/**
- * drm_mode_getconnector - get connector configuration
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Construct a connector configuration structure to return to the user.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_getconnector(struct drm_device *dev, void *data,
-			  struct drm_file *file_priv)
-{
-	struct drm_mode_get_connector *out_resp = data;
-	struct drm_connector *connector;
-	struct drm_encoder *encoder;
-	struct drm_display_mode *mode;
-	int mode_count = 0;
-	int encoders_count = 0;
-	int ret = 0;
-	int copied = 0;
-	int i;
-	struct drm_mode_modeinfo u_mode;
-	struct drm_mode_modeinfo __user *mode_ptr;
-	uint32_t __user *encoder_ptr;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	memset(&u_mode, 0, sizeof(struct drm_mode_modeinfo));
-
-	mutex_lock(&dev->mode_config.mutex);
-
-	connector = drm_connector_lookup(dev, out_resp->connector_id);
-	if (!connector) {
-		ret = -ENOENT;
-		goto out_unlock;
-	}
-
-	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++)
-		if (connector->encoder_ids[i] != 0)
-			encoders_count++;
-
-	if (out_resp->count_modes == 0) {
-		connector->funcs->fill_modes(connector,
-					     dev->mode_config.max_width,
-					     dev->mode_config.max_height);
-	}
-
-	/* delayed so we get modes regardless of pre-fill_modes state */
-	list_for_each_entry(mode, &connector->modes, head)
-		if (drm_mode_expose_to_userspace(mode, file_priv))
-			mode_count++;
-
-	out_resp->connector_id = connector->base.id;
-	out_resp->connector_type = connector->connector_type;
-	out_resp->connector_type_id = connector->connector_type_id;
-	out_resp->mm_width = connector->display_info.width_mm;
-	out_resp->mm_height = connector->display_info.height_mm;
-	out_resp->subpixel = connector->display_info.subpixel_order;
-	out_resp->connection = connector->status;
-
-	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
-	encoder = drm_connector_get_encoder(connector);
-	if (encoder)
-		out_resp->encoder_id = encoder->base.id;
-	else
-		out_resp->encoder_id = 0;
-
-	/*
-	 * This ioctl is called twice, once to determine how much space is
-	 * needed, and the 2nd time to fill it.
-	 */
-	if ((out_resp->count_modes >= mode_count) && mode_count) {
-		copied = 0;
-		mode_ptr = (struct drm_mode_modeinfo __user *)(unsigned long)out_resp->modes_ptr;
-		list_for_each_entry(mode, &connector->modes, head) {
-			if (!drm_mode_expose_to_userspace(mode, file_priv))
-				continue;
-
-			drm_mode_convert_to_umode(&u_mode, mode);
-			if (copy_to_user(mode_ptr + copied,
-					 &u_mode, sizeof(u_mode))) {
-				ret = -EFAULT;
-				goto out;
-			}
-			copied++;
-		}
-	}
-	out_resp->count_modes = mode_count;
-
-	ret = get_properties(&connector->base, file_priv->atomic,
-			(uint32_t __user *)(unsigned long)(out_resp->props_ptr),
-			(uint64_t __user *)(unsigned long)(out_resp->prop_values_ptr),
-			&out_resp->count_props);
-	if (ret)
-		goto out;
-
-	if ((out_resp->count_encoders >= encoders_count) && encoders_count) {
-		copied = 0;
-		encoder_ptr = (uint32_t __user *)(unsigned long)(out_resp->encoders_ptr);
-		for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
-			if (connector->encoder_ids[i] != 0) {
-				if (put_user(connector->encoder_ids[i],
-					     encoder_ptr + copied)) {
-					ret = -EFAULT;
-					goto out;
-				}
-				copied++;
-			}
-		}
-	}
-	out_resp->count_encoders = encoders_count;
-
-out:
-	drm_modeset_unlock(&dev->mode_config.connection_mutex);
-
-	drm_connector_unreference(connector);
-out_unlock:
-	mutex_unlock(&dev->mode_config.mutex);
-
-	return ret;
-}
-
-static struct drm_crtc *drm_encoder_get_crtc(struct drm_encoder *encoder)
-{
-	struct drm_connector *connector;
-	struct drm_device *dev = encoder->dev;
-	bool uses_atomic = false;
-
-	/* For atomic drivers only state objects are synchronously updated and
-	 * protected by modeset locks, so check those first. */
-	drm_for_each_connector(connector, dev) {
-		if (!connector->state)
-			continue;
-
-		uses_atomic = true;
-
-		if (connector->state->best_encoder != encoder)
-			continue;
-
-		return connector->state->crtc;
-	}
-
-	/* Don't return stale data (e.g. pending async disable). */
-	if (uses_atomic)
-		return NULL;
-
-	return encoder->crtc;
-}
-
-/**
- * drm_mode_getencoder - get encoder configuration
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Construct a encoder configuration structure to return to the user.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_getencoder(struct drm_device *dev, void *data,
-			struct drm_file *file_priv)
-{
-	struct drm_mode_get_encoder *enc_resp = data;
-	struct drm_encoder *encoder;
-	struct drm_crtc *crtc;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	encoder = drm_encoder_find(dev, enc_resp->encoder_id);
-	if (!encoder)
-		return -ENOENT;
-
-	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
-	crtc = drm_encoder_get_crtc(encoder);
-	if (crtc)
-		enc_resp->crtc_id = crtc->base.id;
-	else
-		enc_resp->crtc_id = 0;
-	drm_modeset_unlock(&dev->mode_config.connection_mutex);
-
-	enc_resp->encoder_type = encoder->encoder_type;
-	enc_resp->encoder_id = encoder->base.id;
-	enc_resp->possible_crtcs = encoder->possible_crtcs;
-	enc_resp->possible_clones = encoder->possible_clones;
-
-	return 0;
-}
-
-/**
- * drm_mode_getplane_res - enumerate all plane resources
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * Construct a list of plane ids to return to the user.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_getplane_res(struct drm_device *dev, void *data,
-			  struct drm_file *file_priv)
-{
-	struct drm_mode_get_plane_res *plane_resp = data;
-	struct drm_mode_config *config;
-	struct drm_plane *plane;
-	uint32_t __user *plane_ptr;
-	int copied = 0;
-	unsigned num_planes;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	config = &dev->mode_config;
-
-	if (file_priv->universal_planes)
-		num_planes = config->num_total_plane;
-	else
-		num_planes = config->num_overlay_plane;
-
-	/*
-	 * This ioctl is called twice, once to determine how much space is
-	 * needed, and the 2nd time to fill it.
-	 */
-	if (num_planes &&
-	    (plane_resp->count_planes >= num_planes)) {
-		plane_ptr = (uint32_t __user *)(unsigned long)plane_resp->plane_id_ptr;
-
-		/* Plane lists are invariant, no locking needed. */
-		drm_for_each_plane(plane, dev) {
-			/*
-			 * Unless userspace set the 'universal planes'
-			 * capability bit, only advertise overlays.
-			 */
-			if (plane->type != DRM_PLANE_TYPE_OVERLAY &&
-			    !file_priv->universal_planes)
-				continue;
-
-			if (put_user(plane->base.id, plane_ptr + copied))
-				return -EFAULT;
-			copied++;
-		}
-	}
-	plane_resp->count_planes = num_planes;
-
-	return 0;
-}
-
-/**
- * drm_mode_getplane - get plane configuration
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * Construct a plane configuration structure to return to the user.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_getplane(struct drm_device *dev, void *data,
-		      struct drm_file *file_priv)
-{
-	struct drm_mode_get_plane *plane_resp = data;
-	struct drm_plane *plane;
-	uint32_t __user *format_ptr;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	plane = drm_plane_find(dev, plane_resp->plane_id);
-	if (!plane)
-		return -ENOENT;
-
-	drm_modeset_lock(&plane->mutex, NULL);
-	if (plane->crtc)
-		plane_resp->crtc_id = plane->crtc->base.id;
-	else
-		plane_resp->crtc_id = 0;
-
-	if (plane->fb)
-		plane_resp->fb_id = plane->fb->base.id;
-	else
-		plane_resp->fb_id = 0;
-	drm_modeset_unlock(&plane->mutex);
-
-	plane_resp->plane_id = plane->base.id;
-	plane_resp->possible_crtcs = plane->possible_crtcs;
-	plane_resp->gamma_size = 0;
-
-	/*
-	 * This ioctl is called twice, once to determine how much space is
-	 * needed, and the 2nd time to fill it.
-	 */
-	if (plane->format_count &&
-	    (plane_resp->count_format_types >= plane->format_count)) {
-		format_ptr = (uint32_t __user *)(unsigned long)plane_resp->format_type_ptr;
-		if (copy_to_user(format_ptr,
-				 plane->format_types,
-				 sizeof(uint32_t) * plane->format_count)) {
-			return -EFAULT;
-		}
-	}
-	plane_resp->count_format_types = plane->format_count;
-
-	return 0;
-}
-
-/**
- * drm_plane_check_pixel_format - Check if the plane supports the pixel format
- * @plane: plane to check for format support
- * @format: the pixel format
- *
- * Returns:
- * Zero of @plane has @format in its list of supported pixel formats, -EINVAL
- * otherwise.
- */
-int drm_plane_check_pixel_format(const struct drm_plane *plane, u32 format)
-{
-	unsigned int i;
-
-	for (i = 0; i < plane->format_count; i++) {
-		if (format == plane->format_types[i])
-			return 0;
-	}
-
-	return -EINVAL;
-}
-
-static int check_src_coords(uint32_t src_x, uint32_t src_y,
-			    uint32_t src_w, uint32_t src_h,
-			    const struct drm_framebuffer *fb)
-{
-	unsigned int fb_width, fb_height;
-
-	fb_width = fb->width << 16;
-	fb_height = fb->height << 16;
-
-	/* Make sure source coordinates are inside the fb. */
-	if (src_w > fb_width ||
-	    src_x > fb_width - src_w ||
-	    src_h > fb_height ||
-	    src_y > fb_height - src_h) {
-		DRM_DEBUG_KMS("Invalid source coordinates "
-			      "%u.%06ux%u.%06u+%u.%06u+%u.%06u\n",
-			      src_w >> 16, ((src_w & 0xffff) * 15625) >> 10,
-			      src_h >> 16, ((src_h & 0xffff) * 15625) >> 10,
-			      src_x >> 16, ((src_x & 0xffff) * 15625) >> 10,
-			      src_y >> 16, ((src_y & 0xffff) * 15625) >> 10);
-		return -ENOSPC;
-	}
-
-	return 0;
-}
-
-/*
- * setplane_internal - setplane handler for internal callers
- *
- * Note that we assume an extra reference has already been taken on fb.  If the
- * update fails, this reference will be dropped before return; if it succeeds,
- * the previous framebuffer (if any) will be unreferenced instead.
- *
- * src_{x,y,w,h} are provided in 16.16 fixed point format
- */
-static int __setplane_internal(struct drm_plane *plane,
-			       struct drm_crtc *crtc,
-			       struct drm_framebuffer *fb,
-			       int32_t crtc_x, int32_t crtc_y,
-			       uint32_t crtc_w, uint32_t crtc_h,
-			       /* src_{x,y,w,h} values are 16.16 fixed point */
-			       uint32_t src_x, uint32_t src_y,
-			       uint32_t src_w, uint32_t src_h)
-{
-	int ret = 0;
-
-	/* No fb means shut it down */
-	if (!fb) {
-		plane->old_fb = plane->fb;
-		ret = plane->funcs->disable_plane(plane);
-		if (!ret) {
-			plane->crtc = NULL;
-			plane->fb = NULL;
-		} else {
-			plane->old_fb = NULL;
-		}
-		goto out;
-	}
-
-	/* Check whether this plane is usable on this CRTC */
-	if (!(plane->possible_crtcs & drm_crtc_mask(crtc))) {
-		DRM_DEBUG_KMS("Invalid crtc for plane\n");
-		ret = -EINVAL;
-		goto out;
-	}
-
-	/* Check whether this plane supports the fb pixel format. */
-	ret = drm_plane_check_pixel_format(plane, fb->pixel_format);
-	if (ret) {
-		DRM_DEBUG_KMS("Invalid pixel format %s\n",
-			      drm_get_format_name(fb->pixel_format));
-		goto out;
-	}
-
-	/* Give drivers some help against integer overflows */
-	if (crtc_w > INT_MAX ||
-	    crtc_x > INT_MAX - (int32_t) crtc_w ||
-	    crtc_h > INT_MAX ||
-	    crtc_y > INT_MAX - (int32_t) crtc_h) {
-		DRM_DEBUG_KMS("Invalid CRTC coordinates %ux%u+%d+%d\n",
-			      crtc_w, crtc_h, crtc_x, crtc_y);
-		ret = -ERANGE;
-		goto out;
-	}
-
-	ret = check_src_coords(src_x, src_y, src_w, src_h, fb);
-	if (ret)
-		goto out;
-
-	plane->old_fb = plane->fb;
-	ret = plane->funcs->update_plane(plane, crtc, fb,
-					 crtc_x, crtc_y, crtc_w, crtc_h,
-					 src_x, src_y, src_w, src_h);
-	if (!ret) {
-		plane->crtc = crtc;
-		plane->fb = fb;
-		fb = NULL;
-	} else {
-		plane->old_fb = NULL;
-	}
-
-out:
-	if (fb)
-		drm_framebuffer_unreference(fb);
-	if (plane->old_fb)
-		drm_framebuffer_unreference(plane->old_fb);
-	plane->old_fb = NULL;
-
-	return ret;
-}
-
-static int setplane_internal(struct drm_plane *plane,
-			     struct drm_crtc *crtc,
-			     struct drm_framebuffer *fb,
-			     int32_t crtc_x, int32_t crtc_y,
-			     uint32_t crtc_w, uint32_t crtc_h,
-			     /* src_{x,y,w,h} values are 16.16 fixed point */
-			     uint32_t src_x, uint32_t src_y,
-			     uint32_t src_w, uint32_t src_h)
-{
-	int ret;
-
-	drm_modeset_lock_all(plane->dev);
-	ret = __setplane_internal(plane, crtc, fb,
-				  crtc_x, crtc_y, crtc_w, crtc_h,
-				  src_x, src_y, src_w, src_h);
-	drm_modeset_unlock_all(plane->dev);
-
-	return ret;
-}
-
-/**
- * drm_mode_setplane - configure a plane's configuration
- * @dev: DRM device
- * @data: ioctl data*
- * @file_priv: DRM file info
- *
- * Set plane configuration, including placement, fb, scaling, and other factors.
- * Or pass a NULL fb to disable (planes may be disabled without providing a
- * valid crtc).
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_setplane(struct drm_device *dev, void *data,
-		      struct drm_file *file_priv)
-{
-	struct drm_mode_set_plane *plane_req = data;
-	struct drm_plane *plane;
-	struct drm_crtc *crtc = NULL;
-	struct drm_framebuffer *fb = NULL;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	/*
-	 * First, find the plane, crtc, and fb objects.  If not available,
-	 * we don't bother to call the driver.
-	 */
-	plane = drm_plane_find(dev, plane_req->plane_id);
-	if (!plane) {
-		DRM_DEBUG_KMS("Unknown plane ID %d\n",
-			      plane_req->plane_id);
-		return -ENOENT;
-	}
-
-	if (plane_req->fb_id) {
-		fb = drm_framebuffer_lookup(dev, plane_req->fb_id);
-		if (!fb) {
-			DRM_DEBUG_KMS("Unknown framebuffer ID %d\n",
-				      plane_req->fb_id);
-			return -ENOENT;
-		}
-
-		crtc = drm_crtc_find(dev, plane_req->crtc_id);
-		if (!crtc) {
-			DRM_DEBUG_KMS("Unknown crtc ID %d\n",
-				      plane_req->crtc_id);
-			return -ENOENT;
-		}
-	}
-
-	/*
-	 * setplane_internal will take care of deref'ing either the old or new
-	 * framebuffer depending on success.
-	 */
-	return setplane_internal(plane, crtc, fb,
-				 plane_req->crtc_x, plane_req->crtc_y,
-				 plane_req->crtc_w, plane_req->crtc_h,
-				 plane_req->src_x, plane_req->src_y,
-				 plane_req->src_w, plane_req->src_h);
-}
-
 /**
  * drm_mode_set_config_internal - helper to call ->set_config
  * @set: modeset config to set
@@ -2804,12 +695,13 @@ int drm_crtc_check_viewport(const struct drm_crtc *crtc,
 	drm_crtc_get_hv_timing(mode, &hdisplay, &vdisplay);
 
 	if (crtc->state &&
-	    crtc->primary->state->rotation & (BIT(DRM_ROTATE_90) |
-					      BIT(DRM_ROTATE_270)))
+	    crtc->primary->state->rotation & (DRM_ROTATE_90 |
+					      DRM_ROTATE_270))
 		swap(hdisplay, vdisplay);
 
-	return check_src_coords(x << 16, y << 16,
-				hdisplay << 16, vdisplay << 16, fb);
+	return drm_framebuffer_check_src_coords(x << 16, y << 16,
+						hdisplay << 16, vdisplay << 16,
+						fb);
 }
 EXPORT_SYMBOL(drm_crtc_check_viewport);
 
@@ -2904,8 +796,9 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 			ret = drm_plane_check_pixel_format(crtc->primary,
 							   fb->pixel_format);
 			if (ret) {
-				DRM_DEBUG_KMS("Invalid pixel format %s\n",
-					drm_get_format_name(fb->pixel_format));
+				char *format_name = drm_get_format_name(fb->pixel_format);
+				DRM_DEBUG_KMS("Invalid pixel format %s\n", format_name);
+				kfree(format_name);
 				goto out;
 			}
 		}
@@ -2995,2004 +888,9 @@ out:
 	return ret;
 }
 
-/**
- * drm_mode_cursor_universal - translate legacy cursor ioctl call into a
- *     universal plane handler call
- * @crtc: crtc to update cursor for
- * @req: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Legacy cursor ioctl's work directly with driver buffer handles.  To
- * translate legacy ioctl calls into universal plane handler calls, we need to
- * wrap the native buffer handle in a drm_framebuffer.
- *
- * Note that we assume any handle passed to the legacy ioctls was a 32-bit ARGB
- * buffer with a pitch of 4*width; the universal plane interface should be used
- * directly in cases where the hardware can support other buffer settings and
- * userspace wants to make use of these capabilities.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-static int drm_mode_cursor_universal(struct drm_crtc *crtc,
-				     struct drm_mode_cursor2 *req,
-				     struct drm_file *file_priv)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_framebuffer *fb = NULL;
-	struct drm_mode_fb_cmd2 fbreq = {
-		.width = req->width,
-		.height = req->height,
-		.pixel_format = DRM_FORMAT_ARGB8888,
-		.pitches = { req->width * 4 },
-		.handles = { req->handle },
-	};
-	int32_t crtc_x, crtc_y;
-	uint32_t crtc_w = 0, crtc_h = 0;
-	uint32_t src_w = 0, src_h = 0;
-	int ret = 0;
-
-	BUG_ON(!crtc->cursor);
-	WARN_ON(crtc->cursor->crtc != crtc && crtc->cursor->crtc != NULL);
-
-	/*
-	 * Obtain fb we'll be using (either new or existing) and take an extra
-	 * reference to it if fb != null.  setplane will take care of dropping
-	 * the reference if the plane update fails.
-	 */
-	if (req->flags & DRM_MODE_CURSOR_BO) {
-		if (req->handle) {
-			fb = internal_framebuffer_create(dev, &fbreq, file_priv);
-			if (IS_ERR(fb)) {
-				DRM_DEBUG_KMS("failed to wrap cursor buffer in drm framebuffer\n");
-				return PTR_ERR(fb);
-			}
-			fb->hot_x = req->hot_x;
-			fb->hot_y = req->hot_y;
-		} else {
-			fb = NULL;
-		}
-	} else {
-		fb = crtc->cursor->fb;
-		if (fb)
-			drm_framebuffer_reference(fb);
-	}
-
-	if (req->flags & DRM_MODE_CURSOR_MOVE) {
-		crtc_x = req->x;
-		crtc_y = req->y;
-	} else {
-		crtc_x = crtc->cursor_x;
-		crtc_y = crtc->cursor_y;
-	}
-
-	if (fb) {
-		crtc_w = fb->width;
-		crtc_h = fb->height;
-		src_w = fb->width << 16;
-		src_h = fb->height << 16;
-	}
-
-	/*
-	 * setplane_internal will take care of deref'ing either the old or new
-	 * framebuffer depending on success.
-	 */
-	ret = __setplane_internal(crtc->cursor, crtc, fb,
-				crtc_x, crtc_y, crtc_w, crtc_h,
-				0, 0, src_w, src_h);
-
-	/* Update successful; save new cursor position, if necessary */
-	if (ret == 0 && req->flags & DRM_MODE_CURSOR_MOVE) {
-		crtc->cursor_x = req->x;
-		crtc->cursor_y = req->y;
-	}
-
-	return ret;
-}
-
-static int drm_mode_cursor_common(struct drm_device *dev,
-				  struct drm_mode_cursor2 *req,
-				  struct drm_file *file_priv)
-{
-	struct drm_crtc *crtc;
-	int ret = 0;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	if (!req->flags || (~DRM_MODE_CURSOR_FLAGS & req->flags))
-		return -EINVAL;
-
-	crtc = drm_crtc_find(dev, req->crtc_id);
-	if (!crtc) {
-		DRM_DEBUG_KMS("Unknown CRTC ID %d\n", req->crtc_id);
-		return -ENOENT;
-	}
-
-	/*
-	 * If this crtc has a universal cursor plane, call that plane's update
-	 * handler rather than using legacy cursor handlers.
-	 */
-	drm_modeset_lock_crtc(crtc, crtc->cursor);
-	if (crtc->cursor) {
-		ret = drm_mode_cursor_universal(crtc, req, file_priv);
-		goto out;
-	}
-
-	if (req->flags & DRM_MODE_CURSOR_BO) {
-		if (!crtc->funcs->cursor_set && !crtc->funcs->cursor_set2) {
-			ret = -ENXIO;
-			goto out;
-		}
-		/* Turns off the cursor if handle is 0 */
-		if (crtc->funcs->cursor_set2)
-			ret = crtc->funcs->cursor_set2(crtc, file_priv, req->handle,
-						      req->width, req->height, req->hot_x, req->hot_y);
-		else
-			ret = crtc->funcs->cursor_set(crtc, file_priv, req->handle,
-						      req->width, req->height);
-	}
-
-	if (req->flags & DRM_MODE_CURSOR_MOVE) {
-		if (crtc->funcs->cursor_move) {
-			ret = crtc->funcs->cursor_move(crtc, req->x, req->y);
-		} else {
-			ret = -EFAULT;
-			goto out;
-		}
-	}
-out:
-	drm_modeset_unlock_crtc(crtc);
-
-	return ret;
-
-}
-
-
-/**
- * drm_mode_cursor_ioctl - set CRTC's cursor configuration
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Set the cursor configuration based on user request.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_cursor_ioctl(struct drm_device *dev,
-			  void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_cursor *req = data;
-	struct drm_mode_cursor2 new_req;
-
-	memcpy(&new_req, req, sizeof(struct drm_mode_cursor));
-	new_req.hot_x = new_req.hot_y = 0;
-
-	return drm_mode_cursor_common(dev, &new_req, file_priv);
-}
-
-/**
- * drm_mode_cursor2_ioctl - set CRTC's cursor configuration
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Set the cursor configuration based on user request. This implements the 2nd
- * version of the cursor ioctl, which allows userspace to additionally specify
- * the hotspot of the pointer.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_cursor2_ioctl(struct drm_device *dev,
-			   void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_cursor2 *req = data;
-
-	return drm_mode_cursor_common(dev, req, file_priv);
-}
-
-/**
- * drm_mode_legacy_fb_format - compute drm fourcc code from legacy description
- * @bpp: bits per pixels
- * @depth: bit depth per pixel
- *
- * Computes a drm fourcc pixel format code for the given @bpp/@depth values.
- * Useful in fbdev emulation code, since that deals in those values.
- */
-uint32_t drm_mode_legacy_fb_format(uint32_t bpp, uint32_t depth)
-{
-	uint32_t fmt;
-
-	switch (bpp) {
-	case 8:
-		fmt = DRM_FORMAT_C8;
-		break;
-	case 16:
-		if (depth == 15)
-			fmt = DRM_FORMAT_XRGB1555;
-		else
-			fmt = DRM_FORMAT_RGB565;
-		break;
-	case 24:
-		fmt = DRM_FORMAT_RGB888;
-		break;
-	case 32:
-		if (depth == 24)
-			fmt = DRM_FORMAT_XRGB8888;
-		else if (depth == 30)
-			fmt = DRM_FORMAT_XRGB2101010;
-		else
-			fmt = DRM_FORMAT_ARGB8888;
-		break;
-	default:
-		DRM_ERROR("bad bpp, assuming x8r8g8b8 pixel format\n");
-		fmt = DRM_FORMAT_XRGB8888;
-		break;
-	}
-
-	return fmt;
-}
-EXPORT_SYMBOL(drm_mode_legacy_fb_format);
-
-/**
- * drm_mode_addfb - add an FB to the graphics configuration
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Add a new FB to the specified CRTC, given a user request. This is the
- * original addfb ioctl which only supported RGB formats.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_addfb(struct drm_device *dev,
-		   void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_fb_cmd *or = data;
-	struct drm_mode_fb_cmd2 r = {};
-	int ret;
-
-	/* convert to new format and call new ioctl */
-	r.fb_id = or->fb_id;
-	r.width = or->width;
-	r.height = or->height;
-	r.pitches[0] = or->pitch;
-	r.pixel_format = drm_mode_legacy_fb_format(or->bpp, or->depth);
-	r.handles[0] = or->handle;
-
-	ret = drm_mode_addfb2(dev, &r, file_priv);
-	if (ret)
-		return ret;
-
-	or->fb_id = r.fb_id;
-
-	return 0;
-}
-
-static int format_check(const struct drm_mode_fb_cmd2 *r)
-{
-	uint32_t format = r->pixel_format & ~DRM_FORMAT_BIG_ENDIAN;
-
-	switch (format) {
-	case DRM_FORMAT_C8:
-	case DRM_FORMAT_RGB332:
-	case DRM_FORMAT_BGR233:
-	case DRM_FORMAT_XRGB4444:
-	case DRM_FORMAT_XBGR4444:
-	case DRM_FORMAT_RGBX4444:
-	case DRM_FORMAT_BGRX4444:
-	case DRM_FORMAT_ARGB4444:
-	case DRM_FORMAT_ABGR4444:
-	case DRM_FORMAT_RGBA4444:
-	case DRM_FORMAT_BGRA4444:
-	case DRM_FORMAT_XRGB1555:
-	case DRM_FORMAT_XBGR1555:
-	case DRM_FORMAT_RGBX5551:
-	case DRM_FORMAT_BGRX5551:
-	case DRM_FORMAT_ARGB1555:
-	case DRM_FORMAT_ABGR1555:
-	case DRM_FORMAT_RGBA5551:
-	case DRM_FORMAT_BGRA5551:
-	case DRM_FORMAT_RGB565:
-	case DRM_FORMAT_BGR565:
-	case DRM_FORMAT_RGB888:
-	case DRM_FORMAT_BGR888:
-	case DRM_FORMAT_XRGB8888:
-	case DRM_FORMAT_XBGR8888:
-	case DRM_FORMAT_RGBX8888:
-	case DRM_FORMAT_BGRX8888:
-	case DRM_FORMAT_ARGB8888:
-	case DRM_FORMAT_ABGR8888:
-	case DRM_FORMAT_RGBA8888:
-	case DRM_FORMAT_BGRA8888:
-	case DRM_FORMAT_XRGB2101010:
-	case DRM_FORMAT_XBGR2101010:
-	case DRM_FORMAT_RGBX1010102:
-	case DRM_FORMAT_BGRX1010102:
-	case DRM_FORMAT_ARGB2101010:
-	case DRM_FORMAT_ABGR2101010:
-	case DRM_FORMAT_RGBA1010102:
-	case DRM_FORMAT_BGRA1010102:
-	case DRM_FORMAT_YUYV:
-	case DRM_FORMAT_YVYU:
-	case DRM_FORMAT_UYVY:
-	case DRM_FORMAT_VYUY:
-	case DRM_FORMAT_AYUV:
-	case DRM_FORMAT_NV12:
-	case DRM_FORMAT_NV21:
-	case DRM_FORMAT_NV16:
-	case DRM_FORMAT_NV61:
-	case DRM_FORMAT_NV24:
-	case DRM_FORMAT_NV42:
-	case DRM_FORMAT_YUV410:
-	case DRM_FORMAT_YVU410:
-	case DRM_FORMAT_YUV411:
-	case DRM_FORMAT_YVU411:
-	case DRM_FORMAT_YUV420:
-	case DRM_FORMAT_YVU420:
-	case DRM_FORMAT_YUV422:
-	case DRM_FORMAT_YVU422:
-	case DRM_FORMAT_YUV444:
-	case DRM_FORMAT_YVU444:
-		return 0;
-	default:
-		DRM_DEBUG_KMS("invalid pixel format %s\n",
-			      drm_get_format_name(r->pixel_format));
-		return -EINVAL;
-	}
-}
-
-static int framebuffer_check(const struct drm_mode_fb_cmd2 *r)
-{
-	int ret, hsub, vsub, num_planes, i;
-
-	ret = format_check(r);
-	if (ret) {
-		DRM_DEBUG_KMS("bad framebuffer format %s\n",
-			      drm_get_format_name(r->pixel_format));
-		return ret;
-	}
-
-	hsub = drm_format_horz_chroma_subsampling(r->pixel_format);
-	vsub = drm_format_vert_chroma_subsampling(r->pixel_format);
-	num_planes = drm_format_num_planes(r->pixel_format);
-
-	if (r->width == 0 || r->width % hsub) {
-		DRM_DEBUG_KMS("bad framebuffer width %u\n", r->width);
-		return -EINVAL;
-	}
-
-	if (r->height == 0 || r->height % vsub) {
-		DRM_DEBUG_KMS("bad framebuffer height %u\n", r->height);
-		return -EINVAL;
-	}
-
-	for (i = 0; i < num_planes; i++) {
-		unsigned int width = r->width / (i != 0 ? hsub : 1);
-		unsigned int height = r->height / (i != 0 ? vsub : 1);
-		unsigned int cpp = drm_format_plane_cpp(r->pixel_format, i);
-
-		if (!r->handles[i]) {
-			DRM_DEBUG_KMS("no buffer object handle for plane %d\n", i);
-			return -EINVAL;
-		}
-
-		if ((uint64_t) width * cpp > UINT_MAX)
-			return -ERANGE;
-
-		if ((uint64_t) height * r->pitches[i] + r->offsets[i] > UINT_MAX)
-			return -ERANGE;
-
-		if (r->pitches[i] < width * cpp) {
-			DRM_DEBUG_KMS("bad pitch %u for plane %d\n", r->pitches[i], i);
-			return -EINVAL;
-		}
-
-		if (r->modifier[i] && !(r->flags & DRM_MODE_FB_MODIFIERS)) {
-			DRM_DEBUG_KMS("bad fb modifier %llu for plane %d\n",
-				      r->modifier[i], i);
-			return -EINVAL;
-		}
-
-		/* modifier specific checks: */
-		switch (r->modifier[i]) {
-		case DRM_FORMAT_MOD_SAMSUNG_64_32_TILE:
-			/* NOTE: the pitch restriction may be lifted later if it turns
-			 * out that no hw has this restriction:
-			 */
-			if (r->pixel_format != DRM_FORMAT_NV12 ||
-					width % 128 || height % 32 ||
-					r->pitches[i] % 128) {
-				DRM_DEBUG_KMS("bad modifier data for plane %d\n", i);
-				return -EINVAL;
-			}
-			break;
-
-		default:
-			break;
-		}
-	}
-
-	for (i = num_planes; i < 4; i++) {
-		if (r->modifier[i]) {
-			DRM_DEBUG_KMS("non-zero modifier for unused plane %d\n", i);
-			return -EINVAL;
-		}
-
-		/* Pre-FB_MODIFIERS userspace didn't clear the structs properly. */
-		if (!(r->flags & DRM_MODE_FB_MODIFIERS))
-			continue;
-
-		if (r->handles[i]) {
-			DRM_DEBUG_KMS("buffer object handle for unused plane %d\n", i);
-			return -EINVAL;
-		}
-
-		if (r->pitches[i]) {
-			DRM_DEBUG_KMS("non-zero pitch for unused plane %d\n", i);
-			return -EINVAL;
-		}
-
-		if (r->offsets[i]) {
-			DRM_DEBUG_KMS("non-zero offset for unused plane %d\n", i);
-			return -EINVAL;
-		}
-	}
-
-	return 0;
-}
-
-static struct drm_framebuffer *
-internal_framebuffer_create(struct drm_device *dev,
-			    const struct drm_mode_fb_cmd2 *r,
-			    struct drm_file *file_priv)
-{
-	struct drm_mode_config *config = &dev->mode_config;
-	struct drm_framebuffer *fb;
-	int ret;
-
-	if (r->flags & ~(DRM_MODE_FB_INTERLACED | DRM_MODE_FB_MODIFIERS)) {
-		DRM_DEBUG_KMS("bad framebuffer flags 0x%08x\n", r->flags);
-		return ERR_PTR(-EINVAL);
-	}
-
-	if ((config->min_width > r->width) || (r->width > config->max_width)) {
-		DRM_DEBUG_KMS("bad framebuffer width %d, should be >= %d && <= %d\n",
-			  r->width, config->min_width, config->max_width);
-		return ERR_PTR(-EINVAL);
-	}
-	if ((config->min_height > r->height) || (r->height > config->max_height)) {
-		DRM_DEBUG_KMS("bad framebuffer height %d, should be >= %d && <= %d\n",
-			  r->height, config->min_height, config->max_height);
-		return ERR_PTR(-EINVAL);
-	}
-
-	if (r->flags & DRM_MODE_FB_MODIFIERS &&
-	    !dev->mode_config.allow_fb_modifiers) {
-		DRM_DEBUG_KMS("driver does not support fb modifiers\n");
-		return ERR_PTR(-EINVAL);
-	}
-
-	ret = framebuffer_check(r);
-	if (ret)
-		return ERR_PTR(ret);
-
-	fb = dev->mode_config.funcs->fb_create(dev, file_priv, r);
-	if (IS_ERR(fb)) {
-		DRM_DEBUG_KMS("could not create framebuffer\n");
-		return fb;
-	}
-
-	return fb;
-}
-
-/**
- * drm_mode_addfb2 - add an FB to the graphics configuration
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Add a new FB to the specified CRTC, given a user request with format. This is
- * the 2nd version of the addfb ioctl, which supports multi-planar framebuffers
- * and uses fourcc codes as pixel format specifiers.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_addfb2(struct drm_device *dev,
-		    void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_fb_cmd2 *r = data;
-	struct drm_framebuffer *fb;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	fb = internal_framebuffer_create(dev, r, file_priv);
-	if (IS_ERR(fb))
-		return PTR_ERR(fb);
-
-	DRM_DEBUG_KMS("[FB:%d]\n", fb->base.id);
-	r->fb_id = fb->base.id;
-
-	/* Transfer ownership to the filp for reaping on close */
-	mutex_lock(&file_priv->fbs_lock);
-	list_add(&fb->filp_head, &file_priv->fbs);
-	mutex_unlock(&file_priv->fbs_lock);
-
-	return 0;
-}
-
-struct drm_mode_rmfb_work {
-	struct work_struct work;
-	struct list_head fbs;
-};
-
-static void drm_mode_rmfb_work_fn(struct work_struct *w)
-{
-	struct drm_mode_rmfb_work *arg = container_of(w, typeof(*arg), work);
-
-	while (!list_empty(&arg->fbs)) {
-		struct drm_framebuffer *fb =
-			list_first_entry(&arg->fbs, typeof(*fb), filp_head);
-
-		list_del_init(&fb->filp_head);
-		drm_framebuffer_remove(fb);
-	}
-}
-
-/**
- * drm_mode_rmfb - remove an FB from the configuration
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Remove the FB specified by the user.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_rmfb(struct drm_device *dev,
-		   void *data, struct drm_file *file_priv)
-{
-	struct drm_framebuffer *fb = NULL;
-	struct drm_framebuffer *fbl = NULL;
-	uint32_t *id = data;
-	int found = 0;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	fb = drm_framebuffer_lookup(dev, *id);
-	if (!fb)
-		return -ENOENT;
-
-	mutex_lock(&file_priv->fbs_lock);
-	list_for_each_entry(fbl, &file_priv->fbs, filp_head)
-		if (fb == fbl)
-			found = 1;
-	if (!found) {
-		mutex_unlock(&file_priv->fbs_lock);
-		goto fail_unref;
-	}
-
-	list_del_init(&fb->filp_head);
-	mutex_unlock(&file_priv->fbs_lock);
-
-	/* drop the reference we picked up in framebuffer lookup */
-	drm_framebuffer_unreference(fb);
-
-	/*
-	 * we now own the reference that was stored in the fbs list
-	 *
-	 * drm_framebuffer_remove may fail with -EINTR on pending signals,
-	 * so run this in a separate stack as there's no way to correctly
-	 * handle this after the fb is already removed from the lookup table.
-	 */
-	if (drm_framebuffer_read_refcount(fb) > 1) {
-		struct drm_mode_rmfb_work arg;
-
-		INIT_WORK_ONSTACK(&arg.work, drm_mode_rmfb_work_fn);
-		INIT_LIST_HEAD(&arg.fbs);
-		list_add_tail(&fb->filp_head, &arg.fbs);
-
-		schedule_work(&arg.work);
-		flush_work(&arg.work);
-		destroy_work_on_stack(&arg.work);
-	} else
-		drm_framebuffer_unreference(fb);
-
-	return 0;
-
-fail_unref:
-	drm_framebuffer_unreference(fb);
-	return -ENOENT;
-}
-
-/**
- * drm_mode_getfb - get FB info
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Lookup the FB given its ID and return info about it.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_getfb(struct drm_device *dev,
-		   void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_fb_cmd *r = data;
-	struct drm_framebuffer *fb;
-	int ret;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	fb = drm_framebuffer_lookup(dev, r->fb_id);
-	if (!fb)
-		return -ENOENT;
-
-	r->height = fb->height;
-	r->width = fb->width;
-	r->depth = fb->depth;
-	r->bpp = fb->bits_per_pixel;
-	r->pitch = fb->pitches[0];
-	if (fb->funcs->create_handle) {
-		if (drm_is_current_master(file_priv) || capable(CAP_SYS_ADMIN) ||
-		    drm_is_control_client(file_priv)) {
-			ret = fb->funcs->create_handle(fb, file_priv,
-						       &r->handle);
-		} else {
-			/* GET_FB() is an unprivileged ioctl so we must not
-			 * return a buffer-handle to non-master processes! For
-			 * backwards-compatibility reasons, we cannot make
-			 * GET_FB() privileged, so just return an invalid handle
-			 * for non-masters. */
-			r->handle = 0;
-			ret = 0;
-		}
-	} else {
-		ret = -ENODEV;
-	}
-
-	drm_framebuffer_unreference(fb);
-
-	return ret;
-}
-
-/**
- * drm_mode_dirtyfb_ioctl - flush frontbuffer rendering on an FB
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Lookup the FB and flush out the damaged area supplied by userspace as a clip
- * rectangle list. Generic userspace which does frontbuffer rendering must call
- * this ioctl to flush out the changes on manual-update display outputs, e.g.
- * usb display-link, mipi manual update panels or edp panel self refresh modes.
- *
- * Modesetting drivers which always update the frontbuffer do not need to
- * implement the corresponding ->dirty framebuffer callback.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
-			   void *data, struct drm_file *file_priv)
-{
-	struct drm_clip_rect __user *clips_ptr;
-	struct drm_clip_rect *clips = NULL;
-	struct drm_mode_fb_dirty_cmd *r = data;
-	struct drm_framebuffer *fb;
-	unsigned flags;
-	int num_clips;
-	int ret;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	fb = drm_framebuffer_lookup(dev, r->fb_id);
-	if (!fb)
-		return -ENOENT;
-
-	num_clips = r->num_clips;
-	clips_ptr = (struct drm_clip_rect __user *)(unsigned long)r->clips_ptr;
-
-	if (!num_clips != !clips_ptr) {
-		ret = -EINVAL;
-		goto out_err1;
-	}
-
-	flags = DRM_MODE_FB_DIRTY_FLAGS & r->flags;
-
-	/* If userspace annotates copy, clips must come in pairs */
-	if (flags & DRM_MODE_FB_DIRTY_ANNOTATE_COPY && (num_clips % 2)) {
-		ret = -EINVAL;
-		goto out_err1;
-	}
-
-	if (num_clips && clips_ptr) {
-		if (num_clips < 0 || num_clips > DRM_MODE_FB_DIRTY_MAX_CLIPS) {
-			ret = -EINVAL;
-			goto out_err1;
-		}
-		clips = kcalloc(num_clips, sizeof(*clips), GFP_KERNEL);
-		if (!clips) {
-			ret = -ENOMEM;
-			goto out_err1;
-		}
-
-		ret = copy_from_user(clips, clips_ptr,
-				     num_clips * sizeof(*clips));
-		if (ret) {
-			ret = -EFAULT;
-			goto out_err2;
-		}
-	}
-
-	if (fb->funcs->dirty) {
-		ret = fb->funcs->dirty(fb, file_priv, flags, r->color,
-				       clips, num_clips);
-	} else {
-		ret = -ENOSYS;
-	}
-
-out_err2:
-	kfree(clips);
-out_err1:
-	drm_framebuffer_unreference(fb);
-
-	return ret;
-}
-
-/**
- * drm_fb_release - remove and free the FBs on this file
- * @priv: drm file for the ioctl
- *
- * Destroy all the FBs associated with @filp.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-void drm_fb_release(struct drm_file *priv)
-{
-	struct drm_framebuffer *fb, *tfb;
-	struct drm_mode_rmfb_work arg;
-
-	INIT_LIST_HEAD(&arg.fbs);
-
-	/*
-	 * When the file gets released that means no one else can access the fb
-	 * list any more, so no need to grab fpriv->fbs_lock. And we need to
-	 * avoid upsetting lockdep since the universal cursor code adds a
-	 * framebuffer while holding mutex locks.
-	 *
-	 * Note that a real deadlock between fpriv->fbs_lock and the modeset
-	 * locks is impossible here since no one else but this function can get
-	 * at it any more.
-	 */
-	list_for_each_entry_safe(fb, tfb, &priv->fbs, filp_head) {
-		if (drm_framebuffer_read_refcount(fb) > 1) {
-			list_move_tail(&fb->filp_head, &arg.fbs);
-		} else {
-			list_del_init(&fb->filp_head);
-
-			/* This drops the fpriv->fbs reference. */
-			drm_framebuffer_unreference(fb);
-		}
-	}
-
-	if (!list_empty(&arg.fbs)) {
-		INIT_WORK_ONSTACK(&arg.work, drm_mode_rmfb_work_fn);
-
-		schedule_work(&arg.work);
-		flush_work(&arg.work);
-		destroy_work_on_stack(&arg.work);
-	}
-}
-
-static bool drm_property_type_valid(struct drm_property *property)
-{
-	if (property->flags & DRM_MODE_PROP_EXTENDED_TYPE)
-		return !(property->flags & DRM_MODE_PROP_LEGACY_TYPE);
-	return !!(property->flags & DRM_MODE_PROP_LEGACY_TYPE);
-}
-
-/**
- * drm_property_create - create a new property type
- * @dev: drm device
- * @flags: flags specifying the property type
- * @name: name of the property
- * @num_values: number of pre-defined values
- *
- * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy.
- *
- * Note that the DRM core keeps a per-device list of properties and that, if
- * drm_mode_config_cleanup() is called, it will destroy all properties created
- * by the driver.
- *
- * Returns:
- * A pointer to the newly created property on success, NULL on failure.
- */
-struct drm_property *drm_property_create(struct drm_device *dev, int flags,
-					 const char *name, int num_values)
-{
-	struct drm_property *property = NULL;
-	int ret;
-
-	property = kzalloc(sizeof(struct drm_property), GFP_KERNEL);
-	if (!property)
-		return NULL;
-
-	property->dev = dev;
-
-	if (num_values) {
-		property->values = kcalloc(num_values, sizeof(uint64_t),
-					   GFP_KERNEL);
-		if (!property->values)
-			goto fail;
-	}
-
-	ret = drm_mode_object_get(dev, &property->base, DRM_MODE_OBJECT_PROPERTY);
-	if (ret)
-		goto fail;
-
-	property->flags = flags;
-	property->num_values = num_values;
-	INIT_LIST_HEAD(&property->enum_list);
-
-	if (name) {
-		strncpy(property->name, name, DRM_PROP_NAME_LEN);
-		property->name[DRM_PROP_NAME_LEN-1] = '\0';
-	}
-
-	list_add_tail(&property->head, &dev->mode_config.property_list);
-
-	WARN_ON(!drm_property_type_valid(property));
-
-	return property;
-fail:
-	kfree(property->values);
-	kfree(property);
-	return NULL;
-}
-EXPORT_SYMBOL(drm_property_create);
-
-/**
- * drm_property_create_enum - create a new enumeration property type
- * @dev: drm device
- * @flags: flags specifying the property type
- * @name: name of the property
- * @props: enumeration lists with property values
- * @num_values: number of pre-defined values
- *
- * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy.
- *
- * Userspace is only allowed to set one of the predefined values for enumeration
- * properties.
- *
- * Returns:
- * A pointer to the newly created property on success, NULL on failure.
- */
-struct drm_property *drm_property_create_enum(struct drm_device *dev, int flags,
-					 const char *name,
-					 const struct drm_prop_enum_list *props,
-					 int num_values)
-{
-	struct drm_property *property;
-	int i, ret;
-
-	flags |= DRM_MODE_PROP_ENUM;
-
-	property = drm_property_create(dev, flags, name, num_values);
-	if (!property)
-		return NULL;
-
-	for (i = 0; i < num_values; i++) {
-		ret = drm_property_add_enum(property, i,
-				      props[i].type,
-				      props[i].name);
-		if (ret) {
-			drm_property_destroy(dev, property);
-			return NULL;
-		}
-	}
-
-	return property;
-}
-EXPORT_SYMBOL(drm_property_create_enum);
-
-/**
- * drm_property_create_bitmask - create a new bitmask property type
- * @dev: drm device
- * @flags: flags specifying the property type
- * @name: name of the property
- * @props: enumeration lists with property bitflags
- * @num_props: size of the @props array
- * @supported_bits: bitmask of all supported enumeration values
- *
- * This creates a new bitmask drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy.
- *
- * Compared to plain enumeration properties userspace is allowed to set any
- * or'ed together combination of the predefined property bitflag values
- *
- * Returns:
- * A pointer to the newly created property on success, NULL on failure.
- */
-struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
-					 int flags, const char *name,
-					 const struct drm_prop_enum_list *props,
-					 int num_props,
-					 uint64_t supported_bits)
-{
-	struct drm_property *property;
-	int i, ret, index = 0;
-	int num_values = hweight64(supported_bits);
-
-	flags |= DRM_MODE_PROP_BITMASK;
-
-	property = drm_property_create(dev, flags, name, num_values);
-	if (!property)
-		return NULL;
-	for (i = 0; i < num_props; i++) {
-		if (!(supported_bits & (1ULL << props[i].type)))
-			continue;
-
-		if (WARN_ON(index >= num_values)) {
-			drm_property_destroy(dev, property);
-			return NULL;
-		}
-
-		ret = drm_property_add_enum(property, index++,
-				      props[i].type,
-				      props[i].name);
-		if (ret) {
-			drm_property_destroy(dev, property);
-			return NULL;
-		}
-	}
-
-	return property;
-}
-EXPORT_SYMBOL(drm_property_create_bitmask);
-
-static struct drm_property *property_create_range(struct drm_device *dev,
-					 int flags, const char *name,
-					 uint64_t min, uint64_t max)
-{
-	struct drm_property *property;
-
-	property = drm_property_create(dev, flags, name, 2);
-	if (!property)
-		return NULL;
-
-	property->values[0] = min;
-	property->values[1] = max;
-
-	return property;
-}
-
-/**
- * drm_property_create_range - create a new unsigned ranged property type
- * @dev: drm device
- * @flags: flags specifying the property type
- * @name: name of the property
- * @min: minimum value of the property
- * @max: maximum value of the property
- *
- * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy.
- *
- * Userspace is allowed to set any unsigned integer value in the (min, max)
- * range inclusive.
- *
- * Returns:
- * A pointer to the newly created property on success, NULL on failure.
- */
-struct drm_property *drm_property_create_range(struct drm_device *dev, int flags,
-					 const char *name,
-					 uint64_t min, uint64_t max)
-{
-	return property_create_range(dev, DRM_MODE_PROP_RANGE | flags,
-			name, min, max);
-}
-EXPORT_SYMBOL(drm_property_create_range);
-
-/**
- * drm_property_create_signed_range - create a new signed ranged property type
- * @dev: drm device
- * @flags: flags specifying the property type
- * @name: name of the property
- * @min: minimum value of the property
- * @max: maximum value of the property
- *
- * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy.
- *
- * Userspace is allowed to set any signed integer value in the (min, max)
- * range inclusive.
- *
- * Returns:
- * A pointer to the newly created property on success, NULL on failure.
- */
-struct drm_property *drm_property_create_signed_range(struct drm_device *dev,
-					 int flags, const char *name,
-					 int64_t min, int64_t max)
-{
-	return property_create_range(dev, DRM_MODE_PROP_SIGNED_RANGE | flags,
-			name, I642U64(min), I642U64(max));
-}
-EXPORT_SYMBOL(drm_property_create_signed_range);
-
-/**
- * drm_property_create_object - create a new object property type
- * @dev: drm device
- * @flags: flags specifying the property type
- * @name: name of the property
- * @type: object type from DRM_MODE_OBJECT_* defines
- *
- * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy.
- *
- * Userspace is only allowed to set this to any property value of the given
- * @type. Only useful for atomic properties, which is enforced.
- *
- * Returns:
- * A pointer to the newly created property on success, NULL on failure.
- */
-struct drm_property *drm_property_create_object(struct drm_device *dev,
-					 int flags, const char *name, uint32_t type)
-{
-	struct drm_property *property;
-
-	flags |= DRM_MODE_PROP_OBJECT;
-
-	if (WARN_ON(!(flags & DRM_MODE_PROP_ATOMIC)))
-		return NULL;
-
-	property = drm_property_create(dev, flags, name, 1);
-	if (!property)
-		return NULL;
-
-	property->values[0] = type;
-
-	return property;
-}
-EXPORT_SYMBOL(drm_property_create_object);
-
-/**
- * drm_property_create_bool - create a new boolean property type
- * @dev: drm device
- * @flags: flags specifying the property type
- * @name: name of the property
- *
- * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy.
- *
- * This is implemented as a ranged property with only {0, 1} as valid values.
- *
- * Returns:
- * A pointer to the newly created property on success, NULL on failure.
- */
-struct drm_property *drm_property_create_bool(struct drm_device *dev, int flags,
-					 const char *name)
-{
-	return drm_property_create_range(dev, flags, name, 0, 1);
-}
-EXPORT_SYMBOL(drm_property_create_bool);
-
-/**
- * drm_property_add_enum - add a possible value to an enumeration property
- * @property: enumeration property to change
- * @index: index of the new enumeration
- * @value: value of the new enumeration
- * @name: symbolic name of the new enumeration
- *
- * This functions adds enumerations to a property.
- *
- * It's use is deprecated, drivers should use one of the more specific helpers
- * to directly create the property with all enumerations already attached.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_property_add_enum(struct drm_property *property, int index,
-			  uint64_t value, const char *name)
-{
-	struct drm_property_enum *prop_enum;
-
-	if (!(drm_property_type_is(property, DRM_MODE_PROP_ENUM) ||
-			drm_property_type_is(property, DRM_MODE_PROP_BITMASK)))
-		return -EINVAL;
-
-	/*
-	 * Bitmask enum properties have the additional constraint of values
-	 * from 0 to 63
-	 */
-	if (drm_property_type_is(property, DRM_MODE_PROP_BITMASK) &&
-			(value > 63))
-		return -EINVAL;
-
-	if (!list_empty(&property->enum_list)) {
-		list_for_each_entry(prop_enum, &property->enum_list, head) {
-			if (prop_enum->value == value) {
-				strncpy(prop_enum->name, name, DRM_PROP_NAME_LEN);
-				prop_enum->name[DRM_PROP_NAME_LEN-1] = '\0';
-				return 0;
-			}
-		}
-	}
-
-	prop_enum = kzalloc(sizeof(struct drm_property_enum), GFP_KERNEL);
-	if (!prop_enum)
-		return -ENOMEM;
-
-	strncpy(prop_enum->name, name, DRM_PROP_NAME_LEN);
-	prop_enum->name[DRM_PROP_NAME_LEN-1] = '\0';
-	prop_enum->value = value;
-
-	property->values[index] = value;
-	list_add_tail(&prop_enum->head, &property->enum_list);
-	return 0;
-}
-EXPORT_SYMBOL(drm_property_add_enum);
-
-/**
- * drm_property_destroy - destroy a drm property
- * @dev: drm device
- * @property: property to destry
- *
- * This function frees a property including any attached resources like
- * enumeration values.
- */
-void drm_property_destroy(struct drm_device *dev, struct drm_property *property)
-{
-	struct drm_property_enum *prop_enum, *pt;
-
-	list_for_each_entry_safe(prop_enum, pt, &property->enum_list, head) {
-		list_del(&prop_enum->head);
-		kfree(prop_enum);
-	}
-
-	if (property->num_values)
-		kfree(property->values);
-	drm_mode_object_unregister(dev, &property->base);
-	list_del(&property->head);
-	kfree(property);
-}
-EXPORT_SYMBOL(drm_property_destroy);
-
-/**
- * drm_object_attach_property - attach a property to a modeset object
- * @obj: drm modeset object
- * @property: property to attach
- * @init_val: initial value of the property
- *
- * This attaches the given property to the modeset object with the given initial
- * value. Currently this function cannot fail since the properties are stored in
- * a statically sized array.
- */
-void drm_object_attach_property(struct drm_mode_object *obj,
-				struct drm_property *property,
-				uint64_t init_val)
-{
-	int count = obj->properties->count;
-
-	if (count == DRM_OBJECT_MAX_PROPERTY) {
-		WARN(1, "Failed to attach object property (type: 0x%x). Please "
-			"increase DRM_OBJECT_MAX_PROPERTY by 1 for each time "
-			"you see this message on the same object type.\n",
-			obj->type);
-		return;
-	}
-
-	obj->properties->properties[count] = property;
-	obj->properties->values[count] = init_val;
-	obj->properties->count++;
-	if (property->flags & DRM_MODE_PROP_ATOMIC)
-		obj->properties->atomic_count++;
-}
-EXPORT_SYMBOL(drm_object_attach_property);
-
-/**
- * drm_object_property_set_value - set the value of a property
- * @obj: drm mode object to set property value for
- * @property: property to set
- * @val: value the property should be set to
- *
- * This functions sets a given property on a given object. This function only
- * changes the software state of the property, it does not call into the
- * driver's ->set_property callback.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_object_property_set_value(struct drm_mode_object *obj,
-				  struct drm_property *property, uint64_t val)
-{
-	int i;
-
-	for (i = 0; i < obj->properties->count; i++) {
-		if (obj->properties->properties[i] == property) {
-			obj->properties->values[i] = val;
-			return 0;
-		}
-	}
-
-	return -EINVAL;
-}
-EXPORT_SYMBOL(drm_object_property_set_value);
-
-/**
- * drm_object_property_get_value - retrieve the value of a property
- * @obj: drm mode object to get property value from
- * @property: property to retrieve
- * @val: storage for the property value
- *
- * This function retrieves the softare state of the given property for the given
- * property. Since there is no driver callback to retrieve the current property
- * value this might be out of sync with the hardware, depending upon the driver
- * and property.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_object_property_get_value(struct drm_mode_object *obj,
-				  struct drm_property *property, uint64_t *val)
-{
-	int i;
-
-	/* read-only properties bypass atomic mechanism and still store
-	 * their value in obj->properties->values[].. mostly to avoid
-	 * having to deal w/ EDID and similar props in atomic paths:
-	 */
-	if (drm_core_check_feature(property->dev, DRIVER_ATOMIC) &&
-			!(property->flags & DRM_MODE_PROP_IMMUTABLE))
-		return drm_atomic_get_property(obj, property, val);
-
-	for (i = 0; i < obj->properties->count; i++) {
-		if (obj->properties->properties[i] == property) {
-			*val = obj->properties->values[i];
-			return 0;
-		}
-	}
-
-	return -EINVAL;
-}
-EXPORT_SYMBOL(drm_object_property_get_value);
-
-/**
- * drm_mode_getproperty_ioctl - get the property metadata
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * This function retrieves the metadata for a given property, like the different
- * possible values for an enum property or the limits for a range property.
- *
- * Blob properties are special
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_getproperty_ioctl(struct drm_device *dev,
-			       void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_get_property *out_resp = data;
-	struct drm_property *property;
-	int enum_count = 0;
-	int value_count = 0;
-	int ret = 0, i;
-	int copied;
-	struct drm_property_enum *prop_enum;
-	struct drm_mode_property_enum __user *enum_ptr;
-	uint64_t __user *values_ptr;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	drm_modeset_lock_all(dev);
-	property = drm_property_find(dev, out_resp->prop_id);
-	if (!property) {
-		ret = -ENOENT;
-		goto done;
-	}
-
-	if (drm_property_type_is(property, DRM_MODE_PROP_ENUM) ||
-			drm_property_type_is(property, DRM_MODE_PROP_BITMASK)) {
-		list_for_each_entry(prop_enum, &property->enum_list, head)
-			enum_count++;
-	}
-
-	value_count = property->num_values;
-
-	strncpy(out_resp->name, property->name, DRM_PROP_NAME_LEN);
-	out_resp->name[DRM_PROP_NAME_LEN-1] = 0;
-	out_resp->flags = property->flags;
-
-	if ((out_resp->count_values >= value_count) && value_count) {
-		values_ptr = (uint64_t __user *)(unsigned long)out_resp->values_ptr;
-		for (i = 0; i < value_count; i++) {
-			if (copy_to_user(values_ptr + i, &property->values[i], sizeof(uint64_t))) {
-				ret = -EFAULT;
-				goto done;
-			}
-		}
-	}
-	out_resp->count_values = value_count;
-
-	if (drm_property_type_is(property, DRM_MODE_PROP_ENUM) ||
-			drm_property_type_is(property, DRM_MODE_PROP_BITMASK)) {
-		if ((out_resp->count_enum_blobs >= enum_count) && enum_count) {
-			copied = 0;
-			enum_ptr = (struct drm_mode_property_enum __user *)(unsigned long)out_resp->enum_blob_ptr;
-			list_for_each_entry(prop_enum, &property->enum_list, head) {
-
-				if (copy_to_user(&enum_ptr[copied].value, &prop_enum->value, sizeof(uint64_t))) {
-					ret = -EFAULT;
-					goto done;
-				}
-
-				if (copy_to_user(&enum_ptr[copied].name,
-						 &prop_enum->name, DRM_PROP_NAME_LEN)) {
-					ret = -EFAULT;
-					goto done;
-				}
-				copied++;
-			}
-		}
-		out_resp->count_enum_blobs = enum_count;
-	}
-
-	/*
-	 * NOTE: The idea seems to have been to use this to read all the blob
-	 * property values. But nothing ever added them to the corresponding
-	 * list, userspace always used the special-purpose get_blob ioctl to
-	 * read the value for a blob property. It also doesn't make a lot of
-	 * sense to return values here when everything else is just metadata for
-	 * the property itself.
-	 */
-	if (drm_property_type_is(property, DRM_MODE_PROP_BLOB))
-		out_resp->count_enum_blobs = 0;
-done:
-	drm_modeset_unlock_all(dev);
-	return ret;
-}
-
-static void drm_property_free_blob(struct kref *kref)
-{
-	struct drm_property_blob *blob =
-		container_of(kref, struct drm_property_blob, base.refcount);
-
-	mutex_lock(&blob->dev->mode_config.blob_lock);
-	list_del(&blob->head_global);
-	mutex_unlock(&blob->dev->mode_config.blob_lock);
-
-	drm_mode_object_unregister(blob->dev, &blob->base);
-
-	kfree(blob);
-}
-
-/**
- * drm_property_create_blob - Create new blob property
- *
- * Creates a new blob property for a specified DRM device, optionally
- * copying data.
- *
- * @dev: DRM device to create property for
- * @length: Length to allocate for blob data
- * @data: If specified, copies data into blob
- *
- * Returns:
- * New blob property with a single reference on success, or an ERR_PTR
- * value on failure.
- */
-struct drm_property_blob *
-drm_property_create_blob(struct drm_device *dev, size_t length,
-			 const void *data)
-{
-	struct drm_property_blob *blob;
-	int ret;
-
-	if (!length || length > ULONG_MAX - sizeof(struct drm_property_blob))
-		return ERR_PTR(-EINVAL);
-
-	blob = kzalloc(sizeof(struct drm_property_blob)+length, GFP_KERNEL);
-	if (!blob)
-		return ERR_PTR(-ENOMEM);
-
-	/* This must be explicitly initialised, so we can safely call list_del
-	 * on it in the removal handler, even if it isn't in a file list. */
-	INIT_LIST_HEAD(&blob->head_file);
-	blob->length = length;
-	blob->dev = dev;
-
-	if (data)
-		memcpy(blob->data, data, length);
-
-	ret = drm_mode_object_get_reg(dev, &blob->base, DRM_MODE_OBJECT_BLOB,
-				      true, drm_property_free_blob);
-	if (ret) {
-		kfree(blob);
-		return ERR_PTR(-EINVAL);
-	}
-
-	mutex_lock(&dev->mode_config.blob_lock);
-	list_add_tail(&blob->head_global,
-	              &dev->mode_config.property_blob_list);
-	mutex_unlock(&dev->mode_config.blob_lock);
-
-	return blob;
-}
-EXPORT_SYMBOL(drm_property_create_blob);
-
-/**
- * drm_property_unreference_blob - Unreference a blob property
- *
- * Drop a reference on a blob property. May free the object.
- *
- * @blob: Pointer to blob property
- */
-void drm_property_unreference_blob(struct drm_property_blob *blob)
-{
-	if (!blob)
-		return;
-
-	drm_mode_object_unreference(&blob->base);
-}
-EXPORT_SYMBOL(drm_property_unreference_blob);
-
-/**
- * drm_property_destroy_user_blobs - destroy all blobs created by this client
- * @dev:       DRM device
- * @file_priv: destroy all blobs owned by this file handle
- */
-void drm_property_destroy_user_blobs(struct drm_device *dev,
-				     struct drm_file *file_priv)
-{
-	struct drm_property_blob *blob, *bt;
-
-	/*
-	 * When the file gets released that means no one else can access the
-	 * blob list any more, so no need to grab dev->blob_lock.
-	 */
-	list_for_each_entry_safe(blob, bt, &file_priv->blobs, head_file) {
-		list_del_init(&blob->head_file);
-		drm_property_unreference_blob(blob);
-	}
-}
-
-/**
- * drm_property_reference_blob - Take a reference on an existing property
- *
- * Take a new reference on an existing blob property.
- *
- * @blob: Pointer to blob property
- */
-struct drm_property_blob *drm_property_reference_blob(struct drm_property_blob *blob)
-{
-	drm_mode_object_reference(&blob->base);
-	return blob;
-}
-EXPORT_SYMBOL(drm_property_reference_blob);
-
-/**
- * drm_property_lookup_blob - look up a blob property and take a reference
- * @dev: drm device
- * @id: id of the blob property
- *
- * If successful, this takes an additional reference to the blob property.
- * callers need to make sure to eventually unreference the returned property
- * again, using @drm_property_unreference_blob.
- */
-struct drm_property_blob *drm_property_lookup_blob(struct drm_device *dev,
-					           uint32_t id)
-{
-	struct drm_mode_object *obj;
-	struct drm_property_blob *blob = NULL;
-
-	obj = _object_find(dev, id, DRM_MODE_OBJECT_BLOB);
-	if (obj)
-		blob = obj_to_blob(obj);
-	return blob;
-}
-EXPORT_SYMBOL(drm_property_lookup_blob);
-
-/**
- * drm_property_replace_global_blob - atomically replace existing blob property
- * @dev: drm device
- * @replace: location of blob property pointer to be replaced
- * @length: length of data for new blob, or 0 for no data
- * @data: content for new blob, or NULL for no data
- * @obj_holds_id: optional object for property holding blob ID
- * @prop_holds_id: optional property holding blob ID
- * @return 0 on success or error on failure
- *
- * This function will atomically replace a global property in the blob list,
- * optionally updating a property which holds the ID of that property. It is
- * guaranteed to be atomic: no caller will be allowed to see intermediate
- * results, and either the entire operation will succeed and clean up the
- * previous property, or it will fail and the state will be unchanged.
- *
- * If length is 0 or data is NULL, no new blob will be created, and the holding
- * property, if specified, will be set to 0.
- *
- * Access to the replace pointer is assumed to be protected by the caller, e.g.
- * by holding the relevant modesetting object lock for its parent.
- *
- * For example, a drm_connector has a 'PATH' property, which contains the ID
- * of a blob property with the value of the MST path information. Calling this
- * function with replace pointing to the connector's path_blob_ptr, length and
- * data set for the new path information, obj_holds_id set to the connector's
- * base object, and prop_holds_id set to the path property name, will perform
- * a completely atomic update. The access to path_blob_ptr is protected by the
- * caller holding a lock on the connector.
- */
-static int drm_property_replace_global_blob(struct drm_device *dev,
-                                            struct drm_property_blob **replace,
-                                            size_t length,
-                                            const void *data,
-                                            struct drm_mode_object *obj_holds_id,
-                                            struct drm_property *prop_holds_id)
-{
-	struct drm_property_blob *new_blob = NULL;
-	struct drm_property_blob *old_blob = NULL;
-	int ret;
-
-	WARN_ON(replace == NULL);
-
-	old_blob = *replace;
-
-	if (length && data) {
-		new_blob = drm_property_create_blob(dev, length, data);
-		if (IS_ERR(new_blob))
-			return PTR_ERR(new_blob);
-	}
-
-	/* This does not need to be synchronised with blob_lock, as the
-	 * get_properties ioctl locks all modesetting objects, and
-	 * obj_holds_id must be locked before calling here, so we cannot
-	 * have its value out of sync with the list membership modified
-	 * below under blob_lock. */
-	if (obj_holds_id) {
-		ret = drm_object_property_set_value(obj_holds_id,
-						    prop_holds_id,
-						    new_blob ?
-						        new_blob->base.id : 0);
-		if (ret != 0)
-			goto err_created;
-	}
-
-	drm_property_unreference_blob(old_blob);
-	*replace = new_blob;
-
-	return 0;
-
-err_created:
-	drm_property_unreference_blob(new_blob);
-	return ret;
-}
-
-/**
- * drm_mode_getblob_ioctl - get the contents of a blob property value
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * This function retrieves the contents of a blob property. The value stored in
- * an object's blob property is just a normal modeset object id.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_getblob_ioctl(struct drm_device *dev,
-			   void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_get_blob *out_resp = data;
-	struct drm_property_blob *blob;
-	int ret = 0;
-	void __user *blob_ptr;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	blob = drm_property_lookup_blob(dev, out_resp->blob_id);
-	if (!blob)
-		return -ENOENT;
-
-	if (out_resp->length == blob->length) {
-		blob_ptr = (void __user *)(unsigned long)out_resp->data;
-		if (copy_to_user(blob_ptr, blob->data, blob->length)) {
-			ret = -EFAULT;
-			goto unref;
-		}
-	}
-	out_resp->length = blob->length;
-unref:
-	drm_property_unreference_blob(blob);
-
-	return ret;
-}
-
-/**
- * drm_mode_createblob_ioctl - create a new blob property
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * This function creates a new blob property with user-defined values. In order
- * to give us sensible validation and checking when creating, rather than at
- * every potential use, we also require a type to be provided upfront.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_createblob_ioctl(struct drm_device *dev,
-			      void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_create_blob *out_resp = data;
-	struct drm_property_blob *blob;
-	void __user *blob_ptr;
-	int ret = 0;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	blob = drm_property_create_blob(dev, out_resp->length, NULL);
-	if (IS_ERR(blob))
-		return PTR_ERR(blob);
-
-	blob_ptr = (void __user *)(unsigned long)out_resp->data;
-	if (copy_from_user(blob->data, blob_ptr, out_resp->length)) {
-		ret = -EFAULT;
-		goto out_blob;
-	}
-
-	/* Dropping the lock between create_blob and our access here is safe
-	 * as only the same file_priv can remove the blob; at this point, it is
-	 * not associated with any file_priv. */
-	mutex_lock(&dev->mode_config.blob_lock);
-	out_resp->blob_id = blob->base.id;
-	list_add_tail(&blob->head_file, &file_priv->blobs);
-	mutex_unlock(&dev->mode_config.blob_lock);
-
-	return 0;
-
-out_blob:
-	drm_property_unreference_blob(blob);
-	return ret;
-}
-
-/**
- * drm_mode_destroyblob_ioctl - destroy a user blob property
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * Destroy an existing user-defined blob property.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_destroyblob_ioctl(struct drm_device *dev,
-			       void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_destroy_blob *out_resp = data;
-	struct drm_property_blob *blob = NULL, *bt;
-	bool found = false;
-	int ret = 0;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	blob = drm_property_lookup_blob(dev, out_resp->blob_id);
-	if (!blob)
-		return -ENOENT;
-
-	mutex_lock(&dev->mode_config.blob_lock);
-	/* Ensure the property was actually created by this user. */
-	list_for_each_entry(bt, &file_priv->blobs, head_file) {
-		if (bt == blob) {
-			found = true;
-			break;
-		}
-	}
-
-	if (!found) {
-		ret = -EPERM;
-		goto err;
-	}
-
-	/* We must drop head_file here, because we may not be the last
-	 * reference on the blob. */
-	list_del_init(&blob->head_file);
-	mutex_unlock(&dev->mode_config.blob_lock);
-
-	/* One reference from lookup, and one from the filp. */
-	drm_property_unreference_blob(blob);
-	drm_property_unreference_blob(blob);
-
-	return 0;
-
-err:
-	mutex_unlock(&dev->mode_config.blob_lock);
-	drm_property_unreference_blob(blob);
-
-	return ret;
-}
-
-/**
- * drm_mode_connector_set_path_property - set tile property on connector
- * @connector: connector to set property on.
- * @path: path to use for property; must not be NULL.
- *
- * This creates a property to expose to userspace to specify a
- * connector path. This is mainly used for DisplayPort MST where
- * connectors have a topology and we want to allow userspace to give
- * them more meaningful names.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_connector_set_path_property(struct drm_connector *connector,
-					 const char *path)
-{
-	struct drm_device *dev = connector->dev;
-	int ret;
-
-	ret = drm_property_replace_global_blob(dev,
-	                                       &connector->path_blob_ptr,
-	                                       strlen(path) + 1,
-	                                       path,
-	                                       &connector->base,
-	                                       dev->mode_config.path_property);
-	return ret;
-}
-EXPORT_SYMBOL(drm_mode_connector_set_path_property);
-
-/**
- * drm_mode_connector_set_tile_property - set tile property on connector
- * @connector: connector to set property on.
- *
- * This looks up the tile information for a connector, and creates a
- * property for userspace to parse if it exists. The property is of
- * the form of 8 integers using ':' as a separator.
- *
- * Returns:
- * Zero on success, errno on failure.
- */
-int drm_mode_connector_set_tile_property(struct drm_connector *connector)
-{
-	struct drm_device *dev = connector->dev;
-	char tile[256];
-	int ret;
-
-	if (!connector->has_tile) {
-		ret  = drm_property_replace_global_blob(dev,
-		                                        &connector->tile_blob_ptr,
-		                                        0,
-		                                        NULL,
-		                                        &connector->base,
-		                                        dev->mode_config.tile_property);
-		return ret;
-	}
-
-	snprintf(tile, 256, "%d:%d:%d:%d:%d:%d:%d:%d",
-		 connector->tile_group->id, connector->tile_is_single_monitor,
-		 connector->num_h_tile, connector->num_v_tile,
-		 connector->tile_h_loc, connector->tile_v_loc,
-		 connector->tile_h_size, connector->tile_v_size);
-
-	ret = drm_property_replace_global_blob(dev,
-	                                       &connector->tile_blob_ptr,
-	                                       strlen(tile) + 1,
-	                                       tile,
-	                                       &connector->base,
-	                                       dev->mode_config.tile_property);
-	return ret;
-}
-EXPORT_SYMBOL(drm_mode_connector_set_tile_property);
-
-/**
- * drm_mode_connector_update_edid_property - update the edid property of a connector
- * @connector: drm connector
- * @edid: new value of the edid property
- *
- * This function creates a new blob modeset object and assigns its id to the
- * connector's edid property.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_connector_update_edid_property(struct drm_connector *connector,
-					    const struct edid *edid)
-{
-	struct drm_device *dev = connector->dev;
-	size_t size = 0;
-	int ret;
-
-	/* ignore requests to set edid when overridden */
-	if (connector->override_edid)
-		return 0;
-
-	if (edid)
-		size = EDID_LENGTH * (1 + edid->extensions);
-
-	ret = drm_property_replace_global_blob(dev,
-					       &connector->edid_blob_ptr,
-	                                       size,
-	                                       edid,
-	                                       &connector->base,
-	                                       dev->mode_config.edid_property);
-	return ret;
-}
-EXPORT_SYMBOL(drm_mode_connector_update_edid_property);
-
-/* Some properties could refer to dynamic refcnt'd objects, or things that
- * need special locking to handle lifetime issues (ie. to ensure the prop
- * value doesn't become invalid part way through the property update due to
- * race).  The value returned by reference via 'obj' should be passed back
- * to drm_property_change_valid_put() after the property is set (and the
- * object to which the property is attached has a chance to take it's own
- * reference).
- */
-bool drm_property_change_valid_get(struct drm_property *property,
-					 uint64_t value, struct drm_mode_object **ref)
-{
-	int i;
-
-	if (property->flags & DRM_MODE_PROP_IMMUTABLE)
-		return false;
-
-	*ref = NULL;
-
-	if (drm_property_type_is(property, DRM_MODE_PROP_RANGE)) {
-		if (value < property->values[0] || value > property->values[1])
-			return false;
-		return true;
-	} else if (drm_property_type_is(property, DRM_MODE_PROP_SIGNED_RANGE)) {
-		int64_t svalue = U642I64(value);
-
-		if (svalue < U642I64(property->values[0]) ||
-				svalue > U642I64(property->values[1]))
-			return false;
-		return true;
-	} else if (drm_property_type_is(property, DRM_MODE_PROP_BITMASK)) {
-		uint64_t valid_mask = 0;
-
-		for (i = 0; i < property->num_values; i++)
-			valid_mask |= (1ULL << property->values[i]);
-		return !(value & ~valid_mask);
-	} else if (drm_property_type_is(property, DRM_MODE_PROP_BLOB)) {
-		struct drm_property_blob *blob;
-
-		if (value == 0)
-			return true;
-
-		blob = drm_property_lookup_blob(property->dev, value);
-		if (blob) {
-			*ref = &blob->base;
-			return true;
-		} else {
-			return false;
-		}
-	} else if (drm_property_type_is(property, DRM_MODE_PROP_OBJECT)) {
-		/* a zero value for an object property translates to null: */
-		if (value == 0)
-			return true;
-
-		*ref = _object_find(property->dev, value, property->values[0]);
-		return *ref != NULL;
-	}
-
-	for (i = 0; i < property->num_values; i++)
-		if (property->values[i] == value)
-			return true;
-	return false;
-}
-
-void drm_property_change_valid_put(struct drm_property *property,
-		struct drm_mode_object *ref)
-{
-	if (!ref)
-		return;
-
-	if (drm_property_type_is(property, DRM_MODE_PROP_OBJECT)) {
-		drm_mode_object_unreference(ref);
-	} else if (drm_property_type_is(property, DRM_MODE_PROP_BLOB))
-		drm_property_unreference_blob(obj_to_blob(ref));
-}
-
-/**
- * drm_mode_connector_property_set_ioctl - set the current value of a connector property
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * This function sets the current value for a connectors's property. It also
- * calls into a driver's ->set_property callback to update the hardware state
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_connector_property_set_ioctl(struct drm_device *dev,
-				       void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_connector_set_property *conn_set_prop = data;
-	struct drm_mode_obj_set_property obj_set_prop = {
-		.value = conn_set_prop->value,
-		.prop_id = conn_set_prop->prop_id,
-		.obj_id = conn_set_prop->connector_id,
-		.obj_type = DRM_MODE_OBJECT_CONNECTOR
-	};
-
-	/* It does all the locking and checking we need */
-	return drm_mode_obj_set_property_ioctl(dev, &obj_set_prop, file_priv);
-}
-
-static int drm_mode_connector_set_obj_prop(struct drm_mode_object *obj,
-					   struct drm_property *property,
-					   uint64_t value)
-{
-	int ret = -EINVAL;
-	struct drm_connector *connector = obj_to_connector(obj);
-
-	/* Do DPMS ourselves */
-	if (property == connector->dev->mode_config.dpms_property) {
-		ret = (*connector->funcs->dpms)(connector, (int)value);
-	} else if (connector->funcs->set_property)
-		ret = connector->funcs->set_property(connector, property, value);
-
-	/* store the property value if successful */
-	if (!ret)
-		drm_object_property_set_value(&connector->base, property, value);
-	return ret;
-}
-
-static int drm_mode_crtc_set_obj_prop(struct drm_mode_object *obj,
-				      struct drm_property *property,
-				      uint64_t value)
+int drm_mode_crtc_set_obj_prop(struct drm_mode_object *obj,
+			       struct drm_property *property,
+			       uint64_t value)
 {
 	int ret = -EINVAL;
 	struct drm_crtc *crtc = obj_to_crtc(obj);
@@ -5006,494 +904,6 @@ static int drm_mode_crtc_set_obj_prop(struct drm_mode_object *obj,
 }
 
 /**
- * drm_mode_plane_set_obj_prop - set the value of a property
- * @plane: drm plane object to set property value for
- * @property: property to set
- * @value: value the property should be set to
- *
- * This functions sets a given property on a given plane object. This function
- * calls the driver's ->set_property callback and changes the software state of
- * the property if the callback succeeds.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_mode_plane_set_obj_prop(struct drm_plane *plane,
-				struct drm_property *property,
-				uint64_t value)
-{
-	int ret = -EINVAL;
-	struct drm_mode_object *obj = &plane->base;
-
-	if (plane->funcs->set_property)
-		ret = plane->funcs->set_property(plane, property, value);
-	if (!ret)
-		drm_object_property_set_value(obj, property, value);
-
-	return ret;
-}
-EXPORT_SYMBOL(drm_mode_plane_set_obj_prop);
-
-/**
- * drm_mode_obj_get_properties_ioctl - get the current value of a object's property
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * This function retrieves the current value for an object's property. Compared
- * to the connector specific ioctl this one is extended to also work on crtc and
- * plane objects.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data,
-				      struct drm_file *file_priv)
-{
-	struct drm_mode_obj_get_properties *arg = data;
-	struct drm_mode_object *obj;
-	int ret = 0;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	drm_modeset_lock_all(dev);
-
-	obj = drm_mode_object_find(dev, arg->obj_id, arg->obj_type);
-	if (!obj) {
-		ret = -ENOENT;
-		goto out;
-	}
-	if (!obj->properties) {
-		ret = -EINVAL;
-		goto out_unref;
-	}
-
-	ret = get_properties(obj, file_priv->atomic,
-			(uint32_t __user *)(unsigned long)(arg->props_ptr),
-			(uint64_t __user *)(unsigned long)(arg->prop_values_ptr),
-			&arg->count_props);
-
-out_unref:
-	drm_mode_object_unreference(obj);
-out:
-	drm_modeset_unlock_all(dev);
-	return ret;
-}
-
-/**
- * drm_mode_obj_set_property_ioctl - set the current value of an object's property
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * This function sets the current value for an object's property. It also calls
- * into a driver's ->set_property callback to update the hardware state.
- * Compared to the connector specific ioctl this one is extended to also work on
- * crtc and plane objects.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
-				    struct drm_file *file_priv)
-{
-	struct drm_mode_obj_set_property *arg = data;
-	struct drm_mode_object *arg_obj;
-	struct drm_mode_object *prop_obj;
-	struct drm_property *property;
-	int i, ret = -EINVAL;
-	struct drm_mode_object *ref;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	drm_modeset_lock_all(dev);
-
-	arg_obj = drm_mode_object_find(dev, arg->obj_id, arg->obj_type);
-	if (!arg_obj) {
-		ret = -ENOENT;
-		goto out;
-	}
-	if (!arg_obj->properties)
-		goto out_unref;
-
-	for (i = 0; i < arg_obj->properties->count; i++)
-		if (arg_obj->properties->properties[i]->base.id == arg->prop_id)
-			break;
-
-	if (i == arg_obj->properties->count)
-		goto out_unref;
-
-	prop_obj = drm_mode_object_find(dev, arg->prop_id,
-					DRM_MODE_OBJECT_PROPERTY);
-	if (!prop_obj) {
-		ret = -ENOENT;
-		goto out_unref;
-	}
-	property = obj_to_property(prop_obj);
-
-	if (!drm_property_change_valid_get(property, arg->value, &ref))
-		goto out_unref;
-
-	switch (arg_obj->type) {
-	case DRM_MODE_OBJECT_CONNECTOR:
-		ret = drm_mode_connector_set_obj_prop(arg_obj, property,
-						      arg->value);
-		break;
-	case DRM_MODE_OBJECT_CRTC:
-		ret = drm_mode_crtc_set_obj_prop(arg_obj, property, arg->value);
-		break;
-	case DRM_MODE_OBJECT_PLANE:
-		ret = drm_mode_plane_set_obj_prop(obj_to_plane(arg_obj),
-						  property, arg->value);
-		break;
-	}
-
-	drm_property_change_valid_put(property, ref);
-
-out_unref:
-	drm_mode_object_unreference(arg_obj);
-out:
-	drm_modeset_unlock_all(dev);
-	return ret;
-}
-
-/**
- * drm_mode_connector_attach_encoder - attach a connector to an encoder
- * @connector: connector to attach
- * @encoder: encoder to attach @connector to
- *
- * This function links up a connector to an encoder. Note that the routing
- * restrictions between encoders and crtcs are exposed to userspace through the
- * possible_clones and possible_crtcs bitmasks.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_connector_attach_encoder(struct drm_connector *connector,
-				      struct drm_encoder *encoder)
-{
-	int i;
-
-	/*
-	 * In the past, drivers have attempted to model the static association
-	 * of connector to encoder in simple connector/encoder devices using a
-	 * direct assignment of connector->encoder = encoder. This connection
-	 * is a logical one and the responsibility of the core, so drivers are
-	 * expected not to mess with this.
-	 *
-	 * Note that the error return should've been enough here, but a large
-	 * majority of drivers ignores the return value, so add in a big WARN
-	 * to get people's attention.
-	 */
-	if (WARN_ON(connector->encoder))
-		return -EINVAL;
-
-	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
-		if (connector->encoder_ids[i] == 0) {
-			connector->encoder_ids[i] = encoder->base.id;
-			return 0;
-		}
-	}
-	return -ENOMEM;
-}
-EXPORT_SYMBOL(drm_mode_connector_attach_encoder);
-
-/**
- * drm_mode_crtc_set_gamma_size - set the gamma table size
- * @crtc: CRTC to set the gamma table size for
- * @gamma_size: size of the gamma table
- *
- * Drivers which support gamma tables should set this to the supported gamma
- * table size when initializing the CRTC. Currently the drm core only supports a
- * fixed gamma table size.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_crtc_set_gamma_size(struct drm_crtc *crtc,
-				 int gamma_size)
-{
-	uint16_t *r_base, *g_base, *b_base;
-	int i;
-
-	crtc->gamma_size = gamma_size;
-
-	crtc->gamma_store = kcalloc(gamma_size, sizeof(uint16_t) * 3,
-				    GFP_KERNEL);
-	if (!crtc->gamma_store) {
-		crtc->gamma_size = 0;
-		return -ENOMEM;
-	}
-
-	r_base = crtc->gamma_store;
-	g_base = r_base + gamma_size;
-	b_base = g_base + gamma_size;
-	for (i = 0; i < gamma_size; i++) {
-		r_base[i] = i << 8;
-		g_base[i] = i << 8;
-		b_base[i] = i << 8;
-	}
-
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_mode_crtc_set_gamma_size);
-
-/**
- * drm_mode_gamma_set_ioctl - set the gamma table
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * Set the gamma table of a CRTC to the one passed in by the user. Userspace can
- * inquire the required gamma table size through drm_mode_gamma_get_ioctl.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_gamma_set_ioctl(struct drm_device *dev,
-			     void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_crtc_lut *crtc_lut = data;
-	struct drm_crtc *crtc;
-	void *r_base, *g_base, *b_base;
-	int size;
-	int ret = 0;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	drm_modeset_lock_all(dev);
-	crtc = drm_crtc_find(dev, crtc_lut->crtc_id);
-	if (!crtc) {
-		ret = -ENOENT;
-		goto out;
-	}
-
-	if (crtc->funcs->gamma_set == NULL) {
-		ret = -ENOSYS;
-		goto out;
-	}
-
-	/* memcpy into gamma store */
-	if (crtc_lut->gamma_size != crtc->gamma_size) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	size = crtc_lut->gamma_size * (sizeof(uint16_t));
-	r_base = crtc->gamma_store;
-	if (copy_from_user(r_base, (void __user *)(unsigned long)crtc_lut->red, size)) {
-		ret = -EFAULT;
-		goto out;
-	}
-
-	g_base = r_base + size;
-	if (copy_from_user(g_base, (void __user *)(unsigned long)crtc_lut->green, size)) {
-		ret = -EFAULT;
-		goto out;
-	}
-
-	b_base = g_base + size;
-	if (copy_from_user(b_base, (void __user *)(unsigned long)crtc_lut->blue, size)) {
-		ret = -EFAULT;
-		goto out;
-	}
-
-	ret = crtc->funcs->gamma_set(crtc, r_base, g_base, b_base, crtc->gamma_size);
-
-out:
-	drm_modeset_unlock_all(dev);
-	return ret;
-
-}
-
-/**
- * drm_mode_gamma_get_ioctl - get the gamma table
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * Copy the current gamma table into the storage provided. This also provides
- * the gamma table size the driver expects, which can be used to size the
- * allocated storage.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_gamma_get_ioctl(struct drm_device *dev,
-			     void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_crtc_lut *crtc_lut = data;
-	struct drm_crtc *crtc;
-	void *r_base, *g_base, *b_base;
-	int size;
-	int ret = 0;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-	drm_modeset_lock_all(dev);
-	crtc = drm_crtc_find(dev, crtc_lut->crtc_id);
-	if (!crtc) {
-		ret = -ENOENT;
-		goto out;
-	}
-
-	/* memcpy into gamma store */
-	if (crtc_lut->gamma_size != crtc->gamma_size) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	size = crtc_lut->gamma_size * (sizeof(uint16_t));
-	r_base = crtc->gamma_store;
-	if (copy_to_user((void __user *)(unsigned long)crtc_lut->red, r_base, size)) {
-		ret = -EFAULT;
-		goto out;
-	}
-
-	g_base = r_base + size;
-	if (copy_to_user((void __user *)(unsigned long)crtc_lut->green, g_base, size)) {
-		ret = -EFAULT;
-		goto out;
-	}
-
-	b_base = g_base + size;
-	if (copy_to_user((void __user *)(unsigned long)crtc_lut->blue, b_base, size)) {
-		ret = -EFAULT;
-		goto out;
-	}
-out:
-	drm_modeset_unlock_all(dev);
-	return ret;
-}
-
-/**
- * drm_mode_page_flip_ioctl - schedule an asynchronous fb update
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * This schedules an asynchronous update on a given CRTC, called page flip.
- * Optionally a drm event is generated to signal the completion of the event.
- * Generic drivers cannot assume that a pageflip with changed framebuffer
- * properties (including driver specific metadata like tiling layout) will work,
- * but some drivers support e.g. pixel format changes through the pageflip
- * ioctl.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_page_flip_ioctl(struct drm_device *dev,
-			     void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_crtc_page_flip *page_flip = data;
-	struct drm_crtc *crtc;
-	struct drm_framebuffer *fb = NULL;
-	struct drm_pending_vblank_event *e = NULL;
-	int ret = -EINVAL;
-
-	if (page_flip->flags & ~DRM_MODE_PAGE_FLIP_FLAGS ||
-	    page_flip->reserved != 0)
-		return -EINVAL;
-
-	if ((page_flip->flags & DRM_MODE_PAGE_FLIP_ASYNC) && !dev->mode_config.async_page_flip)
-		return -EINVAL;
-
-	crtc = drm_crtc_find(dev, page_flip->crtc_id);
-	if (!crtc)
-		return -ENOENT;
-
-	drm_modeset_lock_crtc(crtc, crtc->primary);
-	if (crtc->primary->fb == NULL) {
-		/* The framebuffer is currently unbound, presumably
-		 * due to a hotplug event, that userspace has not
-		 * yet discovered.
-		 */
-		ret = -EBUSY;
-		goto out;
-	}
-
-	if (crtc->funcs->page_flip == NULL)
-		goto out;
-
-	fb = drm_framebuffer_lookup(dev, page_flip->fb_id);
-	if (!fb) {
-		ret = -ENOENT;
-		goto out;
-	}
-
-	if (crtc->state) {
-		const struct drm_plane_state *state = crtc->primary->state;
-
-		ret = check_src_coords(state->src_x, state->src_y,
-				       state->src_w, state->src_h, fb);
-	} else {
-		ret = drm_crtc_check_viewport(crtc, crtc->x, crtc->y, &crtc->mode, fb);
-	}
-	if (ret)
-		goto out;
-
-	if (crtc->primary->fb->pixel_format != fb->pixel_format) {
-		DRM_DEBUG_KMS("Page flip is not allowed to change frame buffer format.\n");
-		ret = -EINVAL;
-		goto out;
-	}
-
-	if (page_flip->flags & DRM_MODE_PAGE_FLIP_EVENT) {
-		e = kzalloc(sizeof *e, GFP_KERNEL);
-		if (!e) {
-			ret = -ENOMEM;
-			goto out;
-		}
-		e->event.base.type = DRM_EVENT_FLIP_COMPLETE;
-		e->event.base.length = sizeof(e->event);
-		e->event.user_data = page_flip->user_data;
-		ret = drm_event_reserve_init(dev, file_priv, &e->base, &e->event.base);
-		if (ret) {
-			kfree(e);
-			goto out;
-		}
-	}
-
-	crtc->primary->old_fb = crtc->primary->fb;
-	ret = crtc->funcs->page_flip(crtc, fb, e, page_flip->flags);
-	if (ret) {
-		if (page_flip->flags & DRM_MODE_PAGE_FLIP_EVENT)
-			drm_event_cancel_free(dev, &e->base);
-		/* Keep the old fb, don't unref it. */
-		crtc->primary->old_fb = NULL;
-	} else {
-		crtc->primary->fb = fb;
-		/* Unref only the old framebuffer. */
-		fb = NULL;
-	}
-
-out:
-	if (fb)
-		drm_framebuffer_unreference(fb);
-	if (crtc->primary->old_fb)
-		drm_framebuffer_unreference(crtc->primary->old_fb);
-	crtc->primary->old_fb = NULL;
-	drm_modeset_unlock_crtc(crtc);
-
-	return ret;
-}
-
-/**
  * drm_mode_config_reset - call ->reset callbacks
  * @dev: drm device
  *
@@ -5638,37 +1048,6 @@ int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
 }
 
 /**
- * drm_rotation_simplify() - Try to simplify the rotation
- * @rotation: Rotation to be simplified
- * @supported_rotations: Supported rotations
- *
- * Attempt to simplify the rotation to a form that is supported.
- * Eg. if the hardware supports everything except DRM_REFLECT_X
- * one could call this function like this:
- *
- * drm_rotation_simplify(rotation, BIT(DRM_ROTATE_0) |
- *                       BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_180) |
- *                       BIT(DRM_ROTATE_270) | BIT(DRM_REFLECT_Y));
- *
- * to eliminate the DRM_ROTATE_X flag. Depending on what kind of
- * transforms the hardware supports, this function may not
- * be able to produce a supported transform, so the caller should
- * check the result afterwards.
- */
-unsigned int drm_rotation_simplify(unsigned int rotation,
-				   unsigned int supported_rotations)
-{
-	if (rotation & ~supported_rotations) {
-		rotation ^= BIT(DRM_REFLECT_X) | BIT(DRM_REFLECT_Y);
-		rotation = (rotation & DRM_REFLECT_MASK) |
-		           BIT((ffs(rotation & DRM_ROTATE_MASK) + 1) % 4);
-	}
-
-	return rotation;
-}
-EXPORT_SYMBOL(drm_rotation_simplify);
-
-/**
  * drm_mode_config_init - initialize DRM mode_configuration structure
  * @dev: DRM device
  *
@@ -5784,24 +1163,6 @@ void drm_mode_config_cleanup(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_mode_config_cleanup);
 
-struct drm_property *drm_mode_create_rotation_property(struct drm_device *dev,
-						       unsigned int supported_rotations)
-{
-	static const struct drm_prop_enum_list props[] = {
-		{ DRM_ROTATE_0,   "rotate-0" },
-		{ DRM_ROTATE_90,  "rotate-90" },
-		{ DRM_ROTATE_180, "rotate-180" },
-		{ DRM_ROTATE_270, "rotate-270" },
-		{ DRM_REFLECT_X,  "reflect-x" },
-		{ DRM_REFLECT_Y,  "reflect-y" },
-	};
-
-	return drm_property_create_bitmask(dev, 0, "rotation",
-					   props, ARRAY_SIZE(props),
-					   supported_rotations);
-}
-EXPORT_SYMBOL(drm_mode_create_rotation_property);
-
 /**
  * DOC: Tile group
  *
@@ -5900,48 +1261,3 @@ struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
 	return tg;
 }
 EXPORT_SYMBOL(drm_mode_create_tile_group);
-
-/**
- * drm_crtc_enable_color_mgmt - enable color management properties
- * @crtc: DRM CRTC
- * @degamma_lut_size: the size of the degamma lut (before CSC)
- * @has_ctm: whether to attach ctm_property for CSC matrix
- * @gamma_lut_size: the size of the gamma lut (after CSC)
- *
- * This function lets the driver enable the color correction
- * properties on a CRTC. This includes 3 degamma, csc and gamma
- * properties that userspace can set and 2 size properties to inform
- * the userspace of the lut sizes. Each of the properties are
- * optional. The gamma and degamma properties are only attached if
- * their size is not 0 and ctm_property is only attached if has_ctm is
- * true.
- */
-void drm_crtc_enable_color_mgmt(struct drm_crtc *crtc,
-				uint degamma_lut_size,
-				bool has_ctm,
-				uint gamma_lut_size)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_mode_config *config = &dev->mode_config;
-
-	if (degamma_lut_size) {
-		drm_object_attach_property(&crtc->base,
-					   config->degamma_lut_property, 0);
-		drm_object_attach_property(&crtc->base,
-					   config->degamma_lut_size_property,
-					   degamma_lut_size);
-	}
-
-	if (has_ctm)
-		drm_object_attach_property(&crtc->base,
-					   config->ctm_property, 0);
-
-	if (gamma_lut_size) {
-		drm_object_attach_property(&crtc->base,
-					   config->gamma_lut_property, 0);
-		drm_object_attach_property(&crtc->base,
-					   config->gamma_lut_size_property,
-					   gamma_lut_size);
-	}
-}
-EXPORT_SYMBOL(drm_crtc_enable_color_mgmt);
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 604d3ef72ffa..5d2cb138eba6 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -75,35 +75,6 @@
  */
 
 /**
- * drm_helper_move_panel_connectors_to_head() - move panels to the front in the
- * 						connector list
- * @dev: drm device to operate on
- *
- * Some userspace presumes that the first connected connector is the main
- * display, where it's supposed to display e.g. the login screen. For
- * laptops, this should be the main panel. Use this function to sort all
- * (eDP/LVDS) panels to the front of the connector list, instead of
- * painstakingly trying to initialize them in the right order.
- */
-void drm_helper_move_panel_connectors_to_head(struct drm_device *dev)
-{
-	struct drm_connector *connector, *tmp;
-	struct list_head panel_list;
-
-	INIT_LIST_HEAD(&panel_list);
-
-	list_for_each_entry_safe(connector, tmp,
-				 &dev->mode_config.connector_list, head) {
-		if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS ||
-		    connector->connector_type == DRM_MODE_CONNECTOR_eDP)
-			list_move_tail(&connector->head, &panel_list);
-	}
-
-	list_splice(&panel_list, &dev->mode_config.connector_list);
-}
-EXPORT_SYMBOL(drm_helper_move_panel_connectors_to_head);
-
-/**
  * drm_helper_encoder_in_use - check if a given encoder is in use
  * @encoder: encoder to check
  *
@@ -913,33 +884,6 @@ int drm_helper_connector_dpms(struct drm_connector *connector, int mode)
 EXPORT_SYMBOL(drm_helper_connector_dpms);
 
 /**
- * drm_helper_mode_fill_fb_struct - fill out framebuffer metadata
- * @fb: drm_framebuffer object to fill out
- * @mode_cmd: metadata from the userspace fb creation request
- *
- * This helper can be used in a drivers fb_create callback to pre-fill the fb's
- * metadata fields.
- */
-void drm_helper_mode_fill_fb_struct(struct drm_framebuffer *fb,
-				    const struct drm_mode_fb_cmd2 *mode_cmd)
-{
-	int i;
-
-	fb->width = mode_cmd->width;
-	fb->height = mode_cmd->height;
-	for (i = 0; i < 4; i++) {
-		fb->pitches[i] = mode_cmd->pitches[i];
-		fb->offsets[i] = mode_cmd->offsets[i];
-		fb->modifier[i] = mode_cmd->modifier[i];
-	}
-	drm_fb_get_bpp_depth(mode_cmd->pixel_format, &fb->depth,
-				    &fb->bits_per_pixel);
-	fb->pixel_format = mode_cmd->pixel_format;
-	fb->flags = mode_cmd->flags;
-}
-EXPORT_SYMBOL(drm_helper_mode_fill_fb_struct);
-
-/**
  * drm_helper_resume_force_mode - force-restore mode setting configuration
  * @dev: drm_device which should be restored
  *
diff --git a/drivers/gpu/drm/drm_crtc_helper_internal.h b/drivers/gpu/drm/drm_crtc_helper_internal.h
new file mode 100644
index 000000000000..28295e5d0d9e
--- /dev/null
+++ b/drivers/gpu/drm/drm_crtc_helper_internal.h
@@ -0,0 +1,65 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This header file contains mode setting related functions and definitions
+ * which are only used within the drm kms helper module as internal
+ * implementation details and are not exported to drivers.
+ */
+
+#include <drm/drm_dp_helper.h>
+
+/* drm_fb_helper.c */
+#ifdef CONFIG_DRM_FBDEV_EMULATION
+int drm_fb_helper_modinit(void);
+#else
+static inline int drm_fb_helper_modinit(void)
+{
+	return 0;
+}
+#endif
+
+/* drm_dp_aux_dev.c */
+#ifdef CONFIG_DRM_DP_AUX_CHARDEV
+int drm_dp_aux_dev_init(void);
+void drm_dp_aux_dev_exit(void);
+int drm_dp_aux_register_devnode(struct drm_dp_aux *aux);
+void drm_dp_aux_unregister_devnode(struct drm_dp_aux *aux);
+#else
+static inline int drm_dp_aux_dev_init(void)
+{
+	return 0;
+}
+
+static inline void drm_dp_aux_dev_exit(void)
+{
+}
+
+static inline int drm_dp_aux_register_devnode(struct drm_dp_aux *aux)
+{
+	return 0;
+}
+
+static inline void drm_dp_aux_unregister_devnode(struct drm_dp_aux *aux)
+{
+}
+#endif
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index 0c34e6d906d1..c48ba02c5365 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -33,28 +33,15 @@
 
 
 /* drm_crtc.c */
-void drm_connector_ida_init(void);
-void drm_connector_ida_destroy(void);
-int drm_mode_object_get(struct drm_device *dev,
-			struct drm_mode_object *obj, uint32_t obj_type);
-void drm_mode_object_unregister(struct drm_device *dev,
-				struct drm_mode_object *object);
-bool drm_property_change_valid_get(struct drm_property *property,
-				   uint64_t value,
-				   struct drm_mode_object **ref);
-void drm_property_change_valid_put(struct drm_property *property,
-				   struct drm_mode_object *ref);
-
-int drm_plane_check_pixel_format(const struct drm_plane *plane,
-				 u32 format);
+int drm_mode_crtc_set_obj_prop(struct drm_mode_object *obj,
+			       struct drm_property *property,
+			       uint64_t value);
 int drm_crtc_check_viewport(const struct drm_crtc *crtc,
 			    int x, int y,
 			    const struct drm_display_mode *mode,
 			    const struct drm_framebuffer *fb);
 
 void drm_fb_release(struct drm_file *file_priv);
-void drm_property_destroy_user_blobs(struct drm_device *dev,
-				     struct drm_file *file_priv);
 
 /* dumb buffer support IOCTLs */
 int drm_mode_create_dumb_ioctl(struct drm_device *dev,
@@ -64,42 +51,32 @@ int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
 int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
 				void *data, struct drm_file *file_priv);
 
-/* framebuffer IOCTLs */
-extern int drm_mode_addfb(struct drm_device *dev,
-			  void *data, struct drm_file *file_priv);
-extern int drm_mode_addfb2(struct drm_device *dev,
-			   void *data, struct drm_file *file_priv);
-int drm_mode_rmfb(struct drm_device *dev,
-			 void *data, struct drm_file *file_priv);
-int drm_mode_getfb(struct drm_device *dev,
-		   void *data, struct drm_file *file_priv);
-int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
-			   void *data, struct drm_file *file_priv);
-
 /* IOCTLs */
-int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data,
-				      struct drm_file *file_priv);
-int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
-				    struct drm_file *file_priv);
-
 int drm_mode_getresources(struct drm_device *dev,
 			  void *data, struct drm_file *file_priv);
-int drm_mode_getplane_res(struct drm_device *dev, void *data,
-			  struct drm_file *file_priv);
 int drm_mode_getcrtc(struct drm_device *dev,
 		     void *data, struct drm_file *file_priv);
-int drm_mode_getconnector(struct drm_device *dev,
-			  void *data, struct drm_file *file_priv);
 int drm_mode_setcrtc(struct drm_device *dev,
 		     void *data, struct drm_file *file_priv);
-int drm_mode_getplane(struct drm_device *dev,
-		      void *data, struct drm_file *file_priv);
-int drm_mode_setplane(struct drm_device *dev,
-		      void *data, struct drm_file *file_priv);
-int drm_mode_cursor_ioctl(struct drm_device *dev,
-			  void *data, struct drm_file *file_priv);
-int drm_mode_cursor2_ioctl(struct drm_device *dev,
-			   void *data, struct drm_file *file_priv);
+
+/* drm_color_mgmt.c */
+
+/* IOCTLs */
+int drm_mode_gamma_get_ioctl(struct drm_device *dev,
+			     void *data, struct drm_file *file_priv);
+int drm_mode_gamma_set_ioctl(struct drm_device *dev,
+			     void *data, struct drm_file *file_priv);
+
+/* drm_property.c */
+void drm_property_destroy_user_blobs(struct drm_device *dev,
+				     struct drm_file *file_priv);
+bool drm_property_change_valid_get(struct drm_property *property,
+				   uint64_t value,
+				   struct drm_mode_object **ref);
+void drm_property_change_valid_put(struct drm_property *property,
+				   struct drm_mode_object *ref);
+
+/* IOCTL */
 int drm_mode_getproperty_ioctl(struct drm_device *dev,
 			       void *data, struct drm_file *file_priv);
 int drm_mode_getblob_ioctl(struct drm_device *dev,
@@ -108,17 +85,80 @@ int drm_mode_createblob_ioctl(struct drm_device *dev,
 			      void *data, struct drm_file *file_priv);
 int drm_mode_destroyblob_ioctl(struct drm_device *dev,
 			       void *data, struct drm_file *file_priv);
-int drm_mode_connector_property_set_ioctl(struct drm_device *dev,
-					  void *data, struct drm_file *file_priv);
+
+/* drm_mode_object.c */
+int drm_mode_object_get_reg(struct drm_device *dev,
+			    struct drm_mode_object *obj,
+			    uint32_t obj_type,
+			    bool register_obj,
+			    void (*obj_free_cb)(struct kref *kref));
+void drm_mode_object_register(struct drm_device *dev,
+			      struct drm_mode_object *obj);
+int drm_mode_object_get(struct drm_device *dev,
+			struct drm_mode_object *obj, uint32_t obj_type);
+struct drm_mode_object *__drm_mode_object_find(struct drm_device *dev,
+					       uint32_t id, uint32_t type);
+void drm_mode_object_unregister(struct drm_device *dev,
+				struct drm_mode_object *object);
+int drm_mode_object_get_properties(struct drm_mode_object *obj, bool atomic,
+				   uint32_t __user *prop_ptr,
+				   uint64_t __user *prop_values,
+				   uint32_t *arg_count_props);
+struct drm_property *drm_mode_obj_find_prop_id(struct drm_mode_object *obj,
+					       uint32_t prop_id);
+
+/* IOCTL */
+
+int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data,
+				      struct drm_file *file_priv);
+int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv);
+
+/* drm_encoder.c */
+int drm_encoder_register_all(struct drm_device *dev);
+void drm_encoder_unregister_all(struct drm_device *dev);
+
+/* IOCTL */
 int drm_mode_getencoder(struct drm_device *dev,
 			void *data, struct drm_file *file_priv);
-int drm_mode_gamma_get_ioctl(struct drm_device *dev,
-			     void *data, struct drm_file *file_priv);
-int drm_mode_gamma_set_ioctl(struct drm_device *dev,
-			     void *data, struct drm_file *file_priv);
 
-int drm_mode_page_flip_ioctl(struct drm_device *dev,
-			     void *data, struct drm_file *file_priv);
+/* drm_connector.c */
+void drm_connector_ida_init(void);
+void drm_connector_ida_destroy(void);
+void drm_connector_unregister_all(struct drm_device *dev);
+int drm_connector_register_all(struct drm_device *dev);
+int drm_mode_connector_set_obj_prop(struct drm_mode_object *obj,
+				    struct drm_property *property,
+				    uint64_t value);
+int drm_connector_create_standard_properties(struct drm_device *dev);
+
+/* IOCTL */
+int drm_mode_connector_property_set_ioctl(struct drm_device *dev,
+					  void *data, struct drm_file *file_priv);
+int drm_mode_getconnector(struct drm_device *dev,
+			  void *data, struct drm_file *file_priv);
+
+/* drm_framebuffer.c */
+struct drm_framebuffer *
+drm_internal_framebuffer_create(struct drm_device *dev,
+				const struct drm_mode_fb_cmd2 *r,
+				struct drm_file *file_priv);
+void drm_framebuffer_free(struct kref *kref);
+int drm_framebuffer_check_src_coords(uint32_t src_x, uint32_t src_y,
+				     uint32_t src_w, uint32_t src_h,
+				     const struct drm_framebuffer *fb);
+
+/* IOCTL */
+int drm_mode_addfb(struct drm_device *dev,
+		   void *data, struct drm_file *file_priv);
+int drm_mode_addfb2(struct drm_device *dev,
+		    void *data, struct drm_file *file_priv);
+int drm_mode_rmfb(struct drm_device *dev,
+		  void *data, struct drm_file *file_priv);
+int drm_mode_getfb(struct drm_device *dev,
+		   void *data, struct drm_file *file_priv);
+int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
+			   void *data, struct drm_file *file_priv);
 
 /* drm_atomic.c */
 int drm_atomic_get_property(struct drm_mode_object *obj,
@@ -129,6 +169,23 @@ int drm_mode_atomic_ioctl(struct drm_device *dev,
 int drm_modeset_register_all(struct drm_device *dev);
 void drm_modeset_unregister_all(struct drm_device *dev);
 
-/* drm_blend.c */
-int drm_atomic_helper_normalize_zpos(struct drm_device *dev,
-				     struct drm_atomic_state *state);
+
+/* drm_plane.c */
+int drm_plane_register_all(struct drm_device *dev);
+void drm_plane_unregister_all(struct drm_device *dev);
+int drm_plane_check_pixel_format(const struct drm_plane *plane,
+				 u32 format);
+
+/* IOCTL */
+int drm_mode_getplane_res(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv);
+int drm_mode_getplane(struct drm_device *dev,
+		      void *data, struct drm_file *file_priv);
+int drm_mode_setplane(struct drm_device *dev,
+		      void *data, struct drm_file *file_priv);
+int drm_mode_cursor_ioctl(struct drm_device *dev,
+			  void *data, struct drm_file *file_priv);
+int drm_mode_cursor2_ioctl(struct drm_device *dev,
+			   void *data, struct drm_file *file_priv);
+int drm_mode_page_flip_ioctl(struct drm_device *dev,
+			     void *data, struct drm_file *file_priv);
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index fa10cef2ba37..1205790ed960 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -104,8 +104,8 @@ int drm_debugfs_create_files(const struct drm_info_list *files, int count,
 		ent = debugfs_create_file(files[i].name, S_IFREG | S_IRUGO,
 					  root, tmp, &drm_debugfs_fops);
 		if (!ent) {
-			DRM_ERROR("Cannot create /sys/kernel/debug/dri/%s/%s\n",
-				  root->d_name.name, files[i].name);
+			DRM_ERROR("Cannot create /sys/kernel/debug/dri/%pd/%s\n",
+				  root, files[i].name);
 			kfree(tmp);
 			ret = -1;
 			goto fail;
diff --git a/drivers/gpu/drm/drm_dma.c b/drivers/gpu/drm/drm_dma.c
index ea481800ef56..3f83e2ca80ad 100644
--- a/drivers/gpu/drm/drm_dma.c
+++ b/drivers/gpu/drm/drm_dma.c
@@ -50,9 +50,8 @@ int drm_legacy_dma_setup(struct drm_device *dev)
 	int i;
 
 	if (!drm_core_check_feature(dev, DRIVER_HAVE_DMA) ||
-	    drm_core_check_feature(dev, DRIVER_MODESET)) {
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return 0;
-	}
 
 	dev->buf_use = 0;
 	atomic_set(&dev->buf_alloc, 0);
@@ -81,9 +80,8 @@ void drm_legacy_dma_takedown(struct drm_device *dev)
 	int i, j;
 
 	if (!drm_core_check_feature(dev, DRIVER_HAVE_DMA) ||
-	    drm_core_check_feature(dev, DRIVER_MODESET)) {
+	    !drm_core_check_feature(dev, DRIVER_LEGACY))
 		return;
-	}
 
 	if (!dma)
 		return;
diff --git a/drivers/gpu/drm/drm_dp_aux_dev.c b/drivers/gpu/drm/drm_dp_aux_dev.c
index 734f86a345f6..ec1ed94b2390 100644
--- a/drivers/gpu/drm/drm_dp_aux_dev.c
+++ b/drivers/gpu/drm/drm_dp_aux_dev.c
@@ -36,6 +36,8 @@
 #include <drm/drm_crtc.h>
 #include <drm/drmP.h>
 
+#include "drm_crtc_helper_internal.h"
+
 struct drm_dp_aux_dev {
 	unsigned index;
 	struct drm_dp_aux *aux;
@@ -283,12 +285,7 @@ static int auxdev_wait_atomic_t(atomic_t *p)
 	schedule();
 	return 0;
 }
-/**
- * drm_dp_aux_unregister_devnode() - unregister a devnode for this aux channel
- * @aux: DisplayPort AUX channel
- *
- * Returns 0 on success or a negative error code on failure.
- */
+
 void drm_dp_aux_unregister_devnode(struct drm_dp_aux *aux)
 {
 	struct drm_dp_aux_dev *aux_dev;
@@ -314,14 +311,7 @@ void drm_dp_aux_unregister_devnode(struct drm_dp_aux *aux)
 	DRM_DEBUG("drm_dp_aux_dev: aux [%s] unregistering\n", aux->name);
 	kref_put(&aux_dev->refcount, release_drm_dp_aux_dev);
 }
-EXPORT_SYMBOL(drm_dp_aux_unregister_devnode);
 
-/**
- * drm_dp_aux_register_devnode() - register a devnode for this aux channel
- * @aux: DisplayPort AUX channel
- *
- * Returns 0 on success or a negative error code on failure.
- */
 int drm_dp_aux_register_devnode(struct drm_dp_aux *aux)
 {
 	struct drm_dp_aux_dev *aux_dev;
@@ -347,7 +337,6 @@ error:
 	drm_dp_aux_unregister_devnode(aux);
 	return res;
 }
-EXPORT_SYMBOL(drm_dp_aux_register_devnode);
 
 int drm_dp_aux_dev_init(void)
 {
@@ -369,11 +358,9 @@ out:
 	class_destroy(drm_dp_aux_dev_class);
 	return res;
 }
-EXPORT_SYMBOL(drm_dp_aux_dev_init);
 
 void drm_dp_aux_dev_exit(void)
 {
 	unregister_chrdev(drm_dev_major, "aux");
 	class_destroy(drm_dp_aux_dev_class);
 }
-EXPORT_SYMBOL(drm_dp_aux_dev_exit);
diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c
index eae5ef963cb7..3e6fe82c6d64 100644
--- a/drivers/gpu/drm/drm_dp_helper.c
+++ b/drivers/gpu/drm/drm_dp_helper.c
@@ -27,10 +27,12 @@
 #include <linux/errno.h>
 #include <linux/sched.h>
 #include <linux/i2c.h>
+#include <linux/seq_file.h>
 #include <drm/drm_dp_helper.h>
-#include <drm/drm_dp_aux_dev.h>
 #include <drm/drmP.h>
 
+#include "drm_crtc_helper_internal.h"
+
 /**
  * DOC: dp helpers
  *
@@ -223,7 +225,7 @@ static int drm_dp_dpcd_access(struct drm_dp_aux *aux, u8 request,
 			err = ret;
 	}
 
-	DRM_DEBUG_KMS("too many retries, giving up\n");
+	DRM_DEBUG_KMS("Too many retries, giving up. First error: %d\n", err);
 	ret = err;
 
 unlock:
@@ -438,6 +440,179 @@ int drm_dp_link_configure(struct drm_dp_aux *aux, struct drm_dp_link *link)
 }
 EXPORT_SYMBOL(drm_dp_link_configure);
 
+/**
+ * drm_dp_downstream_max_clock() - extract branch device max
+ *                                 pixel rate for legacy VGA
+ *                                 converter or max TMDS clock
+ *                                 rate for others
+ * @dpcd: DisplayPort configuration data
+ * @port_cap: port capabilities
+ *
+ * Returns max clock in kHz on success or 0 if max clock not defined
+ */
+int drm_dp_downstream_max_clock(const u8 dpcd[DP_RECEIVER_CAP_SIZE],
+				const u8 port_cap[4])
+{
+	int type = port_cap[0] & DP_DS_PORT_TYPE_MASK;
+	bool detailed_cap_info = dpcd[DP_DOWNSTREAMPORT_PRESENT] &
+		DP_DETAILED_CAP_INFO_AVAILABLE;
+
+	if (!detailed_cap_info)
+		return 0;
+
+	switch (type) {
+	case DP_DS_PORT_TYPE_VGA:
+		return port_cap[1] * 8 * 1000;
+	case DP_DS_PORT_TYPE_DVI:
+	case DP_DS_PORT_TYPE_HDMI:
+	case DP_DS_PORT_TYPE_DP_DUALMODE:
+		return port_cap[1] * 2500;
+	default:
+		return 0;
+	}
+}
+EXPORT_SYMBOL(drm_dp_downstream_max_clock);
+
+/**
+ * drm_dp_downstream_max_bpc() - extract branch device max
+ *                               bits per component
+ * @dpcd: DisplayPort configuration data
+ * @port_cap: port capabilities
+ *
+ * Returns max bpc on success or 0 if max bpc not defined
+ */
+int drm_dp_downstream_max_bpc(const u8 dpcd[DP_RECEIVER_CAP_SIZE],
+			      const u8 port_cap[4])
+{
+	int type = port_cap[0] & DP_DS_PORT_TYPE_MASK;
+	bool detailed_cap_info = dpcd[DP_DOWNSTREAMPORT_PRESENT] &
+		DP_DETAILED_CAP_INFO_AVAILABLE;
+	int bpc;
+
+	if (!detailed_cap_info)
+		return 0;
+
+	switch (type) {
+	case DP_DS_PORT_TYPE_VGA:
+	case DP_DS_PORT_TYPE_DVI:
+	case DP_DS_PORT_TYPE_HDMI:
+	case DP_DS_PORT_TYPE_DP_DUALMODE:
+		bpc = port_cap[2] & DP_DS_MAX_BPC_MASK;
+
+		switch (bpc) {
+		case DP_DS_8BPC:
+			return 8;
+		case DP_DS_10BPC:
+			return 10;
+		case DP_DS_12BPC:
+			return 12;
+		case DP_DS_16BPC:
+			return 16;
+		}
+	default:
+		return 0;
+	}
+}
+EXPORT_SYMBOL(drm_dp_downstream_max_bpc);
+
+/**
+ * drm_dp_downstream_id() - identify branch device
+ * @aux: DisplayPort AUX channel
+ * @id: DisplayPort branch device id
+ *
+ * Returns branch device id on success or NULL on failure
+ */
+int drm_dp_downstream_id(struct drm_dp_aux *aux, char id[6])
+{
+	return drm_dp_dpcd_read(aux, DP_BRANCH_ID, id, 6);
+}
+EXPORT_SYMBOL(drm_dp_downstream_id);
+
+/**
+ * drm_dp_downstream_debug() - debug DP branch devices
+ * @m: pointer for debugfs file
+ * @dpcd: DisplayPort configuration data
+ * @port_cap: port capabilities
+ * @aux: DisplayPort AUX channel
+ *
+ */
+void drm_dp_downstream_debug(struct seq_file *m,
+			     const u8 dpcd[DP_RECEIVER_CAP_SIZE],
+			     const u8 port_cap[4], struct drm_dp_aux *aux)
+{
+	bool detailed_cap_info = dpcd[DP_DOWNSTREAMPORT_PRESENT] &
+				 DP_DETAILED_CAP_INFO_AVAILABLE;
+	int clk;
+	int bpc;
+	char id[6];
+	int len;
+	uint8_t rev[2];
+	int type = port_cap[0] & DP_DS_PORT_TYPE_MASK;
+	bool branch_device = dpcd[DP_DOWNSTREAMPORT_PRESENT] &
+			     DP_DWN_STRM_PORT_PRESENT;
+
+	seq_printf(m, "\tDP branch device present: %s\n",
+		   branch_device ? "yes" : "no");
+
+	if (!branch_device)
+		return;
+
+	switch (type) {
+	case DP_DS_PORT_TYPE_DP:
+		seq_puts(m, "\t\tType: DisplayPort\n");
+		break;
+	case DP_DS_PORT_TYPE_VGA:
+		seq_puts(m, "\t\tType: VGA\n");
+		break;
+	case DP_DS_PORT_TYPE_DVI:
+		seq_puts(m, "\t\tType: DVI\n");
+		break;
+	case DP_DS_PORT_TYPE_HDMI:
+		seq_puts(m, "\t\tType: HDMI\n");
+		break;
+	case DP_DS_PORT_TYPE_NON_EDID:
+		seq_puts(m, "\t\tType: others without EDID support\n");
+		break;
+	case DP_DS_PORT_TYPE_DP_DUALMODE:
+		seq_puts(m, "\t\tType: DP++\n");
+		break;
+	case DP_DS_PORT_TYPE_WIRELESS:
+		seq_puts(m, "\t\tType: Wireless\n");
+		break;
+	default:
+		seq_puts(m, "\t\tType: N/A\n");
+	}
+
+	drm_dp_downstream_id(aux, id);
+	seq_printf(m, "\t\tID: %s\n", id);
+
+	len = drm_dp_dpcd_read(aux, DP_BRANCH_HW_REV, &rev[0], 1);
+	if (len > 0)
+		seq_printf(m, "\t\tHW: %d.%d\n",
+			   (rev[0] & 0xf0) >> 4, rev[0] & 0xf);
+
+	len = drm_dp_dpcd_read(aux, DP_BRANCH_SW_REV, &rev, 2);
+	if (len > 0)
+		seq_printf(m, "\t\tSW: %d.%d\n", rev[0], rev[1]);
+
+	if (detailed_cap_info) {
+		clk = drm_dp_downstream_max_clock(dpcd, port_cap);
+
+		if (clk > 0) {
+			if (type == DP_DS_PORT_TYPE_VGA)
+				seq_printf(m, "\t\tMax dot clock: %d kHz\n", clk);
+			else
+				seq_printf(m, "\t\tMax TMDS clock: %d kHz\n", clk);
+		}
+
+		bpc = drm_dp_downstream_max_bpc(dpcd, port_cap);
+
+		if (bpc > 0)
+			seq_printf(m, "\t\tMax bpc: %d\n", bpc);
+	}
+}
+EXPORT_SYMBOL(drm_dp_downstream_debug);
+
 /*
  * I2C-over-AUX implementation
  */
@@ -574,7 +749,17 @@ static int drm_dp_i2c_do_msg(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
 			if (ret == -EBUSY)
 				continue;
 
-			DRM_DEBUG_KMS("transaction failed: %d\n", ret);
+			/*
+			 * While timeouts can be errors, they're usually normal
+			 * behavior (for instance, when a driver tries to
+			 * communicate with a non-existant DisplayPort device).
+			 * Avoid spamming the kernel log with timeout errors.
+			 */
+			if (ret == -ETIMEDOUT)
+				DRM_DEBUG_KMS_RATELIMITED("transaction timed out\n");
+			else
+				DRM_DEBUG_KMS("transaction failed: %d\n", ret);
+
 			return ret;
 		}
 
@@ -790,6 +975,12 @@ static void unlock_bus(struct i2c_adapter *i2c, unsigned int flags)
 	mutex_unlock(&i2c_to_aux(i2c)->hw_mutex);
 }
 
+static const struct i2c_lock_operations drm_dp_i2c_lock_ops = {
+	.lock_bus = lock_bus,
+	.trylock_bus = trylock_bus,
+	.unlock_bus = unlock_bus,
+};
+
 /**
  * drm_dp_aux_init() - minimally initialise an aux channel
  * @aux: DisplayPort AUX channel
@@ -807,9 +998,7 @@ void drm_dp_aux_init(struct drm_dp_aux *aux)
 	aux->ddc.algo_data = aux;
 	aux->ddc.retries = 3;
 
-	aux->ddc.lock_bus = lock_bus;
-	aux->ddc.trylock_bus = trylock_bus;
-	aux->ddc.unlock_bus = unlock_bus;
+	aux->ddc.lock_ops = &drm_dp_i2c_lock_ops;
 }
 EXPORT_SYMBOL(drm_dp_aux_init);
 
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index be27ed36f56e..6efdba4993fc 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -33,7 +33,6 @@
 #include <linux/mount.h>
 #include <linux/slab.h>
 #include <drm/drmP.h>
-#include <drm/drm_core.h>
 #include "drm_crtc_internal.h"
 #include "drm_legacy.h"
 #include "drm_internal.h"
@@ -46,8 +45,8 @@
 unsigned int drm_debug = 0;
 EXPORT_SYMBOL(drm_debug);
 
-MODULE_AUTHOR(CORE_AUTHOR);
-MODULE_DESCRIPTION(CORE_DESC);
+MODULE_AUTHOR("Gareth Hughes, Leif Delgass, José Fonseca, Jon Smirl");
+MODULE_DESCRIPTION("DRM shared core routines");
 MODULE_LICENSE("GPL and additional rights");
 MODULE_PARM_DESC(debug, "Enable debug output, where each bit enables a debug category.\n"
 "\t\tBit 0 (0x01) will enable CORE messages (drm core code)\n"
@@ -63,37 +62,52 @@ static struct idr drm_minors_idr;
 
 static struct dentry *drm_debugfs_root;
 
-void drm_err(const char *format, ...)
+#define DRM_PRINTK_FMT "[" DRM_NAME ":%s]%s %pV"
+
+void drm_dev_printk(const struct device *dev, const char *level,
+		    unsigned int category, const char *function_name,
+		    const char *prefix, const char *format, ...)
 {
 	struct va_format vaf;
 	va_list args;
 
-	va_start(args, format);
+	if (category != DRM_UT_NONE && !(drm_debug & category))
+		return;
 
+	va_start(args, format);
 	vaf.fmt = format;
 	vaf.va = &args;
 
-	printk(KERN_ERR "[" DRM_NAME ":%ps] *ERROR* %pV",
-	       __builtin_return_address(0), &vaf);
+	if (dev)
+		dev_printk(level, dev, DRM_PRINTK_FMT, function_name, prefix,
+			   &vaf);
+	else
+		printk("%s" DRM_PRINTK_FMT, level, function_name, prefix, &vaf);
 
 	va_end(args);
 }
-EXPORT_SYMBOL(drm_err);
+EXPORT_SYMBOL(drm_dev_printk);
 
-void drm_ut_debug_printk(const char *function_name, const char *format, ...)
+void drm_printk(const char *level, unsigned int category,
+		const char *format, ...)
 {
 	struct va_format vaf;
 	va_list args;
 
+	if (category != DRM_UT_NONE && !(drm_debug & category))
+		return;
+
 	va_start(args, format);
 	vaf.fmt = format;
 	vaf.va = &args;
 
-	printk(KERN_DEBUG "[" DRM_NAME ":%s] %pV", function_name, &vaf);
+	printk("%s" "[" DRM_NAME ":%ps]%s %pV",
+	       level, __builtin_return_address(0),
+	       strcmp(level, KERN_ERR) == 0 ? " *ERROR*" : "", &vaf);
 
 	va_end(args);
 }
-EXPORT_SYMBOL(drm_ut_debug_printk);
+EXPORT_SYMBOL(drm_printk);
 
 /*
  * DRM Minors
@@ -112,7 +126,7 @@ static struct drm_minor **drm_minor_get_slot(struct drm_device *dev,
 					     unsigned int type)
 {
 	switch (type) {
-	case DRM_MINOR_LEGACY:
+	case DRM_MINOR_PRIMARY:
 		return &dev->primary;
 	case DRM_MINOR_RENDER:
 		return &dev->render;
@@ -325,6 +339,9 @@ void drm_minor_release(struct drm_minor *minor)
 
 static int drm_dev_set_unique(struct drm_device *dev, const char *name)
 {
+	if (!name)
+		return -EINVAL;
+
 	kfree(dev->unique);
 	dev->unique = kstrdup(name, GFP_KERNEL);
 
@@ -512,7 +529,7 @@ int drm_dev_init(struct drm_device *dev,
 			goto err_minors;
 	}
 
-	ret = drm_minor_alloc(dev, DRM_MINOR_LEGACY);
+	ret = drm_minor_alloc(dev, DRM_MINOR_PRIMARY);
 	if (ret)
 		goto err_minors;
 
@@ -545,7 +562,7 @@ err_ctxbitmap:
 	drm_legacy_ctxbitmap_cleanup(dev);
 	drm_ht_remove(&dev->map_hash);
 err_minors:
-	drm_minor_free(dev, DRM_MINOR_LEGACY);
+	drm_minor_free(dev, DRM_MINOR_PRIMARY);
 	drm_minor_free(dev, DRM_MINOR_RENDER);
 	drm_minor_free(dev, DRM_MINOR_CONTROL);
 	drm_fs_inode_free(dev->anon_inode);
@@ -575,7 +592,7 @@ EXPORT_SYMBOL(drm_dev_init);
  * own struct should look at using drm_dev_init() instead.
  *
  * RETURNS:
- * Pointer to new DRM device, or NULL if out of memory.
+ * Pointer to new DRM device, or ERR_PTR on failure.
  */
 struct drm_device *drm_dev_alloc(struct drm_driver *driver,
 				 struct device *parent)
@@ -585,12 +602,12 @@ struct drm_device *drm_dev_alloc(struct drm_driver *driver,
 
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 	if (!dev)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	ret = drm_dev_init(dev, driver, parent);
 	if (ret) {
 		kfree(dev);
-		return NULL;
+		return ERR_PTR(ret);
 	}
 
 	return dev;
@@ -608,7 +625,7 @@ static void drm_dev_release(struct kref *ref)
 	drm_ht_remove(&dev->map_hash);
 	drm_fs_inode_free(dev->anon_inode);
 
-	drm_minor_free(dev, DRM_MINOR_LEGACY);
+	drm_minor_free(dev, DRM_MINOR_PRIMARY);
 	drm_minor_free(dev, DRM_MINOR_RENDER);
 	drm_minor_free(dev, DRM_MINOR_CONTROL);
 
@@ -684,7 +701,7 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 	if (ret)
 		goto err_minors;
 
-	ret = drm_minor_register(dev, DRM_MINOR_LEGACY);
+	ret = drm_minor_register(dev, DRM_MINOR_PRIMARY);
 	if (ret)
 		goto err_minors;
 
@@ -701,7 +718,7 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 	goto out_unlock;
 
 err_minors:
-	drm_minor_unregister(dev, DRM_MINOR_LEGACY);
+	drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
 	drm_minor_unregister(dev, DRM_MINOR_RENDER);
 	drm_minor_unregister(dev, DRM_MINOR_CONTROL);
 out_unlock:
@@ -741,7 +758,7 @@ void drm_dev_unregister(struct drm_device *dev)
 	list_for_each_entry_safe(r_list, list_temp, &dev->maplist, head)
 		drm_legacy_rmmap(dev, r_list->map);
 
-	drm_minor_unregister(dev, DRM_MINOR_LEGACY);
+	drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
 	drm_minor_unregister(dev, DRM_MINOR_RENDER);
 	drm_minor_unregister(dev, DRM_MINOR_CONTROL);
 }
@@ -807,53 +824,48 @@ static const struct file_operations drm_stub_fops = {
 	.llseek = noop_llseek,
 };
 
+static void drm_core_exit(void)
+{
+	unregister_chrdev(DRM_MAJOR, "drm");
+	debugfs_remove(drm_debugfs_root);
+	drm_sysfs_destroy();
+	idr_destroy(&drm_minors_idr);
+	drm_connector_ida_destroy();
+	drm_global_release();
+}
+
 static int __init drm_core_init(void)
 {
-	int ret = -ENOMEM;
+	int ret;
 
 	drm_global_init();
 	drm_connector_ida_init();
 	idr_init(&drm_minors_idr);
 
-	if (register_chrdev(DRM_MAJOR, "drm", &drm_stub_fops))
-		goto err_p1;
-
 	ret = drm_sysfs_init();
 	if (ret < 0) {
-		printk(KERN_ERR "DRM: Error creating drm class.\n");
-		goto err_p2;
+		DRM_ERROR("Cannot create DRM class: %d\n", ret);
+		goto error;
 	}
 
 	drm_debugfs_root = debugfs_create_dir("dri", NULL);
 	if (!drm_debugfs_root) {
-		DRM_ERROR("Cannot create /sys/kernel/debug/dri\n");
-		ret = -1;
-		goto err_p3;
+		ret = -ENOMEM;
+		DRM_ERROR("Cannot create debugfs-root: %d\n", ret);
+		goto error;
 	}
 
-	DRM_INFO("Initialized %s %d.%d.%d %s\n",
-		 CORE_NAME, CORE_MAJOR, CORE_MINOR, CORE_PATCHLEVEL, CORE_DATE);
+	ret = register_chrdev(DRM_MAJOR, "drm", &drm_stub_fops);
+	if (ret < 0)
+		goto error;
+
+	DRM_INFO("Initialized\n");
 	return 0;
-err_p3:
-	drm_sysfs_destroy();
-err_p2:
-	unregister_chrdev(DRM_MAJOR, "drm");
 
-	idr_destroy(&drm_minors_idr);
-err_p1:
+error:
+	drm_core_exit();
 	return ret;
 }
 
-static void __exit drm_core_exit(void)
-{
-	debugfs_remove(drm_debugfs_root);
-	drm_sysfs_destroy();
-
-	unregister_chrdev(DRM_MAJOR, "drm");
-
-	drm_connector_ida_destroy();
-	idr_destroy(&drm_minors_idr);
-}
-
 module_init(drm_core_init);
 module_exit(drm_core_exit);
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 7df26d4b7ad8..ec77bd3e1f08 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -74,6 +74,8 @@
 #define EDID_QUIRK_FORCE_8BPC			(1 << 8)
 /* Force 12bpc */
 #define EDID_QUIRK_FORCE_12BPC			(1 << 9)
+/* Force 6bpc */
+#define EDID_QUIRK_FORCE_6BPC			(1 << 10)
 
 struct detailed_mode_closure {
 	struct drm_connector *connector;
@@ -100,6 +102,9 @@ static struct edid_quirk {
 	/* Unknown Acer */
 	{ "ACR", 2423, EDID_QUIRK_FIRST_DETAILED_PREFERRED },
 
+	/* AEO model 0 reports 8 bpc, but is a 6 bpc panel */
+	{ "AEO", 0, EDID_QUIRK_FORCE_6BPC },
+
 	/* Belinea 10 15 55 */
 	{ "MAX", 1516, EDID_QUIRK_PREFER_LARGE_60 },
 	{ "MAX", 0x77e, EDID_QUIRK_PREFER_LARGE_60 },
@@ -986,7 +991,7 @@ static const struct drm_display_mode edid_cea_modes[] = {
 	 .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, },
 	/* 64 - 1920x1080@100Hz */
 	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2448,
-		   2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
+		   2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
 	 .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, },
 };
@@ -3248,16 +3253,12 @@ static void fixup_detailed_cea_mode_clock(struct drm_display_mode *mode)
 }
 
 static void
-parse_hdmi_vsdb(struct drm_connector *connector, const u8 *db)
+drm_parse_hdmi_vsdb_audio(struct drm_connector *connector, const u8 *db)
 {
 	u8 len = cea_db_payload_len(db);
 
-	if (len >= 6) {
+	if (len >= 6)
 		connector->eld[5] |= (db[6] >> 7) << 1;  /* Supports_AI */
-		connector->dvi_dual = db[6] & 1;
-	}
-	if (len >= 7)
-		connector->max_tmds_clock = db[7] * 5;
 	if (len >= 8) {
 		connector->latency_present[0] = db[8] >> 7;
 		connector->latency_present[1] = (db[8] >> 6) & 1;
@@ -3271,19 +3272,15 @@ parse_hdmi_vsdb(struct drm_connector *connector, const u8 *db)
 	if (len >= 12)
 		connector->audio_latency[1] = db[12];
 
-	DRM_DEBUG_KMS("HDMI: DVI dual %d, "
-		    "max TMDS clock %d, "
-		    "latency present %d %d, "
-		    "video latency %d %d, "
-		    "audio latency %d %d\n",
-		    connector->dvi_dual,
-		    connector->max_tmds_clock,
-	      (int) connector->latency_present[0],
-	      (int) connector->latency_present[1],
-		    connector->video_latency[0],
-		    connector->video_latency[1],
-		    connector->audio_latency[0],
-		    connector->audio_latency[1]);
+	DRM_DEBUG_KMS("HDMI: latency present %d %d, "
+		      "video latency %d %d, "
+		      "audio latency %d %d\n",
+		      connector->latency_present[0],
+		      connector->latency_present[1],
+		      connector->video_latency[0],
+		      connector->video_latency[1],
+		      connector->audio_latency[0],
+		      connector->audio_latency[1]);
 }
 
 static void
@@ -3353,6 +3350,13 @@ void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
 
 	memset(eld, 0, sizeof(connector->eld));
 
+	connector->latency_present[0] = false;
+	connector->latency_present[1] = false;
+	connector->video_latency[0] = 0;
+	connector->audio_latency[0] = 0;
+	connector->video_latency[1] = 0;
+	connector->audio_latency[1] = 0;
+
 	cea = drm_find_cea_extension(edid);
 	if (!cea) {
 		DRM_DEBUG_KMS("ELD: no CEA Extension found\n");
@@ -3402,7 +3406,7 @@ void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
 			case VENDOR_BLOCK:
 				/* HDMI Vendor-Specific Data Block */
 				if (cea_db_is_hdmi_vsdb(db))
-					parse_hdmi_vsdb(connector, db);
+					drm_parse_hdmi_vsdb_audio(connector, db);
 				break;
 			default:
 				break;
@@ -3716,122 +3720,127 @@ bool drm_rgb_quant_range_selectable(struct edid *edid)
 }
 EXPORT_SYMBOL(drm_rgb_quant_range_selectable);
 
-/**
- * drm_assign_hdmi_deep_color_info - detect whether monitor supports
- * hdmi deep color modes and update drm_display_info if so.
- * @edid: monitor EDID information
- * @info: Updated with maximum supported deep color bpc and color format
- *        if deep color supported.
- * @connector: DRM connector, used only for debug output
- *
- * Parse the CEA extension according to CEA-861-B.
- * Return true if HDMI deep color supported, false if not or unknown.
- */
-static bool drm_assign_hdmi_deep_color_info(struct edid *edid,
-                                            struct drm_display_info *info,
-                                            struct drm_connector *connector)
+static void drm_parse_hdmi_deep_color_info(struct drm_connector *connector,
+					   const u8 *hdmi)
 {
-	u8 *edid_ext, *hdmi;
-	int i;
-	int start_offset, end_offset;
+	struct drm_display_info *info = &connector->display_info;
 	unsigned int dc_bpc = 0;
 
-	edid_ext = drm_find_cea_extension(edid);
-	if (!edid_ext)
-		return false;
+	/* HDMI supports at least 8 bpc */
+	info->bpc = 8;
 
-	if (cea_db_offsets(edid_ext, &start_offset, &end_offset))
-		return false;
+	if (cea_db_payload_len(hdmi) < 6)
+		return;
+
+	if (hdmi[6] & DRM_EDID_HDMI_DC_30) {
+		dc_bpc = 10;
+		info->edid_hdmi_dc_modes |= DRM_EDID_HDMI_DC_30;
+		DRM_DEBUG("%s: HDMI sink does deep color 30.\n",
+			  connector->name);
+	}
+
+	if (hdmi[6] & DRM_EDID_HDMI_DC_36) {
+		dc_bpc = 12;
+		info->edid_hdmi_dc_modes |= DRM_EDID_HDMI_DC_36;
+		DRM_DEBUG("%s: HDMI sink does deep color 36.\n",
+			  connector->name);
+	}
+
+	if (hdmi[6] & DRM_EDID_HDMI_DC_48) {
+		dc_bpc = 16;
+		info->edid_hdmi_dc_modes |= DRM_EDID_HDMI_DC_48;
+		DRM_DEBUG("%s: HDMI sink does deep color 48.\n",
+			  connector->name);
+	}
+
+	if (dc_bpc == 0) {
+		DRM_DEBUG("%s: No deep color support on this HDMI sink.\n",
+			  connector->name);
+		return;
+	}
+
+	DRM_DEBUG("%s: Assigning HDMI sink color depth as %d bpc.\n",
+		  connector->name, dc_bpc);
+	info->bpc = dc_bpc;
 
 	/*
-	 * Because HDMI identifier is in Vendor Specific Block,
-	 * search it from all data blocks of CEA extension.
+	 * Deep color support mandates RGB444 support for all video
+	 * modes and forbids YCRCB422 support for all video modes per
+	 * HDMI 1.3 spec.
 	 */
-	for_each_cea_db(edid_ext, i, start_offset, end_offset) {
-		if (cea_db_is_hdmi_vsdb(&edid_ext[i])) {
-			/* HDMI supports at least 8 bpc */
-			info->bpc = 8;
-
-			hdmi = &edid_ext[i];
-			if (cea_db_payload_len(hdmi) < 6)
-				return false;
-
-			if (hdmi[6] & DRM_EDID_HDMI_DC_30) {
-				dc_bpc = 10;
-				info->edid_hdmi_dc_modes |= DRM_EDID_HDMI_DC_30;
-				DRM_DEBUG("%s: HDMI sink does deep color 30.\n",
-						  connector->name);
-			}
+	info->color_formats = DRM_COLOR_FORMAT_RGB444;
 
-			if (hdmi[6] & DRM_EDID_HDMI_DC_36) {
-				dc_bpc = 12;
-				info->edid_hdmi_dc_modes |= DRM_EDID_HDMI_DC_36;
-				DRM_DEBUG("%s: HDMI sink does deep color 36.\n",
-						  connector->name);
-			}
+	/* YCRCB444 is optional according to spec. */
+	if (hdmi[6] & DRM_EDID_HDMI_DC_Y444) {
+		info->color_formats |= DRM_COLOR_FORMAT_YCRCB444;
+		DRM_DEBUG("%s: HDMI sink does YCRCB444 in deep color.\n",
+			  connector->name);
+	}
 
-			if (hdmi[6] & DRM_EDID_HDMI_DC_48) {
-				dc_bpc = 16;
-				info->edid_hdmi_dc_modes |= DRM_EDID_HDMI_DC_48;
-				DRM_DEBUG("%s: HDMI sink does deep color 48.\n",
-						  connector->name);
-			}
+	/*
+	 * Spec says that if any deep color mode is supported at all,
+	 * then deep color 36 bit must be supported.
+	 */
+	if (!(hdmi[6] & DRM_EDID_HDMI_DC_36)) {
+		DRM_DEBUG("%s: HDMI sink should do DC_36, but does not!\n",
+			  connector->name);
+	}
+}
 
-			if (dc_bpc > 0) {
-				DRM_DEBUG("%s: Assigning HDMI sink color depth as %d bpc.\n",
-						  connector->name, dc_bpc);
-				info->bpc = dc_bpc;
-
-				/*
-				 * Deep color support mandates RGB444 support for all video
-				 * modes and forbids YCRCB422 support for all video modes per
-				 * HDMI 1.3 spec.
-				 */
-				info->color_formats = DRM_COLOR_FORMAT_RGB444;
-
-				/* YCRCB444 is optional according to spec. */
-				if (hdmi[6] & DRM_EDID_HDMI_DC_Y444) {
-					info->color_formats |= DRM_COLOR_FORMAT_YCRCB444;
-					DRM_DEBUG("%s: HDMI sink does YCRCB444 in deep color.\n",
-							  connector->name);
-				}
+static void
+drm_parse_hdmi_vsdb_video(struct drm_connector *connector, const u8 *db)
+{
+	struct drm_display_info *info = &connector->display_info;
+	u8 len = cea_db_payload_len(db);
 
-				/*
-				 * Spec says that if any deep color mode is supported at all,
-				 * then deep color 36 bit must be supported.
-				 */
-				if (!(hdmi[6] & DRM_EDID_HDMI_DC_36)) {
-					DRM_DEBUG("%s: HDMI sink should do DC_36, but does not!\n",
-							  connector->name);
-				}
+	if (len >= 6)
+		info->dvi_dual = db[6] & 1;
+	if (len >= 7)
+		info->max_tmds_clock = db[7] * 5000;
 
-				return true;
-			}
-			else {
-				DRM_DEBUG("%s: No deep color support on this HDMI sink.\n",
-						  connector->name);
-			}
-		}
-	}
+	DRM_DEBUG_KMS("HDMI: DVI dual %d, "
+		      "max TMDS clock %d kHz\n",
+		      info->dvi_dual,
+		      info->max_tmds_clock);
 
-	return false;
+	drm_parse_hdmi_deep_color_info(connector, db);
 }
 
-/**
- * drm_add_display_info - pull display info out if present
- * @edid: EDID data
- * @info: display info (attached to connector)
- * @connector: connector whose edid is used to build display info
- *
- * Grab any available display info and stuff it into the drm_display_info
- * structure that's part of the connector.  Useful for tracking bpp and
- * color spaces.
- */
-static void drm_add_display_info(struct edid *edid,
-                                 struct drm_display_info *info,
-                                 struct drm_connector *connector)
+static void drm_parse_cea_ext(struct drm_connector *connector,
+			      struct edid *edid)
 {
-	u8 *edid_ext;
+	struct drm_display_info *info = &connector->display_info;
+	const u8 *edid_ext;
+	int i, start, end;
+
+	edid_ext = drm_find_cea_extension(edid);
+	if (!edid_ext)
+		return;
+
+	info->cea_rev = edid_ext[1];
+
+	/* The existence of a CEA block should imply RGB support */
+	info->color_formats = DRM_COLOR_FORMAT_RGB444;
+	if (edid_ext[3] & EDID_CEA_YCRCB444)
+		info->color_formats |= DRM_COLOR_FORMAT_YCRCB444;
+	if (edid_ext[3] & EDID_CEA_YCRCB422)
+		info->color_formats |= DRM_COLOR_FORMAT_YCRCB422;
+
+	if (cea_db_offsets(edid_ext, &start, &end))
+		return;
+
+	for_each_cea_db(edid_ext, i, start, end) {
+		const u8 *db = &edid_ext[i];
+
+		if (cea_db_is_hdmi_vsdb(db))
+			drm_parse_hdmi_vsdb_video(connector, db);
+	}
+}
+
+static void drm_add_display_info(struct drm_connector *connector,
+				 struct edid *edid)
+{
+	struct drm_display_info *info = &connector->display_info;
 
 	info->width_mm = edid->width_cm * 10;
 	info->height_mm = edid->height_cm * 10;
@@ -3839,6 +3848,9 @@ static void drm_add_display_info(struct edid *edid,
 	/* driver figures it out in this case */
 	info->bpc = 0;
 	info->color_formats = 0;
+	info->cea_rev = 0;
+	info->max_tmds_clock = 0;
+	info->dvi_dual = false;
 
 	if (edid->revision < 3)
 		return;
@@ -3846,21 +3858,21 @@ static void drm_add_display_info(struct edid *edid,
 	if (!(edid->input & DRM_EDID_INPUT_DIGITAL))
 		return;
 
-	/* Get data from CEA blocks if present */
-	edid_ext = drm_find_cea_extension(edid);
-	if (edid_ext) {
-		info->cea_rev = edid_ext[1];
-
-		/* The existence of a CEA block should imply RGB support */
-		info->color_formats = DRM_COLOR_FORMAT_RGB444;
-		if (edid_ext[3] & EDID_CEA_YCRCB444)
-			info->color_formats |= DRM_COLOR_FORMAT_YCRCB444;
-		if (edid_ext[3] & EDID_CEA_YCRCB422)
-			info->color_formats |= DRM_COLOR_FORMAT_YCRCB422;
-	}
+	drm_parse_cea_ext(connector, edid);
 
-	/* HDMI deep color modes supported? Assign to info, if so */
-	drm_assign_hdmi_deep_color_info(edid, info, connector);
+	/*
+	 * Digital sink with "DFP 1.x compliant TMDS" according to EDID 1.3?
+	 *
+	 * For such displays, the DFP spec 1.0, section 3.10 "EDID support"
+	 * tells us to assume 8 bpc color depth if the EDID doesn't have
+	 * extensions which tell otherwise.
+	 */
+	if ((info->bpc == 0) && (edid->revision < 4) &&
+	    (edid->input & DRM_EDID_DIGITAL_TYPE_DVI)) {
+		info->bpc = 8;
+		DRM_DEBUG("%s: Assigning DFP sink color depth as %d bpc.\n",
+			  connector->name, info->bpc);
+	}
 
 	/* Only defined for 1.4 with digital displays */
 	if (edid->revision < 4)
@@ -4033,7 +4045,9 @@ static int add_displayid_detailed_modes(struct drm_connector *connector,
  * @connector: connector we're probing
  * @edid: EDID data
  *
- * Add the specified modes to the connector's mode list.
+ * Add the specified modes to the connector's mode list. Also fills out the
+ * &drm_display_info structure in @connector with any information which can be
+ * derived from the edid.
  *
  * Return: The number of modes added or 0 if we couldn't find any.
  */
@@ -4080,7 +4094,10 @@ int drm_add_edid_modes(struct drm_connector *connector, struct edid *edid)
 	if (quirks & (EDID_QUIRK_PREFER_LARGE_60 | EDID_QUIRK_PREFER_LARGE_75))
 		edid_fixup_preferred(connector, quirks);
 
-	drm_add_display_info(edid, &connector->display_info, connector);
+	drm_add_display_info(connector, edid);
+
+	if (quirks & EDID_QUIRK_FORCE_6BPC)
+		connector->display_info.bpc = 6;
 
 	if (quirks & EDID_QUIRK_FORCE_8BPC)
 		connector->display_info.bpc = 8;
diff --git a/drivers/gpu/drm/drm_encoder.c b/drivers/gpu/drm/drm_encoder.c
new file mode 100644
index 000000000000..5c067719164d
--- /dev/null
+++ b/drivers/gpu/drm/drm_encoder.c
@@ -0,0 +1,233 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <linux/export.h>
+#include <drm/drmP.h>
+#include <drm/drm_encoder.h>
+
+#include "drm_crtc_internal.h"
+
+/**
+ * DOC: overview
+ *
+ * Encoders represent the connecting element between the CRTC (as the overall
+ * pixel pipeline, represented by struct &drm_crtc) and the connectors (as the
+ * generic sink entity, represented by struct &drm_connector). An encoder takes
+ * pixel data from a CRTC and converts it to a format suitable for any attached
+ * connector. Encoders are objects exposed to userspace, originally to allow
+ * userspace to infer cloning and connector/CRTC restrictions. Unfortunately
+ * almost all drivers get this wrong, making the uabi pretty much useless. On
+ * top of that the exposed restrictions are too simple for today's hardware, and
+ * the recommended way to infer restrictions is by using the
+ * DRM_MODE_ATOMIC_TEST_ONLY flag for the atomic IOCTL.
+ *
+ * Otherwise encoders aren't used in the uapi at all (any modeset request from
+ * userspace directly connects a connector with a CRTC), drivers are therefore
+ * free to use them however they wish. Modeset helper libraries make strong use
+ * of encoders to facilitate code sharing. But for more complex settings it is
+ * usually better to move shared code into a separate &drm_bridge. Compared to
+ * encoders, bridges also have the benefit of being purely an internal
+ * abstraction since they are not exposed to userspace at all.
+ *
+ * Encoders are initialized with drm_encoder_init() and cleaned up using
+ * drm_encoder_cleanup().
+ */
+static const struct drm_prop_enum_list drm_encoder_enum_list[] = {
+	{ DRM_MODE_ENCODER_NONE, "None" },
+	{ DRM_MODE_ENCODER_DAC, "DAC" },
+	{ DRM_MODE_ENCODER_TMDS, "TMDS" },
+	{ DRM_MODE_ENCODER_LVDS, "LVDS" },
+	{ DRM_MODE_ENCODER_TVDAC, "TV" },
+	{ DRM_MODE_ENCODER_VIRTUAL, "Virtual" },
+	{ DRM_MODE_ENCODER_DSI, "DSI" },
+	{ DRM_MODE_ENCODER_DPMST, "DP MST" },
+	{ DRM_MODE_ENCODER_DPI, "DPI" },
+};
+
+int drm_encoder_register_all(struct drm_device *dev)
+{
+	struct drm_encoder *encoder;
+	int ret = 0;
+
+	drm_for_each_encoder(encoder, dev) {
+		if (encoder->funcs->late_register)
+			ret = encoder->funcs->late_register(encoder);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+void drm_encoder_unregister_all(struct drm_device *dev)
+{
+	struct drm_encoder *encoder;
+
+	drm_for_each_encoder(encoder, dev) {
+		if (encoder->funcs->early_unregister)
+			encoder->funcs->early_unregister(encoder);
+	}
+}
+
+/**
+ * drm_encoder_init - Init a preallocated encoder
+ * @dev: drm device
+ * @encoder: the encoder to init
+ * @funcs: callbacks for this encoder
+ * @encoder_type: user visible type of the encoder
+ * @name: printf style format string for the encoder name, or NULL for default name
+ *
+ * Initialises a preallocated encoder. Encoder should be subclassed as part of
+ * driver encoder objects. At driver unload time drm_encoder_cleanup() should be
+ * called from the driver's destroy hook in &drm_encoder_funcs.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_encoder_init(struct drm_device *dev,
+		     struct drm_encoder *encoder,
+		     const struct drm_encoder_funcs *funcs,
+		     int encoder_type, const char *name, ...)
+{
+	int ret;
+
+	drm_modeset_lock_all(dev);
+
+	ret = drm_mode_object_get(dev, &encoder->base, DRM_MODE_OBJECT_ENCODER);
+	if (ret)
+		goto out_unlock;
+
+	encoder->dev = dev;
+	encoder->encoder_type = encoder_type;
+	encoder->funcs = funcs;
+	if (name) {
+		va_list ap;
+
+		va_start(ap, name);
+		encoder->name = kvasprintf(GFP_KERNEL, name, ap);
+		va_end(ap);
+	} else {
+		encoder->name = kasprintf(GFP_KERNEL, "%s-%d",
+					  drm_encoder_enum_list[encoder_type].name,
+					  encoder->base.id);
+	}
+	if (!encoder->name) {
+		ret = -ENOMEM;
+		goto out_put;
+	}
+
+	list_add_tail(&encoder->head, &dev->mode_config.encoder_list);
+	encoder->index = dev->mode_config.num_encoder++;
+
+out_put:
+	if (ret)
+		drm_mode_object_unregister(dev, &encoder->base);
+
+out_unlock:
+	drm_modeset_unlock_all(dev);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_encoder_init);
+
+/**
+ * drm_encoder_cleanup - cleans up an initialised encoder
+ * @encoder: encoder to cleanup
+ *
+ * Cleans up the encoder but doesn't free the object.
+ */
+void drm_encoder_cleanup(struct drm_encoder *encoder)
+{
+	struct drm_device *dev = encoder->dev;
+
+	/* Note that the encoder_list is considered to be static; should we
+	 * remove the drm_encoder at runtime we would have to decrement all
+	 * the indices on the drm_encoder after us in the encoder_list.
+	 */
+
+	drm_modeset_lock_all(dev);
+	drm_mode_object_unregister(dev, &encoder->base);
+	kfree(encoder->name);
+	list_del(&encoder->head);
+	dev->mode_config.num_encoder--;
+	drm_modeset_unlock_all(dev);
+
+	memset(encoder, 0, sizeof(*encoder));
+}
+EXPORT_SYMBOL(drm_encoder_cleanup);
+
+static struct drm_crtc *drm_encoder_get_crtc(struct drm_encoder *encoder)
+{
+	struct drm_connector *connector;
+	struct drm_device *dev = encoder->dev;
+	bool uses_atomic = false;
+
+	/* For atomic drivers only state objects are synchronously updated and
+	 * protected by modeset locks, so check those first. */
+	drm_for_each_connector(connector, dev) {
+		if (!connector->state)
+			continue;
+
+		uses_atomic = true;
+
+		if (connector->state->best_encoder != encoder)
+			continue;
+
+		return connector->state->crtc;
+	}
+
+	/* Don't return stale data (e.g. pending async disable). */
+	if (uses_atomic)
+		return NULL;
+
+	return encoder->crtc;
+}
+
+int drm_mode_getencoder(struct drm_device *dev, void *data,
+			struct drm_file *file_priv)
+{
+	struct drm_mode_get_encoder *enc_resp = data;
+	struct drm_encoder *encoder;
+	struct drm_crtc *crtc;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	encoder = drm_encoder_find(dev, enc_resp->encoder_id);
+	if (!encoder)
+		return -ENOENT;
+
+	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
+	crtc = drm_encoder_get_crtc(encoder);
+	if (crtc)
+		enc_resp->crtc_id = crtc->base.id;
+	else
+		enc_resp->crtc_id = 0;
+	drm_modeset_unlock(&dev->mode_config.connection_mutex);
+
+	enc_resp->encoder_type = encoder->encoder_type;
+	enc_resp->encoder_id = encoder->base.id;
+	enc_resp->possible_crtcs = encoder->possible_crtcs;
+	enc_resp->possible_clones = encoder->possible_clones;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index ce54e985d91b..03414bde1f15 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -29,10 +29,10 @@
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/console.h>
 #include <linux/kernel.h>
 #include <linux/sysrq.h>
 #include <linux/slab.h>
-#include <linux/fb.h>
 #include <linux/module.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc.h>
@@ -41,6 +41,8 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 
+#include "drm_crtc_helper_internal.h"
+
 static bool drm_fbdev_emulation = true;
 module_param_named(fbdev_emulation, drm_fbdev_emulation, bool, 0600);
 MODULE_PARM_DESC(fbdev_emulation,
@@ -335,7 +337,7 @@ retry:
 			goto fail;
 		}
 
-		plane_state->rotation = BIT(DRM_ROTATE_0);
+		plane_state->rotation = DRM_ROTATE_0;
 
 		plane->old_fb = plane->fb;
 		plane_mask |= 1 << drm_plane_index(plane);
@@ -395,7 +397,7 @@ static int restore_fbdev_mode(struct drm_fb_helper *fb_helper)
 		if (dev->mode_config.rotation_property) {
 			drm_mode_plane_set_obj_prop(plane,
 						    dev->mode_config.rotation_property,
-						    BIT(DRM_ROTATE_0));
+						    DRM_ROTATE_0);
 		}
 	}
 
@@ -464,7 +466,7 @@ static bool drm_fb_helper_is_bound(struct drm_fb_helper *fb_helper)
 
 	/* Sometimes user space wants everything disabled, so don't steal the
 	 * display if there's a master. */
-	if (lockless_dereference(dev->master))
+	if (READ_ONCE(dev->master))
 		return false;
 
 	drm_for_each_crtc(crtc, dev) {
@@ -618,6 +620,16 @@ static void drm_fb_helper_crtc_free(struct drm_fb_helper *helper)
 	kfree(helper->crtc_info);
 }
 
+static void drm_fb_helper_resume_worker(struct work_struct *work)
+{
+	struct drm_fb_helper *helper = container_of(work, struct drm_fb_helper,
+						    resume_work);
+
+	console_lock();
+	fb_set_suspend(helper->fbdev, 0);
+	console_unlock();
+}
+
 static void drm_fb_helper_dirty_work(struct work_struct *work)
 {
 	struct drm_fb_helper *helper = container_of(work, struct drm_fb_helper,
@@ -649,6 +661,7 @@ void drm_fb_helper_prepare(struct drm_device *dev, struct drm_fb_helper *helper,
 {
 	INIT_LIST_HEAD(&helper->kernel_fb_list);
 	spin_lock_init(&helper->dirty_lock);
+	INIT_WORK(&helper->resume_work, drm_fb_helper_resume_worker);
 	INIT_WORK(&helper->dirty_work, drm_fb_helper_dirty_work);
 	helper->dirty_clip.x1 = helper->dirty_clip.y1 = ~0;
 	helper->funcs = funcs;
@@ -1024,17 +1037,65 @@ EXPORT_SYMBOL(drm_fb_helper_cfb_imageblit);
 /**
  * drm_fb_helper_set_suspend - wrapper around fb_set_suspend
  * @fb_helper: driver-allocated fbdev helper
- * @state: desired state, zero to resume, non-zero to suspend
+ * @suspend: whether to suspend or resume
  *
- * A wrapper around fb_set_suspend implemented by fbdev core
+ * A wrapper around fb_set_suspend implemented by fbdev core.
+ * Use drm_fb_helper_set_suspend_unlocked() if you don't need to take
+ * the lock yourself
  */
-void drm_fb_helper_set_suspend(struct drm_fb_helper *fb_helper, int state)
+void drm_fb_helper_set_suspend(struct drm_fb_helper *fb_helper, bool suspend)
 {
 	if (fb_helper && fb_helper->fbdev)
-		fb_set_suspend(fb_helper->fbdev, state);
+		fb_set_suspend(fb_helper->fbdev, suspend);
 }
 EXPORT_SYMBOL(drm_fb_helper_set_suspend);
 
+/**
+ * drm_fb_helper_set_suspend_unlocked - wrapper around fb_set_suspend that also
+ *                                      takes the console lock
+ * @fb_helper: driver-allocated fbdev helper
+ * @suspend: whether to suspend or resume
+ *
+ * A wrapper around fb_set_suspend() that takes the console lock. If the lock
+ * isn't available on resume, a worker is tasked with waiting for the lock
+ * to become available. The console lock can be pretty contented on resume
+ * due to all the printk activity.
+ *
+ * This function can be called multiple times with the same state since
+ * &fb_info->state is checked to see if fbdev is running or not before locking.
+ *
+ * Use drm_fb_helper_set_suspend() if you need to take the lock yourself.
+ */
+void drm_fb_helper_set_suspend_unlocked(struct drm_fb_helper *fb_helper,
+					bool suspend)
+{
+	if (!fb_helper || !fb_helper->fbdev)
+		return;
+
+	/* make sure there's no pending/ongoing resume */
+	flush_work(&fb_helper->resume_work);
+
+	if (suspend) {
+		if (fb_helper->fbdev->state != FBINFO_STATE_RUNNING)
+			return;
+
+		console_lock();
+
+	} else {
+		if (fb_helper->fbdev->state == FBINFO_STATE_RUNNING)
+			return;
+
+		if (!console_trylock()) {
+			schedule_work(&fb_helper->resume_work);
+			return;
+		}
+	}
+
+	fb_set_suspend(fb_helper->fbdev, suspend);
+	console_unlock();
+}
+EXPORT_SYMBOL(drm_fb_helper_set_suspend_unlocked);
+
 static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
 		     u16 blue, u16 regno, struct fb_info *info)
 {
@@ -2194,7 +2255,7 @@ EXPORT_SYMBOL(drm_fb_helper_initial_config);
  * @fb_helper: the drm_fb_helper
  *
  * Scan the connectors attached to the fb_helper and try to put together a
- * setup after *notification of a change in output configuration.
+ * setup after notification of a change in output configuration.
  *
  * Called at runtime, takes the mode config locks to be able to check/change the
  * modeset configuration. Must be run from process context (which usually means
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index 323c238fcac7..e84faecf5225 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -92,7 +92,7 @@ static int drm_setup(struct drm_device * dev)
 	int ret;
 
 	if (dev->driver->firstopen &&
-	    !drm_core_check_feature(dev, DRIVER_MODESET)) {
+	    drm_core_check_feature(dev, DRIVER_LEGACY)) {
 		ret = dev->driver->firstopen(dev);
 		if (ret != 0)
 			return ret;
@@ -199,7 +199,6 @@ static int drm_open_helper(struct file *filp, struct drm_minor *minor)
 
 	filp->private_data = priv;
 	priv->filp = filp;
-	priv->uid = current_euid();
 	priv->pid = get_pid(task_pid(current));
 	priv->minor = minor;
 
@@ -346,7 +345,7 @@ void drm_lastclose(struct drm_device * dev)
 		dev->driver->lastclose(dev);
 	DRM_DEBUG("driver lastclose completed\n");
 
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+	if (drm_core_check_feature(dev, DRIVER_LEGACY))
 		drm_legacy_dev_reinit(dev);
 }
 
@@ -389,7 +388,7 @@ int drm_release(struct inode *inode, struct file *filp)
 		  (long)old_encode_dev(file_priv->minor->kdev->devt),
 		  dev->open_count);
 
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+	if (drm_core_check_feature(dev, DRIVER_LEGACY))
 		drm_legacy_lock_release(dev, filp);
 
 	if (drm_core_check_feature(dev, DRIVER_HAVE_DMA))
diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c
index 0645c85d5f95..29c56b4331e0 100644
--- a/drivers/gpu/drm/drm_fourcc.c
+++ b/drivers/gpu/drm/drm_fourcc.c
@@ -36,19 +36,60 @@ static char printable_char(int c)
 }
 
 /**
+ * drm_mode_legacy_fb_format - compute drm fourcc code from legacy description
+ * @bpp: bits per pixels
+ * @depth: bit depth per pixel
+ *
+ * Computes a drm fourcc pixel format code for the given @bpp/@depth values.
+ * Useful in fbdev emulation code, since that deals in those values.
+ */
+uint32_t drm_mode_legacy_fb_format(uint32_t bpp, uint32_t depth)
+{
+	uint32_t fmt;
+
+	switch (bpp) {
+	case 8:
+		fmt = DRM_FORMAT_C8;
+		break;
+	case 16:
+		if (depth == 15)
+			fmt = DRM_FORMAT_XRGB1555;
+		else
+			fmt = DRM_FORMAT_RGB565;
+		break;
+	case 24:
+		fmt = DRM_FORMAT_RGB888;
+		break;
+	case 32:
+		if (depth == 24)
+			fmt = DRM_FORMAT_XRGB8888;
+		else if (depth == 30)
+			fmt = DRM_FORMAT_XRGB2101010;
+		else
+			fmt = DRM_FORMAT_ARGB8888;
+		break;
+	default:
+		DRM_ERROR("bad bpp, assuming x8r8g8b8 pixel format\n");
+		fmt = DRM_FORMAT_XRGB8888;
+		break;
+	}
+
+	return fmt;
+}
+EXPORT_SYMBOL(drm_mode_legacy_fb_format);
+
+/**
  * drm_get_format_name - return a string for drm fourcc format
  * @format: format to compute name of
  *
- * Note that the buffer used by this function is globally shared and owned by
- * the function itself.
- *
- * FIXME: This isn't really multithreading safe.
+ * Note that the buffer returned by this function is owned by the caller
+ * and will need to be freed using kfree().
  */
-const char *drm_get_format_name(uint32_t format)
+char *drm_get_format_name(uint32_t format)
 {
-	static char buf[32];
+	char *buf = kmalloc(32, GFP_KERNEL);
 
-	snprintf(buf, sizeof(buf),
+	snprintf(buf, 32,
 		 "%c%c%c%c %s-endian (0x%08x)",
 		 printable_char(format & 0xff),
 		 printable_char((format >> 8) & 0xff),
@@ -73,6 +114,8 @@ EXPORT_SYMBOL(drm_get_format_name);
 void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth,
 			  int *bpp)
 {
+	char *format_name;
+
 	switch (format) {
 	case DRM_FORMAT_C8:
 	case DRM_FORMAT_RGB332:
@@ -127,8 +170,9 @@ void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth,
 		*bpp = 32;
 		break;
 	default:
-		DRM_DEBUG_KMS("unsupported pixel format %s\n",
-			      drm_get_format_name(format));
+		format_name = drm_get_format_name(format);
+		DRM_DEBUG_KMS("unsupported pixel format %s\n", format_name);
+		kfree(format_name);
 		*depth = 0;
 		*bpp = 0;
 		break;
diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c
new file mode 100644
index 000000000000..398efd67cb93
--- /dev/null
+++ b/drivers/gpu/drm/drm_framebuffer.c
@@ -0,0 +1,857 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <linux/export.h>
+#include <drm/drmP.h>
+#include <drm/drm_auth.h>
+#include <drm/drm_framebuffer.h>
+
+#include "drm_crtc_internal.h"
+
+/**
+ * DOC: overview
+ *
+ * Frame buffers are abstract memory objects that provide a source of pixels to
+ * scanout to a CRTC. Applications explicitly request the creation of frame
+ * buffers through the DRM_IOCTL_MODE_ADDFB(2) ioctls and receive an opaque
+ * handle that can be passed to the KMS CRTC control, plane configuration and
+ * page flip functions.
+ *
+ * Frame buffers rely on the underlying memory manager for allocating backing
+ * storage. When creating a frame buffer applications pass a memory handle
+ * (or a list of memory handles for multi-planar formats) through the
+ * struct &drm_mode_fb_cmd2 argument. For drivers using GEM as their userspace
+ * buffer management interface this would be a GEM handle.  Drivers are however
+ * free to use their own backing storage object handles, e.g. vmwgfx directly
+ * exposes special TTM handles to userspace and so expects TTM handles in the
+ * create ioctl and not GEM handles.
+ *
+ * Framebuffers are tracked with struct &drm_framebuffer. They are published
+ * using drm_framebuffer_init() - after calling that function userspace can use
+ * and access the framebuffer object. The helper function
+ * drm_helper_mode_fill_fb_struct() can be used to pre-fill the required
+ * metadata fields.
+ *
+ * The lifetime of a drm framebuffer is controlled with a reference count,
+ * drivers can grab additional references with drm_framebuffer_reference() and
+ * drop them again with drm_framebuffer_unreference(). For driver-private
+ * framebuffers for which the last reference is never dropped (e.g. for the
+ * fbdev framebuffer when the struct struct &drm_framebuffer is embedded into
+ * the fbdev helper struct) drivers can manually clean up a framebuffer at
+ * module unload time with drm_framebuffer_unregister_private(). But doing this
+ * is not recommended, and it's better to have a normal free-standing struct
+ * &drm_framebuffer.
+ */
+
+int drm_framebuffer_check_src_coords(uint32_t src_x, uint32_t src_y,
+				     uint32_t src_w, uint32_t src_h,
+				     const struct drm_framebuffer *fb)
+{
+	unsigned int fb_width, fb_height;
+
+	fb_width = fb->width << 16;
+	fb_height = fb->height << 16;
+
+	/* Make sure source coordinates are inside the fb. */
+	if (src_w > fb_width ||
+	    src_x > fb_width - src_w ||
+	    src_h > fb_height ||
+	    src_y > fb_height - src_h) {
+		DRM_DEBUG_KMS("Invalid source coordinates "
+			      "%u.%06ux%u.%06u+%u.%06u+%u.%06u\n",
+			      src_w >> 16, ((src_w & 0xffff) * 15625) >> 10,
+			      src_h >> 16, ((src_h & 0xffff) * 15625) >> 10,
+			      src_x >> 16, ((src_x & 0xffff) * 15625) >> 10,
+			      src_y >> 16, ((src_y & 0xffff) * 15625) >> 10);
+		return -ENOSPC;
+	}
+
+	return 0;
+}
+
+/**
+ * drm_mode_addfb - add an FB to the graphics configuration
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Add a new FB to the specified CRTC, given a user request. This is the
+ * original addfb ioctl which only supported RGB formats.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_addfb(struct drm_device *dev,
+		   void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_fb_cmd *or = data;
+	struct drm_mode_fb_cmd2 r = {};
+	int ret;
+
+	/* convert to new format and call new ioctl */
+	r.fb_id = or->fb_id;
+	r.width = or->width;
+	r.height = or->height;
+	r.pitches[0] = or->pitch;
+	r.pixel_format = drm_mode_legacy_fb_format(or->bpp, or->depth);
+	r.handles[0] = or->handle;
+
+	ret = drm_mode_addfb2(dev, &r, file_priv);
+	if (ret)
+		return ret;
+
+	or->fb_id = r.fb_id;
+
+	return 0;
+}
+
+static int format_check(const struct drm_mode_fb_cmd2 *r)
+{
+	uint32_t format = r->pixel_format & ~DRM_FORMAT_BIG_ENDIAN;
+	char *format_name;
+
+	switch (format) {
+	case DRM_FORMAT_C8:
+	case DRM_FORMAT_RGB332:
+	case DRM_FORMAT_BGR233:
+	case DRM_FORMAT_XRGB4444:
+	case DRM_FORMAT_XBGR4444:
+	case DRM_FORMAT_RGBX4444:
+	case DRM_FORMAT_BGRX4444:
+	case DRM_FORMAT_ARGB4444:
+	case DRM_FORMAT_ABGR4444:
+	case DRM_FORMAT_RGBA4444:
+	case DRM_FORMAT_BGRA4444:
+	case DRM_FORMAT_XRGB1555:
+	case DRM_FORMAT_XBGR1555:
+	case DRM_FORMAT_RGBX5551:
+	case DRM_FORMAT_BGRX5551:
+	case DRM_FORMAT_ARGB1555:
+	case DRM_FORMAT_ABGR1555:
+	case DRM_FORMAT_RGBA5551:
+	case DRM_FORMAT_BGRA5551:
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_BGR565:
+	case DRM_FORMAT_RGB888:
+	case DRM_FORMAT_BGR888:
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_BGRA8888:
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_RGBX1010102:
+	case DRM_FORMAT_BGRX1010102:
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_ABGR2101010:
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_BGRA1010102:
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_AYUV:
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV21:
+	case DRM_FORMAT_NV16:
+	case DRM_FORMAT_NV61:
+	case DRM_FORMAT_NV24:
+	case DRM_FORMAT_NV42:
+	case DRM_FORMAT_YUV410:
+	case DRM_FORMAT_YVU410:
+	case DRM_FORMAT_YUV411:
+	case DRM_FORMAT_YVU411:
+	case DRM_FORMAT_YUV420:
+	case DRM_FORMAT_YVU420:
+	case DRM_FORMAT_YUV422:
+	case DRM_FORMAT_YVU422:
+	case DRM_FORMAT_YUV444:
+	case DRM_FORMAT_YVU444:
+		return 0;
+	default:
+		format_name = drm_get_format_name(r->pixel_format);
+		DRM_DEBUG_KMS("invalid pixel format %s\n", format_name);
+		kfree(format_name);
+		return -EINVAL;
+	}
+}
+
+static int framebuffer_check(const struct drm_mode_fb_cmd2 *r)
+{
+	int ret, hsub, vsub, num_planes, i;
+
+	ret = format_check(r);
+	if (ret) {
+		char *format_name = drm_get_format_name(r->pixel_format);
+		DRM_DEBUG_KMS("bad framebuffer format %s\n", format_name);
+		kfree(format_name);
+		return ret;
+	}
+
+	hsub = drm_format_horz_chroma_subsampling(r->pixel_format);
+	vsub = drm_format_vert_chroma_subsampling(r->pixel_format);
+	num_planes = drm_format_num_planes(r->pixel_format);
+
+	if (r->width == 0 || r->width % hsub) {
+		DRM_DEBUG_KMS("bad framebuffer width %u\n", r->width);
+		return -EINVAL;
+	}
+
+	if (r->height == 0 || r->height % vsub) {
+		DRM_DEBUG_KMS("bad framebuffer height %u\n", r->height);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < num_planes; i++) {
+		unsigned int width = r->width / (i != 0 ? hsub : 1);
+		unsigned int height = r->height / (i != 0 ? vsub : 1);
+		unsigned int cpp = drm_format_plane_cpp(r->pixel_format, i);
+
+		if (!r->handles[i]) {
+			DRM_DEBUG_KMS("no buffer object handle for plane %d\n", i);
+			return -EINVAL;
+		}
+
+		if ((uint64_t) width * cpp > UINT_MAX)
+			return -ERANGE;
+
+		if ((uint64_t) height * r->pitches[i] + r->offsets[i] > UINT_MAX)
+			return -ERANGE;
+
+		if (r->pitches[i] < width * cpp) {
+			DRM_DEBUG_KMS("bad pitch %u for plane %d\n", r->pitches[i], i);
+			return -EINVAL;
+		}
+
+		if (r->modifier[i] && !(r->flags & DRM_MODE_FB_MODIFIERS)) {
+			DRM_DEBUG_KMS("bad fb modifier %llu for plane %d\n",
+				      r->modifier[i], i);
+			return -EINVAL;
+		}
+
+		/* modifier specific checks: */
+		switch (r->modifier[i]) {
+		case DRM_FORMAT_MOD_SAMSUNG_64_32_TILE:
+			/* NOTE: the pitch restriction may be lifted later if it turns
+			 * out that no hw has this restriction:
+			 */
+			if (r->pixel_format != DRM_FORMAT_NV12 ||
+					width % 128 || height % 32 ||
+					r->pitches[i] % 128) {
+				DRM_DEBUG_KMS("bad modifier data for plane %d\n", i);
+				return -EINVAL;
+			}
+			break;
+
+		default:
+			break;
+		}
+	}
+
+	for (i = num_planes; i < 4; i++) {
+		if (r->modifier[i]) {
+			DRM_DEBUG_KMS("non-zero modifier for unused plane %d\n", i);
+			return -EINVAL;
+		}
+
+		/* Pre-FB_MODIFIERS userspace didn't clear the structs properly. */
+		if (!(r->flags & DRM_MODE_FB_MODIFIERS))
+			continue;
+
+		if (r->handles[i]) {
+			DRM_DEBUG_KMS("buffer object handle for unused plane %d\n", i);
+			return -EINVAL;
+		}
+
+		if (r->pitches[i]) {
+			DRM_DEBUG_KMS("non-zero pitch for unused plane %d\n", i);
+			return -EINVAL;
+		}
+
+		if (r->offsets[i]) {
+			DRM_DEBUG_KMS("non-zero offset for unused plane %d\n", i);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+struct drm_framebuffer *
+drm_internal_framebuffer_create(struct drm_device *dev,
+				const struct drm_mode_fb_cmd2 *r,
+				struct drm_file *file_priv)
+{
+	struct drm_mode_config *config = &dev->mode_config;
+	struct drm_framebuffer *fb;
+	int ret;
+
+	if (r->flags & ~(DRM_MODE_FB_INTERLACED | DRM_MODE_FB_MODIFIERS)) {
+		DRM_DEBUG_KMS("bad framebuffer flags 0x%08x\n", r->flags);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if ((config->min_width > r->width) || (r->width > config->max_width)) {
+		DRM_DEBUG_KMS("bad framebuffer width %d, should be >= %d && <= %d\n",
+			  r->width, config->min_width, config->max_width);
+		return ERR_PTR(-EINVAL);
+	}
+	if ((config->min_height > r->height) || (r->height > config->max_height)) {
+		DRM_DEBUG_KMS("bad framebuffer height %d, should be >= %d && <= %d\n",
+			  r->height, config->min_height, config->max_height);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (r->flags & DRM_MODE_FB_MODIFIERS &&
+	    !dev->mode_config.allow_fb_modifiers) {
+		DRM_DEBUG_KMS("driver does not support fb modifiers\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	ret = framebuffer_check(r);
+	if (ret)
+		return ERR_PTR(ret);
+
+	fb = dev->mode_config.funcs->fb_create(dev, file_priv, r);
+	if (IS_ERR(fb)) {
+		DRM_DEBUG_KMS("could not create framebuffer\n");
+		return fb;
+	}
+
+	return fb;
+}
+
+/**
+ * drm_mode_addfb2 - add an FB to the graphics configuration
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Add a new FB to the specified CRTC, given a user request with format. This is
+ * the 2nd version of the addfb ioctl, which supports multi-planar framebuffers
+ * and uses fourcc codes as pixel format specifiers.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_addfb2(struct drm_device *dev,
+		    void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_fb_cmd2 *r = data;
+	struct drm_framebuffer *fb;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	fb = drm_internal_framebuffer_create(dev, r, file_priv);
+	if (IS_ERR(fb))
+		return PTR_ERR(fb);
+
+	DRM_DEBUG_KMS("[FB:%d]\n", fb->base.id);
+	r->fb_id = fb->base.id;
+
+	/* Transfer ownership to the filp for reaping on close */
+	mutex_lock(&file_priv->fbs_lock);
+	list_add(&fb->filp_head, &file_priv->fbs);
+	mutex_unlock(&file_priv->fbs_lock);
+
+	return 0;
+}
+
+struct drm_mode_rmfb_work {
+	struct work_struct work;
+	struct list_head fbs;
+};
+
+static void drm_mode_rmfb_work_fn(struct work_struct *w)
+{
+	struct drm_mode_rmfb_work *arg = container_of(w, typeof(*arg), work);
+
+	while (!list_empty(&arg->fbs)) {
+		struct drm_framebuffer *fb =
+			list_first_entry(&arg->fbs, typeof(*fb), filp_head);
+
+		list_del_init(&fb->filp_head);
+		drm_framebuffer_remove(fb);
+	}
+}
+
+/**
+ * drm_mode_rmfb - remove an FB from the configuration
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Remove the FB specified by the user.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_rmfb(struct drm_device *dev,
+		   void *data, struct drm_file *file_priv)
+{
+	struct drm_framebuffer *fb = NULL;
+	struct drm_framebuffer *fbl = NULL;
+	uint32_t *id = data;
+	int found = 0;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	fb = drm_framebuffer_lookup(dev, *id);
+	if (!fb)
+		return -ENOENT;
+
+	mutex_lock(&file_priv->fbs_lock);
+	list_for_each_entry(fbl, &file_priv->fbs, filp_head)
+		if (fb == fbl)
+			found = 1;
+	if (!found) {
+		mutex_unlock(&file_priv->fbs_lock);
+		goto fail_unref;
+	}
+
+	list_del_init(&fb->filp_head);
+	mutex_unlock(&file_priv->fbs_lock);
+
+	/* drop the reference we picked up in framebuffer lookup */
+	drm_framebuffer_unreference(fb);
+
+	/*
+	 * we now own the reference that was stored in the fbs list
+	 *
+	 * drm_framebuffer_remove may fail with -EINTR on pending signals,
+	 * so run this in a separate stack as there's no way to correctly
+	 * handle this after the fb is already removed from the lookup table.
+	 */
+	if (drm_framebuffer_read_refcount(fb) > 1) {
+		struct drm_mode_rmfb_work arg;
+
+		INIT_WORK_ONSTACK(&arg.work, drm_mode_rmfb_work_fn);
+		INIT_LIST_HEAD(&arg.fbs);
+		list_add_tail(&fb->filp_head, &arg.fbs);
+
+		schedule_work(&arg.work);
+		flush_work(&arg.work);
+		destroy_work_on_stack(&arg.work);
+	} else
+		drm_framebuffer_unreference(fb);
+
+	return 0;
+
+fail_unref:
+	drm_framebuffer_unreference(fb);
+	return -ENOENT;
+}
+
+/**
+ * drm_mode_getfb - get FB info
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Lookup the FB given its ID and return info about it.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_getfb(struct drm_device *dev,
+		   void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_fb_cmd *r = data;
+	struct drm_framebuffer *fb;
+	int ret;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	fb = drm_framebuffer_lookup(dev, r->fb_id);
+	if (!fb)
+		return -ENOENT;
+
+	r->height = fb->height;
+	r->width = fb->width;
+	r->depth = fb->depth;
+	r->bpp = fb->bits_per_pixel;
+	r->pitch = fb->pitches[0];
+	if (fb->funcs->create_handle) {
+		if (drm_is_current_master(file_priv) || capable(CAP_SYS_ADMIN) ||
+		    drm_is_control_client(file_priv)) {
+			ret = fb->funcs->create_handle(fb, file_priv,
+						       &r->handle);
+		} else {
+			/* GET_FB() is an unprivileged ioctl so we must not
+			 * return a buffer-handle to non-master processes! For
+			 * backwards-compatibility reasons, we cannot make
+			 * GET_FB() privileged, so just return an invalid handle
+			 * for non-masters. */
+			r->handle = 0;
+			ret = 0;
+		}
+	} else {
+		ret = -ENODEV;
+	}
+
+	drm_framebuffer_unreference(fb);
+
+	return ret;
+}
+
+/**
+ * drm_mode_dirtyfb_ioctl - flush frontbuffer rendering on an FB
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Lookup the FB and flush out the damaged area supplied by userspace as a clip
+ * rectangle list. Generic userspace which does frontbuffer rendering must call
+ * this ioctl to flush out the changes on manual-update display outputs, e.g.
+ * usb display-link, mipi manual update panels or edp panel self refresh modes.
+ *
+ * Modesetting drivers which always update the frontbuffer do not need to
+ * implement the corresponding ->dirty framebuffer callback.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
+			   void *data, struct drm_file *file_priv)
+{
+	struct drm_clip_rect __user *clips_ptr;
+	struct drm_clip_rect *clips = NULL;
+	struct drm_mode_fb_dirty_cmd *r = data;
+	struct drm_framebuffer *fb;
+	unsigned flags;
+	int num_clips;
+	int ret;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	fb = drm_framebuffer_lookup(dev, r->fb_id);
+	if (!fb)
+		return -ENOENT;
+
+	num_clips = r->num_clips;
+	clips_ptr = (struct drm_clip_rect __user *)(unsigned long)r->clips_ptr;
+
+	if (!num_clips != !clips_ptr) {
+		ret = -EINVAL;
+		goto out_err1;
+	}
+
+	flags = DRM_MODE_FB_DIRTY_FLAGS & r->flags;
+
+	/* If userspace annotates copy, clips must come in pairs */
+	if (flags & DRM_MODE_FB_DIRTY_ANNOTATE_COPY && (num_clips % 2)) {
+		ret = -EINVAL;
+		goto out_err1;
+	}
+
+	if (num_clips && clips_ptr) {
+		if (num_clips < 0 || num_clips > DRM_MODE_FB_DIRTY_MAX_CLIPS) {
+			ret = -EINVAL;
+			goto out_err1;
+		}
+		clips = kcalloc(num_clips, sizeof(*clips), GFP_KERNEL);
+		if (!clips) {
+			ret = -ENOMEM;
+			goto out_err1;
+		}
+
+		ret = copy_from_user(clips, clips_ptr,
+				     num_clips * sizeof(*clips));
+		if (ret) {
+			ret = -EFAULT;
+			goto out_err2;
+		}
+	}
+
+	if (fb->funcs->dirty) {
+		ret = fb->funcs->dirty(fb, file_priv, flags, r->color,
+				       clips, num_clips);
+	} else {
+		ret = -ENOSYS;
+	}
+
+out_err2:
+	kfree(clips);
+out_err1:
+	drm_framebuffer_unreference(fb);
+
+	return ret;
+}
+
+/**
+ * drm_fb_release - remove and free the FBs on this file
+ * @priv: drm file for the ioctl
+ *
+ * Destroy all the FBs associated with @filp.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+void drm_fb_release(struct drm_file *priv)
+{
+	struct drm_framebuffer *fb, *tfb;
+	struct drm_mode_rmfb_work arg;
+
+	INIT_LIST_HEAD(&arg.fbs);
+
+	/*
+	 * When the file gets released that means no one else can access the fb
+	 * list any more, so no need to grab fpriv->fbs_lock. And we need to
+	 * avoid upsetting lockdep since the universal cursor code adds a
+	 * framebuffer while holding mutex locks.
+	 *
+	 * Note that a real deadlock between fpriv->fbs_lock and the modeset
+	 * locks is impossible here since no one else but this function can get
+	 * at it any more.
+	 */
+	list_for_each_entry_safe(fb, tfb, &priv->fbs, filp_head) {
+		if (drm_framebuffer_read_refcount(fb) > 1) {
+			list_move_tail(&fb->filp_head, &arg.fbs);
+		} else {
+			list_del_init(&fb->filp_head);
+
+			/* This drops the fpriv->fbs reference. */
+			drm_framebuffer_unreference(fb);
+		}
+	}
+
+	if (!list_empty(&arg.fbs)) {
+		INIT_WORK_ONSTACK(&arg.work, drm_mode_rmfb_work_fn);
+
+		schedule_work(&arg.work);
+		flush_work(&arg.work);
+		destroy_work_on_stack(&arg.work);
+	}
+}
+
+void drm_framebuffer_free(struct kref *kref)
+{
+	struct drm_framebuffer *fb =
+			container_of(kref, struct drm_framebuffer, base.refcount);
+	struct drm_device *dev = fb->dev;
+
+	/*
+	 * The lookup idr holds a weak reference, which has not necessarily been
+	 * removed at this point. Check for that.
+	 */
+	drm_mode_object_unregister(dev, &fb->base);
+
+	fb->funcs->destroy(fb);
+}
+
+/**
+ * drm_framebuffer_init - initialize a framebuffer
+ * @dev: DRM device
+ * @fb: framebuffer to be initialized
+ * @funcs: ... with these functions
+ *
+ * Allocates an ID for the framebuffer's parent mode object, sets its mode
+ * functions & device file and adds it to the master fd list.
+ *
+ * IMPORTANT:
+ * This functions publishes the fb and makes it available for concurrent access
+ * by other users. Which means by this point the fb _must_ be fully set up -
+ * since all the fb attributes are invariant over its lifetime, no further
+ * locking but only correct reference counting is required.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_framebuffer_init(struct drm_device *dev, struct drm_framebuffer *fb,
+			 const struct drm_framebuffer_funcs *funcs)
+{
+	int ret;
+
+	INIT_LIST_HEAD(&fb->filp_head);
+	fb->dev = dev;
+	fb->funcs = funcs;
+
+	ret = drm_mode_object_get_reg(dev, &fb->base, DRM_MODE_OBJECT_FB,
+				      false, drm_framebuffer_free);
+	if (ret)
+		goto out;
+
+	mutex_lock(&dev->mode_config.fb_lock);
+	dev->mode_config.num_fb++;
+	list_add(&fb->head, &dev->mode_config.fb_list);
+	mutex_unlock(&dev->mode_config.fb_lock);
+
+	drm_mode_object_register(dev, &fb->base);
+out:
+	return ret;
+}
+EXPORT_SYMBOL(drm_framebuffer_init);
+
+/**
+ * drm_framebuffer_lookup - look up a drm framebuffer and grab a reference
+ * @dev: drm device
+ * @id: id of the fb object
+ *
+ * If successful, this grabs an additional reference to the framebuffer -
+ * callers need to make sure to eventually unreference the returned framebuffer
+ * again, using @drm_framebuffer_unreference.
+ */
+struct drm_framebuffer *drm_framebuffer_lookup(struct drm_device *dev,
+					       uint32_t id)
+{
+	struct drm_mode_object *obj;
+	struct drm_framebuffer *fb = NULL;
+
+	obj = __drm_mode_object_find(dev, id, DRM_MODE_OBJECT_FB);
+	if (obj)
+		fb = obj_to_fb(obj);
+	return fb;
+}
+EXPORT_SYMBOL(drm_framebuffer_lookup);
+
+/**
+ * drm_framebuffer_unregister_private - unregister a private fb from the lookup idr
+ * @fb: fb to unregister
+ *
+ * Drivers need to call this when cleaning up driver-private framebuffers, e.g.
+ * those used for fbdev. Note that the caller must hold a reference of it's own,
+ * i.e. the object may not be destroyed through this call (since it'll lead to a
+ * locking inversion).
+ */
+void drm_framebuffer_unregister_private(struct drm_framebuffer *fb)
+{
+	struct drm_device *dev;
+
+	if (!fb)
+		return;
+
+	dev = fb->dev;
+
+	/* Mark fb as reaped and drop idr ref. */
+	drm_mode_object_unregister(dev, &fb->base);
+}
+EXPORT_SYMBOL(drm_framebuffer_unregister_private);
+
+/**
+ * drm_framebuffer_cleanup - remove a framebuffer object
+ * @fb: framebuffer to remove
+ *
+ * Cleanup framebuffer. This function is intended to be used from the drivers
+ * ->destroy callback. It can also be used to clean up driver private
+ * framebuffers embedded into a larger structure.
+ *
+ * Note that this function does not remove the fb from active usuage - if it is
+ * still used anywhere, hilarity can ensue since userspace could call getfb on
+ * the id and get back -EINVAL. Obviously no concern at driver unload time.
+ *
+ * Also, the framebuffer will not be removed from the lookup idr - for
+ * user-created framebuffers this will happen in in the rmfb ioctl. For
+ * driver-private objects (e.g. for fbdev) drivers need to explicitly call
+ * drm_framebuffer_unregister_private.
+ */
+void drm_framebuffer_cleanup(struct drm_framebuffer *fb)
+{
+	struct drm_device *dev = fb->dev;
+
+	mutex_lock(&dev->mode_config.fb_lock);
+	list_del(&fb->head);
+	dev->mode_config.num_fb--;
+	mutex_unlock(&dev->mode_config.fb_lock);
+}
+EXPORT_SYMBOL(drm_framebuffer_cleanup);
+
+/**
+ * drm_framebuffer_remove - remove and unreference a framebuffer object
+ * @fb: framebuffer to remove
+ *
+ * Scans all the CRTCs and planes in @dev's mode_config.  If they're
+ * using @fb, removes it, setting it to NULL. Then drops the reference to the
+ * passed-in framebuffer. Might take the modeset locks.
+ *
+ * Note that this function optimizes the cleanup away if the caller holds the
+ * last reference to the framebuffer. It is also guaranteed to not take the
+ * modeset locks in this case.
+ */
+void drm_framebuffer_remove(struct drm_framebuffer *fb)
+{
+	struct drm_device *dev;
+	struct drm_crtc *crtc;
+	struct drm_plane *plane;
+
+	if (!fb)
+		return;
+
+	dev = fb->dev;
+
+	WARN_ON(!list_empty(&fb->filp_head));
+
+	/*
+	 * drm ABI mandates that we remove any deleted framebuffers from active
+	 * useage. But since most sane clients only remove framebuffers they no
+	 * longer need, try to optimize this away.
+	 *
+	 * Since we're holding a reference ourselves, observing a refcount of 1
+	 * means that we're the last holder and can skip it. Also, the refcount
+	 * can never increase from 1 again, so we don't need any barriers or
+	 * locks.
+	 *
+	 * Note that userspace could try to race with use and instate a new
+	 * usage _after_ we've cleared all current ones. End result will be an
+	 * in-use fb with fb-id == 0. Userspace is allowed to shoot its own foot
+	 * in this manner.
+	 */
+	if (drm_framebuffer_read_refcount(fb) > 1) {
+		drm_modeset_lock_all(dev);
+		/* remove from any CRTC */
+		drm_for_each_crtc(crtc, dev) {
+			if (crtc->primary->fb == fb) {
+				/* should turn off the crtc */
+				if (drm_crtc_force_disable(crtc))
+					DRM_ERROR("failed to reset crtc %p when fb was deleted\n", crtc);
+			}
+		}
+
+		drm_for_each_plane(plane, dev) {
+			if (plane->fb == fb)
+				drm_plane_force_disable(plane);
+		}
+		drm_modeset_unlock_all(dev);
+	}
+
+	drm_framebuffer_unreference(fb);
+}
+EXPORT_SYMBOL(drm_framebuffer_remove);
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 9134ae134667..465bacd0a630 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -257,7 +257,7 @@ drm_gem_object_release_handle(int id, void *ptr, void *data)
 
 	if (drm_core_check_feature(dev, DRIVER_PRIME))
 		drm_gem_remove_prime_handles(obj, file_priv);
-	drm_vma_node_revoke(&obj->vma_node, file_priv->filp);
+	drm_vma_node_revoke(&obj->vma_node, file_priv);
 
 	if (dev->driver->gem_close_object)
 		dev->driver->gem_close_object(obj, file_priv);
@@ -372,7 +372,7 @@ drm_gem_handle_create_tail(struct drm_file *file_priv,
 
 	handle = ret;
 
-	ret = drm_vma_node_allow(&obj->vma_node, file_priv->filp);
+	ret = drm_vma_node_allow(&obj->vma_node, file_priv);
 	if (ret)
 		goto err_remove;
 
@@ -386,7 +386,7 @@ drm_gem_handle_create_tail(struct drm_file *file_priv,
 	return 0;
 
 err_revoke:
-	drm_vma_node_revoke(&obj->vma_node, file_priv->filp);
+	drm_vma_node_revoke(&obj->vma_node, file_priv);
 err_remove:
 	spin_lock(&file_priv->table_lock);
 	idr_remove(&file_priv->object_idr, handle);
@@ -991,7 +991,7 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 	if (!obj)
 		return -EINVAL;
 
-	if (!drm_vma_node_is_allowed(node, filp)) {
+	if (!drm_vma_node_is_allowed(node, priv)) {
 		drm_gem_object_unreference_unlocked(obj);
 		return -EACCES;
 	}
diff --git a/drivers/gpu/drm/drm_global.c b/drivers/gpu/drm/drm_global.c
index 3d2e91c4d78e..b404287abb97 100644
--- a/drivers/gpu/drm/drm_global.c
+++ b/drivers/gpu/drm/drm_global.c
@@ -65,30 +65,34 @@ void drm_global_release(void)
 
 int drm_global_item_ref(struct drm_global_reference *ref)
 {
-	int ret;
+	int ret = 0;
 	struct drm_global_item *item = &glob[ref->global_type];
 
 	mutex_lock(&item->mutex);
 	if (item->refcount == 0) {
-		item->object = kzalloc(ref->size, GFP_KERNEL);
-		if (unlikely(item->object == NULL)) {
+		ref->object = kzalloc(ref->size, GFP_KERNEL);
+		if (unlikely(ref->object == NULL)) {
 			ret = -ENOMEM;
-			goto out_err;
+			goto error_unlock;
 		}
-
-		ref->object = item->object;
 		ret = ref->init(ref);
 		if (unlikely(ret != 0))
-			goto out_err;
+			goto error_free;
 
+		item->object = ref->object;
+	} else {
+		ref->object = item->object;
 	}
+
 	++item->refcount;
-	ref->object = item->object;
 	mutex_unlock(&item->mutex);
 	return 0;
-out_err:
+
+error_free:
+	kfree(ref->object);
+	ref->object = NULL;
+error_unlock:
 	mutex_unlock(&item->mutex);
-	item->object = NULL;
 	return ret;
 }
 EXPORT_SYMBOL(drm_global_item_ref);
diff --git a/drivers/gpu/drm/drm_hashtab.c b/drivers/gpu/drm/drm_hashtab.c
index 7b30b307674b..dae18e58e79b 100644
--- a/drivers/gpu/drm/drm_hashtab.c
+++ b/drivers/gpu/drm/drm_hashtab.c
@@ -142,7 +142,7 @@ int drm_ht_just_insert_please(struct drm_open_hash *ht, struct drm_hash_item *it
 			      unsigned long add)
 {
 	int ret;
-	unsigned long mask = (1 << bits) - 1;
+	unsigned long mask = (1UL << bits) - 1;
 	unsigned long first, unshifted_key;
 
 	unshifted_key = hash_long(seed, bits);
diff --git a/drivers/gpu/drm/drm_info.c b/drivers/gpu/drm/drm_info.c
index 9ae353f4dd06..1df2d33d0b40 100644
--- a/drivers/gpu/drm/drm_info.c
+++ b/drivers/gpu/drm/drm_info.c
@@ -80,6 +80,7 @@ int drm_clients_info(struct seq_file *m, void *data)
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct drm_file *priv;
+	kuid_t uid;
 
 	seq_printf(m,
 		   "%20s %5s %3s master a %5s %10s\n",
@@ -98,13 +99,14 @@ int drm_clients_info(struct seq_file *m, void *data)
 
 		rcu_read_lock(); /* locks pid_task()->comm */
 		task = pid_task(priv->pid, PIDTYPE_PID);
+		uid = task ? __task_cred(task)->euid : GLOBAL_ROOT_UID;
 		seq_printf(m, "%20s %5d %3d   %c    %c %5d %10u\n",
 			   task ? task->comm : "<unknown>",
 			   pid_vnr(priv->pid),
 			   priv->minor->index,
 			   drm_is_current_master(priv) ? 'y' : 'n',
 			   priv->authenticated ? 'y' : 'n',
-			   from_kuid_munged(seq_user_ns(m), priv->uid),
+			   from_kuid_munged(seq_user_ns(m), uid),
 			   priv->magic);
 		rcu_read_unlock();
 	}
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index b86dc9b921a5..e66af289a016 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -21,6 +21,9 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#define DRM_IF_MAJOR 1
+#define DRM_IF_MINOR 4
+
 /* drm_irq.c */
 extern unsigned int drm_timestamp_monotonic;
 
diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c
index 57676f8d7ecf..867ab8c1582b 100644
--- a/drivers/gpu/drm/drm_ioc32.c
+++ b/drivers/gpu/drm/drm_ioc32.c
@@ -32,7 +32,6 @@
 #include <linux/export.h>
 
 #include <drm/drmP.h>
-#include <drm/drm_core.h>
 
 #define DRM_IOCTL_VERSION32		DRM_IOWR(0x00, drm_version32_t)
 #define DRM_IOCTL_GET_UNIQUE32		DRM_IOWR(0x01, drm_unique32_t)
@@ -346,6 +345,7 @@ static int compat_drm_getstats(struct file *file, unsigned int cmd,
 	struct drm_stats __user *stats;
 	int i, err;
 
+	memset(&s32, 0, sizeof(drm_stats32_t));
 	stats = compat_alloc_user_space(sizeof(*stats));
 	if (!stats)
 		return -EFAULT;
@@ -1015,6 +1015,7 @@ static int compat_drm_wait_vblank(struct file *file, unsigned int cmd,
 	return 0;
 }
 
+#if defined(CONFIG_X86) || defined(CONFIG_IA64)
 typedef struct drm_mode_fb_cmd232 {
 	u32 fb_id;
 	u32 width;
@@ -1071,6 +1072,7 @@ static int compat_drm_mode_addfb2(struct file *file, unsigned int cmd,
 
 	return 0;
 }
+#endif
 
 static drm_ioctl_compat_t *drm_compat_ioctls[] = {
 	[DRM_IOCTL_NR(DRM_IOCTL_VERSION32)] = compat_drm_version,
@@ -1104,7 +1106,9 @@ static drm_ioctl_compat_t *drm_compat_ioctls[] = {
 	[DRM_IOCTL_NR(DRM_IOCTL_UPDATE_DRAW32)] = compat_drm_update_draw,
 #endif
 	[DRM_IOCTL_NR(DRM_IOCTL_WAIT_VBLANK32)] = compat_drm_wait_vblank,
+#if defined(CONFIG_X86) || defined(CONFIG_IA64)
 	[DRM_IOCTL_NR(DRM_IOCTL_MODE_ADDFB232)] = compat_drm_mode_addfb2,
+#endif
 };
 
 /**
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index 33af4a5ddca1..0ad2c47f808f 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -29,7 +29,6 @@
  */
 
 #include <drm/drmP.h>
-#include <drm/drm_core.h>
 #include <drm/drm_auth.h>
 #include "drm_legacy.h"
 #include "drm_internal.h"
@@ -189,9 +188,8 @@ static int drm_getclient(struct drm_device *dev, void *data,
 	 */
 	if (client->idx == 0) {
 		client->auth = file_priv->authenticated;
-		client->pid = pid_vnr(file_priv->pid);
-		client->uid = from_kuid_munged(current_user_ns(),
-					       file_priv->uid);
+		client->pid = task_pid_vnr(current);
+		client->uid = overflowuid;
 		client->magic = 0;
 		client->iocs = 0;
 
@@ -228,6 +226,7 @@ static int drm_getstats(struct drm_device *dev, void *data,
 static int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
 	struct drm_get_cap *req = data;
+	struct drm_crtc *crtc;
 
 	req->value = 0;
 	switch (req->capability) {
@@ -254,6 +253,13 @@ static int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_
 	case DRM_CAP_ASYNC_PAGE_FLIP:
 		req->value = dev->mode_config.async_page_flip;
 		break;
+	case DRM_CAP_PAGE_FLIP_TARGET:
+		req->value = 1;
+		drm_for_each_crtc(crtc, dev) {
+			if (!crtc->funcs->page_flip_target)
+				req->value = 0;
+		}
+		break;
 	case DRM_CAP_CURSOR_WIDTH:
 		if (dev->mode_config.cursor_width)
 			req->value = dev->mode_config.cursor_width;
@@ -714,9 +720,9 @@ long drm_ioctl(struct file *filp,
 	if (ksize > in_size)
 		memset(kdata + in_size, 0, ksize - in_size);
 
-	/* Enforce sane locking for kms driver ioctls. Core ioctls are
+	/* Enforce sane locking for modern driver ioctls. Core ioctls are
 	 * too messy still. */
-	if ((drm_core_check_feature(dev, DRIVER_MODESET) && is_driver_ioctl) ||
+	if ((!drm_core_check_feature(dev, DRIVER_LEGACY) && is_driver_ioctl) ||
 	    (ioctl->flags & DRM_UNLOCKED))
 		retcode = func(dev, kdata, file_priv);
 	else {
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 77f357b2c386..b969a64a1514 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -482,7 +482,7 @@ int drm_irq_install(struct drm_device *dev, int irq)
 		return ret;
 	}
 
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+	if (drm_core_check_feature(dev, DRIVER_LEGACY))
 		vga_client_register(dev->pdev, (void *)dev, drm_irq_vgaarb_nokms, NULL);
 
 	/* After installing handler */
@@ -491,7 +491,7 @@ int drm_irq_install(struct drm_device *dev, int irq)
 
 	if (ret < 0) {
 		dev->irq_enabled = false;
-		if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		if (drm_core_check_feature(dev, DRIVER_LEGACY))
 			vga_client_register(dev->pdev, NULL, NULL, NULL);
 		free_irq(irq, dev);
 	} else {
@@ -557,7 +557,7 @@ int drm_irq_uninstall(struct drm_device *dev)
 
 	DRM_DEBUG("irq=%d\n", dev->irq);
 
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+	if (drm_core_check_feature(dev, DRIVER_LEGACY))
 		vga_client_register(dev->pdev, NULL, NULL, NULL);
 
 	if (dev->driver->irq_uninstall)
@@ -592,7 +592,7 @@ int drm_control(struct drm_device *dev, void *data,
 
 	if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
 		return 0;
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return 0;
 	/* UMS was only ever supported on pci devices. */
 	if (WARN_ON(!dev->pdev))
@@ -713,10 +713,10 @@ EXPORT_SYMBOL(drm_calc_timestamping_constants);
  * Negative value on error, failure or if not supported in current
  * video mode:
  *
- * -EINVAL   - Invalid CRTC.
- * -EAGAIN   - Temporary unavailable, e.g., called before initial modeset.
- * -ENOTSUPP - Function not supported in current display mode.
- * -EIO      - Failed, e.g., due to failed scanout position query.
+ * -EINVAL    Invalid CRTC.
+ * -EAGAIN    Temporary unavailable, e.g., called before initial modeset.
+ * -ENOTSUPP  Function not supported in current display mode.
+ * -EIO       Failed, e.g., due to failed scanout position query.
  *
  * Returns or'ed positive status flags on success:
  *
@@ -1008,6 +1008,31 @@ static void send_vblank_event(struct drm_device *dev,
  * period. This helper function implements exactly the required vblank arming
  * behaviour.
  *
+ * NOTE: Drivers using this to send out the event in struct &drm_crtc_state
+ * as part of an atomic commit must ensure that the next vblank happens at
+ * exactly the same time as the atomic commit is committed to the hardware. This
+ * function itself does **not** protect again the next vblank interrupt racing
+ * with either this function call or the atomic commit operation. A possible
+ * sequence could be:
+ *
+ * 1. Driver commits new hardware state into vblank-synchronized registers.
+ * 2. A vblank happens, committing the hardware state. Also the corresponding
+ *    vblank interrupt is fired off and fully processed by the interrupt
+ *    handler.
+ * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
+ * 4. The event is only send out for the next vblank, which is wrong.
+ *
+ * An equivalent race can happen when the driver calls
+ * drm_crtc_arm_vblank_event() before writing out the new hardware state.
+ *
+ * The only way to make this work safely is to prevent the vblank from firing
+ * (and the hardware from committing anything else) until the entire atomic
+ * commit sequence has run to completion. If the hardware does not have such a
+ * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
+ * Instead drivers need to manually send out the event from their interrupt
+ * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
+ * possible race with the hardware committing the atomic update.
+ *
  * Caller must hold event lock. Caller must also hold a vblank reference for
  * the event @e, which will be dropped when the next vblank arrives.
  */
@@ -1030,8 +1055,11 @@ EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
  * @crtc: the source CRTC of the vblank event
  * @e: the event to send
  *
- * Updates sequence # and timestamp on event, and sends it to userspace.
- * Caller must hold event lock.
+ * Updates sequence # and timestamp on event for the most recently processed
+ * vblank, and sends it to userspace.  Caller must hold event lock.
+ *
+ * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
+ * situation, especially to send out events for atomic commit operations.
  */
 void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
 				struct drm_pending_vblank_event *e)
@@ -1295,7 +1323,7 @@ void drm_vblank_off(struct drm_device *dev, unsigned int pipe)
 		if (e->pipe != pipe)
 			continue;
 		DRM_DEBUG("Sending premature vblank event on disable: "
-			  "wanted %d, current %d\n",
+			  "wanted %u, current %u\n",
 			  e->event.sequence, seq);
 		list_del(&e->base.link);
 		drm_vblank_put(dev, pipe);
@@ -1519,7 +1547,7 @@ int drm_modeset_ctl(struct drm_device *dev, void *data,
 		return 0;
 
 	/* KMS drivers handle this internally */
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return 0;
 
 	pipe = modeset->crtc;
@@ -1585,7 +1613,7 @@ static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
 
 	seq = drm_vblank_count_and_time(dev, pipe, &now);
 
-	DRM_DEBUG("event on vblank count %d, current %d, crtc %u\n",
+	DRM_DEBUG("event on vblank count %u, current %u, crtc %u\n",
 		  vblwait->request.sequence, seq, pipe);
 
 	trace_drm_vblank_event_queued(current->pid, pipe,
@@ -1693,7 +1721,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
 		return drm_queue_vblank_event(dev, pipe, vblwait, file_priv);
 	}
 
-	DRM_DEBUG("waiting on vblank count %d, crtc %u\n",
+	DRM_DEBUG("waiting on vblank count %u, crtc %u\n",
 		  vblwait->request.sequence, pipe);
 	DRM_WAIT_ON(ret, vblank->queue, 3 * HZ,
 		    (((drm_vblank_count(dev, pipe) -
@@ -1708,7 +1736,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
 		vblwait->reply.tval_sec = now.tv_sec;
 		vblwait->reply.tval_usec = now.tv_usec;
 
-		DRM_DEBUG("returning %d to client\n",
+		DRM_DEBUG("returning %u to client\n",
 			  vblwait->reply.sequence);
 	} else {
 		DRM_DEBUG("vblank wait interrupted by signal\n");
@@ -1735,7 +1763,7 @@ static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
 		if ((seq - e->event.sequence) > (1<<23))
 			continue;
 
-		DRM_DEBUG("vblank event on %d, current %d\n",
+		DRM_DEBUG("vblank event on %u, current %u\n",
 			  e->event.sequence, seq);
 
 		list_del(&e->base.link);
@@ -1826,6 +1854,7 @@ EXPORT_SYMBOL(drm_crtc_handle_vblank);
  */
 u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
 {
+	WARN_ON_ONCE(dev->max_vblank_count != 0);
 	return 0;
 }
 EXPORT_SYMBOL(drm_vblank_no_hw_counter);
diff --git a/drivers/gpu/drm/drm_kms_helper_common.c b/drivers/gpu/drm/drm_kms_helper_common.c
index 3187c4bb01cb..45db36cd3d20 100644
--- a/drivers/gpu/drm/drm_kms_helper_common.c
+++ b/drivers/gpu/drm/drm_kms_helper_common.c
@@ -27,7 +27,8 @@
 
 #include <drm/drmP.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_dp_aux_dev.h>
+
+#include "drm_crtc_helper_internal.h"
 
 MODULE_AUTHOR("David Airlie, Jesse Barnes");
 MODULE_DESCRIPTION("DRM KMS helper");
diff --git a/drivers/gpu/drm/drm_lock.c b/drivers/gpu/drm/drm_lock.c
index 48ac0ebbd663..c901f3c5b269 100644
--- a/drivers/gpu/drm/drm_lock.c
+++ b/drivers/gpu/drm/drm_lock.c
@@ -163,7 +163,7 @@ int drm_legacy_lock(struct drm_device *dev, void *data,
 	struct drm_master *master = file_priv->master;
 	int ret = 0;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	++file_priv->lock_count;
@@ -252,7 +252,7 @@ int drm_legacy_unlock(struct drm_device *dev, void *data, struct drm_file *file_
 	struct drm_lock *lock = data;
 	struct drm_master *master = file_priv->master;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (lock->context == DRM_KERNEL_CONTEXT) {
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index af0d471ee246..1160a579e0dc 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -999,6 +999,27 @@ int mipi_dsi_dcs_set_tear_on(struct mipi_dsi_device *dsi,
 EXPORT_SYMBOL(mipi_dsi_dcs_set_tear_on);
 
 /**
+ * mipi_dsi_dcs_set_pixel_format() - sets the pixel format for the RGB image
+ *    data used by the interface
+ * @dsi: DSI peripheral device
+ * @format: pixel format
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int mipi_dsi_dcs_set_pixel_format(struct mipi_dsi_device *dsi, u8 format)
+{
+	ssize_t err;
+
+	err = mipi_dsi_dcs_write(dsi, MIPI_DCS_SET_PIXEL_FORMAT, &format,
+				 sizeof(format));
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+EXPORT_SYMBOL(mipi_dsi_dcs_set_pixel_format);
+
+/**
  * mipi_dsi_dcs_set_tear_scanline() - set the scanline to use as trigger for
  *    the Tearing Effect output signal of the display module
  * @dsi: DSI peripheral device
@@ -1021,25 +1042,53 @@ int mipi_dsi_dcs_set_tear_scanline(struct mipi_dsi_device *dsi, u16 scanline)
 EXPORT_SYMBOL(mipi_dsi_dcs_set_tear_scanline);
 
 /**
- * mipi_dsi_dcs_set_pixel_format() - sets the pixel format for the RGB image
- *    data used by the interface
+ * mipi_dsi_dcs_set_display_brightness() - sets the brightness value of the
+ *    display
  * @dsi: DSI peripheral device
- * @format: pixel format
+ * @brightness: brightness value
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int mipi_dsi_dcs_set_pixel_format(struct mipi_dsi_device *dsi, u8 format)
+int mipi_dsi_dcs_set_display_brightness(struct mipi_dsi_device *dsi,
+					u16 brightness)
 {
+	u8 payload[2] = { brightness & 0xff, brightness >> 8 };
 	ssize_t err;
 
-	err = mipi_dsi_dcs_write(dsi, MIPI_DCS_SET_PIXEL_FORMAT, &format,
-				 sizeof(format));
+	err = mipi_dsi_dcs_write(dsi, MIPI_DCS_SET_DISPLAY_BRIGHTNESS,
+				 payload, sizeof(payload));
 	if (err < 0)
 		return err;
 
 	return 0;
 }
-EXPORT_SYMBOL(mipi_dsi_dcs_set_pixel_format);
+EXPORT_SYMBOL(mipi_dsi_dcs_set_display_brightness);
+
+/**
+ * mipi_dsi_dcs_get_display_brightness() - gets the current brightness value
+ *    of the display
+ * @dsi: DSI peripheral device
+ * @brightness: brightness value
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int mipi_dsi_dcs_get_display_brightness(struct mipi_dsi_device *dsi,
+					u16 *brightness)
+{
+	ssize_t err;
+
+	err = mipi_dsi_dcs_read(dsi, MIPI_DCS_GET_DISPLAY_BRIGHTNESS,
+				brightness, sizeof(*brightness));
+	if (err <= 0) {
+		if (err == 0)
+			err = -ENODATA;
+
+		return err;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(mipi_dsi_dcs_get_display_brightness);
 
 static int mipi_dsi_drv_probe(struct device *dev)
 {
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index cb39f45d6a16..11d44a1e0ab3 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -46,6 +46,7 @@
 #include <linux/slab.h>
 #include <linux/seq_file.h>
 #include <linux/export.h>
+#include <linux/interval_tree_generic.h>
 
 /**
  * DOC: Overview
@@ -103,6 +104,72 @@ static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_
 						u64 end,
 						enum drm_mm_search_flags flags);
 
+#define START(node) ((node)->start)
+#define LAST(node)  ((node)->start + (node)->size - 1)
+
+INTERVAL_TREE_DEFINE(struct drm_mm_node, rb,
+		     u64, __subtree_last,
+		     START, LAST, static inline, drm_mm_interval_tree)
+
+struct drm_mm_node *
+drm_mm_interval_first(struct drm_mm *mm, u64 start, u64 last)
+{
+	return drm_mm_interval_tree_iter_first(&mm->interval_tree,
+					       start, last);
+}
+EXPORT_SYMBOL(drm_mm_interval_first);
+
+struct drm_mm_node *
+drm_mm_interval_next(struct drm_mm_node *node, u64 start, u64 last)
+{
+	return drm_mm_interval_tree_iter_next(node, start, last);
+}
+EXPORT_SYMBOL(drm_mm_interval_next);
+
+static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
+					  struct drm_mm_node *node)
+{
+	struct drm_mm *mm = hole_node->mm;
+	struct rb_node **link, *rb;
+	struct drm_mm_node *parent;
+
+	node->__subtree_last = LAST(node);
+
+	if (hole_node->allocated) {
+		rb = &hole_node->rb;
+		while (rb) {
+			parent = rb_entry(rb, struct drm_mm_node, rb);
+			if (parent->__subtree_last >= node->__subtree_last)
+				break;
+
+			parent->__subtree_last = node->__subtree_last;
+			rb = rb_parent(rb);
+		}
+
+		rb = &hole_node->rb;
+		link = &hole_node->rb.rb_right;
+	} else {
+		rb = NULL;
+		link = &mm->interval_tree.rb_node;
+	}
+
+	while (*link) {
+		rb = *link;
+		parent = rb_entry(rb, struct drm_mm_node, rb);
+		if (parent->__subtree_last < node->__subtree_last)
+			parent->__subtree_last = node->__subtree_last;
+		if (node->start < parent->start)
+			link = &parent->rb.rb_left;
+		else
+			link = &parent->rb.rb_right;
+	}
+
+	rb_link_node(&node->rb, rb, link);
+	rb_insert_augmented(&node->rb,
+			    &mm->interval_tree,
+			    &drm_mm_interval_tree_augment);
+}
+
 static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
 				 struct drm_mm_node *node,
 				 u64 size, unsigned alignment,
@@ -150,9 +217,10 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
 	node->color = color;
 	node->allocated = 1;
 
-	INIT_LIST_HEAD(&node->hole_stack);
 	list_add(&node->node_list, &hole_node->node_list);
 
+	drm_mm_interval_tree_add_node(hole_node, node);
+
 	BUG_ON(node->start + node->size > adj_end);
 
 	node->hole_follows = 0;
@@ -178,41 +246,54 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
  */
 int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
 {
+	u64 end = node->start + node->size;
 	struct drm_mm_node *hole;
-	u64 end;
-	u64 hole_start;
-	u64 hole_end;
+	u64 hole_start, hole_end;
 
-	BUG_ON(node == NULL);
+	if (WARN_ON(node->size == 0))
+		return -EINVAL;
 
 	end = node->start + node->size;
 
 	/* Find the relevant hole to add our node to */
-	drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
-		if (hole_start > node->start || hole_end < end)
-			continue;
+	hole = drm_mm_interval_tree_iter_first(&mm->interval_tree,
+					       node->start, ~(u64)0);
+	if (hole) {
+		if (hole->start < end)
+			return -ENOSPC;
+	} else {
+		hole = list_entry(&mm->head_node.node_list,
+				  typeof(*hole), node_list);
+	}
 
-		node->mm = mm;
-		node->allocated = 1;
+	hole = list_last_entry(&hole->node_list, typeof(*hole), node_list);
+	if (!hole->hole_follows)
+		return -ENOSPC;
 
-		INIT_LIST_HEAD(&node->hole_stack);
-		list_add(&node->node_list, &hole->node_list);
+	hole_start = __drm_mm_hole_node_start(hole);
+	hole_end = __drm_mm_hole_node_end(hole);
+	if (hole_start > node->start || hole_end < end)
+		return -ENOSPC;
 
-		if (node->start == hole_start) {
-			hole->hole_follows = 0;
-			list_del_init(&hole->hole_stack);
-		}
+	node->mm = mm;
+	node->allocated = 1;
 
-		node->hole_follows = 0;
-		if (end != hole_end) {
-			list_add(&node->hole_stack, &mm->hole_stack);
-			node->hole_follows = 1;
-		}
+	list_add(&node->node_list, &hole->node_list);
 
-		return 0;
+	drm_mm_interval_tree_add_node(hole, node);
+
+	if (node->start == hole_start) {
+		hole->hole_follows = 0;
+		list_del(&hole->hole_stack);
 	}
 
-	return -ENOSPC;
+	node->hole_follows = 0;
+	if (end != hole_end) {
+		list_add(&node->hole_stack, &mm->hole_stack);
+		node->hole_follows = 1;
+	}
+
+	return 0;
 }
 EXPORT_SYMBOL(drm_mm_reserve_node);
 
@@ -239,6 +320,9 @@ int drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node,
 {
 	struct drm_mm_node *hole_node;
 
+	if (WARN_ON(size == 0))
+		return -EINVAL;
+
 	hole_node = drm_mm_search_free_generic(mm, size, alignment,
 					       color, sflags);
 	if (!hole_node)
@@ -299,9 +383,10 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
 	node->color = color;
 	node->allocated = 1;
 
-	INIT_LIST_HEAD(&node->hole_stack);
 	list_add(&node->node_list, &hole_node->node_list);
 
+	drm_mm_interval_tree_add_node(hole_node, node);
+
 	BUG_ON(node->start < start);
 	BUG_ON(node->start < adj_start);
 	BUG_ON(node->start + node->size > adj_end);
@@ -340,6 +425,9 @@ int drm_mm_insert_node_in_range_generic(struct drm_mm *mm, struct drm_mm_node *n
 {
 	struct drm_mm_node *hole_node;
 
+	if (WARN_ON(size == 0))
+		return -EINVAL;
+
 	hole_node = drm_mm_search_free_in_range_generic(mm,
 							size, alignment, color,
 							start, end, sflags);
@@ -390,6 +478,7 @@ void drm_mm_remove_node(struct drm_mm_node *node)
 	} else
 		list_move(&prev_node->hole_stack, &mm->hole_stack);
 
+	drm_mm_interval_tree_remove(node, &mm->interval_tree);
 	list_del(&node->node_list);
 	node->allocated = 0;
 }
@@ -516,11 +605,13 @@ void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
 {
 	list_replace(&old->node_list, &new->node_list);
 	list_replace(&old->hole_stack, &new->hole_stack);
+	rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree);
 	new->hole_follows = old->hole_follows;
 	new->mm = old->mm;
 	new->start = old->start;
 	new->size = old->size;
 	new->color = old->color;
+	new->__subtree_last = old->__subtree_last;
 
 	old->allocated = 0;
 	new->allocated = 1;
@@ -748,7 +839,6 @@ void drm_mm_init(struct drm_mm * mm, u64 start, u64 size)
 
 	/* Clever trick to avoid a special case in the free hole tracking. */
 	INIT_LIST_HEAD(&mm->head_node.node_list);
-	INIT_LIST_HEAD(&mm->head_node.hole_stack);
 	mm->head_node.hole_follows = 1;
 	mm->head_node.scanned_block = 0;
 	mm->head_node.scanned_prev_free = 0;
@@ -758,6 +848,8 @@ void drm_mm_init(struct drm_mm * mm, u64 start, u64 size)
 	mm->head_node.size = start - mm->head_node.start;
 	list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack);
 
+	mm->interval_tree = RB_ROOT;
+
 	mm->color_adjust = NULL;
 }
 EXPORT_SYMBOL(drm_mm_init);
diff --git a/drivers/gpu/drm/drm_mode_object.c b/drivers/gpu/drm/drm_mode_object.c
new file mode 100644
index 000000000000..9f17085b1fdd
--- /dev/null
+++ b/drivers/gpu/drm/drm_mode_object.c
@@ -0,0 +1,438 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <linux/export.h>
+#include <drm/drmP.h>
+#include <drm/drm_mode_object.h>
+
+#include "drm_crtc_internal.h"
+
+/*
+ * Internal function to assign a slot in the object idr and optionally
+ * register the object into the idr.
+ */
+int drm_mode_object_get_reg(struct drm_device *dev,
+			    struct drm_mode_object *obj,
+			    uint32_t obj_type,
+			    bool register_obj,
+			    void (*obj_free_cb)(struct kref *kref))
+{
+	int ret;
+
+	mutex_lock(&dev->mode_config.idr_mutex);
+	ret = idr_alloc(&dev->mode_config.crtc_idr, register_obj ? obj : NULL, 1, 0, GFP_KERNEL);
+	if (ret >= 0) {
+		/*
+		 * Set up the object linking under the protection of the idr
+		 * lock so that other users can't see inconsistent state.
+		 */
+		obj->id = ret;
+		obj->type = obj_type;
+		if (obj_free_cb) {
+			obj->free_cb = obj_free_cb;
+			kref_init(&obj->refcount);
+		}
+	}
+	mutex_unlock(&dev->mode_config.idr_mutex);
+
+	return ret < 0 ? ret : 0;
+}
+
+/**
+ * drm_mode_object_get - allocate a new modeset identifier
+ * @dev: DRM device
+ * @obj: object pointer, used to generate unique ID
+ * @obj_type: object type
+ *
+ * Create a unique identifier based on @ptr in @dev's identifier space.  Used
+ * for tracking modes, CRTCs and connectors. Note that despite the _get postfix
+ * modeset identifiers are _not_ reference counted. Hence don't use this for
+ * reference counted modeset objects like framebuffers.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_mode_object_get(struct drm_device *dev,
+			struct drm_mode_object *obj, uint32_t obj_type)
+{
+	return drm_mode_object_get_reg(dev, obj, obj_type, true, NULL);
+}
+
+void drm_mode_object_register(struct drm_device *dev,
+			      struct drm_mode_object *obj)
+{
+	mutex_lock(&dev->mode_config.idr_mutex);
+	idr_replace(&dev->mode_config.crtc_idr, obj, obj->id);
+	mutex_unlock(&dev->mode_config.idr_mutex);
+}
+
+/**
+ * drm_mode_object_unregister - free a modeset identifer
+ * @dev: DRM device
+ * @object: object to free
+ *
+ * Free @id from @dev's unique identifier pool.
+ * This function can be called multiple times, and guards against
+ * multiple removals.
+ * These modeset identifiers are _not_ reference counted. Hence don't use this
+ * for reference counted modeset objects like framebuffers.
+ */
+void drm_mode_object_unregister(struct drm_device *dev,
+				struct drm_mode_object *object)
+{
+	mutex_lock(&dev->mode_config.idr_mutex);
+	if (object->id) {
+		idr_remove(&dev->mode_config.crtc_idr, object->id);
+		object->id = 0;
+	}
+	mutex_unlock(&dev->mode_config.idr_mutex);
+}
+
+struct drm_mode_object *__drm_mode_object_find(struct drm_device *dev,
+					       uint32_t id, uint32_t type)
+{
+	struct drm_mode_object *obj = NULL;
+
+	mutex_lock(&dev->mode_config.idr_mutex);
+	obj = idr_find(&dev->mode_config.crtc_idr, id);
+	if (obj && type != DRM_MODE_OBJECT_ANY && obj->type != type)
+		obj = NULL;
+	if (obj && obj->id != id)
+		obj = NULL;
+
+	if (obj && obj->free_cb) {
+		if (!kref_get_unless_zero(&obj->refcount))
+			obj = NULL;
+	}
+	mutex_unlock(&dev->mode_config.idr_mutex);
+
+	return obj;
+}
+
+/**
+ * drm_mode_object_find - look up a drm object with static lifetime
+ * @dev: drm device
+ * @id: id of the mode object
+ * @type: type of the mode object
+ *
+ * This function is used to look up a modeset object. It will acquire a
+ * reference for reference counted objects. This reference must be dropped again
+ * by callind drm_mode_object_unreference().
+ */
+struct drm_mode_object *drm_mode_object_find(struct drm_device *dev,
+		uint32_t id, uint32_t type)
+{
+	struct drm_mode_object *obj = NULL;
+
+	obj = __drm_mode_object_find(dev, id, type);
+	return obj;
+}
+EXPORT_SYMBOL(drm_mode_object_find);
+
+/**
+ * drm_mode_object_unreference - decr the object refcnt
+ * @obj: mode_object
+ *
+ * This function decrements the object's refcount if it is a refcounted modeset
+ * object. It is a no-op on any other object. This is used to drop references
+ * acquired with drm_mode_object_reference().
+ */
+void drm_mode_object_unreference(struct drm_mode_object *obj)
+{
+	if (obj->free_cb) {
+		DRM_DEBUG("OBJ ID: %d (%d)\n", obj->id, atomic_read(&obj->refcount.refcount));
+		kref_put(&obj->refcount, obj->free_cb);
+	}
+}
+EXPORT_SYMBOL(drm_mode_object_unreference);
+
+/**
+ * drm_mode_object_reference - incr the object refcnt
+ * @obj: mode_object
+ *
+ * This function increments the object's refcount if it is a refcounted modeset
+ * object. It is a no-op on any other object. References should be dropped again
+ * by calling drm_mode_object_unreference().
+ */
+void drm_mode_object_reference(struct drm_mode_object *obj)
+{
+	if (obj->free_cb) {
+		DRM_DEBUG("OBJ ID: %d (%d)\n", obj->id, atomic_read(&obj->refcount.refcount));
+		kref_get(&obj->refcount);
+	}
+}
+EXPORT_SYMBOL(drm_mode_object_reference);
+
+/**
+ * drm_object_attach_property - attach a property to a modeset object
+ * @obj: drm modeset object
+ * @property: property to attach
+ * @init_val: initial value of the property
+ *
+ * This attaches the given property to the modeset object with the given initial
+ * value. Currently this function cannot fail since the properties are stored in
+ * a statically sized array.
+ */
+void drm_object_attach_property(struct drm_mode_object *obj,
+				struct drm_property *property,
+				uint64_t init_val)
+{
+	int count = obj->properties->count;
+
+	if (count == DRM_OBJECT_MAX_PROPERTY) {
+		WARN(1, "Failed to attach object property (type: 0x%x). Please "
+			"increase DRM_OBJECT_MAX_PROPERTY by 1 for each time "
+			"you see this message on the same object type.\n",
+			obj->type);
+		return;
+	}
+
+	obj->properties->properties[count] = property;
+	obj->properties->values[count] = init_val;
+	obj->properties->count++;
+}
+EXPORT_SYMBOL(drm_object_attach_property);
+
+/**
+ * drm_object_property_set_value - set the value of a property
+ * @obj: drm mode object to set property value for
+ * @property: property to set
+ * @val: value the property should be set to
+ *
+ * This function sets a given property on a given object. This function only
+ * changes the software state of the property, it does not call into the
+ * driver's ->set_property callback.
+ *
+ * Note that atomic drivers should not have any need to call this, the core will
+ * ensure consistency of values reported back to userspace through the
+ * appropriate ->atomic_get_property callback. Only legacy drivers should call
+ * this function to update the tracked value (after clamping and other
+ * restrictions have been applied).
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_object_property_set_value(struct drm_mode_object *obj,
+				  struct drm_property *property, uint64_t val)
+{
+	int i;
+
+	for (i = 0; i < obj->properties->count; i++) {
+		if (obj->properties->properties[i] == property) {
+			obj->properties->values[i] = val;
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+EXPORT_SYMBOL(drm_object_property_set_value);
+
+/**
+ * drm_object_property_get_value - retrieve the value of a property
+ * @obj: drm mode object to get property value from
+ * @property: property to retrieve
+ * @val: storage for the property value
+ *
+ * This function retrieves the softare state of the given property for the given
+ * property. Since there is no driver callback to retrieve the current property
+ * value this might be out of sync with the hardware, depending upon the driver
+ * and property.
+ *
+ * Atomic drivers should never call this function directly, the core will read
+ * out property values through the various ->atomic_get_property callbacks.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_object_property_get_value(struct drm_mode_object *obj,
+				  struct drm_property *property, uint64_t *val)
+{
+	int i;
+
+	/* read-only properties bypass atomic mechanism and still store
+	 * their value in obj->properties->values[].. mostly to avoid
+	 * having to deal w/ EDID and similar props in atomic paths:
+	 */
+	if (drm_core_check_feature(property->dev, DRIVER_ATOMIC) &&
+			!(property->flags & DRM_MODE_PROP_IMMUTABLE))
+		return drm_atomic_get_property(obj, property, val);
+
+	for (i = 0; i < obj->properties->count; i++) {
+		if (obj->properties->properties[i] == property) {
+			*val = obj->properties->values[i];
+			return 0;
+		}
+
+	}
+
+	return -EINVAL;
+}
+EXPORT_SYMBOL(drm_object_property_get_value);
+
+/* helper for getconnector and getproperties ioctls */
+int drm_mode_object_get_properties(struct drm_mode_object *obj, bool atomic,
+				   uint32_t __user *prop_ptr,
+				   uint64_t __user *prop_values,
+				   uint32_t *arg_count_props)
+{
+	int i, ret, count;
+
+	for (i = 0, count = 0; i < obj->properties->count; i++) {
+		struct drm_property *prop = obj->properties->properties[i];
+		uint64_t val;
+
+		if ((prop->flags & DRM_MODE_PROP_ATOMIC) && !atomic)
+			continue;
+
+		if (*arg_count_props > count) {
+			ret = drm_object_property_get_value(obj, prop, &val);
+			if (ret)
+				return ret;
+
+			if (put_user(prop->base.id, prop_ptr + count))
+				return -EFAULT;
+
+			if (put_user(val, prop_values + count))
+				return -EFAULT;
+		}
+
+		count++;
+	}
+	*arg_count_props = count;
+
+	return 0;
+}
+
+/**
+ * drm_mode_obj_get_properties_ioctl - get the current value of a object's property
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * This function retrieves the current value for an object's property. Compared
+ * to the connector specific ioctl this one is extended to also work on crtc and
+ * plane objects.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data,
+				      struct drm_file *file_priv)
+{
+	struct drm_mode_obj_get_properties *arg = data;
+	struct drm_mode_object *obj;
+	int ret = 0;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	drm_modeset_lock_all(dev);
+
+	obj = drm_mode_object_find(dev, arg->obj_id, arg->obj_type);
+	if (!obj) {
+		ret = -ENOENT;
+		goto out;
+	}
+	if (!obj->properties) {
+		ret = -EINVAL;
+		goto out_unref;
+	}
+
+	ret = drm_mode_object_get_properties(obj, file_priv->atomic,
+			(uint32_t __user *)(unsigned long)(arg->props_ptr),
+			(uint64_t __user *)(unsigned long)(arg->prop_values_ptr),
+			&arg->count_props);
+
+out_unref:
+	drm_mode_object_unreference(obj);
+out:
+	drm_modeset_unlock_all(dev);
+	return ret;
+}
+
+struct drm_property *drm_mode_obj_find_prop_id(struct drm_mode_object *obj,
+					       uint32_t prop_id)
+{
+	int i;
+
+	for (i = 0; i < obj->properties->count; i++)
+		if (obj->properties->properties[i]->base.id == prop_id)
+			return obj->properties->properties[i];
+
+	return NULL;
+}
+
+int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv)
+{
+	struct drm_mode_obj_set_property *arg = data;
+	struct drm_mode_object *arg_obj;
+	struct drm_property *property;
+	int ret = -EINVAL;
+	struct drm_mode_object *ref;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	drm_modeset_lock_all(dev);
+
+	arg_obj = drm_mode_object_find(dev, arg->obj_id, arg->obj_type);
+	if (!arg_obj) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	if (!arg_obj->properties)
+		goto out_unref;
+
+	property = drm_mode_obj_find_prop_id(arg_obj, arg->prop_id);
+	if (!property)
+		goto out_unref;
+
+	if (!drm_property_change_valid_get(property, arg->value, &ref))
+		goto out_unref;
+
+	switch (arg_obj->type) {
+	case DRM_MODE_OBJECT_CONNECTOR:
+		ret = drm_mode_connector_set_obj_prop(arg_obj, property,
+						      arg->value);
+		break;
+	case DRM_MODE_OBJECT_CRTC:
+		ret = drm_mode_crtc_set_obj_prop(arg_obj, property, arg->value);
+		break;
+	case DRM_MODE_OBJECT_PLANE:
+		ret = drm_mode_plane_set_obj_prop(obj_to_plane(arg_obj),
+						  property, arg->value);
+		break;
+	}
+
+	drm_property_change_valid_put(property, ref);
+
+out_unref:
+	drm_mode_object_unreference(arg_obj);
+out:
+	drm_modeset_unlock_all(dev);
+	return ret;
+}
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index fc5040ae5f25..53f07ac7c174 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -657,11 +657,36 @@ void drm_display_mode_to_videomode(const struct drm_display_mode *dmode,
 }
 EXPORT_SYMBOL_GPL(drm_display_mode_to_videomode);
 
+/**
+ * drm_bus_flags_from_videomode - extract information about pixelclk and
+ * DE polarity from videomode and store it in a separate variable
+ * @vm: videomode structure to use
+ * @bus_flags: information about pixelclk and DE polarity will be stored here
+ *
+ * Sets DRM_BUS_FLAG_DE_(LOW|HIGH) and DRM_BUS_FLAG_PIXDATA_(POS|NEG)EDGE
+ * in @bus_flags according to DISPLAY_FLAGS found in @vm
+ */
+void drm_bus_flags_from_videomode(const struct videomode *vm, u32 *bus_flags)
+{
+	*bus_flags = 0;
+	if (vm->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE)
+		*bus_flags |= DRM_BUS_FLAG_PIXDATA_POSEDGE;
+	if (vm->flags & DISPLAY_FLAGS_PIXDATA_NEGEDGE)
+		*bus_flags |= DRM_BUS_FLAG_PIXDATA_NEGEDGE;
+
+	if (vm->flags & DISPLAY_FLAGS_DE_LOW)
+		*bus_flags |= DRM_BUS_FLAG_DE_LOW;
+	if (vm->flags & DISPLAY_FLAGS_DE_HIGH)
+		*bus_flags |= DRM_BUS_FLAG_DE_HIGH;
+}
+EXPORT_SYMBOL_GPL(drm_bus_flags_from_videomode);
+
 #ifdef CONFIG_OF
 /**
  * of_get_drm_display_mode - get a drm_display_mode from devicetree
  * @np: device_node with the timing specification
  * @dmode: will be set to the return value
+ * @bus_flags: information about pixelclk and DE polarity
  * @index: index into the list of display timings in devicetree
  *
  * This function is expensive and should only be used, if only one mode is to be
@@ -672,7 +697,8 @@ EXPORT_SYMBOL_GPL(drm_display_mode_to_videomode);
  * 0 on success, a negative errno code when no of videomode node was found.
  */
 int of_get_drm_display_mode(struct device_node *np,
-			    struct drm_display_mode *dmode, int index)
+			    struct drm_display_mode *dmode, u32 *bus_flags,
+			    int index)
 {
 	struct videomode vm;
 	int ret;
@@ -682,6 +708,8 @@ int of_get_drm_display_mode(struct device_node *np,
 		return ret;
 
 	drm_display_mode_from_videomode(&vm, dmode);
+	if (bus_flags)
+		drm_bus_flags_from_videomode(&vm, bus_flags);
 
 	pr_debug("%s: got %dx%d display mode from %s\n",
 		of_node_full_name(np), vm.hactive, vm.vactive, np->name);
diff --git a/drivers/gpu/drm/drm_modeset_helper.c b/drivers/gpu/drm/drm_modeset_helper.c
new file mode 100644
index 000000000000..1d45738f8f98
--- /dev/null
+++ b/drivers/gpu/drm/drm_modeset_helper.c
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <drm/drm_modeset_helper.h>
+#include <drm/drm_plane_helper.h>
+
+/**
+ * DOC: aux kms helpers
+ *
+ * This helper library contains various one-off functions which don't really fit
+ * anywhere else in the DRM modeset helper library.
+ */
+
+/**
+ * drm_helper_move_panel_connectors_to_head() - move panels to the front in the
+ * 						connector list
+ * @dev: drm device to operate on
+ *
+ * Some userspace presumes that the first connected connector is the main
+ * display, where it's supposed to display e.g. the login screen. For
+ * laptops, this should be the main panel. Use this function to sort all
+ * (eDP/LVDS) panels to the front of the connector list, instead of
+ * painstakingly trying to initialize them in the right order.
+ */
+void drm_helper_move_panel_connectors_to_head(struct drm_device *dev)
+{
+	struct drm_connector *connector, *tmp;
+	struct list_head panel_list;
+
+	INIT_LIST_HEAD(&panel_list);
+
+	list_for_each_entry_safe(connector, tmp,
+				 &dev->mode_config.connector_list, head) {
+		if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS ||
+		    connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+			list_move_tail(&connector->head, &panel_list);
+	}
+
+	list_splice(&panel_list, &dev->mode_config.connector_list);
+}
+EXPORT_SYMBOL(drm_helper_move_panel_connectors_to_head);
+
+/**
+ * drm_helper_mode_fill_fb_struct - fill out framebuffer metadata
+ * @fb: drm_framebuffer object to fill out
+ * @mode_cmd: metadata from the userspace fb creation request
+ *
+ * This helper can be used in a drivers fb_create callback to pre-fill the fb's
+ * metadata fields.
+ */
+void drm_helper_mode_fill_fb_struct(struct drm_framebuffer *fb,
+				    const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	int i;
+
+	fb->width = mode_cmd->width;
+	fb->height = mode_cmd->height;
+	for (i = 0; i < 4; i++) {
+		fb->pitches[i] = mode_cmd->pitches[i];
+		fb->offsets[i] = mode_cmd->offsets[i];
+		fb->modifier[i] = mode_cmd->modifier[i];
+	}
+	drm_fb_get_bpp_depth(mode_cmd->pixel_format, &fb->depth,
+				    &fb->bits_per_pixel);
+	fb->pixel_format = mode_cmd->pixel_format;
+	fb->flags = mode_cmd->flags;
+}
+EXPORT_SYMBOL(drm_helper_mode_fill_fb_struct);
+
+/*
+ * This is the minimal list of formats that seem to be safe for modeset use
+ * with all current DRM drivers.  Most hardware can actually support more
+ * formats than this and drivers may specify a more accurate list when
+ * creating the primary plane.  However drivers that still call
+ * drm_plane_init() will use this minimal format list as the default.
+ */
+static const uint32_t safe_modeset_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+};
+
+static struct drm_plane *create_primary_plane(struct drm_device *dev)
+{
+	struct drm_plane *primary;
+	int ret;
+
+	primary = kzalloc(sizeof(*primary), GFP_KERNEL);
+	if (primary == NULL) {
+		DRM_DEBUG_KMS("Failed to allocate primary plane\n");
+		return NULL;
+	}
+
+	/*
+	 * Remove the format_default field from drm_plane when dropping
+	 * this helper.
+	 */
+	primary->format_default = true;
+
+	/* possible_crtc's will be filled in later by crtc_init */
+	ret = drm_universal_plane_init(dev, primary, 0,
+				       &drm_primary_helper_funcs,
+				       safe_modeset_formats,
+				       ARRAY_SIZE(safe_modeset_formats),
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		kfree(primary);
+		primary = NULL;
+	}
+
+	return primary;
+}
+
+/**
+ * drm_crtc_init - Legacy CRTC initialization function
+ * @dev: DRM device
+ * @crtc: CRTC object to init
+ * @funcs: callbacks for the new CRTC
+ *
+ * Initialize a CRTC object with a default helper-provided primary plane and no
+ * cursor plane.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
+		  const struct drm_crtc_funcs *funcs)
+{
+	struct drm_plane *primary;
+
+	primary = create_primary_plane(dev);
+	return drm_crtc_init_with_planes(dev, crtc, primary, NULL, funcs,
+					 NULL);
+}
+EXPORT_SYMBOL(drm_crtc_init);
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index b2f8f1062d5f..3ceea9cb9d3e 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -175,7 +175,7 @@ int drm_irq_by_busid(struct drm_device *dev, void *data,
 {
 	struct drm_irq_busid *p = data;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	/* UMS was only ever support on PCI devices. */
@@ -236,8 +236,8 @@ int drm_get_pci_dev(struct pci_dev *pdev, const struct pci_device_id *ent,
 	DRM_DEBUG("\n");
 
 	dev = drm_dev_alloc(driver, &pdev->dev);
-	if (!dev)
-		return -ENOMEM;
+	if (IS_ERR(dev))
+		return PTR_ERR(dev);
 
 	ret = pci_enable_device(pdev);
 	if (ret)
@@ -263,7 +263,7 @@ int drm_get_pci_dev(struct pci_dev *pdev, const struct pci_device_id *ent,
 
 	/* No locking needed since shadow-attach is single-threaded since it may
 	 * only be called from the per-driver module init hook. */
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+	if (drm_core_check_feature(dev, DRIVER_LEGACY))
 		list_add_tail(&dev->legacy_dev_list, &driver->legacy_dev_list);
 
 	return 0;
@@ -299,7 +299,7 @@ int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
 
 	DRM_DEBUG("\n");
 
-	if (driver->driver_features & DRIVER_MODESET)
+	if (!(driver->driver_features & DRIVER_LEGACY))
 		return pci_register_driver(pdriver);
 
 	/* If not using KMS, fall back to stealth mode manual scanning. */
@@ -421,7 +421,7 @@ void drm_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver)
 	struct drm_device *dev, *tmp;
 	DRM_DEBUG("\n");
 
-	if (driver->driver_features & DRIVER_MODESET) {
+	if (!(driver->driver_features & DRIVER_LEGACY)) {
 		pci_unregister_driver(pdriver);
 	} else {
 		list_for_each_entry_safe(dev, tmp, &driver->legacy_dev_list,
diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c
new file mode 100644
index 000000000000..249c0ae52c6d
--- /dev/null
+++ b/drivers/gpu/drm/drm_plane.c
@@ -0,0 +1,906 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_plane.h>
+
+#include "drm_crtc_internal.h"
+
+/**
+ * DOC: overview
+ *
+ * A plane represents an image source that can be blended with or overlayed on
+ * top of a CRTC during the scanout process. Planes take their input data from a
+ * &drm_framebuffer object. The plane itself specifies the cropping and scaling
+ * of that image, and where it is placed on the visible are of a display
+ * pipeline, represented by &drm_crtc. A plane can also have additional
+ * properties that specify how the pixels are positioned and blended, like
+ * rotation or Z-position. All these properties are stored in &drm_plane_state.
+ *
+ * To create a plane, a KMS drivers allocates and zeroes an instances of
+ * struct &drm_plane (possibly as part of a larger structure) and registers it
+ * with a call to drm_universal_plane_init().
+ *
+ * Cursor and overlay planes are optional. All drivers should provide one
+ * primary plane per CRTC to avoid surprising userspace too much. See enum
+ * &drm_plane_type for a more in-depth discussion of these special uapi-relevant
+ * plane types. Special planes are associated with their CRTC by calling
+ * drm_crtc_init_with_planes().
+ *
+ * The type of a plane is exposed in the immutable "type" enumeration property,
+ * which has one of the following values: "Overlay", "Primary", "Cursor".
+ */
+
+static unsigned int drm_num_planes(struct drm_device *dev)
+{
+	unsigned int num = 0;
+	struct drm_plane *tmp;
+
+	drm_for_each_plane(tmp, dev) {
+		num++;
+	}
+
+	return num;
+}
+
+/**
+ * drm_universal_plane_init - Initialize a new universal plane object
+ * @dev: DRM device
+ * @plane: plane object to init
+ * @possible_crtcs: bitmask of possible CRTCs
+ * @funcs: callbacks for the new plane
+ * @formats: array of supported formats (DRM_FORMAT\_\*)
+ * @format_count: number of elements in @formats
+ * @type: type of plane (overlay, primary, cursor)
+ * @name: printf style format string for the plane name, or NULL for default name
+ *
+ * Initializes a plane object of type @type.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
+			     unsigned long possible_crtcs,
+			     const struct drm_plane_funcs *funcs,
+			     const uint32_t *formats, unsigned int format_count,
+			     enum drm_plane_type type,
+			     const char *name, ...)
+{
+	struct drm_mode_config *config = &dev->mode_config;
+	int ret;
+
+	ret = drm_mode_object_get(dev, &plane->base, DRM_MODE_OBJECT_PLANE);
+	if (ret)
+		return ret;
+
+	drm_modeset_lock_init(&plane->mutex);
+
+	plane->base.properties = &plane->properties;
+	plane->dev = dev;
+	plane->funcs = funcs;
+	plane->format_types = kmalloc_array(format_count, sizeof(uint32_t),
+					    GFP_KERNEL);
+	if (!plane->format_types) {
+		DRM_DEBUG_KMS("out of memory when allocating plane\n");
+		drm_mode_object_unregister(dev, &plane->base);
+		return -ENOMEM;
+	}
+
+	if (name) {
+		va_list ap;
+
+		va_start(ap, name);
+		plane->name = kvasprintf(GFP_KERNEL, name, ap);
+		va_end(ap);
+	} else {
+		plane->name = kasprintf(GFP_KERNEL, "plane-%d",
+					drm_num_planes(dev));
+	}
+	if (!plane->name) {
+		kfree(plane->format_types);
+		drm_mode_object_unregister(dev, &plane->base);
+		return -ENOMEM;
+	}
+
+	memcpy(plane->format_types, formats, format_count * sizeof(uint32_t));
+	plane->format_count = format_count;
+	plane->possible_crtcs = possible_crtcs;
+	plane->type = type;
+
+	list_add_tail(&plane->head, &config->plane_list);
+	plane->index = config->num_total_plane++;
+	if (plane->type == DRM_PLANE_TYPE_OVERLAY)
+		config->num_overlay_plane++;
+
+	drm_object_attach_property(&plane->base,
+				   config->plane_type_property,
+				   plane->type);
+
+	if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
+		drm_object_attach_property(&plane->base, config->prop_fb_id, 0);
+		drm_object_attach_property(&plane->base, config->prop_crtc_id, 0);
+		drm_object_attach_property(&plane->base, config->prop_crtc_x, 0);
+		drm_object_attach_property(&plane->base, config->prop_crtc_y, 0);
+		drm_object_attach_property(&plane->base, config->prop_crtc_w, 0);
+		drm_object_attach_property(&plane->base, config->prop_crtc_h, 0);
+		drm_object_attach_property(&plane->base, config->prop_src_x, 0);
+		drm_object_attach_property(&plane->base, config->prop_src_y, 0);
+		drm_object_attach_property(&plane->base, config->prop_src_w, 0);
+		drm_object_attach_property(&plane->base, config->prop_src_h, 0);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_universal_plane_init);
+
+int drm_plane_register_all(struct drm_device *dev)
+{
+	struct drm_plane *plane;
+	int ret = 0;
+
+	drm_for_each_plane(plane, dev) {
+		if (plane->funcs->late_register)
+			ret = plane->funcs->late_register(plane);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+void drm_plane_unregister_all(struct drm_device *dev)
+{
+	struct drm_plane *plane;
+
+	drm_for_each_plane(plane, dev) {
+		if (plane->funcs->early_unregister)
+			plane->funcs->early_unregister(plane);
+	}
+}
+
+/**
+ * drm_plane_init - Initialize a legacy plane
+ * @dev: DRM device
+ * @plane: plane object to init
+ * @possible_crtcs: bitmask of possible CRTCs
+ * @funcs: callbacks for the new plane
+ * @formats: array of supported formats (DRM_FORMAT\_\*)
+ * @format_count: number of elements in @formats
+ * @is_primary: plane type (primary vs overlay)
+ *
+ * Legacy API to initialize a DRM plane.
+ *
+ * New drivers should call drm_universal_plane_init() instead.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
+		   unsigned long possible_crtcs,
+		   const struct drm_plane_funcs *funcs,
+		   const uint32_t *formats, unsigned int format_count,
+		   bool is_primary)
+{
+	enum drm_plane_type type;
+
+	type = is_primary ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
+	return drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
+					formats, format_count, type, NULL);
+}
+EXPORT_SYMBOL(drm_plane_init);
+
+/**
+ * drm_plane_cleanup - Clean up the core plane usage
+ * @plane: plane to cleanup
+ *
+ * This function cleans up @plane and removes it from the DRM mode setting
+ * core. Note that the function does *not* free the plane structure itself,
+ * this is the responsibility of the caller.
+ */
+void drm_plane_cleanup(struct drm_plane *plane)
+{
+	struct drm_device *dev = plane->dev;
+
+	drm_modeset_lock_all(dev);
+	kfree(plane->format_types);
+	drm_mode_object_unregister(dev, &plane->base);
+
+	BUG_ON(list_empty(&plane->head));
+
+	/* Note that the plane_list is considered to be static; should we
+	 * remove the drm_plane at runtime we would have to decrement all
+	 * the indices on the drm_plane after us in the plane_list.
+	 */
+
+	list_del(&plane->head);
+	dev->mode_config.num_total_plane--;
+	if (plane->type == DRM_PLANE_TYPE_OVERLAY)
+		dev->mode_config.num_overlay_plane--;
+	drm_modeset_unlock_all(dev);
+
+	WARN_ON(plane->state && !plane->funcs->atomic_destroy_state);
+	if (plane->state && plane->funcs->atomic_destroy_state)
+		plane->funcs->atomic_destroy_state(plane, plane->state);
+
+	kfree(plane->name);
+
+	memset(plane, 0, sizeof(*plane));
+}
+EXPORT_SYMBOL(drm_plane_cleanup);
+
+/**
+ * drm_plane_from_index - find the registered plane at an index
+ * @dev: DRM device
+ * @idx: index of registered plane to find for
+ *
+ * Given a plane index, return the registered plane from DRM device's
+ * list of planes with matching index.
+ */
+struct drm_plane *
+drm_plane_from_index(struct drm_device *dev, int idx)
+{
+	struct drm_plane *plane;
+
+	drm_for_each_plane(plane, dev)
+		if (idx == plane->index)
+			return plane;
+
+	return NULL;
+}
+EXPORT_SYMBOL(drm_plane_from_index);
+
+/**
+ * drm_plane_force_disable - Forcibly disable a plane
+ * @plane: plane to disable
+ *
+ * Forces the plane to be disabled.
+ *
+ * Used when the plane's current framebuffer is destroyed,
+ * and when restoring fbdev mode.
+ */
+void drm_plane_force_disable(struct drm_plane *plane)
+{
+	int ret;
+
+	if (!plane->fb)
+		return;
+
+	plane->old_fb = plane->fb;
+	ret = plane->funcs->disable_plane(plane);
+	if (ret) {
+		DRM_ERROR("failed to disable plane with busy fb\n");
+		plane->old_fb = NULL;
+		return;
+	}
+	/* disconnect the plane from the fb and crtc: */
+	drm_framebuffer_unreference(plane->old_fb);
+	plane->old_fb = NULL;
+	plane->fb = NULL;
+	plane->crtc = NULL;
+}
+EXPORT_SYMBOL(drm_plane_force_disable);
+
+/**
+ * drm_mode_plane_set_obj_prop - set the value of a property
+ * @plane: drm plane object to set property value for
+ * @property: property to set
+ * @value: value the property should be set to
+ *
+ * This functions sets a given property on a given plane object. This function
+ * calls the driver's ->set_property callback and changes the software state of
+ * the property if the callback succeeds.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_mode_plane_set_obj_prop(struct drm_plane *plane,
+				struct drm_property *property,
+				uint64_t value)
+{
+	int ret = -EINVAL;
+	struct drm_mode_object *obj = &plane->base;
+
+	if (plane->funcs->set_property)
+		ret = plane->funcs->set_property(plane, property, value);
+	if (!ret)
+		drm_object_property_set_value(obj, property, value);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_mode_plane_set_obj_prop);
+
+int drm_mode_getplane_res(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv)
+{
+	struct drm_mode_get_plane_res *plane_resp = data;
+	struct drm_mode_config *config;
+	struct drm_plane *plane;
+	uint32_t __user *plane_ptr;
+	int copied = 0;
+	unsigned num_planes;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	config = &dev->mode_config;
+
+	if (file_priv->universal_planes)
+		num_planes = config->num_total_plane;
+	else
+		num_planes = config->num_overlay_plane;
+
+	/*
+	 * This ioctl is called twice, once to determine how much space is
+	 * needed, and the 2nd time to fill it.
+	 */
+	if (num_planes &&
+	    (plane_resp->count_planes >= num_planes)) {
+		plane_ptr = (uint32_t __user *)(unsigned long)plane_resp->plane_id_ptr;
+
+		/* Plane lists are invariant, no locking needed. */
+		drm_for_each_plane(plane, dev) {
+			/*
+			 * Unless userspace set the 'universal planes'
+			 * capability bit, only advertise overlays.
+			 */
+			if (plane->type != DRM_PLANE_TYPE_OVERLAY &&
+			    !file_priv->universal_planes)
+				continue;
+
+			if (put_user(plane->base.id, plane_ptr + copied))
+				return -EFAULT;
+			copied++;
+		}
+	}
+	plane_resp->count_planes = num_planes;
+
+	return 0;
+}
+
+int drm_mode_getplane(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
+{
+	struct drm_mode_get_plane *plane_resp = data;
+	struct drm_plane *plane;
+	uint32_t __user *format_ptr;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	plane = drm_plane_find(dev, plane_resp->plane_id);
+	if (!plane)
+		return -ENOENT;
+
+	drm_modeset_lock(&plane->mutex, NULL);
+	if (plane->crtc)
+		plane_resp->crtc_id = plane->crtc->base.id;
+	else
+		plane_resp->crtc_id = 0;
+
+	if (plane->fb)
+		plane_resp->fb_id = plane->fb->base.id;
+	else
+		plane_resp->fb_id = 0;
+	drm_modeset_unlock(&plane->mutex);
+
+	plane_resp->plane_id = plane->base.id;
+	plane_resp->possible_crtcs = plane->possible_crtcs;
+	plane_resp->gamma_size = 0;
+
+	/*
+	 * This ioctl is called twice, once to determine how much space is
+	 * needed, and the 2nd time to fill it.
+	 */
+	if (plane->format_count &&
+	    (plane_resp->count_format_types >= plane->format_count)) {
+		format_ptr = (uint32_t __user *)(unsigned long)plane_resp->format_type_ptr;
+		if (copy_to_user(format_ptr,
+				 plane->format_types,
+				 sizeof(uint32_t) * plane->format_count)) {
+			return -EFAULT;
+		}
+	}
+	plane_resp->count_format_types = plane->format_count;
+
+	return 0;
+}
+
+int drm_plane_check_pixel_format(const struct drm_plane *plane, u32 format)
+{
+	unsigned int i;
+
+	for (i = 0; i < plane->format_count; i++) {
+		if (format == plane->format_types[i])
+			return 0;
+	}
+
+	return -EINVAL;
+}
+
+/*
+ * setplane_internal - setplane handler for internal callers
+ *
+ * Note that we assume an extra reference has already been taken on fb.  If the
+ * update fails, this reference will be dropped before return; if it succeeds,
+ * the previous framebuffer (if any) will be unreferenced instead.
+ *
+ * src_{x,y,w,h} are provided in 16.16 fixed point format
+ */
+static int __setplane_internal(struct drm_plane *plane,
+			       struct drm_crtc *crtc,
+			       struct drm_framebuffer *fb,
+			       int32_t crtc_x, int32_t crtc_y,
+			       uint32_t crtc_w, uint32_t crtc_h,
+			       /* src_{x,y,w,h} values are 16.16 fixed point */
+			       uint32_t src_x, uint32_t src_y,
+			       uint32_t src_w, uint32_t src_h)
+{
+	int ret = 0;
+
+	/* No fb means shut it down */
+	if (!fb) {
+		plane->old_fb = plane->fb;
+		ret = plane->funcs->disable_plane(plane);
+		if (!ret) {
+			plane->crtc = NULL;
+			plane->fb = NULL;
+		} else {
+			plane->old_fb = NULL;
+		}
+		goto out;
+	}
+
+	/* Check whether this plane is usable on this CRTC */
+	if (!(plane->possible_crtcs & drm_crtc_mask(crtc))) {
+		DRM_DEBUG_KMS("Invalid crtc for plane\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* Check whether this plane supports the fb pixel format. */
+	ret = drm_plane_check_pixel_format(plane, fb->pixel_format);
+	if (ret) {
+		char *format_name = drm_get_format_name(fb->pixel_format);
+		DRM_DEBUG_KMS("Invalid pixel format %s\n", format_name);
+		kfree(format_name);
+		goto out;
+	}
+
+	/* Give drivers some help against integer overflows */
+	if (crtc_w > INT_MAX ||
+	    crtc_x > INT_MAX - (int32_t) crtc_w ||
+	    crtc_h > INT_MAX ||
+	    crtc_y > INT_MAX - (int32_t) crtc_h) {
+		DRM_DEBUG_KMS("Invalid CRTC coordinates %ux%u+%d+%d\n",
+			      crtc_w, crtc_h, crtc_x, crtc_y);
+		ret = -ERANGE;
+		goto out;
+	}
+
+	ret = drm_framebuffer_check_src_coords(src_x, src_y, src_w, src_h, fb);
+	if (ret)
+		goto out;
+
+	plane->old_fb = plane->fb;
+	ret = plane->funcs->update_plane(plane, crtc, fb,
+					 crtc_x, crtc_y, crtc_w, crtc_h,
+					 src_x, src_y, src_w, src_h);
+	if (!ret) {
+		plane->crtc = crtc;
+		plane->fb = fb;
+		fb = NULL;
+	} else {
+		plane->old_fb = NULL;
+	}
+
+out:
+	if (fb)
+		drm_framebuffer_unreference(fb);
+	if (plane->old_fb)
+		drm_framebuffer_unreference(plane->old_fb);
+	plane->old_fb = NULL;
+
+	return ret;
+}
+
+static int setplane_internal(struct drm_plane *plane,
+			     struct drm_crtc *crtc,
+			     struct drm_framebuffer *fb,
+			     int32_t crtc_x, int32_t crtc_y,
+			     uint32_t crtc_w, uint32_t crtc_h,
+			     /* src_{x,y,w,h} values are 16.16 fixed point */
+			     uint32_t src_x, uint32_t src_y,
+			     uint32_t src_w, uint32_t src_h)
+{
+	int ret;
+
+	drm_modeset_lock_all(plane->dev);
+	ret = __setplane_internal(plane, crtc, fb,
+				  crtc_x, crtc_y, crtc_w, crtc_h,
+				  src_x, src_y, src_w, src_h);
+	drm_modeset_unlock_all(plane->dev);
+
+	return ret;
+}
+
+int drm_mode_setplane(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
+{
+	struct drm_mode_set_plane *plane_req = data;
+	struct drm_plane *plane;
+	struct drm_crtc *crtc = NULL;
+	struct drm_framebuffer *fb = NULL;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	/*
+	 * First, find the plane, crtc, and fb objects.  If not available,
+	 * we don't bother to call the driver.
+	 */
+	plane = drm_plane_find(dev, plane_req->plane_id);
+	if (!plane) {
+		DRM_DEBUG_KMS("Unknown plane ID %d\n",
+			      plane_req->plane_id);
+		return -ENOENT;
+	}
+
+	if (plane_req->fb_id) {
+		fb = drm_framebuffer_lookup(dev, plane_req->fb_id);
+		if (!fb) {
+			DRM_DEBUG_KMS("Unknown framebuffer ID %d\n",
+				      plane_req->fb_id);
+			return -ENOENT;
+		}
+
+		crtc = drm_crtc_find(dev, plane_req->crtc_id);
+		if (!crtc) {
+			DRM_DEBUG_KMS("Unknown crtc ID %d\n",
+				      plane_req->crtc_id);
+			return -ENOENT;
+		}
+	}
+
+	/*
+	 * setplane_internal will take care of deref'ing either the old or new
+	 * framebuffer depending on success.
+	 */
+	return setplane_internal(plane, crtc, fb,
+				 plane_req->crtc_x, plane_req->crtc_y,
+				 plane_req->crtc_w, plane_req->crtc_h,
+				 plane_req->src_x, plane_req->src_y,
+				 plane_req->src_w, plane_req->src_h);
+}
+
+static int drm_mode_cursor_universal(struct drm_crtc *crtc,
+				     struct drm_mode_cursor2 *req,
+				     struct drm_file *file_priv)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_framebuffer *fb = NULL;
+	struct drm_mode_fb_cmd2 fbreq = {
+		.width = req->width,
+		.height = req->height,
+		.pixel_format = DRM_FORMAT_ARGB8888,
+		.pitches = { req->width * 4 },
+		.handles = { req->handle },
+	};
+	int32_t crtc_x, crtc_y;
+	uint32_t crtc_w = 0, crtc_h = 0;
+	uint32_t src_w = 0, src_h = 0;
+	int ret = 0;
+
+	BUG_ON(!crtc->cursor);
+	WARN_ON(crtc->cursor->crtc != crtc && crtc->cursor->crtc != NULL);
+
+	/*
+	 * Obtain fb we'll be using (either new or existing) and take an extra
+	 * reference to it if fb != null.  setplane will take care of dropping
+	 * the reference if the plane update fails.
+	 */
+	if (req->flags & DRM_MODE_CURSOR_BO) {
+		if (req->handle) {
+			fb = drm_internal_framebuffer_create(dev, &fbreq, file_priv);
+			if (IS_ERR(fb)) {
+				DRM_DEBUG_KMS("failed to wrap cursor buffer in drm framebuffer\n");
+				return PTR_ERR(fb);
+			}
+			fb->hot_x = req->hot_x;
+			fb->hot_y = req->hot_y;
+		} else {
+			fb = NULL;
+		}
+	} else {
+		fb = crtc->cursor->fb;
+		if (fb)
+			drm_framebuffer_reference(fb);
+	}
+
+	if (req->flags & DRM_MODE_CURSOR_MOVE) {
+		crtc_x = req->x;
+		crtc_y = req->y;
+	} else {
+		crtc_x = crtc->cursor_x;
+		crtc_y = crtc->cursor_y;
+	}
+
+	if (fb) {
+		crtc_w = fb->width;
+		crtc_h = fb->height;
+		src_w = fb->width << 16;
+		src_h = fb->height << 16;
+	}
+
+	/*
+	 * setplane_internal will take care of deref'ing either the old or new
+	 * framebuffer depending on success.
+	 */
+	ret = __setplane_internal(crtc->cursor, crtc, fb,
+				crtc_x, crtc_y, crtc_w, crtc_h,
+				0, 0, src_w, src_h);
+
+	/* Update successful; save new cursor position, if necessary */
+	if (ret == 0 && req->flags & DRM_MODE_CURSOR_MOVE) {
+		crtc->cursor_x = req->x;
+		crtc->cursor_y = req->y;
+	}
+
+	return ret;
+}
+
+static int drm_mode_cursor_common(struct drm_device *dev,
+				  struct drm_mode_cursor2 *req,
+				  struct drm_file *file_priv)
+{
+	struct drm_crtc *crtc;
+	int ret = 0;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	if (!req->flags || (~DRM_MODE_CURSOR_FLAGS & req->flags))
+		return -EINVAL;
+
+	crtc = drm_crtc_find(dev, req->crtc_id);
+	if (!crtc) {
+		DRM_DEBUG_KMS("Unknown CRTC ID %d\n", req->crtc_id);
+		return -ENOENT;
+	}
+
+	/*
+	 * If this crtc has a universal cursor plane, call that plane's update
+	 * handler rather than using legacy cursor handlers.
+	 */
+	drm_modeset_lock_crtc(crtc, crtc->cursor);
+	if (crtc->cursor) {
+		ret = drm_mode_cursor_universal(crtc, req, file_priv);
+		goto out;
+	}
+
+	if (req->flags & DRM_MODE_CURSOR_BO) {
+		if (!crtc->funcs->cursor_set && !crtc->funcs->cursor_set2) {
+			ret = -ENXIO;
+			goto out;
+		}
+		/* Turns off the cursor if handle is 0 */
+		if (crtc->funcs->cursor_set2)
+			ret = crtc->funcs->cursor_set2(crtc, file_priv, req->handle,
+						      req->width, req->height, req->hot_x, req->hot_y);
+		else
+			ret = crtc->funcs->cursor_set(crtc, file_priv, req->handle,
+						      req->width, req->height);
+	}
+
+	if (req->flags & DRM_MODE_CURSOR_MOVE) {
+		if (crtc->funcs->cursor_move) {
+			ret = crtc->funcs->cursor_move(crtc, req->x, req->y);
+		} else {
+			ret = -EFAULT;
+			goto out;
+		}
+	}
+out:
+	drm_modeset_unlock_crtc(crtc);
+
+	return ret;
+
+}
+
+
+int drm_mode_cursor_ioctl(struct drm_device *dev,
+			  void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_cursor *req = data;
+	struct drm_mode_cursor2 new_req;
+
+	memcpy(&new_req, req, sizeof(struct drm_mode_cursor));
+	new_req.hot_x = new_req.hot_y = 0;
+
+	return drm_mode_cursor_common(dev, &new_req, file_priv);
+}
+
+/*
+ * Set the cursor configuration based on user request. This implements the 2nd
+ * version of the cursor ioctl, which allows userspace to additionally specify
+ * the hotspot of the pointer.
+ */
+int drm_mode_cursor2_ioctl(struct drm_device *dev,
+			   void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_cursor2 *req = data;
+
+	return drm_mode_cursor_common(dev, req, file_priv);
+}
+
+int drm_mode_page_flip_ioctl(struct drm_device *dev,
+			     void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_crtc_page_flip_target *page_flip = data;
+	struct drm_crtc *crtc;
+	struct drm_framebuffer *fb = NULL;
+	struct drm_pending_vblank_event *e = NULL;
+	u32 target_vblank = page_flip->sequence;
+	int ret = -EINVAL;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	if (page_flip->flags & ~DRM_MODE_PAGE_FLIP_FLAGS)
+		return -EINVAL;
+
+	if (page_flip->sequence != 0 && !(page_flip->flags & DRM_MODE_PAGE_FLIP_TARGET))
+		return -EINVAL;
+
+	/* Only one of the DRM_MODE_PAGE_FLIP_TARGET_ABSOLUTE/RELATIVE flags
+	 * can be specified
+	 */
+	if ((page_flip->flags & DRM_MODE_PAGE_FLIP_TARGET) == DRM_MODE_PAGE_FLIP_TARGET)
+		return -EINVAL;
+
+	if ((page_flip->flags & DRM_MODE_PAGE_FLIP_ASYNC) && !dev->mode_config.async_page_flip)
+		return -EINVAL;
+
+	crtc = drm_crtc_find(dev, page_flip->crtc_id);
+	if (!crtc)
+		return -ENOENT;
+
+	if (crtc->funcs->page_flip_target) {
+		u32 current_vblank;
+		int r;
+
+		r = drm_crtc_vblank_get(crtc);
+		if (r)
+			return r;
+
+		current_vblank = drm_crtc_vblank_count(crtc);
+
+		switch (page_flip->flags & DRM_MODE_PAGE_FLIP_TARGET) {
+		case DRM_MODE_PAGE_FLIP_TARGET_ABSOLUTE:
+			if ((int)(target_vblank - current_vblank) > 1) {
+				DRM_DEBUG("Invalid absolute flip target %u, "
+					  "must be <= %u\n", target_vblank,
+					  current_vblank + 1);
+				drm_crtc_vblank_put(crtc);
+				return -EINVAL;
+			}
+			break;
+		case DRM_MODE_PAGE_FLIP_TARGET_RELATIVE:
+			if (target_vblank != 0 && target_vblank != 1) {
+				DRM_DEBUG("Invalid relative flip target %u, "
+					  "must be 0 or 1\n", target_vblank);
+				drm_crtc_vblank_put(crtc);
+				return -EINVAL;
+			}
+			target_vblank += current_vblank;
+			break;
+		default:
+			target_vblank = current_vblank +
+				!(page_flip->flags & DRM_MODE_PAGE_FLIP_ASYNC);
+			break;
+		}
+	} else if (crtc->funcs->page_flip == NULL ||
+		   (page_flip->flags & DRM_MODE_PAGE_FLIP_TARGET)) {
+		return -EINVAL;
+	}
+
+	drm_modeset_lock_crtc(crtc, crtc->primary);
+	if (crtc->primary->fb == NULL) {
+		/* The framebuffer is currently unbound, presumably
+		 * due to a hotplug event, that userspace has not
+		 * yet discovered.
+		 */
+		ret = -EBUSY;
+		goto out;
+	}
+
+	fb = drm_framebuffer_lookup(dev, page_flip->fb_id);
+	if (!fb) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	if (crtc->state) {
+		const struct drm_plane_state *state = crtc->primary->state;
+
+		ret = drm_framebuffer_check_src_coords(state->src_x,
+						       state->src_y,
+						       state->src_w,
+						       state->src_h,
+						       fb);
+	} else {
+		ret = drm_crtc_check_viewport(crtc, crtc->x, crtc->y, &crtc->mode, fb);
+	}
+	if (ret)
+		goto out;
+
+	if (crtc->primary->fb->pixel_format != fb->pixel_format) {
+		DRM_DEBUG_KMS("Page flip is not allowed to change frame buffer format.\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (page_flip->flags & DRM_MODE_PAGE_FLIP_EVENT) {
+		e = kzalloc(sizeof *e, GFP_KERNEL);
+		if (!e) {
+			ret = -ENOMEM;
+			goto out;
+		}
+		e->event.base.type = DRM_EVENT_FLIP_COMPLETE;
+		e->event.base.length = sizeof(e->event);
+		e->event.user_data = page_flip->user_data;
+		ret = drm_event_reserve_init(dev, file_priv, &e->base, &e->event.base);
+		if (ret) {
+			kfree(e);
+			goto out;
+		}
+	}
+
+	crtc->primary->old_fb = crtc->primary->fb;
+	if (crtc->funcs->page_flip_target)
+		ret = crtc->funcs->page_flip_target(crtc, fb, e,
+						    page_flip->flags,
+						    target_vblank);
+	else
+		ret = crtc->funcs->page_flip(crtc, fb, e, page_flip->flags);
+	if (ret) {
+		if (page_flip->flags & DRM_MODE_PAGE_FLIP_EVENT)
+			drm_event_cancel_free(dev, &e->base);
+		/* Keep the old fb, don't unref it. */
+		crtc->primary->old_fb = NULL;
+	} else {
+		crtc->primary->fb = fb;
+		/* Unref only the old framebuffer. */
+		fb = NULL;
+	}
+
+out:
+	if (ret && crtc->funcs->page_flip_target)
+		drm_crtc_vblank_put(crtc);
+	if (fb)
+		drm_framebuffer_unreference(fb);
+	if (crtc->primary->old_fb)
+		drm_framebuffer_unreference(crtc->primary->old_fb);
+	crtc->primary->old_fb = NULL;
+	drm_modeset_unlock_crtc(crtc);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/drm_plane_helper.c b/drivers/gpu/drm/drm_plane_helper.c
index 16c4a7bd7465..7899fc1dcdb0 100644
--- a/drivers/gpu/drm/drm_plane_helper.c
+++ b/drivers/gpu/drm/drm_plane_helper.c
@@ -64,18 +64,6 @@
  */
 
 /*
- * This is the minimal list of formats that seem to be safe for modeset use
- * with all current DRM drivers.  Most hardware can actually support more
- * formats than this and drivers may specify a more accurate list when
- * creating the primary plane.  However drivers that still call
- * drm_plane_init() will use this minimal format list as the default.
- */
-static const uint32_t safe_modeset_formats[] = {
-	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_ARGB8888,
-};
-
-/*
  * Returns the connectors currently associated with a CRTC.  This function
  * should be called twice:  once with a NULL connector list to retrieve
  * the list size, and once with the properly allocated list to be filled in.
@@ -108,14 +96,9 @@ static int get_connectors_for_crtc(struct drm_crtc *crtc,
 }
 
 /**
- * drm_plane_helper_check_update() - Check plane update for validity
- * @plane: plane object to update
- * @crtc: owning CRTC of owning plane
- * @fb: framebuffer to flip onto plane
- * @src: source coordinates in 16.16 fixed point
- * @dest: integer destination coordinates
+ * drm_plane_helper_check_state() - Check plane state for validity
+ * @state: plane state to check
  * @clip: integer clipping coordinates
- * @rotation: plane rotation
  * @min_scale: minimum @src:@dest scaling factor in 16.16 fixed point
  * @max_scale: maximum @src:@dest scaling factor in 16.16 fixed point
  * @can_position: is it legal to position the plane such that it
@@ -123,10 +106,9 @@ static int get_connectors_for_crtc(struct drm_crtc *crtc,
  *                only be false for primary planes.
  * @can_update_disabled: can the plane be updated while the crtc
  *                       is disabled?
- * @visible: output parameter indicating whether plane is still visible after
- *           clipping
  *
- * Checks that a desired plane update is valid.  Drivers that provide
+ * Checks that a desired plane update is valid, and updates various
+ * bits of derived state (clipped coordinates etc.). Drivers that provide
  * their own plane handling rather than helper-provided implementations may
  * still wish to call this function to avoid duplication of error checking
  * code.
@@ -134,29 +116,38 @@ static int get_connectors_for_crtc(struct drm_crtc *crtc,
  * RETURNS:
  * Zero if update appears valid, error code on failure
  */
-int drm_plane_helper_check_update(struct drm_plane *plane,
-				  struct drm_crtc *crtc,
-				  struct drm_framebuffer *fb,
-				  struct drm_rect *src,
-				  struct drm_rect *dest,
-				  const struct drm_rect *clip,
-				  unsigned int rotation,
-				  int min_scale,
-				  int max_scale,
-				  bool can_position,
-				  bool can_update_disabled,
-				  bool *visible)
+int drm_plane_helper_check_state(struct drm_plane_state *state,
+				 const struct drm_rect *clip,
+				 int min_scale,
+				 int max_scale,
+				 bool can_position,
+				 bool can_update_disabled)
 {
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_framebuffer *fb = state->fb;
+	struct drm_rect *src = &state->src;
+	struct drm_rect *dst = &state->dst;
+	unsigned int rotation = state->rotation;
 	int hscale, vscale;
 
+	src->x1 = state->src_x;
+	src->y1 = state->src_y;
+	src->x2 = state->src_x + state->src_w;
+	src->y2 = state->src_y + state->src_h;
+
+	dst->x1 = state->crtc_x;
+	dst->y1 = state->crtc_y;
+	dst->x2 = state->crtc_x + state->crtc_w;
+	dst->y2 = state->crtc_y + state->crtc_h;
+
 	if (!fb) {
-		*visible = false;
+		state->visible = false;
 		return 0;
 	}
 
 	/* crtc should only be NULL when disabling (i.e., !fb) */
 	if (WARN_ON(!crtc)) {
-		*visible = false;
+		state->visible = false;
 		return 0;
 	}
 
@@ -168,20 +159,20 @@ int drm_plane_helper_check_update(struct drm_plane *plane,
 	drm_rect_rotate(src, fb->width << 16, fb->height << 16, rotation);
 
 	/* Check scaling */
-	hscale = drm_rect_calc_hscale(src, dest, min_scale, max_scale);
-	vscale = drm_rect_calc_vscale(src, dest, min_scale, max_scale);
+	hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale);
+	vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale);
 	if (hscale < 0 || vscale < 0) {
 		DRM_DEBUG_KMS("Invalid scaling of plane\n");
-		drm_rect_debug_print("src: ", src, true);
-		drm_rect_debug_print("dst: ", dest, false);
+		drm_rect_debug_print("src: ", &state->src, true);
+		drm_rect_debug_print("dst: ", &state->dst, false);
 		return -ERANGE;
 	}
 
-	*visible = drm_rect_clip_scaled(src, dest, clip, hscale, vscale);
+	state->visible = drm_rect_clip_scaled(src, dst, clip, hscale, vscale);
 
 	drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16, rotation);
 
-	if (!*visible)
+	if (!state->visible)
 		/*
 		 * Plane isn't visible; some drivers can handle this
 		 * so we just return success here.  Drivers that can't
@@ -191,15 +182,87 @@ int drm_plane_helper_check_update(struct drm_plane *plane,
 		 */
 		return 0;
 
-	if (!can_position && !drm_rect_equals(dest, clip)) {
+	if (!can_position && !drm_rect_equals(dst, clip)) {
 		DRM_DEBUG_KMS("Plane must cover entire CRTC\n");
-		drm_rect_debug_print("dst: ", dest, false);
+		drm_rect_debug_print("dst: ", dst, false);
 		drm_rect_debug_print("clip: ", clip, false);
 		return -EINVAL;
 	}
 
 	return 0;
 }
+EXPORT_SYMBOL(drm_plane_helper_check_state);
+
+/**
+ * drm_plane_helper_check_update() - Check plane update for validity
+ * @plane: plane object to update
+ * @crtc: owning CRTC of owning plane
+ * @fb: framebuffer to flip onto plane
+ * @src: source coordinates in 16.16 fixed point
+ * @dst: integer destination coordinates
+ * @clip: integer clipping coordinates
+ * @rotation: plane rotation
+ * @min_scale: minimum @src:@dest scaling factor in 16.16 fixed point
+ * @max_scale: maximum @src:@dest scaling factor in 16.16 fixed point
+ * @can_position: is it legal to position the plane such that it
+ *                doesn't cover the entire crtc?  This will generally
+ *                only be false for primary planes.
+ * @can_update_disabled: can the plane be updated while the crtc
+ *                       is disabled?
+ * @visible: output parameter indicating whether plane is still visible after
+ *           clipping
+ *
+ * Checks that a desired plane update is valid.  Drivers that provide
+ * their own plane handling rather than helper-provided implementations may
+ * still wish to call this function to avoid duplication of error checking
+ * code.
+ *
+ * RETURNS:
+ * Zero if update appears valid, error code on failure
+ */
+int drm_plane_helper_check_update(struct drm_plane *plane,
+				  struct drm_crtc *crtc,
+				  struct drm_framebuffer *fb,
+				  struct drm_rect *src,
+				  struct drm_rect *dst,
+				  const struct drm_rect *clip,
+				  unsigned int rotation,
+				  int min_scale,
+				  int max_scale,
+				  bool can_position,
+				  bool can_update_disabled,
+				  bool *visible)
+{
+	struct drm_plane_state state = {
+		.plane = plane,
+		.crtc = crtc,
+		.fb = fb,
+		.src_x = src->x1,
+		.src_y = src->y1,
+		.src_w = drm_rect_width(src),
+		.src_h = drm_rect_height(src),
+		.crtc_x = dst->x1,
+		.crtc_y = dst->y1,
+		.crtc_w = drm_rect_width(dst),
+		.crtc_h = drm_rect_height(dst),
+		.rotation = rotation,
+		.visible = *visible,
+	};
+	int ret;
+
+	ret = drm_plane_helper_check_state(&state, clip,
+					   min_scale, max_scale,
+					   can_position,
+					   can_update_disabled);
+	if (ret)
+		return ret;
+
+	*src = state.src;
+	*dst = state.dst;
+	*visible = state.visible;
+
+	return 0;
+}
 EXPORT_SYMBOL(drm_plane_helper_check_update);
 
 /**
@@ -274,7 +337,7 @@ int drm_primary_helper_update(struct drm_plane *plane, struct drm_crtc *crtc,
 
 	ret = drm_plane_helper_check_update(plane, crtc, fb,
 					    &src, &dest, &clip,
-					    BIT(DRM_ROTATE_0),
+					    DRM_ROTATE_0,
 					    DRM_PLANE_HELPER_NO_SCALING,
 					    DRM_PLANE_HELPER_NO_SCALING,
 					    false, false, &visible);
@@ -363,60 +426,6 @@ const struct drm_plane_funcs drm_primary_helper_funcs = {
 };
 EXPORT_SYMBOL(drm_primary_helper_funcs);
 
-static struct drm_plane *create_primary_plane(struct drm_device *dev)
-{
-	struct drm_plane *primary;
-	int ret;
-
-	primary = kzalloc(sizeof(*primary), GFP_KERNEL);
-	if (primary == NULL) {
-		DRM_DEBUG_KMS("Failed to allocate primary plane\n");
-		return NULL;
-	}
-
-	/*
-	 * Remove the format_default field from drm_plane when dropping
-	 * this helper.
-	 */
-	primary->format_default = true;
-
-	/* possible_crtc's will be filled in later by crtc_init */
-	ret = drm_universal_plane_init(dev, primary, 0,
-				       &drm_primary_helper_funcs,
-				       safe_modeset_formats,
-				       ARRAY_SIZE(safe_modeset_formats),
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
-	if (ret) {
-		kfree(primary);
-		primary = NULL;
-	}
-
-	return primary;
-}
-
-/**
- * drm_crtc_init - Legacy CRTC initialization function
- * @dev: DRM device
- * @crtc: CRTC object to init
- * @funcs: callbacks for the new CRTC
- *
- * Initialize a CRTC object with a default helper-provided primary plane and no
- * cursor plane.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
-		  const struct drm_crtc_funcs *funcs)
-{
-	struct drm_plane *primary;
-
-	primary = create_primary_plane(dev);
-	return drm_crtc_init_with_planes(dev, crtc, primary, NULL, funcs,
-					 NULL);
-}
-EXPORT_SYMBOL(drm_crtc_init);
-
 int drm_plane_helper_commit(struct drm_plane *plane,
 			    struct drm_plane_state *plane_state,
 			    struct drm_framebuffer *old_fb)
diff --git a/drivers/gpu/drm/drm_platform.c b/drivers/gpu/drm/drm_platform.c
index 2c819ef90090..026269851ce9 100644
--- a/drivers/gpu/drm/drm_platform.c
+++ b/drivers/gpu/drm/drm_platform.c
@@ -48,8 +48,8 @@ static int drm_get_platform_dev(struct platform_device *platdev,
 	DRM_DEBUG("\n");
 
 	dev = drm_dev_alloc(driver, &platdev->dev);
-	if (!dev)
-		return -ENOMEM;
+	if (IS_ERR(dev))
+		return PTR_ERR(dev);
 
 	dev->platformdev = platdev;
 
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index 780589b420a4..b22a94dd7b53 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -28,6 +28,7 @@
 
 #include <linux/export.h>
 #include <linux/dma-buf.h>
+#include <linux/rbtree.h>
 #include <drm/drmP.h>
 #include <drm/drm_gem.h>
 
@@ -61,9 +62,11 @@
  */
 
 struct drm_prime_member {
-	struct list_head entry;
 	struct dma_buf *dma_buf;
 	uint32_t handle;
+
+	struct rb_node dmabuf_rb;
+	struct rb_node handle_rb;
 };
 
 struct drm_prime_attachment {
@@ -75,6 +78,7 @@ static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv,
 				    struct dma_buf *dma_buf, uint32_t handle)
 {
 	struct drm_prime_member *member;
+	struct rb_node **p, *rb;
 
 	member = kmalloc(sizeof(*member), GFP_KERNEL);
 	if (!member)
@@ -83,18 +87,56 @@ static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv,
 	get_dma_buf(dma_buf);
 	member->dma_buf = dma_buf;
 	member->handle = handle;
-	list_add(&member->entry, &prime_fpriv->head);
+
+	rb = NULL;
+	p = &prime_fpriv->dmabufs.rb_node;
+	while (*p) {
+		struct drm_prime_member *pos;
+
+		rb = *p;
+		pos = rb_entry(rb, struct drm_prime_member, dmabuf_rb);
+		if (dma_buf > pos->dma_buf)
+			p = &rb->rb_right;
+		else
+			p = &rb->rb_left;
+	}
+	rb_link_node(&member->dmabuf_rb, rb, p);
+	rb_insert_color(&member->dmabuf_rb, &prime_fpriv->dmabufs);
+
+	rb = NULL;
+	p = &prime_fpriv->handles.rb_node;
+	while (*p) {
+		struct drm_prime_member *pos;
+
+		rb = *p;
+		pos = rb_entry(rb, struct drm_prime_member, handle_rb);
+		if (handle > pos->handle)
+			p = &rb->rb_right;
+		else
+			p = &rb->rb_left;
+	}
+	rb_link_node(&member->handle_rb, rb, p);
+	rb_insert_color(&member->handle_rb, &prime_fpriv->handles);
+
 	return 0;
 }
 
 static struct dma_buf *drm_prime_lookup_buf_by_handle(struct drm_prime_file_private *prime_fpriv,
 						      uint32_t handle)
 {
-	struct drm_prime_member *member;
+	struct rb_node *rb;
+
+	rb = prime_fpriv->handles.rb_node;
+	while (rb) {
+		struct drm_prime_member *member;
 
-	list_for_each_entry(member, &prime_fpriv->head, entry) {
+		member = rb_entry(rb, struct drm_prime_member, handle_rb);
 		if (member->handle == handle)
 			return member->dma_buf;
+		else if (member->handle < handle)
+			rb = rb->rb_right;
+		else
+			rb = rb->rb_left;
 	}
 
 	return NULL;
@@ -104,14 +146,23 @@ static int drm_prime_lookup_buf_handle(struct drm_prime_file_private *prime_fpri
 				       struct dma_buf *dma_buf,
 				       uint32_t *handle)
 {
-	struct drm_prime_member *member;
+	struct rb_node *rb;
+
+	rb = prime_fpriv->dmabufs.rb_node;
+	while (rb) {
+		struct drm_prime_member *member;
 
-	list_for_each_entry(member, &prime_fpriv->head, entry) {
+		member = rb_entry(rb, struct drm_prime_member, dmabuf_rb);
 		if (member->dma_buf == dma_buf) {
 			*handle = member->handle;
 			return 0;
+		} else if (member->dma_buf < dma_buf) {
+			rb = rb->rb_right;
+		} else {
+			rb = rb->rb_left;
 		}
 	}
+
 	return -ENOENT;
 }
 
@@ -166,13 +217,24 @@ static void drm_gem_map_detach(struct dma_buf *dma_buf,
 void drm_prime_remove_buf_handle_locked(struct drm_prime_file_private *prime_fpriv,
 					struct dma_buf *dma_buf)
 {
-	struct drm_prime_member *member, *safe;
+	struct rb_node *rb;
 
-	list_for_each_entry_safe(member, safe, &prime_fpriv->head, entry) {
+	rb = prime_fpriv->dmabufs.rb_node;
+	while (rb) {
+		struct drm_prime_member *member;
+
+		member = rb_entry(rb, struct drm_prime_member, dmabuf_rb);
 		if (member->dma_buf == dma_buf) {
+			rb_erase(&member->handle_rb, &prime_fpriv->handles);
+			rb_erase(&member->dmabuf_rb, &prime_fpriv->dmabufs);
+
 			dma_buf_put(dma_buf);
-			list_del(&member->entry);
 			kfree(member);
+			return;
+		} else if (member->dma_buf < dma_buf) {
+			rb = rb->rb_right;
+		} else {
+			rb = rb->rb_left;
 		}
 	}
 }
@@ -222,18 +284,47 @@ static void drm_gem_unmap_dma_buf(struct dma_buf_attachment *attach,
 }
 
 /**
+ * drm_gem_dmabuf_export - dma_buf export implementation for GEM
+ * @dev: parent device for the exported dmabuf
+ * @exp_info: the export information used by dma_buf_export()
+ *
+ * This wraps dma_buf_export() for use by generic GEM drivers that are using
+ * drm_gem_dmabuf_release(). In addition to calling dma_buf_export(), we take
+ * a reference to the drm_device which is released by drm_gem_dmabuf_release().
+ *
+ * Returns the new dmabuf.
+ */
+struct dma_buf *drm_gem_dmabuf_export(struct drm_device *dev,
+				      struct dma_buf_export_info *exp_info)
+{
+	struct dma_buf *dma_buf;
+
+	dma_buf = dma_buf_export(exp_info);
+	if (!IS_ERR(dma_buf))
+		drm_dev_ref(dev);
+
+	return dma_buf;
+}
+EXPORT_SYMBOL(drm_gem_dmabuf_export);
+
+/**
  * drm_gem_dmabuf_release - dma_buf release implementation for GEM
  * @dma_buf: buffer to be released
  *
  * Generic release function for dma_bufs exported as PRIME buffers. GEM drivers
  * must use this in their dma_buf ops structure as the release callback.
+ * drm_gem_dmabuf_release() should be used in conjunction with
+ * drm_gem_dmabuf_export().
  */
 void drm_gem_dmabuf_release(struct dma_buf *dma_buf)
 {
 	struct drm_gem_object *obj = dma_buf->priv;
+	struct drm_device *dev = obj->dev;
 
 	/* drop the reference on the export fd holds */
 	drm_gem_object_unreference_unlocked(obj);
+
+	drm_dev_unref(dev);
 }
 EXPORT_SYMBOL(drm_gem_dmabuf_release);
 
@@ -335,19 +426,22 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops =  {
  * using the PRIME helpers.
  */
 struct dma_buf *drm_gem_prime_export(struct drm_device *dev,
-				     struct drm_gem_object *obj, int flags)
+				     struct drm_gem_object *obj,
+				     int flags)
 {
-	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
-
-	exp_info.ops = &drm_gem_prime_dmabuf_ops;
-	exp_info.size = obj->size;
-	exp_info.flags = flags;
-	exp_info.priv = obj;
+	struct dma_buf_export_info exp_info = {
+		.exp_name = KBUILD_MODNAME, /* white lie for debug */
+		.owner = dev->driver->fops->owner,
+		.ops = &drm_gem_prime_dmabuf_ops,
+		.size = obj->size,
+		.flags = flags,
+		.priv = obj,
+	};
 
 	if (dev->driver->gem_prime_res_obj)
 		exp_info.resv = dev->driver->gem_prime_res_obj(obj);
 
-	return dma_buf_export(&exp_info);
+	return drm_gem_dmabuf_export(dev, &exp_info);
 }
 EXPORT_SYMBOL(drm_gem_prime_export);
 
@@ -759,12 +853,13 @@ EXPORT_SYMBOL(drm_prime_gem_destroy);
 
 void drm_prime_init_file_private(struct drm_prime_file_private *prime_fpriv)
 {
-	INIT_LIST_HEAD(&prime_fpriv->head);
 	mutex_init(&prime_fpriv->lock);
+	prime_fpriv->dmabufs = RB_ROOT;
+	prime_fpriv->handles = RB_ROOT;
 }
 
 void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv)
 {
 	/* by now drm_gem_release should've made sure the list is empty */
-	WARN_ON(!list_empty(&prime_fpriv->head));
+	WARN_ON(!RB_EMPTY_ROOT(&prime_fpriv->dmabufs));
 }
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index a0df377d7d1c..f6b64d7d3528 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -129,6 +129,7 @@ void drm_kms_helper_poll_enable_locked(struct drm_device *dev)
 {
 	bool poll = false;
 	struct drm_connector *connector;
+	unsigned long delay = DRM_OUTPUT_POLL_PERIOD;
 
 	WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
 
@@ -141,8 +142,13 @@ void drm_kms_helper_poll_enable_locked(struct drm_device *dev)
 			poll = true;
 	}
 
+	if (dev->mode_config.delayed_event) {
+		poll = true;
+		delay = 0;
+	}
+
 	if (poll)
-		schedule_delayed_work(&dev->mode_config.output_poll_work, DRM_OUTPUT_POLL_PERIOD);
+		schedule_delayed_work(&dev->mode_config.output_poll_work, delay);
 }
 EXPORT_SYMBOL(drm_kms_helper_poll_enable_locked);
 
diff --git a/drivers/gpu/drm/drm_property.c b/drivers/gpu/drm/drm_property.c
new file mode 100644
index 000000000000..a4d81cf4ffa0
--- /dev/null
+++ b/drivers/gpu/drm/drm_property.c
@@ -0,0 +1,912 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <linux/export.h>
+#include <drm/drmP.h>
+#include <drm/drm_property.h>
+
+#include "drm_crtc_internal.h"
+
+/**
+ * DOC: overview
+ *
+ * Properties as represented by &drm_property are used to extend the modeset
+ * interface exposed to userspace. For the atomic modeset IOCTL properties are
+ * even the only way to transport metadata about the desired new modeset
+ * configuration from userspace to the kernel. Properties have a well-defined
+ * value range, which is enforced by the drm core. See the documentation of the
+ * flags member of struct &drm_property for an overview of the different
+ * property types and ranges.
+ *
+ * Properties don't store the current value directly, but need to be
+ * instatiated by attaching them to a &drm_mode_object with
+ * drm_object_attach_property().
+ *
+ * Property values are only 64bit. To support bigger piles of data (like gamma
+ * tables, color correction matrizes or large structures) a property can instead
+ * point at a &drm_property_blob with that additional data
+ *
+ * Properties are defined by their symbolic name, userspace must keep a
+ * per-object mapping from those names to the property ID used in the atomic
+ * IOCTL and in the get/set property IOCTL.
+ */
+
+static bool drm_property_type_valid(struct drm_property *property)
+{
+	if (property->flags & DRM_MODE_PROP_EXTENDED_TYPE)
+		return !(property->flags & DRM_MODE_PROP_LEGACY_TYPE);
+	return !!(property->flags & DRM_MODE_PROP_LEGACY_TYPE);
+}
+
+/**
+ * drm_property_create - create a new property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @num_values: number of pre-defined values
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy(), which is done automatically when calling
+ * drm_mode_config_cleanup().
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
+struct drm_property *drm_property_create(struct drm_device *dev, int flags,
+					 const char *name, int num_values)
+{
+	struct drm_property *property = NULL;
+	int ret;
+
+	property = kzalloc(sizeof(struct drm_property), GFP_KERNEL);
+	if (!property)
+		return NULL;
+
+	property->dev = dev;
+
+	if (num_values) {
+		property->values = kcalloc(num_values, sizeof(uint64_t),
+					   GFP_KERNEL);
+		if (!property->values)
+			goto fail;
+	}
+
+	ret = drm_mode_object_get(dev, &property->base, DRM_MODE_OBJECT_PROPERTY);
+	if (ret)
+		goto fail;
+
+	property->flags = flags;
+	property->num_values = num_values;
+	INIT_LIST_HEAD(&property->enum_list);
+
+	if (name) {
+		strncpy(property->name, name, DRM_PROP_NAME_LEN);
+		property->name[DRM_PROP_NAME_LEN-1] = '\0';
+	}
+
+	list_add_tail(&property->head, &dev->mode_config.property_list);
+
+	WARN_ON(!drm_property_type_valid(property));
+
+	return property;
+fail:
+	kfree(property->values);
+	kfree(property);
+	return NULL;
+}
+EXPORT_SYMBOL(drm_property_create);
+
+/**
+ * drm_property_create_enum - create a new enumeration property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @props: enumeration lists with property values
+ * @num_values: number of pre-defined values
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy(), which is done automatically when calling
+ * drm_mode_config_cleanup().
+ *
+ * Userspace is only allowed to set one of the predefined values for enumeration
+ * properties.
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
+struct drm_property *drm_property_create_enum(struct drm_device *dev, int flags,
+					 const char *name,
+					 const struct drm_prop_enum_list *props,
+					 int num_values)
+{
+	struct drm_property *property;
+	int i, ret;
+
+	flags |= DRM_MODE_PROP_ENUM;
+
+	property = drm_property_create(dev, flags, name, num_values);
+	if (!property)
+		return NULL;
+
+	for (i = 0; i < num_values; i++) {
+		ret = drm_property_add_enum(property, i,
+				      props[i].type,
+				      props[i].name);
+		if (ret) {
+			drm_property_destroy(dev, property);
+			return NULL;
+		}
+	}
+
+	return property;
+}
+EXPORT_SYMBOL(drm_property_create_enum);
+
+/**
+ * drm_property_create_bitmask - create a new bitmask property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @props: enumeration lists with property bitflags
+ * @num_props: size of the @props array
+ * @supported_bits: bitmask of all supported enumeration values
+ *
+ * This creates a new bitmask drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy(), which is done automatically when calling
+ * drm_mode_config_cleanup().
+ *
+ * Compared to plain enumeration properties userspace is allowed to set any
+ * or'ed together combination of the predefined property bitflag values
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
+struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
+					 int flags, const char *name,
+					 const struct drm_prop_enum_list *props,
+					 int num_props,
+					 uint64_t supported_bits)
+{
+	struct drm_property *property;
+	int i, ret, index = 0;
+	int num_values = hweight64(supported_bits);
+
+	flags |= DRM_MODE_PROP_BITMASK;
+
+	property = drm_property_create(dev, flags, name, num_values);
+	if (!property)
+		return NULL;
+	for (i = 0; i < num_props; i++) {
+		if (!(supported_bits & (1ULL << props[i].type)))
+			continue;
+
+		if (WARN_ON(index >= num_values)) {
+			drm_property_destroy(dev, property);
+			return NULL;
+		}
+
+		ret = drm_property_add_enum(property, index++,
+				      props[i].type,
+				      props[i].name);
+		if (ret) {
+			drm_property_destroy(dev, property);
+			return NULL;
+		}
+	}
+
+	return property;
+}
+EXPORT_SYMBOL(drm_property_create_bitmask);
+
+static struct drm_property *property_create_range(struct drm_device *dev,
+					 int flags, const char *name,
+					 uint64_t min, uint64_t max)
+{
+	struct drm_property *property;
+
+	property = drm_property_create(dev, flags, name, 2);
+	if (!property)
+		return NULL;
+
+	property->values[0] = min;
+	property->values[1] = max;
+
+	return property;
+}
+
+/**
+ * drm_property_create_range - create a new unsigned ranged property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @min: minimum value of the property
+ * @max: maximum value of the property
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy(), which is done automatically when calling
+ * drm_mode_config_cleanup().
+ *
+ * Userspace is allowed to set any unsigned integer value in the (min, max)
+ * range inclusive.
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
+struct drm_property *drm_property_create_range(struct drm_device *dev, int flags,
+					 const char *name,
+					 uint64_t min, uint64_t max)
+{
+	return property_create_range(dev, DRM_MODE_PROP_RANGE | flags,
+			name, min, max);
+}
+EXPORT_SYMBOL(drm_property_create_range);
+
+/**
+ * drm_property_create_signed_range - create a new signed ranged property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @min: minimum value of the property
+ * @max: maximum value of the property
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy(), which is done automatically when calling
+ * drm_mode_config_cleanup().
+ *
+ * Userspace is allowed to set any signed integer value in the (min, max)
+ * range inclusive.
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
+struct drm_property *drm_property_create_signed_range(struct drm_device *dev,
+					 int flags, const char *name,
+					 int64_t min, int64_t max)
+{
+	return property_create_range(dev, DRM_MODE_PROP_SIGNED_RANGE | flags,
+			name, I642U64(min), I642U64(max));
+}
+EXPORT_SYMBOL(drm_property_create_signed_range);
+
+/**
+ * drm_property_create_object - create a new object property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @type: object type from DRM_MODE_OBJECT_* defines
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy(), which is done automatically when calling
+ * drm_mode_config_cleanup().
+ *
+ * Userspace is only allowed to set this to any property value of the given
+ * @type. Only useful for atomic properties, which is enforced.
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
+struct drm_property *drm_property_create_object(struct drm_device *dev,
+						int flags, const char *name,
+						uint32_t type)
+{
+	struct drm_property *property;
+
+	flags |= DRM_MODE_PROP_OBJECT;
+
+	if (WARN_ON(!(flags & DRM_MODE_PROP_ATOMIC)))
+		return NULL;
+
+	property = drm_property_create(dev, flags, name, 1);
+	if (!property)
+		return NULL;
+
+	property->values[0] = type;
+
+	return property;
+}
+EXPORT_SYMBOL(drm_property_create_object);
+
+/**
+ * drm_property_create_bool - create a new boolean property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy(), which is done automatically when calling
+ * drm_mode_config_cleanup().
+ *
+ * This is implemented as a ranged property with only {0, 1} as valid values.
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
+struct drm_property *drm_property_create_bool(struct drm_device *dev, int flags,
+					      const char *name)
+{
+	return drm_property_create_range(dev, flags, name, 0, 1);
+}
+EXPORT_SYMBOL(drm_property_create_bool);
+
+/**
+ * drm_property_add_enum - add a possible value to an enumeration property
+ * @property: enumeration property to change
+ * @index: index of the new enumeration
+ * @value: value of the new enumeration
+ * @name: symbolic name of the new enumeration
+ *
+ * This functions adds enumerations to a property.
+ *
+ * It's use is deprecated, drivers should use one of the more specific helpers
+ * to directly create the property with all enumerations already attached.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_property_add_enum(struct drm_property *property, int index,
+			  uint64_t value, const char *name)
+{
+	struct drm_property_enum *prop_enum;
+
+	if (!(drm_property_type_is(property, DRM_MODE_PROP_ENUM) ||
+			drm_property_type_is(property, DRM_MODE_PROP_BITMASK)))
+		return -EINVAL;
+
+	/*
+	 * Bitmask enum properties have the additional constraint of values
+	 * from 0 to 63
+	 */
+	if (drm_property_type_is(property, DRM_MODE_PROP_BITMASK) &&
+			(value > 63))
+		return -EINVAL;
+
+	if (!list_empty(&property->enum_list)) {
+		list_for_each_entry(prop_enum, &property->enum_list, head) {
+			if (prop_enum->value == value) {
+				strncpy(prop_enum->name, name, DRM_PROP_NAME_LEN);
+				prop_enum->name[DRM_PROP_NAME_LEN-1] = '\0';
+				return 0;
+			}
+		}
+	}
+
+	prop_enum = kzalloc(sizeof(struct drm_property_enum), GFP_KERNEL);
+	if (!prop_enum)
+		return -ENOMEM;
+
+	strncpy(prop_enum->name, name, DRM_PROP_NAME_LEN);
+	prop_enum->name[DRM_PROP_NAME_LEN-1] = '\0';
+	prop_enum->value = value;
+
+	property->values[index] = value;
+	list_add_tail(&prop_enum->head, &property->enum_list);
+	return 0;
+}
+EXPORT_SYMBOL(drm_property_add_enum);
+
+/**
+ * drm_property_destroy - destroy a drm property
+ * @dev: drm device
+ * @property: property to destry
+ *
+ * This function frees a property including any attached resources like
+ * enumeration values.
+ */
+void drm_property_destroy(struct drm_device *dev, struct drm_property *property)
+{
+	struct drm_property_enum *prop_enum, *pt;
+
+	list_for_each_entry_safe(prop_enum, pt, &property->enum_list, head) {
+		list_del(&prop_enum->head);
+		kfree(prop_enum);
+	}
+
+	if (property->num_values)
+		kfree(property->values);
+	drm_mode_object_unregister(dev, &property->base);
+	list_del(&property->head);
+	kfree(property);
+}
+EXPORT_SYMBOL(drm_property_destroy);
+
+int drm_mode_getproperty_ioctl(struct drm_device *dev,
+			       void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_get_property *out_resp = data;
+	struct drm_property *property;
+	int enum_count = 0;
+	int value_count = 0;
+	int ret = 0, i;
+	int copied;
+	struct drm_property_enum *prop_enum;
+	struct drm_mode_property_enum __user *enum_ptr;
+	uint64_t __user *values_ptr;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	drm_modeset_lock_all(dev);
+	property = drm_property_find(dev, out_resp->prop_id);
+	if (!property) {
+		ret = -ENOENT;
+		goto done;
+	}
+
+	if (drm_property_type_is(property, DRM_MODE_PROP_ENUM) ||
+			drm_property_type_is(property, DRM_MODE_PROP_BITMASK)) {
+		list_for_each_entry(prop_enum, &property->enum_list, head)
+			enum_count++;
+	}
+
+	value_count = property->num_values;
+
+	strncpy(out_resp->name, property->name, DRM_PROP_NAME_LEN);
+	out_resp->name[DRM_PROP_NAME_LEN-1] = 0;
+	out_resp->flags = property->flags;
+
+	if ((out_resp->count_values >= value_count) && value_count) {
+		values_ptr = (uint64_t __user *)(unsigned long)out_resp->values_ptr;
+		for (i = 0; i < value_count; i++) {
+			if (copy_to_user(values_ptr + i, &property->values[i], sizeof(uint64_t))) {
+				ret = -EFAULT;
+				goto done;
+			}
+		}
+	}
+	out_resp->count_values = value_count;
+
+	if (drm_property_type_is(property, DRM_MODE_PROP_ENUM) ||
+			drm_property_type_is(property, DRM_MODE_PROP_BITMASK)) {
+		if ((out_resp->count_enum_blobs >= enum_count) && enum_count) {
+			copied = 0;
+			enum_ptr = (struct drm_mode_property_enum __user *)(unsigned long)out_resp->enum_blob_ptr;
+			list_for_each_entry(prop_enum, &property->enum_list, head) {
+
+				if (copy_to_user(&enum_ptr[copied].value, &prop_enum->value, sizeof(uint64_t))) {
+					ret = -EFAULT;
+					goto done;
+				}
+
+				if (copy_to_user(&enum_ptr[copied].name,
+						 &prop_enum->name, DRM_PROP_NAME_LEN)) {
+					ret = -EFAULT;
+					goto done;
+				}
+				copied++;
+			}
+		}
+		out_resp->count_enum_blobs = enum_count;
+	}
+
+	/*
+	 * NOTE: The idea seems to have been to use this to read all the blob
+	 * property values. But nothing ever added them to the corresponding
+	 * list, userspace always used the special-purpose get_blob ioctl to
+	 * read the value for a blob property. It also doesn't make a lot of
+	 * sense to return values here when everything else is just metadata for
+	 * the property itself.
+	 */
+	if (drm_property_type_is(property, DRM_MODE_PROP_BLOB))
+		out_resp->count_enum_blobs = 0;
+done:
+	drm_modeset_unlock_all(dev);
+	return ret;
+}
+
+static void drm_property_free_blob(struct kref *kref)
+{
+	struct drm_property_blob *blob =
+		container_of(kref, struct drm_property_blob, base.refcount);
+
+	mutex_lock(&blob->dev->mode_config.blob_lock);
+	list_del(&blob->head_global);
+	mutex_unlock(&blob->dev->mode_config.blob_lock);
+
+	drm_mode_object_unregister(blob->dev, &blob->base);
+
+	kfree(blob);
+}
+
+/**
+ * drm_property_create_blob - Create new blob property
+ * @dev: DRM device to create property for
+ * @length: Length to allocate for blob data
+ * @data: If specified, copies data into blob
+ *
+ * Creates a new blob property for a specified DRM device, optionally
+ * copying data. Note that blob properties are meant to be invariant, hence the
+ * data must be filled out before the blob is used as the value of any property.
+ *
+ * Returns:
+ * New blob property with a single reference on success, or an ERR_PTR
+ * value on failure.
+ */
+struct drm_property_blob *
+drm_property_create_blob(struct drm_device *dev, size_t length,
+			 const void *data)
+{
+	struct drm_property_blob *blob;
+	int ret;
+
+	if (!length || length > ULONG_MAX - sizeof(struct drm_property_blob))
+		return ERR_PTR(-EINVAL);
+
+	blob = kzalloc(sizeof(struct drm_property_blob)+length, GFP_KERNEL);
+	if (!blob)
+		return ERR_PTR(-ENOMEM);
+
+	/* This must be explicitly initialised, so we can safely call list_del
+	 * on it in the removal handler, even if it isn't in a file list. */
+	INIT_LIST_HEAD(&blob->head_file);
+	blob->length = length;
+	blob->dev = dev;
+
+	if (data)
+		memcpy(blob->data, data, length);
+
+	ret = drm_mode_object_get_reg(dev, &blob->base, DRM_MODE_OBJECT_BLOB,
+				      true, drm_property_free_blob);
+	if (ret) {
+		kfree(blob);
+		return ERR_PTR(-EINVAL);
+	}
+
+	mutex_lock(&dev->mode_config.blob_lock);
+	list_add_tail(&blob->head_global,
+	              &dev->mode_config.property_blob_list);
+	mutex_unlock(&dev->mode_config.blob_lock);
+
+	return blob;
+}
+EXPORT_SYMBOL(drm_property_create_blob);
+
+/**
+ * drm_property_unreference_blob - Unreference a blob property
+ * @blob: Pointer to blob property
+ *
+ * Drop a reference on a blob property. May free the object.
+ */
+void drm_property_unreference_blob(struct drm_property_blob *blob)
+{
+	if (!blob)
+		return;
+
+	drm_mode_object_unreference(&blob->base);
+}
+EXPORT_SYMBOL(drm_property_unreference_blob);
+
+void drm_property_destroy_user_blobs(struct drm_device *dev,
+				     struct drm_file *file_priv)
+{
+	struct drm_property_blob *blob, *bt;
+
+	/*
+	 * When the file gets released that means no one else can access the
+	 * blob list any more, so no need to grab dev->blob_lock.
+	 */
+	list_for_each_entry_safe(blob, bt, &file_priv->blobs, head_file) {
+		list_del_init(&blob->head_file);
+		drm_property_unreference_blob(blob);
+	}
+}
+
+/**
+ * drm_property_reference_blob - Take a reference on an existing property
+ * @blob: Pointer to blob property
+ *
+ * Take a new reference on an existing blob property. Returns @blob, which
+ * allows this to be used as a shorthand in assignments.
+ */
+struct drm_property_blob *drm_property_reference_blob(struct drm_property_blob *blob)
+{
+	drm_mode_object_reference(&blob->base);
+	return blob;
+}
+EXPORT_SYMBOL(drm_property_reference_blob);
+
+/**
+ * drm_property_lookup_blob - look up a blob property and take a reference
+ * @dev: drm device
+ * @id: id of the blob property
+ *
+ * If successful, this takes an additional reference to the blob property.
+ * callers need to make sure to eventually unreference the returned property
+ * again, using @drm_property_unreference_blob.
+ *
+ * Return:
+ * NULL on failure, pointer to the blob on success.
+ */
+struct drm_property_blob *drm_property_lookup_blob(struct drm_device *dev,
+					           uint32_t id)
+{
+	struct drm_mode_object *obj;
+	struct drm_property_blob *blob = NULL;
+
+	obj = __drm_mode_object_find(dev, id, DRM_MODE_OBJECT_BLOB);
+	if (obj)
+		blob = obj_to_blob(obj);
+	return blob;
+}
+EXPORT_SYMBOL(drm_property_lookup_blob);
+
+/**
+ * drm_property_replace_global_blob - replace existing blob property
+ * @dev: drm device
+ * @replace: location of blob property pointer to be replaced
+ * @length: length of data for new blob, or 0 for no data
+ * @data: content for new blob, or NULL for no data
+ * @obj_holds_id: optional object for property holding blob ID
+ * @prop_holds_id: optional property holding blob ID
+ * @return 0 on success or error on failure
+ *
+ * This function will replace a global property in the blob list, optionally
+ * updating a property which holds the ID of that property.
+ *
+ * If length is 0 or data is NULL, no new blob will be created, and the holding
+ * property, if specified, will be set to 0.
+ *
+ * Access to the replace pointer is assumed to be protected by the caller, e.g.
+ * by holding the relevant modesetting object lock for its parent.
+ *
+ * For example, a drm_connector has a 'PATH' property, which contains the ID
+ * of a blob property with the value of the MST path information. Calling this
+ * function with replace pointing to the connector's path_blob_ptr, length and
+ * data set for the new path information, obj_holds_id set to the connector's
+ * base object, and prop_holds_id set to the path property name, will perform
+ * a completely atomic update. The access to path_blob_ptr is protected by the
+ * caller holding a lock on the connector.
+ */
+int drm_property_replace_global_blob(struct drm_device *dev,
+				     struct drm_property_blob **replace,
+				     size_t length,
+				     const void *data,
+				     struct drm_mode_object *obj_holds_id,
+				     struct drm_property *prop_holds_id)
+{
+	struct drm_property_blob *new_blob = NULL;
+	struct drm_property_blob *old_blob = NULL;
+	int ret;
+
+	WARN_ON(replace == NULL);
+
+	old_blob = *replace;
+
+	if (length && data) {
+		new_blob = drm_property_create_blob(dev, length, data);
+		if (IS_ERR(new_blob))
+			return PTR_ERR(new_blob);
+	}
+
+	if (obj_holds_id) {
+		ret = drm_object_property_set_value(obj_holds_id,
+						    prop_holds_id,
+						    new_blob ?
+						        new_blob->base.id : 0);
+		if (ret != 0)
+			goto err_created;
+	}
+
+	drm_property_unreference_blob(old_blob);
+	*replace = new_blob;
+
+	return 0;
+
+err_created:
+	drm_property_unreference_blob(new_blob);
+	return ret;
+}
+EXPORT_SYMBOL(drm_property_replace_global_blob);
+
+int drm_mode_getblob_ioctl(struct drm_device *dev,
+			   void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_get_blob *out_resp = data;
+	struct drm_property_blob *blob;
+	int ret = 0;
+	void __user *blob_ptr;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	blob = drm_property_lookup_blob(dev, out_resp->blob_id);
+	if (!blob)
+		return -ENOENT;
+
+	if (out_resp->length == blob->length) {
+		blob_ptr = (void __user *)(unsigned long)out_resp->data;
+		if (copy_to_user(blob_ptr, blob->data, blob->length)) {
+			ret = -EFAULT;
+			goto unref;
+		}
+	}
+	out_resp->length = blob->length;
+unref:
+	drm_property_unreference_blob(blob);
+
+	return ret;
+}
+
+int drm_mode_createblob_ioctl(struct drm_device *dev,
+			      void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_create_blob *out_resp = data;
+	struct drm_property_blob *blob;
+	void __user *blob_ptr;
+	int ret = 0;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	blob = drm_property_create_blob(dev, out_resp->length, NULL);
+	if (IS_ERR(blob))
+		return PTR_ERR(blob);
+
+	blob_ptr = (void __user *)(unsigned long)out_resp->data;
+	if (copy_from_user(blob->data, blob_ptr, out_resp->length)) {
+		ret = -EFAULT;
+		goto out_blob;
+	}
+
+	/* Dropping the lock between create_blob and our access here is safe
+	 * as only the same file_priv can remove the blob; at this point, it is
+	 * not associated with any file_priv. */
+	mutex_lock(&dev->mode_config.blob_lock);
+	out_resp->blob_id = blob->base.id;
+	list_add_tail(&blob->head_file, &file_priv->blobs);
+	mutex_unlock(&dev->mode_config.blob_lock);
+
+	return 0;
+
+out_blob:
+	drm_property_unreference_blob(blob);
+	return ret;
+}
+
+int drm_mode_destroyblob_ioctl(struct drm_device *dev,
+			       void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_destroy_blob *out_resp = data;
+	struct drm_property_blob *blob = NULL, *bt;
+	bool found = false;
+	int ret = 0;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	blob = drm_property_lookup_blob(dev, out_resp->blob_id);
+	if (!blob)
+		return -ENOENT;
+
+	mutex_lock(&dev->mode_config.blob_lock);
+	/* Ensure the property was actually created by this user. */
+	list_for_each_entry(bt, &file_priv->blobs, head_file) {
+		if (bt == blob) {
+			found = true;
+			break;
+		}
+	}
+
+	if (!found) {
+		ret = -EPERM;
+		goto err;
+	}
+
+	/* We must drop head_file here, because we may not be the last
+	 * reference on the blob. */
+	list_del_init(&blob->head_file);
+	mutex_unlock(&dev->mode_config.blob_lock);
+
+	/* One reference from lookup, and one from the filp. */
+	drm_property_unreference_blob(blob);
+	drm_property_unreference_blob(blob);
+
+	return 0;
+
+err:
+	mutex_unlock(&dev->mode_config.blob_lock);
+	drm_property_unreference_blob(blob);
+
+	return ret;
+}
+
+/* Some properties could refer to dynamic refcnt'd objects, or things that
+ * need special locking to handle lifetime issues (ie. to ensure the prop
+ * value doesn't become invalid part way through the property update due to
+ * race).  The value returned by reference via 'obj' should be passed back
+ * to drm_property_change_valid_put() after the property is set (and the
+ * object to which the property is attached has a chance to take it's own
+ * reference).
+ */
+bool drm_property_change_valid_get(struct drm_property *property,
+				   uint64_t value, struct drm_mode_object **ref)
+{
+	int i;
+
+	if (property->flags & DRM_MODE_PROP_IMMUTABLE)
+		return false;
+
+	*ref = NULL;
+
+	if (drm_property_type_is(property, DRM_MODE_PROP_RANGE)) {
+		if (value < property->values[0] || value > property->values[1])
+			return false;
+		return true;
+	} else if (drm_property_type_is(property, DRM_MODE_PROP_SIGNED_RANGE)) {
+		int64_t svalue = U642I64(value);
+
+		if (svalue < U642I64(property->values[0]) ||
+				svalue > U642I64(property->values[1]))
+			return false;
+		return true;
+	} else if (drm_property_type_is(property, DRM_MODE_PROP_BITMASK)) {
+		uint64_t valid_mask = 0;
+
+		for (i = 0; i < property->num_values; i++)
+			valid_mask |= (1ULL << property->values[i]);
+		return !(value & ~valid_mask);
+	} else if (drm_property_type_is(property, DRM_MODE_PROP_BLOB)) {
+		struct drm_property_blob *blob;
+
+		if (value == 0)
+			return true;
+
+		blob = drm_property_lookup_blob(property->dev, value);
+		if (blob) {
+			*ref = &blob->base;
+			return true;
+		} else {
+			return false;
+		}
+	} else if (drm_property_type_is(property, DRM_MODE_PROP_OBJECT)) {
+		/* a zero value for an object property translates to null: */
+		if (value == 0)
+			return true;
+
+		*ref = __drm_mode_object_find(property->dev, value,
+					      property->values[0]);
+		return *ref != NULL;
+	}
+
+	for (i = 0; i < property->num_values; i++)
+		if (property->values[i] == value)
+			return true;
+	return false;
+}
+
+void drm_property_change_valid_put(struct drm_property *property,
+		struct drm_mode_object *ref)
+{
+	if (!ref)
+		return;
+
+	if (drm_property_type_is(property, DRM_MODE_PROP_OBJECT)) {
+		drm_mode_object_unreference(ref);
+	} else if (drm_property_type_is(property, DRM_MODE_PROP_BLOB))
+		drm_property_unreference_blob(obj_to_blob(ref));
+}
diff --git a/drivers/gpu/drm/drm_rect.c b/drivers/gpu/drm/drm_rect.c
index a8e2c8603945..73e53a8d1b37 100644
--- a/drivers/gpu/drm/drm_rect.c
+++ b/drivers/gpu/drm/drm_rect.c
@@ -100,7 +100,7 @@ static int drm_calc_scale(int src, int dst)
 {
 	int scale = 0;
 
-	if (src < 0 || dst < 0)
+	if (WARN_ON(src < 0 || dst < 0))
 		return -EINVAL;
 
 	if (dst == 0)
@@ -317,38 +317,38 @@ void drm_rect_rotate(struct drm_rect *r,
 {
 	struct drm_rect tmp;
 
-	if (rotation & (BIT(DRM_REFLECT_X) | BIT(DRM_REFLECT_Y))) {
+	if (rotation & (DRM_REFLECT_X | DRM_REFLECT_Y)) {
 		tmp = *r;
 
-		if (rotation & BIT(DRM_REFLECT_X)) {
+		if (rotation & DRM_REFLECT_X) {
 			r->x1 = width - tmp.x2;
 			r->x2 = width - tmp.x1;
 		}
 
-		if (rotation & BIT(DRM_REFLECT_Y)) {
+		if (rotation & DRM_REFLECT_Y) {
 			r->y1 = height - tmp.y2;
 			r->y2 = height - tmp.y1;
 		}
 	}
 
 	switch (rotation & DRM_ROTATE_MASK) {
-	case BIT(DRM_ROTATE_0):
+	case DRM_ROTATE_0:
 		break;
-	case BIT(DRM_ROTATE_90):
+	case DRM_ROTATE_90:
 		tmp = *r;
 		r->x1 = tmp.y1;
 		r->x2 = tmp.y2;
 		r->y1 = width - tmp.x2;
 		r->y2 = width - tmp.x1;
 		break;
-	case BIT(DRM_ROTATE_180):
+	case DRM_ROTATE_180:
 		tmp = *r;
 		r->x1 = width - tmp.x2;
 		r->x2 = width - tmp.x1;
 		r->y1 = height - tmp.y2;
 		r->y2 = height - tmp.y1;
 		break;
-	case BIT(DRM_ROTATE_270):
+	case DRM_ROTATE_270:
 		tmp = *r;
 		r->x1 = height - tmp.y2;
 		r->x2 = height - tmp.y1;
@@ -392,23 +392,23 @@ void drm_rect_rotate_inv(struct drm_rect *r,
 	struct drm_rect tmp;
 
 	switch (rotation & DRM_ROTATE_MASK) {
-	case BIT(DRM_ROTATE_0):
+	case DRM_ROTATE_0:
 		break;
-	case BIT(DRM_ROTATE_90):
+	case DRM_ROTATE_90:
 		tmp = *r;
 		r->x1 = width - tmp.y2;
 		r->x2 = width - tmp.y1;
 		r->y1 = tmp.x1;
 		r->y2 = tmp.x2;
 		break;
-	case BIT(DRM_ROTATE_180):
+	case DRM_ROTATE_180:
 		tmp = *r;
 		r->x1 = width - tmp.x2;
 		r->x2 = width - tmp.x1;
 		r->y1 = height - tmp.y2;
 		r->y2 = height - tmp.y1;
 		break;
-	case BIT(DRM_ROTATE_270):
+	case DRM_ROTATE_270:
 		tmp = *r;
 		r->x1 = tmp.y1;
 		r->x2 = tmp.y2;
@@ -419,15 +419,15 @@ void drm_rect_rotate_inv(struct drm_rect *r,
 		break;
 	}
 
-	if (rotation & (BIT(DRM_REFLECT_X) | BIT(DRM_REFLECT_Y))) {
+	if (rotation & (DRM_REFLECT_X | DRM_REFLECT_Y)) {
 		tmp = *r;
 
-		if (rotation & BIT(DRM_REFLECT_X)) {
+		if (rotation & DRM_REFLECT_X) {
 			r->x1 = width - tmp.x2;
 			r->x2 = width - tmp.x1;
 		}
 
-		if (rotation & BIT(DRM_REFLECT_Y)) {
+		if (rotation & DRM_REFLECT_Y) {
 			r->y1 = height - tmp.y2;
 			r->y2 = height - tmp.y1;
 		}
diff --git a/drivers/gpu/drm/drm_scatter.c b/drivers/gpu/drm/drm_scatter.c
index bf70431073f6..275bca44f38c 100644
--- a/drivers/gpu/drm/drm_scatter.c
+++ b/drivers/gpu/drm/drm_scatter.c
@@ -68,7 +68,7 @@ static void drm_sg_cleanup(struct drm_sg_mem * entry)
 void drm_legacy_sg_cleanup(struct drm_device *dev)
 {
 	if (drm_core_check_feature(dev, DRIVER_SG) && dev->sg &&
-	    !drm_core_check_feature(dev, DRIVER_MODESET)) {
+	    drm_core_check_feature(dev, DRIVER_LEGACY)) {
 		drm_sg_cleanup(dev->sg);
 		dev->sg = NULL;
 	}
@@ -88,7 +88,7 @@ int drm_legacy_sg_alloc(struct drm_device *dev, void *data,
 
 	DRM_DEBUG("\n");
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (!drm_core_check_feature(dev, DRIVER_SG))
@@ -201,7 +201,7 @@ int drm_legacy_sg_free(struct drm_device *dev, void *data,
 	struct drm_scatter_gather *request = data;
 	struct drm_sg_mem *entry;
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
 		return -EINVAL;
 
 	if (!drm_core_check_feature(dev, DRIVER_SG))
diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c
index 0db36d27e90b..7bae08c2bf0a 100644
--- a/drivers/gpu/drm/drm_simple_kms_helper.c
+++ b/drivers/gpu/drm/drm_simple_kms_helper.c
@@ -34,6 +34,12 @@ static const struct drm_encoder_funcs drm_simple_kms_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
+static int drm_simple_kms_crtc_check(struct drm_crtc *crtc,
+				     struct drm_crtc_state *state)
+{
+	return drm_atomic_add_affected_planes(state->state, crtc);
+}
+
 static void drm_simple_kms_crtc_enable(struct drm_crtc *crtc)
 {
 	struct drm_simple_display_pipe *pipe;
@@ -57,6 +63,7 @@ static void drm_simple_kms_crtc_disable(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_helper_funcs drm_simple_kms_crtc_helper_funcs = {
+	.atomic_check = drm_simple_kms_crtc_check,
 	.disable = drm_simple_kms_crtc_disable,
 	.enable = drm_simple_kms_crtc_enable,
 };
@@ -73,22 +80,9 @@ static const struct drm_crtc_funcs drm_simple_kms_crtc_funcs = {
 static int drm_simple_kms_plane_atomic_check(struct drm_plane *plane,
 					struct drm_plane_state *plane_state)
 {
-	struct drm_rect src = {
-		.x1 = plane_state->src_x,
-		.y1 = plane_state->src_y,
-		.x2 = plane_state->src_x + plane_state->src_w,
-		.y2 = plane_state->src_y + plane_state->src_h,
-	};
-	struct drm_rect dest = {
-		.x1 = plane_state->crtc_x,
-		.y1 = plane_state->crtc_y,
-		.x2 = plane_state->crtc_x + plane_state->crtc_w,
-		.y2 = plane_state->crtc_y + plane_state->crtc_h,
-	};
 	struct drm_rect clip = { 0 };
 	struct drm_simple_display_pipe *pipe;
 	struct drm_crtc_state *crtc_state;
-	bool visible;
 	int ret;
 
 	pipe = container_of(plane, struct drm_simple_display_pipe, plane);
@@ -102,17 +96,15 @@ static int drm_simple_kms_plane_atomic_check(struct drm_plane *plane,
 
 	clip.x2 = crtc_state->adjusted_mode.hdisplay;
 	clip.y2 = crtc_state->adjusted_mode.vdisplay;
-	ret = drm_plane_helper_check_update(plane, &pipe->crtc,
-					    plane_state->fb,
-					    &src, &dest, &clip,
-					    plane_state->rotation,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    false, true, &visible);
+
+	ret = drm_plane_helper_check_state(plane_state, &clip,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   false, true);
 	if (ret)
 		return ret;
 
-	if (!visible)
+	if (!plane_state->visible)
 		return -EINVAL;
 
 	if (!pipe->funcs || !pipe->funcs->check)
@@ -133,7 +125,33 @@ static void drm_simple_kms_plane_atomic_update(struct drm_plane *plane,
 	pipe->funcs->update(pipe, pstate);
 }
 
+static int drm_simple_kms_plane_prepare_fb(struct drm_plane *plane,
+					   struct drm_plane_state *state)
+{
+	struct drm_simple_display_pipe *pipe;
+
+	pipe = container_of(plane, struct drm_simple_display_pipe, plane);
+	if (!pipe->funcs || !pipe->funcs->prepare_fb)
+		return 0;
+
+	return pipe->funcs->prepare_fb(pipe, state);
+}
+
+static void drm_simple_kms_plane_cleanup_fb(struct drm_plane *plane,
+					    struct drm_plane_state *state)
+{
+	struct drm_simple_display_pipe *pipe;
+
+	pipe = container_of(plane, struct drm_simple_display_pipe, plane);
+	if (!pipe->funcs || !pipe->funcs->cleanup_fb)
+		return;
+
+	pipe->funcs->cleanup_fb(pipe, state);
+}
+
 static const struct drm_plane_helper_funcs drm_simple_kms_plane_helper_funcs = {
+	.prepare_fb = drm_simple_kms_plane_prepare_fb,
+	.cleanup_fb = drm_simple_kms_plane_cleanup_fb,
 	.atomic_check = drm_simple_kms_plane_atomic_check,
 	.atomic_update = drm_simple_kms_plane_atomic_update,
 };
@@ -148,16 +166,61 @@ static const struct drm_plane_funcs drm_simple_kms_plane_funcs = {
 };
 
 /**
+ * drm_simple_display_pipe_attach_bridge - Attach a bridge to the display pipe
+ * @pipe: simple display pipe object
+ * @bridge: bridge to attach
+ *
+ * Makes it possible to still use the drm_simple_display_pipe helpers when
+ * a DRM bridge has to be used.
+ *
+ * Note that you probably want to initialize the pipe by passing a NULL
+ * connector to drm_simple_display_pipe_init().
+ *
+ * Returns:
+ * Zero on success, negative error code on failure.
+ */
+int drm_simple_display_pipe_attach_bridge(struct drm_simple_display_pipe *pipe,
+					  struct drm_bridge *bridge)
+{
+	bridge->encoder = &pipe->encoder;
+	pipe->encoder.bridge = bridge;
+	return drm_bridge_attach(pipe->encoder.dev, bridge);
+}
+EXPORT_SYMBOL(drm_simple_display_pipe_attach_bridge);
+
+/**
+ * drm_simple_display_pipe_detach_bridge - Detach the bridge from the display pipe
+ * @pipe: simple display pipe object
+ *
+ * Detaches the drm bridge previously attached with
+ * drm_simple_display_pipe_attach_bridge()
+ */
+void drm_simple_display_pipe_detach_bridge(struct drm_simple_display_pipe *pipe)
+{
+	if (WARN_ON(!pipe->encoder.bridge))
+		return;
+
+	drm_bridge_detach(pipe->encoder.bridge);
+	pipe->encoder.bridge = NULL;
+}
+EXPORT_SYMBOL(drm_simple_display_pipe_detach_bridge);
+
+/**
  * drm_simple_display_pipe_init - Initialize a simple display pipeline
  * @dev: DRM device
  * @pipe: simple display pipe object to initialize
  * @funcs: callbacks for the display pipe (optional)
- * @formats: array of supported formats (%DRM_FORMAT_*)
+ * @formats: array of supported formats (DRM_FORMAT\_\*)
  * @format_count: number of elements in @formats
- * @connector: connector to attach and register
+ * @connector: connector to attach and register (optional)
  *
  * Sets up a display pipeline which consist of a really simple
- * plane-crtc-encoder pipe coupled with the provided connector.
+ * plane-crtc-encoder pipe.
+ *
+ * If a connector is supplied, the pipe will be coupled with the provided
+ * connector. You may supply a NULL connector when using drm bridges, that
+ * handle connectors themselves (see drm_simple_display_pipe_attach_bridge()).
+ *
  * Teardown of a simple display pipe is all handled automatically by the drm
  * core through calling drm_mode_config_cleanup(). Drivers afterwards need to
  * release the memory for the structure themselves.
@@ -196,7 +259,7 @@ int drm_simple_display_pipe_init(struct drm_device *dev,
 	encoder->possible_crtcs = 1 << drm_crtc_index(crtc);
 	ret = drm_encoder_init(dev, encoder, &drm_simple_kms_encoder_funcs,
 			       DRM_MODE_ENCODER_NONE, NULL);
-	if (ret)
+	if (ret || !connector)
 		return ret;
 
 	return drm_mode_connector_attach_encoder(connector, encoder);
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index 32dd821b7202..9a37196c1bf1 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -19,7 +19,6 @@
 #include <linux/export.h>
 
 #include <drm/drm_sysfs.h>
-#include <drm/drm_core.h>
 #include <drm/drmP.h>
 #include "drm_internal.h"
 
@@ -37,12 +36,7 @@ static char *drm_devnode(struct device *dev, umode_t *mode)
 	return kasprintf(GFP_KERNEL, "dri/%s", dev_name(dev));
 }
 
-static CLASS_ATTR_STRING(version, S_IRUGO,
-		CORE_NAME " "
-		__stringify(CORE_MAJOR) "."
-		__stringify(CORE_MINOR) "."
-		__stringify(CORE_PATCHLEVEL) " "
-		CORE_DATE);
+static CLASS_ATTR_STRING(version, S_IRUGO, "drm 1.1.0 20060810");
 
 /**
  * drm_sysfs_init - initialize sysfs helpers
diff --git a/drivers/gpu/drm/drm_vma_manager.c b/drivers/gpu/drm/drm_vma_manager.c
index f306c8855978..20cc33d1bfc1 100644
--- a/drivers/gpu/drm/drm_vma_manager.c
+++ b/drivers/gpu/drm/drm_vma_manager.c
@@ -25,7 +25,6 @@
 #include <drm/drmP.h>
 #include <drm/drm_mm.h>
 #include <drm/drm_vma_manager.h>
-#include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/rbtree.h>
@@ -86,7 +85,6 @@ void drm_vma_offset_manager_init(struct drm_vma_offset_manager *mgr,
 				 unsigned long page_offset, unsigned long size)
 {
 	rwlock_init(&mgr->vm_lock);
-	mgr->vm_addr_space_rb = RB_ROOT;
 	drm_mm_init(&mgr->vm_addr_space_mm, page_offset, size);
 }
 EXPORT_SYMBOL(drm_vma_offset_manager_init);
@@ -145,16 +143,16 @@ struct drm_vma_offset_node *drm_vma_offset_lookup_locked(struct drm_vma_offset_m
 							 unsigned long start,
 							 unsigned long pages)
 {
-	struct drm_vma_offset_node *node, *best;
+	struct drm_mm_node *node, *best;
 	struct rb_node *iter;
 	unsigned long offset;
 
-	iter = mgr->vm_addr_space_rb.rb_node;
+	iter = mgr->vm_addr_space_mm.interval_tree.rb_node;
 	best = NULL;
 
 	while (likely(iter)) {
-		node = rb_entry(iter, struct drm_vma_offset_node, vm_rb);
-		offset = node->vm_node.start;
+		node = rb_entry(iter, struct drm_mm_node, rb);
+		offset = node->start;
 		if (start >= offset) {
 			iter = iter->rb_right;
 			best = node;
@@ -167,38 +165,17 @@ struct drm_vma_offset_node *drm_vma_offset_lookup_locked(struct drm_vma_offset_m
 
 	/* verify that the node spans the requested area */
 	if (best) {
-		offset = best->vm_node.start + best->vm_node.size;
+		offset = best->start + best->size;
 		if (offset < start + pages)
 			best = NULL;
 	}
 
-	return best;
-}
-EXPORT_SYMBOL(drm_vma_offset_lookup_locked);
-
-/* internal helper to link @node into the rb-tree */
-static void _drm_vma_offset_add_rb(struct drm_vma_offset_manager *mgr,
-				   struct drm_vma_offset_node *node)
-{
-	struct rb_node **iter = &mgr->vm_addr_space_rb.rb_node;
-	struct rb_node *parent = NULL;
-	struct drm_vma_offset_node *iter_node;
-
-	while (likely(*iter)) {
-		parent = *iter;
-		iter_node = rb_entry(*iter, struct drm_vma_offset_node, vm_rb);
+	if (!best)
+		return NULL;
 
-		if (node->vm_node.start < iter_node->vm_node.start)
-			iter = &(*iter)->rb_left;
-		else if (node->vm_node.start > iter_node->vm_node.start)
-			iter = &(*iter)->rb_right;
-		else
-			BUG();
-	}
-
-	rb_link_node(&node->vm_rb, parent, iter);
-	rb_insert_color(&node->vm_rb, &mgr->vm_addr_space_rb);
+	return container_of(best, struct drm_vma_offset_node, vm_node);
 }
+EXPORT_SYMBOL(drm_vma_offset_lookup_locked);
 
 /**
  * drm_vma_offset_add() - Add offset node to manager
@@ -240,8 +217,6 @@ int drm_vma_offset_add(struct drm_vma_offset_manager *mgr,
 	if (ret)
 		goto out_unlock;
 
-	_drm_vma_offset_add_rb(mgr, node);
-
 out_unlock:
 	write_unlock(&mgr->vm_lock);
 	return ret;
@@ -265,7 +240,6 @@ void drm_vma_offset_remove(struct drm_vma_offset_manager *mgr,
 	write_lock(&mgr->vm_lock);
 
 	if (drm_mm_node_allocated(&node->vm_node)) {
-		rb_erase(&node->vm_rb, &mgr->vm_addr_space_rb);
 		drm_mm_remove_node(&node->vm_node);
 		memset(&node->vm_node, 0, sizeof(node->vm_node));
 	}
@@ -277,9 +251,9 @@ EXPORT_SYMBOL(drm_vma_offset_remove);
 /**
  * drm_vma_node_allow - Add open-file to list of allowed users
  * @node: Node to modify
- * @filp: Open file to add
+ * @tag: Tag of file to remove
  *
- * Add @filp to the list of allowed open-files for this node. If @filp is
+ * Add @tag to the list of allowed open-files for this node. If @tag is
  * already on this list, the ref-count is incremented.
  *
  * The list of allowed-users is preserved across drm_vma_offset_add() and
@@ -294,7 +268,7 @@ EXPORT_SYMBOL(drm_vma_offset_remove);
  * RETURNS:
  * 0 on success, negative error code on internal failure (out-of-mem)
  */
-int drm_vma_node_allow(struct drm_vma_offset_node *node, struct file *filp)
+int drm_vma_node_allow(struct drm_vma_offset_node *node, struct drm_file *tag)
 {
 	struct rb_node **iter;
 	struct rb_node *parent = NULL;
@@ -315,10 +289,10 @@ int drm_vma_node_allow(struct drm_vma_offset_node *node, struct file *filp)
 		parent = *iter;
 		entry = rb_entry(*iter, struct drm_vma_offset_file, vm_rb);
 
-		if (filp == entry->vm_filp) {
+		if (tag == entry->vm_tag) {
 			entry->vm_count++;
 			goto unlock;
-		} else if (filp > entry->vm_filp) {
+		} else if (tag > entry->vm_tag) {
 			iter = &(*iter)->rb_right;
 		} else {
 			iter = &(*iter)->rb_left;
@@ -330,7 +304,7 @@ int drm_vma_node_allow(struct drm_vma_offset_node *node, struct file *filp)
 		goto unlock;
 	}
 
-	new->vm_filp = filp;
+	new->vm_tag = tag;
 	new->vm_count = 1;
 	rb_link_node(&new->vm_rb, parent, iter);
 	rb_insert_color(&new->vm_rb, &node->vm_files);
@@ -346,17 +320,18 @@ EXPORT_SYMBOL(drm_vma_node_allow);
 /**
  * drm_vma_node_revoke - Remove open-file from list of allowed users
  * @node: Node to modify
- * @filp: Open file to remove
+ * @tag: Tag of file to remove
  *
- * Decrement the ref-count of @filp in the list of allowed open-files on @node.
- * If the ref-count drops to zero, remove @filp from the list. You must call
- * this once for every drm_vma_node_allow() on @filp.
+ * Decrement the ref-count of @tag in the list of allowed open-files on @node.
+ * If the ref-count drops to zero, remove @tag from the list. You must call
+ * this once for every drm_vma_node_allow() on @tag.
  *
  * This is locked against concurrent access internally.
  *
- * If @filp is not on the list, nothing is done.
+ * If @tag is not on the list, nothing is done.
  */
-void drm_vma_node_revoke(struct drm_vma_offset_node *node, struct file *filp)
+void drm_vma_node_revoke(struct drm_vma_offset_node *node,
+			 struct drm_file *tag)
 {
 	struct drm_vma_offset_file *entry;
 	struct rb_node *iter;
@@ -366,13 +341,13 @@ void drm_vma_node_revoke(struct drm_vma_offset_node *node, struct file *filp)
 	iter = node->vm_files.rb_node;
 	while (likely(iter)) {
 		entry = rb_entry(iter, struct drm_vma_offset_file, vm_rb);
-		if (filp == entry->vm_filp) {
+		if (tag == entry->vm_tag) {
 			if (!--entry->vm_count) {
 				rb_erase(&entry->vm_rb, &node->vm_files);
 				kfree(entry);
 			}
 			break;
-		} else if (filp > entry->vm_filp) {
+		} else if (tag > entry->vm_tag) {
 			iter = iter->rb_right;
 		} else {
 			iter = iter->rb_left;
@@ -386,9 +361,9 @@ EXPORT_SYMBOL(drm_vma_node_revoke);
 /**
  * drm_vma_node_is_allowed - Check whether an open-file is granted access
  * @node: Node to check
- * @filp: Open-file to check for
+ * @tag: Tag of file to remove
  *
- * Search the list in @node whether @filp is currently on the list of allowed
+ * Search the list in @node whether @tag is currently on the list of allowed
  * open-files (see drm_vma_node_allow()).
  *
  * This is locked against concurrent access internally.
@@ -397,7 +372,7 @@ EXPORT_SYMBOL(drm_vma_node_revoke);
  * true iff @filp is on the list
  */
 bool drm_vma_node_is_allowed(struct drm_vma_offset_node *node,
-			     struct file *filp)
+			     struct drm_file *tag)
 {
 	struct drm_vma_offset_file *entry;
 	struct rb_node *iter;
@@ -407,9 +382,9 @@ bool drm_vma_node_is_allowed(struct drm_vma_offset_node *node,
 	iter = node->vm_files.rb_node;
 	while (likely(iter)) {
 		entry = rb_entry(iter, struct drm_vma_offset_file, vm_rb);
-		if (filp == entry->vm_filp)
+		if (tag == entry->vm_tag)
 			break;
-		else if (filp > entry->vm_filp)
+		else if (tag > entry->vm_tag)
 			iter = iter->rb_right;
 		else
 			iter = iter->rb_left;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
index d8d556457427..cb86c7e5495c 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
@@ -21,6 +21,7 @@
 
 #include "common.xml.h"
 #include "state.xml.h"
+#include "state_hi.xml.h"
 #include "state_3d.xml.h"
 #include "cmdstream.xml.h"
 
@@ -117,11 +118,6 @@ static void etnaviv_cmd_select_pipe(struct etnaviv_gpu *gpu,
 		       VIVS_GL_PIPE_SELECT_PIPE(pipe));
 }
 
-static u32 gpu_va(struct etnaviv_gpu *gpu, struct etnaviv_cmdbuf *buf)
-{
-	return buf->paddr - gpu->memory_base;
-}
-
 static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu,
 	struct etnaviv_cmdbuf *buf, u32 off, u32 len)
 {
@@ -129,7 +125,7 @@ static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu,
 	u32 *ptr = buf->vaddr + off;
 
 	dev_info(gpu->dev, "virt %p phys 0x%08x free 0x%08x\n",
-			ptr, gpu_va(gpu, buf) + off, size - len * 4 - off);
+			ptr, etnaviv_iommu_get_cmdbuf_va(gpu, buf) + off, size - len * 4 - off);
 
 	print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4,
 			ptr, len * 4, 0);
@@ -162,7 +158,7 @@ static u32 etnaviv_buffer_reserve(struct etnaviv_gpu *gpu,
 	if (buffer->user_size + cmd_dwords * sizeof(u64) > buffer->size)
 		buffer->user_size = 0;
 
-	return gpu_va(gpu, buffer) + buffer->user_size;
+	return etnaviv_iommu_get_cmdbuf_va(gpu, buffer) + buffer->user_size;
 }
 
 u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu)
@@ -173,7 +169,41 @@ u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu)
 	buffer->user_size = 0;
 
 	CMD_WAIT(buffer);
-	CMD_LINK(buffer, 2, gpu_va(gpu, buffer) + buffer->user_size - 4);
+	CMD_LINK(buffer, 2, etnaviv_iommu_get_cmdbuf_va(gpu, buffer) +
+		 buffer->user_size - 4);
+
+	return buffer->user_size / 8;
+}
+
+u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe_addr)
+{
+	struct etnaviv_cmdbuf *buffer = gpu->buffer;
+
+	buffer->user_size = 0;
+
+	if (gpu->identity.features & chipFeatures_PIPE_3D) {
+		CMD_LOAD_STATE(buffer, VIVS_GL_PIPE_SELECT,
+			       VIVS_GL_PIPE_SELECT_PIPE(ETNA_PIPE_3D));
+		CMD_LOAD_STATE(buffer, VIVS_MMUv2_CONFIGURATION,
+			mtlb_addr | VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K);
+		CMD_LOAD_STATE(buffer, VIVS_MMUv2_SAFE_ADDRESS, safe_addr);
+		CMD_SEM(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
+		CMD_STALL(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
+	}
+
+	if (gpu->identity.features & chipFeatures_PIPE_2D) {
+		CMD_LOAD_STATE(buffer, VIVS_GL_PIPE_SELECT,
+			       VIVS_GL_PIPE_SELECT_PIPE(ETNA_PIPE_2D));
+		CMD_LOAD_STATE(buffer, VIVS_MMUv2_CONFIGURATION,
+			mtlb_addr | VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K);
+		CMD_LOAD_STATE(buffer, VIVS_MMUv2_SAFE_ADDRESS, safe_addr);
+		CMD_SEM(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
+		CMD_STALL(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
+	}
+
+	CMD_END(buffer);
+
+	buffer->user_size = ALIGN(buffer->user_size, 8);
 
 	return buffer->user_size / 8;
 }
@@ -231,7 +261,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
 	if (drm_debug & DRM_UT_DRIVER)
 		etnaviv_buffer_dump(gpu, buffer, 0, 0x50);
 
-	link_target = gpu_va(gpu, cmdbuf);
+	link_target = etnaviv_iommu_get_cmdbuf_va(gpu, cmdbuf);
 	link_dwords = cmdbuf->size / 8;
 
 	/*
@@ -246,8 +276,12 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
 		extra_dwords = 1;
 
 		/* flush command */
-		if (gpu->mmu->need_flush)
-			extra_dwords += 1;
+		if (gpu->mmu->need_flush) {
+			if (gpu->mmu->version == ETNAVIV_IOMMU_V1)
+				extra_dwords += 1;
+			else
+				extra_dwords += 3;
+		}
 
 		/* pipe switch commands */
 		if (gpu->switch_context)
@@ -257,12 +291,23 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
 
 		if (gpu->mmu->need_flush) {
 			/* Add the MMU flush */
-			CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_MMU,
-				       VIVS_GL_FLUSH_MMU_FLUSH_FEMMU |
-				       VIVS_GL_FLUSH_MMU_FLUSH_UNK1 |
-				       VIVS_GL_FLUSH_MMU_FLUSH_UNK2 |
-				       VIVS_GL_FLUSH_MMU_FLUSH_PEMMU |
-				       VIVS_GL_FLUSH_MMU_FLUSH_UNK4);
+			if (gpu->mmu->version == ETNAVIV_IOMMU_V1) {
+				CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_MMU,
+					       VIVS_GL_FLUSH_MMU_FLUSH_FEMMU |
+					       VIVS_GL_FLUSH_MMU_FLUSH_UNK1 |
+					       VIVS_GL_FLUSH_MMU_FLUSH_UNK2 |
+					       VIVS_GL_FLUSH_MMU_FLUSH_PEMMU |
+					       VIVS_GL_FLUSH_MMU_FLUSH_UNK4);
+			} else {
+				CMD_LOAD_STATE(buffer, VIVS_MMUv2_CONFIGURATION,
+					VIVS_MMUv2_CONFIGURATION_MODE_MASK |
+					VIVS_MMUv2_CONFIGURATION_ADDRESS_MASK |
+					VIVS_MMUv2_CONFIGURATION_FLUSH_FLUSH);
+				CMD_SEM(buffer, SYNC_RECIPIENT_FE,
+					SYNC_RECIPIENT_PE);
+				CMD_STALL(buffer, SYNC_RECIPIENT_FE,
+					SYNC_RECIPIENT_PE);
+			}
 
 			gpu->mmu->need_flush = false;
 		}
@@ -301,7 +346,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
 
 	if (drm_debug & DRM_UT_DRIVER)
 		pr_info("stream link to 0x%08x @ 0x%08x %p\n",
-			return_target, gpu_va(gpu, cmdbuf), cmdbuf->vaddr);
+			return_target, etnaviv_iommu_get_cmdbuf_va(gpu, cmdbuf), cmdbuf->vaddr);
 
 	if (drm_debug & DRM_UT_DRIVER) {
 		print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4,
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index ffd1b32caa8d..aa687669e22b 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -488,8 +488,7 @@ static const struct file_operations fops = {
 };
 
 static struct drm_driver etnaviv_drm_driver = {
-	.driver_features    = DRIVER_HAVE_IRQ |
-				DRIVER_GEM |
+	.driver_features    = DRIVER_GEM |
 				DRIVER_PRIME |
 				DRIVER_RENDER,
 	.open               = etnaviv_open,
@@ -530,10 +529,8 @@ static int etnaviv_bind(struct device *dev)
 	int ret;
 
 	drm = drm_dev_alloc(&etnaviv_drm_driver, dev);
-	if (!drm)
-		return -ENOMEM;
-
-	drm->platformdev = to_platform_device(dev);
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
 
 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index 115c5bc6d7c8..65e057639653 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -96,6 +96,7 @@ struct drm_gem_object *etnaviv_gem_new(struct drm_device *dev,
 int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file,
 	uintptr_t ptr, u32 size, u32 flags, u32 *handle);
 u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu);
+u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe_addr);
 void etnaviv_buffer_end(struct etnaviv_gpu *gpu);
 void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
 	struct etnaviv_cmdbuf *cmdbuf);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
index 4a29eeadbf1e..2bef501d4a17 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
@@ -175,11 +175,13 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu)
 	etnaviv_core_dump_registers(&iter, gpu);
 	etnaviv_core_dump_mmu(&iter, gpu, mmu_size);
 	etnaviv_core_dump_mem(&iter, ETDUMP_BUF_RING, gpu->buffer->vaddr,
-			      gpu->buffer->size, gpu->buffer->paddr);
+			      gpu->buffer->size,
+			      etnaviv_iommu_get_cmdbuf_va(gpu, gpu->buffer));
 
 	list_for_each_entry(cmd, &gpu->active_cmd_list, node)
 		etnaviv_core_dump_mem(&iter, ETDUMP_BUF_CMD, cmd->vaddr,
-				      cmd->size, cmd->paddr);
+				      cmd->size,
+				      etnaviv_iommu_get_cmdbuf_va(gpu, cmd));
 
 	/* Reserve space for the bomap */
 	if (n_bomap_pages) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index 87ef34150d46..b1254f885fed 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -22,8 +22,6 @@
 #include "etnaviv_gpu.h"
 #include "etnaviv_gem.h"
 #include "etnaviv_mmu.h"
-#include "etnaviv_iommu.h"
-#include "etnaviv_iommu_v2.h"
 #include "common.xml.h"
 #include "state.xml.h"
 #include "state_hi.xml.h"
@@ -329,6 +327,18 @@ static void etnaviv_hw_identify(struct etnaviv_gpu *gpu)
 				gpu->identity.revision = 0x1051;
 			}
 		}
+
+		/*
+		 * NXP likes to call the GPU on the i.MX6QP GC2000+, but in
+		 * reality it's just a re-branded GC3000. We can identify this
+		 * core by the upper half of the revision register being all 1.
+		 * Fix model/rev here, so all other places can refer to this
+		 * core by its real identity.
+		 */
+		if (etnaviv_is_model_rev(gpu, GC2000, 0xffff5450)) {
+			gpu->identity.model = chipModel_GC3000;
+			gpu->identity.revision &= 0xffff;
+		}
 	}
 
 	dev_info(gpu->dev, "model: GC%x, revision: %x\n",
@@ -528,6 +538,14 @@ static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu)
 	gpu_write(gpu, VIVS_PM_MODULE_CONTROLS, pmc);
 }
 
+void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch)
+{
+	gpu_write(gpu, VIVS_FE_COMMAND_ADDRESS, address);
+	gpu_write(gpu, VIVS_FE_COMMAND_CONTROL,
+		  VIVS_FE_COMMAND_CONTROL_ENABLE |
+		  VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch));
+}
+
 static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu)
 {
 	u16 prefetch;
@@ -568,33 +586,20 @@ static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu)
 		gpu_write(gpu, VIVS_MC_BUS_CONFIG, bus_config);
 	}
 
-	/* set base addresses */
-	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_RA, gpu->memory_base);
-	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_FE, gpu->memory_base);
-	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_TX, gpu->memory_base);
-	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PEZ, gpu->memory_base);
-	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PE, gpu->memory_base);
-
-	/* setup the MMU page table pointers */
-	etnaviv_iommu_domain_restore(gpu, gpu->mmu->domain);
+	/* setup the MMU */
+	etnaviv_iommu_restore(gpu);
 
 	/* Start command processor */
 	prefetch = etnaviv_buffer_init(gpu);
 
 	gpu_write(gpu, VIVS_HI_INTR_ENBL, ~0U);
-	gpu_write(gpu, VIVS_FE_COMMAND_ADDRESS,
-		  gpu->buffer->paddr - gpu->memory_base);
-	gpu_write(gpu, VIVS_FE_COMMAND_CONTROL,
-		  VIVS_FE_COMMAND_CONTROL_ENABLE |
-		  VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch));
+	etnaviv_gpu_start_fe(gpu, etnaviv_iommu_get_cmdbuf_va(gpu, gpu->buffer),
+			     prefetch);
 }
 
 int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
 {
 	int ret, i;
-	struct iommu_domain *iommu;
-	enum etnaviv_iommu_version version;
-	bool mmuv2;
 
 	ret = pm_runtime_get_sync(gpu->dev);
 	if (ret < 0) {
@@ -642,32 +647,10 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
 		goto fail;
 	}
 
-	/* Setup IOMMU.. eventually we will (I think) do this once per context
-	 * and have separate page tables per context.  For now, to keep things
-	 * simple and to get something working, just use a single address space:
-	 */
-	mmuv2 = gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION;
-	dev_dbg(gpu->dev, "mmuv2: %d\n", mmuv2);
-
-	if (!mmuv2) {
-		iommu = etnaviv_iommu_domain_alloc(gpu);
-		version = ETNAVIV_IOMMU_V1;
-	} else {
-		iommu = etnaviv_iommu_v2_domain_alloc(gpu);
-		version = ETNAVIV_IOMMU_V2;
-	}
-
-	if (!iommu) {
-		dev_err(gpu->dev, "Failed to allocate GPU IOMMU domain\n");
-		ret = -ENOMEM;
-		goto fail;
-	}
-
-	gpu->mmu = etnaviv_iommu_new(gpu, iommu, version);
-	if (!gpu->mmu) {
+	gpu->mmu = etnaviv_iommu_new(gpu);
+	if (IS_ERR(gpu->mmu)) {
 		dev_err(gpu->dev, "Failed to instantiate GPU IOMMU\n");
-		iommu_domain_free(iommu);
-		ret = -ENOMEM;
+		ret = PTR_ERR(gpu->mmu);
 		goto fail;
 	}
 
@@ -678,7 +661,9 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
 		dev_err(gpu->dev, "could not create command buffer\n");
 		goto destroy_iommu;
 	}
-	if (gpu->buffer->paddr - gpu->memory_base > 0x80000000) {
+
+	if (gpu->mmu->version == ETNAVIV_IOMMU_V1 &&
+	    gpu->buffer->paddr - gpu->memory_base > 0x80000000) {
 		ret = -EINVAL;
 		dev_err(gpu->dev,
 			"command buffer outside valid memory window\n");
@@ -868,45 +853,6 @@ int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m)
 #endif
 
 /*
- * Power Management:
- */
-static int enable_clk(struct etnaviv_gpu *gpu)
-{
-	if (gpu->clk_core)
-		clk_prepare_enable(gpu->clk_core);
-	if (gpu->clk_shader)
-		clk_prepare_enable(gpu->clk_shader);
-
-	return 0;
-}
-
-static int disable_clk(struct etnaviv_gpu *gpu)
-{
-	if (gpu->clk_core)
-		clk_disable_unprepare(gpu->clk_core);
-	if (gpu->clk_shader)
-		clk_disable_unprepare(gpu->clk_shader);
-
-	return 0;
-}
-
-static int enable_axi(struct etnaviv_gpu *gpu)
-{
-	if (gpu->clk_bus)
-		clk_prepare_enable(gpu->clk_bus);
-
-	return 0;
-}
-
-static int disable_axi(struct etnaviv_gpu *gpu)
-{
-	if (gpu->clk_bus)
-		clk_disable_unprepare(gpu->clk_bus);
-
-	return 0;
-}
-
-/*
  * Hangcheck detection for locked gpu:
  */
 static void recover_worker(struct work_struct *work)
@@ -945,7 +891,7 @@ static void recover_worker(struct work_struct *work)
 	gpu->completed_fence = gpu->active_fence;
 
 	etnaviv_gpu_hw_init(gpu);
-	gpu->switch_context = true;
+	gpu->lastctx = NULL;
 	gpu->exec_state = -1;
 
 	mutex_unlock(&gpu->lock);
@@ -1178,6 +1124,9 @@ struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu, u32 size,
 	if (!cmdbuf)
 		return NULL;
 
+	if (gpu->mmu->version == ETNAVIV_IOMMU_V2)
+		size = ALIGN(size, SZ_4K);
+
 	cmdbuf->vaddr = dma_alloc_wc(gpu->dev, size, &cmdbuf->paddr,
 				     GFP_KERNEL);
 	if (!cmdbuf->vaddr) {
@@ -1193,6 +1142,7 @@ struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu, u32 size,
 
 void etnaviv_gpu_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf)
 {
+	etnaviv_iommu_put_cmdbuf_va(cmdbuf->gpu, cmdbuf);
 	dma_free_wc(cmdbuf->gpu->dev, cmdbuf->size, cmdbuf->vaddr,
 		    cmdbuf->paddr);
 	kfree(cmdbuf);
@@ -1333,8 +1283,6 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
 	if (ret < 0)
 		return ret;
 
-	mutex_lock(&gpu->lock);
-
 	/*
 	 * TODO
 	 *
@@ -1348,16 +1296,18 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
 	if (unlikely(event == ~0U)) {
 		DRM_ERROR("no free event\n");
 		ret = -EBUSY;
-		goto out_unlock;
+		goto out_pm_put;
 	}
 
 	fence = etnaviv_gpu_fence_alloc(gpu);
 	if (!fence) {
 		event_free(gpu, event);
 		ret = -ENOMEM;
-		goto out_unlock;
+		goto out_pm_put;
 	}
 
+	mutex_lock(&gpu->lock);
+
 	gpu->event[event].fence = fence;
 	submit->fence = fence->seqno;
 	gpu->active_fence = submit->fence;
@@ -1395,9 +1345,9 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
 	hangcheck_timer_reset(gpu);
 	ret = 0;
 
-out_unlock:
 	mutex_unlock(&gpu->lock);
 
+out_pm_put:
 	etnaviv_gpu_pm_put(gpu);
 
 	return ret;
@@ -1425,6 +1375,21 @@ static irqreturn_t irq_handler(int irq, void *data)
 			intr &= ~VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR;
 		}
 
+		if (intr & VIVS_HI_INTR_ACKNOWLEDGE_MMU_EXCEPTION) {
+			int i;
+
+			dev_err_ratelimited(gpu->dev,
+				"MMU fault status 0x%08x\n",
+				gpu_read(gpu, VIVS_MMUv2_STATUS));
+			for (i = 0; i < 4; i++) {
+				dev_err_ratelimited(gpu->dev,
+					"MMU %d fault addr 0x%08x\n",
+					i, gpu_read(gpu,
+					VIVS_MMUv2_EXCEPTION_ADDR(i)));
+			}
+			intr &= ~VIVS_HI_INTR_ACKNOWLEDGE_MMU_EXCEPTION;
+		}
+
 		while ((event = ffs(intr)) != 0) {
 			struct fence *fence;
 
@@ -1466,39 +1431,72 @@ static int etnaviv_gpu_clk_enable(struct etnaviv_gpu *gpu)
 {
 	int ret;
 
-	ret = enable_clk(gpu);
-	if (ret)
-		return ret;
+	if (gpu->clk_bus) {
+		ret = clk_prepare_enable(gpu->clk_bus);
+		if (ret)
+			return ret;
+	}
 
-	ret = enable_axi(gpu);
-	if (ret) {
-		disable_clk(gpu);
-		return ret;
+	if (gpu->clk_core) {
+		ret = clk_prepare_enable(gpu->clk_core);
+		if (ret)
+			goto disable_clk_bus;
+	}
+
+	if (gpu->clk_shader) {
+		ret = clk_prepare_enable(gpu->clk_shader);
+		if (ret)
+			goto disable_clk_core;
 	}
 
 	return 0;
+
+disable_clk_core:
+	if (gpu->clk_core)
+		clk_disable_unprepare(gpu->clk_core);
+disable_clk_bus:
+	if (gpu->clk_bus)
+		clk_disable_unprepare(gpu->clk_bus);
+
+	return ret;
 }
 
 static int etnaviv_gpu_clk_disable(struct etnaviv_gpu *gpu)
 {
-	int ret;
+	if (gpu->clk_shader)
+		clk_disable_unprepare(gpu->clk_shader);
+	if (gpu->clk_core)
+		clk_disable_unprepare(gpu->clk_core);
+	if (gpu->clk_bus)
+		clk_disable_unprepare(gpu->clk_bus);
 
-	ret = disable_axi(gpu);
-	if (ret)
-		return ret;
+	return 0;
+}
 
-	ret = disable_clk(gpu);
-	if (ret)
-		return ret;
+int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 
-	return 0;
+	do {
+		u32 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
+
+		if ((idle & gpu->idle_mask) == gpu->idle_mask)
+			return 0;
+
+		if (time_is_before_jiffies(timeout)) {
+			dev_warn(gpu->dev,
+				 "timed out waiting for idle: idle=0x%x\n",
+				 idle);
+			return -ETIMEDOUT;
+		}
+
+		udelay(5);
+	} while (1);
 }
 
 static int etnaviv_gpu_hw_suspend(struct etnaviv_gpu *gpu)
 {
 	if (gpu->buffer) {
-		unsigned long timeout;
-
 		/* Replace the last WAIT with END */
 		etnaviv_buffer_end(gpu);
 
@@ -1507,22 +1505,7 @@ static int etnaviv_gpu_hw_suspend(struct etnaviv_gpu *gpu)
 		 * happen quickly (as the WAIT is only 200 cycles).  If
 		 * we fail, just warn and continue.
 		 */
-		timeout = jiffies + msecs_to_jiffies(100);
-		do {
-			u32 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
-
-			if ((idle & gpu->idle_mask) == gpu->idle_mask)
-				break;
-
-			if (time_is_before_jiffies(timeout)) {
-				dev_warn(gpu->dev,
-					 "timed out waiting for idle: idle=0x%x\n",
-					 idle);
-				break;
-			}
-
-			udelay(5);
-		} while (1);
+		etnaviv_gpu_wait_idle(gpu, 100);
 	}
 
 	return etnaviv_gpu_clk_disable(gpu);
@@ -1634,7 +1617,7 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct etnaviv_gpu *gpu;
-	int err = 0;
+	int err;
 
 	gpu = devm_kzalloc(dev, sizeof(*gpu), GFP_KERNEL);
 	if (!gpu)
@@ -1651,16 +1634,15 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 	/* Get Interrupt: */
 	gpu->irq = platform_get_irq(pdev, 0);
 	if (gpu->irq < 0) {
-		err = gpu->irq;
-		dev_err(dev, "failed to get irq: %d\n", err);
-		goto fail;
+		dev_err(dev, "failed to get irq: %d\n", gpu->irq);
+		return gpu->irq;
 	}
 
 	err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0,
 			       dev_name(gpu->dev), gpu);
 	if (err) {
 		dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err);
-		goto fail;
+		return err;
 	}
 
 	/* Get Clocks: */
@@ -1694,13 +1676,10 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 	err = component_add(&pdev->dev, &gpu_ops);
 	if (err < 0) {
 		dev_err(&pdev->dev, "failed to register component: %d\n", err);
-		goto fail;
+		return err;
 	}
 
 	return 0;
-
-fail:
-	return err;
 }
 
 static int etnaviv_gpu_platform_remove(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index a69cdd526bf8..73c278dc3706 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -160,6 +160,8 @@ struct etnaviv_cmdbuf {
 	dma_addr_t paddr;
 	u32 size;
 	u32 user_size;
+	/* vram node used if the cmdbuf is mapped through the MMUv2 */
+	struct drm_mm_node vram_node;
 	/* fence after which this buffer is to be disposed */
 	struct fence *fence;
 	/* target exec state */
@@ -214,6 +216,8 @@ struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu,
 void etnaviv_gpu_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf);
 int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu);
 void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu);
+int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms);
+void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch);
 
 extern struct platform_driver etnaviv_gpu_driver;
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
index 16353ee81651..81f1583a7946 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
@@ -196,12 +196,19 @@ static struct etnaviv_iommu_ops etnaviv_iommu_ops = {
 	.dump = etnaviv_iommuv1_dump,
 };
 
-void etnaviv_iommu_domain_restore(struct etnaviv_gpu *gpu,
-	struct iommu_domain *domain)
+void etnaviv_iommuv1_restore(struct etnaviv_gpu *gpu)
 {
-	struct etnaviv_iommu_domain *etnaviv_domain = to_etnaviv_domain(domain);
+	struct etnaviv_iommu_domain *etnaviv_domain =
+			to_etnaviv_domain(gpu->mmu->domain);
 	u32 pgtable;
 
+	/* set base addresses */
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_RA, gpu->memory_base);
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_FE, gpu->memory_base);
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_TX, gpu->memory_base);
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PEZ, gpu->memory_base);
+	gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PE, gpu->memory_base);
+
 	/* set page table address in MC */
 	pgtable = (u32)etnaviv_domain->pgtable.paddr;
 
@@ -212,7 +219,7 @@ void etnaviv_iommu_domain_restore(struct etnaviv_gpu *gpu,
 	gpu_write(gpu, VIVS_MC_MMU_RA_PAGE_TABLE, pgtable);
 }
 
-struct iommu_domain *etnaviv_iommu_domain_alloc(struct etnaviv_gpu *gpu)
+struct iommu_domain *etnaviv_iommuv1_domain_alloc(struct etnaviv_gpu *gpu)
 {
 	struct etnaviv_iommu_domain *etnaviv_domain;
 	int ret;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.h b/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
index cf45503f6b6f..8b51e7c16feb 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.h
@@ -17,12 +17,12 @@
 #ifndef __ETNAVIV_IOMMU_H__
 #define __ETNAVIV_IOMMU_H__
 
-#include <linux/iommu.h>
 struct etnaviv_gpu;
 
-struct iommu_domain *etnaviv_iommu_domain_alloc(struct etnaviv_gpu *gpu);
-void etnaviv_iommu_domain_restore(struct etnaviv_gpu *gpu,
-	struct iommu_domain *domain);
-struct iommu_domain *etnaviv_iommu_v2_domain_alloc(struct etnaviv_gpu *gpu);
+struct iommu_domain *etnaviv_iommuv1_domain_alloc(struct etnaviv_gpu *gpu);
+void etnaviv_iommuv1_restore(struct etnaviv_gpu *gpu);
+
+struct iommu_domain *etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu);
+void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu);
 
 #endif /* __ETNAVIV_IOMMU_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
index fbb4aed3dc80..7e9c4d210a84 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2014 Christian Gmeiner <christian.gmeiner@gmail.com>
+ * Copyright (C) 2016 Etnaviv Project
   *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published by
@@ -22,12 +22,267 @@
 #include <linux/bitops.h>
 
 #include "etnaviv_gpu.h"
+#include "etnaviv_mmu.h"
 #include "etnaviv_iommu.h"
+#include "state.xml.h"
 #include "state_hi.xml.h"
 
+#define MMUv2_PTE_PRESENT		BIT(0)
+#define MMUv2_PTE_EXCEPTION		BIT(1)
+#define MMUv2_PTE_WRITEABLE		BIT(2)
 
-struct iommu_domain *etnaviv_iommu_v2_domain_alloc(struct etnaviv_gpu *gpu)
+#define MMUv2_MTLB_MASK			0xffc00000
+#define MMUv2_MTLB_SHIFT		22
+#define MMUv2_STLB_MASK			0x003ff000
+#define MMUv2_STLB_SHIFT		12
+
+#define MMUv2_MAX_STLB_ENTRIES		1024
+
+struct etnaviv_iommuv2_domain {
+	struct iommu_domain domain;
+	struct device *dev;
+	void *bad_page_cpu;
+	dma_addr_t bad_page_dma;
+	/* M(aster) TLB aka first level pagetable */
+	u32 *mtlb_cpu;
+	dma_addr_t mtlb_dma;
+	/* S(lave) TLB aka second level pagetable */
+	u32 *stlb_cpu[1024];
+	dma_addr_t stlb_dma[1024];
+};
+
+static struct etnaviv_iommuv2_domain *to_etnaviv_domain(struct iommu_domain *domain)
+{
+	return container_of(domain, struct etnaviv_iommuv2_domain, domain);
+}
+
+static int etnaviv_iommuv2_map(struct iommu_domain *domain, unsigned long iova,
+	   phys_addr_t paddr, size_t size, int prot)
+{
+	struct etnaviv_iommuv2_domain *etnaviv_domain =
+			to_etnaviv_domain(domain);
+	int mtlb_entry, stlb_entry;
+	u32 entry = (u32)paddr | MMUv2_PTE_PRESENT;
+
+	if (size != SZ_4K)
+		return -EINVAL;
+
+	if (prot & IOMMU_WRITE)
+		entry |= MMUv2_PTE_WRITEABLE;
+
+	mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
+	stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
+
+	etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry;
+
+	return 0;
+}
+
+static size_t etnaviv_iommuv2_unmap(struct iommu_domain *domain,
+	unsigned long iova, size_t size)
+{
+	struct etnaviv_iommuv2_domain *etnaviv_domain =
+			to_etnaviv_domain(domain);
+	int mtlb_entry, stlb_entry;
+
+	if (size != SZ_4K)
+		return -EINVAL;
+
+	mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
+	stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
+
+	etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION;
+
+	return SZ_4K;
+}
+
+static phys_addr_t etnaviv_iommuv2_iova_to_phys(struct iommu_domain *domain,
+	dma_addr_t iova)
+{
+	struct etnaviv_iommuv2_domain *etnaviv_domain =
+			to_etnaviv_domain(domain);
+	int mtlb_entry, stlb_entry;
+
+	mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
+	stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
+
+	return etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] & ~(SZ_4K - 1);
+}
+
+static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
+{
+	u32 *p;
+	int ret, i, j;
+
+	/* allocate scratch page */
+	etnaviv_domain->bad_page_cpu = dma_alloc_coherent(etnaviv_domain->dev,
+						  SZ_4K,
+						  &etnaviv_domain->bad_page_dma,
+						  GFP_KERNEL);
+	if (!etnaviv_domain->bad_page_cpu) {
+		ret = -ENOMEM;
+		goto fail_mem;
+	}
+	p = etnaviv_domain->bad_page_cpu;
+	for (i = 0; i < SZ_4K / 4; i++)
+		*p++ = 0xdead55aa;
+
+	etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->dev,
+						  SZ_4K,
+						  &etnaviv_domain->mtlb_dma,
+						  GFP_KERNEL);
+	if (!etnaviv_domain->mtlb_cpu) {
+		ret = -ENOMEM;
+		goto fail_mem;
+	}
+
+	/* pre-populate STLB pages (may want to switch to on-demand later) */
+	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
+		etnaviv_domain->stlb_cpu[i] =
+				dma_alloc_coherent(etnaviv_domain->dev,
+						   SZ_4K,
+						   &etnaviv_domain->stlb_dma[i],
+						   GFP_KERNEL);
+		if (!etnaviv_domain->stlb_cpu[i]) {
+			ret = -ENOMEM;
+			goto fail_mem;
+		}
+		p = etnaviv_domain->stlb_cpu[i];
+		for (j = 0; j < SZ_4K / 4; j++)
+			*p++ = MMUv2_PTE_EXCEPTION;
+
+		etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] |
+					      MMUv2_PTE_PRESENT;
+	}
+
+	return 0;
+
+fail_mem:
+	if (etnaviv_domain->bad_page_cpu)
+		dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+				  etnaviv_domain->bad_page_cpu,
+				  etnaviv_domain->bad_page_dma);
+
+	if (etnaviv_domain->mtlb_cpu)
+		dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+				  etnaviv_domain->mtlb_cpu,
+				  etnaviv_domain->mtlb_dma);
+
+	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
+		if (etnaviv_domain->stlb_cpu[i])
+			dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+					  etnaviv_domain->stlb_cpu[i],
+					  etnaviv_domain->stlb_dma[i]);
+	}
+
+	return ret;
+}
+
+static void etnaviv_iommuv2_domain_free(struct iommu_domain *domain)
+{
+	struct etnaviv_iommuv2_domain *etnaviv_domain =
+			to_etnaviv_domain(domain);
+	int i;
+
+	dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+			  etnaviv_domain->bad_page_cpu,
+			  etnaviv_domain->bad_page_dma);
+
+	dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+			  etnaviv_domain->mtlb_cpu,
+			  etnaviv_domain->mtlb_dma);
+
+	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
+		if (etnaviv_domain->stlb_cpu[i])
+			dma_free_coherent(etnaviv_domain->dev, SZ_4K,
+					  etnaviv_domain->stlb_cpu[i],
+					  etnaviv_domain->stlb_dma[i]);
+	}
+
+	vfree(etnaviv_domain);
+}
+
+static size_t etnaviv_iommuv2_dump_size(struct iommu_domain *domain)
+{
+	struct etnaviv_iommuv2_domain *etnaviv_domain =
+			to_etnaviv_domain(domain);
+	size_t dump_size = SZ_4K;
+	int i;
+
+	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
+		if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
+			dump_size += SZ_4K;
+
+	return dump_size;
+}
+
+static void etnaviv_iommuv2_dump(struct iommu_domain *domain, void *buf)
 {
-	/* TODO */
+	struct etnaviv_iommuv2_domain *etnaviv_domain =
+			to_etnaviv_domain(domain);
+	int i;
+
+	memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K);
+	buf += SZ_4K;
+	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K)
+		if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
+			memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K);
+}
+
+static struct etnaviv_iommu_ops etnaviv_iommu_ops = {
+	.ops = {
+		.domain_free = etnaviv_iommuv2_domain_free,
+		.map = etnaviv_iommuv2_map,
+		.unmap = etnaviv_iommuv2_unmap,
+		.iova_to_phys = etnaviv_iommuv2_iova_to_phys,
+		.pgsize_bitmap = SZ_4K,
+	},
+	.dump_size = etnaviv_iommuv2_dump_size,
+	.dump = etnaviv_iommuv2_dump,
+};
+
+void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu)
+{
+	struct etnaviv_iommuv2_domain *etnaviv_domain =
+			to_etnaviv_domain(gpu->mmu->domain);
+	u16 prefetch;
+
+	/* If the MMU is already enabled the state is still there. */
+	if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE)
+		return;
+
+	prefetch = etnaviv_buffer_config_mmuv2(gpu,
+				(u32)etnaviv_domain->mtlb_dma,
+				(u32)etnaviv_domain->bad_page_dma);
+	etnaviv_gpu_start_fe(gpu, gpu->buffer->paddr, prefetch);
+	etnaviv_gpu_wait_idle(gpu, 100);
+
+	gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE);
+}
+struct iommu_domain *etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu)
+{
+	struct etnaviv_iommuv2_domain *etnaviv_domain;
+	int ret;
+
+	etnaviv_domain = vzalloc(sizeof(*etnaviv_domain));
+	if (!etnaviv_domain)
+		return NULL;
+
+	etnaviv_domain->dev = gpu->dev;
+
+	etnaviv_domain->domain.type = __IOMMU_DOMAIN_PAGING;
+	etnaviv_domain->domain.ops = &etnaviv_iommu_ops.ops;
+	etnaviv_domain->domain.pgsize_bitmap = SZ_4K;
+	etnaviv_domain->domain.geometry.aperture_start = 0;
+	etnaviv_domain->domain.geometry.aperture_end = ~0UL & ~(SZ_4K - 1);
+
+	ret = etnaviv_iommuv2_init(etnaviv_domain);
+	if (ret)
+		goto out_free;
+
+	return &etnaviv_domain->domain;
+
+out_free:
+	vfree(etnaviv_domain);
 	return NULL;
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.h b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.h
deleted file mode 100644
index 603ea41c5389..000000000000
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (C) 2014 Christian Gmeiner <christian.gmeiner@gmail.com>
-  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef __ETNAVIV_IOMMU_V2_H__
-#define __ETNAVIV_IOMMU_V2_H__
-
-#include <linux/iommu.h>
-struct etnaviv_gpu;
-
-struct iommu_domain *etnaviv_iommu_v2_domain_alloc(struct etnaviv_gpu *gpu);
-
-#endif /* __ETNAVIV_IOMMU_V2_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
index 29a723fabc17..d3796ed8d8c5 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
@@ -14,9 +14,11 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "common.xml.h"
 #include "etnaviv_drv.h"
 #include "etnaviv_gem.h"
 #include "etnaviv_gpu.h"
+#include "etnaviv_iommu.h"
 #include "etnaviv_mmu.h"
 
 static int etnaviv_fault_handler(struct iommu_domain *iommu, struct device *dev,
@@ -101,40 +103,21 @@ static void etnaviv_iommu_remove_mapping(struct etnaviv_iommu *mmu,
 	drm_mm_remove_node(&mapping->vram_node);
 }
 
-int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
-	struct etnaviv_gem_object *etnaviv_obj, u32 memory_base,
-	struct etnaviv_vram_mapping *mapping)
+static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu,
+				   struct drm_mm_node *node, size_t size)
 {
 	struct etnaviv_vram_mapping *free = NULL;
-	struct sg_table *sgt = etnaviv_obj->sgt;
-	struct drm_mm_node *node;
 	int ret;
 
-	lockdep_assert_held(&etnaviv_obj->lock);
-
-	mutex_lock(&mmu->lock);
-
-	/* v1 MMU can optimize single entry (contiguous) scatterlists */
-	if (sgt->nents == 1 && !(etnaviv_obj->flags & ETNA_BO_FORCE_MMU)) {
-		u32 iova;
-
-		iova = sg_dma_address(sgt->sgl) - memory_base;
-		if (iova < 0x80000000 - sg_dma_len(sgt->sgl)) {
-			mapping->iova = iova;
-			list_add_tail(&mapping->mmu_node, &mmu->mappings);
-			mutex_unlock(&mmu->lock);
-			return 0;
-		}
-	}
+	lockdep_assert_held(&mmu->lock);
 
-	node = &mapping->vram_node;
 	while (1) {
 		struct etnaviv_vram_mapping *m, *n;
 		struct list_head list;
 		bool found;
 
 		ret = drm_mm_insert_node_in_range(&mmu->mm, node,
-			etnaviv_obj->base.size, 0, mmu->last_iova, ~0UL,
+			size, 0, mmu->last_iova, ~0UL,
 			DRM_MM_SEARCH_DEFAULT);
 
 		if (ret != -ENOSPC)
@@ -151,7 +134,7 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
 		}
 
 		/* Try to retire some entries */
-		drm_mm_init_scan(&mmu->mm, etnaviv_obj->base.size, 0, 0);
+		drm_mm_init_scan(&mmu->mm, size, 0, 0);
 
 		found = 0;
 		INIT_LIST_HEAD(&list);
@@ -212,6 +195,38 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
 		mmu->need_flush = true;
 	}
 
+	return ret;
+}
+
+int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
+	struct etnaviv_gem_object *etnaviv_obj, u32 memory_base,
+	struct etnaviv_vram_mapping *mapping)
+{
+	struct sg_table *sgt = etnaviv_obj->sgt;
+	struct drm_mm_node *node;
+	int ret;
+
+	lockdep_assert_held(&etnaviv_obj->lock);
+
+	mutex_lock(&mmu->lock);
+
+	/* v1 MMU can optimize single entry (contiguous) scatterlists */
+	if (mmu->version == ETNAVIV_IOMMU_V1 &&
+	    sgt->nents == 1 && !(etnaviv_obj->flags & ETNA_BO_FORCE_MMU)) {
+		u32 iova;
+
+		iova = sg_dma_address(sgt->sgl) - memory_base;
+		if (iova < 0x80000000 - sg_dma_len(sgt->sgl)) {
+			mapping->iova = iova;
+			list_add_tail(&mapping->mmu_node, &mmu->mappings);
+			mutex_unlock(&mmu->lock);
+			return 0;
+		}
+	}
+
+	node = &mapping->vram_node;
+
+	ret = etnaviv_iommu_find_iova(mmu, node, etnaviv_obj->base.size);
 	if (ret < 0) {
 		mutex_unlock(&mmu->lock);
 		return ret;
@@ -256,30 +271,102 @@ void etnaviv_iommu_destroy(struct etnaviv_iommu *mmu)
 	kfree(mmu);
 }
 
-struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu,
-	struct iommu_domain *domain, enum etnaviv_iommu_version version)
+struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu)
 {
+	enum etnaviv_iommu_version version;
 	struct etnaviv_iommu *mmu;
 
 	mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);
 	if (!mmu)
 		return ERR_PTR(-ENOMEM);
 
-	mmu->domain = domain;
+	if (!(gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION)) {
+		mmu->domain = etnaviv_iommuv1_domain_alloc(gpu);
+		version = ETNAVIV_IOMMU_V1;
+	} else {
+		mmu->domain = etnaviv_iommuv2_domain_alloc(gpu);
+		version = ETNAVIV_IOMMU_V2;
+	}
+
+	if (!mmu->domain) {
+		dev_err(gpu->dev, "Failed to allocate GPU IOMMU domain\n");
+		kfree(mmu);
+		return ERR_PTR(-ENOMEM);
+	}
+
 	mmu->gpu = gpu;
 	mmu->version = version;
 	mutex_init(&mmu->lock);
 	INIT_LIST_HEAD(&mmu->mappings);
 
-	drm_mm_init(&mmu->mm, domain->geometry.aperture_start,
-		    domain->geometry.aperture_end -
-		      domain->geometry.aperture_start + 1);
+	drm_mm_init(&mmu->mm, mmu->domain->geometry.aperture_start,
+		    mmu->domain->geometry.aperture_end -
+		    mmu->domain->geometry.aperture_start + 1);
 
-	iommu_set_fault_handler(domain, etnaviv_fault_handler, gpu->dev);
+	iommu_set_fault_handler(mmu->domain, etnaviv_fault_handler, gpu->dev);
 
 	return mmu;
 }
 
+void etnaviv_iommu_restore(struct etnaviv_gpu *gpu)
+{
+	if (gpu->mmu->version == ETNAVIV_IOMMU_V1)
+		etnaviv_iommuv1_restore(gpu);
+	else
+		etnaviv_iommuv2_restore(gpu);
+}
+
+u32 etnaviv_iommu_get_cmdbuf_va(struct etnaviv_gpu *gpu,
+				struct etnaviv_cmdbuf *buf)
+{
+	struct etnaviv_iommu *mmu = gpu->mmu;
+
+	if (mmu->version == ETNAVIV_IOMMU_V1) {
+		return buf->paddr - gpu->memory_base;
+	} else {
+		int ret;
+
+		if (buf->vram_node.allocated)
+			return (u32)buf->vram_node.start;
+
+		mutex_lock(&mmu->lock);
+		ret = etnaviv_iommu_find_iova(mmu, &buf->vram_node, buf->size);
+		if (ret < 0) {
+			mutex_unlock(&mmu->lock);
+			return 0;
+		}
+		ret = iommu_map(mmu->domain, buf->vram_node.start, buf->paddr,
+				buf->size, IOMMU_READ);
+		if (ret < 0) {
+			drm_mm_remove_node(&buf->vram_node);
+			mutex_unlock(&mmu->lock);
+			return 0;
+		}
+		/*
+		 * At least on GC3000 the FE MMU doesn't properly flush old TLB
+		 * entries. Make sure to space the command buffers out in a way
+		 * that the FE MMU prefetch won't load invalid entries.
+		 */
+		mmu->last_iova = buf->vram_node.start + buf->size + SZ_64K;
+		gpu->mmu->need_flush = true;
+		mutex_unlock(&mmu->lock);
+
+		return (u32)buf->vram_node.start;
+	}
+}
+
+void etnaviv_iommu_put_cmdbuf_va(struct etnaviv_gpu *gpu,
+				 struct etnaviv_cmdbuf *buf)
+{
+	struct etnaviv_iommu *mmu = gpu->mmu;
+
+	if (mmu->version == ETNAVIV_IOMMU_V2 && buf->vram_node.allocated) {
+		mutex_lock(&mmu->lock);
+		iommu_unmap(mmu->domain, buf->vram_node.start, buf->size);
+		drm_mm_remove_node(&buf->vram_node);
+		mutex_unlock(&mmu->lock);
+	}
+}
 size_t etnaviv_iommu_dump_size(struct etnaviv_iommu *iommu)
 {
 	struct etnaviv_iommu_ops *ops;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h
index fff215a47630..e787e49c9693 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h
@@ -62,10 +62,15 @@ void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu,
 	struct etnaviv_vram_mapping *mapping);
 void etnaviv_iommu_destroy(struct etnaviv_iommu *iommu);
 
+u32 etnaviv_iommu_get_cmdbuf_va(struct etnaviv_gpu *gpu,
+				struct etnaviv_cmdbuf *buf);
+void etnaviv_iommu_put_cmdbuf_va(struct etnaviv_gpu *gpu,
+				 struct etnaviv_cmdbuf *buf);
+
 size_t etnaviv_iommu_dump_size(struct etnaviv_iommu *iommu);
 void etnaviv_iommu_dump(struct etnaviv_iommu *iommu, void *buf);
 
-struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu,
-	struct iommu_domain *domain, enum etnaviv_iommu_version version);
+struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu);
+void etnaviv_iommu_restore(struct etnaviv_gpu *gpu);
 
 #endif /* __ETNAVIV_MMU_H__ */
diff --git a/drivers/gpu/drm/etnaviv/state_hi.xml.h b/drivers/gpu/drm/etnaviv/state_hi.xml.h
index 807a3d9e0dd5..43c73e2ed34f 100644
--- a/drivers/gpu/drm/etnaviv/state_hi.xml.h
+++ b/drivers/gpu/drm/etnaviv/state_hi.xml.h
@@ -8,10 +8,10 @@ http://0x04.net/cgit/index.cgi/rules-ng-ng
 git clone git://0x04.net/rules-ng-ng
 
 The rules-ng-ng source files this header was generated from are:
-- state_hi.xml (  24309 bytes, from 2015-12-12 09:02:53)
-- common.xml   (  18437 bytes, from 2015-12-12 09:02:53)
+- state_hi.xml (  25620 bytes, from 2016-08-19 22:07:37)
+- common.xml   (  20583 bytes, from 2016-06-07 05:22:38)
 
-Copyright (C) 2015
+Copyright (C) 2016
 */
 
 
@@ -78,9 +78,10 @@ Copyright (C) 2015
 #define VIVS_HI_AXI_STATUS_DET_RD_ERR				0x00000200
 
 #define VIVS_HI_INTR_ACKNOWLEDGE				0x00000010
-#define VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC__MASK			0x7fffffff
+#define VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC__MASK			0x3fffffff
 #define VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC__SHIFT		0
 #define VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC(x)			(((x) << VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC__SHIFT) & VIVS_HI_INTR_ACKNOWLEDGE_INTR_VEC__MASK)
+#define VIVS_HI_INTR_ACKNOWLEDGE_MMU_EXCEPTION			0x40000000
 #define VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR			0x80000000
 
 #define VIVS_HI_INTR_ENBL					0x00000014
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index 83f61c513b7e..465d344f3391 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -38,7 +38,6 @@ config DRM_EXYNOS7_DECON
 
 config DRM_EXYNOS_MIXER
 	bool "Mixer"
-	depends on !VIDEO_SAMSUNG_S5P_TV
 	help
 	  Choose this option if you want to use Exynos Mixer for DRM.
 
@@ -77,7 +76,7 @@ config DRM_EXYNOS_DP
 
 config DRM_EXYNOS_HDMI
 	bool "HDMI"
-	depends on !VIDEO_SAMSUNG_S5P_TV && (DRM_EXYNOS_MIXER || DRM_EXYNOS5433_DECON)
+	depends on DRM_EXYNOS_MIXER || DRM_EXYNOS5433_DECON
 	help
 	  Choose this option if you want to use Exynos HDMI for DRM.
 
diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
index ac21b4000835..6ca1f3117fe8 100644
--- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
@@ -551,7 +551,6 @@ static irqreturn_t decon_irq_handler(int irq, void *dev_id)
 {
 	struct decon_context *ctx = dev_id;
 	u32 val;
-	int win;
 
 	if (!test_bit(BIT_CLKS_ENABLED, &ctx->flags))
 		goto out;
@@ -560,16 +559,6 @@ static irqreturn_t decon_irq_handler(int irq, void *dev_id)
 	val &= VIDINTCON1_INTFRMDONEPEND | VIDINTCON1_INTFRMPEND;
 
 	if (val) {
-		for (win = ctx->first_win; win < WINDOWS_NR ; win++) {
-			struct exynos_drm_plane *plane = &ctx->planes[win];
-
-			if (!plane->pending_fb)
-				continue;
-
-			exynos_drm_crtc_finish_update(ctx->crtc, plane);
-		}
-
-		/* clear */
 		writel(val, ctx->addr + DECON_VIDINTCON1);
 		drm_crtc_handle_vblank(&ctx->crtc->base);
 	}
diff --git a/drivers/gpu/drm/exynos/exynos7_drm_decon.c b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
index 7f9901b7777b..f4d5a2133777 100644
--- a/drivers/gpu/drm/exynos/exynos7_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
@@ -603,7 +603,6 @@ static irqreturn_t decon_irq_handler(int irq, void *dev_id)
 {
 	struct decon_context *ctx = (struct decon_context *)dev_id;
 	u32 val, clear_bit;
-	int win;
 
 	val = readl(ctx->regs + VIDINTCON1);
 
@@ -617,14 +616,6 @@ static irqreturn_t decon_irq_handler(int irq, void *dev_id)
 
 	if (!ctx->i80_if) {
 		drm_crtc_handle_vblank(&ctx->crtc->base);
-		for (win = 0 ; win < WINDOWS_NR ; win++) {
-			struct exynos_drm_plane *plane = &ctx->planes[win];
-
-			if (!plane->pending_fb)
-				continue;
-
-			exynos_drm_crtc_finish_update(ctx->crtc, plane);
-		}
 
 		/* set wait vsync event to zero and wake up queue. */
 		if (atomic_read(&ctx->wait_vsync_event)) {
diff --git a/drivers/gpu/drm/exynos/exynos_dp.c b/drivers/gpu/drm/exynos/exynos_dp.c
index 4f0850585b8e..528229faffe4 100644
--- a/drivers/gpu/drm/exynos/exynos_dp.c
+++ b/drivers/gpu/drm/exynos/exynos_dp.c
@@ -43,7 +43,7 @@ struct exynos_dp_device {
 	struct analogix_dp_plat_data plat_data;
 };
 
-int exynos_dp_crtc_clock_enable(struct analogix_dp_plat_data *plat_data,
+static int exynos_dp_crtc_clock_enable(struct analogix_dp_plat_data *plat_data,
 				bool enable)
 {
 	struct exynos_dp_device *dp = to_dp(plat_data);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
index 785ffa6cc309..2530bf57716a 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
@@ -69,8 +69,6 @@ static void exynos_crtc_atomic_begin(struct drm_crtc *crtc,
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
-	exynos_crtc->event = crtc->state->event;
-
 	if (exynos_crtc->ops->atomic_begin)
 		exynos_crtc->ops->atomic_begin(exynos_crtc);
 }
@@ -79,9 +77,24 @@ static void exynos_crtc_atomic_flush(struct drm_crtc *crtc,
 				     struct drm_crtc_state *old_crtc_state)
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
+	struct drm_pending_vblank_event *event;
+	unsigned long flags;
 
 	if (exynos_crtc->ops->atomic_flush)
 		exynos_crtc->ops->atomic_flush(exynos_crtc);
+
+	event = crtc->state->event;
+	if (event) {
+		crtc->state->event = NULL;
+
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		if (drm_crtc_vblank_get(crtc) == 0)
+			drm_crtc_arm_vblank_event(crtc, event);
+		else
+			drm_crtc_send_vblank_event(crtc, event);
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+	}
+
 }
 
 static const struct drm_crtc_helper_funcs exynos_crtc_helper_funcs = {
@@ -134,8 +147,6 @@ struct exynos_drm_crtc *exynos_drm_crtc_create(struct drm_device *drm_dev,
 	exynos_crtc->ops = ops;
 	exynos_crtc->ctx = ctx;
 
-	init_waitqueue_head(&exynos_crtc->wait_update);
-
 	crtc = &exynos_crtc->base;
 
 	private->crtc[pipe] = crtc;
@@ -175,32 +186,6 @@ void exynos_drm_crtc_disable_vblank(struct drm_device *dev, unsigned int pipe)
 		exynos_crtc->ops->disable_vblank(exynos_crtc);
 }
 
-void exynos_drm_crtc_wait_pending_update(struct exynos_drm_crtc *exynos_crtc)
-{
-	wait_event_timeout(exynos_crtc->wait_update,
-			   (atomic_read(&exynos_crtc->pending_update) == 0),
-			   msecs_to_jiffies(50));
-}
-
-void exynos_drm_crtc_finish_update(struct exynos_drm_crtc *exynos_crtc,
-				struct exynos_drm_plane *exynos_plane)
-{
-	struct drm_crtc *crtc = &exynos_crtc->base;
-	unsigned long flags;
-
-	exynos_plane->pending_fb = NULL;
-
-	if (atomic_dec_and_test(&exynos_crtc->pending_update))
-		wake_up(&exynos_crtc->wait_update);
-
-	spin_lock_irqsave(&crtc->dev->event_lock, flags);
-	if (exynos_crtc->event)
-		drm_crtc_send_vblank_event(crtc, exynos_crtc->event);
-
-	exynos_crtc->event = NULL;
-	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
-}
-
 int exynos_drm_crtc_get_pipe_from_type(struct drm_device *drm_dev,
 				       enum exynos_drm_output_type out_type)
 {
@@ -228,20 +213,19 @@ void exynos_drm_crtc_te_handler(struct drm_crtc *crtc)
 void exynos_drm_crtc_cancel_page_flip(struct drm_crtc *crtc,
 					struct drm_file *file)
 {
-	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 	struct drm_pending_vblank_event *e;
 	unsigned long flags;
 
 	spin_lock_irqsave(&crtc->dev->event_lock, flags);
 
-	e = exynos_crtc->event;
-	if (e && e->base.file_priv == file) {
-		exynos_crtc->event = NULL;
-		atomic_dec(&exynos_crtc->pending_update);
-	}
+	e = crtc->state->event;
+	if (e && e->base.file_priv == file)
+		crtc->state->event = NULL;
+	else
+		e = NULL;
 
 	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
 
-	if (e && e->base.file_priv == file)
+	if (e)
 		drm_event_cancel_free(crtc->dev, &e->base);
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index 877d2efa28e2..def78c8c1780 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -45,37 +45,11 @@ struct exynos_atomic_commit {
 	u32			crtcs;
 };
 
-static void exynos_atomic_wait_for_commit(struct drm_atomic_state *state)
-{
-	struct drm_crtc_state *crtc_state;
-	struct drm_crtc *crtc;
-	int i, ret;
-
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
-
-		if (!crtc->state->enable)
-			continue;
-
-		ret = drm_crtc_vblank_get(crtc);
-		if (ret)
-			continue;
-
-		exynos_drm_crtc_wait_pending_update(exynos_crtc);
-		drm_crtc_vblank_put(crtc);
-	}
-}
-
 static void exynos_atomic_commit_complete(struct exynos_atomic_commit *commit)
 {
 	struct drm_device *dev = commit->dev;
 	struct exynos_drm_private *priv = dev->dev_private;
 	struct drm_atomic_state *state = commit->state;
-	struct drm_plane *plane;
-	struct drm_crtc *crtc;
-	struct drm_plane_state *plane_state;
-	struct drm_crtc_state *crtc_state;
-	int i;
 
 	drm_atomic_helper_commit_modeset_disables(dev, state);
 
@@ -89,25 +63,9 @@ static void exynos_atomic_commit_complete(struct exynos_atomic_commit *commit)
 	 * have the relevant clocks enabled to perform the update.
 	 */
 
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
-
-		atomic_set(&exynos_crtc->pending_update, 0);
-	}
-
-	for_each_plane_in_state(state, plane, plane_state, i) {
-		struct exynos_drm_crtc *exynos_crtc =
-						to_exynos_crtc(plane->crtc);
-
-		if (!plane->crtc)
-			continue;
-
-		atomic_inc(&exynos_crtc->pending_update);
-	}
-
-	drm_atomic_helper_commit_planes(dev, state, false);
+	drm_atomic_helper_commit_planes(dev, state, 0);
 
-	exynos_atomic_wait_for_commit(state);
+	drm_atomic_helper_wait_for_vblanks(dev, state);
 
 	drm_atomic_helper_cleanup_planes(dev, state);
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index 7f1a49d5bdbe..d215149e737b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -86,7 +86,6 @@ struct exynos_drm_plane {
 	struct drm_plane base;
 	const struct exynos_drm_plane_config *config;
 	unsigned int index;
-	struct drm_framebuffer *pending_fb;
 };
 
 #define EXYNOS_DRM_PLANE_CAP_DOUBLE	(1 << 0)
@@ -172,9 +171,6 @@ struct exynos_drm_crtc {
 	struct drm_crtc			base;
 	enum exynos_drm_output_type	type;
 	unsigned int			pipe;
-	struct drm_pending_vblank_event	*event;
-	wait_queue_head_t		wait_update;
-	atomic_t			pending_update;
 	const struct exynos_drm_crtc_ops	*ops;
 	void				*ctx;
 	struct exynos_drm_clk		*pipe_clk;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index e0166403b4bd..40ce841eb952 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -55,11 +55,11 @@ static int check_fb_gem_memory_type(struct drm_device *drm_dev,
 	flags = exynos_gem->flags;
 
 	/*
-	 * without iommu support, not support physically non-continuous memory
-	 * for framebuffer.
+	 * Physically non-contiguous memory type for framebuffer is not
+	 * supported without IOMMU.
 	 */
 	if (IS_NONCONTIG_BUFFER(flags)) {
-		DRM_ERROR("cannot use this gem memory type for fb.\n");
+		DRM_ERROR("Non-contiguous GEM memory is not supported.\n");
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimc.c b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
index 0525c56145db..147ef0d298cb 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
@@ -1753,32 +1753,6 @@ static int fimc_clk_ctrl(struct fimc_context *ctx, bool enable)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int fimc_suspend(struct device *dev)
-{
-	struct fimc_context *ctx = get_fimc_context(dev);
-
-	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
-
-	if (pm_runtime_suspended(dev))
-		return 0;
-
-	return fimc_clk_ctrl(ctx, false);
-}
-
-static int fimc_resume(struct device *dev)
-{
-	struct fimc_context *ctx = get_fimc_context(dev);
-
-	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
-
-	if (!pm_runtime_suspended(dev))
-		return fimc_clk_ctrl(ctx, true);
-
-	return 0;
-}
-#endif
-
 static int fimc_runtime_suspend(struct device *dev)
 {
 	struct fimc_context *ctx = get_fimc_context(dev);
@@ -1799,7 +1773,8 @@ static int fimc_runtime_resume(struct device *dev)
 #endif
 
 static const struct dev_pm_ops fimc_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(fimc_suspend, fimc_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 	SET_RUNTIME_PM_OPS(fimc_runtime_suspend, fimc_runtime_resume, NULL)
 };
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index d47216488985..e2e405170d35 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -198,6 +198,7 @@ struct fimd_context {
 	atomic_t			wait_vsync_event;
 	atomic_t			win_updated;
 	atomic_t			triggering;
+	u32				clkdiv;
 
 	const struct fimd_driver_data *driver_data;
 	struct drm_encoder *encoder;
@@ -389,15 +390,18 @@ static void fimd_clear_channels(struct exynos_drm_crtc *crtc)
 	pm_runtime_put(ctx->dev);
 }
 
-static u32 fimd_calc_clkdiv(struct fimd_context *ctx,
-		const struct drm_display_mode *mode)
+
+static int fimd_atomic_check(struct exynos_drm_crtc *crtc,
+		struct drm_crtc_state *state)
 {
-	unsigned long ideal_clk;
+	struct drm_display_mode *mode = &state->adjusted_mode;
+	struct fimd_context *ctx = crtc->ctx;
+	unsigned long ideal_clk, lcd_rate;
 	u32 clkdiv;
 
 	if (mode->clock == 0) {
-		DRM_ERROR("Mode has zero clock value.\n");
-		return 0xff;
+		DRM_INFO("Mode has zero clock value.\n");
+		return -EINVAL;
 	}
 
 	ideal_clk = mode->clock * 1000;
@@ -410,10 +414,23 @@ static u32 fimd_calc_clkdiv(struct fimd_context *ctx,
 		ideal_clk *= 2;
 	}
 
+	lcd_rate = clk_get_rate(ctx->lcd_clk);
+	if (2 * lcd_rate < ideal_clk) {
+		DRM_INFO("sclk_fimd clock too low(%lu) for requested pixel clock(%lu)\n",
+			 lcd_rate, ideal_clk);
+		return -EINVAL;
+	}
+
 	/* Find the clock divider value that gets us closest to ideal_clk */
-	clkdiv = DIV_ROUND_CLOSEST(clk_get_rate(ctx->lcd_clk), ideal_clk);
+	clkdiv = DIV_ROUND_CLOSEST(lcd_rate, ideal_clk);
+	if (clkdiv >= 0x200) {
+		DRM_INFO("requested pixel clock(%lu) too low\n", ideal_clk);
+		return -EINVAL;
+	}
+
+	ctx->clkdiv = (clkdiv < 0x100) ? clkdiv : 0xff;
 
-	return (clkdiv < 0x100) ? clkdiv : 0xff;
+	return 0;
 }
 
 static void fimd_setup_trigger(struct fimd_context *ctx)
@@ -442,7 +459,7 @@ static void fimd_commit(struct exynos_drm_crtc *crtc)
 	struct drm_display_mode *mode = &crtc->base.state->adjusted_mode;
 	const struct fimd_driver_data *driver_data = ctx->driver_data;
 	void *timing_base = ctx->regs + driver_data->timing_base;
-	u32 val, clkdiv;
+	u32 val;
 
 	if (ctx->suspended)
 		return;
@@ -543,9 +560,8 @@ static void fimd_commit(struct exynos_drm_crtc *crtc)
 	if (ctx->driver_data->has_clksel)
 		val |= VIDCON0_CLKSEL_LCD;
 
-	clkdiv = fimd_calc_clkdiv(ctx, mode);
-	if (clkdiv > 1)
-		val |= VIDCON0_CLKVAL_F(clkdiv - 1) | VIDCON0_CLKDIR;
+	if (ctx->clkdiv > 1)
+		val |= VIDCON0_CLKVAL_F(ctx->clkdiv - 1) | VIDCON0_CLKDIR;
 
 	writel(val, ctx->regs + VIDCON0);
 }
@@ -939,14 +955,14 @@ static const struct exynos_drm_crtc_ops fimd_crtc_ops = {
 	.update_plane = fimd_update_plane,
 	.disable_plane = fimd_disable_plane,
 	.atomic_flush = fimd_atomic_flush,
+	.atomic_check = fimd_atomic_check,
 	.te_handler = fimd_te_handler,
 };
 
 static irqreturn_t fimd_irq_handler(int irq, void *dev_id)
 {
 	struct fimd_context *ctx = (struct fimd_context *)dev_id;
-	u32 val, clear_bit, start, start_s;
-	int win;
+	u32 val, clear_bit;
 
 	val = readl(ctx->regs + VIDINTCON1);
 
@@ -961,18 +977,6 @@ static irqreturn_t fimd_irq_handler(int irq, void *dev_id)
 	if (!ctx->i80_if)
 		drm_crtc_handle_vblank(&ctx->crtc->base);
 
-	for (win = 0 ; win < WINDOWS_NR ; win++) {
-		struct exynos_drm_plane *plane = &ctx->planes[win];
-
-		if (!plane->pending_fb)
-			continue;
-
-		start = readl(ctx->regs + VIDWx_BUF_START(win, 0));
-		start_s = readl(ctx->regs + VIDWx_BUF_START_S(win, 0));
-		if (start == start_s)
-			exynos_drm_crtc_finish_update(ctx->crtc, plane);
-	}
-
 	if (ctx->i80_if) {
 		/* Exits triggering mode */
 		atomic_set(&ctx->triggering, 0);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
index 4bf00f57ffe8..aa92decf4233 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
@@ -138,6 +138,18 @@ enum g2d_reg_type {
 	MAX_REG_TYPE_NR
 };
 
+enum g2d_flag_bits {
+	/*
+	 * If set, suspends the runqueue worker after the currently
+	 * processed node is finished.
+	 */
+	G2D_BIT_SUSPEND_RUNQUEUE,
+	/*
+	 * If set, indicates that the engine is currently busy.
+	 */
+	G2D_BIT_ENGINE_BUSY,
+};
+
 /* cmdlist data structure */
 struct g2d_cmdlist {
 	u32		head;
@@ -226,7 +238,7 @@ struct g2d_data {
 	struct workqueue_struct		*g2d_workq;
 	struct work_struct		runqueue_work;
 	struct exynos_drm_subdrv	subdrv;
-	bool				suspended;
+	unsigned long			flags;
 
 	/* cmdlist */
 	struct g2d_cmdlist_node		*cmdlist_node;
@@ -246,6 +258,12 @@ struct g2d_data {
 	unsigned long			max_pool;
 };
 
+static inline void g2d_hw_reset(struct g2d_data *g2d)
+{
+	writel(G2D_R | G2D_SFRCLEAR, g2d->regs + G2D_SOFT_RESET);
+	clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
+}
+
 static int g2d_init_cmdlist(struct g2d_data *g2d)
 {
 	struct device *dev = g2d->dev;
@@ -803,12 +821,8 @@ static void g2d_dma_start(struct g2d_data *g2d,
 	struct g2d_cmdlist_node *node =
 				list_first_entry(&runqueue_node->run_cmdlist,
 						struct g2d_cmdlist_node, list);
-	int ret;
-
-	ret = pm_runtime_get_sync(g2d->dev);
-	if (ret < 0)
-		return;
 
+	set_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
 	writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
 	writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
 }
@@ -831,9 +845,6 @@ static void g2d_free_runqueue_node(struct g2d_data *g2d,
 {
 	struct g2d_cmdlist_node *node;
 
-	if (!runqueue_node)
-		return;
-
 	mutex_lock(&g2d->cmdlist_mutex);
 	/*
 	 * commands in run_cmdlist have been completed so unmap all gem
@@ -847,29 +858,65 @@ static void g2d_free_runqueue_node(struct g2d_data *g2d,
 	kmem_cache_free(g2d->runqueue_slab, runqueue_node);
 }
 
-static void g2d_exec_runqueue(struct g2d_data *g2d)
+/**
+ * g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes
+ * @g2d: G2D state object
+ * @file: if not zero, only remove items with this DRM file
+ *
+ * Has to be called under runqueue lock.
+ */
+static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file* file)
 {
-	g2d->runqueue_node = g2d_get_runqueue_node(g2d);
-	if (g2d->runqueue_node)
-		g2d_dma_start(g2d, g2d->runqueue_node);
+	struct g2d_runqueue_node *node, *n;
+
+	if (list_empty(&g2d->runqueue))
+		return;
+
+	list_for_each_entry_safe(node, n, &g2d->runqueue, list) {
+		if (file && node->filp != file)
+			continue;
+
+		list_del_init(&node->list);
+		g2d_free_runqueue_node(g2d, node);
+	}
 }
 
 static void g2d_runqueue_worker(struct work_struct *work)
 {
 	struct g2d_data *g2d = container_of(work, struct g2d_data,
 					    runqueue_work);
+	struct g2d_runqueue_node *runqueue_node;
+
+	/*
+	 * The engine is busy and the completion of the current node is going
+	 * to poke the runqueue worker, so nothing to do here.
+	 */
+	if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags))
+		return;
 
 	mutex_lock(&g2d->runqueue_mutex);
-	pm_runtime_put_sync(g2d->dev);
 
-	complete(&g2d->runqueue_node->complete);
-	if (g2d->runqueue_node->async)
-		g2d_free_runqueue_node(g2d, g2d->runqueue_node);
+	runqueue_node = g2d->runqueue_node;
+	g2d->runqueue_node = NULL;
+
+	if (runqueue_node) {
+		pm_runtime_mark_last_busy(g2d->dev);
+		pm_runtime_put_autosuspend(g2d->dev);
+
+		complete(&runqueue_node->complete);
+		if (runqueue_node->async)
+			g2d_free_runqueue_node(g2d, runqueue_node);
+	}
+
+	if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) {
+		g2d->runqueue_node = g2d_get_runqueue_node(g2d);
+
+		if (g2d->runqueue_node) {
+			pm_runtime_get_sync(g2d->dev);
+			g2d_dma_start(g2d, g2d->runqueue_node);
+		}
+	}
 
-	if (g2d->suspended)
-		g2d->runqueue_node = NULL;
-	else
-		g2d_exec_runqueue(g2d);
 	mutex_unlock(&g2d->runqueue_mutex);
 }
 
@@ -918,12 +965,72 @@ static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
 		}
 	}
 
-	if (pending & G2D_INTP_ACMD_FIN)
+	if (pending & G2D_INTP_ACMD_FIN) {
+		clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
 		queue_work(g2d->g2d_workq, &g2d->runqueue_work);
+	}
 
 	return IRQ_HANDLED;
 }
 
+/**
+ * g2d_wait_finish - wait for the G2D engine to finish the current runqueue node
+ * @g2d: G2D state object
+ * @file: if not zero, only wait if the current runqueue node belongs
+ *        to the DRM file
+ *
+ * Should the engine not become idle after a 100ms timeout, a hardware
+ * reset is issued.
+ */
+static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file)
+{
+	struct device *dev = g2d->dev;
+
+	struct g2d_runqueue_node *runqueue_node = NULL;
+	unsigned int tries = 10;
+
+	mutex_lock(&g2d->runqueue_mutex);
+
+	/* If no node is currently processed, we have nothing to do. */
+	if (!g2d->runqueue_node)
+		goto out;
+
+	runqueue_node = g2d->runqueue_node;
+
+	/* Check if the currently processed item belongs to us. */
+	if (file && runqueue_node->filp != file)
+		goto out;
+
+	mutex_unlock(&g2d->runqueue_mutex);
+
+	/* Wait for the G2D engine to finish. */
+	while (tries-- && (g2d->runqueue_node == runqueue_node))
+		mdelay(10);
+
+	mutex_lock(&g2d->runqueue_mutex);
+
+	if (g2d->runqueue_node != runqueue_node)
+		goto out;
+
+	dev_err(dev, "wait timed out, resetting engine...\n");
+	g2d_hw_reset(g2d);
+
+	/*
+	 * After the hardware reset of the engine we are going to loose
+	 * the IRQ which triggers the PM runtime put().
+	 * So do this manually here.
+	 */
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+
+	complete(&runqueue_node->complete);
+	if (runqueue_node->async)
+		g2d_free_runqueue_node(g2d, runqueue_node);
+
+out:
+	mutex_unlock(&g2d->runqueue_mutex);
+}
+
 static int g2d_check_reg_offset(struct device *dev,
 				struct g2d_cmdlist_node *node,
 				int nr, bool for_addr)
@@ -1259,10 +1366,11 @@ int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
 	runqueue_node->pid = current->pid;
 	runqueue_node->filp = file;
 	list_add_tail(&runqueue_node->list, &g2d->runqueue);
-	if (!g2d->runqueue_node)
-		g2d_exec_runqueue(g2d);
 	mutex_unlock(&g2d->runqueue_mutex);
 
+	/* Let the runqueue know that there is work to do. */
+	queue_work(g2d->g2d_workq, &g2d->runqueue_work);
+
 	if (runqueue_node->async)
 		goto out;
 
@@ -1339,15 +1447,26 @@ static void g2d_close(struct drm_device *drm_dev, struct device *dev,
 	if (!g2d)
 		return;
 
+	/* Remove the runqueue nodes that belong to us. */
+	mutex_lock(&g2d->runqueue_mutex);
+	g2d_remove_runqueue_nodes(g2d, file);
+	mutex_unlock(&g2d->runqueue_mutex);
+
+	/*
+	 * Wait for the runqueue worker to finish its current node.
+	 * After this the engine should no longer be accessing any
+	 * memory belonging to us.
+	 */
+	g2d_wait_finish(g2d, file);
+
+	/*
+	 * Even after the engine is idle, there might still be stale cmdlists
+	 * (i.e. cmdlisst which we submitted but never executed) around, with
+	 * their corresponding GEM/userptr buffers.
+	 * Properly unmap these buffers here.
+	 */
 	mutex_lock(&g2d->cmdlist_mutex);
 	list_for_each_entry_safe(node, n, &g2d_priv->inuse_cmdlist, list) {
-		/*
-		 * unmap all gem objects not completed.
-		 *
-		 * P.S. if current process was terminated forcely then
-		 * there may be some commands in inuse_cmdlist so unmap
-		 * them.
-		 */
 		g2d_unmap_cmdlist_gem(g2d, node, file);
 		list_move_tail(&node->list, &g2d->free_cmdlist);
 	}
@@ -1399,7 +1518,11 @@ static int g2d_probe(struct platform_device *pdev)
 		goto err_destroy_workqueue;
 	}
 
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_autosuspend_delay(dev, 2000);
 	pm_runtime_enable(dev);
+	clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
+	clear_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 
@@ -1440,7 +1563,7 @@ static int g2d_probe(struct platform_device *pdev)
 		goto err_put_clk;
 	}
 
-	dev_info(dev, "The exynos g2d(ver %d.%d) successfully probed\n",
+	dev_info(dev, "The Exynos G2D (ver %d.%d) successfully probed.\n",
 			G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
 
 	return 0;
@@ -1458,14 +1581,17 @@ static int g2d_remove(struct platform_device *pdev)
 {
 	struct g2d_data *g2d = platform_get_drvdata(pdev);
 
+	/* Suspend operation and wait for engine idle. */
+	set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
+	g2d_wait_finish(g2d, NULL);
+
 	cancel_work_sync(&g2d->runqueue_work);
 	exynos_drm_subdrv_unregister(&g2d->subdrv);
 
-	while (g2d->runqueue_node) {
-		g2d_free_runqueue_node(g2d, g2d->runqueue_node);
-		g2d->runqueue_node = g2d_get_runqueue_node(g2d);
-	}
+	/* There should be no locking needed here. */
+	g2d_remove_runqueue_nodes(g2d, NULL);
 
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
 	g2d_fini_cmdlist(g2d);
@@ -1480,14 +1606,12 @@ static int g2d_suspend(struct device *dev)
 {
 	struct g2d_data *g2d = dev_get_drvdata(dev);
 
-	mutex_lock(&g2d->runqueue_mutex);
-	g2d->suspended = true;
-	mutex_unlock(&g2d->runqueue_mutex);
-
-	while (g2d->runqueue_node)
-		/* FIXME: good range? */
-		usleep_range(500, 1000);
-
+	/*
+	 * Suspend the runqueue worker operation and wait until the G2D
+	 * engine is idle.
+	 */
+	set_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
+	g2d_wait_finish(g2d, NULL);
 	flush_work(&g2d->runqueue_work);
 
 	return 0;
@@ -1497,8 +1621,8 @@ static int g2d_resume(struct device *dev)
 {
 	struct g2d_data *g2d = dev_get_drvdata(dev);
 
-	g2d->suspended = false;
-	g2d_exec_runqueue(g2d);
+	clear_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags);
+	queue_work(g2d->g2d_workq, &g2d->runqueue_work);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.c b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
index 5d20da8f957e..52a9d269484e 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gsc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
@@ -1760,34 +1760,7 @@ static int gsc_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int gsc_suspend(struct device *dev)
-{
-	struct gsc_context *ctx = get_gsc_context(dev);
-
-	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
-
-	if (pm_runtime_suspended(dev))
-		return 0;
-
-	return gsc_clk_ctrl(ctx, false);
-}
-
-static int gsc_resume(struct device *dev)
-{
-	struct gsc_context *ctx = get_gsc_context(dev);
-
-	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
-
-	if (!pm_runtime_suspended(dev))
-		return gsc_clk_ctrl(ctx, true);
-
-	return 0;
-}
-#endif
-
-#ifdef CONFIG_PM
-static int gsc_runtime_suspend(struct device *dev)
+static int __maybe_unused gsc_runtime_suspend(struct device *dev)
 {
 	struct gsc_context *ctx = get_gsc_context(dev);
 
@@ -1796,7 +1769,7 @@ static int gsc_runtime_suspend(struct device *dev)
 	return  gsc_clk_ctrl(ctx, false);
 }
 
-static int gsc_runtime_resume(struct device *dev)
+static int __maybe_unused gsc_runtime_resume(struct device *dev)
 {
 	struct gsc_context *ctx = get_gsc_context(dev);
 
@@ -1804,10 +1777,10 @@ static int gsc_runtime_resume(struct device *dev)
 
 	return  gsc_clk_ctrl(ctx, true);
 }
-#endif
 
 static const struct dev_pm_ops gsc_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(gsc_suspend, gsc_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 	SET_RUNTIME_PM_OPS(gsc_runtime_suspend, gsc_runtime_resume, NULL)
 };
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_iommu.h b/drivers/gpu/drm/exynos/exynos_drm_iommu.h
index c8de4913fdbe..87f6b5672e11 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_iommu.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_iommu.h
@@ -66,7 +66,7 @@ static inline int __exynos_iommu_create_mapping(struct exynos_drm_private *priv,
 	if (ret)
 		goto free_domain;
 
-	ret = iommu_dma_init_domain(domain, start, size);
+	ret = iommu_dma_init_domain(domain, start, size, NULL);
 	if (ret)
 		goto put_cookie;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index 7f32419b25ea..c2f17f30afab 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -238,7 +238,6 @@ static void exynos_plane_atomic_update(struct drm_plane *plane,
 		return;
 
 	plane->crtc = state->crtc;
-	exynos_plane->pending_fb = state->fb;
 
 	if (exynos_crtc->ops->update_plane)
 		exynos_crtc->ops->update_plane(exynos_crtc, exynos_plane);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_rotator.c b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
index 404367a430b5..6591e406084c 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_rotator.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
@@ -794,29 +794,6 @@ static int rotator_clk_crtl(struct rot_context *rot, bool enable)
 	return 0;
 }
 
-
-#ifdef CONFIG_PM_SLEEP
-static int rotator_suspend(struct device *dev)
-{
-	struct rot_context *rot = dev_get_drvdata(dev);
-
-	if (pm_runtime_suspended(dev))
-		return 0;
-
-	return rotator_clk_crtl(rot, false);
-}
-
-static int rotator_resume(struct device *dev)
-{
-	struct rot_context *rot = dev_get_drvdata(dev);
-
-	if (!pm_runtime_suspended(dev))
-		return rotator_clk_crtl(rot, true);
-
-	return 0;
-}
-#endif
-
 static int rotator_runtime_suspend(struct device *dev)
 {
 	struct rot_context *rot = dev_get_drvdata(dev);
@@ -833,7 +810,8 @@ static int rotator_runtime_resume(struct device *dev)
 #endif
 
 static const struct dev_pm_ops rotator_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(rotator_suspend, rotator_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 	SET_RUNTIME_PM_OPS(rotator_runtime_suspend, rotator_runtime_resume,
 									NULL)
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index e8f6c92b2a36..57fe514d5c5b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -15,6 +15,7 @@
 #include <linux/kernel.h>
 #include <linux/platform_device.h>
 #include <linux/component.h>
+#include <linux/timer.h>
 
 #include <drm/exynos_drm.h>
 
@@ -28,6 +29,9 @@
 #include "exynos_drm_plane.h"
 #include "exynos_drm_vidi.h"
 
+/* VIDI uses fixed refresh rate of 50Hz */
+#define VIDI_REFRESH_TIME (1000 / 50)
+
 /* vidi has totally three virtual windows. */
 #define WINDOWS_NR		3
 
@@ -43,12 +47,9 @@ struct vidi_context {
 	struct exynos_drm_plane		planes[WINDOWS_NR];
 	struct edid			*raw_edid;
 	unsigned int			clkdiv;
-	unsigned long			irq_flags;
 	unsigned int			connected;
-	bool				vblank_on;
 	bool				suspended;
-	bool				direct_vblank;
-	struct work_struct		work;
+	struct timer_list		timer;
 	struct mutex			lock;
 	int				pipe;
 };
@@ -102,30 +103,14 @@ static int vidi_enable_vblank(struct exynos_drm_crtc *crtc)
 	if (ctx->suspended)
 		return -EPERM;
 
-	if (!test_and_set_bit(0, &ctx->irq_flags))
-		ctx->vblank_on = true;
-
-	ctx->direct_vblank = true;
-
-	/*
-	 * in case of page flip request, vidi_finish_pageflip function
-	 * will not be called because direct_vblank is true and then
-	 * that function will be called by crtc_ops->update_plane callback
-	 */
-	schedule_work(&ctx->work);
+	mod_timer(&ctx->timer,
+		jiffies + msecs_to_jiffies(VIDI_REFRESH_TIME) - 1);
 
 	return 0;
 }
 
 static void vidi_disable_vblank(struct exynos_drm_crtc *crtc)
 {
-	struct vidi_context *ctx = crtc->ctx;
-
-	if (ctx->suspended)
-		return;
-
-	if (test_and_clear_bit(0, &ctx->irq_flags))
-		ctx->vblank_on = false;
 }
 
 static void vidi_update_plane(struct exynos_drm_crtc *crtc,
@@ -140,9 +125,6 @@ static void vidi_update_plane(struct exynos_drm_crtc *crtc,
 
 	addr = exynos_drm_fb_dma_addr(state->fb, 0);
 	DRM_DEBUG_KMS("dma_addr = %pad\n", &addr);
-
-	if (ctx->vblank_on)
-		schedule_work(&ctx->work);
 }
 
 static void vidi_enable(struct exynos_drm_crtc *crtc)
@@ -153,17 +135,17 @@ static void vidi_enable(struct exynos_drm_crtc *crtc)
 
 	ctx->suspended = false;
 
-	/* if vblank was enabled status, enable it again. */
-	if (test_and_clear_bit(0, &ctx->irq_flags))
-		vidi_enable_vblank(ctx->crtc);
-
 	mutex_unlock(&ctx->lock);
+
+	drm_crtc_vblank_on(&crtc->base);
 }
 
 static void vidi_disable(struct exynos_drm_crtc *crtc)
 {
 	struct vidi_context *ctx = crtc->ctx;
 
+	drm_crtc_vblank_off(&crtc->base);
+
 	mutex_lock(&ctx->lock);
 
 	ctx->suspended = true;
@@ -190,37 +172,16 @@ static const struct exynos_drm_crtc_ops vidi_crtc_ops = {
 	.update_plane = vidi_update_plane,
 };
 
-static void vidi_fake_vblank_handler(struct work_struct *work)
+static void vidi_fake_vblank_timer(unsigned long arg)
 {
-	struct vidi_context *ctx = container_of(work, struct vidi_context,
-					work);
-	int win;
+	struct vidi_context *ctx = (void *)arg;
 
 	if (ctx->pipe < 0)
 		return;
 
-	/* refresh rate is about 50Hz. */
-	usleep_range(16000, 20000);
-
-	mutex_lock(&ctx->lock);
-
-	if (ctx->direct_vblank) {
-		drm_crtc_handle_vblank(&ctx->crtc->base);
-		ctx->direct_vblank = false;
-		mutex_unlock(&ctx->lock);
-		return;
-	}
-
-	mutex_unlock(&ctx->lock);
-
-	for (win = 0 ; win < WINDOWS_NR ; win++) {
-		struct exynos_drm_plane *plane = &ctx->planes[win];
-
-		if (!plane->pending_fb)
-			continue;
-
-		exynos_drm_crtc_finish_update(ctx->crtc, plane);
-	}
+	if (drm_crtc_handle_vblank(&ctx->crtc->base))
+		mod_timer(&ctx->timer,
+			jiffies + msecs_to_jiffies(VIDI_REFRESH_TIME) - 1);
 }
 
 static ssize_t vidi_show_connection(struct device *dev,
@@ -489,6 +450,9 @@ static int vidi_bind(struct device *dev, struct device *master, void *data)
 
 static void vidi_unbind(struct device *dev, struct device *master, void *data)
 {
+	struct vidi_context *ctx = dev_get_drvdata(dev);
+
+	del_timer_sync(&ctx->timer);
 }
 
 static const struct component_ops vidi_component_ops = {
@@ -507,7 +471,7 @@ static int vidi_probe(struct platform_device *pdev)
 
 	ctx->pdev = pdev;
 
-	INIT_WORK(&ctx->work, vidi_fake_vblank_handler);
+	setup_timer(&ctx->timer, vidi_fake_vblank_timer, (unsigned long)ctx);
 
 	mutex_init(&ctx->lock);
 
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index 2275efe41acd..e8fb6ef947ee 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -1669,10 +1669,9 @@ static int hdmi_resources_init(struct hdmi_context *hdata)
 	if (ret)
 		return ret;
 
-	for (i = 0; i < ARRAY_SIZE(supply); ++i) {
+	for (i = 0; i < ARRAY_SIZE(supply); ++i)
 		hdata->regul_bulk[i].supply = supply[i];
-		hdata->regul_bulk[i].consumer = NULL;
-	}
+
 	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(supply), hdata->regul_bulk);
 	if (ret) {
 		if (ret != -EPROBE_DEFER)
@@ -1760,28 +1759,74 @@ static const struct component_ops hdmi_component_ops = {
 	.unbind = hdmi_unbind,
 };
 
-static struct device_node *hdmi_legacy_ddc_dt_binding(struct device *dev)
+static int hdmi_get_ddc_adapter(struct hdmi_context *hdata)
 {
 	const char *compatible_str = "samsung,exynos4210-hdmiddc";
 	struct device_node *np;
+	struct i2c_adapter *adpt;
 
 	np = of_find_compatible_node(NULL, NULL, compatible_str);
 	if (np)
-		return of_get_next_parent(np);
+		np = of_get_next_parent(np);
+	else
+		np = of_parse_phandle(hdata->dev->of_node, "ddc", 0);
+
+	if (!np) {
+		DRM_ERROR("Failed to find ddc node in device tree\n");
+		return -ENODEV;
+	}
 
-	return NULL;
+	adpt = of_find_i2c_adapter_by_node(np);
+	of_node_put(np);
+
+	if (!adpt) {
+		DRM_INFO("Failed to get ddc i2c adapter by node\n");
+		return -EPROBE_DEFER;
+	}
+
+	hdata->ddc_adpt = adpt;
+
+	return 0;
 }
 
-static struct device_node *hdmi_legacy_phy_dt_binding(struct device *dev)
+static int hdmi_get_phy_io(struct hdmi_context *hdata)
 {
 	const char *compatible_str = "samsung,exynos4212-hdmiphy";
+	struct device_node *np;
+	int ret = 0;
+
+	np = of_find_compatible_node(NULL, NULL, compatible_str);
+	if (!np) {
+		np = of_parse_phandle(hdata->dev->of_node, "phy", 0);
+		if (!np) {
+			DRM_ERROR("Failed to find hdmiphy node in device tree\n");
+			return -ENODEV;
+		}
+	}
+
+	if (hdata->drv_data->is_apb_phy) {
+		hdata->regs_hdmiphy = of_iomap(np, 0);
+		if (!hdata->regs_hdmiphy) {
+			DRM_ERROR("failed to ioremap hdmi phy\n");
+			ret = -ENOMEM;
+			goto out;
+		}
+	} else {
+		hdata->hdmiphy_port = of_find_i2c_device_by_node(np);
+		if (!hdata->hdmiphy_port) {
+			DRM_INFO("Failed to get hdmi phy i2c client\n");
+			ret = -EPROBE_DEFER;
+			goto out;
+		}
+	}
 
-	return of_find_compatible_node(NULL, NULL, compatible_str);
+out:
+	of_node_put(np);
+	return ret;
 }
 
 static int hdmi_probe(struct platform_device *pdev)
 {
-	struct device_node *ddc_node, *phy_node;
 	struct device *dev = &pdev->dev;
 	struct hdmi_context *hdata;
 	struct resource *res;
@@ -1811,52 +1856,13 @@ static int hdmi_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	ddc_node = hdmi_legacy_ddc_dt_binding(dev);
-	if (ddc_node)
-		goto out_get_ddc_adpt;
-
-	ddc_node = of_parse_phandle(dev->of_node, "ddc", 0);
-	if (!ddc_node) {
-		DRM_ERROR("Failed to find ddc node in device tree\n");
-		return -ENODEV;
-	}
-	of_node_put(dev->of_node);
-
-out_get_ddc_adpt:
-	hdata->ddc_adpt = of_find_i2c_adapter_by_node(ddc_node);
-	if (!hdata->ddc_adpt) {
-		DRM_ERROR("Failed to get ddc i2c adapter by node\n");
-		return -EPROBE_DEFER;
-	}
-
-	phy_node = hdmi_legacy_phy_dt_binding(dev);
-	if (phy_node)
-		goto out_get_phy_port;
+	ret = hdmi_get_ddc_adapter(hdata);
+	if (ret)
+		return ret;
 
-	phy_node = of_parse_phandle(dev->of_node, "phy", 0);
-	if (!phy_node) {
-		DRM_ERROR("Failed to find hdmiphy node in device tree\n");
-		ret = -ENODEV;
+	ret = hdmi_get_phy_io(hdata);
+	if (ret)
 		goto err_ddc;
-	}
-	of_node_put(dev->of_node);
-
-out_get_phy_port:
-	if (hdata->drv_data->is_apb_phy) {
-		hdata->regs_hdmiphy = of_iomap(phy_node, 0);
-		if (!hdata->regs_hdmiphy) {
-			DRM_ERROR("failed to ioremap hdmi phy\n");
-			ret = -ENOMEM;
-			goto err_ddc;
-		}
-	} else {
-		hdata->hdmiphy_port = of_find_i2c_device_by_node(phy_node);
-		if (!hdata->hdmiphy_port) {
-			DRM_ERROR("Failed to get hdmi phy i2c client\n");
-			ret = -EPROBE_DEFER;
-			goto err_ddc;
-		}
-	}
 
 	INIT_DELAYED_WORK(&hdata->hotplug_work, hdmi_hotplug_work_func);
 
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index e1d47f9435fc..edb20a34c66c 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -73,6 +73,9 @@ enum mixer_version_id {
 enum mixer_flag_bits {
 	MXR_BIT_POWERED,
 	MXR_BIT_VSYNC,
+	MXR_BIT_INTERLACE,
+	MXR_BIT_VP_ENABLED,
+	MXR_BIT_HAS_SCLK,
 };
 
 static const uint32_t mixer_formats[] = {
@@ -98,9 +101,6 @@ struct mixer_context {
 	struct exynos_drm_plane	planes[MIXER_WIN_NR];
 	int			pipe;
 	unsigned long		flags;
-	bool			interlace;
-	bool			vp_enabled;
-	bool			has_sclk;
 
 	struct mixer_resources	mixer_res;
 	enum mixer_version_id	mxr_ver;
@@ -346,7 +346,7 @@ static void mixer_vsync_set_update(struct mixer_context *ctx, bool enable)
 	mixer_reg_writemask(res, MXR_STATUS, enable ?
 			MXR_STATUS_SYNC_ENABLE : 0, MXR_STATUS_SYNC_ENABLE);
 
-	if (ctx->vp_enabled)
+	if (test_bit(MXR_BIT_VP_ENABLED, &ctx->flags))
 		vp_reg_write(res, VP_SHADOW_UPDATE, enable ?
 			VP_SHADOW_UPDATE_ENABLE : 0);
 }
@@ -357,8 +357,8 @@ static void mixer_cfg_scan(struct mixer_context *ctx, unsigned int height)
 	u32 val;
 
 	/* choosing between interlace and progressive mode */
-	val = (ctx->interlace ? MXR_CFG_SCAN_INTERLACE :
-				MXR_CFG_SCAN_PROGRESSIVE);
+	val = test_bit(MXR_BIT_INTERLACE, &ctx->flags) ?
+		MXR_CFG_SCAN_INTERLACE : MXR_CFG_SCAN_PROGRESSIVE;
 
 	if (ctx->mxr_ver != MXR_VER_128_0_0_184) {
 		/* choosing between proper HD and SD mode */
@@ -436,9 +436,10 @@ static void mixer_cfg_layer(struct mixer_context *ctx, unsigned int win,
 		mixer_reg_writemask(res, MXR_LAYER_CFG,
 				    MXR_LAYER_CFG_GRP1_VAL(priority),
 				    MXR_LAYER_CFG_GRP1_MASK);
+
 		break;
 	case VP_DEFAULT_WIN:
-		if (ctx->vp_enabled) {
+		if (test_bit(MXR_BIT_VP_ENABLED, &ctx->flags)) {
 			vp_reg_writemask(res, VP_ENABLE, val, VP_ENABLE_ON);
 			mixer_reg_writemask(res, MXR_CFG, val,
 				MXR_CFG_VP_ENABLE);
@@ -501,7 +502,7 @@ static void vp_video_buffer(struct mixer_context *ctx,
 	chroma_addr[0] = exynos_drm_fb_dma_addr(fb, 1);
 
 	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
-		ctx->interlace = true;
+		__set_bit(MXR_BIT_INTERLACE, &ctx->flags);
 		if (tiled_mode) {
 			luma_addr[1] = luma_addr[0] + 0x40;
 			chroma_addr[1] = chroma_addr[0] + 0x40;
@@ -510,7 +511,7 @@ static void vp_video_buffer(struct mixer_context *ctx,
 			chroma_addr[1] = chroma_addr[0] + fb->pitches[0];
 		}
 	} else {
-		ctx->interlace = false;
+		__clear_bit(MXR_BIT_INTERLACE, &ctx->flags);
 		luma_addr[1] = 0;
 		chroma_addr[1] = 0;
 	}
@@ -518,7 +519,7 @@ static void vp_video_buffer(struct mixer_context *ctx,
 	spin_lock_irqsave(&res->reg_slock, flags);
 
 	/* interlace or progressive scan mode */
-	val = (ctx->interlace ? ~0 : 0);
+	val = (test_bit(MXR_BIT_INTERLACE, &ctx->flags) ? ~0 : 0);
 	vp_reg_writemask(res, VP_MODE, val, VP_MODE_LINE_SKIP);
 
 	/* setup format */
@@ -541,7 +542,7 @@ static void vp_video_buffer(struct mixer_context *ctx,
 
 	vp_reg_write(res, VP_DST_WIDTH, state->crtc.w);
 	vp_reg_write(res, VP_DST_H_POSITION, state->crtc.x);
-	if (ctx->interlace) {
+	if (test_bit(MXR_BIT_INTERLACE, &ctx->flags)) {
 		vp_reg_write(res, VP_DST_HEIGHT, state->crtc.h / 2);
 		vp_reg_write(res, VP_DST_V_POSITION, state->crtc.y / 2);
 	} else {
@@ -636,9 +637,9 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 	src_y_offset = 0;
 
 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-		ctx->interlace = true;
+		__set_bit(MXR_BIT_INTERLACE, &ctx->flags);
 	else
-		ctx->interlace = false;
+		__clear_bit(MXR_BIT_INTERLACE, &ctx->flags);
 
 	spin_lock_irqsave(&res->reg_slock, flags);
 
@@ -697,10 +698,10 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 static void vp_win_reset(struct mixer_context *ctx)
 {
 	struct mixer_resources *res = &ctx->mixer_res;
-	int tries = 100;
+	unsigned int tries = 100;
 
 	vp_reg_write(res, VP_SRESET, VP_SRESET_PROCESSING);
-	for (tries = 100; tries; --tries) {
+	while (tries--) {
 		/* waiting until VP_SRESET_PROCESSING is 0 */
 		if (~vp_reg_read(res, VP_SRESET) & VP_SRESET_PROCESSING)
 			break;
@@ -733,7 +734,7 @@ static void mixer_win_reset(struct mixer_context *ctx)
 	mixer_reg_write(res, MXR_BG_COLOR1, 0x008080);
 	mixer_reg_write(res, MXR_BG_COLOR2, 0x008080);
 
-	if (ctx->vp_enabled) {
+	if (test_bit(MXR_BIT_VP_ENABLED, &ctx->flags)) {
 		/* configuration of Video Processor Registers */
 		vp_win_reset(ctx);
 		vp_default_filter(res);
@@ -742,7 +743,7 @@ static void mixer_win_reset(struct mixer_context *ctx)
 	/* disable all layers */
 	mixer_reg_writemask(res, MXR_CFG, 0, MXR_CFG_GRP0_ENABLE);
 	mixer_reg_writemask(res, MXR_CFG, 0, MXR_CFG_GRP1_ENABLE);
-	if (ctx->vp_enabled)
+	if (test_bit(MXR_BIT_VP_ENABLED, &ctx->flags))
 		mixer_reg_writemask(res, MXR_CFG, 0, MXR_CFG_VP_ENABLE);
 
 	spin_unlock_irqrestore(&res->reg_slock, flags);
@@ -753,7 +754,6 @@ static irqreturn_t mixer_irq_handler(int irq, void *arg)
 	struct mixer_context *ctx = arg;
 	struct mixer_resources *res = &ctx->mixer_res;
 	u32 val, base, shadow;
-	int win;
 
 	spin_lock(&res->reg_slock);
 
@@ -767,7 +767,7 @@ static irqreturn_t mixer_irq_handler(int irq, void *arg)
 		val &= ~MXR_INT_STATUS_VSYNC;
 
 		/* interlace scan need to check shadow register */
-		if (ctx->interlace) {
+		if (test_bit(MXR_BIT_INTERLACE, &ctx->flags)) {
 			base = mixer_reg_read(res, MXR_GRAPHIC_BASE(0));
 			shadow = mixer_reg_read(res, MXR_GRAPHIC_BASE_S(0));
 			if (base != shadow)
@@ -780,14 +780,6 @@ static irqreturn_t mixer_irq_handler(int irq, void *arg)
 		}
 
 		drm_crtc_handle_vblank(&ctx->crtc->base);
-		for (win = 0 ; win < MIXER_WIN_NR ; win++) {
-			struct exynos_drm_plane *plane = &ctx->planes[win];
-
-			if (!plane->pending_fb)
-				continue;
-
-			exynos_drm_crtc_finish_update(ctx->crtc, plane);
-		}
 	}
 
 out:
@@ -867,7 +859,7 @@ static int vp_resources_init(struct mixer_context *mixer_ctx)
 		return -ENODEV;
 	}
 
-	if (mixer_ctx->has_sclk) {
+	if (test_bit(MXR_BIT_HAS_SCLK, &mixer_ctx->flags)) {
 		mixer_res->sclk_mixer = devm_clk_get(dev, "sclk_mixer");
 		if (IS_ERR(mixer_res->sclk_mixer)) {
 			dev_err(dev, "failed to get clock 'sclk_mixer'\n");
@@ -917,7 +909,7 @@ static int mixer_initialize(struct mixer_context *mixer_ctx,
 		return ret;
 	}
 
-	if (mixer_ctx->vp_enabled) {
+	if (test_bit(MXR_BIT_VP_ENABLED, &mixer_ctx->flags)) {
 		/* acquire vp resources: regs, irqs, clocks */
 		ret = vp_resources_init(mixer_ctx);
 		if (ret) {
@@ -1160,7 +1152,8 @@ static int mixer_bind(struct device *dev, struct device *manager, void *data)
 		return ret;
 
 	for (i = 0; i < MIXER_WIN_NR; i++) {
-		if (i == VP_DEFAULT_WIN && !ctx->vp_enabled)
+		if (i == VP_DEFAULT_WIN && !test_bit(MXR_BIT_VP_ENABLED,
+						     &ctx->flags))
 			continue;
 
 		ret = exynos_plane_init(drm_dev, &ctx->planes[i], i,
@@ -1215,10 +1208,13 @@ static int mixer_probe(struct platform_device *pdev)
 
 	ctx->pdev = pdev;
 	ctx->dev = dev;
-	ctx->vp_enabled = drv->is_vp_enabled;
-	ctx->has_sclk = drv->has_sclk;
 	ctx->mxr_ver = drv->version;
 
+	if (drv->is_vp_enabled)
+		__set_bit(MXR_BIT_VP_ENABLED, &ctx->flags);
+	if (drv->has_sclk)
+		__set_bit(MXR_BIT_HAS_SCLK, &ctx->flags);
+
 	platform_set_drvdata(pdev, ctx);
 
 	ret = component_add(&pdev->dev, &mixer_component_ops);
@@ -1244,9 +1240,9 @@ static int __maybe_unused exynos_mixer_suspend(struct device *dev)
 
 	clk_disable_unprepare(res->hdmi);
 	clk_disable_unprepare(res->mixer);
-	if (ctx->vp_enabled) {
+	if (test_bit(MXR_BIT_VP_ENABLED, &ctx->flags)) {
 		clk_disable_unprepare(res->vp);
-		if (ctx->has_sclk)
+		if (test_bit(MXR_BIT_HAS_SCLK, &ctx->flags))
 			clk_disable_unprepare(res->sclk_mixer);
 	}
 
@@ -1269,14 +1265,14 @@ static int __maybe_unused exynos_mixer_resume(struct device *dev)
 		DRM_ERROR("Failed to prepare_enable the hdmi clk [%d]\n", ret);
 		return ret;
 	}
-	if (ctx->vp_enabled) {
+	if (test_bit(MXR_BIT_VP_ENABLED, &ctx->flags)) {
 		ret = clk_prepare_enable(res->vp);
 		if (ret < 0) {
 			DRM_ERROR("Failed to prepare_enable the vp clk [%d]\n",
 				  ret);
 			return ret;
 		}
-		if (ctx->has_sclk) {
+		if (test_bit(MXR_BIT_HAS_SCLK, &ctx->flags)) {
 			ret = clk_prepare_enable(res->sclk_mixer);
 			if (ret < 0) {
 				DRM_ERROR("Failed to prepare_enable the " \
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
index 7882387f9bff..0884c45aefe8 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
@@ -270,7 +270,7 @@ static int fsl_dcu_drm_pm_resume(struct device *dev)
 	ret = clk_prepare_enable(fsl_dev->pix_clk);
 	if (ret < 0) {
 		dev_err(dev, "failed to enable pix clk\n");
-		return ret;
+		goto disable_dcu_clk;
 	}
 
 	fsl_dcu_drm_init_planes(fsl_dev->drm);
@@ -284,6 +284,10 @@ static int fsl_dcu_drm_pm_resume(struct device *dev)
 	enable_irq(fsl_dev->irq);
 
 	return 0;
+
+disable_dcu_clk:
+	clk_disable_unprepare(fsl_dev->clk);
+	return ret;
 }
 #endif
 
@@ -330,6 +334,7 @@ static int fsl_dcu_drm_probe(struct platform_device *pdev)
 	const char *pix_clk_in_name;
 	const struct of_device_id *id;
 	int ret;
+	u8 div_ratio_shift = 0;
 
 	fsl_dev = devm_kzalloc(dev, sizeof(*fsl_dev), GFP_KERNEL);
 	if (!fsl_dev)
@@ -382,11 +387,14 @@ static int fsl_dcu_drm_probe(struct platform_device *pdev)
 		pix_clk_in = fsl_dev->clk;
 	}
 
+	if (of_property_read_bool(dev->of_node, "big-endian"))
+		div_ratio_shift = 24;
+
 	pix_clk_in_name = __clk_get_name(pix_clk_in);
 	snprintf(pix_clk_name, sizeof(pix_clk_name), "%s_pix", pix_clk_in_name);
 	fsl_dev->pix_clk = clk_register_divider(dev, pix_clk_name,
 			pix_clk_in_name, 0, base + DCU_DIV_RATIO,
-			0, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL);
+			div_ratio_shift, 8, CLK_DIVIDER_ROUND_CLOSEST, NULL);
 	if (IS_ERR(fsl_dev->pix_clk)) {
 		dev_err(dev, "failed to register pix clk\n");
 		ret = PTR_ERR(fsl_dev->pix_clk);
@@ -402,8 +410,8 @@ static int fsl_dcu_drm_probe(struct platform_device *pdev)
 	fsl_dev->tcon = fsl_tcon_init(dev);
 
 	drm = drm_dev_alloc(driver, dev);
-	if (!drm) {
-		ret = -ENOMEM;
+	if (IS_ERR(drm)) {
+		ret = PTR_ERR(drm);
 		goto disable_pix_clk;
 	}
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
index e50467a0deb0..a7e5486bd1e9 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
@@ -169,25 +169,10 @@ static void fsl_dcu_drm_plane_atomic_update(struct drm_plane *plane,
 	return;
 }
 
-static void
-fsl_dcu_drm_plane_cleanup_fb(struct drm_plane *plane,
-			     const struct drm_plane_state *new_state)
-{
-}
-
-static int
-fsl_dcu_drm_plane_prepare_fb(struct drm_plane *plane,
-			     const struct drm_plane_state *new_state)
-{
-	return 0;
-}
-
 static const struct drm_plane_helper_funcs fsl_dcu_drm_plane_helper_funcs = {
 	.atomic_check = fsl_dcu_drm_plane_atomic_check,
 	.atomic_disable = fsl_dcu_drm_plane_atomic_disable,
 	.atomic_update = fsl_dcu_drm_plane_atomic_update,
-	.cleanup_fb = fsl_dcu_drm_plane_cleanup_fb,
-	.prepare_fb = fsl_dcu_drm_plane_prepare_fb,
 };
 
 static void fsl_dcu_drm_plane_destroy(struct drm_plane *plane)
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_tcon.c b/drivers/gpu/drm/fsl-dcu/fsl_tcon.c
index bca09ea24632..3194e544ee27 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_tcon.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_tcon.c
@@ -57,10 +57,7 @@ static int fsl_tcon_init_regmap(struct device *dev,
 
 	tcon->regs = devm_regmap_init_mmio(dev, regs,
 					   &fsl_tcon_regmap_config);
-	if (IS_ERR(tcon->regs))
-		return PTR_ERR(tcon->regs);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(tcon->regs);
 }
 
 struct fsl_tcon *fsl_tcon_init(struct device *dev)
diff --git a/drivers/gpu/drm/gma500/accel_2d.c b/drivers/gpu/drm/gma500/accel_2d.c
index db9f7d011832..0d2bb1682508 100644
--- a/drivers/gpu/drm/gma500/accel_2d.c
+++ b/drivers/gpu/drm/gma500/accel_2d.c
@@ -28,7 +28,6 @@
 #include <linux/tty.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include <linux/fb.h>
 #include <linux/init.h>
 #include <linux/console.h>
 
diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
index 38dc89083148..ea733ab5b1e0 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
@@ -415,14 +415,6 @@ static int cdv_intel_lvds_get_modes(struct drm_connector *connector)
 	if (ret)
 		return ret;
 
-	/* Didn't get an EDID, so
-	 * Set wide sync ranges so we get all modes
-	 * handed to valid_mode for checking
-	 */
-	connector->display_info.min_vfreq = 0;
-	connector->display_info.max_vfreq = 200;
-	connector->display_info.min_hfreq = 0;
-	connector->display_info.max_hfreq = 200;
 	if (mode_dev->panel_fixed_mode != NULL) {
 		struct drm_display_mode *mode =
 		    drm_mode_duplicate(dev, mode_dev->panel_fixed_mode);
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 0fcdce0817de..3a44e705db53 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -26,7 +26,6 @@
 #include <linux/tty.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include <linux/fb.h>
 #include <linux/init.h>
 #include <linux/console.h>
 
diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_output.c b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
index 907cb51795c3..acb3848ef1c9 100644
--- a/drivers/gpu/drm/gma500/mdfld_dsi_output.c
+++ b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
@@ -335,11 +335,6 @@ static int mdfld_dsi_connector_get_modes(struct drm_connector *connector)
 	struct drm_display_mode *dup_mode = NULL;
 	struct drm_device *dev = connector->dev;
 
-	connector->display_info.min_vfreq = 0;
-	connector->display_info.max_vfreq = 200;
-	connector->display_info.min_hfreq = 0;
-	connector->display_info.max_hfreq = 200;
-
 	if (fixed_mode) {
 		dev_dbg(dev->dev, "fixed_mode %dx%d\n",
 				fixed_mode->hdisplay, fixed_mode->vdisplay);
diff --git a/drivers/gpu/drm/gma500/opregion.c b/drivers/gpu/drm/gma500/opregion.c
index ab696ca7eeec..eab6d889bde9 100644
--- a/drivers/gpu/drm/gma500/opregion.c
+++ b/drivers/gpu/drm/gma500/opregion.c
@@ -163,10 +163,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
 	if (bclp > 255)
 		return ASLE_BACKLIGHT_FAILED;
 
-	if (config_enabled(CONFIG_BACKLIGHT_CLASS_DEVICE)) {
-		int max = bd->props.max_brightness;
-		gma_backlight_set(dev, bclp * max / 255);
-	}
+	gma_backlight_set(dev, bclp * bd->props.max_brightness / 255);
 
 	asle->cblv = (bclp * 0x64) / 0xff | ASLE_CBLV_VALID;
 
diff --git a/drivers/gpu/drm/gma500/psb_intel_lvds.c b/drivers/gpu/drm/gma500/psb_intel_lvds.c
index e55733ca46d2..fd7c91254841 100644
--- a/drivers/gpu/drm/gma500/psb_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/psb_intel_lvds.c
@@ -530,15 +530,6 @@ static int psb_intel_lvds_get_modes(struct drm_connector *connector)
 	if (ret)
 		return ret;
 
-	/* Didn't get an EDID, so
-	 * Set wide sync ranges so we get all modes
-	 * handed to valid_mode for checking
-	 */
-	connector->display_info.min_vfreq = 0;
-	connector->display_info.max_vfreq = 200;
-	connector->display_info.min_hfreq = 0;
-	connector->display_info.max_hfreq = 200;
-
 	if (mode_dev->panel_fixed_mode != NULL) {
 		struct drm_display_mode *mode =
 		    drm_mode_duplicate(dev, mode_dev->panel_fixed_mode);
diff --git a/drivers/gpu/drm/gma500/psb_intel_modes.c b/drivers/gpu/drm/gma500/psb_intel_modes.c
index 4fca0d6feebe..e5360726d80b 100644
--- a/drivers/gpu/drm/gma500/psb_intel_modes.c
+++ b/drivers/gpu/drm/gma500/psb_intel_modes.c
@@ -18,7 +18,6 @@
  */
 
 #include <linux/i2c.h>
-#include <linux/fb.h>
 #include <drm/drmP.h>
 #include "psb_intel_drv.h"
 
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
index c3707d47cd89..7e7a4d43d6b6 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
@@ -608,15 +608,17 @@ static void ade_rdma_set(void __iomem *base, struct drm_framebuffer *fb,
 			 u32 ch, u32 y, u32 in_h, u32 fmt)
 {
 	struct drm_gem_cma_object *obj = drm_fb_cma_get_gem_obj(fb, 0);
+	char *format_name;
 	u32 reg_ctrl, reg_addr, reg_size, reg_stride, reg_space, reg_en;
 	u32 stride = fb->pitches[0];
 	u32 addr = (u32)obj->paddr + y * stride;
 
 	DRM_DEBUG_DRIVER("rdma%d: (y=%d, height=%d), stride=%d, paddr=0x%x\n",
 			 ch + 1, y, in_h, stride, (u32)obj->paddr);
+	format_name = drm_get_format_name(fb->pixel_format);
 	DRM_DEBUG_DRIVER("addr=0x%x, fb:%dx%d, pixel_format=%d(%s)\n",
-			 addr, fb->width, fb->height, fmt,
-			 drm_get_format_name(fb->pixel_format));
+			 addr, fb->width, fb->height, fmt, format_name);
+	kfree(format_name);
 
 	/* get reg offset */
 	reg_ctrl = RD_CH_CTRL(ch);
@@ -815,19 +817,6 @@ static void ade_disable_channel(struct ade_plane *aplane)
 	ade_compositor_routing_disable(base, ch);
 }
 
-static int ade_plane_prepare_fb(struct drm_plane *plane,
-				const struct drm_plane_state *new_state)
-{
-	/* do nothing */
-	return 0;
-}
-
-static void ade_plane_cleanup_fb(struct drm_plane *plane,
-				 const struct drm_plane_state *old_state)
-{
-	/* do nothing */
-}
-
 static int ade_plane_atomic_check(struct drm_plane *plane,
 				  struct drm_plane_state *state)
 {
@@ -895,8 +884,6 @@ static void ade_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static const struct drm_plane_helper_funcs ade_plane_helper_funcs = {
-	.prepare_fb = ade_plane_prepare_fb,
-	.cleanup_fb = ade_plane_cleanup_fb,
 	.atomic_check = ade_plane_atomic_check,
 	.atomic_update = ade_plane_atomic_update,
 	.atomic_disable = ade_plane_atomic_disable,
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
index 1edd9bc80294..90377a609c98 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
@@ -169,7 +169,7 @@ static int kirin_gem_cma_dumb_create(struct drm_file *file,
 
 static struct drm_driver kirin_drm_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
-				  DRIVER_ATOMIC | DRIVER_HAVE_IRQ,
+				  DRIVER_ATOMIC,
 	.fops			= &kirin_drm_fops,
 
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
@@ -207,8 +207,8 @@ static int kirin_drm_bind(struct device *dev)
 	int ret;
 
 	drm_dev = drm_dev_alloc(driver, dev);
-	if (!drm_dev)
-		return -ENOMEM;
+	if (IS_ERR(drm_dev))
+		return PTR_ERR(drm_dev);
 
 	drm_dev->platformdev = to_platform_device(dev);
 
diff --git a/drivers/gpu/drm/i2c/Kconfig b/drivers/gpu/drm/i2c/Kconfig
index 4d341db462a2..a6c92beb410a 100644
--- a/drivers/gpu/drm/i2c/Kconfig
+++ b/drivers/gpu/drm/i2c/Kconfig
@@ -22,6 +22,7 @@ config DRM_I2C_SIL164
 config DRM_I2C_NXP_TDA998X
 	tristate "NXP Semiconductors TDA998X HDMI encoder"
 	default m if DRM_TILCDC
+	select SND_SOC_HDMI_CODEC if SND_SOC
 	help
 	  Support for NXP Semiconductors TDA998X HDMI encoders.
 
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index f4315bc8d471..9798d400d817 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -20,6 +20,7 @@
 #include <linux/module.h>
 #include <linux/irq.h>
 #include <sound/asoundef.h>
+#include <sound/hdmi-codec.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_atomic_helper.h>
@@ -30,6 +31,11 @@
 
 #define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 
+struct tda998x_audio_port {
+	u8 format;		/* AFMT_xxx */
+	u8 config;		/* AP value */
+};
+
 struct tda998x_priv {
 	struct i2c_client *cec;
 	struct i2c_client *hdmi;
@@ -41,7 +47,10 @@ struct tda998x_priv {
 	u8 vip_cntrl_0;
 	u8 vip_cntrl_1;
 	u8 vip_cntrl_2;
-	struct tda998x_encoder_params params;
+	struct tda998x_audio_params audio_params;
+
+	struct platform_device *audio_pdev;
+	struct mutex audio_mutex;
 
 	wait_queue_head_t wq_edid;
 	volatile int wq_edid_wait;
@@ -53,6 +62,8 @@ struct tda998x_priv {
 
 	struct drm_encoder encoder;
 	struct drm_connector connector;
+
+	struct tda998x_audio_port audio_port[2];
 };
 
 #define conn_to_tda998x_priv(x) \
@@ -666,26 +677,16 @@ tda998x_write_if(struct tda998x_priv *priv, u8 bit, u16 addr,
 	reg_set(priv, REG_DIP_IF_FLAGS, bit);
 }
 
-static void
-tda998x_write_aif(struct tda998x_priv *priv, struct tda998x_encoder_params *p)
+static int tda998x_write_aif(struct tda998x_priv *priv,
+			     struct hdmi_audio_infoframe *cea)
 {
 	union hdmi_infoframe frame;
 
-	hdmi_audio_infoframe_init(&frame.audio);
-
-	frame.audio.channels = p->audio_frame[1] & 0x07;
-	frame.audio.channel_allocation = p->audio_frame[4];
-	frame.audio.level_shift_value = (p->audio_frame[5] & 0x78) >> 3;
-	frame.audio.downmix_inhibit = (p->audio_frame[5] & 0x80) >> 7;
-
-	/*
-	 * L-PCM and IEC61937 compressed audio shall always set sample
-	 * frequency to "refer to stream".  For others, see the HDMI
-	 * specification.
-	 */
-	frame.audio.sample_frequency = (p->audio_frame[2] & 0x1c) >> 2;
+	frame.audio = *cea;
 
 	tda998x_write_if(priv, DIP_IF_FLAGS_IF4, REG_IF4_HB0, &frame);
+
+	return 0;
 }
 
 static void
@@ -710,20 +711,21 @@ static void tda998x_audio_mute(struct tda998x_priv *priv, bool on)
 	}
 }
 
-static void
+static int
 tda998x_configure_audio(struct tda998x_priv *priv,
-		struct drm_display_mode *mode, struct tda998x_encoder_params *p)
+			struct tda998x_audio_params *params,
+			unsigned mode_clock)
 {
 	u8 buf[6], clksel_aip, clksel_fs, cts_n, adiv;
 	u32 n;
 
 	/* Enable audio ports */
-	reg_write(priv, REG_ENA_AP, p->audio_cfg);
-	reg_write(priv, REG_ENA_ACLK, p->audio_clk_cfg);
+	reg_write(priv, REG_ENA_AP, params->config);
 
 	/* Set audio input source */
-	switch (p->audio_format) {
+	switch (params->format) {
 	case AFMT_SPDIF:
+		reg_write(priv, REG_ENA_ACLK, 0);
 		reg_write(priv, REG_MUX_AP, MUX_AP_SELECT_SPDIF);
 		clksel_aip = AIP_CLKSEL_AIP_SPDIF;
 		clksel_fs = AIP_CLKSEL_FS_FS64SPDIF;
@@ -731,15 +733,29 @@ tda998x_configure_audio(struct tda998x_priv *priv,
 		break;
 
 	case AFMT_I2S:
+		reg_write(priv, REG_ENA_ACLK, 1);
 		reg_write(priv, REG_MUX_AP, MUX_AP_SELECT_I2S);
 		clksel_aip = AIP_CLKSEL_AIP_I2S;
 		clksel_fs = AIP_CLKSEL_FS_ACLK;
-		cts_n = CTS_N_M(3) | CTS_N_K(3);
+		switch (params->sample_width) {
+		case 16:
+			cts_n = CTS_N_M(3) | CTS_N_K(1);
+			break;
+		case 18:
+		case 20:
+		case 24:
+			cts_n = CTS_N_M(3) | CTS_N_K(2);
+			break;
+		default:
+		case 32:
+			cts_n = CTS_N_M(3) | CTS_N_K(3);
+			break;
+		}
 		break;
 
 	default:
-		BUG();
-		return;
+		dev_err(&priv->hdmi->dev, "Unsupported I2S format\n");
+		return -EINVAL;
 	}
 
 	reg_write(priv, REG_AIP_CLKSEL, clksel_aip);
@@ -755,11 +771,11 @@ tda998x_configure_audio(struct tda998x_priv *priv,
 	 * assume 100MHz requires larger divider.
 	 */
 	adiv = AUDIO_DIV_SERCLK_8;
-	if (mode->clock > 100000)
+	if (mode_clock > 100000)
 		adiv++;			/* AUDIO_DIV_SERCLK_16 */
 
 	/* S/PDIF asks for a larger divider */
-	if (p->audio_format == AFMT_SPDIF)
+	if (params->format == AFMT_SPDIF)
 		adiv++;			/* AUDIO_DIV_SERCLK_16 or _32 */
 
 	reg_write(priv, REG_AUDIO_DIV, adiv);
@@ -768,7 +784,7 @@ tda998x_configure_audio(struct tda998x_priv *priv,
 	 * This is the approximate value of N, which happens to be
 	 * the recommended values for non-coherent clocks.
 	 */
-	n = 128 * p->audio_sample_rate / 1000;
+	n = 128 * params->sample_rate / 1000;
 
 	/* Write the CTS and N values */
 	buf[0] = 0x44;
@@ -786,20 +802,21 @@ tda998x_configure_audio(struct tda998x_priv *priv,
 	reg_set(priv, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_CTS);
 	reg_clear(priv, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_CTS);
 
-	/* Write the channel status */
-	buf[0] = IEC958_AES0_CON_NOT_COPYRIGHT;
-	buf[1] = 0x00;
-	buf[2] = IEC958_AES3_CON_FS_NOTID;
-	buf[3] = IEC958_AES4_CON_ORIGFS_NOTID |
-			IEC958_AES4_CON_MAX_WORDLEN_24;
+	/* Write the channel status
+	 * The REG_CH_STAT_B-registers skip IEC958 AES2 byte, because
+	 * there is a separate register for each I2S wire.
+	 */
+	buf[0] = params->status[0];
+	buf[1] = params->status[1];
+	buf[2] = params->status[3];
+	buf[3] = params->status[4];
 	reg_write_range(priv, REG_CH_STAT_B(0), buf, 4);
 
 	tda998x_audio_mute(priv, true);
 	msleep(20);
 	tda998x_audio_mute(priv, false);
 
-	/* Write the audio information packet */
-	tda998x_write_aif(priv, p);
+	return tda998x_write_aif(priv, &params->cea);
 }
 
 /* DRM encoder functions */
@@ -820,7 +837,7 @@ static void tda998x_encoder_set_config(struct tda998x_priv *priv,
 			    VIP_CNTRL_2_SWAP_F(p->swap_f) |
 			    (p->mirr_f ? VIP_CNTRL_2_MIRR_F : 0);
 
-	priv->params = *p;
+	priv->audio_params = p->audio_params;
 }
 
 static void tda998x_encoder_dpms(struct drm_encoder *encoder, int mode)
@@ -1057,9 +1074,13 @@ tda998x_encoder_mode_set(struct drm_encoder *encoder,
 
 		tda998x_write_avi(priv, adjusted_mode);
 
-		if (priv->params.audio_cfg)
-			tda998x_configure_audio(priv, adjusted_mode,
-						&priv->params);
+		if (priv->audio_params.format != AFMT_UNUSED) {
+			mutex_lock(&priv->audio_mutex);
+			tda998x_configure_audio(priv,
+						&priv->audio_params,
+						adjusted_mode->clock);
+			mutex_unlock(&priv->audio_mutex);
+		}
 	}
 }
 
@@ -1159,6 +1180,8 @@ static int tda998x_connector_get_modes(struct drm_connector *connector)
 	drm_mode_connector_update_edid_property(connector, edid);
 	n = drm_add_edid_modes(connector, edid);
 	priv->is_hdmi_sink = drm_detect_hdmi_monitor(edid);
+	drm_edid_to_eld(connector, edid);
+
 	kfree(edid);
 
 	return n;
@@ -1180,6 +1203,9 @@ static void tda998x_destroy(struct tda998x_priv *priv)
 	cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
 	reg_clear(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
 
+	if (priv->audio_pdev)
+		platform_device_unregister(priv->audio_pdev);
+
 	if (priv->hdmi->irq)
 		free_irq(priv->hdmi->irq, priv);
 
@@ -1189,8 +1215,189 @@ static void tda998x_destroy(struct tda998x_priv *priv)
 	i2c_unregister_device(priv->cec);
 }
 
+static int tda998x_audio_hw_params(struct device *dev, void *data,
+				   struct hdmi_codec_daifmt *daifmt,
+				   struct hdmi_codec_params *params)
+{
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
+	int i, ret;
+	struct tda998x_audio_params audio = {
+		.sample_width = params->sample_width,
+		.sample_rate = params->sample_rate,
+		.cea = params->cea,
+	};
+
+	if (!priv->encoder.crtc)
+		return -ENODEV;
+
+	memcpy(audio.status, params->iec.status,
+	       min(sizeof(audio.status), sizeof(params->iec.status)));
+
+	switch (daifmt->fmt) {
+	case HDMI_I2S:
+		if (daifmt->bit_clk_inv || daifmt->frame_clk_inv ||
+		    daifmt->bit_clk_master || daifmt->frame_clk_master) {
+			dev_err(dev, "%s: Bad flags %d %d %d %d\n", __func__,
+				daifmt->bit_clk_inv, daifmt->frame_clk_inv,
+				daifmt->bit_clk_master,
+				daifmt->frame_clk_master);
+			return -EINVAL;
+		}
+		for (i = 0; i < ARRAY_SIZE(priv->audio_port); i++)
+			if (priv->audio_port[i].format == AFMT_I2S)
+				audio.config = priv->audio_port[i].config;
+		audio.format = AFMT_I2S;
+		break;
+	case HDMI_SPDIF:
+		for (i = 0; i < ARRAY_SIZE(priv->audio_port); i++)
+			if (priv->audio_port[i].format == AFMT_SPDIF)
+				audio.config = priv->audio_port[i].config;
+		audio.format = AFMT_SPDIF;
+		break;
+	default:
+		dev_err(dev, "%s: Invalid format %d\n", __func__, daifmt->fmt);
+		return -EINVAL;
+	}
+
+	if (audio.config == 0) {
+		dev_err(dev, "%s: No audio configutation found\n", __func__);
+		return -EINVAL;
+	}
+
+	mutex_lock(&priv->audio_mutex);
+	ret = tda998x_configure_audio(priv,
+				      &audio,
+				      priv->encoder.crtc->hwmode.clock);
+
+	if (ret == 0)
+		priv->audio_params = audio;
+	mutex_unlock(&priv->audio_mutex);
+
+	return ret;
+}
+
+static void tda998x_audio_shutdown(struct device *dev, void *data)
+{
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
+
+	mutex_lock(&priv->audio_mutex);
+
+	reg_write(priv, REG_ENA_AP, 0);
+
+	priv->audio_params.format = AFMT_UNUSED;
+
+	mutex_unlock(&priv->audio_mutex);
+}
+
+int tda998x_audio_digital_mute(struct device *dev, void *data, bool enable)
+{
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
+
+	mutex_lock(&priv->audio_mutex);
+
+	tda998x_audio_mute(priv, enable);
+
+	mutex_unlock(&priv->audio_mutex);
+	return 0;
+}
+
+static int tda998x_audio_get_eld(struct device *dev, void *data,
+				 uint8_t *buf, size_t len)
+{
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
+	struct drm_mode_config *config = &priv->encoder.dev->mode_config;
+	struct drm_connector *connector;
+	int ret = -ENODEV;
+
+	mutex_lock(&config->mutex);
+	list_for_each_entry(connector, &config->connector_list, head) {
+		if (&priv->encoder == connector->encoder) {
+			memcpy(buf, connector->eld,
+			       min(sizeof(connector->eld), len));
+			ret = 0;
+		}
+	}
+	mutex_unlock(&config->mutex);
+
+	return ret;
+}
+
+static const struct hdmi_codec_ops audio_codec_ops = {
+	.hw_params = tda998x_audio_hw_params,
+	.audio_shutdown = tda998x_audio_shutdown,
+	.digital_mute = tda998x_audio_digital_mute,
+	.get_eld = tda998x_audio_get_eld,
+};
+
+static int tda998x_audio_codec_init(struct tda998x_priv *priv,
+				    struct device *dev)
+{
+	struct hdmi_codec_pdata codec_data = {
+		.ops = &audio_codec_ops,
+		.max_i2s_channels = 2,
+	};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(priv->audio_port); i++) {
+		if (priv->audio_port[i].format == AFMT_I2S &&
+		    priv->audio_port[i].config != 0)
+			codec_data.i2s = 1;
+		if (priv->audio_port[i].format == AFMT_SPDIF &&
+		    priv->audio_port[i].config != 0)
+			codec_data.spdif = 1;
+	}
+
+	priv->audio_pdev = platform_device_register_data(
+		dev, HDMI_CODEC_DRV_NAME, PLATFORM_DEVID_AUTO,
+		&codec_data, sizeof(codec_data));
+
+	return PTR_ERR_OR_ZERO(priv->audio_pdev);
+}
+
 /* I2C driver functions */
 
+static int tda998x_get_audio_ports(struct tda998x_priv *priv,
+				   struct device_node *np)
+{
+	const u32 *port_data;
+	u32 size;
+	int i;
+
+	port_data = of_get_property(np, "audio-ports", &size);
+	if (!port_data)
+		return 0;
+
+	size /= sizeof(u32);
+	if (size > 2 * ARRAY_SIZE(priv->audio_port) || size % 2 != 0) {
+		dev_err(&priv->hdmi->dev,
+			"Bad number of elements in audio-ports dt-property\n");
+		return -EINVAL;
+	}
+
+	size /= 2;
+
+	for (i = 0; i < size; i++) {
+		u8 afmt = be32_to_cpup(&port_data[2*i]);
+		u8 ena_ap = be32_to_cpup(&port_data[2*i+1]);
+
+		if (afmt != AFMT_SPDIF && afmt != AFMT_I2S) {
+			dev_err(&priv->hdmi->dev,
+				"Bad audio format %u\n", afmt);
+			return -EINVAL;
+		}
+
+		priv->audio_port[i].format = afmt;
+		priv->audio_port[i].config = ena_ap;
+	}
+
+	if (priv->audio_port[0].format == priv->audio_port[1].format) {
+		dev_err(&priv->hdmi->dev,
+			"There can only be on I2S port and one SPDIF port\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
 static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 {
 	struct device_node *np = client->dev.of_node;
@@ -1304,7 +1511,7 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 	if (!np)
 		return 0;		/* non-DT */
 
-	/* get the optional video properties */
+	/* get the device tree parameters */
 	ret = of_property_read_u32(np, "video-ports", &video);
 	if (ret == 0) {
 		priv->vip_cntrl_0 = video >> 16;
@@ -1312,8 +1519,16 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 		priv->vip_cntrl_2 = video;
 	}
 
-	return 0;
+	mutex_init(&priv->audio_mutex); /* Protect access from audio thread */
 
+	ret = tda998x_get_audio_ports(priv, np);
+	if (ret)
+		goto fail;
+
+	if (priv->audio_port[0].format != AFMT_UNUSED)
+		tda998x_audio_codec_init(priv, &client->dev);
+
+	return 0;
 fail:
 	/* if encoder_init fails, the encoder slave is never registered,
 	 * so cleanup here:
diff --git a/drivers/gpu/drm/i810/i810_drv.c b/drivers/gpu/drm/i810/i810_drv.c
index 44f4a131c8dd..0be55dc1ef4b 100644
--- a/drivers/gpu/drm/i810/i810_drv.c
+++ b/drivers/gpu/drm/i810/i810_drv.c
@@ -56,9 +56,7 @@ static const struct file_operations i810_driver_fops = {
 };
 
 static struct drm_driver driver = {
-	.driver_features =
-	    DRIVER_USE_AGP |
-	    DRIVER_HAVE_DMA,
+	.driver_features = DRIVER_USE_AGP | DRIVER_HAVE_DMA | DRIVER_LEGACY,
 	.dev_priv_size = sizeof(drm_i810_buf_priv_t),
 	.load = i810_driver_load,
 	.lastclose = i810_driver_lastclose,
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 684fc1cd08fa..a998c2bce70a 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -3,15 +3,20 @@
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
 subdir-ccflags-$(CONFIG_DRM_I915_WERROR) := -Werror
+subdir-ccflags-y += \
+	$(call as-instr,movntdqa (%eax)$(comma)%xmm0,-DCONFIG_AS_MOVNTDQA)
 
 # Please keep these build lists sorted!
 
 # core driver code
 i915-y := i915_drv.o \
 	  i915_irq.o \
+	  i915_memcpy.o \
+	  i915_mm.o \
 	  i915_params.o \
 	  i915_pci.o \
           i915_suspend.o \
+	  i915_sw_fence.o \
 	  i915_sysfs.o \
 	  intel_csr.o \
 	  intel_device_info.o \
@@ -25,7 +30,6 @@ i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o
 i915-y += i915_cmd_parser.o \
 	  i915_gem_batch_pool.o \
 	  i915_gem_context.o \
-	  i915_gem_debug.o \
 	  i915_gem_dmabuf.o \
 	  i915_gem_evict.o \
 	  i915_gem_execbuffer.o \
@@ -33,6 +37,7 @@ i915-y += i915_cmd_parser.o \
 	  i915_gem_gtt.o \
 	  i915_gem.o \
 	  i915_gem_render_state.o \
+	  i915_gem_request.o \
 	  i915_gem_shrinker.o \
 	  i915_gem_stolen.o \
 	  i915_gem_tiling.o \
@@ -40,6 +45,7 @@ i915-y += i915_cmd_parser.o \
 	  i915_gpu_error.o \
 	  i915_trace_points.o \
 	  intel_breadcrumbs.o \
+	  intel_engine_cs.o \
 	  intel_lrc.o \
 	  intel_mocs.o \
 	  intel_ringbuffer.o \
@@ -109,6 +115,6 @@ i915-y += intel_gvt.o
 include $(src)/gvt/Makefile
 endif
 
-obj-$(CONFIG_DRM_I915)  += i915.o
+obj-$(CONFIG_DRM_I915) += i915.o
 
 CFLAGS_i915_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index b0fd6a7b0603..70980f82a15b 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -62,23 +62,23 @@
  * The parser always rejects such commands.
  *
  * The majority of the problematic commands fall in the MI_* range, with only a
- * few specific commands on each ring (e.g. PIPE_CONTROL and MI_FLUSH_DW).
+ * few specific commands on each engine (e.g. PIPE_CONTROL and MI_FLUSH_DW).
  *
  * Implementation:
- * Each ring maintains tables of commands and registers which the parser uses in
- * scanning batch buffers submitted to that ring.
+ * Each engine maintains tables of commands and registers which the parser
+ * uses in scanning batch buffers submitted to that engine.
  *
  * Since the set of commands that the parser must check for is significantly
  * smaller than the number of commands supported, the parser tables contain only
  * those commands required by the parser. This generally works because command
  * opcode ranges have standard command length encodings. So for commands that
  * the parser does not need to check, it can easily skip them. This is
- * implemented via a per-ring length decoding vfunc.
+ * implemented via a per-engine length decoding vfunc.
  *
  * Unfortunately, there are a number of commands that do not follow the standard
  * length encoding for their opcode range, primarily amongst the MI_* commands.
  * To handle this, the parser provides a way to define explicit "skip" entries
- * in the per-ring command tables.
+ * in the per-engine command tables.
  *
  * Other command table entries map fairly directly to high level categories
  * mentioned above: rejected, master-only, register whitelist. The parser
@@ -86,24 +86,25 @@
  * general bitmasking mechanism.
  */
 
-#define STD_MI_OPCODE_MASK  0xFF800000
-#define STD_3D_OPCODE_MASK  0xFFFF0000
-#define STD_2D_OPCODE_MASK  0xFFC00000
-#define STD_MFX_OPCODE_MASK 0xFFFF0000
+#define STD_MI_OPCODE_SHIFT  (32 - 9)
+#define STD_3D_OPCODE_SHIFT  (32 - 16)
+#define STD_2D_OPCODE_SHIFT  (32 - 10)
+#define STD_MFX_OPCODE_SHIFT (32 - 16)
+#define MIN_OPCODE_SHIFT 16
 
 #define CMD(op, opm, f, lm, fl, ...)				\
 	{							\
 		.flags = (fl) | ((f) ? CMD_DESC_FIXED : 0),	\
-		.cmd = { (op), (opm) },				\
+		.cmd = { (op), ~0u << (opm) },			\
 		.length = { (lm) },				\
 		__VA_ARGS__					\
 	}
 
 /* Convenience macros to compress the tables */
-#define SMI STD_MI_OPCODE_MASK
-#define S3D STD_3D_OPCODE_MASK
-#define S2D STD_2D_OPCODE_MASK
-#define SMFX STD_MFX_OPCODE_MASK
+#define SMI STD_MI_OPCODE_SHIFT
+#define S3D STD_3D_OPCODE_SHIFT
+#define S2D STD_2D_OPCODE_SHIFT
+#define SMFX STD_MFX_OPCODE_SHIFT
 #define F true
 #define S CMD_DESC_SKIP
 #define R CMD_DESC_REJECT
@@ -350,6 +351,9 @@ static const struct drm_i915_cmd_descriptor hsw_blt_cmds[] = {
 	CMD(  MI_LOAD_SCAN_LINES_EXCL,          SMI,   !F,  0x3F,   R  ),
 };
 
+static const struct drm_i915_cmd_descriptor noop_desc =
+	CMD(MI_NOOP, SMI, F, 1, S);
+
 #undef CMD
 #undef SMI
 #undef S3D
@@ -458,6 +462,7 @@ static const struct drm_i915_reg_descriptor gen7_render_regs[] = {
 	REG32(GEN7_GPGPU_DISPATCHDIMX),
 	REG32(GEN7_GPGPU_DISPATCHDIMY),
 	REG32(GEN7_GPGPU_DISPATCHDIMZ),
+	REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),
 	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 0),
 	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 1),
 	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 2),
@@ -473,6 +478,7 @@ static const struct drm_i915_reg_descriptor gen7_render_regs[] = {
 	REG32(GEN7_L3SQCREG1),
 	REG32(GEN7_L3CNTLREG2),
 	REG32(GEN7_L3CNTLREG3),
+	REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
 };
 
 static const struct drm_i915_reg_descriptor hsw_render_regs[] = {
@@ -502,7 +508,10 @@ static const struct drm_i915_reg_descriptor hsw_render_regs[] = {
 };
 
 static const struct drm_i915_reg_descriptor gen7_blt_regs[] = {
+	REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE),
+	REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),
 	REG32(BCS_SWCTRL),
+	REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
 };
 
 static const struct drm_i915_reg_descriptor ivb_master_regs[] = {
@@ -603,7 +612,7 @@ static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header)
 	return 0;
 }
 
-static bool validate_cmds_sorted(struct intel_engine_cs *engine,
+static bool validate_cmds_sorted(const struct intel_engine_cs *engine,
 				 const struct drm_i915_cmd_table *cmd_tables,
 				 int cmd_table_count)
 {
@@ -624,8 +633,10 @@ static bool validate_cmds_sorted(struct intel_engine_cs *engine,
 			u32 curr = desc->cmd.value & desc->cmd.mask;
 
 			if (curr < previous) {
-				DRM_ERROR("CMD: table not sorted ring=%d table=%d entry=%d cmd=0x%08X prev=0x%08X\n",
-					  engine->id, i, j, curr, previous);
+				DRM_ERROR("CMD: %s [%d] command table not sorted: "
+					  "table=%d entry=%d cmd=0x%08X prev=0x%08X\n",
+					  engine->name, engine->id,
+					  i, j, curr, previous);
 				ret = false;
 			}
 
@@ -636,7 +647,7 @@ static bool validate_cmds_sorted(struct intel_engine_cs *engine,
 	return ret;
 }
 
-static bool check_sorted(int ring_id,
+static bool check_sorted(const struct intel_engine_cs *engine,
 			 const struct drm_i915_reg_descriptor *reg_table,
 			 int reg_count)
 {
@@ -648,8 +659,10 @@ static bool check_sorted(int ring_id,
 		u32 curr = i915_mmio_reg_offset(reg_table[i].addr);
 
 		if (curr < previous) {
-			DRM_ERROR("CMD: table not sorted ring=%d entry=%d reg=0x%08X prev=0x%08X\n",
-				  ring_id, i, curr, previous);
+			DRM_ERROR("CMD: %s [%d] register table not sorted: "
+				  "entry=%d reg=0x%08X prev=0x%08X\n",
+				  engine->name, engine->id,
+				  i, curr, previous);
 			ret = false;
 		}
 
@@ -666,7 +679,7 @@ static bool validate_regs_sorted(struct intel_engine_cs *engine)
 
 	for (i = 0; i < engine->reg_table_count; i++) {
 		table = &engine->reg_tables[i];
-		if (!check_sorted(engine->id, table->regs, table->num_regs))
+		if (!check_sorted(engine, table->regs, table->num_regs))
 			return false;
 	}
 
@@ -687,12 +700,26 @@ struct cmd_node {
  * non-opcode bits being set. But if we don't include those bits, some 3D
  * commands may hash to the same bucket due to not including opcode bits that
  * make the command unique. For now, we will risk hashing to the same bucket.
- *
- * If we attempt to generate a perfect hash, we should be able to look at bits
- * 31:29 of a command from a batch buffer and use the full mask for that
- * client. The existing INSTR_CLIENT_MASK/SHIFT defines can be used for this.
  */
-#define CMD_HASH_MASK STD_MI_OPCODE_MASK
+static inline u32 cmd_header_key(u32 x)
+{
+	u32 shift;
+
+	switch (x >> INSTR_CLIENT_SHIFT) {
+	default:
+	case INSTR_MI_CLIENT:
+		shift = STD_MI_OPCODE_SHIFT;
+		break;
+	case INSTR_RC_CLIENT:
+		shift = STD_3D_OPCODE_SHIFT;
+		break;
+	case INSTR_BC_CLIENT:
+		shift = STD_2D_OPCODE_SHIFT;
+		break;
+	}
+
+	return x >> shift;
+}
 
 static int init_hash_table(struct intel_engine_cs *engine,
 			   const struct drm_i915_cmd_table *cmd_tables,
@@ -716,7 +743,7 @@ static int init_hash_table(struct intel_engine_cs *engine,
 
 			desc_node->desc = desc;
 			hash_add(engine->cmd_hash, &desc_node->node,
-				 desc->cmd.value & CMD_HASH_MASK);
+				 cmd_header_key(desc->cmd.value));
 		}
 	}
 
@@ -736,23 +763,21 @@ static void fini_hash_table(struct intel_engine_cs *engine)
 }
 
 /**
- * i915_cmd_parser_init_ring() - set cmd parser related fields for a ringbuffer
+ * intel_engine_init_cmd_parser() - set cmd parser related fields for an engine
  * @engine: the engine to initialize
  *
  * Optionally initializes fields related to batch buffer command parsing in the
  * struct intel_engine_cs based on whether the platform requires software
  * command parsing.
- *
- * Return: non-zero if initialization fails
  */
-int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
+void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
 {
 	const struct drm_i915_cmd_table *cmd_tables;
 	int cmd_table_count;
 	int ret;
 
 	if (!IS_GEN7(engine->i915))
-		return 0;
+		return;
 
 	switch (engine->id) {
 	case RCS:
@@ -806,36 +831,38 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
 		engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
 		break;
 	default:
-		DRM_ERROR("CMD: cmd_parser_init with unknown ring: %d\n",
-			  engine->id);
-		BUG();
+		MISSING_CASE(engine->id);
+		return;
 	}
 
-	BUG_ON(!validate_cmds_sorted(engine, cmd_tables, cmd_table_count));
-	BUG_ON(!validate_regs_sorted(engine));
-
-	WARN_ON(!hash_empty(engine->cmd_hash));
+	if (!validate_cmds_sorted(engine, cmd_tables, cmd_table_count)) {
+		DRM_ERROR("%s: command descriptions are not sorted\n",
+			  engine->name);
+		return;
+	}
+	if (!validate_regs_sorted(engine)) {
+		DRM_ERROR("%s: registers are not sorted\n", engine->name);
+		return;
+	}
 
 	ret = init_hash_table(engine, cmd_tables, cmd_table_count);
 	if (ret) {
-		DRM_ERROR("CMD: cmd_parser_init failed!\n");
+		DRM_ERROR("%s: initialised failed!\n", engine->name);
 		fini_hash_table(engine);
-		return ret;
+		return;
 	}
 
 	engine->needs_cmd_parser = true;
-
-	return 0;
 }
 
 /**
- * i915_cmd_parser_fini_ring() - clean up cmd parser related fields
+ * intel_engine_cleanup_cmd_parser() - clean up cmd parser related fields
  * @engine: the engine to clean up
  *
  * Releases any resources related to command parsing that may have been
- * initialized for the specified ring.
+ * initialized for the specified engine.
  */
-void i915_cmd_parser_fini_ring(struct intel_engine_cs *engine)
+void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine)
 {
 	if (!engine->needs_cmd_parser)
 		return;
@@ -850,12 +877,9 @@ find_cmd_in_table(struct intel_engine_cs *engine,
 	struct cmd_node *desc_node;
 
 	hash_for_each_possible(engine->cmd_hash, desc_node, node,
-			       cmd_header & CMD_HASH_MASK) {
+			       cmd_header_key(cmd_header)) {
 		const struct drm_i915_cmd_descriptor *desc = desc_node->desc;
-		u32 masked_cmd = desc->cmd.mask & cmd_header;
-		u32 masked_value = desc->cmd.value & desc->cmd.mask;
-
-		if (masked_cmd == masked_value)
+		if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0)
 			return desc;
 	}
 
@@ -866,18 +890,21 @@ find_cmd_in_table(struct intel_engine_cs *engine,
  * Returns a pointer to a descriptor for the command specified by cmd_header.
  *
  * The caller must supply space for a default descriptor via the default_desc
- * parameter. If no descriptor for the specified command exists in the ring's
+ * parameter. If no descriptor for the specified command exists in the engine's
  * command parser tables, this function fills in default_desc based on the
- * ring's default length encoding and returns default_desc.
+ * engine's default length encoding and returns default_desc.
  */
 static const struct drm_i915_cmd_descriptor*
 find_cmd(struct intel_engine_cs *engine,
 	 u32 cmd_header,
+	 const struct drm_i915_cmd_descriptor *desc,
 	 struct drm_i915_cmd_descriptor *default_desc)
 {
-	const struct drm_i915_cmd_descriptor *desc;
 	u32 mask;
 
+	if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0)
+		return desc;
+
 	desc = find_cmd_in_table(engine, cmd_header);
 	if (desc)
 		return desc;
@@ -886,152 +913,140 @@ find_cmd(struct intel_engine_cs *engine,
 	if (!mask)
 		return NULL;
 
-	BUG_ON(!default_desc);
-	default_desc->flags = CMD_DESC_SKIP;
+	default_desc->cmd.value = cmd_header;
+	default_desc->cmd.mask = ~0u << MIN_OPCODE_SHIFT;
 	default_desc->length.mask = mask;
-
+	default_desc->flags = CMD_DESC_SKIP;
 	return default_desc;
 }
 
 static const struct drm_i915_reg_descriptor *
-find_reg(const struct drm_i915_reg_descriptor *table,
-	 int count, u32 addr)
+__find_reg(const struct drm_i915_reg_descriptor *table, int count, u32 addr)
 {
-	int i;
-
-	for (i = 0; i < count; i++) {
-		if (i915_mmio_reg_offset(table[i].addr) == addr)
-			return &table[i];
+	int start = 0, end = count;
+	while (start < end) {
+		int mid = start + (end - start) / 2;
+		int ret = addr - i915_mmio_reg_offset(table[mid].addr);
+		if (ret < 0)
+			end = mid;
+		else if (ret > 0)
+			start = mid + 1;
+		else
+			return &table[mid];
 	}
-
 	return NULL;
 }
 
 static const struct drm_i915_reg_descriptor *
-find_reg_in_tables(const struct drm_i915_reg_table *tables,
-		   int count, bool is_master, u32 addr)
+find_reg(const struct intel_engine_cs *engine, bool is_master, u32 addr)
 {
-	int i;
-	const struct drm_i915_reg_table *table;
-	const struct drm_i915_reg_descriptor *reg;
+	const struct drm_i915_reg_table *table = engine->reg_tables;
+	int count = engine->reg_table_count;
 
-	for (i = 0; i < count; i++) {
-		table = &tables[i];
+	do {
 		if (!table->master || is_master) {
-			reg = find_reg(table->regs, table->num_regs,
-				       addr);
+			const struct drm_i915_reg_descriptor *reg;
+
+			reg = __find_reg(table->regs, table->num_regs, addr);
 			if (reg != NULL)
 				return reg;
 		}
-	}
+	} while (table++, --count);
 
 	return NULL;
 }
 
-static u32 *vmap_batch(struct drm_i915_gem_object *obj,
-		       unsigned start, unsigned len)
-{
-	int i;
-	void *addr = NULL;
-	struct sg_page_iter sg_iter;
-	int first_page = start >> PAGE_SHIFT;
-	int last_page = (len + start + 4095) >> PAGE_SHIFT;
-	int npages = last_page - first_page;
-	struct page **pages;
-
-	pages = drm_malloc_ab(npages, sizeof(*pages));
-	if (pages == NULL) {
-		DRM_DEBUG_DRIVER("Failed to get space for pages\n");
-		goto finish;
-	}
-
-	i = 0;
-	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, first_page) {
-		pages[i++] = sg_page_iter_page(&sg_iter);
-		if (i == npages)
-			break;
-	}
-
-	addr = vmap(pages, i, 0, PAGE_KERNEL);
-	if (addr == NULL) {
-		DRM_DEBUG_DRIVER("Failed to vmap pages\n");
-		goto finish;
-	}
-
-finish:
-	if (pages)
-		drm_free_large(pages);
-	return (u32*)addr;
-}
-
-/* Returns a vmap'd pointer to dest_obj, which the caller must unmap */
-static u32 *copy_batch(struct drm_i915_gem_object *dest_obj,
+/* Returns a vmap'd pointer to dst_obj, which the caller must unmap */
+static u32 *copy_batch(struct drm_i915_gem_object *dst_obj,
 		       struct drm_i915_gem_object *src_obj,
 		       u32 batch_start_offset,
-		       u32 batch_len)
+		       u32 batch_len,
+		       bool *needs_clflush_after)
 {
-	int needs_clflush = 0;
-	void *src_base, *src;
-	void *dst = NULL;
+	unsigned int src_needs_clflush;
+	unsigned int dst_needs_clflush;
+	void *dst, *src;
 	int ret;
 
-	if (batch_len > dest_obj->base.size ||
-	    batch_len + batch_start_offset > src_obj->base.size)
-		return ERR_PTR(-E2BIG);
-
-	if (WARN_ON(dest_obj->pages_pin_count == 0))
-		return ERR_PTR(-ENODEV);
-
-	ret = i915_gem_obj_prepare_shmem_read(src_obj, &needs_clflush);
-	if (ret) {
-		DRM_DEBUG_DRIVER("CMD: failed to prepare shadow batch\n");
+	ret = i915_gem_obj_prepare_shmem_read(src_obj, &src_needs_clflush);
+	if (ret)
 		return ERR_PTR(ret);
-	}
 
-	src_base = vmap_batch(src_obj, batch_start_offset, batch_len);
-	if (!src_base) {
-		DRM_DEBUG_DRIVER("CMD: Failed to vmap batch\n");
-		ret = -ENOMEM;
+	ret = i915_gem_obj_prepare_shmem_write(dst_obj, &dst_needs_clflush);
+	if (ret) {
+		dst = ERR_PTR(ret);
 		goto unpin_src;
 	}
 
-	ret = i915_gem_object_set_to_cpu_domain(dest_obj, true);
-	if (ret) {
-		DRM_DEBUG_DRIVER("CMD: Failed to set shadow batch to CPU\n");
-		goto unmap_src;
+	dst = i915_gem_object_pin_map(dst_obj, I915_MAP_WB);
+	if (IS_ERR(dst))
+		goto unpin_dst;
+
+	src = ERR_PTR(-ENODEV);
+	if (src_needs_clflush &&
+	    i915_memcpy_from_wc((void *)(uintptr_t)batch_start_offset, NULL, 0)) {
+		src = i915_gem_object_pin_map(src_obj, I915_MAP_WC);
+		if (!IS_ERR(src)) {
+			i915_memcpy_from_wc(dst,
+					    src + batch_start_offset,
+					    ALIGN(batch_len, 16));
+			i915_gem_object_unpin_map(src_obj);
+		}
 	}
-
-	dst = vmap_batch(dest_obj, 0, batch_len);
-	if (!dst) {
-		DRM_DEBUG_DRIVER("CMD: Failed to vmap shadow batch\n");
-		ret = -ENOMEM;
-		goto unmap_src;
+	if (IS_ERR(src)) {
+		void *ptr;
+		int offset, n;
+
+		offset = offset_in_page(batch_start_offset);
+
+		/* We can avoid clflushing partial cachelines before the write
+		 * if we only every write full cache-lines. Since we know that
+		 * both the source and destination are in multiples of
+		 * PAGE_SIZE, we can simply round up to the next cacheline.
+		 * We don't care about copying too much here as we only
+		 * validate up to the end of the batch.
+		 */
+		if (dst_needs_clflush & CLFLUSH_BEFORE)
+			batch_len = roundup(batch_len,
+					    boot_cpu_data.x86_clflush_size);
+
+		ptr = dst;
+		for (n = batch_start_offset >> PAGE_SHIFT; batch_len; n++) {
+			int len = min_t(int, batch_len, PAGE_SIZE - offset);
+
+			src = kmap_atomic(i915_gem_object_get_page(src_obj, n));
+			if (src_needs_clflush)
+				drm_clflush_virt_range(src + offset, len);
+			memcpy(ptr, src + offset, len);
+			kunmap_atomic(src);
+
+			ptr += len;
+			batch_len -= len;
+			offset = 0;
+		}
 	}
 
-	src = src_base + offset_in_page(batch_start_offset);
-	if (needs_clflush)
-		drm_clflush_virt_range(src, batch_len);
-
-	memcpy(dst, src, batch_len);
+	/* dst_obj is returned with vmap pinned */
+	*needs_clflush_after = dst_needs_clflush & CLFLUSH_AFTER;
 
-unmap_src:
-	vunmap(src_base);
+unpin_dst:
+	i915_gem_obj_finish_shmem_access(dst_obj);
 unpin_src:
-	i915_gem_object_unpin_pages(src_obj);
-
-	return ret ? ERR_PTR(ret) : dst;
+	i915_gem_obj_finish_shmem_access(src_obj);
+	return dst;
 }
 
 /**
- * i915_needs_cmd_parser() - should a given ring use software command parsing?
+ * intel_engine_needs_cmd_parser() - should a given engine use software
+ *                                   command parsing?
  * @engine: the engine in question
  *
  * Only certain platforms require software batch buffer command parsing, and
  * only when enabled via module parameter.
  *
- * Return: true if the ring requires software command parsing
+ * Return: true if the engine requires software command parsing
  */
-bool i915_needs_cmd_parser(struct intel_engine_cs *engine)
+bool intel_engine_needs_cmd_parser(struct intel_engine_cs *engine)
 {
 	if (!engine->needs_cmd_parser)
 		return false;
@@ -1048,6 +1063,9 @@ static bool check_cmd(const struct intel_engine_cs *engine,
 		      const bool is_master,
 		      bool *oacontrol_set)
 {
+	if (desc->flags & CMD_DESC_SKIP)
+		return true;
+
 	if (desc->flags & CMD_DESC_REJECT) {
 		DRM_DEBUG_DRIVER("CMD: Rejected command: 0x%08X\n", *cmd);
 		return false;
@@ -1072,14 +1090,11 @@ static bool check_cmd(const struct intel_engine_cs *engine,
 		     offset += step) {
 			const u32 reg_addr = cmd[offset] & desc->reg.mask;
 			const struct drm_i915_reg_descriptor *reg =
-				find_reg_in_tables(engine->reg_tables,
-						   engine->reg_table_count,
-						   is_master,
-						   reg_addr);
+				find_reg(engine, is_master, reg_addr);
 
 			if (!reg) {
-				DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (ring=%d)\n",
-						 reg_addr, *cmd, engine->id);
+				DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (exec_id=%d)\n",
+						 reg_addr, *cmd, engine->exec_id);
 				return false;
 			}
 
@@ -1159,11 +1174,11 @@ static bool check_cmd(const struct intel_engine_cs *engine,
 				desc->bits[i].mask;
 
 			if (dword != desc->bits[i].expected) {
-				DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (ring=%d)\n",
+				DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (exec_id=%d)\n",
 						 *cmd,
 						 desc->bits[i].mask,
 						 desc->bits[i].expected,
-						 dword, engine->id);
+						 dword, engine->exec_id);
 				return false;
 			}
 		}
@@ -1189,23 +1204,26 @@ static bool check_cmd(const struct intel_engine_cs *engine,
  * Return: non-zero if the parser finds violations or otherwise fails; -EACCES
  * if the batch appears legal but should use hardware parsing
  */
-int i915_parse_cmds(struct intel_engine_cs *engine,
-		    struct drm_i915_gem_object *batch_obj,
-		    struct drm_i915_gem_object *shadow_batch_obj,
-		    u32 batch_start_offset,
-		    u32 batch_len,
-		    bool is_master)
+int intel_engine_cmd_parser(struct intel_engine_cs *engine,
+			    struct drm_i915_gem_object *batch_obj,
+			    struct drm_i915_gem_object *shadow_batch_obj,
+			    u32 batch_start_offset,
+			    u32 batch_len,
+			    bool is_master)
 {
-	u32 *cmd, *batch_base, *batch_end;
-	struct drm_i915_cmd_descriptor default_desc = { 0 };
+	u32 *cmd, *batch_end;
+	struct drm_i915_cmd_descriptor default_desc = noop_desc;
+	const struct drm_i915_cmd_descriptor *desc = &default_desc;
 	bool oacontrol_set = false; /* OACONTROL tracking. See check_cmd() */
+	bool needs_clflush_after = false;
 	int ret = 0;
 
-	batch_base = copy_batch(shadow_batch_obj, batch_obj,
-				batch_start_offset, batch_len);
-	if (IS_ERR(batch_base)) {
+	cmd = copy_batch(shadow_batch_obj, batch_obj,
+			 batch_start_offset, batch_len,
+			 &needs_clflush_after);
+	if (IS_ERR(cmd)) {
 		DRM_DEBUG_DRIVER("CMD: Failed to copy batch\n");
-		return PTR_ERR(batch_base);
+		return PTR_ERR(cmd);
 	}
 
 	/*
@@ -1213,17 +1231,14 @@ int i915_parse_cmds(struct intel_engine_cs *engine,
 	 * large or larger and copy_batch() will write MI_NOPs to the extra
 	 * space. Parsing should be faster in some cases this way.
 	 */
-	batch_end = batch_base + (batch_len / sizeof(*batch_end));
-
-	cmd = batch_base;
+	batch_end = cmd + (batch_len / sizeof(*batch_end));
 	while (cmd < batch_end) {
-		const struct drm_i915_cmd_descriptor *desc;
 		u32 length;
 
 		if (*cmd == MI_BATCH_BUFFER_END)
 			break;
 
-		desc = find_cmd(engine, *cmd, &default_desc);
+		desc = find_cmd(engine, *cmd, desc, &default_desc);
 		if (!desc) {
 			DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n",
 					 *cmd);
@@ -1274,7 +1289,9 @@ int i915_parse_cmds(struct intel_engine_cs *engine,
 		ret = -EINVAL;
 	}
 
-	vunmap(batch_base);
+	if (ret == 0 && needs_clflush_after)
+		drm_clflush_virt_range(shadow_batch_obj->mapping, batch_len);
+	i915_gem_object_unpin_map(shadow_batch_obj);
 
 	return ret;
 }
@@ -1295,7 +1312,7 @@ int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
 
 	/* If the command parser is not enabled, report 0 - unsupported */
 	for_each_engine(engine, dev_priv) {
-		if (i915_needs_cmd_parser(engine)) {
+		if (intel_engine_needs_cmd_parser(engine)) {
 			active = true;
 			break;
 		}
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 844fea795bae..27b0e34dadec 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -40,11 +40,10 @@
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 
-enum {
-	ACTIVE_LIST,
-	INACTIVE_LIST,
-	PINNED_LIST,
-};
+static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
+{
+	return to_i915(node->minor->dev);
+}
 
 /* As the drm_debugfs_init() routines are called before dev->dev_private is
  * allocated we need to hook into the minor for release. */
@@ -63,7 +62,7 @@ drm_add_fake_info_node(struct drm_minor *minor,
 
 	node->minor = minor;
 	node->dent = ent;
-	node->info_ent = (void *) key;
+	node->info_ent = (void *)key;
 
 	mutex_lock(&minor->debugfs_lock);
 	list_add(&node->list, &minor->debugfs_list);
@@ -74,12 +73,11 @@ drm_add_fake_info_node(struct drm_minor *minor,
 
 static int i915_capabilities(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	const struct intel_device_info *info = INTEL_INFO(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	const struct intel_device_info *info = INTEL_INFO(dev_priv);
 
-	seq_printf(m, "gen: %d\n", info->gen);
-	seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev));
+	seq_printf(m, "gen: %d\n", INTEL_GEN(dev_priv));
+	seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev_priv));
 #define PRINT_FLAG(x)  seq_printf(m, #x ": %s\n", yesno(info->x))
 #define SEP_SEMICOLON ;
 	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_SEMICOLON);
@@ -91,7 +89,7 @@ static int i915_capabilities(struct seq_file *m, void *data)
 
 static char get_active_flag(struct drm_i915_gem_object *obj)
 {
-	return obj->active ? '*' : ' ';
+	return i915_gem_object_is_active(obj) ? '*' : ' ';
 }
 
 static char get_pin_flag(struct drm_i915_gem_object *obj)
@@ -101,7 +99,7 @@ static char get_pin_flag(struct drm_i915_gem_object *obj)
 
 static char get_tiling_flag(struct drm_i915_gem_object *obj)
 {
-	switch (obj->tiling_mode) {
+	switch (i915_gem_object_get_tiling(obj)) {
 	default:
 	case I915_TILING_NONE: return ' ';
 	case I915_TILING_X: return 'X';
@@ -111,7 +109,7 @@ static char get_tiling_flag(struct drm_i915_gem_object *obj)
 
 static char get_global_flag(struct drm_i915_gem_object *obj)
 {
-	return i915_gem_obj_to_ggtt(obj) ? 'g' : ' ';
+	return i915_gem_object_to_ggtt(obj, NULL) ?  'g' : ' ';
 }
 
 static char get_pin_mapped_flag(struct drm_i915_gem_object *obj)
@@ -125,7 +123,7 @@ static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj)
 	struct i915_vma *vma;
 
 	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (vma->is_ggtt && drm_mm_node_allocated(&vma->node))
+		if (i915_vma_is_ggtt(vma) && drm_mm_node_allocated(&vma->node))
 			size += vma->node.size;
 	}
 
@@ -138,6 +136,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 	struct intel_engine_cs *engine;
 	struct i915_vma *vma;
+	unsigned int frontbuffer_bits;
 	int pin_count = 0;
 	enum intel_engine_id id;
 
@@ -155,30 +154,36 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 		   obj->base.write_domain);
 	for_each_engine_id(engine, dev_priv, id)
 		seq_printf(m, "%x ",
-				i915_gem_request_get_seqno(obj->last_read_req[id]));
-	seq_printf(m, "] %x %x%s%s%s",
-		   i915_gem_request_get_seqno(obj->last_write_req),
-		   i915_gem_request_get_seqno(obj->last_fenced_req),
-		   i915_cache_level_str(to_i915(obj->base.dev), obj->cache_level),
+			   i915_gem_active_get_seqno(&obj->last_read[id],
+						     &obj->base.dev->struct_mutex));
+	seq_printf(m, "] %x %s%s%s",
+		   i915_gem_active_get_seqno(&obj->last_write,
+					     &obj->base.dev->struct_mutex),
+		   i915_cache_level_str(dev_priv, obj->cache_level),
 		   obj->dirty ? " dirty" : "",
 		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
 	if (obj->base.name)
 		seq_printf(m, " (name: %d)", obj->base.name);
 	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (vma->pin_count > 0)
+		if (i915_vma_is_pinned(vma))
 			pin_count++;
 	}
 	seq_printf(m, " (pinned x %d)", pin_count);
 	if (obj->pin_display)
 		seq_printf(m, " (display)");
-	if (obj->fence_reg != I915_FENCE_REG_NONE)
-		seq_printf(m, " (fence: %d)", obj->fence_reg);
 	list_for_each_entry(vma, &obj->vma_list, obj_link) {
+		if (!drm_mm_node_allocated(&vma->node))
+			continue;
+
 		seq_printf(m, " (%sgtt offset: %08llx, size: %08llx",
-			   vma->is_ggtt ? "g" : "pp",
+			   i915_vma_is_ggtt(vma) ? "g" : "pp",
 			   vma->node.start, vma->node.size);
-		if (vma->is_ggtt)
+		if (i915_vma_is_ggtt(vma))
 			seq_printf(m, ", type: %u", vma->ggtt_view.type);
+		if (vma->fence)
+			seq_printf(m, " , fence: %d%s",
+				   vma->fence->id,
+				   i915_gem_active_isset(&vma->last_fence) ? "*" : "");
 		seq_puts(m, ")");
 	}
 	if (obj->stolen)
@@ -192,58 +197,15 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 		*t = '\0';
 		seq_printf(m, " (%s mappable)", s);
 	}
-	if (obj->last_write_req != NULL)
-		seq_printf(m, " (%s)",
-			   i915_gem_request_get_engine(obj->last_write_req)->name);
-	if (obj->frontbuffer_bits)
-		seq_printf(m, " (frontbuffer: 0x%03x)", obj->frontbuffer_bits);
-}
-
-static int i915_gem_object_list_info(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = m->private;
-	uintptr_t list = (uintptr_t) node->info_ent->data;
-	struct list_head *head;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct i915_vma *vma;
-	u64 total_obj_size, total_gtt_size;
-	int count, ret;
 
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
+	engine = i915_gem_active_get_engine(&obj->last_write,
+					    &dev_priv->drm.struct_mutex);
+	if (engine)
+		seq_printf(m, " (%s)", engine->name);
 
-	/* FIXME: the user of this interface might want more than just GGTT */
-	switch (list) {
-	case ACTIVE_LIST:
-		seq_puts(m, "Active:\n");
-		head = &ggtt->base.active_list;
-		break;
-	case INACTIVE_LIST:
-		seq_puts(m, "Inactive:\n");
-		head = &ggtt->base.inactive_list;
-		break;
-	default:
-		mutex_unlock(&dev->struct_mutex);
-		return -EINVAL;
-	}
-
-	total_obj_size = total_gtt_size = count = 0;
-	list_for_each_entry(vma, head, vm_link) {
-		seq_printf(m, "   ");
-		describe_obj(m, vma->obj);
-		seq_printf(m, "\n");
-		total_obj_size += vma->obj->base.size;
-		total_gtt_size += vma->node.size;
-		count++;
-	}
-	mutex_unlock(&dev->struct_mutex);
-
-	seq_printf(m, "Total %d objects, %llu bytes, %llu GTT size\n",
-		   count, total_obj_size, total_gtt_size);
-	return 0;
+	frontbuffer_bits = atomic_read(&obj->frontbuffer_bits);
+	if (frontbuffer_bits)
+		seq_printf(m, " (frontbuffer: 0x%03x)", frontbuffer_bits);
 }
 
 static int obj_rank_by_stolen(void *priv,
@@ -263,9 +225,8 @@ static int obj_rank_by_stolen(void *priv,
 
 static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct drm_i915_gem_object *obj;
 	u64 total_obj_size, total_gtt_size;
 	LIST_HEAD(stolen);
@@ -311,17 +272,6 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
 	return 0;
 }
 
-#define count_objects(list, member) do { \
-	list_for_each_entry(obj, list, member) { \
-		size += i915_gem_obj_total_ggtt_size(obj); \
-		++count; \
-		if (obj->map_and_fenceable) { \
-			mappable_size += i915_gem_obj_ggtt_size(obj); \
-			++mappable_count; \
-		} \
-	} \
-} while (0)
-
 struct file_stats {
 	struct drm_i915_file_private *file_priv;
 	unsigned long count;
@@ -338,46 +288,29 @@ static int per_file_stats(int id, void *ptr, void *data)
 
 	stats->count++;
 	stats->total += obj->base.size;
-
+	if (!obj->bind_count)
+		stats->unbound += obj->base.size;
 	if (obj->base.name || obj->base.dma_buf)
 		stats->shared += obj->base.size;
 
-	if (USES_FULL_PPGTT(obj->base.dev)) {
-		list_for_each_entry(vma, &obj->vma_list, obj_link) {
-			struct i915_hw_ppgtt *ppgtt;
-
-			if (!drm_mm_node_allocated(&vma->node))
-				continue;
+	list_for_each_entry(vma, &obj->vma_list, obj_link) {
+		if (!drm_mm_node_allocated(&vma->node))
+			continue;
 
-			if (vma->is_ggtt) {
-				stats->global += obj->base.size;
-				continue;
-			}
+		if (i915_vma_is_ggtt(vma)) {
+			stats->global += vma->node.size;
+		} else {
+			struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vma->vm);
 
-			ppgtt = container_of(vma->vm, struct i915_hw_ppgtt, base);
-			if (ppgtt->file_priv != stats->file_priv)
+			if (ppgtt->base.file != stats->file_priv)
 				continue;
-
-			if (obj->active) /* XXX per-vma statistic */
-				stats->active += obj->base.size;
-			else
-				stats->inactive += obj->base.size;
-
-			return 0;
-		}
-	} else {
-		if (i915_gem_obj_ggtt_bound(obj)) {
-			stats->global += obj->base.size;
-			if (obj->active)
-				stats->active += obj->base.size;
-			else
-				stats->inactive += obj->base.size;
-			return 0;
 		}
-	}
 
-	if (!list_empty(&obj->global_list))
-		stats->unbound += obj->base.size;
+		if (i915_vma_is_active(vma))
+			stats->active += vma->node.size;
+		else
+			stats->inactive += vma->node.size;
+	}
 
 	return 0;
 }
@@ -424,9 +357,9 @@ static int per_file_ctx_stats(int id, void *ptr, void *data)
 
 	for (n = 0; n < ARRAY_SIZE(ctx->engine); n++) {
 		if (ctx->engine[n].state)
-			per_file_stats(0, ctx->engine[n].state, data);
-		if (ctx->engine[n].ringbuf)
-			per_file_stats(0, ctx->engine[n].ringbuf->obj, data);
+			per_file_stats(0, ctx->engine[n].state->obj, data);
+		if (ctx->engine[n].ring)
+			per_file_stats(0, ctx->engine[n].ring->vma->obj, data);
 	}
 
 	return 0;
@@ -435,48 +368,34 @@ static int per_file_ctx_stats(int id, void *ptr, void *data)
 static void print_context_stats(struct seq_file *m,
 				struct drm_i915_private *dev_priv)
 {
+	struct drm_device *dev = &dev_priv->drm;
 	struct file_stats stats;
 	struct drm_file *file;
 
 	memset(&stats, 0, sizeof(stats));
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
+	mutex_lock(&dev->struct_mutex);
 	if (dev_priv->kernel_context)
 		per_file_ctx_stats(0, dev_priv->kernel_context, &stats);
 
-	list_for_each_entry(file, &dev_priv->drm.filelist, lhead) {
+	list_for_each_entry(file, &dev->filelist, lhead) {
 		struct drm_i915_file_private *fpriv = file->driver_priv;
 		idr_for_each(&fpriv->context_idr, per_file_ctx_stats, &stats);
 	}
-	mutex_unlock(&dev_priv->drm.struct_mutex);
+	mutex_unlock(&dev->struct_mutex);
 
 	print_file_stats(m, "[k]contexts", stats);
 }
 
-#define count_vmas(list, member) do { \
-	list_for_each_entry(vma, list, member) { \
-		size += i915_gem_obj_total_ggtt_size(vma->obj); \
-		++count; \
-		if (vma->obj->map_and_fenceable) { \
-			mappable_size += i915_gem_obj_ggtt_size(vma->obj); \
-			++mappable_count; \
-		} \
-	} \
-} while (0)
-
-static int i915_gem_object_info(struct seq_file *m, void* data)
+static int i915_gem_object_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	u32 count, mappable_count, purgeable_count;
-	u64 size, mappable_size, purgeable_size;
-	unsigned long pin_mapped_count = 0, pin_mapped_purgeable_count = 0;
-	u64 pin_mapped_size = 0, pin_mapped_purgeable_size = 0;
+	u32 count, mapped_count, purgeable_count, dpy_count;
+	u64 size, mapped_size, purgeable_size, dpy_size;
 	struct drm_i915_gem_object *obj;
 	struct drm_file *file;
-	struct i915_vma *vma;
 	int ret;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -487,70 +406,53 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
 		   dev_priv->mm.object_count,
 		   dev_priv->mm.object_memory);
 
-	size = count = mappable_size = mappable_count = 0;
-	count_objects(&dev_priv->mm.bound_list, global_list);
-	seq_printf(m, "%u [%u] objects, %llu [%llu] bytes in gtt\n",
-		   count, mappable_count, size, mappable_size);
-
-	size = count = mappable_size = mappable_count = 0;
-	count_vmas(&ggtt->base.active_list, vm_link);
-	seq_printf(m, "  %u [%u] active objects, %llu [%llu] bytes\n",
-		   count, mappable_count, size, mappable_size);
+	size = count = 0;
+	mapped_size = mapped_count = 0;
+	purgeable_size = purgeable_count = 0;
+	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
+		size += obj->base.size;
+		++count;
 
-	size = count = mappable_size = mappable_count = 0;
-	count_vmas(&ggtt->base.inactive_list, vm_link);
-	seq_printf(m, "  %u [%u] inactive objects, %llu [%llu] bytes\n",
-		   count, mappable_count, size, mappable_size);
+		if (obj->madv == I915_MADV_DONTNEED) {
+			purgeable_size += obj->base.size;
+			++purgeable_count;
+		}
 
-	size = count = purgeable_size = purgeable_count = 0;
-	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
-		size += obj->base.size, ++count;
-		if (obj->madv == I915_MADV_DONTNEED)
-			purgeable_size += obj->base.size, ++purgeable_count;
 		if (obj->mapping) {
-			pin_mapped_count++;
-			pin_mapped_size += obj->base.size;
-			if (obj->pages_pin_count == 0) {
-				pin_mapped_purgeable_count++;
-				pin_mapped_purgeable_size += obj->base.size;
-			}
+			mapped_count++;
+			mapped_size += obj->base.size;
 		}
 	}
 	seq_printf(m, "%u unbound objects, %llu bytes\n", count, size);
 
-	size = count = mappable_size = mappable_count = 0;
+	size = count = dpy_size = dpy_count = 0;
 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-		if (obj->fault_mappable) {
-			size += i915_gem_obj_ggtt_size(obj);
-			++count;
-		}
+		size += obj->base.size;
+		++count;
+
 		if (obj->pin_display) {
-			mappable_size += i915_gem_obj_ggtt_size(obj);
-			++mappable_count;
+			dpy_size += obj->base.size;
+			++dpy_count;
 		}
+
 		if (obj->madv == I915_MADV_DONTNEED) {
 			purgeable_size += obj->base.size;
 			++purgeable_count;
 		}
+
 		if (obj->mapping) {
-			pin_mapped_count++;
-			pin_mapped_size += obj->base.size;
-			if (obj->pages_pin_count == 0) {
-				pin_mapped_purgeable_count++;
-				pin_mapped_purgeable_size += obj->base.size;
-			}
+			mapped_count++;
+			mapped_size += obj->base.size;
 		}
 	}
+	seq_printf(m, "%u bound objects, %llu bytes\n",
+		   count, size);
 	seq_printf(m, "%u purgeable objects, %llu bytes\n",
 		   purgeable_count, purgeable_size);
-	seq_printf(m, "%u pinned mappable objects, %llu bytes\n",
-		   mappable_count, mappable_size);
-	seq_printf(m, "%u fault mappable objects, %llu bytes\n",
-		   count, size);
-	seq_printf(m,
-		   "%lu [%lu] pin mapped objects, %llu [%llu] bytes [purgeable]\n",
-		   pin_mapped_count, pin_mapped_purgeable_count,
-		   pin_mapped_size, pin_mapped_purgeable_size);
+	seq_printf(m, "%u mapped objects, %llu bytes\n",
+		   mapped_count, mapped_size);
+	seq_printf(m, "%u display objects (pinned), %llu bytes\n",
+		   dpy_count, dpy_size);
 
 	seq_printf(m, "%llu [%llu] gtt total\n",
 		   ggtt->base.total, ggtt->mappable_end - ggtt->base.start);
@@ -563,6 +465,8 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
 	print_context_stats(m, dev_priv);
 	list_for_each_entry_reverse(file, &dev->filelist, lhead) {
 		struct file_stats stats;
+		struct drm_i915_file_private *file_priv = file->driver_priv;
+		struct drm_i915_gem_request *request;
 		struct task_struct *task;
 
 		memset(&stats, 0, sizeof(stats));
@@ -576,10 +480,17 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
 		 * still alive (e.g. get_pid(current) => fork() => exit()).
 		 * Therefore, we need to protect this ->comm access using RCU.
 		 */
+		mutex_lock(&dev->struct_mutex);
+		request = list_first_entry_or_null(&file_priv->mm.request_list,
+						   struct drm_i915_gem_request,
+						   client_list);
 		rcu_read_lock();
-		task = pid_task(file->pid, PIDTYPE_PID);
+		task = pid_task(request && request->ctx->pid ?
+				request->ctx->pid : file->pid,
+				PIDTYPE_PID);
 		print_file_stats(m, task ? task->comm : "<unknown>", stats);
 		rcu_read_unlock();
+		mutex_unlock(&dev->struct_mutex);
 	}
 	mutex_unlock(&dev->filelist_mutex);
 
@@ -589,9 +500,9 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
 static int i915_gem_gtt_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	uintptr_t list = (uintptr_t) node->info_ent->data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(node);
+	struct drm_device *dev = &dev_priv->drm;
+	bool show_pin_display_only = !!node->info_ent->data;
 	struct drm_i915_gem_object *obj;
 	u64 total_obj_size, total_gtt_size;
 	int count, ret;
@@ -602,7 +513,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data)
 
 	total_obj_size = total_gtt_size = count = 0;
 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-		if (list == PINNED_LIST && !i915_gem_obj_is_pinned(obj))
+		if (show_pin_display_only && !obj->pin_display)
 			continue;
 
 		seq_puts(m, "   ");
@@ -623,9 +534,8 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data)
 
 static int i915_gem_pageflip_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_crtc *crtc;
 	int ret;
 
@@ -672,7 +582,7 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
 				   intel_crtc_get_vblank_counter(crtc));
 			seq_printf(m, "%d prepares\n", atomic_read(&work->pending));
 
-			if (INTEL_INFO(dev)->gen >= 4)
+			if (INTEL_GEN(dev_priv) >= 4)
 				addr = I915_HI_DISPBASE(I915_READ(DSPSURF(crtc->plane)));
 			else
 				addr = I915_READ(DSPADDR(crtc->plane));
@@ -693,9 +603,8 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
 
 static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct drm_i915_gem_object *obj;
 	struct intel_engine_cs *engine;
 	int total = 0;
@@ -738,9 +647,8 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
 
 static int i915_gem_request_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_engine_cs *engine;
 	struct drm_i915_gem_request *req;
 	int ret, any;
@@ -754,21 +662,20 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
 		int count;
 
 		count = 0;
-		list_for_each_entry(req, &engine->request_list, list)
+		list_for_each_entry(req, &engine->request_list, link)
 			count++;
 		if (count == 0)
 			continue;
 
 		seq_printf(m, "%s requests: %d\n", engine->name, count);
-		list_for_each_entry(req, &engine->request_list, list) {
+		list_for_each_entry(req, &engine->request_list, link) {
+			struct pid *pid = req->ctx->pid;
 			struct task_struct *task;
 
 			rcu_read_lock();
-			task = NULL;
-			if (req->pid)
-				task = pid_task(req->pid, PIDTYPE_PID);
+			task = pid ? pid_task(pid, PIDTYPE_PID) : NULL;
 			seq_printf(m, "    %x @ %d: %s [%d]\n",
-				   req->seqno,
+				   req->fence.seqno,
 				   (int) (jiffies - req->emitted_jiffies),
 				   task ? task->comm : "<unknown>",
 				   task ? task->pid : -1);
@@ -793,8 +700,6 @@ static void i915_ring_seqno_info(struct seq_file *m,
 
 	seq_printf(m, "Current sequence (%s): %x\n",
 		   engine->name, intel_engine_get_seqno(engine));
-	seq_printf(m, "Current user interrupts (%s): %lx\n",
-		   engine->name, READ_ONCE(engine->breadcrumbs.irq_wakeups));
 
 	spin_lock(&b->lock);
 	for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
@@ -808,41 +713,25 @@ static void i915_ring_seqno_info(struct seq_file *m,
 
 static int i915_gem_seqno_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_engine_cs *engine;
-	int ret;
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-	intel_runtime_pm_get(dev_priv);
 
 	for_each_engine(engine, dev_priv)
 		i915_ring_seqno_info(m, engine);
 
-	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
-
 	return 0;
 }
 
 
 static int i915_interrupt_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_engine_cs *engine;
-	int ret, i, pipe;
+	int i, pipe;
 
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
 	intel_runtime_pm_get(dev_priv);
 
-	if (IS_CHERRYVIEW(dev)) {
+	if (IS_CHERRYVIEW(dev_priv)) {
 		seq_printf(m, "Master Interrupt Control:\t%08x\n",
 			   I915_READ(GEN8_MASTER_IRQ));
 
@@ -881,7 +770,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 			   I915_READ(GEN8_PCU_IIR));
 		seq_printf(m, "PCU interrupt enable:\t%08x\n",
 			   I915_READ(GEN8_PCU_IER));
-	} else if (INTEL_INFO(dev)->gen >= 8) {
+	} else if (INTEL_GEN(dev_priv) >= 8) {
 		seq_printf(m, "Master Interrupt Control:\t%08x\n",
 			   I915_READ(GEN8_MASTER_IRQ));
 
@@ -937,7 +826,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 			   I915_READ(GEN8_PCU_IIR));
 		seq_printf(m, "PCU interrupt enable:\t%08x\n",
 			   I915_READ(GEN8_PCU_IER));
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv)) {
 		seq_printf(m, "Display IER:\t%08x\n",
 			   I915_READ(VLV_IER));
 		seq_printf(m, "Display IIR:\t%08x\n",
@@ -975,7 +864,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 		seq_printf(m, "DPINVGTT:\t%08x\n",
 			   I915_READ(DPINVGTT));
 
-	} else if (!HAS_PCH_SPLIT(dev)) {
+	} else if (!HAS_PCH_SPLIT(dev_priv)) {
 		seq_printf(m, "Interrupt enable:    %08x\n",
 			   I915_READ(IER));
 		seq_printf(m, "Interrupt identity:  %08x\n",
@@ -1007,7 +896,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 			   I915_READ(GTIMR));
 	}
 	for_each_engine(engine, dev_priv) {
-		if (INTEL_INFO(dev)->gen >= 6) {
+		if (INTEL_GEN(dev_priv) >= 6) {
 			seq_printf(m,
 				   "Graphics Interrupt mask (%s):	%08x\n",
 				   engine->name, I915_READ_IMR(engine));
@@ -1015,16 +904,14 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 		i915_ring_seqno_info(m, engine);
 	}
 	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
 
 	return 0;
 }
 
 static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	int i, ret;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1033,14 +920,14 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
 
 	seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs);
 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
-		struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj;
+		struct i915_vma *vma = dev_priv->fence_regs[i].vma;
 
 		seq_printf(m, "Fence %d, pin count = %d, object = ",
 			   i, dev_priv->fence_regs[i].pin_count);
-		if (obj == NULL)
+		if (!vma)
 			seq_puts(m, "unused");
 		else
-			describe_obj(m, obj);
+			describe_obj(m, vma->obj);
 		seq_putc(m, '\n');
 	}
 
@@ -1051,8 +938,7 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
 static int i915_hws_info(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(node);
 	struct intel_engine_cs *engine;
 	const u32 *hws;
 	int i;
@@ -1077,33 +963,25 @@ i915_error_state_write(struct file *filp,
 		       loff_t *ppos)
 {
 	struct i915_error_state_file_priv *error_priv = filp->private_data;
-	struct drm_device *dev = error_priv->dev;
-	int ret;
 
 	DRM_DEBUG_DRIVER("Resetting error state\n");
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-
-	i915_destroy_error_state(dev);
-	mutex_unlock(&dev->struct_mutex);
+	i915_destroy_error_state(error_priv->dev);
 
 	return cnt;
 }
 
 static int i915_error_state_open(struct inode *inode, struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
+	struct drm_i915_private *dev_priv = inode->i_private;
 	struct i915_error_state_file_priv *error_priv;
 
 	error_priv = kzalloc(sizeof(*error_priv), GFP_KERNEL);
 	if (!error_priv)
 		return -ENOMEM;
 
-	error_priv->dev = dev;
+	error_priv->dev = &dev_priv->drm;
 
-	i915_error_state_get(dev, error_priv);
+	i915_error_state_get(&dev_priv->drm, error_priv);
 
 	file->private_data = error_priv;
 
@@ -1129,7 +1007,8 @@ static ssize_t i915_error_state_read(struct file *file, char __user *userbuf,
 	ssize_t ret_count = 0;
 	int ret;
 
-	ret = i915_error_state_buf_init(&error_str, to_i915(error_priv->dev), count, *pos);
+	ret = i915_error_state_buf_init(&error_str,
+					to_i915(error_priv->dev), count, *pos);
 	if (ret)
 		return ret;
 
@@ -1162,16 +1041,15 @@ static const struct file_operations i915_error_state_fops = {
 static int
 i915_next_seqno_get(void *data, u64 *val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 	int ret;
 
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
 	if (ret)
 		return ret;
 
 	*val = dev_priv->next_seqno;
-	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	return 0;
 }
@@ -1179,7 +1057,8 @@ i915_next_seqno_get(void *data, u64 *val)
 static int
 i915_next_seqno_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
+	struct drm_i915_private *dev_priv = data;
+	struct drm_device *dev = &dev_priv->drm;
 	int ret;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1198,16 +1077,13 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_next_seqno_fops,
 
 static int i915_frequency_info(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	int ret = 0;
 
 	intel_runtime_pm_get(dev_priv);
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
-	if (IS_GEN5(dev)) {
+	if (IS_GEN5(dev_priv)) {
 		u16 rgvswctl = I915_READ16(MEMSWCTL);
 		u16 rgvstat = I915_READ16(MEMSTAT_ILK);
 
@@ -1217,7 +1093,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 			   MEMSTAT_VID_SHIFT);
 		seq_printf(m, "Current P-state: %d\n",
 			   (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
-	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		u32 freq_sts;
 
 		mutex_lock(&dev_priv->rps.hw_lock);
@@ -1244,7 +1120,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 			   "efficient (RPe) frequency: %d MHz\n",
 			   intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq));
 		mutex_unlock(&dev_priv->rps.hw_lock);
-	} else if (INTEL_INFO(dev)->gen >= 6) {
+	} else if (INTEL_GEN(dev_priv) >= 6) {
 		u32 rp_state_limits;
 		u32 gt_perf_status;
 		u32 rp_state_cap;
@@ -1256,7 +1132,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		int max_freq;
 
 		rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS);
-		if (IS_BROXTON(dev)) {
+		if (IS_BROXTON(dev_priv)) {
 			rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
 			gt_perf_status = I915_READ(BXT_GT_PERF_STATUS);
 		} else {
@@ -1272,11 +1148,11 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
 		reqf = I915_READ(GEN6_RPNSWREQ);
-		if (IS_GEN9(dev))
+		if (IS_GEN9(dev_priv))
 			reqf >>= 23;
 		else {
 			reqf &= ~GEN6_TURBO_DISABLE;
-			if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+			if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 				reqf >>= 24;
 			else
 				reqf >>= 25;
@@ -1294,9 +1170,9 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
 		rpcurdown = I915_READ(GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
 		rpprevdown = I915_READ(GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
-		if (IS_GEN9(dev))
+		if (IS_GEN9(dev_priv))
 			cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
-		else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+		else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 			cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
 		else
 			cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
@@ -1305,7 +1181,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 		mutex_unlock(&dev->struct_mutex);
 
-		if (IS_GEN6(dev) || IS_GEN7(dev)) {
+		if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
 			pm_ier = I915_READ(GEN6_PMIER);
 			pm_imr = I915_READ(GEN6_PMIMR);
 			pm_isr = I915_READ(GEN6_PMISR);
@@ -1323,7 +1199,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		seq_printf(m, "pm_intr_keep: 0x%08x\n", dev_priv->rps.pm_intr_keep);
 		seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
 		seq_printf(m, "Render p-state ratio: %d\n",
-			   (gt_perf_status & (IS_GEN9(dev) ? 0x1ff00 : 0xff00)) >> 8);
+			   (gt_perf_status & (IS_GEN9(dev_priv) ? 0x1ff00 : 0xff00)) >> 8);
 		seq_printf(m, "Render p-state VID: %d\n",
 			   gt_perf_status & 0xff);
 		seq_printf(m, "Render p-state limit: %d\n",
@@ -1352,22 +1228,22 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		seq_printf(m, "Down threshold: %d%%\n",
 			   dev_priv->rps.down_threshold);
 
-		max_freq = (IS_BROXTON(dev) ? rp_state_cap >> 0 :
+		max_freq = (IS_BROXTON(dev_priv) ? rp_state_cap >> 0 :
 			    rp_state_cap >> 16) & 0xff;
-		max_freq *= (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ?
+		max_freq *= (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) ?
 			     GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 
 		max_freq = (rp_state_cap & 0xff00) >> 8;
-		max_freq *= (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ?
+		max_freq *= (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) ?
 			     GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 
-		max_freq = (IS_BROXTON(dev) ? rp_state_cap >> 16 :
+		max_freq = (IS_BROXTON(dev_priv) ? rp_state_cap >> 16 :
 			    rp_state_cap >> 0) & 0xff;
-		max_freq *= (IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ?
+		max_freq *= (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) ?
 			     GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
@@ -1381,6 +1257,8 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 			   intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq));
 		seq_printf(m, "Min freq: %d MHz\n",
 			   intel_gpu_freq(dev_priv, dev_priv->rps.min_freq));
+		seq_printf(m, "Boost freq: %d MHz\n",
+			   intel_gpu_freq(dev_priv, dev_priv->rps.boost_freq));
 		seq_printf(m, "Max freq: %d MHz\n",
 			   intel_gpu_freq(dev_priv, dev_priv->rps.max_freq));
 		seq_printf(m,
@@ -1401,9 +1279,7 @@ out:
 
 static int i915_hangcheck_info(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_engine_cs *engine;
 	u64 acthd[I915_NUM_ENGINES];
 	u32 seqno[I915_NUM_ENGINES];
@@ -1411,6 +1287,15 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 	enum intel_engine_id id;
 	int j;
 
+	if (test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
+		seq_printf(m, "Wedged\n");
+	if (test_bit(I915_RESET_IN_PROGRESS, &dev_priv->gpu_error.flags))
+		seq_printf(m, "Reset in progress\n");
+	if (waitqueue_active(&dev_priv->gpu_error.wait_queue))
+		seq_printf(m, "Waiter holding struct mutex\n");
+	if (waitqueue_active(&dev_priv->gpu_error.reset_queue))
+		seq_printf(m, "struct_mutex blocked for reset\n");
+
 	if (!i915.enable_hangcheck) {
 		seq_printf(m, "Hangcheck disabled\n");
 		return 0;
@@ -1419,7 +1304,7 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 	intel_runtime_pm_get(dev_priv);
 
 	for_each_engine_id(engine, dev_priv, id) {
-		acthd[id] = intel_ring_get_active_head(engine);
+		acthd[id] = intel_engine_get_active_head(engine);
 		seqno[id] = intel_engine_get_seqno(engine);
 	}
 
@@ -1440,11 +1325,10 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 			   engine->hangcheck.seqno,
 			   seqno[id],
 			   engine->last_submitted_seqno);
-		seq_printf(m, "\twaiters? %d\n",
-			   intel_engine_has_waiter(engine));
-		seq_printf(m, "\tuser interrupts = %lx [current %lx]\n",
-			   engine->hangcheck.user_interrupts,
-			   READ_ONCE(engine->breadcrumbs.irq_wakeups));
+		seq_printf(m, "\twaiters? %s, fake irq active? %s\n",
+			   yesno(intel_engine_has_waiter(engine)),
+			   yesno(test_bit(engine->id,
+					  &dev_priv->gpu_error.missed_irq_rings)));
 		seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n",
 			   (long long)engine->hangcheck.acthd,
 			   (long long)acthd[id]);
@@ -1472,9 +1356,8 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 
 static int ironlake_drpc_info(struct seq_file *m)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	u32 rgvmodectl, rstdbyctl;
 	u16 crstandvid;
 	int ret;
@@ -1540,9 +1423,7 @@ static int ironlake_drpc_info(struct seq_file *m)
 
 static int i915_forcewake_domains(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_uncore_forcewake_domain *fw_domain;
 
 	spin_lock_irq(&dev_priv->uncore.lock);
@@ -1558,9 +1439,7 @@ static int i915_forcewake_domains(struct seq_file *m, void *data)
 
 static int vlv_drpc_info(struct seq_file *m)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	u32 rpmodectl1, rcctl1, pw_status;
 
 	intel_runtime_pm_get(dev_priv);
@@ -1598,10 +1477,10 @@ static int vlv_drpc_info(struct seq_file *m)
 
 static int gen6_drpc_info(struct seq_file *m)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	u32 rpmodectl1, gt_core_status, rcctl1, rc6vids = 0;
+	u32 gen9_powergate_enable = 0, gen9_powergate_status = 0;
 	unsigned forcewake_count;
 	int count = 0, ret;
 
@@ -1629,6 +1508,10 @@ static int gen6_drpc_info(struct seq_file *m)
 
 	rpmodectl1 = I915_READ(GEN6_RP_CONTROL);
 	rcctl1 = I915_READ(GEN6_RC_CONTROL);
+	if (INTEL_GEN(dev_priv) >= 9) {
+		gen9_powergate_enable = I915_READ(GEN9_PG_ENABLE);
+		gen9_powergate_status = I915_READ(GEN9_PWRGT_DOMAIN_STATUS);
+	}
 	mutex_unlock(&dev->struct_mutex);
 	mutex_lock(&dev_priv->rps.hw_lock);
 	sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
@@ -1647,6 +1530,12 @@ static int gen6_drpc_info(struct seq_file *m)
 		   yesno(rcctl1 & GEN6_RC_CTL_RC1e_ENABLE));
 	seq_printf(m, "RC6 Enabled: %s\n",
 		   yesno(rcctl1 & GEN6_RC_CTL_RC6_ENABLE));
+	if (INTEL_GEN(dev_priv) >= 9) {
+		seq_printf(m, "Render Well Gating Enabled: %s\n",
+			yesno(gen9_powergate_enable & GEN9_RENDER_PG_ENABLE));
+		seq_printf(m, "Media Well Gating Enabled: %s\n",
+			yesno(gen9_powergate_enable & GEN9_MEDIA_PG_ENABLE));
+	}
 	seq_printf(m, "Deep RC6 Enabled: %s\n",
 		   yesno(rcctl1 & GEN6_RC_CTL_RC6p_ENABLE));
 	seq_printf(m, "Deepest RC6 Enabled: %s\n",
@@ -1675,6 +1564,14 @@ static int gen6_drpc_info(struct seq_file *m)
 
 	seq_printf(m, "Core Power Down: %s\n",
 		   yesno(gt_core_status & GEN6_CORE_CPD_STATE_MASK));
+	if (INTEL_GEN(dev_priv) >= 9) {
+		seq_printf(m, "Render Power Well: %s\n",
+			(gen9_powergate_status &
+			 GEN9_PWRGT_RENDER_STATUS_MASK) ? "Up" : "Down");
+		seq_printf(m, "Media Power Well: %s\n",
+			(gen9_powergate_status &
+			 GEN9_PWRGT_MEDIA_STATUS_MASK) ? "Up" : "Down");
+	}
 
 	/* Not exactly sure what this is */
 	seq_printf(m, "RC6 \"Locked to RPn\" residency since boot: %u\n",
@@ -1692,17 +1589,16 @@ static int gen6_drpc_info(struct seq_file *m)
 		   GEN6_DECODE_RC6_VID(((rc6vids >> 8) & 0xff)));
 	seq_printf(m, "RC6++ voltage: %dmV\n",
 		   GEN6_DECODE_RC6_VID(((rc6vids >> 16) & 0xff)));
-	return 0;
+	return i915_forcewake_domains(m, NULL);
 }
 
 static int i915_drpc_info(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		return vlv_drpc_info(m);
-	else if (INTEL_INFO(dev)->gen >= 6)
+	else if (INTEL_GEN(dev_priv) >= 6)
 		return gen6_drpc_info(m);
 	else
 		return ironlake_drpc_info(m);
@@ -1710,9 +1606,7 @@ static int i915_drpc_info(struct seq_file *m, void *unused)
 
 static int i915_frontbuffer_tracking(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 
 	seq_printf(m, "FB tracking busy bits: 0x%08x\n",
 		   dev_priv->fb_tracking.busy_bits);
@@ -1725,11 +1619,9 @@ static int i915_frontbuffer_tracking(struct seq_file *m, void *unused)
 
 static int i915_fbc_status(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 
-	if (!HAS_FBC(dev)) {
+	if (!HAS_FBC(dev_priv)) {
 		seq_puts(m, "FBC unsupported on this chipset\n");
 		return 0;
 	}
@@ -1743,7 +1635,7 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 		seq_printf(m, "FBC disabled: %s\n",
 			   dev_priv->fbc.no_fbc_reason);
 
-	if (INTEL_INFO(dev_priv)->gen >= 7)
+	if (INTEL_GEN(dev_priv) >= 7)
 		seq_printf(m, "Compressing: %s\n",
 			   yesno(I915_READ(FBC_STATUS2) &
 				 FBC_COMPRESSION_MASK));
@@ -1756,10 +1648,9 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 
 static int i915_fbc_fc_get(void *data, u64 *val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 
-	if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev))
+	if (INTEL_GEN(dev_priv) < 7 || !HAS_FBC(dev_priv))
 		return -ENODEV;
 
 	*val = dev_priv->fbc.false_color;
@@ -1769,11 +1660,10 @@ static int i915_fbc_fc_get(void *data, u64 *val)
 
 static int i915_fbc_fc_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 	u32 reg;
 
-	if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev))
+	if (INTEL_GEN(dev_priv) < 7 || !HAS_FBC(dev_priv))
 		return -ENODEV;
 
 	mutex_lock(&dev_priv->fbc.lock);
@@ -1795,11 +1685,9 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_fbc_fc_fops,
 
 static int i915_ips_status(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 
-	if (!HAS_IPS(dev)) {
+	if (!HAS_IPS(dev_priv)) {
 		seq_puts(m, "not supported\n");
 		return 0;
 	}
@@ -1809,7 +1697,7 @@ static int i915_ips_status(struct seq_file *m, void *unused)
 	seq_printf(m, "Enabled by kernel parameter: %s\n",
 		   yesno(i915.enable_ips));
 
-	if (INTEL_INFO(dev)->gen >= 8) {
+	if (INTEL_GEN(dev_priv) >= 8) {
 		seq_puts(m, "Currently: unknown\n");
 	} else {
 		if (I915_READ(IPS_CTL) & IPS_ENABLE)
@@ -1825,23 +1713,21 @@ static int i915_ips_status(struct seq_file *m, void *unused)
 
 static int i915_sr_status(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	bool sr_enabled = false;
 
 	intel_runtime_pm_get(dev_priv);
 
-	if (HAS_PCH_SPLIT(dev))
+	if (HAS_PCH_SPLIT(dev_priv))
 		sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN;
-	else if (IS_CRESTLINE(dev) || IS_G4X(dev) ||
-		 IS_I945G(dev) || IS_I945GM(dev))
+	else if (IS_CRESTLINE(dev_priv) || IS_G4X(dev_priv) ||
+		 IS_I945G(dev_priv) || IS_I945GM(dev_priv))
 		sr_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
-	else if (IS_I915GM(dev))
+	else if (IS_I915GM(dev_priv))
 		sr_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
-	else if (IS_PINEVIEW(dev))
+	else if (IS_PINEVIEW(dev_priv))
 		sr_enabled = I915_READ(DSPFW3) & PINEVIEW_SELF_REFRESH_EN;
-	else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		sr_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 
 	intel_runtime_pm_put(dev_priv);
@@ -1854,13 +1740,12 @@ static int i915_sr_status(struct seq_file *m, void *unused)
 
 static int i915_emon_status(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	unsigned long temp, chipset, gfx;
 	int ret;
 
-	if (!IS_GEN5(dev))
+	if (!IS_GEN5(dev_priv))
 		return -ENODEV;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1882,27 +1767,23 @@ static int i915_emon_status(struct seq_file *m, void *unused)
 
 static int i915_ring_freq_table(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	int ret = 0;
 	int gpu_freq, ia_freq;
 	unsigned int max_gpu_freq, min_gpu_freq;
 
-	if (!HAS_CORE_RING_FREQ(dev)) {
+	if (!HAS_LLC(dev_priv)) {
 		seq_puts(m, "unsupported on this chipset\n");
 		return 0;
 	}
 
 	intel_runtime_pm_get(dev_priv);
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
 	ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
 	if (ret)
 		goto out;
 
-	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		/* Convert GT frequency to 50 HZ units */
 		min_gpu_freq =
 			dev_priv->rps.min_freq_softlimit / GEN9_FREQ_SCALER;
@@ -1922,7 +1803,7 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 				       &ia_freq);
 		seq_printf(m, "%d\t\t%d\t\t\t\t%d\n",
 			   intel_gpu_freq(dev_priv, (gpu_freq *
-				(IS_SKYLAKE(dev) || IS_KABYLAKE(dev) ?
+				(IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) ?
 				 GEN9_FREQ_SCALER : 1))),
 			   ((ia_freq >> 0) & 0xff) * 100,
 			   ((ia_freq >> 8) & 0xff) * 100);
@@ -1937,9 +1818,8 @@ out:
 
 static int i915_opregion(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_opregion *opregion = &dev_priv->opregion;
 	int ret;
 
@@ -1958,10 +1838,7 @@ out:
 
 static int i915_vbt(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_opregion *opregion = &dev_priv->opregion;
+	struct intel_opregion *opregion = &node_to_i915(m->private)->opregion;
 
 	if (opregion->vbt)
 		seq_write(m, opregion->vbt, opregion->vbt_size);
@@ -1971,8 +1848,8 @@ static int i915_vbt(struct seq_file *m, void *unused)
 
 static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_framebuffer *fbdev_fb = NULL;
 	struct drm_framebuffer *drm_fb;
 	int ret;
@@ -1982,8 +1859,8 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 		return ret;
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-	if (to_i915(dev)->fbdev) {
-		fbdev_fb = to_intel_framebuffer(to_i915(dev)->fbdev->helper.fb);
+	if (dev_priv->fbdev) {
+		fbdev_fb = to_intel_framebuffer(dev_priv->fbdev->helper.fb);
 
 		seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ",
 			   fbdev_fb->base.width,
@@ -2019,19 +1896,17 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 	return 0;
 }
 
-static void describe_ctx_ringbuf(struct seq_file *m,
-				 struct intel_ringbuffer *ringbuf)
+static void describe_ctx_ring(struct seq_file *m, struct intel_ring *ring)
 {
 	seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u, last head: %d)",
-		   ringbuf->space, ringbuf->head, ringbuf->tail,
-		   ringbuf->last_retired_head);
+		   ring->space, ring->head, ring->tail,
+		   ring->last_retired_head);
 }
 
 static int i915_context_status(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_engine_cs *engine;
 	struct i915_gem_context *ctx;
 	int ret;
@@ -2042,18 +1917,17 @@ static int i915_context_status(struct seq_file *m, void *unused)
 
 	list_for_each_entry(ctx, &dev_priv->context_list, link) {
 		seq_printf(m, "HW context %u ", ctx->hw_id);
-		if (IS_ERR(ctx->file_priv)) {
-			seq_puts(m, "(deleted) ");
-		} else if (ctx->file_priv) {
-			struct pid *pid = ctx->file_priv->file->pid;
+		if (ctx->pid) {
 			struct task_struct *task;
 
-			task = get_pid_task(pid, PIDTYPE_PID);
+			task = get_pid_task(ctx->pid, PIDTYPE_PID);
 			if (task) {
 				seq_printf(m, "(%s [%d]) ",
 					   task->comm, task->pid);
 				put_task_struct(task);
 			}
+		} else if (IS_ERR(ctx->file_priv)) {
+			seq_puts(m, "(deleted) ");
 		} else {
 			seq_puts(m, "(kernel) ");
 		}
@@ -2067,9 +1941,9 @@ static int i915_context_status(struct seq_file *m, void *unused)
 			seq_printf(m, "%s: ", engine->name);
 			seq_putc(m, ce->initialised ? 'I' : 'i');
 			if (ce->state)
-				describe_obj(m, ce->state);
-			if (ce->ringbuf)
-				describe_ctx_ringbuf(m, ce->ringbuf);
+				describe_obj(m, ce->state->obj);
+			if (ce->ring)
+				describe_ctx_ring(m, ce->ring);
 			seq_putc(m, '\n');
 		}
 
@@ -2085,36 +1959,34 @@ static void i915_dump_lrc_obj(struct seq_file *m,
 			      struct i915_gem_context *ctx,
 			      struct intel_engine_cs *engine)
 {
-	struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state;
+	struct i915_vma *vma = ctx->engine[engine->id].state;
 	struct page *page;
-	uint32_t *reg_state;
 	int j;
-	unsigned long ggtt_offset = 0;
 
 	seq_printf(m, "CONTEXT: %s %u\n", engine->name, ctx->hw_id);
 
-	if (ctx_obj == NULL) {
-		seq_puts(m, "\tNot allocated\n");
+	if (!vma) {
+		seq_puts(m, "\tFake context\n");
 		return;
 	}
 
-	if (!i915_gem_obj_ggtt_bound(ctx_obj))
-		seq_puts(m, "\tNot bound in GGTT\n");
-	else
-		ggtt_offset = i915_gem_obj_ggtt_offset(ctx_obj);
+	if (vma->flags & I915_VMA_GLOBAL_BIND)
+		seq_printf(m, "\tBound in GGTT at 0x%08x\n",
+			   i915_ggtt_offset(vma));
 
-	if (i915_gem_object_get_pages(ctx_obj)) {
-		seq_puts(m, "\tFailed to get pages for context object\n");
+	if (i915_gem_object_get_pages(vma->obj)) {
+		seq_puts(m, "\tFailed to get pages for context object\n\n");
 		return;
 	}
 
-	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
-	if (!WARN_ON(page == NULL)) {
-		reg_state = kmap_atomic(page);
+	page = i915_gem_object_get_page(vma->obj, LRC_STATE_PN);
+	if (page) {
+		u32 *reg_state = kmap_atomic(page);
 
 		for (j = 0; j < 0x600 / sizeof(u32) / 4; j += 4) {
-			seq_printf(m, "\t[0x%08lx] 0x%08x 0x%08x 0x%08x 0x%08x\n",
-				   ggtt_offset + 4096 + (j * 4),
+			seq_printf(m,
+				   "\t[0x%04x] 0x%08x 0x%08x 0x%08x 0x%08x\n",
+				   j * 4,
 				   reg_state[j], reg_state[j + 1],
 				   reg_state[j + 2], reg_state[j + 3]);
 		}
@@ -2126,9 +1998,8 @@ static void i915_dump_lrc_obj(struct seq_file *m,
 
 static int i915_dump_lrc(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_engine_cs *engine;
 	struct i915_gem_context *ctx;
 	int ret;
@@ -2153,9 +2024,8 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
 
 static int i915_execlists(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = (struct drm_info_node *)m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_engine_cs *engine;
 	u32 status_pointer;
 	u8 read_pointer;
@@ -2190,7 +2060,7 @@ static int i915_execlists(struct seq_file *m, void *data)
 		status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
 		seq_printf(m, "\tStatus pointer: 0x%08X\n", status_pointer);
 
-		read_pointer = engine->next_context_status_buffer;
+		read_pointer = GEN8_CSB_READ_PTR(status_pointer);
 		write_pointer = GEN8_CSB_WRITE_PTR(status_pointer);
 		if (read_pointer > write_pointer)
 			write_pointer += GEN8_CSB_ENTRIES;
@@ -2256,9 +2126,8 @@ static const char *swizzle_string(unsigned swizzle)
 
 static int i915_swizzle_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	int ret;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -2271,7 +2140,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 	seq_printf(m, "bit6 swizzle for Y-tiling = %s\n",
 		   swizzle_string(dev_priv->mm.bit_6_swizzle_y));
 
-	if (IS_GEN3(dev) || IS_GEN4(dev)) {
+	if (IS_GEN3(dev_priv) || IS_GEN4(dev_priv)) {
 		seq_printf(m, "DDC = 0x%08x\n",
 			   I915_READ(DCC));
 		seq_printf(m, "DDC2 = 0x%08x\n",
@@ -2280,7 +2149,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 			   I915_READ16(C0DRB3));
 		seq_printf(m, "C1DRB3 = 0x%04x\n",
 			   I915_READ16(C1DRB3));
-	} else if (INTEL_INFO(dev)->gen >= 6) {
+	} else if (INTEL_GEN(dev_priv) >= 6) {
 		seq_printf(m, "MAD_DIMM_C0 = 0x%08x\n",
 			   I915_READ(MAD_DIMM_C0));
 		seq_printf(m, "MAD_DIMM_C1 = 0x%08x\n",
@@ -2289,7 +2158,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 			   I915_READ(MAD_DIMM_C2));
 		seq_printf(m, "TILECTL = 0x%08x\n",
 			   I915_READ(TILECTL));
-		if (INTEL_INFO(dev)->gen >= 8)
+		if (INTEL_GEN(dev_priv) >= 8)
 			seq_printf(m, "GAMTARBMODE = 0x%08x\n",
 				   I915_READ(GAMTARBMODE));
 		else
@@ -2329,9 +2198,9 @@ static int per_file_ctx(int id, void *ptr, void *data)
 	return 0;
 }
 
-static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
+static void gen8_ppgtt_info(struct seq_file *m,
+			    struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_engine_cs *engine;
 	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 	int i;
@@ -2350,9 +2219,9 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 	}
 }
 
-static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
+static void gen6_ppgtt_info(struct seq_file *m,
+			    struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_engine_cs *engine;
 
 	if (IS_GEN6(dev_priv))
@@ -2384,22 +2253,23 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 
 static int i915_ppgtt_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct drm_file *file;
+	int ret;
 
-	int ret = mutex_lock_interruptible(&dev->struct_mutex);
+	mutex_lock(&dev->filelist_mutex);
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
-		return ret;
+		goto out_unlock;
+
 	intel_runtime_pm_get(dev_priv);
 
-	if (INTEL_INFO(dev)->gen >= 8)
-		gen8_ppgtt_info(m, dev);
-	else if (INTEL_INFO(dev)->gen >= 6)
-		gen6_ppgtt_info(m, dev);
+	if (INTEL_GEN(dev_priv) >= 8)
+		gen8_ppgtt_info(m, dev_priv);
+	else if (INTEL_GEN(dev_priv) >= 6)
+		gen6_ppgtt_info(m, dev_priv);
 
-	mutex_lock(&dev->filelist_mutex);
 	list_for_each_entry_reverse(file, &dev->filelist, lhead) {
 		struct drm_i915_file_private *file_priv = file->driver_priv;
 		struct task_struct *task;
@@ -2407,19 +2277,19 @@ static int i915_ppgtt_info(struct seq_file *m, void *data)
 		task = get_pid_task(file->pid, PIDTYPE_PID);
 		if (!task) {
 			ret = -ESRCH;
-			goto out_unlock;
+			goto out_rpm;
 		}
 		seq_printf(m, "\nproc: %s\n", task->comm);
 		put_task_struct(task);
 		idr_for_each(&file_priv->context_idr, per_file_ctx,
 			     (void *)(unsigned long)m);
 	}
-out_unlock:
-	mutex_unlock(&dev->filelist_mutex);
 
+out_rpm:
 	intel_runtime_pm_put(dev_priv);
 	mutex_unlock(&dev->struct_mutex);
-
+out_unlock:
+	mutex_unlock(&dev->filelist_mutex);
 	return ret;
 }
 
@@ -2434,23 +2304,41 @@ static int count_irq_waiters(struct drm_i915_private *i915)
 	return count;
 }
 
+static const char *rps_power_to_str(unsigned int power)
+{
+	static const char * const strings[] = {
+		[LOW_POWER] = "low power",
+		[BETWEEN] = "mixed",
+		[HIGH_POWER] = "high power",
+	};
+
+	if (power >= ARRAY_SIZE(strings) || !strings[power])
+		return "unknown";
+
+	return strings[power];
+}
+
 static int i915_rps_boost_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct drm_file *file;
 
 	seq_printf(m, "RPS enabled? %d\n", dev_priv->rps.enabled);
 	seq_printf(m, "GPU busy? %s [%x]\n",
 		   yesno(dev_priv->gt.awake), dev_priv->gt.active_engines);
 	seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv));
-	seq_printf(m, "Frequency requested %d; min hard:%d, soft:%d; max soft:%d, hard:%d\n",
-		   intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
+	seq_printf(m, "Frequency requested %d\n",
+		   intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq));
+	seq_printf(m, "  min hard:%d, soft:%d; max soft:%d, hard:%d\n",
 		   intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
 		   intel_gpu_freq(dev_priv, dev_priv->rps.min_freq_softlimit),
 		   intel_gpu_freq(dev_priv, dev_priv->rps.max_freq_softlimit),
 		   intel_gpu_freq(dev_priv, dev_priv->rps.max_freq));
+	seq_printf(m, "  idle:%d, efficient:%d, boost:%d\n",
+		   intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq),
+		   intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
+		   intel_gpu_freq(dev_priv, dev_priv->rps.boost_freq));
 
 	mutex_lock(&dev->filelist_mutex);
 	spin_lock(&dev_priv->rps.client_lock);
@@ -2467,27 +2355,44 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 			   list_empty(&file_priv->rps.link) ? "" : ", active");
 		rcu_read_unlock();
 	}
-	seq_printf(m, "Semaphore boosts: %d%s\n",
-		   dev_priv->rps.semaphores.boosts,
-		   list_empty(&dev_priv->rps.semaphores.link) ? "" : ", active");
-	seq_printf(m, "MMIO flip boosts: %d%s\n",
-		   dev_priv->rps.mmioflips.boosts,
-		   list_empty(&dev_priv->rps.mmioflips.link) ? "" : ", active");
-	seq_printf(m, "Kernel boosts: %d\n", dev_priv->rps.boosts);
+	seq_printf(m, "Kernel (anonymous) boosts: %d\n", dev_priv->rps.boosts);
 	spin_unlock(&dev_priv->rps.client_lock);
 	mutex_unlock(&dev->filelist_mutex);
 
+	if (INTEL_GEN(dev_priv) >= 6 &&
+	    dev_priv->rps.enabled &&
+	    dev_priv->gt.active_engines) {
+		u32 rpup, rpupei;
+		u32 rpdown, rpdownei;
+
+		intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+		rpup = I915_READ_FW(GEN6_RP_CUR_UP) & GEN6_RP_EI_MASK;
+		rpupei = I915_READ_FW(GEN6_RP_CUR_UP_EI) & GEN6_RP_EI_MASK;
+		rpdown = I915_READ_FW(GEN6_RP_CUR_DOWN) & GEN6_RP_EI_MASK;
+		rpdownei = I915_READ_FW(GEN6_RP_CUR_DOWN_EI) & GEN6_RP_EI_MASK;
+		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+
+		seq_printf(m, "\nRPS Autotuning (current \"%s\" window):\n",
+			   rps_power_to_str(dev_priv->rps.power));
+		seq_printf(m, "  Avg. up: %d%% [above threshold? %d%%]\n",
+			   100 * rpup / rpupei,
+			   dev_priv->rps.up_threshold);
+		seq_printf(m, "  Avg. down: %d%% [below threshold? %d%%]\n",
+			   100 * rpdown / rpdownei,
+			   dev_priv->rps.down_threshold);
+	} else {
+		seq_puts(m, "\nRPS Autotuning inactive\n");
+	}
+
 	return 0;
 }
 
 static int i915_llc(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	const bool edram = INTEL_GEN(dev_priv) > 8;
 
-	seq_printf(m, "LLC: %s\n", yesno(HAS_LLC(dev)));
+	seq_printf(m, "LLC: %s\n", yesno(HAS_LLC(dev_priv)));
 	seq_printf(m, "%s: %lluMB\n", edram ? "eDRAM" : "eLLC",
 		   intel_uncore_edram_size(dev_priv)/1024/1024);
 
@@ -2496,8 +2401,7 @@ static int i915_llc(struct seq_file *m, void *data)
 
 static int i915_guc_load_status_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_i915_private *dev_priv = to_i915(node->minor->dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
 	u32 tmp, i;
 
@@ -2543,6 +2447,7 @@ static void i915_guc_client_info(struct seq_file *m,
 				 struct i915_guc_client *client)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	uint64_t tot = 0;
 
 	seq_printf(m, "\tPriority %d, GuC ctx index: %u, PD offset 0x%x\n",
@@ -2553,27 +2458,26 @@ static void i915_guc_client_info(struct seq_file *m,
 		client->wq_size, client->wq_offset, client->wq_tail);
 
 	seq_printf(m, "\tWork queue full: %u\n", client->no_wq_space);
-	seq_printf(m, "\tFailed to queue: %u\n", client->q_fail);
 	seq_printf(m, "\tFailed doorbell: %u\n", client->b_fail);
 	seq_printf(m, "\tLast submission result: %d\n", client->retcode);
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine_id(engine, dev_priv, id) {
+		u64 submissions = client->submissions[id];
+		tot += submissions;
 		seq_printf(m, "\tSubmissions: %llu %s\n",
-				client->submissions[engine->id],
-				engine->name);
-		tot += client->submissions[engine->id];
+				submissions, engine->name);
 	}
 	seq_printf(m, "\tTotal: %llu\n", tot);
 }
 
 static int i915_guc_info(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_guc guc;
 	struct i915_guc_client client = {};
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	u64 total = 0;
 
 	if (!HAS_GUC_SCHED(dev_priv))
@@ -2600,11 +2504,11 @@ static int i915_guc_info(struct seq_file *m, void *data)
 	seq_printf(m, "GuC last action error code: %d\n", guc.action_err);
 
 	seq_printf(m, "\nGuC submissions:\n");
-	for_each_engine(engine, dev_priv) {
+	for_each_engine_id(engine, dev_priv, id) {
+		u64 submissions = guc.submissions[id];
+		total += submissions;
 		seq_printf(m, "\t%-24s: %10llu, last seqno 0x%08x\n",
-			engine->name, guc.submissions[engine->id],
-			guc.last_seqno[engine->id]);
-		total += guc.submissions[engine->id];
+			engine->name, submissions, guc.last_seqno[id]);
 	}
 	seq_printf(m, "\t%s: %llu\n", "Total", total);
 
@@ -2618,18 +2522,16 @@ static int i915_guc_info(struct seq_file *m, void *data)
 
 static int i915_guc_log_dump(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_i915_gem_object *log_obj = dev_priv->guc.log_obj;
-	u32 *log;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_i915_gem_object *obj;
 	int i = 0, pg;
 
-	if (!log_obj)
+	if (!dev_priv->guc.log_vma)
 		return 0;
 
-	for (pg = 0; pg < log_obj->base.size / PAGE_SIZE; pg++) {
-		log = kmap_atomic(i915_gem_object_get_page(log_obj, pg));
+	obj = dev_priv->guc.log_vma->obj;
+	for (pg = 0; pg < obj->base.size / PAGE_SIZE; pg++) {
+		u32 *log = kmap_atomic(i915_gem_object_get_page(obj, pg));
 
 		for (i = 0; i < PAGE_SIZE / sizeof(u32); i += 4)
 			seq_printf(m, "0x%08x 0x%08x 0x%08x 0x%08x\n",
@@ -2646,15 +2548,13 @@ static int i915_guc_log_dump(struct seq_file *m, void *data)
 
 static int i915_edp_psr_status(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	u32 psrperf = 0;
 	u32 stat[3];
 	enum pipe pipe;
 	bool enabled = false;
 
-	if (!HAS_PSR(dev)) {
+	if (!HAS_PSR(dev_priv)) {
 		seq_puts(m, "PSR not supported\n");
 		return 0;
 	}
@@ -2671,7 +2571,7 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 	seq_printf(m, "Re-enable work scheduled: %s\n",
 		   yesno(work_busy(&dev_priv->psr.work.work)));
 
-	if (HAS_DDI(dev))
+	if (HAS_DDI(dev_priv))
 		enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE;
 	else {
 		for_each_pipe(dev_priv, pipe) {
@@ -2688,7 +2588,7 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 
 	seq_printf(m, "HW Enabled & Active bit: %s", yesno(enabled));
 
-	if (!HAS_DDI(dev))
+	if (!HAS_DDI(dev_priv))
 		for_each_pipe(dev_priv, pipe) {
 			if ((stat[pipe] == VLV_EDP_PSR_ACTIVE_NORFB_UP) ||
 			    (stat[pipe] == VLV_EDP_PSR_ACTIVE_SF_UPDATE))
@@ -2700,7 +2600,7 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 	 * VLV/CHV PSR has no kind of performance counter
 	 * SKL+ Perf counter is reset to 0 everytime DC state is entered
 	 */
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		psrperf = I915_READ(EDP_PSR_PERF_CNT) &
 			EDP_PSR_PERF_CNT_MASK;
 
@@ -2714,8 +2614,8 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 
 static int i915_sink_crc(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_connector *connector;
 	struct intel_dp *intel_dp = NULL;
 	int ret;
@@ -2754,13 +2654,11 @@ out:
 
 static int i915_energy_uJ(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	u64 power;
 	u32 units;
 
-	if (INTEL_INFO(dev)->gen < 6)
+	if (INTEL_GEN(dev_priv) < 6)
 		return -ENODEV;
 
 	intel_runtime_pm_get(dev_priv);
@@ -2780,9 +2678,8 @@ static int i915_energy_uJ(struct seq_file *m, void *data)
 
 static int i915_runtime_pm_status(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 
 	if (!HAS_RUNTIME_PM(dev_priv))
 		seq_puts(m, "Runtime power management not supported\n");
@@ -2792,22 +2689,20 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused)
 		   yesno(!intel_irqs_enabled(dev_priv)));
 #ifdef CONFIG_PM
 	seq_printf(m, "Usage count: %d\n",
-		   atomic_read(&dev->dev->power.usage_count));
+		   atomic_read(&dev_priv->drm.dev->power.usage_count));
 #else
 	seq_printf(m, "Device Power Management (CONFIG_PM) disabled\n");
 #endif
 	seq_printf(m, "PCI device power state: %s [%d]\n",
-		   pci_power_name(dev_priv->drm.pdev->current_state),
-		   dev_priv->drm.pdev->current_state);
+		   pci_power_name(pdev->current_state),
+		   pdev->current_state);
 
 	return 0;
 }
 
 static int i915_power_domain_info(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct i915_power_domains *power_domains = &dev_priv->power_domains;
 	int i;
 
@@ -2840,12 +2735,10 @@ static int i915_power_domain_info(struct seq_file *m, void *unused)
 
 static int i915_dmc_info(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_csr *csr;
 
-	if (!HAS_CSR(dev)) {
+	if (!HAS_CSR(dev_priv)) {
 		seq_puts(m, "not supported\n");
 		return 0;
 	}
@@ -2863,12 +2756,12 @@ static int i915_dmc_info(struct seq_file *m, void *unused)
 	seq_printf(m, "version: %d.%d\n", CSR_VERSION_MAJOR(csr->version),
 		   CSR_VERSION_MINOR(csr->version));
 
-	if (IS_SKYLAKE(dev) && csr->version >= CSR_VERSION(1, 6)) {
+	if (IS_SKYLAKE(dev_priv) && csr->version >= CSR_VERSION(1, 6)) {
 		seq_printf(m, "DC3 -> DC5 count: %d\n",
 			   I915_READ(SKL_CSR_DC3_DC5_COUNT));
 		seq_printf(m, "DC5 -> DC6 count: %d\n",
 			   I915_READ(SKL_CSR_DC5_DC6_COUNT));
-	} else if (IS_BROXTON(dev) && csr->version >= CSR_VERSION(1, 4)) {
+	} else if (IS_BROXTON(dev_priv) && csr->version >= CSR_VERSION(1, 4)) {
 		seq_printf(m, "DC3 -> DC5 count: %d\n",
 			   I915_READ(BXT_CSR_DC3_DC5_COUNT));
 	}
@@ -2905,8 +2798,8 @@ static void intel_encoder_info(struct seq_file *m,
 			       struct intel_crtc *intel_crtc,
 			       struct intel_encoder *intel_encoder)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct drm_crtc *crtc = &intel_crtc->base;
 	struct intel_connector *intel_connector;
 	struct drm_encoder *encoder;
@@ -2932,8 +2825,8 @@ static void intel_encoder_info(struct seq_file *m,
 
 static void intel_crtc_info(struct seq_file *m, struct intel_crtc *intel_crtc)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct drm_crtc *crtc = &intel_crtc->base;
 	struct intel_encoder *intel_encoder;
 	struct drm_plane_state *plane_state = crtc->primary->state;
@@ -2967,6 +2860,9 @@ static void intel_dp_info(struct seq_file *m,
 	seq_printf(m, "\taudio support: %s\n", yesno(intel_dp->has_audio));
 	if (intel_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)
 		intel_panel_info(m, &intel_connector->panel);
+
+	drm_dp_downstream_debug(m, intel_dp->dpcd, intel_dp->downstream_ports,
+				&intel_dp->aux);
 }
 
 static void intel_hdmi_info(struct seq_file *m,
@@ -3031,12 +2927,11 @@ static void intel_connector_info(struct seq_file *m,
 		intel_seq_print_mode(m, 2, mode);
 }
 
-static bool cursor_active(struct drm_device *dev, int pipe)
+static bool cursor_active(struct drm_i915_private *dev_priv, int pipe)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 state;
 
-	if (IS_845G(dev) || IS_I865G(dev))
+	if (IS_845G(dev_priv) || IS_I865G(dev_priv))
 		state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
 	else
 		state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
@@ -3044,9 +2939,9 @@ static bool cursor_active(struct drm_device *dev, int pipe)
 	return state;
 }
 
-static bool cursor_position(struct drm_device *dev, int pipe, int *x, int *y)
+static bool cursor_position(struct drm_i915_private *dev_priv,
+			    int pipe, int *x, int *y)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 pos;
 
 	pos = I915_READ(CURPOS(pipe));
@@ -3059,7 +2954,7 @@ static bool cursor_position(struct drm_device *dev, int pipe, int *x, int *y)
 	if (pos & (CURSOR_POS_SIGN << CURSOR_Y_SHIFT))
 		*y = -*y;
 
-	return cursor_active(dev, pipe);
+	return cursor_active(dev_priv, pipe);
 }
 
 static const char *plane_type(enum drm_plane_type type)
@@ -3089,12 +2984,12 @@ static const char *plane_rotation(unsigned int rotation)
 	 */
 	snprintf(buf, sizeof(buf),
 		 "%s%s%s%s%s%s(0x%08x)",
-		 (rotation & BIT(DRM_ROTATE_0)) ? "0 " : "",
-		 (rotation & BIT(DRM_ROTATE_90)) ? "90 " : "",
-		 (rotation & BIT(DRM_ROTATE_180)) ? "180 " : "",
-		 (rotation & BIT(DRM_ROTATE_270)) ? "270 " : "",
-		 (rotation & BIT(DRM_REFLECT_X)) ? "FLIPX " : "",
-		 (rotation & BIT(DRM_REFLECT_Y)) ? "FLIPY " : "",
+		 (rotation & DRM_ROTATE_0) ? "0 " : "",
+		 (rotation & DRM_ROTATE_90) ? "90 " : "",
+		 (rotation & DRM_ROTATE_180) ? "180 " : "",
+		 (rotation & DRM_ROTATE_270) ? "270 " : "",
+		 (rotation & DRM_REFLECT_X) ? "FLIPX " : "",
+		 (rotation & DRM_REFLECT_Y) ? "FLIPY " : "",
 		 rotation);
 
 	return buf;
@@ -3102,13 +2997,14 @@ static const char *plane_rotation(unsigned int rotation)
 
 static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_plane *intel_plane;
 
 	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
 		struct drm_plane_state *state;
 		struct drm_plane *plane = &intel_plane->base;
+		char *format_name;
 
 		if (!plane->state) {
 			seq_puts(m, "plane->state is NULL!\n");
@@ -3117,6 +3013,12 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc)
 
 		state = plane->state;
 
+		if (state->fb) {
+			format_name = drm_get_format_name(state->fb->pixel_format);
+		} else {
+			format_name = kstrdup("N/A", GFP_KERNEL);
+		}
+
 		seq_printf(m, "\t--Plane id %d: type=%s, crtc_pos=%4dx%4d, crtc_size=%4dx%4d, src_pos=%d.%04ux%d.%04u, src_size=%d.%04ux%d.%04u, format=%s, rotation=%s\n",
 			   plane->base.id,
 			   plane_type(intel_plane->base.type),
@@ -3130,8 +3032,10 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc)
 			   ((state->src_w & 0xffff) * 15625) >> 10,
 			   (state->src_h >> 16),
 			   ((state->src_h & 0xffff) * 15625) >> 10,
-			   state->fb ? drm_get_format_name(state->fb->pixel_format) : "N/A",
+			   format_name,
 			   plane_rotation(state->rotation));
+
+		kfree(format_name);
 	}
 }
 
@@ -3165,9 +3069,8 @@ static void intel_scaler_info(struct seq_file *m, struct intel_crtc *intel_crtc)
 
 static int i915_display_info(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_crtc *crtc;
 	struct drm_connector *connector;
 
@@ -3191,7 +3094,7 @@ static int i915_display_info(struct seq_file *m, void *unused)
 		if (pipe_config->base.active) {
 			intel_crtc_info(m, crtc);
 
-			active = cursor_position(dev, crtc->pipe, &x, &y);
+			active = cursor_position(dev_priv, crtc->pipe, &x, &y);
 			seq_printf(m, "\tcursor visible? %s, position (%d, %d), size %dx%d, addr 0x%08x, active? %s\n",
 				   yesno(crtc->cursor_base),
 				   x, y, crtc->base.cursor->state->crtc_w,
@@ -3220,15 +3123,14 @@ static int i915_display_info(struct seq_file *m, void *unused)
 
 static int i915_semaphore_status(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_engine_cs *engine;
-	int num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
+	int num_rings = INTEL_INFO(dev_priv)->num_rings;
 	enum intel_engine_id id;
 	int j, ret;
 
-	if (!i915_semaphore_is_enabled(dev_priv)) {
+	if (!i915.semaphores) {
 		seq_puts(m, "Semaphores are disabled\n");
 		return 0;
 	}
@@ -3238,11 +3140,11 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
 		return ret;
 	intel_runtime_pm_get(dev_priv);
 
-	if (IS_BROADWELL(dev)) {
+	if (IS_BROADWELL(dev_priv)) {
 		struct page *page;
 		uint64_t *seqno;
 
-		page = i915_gem_object_get_page(dev_priv->semaphore_obj, 0);
+		page = i915_gem_object_get_page(dev_priv->semaphore->obj, 0);
 
 		seqno = (uint64_t *)kmap_atomic(page);
 		for_each_engine_id(engine, dev_priv, id) {
@@ -3293,9 +3195,8 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
 
 static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	int i;
 
 	drm_modeset_lock_all(dev);
@@ -3323,9 +3224,8 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
 	int i;
 	int ret;
 	struct intel_engine_cs *engine;
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct i915_workarounds *workarounds = &dev_priv->workarounds;
 	enum intel_engine_id id;
 
@@ -3361,15 +3261,14 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
 
 static int i915_ddb_info(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct skl_ddb_allocation *ddb;
 	struct skl_ddb_entry *entry;
 	enum pipe pipe;
 	int plane;
 
-	if (INTEL_INFO(dev)->gen < 9)
+	if (INTEL_GEN(dev_priv) < 9)
 		return 0;
 
 	drm_modeset_lock_all(dev);
@@ -3399,7 +3298,8 @@ static int i915_ddb_info(struct seq_file *m, void *unused)
 }
 
 static void drrs_status_per_crtc(struct seq_file *m,
-		struct drm_device *dev, struct intel_crtc *intel_crtc)
+				 struct drm_device *dev,
+				 struct intel_crtc *intel_crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_drrs *drrs = &dev_priv->drrs;
@@ -3468,8 +3368,8 @@ static void drrs_status_per_crtc(struct seq_file *m,
 
 static int i915_drrs_status(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_crtc *intel_crtc;
 	int active_crtc_cnt = 0;
 
@@ -3492,14 +3392,14 @@ static int i915_drrs_status(struct seq_file *m, void *unused)
 
 struct pipe_crc_info {
 	const char *name;
-	struct drm_device *dev;
+	struct drm_i915_private *dev_priv;
 	enum pipe pipe;
 };
 
 static int i915_dp_mst_info(struct seq_file *m, void *unused)
 {
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_encoder *intel_encoder;
 	struct intel_digital_port *intel_dig_port;
 	struct drm_connector *connector;
@@ -3528,10 +3428,10 @@ static int i915_dp_mst_info(struct seq_file *m, void *unused)
 static int i915_pipe_crc_open(struct inode *inode, struct file *filep)
 {
 	struct pipe_crc_info *info = inode->i_private;
-	struct drm_i915_private *dev_priv = to_i915(info->dev);
+	struct drm_i915_private *dev_priv = info->dev_priv;
 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
 
-	if (info->pipe >= INTEL_INFO(info->dev)->num_pipes)
+	if (info->pipe >= INTEL_INFO(dev_priv)->num_pipes)
 		return -ENODEV;
 
 	spin_lock_irq(&pipe_crc->lock);
@@ -3552,7 +3452,7 @@ static int i915_pipe_crc_open(struct inode *inode, struct file *filep)
 static int i915_pipe_crc_release(struct inode *inode, struct file *filep)
 {
 	struct pipe_crc_info *info = inode->i_private;
-	struct drm_i915_private *dev_priv = to_i915(info->dev);
+	struct drm_i915_private *dev_priv = info->dev_priv;
 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
 
 	spin_lock_irq(&pipe_crc->lock);
@@ -3579,8 +3479,7 @@ i915_pipe_crc_read(struct file *filep, char __user *user_buf, size_t count,
 		   loff_t *pos)
 {
 	struct pipe_crc_info *info = filep->private_data;
-	struct drm_device *dev = info->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = info->dev_priv;
 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
 	char buf[PIPE_CRC_BUFFER_LEN];
 	int n_entries;
@@ -3621,7 +3520,6 @@ i915_pipe_crc_read(struct file *filep, char __user *user_buf, size_t count,
 	while (n_entries > 0) {
 		struct intel_pipe_crc_entry *entry =
 			&pipe_crc->entries[pipe_crc->tail];
-		int ret;
 
 		if (CIRC_CNT(pipe_crc->head, pipe_crc->tail,
 			     INTEL_PIPE_CRC_ENTRIES_NR) < 1)
@@ -3638,8 +3536,7 @@ i915_pipe_crc_read(struct file *filep, char __user *user_buf, size_t count,
 
 		spin_unlock_irq(&pipe_crc->lock);
 
-		ret = copy_to_user(user_buf, buf, PIPE_CRC_LINE_LEN);
-		if (ret == PIPE_CRC_LINE_LEN)
+		if (copy_to_user(user_buf, buf, PIPE_CRC_LINE_LEN))
 			return -EFAULT;
 
 		user_buf += PIPE_CRC_LINE_LEN;
@@ -3678,11 +3575,11 @@ static struct pipe_crc_info i915_pipe_crc_data[I915_MAX_PIPES] = {
 static int i915_pipe_crc_create(struct dentry *root, struct drm_minor *minor,
 				enum pipe pipe)
 {
-	struct drm_device *dev = minor->dev;
+	struct drm_i915_private *dev_priv = to_i915(minor->dev);
 	struct dentry *ent;
 	struct pipe_crc_info *info = &i915_pipe_crc_data[pipe];
 
-	info->dev = dev;
+	info->dev_priv = dev_priv;
 	ent = debugfs_create_file(info->name, S_IRUGO, root, info,
 				  &i915_pipe_crc_fops);
 	if (!ent)
@@ -3712,8 +3609,7 @@ static const char *pipe_crc_source_name(enum intel_pipe_crc_source source)
 
 static int display_crc_ctl_show(struct seq_file *m, void *data)
 {
-	struct drm_device *dev = m->private;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = m->private;
 	int i;
 
 	for (i = 0; i < I915_MAX_PIPES; i++)
@@ -3725,9 +3621,7 @@ static int display_crc_ctl_show(struct seq_file *m, void *data)
 
 static int display_crc_ctl_open(struct inode *inode, struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
-
-	return single_open(file, display_crc_ctl_show, dev);
+	return single_open(file, display_crc_ctl_show, inode->i_private);
 }
 
 static int i8xx_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source,
@@ -3750,9 +3644,11 @@ static int i8xx_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source,
 	return 0;
 }
 
-static int i9xx_pipe_crc_auto_source(struct drm_device *dev, enum pipe pipe,
+static int i9xx_pipe_crc_auto_source(struct drm_i915_private *dev_priv,
+				     enum pipe pipe,
 				     enum intel_pipe_crc_source *source)
 {
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_encoder *encoder;
 	struct intel_crtc *crtc;
 	struct intel_digital_port *dig_port;
@@ -3802,16 +3698,15 @@ static int i9xx_pipe_crc_auto_source(struct drm_device *dev, enum pipe pipe,
 	return ret;
 }
 
-static int vlv_pipe_crc_ctl_reg(struct drm_device *dev,
+static int vlv_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 				enum pipe pipe,
 				enum intel_pipe_crc_source *source,
 				uint32_t *val)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	bool need_stable_symbols = false;
 
 	if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) {
-		int ret = i9xx_pipe_crc_auto_source(dev, pipe, source);
+		int ret = i9xx_pipe_crc_auto_source(dev_priv, pipe, source);
 		if (ret)
 			return ret;
 	}
@@ -3829,7 +3724,7 @@ static int vlv_pipe_crc_ctl_reg(struct drm_device *dev,
 		need_stable_symbols = true;
 		break;
 	case INTEL_PIPE_CRC_SOURCE_DP_D:
-		if (!IS_CHERRYVIEW(dev))
+		if (!IS_CHERRYVIEW(dev_priv))
 			return -EINVAL;
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_DP_D_VLV;
 		need_stable_symbols = true;
@@ -3873,16 +3768,15 @@ static int vlv_pipe_crc_ctl_reg(struct drm_device *dev,
 	return 0;
 }
 
-static int i9xx_pipe_crc_ctl_reg(struct drm_device *dev,
+static int i9xx_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 				 enum pipe pipe,
 				 enum intel_pipe_crc_source *source,
 				 uint32_t *val)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	bool need_stable_symbols = false;
 
 	if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) {
-		int ret = i9xx_pipe_crc_auto_source(dev, pipe, source);
+		int ret = i9xx_pipe_crc_auto_source(dev_priv, pipe, source);
 		if (ret)
 			return ret;
 	}
@@ -3892,24 +3786,24 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_device *dev,
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PIPE_I9XX;
 		break;
 	case INTEL_PIPE_CRC_SOURCE_TV:
-		if (!SUPPORTS_TV(dev))
+		if (!SUPPORTS_TV(dev_priv))
 			return -EINVAL;
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_TV_PRE;
 		break;
 	case INTEL_PIPE_CRC_SOURCE_DP_B:
-		if (!IS_G4X(dev))
+		if (!IS_G4X(dev_priv))
 			return -EINVAL;
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_DP_B_G4X;
 		need_stable_symbols = true;
 		break;
 	case INTEL_PIPE_CRC_SOURCE_DP_C:
-		if (!IS_G4X(dev))
+		if (!IS_G4X(dev_priv))
 			return -EINVAL;
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_DP_C_G4X;
 		need_stable_symbols = true;
 		break;
 	case INTEL_PIPE_CRC_SOURCE_DP_D:
-		if (!IS_G4X(dev))
+		if (!IS_G4X(dev_priv))
 			return -EINVAL;
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_DP_D_G4X;
 		need_stable_symbols = true;
@@ -3933,7 +3827,7 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_device *dev,
 	if (need_stable_symbols) {
 		uint32_t tmp = I915_READ(PORT_DFT2_G4X);
 
-		WARN_ON(!IS_G4X(dev));
+		WARN_ON(!IS_G4X(dev_priv));
 
 		I915_WRITE(PORT_DFT_I9XX,
 			   I915_READ(PORT_DFT_I9XX) | DC_BALANCE_RESET);
@@ -3949,10 +3843,9 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_device *dev,
 	return 0;
 }
 
-static void vlv_undo_pipe_scramble_reset(struct drm_device *dev,
+static void vlv_undo_pipe_scramble_reset(struct drm_i915_private *dev_priv,
 					 enum pipe pipe)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t tmp = I915_READ(PORT_DFT2_G4X);
 
 	switch (pipe) {
@@ -3974,10 +3867,9 @@ static void vlv_undo_pipe_scramble_reset(struct drm_device *dev,
 
 }
 
-static void g4x_undo_pipe_scramble_reset(struct drm_device *dev,
+static void g4x_undo_pipe_scramble_reset(struct drm_i915_private *dev_priv,
 					 enum pipe pipe)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t tmp = I915_READ(PORT_DFT2_G4X);
 
 	if (pipe == PIPE_A)
@@ -4018,9 +3910,10 @@ static int ilk_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source,
 	return 0;
 }
 
-static void hsw_trans_edp_pipe_A_crc_wa(struct drm_device *dev, bool enable)
+static void hsw_trans_edp_pipe_A_crc_wa(struct drm_i915_private *dev_priv,
+					bool enable)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_crtc *crtc =
 		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_A]);
 	struct intel_crtc_state *pipe_config;
@@ -4054,7 +3947,7 @@ out:
 		drm_atomic_state_free(state);
 }
 
-static int ivb_pipe_crc_ctl_reg(struct drm_device *dev,
+static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 				enum pipe pipe,
 				enum intel_pipe_crc_source *source,
 				uint32_t *val)
@@ -4070,8 +3963,8 @@ static int ivb_pipe_crc_ctl_reg(struct drm_device *dev,
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_SPRITE_IVB;
 		break;
 	case INTEL_PIPE_CRC_SOURCE_PF:
-		if (IS_HASWELL(dev) && pipe == PIPE_A)
-			hsw_trans_edp_pipe_A_crc_wa(dev, true);
+		if (IS_HASWELL(dev_priv) && pipe == PIPE_A)
+			hsw_trans_edp_pipe_A_crc_wa(dev_priv, true);
 
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PF_IVB;
 		break;
@@ -4085,13 +3978,14 @@ static int ivb_pipe_crc_ctl_reg(struct drm_device *dev,
 	return 0;
 }
 
-static int pipe_crc_set_source(struct drm_device *dev, enum pipe pipe,
+static int pipe_crc_set_source(struct drm_i915_private *dev_priv,
+			       enum pipe pipe,
 			       enum intel_pipe_crc_source source)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_device *dev = &dev_priv->drm;
 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
-	struct intel_crtc *crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev,
-									pipe));
+	struct intel_crtc *crtc =
+			to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
 	enum intel_display_power_domain power_domain;
 	u32 val = 0; /* shut up gcc */
 	int ret;
@@ -4109,16 +4003,16 @@ static int pipe_crc_set_source(struct drm_device *dev, enum pipe pipe,
 		return -EIO;
 	}
 
-	if (IS_GEN2(dev))
+	if (IS_GEN2(dev_priv))
 		ret = i8xx_pipe_crc_ctl_reg(&source, &val);
-	else if (INTEL_INFO(dev)->gen < 5)
-		ret = i9xx_pipe_crc_ctl_reg(dev, pipe, &source, &val);
-	else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
-		ret = vlv_pipe_crc_ctl_reg(dev, pipe, &source, &val);
-	else if (IS_GEN5(dev) || IS_GEN6(dev))
+	else if (INTEL_GEN(dev_priv) < 5)
+		ret = i9xx_pipe_crc_ctl_reg(dev_priv, pipe, &source, &val);
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		ret = vlv_pipe_crc_ctl_reg(dev_priv, pipe, &source, &val);
+	else if (IS_GEN5(dev_priv) || IS_GEN6(dev_priv))
 		ret = ilk_pipe_crc_ctl_reg(&source, &val);
 	else
-		ret = ivb_pipe_crc_ctl_reg(dev, pipe, &source, &val);
+		ret = ivb_pipe_crc_ctl_reg(dev_priv, pipe, &source, &val);
 
 	if (ret != 0)
 		goto out;
@@ -4182,12 +4076,12 @@ static int pipe_crc_set_source(struct drm_device *dev, enum pipe pipe,
 
 		kfree(entries);
 
-		if (IS_G4X(dev))
-			g4x_undo_pipe_scramble_reset(dev, pipe);
-		else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
-			vlv_undo_pipe_scramble_reset(dev, pipe);
-		else if (IS_HASWELL(dev) && pipe == PIPE_A)
-			hsw_trans_edp_pipe_A_crc_wa(dev, false);
+		if (IS_G4X(dev_priv))
+			g4x_undo_pipe_scramble_reset(dev_priv, pipe);
+		else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+			vlv_undo_pipe_scramble_reset(dev_priv, pipe);
+		else if (IS_HASWELL(dev_priv) && pipe == PIPE_A)
+			hsw_trans_edp_pipe_A_crc_wa(dev_priv, false);
 
 		hsw_enable_ips(crtc);
 	}
@@ -4291,7 +4185,8 @@ display_crc_ctl_parse_source(const char *buf, enum intel_pipe_crc_source *s)
 	return -EINVAL;
 }
 
-static int display_crc_ctl_parse(struct drm_device *dev, char *buf, size_t len)
+static int display_crc_ctl_parse(struct drm_i915_private *dev_priv,
+				 char *buf, size_t len)
 {
 #define N_WORDS 3
 	int n_words;
@@ -4322,14 +4217,14 @@ static int display_crc_ctl_parse(struct drm_device *dev, char *buf, size_t len)
 		return -EINVAL;
 	}
 
-	return pipe_crc_set_source(dev, pipe, source);
+	return pipe_crc_set_source(dev_priv, pipe, source);
 }
 
 static ssize_t display_crc_ctl_write(struct file *file, const char __user *ubuf,
 				     size_t len, loff_t *offp)
 {
 	struct seq_file *m = file->private_data;
-	struct drm_device *dev = m->private;
+	struct drm_i915_private *dev_priv = m->private;
 	char *tmpbuf;
 	int ret;
 
@@ -4352,7 +4247,7 @@ static ssize_t display_crc_ctl_write(struct file *file, const char __user *ubuf,
 	}
 	tmpbuf[len] = '\0';
 
-	ret = display_crc_ctl_parse(dev, tmpbuf, len);
+	ret = display_crc_ctl_parse(dev_priv, tmpbuf, len);
 
 out:
 	kfree(tmpbuf);
@@ -4373,8 +4268,8 @@ static const struct file_operations i915_display_crc_ctl_fops = {
 };
 
 static ssize_t i915_displayport_test_active_write(struct file *file,
-					    const char __user *ubuf,
-					    size_t len, loff_t *offp)
+						  const char __user *ubuf,
+						  size_t len, loff_t *offp)
 {
 	char *input_buffer;
 	int status = 0;
@@ -4404,7 +4299,6 @@ static ssize_t i915_displayport_test_active_write(struct file *file,
 	DRM_DEBUG_DRIVER("Copied %d bytes from user\n", (unsigned int)len);
 
 	list_for_each_entry(connector, connector_list, head) {
-
 		if (connector->connector_type !=
 		    DRM_MODE_CONNECTOR_DisplayPort)
 			continue;
@@ -4442,7 +4336,6 @@ static int i915_displayport_test_active_show(struct seq_file *m, void *data)
 	struct intel_dp *intel_dp;
 
 	list_for_each_entry(connector, connector_list, head) {
-
 		if (connector->connector_type !=
 		    DRM_MODE_CONNECTOR_DisplayPort)
 			continue;
@@ -4462,11 +4355,12 @@ static int i915_displayport_test_active_show(struct seq_file *m, void *data)
 }
 
 static int i915_displayport_test_active_open(struct inode *inode,
-				       struct file *file)
+					     struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
+	struct drm_i915_private *dev_priv = inode->i_private;
 
-	return single_open(file, i915_displayport_test_active_show, dev);
+	return single_open(file, i915_displayport_test_active_show,
+			   &dev_priv->drm);
 }
 
 static const struct file_operations i915_displayport_test_active_fops = {
@@ -4486,7 +4380,6 @@ static int i915_displayport_test_data_show(struct seq_file *m, void *data)
 	struct intel_dp *intel_dp;
 
 	list_for_each_entry(connector, connector_list, head) {
-
 		if (connector->connector_type !=
 		    DRM_MODE_CONNECTOR_DisplayPort)
 			continue;
@@ -4502,11 +4395,12 @@ static int i915_displayport_test_data_show(struct seq_file *m, void *data)
 	return 0;
 }
 static int i915_displayport_test_data_open(struct inode *inode,
-				       struct file *file)
+					   struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
+	struct drm_i915_private *dev_priv = inode->i_private;
 
-	return single_open(file, i915_displayport_test_data_show, dev);
+	return single_open(file, i915_displayport_test_data_show,
+			   &dev_priv->drm);
 }
 
 static const struct file_operations i915_displayport_test_data_fops = {
@@ -4525,7 +4419,6 @@ static int i915_displayport_test_type_show(struct seq_file *m, void *data)
 	struct intel_dp *intel_dp;
 
 	list_for_each_entry(connector, connector_list, head) {
-
 		if (connector->connector_type !=
 		    DRM_MODE_CONNECTOR_DisplayPort)
 			continue;
@@ -4544,9 +4437,10 @@ static int i915_displayport_test_type_show(struct seq_file *m, void *data)
 static int i915_displayport_test_type_open(struct inode *inode,
 				       struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
+	struct drm_i915_private *dev_priv = inode->i_private;
 
-	return single_open(file, i915_displayport_test_type_show, dev);
+	return single_open(file, i915_displayport_test_type_show,
+			   &dev_priv->drm);
 }
 
 static const struct file_operations i915_displayport_test_type_fops = {
@@ -4559,13 +4453,14 @@ static const struct file_operations i915_displayport_test_type_fops = {
 
 static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 {
-	struct drm_device *dev = m->private;
+	struct drm_i915_private *dev_priv = m->private;
+	struct drm_device *dev = &dev_priv->drm;
 	int level;
 	int num_levels;
 
-	if (IS_CHERRYVIEW(dev))
+	if (IS_CHERRYVIEW(dev_priv))
 		num_levels = 3;
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev_priv))
 		num_levels = 1;
 	else
 		num_levels = ilk_wm_max_level(dev) + 1;
@@ -4579,8 +4474,8 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 		 * - WM1+ latency values in 0.5us units
 		 * - latencies are in us on gen9/vlv/chv
 		 */
-		if (INTEL_INFO(dev)->gen >= 9 || IS_VALLEYVIEW(dev) ||
-		    IS_CHERRYVIEW(dev))
+		if (INTEL_GEN(dev_priv) >= 9 || IS_VALLEYVIEW(dev_priv) ||
+		    IS_CHERRYVIEW(dev_priv))
 			latency *= 10;
 		else if (level > 0)
 			latency *= 5;
@@ -4594,14 +4489,13 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 
 static int pri_wm_latency_show(struct seq_file *m, void *data)
 {
-	struct drm_device *dev = m->private;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = m->private;
 	const uint16_t *latencies;
 
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		latencies = dev_priv->wm.skl_latency;
 	else
-		latencies = to_i915(dev)->wm.pri_latency;
+		latencies = dev_priv->wm.pri_latency;
 
 	wm_latency_show(m, latencies);
 
@@ -4610,14 +4504,13 @@ static int pri_wm_latency_show(struct seq_file *m, void *data)
 
 static int spr_wm_latency_show(struct seq_file *m, void *data)
 {
-	struct drm_device *dev = m->private;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = m->private;
 	const uint16_t *latencies;
 
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		latencies = dev_priv->wm.skl_latency;
 	else
-		latencies = to_i915(dev)->wm.spr_latency;
+		latencies = dev_priv->wm.spr_latency;
 
 	wm_latency_show(m, latencies);
 
@@ -4626,14 +4519,13 @@ static int spr_wm_latency_show(struct seq_file *m, void *data)
 
 static int cur_wm_latency_show(struct seq_file *m, void *data)
 {
-	struct drm_device *dev = m->private;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = m->private;
 	const uint16_t *latencies;
 
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		latencies = dev_priv->wm.skl_latency;
 	else
-		latencies = to_i915(dev)->wm.cur_latency;
+		latencies = dev_priv->wm.cur_latency;
 
 	wm_latency_show(m, latencies);
 
@@ -4642,48 +4534,49 @@ static int cur_wm_latency_show(struct seq_file *m, void *data)
 
 static int pri_wm_latency_open(struct inode *inode, struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
+	struct drm_i915_private *dev_priv = inode->i_private;
 
-	if (INTEL_INFO(dev)->gen < 5)
+	if (INTEL_GEN(dev_priv) < 5)
 		return -ENODEV;
 
-	return single_open(file, pri_wm_latency_show, dev);
+	return single_open(file, pri_wm_latency_show, dev_priv);
 }
 
 static int spr_wm_latency_open(struct inode *inode, struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
+	struct drm_i915_private *dev_priv = inode->i_private;
 
-	if (HAS_GMCH_DISPLAY(dev))
+	if (HAS_GMCH_DISPLAY(dev_priv))
 		return -ENODEV;
 
-	return single_open(file, spr_wm_latency_show, dev);
+	return single_open(file, spr_wm_latency_show, dev_priv);
 }
 
 static int cur_wm_latency_open(struct inode *inode, struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
+	struct drm_i915_private *dev_priv = inode->i_private;
 
-	if (HAS_GMCH_DISPLAY(dev))
+	if (HAS_GMCH_DISPLAY(dev_priv))
 		return -ENODEV;
 
-	return single_open(file, cur_wm_latency_show, dev);
+	return single_open(file, cur_wm_latency_show, dev_priv);
 }
 
 static ssize_t wm_latency_write(struct file *file, const char __user *ubuf,
 				size_t len, loff_t *offp, uint16_t wm[8])
 {
 	struct seq_file *m = file->private_data;
-	struct drm_device *dev = m->private;
+	struct drm_i915_private *dev_priv = m->private;
+	struct drm_device *dev = &dev_priv->drm;
 	uint16_t new[8] = { 0 };
 	int num_levels;
 	int level;
 	int ret;
 	char tmp[32];
 
-	if (IS_CHERRYVIEW(dev))
+	if (IS_CHERRYVIEW(dev_priv))
 		num_levels = 3;
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev_priv))
 		num_levels = 1;
 	else
 		num_levels = ilk_wm_max_level(dev) + 1;
@@ -4717,14 +4610,13 @@ static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf,
 				    size_t len, loff_t *offp)
 {
 	struct seq_file *m = file->private_data;
-	struct drm_device *dev = m->private;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = m->private;
 	uint16_t *latencies;
 
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		latencies = dev_priv->wm.skl_latency;
 	else
-		latencies = to_i915(dev)->wm.pri_latency;
+		latencies = dev_priv->wm.pri_latency;
 
 	return wm_latency_write(file, ubuf, len, offp, latencies);
 }
@@ -4733,14 +4625,13 @@ static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf,
 				    size_t len, loff_t *offp)
 {
 	struct seq_file *m = file->private_data;
-	struct drm_device *dev = m->private;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = m->private;
 	uint16_t *latencies;
 
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		latencies = dev_priv->wm.skl_latency;
 	else
-		latencies = to_i915(dev)->wm.spr_latency;
+		latencies = dev_priv->wm.spr_latency;
 
 	return wm_latency_write(file, ubuf, len, offp, latencies);
 }
@@ -4749,14 +4640,13 @@ static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf,
 				    size_t len, loff_t *offp)
 {
 	struct seq_file *m = file->private_data;
-	struct drm_device *dev = m->private;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = m->private;
 	uint16_t *latencies;
 
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		latencies = dev_priv->wm.skl_latency;
 	else
-		latencies = to_i915(dev)->wm.cur_latency;
+		latencies = dev_priv->wm.cur_latency;
 
 	return wm_latency_write(file, ubuf, len, offp, latencies);
 }
@@ -4791,8 +4681,7 @@ static const struct file_operations i915_cur_wm_latency_fops = {
 static int
 i915_wedged_get(void *data, u64 *val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 
 	*val = i915_terminally_wedged(&dev_priv->gpu_error);
 
@@ -4802,8 +4691,7 @@ i915_wedged_get(void *data, u64 *val)
 static int
 i915_wedged_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 
 	/*
 	 * There is no safeguard against this debugfs entry colliding
@@ -4833,8 +4721,7 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_wedged_fops,
 static int
 i915_ring_missed_irq_get(void *data, u64 *val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 
 	*val = dev_priv->gpu_error.missed_irq_rings;
 	return 0;
@@ -4843,8 +4730,8 @@ i915_ring_missed_irq_get(void *data, u64 *val)
 static int
 i915_ring_missed_irq_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
+	struct drm_device *dev = &dev_priv->drm;
 	int ret;
 
 	/* Lock against concurrent debugfs callers */
@@ -4864,8 +4751,7 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_ring_missed_irq_fops,
 static int
 i915_ring_test_irq_get(void *data, u64 *val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 
 	*val = dev_priv->gpu_error.test_irq_rings;
 
@@ -4875,8 +4761,7 @@ i915_ring_test_irq_get(void *data, u64 *val)
 static int
 i915_ring_test_irq_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 
 	val &= INTEL_INFO(dev_priv)->ring_mask;
 	DRM_DEBUG_DRIVER("Masking interrupts on rings 0x%08llx\n", val);
@@ -4908,8 +4793,8 @@ i915_drop_caches_get(void *data, u64 *val)
 static int
 i915_drop_caches_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
+	struct drm_device *dev = &dev_priv->drm;
 	int ret;
 
 	DRM_DEBUG("Dropping caches: 0x%08llx\n", val);
@@ -4921,7 +4806,9 @@ i915_drop_caches_set(void *data, u64 val)
 		return ret;
 
 	if (val & DROP_ACTIVE) {
-		ret = i915_gem_wait_for_idle(dev_priv);
+		ret = i915_gem_wait_for_idle(dev_priv,
+					     I915_WAIT_INTERRUPTIBLE |
+					     I915_WAIT_LOCKED);
 		if (ret)
 			goto unlock;
 	}
@@ -4948,38 +4835,25 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_drop_caches_fops,
 static int
 i915_max_freq_get(void *data, u64 *val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret;
+	struct drm_i915_private *dev_priv = data;
 
-	if (INTEL_INFO(dev)->gen < 6)
+	if (INTEL_GEN(dev_priv) < 6)
 		return -ENODEV;
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
-	ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
-	if (ret)
-		return ret;
-
 	*val = intel_gpu_freq(dev_priv, dev_priv->rps.max_freq_softlimit);
-	mutex_unlock(&dev_priv->rps.hw_lock);
-
 	return 0;
 }
 
 static int
 i915_max_freq_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 	u32 hw_max, hw_min;
 	int ret;
 
-	if (INTEL_INFO(dev)->gen < 6)
+	if (INTEL_GEN(dev_priv) < 6)
 		return -ENODEV;
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
 	DRM_DEBUG_DRIVER("Manually setting max freq to %llu\n", val);
 
 	ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
@@ -5015,38 +4889,25 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_max_freq_fops,
 static int
 i915_min_freq_get(void *data, u64 *val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret;
+	struct drm_i915_private *dev_priv = data;
 
-	if (INTEL_INFO(dev)->gen < 6)
+	if (INTEL_GEN(dev_priv) < 6)
 		return -ENODEV;
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
-	ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
-	if (ret)
-		return ret;
-
 	*val = intel_gpu_freq(dev_priv, dev_priv->rps.min_freq_softlimit);
-	mutex_unlock(&dev_priv->rps.hw_lock);
-
 	return 0;
 }
 
 static int
 i915_min_freq_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 	u32 hw_max, hw_min;
 	int ret;
 
-	if (INTEL_INFO(dev)->gen < 6)
+	if (INTEL_GEN(dev_priv) < 6)
 		return -ENODEV;
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
 	DRM_DEBUG_DRIVER("Manually setting min freq to %llu\n", val);
 
 	ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
@@ -5061,7 +4922,8 @@ i915_min_freq_set(void *data, u64 val)
 	hw_max = dev_priv->rps.max_freq;
 	hw_min = dev_priv->rps.min_freq;
 
-	if (val < hw_min || val > hw_max || val > dev_priv->rps.max_freq_softlimit) {
+	if (val < hw_min ||
+	    val > hw_max || val > dev_priv->rps.max_freq_softlimit) {
 		mutex_unlock(&dev_priv->rps.hw_lock);
 		return -EINVAL;
 	}
@@ -5082,12 +4944,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_min_freq_fops,
 static int
 i915_cache_sharing_get(void *data, u64 *val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
+	struct drm_device *dev = &dev_priv->drm;
 	u32 snpcr;
 	int ret;
 
-	if (!(IS_GEN6(dev) || IS_GEN7(dev)))
+	if (!(IS_GEN6(dev_priv) || IS_GEN7(dev_priv)))
 		return -ENODEV;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -5098,7 +4960,7 @@ i915_cache_sharing_get(void *data, u64 *val)
 	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
 
 	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev_priv->drm.struct_mutex);
+	mutex_unlock(&dev->struct_mutex);
 
 	*val = (snpcr & GEN6_MBC_SNPCR_MASK) >> GEN6_MBC_SNPCR_SHIFT;
 
@@ -5108,11 +4970,10 @@ i915_cache_sharing_get(void *data, u64 *val)
 static int
 i915_cache_sharing_set(void *data, u64 val)
 {
-	struct drm_device *dev = data;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = data;
 	u32 snpcr;
 
-	if (!(IS_GEN6(dev) || IS_GEN7(dev)))
+	if (!(IS_GEN6(dev_priv) || IS_GEN7(dev_priv)))
 		return -ENODEV;
 
 	if (val > 3)
@@ -5135,18 +4996,9 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_cache_sharing_fops,
 			i915_cache_sharing_get, i915_cache_sharing_set,
 			"%llu\n");
 
-struct sseu_dev_status {
-	unsigned int slice_total;
-	unsigned int subslice_total;
-	unsigned int subslice_per_slice;
-	unsigned int eu_total;
-	unsigned int eu_per_subslice;
-};
-
-static void cherryview_sseu_device_status(struct drm_device *dev,
-					  struct sseu_dev_status *stat)
+static void cherryview_sseu_device_status(struct drm_i915_private *dev_priv,
+					  struct sseu_dev_info *sseu)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	int ss_max = 2;
 	int ss;
 	u32 sig1[ss_max], sig2[ss_max];
@@ -5163,28 +5015,27 @@ static void cherryview_sseu_device_status(struct drm_device *dev,
 			/* skip disabled subslice */
 			continue;
 
-		stat->slice_total = 1;
-		stat->subslice_per_slice++;
+		sseu->slice_mask = BIT(0);
+		sseu->subslice_mask |= BIT(ss);
 		eu_cnt = ((sig1[ss] & CHV_EU08_PG_ENABLE) ? 0 : 2) +
 			 ((sig1[ss] & CHV_EU19_PG_ENABLE) ? 0 : 2) +
 			 ((sig1[ss] & CHV_EU210_PG_ENABLE) ? 0 : 2) +
 			 ((sig2[ss] & CHV_EU311_PG_ENABLE) ? 0 : 2);
-		stat->eu_total += eu_cnt;
-		stat->eu_per_subslice = max(stat->eu_per_subslice, eu_cnt);
+		sseu->eu_total += eu_cnt;
+		sseu->eu_per_subslice = max_t(unsigned int,
+					      sseu->eu_per_subslice, eu_cnt);
 	}
-	stat->subslice_total = stat->subslice_per_slice;
 }
 
-static void gen9_sseu_device_status(struct drm_device *dev,
-				    struct sseu_dev_status *stat)
+static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
+				    struct sseu_dev_info *sseu)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	int s_max = 3, ss_max = 4;
 	int s, ss;
 	u32 s_reg[s_max], eu_reg[2*s_max], eu_mask[2];
 
 	/* BXT has a single slice and at most 3 subslices. */
-	if (IS_BROXTON(dev)) {
+	if (IS_BROXTON(dev_priv)) {
 		s_max = 1;
 		ss_max = 3;
 	}
@@ -5205,126 +5056,134 @@ static void gen9_sseu_device_status(struct drm_device *dev,
 		     GEN9_PGCTL_SSB_EU311_ACK;
 
 	for (s = 0; s < s_max; s++) {
-		unsigned int ss_cnt = 0;
-
 		if ((s_reg[s] & GEN9_PGCTL_SLICE_ACK) == 0)
 			/* skip disabled slice */
 			continue;
 
-		stat->slice_total++;
+		sseu->slice_mask |= BIT(s);
 
-		if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
-			ss_cnt = INTEL_INFO(dev)->subslice_per_slice;
+		if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+			sseu->subslice_mask =
+				INTEL_INFO(dev_priv)->sseu.subslice_mask;
 
 		for (ss = 0; ss < ss_max; ss++) {
 			unsigned int eu_cnt;
 
-			if (IS_BROXTON(dev) &&
-			    !(s_reg[s] & (GEN9_PGCTL_SS_ACK(ss))))
-				/* skip disabled subslice */
-				continue;
+			if (IS_BROXTON(dev_priv)) {
+				if (!(s_reg[s] & (GEN9_PGCTL_SS_ACK(ss))))
+					/* skip disabled subslice */
+					continue;
 
-			if (IS_BROXTON(dev))
-				ss_cnt++;
+				sseu->subslice_mask |= BIT(ss);
+			}
 
 			eu_cnt = 2 * hweight32(eu_reg[2*s + ss/2] &
 					       eu_mask[ss%2]);
-			stat->eu_total += eu_cnt;
-			stat->eu_per_subslice = max(stat->eu_per_subslice,
-						    eu_cnt);
+			sseu->eu_total += eu_cnt;
+			sseu->eu_per_subslice = max_t(unsigned int,
+						      sseu->eu_per_subslice,
+						      eu_cnt);
 		}
-
-		stat->subslice_total += ss_cnt;
-		stat->subslice_per_slice = max(stat->subslice_per_slice,
-					       ss_cnt);
 	}
 }
 
-static void broadwell_sseu_device_status(struct drm_device *dev,
-					 struct sseu_dev_status *stat)
+static void broadwell_sseu_device_status(struct drm_i915_private *dev_priv,
+					 struct sseu_dev_info *sseu)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int s;
 	u32 slice_info = I915_READ(GEN8_GT_SLICE_INFO);
+	int s;
 
-	stat->slice_total = hweight32(slice_info & GEN8_LSLICESTAT_MASK);
+	sseu->slice_mask = slice_info & GEN8_LSLICESTAT_MASK;
 
-	if (stat->slice_total) {
-		stat->subslice_per_slice = INTEL_INFO(dev)->subslice_per_slice;
-		stat->subslice_total = stat->slice_total *
-				       stat->subslice_per_slice;
-		stat->eu_per_subslice = INTEL_INFO(dev)->eu_per_subslice;
-		stat->eu_total = stat->eu_per_subslice * stat->subslice_total;
+	if (sseu->slice_mask) {
+		sseu->subslice_mask = INTEL_INFO(dev_priv)->sseu.subslice_mask;
+		sseu->eu_per_subslice =
+				INTEL_INFO(dev_priv)->sseu.eu_per_subslice;
+		sseu->eu_total = sseu->eu_per_subslice *
+				 sseu_subslice_total(sseu);
 
 		/* subtract fused off EU(s) from enabled slice(s) */
-		for (s = 0; s < stat->slice_total; s++) {
-			u8 subslice_7eu = INTEL_INFO(dev)->subslice_7eu[s];
+		for (s = 0; s < fls(sseu->slice_mask); s++) {
+			u8 subslice_7eu =
+				INTEL_INFO(dev_priv)->sseu.subslice_7eu[s];
 
-			stat->eu_total -= hweight8(subslice_7eu);
+			sseu->eu_total -= hweight8(subslice_7eu);
 		}
 	}
 }
 
-static int i915_sseu_status(struct seq_file *m, void *unused)
+static void i915_print_sseu_info(struct seq_file *m, bool is_available_info,
+				 const struct sseu_dev_info *sseu)
 {
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct sseu_dev_status stat;
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	const char *type = is_available_info ? "Available" : "Enabled";
 
-	if (INTEL_INFO(dev)->gen < 8)
-		return -ENODEV;
+	seq_printf(m, "  %s Slice Mask: %04x\n", type,
+		   sseu->slice_mask);
+	seq_printf(m, "  %s Slice Total: %u\n", type,
+		   hweight8(sseu->slice_mask));
+	seq_printf(m, "  %s Subslice Total: %u\n", type,
+		   sseu_subslice_total(sseu));
+	seq_printf(m, "  %s Subslice Mask: %04x\n", type,
+		   sseu->subslice_mask);
+	seq_printf(m, "  %s Subslice Per Slice: %u\n", type,
+		   hweight8(sseu->subslice_mask));
+	seq_printf(m, "  %s EU Total: %u\n", type,
+		   sseu->eu_total);
+	seq_printf(m, "  %s EU Per Subslice: %u\n", type,
+		   sseu->eu_per_subslice);
+
+	if (!is_available_info)
+		return;
+
+	seq_printf(m, "  Has Pooled EU: %s\n", yesno(HAS_POOLED_EU(dev_priv)));
+	if (HAS_POOLED_EU(dev_priv))
+		seq_printf(m, "  Min EU in pool: %u\n", sseu->min_eu_in_pool);
 
-	seq_puts(m, "SSEU Device Info\n");
-	seq_printf(m, "  Available Slice Total: %u\n",
-		   INTEL_INFO(dev)->slice_total);
-	seq_printf(m, "  Available Subslice Total: %u\n",
-		   INTEL_INFO(dev)->subslice_total);
-	seq_printf(m, "  Available Subslice Per Slice: %u\n",
-		   INTEL_INFO(dev)->subslice_per_slice);
-	seq_printf(m, "  Available EU Total: %u\n",
-		   INTEL_INFO(dev)->eu_total);
-	seq_printf(m, "  Available EU Per Subslice: %u\n",
-		   INTEL_INFO(dev)->eu_per_subslice);
-	seq_printf(m, "  Has Pooled EU: %s\n", yesno(HAS_POOLED_EU(dev)));
-	if (HAS_POOLED_EU(dev))
-		seq_printf(m, "  Min EU in pool: %u\n",
-			   INTEL_INFO(dev)->min_eu_in_pool);
 	seq_printf(m, "  Has Slice Power Gating: %s\n",
-		   yesno(INTEL_INFO(dev)->has_slice_pg));
+		   yesno(sseu->has_slice_pg));
 	seq_printf(m, "  Has Subslice Power Gating: %s\n",
-		   yesno(INTEL_INFO(dev)->has_subslice_pg));
+		   yesno(sseu->has_subslice_pg));
 	seq_printf(m, "  Has EU Power Gating: %s\n",
-		   yesno(INTEL_INFO(dev)->has_eu_pg));
+		   yesno(sseu->has_eu_pg));
+}
+
+static int i915_sseu_status(struct seq_file *m, void *unused)
+{
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct sseu_dev_info sseu;
+
+	if (INTEL_GEN(dev_priv) < 8)
+		return -ENODEV;
+
+	seq_puts(m, "SSEU Device Info\n");
+	i915_print_sseu_info(m, true, &INTEL_INFO(dev_priv)->sseu);
 
 	seq_puts(m, "SSEU Device Status\n");
-	memset(&stat, 0, sizeof(stat));
-	if (IS_CHERRYVIEW(dev)) {
-		cherryview_sseu_device_status(dev, &stat);
-	} else if (IS_BROADWELL(dev)) {
-		broadwell_sseu_device_status(dev, &stat);
-	} else if (INTEL_INFO(dev)->gen >= 9) {
-		gen9_sseu_device_status(dev, &stat);
-	}
-	seq_printf(m, "  Enabled Slice Total: %u\n",
-		   stat.slice_total);
-	seq_printf(m, "  Enabled Subslice Total: %u\n",
-		   stat.subslice_total);
-	seq_printf(m, "  Enabled Subslice Per Slice: %u\n",
-		   stat.subslice_per_slice);
-	seq_printf(m, "  Enabled EU Total: %u\n",
-		   stat.eu_total);
-	seq_printf(m, "  Enabled EU Per Subslice: %u\n",
-		   stat.eu_per_subslice);
+	memset(&sseu, 0, sizeof(sseu));
+
+	intel_runtime_pm_get(dev_priv);
+
+	if (IS_CHERRYVIEW(dev_priv)) {
+		cherryview_sseu_device_status(dev_priv, &sseu);
+	} else if (IS_BROADWELL(dev_priv)) {
+		broadwell_sseu_device_status(dev_priv, &sseu);
+	} else if (INTEL_GEN(dev_priv) >= 9) {
+		gen9_sseu_device_status(dev_priv, &sseu);
+	}
+
+	intel_runtime_pm_put(dev_priv);
+
+	i915_print_sseu_info(m, false, &sseu);
 
 	return 0;
 }
 
 static int i915_forcewake_open(struct inode *inode, struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = inode->i_private;
 
-	if (INTEL_INFO(dev)->gen < 6)
+	if (INTEL_GEN(dev_priv) < 6)
 		return 0;
 
 	intel_runtime_pm_get(dev_priv);
@@ -5335,10 +5194,9 @@ static int i915_forcewake_open(struct inode *inode, struct file *file)
 
 static int i915_forcewake_release(struct inode *inode, struct file *file)
 {
-	struct drm_device *dev = inode->i_private;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = inode->i_private;
 
-	if (INTEL_INFO(dev)->gen < 6)
+	if (INTEL_GEN(dev_priv) < 6)
 		return 0;
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
@@ -5355,12 +5213,11 @@ static const struct file_operations i915_forcewake_fops = {
 
 static int i915_forcewake_create(struct dentry *root, struct drm_minor *minor)
 {
-	struct drm_device *dev = minor->dev;
 	struct dentry *ent;
 
 	ent = debugfs_create_file("i915_forcewake_user",
 				  S_IRUSR,
-				  root, dev,
+				  root, to_i915(minor->dev),
 				  &i915_forcewake_fops);
 	if (!ent)
 		return -ENOMEM;
@@ -5373,12 +5230,11 @@ static int i915_debugfs_create(struct dentry *root,
 			       const char *name,
 			       const struct file_operations *fops)
 {
-	struct drm_device *dev = minor->dev;
 	struct dentry *ent;
 
 	ent = debugfs_create_file(name,
 				  S_IRUGO | S_IWUSR,
-				  root, dev,
+				  root, to_i915(minor->dev),
 				  fops);
 	if (!ent)
 		return -ENOMEM;
@@ -5390,9 +5246,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_capabilities", i915_capabilities, 0},
 	{"i915_gem_objects", i915_gem_object_info, 0},
 	{"i915_gem_gtt", i915_gem_gtt_info, 0},
-	{"i915_gem_pinned", i915_gem_gtt_info, 0, (void *) PINNED_LIST},
-	{"i915_gem_active", i915_gem_object_list_info, 0, (void *) ACTIVE_LIST},
-	{"i915_gem_inactive", i915_gem_object_list_info, 0, (void *) INACTIVE_LIST},
+	{"i915_gem_pin_display", i915_gem_gtt_info, 0, (void *)1},
 	{"i915_gem_stolen", i915_gem_stolen_list_info },
 	{"i915_gem_pageflip", i915_gem_pageflip_info, 0},
 	{"i915_gem_request", i915_gem_request_info, 0},
@@ -5467,9 +5321,8 @@ static const struct i915_debugfs_files {
 	{"i915_dp_test_active", &i915_displayport_test_active_fops}
 };
 
-void intel_display_crc_init(struct drm_device *dev)
+void intel_display_crc_init(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum pipe pipe;
 
 	for_each_pipe(dev_priv, pipe) {
@@ -5517,7 +5370,7 @@ void i915_debugfs_unregister(struct drm_i915_private *dev_priv)
 	drm_debugfs_remove_files(i915_debugfs_list,
 				 I915_DEBUGFS_ENTRIES, minor);
 
-	drm_debugfs_remove_files((struct drm_info_list *) &i915_forcewake_fops,
+	drm_debugfs_remove_files((struct drm_info_list *)&i915_forcewake_fops,
 				 1, minor);
 
 	for (i = 0; i < ARRAY_SIZE(i915_pipe_crc_data); i++) {
@@ -5529,7 +5382,7 @@ void i915_debugfs_unregister(struct drm_i915_private *dev_priv)
 
 	for (i = 0; i < ARRAY_SIZE(i915_debugfs_files); i++) {
 		struct drm_info_list *info_list =
-			(struct drm_info_list *) i915_debugfs_files[i].fops;
+			(struct drm_info_list *)i915_debugfs_files[i].fops;
 
 		drm_debugfs_remove_files(info_list, 1, minor);
 	}
@@ -5609,6 +5462,40 @@ static const struct file_operations i915_dpcd_fops = {
 	.release = single_release,
 };
 
+static int i915_panel_show(struct seq_file *m, void *data)
+{
+	struct drm_connector *connector = m->private;
+	struct intel_dp *intel_dp =
+		enc_to_intel_dp(&intel_attached_encoder(connector)->base);
+
+	if (connector->status != connector_status_connected)
+		return -ENODEV;
+
+	seq_printf(m, "Panel power up delay: %d\n",
+		   intel_dp->panel_power_up_delay);
+	seq_printf(m, "Panel power down delay: %d\n",
+		   intel_dp->panel_power_down_delay);
+	seq_printf(m, "Backlight on delay: %d\n",
+		   intel_dp->backlight_on_delay);
+	seq_printf(m, "Backlight off delay: %d\n",
+		   intel_dp->backlight_off_delay);
+
+	return 0;
+}
+
+static int i915_panel_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, i915_panel_show, inode->i_private);
+}
+
+static const struct file_operations i915_panel_fops = {
+	.owner = THIS_MODULE,
+	.open = i915_panel_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
 /**
  * i915_debugfs_connector_add - add i915 specific connector debugfs files
  * @connector: pointer to a registered drm_connector
@@ -5628,8 +5515,12 @@ int i915_debugfs_connector_add(struct drm_connector *connector)
 
 	if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
 	    connector->connector_type == DRM_MODE_CONNECTOR_eDP)
-		debugfs_create_file("i915_dpcd", S_IRUGO, root, connector,
-				    &i915_dpcd_fops);
+		debugfs_create_file("i915_dpcd", S_IRUGO, root,
+				    connector, &i915_dpcd_fops);
+
+	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+		debugfs_create_file("i915_panel_timings", S_IRUGO, root,
+				    connector, &i915_panel_fops);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 95ddd56b89f0..bfb2efd8d4d4 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -77,7 +77,7 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level,
 	      const char *fmt, ...)
 {
 	static bool shown_bug_once;
-	struct device *dev = dev_priv->drm.dev;
+	struct device *kdev = dev_priv->drm.dev;
 	bool is_error = level[1] <= KERN_ERR[1];
 	bool is_debug = level[1] == KERN_DEBUG[1];
 	struct va_format vaf;
@@ -91,11 +91,11 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level,
 	vaf.fmt = fmt;
 	vaf.va = &args;
 
-	dev_printk(level, dev, "[" DRM_NAME ":%ps] %pV",
+	dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV",
 		   __builtin_return_address(0), &vaf);
 
 	if (is_error && !shown_bug_once) {
-		dev_notice(dev, "%s", FDO_BUG_MSG);
+		dev_notice(kdev, "%s", FDO_BUG_MSG);
 		shown_bug_once = true;
 	}
 
@@ -228,31 +228,11 @@ static void intel_detect_pch(struct drm_device *dev)
 	pci_dev_put(pch);
 }
 
-bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv)
-{
-	if (INTEL_GEN(dev_priv) < 6)
-		return false;
-
-	if (i915.semaphores >= 0)
-		return i915.semaphores;
-
-	/* TODO: make semaphores and Execlists play nicely together */
-	if (i915.enable_execlists)
-		return false;
-
-#ifdef CONFIG_INTEL_IOMMU
-	/* Enable semaphores on SNB when IO remapping is off */
-	if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped)
-		return false;
-#endif
-
-	return true;
-}
-
 static int i915_getparam(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	drm_i915_getparam_t *param = data;
 	int value;
 
@@ -263,13 +243,10 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		/* Reject all old ums/dri params. */
 		return -ENODEV;
 	case I915_PARAM_CHIPSET_ID:
-		value = dev->pdev->device;
+		value = pdev->device;
 		break;
 	case I915_PARAM_REVISION:
-		value = dev->pdev->revision;
-		break;
-	case I915_PARAM_HAS_GEM:
-		value = 1;
+		value = pdev->revision;
 		break;
 	case I915_PARAM_NUM_FENCES_AVAIL:
 		value = dev_priv->num_fence_regs;
@@ -277,13 +254,6 @@ static int i915_getparam(struct drm_device *dev, void *data,
 	case I915_PARAM_HAS_OVERLAY:
 		value = dev_priv->overlay ? 1 : 0;
 		break;
-	case I915_PARAM_HAS_PAGEFLIPPING:
-		value = 1;
-		break;
-	case I915_PARAM_HAS_EXECBUF2:
-		/* depends on GEM */
-		value = 1;
-		break;
 	case I915_PARAM_HAS_BSD:
 		value = intel_engine_initialized(&dev_priv->engine[VCS]);
 		break;
@@ -296,67 +266,34 @@ static int i915_getparam(struct drm_device *dev, void *data,
 	case I915_PARAM_HAS_BSD2:
 		value = intel_engine_initialized(&dev_priv->engine[VCS2]);
 		break;
-	case I915_PARAM_HAS_RELAXED_FENCING:
-		value = 1;
-		break;
-	case I915_PARAM_HAS_COHERENT_RINGS:
-		value = 1;
-		break;
 	case I915_PARAM_HAS_EXEC_CONSTANTS:
-		value = INTEL_INFO(dev)->gen >= 4;
-		break;
-	case I915_PARAM_HAS_RELAXED_DELTA:
-		value = 1;
-		break;
-	case I915_PARAM_HAS_GEN7_SOL_RESET:
-		value = 1;
+		value = INTEL_GEN(dev_priv) >= 4;
 		break;
 	case I915_PARAM_HAS_LLC:
-		value = HAS_LLC(dev);
+		value = HAS_LLC(dev_priv);
 		break;
 	case I915_PARAM_HAS_WT:
-		value = HAS_WT(dev);
+		value = HAS_WT(dev_priv);
 		break;
 	case I915_PARAM_HAS_ALIASING_PPGTT:
-		value = USES_PPGTT(dev);
-		break;
-	case I915_PARAM_HAS_WAIT_TIMEOUT:
-		value = 1;
+		value = USES_PPGTT(dev_priv);
 		break;
 	case I915_PARAM_HAS_SEMAPHORES:
-		value = i915_semaphore_is_enabled(dev_priv);
-		break;
-	case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
-		value = 1;
+		value = i915.semaphores;
 		break;
 	case I915_PARAM_HAS_SECURE_BATCHES:
 		value = capable(CAP_SYS_ADMIN);
 		break;
-	case I915_PARAM_HAS_PINNED_BATCHES:
-		value = 1;
-		break;
-	case I915_PARAM_HAS_EXEC_NO_RELOC:
-		value = 1;
-		break;
-	case I915_PARAM_HAS_EXEC_HANDLE_LUT:
-		value = 1;
-		break;
 	case I915_PARAM_CMD_PARSER_VERSION:
 		value = i915_cmd_parser_get_version(dev_priv);
 		break;
-	case I915_PARAM_HAS_COHERENT_PHYS_GTT:
-		value = 1;
-		break;
-	case I915_PARAM_MMAP_VERSION:
-		value = 1;
-		break;
 	case I915_PARAM_SUBSLICE_TOTAL:
-		value = INTEL_INFO(dev)->subslice_total;
+		value = sseu_subslice_total(&INTEL_INFO(dev_priv)->sseu);
 		if (!value)
 			return -ENODEV;
 		break;
 	case I915_PARAM_EU_TOTAL:
-		value = INTEL_INFO(dev)->eu_total;
+		value = INTEL_INFO(dev_priv)->sseu.eu_total;
 		if (!value)
 			return -ENODEV;
 		break;
@@ -364,16 +301,43 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv);
 		break;
 	case I915_PARAM_HAS_RESOURCE_STREAMER:
-		value = HAS_RESOURCE_STREAMER(dev);
-		break;
-	case I915_PARAM_HAS_EXEC_SOFTPIN:
-		value = 1;
+		value = HAS_RESOURCE_STREAMER(dev_priv);
 		break;
 	case I915_PARAM_HAS_POOLED_EU:
-		value = HAS_POOLED_EU(dev);
+		value = HAS_POOLED_EU(dev_priv);
 		break;
 	case I915_PARAM_MIN_EU_IN_POOL:
-		value = INTEL_INFO(dev)->min_eu_in_pool;
+		value = INTEL_INFO(dev_priv)->sseu.min_eu_in_pool;
+		break;
+	case I915_PARAM_MMAP_GTT_VERSION:
+		/* Though we've started our numbering from 1, and so class all
+		 * earlier versions as 0, in effect their value is undefined as
+		 * the ioctl will report EINVAL for the unknown param!
+		 */
+		value = i915_gem_mmap_gtt_version();
+		break;
+	case I915_PARAM_MMAP_VERSION:
+		/* Remember to bump this if the version changes! */
+	case I915_PARAM_HAS_GEM:
+	case I915_PARAM_HAS_PAGEFLIPPING:
+	case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */
+	case I915_PARAM_HAS_RELAXED_FENCING:
+	case I915_PARAM_HAS_COHERENT_RINGS:
+	case I915_PARAM_HAS_RELAXED_DELTA:
+	case I915_PARAM_HAS_GEN7_SOL_RESET:
+	case I915_PARAM_HAS_WAIT_TIMEOUT:
+	case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
+	case I915_PARAM_HAS_PINNED_BATCHES:
+	case I915_PARAM_HAS_EXEC_NO_RELOC:
+	case I915_PARAM_HAS_EXEC_HANDLE_LUT:
+	case I915_PARAM_HAS_COHERENT_PHYS_GTT:
+	case I915_PARAM_HAS_EXEC_SOFTPIN:
+		/* For the time being all of these are always true;
+		 * if some supported hardware does not have one of these
+		 * features this value needs to be provided from
+		 * INTEL_INFO(), a feature macro, or similar.
+		 */
+		value = 1;
 		break;
 	default:
 		DRM_DEBUG("Unknown parameter %d\n", param->param);
@@ -537,7 +501,7 @@ static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_
 		pr_info("switched on\n");
 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 		/* i915 resume handler doesn't set to D0 */
-		pci_set_power_state(dev->pdev, PCI_D0);
+		pci_set_power_state(pdev, PCI_D0);
 		i915_resume_switcheroo(dev);
 		dev->switch_power_state = DRM_SWITCH_POWER_ON;
 	} else {
@@ -595,7 +559,6 @@ static void i915_gem_fini(struct drm_device *dev)
 	}
 
 	mutex_lock(&dev->struct_mutex);
-	i915_gem_reset(dev);
 	i915_gem_cleanup_engines(dev);
 	i915_gem_context_fini(dev);
 	mutex_unlock(&dev->struct_mutex);
@@ -606,6 +569,7 @@ static void i915_gem_fini(struct drm_device *dev)
 static int i915_load_modeset_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	int ret;
 
 	if (i915_inject_load_failure())
@@ -622,13 +586,13 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	 * then we do not take part in VGA arbitration and the
 	 * vga_client_register() fails with -ENODEV.
 	 */
-	ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
+	ret = vga_client_register(pdev, dev, NULL, i915_vga_set_decode);
 	if (ret && ret != -ENODEV)
 		goto out;
 
 	intel_register_dsm_handler();
 
-	ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false);
+	ret = vga_switcheroo_register_client(pdev, &i915_switcheroo_ops, false);
 	if (ret)
 		goto cleanup_vga_client;
 
@@ -680,9 +644,9 @@ cleanup_irq:
 cleanup_csr:
 	intel_csr_ucode_fini(dev_priv);
 	intel_power_domains_fini(dev_priv);
-	vga_switcheroo_unregister_client(dev->pdev);
+	vga_switcheroo_unregister_client(pdev);
 cleanup_vga_client:
-	vga_client_register(dev->pdev, NULL, NULL, NULL);
+	vga_client_register(pdev, NULL, NULL, NULL);
 out:
 	return ret;
 }
@@ -706,7 +670,7 @@ static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
 	primary =
 		pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
 
-	ret = remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
+	ret = drm_fb_helper_remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
 
 	kfree(ap);
 
@@ -848,6 +812,8 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	mutex_init(&dev_priv->wm.wm_mutex);
 	mutex_init(&dev_priv->pps_mutex);
 
+	i915_memcpy_init_early(dev_priv);
+
 	ret = i915_workqueues_init(dev_priv);
 	if (ret < 0)
 		return ret;
@@ -868,7 +834,7 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	intel_init_audio_hooks(dev_priv);
 	i915_gem_load_init(&dev_priv->drm);
 
-	intel_display_crc_init(&dev_priv->drm);
+	intel_display_crc_init(dev_priv);
 
 	intel_device_info_dump(dev_priv);
 
@@ -900,6 +866,7 @@ static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
 static int i915_mmio_setup(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	int mmio_bar;
 	int mmio_size;
 
@@ -916,7 +883,7 @@ static int i915_mmio_setup(struct drm_device *dev)
 		mmio_size = 512 * 1024;
 	else
 		mmio_size = 2 * 1024 * 1024;
-	dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size);
+	dev_priv->regs = pci_iomap(pdev, mmio_bar, mmio_size);
 	if (dev_priv->regs == NULL) {
 		DRM_ERROR("failed to map registers\n");
 
@@ -932,9 +899,10 @@ static int i915_mmio_setup(struct drm_device *dev)
 static void i915_mmio_cleanup(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 
 	intel_teardown_mchbar(dev);
-	pci_iounmap(dev->pdev, dev_priv->regs);
+	pci_iounmap(pdev, dev_priv->regs);
 }
 
 /**
@@ -999,6 +967,9 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
 	i915.enable_ppgtt =
 		intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
 	DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
+
+	i915.semaphores = intel_sanitize_semaphores(dev_priv, i915.semaphores);
+	DRM_DEBUG_DRIVER("use GPU sempahores? %s\n", yesno(i915.semaphores));
 }
 
 /**
@@ -1010,9 +981,8 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
  */
 static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 {
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct drm_device *dev = &dev_priv->drm;
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	uint32_t aperture_size;
 	int ret;
 
 	if (i915_inject_load_failure())
@@ -1022,16 +992,10 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 
 	intel_sanitize_options(dev_priv);
 
-	ret = i915_ggtt_init_hw(dev);
+	ret = i915_ggtt_probe_hw(dev_priv);
 	if (ret)
 		return ret;
 
-	ret = i915_ggtt_enable_hw(dev);
-	if (ret) {
-		DRM_ERROR("failed to enable GGTT\n");
-		goto out_ggtt;
-	}
-
 	/* WARNING: Apparently we must kick fbdev drivers before vgacon,
 	 * otherwise the vga fbdev driver falls over. */
 	ret = i915_kick_out_firmware_fb(dev_priv);
@@ -1046,11 +1010,21 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 		goto out_ggtt;
 	}
 
-	pci_set_master(dev->pdev);
+	ret = i915_ggtt_init_hw(dev_priv);
+	if (ret)
+		return ret;
+
+	ret = i915_ggtt_enable_hw(dev_priv);
+	if (ret) {
+		DRM_ERROR("failed to enable GGTT\n");
+		goto out_ggtt;
+	}
+
+	pci_set_master(pdev);
 
 	/* overlay on gen2 is broken and can't address above 1G */
 	if (IS_GEN2(dev)) {
-		ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
+		ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30));
 		if (ret) {
 			DRM_ERROR("failed to set DMA mask\n");
 
@@ -1058,7 +1032,6 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 		}
 	}
 
-
 	/* 965GM sometimes incorrectly writes to hardware status page (HWS)
 	 * using 32bit addressing, overwriting memory if HWS is located
 	 * above 4GB.
@@ -1068,7 +1041,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 	 * which also needs to be handled carefully.
 	 */
 	if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) {
-		ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
+		ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
 
 		if (ret) {
 			DRM_ERROR("failed to set DMA mask\n");
@@ -1077,19 +1050,6 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 		}
 	}
 
-	aperture_size = ggtt->mappable_end;
-
-	ggtt->mappable =
-		io_mapping_create_wc(ggtt->mappable_base,
-				     aperture_size);
-	if (!ggtt->mappable) {
-		ret = -EIO;
-		goto out_ggtt;
-	}
-
-	ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base,
-					      aperture_size);
-
 	pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
 			   PM_QOS_DEFAULT_VALUE);
 
@@ -1111,14 +1071,14 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 	 * stuck interrupts on some machines.
 	 */
 	if (!IS_I945G(dev) && !IS_I945GM(dev)) {
-		if (pci_enable_msi(dev->pdev) < 0)
+		if (pci_enable_msi(pdev) < 0)
 			DRM_DEBUG_DRIVER("can't enable MSI");
 	}
 
 	return 0;
 
 out_ggtt:
-	i915_ggtt_cleanup_hw(dev);
+	i915_ggtt_cleanup_hw(dev_priv);
 
 	return ret;
 }
@@ -1129,16 +1089,13 @@ out_ggtt:
  */
 static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 
-	if (dev->pdev->msi_enabled)
-		pci_disable_msi(dev->pdev);
+	if (pdev->msi_enabled)
+		pci_disable_msi(pdev);
 
 	pm_qos_remove_request(&dev_priv->pm_qos);
-	arch_phys_wc_del(ggtt->mtrr);
-	io_mapping_free(ggtt->mappable);
-	i915_ggtt_cleanup_hw(dev);
+	i915_ggtt_cleanup_hw(dev_priv);
 }
 
 /**
@@ -1164,7 +1121,7 @@ static void i915_driver_register(struct drm_i915_private *dev_priv)
 	/* Reveal our presence to userspace */
 	if (drm_dev_register(dev, 0) == 0) {
 		i915_debugfs_register(dev_priv);
-		i915_setup_sysfs(dev);
+		i915_setup_sysfs(dev_priv);
 	} else
 		DRM_ERROR("Failed to register driver for userspace access!\n");
 
@@ -1201,7 +1158,7 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 	acpi_video_unregister();
 	intel_opregion_unregister(dev_priv);
 
-	i915_teardown_sysfs(&dev_priv->drm);
+	i915_teardown_sysfs(dev_priv);
 	i915_debugfs_unregister(dev_priv);
 	drm_dev_unregister(&dev_priv->drm);
 
@@ -1281,6 +1238,11 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	intel_runtime_pm_enable(dev_priv);
 
+	/* Everything is in place, we can now relax! */
+	DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n",
+		 driver.name, driver.major, driver.minor, driver.patchlevel,
+		 driver.date, pci_name(pdev), dev_priv->drm.primary->index);
+
 	intel_runtime_pm_put(dev_priv);
 
 	return 0;
@@ -1305,6 +1267,7 @@ out_free_priv:
 void i915_driver_unload(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 
 	intel_fbdev_fini(dev);
 
@@ -1333,8 +1296,8 @@ void i915_driver_unload(struct drm_device *dev)
 	kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
 	dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
 
-	vga_switcheroo_unregister_client(dev->pdev);
-	vga_client_register(dev->pdev, NULL, NULL, NULL);
+	vga_switcheroo_unregister_client(pdev);
+	vga_client_register(pdev, NULL, NULL, NULL);
 
 	intel_csr_ucode_fini(dev_priv);
 
@@ -1343,7 +1306,7 @@ void i915_driver_unload(struct drm_device *dev)
 	i915_destroy_error_state(dev);
 
 	/* Flush any outstanding unpin_work. */
-	flush_workqueue(dev_priv->wq);
+	drain_workqueue(dev_priv->wq);
 
 	intel_guc_fini(dev);
 	i915_gem_fini(dev);
@@ -1431,6 +1394,7 @@ static bool suspend_to_idle(struct drm_i915_private *dev_priv)
 static int i915_drm_suspend(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	pci_power_t opregion_target_state;
 	int error;
 
@@ -1447,19 +1411,17 @@ static int i915_drm_suspend(struct drm_device *dev)
 
 	drm_kms_helper_poll_disable(dev);
 
-	pci_save_state(dev->pdev);
+	pci_save_state(pdev);
 
 	error = i915_gem_suspend(dev);
 	if (error) {
-		dev_err(&dev->pdev->dev,
+		dev_err(&pdev->dev,
 			"GEM idle failed, resume might fail\n");
 		goto out;
 	}
 
 	intel_guc_suspend(dev);
 
-	intel_suspend_gt_powersave(dev_priv);
-
 	intel_display_suspend(dev);
 
 	intel_dp_mst_suspend(dev);
@@ -1495,9 +1457,10 @@ out:
 	return error;
 }
 
-static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
+static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
 {
-	struct drm_i915_private *dev_priv = to_i915(drm_dev);
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	bool fw_csr;
 	int ret;
 
@@ -1531,7 +1494,7 @@ static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
 		goto out;
 	}
 
-	pci_disable_device(drm_dev->pdev);
+	pci_disable_device(pdev);
 	/*
 	 * During hibernation on some platforms the BIOS may try to access
 	 * the device even though it's already in D3 and hang the machine. So
@@ -1545,7 +1508,7 @@ static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
 	 * Acer Aspire 1830T
 	 */
 	if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
-		pci_set_power_state(drm_dev->pdev, PCI_D3hot);
+		pci_set_power_state(pdev, PCI_D3hot);
 
 	dev_priv->suspended_to_idle = suspend_to_idle(dev_priv);
 
@@ -1585,18 +1548,18 @@ static int i915_drm_resume(struct drm_device *dev)
 	int ret;
 
 	disable_rpm_wakeref_asserts(dev_priv);
+	intel_sanitize_gt_powersave(dev_priv);
 
-	ret = i915_ggtt_enable_hw(dev);
+	ret = i915_ggtt_enable_hw(dev_priv);
 	if (ret)
 		DRM_ERROR("failed to re-enable GGTT\n");
 
 	intel_csr_ucode_resume(dev_priv);
 
-	mutex_lock(&dev->struct_mutex);
-	i915_gem_restore_gtt_mappings(dev);
-	mutex_unlock(&dev->struct_mutex);
+	i915_gem_resume(dev);
 
 	i915_restore_state(dev);
+	intel_pps_unlock_regs_wa(dev_priv);
 	intel_opregion_setup(dev_priv);
 
 	intel_init_pch_refclk(dev);
@@ -1615,7 +1578,7 @@ static int i915_drm_resume(struct drm_device *dev)
 	mutex_lock(&dev->struct_mutex);
 	if (i915_gem_init_hw(dev)) {
 		DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
-		atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
+		i915_gem_set_wedged(dev_priv);
 	}
 	mutex_unlock(&dev->struct_mutex);
 
@@ -1652,6 +1615,7 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	intel_opregion_notify_adapter(dev_priv, PCI_D0);
 
+	intel_autoenable_gt_powersave(dev_priv);
 	drm_kms_helper_poll_enable(dev);
 
 	enable_rpm_wakeref_asserts(dev_priv);
@@ -1662,6 +1626,7 @@ static int i915_drm_resume(struct drm_device *dev)
 static int i915_drm_resume_early(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	int ret;
 
 	/*
@@ -1684,7 +1649,7 @@ static int i915_drm_resume_early(struct drm_device *dev)
 	 * the device powered we can also remove the following set power state
 	 * call.
 	 */
-	ret = pci_set_power_state(dev->pdev, PCI_D0);
+	ret = pci_set_power_state(pdev, PCI_D0);
 	if (ret) {
 		DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret);
 		goto out;
@@ -1703,12 +1668,12 @@ static int i915_drm_resume_early(struct drm_device *dev)
 	 * depend on the device enable refcount we can't anyway depend on them
 	 * disabling/enabling the device.
 	 */
-	if (pci_enable_device(dev->pdev)) {
+	if (pci_enable_device(pdev)) {
 		ret = -EIO;
 		goto out;
 	}
 
-	pci_set_master(dev->pdev);
+	pci_set_master(pdev);
 
 	disable_rpm_wakeref_asserts(dev_priv);
 
@@ -1760,8 +1725,10 @@ int i915_resume_switcheroo(struct drm_device *dev)
  * i915_reset - reset chip after a hang
  * @dev: drm device to reset
  *
- * Reset the chip.  Useful if a hang is detected. Returns zero on successful
- * reset or otherwise an error code.
+ * Reset the chip.  Useful if a hang is detected. Marks the device as wedged
+ * on failure.
+ *
+ * Caller must hold the struct_mutex.
  *
  * Procedure is fairly simple:
  *   - reset the chip using the reset reg
@@ -1771,31 +1738,22 @@ int i915_resume_switcheroo(struct drm_device *dev)
  *   - re-init interrupt state
  *   - re-init display
  */
-int i915_reset(struct drm_i915_private *dev_priv)
+void i915_reset(struct drm_i915_private *dev_priv)
 {
 	struct drm_device *dev = &dev_priv->drm;
 	struct i915_gpu_error *error = &dev_priv->gpu_error;
-	unsigned reset_counter;
 	int ret;
 
-	intel_reset_gt_powersave(dev_priv);
+	lockdep_assert_held(&dev->struct_mutex);
 
-	mutex_lock(&dev->struct_mutex);
+	if (!test_and_clear_bit(I915_RESET_IN_PROGRESS, &error->flags))
+		return;
 
 	/* Clear any previous failed attempts at recovery. Time to try again. */
-	atomic_andnot(I915_WEDGED, &error->reset_counter);
-
-	/* Clear the reset-in-progress flag and increment the reset epoch. */
-	reset_counter = atomic_inc_return(&error->reset_counter);
-	if (WARN_ON(__i915_reset_in_progress(reset_counter))) {
-		ret = -EIO;
-		goto error;
-	}
+	__clear_bit(I915_WEDGED, &error->flags);
+	error->reset_count++;
 
 	pr_notice("drm/i915: Resetting chip after gpu hang\n");
-
-	i915_gem_reset(dev);
-
 	ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
 	if (ret) {
 		if (ret != -ENODEV)
@@ -1805,6 +1763,7 @@ int i915_reset(struct drm_i915_private *dev_priv)
 		goto error;
 	}
 
+	i915_gem_reset(dev_priv);
 	intel_overlay_reset(dev_priv);
 
 	/* Ok, now get things going again... */
@@ -1827,44 +1786,34 @@ int i915_reset(struct drm_i915_private *dev_priv)
 		goto error;
 	}
 
-	mutex_unlock(&dev->struct_mutex);
-
-	/*
-	 * rps/rc6 re-init is necessary to restore state lost after the
-	 * reset and the re-install of gt irqs. Skip for ironlake per
-	 * previous concerns that it doesn't respond well to some forms
-	 * of re-init after reset.
-	 */
-	if (INTEL_INFO(dev)->gen > 5)
-		intel_enable_gt_powersave(dev_priv);
-
-	return 0;
+wakeup:
+	wake_up_bit(&error->flags, I915_RESET_IN_PROGRESS);
+	return;
 
 error:
-	atomic_or(I915_WEDGED, &error->reset_counter);
-	mutex_unlock(&dev->struct_mutex);
-	return ret;
+	i915_gem_set_wedged(dev_priv);
+	goto wakeup;
 }
 
-static int i915_pm_suspend(struct device *dev)
+static int i915_pm_suspend(struct device *kdev)
 {
-	struct pci_dev *pdev = to_pci_dev(dev);
-	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	struct pci_dev *pdev = to_pci_dev(kdev);
+	struct drm_device *dev = pci_get_drvdata(pdev);
 
-	if (!drm_dev) {
-		dev_err(dev, "DRM not initialized, aborting suspend.\n");
+	if (!dev) {
+		dev_err(kdev, "DRM not initialized, aborting suspend.\n");
 		return -ENODEV;
 	}
 
-	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
-	return i915_drm_suspend(drm_dev);
+	return i915_drm_suspend(dev);
 }
 
-static int i915_pm_suspend_late(struct device *dev)
+static int i915_pm_suspend_late(struct device *kdev)
 {
-	struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
+	struct drm_device *dev = &kdev_to_i915(kdev)->drm;
 
 	/*
 	 * We have a suspend ordering issue with the snd-hda driver also
@@ -1875,57 +1824,67 @@ static int i915_pm_suspend_late(struct device *dev)
 	 * FIXME: This should be solved with a special hdmi sink device or
 	 * similar so that power domains can be employed.
 	 */
-	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
-	return i915_drm_suspend_late(drm_dev, false);
+	return i915_drm_suspend_late(dev, false);
 }
 
-static int i915_pm_poweroff_late(struct device *dev)
+static int i915_pm_poweroff_late(struct device *kdev)
 {
-	struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
+	struct drm_device *dev = &kdev_to_i915(kdev)->drm;
 
-	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
-	return i915_drm_suspend_late(drm_dev, true);
+	return i915_drm_suspend_late(dev, true);
 }
 
-static int i915_pm_resume_early(struct device *dev)
+static int i915_pm_resume_early(struct device *kdev)
 {
-	struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
+	struct drm_device *dev = &kdev_to_i915(kdev)->drm;
 
-	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
-	return i915_drm_resume_early(drm_dev);
+	return i915_drm_resume_early(dev);
 }
 
-static int i915_pm_resume(struct device *dev)
+static int i915_pm_resume(struct device *kdev)
 {
-	struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
+	struct drm_device *dev = &kdev_to_i915(kdev)->drm;
 
-	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
-	return i915_drm_resume(drm_dev);
+	return i915_drm_resume(dev);
 }
 
 /* freeze: before creating the hibernation_image */
-static int i915_pm_freeze(struct device *dev)
+static int i915_pm_freeze(struct device *kdev)
 {
-	return i915_pm_suspend(dev);
+	int ret;
+
+	ret = i915_pm_suspend(kdev);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_freeze(kdev_to_i915(kdev));
+	if (ret)
+		return ret;
+
+	return 0;
 }
 
-static int i915_pm_freeze_late(struct device *dev)
+static int i915_pm_freeze_late(struct device *kdev)
 {
 	int ret;
 
-	ret = i915_pm_suspend_late(dev);
+	ret = i915_pm_suspend_late(kdev);
 	if (ret)
 		return ret;
 
-	ret = i915_gem_freeze_late(dev_to_i915(dev));
+	ret = i915_gem_freeze_late(kdev_to_i915(kdev));
 	if (ret)
 		return ret;
 
@@ -1933,25 +1892,25 @@ static int i915_pm_freeze_late(struct device *dev)
 }
 
 /* thaw: called after creating the hibernation image, but before turning off. */
-static int i915_pm_thaw_early(struct device *dev)
+static int i915_pm_thaw_early(struct device *kdev)
 {
-	return i915_pm_resume_early(dev);
+	return i915_pm_resume_early(kdev);
 }
 
-static int i915_pm_thaw(struct device *dev)
+static int i915_pm_thaw(struct device *kdev)
 {
-	return i915_pm_resume(dev);
+	return i915_pm_resume(kdev);
 }
 
 /* restore: called after loading the hibernation image. */
-static int i915_pm_restore_early(struct device *dev)
+static int i915_pm_restore_early(struct device *kdev)
 {
-	return i915_pm_resume_early(dev);
+	return i915_pm_resume_early(kdev);
 }
 
-static int i915_pm_restore(struct device *dev)
+static int i915_pm_restore(struct device *kdev)
 {
-	return i915_pm_resume(dev);
+	return i915_pm_resume(kdev);
 }
 
 /*
@@ -2313,9 +2272,9 @@ static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
 	return ret;
 }
 
-static int intel_runtime_suspend(struct device *device)
+static int intel_runtime_suspend(struct device *kdev)
 {
-	struct pci_dev *pdev = to_pci_dev(device);
+	struct pci_dev *pdev = to_pci_dev(kdev);
 	struct drm_device *dev = pci_get_drvdata(pdev);
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	int ret;
@@ -2341,7 +2300,7 @@ static int intel_runtime_suspend(struct device *device)
 		 * Bump the expiration timestamp, otherwise the suspend won't
 		 * be rescheduled.
 		 */
-		pm_runtime_mark_last_busy(device);
+		pm_runtime_mark_last_busy(kdev);
 
 		return -EAGAIN;
 	}
@@ -2420,9 +2379,9 @@ static int intel_runtime_suspend(struct device *device)
 	return 0;
 }
 
-static int intel_runtime_resume(struct device *device)
+static int intel_runtime_resume(struct device *kdev)
 {
-	struct pci_dev *pdev = to_pci_dev(device);
+	struct pci_dev *pdev = to_pci_dev(kdev);
 	struct drm_device *dev = pci_get_drvdata(pdev);
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	int ret = 0;
@@ -2462,7 +2421,6 @@ static int intel_runtime_resume(struct device *device)
 	 * we can do is to hope that things will still work (and disable RPM).
 	 */
 	i915_gem_init_swizzling(dev);
-	gen6_update_ring_freq(dev_priv);
 
 	intel_runtime_pm_enable_interrupts(dev_priv);
 
@@ -2618,6 +2576,7 @@ static struct drm_driver driver = {
 	.postclose = i915_driver_postclose,
 	.set_busid = drm_pci_set_busid,
 
+	.gem_close_object = i915_gem_close_object,
 	.gem_free_object = i915_gem_free_object,
 	.gem_vm_ops = &i915_gem_vm_ops,
 
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 21f939074abc..8b9ee4e390c0 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -61,6 +61,7 @@
 #include "i915_gem.h"
 #include "i915_gem_gtt.h"
 #include "i915_gem_render_state.h"
+#include "i915_gem_request.h"
 
 #include "intel_gvt.h"
 
@@ -69,7 +70,7 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20160711"
+#define DRIVER_DATE		"20160919"
 
 #undef WARN_ON
 /* Many gcc seem to no see through this and fall over :( */
@@ -401,7 +402,7 @@ struct drm_i915_file_private {
 		unsigned boosts;
 	} rps;
 
-	unsigned int bsd_ring;
+	unsigned int bsd_engine;
 };
 
 /* Used by dp and fdi links */
@@ -431,8 +432,6 @@ void intel_link_compute_m_n(int bpp, int nlanes,
 #define DRIVER_MINOR		6
 #define DRIVER_PATCHLEVEL	0
 
-#define WATCH_LISTS	0
-
 struct opregion_header;
 struct opregion_acpi;
 struct opregion_swsci;
@@ -456,15 +455,21 @@ struct intel_opregion {
 struct intel_overlay;
 struct intel_overlay_error_state;
 
-#define I915_FENCE_REG_NONE -1
-#define I915_MAX_NUM_FENCES 32
-/* 32 fences + sign bit for FENCE_REG_NONE */
-#define I915_MAX_NUM_FENCE_BITS 6
-
 struct drm_i915_fence_reg {
-	struct list_head lru_list;
-	struct drm_i915_gem_object *obj;
+	struct list_head link;
+	struct drm_i915_private *i915;
+	struct i915_vma *vma;
 	int pin_count;
+	int id;
+	/**
+	 * Whether the tiling parameters for the currently
+	 * associated fence register have changed. Note that
+	 * for the purposes of tracking tiling changes we also
+	 * treat the unfenced register, the register slot that
+	 * the object occupies whilst it executes a fenced
+	 * command (such as BLT on gen2/3), as a "fence".
+	 */
+	bool dirty;
 };
 
 struct sdvo_device_mapping {
@@ -476,130 +481,6 @@ struct sdvo_device_mapping {
 	u8 ddc_pin;
 };
 
-struct intel_display_error_state;
-
-struct drm_i915_error_state {
-	struct kref ref;
-	struct timeval time;
-
-	char error_msg[128];
-	bool simulated;
-	int iommu;
-	u32 reset_count;
-	u32 suspend_count;
-
-	/* Generic register state */
-	u32 eir;
-	u32 pgtbl_er;
-	u32 ier;
-	u32 gtier[4];
-	u32 ccid;
-	u32 derrmr;
-	u32 forcewake;
-	u32 error; /* gen6+ */
-	u32 err_int; /* gen7 */
-	u32 fault_data0; /* gen8, gen9 */
-	u32 fault_data1; /* gen8, gen9 */
-	u32 done_reg;
-	u32 gac_eco;
-	u32 gam_ecochk;
-	u32 gab_ctl;
-	u32 gfx_mode;
-	u32 extra_instdone[I915_NUM_INSTDONE_REG];
-	u64 fence[I915_MAX_NUM_FENCES];
-	struct intel_overlay_error_state *overlay;
-	struct intel_display_error_state *display;
-	struct drm_i915_error_object *semaphore_obj;
-
-	struct drm_i915_error_ring {
-		bool valid;
-		/* Software tracked state */
-		bool waiting;
-		int num_waiters;
-		int hangcheck_score;
-		enum intel_ring_hangcheck_action hangcheck_action;
-		int num_requests;
-
-		/* our own tracking of ring head and tail */
-		u32 cpu_ring_head;
-		u32 cpu_ring_tail;
-
-		u32 last_seqno;
-		u32 semaphore_seqno[I915_NUM_ENGINES - 1];
-
-		/* Register state */
-		u32 start;
-		u32 tail;
-		u32 head;
-		u32 ctl;
-		u32 hws;
-		u32 ipeir;
-		u32 ipehr;
-		u32 instdone;
-		u32 bbstate;
-		u32 instpm;
-		u32 instps;
-		u32 seqno;
-		u64 bbaddr;
-		u64 acthd;
-		u32 fault_reg;
-		u64 faddr;
-		u32 rc_psmi; /* sleep state */
-		u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
-
-		struct drm_i915_error_object {
-			int page_count;
-			u64 gtt_offset;
-			u32 *pages[0];
-		} *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
-
-		struct drm_i915_error_object *wa_ctx;
-
-		struct drm_i915_error_request {
-			long jiffies;
-			u32 seqno;
-			u32 tail;
-		} *requests;
-
-		struct drm_i915_error_waiter {
-			char comm[TASK_COMM_LEN];
-			pid_t pid;
-			u32 seqno;
-		} *waiters;
-
-		struct {
-			u32 gfx_mode;
-			union {
-				u64 pdp[4];
-				u32 pp_dir_base;
-			};
-		} vm_info;
-
-		pid_t pid;
-		char comm[TASK_COMM_LEN];
-	} ring[I915_NUM_ENGINES];
-
-	struct drm_i915_error_buffer {
-		u32 size;
-		u32 name;
-		u32 rseqno[I915_NUM_ENGINES], wseqno;
-		u64 gtt_offset;
-		u32 read_domains;
-		u32 write_domain;
-		s32 fence_reg:I915_MAX_NUM_FENCE_BITS;
-		s32 pinned:2;
-		u32 tiling:2;
-		u32 dirty:1;
-		u32 purgeable:1;
-		u32 userptr:1;
-		s32 ring:4;
-		u32 cache_level:3;
-	} **active_bo, **pinned_bo;
-
-	u32 *active_bo_count, *pinned_bo_count;
-	u32 vm_count;
-};
-
 struct intel_connector;
 struct intel_encoder;
 struct intel_crtc_state;
@@ -629,8 +510,12 @@ struct drm_i915_display_funcs {
 					 struct intel_initial_plane_config *);
 	int (*crtc_compute_clock)(struct intel_crtc *crtc,
 				  struct intel_crtc_state *crtc_state);
-	void (*crtc_enable)(struct drm_crtc *crtc);
-	void (*crtc_disable)(struct drm_crtc *crtc);
+	void (*crtc_enable)(struct intel_crtc_state *pipe_config,
+			    struct drm_atomic_state *old_state);
+	void (*crtc_disable)(struct intel_crtc_state *old_crtc_state,
+			     struct drm_atomic_state *old_state);
+	void (*update_crtcs)(struct drm_atomic_state *state,
+			     unsigned int *crtc_vblank_mask);
 	void (*audio_codec_enable)(struct drm_connector *connector,
 				   struct intel_encoder *encoder,
 				   const struct drm_display_mode *adjusted_mode);
@@ -694,8 +579,6 @@ struct intel_uncore_funcs {
 				uint16_t val, bool trace);
 	void (*mmio_writel)(struct drm_i915_private *dev_priv, i915_reg_t r,
 				uint32_t val, bool trace);
-	void (*mmio_writeq)(struct drm_i915_private *dev_priv, i915_reg_t r,
-				uint64_t val, bool trace);
 };
 
 struct intel_uncore {
@@ -756,7 +639,7 @@ struct intel_csr {
 	func(is_i915g) sep \
 	func(is_i945gm) sep \
 	func(is_g33) sep \
-	func(need_gfx_hws) sep \
+	func(hws_needs_physical) sep \
 	func(is_g4x) sep \
 	func(is_pineview) sep \
 	func(is_broadwater) sep \
@@ -771,6 +654,19 @@ struct intel_csr {
 	func(is_kabylake) sep \
 	func(is_preliminary) sep \
 	func(has_fbc) sep \
+	func(has_psr) sep \
+	func(has_runtime_pm) sep \
+	func(has_csr) sep \
+	func(has_resource_streamer) sep \
+	func(has_rc6) sep \
+	func(has_rc6p) sep \
+	func(has_dp_mst) sep \
+	func(has_gmbus_irq) sep \
+	func(has_hw_contexts) sep \
+	func(has_logical_ring_contexts) sep \
+	func(has_l3_dpf) sep \
+	func(has_gmch_display) sep \
+	func(has_guc) sep \
 	func(has_pipe_cxsr) sep \
 	func(has_hotplug) sep \
 	func(cursor_needs_physical) sep \
@@ -786,6 +682,24 @@ struct intel_csr {
 #define DEFINE_FLAG(name) u8 name:1
 #define SEP_SEMICOLON ;
 
+struct sseu_dev_info {
+	u8 slice_mask;
+	u8 subslice_mask;
+	u8 eu_total;
+	u8 eu_per_subslice;
+	u8 min_eu_in_pool;
+	/* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */
+	u8 subslice_7eu[3];
+	u8 has_slice_pg:1;
+	u8 has_subslice_pg:1;
+	u8 has_eu_pg:1;
+};
+
+static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu)
+{
+	return hweight8(sseu->slice_mask) * hweight8(sseu->subslice_mask);
+}
+
 struct intel_device_info {
 	u32 display_mmio_offset;
 	u16 device_id;
@@ -794,7 +708,9 @@ struct intel_device_info {
 	u8 gen;
 	u16 gen_mask;
 	u8 ring_mask; /* Rings supported by the HW */
+	u8 num_rings;
 	DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG, SEP_SEMICOLON);
+	u16 ddb_size; /* in blocks */
 	/* Register offsets for the various display pipes and transcoders */
 	int pipe_offsets[I915_MAX_TRANSCODERS];
 	int trans_offsets[I915_MAX_TRANSCODERS];
@@ -802,17 +718,7 @@ struct intel_device_info {
 	int cursor_offsets[I915_MAX_PIPES];
 
 	/* Slice/subslice/EU info */
-	u8 slice_total;
-	u8 subslice_total;
-	u8 subslice_per_slice;
-	u8 eu_total;
-	u8 eu_per_subslice;
-	u8 min_eu_in_pool;
-	/* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */
-	u8 subslice_7eu[3];
-	u8 has_slice_pg:1;
-	u8 has_subslice_pg:1;
-	u8 has_eu_pg:1;
+	struct sseu_dev_info sseu;
 
 	struct color_luts {
 		u16 degamma_lut_size;
@@ -823,6 +729,134 @@ struct intel_device_info {
 #undef DEFINE_FLAG
 #undef SEP_SEMICOLON
 
+struct intel_display_error_state;
+
+struct drm_i915_error_state {
+	struct kref ref;
+	struct timeval time;
+
+	char error_msg[128];
+	bool simulated;
+	int iommu;
+	u32 reset_count;
+	u32 suspend_count;
+	struct intel_device_info device_info;
+
+	/* Generic register state */
+	u32 eir;
+	u32 pgtbl_er;
+	u32 ier;
+	u32 gtier[4];
+	u32 ccid;
+	u32 derrmr;
+	u32 forcewake;
+	u32 error; /* gen6+ */
+	u32 err_int; /* gen7 */
+	u32 fault_data0; /* gen8, gen9 */
+	u32 fault_data1; /* gen8, gen9 */
+	u32 done_reg;
+	u32 gac_eco;
+	u32 gam_ecochk;
+	u32 gab_ctl;
+	u32 gfx_mode;
+	u32 extra_instdone[I915_NUM_INSTDONE_REG];
+	u64 fence[I915_MAX_NUM_FENCES];
+	struct intel_overlay_error_state *overlay;
+	struct intel_display_error_state *display;
+	struct drm_i915_error_object *semaphore;
+
+	struct drm_i915_error_engine {
+		int engine_id;
+		/* Software tracked state */
+		bool waiting;
+		int num_waiters;
+		int hangcheck_score;
+		enum intel_engine_hangcheck_action hangcheck_action;
+		struct i915_address_space *vm;
+		int num_requests;
+
+		/* our own tracking of ring head and tail */
+		u32 cpu_ring_head;
+		u32 cpu_ring_tail;
+
+		u32 last_seqno;
+		u32 semaphore_seqno[I915_NUM_ENGINES - 1];
+
+		/* Register state */
+		u32 start;
+		u32 tail;
+		u32 head;
+		u32 ctl;
+		u32 mode;
+		u32 hws;
+		u32 ipeir;
+		u32 ipehr;
+		u32 instdone;
+		u32 bbstate;
+		u32 instpm;
+		u32 instps;
+		u32 seqno;
+		u64 bbaddr;
+		u64 acthd;
+		u32 fault_reg;
+		u64 faddr;
+		u32 rc_psmi; /* sleep state */
+		u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
+
+		struct drm_i915_error_object {
+			int page_count;
+			u64 gtt_offset;
+			u64 gtt_size;
+			u32 *pages[0];
+		} *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
+
+		struct drm_i915_error_object *wa_ctx;
+
+		struct drm_i915_error_request {
+			long jiffies;
+			pid_t pid;
+			u32 seqno;
+			u32 head;
+			u32 tail;
+		} *requests;
+
+		struct drm_i915_error_waiter {
+			char comm[TASK_COMM_LEN];
+			pid_t pid;
+			u32 seqno;
+		} *waiters;
+
+		struct {
+			u32 gfx_mode;
+			union {
+				u64 pdp[4];
+				u32 pp_dir_base;
+			};
+		} vm_info;
+
+		pid_t pid;
+		char comm[TASK_COMM_LEN];
+	} engine[I915_NUM_ENGINES];
+
+	struct drm_i915_error_buffer {
+		u32 size;
+		u32 name;
+		u32 rseqno[I915_NUM_ENGINES], wseqno;
+		u64 gtt_offset;
+		u32 read_domains;
+		u32 write_domain;
+		s32 fence_reg:I915_MAX_NUM_FENCE_BITS;
+		u32 tiling:2;
+		u32 dirty:1;
+		u32 purgeable:1;
+		u32 userptr:1;
+		s32 engine:4;
+		u32 cache_level:3;
+	} *active_bo[I915_NUM_ENGINES], *pinned_bo;
+	u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count;
+	struct i915_address_space *active_vm[I915_NUM_ENGINES];
+};
+
 enum i915_cache_level {
 	I915_CACHE_NONE = 0,
 	I915_CACHE_LLC, /* also used for snoopable memory on non-LLC */
@@ -879,22 +913,23 @@ struct i915_gem_context {
 	struct drm_i915_private *i915;
 	struct drm_i915_file_private *file_priv;
 	struct i915_hw_ppgtt *ppgtt;
+	struct pid *pid;
 
 	struct i915_ctx_hang_stats hang_stats;
 
-	/* Unique identifier for this context, used by the hw for tracking */
 	unsigned long flags;
 #define CONTEXT_NO_ZEROMAP		BIT(0)
 #define CONTEXT_NO_ERROR_CAPTURE	BIT(1)
-	unsigned hw_id;
+
+	/* Unique identifier for this context, used by the hw for tracking */
+	unsigned int hw_id;
 	u32 user_handle;
 
 	u32 ggtt_alignment;
 
 	struct intel_context {
-		struct drm_i915_gem_object *state;
-		struct intel_ringbuffer *ringbuf;
-		struct i915_vma *lrc_vma;
+		struct i915_vma *state;
+		struct intel_ring *ring;
 		uint32_t *lrc_reg_state;
 		u64 lrc_desc;
 		int pin_count;
@@ -908,6 +943,7 @@ struct i915_gem_context {
 	struct list_head link;
 
 	u8 remap_slice;
+	bool closed:1;
 };
 
 enum fb_op_origin {
@@ -1061,13 +1097,6 @@ struct intel_gmbus {
 
 struct i915_suspend_saved_registers {
 	u32 saveDSPARB;
-	u32 saveLVDS;
-	u32 savePP_ON_DELAYS;
-	u32 savePP_OFF_DELAYS;
-	u32 savePP_ON;
-	u32 savePP_OFF;
-	u32 savePP_CONTROL;
-	u32 savePP_DIVISOR;
 	u32 saveFBC_CONTROL;
 	u32 saveCACHE_MODE_0;
 	u32 saveMI_ARB_STATE;
@@ -1156,6 +1185,7 @@ struct intel_gen6_power_mgmt {
 	bool interrupts_enabled;
 	u32 pm_iir;
 
+	/* PM interrupt bits that should never be masked */
 	u32 pm_intr_keep;
 
 	/* Frequencies are stored in potentially platform dependent multiples.
@@ -1173,6 +1203,7 @@ struct intel_gen6_power_mgmt {
 	u8 max_freq_softlimit;	/* Max frequency permitted by the driver */
 	u8 max_freq;		/* Maximum frequency, RP0 if not overclocking */
 	u8 min_freq;		/* AKA RPn. Minimum frequency */
+	u8 boost_freq;		/* Frequency to request when wait boosting */
 	u8 idle_freq;		/* Frequency to request when we are idle */
 	u8 efficient_freq;	/* AKA RPe. Pre-determined balanced frequency */
 	u8 rp1_freq;		/* "less than" RP0 power/freqency */
@@ -1190,11 +1221,9 @@ struct intel_gen6_power_mgmt {
 	bool client_boost;
 
 	bool enabled;
-	struct delayed_work delayed_resume_work;
+	struct delayed_work autoenable_work;
 	unsigned boosts;
 
-	struct intel_rps_client semaphores, mmioflips;
-
 	/* manual wa residency calculations */
 	struct intel_rps_ei up_ei, down_ei;
 
@@ -1319,7 +1348,6 @@ struct i915_gem_mm {
 	struct notifier_block oom_notifier;
 	struct notifier_block vmap_notifier;
 	struct shrinker shrinker;
-	bool shrinker_no_lock_stealing;
 
 	/** LRU list of objects with fence regs on them. */
 	struct list_head fence_list;
@@ -1331,7 +1359,7 @@ struct i915_gem_mm {
 	bool interruptible;
 
 	/* the indicator for dispatch video commands on two BSD rings */
-	unsigned int bsd_ring_dispatch_index;
+	atomic_t bsd_engine_dispatch_index;
 
 	/** Bit 6 swizzling required for X tiling */
 	uint32_t bit_6_swizzle_x;
@@ -1379,9 +1407,10 @@ struct i915_gpu_error {
 	 * State variable controlling the reset flow and count
 	 *
 	 * This is a counter which gets incremented when reset is triggered,
-	 * and again when reset has been handled. So odd values (lowest bit set)
-	 * means that reset is in progress and even values that
-	 * (reset_counter >> 1):th reset was successfully completed.
+	 *
+	 * Before the reset commences, the I915_RESET_IN_PROGRESS bit is set
+	 * meaning that any waiters holding onto the struct_mutex should
+	 * relinquish the lock immediately in order for the reset to start.
 	 *
 	 * If reset is not completed succesfully, the I915_WEDGE bit is
 	 * set meaning that hardware is terminally sour and there is no
@@ -1396,10 +1425,11 @@ struct i915_gpu_error {
 	 * naturally enforces the correct ordering between the bail-out of the
 	 * waiter and the gpu reset work code.
 	 */
-	atomic_t reset_counter;
+	unsigned long reset_count;
 
-#define I915_RESET_IN_PROGRESS_FLAG	1
-#define I915_WEDGED			(1 << 31)
+	unsigned long flags;
+#define I915_RESET_IN_PROGRESS	0
+#define I915_WEDGED		(BITS_PER_LONG - 1)
 
 	/**
 	 * Waitqueue to signal when a hang is detected. Used to for waiters
@@ -1670,7 +1700,7 @@ struct intel_pipe_crc {
 };
 
 struct i915_frontbuffer_tracking {
-	struct mutex lock;
+	spinlock_t lock;
 
 	/*
 	 * Tracking bits for delayed frontbuffer flushing du to gpu activity or
@@ -1705,18 +1735,6 @@ struct i915_virtual_gpu {
 	bool active;
 };
 
-struct i915_execbuffer_params {
-	struct drm_device               *dev;
-	struct drm_file                 *file;
-	uint32_t                        dispatch_flags;
-	uint32_t                        args_batch_start_offset;
-	uint64_t                        batch_obj_vm_offset;
-	struct intel_engine_cs *engine;
-	struct drm_i915_gem_object      *batch_obj;
-	struct i915_gem_context            *ctx;
-	struct drm_i915_gem_request     *request;
-};
-
 /* used in computing the new watermarks state */
 struct intel_wm_config {
 	unsigned int num_pipes_active;
@@ -1763,13 +1781,15 @@ struct drm_i915_private {
 
 	uint32_t psr_mmio_base;
 
+	uint32_t pps_mmio_base;
+
 	wait_queue_head_t gmbus_wait_queue;
 
 	struct pci_dev *bridge_dev;
 	struct i915_gem_context *kernel_context;
 	struct intel_engine_cs engine[I915_NUM_ENGINES];
-	struct drm_i915_gem_object *semaphore_obj;
-	uint32_t last_seqno, next_seqno;
+	struct i915_vma *semaphore;
+	u32 next_seqno;
 
 	struct drm_dma_handle *status_page_dmah;
 	struct resource mch_res;
@@ -1854,6 +1874,7 @@ struct drm_i915_private {
 	enum modeset_restore modeset_restore;
 	struct mutex modeset_restore_lock;
 	struct drm_atomic_state *modeset_restore_state;
+	struct drm_modeset_acquire_ctx reset_ctx;
 
 	struct list_head vm_list; /* Global list of all address spaces */
 	struct i915_ggtt ggtt; /* VM representing the global address space */
@@ -1962,6 +1983,13 @@ struct drm_i915_private {
 	struct i915_suspend_saved_registers regfile;
 	struct vlv_s0ix_state vlv_s0ix_state;
 
+	enum {
+		I915_SAGV_UNKNOWN = 0,
+		I915_SAGV_DISABLED,
+		I915_SAGV_ENABLED,
+		I915_SAGV_NOT_CONTROLLED
+	} sagv_status;
+
 	struct {
 		/*
 		 * Raw watermark latency values:
@@ -2016,12 +2044,8 @@ struct drm_i915_private {
 
 	/* Abstract the submission mechanism (legacy ringbuffer or execlists) away */
 	struct {
-		int (*execbuf_submit)(struct i915_execbuffer_params *params,
-				      struct drm_i915_gem_execbuffer2 *args,
-				      struct list_head *vmas);
-		int (*init_engines)(struct drm_device *dev);
+		void (*resume)(struct drm_i915_private *);
 		void (*cleanup_engine)(struct intel_engine_cs *engine);
-		void (*stop_engine)(struct intel_engine_cs *engine);
 
 		/**
 		 * Is the GPU currently considered idle, or busy executing
@@ -2068,9 +2092,9 @@ static inline struct drm_i915_private *to_i915(const struct drm_device *dev)
 	return container_of(dev, struct drm_i915_private, drm);
 }
 
-static inline struct drm_i915_private *dev_to_i915(struct device *dev)
+static inline struct drm_i915_private *kdev_to_i915(struct device *kdev)
 {
-	return to_i915(dev_get_drvdata(dev));
+	return to_i915(dev_get_drvdata(kdev));
 }
 
 static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
@@ -2093,13 +2117,16 @@ static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
 		for_each_if (((id__) = (engine__)->id, \
 			      intel_engine_initialized(engine__)))
 
+#define __mask_next_bit(mask) ({					\
+	int __idx = ffs(mask) - 1;					\
+	mask &= ~BIT(__idx);						\
+	__idx;								\
+})
+
 /* Iterator over subset of engines selected by mask */
-#define for_each_engine_masked(engine__, dev_priv__, mask__) \
-	for ((engine__) = &(dev_priv__)->engine[0]; \
-	     (engine__) < &(dev_priv__)->engine[I915_NUM_ENGINES]; \
-	     (engine__)++) \
-		for_each_if (((mask__) & intel_engine_flag(engine__)) && \
-			     intel_engine_initialized(engine__))
+#define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \
+	for (tmp__ = mask__ & INTEL_INFO(dev_priv__)->ring_mask;	\
+	     tmp__ ? (engine__ = &(dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; )
 
 enum hdmi_force_audio {
 	HDMI_AUDIO_OFF_DVI = -2,	/* no aux data for HDMI-DVI converter */
@@ -2144,8 +2171,6 @@ struct drm_i915_gem_object_ops {
  */
 #define INTEL_MAX_SPRITE_BITS_PER_PIPE 5
 #define INTEL_FRONTBUFFER_BITS_PER_PIPE 8
-#define INTEL_FRONTBUFFER_BITS \
-	(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES)
 #define INTEL_FRONTBUFFER_PRIMARY(pipe) \
 	(1 << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))
 #define INTEL_FRONTBUFFER_CURSOR(pipe) \
@@ -2169,18 +2194,21 @@ struct drm_i915_gem_object {
 	struct drm_mm_node *stolen;
 	struct list_head global_list;
 
-	struct list_head engine_list[I915_NUM_ENGINES];
 	/** Used in execbuf to temporarily hold a ref */
 	struct list_head obj_exec_link;
 
 	struct list_head batch_pool_link;
 
+	unsigned long flags;
 	/**
 	 * This is set if the object is on the active lists (has pending
 	 * rendering and so a non-zero seqno), and is not set if it i s on
 	 * inactive (ready to be unbound) list.
 	 */
-	unsigned int active:I915_NUM_ENGINES;
+#define I915_BO_ACTIVE_SHIFT 0
+#define I915_BO_ACTIVE_MASK ((1 << I915_NUM_ENGINES) - 1)
+#define __I915_BO_ACTIVE(bo) \
+	((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK)
 
 	/**
 	 * This is set if the object has been written to since last bound
@@ -2189,37 +2217,11 @@ struct drm_i915_gem_object {
 	unsigned int dirty:1;
 
 	/**
-	 * Fence register bits (if any) for this object.  Will be set
-	 * as needed when mapped into the GTT.
-	 * Protected by dev->struct_mutex.
-	 */
-	signed int fence_reg:I915_MAX_NUM_FENCE_BITS;
-
-	/**
 	 * Advice: are the backing pages purgeable?
 	 */
 	unsigned int madv:2;
 
 	/**
-	 * Current tiling mode for the object.
-	 */
-	unsigned int tiling_mode:2;
-	/**
-	 * Whether the tiling parameters for the currently associated fence
-	 * register have changed. Note that for the purposes of tracking
-	 * tiling changes we also treat the unfenced register, the register
-	 * slot that the object occupies whilst it executes a fenced
-	 * command (such as BLT on gen2/3), as a "fence".
-	 */
-	unsigned int fence_dirty:1;
-
-	/**
-	 * Is the object at the current location in the gtt mappable and
-	 * fenceable? Used to avoid costly recalculations.
-	 */
-	unsigned int map_and_fenceable:1;
-
-	/**
 	 * Whether the current gtt mapping needs to be mappable (and isn't just
 	 * mappable by accident). Track pin and fault separate for a more
 	 * accurate mappable working set.
@@ -2234,9 +2236,17 @@ struct drm_i915_gem_object {
 	unsigned int cache_level:3;
 	unsigned int cache_dirty:1;
 
-	unsigned int frontbuffer_bits:INTEL_FRONTBUFFER_BITS;
+	atomic_t frontbuffer_bits;
+	unsigned int frontbuffer_ggtt_origin; /* write once */
 
-	unsigned int has_wc_mmap;
+	/** Current tiling stride for the object, if it's tiled. */
+	unsigned int tiling_and_stride;
+#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */
+#define TILING_MASK (FENCE_MINIMUM_STRIDE-1)
+#define STRIDE_MASK (~TILING_MASK)
+
+	/** Count of VMA actually bound by this object */
+	unsigned int bind_count;
 	unsigned int pin_display;
 
 	struct sg_table *pages;
@@ -2256,14 +2266,9 @@ struct drm_i915_gem_object {
 	 * requests on one ring where the write request is older than the
 	 * read request. This allows for the CPU to read from an active
 	 * buffer by only waiting for the write to complete.
-	 * */
-	struct drm_i915_gem_request *last_read_req[I915_NUM_ENGINES];
-	struct drm_i915_gem_request *last_write_req;
-	/** Breadcrumb of last fenced GPU access to the buffer. */
-	struct drm_i915_gem_request *last_fenced_req;
-
-	/** Current tiling stride for the object, if it's tiled. */
-	uint32_t stride;
+	 */
+	struct i915_gem_active last_read[I915_NUM_ENGINES];
+	struct i915_gem_active last_write;
 
 	/** References from framebuffers, locks out tiling changes. */
 	unsigned long framebuffer_references;
@@ -2271,23 +2276,70 @@ struct drm_i915_gem_object {
 	/** Record of address bit 17 of each page at last unbind. */
 	unsigned long *bit_17;
 
-	union {
-		/** for phy allocated objects */
-		struct drm_dma_handle *phys_handle;
-
-		struct i915_gem_userptr {
-			uintptr_t ptr;
-			unsigned read_only :1;
-			unsigned workers :4;
+	struct i915_gem_userptr {
+		uintptr_t ptr;
+		unsigned read_only :1;
+		unsigned workers :4;
 #define I915_GEM_USERPTR_MAX_WORKERS 15
 
-			struct i915_mm_struct *mm;
-			struct i915_mmu_object *mmu_object;
-			struct work_struct *work;
-		} userptr;
-	};
+		struct i915_mm_struct *mm;
+		struct i915_mmu_object *mmu_object;
+		struct work_struct *work;
+	} userptr;
+
+	/** for phys allocated objects */
+	struct drm_dma_handle *phys_handle;
 };
-#define to_intel_bo(x) container_of(x, struct drm_i915_gem_object, base)
+
+static inline struct drm_i915_gem_object *
+to_intel_bo(struct drm_gem_object *gem)
+{
+	/* Assert that to_intel_bo(NULL) == NULL */
+	BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base));
+
+	return container_of(gem, struct drm_i915_gem_object, base);
+}
+
+static inline struct drm_i915_gem_object *
+i915_gem_object_lookup(struct drm_file *file, u32 handle)
+{
+	return to_intel_bo(drm_gem_object_lookup(file, handle));
+}
+
+__deprecated
+extern struct drm_gem_object *
+drm_gem_object_lookup(struct drm_file *file, u32 handle);
+
+__attribute__((nonnull))
+static inline struct drm_i915_gem_object *
+i915_gem_object_get(struct drm_i915_gem_object *obj)
+{
+	drm_gem_object_reference(&obj->base);
+	return obj;
+}
+
+__deprecated
+extern void drm_gem_object_reference(struct drm_gem_object *);
+
+__attribute__((nonnull))
+static inline void
+i915_gem_object_put(struct drm_i915_gem_object *obj)
+{
+	drm_gem_object_unreference(&obj->base);
+}
+
+__deprecated
+extern void drm_gem_object_unreference(struct drm_gem_object *);
+
+__attribute__((nonnull))
+static inline void
+i915_gem_object_put_unlocked(struct drm_i915_gem_object *obj)
+{
+	drm_gem_object_unreference_unlocked(&obj->base);
+}
+
+__deprecated
+extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
 
 static inline bool
 i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
@@ -2295,6 +2347,67 @@ i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
 	return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
 }
 
+static inline unsigned long
+i915_gem_object_get_active(const struct drm_i915_gem_object *obj)
+{
+	return (obj->flags >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK;
+}
+
+static inline bool
+i915_gem_object_is_active(const struct drm_i915_gem_object *obj)
+{
+	return i915_gem_object_get_active(obj);
+}
+
+static inline void
+i915_gem_object_set_active(struct drm_i915_gem_object *obj, int engine)
+{
+	obj->flags |= BIT(engine + I915_BO_ACTIVE_SHIFT);
+}
+
+static inline void
+i915_gem_object_clear_active(struct drm_i915_gem_object *obj, int engine)
+{
+	obj->flags &= ~BIT(engine + I915_BO_ACTIVE_SHIFT);
+}
+
+static inline bool
+i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,
+				  int engine)
+{
+	return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT);
+}
+
+static inline unsigned int
+i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
+{
+	return obj->tiling_and_stride & TILING_MASK;
+}
+
+static inline bool
+i915_gem_object_is_tiled(struct drm_i915_gem_object *obj)
+{
+	return i915_gem_object_get_tiling(obj) != I915_TILING_NONE;
+}
+
+static inline unsigned int
+i915_gem_object_get_stride(struct drm_i915_gem_object *obj)
+{
+	return obj->tiling_and_stride & STRIDE_MASK;
+}
+
+static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)
+{
+	i915_gem_object_get(vma->obj);
+	return vma;
+}
+
+static inline void i915_vma_put(struct i915_vma *vma)
+{
+	lockdep_assert_held(&vma->vm->dev->struct_mutex);
+	i915_gem_object_put(vma->obj);
+}
+
 /*
  * Optimised SGL iterator for GEM objects
  */
@@ -2365,171 +2478,6 @@ static inline struct scatterlist *__sg_next(struct scatterlist *sg)
 	     (((__iter).curr += PAGE_SIZE) < (__iter).max) ||		\
 	     ((__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0))
 
-/**
- * Request queue structure.
- *
- * The request queue allows us to note sequence numbers that have been emitted
- * and may be associated with active buffers to be retired.
- *
- * By keeping this list, we can avoid having to do questionable sequence
- * number comparisons on buffer last_read|write_seqno. It also allows an
- * emission time to be associated with the request for tracking how far ahead
- * of the GPU the submission is.
- *
- * The requests are reference counted, so upon creation they should have an
- * initial reference taken using kref_init
- */
-struct drm_i915_gem_request {
-	struct kref ref;
-
-	/** On Which ring this request was generated */
-	struct drm_i915_private *i915;
-	struct intel_engine_cs *engine;
-	struct intel_signal_node signaling;
-
-	 /** GEM sequence number associated with the previous request,
-	  * when the HWS breadcrumb is equal to this the GPU is processing
-	  * this request.
-	  */
-	u32 previous_seqno;
-
-	 /** GEM sequence number associated with this request,
-	  * when the HWS breadcrumb is equal or greater than this the GPU
-	  * has finished processing this request.
-	  */
-	u32 seqno;
-
-	/** Position in the ringbuffer of the start of the request */
-	u32 head;
-
-	/**
-	 * Position in the ringbuffer of the start of the postfix.
-	 * This is required to calculate the maximum available ringbuffer
-	 * space without overwriting the postfix.
-	 */
-	 u32 postfix;
-
-	/** Position in the ringbuffer of the end of the whole request */
-	u32 tail;
-
-	/** Preallocate space in the ringbuffer for the emitting the request */
-	u32 reserved_space;
-
-	/**
-	 * Context and ring buffer related to this request
-	 * Contexts are refcounted, so when this request is associated with a
-	 * context, we must increment the context's refcount, to guarantee that
-	 * it persists while any request is linked to it. Requests themselves
-	 * are also refcounted, so the request will only be freed when the last
-	 * reference to it is dismissed, and the code in
-	 * i915_gem_request_free() will then decrement the refcount on the
-	 * context.
-	 */
-	struct i915_gem_context *ctx;
-	struct intel_ringbuffer *ringbuf;
-
-	/**
-	 * Context related to the previous request.
-	 * As the contexts are accessed by the hardware until the switch is
-	 * completed to a new context, the hardware may still be writing
-	 * to the context object after the breadcrumb is visible. We must
-	 * not unpin/unbind/prune that object whilst still active and so
-	 * we keep the previous context pinned until the following (this)
-	 * request is retired.
-	 */
-	struct i915_gem_context *previous_context;
-
-	/** Batch buffer related to this request if any (used for
-	    error state dump only) */
-	struct drm_i915_gem_object *batch_obj;
-
-	/** Time at which this request was emitted, in jiffies. */
-	unsigned long emitted_jiffies;
-
-	/** global list entry for this request */
-	struct list_head list;
-
-	struct drm_i915_file_private *file_priv;
-	/** file_priv list entry for this request */
-	struct list_head client_list;
-
-	/** process identifier submitting this request */
-	struct pid *pid;
-
-	/**
-	 * The ELSP only accepts two elements at a time, so we queue
-	 * context/tail pairs on a given queue (ring->execlist_queue) until the
-	 * hardware is available. The queue serves a double purpose: we also use
-	 * it to keep track of the up to 2 contexts currently in the hardware
-	 * (usually one in execution and the other queued up by the GPU): We
-	 * only remove elements from the head of the queue when the hardware
-	 * informs us that an element has been completed.
-	 *
-	 * All accesses to the queue are mediated by a spinlock
-	 * (ring->execlist_lock).
-	 */
-
-	/** Execlist link in the submission queue.*/
-	struct list_head execlist_link;
-
-	/** Execlists no. of times this request has been sent to the ELSP */
-	int elsp_submitted;
-
-	/** Execlists context hardware id. */
-	unsigned ctx_hw_id;
-};
-
-struct drm_i915_gem_request * __must_check
-i915_gem_request_alloc(struct intel_engine_cs *engine,
-		       struct i915_gem_context *ctx);
-void i915_gem_request_free(struct kref *req_ref);
-int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
-				   struct drm_file *file);
-
-static inline uint32_t
-i915_gem_request_get_seqno(struct drm_i915_gem_request *req)
-{
-	return req ? req->seqno : 0;
-}
-
-static inline struct intel_engine_cs *
-i915_gem_request_get_engine(struct drm_i915_gem_request *req)
-{
-	return req ? req->engine : NULL;
-}
-
-static inline struct drm_i915_gem_request *
-i915_gem_request_reference(struct drm_i915_gem_request *req)
-{
-	if (req)
-		kref_get(&req->ref);
-	return req;
-}
-
-static inline void
-i915_gem_request_unreference(struct drm_i915_gem_request *req)
-{
-	kref_put(&req->ref, i915_gem_request_free);
-}
-
-static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
-					   struct drm_i915_gem_request *src)
-{
-	if (src)
-		i915_gem_request_reference(src);
-
-	if (*pdst)
-		i915_gem_request_unreference(*pdst);
-
-	*pdst = src;
-}
-
-/*
- * XXX: i915_gem_request_completed should be here but currently needs the
- * definition of i915_seqno_passed() which is below. It will be moved in
- * a later patch when the call to i915_seqno_passed() is obsoleted...
- */
-
 /*
  * A command that requires special handling by the command parser.
  */
@@ -2617,8 +2565,9 @@ struct drm_i915_cmd_descriptor {
 /*
  * A table of commands requiring special handling by the command parser.
  *
- * Each ring has an array of tables. Each table consists of an array of command
- * descriptors, which must be sorted with command opcodes in ascending order.
+ * Each engine has an array of tables. Each table consists of an array of
+ * command descriptors, which must be sorted with command opcodes in
+ * ascending order.
  */
 struct drm_i915_cmd_table {
 	const struct drm_i915_cmd_descriptor *table;
@@ -2636,7 +2585,7 @@ struct drm_i915_cmd_table {
 		BUILD_BUG(); \
 	__p; \
 })
-#define INTEL_INFO(p) 	(&__I915__(p)->info)
+#define INTEL_INFO(p)	(&__I915__(p)->info)
 #define INTEL_GEN(p)	(INTEL_INFO(p)->gen)
 #define INTEL_DEVID(p)	(INTEL_INFO(p)->device_id)
 
@@ -2803,10 +2752,10 @@ struct drm_i915_cmd_table {
 #define HAS_EDRAM(dev)		(!!(__I915__(dev)->edram_cap & EDRAM_ENABLED))
 #define HAS_WT(dev)		((IS_HASWELL(dev) || IS_BROADWELL(dev)) && \
 				 HAS_EDRAM(dev))
-#define I915_NEED_GFX_HWS(dev)	(INTEL_INFO(dev)->need_gfx_hws)
+#define HWS_NEEDS_PHYSICAL(dev)	(INTEL_INFO(dev)->hws_needs_physical)
 
-#define HAS_HW_CONTEXTS(dev)	(INTEL_INFO(dev)->gen >= 6)
-#define HAS_LOGICAL_RING_CONTEXTS(dev)	(INTEL_INFO(dev)->gen >= 8)
+#define HAS_HW_CONTEXTS(dev)	(INTEL_INFO(dev)->has_hw_contexts)
+#define HAS_LOGICAL_RING_CONTEXTS(dev)	(INTEL_INFO(dev)->has_logical_ring_contexts)
 #define USES_PPGTT(dev)		(i915.enable_ppgtt)
 #define USES_FULL_PPGTT(dev)	(i915.enable_ppgtt >= 2)
 #define USES_FULL_48BIT_PPGTT(dev)	(i915.enable_ppgtt == 3)
@@ -2830,7 +2779,7 @@ struct drm_i915_cmd_table {
  * interrupt source and so prevents the other device from working properly.
  */
 #define HAS_AUX_IRQ(dev) (INTEL_INFO(dev)->gen >= 5)
-#define HAS_GMBUS_IRQ(dev) (INTEL_INFO(dev)->gen >= 5)
+#define HAS_GMBUS_IRQ(dev) (INTEL_INFO(dev)->has_gmbus_irq)
 
 /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
  * rows, which changed the alignment requirements and fence programming.
@@ -2846,38 +2795,27 @@ struct drm_i915_cmd_table {
 
 #define HAS_IPS(dev)		(IS_HSW_ULT(dev) || IS_BROADWELL(dev))
 
-#define HAS_DP_MST(dev)		(IS_HASWELL(dev) || IS_BROADWELL(dev) || \
-				 INTEL_INFO(dev)->gen >= 9)
+#define HAS_DP_MST(dev)	(INTEL_INFO(dev)->has_dp_mst)
 
 #define HAS_DDI(dev)		(INTEL_INFO(dev)->has_ddi)
 #define HAS_FPGA_DBG_UNCLAIMED(dev)	(INTEL_INFO(dev)->has_fpga_dbg)
-#define HAS_PSR(dev)		(IS_HASWELL(dev) || IS_BROADWELL(dev) || \
-				 IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev) || \
-				 IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
-#define HAS_RUNTIME_PM(dev)	(IS_GEN6(dev) || IS_HASWELL(dev) || \
-				 IS_BROADWELL(dev) || IS_VALLEYVIEW(dev) || \
-				 IS_CHERRYVIEW(dev) || IS_SKYLAKE(dev) || \
-				 IS_KABYLAKE(dev) || IS_BROXTON(dev))
-#define HAS_RC6(dev)		(INTEL_INFO(dev)->gen >= 6)
-#define HAS_RC6p(dev)		(IS_GEN6(dev) || IS_IVYBRIDGE(dev))
-
-#define HAS_CSR(dev)	(IS_GEN9(dev))
+#define HAS_PSR(dev)		(INTEL_INFO(dev)->has_psr)
+#define HAS_RUNTIME_PM(dev)	(INTEL_INFO(dev)->has_runtime_pm)
+#define HAS_RC6(dev)		(INTEL_INFO(dev)->has_rc6)
+#define HAS_RC6p(dev)		(INTEL_INFO(dev)->has_rc6p)
+
+#define HAS_CSR(dev)	(INTEL_INFO(dev)->has_csr)
 
 /*
  * For now, anything with a GuC requires uCode loading, and then supports
  * command submission once loaded. But these are logically independent
  * properties, so we have separate macros to test them.
  */
-#define HAS_GUC(dev)		(IS_GEN9(dev))
+#define HAS_GUC(dev)		(INTEL_INFO(dev)->has_guc)
 #define HAS_GUC_UCODE(dev)	(HAS_GUC(dev))
 #define HAS_GUC_SCHED(dev)	(HAS_GUC(dev))
 
-#define HAS_RESOURCE_STREAMER(dev) (IS_HASWELL(dev) || \
-				    INTEL_INFO(dev)->gen >= 8)
-
-#define HAS_CORE_RING_FREQ(dev)	(INTEL_INFO(dev)->gen >= 6 && \
-				 !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && \
-				 !IS_BROXTON(dev))
+#define HAS_RESOURCE_STREAMER(dev) (INTEL_INFO(dev)->has_resource_streamer)
 
 #define HAS_POOLED_EU(dev)	(INTEL_INFO(dev)->has_pooled_eu)
 
@@ -2905,11 +2843,10 @@ struct drm_i915_cmd_table {
 #define HAS_PCH_NOP(dev) (INTEL_PCH_TYPE(dev) == PCH_NOP)
 #define HAS_PCH_SPLIT(dev) (INTEL_PCH_TYPE(dev) != PCH_NONE)
 
-#define HAS_GMCH_DISPLAY(dev) (INTEL_INFO(dev)->gen < 5 || \
-			       IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+#define HAS_GMCH_DISPLAY(dev) (INTEL_INFO(dev)->has_gmch_display)
 
 /* DPF == dynamic parity feature */
-#define HAS_L3_DPF(dev) (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
+#define HAS_L3_DPF(dev) (INTEL_INFO(dev)->has_l3_dpf)
 #define NUM_L3_SLICES(dev) (IS_HSW_GT3(dev) ? 2 : HAS_L3_DPF(dev))
 
 #define GT_FREQUENCY_MULTIPLIER 50
@@ -2930,7 +2867,9 @@ extern int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state);
 extern int i915_resume_switcheroo(struct drm_device *dev);
 
 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
-			       	int enable_ppgtt);
+				int enable_ppgtt);
+
+bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value);
 
 /* i915_drv.c */
 void __printf(3, 4)
@@ -2946,7 +2885,7 @@ extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
 #endif
 extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask);
 extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv);
-extern int i915_reset(struct drm_i915_private *dev_priv);
+extern void i915_reset(struct drm_i915_private *dev_priv);
 extern int intel_guc_reset(struct drm_i915_private *dev_priv);
 extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine);
 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
@@ -3107,11 +3046,6 @@ int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
 int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file_priv);
-void i915_gem_execbuffer_move_to_active(struct list_head *vmas,
-					struct drm_i915_gem_request *req);
-int i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
-				   struct drm_i915_gem_execbuffer2 *args,
-				   struct list_head *vmas);
 int i915_gem_execbuffer(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int i915_gem_execbuffer2(struct drm_device *dev, void *data,
@@ -3140,6 +3074,7 @@ int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
 void i915_gem_load_init(struct drm_device *dev);
 void i915_gem_load_cleanup(struct drm_device *dev);
 void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
+int i915_gem_freeze(struct drm_i915_private *dev_priv);
 int i915_gem_freeze_late(struct drm_i915_private *dev_priv);
 
 void *i915_gem_object_alloc(struct drm_device *dev);
@@ -3150,47 +3085,28 @@ struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
 						  size_t size);
 struct drm_i915_gem_object *i915_gem_object_create_from_data(
 		struct drm_device *dev, const void *data, size_t size);
+void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file);
 void i915_gem_free_object(struct drm_gem_object *obj);
-void i915_gem_vma_destroy(struct i915_vma *vma);
-
-/* Flags used by pin/bind&friends. */
-#define PIN_MAPPABLE	(1<<0)
-#define PIN_NONBLOCK	(1<<1)
-#define PIN_GLOBAL	(1<<2)
-#define PIN_OFFSET_BIAS	(1<<3)
-#define PIN_USER	(1<<4)
-#define PIN_UPDATE	(1<<5)
-#define PIN_ZONE_4G	(1<<6)
-#define PIN_HIGH	(1<<7)
-#define PIN_OFFSET_FIXED	(1<<8)
-#define PIN_OFFSET_MASK (~4095)
-int __must_check
-i915_gem_object_pin(struct drm_i915_gem_object *obj,
-		    struct i915_address_space *vm,
-		    uint32_t alignment,
-		    uint64_t flags);
-int __must_check
+
+struct i915_vma * __must_check
 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 			 const struct i915_ggtt_view *view,
-			 uint32_t alignment,
-			 uint64_t flags);
+			 u64 size,
+			 u64 alignment,
+			 u64 flags);
 
 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
 		  u32 flags);
 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
 int __must_check i915_vma_unbind(struct i915_vma *vma);
-/*
- * BEWARE: Do not use the function below unless you can _absolutely_
- * _guarantee_ VMA in question is _not in use_ anywhere.
- */
-int __must_check __i915_vma_unbind_no_wait(struct i915_vma *vma);
+void i915_vma_close(struct i915_vma *vma);
+void i915_vma_destroy(struct i915_vma *vma);
+
+int i915_gem_object_unbind(struct drm_i915_gem_object *obj);
 int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
 void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv);
 void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 
-int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
-				    int *needs_clflush);
-
 int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
 
 static inline int __sg_page_count(struct scatterlist *sg)
@@ -3250,13 +3166,20 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
 	obj->pages_pin_count--;
 }
 
+enum i915_map_type {
+	I915_MAP_WB = 0,
+	I915_MAP_WC,
+};
+
 /**
  * i915_gem_object_pin_map - return a contiguous mapping of the entire object
  * @obj - the object to map into kernel address space
+ * @type - the type of mapping, used to select pgprot_t
  *
  * Calls i915_gem_object_pin_pages() to prevent reaping of the object's
  * pages and then returns a contiguous mapping of the backing storage into
- * the kernel address space.
+ * the kernel address space. Based on the @type of mapping, the PTE will be
+ * set to either WriteBack or WriteCombine (via pgprot_t).
  *
  * The caller must hold the struct_mutex, and is responsible for calling
  * i915_gem_object_unpin_map() when the mapping is no longer required.
@@ -3264,7 +3187,8 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
  * Returns the pointer through which to access the mapped object, or an
  * ERR_PTR() on error.
  */
-void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj);
+void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
+					   enum i915_map_type type);
 
 /**
  * i915_gem_object_unpin_map - releases an earlier mapping
@@ -3283,122 +3207,73 @@ static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj)
 	i915_gem_object_unpin_pages(obj);
 }
 
+int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
+				    unsigned int *needs_clflush);
+int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
+				     unsigned int *needs_clflush);
+#define CLFLUSH_BEFORE 0x1
+#define CLFLUSH_AFTER 0x2
+#define CLFLUSH_FLAGS (CLFLUSH_BEFORE | CLFLUSH_AFTER)
+
+static inline void
+i915_gem_obj_finish_shmem_access(struct drm_i915_gem_object *obj)
+{
+	i915_gem_object_unpin_pages(obj);
+}
+
 int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
-int i915_gem_object_sync(struct drm_i915_gem_object *obj,
-			 struct intel_engine_cs *to,
-			 struct drm_i915_gem_request **to_req);
 void i915_vma_move_to_active(struct i915_vma *vma,
-			     struct drm_i915_gem_request *req);
+			     struct drm_i915_gem_request *req,
+			     unsigned int flags);
 int i915_gem_dumb_create(struct drm_file *file_priv,
 			 struct drm_device *dev,
 			 struct drm_mode_create_dumb *args);
 int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev,
 		      uint32_t handle, uint64_t *offset);
+int i915_gem_mmap_gtt_version(void);
 
 void i915_gem_track_fb(struct drm_i915_gem_object *old,
 		       struct drm_i915_gem_object *new,
 		       unsigned frontbuffer_bits);
 
-/**
- * Returns true if seq1 is later than seq2.
- */
-static inline bool
-i915_seqno_passed(uint32_t seq1, uint32_t seq2)
-{
-	return (int32_t)(seq1 - seq2) >= 0;
-}
-
-static inline bool i915_gem_request_started(const struct drm_i915_gem_request *req)
-{
-	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
-				 req->previous_seqno);
-}
-
-static inline bool i915_gem_request_completed(const struct drm_i915_gem_request *req)
-{
-	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
-				 req->seqno);
-}
-
-bool __i915_spin_request(const struct drm_i915_gem_request *request,
-			 int state, unsigned long timeout_us);
-static inline bool i915_spin_request(const struct drm_i915_gem_request *request,
-				     int state, unsigned long timeout_us)
-{
-	return (i915_gem_request_started(request) &&
-		__i915_spin_request(request, state, timeout_us));
-}
-
-int __must_check i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno);
 int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno);
 
 struct drm_i915_gem_request *
 i915_gem_find_active_request(struct intel_engine_cs *engine);
 
 void i915_gem_retire_requests(struct drm_i915_private *dev_priv);
-void i915_gem_retire_requests_ring(struct intel_engine_cs *engine);
-
-static inline u32 i915_reset_counter(struct i915_gpu_error *error)
-{
-	return atomic_read(&error->reset_counter);
-}
-
-static inline bool __i915_reset_in_progress(u32 reset)
-{
-	return unlikely(reset & I915_RESET_IN_PROGRESS_FLAG);
-}
-
-static inline bool __i915_reset_in_progress_or_wedged(u32 reset)
-{
-	return unlikely(reset & (I915_RESET_IN_PROGRESS_FLAG | I915_WEDGED));
-}
-
-static inline bool __i915_terminally_wedged(u32 reset)
-{
-	return unlikely(reset & I915_WEDGED);
-}
 
 static inline bool i915_reset_in_progress(struct i915_gpu_error *error)
 {
-	return __i915_reset_in_progress(i915_reset_counter(error));
+	return unlikely(test_bit(I915_RESET_IN_PROGRESS, &error->flags));
 }
 
-static inline bool i915_reset_in_progress_or_wedged(struct i915_gpu_error *error)
+static inline bool i915_terminally_wedged(struct i915_gpu_error *error)
 {
-	return __i915_reset_in_progress_or_wedged(i915_reset_counter(error));
+	return unlikely(test_bit(I915_WEDGED, &error->flags));
 }
 
-static inline bool i915_terminally_wedged(struct i915_gpu_error *error)
+static inline bool i915_reset_in_progress_or_wedged(struct i915_gpu_error *error)
 {
-	return __i915_terminally_wedged(i915_reset_counter(error));
+	return i915_reset_in_progress(error) | i915_terminally_wedged(error);
 }
 
 static inline u32 i915_reset_count(struct i915_gpu_error *error)
 {
-	return ((i915_reset_counter(error) & ~I915_WEDGED) + 1) / 2;
+	return READ_ONCE(error->reset_count);
 }
 
-void i915_gem_reset(struct drm_device *dev);
+void i915_gem_reset(struct drm_i915_private *dev_priv);
+void i915_gem_set_wedged(struct drm_i915_private *dev_priv);
 bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force);
 int __must_check i915_gem_init(struct drm_device *dev);
-int i915_gem_init_engines(struct drm_device *dev);
 int __must_check i915_gem_init_hw(struct drm_device *dev);
 void i915_gem_init_swizzling(struct drm_device *dev);
 void i915_gem_cleanup_engines(struct drm_device *dev);
-int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv);
+int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
+					unsigned int flags);
 int __must_check i915_gem_suspend(struct drm_device *dev);
-void __i915_add_request(struct drm_i915_gem_request *req,
-			struct drm_i915_gem_object *batch_obj,
-			bool flush_caches);
-#define i915_add_request(req) \
-	__i915_add_request(req, NULL, true)
-#define i915_add_request_no_flush(req) \
-	__i915_add_request(req, NULL, false)
-int __i915_wait_request(struct drm_i915_gem_request *req,
-			bool interruptible,
-			s64 *timeout,
-			struct intel_rps_client *rps);
-int __must_check i915_wait_request(struct drm_i915_gem_request *req);
+void i915_gem_resume(struct drm_device *dev);
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
 int __must_check
 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
@@ -3408,22 +3283,20 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
 				  bool write);
 int __must_check
 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write);
-int __must_check
+struct i915_vma * __must_check
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 				     u32 alignment,
 				     const struct i915_ggtt_view *view);
-void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
-					      const struct i915_ggtt_view *view);
+void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma);
 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 				int align);
 int i915_gem_open(struct drm_device *dev, struct drm_file *file);
 void i915_gem_release(struct drm_device *dev, struct drm_file *file);
 
-uint32_t
-i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode);
-uint32_t
-i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
-			    int tiling_mode, bool fenced);
+u64 i915_gem_get_ggtt_size(struct drm_i915_private *dev_priv, u64 size,
+			   int tiling_mode);
+u64 i915_gem_get_ggtt_alignment(struct drm_i915_private *dev_priv, u64 size,
+				int tiling_mode, bool fenced);
 
 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 				    enum i915_cache_level cache_level);
@@ -3434,86 +3307,82 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
 struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
 				struct drm_gem_object *gem_obj, int flags);
 
-u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
-				  const struct i915_ggtt_view *view);
-u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
-			struct i915_address_space *vm);
-static inline u64
-i915_gem_obj_ggtt_offset(struct drm_i915_gem_object *o)
-{
-	return i915_gem_obj_ggtt_offset_view(o, &i915_ggtt_view_normal);
-}
-
-bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o);
-bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
-				  const struct i915_ggtt_view *view);
-bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
-			struct i915_address_space *vm);
-
 struct i915_vma *
 i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
-		    struct i915_address_space *vm);
-struct i915_vma *
-i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
-			  const struct i915_ggtt_view *view);
+		     struct i915_address_space *vm,
+		     const struct i915_ggtt_view *view);
 
 struct i915_vma *
 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
-				  struct i915_address_space *vm);
-struct i915_vma *
-i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj,
-				       const struct i915_ggtt_view *view);
-
-static inline struct i915_vma *
-i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
-{
-	return i915_gem_obj_to_ggtt_view(obj, &i915_ggtt_view_normal);
-}
-bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj);
+				  struct i915_address_space *vm,
+				  const struct i915_ggtt_view *view);
 
-/* Some GGTT VM helpers */
 static inline struct i915_hw_ppgtt *
 i915_vm_to_ppgtt(struct i915_address_space *vm)
 {
 	return container_of(vm, struct i915_hw_ppgtt, base);
 }
 
+static inline struct i915_vma *
+i915_gem_object_to_ggtt(struct drm_i915_gem_object *obj,
+			const struct i915_ggtt_view *view)
+{
+	return i915_gem_obj_to_vma(obj, &to_i915(obj->base.dev)->ggtt.base, view);
+}
 
-static inline bool i915_gem_obj_ggtt_bound(struct drm_i915_gem_object *obj)
+static inline unsigned long
+i915_gem_object_ggtt_offset(struct drm_i915_gem_object *o,
+			    const struct i915_ggtt_view *view)
 {
-	return i915_gem_obj_ggtt_bound_view(obj, &i915_ggtt_view_normal);
+	return i915_ggtt_offset(i915_gem_object_to_ggtt(o, view));
 }
 
-unsigned long
-i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj);
+/* i915_gem_fence.c */
+int __must_check i915_vma_get_fence(struct i915_vma *vma);
+int __must_check i915_vma_put_fence(struct i915_vma *vma);
 
-static inline int __must_check
-i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj,
-		      uint32_t alignment,
-		      unsigned flags)
+/**
+ * i915_vma_pin_fence - pin fencing state
+ * @vma: vma to pin fencing for
+ *
+ * This pins the fencing state (whether tiled or untiled) to make sure the
+ * vma (and its object) is ready to be used as a scanout target. Fencing
+ * status must be synchronize first by calling i915_vma_get_fence():
+ *
+ * The resulting fence pin reference must be released again with
+ * i915_vma_unpin_fence().
+ *
+ * Returns:
+ *
+ * True if the vma has a fence, false otherwise.
+ */
+static inline bool
+i915_vma_pin_fence(struct i915_vma *vma)
 {
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-
-	return i915_gem_object_pin(obj, &ggtt->base,
-				   alignment, flags | PIN_GLOBAL);
+	if (vma->fence) {
+		vma->fence->pin_count++;
+		return true;
+	} else
+		return false;
 }
 
-void i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
-				     const struct i915_ggtt_view *view);
+/**
+ * i915_vma_unpin_fence - unpin fencing state
+ * @vma: vma to unpin fencing for
+ *
+ * This releases the fence pin reference acquired through
+ * i915_vma_pin_fence. It will handle both objects with and without an
+ * attached fence correctly, callers do not need to distinguish this.
+ */
 static inline void
-i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
+i915_vma_unpin_fence(struct i915_vma *vma)
 {
-	i915_gem_object_ggtt_unpin_view(obj, &i915_ggtt_view_normal);
+	if (vma->fence) {
+		GEM_BUG_ON(vma->fence->pin_count <= 0);
+		vma->fence->pin_count--;
+	}
 }
 
-/* i915_gem_fence.c */
-int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj);
-int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
-
-bool i915_gem_object_pin_fence(struct drm_i915_gem_object *obj);
-void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj);
-
 void i915_gem_restore_fences(struct drm_device *dev);
 
 void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
@@ -3524,10 +3393,10 @@ void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_context_init(struct drm_device *dev);
 void i915_gem_context_lost(struct drm_i915_private *dev_priv);
 void i915_gem_context_fini(struct drm_device *dev);
-void i915_gem_context_reset(struct drm_device *dev);
 int i915_gem_context_open(struct drm_device *dev, struct drm_file *file);
 void i915_gem_context_close(struct drm_device *dev, struct drm_file *file);
 int i915_switch_context(struct drm_i915_gem_request *req);
+int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv);
 void i915_gem_context_free(struct kref *ctx_ref);
 struct drm_i915_gem_object *
 i915_gem_alloc_context_obj(struct drm_device *dev, size_t size);
@@ -3548,12 +3417,14 @@ i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id)
 	return ctx;
 }
 
-static inline void i915_gem_context_reference(struct i915_gem_context *ctx)
+static inline struct i915_gem_context *
+i915_gem_context_get(struct i915_gem_context *ctx)
 {
 	kref_get(&ctx->ref);
+	return ctx;
 }
 
-static inline void i915_gem_context_unreference(struct i915_gem_context *ctx)
+static inline void i915_gem_context_put(struct i915_gem_context *ctx)
 {
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
 	kref_put(&ctx->ref, i915_gem_context_free);
@@ -3576,13 +3447,10 @@ int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data,
 				       struct drm_file *file);
 
 /* i915_gem_evict.c */
-int __must_check i915_gem_evict_something(struct drm_device *dev,
-					  struct i915_address_space *vm,
-					  int min_size,
-					  unsigned alignment,
+int __must_check i915_gem_evict_something(struct i915_address_space *vm,
+					  u64 min_size, u64 alignment,
 					  unsigned cache_level,
-					  unsigned long start,
-					  unsigned long end,
+					  u64 start, u64 end,
 					  unsigned flags);
 int __must_check i915_gem_evict_for_vma(struct i915_vma *target);
 int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
@@ -3590,6 +3458,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
 /* belongs in i915_gem_gtt.h */
 static inline void i915_gem_chipset_flush(struct drm_i915_private *dev_priv)
 {
+	wmb();
 	if (INTEL_GEN(dev_priv) < 6)
 		intel_gtt_chipset_flush();
 }
@@ -3634,28 +3503,21 @@ static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_objec
 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 
 	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
-		obj->tiling_mode != I915_TILING_NONE;
+		i915_gem_object_is_tiled(obj);
 }
 
-/* i915_gem_debug.c */
-#if WATCH_LISTS
-int i915_verify_lists(struct drm_device *dev);
-#else
-#define i915_verify_lists(dev) 0
-#endif
-
 /* i915_debugfs.c */
 #ifdef CONFIG_DEBUG_FS
 int i915_debugfs_register(struct drm_i915_private *dev_priv);
 void i915_debugfs_unregister(struct drm_i915_private *dev_priv);
 int i915_debugfs_connector_add(struct drm_connector *connector);
-void intel_display_crc_init(struct drm_device *dev);
+void intel_display_crc_init(struct drm_i915_private *dev_priv);
 #else
 static inline int i915_debugfs_register(struct drm_i915_private *dev_priv) {return 0;}
 static inline void i915_debugfs_unregister(struct drm_i915_private *dev_priv) {}
 static inline int i915_debugfs_connector_add(struct drm_connector *connector)
 { return 0; }
-static inline void intel_display_crc_init(struct drm_device *dev) {}
+static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {}
 #endif
 
 /* i915_gpu_error.c */
@@ -3684,23 +3546,23 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
 
 /* i915_cmd_parser.c */
 int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv);
-int i915_cmd_parser_init_ring(struct intel_engine_cs *engine);
-void i915_cmd_parser_fini_ring(struct intel_engine_cs *engine);
-bool i915_needs_cmd_parser(struct intel_engine_cs *engine);
-int i915_parse_cmds(struct intel_engine_cs *engine,
-		    struct drm_i915_gem_object *batch_obj,
-		    struct drm_i915_gem_object *shadow_batch_obj,
-		    u32 batch_start_offset,
-		    u32 batch_len,
-		    bool is_master);
+void intel_engine_init_cmd_parser(struct intel_engine_cs *engine);
+void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine);
+bool intel_engine_needs_cmd_parser(struct intel_engine_cs *engine);
+int intel_engine_cmd_parser(struct intel_engine_cs *engine,
+			    struct drm_i915_gem_object *batch_obj,
+			    struct drm_i915_gem_object *shadow_batch_obj,
+			    u32 batch_start_offset,
+			    u32 batch_len,
+			    bool is_master);
 
 /* i915_suspend.c */
 extern int i915_save_state(struct drm_device *dev);
 extern int i915_restore_state(struct drm_device *dev);
 
 /* i915_sysfs.c */
-void i915_setup_sysfs(struct drm_device *dev_priv);
-void i915_teardown_sysfs(struct drm_device *dev_priv);
+void i915_setup_sysfs(struct drm_i915_private *dev_priv);
+void i915_teardown_sysfs(struct drm_i915_private *dev_priv);
 
 /* intel_i2c.c */
 extern int intel_setup_gmbus(struct drm_device *dev);
@@ -3800,7 +3662,6 @@ extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
 extern void intel_set_memory_cxsr(struct drm_i915_private *dev_priv,
 				  bool enable);
 
-extern bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv);
 int i915_reg_read_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file);
 
@@ -3878,9 +3739,16 @@ int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
  * will be implemented using 2 32-bit writes in an arbitrary order with
  * an arbitrary delay between them. This can cause the hardware to
  * act upon the intermediate value, possibly leading to corruption and
- * machine death. You have been warned.
+ * machine death. For this reason we do not support I915_WRITE64, or
+ * dev_priv->uncore.funcs.mmio_writeq.
+ *
+ * When reading a 64-bit value as two 32-bit values, the delay may cause
+ * the two reads to mismatch, e.g. a timestamp overflowing. Also note that
+ * occasionally a 64-bit register does not actualy support a full readq
+ * and must be read using two 32-bit reads.
+ *
+ * You have been warned.
  */
-#define I915_WRITE64(reg, val)	dev_priv->uncore.funcs.mmio_writeq(dev_priv, (reg), (val), true)
 #define I915_READ64(reg)	dev_priv->uncore.funcs.mmio_readq(dev_priv, (reg), true)
 
 #define I915_READ64_2x32(lower_reg, upper_reg) ({			\
@@ -3923,7 +3791,7 @@ __raw_write(64, q)
 #undef __raw_write
 
 /* These are untraced mmio-accessors that are only valid to be used inside
- * criticial sections inside IRQ handlers where forcewake is explicitly
+ * critical sections inside IRQ handlers where forcewake is explicitly
  * controlled.
  * Think twice, and think again, before using these.
  * Note: Should only be used between intel_uncore_forcewake_irqlock() and
@@ -3995,7 +3863,9 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms)
 			    schedule_timeout_uninterruptible(remaining_jiffies);
 	}
 }
-static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req)
+
+static inline bool
+__i915_request_irq_complete(struct drm_i915_gem_request *req)
 {
 	struct intel_engine_cs *engine = req->engine;
 
@@ -4017,7 +3887,7 @@ static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req)
 	 * is woken.
 	 */
 	if (engine->irq_seqno_barrier &&
-	    READ_ONCE(engine->breadcrumbs.irq_seqno_bh) == current &&
+	    rcu_access_pointer(engine->breadcrumbs.irq_seqno_bh) == current &&
 	    cmpxchg_relaxed(&engine->breadcrumbs.irq_posted, 1, 0)) {
 		struct task_struct *tsk;
 
@@ -4042,7 +3912,7 @@ static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req)
 		 * irq_posted == false but we are still running).
 		 */
 		rcu_read_lock();
-		tsk = READ_ONCE(engine->breadcrumbs.irq_seqno_bh);
+		tsk = rcu_dereference(engine->breadcrumbs.irq_seqno_bh);
 		if (tsk && tsk != current)
 			/* Note that if the bottom-half is changed as we
 			 * are sending the wake-up, the new bottom-half will
@@ -4057,18 +3927,35 @@ static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req)
 			return true;
 	}
 
-	/* We need to check whether any gpu reset happened in between
-	 * the request being submitted and now. If a reset has occurred,
-	 * the seqno will have been advance past ours and our request
-	 * is complete. If we are in the process of handling a reset,
-	 * the request is effectively complete as the rendering will
-	 * be discarded, but we need to return in order to drop the
-	 * struct_mutex.
-	 */
-	if (i915_reset_in_progress(&req->i915->gpu_error))
-		return true;
-
 	return false;
 }
 
+void i915_memcpy_init_early(struct drm_i915_private *dev_priv);
+bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len);
+
+/* i915_mm.c */
+int remap_io_mapping(struct vm_area_struct *vma,
+		     unsigned long addr, unsigned long pfn, unsigned long size,
+		     struct io_mapping *iomap);
+
+#define ptr_mask_bits(ptr) ({						\
+	unsigned long __v = (unsigned long)(ptr);			\
+	(typeof(ptr))(__v & PAGE_MASK);					\
+})
+
+#define ptr_unpack_bits(ptr, bits) ({					\
+	unsigned long __v = (unsigned long)(ptr);			\
+	(bits) = __v & ~PAGE_MASK;					\
+	(typeof(ptr))(__v & PAGE_MASK);					\
+})
+
+#define ptr_pack_bits(ptr, bits)					\
+	((typeof(ptr))((unsigned long)(ptr) | (bits)))
+
+#define fetch_and_zero(ptr) ({						\
+	typeof(*ptr) __T = *(ptr);					\
+	*(ptr) = (typeof(*ptr))0;					\
+	__T;								\
+})
+
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 11681501d7b1..947e82c2b175 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -29,10 +29,13 @@
 #include <drm/drm_vma_manager.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
+#include "i915_gem_dmabuf.h"
 #include "i915_vgpu.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
+#include "intel_frontbuffer.h"
 #include "intel_mocs.h"
+#include <linux/reservation.h>
 #include <linux/shmem_fs.h>
 #include <linux/slab.h>
 #include <linux/swap.h>
@@ -41,10 +44,6 @@
 
 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
-static void
-i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
-static void
-i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
 
 static bool cpu_cache_is_coherent(struct drm_device *dev,
 				  enum i915_cache_level level)
@@ -139,7 +138,6 @@ int i915_mutex_lock_interruptible(struct drm_device *dev)
 	if (ret)
 		return ret;
 
-	WARN_ON(i915_verify_lists(dev));
 	return 0;
 }
 
@@ -156,10 +154,10 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 	pinned = 0;
 	mutex_lock(&dev->struct_mutex);
 	list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
-		if (vma->pin_count)
+		if (i915_vma_is_pinned(vma))
 			pinned += vma->node.size;
 	list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
-		if (vma->pin_count)
+		if (i915_vma_is_pinned(vma))
 			pinned += vma->node.size;
 	mutex_unlock(&dev->struct_mutex);
 
@@ -281,23 +279,129 @@ static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 	.release = i915_gem_object_release_phys,
 };
 
-static int
-drop_pages(struct drm_i915_gem_object *obj)
+int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 {
-	struct i915_vma *vma, *next;
+	struct i915_vma *vma;
+	LIST_HEAD(still_in_list);
 	int ret;
 
-	drm_gem_object_reference(&obj->base);
-	list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
-		if (i915_vma_unbind(vma))
-			break;
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-	ret = i915_gem_object_put_pages(obj);
-	drm_gem_object_unreference(&obj->base);
+	/* Closed vma are removed from the obj->vma_list - but they may
+	 * still have an active binding on the object. To remove those we
+	 * must wait for all rendering to complete to the object (as unbinding
+	 * must anyway), and retire the requests.
+	 */
+	ret = i915_gem_object_wait_rendering(obj, false);
+	if (ret)
+		return ret;
+
+	i915_gem_retire_requests(to_i915(obj->base.dev));
+
+	while ((vma = list_first_entry_or_null(&obj->vma_list,
+					       struct i915_vma,
+					       obj_link))) {
+		list_move_tail(&vma->obj_link, &still_in_list);
+		ret = i915_vma_unbind(vma);
+		if (ret)
+			break;
+	}
+	list_splice(&still_in_list, &obj->vma_list);
 
 	return ret;
 }
 
+/**
+ * Ensures that all rendering to the object has completed and the object is
+ * safe to unbind from the GTT or access from the CPU.
+ * @obj: i915 gem object
+ * @readonly: waiting for just read access or read-write access
+ */
+int
+i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
+			       bool readonly)
+{
+	struct reservation_object *resv;
+	struct i915_gem_active *active;
+	unsigned long active_mask;
+	int idx;
+
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+	if (!readonly) {
+		active = obj->last_read;
+		active_mask = i915_gem_object_get_active(obj);
+	} else {
+		active_mask = 1;
+		active = &obj->last_write;
+	}
+
+	for_each_active(active_mask, idx) {
+		int ret;
+
+		ret = i915_gem_active_wait(&active[idx],
+					   &obj->base.dev->struct_mutex);
+		if (ret)
+			return ret;
+	}
+
+	resv = i915_gem_object_get_dmabuf_resv(obj);
+	if (resv) {
+		long err;
+
+		err = reservation_object_wait_timeout_rcu(resv, !readonly, true,
+							  MAX_SCHEDULE_TIMEOUT);
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+
+/* A nonblocking variant of the above wait. Must be called prior to
+ * acquiring the mutex for the object, as the object state may change
+ * during this call. A reference must be held by the caller for the object.
+ */
+static __must_check int
+__unsafe_wait_rendering(struct drm_i915_gem_object *obj,
+			struct intel_rps_client *rps,
+			bool readonly)
+{
+	struct i915_gem_active *active;
+	unsigned long active_mask;
+	int idx;
+
+	active_mask = __I915_BO_ACTIVE(obj);
+	if (!active_mask)
+		return 0;
+
+	if (!readonly) {
+		active = obj->last_read;
+	} else {
+		active_mask = 1;
+		active = &obj->last_write;
+	}
+
+	for_each_active(active_mask, idx) {
+		int ret;
+
+		ret = i915_gem_active_wait_unlocked(&active[idx],
+						    I915_WAIT_INTERRUPTIBLE,
+						    NULL, rps);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static struct intel_rps_client *to_rps_client(struct drm_file *file)
+{
+	struct drm_i915_file_private *fpriv = file->driver_priv;
+
+	return &fpriv->rps;
+}
+
 int
 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 			    int align)
@@ -318,7 +422,11 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 	if (obj->base.filp == NULL)
 		return -EINVAL;
 
-	ret = drop_pages(obj);
+	ret = i915_gem_object_unbind(obj);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_object_put_pages(obj);
 	if (ret)
 		return ret;
 
@@ -408,7 +516,7 @@ i915_gem_create(struct drm_file *file,
 
 	ret = drm_gem_handle_create(file, &obj->base, &handle);
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(&obj->base);
+	i915_gem_object_put_unlocked(obj);
 	if (ret)
 		return ret;
 
@@ -502,33 +610,106 @@ __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
  * flush the object from the CPU cache.
  */
 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
-				    int *needs_clflush)
+				    unsigned int *needs_clflush)
 {
 	int ret;
 
 	*needs_clflush = 0;
 
-	if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
-		return -EINVAL;
+	if (!i915_gem_object_has_struct_page(obj))
+		return -ENODEV;
+
+	ret = i915_gem_object_wait_rendering(obj, true);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ret;
+
+	i915_gem_object_pin_pages(obj);
+
+	i915_gem_object_flush_gtt_write_domain(obj);
 
-	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
-		/* If we're not in the cpu read domain, set ourself into the gtt
-		 * read domain and manually flush cachelines (if required). This
-		 * optimizes for the case when the gpu will dirty the data
-		 * anyway again before the next pread happens. */
+	/* If we're not in the cpu read domain, set ourself into the gtt
+	 * read domain and manually flush cachelines (if required). This
+	 * optimizes for the case when the gpu will dirty the data
+	 * anyway again before the next pread happens.
+	 */
+	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
 		*needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 							obj->cache_level);
-		ret = i915_gem_object_wait_rendering(obj, true);
+
+	if (*needs_clflush && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+		ret = i915_gem_object_set_to_cpu_domain(obj, false);
 		if (ret)
-			return ret;
+			goto err_unpin;
+
+		*needs_clflush = 0;
 	}
 
+	/* return with the pages pinned */
+	return 0;
+
+err_unpin:
+	i915_gem_object_unpin_pages(obj);
+	return ret;
+}
+
+int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
+				     unsigned int *needs_clflush)
+{
+	int ret;
+
+	*needs_clflush = 0;
+	if (!i915_gem_object_has_struct_page(obj))
+		return -ENODEV;
+
+	ret = i915_gem_object_wait_rendering(obj, false);
+	if (ret)
+		return ret;
+
 	ret = i915_gem_object_get_pages(obj);
 	if (ret)
 		return ret;
 
 	i915_gem_object_pin_pages(obj);
 
+	i915_gem_object_flush_gtt_write_domain(obj);
+
+	/* If we're not in the cpu write domain, set ourself into the
+	 * gtt write domain and manually flush cachelines (as required).
+	 * This optimizes for the case when the gpu will use the data
+	 * right away and we therefore have to clflush anyway.
+	 */
+	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
+		*needs_clflush |= cpu_write_needs_clflush(obj) << 1;
+
+	/* Same trick applies to invalidate partially written cachelines read
+	 * before writing.
+	 */
+	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
+		*needs_clflush |= !cpu_cache_is_coherent(obj->base.dev,
+							 obj->cache_level);
+
+	if (*needs_clflush && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+		ret = i915_gem_object_set_to_cpu_domain(obj, true);
+		if (ret)
+			goto err_unpin;
+
+		*needs_clflush = 0;
+	}
+
+	if ((*needs_clflush & CLFLUSH_AFTER) == 0)
+		obj->cache_dirty = true;
+
+	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
+	obj->dirty = 1;
+	/* return with the pages pinned */
+	return 0;
+
+err_unpin:
+	i915_gem_object_unpin_pages(obj);
 	return ret;
 }
 
@@ -638,14 +819,24 @@ i915_gem_gtt_pread(struct drm_device *dev,
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
+	struct i915_vma *vma;
 	struct drm_mm_node node;
 	char __user *user_data;
 	uint64_t remain;
 	uint64_t offset;
 	int ret;
 
-	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
-	if (ret) {
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
+	if (!IS_ERR(vma)) {
+		node.start = i915_ggtt_offset(vma);
+		node.allocated = false;
+		ret = i915_vma_put_fence(vma);
+		if (ret) {
+			i915_vma_unpin(vma);
+			vma = ERR_PTR(ret);
+		}
+	}
+	if (IS_ERR(vma)) {
 		ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
 		if (ret)
 			goto out;
@@ -657,12 +848,6 @@ i915_gem_gtt_pread(struct drm_device *dev,
 		}
 
 		i915_gem_object_pin_pages(obj);
-	} else {
-		node.start = i915_gem_obj_ggtt_offset(obj);
-		node.allocated = false;
-		ret = i915_gem_object_put_fence(obj);
-		if (ret)
-			goto out_unpin;
 	}
 
 	ret = i915_gem_object_set_to_gtt_domain(obj, false);
@@ -675,7 +860,7 @@ i915_gem_gtt_pread(struct drm_device *dev,
 
 	mutex_unlock(&dev->struct_mutex);
 	if (likely(!i915.prefault_disable)) {
-		ret = fault_in_multipages_writeable(user_data, remain);
+		ret = fault_in_pages_writeable(user_data, remain);
 		if (ret) {
 			mutex_lock(&dev->struct_mutex);
 			goto out_unpin;
@@ -707,7 +892,7 @@ i915_gem_gtt_pread(struct drm_device *dev,
 		 * and write to user memory which may result into page
 		 * faults, and so we cannot perform this under struct_mutex.
 		 */
-		if (slow_user_access(ggtt->mappable, page_base,
+		if (slow_user_access(&ggtt->mappable, page_base,
 				     page_offset, user_data,
 				     page_length, false)) {
 			ret = -EFAULT;
@@ -739,7 +924,7 @@ out_unpin:
 		i915_gem_object_unpin_pages(obj);
 		remove_mappable_node(&node);
 	} else {
-		i915_gem_object_ggtt_unpin(obj);
+		i915_vma_unpin(vma);
 	}
 out:
 	return ret;
@@ -760,19 +945,14 @@ i915_gem_shmem_pread(struct drm_device *dev,
 	int needs_clflush = 0;
 	struct sg_page_iter sg_iter;
 
-	if (!i915_gem_object_has_struct_page(obj))
-		return -ENODEV;
-
-	user_data = u64_to_user_ptr(args->data_ptr);
-	remain = args->size;
-
-	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-
 	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 	if (ret)
 		return ret;
 
+	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
+	user_data = u64_to_user_ptr(args->data_ptr);
 	offset = args->offset;
+	remain = args->size;
 
 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 			 offset >> PAGE_SHIFT) {
@@ -803,7 +983,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
 		mutex_unlock(&dev->struct_mutex);
 
 		if (likely(!i915.prefault_disable) && !prefaulted) {
-			ret = fault_in_multipages_writeable(user_data, remain);
+			ret = fault_in_pages_writeable(user_data, remain);
 			/* Userspace is tricking us, but we've already clobbered
 			 * its pages with the prefault and promised to write the
 			 * data up to the first fault. Hence ignore any errors
@@ -828,7 +1008,7 @@ next_page:
 	}
 
 out:
-	i915_gem_object_unpin_pages(obj);
+	i915_gem_obj_finish_shmem_access(obj);
 
 	return ret;
 }
@@ -857,36 +1037,44 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 		       args->size))
 		return -EFAULT;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
-
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
-	if (&obj->base == NULL) {
-		ret = -ENOENT;
-		goto unlock;
-	}
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj)
+		return -ENOENT;
 
 	/* Bounds check source.  */
 	if (args->offset > obj->base.size ||
 	    args->size > obj->base.size - args->offset) {
 		ret = -EINVAL;
-		goto out;
+		goto err;
 	}
 
 	trace_i915_gem_object_pread(obj, args->offset, args->size);
 
+	ret = __unsafe_wait_rendering(obj, to_rps_client(file), true);
+	if (ret)
+		goto err;
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		goto err;
+
 	ret = i915_gem_shmem_pread(dev, obj, args, file);
 
 	/* pread for non shmem backed objects */
-	if (ret == -EFAULT || ret == -ENODEV)
+	if (ret == -EFAULT || ret == -ENODEV) {
+		intel_runtime_pm_get(to_i915(dev));
 		ret = i915_gem_gtt_pread(dev, obj, args->size,
 					args->offset, args->data_ptr);
+		intel_runtime_pm_put(to_i915(dev));
+	}
 
-out:
-	drm_gem_object_unreference(&obj->base);
-unlock:
+	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
+
+	return ret;
+
+err:
+	i915_gem_object_put_unlocked(obj);
 	return ret;
 }
 
@@ -916,7 +1104,7 @@ fast_user_write(struct io_mapping *mapping,
 /**
  * This is the fast pwrite path, where we copy the data directly from the
  * user into the GTT, uncached.
- * @dev: drm device pointer
+ * @i915: i915 device private data
  * @obj: i915 gem object
  * @args: pwrite arguments structure
  * @file: drm file pointer
@@ -929,17 +1117,28 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 {
 	struct i915_ggtt *ggtt = &i915->ggtt;
 	struct drm_device *dev = obj->base.dev;
+	struct i915_vma *vma;
 	struct drm_mm_node node;
 	uint64_t remain, offset;
 	char __user *user_data;
 	int ret;
 	bool hit_slow_path = false;
 
-	if (obj->tiling_mode != I915_TILING_NONE)
+	if (i915_gem_object_is_tiled(obj))
 		return -EFAULT;
 
-	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
-	if (ret) {
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+				       PIN_MAPPABLE | PIN_NONBLOCK);
+	if (!IS_ERR(vma)) {
+		node.start = i915_ggtt_offset(vma);
+		node.allocated = false;
+		ret = i915_vma_put_fence(vma);
+		if (ret) {
+			i915_vma_unpin(vma);
+			vma = ERR_PTR(ret);
+		}
+	}
+	if (IS_ERR(vma)) {
 		ret = insert_mappable_node(i915, &node, PAGE_SIZE);
 		if (ret)
 			goto out;
@@ -951,19 +1150,13 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 		}
 
 		i915_gem_object_pin_pages(obj);
-	} else {
-		node.start = i915_gem_obj_ggtt_offset(obj);
-		node.allocated = false;
-		ret = i915_gem_object_put_fence(obj);
-		if (ret)
-			goto out_unpin;
 	}
 
 	ret = i915_gem_object_set_to_gtt_domain(obj, true);
 	if (ret)
 		goto out_unpin;
 
-	intel_fb_obj_invalidate(obj, ORIGIN_GTT);
+	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 	obj->dirty = true;
 
 	user_data = u64_to_user_ptr(args->data_ptr);
@@ -995,11 +1188,11 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 		 * If the object is non-shmem backed, we retry again with the
 		 * path that handles page fault.
 		 */
-		if (fast_user_write(ggtt->mappable, page_base,
+		if (fast_user_write(&ggtt->mappable, page_base,
 				    page_offset, user_data, page_length)) {
 			hit_slow_path = true;
 			mutex_unlock(&dev->struct_mutex);
-			if (slow_user_access(ggtt->mappable,
+			if (slow_user_access(&ggtt->mappable,
 					     page_base,
 					     page_offset, user_data,
 					     page_length, true)) {
@@ -1030,7 +1223,7 @@ out_flush:
 		}
 	}
 
-	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
+	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 out_unpin:
 	if (node.allocated) {
 		wmb();
@@ -1040,7 +1233,7 @@ out_unpin:
 		i915_gem_object_unpin_pages(obj);
 		remove_mappable_node(&node);
 	} else {
-		i915_gem_object_ggtt_unpin(obj);
+		i915_vma_unpin(vma);
 	}
 out:
 	return ret;
@@ -1123,41 +1316,17 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 	int shmem_page_offset, page_length, ret = 0;
 	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 	int hit_slowpath = 0;
-	int needs_clflush_after = 0;
-	int needs_clflush_before = 0;
+	unsigned int needs_clflush;
 	struct sg_page_iter sg_iter;
 
-	user_data = u64_to_user_ptr(args->data_ptr);
-	remain = args->size;
-
-	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-
-	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
-		/* If we're not in the cpu write domain, set ourself into the gtt
-		 * write domain and manually flush cachelines (if required). This
-		 * optimizes for the case when the gpu will use the data
-		 * right away and we therefore have to clflush anyway. */
-		needs_clflush_after = cpu_write_needs_clflush(obj);
-		ret = i915_gem_object_wait_rendering(obj, false);
-		if (ret)
-			return ret;
-	}
-	/* Same trick applies to invalidate partially written cachelines read
-	 * before writing. */
-	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
-		needs_clflush_before =
-			!cpu_cache_is_coherent(dev, obj->cache_level);
-
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
 	if (ret)
 		return ret;
 
-	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
-
-	i915_gem_object_pin_pages(obj);
-
+	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
+	user_data = u64_to_user_ptr(args->data_ptr);
 	offset = args->offset;
-	obj->dirty = 1;
+	remain = args->size;
 
 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 			 offset >> PAGE_SHIFT) {
@@ -1181,7 +1350,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 		/* If we don't overwrite a cacheline completely we need to be
 		 * careful to have up-to-date data by first clflushing. Don't
 		 * overcomplicate things and flush the entire patch. */
-		partial_cacheline_write = needs_clflush_before &&
+		partial_cacheline_write = needs_clflush & CLFLUSH_BEFORE &&
 			((shmem_page_offset | page_length)
 				& (boot_cpu_data.x86_clflush_size - 1));
 
@@ -1191,7 +1360,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 					user_data, page_do_bit17_swizzling,
 					partial_cacheline_write,
-					needs_clflush_after);
+					needs_clflush & CLFLUSH_AFTER);
 		if (ret == 0)
 			goto next_page;
 
@@ -1200,7 +1369,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 					user_data, page_do_bit17_swizzling,
 					partial_cacheline_write,
-					needs_clflush_after);
+					needs_clflush & CLFLUSH_AFTER);
 
 		mutex_lock(&dev->struct_mutex);
 
@@ -1214,7 +1383,7 @@ next_page:
 	}
 
 out:
-	i915_gem_object_unpin_pages(obj);
+	i915_gem_obj_finish_shmem_access(obj);
 
 	if (hit_slowpath) {
 		/*
@@ -1222,17 +1391,15 @@ out:
 		 * cachelines in-line while writing and the object moved
 		 * out of the cpu write domain while we've dropped the lock.
 		 */
-		if (!needs_clflush_after &&
+		if (!(needs_clflush & CLFLUSH_AFTER) &&
 		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 			if (i915_gem_clflush_object(obj, obj->pin_display))
-				needs_clflush_after = true;
+				needs_clflush |= CLFLUSH_AFTER;
 		}
 	}
 
-	if (needs_clflush_after)
+	if (needs_clflush & CLFLUSH_AFTER)
 		i915_gem_chipset_flush(to_i915(dev));
-	else
-		obj->cache_dirty = true;
 
 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 	return ret;
@@ -1264,33 +1431,35 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		return -EFAULT;
 
 	if (likely(!i915.prefault_disable)) {
-		ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
+		ret = fault_in_pages_readable(u64_to_user_ptr(args->data_ptr),
 						   args->size);
 		if (ret)
 			return -EFAULT;
 	}
 
-	intel_runtime_pm_get(dev_priv);
-
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		goto put_rpm;
-
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
-	if (&obj->base == NULL) {
-		ret = -ENOENT;
-		goto unlock;
-	}
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj)
+		return -ENOENT;
 
 	/* Bounds check destination. */
 	if (args->offset > obj->base.size ||
 	    args->size > obj->base.size - args->offset) {
 		ret = -EINVAL;
-		goto out;
+		goto err;
 	}
 
 	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 
+	ret = __unsafe_wait_rendering(obj, to_rps_client(file), false);
+	if (ret)
+		goto err;
+
+	intel_runtime_pm_get(dev_priv);
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		goto err_rpm;
+
 	ret = -EFAULT;
 	/* We can only do the GTT pwrite on untiled buffers, as otherwise
 	 * it would end up going through the fenced access, and we'll get
@@ -1306,508 +1475,31 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		 * textures). Fallback to the shmem path in that case. */
 	}
 
-	if (ret == -EFAULT) {
+	if (ret == -EFAULT || ret == -ENOSPC) {
 		if (obj->phys_handle)
 			ret = i915_gem_phys_pwrite(obj, args, file);
-		else if (i915_gem_object_has_struct_page(obj))
-			ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 		else
-			ret = -ENODEV;
+			ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 	}
 
-out:
-	drm_gem_object_unreference(&obj->base);
-unlock:
+	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
-put_rpm:
 	intel_runtime_pm_put(dev_priv);
 
 	return ret;
-}
-
-static int
-i915_gem_check_wedge(unsigned reset_counter, bool interruptible)
-{
-	if (__i915_terminally_wedged(reset_counter))
-		return -EIO;
-
-	if (__i915_reset_in_progress(reset_counter)) {
-		/* Non-interruptible callers can't handle -EAGAIN, hence return
-		 * -EIO unconditionally for these. */
-		if (!interruptible)
-			return -EIO;
-
-		return -EAGAIN;
-	}
-
-	return 0;
-}
-
-static unsigned long local_clock_us(unsigned *cpu)
-{
-	unsigned long t;
-
-	/* Cheaply and approximately convert from nanoseconds to microseconds.
-	 * The result and subsequent calculations are also defined in the same
-	 * approximate microseconds units. The principal source of timing
-	 * error here is from the simple truncation.
-	 *
-	 * Note that local_clock() is only defined wrt to the current CPU;
-	 * the comparisons are no longer valid if we switch CPUs. Instead of
-	 * blocking preemption for the entire busywait, we can detect the CPU
-	 * switch and use that as indicator of system load and a reason to
-	 * stop busywaiting, see busywait_stop().
-	 */
-	*cpu = get_cpu();
-	t = local_clock() >> 10;
-	put_cpu();
-
-	return t;
-}
-
-static bool busywait_stop(unsigned long timeout, unsigned cpu)
-{
-	unsigned this_cpu;
-
-	if (time_after(local_clock_us(&this_cpu), timeout))
-		return true;
-
-	return this_cpu != cpu;
-}
-
-bool __i915_spin_request(const struct drm_i915_gem_request *req,
-			 int state, unsigned long timeout_us)
-{
-	unsigned cpu;
-
-	/* When waiting for high frequency requests, e.g. during synchronous
-	 * rendering split between the CPU and GPU, the finite amount of time
-	 * required to set up the irq and wait upon it limits the response
-	 * rate. By busywaiting on the request completion for a short while we
-	 * can service the high frequency waits as quick as possible. However,
-	 * if it is a slow request, we want to sleep as quickly as possible.
-	 * The tradeoff between waiting and sleeping is roughly the time it
-	 * takes to sleep on a request, on the order of a microsecond.
-	 */
-
-	timeout_us += local_clock_us(&cpu);
-	do {
-		if (i915_gem_request_completed(req))
-			return true;
-
-		if (signal_pending_state(state, current))
-			break;
-
-		if (busywait_stop(timeout_us, cpu))
-			break;
-
-		cpu_relax_lowlatency();
-	} while (!need_resched());
-
-	return false;
-}
-
-/**
- * __i915_wait_request - wait until execution of request has finished
- * @req: duh!
- * @interruptible: do an interruptible wait (normally yes)
- * @timeout: in - how long to wait (NULL forever); out - how much time remaining
- * @rps: RPS client
- *
- * Note: It is of utmost importance that the passed in seqno and reset_counter
- * values have been read by the caller in an smp safe manner. Where read-side
- * locks are involved, it is sufficient to read the reset_counter before
- * unlocking the lock that protects the seqno. For lockless tricks, the
- * reset_counter _must_ be read before, and an appropriate smp_rmb must be
- * inserted.
- *
- * Returns 0 if the request was found within the alloted time. Else returns the
- * errno with remaining time filled in timeout argument.
- */
-int __i915_wait_request(struct drm_i915_gem_request *req,
-			bool interruptible,
-			s64 *timeout,
-			struct intel_rps_client *rps)
-{
-	int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
-	DEFINE_WAIT(reset);
-	struct intel_wait wait;
-	unsigned long timeout_remain;
-	s64 before = 0; /* Only to silence a compiler warning. */
-	int ret = 0;
-
-	might_sleep();
-
-	if (list_empty(&req->list))
-		return 0;
-
-	if (i915_gem_request_completed(req))
-		return 0;
-
-	timeout_remain = MAX_SCHEDULE_TIMEOUT;
-	if (timeout) {
-		if (WARN_ON(*timeout < 0))
-			return -EINVAL;
-
-		if (*timeout == 0)
-			return -ETIME;
-
-		timeout_remain = nsecs_to_jiffies_timeout(*timeout);
-
-		/*
-		 * Record current time in case interrupted by signal, or wedged.
-		 */
-		before = ktime_get_raw_ns();
-	}
-
-	trace_i915_gem_request_wait_begin(req);
-
-	/* This client is about to stall waiting for the GPU. In many cases
-	 * this is undesirable and limits the throughput of the system, as
-	 * many clients cannot continue processing user input/output whilst
-	 * blocked. RPS autotuning may take tens of milliseconds to respond
-	 * to the GPU load and thus incurs additional latency for the client.
-	 * We can circumvent that by promoting the GPU frequency to maximum
-	 * before we wait. This makes the GPU throttle up much more quickly
-	 * (good for benchmarks and user experience, e.g. window animations),
-	 * but at a cost of spending more power processing the workload
-	 * (bad for battery). Not all clients even want their results
-	 * immediately and for them we should just let the GPU select its own
-	 * frequency to maximise efficiency. To prevent a single client from
-	 * forcing the clocks too high for the whole system, we only allow
-	 * each client to waitboost once in a busy period.
-	 */
-	if (INTEL_INFO(req->i915)->gen >= 6)
-		gen6_rps_boost(req->i915, rps, req->emitted_jiffies);
-
-	/* Optimistic spin for the next ~jiffie before touching IRQs */
-	if (i915_spin_request(req, state, 5))
-		goto complete;
-
-	set_current_state(state);
-	add_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
-
-	intel_wait_init(&wait, req->seqno);
-	if (intel_engine_add_wait(req->engine, &wait))
-		/* In order to check that we haven't missed the interrupt
-		 * as we enabled it, we need to kick ourselves to do a
-		 * coherent check on the seqno before we sleep.
-		 */
-		goto wakeup;
-
-	for (;;) {
-		if (signal_pending_state(state, current)) {
-			ret = -ERESTARTSYS;
-			break;
-		}
-
-		timeout_remain = io_schedule_timeout(timeout_remain);
-		if (timeout_remain == 0) {
-			ret = -ETIME;
-			break;
-		}
-
-		if (intel_wait_complete(&wait))
-			break;
-
-		set_current_state(state);
-
-wakeup:
-		/* Carefully check if the request is complete, giving time
-		 * for the seqno to be visible following the interrupt.
-		 * We also have to check in case we are kicked by the GPU
-		 * reset in order to drop the struct_mutex.
-		 */
-		if (__i915_request_irq_complete(req))
-			break;
-
-		/* Only spin if we know the GPU is processing this request */
-		if (i915_spin_request(req, state, 2))
-			break;
-	}
-	remove_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
-
-	intel_engine_remove_wait(req->engine, &wait);
-	__set_current_state(TASK_RUNNING);
-complete:
-	trace_i915_gem_request_wait_end(req);
-
-	if (timeout) {
-		s64 tres = *timeout - (ktime_get_raw_ns() - before);
-
-		*timeout = tres < 0 ? 0 : tres;
-
-		/*
-		 * Apparently ktime isn't accurate enough and occasionally has a
-		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
-		 * things up to make the test happy. We allow up to 1 jiffy.
-		 *
-		 * This is a regrssion from the timespec->ktime conversion.
-		 */
-		if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
-			*timeout = 0;
-	}
-
-	if (rps && req->seqno == req->engine->last_submitted_seqno) {
-		/* The GPU is now idle and this client has stalled.
-		 * Since no other client has submitted a request in the
-		 * meantime, assume that this client is the only one
-		 * supplying work to the GPU but is unable to keep that
-		 * work supplied because it is waiting. Since the GPU is
-		 * then never kept fully busy, RPS autoclocking will
-		 * keep the clocks relatively low, causing further delays.
-		 * Compensate by giving the synchronous client credit for
-		 * a waitboost next time.
-		 */
-		spin_lock(&req->i915->rps.client_lock);
-		list_del_init(&rps->link);
-		spin_unlock(&req->i915->rps.client_lock);
-	}
-
-	return ret;
-}
-
-int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
-				   struct drm_file *file)
-{
-	struct drm_i915_file_private *file_priv;
-
-	WARN_ON(!req || !file || req->file_priv);
-
-	if (!req || !file)
-		return -EINVAL;
-
-	if (req->file_priv)
-		return -EINVAL;
-
-	file_priv = file->driver_priv;
-
-	spin_lock(&file_priv->mm.lock);
-	req->file_priv = file_priv;
-	list_add_tail(&req->client_list, &file_priv->mm.request_list);
-	spin_unlock(&file_priv->mm.lock);
-
-	req->pid = get_pid(task_pid(current));
-
-	return 0;
-}
-
-static inline void
-i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
-{
-	struct drm_i915_file_private *file_priv = request->file_priv;
-
-	if (!file_priv)
-		return;
-
-	spin_lock(&file_priv->mm.lock);
-	list_del(&request->client_list);
-	request->file_priv = NULL;
-	spin_unlock(&file_priv->mm.lock);
-
-	put_pid(request->pid);
-	request->pid = NULL;
-}
-
-static void i915_gem_request_retire(struct drm_i915_gem_request *request)
-{
-	trace_i915_gem_request_retire(request);
-
-	/* We know the GPU must have read the request to have
-	 * sent us the seqno + interrupt, so use the position
-	 * of tail of the request to update the last known position
-	 * of the GPU head.
-	 *
-	 * Note this requires that we are always called in request
-	 * completion order.
-	 */
-	request->ringbuf->last_retired_head = request->postfix;
-
-	list_del_init(&request->list);
-	i915_gem_request_remove_from_client(request);
-
-	if (request->previous_context) {
-		if (i915.enable_execlists)
-			intel_lr_context_unpin(request->previous_context,
-					       request->engine);
-	}
-
-	i915_gem_context_unreference(request->ctx);
-	i915_gem_request_unreference(request);
-}
-
-static void
-__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
-{
-	struct intel_engine_cs *engine = req->engine;
-	struct drm_i915_gem_request *tmp;
-
-	lockdep_assert_held(&engine->i915->drm.struct_mutex);
-
-	if (list_empty(&req->list))
-		return;
-
-	do {
-		tmp = list_first_entry(&engine->request_list,
-				       typeof(*tmp), list);
-
-		i915_gem_request_retire(tmp);
-	} while (tmp != req);
-
-	WARN_ON(i915_verify_lists(engine->dev));
-}
-
-/**
- * Waits for a request to be signaled, and cleans up the
- * request and object lists appropriately for that event.
- * @req: request to wait on
- */
-int
-i915_wait_request(struct drm_i915_gem_request *req)
-{
-	struct drm_i915_private *dev_priv = req->i915;
-	bool interruptible;
-	int ret;
-
-	interruptible = dev_priv->mm.interruptible;
-
-	BUG_ON(!mutex_is_locked(&dev_priv->drm.struct_mutex));
-
-	ret = __i915_wait_request(req, interruptible, NULL, NULL);
-	if (ret)
-		return ret;
-
-	/* If the GPU hung, we want to keep the requests to find the guilty. */
-	if (!i915_reset_in_progress(&dev_priv->gpu_error))
-		__i915_gem_request_retire__upto(req);
-
-	return 0;
-}
-
-/**
- * Ensures that all rendering to the object has completed and the object is
- * safe to unbind from the GTT or access from the CPU.
- * @obj: i915 gem object
- * @readonly: waiting for read access or write
- */
-int
-i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
-			       bool readonly)
-{
-	int ret, i;
-
-	if (!obj->active)
-		return 0;
-
-	if (readonly) {
-		if (obj->last_write_req != NULL) {
-			ret = i915_wait_request(obj->last_write_req);
-			if (ret)
-				return ret;
-
-			i = obj->last_write_req->engine->id;
-			if (obj->last_read_req[i] == obj->last_write_req)
-				i915_gem_object_retire__read(obj, i);
-			else
-				i915_gem_object_retire__write(obj);
-		}
-	} else {
-		for (i = 0; i < I915_NUM_ENGINES; i++) {
-			if (obj->last_read_req[i] == NULL)
-				continue;
-
-			ret = i915_wait_request(obj->last_read_req[i]);
-			if (ret)
-				return ret;
-
-			i915_gem_object_retire__read(obj, i);
-		}
-		GEM_BUG_ON(obj->active);
-	}
-
-	return 0;
-}
-
-static void
-i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
-			       struct drm_i915_gem_request *req)
-{
-	int ring = req->engine->id;
-
-	if (obj->last_read_req[ring] == req)
-		i915_gem_object_retire__read(obj, ring);
-	else if (obj->last_write_req == req)
-		i915_gem_object_retire__write(obj);
-
-	if (!i915_reset_in_progress(&req->i915->gpu_error))
-		__i915_gem_request_retire__upto(req);
-}
-
-/* A nonblocking variant of the above wait. This is a highly dangerous routine
- * as the object state may change during this call.
- */
-static __must_check int
-i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
-					    struct intel_rps_client *rps,
-					    bool readonly)
-{
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
-	int ret, i, n = 0;
-
-	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
-	BUG_ON(!dev_priv->mm.interruptible);
-
-	if (!obj->active)
-		return 0;
-
-	if (readonly) {
-		struct drm_i915_gem_request *req;
-
-		req = obj->last_write_req;
-		if (req == NULL)
-			return 0;
-
-		requests[n++] = i915_gem_request_reference(req);
-	} else {
-		for (i = 0; i < I915_NUM_ENGINES; i++) {
-			struct drm_i915_gem_request *req;
-
-			req = obj->last_read_req[i];
-			if (req == NULL)
-				continue;
-
-			requests[n++] = i915_gem_request_reference(req);
-		}
-	}
-
-	mutex_unlock(&dev->struct_mutex);
-	ret = 0;
-	for (i = 0; ret == 0 && i < n; i++)
-		ret = __i915_wait_request(requests[i], true, NULL, rps);
-	mutex_lock(&dev->struct_mutex);
-
-	for (i = 0; i < n; i++) {
-		if (ret == 0)
-			i915_gem_object_retire_request(obj, requests[i]);
-		i915_gem_request_unreference(requests[i]);
-	}
 
+err_rpm:
+	intel_runtime_pm_put(dev_priv);
+err:
+	i915_gem_object_put_unlocked(obj);
 	return ret;
 }
 
-static struct intel_rps_client *to_rps_client(struct drm_file *file)
-{
-	struct drm_i915_file_private *fpriv = file->driver_priv;
-	return &fpriv->rps;
-}
-
-static enum fb_op_origin
+static inline enum fb_op_origin
 write_origin(struct drm_i915_gem_object *obj, unsigned domain)
 {
-	return domain == I915_GEM_DOMAIN_GTT && !obj->has_wc_mmap ?
-	       ORIGIN_GTT : ORIGIN_CPU;
+	return (domain == I915_GEM_DOMAIN_GTT ?
+		obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
 }
 
 /**
@@ -1828,10 +1520,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	int ret;
 
 	/* Only handle setting domains to types used by the CPU. */
-	if (write_domain & I915_GEM_GPU_DOMAINS)
-		return -EINVAL;
-
-	if (read_domains & I915_GEM_GPU_DOMAINS)
+	if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
 		return -EINVAL;
 
 	/* Having something in the write domain implies it's in the read
@@ -1840,25 +1529,21 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	if (write_domain != 0 && read_domains != write_domain)
 		return -EINVAL;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
-
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
-	if (&obj->base == NULL) {
-		ret = -ENOENT;
-		goto unlock;
-	}
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj)
+		return -ENOENT;
 
 	/* Try to flush the object off the GPU without holding the lock.
 	 * We will repeat the flush holding the lock in the normal manner
 	 * to catch cases where we are gazumped.
 	 */
-	ret = i915_gem_object_wait_rendering__nonblocking(obj,
-							  to_rps_client(file),
-							  !write_domain);
+	ret = __unsafe_wait_rendering(obj, to_rps_client(file), !write_domain);
+	if (ret)
+		goto err;
+
+	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
-		goto unref;
+		goto err;
 
 	if (read_domains & I915_GEM_DOMAIN_GTT)
 		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
@@ -1868,11 +1553,13 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	if (write_domain != 0)
 		intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
 
-unref:
-	drm_gem_object_unreference(&obj->base);
-unlock:
+	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
+
+err:
+	i915_gem_object_put_unlocked(obj);
+	return ret;
 }
 
 /**
@@ -1887,26 +1574,23 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_sw_finish *args = data;
 	struct drm_i915_gem_object *obj;
-	int ret = 0;
-
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
+	int err = 0;
 
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
-	if (&obj->base == NULL) {
-		ret = -ENOENT;
-		goto unlock;
-	}
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj)
+		return -ENOENT;
 
 	/* Pinned buffers may be scanout, so flush the cache */
-	if (obj->pin_display)
-		i915_gem_object_flush_cpu_write_domain(obj);
+	if (READ_ONCE(obj->pin_display)) {
+		err = i915_mutex_lock_interruptible(dev);
+		if (!err) {
+			i915_gem_object_flush_cpu_write_domain(obj);
+			mutex_unlock(&dev->struct_mutex);
+		}
+	}
 
-	drm_gem_object_unreference(&obj->base);
-unlock:
-	mutex_unlock(&dev->struct_mutex);
-	return ret;
+	i915_gem_object_put_unlocked(obj);
+	return err;
 }
 
 /**
@@ -1934,7 +1618,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 		    struct drm_file *file)
 {
 	struct drm_i915_gem_mmap *args = data;
-	struct drm_gem_object *obj;
+	struct drm_i915_gem_object *obj;
 	unsigned long addr;
 
 	if (args->flags & ~(I915_MMAP_WC))
@@ -1943,19 +1627,19 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 	if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
 		return -ENODEV;
 
-	obj = drm_gem_object_lookup(file, args->handle);
-	if (obj == NULL)
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj)
 		return -ENOENT;
 
 	/* prime objects have no backing filp to GEM mmap
 	 * pages from.
 	 */
-	if (!obj->filp) {
-		drm_gem_object_unreference_unlocked(obj);
+	if (!obj->base.filp) {
+		i915_gem_object_put_unlocked(obj);
 		return -EINVAL;
 	}
 
-	addr = vm_mmap(obj->filp, 0, args->size,
+	addr = vm_mmap(obj->base.filp, 0, args->size,
 		       PROT_READ | PROT_WRITE, MAP_SHARED,
 		       args->offset);
 	if (args->flags & I915_MMAP_WC) {
@@ -1963,7 +1647,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 		struct vm_area_struct *vma;
 
 		if (down_write_killable(&mm->mmap_sem)) {
-			drm_gem_object_unreference_unlocked(obj);
+			i915_gem_object_put_unlocked(obj);
 			return -EINTR;
 		}
 		vma = find_vma(mm, addr);
@@ -1975,9 +1659,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 		up_write(&mm->mmap_sem);
 
 		/* This may race, but that's ok, it only gets set */
-		WRITE_ONCE(to_intel_bo(obj)->has_wc_mmap, true);
+		WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
 	}
-	drm_gem_object_unreference_unlocked(obj);
+	i915_gem_object_put_unlocked(obj);
 	if (IS_ERR((void *)addr))
 		return addr;
 
@@ -1986,9 +1670,69 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 	return 0;
 }
 
+static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
+{
+	u64 size;
+
+	size = i915_gem_object_get_stride(obj);
+	size *= i915_gem_object_get_tiling(obj) == I915_TILING_Y ? 32 : 8;
+
+	return size >> PAGE_SHIFT;
+}
+
+/**
+ * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
+ *
+ * A history of the GTT mmap interface:
+ *
+ * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
+ *     aligned and suitable for fencing, and still fit into the available
+ *     mappable space left by the pinned display objects. A classic problem
+ *     we called the page-fault-of-doom where we would ping-pong between
+ *     two objects that could not fit inside the GTT and so the memcpy
+ *     would page one object in at the expense of the other between every
+ *     single byte.
+ *
+ * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
+ *     as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
+ *     object is too large for the available space (or simply too large
+ *     for the mappable aperture!), a view is created instead and faulted
+ *     into userspace. (This view is aligned and sized appropriately for
+ *     fenced access.)
+ *
+ * Restrictions:
+ *
+ *  * snoopable objects cannot be accessed via the GTT. It can cause machine
+ *    hangs on some architectures, corruption on others. An attempt to service
+ *    a GTT page fault from a snoopable object will generate a SIGBUS.
+ *
+ *  * the object must be able to fit into RAM (physical memory, though no
+ *    limited to the mappable aperture).
+ *
+ *
+ * Caveats:
+ *
+ *  * a new GTT page fault will synchronize rendering from the GPU and flush
+ *    all data to system memory. Subsequent access will not be synchronized.
+ *
+ *  * all mappings are revoked on runtime device suspend.
+ *
+ *  * there are only 8, 16 or 32 fence registers to share between all users
+ *    (older machines require fence register for display and blitter access
+ *    as well). Contention of the fence registers will cause the previous users
+ *    to be unmapped and any new access will generate new page faults.
+ *
+ *  * running out of memory while servicing a fault may generate a SIGBUS,
+ *    rather than the expected SIGSEGV.
+ */
+int i915_gem_mmap_gtt_version(void)
+{
+	return 1;
+}
+
 /**
  * i915_gem_fault - fault a page into the GTT
- * @vma: VMA in question
+ * @area: CPU VMA in question
  * @vmf: fault info
  *
  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
@@ -2001,122 +1745,120 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
  * from the GTT and/or fence registers to make room.  So performance may
  * suffer if the GTT working set is large or there are few fence registers
  * left.
+ *
+ * The current feature set supported by i915_gem_fault() and thus GTT mmaps
+ * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
  */
-int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
 {
-	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
+#define MIN_CHUNK_PAGES ((1 << 20) >> PAGE_SHIFT) /* 1 MiB */
+	struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
 	struct drm_device *dev = obj->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct i915_ggtt_view view = i915_ggtt_view_normal;
-	pgoff_t page_offset;
-	unsigned long pfn;
-	int ret = 0;
 	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
-
-	intel_runtime_pm_get(dev_priv);
+	struct i915_vma *vma;
+	pgoff_t page_offset;
+	unsigned int flags;
+	int ret;
 
 	/* We don't use vmf->pgoff since that has the fake offset */
-	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
+	page_offset = ((unsigned long)vmf->virtual_address - area->vm_start) >>
 		PAGE_SHIFT;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		goto out;
-
 	trace_i915_gem_object_fault(obj, page_offset, true, write);
 
 	/* Try to flush the object off the GPU first without holding the lock.
-	 * Upon reacquiring the lock, we will perform our sanity checks and then
+	 * Upon acquiring the lock, we will perform our sanity checks and then
 	 * repeat the flush holding the lock in the normal manner to catch cases
 	 * where we are gazumped.
 	 */
-	ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
+	ret = __unsafe_wait_rendering(obj, NULL, !write);
 	if (ret)
-		goto unlock;
+		goto err;
+
+	intel_runtime_pm_get(dev_priv);
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		goto err_rpm;
 
 	/* Access to snoopable pages through the GTT is incoherent. */
 	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
 		ret = -EFAULT;
-		goto unlock;
+		goto err_unlock;
 	}
 
-	/* Use a partial view if the object is bigger than the aperture. */
-	if (obj->base.size >= ggtt->mappable_end &&
-	    obj->tiling_mode == I915_TILING_NONE) {
-		static const unsigned int chunk_size = 256; // 1 MiB
+	/* If the object is smaller than a couple of partial vma, it is
+	 * not worth only creating a single partial vma - we may as well
+	 * clear enough space for the full object.
+	 */
+	flags = PIN_MAPPABLE;
+	if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT)
+		flags |= PIN_NONBLOCK | PIN_NONFAULT;
+
+	/* Now pin it into the GTT as needed */
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags);
+	if (IS_ERR(vma)) {
+		struct i915_ggtt_view view;
+		unsigned int chunk_size;
+
+		/* Use a partial view if it is bigger than available space */
+		chunk_size = MIN_CHUNK_PAGES;
+		if (i915_gem_object_is_tiled(obj))
+			chunk_size = max(chunk_size, tile_row_pages(obj));
 
 		memset(&view, 0, sizeof(view));
 		view.type = I915_GGTT_VIEW_PARTIAL;
 		view.params.partial.offset = rounddown(page_offset, chunk_size);
 		view.params.partial.size =
-			min_t(unsigned int,
-			      chunk_size,
-			      (vma->vm_end - vma->vm_start)/PAGE_SIZE -
+			min_t(unsigned int, chunk_size,
+			      (area->vm_end - area->vm_start) / PAGE_SIZE -
 			      view.params.partial.offset);
-	}
 
-	/* Now pin it into the GTT if needed */
-	ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
-	if (ret)
-		goto unlock;
+		/* If the partial covers the entire object, just create a
+		 * normal VMA.
+		 */
+		if (chunk_size >= obj->base.size >> PAGE_SHIFT)
+			view.type = I915_GGTT_VIEW_NORMAL;
+
+		/* Userspace is now writing through an untracked VMA, abandon
+		 * all hope that the hardware is able to track future writes.
+		 */
+		obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
+
+		vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
+	}
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
+		goto err_unlock;
+	}
 
 	ret = i915_gem_object_set_to_gtt_domain(obj, write);
 	if (ret)
-		goto unpin;
+		goto err_unpin;
 
-	ret = i915_gem_object_get_fence(obj);
+	ret = i915_vma_get_fence(vma);
 	if (ret)
-		goto unpin;
+		goto err_unpin;
 
 	/* Finally, remap it using the new GTT offset */
-	pfn = ggtt->mappable_base +
-		i915_gem_obj_ggtt_offset_view(obj, &view);
-	pfn >>= PAGE_SHIFT;
-
-	if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
-		/* Overriding existing pages in partial view does not cause
-		 * us any trouble as TLBs are still valid because the fault
-		 * is due to userspace losing part of the mapping or never
-		 * having accessed it before (at this partials' range).
-		 */
-		unsigned long base = vma->vm_start +
-				     (view.params.partial.offset << PAGE_SHIFT);
-		unsigned int i;
-
-		for (i = 0; i < view.params.partial.size; i++) {
-			ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
-			if (ret)
-				break;
-		}
-
-		obj->fault_mappable = true;
-	} else {
-		if (!obj->fault_mappable) {
-			unsigned long size = min_t(unsigned long,
-						   vma->vm_end - vma->vm_start,
-						   obj->base.size);
-			int i;
-
-			for (i = 0; i < size >> PAGE_SHIFT; i++) {
-				ret = vm_insert_pfn(vma,
-						    (unsigned long)vma->vm_start + i * PAGE_SIZE,
-						    pfn + i);
-				if (ret)
-					break;
-			}
+	ret = remap_io_mapping(area,
+			       area->vm_start + (vma->ggtt_view.params.partial.offset << PAGE_SHIFT),
+			       (ggtt->mappable_base + vma->node.start) >> PAGE_SHIFT,
+			       min_t(u64, vma->size, area->vm_end - area->vm_start),
+			       &ggtt->mappable);
+	if (ret)
+		goto err_unpin;
 
-			obj->fault_mappable = true;
-		} else
-			ret = vm_insert_pfn(vma,
-					    (unsigned long)vmf->virtual_address,
-					    pfn + page_offset);
-	}
-unpin:
-	i915_gem_object_ggtt_unpin_view(obj, &view);
-unlock:
+	obj->fault_mappable = true;
+err_unpin:
+	__i915_vma_unpin(vma);
+err_unlock:
 	mutex_unlock(&dev->struct_mutex);
-out:
+err_rpm:
+	intel_runtime_pm_put(dev_priv);
+err:
 	switch (ret) {
 	case -EIO:
 		/*
@@ -2157,8 +1899,6 @@ out:
 		ret = VM_FAULT_SIGBUS;
 		break;
 	}
-
-	intel_runtime_pm_put(dev_priv);
 	return ret;
 }
 
@@ -2212,46 +1952,58 @@ i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
 		i915_gem_release_mmap(obj);
 }
 
-uint32_t
-i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
+/**
+ * i915_gem_get_ggtt_size - return required global GTT size for an object
+ * @dev_priv: i915 device
+ * @size: object size
+ * @tiling_mode: tiling mode
+ *
+ * Return the required global GTT size for an object, taking into account
+ * potential fence register mapping.
+ */
+u64 i915_gem_get_ggtt_size(struct drm_i915_private *dev_priv,
+			   u64 size, int tiling_mode)
 {
-	uint32_t gtt_size;
+	u64 ggtt_size;
 
-	if (INTEL_INFO(dev)->gen >= 4 ||
+	GEM_BUG_ON(size == 0);
+
+	if (INTEL_GEN(dev_priv) >= 4 ||
 	    tiling_mode == I915_TILING_NONE)
 		return size;
 
 	/* Previous chips need a power-of-two fence region when tiling */
-	if (IS_GEN3(dev))
-		gtt_size = 1024*1024;
+	if (IS_GEN3(dev_priv))
+		ggtt_size = 1024*1024;
 	else
-		gtt_size = 512*1024;
+		ggtt_size = 512*1024;
 
-	while (gtt_size < size)
-		gtt_size <<= 1;
+	while (ggtt_size < size)
+		ggtt_size <<= 1;
 
-	return gtt_size;
+	return ggtt_size;
 }
 
 /**
- * i915_gem_get_gtt_alignment - return required GTT alignment for an object
- * @dev: drm device
+ * i915_gem_get_ggtt_alignment - return required global GTT alignment
+ * @dev_priv: i915 device
  * @size: object size
  * @tiling_mode: tiling mode
- * @fenced: is fenced alignemned required or not
+ * @fenced: is fenced alignment required or not
  *
- * Return the required GTT alignment for an object, taking into account
+ * Return the required global GTT alignment for an object, taking into account
  * potential fence register mapping.
  */
-uint32_t
-i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
-			   int tiling_mode, bool fenced)
+u64 i915_gem_get_ggtt_alignment(struct drm_i915_private *dev_priv, u64 size,
+				int tiling_mode, bool fenced)
 {
+	GEM_BUG_ON(size == 0);
+
 	/*
 	 * Minimum alignment is 4k (GTT page size), but might be greater
 	 * if a fence register is needed for the object.
 	 */
-	if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
+	if (INTEL_GEN(dev_priv) >= 4 || (!fenced && IS_G33(dev_priv)) ||
 	    tiling_mode == I915_TILING_NONE)
 		return 4096;
 
@@ -2259,42 +2011,34 @@ i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
 	 * Previous chips need to be aligned to the size of the smallest
 	 * fence register that can contain the object.
 	 */
-	return i915_gem_get_gtt_size(dev, size, tiling_mode);
+	return i915_gem_get_ggtt_size(dev_priv, size, tiling_mode);
 }
 
 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
 {
 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	int ret;
-
-	dev_priv->mm.shrinker_no_lock_stealing = true;
+	int err;
 
-	ret = drm_gem_create_mmap_offset(&obj->base);
-	if (ret != -ENOSPC)
-		goto out;
+	err = drm_gem_create_mmap_offset(&obj->base);
+	if (!err)
+		return 0;
 
-	/* Badly fragmented mmap space? The only way we can recover
-	 * space is by destroying unwanted objects. We can't randomly release
-	 * mmap_offsets as userspace expects them to be persistent for the
-	 * lifetime of the objects. The closest we can is to release the
-	 * offsets on purgeable objects by truncating it and marking it purged,
-	 * which prevents userspace from ever using that object again.
+	/* We can idle the GPU locklessly to flush stale objects, but in order
+	 * to claim that space for ourselves, we need to take the big
+	 * struct_mutex to free the requests+objects and allocate our slot.
 	 */
-	i915_gem_shrink(dev_priv,
-			obj->base.size >> PAGE_SHIFT,
-			I915_SHRINK_BOUND |
-			I915_SHRINK_UNBOUND |
-			I915_SHRINK_PURGEABLE);
-	ret = drm_gem_create_mmap_offset(&obj->base);
-	if (ret != -ENOSPC)
-		goto out;
+	err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE);
+	if (err)
+		return err;
 
-	i915_gem_shrink_all(dev_priv);
-	ret = drm_gem_create_mmap_offset(&obj->base);
-out:
-	dev_priv->mm.shrinker_no_lock_stealing = false;
+	err = i915_mutex_lock_interruptible(&dev_priv->drm);
+	if (!err) {
+		i915_gem_retire_requests(dev_priv);
+		err = drm_gem_create_mmap_offset(&obj->base);
+		mutex_unlock(&dev_priv->drm.struct_mutex);
+	}
 
-	return ret;
+	return err;
 }
 
 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
@@ -2311,32 +2055,15 @@ i915_gem_mmap_gtt(struct drm_file *file,
 	struct drm_i915_gem_object *obj;
 	int ret;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
-
-	obj = to_intel_bo(drm_gem_object_lookup(file, handle));
-	if (&obj->base == NULL) {
-		ret = -ENOENT;
-		goto unlock;
-	}
-
-	if (obj->madv != I915_MADV_WILLNEED) {
-		DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
-		ret = -EFAULT;
-		goto out;
-	}
+	obj = i915_gem_object_lookup(file, handle);
+	if (!obj)
+		return -ENOENT;
 
 	ret = i915_gem_object_create_mmap_offset(obj);
-	if (ret)
-		goto out;
+	if (ret == 0)
+		*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
 
-	*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
-
-out:
-	drm_gem_object_unreference(&obj->base);
-unlock:
-	mutex_unlock(&dev->struct_mutex);
+	i915_gem_object_put_unlocked(obj);
 	return ret;
 }
 
@@ -2454,7 +2181,7 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
 	if (obj->pages_pin_count)
 		return -EBUSY;
 
-	BUG_ON(i915_gem_obj_bound_any(obj));
+	GEM_BUG_ON(obj->bind_count);
 
 	/* ->put_pages might need to allocate memory for the bit17 swizzle
 	 * array, hence protect them from being reaped by removing them from gtt
@@ -2462,10 +2189,14 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
 	list_del(&obj->global_list);
 
 	if (obj->mapping) {
-		if (is_vmalloc_addr(obj->mapping))
-			vunmap(obj->mapping);
+		void *ptr;
+
+		ptr = ptr_mask_bits(obj->mapping);
+		if (is_vmalloc_addr(ptr))
+			vunmap(ptr);
 		else
-			kunmap(kmap_to_page(obj->mapping));
+			kunmap(kmap_to_page(ptr));
+
 		obj->mapping = NULL;
 	}
 
@@ -2574,7 +2305,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 	if (i915_gem_object_needs_bit17_swizzle(obj))
 		i915_gem_object_do_bit_17_swizzle(obj);
 
-	if (obj->tiling_mode != I915_TILING_NONE &&
+	if (i915_gem_object_is_tiled(obj) &&
 	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
 		i915_gem_object_pin_pages(obj);
 
@@ -2638,7 +2369,8 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
 }
 
 /* The 'mapping' part of i915_gem_object_pin_map() below */
-static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
+static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
+				 enum i915_map_type type)
 {
 	unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
 	struct sg_table *sgt = obj->pages;
@@ -2647,10 +2379,11 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
 	struct page *stack_pages[32];
 	struct page **pages = stack_pages;
 	unsigned long i = 0;
+	pgprot_t pgprot;
 	void *addr;
 
 	/* A single page can always be kmapped */
-	if (n_pages == 1)
+	if (n_pages == 1 && type == I915_MAP_WB)
 		return kmap(sg_page(sgt->sgl));
 
 	if (n_pages > ARRAY_SIZE(stack_pages)) {
@@ -2666,7 +2399,15 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
 	/* Check that we have the expected number of pages */
 	GEM_BUG_ON(i != n_pages);
 
-	addr = vmap(pages, n_pages, 0, PAGE_KERNEL);
+	switch (type) {
+	case I915_MAP_WB:
+		pgprot = PAGE_KERNEL;
+		break;
+	case I915_MAP_WC:
+		pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
+		break;
+	}
+	addr = vmap(pages, n_pages, 0, pgprot);
 
 	if (pages != stack_pages)
 		drm_free_large(pages);
@@ -2675,276 +2416,89 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
 }
 
 /* get, pin, and map the pages of the object into kernel space */
-void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
+void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
+			      enum i915_map_type type)
 {
+	enum i915_map_type has_type;
+	bool pinned;
+	void *ptr;
 	int ret;
 
 	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
 
 	ret = i915_gem_object_get_pages(obj);
 	if (ret)
 		return ERR_PTR(ret);
 
 	i915_gem_object_pin_pages(obj);
+	pinned = obj->pages_pin_count > 1;
 
-	if (!obj->mapping) {
-		obj->mapping = i915_gem_object_map(obj);
-		if (!obj->mapping) {
-			i915_gem_object_unpin_pages(obj);
-			return ERR_PTR(-ENOMEM);
+	ptr = ptr_unpack_bits(obj->mapping, has_type);
+	if (ptr && has_type != type) {
+		if (pinned) {
+			ret = -EBUSY;
+			goto err;
 		}
-	}
 
-	return obj->mapping;
-}
+		if (is_vmalloc_addr(ptr))
+			vunmap(ptr);
+		else
+			kunmap(kmap_to_page(ptr));
 
-void i915_vma_move_to_active(struct i915_vma *vma,
-			     struct drm_i915_gem_request *req)
-{
-	struct drm_i915_gem_object *obj = vma->obj;
-	struct intel_engine_cs *engine;
+		ptr = obj->mapping = NULL;
+	}
 
-	engine = i915_gem_request_get_engine(req);
+	if (!ptr) {
+		ptr = i915_gem_object_map(obj, type);
+		if (!ptr) {
+			ret = -ENOMEM;
+			goto err;
+		}
 
-	/* Add a reference if we're newly entering the active list. */
-	if (obj->active == 0)
-		drm_gem_object_reference(&obj->base);
-	obj->active |= intel_engine_flag(engine);
+		obj->mapping = ptr_pack_bits(ptr, type);
+	}
 
-	list_move_tail(&obj->engine_list[engine->id], &engine->active_list);
-	i915_gem_request_assign(&obj->last_read_req[engine->id], req);
+	return ptr;
 
-	list_move_tail(&vma->vm_link, &vma->vm->active_list);
+err:
+	i915_gem_object_unpin_pages(obj);
+	return ERR_PTR(ret);
 }
 
 static void
-i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
+i915_gem_object_retire__write(struct i915_gem_active *active,
+			      struct drm_i915_gem_request *request)
 {
-	GEM_BUG_ON(obj->last_write_req == NULL);
-	GEM_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine)));
+	struct drm_i915_gem_object *obj =
+		container_of(active, struct drm_i915_gem_object, last_write);
 
-	i915_gem_request_assign(&obj->last_write_req, NULL);
 	intel_fb_obj_flush(obj, true, ORIGIN_CS);
 }
 
 static void
-i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
+i915_gem_object_retire__read(struct i915_gem_active *active,
+			     struct drm_i915_gem_request *request)
 {
-	struct i915_vma *vma;
-
-	GEM_BUG_ON(obj->last_read_req[ring] == NULL);
-	GEM_BUG_ON(!(obj->active & (1 << ring)));
-
-	list_del_init(&obj->engine_list[ring]);
-	i915_gem_request_assign(&obj->last_read_req[ring], NULL);
+	int idx = request->engine->id;
+	struct drm_i915_gem_object *obj =
+		container_of(active, struct drm_i915_gem_object, last_read[idx]);
 
-	if (obj->last_write_req && obj->last_write_req->engine->id == ring)
-		i915_gem_object_retire__write(obj);
+	GEM_BUG_ON(!i915_gem_object_has_active_engine(obj, idx));
 
-	obj->active &= ~(1 << ring);
-	if (obj->active)
+	i915_gem_object_clear_active(obj, idx);
+	if (i915_gem_object_is_active(obj))
 		return;
 
 	/* Bump our place on the bound list to keep it roughly in LRU order
 	 * so that we don't steal from recently used but inactive objects
 	 * (unless we are forced to ofc!)
 	 */
-	list_move_tail(&obj->global_list,
-		       &to_i915(obj->base.dev)->mm.bound_list);
-
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (!list_empty(&vma->vm_link))
-			list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
-	}
+	if (obj->bind_count)
+		list_move_tail(&obj->global_list,
+			       &request->i915->mm.bound_list);
 
-	i915_gem_request_assign(&obj->last_fenced_req, NULL);
-	drm_gem_object_unreference(&obj->base);
-}
-
-static int
-i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
-{
-	struct intel_engine_cs *engine;
-	int ret;
-
-	/* Carefully retire all requests without writing to the rings */
-	for_each_engine(engine, dev_priv) {
-		ret = intel_engine_idle(engine);
-		if (ret)
-			return ret;
-	}
-	i915_gem_retire_requests(dev_priv);
-
-	/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
-	if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) {
-		while (intel_kick_waiters(dev_priv) ||
-		       intel_kick_signalers(dev_priv))
-			yield();
-	}
-
-	/* Finally reset hw state */
-	for_each_engine(engine, dev_priv)
-		intel_ring_init_seqno(engine, seqno);
-
-	return 0;
-}
-
-int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret;
-
-	if (seqno == 0)
-		return -EINVAL;
-
-	/* HWS page needs to be set less than what we
-	 * will inject to ring
-	 */
-	ret = i915_gem_init_seqno(dev_priv, seqno - 1);
-	if (ret)
-		return ret;
-
-	/* Carefully set the last_seqno value so that wrap
-	 * detection still works
-	 */
-	dev_priv->next_seqno = seqno;
-	dev_priv->last_seqno = seqno - 1;
-	if (dev_priv->last_seqno == 0)
-		dev_priv->last_seqno--;
-
-	return 0;
-}
-
-int
-i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
-{
-	/* reserve 0 for non-seqno */
-	if (dev_priv->next_seqno == 0) {
-		int ret = i915_gem_init_seqno(dev_priv, 0);
-		if (ret)
-			return ret;
-
-		dev_priv->next_seqno = 1;
-	}
-
-	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
-	return 0;
-}
-
-static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	dev_priv->gt.active_engines |= intel_engine_flag(engine);
-	if (dev_priv->gt.awake)
-		return;
-
-	intel_runtime_pm_get_noresume(dev_priv);
-	dev_priv->gt.awake = true;
-
-	i915_update_gfx_val(dev_priv);
-	if (INTEL_GEN(dev_priv) >= 6)
-		gen6_rps_busy(dev_priv);
-
-	queue_delayed_work(dev_priv->wq,
-			   &dev_priv->gt.retire_work,
-			   round_jiffies_up_relative(HZ));
-}
-
-/*
- * NB: This function is not allowed to fail. Doing so would mean the the
- * request is not being tracked for completion but the work itself is
- * going to happen on the hardware. This would be a Bad Thing(tm).
- */
-void __i915_add_request(struct drm_i915_gem_request *request,
-			struct drm_i915_gem_object *obj,
-			bool flush_caches)
-{
-	struct intel_engine_cs *engine;
-	struct intel_ringbuffer *ringbuf;
-	u32 request_start;
-	u32 reserved_tail;
-	int ret;
-
-	if (WARN_ON(request == NULL))
-		return;
-
-	engine = request->engine;
-	ringbuf = request->ringbuf;
-
-	/*
-	 * To ensure that this call will not fail, space for its emissions
-	 * should already have been reserved in the ring buffer. Let the ring
-	 * know that it is time to use that space up.
-	 */
-	request_start = intel_ring_get_tail(ringbuf);
-	reserved_tail = request->reserved_space;
-	request->reserved_space = 0;
-
-	/*
-	 * Emit any outstanding flushes - execbuf can fail to emit the flush
-	 * after having emitted the batchbuffer command. Hence we need to fix
-	 * things up similar to emitting the lazy request. The difference here
-	 * is that the flush _must_ happen before the next request, no matter
-	 * what.
-	 */
-	if (flush_caches) {
-		if (i915.enable_execlists)
-			ret = logical_ring_flush_all_caches(request);
-		else
-			ret = intel_ring_flush_all_caches(request);
-		/* Not allowed to fail! */
-		WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
-	}
-
-	trace_i915_gem_request_add(request);
-
-	request->head = request_start;
-
-	/* Whilst this request exists, batch_obj will be on the
-	 * active_list, and so will hold the active reference. Only when this
-	 * request is retired will the the batch_obj be moved onto the
-	 * inactive_list and lose its active reference. Hence we do not need
-	 * to explicitly hold another reference here.
-	 */
-	request->batch_obj = obj;
-
-	/* Seal the request and mark it as pending execution. Note that
-	 * we may inspect this state, without holding any locks, during
-	 * hangcheck. Hence we apply the barrier to ensure that we do not
-	 * see a more recent value in the hws than we are tracking.
-	 */
-	request->emitted_jiffies = jiffies;
-	request->previous_seqno = engine->last_submitted_seqno;
-	smp_store_mb(engine->last_submitted_seqno, request->seqno);
-	list_add_tail(&request->list, &engine->request_list);
-
-	/* Record the position of the start of the request so that
-	 * should we detect the updated seqno part-way through the
-	 * GPU processing the request, we never over-estimate the
-	 * position of the head.
-	 */
-	request->postfix = intel_ring_get_tail(ringbuf);
-
-	if (i915.enable_execlists)
-		ret = engine->emit_request(request);
-	else {
-		ret = engine->add_request(request);
-
-		request->tail = intel_ring_get_tail(ringbuf);
-	}
-	/* Not allowed to fail! */
-	WARN(ret, "emit|add_request failed: %d!\n", ret);
-	/* Sanity check that the reserved size was large enough. */
-	ret = intel_ring_get_tail(ringbuf) - request_start;
-	if (ret < 0)
-		ret += ringbuf->size;
-	WARN_ONCE(ret > reserved_tail,
-		  "Not enough space reserved (%d bytes) "
-		  "for adding the request (%d bytes)\n",
-		  reserved_tail, ret);
-
-	i915_gem_mark_busy(engine);
+	i915_gem_object_put(obj);
 }
 
 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
@@ -2978,101 +2532,6 @@ static void i915_set_reset_status(struct i915_gem_context *ctx,
 	}
 }
 
-void i915_gem_request_free(struct kref *req_ref)
-{
-	struct drm_i915_gem_request *req = container_of(req_ref,
-						 typeof(*req), ref);
-	kmem_cache_free(req->i915->requests, req);
-}
-
-static inline int
-__i915_gem_request_alloc(struct intel_engine_cs *engine,
-			 struct i915_gem_context *ctx,
-			 struct drm_i915_gem_request **req_out)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error);
-	struct drm_i915_gem_request *req;
-	int ret;
-
-	if (!req_out)
-		return -EINVAL;
-
-	*req_out = NULL;
-
-	/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
-	 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
-	 * and restart.
-	 */
-	ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible);
-	if (ret)
-		return ret;
-
-	req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
-	if (req == NULL)
-		return -ENOMEM;
-
-	ret = i915_gem_get_seqno(engine->i915, &req->seqno);
-	if (ret)
-		goto err;
-
-	kref_init(&req->ref);
-	req->i915 = dev_priv;
-	req->engine = engine;
-	req->ctx  = ctx;
-	i915_gem_context_reference(req->ctx);
-
-	/*
-	 * Reserve space in the ring buffer for all the commands required to
-	 * eventually emit this request. This is to guarantee that the
-	 * i915_add_request() call can't fail. Note that the reserve may need
-	 * to be redone if the request is not actually submitted straight
-	 * away, e.g. because a GPU scheduler has deferred it.
-	 */
-	req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
-
-	if (i915.enable_execlists)
-		ret = intel_logical_ring_alloc_request_extras(req);
-	else
-		ret = intel_ring_alloc_request_extras(req);
-	if (ret)
-		goto err_ctx;
-
-	*req_out = req;
-	return 0;
-
-err_ctx:
-	i915_gem_context_unreference(ctx);
-err:
-	kmem_cache_free(dev_priv->requests, req);
-	return ret;
-}
-
-/**
- * i915_gem_request_alloc - allocate a request structure
- *
- * @engine: engine that we wish to issue the request on.
- * @ctx: context that the request will be associated with.
- *       This can be NULL if the request is not directly related to
- *       any specific user context, in which case this function will
- *       choose an appropriate context to use.
- *
- * Returns a pointer to the allocated request if successful,
- * or an error code if not.
- */
-struct drm_i915_gem_request *
-i915_gem_request_alloc(struct intel_engine_cs *engine,
-		       struct i915_gem_context *ctx)
-{
-	struct drm_i915_gem_request *req;
-	int err;
-
-	if (ctx == NULL)
-		ctx = engine->i915->kernel_context;
-	err = __i915_gem_request_alloc(engine, ctx, &req);
-	return err ? ERR_PTR(err) : req;
-}
-
 struct drm_i915_gem_request *
 i915_gem_find_active_request(struct intel_engine_cs *engine)
 {
@@ -3086,182 +2545,143 @@ i915_gem_find_active_request(struct intel_engine_cs *engine)
 	 * extra delay for a recent interrupt is pointless. Hence, we do
 	 * not need an engine->irq_seqno_barrier() before the seqno reads.
 	 */
-	list_for_each_entry(request, &engine->request_list, list) {
+	list_for_each_entry(request, &engine->request_list, link) {
 		if (i915_gem_request_completed(request))
 			continue;
 
+		if (!i915_sw_fence_done(&request->submit))
+			break;
+
 		return request;
 	}
 
 	return NULL;
 }
 
-static void i915_gem_reset_engine_status(struct intel_engine_cs *engine)
+static void reset_request(struct drm_i915_gem_request *request)
+{
+	void *vaddr = request->ring->vaddr;
+	u32 head;
+
+	/* As this request likely depends on state from the lost
+	 * context, clear out all the user operations leaving the
+	 * breadcrumb at the end (so we get the fence notifications).
+	 */
+	head = request->head;
+	if (request->postfix < head) {
+		memset(vaddr + head, 0, request->ring->size - head);
+		head = 0;
+	}
+	memset(vaddr + head, 0, request->postfix - head);
+}
+
+static void i915_gem_reset_engine(struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_request *request;
+	struct i915_gem_context *incomplete_ctx;
 	bool ring_hung;
 
+	/* Ensure irq handler finishes, and not run again. */
+	tasklet_kill(&engine->irq_tasklet);
+	if (engine->irq_seqno_barrier)
+		engine->irq_seqno_barrier(engine);
+
 	request = i915_gem_find_active_request(engine);
-	if (request == NULL)
+	if (!request)
 		return;
 
 	ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
-
 	i915_set_reset_status(request->ctx, ring_hung);
-	list_for_each_entry_continue(request, &engine->request_list, list)
-		i915_set_reset_status(request->ctx, false);
-}
-
-static void i915_gem_reset_engine_cleanup(struct intel_engine_cs *engine)
-{
-	struct intel_ringbuffer *buffer;
-
-	while (!list_empty(&engine->active_list)) {
-		struct drm_i915_gem_object *obj;
-
-		obj = list_first_entry(&engine->active_list,
-				       struct drm_i915_gem_object,
-				       engine_list[engine->id]);
-
-		i915_gem_object_retire__read(obj, engine->id);
-	}
-
-	/*
-	 * Clear the execlists queue up before freeing the requests, as those
-	 * are the ones that keep the context and ringbuffer backing objects
-	 * pinned in place.
-	 */
-
-	if (i915.enable_execlists) {
-		/* Ensure irq handler finishes or is cancelled. */
-		tasklet_kill(&engine->irq_tasklet);
-
-		intel_execlists_cancel_requests(engine);
-	}
-
-	/*
-	 * We must free the requests after all the corresponding objects have
-	 * been moved off active lists. Which is the same order as the normal
-	 * retire_requests function does. This is important if object hold
-	 * implicit references on things like e.g. ppgtt address spaces through
-	 * the request.
-	 */
-	while (!list_empty(&engine->request_list)) {
-		struct drm_i915_gem_request *request;
+	if (!ring_hung)
+		return;
 
-		request = list_first_entry(&engine->request_list,
-					   struct drm_i915_gem_request,
-					   list);
+	DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
+			 engine->name, request->fence.seqno);
 
-		i915_gem_request_retire(request);
-	}
+	/* Setup the CS to resume from the breadcrumb of the hung request */
+	engine->reset_hw(engine, request);
 
-	/* Having flushed all requests from all queues, we know that all
-	 * ringbuffers must now be empty. However, since we do not reclaim
-	 * all space when retiring the request (to prevent HEADs colliding
-	 * with rapid ringbuffer wraparound) the amount of available space
-	 * upon reset is less than when we start. Do one more pass over
-	 * all the ringbuffers to reset last_retired_head.
+	/* Users of the default context do not rely on logical state
+	 * preserved between batches. They have to emit full state on
+	 * every batch and so it is safe to execute queued requests following
+	 * the hang.
+	 *
+	 * Other contexts preserve state, now corrupt. We want to skip all
+	 * queued requests that reference the corrupt context.
 	 */
-	list_for_each_entry(buffer, &engine->buffers, link) {
-		buffer->last_retired_head = buffer->tail;
-		intel_ring_update_space(buffer);
-	}
+	incomplete_ctx = request->ctx;
+	if (i915_gem_context_is_default(incomplete_ctx))
+		return;
 
-	intel_ring_init_seqno(engine, engine->last_submitted_seqno);
+	list_for_each_entry_continue(request, &engine->request_list, link)
+		if (request->ctx == incomplete_ctx)
+			reset_request(request);
 }
 
-void i915_gem_reset(struct drm_device *dev)
+void i915_gem_reset(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_engine_cs *engine;
 
-	/*
-	 * Before we free the objects from the requests, we need to inspect
-	 * them for finding the guilty party. As the requests only borrow
-	 * their reference to the objects, the inspection must be done first.
-	 */
-	for_each_engine(engine, dev_priv)
-		i915_gem_reset_engine_status(engine);
+	i915_gem_retire_requests(dev_priv);
 
 	for_each_engine(engine, dev_priv)
-		i915_gem_reset_engine_cleanup(engine);
+		i915_gem_reset_engine(engine);
 
-	i915_gem_context_reset(dev);
+	i915_gem_restore_fences(&dev_priv->drm);
 
-	i915_gem_restore_fences(dev);
+	if (dev_priv->gt.awake) {
+		intel_sanitize_gt_powersave(dev_priv);
+		intel_enable_gt_powersave(dev_priv);
+		if (INTEL_GEN(dev_priv) >= 6)
+			gen6_rps_busy(dev_priv);
+	}
+}
 
-	WARN_ON(i915_verify_lists(dev));
+static void nop_submit_request(struct drm_i915_gem_request *request)
+{
 }
 
-/**
- * This function clears the request list as sequence numbers are passed.
- * @engine: engine to retire requests on
- */
-void
-i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
+static void i915_gem_cleanup_engine(struct intel_engine_cs *engine)
 {
-	WARN_ON(i915_verify_lists(engine->dev));
+	engine->submit_request = nop_submit_request;
 
-	/* Retire requests first as we use it above for the early return.
-	 * If we retire requests last, we may use a later seqno and so clear
-	 * the requests lists without clearing the active list, leading to
-	 * confusion.
+	/* Mark all pending requests as complete so that any concurrent
+	 * (lockless) lookup doesn't try and wait upon the request as we
+	 * reset it.
 	 */
-	while (!list_empty(&engine->request_list)) {
-		struct drm_i915_gem_request *request;
-
-		request = list_first_entry(&engine->request_list,
-					   struct drm_i915_gem_request,
-					   list);
+	intel_engine_init_seqno(engine, engine->last_submitted_seqno);
 
-		if (!i915_gem_request_completed(request))
-			break;
-
-		i915_gem_request_retire(request);
-	}
-
-	/* Move any buffers on the active list that are no longer referenced
-	 * by the ringbuffer to the flushing/inactive lists as appropriate,
-	 * before we free the context associated with the requests.
+	/*
+	 * Clear the execlists queue up before freeing the requests, as those
+	 * are the ones that keep the context and ringbuffer backing objects
+	 * pinned in place.
 	 */
-	while (!list_empty(&engine->active_list)) {
-		struct drm_i915_gem_object *obj;
 
-		obj = list_first_entry(&engine->active_list,
-				       struct drm_i915_gem_object,
-				       engine_list[engine->id]);
-
-		if (!list_empty(&obj->last_read_req[engine->id]->list))
-			break;
-
-		i915_gem_object_retire__read(obj, engine->id);
+	if (i915.enable_execlists) {
+		spin_lock(&engine->execlist_lock);
+		INIT_LIST_HEAD(&engine->execlist_queue);
+		i915_gem_request_put(engine->execlist_port[0].request);
+		i915_gem_request_put(engine->execlist_port[1].request);
+		memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
+		spin_unlock(&engine->execlist_lock);
 	}
 
-	WARN_ON(i915_verify_lists(engine->dev));
+	engine->i915->gt.active_engines &= ~intel_engine_flag(engine);
 }
 
-void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
+void i915_gem_set_wedged(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+	set_bit(I915_WEDGED, &dev_priv->gpu_error.flags);
 
-	if (dev_priv->gt.active_engines == 0)
-		return;
-
-	GEM_BUG_ON(!dev_priv->gt.awake);
-
-	for_each_engine(engine, dev_priv) {
-		i915_gem_retire_requests_ring(engine);
-		if (list_empty(&engine->request_list))
-			dev_priv->gt.active_engines &= ~intel_engine_flag(engine);
-	}
+	i915_gem_context_lost(dev_priv);
+	for_each_engine(engine, dev_priv)
+		i915_gem_cleanup_engine(engine);
+	mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
 
-	if (dev_priv->gt.active_engines == 0)
-		queue_delayed_work(dev_priv->wq,
-				   &dev_priv->gt.idle_work,
-				   msecs_to_jiffies(100));
+	i915_gem_retire_requests(dev_priv);
 }
 
 static void
@@ -3281,10 +2701,12 @@ i915_gem_retire_work_handler(struct work_struct *work)
 	 * We do not need to do this test under locking as in the worst-case
 	 * we queue the retire worker once too often.
 	 */
-	if (READ_ONCE(dev_priv->gt.awake))
+	if (READ_ONCE(dev_priv->gt.awake)) {
+		i915_queue_hangcheck(dev_priv);
 		queue_delayed_work(dev_priv->wq,
 				   &dev_priv->gt.retire_work,
 				   round_jiffies_up_relative(HZ));
+	}
 }
 
 static void
@@ -3294,7 +2716,6 @@ i915_gem_idle_work_handler(struct work_struct *work)
 		container_of(work, typeof(*dev_priv), gt.idle_work.work);
 	struct drm_device *dev = &dev_priv->drm;
 	struct intel_engine_cs *engine;
-	unsigned int stuck_engines;
 	bool rearm_hangcheck;
 
 	if (!READ_ONCE(dev_priv->gt.awake))
@@ -3324,12 +2745,6 @@ i915_gem_idle_work_handler(struct work_struct *work)
 	dev_priv->gt.awake = false;
 	rearm_hangcheck = false;
 
-	stuck_engines = intel_kick_waiters(dev_priv);
-	if (unlikely(stuck_engines)) {
-		DRM_DEBUG_DRIVER("kicked stuck waiters...missed irq\n");
-		dev_priv->gpu_error.missed_irq_rings |= stuck_engines;
-	}
-
 	if (INTEL_GEN(dev_priv) >= 6)
 		gen6_rps_idle(dev_priv);
 	intel_runtime_pm_put(dev_priv);
@@ -3343,32 +2758,17 @@ out_rearm:
 	}
 }
 
-/**
- * Ensures that an object will eventually get non-busy by flushing any required
- * write domains, emitting any outstanding lazy request and retiring and
- * completed requests.
- * @obj: object to flush
- */
-static int
-i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
+void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
 {
-	int i;
-
-	if (!obj->active)
-		return 0;
-
-	for (i = 0; i < I915_NUM_ENGINES; i++) {
-		struct drm_i915_gem_request *req;
-
-		req = obj->last_read_req[i];
-		if (req == NULL)
-			continue;
-
-		if (i915_gem_request_completed(req))
-			i915_gem_object_retire__read(obj, i);
-	}
+	struct drm_i915_gem_object *obj = to_intel_bo(gem);
+	struct drm_i915_file_private *fpriv = file->driver_priv;
+	struct i915_vma *vma, *vn;
 
-	return 0;
+	mutex_lock(&obj->base.dev->struct_mutex);
+	list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
+		if (vma->vm->file == fpriv)
+			i915_vma_close(vma);
+	mutex_unlock(&obj->base.dev->struct_mutex);
 }
 
 /**
@@ -3399,219 +2799,35 @@ int
 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
 	struct drm_i915_gem_wait *args = data;
+	struct intel_rps_client *rps = to_rps_client(file);
 	struct drm_i915_gem_object *obj;
-	struct drm_i915_gem_request *req[I915_NUM_ENGINES];
-	int i, n = 0;
-	int ret;
+	unsigned long active;
+	int idx, ret = 0;
 
 	if (args->flags != 0)
 		return -EINVAL;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
-
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->bo_handle));
-	if (&obj->base == NULL) {
-		mutex_unlock(&dev->struct_mutex);
+	obj = i915_gem_object_lookup(file, args->bo_handle);
+	if (!obj)
 		return -ENOENT;
-	}
-
-	/* Need to make sure the object gets inactive eventually. */
-	ret = i915_gem_object_flush_active(obj);
-	if (ret)
-		goto out;
-
-	if (!obj->active)
-		goto out;
-
-	/* Do this after OLR check to make sure we make forward progress polling
-	 * on this IOCTL with a timeout == 0 (like busy ioctl)
-	 */
-	if (args->timeout_ns == 0) {
-		ret = -ETIME;
-		goto out;
-	}
 
-	drm_gem_object_unreference(&obj->base);
-
-	for (i = 0; i < I915_NUM_ENGINES; i++) {
-		if (obj->last_read_req[i] == NULL)
-			continue;
-
-		req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
-	}
-
-	mutex_unlock(&dev->struct_mutex);
-
-	for (i = 0; i < n; i++) {
-		if (ret == 0)
-			ret = __i915_wait_request(req[i], true,
-						  args->timeout_ns > 0 ? &args->timeout_ns : NULL,
-						  to_rps_client(file));
-		i915_gem_request_unreference(req[i]);
-	}
-	return ret;
-
-out:
-	drm_gem_object_unreference(&obj->base);
-	mutex_unlock(&dev->struct_mutex);
-	return ret;
-}
-
-static int
-__i915_gem_object_sync(struct drm_i915_gem_object *obj,
-		       struct intel_engine_cs *to,
-		       struct drm_i915_gem_request *from_req,
-		       struct drm_i915_gem_request **to_req)
-{
-	struct intel_engine_cs *from;
-	int ret;
-
-	from = i915_gem_request_get_engine(from_req);
-	if (to == from)
-		return 0;
-
-	if (i915_gem_request_completed(from_req))
-		return 0;
-
-	if (!i915_semaphore_is_enabled(to_i915(obj->base.dev))) {
-		struct drm_i915_private *i915 = to_i915(obj->base.dev);
-		ret = __i915_wait_request(from_req,
-					  i915->mm.interruptible,
-					  NULL,
-					  &i915->rps.semaphores);
-		if (ret)
-			return ret;
-
-		i915_gem_object_retire_request(obj, from_req);
-	} else {
-		int idx = intel_ring_sync_index(from, to);
-		u32 seqno = i915_gem_request_get_seqno(from_req);
-
-		WARN_ON(!to_req);
-
-		if (seqno <= from->semaphore.sync_seqno[idx])
-			return 0;
-
-		if (*to_req == NULL) {
-			struct drm_i915_gem_request *req;
-
-			req = i915_gem_request_alloc(to, NULL);
-			if (IS_ERR(req))
-				return PTR_ERR(req);
-
-			*to_req = req;
-		}
-
-		trace_i915_gem_ring_sync_to(*to_req, from, from_req);
-		ret = to->semaphore.sync_to(*to_req, from, seqno);
-		if (ret)
-			return ret;
-
-		/* We use last_read_req because sync_to()
-		 * might have just caused seqno wrap under
-		 * the radar.
-		 */
-		from->semaphore.sync_seqno[idx] =
-			i915_gem_request_get_seqno(obj->last_read_req[from->id]);
-	}
-
-	return 0;
-}
-
-/**
- * i915_gem_object_sync - sync an object to a ring.
- *
- * @obj: object which may be in use on another ring.
- * @to: ring we wish to use the object on. May be NULL.
- * @to_req: request we wish to use the object for. See below.
- *          This will be allocated and returned if a request is
- *          required but not passed in.
- *
- * This code is meant to abstract object synchronization with the GPU.
- * Calling with NULL implies synchronizing the object with the CPU
- * rather than a particular GPU ring. Conceptually we serialise writes
- * between engines inside the GPU. We only allow one engine to write
- * into a buffer at any time, but multiple readers. To ensure each has
- * a coherent view of memory, we must:
- *
- * - If there is an outstanding write request to the object, the new
- *   request must wait for it to complete (either CPU or in hw, requests
- *   on the same ring will be naturally ordered).
- *
- * - If we are a write request (pending_write_domain is set), the new
- *   request must wait for outstanding read requests to complete.
- *
- * For CPU synchronisation (NULL to) no request is required. For syncing with
- * rings to_req must be non-NULL. However, a request does not have to be
- * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
- * request will be allocated automatically and returned through *to_req. Note
- * that it is not guaranteed that commands will be emitted (because the system
- * might already be idle). Hence there is no need to create a request that
- * might never have any work submitted. Note further that if a request is
- * returned in *to_req, it is the responsibility of the caller to submit
- * that request (after potentially adding more work to it).
- *
- * Returns 0 if successful, else propagates up the lower layer error.
- */
-int
-i915_gem_object_sync(struct drm_i915_gem_object *obj,
-		     struct intel_engine_cs *to,
-		     struct drm_i915_gem_request **to_req)
-{
-	const bool readonly = obj->base.pending_write_domain == 0;
-	struct drm_i915_gem_request *req[I915_NUM_ENGINES];
-	int ret, i, n;
-
-	if (!obj->active)
-		return 0;
-
-	if (to == NULL)
-		return i915_gem_object_wait_rendering(obj, readonly);
-
-	n = 0;
-	if (readonly) {
-		if (obj->last_write_req)
-			req[n++] = obj->last_write_req;
-	} else {
-		for (i = 0; i < I915_NUM_ENGINES; i++)
-			if (obj->last_read_req[i])
-				req[n++] = obj->last_read_req[i];
-	}
-	for (i = 0; i < n; i++) {
-		ret = __i915_gem_object_sync(obj, to, req[i], to_req);
+	active = __I915_BO_ACTIVE(obj);
+	for_each_active(active, idx) {
+		s64 *timeout = args->timeout_ns >= 0 ? &args->timeout_ns : NULL;
+		ret = i915_gem_active_wait_unlocked(&obj->last_read[idx],
+						    I915_WAIT_INTERRUPTIBLE,
+						    timeout, rps);
 		if (ret)
-			return ret;
+			break;
 	}
 
-	return 0;
-}
-
-static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
-{
-	u32 old_write_domain, old_read_domains;
-
-	/* Force a pagefault for domain tracking on next user access */
-	i915_gem_release_mmap(obj);
-
-	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
-		return;
-
-	old_read_domains = obj->base.read_domains;
-	old_write_domain = obj->base.write_domain;
-
-	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
-	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
-
-	trace_i915_gem_object_change_domain(obj,
-					    old_read_domains,
-					    old_write_domain);
+	i915_gem_object_put_unlocked(obj);
+	return ret;
 }
 
 static void __i915_vma_iounmap(struct i915_vma *vma)
 {
-	GEM_BUG_ON(vma->pin_count);
+	GEM_BUG_ON(i915_vma_is_pinned(vma));
 
 	if (vma->iomap == NULL)
 		return;
@@ -3620,65 +2836,83 @@ static void __i915_vma_iounmap(struct i915_vma *vma)
 	vma->iomap = NULL;
 }
 
-static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
+int i915_vma_unbind(struct i915_vma *vma)
 {
 	struct drm_i915_gem_object *obj = vma->obj;
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+	unsigned long active;
 	int ret;
 
-	if (list_empty(&vma->obj_link))
-		return 0;
-
-	if (!drm_mm_node_allocated(&vma->node)) {
-		i915_gem_vma_destroy(vma);
-		return 0;
-	}
-
-	if (vma->pin_count)
-		return -EBUSY;
+	/* First wait upon any activity as retiring the request may
+	 * have side-effects such as unpinning or even unbinding this vma.
+	 */
+	active = i915_vma_get_active(vma);
+	if (active) {
+		int idx;
+
+		/* When a closed VMA is retired, it is unbound - eek.
+		 * In order to prevent it from being recursively closed,
+		 * take a pin on the vma so that the second unbind is
+		 * aborted.
+		 */
+		__i915_vma_pin(vma);
 
-	BUG_ON(obj->pages == NULL);
+		for_each_active(active, idx) {
+			ret = i915_gem_active_retire(&vma->last_read[idx],
+						   &vma->vm->dev->struct_mutex);
+			if (ret)
+				break;
+		}
 
-	if (wait) {
-		ret = i915_gem_object_wait_rendering(obj, false);
+		__i915_vma_unpin(vma);
 		if (ret)
 			return ret;
+
+		GEM_BUG_ON(i915_vma_is_active(vma));
 	}
 
-	if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
-		i915_gem_object_finish_gtt(obj);
+	if (i915_vma_is_pinned(vma))
+		return -EBUSY;
+
+	if (!drm_mm_node_allocated(&vma->node))
+		goto destroy;
+
+	GEM_BUG_ON(obj->bind_count == 0);
+	GEM_BUG_ON(!obj->pages);
 
+	if (i915_vma_is_map_and_fenceable(vma)) {
 		/* release the fence reg _after_ flushing */
-		ret = i915_gem_object_put_fence(obj);
+		ret = i915_vma_put_fence(vma);
 		if (ret)
 			return ret;
 
+		/* Force a pagefault for domain tracking on next user access */
+		i915_gem_release_mmap(obj);
+
 		__i915_vma_iounmap(vma);
+		vma->flags &= ~I915_VMA_CAN_FENCE;
 	}
 
-	trace_i915_vma_unbind(vma);
-
-	vma->vm->unbind_vma(vma);
-	vma->bound = 0;
-
-	list_del_init(&vma->vm_link);
-	if (vma->is_ggtt) {
-		if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
-			obj->map_and_fenceable = false;
-		} else if (vma->ggtt_view.pages) {
-			sg_free_table(vma->ggtt_view.pages);
-			kfree(vma->ggtt_view.pages);
-		}
-		vma->ggtt_view.pages = NULL;
+	if (likely(!vma->vm->closed)) {
+		trace_i915_vma_unbind(vma);
+		vma->vm->unbind_vma(vma);
 	}
+	vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
 
 	drm_mm_remove_node(&vma->node);
-	i915_gem_vma_destroy(vma);
+	list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
+
+	if (vma->pages != obj->pages) {
+		GEM_BUG_ON(!vma->pages);
+		sg_free_table(vma->pages);
+		kfree(vma->pages);
+	}
+	vma->pages = NULL;
 
 	/* Since the unbound list is global, only move to that list if
 	 * no more VMAs exist. */
-	if (list_empty(&obj->vma_list))
-		list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
+	if (--obj->bind_count == 0)
+		list_move_tail(&obj->global_list,
+			       &to_i915(obj->base.dev)->mm.unbound_list);
 
 	/* And finally now the object is completely decoupled from this vma,
 	 * we can drop its hold on the backing storage and allow it to be
@@ -3686,36 +2920,28 @@ static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
 	 */
 	i915_gem_object_unpin_pages(obj);
 
-	return 0;
-}
+destroy:
+	if (unlikely(i915_vma_is_closed(vma)))
+		i915_vma_destroy(vma);
 
-int i915_vma_unbind(struct i915_vma *vma)
-{
-	return __i915_vma_unbind(vma, true);
-}
-
-int __i915_vma_unbind_no_wait(struct i915_vma *vma)
-{
-	return __i915_vma_unbind(vma, false);
+	return 0;
 }
 
-int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv)
+int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
+			   unsigned int flags)
 {
 	struct intel_engine_cs *engine;
 	int ret;
 
-	lockdep_assert_held(&dev_priv->drm.struct_mutex);
-
 	for_each_engine(engine, dev_priv) {
 		if (engine->last_context == NULL)
 			continue;
 
-		ret = intel_engine_idle(engine);
+		ret = intel_engine_idle(engine, flags);
 		if (ret)
 			return ret;
 	}
 
-	WARN_ON(i915_verify_lists(dev));
 	return 0;
 }
 
@@ -3753,128 +2979,87 @@ static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
 }
 
 /**
- * Finds free space in the GTT aperture and binds the object or a view of it
- * there.
- * @obj: object to bind
- * @vm: address space to bind into
- * @ggtt_view: global gtt view if applicable
- * @alignment: requested alignment
+ * i915_vma_insert - finds a slot for the vma in its address space
+ * @vma: the vma
+ * @size: requested size in bytes (can be larger than the VMA)
+ * @alignment: required alignment
  * @flags: mask of PIN_* flags to use
+ *
+ * First we try to allocate some free space that meets the requirements for
+ * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
+ * preferrably the oldest idle entry to make room for the new VMA.
+ *
+ * Returns:
+ * 0 on success, negative error code otherwise.
  */
-static struct i915_vma *
-i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
-			   struct i915_address_space *vm,
-			   const struct i915_ggtt_view *ggtt_view,
-			   unsigned alignment,
-			   uint64_t flags)
+static int
+i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	u32 fence_alignment, unfenced_alignment;
-	u32 search_flag, alloc_flag;
+	struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
+	struct drm_i915_gem_object *obj = vma->obj;
 	u64 start, end;
-	u64 size, fence_size;
-	struct i915_vma *vma;
 	int ret;
 
-	if (i915_is_ggtt(vm)) {
-		u32 view_size;
-
-		if (WARN_ON(!ggtt_view))
-			return ERR_PTR(-EINVAL);
-
-		view_size = i915_ggtt_view_size(obj, ggtt_view);
-
-		fence_size = i915_gem_get_gtt_size(dev,
-						   view_size,
-						   obj->tiling_mode);
-		fence_alignment = i915_gem_get_gtt_alignment(dev,
-							     view_size,
-							     obj->tiling_mode,
-							     true);
-		unfenced_alignment = i915_gem_get_gtt_alignment(dev,
-								view_size,
-								obj->tiling_mode,
-								false);
-		size = flags & PIN_MAPPABLE ? fence_size : view_size;
-	} else {
-		fence_size = i915_gem_get_gtt_size(dev,
-						   obj->base.size,
-						   obj->tiling_mode);
-		fence_alignment = i915_gem_get_gtt_alignment(dev,
-							     obj->base.size,
-							     obj->tiling_mode,
-							     true);
-		unfenced_alignment =
-			i915_gem_get_gtt_alignment(dev,
-						   obj->base.size,
-						   obj->tiling_mode,
-						   false);
-		size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
-	}
+	GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
+	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
+
+	size = max(size, vma->size);
+	if (flags & PIN_MAPPABLE)
+		size = i915_gem_get_ggtt_size(dev_priv, size,
+					      i915_gem_object_get_tiling(obj));
+
+	alignment = max(max(alignment, vma->display_alignment),
+			i915_gem_get_ggtt_alignment(dev_priv, size,
+						    i915_gem_object_get_tiling(obj),
+						    flags & PIN_MAPPABLE));
 
 	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
-	end = vm->total;
+
+	end = vma->vm->total;
 	if (flags & PIN_MAPPABLE)
-		end = min_t(u64, end, ggtt->mappable_end);
+		end = min_t(u64, end, dev_priv->ggtt.mappable_end);
 	if (flags & PIN_ZONE_4G)
 		end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
 
-	if (alignment == 0)
-		alignment = flags & PIN_MAPPABLE ? fence_alignment :
-						unfenced_alignment;
-	if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
-		DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
-			  ggtt_view ? ggtt_view->type : 0,
-			  alignment);
-		return ERR_PTR(-EINVAL);
-	}
-
 	/* If binding the object/GGTT view requires more space than the entire
 	 * aperture has, reject it early before evicting everything in a vain
 	 * attempt to find space.
 	 */
 	if (size > end) {
-		DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
-			  ggtt_view ? ggtt_view->type : 0,
-			  size,
+		DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",
+			  size, obj->base.size,
 			  flags & PIN_MAPPABLE ? "mappable" : "total",
 			  end);
-		return ERR_PTR(-E2BIG);
+		return -E2BIG;
 	}
 
 	ret = i915_gem_object_get_pages(obj);
 	if (ret)
-		return ERR_PTR(ret);
+		return ret;
 
 	i915_gem_object_pin_pages(obj);
 
-	vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
-			  i915_gem_obj_lookup_or_create_vma(obj, vm);
-
-	if (IS_ERR(vma))
-		goto err_unpin;
-
 	if (flags & PIN_OFFSET_FIXED) {
-		uint64_t offset = flags & PIN_OFFSET_MASK;
-
-		if (offset & (alignment - 1) || offset + size > end) {
+		u64 offset = flags & PIN_OFFSET_MASK;
+		if (offset & (alignment - 1) || offset > end - size) {
 			ret = -EINVAL;
-			goto err_free_vma;
+			goto err_unpin;
 		}
+
 		vma->node.start = offset;
 		vma->node.size = size;
 		vma->node.color = obj->cache_level;
-		ret = drm_mm_reserve_node(&vm->mm, &vma->node);
+		ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
 		if (ret) {
 			ret = i915_gem_evict_for_vma(vma);
 			if (ret == 0)
-				ret = drm_mm_reserve_node(&vm->mm, &vma->node);
+				ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
+			if (ret)
+				goto err_unpin;
 		}
-		if (ret)
-			goto err_free_vma;
 	} else {
+		u32 search_flag, alloc_flag;
+
 		if (flags & PIN_HIGH) {
 			search_flag = DRM_MM_SEARCH_BELOW;
 			alloc_flag = DRM_MM_CREATE_TOP;
@@ -3883,47 +3068,45 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
 			alloc_flag = DRM_MM_CREATE_DEFAULT;
 		}
 
+		/* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
+		 * so we know that we always have a minimum alignment of 4096.
+		 * The drm_mm range manager is optimised to return results
+		 * with zero alignment, so where possible use the optimal
+		 * path.
+		 */
+		if (alignment <= 4096)
+			alignment = 0;
+
 search_free:
-		ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
+		ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm,
+							  &vma->node,
 							  size, alignment,
 							  obj->cache_level,
 							  start, end,
 							  search_flag,
 							  alloc_flag);
 		if (ret) {
-			ret = i915_gem_evict_something(dev, vm, size, alignment,
+			ret = i915_gem_evict_something(vma->vm, size, alignment,
 						       obj->cache_level,
 						       start, end,
 						       flags);
 			if (ret == 0)
 				goto search_free;
 
-			goto err_free_vma;
+			goto err_unpin;
 		}
 	}
-	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
-		ret = -EINVAL;
-		goto err_remove_node;
-	}
-
-	trace_i915_vma_bind(vma, flags);
-	ret = i915_vma_bind(vma, obj->cache_level, flags);
-	if (ret)
-		goto err_remove_node;
+	GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
 
 	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
-	list_add_tail(&vma->vm_link, &vm->inactive_list);
+	list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+	obj->bind_count++;
 
-	return vma;
+	return 0;
 
-err_remove_node:
-	drm_mm_remove_node(&vma->node);
-err_free_vma:
-	i915_gem_vma_destroy(vma);
-	vma = ERR_PTR(ret);
 err_unpin:
 	i915_gem_object_unpin_pages(obj);
-	return vma;
+	return ret;
 }
 
 bool
@@ -3968,51 +3151,72 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,
 static void
 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
 {
-	uint32_t old_write_domain;
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 
 	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
 		return;
 
 	/* No actual flushing is required for the GTT write domain.  Writes
-	 * to it immediately go to main memory as far as we know, so there's
+	 * to it "immediately" go to main memory as far as we know, so there's
 	 * no chipset flush.  It also doesn't land in render cache.
 	 *
 	 * However, we do have to enforce the order so that all writes through
 	 * the GTT land before any writes to the device, such as updates to
 	 * the GATT itself.
+	 *
+	 * We also have to wait a bit for the writes to land from the GTT.
+	 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
+	 * timing. This issue has only been observed when switching quickly
+	 * between GTT writes and CPU reads from inside the kernel on recent hw,
+	 * and it appears to only affect discrete GTT blocks (i.e. on LLC
+	 * system agents we cannot reproduce this behaviour).
 	 */
 	wmb();
+	if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv))
+		POSTING_READ(RING_ACTHD(dev_priv->engine[RCS].mmio_base));
 
-	old_write_domain = obj->base.write_domain;
-	obj->base.write_domain = 0;
-
-	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
+	intel_fb_obj_flush(obj, false, write_origin(obj, I915_GEM_DOMAIN_GTT));
 
+	obj->base.write_domain = 0;
 	trace_i915_gem_object_change_domain(obj,
 					    obj->base.read_domains,
-					    old_write_domain);
+					    I915_GEM_DOMAIN_GTT);
 }
 
 /** Flushes the CPU write domain for the object if it's dirty. */
 static void
 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
 {
-	uint32_t old_write_domain;
-
 	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
 		return;
 
 	if (i915_gem_clflush_object(obj, obj->pin_display))
 		i915_gem_chipset_flush(to_i915(obj->base.dev));
 
-	old_write_domain = obj->base.write_domain;
-	obj->base.write_domain = 0;
-
 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 
+	obj->base.write_domain = 0;
 	trace_i915_gem_object_change_domain(obj,
 					    obj->base.read_domains,
-					    old_write_domain);
+					    I915_GEM_DOMAIN_CPU);
+}
+
+static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
+{
+	struct i915_vma *vma;
+
+	list_for_each_entry(vma, &obj->vma_list, obj_link) {
+		if (!i915_vma_is_ggtt(vma))
+			continue;
+
+		if (i915_vma_is_active(vma))
+			continue;
+
+		if (!drm_mm_node_allocated(&vma->node))
+			continue;
+
+		list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+	}
 }
 
 /**
@@ -4026,20 +3230,16 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
 int
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	uint32_t old_write_domain, old_read_domains;
-	struct i915_vma *vma;
 	int ret;
 
-	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
-		return 0;
-
 	ret = i915_gem_object_wait_rendering(obj, !write);
 	if (ret)
 		return ret;
 
+	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
+		return 0;
+
 	/* Flush and acquire obj->pages so that we are coherent through
 	 * direct access in memory with previous cached writes through
 	 * shmemfs and that our cache domain tracking remains valid.
@@ -4080,10 +3280,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 					    old_write_domain);
 
 	/* And bump the LRU for this access */
-	vma = i915_gem_obj_to_ggtt(obj);
-	if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
-		list_move_tail(&vma->vm_link,
-			       &ggtt->base.inactive_list);
+	i915_gem_object_bump_inactive_ggtt(obj);
 
 	return 0;
 }
@@ -4106,9 +3303,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 				    enum i915_cache_level cache_level)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct i915_vma *vma, *next;
-	bool bound = false;
+	struct i915_vma *vma;
 	int ret = 0;
 
 	if (obj->cache_level == cache_level)
@@ -4119,21 +3314,28 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 	 * catch the issue of the CS prefetch crossing page boundaries and
 	 * reading an invalid PTE on older architectures.
 	 */
-	list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
+restart:
+	list_for_each_entry(vma, &obj->vma_list, obj_link) {
 		if (!drm_mm_node_allocated(&vma->node))
 			continue;
 
-		if (vma->pin_count) {
+		if (i915_vma_is_pinned(vma)) {
 			DRM_DEBUG("can not change the cache level of pinned objects\n");
 			return -EBUSY;
 		}
 
-		if (!i915_gem_valid_gtt_space(vma, cache_level)) {
-			ret = i915_vma_unbind(vma);
-			if (ret)
-				return ret;
-		} else
-			bound = true;
+		if (i915_gem_valid_gtt_space(vma, cache_level))
+			continue;
+
+		ret = i915_vma_unbind(vma);
+		if (ret)
+			return ret;
+
+		/* As unbinding may affect other elements in the
+		 * obj->vma_list (due to side-effects from retiring
+		 * an active vma), play safe and restart the iterator.
+		 */
+		goto restart;
 	}
 
 	/* We can reuse the existing drm_mm nodes but need to change the
@@ -4143,7 +3345,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 	 * rewrite the PTE in the belief that doing so tramples upon less
 	 * state and so involves less work.
 	 */
-	if (bound) {
+	if (obj->bind_count) {
 		/* Before we change the PTE, the GPU must not be accessing it.
 		 * If we wait upon the object, we know that all the bound
 		 * VMA are no longer active.
@@ -4152,7 +3354,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 		if (ret)
 			return ret;
 
-		if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
+		if (!HAS_LLC(obj->base.dev) && cache_level != I915_CACHE_NONE) {
 			/* Access to snoopable pages through the GTT is
 			 * incoherent and on some machines causes a hard
 			 * lockup. Relinquish the CPU mmaping to force
@@ -4169,9 +3371,11 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 			 * dropped the fence as all snoopable access is
 			 * supposed to be linear.
 			 */
-			ret = i915_gem_object_put_fence(obj);
-			if (ret)
-				return ret;
+			list_for_each_entry(vma, &obj->vma_list, obj_link) {
+				ret = i915_vma_put_fence(vma);
+				if (ret)
+					return ret;
+			}
 		} else {
 			/* We either have incoherent backing store and
 			 * so no GTT access or the architecture is fully
@@ -4215,8 +3419,8 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
 	struct drm_i915_gem_caching *args = data;
 	struct drm_i915_gem_object *obj;
 
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
-	if (&obj->base == NULL)
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj)
 		return -ENOENT;
 
 	switch (obj->cache_level) {
@@ -4234,7 +3438,7 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
 		break;
 	}
 
-	drm_gem_object_unreference_unlocked(&obj->base);
+	i915_gem_object_put_unlocked(obj);
 	return 0;
 }
 
@@ -4276,15 +3480,15 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 	if (ret)
 		goto rpm_put;
 
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
-	if (&obj->base == NULL) {
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj) {
 		ret = -ENOENT;
 		goto unlock;
 	}
 
 	ret = i915_gem_object_set_cache_level(obj, level);
 
-	drm_gem_object_unreference(&obj->base);
+	i915_gem_object_put(obj);
 unlock:
 	mutex_unlock(&dev->struct_mutex);
 rpm_put:
@@ -4298,11 +3502,12 @@ rpm_put:
  * Can be called from an uninterruptible phase (modesetting) and allows
  * any flushes to be pipelined (for pageflips).
  */
-int
+struct i915_vma *
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 				     u32 alignment,
 				     const struct i915_ggtt_view *view)
 {
+	struct i915_vma *vma;
 	u32 old_read_domains, old_write_domain;
 	int ret;
 
@@ -4322,19 +3527,31 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 	 */
 	ret = i915_gem_object_set_cache_level(obj,
 					      HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
-	if (ret)
+	if (ret) {
+		vma = ERR_PTR(ret);
 		goto err_unpin_display;
+	}
 
 	/* As the user may map the buffer once pinned in the display plane
 	 * (e.g. libkms for the bootup splash), we have to ensure that we
-	 * always use map_and_fenceable for all scanout buffers.
+	 * always use map_and_fenceable for all scanout buffers. However,
+	 * it may simply be too big to fit into mappable, in which case
+	 * put it anyway and hope that userspace can cope (but always first
+	 * try to preserve the existing ABI).
 	 */
-	ret = i915_gem_object_ggtt_pin(obj, view, alignment,
-				       view->type == I915_GGTT_VIEW_NORMAL ?
-				       PIN_MAPPABLE : 0);
-	if (ret)
+	vma = ERR_PTR(-ENOSPC);
+	if (view->type == I915_GGTT_VIEW_NORMAL)
+		vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
+					       PIN_MAPPABLE | PIN_NONBLOCK);
+	if (IS_ERR(vma))
+		vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, 0);
+	if (IS_ERR(vma))
 		goto err_unpin_display;
 
+	vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
+
+	WARN_ON(obj->pin_display > i915_vma_pin_count(vma));
+
 	i915_gem_object_flush_cpu_write_domain(obj);
 
 	old_write_domain = obj->base.write_domain;
@@ -4350,23 +3567,28 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 					    old_read_domains,
 					    old_write_domain);
 
-	return 0;
+	return vma;
 
 err_unpin_display:
 	obj->pin_display--;
-	return ret;
+	return vma;
 }
 
 void
-i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
-					 const struct i915_ggtt_view *view)
+i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
 {
-	if (WARN_ON(obj->pin_display == 0))
+	if (WARN_ON(vma->obj->pin_display == 0))
 		return;
 
-	i915_gem_object_ggtt_unpin_view(obj, view);
+	if (--vma->obj->pin_display == 0)
+		vma->display_alignment = 0;
 
-	obj->pin_display--;
+	/* Bump the LRU to try and avoid premature eviction whilst flipping  */
+	if (!i915_vma_is_active(vma))
+		list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+
+	i915_vma_unpin(vma);
+	WARN_ON(vma->obj->pin_display > i915_vma_pin_count(vma));
 }
 
 /**
@@ -4383,13 +3605,13 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	uint32_t old_write_domain, old_read_domains;
 	int ret;
 
-	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
-		return 0;
-
 	ret = i915_gem_object_wait_rendering(obj, !write);
 	if (ret)
 		return ret;
 
+	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
+		return 0;
+
 	i915_gem_object_flush_gtt_write_domain(obj);
 
 	old_write_domain = obj->base.write_domain;
@@ -4464,28 +3686,31 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 		target = request;
 	}
 	if (target)
-		i915_gem_request_reference(target);
+		i915_gem_request_get(target);
 	spin_unlock(&file_priv->mm.lock);
 
 	if (target == NULL)
 		return 0;
 
-	ret = __i915_wait_request(target, true, NULL, NULL);
-	i915_gem_request_unreference(target);
+	ret = i915_wait_request(target, I915_WAIT_INTERRUPTIBLE, NULL, NULL);
+	i915_gem_request_put(target);
 
 	return ret;
 }
 
 static bool
-i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
+i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 {
-	struct drm_i915_gem_object *obj = vma->obj;
+	if (!drm_mm_node_allocated(&vma->node))
+		return false;
 
-	if (alignment &&
-	    vma->node.start & (alignment - 1))
+	if (vma->node.size < size)
 		return true;
 
-	if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
+	if (alignment && vma->node.start & (alignment - 1))
+		return true;
+
+	if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
 		return true;
 
 	if (flags & PIN_OFFSET_BIAS &&
@@ -4502,135 +3727,208 @@ i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
 {
 	struct drm_i915_gem_object *obj = vma->obj;
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 	bool mappable, fenceable;
 	u32 fence_size, fence_alignment;
 
-	fence_size = i915_gem_get_gtt_size(obj->base.dev,
-					   obj->base.size,
-					   obj->tiling_mode);
-	fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
-						     obj->base.size,
-						     obj->tiling_mode,
-						     true);
+	fence_size = i915_gem_get_ggtt_size(dev_priv,
+					    vma->size,
+					    i915_gem_object_get_tiling(obj));
+	fence_alignment = i915_gem_get_ggtt_alignment(dev_priv,
+						      vma->size,
+						      i915_gem_object_get_tiling(obj),
+						      true);
 
 	fenceable = (vma->node.size == fence_size &&
 		     (vma->node.start & (fence_alignment - 1)) == 0);
 
 	mappable = (vma->node.start + fence_size <=
-		    to_i915(obj->base.dev)->ggtt.mappable_end);
+		    dev_priv->ggtt.mappable_end);
 
-	obj->map_and_fenceable = mappable && fenceable;
+	if (mappable && fenceable)
+		vma->flags |= I915_VMA_CAN_FENCE;
+	else
+		vma->flags &= ~I915_VMA_CAN_FENCE;
 }
 
-static int
-i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
-		       struct i915_address_space *vm,
-		       const struct i915_ggtt_view *ggtt_view,
-		       uint32_t alignment,
-		       uint64_t flags)
+int __i915_vma_do_pin(struct i915_vma *vma,
+		      u64 size, u64 alignment, u64 flags)
 {
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	struct i915_vma *vma;
-	unsigned bound;
+	unsigned int bound = vma->flags;
 	int ret;
 
-	if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
-		return -ENODEV;
+	GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
+	GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
 
-	if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
-		return -EINVAL;
+	if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
+		ret = -EBUSY;
+		goto err;
+	}
 
-	if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
-		return -EINVAL;
+	if ((bound & I915_VMA_BIND_MASK) == 0) {
+		ret = i915_vma_insert(vma, size, alignment, flags);
+		if (ret)
+			goto err;
+	}
 
-	if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
-		return -EINVAL;
+	ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
+	if (ret)
+		goto err;
 
-	vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
-			  i915_gem_obj_to_vma(obj, vm);
+	if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
+		__i915_vma_set_map_and_fenceable(vma);
 
-	if (vma) {
-		if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
-			return -EBUSY;
+	GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
+	return 0;
 
-		if (i915_vma_misplaced(vma, alignment, flags)) {
-			WARN(vma->pin_count,
-			     "bo is already pinned in %s with incorrect alignment:"
-			     " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
-			     " obj->map_and_fenceable=%d\n",
-			     ggtt_view ? "ggtt" : "ppgtt",
-			     upper_32_bits(vma->node.start),
-			     lower_32_bits(vma->node.start),
-			     alignment,
-			     !!(flags & PIN_MAPPABLE),
-			     obj->map_and_fenceable);
-			ret = i915_vma_unbind(vma);
-			if (ret)
-				return ret;
+err:
+	__i915_vma_unpin(vma);
+	return ret;
+}
 
-			vma = NULL;
-		}
-	}
+struct i915_vma *
+i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
+			 const struct i915_ggtt_view *view,
+			 u64 size,
+			 u64 alignment,
+			 u64 flags)
+{
+	struct i915_address_space *vm = &to_i915(obj->base.dev)->ggtt.base;
+	struct i915_vma *vma;
+	int ret;
 
-	bound = vma ? vma->bound : 0;
-	if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
-		vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
-						 flags);
-		if (IS_ERR(vma))
-			return PTR_ERR(vma);
-	} else {
-		ret = i915_vma_bind(vma, obj->cache_level, flags);
+	vma = i915_gem_obj_lookup_or_create_vma(obj, vm, view);
+	if (IS_ERR(vma))
+		return vma;
+
+	if (i915_vma_misplaced(vma, size, alignment, flags)) {
+		if (flags & PIN_NONBLOCK &&
+		    (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)))
+			return ERR_PTR(-ENOSPC);
+
+		WARN(i915_vma_is_pinned(vma),
+		     "bo is already pinned in ggtt with incorrect alignment:"
+		     " offset=%08x, req.alignment=%llx,"
+		     " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
+		     i915_ggtt_offset(vma), alignment,
+		     !!(flags & PIN_MAPPABLE),
+		     i915_vma_is_map_and_fenceable(vma));
+		ret = i915_vma_unbind(vma);
 		if (ret)
-			return ret;
+			return ERR_PTR(ret);
 	}
 
-	if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
-	    (bound ^ vma->bound) & GLOBAL_BIND) {
-		__i915_vma_set_map_and_fenceable(vma);
-		WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
-	}
+	ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
+	if (ret)
+		return ERR_PTR(ret);
 
-	vma->pin_count++;
-	return 0;
+	return vma;
 }
 
-int
-i915_gem_object_pin(struct drm_i915_gem_object *obj,
-		    struct i915_address_space *vm,
-		    uint32_t alignment,
-		    uint64_t flags)
+static __always_inline unsigned int __busy_read_flag(unsigned int id)
 {
-	return i915_gem_object_do_pin(obj, vm,
-				      i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
-				      alignment, flags);
+	/* Note that we could alias engines in the execbuf API, but
+	 * that would be very unwise as it prevents userspace from
+	 * fine control over engine selection. Ahem.
+	 *
+	 * This should be something like EXEC_MAX_ENGINE instead of
+	 * I915_NUM_ENGINES.
+	 */
+	BUILD_BUG_ON(I915_NUM_ENGINES > 16);
+	return 0x10000 << id;
 }
 
-int
-i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
-			 const struct i915_ggtt_view *view,
-			 uint32_t alignment,
-			 uint64_t flags)
+static __always_inline unsigned int __busy_write_id(unsigned int id)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
+	/* The uABI guarantees an active writer is also amongst the read
+	 * engines. This would be true if we accessed the activity tracking
+	 * under the lock, but as we perform the lookup of the object and
+	 * its activity locklessly we can not guarantee that the last_write
+	 * being active implies that we have set the same engine flag from
+	 * last_read - hence we always set both read and write busy for
+	 * last_write.
+	 */
+	return id | __busy_read_flag(id);
+}
 
-	BUG_ON(!view);
+static __always_inline unsigned int
+__busy_set_if_active(const struct i915_gem_active *active,
+		     unsigned int (*flag)(unsigned int id))
+{
+	struct drm_i915_gem_request *request;
+
+	request = rcu_dereference(active->request);
+	if (!request || i915_gem_request_completed(request))
+		return 0;
 
-	return i915_gem_object_do_pin(obj, &ggtt->base, view,
-				      alignment, flags | PIN_GLOBAL);
+	/* This is racy. See __i915_gem_active_get_rcu() for an in detail
+	 * discussion of how to handle the race correctly, but for reporting
+	 * the busy state we err on the side of potentially reporting the
+	 * wrong engine as being busy (but we guarantee that the result
+	 * is at least self-consistent).
+	 *
+	 * As we use SLAB_DESTROY_BY_RCU, the request may be reallocated
+	 * whilst we are inspecting it, even under the RCU read lock as we are.
+	 * This means that there is a small window for the engine and/or the
+	 * seqno to have been overwritten. The seqno will always be in the
+	 * future compared to the intended, and so we know that if that
+	 * seqno is idle (on whatever engine) our request is idle and the
+	 * return 0 above is correct.
+	 *
+	 * The issue is that if the engine is switched, it is just as likely
+	 * to report that it is busy (but since the switch happened, we know
+	 * the request should be idle). So there is a small chance that a busy
+	 * result is actually the wrong engine.
+	 *
+	 * So why don't we care?
+	 *
+	 * For starters, the busy ioctl is a heuristic that is by definition
+	 * racy. Even with perfect serialisation in the driver, the hardware
+	 * state is constantly advancing - the state we report to the user
+	 * is stale.
+	 *
+	 * The critical information for the busy-ioctl is whether the object
+	 * is idle as userspace relies on that to detect whether its next
+	 * access will stall, or if it has missed submitting commands to
+	 * the hardware allowing the GPU to stall. We never generate a
+	 * false-positive for idleness, thus busy-ioctl is reliable at the
+	 * most fundamental level, and we maintain the guarantee that a
+	 * busy object left to itself will eventually become idle (and stay
+	 * idle!).
+	 *
+	 * We allow ourselves the leeway of potentially misreporting the busy
+	 * state because that is an optimisation heuristic that is constantly
+	 * in flux. Being quickly able to detect the busy/idle state is much
+	 * more important than accurate logging of exactly which engines were
+	 * busy.
+	 *
+	 * For accuracy in reporting the engine, we could use
+	 *
+	 *	result = 0;
+	 *	request = __i915_gem_active_get_rcu(active);
+	 *	if (request) {
+	 *		if (!i915_gem_request_completed(request))
+	 *			result = flag(request->engine->exec_id);
+	 *		i915_gem_request_put(request);
+	 *	}
+	 *
+	 * but that still remains susceptible to both hardware and userspace
+	 * races. So we accept making the result of that race slightly worse,
+	 * given the rarity of the race and its low impact on the result.
+	 */
+	return flag(READ_ONCE(request->engine->exec_id));
 }
 
-void
-i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
-				const struct i915_ggtt_view *view)
+static __always_inline unsigned int
+busy_check_reader(const struct i915_gem_active *active)
 {
-	struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
-
-	WARN_ON(vma->pin_count == 0);
-	WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
+	return __busy_set_if_active(active, __busy_read_flag);
+}
 
-	--vma->pin_count;
+static __always_inline unsigned int
+busy_check_writer(const struct i915_gem_active *active)
+{
+	return __busy_set_if_active(active, __busy_write_id);
 }
 
 int
@@ -4639,47 +3937,64 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_busy *args = data;
 	struct drm_i915_gem_object *obj;
-	int ret;
+	unsigned long active;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj)
+		return -ENOENT;
 
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
-	if (&obj->base == NULL) {
-		ret = -ENOENT;
-		goto unlock;
-	}
+	args->busy = 0;
+	active = __I915_BO_ACTIVE(obj);
+	if (active) {
+		int idx;
 
-	/* Count all active objects as busy, even if they are currently not used
-	 * by the gpu. Users of this interface expect objects to eventually
-	 * become non-busy without any further actions, therefore emit any
-	 * necessary flushes here.
-	 */
-	ret = i915_gem_object_flush_active(obj);
-	if (ret)
-		goto unref;
+		/* Yes, the lookups are intentionally racy.
+		 *
+		 * First, we cannot simply rely on __I915_BO_ACTIVE. We have
+		 * to regard the value as stale and as our ABI guarantees
+		 * forward progress, we confirm the status of each active
+		 * request with the hardware.
+		 *
+		 * Even though we guard the pointer lookup by RCU, that only
+		 * guarantees that the pointer and its contents remain
+		 * dereferencable and does *not* mean that the request we
+		 * have is the same as the one being tracked by the object.
+		 *
+		 * Consider that we lookup the request just as it is being
+		 * retired and freed. We take a local copy of the pointer,
+		 * but before we add its engine into the busy set, the other
+		 * thread reallocates it and assigns it to a task on another
+		 * engine with a fresh and incomplete seqno. Guarding against
+		 * that requires careful serialisation and reference counting,
+		 * i.e. using __i915_gem_active_get_request_rcu(). We don't,
+		 * instead we expect that if the result is busy, which engines
+		 * are busy is not completely reliable - we only guarantee
+		 * that the object was busy.
+		 */
+		rcu_read_lock();
 
-	args->busy = 0;
-	if (obj->active) {
-		int i;
+		for_each_active(active, idx)
+			args->busy |= busy_check_reader(&obj->last_read[idx]);
 
-		for (i = 0; i < I915_NUM_ENGINES; i++) {
-			struct drm_i915_gem_request *req;
+		/* For ABI sanity, we only care that the write engine is in
+		 * the set of read engines. This should be ensured by the
+		 * ordering of setting last_read/last_write in
+		 * i915_vma_move_to_active(), and then in reverse in retire.
+		 * However, for good measure, we always report the last_write
+		 * request as a busy read as well as being a busy write.
+		 *
+		 * We don't care that the set of active read/write engines
+		 * may change during construction of the result, as it is
+		 * equally liable to change before userspace can inspect
+		 * the result.
+		 */
+		args->busy |= busy_check_writer(&obj->last_write);
 
-			req = obj->last_read_req[i];
-			if (req)
-				args->busy |= 1 << (16 + req->engine->exec_id);
-		}
-		if (obj->last_write_req)
-			args->busy |= obj->last_write_req->engine->exec_id;
+		rcu_read_unlock();
 	}
 
-unref:
-	drm_gem_object_unreference(&obj->base);
-unlock:
-	mutex_unlock(&dev->struct_mutex);
-	return ret;
+	i915_gem_object_put_unlocked(obj);
+	return 0;
 }
 
 int
@@ -4710,19 +4025,14 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 	if (ret)
 		return ret;
 
-	obj = to_intel_bo(drm_gem_object_lookup(file_priv, args->handle));
-	if (&obj->base == NULL) {
+	obj = i915_gem_object_lookup(file_priv, args->handle);
+	if (!obj) {
 		ret = -ENOENT;
 		goto unlock;
 	}
 
-	if (i915_gem_obj_is_pinned(obj)) {
-		ret = -EINVAL;
-		goto out;
-	}
-
 	if (obj->pages &&
-	    obj->tiling_mode != I915_TILING_NONE &&
+	    i915_gem_object_is_tiled(obj) &&
 	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
 		if (obj->madv == I915_MADV_WILLNEED)
 			i915_gem_object_unpin_pages(obj);
@@ -4739,8 +4049,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 
 	args->retained = obj->madv != __I915_MADV_PURGED;
 
-out:
-	drm_gem_object_unreference(&obj->base);
+	i915_gem_object_put(obj);
 unlock:
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
@@ -4753,14 +4062,17 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
 
 	INIT_LIST_HEAD(&obj->global_list);
 	for (i = 0; i < I915_NUM_ENGINES; i++)
-		INIT_LIST_HEAD(&obj->engine_list[i]);
+		init_request_active(&obj->last_read[i],
+				    i915_gem_object_retire__read);
+	init_request_active(&obj->last_write,
+			    i915_gem_object_retire__write);
 	INIT_LIST_HEAD(&obj->obj_exec_link);
 	INIT_LIST_HEAD(&obj->vma_list);
 	INIT_LIST_HEAD(&obj->batch_pool_link);
 
 	obj->ops = ops;
 
-	obj->fence_reg = I915_FENCE_REG_NONE;
+	obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
 	obj->madv = I915_MADV_WILLNEED;
 
 	i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
@@ -4865,33 +4177,31 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 
 	trace_i915_gem_object_destroy(obj);
 
+	/* All file-owned VMA should have been released by this point through
+	 * i915_gem_close_object(), or earlier by i915_gem_context_close().
+	 * However, the object may also be bound into the global GTT (e.g.
+	 * older GPUs without per-process support, or for direct access through
+	 * the GTT either for the user or for scanout). Those VMA still need to
+	 * unbound now.
+	 */
 	list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
-		int ret;
-
-		vma->pin_count = 0;
-		ret = i915_vma_unbind(vma);
-		if (WARN_ON(ret == -ERESTARTSYS)) {
-			bool was_interruptible;
-
-			was_interruptible = dev_priv->mm.interruptible;
-			dev_priv->mm.interruptible = false;
-
-			WARN_ON(i915_vma_unbind(vma));
-
-			dev_priv->mm.interruptible = was_interruptible;
-		}
+		GEM_BUG_ON(!i915_vma_is_ggtt(vma));
+		GEM_BUG_ON(i915_vma_is_active(vma));
+		vma->flags &= ~I915_VMA_PIN_MASK;
+		i915_vma_close(vma);
 	}
+	GEM_BUG_ON(obj->bind_count);
 
 	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
 	 * before progressing. */
 	if (obj->stolen)
 		i915_gem_object_unpin_pages(obj);
 
-	WARN_ON(obj->frontbuffer_bits);
+	WARN_ON(atomic_read(&obj->frontbuffer_bits));
 
 	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
 	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
-	    obj->tiling_mode != I915_TILING_NONE)
+	    i915_gem_object_is_tiled(obj))
 		i915_gem_object_unpin_pages(obj);
 
 	if (WARN_ON(obj->pages_pin_count))
@@ -4899,7 +4209,6 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 	if (discard_backing_storage(obj))
 		obj->madv = I915_MADV_DONTNEED;
 	i915_gem_object_put_pages(obj);
-	i915_gem_object_free_mmap_offset(obj);
 
 	BUG_ON(obj->pages);
 
@@ -4918,71 +4227,35 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 	intel_runtime_pm_put(dev_priv);
 }
 
-struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
-				     struct i915_address_space *vm)
-{
-	struct i915_vma *vma;
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
-		    vma->vm == vm)
-			return vma;
-	}
-	return NULL;
-}
-
-struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
-					   const struct i915_ggtt_view *view)
-{
-	struct i915_vma *vma;
-
-	GEM_BUG_ON(!view);
-
-	list_for_each_entry(vma, &obj->vma_list, obj_link)
-		if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
-			return vma;
-	return NULL;
-}
-
-void i915_gem_vma_destroy(struct i915_vma *vma)
-{
-	WARN_ON(vma->node.allocated);
-
-	/* Keep the vma as a placeholder in the execbuffer reservation lists */
-	if (!list_empty(&vma->exec_list))
-		return;
-
-	if (!vma->is_ggtt)
-		i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
-
-	list_del(&vma->obj_link);
-
-	kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
-}
-
-static void
-i915_gem_stop_engines(struct drm_device *dev)
+int i915_gem_suspend(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_engine_cs *engine;
+	int ret;
 
-	for_each_engine(engine, dev_priv)
-		dev_priv->gt.stop_engine(engine);
-}
-
-int
-i915_gem_suspend(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret = 0;
+	intel_suspend_gt_powersave(dev_priv);
 
 	mutex_lock(&dev->struct_mutex);
-	ret = i915_gem_wait_for_idle(dev_priv);
+
+	/* We have to flush all the executing contexts to main memory so
+	 * that they can saved in the hibernation image. To ensure the last
+	 * context image is coherent, we have to switch away from it. That
+	 * leaves the dev_priv->kernel_context still active when
+	 * we actually suspend, and its image in memory may not match the GPU
+	 * state. Fortunately, the kernel_context is disposable and we do
+	 * not rely on its state.
+	 */
+	ret = i915_gem_switch_to_kernel_context(dev_priv);
+	if (ret)
+		goto err;
+
+	ret = i915_gem_wait_for_idle(dev_priv,
+				     I915_WAIT_INTERRUPTIBLE |
+				     I915_WAIT_LOCKED);
 	if (ret)
 		goto err;
 
 	i915_gem_retire_requests(dev_priv);
 
-	i915_gem_stop_engines(dev);
 	i915_gem_context_lost(dev_priv);
 	mutex_unlock(&dev->struct_mutex);
 
@@ -5002,6 +4275,22 @@ err:
 	return ret;
 }
 
+void i915_gem_resume(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	mutex_lock(&dev->struct_mutex);
+	i915_gem_restore_gtt_mappings(dev);
+
+	/* As we didn't flush the kernel context before suspend, we cannot
+	 * guarantee that the context image is complete. So let's just reset
+	 * it and start again.
+	 */
+	dev_priv->gt.resume(dev_priv);
+
+	mutex_unlock(&dev->struct_mutex);
+}
+
 void i915_gem_init_swizzling(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -5054,53 +4343,6 @@ static void init_unused_rings(struct drm_device *dev)
 	}
 }
 
-int i915_gem_init_engines(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret;
-
-	ret = intel_init_render_ring_buffer(dev);
-	if (ret)
-		return ret;
-
-	if (HAS_BSD(dev)) {
-		ret = intel_init_bsd_ring_buffer(dev);
-		if (ret)
-			goto cleanup_render_ring;
-	}
-
-	if (HAS_BLT(dev)) {
-		ret = intel_init_blt_ring_buffer(dev);
-		if (ret)
-			goto cleanup_bsd_ring;
-	}
-
-	if (HAS_VEBOX(dev)) {
-		ret = intel_init_vebox_ring_buffer(dev);
-		if (ret)
-			goto cleanup_blt_ring;
-	}
-
-	if (HAS_BSD2(dev)) {
-		ret = intel_init_bsd2_ring_buffer(dev);
-		if (ret)
-			goto cleanup_vebox_ring;
-	}
-
-	return 0;
-
-cleanup_vebox_ring:
-	intel_cleanup_engine(&dev_priv->engine[VECS]);
-cleanup_blt_ring:
-	intel_cleanup_engine(&dev_priv->engine[BCS]);
-cleanup_bsd_ring:
-	intel_cleanup_engine(&dev_priv->engine[VCS]);
-cleanup_render_ring:
-	intel_cleanup_engine(&dev_priv->engine[RCS]);
-
-	return ret;
-}
-
 int
 i915_gem_init_hw(struct drm_device *dev)
 {
@@ -5167,6 +4409,27 @@ out:
 	return ret;
 }
 
+bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
+{
+	if (INTEL_INFO(dev_priv)->gen < 6)
+		return false;
+
+	/* TODO: make semaphores and Execlists play nicely together */
+	if (i915.enable_execlists)
+		return false;
+
+	if (value >= 0)
+		return value;
+
+#ifdef CONFIG_INTEL_IOMMU
+	/* Enable semaphores on SNB when IO remapping is off */
+	if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
+		return false;
+#endif
+
+	return true;
+}
+
 int i915_gem_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -5175,15 +4438,11 @@ int i915_gem_init(struct drm_device *dev)
 	mutex_lock(&dev->struct_mutex);
 
 	if (!i915.enable_execlists) {
-		dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
-		dev_priv->gt.init_engines = i915_gem_init_engines;
-		dev_priv->gt.cleanup_engine = intel_cleanup_engine;
-		dev_priv->gt.stop_engine = intel_stop_engine;
+		dev_priv->gt.resume = intel_legacy_submission_resume;
+		dev_priv->gt.cleanup_engine = intel_engine_cleanup;
 	} else {
-		dev_priv->gt.execbuf_submit = intel_execlists_submission;
-		dev_priv->gt.init_engines = intel_logical_rings_init;
+		dev_priv->gt.resume = intel_lr_context_resume;
 		dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
-		dev_priv->gt.stop_engine = intel_logical_ring_stop;
 	}
 
 	/* This is just a security blanket to placate dragons.
@@ -5195,24 +4454,27 @@ int i915_gem_init(struct drm_device *dev)
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
 	i915_gem_init_userptr(dev_priv);
-	i915_gem_init_ggtt(dev);
+
+	ret = i915_gem_init_ggtt(dev_priv);
+	if (ret)
+		goto out_unlock;
 
 	ret = i915_gem_context_init(dev);
 	if (ret)
 		goto out_unlock;
 
-	ret = dev_priv->gt.init_engines(dev);
+	ret = intel_engines_init(dev);
 	if (ret)
 		goto out_unlock;
 
 	ret = i915_gem_init_hw(dev);
 	if (ret == -EIO) {
-		/* Allow ring initialisation to fail by marking the GPU as
+		/* Allow engine initialisation to fail by marking the GPU as
 		 * wedged. But we only want to do this where the GPU is angry,
 		 * for all other failure, such as an allocation failure, bail.
 		 */
 		DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
-		atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
+		i915_gem_set_wedged(dev_priv);
 		ret = 0;
 	}
 
@@ -5236,7 +4498,6 @@ i915_gem_cleanup_engines(struct drm_device *dev)
 static void
 init_engine_lists(struct intel_engine_cs *engine)
 {
-	INIT_LIST_HEAD(&engine->active_list);
 	INIT_LIST_HEAD(&engine->request_list);
 }
 
@@ -5244,6 +4505,7 @@ void
 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
 {
 	struct drm_device *dev = &dev_priv->drm;
+	int i;
 
 	if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
 	    !IS_CHERRYVIEW(dev_priv))
@@ -5259,6 +4521,13 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
 				I915_READ(vgtif_reg(avail_rs.fence_num));
 
 	/* Initialize fence registers to zero */
+	for (i = 0; i < dev_priv->num_fence_regs; i++) {
+		struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
+
+		fence->i915 = dev_priv;
+		fence->id = i;
+		list_add_tail(&fence->link, &dev_priv->mm.fence_list);
+	}
 	i915_gem_restore_fences(dev);
 
 	i915_gem_detect_bit_6_swizzle(dev);
@@ -5283,18 +4552,17 @@ i915_gem_load_init(struct drm_device *dev)
 	dev_priv->requests =
 		kmem_cache_create("i915_gem_request",
 				  sizeof(struct drm_i915_gem_request), 0,
-				  SLAB_HWCACHE_ALIGN,
+				  SLAB_HWCACHE_ALIGN |
+				  SLAB_RECLAIM_ACCOUNT |
+				  SLAB_DESTROY_BY_RCU,
 				  NULL);
 
-	INIT_LIST_HEAD(&dev_priv->vm_list);
 	INIT_LIST_HEAD(&dev_priv->context_list);
 	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
 	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
 	for (i = 0; i < I915_NUM_ENGINES; i++)
 		init_engine_lists(&dev_priv->engine[i]);
-	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
-		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
 	INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
 			  i915_gem_retire_work_handler);
 	INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
@@ -5304,13 +4572,13 @@ i915_gem_load_init(struct drm_device *dev)
 
 	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
 
-	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
-
 	init_waitqueue_head(&dev_priv->pending_flip_queue);
 
 	dev_priv->mm.interruptible = true;
 
-	mutex_init(&dev_priv->fb_tracking.lock);
+	atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
+
+	spin_lock_init(&dev_priv->fb_tracking.lock);
 }
 
 void i915_gem_load_cleanup(struct drm_device *dev)
@@ -5320,11 +4588,32 @@ void i915_gem_load_cleanup(struct drm_device *dev)
 	kmem_cache_destroy(dev_priv->requests);
 	kmem_cache_destroy(dev_priv->vmas);
 	kmem_cache_destroy(dev_priv->objects);
+
+	/* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
+	rcu_barrier();
+}
+
+int i915_gem_freeze(struct drm_i915_private *dev_priv)
+{
+	intel_runtime_pm_get(dev_priv);
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	i915_gem_shrink_all(dev_priv);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+
+	intel_runtime_pm_put(dev_priv);
+
+	return 0;
 }
 
 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
 {
 	struct drm_i915_gem_object *obj;
+	struct list_head *phases[] = {
+		&dev_priv->mm.unbound_list,
+		&dev_priv->mm.bound_list,
+		NULL
+	}, **p;
 
 	/* Called just before we write the hibernation image.
 	 *
@@ -5335,17 +4624,21 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
 	 *
 	 * To make sure the hibernation image contains the latest state,
 	 * we update that state just before writing out the image.
+	 *
+	 * To try and reduce the hibernation image, we manually shrink
+	 * the objects as well.
 	 */
 
-	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
-		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
-		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	}
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND);
 
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
-		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	for (p = phases; *p; p++) {
+		list_for_each_entry(obj, *p, global_list) {
+			obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+			obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+		}
 	}
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	return 0;
 }
@@ -5353,21 +4646,15 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv = file->driver_priv;
+	struct drm_i915_gem_request *request;
 
 	/* Clean up our request list when the client is going away, so that
 	 * later retire_requests won't dereference our soon-to-be-gone
 	 * file_priv.
 	 */
 	spin_lock(&file_priv->mm.lock);
-	while (!list_empty(&file_priv->mm.request_list)) {
-		struct drm_i915_gem_request *request;
-
-		request = list_first_entry(&file_priv->mm.request_list,
-					   struct drm_i915_gem_request,
-					   client_list);
-		list_del(&request->client_list);
+	list_for_each_entry(request, &file_priv->mm.request_list, client_list)
 		request->file_priv = NULL;
-	}
 	spin_unlock(&file_priv->mm.lock);
 
 	if (!list_empty(&file_priv->rps.link)) {
@@ -5396,7 +4683,7 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file)
 	spin_lock_init(&file_priv->mm.lock);
 	INIT_LIST_HEAD(&file_priv->mm.request_list);
 
-	file_priv->bsd_ring = -1;
+	file_priv->bsd_engine = -1;
 
 	ret = i915_gem_context_open(dev, file);
 	if (ret)
@@ -5418,120 +4705,26 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
 		       struct drm_i915_gem_object *new,
 		       unsigned frontbuffer_bits)
 {
+	/* Control of individual bits within the mask are guarded by
+	 * the owning plane->mutex, i.e. we can never see concurrent
+	 * manipulation of individual bits. But since the bitfield as a whole
+	 * is updated using RMW, we need to use atomics in order to update
+	 * the bits.
+	 */
+	BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
+		     sizeof(atomic_t) * BITS_PER_BYTE);
+
 	if (old) {
-		WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
-		WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
-		old->frontbuffer_bits &= ~frontbuffer_bits;
+		WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
+		atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
 	}
 
 	if (new) {
-		WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
-		WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
-		new->frontbuffer_bits |= frontbuffer_bits;
+		WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
+		atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
 	}
 }
 
-/* All the new VM stuff */
-u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
-			struct i915_address_space *vm)
-{
-	struct drm_i915_private *dev_priv = to_i915(o->base.dev);
-	struct i915_vma *vma;
-
-	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
-
-	list_for_each_entry(vma, &o->vma_list, obj_link) {
-		if (vma->is_ggtt &&
-		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
-			continue;
-		if (vma->vm == vm)
-			return vma->node.start;
-	}
-
-	WARN(1, "%s vma for this object not found.\n",
-	     i915_is_ggtt(vm) ? "global" : "ppgtt");
-	return -1;
-}
-
-u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
-				  const struct i915_ggtt_view *view)
-{
-	struct i915_vma *vma;
-
-	list_for_each_entry(vma, &o->vma_list, obj_link)
-		if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
-			return vma->node.start;
-
-	WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
-	return -1;
-}
-
-bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
-			struct i915_address_space *vm)
-{
-	struct i915_vma *vma;
-
-	list_for_each_entry(vma, &o->vma_list, obj_link) {
-		if (vma->is_ggtt &&
-		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
-			continue;
-		if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
-			return true;
-	}
-
-	return false;
-}
-
-bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
-				  const struct i915_ggtt_view *view)
-{
-	struct i915_vma *vma;
-
-	list_for_each_entry(vma, &o->vma_list, obj_link)
-		if (vma->is_ggtt &&
-		    i915_ggtt_view_equal(&vma->ggtt_view, view) &&
-		    drm_mm_node_allocated(&vma->node))
-			return true;
-
-	return false;
-}
-
-bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
-{
-	struct i915_vma *vma;
-
-	list_for_each_entry(vma, &o->vma_list, obj_link)
-		if (drm_mm_node_allocated(&vma->node))
-			return true;
-
-	return false;
-}
-
-unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o)
-{
-	struct i915_vma *vma;
-
-	GEM_BUG_ON(list_empty(&o->vma_list));
-
-	list_for_each_entry(vma, &o->vma_list, obj_link) {
-		if (vma->is_ggtt &&
-		    vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
-			return vma->node.size;
-	}
-
-	return 0;
-}
-
-bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
-{
-	struct i915_vma *vma;
-	list_for_each_entry(vma, &obj->vma_list, obj_link)
-		if (vma->pin_count > 0)
-			return true;
-
-	return false;
-}
-
 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
 struct page *
 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
@@ -5584,6 +4777,6 @@ i915_gem_object_create_from_data(struct drm_device *dev,
 	return obj;
 
 fail:
-	drm_gem_object_unreference(&obj->base);
+	i915_gem_object_put(obj);
 	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
index 3752d5daa4b2..ed989596d9a3 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
@@ -41,15 +41,15 @@
 
 /**
  * i915_gem_batch_pool_init() - initialize a batch buffer pool
- * @dev: the drm device
+ * @engine: the associated request submission engine
  * @pool: the batch buffer pool
  */
-void i915_gem_batch_pool_init(struct drm_device *dev,
+void i915_gem_batch_pool_init(struct intel_engine_cs *engine,
 			      struct i915_gem_batch_pool *pool)
 {
 	int n;
 
-	pool->dev = dev;
+	pool->engine = engine;
 
 	for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
 		INIT_LIST_HEAD(&pool->cache_list[n]);
@@ -65,18 +65,17 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)
 {
 	int n;
 
-	WARN_ON(!mutex_is_locked(&pool->dev->struct_mutex));
+	lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
 
 	for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) {
-		while (!list_empty(&pool->cache_list[n])) {
-			struct drm_i915_gem_object *obj =
-				list_first_entry(&pool->cache_list[n],
-						 struct drm_i915_gem_object,
-						 batch_pool_link);
-
-			list_del(&obj->batch_pool_link);
-			drm_gem_object_unreference(&obj->base);
-		}
+		struct drm_i915_gem_object *obj, *next;
+
+		list_for_each_entry_safe(obj, next,
+					 &pool->cache_list[n],
+					 batch_pool_link)
+			i915_gem_object_put(obj);
+
+		INIT_LIST_HEAD(&pool->cache_list[n]);
 	}
 }
 
@@ -102,7 +101,7 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 	struct list_head *list;
 	int n;
 
-	WARN_ON(!mutex_is_locked(&pool->dev->struct_mutex));
+	lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
 
 	/* Compute a power-of-two bucket, but throw everything greater than
 	 * 16KiB into the same bucket: i.e. the the buckets hold objects of
@@ -115,13 +114,14 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 
 	list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
 		/* The batches are strictly LRU ordered */
-		if (tmp->active)
+		if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
+					     &tmp->base.dev->struct_mutex))
 			break;
 
 		/* While we're looping, do some clean up */
 		if (tmp->madv == __I915_MADV_PURGED) {
 			list_del(&tmp->batch_pool_link);
-			drm_gem_object_unreference(&tmp->base);
+			i915_gem_object_put(tmp);
 			continue;
 		}
 
@@ -134,7 +134,7 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 	if (obj == NULL) {
 		int ret;
 
-		obj = i915_gem_object_create(pool->dev, size);
+		obj = i915_gem_object_create(&pool->engine->i915->drm, size);
 		if (IS_ERR(obj))
 			return obj;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.h b/drivers/gpu/drm/i915/i915_gem_batch_pool.h
index 848e90703eed..10d5ac4c00d3 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.h
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.h
@@ -27,13 +27,15 @@
 
 #include "i915_drv.h"
 
+struct intel_engine_cs;
+
 struct i915_gem_batch_pool {
-	struct drm_device *dev;
+	struct intel_engine_cs *engine;
 	struct list_head cache_list[4];
 };
 
 /* i915_gem_batch_pool.c */
-void i915_gem_batch_pool_init(struct drm_device *dev,
+void i915_gem_batch_pool_init(struct intel_engine_cs *engine,
 			      struct i915_gem_batch_pool *pool);
 void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool);
 struct drm_i915_gem_object*
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 3c97f0e7a003..df10f4e95736 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -134,21 +134,6 @@ static int get_context_size(struct drm_i915_private *dev_priv)
 	return ret;
 }
 
-static void i915_gem_context_clean(struct i915_gem_context *ctx)
-{
-	struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
-	struct i915_vma *vma, *next;
-
-	if (!ppgtt)
-		return;
-
-	list_for_each_entry_safe(vma, next, &ppgtt->base.inactive_list,
-				 vm_link) {
-		if (WARN_ON(__i915_vma_unbind_no_wait(vma)))
-			break;
-	}
-}
-
 void i915_gem_context_free(struct kref *ctx_ref)
 {
 	struct i915_gem_context *ctx = container_of(ctx_ref, typeof(*ctx), ref);
@@ -156,13 +141,7 @@ void i915_gem_context_free(struct kref *ctx_ref)
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
 	trace_i915_context_free(ctx);
-
-	/*
-	 * This context is going away and we need to remove all VMAs still
-	 * around. This is to handle imported shared objects for which
-	 * destructor did not run when their handles were closed.
-	 */
-	i915_gem_context_clean(ctx);
+	GEM_BUG_ON(!ctx->closed);
 
 	i915_ppgtt_put(ctx->ppgtt);
 
@@ -173,12 +152,13 @@ void i915_gem_context_free(struct kref *ctx_ref)
 			continue;
 
 		WARN_ON(ce->pin_count);
-		if (ce->ringbuf)
-			intel_ringbuffer_free(ce->ringbuf);
+		if (ce->ring)
+			intel_ring_free(ce->ring);
 
-		drm_gem_object_unreference(&ce->state->base);
+		i915_vma_put(ce->state);
 	}
 
+	put_pid(ctx->pid);
 	list_del(&ctx->link);
 
 	ida_simple_remove(&ctx->i915->context_hw_ida, ctx->hw_id);
@@ -216,7 +196,7 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
 		ret = i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
 		/* Failure shouldn't ever happen this early */
 		if (WARN_ON(ret)) {
-			drm_gem_object_unreference(&obj->base);
+			i915_gem_object_put(obj);
 			return ERR_PTR(ret);
 		}
 	}
@@ -224,6 +204,37 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
 	return obj;
 }
 
+static void i915_ppgtt_close(struct i915_address_space *vm)
+{
+	struct list_head *phases[] = {
+		&vm->active_list,
+		&vm->inactive_list,
+		&vm->unbound_list,
+		NULL,
+	}, **phase;
+
+	GEM_BUG_ON(vm->closed);
+	vm->closed = true;
+
+	for (phase = phases; *phase; phase++) {
+		struct i915_vma *vma, *vn;
+
+		list_for_each_entry_safe(vma, vn, *phase, vm_link)
+			if (!i915_vma_is_closed(vma))
+				i915_vma_close(vma);
+	}
+}
+
+static void context_close(struct i915_gem_context *ctx)
+{
+	GEM_BUG_ON(ctx->closed);
+	ctx->closed = true;
+	if (ctx->ppgtt)
+		i915_ppgtt_close(&ctx->ppgtt->base);
+	ctx->file_priv = ERR_PTR(-EBADF);
+	i915_gem_context_put(ctx);
+}
+
 static int assign_hw_id(struct drm_i915_private *dev_priv, unsigned *out)
 {
 	int ret;
@@ -271,13 +282,24 @@ __create_hw_context(struct drm_device *dev,
 	ctx->ggtt_alignment = get_context_alignment(dev_priv);
 
 	if (dev_priv->hw_context_size) {
-		struct drm_i915_gem_object *obj =
-				i915_gem_alloc_context_obj(dev, dev_priv->hw_context_size);
+		struct drm_i915_gem_object *obj;
+		struct i915_vma *vma;
+
+		obj = i915_gem_alloc_context_obj(dev,
+						 dev_priv->hw_context_size);
 		if (IS_ERR(obj)) {
 			ret = PTR_ERR(obj);
 			goto err_out;
 		}
-		ctx->engine[RCS].state = obj;
+
+		vma = i915_vma_create(obj, &dev_priv->ggtt.base, NULL);
+		if (IS_ERR(vma)) {
+			i915_gem_object_put(obj);
+			ret = PTR_ERR(vma);
+			goto err_out;
+		}
+
+		ctx->engine[RCS].state = vma;
 	}
 
 	/* Default context will never have a file_priv */
@@ -290,6 +312,9 @@ __create_hw_context(struct drm_device *dev,
 		ret = DEFAULT_CONTEXT_HANDLE;
 
 	ctx->file_priv = file_priv;
+	if (file_priv)
+		ctx->pid = get_task_pid(current, PIDTYPE_PID);
+
 	ctx->user_handle = ret;
 	/* NB: Mark all slices as needing a remap so that when the context first
 	 * loads it will restore whatever remap state already exists. If there
@@ -305,7 +330,7 @@ __create_hw_context(struct drm_device *dev,
 	return ctx;
 
 err_out:
-	i915_gem_context_unreference(ctx);
+	context_close(ctx);
 	return ERR_PTR(ret);
 }
 
@@ -327,13 +352,14 @@ i915_gem_create_context(struct drm_device *dev,
 		return ctx;
 
 	if (USES_FULL_PPGTT(dev)) {
-		struct i915_hw_ppgtt *ppgtt = i915_ppgtt_create(dev, file_priv);
+		struct i915_hw_ppgtt *ppgtt =
+			i915_ppgtt_create(to_i915(dev), file_priv);
 
 		if (IS_ERR(ppgtt)) {
 			DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
 					 PTR_ERR(ppgtt));
 			idr_remove(&file_priv->context_idr, ctx->user_handle);
-			i915_gem_context_unreference(ctx);
+			context_close(ctx);
 			return ERR_CAST(ppgtt);
 		}
 
@@ -388,28 +414,12 @@ static void i915_gem_context_unpin(struct i915_gem_context *ctx,
 		struct intel_context *ce = &ctx->engine[engine->id];
 
 		if (ce->state)
-			i915_gem_object_ggtt_unpin(ce->state);
+			i915_vma_unpin(ce->state);
 
-		i915_gem_context_unreference(ctx);
+		i915_gem_context_put(ctx);
 	}
 }
 
-void i915_gem_context_reset(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	lockdep_assert_held(&dev->struct_mutex);
-
-	if (i915.enable_execlists) {
-		struct i915_gem_context *ctx;
-
-		list_for_each_entry(ctx, &dev_priv->context_list, link)
-			intel_lr_context_reset(dev_priv, ctx);
-	}
-
-	i915_gem_context_lost(dev_priv);
-}
-
 int i915_gem_context_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -504,7 +514,7 @@ void i915_gem_context_fini(struct drm_device *dev)
 
 	lockdep_assert_held(&dev->struct_mutex);
 
-	i915_gem_context_unreference(dctx);
+	context_close(dctx);
 	dev_priv->kernel_context = NULL;
 
 	ida_destroy(&dev_priv->context_hw_ida);
@@ -514,8 +524,7 @@ static int context_idr_cleanup(int id, void *p, void *data)
 {
 	struct i915_gem_context *ctx = p;
 
-	ctx->file_priv = ERR_PTR(-EBADF);
-	i915_gem_context_unreference(ctx);
+	context_close(ctx);
 	return 0;
 }
 
@@ -552,12 +561,13 @@ static inline int
 mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 {
 	struct drm_i915_private *dev_priv = req->i915;
+	struct intel_ring *ring = req->ring;
 	struct intel_engine_cs *engine = req->engine;
 	u32 flags = hw_flags | MI_MM_SPACE_GTT;
 	const int num_rings =
 		/* Use an extended w/a on ivb+ if signalling from other rings */
-		i915_semaphore_is_enabled(dev_priv) ?
-		hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1 :
+		i915.semaphores ?
+		INTEL_INFO(dev_priv)->num_rings - 1 :
 		0;
 	int len, ret;
 
@@ -567,7 +577,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 	 * itlb_before_ctx_switch.
 	 */
 	if (IS_GEN6(dev_priv)) {
-		ret = engine->flush(req, I915_GEM_GPU_DOMAINS, 0);
+		ret = engine->emit_flush(req, EMIT_INVALIDATE);
 		if (ret)
 			return ret;
 	}
@@ -589,64 +599,64 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 
 	/* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
 	if (INTEL_GEN(dev_priv) >= 7) {
-		intel_ring_emit(engine, MI_ARB_ON_OFF | MI_ARB_DISABLE);
+		intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_DISABLE);
 		if (num_rings) {
 			struct intel_engine_cs *signaller;
 
-			intel_ring_emit(engine,
+			intel_ring_emit(ring,
 					MI_LOAD_REGISTER_IMM(num_rings));
 			for_each_engine(signaller, dev_priv) {
 				if (signaller == engine)
 					continue;
 
-				intel_ring_emit_reg(engine,
+				intel_ring_emit_reg(ring,
 						    RING_PSMI_CTL(signaller->mmio_base));
-				intel_ring_emit(engine,
+				intel_ring_emit(ring,
 						_MASKED_BIT_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
 			}
 		}
 	}
 
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_emit(engine, MI_SET_CONTEXT);
-	intel_ring_emit(engine,
-			i915_gem_obj_ggtt_offset(req->ctx->engine[RCS].state) |
-			flags);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_emit(ring, MI_SET_CONTEXT);
+	intel_ring_emit(ring,
+			i915_ggtt_offset(req->ctx->engine[RCS].state) | flags);
 	/*
 	 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
 	 * WaMiSetContext_Hang:snb,ivb,vlv
 	 */
-	intel_ring_emit(engine, MI_NOOP);
+	intel_ring_emit(ring, MI_NOOP);
 
 	if (INTEL_GEN(dev_priv) >= 7) {
 		if (num_rings) {
 			struct intel_engine_cs *signaller;
 			i915_reg_t last_reg = {}; /* keep gcc quiet */
 
-			intel_ring_emit(engine,
+			intel_ring_emit(ring,
 					MI_LOAD_REGISTER_IMM(num_rings));
 			for_each_engine(signaller, dev_priv) {
 				if (signaller == engine)
 					continue;
 
 				last_reg = RING_PSMI_CTL(signaller->mmio_base);
-				intel_ring_emit_reg(engine, last_reg);
-				intel_ring_emit(engine,
+				intel_ring_emit_reg(ring, last_reg);
+				intel_ring_emit(ring,
 						_MASKED_BIT_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
 			}
 
 			/* Insert a delay before the next switch! */
-			intel_ring_emit(engine,
+			intel_ring_emit(ring,
 					MI_STORE_REGISTER_MEM |
 					MI_SRM_LRM_GLOBAL_GTT);
-			intel_ring_emit_reg(engine, last_reg);
-			intel_ring_emit(engine, engine->scratch.gtt_offset);
-			intel_ring_emit(engine, MI_NOOP);
+			intel_ring_emit_reg(ring, last_reg);
+			intel_ring_emit(ring,
+					i915_ggtt_offset(engine->scratch));
+			intel_ring_emit(ring, MI_NOOP);
 		}
-		intel_ring_emit(engine, MI_ARB_ON_OFF | MI_ARB_ENABLE);
+		intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_ENABLE);
 	}
 
-	intel_ring_advance(engine);
+	intel_ring_advance(ring);
 
 	return ret;
 }
@@ -654,7 +664,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 static int remap_l3(struct drm_i915_gem_request *req, int slice)
 {
 	u32 *remap_info = req->i915->l3_parity.remap_info[slice];
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	int i, ret;
 
 	if (!remap_info)
@@ -669,13 +679,13 @@ static int remap_l3(struct drm_i915_gem_request *req, int slice)
 	 * here because no other code should access these registers other than
 	 * at initialization time.
 	 */
-	intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4));
+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4));
 	for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) {
-		intel_ring_emit_reg(engine, GEN7_L3LOG(slice, i));
-		intel_ring_emit(engine, remap_info[i]);
+		intel_ring_emit_reg(ring, GEN7_L3LOG(slice, i));
+		intel_ring_emit(ring, remap_info[i]);
 	}
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -744,6 +754,7 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 	struct i915_gem_context *to = req->ctx;
 	struct intel_engine_cs *engine = req->engine;
 	struct i915_hw_ppgtt *ppgtt = to->ppgtt ?: req->i915->mm.aliasing_ppgtt;
+	struct i915_vma *vma = to->engine[RCS].state;
 	struct i915_gem_context *from;
 	u32 hw_flags;
 	int ret, i;
@@ -751,10 +762,15 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 	if (skip_rcs_switch(ppgtt, engine, to))
 		return 0;
 
+	/* Clear this page out of any CPU caches for coherent swap-in/out. */
+	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
+		ret = i915_gem_object_set_to_gtt_domain(vma->obj, false);
+		if (ret)
+			return ret;
+	}
+
 	/* Trying to pin first makes error handling easier. */
-	ret = i915_gem_obj_ggtt_pin(to->engine[RCS].state,
-				    to->ggtt_alignment,
-				    0);
+	ret = i915_vma_pin(vma, 0, to->ggtt_alignment, PIN_GLOBAL);
 	if (ret)
 		return ret;
 
@@ -767,18 +783,6 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 	 */
 	from = engine->last_context;
 
-	/*
-	 * Clear this page out of any CPU caches for coherent swap-in/out. Note
-	 * that thanks to write = false in this call and us not setting any gpu
-	 * write domains when putting a context object onto the active list
-	 * (when switching away from it), this won't block.
-	 *
-	 * XXX: We need a real interface to do this instead of trickery.
-	 */
-	ret = i915_gem_object_set_to_gtt_domain(to->engine[RCS].state, false);
-	if (ret)
-		goto unpin_out;
-
 	if (needs_pd_load_pre(ppgtt, engine, to)) {
 		/* Older GENs and non render rings still want the load first,
 		 * "PP_DCLV followed by PP_DIR_BASE register through Load
@@ -787,7 +791,7 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 		trace_switch_mm(engine, to);
 		ret = ppgtt->switch_mm(ppgtt, req);
 		if (ret)
-			goto unpin_out;
+			goto err;
 	}
 
 	if (!to->engine[RCS].initialised || i915_gem_context_is_default(to))
@@ -804,7 +808,7 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 	if (to != from || (hw_flags & MI_FORCE_RESTORE)) {
 		ret = mi_set_context(req, hw_flags);
 		if (ret)
-			goto unpin_out;
+			goto err;
 	}
 
 	/* The backing object for the context is done after switching to the
@@ -814,8 +818,6 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 	 * MI_SET_CONTEXT instead of when the next seqno has completed.
 	 */
 	if (from != NULL) {
-		from->engine[RCS].state->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
-		i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->engine[RCS].state), req);
 		/* As long as MI_SET_CONTEXT is serializing, ie. it flushes the
 		 * whole damn pipeline, we don't need to explicitly mark the
 		 * object dirty. The only exception is that the context must be
@@ -823,14 +825,12 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 		 * able to defer doing this until we know the object would be
 		 * swapped, but there is no way to do that yet.
 		 */
-		from->engine[RCS].state->dirty = 1;
-
-		/* obj is kept alive until the next request by its active ref */
-		i915_gem_object_ggtt_unpin(from->engine[RCS].state);
-		i915_gem_context_unreference(from);
+		i915_vma_move_to_active(from->engine[RCS].state, req, 0);
+		/* state is kept alive until the next request */
+		i915_vma_unpin(from->engine[RCS].state);
+		i915_gem_context_put(from);
 	}
-	i915_gem_context_reference(to);
-	engine->last_context = to;
+	engine->last_context = i915_gem_context_get(to);
 
 	/* GEN8 does *not* require an explicit reload if the PDPs have been
 	 * setup, and we do not wish to move them.
@@ -872,8 +872,8 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 
 	return 0;
 
-unpin_out:
-	i915_gem_object_ggtt_unpin(to->engine[RCS].state);
+err:
+	i915_vma_unpin(vma);
 	return ret;
 }
 
@@ -894,8 +894,9 @@ int i915_switch_context(struct drm_i915_gem_request *req)
 {
 	struct intel_engine_cs *engine = req->engine;
 
-	WARN_ON(i915.enable_execlists);
 	lockdep_assert_held(&req->i915->drm.struct_mutex);
+	if (i915.enable_execlists)
+		return 0;
 
 	if (!req->ctx->engine[engine->id].state) {
 		struct i915_gem_context *to = req->ctx;
@@ -914,10 +915,9 @@ int i915_switch_context(struct drm_i915_gem_request *req)
 		}
 
 		if (to != engine->last_context) {
-			i915_gem_context_reference(to);
 			if (engine->last_context)
-				i915_gem_context_unreference(engine->last_context);
-			engine->last_context = to;
+				i915_gem_context_put(engine->last_context);
+			engine->last_context = i915_gem_context_get(to);
 		}
 
 		return 0;
@@ -926,6 +926,33 @@ int i915_switch_context(struct drm_i915_gem_request *req)
 	return do_rcs_switch(req);
 }
 
+int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+
+	for_each_engine(engine, dev_priv) {
+		struct drm_i915_gem_request *req;
+		int ret;
+
+		if (engine->last_context == NULL)
+			continue;
+
+		if (engine->last_context == dev_priv->kernel_context)
+			continue;
+
+		req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
+		if (IS_ERR(req))
+			return PTR_ERR(req);
+
+		ret = i915_switch_context(req);
+		i915_add_request_no_flush(req);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static bool contexts_enabled(struct drm_device *dev)
 {
 	return i915.enable_execlists || to_i915(dev)->hw_context_size;
@@ -985,7 +1012,7 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
 	}
 
 	idr_remove(&file_priv->context_idr, ctx->user_handle);
-	i915_gem_context_unreference(ctx);
+	context_close(ctx);
 	mutex_unlock(&dev->struct_mutex);
 
 	DRM_DEBUG_DRIVER("HW context %d destroyed\n", args->ctx_id);
diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c
deleted file mode 100644
index a56516482394..000000000000
--- a/drivers/gpu/drm/i915/i915_gem_debug.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright © 2008 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- *
- * Authors:
- *    Keith Packard <keithp@keithp.com>
- *
- */
-
-#include <drm/drmP.h>
-#include <drm/i915_drm.h>
-#include "i915_drv.h"
-
-#if WATCH_LISTS
-int
-i915_verify_lists(struct drm_device *dev)
-{
-	static int warned;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_i915_gem_object *obj;
-	struct intel_engine_cs *engine;
-	int err = 0;
-
-	if (warned)
-		return 0;
-
-	for_each_engine(engine, dev_priv) {
-		list_for_each_entry(obj, &engine->active_list,
-				    engine_list[engine->id]) {
-			if (obj->base.dev != dev ||
-			    !atomic_read(&obj->base.refcount.refcount)) {
-				DRM_ERROR("%s: freed active obj %p\n",
-					  engine->name, obj);
-				err++;
-				break;
-			} else if (!obj->active ||
-				   obj->last_read_req[engine->id] == NULL) {
-				DRM_ERROR("%s: invalid active obj %p\n",
-					  engine->name, obj);
-				err++;
-			} else if (obj->base.write_domain) {
-				DRM_ERROR("%s: invalid write obj %p (w %x)\n",
-					  engine->name,
-					  obj, obj->base.write_domain);
-				err++;
-			}
-		}
-	}
-
-	return warned = err;
-}
-#endif /* WATCH_LIST */
diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
index 80bbe43a2e92..97c9d68b45df 100644
--- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
@@ -23,9 +23,13 @@
  * Authors:
  *	Dave Airlie <airlied@redhat.com>
  */
+
+#include <linux/dma-buf.h>
+#include <linux/reservation.h>
+
 #include <drm/drmP.h>
+
 #include "i915_drv.h"
-#include <linux/dma-buf.h>
 
 static struct drm_i915_gem_object *dma_buf_to_obj(struct dma_buf *buf)
 {
@@ -115,7 +119,7 @@ static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf)
 	if (ret)
 		return ERR_PTR(ret);
 
-	addr = i915_gem_object_pin_map(obj);
+	addr = i915_gem_object_pin_map(obj, I915_MAP_WB);
 	mutex_unlock(&dev->struct_mutex);
 
 	return addr;
@@ -218,25 +222,73 @@ static const struct dma_buf_ops i915_dmabuf_ops =  {
 	.end_cpu_access = i915_gem_end_cpu_access,
 };
 
+static void export_fences(struct drm_i915_gem_object *obj,
+			  struct dma_buf *dma_buf)
+{
+	struct reservation_object *resv = dma_buf->resv;
+	struct drm_i915_gem_request *req;
+	unsigned long active;
+	int idx;
+
+	active = __I915_BO_ACTIVE(obj);
+	if (!active)
+		return;
+
+	/* Serialise with execbuf to prevent concurrent fence-loops */
+	mutex_lock(&obj->base.dev->struct_mutex);
+
+	/* Mark the object for future fences before racily adding old fences */
+	obj->base.dma_buf = dma_buf;
+
+	ww_mutex_lock(&resv->lock, NULL);
+
+	for_each_active(active, idx) {
+		req = i915_gem_active_get(&obj->last_read[idx],
+					  &obj->base.dev->struct_mutex);
+		if (!req)
+			continue;
+
+		if (reservation_object_reserve_shared(resv) == 0)
+			reservation_object_add_shared_fence(resv, &req->fence);
+
+		i915_gem_request_put(req);
+	}
+
+	req = i915_gem_active_get(&obj->last_write,
+				  &obj->base.dev->struct_mutex);
+	if (req) {
+		reservation_object_add_excl_fence(resv, &req->fence);
+		i915_gem_request_put(req);
+	}
+
+	ww_mutex_unlock(&resv->lock);
+	mutex_unlock(&obj->base.dev->struct_mutex);
+}
+
 struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
 				      struct drm_gem_object *gem_obj, int flags)
 {
 	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	struct dma_buf *dma_buf;
 
 	exp_info.ops = &i915_dmabuf_ops;
 	exp_info.size = gem_obj->size;
 	exp_info.flags = flags;
 	exp_info.priv = gem_obj;
 
-
 	if (obj->ops->dmabuf_export) {
 		int ret = obj->ops->dmabuf_export(obj);
 		if (ret)
 			return ERR_PTR(ret);
 	}
 
-	return dma_buf_export(&exp_info);
+	dma_buf = drm_gem_dmabuf_export(dev, &exp_info);
+	if (IS_ERR(dma_buf))
+		return dma_buf;
+
+	export_fences(obj, dma_buf);
+	return dma_buf;
 }
 
 static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
@@ -278,8 +330,7 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
 			 * Importing dmabuf exported from out own gem increases
 			 * refcount on gem itself instead of f_count of dmabuf.
 			 */
-			drm_gem_object_reference(&obj->base);
-			return &obj->base;
+			return &i915_gem_object_get(obj)->base;
 		}
 	}
 
@@ -300,6 +351,16 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
 	i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops);
 	obj->base.import_attach = attach;
 
+	/* We use GTT as shorthand for a coherent domain, one that is
+	 * neither in the GPU cache nor in the CPU cache, where all
+	 * writes are immediately visible in memory. (That's not strictly
+	 * true, but it's close! There are internal buffers such as the
+	 * write-combined buffer or a delay through the chipset for GTT
+	 * writes that do require us to treat GTT as a separate cache domain.)
+	 */
+	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
+	obj->base.write_domain = 0;
+
 	return &obj->base;
 
 fail_detach:
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 3c1280ec7ff6..5b6f81c1dbca 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -33,53 +33,37 @@
 #include "intel_drv.h"
 #include "i915_trace.h"
 
-static int switch_to_pinned_context(struct drm_i915_private *dev_priv)
+static bool
+gpu_is_idle(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 
-	if (i915.enable_execlists)
-		return 0;
-
 	for_each_engine(engine, dev_priv) {
-		struct drm_i915_gem_request *req;
-		int ret;
-
-		if (engine->last_context == NULL)
-			continue;
-
-		if (engine->last_context == dev_priv->kernel_context)
-			continue;
-
-		req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
-		if (IS_ERR(req))
-			return PTR_ERR(req);
-
-		ret = i915_switch_context(req);
-		i915_add_request_no_flush(req);
-		if (ret)
-			return ret;
+		if (intel_engine_is_active(engine))
+			return false;
 	}
 
-	return 0;
+	return true;
 }
 
-
 static bool
-mark_free(struct i915_vma *vma, struct list_head *unwind)
+mark_free(struct i915_vma *vma, unsigned int flags, struct list_head *unwind)
 {
-	if (vma->pin_count)
+	if (i915_vma_is_pinned(vma))
 		return false;
 
 	if (WARN_ON(!list_empty(&vma->exec_list)))
 		return false;
 
+	if (flags & PIN_NONFAULT && vma->obj->fault_mappable)
+		return false;
+
 	list_add(&vma->exec_list, unwind);
 	return drm_mm_scan_add_block(&vma->node);
 }
 
 /**
  * i915_gem_evict_something - Evict vmas to make room for binding a new one
- * @dev: drm_device
  * @vm: address space to evict from
  * @min_size: size of the desired free space
  * @alignment: alignment constraint of the desired free space
@@ -102,42 +86,37 @@ mark_free(struct i915_vma *vma, struct list_head *unwind)
  * memory in e.g. the shrinker.
  */
 int
-i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm,
-			 int min_size, unsigned alignment, unsigned cache_level,
-			 unsigned long start, unsigned long end,
+i915_gem_evict_something(struct i915_address_space *vm,
+			 u64 min_size, u64 alignment,
+			 unsigned cache_level,
+			 u64 start, u64 end,
 			 unsigned flags)
 {
-	struct list_head eviction_list, unwind_list;
-	struct i915_vma *vma;
-	int ret = 0;
-	int pass = 0;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
+	struct list_head eviction_list;
+	struct list_head *phases[] = {
+		&vm->inactive_list,
+		&vm->active_list,
+		NULL,
+	}, **phase;
+	struct i915_vma *vma, *next;
+	int ret;
 
-	trace_i915_gem_evict(dev, min_size, alignment, flags);
+	trace_i915_gem_evict(vm, min_size, alignment, flags);
 
 	/*
 	 * The goal is to evict objects and amalgamate space in LRU order.
 	 * The oldest idle objects reside on the inactive list, which is in
-	 * retirement order. The next objects to retire are those on the (per
-	 * ring) active list that do not have an outstanding flush. Once the
-	 * hardware reports completion (the seqno is updated after the
-	 * batchbuffer has been finished) the clean buffer objects would
-	 * be retired to the inactive list. Any dirty objects would be added
-	 * to the tail of the flushing list. So after processing the clean
-	 * active objects we need to emit a MI_FLUSH to retire the flushing
-	 * list, hence the retirement order of the flushing list is in
-	 * advance of the dirty objects on the active lists.
+	 * retirement order. The next objects to retire are those in flight,
+	 * on the active list, again in retirement order.
 	 *
 	 * The retirement sequence is thus:
 	 *   1. Inactive objects (already retired)
-	 *   2. Clean active objects
-	 *   3. Flushing list
-	 *   4. Dirty active objects.
+	 *   2. Active objects (will stall on unbinding)
 	 *
 	 * On each list, the oldest objects lie at the HEAD with the freshest
 	 * object on the TAIL.
 	 */
-
-	INIT_LIST_HEAD(&unwind_list);
 	if (start != 0 || end != vm->total) {
 		drm_mm_init_scan_with_range(&vm->mm, min_size,
 					    alignment, cache_level,
@@ -145,96 +124,86 @@ i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm,
 	} else
 		drm_mm_init_scan(&vm->mm, min_size, alignment, cache_level);
 
-search_again:
-	/* First see if there is a large enough contiguous idle region... */
-	list_for_each_entry(vma, &vm->inactive_list, vm_link) {
-		if (mark_free(vma, &unwind_list))
-			goto found;
-	}
-
 	if (flags & PIN_NONBLOCK)
-		goto none;
+		phases[1] = NULL;
 
-	/* Now merge in the soon-to-be-expired objects... */
-	list_for_each_entry(vma, &vm->active_list, vm_link) {
-		if (mark_free(vma, &unwind_list))
-			goto found;
-	}
+search_again:
+	INIT_LIST_HEAD(&eviction_list);
+	phase = phases;
+	do {
+		list_for_each_entry(vma, *phase, vm_link)
+			if (mark_free(vma, flags, &eviction_list))
+				goto found;
+	} while (*++phase);
 
-none:
 	/* Nothing found, clean up and bail out! */
-	while (!list_empty(&unwind_list)) {
-		vma = list_first_entry(&unwind_list,
-				       struct i915_vma,
-				       exec_list);
+	list_for_each_entry_safe(vma, next, &eviction_list, exec_list) {
 		ret = drm_mm_scan_remove_block(&vma->node);
 		BUG_ON(ret);
 
-		list_del_init(&vma->exec_list);
+		INIT_LIST_HEAD(&vma->exec_list);
 	}
 
 	/* Can we unpin some objects such as idle hw contents,
-	 * or pending flips?
+	 * or pending flips? But since only the GGTT has global entries
+	 * such as scanouts, rinbuffers and contexts, we can skip the
+	 * purge when inspecting per-process local address spaces.
 	 */
-	if (flags & PIN_NONBLOCK)
+	if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK)
 		return -ENOSPC;
 
-	/* Only idle the GPU and repeat the search once */
-	if (pass++ == 0) {
-		struct drm_i915_private *dev_priv = to_i915(dev);
-
-		if (i915_is_ggtt(vm)) {
-			ret = switch_to_pinned_context(dev_priv);
-			if (ret)
-				return ret;
-		}
-
-		ret = i915_gem_wait_for_idle(dev_priv);
-		if (ret)
-			return ret;
-
-		i915_gem_retire_requests(dev_priv);
-		goto search_again;
+	if (gpu_is_idle(dev_priv)) {
+		/* If we still have pending pageflip completions, drop
+		 * back to userspace to give our workqueues time to
+		 * acquire our locks and unpin the old scanouts.
+		 */
+		return intel_has_pending_fb_unpin(vm->dev) ? -EAGAIN : -ENOSPC;
 	}
 
-	/* If we still have pending pageflip completions, drop
-	 * back to userspace to give our workqueues time to
-	 * acquire our locks and unpin the old scanouts.
+	/* Not everything in the GGTT is tracked via vma (otherwise we
+	 * could evict as required with minimal stalling) so we are forced
+	 * to idle the GPU and explicitly retire outstanding requests in
+	 * the hopes that we can then remove contexts and the like only
+	 * bound by their active reference.
 	 */
-	return intel_has_pending_fb_unpin(dev) ? -EAGAIN : -ENOSPC;
+	ret = i915_gem_switch_to_kernel_context(dev_priv);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_wait_for_idle(dev_priv,
+				     I915_WAIT_INTERRUPTIBLE |
+				     I915_WAIT_LOCKED);
+	if (ret)
+		return ret;
+
+	i915_gem_retire_requests(dev_priv);
+	goto search_again;
 
 found:
 	/* drm_mm doesn't allow any other other operations while
-	 * scanning, therefore store to be evicted objects on a
-	 * temporary list. */
-	INIT_LIST_HEAD(&eviction_list);
-	while (!list_empty(&unwind_list)) {
-		vma = list_first_entry(&unwind_list,
-				       struct i915_vma,
-				       exec_list);
-		if (drm_mm_scan_remove_block(&vma->node)) {
-			list_move(&vma->exec_list, &eviction_list);
-			drm_gem_object_reference(&vma->obj->base);
-			continue;
-		}
-		list_del_init(&vma->exec_list);
+	 * scanning, therefore store to-be-evicted objects on a
+	 * temporary list and take a reference for all before
+	 * calling unbind (which may remove the active reference
+	 * of any of our objects, thus corrupting the list).
+	 */
+	list_for_each_entry_safe(vma, next, &eviction_list, exec_list) {
+		if (drm_mm_scan_remove_block(&vma->node))
+			__i915_vma_pin(vma);
+		else
+			list_del_init(&vma->exec_list);
 	}
 
 	/* Unbinding will emit any required flushes */
 	while (!list_empty(&eviction_list)) {
-		struct drm_gem_object *obj;
 		vma = list_first_entry(&eviction_list,
 				       struct i915_vma,
 				       exec_list);
 
-		obj =  &vma->obj->base;
 		list_del_init(&vma->exec_list);
+		__i915_vma_unpin(vma);
 		if (ret == 0)
 			ret = i915_vma_unbind(vma);
-
-		drm_gem_object_unreference(obj);
 	}
-
 	return ret;
 }
 
@@ -256,8 +225,8 @@ i915_gem_evict_for_vma(struct i915_vma *target)
 
 		vma = container_of(node, typeof(*vma), node);
 
-		if (vma->pin_count) {
-			if (!vma->exec_entry || (vma->pin_count > 1))
+		if (i915_vma_is_pinned(vma)) {
+			if (!vma->exec_entry || i915_vma_pin_count(vma) > 1)
 				/* Object is pinned for some other use */
 				return -EBUSY;
 
@@ -303,22 +272,23 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
 		struct drm_i915_private *dev_priv = to_i915(vm->dev);
 
 		if (i915_is_ggtt(vm)) {
-			ret = switch_to_pinned_context(dev_priv);
+			ret = i915_gem_switch_to_kernel_context(dev_priv);
 			if (ret)
 				return ret;
 		}
 
-		ret = i915_gem_wait_for_idle(dev_priv);
+		ret = i915_gem_wait_for_idle(dev_priv,
+					     I915_WAIT_INTERRUPTIBLE |
+					     I915_WAIT_LOCKED);
 		if (ret)
 			return ret;
 
 		i915_gem_retire_requests(dev_priv);
-
 		WARN_ON(!list_empty(&vm->active_list));
 	}
 
 	list_for_each_entry_safe(vma, next, &vm->inactive_list, vm_link)
-		if (vma->pin_count == 0)
+		if (!i915_vma_is_pinned(vma))
 			WARN_ON(i915_vma_unbind(vma));
 
 	return 0;
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 1978633e7549..7adb4c77cc7f 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -26,22 +26,42 @@
  *
  */
 
+#include <linux/dma_remapping.h>
+#include <linux/reservation.h>
+#include <linux/uaccess.h>
+
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
+
 #include "i915_drv.h"
+#include "i915_gem_dmabuf.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
-#include <linux/dma_remapping.h>
-#include <linux/uaccess.h>
+#include "intel_frontbuffer.h"
+
+#define DBG_USE_CPU_RELOC 0 /* -1 force GTT relocs; 1 force CPU relocs */
 
-#define  __EXEC_OBJECT_HAS_PIN (1<<31)
-#define  __EXEC_OBJECT_HAS_FENCE (1<<30)
-#define  __EXEC_OBJECT_NEEDS_MAP (1<<29)
-#define  __EXEC_OBJECT_NEEDS_BIAS (1<<28)
+#define  __EXEC_OBJECT_HAS_PIN		(1<<31)
+#define  __EXEC_OBJECT_HAS_FENCE	(1<<30)
+#define  __EXEC_OBJECT_NEEDS_MAP	(1<<29)
+#define  __EXEC_OBJECT_NEEDS_BIAS	(1<<28)
+#define  __EXEC_OBJECT_INTERNAL_FLAGS (0xf<<28) /* all of the above */
 
 #define BATCH_OFFSET_BIAS (256*1024)
 
+struct i915_execbuffer_params {
+	struct drm_device               *dev;
+	struct drm_file                 *file;
+	struct i915_vma			*batch;
+	u32				dispatch_flags;
+	u32				args_batch_start_offset;
+	struct intel_engine_cs          *engine;
+	struct i915_gem_context         *ctx;
+	struct drm_i915_gem_request     *request;
+};
+
 struct eb_vmas {
+	struct drm_i915_private *i915;
 	struct list_head vmas;
 	int and;
 	union {
@@ -51,7 +71,8 @@ struct eb_vmas {
 };
 
 static struct eb_vmas *
-eb_create(struct drm_i915_gem_execbuffer2 *args)
+eb_create(struct drm_i915_private *i915,
+	  struct drm_i915_gem_execbuffer2 *args)
 {
 	struct eb_vmas *eb = NULL;
 
@@ -78,6 +99,7 @@ eb_create(struct drm_i915_gem_execbuffer2 *args)
 	} else
 		eb->and = -args->buffer_count;
 
+	eb->i915 = i915;
 	INIT_LIST_HEAD(&eb->vmas);
 	return eb;
 }
@@ -89,6 +111,26 @@ eb_reset(struct eb_vmas *eb)
 		memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
 }
 
+static struct i915_vma *
+eb_get_batch(struct eb_vmas *eb)
+{
+	struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
+
+	/*
+	 * SNA is doing fancy tricks with compressing batch buffers, which leads
+	 * to negative relocation deltas. Usually that works out ok since the
+	 * relocate address is still positive, except when the batch is placed
+	 * very low in the GTT. Ensure this doesn't happen.
+	 *
+	 * Note that actual hangs have only been observed on gen7, but for
+	 * paranoia do it everywhere.
+	 */
+	if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
+		vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
+
+	return vma;
+}
+
 static int
 eb_lookup_vmas(struct eb_vmas *eb,
 	       struct drm_i915_gem_exec_object2 *exec,
@@ -122,7 +164,7 @@ eb_lookup_vmas(struct eb_vmas *eb,
 			goto err;
 		}
 
-		drm_gem_object_reference(&obj->base);
+		i915_gem_object_get(obj);
 		list_add_tail(&obj->obj_exec_link, &objects);
 	}
 	spin_unlock(&file->table_lock);
@@ -143,8 +185,8 @@ eb_lookup_vmas(struct eb_vmas *eb,
 		 * from the (obj, vm) we don't run the risk of creating
 		 * duplicated vmas for the same vm.
 		 */
-		vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
-		if (IS_ERR(vma)) {
+		vma = i915_gem_obj_lookup_or_create_vma(obj, vm, NULL);
+		if (unlikely(IS_ERR(vma))) {
 			DRM_DEBUG("Failed to lookup VMA\n");
 			ret = PTR_ERR(vma);
 			goto err;
@@ -175,7 +217,7 @@ err:
 				       struct drm_i915_gem_object,
 				       obj_exec_link);
 		list_del_init(&obj->obj_exec_link);
-		drm_gem_object_unreference(&obj->base);
+		i915_gem_object_put(obj);
 	}
 	/*
 	 * Objects already transfered to the vmas list will be unreferenced by
@@ -208,7 +250,6 @@ static void
 i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
 {
 	struct drm_i915_gem_exec_object2 *entry;
-	struct drm_i915_gem_object *obj = vma->obj;
 
 	if (!drm_mm_node_allocated(&vma->node))
 		return;
@@ -216,10 +257,10 @@ i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
 	entry = vma->exec_entry;
 
 	if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
-		i915_gem_object_unpin_fence(obj);
+		i915_vma_unpin_fence(vma);
 
 	if (entry->flags & __EXEC_OBJECT_HAS_PIN)
-		vma->pin_count--;
+		__i915_vma_unpin(vma);
 
 	entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
 }
@@ -234,13 +275,19 @@ static void eb_destroy(struct eb_vmas *eb)
 				       exec_list);
 		list_del_init(&vma->exec_list);
 		i915_gem_execbuffer_unreserve_vma(vma);
-		drm_gem_object_unreference(&vma->obj->base);
+		i915_vma_put(vma);
 	}
 	kfree(eb);
 }
 
 static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
 {
+	if (!i915_gem_object_has_struct_page(obj))
+		return false;
+
+	if (DBG_USE_CPU_RELOC)
+		return DBG_USE_CPU_RELOC > 0;
+
 	return (HAS_LLC(obj->base.dev) ||
 		obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
 		obj->cache_level != I915_CACHE_NONE);
@@ -265,144 +312,265 @@ static inline uint64_t gen8_noncanonical_addr(uint64_t address)
 }
 
 static inline uint64_t
-relocation_target(struct drm_i915_gem_relocation_entry *reloc,
+relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
 		  uint64_t target_offset)
 {
 	return gen8_canonical_addr((int)reloc->delta + target_offset);
 }
 
-static int
-relocate_entry_cpu(struct drm_i915_gem_object *obj,
-		   struct drm_i915_gem_relocation_entry *reloc,
-		   uint64_t target_offset)
+struct reloc_cache {
+	struct drm_i915_private *i915;
+	struct drm_mm_node node;
+	unsigned long vaddr;
+	unsigned int page;
+	bool use_64bit_reloc;
+};
+
+static void reloc_cache_init(struct reloc_cache *cache,
+			     struct drm_i915_private *i915)
 {
-	struct drm_device *dev = obj->base.dev;
-	uint32_t page_offset = offset_in_page(reloc->offset);
-	uint64_t delta = relocation_target(reloc, target_offset);
-	char *vaddr;
-	int ret;
+	cache->page = -1;
+	cache->vaddr = 0;
+	cache->i915 = i915;
+	cache->use_64bit_reloc = INTEL_GEN(cache->i915) >= 8;
+	cache->node.allocated = false;
+}
 
-	ret = i915_gem_object_set_to_cpu_domain(obj, true);
-	if (ret)
-		return ret;
+static inline void *unmask_page(unsigned long p)
+{
+	return (void *)(uintptr_t)(p & PAGE_MASK);
+}
 
-	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
-				reloc->offset >> PAGE_SHIFT));
-	*(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta);
+static inline unsigned int unmask_flags(unsigned long p)
+{
+	return p & ~PAGE_MASK;
+}
+
+#define KMAP 0x4 /* after CLFLUSH_FLAGS */
+
+static void reloc_cache_fini(struct reloc_cache *cache)
+{
+	void *vaddr;
+
+	if (!cache->vaddr)
+		return;
 
-	if (INTEL_INFO(dev)->gen >= 8) {
-		page_offset = offset_in_page(page_offset + sizeof(uint32_t));
+	vaddr = unmask_page(cache->vaddr);
+	if (cache->vaddr & KMAP) {
+		if (cache->vaddr & CLFLUSH_AFTER)
+			mb();
 
-		if (page_offset == 0) {
-			kunmap_atomic(vaddr);
-			vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
-			    (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
+		kunmap_atomic(vaddr);
+		i915_gem_obj_finish_shmem_access((struct drm_i915_gem_object *)cache->node.mm);
+	} else {
+		wmb();
+		io_mapping_unmap_atomic((void __iomem *)vaddr);
+		if (cache->node.allocated) {
+			struct i915_ggtt *ggtt = &cache->i915->ggtt;
+
+			ggtt->base.clear_range(&ggtt->base,
+					       cache->node.start,
+					       cache->node.size,
+					       true);
+			drm_mm_remove_node(&cache->node);
+		} else {
+			i915_vma_unpin((struct i915_vma *)cache->node.mm);
 		}
+	}
+}
+
+static void *reloc_kmap(struct drm_i915_gem_object *obj,
+			struct reloc_cache *cache,
+			int page)
+{
+	void *vaddr;
+
+	if (cache->vaddr) {
+		kunmap_atomic(unmask_page(cache->vaddr));
+	} else {
+		unsigned int flushes;
+		int ret;
 
-		*(uint32_t *)(vaddr + page_offset) = upper_32_bits(delta);
+		ret = i915_gem_obj_prepare_shmem_write(obj, &flushes);
+		if (ret)
+			return ERR_PTR(ret);
+
+		BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
+		BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
+
+		cache->vaddr = flushes | KMAP;
+		cache->node.mm = (void *)obj;
+		if (flushes)
+			mb();
 	}
 
-	kunmap_atomic(vaddr);
+	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj, page));
+	cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;
+	cache->page = page;
 
-	return 0;
+	return vaddr;
 }
 
-static int
-relocate_entry_gtt(struct drm_i915_gem_object *obj,
-		   struct drm_i915_gem_relocation_entry *reloc,
-		   uint64_t target_offset)
+static void *reloc_iomap(struct drm_i915_gem_object *obj,
+			 struct reloc_cache *cache,
+			 int page)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	uint64_t delta = relocation_target(reloc, target_offset);
-	uint64_t offset;
-	void __iomem *reloc_page;
-	int ret;
+	struct i915_ggtt *ggtt = &cache->i915->ggtt;
+	unsigned long offset;
+	void *vaddr;
 
-	ret = i915_gem_object_set_to_gtt_domain(obj, true);
-	if (ret)
-		return ret;
+	if (cache->node.allocated) {
+		wmb();
+		ggtt->base.insert_page(&ggtt->base,
+				       i915_gem_object_get_dma_address(obj, page),
+				       cache->node.start, I915_CACHE_NONE, 0);
+		cache->page = page;
+		return unmask_page(cache->vaddr);
+	}
 
-	ret = i915_gem_object_put_fence(obj);
-	if (ret)
-		return ret;
+	if (cache->vaddr) {
+		io_mapping_unmap_atomic(unmask_page(cache->vaddr));
+	} else {
+		struct i915_vma *vma;
+		int ret;
 
-	/* Map the page containing the relocation we're going to perform.  */
-	offset = i915_gem_obj_ggtt_offset(obj);
-	offset += reloc->offset;
-	reloc_page = io_mapping_map_atomic_wc(ggtt->mappable,
-					      offset & PAGE_MASK);
-	iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
-
-	if (INTEL_INFO(dev)->gen >= 8) {
-		offset += sizeof(uint32_t);
-
-		if (offset_in_page(offset) == 0) {
-			io_mapping_unmap_atomic(reloc_page);
-			reloc_page =
-				io_mapping_map_atomic_wc(ggtt->mappable,
-							 offset);
+		if (use_cpu_reloc(obj))
+			return NULL;
+
+		ret = i915_gem_object_set_to_gtt_domain(obj, true);
+		if (ret)
+			return ERR_PTR(ret);
+
+		vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+					       PIN_MAPPABLE | PIN_NONBLOCK);
+		if (IS_ERR(vma)) {
+			memset(&cache->node, 0, sizeof(cache->node));
+			ret = drm_mm_insert_node_in_range_generic
+				(&ggtt->base.mm, &cache->node,
+				 4096, 0, 0,
+				 0, ggtt->mappable_end,
+				 DRM_MM_SEARCH_DEFAULT,
+				 DRM_MM_CREATE_DEFAULT);
+			if (ret) /* no inactive aperture space, use cpu reloc */
+				return NULL;
+		} else {
+			ret = i915_vma_put_fence(vma);
+			if (ret) {
+				i915_vma_unpin(vma);
+				return ERR_PTR(ret);
+			}
+
+			cache->node.start = vma->node.start;
+			cache->node.mm = (void *)vma;
 		}
+	}
 
-		iowrite32(upper_32_bits(delta),
-			  reloc_page + offset_in_page(offset));
+	offset = cache->node.start;
+	if (cache->node.allocated) {
+		ggtt->base.insert_page(&ggtt->base,
+				       i915_gem_object_get_dma_address(obj, page),
+				       offset, I915_CACHE_NONE, 0);
+	} else {
+		offset += page << PAGE_SHIFT;
 	}
 
-	io_mapping_unmap_atomic(reloc_page);
+	vaddr = io_mapping_map_atomic_wc(&cache->i915->ggtt.mappable, offset);
+	cache->page = page;
+	cache->vaddr = (unsigned long)vaddr;
 
-	return 0;
+	return vaddr;
 }
 
-static void
-clflush_write32(void *addr, uint32_t value)
+static void *reloc_vaddr(struct drm_i915_gem_object *obj,
+			 struct reloc_cache *cache,
+			 int page)
 {
-	/* This is not a fast path, so KISS. */
-	drm_clflush_virt_range(addr, sizeof(uint32_t));
-	*(uint32_t *)addr = value;
-	drm_clflush_virt_range(addr, sizeof(uint32_t));
+	void *vaddr;
+
+	if (cache->page == page) {
+		vaddr = unmask_page(cache->vaddr);
+	} else {
+		vaddr = NULL;
+		if ((cache->vaddr & KMAP) == 0)
+			vaddr = reloc_iomap(obj, cache, page);
+		if (!vaddr)
+			vaddr = reloc_kmap(obj, cache, page);
+	}
+
+	return vaddr;
 }
 
-static int
-relocate_entry_clflush(struct drm_i915_gem_object *obj,
-		       struct drm_i915_gem_relocation_entry *reloc,
-		       uint64_t target_offset)
+static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
 {
-	struct drm_device *dev = obj->base.dev;
-	uint32_t page_offset = offset_in_page(reloc->offset);
-	uint64_t delta = relocation_target(reloc, target_offset);
-	char *vaddr;
-	int ret;
+	if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) {
+		if (flushes & CLFLUSH_BEFORE) {
+			clflushopt(addr);
+			mb();
+		}
 
-	ret = i915_gem_object_set_to_gtt_domain(obj, true);
-	if (ret)
-		return ret;
+		*addr = value;
 
-	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
-				reloc->offset >> PAGE_SHIFT));
-	clflush_write32(vaddr + page_offset, lower_32_bits(delta));
+		/* Writes to the same cacheline are serialised by the CPU
+		 * (including clflush). On the write path, we only require
+		 * that it hits memory in an orderly fashion and place
+		 * mb barriers at the start and end of the relocation phase
+		 * to ensure ordering of clflush wrt to the system.
+		 */
+		if (flushes & CLFLUSH_AFTER)
+			clflushopt(addr);
+	} else
+		*addr = value;
+}
 
-	if (INTEL_INFO(dev)->gen >= 8) {
-		page_offset = offset_in_page(page_offset + sizeof(uint32_t));
+static int
+relocate_entry(struct drm_i915_gem_object *obj,
+	       const struct drm_i915_gem_relocation_entry *reloc,
+	       struct reloc_cache *cache,
+	       u64 target_offset)
+{
+	u64 offset = reloc->offset;
+	bool wide = cache->use_64bit_reloc;
+	void *vaddr;
+
+	target_offset = relocation_target(reloc, target_offset);
+repeat:
+	vaddr = reloc_vaddr(obj, cache, offset >> PAGE_SHIFT);
+	if (IS_ERR(vaddr))
+		return PTR_ERR(vaddr);
+
+	clflush_write32(vaddr + offset_in_page(offset),
+			lower_32_bits(target_offset),
+			cache->vaddr);
+
+	if (wide) {
+		offset += sizeof(u32);
+		target_offset >>= 32;
+		wide = false;
+		goto repeat;
+	}
 
-		if (page_offset == 0) {
-			kunmap_atomic(vaddr);
-			vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
-			    (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
-		}
+	return 0;
+}
 
-		clflush_write32(vaddr + page_offset, upper_32_bits(delta));
-	}
+static bool object_is_idle(struct drm_i915_gem_object *obj)
+{
+	unsigned long active = i915_gem_object_get_active(obj);
+	int idx;
 
-	kunmap_atomic(vaddr);
+	for_each_active(active, idx) {
+		if (!i915_gem_active_is_idle(&obj->last_read[idx],
+					     &obj->base.dev->struct_mutex))
+			return false;
+	}
 
-	return 0;
+	return true;
 }
 
 static int
 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 				   struct eb_vmas *eb,
-				   struct drm_i915_gem_relocation_entry *reloc)
+				   struct drm_i915_gem_relocation_entry *reloc,
+				   struct reloc_cache *cache)
 {
 	struct drm_device *dev = obj->base.dev;
 	struct drm_gem_object *target_obj;
@@ -465,7 +633,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 
 	/* Check that the relocation address is valid... */
 	if (unlikely(reloc->offset >
-		obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
+		     obj->base.size - (cache->use_64bit_reloc ? 8 : 4))) {
 		DRM_DEBUG("Relocation beyond object bounds: "
 			  "obj %p target %d offset %d size %d.\n",
 			  obj, reloc->target_handle,
@@ -482,26 +650,15 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 	}
 
 	/* We can't wait for rendering with pagefaults disabled */
-	if (obj->active && pagefault_disabled())
+	if (pagefault_disabled() && !object_is_idle(obj))
 		return -EFAULT;
 
-	if (use_cpu_reloc(obj))
-		ret = relocate_entry_cpu(obj, reloc, target_offset);
-	else if (obj->map_and_fenceable)
-		ret = relocate_entry_gtt(obj, reloc, target_offset);
-	else if (static_cpu_has(X86_FEATURE_CLFLUSH))
-		ret = relocate_entry_clflush(obj, reloc, target_offset);
-	else {
-		WARN_ONCE(1, "Impossible case in relocation handling\n");
-		ret = -ENODEV;
-	}
-
+	ret = relocate_entry(obj, reloc, cache, target_offset);
 	if (ret)
 		return ret;
 
 	/* and update the user's relocation entry */
 	reloc->presumed_offset = target_offset;
-
 	return 0;
 }
 
@@ -513,9 +670,11 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
 	struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
 	struct drm_i915_gem_relocation_entry __user *user_relocs;
 	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-	int remain, ret;
+	struct reloc_cache cache;
+	int remain, ret = 0;
 
 	user_relocs = u64_to_user_ptr(entry->relocs_ptr);
+	reloc_cache_init(&cache, eb->i915);
 
 	remain = entry->relocation_count;
 	while (remain) {
@@ -525,19 +684,23 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
 			count = ARRAY_SIZE(stack_reloc);
 		remain -= count;
 
-		if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
-			return -EFAULT;
+		if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]))) {
+			ret = -EFAULT;
+			goto out;
+		}
 
 		do {
 			u64 offset = r->presumed_offset;
 
-			ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
+			ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r, &cache);
 			if (ret)
-				return ret;
+				goto out;
 
 			if (r->presumed_offset != offset &&
-			    __put_user(r->presumed_offset, &user_relocs->presumed_offset)) {
-				return -EFAULT;
+			    __put_user(r->presumed_offset,
+				       &user_relocs->presumed_offset)) {
+				ret = -EFAULT;
+				goto out;
 			}
 
 			user_relocs++;
@@ -545,7 +708,9 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
 		} while (--count);
 	}
 
-	return 0;
+out:
+	reloc_cache_fini(&cache);
+	return ret;
 #undef N_RELOC
 }
 
@@ -555,15 +720,18 @@ i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
 				      struct drm_i915_gem_relocation_entry *relocs)
 {
 	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-	int i, ret;
+	struct reloc_cache cache;
+	int i, ret = 0;
 
+	reloc_cache_init(&cache, eb->i915);
 	for (i = 0; i < entry->relocation_count; i++) {
-		ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
+		ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i], &cache);
 		if (ret)
-			return ret;
+			break;
 	}
+	reloc_cache_fini(&cache);
 
-	return 0;
+	return ret;
 }
 
 static int
@@ -626,23 +794,27 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
 			flags |= PIN_HIGH;
 	}
 
-	ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
-	if ((ret == -ENOSPC  || ret == -E2BIG) &&
+	ret = i915_vma_pin(vma,
+			   entry->pad_to_size,
+			   entry->alignment,
+			   flags);
+	if ((ret == -ENOSPC || ret == -E2BIG) &&
 	    only_mappable_for_reloc(entry->flags))
-		ret = i915_gem_object_pin(obj, vma->vm,
-					  entry->alignment,
-					  flags & ~PIN_MAPPABLE);
+		ret = i915_vma_pin(vma,
+				   entry->pad_to_size,
+				   entry->alignment,
+				   flags & ~PIN_MAPPABLE);
 	if (ret)
 		return ret;
 
 	entry->flags |= __EXEC_OBJECT_HAS_PIN;
 
 	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
-		ret = i915_gem_object_get_fence(obj);
+		ret = i915_vma_get_fence(vma);
 		if (ret)
 			return ret;
 
-		if (i915_gem_object_pin_fence(obj))
+		if (i915_vma_pin_fence(vma))
 			entry->flags |= __EXEC_OBJECT_HAS_FENCE;
 	}
 
@@ -667,7 +839,7 @@ need_reloc_mappable(struct i915_vma *vma)
 	if (entry->relocation_count == 0)
 		return false;
 
-	if (!vma->is_ggtt)
+	if (!i915_vma_is_ggtt(vma))
 		return false;
 
 	/* See also use_cpu_reloc() */
@@ -684,14 +856,17 @@ static bool
 eb_vma_misplaced(struct i915_vma *vma)
 {
 	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-	struct drm_i915_gem_object *obj = vma->obj;
 
-	WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP && !vma->is_ggtt);
+	WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
+		!i915_vma_is_ggtt(vma));
 
 	if (entry->alignment &&
 	    vma->node.start & (entry->alignment - 1))
 		return true;
 
+	if (vma->node.size < entry->pad_to_size)
+		return true;
+
 	if (entry->flags & EXEC_OBJECT_PINNED &&
 	    vma->node.start != entry->offset)
 		return true;
@@ -701,7 +876,8 @@ eb_vma_misplaced(struct i915_vma *vma)
 		return true;
 
 	/* avoid costly ping-pong once a batch bo ended up non-mappable */
-	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
+	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
+	    !i915_vma_is_map_and_fenceable(vma))
 		return !only_mappable_for_reloc(entry->flags);
 
 	if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
@@ -725,8 +901,6 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
 	bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
 	int retry;
 
-	i915_gem_retire_requests_ring(engine);
-
 	vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
 
 	INIT_LIST_HEAD(&ordered_vmas);
@@ -746,7 +920,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
 			entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
 		need_fence =
 			entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
-			obj->tiling_mode != I915_TILING_NONE;
+			i915_gem_object_is_tiled(obj);
 		need_mappable = need_fence || need_reloc_mappable(vma);
 
 		if (entry->flags & EXEC_OBJECT_PINNED)
@@ -843,7 +1017,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
 		vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
 		list_del_init(&vma->exec_list);
 		i915_gem_execbuffer_unreserve_vma(vma);
-		drm_gem_object_unreference(&vma->obj->base);
+		i915_vma_put(vma);
 	}
 
 	mutex_unlock(&dev->struct_mutex);
@@ -937,41 +1111,54 @@ err:
 	return ret;
 }
 
+static unsigned int eb_other_engines(struct drm_i915_gem_request *req)
+{
+	unsigned int mask;
+
+	mask = ~intel_engine_flag(req->engine) & I915_BO_ACTIVE_MASK;
+	mask <<= I915_BO_ACTIVE_SHIFT;
+
+	return mask;
+}
+
 static int
 i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
 				struct list_head *vmas)
 {
-	const unsigned other_rings = ~intel_engine_flag(req->engine);
+	const unsigned int other_rings = eb_other_engines(req);
 	struct i915_vma *vma;
-	uint32_t flush_domains = 0;
-	bool flush_chipset = false;
 	int ret;
 
 	list_for_each_entry(vma, vmas, exec_list) {
 		struct drm_i915_gem_object *obj = vma->obj;
+		struct reservation_object *resv;
 
-		if (obj->active & other_rings) {
-			ret = i915_gem_object_sync(obj, req->engine, &req);
+		if (obj->flags & other_rings) {
+			ret = i915_gem_request_await_object
+				(req, obj, obj->base.pending_write_domain);
 			if (ret)
 				return ret;
 		}
 
-		if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
-			flush_chipset |= i915_gem_clflush_object(obj, false);
+		resv = i915_gem_object_get_dmabuf_resv(obj);
+		if (resv) {
+			ret = i915_sw_fence_await_reservation
+				(&req->submit, resv, &i915_fence_ops,
+				 obj->base.pending_write_domain, 10*HZ,
+				 GFP_KERNEL | __GFP_NOWARN);
+			if (ret < 0)
+				return ret;
+		}
 
-		flush_domains |= obj->base.write_domain;
+		if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
+			i915_gem_clflush_object(obj, false);
 	}
 
-	if (flush_chipset)
-		i915_gem_chipset_flush(req->engine->i915);
-
-	if (flush_domains & I915_GEM_DOMAIN_GTT)
-		wmb();
+	/* Unconditionally flush any chipset caches (for streaming writes). */
+	i915_gem_chipset_flush(req->engine->i915);
 
-	/* Unconditionally invalidate gpu caches and ensure that we do flush
-	 * any residual writes from the previous batch.
-	 */
-	return intel_ring_invalidate_all_caches(req);
+	/* Unconditionally invalidate GPU caches and TLBs. */
+	return req->engine->emit_flush(req, EMIT_INVALIDATE);
 }
 
 static bool
@@ -1007,6 +1194,9 @@ validate_exec_list(struct drm_device *dev,
 	unsigned invalid_flags;
 	int i;
 
+	/* INTERNAL flags must not overlap with external ones */
+	BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ~__EXEC_OBJECT_UNKNOWN_FLAGS);
+
 	invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
 	if (USES_FULL_PPGTT(dev))
 		invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
@@ -1036,6 +1226,14 @@ validate_exec_list(struct drm_device *dev,
 		if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
 			return -EINVAL;
 
+		/* pad_to_size was once a reserved field, so sanitize it */
+		if (exec[i].flags & EXEC_OBJECT_PAD_TO_SIZE) {
+			if (offset_in_page(exec[i].pad_to_size))
+				return -EINVAL;
+		} else {
+			exec[i].pad_to_size = 0;
+		}
+
 		/* First check for malicious input causing overflow in
 		 * the worst case where we need to allocate the entire
 		 * relocation tree as a single array.
@@ -1055,7 +1253,7 @@ validate_exec_list(struct drm_device *dev,
 			return -EFAULT;
 
 		if (likely(!i915.prefault_disable)) {
-			if (fault_in_multipages_readable(ptr, length))
+			if (fault_in_pages_readable(ptr, length))
 				return -EFAULT;
 		}
 	}
@@ -1067,12 +1265,9 @@ static struct i915_gem_context *
 i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
 			  struct intel_engine_cs *engine, const u32 ctx_id)
 {
-	struct i915_gem_context *ctx = NULL;
+	struct i915_gem_context *ctx;
 	struct i915_ctx_hang_stats *hs;
 
-	if (engine->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
-		return ERR_PTR(-EINVAL);
-
 	ctx = i915_gem_context_lookup(file->driver_priv, ctx_id);
 	if (IS_ERR(ctx))
 		return ctx;
@@ -1086,66 +1281,99 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
 	return ctx;
 }
 
-void
+void i915_vma_move_to_active(struct i915_vma *vma,
+			     struct drm_i915_gem_request *req,
+			     unsigned int flags)
+{
+	struct drm_i915_gem_object *obj = vma->obj;
+	const unsigned int idx = req->engine->id;
+
+	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+
+	obj->dirty = 1; /* be paranoid  */
+
+	/* Add a reference if we're newly entering the active list.
+	 * The order in which we add operations to the retirement queue is
+	 * vital here: mark_active adds to the start of the callback list,
+	 * such that subsequent callbacks are called first. Therefore we
+	 * add the active reference first and queue for it to be dropped
+	 * *last*.
+	 */
+	if (!i915_gem_object_is_active(obj))
+		i915_gem_object_get(obj);
+	i915_gem_object_set_active(obj, idx);
+	i915_gem_active_set(&obj->last_read[idx], req);
+
+	if (flags & EXEC_OBJECT_WRITE) {
+		i915_gem_active_set(&obj->last_write, req);
+
+		intel_fb_obj_invalidate(obj, ORIGIN_CS);
+
+		/* update for the implicit flush after a batch */
+		obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
+	}
+
+	if (flags & EXEC_OBJECT_NEEDS_FENCE)
+		i915_gem_active_set(&vma->last_fence, req);
+
+	i915_vma_set_active(vma, idx);
+	i915_gem_active_set(&vma->last_read[idx], req);
+	list_move_tail(&vma->vm_link, &vma->vm->active_list);
+}
+
+static void eb_export_fence(struct drm_i915_gem_object *obj,
+			    struct drm_i915_gem_request *req,
+			    unsigned int flags)
+{
+	struct reservation_object *resv;
+
+	resv = i915_gem_object_get_dmabuf_resv(obj);
+	if (!resv)
+		return;
+
+	/* Ignore errors from failing to allocate the new fence, we can't
+	 * handle an error right now. Worst case should be missed
+	 * synchronisation leading to rendering corruption.
+	 */
+	ww_mutex_lock(&resv->lock, NULL);
+	if (flags & EXEC_OBJECT_WRITE)
+		reservation_object_add_excl_fence(resv, &req->fence);
+	else if (reservation_object_reserve_shared(resv) == 0)
+		reservation_object_add_shared_fence(resv, &req->fence);
+	ww_mutex_unlock(&resv->lock);
+}
+
+static void
 i915_gem_execbuffer_move_to_active(struct list_head *vmas,
 				   struct drm_i915_gem_request *req)
 {
-	struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
 	struct i915_vma *vma;
 
 	list_for_each_entry(vma, vmas, exec_list) {
-		struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 		struct drm_i915_gem_object *obj = vma->obj;
 		u32 old_read = obj->base.read_domains;
 		u32 old_write = obj->base.write_domain;
 
-		obj->dirty = 1; /* be paranoid  */
 		obj->base.write_domain = obj->base.pending_write_domain;
-		if (obj->base.write_domain == 0)
+		if (obj->base.write_domain)
+			vma->exec_entry->flags |= EXEC_OBJECT_WRITE;
+		else
 			obj->base.pending_read_domains |= obj->base.read_domains;
 		obj->base.read_domains = obj->base.pending_read_domains;
 
-		i915_vma_move_to_active(vma, req);
-		if (obj->base.write_domain) {
-			i915_gem_request_assign(&obj->last_write_req, req);
-
-			intel_fb_obj_invalidate(obj, ORIGIN_CS);
-
-			/* update for the implicit flush after a batch */
-			obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
-		}
-		if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
-			i915_gem_request_assign(&obj->last_fenced_req, req);
-			if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
-				struct drm_i915_private *dev_priv = engine->i915;
-				list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
-					       &dev_priv->mm.fence_list);
-			}
-		}
-
+		i915_vma_move_to_active(vma, req, vma->exec_entry->flags);
+		eb_export_fence(obj, req, vma->exec_entry->flags);
 		trace_i915_gem_object_change_domain(obj, old_read, old_write);
 	}
 }
 
-static void
-i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params)
-{
-	/* Unconditionally force add_request to emit a full flush. */
-	params->engine->gpu_caches_dirty = true;
-
-	/* Add a breadcrumb for the completion of the batch buffer */
-	__i915_add_request(params->request, params->batch_obj, true);
-}
-
 static int
-i915_reset_gen7_sol_offsets(struct drm_device *dev,
-			    struct drm_i915_gem_request *req)
+i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
 {
-	struct intel_engine_cs *engine = req->engine;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_ring *ring = req->ring;
 	int ret, i;
 
-	if (!IS_GEN7(dev) || engine != &dev_priv->engine[RCS]) {
+	if (!IS_GEN7(req->i915) || req->engine->id != RCS) {
 		DRM_DEBUG("sol reset is gen7/rcs only\n");
 		return -EINVAL;
 	}
@@ -1155,21 +1383,21 @@ i915_reset_gen7_sol_offsets(struct drm_device *dev,
 		return ret;
 
 	for (i = 0; i < 4; i++) {
-		intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit_reg(engine, GEN7_SO_WRITE_OFFSET(i));
-		intel_ring_emit(engine, 0);
+		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+		intel_ring_emit_reg(ring, GEN7_SO_WRITE_OFFSET(i));
+		intel_ring_emit(ring, 0);
 	}
 
-	intel_ring_advance(engine);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
-static struct drm_i915_gem_object*
+static struct i915_vma *
 i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
 			  struct drm_i915_gem_exec_object2 *shadow_exec_entry,
-			  struct eb_vmas *eb,
 			  struct drm_i915_gem_object *batch_obj,
+			  struct eb_vmas *eb,
 			  u32 batch_start_offset,
 			  u32 batch_len,
 			  bool is_master)
@@ -1181,51 +1409,44 @@ i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
 	shadow_batch_obj = i915_gem_batch_pool_get(&engine->batch_pool,
 						   PAGE_ALIGN(batch_len));
 	if (IS_ERR(shadow_batch_obj))
-		return shadow_batch_obj;
-
-	ret = i915_parse_cmds(engine,
-			      batch_obj,
-			      shadow_batch_obj,
-			      batch_start_offset,
-			      batch_len,
-			      is_master);
-	if (ret)
-		goto err;
-
-	ret = i915_gem_obj_ggtt_pin(shadow_batch_obj, 0, 0);
-	if (ret)
-		goto err;
+		return ERR_CAST(shadow_batch_obj);
+
+	ret = intel_engine_cmd_parser(engine,
+				      batch_obj,
+				      shadow_batch_obj,
+				      batch_start_offset,
+				      batch_len,
+				      is_master);
+	if (ret) {
+		if (ret == -EACCES) /* unhandled chained batch */
+			vma = NULL;
+		else
+			vma = ERR_PTR(ret);
+		goto out;
+	}
 
-	i915_gem_object_unpin_pages(shadow_batch_obj);
+	vma = i915_gem_object_ggtt_pin(shadow_batch_obj, NULL, 0, 0, 0);
+	if (IS_ERR(vma))
+		goto out;
 
 	memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
 
-	vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
 	vma->exec_entry = shadow_exec_entry;
 	vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
-	drm_gem_object_reference(&shadow_batch_obj->base);
+	i915_gem_object_get(shadow_batch_obj);
 	list_add_tail(&vma->exec_list, &eb->vmas);
 
-	shadow_batch_obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;
-
-	return shadow_batch_obj;
-
-err:
+out:
 	i915_gem_object_unpin_pages(shadow_batch_obj);
-	if (ret == -EACCES) /* unhandled chained batch */
-		return batch_obj;
-	else
-		return ERR_PTR(ret);
+	return vma;
 }
 
-int
-i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
-			       struct drm_i915_gem_execbuffer2 *args,
-			       struct list_head *vmas)
+static int
+execbuf_submit(struct i915_execbuffer_params *params,
+	       struct drm_i915_gem_execbuffer2 *args,
+	       struct list_head *vmas)
 {
-	struct drm_device *dev = params->dev;
-	struct intel_engine_cs *engine = params->engine;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = params->request->i915;
 	u64 exec_start, exec_len;
 	int instp_mode;
 	u32 instp_mask;
@@ -1239,34 +1460,31 @@ i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
 	if (ret)
 		return ret;
 
-	WARN(params->ctx->ppgtt && params->ctx->ppgtt->pd_dirty_rings & (1<<engine->id),
-	     "%s didn't clear reload\n", engine->name);
-
 	instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
 	instp_mask = I915_EXEC_CONSTANTS_MASK;
 	switch (instp_mode) {
 	case I915_EXEC_CONSTANTS_REL_GENERAL:
 	case I915_EXEC_CONSTANTS_ABSOLUTE:
 	case I915_EXEC_CONSTANTS_REL_SURFACE:
-		if (instp_mode != 0 && engine != &dev_priv->engine[RCS]) {
+		if (instp_mode != 0 && params->engine->id != RCS) {
 			DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
 			return -EINVAL;
 		}
 
 		if (instp_mode != dev_priv->relative_constants_mode) {
-			if (INTEL_INFO(dev)->gen < 4) {
+			if (INTEL_INFO(dev_priv)->gen < 4) {
 				DRM_DEBUG("no rel constants on pre-gen4\n");
 				return -EINVAL;
 			}
 
-			if (INTEL_INFO(dev)->gen > 5 &&
+			if (INTEL_INFO(dev_priv)->gen > 5 &&
 			    instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
 				DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
 				return -EINVAL;
 			}
 
 			/* The HW changed the meaning on this bit on gen6 */
-			if (INTEL_INFO(dev)->gen >= 6)
+			if (INTEL_INFO(dev_priv)->gen >= 6)
 				instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
 		}
 		break;
@@ -1275,37 +1493,39 @@ i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
 		return -EINVAL;
 	}
 
-	if (engine == &dev_priv->engine[RCS] &&
+	if (params->engine->id == RCS &&
 	    instp_mode != dev_priv->relative_constants_mode) {
+		struct intel_ring *ring = params->request->ring;
+
 		ret = intel_ring_begin(params->request, 4);
 		if (ret)
 			return ret;
 
-		intel_ring_emit(engine, MI_NOOP);
-		intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit_reg(engine, INSTPM);
-		intel_ring_emit(engine, instp_mask << 16 | instp_mode);
-		intel_ring_advance(engine);
+		intel_ring_emit(ring, MI_NOOP);
+		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+		intel_ring_emit_reg(ring, INSTPM);
+		intel_ring_emit(ring, instp_mask << 16 | instp_mode);
+		intel_ring_advance(ring);
 
 		dev_priv->relative_constants_mode = instp_mode;
 	}
 
 	if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
-		ret = i915_reset_gen7_sol_offsets(dev, params->request);
+		ret = i915_reset_gen7_sol_offsets(params->request);
 		if (ret)
 			return ret;
 	}
 
 	exec_len   = args->batch_len;
-	exec_start = params->batch_obj_vm_offset +
+	exec_start = params->batch->node.start +
 		     params->args_batch_start_offset;
 
 	if (exec_len == 0)
-		exec_len = params->batch_obj->base.size;
+		exec_len = params->batch->size - params->args_batch_start_offset;
 
-	ret = engine->dispatch_execbuffer(params->request,
-					exec_start, exec_len,
-					params->dispatch_flags);
+	ret = params->engine->emit_bb_start(params->request,
+					    exec_start, exec_len,
+					    params->dispatch_flags);
 	if (ret)
 		return ret;
 
@@ -1318,43 +1538,20 @@ i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
 
 /**
  * Find one BSD ring to dispatch the corresponding BSD command.
- * The ring index is returned.
+ * The engine index is returned.
  */
 static unsigned int
-gen8_dispatch_bsd_ring(struct drm_i915_private *dev_priv, struct drm_file *file)
+gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
+			 struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 
 	/* Check whether the file_priv has already selected one ring. */
-	if ((int)file_priv->bsd_ring < 0) {
-		/* If not, use the ping-pong mechanism to select one. */
-		mutex_lock(&dev_priv->drm.struct_mutex);
-		file_priv->bsd_ring = dev_priv->mm.bsd_ring_dispatch_index;
-		dev_priv->mm.bsd_ring_dispatch_index ^= 1;
-		mutex_unlock(&dev_priv->drm.struct_mutex);
-	}
+	if ((int)file_priv->bsd_engine < 0)
+		file_priv->bsd_engine = atomic_fetch_xor(1,
+			 &dev_priv->mm.bsd_engine_dispatch_index);
 
-	return file_priv->bsd_ring;
-}
-
-static struct drm_i915_gem_object *
-eb_get_batch(struct eb_vmas *eb)
-{
-	struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
-
-	/*
-	 * SNA is doing fancy tricks with compressing batch buffers, which leads
-	 * to negative relocation deltas. Usually that works out ok since the
-	 * relocate address is still positive, except when the batch is placed
-	 * very low in the GTT. Ensure this doesn't happen.
-	 *
-	 * Note that actual hangs have only been observed on gen7, but for
-	 * paranoia do it everywhere.
-	 */
-	if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
-		vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
-
-	return vma->obj;
+	return file_priv->bsd_engine;
 }
 
 #define I915_USER_RINGS (4)
@@ -1367,31 +1564,31 @@ static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
 	[I915_EXEC_VEBOX]	= VECS
 };
 
-static int
-eb_select_ring(struct drm_i915_private *dev_priv,
-	       struct drm_file *file,
-	       struct drm_i915_gem_execbuffer2 *args,
-	       struct intel_engine_cs **ring)
+static struct intel_engine_cs *
+eb_select_engine(struct drm_i915_private *dev_priv,
+		 struct drm_file *file,
+		 struct drm_i915_gem_execbuffer2 *args)
 {
 	unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
+	struct intel_engine_cs *engine;
 
 	if (user_ring_id > I915_USER_RINGS) {
 		DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
-		return -EINVAL;
+		return NULL;
 	}
 
 	if ((user_ring_id != I915_EXEC_BSD) &&
 	    ((args->flags & I915_EXEC_BSD_MASK) != 0)) {
 		DRM_DEBUG("execbuf with non bsd ring but with invalid "
 			  "bsd dispatch flags: %d\n", (int)(args->flags));
-		return -EINVAL;
+		return NULL;
 	}
 
 	if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
 		unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
 
 		if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
-			bsd_idx = gen8_dispatch_bsd_ring(dev_priv, file);
+			bsd_idx = gen8_dispatch_bsd_engine(dev_priv, file);
 		} else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
 			   bsd_idx <= I915_EXEC_BSD_RING2) {
 			bsd_idx >>= I915_EXEC_BSD_SHIFT;
@@ -1399,20 +1596,20 @@ eb_select_ring(struct drm_i915_private *dev_priv,
 		} else {
 			DRM_DEBUG("execbuf with unknown bsd ring: %u\n",
 				  bsd_idx);
-			return -EINVAL;
+			return NULL;
 		}
 
-		*ring = &dev_priv->engine[_VCS(bsd_idx)];
+		engine = &dev_priv->engine[_VCS(bsd_idx)];
 	} else {
-		*ring = &dev_priv->engine[user_ring_map[user_ring_id]];
+		engine = &dev_priv->engine[user_ring_map[user_ring_id]];
 	}
 
-	if (!intel_engine_initialized(*ring)) {
+	if (!intel_engine_initialized(engine)) {
 		DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id);
-		return -EINVAL;
+		return NULL;
 	}
 
-	return 0;
+	return engine;
 }
 
 static int
@@ -1423,9 +1620,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct drm_i915_gem_request *req = NULL;
 	struct eb_vmas *eb;
-	struct drm_i915_gem_object *batch_obj;
 	struct drm_i915_gem_exec_object2 shadow_exec_entry;
 	struct intel_engine_cs *engine;
 	struct i915_gem_context *ctx;
@@ -1454,9 +1649,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 	if (args->flags & I915_EXEC_IS_PINNED)
 		dispatch_flags |= I915_DISPATCH_PINNED;
 
-	ret = eb_select_ring(dev_priv, file, args, &engine);
-	if (ret)
-		return ret;
+	engine = eb_select_engine(dev_priv, file, args);
+	if (!engine)
+		return -EINVAL;
 
 	if (args->buffer_count < 1) {
 		DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
@@ -1496,7 +1691,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 		goto pre_mutex_err;
 	}
 
-	i915_gem_context_reference(ctx);
+	i915_gem_context_get(ctx);
 
 	if (ctx->ppgtt)
 		vm = &ctx->ppgtt->base;
@@ -1505,9 +1700,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 
 	memset(&params_master, 0x00, sizeof(params_master));
 
-	eb = eb_create(args);
+	eb = eb_create(dev_priv, args);
 	if (eb == NULL) {
-		i915_gem_context_unreference(ctx);
+		i915_gem_context_put(ctx);
 		mutex_unlock(&dev->struct_mutex);
 		ret = -ENOMEM;
 		goto pre_mutex_err;
@@ -1519,7 +1714,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 		goto err;
 
 	/* take note of the batch buffer before we might reorder the lists */
-	batch_obj = eb_get_batch(eb);
+	params->batch = eb_get_batch(eb);
 
 	/* Move the objects en-masse into the GTT, evicting if necessary. */
 	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
@@ -1543,34 +1738,34 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 	}
 
 	/* Set the pending read domains for the batch buffer to COMMAND */
-	if (batch_obj->base.pending_write_domain) {
+	if (params->batch->obj->base.pending_write_domain) {
 		DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
 		ret = -EINVAL;
 		goto err;
 	}
+	if (args->batch_start_offset > params->batch->size ||
+	    args->batch_len > params->batch->size - args->batch_start_offset) {
+		DRM_DEBUG("Attempting to use out-of-bounds batch\n");
+		ret = -EINVAL;
+		goto err;
+	}
 
 	params->args_batch_start_offset = args->batch_start_offset;
-	if (i915_needs_cmd_parser(engine) && args->batch_len) {
-		struct drm_i915_gem_object *parsed_batch_obj;
-
-		parsed_batch_obj = i915_gem_execbuffer_parse(engine,
-							     &shadow_exec_entry,
-							     eb,
-							     batch_obj,
-							     args->batch_start_offset,
-							     args->batch_len,
-							     drm_is_current_master(file));
-		if (IS_ERR(parsed_batch_obj)) {
-			ret = PTR_ERR(parsed_batch_obj);
+	if (intel_engine_needs_cmd_parser(engine) && args->batch_len) {
+		struct i915_vma *vma;
+
+		vma = i915_gem_execbuffer_parse(engine, &shadow_exec_entry,
+						params->batch->obj,
+						eb,
+						args->batch_start_offset,
+						args->batch_len,
+						drm_is_current_master(file));
+		if (IS_ERR(vma)) {
+			ret = PTR_ERR(vma);
 			goto err;
 		}
 
-		/*
-		 * parsed_batch_obj == batch_obj means batch not fully parsed:
-		 * Accept, but don't promote to secure.
-		 */
-
-		if (parsed_batch_obj != batch_obj) {
+		if (vma) {
 			/*
 			 * Batch parsed and accepted:
 			 *
@@ -1582,16 +1777,19 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 			 */
 			dispatch_flags |= I915_DISPATCH_SECURE;
 			params->args_batch_start_offset = 0;
-			batch_obj = parsed_batch_obj;
+			params->batch = vma;
 		}
 	}
 
-	batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
+	params->batch->obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
 
 	/* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
 	 * batch" bit. Hence we need to pin secure batches into the global gtt.
 	 * hsw should have this fixed, but bdw mucks it up again. */
 	if (dispatch_flags & I915_DISPATCH_SECURE) {
+		struct drm_i915_gem_object *obj = params->batch->obj;
+		struct i915_vma *vma;
+
 		/*
 		 * So on first glance it looks freaky that we pin the batch here
 		 * outside of the reservation loop. But:
@@ -1602,22 +1800,31 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 		 *   fitting due to fragmentation.
 		 * So this is actually safe.
 		 */
-		ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
-		if (ret)
+		vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
+		if (IS_ERR(vma)) {
+			ret = PTR_ERR(vma);
 			goto err;
+		}
 
-		params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj);
-	} else
-		params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm);
+		params->batch = vma;
+	}
 
 	/* Allocate a request for this batch buffer nice and early. */
-	req = i915_gem_request_alloc(engine, ctx);
-	if (IS_ERR(req)) {
-		ret = PTR_ERR(req);
+	params->request = i915_gem_request_alloc(engine, ctx);
+	if (IS_ERR(params->request)) {
+		ret = PTR_ERR(params->request);
 		goto err_batch_unpin;
 	}
 
-	ret = i915_gem_request_add_to_client(req, file);
+	/* Whilst this request exists, batch_obj will be on the
+	 * active_list, and so will hold the active reference. Only when this
+	 * request is retired will the the batch_obj be moved onto the
+	 * inactive_list and lose its active reference. Hence we do not need
+	 * to explicitly hold another reference here.
+	 */
+	params->request->batch = params->batch;
+
+	ret = i915_gem_request_add_to_client(params->request, file);
 	if (ret)
 		goto err_request;
 
@@ -1631,13 +1838,11 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 	params->file                    = file;
 	params->engine                    = engine;
 	params->dispatch_flags          = dispatch_flags;
-	params->batch_obj               = batch_obj;
 	params->ctx                     = ctx;
-	params->request                 = req;
 
-	ret = dev_priv->gt.execbuf_submit(params, args, &eb->vmas);
+	ret = execbuf_submit(params, args, &eb->vmas);
 err_request:
-	i915_gem_execbuffer_retire_commands(params);
+	__i915_add_request(params->request, ret == 0);
 
 err_batch_unpin:
 	/*
@@ -1647,11 +1852,10 @@ err_batch_unpin:
 	 * active.
 	 */
 	if (dispatch_flags & I915_DISPATCH_SECURE)
-		i915_gem_object_ggtt_unpin(batch_obj);
-
+		i915_vma_unpin(params->batch);
 err:
 	/* the request owns the ref now */
-	i915_gem_context_unreference(ctx);
+	i915_gem_context_put(ctx);
 	eb_destroy(eb);
 
 	mutex_unlock(&dev->struct_mutex);
diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c
index 251d7a95af89..8df1fa7234e8 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence.c
@@ -55,226 +55,228 @@
  * CPU ptes into GTT mmaps (not the GTT ptes themselves) as needed.
  */
 
-static void i965_write_fence_reg(struct drm_device *dev, int reg,
-				 struct drm_i915_gem_object *obj)
+#define pipelined 0
+
+static void i965_write_fence_reg(struct drm_i915_fence_reg *fence,
+				 struct i915_vma *vma)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	i915_reg_t fence_reg_lo, fence_reg_hi;
 	int fence_pitch_shift;
+	u64 val;
 
-	if (INTEL_INFO(dev)->gen >= 6) {
-		fence_reg_lo = FENCE_REG_GEN6_LO(reg);
-		fence_reg_hi = FENCE_REG_GEN6_HI(reg);
+	if (INTEL_INFO(fence->i915)->gen >= 6) {
+		fence_reg_lo = FENCE_REG_GEN6_LO(fence->id);
+		fence_reg_hi = FENCE_REG_GEN6_HI(fence->id);
 		fence_pitch_shift = GEN6_FENCE_PITCH_SHIFT;
+
 	} else {
-		fence_reg_lo = FENCE_REG_965_LO(reg);
-		fence_reg_hi = FENCE_REG_965_HI(reg);
+		fence_reg_lo = FENCE_REG_965_LO(fence->id);
+		fence_reg_hi = FENCE_REG_965_HI(fence->id);
 		fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
 	}
 
-	/* To w/a incoherency with non-atomic 64-bit register updates,
-	 * we split the 64-bit update into two 32-bit writes. In order
-	 * for a partial fence not to be evaluated between writes, we
-	 * precede the update with write to turn off the fence register,
-	 * and only enable the fence as the last step.
-	 *
-	 * For extra levels of paranoia, we make sure each step lands
-	 * before applying the next step.
-	 */
-	I915_WRITE(fence_reg_lo, 0);
-	POSTING_READ(fence_reg_lo);
-
-	if (obj) {
-		u32 size = i915_gem_obj_ggtt_size(obj);
-		uint64_t val;
-
-		/* Adjust fence size to match tiled area */
-		if (obj->tiling_mode != I915_TILING_NONE) {
-			uint32_t row_size = obj->stride *
-				(obj->tiling_mode == I915_TILING_Y ? 32 : 8);
-			size = (size / row_size) * row_size;
-		}
-
-		val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
-				 0xfffff000) << 32;
-		val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
-		val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
-		if (obj->tiling_mode == I915_TILING_Y)
-			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
+	val = 0;
+	if (vma) {
+		unsigned int tiling = i915_gem_object_get_tiling(vma->obj);
+		bool is_y_tiled = tiling == I915_TILING_Y;
+		unsigned int stride = i915_gem_object_get_stride(vma->obj);
+		u32 row_size = stride * (is_y_tiled ? 32 : 8);
+		u32 size = rounddown((u32)vma->node.size, row_size);
+
+		val = ((vma->node.start + size - 4096) & 0xfffff000) << 32;
+		val |= vma->node.start & 0xfffff000;
+		val |= (u64)((stride / 128) - 1) << fence_pitch_shift;
+		if (is_y_tiled)
+			val |= BIT(I965_FENCE_TILING_Y_SHIFT);
 		val |= I965_FENCE_REG_VALID;
+	}
 
-		I915_WRITE(fence_reg_hi, val >> 32);
-		POSTING_READ(fence_reg_hi);
+	if (!pipelined) {
+		struct drm_i915_private *dev_priv = fence->i915;
 
-		I915_WRITE(fence_reg_lo, val);
+		/* To w/a incoherency with non-atomic 64-bit register updates,
+		 * we split the 64-bit update into two 32-bit writes. In order
+		 * for a partial fence not to be evaluated between writes, we
+		 * precede the update with write to turn off the fence register,
+		 * and only enable the fence as the last step.
+		 *
+		 * For extra levels of paranoia, we make sure each step lands
+		 * before applying the next step.
+		 */
+		I915_WRITE(fence_reg_lo, 0);
+		POSTING_READ(fence_reg_lo);
+
+		I915_WRITE(fence_reg_hi, upper_32_bits(val));
+		I915_WRITE(fence_reg_lo, lower_32_bits(val));
 		POSTING_READ(fence_reg_lo);
-	} else {
-		I915_WRITE(fence_reg_hi, 0);
-		POSTING_READ(fence_reg_hi);
 	}
 }
 
-static void i915_write_fence_reg(struct drm_device *dev, int reg,
-				 struct drm_i915_gem_object *obj)
+static void i915_write_fence_reg(struct drm_i915_fence_reg *fence,
+				 struct i915_vma *vma)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 val;
 
-	if (obj) {
-		u32 size = i915_gem_obj_ggtt_size(obj);
+	val = 0;
+	if (vma) {
+		unsigned int tiling = i915_gem_object_get_tiling(vma->obj);
+		bool is_y_tiled = tiling == I915_TILING_Y;
+		unsigned int stride = i915_gem_object_get_stride(vma->obj);
 		int pitch_val;
 		int tile_width;
 
-		WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
-		     (size & -size) != size ||
-		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
-		     "object 0x%08llx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
-		     i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
+		WARN((vma->node.start & ~I915_FENCE_START_MASK) ||
+		     !is_power_of_2(vma->node.size) ||
+		     (vma->node.start & (vma->node.size - 1)),
+		     "object 0x%08llx [fenceable? %d] not 1M or pot-size (0x%08llx) aligned\n",
+		     vma->node.start,
+		     i915_vma_is_map_and_fenceable(vma),
+		     vma->node.size);
 
-		if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
+		if (is_y_tiled && HAS_128_BYTE_Y_TILING(fence->i915))
 			tile_width = 128;
 		else
 			tile_width = 512;
 
 		/* Note: pitch better be a power of two tile widths */
-		pitch_val = obj->stride / tile_width;
+		pitch_val = stride / tile_width;
 		pitch_val = ffs(pitch_val) - 1;
 
-		val = i915_gem_obj_ggtt_offset(obj);
-		if (obj->tiling_mode == I915_TILING_Y)
-			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-		val |= I915_FENCE_SIZE_BITS(size);
+		val = vma->node.start;
+		if (is_y_tiled)
+			val |= BIT(I830_FENCE_TILING_Y_SHIFT);
+		val |= I915_FENCE_SIZE_BITS(vma->node.size);
 		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
 		val |= I830_FENCE_REG_VALID;
-	} else
-		val = 0;
+	}
+
+	if (!pipelined) {
+		struct drm_i915_private *dev_priv = fence->i915;
+		i915_reg_t reg = FENCE_REG(fence->id);
 
-	I915_WRITE(FENCE_REG(reg), val);
-	POSTING_READ(FENCE_REG(reg));
+		I915_WRITE(reg, val);
+		POSTING_READ(reg);
+	}
 }
 
-static void i830_write_fence_reg(struct drm_device *dev, int reg,
-				struct drm_i915_gem_object *obj)
+static void i830_write_fence_reg(struct drm_i915_fence_reg *fence,
+				 struct i915_vma *vma)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	uint32_t val;
+	u32 val;
 
-	if (obj) {
-		u32 size = i915_gem_obj_ggtt_size(obj);
-		uint32_t pitch_val;
+	val = 0;
+	if (vma) {
+		unsigned int tiling = i915_gem_object_get_tiling(vma->obj);
+		bool is_y_tiled = tiling == I915_TILING_Y;
+		unsigned int stride = i915_gem_object_get_stride(vma->obj);
+		u32 pitch_val;
 
-		WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
-		     (size & -size) != size ||
-		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
-		     "object 0x%08llx not 512K or pot-size 0x%08x aligned\n",
-		     i915_gem_obj_ggtt_offset(obj), size);
+		WARN((vma->node.start & ~I830_FENCE_START_MASK) ||
+		     !is_power_of_2(vma->node.size) ||
+		     (vma->node.start & (vma->node.size - 1)),
+		     "object 0x%08llx not 512K or pot-size 0x%08llx aligned\n",
+		     vma->node.start, vma->node.size);
 
-		pitch_val = obj->stride / 128;
+		pitch_val = stride / 128;
 		pitch_val = ffs(pitch_val) - 1;
 
-		val = i915_gem_obj_ggtt_offset(obj);
-		if (obj->tiling_mode == I915_TILING_Y)
-			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-		val |= I830_FENCE_SIZE_BITS(size);
+		val = vma->node.start;
+		if (is_y_tiled)
+			val |= BIT(I830_FENCE_TILING_Y_SHIFT);
+		val |= I830_FENCE_SIZE_BITS(vma->node.size);
 		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
 		val |= I830_FENCE_REG_VALID;
-	} else
-		val = 0;
+	}
 
-	I915_WRITE(FENCE_REG(reg), val);
-	POSTING_READ(FENCE_REG(reg));
-}
+	if (!pipelined) {
+		struct drm_i915_private *dev_priv = fence->i915;
+		i915_reg_t reg = FENCE_REG(fence->id);
 
-inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
-{
-	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
+		I915_WRITE(reg, val);
+		POSTING_READ(reg);
+	}
 }
 
-static void i915_gem_write_fence(struct drm_device *dev, int reg,
-				 struct drm_i915_gem_object *obj)
+static void fence_write(struct drm_i915_fence_reg *fence,
+			struct i915_vma *vma)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	/* Ensure that all CPU reads are completed before installing a fence
-	 * and all writes before removing the fence.
+	/* Previous access through the fence register is marshalled by
+	 * the mb() inside the fault handlers (i915_gem_release_mmaps)
+	 * and explicitly managed for internal users.
 	 */
-	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
-		mb();
-
-	WARN(obj && (!obj->stride || !obj->tiling_mode),
-	     "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
-	     obj->stride, obj->tiling_mode);
-
-	if (IS_GEN2(dev))
-		i830_write_fence_reg(dev, reg, obj);
-	else if (IS_GEN3(dev))
-		i915_write_fence_reg(dev, reg, obj);
-	else if (INTEL_INFO(dev)->gen >= 4)
-		i965_write_fence_reg(dev, reg, obj);
-
-	/* And similarly be paranoid that no direct access to this region
-	 * is reordered to before the fence is installed.
+
+	if (IS_GEN2(fence->i915))
+		i830_write_fence_reg(fence, vma);
+	else if (IS_GEN3(fence->i915))
+		i915_write_fence_reg(fence, vma);
+	else
+		i965_write_fence_reg(fence, vma);
+
+	/* Access through the fenced region afterwards is
+	 * ordered by the posting reads whilst writing the registers.
 	 */
-	if (i915_gem_object_needs_mb(obj))
-		mb();
-}
 
-static inline int fence_number(struct drm_i915_private *dev_priv,
-			       struct drm_i915_fence_reg *fence)
-{
-	return fence - dev_priv->fence_regs;
+	fence->dirty = false;
 }
 
-static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
-					 struct drm_i915_fence_reg *fence,
-					 bool enable)
+static int fence_update(struct drm_i915_fence_reg *fence,
+			struct i915_vma *vma)
 {
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	int reg = fence_number(dev_priv, fence);
-
-	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
+	int ret;
 
-	if (enable) {
-		obj->fence_reg = reg;
-		fence->obj = obj;
-		list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
-	} else {
-		obj->fence_reg = I915_FENCE_REG_NONE;
-		fence->obj = NULL;
-		list_del_init(&fence->lru_list);
-	}
-	obj->fence_dirty = false;
-}
+	if (vma) {
+		if (!i915_vma_is_map_and_fenceable(vma))
+			return -EINVAL;
 
-static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
-{
-	if (obj->tiling_mode)
-		i915_gem_release_mmap(obj);
+		if (WARN(!i915_gem_object_get_stride(vma->obj) ||
+			 !i915_gem_object_get_tiling(vma->obj),
+			 "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
+			 i915_gem_object_get_stride(vma->obj),
+			 i915_gem_object_get_tiling(vma->obj)))
+			return -EINVAL;
 
-	/* As we do not have an associated fence register, we will force
-	 * a tiling change if we ever need to acquire one.
-	 */
-	obj->fence_dirty = false;
-	obj->fence_reg = I915_FENCE_REG_NONE;
-}
+		ret = i915_gem_active_retire(&vma->last_fence,
+					     &vma->obj->base.dev->struct_mutex);
+		if (ret)
+			return ret;
+	}
 
-static int
-i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
-{
-	if (obj->last_fenced_req) {
-		int ret = i915_wait_request(obj->last_fenced_req);
+	if (fence->vma) {
+		ret = i915_gem_active_retire(&fence->vma->last_fence,
+				      &fence->vma->obj->base.dev->struct_mutex);
 		if (ret)
 			return ret;
+	}
+
+	if (fence->vma && fence->vma != vma) {
+		/* Ensure that all userspace CPU access is completed before
+		 * stealing the fence.
+		 */
+		i915_gem_release_mmap(fence->vma->obj);
+
+		fence->vma->fence = NULL;
+		fence->vma = NULL;
+
+		list_move(&fence->link, &fence->i915->mm.fence_list);
+	}
+
+	fence_write(fence, vma);
+
+	if (vma) {
+		if (fence->vma != vma) {
+			vma->fence = fence;
+			fence->vma = vma;
+		}
 
-		i915_gem_request_assign(&obj->last_fenced_req, NULL);
+		list_move_tail(&fence->link, &fence->i915->mm.fence_list);
 	}
 
 	return 0;
 }
 
 /**
- * i915_gem_object_put_fence - force-remove fence for an object
- * @obj: object to map through a fence reg
+ * i915_vma_put_fence - force-remove fence for a VMA
+ * @vma: vma to map linearly (not through a fence reg)
  *
  * This function force-removes any fence from the given object, which is useful
  * if the kernel wants to do untiled GTT access.
@@ -284,70 +286,40 @@ i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
  * 0 on success, negative error code on failure.
  */
 int
-i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
+i915_vma_put_fence(struct i915_vma *vma)
 {
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	struct drm_i915_fence_reg *fence;
-	int ret;
+	struct drm_i915_fence_reg *fence = vma->fence;
 
-	ret = i915_gem_object_wait_fence(obj);
-	if (ret)
-		return ret;
-
-	if (obj->fence_reg == I915_FENCE_REG_NONE)
+	if (!fence)
 		return 0;
 
-	fence = &dev_priv->fence_regs[obj->fence_reg];
-
-	if (WARN_ON(fence->pin_count))
+	if (fence->pin_count)
 		return -EBUSY;
 
-	i915_gem_object_fence_lost(obj);
-	i915_gem_object_update_fence(obj, fence, false);
-
-	return 0;
+	return fence_update(fence, NULL);
 }
 
-static struct drm_i915_fence_reg *
-i915_find_fence_reg(struct drm_device *dev)
+static struct drm_i915_fence_reg *fence_find(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_i915_fence_reg *reg, *avail;
-	int i;
-
-	/* First try to find a free reg */
-	avail = NULL;
-	for (i = 0; i < dev_priv->num_fence_regs; i++) {
-		reg = &dev_priv->fence_regs[i];
-		if (!reg->obj)
-			return reg;
-
-		if (!reg->pin_count)
-			avail = reg;
-	}
-
-	if (avail == NULL)
-		goto deadlock;
+	struct drm_i915_fence_reg *fence;
 
-	/* None available, try to steal one or wait for a user to finish */
-	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
-		if (reg->pin_count)
+	list_for_each_entry(fence, &dev_priv->mm.fence_list, link) {
+		if (fence->pin_count)
 			continue;
 
-		return reg;
+		return fence;
 	}
 
-deadlock:
 	/* Wait for completion of pending flips which consume fences */
-	if (intel_has_pending_fb_unpin(dev))
+	if (intel_has_pending_fb_unpin(&dev_priv->drm))
 		return ERR_PTR(-EAGAIN);
 
 	return ERR_PTR(-EDEADLK);
 }
 
 /**
- * i915_gem_object_get_fence - set up fencing for an object
- * @obj: object to map through a fence reg
+ * i915_vma_get_fence - set up fencing for a vma
+ * @vma: vma to map through a fence reg
  *
  * When mapping objects through the GTT, userspace wants to be able to write
  * to them without having to worry about swizzling if the object is tiled.
@@ -364,103 +336,27 @@ deadlock:
  * 0 on success, negative error code on failure.
  */
 int
-i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
+i915_vma_get_fence(struct i915_vma *vma)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	bool enable = obj->tiling_mode != I915_TILING_NONE;
-	struct drm_i915_fence_reg *reg;
-	int ret;
-
-	/* Have we updated the tiling parameters upon the object and so
-	 * will need to serialise the write to the associated fence register?
-	 */
-	if (obj->fence_dirty) {
-		ret = i915_gem_object_wait_fence(obj);
-		if (ret)
-			return ret;
-	}
+	struct drm_i915_fence_reg *fence;
+	struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL;
 
 	/* Just update our place in the LRU if our fence is getting reused. */
-	if (obj->fence_reg != I915_FENCE_REG_NONE) {
-		reg = &dev_priv->fence_regs[obj->fence_reg];
-		if (!obj->fence_dirty) {
-			list_move_tail(&reg->lru_list,
-				       &dev_priv->mm.fence_list);
+	if (vma->fence) {
+		fence = vma->fence;
+		if (!fence->dirty) {
+			list_move_tail(&fence->link,
+				       &fence->i915->mm.fence_list);
 			return 0;
 		}
-	} else if (enable) {
-		if (WARN_ON(!obj->map_and_fenceable))
-			return -EINVAL;
-
-		reg = i915_find_fence_reg(dev);
-		if (IS_ERR(reg))
-			return PTR_ERR(reg);
-
-		if (reg->obj) {
-			struct drm_i915_gem_object *old = reg->obj;
-
-			ret = i915_gem_object_wait_fence(old);
-			if (ret)
-				return ret;
-
-			i915_gem_object_fence_lost(old);
-		}
+	} else if (set) {
+		fence = fence_find(to_i915(vma->vm->dev));
+		if (IS_ERR(fence))
+			return PTR_ERR(fence);
 	} else
 		return 0;
 
-	i915_gem_object_update_fence(obj, reg, enable);
-
-	return 0;
-}
-
-/**
- * i915_gem_object_pin_fence - pin fencing state
- * @obj: object to pin fencing for
- *
- * This pins the fencing state (whether tiled or untiled) to make sure the
- * object is ready to be used as a scanout target. Fencing status must be
- * synchronize first by calling i915_gem_object_get_fence():
- *
- * The resulting fence pin reference must be released again with
- * i915_gem_object_unpin_fence().
- *
- * Returns:
- *
- * True if the object has a fence, false otherwise.
- */
-bool
-i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
-{
-	if (obj->fence_reg != I915_FENCE_REG_NONE) {
-		struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-		struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj);
-
-		WARN_ON(!ggtt_vma ||
-			dev_priv->fence_regs[obj->fence_reg].pin_count >
-			ggtt_vma->pin_count);
-		dev_priv->fence_regs[obj->fence_reg].pin_count++;
-		return true;
-	} else
-		return false;
-}
-
-/**
- * i915_gem_object_unpin_fence - unpin fencing state
- * @obj: object to unpin fencing for
- *
- * This releases the fence pin reference acquired through
- * i915_gem_object_pin_fence. It will handle both objects with and without an
- * attached fence correctly, callers do not need to distinguish this.
- */
-void
-i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
-{
-	if (obj->fence_reg != I915_FENCE_REG_NONE) {
-		struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-		WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0);
-		dev_priv->fence_regs[obj->fence_reg].pin_count--;
-	}
+	return fence_update(fence, set);
 }
 
 /**
@@ -477,17 +373,16 @@ void i915_gem_restore_fences(struct drm_device *dev)
 
 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
 		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
+		struct i915_vma *vma = reg->vma;
 
 		/*
 		 * Commit delayed tiling changes if we have an object still
 		 * attached to the fence, otherwise just clear the fence.
 		 */
-		if (reg->obj) {
-			i915_gem_object_update_fence(reg->obj, reg,
-						     reg->obj->tiling_mode);
-		} else {
-			i915_gem_write_fence(dev, i, NULL);
-		}
+		if (vma && !i915_gem_object_is_tiled(vma->obj))
+			vma = NULL;
+
+		fence_update(reg, vma);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 10f1e32767e6..0bb4232f66bc 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -32,6 +32,8 @@
 #include "i915_trace.h"
 #include "intel_drv.h"
 
+#define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
+
 /**
  * DOC: Global GTT views
  *
@@ -122,8 +124,11 @@ int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 	has_full_48bit_ppgtt =
 	       	IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9;
 
-	if (intel_vgpu_active(dev_priv))
-		has_full_ppgtt = false; /* emulation is too hard */
+	if (intel_vgpu_active(dev_priv)) {
+		/* emulation is too hard */
+		has_full_ppgtt = false;
+		has_full_48bit_ppgtt = false;
+	}
 
 	if (!has_aliasing_ppgtt)
 		return 0;
@@ -158,7 +163,7 @@ int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 		return 0;
 	}
 
-	if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists)
+	if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt)
 		return has_full_48bit_ppgtt ? 3 : 2;
 	else
 		return has_aliasing_ppgtt ? 1 : 0;
@@ -170,11 +175,13 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
 {
 	u32 pte_flags = 0;
 
+	vma->pages = vma->obj->pages;
+
 	/* Currently applicable only to VLV */
 	if (vma->obj->gt_ro)
 		pte_flags |= PTE_READ_ONLY;
 
-	vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
+	vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
 				cache_level, pte_flags);
 
 	return 0;
@@ -184,7 +191,7 @@ static void ppgtt_unbind_vma(struct i915_vma *vma)
 {
 	vma->vm->clear_range(vma->vm,
 			     vma->node.start,
-			     vma->obj->base.size,
+			     vma->size,
 			     true);
 }
 
@@ -324,16 +331,16 @@ static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 static int __setup_page_dma(struct drm_device *dev,
 			    struct i915_page_dma *p, gfp_t flags)
 {
-	struct device *device = &dev->pdev->dev;
+	struct device *kdev = &dev->pdev->dev;
 
 	p->page = alloc_page(flags);
 	if (!p->page)
 		return -ENOMEM;
 
-	p->daddr = dma_map_page(device,
+	p->daddr = dma_map_page(kdev,
 				p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
 
-	if (dma_mapping_error(device, p->daddr)) {
+	if (dma_mapping_error(kdev, p->daddr)) {
 		__free_page(p->page);
 		return -EINVAL;
 	}
@@ -343,15 +350,17 @@ static int __setup_page_dma(struct drm_device *dev,
 
 static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
 {
-	return __setup_page_dma(dev, p, GFP_KERNEL);
+	return __setup_page_dma(dev, p, I915_GFP_DMA);
 }
 
 static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
 {
+	struct pci_dev *pdev = dev->pdev;
+
 	if (WARN_ON(!p->page))
 		return;
 
-	dma_unmap_page(&dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL);
+	dma_unmap_page(&pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL);
 	__free_page(p->page);
 	memset(p, 0, sizeof(*p));
 }
@@ -405,33 +414,18 @@ static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p,
 	fill_page_dma(dev, p, v);
 }
 
-static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev)
+static int
+setup_scratch_page(struct drm_device *dev,
+		   struct i915_page_dma *scratch,
+		   gfp_t gfp)
 {
-	struct i915_page_scratch *sp;
-	int ret;
-
-	sp = kzalloc(sizeof(*sp), GFP_KERNEL);
-	if (sp == NULL)
-		return ERR_PTR(-ENOMEM);
-
-	ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO);
-	if (ret) {
-		kfree(sp);
-		return ERR_PTR(ret);
-	}
-
-	set_pages_uc(px_page(sp), 1);
-
-	return sp;
+	return __setup_page_dma(dev, scratch, gfp | __GFP_ZERO);
 }
 
-static void free_scratch_page(struct drm_device *dev,
-			      struct i915_page_scratch *sp)
+static void cleanup_scratch_page(struct drm_device *dev,
+				 struct i915_page_dma *scratch)
 {
-	set_pages_wb(px_page(sp), 1);
-
-	cleanup_px(dev, sp);
-	kfree(sp);
+	cleanup_page_dma(dev, scratch);
 }
 
 static struct i915_page_table *alloc_pt(struct drm_device *dev)
@@ -477,7 +471,7 @@ static void gen8_initialize_pt(struct i915_address_space *vm,
 {
 	gen8_pte_t scratch_pte;
 
-	scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+	scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
 				      I915_CACHE_LLC, true);
 
 	fill_px(vm->dev, pt, scratch_pte);
@@ -488,9 +482,9 @@ static void gen6_initialize_pt(struct i915_address_space *vm,
 {
 	gen6_pte_t scratch_pte;
 
-	WARN_ON(px_dma(vm->scratch_page) == 0);
+	WARN_ON(vm->scratch_page.daddr == 0);
 
-	scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
+	scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
 				     I915_CACHE_LLC, true, 0);
 
 	fill32_px(vm->dev, pt, scratch_pte);
@@ -669,6 +663,7 @@ static int gen8_write_pdp(struct drm_i915_gem_request *req,
 			  unsigned entry,
 			  dma_addr_t addr)
 {
+	struct intel_ring *ring = req->ring;
 	struct intel_engine_cs *engine = req->engine;
 	int ret;
 
@@ -678,13 +673,13 @@ static int gen8_write_pdp(struct drm_i915_gem_request *req,
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
-	intel_ring_emit_reg(engine, GEN8_RING_PDP_UDW(engine, entry));
-	intel_ring_emit(engine, upper_32_bits(addr));
-	intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
-	intel_ring_emit_reg(engine, GEN8_RING_PDP_LDW(engine, entry));
-	intel_ring_emit(engine, lower_32_bits(addr));
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+	intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(engine, entry));
+	intel_ring_emit(ring, upper_32_bits(addr));
+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+	intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(engine, entry));
+	intel_ring_emit(ring, lower_32_bits(addr));
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -773,7 +768,7 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 				   bool use_scratch)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
-	gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+	gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
 						 I915_CACHE_LLC, use_scratch);
 
 	if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
@@ -879,9 +874,9 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 	struct drm_device *dev = vm->dev;
 	int ret;
 
-	vm->scratch_page = alloc_scratch_page(dev);
-	if (IS_ERR(vm->scratch_page))
-		return PTR_ERR(vm->scratch_page);
+	ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA);
+	if (ret)
+		return ret;
 
 	vm->scratch_pt = alloc_pt(dev);
 	if (IS_ERR(vm->scratch_pt)) {
@@ -915,7 +910,7 @@ free_pd:
 free_pt:
 	free_pt(dev, vm->scratch_pt);
 free_scratch_page:
-	free_scratch_page(dev, vm->scratch_page);
+	cleanup_scratch_page(dev, &vm->scratch_page);
 
 	return ret;
 }
@@ -959,7 +954,7 @@ static void gen8_free_scratch(struct i915_address_space *vm)
 		free_pdp(dev, vm->scratch_pdp);
 	free_pd(dev, vm->scratch_pd);
 	free_pt(dev, vm->scratch_pt);
-	free_scratch_page(dev, vm->scratch_page);
+	cleanup_scratch_page(dev, &vm->scratch_page);
 }
 
 static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev,
@@ -1456,7 +1451,7 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 	struct i915_address_space *vm = &ppgtt->base;
 	uint64_t start = ppgtt->base.start;
 	uint64_t length = ppgtt->base.total;
-	gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+	gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
 						 I915_CACHE_LLC, true);
 
 	if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
@@ -1573,7 +1568,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 	uint32_t  pte, pde;
 	uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
 
-	scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
+	scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
 				     I915_CACHE_LLC, true, 0);
 
 	gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
@@ -1660,11 +1655,12 @@ static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
 			 struct drm_i915_gem_request *req)
 {
+	struct intel_ring *ring = req->ring;
 	struct intel_engine_cs *engine = req->engine;
 	int ret;
 
 	/* NB: TLBs must be flushed and invalidated before a switch */
-	ret = engine->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+	ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
 	if (ret)
 		return ret;
 
@@ -1672,13 +1668,13 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(2));
-	intel_ring_emit_reg(engine, RING_PP_DIR_DCLV(engine));
-	intel_ring_emit(engine, PP_DIR_DCLV_2G);
-	intel_ring_emit_reg(engine, RING_PP_DIR_BASE(engine));
-	intel_ring_emit(engine, get_pd_offset(ppgtt));
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
+	intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(engine));
+	intel_ring_emit(ring, PP_DIR_DCLV_2G);
+	intel_ring_emit_reg(ring, RING_PP_DIR_BASE(engine));
+	intel_ring_emit(ring, get_pd_offset(ppgtt));
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -1686,11 +1682,12 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
 			  struct drm_i915_gem_request *req)
 {
+	struct intel_ring *ring = req->ring;
 	struct intel_engine_cs *engine = req->engine;
 	int ret;
 
 	/* NB: TLBs must be flushed and invalidated before a switch */
-	ret = engine->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+	ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
 	if (ret)
 		return ret;
 
@@ -1698,17 +1695,17 @@ static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(2));
-	intel_ring_emit_reg(engine, RING_PP_DIR_DCLV(engine));
-	intel_ring_emit(engine, PP_DIR_DCLV_2G);
-	intel_ring_emit_reg(engine, RING_PP_DIR_BASE(engine));
-	intel_ring_emit(engine, get_pd_offset(ppgtt));
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
+	intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(engine));
+	intel_ring_emit(ring, PP_DIR_DCLV_2G);
+	intel_ring_emit_reg(ring, RING_PP_DIR_BASE(engine));
+	intel_ring_emit(ring, get_pd_offset(ppgtt));
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	/* XXX: RCS is the only one to auto invalidate the TLBs? */
 	if (engine->id != RCS) {
-		ret = engine->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+		ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
 		if (ret)
 			return ret;
 	}
@@ -1796,7 +1793,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 	unsigned first_pte = first_entry % GEN6_PTES;
 	unsigned last_pte, i;
 
-	scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
+	scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
 				     I915_CACHE_LLC, true, 0);
 
 	while (num_entries) {
@@ -1942,14 +1939,15 @@ unwind_out:
 static int gen6_init_scratch(struct i915_address_space *vm)
 {
 	struct drm_device *dev = vm->dev;
+	int ret;
 
-	vm->scratch_page = alloc_scratch_page(dev);
-	if (IS_ERR(vm->scratch_page))
-		return PTR_ERR(vm->scratch_page);
+	ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA);
+	if (ret)
+		return ret;
 
 	vm->scratch_pt = alloc_pt(dev);
 	if (IS_ERR(vm->scratch_pt)) {
-		free_scratch_page(dev, vm->scratch_page);
+		cleanup_scratch_page(dev, &vm->scratch_page);
 		return PTR_ERR(vm->scratch_pt);
 	}
 
@@ -1963,7 +1961,7 @@ static void gen6_free_scratch(struct i915_address_space *vm)
 	struct drm_device *dev = vm->dev;
 
 	free_pt(dev, vm->scratch_pt);
-	free_scratch_page(dev, vm->scratch_page);
+	cleanup_scratch_page(dev, &vm->scratch_page);
 }
 
 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
@@ -2009,7 +2007,7 @@ alloc:
 						  0, ggtt->base.total,
 						  DRM_MM_TOPDOWN);
 	if (ret == -ENOSPC && !retried) {
-		ret = i915_gem_evict_something(dev, &ggtt->base,
+		ret = i915_gem_evict_something(&ggtt->base,
 					       GEN6_PD_SIZE, GEN6_PD_ALIGN,
 					       I915_CACHE_NONE,
 					       0, ggtt->base.total,
@@ -2101,11 +2099,12 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	return 0;
 }
 
-static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
+static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
+			   struct drm_i915_private *dev_priv)
 {
-	ppgtt->base.dev = dev;
+	ppgtt->base.dev = &dev_priv->drm;
 
-	if (INTEL_INFO(dev)->gen < 8)
+	if (INTEL_INFO(dev_priv)->gen < 8)
 		return gen6_ppgtt_init(ppgtt);
 	else
 		return gen8_ppgtt_init(ppgtt);
@@ -2115,9 +2114,9 @@ static void i915_address_space_init(struct i915_address_space *vm,
 				    struct drm_i915_private *dev_priv)
 {
 	drm_mm_init(&vm->mm, vm->start, vm->total);
-	vm->dev = &dev_priv->drm;
 	INIT_LIST_HEAD(&vm->active_list);
 	INIT_LIST_HEAD(&vm->inactive_list);
+	INIT_LIST_HEAD(&vm->unbound_list);
 	list_add_tail(&vm->global_link, &dev_priv->vm_list);
 }
 
@@ -2140,15 +2139,17 @@ static void gtt_write_workarounds(struct drm_device *dev)
 		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
 }
 
-static int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
+static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
+			   struct drm_i915_private *dev_priv,
+			   struct drm_i915_file_private *file_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret = 0;
+	int ret;
 
-	ret = __hw_ppgtt_init(dev, ppgtt);
+	ret = __hw_ppgtt_init(ppgtt, dev_priv);
 	if (ret == 0) {
 		kref_init(&ppgtt->ref);
 		i915_address_space_init(&ppgtt->base, dev_priv);
+		ppgtt->base.file = file_priv;
 	}
 
 	return ret;
@@ -2180,7 +2181,8 @@ int i915_ppgtt_init_hw(struct drm_device *dev)
 }
 
 struct i915_hw_ppgtt *
-i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
+i915_ppgtt_create(struct drm_i915_private *dev_priv,
+		  struct drm_i915_file_private *fpriv)
 {
 	struct i915_hw_ppgtt *ppgtt;
 	int ret;
@@ -2189,14 +2191,12 @@ i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
 	if (!ppgtt)
 		return ERR_PTR(-ENOMEM);
 
-	ret = i915_ppgtt_init(dev, ppgtt);
+	ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv);
 	if (ret) {
 		kfree(ppgtt);
 		return ERR_PTR(ret);
 	}
 
-	ppgtt->file_priv = fpriv;
-
 	trace_i915_ppgtt_create(&ppgtt->base);
 
 	return ppgtt;
@@ -2209,9 +2209,10 @@ void  i915_ppgtt_release(struct kref *kref)
 
 	trace_i915_ppgtt_release(&ppgtt->base);
 
-	/* vmas should already be unbound */
+	/* vmas should already be unbound and destroyed */
 	WARN_ON(!list_empty(&ppgtt->base.active_list));
 	WARN_ON(!list_empty(&ppgtt->base.inactive_list));
+	WARN_ON(!list_empty(&ppgtt->base.unbound_list));
 
 	list_del(&ppgtt->base.global_link);
 	drm_mm_takedown(&ppgtt->base.mm);
@@ -2220,47 +2221,21 @@ void  i915_ppgtt_release(struct kref *kref)
 	kfree(ppgtt);
 }
 
-extern int intel_iommu_gfx_mapped;
 /* Certain Gen5 chipsets require require idling the GPU before
  * unmapping anything from the GTT when VT-d is enabled.
  */
-static bool needs_idle_maps(struct drm_device *dev)
+static bool needs_idle_maps(struct drm_i915_private *dev_priv)
 {
 #ifdef CONFIG_INTEL_IOMMU
 	/* Query intel_iommu to see if we need the workaround. Presumably that
 	 * was loaded first.
 	 */
-	if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
+	if (IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_iommu_gfx_mapped)
 		return true;
 #endif
 	return false;
 }
 
-static bool do_idling(struct drm_i915_private *dev_priv)
-{
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	bool ret = dev_priv->mm.interruptible;
-
-	if (unlikely(ggtt->do_idle_maps)) {
-		dev_priv->mm.interruptible = false;
-		if (i915_gem_wait_for_idle(dev_priv)) {
-			DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
-			/* Wait a bit, in hopes it avoids the hang */
-			udelay(10);
-		}
-	}
-
-	return ret;
-}
-
-static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
-{
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-
-	if (unlikely(ggtt->do_idle_maps))
-		dev_priv->mm.interruptible = interruptible;
-}
-
 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
@@ -2329,12 +2304,7 @@ int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 
 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
 {
-#ifdef writeq
 	writeq(pte, addr);
-#else
-	iowrite32((u32)pte, addr);
-	iowrite32(pte >> 32, addr + 4);
-#endif
 }
 
 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
@@ -2527,7 +2497,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 		 first_entry, num_entries, max_entries))
 		num_entries = max_entries;
 
-	scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
+	scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
 				      I915_CACHE_LLC,
 				      use_scratch);
 	for (i = 0; i < num_entries; i++)
@@ -2559,7 +2529,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 		 first_entry, num_entries, max_entries))
 		num_entries = max_entries;
 
-	scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
+	scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
 				     I915_CACHE_LLC, use_scratch, 0);
 
 	for (i = 0; i < num_entries; i++)
@@ -2638,8 +2608,7 @@ static int ggtt_bind_vma(struct i915_vma *vma,
 	if (obj->gt_ro)
 		pte_flags |= PTE_READ_ONLY;
 
-	vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages,
-				vma->node.start,
+	vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
 				cache_level, pte_flags);
 
 	/*
@@ -2647,7 +2616,7 @@ static int ggtt_bind_vma(struct i915_vma *vma,
 	 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
 	 * upgrade to both bound if we bind either to avoid double-binding.
 	 */
-	vma->bound |= GLOBAL_BIND | LOCAL_BIND;
+	vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 
 	return 0;
 }
@@ -2669,19 +2638,17 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 		pte_flags |= PTE_READ_ONLY;
 
 
-	if (flags & GLOBAL_BIND) {
+	if (flags & I915_VMA_GLOBAL_BIND) {
 		vma->vm->insert_entries(vma->vm,
-					vma->ggtt_view.pages,
-					vma->node.start,
+					vma->pages, vma->node.start,
 					cache_level, pte_flags);
 	}
 
-	if (flags & LOCAL_BIND) {
+	if (flags & I915_VMA_LOCAL_BIND) {
 		struct i915_hw_ppgtt *appgtt =
 			to_i915(vma->vm->dev)->mm.aliasing_ppgtt;
 		appgtt->base.insert_entries(&appgtt->base,
-					    vma->ggtt_view.pages,
-					    vma->node.start,
+					    vma->pages, vma->node.start,
 					    cache_level, pte_flags);
 	}
 
@@ -2690,42 +2657,36 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 
 static void ggtt_unbind_vma(struct i915_vma *vma)
 {
-	struct drm_device *dev = vma->vm->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_i915_gem_object *obj = vma->obj;
-	const uint64_t size = min_t(uint64_t,
-				    obj->base.size,
-				    vma->node.size);
+	struct i915_hw_ppgtt *appgtt = to_i915(vma->vm->dev)->mm.aliasing_ppgtt;
+	const u64 size = min(vma->size, vma->node.size);
 
-	if (vma->bound & GLOBAL_BIND) {
+	if (vma->flags & I915_VMA_GLOBAL_BIND)
 		vma->vm->clear_range(vma->vm,
-				     vma->node.start,
-				     size,
+				     vma->node.start, size,
 				     true);
-	}
-
-	if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) {
-		struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
 
+	if (vma->flags & I915_VMA_LOCAL_BIND && appgtt)
 		appgtt->base.clear_range(&appgtt->base,
-					 vma->node.start,
-					 size,
+					 vma->node.start, size,
 					 true);
-	}
 }
 
 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	bool interruptible;
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+	struct device *kdev = &dev_priv->drm.pdev->dev;
+	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 
-	interruptible = do_idling(dev_priv);
+	if (unlikely(ggtt->do_idle_maps)) {
+		if (i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED)) {
+			DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
+			/* Wait a bit, in hopes it avoids the hang */
+			udelay(10);
+		}
+	}
 
-	dma_unmap_sg(&dev->pdev->dev, obj->pages->sgl, obj->pages->nents,
+	dma_unmap_sg(kdev, obj->pages->sgl, obj->pages->nents,
 		     PCI_DMA_BIDIRECTIONAL);
-
-	undo_idling(dev_priv, interruptible);
 }
 
 static void i915_gtt_color_adjust(struct drm_mm_node *node,
@@ -2736,19 +2697,14 @@ static void i915_gtt_color_adjust(struct drm_mm_node *node,
 	if (node->color != color)
 		*start += 4096;
 
-	if (!list_empty(&node->node_list)) {
-		node = list_entry(node->node_list.next,
-				  struct drm_mm_node,
-				  node_list);
-		if (node->allocated && node->color != color)
-			*end -= 4096;
-	}
+	node = list_first_entry_or_null(&node->node_list,
+					struct drm_mm_node,
+					node_list);
+	if (node && node->allocated && node->color != color)
+		*end -= 4096;
 }
 
-static int i915_gem_setup_global_gtt(struct drm_device *dev,
-				     u64 start,
-				     u64 mappable_end,
-				     u64 end)
+int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
 {
 	/* Let GEM Manage all of the aperture.
 	 *
@@ -2759,48 +2715,15 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev,
 	 * aperture.  One page should be enough to keep any prefetching inside
 	 * of the aperture.
 	 */
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct drm_mm_node *entry;
-	struct drm_i915_gem_object *obj;
 	unsigned long hole_start, hole_end;
+	struct drm_mm_node *entry;
 	int ret;
 
-	BUG_ON(mappable_end > end);
-
-	ggtt->base.start = start;
-
-	/* Subtract the guard page before address space initialization to
-	 * shrink the range used by drm_mm */
-	ggtt->base.total = end - start - PAGE_SIZE;
-	i915_address_space_init(&ggtt->base, dev_priv);
-	ggtt->base.total += PAGE_SIZE;
-
 	ret = intel_vgt_balloon(dev_priv);
 	if (ret)
 		return ret;
 
-	if (!HAS_LLC(dev))
-		ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
-
-	/* Mark any preallocated objects as occupied */
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-		struct i915_vma *vma = i915_gem_obj_to_vma(obj, &ggtt->base);
-
-		DRM_DEBUG_KMS("reserving preallocated space: %llx + %zx\n",
-			      i915_gem_obj_ggtt_offset(obj), obj->base.size);
-
-		WARN_ON(i915_gem_obj_ggtt_bound(obj));
-		ret = drm_mm_reserve_node(&ggtt->base.mm, &vma->node);
-		if (ret) {
-			DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
-			return ret;
-		}
-		vma->bound |= GLOBAL_BIND;
-		__i915_vma_set_map_and_fenceable(vma);
-		list_add_tail(&vma->vm_link, &ggtt->base.inactive_list);
-	}
-
 	/* Clear any non-preallocated blocks */
 	drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
 		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
@@ -2810,18 +2733,19 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev,
 	}
 
 	/* And finally clear the reserved guard page */
-	ggtt->base.clear_range(&ggtt->base, end - PAGE_SIZE, PAGE_SIZE, true);
+	ggtt->base.clear_range(&ggtt->base,
+			       ggtt->base.total - PAGE_SIZE, PAGE_SIZE,
+			       true);
 
-	if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
+	if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
 		struct i915_hw_ppgtt *ppgtt;
 
 		ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
 		if (!ppgtt)
 			return -ENOMEM;
 
-		ret = __hw_ppgtt_init(dev, ppgtt);
+		ret = __hw_ppgtt_init(ppgtt, dev_priv);
 		if (ret) {
-			ppgtt->base.cleanup(&ppgtt->base);
 			kfree(ppgtt);
 			return ret;
 		}
@@ -2849,33 +2773,20 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev,
 }
 
 /**
- * i915_gem_init_ggtt - Initialize GEM for Global GTT
- * @dev: DRM device
- */
-void i915_gem_init_ggtt(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-
-	i915_gem_setup_global_gtt(dev, 0, ggtt->mappable_end, ggtt->base.total);
-}
-
-/**
  * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
- * @dev: DRM device
+ * @dev_priv: i915 device
  */
-void i915_ggtt_cleanup_hw(struct drm_device *dev)
+void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 
 	if (dev_priv->mm.aliasing_ppgtt) {
 		struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
-
 		ppgtt->base.cleanup(&ppgtt->base);
+		kfree(ppgtt);
 	}
 
-	i915_gem_cleanup_stolen(dev);
+	i915_gem_cleanup_stolen(&dev_priv->drm);
 
 	if (drm_mm_initialized(&ggtt->base.mm)) {
 		intel_vgt_deballoon(dev_priv);
@@ -2885,6 +2796,9 @@ void i915_ggtt_cleanup_hw(struct drm_device *dev)
 	}
 
 	ggtt->base.cleanup(&ggtt->base);
+
+	arch_phys_wc_del(ggtt->mtrr);
+	io_mapping_fini(&ggtt->mappable);
 }
 
 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
@@ -2965,17 +2879,14 @@ static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
 		return (gen9_gmch_ctl - 0xf0 + 1) << 22;
 }
 
-static int ggtt_probe_common(struct drm_device *dev,
-			     size_t gtt_size)
+static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct i915_page_scratch *scratch_page;
-	phys_addr_t ggtt_phys_addr;
+	struct pci_dev *pdev = ggtt->base.dev->pdev;
+	phys_addr_t phys_addr;
+	int ret;
 
 	/* For Modern GENs the PTEs and register space are split in the BAR */
-	ggtt_phys_addr = pci_resource_start(dev->pdev, 0) +
-			 (pci_resource_len(dev->pdev, 0) / 2);
+	phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
 
 	/*
 	 * On BXT writes larger than 64 bit to the GTT pagetable range will be
@@ -2984,25 +2895,25 @@ static int ggtt_probe_common(struct drm_device *dev,
 	 * resort to an uncached mapping. The WC issue is easily caught by the
 	 * readback check when writing GTT PTE entries.
 	 */
-	if (IS_BROXTON(dev))
-		ggtt->gsm = ioremap_nocache(ggtt_phys_addr, gtt_size);
+	if (IS_BROXTON(ggtt->base.dev))
+		ggtt->gsm = ioremap_nocache(phys_addr, size);
 	else
-		ggtt->gsm = ioremap_wc(ggtt_phys_addr, gtt_size);
+		ggtt->gsm = ioremap_wc(phys_addr, size);
 	if (!ggtt->gsm) {
-		DRM_ERROR("Failed to map the gtt page table\n");
+		DRM_ERROR("Failed to map the ggtt page table\n");
 		return -ENOMEM;
 	}
 
-	scratch_page = alloc_scratch_page(dev);
-	if (IS_ERR(scratch_page)) {
+	ret = setup_scratch_page(ggtt->base.dev,
+				 &ggtt->base.scratch_page,
+				 GFP_DMA32);
+	if (ret) {
 		DRM_ERROR("Scratch setup failed\n");
 		/* iounmap will also get called at remove, but meh */
 		iounmap(ggtt->gsm);
-		return PTR_ERR(scratch_page);
+		return ret;
 	}
 
-	ggtt->base.scratch_page = scratch_page;
-
 	return 0;
 }
 
@@ -3079,42 +2990,49 @@ static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
 }
 
+static void gen6_gmch_remove(struct i915_address_space *vm)
+{
+	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
+
+	iounmap(ggtt->gsm);
+	cleanup_scratch_page(vm->dev, &vm->scratch_page);
+}
+
 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 {
-	struct drm_device *dev = ggtt->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(ggtt->base.dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
+	unsigned int size;
 	u16 snb_gmch_ctl;
-	int ret;
 
 	/* TODO: We're not aware of mappable constraints on gen8 yet */
-	ggtt->mappable_base = pci_resource_start(dev->pdev, 2);
-	ggtt->mappable_end = pci_resource_len(dev->pdev, 2);
+	ggtt->mappable_base = pci_resource_start(pdev, 2);
+	ggtt->mappable_end = pci_resource_len(pdev, 2);
 
-	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
-		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
+	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(39)))
+		pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
 
-	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
+	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 
-	if (INTEL_INFO(dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl);
-		ggtt->size = gen8_get_total_gtt_size(snb_gmch_ctl);
-	} else if (IS_CHERRYVIEW(dev)) {
+		size = gen8_get_total_gtt_size(snb_gmch_ctl);
+	} else if (IS_CHERRYVIEW(dev_priv)) {
 		ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl);
-		ggtt->size = chv_get_total_gtt_size(snb_gmch_ctl);
+		size = chv_get_total_gtt_size(snb_gmch_ctl);
 	} else {
 		ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl);
-		ggtt->size = gen8_get_total_gtt_size(snb_gmch_ctl);
+		size = gen8_get_total_gtt_size(snb_gmch_ctl);
 	}
 
-	ggtt->base.total = (ggtt->size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
+	ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
 
-	if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
+	if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
 		chv_setup_private_ppat(dev_priv);
 	else
 		bdw_setup_private_ppat(dev_priv);
 
-	ret = ggtt_probe_common(dev, ggtt->size);
-
+	ggtt->base.cleanup = gen6_gmch_remove;
 	ggtt->base.bind_vma = ggtt_bind_vma;
 	ggtt->base.unbind_vma = ggtt_unbind_vma;
 	ggtt->base.insert_page = gen8_ggtt_insert_page;
@@ -3126,57 +3044,65 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 	if (IS_CHERRYVIEW(dev_priv))
 		ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL;
 
-	return ret;
+	return ggtt_probe_common(ggtt, size);
 }
 
 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
 {
-	struct drm_device *dev = ggtt->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(ggtt->base.dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
+	unsigned int size;
 	u16 snb_gmch_ctl;
-	int ret;
 
-	ggtt->mappable_base = pci_resource_start(dev->pdev, 2);
-	ggtt->mappable_end = pci_resource_len(dev->pdev, 2);
+	ggtt->mappable_base = pci_resource_start(pdev, 2);
+	ggtt->mappable_end = pci_resource_len(pdev, 2);
 
 	/* 64/512MB is the current min/max we actually know of, but this is just
 	 * a coarse sanity check.
 	 */
-	if ((ggtt->mappable_end < (64<<20) || (ggtt->mappable_end > (512<<20)))) {
+	if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
 		DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end);
 		return -ENXIO;
 	}
 
-	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
-		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
-	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
+	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(40)))
+		pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
+	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 
 	ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
-	ggtt->size = gen6_get_total_gtt_size(snb_gmch_ctl);
-	ggtt->base.total = (ggtt->size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
 
-	ret = ggtt_probe_common(dev, ggtt->size);
+	size = gen6_get_total_gtt_size(snb_gmch_ctl);
+	ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
 
 	ggtt->base.clear_range = gen6_ggtt_clear_range;
 	ggtt->base.insert_page = gen6_ggtt_insert_page;
 	ggtt->base.insert_entries = gen6_ggtt_insert_entries;
 	ggtt->base.bind_vma = ggtt_bind_vma;
 	ggtt->base.unbind_vma = ggtt_unbind_vma;
+	ggtt->base.cleanup = gen6_gmch_remove;
+
+	if (HAS_EDRAM(dev_priv))
+		ggtt->base.pte_encode = iris_pte_encode;
+	else if (IS_HASWELL(dev_priv))
+		ggtt->base.pte_encode = hsw_pte_encode;
+	else if (IS_VALLEYVIEW(dev_priv))
+		ggtt->base.pte_encode = byt_pte_encode;
+	else if (INTEL_GEN(dev_priv) >= 7)
+		ggtt->base.pte_encode = ivb_pte_encode;
+	else
+		ggtt->base.pte_encode = snb_pte_encode;
 
-	return ret;
+	return ggtt_probe_common(ggtt, size);
 }
 
-static void gen6_gmch_remove(struct i915_address_space *vm)
+static void i915_gmch_remove(struct i915_address_space *vm)
 {
-	struct i915_ggtt *ggtt = container_of(vm, struct i915_ggtt, base);
-
-	iounmap(ggtt->gsm);
-	free_scratch_page(vm->dev, vm->scratch_page);
+	intel_gmch_remove();
 }
 
 static int i915_gmch_probe(struct i915_ggtt *ggtt)
 {
-	struct drm_device *dev = ggtt->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(ggtt->base.dev);
 	int ret;
 
 	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
@@ -3188,12 +3114,13 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt)
 	intel_gtt_get(&ggtt->base.total, &ggtt->stolen_size,
 		      &ggtt->mappable_base, &ggtt->mappable_end);
 
-	ggtt->do_idle_maps = needs_idle_maps(&dev_priv->drm);
+	ggtt->do_idle_maps = needs_idle_maps(dev_priv);
 	ggtt->base.insert_page = i915_ggtt_insert_page;
 	ggtt->base.insert_entries = i915_ggtt_insert_entries;
 	ggtt->base.clear_range = i915_ggtt_clear_range;
 	ggtt->base.bind_vma = ggtt_bind_vma;
 	ggtt->base.unbind_vma = ggtt_unbind_vma;
+	ggtt->base.cleanup = i915_gmch_remove;
 
 	if (unlikely(ggtt->do_idle_maps))
 		DRM_INFO("applying Ironlake quirks for intel_iommu\n");
@@ -3201,65 +3128,40 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt)
 	return 0;
 }
 
-static void i915_gmch_remove(struct i915_address_space *vm)
-{
-	intel_gmch_remove();
-}
-
 /**
- * i915_ggtt_init_hw - Initialize GGTT hardware
- * @dev: DRM device
+ * i915_ggtt_probe_hw - Probe GGTT hardware location
+ * @dev_priv: i915 device
  */
-int i915_ggtt_init_hw(struct drm_device *dev)
+int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	int ret;
 
-	if (INTEL_INFO(dev)->gen <= 5) {
-		ggtt->probe = i915_gmch_probe;
-		ggtt->base.cleanup = i915_gmch_remove;
-	} else if (INTEL_INFO(dev)->gen < 8) {
-		ggtt->probe = gen6_gmch_probe;
-		ggtt->base.cleanup = gen6_gmch_remove;
-
-		if (HAS_EDRAM(dev))
-			ggtt->base.pte_encode = iris_pte_encode;
-		else if (IS_HASWELL(dev))
-			ggtt->base.pte_encode = hsw_pte_encode;
-		else if (IS_VALLEYVIEW(dev))
-			ggtt->base.pte_encode = byt_pte_encode;
-		else if (INTEL_INFO(dev)->gen >= 7)
-			ggtt->base.pte_encode = ivb_pte_encode;
-		else
-			ggtt->base.pte_encode = snb_pte_encode;
-	} else {
-		ggtt->probe = gen8_gmch_probe;
-		ggtt->base.cleanup = gen6_gmch_remove;
-	}
-
-	ggtt->base.dev = dev;
-	ggtt->base.is_ggtt = true;
+	ggtt->base.dev = &dev_priv->drm;
 
-	ret = ggtt->probe(ggtt);
+	if (INTEL_GEN(dev_priv) <= 5)
+		ret = i915_gmch_probe(ggtt);
+	else if (INTEL_GEN(dev_priv) < 8)
+		ret = gen6_gmch_probe(ggtt);
+	else
+		ret = gen8_gmch_probe(ggtt);
 	if (ret)
 		return ret;
 
 	if ((ggtt->base.total - 1) >> 32) {
 		DRM_ERROR("We never expected a Global GTT with more than 32bits"
-			  "of address space! Found %lldM!\n",
+			  " of address space! Found %lldM!\n",
 			  ggtt->base.total >> 20);
 		ggtt->base.total = 1ULL << 32;
 		ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
 	}
 
-	/*
-	 * Initialise stolen early so that we may reserve preallocated
-	 * objects for the BIOS to KMS transition.
-	 */
-	ret = i915_gem_init_stolen(dev);
-	if (ret)
-		goto out_gtt_cleanup;
+	if (ggtt->mappable_end > ggtt->base.total) {
+		DRM_ERROR("mappable aperture extends past end of GGTT,"
+			  " aperture=%llx, total=%llx\n",
+			  ggtt->mappable_end, ggtt->base.total);
+		ggtt->mappable_end = ggtt->base.total;
+	}
 
 	/* GMADR is the PCI mmio aperture into the global GTT. */
 	DRM_INFO("Memory usable by graphics device = %lluM\n",
@@ -3272,16 +3174,55 @@ int i915_ggtt_init_hw(struct drm_device *dev)
 #endif
 
 	return 0;
+}
+
+/**
+ * i915_ggtt_init_hw - Initialize GGTT hardware
+ * @dev_priv: i915 device
+ */
+int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
+{
+	struct i915_ggtt *ggtt = &dev_priv->ggtt;
+	int ret;
+
+	INIT_LIST_HEAD(&dev_priv->vm_list);
+
+	/* Subtract the guard page before address space initialization to
+	 * shrink the range used by drm_mm.
+	 */
+	ggtt->base.total -= PAGE_SIZE;
+	i915_address_space_init(&ggtt->base, dev_priv);
+	ggtt->base.total += PAGE_SIZE;
+	if (!HAS_LLC(dev_priv))
+		ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
+
+	if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
+				dev_priv->ggtt.mappable_base,
+				dev_priv->ggtt.mappable_end)) {
+		ret = -EIO;
+		goto out_gtt_cleanup;
+	}
+
+	ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, ggtt->mappable_end);
+
+	/*
+	 * Initialise stolen early so that we may reserve preallocated
+	 * objects for the BIOS to KMS transition.
+	 */
+	ret = i915_gem_init_stolen(&dev_priv->drm);
+	if (ret)
+		goto out_gtt_cleanup;
+
+	return 0;
 
 out_gtt_cleanup:
 	ggtt->base.cleanup(&ggtt->base);
-
 	return ret;
 }
 
-int i915_ggtt_enable_hw(struct drm_device *dev)
+int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
 {
-	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
+	if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
 		return -EIO;
 
 	return 0;
@@ -3291,8 +3232,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct drm_i915_gem_object *obj;
-	struct i915_vma *vma;
+	struct drm_i915_gem_object *obj, *on;
 
 	i915_check_and_clear_faults(dev_priv);
 
@@ -3300,20 +3240,32 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 	ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total,
 			       true);
 
-	/* Cache flush objects bound into GGTT and rebind them. */
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+	ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */
+
+	/* clflush objects bound into the GGTT and rebind them. */
+	list_for_each_entry_safe(obj, on,
+				 &dev_priv->mm.bound_list, global_list) {
+		bool ggtt_bound = false;
+		struct i915_vma *vma;
+
 		list_for_each_entry(vma, &obj->vma_list, obj_link) {
 			if (vma->vm != &ggtt->base)
 				continue;
 
+			if (!i915_vma_unbind(vma))
+				continue;
+
 			WARN_ON(i915_vma_bind(vma, obj->cache_level,
 					      PIN_UPDATE));
+			ggtt_bound = true;
 		}
 
-		if (obj->pin_display)
+		if (ggtt_bound)
 			WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
 	}
 
+	ggtt->base.closed = false;
+
 	if (INTEL_INFO(dev)->gen >= 8) {
 		if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
 			chv_setup_private_ppat(dev_priv);
@@ -3331,7 +3283,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 
 			struct i915_hw_ppgtt *ppgtt;
 
-			if (vm->is_ggtt)
+			if (i915_is_ggtt(vm))
 				ppgtt = dev_priv->mm.aliasing_ppgtt;
 			else
 				ppgtt = i915_vm_to_ppgtt(vm);
@@ -3344,65 +3296,155 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 	i915_ggtt_flush(dev_priv);
 }
 
+static void
+i915_vma_retire(struct i915_gem_active *active,
+		struct drm_i915_gem_request *rq)
+{
+	const unsigned int idx = rq->engine->id;
+	struct i915_vma *vma =
+		container_of(active, struct i915_vma, last_read[idx]);
+
+	GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx));
+
+	i915_vma_clear_active(vma, idx);
+	if (i915_vma_is_active(vma))
+		return;
+
+	list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+	if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma)))
+		WARN_ON(i915_vma_unbind(vma));
+}
+
+void i915_vma_destroy(struct i915_vma *vma)
+{
+	GEM_BUG_ON(vma->node.allocated);
+	GEM_BUG_ON(i915_vma_is_active(vma));
+	GEM_BUG_ON(!i915_vma_is_closed(vma));
+	GEM_BUG_ON(vma->fence);
+
+	list_del(&vma->vm_link);
+	if (!i915_vma_is_ggtt(vma))
+		i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
+
+	kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
+}
+
+void i915_vma_close(struct i915_vma *vma)
+{
+	GEM_BUG_ON(i915_vma_is_closed(vma));
+	vma->flags |= I915_VMA_CLOSED;
+
+	list_del_init(&vma->obj_link);
+	if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma))
+		WARN_ON(i915_vma_unbind(vma));
+}
+
 static struct i915_vma *
-__i915_gem_vma_create(struct drm_i915_gem_object *obj,
-		      struct i915_address_space *vm,
-		      const struct i915_ggtt_view *ggtt_view)
+__i915_vma_create(struct drm_i915_gem_object *obj,
+		  struct i915_address_space *vm,
+		  const struct i915_ggtt_view *view)
 {
 	struct i915_vma *vma;
+	int i;
 
-	if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
-		return ERR_PTR(-EINVAL);
+	GEM_BUG_ON(vm->closed);
 
 	vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL);
 	if (vma == NULL)
 		return ERR_PTR(-ENOMEM);
 
-	INIT_LIST_HEAD(&vma->vm_link);
-	INIT_LIST_HEAD(&vma->obj_link);
 	INIT_LIST_HEAD(&vma->exec_list);
+	for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
+		init_request_active(&vma->last_read[i], i915_vma_retire);
+	init_request_active(&vma->last_fence, NULL);
+	list_add(&vma->vm_link, &vm->unbound_list);
 	vma->vm = vm;
 	vma->obj = obj;
-	vma->is_ggtt = i915_is_ggtt(vm);
+	vma->size = obj->base.size;
+
+	if (view) {
+		vma->ggtt_view = *view;
+		if (view->type == I915_GGTT_VIEW_PARTIAL) {
+			vma->size = view->params.partial.size;
+			vma->size <<= PAGE_SHIFT;
+		} else if (view->type == I915_GGTT_VIEW_ROTATED) {
+			vma->size =
+				intel_rotation_info_size(&view->params.rotated);
+			vma->size <<= PAGE_SHIFT;
+		}
+	}
 
-	if (i915_is_ggtt(vm))
-		vma->ggtt_view = *ggtt_view;
-	else
+	if (i915_is_ggtt(vm)) {
+		vma->flags |= I915_VMA_GGTT;
+	} else {
 		i915_ppgtt_get(i915_vm_to_ppgtt(vm));
+	}
 
 	list_add_tail(&vma->obj_link, &obj->vma_list);
-
 	return vma;
 }
 
+static inline bool vma_matches(struct i915_vma *vma,
+			       struct i915_address_space *vm,
+			       const struct i915_ggtt_view *view)
+{
+	if (vma->vm != vm)
+		return false;
+
+	if (!i915_vma_is_ggtt(vma))
+		return true;
+
+	if (!view)
+		return vma->ggtt_view.type == 0;
+
+	if (vma->ggtt_view.type != view->type)
+		return false;
+
+	return memcmp(&vma->ggtt_view.params,
+		      &view->params,
+		      sizeof(view->params)) == 0;
+}
+
 struct i915_vma *
-i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
-				  struct i915_address_space *vm)
+i915_vma_create(struct drm_i915_gem_object *obj,
+		struct i915_address_space *vm,
+		const struct i915_ggtt_view *view)
+{
+	GEM_BUG_ON(view && !i915_is_ggtt(vm));
+	GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view));
+
+	return __i915_vma_create(obj, vm, view);
+}
+
+struct i915_vma *
+i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
+		    struct i915_address_space *vm,
+		    const struct i915_ggtt_view *view)
 {
 	struct i915_vma *vma;
 
-	vma = i915_gem_obj_to_vma(obj, vm);
-	if (!vma)
-		vma = __i915_gem_vma_create(obj, vm,
-					    i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL);
+	list_for_each_entry_reverse(vma, &obj->vma_list, obj_link)
+		if (vma_matches(vma, vm, view))
+			return vma;
 
-	return vma;
+	return NULL;
 }
 
 struct i915_vma *
-i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj,
-				       const struct i915_ggtt_view *view)
+i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
+				  struct i915_address_space *vm,
+				  const struct i915_ggtt_view *view)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
+	struct i915_vma *vma;
 
+	GEM_BUG_ON(view && !i915_is_ggtt(vm));
+
+	vma = i915_gem_obj_to_vma(obj, vm, view);
 	if (!vma)
-		vma = __i915_gem_vma_create(obj, &ggtt->base, view);
+		vma = __i915_vma_create(obj, vm, view);
 
+	GEM_BUG_ON(i915_vma_is_closed(vma));
 	return vma;
-
 }
 
 static struct scatterlist *
@@ -3434,18 +3476,16 @@ rotate_pages(const dma_addr_t *in, unsigned int offset,
 }
 
 static struct sg_table *
-intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info,
+intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,
 			  struct drm_i915_gem_object *obj)
 {
 	const size_t n_pages = obj->base.size / PAGE_SIZE;
-	unsigned int size_pages = rot_info->plane[0].width * rot_info->plane[0].height;
-	unsigned int size_pages_uv;
+	unsigned int size = intel_rotation_info_size(rot_info);
 	struct sgt_iter sgt_iter;
 	dma_addr_t dma_addr;
 	unsigned long i;
 	dma_addr_t *page_addr_list;
 	struct sg_table *st;
-	unsigned int uv_start_page;
 	struct scatterlist *sg;
 	int ret = -ENOMEM;
 
@@ -3456,18 +3496,12 @@ intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info,
 	if (!page_addr_list)
 		return ERR_PTR(ret);
 
-	/* Account for UV plane with NV12. */
-	if (rot_info->pixel_format == DRM_FORMAT_NV12)
-		size_pages_uv = rot_info->plane[1].width * rot_info->plane[1].height;
-	else
-		size_pages_uv = 0;
-
 	/* Allocate target SG list. */
 	st = kmalloc(sizeof(*st), GFP_KERNEL);
 	if (!st)
 		goto err_st_alloc;
 
-	ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL);
+	ret = sg_alloc_table(st, size, GFP_KERNEL);
 	if (ret)
 		goto err_sg_alloc;
 
@@ -3480,32 +3514,14 @@ intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info,
 	st->nents = 0;
 	sg = st->sgl;
 
-	/* Rotate the pages. */
-	sg = rotate_pages(page_addr_list, 0,
-			  rot_info->plane[0].width, rot_info->plane[0].height,
-			  rot_info->plane[0].width,
-			  st, sg);
-
-	/* Append the UV plane if NV12. */
-	if (rot_info->pixel_format == DRM_FORMAT_NV12) {
-		uv_start_page = size_pages;
-
-		/* Check for tile-row un-alignment. */
-		if (offset_in_page(rot_info->uv_offset))
-			uv_start_page--;
-
-		rot_info->uv_start_page = uv_start_page;
-
-		sg = rotate_pages(page_addr_list, rot_info->uv_start_page,
-				  rot_info->plane[1].width, rot_info->plane[1].height,
-				  rot_info->plane[1].width,
-				  st, sg);
+	for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
+		sg = rotate_pages(page_addr_list, rot_info->plane[i].offset,
+				  rot_info->plane[i].width, rot_info->plane[i].height,
+				  rot_info->plane[i].stride, st, sg);
 	}
 
-	DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages (%u plane 0)).\n",
-		      obj->base.size, rot_info->plane[0].width,
-		      rot_info->plane[0].height, size_pages + size_pages_uv,
-		      size_pages);
+	DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n",
+		      obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
 
 	drm_free_large(page_addr_list);
 
@@ -3516,10 +3532,9 @@ err_sg_alloc:
 err_st_alloc:
 	drm_free_large(page_addr_list);
 
-	DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%d) (%ux%u tiles, %u pages (%u plane 0))\n",
-		      obj->base.size, ret, rot_info->plane[0].width,
-		      rot_info->plane[0].height, size_pages + size_pages_uv,
-		      size_pages);
+	DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
+		      obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
+
 	return ERR_PTR(ret);
 }
 
@@ -3569,28 +3584,27 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
 {
 	int ret = 0;
 
-	if (vma->ggtt_view.pages)
+	if (vma->pages)
 		return 0;
 
 	if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
-		vma->ggtt_view.pages = vma->obj->pages;
+		vma->pages = vma->obj->pages;
 	else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
-		vma->ggtt_view.pages =
+		vma->pages =
 			intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj);
 	else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL)
-		vma->ggtt_view.pages =
-			intel_partial_pages(&vma->ggtt_view, vma->obj);
+		vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
 	else
 		WARN_ONCE(1, "GGTT view %u not implemented!\n",
 			  vma->ggtt_view.type);
 
-	if (!vma->ggtt_view.pages) {
+	if (!vma->pages) {
 		DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
 			  vma->ggtt_view.type);
 		ret = -EINVAL;
-	} else if (IS_ERR(vma->ggtt_view.pages)) {
-		ret = PTR_ERR(vma->ggtt_view.pages);
-		vma->ggtt_view.pages = NULL;
+	} else if (IS_ERR(vma->pages)) {
+		ret = PTR_ERR(vma->pages);
+		vma->pages = NULL;
 		DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
 			  vma->ggtt_view.type, ret);
 	}
@@ -3611,34 +3625,32 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
 		  u32 flags)
 {
-	int ret;
 	u32 bind_flags;
+	u32 vma_flags;
+	int ret;
 
 	if (WARN_ON(flags == 0))
 		return -EINVAL;
 
 	bind_flags = 0;
 	if (flags & PIN_GLOBAL)
-		bind_flags |= GLOBAL_BIND;
+		bind_flags |= I915_VMA_GLOBAL_BIND;
 	if (flags & PIN_USER)
-		bind_flags |= LOCAL_BIND;
+		bind_flags |= I915_VMA_LOCAL_BIND;
 
+	vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
 	if (flags & PIN_UPDATE)
-		bind_flags |= vma->bound;
+		bind_flags |= vma_flags;
 	else
-		bind_flags &= ~vma->bound;
-
+		bind_flags &= ~vma_flags;
 	if (bind_flags == 0)
 		return 0;
 
-	if (vma->bound == 0 && vma->vm->allocate_va_range) {
-		/* XXX: i915_vma_pin() will fix this +- hack */
-		vma->pin_count++;
+	if (vma_flags == 0 && vma->vm->allocate_va_range) {
 		trace_i915_va_alloc(vma);
 		ret = vma->vm->allocate_va_range(vma->vm,
 						 vma->node.start,
 						 vma->node.size);
-		vma->pin_count--;
 		if (ret)
 			return ret;
 	}
@@ -3647,56 +3659,47 @@ int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
 	if (ret)
 		return ret;
 
-	vma->bound |= bind_flags;
-
+	vma->flags |= bind_flags;
 	return 0;
 }
 
-/**
- * i915_ggtt_view_size - Get the size of a GGTT view.
- * @obj: Object the view is of.
- * @view: The view in question.
- *
- * @return The size of the GGTT view in bytes.
- */
-size_t
-i915_ggtt_view_size(struct drm_i915_gem_object *obj,
-		    const struct i915_ggtt_view *view)
-{
-	if (view->type == I915_GGTT_VIEW_NORMAL) {
-		return obj->base.size;
-	} else if (view->type == I915_GGTT_VIEW_ROTATED) {
-		return intel_rotation_info_size(&view->params.rotated) << PAGE_SHIFT;
-	} else if (view->type == I915_GGTT_VIEW_PARTIAL) {
-		return view->params.partial.size << PAGE_SHIFT;
-	} else {
-		WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type);
-		return obj->base.size;
-	}
-}
-
 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
 {
 	void __iomem *ptr;
 
+	/* Access through the GTT requires the device to be awake. */
+	assert_rpm_wakelock_held(to_i915(vma->vm->dev));
+
 	lockdep_assert_held(&vma->vm->dev->struct_mutex);
-	if (WARN_ON(!vma->obj->map_and_fenceable))
-		return ERR_PTR(-ENODEV);
+	if (WARN_ON(!i915_vma_is_map_and_fenceable(vma)))
+		return IO_ERR_PTR(-ENODEV);
 
-	GEM_BUG_ON(!vma->is_ggtt);
-	GEM_BUG_ON((vma->bound & GLOBAL_BIND) == 0);
+	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
+	GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0);
 
 	ptr = vma->iomap;
 	if (ptr == NULL) {
-		ptr = io_mapping_map_wc(i915_vm_to_ggtt(vma->vm)->mappable,
+		ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable,
 					vma->node.start,
 					vma->node.size);
 		if (ptr == NULL)
-			return ERR_PTR(-ENOMEM);
+			return IO_ERR_PTR(-ENOMEM);
 
 		vma->iomap = ptr;
 	}
 
-	vma->pin_count++;
+	__i915_vma_pin(vma);
 	return ptr;
 }
+
+void i915_vma_unpin_and_release(struct i915_vma **p_vma)
+{
+	struct i915_vma *vma;
+
+	vma = fetch_and_zero(p_vma);
+	if (!vma)
+		return;
+
+	i915_vma_unpin(vma);
+	i915_vma_put(vma);
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index aa5f31d1c2ed..ec78be2f8c77 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -36,7 +36,15 @@
 
 #include <linux/io-mapping.h>
 
+#include "i915_gem_request.h"
+
+#define I915_FENCE_REG_NONE -1
+#define I915_MAX_NUM_FENCES 32
+/* 32 fences + sign bit for FENCE_REG_NONE */
+#define I915_MAX_NUM_FENCE_BITS 6
+
 struct drm_i915_file_private;
+struct drm_i915_fence_reg;
 
 typedef uint32_t gen6_pte_t;
 typedef uint64_t gen8_pte_t;
@@ -137,12 +145,9 @@ enum i915_ggtt_view_type {
 };
 
 struct intel_rotation_info {
-	unsigned int uv_offset;
-	uint32_t pixel_format;
-	unsigned int uv_start_page;
 	struct {
 		/* tiles */
-		unsigned int width, height;
+		unsigned int width, height, stride, offset;
 	} plane[2];
 };
 
@@ -156,8 +161,6 @@ struct i915_ggtt_view {
 		} partial;
 		struct intel_rotation_info rotated;
 	} params;
-
-	struct sg_table *pages;
 };
 
 extern const struct i915_ggtt_view i915_ggtt_view_normal;
@@ -177,13 +180,38 @@ struct i915_vma {
 	struct drm_mm_node node;
 	struct drm_i915_gem_object *obj;
 	struct i915_address_space *vm;
+	struct drm_i915_fence_reg *fence;
+	struct sg_table *pages;
 	void __iomem *iomap;
+	u64 size;
+	u64 display_alignment;
+
+	unsigned int flags;
+	/**
+	 * How many users have pinned this object in GTT space. The following
+	 * users can each hold at most one reference: pwrite/pread, execbuffer
+	 * (objects are not allowed multiple times for the same batchbuffer),
+	 * and the framebuffer code. When switching/pageflipping, the
+	 * framebuffer code has at most two buffers pinned per crtc.
+	 *
+	 * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
+	 * bits with absolutely no headroom. So use 4 bits.
+	 */
+#define I915_VMA_PIN_MASK 0xf
+#define I915_VMA_PIN_OVERFLOW	BIT(5)
 
 	/** Flags and address space this VMA is bound to */
-#define GLOBAL_BIND	(1<<0)
-#define LOCAL_BIND	(1<<1)
-	unsigned int bound : 4;
-	bool is_ggtt : 1;
+#define I915_VMA_GLOBAL_BIND	BIT(6)
+#define I915_VMA_LOCAL_BIND	BIT(7)
+#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
+
+#define I915_VMA_GGTT		BIT(8)
+#define I915_VMA_CAN_FENCE	BIT(9)
+#define I915_VMA_CLOSED		BIT(10)
+
+	unsigned int active;
+	struct i915_gem_active last_read[I915_NUM_ENGINES];
+	struct i915_gem_active last_fence;
 
 	/**
 	 * Support different GGTT views into the same object.
@@ -208,20 +236,66 @@ struct i915_vma {
 	struct hlist_node exec_node;
 	unsigned long exec_handle;
 	struct drm_i915_gem_exec_object2 *exec_entry;
-
-	/**
-	 * How many users have pinned this object in GTT space. The following
-	 * users can each hold at most one reference: pwrite/pread, execbuffer
-	 * (objects are not allowed multiple times for the same batchbuffer),
-	 * and the framebuffer code. When switching/pageflipping, the
-	 * framebuffer code has at most two buffers pinned per crtc.
-	 *
-	 * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
-	 * bits with absolutely no headroom. So use 4 bits. */
-	unsigned int pin_count:4;
-#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
 };
 
+struct i915_vma *
+i915_vma_create(struct drm_i915_gem_object *obj,
+		struct i915_address_space *vm,
+		const struct i915_ggtt_view *view);
+void i915_vma_unpin_and_release(struct i915_vma **p_vma);
+
+static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
+{
+	return vma->flags & I915_VMA_GGTT;
+}
+
+static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
+{
+	return vma->flags & I915_VMA_CAN_FENCE;
+}
+
+static inline bool i915_vma_is_closed(const struct i915_vma *vma)
+{
+	return vma->flags & I915_VMA_CLOSED;
+}
+
+static inline unsigned int i915_vma_get_active(const struct i915_vma *vma)
+{
+	return vma->active;
+}
+
+static inline bool i915_vma_is_active(const struct i915_vma *vma)
+{
+	return i915_vma_get_active(vma);
+}
+
+static inline void i915_vma_set_active(struct i915_vma *vma,
+				       unsigned int engine)
+{
+	vma->active |= BIT(engine);
+}
+
+static inline void i915_vma_clear_active(struct i915_vma *vma,
+					 unsigned int engine)
+{
+	vma->active &= ~BIT(engine);
+}
+
+static inline bool i915_vma_has_active_engine(const struct i915_vma *vma,
+					      unsigned int engine)
+{
+	return vma->active & BIT(engine);
+}
+
+static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
+{
+	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
+	GEM_BUG_ON(!vma->node.allocated);
+	GEM_BUG_ON(upper_32_bits(vma->node.start));
+	GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1));
+	return lower_32_bits(vma->node.start);
+}
+
 struct i915_page_dma {
 	struct page *page;
 	union {
@@ -238,10 +312,6 @@ struct i915_page_dma {
 #define px_page(px) (px_base(px)->page)
 #define px_dma(px) (px_base(px)->daddr)
 
-struct i915_page_scratch {
-	struct i915_page_dma base;
-};
-
 struct i915_page_table {
 	struct i915_page_dma base;
 
@@ -272,13 +342,22 @@ struct i915_pml4 {
 struct i915_address_space {
 	struct drm_mm mm;
 	struct drm_device *dev;
+	/* Every address space belongs to a struct file - except for the global
+	 * GTT that is owned by the driver (and so @file is set to NULL). In
+	 * principle, no information should leak from one context to another
+	 * (or between files/processes etc) unless explicitly shared by the
+	 * owner. Tracking the owner is important in order to free up per-file
+	 * objects along with the file, to aide resource tracking, and to
+	 * assign blame.
+	 */
+	struct drm_i915_file_private *file;
 	struct list_head global_link;
 	u64 start;		/* Start offset always 0 for dri2 */
 	u64 total;		/* size addr space maps (ex. 2GB for ggtt) */
 
-	bool is_ggtt;
+	bool closed;
 
-	struct i915_page_scratch *scratch_page;
+	struct i915_page_dma scratch_page;
 	struct i915_page_table *scratch_pt;
 	struct i915_page_directory *scratch_pd;
 	struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */
@@ -306,6 +385,13 @@ struct i915_address_space {
 	 */
 	struct list_head inactive_list;
 
+	/**
+	 * List of vma that have been unbound.
+	 *
+	 * A reference is not held on the buffer while on this list.
+	 */
+	struct list_head unbound_list;
+
 	/* FIXME: Need a more generic return type */
 	gen6_pte_t (*pte_encode)(dma_addr_t addr,
 				 enum i915_cache_level level,
@@ -338,7 +424,7 @@ struct i915_address_space {
 			u32 flags);
 };
 
-#define i915_is_ggtt(V) ((V)->is_ggtt)
+#define i915_is_ggtt(V) (!(V)->file)
 
 /* The Graphics Translation Table is the way in which GEN hardware translates a
  * Graphics Virtual Address into a Physical Address. In addition to the normal
@@ -349,14 +435,13 @@ struct i915_address_space {
  */
 struct i915_ggtt {
 	struct i915_address_space base;
+	struct io_mapping mappable;	/* Mapping to our CPU mappable region */
 
 	size_t stolen_size;		/* Total size of stolen memory */
 	size_t stolen_usable_size;	/* Total size minus BIOS reserved */
 	size_t stolen_reserved_base;
 	size_t stolen_reserved_size;
-	size_t size;			/* Total size of Global GTT */
 	u64 mappable_end;		/* End offset that we can CPU map */
-	struct io_mapping *mappable;	/* Mapping to our CPU mappable region */
 	phys_addr_t mappable_base;	/* PA of our GMADR */
 
 	/** "Graphics Stolen Memory" holds the global PTEs */
@@ -365,8 +450,6 @@ struct i915_ggtt {
 	bool do_idle_maps;
 
 	int mtrr;
-
-	int (*probe)(struct i915_ggtt *ggtt);
 };
 
 struct i915_hw_ppgtt {
@@ -380,8 +463,6 @@ struct i915_hw_ppgtt {
 		struct i915_page_directory pd;		/* GEN6-7 */
 	};
 
-	struct drm_i915_file_private *file_priv;
-
 	gen6_pte_t __iomem *pd_addr;
 
 	int (*enable)(struct i915_hw_ppgtt *ppgtt);
@@ -521,14 +602,15 @@ i915_page_dir_dma_addr(const struct i915_hw_ppgtt *ppgtt, const unsigned n)
 		px_dma(ppgtt->base.scratch_pd);
 }
 
-int i915_ggtt_init_hw(struct drm_device *dev);
-int i915_ggtt_enable_hw(struct drm_device *dev);
-void i915_gem_init_ggtt(struct drm_device *dev);
-void i915_ggtt_cleanup_hw(struct drm_device *dev);
+int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv);
+int i915_ggtt_init_hw(struct drm_i915_private *dev_priv);
+int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv);
+int i915_gem_init_ggtt(struct drm_i915_private *dev_priv);
+void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv);
 
 int i915_ppgtt_init_hw(struct drm_device *dev);
 void i915_ppgtt_release(struct kref *kref);
-struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_device *dev,
+struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv,
 					struct drm_i915_file_private *fpriv);
 static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
 {
@@ -548,23 +630,67 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev);
 int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj);
 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj);
 
-static inline bool
-i915_ggtt_view_equal(const struct i915_ggtt_view *a,
-                     const struct i915_ggtt_view *b)
+/* Flags used by pin/bind&friends. */
+#define PIN_NONBLOCK		BIT(0)
+#define PIN_MAPPABLE		BIT(1)
+#define PIN_ZONE_4G		BIT(2)
+#define PIN_NONFAULT		BIT(3)
+
+#define PIN_MBZ			BIT(5) /* I915_VMA_PIN_OVERFLOW */
+#define PIN_GLOBAL		BIT(6) /* I915_VMA_GLOBAL_BIND */
+#define PIN_USER		BIT(7) /* I915_VMA_LOCAL_BIND */
+#define PIN_UPDATE		BIT(8)
+
+#define PIN_HIGH		BIT(9)
+#define PIN_OFFSET_BIAS		BIT(10)
+#define PIN_OFFSET_FIXED	BIT(11)
+#define PIN_OFFSET_MASK		(~4095)
+
+int __i915_vma_do_pin(struct i915_vma *vma,
+		      u64 size, u64 alignment, u64 flags);
+static inline int __must_check
+i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 {
-	if (WARN_ON(!a || !b))
-		return false;
-
-	if (a->type != b->type)
-		return false;
-	if (a->type != I915_GGTT_VIEW_NORMAL)
-		return !memcmp(&a->params, &b->params, sizeof(a->params));
-	return true;
+	BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
+	BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
+	BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
+
+	/* Pin early to prevent the shrinker/eviction logic from destroying
+	 * our vma as we insert and bind.
+	 */
+	if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0))
+		return 0;
+
+	return __i915_vma_do_pin(vma, size, alignment, flags);
 }
 
-size_t
-i915_ggtt_view_size(struct drm_i915_gem_object *obj,
-		    const struct i915_ggtt_view *view);
+static inline int i915_vma_pin_count(const struct i915_vma *vma)
+{
+	return vma->flags & I915_VMA_PIN_MASK;
+}
+
+static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
+{
+	return i915_vma_pin_count(vma);
+}
+
+static inline void __i915_vma_pin(struct i915_vma *vma)
+{
+	vma->flags++;
+	GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+}
+
+static inline void __i915_vma_unpin(struct i915_vma *vma)
+{
+	GEM_BUG_ON(!i915_vma_is_pinned(vma));
+	vma->flags--;
+}
+
+static inline void i915_vma_unpin(struct i915_vma *vma)
+{
+	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+	__i915_vma_unpin(vma);
+}
 
 /**
  * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
@@ -580,6 +706,7 @@ i915_ggtt_view_size(struct drm_i915_gem_object *obj,
  * Returns a valid iomapped pointer or ERR_PTR.
  */
 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
+#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
 
 /**
  * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
@@ -593,9 +720,14 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
 static inline void i915_vma_unpin_iomap(struct i915_vma *vma)
 {
 	lockdep_assert_held(&vma->vm->dev->struct_mutex);
-	GEM_BUG_ON(vma->pin_count == 0);
 	GEM_BUG_ON(vma->iomap == NULL);
-	vma->pin_count--;
+	i915_vma_unpin(vma);
+}
+
+static inline struct page *i915_vma_first_page(struct i915_vma *vma)
+{
+	GEM_BUG_ON(!vma->pages);
+	return sg_page(vma->pages->sgl);
 }
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index f75bbd67a13a..95b7e9afd5f8 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -28,10 +28,17 @@
 #include "i915_drv.h"
 #include "intel_renderstate.h"
 
+struct render_state {
+	const struct intel_renderstate_rodata *rodata;
+	struct i915_vma *vma;
+	u32 aux_batch_size;
+	u32 aux_batch_offset;
+};
+
 static const struct intel_renderstate_rodata *
-render_state_get_rodata(const int gen)
+render_state_get_rodata(const struct drm_i915_gem_request *req)
 {
-	switch (gen) {
+	switch (INTEL_GEN(req->i915)) {
 	case 6:
 		return &gen6_null_state;
 	case 7:
@@ -45,35 +52,6 @@ render_state_get_rodata(const int gen)
 	return NULL;
 }
 
-static int render_state_init(struct render_state *so,
-			     struct drm_i915_private *dev_priv)
-{
-	int ret;
-
-	so->gen = INTEL_GEN(dev_priv);
-	so->rodata = render_state_get_rodata(so->gen);
-	if (so->rodata == NULL)
-		return 0;
-
-	if (so->rodata->batch_items * 4 > 4096)
-		return -EINVAL;
-
-	so->obj = i915_gem_object_create(&dev_priv->drm, 4096);
-	if (IS_ERR(so->obj))
-		return PTR_ERR(so->obj);
-
-	ret = i915_gem_obj_ggtt_pin(so->obj, 4096, 0);
-	if (ret)
-		goto free_gem;
-
-	so->ggtt_offset = i915_gem_obj_ggtt_offset(so->obj);
-	return 0;
-
-free_gem:
-	drm_gem_object_unreference(&so->obj->base);
-	return ret;
-}
-
 /*
  * Macro to add commands to auxiliary batch.
  * This macro only checks for page overflow before inserting the commands,
@@ -94,27 +72,28 @@ free_gem:
 
 static int render_state_setup(struct render_state *so)
 {
-	struct drm_device *dev = so->obj->base.dev;
+	struct drm_device *dev = so->vma->vm->dev;
 	const struct intel_renderstate_rodata *rodata = so->rodata;
+	const bool has_64bit_reloc = INTEL_GEN(dev) >= 8;
 	unsigned int i = 0, reloc_index = 0;
 	struct page *page;
 	u32 *d;
 	int ret;
 
-	ret = i915_gem_object_set_to_cpu_domain(so->obj, true);
+	ret = i915_gem_object_set_to_cpu_domain(so->vma->obj, true);
 	if (ret)
 		return ret;
 
-	page = i915_gem_object_get_dirty_page(so->obj, 0);
+	page = i915_gem_object_get_dirty_page(so->vma->obj, 0);
 	d = kmap(page);
 
 	while (i < rodata->batch_items) {
 		u32 s = rodata->batch[i];
 
 		if (i * 4  == rodata->reloc[reloc_index]) {
-			u64 r = s + so->ggtt_offset;
+			u64 r = s + so->vma->node.start;
 			s = lower_32_bits(r);
-			if (so->gen >= 8) {
+			if (has_64bit_reloc) {
 				if (i + 1 >= rodata->batch_items ||
 				    rodata->batch[i + 1] != 0) {
 					ret = -EINVAL;
@@ -174,7 +153,7 @@ static int render_state_setup(struct render_state *so)
 
 	kunmap(page);
 
-	ret = i915_gem_object_set_to_gtt_domain(so->obj, false);
+	ret = i915_gem_object_set_to_gtt_domain(so->vma->obj, false);
 	if (ret)
 		return ret;
 
@@ -192,67 +171,60 @@ err_out:
 
 #undef OUT_BATCH
 
-void i915_gem_render_state_fini(struct render_state *so)
-{
-	i915_gem_object_ggtt_unpin(so->obj);
-	drm_gem_object_unreference(&so->obj->base);
-}
-
-int i915_gem_render_state_prepare(struct intel_engine_cs *engine,
-				  struct render_state *so)
+int i915_gem_render_state_init(struct drm_i915_gem_request *req)
 {
+	struct render_state so;
+	struct drm_i915_gem_object *obj;
 	int ret;
 
-	if (WARN_ON(engine->id != RCS))
+	if (WARN_ON(req->engine->id != RCS))
 		return -ENOENT;
 
-	ret = render_state_init(so, engine->i915);
-	if (ret)
-		return ret;
-
-	if (so->rodata == NULL)
+	so.rodata = render_state_get_rodata(req);
+	if (!so.rodata)
 		return 0;
 
-	ret = render_state_setup(so);
-	if (ret) {
-		i915_gem_render_state_fini(so);
-		return ret;
-	}
+	if (so.rodata->batch_items * 4 > 4096)
+		return -EINVAL;
 
-	return 0;
-}
+	obj = i915_gem_object_create(&req->i915->drm, 4096);
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
 
-int i915_gem_render_state_init(struct drm_i915_gem_request *req)
-{
-	struct render_state so;
-	int ret;
+	so.vma = i915_vma_create(obj, &req->i915->ggtt.base, NULL);
+	if (IS_ERR(so.vma)) {
+		ret = PTR_ERR(so.vma);
+		goto err_obj;
+	}
 
-	ret = i915_gem_render_state_prepare(req->engine, &so);
+	ret = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL);
 	if (ret)
-		return ret;
+		goto err_obj;
 
-	if (so.rodata == NULL)
-		return 0;
+	ret = render_state_setup(&so);
+	if (ret)
+		goto err_unpin;
 
-	ret = req->engine->dispatch_execbuffer(req, so.ggtt_offset,
-					     so.rodata->batch_items * 4,
-					     I915_DISPATCH_SECURE);
+	ret = req->engine->emit_bb_start(req, so.vma->node.start,
+					 so.rodata->batch_items * 4,
+					 I915_DISPATCH_SECURE);
 	if (ret)
-		goto out;
+		goto err_unpin;
 
 	if (so.aux_batch_size > 8) {
-		ret = req->engine->dispatch_execbuffer(req,
-						     (so.ggtt_offset +
-						      so.aux_batch_offset),
-						     so.aux_batch_size,
-						     I915_DISPATCH_SECURE);
+		ret = req->engine->emit_bb_start(req,
+						 (so.vma->node.start +
+						  so.aux_batch_offset),
+						 so.aux_batch_size,
+						 I915_DISPATCH_SECURE);
 		if (ret)
-			goto out;
+			goto err_unpin;
 	}
 
-	i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), req);
-
-out:
-	i915_gem_render_state_fini(&so);
+	i915_vma_move_to_active(so.vma, req, 0);
+err_unpin:
+	i915_vma_unpin(so.vma);
+err_obj:
+	i915_gem_object_put(obj);
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h
index 6aaa3a10a630..18cce3f06e9c 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.h
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.h
@@ -24,26 +24,8 @@
 #ifndef _I915_GEM_RENDER_STATE_H_
 #define _I915_GEM_RENDER_STATE_H_
 
-#include <linux/types.h>
-
-struct intel_renderstate_rodata {
-	const u32 *reloc;
-	const u32 *batch;
-	const u32 batch_items;
-};
-
-struct render_state {
-	const struct intel_renderstate_rodata *rodata;
-	struct drm_i915_gem_object *obj;
-	u64 ggtt_offset;
-	int gen;
-	u32 aux_batch_size;
-	u32 aux_batch_offset;
-};
+struct drm_i915_gem_request;
 
 int i915_gem_render_state_init(struct drm_i915_gem_request *req);
-void i915_gem_render_state_fini(struct render_state *so);
-int i915_gem_render_state_prepare(struct intel_engine_cs *engine,
-				  struct render_state *so);
 
 #endif /* _I915_GEM_RENDER_STATE_H_ */
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
new file mode 100644
index 000000000000..8832f8ec1583
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -0,0 +1,947 @@
+/*
+ * Copyright © 2008-2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/prefetch.h>
+
+#include "i915_drv.h"
+
+static const char *i915_fence_get_driver_name(struct fence *fence)
+{
+	return "i915";
+}
+
+static const char *i915_fence_get_timeline_name(struct fence *fence)
+{
+	/* Timelines are bound by eviction to a VM. However, since
+	 * we only have a global seqno at the moment, we only have
+	 * a single timeline. Note that each timeline will have
+	 * multiple execution contexts (fence contexts) as we allow
+	 * engines within a single timeline to execute in parallel.
+	 */
+	return "global";
+}
+
+static bool i915_fence_signaled(struct fence *fence)
+{
+	return i915_gem_request_completed(to_request(fence));
+}
+
+static bool i915_fence_enable_signaling(struct fence *fence)
+{
+	if (i915_fence_signaled(fence))
+		return false;
+
+	intel_engine_enable_signaling(to_request(fence));
+	return true;
+}
+
+static signed long i915_fence_wait(struct fence *fence,
+				   bool interruptible,
+				   signed long timeout_jiffies)
+{
+	s64 timeout_ns, *timeout;
+	int ret;
+
+	if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT) {
+		timeout_ns = jiffies_to_nsecs(timeout_jiffies);
+		timeout = &timeout_ns;
+	} else {
+		timeout = NULL;
+	}
+
+	ret = i915_wait_request(to_request(fence),
+				interruptible, timeout,
+				NO_WAITBOOST);
+	if (ret == -ETIME)
+		return 0;
+
+	if (ret < 0)
+		return ret;
+
+	if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT)
+		timeout_jiffies = nsecs_to_jiffies(timeout_ns);
+
+	return timeout_jiffies;
+}
+
+static void i915_fence_value_str(struct fence *fence, char *str, int size)
+{
+	snprintf(str, size, "%u", fence->seqno);
+}
+
+static void i915_fence_timeline_value_str(struct fence *fence, char *str,
+					  int size)
+{
+	snprintf(str, size, "%u",
+		 intel_engine_get_seqno(to_request(fence)->engine));
+}
+
+static void i915_fence_release(struct fence *fence)
+{
+	struct drm_i915_gem_request *req = to_request(fence);
+
+	kmem_cache_free(req->i915->requests, req);
+}
+
+const struct fence_ops i915_fence_ops = {
+	.get_driver_name = i915_fence_get_driver_name,
+	.get_timeline_name = i915_fence_get_timeline_name,
+	.enable_signaling = i915_fence_enable_signaling,
+	.signaled = i915_fence_signaled,
+	.wait = i915_fence_wait,
+	.release = i915_fence_release,
+	.fence_value_str = i915_fence_value_str,
+	.timeline_value_str = i915_fence_timeline_value_str,
+};
+
+int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
+				   struct drm_file *file)
+{
+	struct drm_i915_private *dev_private;
+	struct drm_i915_file_private *file_priv;
+
+	WARN_ON(!req || !file || req->file_priv);
+
+	if (!req || !file)
+		return -EINVAL;
+
+	if (req->file_priv)
+		return -EINVAL;
+
+	dev_private = req->i915;
+	file_priv = file->driver_priv;
+
+	spin_lock(&file_priv->mm.lock);
+	req->file_priv = file_priv;
+	list_add_tail(&req->client_list, &file_priv->mm.request_list);
+	spin_unlock(&file_priv->mm.lock);
+
+	return 0;
+}
+
+static inline void
+i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
+{
+	struct drm_i915_file_private *file_priv = request->file_priv;
+
+	if (!file_priv)
+		return;
+
+	spin_lock(&file_priv->mm.lock);
+	list_del(&request->client_list);
+	request->file_priv = NULL;
+	spin_unlock(&file_priv->mm.lock);
+}
+
+void i915_gem_retire_noop(struct i915_gem_active *active,
+			  struct drm_i915_gem_request *request)
+{
+	/* Space left intentionally blank */
+}
+
+static void i915_gem_request_retire(struct drm_i915_gem_request *request)
+{
+	struct i915_gem_active *active, *next;
+
+	trace_i915_gem_request_retire(request);
+	list_del(&request->link);
+
+	/* We know the GPU must have read the request to have
+	 * sent us the seqno + interrupt, so use the position
+	 * of tail of the request to update the last known position
+	 * of the GPU head.
+	 *
+	 * Note this requires that we are always called in request
+	 * completion order.
+	 */
+	list_del(&request->ring_link);
+	request->ring->last_retired_head = request->postfix;
+
+	/* Walk through the active list, calling retire on each. This allows
+	 * objects to track their GPU activity and mark themselves as idle
+	 * when their *last* active request is completed (updating state
+	 * tracking lists for eviction, active references for GEM, etc).
+	 *
+	 * As the ->retire() may free the node, we decouple it first and
+	 * pass along the auxiliary information (to avoid dereferencing
+	 * the node after the callback).
+	 */
+	list_for_each_entry_safe(active, next, &request->active_list, link) {
+		/* In microbenchmarks or focusing upon time inside the kernel,
+		 * we may spend an inordinate amount of time simply handling
+		 * the retirement of requests and processing their callbacks.
+		 * Of which, this loop itself is particularly hot due to the
+		 * cache misses when jumping around the list of i915_gem_active.
+		 * So we try to keep this loop as streamlined as possible and
+		 * also prefetch the next i915_gem_active to try and hide
+		 * the likely cache miss.
+		 */
+		prefetchw(next);
+
+		INIT_LIST_HEAD(&active->link);
+		RCU_INIT_POINTER(active->request, NULL);
+
+		active->retire(active, request);
+	}
+
+	i915_gem_request_remove_from_client(request);
+
+	if (request->previous_context) {
+		if (i915.enable_execlists)
+			intel_lr_context_unpin(request->previous_context,
+					       request->engine);
+	}
+
+	i915_gem_context_put(request->ctx);
+	i915_gem_request_put(request);
+}
+
+void i915_gem_request_retire_upto(struct drm_i915_gem_request *req)
+{
+	struct intel_engine_cs *engine = req->engine;
+	struct drm_i915_gem_request *tmp;
+
+	lockdep_assert_held(&req->i915->drm.struct_mutex);
+	GEM_BUG_ON(list_empty(&req->link));
+
+	do {
+		tmp = list_first_entry(&engine->request_list,
+				       typeof(*tmp), link);
+
+		i915_gem_request_retire(tmp);
+	} while (tmp != req);
+}
+
+static int i915_gem_check_wedge(struct drm_i915_private *dev_priv)
+{
+	struct i915_gpu_error *error = &dev_priv->gpu_error;
+
+	if (i915_terminally_wedged(error))
+		return -EIO;
+
+	if (i915_reset_in_progress(error)) {
+		/* Non-interruptible callers can't handle -EAGAIN, hence return
+		 * -EIO unconditionally for these.
+		 */
+		if (!dev_priv->mm.interruptible)
+			return -EIO;
+
+		return -EAGAIN;
+	}
+
+	return 0;
+}
+
+static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
+{
+	struct intel_engine_cs *engine;
+	int ret;
+
+	/* Carefully retire all requests without writing to the rings */
+	for_each_engine(engine, dev_priv) {
+		ret = intel_engine_idle(engine,
+					I915_WAIT_INTERRUPTIBLE |
+					I915_WAIT_LOCKED);
+		if (ret)
+			return ret;
+	}
+	i915_gem_retire_requests(dev_priv);
+
+	/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
+	if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) {
+		while (intel_kick_waiters(dev_priv) ||
+		       intel_kick_signalers(dev_priv))
+			yield();
+	}
+
+	/* Finally reset hw state */
+	for_each_engine(engine, dev_priv)
+		intel_engine_init_seqno(engine, seqno);
+
+	return 0;
+}
+
+int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	int ret;
+
+	if (seqno == 0)
+		return -EINVAL;
+
+	/* HWS page needs to be set less than what we
+	 * will inject to ring
+	 */
+	ret = i915_gem_init_seqno(dev_priv, seqno - 1);
+	if (ret)
+		return ret;
+
+	dev_priv->next_seqno = seqno;
+	return 0;
+}
+
+static int i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
+{
+	/* reserve 0 for non-seqno */
+	if (unlikely(dev_priv->next_seqno == 0)) {
+		int ret;
+
+		ret = i915_gem_init_seqno(dev_priv, 0);
+		if (ret)
+			return ret;
+
+		dev_priv->next_seqno = 1;
+	}
+
+	*seqno = dev_priv->next_seqno++;
+	return 0;
+}
+
+static int __i915_sw_fence_call
+submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
+{
+	struct drm_i915_gem_request *request =
+		container_of(fence, typeof(*request), submit);
+
+	/* Will be called from irq-context when using foreign DMA fences */
+
+	switch (state) {
+	case FENCE_COMPLETE:
+		request->engine->last_submitted_seqno = request->fence.seqno;
+		request->engine->submit_request(request);
+		break;
+
+	case FENCE_FREE:
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+/**
+ * i915_gem_request_alloc - allocate a request structure
+ *
+ * @engine: engine that we wish to issue the request on.
+ * @ctx: context that the request will be associated with.
+ *       This can be NULL if the request is not directly related to
+ *       any specific user context, in which case this function will
+ *       choose an appropriate context to use.
+ *
+ * Returns a pointer to the allocated request if successful,
+ * or an error code if not.
+ */
+struct drm_i915_gem_request *
+i915_gem_request_alloc(struct intel_engine_cs *engine,
+		       struct i915_gem_context *ctx)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	struct drm_i915_gem_request *req;
+	u32 seqno;
+	int ret;
+
+	/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
+	 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
+	 * and restart.
+	 */
+	ret = i915_gem_check_wedge(dev_priv);
+	if (ret)
+		return ERR_PTR(ret);
+
+	/* Move the oldest request to the slab-cache (if not in use!) */
+	req = list_first_entry_or_null(&engine->request_list,
+				       typeof(*req), link);
+	if (req && i915_gem_request_completed(req))
+		i915_gem_request_retire(req);
+
+	/* Beware: Dragons be flying overhead.
+	 *
+	 * We use RCU to look up requests in flight. The lookups may
+	 * race with the request being allocated from the slab freelist.
+	 * That is the request we are writing to here, may be in the process
+	 * of being read by __i915_gem_active_get_rcu(). As such,
+	 * we have to be very careful when overwriting the contents. During
+	 * the RCU lookup, we change chase the request->engine pointer,
+	 * read the request->fence.seqno and increment the reference count.
+	 *
+	 * The reference count is incremented atomically. If it is zero,
+	 * the lookup knows the request is unallocated and complete. Otherwise,
+	 * it is either still in use, or has been reallocated and reset
+	 * with fence_init(). This increment is safe for release as we check
+	 * that the request we have a reference to and matches the active
+	 * request.
+	 *
+	 * Before we increment the refcount, we chase the request->engine
+	 * pointer. We must not call kmem_cache_zalloc() or else we set
+	 * that pointer to NULL and cause a crash during the lookup. If
+	 * we see the request is completed (based on the value of the
+	 * old engine and seqno), the lookup is complete and reports NULL.
+	 * If we decide the request is not completed (new engine or seqno),
+	 * then we grab a reference and double check that it is still the
+	 * active request - which it won't be and restart the lookup.
+	 *
+	 * Do not use kmem_cache_zalloc() here!
+	 */
+	req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL);
+	if (!req)
+		return ERR_PTR(-ENOMEM);
+
+	ret = i915_gem_get_seqno(dev_priv, &seqno);
+	if (ret)
+		goto err;
+
+	spin_lock_init(&req->lock);
+	fence_init(&req->fence,
+		   &i915_fence_ops,
+		   &req->lock,
+		   engine->fence_context,
+		   seqno);
+
+	i915_sw_fence_init(&req->submit, submit_notify);
+
+	INIT_LIST_HEAD(&req->active_list);
+	req->i915 = dev_priv;
+	req->engine = engine;
+	req->ctx = i915_gem_context_get(ctx);
+
+	/* No zalloc, must clear what we need by hand */
+	req->previous_context = NULL;
+	req->file_priv = NULL;
+	req->batch = NULL;
+
+	/*
+	 * Reserve space in the ring buffer for all the commands required to
+	 * eventually emit this request. This is to guarantee that the
+	 * i915_add_request() call can't fail. Note that the reserve may need
+	 * to be redone if the request is not actually submitted straight
+	 * away, e.g. because a GPU scheduler has deferred it.
+	 */
+	req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
+
+	if (i915.enable_execlists)
+		ret = intel_logical_ring_alloc_request_extras(req);
+	else
+		ret = intel_ring_alloc_request_extras(req);
+	if (ret)
+		goto err_ctx;
+
+	/* Record the position of the start of the request so that
+	 * should we detect the updated seqno part-way through the
+	 * GPU processing the request, we never over-estimate the
+	 * position of the head.
+	 */
+	req->head = req->ring->tail;
+
+	return req;
+
+err_ctx:
+	i915_gem_context_put(ctx);
+err:
+	kmem_cache_free(dev_priv->requests, req);
+	return ERR_PTR(ret);
+}
+
+static int
+i915_gem_request_await_request(struct drm_i915_gem_request *to,
+			       struct drm_i915_gem_request *from)
+{
+	int idx, ret;
+
+	GEM_BUG_ON(to == from);
+
+	if (to->engine == from->engine)
+		return 0;
+
+	idx = intel_engine_sync_index(from->engine, to->engine);
+	if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx])
+		return 0;
+
+	trace_i915_gem_ring_sync_to(to, from);
+	if (!i915.semaphores) {
+		if (!i915_spin_request(from, TASK_INTERRUPTIBLE, 2)) {
+			ret = i915_sw_fence_await_dma_fence(&to->submit,
+							    &from->fence, 0,
+							    GFP_KERNEL);
+			if (ret < 0)
+				return ret;
+		}
+	} else {
+		ret = to->engine->semaphore.sync_to(to, from);
+		if (ret)
+			return ret;
+	}
+
+	from->engine->semaphore.sync_seqno[idx] = from->fence.seqno;
+	return 0;
+}
+
+/**
+ * i915_gem_request_await_object - set this request to (async) wait upon a bo
+ *
+ * @to: request we are wishing to use
+ * @obj: object which may be in use on another ring.
+ *
+ * This code is meant to abstract object synchronization with the GPU.
+ * Conceptually we serialise writes between engines inside the GPU.
+ * We only allow one engine to write into a buffer at any time, but
+ * multiple readers. To ensure each has a coherent view of memory, we must:
+ *
+ * - If there is an outstanding write request to the object, the new
+ *   request must wait for it to complete (either CPU or in hw, requests
+ *   on the same ring will be naturally ordered).
+ *
+ * - If we are a write request (pending_write_domain is set), the new
+ *   request must wait for outstanding read requests to complete.
+ *
+ * Returns 0 if successful, else propagates up the lower layer error.
+ */
+int
+i915_gem_request_await_object(struct drm_i915_gem_request *to,
+			      struct drm_i915_gem_object *obj,
+			      bool write)
+{
+	struct i915_gem_active *active;
+	unsigned long active_mask;
+	int idx;
+
+	if (write) {
+		active_mask = i915_gem_object_get_active(obj);
+		active = obj->last_read;
+	} else {
+		active_mask = 1;
+		active = &obj->last_write;
+	}
+
+	for_each_active(active_mask, idx) {
+		struct drm_i915_gem_request *request;
+		int ret;
+
+		request = i915_gem_active_peek(&active[idx],
+					       &obj->base.dev->struct_mutex);
+		if (!request)
+			continue;
+
+		ret = i915_gem_request_await_request(to, request);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+
+	dev_priv->gt.active_engines |= intel_engine_flag(engine);
+	if (dev_priv->gt.awake)
+		return;
+
+	intel_runtime_pm_get_noresume(dev_priv);
+	dev_priv->gt.awake = true;
+
+	intel_enable_gt_powersave(dev_priv);
+	i915_update_gfx_val(dev_priv);
+	if (INTEL_GEN(dev_priv) >= 6)
+		gen6_rps_busy(dev_priv);
+
+	queue_delayed_work(dev_priv->wq,
+			   &dev_priv->gt.retire_work,
+			   round_jiffies_up_relative(HZ));
+}
+
+/*
+ * NB: This function is not allowed to fail. Doing so would mean the the
+ * request is not being tracked for completion but the work itself is
+ * going to happen on the hardware. This would be a Bad Thing(tm).
+ */
+void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
+{
+	struct intel_engine_cs *engine = request->engine;
+	struct intel_ring *ring = request->ring;
+	struct drm_i915_gem_request *prev;
+	u32 request_start;
+	u32 reserved_tail;
+	int ret;
+
+	trace_i915_gem_request_add(request);
+
+	/*
+	 * To ensure that this call will not fail, space for its emissions
+	 * should already have been reserved in the ring buffer. Let the ring
+	 * know that it is time to use that space up.
+	 */
+	request_start = ring->tail;
+	reserved_tail = request->reserved_space;
+	request->reserved_space = 0;
+
+	/*
+	 * Emit any outstanding flushes - execbuf can fail to emit the flush
+	 * after having emitted the batchbuffer command. Hence we need to fix
+	 * things up similar to emitting the lazy request. The difference here
+	 * is that the flush _must_ happen before the next request, no matter
+	 * what.
+	 */
+	if (flush_caches) {
+		ret = engine->emit_flush(request, EMIT_FLUSH);
+
+		/* Not allowed to fail! */
+		WARN(ret, "engine->emit_flush() failed: %d!\n", ret);
+	}
+
+	/* Record the position of the start of the breadcrumb so that
+	 * should we detect the updated seqno part-way through the
+	 * GPU processing the request, we never over-estimate the
+	 * position of the ring's HEAD.
+	 */
+	request->postfix = ring->tail;
+
+	/* Not allowed to fail! */
+	ret = engine->emit_request(request);
+	WARN(ret, "(%s)->emit_request failed: %d!\n", engine->name, ret);
+
+	/* Sanity check that the reserved size was large enough. */
+	ret = ring->tail - request_start;
+	if (ret < 0)
+		ret += ring->size;
+	WARN_ONCE(ret > reserved_tail,
+		  "Not enough space reserved (%d bytes) "
+		  "for adding the request (%d bytes)\n",
+		  reserved_tail, ret);
+
+	/* Seal the request and mark it as pending execution. Note that
+	 * we may inspect this state, without holding any locks, during
+	 * hangcheck. Hence we apply the barrier to ensure that we do not
+	 * see a more recent value in the hws than we are tracking.
+	 */
+
+	prev = i915_gem_active_raw(&engine->last_request,
+				   &request->i915->drm.struct_mutex);
+	if (prev)
+		i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
+					     &request->submitq);
+
+	request->emitted_jiffies = jiffies;
+	request->previous_seqno = engine->last_pending_seqno;
+	engine->last_pending_seqno = request->fence.seqno;
+	i915_gem_active_set(&engine->last_request, request);
+	list_add_tail(&request->link, &engine->request_list);
+	list_add_tail(&request->ring_link, &ring->request_list);
+
+	i915_gem_mark_busy(engine);
+
+	local_bh_disable();
+	i915_sw_fence_commit(&request->submit);
+	local_bh_enable(); /* Kick the execlists tasklet if just scheduled */
+}
+
+static void reset_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&q->lock, flags);
+	if (list_empty(&wait->task_list))
+		__add_wait_queue(q, wait);
+	spin_unlock_irqrestore(&q->lock, flags);
+}
+
+static unsigned long local_clock_us(unsigned int *cpu)
+{
+	unsigned long t;
+
+	/* Cheaply and approximately convert from nanoseconds to microseconds.
+	 * The result and subsequent calculations are also defined in the same
+	 * approximate microseconds units. The principal source of timing
+	 * error here is from the simple truncation.
+	 *
+	 * Note that local_clock() is only defined wrt to the current CPU;
+	 * the comparisons are no longer valid if we switch CPUs. Instead of
+	 * blocking preemption for the entire busywait, we can detect the CPU
+	 * switch and use that as indicator of system load and a reason to
+	 * stop busywaiting, see busywait_stop().
+	 */
+	*cpu = get_cpu();
+	t = local_clock() >> 10;
+	put_cpu();
+
+	return t;
+}
+
+static bool busywait_stop(unsigned long timeout, unsigned int cpu)
+{
+	unsigned int this_cpu;
+
+	if (time_after(local_clock_us(&this_cpu), timeout))
+		return true;
+
+	return this_cpu != cpu;
+}
+
+bool __i915_spin_request(const struct drm_i915_gem_request *req,
+			 int state, unsigned long timeout_us)
+{
+	unsigned int cpu;
+
+	/* When waiting for high frequency requests, e.g. during synchronous
+	 * rendering split between the CPU and GPU, the finite amount of time
+	 * required to set up the irq and wait upon it limits the response
+	 * rate. By busywaiting on the request completion for a short while we
+	 * can service the high frequency waits as quick as possible. However,
+	 * if it is a slow request, we want to sleep as quickly as possible.
+	 * The tradeoff between waiting and sleeping is roughly the time it
+	 * takes to sleep on a request, on the order of a microsecond.
+	 */
+
+	timeout_us += local_clock_us(&cpu);
+	do {
+		if (i915_gem_request_completed(req))
+			return true;
+
+		if (signal_pending_state(state, current))
+			break;
+
+		if (busywait_stop(timeout_us, cpu))
+			break;
+
+		cpu_relax_lowlatency();
+	} while (!need_resched());
+
+	return false;
+}
+
+/**
+ * i915_wait_request - wait until execution of request has finished
+ * @req: duh!
+ * @flags: how to wait
+ * @timeout: in - how long to wait (NULL forever); out - how much time remaining
+ * @rps: client to charge for RPS boosting
+ *
+ * Note: It is of utmost importance that the passed in seqno and reset_counter
+ * values have been read by the caller in an smp safe manner. Where read-side
+ * locks are involved, it is sufficient to read the reset_counter before
+ * unlocking the lock that protects the seqno. For lockless tricks, the
+ * reset_counter _must_ be read before, and an appropriate smp_rmb must be
+ * inserted.
+ *
+ * Returns 0 if the request was found within the alloted time. Else returns the
+ * errno with remaining time filled in timeout argument.
+ */
+int i915_wait_request(struct drm_i915_gem_request *req,
+		      unsigned int flags,
+		      s64 *timeout,
+		      struct intel_rps_client *rps)
+{
+	const int state = flags & I915_WAIT_INTERRUPTIBLE ?
+		TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
+	DEFINE_WAIT(reset);
+	struct intel_wait wait;
+	unsigned long timeout_remain;
+	int ret = 0;
+
+	might_sleep();
+#if IS_ENABLED(CONFIG_LOCKDEP)
+	GEM_BUG_ON(!!lockdep_is_held(&req->i915->drm.struct_mutex) !=
+		   !!(flags & I915_WAIT_LOCKED));
+#endif
+
+	if (i915_gem_request_completed(req))
+		return 0;
+
+	timeout_remain = MAX_SCHEDULE_TIMEOUT;
+	if (timeout) {
+		if (WARN_ON(*timeout < 0))
+			return -EINVAL;
+
+		if (*timeout == 0)
+			return -ETIME;
+
+		/* Record current time in case interrupted, or wedged */
+		timeout_remain = nsecs_to_jiffies_timeout(*timeout);
+		*timeout += ktime_get_raw_ns();
+	}
+
+	trace_i915_gem_request_wait_begin(req);
+
+	/* This client is about to stall waiting for the GPU. In many cases
+	 * this is undesirable and limits the throughput of the system, as
+	 * many clients cannot continue processing user input/output whilst
+	 * blocked. RPS autotuning may take tens of milliseconds to respond
+	 * to the GPU load and thus incurs additional latency for the client.
+	 * We can circumvent that by promoting the GPU frequency to maximum
+	 * before we wait. This makes the GPU throttle up much more quickly
+	 * (good for benchmarks and user experience, e.g. window animations),
+	 * but at a cost of spending more power processing the workload
+	 * (bad for battery). Not all clients even want their results
+	 * immediately and for them we should just let the GPU select its own
+	 * frequency to maximise efficiency. To prevent a single client from
+	 * forcing the clocks too high for the whole system, we only allow
+	 * each client to waitboost once in a busy period.
+	 */
+	if (IS_RPS_CLIENT(rps) && INTEL_GEN(req->i915) >= 6)
+		gen6_rps_boost(req->i915, rps, req->emitted_jiffies);
+
+	/* Optimistic short spin before touching IRQs */
+	if (i915_spin_request(req, state, 5))
+		goto complete;
+
+	set_current_state(state);
+	if (flags & I915_WAIT_LOCKED)
+		add_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
+
+	intel_wait_init(&wait, req->fence.seqno);
+	if (intel_engine_add_wait(req->engine, &wait))
+		/* In order to check that we haven't missed the interrupt
+		 * as we enabled it, we need to kick ourselves to do a
+		 * coherent check on the seqno before we sleep.
+		 */
+		goto wakeup;
+
+	for (;;) {
+		if (signal_pending_state(state, current)) {
+			ret = -ERESTARTSYS;
+			break;
+		}
+
+		timeout_remain = io_schedule_timeout(timeout_remain);
+		if (timeout_remain == 0) {
+			ret = -ETIME;
+			break;
+		}
+
+		if (intel_wait_complete(&wait))
+			break;
+
+		set_current_state(state);
+
+wakeup:
+		/* Carefully check if the request is complete, giving time
+		 * for the seqno to be visible following the interrupt.
+		 * We also have to check in case we are kicked by the GPU
+		 * reset in order to drop the struct_mutex.
+		 */
+		if (__i915_request_irq_complete(req))
+			break;
+
+		/* If the GPU is hung, and we hold the lock, reset the GPU
+		 * and then check for completion. On a full reset, the engine's
+		 * HW seqno will be advanced passed us and we are complete.
+		 * If we do a partial reset, we have to wait for the GPU to
+		 * resume and update the breadcrumb.
+		 *
+		 * If we don't hold the mutex, we can just wait for the worker
+		 * to come along and update the breadcrumb (either directly
+		 * itself, or indirectly by recovering the GPU).
+		 */
+		if (flags & I915_WAIT_LOCKED &&
+		    i915_reset_in_progress(&req->i915->gpu_error)) {
+			__set_current_state(TASK_RUNNING);
+			i915_reset(req->i915);
+			reset_wait_queue(&req->i915->gpu_error.wait_queue,
+					 &reset);
+			continue;
+		}
+
+		/* Only spin if we know the GPU is processing this request */
+		if (i915_spin_request(req, state, 2))
+			break;
+	}
+
+	intel_engine_remove_wait(req->engine, &wait);
+	if (flags & I915_WAIT_LOCKED)
+		remove_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
+	__set_current_state(TASK_RUNNING);
+
+complete:
+	trace_i915_gem_request_wait_end(req);
+
+	if (timeout) {
+		*timeout -= ktime_get_raw_ns();
+		if (*timeout < 0)
+			*timeout = 0;
+
+		/*
+		 * Apparently ktime isn't accurate enough and occasionally has a
+		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
+		 * things up to make the test happy. We allow up to 1 jiffy.
+		 *
+		 * This is a regrssion from the timespec->ktime conversion.
+		 */
+		if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
+			*timeout = 0;
+	}
+
+	if (IS_RPS_USER(rps) &&
+	    req->fence.seqno == req->engine->last_submitted_seqno) {
+		/* The GPU is now idle and this client has stalled.
+		 * Since no other client has submitted a request in the
+		 * meantime, assume that this client is the only one
+		 * supplying work to the GPU but is unable to keep that
+		 * work supplied because it is waiting. Since the GPU is
+		 * then never kept fully busy, RPS autoclocking will
+		 * keep the clocks relatively low, causing further delays.
+		 * Compensate by giving the synchronous client credit for
+		 * a waitboost next time.
+		 */
+		spin_lock(&req->i915->rps.client_lock);
+		list_del_init(&rps->link);
+		spin_unlock(&req->i915->rps.client_lock);
+	}
+
+	return ret;
+}
+
+static bool engine_retire_requests(struct intel_engine_cs *engine)
+{
+	struct drm_i915_gem_request *request, *next;
+
+	list_for_each_entry_safe(request, next, &engine->request_list, link) {
+		if (!i915_gem_request_completed(request))
+			return false;
+
+		i915_gem_request_retire(request);
+	}
+
+	return true;
+}
+
+void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+	unsigned int tmp;
+
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
+	if (dev_priv->gt.active_engines == 0)
+		return;
+
+	GEM_BUG_ON(!dev_priv->gt.awake);
+
+	for_each_engine_masked(engine, dev_priv, dev_priv->gt.active_engines, tmp)
+		if (engine_retire_requests(engine))
+			dev_priv->gt.active_engines &= ~intel_engine_flag(engine);
+
+	if (dev_priv->gt.active_engines == 0)
+		queue_delayed_work(dev_priv->wq,
+				   &dev_priv->gt.idle_work,
+				   msecs_to_jiffies(100));
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h
new file mode 100644
index 000000000000..974bd7bcc801
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_request.h
@@ -0,0 +1,689 @@
+/*
+ * Copyright © 2008-2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef I915_GEM_REQUEST_H
+#define I915_GEM_REQUEST_H
+
+#include <linux/fence.h>
+
+#include "i915_gem.h"
+#include "i915_sw_fence.h"
+
+struct intel_wait {
+	struct rb_node node;
+	struct task_struct *tsk;
+	u32 seqno;
+};
+
+struct intel_signal_node {
+	struct rb_node node;
+	struct intel_wait wait;
+};
+
+/**
+ * Request queue structure.
+ *
+ * The request queue allows us to note sequence numbers that have been emitted
+ * and may be associated with active buffers to be retired.
+ *
+ * By keeping this list, we can avoid having to do questionable sequence
+ * number comparisons on buffer last_read|write_seqno. It also allows an
+ * emission time to be associated with the request for tracking how far ahead
+ * of the GPU the submission is.
+ *
+ * When modifying this structure be very aware that we perform a lockless
+ * RCU lookup of it that may race against reallocation of the struct
+ * from the slab freelist. We intentionally do not zero the structure on
+ * allocation so that the lookup can use the dangling pointers (and is
+ * cogniscent that those pointers may be wrong). Instead, everything that
+ * needs to be initialised must be done so explicitly.
+ *
+ * The requests are reference counted.
+ */
+struct drm_i915_gem_request {
+	struct fence fence;
+	spinlock_t lock;
+
+	/** On Which ring this request was generated */
+	struct drm_i915_private *i915;
+
+	/**
+	 * Context and ring buffer related to this request
+	 * Contexts are refcounted, so when this request is associated with a
+	 * context, we must increment the context's refcount, to guarantee that
+	 * it persists while any request is linked to it. Requests themselves
+	 * are also refcounted, so the request will only be freed when the last
+	 * reference to it is dismissed, and the code in
+	 * i915_gem_request_free() will then decrement the refcount on the
+	 * context.
+	 */
+	struct i915_gem_context *ctx;
+	struct intel_engine_cs *engine;
+	struct intel_ring *ring;
+	struct intel_signal_node signaling;
+
+	struct i915_sw_fence submit;
+	wait_queue_t submitq;
+
+	/** GEM sequence number associated with the previous request,
+	 * when the HWS breadcrumb is equal to this the GPU is processing
+	 * this request.
+	 */
+	u32 previous_seqno;
+
+	/** Position in the ring of the start of the request */
+	u32 head;
+
+	/**
+	 * Position in the ring of the start of the postfix.
+	 * This is required to calculate the maximum available ring space
+	 * without overwriting the postfix.
+	 */
+	u32 postfix;
+
+	/** Position in the ring of the end of the whole request */
+	u32 tail;
+
+	/** Position in the ring of the end of any workarounds after the tail */
+	u32 wa_tail;
+
+	/** Preallocate space in the ring for the emitting the request */
+	u32 reserved_space;
+
+	/**
+	 * Context related to the previous request.
+	 * As the contexts are accessed by the hardware until the switch is
+	 * completed to a new context, the hardware may still be writing
+	 * to the context object after the breadcrumb is visible. We must
+	 * not unpin/unbind/prune that object whilst still active and so
+	 * we keep the previous context pinned until the following (this)
+	 * request is retired.
+	 */
+	struct i915_gem_context *previous_context;
+
+	/** Batch buffer related to this request if any (used for
+	 * error state dump only).
+	 */
+	struct i915_vma *batch;
+	struct list_head active_list;
+
+	/** Time at which this request was emitted, in jiffies. */
+	unsigned long emitted_jiffies;
+
+	/** engine->request_list entry for this request */
+	struct list_head link;
+
+	/** ring->request_list entry for this request */
+	struct list_head ring_link;
+
+	struct drm_i915_file_private *file_priv;
+	/** file_priv list entry for this request */
+	struct list_head client_list;
+
+	/** Link in the execlist submission queue, guarded by execlist_lock. */
+	struct list_head execlist_link;
+};
+
+extern const struct fence_ops i915_fence_ops;
+
+static inline bool fence_is_i915(struct fence *fence)
+{
+	return fence->ops == &i915_fence_ops;
+}
+
+struct drm_i915_gem_request * __must_check
+i915_gem_request_alloc(struct intel_engine_cs *engine,
+		       struct i915_gem_context *ctx);
+int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
+				   struct drm_file *file);
+void i915_gem_request_retire_upto(struct drm_i915_gem_request *req);
+
+static inline u32
+i915_gem_request_get_seqno(struct drm_i915_gem_request *req)
+{
+	return req ? req->fence.seqno : 0;
+}
+
+static inline struct intel_engine_cs *
+i915_gem_request_get_engine(struct drm_i915_gem_request *req)
+{
+	return req ? req->engine : NULL;
+}
+
+static inline struct drm_i915_gem_request *
+to_request(struct fence *fence)
+{
+	/* We assume that NULL fence/request are interoperable */
+	BUILD_BUG_ON(offsetof(struct drm_i915_gem_request, fence) != 0);
+	GEM_BUG_ON(fence && !fence_is_i915(fence));
+	return container_of(fence, struct drm_i915_gem_request, fence);
+}
+
+static inline struct drm_i915_gem_request *
+i915_gem_request_get(struct drm_i915_gem_request *req)
+{
+	return to_request(fence_get(&req->fence));
+}
+
+static inline struct drm_i915_gem_request *
+i915_gem_request_get_rcu(struct drm_i915_gem_request *req)
+{
+	return to_request(fence_get_rcu(&req->fence));
+}
+
+static inline void
+i915_gem_request_put(struct drm_i915_gem_request *req)
+{
+	fence_put(&req->fence);
+}
+
+static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
+					   struct drm_i915_gem_request *src)
+{
+	if (src)
+		i915_gem_request_get(src);
+
+	if (*pdst)
+		i915_gem_request_put(*pdst);
+
+	*pdst = src;
+}
+
+int
+i915_gem_request_await_object(struct drm_i915_gem_request *to,
+			      struct drm_i915_gem_object *obj,
+			      bool write);
+
+void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches);
+#define i915_add_request(req) \
+	__i915_add_request(req, true)
+#define i915_add_request_no_flush(req) \
+	__i915_add_request(req, false)
+
+struct intel_rps_client;
+#define NO_WAITBOOST ERR_PTR(-1)
+#define IS_RPS_CLIENT(p) (!IS_ERR(p))
+#define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p))
+
+int i915_wait_request(struct drm_i915_gem_request *req,
+		      unsigned int flags,
+		      s64 *timeout,
+		      struct intel_rps_client *rps)
+	__attribute__((nonnull(1)));
+#define I915_WAIT_INTERRUPTIBLE	BIT(0)
+#define I915_WAIT_LOCKED	BIT(1) /* struct_mutex held, handle GPU reset */
+
+static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine);
+
+/**
+ * Returns true if seq1 is later than seq2.
+ */
+static inline bool i915_seqno_passed(u32 seq1, u32 seq2)
+{
+	return (s32)(seq1 - seq2) >= 0;
+}
+
+static inline bool
+i915_gem_request_started(const struct drm_i915_gem_request *req)
+{
+	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
+				 req->previous_seqno);
+}
+
+static inline bool
+i915_gem_request_completed(const struct drm_i915_gem_request *req)
+{
+	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
+				 req->fence.seqno);
+}
+
+bool __i915_spin_request(const struct drm_i915_gem_request *request,
+			 int state, unsigned long timeout_us);
+static inline bool i915_spin_request(const struct drm_i915_gem_request *request,
+				     int state, unsigned long timeout_us)
+{
+	return (i915_gem_request_started(request) &&
+		__i915_spin_request(request, state, timeout_us));
+}
+
+/* We treat requests as fences. This is not be to confused with our
+ * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync.
+ * We use the fences to synchronize access from the CPU with activity on the
+ * GPU, for example, we should not rewrite an object's PTE whilst the GPU
+ * is reading them. We also track fences at a higher level to provide
+ * implicit synchronisation around GEM objects, e.g. set-domain will wait
+ * for outstanding GPU rendering before marking the object ready for CPU
+ * access, or a pageflip will wait until the GPU is complete before showing
+ * the frame on the scanout.
+ *
+ * In order to use a fence, the object must track the fence it needs to
+ * serialise with. For example, GEM objects want to track both read and
+ * write access so that we can perform concurrent read operations between
+ * the CPU and GPU engines, as well as waiting for all rendering to
+ * complete, or waiting for the last GPU user of a "fence register". The
+ * object then embeds a #i915_gem_active to track the most recent (in
+ * retirement order) request relevant for the desired mode of access.
+ * The #i915_gem_active is updated with i915_gem_active_set() to track the
+ * most recent fence request, typically this is done as part of
+ * i915_vma_move_to_active().
+ *
+ * When the #i915_gem_active completes (is retired), it will
+ * signal its completion to the owner through a callback as well as mark
+ * itself as idle (i915_gem_active.request == NULL). The owner
+ * can then perform any action, such as delayed freeing of an active
+ * resource including itself.
+ */
+struct i915_gem_active;
+
+typedef void (*i915_gem_retire_fn)(struct i915_gem_active *,
+				   struct drm_i915_gem_request *);
+
+struct i915_gem_active {
+	struct drm_i915_gem_request __rcu *request;
+	struct list_head link;
+	i915_gem_retire_fn retire;
+};
+
+void i915_gem_retire_noop(struct i915_gem_active *,
+			  struct drm_i915_gem_request *request);
+
+/**
+ * init_request_active - prepares the activity tracker for use
+ * @active - the active tracker
+ * @func - a callback when then the tracker is retired (becomes idle),
+ *         can be NULL
+ *
+ * init_request_active() prepares the embedded @active struct for use as
+ * an activity tracker, that is for tracking the last known active request
+ * associated with it. When the last request becomes idle, when it is retired
+ * after completion, the optional callback @func is invoked.
+ */
+static inline void
+init_request_active(struct i915_gem_active *active,
+		    i915_gem_retire_fn retire)
+{
+	INIT_LIST_HEAD(&active->link);
+	active->retire = retire ?: i915_gem_retire_noop;
+}
+
+/**
+ * i915_gem_active_set - updates the tracker to watch the current request
+ * @active - the active tracker
+ * @request - the request to watch
+ *
+ * i915_gem_active_set() watches the given @request for completion. Whilst
+ * that @request is busy, the @active reports busy. When that @request is
+ * retired, the @active tracker is updated to report idle.
+ */
+static inline void
+i915_gem_active_set(struct i915_gem_active *active,
+		    struct drm_i915_gem_request *request)
+{
+	list_move(&active->link, &request->active_list);
+	rcu_assign_pointer(active->request, request);
+}
+
+static inline struct drm_i915_gem_request *
+__i915_gem_active_peek(const struct i915_gem_active *active)
+{
+	/* Inside the error capture (running with the driver in an unknown
+	 * state), we want to bend the rules slightly (a lot).
+	 *
+	 * Work is in progress to make it safer, in the meantime this keeps
+	 * the known issue from spamming the logs.
+	 */
+	return rcu_dereference_protected(active->request, 1);
+}
+
+/**
+ * i915_gem_active_raw - return the active request
+ * @active - the active tracker
+ *
+ * i915_gem_active_raw() returns the current request being tracked, or NULL.
+ * It does not obtain a reference on the request for the caller, so the caller
+ * must hold struct_mutex.
+ */
+static inline struct drm_i915_gem_request *
+i915_gem_active_raw(const struct i915_gem_active *active, struct mutex *mutex)
+{
+	return rcu_dereference_protected(active->request,
+					 lockdep_is_held(mutex));
+}
+
+/**
+ * i915_gem_active_peek - report the active request being monitored
+ * @active - the active tracker
+ *
+ * i915_gem_active_peek() returns the current request being tracked if
+ * still active, or NULL. It does not obtain a reference on the request
+ * for the caller, so the caller must hold struct_mutex.
+ */
+static inline struct drm_i915_gem_request *
+i915_gem_active_peek(const struct i915_gem_active *active, struct mutex *mutex)
+{
+	struct drm_i915_gem_request *request;
+
+	request = i915_gem_active_raw(active, mutex);
+	if (!request || i915_gem_request_completed(request))
+		return NULL;
+
+	return request;
+}
+
+/**
+ * i915_gem_active_get - return a reference to the active request
+ * @active - the active tracker
+ *
+ * i915_gem_active_get() returns a reference to the active request, or NULL
+ * if the active tracker is idle. The caller must hold struct_mutex.
+ */
+static inline struct drm_i915_gem_request *
+i915_gem_active_get(const struct i915_gem_active *active, struct mutex *mutex)
+{
+	return i915_gem_request_get(i915_gem_active_peek(active, mutex));
+}
+
+/**
+ * __i915_gem_active_get_rcu - return a reference to the active request
+ * @active - the active tracker
+ *
+ * __i915_gem_active_get() returns a reference to the active request, or NULL
+ * if the active tracker is idle. The caller must hold the RCU read lock, but
+ * the returned pointer is safe to use outside of RCU.
+ */
+static inline struct drm_i915_gem_request *
+__i915_gem_active_get_rcu(const struct i915_gem_active *active)
+{
+	/* Performing a lockless retrieval of the active request is super
+	 * tricky. SLAB_DESTROY_BY_RCU merely guarantees that the backing
+	 * slab of request objects will not be freed whilst we hold the
+	 * RCU read lock. It does not guarantee that the request itself
+	 * will not be freed and then *reused*. Viz,
+	 *
+	 * Thread A			Thread B
+	 *
+	 * req = active.request
+	 *				retire(req) -> free(req);
+	 *				(req is now first on the slab freelist)
+	 *				active.request = NULL
+	 *
+	 *				req = new submission on a new object
+	 * ref(req)
+	 *
+	 * To prevent the request from being reused whilst the caller
+	 * uses it, we take a reference like normal. Whilst acquiring
+	 * the reference we check that it is not in a destroyed state
+	 * (refcnt == 0). That prevents the request being reallocated
+	 * whilst the caller holds on to it. To check that the request
+	 * was not reallocated as we acquired the reference we have to
+	 * check that our request remains the active request across
+	 * the lookup, in the same manner as a seqlock. The visibility
+	 * of the pointer versus the reference counting is controlled
+	 * by using RCU barriers (rcu_dereference and rcu_assign_pointer).
+	 *
+	 * In the middle of all that, we inspect whether the request is
+	 * complete. Retiring is lazy so the request may be completed long
+	 * before the active tracker is updated. Querying whether the
+	 * request is complete is far cheaper (as it involves no locked
+	 * instructions setting cachelines to exclusive) than acquiring
+	 * the reference, so we do it first. The RCU read lock ensures the
+	 * pointer dereference is valid, but does not ensure that the
+	 * seqno nor HWS is the right one! However, if the request was
+	 * reallocated, that means the active tracker's request was complete.
+	 * If the new request is also complete, then both are and we can
+	 * just report the active tracker is idle. If the new request is
+	 * incomplete, then we acquire a reference on it and check that
+	 * it remained the active request.
+	 *
+	 * It is then imperative that we do not zero the request on
+	 * reallocation, so that we can chase the dangling pointers!
+	 * See i915_gem_request_alloc().
+	 */
+	do {
+		struct drm_i915_gem_request *request;
+
+		request = rcu_dereference(active->request);
+		if (!request || i915_gem_request_completed(request))
+			return NULL;
+
+		/* An especially silly compiler could decide to recompute the
+		 * result of i915_gem_request_completed, more specifically
+		 * re-emit the load for request->fence.seqno. A race would catch
+		 * a later seqno value, which could flip the result from true to
+		 * false. Which means part of the instructions below might not
+		 * be executed, while later on instructions are executed. Due to
+		 * barriers within the refcounting the inconsistency can't reach
+		 * past the call to i915_gem_request_get_rcu, but not executing
+		 * that while still executing i915_gem_request_put() creates
+		 * havoc enough.  Prevent this with a compiler barrier.
+		 */
+		barrier();
+
+		request = i915_gem_request_get_rcu(request);
+
+		/* What stops the following rcu_access_pointer() from occurring
+		 * before the above i915_gem_request_get_rcu()? If we were
+		 * to read the value before pausing to get the reference to
+		 * the request, we may not notice a change in the active
+		 * tracker.
+		 *
+		 * The rcu_access_pointer() is a mere compiler barrier, which
+		 * means both the CPU and compiler are free to perform the
+		 * memory read without constraint. The compiler only has to
+		 * ensure that any operations after the rcu_access_pointer()
+		 * occur afterwards in program order. This means the read may
+		 * be performed earlier by an out-of-order CPU, or adventurous
+		 * compiler.
+		 *
+		 * The atomic operation at the heart of
+		 * i915_gem_request_get_rcu(), see fence_get_rcu(), is
+		 * atomic_inc_not_zero() which is only a full memory barrier
+		 * when successful. That is, if i915_gem_request_get_rcu()
+		 * returns the request (and so with the reference counted
+		 * incremented) then the following read for rcu_access_pointer()
+		 * must occur after the atomic operation and so confirm
+		 * that this request is the one currently being tracked.
+		 *
+		 * The corresponding write barrier is part of
+		 * rcu_assign_pointer().
+		 */
+		if (!request || request == rcu_access_pointer(active->request))
+			return rcu_pointer_handoff(request);
+
+		i915_gem_request_put(request);
+	} while (1);
+}
+
+/**
+ * i915_gem_active_get_unlocked - return a reference to the active request
+ * @active - the active tracker
+ *
+ * i915_gem_active_get_unlocked() returns a reference to the active request,
+ * or NULL if the active tracker is idle. The reference is obtained under RCU,
+ * so no locking is required by the caller.
+ *
+ * The reference should be freed with i915_gem_request_put().
+ */
+static inline struct drm_i915_gem_request *
+i915_gem_active_get_unlocked(const struct i915_gem_active *active)
+{
+	struct drm_i915_gem_request *request;
+
+	rcu_read_lock();
+	request = __i915_gem_active_get_rcu(active);
+	rcu_read_unlock();
+
+	return request;
+}
+
+/**
+ * i915_gem_active_isset - report whether the active tracker is assigned
+ * @active - the active tracker
+ *
+ * i915_gem_active_isset() returns true if the active tracker is currently
+ * assigned to a request. Due to the lazy retiring, that request may be idle
+ * and this may report stale information.
+ */
+static inline bool
+i915_gem_active_isset(const struct i915_gem_active *active)
+{
+	return rcu_access_pointer(active->request);
+}
+
+/**
+ * i915_gem_active_is_idle - report whether the active tracker is idle
+ * @active - the active tracker
+ *
+ * i915_gem_active_is_idle() returns true if the active tracker is currently
+ * unassigned or if the request is complete (but not yet retired). Requires
+ * the caller to hold struct_mutex (but that can be relaxed if desired).
+ */
+static inline bool
+i915_gem_active_is_idle(const struct i915_gem_active *active,
+			struct mutex *mutex)
+{
+	return !i915_gem_active_peek(active, mutex);
+}
+
+/**
+ * i915_gem_active_wait - waits until the request is completed
+ * @active - the active request on which to wait
+ *
+ * i915_gem_active_wait() waits until the request is completed before
+ * returning. Note that it does not guarantee that the request is
+ * retired first, see i915_gem_active_retire().
+ *
+ * i915_gem_active_wait() returns immediately if the active
+ * request is already complete.
+ */
+static inline int __must_check
+i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex)
+{
+	struct drm_i915_gem_request *request;
+
+	request = i915_gem_active_peek(active, mutex);
+	if (!request)
+		return 0;
+
+	return i915_wait_request(request,
+				 I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
+				 NULL, NULL);
+}
+
+/**
+ * i915_gem_active_wait_unlocked - waits until the request is completed
+ * @active - the active request on which to wait
+ * @flags - how to wait
+ * @timeout - how long to wait at most
+ * @rps - userspace client to charge for a waitboost
+ *
+ * i915_gem_active_wait_unlocked() waits until the request is completed before
+ * returning, without requiring any locks to be held. Note that it does not
+ * retire any requests before returning.
+ *
+ * This function relies on RCU in order to acquire the reference to the active
+ * request without holding any locks. See __i915_gem_active_get_rcu() for the
+ * glory details on how that is managed. Once the reference is acquired, we
+ * can then wait upon the request, and afterwards release our reference,
+ * free of any locking.
+ *
+ * This function wraps i915_wait_request(), see it for the full details on
+ * the arguments.
+ *
+ * Returns 0 if successful, or a negative error code.
+ */
+static inline int
+i915_gem_active_wait_unlocked(const struct i915_gem_active *active,
+			      unsigned int flags,
+			      s64 *timeout,
+			      struct intel_rps_client *rps)
+{
+	struct drm_i915_gem_request *request;
+	int ret = 0;
+
+	request = i915_gem_active_get_unlocked(active);
+	if (request) {
+		ret = i915_wait_request(request, flags, timeout, rps);
+		i915_gem_request_put(request);
+	}
+
+	return ret;
+}
+
+/**
+ * i915_gem_active_retire - waits until the request is retired
+ * @active - the active request on which to wait
+ *
+ * i915_gem_active_retire() waits until the request is completed,
+ * and then ensures that at least the retirement handler for this
+ * @active tracker is called before returning. If the @active
+ * tracker is idle, the function returns immediately.
+ */
+static inline int __must_check
+i915_gem_active_retire(struct i915_gem_active *active,
+		       struct mutex *mutex)
+{
+	struct drm_i915_gem_request *request;
+	int ret;
+
+	request = i915_gem_active_raw(active, mutex);
+	if (!request)
+		return 0;
+
+	ret = i915_wait_request(request,
+				I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
+				NULL, NULL);
+	if (ret)
+		return ret;
+
+	list_del_init(&active->link);
+	RCU_INIT_POINTER(active->request, NULL);
+
+	active->retire(active, request);
+
+	return 0;
+}
+
+/* Convenience functions for peeking at state inside active's request whilst
+ * guarded by the struct_mutex.
+ */
+
+static inline uint32_t
+i915_gem_active_get_seqno(const struct i915_gem_active *active,
+			  struct mutex *mutex)
+{
+	return i915_gem_request_get_seqno(i915_gem_active_peek(active, mutex));
+}
+
+static inline struct intel_engine_cs *
+i915_gem_active_get_engine(const struct i915_gem_active *active,
+			   struct mutex *mutex)
+{
+	return i915_gem_request_get_engine(i915_gem_active_peek(active, mutex));
+}
+
+#define for_each_active(mask, idx) \
+	for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx))
+
+#endif /* I915_GEM_REQUEST_H */
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 6f10b421487b..1c237d02f30b 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -48,19 +48,15 @@ static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
 #endif
 }
 
-static int num_vma_bound(struct drm_i915_gem_object *obj)
+static bool any_vma_pinned(struct drm_i915_gem_object *obj)
 {
 	struct i915_vma *vma;
-	int count = 0;
 
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (drm_mm_node_allocated(&vma->node))
-			count++;
-		if (vma->pin_count)
-			count++;
-	}
+	list_for_each_entry(vma, &obj->vma_list, obj_link)
+		if (i915_vma_is_pinned(vma))
+			return true;
 
-	return count;
+	return false;
 }
 
 static bool swap_available(void)
@@ -82,7 +78,10 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
 	 * to the GPU, simply unbinding from the GPU is not going to succeed
 	 * in releasing our pin count on the pages themselves.
 	 */
-	if (obj->pages_pin_count != num_vma_bound(obj))
+	if (obj->pages_pin_count > obj->bind_count)
+		return false;
+
+	if (any_vma_pinned(obj))
 		return false;
 
 	/* We can only return physical pages to the system if we can either
@@ -163,17 +162,16 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 	 */
 	for (phase = phases; phase->list; phase++) {
 		struct list_head still_in_list;
+		struct drm_i915_gem_object *obj;
 
 		if ((flags & phase->bit) == 0)
 			continue;
 
 		INIT_LIST_HEAD(&still_in_list);
-		while (count < target && !list_empty(phase->list)) {
-			struct drm_i915_gem_object *obj;
-			struct i915_vma *vma, *v;
-
-			obj = list_first_entry(phase->list,
-					       typeof(*obj), global_list);
+		while (count < target &&
+		       (obj = list_first_entry_or_null(phase->list,
+						       typeof(*obj),
+						       global_list))) {
 			list_move_tail(&obj->global_list, &still_in_list);
 
 			if (flags & I915_SHRINK_PURGEABLE &&
@@ -184,24 +182,21 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 			    !is_vmalloc_addr(obj->mapping))
 				continue;
 
-			if ((flags & I915_SHRINK_ACTIVE) == 0 && obj->active)
+			if ((flags & I915_SHRINK_ACTIVE) == 0 &&
+			    i915_gem_object_is_active(obj))
 				continue;
 
 			if (!can_release_pages(obj))
 				continue;
 
-			drm_gem_object_reference(&obj->base);
+			i915_gem_object_get(obj);
 
 			/* For the unbound phase, this should be a no-op! */
-			list_for_each_entry_safe(vma, v,
-						 &obj->vma_list, obj_link)
-				if (i915_vma_unbind(vma))
-					break;
-
+			i915_gem_object_unbind(obj);
 			if (i915_gem_object_put_pages(obj) == 0)
 				count += obj->base.size >> PAGE_SHIFT;
 
-			drm_gem_object_unreference(&obj->base);
+			i915_gem_object_put(obj);
 		}
 		list_splice(&still_in_list, phase->list);
 	}
@@ -210,6 +205,8 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 		intel_runtime_pm_put(dev_priv);
 
 	i915_gem_retire_requests(dev_priv);
+	/* expedite the RCU grace period to free some request slabs */
+	synchronize_rcu_expedited();
 
 	return count;
 }
@@ -230,10 +227,15 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
  */
 unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
 {
-	return i915_gem_shrink(dev_priv, -1UL,
-			       I915_SHRINK_BOUND |
-			       I915_SHRINK_UNBOUND |
-			       I915_SHRINK_ACTIVE);
+	unsigned long freed;
+
+	freed = i915_gem_shrink(dev_priv, -1UL,
+				I915_SHRINK_BOUND |
+				I915_SHRINK_UNBOUND |
+				I915_SHRINK_ACTIVE);
+	rcu_barrier(); /* wait until our RCU delayed slab frees are completed */
+
+	return freed;
 }
 
 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
@@ -242,9 +244,6 @@ static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
 		if (!mutex_is_locked_by(&dev->struct_mutex, current))
 			return false;
 
-		if (to_i915(dev)->mm.shrinker_no_lock_stealing)
-			return false;
-
 		*unlock = false;
 	} else
 		*unlock = true;
@@ -273,7 +272,7 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 			count += obj->base.size >> PAGE_SHIFT;
 
 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-		if (!obj->active && can_release_pages(obj))
+		if (!i915_gem_object_is_active(obj) && can_release_pages(obj))
 			count += obj->base.size >> PAGE_SHIFT;
 	}
 
@@ -321,17 +320,22 @@ i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv,
 				       struct shrinker_lock_uninterruptible *slu,
 				       int timeout_ms)
 {
-	unsigned long timeout = msecs_to_jiffies(timeout_ms) + 1;
+	unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
+
+	do {
+		if (i915_gem_wait_for_idle(dev_priv, 0) == 0 &&
+		    i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock))
+			break;
 
-	while (!i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock)) {
 		schedule_timeout_killable(1);
 		if (fatal_signal_pending(current))
 			return false;
-		if (--timeout == 0) {
+
+		if (time_after(jiffies, timeout)) {
 			pr_err("Unable to lock GPU to purge memory.\n");
 			return false;
 		}
-	}
+	} while (1);
 
 	slu->was_interruptible = dev_priv->mm.interruptible;
 	dev_priv->mm.interruptible = false;
@@ -410,7 +414,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
 		return NOTIFY_DONE;
 
 	/* Force everything onto the inactive lists */
-	ret = i915_gem_wait_for_idle(dev_priv);
+	ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
 	if (ret)
 		goto out;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 66be299a1486..59989e8ee5dc 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -92,6 +92,7 @@ void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
 static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct resource *r;
 	u32 base;
@@ -111,33 +112,44 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 	if (INTEL_INFO(dev)->gen >= 3) {
 		u32 bsm;
 
-		pci_read_config_dword(dev->pdev, INTEL_BSM, &bsm);
+		pci_read_config_dword(pdev, INTEL_BSM, &bsm);
 
 		base = bsm & INTEL_BSM_MASK;
 	} else if (IS_I865G(dev)) {
+		u32 tseg_size = 0;
 		u16 toud = 0;
+		u8 tmp;
 
-		/*
-		 * FIXME is the graphics stolen memory region
-		 * always at TOUD? Ie. is it always the last
-		 * one to be allocated by the BIOS?
-		 */
-		pci_bus_read_config_word(dev->pdev->bus, PCI_DEVFN(0, 0),
+		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
+					 I845_ESMRAMC, &tmp);
+
+		if (tmp & TSEG_ENABLE) {
+			switch (tmp & I845_TSEG_SIZE_MASK) {
+			case I845_TSEG_SIZE_512K:
+				tseg_size = KB(512);
+				break;
+			case I845_TSEG_SIZE_1M:
+				tseg_size = MB(1);
+				break;
+			}
+		}
+
+		pci_bus_read_config_word(pdev->bus, PCI_DEVFN(0, 0),
 					 I865_TOUD, &toud);
 
-		base = toud << 16;
+		base = (toud << 16) + tseg_size;
 	} else if (IS_I85X(dev)) {
 		u32 tseg_size = 0;
 		u32 tom;
 		u8 tmp;
 
-		pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
 					 I85X_ESMRAMC, &tmp);
 
 		if (tmp & TSEG_ENABLE)
 			tseg_size = MB(1);
 
-		pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 1),
+		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 1),
 					 I85X_DRB3, &tmp);
 		tom = tmp * MB(32);
 
@@ -147,7 +159,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 		u32 tom;
 		u8 tmp;
 
-		pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
 					 I845_ESMRAMC, &tmp);
 
 		if (tmp & TSEG_ENABLE) {
@@ -161,7 +173,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 			}
 		}
 
-		pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
 					 I830_DRB3, &tmp);
 		tom = tmp * MB(32);
 
@@ -171,7 +183,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 		u32 tom;
 		u8 tmp;
 
-		pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
 					 I830_ESMRAMC, &tmp);
 
 		if (tmp & TSEG_ENABLE) {
@@ -181,7 +193,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 				tseg_size = KB(512);
 		}
 
-		pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0),
+		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
 					 I830_DRB3, &tmp);
 		tom = tmp * MB(32);
 
@@ -685,7 +697,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 	if (gtt_offset == I915_GTT_OFFSET_NONE)
 		return obj;
 
-	vma = i915_gem_obj_lookup_or_create_vma(obj, &ggtt->base);
+	vma = i915_gem_obj_lookup_or_create_vma(obj, &ggtt->base, NULL);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		goto err;
@@ -698,24 +710,25 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 	 */
 	vma->node.start = gtt_offset;
 	vma->node.size = size;
-	if (drm_mm_initialized(&ggtt->base.mm)) {
-		ret = drm_mm_reserve_node(&ggtt->base.mm, &vma->node);
-		if (ret) {
-			DRM_DEBUG_KMS("failed to allocate stolen GTT space\n");
-			goto err;
-		}
 
-		vma->bound |= GLOBAL_BIND;
-		__i915_vma_set_map_and_fenceable(vma);
-		list_add_tail(&vma->vm_link, &ggtt->base.inactive_list);
+	ret = drm_mm_reserve_node(&ggtt->base.mm, &vma->node);
+	if (ret) {
+		DRM_DEBUG_KMS("failed to allocate stolen GTT space\n");
+		goto err;
 	}
 
+	vma->pages = obj->pages;
+	vma->flags |= I915_VMA_GLOBAL_BIND;
+	__i915_vma_set_map_and_fenceable(vma);
+	list_move_tail(&vma->vm_link, &ggtt->base.inactive_list);
+	obj->bind_count++;
+
 	list_add_tail(&obj->global_list, &dev_priv->mm.bound_list);
 	i915_gem_object_pin_pages(obj);
 
 	return obj;
 
 err:
-	drm_gem_object_unreference(&obj->base);
+	i915_gem_object_put(obj);
 	return NULL;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index 8030199731db..a14b1e3d4c78 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -68,6 +68,9 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
 	if (tiling_mode == I915_TILING_NONE)
 		return true;
 
+	if (tiling_mode > I915_TILING_LAST)
+		return false;
+
 	if (IS_GEN2(dev) ||
 	    (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
 		tile_width = 128;
@@ -113,36 +116,58 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
 	return true;
 }
 
-/* Is the current GTT allocation valid for the change in tiling? */
-static bool
-i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
+static bool i915_vma_fence_prepare(struct i915_vma *vma, int tiling_mode)
 {
+	struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
 	u32 size;
 
-	if (tiling_mode == I915_TILING_NONE)
-		return true;
-
-	if (INTEL_INFO(obj->base.dev)->gen >= 4)
+	if (!i915_vma_is_map_and_fenceable(vma))
 		return true;
 
-	if (IS_GEN3(obj->base.dev)) {
-		if (i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK)
+	if (INTEL_GEN(dev_priv) == 3) {
+		if (vma->node.start & ~I915_FENCE_START_MASK)
 			return false;
 	} else {
-		if (i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK)
+		if (vma->node.start & ~I830_FENCE_START_MASK)
 			return false;
 	}
 
-	size = i915_gem_get_gtt_size(obj->base.dev, obj->base.size, tiling_mode);
-	if (i915_gem_obj_ggtt_size(obj) != size)
+	size = i915_gem_get_ggtt_size(dev_priv, vma->size, tiling_mode);
+	if (vma->node.size < size)
 		return false;
 
-	if (i915_gem_obj_ggtt_offset(obj) & (size - 1))
+	if (vma->node.start & (size - 1))
 		return false;
 
 	return true;
 }
 
+/* Make the current GTT allocation valid for the change in tiling. */
+static int
+i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj, int tiling_mode)
+{
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+	struct i915_vma *vma;
+	int ret;
+
+	if (tiling_mode == I915_TILING_NONE)
+		return 0;
+
+	if (INTEL_GEN(dev_priv) >= 4)
+		return 0;
+
+	list_for_each_entry(vma, &obj->vma_list, obj_link) {
+		if (i915_vma_fence_prepare(vma, tiling_mode))
+			continue;
+
+		ret = i915_vma_unbind(vma);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 /**
  * i915_gem_set_tiling - IOCTL handler to set tiling mode
  * @dev: DRM device
@@ -164,15 +189,18 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 	struct drm_i915_gem_set_tiling *args = data;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_object *obj;
-	int ret = 0;
+	int err = 0;
+
+	/* Make sure we don't cross-contaminate obj->tiling_and_stride */
+	BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);
 
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
-	if (&obj->base == NULL)
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj)
 		return -ENOENT;
 
 	if (!i915_tiling_ok(dev,
 			    args->stride, obj->base.size, args->tiling_mode)) {
-		drm_gem_object_unreference_unlocked(&obj->base);
+		i915_gem_object_put_unlocked(obj);
 		return -EINVAL;
 	}
 
@@ -180,7 +208,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 
 	mutex_lock(&dev->struct_mutex);
 	if (obj->pin_display || obj->framebuffer_references) {
-		ret = -EBUSY;
+		err = -EBUSY;
 		goto err;
 	}
 
@@ -213,8 +241,8 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 		}
 	}
 
-	if (args->tiling_mode != obj->tiling_mode ||
-	    args->stride != obj->stride) {
+	if (args->tiling_mode != i915_gem_object_get_tiling(obj) ||
+	    args->stride != i915_gem_object_get_stride(obj)) {
 		/* We need to rebind the object if its current allocation
 		 * no longer meets the alignment restrictions for its new
 		 * tiling mode. Otherwise we can just leave it alone, but
@@ -227,34 +255,36 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 		 * has to also include the unfenced register the GPU uses
 		 * whilst executing a fenced command for an untiled object.
 		 */
-		if (obj->map_and_fenceable &&
-		    !i915_gem_object_fence_ok(obj, args->tiling_mode))
-			ret = i915_vma_unbind(i915_gem_obj_to_ggtt(obj));
 
-		if (ret == 0) {
+		err = i915_gem_object_fence_prepare(obj, args->tiling_mode);
+		if (!err) {
+			struct i915_vma *vma;
+
 			if (obj->pages &&
 			    obj->madv == I915_MADV_WILLNEED &&
 			    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
 				if (args->tiling_mode == I915_TILING_NONE)
 					i915_gem_object_unpin_pages(obj);
-				if (obj->tiling_mode == I915_TILING_NONE)
+				if (!i915_gem_object_is_tiled(obj))
 					i915_gem_object_pin_pages(obj);
 			}
 
-			obj->fence_dirty =
-				obj->last_fenced_req ||
-				obj->fence_reg != I915_FENCE_REG_NONE;
+			list_for_each_entry(vma, &obj->vma_list, obj_link) {
+				if (!vma->fence)
+					continue;
 
-			obj->tiling_mode = args->tiling_mode;
-			obj->stride = args->stride;
+				vma->fence->dirty = true;
+			}
+			obj->tiling_and_stride =
+				args->stride | args->tiling_mode;
 
 			/* Force the fence to be reacquired for GTT access */
 			i915_gem_release_mmap(obj);
 		}
 	}
 	/* we have to maintain this existing ABI... */
-	args->stride = obj->stride;
-	args->tiling_mode = obj->tiling_mode;
+	args->stride = i915_gem_object_get_stride(obj);
+	args->tiling_mode = i915_gem_object_get_tiling(obj);
 
 	/* Try to preallocate memory required to save swizzling on put-pages */
 	if (i915_gem_object_needs_bit17_swizzle(obj)) {
@@ -268,12 +298,12 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 	}
 
 err:
-	drm_gem_object_unreference(&obj->base);
+	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
 
 	intel_runtime_pm_put(dev_priv);
 
-	return ret;
+	return err;
 }
 
 /**
@@ -297,14 +327,12 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_object *obj;
 
-	obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
-	if (&obj->base == NULL)
+	obj = i915_gem_object_lookup(file, args->handle);
+	if (!obj)
 		return -ENOENT;
 
-	mutex_lock(&dev->struct_mutex);
-
-	args->tiling_mode = obj->tiling_mode;
-	switch (obj->tiling_mode) {
+	args->tiling_mode = READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
+	switch (args->tiling_mode) {
 	case I915_TILING_X:
 		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
 		break;
@@ -328,8 +356,6 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
 		args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
 
-	drm_gem_object_unreference(&obj->base);
-	mutex_unlock(&dev->struct_mutex);
-
+	i915_gem_object_put_unlocked(obj);
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index 2314c88323e3..e537930c64b5 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -63,33 +63,12 @@ struct i915_mmu_object {
 
 static void wait_rendering(struct drm_i915_gem_object *obj)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
-	int i, n;
-
-	if (!obj->active)
-		return;
-
-	n = 0;
-	for (i = 0; i < I915_NUM_ENGINES; i++) {
-		struct drm_i915_gem_request *req;
-
-		req = obj->last_read_req[i];
-		if (req == NULL)
-			continue;
-
-		requests[n++] = i915_gem_request_reference(req);
-	}
-
-	mutex_unlock(&dev->struct_mutex);
-
-	for (i = 0; i < n; i++)
-		__i915_wait_request(requests[i], false, NULL, NULL);
-
-	mutex_lock(&dev->struct_mutex);
+	unsigned long active = __I915_BO_ACTIVE(obj);
+	int idx;
 
-	for (i = 0; i < n; i++)
-		i915_gem_request_unreference(requests[i]);
+	for_each_active(active, idx)
+		i915_gem_active_wait_unlocked(&obj->last_read[idx],
+					      0, NULL, NULL);
 }
 
 static void cancel_userptr(struct work_struct *work)
@@ -98,28 +77,19 @@ static void cancel_userptr(struct work_struct *work)
 	struct drm_i915_gem_object *obj = mo->obj;
 	struct drm_device *dev = obj->base.dev;
 
+	wait_rendering(obj);
+
 	mutex_lock(&dev->struct_mutex);
 	/* Cancel any active worker and force us to re-evaluate gup */
 	obj->userptr.work = NULL;
 
 	if (obj->pages != NULL) {
-		struct drm_i915_private *dev_priv = to_i915(dev);
-		struct i915_vma *vma, *tmp;
-		bool was_interruptible;
-
-		wait_rendering(obj);
-
-		was_interruptible = dev_priv->mm.interruptible;
-		dev_priv->mm.interruptible = false;
-
-		list_for_each_entry_safe(vma, tmp, &obj->vma_list, obj_link)
-			WARN_ON(i915_vma_unbind(vma));
+		/* We are inside a kthread context and can't be interrupted */
+		WARN_ON(i915_gem_object_unbind(obj));
 		WARN_ON(i915_gem_object_put_pages(obj));
-
-		dev_priv->mm.interruptible = was_interruptible;
 	}
 
-	drm_gem_object_unreference(&obj->base);
+	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
 }
 
@@ -572,12 +542,10 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 			}
 		}
 		obj->userptr.work = ERR_PTR(ret);
-		if (ret)
-			__i915_gem_userptr_set_active(obj, false);
 	}
 
 	obj->userptr.workers--;
-	drm_gem_object_unreference(&obj->base);
+	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
 
 	release_pages(pvec, pinned, 0);
@@ -622,8 +590,7 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,
 	obj->userptr.work = &work->work;
 	obj->userptr.workers++;
 
-	work->obj = obj;
-	drm_gem_object_reference(&obj->base);
+	work->obj = i915_gem_object_get(obj);
 
 	work->task = current;
 	get_task_struct(work->task);
@@ -659,15 +626,14 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 	 * to the vma (discard or cloning) which should prevent the more
 	 * egregious cases from causing harm.
 	 */
-	if (IS_ERR(obj->userptr.work)) {
-		/* active flag will have been dropped already by the worker */
-		ret = PTR_ERR(obj->userptr.work);
-		obj->userptr.work = NULL;
-		return ret;
-	}
-	if (obj->userptr.work)
+
+	if (obj->userptr.work) {
 		/* active flag should still be held for the pending work */
-		return -EAGAIN;
+		if (IS_ERR(obj->userptr.work))
+			return PTR_ERR(obj->userptr.work);
+		else
+			return -EAGAIN;
+	}
 
 	/* Let the mmu-notifier know that we have begun and need cancellation */
 	ret = __i915_gem_userptr_set_active(obj, true);
@@ -846,7 +812,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file
 		ret = drm_gem_handle_create(file, &obj->base, &handle);
 
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(&obj->base);
+	i915_gem_object_put_unlocked(obj);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 9d73d2216adc..334f15df7c8d 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -30,9 +30,9 @@
 #include <generated/utsrelease.h>
 #include "i915_drv.h"
 
-static const char *ring_str(int ring)
+static const char *engine_str(int engine)
 {
-	switch (ring) {
+	switch (engine) {
 	case RCS: return "render";
 	case VCS: return "bsd";
 	case BCS: return "blt";
@@ -42,16 +42,6 @@ static const char *ring_str(int ring)
 	}
 }
 
-static const char *pin_flag(int pinned)
-{
-	if (pinned > 0)
-		return " P";
-	else if (pinned < 0)
-		return " p";
-	else
-		return "";
-}
-
 static const char *tiling_flag(int tiling)
 {
 	switch (tiling) {
@@ -189,7 +179,7 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
 {
 	int i;
 
-	err_printf(m, "  %s [%d]:\n", name, count);
+	err_printf(m, "%s [%d]:\n", name, count);
 
 	while (count--) {
 		err_printf(m, "    %08x_%08x %8u %02x %02x [ ",
@@ -202,13 +192,12 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
 			err_printf(m, "%02x ", err->rseqno[i]);
 
 		err_printf(m, "] %02x", err->wseqno);
-		err_puts(m, pin_flag(err->pinned));
 		err_puts(m, tiling_flag(err->tiling));
 		err_puts(m, dirty_flag(err->dirty));
 		err_puts(m, purgeable_flag(err->purgeable));
 		err_puts(m, err->userptr ? " userptr" : "");
-		err_puts(m, err->ring != -1 ? " " : "");
-		err_puts(m, ring_str(err->ring));
+		err_puts(m, err->engine != -1 ? " " : "");
+		err_puts(m, engine_str(err->engine));
 		err_puts(m, i915_cache_level_str(m->i915, err->cache_level));
 
 		if (err->name)
@@ -221,7 +210,7 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
 	}
 }
 
-static const char *hangcheck_action_to_str(enum intel_ring_hangcheck_action a)
+static const char *hangcheck_action_to_str(enum intel_engine_hangcheck_action a)
 {
 	switch (a) {
 	case HANGCHECK_IDLE:
@@ -239,70 +228,74 @@ static const char *hangcheck_action_to_str(enum intel_ring_hangcheck_action a)
 	return "unknown";
 }
 
-static void i915_ring_error_state(struct drm_i915_error_state_buf *m,
-				  struct drm_device *dev,
-				  struct drm_i915_error_state *error,
-				  int ring_idx)
+static void error_print_engine(struct drm_i915_error_state_buf *m,
+			       struct drm_i915_error_engine *ee)
 {
-	struct drm_i915_error_ring *ring = &error->ring[ring_idx];
-
-	if (!ring->valid)
-		return;
-
-	err_printf(m, "%s command stream:\n", ring_str(ring_idx));
-	err_printf(m, "  START: 0x%08x\n", ring->start);
-	err_printf(m, "  HEAD:  0x%08x\n", ring->head);
-	err_printf(m, "  TAIL:  0x%08x\n", ring->tail);
-	err_printf(m, "  CTL:   0x%08x\n", ring->ctl);
-	err_printf(m, "  HWS:   0x%08x\n", ring->hws);
-	err_printf(m, "  ACTHD: 0x%08x %08x\n", (u32)(ring->acthd>>32), (u32)ring->acthd);
-	err_printf(m, "  IPEIR: 0x%08x\n", ring->ipeir);
-	err_printf(m, "  IPEHR: 0x%08x\n", ring->ipehr);
-	err_printf(m, "  INSTDONE: 0x%08x\n", ring->instdone);
-	if (INTEL_INFO(dev)->gen >= 4) {
-		err_printf(m, "  BBADDR: 0x%08x %08x\n", (u32)(ring->bbaddr>>32), (u32)ring->bbaddr);
-		err_printf(m, "  BB_STATE: 0x%08x\n", ring->bbstate);
-		err_printf(m, "  INSTPS: 0x%08x\n", ring->instps);
+	err_printf(m, "%s command stream:\n", engine_str(ee->engine_id));
+	err_printf(m, "  START: 0x%08x\n", ee->start);
+	err_printf(m, "  HEAD:  0x%08x\n", ee->head);
+	err_printf(m, "  TAIL:  0x%08x\n", ee->tail);
+	err_printf(m, "  CTL:   0x%08x\n", ee->ctl);
+	err_printf(m, "  MODE:  0x%08x\n", ee->mode);
+	err_printf(m, "  HWS:   0x%08x\n", ee->hws);
+	err_printf(m, "  ACTHD: 0x%08x %08x\n",
+		   (u32)(ee->acthd>>32), (u32)ee->acthd);
+	err_printf(m, "  IPEIR: 0x%08x\n", ee->ipeir);
+	err_printf(m, "  IPEHR: 0x%08x\n", ee->ipehr);
+	err_printf(m, "  INSTDONE: 0x%08x\n", ee->instdone);
+	if (ee->batchbuffer) {
+		u64 start = ee->batchbuffer->gtt_offset;
+		u64 end = start + ee->batchbuffer->gtt_size;
+
+		err_printf(m, "  batch: [0x%08x_%08x, 0x%08x_%08x]\n",
+			   upper_32_bits(start), lower_32_bits(start),
+			   upper_32_bits(end), lower_32_bits(end));
 	}
-	err_printf(m, "  INSTPM: 0x%08x\n", ring->instpm);
-	err_printf(m, "  FADDR: 0x%08x %08x\n", upper_32_bits(ring->faddr),
-		   lower_32_bits(ring->faddr));
-	if (INTEL_INFO(dev)->gen >= 6) {
-		err_printf(m, "  RC PSMI: 0x%08x\n", ring->rc_psmi);
-		err_printf(m, "  FAULT_REG: 0x%08x\n", ring->fault_reg);
+	if (INTEL_GEN(m->i915) >= 4) {
+		err_printf(m, "  BBADDR: 0x%08x_%08x\n",
+			   (u32)(ee->bbaddr>>32), (u32)ee->bbaddr);
+		err_printf(m, "  BB_STATE: 0x%08x\n", ee->bbstate);
+		err_printf(m, "  INSTPS: 0x%08x\n", ee->instps);
+	}
+	err_printf(m, "  INSTPM: 0x%08x\n", ee->instpm);
+	err_printf(m, "  FADDR: 0x%08x %08x\n", upper_32_bits(ee->faddr),
+		   lower_32_bits(ee->faddr));
+	if (INTEL_GEN(m->i915) >= 6) {
+		err_printf(m, "  RC PSMI: 0x%08x\n", ee->rc_psmi);
+		err_printf(m, "  FAULT_REG: 0x%08x\n", ee->fault_reg);
 		err_printf(m, "  SYNC_0: 0x%08x [last synced 0x%08x]\n",
-			   ring->semaphore_mboxes[0],
-			   ring->semaphore_seqno[0]);
+			   ee->semaphore_mboxes[0],
+			   ee->semaphore_seqno[0]);
 		err_printf(m, "  SYNC_1: 0x%08x [last synced 0x%08x]\n",
-			   ring->semaphore_mboxes[1],
-			   ring->semaphore_seqno[1]);
-		if (HAS_VEBOX(dev)) {
+			   ee->semaphore_mboxes[1],
+			   ee->semaphore_seqno[1]);
+		if (HAS_VEBOX(m->i915)) {
 			err_printf(m, "  SYNC_2: 0x%08x [last synced 0x%08x]\n",
-				   ring->semaphore_mboxes[2],
-				   ring->semaphore_seqno[2]);
+				   ee->semaphore_mboxes[2],
+				   ee->semaphore_seqno[2]);
 		}
 	}
-	if (USES_PPGTT(dev)) {
-		err_printf(m, "  GFX_MODE: 0x%08x\n", ring->vm_info.gfx_mode);
+	if (USES_PPGTT(m->i915)) {
+		err_printf(m, "  GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode);
 
-		if (INTEL_INFO(dev)->gen >= 8) {
+		if (INTEL_GEN(m->i915) >= 8) {
 			int i;
 			for (i = 0; i < 4; i++)
 				err_printf(m, "  PDP%d: 0x%016llx\n",
-					   i, ring->vm_info.pdp[i]);
+					   i, ee->vm_info.pdp[i]);
 		} else {
 			err_printf(m, "  PP_DIR_BASE: 0x%08x\n",
-				   ring->vm_info.pp_dir_base);
+				   ee->vm_info.pp_dir_base);
 		}
 	}
-	err_printf(m, "  seqno: 0x%08x\n", ring->seqno);
-	err_printf(m, "  last_seqno: 0x%08x\n", ring->last_seqno);
-	err_printf(m, "  waiting: %s\n", yesno(ring->waiting));
-	err_printf(m, "  ring->head: 0x%08x\n", ring->cpu_ring_head);
-	err_printf(m, "  ring->tail: 0x%08x\n", ring->cpu_ring_tail);
+	err_printf(m, "  seqno: 0x%08x\n", ee->seqno);
+	err_printf(m, "  last_seqno: 0x%08x\n", ee->last_seqno);
+	err_printf(m, "  waiting: %s\n", yesno(ee->waiting));
+	err_printf(m, "  ring->head: 0x%08x\n", ee->cpu_ring_head);
+	err_printf(m, "  ring->tail: 0x%08x\n", ee->cpu_ring_tail);
 	err_printf(m, "  hangcheck: %s [%d]\n",
-		   hangcheck_action_to_str(ring->hangcheck_action),
-		   ring->hangcheck_score);
+		   hangcheck_action_to_str(ee->hangcheck_action),
+		   ee->hangcheck_score);
 }
 
 void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
@@ -328,11 +321,22 @@ static void print_error_obj(struct drm_i915_error_state_buf *m,
 	}
 }
 
+static void err_print_capabilities(struct drm_i915_error_state_buf *m,
+				   const struct intel_device_info *info)
+{
+#define PRINT_FLAG(x)  err_printf(m, #x ": %s\n", yesno(info->x))
+#define SEP_SEMICOLON ;
+	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_SEMICOLON);
+#undef PRINT_FLAG
+#undef SEP_SEMICOLON
+}
+
 int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			    const struct i915_error_state_file_priv *error_priv)
 {
 	struct drm_device *dev = error_priv->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct drm_i915_error_state *error = error_priv->error;
 	struct drm_i915_error_object *obj;
 	int i, j, offset, elt;
@@ -347,27 +351,28 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 	err_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec,
 		   error->time.tv_usec);
 	err_printf(m, "Kernel: " UTS_RELEASE "\n");
+	err_print_capabilities(m, &error->device_info);
 	max_hangcheck_score = 0;
-	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
-		if (error->ring[i].hangcheck_score > max_hangcheck_score)
-			max_hangcheck_score = error->ring[i].hangcheck_score;
+	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
+		if (error->engine[i].hangcheck_score > max_hangcheck_score)
+			max_hangcheck_score = error->engine[i].hangcheck_score;
 	}
-	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
-		if (error->ring[i].hangcheck_score == max_hangcheck_score &&
-		    error->ring[i].pid != -1) {
+	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
+		if (error->engine[i].hangcheck_score == max_hangcheck_score &&
+		    error->engine[i].pid != -1) {
 			err_printf(m, "Active process (on ring %s): %s [%d]\n",
-				   ring_str(i),
-				   error->ring[i].comm,
-				   error->ring[i].pid);
+				   engine_str(i),
+				   error->engine[i].comm,
+				   error->engine[i].pid);
 		}
 	}
 	err_printf(m, "Reset count: %u\n", error->reset_count);
 	err_printf(m, "Suspend count: %u\n", error->suspend_count);
-	err_printf(m, "PCI ID: 0x%04x\n", dev->pdev->device);
-	err_printf(m, "PCI Revision: 0x%02x\n", dev->pdev->revision);
+	err_printf(m, "PCI ID: 0x%04x\n", pdev->device);
+	err_printf(m, "PCI Revision: 0x%02x\n", pdev->revision);
 	err_printf(m, "PCI Subsystem: %04x:%04x\n",
-		   dev->pdev->subsystem_vendor,
-		   dev->pdev->subsystem_device);
+		   pdev->subsystem_vendor,
+		   pdev->subsystem_device);
 	err_printf(m, "IOMMU enabled?: %d\n", error->iommu);
 
 	if (HAS_CSR(dev)) {
@@ -414,36 +419,55 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 	if (IS_GEN7(dev))
 		err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);
 
-	for (i = 0; i < ARRAY_SIZE(error->ring); i++)
-		i915_ring_error_state(m, dev, error, i);
+	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
+		if (error->engine[i].engine_id != -1)
+			error_print_engine(m, &error->engine[i]);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(error->active_vm); i++) {
+		char buf[128];
+		int len, first = 1;
+
+		if (!error->active_vm[i])
+			break;
 
-	for (i = 0; i < error->vm_count; i++) {
-		err_printf(m, "vm[%d]\n", i);
+		len = scnprintf(buf, sizeof(buf), "Active (");
+		for (j = 0; j < ARRAY_SIZE(error->engine); j++) {
+			if (error->engine[j].vm != error->active_vm[i])
+				continue;
 
-		print_error_buffers(m, "Active",
+			len += scnprintf(buf + len, sizeof(buf), "%s%s",
+					 first ? "" : ", ",
+					 dev_priv->engine[j].name);
+			first = 0;
+		}
+		scnprintf(buf + len, sizeof(buf), ")");
+		print_error_buffers(m, buf,
 				    error->active_bo[i],
 				    error->active_bo_count[i]);
-
-		print_error_buffers(m, "Pinned",
-				    error->pinned_bo[i],
-				    error->pinned_bo_count[i]);
 	}
 
-	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
-		obj = error->ring[i].batchbuffer;
+	print_error_buffers(m, "Pinned (global)",
+			    error->pinned_bo,
+			    error->pinned_bo_count);
+
+	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
+		struct drm_i915_error_engine *ee = &error->engine[i];
+
+		obj = ee->batchbuffer;
 		if (obj) {
 			err_puts(m, dev_priv->engine[i].name);
-			if (error->ring[i].pid != -1)
+			if (ee->pid != -1)
 				err_printf(m, " (submitted by %s [%d])",
-					   error->ring[i].comm,
-					   error->ring[i].pid);
+					   ee->comm,
+					   ee->pid);
 			err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
 				   upper_32_bits(obj->gtt_offset),
 				   lower_32_bits(obj->gtt_offset));
 			print_error_obj(m, obj);
 		}
 
-		obj = error->ring[i].wa_batchbuffer;
+		obj = ee->wa_batchbuffer;
 		if (obj) {
 			err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n",
 				   dev_priv->engine[i].name,
@@ -451,38 +475,43 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			print_error_obj(m, obj);
 		}
 
-		if (error->ring[i].num_requests) {
+		if (ee->num_requests) {
 			err_printf(m, "%s --- %d requests\n",
 				   dev_priv->engine[i].name,
-				   error->ring[i].num_requests);
-			for (j = 0; j < error->ring[i].num_requests; j++) {
-				err_printf(m, "  seqno 0x%08x, emitted %ld, tail 0x%08x\n",
-					   error->ring[i].requests[j].seqno,
-					   error->ring[i].requests[j].jiffies,
-					   error->ring[i].requests[j].tail);
+				   ee->num_requests);
+			for (j = 0; j < ee->num_requests; j++) {
+				err_printf(m, "  pid %d, seqno 0x%08x, emitted %ld, head 0x%08x, tail 0x%08x\n",
+					   ee->requests[j].pid,
+					   ee->requests[j].seqno,
+					   ee->requests[j].jiffies,
+					   ee->requests[j].head,
+					   ee->requests[j].tail);
 			}
 		}
 
-		if (error->ring[i].num_waiters) {
+		if (IS_ERR(ee->waiters)) {
+			err_printf(m, "%s --- ? waiters [unable to acquire spinlock]\n",
+				   dev_priv->engine[i].name);
+		} else if (ee->num_waiters) {
 			err_printf(m, "%s --- %d waiters\n",
 				   dev_priv->engine[i].name,
-				   error->ring[i].num_waiters);
-			for (j = 0; j < error->ring[i].num_waiters; j++) {
+				   ee->num_waiters);
+			for (j = 0; j < ee->num_waiters; j++) {
 				err_printf(m, " seqno 0x%08x for %s [%d]\n",
-					   error->ring[i].waiters[j].seqno,
-					   error->ring[i].waiters[j].comm,
-					   error->ring[i].waiters[j].pid);
+					   ee->waiters[j].seqno,
+					   ee->waiters[j].comm,
+					   ee->waiters[j].pid);
 			}
 		}
 
-		if ((obj = error->ring[i].ringbuffer)) {
+		if ((obj = ee->ringbuffer)) {
 			err_printf(m, "%s --- ringbuffer = 0x%08x\n",
 				   dev_priv->engine[i].name,
 				   lower_32_bits(obj->gtt_offset));
 			print_error_obj(m, obj);
 		}
 
-		if ((obj = error->ring[i].hws_page)) {
+		if ((obj = ee->hws_page)) {
 			u64 hws_offset = obj->gtt_offset;
 			u32 *hws_page = &obj->pages[0][0];
 
@@ -504,7 +533,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			}
 		}
 
-		obj = error->ring[i].wa_ctx;
+		obj = ee->wa_ctx;
 		if (obj) {
 			u64 wa_ctx_offset = obj->gtt_offset;
 			u32 *wa_ctx_page = &obj->pages[0][0];
@@ -526,7 +555,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			}
 		}
 
-		if ((obj = error->ring[i].ctx)) {
+		if ((obj = ee->ctx)) {
 			err_printf(m, "%s --- HW Context = 0x%08x\n",
 				   dev_priv->engine[i].name,
 				   lower_32_bits(obj->gtt_offset));
@@ -534,7 +563,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 		}
 	}
 
-	if ((obj = error->semaphore_obj)) {
+	if ((obj = error->semaphore)) {
 		err_printf(m, "Semaphore page = 0x%08x\n",
 			   lower_32_bits(obj->gtt_offset));
 		for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
@@ -611,26 +640,27 @@ static void i915_error_state_free(struct kref *error_ref)
 							  typeof(*error), ref);
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
-		i915_error_object_free(error->ring[i].batchbuffer);
-		i915_error_object_free(error->ring[i].wa_batchbuffer);
-		i915_error_object_free(error->ring[i].ringbuffer);
-		i915_error_object_free(error->ring[i].hws_page);
-		i915_error_object_free(error->ring[i].ctx);
-		i915_error_object_free(error->ring[i].wa_ctx);
-		kfree(error->ring[i].requests);
-		kfree(error->ring[i].waiters);
+	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
+		struct drm_i915_error_engine *ee = &error->engine[i];
+
+		i915_error_object_free(ee->batchbuffer);
+		i915_error_object_free(ee->wa_batchbuffer);
+		i915_error_object_free(ee->ringbuffer);
+		i915_error_object_free(ee->hws_page);
+		i915_error_object_free(ee->ctx);
+		i915_error_object_free(ee->wa_ctx);
+
+		kfree(ee->requests);
+		if (!IS_ERR_OR_NULL(ee->waiters))
+			kfree(ee->waiters);
 	}
 
-	i915_error_object_free(error->semaphore_obj);
+	i915_error_object_free(error->semaphore);
 
-	for (i = 0; i < error->vm_count; i++)
+	for (i = 0; i < ARRAY_SIZE(error->active_bo); i++)
 		kfree(error->active_bo[i]);
-
-	kfree(error->active_bo);
-	kfree(error->active_bo_count);
 	kfree(error->pinned_bo);
-	kfree(error->pinned_bo_count);
+
 	kfree(error->overlay);
 	kfree(error->display);
 	kfree(error);
@@ -638,46 +668,45 @@ static void i915_error_state_free(struct kref *error_ref)
 
 static struct drm_i915_error_object *
 i915_error_object_create(struct drm_i915_private *dev_priv,
-			 struct drm_i915_gem_object *src,
-			 struct i915_address_space *vm)
+			 struct i915_vma *vma)
 {
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
+	struct drm_i915_gem_object *src;
 	struct drm_i915_error_object *dst;
-	struct i915_vma *vma = NULL;
 	int num_pages;
 	bool use_ggtt;
 	int i = 0;
 	u64 reloc_offset;
 
-	if (src == NULL || src->pages == NULL)
+	if (!vma)
+		return NULL;
+
+	src = vma->obj;
+	if (!src->pages)
 		return NULL;
 
 	num_pages = src->base.size >> PAGE_SHIFT;
 
 	dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), GFP_ATOMIC);
-	if (dst == NULL)
+	if (!dst)
 		return NULL;
 
-	if (i915_gem_obj_bound(src, vm))
-		dst->gtt_offset = i915_gem_obj_offset(src, vm);
-	else
-		dst->gtt_offset = -1;
+	dst->gtt_offset = vma->node.start;
+	dst->gtt_size = vma->node.size;
 
 	reloc_offset = dst->gtt_offset;
-	if (i915_is_ggtt(vm))
-		vma = i915_gem_obj_to_ggtt(src);
 	use_ggtt = (src->cache_level == I915_CACHE_NONE &&
-		   vma && (vma->bound & GLOBAL_BIND) &&
+		   (vma->flags & I915_VMA_GLOBAL_BIND) &&
 		   reloc_offset + num_pages * PAGE_SIZE <= ggtt->mappable_end);
 
 	/* Cannot access stolen address directly, try to use the aperture */
 	if (src->stolen) {
 		use_ggtt = true;
 
-		if (!(vma && vma->bound & GLOBAL_BIND))
+		if (!(vma->flags & I915_VMA_GLOBAL_BIND))
 			goto unwind;
 
-		reloc_offset = i915_gem_obj_ggtt_offset(src);
+		reloc_offset = vma->node.start;
 		if (reloc_offset + num_pages * PAGE_SIZE > ggtt->mappable_end)
 			goto unwind;
 	}
@@ -705,7 +734,7 @@ i915_error_object_create(struct drm_i915_private *dev_priv,
 			 * captures what the GPU read.
 			 */
 
-			s = io_mapping_map_atomic_wc(ggtt->mappable,
+			s = io_mapping_map_atomic_wc(&ggtt->mappable,
 						     reloc_offset);
 			memcpy_fromio(d, s, PAGE_SIZE);
 			io_mapping_unmap_atomic(s);
@@ -737,8 +766,24 @@ unwind:
 	kfree(dst);
 	return NULL;
 }
-#define i915_error_ggtt_object_create(dev_priv, src) \
-	i915_error_object_create((dev_priv), (src), &(dev_priv)->ggtt.base)
+
+/* The error capture is special as tries to run underneath the normal
+ * locking rules - so we use the raw version of the i915_gem_active lookup.
+ */
+static inline uint32_t
+__active_get_seqno(struct i915_gem_active *active)
+{
+	return i915_gem_request_get_seqno(__i915_gem_active_peek(active));
+}
+
+static inline int
+__active_get_engine_id(struct i915_gem_active *active)
+{
+	struct intel_engine_cs *engine;
+
+	engine = i915_gem_request_get_engine(__i915_gem_active_peek(active));
+	return engine ? engine->id : -1;
+}
 
 static void capture_bo(struct drm_i915_error_buffer *err,
 		       struct i915_vma *vma)
@@ -748,32 +793,34 @@ static void capture_bo(struct drm_i915_error_buffer *err,
 
 	err->size = obj->base.size;
 	err->name = obj->base.name;
+
 	for (i = 0; i < I915_NUM_ENGINES; i++)
-		err->rseqno[i] = i915_gem_request_get_seqno(obj->last_read_req[i]);
-	err->wseqno = i915_gem_request_get_seqno(obj->last_write_req);
+		err->rseqno[i] = __active_get_seqno(&obj->last_read[i]);
+	err->wseqno = __active_get_seqno(&obj->last_write);
+	err->engine = __active_get_engine_id(&obj->last_write);
+
 	err->gtt_offset = vma->node.start;
 	err->read_domains = obj->base.read_domains;
 	err->write_domain = obj->base.write_domain;
-	err->fence_reg = obj->fence_reg;
-	err->pinned = 0;
-	if (i915_gem_obj_is_pinned(obj))
-		err->pinned = 1;
-	err->tiling = obj->tiling_mode;
+	err->fence_reg = vma->fence ? vma->fence->id : -1;
+	err->tiling = i915_gem_object_get_tiling(obj);
 	err->dirty = obj->dirty;
 	err->purgeable = obj->madv != I915_MADV_WILLNEED;
 	err->userptr = obj->userptr.mm != NULL;
-	err->ring = obj->last_write_req ?
-			i915_gem_request_get_engine(obj->last_write_req)->id : -1;
 	err->cache_level = obj->cache_level;
 }
 
-static u32 capture_active_bo(struct drm_i915_error_buffer *err,
-			     int count, struct list_head *head)
+static u32 capture_error_bo(struct drm_i915_error_buffer *err,
+			    int count, struct list_head *head,
+			    bool pinned_only)
 {
 	struct i915_vma *vma;
 	int i = 0;
 
 	list_for_each_entry(vma, head, vm_link) {
+		if (pinned_only && !i915_vma_is_pinned(vma))
+			continue;
+
 		capture_bo(err++, vma);
 		if (++i == count)
 			break;
@@ -782,28 +829,6 @@ static u32 capture_active_bo(struct drm_i915_error_buffer *err,
 	return i;
 }
 
-static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
-			     int count, struct list_head *head,
-			     struct i915_address_space *vm)
-{
-	struct drm_i915_gem_object *obj;
-	struct drm_i915_error_buffer * const first = err;
-	struct drm_i915_error_buffer * const last = err + count;
-
-	list_for_each_entry(obj, head, global_list) {
-		struct i915_vma *vma;
-
-		if (err == last)
-			break;
-
-		list_for_each_entry(vma, &obj->vma_list, obj_link)
-			if (vma->vm == vm && vma->pin_count > 0)
-				capture_bo(err++, vma);
-	}
-
-	return err - first;
-}
-
 /* Generate a semi-unique error code. The code is not meant to have meaning, The
  * code's only purpose is to try to prevent false duplicated bug reports by
  * grossly estimating a GPU error state.
@@ -815,7 +840,7 @@ static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
  */
 static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
 					 struct drm_i915_error_state *error,
-					 int *ring_id)
+					 int *engine_id)
 {
 	uint32_t error_code = 0;
 	int i;
@@ -826,11 +851,11 @@ static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
 	 * strictly a client bug. Use instdone to differentiate those some.
 	 */
 	for (i = 0; i < I915_NUM_ENGINES; i++) {
-		if (error->ring[i].hangcheck_action == HANGCHECK_HUNG) {
-			if (ring_id)
-				*ring_id = i;
+		if (error->engine[i].hangcheck_action == HANGCHECK_HUNG) {
+			if (engine_id)
+				*engine_id = i;
 
-			return error->ring[i].ipehr ^ error->ring[i].instdone;
+			return error->engine[i].ipehr ^ error->engine[i].instdone;
 		}
 	}
 
@@ -855,22 +880,17 @@ static void i915_gem_record_fences(struct drm_i915_private *dev_priv,
 }
 
 
-static void gen8_record_semaphore_state(struct drm_i915_private *dev_priv,
-					struct drm_i915_error_state *error,
+static void gen8_record_semaphore_state(struct drm_i915_error_state *error,
 					struct intel_engine_cs *engine,
-					struct drm_i915_error_ring *ering)
+					struct drm_i915_error_engine *ee)
 {
+	struct drm_i915_private *dev_priv = engine->i915;
 	struct intel_engine_cs *to;
 	enum intel_engine_id id;
 
-	if (!i915_semaphore_is_enabled(dev_priv))
+	if (!error->semaphore)
 		return;
 
-	if (!error->semaphore_obj)
-		error->semaphore_obj =
-			i915_error_ggtt_object_create(dev_priv,
-						      dev_priv->semaphore_obj);
-
 	for_each_engine_id(to, dev_priv, id) {
 		int idx;
 		u16 signal_offset;
@@ -879,44 +899,52 @@ static void gen8_record_semaphore_state(struct drm_i915_private *dev_priv,
 		if (engine == to)
 			continue;
 
-		signal_offset = (GEN8_SIGNAL_OFFSET(engine, id) & (PAGE_SIZE - 1))
-				/ 4;
-		tmp = error->semaphore_obj->pages[0];
-		idx = intel_ring_sync_index(engine, to);
+		signal_offset =
+			(GEN8_SIGNAL_OFFSET(engine, id) & (PAGE_SIZE - 1)) / 4;
+		tmp = error->semaphore->pages[0];
+		idx = intel_engine_sync_index(engine, to);
 
-		ering->semaphore_mboxes[idx] = tmp[signal_offset];
-		ering->semaphore_seqno[idx] = engine->semaphore.sync_seqno[idx];
+		ee->semaphore_mboxes[idx] = tmp[signal_offset];
+		ee->semaphore_seqno[idx] = engine->semaphore.sync_seqno[idx];
 	}
 }
 
-static void gen6_record_semaphore_state(struct drm_i915_private *dev_priv,
-					struct intel_engine_cs *engine,
-					struct drm_i915_error_ring *ering)
+static void gen6_record_semaphore_state(struct intel_engine_cs *engine,
+					struct drm_i915_error_engine *ee)
 {
-	ering->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base));
-	ering->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base));
-	ering->semaphore_seqno[0] = engine->semaphore.sync_seqno[0];
-	ering->semaphore_seqno[1] = engine->semaphore.sync_seqno[1];
+	struct drm_i915_private *dev_priv = engine->i915;
+
+	ee->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base));
+	ee->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base));
+	ee->semaphore_seqno[0] = engine->semaphore.sync_seqno[0];
+	ee->semaphore_seqno[1] = engine->semaphore.sync_seqno[1];
 
 	if (HAS_VEBOX(dev_priv)) {
-		ering->semaphore_mboxes[2] =
+		ee->semaphore_mboxes[2] =
 			I915_READ(RING_SYNC_2(engine->mmio_base));
-		ering->semaphore_seqno[2] = engine->semaphore.sync_seqno[2];
+		ee->semaphore_seqno[2] = engine->semaphore.sync_seqno[2];
 	}
 }
 
-static void engine_record_waiters(struct intel_engine_cs *engine,
-				  struct drm_i915_error_ring *ering)
+static void error_record_engine_waiters(struct intel_engine_cs *engine,
+					struct drm_i915_error_engine *ee)
 {
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 	struct drm_i915_error_waiter *waiter;
 	struct rb_node *rb;
 	int count;
 
-	ering->num_waiters = 0;
-	ering->waiters = NULL;
+	ee->num_waiters = 0;
+	ee->waiters = NULL;
+
+	if (RB_EMPTY_ROOT(&b->waiters))
+		return;
+
+	if (!spin_trylock(&b->lock)) {
+		ee->waiters = ERR_PTR(-EDEADLK);
+		return;
+	}
 
-	spin_lock(&b->lock);
 	count = 0;
 	for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb))
 		count++;
@@ -930,9 +958,13 @@ static void engine_record_waiters(struct intel_engine_cs *engine,
 	if (!waiter)
 		return;
 
-	ering->waiters = waiter;
+	if (!spin_trylock(&b->lock)) {
+		kfree(waiter);
+		ee->waiters = ERR_PTR(-EDEADLK);
+		return;
+	}
 
-	spin_lock(&b->lock);
+	ee->waiters = waiter;
 	for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
 		struct intel_wait *w = container_of(rb, typeof(*w), node);
 
@@ -941,57 +973,59 @@ static void engine_record_waiters(struct intel_engine_cs *engine,
 		waiter->seqno = w->seqno;
 		waiter++;
 
-		if (++ering->num_waiters == count)
+		if (++ee->num_waiters == count)
 			break;
 	}
 	spin_unlock(&b->lock);
 }
 
-static void i915_record_ring_state(struct drm_i915_private *dev_priv,
-				   struct drm_i915_error_state *error,
-				   struct intel_engine_cs *engine,
-				   struct drm_i915_error_ring *ering)
+static void error_record_engine_registers(struct drm_i915_error_state *error,
+					  struct intel_engine_cs *engine,
+					  struct drm_i915_error_engine *ee)
 {
+	struct drm_i915_private *dev_priv = engine->i915;
+
 	if (INTEL_GEN(dev_priv) >= 6) {
-		ering->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base));
-		ering->fault_reg = I915_READ(RING_FAULT_REG(engine));
+		ee->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base));
+		ee->fault_reg = I915_READ(RING_FAULT_REG(engine));
 		if (INTEL_GEN(dev_priv) >= 8)
-			gen8_record_semaphore_state(dev_priv, error, engine,
-						    ering);
+			gen8_record_semaphore_state(error, engine, ee);
 		else
-			gen6_record_semaphore_state(dev_priv, engine, ering);
+			gen6_record_semaphore_state(engine, ee);
 	}
 
 	if (INTEL_GEN(dev_priv) >= 4) {
-		ering->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base));
-		ering->ipeir = I915_READ(RING_IPEIR(engine->mmio_base));
-		ering->ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
-		ering->instdone = I915_READ(RING_INSTDONE(engine->mmio_base));
-		ering->instps = I915_READ(RING_INSTPS(engine->mmio_base));
-		ering->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
+		ee->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base));
+		ee->ipeir = I915_READ(RING_IPEIR(engine->mmio_base));
+		ee->ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
+		ee->instdone = I915_READ(RING_INSTDONE(engine->mmio_base));
+		ee->instps = I915_READ(RING_INSTPS(engine->mmio_base));
+		ee->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
 		if (INTEL_GEN(dev_priv) >= 8) {
-			ering->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(engine->mmio_base)) << 32;
-			ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(engine->mmio_base)) << 32;
+			ee->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(engine->mmio_base)) << 32;
+			ee->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(engine->mmio_base)) << 32;
 		}
-		ering->bbstate = I915_READ(RING_BBSTATE(engine->mmio_base));
+		ee->bbstate = I915_READ(RING_BBSTATE(engine->mmio_base));
 	} else {
-		ering->faddr = I915_READ(DMA_FADD_I8XX);
-		ering->ipeir = I915_READ(IPEIR);
-		ering->ipehr = I915_READ(IPEHR);
-		ering->instdone = I915_READ(GEN2_INSTDONE);
+		ee->faddr = I915_READ(DMA_FADD_I8XX);
+		ee->ipeir = I915_READ(IPEIR);
+		ee->ipehr = I915_READ(IPEHR);
+		ee->instdone = I915_READ(GEN2_INSTDONE);
 	}
 
-	ering->waiting = intel_engine_has_waiter(engine);
-	ering->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
-	ering->acthd = intel_ring_get_active_head(engine);
-	ering->seqno = intel_engine_get_seqno(engine);
-	ering->last_seqno = engine->last_submitted_seqno;
-	ering->start = I915_READ_START(engine);
-	ering->head = I915_READ_HEAD(engine);
-	ering->tail = I915_READ_TAIL(engine);
-	ering->ctl = I915_READ_CTL(engine);
-
-	if (I915_NEED_GFX_HWS(dev_priv)) {
+	ee->waiting = intel_engine_has_waiter(engine);
+	ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
+	ee->acthd = intel_engine_get_active_head(engine);
+	ee->seqno = intel_engine_get_seqno(engine);
+	ee->last_seqno = engine->last_submitted_seqno;
+	ee->start = I915_READ_START(engine);
+	ee->head = I915_READ_HEAD(engine);
+	ee->tail = I915_READ_TAIL(engine);
+	ee->ctl = I915_READ_CTL(engine);
+	if (INTEL_GEN(dev_priv) > 2)
+		ee->mode = I915_READ_MODE(engine);
+
+	if (!HWS_NEEDS_PHYSICAL(dev_priv)) {
 		i915_reg_t mmio;
 
 		if (IS_GEN7(dev_priv)) {
@@ -1017,107 +1051,150 @@ static void i915_record_ring_state(struct drm_i915_private *dev_priv,
 			mmio = RING_HWS_PGA(engine->mmio_base);
 		}
 
-		ering->hws = I915_READ(mmio);
+		ee->hws = I915_READ(mmio);
 	}
 
-	ering->hangcheck_score = engine->hangcheck.score;
-	ering->hangcheck_action = engine->hangcheck.action;
+	ee->hangcheck_score = engine->hangcheck.score;
+	ee->hangcheck_action = engine->hangcheck.action;
 
 	if (USES_PPGTT(dev_priv)) {
 		int i;
 
-		ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine));
+		ee->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine));
 
 		if (IS_GEN6(dev_priv))
-			ering->vm_info.pp_dir_base =
+			ee->vm_info.pp_dir_base =
 				I915_READ(RING_PP_DIR_BASE_READ(engine));
 		else if (IS_GEN7(dev_priv))
-			ering->vm_info.pp_dir_base =
+			ee->vm_info.pp_dir_base =
 				I915_READ(RING_PP_DIR_BASE(engine));
 		else if (INTEL_GEN(dev_priv) >= 8)
 			for (i = 0; i < 4; i++) {
-				ering->vm_info.pdp[i] =
+				ee->vm_info.pdp[i] =
 					I915_READ(GEN8_RING_PDP_UDW(engine, i));
-				ering->vm_info.pdp[i] <<= 32;
-				ering->vm_info.pdp[i] |=
+				ee->vm_info.pdp[i] <<= 32;
+				ee->vm_info.pdp[i] |=
 					I915_READ(GEN8_RING_PDP_LDW(engine, i));
 			}
 	}
 }
 
-
-static void i915_gem_record_active_context(struct intel_engine_cs *engine,
-					   struct drm_i915_error_state *error,
-					   struct drm_i915_error_ring *ering)
+static void engine_record_requests(struct intel_engine_cs *engine,
+				   struct drm_i915_gem_request *first,
+				   struct drm_i915_error_engine *ee)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	struct drm_i915_gem_object *obj;
+	struct drm_i915_gem_request *request;
+	int count;
 
-	/* Currently render ring is the only HW context user */
-	if (engine->id != RCS || !error->ccid)
+	count = 0;
+	request = first;
+	list_for_each_entry_from(request, &engine->request_list, link)
+		count++;
+	if (!count)
 		return;
 
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-		if (!i915_gem_obj_ggtt_bound(obj))
-			continue;
+	ee->requests = kcalloc(count, sizeof(*ee->requests), GFP_ATOMIC);
+	if (!ee->requests)
+		return;
 
-		if ((error->ccid & PAGE_MASK) == i915_gem_obj_ggtt_offset(obj)) {
-			ering->ctx = i915_error_ggtt_object_create(dev_priv, obj);
+	ee->num_requests = count;
+
+	count = 0;
+	request = first;
+	list_for_each_entry_from(request, &engine->request_list, link) {
+		struct drm_i915_error_request *erq;
+
+		if (count >= ee->num_requests) {
+			/*
+			 * If the ring request list was changed in
+			 * between the point where the error request
+			 * list was created and dimensioned and this
+			 * point then just exit early to avoid crashes.
+			 *
+			 * We don't need to communicate that the
+			 * request list changed state during error
+			 * state capture and that the error state is
+			 * slightly incorrect as a consequence since we
+			 * are typically only interested in the request
+			 * list state at the point of error state
+			 * capture, not in any changes happening during
+			 * the capture.
+			 */
 			break;
 		}
+
+		erq = &ee->requests[count++];
+		erq->seqno = request->fence.seqno;
+		erq->jiffies = request->emitted_jiffies;
+		erq->head = request->head;
+		erq->tail = request->tail;
+
+		rcu_read_lock();
+		erq->pid = request->ctx->pid ? pid_nr(request->ctx->pid) : 0;
+		rcu_read_unlock();
 	}
+	ee->num_requests = count;
 }
 
 static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
 				  struct drm_i915_error_state *error)
 {
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct drm_i915_gem_request *request;
-	int i, count;
+	int i;
+
+	error->semaphore =
+		i915_error_object_create(dev_priv, dev_priv->semaphore);
 
 	for (i = 0; i < I915_NUM_ENGINES; i++) {
 		struct intel_engine_cs *engine = &dev_priv->engine[i];
+		struct drm_i915_error_engine *ee = &error->engine[i];
+		struct drm_i915_gem_request *request;
 
-		error->ring[i].pid = -1;
+		ee->pid = -1;
+		ee->engine_id = -1;
 
 		if (!intel_engine_initialized(engine))
 			continue;
 
-		error->ring[i].valid = true;
+		ee->engine_id = i;
 
-		i915_record_ring_state(dev_priv, error, engine, &error->ring[i]);
-		engine_record_waiters(engine, &error->ring[i]);
+		error_record_engine_registers(error, engine, ee);
+		error_record_engine_waiters(engine, ee);
 
 		request = i915_gem_find_active_request(engine);
 		if (request) {
-			struct i915_address_space *vm;
-			struct intel_ringbuffer *rb;
+			struct intel_ring *ring;
+			struct pid *pid;
 
-			vm = request->ctx->ppgtt ?
+			ee->vm = request->ctx->ppgtt ?
 				&request->ctx->ppgtt->base : &ggtt->base;
 
 			/* We need to copy these to an anonymous buffer
 			 * as the simplest method to avoid being overwritten
 			 * by userspace.
 			 */
-			error->ring[i].batchbuffer =
+			ee->batchbuffer =
 				i915_error_object_create(dev_priv,
-							 request->batch_obj,
-							 vm);
+							 request->batch);
 
 			if (HAS_BROKEN_CS_TLB(dev_priv))
-				error->ring[i].wa_batchbuffer =
-					i915_error_ggtt_object_create(dev_priv,
-							     engine->scratch.obj);
+				ee->wa_batchbuffer =
+					i915_error_object_create(dev_priv,
+								 engine->scratch);
+
+			ee->ctx =
+				i915_error_object_create(dev_priv,
+							 request->ctx->engine[i].state);
 
-			if (request->pid) {
+			pid = request->ctx->pid;
+			if (pid) {
 				struct task_struct *task;
 
 				rcu_read_lock();
-				task = pid_task(request->pid, PIDTYPE_PID);
+				task = pid_task(pid, PIDTYPE_PID);
 				if (task) {
-					strcpy(error->ring[i].comm, task->comm);
-					error->ring[i].pid = task->pid;
+					strcpy(ee->comm, task->comm);
+					ee->pid = task->pid;
 				}
 				rcu_read_unlock();
 			}
@@ -1125,153 +1202,106 @@ static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
 			error->simulated |=
 				request->ctx->flags & CONTEXT_NO_ERROR_CAPTURE;
 
-			rb = request->ringbuf;
-			error->ring[i].cpu_ring_head = rb->head;
-			error->ring[i].cpu_ring_tail = rb->tail;
-			error->ring[i].ringbuffer =
-				i915_error_ggtt_object_create(dev_priv,
-							      rb->obj);
-		}
-
-		error->ring[i].hws_page =
-			i915_error_ggtt_object_create(dev_priv,
-						      engine->status_page.obj);
+			ring = request->ring;
+			ee->cpu_ring_head = ring->head;
+			ee->cpu_ring_tail = ring->tail;
+			ee->ringbuffer =
+				i915_error_object_create(dev_priv, ring->vma);
 
-		if (engine->wa_ctx.obj) {
-			error->ring[i].wa_ctx =
-				i915_error_ggtt_object_create(dev_priv,
-							      engine->wa_ctx.obj);
+			engine_record_requests(engine, request, ee);
 		}
 
-		i915_gem_record_active_context(engine, error, &error->ring[i]);
-
-		count = 0;
-		list_for_each_entry(request, &engine->request_list, list)
-			count++;
-
-		error->ring[i].num_requests = count;
-		error->ring[i].requests =
-			kcalloc(count, sizeof(*error->ring[i].requests),
-				GFP_ATOMIC);
-		if (error->ring[i].requests == NULL) {
-			error->ring[i].num_requests = 0;
-			continue;
-		}
-
-		count = 0;
-		list_for_each_entry(request, &engine->request_list, list) {
-			struct drm_i915_error_request *erq;
-
-			if (count >= error->ring[i].num_requests) {
-				/*
-				 * If the ring request list was changed in
-				 * between the point where the error request
-				 * list was created and dimensioned and this
-				 * point then just exit early to avoid crashes.
-				 *
-				 * We don't need to communicate that the
-				 * request list changed state during error
-				 * state capture and that the error state is
-				 * slightly incorrect as a consequence since we
-				 * are typically only interested in the request
-				 * list state at the point of error state
-				 * capture, not in any changes happening during
-				 * the capture.
-				 */
-				break;
-			}
+		ee->hws_page =
+			i915_error_object_create(dev_priv,
+						 engine->status_page.vma);
 
-			erq = &error->ring[i].requests[count++];
-			erq->seqno = request->seqno;
-			erq->jiffies = request->emitted_jiffies;
-			erq->tail = request->postfix;
-		}
+		ee->wa_ctx =
+			i915_error_object_create(dev_priv, engine->wa_ctx.vma);
 	}
 }
 
-/* FIXME: Since pin count/bound list is global, we duplicate what we capture per
- * VM.
- */
 static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
 				struct drm_i915_error_state *error,
 				struct i915_address_space *vm,
-				const int ndx)
+				int idx)
 {
-	struct drm_i915_error_buffer *active_bo = NULL, *pinned_bo = NULL;
-	struct drm_i915_gem_object *obj;
+	struct drm_i915_error_buffer *active_bo;
 	struct i915_vma *vma;
-	int i;
+	int count;
 
-	i = 0;
+	count = 0;
 	list_for_each_entry(vma, &vm->active_list, vm_link)
-		i++;
-	error->active_bo_count[ndx] = i;
-
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-		list_for_each_entry(vma, &obj->vma_list, obj_link)
-			if (vma->vm == vm && vma->pin_count > 0)
-				i++;
-	}
-	error->pinned_bo_count[ndx] = i - error->active_bo_count[ndx];
-
-	if (i) {
-		active_bo = kcalloc(i, sizeof(*active_bo), GFP_ATOMIC);
-		if (active_bo)
-			pinned_bo = active_bo + error->active_bo_count[ndx];
-	}
+		count++;
 
+	active_bo = NULL;
+	if (count)
+		active_bo = kcalloc(count, sizeof(*active_bo), GFP_ATOMIC);
 	if (active_bo)
-		error->active_bo_count[ndx] =
-			capture_active_bo(active_bo,
-					  error->active_bo_count[ndx],
-					  &vm->active_list);
-
-	if (pinned_bo)
-		error->pinned_bo_count[ndx] =
-			capture_pinned_bo(pinned_bo,
-					  error->pinned_bo_count[ndx],
-					  &dev_priv->mm.bound_list, vm);
-	error->active_bo[ndx] = active_bo;
-	error->pinned_bo[ndx] = pinned_bo;
+		count = capture_error_bo(active_bo, count, &vm->active_list, false);
+	else
+		count = 0;
+
+	error->active_vm[idx] = vm;
+	error->active_bo[idx] = active_bo;
+	error->active_bo_count[idx] = count;
 }
 
-static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv,
-				     struct drm_i915_error_state *error)
+static void i915_capture_active_buffers(struct drm_i915_private *dev_priv,
+					struct drm_i915_error_state *error)
 {
-	struct i915_address_space *vm;
-	int cnt = 0, i = 0;
-
-	list_for_each_entry(vm, &dev_priv->vm_list, global_link)
-		cnt++;
-
-	error->active_bo = kcalloc(cnt, sizeof(*error->active_bo), GFP_ATOMIC);
-	error->pinned_bo = kcalloc(cnt, sizeof(*error->pinned_bo), GFP_ATOMIC);
-	error->active_bo_count = kcalloc(cnt, sizeof(*error->active_bo_count),
-					 GFP_ATOMIC);
-	error->pinned_bo_count = kcalloc(cnt, sizeof(*error->pinned_bo_count),
-					 GFP_ATOMIC);
-
-	if (error->active_bo == NULL ||
-	    error->pinned_bo == NULL ||
-	    error->active_bo_count == NULL ||
-	    error->pinned_bo_count == NULL) {
-		kfree(error->active_bo);
-		kfree(error->active_bo_count);
-		kfree(error->pinned_bo);
-		kfree(error->pinned_bo_count);
-
-		error->active_bo = NULL;
-		error->active_bo_count = NULL;
-		error->pinned_bo = NULL;
-		error->pinned_bo_count = NULL;
-	} else {
-		list_for_each_entry(vm, &dev_priv->vm_list, global_link)
-			i915_gem_capture_vm(dev_priv, error, vm, i++);
+	int cnt = 0, i, j;
 
-		error->vm_count = cnt;
+	BUILD_BUG_ON(ARRAY_SIZE(error->engine) > ARRAY_SIZE(error->active_bo));
+	BUILD_BUG_ON(ARRAY_SIZE(error->active_bo) != ARRAY_SIZE(error->active_vm));
+	BUILD_BUG_ON(ARRAY_SIZE(error->active_bo) != ARRAY_SIZE(error->active_bo_count));
+
+	/* Scan each engine looking for unique active contexts/vm */
+	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
+		struct drm_i915_error_engine *ee = &error->engine[i];
+		bool found;
+
+		if (!ee->vm)
+			continue;
+
+		found = false;
+		for (j = 0; j < i && !found; j++)
+			found = error->engine[j].vm == ee->vm;
+		if (!found)
+			i915_gem_capture_vm(dev_priv, error, ee->vm, cnt++);
 	}
 }
 
+static void i915_capture_pinned_buffers(struct drm_i915_private *dev_priv,
+					struct drm_i915_error_state *error)
+{
+	struct i915_address_space *vm = &dev_priv->ggtt.base;
+	struct drm_i915_error_buffer *bo;
+	struct i915_vma *vma;
+	int count_inactive, count_active;
+
+	count_inactive = 0;
+	list_for_each_entry(vma, &vm->active_list, vm_link)
+		count_inactive++;
+
+	count_active = 0;
+	list_for_each_entry(vma, &vm->inactive_list, vm_link)
+		count_active++;
+
+	bo = NULL;
+	if (count_inactive + count_active)
+		bo = kcalloc(count_inactive + count_active,
+			     sizeof(*bo), GFP_ATOMIC);
+	if (!bo)
+		return;
+
+	count_inactive = capture_error_bo(bo, count_inactive,
+					  &vm->active_list, true);
+	count_active = capture_error_bo(bo + count_inactive, count_active,
+					&vm->inactive_list, true);
+	error->pinned_bo_count = count_inactive + count_active;
+	error->pinned_bo = bo;
+}
+
 /* Capture all registers which don't fit into another category. */
 static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
 				   struct drm_i915_error_state *error)
@@ -1352,20 +1382,20 @@ static void i915_error_capture_msg(struct drm_i915_private *dev_priv,
 				   const char *error_msg)
 {
 	u32 ecode;
-	int ring_id = -1, len;
+	int engine_id = -1, len;
 
-	ecode = i915_error_generate_code(dev_priv, error, &ring_id);
+	ecode = i915_error_generate_code(dev_priv, error, &engine_id);
 
 	len = scnprintf(error->error_msg, sizeof(error->error_msg),
 			"GPU HANG: ecode %d:%d:0x%08x",
-			INTEL_GEN(dev_priv), ring_id, ecode);
+			INTEL_GEN(dev_priv), engine_id, ecode);
 
-	if (ring_id != -1 && error->ring[ring_id].pid != -1)
+	if (engine_id != -1 && error->engine[engine_id].pid != -1)
 		len += scnprintf(error->error_msg + len,
 				 sizeof(error->error_msg) - len,
 				 ", in %s [%d]",
-				 error->ring[ring_id].comm,
-				 error->ring[ring_id].pid);
+				 error->engine[engine_id].comm,
+				 error->engine[engine_id].pid);
 
 	scnprintf(error->error_msg + len, sizeof(error->error_msg) - len,
 		  ", reason: %s, action: %s",
@@ -1382,6 +1412,10 @@ static void i915_capture_gen_state(struct drm_i915_private *dev_priv,
 #endif
 	error->reset_count = i915_reset_count(&dev_priv->gpu_error);
 	error->suspend_count = dev_priv->suspend_count;
+
+	memcpy(&error->device_info,
+	       INTEL_INFO(dev_priv),
+	       sizeof(error->device_info));
 }
 
 /**
@@ -1415,9 +1449,10 @@ void i915_capture_error_state(struct drm_i915_private *dev_priv,
 
 	i915_capture_gen_state(dev_priv, error);
 	i915_capture_reg_state(dev_priv, error);
-	i915_gem_capture_buffers(dev_priv, error);
 	i915_gem_record_fences(dev_priv, error);
 	i915_gem_record_rings(dev_priv, error);
+	i915_capture_active_buffers(dev_priv, error);
+	i915_capture_pinned_buffers(dev_priv, error);
 
 	do_gettimeofday(&error->time);
 
diff --git a/drivers/gpu/drm/i915/i915_guc_reg.h b/drivers/gpu/drm/i915/i915_guc_reg.h
index cf5a65be4fe0..a47e1e4aec03 100644
--- a/drivers/gpu/drm/i915/i915_guc_reg.h
+++ b/drivers/gpu/drm/i915/i915_guc_reg.h
@@ -103,9 +103,6 @@
 #define HOST2GUC_INTERRUPT		_MMIO(0xc4c8)
 #define   HOST2GUC_TRIGGER		  (1<<0)
 
-#define DRBMISC1			0x1984
-#define   DOORBELL_ENABLE		  (1<<0)
-
 #define GEN8_DRBREGL(x)			_MMIO(0x1000 + (x) * 8)
 #define   GEN8_DRB_VALID		  (1<<0)
 #define GEN8_DRBREGU(x)			_MMIO(0x1000 + (x) * 8 + 4)
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c
index 2112e029db6a..3106dcc06fe9 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.c
+++ b/drivers/gpu/drm/i915/i915_guc_submission.c
@@ -59,7 +59,7 @@
  * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which
  * represents in-order queue. The kernel driver packs ring tail pointer and an
  * ELSP context descriptor dword into Work Item.
- * See guc_add_workqueue_item()
+ * See guc_wq_item_append()
  *
  */
 
@@ -114,10 +114,8 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len)
 		if (ret != -ETIMEDOUT)
 			ret = -EIO;
 
-		DRM_ERROR("GUC: host2guc action 0x%X failed. ret=%d "
-				"status=0x%08X response=0x%08X\n",
-				data[0], ret, status,
-				I915_READ(SOFT_SCRATCH(15)));
+		DRM_WARN("Action 0x%X failed; ret=%d status=0x%08X response=0x%08X\n",
+			 data[0], ret, status, I915_READ(SOFT_SCRATCH(15)));
 
 		dev_priv->guc.action_fail += 1;
 		dev_priv->guc.action_err = ret;
@@ -183,7 +181,7 @@ static int guc_update_doorbell_id(struct intel_guc *guc,
 				  struct i915_guc_client *client,
 				  u16 new_id)
 {
-	struct sg_table *sg = guc->ctx_pool_obj->pages;
+	struct sg_table *sg = guc->ctx_pool_vma->pages;
 	void *doorbell_bitmap = guc->doorbell_bitmap;
 	struct guc_doorbell_info *doorbell;
 	struct guc_context_desc desc;
@@ -290,7 +288,7 @@ static uint32_t select_doorbell_cacheline(struct intel_guc *guc)
 /*
  * Initialise the process descriptor shared with the GuC firmware.
  */
-static void guc_init_proc_desc(struct intel_guc *guc,
+static void guc_proc_desc_init(struct intel_guc *guc,
 			       struct i915_guc_client *client)
 {
 	struct guc_process_desc *desc;
@@ -322,15 +320,15 @@ static void guc_init_proc_desc(struct intel_guc *guc,
  * write queue, etc).
  */
 
-static void guc_init_ctx_desc(struct intel_guc *guc,
+static void guc_ctx_desc_init(struct intel_guc *guc,
 			      struct i915_guc_client *client)
 {
-	struct drm_i915_gem_object *client_obj = client->client_obj;
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 	struct intel_engine_cs *engine;
 	struct i915_gem_context *ctx = client->owner;
 	struct guc_context_desc desc;
 	struct sg_table *sg;
+	unsigned int tmp;
 	u32 gfx_addr;
 
 	memset(&desc, 0, sizeof(desc));
@@ -340,10 +338,10 @@ static void guc_init_ctx_desc(struct intel_guc *guc,
 	desc.priority = client->priority;
 	desc.db_id = client->doorbell_id;
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine_masked(engine, dev_priv, client->engines, tmp) {
 		struct intel_context *ce = &ctx->engine[engine->id];
-		struct guc_execlist_context *lrc = &desc.lrc[engine->guc_id];
-		struct drm_i915_gem_object *obj;
+		uint32_t guc_engine_id = engine->guc_id;
+		struct guc_execlist_context *lrc = &desc.lrc[guc_engine_id];
 
 		/* TODO: We have a design issue to be solved here. Only when we
 		 * receive the first batch, we know which engine is used by the
@@ -358,30 +356,29 @@ static void guc_init_ctx_desc(struct intel_guc *guc,
 		lrc->context_desc = lower_32_bits(ce->lrc_desc);
 
 		/* The state page is after PPHWSP */
-		gfx_addr = i915_gem_obj_ggtt_offset(ce->state);
-		lrc->ring_lcra = gfx_addr + LRC_STATE_PN * PAGE_SIZE;
+		lrc->ring_lcra =
+			i915_ggtt_offset(ce->state) + LRC_STATE_PN * PAGE_SIZE;
 		lrc->context_id = (client->ctx_index << GUC_ELC_CTXID_OFFSET) |
-				(engine->guc_id << GUC_ELC_ENGINE_OFFSET);
-
-		obj = ce->ringbuf->obj;
-		gfx_addr = i915_gem_obj_ggtt_offset(obj);
+				(guc_engine_id << GUC_ELC_ENGINE_OFFSET);
 
-		lrc->ring_begin = gfx_addr;
-		lrc->ring_end = gfx_addr + obj->base.size - 1;
-		lrc->ring_next_free_location = gfx_addr;
+		lrc->ring_begin = i915_ggtt_offset(ce->ring->vma);
+		lrc->ring_end = lrc->ring_begin + ce->ring->size - 1;
+		lrc->ring_next_free_location = lrc->ring_begin;
 		lrc->ring_current_tail_pointer_value = 0;
 
-		desc.engines_used |= (1 << engine->guc_id);
+		desc.engines_used |= (1 << guc_engine_id);
 	}
 
+	DRM_DEBUG_DRIVER("Host engines 0x%x => GuC engines used 0x%x\n",
+			client->engines, desc.engines_used);
 	WARN_ON(desc.engines_used == 0);
 
 	/*
 	 * The doorbell, process descriptor, and workqueue are all parts
 	 * of the client object, which the GuC will reference via the GGTT
 	 */
-	gfx_addr = i915_gem_obj_ggtt_offset(client_obj);
-	desc.db_trigger_phy = sg_dma_address(client_obj->pages->sgl) +
+	gfx_addr = i915_ggtt_offset(client->vma);
+	desc.db_trigger_phy = sg_dma_address(client->vma->pages->sgl) +
 				client->doorbell_offset;
 	desc.db_trigger_cpu = (uintptr_t)client->client_base +
 				client->doorbell_offset;
@@ -397,12 +394,12 @@ static void guc_init_ctx_desc(struct intel_guc *guc,
 	desc.desc_private = (uintptr_t)client;
 
 	/* Pool context is pinned already */
-	sg = guc->ctx_pool_obj->pages;
+	sg = guc->ctx_pool_vma->pages;
 	sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
 			     sizeof(desc) * client->ctx_index);
 }
 
-static void guc_fini_ctx_desc(struct intel_guc *guc,
+static void guc_ctx_desc_fini(struct intel_guc *guc,
 			      struct i915_guc_client *client)
 {
 	struct guc_context_desc desc;
@@ -410,13 +407,13 @@ static void guc_fini_ctx_desc(struct intel_guc *guc,
 
 	memset(&desc, 0, sizeof(desc));
 
-	sg = guc->ctx_pool_obj->pages;
+	sg = guc->ctx_pool_vma->pages;
 	sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
 			     sizeof(desc) * client->ctx_index);
 }
 
 /**
- * i915_guc_wq_check_space() - check that the GuC can accept a request
+ * i915_guc_wq_reserve() - reserve space in the GuC's workqueue
  * @request:	request associated with the commands
  *
  * Return:	0 if space is available
@@ -424,39 +421,56 @@ static void guc_fini_ctx_desc(struct intel_guc *guc,
  *
  * This function must be called (and must return 0) before a request
  * is submitted to the GuC via i915_guc_submit() below. Once a result
- * of 0 has been returned, it remains valid until (but only until)
- * the next call to submit().
+ * of 0 has been returned, it must be balanced by a corresponding
+ * call to submit().
  *
- * This precheck allows the caller to determine in advance that space
+ * Reservation allows the caller to determine in advance that space
  * will be available for the next submission before committing resources
  * to it, and helps avoid late failures with complicated recovery paths.
  */
-int i915_guc_wq_check_space(struct drm_i915_gem_request *request)
+int i915_guc_wq_reserve(struct drm_i915_gem_request *request)
 {
 	const size_t wqi_size = sizeof(struct guc_wq_item);
 	struct i915_guc_client *gc = request->i915->guc.execbuf_client;
-	struct guc_process_desc *desc;
+	struct guc_process_desc *desc = gc->client_base + gc->proc_desc_offset;
 	u32 freespace;
+	int ret;
 
-	GEM_BUG_ON(gc == NULL);
+	spin_lock(&gc->wq_lock);
+	freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size);
+	freespace -= gc->wq_rsvd;
+	if (likely(freespace >= wqi_size)) {
+		gc->wq_rsvd += wqi_size;
+		ret = 0;
+	} else {
+		gc->no_wq_space++;
+		ret = -EAGAIN;
+	}
+	spin_unlock(&gc->wq_lock);
 
-	desc = gc->client_base + gc->proc_desc_offset;
+	return ret;
+}
 
-	freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size);
-	if (likely(freespace >= wqi_size))
-		return 0;
+void i915_guc_wq_unreserve(struct drm_i915_gem_request *request)
+{
+	const size_t wqi_size = sizeof(struct guc_wq_item);
+	struct i915_guc_client *gc = request->i915->guc.execbuf_client;
 
-	gc->no_wq_space += 1;
+	GEM_BUG_ON(READ_ONCE(gc->wq_rsvd) < wqi_size);
 
-	return -EAGAIN;
+	spin_lock(&gc->wq_lock);
+	gc->wq_rsvd -= wqi_size;
+	spin_unlock(&gc->wq_lock);
 }
 
-static void guc_add_workqueue_item(struct i915_guc_client *gc,
-				   struct drm_i915_gem_request *rq)
+/* Construct a Work Item and append it to the GuC's Work Queue */
+static void guc_wq_item_append(struct i915_guc_client *gc,
+			       struct drm_i915_gem_request *rq)
 {
 	/* wqi_len is in DWords, and does not include the one-word header */
 	const size_t wqi_size = sizeof(struct guc_wq_item);
 	const u32 wqi_len = wqi_size/sizeof(u32) - 1;
+	struct intel_engine_cs *engine = rq->engine;
 	struct guc_process_desc *desc;
 	struct guc_wq_item *wqi;
 	void *base;
@@ -464,7 +478,7 @@ static void guc_add_workqueue_item(struct i915_guc_client *gc,
 
 	desc = gc->client_base + gc->proc_desc_offset;
 
-	/* Free space is guaranteed, see i915_guc_wq_check_space() above */
+	/* Free space is guaranteed, see i915_guc_wq_reserve() above */
 	freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size);
 	GEM_BUG_ON(freespace < wqi_size);
 
@@ -482,31 +496,32 @@ static void guc_add_workqueue_item(struct i915_guc_client *gc,
 	 * workqueue buffer dw by dw.
 	 */
 	BUILD_BUG_ON(wqi_size != 16);
+	GEM_BUG_ON(gc->wq_rsvd < wqi_size);
 
 	/* postincrement WQ tail for next time */
 	wq_off = gc->wq_tail;
+	GEM_BUG_ON(wq_off & (wqi_size - 1));
 	gc->wq_tail += wqi_size;
 	gc->wq_tail &= gc->wq_size - 1;
-	GEM_BUG_ON(wq_off & (wqi_size - 1));
+	gc->wq_rsvd -= wqi_size;
 
 	/* WQ starts from the page after doorbell / process_desc */
 	wq_page = (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT;
 	wq_off &= PAGE_SIZE - 1;
-	base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, wq_page));
+	base = kmap_atomic(i915_gem_object_get_page(gc->vma->obj, wq_page));
 	wqi = (struct guc_wq_item *)((char *)base + wq_off);
 
 	/* Now fill in the 4-word work queue item */
 	wqi->header = WQ_TYPE_INORDER |
 			(wqi_len << WQ_LEN_SHIFT) |
-			(rq->engine->guc_id << WQ_TARGET_SHIFT) |
+			(engine->guc_id << WQ_TARGET_SHIFT) |
 			WQ_NO_WCFLUSH_WAIT;
 
 	/* The GuC wants only the low-order word of the context descriptor */
-	wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx,
-							     rq->engine);
+	wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, engine);
 
 	wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT;
-	wqi->fence_id = rq->seqno;
+	wqi->fence_id = rq->fence.seqno;
 
 	kunmap_atomic(base);
 }
@@ -553,8 +568,8 @@ static int guc_ring_doorbell(struct i915_guc_client *gc)
 		if (db_ret.db_status == GUC_DOORBELL_DISABLED)
 			break;
 
-		DRM_ERROR("Cookie mismatch. Expected %d, returned %d\n",
-			  db_cmp.cookie, db_ret.cookie);
+		DRM_WARN("Cookie mismatch. Expected %d, found %d\n",
+			 db_cmp.cookie, db_ret.cookie);
 
 		/* update the cookie to newly read cookie from GuC */
 		db_cmp.cookie = db_ret.cookie;
@@ -573,26 +588,26 @@ static int guc_ring_doorbell(struct i915_guc_client *gc)
  * Return:	0 on success, otherwise an errno.
  * 		(Note: nonzero really shouldn't happen!)
  *
- * The caller must have already called i915_guc_wq_check_space() above
- * with a result of 0 (success) since the last request submission. This
- * guarantees that there is space in the work queue for the new request,
- * so enqueuing the item cannot fail.
+ * The caller must have already called i915_guc_wq_reserve() above with
+ * a result of 0 (success), guaranteeing that there is space in the work
+ * queue for the new request, so enqueuing the item cannot fail.
  *
  * Bad Things Will Happen if the caller violates this protocol e.g. calls
- * submit() when check() says there's no space, or calls submit() multiple
- * times with no intervening check().
+ * submit() when _reserve() says there's no space, or calls _submit()
+ * a different number of times from (successful) calls to _reserve().
  *
  * The only error here arises if the doorbell hardware isn't functioning
  * as expected, which really shouln't happen.
  */
-int i915_guc_submit(struct drm_i915_gem_request *rq)
+static void i915_guc_submit(struct drm_i915_gem_request *rq)
 {
 	unsigned int engine_id = rq->engine->id;
 	struct intel_guc *guc = &rq->i915->guc;
 	struct i915_guc_client *client = guc->execbuf_client;
 	int b_ret;
 
-	guc_add_workqueue_item(client, rq);
+	spin_lock(&client->wq_lock);
+	guc_wq_item_append(client, rq);
 	b_ret = guc_ring_doorbell(client);
 
 	client->submissions[engine_id] += 1;
@@ -601,9 +616,8 @@ int i915_guc_submit(struct drm_i915_gem_request *rq)
 		client->b_fail += 1;
 
 	guc->submissions[engine_id] += 1;
-	guc->last_seqno[engine_id] = rq->seqno;
-
-	return b_ret;
+	guc->last_seqno[engine_id] = rq->fence.seqno;
+	spin_unlock(&client->wq_lock);
 }
 
 /*
@@ -613,55 +627,48 @@ int i915_guc_submit(struct drm_i915_gem_request *rq)
  */
 
 /**
- * gem_allocate_guc_obj() - Allocate gem object for GuC usage
- * @dev_priv:	driver private data structure
- * @size:	size of object
+ * guc_allocate_vma() - Allocate a GGTT VMA for GuC usage
+ * @guc:	the guc
+ * @size:	size of area to allocate (both virtual space and memory)
  *
- * This is a wrapper to create a gem obj. In order to use it inside GuC, the
- * object needs to be pinned lifetime. Also we must pin it to gtt space other
- * than [0, GUC_WOPCM_TOP) because this range is reserved inside GuC.
+ * This is a wrapper to create an object for use with the GuC. In order to
+ * use it inside the GuC, an object needs to be pinned lifetime, so we allocate
+ * both some backing storage and a range inside the Global GTT. We must pin
+ * it in the GGTT somewhere other than than [0, GUC_WOPCM_TOP) because that
+ * range is reserved inside GuC.
  *
- * Return:	A drm_i915_gem_object if successful, otherwise NULL.
+ * Return:	A i915_vma if successful, otherwise an ERR_PTR.
  */
-static struct drm_i915_gem_object *
-gem_allocate_guc_obj(struct drm_i915_private *dev_priv, u32 size)
+static struct i915_vma *guc_allocate_vma(struct intel_guc *guc, u32 size)
 {
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	int ret;
 
 	obj = i915_gem_object_create(&dev_priv->drm, size);
 	if (IS_ERR(obj))
-		return NULL;
+		return ERR_CAST(obj);
 
-	if (i915_gem_object_get_pages(obj)) {
-		drm_gem_object_unreference(&obj->base);
-		return NULL;
-	}
+	vma = i915_vma_create(obj, &dev_priv->ggtt.base, NULL);
+	if (IS_ERR(vma))
+		goto err;
 
-	if (i915_gem_obj_ggtt_pin(obj, PAGE_SIZE,
-			PIN_OFFSET_BIAS | GUC_WOPCM_TOP)) {
-		drm_gem_object_unreference(&obj->base);
-		return NULL;
+	ret = i915_vma_pin(vma, 0, PAGE_SIZE,
+			   PIN_GLOBAL | PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
+	if (ret) {
+		vma = ERR_PTR(ret);
+		goto err;
 	}
 
 	/* Invalidate GuC TLB to let GuC take the latest updates to GTT. */
 	I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 
-	return obj;
-}
+	return vma;
 
-/**
- * gem_release_guc_obj() - Release gem object allocated for GuC usage
- * @obj:	gem obj to be released
- */
-static void gem_release_guc_obj(struct drm_i915_gem_object *obj)
-{
-	if (!obj)
-		return;
-
-	if (i915_gem_obj_is_pinned(obj))
-		i915_gem_object_ggtt_unpin(obj);
-
-	drm_gem_object_unreference(&obj->base);
+err:
+	i915_gem_object_put(obj);
+	return vma;
 }
 
 static void
@@ -688,61 +695,74 @@ guc_client_free(struct drm_i915_private *dev_priv,
 		kunmap(kmap_to_page(client->client_base));
 	}
 
-	gem_release_guc_obj(client->client_obj);
+	i915_vma_unpin_and_release(&client->vma);
 
 	if (client->ctx_index != GUC_INVALID_CTX_ID) {
-		guc_fini_ctx_desc(guc, client);
+		guc_ctx_desc_fini(guc, client);
 		ida_simple_remove(&guc->ctx_ids, client->ctx_index);
 	}
 
 	kfree(client);
 }
 
+/* Check that a doorbell register is in the expected state */
+static bool guc_doorbell_check(struct intel_guc *guc, uint16_t db_id)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	i915_reg_t drbreg = GEN8_DRBREGL(db_id);
+	uint32_t value = I915_READ(drbreg);
+	bool enabled = (value & GUC_DOORBELL_ENABLED) != 0;
+	bool expected = test_bit(db_id, guc->doorbell_bitmap);
+
+	if (enabled == expected)
+		return true;
+
+	DRM_DEBUG_DRIVER("Doorbell %d (reg 0x%x) 0x%x, should be %s\n",
+			 db_id, drbreg.reg, value,
+			 expected ? "active" : "inactive");
+
+	return false;
+}
+
 /*
- * Borrow the first client to set up & tear down every doorbell
+ * Borrow the first client to set up & tear down each unused doorbell
  * in turn, to ensure that all doorbell h/w is (re)initialised.
  */
 static void guc_init_doorbell_hw(struct intel_guc *guc)
 {
-	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 	struct i915_guc_client *client = guc->execbuf_client;
-	uint16_t db_id, i;
-	int err;
+	uint16_t db_id;
+	int i, err;
 
+	/* Save client's original doorbell selection */
 	db_id = client->doorbell_id;
 
 	for (i = 0; i < GUC_MAX_DOORBELLS; ++i) {
-		i915_reg_t drbreg = GEN8_DRBREGL(i);
-		u32 value = I915_READ(drbreg);
+		/* Skip if doorbell is OK */
+		if (guc_doorbell_check(guc, i))
+			continue;
 
 		err = guc_update_doorbell_id(guc, client, i);
-
-		/* Report update failure or unexpectedly active doorbell */
-		if (err || (i != db_id && (value & GUC_DOORBELL_ENABLED)))
-			DRM_DEBUG_DRIVER("Doorbell %d (reg 0x%x) was 0x%x, err %d\n",
-					  i, drbreg.reg, value, err);
+		if (err)
+			DRM_DEBUG_DRIVER("Doorbell %d update failed, err %d\n",
+					i, err);
 	}
 
 	/* Restore to original value */
 	err = guc_update_doorbell_id(guc, client, db_id);
 	if (err)
-		DRM_ERROR("Failed to restore doorbell to %d, err %d\n",
-			db_id, err);
+		DRM_WARN("Failed to restore doorbell to %d, err %d\n",
+			 db_id, err);
 
-	for (i = 0; i < GUC_MAX_DOORBELLS; ++i) {
-		i915_reg_t drbreg = GEN8_DRBREGL(i);
-		u32 value = I915_READ(drbreg);
-
-		if (i != db_id && (value & GUC_DOORBELL_ENABLED))
-			DRM_DEBUG_DRIVER("Doorbell %d (reg 0x%x) finally 0x%x\n",
-					  i, drbreg.reg, value);
-
-	}
+	/* Read back & verify all doorbell registers */
+	for (i = 0; i < GUC_MAX_DOORBELLS; ++i)
+		(void)guc_doorbell_check(guc, i);
 }
 
 /**
  * guc_client_alloc() - Allocate an i915_guc_client
  * @dev_priv:	driver private data structure
+ * @engines:	The set of engines to enable for this client
  * @priority:	four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW
  * 		The kernel client to replace ExecList submission is created with
  * 		NORMAL priority. Priority of a client for scheduler can be HIGH,
@@ -754,22 +774,24 @@ static void guc_init_doorbell_hw(struct intel_guc *guc)
  */
 static struct i915_guc_client *
 guc_client_alloc(struct drm_i915_private *dev_priv,
+		 uint32_t engines,
 		 uint32_t priority,
 		 struct i915_gem_context *ctx)
 {
 	struct i915_guc_client *client;
 	struct intel_guc *guc = &dev_priv->guc;
-	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
 	uint16_t db_id;
 
 	client = kzalloc(sizeof(*client), GFP_KERNEL);
 	if (!client)
 		return NULL;
 
-	client->doorbell_id = GUC_INVALID_DOORBELL_ID;
-	client->priority = priority;
 	client->owner = ctx;
 	client->guc = guc;
+	client->engines = engines;
+	client->priority = priority;
+	client->doorbell_id = GUC_INVALID_DOORBELL_ID;
 
 	client->ctx_index = (uint32_t)ida_simple_get(&guc->ctx_ids, 0,
 			GUC_MAX_GPU_CONTEXTS, GFP_KERNEL);
@@ -779,13 +801,15 @@ guc_client_alloc(struct drm_i915_private *dev_priv,
 	}
 
 	/* The first page is doorbell/proc_desc. Two followed pages are wq. */
-	obj = gem_allocate_guc_obj(dev_priv, GUC_DB_SIZE + GUC_WQ_SIZE);
-	if (!obj)
+	vma = guc_allocate_vma(guc, GUC_DB_SIZE + GUC_WQ_SIZE);
+	if (IS_ERR(vma))
 		goto err;
 
 	/* We'll keep just the first (doorbell/proc) page permanently kmap'd. */
-	client->client_obj = obj;
-	client->client_base = kmap(i915_gem_object_get_page(obj, 0));
+	client->vma = vma;
+	client->client_base = kmap(i915_vma_first_page(vma));
+
+	spin_lock_init(&client->wq_lock);
 	client->wq_offset = GUC_DB_SIZE;
 	client->wq_size = GUC_WQ_SIZE;
 
@@ -806,29 +830,26 @@ guc_client_alloc(struct drm_i915_private *dev_priv,
 	else
 		client->proc_desc_offset = (GUC_DB_SIZE / 2);
 
-	guc_init_proc_desc(guc, client);
-	guc_init_ctx_desc(guc, client);
+	guc_proc_desc_init(guc, client);
+	guc_ctx_desc_init(guc, client);
 	if (guc_init_doorbell(guc, client, db_id))
 		goto err;
 
-	DRM_DEBUG_DRIVER("new priority %u client %p: ctx_index %u\n",
-		priority, client, client->ctx_index);
+	DRM_DEBUG_DRIVER("new priority %u client %p for engine(s) 0x%x: ctx_index %u\n",
+		priority, client, client->engines, client->ctx_index);
 	DRM_DEBUG_DRIVER("doorbell id %u, cacheline offset 0x%x\n",
 		client->doorbell_id, client->doorbell_offset);
 
 	return client;
 
 err:
-	DRM_ERROR("FAILED to create priority %u GuC client!\n", priority);
-
 	guc_client_free(dev_priv, client);
 	return NULL;
 }
 
-static void guc_create_log(struct intel_guc *guc)
+static void guc_log_create(struct intel_guc *guc)
 {
-	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
 	unsigned long offset;
 	uint32_t size, flags;
 
@@ -844,16 +865,16 @@ static void guc_create_log(struct intel_guc *guc)
 		GUC_LOG_ISR_PAGES + 1 +
 		GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT;
 
-	obj = guc->log_obj;
-	if (!obj) {
-		obj = gem_allocate_guc_obj(dev_priv, size);
-		if (!obj) {
+	vma = guc->log_vma;
+	if (!vma) {
+		vma = guc_allocate_vma(guc, size);
+		if (IS_ERR(vma)) {
 			/* logging will be off */
 			i915.guc_log_level = -1;
 			return;
 		}
 
-		guc->log_obj = obj;
+		guc->log_vma = vma;
 	}
 
 	/* each allocated unit is a page */
@@ -862,11 +883,11 @@ static void guc_create_log(struct intel_guc *guc)
 		(GUC_LOG_ISR_PAGES << GUC_LOG_ISR_SHIFT) |
 		(GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT);
 
-	offset = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; /* in pages */
+	offset = i915_ggtt_offset(vma) >> PAGE_SHIFT; /* in pages */
 	guc->log_flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags;
 }
 
-static void init_guc_policies(struct guc_policies *policies)
+static void guc_policies_init(struct guc_policies *policies)
 {
 	struct guc_policy *policy;
 	u32 p, i;
@@ -888,10 +909,10 @@ static void init_guc_policies(struct guc_policies *policies)
 	policies->is_valid = 1;
 }
 
-static void guc_create_ads(struct intel_guc *guc)
+static void guc_addon_create(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
-	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
 	struct guc_ads *ads;
 	struct guc_policies *policies;
 	struct guc_mmio_reg_state *reg_state;
@@ -904,16 +925,16 @@ static void guc_create_ads(struct intel_guc *guc)
 			sizeof(struct guc_mmio_reg_state) +
 			GUC_S3_SAVE_SPACE_PAGES * PAGE_SIZE;
 
-	obj = guc->ads_obj;
-	if (!obj) {
-		obj = gem_allocate_guc_obj(dev_priv, PAGE_ALIGN(size));
-		if (!obj)
+	vma = guc->ads_vma;
+	if (!vma) {
+		vma = guc_allocate_vma(guc, PAGE_ALIGN(size));
+		if (IS_ERR(vma))
 			return;
 
-		guc->ads_obj = obj;
+		guc->ads_vma = vma;
 	}
 
-	page = i915_gem_object_get_page(obj, 0);
+	page = i915_vma_first_page(vma);
 	ads = kmap(page);
 
 	/*
@@ -924,17 +945,17 @@ static void guc_create_ads(struct intel_guc *guc)
 	 * to find it.
 	 */
 	engine = &dev_priv->engine[RCS];
-	ads->golden_context_lrca = engine->status_page.gfx_addr;
+	ads->golden_context_lrca = engine->status_page.ggtt_offset;
 
 	for_each_engine(engine, dev_priv)
 		ads->eng_state_size[engine->guc_id] = intel_lr_context_size(engine);
 
 	/* GuC scheduling policies */
 	policies = (void *)ads + sizeof(struct guc_ads);
-	init_guc_policies(policies);
+	guc_policies_init(policies);
 
-	ads->scheduler_policies = i915_gem_obj_ggtt_offset(obj) +
-			sizeof(struct guc_ads);
+	ads->scheduler_policies =
+		i915_ggtt_offset(vma) + sizeof(struct guc_ads);
 
 	/* MMIO reg state */
 	reg_state = (void *)policies + sizeof(struct guc_policies);
@@ -966,6 +987,7 @@ int i915_guc_submission_init(struct drm_i915_private *dev_priv)
 	const size_t poolsize = GUC_MAX_GPU_CONTEXTS * ctxsize;
 	const size_t gemsize = round_up(poolsize, PAGE_SIZE);
 	struct intel_guc *guc = &dev_priv->guc;
+	struct i915_vma *vma;
 
 	/* Wipe bitmap & delete client in case of reinitialisation */
 	bitmap_clear(guc->doorbell_bitmap, 0, GUC_MAX_DOORBELLS);
@@ -974,16 +996,17 @@ int i915_guc_submission_init(struct drm_i915_private *dev_priv)
 	if (!i915.enable_guc_submission)
 		return 0; /* not enabled  */
 
-	if (guc->ctx_pool_obj)
+	if (guc->ctx_pool_vma)
 		return 0; /* already allocated */
 
-	guc->ctx_pool_obj = gem_allocate_guc_obj(dev_priv, gemsize);
-	if (!guc->ctx_pool_obj)
-		return -ENOMEM;
+	vma = guc_allocate_vma(guc, gemsize);
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
 
+	guc->ctx_pool_vma = vma;
 	ida_init(&guc->ctx_ids);
-	guc_create_log(guc);
-	guc_create_ads(guc);
+	guc_log_create(guc);
+	guc_addon_create(guc);
 
 	return 0;
 }
@@ -992,13 +1015,16 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
 	struct i915_guc_client *client;
+	struct intel_engine_cs *engine;
+	struct drm_i915_gem_request *request;
 
 	/* client for execbuf submission */
 	client = guc_client_alloc(dev_priv,
+				  INTEL_INFO(dev_priv)->ring_mask,
 				  GUC_CTX_PRIORITY_KMD_NORMAL,
 				  dev_priv->kernel_context);
 	if (!client) {
-		DRM_ERROR("Failed to create execbuf guc_client\n");
+		DRM_ERROR("Failed to create normal GuC client!\n");
 		return -ENOMEM;
 	}
 
@@ -1006,6 +1032,18 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
 	host2guc_sample_forcewake(guc, client);
 	guc_init_doorbell_hw(guc);
 
+	/* Take over from manual control of ELSP (execlists) */
+	for_each_engine(engine, dev_priv) {
+		engine->submit_request = i915_guc_submit;
+
+		/* Replay the current set of previously submitted requests */
+		list_for_each_entry(request, &engine->request_list, link) {
+			client->wq_rsvd += sizeof(struct guc_wq_item);
+			if (i915_sw_fence_done(&request->submit))
+				i915_guc_submit(request);
+		}
+	}
+
 	return 0;
 }
 
@@ -1013,6 +1051,12 @@ void i915_guc_submission_disable(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
 
+	if (!guc->execbuf_client)
+		return;
+
+	/* Revert back to manual ELSP submission */
+	intel_execlists_enable_submission(dev_priv);
+
 	guc_client_free(dev_priv, guc->execbuf_client);
 	guc->execbuf_client = NULL;
 }
@@ -1021,16 +1065,12 @@ void i915_guc_submission_fini(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
 
-	gem_release_guc_obj(dev_priv->guc.ads_obj);
-	guc->ads_obj = NULL;
-
-	gem_release_guc_obj(dev_priv->guc.log_obj);
-	guc->log_obj = NULL;
+	i915_vma_unpin_and_release(&guc->ads_vma);
+	i915_vma_unpin_and_release(&guc->log_vma);
 
-	if (guc->ctx_pool_obj)
+	if (guc->ctx_pool_vma)
 		ida_destroy(&guc->ctx_ids);
-	gem_release_guc_obj(guc->ctx_pool_obj);
-	guc->ctx_pool_obj = NULL;
+	i915_vma_unpin_and_release(&guc->ctx_pool_vma);
 }
 
 /**
@@ -1053,7 +1093,7 @@ int intel_guc_suspend(struct drm_device *dev)
 	/* any value greater than GUC_POWER_D0 */
 	data[1] = GUC_POWER_D1;
 	/* first page is shared data with GuC */
-	data[2] = i915_gem_obj_ggtt_offset(ctx->engine[RCS].state);
+	data[2] = i915_ggtt_offset(ctx->engine[RCS].state);
 
 	return host2guc_action(guc, data, ARRAY_SIZE(data));
 }
@@ -1078,7 +1118,7 @@ int intel_guc_resume(struct drm_device *dev)
 	data[0] = HOST2GUC_ACTION_EXIT_S_STATE;
 	data[1] = GUC_POWER_D0;
 	/* first page is shared data with GuC */
-	data[2] = i915_gem_obj_ggtt_offset(ctx->engine[RCS].state);
+	data[2] = i915_ggtt_offset(ctx->engine[RCS].state);
 
 	return host2guc_action(guc, data, ARRAY_SIZE(data));
 }
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 1c2aec392412..3fc286cd1157 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -350,6 +350,9 @@ void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
 
 void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
 {
+	if (READ_ONCE(dev_priv->rps.interrupts_enabled))
+		return;
+
 	spin_lock_irq(&dev_priv->irq_lock);
 	WARN_ON_ONCE(dev_priv->rps.pm_iir);
 	WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
@@ -368,10 +371,13 @@ u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
 
 void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
 {
+	if (!READ_ONCE(dev_priv->rps.interrupts_enabled))
+		return;
+
 	spin_lock_irq(&dev_priv->irq_lock);
 	dev_priv->rps.interrupts_enabled = false;
 
-	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
+	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
 
 	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
@@ -656,12 +662,6 @@ static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
  *   of horizontal active on the first line of vertical active
  */
 
-static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
-{
-	/* Gen2 doesn't have a hardware frame counter */
-	return 0;
-}
-
 /* Called from drm generic code, passed a 'crtc', which
  * we use as a pipe index
  */
@@ -978,10 +978,8 @@ static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
 static void notify_ring(struct intel_engine_cs *engine)
 {
 	smp_store_mb(engine->breadcrumbs.irq_posted, true);
-	if (intel_engine_wakeup(engine)) {
+	if (intel_engine_wakeup(engine))
 		trace_i915_gem_request_notify(engine);
-		engine->breadcrumbs.irq_wakeups++;
-	}
 }
 
 static void vlv_c0_read(struct drm_i915_private *dev_priv,
@@ -1105,9 +1103,10 @@ static void gen6_pm_rps_work(struct work_struct *work)
 	new_delay = dev_priv->rps.cur_freq;
 	min = dev_priv->rps.min_freq_softlimit;
 	max = dev_priv->rps.max_freq_softlimit;
-
-	if (client_boost) {
-		new_delay = dev_priv->rps.max_freq_softlimit;
+	if (client_boost || any_waiters(dev_priv))
+		max = dev_priv->rps.max_freq;
+	if (client_boost && new_delay < dev_priv->rps.boost_freq) {
+		new_delay = dev_priv->rps.boost_freq;
 		adj = 0;
 	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
 		if (adj > 0)
@@ -1122,7 +1121,7 @@ static void gen6_pm_rps_work(struct work_struct *work)
 			new_delay = dev_priv->rps.efficient_freq;
 			adj = 0;
 		}
-	} else if (any_waiters(dev_priv)) {
+	} else if (client_boost || any_waiters(dev_priv)) {
 		adj = 0;
 	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
 		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
@@ -2504,57 +2503,52 @@ static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv)
 	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
 	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
 	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
-	int ret;
 
 	kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
 
+	DRM_DEBUG_DRIVER("resetting chip\n");
+	kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);
+
 	/*
-	 * Note that there's only one work item which does gpu resets, so we
-	 * need not worry about concurrent gpu resets potentially incrementing
-	 * error->reset_counter twice. We only need to take care of another
-	 * racing irq/hangcheck declaring the gpu dead for a second time. A
-	 * quick check for that is good enough: schedule_work ensures the
-	 * correct ordering between hang detection and this work item, and since
-	 * the reset in-progress bit is only ever set by code outside of this
-	 * work we don't need to worry about any other races.
+	 * In most cases it's guaranteed that we get here with an RPM
+	 * reference held, for example because there is a pending GPU
+	 * request that won't finish until the reset is done. This
+	 * isn't the case at least when we get here by doing a
+	 * simulated reset via debugs, so get an RPM reference.
 	 */
-	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
-		DRM_DEBUG_DRIVER("resetting chip\n");
-		kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);
-
-		/*
-		 * In most cases it's guaranteed that we get here with an RPM
-		 * reference held, for example because there is a pending GPU
-		 * request that won't finish until the reset is done. This
-		 * isn't the case at least when we get here by doing a
-		 * simulated reset via debugs, so get an RPM reference.
-		 */
-		intel_runtime_pm_get(dev_priv);
-
-		intel_prepare_reset(dev_priv);
+	intel_runtime_pm_get(dev_priv);
+	intel_prepare_reset(dev_priv);
 
+	do {
 		/*
 		 * All state reset _must_ be completed before we update the
 		 * reset counter, for otherwise waiters might miss the reset
 		 * pending state and not properly drop locks, resulting in
 		 * deadlocks with the reset work.
 		 */
-		ret = i915_reset(dev_priv);
+		if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
+			i915_reset(dev_priv);
+			mutex_unlock(&dev_priv->drm.struct_mutex);
+		}
 
-		intel_finish_reset(dev_priv);
+		/* We need to wait for anyone holding the lock to wakeup */
+	} while (wait_on_bit_timeout(&dev_priv->gpu_error.flags,
+				     I915_RESET_IN_PROGRESS,
+				     TASK_UNINTERRUPTIBLE,
+				     HZ));
 
-		intel_runtime_pm_put(dev_priv);
+	intel_finish_reset(dev_priv);
+	intel_runtime_pm_put(dev_priv);
 
-		if (ret == 0)
-			kobject_uevent_env(kobj,
-					   KOBJ_CHANGE, reset_done_event);
+	if (!test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
+		kobject_uevent_env(kobj,
+				   KOBJ_CHANGE, reset_done_event);
 
-		/*
-		 * Note: The wake_up also serves as a memory barrier so that
-		 * waiters see the update value of the reset counter atomic_t.
-		 */
-		wake_up_all(&dev_priv->gpu_error.reset_queue);
-	}
+	/*
+	 * Note: The wake_up also serves as a memory barrier so that
+	 * waiters see the updated value of the dev_priv->gpu_error.
+	 */
+	wake_up_all(&dev_priv->gpu_error.reset_queue);
 }
 
 static void i915_report_and_clear_eir(struct drm_i915_private *dev_priv)
@@ -2673,25 +2667,26 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 	i915_capture_error_state(dev_priv, engine_mask, error_msg);
 	i915_report_and_clear_eir(dev_priv);
 
-	if (engine_mask) {
-		atomic_or(I915_RESET_IN_PROGRESS_FLAG,
-				&dev_priv->gpu_error.reset_counter);
+	if (!engine_mask)
+		return;
 
-		/*
-		 * Wakeup waiting processes so that the reset function
-		 * i915_reset_and_wakeup doesn't deadlock trying to grab
-		 * various locks. By bumping the reset counter first, the woken
-		 * processes will see a reset in progress and back off,
-		 * releasing their locks and then wait for the reset completion.
-		 * We must do this for _all_ gpu waiters that might hold locks
-		 * that the reset work needs to acquire.
-		 *
-		 * Note: The wake_up serves as the required memory barrier to
-		 * ensure that the waiters see the updated value of the reset
-		 * counter atomic_t.
-		 */
-		i915_error_wake_up(dev_priv);
-	}
+	if (test_and_set_bit(I915_RESET_IN_PROGRESS,
+			     &dev_priv->gpu_error.flags))
+		return;
+
+	/*
+	 * Wakeup waiting processes so that the reset function
+	 * i915_reset_and_wakeup doesn't deadlock trying to grab
+	 * various locks. By bumping the reset counter first, the woken
+	 * processes will see a reset in progress and back off,
+	 * releasing their locks and then wait for the reset completion.
+	 * We must do this for _all_ gpu waiters that might hold locks
+	 * that the reset work needs to acquire.
+	 *
+	 * Note: The wake_up also provides a memory barrier to ensure that the
+	 * waiters see the updated value of the reset flags.
+	 */
+	i915_error_wake_up(dev_priv);
 
 	i915_reset_and_wakeup(dev_priv);
 }
@@ -2804,13 +2799,6 @@ static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
 }
 
 static bool
-ring_idle(struct intel_engine_cs *engine, u32 seqno)
-{
-	return i915_seqno_passed(seqno,
-				 READ_ONCE(engine->last_submitted_seqno));
-}
-
-static bool
 ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr)
 {
 	if (INTEL_GEN(engine->i915) >= 8) {
@@ -2834,7 +2822,7 @@ semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
 			if (engine == signaller)
 				continue;
 
-			if (offset == signaller->semaphore.signal_ggtt[engine->id])
+			if (offset == signaller->semaphore.signal_ggtt[engine->hw_id])
 				return signaller;
 		}
 	} else {
@@ -2844,21 +2832,22 @@ semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
 			if(engine == signaller)
 				continue;
 
-			if (sync_bits == signaller->semaphore.mbox.wait[engine->id])
+			if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id])
 				return signaller;
 		}
 	}
 
-	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
-		  engine->id, ipehr, offset);
+	DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x, offset 0x%016llx\n",
+			 engine->name, ipehr, offset);
 
-	return NULL;
+	return ERR_PTR(-ENODEV);
 }
 
 static struct intel_engine_cs *
 semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
+	void __iomem *vaddr;
 	u32 cmd, ipehr, head;
 	u64 offset = 0;
 	int i, backwards;
@@ -2897,6 +2886,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
 	 */
 	head = I915_READ_HEAD(engine) & HEAD_ADDR;
 	backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4;
+	vaddr = (void __iomem *)engine->buffer->vaddr;
 
 	for (i = backwards; i; --i) {
 		/*
@@ -2907,7 +2897,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
 		head &= engine->buffer->size - 1;
 
 		/* This here seems to blow up */
-		cmd = ioread32(engine->buffer->virtual_start + head);
+		cmd = ioread32(vaddr + head);
 		if (cmd == ipehr)
 			break;
 
@@ -2917,11 +2907,11 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
 	if (!i)
 		return NULL;
 
-	*seqno = ioread32(engine->buffer->virtual_start + head + 4) + 1;
+	*seqno = ioread32(vaddr + head + 4) + 1;
 	if (INTEL_GEN(dev_priv) >= 8) {
-		offset = ioread32(engine->buffer->virtual_start + head + 12);
+		offset = ioread32(vaddr + head + 12);
 		offset <<= 32;
-		offset = ioread32(engine->buffer->virtual_start + head + 8);
+		offset |= ioread32(vaddr + head + 8);
 	}
 	return semaphore_wait_to_signaller_ring(engine, ipehr, offset);
 }
@@ -2938,6 +2928,9 @@ static int semaphore_passed(struct intel_engine_cs *engine)
 	if (signaller == NULL)
 		return -1;
 
+	if (IS_ERR(signaller))
+		return 0;
+
 	/* Prevent pathological recursion due to driver bugs */
 	if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES)
 		return -1;
@@ -2990,7 +2983,7 @@ static bool subunits_stuck(struct intel_engine_cs *engine)
 	return stuck;
 }
 
-static enum intel_ring_hangcheck_action
+static enum intel_engine_hangcheck_action
 head_stuck(struct intel_engine_cs *engine, u64 acthd)
 {
 	if (acthd != engine->hangcheck.acthd) {
@@ -3008,11 +3001,11 @@ head_stuck(struct intel_engine_cs *engine, u64 acthd)
 	return HANGCHECK_HUNG;
 }
 
-static enum intel_ring_hangcheck_action
-ring_stuck(struct intel_engine_cs *engine, u64 acthd)
+static enum intel_engine_hangcheck_action
+engine_stuck(struct intel_engine_cs *engine, u64 acthd)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
-	enum intel_ring_hangcheck_action ha;
+	enum intel_engine_hangcheck_action ha;
 	u32 tmp;
 
 	ha = head_stuck(engine, acthd);
@@ -3054,22 +3047,6 @@ ring_stuck(struct intel_engine_cs *engine, u64 acthd)
 	return HANGCHECK_HUNG;
 }
 
-static unsigned long kick_waiters(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *i915 = engine->i915;
-	unsigned long irq_count = READ_ONCE(engine->breadcrumbs.irq_wakeups);
-
-	if (engine->hangcheck.user_interrupts == irq_count &&
-	    !test_and_set_bit(engine->id, &i915->gpu_error.missed_irq_rings)) {
-		if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings))
-			DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
-				  engine->name);
-
-		intel_engine_enable_fake_irq(engine);
-	}
-
-	return irq_count;
-}
 /*
  * This is called when the chip hasn't reported back with completed
  * batchbuffers in a long time. We keep track per ring seqno progress and
@@ -3107,7 +3084,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 		bool busy = intel_engine_has_waiter(engine);
 		u64 acthd;
 		u32 seqno;
-		unsigned user_interrupts;
+		u32 submit;
 
 		semaphore_clear_deadlocks(dev_priv);
 
@@ -3121,29 +3098,22 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 		if (engine->irq_seqno_barrier)
 			engine->irq_seqno_barrier(engine);
 
-		acthd = intel_ring_get_active_head(engine);
+		acthd = intel_engine_get_active_head(engine);
 		seqno = intel_engine_get_seqno(engine);
-
-		/* Reset stuck interrupts between batch advances */
-		user_interrupts = 0;
+		submit = READ_ONCE(engine->last_submitted_seqno);
 
 		if (engine->hangcheck.seqno == seqno) {
-			if (ring_idle(engine, seqno)) {
+			if (i915_seqno_passed(seqno, submit)) {
 				engine->hangcheck.action = HANGCHECK_IDLE;
-				if (busy) {
-					/* Safeguard against driver failure */
-					user_interrupts = kick_waiters(engine);
-					engine->hangcheck.score += BUSY;
-				}
 			} else {
 				/* We always increment the hangcheck score
-				 * if the ring is busy and still processing
+				 * if the engine is busy and still processing
 				 * the same request, so that no single request
 				 * can run indefinitely (such as a chain of
 				 * batches). The only time we do not increment
 				 * the hangcheck score on this ring, if this
-				 * ring is in a legitimate wait for another
-				 * ring. In that case the waiting ring is a
+				 * engine is in a legitimate wait for another
+				 * engine. In that case the waiting engine is a
 				 * victim and we want to be sure we catch the
 				 * right culprit. Then every time we do kick
 				 * the ring, add a small increment to the
@@ -3151,8 +3121,8 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 				 * being repeatedly kicked and so responsible
 				 * for stalling the machine.
 				 */
-				engine->hangcheck.action = ring_stuck(engine,
-								      acthd);
+				engine->hangcheck.action =
+					engine_stuck(engine, acthd);
 
 				switch (engine->hangcheck.action) {
 				case HANGCHECK_IDLE:
@@ -3195,12 +3165,12 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 
 		engine->hangcheck.seqno = seqno;
 		engine->hangcheck.acthd = acthd;
-		engine->hangcheck.user_interrupts = user_interrupts;
 		busy_count += busy;
 	}
 
 	if (hung) {
 		char msg[80];
+		unsigned int tmp;
 		int len;
 
 		/* If some rings hung but others were still busy, only
@@ -3210,7 +3180,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 			hung &= ~stuck;
 		len = scnprintf(msg, sizeof(msg),
 				"%s on ", stuck == hung ? "No progress" : "Hang");
-		for_each_engine_masked(engine, dev_priv, hung)
+		for_each_engine_masked(engine, dev_priv, hung, tmp)
 			len += scnprintf(msg + len, sizeof(msg) - len,
 					 "%s, ", engine->name);
 		msg[len-2] = '\0';
@@ -4536,14 +4506,15 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 		dev_priv->rps.pm_intr_keep |= GEN6_PM_RP_UP_EI_EXPIRED;
 
 	if (INTEL_INFO(dev_priv)->gen >= 8)
-		dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_NON_DISP;
+		dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_GUC;
 
 	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
 			  i915_hangcheck_elapsed);
 
 	if (IS_GEN2(dev_priv)) {
+		/* Gen2 doesn't have a hardware frame counter */
 		dev->max_vblank_count = 0;
-		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
+		dev->driver->get_vblank_counter = drm_vblank_no_hw_counter;
 	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
 		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
 		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
diff --git a/drivers/gpu/drm/i915/i915_memcpy.c b/drivers/gpu/drm/i915/i915_memcpy.c
new file mode 100644
index 000000000000..49a079494b68
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_memcpy.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <asm/fpu/api.h>
+
+#include "i915_drv.h"
+
+static DEFINE_STATIC_KEY_FALSE(has_movntdqa);
+
+#ifdef CONFIG_AS_MOVNTDQA
+static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
+{
+	kernel_fpu_begin();
+
+	len >>= 4;
+	while (len >= 4) {
+		asm("movntdqa   (%0), %%xmm0\n"
+		    "movntdqa 16(%0), %%xmm1\n"
+		    "movntdqa 32(%0), %%xmm2\n"
+		    "movntdqa 48(%0), %%xmm3\n"
+		    "movaps %%xmm0,   (%1)\n"
+		    "movaps %%xmm1, 16(%1)\n"
+		    "movaps %%xmm2, 32(%1)\n"
+		    "movaps %%xmm3, 48(%1)\n"
+		    :: "r" (src), "r" (dst) : "memory");
+		src += 64;
+		dst += 64;
+		len -= 4;
+	}
+	while (len--) {
+		asm("movntdqa (%0), %%xmm0\n"
+		    "movaps %%xmm0, (%1)\n"
+		    :: "r" (src), "r" (dst) : "memory");
+		src += 16;
+		dst += 16;
+	}
+
+	kernel_fpu_end();
+}
+#endif
+
+/**
+ * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC
+ * @dst: destination pointer
+ * @src: source pointer
+ * @len: how many bytes to copy
+ *
+ * i915_memcpy_from_wc copies @len bytes from @src to @dst using
+ * non-temporal instructions where available. Note that all arguments
+ * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
+ * of 16.
+ *
+ * To test whether accelerated reads from WC are supported, use
+ * i915_memcpy_from_wc(NULL, NULL, 0);
+ *
+ * Returns true if the copy was successful, false if the preconditions
+ * are not met.
+ */
+bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
+{
+	if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))
+		return false;
+
+#ifdef CONFIG_AS_MOVNTDQA
+	if (static_branch_likely(&has_movntdqa)) {
+		if (likely(len))
+			__memcpy_ntdqa(dst, src, len);
+		return true;
+	}
+#endif
+
+	return false;
+}
+
+void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
+{
+	if (static_cpu_has(X86_FEATURE_XMM4_1))
+		static_branch_enable(&has_movntdqa);
+}
diff --git a/drivers/gpu/drm/i915/i915_mm.c b/drivers/gpu/drm/i915/i915_mm.c
new file mode 100644
index 000000000000..e4935dd1fd37
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_mm.c
@@ -0,0 +1,84 @@
+/*
+ * Copyright © 2014 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/mm.h>
+#include <linux/io-mapping.h>
+
+#include <asm/pgtable.h>
+
+#include "i915_drv.h"
+
+struct remap_pfn {
+	struct mm_struct *mm;
+	unsigned long pfn;
+	pgprot_t prot;
+};
+
+static int remap_pfn(pte_t *pte, pgtable_t token,
+		     unsigned long addr, void *data)
+{
+	struct remap_pfn *r = data;
+
+	/* Special PTE are not associated with any struct page */
+	set_pte_at(r->mm, addr, pte, pte_mkspecial(pfn_pte(r->pfn, r->prot)));
+	r->pfn++;
+
+	return 0;
+}
+
+/**
+ * remap_io_mapping - remap an IO mapping to userspace
+ * @vma: user vma to map to
+ * @addr: target user address to start at
+ * @pfn: physical address of kernel memory
+ * @size: size of map area
+ * @iomap: the source io_mapping
+ *
+ *  Note: this is only safe if the mm semaphore is held when called.
+ */
+int remap_io_mapping(struct vm_area_struct *vma,
+		     unsigned long addr, unsigned long pfn, unsigned long size,
+		     struct io_mapping *iomap)
+{
+	struct remap_pfn r;
+	int err;
+
+	GEM_BUG_ON((vma->vm_flags &
+		    (VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP)) !=
+		   (VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP));
+
+	/* We rely on prevalidation of the io-mapping to skip track_pfn(). */
+	r.mm = vma->vm_mm;
+	r.pfn = pfn;
+	r.prot = __pgprot((pgprot_val(iomap->prot) & _PAGE_CACHE_MASK) |
+			  (pgprot_val(vma->vm_page_prot) & ~_PAGE_CACHE_MASK));
+
+	err = apply_to_page_range(r.mm, addr, size, remap_pfn, &r);
+	if (unlikely(err)) {
+		zap_vma_ptes(vma, addr, (r.pfn - pfn) << PAGE_SHIFT);
+		return err;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index b6e404c91eed..768ad89d9cd4 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -45,6 +45,7 @@ struct i915_params i915 __read_mostly = {
 	.fastboot = 0,
 	.prefault_disable = 0,
 	.load_detect_test = 0,
+	.force_reset_modeset_test = 0,
 	.reset = true,
 	.invert_brightness = 0,
 	.disable_display = 0,
@@ -161,6 +162,11 @@ MODULE_PARM_DESC(load_detect_test,
 	"Force-enable the VGA load detect code for testing (default:false). "
 	"For developers only.");
 
+module_param_named_unsafe(force_reset_modeset_test, i915.force_reset_modeset_test, bool, 0600);
+MODULE_PARM_DESC(force_reset_modeset_test,
+	"Force a modeset during gpu reset for testing (default:false). "
+	"For developers only.");
+
 module_param_named_unsafe(invert_brightness, i915.invert_brightness, int, 0600);
 MODULE_PARM_DESC(invert_brightness,
 	"Invert backlight brightness "
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 0ad020b4a925..3a0dd78ddb38 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -57,6 +57,7 @@ struct i915_params {
 	bool fastboot;
 	bool prefault_disable;
 	bool load_detect_test;
+	bool force_reset_modeset_test;
 	bool reset;
 	bool disable_display;
 	bool verbose_state_checks;
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 949c01686a66..687c768833b3 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -54,207 +54,216 @@
 #define CHV_COLORS \
 	.color = { .degamma_lut_size = 65, .gamma_lut_size = 257 }
 
+#define GEN2_FEATURES \
+	.gen = 2, .num_pipes = 1, \
+	.has_overlay = 1, .overlay_needs_physical = 1, \
+	.has_gmch_display = 1, \
+	.hws_needs_physical = 1, \
+	.ring_mask = RENDER_RING, \
+	GEN_DEFAULT_PIPEOFFSETS, \
+	CURSOR_OFFSETS
+
 static const struct intel_device_info intel_i830_info = {
-	.gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2,
-	.has_overlay = 1, .overlay_needs_physical = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	GEN2_FEATURES,
+	.is_mobile = 1, .cursor_needs_physical = 1,
+	.num_pipes = 2, /* legal, last one wins */
 };
 
 static const struct intel_device_info intel_845g_info = {
-	.gen = 2, .num_pipes = 1,
-	.has_overlay = 1, .overlay_needs_physical = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	GEN2_FEATURES,
 };
 
 static const struct intel_device_info intel_i85x_info = {
-	.gen = 2, .is_i85x = 1, .is_mobile = 1, .num_pipes = 2,
+	GEN2_FEATURES,
+	.is_i85x = 1, .is_mobile = 1,
+	.num_pipes = 2, /* legal, last one wins */
 	.cursor_needs_physical = 1,
-	.has_overlay = 1, .overlay_needs_physical = 1,
 	.has_fbc = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_i865g_info = {
-	.gen = 2, .num_pipes = 1,
-	.has_overlay = 1, .overlay_needs_physical = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	GEN2_FEATURES,
 };
 
+#define GEN3_FEATURES \
+	.gen = 3, .num_pipes = 2, \
+	.has_gmch_display = 1, \
+	.ring_mask = RENDER_RING, \
+	GEN_DEFAULT_PIPEOFFSETS, \
+	CURSOR_OFFSETS
+
 static const struct intel_device_info intel_i915g_info = {
-	.gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .num_pipes = 2,
+	GEN3_FEATURES,
+	.is_i915g = 1, .cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	.hws_needs_physical = 1,
 };
 static const struct intel_device_info intel_i915gm_info = {
-	.gen = 3, .is_mobile = 1, .num_pipes = 2,
+	GEN3_FEATURES,
+	.is_mobile = 1,
 	.cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 	.supports_tv = 1,
 	.has_fbc = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	.hws_needs_physical = 1,
 };
 static const struct intel_device_info intel_i945g_info = {
-	.gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2,
+	GEN3_FEATURES,
+	.has_hotplug = 1, .cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	.hws_needs_physical = 1,
 };
 static const struct intel_device_info intel_i945gm_info = {
-	.gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2,
+	GEN3_FEATURES,
+	.is_i945gm = 1, .is_mobile = 1,
 	.has_hotplug = 1, .cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
 	.supports_tv = 1,
 	.has_fbc = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	.hws_needs_physical = 1,
 };
 
+#define GEN4_FEATURES \
+	.gen = 4, .num_pipes = 2, \
+	.has_hotplug = 1, \
+	.has_gmch_display = 1, \
+	.ring_mask = RENDER_RING, \
+	GEN_DEFAULT_PIPEOFFSETS, \
+	CURSOR_OFFSETS
+
 static const struct intel_device_info intel_i965g_info = {
-	.gen = 4, .is_broadwater = 1, .num_pipes = 2,
-	.has_hotplug = 1,
+	GEN4_FEATURES,
+	.is_broadwater = 1,
 	.has_overlay = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	.hws_needs_physical = 1,
 };
 
 static const struct intel_device_info intel_i965gm_info = {
-	.gen = 4, .is_crestline = 1, .num_pipes = 2,
-	.is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
+	GEN4_FEATURES,
+	.is_crestline = 1,
+	.is_mobile = 1, .has_fbc = 1,
 	.has_overlay = 1,
 	.supports_tv = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	.hws_needs_physical = 1,
 };
 
 static const struct intel_device_info intel_g33_info = {
-	.gen = 3, .is_g33 = 1, .num_pipes = 2,
-	.need_gfx_hws = 1, .has_hotplug = 1,
+	GEN3_FEATURES,
+	.is_g33 = 1,
+	.has_hotplug = 1,
 	.has_overlay = 1,
-	.ring_mask = RENDER_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_g45_info = {
-	.gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .num_pipes = 2,
-	.has_pipe_cxsr = 1, .has_hotplug = 1,
+	GEN4_FEATURES,
+	.is_g4x = 1,
+	.has_pipe_cxsr = 1,
 	.ring_mask = RENDER_RING | BSD_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_gm45_info = {
-	.gen = 4, .is_g4x = 1, .num_pipes = 2,
-	.is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
-	.has_pipe_cxsr = 1, .has_hotplug = 1,
+	GEN4_FEATURES,
+	.is_g4x = 1,
+	.is_mobile = 1, .has_fbc = 1,
+	.has_pipe_cxsr = 1,
 	.supports_tv = 1,
 	.ring_mask = RENDER_RING | BSD_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
 };
 
 static const struct intel_device_info intel_pineview_info = {
-	.gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .num_pipes = 2,
-	.need_gfx_hws = 1, .has_hotplug = 1,
+	GEN3_FEATURES,
+	.is_g33 = 1, .is_pineview = 1, .is_mobile = 1,
+	.has_hotplug = 1,
 	.has_overlay = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
 };
 
+#define GEN5_FEATURES \
+	.gen = 5, .num_pipes = 2, \
+	.has_hotplug = 1, \
+	.has_gmbus_irq = 1, \
+	.ring_mask = RENDER_RING | BSD_RING, \
+	GEN_DEFAULT_PIPEOFFSETS, \
+	CURSOR_OFFSETS
+
 static const struct intel_device_info intel_ironlake_d_info = {
-	.gen = 5, .num_pipes = 2,
-	.need_gfx_hws = 1, .has_hotplug = 1,
-	.ring_mask = RENDER_RING | BSD_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	GEN5_FEATURES,
 };
 
 static const struct intel_device_info intel_ironlake_m_info = {
-	.gen = 5, .is_mobile = 1, .num_pipes = 2,
-	.need_gfx_hws = 1, .has_hotplug = 1,
-	.has_fbc = 1,
-	.ring_mask = RENDER_RING | BSD_RING,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	GEN5_FEATURES,
+	.is_mobile = 1,
 };
 
+#define GEN6_FEATURES \
+	.gen = 6, .num_pipes = 2, \
+	.has_hotplug = 1, \
+	.has_fbc = 1, \
+	.ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
+	.has_llc = 1, \
+	.has_rc6 = 1, \
+	.has_rc6p = 1, \
+	.has_gmbus_irq = 1, \
+	.has_hw_contexts = 1, \
+	GEN_DEFAULT_PIPEOFFSETS, \
+	CURSOR_OFFSETS
+
 static const struct intel_device_info intel_sandybridge_d_info = {
-	.gen = 6, .num_pipes = 2,
-	.need_gfx_hws = 1, .has_hotplug = 1,
-	.has_fbc = 1,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING,
-	.has_llc = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	GEN6_FEATURES,
 };
 
 static const struct intel_device_info intel_sandybridge_m_info = {
-	.gen = 6, .is_mobile = 1, .num_pipes = 2,
-	.need_gfx_hws = 1, .has_hotplug = 1,
-	.has_fbc = 1,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING,
-	.has_llc = 1,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	GEN6_FEATURES,
+	.is_mobile = 1,
 };
 
 #define GEN7_FEATURES  \
 	.gen = 7, .num_pipes = 3, \
-	.need_gfx_hws = 1, .has_hotplug = 1, \
+	.has_hotplug = 1, \
 	.has_fbc = 1, \
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
 	.has_llc = 1, \
+	.has_rc6 = 1, \
+	.has_rc6p = 1, \
+	.has_gmbus_irq = 1, \
+	.has_hw_contexts = 1, \
 	GEN_DEFAULT_PIPEOFFSETS, \
 	IVB_CURSOR_OFFSETS
 
 static const struct intel_device_info intel_ivybridge_d_info = {
 	GEN7_FEATURES,
 	.is_ivybridge = 1,
+	.has_l3_dpf = 1,
 };
 
 static const struct intel_device_info intel_ivybridge_m_info = {
 	GEN7_FEATURES,
 	.is_ivybridge = 1,
 	.is_mobile = 1,
+	.has_l3_dpf = 1,
 };
 
 static const struct intel_device_info intel_ivybridge_q_info = {
 	GEN7_FEATURES,
 	.is_ivybridge = 1,
 	.num_pipes = 0, /* legal, last one wins */
+	.has_l3_dpf = 1,
 };
 
 #define VLV_FEATURES  \
 	.gen = 7, .num_pipes = 2, \
-	.need_gfx_hws = 1, .has_hotplug = 1, \
+	.has_psr = 1, \
+	.has_runtime_pm = 1, \
+	.has_rc6 = 1, \
+	.has_gmbus_irq = 1, \
+	.has_hw_contexts = 1, \
+	.has_gmch_display = 1, \
+	.has_hotplug = 1, \
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
 	.display_mmio_offset = VLV_DISPLAY_BASE, \
 	GEN_DEFAULT_PIPEOFFSETS, \
 	CURSOR_OFFSETS
 
-static const struct intel_device_info intel_valleyview_m_info = {
-	VLV_FEATURES,
-	.is_valleyview = 1,
-	.is_mobile = 1,
-};
-
-static const struct intel_device_info intel_valleyview_d_info = {
+static const struct intel_device_info intel_valleyview_info = {
 	VLV_FEATURES,
 	.is_valleyview = 1,
 };
@@ -263,54 +272,50 @@ static const struct intel_device_info intel_valleyview_d_info = {
 	GEN7_FEATURES, \
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \
 	.has_ddi = 1, \
-	.has_fpga_dbg = 1
-
-static const struct intel_device_info intel_haswell_d_info = {
-	HSW_FEATURES,
-	.is_haswell = 1,
-};
-
-static const struct intel_device_info intel_haswell_m_info = {
+	.has_fpga_dbg = 1, \
+	.has_psr = 1, \
+	.has_resource_streamer = 1, \
+	.has_dp_mst = 1, \
+	.has_rc6p = 0 /* RC6p removed-by HSW */, \
+	.has_runtime_pm = 1
+
+static const struct intel_device_info intel_haswell_info = {
 	HSW_FEATURES,
 	.is_haswell = 1,
-	.is_mobile = 1,
+	.has_l3_dpf = 1,
 };
 
 #define BDW_FEATURES \
 	HSW_FEATURES, \
-	BDW_COLORS
+	BDW_COLORS, \
+	.has_logical_ring_contexts = 1
 
-static const struct intel_device_info intel_broadwell_d_info = {
+static const struct intel_device_info intel_broadwell_info = {
 	BDW_FEATURES,
 	.gen = 8,
 	.is_broadwell = 1,
 };
 
-static const struct intel_device_info intel_broadwell_m_info = {
-	BDW_FEATURES,
-	.gen = 8, .is_mobile = 1,
-	.is_broadwell = 1,
-};
-
-static const struct intel_device_info intel_broadwell_gt3d_info = {
+static const struct intel_device_info intel_broadwell_gt3_info = {
 	BDW_FEATURES,
 	.gen = 8,
 	.is_broadwell = 1,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 };
 
-static const struct intel_device_info intel_broadwell_gt3m_info = {
-	BDW_FEATURES,
-	.gen = 8, .is_mobile = 1,
-	.is_broadwell = 1,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
-};
-
 static const struct intel_device_info intel_cherryview_info = {
 	.gen = 8, .num_pipes = 3,
-	.need_gfx_hws = 1, .has_hotplug = 1,
+	.has_hotplug = 1,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 	.is_cherryview = 1,
+	.has_psr = 1,
+	.has_runtime_pm = 1,
+	.has_resource_streamer = 1,
+	.has_rc6 = 1,
+	.has_gmbus_irq = 1,
+	.has_hw_contexts = 1,
+	.has_logical_ring_contexts = 1,
+	.has_gmch_display = 1,
 	.display_mmio_offset = VLV_DISPLAY_BASE,
 	GEN_CHV_PIPEOFFSETS,
 	CURSOR_OFFSETS,
@@ -321,25 +326,41 @@ static const struct intel_device_info intel_skylake_info = {
 	BDW_FEATURES,
 	.is_skylake = 1,
 	.gen = 9,
+	.has_csr = 1,
+	.has_guc = 1,
+	.ddb_size = 896,
 };
 
 static const struct intel_device_info intel_skylake_gt3_info = {
 	BDW_FEATURES,
 	.is_skylake = 1,
 	.gen = 9,
+	.has_csr = 1,
+	.has_guc = 1,
+	.ddb_size = 896,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 };
 
 static const struct intel_device_info intel_broxton_info = {
 	.is_broxton = 1,
 	.gen = 9,
-	.need_gfx_hws = 1, .has_hotplug = 1,
+	.has_hotplug = 1,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 	.num_pipes = 3,
 	.has_ddi = 1,
 	.has_fpga_dbg = 1,
 	.has_fbc = 1,
+	.has_runtime_pm = 1,
 	.has_pooled_eu = 0,
+	.has_csr = 1,
+	.has_resource_streamer = 1,
+	.has_rc6 = 1,
+	.has_dp_mst = 1,
+	.has_gmbus_irq = 1,
+	.has_hw_contexts = 1,
+	.has_logical_ring_contexts = 1,
+	.has_guc = 1,
+	.ddb_size = 512,
 	GEN_DEFAULT_PIPEOFFSETS,
 	IVB_CURSOR_OFFSETS,
 	BDW_COLORS,
@@ -349,12 +370,18 @@ static const struct intel_device_info intel_kabylake_info = {
 	BDW_FEATURES,
 	.is_kabylake = 1,
 	.gen = 9,
+	.has_csr = 1,
+	.has_guc = 1,
+	.ddb_size = 896,
 };
 
 static const struct intel_device_info intel_kabylake_gt3_info = {
 	BDW_FEATURES,
 	.is_kabylake = 1,
 	.gen = 9,
+	.has_csr = 1,
+	.has_guc = 1,
+	.ddb_size = 896,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 };
 
@@ -386,14 +413,10 @@ static const struct pci_device_id pciidlist[] = {
 	INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */
 	INTEL_IVB_M_IDS(&intel_ivybridge_m_info),
 	INTEL_IVB_D_IDS(&intel_ivybridge_d_info),
-	INTEL_HSW_D_IDS(&intel_haswell_d_info),
-	INTEL_HSW_M_IDS(&intel_haswell_m_info),
-	INTEL_VLV_M_IDS(&intel_valleyview_m_info),
-	INTEL_VLV_D_IDS(&intel_valleyview_d_info),
-	INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info),
-	INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info),
-	INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info),
-	INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info),
+	INTEL_HSW_IDS(&intel_haswell_info),
+	INTEL_VLV_IDS(&intel_valleyview_info),
+	INTEL_BDW_GT12_IDS(&intel_broadwell_info),
+	INTEL_BDW_GT3_IDS(&intel_broadwell_gt3_info),
 	INTEL_CHV_IDS(&intel_cherryview_info),
 	INTEL_SKL_GT1_IDS(&intel_skylake_info),
 	INTEL_SKL_GT2_IDS(&intel_skylake_info),
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index ce14fe09d962..70d96162def6 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -186,13 +186,13 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  GEN9_GRDOM_GUC			(1 << 5)
 #define  GEN8_GRDOM_MEDIA2		(1 << 7)
 
-#define RING_PP_DIR_BASE(ring)		_MMIO((ring)->mmio_base+0x228)
-#define RING_PP_DIR_BASE_READ(ring)	_MMIO((ring)->mmio_base+0x518)
-#define RING_PP_DIR_DCLV(ring)		_MMIO((ring)->mmio_base+0x220)
+#define RING_PP_DIR_BASE(engine)	_MMIO((engine)->mmio_base+0x228)
+#define RING_PP_DIR_BASE_READ(engine)	_MMIO((engine)->mmio_base+0x518)
+#define RING_PP_DIR_DCLV(engine)	_MMIO((engine)->mmio_base+0x220)
 #define   PP_DIR_DCLV_2G		0xffffffff
 
-#define GEN8_RING_PDP_UDW(ring, n)	_MMIO((ring)->mmio_base+0x270 + (n) * 8 + 4)
-#define GEN8_RING_PDP_LDW(ring, n)	_MMIO((ring)->mmio_base+0x270 + (n) * 8)
+#define GEN8_RING_PDP_UDW(engine, n)	_MMIO((engine)->mmio_base+0x270 + (n) * 8 + 4)
+#define GEN8_RING_PDP_LDW(engine, n)	_MMIO((engine)->mmio_base+0x270 + (n) * 8)
 
 #define GEN8_R_PWR_CLK_STATE		_MMIO(0x20C8)
 #define   GEN8_RPCS_ENABLE		(1 << 31)
@@ -1536,6 +1536,7 @@ enum skl_disp_power_wells {
 #define BALANCE_LEG_MASK(port)		(7<<(8+3*(port)))
 /* Balance leg disable bits */
 #define BALANCE_LEG_DISABLE_SHIFT	23
+#define BALANCE_LEG_DISABLE(port)	(1 << (23 + (port)))
 
 /*
  * Fence registers
@@ -1647,7 +1648,7 @@ enum skl_disp_power_wells {
 #define   ARB_MODE_BWGTLB_DISABLE (1<<9)
 #define   ARB_MODE_SWIZZLE_BDW	(1<<1)
 #define RENDER_HWS_PGA_GEN7	_MMIO(0x04080)
-#define RING_FAULT_REG(ring)	_MMIO(0x4094 + 0x100*(ring)->id)
+#define RING_FAULT_REG(engine)	_MMIO(0x4094 + 0x100*(engine)->hw_id)
 #define   RING_FAULT_GTTSEL_MASK (1<<11)
 #define   RING_FAULT_SRCID(x)	(((x) >> 3) & 0xff)
 #define   RING_FAULT_FAULT_TYPE(x) (((x) >> 1) & 0x3)
@@ -1845,7 +1846,7 @@ enum skl_disp_power_wells {
 
 #define GFX_MODE	_MMIO(0x2520)
 #define GFX_MODE_GEN7	_MMIO(0x229c)
-#define RING_MODE_GEN7(ring)	_MMIO((ring)->mmio_base+0x29c)
+#define RING_MODE_GEN7(engine)	_MMIO((engine)->mmio_base+0x29c)
 #define   GFX_RUN_LIST_ENABLE		(1<<15)
 #define   GFX_INTERRUPT_STEERING	(1<<14)
 #define   GFX_TLB_INVALIDATE_EXPLICIT	(1<<13)
@@ -3659,8 +3660,17 @@ enum {
 #define   VIDEO_DIP_ENABLE_SPD_HSW	(1 << 0)
 
 /* Panel power sequencing */
-#define PP_STATUS	_MMIO(0x61200)
-#define   PP_ON		(1 << 31)
+#define PPS_BASE			0x61200
+#define VLV_PPS_BASE			(VLV_DISPLAY_BASE + PPS_BASE)
+#define PCH_PPS_BASE			0xC7200
+
+#define _MMIO_PPS(pps_idx, reg)		_MMIO(dev_priv->pps_mmio_base -	\
+					      PPS_BASE + (reg) +	\
+					      (pps_idx) * 0x100)
+
+#define _PP_STATUS			0x61200
+#define PP_STATUS(pps_idx)		_MMIO_PPS(pps_idx, _PP_STATUS)
+#define   PP_ON				(1 << 31)
 /*
  * Indicates that all dependencies of the panel are on:
  *
@@ -3668,14 +3678,14 @@ enum {
  * - pipe enabled
  * - LVDS/DVOB/DVOC on
  */
-#define   PP_READY		(1 << 30)
-#define   PP_SEQUENCE_NONE	(0 << 28)
-#define   PP_SEQUENCE_POWER_UP	(1 << 28)
-#define   PP_SEQUENCE_POWER_DOWN (2 << 28)
-#define   PP_SEQUENCE_MASK	(3 << 28)
-#define   PP_SEQUENCE_SHIFT	28
-#define   PP_CYCLE_DELAY_ACTIVE	(1 << 27)
-#define   PP_SEQUENCE_STATE_MASK 0x0000000f
+#define   PP_READY			(1 << 30)
+#define   PP_SEQUENCE_NONE		(0 << 28)
+#define   PP_SEQUENCE_POWER_UP		(1 << 28)
+#define   PP_SEQUENCE_POWER_DOWN	(2 << 28)
+#define   PP_SEQUENCE_MASK		(3 << 28)
+#define   PP_SEQUENCE_SHIFT		28
+#define   PP_CYCLE_DELAY_ACTIVE		(1 << 27)
+#define   PP_SEQUENCE_STATE_MASK	0x0000000f
 #define   PP_SEQUENCE_STATE_OFF_IDLE	(0x0 << 0)
 #define   PP_SEQUENCE_STATE_OFF_S0_1	(0x1 << 0)
 #define   PP_SEQUENCE_STATE_OFF_S0_2	(0x2 << 0)
@@ -3685,11 +3695,46 @@ enum {
 #define   PP_SEQUENCE_STATE_ON_S1_2	(0xa << 0)
 #define   PP_SEQUENCE_STATE_ON_S1_3	(0xb << 0)
 #define   PP_SEQUENCE_STATE_RESET	(0xf << 0)
-#define PP_CONTROL	_MMIO(0x61204)
-#define   POWER_TARGET_ON	(1 << 0)
-#define PP_ON_DELAYS	_MMIO(0x61208)
-#define PP_OFF_DELAYS	_MMIO(0x6120c)
-#define PP_DIVISOR	_MMIO(0x61210)
+
+#define _PP_CONTROL			0x61204
+#define PP_CONTROL(pps_idx)		_MMIO_PPS(pps_idx, _PP_CONTROL)
+#define  PANEL_UNLOCK_REGS		(0xabcd << 16)
+#define  PANEL_UNLOCK_MASK		(0xffff << 16)
+#define  BXT_POWER_CYCLE_DELAY_MASK	0x1f0
+#define  BXT_POWER_CYCLE_DELAY_SHIFT	4
+#define  EDP_FORCE_VDD			(1 << 3)
+#define  EDP_BLC_ENABLE			(1 << 2)
+#define  PANEL_POWER_RESET		(1 << 1)
+#define  PANEL_POWER_OFF		(0 << 0)
+#define  PANEL_POWER_ON			(1 << 0)
+
+#define _PP_ON_DELAYS			0x61208
+#define PP_ON_DELAYS(pps_idx)		_MMIO_PPS(pps_idx, _PP_ON_DELAYS)
+#define  PANEL_PORT_SELECT_SHIFT	30
+#define  PANEL_PORT_SELECT_MASK		(3 << 30)
+#define  PANEL_PORT_SELECT_LVDS		(0 << 30)
+#define  PANEL_PORT_SELECT_DPA		(1 << 30)
+#define  PANEL_PORT_SELECT_DPC		(2 << 30)
+#define  PANEL_PORT_SELECT_DPD		(3 << 30)
+#define  PANEL_PORT_SELECT_VLV(port)	((port) << 30)
+#define  PANEL_POWER_UP_DELAY_MASK	0x1fff0000
+#define  PANEL_POWER_UP_DELAY_SHIFT	16
+#define  PANEL_LIGHT_ON_DELAY_MASK	0x1fff
+#define  PANEL_LIGHT_ON_DELAY_SHIFT	0
+
+#define _PP_OFF_DELAYS			0x6120C
+#define PP_OFF_DELAYS(pps_idx)		_MMIO_PPS(pps_idx, _PP_OFF_DELAYS)
+#define  PANEL_POWER_DOWN_DELAY_MASK	0x1fff0000
+#define  PANEL_POWER_DOWN_DELAY_SHIFT	16
+#define  PANEL_LIGHT_OFF_DELAY_MASK	0x1fff
+#define  PANEL_LIGHT_OFF_DELAY_SHIFT	0
+
+#define _PP_DIVISOR			0x61210
+#define PP_DIVISOR(pps_idx)		_MMIO_PPS(pps_idx, _PP_DIVISOR)
+#define  PP_REFERENCE_DIVIDER_MASK	0xffffff00
+#define  PP_REFERENCE_DIVIDER_SHIFT	8
+#define  PANEL_POWER_CYCLE_DELAY_MASK	0x1f
+#define  PANEL_POWER_CYCLE_DELAY_SHIFT	0
 
 /* Panel fitting */
 #define PFIT_CONTROL	_MMIO(dev_priv->info.display_mmio_offset + 0x61230)
@@ -6132,6 +6177,7 @@ enum {
 # define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC	((1<<10) | (1<<26))
 # define GEN9_RHWO_OPTIMIZATION_DISABLE		(1<<14)
 #define COMMON_SLICE_CHICKEN2			_MMIO(0x7014)
+# define GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE (1<<12)
 # define GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION (1<<8)
 # define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE	(1<<0)
 
@@ -6748,77 +6794,6 @@ enum {
 #define PCH_LVDS	_MMIO(0xe1180)
 #define  LVDS_DETECTED	(1 << 1)
 
-/* vlv has 2 sets of panel control regs. */
-#define _PIPEA_PP_STATUS         (VLV_DISPLAY_BASE + 0x61200)
-#define _PIPEA_PP_CONTROL        (VLV_DISPLAY_BASE + 0x61204)
-#define _PIPEA_PP_ON_DELAYS      (VLV_DISPLAY_BASE + 0x61208)
-#define  PANEL_PORT_SELECT_VLV(port)	((port) << 30)
-#define _PIPEA_PP_OFF_DELAYS     (VLV_DISPLAY_BASE + 0x6120c)
-#define _PIPEA_PP_DIVISOR        (VLV_DISPLAY_BASE + 0x61210)
-
-#define _PIPEB_PP_STATUS         (VLV_DISPLAY_BASE + 0x61300)
-#define _PIPEB_PP_CONTROL        (VLV_DISPLAY_BASE + 0x61304)
-#define _PIPEB_PP_ON_DELAYS      (VLV_DISPLAY_BASE + 0x61308)
-#define _PIPEB_PP_OFF_DELAYS     (VLV_DISPLAY_BASE + 0x6130c)
-#define _PIPEB_PP_DIVISOR        (VLV_DISPLAY_BASE + 0x61310)
-
-#define VLV_PIPE_PP_STATUS(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_STATUS, _PIPEB_PP_STATUS)
-#define VLV_PIPE_PP_CONTROL(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_CONTROL, _PIPEB_PP_CONTROL)
-#define VLV_PIPE_PP_ON_DELAYS(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_ON_DELAYS, _PIPEB_PP_ON_DELAYS)
-#define VLV_PIPE_PP_OFF_DELAYS(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_OFF_DELAYS, _PIPEB_PP_OFF_DELAYS)
-#define VLV_PIPE_PP_DIVISOR(pipe)	_MMIO_PIPE(pipe, _PIPEA_PP_DIVISOR, _PIPEB_PP_DIVISOR)
-
-#define _PCH_PP_STATUS		0xc7200
-#define _PCH_PP_CONTROL		0xc7204
-#define  PANEL_UNLOCK_REGS	(0xabcd << 16)
-#define  PANEL_UNLOCK_MASK	(0xffff << 16)
-#define  BXT_POWER_CYCLE_DELAY_MASK	(0x1f0)
-#define  BXT_POWER_CYCLE_DELAY_SHIFT	4
-#define  EDP_FORCE_VDD		(1 << 3)
-#define  EDP_BLC_ENABLE		(1 << 2)
-#define  PANEL_POWER_RESET	(1 << 1)
-#define  PANEL_POWER_OFF	(0 << 0)
-#define  PANEL_POWER_ON		(1 << 0)
-#define _PCH_PP_ON_DELAYS	0xc7208
-#define  PANEL_PORT_SELECT_MASK	(3 << 30)
-#define  PANEL_PORT_SELECT_LVDS	(0 << 30)
-#define  PANEL_PORT_SELECT_DPA	(1 << 30)
-#define  PANEL_PORT_SELECT_DPC	(2 << 30)
-#define  PANEL_PORT_SELECT_DPD	(3 << 30)
-#define  PANEL_POWER_UP_DELAY_MASK	(0x1fff0000)
-#define  PANEL_POWER_UP_DELAY_SHIFT	16
-#define  PANEL_LIGHT_ON_DELAY_MASK	(0x1fff)
-#define  PANEL_LIGHT_ON_DELAY_SHIFT	0
-
-#define _PCH_PP_OFF_DELAYS		0xc720c
-#define  PANEL_POWER_DOWN_DELAY_MASK	(0x1fff0000)
-#define  PANEL_POWER_DOWN_DELAY_SHIFT	16
-#define  PANEL_LIGHT_OFF_DELAY_MASK	(0x1fff)
-#define  PANEL_LIGHT_OFF_DELAY_SHIFT	0
-
-#define _PCH_PP_DIVISOR			0xc7210
-#define  PP_REFERENCE_DIVIDER_MASK	(0xffffff00)
-#define  PP_REFERENCE_DIVIDER_SHIFT	8
-#define  PANEL_POWER_CYCLE_DELAY_MASK	(0x1f)
-#define  PANEL_POWER_CYCLE_DELAY_SHIFT	0
-
-#define PCH_PP_STATUS			_MMIO(_PCH_PP_STATUS)
-#define PCH_PP_CONTROL			_MMIO(_PCH_PP_CONTROL)
-#define PCH_PP_ON_DELAYS		_MMIO(_PCH_PP_ON_DELAYS)
-#define PCH_PP_OFF_DELAYS		_MMIO(_PCH_PP_OFF_DELAYS)
-#define PCH_PP_DIVISOR			_MMIO(_PCH_PP_DIVISOR)
-
-/* BXT PPS changes - 2nd set of PPS registers */
-#define _BXT_PP_STATUS2 	0xc7300
-#define _BXT_PP_CONTROL2 	0xc7304
-#define _BXT_PP_ON_DELAYS2	0xc7308
-#define _BXT_PP_OFF_DELAYS2	0xc730c
-
-#define BXT_PP_STATUS(n)	_MMIO_PIPE(n, _PCH_PP_STATUS, _BXT_PP_STATUS2)
-#define BXT_PP_CONTROL(n)	_MMIO_PIPE(n, _PCH_PP_CONTROL, _BXT_PP_CONTROL2)
-#define BXT_PP_ON_DELAYS(n)	_MMIO_PIPE(n, _PCH_PP_ON_DELAYS, _BXT_PP_ON_DELAYS2)
-#define BXT_PP_OFF_DELAYS(n)	_MMIO_PIPE(n, _PCH_PP_OFF_DELAYS, _BXT_PP_OFF_DELAYS2)
-
 #define _PCH_DP_B		0xe4100
 #define PCH_DP_B		_MMIO(_PCH_DP_B)
 #define _PCH_DPB_AUX_CH_CTL	0xe4110
@@ -6958,6 +6933,9 @@ enum {
 #define  ECOBUS					_MMIO(0xa180)
 #define    FORCEWAKE_MT_ENABLE			(1<<5)
 #define  VLV_SPAREG2H				_MMIO(0xA194)
+#define  GEN9_PWRGT_DOMAIN_STATUS		_MMIO(0xA2A0)
+#define   GEN9_PWRGT_MEDIA_STATUS_MASK		(1 << 0)
+#define   GEN9_PWRGT_RENDER_STATUS_MASK		(1 << 1)
 
 #define  GTFIFODBG				_MMIO(0x120000)
 #define    GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV	(0x1f << 20)
@@ -7058,12 +7036,13 @@ enum {
 #define GEN6_RP_UP_THRESHOLD			_MMIO(0xA02C)
 #define GEN6_RP_DOWN_THRESHOLD			_MMIO(0xA030)
 #define GEN6_RP_CUR_UP_EI			_MMIO(0xA050)
-#define   GEN6_CURICONT_MASK			0xffffff
+#define   GEN6_RP_EI_MASK			0xffffff
+#define   GEN6_CURICONT_MASK			GEN6_RP_EI_MASK
 #define GEN6_RP_CUR_UP				_MMIO(0xA054)
-#define   GEN6_CURBSYTAVG_MASK			0xffffff
+#define   GEN6_CURBSYTAVG_MASK			GEN6_RP_EI_MASK
 #define GEN6_RP_PREV_UP				_MMIO(0xA058)
 #define GEN6_RP_CUR_DOWN_EI			_MMIO(0xA05C)
-#define   GEN6_CURIAVG_MASK			0xffffff
+#define   GEN6_CURIAVG_MASK			GEN6_RP_EI_MASK
 #define GEN6_RP_CUR_DOWN			_MMIO(0xA060)
 #define GEN6_RP_PREV_DOWN			_MMIO(0xA064)
 #define GEN6_RP_UP_EI				_MMIO(0xA068)
@@ -7088,7 +7067,7 @@ enum {
 #define VLV_RCEDATA				_MMIO(0xA0BC)
 #define GEN6_RC6pp_THRESHOLD			_MMIO(0xA0C0)
 #define GEN6_PMINTRMSK				_MMIO(0xA168)
-#define   GEN8_PMINTR_REDIRECT_TO_NON_DISP	(1<<31)
+#define   GEN8_PMINTR_REDIRECT_TO_GUC		  (1<<31)
 #define GEN8_MISC_CTRL0				_MMIO(0xA180)
 #define VLV_PWRDWNUPCTL				_MMIO(0xA294)
 #define GEN9_MEDIA_PG_IDLE_HYSTERESIS		_MMIO(0xA0C4)
@@ -7144,6 +7123,15 @@ enum {
 
 #define GEN6_PCODE_MAILBOX			_MMIO(0x138124)
 #define   GEN6_PCODE_READY			(1<<31)
+#define   GEN6_PCODE_ERROR_MASK			0xFF
+#define     GEN6_PCODE_SUCCESS			0x0
+#define     GEN6_PCODE_ILLEGAL_CMD		0x1
+#define     GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x2
+#define     GEN6_PCODE_TIMEOUT			0x3
+#define     GEN6_PCODE_UNIMPLEMENTED_CMD	0xFF
+#define     GEN7_PCODE_TIMEOUT			0x2
+#define     GEN7_PCODE_ILLEGAL_DATA		0x3
+#define     GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x10
 #define	  GEN6_PCODE_WRITE_RC6VIDS		0x4
 #define	  GEN6_PCODE_READ_RC6VIDS		0x5
 #define     GEN6_ENCODE_RC6_VID(mv)		(((mv) - 245) / 5)
@@ -7165,6 +7153,10 @@ enum {
 #define   HSW_PCODE_DE_WRITE_FREQ_REQ		0x17
 #define   DISPLAY_IPS_CONTROL			0x19
 #define	  HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL	0x1A
+#define   GEN9_PCODE_SAGV_CONTROL		0x21
+#define     GEN9_SAGV_DISABLE			0x0
+#define     GEN9_SAGV_IS_DISABLED		0x1
+#define     GEN9_SAGV_ENABLE			0x3
 #define GEN6_PCODE_DATA				_MMIO(0x138128)
 #define   GEN6_PCODE_FREQ_IA_RATIO_SHIFT	8
 #define   GEN6_PCODE_FREQ_RING_RATIO_SHIFT	16
@@ -7485,6 +7477,7 @@ enum {
 #define _DDI_BUF_TRANS_A		0x64E00
 #define _DDI_BUF_TRANS_B		0x64E60
 #define DDI_BUF_TRANS_LO(port, i)	_MMIO(_PORT(port, _DDI_BUF_TRANS_A, _DDI_BUF_TRANS_B) + (i) * 8)
+#define  DDI_BUF_BALANCE_LEG_ENABLE	(1 << 31)
 #define DDI_BUF_TRANS_HI(port, i)	_MMIO(_PORT(port, _DDI_BUF_TRANS_A, _DDI_BUF_TRANS_B) + (i) * 8 + 4)
 
 /* Sideband Interface (SBI) is programmed indirectly, via
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 5cfe4c7716b4..a0af170062b1 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -37,25 +37,6 @@ static void i915_save_display(struct drm_device *dev)
 	if (INTEL_INFO(dev)->gen <= 4)
 		dev_priv->regfile.saveDSPARB = I915_READ(DSPARB);
 
-	/* LVDS state */
-	if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
-		dev_priv->regfile.saveLVDS = I915_READ(PCH_LVDS);
-	else if (INTEL_INFO(dev)->gen <= 4 && IS_MOBILE(dev) && !IS_I830(dev))
-		dev_priv->regfile.saveLVDS = I915_READ(LVDS);
-
-	/* Panel power sequencer */
-	if (HAS_PCH_SPLIT(dev)) {
-		dev_priv->regfile.savePP_CONTROL = I915_READ(PCH_PP_CONTROL);
-		dev_priv->regfile.savePP_ON_DELAYS = I915_READ(PCH_PP_ON_DELAYS);
-		dev_priv->regfile.savePP_OFF_DELAYS = I915_READ(PCH_PP_OFF_DELAYS);
-		dev_priv->regfile.savePP_DIVISOR = I915_READ(PCH_PP_DIVISOR);
-	} else if (INTEL_INFO(dev)->gen <= 4) {
-		dev_priv->regfile.savePP_CONTROL = I915_READ(PP_CONTROL);
-		dev_priv->regfile.savePP_ON_DELAYS = I915_READ(PP_ON_DELAYS);
-		dev_priv->regfile.savePP_OFF_DELAYS = I915_READ(PP_OFF_DELAYS);
-		dev_priv->regfile.savePP_DIVISOR = I915_READ(PP_DIVISOR);
-	}
-
 	/* save FBC interval */
 	if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev))
 		dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL);
@@ -64,33 +45,11 @@ static void i915_save_display(struct drm_device *dev)
 static void i915_restore_display(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	u32 mask = 0xffffffff;
 
 	/* Display arbitration */
 	if (INTEL_INFO(dev)->gen <= 4)
 		I915_WRITE(DSPARB, dev_priv->regfile.saveDSPARB);
 
-	mask = ~LVDS_PORT_EN;
-
-	/* LVDS state */
-	if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
-		I915_WRITE(PCH_LVDS, dev_priv->regfile.saveLVDS & mask);
-	else if (INTEL_INFO(dev)->gen <= 4 && IS_MOBILE(dev) && !IS_I830(dev))
-		I915_WRITE(LVDS, dev_priv->regfile.saveLVDS & mask);
-
-	/* Panel power sequencer */
-	if (HAS_PCH_SPLIT(dev)) {
-		I915_WRITE(PCH_PP_ON_DELAYS, dev_priv->regfile.savePP_ON_DELAYS);
-		I915_WRITE(PCH_PP_OFF_DELAYS, dev_priv->regfile.savePP_OFF_DELAYS);
-		I915_WRITE(PCH_PP_DIVISOR, dev_priv->regfile.savePP_DIVISOR);
-		I915_WRITE(PCH_PP_CONTROL, dev_priv->regfile.savePP_CONTROL);
-	} else if (INTEL_INFO(dev)->gen <= 4) {
-		I915_WRITE(PP_ON_DELAYS, dev_priv->regfile.savePP_ON_DELAYS);
-		I915_WRITE(PP_OFF_DELAYS, dev_priv->regfile.savePP_OFF_DELAYS);
-		I915_WRITE(PP_DIVISOR, dev_priv->regfile.savePP_DIVISOR);
-		I915_WRITE(PP_CONTROL, dev_priv->regfile.savePP_CONTROL);
-	}
-
 	/* only restore FBC info on the platform that supports FBC*/
 	intel_fbc_global_disable(dev_priv);
 
@@ -104,6 +63,7 @@ static void i915_restore_display(struct drm_device *dev)
 int i915_save_state(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	int i;
 
 	mutex_lock(&dev->struct_mutex);
@@ -111,7 +71,7 @@ int i915_save_state(struct drm_device *dev)
 	i915_save_display(dev);
 
 	if (IS_GEN4(dev))
-		pci_read_config_word(dev->pdev, GCDGMBUS,
+		pci_read_config_word(pdev, GCDGMBUS,
 				     &dev_priv->regfile.saveGCDGMBUS);
 
 	/* Cache mode state */
@@ -149,6 +109,7 @@ int i915_save_state(struct drm_device *dev)
 int i915_restore_state(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	int i;
 
 	mutex_lock(&dev->struct_mutex);
@@ -156,7 +117,7 @@ int i915_restore_state(struct drm_device *dev)
 	i915_gem_restore_fences(dev);
 
 	if (IS_GEN4(dev))
-		pci_write_config_word(dev->pdev, GCDGMBUS,
+		pci_write_config_word(pdev, GCDGMBUS,
 				      dev_priv->regfile.saveGCDGMBUS);
 	i915_restore_display(dev);
 
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c
new file mode 100644
index 000000000000..1e5cbc585ca2
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_sw_fence.c
@@ -0,0 +1,362 @@
+/*
+ * (C) Copyright 2016 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+
+#include <linux/slab.h>
+#include <linux/fence.h>
+#include <linux/reservation.h>
+
+#include "i915_sw_fence.h"
+
+static DEFINE_SPINLOCK(i915_sw_fence_lock);
+
+static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
+				  enum i915_sw_fence_notify state)
+{
+	i915_sw_fence_notify_t fn;
+
+	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
+	return fn(fence, state);
+}
+
+static void i915_sw_fence_free(struct kref *kref)
+{
+	struct i915_sw_fence *fence = container_of(kref, typeof(*fence), kref);
+
+	WARN_ON(atomic_read(&fence->pending) > 0);
+
+	if (fence->flags & I915_SW_FENCE_MASK)
+		__i915_sw_fence_notify(fence, FENCE_FREE);
+	else
+		kfree(fence);
+}
+
+static void i915_sw_fence_put(struct i915_sw_fence *fence)
+{
+	kref_put(&fence->kref, i915_sw_fence_free);
+}
+
+static struct i915_sw_fence *i915_sw_fence_get(struct i915_sw_fence *fence)
+{
+	kref_get(&fence->kref);
+	return fence;
+}
+
+static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
+					struct list_head *continuation)
+{
+	wait_queue_head_t *x = &fence->wait;
+	wait_queue_t *pos, *next;
+	unsigned long flags;
+
+	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
+
+	/*
+	 * To prevent unbounded recursion as we traverse the graph of
+	 * i915_sw_fences, we move the task_list from this, the next ready
+	 * fence, to the tail of the original fence's task_list
+	 * (and so added to the list to be woken).
+	 */
+
+	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
+	if (continuation) {
+		list_for_each_entry_safe(pos, next, &x->task_list, task_list) {
+			if (pos->func == autoremove_wake_function)
+				pos->func(pos, TASK_NORMAL, 0, continuation);
+			else
+				list_move_tail(&pos->task_list, continuation);
+		}
+	} else {
+		LIST_HEAD(extra);
+
+		do {
+			list_for_each_entry_safe(pos, next,
+						 &x->task_list, task_list)
+				pos->func(pos, TASK_NORMAL, 0, &extra);
+
+			if (list_empty(&extra))
+				break;
+
+			list_splice_tail_init(&extra, &x->task_list);
+		} while (1);
+	}
+	spin_unlock_irqrestore(&x->lock, flags);
+}
+
+static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
+				     struct list_head *continuation)
+{
+	if (!atomic_dec_and_test(&fence->pending))
+		return;
+
+	if (fence->flags & I915_SW_FENCE_MASK &&
+	    __i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
+		return;
+
+	__i915_sw_fence_wake_up_all(fence, continuation);
+}
+
+static void i915_sw_fence_complete(struct i915_sw_fence *fence)
+{
+	if (WARN_ON(i915_sw_fence_done(fence)))
+		return;
+
+	__i915_sw_fence_complete(fence, NULL);
+}
+
+static void i915_sw_fence_await(struct i915_sw_fence *fence)
+{
+	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
+}
+
+void i915_sw_fence_init(struct i915_sw_fence *fence, i915_sw_fence_notify_t fn)
+{
+	BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);
+
+	init_waitqueue_head(&fence->wait);
+	kref_init(&fence->kref);
+	atomic_set(&fence->pending, 1);
+	fence->flags = (unsigned long)fn;
+}
+
+void i915_sw_fence_commit(struct i915_sw_fence *fence)
+{
+	i915_sw_fence_complete(fence);
+	i915_sw_fence_put(fence);
+}
+
+static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *key)
+{
+	list_del(&wq->task_list);
+	__i915_sw_fence_complete(wq->private, key);
+	i915_sw_fence_put(wq->private);
+	return 0;
+}
+
+static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
+				    const struct i915_sw_fence * const signaler)
+{
+	wait_queue_t *wq;
+
+	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
+		return false;
+
+	if (fence == signaler)
+		return true;
+
+	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
+		if (wq->func != i915_sw_fence_wake)
+			continue;
+
+		if (__i915_sw_fence_check_if_after(wq->private, signaler))
+			return true;
+	}
+
+	return false;
+}
+
+static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
+{
+	wait_queue_t *wq;
+
+	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
+		return;
+
+	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
+		if (wq->func != i915_sw_fence_wake)
+			continue;
+
+		__i915_sw_fence_clear_checked_bit(wq->private);
+	}
+}
+
+static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
+				  const struct i915_sw_fence * const signaler)
+{
+	unsigned long flags;
+	bool err;
+
+	if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG))
+		return false;
+
+	spin_lock_irqsave(&i915_sw_fence_lock, flags);
+	err = __i915_sw_fence_check_if_after(fence, signaler);
+	__i915_sw_fence_clear_checked_bit(fence);
+	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
+
+	return err;
+}
+
+int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
+				 struct i915_sw_fence *signaler,
+				 wait_queue_t *wq)
+{
+	unsigned long flags;
+	int pending;
+
+	if (i915_sw_fence_done(signaler))
+		return 0;
+
+	/* The dependency graph must be acyclic. */
+	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
+		return -EINVAL;
+
+	INIT_LIST_HEAD(&wq->task_list);
+	wq->flags = 0;
+	wq->func = i915_sw_fence_wake;
+	wq->private = i915_sw_fence_get(fence);
+
+	i915_sw_fence_await(fence);
+
+	spin_lock_irqsave(&signaler->wait.lock, flags);
+	if (likely(!i915_sw_fence_done(signaler))) {
+		__add_wait_queue_tail(&signaler->wait, wq);
+		pending = 1;
+	} else {
+		i915_sw_fence_wake(wq, 0, 0, NULL);
+		pending = 0;
+	}
+	spin_unlock_irqrestore(&signaler->wait.lock, flags);
+
+	return pending;
+}
+
+struct dma_fence_cb {
+	struct fence_cb base;
+	struct i915_sw_fence *fence;
+	struct fence *dma;
+	struct timer_list timer;
+};
+
+static void timer_i915_sw_fence_wake(unsigned long data)
+{
+	struct dma_fence_cb *cb = (struct dma_fence_cb *)data;
+
+	printk(KERN_WARNING "asynchronous wait on fence %s:%s:%x timed out\n",
+	       cb->dma->ops->get_driver_name(cb->dma),
+	       cb->dma->ops->get_timeline_name(cb->dma),
+	       cb->dma->seqno);
+	fence_put(cb->dma);
+	cb->dma = NULL;
+
+	i915_sw_fence_commit(cb->fence);
+	cb->timer.function = NULL;
+}
+
+static void dma_i915_sw_fence_wake(struct fence *dma, struct fence_cb *data)
+{
+	struct dma_fence_cb *cb = container_of(data, typeof(*cb), base);
+
+	del_timer_sync(&cb->timer);
+	if (cb->timer.function)
+		i915_sw_fence_commit(cb->fence);
+	fence_put(cb->dma);
+
+	kfree(cb);
+}
+
+int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
+				  struct fence *dma,
+				  unsigned long timeout,
+				  gfp_t gfp)
+{
+	struct dma_fence_cb *cb;
+	int ret;
+
+	if (fence_is_signaled(dma))
+		return 0;
+
+	cb = kmalloc(sizeof(*cb), gfp);
+	if (!cb) {
+		if (!gfpflags_allow_blocking(gfp))
+			return -ENOMEM;
+
+		return fence_wait(dma, false);
+	}
+
+	cb->fence = i915_sw_fence_get(fence);
+	i915_sw_fence_await(fence);
+
+	cb->dma = NULL;
+	__setup_timer(&cb->timer,
+		      timer_i915_sw_fence_wake, (unsigned long)cb,
+		      TIMER_IRQSAFE);
+	if (timeout) {
+		cb->dma = fence_get(dma);
+		mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
+	}
+
+	ret = fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
+	if (ret == 0) {
+		ret = 1;
+	} else {
+		dma_i915_sw_fence_wake(dma, &cb->base);
+		if (ret == -ENOENT) /* fence already signaled */
+			ret = 0;
+	}
+
+	return ret;
+}
+
+int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
+				    struct reservation_object *resv,
+				    const struct fence_ops *exclude,
+				    bool write,
+				    unsigned long timeout,
+				    gfp_t gfp)
+{
+	struct fence *excl;
+	int ret = 0, pending;
+
+	if (write) {
+		struct fence **shared;
+		unsigned int count, i;
+
+		ret = reservation_object_get_fences_rcu(resv,
+							&excl, &count, &shared);
+		if (ret)
+			return ret;
+
+		for (i = 0; i < count; i++) {
+			if (shared[i]->ops == exclude)
+				continue;
+
+			pending = i915_sw_fence_await_dma_fence(fence,
+								shared[i],
+								timeout,
+								gfp);
+			if (pending < 0) {
+				ret = pending;
+				break;
+			}
+
+			ret |= pending;
+		}
+
+		for (i = 0; i < count; i++)
+			fence_put(shared[i]);
+		kfree(shared);
+	} else {
+		excl = reservation_object_get_excl_rcu(resv);
+	}
+
+	if (ret >= 0 && excl && excl->ops != exclude) {
+		pending = i915_sw_fence_await_dma_fence(fence,
+							excl,
+							timeout,
+							gfp);
+		if (pending < 0)
+			ret = pending;
+		else
+			ret |= pending;
+	}
+
+	fence_put(excl);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h
new file mode 100644
index 000000000000..373141602ca4
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_sw_fence.h
@@ -0,0 +1,65 @@
+/*
+ * i915_sw_fence.h - library routines for N:M synchronisation points
+ *
+ * Copyright (C) 2016 Intel Corporation
+ *
+ * This file is released under the GPLv2.
+ *
+ */
+
+#ifndef _I915_SW_FENCE_H_
+#define _I915_SW_FENCE_H_
+
+#include <linux/gfp.h>
+#include <linux/kref.h>
+#include <linux/notifier.h> /* for NOTIFY_DONE */
+#include <linux/wait.h>
+
+struct completion;
+struct fence;
+struct fence_ops;
+struct reservation_object;
+
+struct i915_sw_fence {
+	wait_queue_head_t wait;
+	unsigned long flags;
+	struct kref kref;
+	atomic_t pending;
+};
+
+#define I915_SW_FENCE_CHECKED_BIT	0 /* used internally for DAG checking */
+#define I915_SW_FENCE_PRIVATE_BIT	1 /* available for use by owner */
+#define I915_SW_FENCE_MASK		(~3)
+
+enum i915_sw_fence_notify {
+	FENCE_COMPLETE,
+	FENCE_FREE
+};
+
+typedef int (*i915_sw_fence_notify_t)(struct i915_sw_fence *,
+				      enum i915_sw_fence_notify state);
+#define __i915_sw_fence_call __aligned(4)
+
+void i915_sw_fence_init(struct i915_sw_fence *fence, i915_sw_fence_notify_t fn);
+void i915_sw_fence_commit(struct i915_sw_fence *fence);
+
+int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
+				 struct i915_sw_fence *after,
+				 wait_queue_t *wq);
+int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
+				  struct fence *dma,
+				  unsigned long timeout,
+				  gfp_t gfp);
+int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
+				    struct reservation_object *resv,
+				    const struct fence_ops *exclude,
+				    bool write,
+				    unsigned long timeout,
+				    gfp_t gfp);
+
+static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence)
+{
+	return atomic_read(&fence->pending) < 0;
+}
+
+#endif /* _I915_SW_FENCE_H_ */
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index d61829e54f93..1012eeea1324 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -32,13 +32,16 @@
 #include "intel_drv.h"
 #include "i915_drv.h"
 
-#define dev_to_drm_minor(d) dev_get_drvdata((d))
+static inline struct drm_i915_private *kdev_minor_to_i915(struct device *kdev)
+{
+	struct drm_minor *minor = dev_get_drvdata(kdev);
+	return to_i915(minor->dev);
+}
 
 #ifdef CONFIG_PM
-static u32 calc_residency(struct drm_device *dev,
+static u32 calc_residency(struct drm_i915_private *dev_priv,
 			  i915_reg_t reg)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	u64 raw_time; /* 32b value may overflow during fixed point math */
 	u64 units = 128ULL, div = 100000ULL;
 	u32 ret;
@@ -49,13 +52,13 @@ static u32 calc_residency(struct drm_device *dev,
 	intel_runtime_pm_get(dev_priv);
 
 	/* On VLV and CHV, residency time is in CZ units rather than 1.28us */
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		units = 1;
 		div = dev_priv->czclk_freq;
 
 		if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
 			units <<= 8;
-	} else if (IS_BROXTON(dev)) {
+	} else if (IS_BROXTON(dev_priv)) {
 		units = 1;
 		div = 1200;		/* 833.33ns */
 	}
@@ -76,32 +79,32 @@ show_rc6_mask(struct device *kdev, struct device_attribute *attr, char *buf)
 static ssize_t
 show_rc6_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *dminor = dev_get_drvdata(kdev);
-	u32 rc6_residency = calc_residency(dminor->dev, GEN6_GT_GFX_RC6);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+	u32 rc6_residency = calc_residency(dev_priv, GEN6_GT_GFX_RC6);
 	return snprintf(buf, PAGE_SIZE, "%u\n", rc6_residency);
 }
 
 static ssize_t
 show_rc6p_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *dminor = dev_to_drm_minor(kdev);
-	u32 rc6p_residency = calc_residency(dminor->dev, GEN6_GT_GFX_RC6p);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+	u32 rc6p_residency = calc_residency(dev_priv, GEN6_GT_GFX_RC6p);
 	return snprintf(buf, PAGE_SIZE, "%u\n", rc6p_residency);
 }
 
 static ssize_t
 show_rc6pp_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *dminor = dev_to_drm_minor(kdev);
-	u32 rc6pp_residency = calc_residency(dminor->dev, GEN6_GT_GFX_RC6pp);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+	u32 rc6pp_residency = calc_residency(dev_priv, GEN6_GT_GFX_RC6pp);
 	return snprintf(buf, PAGE_SIZE, "%u\n", rc6pp_residency);
 }
 
 static ssize_t
 show_media_rc6_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *dminor = dev_get_drvdata(kdev);
-	u32 rc6_residency = calc_residency(dminor->dev, VLV_GT_MEDIA_RC6);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+	u32 rc6_residency = calc_residency(dev_priv, VLV_GT_MEDIA_RC6);
 	return snprintf(buf, PAGE_SIZE, "%u\n", rc6_residency);
 }
 
@@ -144,9 +147,9 @@ static struct attribute_group media_rc6_attr_group = {
 };
 #endif
 
-static int l3_access_valid(struct drm_device *dev, loff_t offset)
+static int l3_access_valid(struct drm_i915_private *dev_priv, loff_t offset)
 {
-	if (!HAS_L3_DPF(dev))
+	if (!HAS_L3_DPF(dev_priv))
 		return -EPERM;
 
 	if (offset % 4 != 0)
@@ -163,22 +166,21 @@ i915_l3_read(struct file *filp, struct kobject *kobj,
 	     struct bin_attribute *attr, char *buf,
 	     loff_t offset, size_t count)
 {
-	struct device *dev = kobj_to_dev(kobj);
-	struct drm_minor *dminor = dev_to_drm_minor(dev);
-	struct drm_device *drm_dev = dminor->dev;
-	struct drm_i915_private *dev_priv = to_i915(drm_dev);
+	struct device *kdev = kobj_to_dev(kobj);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+	struct drm_device *dev = &dev_priv->drm;
 	int slice = (int)(uintptr_t)attr->private;
 	int ret;
 
 	count = round_down(count, 4);
 
-	ret = l3_access_valid(drm_dev, offset);
+	ret = l3_access_valid(dev_priv, offset);
 	if (ret)
 		return ret;
 
 	count = min_t(size_t, GEN7_L3LOG_SIZE - offset, count);
 
-	ret = i915_mutex_lock_interruptible(drm_dev);
+	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
 		return ret;
 
@@ -189,7 +191,7 @@ i915_l3_read(struct file *filp, struct kobject *kobj,
 	else
 		memset(buf, 0, count);
 
-	mutex_unlock(&drm_dev->struct_mutex);
+	mutex_unlock(&dev->struct_mutex);
 
 	return count;
 }
@@ -199,30 +201,29 @@ i915_l3_write(struct file *filp, struct kobject *kobj,
 	      struct bin_attribute *attr, char *buf,
 	      loff_t offset, size_t count)
 {
-	struct device *dev = kobj_to_dev(kobj);
-	struct drm_minor *dminor = dev_to_drm_minor(dev);
-	struct drm_device *drm_dev = dminor->dev;
-	struct drm_i915_private *dev_priv = to_i915(drm_dev);
+	struct device *kdev = kobj_to_dev(kobj);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+	struct drm_device *dev = &dev_priv->drm;
 	struct i915_gem_context *ctx;
 	u32 *temp = NULL; /* Just here to make handling failures easy */
 	int slice = (int)(uintptr_t)attr->private;
 	int ret;
 
-	if (!HAS_HW_CONTEXTS(drm_dev))
+	if (!HAS_HW_CONTEXTS(dev_priv))
 		return -ENXIO;
 
-	ret = l3_access_valid(drm_dev, offset);
+	ret = l3_access_valid(dev_priv, offset);
 	if (ret)
 		return ret;
 
-	ret = i915_mutex_lock_interruptible(drm_dev);
+	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
 		return ret;
 
 	if (!dev_priv->l3_parity.remap_info[slice]) {
 		temp = kzalloc(GEN7_L3LOG_SIZE, GFP_KERNEL);
 		if (!temp) {
-			mutex_unlock(&drm_dev->struct_mutex);
+			mutex_unlock(&dev->struct_mutex);
 			return -ENOMEM;
 		}
 	}
@@ -240,7 +241,7 @@ i915_l3_write(struct file *filp, struct kobject *kobj,
 	list_for_each_entry(ctx, &dev_priv->context_list, link)
 		ctx->remap_slice |= (1<<slice);
 
-	mutex_unlock(&drm_dev->struct_mutex);
+	mutex_unlock(&dev->struct_mutex);
 
 	return count;
 }
@@ -266,13 +267,9 @@ static struct bin_attribute dpf_attrs_1 = {
 static ssize_t gt_act_freq_mhz_show(struct device *kdev,
 				    struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 	int ret;
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
 	intel_runtime_pm_get(dev_priv);
 
 	mutex_lock(&dev_priv->rps.hw_lock);
@@ -300,59 +297,70 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev,
 static ssize_t gt_cur_freq_mhz_show(struct device *kdev,
 				    struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret;
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			intel_gpu_freq(dev_priv,
+				       dev_priv->rps.cur_freq));
+}
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
+static ssize_t gt_boost_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 
-	intel_runtime_pm_get(dev_priv);
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			intel_gpu_freq(dev_priv,
+				       dev_priv->rps.boost_freq));
+}
+
+static ssize_t gt_boost_freq_mhz_store(struct device *kdev,
+				       struct device_attribute *attr,
+				       const char *buf, size_t count)
+{
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+	u32 val;
+	ssize_t ret;
+
+	ret = kstrtou32(buf, 0, &val);
+	if (ret)
+		return ret;
+
+	/* Validate against (static) hardware limits */
+	val = intel_freq_opcode(dev_priv, val);
+	if (val < dev_priv->rps.min_freq || val > dev_priv->rps.max_freq)
+		return -EINVAL;
 
 	mutex_lock(&dev_priv->rps.hw_lock);
-	ret = intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq);
+	dev_priv->rps.boost_freq = val;
 	mutex_unlock(&dev_priv->rps.hw_lock);
 
-	intel_runtime_pm_put(dev_priv);
-
-	return snprintf(buf, PAGE_SIZE, "%d\n", ret);
+	return count;
 }
 
 static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev,
 				     struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 
-	return snprintf(buf, PAGE_SIZE,
-			"%d\n",
-			intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq));
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			intel_gpu_freq(dev_priv,
+				       dev_priv->rps.efficient_freq));
 }
 
 static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret;
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
-	mutex_lock(&dev_priv->rps.hw_lock);
-	ret = intel_gpu_freq(dev_priv, dev_priv->rps.max_freq_softlimit);
-	mutex_unlock(&dev_priv->rps.hw_lock);
-
-	return snprintf(buf, PAGE_SIZE, "%d\n", ret);
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			intel_gpu_freq(dev_priv,
+				       dev_priv->rps.max_freq_softlimit));
 }
 
 static ssize_t gt_max_freq_mhz_store(struct device *kdev,
 				     struct device_attribute *attr,
 				     const char *buf, size_t count)
 {
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 	u32 val;
 	ssize_t ret;
 
@@ -360,8 +368,6 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
 	if (ret)
 		return ret;
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
 	intel_runtime_pm_get(dev_priv);
 
 	mutex_lock(&dev_priv->rps.hw_lock);
@@ -400,27 +406,18 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
 
 static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret;
-
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 
-	mutex_lock(&dev_priv->rps.hw_lock);
-	ret = intel_gpu_freq(dev_priv, dev_priv->rps.min_freq_softlimit);
-	mutex_unlock(&dev_priv->rps.hw_lock);
-
-	return snprintf(buf, PAGE_SIZE, "%d\n", ret);
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			intel_gpu_freq(dev_priv,
+				       dev_priv->rps.min_freq_softlimit));
 }
 
 static ssize_t gt_min_freq_mhz_store(struct device *kdev,
 				     struct device_attribute *attr,
 				     const char *buf, size_t count)
 {
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 	u32 val;
 	ssize_t ret;
 
@@ -428,8 +425,6 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
 	if (ret)
 		return ret;
 
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
 	intel_runtime_pm_get(dev_priv);
 
 	mutex_lock(&dev_priv->rps.hw_lock);
@@ -465,6 +460,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
 
 static DEVICE_ATTR(gt_act_freq_mhz, S_IRUGO, gt_act_freq_mhz_show, NULL);
 static DEVICE_ATTR(gt_cur_freq_mhz, S_IRUGO, gt_cur_freq_mhz_show, NULL);
+static DEVICE_ATTR(gt_boost_freq_mhz, S_IRUGO, gt_boost_freq_mhz_show, gt_boost_freq_mhz_store);
 static DEVICE_ATTR(gt_max_freq_mhz, S_IRUGO | S_IWUSR, gt_max_freq_mhz_show, gt_max_freq_mhz_store);
 static DEVICE_ATTR(gt_min_freq_mhz, S_IRUGO | S_IWUSR, gt_min_freq_mhz_show, gt_min_freq_mhz_store);
 
@@ -478,9 +474,7 @@ static DEVICE_ATTR(gt_RPn_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL);
 /* For now we have a static number of RP states */
 static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 	u32 val;
 
 	if (attr == &dev_attr_gt_RP0_freq_mhz)
@@ -498,6 +492,7 @@ static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr
 static const struct attribute *gen6_attrs[] = {
 	&dev_attr_gt_act_freq_mhz.attr,
 	&dev_attr_gt_cur_freq_mhz.attr,
+	&dev_attr_gt_boost_freq_mhz.attr,
 	&dev_attr_gt_max_freq_mhz.attr,
 	&dev_attr_gt_min_freq_mhz.attr,
 	&dev_attr_gt_RP0_freq_mhz.attr,
@@ -509,6 +504,7 @@ static const struct attribute *gen6_attrs[] = {
 static const struct attribute *vlv_attrs[] = {
 	&dev_attr_gt_act_freq_mhz.attr,
 	&dev_attr_gt_cur_freq_mhz.attr,
+	&dev_attr_gt_boost_freq_mhz.attr,
 	&dev_attr_gt_max_freq_mhz.attr,
 	&dev_attr_gt_min_freq_mhz.attr,
 	&dev_attr_gt_RP0_freq_mhz.attr,
@@ -524,8 +520,8 @@ static ssize_t error_state_read(struct file *filp, struct kobject *kobj,
 {
 
 	struct device *kdev = kobj_to_dev(kobj);
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+	struct drm_device *dev = &dev_priv->drm;
 	struct i915_error_state_file_priv error_priv;
 	struct drm_i915_error_state_buf error_str;
 	ssize_t ret_count = 0;
@@ -559,18 +555,10 @@ static ssize_t error_state_write(struct file *file, struct kobject *kobj,
 				 loff_t off, size_t count)
 {
 	struct device *kdev = kobj_to_dev(kobj);
-	struct drm_minor *minor = dev_to_drm_minor(kdev);
-	struct drm_device *dev = minor->dev;
-	int ret;
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 
 	DRM_DEBUG_DRIVER("Resetting error state\n");
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-
-	i915_destroy_error_state(dev);
-	mutex_unlock(&dev->struct_mutex);
+	i915_destroy_error_state(&dev_priv->drm);
 
 	return count;
 }
@@ -583,37 +571,38 @@ static struct bin_attribute error_state_attr = {
 	.write = error_state_write,
 };
 
-void i915_setup_sysfs(struct drm_device *dev)
+void i915_setup_sysfs(struct drm_i915_private *dev_priv)
 {
+	struct device *kdev = dev_priv->drm.primary->kdev;
 	int ret;
 
 #ifdef CONFIG_PM
-	if (HAS_RC6(dev)) {
-		ret = sysfs_merge_group(&dev->primary->kdev->kobj,
+	if (HAS_RC6(dev_priv)) {
+		ret = sysfs_merge_group(&kdev->kobj,
 					&rc6_attr_group);
 		if (ret)
 			DRM_ERROR("RC6 residency sysfs setup failed\n");
 	}
-	if (HAS_RC6p(dev)) {
-		ret = sysfs_merge_group(&dev->primary->kdev->kobj,
+	if (HAS_RC6p(dev_priv)) {
+		ret = sysfs_merge_group(&kdev->kobj,
 					&rc6p_attr_group);
 		if (ret)
 			DRM_ERROR("RC6p residency sysfs setup failed\n");
 	}
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
-		ret = sysfs_merge_group(&dev->primary->kdev->kobj,
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+		ret = sysfs_merge_group(&kdev->kobj,
 					&media_rc6_attr_group);
 		if (ret)
 			DRM_ERROR("Media RC6 residency sysfs setup failed\n");
 	}
 #endif
-	if (HAS_L3_DPF(dev)) {
-		ret = device_create_bin_file(dev->primary->kdev, &dpf_attrs);
+	if (HAS_L3_DPF(dev_priv)) {
+		ret = device_create_bin_file(kdev, &dpf_attrs);
 		if (ret)
 			DRM_ERROR("l3 parity sysfs setup failed\n");
 
-		if (NUM_L3_SLICES(dev) > 1) {
-			ret = device_create_bin_file(dev->primary->kdev,
+		if (NUM_L3_SLICES(dev_priv) > 1) {
+			ret = device_create_bin_file(kdev,
 						     &dpf_attrs_1);
 			if (ret)
 				DRM_ERROR("l3 parity slice 1 setup failed\n");
@@ -621,30 +610,32 @@ void i915_setup_sysfs(struct drm_device *dev)
 	}
 
 	ret = 0;
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
-		ret = sysfs_create_files(&dev->primary->kdev->kobj, vlv_attrs);
-	else if (INTEL_INFO(dev)->gen >= 6)
-		ret = sysfs_create_files(&dev->primary->kdev->kobj, gen6_attrs);
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		ret = sysfs_create_files(&kdev->kobj, vlv_attrs);
+	else if (INTEL_GEN(dev_priv) >= 6)
+		ret = sysfs_create_files(&kdev->kobj, gen6_attrs);
 	if (ret)
 		DRM_ERROR("RPS sysfs setup failed\n");
 
-	ret = sysfs_create_bin_file(&dev->primary->kdev->kobj,
+	ret = sysfs_create_bin_file(&kdev->kobj,
 				    &error_state_attr);
 	if (ret)
 		DRM_ERROR("error_state sysfs setup failed\n");
 }
 
-void i915_teardown_sysfs(struct drm_device *dev)
+void i915_teardown_sysfs(struct drm_i915_private *dev_priv)
 {
-	sysfs_remove_bin_file(&dev->primary->kdev->kobj, &error_state_attr);
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
-		sysfs_remove_files(&dev->primary->kdev->kobj, vlv_attrs);
+	struct device *kdev = dev_priv->drm.primary->kdev;
+
+	sysfs_remove_bin_file(&kdev->kobj, &error_state_attr);
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		sysfs_remove_files(&kdev->kobj, vlv_attrs);
 	else
-		sysfs_remove_files(&dev->primary->kdev->kobj, gen6_attrs);
-	device_remove_bin_file(dev->primary->kdev,  &dpf_attrs_1);
-	device_remove_bin_file(dev->primary->kdev,  &dpf_attrs);
+		sysfs_remove_files(&kdev->kobj, gen6_attrs);
+	device_remove_bin_file(kdev,  &dpf_attrs_1);
+	device_remove_bin_file(kdev,  &dpf_attrs);
 #ifdef CONFIG_PM
-	sysfs_unmerge_group(&dev->primary->kdev->kobj, &rc6_attr_group);
-	sysfs_unmerge_group(&dev->primary->kdev->kobj, &rc6p_attr_group);
+	sysfs_unmerge_group(&kdev->kobj, &rc6_attr_group);
+	sysfs_unmerge_group(&kdev->kobj, &rc6p_attr_group);
 #endif
 }
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index 534154e05fbe..178798002a73 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -394,25 +394,27 @@ DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy,
 );
 
 TRACE_EVENT(i915_gem_evict,
-	    TP_PROTO(struct drm_device *dev, u32 size, u32 align, unsigned flags),
-	    TP_ARGS(dev, size, align, flags),
+	    TP_PROTO(struct i915_address_space *vm, u32 size, u32 align, unsigned int flags),
+	    TP_ARGS(vm, size, align, flags),
 
 	    TP_STRUCT__entry(
 			     __field(u32, dev)
+			     __field(struct i915_address_space *, vm)
 			     __field(u32, size)
 			     __field(u32, align)
-			     __field(unsigned, flags)
+			     __field(unsigned int, flags)
 			    ),
 
 	    TP_fast_assign(
-			   __entry->dev = dev->primary->index;
+			   __entry->dev = vm->dev->primary->index;
+			   __entry->vm = vm;
 			   __entry->size = size;
 			   __entry->align = align;
 			   __entry->flags = flags;
 			  ),
 
-	    TP_printk("dev=%d, size=%d, align=%d %s",
-		      __entry->dev, __entry->size, __entry->align,
+	    TP_printk("dev=%d, vm=%p, size=%d, align=%d %s",
+		      __entry->dev, __entry->vm, __entry->size, __entry->align,
 		      __entry->flags & PIN_MAPPABLE ? ", mappable" : "")
 );
 
@@ -449,10 +451,9 @@ TRACE_EVENT(i915_gem_evict_vm,
 );
 
 TRACE_EVENT(i915_gem_ring_sync_to,
-	    TP_PROTO(struct drm_i915_gem_request *to_req,
-		     struct intel_engine_cs *from,
-		     struct drm_i915_gem_request *req),
-	    TP_ARGS(to_req, from, req),
+	    TP_PROTO(struct drm_i915_gem_request *to,
+		     struct drm_i915_gem_request *from),
+	    TP_ARGS(to, from),
 
 	    TP_STRUCT__entry(
 			     __field(u32, dev)
@@ -463,9 +464,9 @@ TRACE_EVENT(i915_gem_ring_sync_to,
 
 	    TP_fast_assign(
 			   __entry->dev = from->i915->drm.primary->index;
-			   __entry->sync_from = from->id;
-			   __entry->sync_to = to_req->engine->id;
-			   __entry->seqno = i915_gem_request_get_seqno(req);
+			   __entry->sync_from = from->engine->id;
+			   __entry->sync_to = to->engine->id;
+			   __entry->seqno = from->fence.seqno;
 			   ),
 
 	    TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%u",
@@ -488,9 +489,9 @@ TRACE_EVENT(i915_gem_ring_dispatch,
 	    TP_fast_assign(
 			   __entry->dev = req->i915->drm.primary->index;
 			   __entry->ring = req->engine->id;
-			   __entry->seqno = req->seqno;
+			   __entry->seqno = req->fence.seqno;
 			   __entry->flags = flags;
-			   intel_engine_enable_signaling(req);
+			   fence_enable_sw_signaling(&req->fence);
 			   ),
 
 	    TP_printk("dev=%u, ring=%u, seqno=%u, flags=%x",
@@ -533,7 +534,7 @@ DECLARE_EVENT_CLASS(i915_gem_request,
 	    TP_fast_assign(
 			   __entry->dev = req->i915->drm.primary->index;
 			   __entry->ring = req->engine->id;
-			   __entry->seqno = req->seqno;
+			   __entry->seqno = req->fence.seqno;
 			   ),
 
 	    TP_printk("dev=%u, ring=%u, seqno=%u",
@@ -595,7 +596,7 @@ TRACE_EVENT(i915_gem_request_wait_begin,
 	    TP_fast_assign(
 			   __entry->dev = req->i915->drm.primary->index;
 			   __entry->ring = req->engine->id;
-			   __entry->seqno = req->seqno;
+			   __entry->seqno = req->fence.seqno;
 			   __entry->blocking =
 				     mutex_is_locked(&req->i915->drm.struct_mutex);
 			   ),
diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c
index f6acb5a0e701..dae340cfc6c7 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.c
+++ b/drivers/gpu/drm/i915/i915_vgpu.c
@@ -65,9 +65,6 @@ void i915_check_vgpu(struct drm_i915_private *dev_priv)
 
 	BUILD_BUG_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE);
 
-	if (!IS_HASWELL(dev_priv))
-		return;
-
 	magic = __raw_i915_read64(dev_priv, vgtif_reg(magic));
 	if (magic != VGT_MAGIC)
 		return;
@@ -97,6 +94,7 @@ static struct _balloon_info_ bl_info;
 
 /**
  * intel_vgt_deballoon - deballoon reserved graphics address trunks
+ * @dev_priv: i915 device private data
  *
  * This function is called to deallocate the ballooned-out graphic memory, when
  * driver is unloaded or when ballooning fails.
@@ -138,7 +136,7 @@ static int vgt_balloon_space(struct drm_mm *mm,
 
 /**
  * intel_vgt_balloon - balloon out reserved graphics address trunks
- * @dev: drm device
+ * @dev_priv: i915 device private data
  *
  * This function is called at the initialization stage, to balloon out the
  * graphic address space allocated to other vGPUs, by marking these spaces as
@@ -155,27 +153,27 @@ static int vgt_balloon_space(struct drm_mm *mm,
  * host point of view, the graphic address space is partitioned by multiple
  * vGPUs in different VMs. ::
  *
- *                        vGPU1 view         Host view
- *             0 ------> +-----------+     +-----------+
- *               ^       |###########|     |   vGPU3   |
- *               |       |###########|     +-----------+
- *               |       |###########|     |   vGPU2   |
- *               |       +-----------+     +-----------+
- *        mappable GM    | available | ==> |   vGPU1   |
- *               |       +-----------+     +-----------+
- *               |       |###########|     |           |
- *               v       |###########|     |   Host    |
- *               +=======+===========+     +===========+
- *               ^       |###########|     |   vGPU3   |
- *               |       |###########|     +-----------+
- *               |       |###########|     |   vGPU2   |
- *               |       +-----------+     +-----------+
- *      unmappable GM    | available | ==> |   vGPU1   |
- *               |       +-----------+     +-----------+
- *               |       |###########|     |           |
- *               |       |###########|     |   Host    |
- *               v       |###########|     |           |
- * total GM size ------> +-----------+     +-----------+
+ *                         vGPU1 view         Host view
+ *              0 ------> +-----------+     +-----------+
+ *                ^       |###########|     |   vGPU3   |
+ *                |       |###########|     +-----------+
+ *                |       |###########|     |   vGPU2   |
+ *                |       +-----------+     +-----------+
+ *         mappable GM    | available | ==> |   vGPU1   |
+ *                |       +-----------+     +-----------+
+ *                |       |###########|     |           |
+ *                v       |###########|     |   Host    |
+ *                +=======+===========+     +===========+
+ *                ^       |###########|     |   vGPU3   |
+ *                |       |###########|     +-----------+
+ *                |       |###########|     |   vGPU2   |
+ *                |       +-----------+     +-----------+
+ *       unmappable GM    | available | ==> |   vGPU1   |
+ *                |       +-----------+     +-----------+
+ *                |       |###########|     |           |
+ *                |       |###########|     |   Host    |
+ *                v       |###########|     |           |
+ *  total GM size ------> +-----------+     +-----------+
  *
  * Returns:
  * zero on success, non-zero if configuration invalid or ballooning failed
diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c
index 7de7721f65bc..b82de3072d4f 100644
--- a/drivers/gpu/drm/i915/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/intel_atomic_plane.c
@@ -55,7 +55,7 @@ intel_create_plane_state(struct drm_plane *plane)
 		return NULL;
 
 	state->base.plane = plane;
-	state->base.rotation = BIT(DRM_ROTATE_0);
+	state->base.rotation = DRM_ROTATE_0;
 	state->ckey.flags = I915_SET_COLORKEY_NONE;
 
 	return state;
@@ -134,20 +134,6 @@ static int intel_plane_atomic_check(struct drm_plane *plane,
 
 	crtc_state = to_intel_crtc_state(drm_crtc_state);
 
-	/*
-	 * The original src/dest coordinates are stored in state->base, but
-	 * we want to keep another copy internal to our driver that we can
-	 * clip/modify ourselves.
-	 */
-	intel_state->src.x1 = state->src_x;
-	intel_state->src.y1 = state->src_y;
-	intel_state->src.x2 = state->src_x + state->src_w;
-	intel_state->src.y2 = state->src_y + state->src_h;
-	intel_state->dst.x1 = state->crtc_x;
-	intel_state->dst.y1 = state->crtc_y;
-	intel_state->dst.x2 = state->crtc_x + state->crtc_w;
-	intel_state->dst.y2 = state->crtc_y + state->crtc_h;
-
 	/* Clip all planes to CRTC size, or 0x0 if CRTC is disabled */
 	intel_state->clip.x1 = 0;
 	intel_state->clip.y1 = 0;
@@ -157,6 +143,7 @@ static int intel_plane_atomic_check(struct drm_plane *plane,
 		crtc_state->base.enable ? crtc_state->pipe_src_h : 0;
 
 	if (state->fb && intel_rotation_90_or_270(state->rotation)) {
+		char *format_name;
 		if (!(state->fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
 			state->fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)) {
 			DRM_DEBUG_KMS("Y/Yf tiling required for 90/270!\n");
@@ -171,8 +158,9 @@ static int intel_plane_atomic_check(struct drm_plane *plane,
 		switch (state->fb->pixel_format) {
 		case DRM_FORMAT_C8:
 		case DRM_FORMAT_RGB565:
-			DRM_DEBUG_KMS("Unsupported pixel format %s for 90/270!\n",
-					drm_get_format_name(state->fb->pixel_format));
+			format_name = drm_get_format_name(state->fb->pixel_format);
+			DRM_DEBUG_KMS("Unsupported pixel format %s for 90/270!\n", format_name);
+			kfree(format_name);
 			return -EINVAL;
 
 		default:
@@ -180,7 +168,7 @@ static int intel_plane_atomic_check(struct drm_plane *plane,
 		}
 	}
 
-	intel_state->visible = false;
+	intel_state->base.visible = false;
 	ret = intel_plane->check_plane(plane, crtc_state, intel_state);
 	if (ret)
 		return ret;
@@ -196,7 +184,7 @@ static void intel_plane_atomic_update(struct drm_plane *plane,
 		to_intel_plane_state(plane->state);
 	struct drm_crtc *crtc = plane->state->crtc ?: old_state->crtc;
 
-	if (intel_state->visible)
+	if (intel_state->base.visible)
 		intel_plane->update_plane(plane,
 					  to_intel_crtc_state(crtc->state),
 					  intel_state);
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 6700a7be7f78..6c70a5bfd7d8 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -51,10 +51,10 @@
  * related registers. (The notable exception is the power management, not
  * covered here.)
  *
- * The struct i915_audio_component is used to interact between the graphics
- * and audio drivers. The struct i915_audio_component_ops *ops in it is
+ * The struct &i915_audio_component is used to interact between the graphics
+ * and audio drivers. The struct &i915_audio_component_ops @ops in it is
  * defined in graphics driver and called in audio driver. The
- * struct i915_audio_component_audio_ops *audio_ops is called from i915 driver.
+ * struct &i915_audio_component_audio_ops @audio_ops is called from i915 driver.
  */
 
 static const struct {
@@ -359,9 +359,7 @@ static void ilk_audio_codec_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-	struct intel_digital_port *intel_dig_port =
-		enc_to_dig_port(&encoder->base);
-	enum port port = intel_dig_port->port;
+	enum port port = enc_to_dig_port(&encoder->base)->port;
 	enum pipe pipe = intel_crtc->pipe;
 	uint32_t tmp, eldv;
 	i915_reg_t aud_config, aud_cntrl_st2;
@@ -407,13 +405,10 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-	struct intel_digital_port *intel_dig_port =
-		enc_to_dig_port(&encoder->base);
-	enum port port = intel_dig_port->port;
+	enum port port = enc_to_dig_port(&encoder->base)->port;
 	enum pipe pipe = intel_crtc->pipe;
 	uint8_t *eld = connector->eld;
-	uint32_t eldv;
-	uint32_t tmp;
+	uint32_t tmp, eldv;
 	int len, i;
 	i915_reg_t hdmiw_hdmiedid, aud_config, aud_cntl_st, aud_cntrl_st2;
 
@@ -581,25 +576,27 @@ void intel_init_audio_hooks(struct drm_i915_private *dev_priv)
 	}
 }
 
-static void i915_audio_component_get_power(struct device *dev)
+static void i915_audio_component_get_power(struct device *kdev)
 {
-	intel_display_power_get(dev_to_i915(dev), POWER_DOMAIN_AUDIO);
+	intel_display_power_get(kdev_to_i915(kdev), POWER_DOMAIN_AUDIO);
 }
 
-static void i915_audio_component_put_power(struct device *dev)
+static void i915_audio_component_put_power(struct device *kdev)
 {
-	intel_display_power_put(dev_to_i915(dev), POWER_DOMAIN_AUDIO);
+	intel_display_power_put(kdev_to_i915(kdev), POWER_DOMAIN_AUDIO);
 }
 
-static void i915_audio_component_codec_wake_override(struct device *dev,
+static void i915_audio_component_codec_wake_override(struct device *kdev,
 						     bool enable)
 {
-	struct drm_i915_private *dev_priv = dev_to_i915(dev);
+	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
 	u32 tmp;
 
 	if (!IS_SKYLAKE(dev_priv) && !IS_KABYLAKE(dev_priv))
 		return;
 
+	i915_audio_component_get_power(kdev);
+
 	/*
 	 * Enable/disable generating the codec wake signal, overriding the
 	 * internal logic to generate the codec wake to controller.
@@ -615,12 +612,14 @@ static void i915_audio_component_codec_wake_override(struct device *dev,
 		I915_WRITE(HSW_AUD_CHICKENBIT, tmp);
 		usleep_range(1000, 1500);
 	}
+
+	i915_audio_component_put_power(kdev);
 }
 
 /* Get CDCLK in kHz  */
-static int i915_audio_component_get_cdclk_freq(struct device *dev)
+static int i915_audio_component_get_cdclk_freq(struct device *kdev)
 {
-	struct drm_i915_private *dev_priv = dev_to_i915(dev);
+	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
 
 	if (WARN_ON_ONCE(!HAS_DDI(dev_priv)))
 		return -ENODEV;
@@ -628,10 +627,10 @@ static int i915_audio_component_get_cdclk_freq(struct device *dev)
 	return dev_priv->cdclk_freq;
 }
 
-static int i915_audio_component_sync_audio_rate(struct device *dev,
+static int i915_audio_component_sync_audio_rate(struct device *kdev,
 						int port, int rate)
 {
-	struct drm_i915_private *dev_priv = dev_to_i915(dev);
+	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
 	struct intel_encoder *intel_encoder;
 	struct intel_crtc *crtc;
 	struct drm_display_mode *mode;
@@ -648,6 +647,7 @@ static int i915_audio_component_sync_audio_rate(struct device *dev,
 	    !IS_HASWELL(dev_priv))
 		return 0;
 
+	i915_audio_component_get_power(kdev);
 	mutex_lock(&dev_priv->av_mutex);
 	/* 1. get the pipe */
 	intel_encoder = dev_priv->dig_port_map[port];
@@ -698,14 +698,15 @@ static int i915_audio_component_sync_audio_rate(struct device *dev,
 
  unlock:
 	mutex_unlock(&dev_priv->av_mutex);
+	i915_audio_component_put_power(kdev);
 	return err;
 }
 
-static int i915_audio_component_get_eld(struct device *dev, int port,
+static int i915_audio_component_get_eld(struct device *kdev, int port,
 					bool *enabled,
 					unsigned char *buf, int max_bytes)
 {
-	struct drm_i915_private *dev_priv = dev_to_i915(dev);
+	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
 	struct intel_encoder *intel_encoder;
 	struct intel_digital_port *intel_dig_port;
 	const u8 *eld;
@@ -739,11 +740,11 @@ static const struct i915_audio_component_ops i915_audio_component_ops = {
 	.get_eld	= i915_audio_component_get_eld,
 };
 
-static int i915_audio_component_bind(struct device *i915_dev,
-				     struct device *hda_dev, void *data)
+static int i915_audio_component_bind(struct device *i915_kdev,
+				     struct device *hda_kdev, void *data)
 {
 	struct i915_audio_component *acomp = data;
-	struct drm_i915_private *dev_priv = dev_to_i915(i915_dev);
+	struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev);
 	int i;
 
 	if (WARN_ON(acomp->ops || acomp->dev))
@@ -751,7 +752,7 @@ static int i915_audio_component_bind(struct device *i915_dev,
 
 	drm_modeset_lock_all(&dev_priv->drm);
 	acomp->ops = &i915_audio_component_ops;
-	acomp->dev = i915_dev;
+	acomp->dev = i915_kdev;
 	BUILD_BUG_ON(MAX_PORTS != I915_MAX_PORTS);
 	for (i = 0; i < ARRAY_SIZE(acomp->aud_sample_rate); i++)
 		acomp->aud_sample_rate[i] = 0;
@@ -761,11 +762,11 @@ static int i915_audio_component_bind(struct device *i915_dev,
 	return 0;
 }
 
-static void i915_audio_component_unbind(struct device *i915_dev,
-					struct device *hda_dev, void *data)
+static void i915_audio_component_unbind(struct device *i915_kdev,
+					struct device *hda_kdev, void *data)
 {
 	struct i915_audio_component *acomp = data;
-	struct drm_i915_private *dev_priv = dev_to_i915(i915_dev);
+	struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev);
 
 	drm_modeset_lock_all(&dev_priv->drm);
 	acomp->ops = NULL;
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index b074f3d6d127..495611b7068d 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -26,6 +26,40 @@
 
 #include "i915_drv.h"
 
+static void intel_breadcrumbs_hangcheck(unsigned long data)
+{
+	struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
+	struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+	if (!b->irq_enabled)
+		return;
+
+	if (time_before(jiffies, b->timeout)) {
+		mod_timer(&b->hangcheck, b->timeout);
+		return;
+	}
+
+	DRM_DEBUG("Hangcheck timer elapsed... %s idle\n", engine->name);
+	set_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
+	mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1);
+
+	/* Ensure that even if the GPU hangs, we get woken up.
+	 *
+	 * However, note that if no one is waiting, we never notice
+	 * a gpu hang. Eventually, we will have to wait for a resource
+	 * held by the GPU and so trigger a hangcheck. In the most
+	 * pathological case, this will be upon memory starvation! To
+	 * prevent this, we also queue the hangcheck from the retire
+	 * worker.
+	 */
+	i915_queue_hangcheck(engine->i915);
+}
+
+static unsigned long wait_timeout(void)
+{
+	return round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES);
+}
+
 static void intel_breadcrumbs_fake_irq(unsigned long data)
 {
 	struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
@@ -37,10 +71,8 @@ static void intel_breadcrumbs_fake_irq(unsigned long data)
 	 * every jiffie in order to kick the oldest waiter to do the
 	 * coherent seqno check.
 	 */
-	rcu_read_lock();
 	if (intel_engine_wakeup(engine))
 		mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1);
-	rcu_read_unlock();
 }
 
 static void irq_enable(struct intel_engine_cs *engine)
@@ -91,17 +123,13 @@ static void __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
 	}
 
 	if (!b->irq_enabled ||
-	    test_bit(engine->id, &i915->gpu_error.missed_irq_rings))
+	    test_bit(engine->id, &i915->gpu_error.missed_irq_rings)) {
 		mod_timer(&b->fake_irq, jiffies + 1);
-
-	/* Ensure that even if the GPU hangs, we get woken up.
-	 *
-	 * However, note that if no one is waiting, we never notice
-	 * a gpu hang. Eventually, we will have to wait for a resource
-	 * held by the GPU and so trigger a hangcheck. In the most
-	 * pathological case, this will be upon memory starvation!
-	 */
-	i915_queue_hangcheck(i915);
+	} else {
+		/* Ensure we never sleep indefinitely */
+		GEM_BUG_ON(!time_after(b->timeout, jiffies));
+		mod_timer(&b->hangcheck, b->timeout);
+	}
 }
 
 static void __intel_breadcrumbs_disable_irq(struct intel_breadcrumbs *b)
@@ -204,7 +232,7 @@ static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 	}
 	rb_link_node(&wait->node, parent, p);
 	rb_insert_color(&wait->node, &b->waiters);
-	GEM_BUG_ON(!first && !b->irq_seqno_bh);
+	GEM_BUG_ON(!first && !rcu_access_pointer(b->irq_seqno_bh));
 
 	if (completed) {
 		struct rb_node *next = rb_next(completed);
@@ -212,8 +240,9 @@ static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 		GEM_BUG_ON(!next && !first);
 		if (next && next != &wait->node) {
 			GEM_BUG_ON(first);
+			b->timeout = wait_timeout();
 			b->first_wait = to_wait(next);
-			smp_store_mb(b->irq_seqno_bh, b->first_wait->tsk);
+			rcu_assign_pointer(b->irq_seqno_bh, b->first_wait->tsk);
 			/* As there is a delay between reading the current
 			 * seqno, processing the completed tasks and selecting
 			 * the next waiter, we may have missed the interrupt
@@ -238,8 +267,9 @@ static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 
 	if (first) {
 		GEM_BUG_ON(rb_first(&b->waiters) != &wait->node);
+		b->timeout = wait_timeout();
 		b->first_wait = wait;
-		smp_store_mb(b->irq_seqno_bh, wait->tsk);
+		rcu_assign_pointer(b->irq_seqno_bh, wait->tsk);
 		/* After assigning ourselves as the new bottom-half, we must
 		 * perform a cursory check to prevent a missed interrupt.
 		 * Either we miss the interrupt whilst programming the hardware,
@@ -250,7 +280,7 @@ static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 		 */
 		__intel_breadcrumbs_enable_irq(b);
 	}
-	GEM_BUG_ON(!b->irq_seqno_bh);
+	GEM_BUG_ON(!rcu_access_pointer(b->irq_seqno_bh));
 	GEM_BUG_ON(!b->first_wait);
 	GEM_BUG_ON(rb_first(&b->waiters) != &b->first_wait->node);
 
@@ -270,11 +300,6 @@ bool intel_engine_add_wait(struct intel_engine_cs *engine,
 	return first;
 }
 
-void intel_engine_enable_fake_irq(struct intel_engine_cs *engine)
-{
-	mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1);
-}
-
 static inline bool chain_wakeup(struct rb_node *rb, int priority)
 {
 	return rb && to_wait(rb)->tsk->prio <= priority;
@@ -310,7 +335,7 @@ void intel_engine_remove_wait(struct intel_engine_cs *engine,
 		const int priority = wakeup_priority(b, wait->tsk);
 		struct rb_node *next;
 
-		GEM_BUG_ON(b->irq_seqno_bh != wait->tsk);
+		GEM_BUG_ON(rcu_access_pointer(b->irq_seqno_bh) != wait->tsk);
 
 		/* We are the current bottom-half. Find the next candidate,
 		 * the first waiter in the queue on the remaining oldest
@@ -352,14 +377,15 @@ void intel_engine_remove_wait(struct intel_engine_cs *engine,
 			 * the interrupt, or if we have to handle an
 			 * exception rather than a seqno completion.
 			 */
+			b->timeout = wait_timeout();
 			b->first_wait = to_wait(next);
-			smp_store_mb(b->irq_seqno_bh, b->first_wait->tsk);
+			rcu_assign_pointer(b->irq_seqno_bh, b->first_wait->tsk);
 			if (b->first_wait->seqno != wait->seqno)
 				__intel_breadcrumbs_enable_irq(b);
-			wake_up_process(b->irq_seqno_bh);
+			wake_up_process(b->first_wait->tsk);
 		} else {
 			b->first_wait = NULL;
-			WRITE_ONCE(b->irq_seqno_bh, NULL);
+			rcu_assign_pointer(b->irq_seqno_bh, NULL);
 			__intel_breadcrumbs_disable_irq(b);
 		}
 	} else {
@@ -373,7 +399,7 @@ out_unlock:
 	GEM_BUG_ON(b->first_wait == wait);
 	GEM_BUG_ON(rb_first(&b->waiters) !=
 		   (b->first_wait ? &b->first_wait->node : NULL));
-	GEM_BUG_ON(!b->irq_seqno_bh ^ RB_EMPTY_ROOT(&b->waiters));
+	GEM_BUG_ON(!rcu_access_pointer(b->irq_seqno_bh) ^ RB_EMPTY_ROOT(&b->waiters));
 	spin_unlock(&b->lock);
 }
 
@@ -437,6 +463,10 @@ static int intel_breadcrumbs_signaler(void *arg)
 			intel_engine_remove_wait(engine,
 						 &request->signaling.wait);
 
+			local_bh_disable();
+			fence_signal(&request->fence);
+			local_bh_enable(); /* kick start the tasklets */
+
 			/* Find the next oldest signal. Note that as we have
 			 * not been holding the lock, another client may
 			 * have installed an even older signal than the one
@@ -452,7 +482,7 @@ static int intel_breadcrumbs_signaler(void *arg)
 			rb_erase(&request->signaling.node, &b->signals);
 			spin_unlock(&b->lock);
 
-			i915_gem_request_unreference(request);
+			i915_gem_request_put(request);
 		} else {
 			if (kthread_should_stop())
 				break;
@@ -472,18 +502,14 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request)
 	struct rb_node *parent, **p;
 	bool first, wakeup;
 
-	if (unlikely(READ_ONCE(request->signaling.wait.tsk)))
-		return;
-
-	spin_lock(&b->lock);
-	if (unlikely(request->signaling.wait.tsk)) {
-		wakeup = false;
-		goto unlock;
-	}
+	/* locked by fence_enable_sw_signaling() */
+	assert_spin_locked(&request->lock);
 
 	request->signaling.wait.tsk = b->signaler;
-	request->signaling.wait.seqno = request->seqno;
-	i915_gem_request_reference(request);
+	request->signaling.wait.seqno = request->fence.seqno;
+	i915_gem_request_get(request);
+
+	spin_lock(&b->lock);
 
 	/* First add ourselves into the list of waiters, but register our
 	 * bottom-half as the signaller thread. As per usual, only the oldest
@@ -504,8 +530,8 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request)
 	p = &b->signals.rb_node;
 	while (*p) {
 		parent = *p;
-		if (i915_seqno_passed(request->seqno,
-				      to_signaler(parent)->seqno)) {
+		if (i915_seqno_passed(request->fence.seqno,
+				      to_signaler(parent)->fence.seqno)) {
 			p = &parent->rb_right;
 			first = false;
 		} else {
@@ -517,7 +543,6 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request)
 	if (first)
 		smp_store_mb(b->first_signal, request);
 
-unlock:
 	spin_unlock(&b->lock);
 
 	if (wakeup)
@@ -533,6 +558,9 @@ int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
 	setup_timer(&b->fake_irq,
 		    intel_breadcrumbs_fake_irq,
 		    (unsigned long)engine);
+	setup_timer(&b->hangcheck,
+		    intel_breadcrumbs_hangcheck,
+		    (unsigned long)engine);
 
 	/* Spawn a thread to provide a common bottom-half for all signals.
 	 * As this is an asynchronous interface we cannot steal the current
@@ -550,6 +578,36 @@ int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
 	return 0;
 }
 
+static void cancel_fake_irq(struct intel_engine_cs *engine)
+{
+	struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+	del_timer_sync(&b->hangcheck);
+	del_timer_sync(&b->fake_irq);
+	clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
+}
+
+void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
+{
+	struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+	cancel_fake_irq(engine);
+	spin_lock(&b->lock);
+
+	__intel_breadcrumbs_disable_irq(b);
+	if (intel_engine_has_waiter(engine)) {
+		b->timeout = wait_timeout();
+		__intel_breadcrumbs_enable_irq(b);
+		if (READ_ONCE(b->irq_posted))
+			wake_up_process(b->first_wait->tsk);
+	} else {
+		/* sanitize the IMR and unmask any auxiliary interrupts */
+		irq_disable(engine);
+	}
+
+	spin_unlock(&b->lock);
+}
+
 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
 {
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
@@ -557,7 +615,7 @@ void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
 	if (!IS_ERR_OR_NULL(b->signaler))
 		kthread_stop(b->signaler);
 
-	del_timer_sync(&b->fake_irq);
+	cancel_fake_irq(engine);
 }
 
 unsigned int intel_kick_waiters(struct drm_i915_private *i915)
@@ -570,11 +628,9 @@ unsigned int intel_kick_waiters(struct drm_i915_private *i915)
 	 * RCU lock, i.e. as we call wake_up_process() we must be holding the
 	 * rcu_read_lock().
 	 */
-	rcu_read_lock();
 	for_each_engine(engine, i915)
 		if (unlikely(intel_engine_wakeup(engine)))
 			mask |= intel_engine_flag(engine);
-	rcu_read_unlock();
 
 	return mask;
 }
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index bc0fef3d3335..95a72771eea6 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -100,13 +100,14 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int i, pipe = intel_crtc->pipe;
 	uint16_t coeffs[9] = { 0, };
+	struct intel_crtc_state *intel_crtc_state = to_intel_crtc_state(crtc_state);
 
 	if (crtc_state->ctm) {
 		struct drm_color_ctm *ctm =
 			(struct drm_color_ctm *)crtc_state->ctm->data;
 		uint64_t input[9] = { 0, };
 
-		if (intel_crtc->config->limited_color_range) {
+		if (intel_crtc_state->limited_color_range) {
 			ctm_mult_by_limited(input, ctm->matrix);
 		} else {
 			for (i = 0; i < ARRAY_SIZE(input); i++)
@@ -158,7 +159,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
 		 * into consideration.
 		 */
 		for (i = 0; i < 3; i++) {
-			if (intel_crtc->config->limited_color_range)
+			if (intel_crtc_state->limited_color_range)
 				coeffs[i * 3 + i] =
 					I9XX_CSC_COEFF_LIMITED_RANGE;
 			else
@@ -182,7 +183,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
 	if (INTEL_INFO(dev)->gen > 6) {
 		uint16_t postoff = 0;
 
-		if (intel_crtc->config->limited_color_range)
+		if (intel_crtc_state->limited_color_range)
 			postoff = (16 * (1 << 12) / 255) & 0x1fff;
 
 		I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), postoff);
@@ -193,7 +194,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
 	} else {
 		uint32_t mode = CSC_MODE_YUV_TO_RGB;
 
-		if (intel_crtc->config->limited_color_range)
+		if (intel_crtc_state->limited_color_range)
 			mode |= CSC_BLACK_SCREEN_OFFSET;
 
 		I915_WRITE(PIPE_CSC_MODE(pipe), mode);
@@ -263,7 +264,8 @@ void intel_color_set_csc(struct drm_crtc_state *crtc_state)
 
 /* Loads the legacy palette/gamma unit for the CRTC. */
 static void i9xx_load_luts_internal(struct drm_crtc *crtc,
-				    struct drm_property_blob *blob)
+				    struct drm_property_blob *blob,
+				    struct intel_crtc_state *crtc_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -272,7 +274,7 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc,
 	int i;
 
 	if (HAS_GMCH_DISPLAY(dev)) {
-		if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI))
+		if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
 			assert_dsi_pll_enabled(dev_priv);
 		else
 			assert_pll_enabled(dev_priv, pipe);
@@ -305,7 +307,8 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc,
 
 static void i9xx_load_luts(struct drm_crtc_state *crtc_state)
 {
-	i9xx_load_luts_internal(crtc_state->crtc, crtc_state->gamma_lut);
+	i9xx_load_luts_internal(crtc_state->crtc, crtc_state->gamma_lut,
+				to_intel_crtc_state(crtc_state));
 }
 
 /* Loads the legacy palette/gamma unit for the CRTC on Haswell. */
@@ -323,7 +326,7 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
 	 * Workaround : Do not read or write the pipe palette/gamma data while
 	 * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
 	 */
-	if (IS_HASWELL(dev) && intel_crtc->config->ips_enabled &&
+	if (IS_HASWELL(dev) && intel_crtc_state->ips_enabled &&
 	    (intel_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)) {
 		hsw_disable_ips(intel_crtc);
 		reenable_ips = true;
@@ -436,7 +439,8 @@ static void cherryview_load_luts(struct drm_crtc_state *state)
 		/* Turn off degamma/gamma on CGM block. */
 		I915_WRITE(CGM_PIPE_MODE(pipe),
 			   (state->ctm ? CGM_PIPE_MODE_CSC : 0));
-		i9xx_load_luts_internal(crtc, state->gamma_lut);
+		i9xx_load_luts_internal(crtc, state->gamma_lut,
+					to_intel_crtc_state(state));
 		return;
 	}
 
@@ -479,7 +483,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state)
 	 * Also program a linear LUT in the legacy block (behind the
 	 * CGM block).
 	 */
-	i9xx_load_luts_internal(crtc, NULL);
+	i9xx_load_luts_internal(crtc, NULL, to_intel_crtc_state(state));
 }
 
 void intel_color_load_luts(struct drm_crtc_state *crtc_state)
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 827b6ef4e9ae..dfbcf16b41df 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -143,13 +143,15 @@ static void hsw_crt_get_config(struct intel_encoder *encoder,
 
 /* Note: The caller is required to filter out dpms modes not supported by the
  * platform. */
-static void intel_crt_set_dpms(struct intel_encoder *encoder, int mode)
+static void intel_crt_set_dpms(struct intel_encoder *encoder,
+			       struct intel_crtc_state *crtc_state,
+			       int mode)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crt *crt = intel_encoder_to_crt(encoder);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-	const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
 	u32 adpa;
 
 	if (INTEL_INFO(dev)->gen >= 5)
@@ -193,23 +195,45 @@ static void intel_crt_set_dpms(struct intel_encoder *encoder, int mode)
 	I915_WRITE(crt->adpa_reg, adpa);
 }
 
-static void intel_disable_crt(struct intel_encoder *encoder)
+static void intel_disable_crt(struct intel_encoder *encoder,
+			      struct intel_crtc_state *old_crtc_state,
+			      struct drm_connector_state *old_conn_state)
 {
-	intel_crt_set_dpms(encoder, DRM_MODE_DPMS_OFF);
+	intel_crt_set_dpms(encoder, old_crtc_state, DRM_MODE_DPMS_OFF);
 }
 
-static void pch_disable_crt(struct intel_encoder *encoder)
+static void pch_disable_crt(struct intel_encoder *encoder,
+			    struct intel_crtc_state *old_crtc_state,
+			    struct drm_connector_state *old_conn_state)
 {
 }
 
-static void pch_post_disable_crt(struct intel_encoder *encoder)
+static void pch_post_disable_crt(struct intel_encoder *encoder,
+				 struct intel_crtc_state *old_crtc_state,
+				 struct drm_connector_state *old_conn_state)
 {
-	intel_disable_crt(encoder);
+	intel_disable_crt(encoder, old_crtc_state, old_conn_state);
 }
 
-static void intel_enable_crt(struct intel_encoder *encoder)
+static void hsw_post_disable_crt(struct intel_encoder *encoder,
+				 struct intel_crtc_state *old_crtc_state,
+				 struct drm_connector_state *old_conn_state)
 {
-	intel_crt_set_dpms(encoder, DRM_MODE_DPMS_ON);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+
+	pch_post_disable_crt(encoder, old_crtc_state, old_conn_state);
+
+	lpt_disable_pch_transcoder(dev_priv);
+	lpt_disable_iclkip(dev_priv);
+
+	intel_ddi_fdi_post_disable(encoder, old_crtc_state, old_conn_state);
+}
+
+static void intel_enable_crt(struct intel_encoder *encoder,
+			     struct intel_crtc_state *pipe_config,
+			     struct drm_connector_state *conn_state)
+{
+	intel_crt_set_dpms(encoder, pipe_config, DRM_MODE_DPMS_ON);
 }
 
 static enum drm_mode_status
@@ -253,7 +277,8 @@ intel_crt_mode_valid(struct drm_connector *connector,
 }
 
 static bool intel_crt_compute_config(struct intel_encoder *encoder,
-				     struct intel_crtc_state *pipe_config)
+				     struct intel_crtc_state *pipe_config,
+				     struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 
@@ -894,6 +919,7 @@ void intel_crt_init(struct drm_device *dev)
 	if (HAS_DDI(dev)) {
 		crt->base.get_config = hsw_crt_get_config;
 		crt->base.get_hw_state = intel_ddi_get_hw_state;
+		crt->base.post_disable = hsw_post_disable_crt;
 	} else {
 		crt->base.get_config = intel_crt_get_config;
 		crt->base.get_hw_state = intel_crt_get_hw_state;
diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c
index 3edb9580928e..1ea0e1f43397 100644
--- a/drivers/gpu/drm/i915/intel_csr.c
+++ b/drivers/gpu/drm/i915/intel_csr.c
@@ -32,24 +32,17 @@
  * onwards to drive newly added DMC (Display microcontroller) in display
  * engine to save and restore the state of display engine when it enter into
  * low-power state and comes back to normal.
- *
- * Firmware loading status will be one of the below states: FW_UNINITIALIZED,
- * FW_LOADED, FW_FAILED.
- *
- * Once the firmware is written into the registers status will be moved from
- * FW_UNINITIALIZED to FW_LOADED and for any erroneous condition status will
- * be moved to FW_FAILED.
  */
 
-#define I915_CSR_KBL "i915/kbl_dmc_ver1.bin"
+#define I915_CSR_KBL "i915/kbl_dmc_ver1_01.bin"
 MODULE_FIRMWARE(I915_CSR_KBL);
 #define KBL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 1)
 
-#define I915_CSR_SKL "i915/skl_dmc_ver1.bin"
+#define I915_CSR_SKL "i915/skl_dmc_ver1_26.bin"
 MODULE_FIRMWARE(I915_CSR_SKL);
-#define SKL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 23)
+#define SKL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 26)
 
-#define I915_CSR_BXT "i915/bxt_dmc_ver1.bin"
+#define I915_CSR_BXT "i915/bxt_dmc_ver1_07.bin"
 MODULE_FIRMWARE(I915_CSR_BXT);
 #define BXT_CSR_VERSION_REQUIRED	CSR_VERSION(1, 7)
 
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index dd1d6fe12297..15d47c87def6 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -145,7 +145,7 @@ static const struct ddi_buf_trans skl_ddi_translations_dp[] = {
 static const struct ddi_buf_trans skl_u_ddi_translations_dp[] = {
 	{ 0x0000201B, 0x000000A2, 0x0 },
 	{ 0x00005012, 0x00000088, 0x0 },
-	{ 0x80007011, 0x000000CD, 0x0 },
+	{ 0x80007011, 0x000000CD, 0x1 },
 	{ 0x80009010, 0x000000C0, 0x1 },
 	{ 0x0000201B, 0x0000009D, 0x0 },
 	{ 0x80005012, 0x000000C0, 0x1 },
@@ -158,7 +158,7 @@ static const struct ddi_buf_trans skl_u_ddi_translations_dp[] = {
 static const struct ddi_buf_trans skl_y_ddi_translations_dp[] = {
 	{ 0x00000018, 0x000000A2, 0x0 },
 	{ 0x00005012, 0x00000088, 0x0 },
-	{ 0x80007011, 0x000000CD, 0x0 },
+	{ 0x80007011, 0x000000CD, 0x3 },
 	{ 0x80009010, 0x000000C0, 0x3 },
 	{ 0x00000018, 0x0000009D, 0x0 },
 	{ 0x80005012, 0x000000C0, 0x3 },
@@ -301,45 +301,34 @@ static const struct bxt_ddi_buf_trans bxt_ddi_translations_hdmi[] = {
 	{ 154, 0x9A, 1, 128, true },	/* 9:	1200		0   */
 };
 
-static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv,
-				    u32 level, enum port port, int type);
-
-static void ddi_get_encoder_port(struct intel_encoder *intel_encoder,
-				 struct intel_digital_port **dig_port,
-				 enum port *port)
+enum port intel_ddi_get_encoder_port(struct intel_encoder *encoder)
 {
-	struct drm_encoder *encoder = &intel_encoder->base;
-
-	switch (intel_encoder->type) {
+	switch (encoder->type) {
 	case INTEL_OUTPUT_DP_MST:
-		*dig_port = enc_to_mst(encoder)->primary;
-		*port = (*dig_port)->port;
-		break;
-	default:
-		WARN(1, "Invalid DDI encoder type %d\n", intel_encoder->type);
-		/* fallthrough and treat as unknown */
+		return enc_to_mst(&encoder->base)->primary->port;
 	case INTEL_OUTPUT_DP:
 	case INTEL_OUTPUT_EDP:
 	case INTEL_OUTPUT_HDMI:
 	case INTEL_OUTPUT_UNKNOWN:
-		*dig_port = enc_to_dig_port(encoder);
-		*port = (*dig_port)->port;
-		break;
+		return enc_to_dig_port(&encoder->base)->port;
 	case INTEL_OUTPUT_ANALOG:
-		*dig_port = NULL;
-		*port = PORT_E;
-		break;
+		return PORT_E;
+	default:
+		MISSING_CASE(encoder->type);
+		return PORT_A;
 	}
 }
 
-enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder)
+static const struct ddi_buf_trans *
+bdw_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 {
-	struct intel_digital_port *dig_port;
-	enum port port;
-
-	ddi_get_encoder_port(intel_encoder, &dig_port, &port);
-
-	return port;
+	if (dev_priv->vbt.edp.low_vswing) {
+		*n_entries = ARRAY_SIZE(bdw_ddi_translations_edp);
+		return bdw_ddi_translations_edp;
+	} else {
+		*n_entries = ARRAY_SIZE(bdw_ddi_translations_dp);
+		return bdw_ddi_translations_dp;
+	}
 }
 
 static const struct ddi_buf_trans *
@@ -388,39 +377,58 @@ skl_get_buf_trans_hdmi(struct drm_i915_private *dev_priv, int *n_entries)
 	}
 }
 
+static int intel_ddi_hdmi_level(struct drm_i915_private *dev_priv, enum port port)
+{
+	int n_hdmi_entries;
+	int hdmi_level;
+	int hdmi_default_entry;
+
+	hdmi_level = dev_priv->vbt.ddi_port_info[port].hdmi_level_shift;
+
+	if (IS_BROXTON(dev_priv))
+		return hdmi_level;
+
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+		skl_get_buf_trans_hdmi(dev_priv, &n_hdmi_entries);
+		hdmi_default_entry = 8;
+	} else if (IS_BROADWELL(dev_priv)) {
+		n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
+		hdmi_default_entry = 7;
+	} else if (IS_HASWELL(dev_priv)) {
+		n_hdmi_entries = ARRAY_SIZE(hsw_ddi_translations_hdmi);
+		hdmi_default_entry = 6;
+	} else {
+		WARN(1, "ddi translation table missing\n");
+		n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
+		hdmi_default_entry = 7;
+	}
+
+	/* Choose a good default if VBT is badly populated */
+	if (hdmi_level == HDMI_LEVEL_SHIFT_UNKNOWN ||
+	    hdmi_level >= n_hdmi_entries)
+		hdmi_level = hdmi_default_entry;
+
+	return hdmi_level;
+}
+
 /*
  * Starting with Haswell, DDI port buffers must be programmed with correct
- * values in advance. The buffer values are different for FDI and DP modes,
- * but the HDMI/DVI fields are shared among those. So we program the DDI
- * in either FDI or DP modes only, as HDMI connections will work with both
- * of those
+ * values in advance. This function programs the correct values for
+ * DP/eDP/FDI use cases.
  */
-void intel_prepare_ddi_buffer(struct intel_encoder *encoder)
+void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	u32 iboost_bit = 0;
-	int i, n_hdmi_entries, n_dp_entries, n_edp_entries, hdmi_default_entry,
-	    size;
-	int hdmi_level;
-	enum port port;
+	int i, n_dp_entries, n_edp_entries, size;
+	enum port port = intel_ddi_get_encoder_port(encoder);
 	const struct ddi_buf_trans *ddi_translations_fdi;
 	const struct ddi_buf_trans *ddi_translations_dp;
 	const struct ddi_buf_trans *ddi_translations_edp;
-	const struct ddi_buf_trans *ddi_translations_hdmi;
 	const struct ddi_buf_trans *ddi_translations;
 
-	port = intel_ddi_get_encoder_port(encoder);
-	hdmi_level = dev_priv->vbt.ddi_port_info[port].hdmi_level_shift;
-
-	if (IS_BROXTON(dev_priv)) {
-		if (encoder->type != INTEL_OUTPUT_HDMI)
-			return;
-
-		/* Vswing programming for HDMI */
-		bxt_ddi_vswing_sequence(dev_priv, hdmi_level, port,
-					INTEL_OUTPUT_HDMI);
+	if (IS_BROXTON(dev_priv))
 		return;
-	}
 
 	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		ddi_translations_fdi = NULL;
@@ -428,13 +436,10 @@ void intel_prepare_ddi_buffer(struct intel_encoder *encoder)
 				skl_get_buf_trans_dp(dev_priv, &n_dp_entries);
 		ddi_translations_edp =
 				skl_get_buf_trans_edp(dev_priv, &n_edp_entries);
-		ddi_translations_hdmi =
-				skl_get_buf_trans_hdmi(dev_priv, &n_hdmi_entries);
-		hdmi_default_entry = 8;
+
 		/* If we're boosting the current, set bit 31 of trans1 */
-		if (dev_priv->vbt.ddi_port_info[port].hdmi_boost_level ||
-		    dev_priv->vbt.ddi_port_info[port].dp_boost_level)
-			iboost_bit = 1<<31;
+		if (dev_priv->vbt.ddi_port_info[port].dp_boost_level)
+			iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE;
 
 		if (WARN_ON(encoder->type == INTEL_OUTPUT_EDP &&
 			    port != PORT_A && port != PORT_E &&
@@ -443,38 +448,20 @@ void intel_prepare_ddi_buffer(struct intel_encoder *encoder)
 	} else if (IS_BROADWELL(dev_priv)) {
 		ddi_translations_fdi = bdw_ddi_translations_fdi;
 		ddi_translations_dp = bdw_ddi_translations_dp;
-
-		if (dev_priv->vbt.edp.low_vswing) {
-			ddi_translations_edp = bdw_ddi_translations_edp;
-			n_edp_entries = ARRAY_SIZE(bdw_ddi_translations_edp);
-		} else {
-			ddi_translations_edp = bdw_ddi_translations_dp;
-			n_edp_entries = ARRAY_SIZE(bdw_ddi_translations_dp);
-		}
-
-		ddi_translations_hdmi = bdw_ddi_translations_hdmi;
-
+		ddi_translations_edp = bdw_get_buf_trans_edp(dev_priv, &n_edp_entries);
 		n_dp_entries = ARRAY_SIZE(bdw_ddi_translations_dp);
-		n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
-		hdmi_default_entry = 7;
 	} else if (IS_HASWELL(dev_priv)) {
 		ddi_translations_fdi = hsw_ddi_translations_fdi;
 		ddi_translations_dp = hsw_ddi_translations_dp;
 		ddi_translations_edp = hsw_ddi_translations_dp;
-		ddi_translations_hdmi = hsw_ddi_translations_hdmi;
 		n_dp_entries = n_edp_entries = ARRAY_SIZE(hsw_ddi_translations_dp);
-		n_hdmi_entries = ARRAY_SIZE(hsw_ddi_translations_hdmi);
-		hdmi_default_entry = 6;
 	} else {
 		WARN(1, "ddi translation table missing\n");
 		ddi_translations_edp = bdw_ddi_translations_dp;
 		ddi_translations_fdi = bdw_ddi_translations_fdi;
 		ddi_translations_dp = bdw_ddi_translations_dp;
-		ddi_translations_hdmi = bdw_ddi_translations_hdmi;
 		n_edp_entries = ARRAY_SIZE(bdw_ddi_translations_edp);
 		n_dp_entries = ARRAY_SIZE(bdw_ddi_translations_dp);
-		n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
-		hdmi_default_entry = 7;
 	}
 
 	switch (encoder->type) {
@@ -483,7 +470,6 @@ void intel_prepare_ddi_buffer(struct intel_encoder *encoder)
 		size = n_edp_entries;
 		break;
 	case INTEL_OUTPUT_DP:
-	case INTEL_OUTPUT_HDMI:
 		ddi_translations = ddi_translations_dp;
 		size = n_dp_entries;
 		break;
@@ -501,19 +487,48 @@ void intel_prepare_ddi_buffer(struct intel_encoder *encoder)
 		I915_WRITE(DDI_BUF_TRANS_HI(port, i),
 			   ddi_translations[i].trans2);
 	}
+}
 
-	if (encoder->type != INTEL_OUTPUT_HDMI)
+/*
+ * Starting with Haswell, DDI port buffers must be programmed with correct
+ * values in advance. This function programs the correct values for
+ * HDMI/DVI use cases.
+ */
+static void intel_prepare_hdmi_ddi_buffers(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	u32 iboost_bit = 0;
+	int n_hdmi_entries, hdmi_level;
+	enum port port = intel_ddi_get_encoder_port(encoder);
+	const struct ddi_buf_trans *ddi_translations_hdmi;
+
+	if (IS_BROXTON(dev_priv))
 		return;
 
-	/* Choose a good default if VBT is badly populated */
-	if (hdmi_level == HDMI_LEVEL_SHIFT_UNKNOWN ||
-	    hdmi_level >= n_hdmi_entries)
-		hdmi_level = hdmi_default_entry;
+	hdmi_level = intel_ddi_hdmi_level(dev_priv, port);
+
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+		ddi_translations_hdmi = skl_get_buf_trans_hdmi(dev_priv, &n_hdmi_entries);
+
+		/* If we're boosting the current, set bit 31 of trans1 */
+		if (dev_priv->vbt.ddi_port_info[port].hdmi_boost_level)
+			iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE;
+	} else if (IS_BROADWELL(dev_priv)) {
+		ddi_translations_hdmi = bdw_ddi_translations_hdmi;
+		n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
+	} else if (IS_HASWELL(dev_priv)) {
+		ddi_translations_hdmi = hsw_ddi_translations_hdmi;
+		n_hdmi_entries = ARRAY_SIZE(hsw_ddi_translations_hdmi);
+	} else {
+		WARN(1, "ddi translation table missing\n");
+		ddi_translations_hdmi = bdw_ddi_translations_hdmi;
+		n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
+	}
 
 	/* Entry 9 is for HDMI: */
-	I915_WRITE(DDI_BUF_TRANS_LO(port, i),
+	I915_WRITE(DDI_BUF_TRANS_LO(port, 9),
 		   ddi_translations_hdmi[hdmi_level].trans1 | iboost_bit);
-	I915_WRITE(DDI_BUF_TRANS_HI(port, i),
+	I915_WRITE(DDI_BUF_TRANS_HI(port, 9),
 		   ddi_translations_hdmi[hdmi_level].trans2);
 }
 
@@ -531,6 +546,27 @@ static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv,
 	DRM_ERROR("Timeout waiting for DDI BUF %c idle bit\n", port_name(port));
 }
 
+static uint32_t hsw_pll_to_ddi_pll_sel(struct intel_shared_dpll *pll)
+{
+	switch (pll->id) {
+	case DPLL_ID_WRPLL1:
+		return PORT_CLK_SEL_WRPLL1;
+	case DPLL_ID_WRPLL2:
+		return PORT_CLK_SEL_WRPLL2;
+	case DPLL_ID_SPLL:
+		return PORT_CLK_SEL_SPLL;
+	case DPLL_ID_LCPLL_810:
+		return PORT_CLK_SEL_LCPLL_810;
+	case DPLL_ID_LCPLL_1350:
+		return PORT_CLK_SEL_LCPLL_1350;
+	case DPLL_ID_LCPLL_2700:
+		return PORT_CLK_SEL_LCPLL_2700;
+	default:
+		MISSING_CASE(pll->id);
+		return PORT_CLK_SEL_NONE;
+	}
+}
+
 /* Starting with Haswell, different DDI ports can work in FDI mode for
  * connection to the PCH-located connectors. For this, it is necessary to train
  * both the DDI port and PCH receiver for the desired DDI buffer settings.
@@ -546,11 +582,11 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_encoder *encoder;
-	u32 temp, i, rx_ctl_val;
+	u32 temp, i, rx_ctl_val, ddi_pll_sel;
 
 	for_each_encoder_on_crtc(dev, crtc, encoder) {
 		WARN_ON(encoder->type != INTEL_OUTPUT_ANALOG);
-		intel_prepare_ddi_buffer(encoder);
+		intel_prepare_dp_ddi_buffers(encoder);
 	}
 
 	/* Set the FDI_RX_MISC pwrdn lanes and the 2 workarounds listed at the
@@ -577,8 +613,9 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
 	I915_WRITE(FDI_RX_CTL(PIPE_A), rx_ctl_val);
 
 	/* Configure Port Clock Select */
-	I915_WRITE(PORT_CLK_SEL(PORT_E), intel_crtc->config->ddi_pll_sel);
-	WARN_ON(intel_crtc->config->ddi_pll_sel != PORT_CLK_SEL_SPLL);
+	ddi_pll_sel = hsw_pll_to_ddi_pll_sel(intel_crtc->config->shared_dpll);
+	I915_WRITE(PORT_CLK_SEL(PORT_E), ddi_pll_sel);
+	WARN_ON(ddi_pll_sel != PORT_CLK_SEL_SPLL);
 
 	/* Start the training iterating through available voltages and emphasis,
 	 * testing each value twice. */
@@ -855,7 +892,7 @@ static void skl_ddi_clock_get(struct intel_encoder *encoder,
 	int link_clock = 0;
 	uint32_t dpll_ctl1, dpll;
 
-	dpll = pipe_config->ddi_pll_sel;
+	dpll = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll);
 
 	dpll_ctl1 = I915_READ(DPLL_CTRL1);
 
@@ -903,7 +940,7 @@ static void hsw_ddi_clock_get(struct intel_encoder *encoder,
 	int link_clock = 0;
 	u32 val, pll;
 
-	val = pipe_config->ddi_pll_sel;
+	val = hsw_pll_to_ddi_pll_sel(pipe_config->shared_dpll);
 	switch (val & PORT_CLK_SEL_MASK) {
 	case PORT_CLK_SEL_LCPLL_810:
 		link_clock = 81000;
@@ -1111,7 +1148,6 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
 {
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
-	struct drm_encoder *encoder = &intel_encoder->base;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum pipe pipe = intel_crtc->pipe;
@@ -1177,29 +1213,15 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
 			temp |= TRANS_DDI_MODE_SELECT_HDMI;
 		else
 			temp |= TRANS_DDI_MODE_SELECT_DVI;
-
 	} else if (type == INTEL_OUTPUT_ANALOG) {
 		temp |= TRANS_DDI_MODE_SELECT_FDI;
 		temp |= (intel_crtc->config->fdi_lanes - 1) << 1;
-
 	} else if (type == INTEL_OUTPUT_DP ||
 		   type == INTEL_OUTPUT_EDP) {
-		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-
-		if (intel_dp->is_mst) {
-			temp |= TRANS_DDI_MODE_SELECT_DP_MST;
-		} else
-			temp |= TRANS_DDI_MODE_SELECT_DP_SST;
-
+		temp |= TRANS_DDI_MODE_SELECT_DP_SST;
 		temp |= DDI_PORT_WIDTH(intel_crtc->config->lane_count);
 	} else if (type == INTEL_OUTPUT_DP_MST) {
-		struct intel_dp *intel_dp = &enc_to_mst(encoder)->primary->dp;
-
-		if (intel_dp->is_mst) {
-			temp |= TRANS_DDI_MODE_SELECT_DP_MST;
-		} else
-			temp |= TRANS_DDI_MODE_SELECT_DP_SST;
-
+		temp |= TRANS_DDI_MODE_SELECT_DP_MST;
 		temp |= DDI_PORT_WIDTH(intel_crtc->config->lane_count);
 	} else {
 		WARN(1, "Invalid encoder type %d for pipe %c\n",
@@ -1379,14 +1401,30 @@ void intel_ddi_disable_pipe_clock(struct intel_crtc *intel_crtc)
 			   TRANS_CLK_SEL_DISABLED);
 }
 
-static void skl_ddi_set_iboost(struct drm_i915_private *dev_priv,
-			       u32 level, enum port port, int type)
+static void _skl_ddi_set_iboost(struct drm_i915_private *dev_priv,
+				enum port port, uint8_t iboost)
+{
+	u32 tmp;
+
+	tmp = I915_READ(DISPIO_CR_TX_BMU_CR0);
+	tmp &= ~(BALANCE_LEG_MASK(port) | BALANCE_LEG_DISABLE(port));
+	if (iboost)
+		tmp |= iboost << BALANCE_LEG_SHIFT(port);
+	else
+		tmp |= BALANCE_LEG_DISABLE(port);
+	I915_WRITE(DISPIO_CR_TX_BMU_CR0, tmp);
+}
+
+static void skl_ddi_set_iboost(struct intel_encoder *encoder, u32 level)
 {
+	struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base);
+	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
+	enum port port = intel_dig_port->port;
+	int type = encoder->type;
 	const struct ddi_buf_trans *ddi_translations;
 	uint8_t iboost;
 	uint8_t dp_iboost, hdmi_iboost;
 	int n_entries;
-	u32 reg;
 
 	/* VBT may override standard boost values */
 	dp_iboost = dev_priv->vbt.ddi_port_info[port].dp_boost_level;
@@ -1428,16 +1466,10 @@ static void skl_ddi_set_iboost(struct drm_i915_private *dev_priv,
 		return;
 	}
 
-	reg = I915_READ(DISPIO_CR_TX_BMU_CR0);
-	reg &= ~BALANCE_LEG_MASK(port);
-	reg &= ~(1 << (BALANCE_LEG_DISABLE_SHIFT + port));
+	_skl_ddi_set_iboost(dev_priv, port, iboost);
 
-	if (iboost)
-		reg |= iboost << BALANCE_LEG_SHIFT(port);
-	else
-		reg |= 1 << (BALANCE_LEG_DISABLE_SHIFT + port);
-
-	I915_WRITE(DISPIO_CR_TX_BMU_CR0, reg);
+	if (port == PORT_A && intel_dig_port->max_lanes == 4)
+		_skl_ddi_set_iboost(dev_priv, PORT_E, iboost);
 }
 
 static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv,
@@ -1568,7 +1600,7 @@ uint32_t ddi_signal_levels(struct intel_dp *intel_dp)
 	level = translate_signal_level(signal_levels);
 
 	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
-		skl_ddi_set_iboost(dev_priv, level, port, encoder->type);
+		skl_ddi_set_iboost(encoder, level);
 	else if (IS_BROXTON(dev_priv))
 		bxt_ddi_vswing_sequence(dev_priv, level, port, encoder->type);
 
@@ -1576,13 +1608,15 @@ uint32_t ddi_signal_levels(struct intel_dp *intel_dp)
 }
 
 void intel_ddi_clk_select(struct intel_encoder *encoder,
-			  const struct intel_crtc_state *pipe_config)
+			  struct intel_shared_dpll *pll)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum port port = intel_ddi_get_encoder_port(encoder);
 
+	if (WARN_ON(!pll))
+		return;
+
 	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
-		uint32_t dpll = pipe_config->ddi_pll_sel;
 		uint32_t val;
 
 		/* DDI -> PLL mapping  */
@@ -1590,61 +1624,91 @@ void intel_ddi_clk_select(struct intel_encoder *encoder,
 
 		val &= ~(DPLL_CTRL2_DDI_CLK_OFF(port) |
 			DPLL_CTRL2_DDI_CLK_SEL_MASK(port));
-		val |= (DPLL_CTRL2_DDI_CLK_SEL(dpll, port) |
+		val |= (DPLL_CTRL2_DDI_CLK_SEL(pll->id, port) |
 			DPLL_CTRL2_DDI_SEL_OVERRIDE(port));
 
 		I915_WRITE(DPLL_CTRL2, val);
 
 	} else if (INTEL_INFO(dev_priv)->gen < 9) {
-		WARN_ON(pipe_config->ddi_pll_sel == PORT_CLK_SEL_NONE);
-		I915_WRITE(PORT_CLK_SEL(port), pipe_config->ddi_pll_sel);
+		I915_WRITE(PORT_CLK_SEL(port), hsw_pll_to_ddi_pll_sel(pll));
 	}
 }
 
-static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder)
+static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
+				    int link_rate, uint32_t lane_count,
+				    struct intel_shared_dpll *pll,
+				    bool link_mst)
 {
-	struct drm_encoder *encoder = &intel_encoder->base;
-	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
-	enum port port = intel_ddi_get_encoder_port(intel_encoder);
-	int type = intel_encoder->type;
-
-	if (type == INTEL_OUTPUT_HDMI) {
-		struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-
-		intel_dp_dual_mode_set_tmds_output(intel_hdmi, true);
-	}
-
-	intel_prepare_ddi_buffer(intel_encoder);
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum port port = intel_ddi_get_encoder_port(encoder);
 
-	if (type == INTEL_OUTPUT_EDP) {
-		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	intel_dp_set_link_params(intel_dp, link_rate, lane_count,
+				 link_mst);
+	if (encoder->type == INTEL_OUTPUT_EDP)
 		intel_edp_panel_on(intel_dp);
-	}
 
-	intel_ddi_clk_select(intel_encoder, crtc->config);
+	intel_ddi_clk_select(encoder, pll);
+	intel_prepare_dp_ddi_buffers(encoder);
+	intel_ddi_init_dp_buf_reg(encoder);
+	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
+	intel_dp_start_link_train(intel_dp);
+	if (port != PORT_A || INTEL_GEN(dev_priv) >= 9)
+		intel_dp_stop_link_train(intel_dp);
+}
 
-	if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) {
-		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder,
+				      bool has_hdmi_sink,
+				      struct drm_display_mode *adjusted_mode,
+				      struct intel_shared_dpll *pll)
+{
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct drm_encoder *drm_encoder = &encoder->base;
+	enum port port = intel_ddi_get_encoder_port(encoder);
+	int level = intel_ddi_hdmi_level(dev_priv, port);
 
-		intel_dp_set_link_params(intel_dp, crtc->config);
+	intel_dp_dual_mode_set_tmds_output(intel_hdmi, true);
+	intel_ddi_clk_select(encoder, pll);
+	intel_prepare_hdmi_ddi_buffers(encoder);
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+		skl_ddi_set_iboost(encoder, level);
+	else if (IS_BROXTON(dev_priv))
+		bxt_ddi_vswing_sequence(dev_priv, level, port,
+					INTEL_OUTPUT_HDMI);
 
-		intel_ddi_init_dp_buf_reg(intel_encoder);
+	intel_hdmi->set_infoframes(drm_encoder,
+				   has_hdmi_sink,
+				   adjusted_mode);
+}
 
-		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
-		intel_dp_start_link_train(intel_dp);
-		if (port != PORT_A || INTEL_INFO(dev_priv)->gen >= 9)
-			intel_dp_stop_link_train(intel_dp);
-	} else if (type == INTEL_OUTPUT_HDMI) {
-		struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
+static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder,
+				 struct intel_crtc_state *pipe_config,
+				 struct drm_connector_state *conn_state)
+{
+	struct drm_encoder *encoder = &intel_encoder->base;
+	struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
+	int type = intel_encoder->type;
 
-		intel_hdmi->set_infoframes(encoder,
-					   crtc->config->has_hdmi_sink,
-					   &crtc->config->base.adjusted_mode);
+	if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) {
+		intel_ddi_pre_enable_dp(intel_encoder,
+					crtc->config->port_clock,
+					crtc->config->lane_count,
+					crtc->config->shared_dpll,
+					intel_crtc_has_type(crtc->config,
+							    INTEL_OUTPUT_DP_MST));
+	}
+	if (type == INTEL_OUTPUT_HDMI) {
+		intel_ddi_pre_enable_hdmi(intel_encoder,
+					  crtc->config->has_hdmi_sink,
+					  &crtc->config->base.adjusted_mode,
+					  crtc->config->shared_dpll);
 	}
 }
 
-static void intel_ddi_post_disable(struct intel_encoder *intel_encoder)
+static void intel_ddi_post_disable(struct intel_encoder *intel_encoder,
+				   struct intel_crtc_state *old_crtc_state,
+				   struct drm_connector_state *old_conn_state)
 {
 	struct drm_encoder *encoder = &intel_encoder->base;
 	struct drm_device *dev = encoder->dev;
@@ -1654,6 +1718,8 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder)
 	uint32_t val;
 	bool wait = false;
 
+	/* old_crtc_state and old_conn_state are NULL when called from DP_MST */
+
 	val = I915_READ(DDI_BUF_CTL(port));
 	if (val & DDI_BUF_CTL_ENABLE) {
 		val &= ~DDI_BUF_CTL_ENABLE;
@@ -1689,7 +1755,42 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder)
 	}
 }
 
-static void intel_enable_ddi(struct intel_encoder *intel_encoder)
+void intel_ddi_fdi_post_disable(struct intel_encoder *intel_encoder,
+				struct intel_crtc_state *old_crtc_state,
+				struct drm_connector_state *old_conn_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
+	uint32_t val;
+
+	/*
+	 * Bspec lists this as both step 13 (before DDI_BUF_CTL disable)
+	 * and step 18 (after clearing PORT_CLK_SEL). Based on a BUN,
+	 * step 13 is the correct place for it. Step 18 is where it was
+	 * originally before the BUN.
+	 */
+	val = I915_READ(FDI_RX_CTL(PIPE_A));
+	val &= ~FDI_RX_ENABLE;
+	I915_WRITE(FDI_RX_CTL(PIPE_A), val);
+
+	intel_ddi_post_disable(intel_encoder, old_crtc_state, old_conn_state);
+
+	val = I915_READ(FDI_RX_MISC(PIPE_A));
+	val &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK);
+	val |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2);
+	I915_WRITE(FDI_RX_MISC(PIPE_A), val);
+
+	val = I915_READ(FDI_RX_CTL(PIPE_A));
+	val &= ~FDI_PCDCLK;
+	I915_WRITE(FDI_RX_CTL(PIPE_A), val);
+
+	val = I915_READ(FDI_RX_CTL(PIPE_A));
+	val &= ~FDI_RX_PLL_ENABLE;
+	I915_WRITE(FDI_RX_CTL(PIPE_A), val);
+}
+
+static void intel_enable_ddi(struct intel_encoder *intel_encoder,
+			     struct intel_crtc_state *pipe_config,
+			     struct drm_connector_state *conn_state)
 {
 	struct drm_encoder *encoder = &intel_encoder->base;
 	struct drm_crtc *crtc = encoder->crtc;
@@ -1718,7 +1819,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder)
 
 		intel_edp_backlight_on(intel_dp);
 		intel_psr_enable(intel_dp);
-		intel_edp_drrs_enable(intel_dp);
+		intel_edp_drrs_enable(intel_dp, pipe_config);
 	}
 
 	if (intel_crtc->config->has_audio) {
@@ -1727,7 +1828,9 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder)
 	}
 }
 
-static void intel_disable_ddi(struct intel_encoder *intel_encoder)
+static void intel_disable_ddi(struct intel_encoder *intel_encoder,
+			      struct intel_crtc_state *old_crtc_state,
+			      struct drm_connector_state *old_conn_state)
 {
 	struct drm_encoder *encoder = &intel_encoder->base;
 	struct drm_crtc *crtc = encoder->crtc;
@@ -1744,7 +1847,7 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder)
 	if (type == INTEL_OUTPUT_EDP) {
 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
-		intel_edp_drrs_disable(intel_dp);
+		intel_edp_drrs_disable(intel_dp, old_crtc_state);
 		intel_psr_disable(intel_dp);
 		intel_edp_backlight_off(intel_dp);
 	}
@@ -2033,7 +2136,9 @@ bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
 	}
 }
 
-static void bxt_ddi_pre_pll_enable(struct intel_encoder *encoder)
+static void bxt_ddi_pre_pll_enable(struct intel_encoder *encoder,
+				   struct intel_crtc_state *pipe_config,
+				   struct drm_connector_state *conn_state)
 {
 	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
 	struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
@@ -2105,7 +2210,7 @@ void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp)
 
 	val = DP_TP_CTL_ENABLE |
 	      DP_TP_CTL_LINK_TRAIN_PAT1 | DP_TP_CTL_SCRAMBLE_DISABLE;
-	if (intel_dp->is_mst)
+	if (intel_dp->link_mst)
 		val |= DP_TP_CTL_MODE_MST;
 	else {
 		val |= DP_TP_CTL_MODE_SST;
@@ -2122,38 +2227,6 @@ void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp)
 	udelay(600);
 }
 
-void intel_ddi_fdi_disable(struct drm_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-	struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
-	uint32_t val;
-
-	/*
-	 * Bspec lists this as both step 13 (before DDI_BUF_CTL disable)
-	 * and step 18 (after clearing PORT_CLK_SEL). Based on a BUN,
-	 * step 13 is the correct place for it. Step 18 is where it was
-	 * originally before the BUN.
-	 */
-	val = I915_READ(FDI_RX_CTL(PIPE_A));
-	val &= ~FDI_RX_ENABLE;
-	I915_WRITE(FDI_RX_CTL(PIPE_A), val);
-
-	intel_ddi_post_disable(intel_encoder);
-
-	val = I915_READ(FDI_RX_MISC(PIPE_A));
-	val &= ~(FDI_RX_PWRDN_LANE1_MASK | FDI_RX_PWRDN_LANE0_MASK);
-	val |= FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2);
-	I915_WRITE(FDI_RX_MISC(PIPE_A), val);
-
-	val = I915_READ(FDI_RX_CTL(PIPE_A));
-	val &= ~FDI_PCDCLK;
-	I915_WRITE(FDI_RX_CTL(PIPE_A), val);
-
-	val = I915_READ(FDI_RX_CTL(PIPE_A));
-	val &= ~FDI_RX_PLL_ENABLE;
-	I915_WRITE(FDI_RX_CTL(PIPE_A), val);
-}
-
 void intel_ddi_get_config(struct intel_encoder *encoder,
 			  struct intel_crtc_state *pipe_config)
 {
@@ -2253,7 +2326,8 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
 }
 
 static bool intel_ddi_compute_config(struct intel_encoder *encoder,
-				     struct intel_crtc_state *pipe_config)
+				     struct intel_crtc_state *pipe_config,
+				     struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	int type = encoder->type;
@@ -2266,9 +2340,9 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder,
 		pipe_config->cpu_transcoder = TRANSCODER_EDP;
 
 	if (type == INTEL_OUTPUT_HDMI)
-		ret = intel_hdmi_compute_config(encoder, pipe_config);
+		ret = intel_hdmi_compute_config(encoder, pipe_config, conn_state);
 	else
-		ret = intel_dp_compute_config(encoder, pipe_config);
+		ret = intel_dp_compute_config(encoder, pipe_config, conn_state);
 
 	if (IS_BROXTON(dev_priv) && ret)
 		pipe_config->lane_lat_optim_mask =
@@ -2319,6 +2393,45 @@ intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port)
 	return connector;
 }
 
+struct intel_shared_dpll *
+intel_ddi_get_link_dpll(struct intel_dp *intel_dp, int clock)
+{
+	struct intel_connector *connector = intel_dp->attached_connector;
+	struct intel_encoder *encoder = connector->encoder;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct intel_shared_dpll *pll = NULL;
+	struct intel_shared_dpll_config tmp_pll_config;
+	enum intel_dpll_id dpll_id;
+
+	if (IS_BROXTON(dev_priv)) {
+		dpll_id =  (enum intel_dpll_id)dig_port->port;
+		/*
+		 * Select the required PLL. This works for platforms where
+		 * there is no shared DPLL.
+		 */
+		pll = &dev_priv->shared_dplls[dpll_id];
+		if (WARN_ON(pll->active_mask)) {
+
+			DRM_ERROR("Shared DPLL in use. active_mask:%x\n",
+				  pll->active_mask);
+			return NULL;
+		}
+		tmp_pll_config = pll->config;
+		if (!bxt_ddi_dp_set_dpll_hw_state(clock,
+						  &pll->config.hw_state)) {
+			DRM_ERROR("Could not setup DPLL\n");
+			pll->config = tmp_pll_config;
+			return NULL;
+		}
+	} else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+		pll = skl_find_link_pll(dev_priv, clock);
+	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
+		pll = hsw_ddi_dp_get_dpll(encoder, clock);
+	}
+	return pll;
+}
+
 void intel_ddi_init(struct drm_device *dev, enum port port)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index cba137f9ad3e..73b6858600ac 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -46,71 +46,70 @@ void intel_device_info_dump(struct drm_i915_private *dev_priv)
 
 static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv)
 {
-	struct intel_device_info *info = mkwrite_device_info(dev_priv);
+	struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
 	u32 fuse, eu_dis;
 
 	fuse = I915_READ(CHV_FUSE_GT);
 
-	info->slice_total = 1;
+	sseu->slice_mask = BIT(0);
 
 	if (!(fuse & CHV_FGT_DISABLE_SS0)) {
-		info->subslice_per_slice++;
+		sseu->subslice_mask |= BIT(0);
 		eu_dis = fuse & (CHV_FGT_EU_DIS_SS0_R0_MASK |
 				 CHV_FGT_EU_DIS_SS0_R1_MASK);
-		info->eu_total += 8 - hweight32(eu_dis);
+		sseu->eu_total += 8 - hweight32(eu_dis);
 	}
 
 	if (!(fuse & CHV_FGT_DISABLE_SS1)) {
-		info->subslice_per_slice++;
+		sseu->subslice_mask |= BIT(1);
 		eu_dis = fuse & (CHV_FGT_EU_DIS_SS1_R0_MASK |
 				 CHV_FGT_EU_DIS_SS1_R1_MASK);
-		info->eu_total += 8 - hweight32(eu_dis);
+		sseu->eu_total += 8 - hweight32(eu_dis);
 	}
 
-	info->subslice_total = info->subslice_per_slice;
 	/*
 	 * CHV expected to always have a uniform distribution of EU
 	 * across subslices.
 	*/
-	info->eu_per_subslice = info->subslice_total ?
-				info->eu_total / info->subslice_total :
+	sseu->eu_per_subslice = sseu_subslice_total(sseu) ?
+				sseu->eu_total / sseu_subslice_total(sseu) :
 				0;
 	/*
 	 * CHV supports subslice power gating on devices with more than
 	 * one subslice, and supports EU power gating on devices with
 	 * more than one EU pair per subslice.
 	*/
-	info->has_slice_pg = 0;
-	info->has_subslice_pg = (info->subslice_total > 1);
-	info->has_eu_pg = (info->eu_per_subslice > 2);
+	sseu->has_slice_pg = 0;
+	sseu->has_subslice_pg = sseu_subslice_total(sseu) > 1;
+	sseu->has_eu_pg = (sseu->eu_per_subslice > 2);
 }
 
 static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 {
 	struct intel_device_info *info = mkwrite_device_info(dev_priv);
+	struct sseu_dev_info *sseu = &info->sseu;
 	int s_max = 3, ss_max = 4, eu_max = 8;
 	int s, ss;
-	u32 fuse2, s_enable, ss_disable, eu_disable;
+	u32 fuse2, eu_disable;
 	u8 eu_mask = 0xff;
 
 	fuse2 = I915_READ(GEN8_FUSE2);
-	s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
-	ss_disable = (fuse2 & GEN9_F2_SS_DIS_MASK) >> GEN9_F2_SS_DIS_SHIFT;
+	sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
 
-	info->slice_total = hweight32(s_enable);
 	/*
 	 * The subslice disable field is global, i.e. it applies
 	 * to each of the enabled slices.
 	*/
-	info->subslice_per_slice = ss_max - hweight32(ss_disable);
-	info->subslice_total = info->slice_total * info->subslice_per_slice;
+	sseu->subslice_mask = (1 << ss_max) - 1;
+	sseu->subslice_mask &= ~((fuse2 & GEN9_F2_SS_DIS_MASK) >>
+				 GEN9_F2_SS_DIS_SHIFT);
 
 	/*
 	 * Iterate through enabled slices and subslices to
 	 * count the total enabled EU.
 	*/
 	for (s = 0; s < s_max; s++) {
-		if (!(s_enable & BIT(s)))
+		if (!(sseu->slice_mask & BIT(s)))
 			/* skip disabled slice */
 			continue;
 
@@ -118,7 +117,7 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 		for (ss = 0; ss < ss_max; ss++) {
 			int eu_per_ss;
 
-			if (ss_disable & BIT(ss))
+			if (!(sseu->subslice_mask & BIT(ss)))
 				/* skip disabled subslice */
 				continue;
 
@@ -131,9 +130,9 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 			 * subslices if they are unbalanced.
 			 */
 			if (eu_per_ss == 7)
-				info->subslice_7eu[s] |= BIT(ss);
+				sseu->subslice_7eu[s] |= BIT(ss);
 
-			info->eu_total += eu_per_ss;
+			sseu->eu_total += eu_per_ss;
 		}
 	}
 
@@ -144,9 +143,9 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 	 * recovery. BXT is expected to be perfectly uniform in EU
 	 * distribution.
 	*/
-	info->eu_per_subslice = info->subslice_total ?
-				DIV_ROUND_UP(info->eu_total,
-					     info->subslice_total) : 0;
+	sseu->eu_per_subslice = sseu_subslice_total(sseu) ?
+				DIV_ROUND_UP(sseu->eu_total,
+					     sseu_subslice_total(sseu)) : 0;
 	/*
 	 * SKL supports slice power gating on devices with more than
 	 * one slice, and supports EU power gating on devices with
@@ -155,15 +154,15 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 	 * supports EU power gating on devices with more than one EU
 	 * pair per subslice.
 	*/
-	info->has_slice_pg =
+	sseu->has_slice_pg =
 		(IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) &&
-		info->slice_total > 1;
-	info->has_subslice_pg =
-		IS_BROXTON(dev_priv) && info->subslice_total > 1;
-	info->has_eu_pg = info->eu_per_subslice > 2;
+		hweight8(sseu->slice_mask) > 1;
+	sseu->has_subslice_pg =
+		IS_BROXTON(dev_priv) && sseu_subslice_total(sseu) > 1;
+	sseu->has_eu_pg = sseu->eu_per_subslice > 2;
 
 	if (IS_BROXTON(dev_priv)) {
-#define IS_SS_DISABLED(_ss_disable, ss)    (_ss_disable & BIT(ss))
+#define IS_SS_DISABLED(ss)	(!(sseu->subslice_mask & BIT(ss)))
 		/*
 		 * There is a HW issue in 2x6 fused down parts that requires
 		 * Pooled EU to be enabled as a WA. The pool configuration
@@ -171,19 +170,18 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 		 * doesn't affect if the device has all 3 subslices enabled.
 		 */
 		/* WaEnablePooledEuFor2x6:bxt */
-		info->has_pooled_eu = ((info->subslice_per_slice == 3) ||
-				       (info->subslice_per_slice == 2 &&
+		info->has_pooled_eu = ((hweight8(sseu->subslice_mask) == 3) ||
+				       (hweight8(sseu->subslice_mask) == 2 &&
 					INTEL_REVID(dev_priv) < BXT_REVID_C0));
 
-		info->min_eu_in_pool = 0;
+		sseu->min_eu_in_pool = 0;
 		if (info->has_pooled_eu) {
-			if (IS_SS_DISABLED(ss_disable, 0) ||
-			    IS_SS_DISABLED(ss_disable, 2))
-				info->min_eu_in_pool = 3;
-			else if (IS_SS_DISABLED(ss_disable, 1))
-				info->min_eu_in_pool = 6;
+			if (IS_SS_DISABLED(2) || IS_SS_DISABLED(0))
+				sseu->min_eu_in_pool = 3;
+			else if (IS_SS_DISABLED(1))
+				sseu->min_eu_in_pool = 6;
 			else
-				info->min_eu_in_pool = 9;
+				sseu->min_eu_in_pool = 9;
 		}
 #undef IS_SS_DISABLED
 	}
@@ -191,14 +189,20 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 
 static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
 {
-	struct intel_device_info *info = mkwrite_device_info(dev_priv);
+	struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
 	const int s_max = 3, ss_max = 3, eu_max = 8;
 	int s, ss;
-	u32 fuse2, eu_disable[s_max], s_enable, ss_disable;
+	u32 fuse2, eu_disable[s_max];
 
 	fuse2 = I915_READ(GEN8_FUSE2);
-	s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
-	ss_disable = (fuse2 & GEN8_F2_SS_DIS_MASK) >> GEN8_F2_SS_DIS_SHIFT;
+	sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
+	/*
+	 * The subslice disable field is global, i.e. it applies
+	 * to each of the enabled slices.
+	 */
+	sseu->subslice_mask = BIT(ss_max) - 1;
+	sseu->subslice_mask &= ~((fuse2 & GEN8_F2_SS_DIS_MASK) >>
+				 GEN8_F2_SS_DIS_SHIFT);
 
 	eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK;
 	eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) |
@@ -208,28 +212,19 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
 			((I915_READ(GEN8_EU_DISABLE2) & GEN8_EU_DIS2_S2_MASK) <<
 			 (32 - GEN8_EU_DIS1_S2_SHIFT));
 
-	info->slice_total = hweight32(s_enable);
-
-	/*
-	 * The subslice disable field is global, i.e. it applies
-	 * to each of the enabled slices.
-	 */
-	info->subslice_per_slice = ss_max - hweight32(ss_disable);
-	info->subslice_total = info->slice_total * info->subslice_per_slice;
-
 	/*
 	 * Iterate through enabled slices and subslices to
 	 * count the total enabled EU.
 	 */
 	for (s = 0; s < s_max; s++) {
-		if (!(s_enable & (0x1 << s)))
+		if (!(sseu->slice_mask & BIT(s)))
 			/* skip disabled slice */
 			continue;
 
 		for (ss = 0; ss < ss_max; ss++) {
 			u32 n_disabled;
 
-			if (ss_disable & (0x1 << ss))
+			if (!(sseu->subslice_mask & BIT(ss)))
 				/* skip disabled subslice */
 				continue;
 
@@ -239,9 +234,9 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
 			 * Record which subslices have 7 EUs.
 			 */
 			if (eu_max - n_disabled == 7)
-				info->subslice_7eu[s] |= 1 << ss;
+				sseu->subslice_7eu[s] |= 1 << ss;
 
-			info->eu_total += eu_max - n_disabled;
+			sseu->eu_total += eu_max - n_disabled;
 		}
 	}
 
@@ -250,16 +245,17 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
 	 * subslices with the exception that any one EU in any one subslice may
 	 * be fused off for die recovery.
 	 */
-	info->eu_per_subslice = info->subslice_total ?
-		DIV_ROUND_UP(info->eu_total, info->subslice_total) : 0;
+	sseu->eu_per_subslice = sseu_subslice_total(sseu) ?
+				DIV_ROUND_UP(sseu->eu_total,
+					     sseu_subslice_total(sseu)) : 0;
 
 	/*
 	 * BDW supports slice power gating on devices with more than
 	 * one slice.
 	 */
-	info->has_slice_pg = (info->slice_total > 1);
-	info->has_subslice_pg = 0;
-	info->has_eu_pg = 0;
+	sseu->has_slice_pg = hweight8(sseu->slice_mask) > 1;
+	sseu->has_subslice_pg = 0;
+	sseu->has_eu_pg = 0;
 }
 
 /*
@@ -374,15 +370,19 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 		info->has_snoop = false;
 
-	DRM_DEBUG_DRIVER("slice total: %u\n", info->slice_total);
-	DRM_DEBUG_DRIVER("subslice total: %u\n", info->subslice_total);
-	DRM_DEBUG_DRIVER("subslice per slice: %u\n", info->subslice_per_slice);
-	DRM_DEBUG_DRIVER("EU total: %u\n", info->eu_total);
-	DRM_DEBUG_DRIVER("EU per subslice: %u\n", info->eu_per_subslice);
+	DRM_DEBUG_DRIVER("slice mask: %04x\n", info->sseu.slice_mask);
+	DRM_DEBUG_DRIVER("slice total: %u\n", hweight8(info->sseu.slice_mask));
+	DRM_DEBUG_DRIVER("subslice total: %u\n",
+			 sseu_subslice_total(&info->sseu));
+	DRM_DEBUG_DRIVER("subslice mask %04x\n", info->sseu.subslice_mask);
+	DRM_DEBUG_DRIVER("subslice per slice: %u\n",
+			 hweight8(info->sseu.subslice_mask));
+	DRM_DEBUG_DRIVER("EU total: %u\n", info->sseu.eu_total);
+	DRM_DEBUG_DRIVER("EU per subslice: %u\n", info->sseu.eu_per_subslice);
 	DRM_DEBUG_DRIVER("has slice power gating: %s\n",
-			 info->has_slice_pg ? "y" : "n");
+			 info->sseu.has_slice_pg ? "y" : "n");
 	DRM_DEBUG_DRIVER("has subslice power gating: %s\n",
-			 info->has_subslice_pg ? "y" : "n");
+			 info->sseu.has_subslice_pg ? "y" : "n");
 	DRM_DEBUG_DRIVER("has EU power gating: %s\n",
-			 info->has_eu_pg ? "y" : "n");
+			 info->sseu.has_eu_pg ? "y" : "n");
 }
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index c457eed76f1f..fbcfed63a76e 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -34,6 +34,7 @@
 #include <drm/drm_edid.h>
 #include <drm/drmP.h>
 #include "intel_drv.h"
+#include "intel_frontbuffer.h"
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_gem_dmabuf.h"
@@ -1201,8 +1202,8 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 	if (HAS_PCH_SPLIT(dev)) {
 		u32 port_sel;
 
-		pp_reg = PCH_PP_CONTROL;
-		port_sel = I915_READ(PCH_PP_ON_DELAYS) & PANEL_PORT_SELECT_MASK;
+		pp_reg = PP_CONTROL(0);
+		port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
 
 		if (port_sel == PANEL_PORT_SELECT_LVDS &&
 		    I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
@@ -1210,10 +1211,10 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 		/* XXX: else fix for eDP */
 	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		/* presumably write lock depends on pipe, not port select */
-		pp_reg = VLV_PIPE_PP_CONTROL(pipe);
+		pp_reg = PP_CONTROL(pipe);
 		panel_pipe = pipe;
 	} else {
-		pp_reg = PP_CONTROL;
+		pp_reg = PP_CONTROL(0);
 		if (I915_READ(LVDS) & LVDS_PIPEB_SELECT)
 			panel_pipe = PIPE_B;
 	}
@@ -1906,7 +1907,7 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
 	}
 }
 
-static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
+void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 {
 	u32 val;
 
@@ -1958,12 +1959,12 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 	 * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
 	 * need the check.
 	 */
-	if (HAS_GMCH_DISPLAY(dev_priv))
+	if (HAS_GMCH_DISPLAY(dev_priv)) {
 		if (intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DSI))
 			assert_dsi_pll_enabled(dev_priv);
 		else
 			assert_pll_enabled(dev_priv, pipe);
-	else {
+	} else {
 		if (crtc->config->has_pch_encoder) {
 			/* if driving the PCH, we need FDI enabled */
 			assert_fdi_rx_pll_enabled(dev_priv, pch_transcoder);
@@ -2146,33 +2147,6 @@ intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
 	}
 }
 
-static void
-intel_fill_fb_info(struct drm_i915_private *dev_priv,
-		   struct drm_framebuffer *fb)
-{
-	struct intel_rotation_info *info = &to_intel_framebuffer(fb)->rot_info;
-	unsigned int tile_size, tile_width, tile_height, cpp;
-
-	tile_size = intel_tile_size(dev_priv);
-
-	cpp = drm_format_plane_cpp(fb->pixel_format, 0);
-	intel_tile_dims(dev_priv, &tile_width, &tile_height,
-			fb->modifier[0], cpp);
-
-	info->plane[0].width = DIV_ROUND_UP(fb->pitches[0], tile_width * cpp);
-	info->plane[0].height = DIV_ROUND_UP(fb->height, tile_height);
-
-	if (info->pixel_format == DRM_FORMAT_NV12) {
-		cpp = drm_format_plane_cpp(fb->pixel_format, 1);
-		intel_tile_dims(dev_priv, &tile_width, &tile_height,
-				fb->modifier[1], cpp);
-
-		info->uv_offset = fb->offsets[1];
-		info->plane[1].width = DIV_ROUND_UP(fb->pitches[1], tile_width * cpp);
-		info->plane[1].height = DIV_ROUND_UP(fb->height / 2, tile_height);
-	}
-}
-
 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
 {
 	if (INTEL_INFO(dev_priv)->gen >= 9)
@@ -2205,16 +2179,15 @@ static unsigned int intel_surf_alignment(const struct drm_i915_private *dev_priv
 	}
 }
 
-int
-intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
-			   unsigned int rotation)
+struct i915_vma *
+intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
 {
 	struct drm_device *dev = fb->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	struct i915_ggtt_view view;
+	struct i915_vma *vma;
 	u32 alignment;
-	int ret;
 
 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
@@ -2239,75 +2212,112 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
 	 */
 	intel_runtime_pm_get(dev_priv);
 
-	ret = i915_gem_object_pin_to_display_plane(obj, alignment,
-						   &view);
-	if (ret)
-		goto err_pm;
-
-	/* Install a fence for tiled scan-out. Pre-i965 always needs a
-	 * fence, whereas 965+ only requires a fence if using
-	 * framebuffer compression.  For simplicity, we always install
-	 * a fence as the cost is not that onerous.
-	 */
-	if (view.type == I915_GGTT_VIEW_NORMAL) {
-		ret = i915_gem_object_get_fence(obj);
-		if (ret == -EDEADLK) {
-			/*
-			 * -EDEADLK means there are no free fences
-			 * no pending flips.
-			 *
-			 * This is propagated to atomic, but it uses
-			 * -EDEADLK to force a locking recovery, so
-			 * change the returned error to -EBUSY.
-			 */
-			ret = -EBUSY;
-			goto err_unpin;
-		} else if (ret)
-			goto err_unpin;
+	vma = i915_gem_object_pin_to_display_plane(obj, alignment, &view);
+	if (IS_ERR(vma))
+		goto err;
 
-		i915_gem_object_pin_fence(obj);
+	if (i915_vma_is_map_and_fenceable(vma)) {
+		/* Install a fence for tiled scan-out. Pre-i965 always needs a
+		 * fence, whereas 965+ only requires a fence if using
+		 * framebuffer compression.  For simplicity, we always, when
+		 * possible, install a fence as the cost is not that onerous.
+		 *
+		 * If we fail to fence the tiled scanout, then either the
+		 * modeset will reject the change (which is highly unlikely as
+		 * the affected systems, all but one, do not have unmappable
+		 * space) or we will not be able to enable full powersaving
+		 * techniques (also likely not to apply due to various limits
+		 * FBC and the like impose on the size of the buffer, which
+		 * presumably we violated anyway with this unmappable buffer).
+		 * Anyway, it is presumably better to stumble onwards with
+		 * something and try to run the system in a "less than optimal"
+		 * mode that matches the user configuration.
+		 */
+		if (i915_vma_get_fence(vma) == 0)
+			i915_vma_pin_fence(vma);
 	}
 
+err:
 	intel_runtime_pm_put(dev_priv);
-	return 0;
-
-err_unpin:
-	i915_gem_object_unpin_from_display_plane(obj, &view);
-err_pm:
-	intel_runtime_pm_put(dev_priv);
-	return ret;
+	return vma;
 }
 
 void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
 {
 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	struct i915_ggtt_view view;
+	struct i915_vma *vma;
 
 	WARN_ON(!mutex_is_locked(&obj->base.dev->struct_mutex));
 
 	intel_fill_fb_ggtt_view(&view, fb, rotation);
+	vma = i915_gem_object_to_ggtt(obj, &view);
 
-	if (view.type == I915_GGTT_VIEW_NORMAL)
-		i915_gem_object_unpin_fence(obj);
+	i915_vma_unpin_fence(vma);
+	i915_gem_object_unpin_from_display_plane(vma);
+}
 
-	i915_gem_object_unpin_from_display_plane(obj, &view);
+static int intel_fb_pitch(const struct drm_framebuffer *fb, int plane,
+			  unsigned int rotation)
+{
+	if (intel_rotation_90_or_270(rotation))
+		return to_intel_framebuffer(fb)->rotated[plane].pitch;
+	else
+		return fb->pitches[plane];
+}
+
+/*
+ * Convert the x/y offsets into a linear offset.
+ * Only valid with 0/180 degree rotation, which is fine since linear
+ * offset is only used with linear buffers on pre-hsw and tiled buffers
+ * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
+ */
+u32 intel_fb_xy_to_linear(int x, int y,
+			  const struct intel_plane_state *state,
+			  int plane)
+{
+	const struct drm_framebuffer *fb = state->base.fb;
+	unsigned int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
+	unsigned int pitch = fb->pitches[plane];
+
+	return y * pitch + x * cpp;
+}
+
+/*
+ * Add the x/y offsets derived from fb->offsets[] to the user
+ * specified plane src x/y offsets. The resulting x/y offsets
+ * specify the start of scanout from the beginning of the gtt mapping.
+ */
+void intel_add_fb_offsets(int *x, int *y,
+			  const struct intel_plane_state *state,
+			  int plane)
+
+{
+	const struct intel_framebuffer *intel_fb = to_intel_framebuffer(state->base.fb);
+	unsigned int rotation = state->base.rotation;
+
+	if (intel_rotation_90_or_270(rotation)) {
+		*x += intel_fb->rotated[plane].x;
+		*y += intel_fb->rotated[plane].y;
+	} else {
+		*x += intel_fb->normal[plane].x;
+		*y += intel_fb->normal[plane].y;
+	}
 }
 
 /*
- * Adjust the tile offset by moving the difference into
- * the x/y offsets.
- *
  * Input tile dimensions and pitch must already be
  * rotated to match x and y, and in pixel units.
  */
-static u32 intel_adjust_tile_offset(int *x, int *y,
-				    unsigned int tile_width,
-				    unsigned int tile_height,
-				    unsigned int tile_size,
-				    unsigned int pitch_tiles,
-				    u32 old_offset,
-				    u32 new_offset)
-{
+static u32 _intel_adjust_tile_offset(int *x, int *y,
+				     unsigned int tile_width,
+				     unsigned int tile_height,
+				     unsigned int tile_size,
+				     unsigned int pitch_tiles,
+				     u32 old_offset,
+				     u32 new_offset)
+{
+	unsigned int pitch_pixels = pitch_tiles * tile_width;
 	unsigned int tiles;
 
 	WARN_ON(old_offset & (tile_size - 1));
@@ -2319,6 +2329,54 @@ static u32 intel_adjust_tile_offset(int *x, int *y,
 	*y += tiles / pitch_tiles * tile_height;
 	*x += tiles % pitch_tiles * tile_width;
 
+	/* minimize x in case it got needlessly big */
+	*y += *x / pitch_pixels * tile_height;
+	*x %= pitch_pixels;
+
+	return new_offset;
+}
+
+/*
+ * Adjust the tile offset by moving the difference into
+ * the x/y offsets.
+ */
+static u32 intel_adjust_tile_offset(int *x, int *y,
+				    const struct intel_plane_state *state, int plane,
+				    u32 old_offset, u32 new_offset)
+{
+	const struct drm_i915_private *dev_priv = to_i915(state->base.plane->dev);
+	const struct drm_framebuffer *fb = state->base.fb;
+	unsigned int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
+	unsigned int rotation = state->base.rotation;
+	unsigned int pitch = intel_fb_pitch(fb, plane, rotation);
+
+	WARN_ON(new_offset > old_offset);
+
+	if (fb->modifier[plane] != DRM_FORMAT_MOD_NONE) {
+		unsigned int tile_size, tile_width, tile_height;
+		unsigned int pitch_tiles;
+
+		tile_size = intel_tile_size(dev_priv);
+		intel_tile_dims(dev_priv, &tile_width, &tile_height,
+				fb->modifier[plane], cpp);
+
+		if (intel_rotation_90_or_270(rotation)) {
+			pitch_tiles = pitch / tile_height;
+			swap(tile_width, tile_height);
+		} else {
+			pitch_tiles = pitch / (tile_width * cpp);
+		}
+
+		_intel_adjust_tile_offset(x, y, tile_width, tile_height,
+					  tile_size, pitch_tiles,
+					  old_offset, new_offset);
+	} else {
+		old_offset += *y * pitch + *x * cpp;
+
+		*y = (old_offset - new_offset) / pitch;
+		*x = ((old_offset - new_offset) - *y * pitch) / cpp;
+	}
+
 	return new_offset;
 }
 
@@ -2329,18 +2387,24 @@ static u32 intel_adjust_tile_offset(int *x, int *y,
  * In the 90/270 rotated case, x and y are assumed
  * to be already rotated to match the rotated GTT view, and
  * pitch is the tile_height aligned framebuffer height.
+ *
+ * This function is used when computing the derived information
+ * under intel_framebuffer, so using any of that information
+ * here is not allowed. Anything under drm_framebuffer can be
+ * used. This is why the user has to pass in the pitch since it
+ * is specified in the rotated orientation.
  */
-u32 intel_compute_tile_offset(int *x, int *y,
-			      const struct drm_framebuffer *fb, int plane,
-			      unsigned int pitch,
-			      unsigned int rotation)
+static u32 _intel_compute_tile_offset(const struct drm_i915_private *dev_priv,
+				      int *x, int *y,
+				      const struct drm_framebuffer *fb, int plane,
+				      unsigned int pitch,
+				      unsigned int rotation,
+				      u32 alignment)
 {
-	const struct drm_i915_private *dev_priv = to_i915(fb->dev);
 	uint64_t fb_modifier = fb->modifier[plane];
 	unsigned int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
-	u32 offset, offset_aligned, alignment;
+	u32 offset, offset_aligned;
 
-	alignment = intel_surf_alignment(dev_priv, fb_modifier);
 	if (alignment)
 		alignment--;
 
@@ -2368,9 +2432,9 @@ u32 intel_compute_tile_offset(int *x, int *y,
 		offset = (tile_rows * pitch_tiles + tiles) * tile_size;
 		offset_aligned = offset & ~alignment;
 
-		intel_adjust_tile_offset(x, y, tile_width, tile_height,
-					 tile_size, pitch_tiles,
-					 offset, offset_aligned);
+		_intel_adjust_tile_offset(x, y, tile_width, tile_height,
+					  tile_size, pitch_tiles,
+					  offset, offset_aligned);
 	} else {
 		offset = *y * pitch + *x * cpp;
 		offset_aligned = offset & ~alignment;
@@ -2382,6 +2446,177 @@ u32 intel_compute_tile_offset(int *x, int *y,
 	return offset_aligned;
 }
 
+u32 intel_compute_tile_offset(int *x, int *y,
+			      const struct intel_plane_state *state,
+			      int plane)
+{
+	const struct drm_i915_private *dev_priv = to_i915(state->base.plane->dev);
+	const struct drm_framebuffer *fb = state->base.fb;
+	unsigned int rotation = state->base.rotation;
+	int pitch = intel_fb_pitch(fb, plane, rotation);
+	u32 alignment;
+
+	/* AUX_DIST needs only 4K alignment */
+	if (fb->pixel_format == DRM_FORMAT_NV12 && plane == 1)
+		alignment = 4096;
+	else
+		alignment = intel_surf_alignment(dev_priv, fb->modifier[plane]);
+
+	return _intel_compute_tile_offset(dev_priv, x, y, fb, plane, pitch,
+					  rotation, alignment);
+}
+
+/* Convert the fb->offset[] linear offset into x/y offsets */
+static void intel_fb_offset_to_xy(int *x, int *y,
+				  const struct drm_framebuffer *fb, int plane)
+{
+	unsigned int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
+	unsigned int pitch = fb->pitches[plane];
+	u32 linear_offset = fb->offsets[plane];
+
+	*y = linear_offset / pitch;
+	*x = linear_offset % pitch / cpp;
+}
+
+static unsigned int intel_fb_modifier_to_tiling(uint64_t fb_modifier)
+{
+	switch (fb_modifier) {
+	case I915_FORMAT_MOD_X_TILED:
+		return I915_TILING_X;
+	case I915_FORMAT_MOD_Y_TILED:
+		return I915_TILING_Y;
+	default:
+		return I915_TILING_NONE;
+	}
+}
+
+static int
+intel_fill_fb_info(struct drm_i915_private *dev_priv,
+		   struct drm_framebuffer *fb)
+{
+	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
+	struct intel_rotation_info *rot_info = &intel_fb->rot_info;
+	u32 gtt_offset_rotated = 0;
+	unsigned int max_size = 0;
+	uint32_t format = fb->pixel_format;
+	int i, num_planes = drm_format_num_planes(format);
+	unsigned int tile_size = intel_tile_size(dev_priv);
+
+	for (i = 0; i < num_planes; i++) {
+		unsigned int width, height;
+		unsigned int cpp, size;
+		u32 offset;
+		int x, y;
+
+		cpp = drm_format_plane_cpp(format, i);
+		width = drm_format_plane_width(fb->width, format, i);
+		height = drm_format_plane_height(fb->height, format, i);
+
+		intel_fb_offset_to_xy(&x, &y, fb, i);
+
+		/*
+		 * The fence (if used) is aligned to the start of the object
+		 * so having the framebuffer wrap around across the edge of the
+		 * fenced region doesn't really work. We have no API to configure
+		 * the fence start offset within the object (nor could we probably
+		 * on gen2/3). So it's just easier if we just require that the
+		 * fb layout agrees with the fence layout. We already check that the
+		 * fb stride matches the fence stride elsewhere.
+		 */
+		if (i915_gem_object_is_tiled(intel_fb->obj) &&
+		    (x + width) * cpp > fb->pitches[i]) {
+			DRM_DEBUG("bad fb plane %d offset: 0x%x\n",
+				  i, fb->offsets[i]);
+			return -EINVAL;
+		}
+
+		/*
+		 * First pixel of the framebuffer from
+		 * the start of the normal gtt mapping.
+		 */
+		intel_fb->normal[i].x = x;
+		intel_fb->normal[i].y = y;
+
+		offset = _intel_compute_tile_offset(dev_priv, &x, &y,
+						    fb, 0, fb->pitches[i],
+						    DRM_ROTATE_0, tile_size);
+		offset /= tile_size;
+
+		if (fb->modifier[i] != DRM_FORMAT_MOD_NONE) {
+			unsigned int tile_width, tile_height;
+			unsigned int pitch_tiles;
+			struct drm_rect r;
+
+			intel_tile_dims(dev_priv, &tile_width, &tile_height,
+					fb->modifier[i], cpp);
+
+			rot_info->plane[i].offset = offset;
+			rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
+			rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
+			rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
+
+			intel_fb->rotated[i].pitch =
+				rot_info->plane[i].height * tile_height;
+
+			/* how many tiles does this plane need */
+			size = rot_info->plane[i].stride * rot_info->plane[i].height;
+			/*
+			 * If the plane isn't horizontally tile aligned,
+			 * we need one more tile.
+			 */
+			if (x != 0)
+				size++;
+
+			/* rotate the x/y offsets to match the GTT view */
+			r.x1 = x;
+			r.y1 = y;
+			r.x2 = x + width;
+			r.y2 = y + height;
+			drm_rect_rotate(&r,
+					rot_info->plane[i].width * tile_width,
+					rot_info->plane[i].height * tile_height,
+					DRM_ROTATE_270);
+			x = r.x1;
+			y = r.y1;
+
+			/* rotate the tile dimensions to match the GTT view */
+			pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
+			swap(tile_width, tile_height);
+
+			/*
+			 * We only keep the x/y offsets, so push all of the
+			 * gtt offset into the x/y offsets.
+			 */
+			_intel_adjust_tile_offset(&x, &y, tile_size,
+						  tile_width, tile_height, pitch_tiles,
+						  gtt_offset_rotated * tile_size, 0);
+
+			gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
+
+			/*
+			 * First pixel of the framebuffer from
+			 * the start of the rotated gtt mapping.
+			 */
+			intel_fb->rotated[i].x = x;
+			intel_fb->rotated[i].y = y;
+		} else {
+			size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
+					    x * cpp, tile_size);
+		}
+
+		/* how many tiles in total needed in the bo */
+		max_size = max(max_size, offset + size);
+	}
+
+	if (max_size * tile_size > to_intel_framebuffer(fb)->obj->base.size) {
+		DRM_DEBUG("fb too big for bo (need %u bytes, have %zu bytes)\n",
+			  max_size * tile_size, to_intel_framebuffer(fb)->obj->base.size);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int i9xx_format_to_fourcc(int format)
 {
 	switch (format) {
@@ -2465,9 +2700,8 @@ intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
 		return false;
 	}
 
-	obj->tiling_mode = plane_config->tiling;
-	if (obj->tiling_mode == I915_TILING_X)
-		obj->stride = fb->pitches[0];
+	if (plane_config->tiling == I915_TILING_X)
+		obj->tiling_and_stride = fb->pitches[0] | I915_TILING_X;
 
 	mode_cmd.pixel_format = fb->pixel_format;
 	mode_cmd.width = fb->width;
@@ -2488,7 +2722,7 @@ intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
 	return true;
 
 out_unref_obj:
-	drm_gem_object_unreference(&obj->base);
+	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
 	return false;
 }
@@ -2552,7 +2786,7 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
 			continue;
 
 		obj = intel_fb_obj(fb);
-		if (i915_gem_obj_ggtt_offset(obj) == plane_config->base) {
+		if (i915_gem_object_ggtt_offset(obj, NULL) == plane_config->base) {
 			drm_framebuffer_reference(fb);
 			goto valid_fb;
 		}
@@ -2565,7 +2799,7 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
 	 * simplest solution is to just disable the primary plane now and
 	 * pretend the BIOS never had it enabled.
 	 */
-	to_intel_plane_state(plane_state)->visible = false;
+	to_intel_plane_state(plane_state)->base.visible = false;
 	crtc_state->plane_mask &= ~(1 << drm_plane_index(primary));
 	intel_pre_disable_primary_noatomic(&intel_crtc->base);
 	intel_plane->disable_plane(primary, &intel_crtc->base);
@@ -2583,24 +2817,188 @@ valid_fb:
 	plane_state->crtc_w = fb->width;
 	plane_state->crtc_h = fb->height;
 
-	intel_state->src.x1 = plane_state->src_x;
-	intel_state->src.y1 = plane_state->src_y;
-	intel_state->src.x2 = plane_state->src_x + plane_state->src_w;
-	intel_state->src.y2 = plane_state->src_y + plane_state->src_h;
-	intel_state->dst.x1 = plane_state->crtc_x;
-	intel_state->dst.y1 = plane_state->crtc_y;
-	intel_state->dst.x2 = plane_state->crtc_x + plane_state->crtc_w;
-	intel_state->dst.y2 = plane_state->crtc_y + plane_state->crtc_h;
+	intel_state->base.src.x1 = plane_state->src_x;
+	intel_state->base.src.y1 = plane_state->src_y;
+	intel_state->base.src.x2 = plane_state->src_x + plane_state->src_w;
+	intel_state->base.src.y2 = plane_state->src_y + plane_state->src_h;
+	intel_state->base.dst.x1 = plane_state->crtc_x;
+	intel_state->base.dst.y1 = plane_state->crtc_y;
+	intel_state->base.dst.x2 = plane_state->crtc_x + plane_state->crtc_w;
+	intel_state->base.dst.y2 = plane_state->crtc_y + plane_state->crtc_h;
 
 	obj = intel_fb_obj(fb);
-	if (obj->tiling_mode != I915_TILING_NONE)
+	if (i915_gem_object_is_tiled(obj))
 		dev_priv->preserve_bios_swizzle = true;
 
 	drm_framebuffer_reference(fb);
 	primary->fb = primary->state->fb = fb;
 	primary->crtc = primary->state->crtc = &intel_crtc->base;
 	intel_crtc->base.state->plane_mask |= (1 << drm_plane_index(primary));
-	obj->frontbuffer_bits |= to_intel_plane(primary)->frontbuffer_bit;
+	atomic_or(to_intel_plane(primary)->frontbuffer_bit,
+		  &obj->frontbuffer_bits);
+}
+
+static int skl_max_plane_width(const struct drm_framebuffer *fb, int plane,
+			       unsigned int rotation)
+{
+	int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
+
+	switch (fb->modifier[plane]) {
+	case DRM_FORMAT_MOD_NONE:
+	case I915_FORMAT_MOD_X_TILED:
+		switch (cpp) {
+		case 8:
+			return 4096;
+		case 4:
+		case 2:
+		case 1:
+			return 8192;
+		default:
+			MISSING_CASE(cpp);
+			break;
+		}
+		break;
+	case I915_FORMAT_MOD_Y_TILED:
+	case I915_FORMAT_MOD_Yf_TILED:
+		switch (cpp) {
+		case 8:
+			return 2048;
+		case 4:
+			return 4096;
+		case 2:
+		case 1:
+			return 8192;
+		default:
+			MISSING_CASE(cpp);
+			break;
+		}
+		break;
+	default:
+		MISSING_CASE(fb->modifier[plane]);
+	}
+
+	return 2048;
+}
+
+static int skl_check_main_surface(struct intel_plane_state *plane_state)
+{
+	const struct drm_i915_private *dev_priv = to_i915(plane_state->base.plane->dev);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	unsigned int rotation = plane_state->base.rotation;
+	int x = plane_state->base.src.x1 >> 16;
+	int y = plane_state->base.src.y1 >> 16;
+	int w = drm_rect_width(&plane_state->base.src) >> 16;
+	int h = drm_rect_height(&plane_state->base.src) >> 16;
+	int max_width = skl_max_plane_width(fb, 0, rotation);
+	int max_height = 4096;
+	u32 alignment, offset, aux_offset = plane_state->aux.offset;
+
+	if (w > max_width || h > max_height) {
+		DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
+			      w, h, max_width, max_height);
+		return -EINVAL;
+	}
+
+	intel_add_fb_offsets(&x, &y, plane_state, 0);
+	offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
+
+	alignment = intel_surf_alignment(dev_priv, fb->modifier[0]);
+
+	/*
+	 * AUX surface offset is specified as the distance from the
+	 * main surface offset, and it must be non-negative. Make
+	 * sure that is what we will get.
+	 */
+	if (offset > aux_offset)
+		offset = intel_adjust_tile_offset(&x, &y, plane_state, 0,
+						  offset, aux_offset & ~(alignment - 1));
+
+	/*
+	 * When using an X-tiled surface, the plane blows up
+	 * if the x offset + width exceed the stride.
+	 *
+	 * TODO: linear and Y-tiled seem fine, Yf untested,
+	 */
+	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) {
+		int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
+
+		while ((x + w) * cpp > fb->pitches[0]) {
+			if (offset == 0) {
+				DRM_DEBUG_KMS("Unable to find suitable display surface offset\n");
+				return -EINVAL;
+			}
+
+			offset = intel_adjust_tile_offset(&x, &y, plane_state, 0,
+							  offset, offset - alignment);
+		}
+	}
+
+	plane_state->main.offset = offset;
+	plane_state->main.x = x;
+	plane_state->main.y = y;
+
+	return 0;
+}
+
+static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
+{
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	unsigned int rotation = plane_state->base.rotation;
+	int max_width = skl_max_plane_width(fb, 1, rotation);
+	int max_height = 4096;
+	int x = plane_state->base.src.x1 >> 17;
+	int y = plane_state->base.src.y1 >> 17;
+	int w = drm_rect_width(&plane_state->base.src) >> 17;
+	int h = drm_rect_height(&plane_state->base.src) >> 17;
+	u32 offset;
+
+	intel_add_fb_offsets(&x, &y, plane_state, 1);
+	offset = intel_compute_tile_offset(&x, &y, plane_state, 1);
+
+	/* FIXME not quite sure how/if these apply to the chroma plane */
+	if (w > max_width || h > max_height) {
+		DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
+			      w, h, max_width, max_height);
+		return -EINVAL;
+	}
+
+	plane_state->aux.offset = offset;
+	plane_state->aux.x = x;
+	plane_state->aux.y = y;
+
+	return 0;
+}
+
+int skl_check_plane_surface(struct intel_plane_state *plane_state)
+{
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	unsigned int rotation = plane_state->base.rotation;
+	int ret;
+
+	/* Rotate src coordinates to match rotated GTT view */
+	if (intel_rotation_90_or_270(rotation))
+		drm_rect_rotate(&plane_state->base.src,
+				fb->width, fb->height, DRM_ROTATE_270);
+
+	/*
+	 * Handle the AUX surface first since
+	 * the main surface setup depends on it.
+	 */
+	if (fb->pixel_format == DRM_FORMAT_NV12) {
+		ret = skl_check_nv12_aux_surface(plane_state);
+		if (ret)
+			return ret;
+	} else {
+		plane_state->aux.offset = ~0xfff;
+		plane_state->aux.x = 0;
+		plane_state->aux.y = 0;
+	}
+
+	ret = skl_check_main_surface(plane_state);
+	if (ret)
+		return ret;
+
+	return 0;
 }
 
 static void i9xx_update_primary_plane(struct drm_plane *primary,
@@ -2617,9 +3015,8 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 	u32 dspcntr;
 	i915_reg_t reg = DSPCNTR(plane);
 	unsigned int rotation = plane_state->base.rotation;
-	int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
-	int x = plane_state->src.x1 >> 16;
-	int y = plane_state->src.y1 >> 16;
+	int x = plane_state->base.src.x1 >> 16;
+	int y = plane_state->base.src.y1 >> 16;
 
 	dspcntr = DISPPLANE_GAMMA_ENABLE;
 
@@ -2670,37 +3067,31 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 		BUG();
 	}
 
-	if (INTEL_INFO(dev)->gen >= 4 &&
-	    obj->tiling_mode != I915_TILING_NONE)
+	if (INTEL_GEN(dev_priv) >= 4 &&
+	    fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
 		dspcntr |= DISPPLANE_TILED;
 
 	if (IS_G4X(dev))
 		dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
 
-	linear_offset = y * fb->pitches[0] + x * cpp;
+	intel_add_fb_offsets(&x, &y, plane_state, 0);
 
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_INFO(dev)->gen >= 4)
 		intel_crtc->dspaddr_offset =
-			intel_compute_tile_offset(&x, &y, fb, 0,
-						  fb->pitches[0], rotation);
-		linear_offset -= intel_crtc->dspaddr_offset;
-	} else {
-		intel_crtc->dspaddr_offset = linear_offset;
-	}
+			intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	if (rotation == BIT(DRM_ROTATE_180)) {
+	if (rotation == DRM_ROTATE_180) {
 		dspcntr |= DISPPLANE_ROTATE_180;
 
 		x += (crtc_state->pipe_src_w - 1);
 		y += (crtc_state->pipe_src_h - 1);
-
-		/* Finding the last pixel of the last line of the display
-		data and adding to linear_offset*/
-		linear_offset +=
-			(crtc_state->pipe_src_h - 1) * fb->pitches[0] +
-			(crtc_state->pipe_src_w - 1) * cpp;
 	}
 
+	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
+
+	if (INTEL_INFO(dev)->gen < 4)
+		intel_crtc->dspaddr_offset = linear_offset;
+
 	intel_crtc->adjusted_x = x;
 	intel_crtc->adjusted_y = y;
 
@@ -2709,11 +3100,12 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 	I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
 	if (INTEL_INFO(dev)->gen >= 4) {
 		I915_WRITE(DSPSURF(plane),
-			   i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
+			   intel_fb_gtt_offset(fb, rotation) +
+			   intel_crtc->dspaddr_offset);
 		I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
 		I915_WRITE(DSPLINOFF(plane), linear_offset);
 	} else
-		I915_WRITE(DSPADDR(plane), i915_gem_obj_ggtt_offset(obj) + linear_offset);
+		I915_WRITE(DSPADDR(plane), i915_gem_object_ggtt_offset(obj, NULL) + linear_offset);
 	POSTING_READ(reg);
 }
 
@@ -2741,15 +3133,13 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_framebuffer *fb = plane_state->base.fb;
-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	int plane = intel_crtc->plane;
 	u32 linear_offset;
 	u32 dspcntr;
 	i915_reg_t reg = DSPCNTR(plane);
 	unsigned int rotation = plane_state->base.rotation;
-	int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
-	int x = plane_state->src.x1 >> 16;
-	int y = plane_state->src.y1 >> 16;
+	int x = plane_state->base.src.x1 >> 16;
+	int y = plane_state->base.src.y1 >> 16;
 
 	dspcntr = DISPPLANE_GAMMA_ENABLE;
 	dspcntr |= DISPLAY_PLANE_ENABLE;
@@ -2780,32 +3170,28 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
 		BUG();
 	}
 
-	if (obj->tiling_mode != I915_TILING_NONE)
+	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
 		dspcntr |= DISPPLANE_TILED;
 
 	if (!IS_HASWELL(dev) && !IS_BROADWELL(dev))
 		dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
 
-	linear_offset = y * fb->pitches[0] + x * cpp;
+	intel_add_fb_offsets(&x, &y, plane_state, 0);
+
 	intel_crtc->dspaddr_offset =
-		intel_compute_tile_offset(&x, &y, fb, 0,
-					  fb->pitches[0], rotation);
-	linear_offset -= intel_crtc->dspaddr_offset;
-	if (rotation == BIT(DRM_ROTATE_180)) {
+		intel_compute_tile_offset(&x, &y, plane_state, 0);
+
+	if (rotation == DRM_ROTATE_180) {
 		dspcntr |= DISPPLANE_ROTATE_180;
 
 		if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
 			x += (crtc_state->pipe_src_w - 1);
 			y += (crtc_state->pipe_src_h - 1);
-
-			/* Finding the last pixel of the last line of the display
-			data and adding to linear_offset*/
-			linear_offset +=
-				(crtc_state->pipe_src_h - 1) * fb->pitches[0] +
-				(crtc_state->pipe_src_w - 1) * cpp;
 		}
 	}
 
+	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
+
 	intel_crtc->adjusted_x = x;
 	intel_crtc->adjusted_y = y;
 
@@ -2813,7 +3199,8 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
 
 	I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
 	I915_WRITE(DSPSURF(plane),
-		   i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
+		   intel_fb_gtt_offset(fb, rotation) +
+		   intel_crtc->dspaddr_offset);
 	if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
 		I915_WRITE(DSPOFFSET(plane), (y << 16) | x);
 	} else {
@@ -2835,32 +3222,21 @@ u32 intel_fb_stride_alignment(const struct drm_i915_private *dev_priv,
 	}
 }
 
-u32 intel_plane_obj_offset(struct intel_plane *intel_plane,
-			   struct drm_i915_gem_object *obj,
-			   unsigned int plane)
+u32 intel_fb_gtt_offset(struct drm_framebuffer *fb,
+			unsigned int rotation)
 {
+	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	struct i915_ggtt_view view;
 	struct i915_vma *vma;
-	u64 offset;
 
-	intel_fill_fb_ggtt_view(&view, intel_plane->base.state->fb,
-				intel_plane->base.state->rotation);
+	intel_fill_fb_ggtt_view(&view, fb, rotation);
 
-	vma = i915_gem_obj_to_ggtt_view(obj, &view);
+	vma = i915_gem_object_to_ggtt(obj, &view);
 	if (WARN(!vma, "ggtt vma for display object not found! (view=%u)\n",
-		view.type))
+		 view.type))
 		return -1;
 
-	offset = vma->node.start;
-
-	if (plane == 1) {
-		offset += vma->ggtt_view.params.rotated.uv_start_page *
-			  PAGE_SIZE;
-	}
-
-	WARN_ON(upper_32_bits(offset));
-
-	return lower_32_bits(offset);
+	return i915_ggtt_offset(vma);
 }
 
 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
@@ -2890,6 +3266,28 @@ static void skl_detach_scalers(struct intel_crtc *intel_crtc)
 	}
 }
 
+u32 skl_plane_stride(const struct drm_framebuffer *fb, int plane,
+		     unsigned int rotation)
+{
+	const struct drm_i915_private *dev_priv = to_i915(fb->dev);
+	u32 stride = intel_fb_pitch(fb, plane, rotation);
+
+	/*
+	 * The stride is either expressed as a multiple of 64 bytes chunks for
+	 * linear buffers or in number of tiles for tiled buffers.
+	 */
+	if (intel_rotation_90_or_270(rotation)) {
+		int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
+
+		stride /= intel_tile_height(dev_priv, fb->modifier[0], cpp);
+	} else {
+		stride /= intel_fb_stride_alignment(dev_priv, fb->modifier[0],
+						    fb->pixel_format);
+	}
+
+	return stride;
+}
+
 u32 skl_plane_ctl_format(uint32_t pixel_format)
 {
 	switch (pixel_format) {
@@ -2952,17 +3350,17 @@ u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
 u32 skl_plane_ctl_rotation(unsigned int rotation)
 {
 	switch (rotation) {
-	case BIT(DRM_ROTATE_0):
+	case DRM_ROTATE_0:
 		break;
 	/*
 	 * DRM_ROTATE_ is counter clockwise to stay compatible with Xrandr
 	 * while i915 HW rotation is clockwise, thats why this swapping.
 	 */
-	case BIT(DRM_ROTATE_90):
+	case DRM_ROTATE_90:
 		return PLANE_CTL_ROTATE_270;
-	case BIT(DRM_ROTATE_180):
+	case DRM_ROTATE_180:
 		return PLANE_CTL_ROTATE_180;
-	case BIT(DRM_ROTATE_270):
+	case DRM_ROTATE_270:
 		return PLANE_CTL_ROTATE_90;
 	default:
 		MISSING_CASE(rotation);
@@ -2979,22 +3377,21 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_framebuffer *fb = plane_state->base.fb;
-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
+	const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
 	int pipe = intel_crtc->pipe;
-	u32 plane_ctl, stride_div, stride;
-	u32 tile_height, plane_offset, plane_size;
+	u32 plane_ctl;
 	unsigned int rotation = plane_state->base.rotation;
-	int x_offset, y_offset;
-	u32 surf_addr;
+	u32 stride = skl_plane_stride(fb, 0, rotation);
+	u32 surf_addr = plane_state->main.offset;
 	int scaler_id = plane_state->scaler_id;
-	int src_x = plane_state->src.x1 >> 16;
-	int src_y = plane_state->src.y1 >> 16;
-	int src_w = drm_rect_width(&plane_state->src) >> 16;
-	int src_h = drm_rect_height(&plane_state->src) >> 16;
-	int dst_x = plane_state->dst.x1;
-	int dst_y = plane_state->dst.y1;
-	int dst_w = drm_rect_width(&plane_state->dst);
-	int dst_h = drm_rect_height(&plane_state->dst);
+	int src_x = plane_state->main.x;
+	int src_y = plane_state->main.y;
+	int src_w = drm_rect_width(&plane_state->base.src) >> 16;
+	int src_h = drm_rect_height(&plane_state->base.src) >> 16;
+	int dst_x = plane_state->base.dst.x1;
+	int dst_y = plane_state->base.dst.y1;
+	int dst_w = drm_rect_width(&plane_state->base.dst);
+	int dst_h = drm_rect_height(&plane_state->base.dst);
 
 	plane_ctl = PLANE_CTL_ENABLE |
 		    PLANE_CTL_PIPE_GAMMA_ENABLE |
@@ -3005,36 +3402,24 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
 	plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
 	plane_ctl |= skl_plane_ctl_rotation(rotation);
 
-	stride_div = intel_fb_stride_alignment(dev_priv, fb->modifier[0],
-					       fb->pixel_format);
-	surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj, 0);
+	/* Sizes are 0 based */
+	src_w--;
+	src_h--;
+	dst_w--;
+	dst_h--;
 
-	WARN_ON(drm_rect_width(&plane_state->src) == 0);
-
-	if (intel_rotation_90_or_270(rotation)) {
-		int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
+	intel_crtc->dspaddr_offset = surf_addr;
 
-		/* stride = Surface height in tiles */
-		tile_height = intel_tile_height(dev_priv, fb->modifier[0], cpp);
-		stride = DIV_ROUND_UP(fb->height, tile_height);
-		x_offset = stride * tile_height - src_y - src_h;
-		y_offset = src_x;
-		plane_size = (src_w - 1) << 16 | (src_h - 1);
-	} else {
-		stride = fb->pitches[0] / stride_div;
-		x_offset = src_x;
-		y_offset = src_y;
-		plane_size = (src_h - 1) << 16 | (src_w - 1);
-	}
-	plane_offset = y_offset << 16 | x_offset;
+	intel_crtc->adjusted_x = src_x;
+	intel_crtc->adjusted_y = src_y;
 
-	intel_crtc->adjusted_x = x_offset;
-	intel_crtc->adjusted_y = y_offset;
+	if (wm->dirty_pipes & drm_crtc_mask(&intel_crtc->base))
+		skl_write_plane_wm(intel_crtc, wm, 0);
 
 	I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl);
-	I915_WRITE(PLANE_OFFSET(pipe, 0), plane_offset);
-	I915_WRITE(PLANE_SIZE(pipe, 0), plane_size);
+	I915_WRITE(PLANE_OFFSET(pipe, 0), (src_y << 16) | src_x);
 	I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
+	I915_WRITE(PLANE_SIZE(pipe, 0), (src_h << 16) | src_w);
 
 	if (scaler_id >= 0) {
 		uint32_t ps_ctrl = 0;
@@ -3051,7 +3436,8 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
 		I915_WRITE(PLANE_POS(pipe, 0), (dst_y << 16) | dst_x);
 	}
 
-	I915_WRITE(PLANE_SURF(pipe, 0), surf_addr);
+	I915_WRITE(PLANE_SURF(pipe, 0),
+		   intel_fb_gtt_offset(fb, rotation) + surf_addr);
 
 	POSTING_READ(PLANE_SURF(pipe, 0));
 }
@@ -3061,7 +3447,15 @@ static void skylake_disable_primary_plane(struct drm_plane *primary,
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int pipe = to_intel_crtc(crtc)->pipe;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	int pipe = intel_crtc->pipe;
+
+	/*
+	 * We only populate skl_results on watermark updates, and if the
+	 * plane's visiblity isn't actually changing neither is its watermarks.
+	 */
+	if (!crtc->primary->state->visible)
+		skl_write_plane_wm(intel_crtc, &dev_priv->wm.skl_results, 0);
 
 	I915_WRITE(PLANE_CTL(pipe, 0), 0);
 	I915_WRITE(PLANE_SURF(pipe, 0), 0);
@@ -3093,40 +3487,113 @@ static void intel_update_primary_planes(struct drm_device *dev)
 
 	for_each_crtc(dev, crtc) {
 		struct intel_plane *plane = to_intel_plane(crtc->primary);
-		struct intel_plane_state *plane_state;
-
-		drm_modeset_lock_crtc(crtc, &plane->base);
-		plane_state = to_intel_plane_state(plane->base.state);
+		struct intel_plane_state *plane_state =
+			to_intel_plane_state(plane->base.state);
 
-		if (plane_state->visible)
+		if (plane_state->base.visible)
 			plane->update_plane(&plane->base,
 					    to_intel_crtc_state(crtc->state),
 					    plane_state);
+	}
+}
+
+static int
+__intel_display_resume(struct drm_device *dev,
+		       struct drm_atomic_state *state)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_crtc *crtc;
+	int i, ret;
 
-		drm_modeset_unlock_crtc(crtc);
+	intel_modeset_setup_hw_state(dev);
+	i915_redisable_vga(dev);
+
+	if (!state)
+		return 0;
+
+	for_each_crtc_in_state(state, crtc, crtc_state, i) {
+		/*
+		 * Force recalculation even if we restore
+		 * current state. With fast modeset this may not result
+		 * in a modeset when the state is compatible.
+		 */
+		crtc_state->mode_changed = true;
 	}
+
+	/* ignore any reset values/BIOS leftovers in the WM registers */
+	to_intel_atomic_state(state)->skip_intermediate_wm = true;
+
+	ret = drm_atomic_commit(state);
+
+	WARN_ON(ret == -EDEADLK);
+	return ret;
+}
+
+static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
+{
+	return intel_has_gpu_reset(dev_priv) &&
+		INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv);
 }
 
 void intel_prepare_reset(struct drm_i915_private *dev_priv)
 {
-	/* no reset support for gen2 */
-	if (IS_GEN2(dev_priv))
-		return;
+	struct drm_device *dev = &dev_priv->drm;
+	struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
+	struct drm_atomic_state *state;
+	int ret;
 
-	/* reset doesn't touch the display */
-	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
+	/*
+	 * Need mode_config.mutex so that we don't
+	 * trample ongoing ->detect() and whatnot.
+	 */
+	mutex_lock(&dev->mode_config.mutex);
+	drm_modeset_acquire_init(ctx, 0);
+	while (1) {
+		ret = drm_modeset_lock_all_ctx(dev, ctx);
+		if (ret != -EDEADLK)
+			break;
+
+		drm_modeset_backoff(ctx);
+	}
+
+	/* reset doesn't touch the display, but flips might get nuked anyway, */
+	if (!i915.force_reset_modeset_test &&
+	    !gpu_reset_clobbers_display(dev_priv))
 		return;
 
-	drm_modeset_lock_all(&dev_priv->drm);
 	/*
 	 * Disabling the crtcs gracefully seems nicer. Also the
 	 * g33 docs say we should at least disable all the planes.
 	 */
-	intel_display_suspend(&dev_priv->drm);
+	state = drm_atomic_helper_duplicate_state(dev, ctx);
+	if (IS_ERR(state)) {
+		ret = PTR_ERR(state);
+		state = NULL;
+		DRM_ERROR("Duplicating state failed with %i\n", ret);
+		goto err;
+	}
+
+	ret = drm_atomic_helper_disable_all(dev, ctx);
+	if (ret) {
+		DRM_ERROR("Suspending crtc's failed with %i\n", ret);
+		goto err;
+	}
+
+	dev_priv->modeset_restore_state = state;
+	state->acquire_ctx = ctx;
+	return;
+
+err:
+	drm_atomic_state_free(state);
 }
 
 void intel_finish_reset(struct drm_i915_private *dev_priv)
 {
+	struct drm_device *dev = &dev_priv->drm;
+	struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
+	struct drm_atomic_state *state = dev_priv->modeset_restore_state;
+	int ret;
+
 	/*
 	 * Flips in the rings will be nuked by the reset,
 	 * so complete all pending flips so that user space
@@ -3134,55 +3601,76 @@ void intel_finish_reset(struct drm_i915_private *dev_priv)
 	 */
 	intel_complete_page_flips(dev_priv);
 
-	/* no reset support for gen2 */
-	if (IS_GEN2(dev_priv))
-		return;
+	dev_priv->modeset_restore_state = NULL;
+
+	dev_priv->modeset_restore_state = NULL;
 
 	/* reset doesn't touch the display */
-	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) {
+	if (!gpu_reset_clobbers_display(dev_priv)) {
+		if (!state) {
+			/*
+			 * Flips in the rings have been nuked by the reset,
+			 * so update the base address of all primary
+			 * planes to the the last fb to make sure we're
+			 * showing the correct fb after a reset.
+			 *
+			 * FIXME: Atomic will make this obsolete since we won't schedule
+			 * CS-based flips (which might get lost in gpu resets) any more.
+			 */
+			intel_update_primary_planes(dev);
+		} else {
+			ret = __intel_display_resume(dev, state);
+			if (ret)
+				DRM_ERROR("Restoring old state failed with %i\n", ret);
+		}
+	} else {
 		/*
-		 * Flips in the rings have been nuked by the reset,
-		 * so update the base address of all primary
-		 * planes to the the last fb to make sure we're
-		 * showing the correct fb after a reset.
-		 *
-		 * FIXME: Atomic will make this obsolete since we won't schedule
-		 * CS-based flips (which might get lost in gpu resets) any more.
+		 * The display has been reset as well,
+		 * so need a full re-initialization.
 		 */
-		intel_update_primary_planes(&dev_priv->drm);
-		return;
-	}
+		intel_runtime_pm_disable_interrupts(dev_priv);
+		intel_runtime_pm_enable_interrupts(dev_priv);
 
-	/*
-	 * The display has been reset as well,
-	 * so need a full re-initialization.
-	 */
-	intel_runtime_pm_disable_interrupts(dev_priv);
-	intel_runtime_pm_enable_interrupts(dev_priv);
+		intel_pps_unlock_regs_wa(dev_priv);
+		intel_modeset_init_hw(dev);
+
+		spin_lock_irq(&dev_priv->irq_lock);
+		if (dev_priv->display.hpd_irq_setup)
+			dev_priv->display.hpd_irq_setup(dev_priv);
+		spin_unlock_irq(&dev_priv->irq_lock);
+
+		ret = __intel_display_resume(dev, state);
+		if (ret)
+			DRM_ERROR("Restoring old state failed with %i\n", ret);
+
+		intel_hpd_init(dev_priv);
+	}
 
-	intel_modeset_init_hw(&dev_priv->drm);
+	drm_modeset_drop_locks(ctx);
+	drm_modeset_acquire_fini(ctx);
+	mutex_unlock(&dev->mode_config.mutex);
+}
 
-	spin_lock_irq(&dev_priv->irq_lock);
-	if (dev_priv->display.hpd_irq_setup)
-		dev_priv->display.hpd_irq_setup(dev_priv);
-	spin_unlock_irq(&dev_priv->irq_lock);
+static bool abort_flip_on_reset(struct intel_crtc *crtc)
+{
+	struct i915_gpu_error *error = &to_i915(crtc->base.dev)->gpu_error;
 
-	intel_display_resume(&dev_priv->drm);
+	if (i915_reset_in_progress(error))
+		return true;
 
-	intel_hpd_init(dev_priv);
+	if (crtc->reset_count != i915_reset_count(error))
+		return true;
 
-	drm_modeset_unlock_all(&dev_priv->drm);
+	return false;
 }
 
 static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	unsigned reset_counter;
 	bool pending;
 
-	reset_counter = i915_reset_counter(&to_i915(dev)->gpu_error);
-	if (intel_crtc->reset_counter != reset_counter)
+	if (abort_flip_on_reset(intel_crtc))
 		return false;
 
 	spin_lock_irq(&dev->event_lock);
@@ -3825,7 +4313,7 @@ static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
 	return 0;
 }
 
-static void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
+void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
 {
 	u32 temp;
 
@@ -4248,7 +4736,7 @@ int skl_update_scaler_crtc(struct intel_crtc_state *state)
 		      intel_crtc->pipe, SKL_CRTC_INDEX);
 
 	return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
-		&state->scaler_state.scaler_id, BIT(DRM_ROTATE_0),
+		&state->scaler_state.scaler_id, DRM_ROTATE_0,
 		state->pipe_src_w, state->pipe_src_h,
 		adjusted_mode->crtc_hdisplay, adjusted_mode->crtc_vdisplay);
 }
@@ -4273,7 +4761,7 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 	struct drm_framebuffer *fb = plane_state->base.fb;
 	int ret;
 
-	bool force_detach = !fb || !plane_state->visible;
+	bool force_detach = !fb || !plane_state->base.visible;
 
 	DRM_DEBUG_KMS("Updating scaler for [PLANE:%d:%s] scaler_user index %u.%u\n",
 		      intel_plane->base.base.id, intel_plane->base.name,
@@ -4283,10 +4771,10 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 				drm_plane_index(&intel_plane->base),
 				&plane_state->scaler_id,
 				plane_state->base.rotation,
-				drm_rect_width(&plane_state->src) >> 16,
-				drm_rect_height(&plane_state->src) >> 16,
-				drm_rect_width(&plane_state->dst),
-				drm_rect_height(&plane_state->dst));
+				drm_rect_width(&plane_state->base.src) >> 16,
+				drm_rect_height(&plane_state->base.src) >> 16,
+				drm_rect_width(&plane_state->base.dst),
+				drm_rect_height(&plane_state->base.dst));
 
 	if (ret || plane_state->scaler_id < 0)
 		return ret;
@@ -4564,12 +5052,11 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
 	struct drm_atomic_state *old_state = old_crtc_state->base.state;
 	struct intel_crtc_state *pipe_config =
 		to_intel_crtc_state(crtc->base.state);
-	struct drm_device *dev = crtc->base.dev;
 	struct drm_plane *primary = crtc->base.primary;
 	struct drm_plane_state *old_pri_state =
 		drm_atomic_get_existing_plane_state(old_state, primary);
 
-	intel_frontbuffer_flip(dev, pipe_config->fb_bits);
+	intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
 
 	crtc->wm.cxsr_allowed = true;
 
@@ -4584,9 +5071,9 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
 
 		intel_fbc_post_update(crtc);
 
-		if (primary_state->visible &&
+		if (primary_state->base.visible &&
 		    (needs_modeset(&pipe_config->base) ||
-		     !old_primary_state->visible))
+		     !old_primary_state->base.visible))
 			intel_post_enable_primary(&crtc->base);
 	}
 }
@@ -4612,8 +5099,8 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
 
 		intel_fbc_pre_update(crtc, pipe_config, primary_state);
 
-		if (old_primary_state->visible &&
-		    (modeset || !primary_state->visible))
+		if (old_primary_state->base.visible &&
+		    (modeset || !primary_state->base.visible))
 			intel_pre_disable_primary(&crtc->base);
 	}
 
@@ -4692,18 +5179,140 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask
 	 * to compute the mask of flip planes precisely. For the time being
 	 * consider this a flip to a NULL plane.
 	 */
-	intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe));
+	intel_frontbuffer_flip(to_i915(dev), INTEL_FRONTBUFFER_ALL_MASK(pipe));
+}
+
+static void intel_encoders_pre_pll_enable(struct drm_crtc *crtc,
+					  struct intel_crtc_state *crtc_state,
+					  struct drm_atomic_state *old_state)
+{
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector *conn;
+	int i;
+
+	for_each_connector_in_state(old_state, conn, old_conn_state, i) {
+		struct drm_connector_state *conn_state = conn->state;
+		struct intel_encoder *encoder =
+			to_intel_encoder(conn_state->best_encoder);
+
+		if (conn_state->crtc != crtc)
+			continue;
+
+		if (encoder->pre_pll_enable)
+			encoder->pre_pll_enable(encoder, crtc_state, conn_state);
+	}
+}
+
+static void intel_encoders_pre_enable(struct drm_crtc *crtc,
+				      struct intel_crtc_state *crtc_state,
+				      struct drm_atomic_state *old_state)
+{
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector *conn;
+	int i;
+
+	for_each_connector_in_state(old_state, conn, old_conn_state, i) {
+		struct drm_connector_state *conn_state = conn->state;
+		struct intel_encoder *encoder =
+			to_intel_encoder(conn_state->best_encoder);
+
+		if (conn_state->crtc != crtc)
+			continue;
+
+		if (encoder->pre_enable)
+			encoder->pre_enable(encoder, crtc_state, conn_state);
+	}
 }
 
-static void ironlake_crtc_enable(struct drm_crtc *crtc)
+static void intel_encoders_enable(struct drm_crtc *crtc,
+				  struct intel_crtc_state *crtc_state,
+				  struct drm_atomic_state *old_state)
 {
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector *conn;
+	int i;
+
+	for_each_connector_in_state(old_state, conn, old_conn_state, i) {
+		struct drm_connector_state *conn_state = conn->state;
+		struct intel_encoder *encoder =
+			to_intel_encoder(conn_state->best_encoder);
+
+		if (conn_state->crtc != crtc)
+			continue;
+
+		encoder->enable(encoder, crtc_state, conn_state);
+		intel_opregion_notify_encoder(encoder, true);
+	}
+}
+
+static void intel_encoders_disable(struct drm_crtc *crtc,
+				   struct intel_crtc_state *old_crtc_state,
+				   struct drm_atomic_state *old_state)
+{
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector *conn;
+	int i;
+
+	for_each_connector_in_state(old_state, conn, old_conn_state, i) {
+		struct intel_encoder *encoder =
+			to_intel_encoder(old_conn_state->best_encoder);
+
+		if (old_conn_state->crtc != crtc)
+			continue;
+
+		intel_opregion_notify_encoder(encoder, false);
+		encoder->disable(encoder, old_crtc_state, old_conn_state);
+	}
+}
+
+static void intel_encoders_post_disable(struct drm_crtc *crtc,
+					struct intel_crtc_state *old_crtc_state,
+					struct drm_atomic_state *old_state)
+{
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector *conn;
+	int i;
+
+	for_each_connector_in_state(old_state, conn, old_conn_state, i) {
+		struct intel_encoder *encoder =
+			to_intel_encoder(old_conn_state->best_encoder);
+
+		if (old_conn_state->crtc != crtc)
+			continue;
+
+		if (encoder->post_disable)
+			encoder->post_disable(encoder, old_crtc_state, old_conn_state);
+	}
+}
+
+static void intel_encoders_post_pll_disable(struct drm_crtc *crtc,
+					    struct intel_crtc_state *old_crtc_state,
+					    struct drm_atomic_state *old_state)
+{
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector *conn;
+	int i;
+
+	for_each_connector_in_state(old_state, conn, old_conn_state, i) {
+		struct intel_encoder *encoder =
+			to_intel_encoder(old_conn_state->best_encoder);
+
+		if (old_conn_state->crtc != crtc)
+			continue;
+
+		if (encoder->post_pll_disable)
+			encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
+	}
+}
+
+static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
+				 struct drm_atomic_state *old_state)
+{
+	struct drm_crtc *crtc = pipe_config->base.crtc;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe;
-	struct intel_crtc_state *pipe_config =
-		to_intel_crtc_state(crtc->state);
 
 	if (WARN_ON(intel_crtc->active))
 		return;
@@ -4741,9 +5350,7 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 
 	intel_crtc->active = true;
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->pre_enable)
-			encoder->pre_enable(encoder);
+	intel_encoders_pre_enable(crtc, pipe_config, old_state);
 
 	if (intel_crtc->config->has_pch_encoder) {
 		/* Note: FDI PLL enabling _must_ be done before we enable the
@@ -4773,8 +5380,7 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 	assert_vblank_disabled(crtc);
 	drm_crtc_vblank_on(crtc);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		encoder->enable(encoder);
+	intel_encoders_enable(crtc, pipe_config, old_state);
 
 	if (HAS_PCH_CPT(dev))
 		cpt_verify_modeset(dev, intel_crtc->pipe);
@@ -4792,16 +5398,15 @@ static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
 	return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A;
 }
 
-static void haswell_crtc_enable(struct drm_crtc *crtc)
+static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
+				struct drm_atomic_state *old_state)
 {
+	struct drm_crtc *crtc = pipe_config->base.crtc;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe, hsw_workaround_pipe;
 	enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
-	struct intel_crtc_state *pipe_config =
-		to_intel_crtc_state(crtc->state);
 
 	if (WARN_ON(intel_crtc->active))
 		return;
@@ -4810,9 +5415,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
 						      false);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->pre_pll_enable)
-			encoder->pre_pll_enable(encoder);
+	intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
 
 	if (intel_crtc->config->shared_dpll)
 		intel_enable_shared_dpll(intel_crtc);
@@ -4850,10 +5453,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 	else
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder) {
-		if (encoder->pre_enable)
-			encoder->pre_enable(encoder);
-	}
+	intel_encoders_pre_enable(crtc, pipe_config, old_state);
 
 	if (intel_crtc->config->has_pch_encoder)
 		dev_priv->display.fdi_link_train(crtc);
@@ -4894,10 +5494,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 	assert_vblank_disabled(crtc);
 	drm_crtc_vblank_on(crtc);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder) {
-		encoder->enable(encoder);
-		intel_opregion_notify_encoder(encoder, true);
-	}
+	intel_encoders_enable(crtc, pipe_config, old_state);
 
 	if (intel_crtc->config->has_pch_encoder) {
 		intel_wait_for_vblank(dev, pipe);
@@ -4931,12 +5528,13 @@ static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
 	}
 }
 
-static void ironlake_crtc_disable(struct drm_crtc *crtc)
+static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
+				  struct drm_atomic_state *old_state)
 {
+	struct drm_crtc *crtc = old_crtc_state->base.crtc;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe;
 
 	/*
@@ -4949,8 +5547,7 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 		intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
 	}
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		encoder->disable(encoder);
+	intel_encoders_disable(crtc, old_crtc_state, old_state);
 
 	drm_crtc_vblank_off(crtc);
 	assert_vblank_disabled(crtc);
@@ -4962,9 +5559,7 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	if (intel_crtc->config->has_pch_encoder)
 		ironlake_fdi_disable(crtc);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->post_disable)
-			encoder->post_disable(encoder);
+	intel_encoders_post_disable(crtc, old_crtc_state, old_state);
 
 	if (intel_crtc->config->has_pch_encoder) {
 		ironlake_disable_pch_transcoder(dev_priv, pipe);
@@ -4994,22 +5589,20 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
 }
 
-static void haswell_crtc_disable(struct drm_crtc *crtc)
+static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
+				 struct drm_atomic_state *old_state)
 {
+	struct drm_crtc *crtc = old_crtc_state->base.crtc;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_encoder *encoder;
 	enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
 
 	if (intel_crtc->config->has_pch_encoder)
 		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
 						      false);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder) {
-		intel_opregion_notify_encoder(encoder, false);
-		encoder->disable(encoder);
-	}
+	intel_encoders_disable(crtc, old_crtc_state, old_state);
 
 	drm_crtc_vblank_off(crtc);
 	assert_vblank_disabled(crtc);
@@ -5032,18 +5625,11 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 	if (!transcoder_is_dsi(cpu_transcoder))
 		intel_ddi_disable_pipe_clock(intel_crtc);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->post_disable)
-			encoder->post_disable(encoder);
-
-	if (intel_crtc->config->has_pch_encoder) {
-		lpt_disable_pch_transcoder(dev_priv);
-		lpt_disable_iclkip(dev_priv);
-		intel_ddi_fdi_disable(crtc);
+	intel_encoders_post_disable(crtc, old_crtc_state, old_state);
 
+	if (old_crtc_state->has_pch_encoder)
 		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
 						      true);
-	}
 }
 
 static void i9xx_pfit_enable(struct intel_crtc *crtc)
@@ -5691,15 +6277,7 @@ static bool skl_cdclk_pcu_ready(struct drm_i915_private *dev_priv)
 
 static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv)
 {
-	unsigned int i;
-
-	for (i = 0; i < 15; i++) {
-		if (skl_cdclk_pcu_ready(dev_priv))
-			return true;
-		udelay(10);
-	}
-
-	return false;
+	return _wait_for(skl_cdclk_pcu_ready(dev_priv), 3000, 10) == 0;
 }
 
 static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco)
@@ -6107,14 +6685,13 @@ static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state)
 	intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A);
 }
 
-static void valleyview_crtc_enable(struct drm_crtc *crtc)
+static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
+				   struct drm_atomic_state *old_state)
 {
+	struct drm_crtc *crtc = pipe_config->base.crtc;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_encoder *encoder;
-	struct intel_crtc_state *pipe_config =
-		to_intel_crtc_state(crtc->state);
 	int pipe = intel_crtc->pipe;
 
 	if (WARN_ON(intel_crtc->active))
@@ -6139,9 +6716,7 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
 
 	intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->pre_pll_enable)
-			encoder->pre_pll_enable(encoder);
+	intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
 
 	if (IS_CHERRYVIEW(dev)) {
 		chv_prepare_pll(intel_crtc, intel_crtc->config);
@@ -6151,9 +6726,7 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
 		vlv_enable_pll(intel_crtc, intel_crtc->config);
 	}
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->pre_enable)
-			encoder->pre_enable(encoder);
+	intel_encoders_pre_enable(crtc, pipe_config, old_state);
 
 	i9xx_pfit_enable(intel_crtc);
 
@@ -6165,8 +6738,7 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
 	assert_vblank_disabled(crtc);
 	drm_crtc_vblank_on(crtc);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		encoder->enable(encoder);
+	intel_encoders_enable(crtc, pipe_config, old_state);
 }
 
 static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
@@ -6178,14 +6750,13 @@ static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
 	I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1);
 }
 
-static void i9xx_crtc_enable(struct drm_crtc *crtc)
+static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
+			     struct drm_atomic_state *old_state)
 {
+	struct drm_crtc *crtc = pipe_config->base.crtc;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_encoder *encoder;
-	struct intel_crtc_state *pipe_config =
-		to_intel_crtc_state(crtc->state);
 	enum pipe pipe = intel_crtc->pipe;
 
 	if (WARN_ON(intel_crtc->active))
@@ -6206,9 +6777,7 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 	if (!IS_GEN2(dev))
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->pre_enable)
-			encoder->pre_enable(encoder);
+	intel_encoders_pre_enable(crtc, pipe_config, old_state);
 
 	i9xx_enable_pll(intel_crtc);
 
@@ -6222,8 +6791,7 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 	assert_vblank_disabled(crtc);
 	drm_crtc_vblank_on(crtc);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		encoder->enable(encoder);
+	intel_encoders_enable(crtc, pipe_config, old_state);
 }
 
 static void i9xx_pfit_disable(struct intel_crtc *crtc)
@@ -6241,12 +6809,13 @@ static void i9xx_pfit_disable(struct intel_crtc *crtc)
 	I915_WRITE(PFIT_CONTROL, 0);
 }
 
-static void i9xx_crtc_disable(struct drm_crtc *crtc)
+static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
+			      struct drm_atomic_state *old_state)
 {
+	struct drm_crtc *crtc = old_crtc_state->base.crtc;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe;
 
 	/*
@@ -6256,8 +6825,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 	if (IS_GEN2(dev))
 		intel_wait_for_vblank(dev, pipe);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		encoder->disable(encoder);
+	intel_encoders_disable(crtc, old_crtc_state, old_state);
 
 	drm_crtc_vblank_off(crtc);
 	assert_vblank_disabled(crtc);
@@ -6266,9 +6834,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 
 	i9xx_pfit_disable(intel_crtc);
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->post_disable)
-			encoder->post_disable(encoder);
+	intel_encoders_post_disable(crtc, old_crtc_state, old_state);
 
 	if (!intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI)) {
 		if (IS_CHERRYVIEW(dev))
@@ -6279,9 +6845,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 			i9xx_disable_pll(intel_crtc);
 	}
 
-	for_each_encoder_on_crtc(dev, crtc, encoder)
-		if (encoder->post_pll_disable)
-			encoder->post_pll_disable(encoder);
+	intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state);
 
 	if (!IS_GEN2(dev))
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
@@ -6294,20 +6858,34 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	enum intel_display_power_domain domain;
 	unsigned long domains;
+	struct drm_atomic_state *state;
+	struct intel_crtc_state *crtc_state;
+	int ret;
 
 	if (!intel_crtc->active)
 		return;
 
-	if (to_intel_plane_state(crtc->primary->state)->visible) {
+	if (to_intel_plane_state(crtc->primary->state)->base.visible) {
 		WARN_ON(intel_crtc->flip_work);
 
 		intel_pre_disable_primary_noatomic(crtc);
 
 		intel_crtc_disable_planes(crtc, 1 << drm_plane_index(crtc->primary));
-		to_intel_plane_state(crtc->primary->state)->visible = false;
+		to_intel_plane_state(crtc->primary->state)->base.visible = false;
 	}
 
-	dev_priv->display.crtc_disable(crtc);
+	state = drm_atomic_state_alloc(crtc->dev);
+	state->acquire_ctx = crtc->dev->mode_config.acquire_ctx;
+
+	/* Everything's already locked, -EDEADLK can't happen. */
+	crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
+	ret = drm_atomic_add_affected_connectors(state, crtc);
+
+	WARN_ON(IS_ERR(crtc_state) || ret);
+
+	dev_priv->display.crtc_disable(crtc_state, state);
+
+	drm_atomic_state_free(state);
 
 	DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
 		      crtc->base.id, crtc->name);
@@ -6822,9 +7400,10 @@ static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
 
 static int pnv_get_display_clock_speed(struct drm_device *dev)
 {
+	struct pci_dev *pdev = dev->pdev;
 	u16 gcfgc = 0;
 
-	pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
+	pci_read_config_word(pdev, GCFGC, &gcfgc);
 
 	switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 	case GC_DISPLAY_CLOCK_267_MHZ_PNV:
@@ -6846,9 +7425,10 @@ static int pnv_get_display_clock_speed(struct drm_device *dev)
 
 static int i915gm_get_display_clock_speed(struct drm_device *dev)
 {
+	struct pci_dev *pdev = dev->pdev;
 	u16 gcfgc = 0;
 
-	pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
+	pci_read_config_word(pdev, GCFGC, &gcfgc);
 
 	if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
 		return 133333;
@@ -6870,6 +7450,7 @@ static int i865_get_display_clock_speed(struct drm_device *dev)
 
 static int i85x_get_display_clock_speed(struct drm_device *dev)
 {
+	struct pci_dev *pdev = dev->pdev;
 	u16 hpllcc = 0;
 
 	/*
@@ -6877,10 +7458,10 @@ static int i85x_get_display_clock_speed(struct drm_device *dev)
 	 * encoding is different :(
 	 * FIXME is this the right way to detect 852GM/852GMV?
 	 */
-	if (dev->pdev->revision == 0x1)
+	if (pdev->revision == 0x1)
 		return 133333;
 
-	pci_bus_read_config_word(dev->pdev->bus,
+	pci_bus_read_config_word(pdev->bus,
 				 PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 
 	/* Assume that the hardware is in the high speed state.  This
@@ -6981,10 +7562,11 @@ static unsigned int intel_hpll_vco(struct drm_device *dev)
 
 static int gm45_get_display_clock_speed(struct drm_device *dev)
 {
+	struct pci_dev *pdev = dev->pdev;
 	unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
 	uint16_t tmp = 0;
 
-	pci_read_config_word(dev->pdev, GCFGC, &tmp);
+	pci_read_config_word(pdev, GCFGC, &tmp);
 
 	cdclk_sel = (tmp >> 12) & 0x1;
 
@@ -7003,6 +7585,7 @@ static int gm45_get_display_clock_speed(struct drm_device *dev)
 
 static int i965gm_get_display_clock_speed(struct drm_device *dev)
 {
+	struct pci_dev *pdev = dev->pdev;
 	static const uint8_t div_3200[] = { 16, 10,  8 };
 	static const uint8_t div_4000[] = { 20, 12, 10 };
 	static const uint8_t div_5333[] = { 24, 16, 14 };
@@ -7010,7 +7593,7 @@ static int i965gm_get_display_clock_speed(struct drm_device *dev)
 	unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
 	uint16_t tmp = 0;
 
-	pci_read_config_word(dev->pdev, GCFGC, &tmp);
+	pci_read_config_word(pdev, GCFGC, &tmp);
 
 	cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 
@@ -7040,6 +7623,7 @@ fail:
 
 static int g33_get_display_clock_speed(struct drm_device *dev)
 {
+	struct pci_dev *pdev = dev->pdev;
 	static const uint8_t div_3200[] = { 12, 10,  8,  7, 5, 16 };
 	static const uint8_t div_4000[] = { 14, 12, 10,  8, 6, 20 };
 	static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 };
@@ -7048,7 +7632,7 @@ static int g33_get_display_clock_speed(struct drm_device *dev)
 	unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
 	uint16_t tmp = 0;
 
-	pci_read_config_word(dev->pdev, GCFGC, &tmp);
+	pci_read_config_word(pdev, GCFGC, &tmp);
 
 	cdclk_sel = (tmp >> 4) & 0x7;
 
@@ -8928,6 +9512,24 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
 	if (intel_crtc_has_dp_encoder(crtc_state))
 		dpll |= DPLL_SDVO_HIGH_SPEED;
 
+	/*
+	 * The high speed IO clock is only really required for
+	 * SDVO/HDMI/DP, but we also enable it for CRT to make it
+	 * possible to share the DPLL between CRT and HDMI. Enabling
+	 * the clock needlessly does no real harm, except use up a
+	 * bit of power potentially.
+	 *
+	 * We'll limit this to IVB with 3 pipes, since it has only two
+	 * DPLLs and so DPLL sharing is the only way to get three pipes
+	 * driving PCH ports at the same time. On SNB we could do this,
+	 * and potentially avoid enabling the second DPLL, but it's not
+	 * clear if it''s a win or loss power wise. No point in doing
+	 * this on ILK at all since it has a fixed DPLL<->pipe mapping.
+	 */
+	if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
+	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
+		dpll |= DPLL_SDVO_HIGH_SPEED;
+
 	/* compute bitmask from p1 value */
 	dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
 	/* also FPA1 */
@@ -9214,7 +9816,7 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
 	return;
 
 error:
-	kfree(fb);
+	kfree(intel_fb);
 }
 
 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
@@ -9420,7 +10022,7 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
 	I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
 	I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
 	I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
-	I915_STATE_WARN(I915_READ(PCH_PP_STATUS) & PP_ON, "Panel power on\n");
+	I915_STATE_WARN(I915_READ(PP_STATUS(0)) & PP_ON, "Panel power on\n");
 	I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
 	     "CPU PWM1 enabled\n");
 	if (IS_HASWELL(dev))
@@ -9459,7 +10061,7 @@ static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
 		mutex_lock(&dev_priv->rps.hw_lock);
 		if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
 					    val))
-			DRM_ERROR("Failed to write to D_COMP\n");
+			DRM_DEBUG_KMS("Failed to write to D_COMP\n");
 		mutex_unlock(&dev_priv->rps.hw_lock);
 	} else {
 		I915_WRITE(D_COMP_BDW, val);
@@ -9867,15 +10469,12 @@ static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
 
 	switch (port) {
 	case PORT_A:
-		pipe_config->ddi_pll_sel = SKL_DPLL0;
 		id = DPLL_ID_SKL_DPLL0;
 		break;
 	case PORT_B:
-		pipe_config->ddi_pll_sel = SKL_DPLL1;
 		id = DPLL_ID_SKL_DPLL1;
 		break;
 	case PORT_C:
-		pipe_config->ddi_pll_sel = SKL_DPLL2;
 		id = DPLL_ID_SKL_DPLL2;
 		break;
 	default:
@@ -9894,25 +10493,10 @@ static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
 	u32 temp;
 
 	temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
-	pipe_config->ddi_pll_sel = temp >> (port * 3 + 1);
+	id = temp >> (port * 3 + 1);
 
-	switch (pipe_config->ddi_pll_sel) {
-	case SKL_DPLL0:
-		id = DPLL_ID_SKL_DPLL0;
-		break;
-	case SKL_DPLL1:
-		id = DPLL_ID_SKL_DPLL1;
-		break;
-	case SKL_DPLL2:
-		id = DPLL_ID_SKL_DPLL2;
-		break;
-	case SKL_DPLL3:
-		id = DPLL_ID_SKL_DPLL3;
-		break;
-	default:
-		MISSING_CASE(pipe_config->ddi_pll_sel);
+	if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
 		return;
-	}
 
 	pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
 }
@@ -9922,10 +10506,9 @@ static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
 				struct intel_crtc_state *pipe_config)
 {
 	enum intel_dpll_id id;
+	uint32_t ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
 
-	pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
-
-	switch (pipe_config->ddi_pll_sel) {
+	switch (ddi_pll_sel) {
 	case PORT_CLK_SEL_WRPLL1:
 		id = DPLL_ID_WRPLL1;
 		break;
@@ -9945,7 +10528,7 @@ static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
 		id = DPLL_ID_LCPLL_2700;
 		break;
 	default:
-		MISSING_CASE(pipe_config->ddi_pll_sel);
+		MISSING_CASE(ddi_pll_sel);
 		/* fall through */
 	case PORT_CLK_SEL_NONE:
 		return;
@@ -10178,7 +10761,7 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base,
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	uint32_t cntl = 0, size = 0;
 
-	if (plane_state && plane_state->visible) {
+	if (plane_state && plane_state->base.visible) {
 		unsigned int width = plane_state->base.crtc_w;
 		unsigned int height = plane_state->base.crtc_h;
 		unsigned int stride = roundup_pow_of_two(width) * 4;
@@ -10239,10 +10822,14 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
 	int pipe = intel_crtc->pipe;
 	uint32_t cntl = 0;
 
-	if (plane_state && plane_state->visible) {
+	if (INTEL_GEN(dev_priv) >= 9 && wm->dirty_pipes & drm_crtc_mask(crtc))
+		skl_write_cursor_wm(intel_crtc, wm);
+
+	if (plane_state && plane_state->base.visible) {
 		cntl = MCURSOR_GAMMA_ENABLE;
 		switch (plane_state->base.crtc_w) {
 			case 64:
@@ -10263,7 +10850,7 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
 		if (HAS_DDI(dev))
 			cntl |= CURSOR_PIPE_CSC_ENABLE;
 
-		if (plane_state->base.rotation == BIT(DRM_ROTATE_180))
+		if (plane_state->base.rotation == DRM_ROTATE_180)
 			cntl |= CURSOR_ROTATE_180;
 	}
 
@@ -10309,7 +10896,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
 
 		/* ILK+ do this automagically */
 		if (HAS_GMCH_DISPLAY(dev) &&
-		    plane_state->base.rotation == BIT(DRM_ROTATE_180)) {
+		    plane_state->base.rotation == DRM_ROTATE_180) {
 			base += (plane_state->base.crtc_h *
 				 plane_state->base.crtc_w - 1) * 4;
 		}
@@ -10442,7 +11029,7 @@ intel_framebuffer_create_for_mode(struct drm_device *dev,
 
 	fb = intel_framebuffer_create(dev, &mode_cmd, obj);
 	if (IS_ERR(fb))
-		drm_gem_object_unreference_unlocked(&obj->base);
+		i915_gem_object_put_unlocked(obj);
 
 	return fb;
 }
@@ -10953,13 +11540,13 @@ static void intel_unpin_work_fn(struct work_struct *__work)
 
 	mutex_lock(&dev->struct_mutex);
 	intel_unpin_fb_obj(work->old_fb, primary->state->rotation);
-	drm_gem_object_unreference(&work->pending_flip_obj->base);
-
-	if (work->flip_queued_req)
-		i915_gem_request_assign(&work->flip_queued_req, NULL);
+	i915_gem_object_put(work->pending_flip_obj);
 	mutex_unlock(&dev->struct_mutex);
 
-	intel_frontbuffer_flip_complete(dev, to_intel_plane(primary)->frontbuffer_bit);
+	i915_gem_request_put(work->flip_queued_req);
+
+	intel_frontbuffer_flip_complete(to_i915(dev),
+					to_intel_plane(primary)->frontbuffer_bit);
 	intel_fbc_post_update(crtc);
 	drm_framebuffer_unreference(work->old_fb);
 
@@ -10980,10 +11567,8 @@ static bool __pageflip_finished_cs(struct intel_crtc *crtc,
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	unsigned reset_counter;
 
-	reset_counter = i915_reset_counter(&dev_priv->gpu_error);
-	if (crtc->reset_counter != reset_counter)
+	if (abort_flip_on_reset(crtc))
 		return true;
 
 	/*
@@ -11124,7 +11709,7 @@ static int intel_gen2_queue_flip(struct drm_device *dev,
 				 struct drm_i915_gem_request *req,
 				 uint32_t flags)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	u32 flip_mask;
 	int ret;
@@ -11140,13 +11725,13 @@ static int intel_gen2_queue_flip(struct drm_device *dev,
 		flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
 	else
 		flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
-	intel_ring_emit(engine, MI_WAIT_FOR_EVENT | flip_mask);
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_emit(engine, MI_DISPLAY_FLIP |
+	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_emit(ring, MI_DISPLAY_FLIP |
 			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-	intel_ring_emit(engine, fb->pitches[0]);
-	intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
-	intel_ring_emit(engine, 0); /* aux display base address, unused */
+	intel_ring_emit(ring, fb->pitches[0]);
+	intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset);
+	intel_ring_emit(ring, 0); /* aux display base address, unused */
 
 	return 0;
 }
@@ -11158,7 +11743,7 @@ static int intel_gen3_queue_flip(struct drm_device *dev,
 				 struct drm_i915_gem_request *req,
 				 uint32_t flags)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	u32 flip_mask;
 	int ret;
@@ -11171,13 +11756,13 @@ static int intel_gen3_queue_flip(struct drm_device *dev,
 		flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
 	else
 		flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
-	intel_ring_emit(engine, MI_WAIT_FOR_EVENT | flip_mask);
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_emit(engine, MI_DISPLAY_FLIP_I915 |
+	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 |
 			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-	intel_ring_emit(engine, fb->pitches[0]);
-	intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
-	intel_ring_emit(engine, MI_NOOP);
+	intel_ring_emit(ring, fb->pitches[0]);
+	intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset);
+	intel_ring_emit(ring, MI_NOOP);
 
 	return 0;
 }
@@ -11189,7 +11774,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
 				 struct drm_i915_gem_request *req,
 				 uint32_t flags)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	uint32_t pf, pipesrc;
@@ -11203,11 +11788,11 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
 	 * Display Registers (which do not change across a page-flip)
 	 * so we need only reprogram the base address.
 	 */
-	intel_ring_emit(engine, MI_DISPLAY_FLIP |
+	intel_ring_emit(ring, MI_DISPLAY_FLIP |
 			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-	intel_ring_emit(engine, fb->pitches[0]);
-	intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset |
-			obj->tiling_mode);
+	intel_ring_emit(ring, fb->pitches[0]);
+	intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset |
+			intel_fb_modifier_to_tiling(fb->modifier[0]));
 
 	/* XXX Enabling the panel-fitter across page-flip is so far
 	 * untested on non-native modes, so ignore it for now.
@@ -11215,7 +11800,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
 	 */
 	pf = 0;
 	pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
-	intel_ring_emit(engine, pf | pipesrc);
+	intel_ring_emit(ring, pf | pipesrc);
 
 	return 0;
 }
@@ -11227,7 +11812,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
 				 struct drm_i915_gem_request *req,
 				 uint32_t flags)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	uint32_t pf, pipesrc;
@@ -11237,10 +11822,11 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, MI_DISPLAY_FLIP |
+	intel_ring_emit(ring, MI_DISPLAY_FLIP |
 			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-	intel_ring_emit(engine, fb->pitches[0] | obj->tiling_mode);
-	intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
+	intel_ring_emit(ring, fb->pitches[0] |
+			intel_fb_modifier_to_tiling(fb->modifier[0]));
+	intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset);
 
 	/* Contrary to the suggestions in the documentation,
 	 * "Enable Panel Fitter" does not seem to be required when page
@@ -11250,7 +11836,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
 	 */
 	pf = 0;
 	pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
-	intel_ring_emit(engine, pf | pipesrc);
+	intel_ring_emit(ring, pf | pipesrc);
 
 	return 0;
 }
@@ -11262,7 +11848,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 				 struct drm_i915_gem_request *req,
 				 uint32_t flags)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	uint32_t plane_bit = 0;
 	int len, ret;
@@ -11283,7 +11869,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 	}
 
 	len = 4;
-	if (engine->id == RCS) {
+	if (req->engine->id == RCS) {
 		len += 6;
 		/*
 		 * On Gen 8, SRM is now taking an extra dword to accommodate
@@ -11321,30 +11907,32 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 	 * for the RCS also doesn't appear to drop events. Setting the DERRMR
 	 * to zero does lead to lockups within MI_DISPLAY_FLIP.
 	 */
-	if (engine->id == RCS) {
-		intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit_reg(engine, DERRMR);
-		intel_ring_emit(engine, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
+	if (req->engine->id == RCS) {
+		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+		intel_ring_emit_reg(ring, DERRMR);
+		intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
 					  DERRMR_PIPEB_PRI_FLIP_DONE |
 					  DERRMR_PIPEC_PRI_FLIP_DONE));
 		if (IS_GEN8(dev))
-			intel_ring_emit(engine, MI_STORE_REGISTER_MEM_GEN8 |
+			intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8 |
 					      MI_SRM_LRM_GLOBAL_GTT);
 		else
-			intel_ring_emit(engine, MI_STORE_REGISTER_MEM |
+			intel_ring_emit(ring, MI_STORE_REGISTER_MEM |
 					      MI_SRM_LRM_GLOBAL_GTT);
-		intel_ring_emit_reg(engine, DERRMR);
-		intel_ring_emit(engine, engine->scratch.gtt_offset + 256);
+		intel_ring_emit_reg(ring, DERRMR);
+		intel_ring_emit(ring,
+				i915_ggtt_offset(req->engine->scratch) + 256);
 		if (IS_GEN8(dev)) {
-			intel_ring_emit(engine, 0);
-			intel_ring_emit(engine, MI_NOOP);
+			intel_ring_emit(ring, 0);
+			intel_ring_emit(ring, MI_NOOP);
 		}
 	}
 
-	intel_ring_emit(engine, MI_DISPLAY_FLIP_I915 | plane_bit);
-	intel_ring_emit(engine, (fb->pitches[0] | obj->tiling_mode));
-	intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
-	intel_ring_emit(engine, (MI_NOOP));
+	intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | plane_bit);
+	intel_ring_emit(ring, fb->pitches[0] |
+			intel_fb_modifier_to_tiling(fb->modifier[0]));
+	intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset);
+	intel_ring_emit(ring, (MI_NOOP));
 
 	return 0;
 }
@@ -11379,7 +11967,8 @@ static bool use_mmio_flip(struct intel_engine_cs *engine,
 	if (resv && !reservation_object_test_signaled_rcu(resv, false))
 		return true;
 
-	return engine != i915_gem_request_get_engine(obj->last_write_req);
+	return engine != i915_gem_active_get_engine(&obj->last_write,
+						    &obj->base.dev->struct_mutex);
 }
 
 static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
@@ -11390,7 +11979,7 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
 	const enum pipe pipe = intel_crtc->pipe;
-	u32 ctl, stride, tile_height;
+	u32 ctl, stride = skl_plane_stride(fb, 0, rotation);
 
 	ctl = I915_READ(PLANE_CTL(pipe, 0));
 	ctl &= ~PLANE_CTL_TILED_MASK;
@@ -11411,20 +12000,6 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
 	}
 
 	/*
-	 * The stride is either expressed as a multiple of 64 bytes chunks for
-	 * linear buffers or in number of tiles for tiled buffers.
-	 */
-	if (intel_rotation_90_or_270(rotation)) {
-		/* stride = Surface height in tiles */
-		tile_height = intel_tile_height(dev_priv, fb->modifier[0], 0);
-		stride = DIV_ROUND_UP(fb->height, tile_height);
-	} else {
-		stride = fb->pitches[0] /
-			intel_fb_stride_alignment(dev_priv, fb->modifier[0],
-						  fb->pixel_format);
-	}
-
-	/*
 	 * Both PLANE_CTL and PLANE_STRIDE are not updated on vblank but on
 	 * PLANE_SURF updates, the update is then guaranteed to be atomic.
 	 */
@@ -11440,15 +12015,13 @@ static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
 {
 	struct drm_device *dev = intel_crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_framebuffer *intel_fb =
-		to_intel_framebuffer(intel_crtc->base.primary->fb);
-	struct drm_i915_gem_object *obj = intel_fb->obj;
+	struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
 	i915_reg_t reg = DSPCNTR(intel_crtc->plane);
 	u32 dspcntr;
 
 	dspcntr = I915_READ(reg);
 
-	if (obj->tiling_mode != I915_TILING_NONE)
+	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
 		dspcntr |= DISPPLANE_TILED;
 	else
 		dspcntr &= ~DISPPLANE_TILED;
@@ -11471,9 +12044,8 @@ static void intel_mmio_flip_work_func(struct work_struct *w)
 	struct reservation_object *resv;
 
 	if (work->flip_queued_req)
-		WARN_ON(__i915_wait_request(work->flip_queued_req,
-					    false, NULL,
-					    &dev_priv->rps.mmioflips));
+		WARN_ON(i915_wait_request(work->flip_queued_req,
+					  0, NULL, NO_WAITBOOST));
 
 	/* For framebuffer backed by dmabuf, wait for fence */
 	resv = i915_gem_object_get_dmabuf_resv(obj);
@@ -11584,7 +12156,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 	struct intel_flip_work *work;
 	struct intel_engine_cs *engine;
 	bool mmio_flip;
-	struct drm_i915_gem_request *request = NULL;
+	struct drm_i915_gem_request *request;
+	struct i915_vma *vma;
 	int ret;
 
 	/*
@@ -11650,22 +12223,18 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
 	/* Reference the objects for the scheduled work. */
 	drm_framebuffer_reference(work->old_fb);
-	drm_gem_object_reference(&obj->base);
 
 	crtc->primary->fb = fb;
 	update_state_fb(crtc->primary);
 
-	intel_fbc_pre_update(intel_crtc, intel_crtc->config,
-			     to_intel_plane_state(primary->state));
-
-	work->pending_flip_obj = obj;
+	work->pending_flip_obj = i915_gem_object_get(obj);
 
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
 		goto cleanup;
 
-	intel_crtc->reset_counter = i915_reset_counter(&dev_priv->gpu_error);
-	if (__i915_reset_in_progress_or_wedged(intel_crtc->reset_counter)) {
+	intel_crtc->reset_count = i915_reset_count(&dev_priv->gpu_error);
+	if (i915_reset_in_progress_or_wedged(&dev_priv->gpu_error)) {
 		ret = -EIO;
 		goto cleanup;
 	}
@@ -11677,13 +12246,14 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
 	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
 		engine = &dev_priv->engine[BCS];
-		if (obj->tiling_mode != intel_fb_obj(work->old_fb)->tiling_mode)
+		if (fb->modifier[0] != old_fb->modifier[0])
 			/* vlv: DISPLAY_FLIP fails to change tiling */
 			engine = NULL;
 	} else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
 		engine = &dev_priv->engine[BCS];
 	} else if (INTEL_INFO(dev)->gen >= 7) {
-		engine = i915_gem_request_get_engine(obj->last_write_req);
+		engine = i915_gem_active_get_engine(&obj->last_write,
+						    &obj->base.dev->struct_mutex);
 		if (engine == NULL || engine->id != RCS)
 			engine = &dev_priv->engine[BCS];
 	} else {
@@ -11692,47 +12262,52 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
 	mmio_flip = use_mmio_flip(engine, obj);
 
-	/* When using CS flips, we want to emit semaphores between rings.
-	 * However, when using mmio flips we will create a task to do the
-	 * synchronisation, so all we want here is to pin the framebuffer
-	 * into the display plane and skip any waits.
-	 */
-	if (!mmio_flip) {
-		ret = i915_gem_object_sync(obj, engine, &request);
-		if (!ret && !request) {
-			request = i915_gem_request_alloc(engine, NULL);
-			ret = PTR_ERR_OR_ZERO(request);
-		}
-
-		if (ret)
-			goto cleanup_pending;
-	}
-
-	ret = intel_pin_and_fence_fb_obj(fb, primary->state->rotation);
-	if (ret)
+	vma = intel_pin_and_fence_fb_obj(fb, primary->state->rotation);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
 		goto cleanup_pending;
+	}
 
-	work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary),
-						  obj, 0);
+	work->gtt_offset = intel_fb_gtt_offset(fb, primary->state->rotation);
 	work->gtt_offset += intel_crtc->dspaddr_offset;
 	work->rotation = crtc->primary->state->rotation;
 
+	/*
+	 * There's the potential that the next frame will not be compatible with
+	 * FBC, so we want to call pre_update() before the actual page flip.
+	 * The problem is that pre_update() caches some information about the fb
+	 * object, so we want to do this only after the object is pinned. Let's
+	 * be on the safe side and do this immediately before scheduling the
+	 * flip.
+	 */
+	intel_fbc_pre_update(intel_crtc, intel_crtc->config,
+			     to_intel_plane_state(primary->state));
+
 	if (mmio_flip) {
 		INIT_WORK(&work->mmio_work, intel_mmio_flip_work_func);
 
-		i915_gem_request_assign(&work->flip_queued_req,
-					obj->last_write_req);
-
+		work->flip_queued_req = i915_gem_active_get(&obj->last_write,
+							    &obj->base.dev->struct_mutex);
 		schedule_work(&work->mmio_work);
 	} else {
-		i915_gem_request_assign(&work->flip_queued_req, request);
+		request = i915_gem_request_alloc(engine, engine->last_context);
+		if (IS_ERR(request)) {
+			ret = PTR_ERR(request);
+			goto cleanup_unpin;
+		}
+
+		ret = i915_gem_request_await_object(request, obj, false);
+		if (ret)
+			goto cleanup_request;
+
 		ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, request,
 						   page_flip_flags);
 		if (ret)
-			goto cleanup_unpin;
+			goto cleanup_request;
 
 		intel_mark_page_flip_active(intel_crtc, work);
 
+		work->flip_queued_req = i915_gem_request_get(request);
 		i915_add_request_no_flush(request);
 	}
 
@@ -11740,25 +12315,25 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 			  to_intel_plane(primary)->frontbuffer_bit);
 	mutex_unlock(&dev->struct_mutex);
 
-	intel_frontbuffer_flip_prepare(dev,
+	intel_frontbuffer_flip_prepare(to_i915(dev),
 				       to_intel_plane(primary)->frontbuffer_bit);
 
 	trace_i915_flip_request(intel_crtc->plane, obj);
 
 	return 0;
 
+cleanup_request:
+	i915_add_request_no_flush(request);
 cleanup_unpin:
 	intel_unpin_fb_obj(fb, crtc->primary->state->rotation);
 cleanup_pending:
-	if (!IS_ERR_OR_NULL(request))
-		i915_add_request_no_flush(request);
 	atomic_dec(&intel_crtc->unpin_work_count);
 	mutex_unlock(&dev->struct_mutex);
 cleanup:
 	crtc->primary->fb = old_fb;
 	update_state_fb(crtc->primary);
 
-	drm_gem_object_unreference_unlocked(&obj->base);
+	i915_gem_object_put_unlocked(obj);
 	drm_framebuffer_unreference(work->old_fb);
 
 	spin_lock_irq(&dev->event_lock);
@@ -11826,7 +12401,7 @@ static bool intel_wm_need_update(struct drm_plane *plane,
 	struct intel_plane_state *cur = to_intel_plane_state(plane->state);
 
 	/* Update watermarks on tiling or size changes. */
-	if (new->visible != cur->visible)
+	if (new->base.visible != cur->base.visible)
 		return true;
 
 	if (!cur->base.fb || !new->base.fb)
@@ -11834,10 +12409,10 @@ static bool intel_wm_need_update(struct drm_plane *plane,
 
 	if (cur->base.fb->modifier[0] != new->base.fb->modifier[0] ||
 	    cur->base.rotation != new->base.rotation ||
-	    drm_rect_width(&new->src) != drm_rect_width(&cur->src) ||
-	    drm_rect_height(&new->src) != drm_rect_height(&cur->src) ||
-	    drm_rect_width(&new->dst) != drm_rect_width(&cur->dst) ||
-	    drm_rect_height(&new->dst) != drm_rect_height(&cur->dst))
+	    drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
+	    drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
+	    drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) ||
+	    drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst))
 		return true;
 
 	return false;
@@ -11845,10 +12420,10 @@ static bool intel_wm_need_update(struct drm_plane *plane,
 
 static bool needs_scaling(struct intel_plane_state *state)
 {
-	int src_w = drm_rect_width(&state->src) >> 16;
-	int src_h = drm_rect_height(&state->src) >> 16;
-	int dst_w = drm_rect_width(&state->dst);
-	int dst_h = drm_rect_height(&state->dst);
+	int src_w = drm_rect_width(&state->base.src) >> 16;
+	int src_h = drm_rect_height(&state->base.src) >> 16;
+	int dst_w = drm_rect_width(&state->base.dst);
+	int dst_h = drm_rect_height(&state->base.dst);
 
 	return (src_w != dst_w || src_h != dst_h);
 }
@@ -11879,8 +12454,8 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 			return ret;
 	}
 
-	was_visible = old_plane_state->visible;
-	visible = to_intel_plane_state(plane_state)->visible;
+	was_visible = old_plane_state->base.visible;
+	visible = to_intel_plane_state(plane_state)->base.visible;
 
 	if (!was_crtc_enabled && WARN_ON(was_visible))
 		was_visible = false;
@@ -11896,7 +12471,7 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 	 * only combine the results from all planes in the current place?
 	 */
 	if (!is_crtc_enabled)
-		to_intel_plane_state(plane_state)->visible = visible = false;
+		to_intel_plane_state(plane_state)->base.visible = visible = false;
 
 	if (!was_visible && !visible)
 		return 0;
@@ -12100,35 +12675,25 @@ static void
 connected_sink_compute_bpp(struct intel_connector *connector,
 			   struct intel_crtc_state *pipe_config)
 {
+	const struct drm_display_info *info = &connector->base.display_info;
 	int bpp = pipe_config->pipe_bpp;
 
 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n",
-		connector->base.base.id,
-		connector->base.name);
+		      connector->base.base.id,
+		      connector->base.name);
 
 	/* Don't use an invalid EDID bpc value */
-	if (connector->base.display_info.bpc &&
-	    connector->base.display_info.bpc * 3 < bpp) {
+	if (info->bpc != 0 && info->bpc * 3 < bpp) {
 		DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n",
-			      bpp, connector->base.display_info.bpc*3);
-		pipe_config->pipe_bpp = connector->base.display_info.bpc*3;
+			      bpp, info->bpc * 3);
+		pipe_config->pipe_bpp = info->bpc * 3;
 	}
 
-	/* Clamp bpp to default limit on screens without EDID 1.4 */
-	if (connector->base.display_info.bpc == 0) {
-		int type = connector->base.connector_type;
-		int clamp_bpp = 24;
-
-		/* Fall back to 18 bpp when DP sink capability is unknown. */
-		if (type == DRM_MODE_CONNECTOR_DisplayPort ||
-		    type == DRM_MODE_CONNECTOR_eDP)
-			clamp_bpp = 18;
-
-		if (bpp > clamp_bpp) {
-			DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of %d\n",
-				      bpp, clamp_bpp);
-			pipe_config->pipe_bpp = clamp_bpp;
-		}
+	/* Clamp bpp to 8 on screens without EDID 1.4 */
+	if (info->bpc == 0 && bpp > 24) {
+		DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of 24\n",
+			      bpp);
+		pipe_config->pipe_bpp = 24;
 	}
 }
 
@@ -12244,10 +12809,9 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 	DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide);
 
 	if (IS_BROXTON(dev)) {
-		DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
+		DRM_DEBUG_KMS("dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
 			      "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, "
 			      "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n",
-			      pipe_config->ddi_pll_sel,
 			      pipe_config->dpll_hw_state.ebb0,
 			      pipe_config->dpll_hw_state.ebb4,
 			      pipe_config->dpll_hw_state.pll0,
@@ -12260,15 +12824,13 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 			      pipe_config->dpll_hw_state.pll10,
 			      pipe_config->dpll_hw_state.pcsdw12);
 	} else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
-		DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: "
+		DRM_DEBUG_KMS("dpll_hw_state: "
 			      "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
-			      pipe_config->ddi_pll_sel,
 			      pipe_config->dpll_hw_state.ctrl1,
 			      pipe_config->dpll_hw_state.cfgcr1,
 			      pipe_config->dpll_hw_state.cfgcr2);
 	} else if (HAS_DDI(dev)) {
-		DRM_DEBUG_KMS("ddi_pll_sel: 0x%x; dpll_hw_state: wrpll: 0x%x spll: 0x%x\n",
-			      pipe_config->ddi_pll_sel,
+		DRM_DEBUG_KMS("dpll_hw_state: wrpll: 0x%x spll: 0x%x\n",
 			      pipe_config->dpll_hw_state.wrpll,
 			      pipe_config->dpll_hw_state.spll);
 	} else {
@@ -12282,6 +12844,7 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 
 	DRM_DEBUG_KMS("planes on this crtc\n");
 	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+		char *format_name;
 		intel_plane = to_intel_plane(plane);
 		if (intel_plane->pipe != crtc->pipe)
 			continue;
@@ -12294,19 +12857,23 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 			continue;
 		}
 
+		format_name = drm_get_format_name(fb->pixel_format);
+
 		DRM_DEBUG_KMS("[PLANE:%d:%s] enabled",
 			      plane->base.id, plane->name);
 		DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = %s",
-			      fb->base.id, fb->width, fb->height,
-			      drm_get_format_name(fb->pixel_format));
+			      fb->base.id, fb->width, fb->height, format_name);
 		DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
 			      state->scaler_id,
-			      state->src.x1 >> 16, state->src.y1 >> 16,
-			      drm_rect_width(&state->src) >> 16,
-			      drm_rect_height(&state->src) >> 16,
-			      state->dst.x1, state->dst.y1,
-			      drm_rect_width(&state->dst),
-			      drm_rect_height(&state->dst));
+			      state->base.src.x1 >> 16,
+			      state->base.src.y1 >> 16,
+			      drm_rect_width(&state->base.src) >> 16,
+			      drm_rect_height(&state->base.src) >> 16,
+			      state->base.dst.x1, state->base.dst.y1,
+			      drm_rect_width(&state->base.dst),
+			      drm_rect_height(&state->base.dst));
+
+		kfree(format_name);
 	}
 }
 
@@ -12315,6 +12882,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 	struct drm_device *dev = state->dev;
 	struct drm_connector *connector;
 	unsigned int used_ports = 0;
+	unsigned int used_mst_ports = 0;
 
 	/*
 	 * Walk the connector list instead of the encoder
@@ -12351,11 +12919,20 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 				return false;
 
 			used_ports |= port_mask;
+			break;
+		case INTEL_OUTPUT_DP_MST:
+			used_mst_ports |=
+				1 << enc_to_mst(&encoder->base)->primary->port;
+			break;
 		default:
 			break;
 		}
 	}
 
+	/* can't mix MST and SST/HDMI on the same port */
+	if (used_ports & used_mst_ports)
+		return false;
+
 	return true;
 }
 
@@ -12366,7 +12943,6 @@ clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
 	struct intel_crtc_scaler_state scaler_state;
 	struct intel_dpll_hw_state dpll_hw_state;
 	struct intel_shared_dpll *shared_dpll;
-	uint32_t ddi_pll_sel;
 	bool force_thru;
 
 	/* FIXME: before the switch to atomic started, a new pipe_config was
@@ -12378,7 +12954,6 @@ clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
 	scaler_state = crtc_state->scaler_state;
 	shared_dpll = crtc_state->shared_dpll;
 	dpll_hw_state = crtc_state->dpll_hw_state;
-	ddi_pll_sel = crtc_state->ddi_pll_sel;
 	force_thru = crtc_state->pch_pfit.force_thru;
 
 	memset(crtc_state, 0, sizeof *crtc_state);
@@ -12387,7 +12962,6 @@ clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
 	crtc_state->scaler_state = scaler_state;
 	crtc_state->shared_dpll = shared_dpll;
 	crtc_state->dpll_hw_state = dpll_hw_state;
-	crtc_state->ddi_pll_sel = ddi_pll_sel;
 	crtc_state->pch_pfit.force_thru = force_thru;
 }
 
@@ -12475,7 +13049,7 @@ encoder_retry:
 
 		encoder = to_intel_encoder(connector_state->best_encoder);
 
-		if (!(encoder->compute_config(encoder, pipe_config))) {
+		if (!(encoder->compute_config(encoder, pipe_config, connector_state))) {
 			DRM_DEBUG_KMS("Encoder config failure\n");
 			goto fail;
 		}
@@ -12563,12 +13137,6 @@ static bool intel_fuzzy_clock_check(int clock1, int clock2)
 	return false;
 }
 
-#define for_each_intel_crtc_masked(dev, mask, intel_crtc) \
-	list_for_each_entry((intel_crtc), \
-			    &(dev)->mode_config.crtc_list, \
-			    base.head) \
-		for_each_if (mask & (1 <<(intel_crtc)->pipe))
-
 static bool
 intel_compare_m_n(unsigned int m, unsigned int n,
 		  unsigned int m2, unsigned int n2,
@@ -12816,8 +13384,6 @@ intel_pipe_config_compare(struct drm_device *dev,
 
 	PIPE_CONF_CHECK_I(double_wide);
 
-	PIPE_CONF_CHECK_X(ddi_pll_sel);
-
 	PIPE_CONF_CHECK_P(shared_dpll);
 	PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
 	PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
@@ -12899,16 +13465,23 @@ static void verify_wm_state(struct drm_crtc *crtc,
 			  hw_entry->start, hw_entry->end);
 	}
 
-	/* cursor */
-	hw_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
-	sw_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
-
-	if (!skl_ddb_entry_equal(hw_entry, sw_entry)) {
-		DRM_ERROR("mismatch in DDB state pipe %c cursor "
-			  "(expected (%u,%u), found (%u,%u))\n",
-			  pipe_name(pipe),
-			  sw_entry->start, sw_entry->end,
-			  hw_entry->start, hw_entry->end);
+	/*
+	 * cursor
+	 * If the cursor plane isn't active, we may not have updated it's ddb
+	 * allocation. In that case since the ddb allocation will be updated
+	 * once the plane becomes visible, we can skip this check
+	 */
+	if (intel_crtc->cursor_addr) {
+		hw_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
+		sw_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
+
+		if (!skl_ddb_entry_equal(hw_entry, sw_entry)) {
+			DRM_ERROR("mismatch in DDB state pipe %c cursor "
+				  "(expected (%u,%u), found (%u,%u))\n",
+				  pipe_name(pipe),
+				  sw_entry->start, sw_entry->end,
+				  hw_entry->start, hw_entry->end);
+		}
 	}
 }
 
@@ -13523,8 +14096,9 @@ static int intel_atomic_prepare_commit(struct drm_device *dev,
 			if (!intel_plane_state->wait_req)
 				continue;
 
-			ret = __i915_wait_request(intel_plane_state->wait_req,
-						  true, NULL, NULL);
+			ret = i915_wait_request(intel_plane_state->wait_req,
+						I915_WAIT_INTERRUPTIBLE,
+						NULL, NULL);
 			if (ret) {
 				/* Any hang should be swallowed by the wait */
 				WARN_ON(ret == -EIO);
@@ -13614,6 +14188,111 @@ static bool needs_vblank_wait(struct intel_crtc_state *crtc_state)
 	return false;
 }
 
+static void intel_update_crtc(struct drm_crtc *crtc,
+			      struct drm_atomic_state *state,
+			      struct drm_crtc_state *old_crtc_state,
+			      unsigned int *crtc_vblank_mask)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc->state);
+	bool modeset = needs_modeset(crtc->state);
+
+	if (modeset) {
+		update_scanline_offset(intel_crtc);
+		dev_priv->display.crtc_enable(pipe_config, state);
+	} else {
+		intel_pre_plane_update(to_intel_crtc_state(old_crtc_state));
+	}
+
+	if (drm_atomic_get_existing_plane_state(state, crtc->primary)) {
+		intel_fbc_enable(
+		    intel_crtc, pipe_config,
+		    to_intel_plane_state(crtc->primary->state));
+	}
+
+	drm_atomic_helper_commit_planes_on_crtc(old_crtc_state);
+
+	if (needs_vblank_wait(pipe_config))
+		*crtc_vblank_mask |= drm_crtc_mask(crtc);
+}
+
+static void intel_update_crtcs(struct drm_atomic_state *state,
+			       unsigned int *crtc_vblank_mask)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
+	int i;
+
+	for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
+		if (!crtc->state->active)
+			continue;
+
+		intel_update_crtc(crtc, state, old_crtc_state,
+				  crtc_vblank_mask);
+	}
+}
+
+static void skl_update_crtcs(struct drm_atomic_state *state,
+			     unsigned int *crtc_vblank_mask)
+{
+	struct drm_device *dev = state->dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
+	struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
+	struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
+	unsigned int updated = 0;
+	bool progress;
+	enum pipe pipe;
+
+	/*
+	 * Whenever the number of active pipes changes, we need to make sure we
+	 * update the pipes in the right order so that their ddb allocations
+	 * never overlap with eachother inbetween CRTC updates. Otherwise we'll
+	 * cause pipe underruns and other bad stuff.
+	 */
+	do {
+		int i;
+		progress = false;
+
+		for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
+			bool vbl_wait = false;
+			unsigned int cmask = drm_crtc_mask(crtc);
+			pipe = to_intel_crtc(crtc)->pipe;
+
+			if (updated & cmask || !crtc->state->active)
+				continue;
+			if (skl_ddb_allocation_overlaps(state, cur_ddb, new_ddb,
+							pipe))
+				continue;
+
+			updated |= cmask;
+
+			/*
+			 * If this is an already active pipe, it's DDB changed,
+			 * and this isn't the last pipe that needs updating
+			 * then we need to wait for a vblank to pass for the
+			 * new ddb allocation to take effect.
+			 */
+			if (!skl_ddb_allocation_equals(cur_ddb, new_ddb, pipe) &&
+			    !crtc->state->active_changed &&
+			    intel_state->wm_results.dirty_pipes != updated)
+				vbl_wait = true;
+
+			intel_update_crtc(crtc, state, old_crtc_state,
+					  crtc_vblank_mask);
+
+			if (vbl_wait)
+				intel_wait_for_vblank(dev, pipe);
+
+			progress = true;
+		}
+	} while (progress);
+}
+
 static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
@@ -13636,8 +14315,8 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		if (!intel_plane_state->wait_req)
 			continue;
 
-		ret = __i915_wait_request(intel_plane_state->wait_req,
-					  true, NULL, NULL);
+		ret = i915_wait_request(intel_plane_state->wait_req,
+					0, NULL, NULL);
 		/* EIO should be eaten, and we can't get interrupted in the
 		 * worker, and blocking commits have waited already. */
 		WARN_ON(ret);
@@ -13673,7 +14352,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 
 		if (old_crtc_state->active) {
 			intel_crtc_disable_planes(crtc, old_crtc_state->plane_mask);
-			dev_priv->display.crtc_disable(crtc);
+			dev_priv->display.crtc_disable(to_intel_crtc_state(old_crtc_state), state);
 			intel_crtc->active = false;
 			intel_fbc_disable(intel_crtc);
 			intel_disable_shared_dpll(intel_crtc);
@@ -13702,20 +14381,19 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		     intel_state->cdclk_pll_vco != dev_priv->cdclk_pll.vco))
 			dev_priv->display.modeset_commit_cdclk(state);
 
+		/*
+		 * SKL workaround: bspec recommends we disable the SAGV when we
+		 * have more then one pipe enabled
+		 */
+		if (!intel_can_enable_sagv(state))
+			intel_disable_sagv(dev_priv);
+
 		intel_modeset_verify_disabled(dev);
 	}
 
-	/* Now enable the clocks, plane, pipe, and connectors that we set up. */
+	/* Complete the events for pipes that have now been disabled */
 	for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
-		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 		bool modeset = needs_modeset(crtc->state);
-		struct intel_crtc_state *pipe_config =
-			to_intel_crtc_state(crtc->state);
-
-		if (modeset && crtc->state->active) {
-			update_scanline_offset(to_intel_crtc(crtc));
-			dev_priv->display.crtc_enable(crtc);
-		}
 
 		/* Complete events for now disable pipes here. */
 		if (modeset && !crtc->state->active && crtc->state->event) {
@@ -13725,21 +14403,11 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 
 			crtc->state->event = NULL;
 		}
-
-		if (!modeset)
-			intel_pre_plane_update(to_intel_crtc_state(old_crtc_state));
-
-		if (crtc->state->active &&
-		    drm_atomic_get_existing_plane_state(state, crtc->primary))
-			intel_fbc_enable(intel_crtc, pipe_config, to_intel_plane_state(crtc->primary->state));
-
-		if (crtc->state->active)
-			drm_atomic_helper_commit_planes_on_crtc(old_crtc_state);
-
-		if (pipe_config->base.active && needs_vblank_wait(pipe_config))
-			crtc_vblank_mask |= 1 << i;
 	}
 
+	/* Now enable the clocks, plane, pipe, and connectors that we set up. */
+	dev_priv->display.update_crtcs(state, &crtc_vblank_mask);
+
 	/* FIXME: We should call drm_atomic_helper_commit_hw_done() here
 	 * already, but still need the state for the delayed optimization. To
 	 * fix this:
@@ -13775,6 +14443,9 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		intel_modeset_verify_crtc(crtc, old_crtc_state, crtc->state);
 	}
 
+	if (intel_state->modeset && intel_can_enable_sagv(state))
+		intel_enable_sagv(dev_priv);
+
 	drm_atomic_helper_commit_hw_done(state);
 
 	if (intel_state->modeset)
@@ -13814,19 +14485,12 @@ static void intel_atomic_track_fbs(struct drm_atomic_state *state)
 {
 	struct drm_plane_state *old_plane_state;
 	struct drm_plane *plane;
-	struct drm_i915_gem_object *obj, *old_obj;
-	struct intel_plane *intel_plane;
 	int i;
 
-	mutex_lock(&state->dev->struct_mutex);
-	for_each_plane_in_state(state, plane, old_plane_state, i) {
-		obj = intel_fb_obj(plane->state->fb);
-		old_obj = intel_fb_obj(old_plane_state->fb);
-		intel_plane = to_intel_plane(plane);
-
-		i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
-	}
-	mutex_unlock(&state->dev->struct_mutex);
+	for_each_plane_in_state(state, plane, old_plane_state, i)
+		i915_gem_track_fb(intel_fb_obj(old_plane_state->fb),
+				  intel_fb_obj(plane->state->fb),
+				  to_intel_plane(plane)->frontbuffer_bit);
 }
 
 /**
@@ -13922,8 +14586,6 @@ out:
 		drm_atomic_state_free(state);
 }
 
-#undef for_each_intel_crtc_masked
-
 /*
  * FIXME: Remove this once i915 is fully DRIVER_ATOMIC by calling
  *        drm_atomic_helper_legacy_gamma_set() directly.
@@ -13992,7 +14654,7 @@ static const struct drm_crtc_funcs intel_crtc_funcs = {
  */
 int
 intel_prepare_plane_fb(struct drm_plane *plane,
-		       const struct drm_plane_state *new_state)
+		       struct drm_plane_state *new_state)
 {
 	struct drm_device *dev = plane->dev;
 	struct drm_framebuffer *fb = new_state->fb;
@@ -14051,15 +14713,17 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 		if (ret)
 			DRM_DEBUG_KMS("failed to attach phys object\n");
 	} else {
-		ret = intel_pin_and_fence_fb_obj(fb, new_state->rotation);
+		struct i915_vma *vma;
+
+		vma = intel_pin_and_fence_fb_obj(fb, new_state->rotation);
+		if (IS_ERR(vma))
+			ret = PTR_ERR(vma);
 	}
 
 	if (ret == 0) {
-		struct intel_plane_state *plane_state =
-			to_intel_plane_state(new_state);
-
-		i915_gem_request_assign(&plane_state->wait_req,
-					obj->last_write_req);
+		to_intel_plane_state(new_state)->wait_req =
+			i915_gem_active_get(&obj->last_write,
+					    &obj->base.dev->struct_mutex);
 	}
 
 	return ret;
@@ -14076,10 +14740,11 @@ intel_prepare_plane_fb(struct drm_plane *plane,
  */
 void
 intel_cleanup_plane_fb(struct drm_plane *plane,
-		       const struct drm_plane_state *old_state)
+		       struct drm_plane_state *old_state)
 {
 	struct drm_device *dev = plane->dev;
 	struct intel_plane_state *old_intel_state;
+	struct intel_plane_state *intel_state = to_intel_plane_state(plane->state);
 	struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb);
 	struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb);
 
@@ -14092,6 +14757,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
 	    !INTEL_INFO(dev)->cursor_needs_physical))
 		intel_unpin_fb_obj(old_state->fb, old_state->rotation);
 
+	i915_gem_request_assign(&intel_state->wait_req, NULL);
 	i915_gem_request_assign(&old_intel_state->wait_req, NULL);
 }
 
@@ -14126,13 +14792,14 @@ intel_check_primary_plane(struct drm_plane *plane,
 			  struct intel_crtc_state *crtc_state,
 			  struct intel_plane_state *state)
 {
+	struct drm_i915_private *dev_priv = to_i915(plane->dev);
 	struct drm_crtc *crtc = state->base.crtc;
-	struct drm_framebuffer *fb = state->base.fb;
 	int min_scale = DRM_PLANE_HELPER_NO_SCALING;
 	int max_scale = DRM_PLANE_HELPER_NO_SCALING;
 	bool can_position = false;
+	int ret;
 
-	if (INTEL_INFO(plane->dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		/* use scaler when colorkey is not required */
 		if (state->ckey.flags == I915_SET_COLORKEY_NONE) {
 			min_scale = 1;
@@ -14141,22 +14808,35 @@ intel_check_primary_plane(struct drm_plane *plane,
 		can_position = true;
 	}
 
-	return drm_plane_helper_check_update(plane, crtc, fb, &state->src,
-					     &state->dst, &state->clip,
-					     state->base.rotation,
-					     min_scale, max_scale,
-					     can_position, true,
-					     &state->visible);
+	ret = drm_plane_helper_check_state(&state->base,
+					   &state->clip,
+					   min_scale, max_scale,
+					   can_position, true);
+	if (ret)
+		return ret;
+
+	if (!state->base.fb)
+		return 0;
+
+	if (INTEL_GEN(dev_priv) >= 9) {
+		ret = skl_check_plane_surface(state);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 static void intel_begin_crtc_commit(struct drm_crtc *crtc,
 				    struct drm_crtc_state *old_crtc_state)
 {
 	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_crtc_state *old_intel_state =
 		to_intel_crtc_state(old_crtc_state);
 	bool modeset = needs_modeset(crtc->state);
+	enum pipe pipe = intel_crtc->pipe;
 
 	/* Perform vblank evasion around commit operation */
 	intel_pipe_update_start(intel_crtc);
@@ -14171,8 +14851,12 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc,
 
 	if (to_intel_crtc_state(crtc->state)->update_pipe)
 		intel_update_pipe_config(intel_crtc, old_intel_state);
-	else if (INTEL_INFO(dev)->gen >= 9)
+	else if (INTEL_GEN(dev_priv) >= 9) {
 		skl_detach_scalers(intel_crtc);
+
+		I915_WRITE(PIPE_WM_LINETIME(pipe),
+			   dev_priv->wm.skl_hw.wm_linetime[pipe]);
+	}
 }
 
 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
@@ -14306,11 +14990,11 @@ fail:
 void intel_create_rotation_property(struct drm_device *dev, struct intel_plane *plane)
 {
 	if (!dev->mode_config.rotation_property) {
-		unsigned long flags = BIT(DRM_ROTATE_0) |
-			BIT(DRM_ROTATE_180);
+		unsigned long flags = DRM_ROTATE_0 |
+			DRM_ROTATE_180;
 
 		if (INTEL_INFO(dev)->gen >= 9)
-			flags |= BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270);
+			flags |= DRM_ROTATE_90 | DRM_ROTATE_270;
 
 		dev->mode_config.rotation_property =
 			drm_mode_create_rotation_property(dev, flags);
@@ -14326,19 +15010,17 @@ intel_check_cursor_plane(struct drm_plane *plane,
 			 struct intel_crtc_state *crtc_state,
 			 struct intel_plane_state *state)
 {
-	struct drm_crtc *crtc = crtc_state->base.crtc;
 	struct drm_framebuffer *fb = state->base.fb;
 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	enum pipe pipe = to_intel_plane(plane)->pipe;
 	unsigned stride;
 	int ret;
 
-	ret = drm_plane_helper_check_update(plane, crtc, fb, &state->src,
-					    &state->dst, &state->clip,
-					    state->base.rotation,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    true, true, &state->visible);
+	ret = drm_plane_helper_check_state(&state->base,
+					   &state->clip,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   true, true);
 	if (ret)
 		return ret;
 
@@ -14375,7 +15057,7 @@ intel_check_cursor_plane(struct drm_plane *plane,
 	 * Refuse the put the cursor into that compromised position.
 	 */
 	if (IS_CHERRYVIEW(plane->dev) && pipe == PIPE_C &&
-	    state->visible && state->base.crtc_x < 0) {
+	    state->base.visible && state->base.crtc_x < 0) {
 		DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
 		return -EINVAL;
 	}
@@ -14407,7 +15089,7 @@ intel_update_cursor_plane(struct drm_plane *plane,
 	if (!obj)
 		addr = 0;
 	else if (!INTEL_INFO(dev)->cursor_needs_physical)
-		addr = i915_gem_obj_ggtt_offset(obj);
+		addr = i915_gem_object_ggtt_offset(obj, NULL);
 	else
 		addr = obj->phys_handle->busaddr;
 
@@ -14453,8 +15135,8 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
 		if (!dev->mode_config.rotation_property)
 			dev->mode_config.rotation_property =
 				drm_mode_create_rotation_property(dev,
-							BIT(DRM_ROTATE_0) |
-							BIT(DRM_ROTATE_180));
+							DRM_ROTATE_0 |
+							DRM_ROTATE_180);
 		if (dev->mode_config.rotation_property)
 			drm_object_attach_property(&cursor->base.base,
 				dev->mode_config.rotation_property,
@@ -14660,12 +15342,50 @@ static bool intel_crt_present(struct drm_device *dev)
 	return true;
 }
 
+void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
+{
+	int pps_num;
+	int pps_idx;
+
+	if (HAS_DDI(dev_priv))
+		return;
+	/*
+	 * This w/a is needed at least on CPT/PPT, but to be sure apply it
+	 * everywhere where registers can be write protected.
+	 */
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		pps_num = 2;
+	else
+		pps_num = 1;
+
+	for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
+		u32 val = I915_READ(PP_CONTROL(pps_idx));
+
+		val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
+		I915_WRITE(PP_CONTROL(pps_idx), val);
+	}
+}
+
+static void intel_pps_init(struct drm_i915_private *dev_priv)
+{
+	if (HAS_PCH_SPLIT(dev_priv) || IS_BROXTON(dev_priv))
+		dev_priv->pps_mmio_base = PCH_PPS_BASE;
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		dev_priv->pps_mmio_base = VLV_PPS_BASE;
+	else
+		dev_priv->pps_mmio_base = PPS_BASE;
+
+	intel_pps_unlock_regs_wa(dev_priv);
+}
+
 static void intel_setup_outputs(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_encoder *encoder;
 	bool dpd_is_edp = false;
 
+	intel_pps_init(dev_priv);
+
 	/*
 	 * intel_edp_init_connector() depends on this completing first, to
 	 * prevent the registeration of both eDP and LVDS and the incorrect
@@ -14853,7 +15573,7 @@ static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
 	drm_framebuffer_cleanup(fb);
 	mutex_lock(&dev->struct_mutex);
 	WARN_ON(!intel_fb->obj->framebuffer_references--);
-	drm_gem_object_unreference(&intel_fb->obj->base);
+	i915_gem_object_put(intel_fb->obj);
 	mutex_unlock(&dev->struct_mutex);
 	kfree(intel_fb);
 }
@@ -14933,24 +15653,27 @@ static int intel_framebuffer_init(struct drm_device *dev,
 				  struct drm_i915_gem_object *obj)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	unsigned int aligned_height;
+	unsigned int tiling = i915_gem_object_get_tiling(obj);
 	int ret;
 	u32 pitch_limit, stride_alignment;
+	char *format_name;
 
 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
 	if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
-		/* Enforce that fb modifier and tiling mode match, but only for
-		 * X-tiled. This is needed for FBC. */
-		if (!!(obj->tiling_mode == I915_TILING_X) !=
-		    !!(mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED)) {
+		/*
+		 * If there's a fence, enforce that
+		 * the fb modifier and tiling mode match.
+		 */
+		if (tiling != I915_TILING_NONE &&
+		    tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
 			DRM_DEBUG("tiling_mode doesn't match fb modifier\n");
 			return -EINVAL;
 		}
 	} else {
-		if (obj->tiling_mode == I915_TILING_X)
+		if (tiling == I915_TILING_X) {
 			mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
-		else if (obj->tiling_mode == I915_TILING_Y) {
+		} else if (tiling == I915_TILING_Y) {
 			DRM_DEBUG("No Y tiling for legacy addfb\n");
 			return -EINVAL;
 		}
@@ -14974,6 +15697,16 @@ static int intel_framebuffer_init(struct drm_device *dev,
 		return -EINVAL;
 	}
 
+	/*
+	 * gen2/3 display engine uses the fence if present,
+	 * so the tiling mode must match the fb modifier exactly.
+	 */
+	if (INTEL_INFO(dev_priv)->gen < 4 &&
+	    tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
+		DRM_DEBUG("tiling_mode must match fb modifier exactly on gen2/3\n");
+		return -EINVAL;
+	}
+
 	stride_alignment = intel_fb_stride_alignment(dev_priv,
 						     mode_cmd->modifier[0],
 						     mode_cmd->pixel_format);
@@ -14993,10 +15726,15 @@ static int intel_framebuffer_init(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	if (mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED &&
-	    mode_cmd->pitches[0] != obj->stride) {
+	/*
+	 * If there's a fence, enforce that
+	 * the fb pitch and fence stride match.
+	 */
+	if (tiling != I915_TILING_NONE &&
+	    mode_cmd->pitches[0] != i915_gem_object_get_stride(obj)) {
 		DRM_DEBUG("pitch (%d) must match tiling stride (%d)\n",
-			  mode_cmd->pitches[0], obj->stride);
+			  mode_cmd->pitches[0],
+			  i915_gem_object_get_stride(obj));
 		return -EINVAL;
 	}
 
@@ -15009,16 +15747,18 @@ static int intel_framebuffer_init(struct drm_device *dev,
 		break;
 	case DRM_FORMAT_XRGB1555:
 		if (INTEL_INFO(dev)->gen > 3) {
-			DRM_DEBUG("unsupported pixel format: %s\n",
-				  drm_get_format_name(mode_cmd->pixel_format));
+			format_name = drm_get_format_name(mode_cmd->pixel_format);
+			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
+			kfree(format_name);
 			return -EINVAL;
 		}
 		break;
 	case DRM_FORMAT_ABGR8888:
 		if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
 		    INTEL_INFO(dev)->gen < 9) {
-			DRM_DEBUG("unsupported pixel format: %s\n",
-				  drm_get_format_name(mode_cmd->pixel_format));
+			format_name = drm_get_format_name(mode_cmd->pixel_format);
+			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
+			kfree(format_name);
 			return -EINVAL;
 		}
 		break;
@@ -15026,15 +15766,17 @@ static int intel_framebuffer_init(struct drm_device *dev,
 	case DRM_FORMAT_XRGB2101010:
 	case DRM_FORMAT_XBGR2101010:
 		if (INTEL_INFO(dev)->gen < 4) {
-			DRM_DEBUG("unsupported pixel format: %s\n",
-				  drm_get_format_name(mode_cmd->pixel_format));
+			format_name = drm_get_format_name(mode_cmd->pixel_format);
+			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
+			kfree(format_name);
 			return -EINVAL;
 		}
 		break;
 	case DRM_FORMAT_ABGR2101010:
 		if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
-			DRM_DEBUG("unsupported pixel format: %s\n",
-				  drm_get_format_name(mode_cmd->pixel_format));
+			format_name = drm_get_format_name(mode_cmd->pixel_format);
+			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
+			kfree(format_name);
 			return -EINVAL;
 		}
 		break;
@@ -15043,14 +15785,16 @@ static int intel_framebuffer_init(struct drm_device *dev,
 	case DRM_FORMAT_YVYU:
 	case DRM_FORMAT_VYUY:
 		if (INTEL_INFO(dev)->gen < 5) {
-			DRM_DEBUG("unsupported pixel format: %s\n",
-				  drm_get_format_name(mode_cmd->pixel_format));
+			format_name = drm_get_format_name(mode_cmd->pixel_format);
+			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
+			kfree(format_name);
 			return -EINVAL;
 		}
 		break;
 	default:
-		DRM_DEBUG("unsupported pixel format: %s\n",
-			  drm_get_format_name(mode_cmd->pixel_format));
+		format_name = drm_get_format_name(mode_cmd->pixel_format);
+		DRM_DEBUG("unsupported pixel format: %s\n", format_name);
+		kfree(format_name);
 		return -EINVAL;
 	}
 
@@ -15058,17 +15802,12 @@ static int intel_framebuffer_init(struct drm_device *dev,
 	if (mode_cmd->offsets[0] != 0)
 		return -EINVAL;
 
-	aligned_height = intel_fb_align_height(dev, mode_cmd->height,
-					       mode_cmd->pixel_format,
-					       mode_cmd->modifier[0]);
-	/* FIXME drm helper for size checks (especially planar formats)? */
-	if (obj->base.size < aligned_height * mode_cmd->pitches[0])
-		return -EINVAL;
-
 	drm_helper_mode_fill_fb_struct(&intel_fb->base, mode_cmd);
 	intel_fb->obj = obj;
 
-	intel_fill_fb_info(dev_priv, &intel_fb->base);
+	ret = intel_fill_fb_info(dev_priv, &intel_fb->base);
+	if (ret)
+		return ret;
 
 	ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs);
 	if (ret) {
@@ -15090,13 +15829,13 @@ intel_user_framebuffer_create(struct drm_device *dev,
 	struct drm_i915_gem_object *obj;
 	struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
 
-	obj = to_intel_bo(drm_gem_object_lookup(filp, mode_cmd.handles[0]));
-	if (&obj->base == NULL)
+	obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
+	if (!obj)
 		return ERR_PTR(-ENOENT);
 
 	fb = intel_framebuffer_create(dev, &mode_cmd, obj);
 	if (IS_ERR(fb))
-		drm_gem_object_unreference_unlocked(&obj->base);
+		i915_gem_object_put_unlocked(obj);
 
 	return fb;
 }
@@ -15279,6 +16018,11 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv)
 			skl_modeset_calc_cdclk;
 	}
 
+	if (dev_priv->info.gen >= 9)
+		dev_priv->display.update_crtcs = skl_update_crtcs;
+	else
+		dev_priv->display.update_crtcs = intel_update_crtcs;
+
 	switch (INTEL_INFO(dev_priv)->gen) {
 	case 2:
 		dev_priv->display.queue_flip = intel_gen2_queue_flip;
@@ -15480,15 +16224,16 @@ static void intel_init_quirks(struct drm_device *dev)
 static void i915_disable_vga(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	u8 sr1;
 	i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
 
 	/* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
-	vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
+	vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 	outb(SR01, VGA_SR_INDEX);
 	sr1 = inb(VGA_SR_DATA);
 	outb(sr1 | 1<<5, VGA_SR_DATA);
-	vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
+	vga_put(pdev, VGA_RSRC_LEGACY_IO);
 	udelay(300);
 
 	I915_WRITE(vga_reg, VGA_DISP_DISABLE);
@@ -15504,7 +16249,6 @@ void intel_modeset_init_hw(struct drm_device *dev)
 	dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq;
 
 	intel_init_clock_gating(dev);
-	intel_enable_gt_powersave(dev_priv);
 }
 
 /*
@@ -15771,15 +16515,22 @@ static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
 	return false;
 }
 
-static bool intel_encoder_has_connectors(struct intel_encoder *encoder)
+static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct intel_connector *connector;
 
 	for_each_connector_on_encoder(dev, &encoder->base, connector)
-		return true;
+		return connector;
 
-	return false;
+	return NULL;
+}
+
+static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
+			      enum transcoder pch_transcoder)
+{
+	return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
+		(HAS_PCH_LPT_H(dev_priv) && pch_transcoder == TRANSCODER_A);
 }
 
 static void intel_sanitize_crtc(struct intel_crtc *crtc)
@@ -15825,7 +16576,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
 		 * Temporarily change the plane mapping and disable everything
 		 * ...  */
 		plane = crtc->plane;
-		to_intel_plane_state(crtc->base.primary->state)->visible = true;
+		to_intel_plane_state(crtc->base.primary->state)->base.visible = true;
 		crtc->plane = !plane;
 		intel_crtc_disable_noatomic(&crtc->base);
 		crtc->plane = plane;
@@ -15860,14 +16611,23 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
 		 * worst a fifo underrun happens which also sets this to false.
 		 */
 		crtc->cpu_fifo_underrun_disabled = true;
-		crtc->pch_fifo_underrun_disabled = true;
+		/*
+		 * We track the PCH trancoder underrun reporting state
+		 * within the crtc. With crtc for pipe A housing the underrun
+		 * reporting state for PCH transcoder A, crtc for pipe B housing
+		 * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
+		 * and marking underrun reporting as disabled for the non-existing
+		 * PCH transcoders B and C would prevent enabling the south
+		 * error interrupt (see cpt_can_enable_serr_int()).
+		 */
+		if (has_pch_trancoder(dev_priv, (enum transcoder)crtc->pipe))
+			crtc->pch_fifo_underrun_disabled = true;
 	}
 }
 
 static void intel_sanitize_encoder(struct intel_encoder *encoder)
 {
 	struct intel_connector *connector;
-	struct drm_device *dev = encoder->base.dev;
 
 	/* We need to check both for a crtc link (meaning that the
 	 * encoder is active and trying to read from a pipe) and the
@@ -15875,7 +16635,8 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
 	bool has_active_crtc = encoder->base.crtc &&
 		to_intel_crtc(encoder->base.crtc)->active;
 
-	if (intel_encoder_has_connectors(encoder) && !has_active_crtc) {
+	connector = intel_encoder_find_connector(encoder);
+	if (connector && !has_active_crtc) {
 		DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
 			      encoder->base.base.id,
 			      encoder->base.name);
@@ -15884,12 +16645,14 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
 		 * fallout from our resume register restoring. Disable
 		 * the encoder manually again. */
 		if (encoder->base.crtc) {
+			struct drm_crtc_state *crtc_state = encoder->base.crtc->state;
+
 			DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
 				      encoder->base.base.id,
 				      encoder->base.name);
-			encoder->disable(encoder);
+			encoder->disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state);
 			if (encoder->post_disable)
-				encoder->post_disable(encoder);
+				encoder->post_disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state);
 		}
 		encoder->base.crtc = NULL;
 
@@ -15897,12 +16660,9 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
 		 * a bug in one of the get_hw_state functions. Or someplace else
 		 * in our code, like the register restore mess on resume. Clamp
 		 * things to off as a safer default. */
-		for_each_intel_connector(dev, connector) {
-			if (connector->encoder != encoder)
-				continue;
-			connector->base.dpms = DRM_MODE_DPMS_OFF;
-			connector->base.encoder = NULL;
-		}
+
+		connector->base.dpms = DRM_MODE_DPMS_OFF;
+		connector->base.encoder = NULL;
 	}
 	/* Enabled encoders without active connectors will be fixed in
 	 * the crtc fixup. */
@@ -15952,10 +16712,10 @@ static void readout_plane_state(struct intel_crtc *crtc)
 	struct intel_plane_state *plane_state =
 		to_intel_plane_state(primary->state);
 
-	plane_state->visible = crtc->active &&
+	plane_state->base.visible = crtc->active &&
 		primary_get_hw_state(to_intel_plane(primary));
 
-	if (plane_state->visible)
+	if (plane_state->base.visible)
 		crtc->base.state->plane_mask |= 1 << drm_plane_index(primary);
 }
 
@@ -16174,9 +16934,10 @@ void intel_display_resume(struct drm_device *dev)
 	struct drm_atomic_state *state = dev_priv->modeset_restore_state;
 	struct drm_modeset_acquire_ctx ctx;
 	int ret;
-	bool setup = false;
 
 	dev_priv->modeset_restore_state = NULL;
+	if (state)
+		state->acquire_ctx = &ctx;
 
 	/*
 	 * This is a cludge because with real atomic modeset mode_config.mutex
@@ -16187,43 +16948,17 @@ void intel_display_resume(struct drm_device *dev)
 	mutex_lock(&dev->mode_config.mutex);
 	drm_modeset_acquire_init(&ctx, 0);
 
-retry:
-	ret = drm_modeset_lock_all_ctx(dev, &ctx);
-
-	if (ret == 0 && !setup) {
-		setup = true;
-
-		intel_modeset_setup_hw_state(dev);
-		i915_redisable_vga(dev);
-	}
-
-	if (ret == 0 && state) {
-		struct drm_crtc_state *crtc_state;
-		struct drm_crtc *crtc;
-		int i;
-
-		state->acquire_ctx = &ctx;
-
-		/* ignore any reset values/BIOS leftovers in the WM registers */
-		to_intel_atomic_state(state)->skip_intermediate_wm = true;
-
-		for_each_crtc_in_state(state, crtc, crtc_state, i) {
-			/*
-			 * Force recalculation even if we restore
-			 * current state. With fast modeset this may not result
-			 * in a modeset when the state is compatible.
-			 */
-			crtc_state->mode_changed = true;
-		}
-
-		ret = drm_atomic_commit(state);
-	}
+	while (1) {
+		ret = drm_modeset_lock_all_ctx(dev, &ctx);
+		if (ret != -EDEADLK)
+			break;
 
-	if (ret == -EDEADLK) {
 		drm_modeset_backoff(&ctx);
-		goto retry;
 	}
 
+	if (!ret)
+		ret = __intel_display_resume(dev, state);
+
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
 	mutex_unlock(&dev->mode_config.mutex);
@@ -16239,7 +16974,6 @@ void intel_modeset_gem_init(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_crtc *c;
 	struct drm_i915_gem_object *obj;
-	int ret;
 
 	intel_init_gt_powersave(dev_priv);
 
@@ -16253,15 +16987,17 @@ void intel_modeset_gem_init(struct drm_device *dev)
 	 * for this.
 	 */
 	for_each_crtc(dev, c) {
+		struct i915_vma *vma;
+
 		obj = intel_fb_obj(c->primary->fb);
 		if (obj == NULL)
 			continue;
 
 		mutex_lock(&dev->struct_mutex);
-		ret = intel_pin_and_fence_fb_obj(c->primary->fb,
+		vma = intel_pin_and_fence_fb_obj(c->primary->fb,
 						 c->primary->state->rotation);
 		mutex_unlock(&dev->struct_mutex);
-		if (ret) {
+		if (IS_ERR(vma)) {
 			DRM_ERROR("failed to pin boot fb on pipe %d\n",
 				  to_intel_crtc(c)->pipe);
 			drm_framebuffer_unreference(c->primary->fb);
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 21b04c3eda41..14a3cf0b7213 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -190,6 +190,29 @@ intel_dp_max_data_rate(int max_link_clock, int max_lanes)
 	return (max_link_clock * max_lanes * 8) / 10;
 }
 
+static int
+intel_dp_downstream_max_dotclock(struct intel_dp *intel_dp)
+{
+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+	struct intel_encoder *encoder = &intel_dig_port->base;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	int max_dotclk = dev_priv->max_dotclk_freq;
+	int ds_max_dotclk;
+
+	int type = intel_dp->downstream_ports[0] & DP_DS_PORT_TYPE_MASK;
+
+	if (type != DP_DS_PORT_TYPE_VGA)
+		return max_dotclk;
+
+	ds_max_dotclk = drm_dp_downstream_max_clock(intel_dp->dpcd,
+						    intel_dp->downstream_ports);
+
+	if (ds_max_dotclk != 0)
+		max_dotclk = min(max_dotclk, ds_max_dotclk);
+
+	return max_dotclk;
+}
+
 static enum drm_mode_status
 intel_dp_mode_valid(struct drm_connector *connector,
 		    struct drm_display_mode *mode)
@@ -199,7 +222,9 @@ intel_dp_mode_valid(struct drm_connector *connector,
 	struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode;
 	int target_clock = mode->clock;
 	int max_rate, mode_rate, max_lanes, max_link_clock;
-	int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
+	int max_dotclk;
+
+	max_dotclk = intel_dp_downstream_max_dotclock(intel_dp);
 
 	if (is_edp(intel_dp) && fixed_mode) {
 		if (mode->hdisplay > fixed_mode->hdisplay)
@@ -256,6 +281,8 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
 static void
 intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 					      struct intel_dp *intel_dp);
+static void
+intel_dp_pps_init(struct drm_device *dev, struct intel_dp *intel_dp);
 
 static void pps_lock(struct intel_dp *intel_dp)
 {
@@ -463,13 +490,13 @@ typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv,
 static bool vlv_pipe_has_pp_on(struct drm_i915_private *dev_priv,
 			       enum pipe pipe)
 {
-	return I915_READ(VLV_PIPE_PP_STATUS(pipe)) & PP_ON;
+	return I915_READ(PP_STATUS(pipe)) & PP_ON;
 }
 
 static bool vlv_pipe_has_vdd_on(struct drm_i915_private *dev_priv,
 				enum pipe pipe)
 {
-	return I915_READ(VLV_PIPE_PP_CONTROL(pipe)) & EDP_FORCE_VDD;
+	return I915_READ(PP_CONTROL(pipe)) & EDP_FORCE_VDD;
 }
 
 static bool vlv_pipe_any(struct drm_i915_private *dev_priv,
@@ -486,7 +513,7 @@ vlv_initial_pps_pipe(struct drm_i915_private *dev_priv,
 	enum pipe pipe;
 
 	for (pipe = PIPE_A; pipe <= PIPE_B; pipe++) {
-		u32 port_sel = I915_READ(VLV_PIPE_PP_ON_DELAYS(pipe)) &
+		u32 port_sel = I915_READ(PP_ON_DELAYS(pipe)) &
 			PANEL_PORT_SELECT_MASK;
 
 		if (port_sel != PANEL_PORT_SELECT_VLV(port))
@@ -583,30 +610,21 @@ static void intel_pps_get_registers(struct drm_i915_private *dev_priv,
 				    struct intel_dp *intel_dp,
 				    struct pps_registers *regs)
 {
+	int pps_idx = 0;
+
 	memset(regs, 0, sizeof(*regs));
 
-	if (IS_BROXTON(dev_priv)) {
-		int idx = bxt_power_sequencer_idx(intel_dp);
-
-		regs->pp_ctrl = BXT_PP_CONTROL(idx);
-		regs->pp_stat = BXT_PP_STATUS(idx);
-		regs->pp_on = BXT_PP_ON_DELAYS(idx);
-		regs->pp_off = BXT_PP_OFF_DELAYS(idx);
-	} else if (HAS_PCH_SPLIT(dev_priv)) {
-		regs->pp_ctrl = PCH_PP_CONTROL;
-		regs->pp_stat = PCH_PP_STATUS;
-		regs->pp_on = PCH_PP_ON_DELAYS;
-		regs->pp_off = PCH_PP_OFF_DELAYS;
-		regs->pp_div = PCH_PP_DIVISOR;
-	} else {
-		enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
+	if (IS_BROXTON(dev_priv))
+		pps_idx = bxt_power_sequencer_idx(intel_dp);
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		pps_idx = vlv_power_sequencer_pipe(intel_dp);
 
-		regs->pp_ctrl = VLV_PIPE_PP_CONTROL(pipe);
-		regs->pp_stat = VLV_PIPE_PP_STATUS(pipe);
-		regs->pp_on = VLV_PIPE_PP_ON_DELAYS(pipe);
-		regs->pp_off = VLV_PIPE_PP_OFF_DELAYS(pipe);
-		regs->pp_div = VLV_PIPE_PP_DIVISOR(pipe);
-	}
+	regs->pp_ctrl = PP_CONTROL(pps_idx);
+	regs->pp_stat = PP_STATUS(pps_idx);
+	regs->pp_on = PP_ON_DELAYS(pps_idx);
+	regs->pp_off = PP_OFF_DELAYS(pps_idx);
+	if (!IS_BROXTON(dev_priv))
+		regs->pp_div = PP_DIVISOR(pps_idx);
 }
 
 static i915_reg_t
@@ -651,8 +669,8 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
 		i915_reg_t pp_ctrl_reg, pp_div_reg;
 		u32 pp_div;
 
-		pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe);
-		pp_div_reg  = VLV_PIPE_PP_DIVISOR(pipe);
+		pp_ctrl_reg = PP_CONTROL(pipe);
+		pp_div_reg  = PP_DIVISOR(pipe);
 		pp_div = I915_READ(pp_div_reg);
 		pp_div &= PP_REFERENCE_DIVIDER_MASK;
 
@@ -1041,10 +1059,10 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
 		if (WARN_ON(txsize > 20))
 			return -E2BIG;
 
+		WARN_ON(!msg->buffer != !msg->size);
+
 		if (msg->buffer)
 			memcpy(txbuf + HEADER_SIZE, msg->buffer, msg->size);
-		else
-			WARN_ON(msg->size);
 
 		ret = intel_dp_aux_ch(intel_dp, txbuf, txsize, rxbuf, rxsize);
 		if (ret > 0) {
@@ -1250,7 +1268,7 @@ intel_dp_aux_fini(struct intel_dp *intel_dp)
 }
 
 static void
-intel_dp_aux_init(struct intel_dp *intel_dp, struct intel_connector *connector)
+intel_dp_aux_init(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	enum port port = intel_dig_port->port;
@@ -1426,6 +1444,44 @@ static void intel_dp_print_rates(struct intel_dp *intel_dp)
 	DRM_DEBUG_KMS("common rates: %s\n", str);
 }
 
+static void intel_dp_print_hw_revision(struct intel_dp *intel_dp)
+{
+	uint8_t rev;
+	int len;
+
+	if ((drm_debug & DRM_UT_KMS) == 0)
+		return;
+
+	if (!(intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] &
+	      DP_DWN_STRM_PORT_PRESENT))
+		return;
+
+	len = drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_HW_REV, &rev, 1);
+	if (len < 0)
+		return;
+
+	DRM_DEBUG_KMS("sink hw revision: %d.%d\n", (rev & 0xf0) >> 4, rev & 0xf);
+}
+
+static void intel_dp_print_sw_revision(struct intel_dp *intel_dp)
+{
+	uint8_t rev[2];
+	int len;
+
+	if ((drm_debug & DRM_UT_KMS) == 0)
+		return;
+
+	if (!(intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] &
+	      DP_DWN_STRM_PORT_PRESENT))
+		return;
+
+	len = drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_SW_REV, &rev, 2);
+	if (len < 0)
+		return;
+
+	DRM_DEBUG_KMS("sink sw revision: %d.%d\n", rev[0], rev[1]);
+}
+
 static int rate_to_index(int find, const int *rates)
 {
 	int i = 0;
@@ -1447,7 +1503,7 @@ intel_dp_max_link_rate(struct intel_dp *intel_dp)
 	if (WARN_ON(len <= 0))
 		return 162000;
 
-	return rates[rate_to_index(0, rates) - 1];
+	return rates[len - 1];
 }
 
 int intel_dp_rate_select(struct intel_dp *intel_dp, int rate)
@@ -1468,9 +1524,24 @@ void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
 	}
 }
 
+static int intel_dp_compute_bpp(struct intel_dp *intel_dp,
+				struct intel_crtc_state *pipe_config)
+{
+	int bpp, bpc;
+
+	bpp = pipe_config->pipe_bpp;
+	bpc = drm_dp_downstream_max_bpc(intel_dp->dpcd, intel_dp->downstream_ports);
+
+	if (bpc > 0)
+		bpp = min(bpp, 3*bpc);
+
+	return bpp;
+}
+
 bool
 intel_dp_compute_config(struct intel_encoder *encoder,
-			struct intel_crtc_state *pipe_config)
+			struct intel_crtc_state *pipe_config,
+			struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -1533,7 +1604,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
 	/* Walk through all bpp values. Luckily they're all nicely spaced with 2
 	 * bpc in between. */
-	bpp = pipe_config->pipe_bpp;
+	bpp = intel_dp_compute_bpp(intel_dp, pipe_config);
 	if (is_edp(intel_dp)) {
 
 		/* Get bpp from vbt only for panels that dont have bpp in edid */
@@ -1647,22 +1718,28 @@ found:
 }
 
 void intel_dp_set_link_params(struct intel_dp *intel_dp,
-			      const struct intel_crtc_state *pipe_config)
+			      int link_rate, uint8_t lane_count,
+			      bool link_mst)
 {
-	intel_dp->link_rate = pipe_config->port_clock;
-	intel_dp->lane_count = pipe_config->lane_count;
+	intel_dp->link_rate = link_rate;
+	intel_dp->lane_count = lane_count;
+	intel_dp->link_mst = link_mst;
 }
 
-static void intel_dp_prepare(struct intel_encoder *encoder)
+static void intel_dp_prepare(struct intel_encoder *encoder,
+			     struct intel_crtc_state *pipe_config)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	enum port port = dp_to_dig_port(intel_dp)->port;
 	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-	const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 
-	intel_dp_set_link_params(intel_dp, crtc->config);
+	intel_dp_set_link_params(intel_dp, pipe_config->port_clock,
+				 pipe_config->lane_count,
+				 intel_crtc_has_type(pipe_config,
+						     INTEL_OUTPUT_DP_MST));
 
 	/*
 	 * There are four kinds of DP registers:
@@ -1688,7 +1765,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder)
 
 	/* Handle DP bits in common between all three register formats */
 	intel_dp->DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0;
-	intel_dp->DP |= DP_PORT_WIDTH(crtc->config->lane_count);
+	intel_dp->DP |= DP_PORT_WIDTH(pipe_config->lane_count);
 
 	/* Split out the IBX/CPU vs CPT settings */
 
@@ -1716,7 +1793,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder)
 		I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp);
 	} else {
 		if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev) &&
-		    !IS_CHERRYVIEW(dev) && crtc->config->limited_color_range)
+		    !IS_CHERRYVIEW(dev) && pipe_config->limited_color_range)
 			intel_dp->DP |= DP_COLOR_RANGE_16_235;
 
 		if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
@@ -1835,7 +1912,8 @@ static  u32 ironlake_get_pp_control(struct intel_dp *intel_dp)
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
 	control = I915_READ(_pp_ctrl_reg(intel_dp));
-	if (!IS_BROXTON(dev)) {
+	if (WARN_ON(!HAS_DDI(dev_priv) &&
+		    (control & PANEL_UNLOCK_MASK) != PANEL_UNLOCK_REGS)) {
 		control &= ~PANEL_UNLOCK_MASK;
 		control |= PANEL_UNLOCK_REGS;
 	}
@@ -1956,7 +2034,7 @@ static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
 	DRM_DEBUG_KMS("PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n",
 	I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg));
 
-	if ((pp & POWER_TARGET_ON) == 0)
+	if ((pp & PANEL_POWER_ON) == 0)
 		intel_dp->panel_power_off_time = ktime_get_boottime();
 
 	power_domain = intel_display_port_aux_power_domain(intel_encoder);
@@ -2043,7 +2121,7 @@ static void edp_panel_on(struct intel_dp *intel_dp)
 		POSTING_READ(pp_ctrl_reg);
 	}
 
-	pp |= POWER_TARGET_ON;
+	pp |= PANEL_POWER_ON;
 	if (!IS_GEN5(dev))
 		pp |= PANEL_POWER_RESET;
 
@@ -2095,7 +2173,7 @@ static void edp_panel_off(struct intel_dp *intel_dp)
 	pp = ironlake_get_pp_control(intel_dp);
 	/* We need to switch off panel power _and_ force vdd, for otherwise some
 	 * panels get very unhappy and cease to work. */
-	pp &= ~(POWER_TARGET_ON | PANEL_POWER_RESET | EDP_FORCE_VDD |
+	pp &= ~(PANEL_POWER_ON | PANEL_POWER_RESET | EDP_FORCE_VDD |
 		EDP_BLC_ENABLE);
 
 	pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
@@ -2254,10 +2332,10 @@ static void assert_edp_pll(struct drm_i915_private *dev_priv, bool state)
 #define assert_edp_pll_enabled(d) assert_edp_pll((d), true)
 #define assert_edp_pll_disabled(d) assert_edp_pll((d), false)
 
-static void ironlake_edp_pll_on(struct intel_dp *intel_dp)
+static void ironlake_edp_pll_on(struct intel_dp *intel_dp,
+				struct intel_crtc_state *pipe_config)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
 	assert_pipe_disabled(dev_priv, crtc->pipe);
@@ -2265,11 +2343,11 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp)
 	assert_edp_pll_disabled(dev_priv);
 
 	DRM_DEBUG_KMS("enabling eDP PLL for clock %d\n",
-		      crtc->config->port_clock);
+		      pipe_config->port_clock);
 
 	intel_dp->DP &= ~DP_PLL_FREQ_MASK;
 
-	if (crtc->config->port_clock == 162000)
+	if (pipe_config->port_clock == 162000)
 		intel_dp->DP |= DP_PLL_FREQ_162MHZ;
 	else
 		intel_dp->DP |= DP_PLL_FREQ_270MHZ;
@@ -2478,16 +2556,17 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
 	}
 }
 
-static void intel_disable_dp(struct intel_encoder *encoder)
+static void intel_disable_dp(struct intel_encoder *encoder,
+			     struct intel_crtc_state *old_crtc_state,
+			     struct drm_connector_state *old_conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	struct drm_device *dev = encoder->base.dev;
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (crtc->config->has_audio)
+	if (old_crtc_state->has_audio)
 		intel_audio_codec_disable(encoder);
 
-	if (HAS_PSR(dev) && !HAS_DDI(dev))
+	if (HAS_PSR(dev_priv) && !HAS_DDI(dev_priv))
 		intel_psr_disable(intel_dp);
 
 	/* Make sure the panel is off before trying to change the mode. But also
@@ -2498,11 +2577,13 @@ static void intel_disable_dp(struct intel_encoder *encoder)
 	intel_edp_panel_off(intel_dp);
 
 	/* disable the port before the pipe on g4x */
-	if (INTEL_INFO(dev)->gen < 5)
+	if (INTEL_GEN(dev_priv) < 5)
 		intel_dp_link_down(intel_dp);
 }
 
-static void ilk_post_disable_dp(struct intel_encoder *encoder)
+static void ilk_post_disable_dp(struct intel_encoder *encoder,
+				struct intel_crtc_state *old_crtc_state,
+				struct drm_connector_state *old_conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	enum port port = dp_to_dig_port(intel_dp)->port;
@@ -2514,14 +2595,18 @@ static void ilk_post_disable_dp(struct intel_encoder *encoder)
 		ironlake_edp_pll_off(intel_dp);
 }
 
-static void vlv_post_disable_dp(struct intel_encoder *encoder)
+static void vlv_post_disable_dp(struct intel_encoder *encoder,
+				struct intel_crtc_state *old_crtc_state,
+				struct drm_connector_state *old_conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
 	intel_dp_link_down(intel_dp);
 }
 
-static void chv_post_disable_dp(struct intel_encoder *encoder)
+static void chv_post_disable_dp(struct intel_encoder *encoder,
+				struct intel_crtc_state *old_crtc_state,
+				struct drm_connector_state *old_conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	struct drm_device *dev = encoder->base.dev;
@@ -2547,6 +2632,10 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum port port = intel_dig_port->port;
 
+	if (dp_train_pat & DP_TRAINING_PATTERN_MASK)
+		DRM_DEBUG_KMS("Using DP training pattern TPS%d\n",
+			      dp_train_pat & DP_TRAINING_PATTERN_MASK);
+
 	if (HAS_DDI(dev)) {
 		uint32_t temp = I915_READ(DP_TP_CTL(port));
 
@@ -2588,7 +2677,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 			*DP |= DP_LINK_TRAIN_PAT_2_CPT;
 			break;
 		case DP_TRAINING_PATTERN_3:
-			DRM_ERROR("DP training pattern 3 not supported\n");
+			DRM_DEBUG_KMS("TPS3 not supported, using TPS2 instead\n");
 			*DP |= DP_LINK_TRAIN_PAT_2_CPT;
 			break;
 		}
@@ -2613,7 +2702,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 			if (IS_CHERRYVIEW(dev)) {
 				*DP |= DP_LINK_TRAIN_PAT_3_CHV;
 			} else {
-				DRM_ERROR("DP training pattern 3 not supported\n");
+				DRM_DEBUG_KMS("TPS3 not supported, using TPS2 instead\n");
 				*DP |= DP_LINK_TRAIN_PAT_2;
 			}
 			break;
@@ -2621,19 +2710,15 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 	}
 }
 
-static void intel_dp_enable_port(struct intel_dp *intel_dp)
+static void intel_dp_enable_port(struct intel_dp *intel_dp,
+				 struct intel_crtc_state *old_crtc_state)
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *crtc =
-		to_intel_crtc(dp_to_dig_port(intel_dp)->base.base.crtc);
 
 	/* enable with pattern 1 (as per spec) */
-	_intel_dp_set_link_train(intel_dp, &intel_dp->DP,
-				 DP_TRAINING_PATTERN_1);
 
-	I915_WRITE(intel_dp->output_reg, intel_dp->DP);
-	POSTING_READ(intel_dp->output_reg);
+	intel_dp_program_link_training_pattern(intel_dp, DP_TRAINING_PATTERN_1);
 
 	/*
 	 * Magic for VLV/CHV. We _must_ first set up the register
@@ -2642,14 +2727,15 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp)
 	 * fail when the power sequencer is freshly used for this port.
 	 */
 	intel_dp->DP |= DP_PORT_EN;
-	if (crtc->config->has_audio)
+	if (old_crtc_state->has_audio)
 		intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE;
 
 	I915_WRITE(intel_dp->output_reg, intel_dp->DP);
 	POSTING_READ(intel_dp->output_reg);
 }
 
-static void intel_enable_dp(struct intel_encoder *encoder)
+static void intel_enable_dp(struct intel_encoder *encoder,
+			    struct intel_crtc_state *pipe_config)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	struct drm_device *dev = encoder->base.dev;
@@ -2666,7 +2752,7 @@ static void intel_enable_dp(struct intel_encoder *encoder)
 	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
 		vlv_init_panel_power_sequencer(intel_dp);
 
-	intel_dp_enable_port(intel_dp);
+	intel_dp_enable_port(intel_dp, pipe_config);
 
 	edp_panel_vdd_on(intel_dp);
 	edp_panel_on(intel_dp);
@@ -2678,7 +2764,7 @@ static void intel_enable_dp(struct intel_encoder *encoder)
 		unsigned int lane_mask = 0x0;
 
 		if (IS_CHERRYVIEW(dev))
-			lane_mask = intel_dp_unused_lane_mask(crtc->config->lane_count);
+			lane_mask = intel_dp_unused_lane_mask(pipe_config->lane_count);
 
 		vlv_wait_port_ready(dev_priv, dp_to_dig_port(intel_dp),
 				    lane_mask);
@@ -2688,22 +2774,26 @@ static void intel_enable_dp(struct intel_encoder *encoder)
 	intel_dp_start_link_train(intel_dp);
 	intel_dp_stop_link_train(intel_dp);
 
-	if (crtc->config->has_audio) {
+	if (pipe_config->has_audio) {
 		DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n",
 				 pipe_name(pipe));
 		intel_audio_codec_enable(encoder);
 	}
 }
 
-static void g4x_enable_dp(struct intel_encoder *encoder)
+static void g4x_enable_dp(struct intel_encoder *encoder,
+			  struct intel_crtc_state *pipe_config,
+			  struct drm_connector_state *conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
-	intel_enable_dp(encoder);
+	intel_enable_dp(encoder, pipe_config);
 	intel_edp_backlight_on(intel_dp);
 }
 
-static void vlv_enable_dp(struct intel_encoder *encoder)
+static void vlv_enable_dp(struct intel_encoder *encoder,
+			  struct intel_crtc_state *pipe_config,
+			  struct drm_connector_state *conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
@@ -2711,16 +2801,18 @@ static void vlv_enable_dp(struct intel_encoder *encoder)
 	intel_psr_enable(intel_dp);
 }
 
-static void g4x_pre_enable_dp(struct intel_encoder *encoder)
+static void g4x_pre_enable_dp(struct intel_encoder *encoder,
+			      struct intel_crtc_state *pipe_config,
+			      struct drm_connector_state *conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	enum port port = dp_to_dig_port(intel_dp)->port;
 
-	intel_dp_prepare(encoder);
+	intel_dp_prepare(encoder, pipe_config);
 
 	/* Only ilk+ has port A */
 	if (port == PORT_A)
-		ironlake_edp_pll_on(intel_dp);
+		ironlake_edp_pll_on(intel_dp, pipe_config);
 }
 
 static void vlv_detach_power_sequencer(struct intel_dp *intel_dp)
@@ -2728,7 +2820,7 @@ static void vlv_detach_power_sequencer(struct intel_dp *intel_dp)
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
 	enum pipe pipe = intel_dp->pps_pipe;
-	i915_reg_t pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe);
+	i915_reg_t pp_on_reg = PP_ON_DELAYS(pipe);
 
 	edp_panel_vdd_off_sync(intel_dp);
 
@@ -2826,38 +2918,48 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
 	intel_dp_init_panel_power_sequencer_registers(dev, intel_dp);
 }
 
-static void vlv_pre_enable_dp(struct intel_encoder *encoder)
+static void vlv_pre_enable_dp(struct intel_encoder *encoder,
+			      struct intel_crtc_state *pipe_config,
+			      struct drm_connector_state *conn_state)
 {
 	vlv_phy_pre_encoder_enable(encoder);
 
-	intel_enable_dp(encoder);
+	intel_enable_dp(encoder, pipe_config);
 }
 
-static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder)
+static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder,
+				  struct intel_crtc_state *pipe_config,
+				  struct drm_connector_state *conn_state)
 {
-	intel_dp_prepare(encoder);
+	intel_dp_prepare(encoder, pipe_config);
 
 	vlv_phy_pre_pll_enable(encoder);
 }
 
-static void chv_pre_enable_dp(struct intel_encoder *encoder)
+static void chv_pre_enable_dp(struct intel_encoder *encoder,
+			      struct intel_crtc_state *pipe_config,
+			      struct drm_connector_state *conn_state)
 {
 	chv_phy_pre_encoder_enable(encoder);
 
-	intel_enable_dp(encoder);
+	intel_enable_dp(encoder, pipe_config);
 
 	/* Second common lane will stay alive on its own now */
 	chv_phy_release_cl2_override(encoder);
 }
 
-static void chv_dp_pre_pll_enable(struct intel_encoder *encoder)
+static void chv_dp_pre_pll_enable(struct intel_encoder *encoder,
+				  struct intel_crtc_state *pipe_config,
+				  struct drm_connector_state *conn_state)
 {
-	intel_dp_prepare(encoder);
+	intel_dp_prepare(encoder, pipe_config);
 
 	chv_phy_pre_pll_enable(encoder);
 }
 
-static void chv_dp_post_pll_disable(struct intel_encoder *encoder)
+static void chv_dp_post_pll_disable(struct intel_encoder *encoder,
+				    struct intel_crtc_state *pipe_config,
+				    struct drm_connector_state *conn_state)
 {
 	chv_phy_post_pll_disable(encoder);
 }
@@ -3395,84 +3497,67 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 }
 
 static bool
-intel_dp_get_dpcd(struct intel_dp *intel_dp)
+intel_dp_read_dpcd(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	if (drm_dp_dpcd_read(&intel_dp->aux, 0x000, intel_dp->dpcd,
 			     sizeof(intel_dp->dpcd)) < 0)
 		return false; /* aux transfer failed */
 
 	DRM_DEBUG_KMS("DPCD: %*ph\n", (int) sizeof(intel_dp->dpcd), intel_dp->dpcd);
 
-	if (intel_dp->dpcd[DP_DPCD_REV] == 0)
-		return false; /* DPCD not present */
+	return intel_dp->dpcd[DP_DPCD_REV] != 0;
+}
 
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT,
-			     &intel_dp->sink_count, 1) < 0)
-		return false;
+static bool
+intel_edp_init_dpcd(struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *dev_priv =
+		to_i915(dp_to_dig_port(intel_dp)->base.base.dev);
 
-	/*
-	 * Sink count can change between short pulse hpd hence
-	 * a member variable in intel_dp will track any changes
-	 * between short pulse interrupts.
-	 */
-	intel_dp->sink_count = DP_GET_SINK_COUNT(intel_dp->sink_count);
+	/* this function is meant to be called only once */
+	WARN_ON(intel_dp->dpcd[DP_DPCD_REV] != 0);
 
-	/*
-	 * SINK_COUNT == 0 and DOWNSTREAM_PORT_PRESENT == 1 implies that
-	 * a dongle is present but no display. Unless we require to know
-	 * if a dongle is present or not, we don't need to update
-	 * downstream port information. So, an early return here saves
-	 * time from performing other operations which are not required.
-	 */
-	if (!is_edp(intel_dp) && !intel_dp->sink_count)
+	if (!intel_dp_read_dpcd(intel_dp))
 		return false;
 
-	/* Check if the panel supports PSR */
-	memset(intel_dp->psr_dpcd, 0, sizeof(intel_dp->psr_dpcd));
-	if (is_edp(intel_dp)) {
-		drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT,
-				 intel_dp->psr_dpcd,
-				 sizeof(intel_dp->psr_dpcd));
-		if (intel_dp->psr_dpcd[0] & DP_PSR_IS_SUPPORTED) {
-			dev_priv->psr.sink_support = true;
-			DRM_DEBUG_KMS("Detected EDP PSR Panel.\n");
-		}
+	if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
+		dev_priv->no_aux_handshake = intel_dp->dpcd[DP_MAX_DOWNSPREAD] &
+			DP_NO_AUX_HANDSHAKE_LINK_TRAINING;
 
-		if (INTEL_INFO(dev)->gen >= 9 &&
-			(intel_dp->psr_dpcd[0] & DP_PSR2_IS_SUPPORTED)) {
-			uint8_t frame_sync_cap;
-
-			dev_priv->psr.sink_support = true;
-			drm_dp_dpcd_read(&intel_dp->aux,
-					 DP_SINK_DEVICE_AUX_FRAME_SYNC_CAP,
-					 &frame_sync_cap, 1);
-			dev_priv->psr.aux_frame_sync = frame_sync_cap ? true : false;
-			/* PSR2 needs frame sync as well */
-			dev_priv->psr.psr2_support = dev_priv->psr.aux_frame_sync;
-			DRM_DEBUG_KMS("PSR2 %s on sink",
-				dev_priv->psr.psr2_support ? "supported" : "not supported");
-		}
-
-		/* Read the eDP Display control capabilities registers */
-		memset(intel_dp->edp_dpcd, 0, sizeof(intel_dp->edp_dpcd));
-		if ((intel_dp->dpcd[DP_EDP_CONFIGURATION_CAP] & DP_DPCD_DISPLAY_CONTROL_CAPABLE) &&
-				(drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV,
-						intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd)) ==
-								sizeof(intel_dp->edp_dpcd)))
-			DRM_DEBUG_KMS("EDP DPCD : %*ph\n", (int) sizeof(intel_dp->edp_dpcd),
-					intel_dp->edp_dpcd);
-	}
-
-	DRM_DEBUG_KMS("Display Port TPS3 support: source %s, sink %s\n",
-		      yesno(intel_dp_source_supports_hbr2(intel_dp)),
-		      yesno(drm_dp_tps3_supported(intel_dp->dpcd)));
+	/* Check if the panel supports PSR */
+	drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT,
+			 intel_dp->psr_dpcd,
+			 sizeof(intel_dp->psr_dpcd));
+	if (intel_dp->psr_dpcd[0] & DP_PSR_IS_SUPPORTED) {
+		dev_priv->psr.sink_support = true;
+		DRM_DEBUG_KMS("Detected EDP PSR Panel.\n");
+	}
+
+	if (INTEL_GEN(dev_priv) >= 9 &&
+	    (intel_dp->psr_dpcd[0] & DP_PSR2_IS_SUPPORTED)) {
+		uint8_t frame_sync_cap;
+
+		dev_priv->psr.sink_support = true;
+		drm_dp_dpcd_read(&intel_dp->aux,
+				 DP_SINK_DEVICE_AUX_FRAME_SYNC_CAP,
+				 &frame_sync_cap, 1);
+		dev_priv->psr.aux_frame_sync = frame_sync_cap ? true : false;
+		/* PSR2 needs frame sync as well */
+		dev_priv->psr.psr2_support = dev_priv->psr.aux_frame_sync;
+		DRM_DEBUG_KMS("PSR2 %s on sink",
+			      dev_priv->psr.psr2_support ? "supported" : "not supported");
+	}
+
+	/* Read the eDP Display control capabilities registers */
+	if ((intel_dp->dpcd[DP_EDP_CONFIGURATION_CAP] & DP_DPCD_DISPLAY_CONTROL_CAPABLE) &&
+	    drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV,
+			     intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd) ==
+			     sizeof(intel_dp->edp_dpcd)))
+		DRM_DEBUG_KMS("EDP DPCD : %*ph\n", (int) sizeof(intel_dp->edp_dpcd),
+			      intel_dp->edp_dpcd);
 
 	/* Intermediate frequency support */
-	if (is_edp(intel_dp) && (intel_dp->edp_dpcd[0] >= 0x03)) { /* eDp v1.4 or higher */
+	if (intel_dp->edp_dpcd[0] >= 0x03) { /* eDp v1.4 or higher */
 		__le16 sink_rates[DP_MAX_SUPPORTED_RATES];
 		int i;
 
@@ -3491,7 +3576,36 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 		intel_dp->num_sink_rates = i;
 	}
 
-	intel_dp_print_rates(intel_dp);
+	return true;
+}
+
+
+static bool
+intel_dp_get_dpcd(struct intel_dp *intel_dp)
+{
+	if (!intel_dp_read_dpcd(intel_dp))
+		return false;
+
+	if (drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT,
+			     &intel_dp->sink_count, 1) < 0)
+		return false;
+
+	/*
+	 * Sink count can change between short pulse hpd hence
+	 * a member variable in intel_dp will track any changes
+	 * between short pulse interrupts.
+	 */
+	intel_dp->sink_count = DP_GET_SINK_COUNT(intel_dp->sink_count);
+
+	/*
+	 * SINK_COUNT == 0 and DOWNSTREAM_PORT_PRESENT == 1 implies that
+	 * a dongle is present but no display. Unless we require to know
+	 * if a dongle is present or not, we don't need to update
+	 * downstream port information. So, an early return here saves
+	 * time from performing other operations which are not required.
+	 */
+	if (!is_edp(intel_dp) && !intel_dp->sink_count)
+		return false;
 
 	if (!(intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] &
 	      DP_DWN_STRM_PORT_PRESENT))
@@ -3526,7 +3640,7 @@ intel_dp_probe_oui(struct intel_dp *intel_dp)
 }
 
 static bool
-intel_dp_probe_mst(struct intel_dp *intel_dp)
+intel_dp_can_mst(struct intel_dp *intel_dp)
 {
 	u8 buf[1];
 
@@ -3539,18 +3653,30 @@ intel_dp_probe_mst(struct intel_dp *intel_dp)
 	if (intel_dp->dpcd[DP_DPCD_REV] < 0x12)
 		return false;
 
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_MSTM_CAP, buf, 1)) {
-		if (buf[0] & DP_MST_CAP) {
-			DRM_DEBUG_KMS("Sink is MST capable\n");
-			intel_dp->is_mst = true;
-		} else {
-			DRM_DEBUG_KMS("Sink is not MST capable\n");
-			intel_dp->is_mst = false;
-		}
-	}
+	if (drm_dp_dpcd_read(&intel_dp->aux, DP_MSTM_CAP, buf, 1) != 1)
+		return false;
+
+	return buf[0] & DP_MST_CAP;
+}
+
+static void
+intel_dp_configure_mst(struct intel_dp *intel_dp)
+{
+	if (!i915.enable_dp_mst)
+		return;
+
+	if (!intel_dp->can_mst)
+		return;
 
-	drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst);
-	return intel_dp->is_mst;
+	intel_dp->is_mst = intel_dp_can_mst(intel_dp);
+
+	if (intel_dp->is_mst)
+		DRM_DEBUG_KMS("Sink is MST capable\n");
+	else
+		DRM_DEBUG_KMS("Sink is not MST capable\n");
+
+	drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr,
+					intel_dp->is_mst);
 }
 
 static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
@@ -3909,7 +4035,7 @@ static bool
 intel_dp_short_pulse(struct intel_dp *intel_dp)
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	u8 sink_irq_vector;
+	u8 sink_irq_vector = 0;
 	u8 old_sink_count = intel_dp->sink_count;
 	bool ret;
 
@@ -3936,7 +4062,8 @@ intel_dp_short_pulse(struct intel_dp *intel_dp)
 
 	/* Try to read the source of the interrupt */
 	if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 &&
-	    intel_dp_get_sink_irq(intel_dp, &sink_irq_vector)) {
+	    intel_dp_get_sink_irq(intel_dp, &sink_irq_vector) &&
+	    sink_irq_vector != 0) {
 		/* Clear interrupt source */
 		drm_dp_dpcd_writeb(&intel_dp->aux,
 				   DP_DEVICE_SERVICE_IRQ_VECTOR,
@@ -3980,6 +4107,9 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
 		connector_status_connected : connector_status_disconnected;
 	}
 
+	if (intel_dp_can_mst(intel_dp))
+		return connector_status_connected;
+
 	/* If no HPD, poke DDC gently */
 	if (drm_probe_ddc(&intel_dp->aux.ddc))
 		return connector_status_connected;
@@ -4148,7 +4278,7 @@ static bool bxt_digital_port_connected(struct drm_i915_private *dev_priv,
  *
  * Return %true if @port is connected, %false otherwise.
  */
-bool intel_digital_port_connected(struct drm_i915_private *dev_priv,
+static bool intel_digital_port_connected(struct drm_i915_private *dev_priv,
 					 struct intel_digital_port *port)
 {
 	if (HAS_PCH_IBX(dev_priv))
@@ -4207,7 +4337,7 @@ intel_dp_unset_edid(struct intel_dp *intel_dp)
 	intel_dp->has_audio = false;
 }
 
-static void
+static enum drm_connector_status
 intel_dp_long_pulse(struct intel_connector *intel_connector)
 {
 	struct drm_connector *connector = &intel_connector->base;
@@ -4217,8 +4347,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 	struct drm_device *dev = connector->dev;
 	enum drm_connector_status status;
 	enum intel_display_power_domain power_domain;
-	bool ret;
-	u8 sink_irq_vector;
+	u8 sink_irq_vector = 0;
 
 	power_domain = intel_display_port_aux_power_domain(intel_encoder);
 	intel_display_power_get(to_i915(dev), power_domain);
@@ -4232,7 +4361,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 	else
 		status = connector_status_disconnected;
 
-	if (status != connector_status_connected) {
+	if (status == connector_status_disconnected) {
 		intel_dp->compliance_test_active = 0;
 		intel_dp->compliance_test_type = 0;
 		intel_dp->compliance_test_data = 0;
@@ -4252,10 +4381,20 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 	if (intel_encoder->type != INTEL_OUTPUT_EDP)
 		intel_encoder->type = INTEL_OUTPUT_DP;
 
+	DRM_DEBUG_KMS("Display Port TPS3 support: source %s, sink %s\n",
+		      yesno(intel_dp_source_supports_hbr2(intel_dp)),
+		      yesno(drm_dp_tps3_supported(intel_dp->dpcd)));
+
+	intel_dp_print_rates(intel_dp);
+
 	intel_dp_probe_oui(intel_dp);
 
-	ret = intel_dp_probe_mst(intel_dp);
-	if (ret) {
+	intel_dp_print_hw_revision(intel_dp);
+	intel_dp_print_sw_revision(intel_dp);
+
+	intel_dp_configure_mst(intel_dp);
+
+	if (intel_dp->is_mst) {
 		/*
 		 * If we are in MST mode then this connector
 		 * won't appear connected or have anything
@@ -4284,13 +4423,14 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 	intel_dp->aux.i2c_defer_count = 0;
 
 	intel_dp_set_edid(intel_dp);
-
-	status = connector_status_connected;
+	if (is_edp(intel_dp) || intel_connector->detect_edid)
+		status = connector_status_connected;
 	intel_dp->detect_done = true;
 
 	/* Try to read the source of the interrupt */
 	if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 &&
-	    intel_dp_get_sink_irq(intel_dp, &sink_irq_vector)) {
+	    intel_dp_get_sink_irq(intel_dp, &sink_irq_vector) &&
+	    sink_irq_vector != 0) {
 		/* Clear interrupt source */
 		drm_dp_dpcd_writeb(&intel_dp->aux,
 				   DP_DEVICE_SERVICE_IRQ_VECTOR,
@@ -4303,12 +4443,11 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 	}
 
 out:
-	if ((status != connector_status_connected) &&
-	    (intel_dp->is_mst == false))
+	if (status != connector_status_connected && !intel_dp->is_mst)
 		intel_dp_unset_edid(intel_dp);
 
 	intel_display_power_put(to_i915(dev), power_domain);
-	return;
+	return status;
 }
 
 static enum drm_connector_status
@@ -4317,7 +4456,7 @@ intel_dp_detect(struct drm_connector *connector, bool force)
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct intel_encoder *intel_encoder = &intel_dig_port->base;
-	struct intel_connector *intel_connector = to_intel_connector(connector);
+	enum drm_connector_status status = connector->status;
 
 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
 		      connector->base.id, connector->name);
@@ -4332,14 +4471,11 @@ intel_dp_detect(struct drm_connector *connector, bool force)
 
 	/* If full detect is not performed yet, do a full detect */
 	if (!intel_dp->detect_done)
-		intel_dp_long_pulse(intel_dp->attached_connector);
+		status = intel_dp_long_pulse(intel_dp->attached_connector);
 
 	intel_dp->detect_done = false;
 
-	if (is_edp(intel_dp) || intel_connector->detect_edid)
-		return connector_status_connected;
-	else
-		return connector_status_disconnected;
+	return status;
 }
 
 static void
@@ -4630,13 +4766,8 @@ void intel_dp_encoder_reset(struct drm_encoder *encoder)
 
 	pps_lock(intel_dp);
 
-	/*
-	 * Read out the current power sequencer assignment,
-	 * in case the BIOS did something with it.
-	 */
-	if (IS_VALLEYVIEW(encoder->dev) || IS_CHERRYVIEW(encoder->dev))
-		vlv_initial_power_sequencer_setup(intel_dp);
-
+	/* Reinit the power sequencer, in case BIOS did something with it. */
+	intel_dp_pps_init(encoder->dev, intel_dp);
 	intel_edp_panel_vdd_sanitize(intel_dp);
 
 	pps_unlock(intel_dp);
@@ -4696,36 +4827,34 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
 		      port_name(intel_dig_port->port),
 		      long_hpd ? "long" : "short");
 
+	if (long_hpd) {
+		intel_dp->detect_done = false;
+		return IRQ_NONE;
+	}
+
 	power_domain = intel_display_port_aux_power_domain(intel_encoder);
 	intel_display_power_get(dev_priv, power_domain);
 
-	if (long_hpd) {
-		intel_dp_long_pulse(intel_dp->attached_connector);
-		if (intel_dp->is_mst)
-			ret = IRQ_HANDLED;
-		goto put_power;
-
-	} else {
-		if (intel_dp->is_mst) {
-			if (intel_dp_check_mst_status(intel_dp) == -EINVAL) {
-				/*
-				 * If we were in MST mode, and device is not
-				 * there, get out of MST mode
-				 */
-				DRM_DEBUG_KMS("MST device may have disappeared %d vs %d\n",
-					      intel_dp->is_mst, intel_dp->mst_mgr.mst_state);
-				intel_dp->is_mst = false;
-				drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr,
-								intel_dp->is_mst);
-				goto put_power;
-			}
+	if (intel_dp->is_mst) {
+		if (intel_dp_check_mst_status(intel_dp) == -EINVAL) {
+			/*
+			 * If we were in MST mode, and device is not
+			 * there, get out of MST mode
+			 */
+			DRM_DEBUG_KMS("MST device may have disappeared %d vs %d\n",
+				      intel_dp->is_mst, intel_dp->mst_mgr.mst_state);
+			intel_dp->is_mst = false;
+			drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr,
+							intel_dp->is_mst);
+			intel_dp->detect_done = false;
+			goto put_power;
 		}
+	}
 
-		if (!intel_dp->is_mst) {
-			if (!intel_dp_short_pulse(intel_dp)) {
-				intel_dp_long_pulse(intel_dp->attached_connector);
-				goto put_power;
-			}
+	if (!intel_dp->is_mst) {
+		if (!intel_dp_short_pulse(intel_dp)) {
+			intel_dp->detect_done = false;
+			goto put_power;
 		}
 	}
 
@@ -4984,9 +5113,21 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 		      I915_READ(regs.pp_div));
 }
 
+static void intel_dp_pps_init(struct drm_device *dev,
+			      struct intel_dp *intel_dp)
+{
+	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+		vlv_initial_power_sequencer_setup(intel_dp);
+	} else {
+		intel_dp_init_panel_power_sequencer(dev, intel_dp);
+		intel_dp_init_panel_power_sequencer_registers(dev, intel_dp);
+	}
+}
+
 /**
  * intel_dp_set_drrs_state - program registers for RR switch to take effect
- * @dev: DRM device
+ * @dev_priv: i915 device
+ * @crtc_state: a pointer to the active intel_crtc_state
  * @refresh_rate: RR to be programmed
  *
  * This function gets called when refresh rate (RR) has to be changed from
@@ -4996,14 +5137,14 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
  *
  * The caller of this function needs to take a lock on dev_priv->drrs.
  */
-static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
+static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv,
+				    struct intel_crtc_state *crtc_state,
+				    int refresh_rate)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_encoder *encoder;
 	struct intel_digital_port *dig_port = NULL;
 	struct intel_dp *intel_dp = dev_priv->drrs.dp;
-	struct intel_crtc_state *config = NULL;
-	struct intel_crtc *intel_crtc = NULL;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
 	enum drrs_refresh_rate_type index = DRRS_HIGH_RR;
 
 	if (refresh_rate <= 0) {
@@ -5030,8 +5171,6 @@ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
 		return;
 	}
 
-	config = intel_crtc->config;
-
 	if (dev_priv->drrs.type < SEAMLESS_DRRS_SUPPORT) {
 		DRM_DEBUG_KMS("Only Seamless DRRS supported.\n");
 		return;
@@ -5047,12 +5186,12 @@ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
 		return;
 	}
 
-	if (!intel_crtc->active) {
+	if (!crtc_state->base.active) {
 		DRM_DEBUG_KMS("eDP encoder disabled. CRTC not Active\n");
 		return;
 	}
 
-	if (INTEL_INFO(dev)->gen >= 8 && !IS_CHERRYVIEW(dev)) {
+	if (INTEL_GEN(dev_priv) >= 8 && !IS_CHERRYVIEW(dev_priv)) {
 		switch (index) {
 		case DRRS_HIGH_RR:
 			intel_dp_set_m_n(intel_crtc, M1_N1);
@@ -5064,18 +5203,18 @@ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
 		default:
 			DRM_ERROR("Unsupported refreshrate type\n");
 		}
-	} else if (INTEL_INFO(dev)->gen > 6) {
-		i915_reg_t reg = PIPECONF(intel_crtc->config->cpu_transcoder);
+	} else if (INTEL_GEN(dev_priv) > 6) {
+		i915_reg_t reg = PIPECONF(crtc_state->cpu_transcoder);
 		u32 val;
 
 		val = I915_READ(reg);
 		if (index > DRRS_HIGH_RR) {
-			if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+			if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 				val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV;
 			else
 				val |= PIPECONF_EDP_RR_MODE_SWITCH;
 		} else {
-			if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+			if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 				val &= ~PIPECONF_EDP_RR_MODE_SWITCH_VLV;
 			else
 				val &= ~PIPECONF_EDP_RR_MODE_SWITCH;
@@ -5091,18 +5230,17 @@ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
 /**
  * intel_edp_drrs_enable - init drrs struct if supported
  * @intel_dp: DP struct
+ * @crtc_state: A pointer to the active crtc state.
  *
  * Initializes frontbuffer_bits and drrs.dp
  */
-void intel_edp_drrs_enable(struct intel_dp *intel_dp)
+void intel_edp_drrs_enable(struct intel_dp *intel_dp,
+			   struct intel_crtc_state *crtc_state)
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_crtc *crtc = dig_port->base.base.crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
-	if (!intel_crtc->config->has_drrs) {
+	if (!crtc_state->has_drrs) {
 		DRM_DEBUG_KMS("Panel doesn't support DRRS\n");
 		return;
 	}
@@ -5124,17 +5262,16 @@ unlock:
 /**
  * intel_edp_drrs_disable - Disable DRRS
  * @intel_dp: DP struct
+ * @old_crtc_state: Pointer to old crtc_state.
  *
  */
-void intel_edp_drrs_disable(struct intel_dp *intel_dp)
+void intel_edp_drrs_disable(struct intel_dp *intel_dp,
+			    struct intel_crtc_state *old_crtc_state)
 {
 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_crtc *crtc = dig_port->base.base.crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
-	if (!intel_crtc->config->has_drrs)
+	if (!old_crtc_state->has_drrs)
 		return;
 
 	mutex_lock(&dev_priv->drrs.mutex);
@@ -5144,9 +5281,8 @@ void intel_edp_drrs_disable(struct intel_dp *intel_dp)
 	}
 
 	if (dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
-		intel_dp_set_drrs_state(&dev_priv->drm,
-					intel_dp->attached_connector->panel.
-					fixed_mode->vrefresh);
+		intel_dp_set_drrs_state(dev_priv, old_crtc_state,
+			intel_dp->attached_connector->panel.fixed_mode->vrefresh);
 
 	dev_priv->drrs.dp = NULL;
 	mutex_unlock(&dev_priv->drrs.mutex);
@@ -5175,10 +5311,12 @@ static void intel_edp_drrs_downclock_work(struct work_struct *work)
 	if (dev_priv->drrs.busy_frontbuffer_bits)
 		goto unlock;
 
-	if (dev_priv->drrs.refresh_rate_type != DRRS_LOW_RR)
-		intel_dp_set_drrs_state(&dev_priv->drm,
-					intel_dp->attached_connector->panel.
-					downclock_mode->vrefresh);
+	if (dev_priv->drrs.refresh_rate_type != DRRS_LOW_RR) {
+		struct drm_crtc *crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
+
+		intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config,
+			intel_dp->attached_connector->panel.downclock_mode->vrefresh);
+	}
 
 unlock:
 	mutex_unlock(&dev_priv->drrs.mutex);
@@ -5186,7 +5324,7 @@ unlock:
 
 /**
  * intel_edp_drrs_invalidate - Disable Idleness DRRS
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
  *
  * This function gets called everytime rendering on the given planes start.
@@ -5194,10 +5332,9 @@ unlock:
  *
  * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits.
  */
-void intel_edp_drrs_invalidate(struct drm_device *dev,
-		unsigned frontbuffer_bits)
+void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv,
+			       unsigned int frontbuffer_bits)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_crtc *crtc;
 	enum pipe pipe;
 
@@ -5220,16 +5357,15 @@ void intel_edp_drrs_invalidate(struct drm_device *dev,
 
 	/* invalidate means busy screen hence upclock */
 	if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
-		intel_dp_set_drrs_state(&dev_priv->drm,
-					dev_priv->drrs.dp->attached_connector->panel.
-					fixed_mode->vrefresh);
+		intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config,
+			dev_priv->drrs.dp->attached_connector->panel.fixed_mode->vrefresh);
 
 	mutex_unlock(&dev_priv->drrs.mutex);
 }
 
 /**
  * intel_edp_drrs_flush - Restart Idleness DRRS
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
  *
  * This function gets called every time rendering on the given planes has
@@ -5239,10 +5375,9 @@ void intel_edp_drrs_invalidate(struct drm_device *dev,
  *
  * Dirty frontbuffers relevant to DRRS are tracked in busy_frontbuffer_bits.
  */
-void intel_edp_drrs_flush(struct drm_device *dev,
-		unsigned frontbuffer_bits)
+void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
+			  unsigned int frontbuffer_bits)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_crtc *crtc;
 	enum pipe pipe;
 
@@ -5265,9 +5400,8 @@ void intel_edp_drrs_flush(struct drm_device *dev,
 
 	/* flush means busy screen hence upclock */
 	if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
-		intel_dp_set_drrs_state(&dev_priv->drm,
-					dev_priv->drrs.dp->attached_connector->panel.
-					fixed_mode->vrefresh);
+		intel_dp_set_drrs_state(dev_priv, to_intel_crtc(crtc)->config,
+				dev_priv->drrs.dp->attached_connector->panel.fixed_mode->vrefresh);
 
 	/*
 	 * flush also means no more activity hence schedule downclock, if all
@@ -5400,27 +5534,15 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	pps_lock(intel_dp);
 
 	intel_dp_init_panel_power_timestamps(intel_dp);
-
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
-		vlv_initial_power_sequencer_setup(intel_dp);
-	} else {
-		intel_dp_init_panel_power_sequencer(dev, intel_dp);
-		intel_dp_init_panel_power_sequencer_registers(dev, intel_dp);
-	}
-
+	intel_dp_pps_init(dev, intel_dp);
 	intel_edp_panel_vdd_sanitize(intel_dp);
 
 	pps_unlock(intel_dp);
 
 	/* Cache DPCD and EDID for edp. */
-	has_dpcd = intel_dp_get_dpcd(intel_dp);
+	has_dpcd = intel_edp_init_dpcd(intel_dp);
 
-	if (has_dpcd) {
-		if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
-			dev_priv->no_aux_handshake =
-				intel_dp->dpcd[DP_MAX_DOWNSPREAD] &
-				DP_NO_AUX_HANDSHAKE_LINK_TRAINING;
-	} else {
+	if (!has_dpcd) {
 		/* if this fails, presume the device is a ghost */
 		DRM_INFO("failed to retrieve link info, disabling eDP\n");
 		goto out_vdd_off;
@@ -5576,7 +5698,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	connector->interlace_allowed = true;
 	connector->doublescan_allowed = 0;
 
-	intel_dp_aux_init(intel_dp, intel_connector);
+	intel_dp_aux_init(intel_dp);
 
 	INIT_DELAYED_WORK(&intel_dp->panel_vdd_work,
 			  edp_panel_vdd_work);
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
index 60fb39cd220b..c438b02184cb 100644
--- a/drivers/gpu/drm/i915/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -24,6 +24,15 @@
 #include "intel_drv.h"
 
 static void
+intel_dp_dump_link_status(const uint8_t link_status[DP_LINK_STATUS_SIZE])
+{
+
+	DRM_DEBUG_KMS("ln0_1:0x%x ln2_3:0x%x align:0x%x sink:0x%x adj_req0_1:0x%x adj_req2_3:0x%x",
+		      link_status[0], link_status[1], link_status[2],
+		      link_status[3], link_status[4], link_status[5]);
+}
+
+static void
 intel_get_adjust_train(struct intel_dp *intel_dp,
 		       const uint8_t link_status[DP_LINK_STATUS_SIZE])
 {
@@ -103,13 +112,24 @@ intel_dp_update_link_train(struct intel_dp *intel_dp)
 	return ret == intel_dp->lane_count;
 }
 
+static bool intel_dp_link_max_vswing_reached(struct intel_dp *intel_dp)
+{
+	int lane;
+
+	for (lane = 0; lane < intel_dp->lane_count; lane++)
+		if ((intel_dp->train_set[lane] &
+		     DP_TRAIN_MAX_SWING_REACHED) == 0)
+			return false;
+
+	return true;
+}
+
 /* Enable corresponding port and start training pattern 1 */
-static void
+static bool
 intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp)
 {
-	int i;
 	uint8_t voltage;
-	int voltage_tries, loop_tries;
+	int voltage_tries, max_vswing_tries;
 	uint8_t link_config[2];
 	uint8_t link_bw, rate_select;
 
@@ -125,6 +145,7 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp)
 	if (drm_dp_enhanced_frame_cap(intel_dp->dpcd))
 		link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
 	drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2);
+
 	if (intel_dp->num_sink_rates)
 		drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_RATE_SET,
 				  &rate_select, 1);
@@ -140,60 +161,54 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp)
 				       DP_TRAINING_PATTERN_1 |
 				       DP_LINK_SCRAMBLING_DISABLE)) {
 		DRM_ERROR("failed to enable link training\n");
-		return;
+		return false;
 	}
 
-	voltage = 0xff;
-	voltage_tries = 0;
-	loop_tries = 0;
+	voltage_tries = 1;
+	max_vswing_tries = 0;
 	for (;;) {
 		uint8_t link_status[DP_LINK_STATUS_SIZE];
 
 		drm_dp_link_train_clock_recovery_delay(intel_dp->dpcd);
+
 		if (!intel_dp_get_link_status(intel_dp, link_status)) {
 			DRM_ERROR("failed to get link status\n");
-			break;
+			return false;
 		}
 
 		if (drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) {
 			DRM_DEBUG_KMS("clock recovery OK\n");
-			break;
+			return true;
 		}
 
-		/* Check to see if we've tried the max voltage */
-		for (i = 0; i < intel_dp->lane_count; i++)
-			if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
-				break;
-		if (i == intel_dp->lane_count) {
-			++loop_tries;
-			if (loop_tries == 5) {
-				DRM_ERROR("too many full retries, give up\n");
-				break;
-			}
-			intel_dp_reset_link_train(intel_dp,
-						  DP_TRAINING_PATTERN_1 |
-						  DP_LINK_SCRAMBLING_DISABLE);
-			voltage_tries = 0;
-			continue;
+		if (voltage_tries == 5) {
+			DRM_DEBUG_KMS("Same voltage tried 5 times\n");
+			return false;
+		}
+
+		if (max_vswing_tries == 1) {
+			DRM_DEBUG_KMS("Max Voltage Swing reached\n");
+			return false;
 		}
 
-		/* Check to see if we've tried the same voltage 5 times */
-		if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
-			++voltage_tries;
-			if (voltage_tries == 5) {
-				DRM_ERROR("too many voltage retries, give up\n");
-				break;
-			}
-		} else
-			voltage_tries = 0;
 		voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
 
 		/* Update training set as requested by target */
 		intel_get_adjust_train(intel_dp, link_status);
 		if (!intel_dp_update_link_train(intel_dp)) {
 			DRM_ERROR("failed to update link training\n");
-			break;
+			return false;
 		}
+
+		if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) ==
+		    voltage)
+			++voltage_tries;
+		else
+			voltage_tries = 1;
+
+		if (intel_dp_link_max_vswing_reached(intel_dp))
+			++max_vswing_tries;
+
 	}
 }
 
@@ -229,12 +244,12 @@ static u32 intel_dp_training_pattern(struct intel_dp *intel_dp)
 	return training_pattern;
 }
 
-static void
+static bool
 intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
 {
-	bool channel_eq = false;
-	int tries, cr_tries;
+	int tries;
 	u32 training_pattern;
+	uint8_t link_status[DP_LINK_STATUS_SIZE];
 
 	training_pattern = intel_dp_training_pattern(intel_dp);
 
@@ -243,19 +258,11 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
 				     training_pattern |
 				     DP_LINK_SCRAMBLING_DISABLE)) {
 		DRM_ERROR("failed to start channel equalization\n");
-		return;
+		return false;
 	}
 
-	tries = 0;
-	cr_tries = 0;
-	channel_eq = false;
-	for (;;) {
-		uint8_t link_status[DP_LINK_STATUS_SIZE];
-
-		if (cr_tries > 5) {
-			DRM_ERROR("failed to train DP, aborting\n");
-			break;
-		}
+	intel_dp->channel_eq_status = false;
+	for (tries = 0; tries < 5; tries++) {
 
 		drm_dp_link_train_channel_eq_delay(intel_dp->dpcd);
 		if (!intel_dp_get_link_status(intel_dp, link_status)) {
@@ -266,44 +273,38 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
 		/* Make sure clock is still ok */
 		if (!drm_dp_clock_recovery_ok(link_status,
 					      intel_dp->lane_count)) {
-			intel_dp_link_training_clock_recovery(intel_dp);
-			intel_dp_set_link_train(intel_dp,
-						training_pattern |
-						DP_LINK_SCRAMBLING_DISABLE);
-			cr_tries++;
-			continue;
+			intel_dp_dump_link_status(link_status);
+			DRM_DEBUG_KMS("Clock recovery check failed, cannot "
+				      "continue channel equalization\n");
+			break;
 		}
 
 		if (drm_dp_channel_eq_ok(link_status,
 					 intel_dp->lane_count)) {
-			channel_eq = true;
+			intel_dp->channel_eq_status = true;
+			DRM_DEBUG_KMS("Channel EQ done. DP Training "
+				      "successful\n");
 			break;
 		}
 
-		/* Try 5 times, then try clock recovery if that fails */
-		if (tries > 5) {
-			intel_dp_link_training_clock_recovery(intel_dp);
-			intel_dp_set_link_train(intel_dp,
-						training_pattern |
-						DP_LINK_SCRAMBLING_DISABLE);
-			tries = 0;
-			cr_tries++;
-			continue;
-		}
-
 		/* Update training set as requested by target */
 		intel_get_adjust_train(intel_dp, link_status);
 		if (!intel_dp_update_link_train(intel_dp)) {
 			DRM_ERROR("failed to update link training\n");
 			break;
 		}
-		++tries;
+	}
+
+	/* Try 5 times, else fail and try at lower BW */
+	if (tries == 5) {
+		intel_dp_dump_link_status(link_status);
+		DRM_DEBUG_KMS("Channel equalization failed 5 times\n");
 	}
 
 	intel_dp_set_idle_link_train(intel_dp);
 
-	if (channel_eq)
-		DRM_DEBUG_KMS("Channel EQ done. DP Training successful\n");
+	return intel_dp->channel_eq_status;
+
 }
 
 void intel_dp_stop_link_train(struct intel_dp *intel_dp)
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 68a005d729e9..54a9d7610d8f 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -31,18 +31,16 @@
 #include <drm/drm_edid.h>
 
 static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
-					struct intel_crtc_state *pipe_config)
+					struct intel_crtc_state *pipe_config,
+					struct drm_connector_state *conn_state)
 {
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
 	struct drm_atomic_state *state;
-	int bpp, i;
+	int bpp;
 	int lane_count, slots;
 	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
-	struct drm_connector *drm_connector;
-	struct intel_connector *connector, *found = NULL;
-	struct drm_connector_state *connector_state;
 	int mst_pbn;
 
 	pipe_config->dp_encoder_is_mst = true;
@@ -54,7 +52,6 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 	 */
 	lane_count = drm_dp_max_lane_count(intel_dp->dpcd);
 
-
 	pipe_config->lane_count = lane_count;
 
 	pipe_config->pipe_bpp = 24;
@@ -62,20 +59,6 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 
 	state = pipe_config->base.state;
 
-	for_each_connector_in_state(state, drm_connector, connector_state, i) {
-		connector = to_intel_connector(drm_connector);
-
-		if (connector_state->best_encoder == &encoder->base) {
-			found = connector;
-			break;
-		}
-	}
-
-	if (!found) {
-		DRM_ERROR("can't find connector\n");
-		return false;
-	}
-
 	mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp);
 
 	pipe_config->pbn = mst_pbn;
@@ -92,16 +75,20 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 
 }
 
-static void intel_mst_disable_dp(struct intel_encoder *encoder)
+static void intel_mst_disable_dp(struct intel_encoder *encoder,
+				 struct intel_crtc_state *old_crtc_state,
+				 struct drm_connector_state *old_conn_state)
 {
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
+	struct intel_connector *connector =
+		to_intel_connector(old_conn_state->connector);
 	int ret;
 
 	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
 
-	drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, intel_mst->connector->port);
+	drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, connector->port);
 
 	ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr);
 	if (ret) {
@@ -109,11 +96,15 @@ static void intel_mst_disable_dp(struct intel_encoder *encoder)
 	}
 }
 
-static void intel_mst_post_disable_dp(struct intel_encoder *encoder)
+static void intel_mst_post_disable_dp(struct intel_encoder *encoder,
+				      struct intel_crtc_state *old_crtc_state,
+				      struct drm_connector_state *old_conn_state)
 {
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
+	struct intel_connector *connector =
+		to_intel_connector(old_conn_state->connector);
 
 	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
 
@@ -122,59 +113,51 @@ static void intel_mst_post_disable_dp(struct intel_encoder *encoder)
 	/* and this can also fail */
 	drm_dp_update_payload_part2(&intel_dp->mst_mgr);
 
-	drm_dp_mst_deallocate_vcpi(&intel_dp->mst_mgr, intel_mst->connector->port);
+	drm_dp_mst_deallocate_vcpi(&intel_dp->mst_mgr, connector->port);
 
 	intel_dp->active_mst_links--;
 
 	intel_mst->connector = NULL;
 	if (intel_dp->active_mst_links == 0) {
-		intel_dig_port->base.post_disable(&intel_dig_port->base);
+		intel_dig_port->base.post_disable(&intel_dig_port->base,
+						  NULL, NULL);
+
 		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
 	}
 }
 
-static void intel_mst_pre_enable_dp(struct intel_encoder *encoder)
+static void intel_mst_pre_enable_dp(struct intel_encoder *encoder,
+				    struct intel_crtc_state *pipe_config,
+				    struct drm_connector_state *conn_state)
 {
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum port port = intel_dig_port->port;
+	struct intel_connector *connector =
+		to_intel_connector(conn_state->connector);
 	int ret;
 	uint32_t temp;
-	struct intel_connector *found = NULL, *connector;
 	int slots;
-	struct drm_crtc *crtc = encoder->base.crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-
-	for_each_intel_connector(dev, connector) {
-		if (connector->base.state->best_encoder == &encoder->base) {
-			found = connector;
-			break;
-		}
-	}
-
-	if (!found) {
-		DRM_ERROR("can't find connector\n");
-		return;
-	}
 
 	/* MST encoders are bound to a crtc, not to a connector,
 	 * force the mapping here for get_hw_state.
 	 */
-	found->encoder = encoder;
+	connector->encoder = encoder;
+	intel_mst->connector = connector;
 
 	DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
 
-	intel_mst->connector = found;
-
 	if (intel_dp->active_mst_links == 0) {
-		intel_prepare_ddi_buffer(&intel_dig_port->base);
-
-		intel_ddi_clk_select(&intel_dig_port->base, intel_crtc->config);
+		intel_ddi_clk_select(&intel_dig_port->base,
+				     pipe_config->shared_dpll);
 
-		intel_dp_set_link_params(intel_dp, intel_crtc->config);
+		intel_prepare_dp_ddi_buffers(&intel_dig_port->base);
+		intel_dp_set_link_params(intel_dp,
+					 pipe_config->port_clock,
+					 pipe_config->lane_count,
+					 true);
 
 		intel_ddi_init_dp_buf_reg(&intel_dig_port->base);
 
@@ -185,8 +168,8 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder)
 	}
 
 	ret = drm_dp_mst_allocate_vcpi(&intel_dp->mst_mgr,
-				       intel_mst->connector->port,
-				       intel_crtc->config->pbn, &slots);
+				       connector->port,
+				       pipe_config->pbn, &slots);
 	if (ret == false) {
 		DRM_ERROR("failed to allocate vcpi\n");
 		return;
@@ -200,13 +183,14 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder)
 	ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr);
 }
 
-static void intel_mst_enable_dp(struct intel_encoder *encoder)
+static void intel_mst_enable_dp(struct intel_encoder *encoder,
+				struct intel_crtc_state *pipe_config,
+				struct drm_connector_state *conn_state)
 {
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum port port = intel_dig_port->port;
 	int ret;
 
@@ -239,9 +223,8 @@ static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder,
 {
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
 	u32 temp, flags = 0;
 
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index 5c1f2d235ffa..1c59ca50c430 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -24,6 +24,44 @@
 #include "intel_drv.h"
 
 struct intel_shared_dpll *
+skl_find_link_pll(struct drm_i915_private *dev_priv, int clock)
+{
+	struct intel_shared_dpll *pll = NULL;
+	struct intel_dpll_hw_state dpll_hw_state;
+	enum intel_dpll_id i;
+	bool found = false;
+
+	if (!skl_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state))
+		return pll;
+
+	for (i = DPLL_ID_SKL_DPLL1; i <= DPLL_ID_SKL_DPLL3; i++) {
+		pll = &dev_priv->shared_dplls[i];
+
+		/* Only want to check enabled timings first */
+		if (pll->config.crtc_mask == 0)
+			continue;
+
+		if (memcmp(&dpll_hw_state, &pll->config.hw_state,
+			   sizeof(pll->config.hw_state)) == 0) {
+			found = true;
+			break;
+		}
+	}
+
+	/* Ok no matching timings, maybe there's a free one? */
+	for (i = DPLL_ID_SKL_DPLL1;
+	     ((found == false) && (i <= DPLL_ID_SKL_DPLL3)); i++) {
+		pll = &dev_priv->shared_dplls[i];
+		if (pll->config.crtc_mask == 0) {
+			pll->config.hw_state = dpll_hw_state;
+			break;
+		}
+	}
+
+	return pll;
+}
+
+struct intel_shared_dpll *
 intel_get_shared_dpll_by_id(struct drm_i915_private *dev_priv,
 			    enum intel_dpll_id id)
 {
@@ -452,26 +490,6 @@ static bool hsw_ddi_spll_get_hw_state(struct drm_i915_private *dev_priv,
 	return val & SPLL_PLL_ENABLE;
 }
 
-static uint32_t hsw_pll_to_ddi_pll_sel(struct intel_shared_dpll *pll)
-{
-	switch (pll->id) {
-	case DPLL_ID_WRPLL1:
-		return PORT_CLK_SEL_WRPLL1;
-	case DPLL_ID_WRPLL2:
-		return PORT_CLK_SEL_WRPLL2;
-	case DPLL_ID_SPLL:
-		return PORT_CLK_SEL_SPLL;
-	case DPLL_ID_LCPLL_810:
-		return PORT_CLK_SEL_LCPLL_810;
-	case DPLL_ID_LCPLL_1350:
-		return PORT_CLK_SEL_LCPLL_1350;
-	case DPLL_ID_LCPLL_2700:
-		return PORT_CLK_SEL_LCPLL_2700;
-	default:
-		return PORT_CLK_SEL_NONE;
-	}
-}
-
 #define LC_FREQ 2700
 #define LC_FREQ_2K U64_C(LC_FREQ * 2000)
 
@@ -687,11 +705,65 @@ hsw_ddi_calculate_wrpll(int clock /* in Hz */,
 	*r2_out = best.r2;
 }
 
+static struct intel_shared_dpll *hsw_ddi_hdmi_get_dpll(int clock,
+						       struct intel_crtc *crtc,
+						       struct intel_crtc_state *crtc_state)
+{
+	struct intel_shared_dpll *pll;
+	uint32_t val;
+	unsigned int p, n2, r2;
+
+	hsw_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p);
+
+	val = WRPLL_PLL_ENABLE | WRPLL_PLL_LCPLL |
+	      WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) |
+	      WRPLL_DIVIDER_POST(p);
+
+	crtc_state->dpll_hw_state.wrpll = val;
+
+	pll = intel_find_shared_dpll(crtc, crtc_state,
+				     DPLL_ID_WRPLL1, DPLL_ID_WRPLL2);
+
+	if (!pll)
+		return NULL;
+
+	return pll;
+}
+
+struct intel_shared_dpll *hsw_ddi_dp_get_dpll(struct intel_encoder *encoder,
+					      int clock)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_shared_dpll *pll;
+	enum intel_dpll_id pll_id;
+
+	switch (clock / 2) {
+	case 81000:
+		pll_id = DPLL_ID_LCPLL_810;
+		break;
+	case 135000:
+		pll_id = DPLL_ID_LCPLL_1350;
+		break;
+	case 270000:
+		pll_id = DPLL_ID_LCPLL_2700;
+		break;
+	default:
+		DRM_DEBUG_KMS("Invalid clock for DP: %d\n", clock);
+		return NULL;
+	}
+
+	pll = intel_get_shared_dpll_by_id(dev_priv, pll_id);
+
+	if (!pll)
+		return NULL;
+
+	return pll;
+}
+
 static struct intel_shared_dpll *
 hsw_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 	     struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_shared_dpll *pll;
 	int clock = crtc_state->port_clock;
 
@@ -699,41 +771,12 @@ hsw_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 	       sizeof(crtc_state->dpll_hw_state));
 
 	if (encoder->type == INTEL_OUTPUT_HDMI) {
-		uint32_t val;
-		unsigned p, n2, r2;
-
-		hsw_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p);
-
-		val = WRPLL_PLL_ENABLE | WRPLL_PLL_LCPLL |
-		      WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) |
-		      WRPLL_DIVIDER_POST(p);
-
-		crtc_state->dpll_hw_state.wrpll = val;
-
-		pll = intel_find_shared_dpll(crtc, crtc_state,
-					     DPLL_ID_WRPLL1, DPLL_ID_WRPLL2);
+		pll = hsw_ddi_hdmi_get_dpll(clock, crtc, crtc_state);
 
 	} else if (encoder->type == INTEL_OUTPUT_DP ||
 		   encoder->type == INTEL_OUTPUT_DP_MST ||
 		   encoder->type == INTEL_OUTPUT_EDP) {
-		enum intel_dpll_id pll_id;
-
-		switch (clock / 2) {
-		case 81000:
-			pll_id = DPLL_ID_LCPLL_810;
-			break;
-		case 135000:
-			pll_id = DPLL_ID_LCPLL_1350;
-			break;
-		case 270000:
-			pll_id = DPLL_ID_LCPLL_2700;
-			break;
-		default:
-			DRM_DEBUG_KMS("Invalid clock for DP: %d\n", clock);
-			return NULL;
-		}
-
-		pll = intel_get_shared_dpll_by_id(dev_priv, pll_id);
+		pll = hsw_ddi_dp_get_dpll(encoder, clock);
 
 	} else if (encoder->type == INTEL_OUTPUT_ANALOG) {
 		if (WARN_ON(crtc_state->port_clock / 2 != 135000))
@@ -751,14 +794,11 @@ hsw_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 	if (!pll)
 		return NULL;
 
-	crtc_state->ddi_pll_sel = hsw_pll_to_ddi_pll_sel(pll);
-
 	intel_reference_shared_dpll(pll, crtc_state);
 
 	return pll;
 }
 
-
 static const struct intel_shared_dpll_funcs hsw_ddi_wrpll_funcs = {
 	.enable = hsw_ddi_wrpll_enable,
 	.disable = hsw_ddi_wrpll_disable,
@@ -1194,75 +1234,110 @@ skip_remaining_dividers:
 	return true;
 }
 
-static struct intel_shared_dpll *
-skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
-	     struct intel_encoder *encoder)
+static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
+				      struct intel_crtc_state *crtc_state,
+				      int clock)
 {
-	struct intel_shared_dpll *pll;
 	uint32_t ctrl1, cfgcr1, cfgcr2;
-	int clock = crtc_state->port_clock;
+	struct skl_wrpll_params wrpll_params = { 0, };
 
 	/*
 	 * See comment in intel_dpll_hw_state to understand why we always use 0
 	 * as the DPLL id in this function.
 	 */
-
 	ctrl1 = DPLL_CTRL1_OVERRIDE(0);
 
-	if (encoder->type == INTEL_OUTPUT_HDMI) {
-		struct skl_wrpll_params wrpll_params = { 0, };
+	ctrl1 |= DPLL_CTRL1_HDMI_MODE(0);
+
+	if (!skl_ddi_calculate_wrpll(clock * 1000, &wrpll_params))
+		return false;
 
-		ctrl1 |= DPLL_CTRL1_HDMI_MODE(0);
+	cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE |
+		DPLL_CFGCR1_DCO_FRACTION(wrpll_params.dco_fraction) |
+		wrpll_params.dco_integer;
 
-		if (!skl_ddi_calculate_wrpll(clock * 1000, &wrpll_params))
-			return NULL;
+	cfgcr2 = DPLL_CFGCR2_QDIV_RATIO(wrpll_params.qdiv_ratio) |
+		DPLL_CFGCR2_QDIV_MODE(wrpll_params.qdiv_mode) |
+		DPLL_CFGCR2_KDIV(wrpll_params.kdiv) |
+		DPLL_CFGCR2_PDIV(wrpll_params.pdiv) |
+		wrpll_params.central_freq;
+
+	memset(&crtc_state->dpll_hw_state, 0,
+	       sizeof(crtc_state->dpll_hw_state));
+
+	crtc_state->dpll_hw_state.ctrl1 = ctrl1;
+	crtc_state->dpll_hw_state.cfgcr1 = cfgcr1;
+	crtc_state->dpll_hw_state.cfgcr2 = cfgcr2;
+	return true;
+}
+
+
+bool skl_ddi_dp_set_dpll_hw_state(int clock,
+				  struct intel_dpll_hw_state *dpll_hw_state)
+{
+	uint32_t ctrl1;
+
+	/*
+	 * See comment in intel_dpll_hw_state to understand why we always use 0
+	 * as the DPLL id in this function.
+	 */
+	ctrl1 = DPLL_CTRL1_OVERRIDE(0);
+	switch (clock / 2) {
+	case 81000:
+		ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, 0);
+		break;
+	case 135000:
+		ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, 0);
+		break;
+	case 270000:
+		ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, 0);
+		break;
+		/* eDP 1.4 rates */
+	case 162000:
+		ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, 0);
+		break;
+	case 108000:
+		ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, 0);
+		break;
+	case 216000:
+		ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, 0);
+		break;
+	}
+
+	dpll_hw_state->ctrl1 = ctrl1;
+	return true;
+}
+
+static struct intel_shared_dpll *
+skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
+	     struct intel_encoder *encoder)
+{
+	struct intel_shared_dpll *pll;
+	int clock = crtc_state->port_clock;
+	bool bret;
+	struct intel_dpll_hw_state dpll_hw_state;
 
-		cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE |
-			 DPLL_CFGCR1_DCO_FRACTION(wrpll_params.dco_fraction) |
-			 wrpll_params.dco_integer;
+	memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
 
-		cfgcr2 = DPLL_CFGCR2_QDIV_RATIO(wrpll_params.qdiv_ratio) |
-			 DPLL_CFGCR2_QDIV_MODE(wrpll_params.qdiv_mode) |
-			 DPLL_CFGCR2_KDIV(wrpll_params.kdiv) |
-			 DPLL_CFGCR2_PDIV(wrpll_params.pdiv) |
-			 wrpll_params.central_freq;
+	if (encoder->type == INTEL_OUTPUT_HDMI) {
+		bret = skl_ddi_hdmi_pll_dividers(crtc, crtc_state, clock);
+		if (!bret) {
+			DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n");
+			return NULL;
+		}
 	} else if (encoder->type == INTEL_OUTPUT_DP ||
 		   encoder->type == INTEL_OUTPUT_DP_MST ||
 		   encoder->type == INTEL_OUTPUT_EDP) {
-		switch (crtc_state->port_clock / 2) {
-		case 81000:
-			ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, 0);
-			break;
-		case 135000:
-			ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, 0);
-			break;
-		case 270000:
-			ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, 0);
-			break;
-		/* eDP 1.4 rates */
-		case 162000:
-			ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, 0);
-			break;
-		case 108000:
-			ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, 0);
-			break;
-		case 216000:
-			ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, 0);
-			break;
+		bret = skl_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state);
+		if (!bret) {
+			DRM_DEBUG_KMS("Could not set DP dpll HW state.\n");
+			return NULL;
 		}
-
-		cfgcr1 = cfgcr2 = 0;
+		crtc_state->dpll_hw_state = dpll_hw_state;
 	} else {
 		return NULL;
 	}
 
-	memset(&crtc_state->dpll_hw_state, 0,
-	       sizeof(crtc_state->dpll_hw_state));
-
-	crtc_state->dpll_hw_state.ctrl1 = ctrl1;
-	crtc_state->dpll_hw_state.cfgcr1 = cfgcr1;
-	crtc_state->dpll_hw_state.cfgcr2 = cfgcr2;
-
 	if (encoder->type == INTEL_OUTPUT_EDP)
 		pll = intel_find_shared_dpll(crtc, crtc_state,
 					     DPLL_ID_SKL_DPLL0,
@@ -1274,8 +1349,6 @@ skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 	if (!pll)
 		return NULL;
 
-	crtc_state->ddi_pll_sel = pll->id;
-
 	intel_reference_shared_dpll(pll, crtc_state);
 
 	return pll;
@@ -1484,6 +1557,8 @@ struct bxt_clk_div {
 	uint32_t m2_frac;
 	bool m2_frac_en;
 	uint32_t n;
+
+	int vco;
 };
 
 /* pre-calculated values for DP linkrates */
@@ -1497,57 +1572,60 @@ static const struct bxt_clk_div bxt_dp_clk_val[] = {
 	{432000, 3, 1, 32, 1677722, 1, 1}
 };
 
-static struct intel_shared_dpll *
-bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
-	     struct intel_encoder *encoder)
+static bool
+bxt_ddi_hdmi_pll_dividers(struct intel_crtc *intel_crtc,
+			  struct intel_crtc_state *crtc_state, int clock,
+			  struct bxt_clk_div *clk_div)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct intel_shared_dpll *pll;
-	enum intel_dpll_id i;
-	struct intel_digital_port *intel_dig_port;
-	struct bxt_clk_div clk_div = {0};
-	int vco = 0;
-	uint32_t prop_coef, int_coef, gain_ctl, targ_cnt;
-	uint32_t lanestagger;
-	int clock = crtc_state->port_clock;
+	struct dpll best_clock;
 
-	if (encoder->type == INTEL_OUTPUT_HDMI) {
-		struct dpll best_clock;
+	/* Calculate HDMI div */
+	/*
+	 * FIXME: tie the following calculation into
+	 * i9xx_crtc_compute_clock
+	 */
+	if (!bxt_find_best_dpll(crtc_state, clock, &best_clock)) {
+		DRM_DEBUG_DRIVER("no PLL dividers found for clock %d pipe %c\n",
+				 clock, pipe_name(intel_crtc->pipe));
+		return false;
+	}
 
-		/* Calculate HDMI div */
-		/*
-		 * FIXME: tie the following calculation into
-		 * i9xx_crtc_compute_clock
-		 */
-		if (!bxt_find_best_dpll(crtc_state, clock, &best_clock)) {
-			DRM_DEBUG_DRIVER("no PLL dividers found for clock %d pipe %c\n",
-					 clock, pipe_name(crtc->pipe));
-			return NULL;
-		}
+	clk_div->p1 = best_clock.p1;
+	clk_div->p2 = best_clock.p2;
+	WARN_ON(best_clock.m1 != 2);
+	clk_div->n = best_clock.n;
+	clk_div->m2_int = best_clock.m2 >> 22;
+	clk_div->m2_frac = best_clock.m2 & ((1 << 22) - 1);
+	clk_div->m2_frac_en = clk_div->m2_frac != 0;
 
-		clk_div.p1 = best_clock.p1;
-		clk_div.p2 = best_clock.p2;
-		WARN_ON(best_clock.m1 != 2);
-		clk_div.n = best_clock.n;
-		clk_div.m2_int = best_clock.m2 >> 22;
-		clk_div.m2_frac = best_clock.m2 & ((1 << 22) - 1);
-		clk_div.m2_frac_en = clk_div.m2_frac != 0;
+	clk_div->vco = best_clock.vco;
 
-		vco = best_clock.vco;
-	} else if (encoder->type == INTEL_OUTPUT_DP ||
-		   encoder->type == INTEL_OUTPUT_EDP) {
-		int i;
+	return true;
+}
 
-		clk_div = bxt_dp_clk_val[0];
-		for (i = 0; i < ARRAY_SIZE(bxt_dp_clk_val); ++i) {
-			if (bxt_dp_clk_val[i].clock == clock) {
-				clk_div = bxt_dp_clk_val[i];
-				break;
-			}
+static void bxt_ddi_dp_pll_dividers(int clock, struct bxt_clk_div *clk_div)
+{
+	int i;
+
+	*clk_div = bxt_dp_clk_val[0];
+	for (i = 0; i < ARRAY_SIZE(bxt_dp_clk_val); ++i) {
+		if (bxt_dp_clk_val[i].clock == clock) {
+			*clk_div = bxt_dp_clk_val[i];
+			break;
 		}
-		vco = clock * 10 / 2 * clk_div.p1 * clk_div.p2;
 	}
 
+	clk_div->vco = clock * 10 / 2 * clk_div->p1 * clk_div->p2;
+}
+
+static bool bxt_ddi_set_dpll_hw_state(int clock,
+			  struct bxt_clk_div *clk_div,
+			  struct intel_dpll_hw_state *dpll_hw_state)
+{
+	int vco = clk_div->vco;
+	uint32_t prop_coef, int_coef, gain_ctl, targ_cnt;
+	uint32_t lanestagger;
+
 	if (vco >= 6200000 && vco <= 6700000) {
 		prop_coef = 4;
 		int_coef = 9;
@@ -1566,12 +1644,9 @@ bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 		targ_cnt = 9;
 	} else {
 		DRM_ERROR("Invalid VCO\n");
-		return NULL;
+		return false;
 	}
 
-	memset(&crtc_state->dpll_hw_state, 0,
-	       sizeof(crtc_state->dpll_hw_state));
-
 	if (clock > 270000)
 		lanestagger = 0x18;
 	else if (clock > 135000)
@@ -1583,35 +1658,86 @@ bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 	else
 		lanestagger = 0x02;
 
-	crtc_state->dpll_hw_state.ebb0 =
-		PORT_PLL_P1(clk_div.p1) | PORT_PLL_P2(clk_div.p2);
-	crtc_state->dpll_hw_state.pll0 = clk_div.m2_int;
-	crtc_state->dpll_hw_state.pll1 = PORT_PLL_N(clk_div.n);
-	crtc_state->dpll_hw_state.pll2 = clk_div.m2_frac;
+	dpll_hw_state->ebb0 = PORT_PLL_P1(clk_div->p1) | PORT_PLL_P2(clk_div->p2);
+	dpll_hw_state->pll0 = clk_div->m2_int;
+	dpll_hw_state->pll1 = PORT_PLL_N(clk_div->n);
+	dpll_hw_state->pll2 = clk_div->m2_frac;
 
-	if (clk_div.m2_frac_en)
-		crtc_state->dpll_hw_state.pll3 =
-			PORT_PLL_M2_FRAC_ENABLE;
+	if (clk_div->m2_frac_en)
+		dpll_hw_state->pll3 = PORT_PLL_M2_FRAC_ENABLE;
 
-	crtc_state->dpll_hw_state.pll6 =
-		prop_coef | PORT_PLL_INT_COEFF(int_coef);
-	crtc_state->dpll_hw_state.pll6 |=
-		PORT_PLL_GAIN_CTL(gain_ctl);
+	dpll_hw_state->pll6 = prop_coef | PORT_PLL_INT_COEFF(int_coef);
+	dpll_hw_state->pll6 |= PORT_PLL_GAIN_CTL(gain_ctl);
 
-	crtc_state->dpll_hw_state.pll8 = targ_cnt;
+	dpll_hw_state->pll8 = targ_cnt;
 
-	crtc_state->dpll_hw_state.pll9 = 5 << PORT_PLL_LOCK_THRESHOLD_SHIFT;
+	dpll_hw_state->pll9 = 5 << PORT_PLL_LOCK_THRESHOLD_SHIFT;
 
-	crtc_state->dpll_hw_state.pll10 =
+	dpll_hw_state->pll10 =
 		PORT_PLL_DCO_AMP(PORT_PLL_DCO_AMP_DEFAULT)
 		| PORT_PLL_DCO_AMP_OVR_EN_H;
 
-	crtc_state->dpll_hw_state.ebb4 = PORT_PLL_10BIT_CLK_ENABLE;
+	dpll_hw_state->ebb4 = PORT_PLL_10BIT_CLK_ENABLE;
+
+	dpll_hw_state->pcsdw12 = LANESTAGGER_STRAP_OVRD | lanestagger;
 
-	crtc_state->dpll_hw_state.pcsdw12 =
-		LANESTAGGER_STRAP_OVRD | lanestagger;
+	return true;
+}
 
-	intel_dig_port = enc_to_dig_port(&encoder->base);
+bool bxt_ddi_dp_set_dpll_hw_state(int clock,
+			  struct intel_dpll_hw_state *dpll_hw_state)
+{
+	struct bxt_clk_div clk_div = {0};
+
+	bxt_ddi_dp_pll_dividers(clock, &clk_div);
+
+	return bxt_ddi_set_dpll_hw_state(clock, &clk_div, dpll_hw_state);
+}
+
+static bool
+bxt_ddi_hdmi_set_dpll_hw_state(struct intel_crtc *intel_crtc,
+			       struct intel_crtc_state *crtc_state, int clock,
+			       struct intel_dpll_hw_state *dpll_hw_state)
+{
+	struct bxt_clk_div clk_div = { };
+
+	bxt_ddi_hdmi_pll_dividers(intel_crtc, crtc_state, clock, &clk_div);
+
+	return bxt_ddi_set_dpll_hw_state(clock, &clk_div, dpll_hw_state);
+}
+
+static struct intel_shared_dpll *
+bxt_get_dpll(struct intel_crtc *crtc,
+		struct intel_crtc_state *crtc_state,
+		struct intel_encoder *encoder)
+{
+	struct intel_dpll_hw_state dpll_hw_state = { };
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	struct intel_digital_port *intel_dig_port;
+	struct intel_shared_dpll *pll;
+	int i, clock = crtc_state->port_clock;
+
+	if (encoder->type == INTEL_OUTPUT_HDMI &&
+	    !bxt_ddi_hdmi_set_dpll_hw_state(crtc, crtc_state, clock,
+					    &dpll_hw_state))
+		return NULL;
+
+	if ((encoder->type == INTEL_OUTPUT_DP ||
+	     encoder->type == INTEL_OUTPUT_EDP) &&
+	    !bxt_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state))
+		return NULL;
+
+	memset(&crtc_state->dpll_hw_state, 0,
+	       sizeof(crtc_state->dpll_hw_state));
+
+	crtc_state->dpll_hw_state = dpll_hw_state;
+
+	if (encoder->type == INTEL_OUTPUT_DP_MST) {
+		struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
+
+		intel_dig_port = intel_mst->primary;
+	} else
+		intel_dig_port = enc_to_dig_port(&encoder->base);
 
 	/* 1:1 mapping between ports and PLLs */
 	i = (enum intel_dpll_id) intel_dig_port->port;
@@ -1622,9 +1748,6 @@ bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 
 	intel_reference_shared_dpll(pll, crtc_state);
 
-	/* shared DPLL id 0 is DPLL A */
-	crtc_state->ddi_pll_sel = pll->id;
-
 	return pll;
 }
 
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.h b/drivers/gpu/drm/i915/intel_dpll_mgr.h
index 89c5ada1a315..f4385353bc11 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.h
@@ -160,5 +160,20 @@ void intel_disable_shared_dpll(struct intel_crtc *crtc);
 void intel_shared_dpll_commit(struct drm_atomic_state *state);
 void intel_shared_dpll_init(struct drm_device *dev);
 
+/* BXT dpll related functions */
+bool bxt_ddi_dp_set_dpll_hw_state(int clock,
+			  struct intel_dpll_hw_state *dpll_hw_state);
+
+
+/* SKL dpll related functions */
+bool skl_ddi_dp_set_dpll_hw_state(int clock,
+				  struct intel_dpll_hw_state *dpll_hw_state);
+struct intel_shared_dpll *skl_find_link_pll(struct drm_i915_private *dev_priv,
+					    int clock);
+
+
+/* HSW dpll related functions */
+struct intel_shared_dpll *hsw_ddi_dp_get_dpll(struct intel_encoder *encoder,
+					      int clock);
 
 #endif /* _INTEL_DPLL_MGR_H_ */
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index cc937a19b1ba..a19ec06f9e42 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -52,11 +52,15 @@
  */
 #define _wait_for(COND, US, W) ({ \
 	unsigned long timeout__ = jiffies + usecs_to_jiffies(US) + 1;	\
-	int ret__ = 0;							\
-	while (!(COND)) {						\
-		if (time_after(jiffies, timeout__)) {			\
-			if (!(COND))					\
-				ret__ = -ETIMEDOUT;			\
+	int ret__;							\
+	for (;;) {							\
+		bool expired__ = time_after(jiffies, timeout__);	\
+		if (COND) {						\
+			ret__ = 0;					\
+			break;						\
+		}							\
+		if (expired__) {					\
+			ret__ = -ETIMEDOUT;				\
 			break;						\
 		}							\
 		if ((W) && drm_can_sleep()) {				\
@@ -178,11 +182,22 @@ struct intel_framebuffer {
 	struct drm_framebuffer base;
 	struct drm_i915_gem_object *obj;
 	struct intel_rotation_info rot_info;
+
+	/* for each plane in the normal GTT view */
+	struct {
+		unsigned int x, y;
+	} normal[2];
+	/* for each plane in the rotated GTT view */
+	struct {
+		unsigned int x, y;
+		unsigned int pitch; /* pixels */
+	} rotated[2];
 };
 
 struct intel_fbdev {
 	struct drm_fb_helper helper;
 	struct intel_framebuffer *fb;
+	struct i915_vma *vma;
 	async_cookie_t cookie;
 	int preferred_bpp;
 };
@@ -194,14 +209,26 @@ struct intel_encoder {
 	unsigned int cloneable;
 	void (*hot_plug)(struct intel_encoder *);
 	bool (*compute_config)(struct intel_encoder *,
-			       struct intel_crtc_state *);
-	void (*pre_pll_enable)(struct intel_encoder *);
-	void (*pre_enable)(struct intel_encoder *);
-	void (*enable)(struct intel_encoder *);
-	void (*mode_set)(struct intel_encoder *intel_encoder);
-	void (*disable)(struct intel_encoder *);
-	void (*post_disable)(struct intel_encoder *);
-	void (*post_pll_disable)(struct intel_encoder *);
+			       struct intel_crtc_state *,
+			       struct drm_connector_state *);
+	void (*pre_pll_enable)(struct intel_encoder *,
+			       struct intel_crtc_state *,
+			       struct drm_connector_state *);
+	void (*pre_enable)(struct intel_encoder *,
+			   struct intel_crtc_state *,
+			   struct drm_connector_state *);
+	void (*enable)(struct intel_encoder *,
+		       struct intel_crtc_state *,
+		       struct drm_connector_state *);
+	void (*disable)(struct intel_encoder *,
+			struct intel_crtc_state *,
+			struct drm_connector_state *);
+	void (*post_disable)(struct intel_encoder *,
+			     struct intel_crtc_state *,
+			     struct drm_connector_state *);
+	void (*post_pll_disable)(struct intel_encoder *,
+				 struct intel_crtc_state *,
+				 struct drm_connector_state *);
 	/* Read out the current hw state of this connector, returning true if
 	 * the encoder is active. If the encoder is enabled it also set the pipe
 	 * it is connected to in the pipe parameter. */
@@ -236,6 +263,7 @@ struct intel_panel {
 		bool enabled;
 		bool combination_mode;	/* gen 2/4 only */
 		bool active_low_pwm;
+		bool alternate_pwm_increment;	/* lpt+ */
 
 		/* PWM chip */
 		bool util_pin_active_low;	/* bxt+ */
@@ -338,10 +366,16 @@ struct intel_atomic_state {
 
 struct intel_plane_state {
 	struct drm_plane_state base;
-	struct drm_rect src;
-	struct drm_rect dst;
 	struct drm_rect clip;
-	bool visible;
+
+	struct {
+		u32 offset;
+		int x, y;
+	} main;
+	struct {
+		u32 offset;
+		int x, y;
+	} aux;
 
 	/*
 	 * scaler_id
@@ -561,12 +595,6 @@ struct intel_crtc_state {
 	/* Selected dpll when shared or NULL. */
 	struct intel_shared_dpll *shared_dpll;
 
-	/*
-	 * - PORT_CLK_SEL for DDI ports on HSW/BDW.
-	 * - enum skl_dpll on SKL
-	 */
-	uint32_t ddi_pll_sel;
-
 	/* Actual register state of the dpll, for shared dpll cross-checking. */
 	struct intel_dpll_hw_state dpll_hw_state;
 
@@ -683,8 +711,8 @@ struct intel_crtc {
 
 	struct intel_crtc_state *config;
 
-	/* reset counter value when the last flip was submitted */
-	unsigned int reset_counter;
+	/* global reset count when the last flip was submitted */
+	unsigned int reset_count;
 
 	/* Access to these should be protected by dev_priv->irq_lock. */
 	bool cpu_fifo_underrun_disabled;
@@ -852,8 +880,10 @@ struct intel_dp {
 	int link_rate;
 	uint8_t lane_count;
 	uint8_t sink_count;
+	bool link_mst;
 	bool has_audio;
 	bool detect_done;
+	bool channel_eq_status;
 	enum hdmi_force_audio force_audio;
 	bool limited_color_range;
 	bool color_range_auto;
@@ -1106,8 +1136,11 @@ void intel_crt_reset(struct drm_encoder *encoder);
 
 /* intel_ddi.c */
 void intel_ddi_clk_select(struct intel_encoder *encoder,
-			  const struct intel_crtc_state *pipe_config);
-void intel_prepare_ddi_buffer(struct intel_encoder *encoder);
+			  struct intel_shared_dpll *pll);
+void intel_ddi_fdi_post_disable(struct intel_encoder *intel_encoder,
+				struct intel_crtc_state *old_crtc_state,
+				struct drm_connector_state *old_conn_state);
+void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder);
 void hsw_fdi_link_train(struct drm_crtc *crtc);
 void intel_ddi_init(struct drm_device *dev, enum port port);
 enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder);
@@ -1122,7 +1155,6 @@ bool intel_ddi_pll_select(struct intel_crtc *crtc,
 void intel_ddi_set_pipe_settings(struct drm_crtc *crtc);
 void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp);
 bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector);
-void intel_ddi_fdi_disable(struct drm_crtc *crtc);
 void intel_ddi_get_config(struct intel_encoder *encoder,
 			  struct intel_crtc_state *pipe_config);
 struct intel_encoder *
@@ -1133,22 +1165,12 @@ void intel_ddi_clock_get(struct intel_encoder *encoder,
 			 struct intel_crtc_state *pipe_config);
 void intel_ddi_set_vc_payload_alloc(struct drm_crtc *crtc, bool state);
 uint32_t ddi_signal_levels(struct intel_dp *intel_dp);
-
-/* intel_frontbuffer.c */
-void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
-			     enum fb_op_origin origin);
-void intel_frontbuffer_flip_prepare(struct drm_device *dev,
-				    unsigned frontbuffer_bits);
-void intel_frontbuffer_flip_complete(struct drm_device *dev,
-				     unsigned frontbuffer_bits);
-void intel_frontbuffer_flip(struct drm_device *dev,
-			    unsigned frontbuffer_bits);
+struct intel_shared_dpll *intel_ddi_get_link_dpll(struct intel_dp *intel_dp,
+						  int clock);
 unsigned int intel_fb_align_height(struct drm_device *dev,
 				   unsigned int height,
 				   uint32_t pixel_format,
 				   uint64_t fb_format_modifier);
-void intel_fb_obj_flush(struct drm_i915_gem_object *obj, bool retire,
-			enum fb_op_origin origin);
 u32 intel_fb_stride_alignment(const struct drm_i915_private *dev_priv,
 			      uint64_t fb_modifier, uint32_t pixel_format);
 
@@ -1164,14 +1186,22 @@ void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco);
 void intel_update_rawclk(struct drm_i915_private *dev_priv);
 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 		      const char *name, u32 reg, int ref_freq);
+void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv);
+void lpt_disable_iclkip(struct drm_i915_private *dev_priv);
 extern const struct drm_plane_funcs intel_plane_funcs;
 void intel_init_display_hooks(struct drm_i915_private *dev_priv);
+unsigned int intel_fb_xy_to_linear(int x, int y,
+				   const struct intel_plane_state *state,
+				   int plane);
+void intel_add_fb_offsets(int *x, int *y,
+			  const struct intel_plane_state *state, int plane);
 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info);
 bool intel_has_pending_fb_unpin(struct drm_device *dev);
 void intel_mark_busy(struct drm_i915_private *dev_priv);
 void intel_mark_idle(struct drm_i915_private *dev_priv);
 void intel_crtc_restore_mode(struct drm_crtc *crtc);
 int intel_display_suspend(struct drm_device *dev);
+void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv);
 void intel_encoder_destroy(struct drm_encoder *encoder);
 int intel_connector_init(struct intel_connector *);
 struct intel_connector *intel_connector_alloc(void);
@@ -1227,8 +1257,8 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector,
 void intel_release_load_detect_pipe(struct drm_connector *connector,
 				    struct intel_load_detect_pipe *old,
 				    struct drm_modeset_acquire_ctx *ctx);
-int intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
-			       unsigned int rotation);
+struct i915_vma *
+intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation);
 void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation);
 struct drm_framebuffer *
 __intel_framebuffer_create(struct drm_device *dev,
@@ -1238,9 +1268,9 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe);
 void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe);
 void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe);
 int intel_prepare_plane_fb(struct drm_plane *plane,
-			   const struct drm_plane_state *new_state);
+			   struct drm_plane_state *new_state);
 void intel_cleanup_plane_fb(struct drm_plane *plane,
-			    const struct drm_plane_state *old_state);
+			    struct drm_plane_state *old_state);
 int intel_plane_atomic_get_property(struct drm_plane *plane,
 				    const struct drm_plane_state *state,
 				    struct drm_property *property,
@@ -1258,7 +1288,7 @@ unsigned int intel_tile_height(const struct drm_i915_private *dev_priv,
 static inline bool
 intel_rotation_90_or_270(unsigned int rotation)
 {
-	return rotation & (BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270));
+	return rotation & (DRM_ROTATE_90 | DRM_ROTATE_270);
 }
 
 void intel_create_rotation_property(struct drm_device *dev,
@@ -1290,9 +1320,7 @@ void assert_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, bool state);
 #define assert_pipe_enabled(d, p) assert_pipe(d, p, true)
 #define assert_pipe_disabled(d, p) assert_pipe(d, p, false)
 u32 intel_compute_tile_offset(int *x, int *y,
-			      const struct drm_framebuffer *fb, int plane,
-			      unsigned int pitch,
-			      unsigned int rotation);
+			      const struct intel_plane_state *state, int plane);
 void intel_prepare_reset(struct drm_i915_private *dev_priv);
 void intel_finish_reset(struct drm_i915_private *dev_priv);
 void hsw_enable_pc8(struct drm_i915_private *dev_priv);
@@ -1335,13 +1363,14 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode,
 int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state);
 int skl_max_scale(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state);
 
-u32 intel_plane_obj_offset(struct intel_plane *intel_plane,
-			   struct drm_i915_gem_object *obj,
-			   unsigned int plane);
+u32 intel_fb_gtt_offset(struct drm_framebuffer *fb, unsigned int rotation);
 
 u32 skl_plane_ctl_format(uint32_t pixel_format);
 u32 skl_plane_ctl_tiling(uint64_t fb_modifier);
 u32 skl_plane_ctl_rotation(unsigned int rotation);
+u32 skl_plane_stride(const struct drm_framebuffer *fb, int plane,
+		     unsigned int rotation);
+int skl_check_plane_surface(struct intel_plane_state *plane_state);
 
 /* intel_csr.c */
 void intel_csr_ucode_init(struct drm_i915_private *);
@@ -1355,7 +1384,8 @@ bool intel_dp_init(struct drm_device *dev, i915_reg_t output_reg, enum port port
 bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 			     struct intel_connector *intel_connector);
 void intel_dp_set_link_params(struct intel_dp *intel_dp,
-			      const struct intel_crtc_state *pipe_config);
+			      int link_rate, uint8_t lane_count,
+			      bool link_mst);
 void intel_dp_start_link_train(struct intel_dp *intel_dp);
 void intel_dp_stop_link_train(struct intel_dp *intel_dp);
 void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode);
@@ -1364,7 +1394,8 @@ void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder);
 void intel_dp_encoder_destroy(struct drm_encoder *encoder);
 int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc);
 bool intel_dp_compute_config(struct intel_encoder *encoder,
-			     struct intel_crtc_state *pipe_config);
+			     struct intel_crtc_state *pipe_config,
+			     struct drm_connector_state *conn_state);
 bool intel_dp_is_edp(struct drm_device *dev, enum port port);
 enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port,
 				  bool long_hpd);
@@ -1382,13 +1413,14 @@ void intel_dp_hot_plug(struct intel_encoder *intel_encoder);
 void intel_power_sequencer_reset(struct drm_i915_private *dev_priv);
 uint32_t intel_dp_pack_aux(const uint8_t *src, int src_bytes);
 void intel_plane_destroy(struct drm_plane *plane);
-void intel_edp_drrs_enable(struct intel_dp *intel_dp);
-void intel_edp_drrs_disable(struct intel_dp *intel_dp);
-void intel_edp_drrs_invalidate(struct drm_device *dev,
-		unsigned frontbuffer_bits);
-void intel_edp_drrs_flush(struct drm_device *dev, unsigned frontbuffer_bits);
-bool intel_digital_port_connected(struct drm_i915_private *dev_priv,
-					 struct intel_digital_port *port);
+void intel_edp_drrs_enable(struct intel_dp *intel_dp,
+			   struct intel_crtc_state *crtc_state);
+void intel_edp_drrs_disable(struct intel_dp *intel_dp,
+			   struct intel_crtc_state *crtc_state);
+void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv,
+			       unsigned int frontbuffer_bits);
+void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
+			  unsigned int frontbuffer_bits);
 
 void
 intel_dp_program_link_training_pattern(struct intel_dp *intel_dp,
@@ -1488,7 +1520,8 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 			       struct intel_connector *intel_connector);
 struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder);
 bool intel_hdmi_compute_config(struct intel_encoder *encoder,
-			       struct intel_crtc_state *pipe_config);
+			       struct intel_crtc_state *pipe_config,
+			       struct drm_connector_state *conn_state);
 void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable);
 
 
@@ -1561,13 +1594,13 @@ static inline void intel_backlight_device_unregister(struct intel_connector *con
 /* intel_psr.c */
 void intel_psr_enable(struct intel_dp *intel_dp);
 void intel_psr_disable(struct intel_dp *intel_dp);
-void intel_psr_invalidate(struct drm_device *dev,
+void intel_psr_invalidate(struct drm_i915_private *dev_priv,
 			  unsigned frontbuffer_bits);
-void intel_psr_flush(struct drm_device *dev,
+void intel_psr_flush(struct drm_i915_private *dev_priv,
 		     unsigned frontbuffer_bits,
 		     enum fb_op_origin origin);
 void intel_psr_init(struct drm_device *dev);
-void intel_psr_single_frame_update(struct drm_device *dev,
+void intel_psr_single_frame_update(struct drm_i915_private *dev_priv,
 				   unsigned frontbuffer_bits);
 
 /* intel_runtime_pm.c */
@@ -1667,13 +1700,6 @@ enable_rpm_wakeref_asserts(struct drm_i915_private *dev_priv)
 	atomic_dec(&dev_priv->pm.wakeref_count);
 }
 
-/* TODO: convert users of these to rely instead on proper RPM refcounting */
-#define DISABLE_RPM_WAKEREF_ASSERTS(dev_priv)	\
-	disable_rpm_wakeref_asserts(dev_priv)
-
-#define ENABLE_RPM_WAKEREF_ASSERTS(dev_priv)	\
-	enable_rpm_wakeref_asserts(dev_priv)
-
 void intel_runtime_pm_get(struct drm_i915_private *dev_priv);
 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv);
 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv);
@@ -1699,11 +1725,11 @@ void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
 void intel_gpu_ips_teardown(void);
 void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv);
-void intel_reset_gt_powersave(struct drm_i915_private *dev_priv);
-void gen6_update_ring_freq(struct drm_i915_private *dev_priv);
 void gen6_rps_busy(struct drm_i915_private *dev_priv);
 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv);
 void gen6_rps_idle(struct drm_i915_private *dev_priv);
@@ -1716,6 +1742,21 @@ void ilk_wm_get_hw_state(struct drm_device *dev);
 void skl_wm_get_hw_state(struct drm_device *dev);
 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
 			  struct skl_ddb_allocation *ddb /* out */);
+bool intel_can_enable_sagv(struct drm_atomic_state *state);
+int intel_enable_sagv(struct drm_i915_private *dev_priv);
+int intel_disable_sagv(struct drm_i915_private *dev_priv);
+bool skl_ddb_allocation_equals(const struct skl_ddb_allocation *old,
+			       const struct skl_ddb_allocation *new,
+			       enum pipe pipe);
+bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state,
+				 const struct skl_ddb_allocation *old,
+				 const struct skl_ddb_allocation *new,
+				 enum pipe pipe);
+void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
+			 const struct skl_wm_values *wm);
+void skl_write_plane_wm(struct intel_crtc *intel_crtc,
+			const struct skl_wm_values *wm,
+			int plane);
 uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config);
 bool ilk_disable_lp_wm(struct drm_device *dev);
 int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6);
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index de8e9fb51595..b2e3d3a334f7 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -312,7 +312,8 @@ static inline bool is_cmd_mode(struct intel_dsi *intel_dsi)
 }
 
 static bool intel_dsi_compute_config(struct intel_encoder *encoder,
-				     struct intel_crtc_state *pipe_config)
+				     struct intel_crtc_state *pipe_config,
+				     struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi,
@@ -533,14 +534,15 @@ static void intel_dsi_enable(struct intel_encoder *encoder)
 	intel_panel_enable_backlight(intel_dsi->attached_connector);
 }
 
-static void intel_dsi_prepare(struct intel_encoder *intel_encoder);
+static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
+			      struct intel_crtc_state *pipe_config);
 
-static void intel_dsi_pre_enable(struct intel_encoder *encoder)
+static void intel_dsi_pre_enable(struct intel_encoder *encoder,
+				 struct intel_crtc_state *pipe_config,
+				 struct drm_connector_state *conn_state)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 	enum port port;
 
 	DRM_DEBUG_KMS("\n");
@@ -550,9 +552,9 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder)
 	 * lock. It needs to be fully powered down to fix it.
 	 */
 	intel_disable_dsi_pll(encoder);
-	intel_enable_dsi_pll(encoder, crtc->config);
+	intel_enable_dsi_pll(encoder, pipe_config);
 
-	intel_dsi_prepare(encoder);
+	intel_dsi_prepare(encoder, pipe_config);
 
 	/* Panel Enable over CRC PMIC */
 	if (intel_dsi->gpio_panel)
@@ -582,7 +584,9 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder)
 	intel_dsi_enable(encoder);
 }
 
-static void intel_dsi_enable_nop(struct intel_encoder *encoder)
+static void intel_dsi_enable_nop(struct intel_encoder *encoder,
+				 struct intel_crtc_state *pipe_config,
+				 struct drm_connector_state *conn_state)
 {
 	DRM_DEBUG_KMS("\n");
 
@@ -592,7 +596,9 @@ static void intel_dsi_enable_nop(struct intel_encoder *encoder)
 	 */
 }
 
-static void intel_dsi_pre_disable(struct intel_encoder *encoder)
+static void intel_dsi_pre_disable(struct intel_encoder *encoder,
+				  struct intel_crtc_state *old_crtc_state,
+				  struct drm_connector_state *old_conn_state)
 {
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	enum port port;
@@ -694,7 +700,9 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 	intel_disable_dsi_pll(encoder);
 }
 
-static void intel_dsi_post_disable(struct intel_encoder *encoder)
+static void intel_dsi_post_disable(struct intel_encoder *encoder,
+				   struct intel_crtc_state *pipe_config,
+				   struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
@@ -819,6 +827,7 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 	u16 crtc_htotal_sw, crtc_hsync_start_sw, crtc_hsync_end_sw,
 				crtc_hblank_start_sw, crtc_hblank_end_sw;
 
+	/* FIXME: hw readout should not depend on SW state */
 	intel_crtc = to_intel_crtc(encoder->base.crtc);
 	adjusted_mode_sw = &intel_crtc->config->base.adjusted_mode;
 
@@ -1104,14 +1113,15 @@ static u32 pixel_format_to_reg(enum mipi_dsi_pixel_format fmt)
 	}
 }
 
-static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
+static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
+			      struct intel_crtc_state *pipe_config)
 {
 	struct drm_encoder *encoder = &intel_encoder->base;
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
-	const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
+	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	enum port port;
 	unsigned int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
 	u32 val, tmp;
@@ -1348,7 +1358,7 @@ static int intel_dsi_set_property(struct drm_connector *connector,
 		intel_connector->panel.fitting_mode = val;
 	}
 
-	crtc = intel_attached_encoder(connector)->base.crtc;
+	crtc = connector->state->crtc;
 	if (crtc && crtc->state->enable) {
 		/*
 		 * If the CRTC is enabled, the display will be changed
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 47bdf9dad0d3..2e452c505e7e 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -174,7 +174,9 @@ static void intel_dvo_get_config(struct intel_encoder *encoder,
 	pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock;
 }
 
-static void intel_disable_dvo(struct intel_encoder *encoder)
+static void intel_disable_dvo(struct intel_encoder *encoder,
+			      struct intel_crtc_state *old_crtc_state,
+			      struct drm_connector_state *old_conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
@@ -186,17 +188,18 @@ static void intel_disable_dvo(struct intel_encoder *encoder)
 	I915_READ(dvo_reg);
 }
 
-static void intel_enable_dvo(struct intel_encoder *encoder)
+static void intel_enable_dvo(struct intel_encoder *encoder,
+			     struct intel_crtc_state *pipe_config,
+			     struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 	i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg;
 	u32 temp = I915_READ(dvo_reg);
 
 	intel_dvo->dev.dev_ops->mode_set(&intel_dvo->dev,
-					 &crtc->config->base.mode,
-					 &crtc->config->base.adjusted_mode);
+					 &pipe_config->base.mode,
+					 &pipe_config->base.adjusted_mode);
 
 	I915_WRITE(dvo_reg, temp | DVO_ENABLE);
 	I915_READ(dvo_reg);
@@ -235,7 +238,8 @@ intel_dvo_mode_valid(struct drm_connector *connector,
 }
 
 static bool intel_dvo_compute_config(struct intel_encoder *encoder,
-				     struct intel_crtc_state *pipe_config)
+				     struct intel_crtc_state *pipe_config,
+				     struct drm_connector_state *conn_state)
 {
 	struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
 	const struct drm_display_mode *fixed_mode =
@@ -253,12 +257,13 @@ static bool intel_dvo_compute_config(struct intel_encoder *encoder,
 	return true;
 }
 
-static void intel_dvo_pre_enable(struct intel_encoder *encoder)
+static void intel_dvo_pre_enable(struct intel_encoder *encoder,
+				 struct intel_crtc_state *pipe_config,
+				 struct drm_connector_state *conn_state)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-	const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
+	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
 	int pipe = crtc->pipe;
 	u32 dvo_val;
@@ -554,7 +559,6 @@ void intel_dvo_init(struct drm_device *dev)
 		return;
 	}
 
-	drm_encoder_cleanup(&intel_encoder->base);
 	kfree(intel_dvo);
 	kfree(intel_connector);
 }
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
new file mode 100644
index 000000000000..025e232a4205
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -0,0 +1,321 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "i915_drv.h"
+#include "intel_ringbuffer.h"
+#include "intel_lrc.h"
+
+static const struct engine_info {
+	const char *name;
+	unsigned exec_id;
+	enum intel_engine_hw_id hw_id;
+	u32 mmio_base;
+	unsigned irq_shift;
+	int (*init_legacy)(struct intel_engine_cs *engine);
+	int (*init_execlists)(struct intel_engine_cs *engine);
+} intel_engines[] = {
+	[RCS] = {
+		.name = "render ring",
+		.exec_id = I915_EXEC_RENDER,
+		.hw_id = RCS_HW,
+		.mmio_base = RENDER_RING_BASE,
+		.irq_shift = GEN8_RCS_IRQ_SHIFT,
+		.init_execlists = logical_render_ring_init,
+		.init_legacy = intel_init_render_ring_buffer,
+	},
+	[BCS] = {
+		.name = "blitter ring",
+		.exec_id = I915_EXEC_BLT,
+		.hw_id = BCS_HW,
+		.mmio_base = BLT_RING_BASE,
+		.irq_shift = GEN8_BCS_IRQ_SHIFT,
+		.init_execlists = logical_xcs_ring_init,
+		.init_legacy = intel_init_blt_ring_buffer,
+	},
+	[VCS] = {
+		.name = "bsd ring",
+		.exec_id = I915_EXEC_BSD,
+		.hw_id = VCS_HW,
+		.mmio_base = GEN6_BSD_RING_BASE,
+		.irq_shift = GEN8_VCS1_IRQ_SHIFT,
+		.init_execlists = logical_xcs_ring_init,
+		.init_legacy = intel_init_bsd_ring_buffer,
+	},
+	[VCS2] = {
+		.name = "bsd2 ring",
+		.exec_id = I915_EXEC_BSD,
+		.hw_id = VCS2_HW,
+		.mmio_base = GEN8_BSD2_RING_BASE,
+		.irq_shift = GEN8_VCS2_IRQ_SHIFT,
+		.init_execlists = logical_xcs_ring_init,
+		.init_legacy = intel_init_bsd2_ring_buffer,
+	},
+	[VECS] = {
+		.name = "video enhancement ring",
+		.exec_id = I915_EXEC_VEBOX,
+		.hw_id = VECS_HW,
+		.mmio_base = VEBOX_RING_BASE,
+		.irq_shift = GEN8_VECS_IRQ_SHIFT,
+		.init_execlists = logical_xcs_ring_init,
+		.init_legacy = intel_init_vebox_ring_buffer,
+	},
+};
+
+static struct intel_engine_cs *
+intel_engine_setup(struct drm_i915_private *dev_priv,
+		   enum intel_engine_id id)
+{
+	const struct engine_info *info = &intel_engines[id];
+	struct intel_engine_cs *engine = &dev_priv->engine[id];
+
+	engine->id = id;
+	engine->i915 = dev_priv;
+	engine->name = info->name;
+	engine->exec_id = info->exec_id;
+	engine->hw_id = engine->guc_id = info->hw_id;
+	engine->mmio_base = info->mmio_base;
+	engine->irq_shift = info->irq_shift;
+
+	return engine;
+}
+
+/**
+ * intel_engines_init() - allocate, populate and init the Engine Command Streamers
+ * @dev: DRM device.
+ *
+ * Return: non-zero if the initialization failed.
+ */
+int intel_engines_init(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
+	unsigned int mask = 0;
+	int (*init)(struct intel_engine_cs *engine);
+	unsigned int i;
+	int ret;
+
+	WARN_ON(INTEL_INFO(dev_priv)->ring_mask == 0);
+	WARN_ON(INTEL_INFO(dev_priv)->ring_mask &
+		GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
+
+	for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
+		if (!HAS_ENGINE(dev_priv, i))
+			continue;
+
+		if (i915.enable_execlists)
+			init = intel_engines[i].init_execlists;
+		else
+			init = intel_engines[i].init_legacy;
+
+		if (!init)
+			continue;
+
+		ret = init(intel_engine_setup(dev_priv, i));
+		if (ret)
+			goto cleanup;
+
+		mask |= ENGINE_MASK(i);
+	}
+
+	/*
+	 * Catch failures to update intel_engines table when the new engines
+	 * are added to the driver by a warning and disabling the forgotten
+	 * engines.
+	 */
+	if (WARN_ON(mask != INTEL_INFO(dev_priv)->ring_mask))
+		device_info->ring_mask = mask;
+
+	device_info->num_rings = hweight32(mask);
+
+	return 0;
+
+cleanup:
+	for (i = 0; i < I915_NUM_ENGINES; i++) {
+		if (i915.enable_execlists)
+			intel_logical_ring_cleanup(&dev_priv->engine[i]);
+		else
+			intel_engine_cleanup(&dev_priv->engine[i]);
+	}
+
+	return ret;
+}
+
+void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+
+	/* Our semaphore implementation is strictly monotonic (i.e. we proceed
+	 * so long as the semaphore value in the register/page is greater
+	 * than the sync value), so whenever we reset the seqno,
+	 * so long as we reset the tracking semaphore value to 0, it will
+	 * always be before the next request's seqno. If we don't reset
+	 * the semaphore value, then when the seqno moves backwards all
+	 * future waits will complete instantly (causing rendering corruption).
+	 */
+	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
+		I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
+		I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
+		if (HAS_VEBOX(dev_priv))
+			I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
+	}
+	if (dev_priv->semaphore) {
+		struct page *page = i915_vma_first_page(dev_priv->semaphore);
+		void *semaphores;
+
+		/* Semaphores are in noncoherent memory, flush to be safe */
+		semaphores = kmap(page);
+		memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
+		       0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
+		drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
+				       I915_NUM_ENGINES * gen8_semaphore_seqno_size);
+		kunmap(page);
+	}
+	memset(engine->semaphore.sync_seqno, 0,
+	       sizeof(engine->semaphore.sync_seqno));
+
+	intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
+	if (engine->irq_seqno_barrier)
+		engine->irq_seqno_barrier(engine);
+	engine->last_submitted_seqno = seqno;
+
+	engine->hangcheck.seqno = seqno;
+
+	/* After manually advancing the seqno, fake the interrupt in case
+	 * there are any waiters for that seqno.
+	 */
+	intel_engine_wakeup(engine);
+}
+
+void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
+{
+	memset(&engine->hangcheck, 0, sizeof(engine->hangcheck));
+}
+
+static void intel_engine_init_requests(struct intel_engine_cs *engine)
+{
+	init_request_active(&engine->last_request, NULL);
+	INIT_LIST_HEAD(&engine->request_list);
+}
+
+/**
+ * intel_engines_setup_common - setup engine state not requiring hw access
+ * @engine: Engine to setup.
+ *
+ * Initializes @engine@ structure members shared between legacy and execlists
+ * submission modes which do not require hardware access.
+ *
+ * Typically done early in the submission mode specific engine setup stage.
+ */
+void intel_engine_setup_common(struct intel_engine_cs *engine)
+{
+	INIT_LIST_HEAD(&engine->execlist_queue);
+	spin_lock_init(&engine->execlist_lock);
+
+	engine->fence_context = fence_context_alloc(1);
+
+	intel_engine_init_requests(engine);
+	intel_engine_init_hangcheck(engine);
+	i915_gem_batch_pool_init(engine, &engine->batch_pool);
+
+	intel_engine_init_cmd_parser(engine);
+}
+
+int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
+{
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	int ret;
+
+	WARN_ON(engine->scratch);
+
+	obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
+	if (!obj)
+		obj = i915_gem_object_create(&engine->i915->drm, size);
+	if (IS_ERR(obj)) {
+		DRM_ERROR("Failed to allocate scratch page\n");
+		return PTR_ERR(obj);
+	}
+
+	vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
+		goto err_unref;
+	}
+
+	ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL | PIN_HIGH);
+	if (ret)
+		goto err_unref;
+
+	engine->scratch = vma;
+	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
+			 engine->name, i915_ggtt_offset(vma));
+	return 0;
+
+err_unref:
+	i915_gem_object_put(obj);
+	return ret;
+}
+
+static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine)
+{
+	i915_vma_unpin_and_release(&engine->scratch);
+}
+
+/**
+ * intel_engines_init_common - initialize cengine state which might require hw access
+ * @engine: Engine to initialize.
+ *
+ * Initializes @engine@ structure members shared between legacy and execlists
+ * submission modes which do require hardware access.
+ *
+ * Typcally done at later stages of submission mode specific engine setup.
+ *
+ * Returns zero on success or an error code on failure.
+ */
+int intel_engine_init_common(struct intel_engine_cs *engine)
+{
+	int ret;
+
+	ret = intel_engine_init_breadcrumbs(engine);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+/**
+ * intel_engines_cleanup_common - cleans up the engine state created by
+ *                                the common initiailizers.
+ * @engine: Engine to cleanup.
+ *
+ * This cleans up everything created by the common helpers.
+ */
+void intel_engine_cleanup_common(struct intel_engine_cs *engine)
+{
+	intel_engine_cleanup_scratch(engine);
+
+	intel_engine_fini_breadcrumbs(engine);
+	intel_engine_cleanup_cmd_parser(engine);
+	i915_gem_batch_pool_fini(&engine->batch_pool);
+}
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index 6a7ad3ed1463..faa67624e1ed 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -190,9 +190,13 @@ static void g4x_fbc_activate(struct drm_i915_private *dev_priv)
 		dpfc_ctl |= DPFC_CTL_LIMIT_2X;
 	else
 		dpfc_ctl |= DPFC_CTL_LIMIT_1X;
-	dpfc_ctl |= DPFC_CTL_FENCE_EN | params->fb.fence_reg;
 
-	I915_WRITE(DPFC_FENCE_YOFF, params->crtc.fence_y_offset);
+	if (params->fb.fence_reg != I915_FENCE_REG_NONE) {
+		dpfc_ctl |= DPFC_CTL_FENCE_EN | params->fb.fence_reg;
+		I915_WRITE(DPFC_FENCE_YOFF, params->crtc.fence_y_offset);
+	} else {
+		I915_WRITE(DPFC_FENCE_YOFF, 0);
+	}
 
 	/* enable it... */
 	I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
@@ -244,21 +248,29 @@ static void ilk_fbc_activate(struct drm_i915_private *dev_priv)
 		dpfc_ctl |= DPFC_CTL_LIMIT_1X;
 		break;
 	}
-	dpfc_ctl |= DPFC_CTL_FENCE_EN;
-	if (IS_GEN5(dev_priv))
-		dpfc_ctl |= params->fb.fence_reg;
+
+	if (params->fb.fence_reg != I915_FENCE_REG_NONE) {
+		dpfc_ctl |= DPFC_CTL_FENCE_EN;
+		if (IS_GEN5(dev_priv))
+			dpfc_ctl |= params->fb.fence_reg;
+		if (IS_GEN6(dev_priv)) {
+			I915_WRITE(SNB_DPFC_CTL_SA,
+				   SNB_CPU_FENCE_ENABLE | params->fb.fence_reg);
+			I915_WRITE(DPFC_CPU_FENCE_OFFSET,
+				   params->crtc.fence_y_offset);
+		}
+	} else {
+		if (IS_GEN6(dev_priv)) {
+			I915_WRITE(SNB_DPFC_CTL_SA, 0);
+			I915_WRITE(DPFC_CPU_FENCE_OFFSET, 0);
+		}
+	}
 
 	I915_WRITE(ILK_DPFC_FENCE_YOFF, params->crtc.fence_y_offset);
 	I915_WRITE(ILK_FBC_RT_BASE, params->fb.ggtt_offset | ILK_FBC_RT_VALID);
 	/* enable it... */
 	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 
-	if (IS_GEN6(dev_priv)) {
-		I915_WRITE(SNB_DPFC_CTL_SA,
-			   SNB_CPU_FENCE_ENABLE | params->fb.fence_reg);
-		I915_WRITE(DPFC_CPU_FENCE_OFFSET, params->crtc.fence_y_offset);
-	}
-
 	intel_fbc_recompress(dev_priv);
 }
 
@@ -305,7 +317,15 @@ static void gen7_fbc_activate(struct drm_i915_private *dev_priv)
 		break;
 	}
 
-	dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN;
+	if (params->fb.fence_reg != I915_FENCE_REG_NONE) {
+		dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN;
+		I915_WRITE(SNB_DPFC_CTL_SA,
+			   SNB_CPU_FENCE_ENABLE | params->fb.fence_reg);
+		I915_WRITE(DPFC_CPU_FENCE_OFFSET, params->crtc.fence_y_offset);
+	} else {
+		I915_WRITE(SNB_DPFC_CTL_SA,0);
+		I915_WRITE(DPFC_CPU_FENCE_OFFSET, 0);
+	}
 
 	if (dev_priv->fbc.false_color)
 		dpfc_ctl |= FBC_CTL_FALSE_COLOR;
@@ -324,10 +344,6 @@ static void gen7_fbc_activate(struct drm_i915_private *dev_priv)
 
 	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 
-	I915_WRITE(SNB_DPFC_CTL_SA,
-		   SNB_CPU_FENCE_ENABLE | params->fb.fence_reg);
-	I915_WRITE(DPFC_CPU_FENCE_OFFSET, params->crtc.fence_y_offset);
-
 	intel_fbc_recompress(dev_priv);
 }
 
@@ -494,7 +510,7 @@ static bool multiple_pipes_ok(struct intel_crtc *crtc,
 	if (!no_fbc_on_multiple_pipes(dev_priv))
 		return true;
 
-	if (plane_state->visible)
+	if (plane_state->base.visible)
 		fbc->visible_pipes_mask |= (1 << pipe);
 	else
 		fbc->visible_pipes_mask &= ~(1 << pipe);
@@ -709,6 +725,14 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
 	return effective_w <= max_w && effective_h <= max_h;
 }
 
+/* XXX replace me when we have VMA tracking for intel_plane_state */
+static int get_fence_id(struct drm_framebuffer *fb)
+{
+	struct i915_vma *vma = i915_gem_object_to_ggtt(intel_fb_obj(fb), NULL);
+
+	return vma && vma->fence ? vma->fence->id : I915_FENCE_REG_NONE;
+}
+
 static void intel_fbc_update_state_cache(struct intel_crtc *crtc,
 					 struct intel_crtc_state *crtc_state,
 					 struct intel_plane_state *plane_state)
@@ -725,9 +749,9 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc,
 			ilk_pipe_pixel_rate(crtc_state);
 
 	cache->plane.rotation = plane_state->base.rotation;
-	cache->plane.src_w = drm_rect_width(&plane_state->src) >> 16;
-	cache->plane.src_h = drm_rect_height(&plane_state->src) >> 16;
-	cache->plane.visible = plane_state->visible;
+	cache->plane.src_w = drm_rect_width(&plane_state->base.src) >> 16;
+	cache->plane.src_h = drm_rect_height(&plane_state->base.src) >> 16;
+	cache->plane.visible = plane_state->base.visible;
 
 	if (!cache->plane.visible)
 		return;
@@ -737,11 +761,11 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc,
 	/* FIXME: We lack the proper locking here, so only run this on the
 	 * platforms that need. */
 	if (IS_GEN(dev_priv, 5, 6))
-		cache->fb.ilk_ggtt_offset = i915_gem_obj_ggtt_offset(obj);
+		cache->fb.ilk_ggtt_offset = i915_gem_object_ggtt_offset(obj, NULL);
 	cache->fb.pixel_format = fb->pixel_format;
 	cache->fb.stride = fb->pitches[0];
-	cache->fb.fence_reg = obj->fence_reg;
-	cache->fb.tiling_mode = obj->tiling_mode;
+	cache->fb.fence_reg = get_fence_id(fb);
+	cache->fb.tiling_mode = i915_gem_object_get_tiling(obj);
 }
 
 static bool intel_fbc_can_activate(struct intel_crtc *crtc)
@@ -768,6 +792,10 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
 
 	/* The use of a CPU fence is mandatory in order to detect writes
 	 * by the CPU to the scanout and trigger updates to the FBC.
+	 *
+	 * Note that is possible for a tiled surface to be unmappable (and
+	 * so have no fence associated with it) due to aperture constaints
+	 * at the time of pinning.
 	 */
 	if (cache->fb.tiling_mode != I915_TILING_X ||
 	    cache->fb.fence_reg == I915_FENCE_REG_NONE) {
@@ -775,7 +803,7 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
 		return false;
 	}
 	if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_G4X(dev_priv) &&
-	    cache->plane.rotation != BIT(DRM_ROTATE_0)) {
+	    cache->plane.rotation != DRM_ROTATE_0) {
 		fbc->no_fbc_reason = "rotation unsupported";
 		return false;
 	}
@@ -1050,7 +1078,7 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
 		struct intel_plane_state *intel_plane_state =
 			to_intel_plane_state(plane_state);
 
-		if (!intel_plane_state->visible)
+		if (!intel_plane_state->base.visible)
 			continue;
 
 		for_each_crtc_in_state(state, crtc, crtc_state, j) {
@@ -1075,6 +1103,8 @@ out:
 /**
  * intel_fbc_enable: tries to enable FBC on the CRTC
  * @crtc: the CRTC
+ * @crtc_state: corresponding &drm_crtc_state for @crtc
+ * @plane_state: corresponding &drm_plane_state for the primary plane of @crtc
  *
  * This function checks if the given CRTC was chosen for FBC, then enables it if
  * possible. Notice that it doesn't activate FBC. It is valid to call
@@ -1163,11 +1193,8 @@ void intel_fbc_disable(struct intel_crtc *crtc)
 		return;
 
 	mutex_lock(&fbc->lock);
-	if (fbc->crtc == crtc) {
-		WARN_ON(!fbc->enabled);
-		WARN_ON(fbc->active);
+	if (fbc->crtc == crtc)
 		__intel_fbc_disable(dev_priv);
-	}
 	mutex_unlock(&fbc->lock);
 
 	cancel_work_sync(&fbc->work.work);
@@ -1212,7 +1239,7 @@ void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv)
 
 	for_each_intel_crtc(&dev_priv->drm, crtc)
 		if (intel_crtc_active(&crtc->base) &&
-		    to_intel_plane_state(crtc->base.primary->state)->visible)
+		    to_intel_plane_state(crtc->base.primary->state)->base.visible)
 			dev_priv->fbc.visible_pipes_mask |= (1 << crtc->pipe);
 }
 
@@ -1230,12 +1257,29 @@ static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv)
 	if (i915.enable_fbc >= 0)
 		return !!i915.enable_fbc;
 
+	if (!HAS_FBC(dev_priv))
+		return 0;
+
 	if (IS_BROADWELL(dev_priv))
 		return 1;
 
 	return 0;
 }
 
+static bool need_fbc_vtd_wa(struct drm_i915_private *dev_priv)
+{
+#ifdef CONFIG_INTEL_IOMMU
+	/* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
+	if (intel_iommu_gfx_mapped &&
+	    (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))) {
+		DRM_INFO("Disabling framebuffer compression (FBC) to prevent screen flicker with VT-d enabled\n");
+		return true;
+	}
+#endif
+
+	return false;
+}
+
 /**
  * intel_fbc_init - Initialize FBC
  * @dev_priv: the i915 device
@@ -1253,6 +1297,9 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
 	fbc->active = false;
 	fbc->work.scheduled = false;
 
+	if (need_fbc_vtd_wa(dev_priv))
+		mkwrite_device_info(dev_priv)->has_fbc = false;
+
 	i915.enable_fbc = intel_sanitize_fbc_option(dev_priv);
 	DRM_DEBUG_KMS("Sanitized enable_fbc value: %d\n", i915.enable_fbc);
 
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index 86b00c6db1a6..b7098f98bb67 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -34,7 +34,6 @@
 #include <linux/tty.h>
 #include <linux/sysrq.h>
 #include <linux/delay.h>
-#include <linux/fb.h>
 #include <linux/init.h>
 #include <linux/vga_switcheroo.h>
 
@@ -42,6 +41,7 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_fb_helper.h>
 #include "intel_drv.h"
+#include "intel_frontbuffer.h"
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 
@@ -159,7 +159,7 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
 
 	fb = __intel_framebuffer_create(dev, &mode_cmd, obj);
 	if (IS_ERR(fb)) {
-		drm_gem_object_unreference(&obj->base);
+		i915_gem_object_put(obj);
 		ret = PTR_ERR(fb);
 		goto out;
 	}
@@ -183,13 +183,13 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	struct intel_framebuffer *intel_fb = ifbdev->fb;
 	struct drm_device *dev = helper->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct fb_info *info;
 	struct drm_framebuffer *fb;
 	struct i915_vma *vma;
-	struct drm_i915_gem_object *obj;
 	bool prealloc = false;
-	void *vaddr;
+	void __iomem *vaddr;
 	int ret;
 
 	if (intel_fb &&
@@ -215,17 +215,17 @@ static int intelfb_create(struct drm_fb_helper *helper,
 		sizes->fb_height = intel_fb->base.height;
 	}
 
-	obj = intel_fb->obj;
-
 	mutex_lock(&dev->struct_mutex);
 
 	/* Pin the GGTT vma for our access via info->screen_base.
 	 * This also validates that any existing fb inherited from the
 	 * BIOS is suitable for own access.
 	 */
-	ret = intel_pin_and_fence_fb_obj(&ifbdev->fb->base, BIT(DRM_ROTATE_0));
-	if (ret)
+	vma = intel_pin_and_fence_fb_obj(&ifbdev->fb->base, DRM_ROTATE_0);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
 		goto out_unlock;
+	}
 
 	info = drm_fb_helper_alloc_fbi(helper);
 	if (IS_ERR(info)) {
@@ -245,13 +245,11 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &intelfb_ops;
 
-	vma = i915_gem_obj_to_ggtt(obj);
-
 	/* setup aperture base/size for vesafb takeover */
 	info->apertures->ranges[0].base = dev->mode_config.fb_base;
 	info->apertures->ranges[0].size = ggtt->mappable_end;
 
-	info->fix.smem_start = dev->mode_config.fb_base + vma->node.start;
+	info->fix.smem_start = dev->mode_config.fb_base + i915_ggtt_offset(vma);
 	info->fix.smem_len = vma->node.size;
 
 	vaddr = i915_vma_pin_iomap(vma);
@@ -273,23 +271,23 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	 * If the object is stolen however, it will be full of whatever
 	 * garbage was left in there.
 	 */
-	if (ifbdev->fb->obj->stolen && !prealloc)
+	if (intel_fb->obj->stolen && !prealloc)
 		memset_io(info->screen_base, 0, info->screen_size);
 
 	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
 
-	DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08llx, bo %p\n",
-		      fb->width, fb->height,
-		      i915_gem_obj_ggtt_offset(obj), obj);
+	DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08x\n",
+		      fb->width, fb->height, i915_ggtt_offset(vma));
+	ifbdev->vma = vma;
 
 	mutex_unlock(&dev->struct_mutex);
-	vga_switcheroo_client_fb_set(dev->pdev, info);
+	vga_switcheroo_client_fb_set(pdev, info);
 	return 0;
 
 out_destroy_fbi:
 	drm_fb_helper_release_fbi(helper);
 out_unpin:
-	intel_unpin_fb_obj(&ifbdev->fb->base, BIT(DRM_ROTATE_0));
+	intel_unpin_fb_obj(&ifbdev->fb->base, DRM_ROTATE_0);
 out_unlock:
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
@@ -554,7 +552,7 @@ static void intel_fbdev_destroy(struct intel_fbdev *ifbdev)
 
 	if (ifbdev->fb) {
 		mutex_lock(&ifbdev->helper.dev->struct_mutex);
-		intel_unpin_fb_obj(&ifbdev->fb->base, BIT(DRM_ROTATE_0));
+		intel_unpin_fb_obj(&ifbdev->fb->base, DRM_ROTATE_0);
 		mutex_unlock(&ifbdev->helper.dev->struct_mutex);
 
 		drm_framebuffer_remove(&ifbdev->fb->base);
@@ -768,7 +766,7 @@ void intel_fbdev_fini(struct drm_device *dev)
 	if (!ifbdev)
 		return;
 
-	flush_work(&dev_priv->fbdev_suspend_work);
+	cancel_work_sync(&dev_priv->fbdev_suspend_work);
 	if (!current_is_async())
 		intel_fbdev_sync(ifbdev);
 
@@ -782,7 +780,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous
 	struct intel_fbdev *ifbdev = dev_priv->fbdev;
 	struct fb_info *info;
 
-	if (!ifbdev)
+	if (!ifbdev || !ifbdev->fb)
 		return;
 
 	info = ifbdev->helper.fbdev;
@@ -827,31 +825,28 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous
 
 void intel_fbdev_output_poll_changed(struct drm_device *dev)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	if (dev_priv->fbdev)
-		drm_fb_helper_hotplug_event(&dev_priv->fbdev->helper);
+	struct intel_fbdev *ifbdev = to_i915(dev)->fbdev;
+
+	if (ifbdev && ifbdev->fb)
+		drm_fb_helper_hotplug_event(&ifbdev->helper);
 }
 
 void intel_fbdev_restore_mode(struct drm_device *dev)
 {
-	int ret;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_fbdev *ifbdev = dev_priv->fbdev;
-	struct drm_fb_helper *fb_helper;
+	struct intel_fbdev *ifbdev = to_i915(dev)->fbdev;
 
 	if (!ifbdev)
 		return;
 
 	intel_fbdev_sync(ifbdev);
+	if (!ifbdev->fb)
+		return;
 
-	fb_helper = &ifbdev->helper;
-
-	ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper);
-	if (ret) {
+	if (drm_fb_helper_restore_fbdev_mode_unlocked(&ifbdev->helper)) {
 		DRM_DEBUG("failed to restore crtc mode\n");
 	} else {
-		mutex_lock(&fb_helper->dev->struct_mutex);
+		mutex_lock(&dev->struct_mutex);
 		intel_fb_obj_invalidate(ifbdev->fb->obj, ORIGIN_GTT);
-		mutex_unlock(&fb_helper->dev->struct_mutex);
+		mutex_unlock(&dev->struct_mutex);
 	}
 }
diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.c b/drivers/gpu/drm/i915/intel_frontbuffer.c
index ac85357010b4..966de4c7c7a2 100644
--- a/drivers/gpu/drm/i915/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/intel_frontbuffer.c
@@ -63,47 +63,30 @@
 #include <drm/drmP.h>
 
 #include "intel_drv.h"
+#include "intel_frontbuffer.h"
 #include "i915_drv.h"
 
-/**
- * intel_fb_obj_invalidate - invalidate frontbuffer object
- * @obj: GEM object to invalidate
- * @origin: which operation caused the invalidation
- *
- * This function gets called every time rendering on the given object starts and
- * frontbuffer caching (fbc, low refresh rate for DRRS, panel self refresh) must
- * be invalidated. For ORIGIN_CS any subsequent invalidation will be delayed
- * until the rendering completes or a flip on this frontbuffer plane is
- * scheduled.
- */
-void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
-			     enum fb_op_origin origin)
+void __intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
+			       enum fb_op_origin origin,
+			       unsigned int frontbuffer_bits)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
-
-	if (!obj->frontbuffer_bits)
-		return;
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 
 	if (origin == ORIGIN_CS) {
-		mutex_lock(&dev_priv->fb_tracking.lock);
-		dev_priv->fb_tracking.busy_bits
-			|= obj->frontbuffer_bits;
-		dev_priv->fb_tracking.flip_bits
-			&= ~obj->frontbuffer_bits;
-		mutex_unlock(&dev_priv->fb_tracking.lock);
+		spin_lock(&dev_priv->fb_tracking.lock);
+		dev_priv->fb_tracking.busy_bits |= frontbuffer_bits;
+		dev_priv->fb_tracking.flip_bits &= ~frontbuffer_bits;
+		spin_unlock(&dev_priv->fb_tracking.lock);
 	}
 
-	intel_psr_invalidate(dev, obj->frontbuffer_bits);
-	intel_edp_drrs_invalidate(dev, obj->frontbuffer_bits);
-	intel_fbc_invalidate(dev_priv, obj->frontbuffer_bits, origin);
+	intel_psr_invalidate(dev_priv, frontbuffer_bits);
+	intel_edp_drrs_invalidate(dev_priv, frontbuffer_bits);
+	intel_fbc_invalidate(dev_priv, frontbuffer_bits, origin);
 }
 
 /**
  * intel_frontbuffer_flush - flush frontbuffer
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
  * @origin: which operation caused the flush
  *
@@ -113,64 +96,45 @@ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
  *
  * Can be called without any locks held.
  */
-static void intel_frontbuffer_flush(struct drm_device *dev,
+static void intel_frontbuffer_flush(struct drm_i915_private *dev_priv,
 				    unsigned frontbuffer_bits,
 				    enum fb_op_origin origin)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/* Delay flushing when rings are still busy.*/
-	mutex_lock(&dev_priv->fb_tracking.lock);
+	spin_lock(&dev_priv->fb_tracking.lock);
 	frontbuffer_bits &= ~dev_priv->fb_tracking.busy_bits;
-	mutex_unlock(&dev_priv->fb_tracking.lock);
+	spin_unlock(&dev_priv->fb_tracking.lock);
 
 	if (!frontbuffer_bits)
 		return;
 
-	intel_edp_drrs_flush(dev, frontbuffer_bits);
-	intel_psr_flush(dev, frontbuffer_bits, origin);
+	intel_edp_drrs_flush(dev_priv, frontbuffer_bits);
+	intel_psr_flush(dev_priv, frontbuffer_bits, origin);
 	intel_fbc_flush(dev_priv, frontbuffer_bits, origin);
 }
 
-/**
- * intel_fb_obj_flush - flush frontbuffer object
- * @obj: GEM object to flush
- * @retire: set when retiring asynchronous rendering
- * @origin: which operation caused the flush
- *
- * This function gets called every time rendering on the given object has
- * completed and frontbuffer caching can be started again. If @retire is true
- * then any delayed flushes will be unblocked.
- */
-void intel_fb_obj_flush(struct drm_i915_gem_object *obj,
-			bool retire, enum fb_op_origin origin)
+void __intel_fb_obj_flush(struct drm_i915_gem_object *obj,
+			  bool retire,
+			  enum fb_op_origin origin,
+			  unsigned int frontbuffer_bits)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	unsigned frontbuffer_bits;
-
-	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
-
-	if (!obj->frontbuffer_bits)
-		return;
-
-	frontbuffer_bits = obj->frontbuffer_bits;
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 
 	if (retire) {
-		mutex_lock(&dev_priv->fb_tracking.lock);
+		spin_lock(&dev_priv->fb_tracking.lock);
 		/* Filter out new bits since rendering started. */
 		frontbuffer_bits &= dev_priv->fb_tracking.busy_bits;
-
 		dev_priv->fb_tracking.busy_bits &= ~frontbuffer_bits;
-		mutex_unlock(&dev_priv->fb_tracking.lock);
+		spin_unlock(&dev_priv->fb_tracking.lock);
 	}
 
-	intel_frontbuffer_flush(dev, frontbuffer_bits, origin);
+	if (frontbuffer_bits)
+		intel_frontbuffer_flush(dev_priv, frontbuffer_bits, origin);
 }
 
 /**
  * intel_frontbuffer_flip_prepare - prepare asynchronous frontbuffer flip
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
  *
  * This function gets called after scheduling a flip on @obj. The actual
@@ -180,23 +144,21 @@ void intel_fb_obj_flush(struct drm_i915_gem_object *obj,
  *
  * Can be called without any locks held.
  */
-void intel_frontbuffer_flip_prepare(struct drm_device *dev,
+void intel_frontbuffer_flip_prepare(struct drm_i915_private *dev_priv,
 				    unsigned frontbuffer_bits)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	mutex_lock(&dev_priv->fb_tracking.lock);
+	spin_lock(&dev_priv->fb_tracking.lock);
 	dev_priv->fb_tracking.flip_bits |= frontbuffer_bits;
 	/* Remove stale busy bits due to the old buffer. */
 	dev_priv->fb_tracking.busy_bits &= ~frontbuffer_bits;
-	mutex_unlock(&dev_priv->fb_tracking.lock);
+	spin_unlock(&dev_priv->fb_tracking.lock);
 
-	intel_psr_single_frame_update(dev, frontbuffer_bits);
+	intel_psr_single_frame_update(dev_priv, frontbuffer_bits);
 }
 
 /**
  * intel_frontbuffer_flip_complete - complete asynchronous frontbuffer flip
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
  *
  * This function gets called after the flip has been latched and will complete
@@ -204,23 +166,23 @@ void intel_frontbuffer_flip_prepare(struct drm_device *dev,
  *
  * Can be called without any locks held.
  */
-void intel_frontbuffer_flip_complete(struct drm_device *dev,
+void intel_frontbuffer_flip_complete(struct drm_i915_private *dev_priv,
 				     unsigned frontbuffer_bits)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	mutex_lock(&dev_priv->fb_tracking.lock);
+	spin_lock(&dev_priv->fb_tracking.lock);
 	/* Mask any cancelled flips. */
 	frontbuffer_bits &= dev_priv->fb_tracking.flip_bits;
 	dev_priv->fb_tracking.flip_bits &= ~frontbuffer_bits;
-	mutex_unlock(&dev_priv->fb_tracking.lock);
+	spin_unlock(&dev_priv->fb_tracking.lock);
 
-	intel_frontbuffer_flush(dev, frontbuffer_bits, ORIGIN_FLIP);
+	if (frontbuffer_bits)
+		intel_frontbuffer_flush(dev_priv,
+					frontbuffer_bits, ORIGIN_FLIP);
 }
 
 /**
  * intel_frontbuffer_flip - synchronous frontbuffer flip
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
  *
  * This function gets called after scheduling a flip on @obj. This is for
@@ -229,15 +191,13 @@ void intel_frontbuffer_flip_complete(struct drm_device *dev,
  *
  * Can be called without any locks held.
  */
-void intel_frontbuffer_flip(struct drm_device *dev,
+void intel_frontbuffer_flip(struct drm_i915_private *dev_priv,
 			    unsigned frontbuffer_bits)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	mutex_lock(&dev_priv->fb_tracking.lock);
+	spin_lock(&dev_priv->fb_tracking.lock);
 	/* Remove stale busy bits due to the old buffer. */
 	dev_priv->fb_tracking.busy_bits &= ~frontbuffer_bits;
-	mutex_unlock(&dev_priv->fb_tracking.lock);
+	spin_unlock(&dev_priv->fb_tracking.lock);
 
-	intel_frontbuffer_flush(dev, frontbuffer_bits, ORIGIN_FLIP);
+	intel_frontbuffer_flush(dev_priv, frontbuffer_bits, ORIGIN_FLIP);
 }
diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.h b/drivers/gpu/drm/i915/intel_frontbuffer.h
new file mode 100644
index 000000000000..76ceb539f9f0
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_frontbuffer.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2014-2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef __INTEL_FRONTBUFFER_H__
+#define __INTEL_FRONTBUFFER_H__
+
+struct drm_i915_private;
+struct drm_i915_gem_object;
+
+void intel_frontbuffer_flip_prepare(struct drm_i915_private *dev_priv,
+				    unsigned frontbuffer_bits);
+void intel_frontbuffer_flip_complete(struct drm_i915_private *dev_priv,
+				     unsigned frontbuffer_bits);
+void intel_frontbuffer_flip(struct drm_i915_private *dev_priv,
+			    unsigned frontbuffer_bits);
+
+void __intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
+			       enum fb_op_origin origin,
+			       unsigned int frontbuffer_bits);
+void __intel_fb_obj_flush(struct drm_i915_gem_object *obj,
+			  bool retire,
+			  enum fb_op_origin origin,
+			  unsigned int frontbuffer_bits);
+
+/**
+ * intel_fb_obj_invalidate - invalidate frontbuffer object
+ * @obj: GEM object to invalidate
+ * @origin: which operation caused the invalidation
+ *
+ * This function gets called every time rendering on the given object starts and
+ * frontbuffer caching (fbc, low refresh rate for DRRS, panel self refresh) must
+ * be invalidated. For ORIGIN_CS any subsequent invalidation will be delayed
+ * until the rendering completes or a flip on this frontbuffer plane is
+ * scheduled.
+ */
+static inline void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
+					   enum fb_op_origin origin)
+{
+	unsigned int frontbuffer_bits;
+
+	frontbuffer_bits = atomic_read(&obj->frontbuffer_bits);
+	if (!frontbuffer_bits)
+		return;
+
+	__intel_fb_obj_invalidate(obj, origin, frontbuffer_bits);
+}
+
+/**
+ * intel_fb_obj_flush - flush frontbuffer object
+ * @obj: GEM object to flush
+ * @retire: set when retiring asynchronous rendering
+ * @origin: which operation caused the flush
+ *
+ * This function gets called every time rendering on the given object has
+ * completed and frontbuffer caching can be started again. If @retire is true
+ * then any delayed flushes will be unblocked.
+ */
+static inline void intel_fb_obj_flush(struct drm_i915_gem_object *obj,
+				      bool retire,
+				      enum fb_op_origin origin)
+{
+	unsigned int frontbuffer_bits;
+
+	frontbuffer_bits = atomic_read(&obj->frontbuffer_bits);
+	if (!frontbuffer_bits)
+		return;
+
+	__intel_fb_obj_flush(obj, retire, origin, frontbuffer_bits);
+}
+
+#endif /* __INTEL_FRONTBUFFER_H__ */
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index 3e3e743740c0..5cdf7aa75be5 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -63,26 +63,27 @@ struct drm_i915_gem_request;
  *   retcode: errno from last guc_submit()
  */
 struct i915_guc_client {
-	struct drm_i915_gem_object *client_obj;
+	struct i915_vma *vma;
 	void *client_base;		/* first page (only) of above	*/
 	struct i915_gem_context *owner;
 	struct intel_guc *guc;
+
+	uint32_t engines;		/* bitmap of (host) engine ids	*/
 	uint32_t priority;
 	uint32_t ctx_index;
-
 	uint32_t proc_desc_offset;
+
 	uint32_t doorbell_offset;
 	uint32_t cookie;
 	uint16_t doorbell_id;
-	uint16_t padding;		/* Maintain alignment		*/
+	uint16_t padding[3];		/* Maintain alignment		*/
 
+	spinlock_t wq_lock;
 	uint32_t wq_offset;
 	uint32_t wq_size;
 	uint32_t wq_tail;
-	uint32_t unused;		/* Was 'wq_head'		*/
-
+	uint32_t wq_rsvd;
 	uint32_t no_wq_space;
-	uint32_t q_fail;		/* No longer used		*/
 	uint32_t b_fail;
 	int retcode;
 
@@ -125,11 +126,10 @@ struct intel_guc_fw {
 struct intel_guc {
 	struct intel_guc_fw guc_fw;
 	uint32_t log_flags;
-	struct drm_i915_gem_object *log_obj;
-
-	struct drm_i915_gem_object *ads_obj;
+	struct i915_vma *log_vma;
 
-	struct drm_i915_gem_object *ctx_pool_obj;
+	struct i915_vma *ads_vma;
+	struct i915_vma *ctx_pool_vma;
 	struct ida ctx_ids;
 
 	struct i915_guc_client *execbuf_client;
@@ -159,8 +159,8 @@ extern int intel_guc_resume(struct drm_device *dev);
 /* i915_guc_submission.c */
 int i915_guc_submission_init(struct drm_i915_private *dev_priv);
 int i915_guc_submission_enable(struct drm_i915_private *dev_priv);
-int i915_guc_wq_check_space(struct drm_i915_gem_request *rq);
-int i915_guc_submit(struct drm_i915_gem_request *rq);
+int i915_guc_wq_reserve(struct drm_i915_gem_request *rq);
+void i915_guc_wq_unreserve(struct drm_i915_gem_request *request);
 void i915_guc_submission_disable(struct drm_i915_private *dev_priv);
 void i915_guc_submission_fini(struct drm_i915_private *dev_priv);
 
diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h
index 944786d7075b..e40db2d2ae99 100644
--- a/drivers/gpu/drm/i915/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/intel_guc_fwif.h
@@ -155,6 +155,7 @@
  *
  *     +-------------------------------+
  *     |        guc_css_header         |
+ *     |                               |
  *     | contains major/minor version  |
  *     +-------------------------------+
  *     |             uCode             |
@@ -176,10 +177,10 @@
  *
  * 1. Header, uCode and RSA are must-have components.
  * 2. All firmware components, if they present, are in the sequence illustrated
- * in the layout table above.
+ *    in the layout table above.
  * 3. Length info of each component can be found in header, in dwords.
  * 4. Modulus and exponent key are not required by driver. They may not appear
- * in fw. So driver will load a truncated firmware in this case.
+ *    in fw. So driver will load a truncated firmware in this case.
  */
 
 struct guc_css_header {
diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c
index 605c69658d2c..6fd39efb7894 100644
--- a/drivers/gpu/drm/i915/intel_guc_loader.c
+++ b/drivers/gpu/drm/i915/intel_guc_loader.c
@@ -59,13 +59,25 @@
  *
  */
 
-#define I915_SKL_GUC_UCODE "i915/skl_guc_ver6_1.bin"
+#define SKL_FW_MAJOR 6
+#define SKL_FW_MINOR 1
+
+#define BXT_FW_MAJOR 8
+#define BXT_FW_MINOR 7
+
+#define KBL_FW_MAJOR 9
+#define KBL_FW_MINOR 14
+
+#define GUC_FW_PATH(platform, major, minor) \
+       "i915/" __stringify(platform) "_guc_ver" __stringify(major) "_" __stringify(minor) ".bin"
+
+#define I915_SKL_GUC_UCODE GUC_FW_PATH(skl, SKL_FW_MAJOR, SKL_FW_MINOR)
 MODULE_FIRMWARE(I915_SKL_GUC_UCODE);
 
-#define I915_BXT_GUC_UCODE "i915/bxt_guc_ver8_7.bin"
+#define I915_BXT_GUC_UCODE GUC_FW_PATH(bxt, BXT_FW_MAJOR, BXT_FW_MINOR)
 MODULE_FIRMWARE(I915_BXT_GUC_UCODE);
 
-#define I915_KBL_GUC_UCODE "i915/kbl_guc_ver9_14.bin"
+#define I915_KBL_GUC_UCODE GUC_FW_PATH(kbl, KBL_FW_MAJOR, KBL_FW_MINOR)
 MODULE_FIRMWARE(I915_KBL_GUC_UCODE);
 
 /* User-friendly representation of an enum */
@@ -85,7 +97,7 @@ const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status)
 	}
 };
 
-static void direct_interrupts_to_host(struct drm_i915_private *dev_priv)
+static void guc_interrupts_release(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 	int irqs;
@@ -102,7 +114,7 @@ static void direct_interrupts_to_host(struct drm_i915_private *dev_priv)
 	I915_WRITE(GUC_WD_VECS_IER, 0);
 }
 
-static void direct_interrupts_to_guc(struct drm_i915_private *dev_priv)
+static void guc_interrupts_capture(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 	int irqs;
@@ -122,13 +134,28 @@ static void direct_interrupts_to_guc(struct drm_i915_private *dev_priv)
 	I915_WRITE(GUC_WD_VECS_IER, ~irqs);
 
 	/*
-	 * If GuC has routed PM interrupts to itself, don't keep it.
-	 * and keep other interrupts those are unmasked by GuC.
-	*/
+	 * The REDIRECT_TO_GUC bit of the PMINTRMSK register directs all
+	 * (unmasked) PM interrupts to the GuC. All other bits of this
+	 * register *disable* generation of a specific interrupt.
+	 *
+	 * 'pm_intr_keep' indicates bits that are NOT to be set when
+	 * writing to the PM interrupt mask register, i.e. interrupts
+	 * that must not be disabled.
+	 *
+	 * If the GuC is handling these interrupts, then we must not let
+	 * the PM code disable ANY interrupt that the GuC is expecting.
+	 * So for each ENABLED (0) bit in this register, we must SET the
+	 * bit in pm_intr_keep so that it's left enabled for the GuC.
+	 *
+	 * OTOH the REDIRECT_TO_GUC bit is initially SET in pm_intr_keep
+	 * (so interrupts go to the DISPLAY unit at first); but here we
+	 * need to CLEAR that bit, which will result in the register bit
+	 * being left SET!
+	 */
 	tmp = I915_READ(GEN6_PMINTRMSK);
-	if (tmp & GEN8_PMINTR_REDIRECT_TO_NON_DISP) {
-		dev_priv->rps.pm_intr_keep |= ~(tmp & ~GEN8_PMINTR_REDIRECT_TO_NON_DISP);
-		dev_priv->rps.pm_intr_keep &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
+	if (tmp & GEN8_PMINTR_REDIRECT_TO_GUC) {
+		dev_priv->rps.pm_intr_keep |= ~tmp;
+		dev_priv->rps.pm_intr_keep &= ~GEN8_PMINTR_REDIRECT_TO_GUC;
 	}
 }
 
@@ -140,17 +167,24 @@ static u32 get_gttype(struct drm_i915_private *dev_priv)
 
 static u32 get_core_family(struct drm_i915_private *dev_priv)
 {
-	switch (INTEL_INFO(dev_priv)->gen) {
+	u32 gen = INTEL_GEN(dev_priv);
+
+	switch (gen) {
 	case 9:
 		return GFXCORE_FAMILY_GEN9;
 
 	default:
-		DRM_ERROR("GUC: unsupported core family\n");
+		WARN(1, "GEN%d does not support GuC operation!\n", gen);
 		return GFXCORE_FAMILY_UNKNOWN;
 	}
 }
 
-static void set_guc_init_params(struct drm_i915_private *dev_priv)
+/*
+ * Initialise the GuC parameter block before starting the firmware
+ * transfer. These parameters are read by the firmware on startup
+ * and cannot be changed thereafter.
+ */
+static void guc_params_init(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
 	u32 params[GUC_CTL_MAX_DWORDS];
@@ -181,16 +215,15 @@ static void set_guc_init_params(struct drm_i915_private *dev_priv)
 			i915.guc_log_level << GUC_LOG_VERBOSITY_SHIFT;
 	}
 
-	if (guc->ads_obj) {
-		u32 ads = (u32)i915_gem_obj_ggtt_offset(guc->ads_obj)
-				>> PAGE_SHIFT;
+	if (guc->ads_vma) {
+		u32 ads = i915_ggtt_offset(guc->ads_vma) >> PAGE_SHIFT;
 		params[GUC_CTL_DEBUG] |= ads << GUC_ADS_ADDR_SHIFT;
 		params[GUC_CTL_DEBUG] |= GUC_ADS_ENABLED;
 	}
 
 	/* If GuC submission is enabled, set up additional parameters here */
 	if (i915.enable_guc_submission) {
-		u32 pgs = i915_gem_obj_ggtt_offset(dev_priv->guc.ctx_pool_obj);
+		u32 pgs = i915_ggtt_offset(dev_priv->guc.ctx_pool_vma);
 		u32 ctx_in_16 = GUC_MAX_GPU_CONTEXTS / 16;
 
 		pgs >>= PAGE_SHIFT;
@@ -238,12 +271,12 @@ static inline bool guc_ucode_response(struct drm_i915_private *dev_priv,
  * Note that GuC needs the CSS header plus uKernel code to be copied by the
  * DMA engine in one operation, whereas the RSA signature is loaded via MMIO.
  */
-static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv)
+static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv,
+			      struct i915_vma *vma)
 {
 	struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
-	struct drm_i915_gem_object *fw_obj = guc_fw->guc_fw_obj;
 	unsigned long offset;
-	struct sg_table *sg = fw_obj->pages;
+	struct sg_table *sg = vma->pages;
 	u32 status, rsa[UOS_RSA_SCRATCH_MAX_COUNT];
 	int i, ret = 0;
 
@@ -260,7 +293,7 @@ static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv)
 	I915_WRITE(DMA_COPY_SIZE, guc_fw->header_size + guc_fw->ucode_size);
 
 	/* Set the source address for the new blob */
-	offset = i915_gem_obj_ggtt_offset(fw_obj) + guc_fw->header_offset;
+	offset = i915_ggtt_offset(vma) + guc_fw->header_offset;
 	I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset));
 	I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF);
 
@@ -315,6 +348,7 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
 	struct drm_device *dev = &dev_priv->drm;
+	struct i915_vma *vma;
 	int ret;
 
 	ret = i915_gem_object_set_to_gtt_domain(guc_fw->guc_fw_obj, false);
@@ -323,10 +357,10 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 		return ret;
 	}
 
-	ret = i915_gem_obj_ggtt_pin(guc_fw->guc_fw_obj, 0, 0);
-	if (ret) {
-		DRM_DEBUG_DRIVER("pin failed %d\n", ret);
-		return ret;
+	vma = i915_gem_object_ggtt_pin(guc_fw->guc_fw_obj, NULL, 0, 0, 0);
+	if (IS_ERR(vma)) {
+		DRM_DEBUG_DRIVER("pin failed %d\n", (int)PTR_ERR(vma));
+		return PTR_ERR(vma);
 	}
 
 	/* Invalidate GuC TLB to let GuC take the latest updates to GTT. */
@@ -349,7 +383,9 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 	}
 
 	/* WaC6DisallowByGfxPause*/
-	I915_WRITE(GEN6_GFXPAUSE, 0x30FFF);
+	if (IS_SKL_REVID(dev, 0, SKL_REVID_C0) ||
+	    IS_BXT_REVID(dev, 0, BXT_REVID_B0))
+		I915_WRITE(GEN6_GFXPAUSE, 0x30FFF);
 
 	if (IS_BROXTON(dev))
 		I915_WRITE(GEN9LP_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
@@ -361,13 +397,13 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 		I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE |
 					    I915_READ(GEN7_MISCCPCTL)));
 
-		/* allows for 5us before GT can go to RC6 */
+		/* allows for 5us (in 10ns units) before GT can go to RC6 */
 		I915_WRITE(GUC_ARAT_C6DIS, 0x1FF);
 	}
 
-	set_guc_init_params(dev_priv);
+	guc_params_init(dev_priv);
 
-	ret = guc_ucode_xfer_dma(dev_priv);
+	ret = guc_ucode_xfer_dma(dev_priv, vma);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
@@ -375,12 +411,12 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 	 * We keep the object pages for reuse during resume. But we can unpin it
 	 * now that DMA has completed, so it doesn't continue to take up space.
 	 */
-	i915_gem_object_ggtt_unpin(guc_fw->guc_fw_obj);
+	i915_vma_unpin(vma);
 
 	return ret;
 }
 
-static int i915_reset_guc(struct drm_i915_private *dev_priv)
+static int guc_hw_reset(struct drm_i915_private *dev_priv)
 {
 	int ret;
 	u32 guc_status;
@@ -433,7 +469,7 @@ int intel_guc_setup(struct drm_device *dev)
 		goto fail;
 	} else if (*fw_path == '\0') {
 		/* Device has a GuC but we don't know what f/w to load? */
-		DRM_INFO("No GuC firmware known for this platform\n");
+		WARN(1, "No GuC firmware known for this platform!\n");
 		err = -ENODEV;
 		goto fail;
 	}
@@ -447,7 +483,7 @@ int intel_guc_setup(struct drm_device *dev)
 		goto fail;
 	}
 
-	direct_interrupts_to_host(dev_priv);
+	guc_interrupts_release(dev_priv);
 
 	guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING;
 
@@ -470,11 +506,9 @@ int intel_guc_setup(struct drm_device *dev)
 		 * Always reset the GuC just before (re)loading, so
 		 * that the state and timing are fairly predictable
 		 */
-		err = i915_reset_guc(dev_priv);
-		if (err) {
-			DRM_ERROR("GuC reset failed: %d\n", err);
+		err = guc_hw_reset(dev_priv);
+		if (err)
 			goto fail;
-		}
 
 		err = guc_ucode_xfer(dev_priv);
 		if (!err)
@@ -497,7 +531,7 @@ int intel_guc_setup(struct drm_device *dev)
 		err = i915_guc_submission_enable(dev_priv);
 		if (err)
 			goto fail;
-		direct_interrupts_to_guc(dev_priv);
+		guc_interrupts_capture(dev_priv);
 	}
 
 	return 0;
@@ -506,7 +540,7 @@ fail:
 	if (guc_fw->guc_fw_load_status == GUC_FIRMWARE_PENDING)
 		guc_fw->guc_fw_load_status = GUC_FIRMWARE_FAIL;
 
-	direct_interrupts_to_host(dev_priv);
+	guc_interrupts_release(dev_priv);
 	i915_guc_submission_disable(dev_priv);
 	i915_guc_submission_fini(dev_priv);
 
@@ -532,15 +566,15 @@ fail:
 	else if (err == 0)
 		DRM_INFO("GuC firmware load skipped\n");
 	else if (ret != -EIO)
-		DRM_INFO("GuC firmware load failed: %d\n", err);
+		DRM_NOTE("GuC firmware load failed: %d\n", err);
 	else
-		DRM_ERROR("GuC firmware load failed: %d\n", err);
+		DRM_WARN("GuC firmware load failed: %d\n", err);
 
 	if (i915.enable_guc_submission) {
 		if (fw_path == NULL)
 			DRM_INFO("GuC submission without firmware not supported\n");
 		if (ret == 0)
-			DRM_INFO("Falling back from GuC submission to execlist mode\n");
+			DRM_NOTE("Falling back from GuC submission to execlist mode\n");
 		else
 			DRM_ERROR("GuC init failed: %d\n", ret);
 	}
@@ -551,6 +585,7 @@ fail:
 
 static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 {
+	struct pci_dev *pdev = dev->pdev;
 	struct drm_i915_gem_object *obj;
 	const struct firmware *fw;
 	struct guc_css_header *css;
@@ -560,7 +595,7 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 	DRM_DEBUG_DRIVER("before requesting firmware: GuC fw fetch status %s\n",
 		intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status));
 
-	err = request_firmware(&fw, guc_fw->guc_fw_path, &dev->pdev->dev);
+	err = request_firmware(&fw, guc_fw->guc_fw_path, &pdev->dev);
 	if (err)
 		goto fail;
 	if (!fw)
@@ -571,7 +606,7 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 
 	/* Check the size of the blob before examining buffer contents */
 	if (fw->size < sizeof(struct guc_css_header)) {
-		DRM_ERROR("Firmware header is missing\n");
+		DRM_NOTE("Firmware header is missing\n");
 		goto fail;
 	}
 
@@ -583,7 +618,7 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 		css->key_size_dw - css->exponent_size_dw) * sizeof(u32);
 
 	if (guc_fw->header_size != sizeof(struct guc_css_header)) {
-		DRM_ERROR("CSS header definition mismatch\n");
+		DRM_NOTE("CSS header definition mismatch\n");
 		goto fail;
 	}
 
@@ -593,7 +628,7 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 
 	/* now RSA */
 	if (css->key_size_dw != UOS_RSA_SCRATCH_MAX_COUNT) {
-		DRM_ERROR("RSA key size is bad\n");
+		DRM_NOTE("RSA key size is bad\n");
 		goto fail;
 	}
 	guc_fw->rsa_offset = guc_fw->ucode_offset + guc_fw->ucode_size;
@@ -602,14 +637,14 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 	/* At least, it should have header, uCode and RSA. Size of all three. */
 	size = guc_fw->header_size + guc_fw->ucode_size + guc_fw->rsa_size;
 	if (fw->size < size) {
-		DRM_ERROR("Missing firmware components\n");
+		DRM_NOTE("Missing firmware components\n");
 		goto fail;
 	}
 
 	/* Header and uCode will be loaded to WOPCM. Size of the two. */
 	size = guc_fw->header_size + guc_fw->ucode_size;
 	if (size > guc_wopcm_size(to_i915(dev))) {
-		DRM_ERROR("Firmware is too large to fit in WOPCM\n");
+		DRM_NOTE("Firmware is too large to fit in WOPCM\n");
 		goto fail;
 	}
 
@@ -624,7 +659,7 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 
 	if (guc_fw->guc_fw_major_found != guc_fw->guc_fw_major_wanted ||
 	    guc_fw->guc_fw_minor_found < guc_fw->guc_fw_minor_wanted) {
-		DRM_ERROR("GuC firmware version %d.%d, required %d.%d\n",
+		DRM_NOTE("GuC firmware version %d.%d, required %d.%d\n",
 			guc_fw->guc_fw_major_found, guc_fw->guc_fw_minor_found,
 			guc_fw->guc_fw_major_wanted, guc_fw->guc_fw_minor_wanted);
 		err = -ENOEXEC;
@@ -654,15 +689,15 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 	return;
 
 fail:
+	DRM_WARN("Failed to fetch valid GuC firmware from %s (error %d)\n",
+		 guc_fw->guc_fw_path, err);
 	DRM_DEBUG_DRIVER("GuC fw fetch status FAIL; err %d, fw %p, obj %p\n",
 		err, fw, guc_fw->guc_fw_obj);
-	DRM_ERROR("Failed to fetch GuC firmware from %s (error %d)\n",
-		  guc_fw->guc_fw_path, err);
 
 	mutex_lock(&dev->struct_mutex);
 	obj = guc_fw->guc_fw_obj;
 	if (obj)
-		drm_gem_object_unreference(&obj->base);
+		i915_gem_object_put(obj);
 	guc_fw->guc_fw_obj = NULL;
 	mutex_unlock(&dev->struct_mutex);
 
@@ -695,16 +730,16 @@ void intel_guc_init(struct drm_device *dev)
 		fw_path = NULL;
 	} else if (IS_SKYLAKE(dev)) {
 		fw_path = I915_SKL_GUC_UCODE;
-		guc_fw->guc_fw_major_wanted = 6;
-		guc_fw->guc_fw_minor_wanted = 1;
+		guc_fw->guc_fw_major_wanted = SKL_FW_MAJOR;
+		guc_fw->guc_fw_minor_wanted = SKL_FW_MINOR;
 	} else if (IS_BROXTON(dev)) {
 		fw_path = I915_BXT_GUC_UCODE;
-		guc_fw->guc_fw_major_wanted = 8;
-		guc_fw->guc_fw_minor_wanted = 7;
+		guc_fw->guc_fw_major_wanted = BXT_FW_MAJOR;
+		guc_fw->guc_fw_minor_wanted = BXT_FW_MINOR;
 	} else if (IS_KABYLAKE(dev)) {
 		fw_path = I915_KBL_GUC_UCODE;
-		guc_fw->guc_fw_major_wanted = 9;
-		guc_fw->guc_fw_minor_wanted = 14;
+		guc_fw->guc_fw_major_wanted = KBL_FW_MAJOR;
+		guc_fw->guc_fw_minor_wanted = KBL_FW_MINOR;
 	} else {
 		fw_path = "";	/* unknown device */
 	}
@@ -738,12 +773,12 @@ void intel_guc_fini(struct drm_device *dev)
 	struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
 
 	mutex_lock(&dev->struct_mutex);
-	direct_interrupts_to_host(dev_priv);
+	guc_interrupts_release(dev_priv);
 	i915_guc_submission_disable(dev_priv);
 	i915_guc_submission_fini(dev_priv);
 
 	if (guc_fw->guc_fw_obj)
-		drm_gem_object_unreference(&guc_fw->guc_fw_obj->base);
+		i915_gem_object_put(guc_fw->guc_fw_obj);
 	guc_fw->guc_fw_obj = NULL;
 	mutex_unlock(&dev->struct_mutex);
 
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 4df9f384910c..f40a35f2913a 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -985,7 +985,9 @@ static void intel_enable_hdmi_audio(struct intel_encoder *encoder)
 	intel_audio_codec_enable(encoder);
 }
 
-static void g4x_enable_hdmi(struct intel_encoder *encoder)
+static void g4x_enable_hdmi(struct intel_encoder *encoder,
+			    struct intel_crtc_state *pipe_config,
+			    struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -1006,7 +1008,9 @@ static void g4x_enable_hdmi(struct intel_encoder *encoder)
 		intel_enable_hdmi_audio(encoder);
 }
 
-static void ibx_enable_hdmi(struct intel_encoder *encoder)
+static void ibx_enable_hdmi(struct intel_encoder *encoder,
+			    struct intel_crtc_state *pipe_config,
+			    struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -1055,7 +1059,9 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder)
 		intel_enable_hdmi_audio(encoder);
 }
 
-static void cpt_enable_hdmi(struct intel_encoder *encoder)
+static void cpt_enable_hdmi(struct intel_encoder *encoder,
+			    struct intel_crtc_state *pipe_config,
+			    struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -1108,11 +1114,15 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder)
 		intel_enable_hdmi_audio(encoder);
 }
 
-static void vlv_enable_hdmi(struct intel_encoder *encoder)
+static void vlv_enable_hdmi(struct intel_encoder *encoder,
+			    struct intel_crtc_state *pipe_config,
+			    struct drm_connector_state *conn_state)
 {
 }
 
-static void intel_disable_hdmi(struct intel_encoder *encoder)
+static void intel_disable_hdmi(struct intel_encoder *encoder,
+			       struct intel_crtc_state *old_crtc_state,
+			       struct drm_connector_state *old_conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -1164,17 +1174,21 @@ static void intel_disable_hdmi(struct intel_encoder *encoder)
 	intel_dp_dual_mode_set_tmds_output(intel_hdmi, false);
 }
 
-static void g4x_disable_hdmi(struct intel_encoder *encoder)
+static void g4x_disable_hdmi(struct intel_encoder *encoder,
+			     struct intel_crtc_state *old_crtc_state,
+			     struct drm_connector_state *old_conn_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 
 	if (crtc->config->has_audio)
 		intel_audio_codec_disable(encoder);
 
-	intel_disable_hdmi(encoder);
+	intel_disable_hdmi(encoder, old_crtc_state, old_conn_state);
 }
 
-static void pch_disable_hdmi(struct intel_encoder *encoder)
+static void pch_disable_hdmi(struct intel_encoder *encoder,
+			     struct intel_crtc_state *old_crtc_state,
+			     struct drm_connector_state *old_conn_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 
@@ -1182,9 +1196,11 @@ static void pch_disable_hdmi(struct intel_encoder *encoder)
 		intel_audio_codec_disable(encoder);
 }
 
-static void pch_post_disable_hdmi(struct intel_encoder *encoder)
+static void pch_post_disable_hdmi(struct intel_encoder *encoder,
+				  struct intel_crtc_state *old_crtc_state,
+				  struct drm_connector_state *old_conn_state)
 {
-	intel_disable_hdmi(encoder);
+	intel_disable_hdmi(encoder, old_crtc_state, old_conn_state);
 }
 
 static int intel_hdmi_source_max_tmds_clock(struct drm_i915_private *dev_priv)
@@ -1204,10 +1220,17 @@ static int hdmi_port_clock_limit(struct intel_hdmi *hdmi,
 	int max_tmds_clock = intel_hdmi_source_max_tmds_clock(to_i915(dev));
 
 	if (respect_downstream_limits) {
+		struct intel_connector *connector = hdmi->attached_connector;
+		const struct drm_display_info *info = &connector->base.display_info;
+
 		if (hdmi->dp_dual_mode.max_tmds_clock)
 			max_tmds_clock = min(max_tmds_clock,
 					     hdmi->dp_dual_mode.max_tmds_clock);
-		if (!hdmi->has_hdmi_sink)
+
+		if (info->max_tmds_clock)
+			max_tmds_clock = min(max_tmds_clock,
+					     info->max_tmds_clock);
+		else if (!hdmi->has_hdmi_sink)
 			max_tmds_clock = min(max_tmds_clock, 165000);
 	}
 
@@ -1285,7 +1308,8 @@ static bool hdmi_12bpc_possible(struct intel_crtc_state *crtc_state)
 }
 
 bool intel_hdmi_compute_config(struct intel_encoder *encoder,
-			       struct intel_crtc_state *pipe_config)
+			       struct intel_crtc_state *pipe_config,
+			       struct drm_connector_state *conn_state)
 {
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
 	struct drm_device *dev = encoder->base.dev;
@@ -1422,24 +1446,22 @@ intel_hdmi_dp_dual_mode_detect(struct drm_connector *connector, bool has_edid)
 }
 
 static bool
-intel_hdmi_set_edid(struct drm_connector *connector, bool force)
+intel_hdmi_set_edid(struct drm_connector *connector)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->dev);
 	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
-	struct edid *edid = NULL;
+	struct edid *edid;
 	bool connected = false;
 
-	if (force) {
-		intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
+	intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
 
-		edid = drm_get_edid(connector,
-				    intel_gmbus_get_adapter(dev_priv,
-				    intel_hdmi->ddc_bus));
+	edid = drm_get_edid(connector,
+			    intel_gmbus_get_adapter(dev_priv,
+			    intel_hdmi->ddc_bus));
 
-		intel_hdmi_dp_dual_mode_detect(connector, edid != NULL);
+	intel_hdmi_dp_dual_mode_detect(connector, edid != NULL);
 
-		intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS);
-	}
+	intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS);
 
 	to_intel_connector(connector)->detect_edid = edid;
 	if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) {
@@ -1465,37 +1487,16 @@ static enum drm_connector_status
 intel_hdmi_detect(struct drm_connector *connector, bool force)
 {
 	enum drm_connector_status status;
-	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
 	struct drm_i915_private *dev_priv = to_i915(connector->dev);
-	bool live_status = false;
-	unsigned int try;
 
 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
 		      connector->base.id, connector->name);
 
 	intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
 
-	for (try = 0; !live_status && try < 9; try++) {
-		if (try)
-			msleep(10);
-		live_status = intel_digital_port_connected(dev_priv,
-				hdmi_to_dig_port(intel_hdmi));
-	}
-
-	if (!live_status) {
-		DRM_DEBUG_KMS("HDMI live status down\n");
-		/*
-		 * Live status register is not reliable on all intel platforms.
-		 * So consider live_status only for certain platforms, for
-		 * others, read EDID to determine presence of sink.
-		 */
-		if (INTEL_INFO(dev_priv)->gen < 7 || IS_IVYBRIDGE(dev_priv))
-			live_status = true;
-	}
-
 	intel_hdmi_unset_edid(connector);
 
-	if (intel_hdmi_set_edid(connector, live_status)) {
+	if (intel_hdmi_set_edid(connector)) {
 		struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
 
 		hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
@@ -1521,7 +1522,7 @@ intel_hdmi_force(struct drm_connector *connector)
 	if (connector->status != connector_status_connected)
 		return;
 
-	intel_hdmi_set_edid(connector, true);
+	intel_hdmi_set_edid(connector);
 	hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
 }
 
@@ -1638,7 +1639,9 @@ done:
 	return 0;
 }
 
-static void intel_hdmi_pre_enable(struct intel_encoder *encoder)
+static void intel_hdmi_pre_enable(struct intel_encoder *encoder,
+				  struct intel_crtc_state *pipe_config,
+				  struct drm_connector_state *conn_state)
 {
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
@@ -1651,7 +1654,9 @@ static void intel_hdmi_pre_enable(struct intel_encoder *encoder)
 				   adjusted_mode);
 }
 
-static void vlv_hdmi_pre_enable(struct intel_encoder *encoder)
+static void vlv_hdmi_pre_enable(struct intel_encoder *encoder,
+				struct intel_crtc_state *pipe_config,
+				struct drm_connector_state *conn_state)
 {
 	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
 	struct intel_hdmi *intel_hdmi = &dport->hdmi;
@@ -1671,37 +1676,47 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder)
 				   intel_crtc->config->has_hdmi_sink,
 				   adjusted_mode);
 
-	g4x_enable_hdmi(encoder);
+	g4x_enable_hdmi(encoder, pipe_config, conn_state);
 
 	vlv_wait_port_ready(dev_priv, dport, 0x0);
 }
 
-static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder)
+static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder,
+				    struct intel_crtc_state *pipe_config,
+				    struct drm_connector_state *conn_state)
 {
 	intel_hdmi_prepare(encoder);
 
 	vlv_phy_pre_pll_enable(encoder);
 }
 
-static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder)
+static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder,
+				    struct intel_crtc_state *pipe_config,
+				    struct drm_connector_state *conn_state)
 {
 	intel_hdmi_prepare(encoder);
 
 	chv_phy_pre_pll_enable(encoder);
 }
 
-static void chv_hdmi_post_pll_disable(struct intel_encoder *encoder)
+static void chv_hdmi_post_pll_disable(struct intel_encoder *encoder,
+				      struct intel_crtc_state *old_crtc_state,
+				      struct drm_connector_state *old_conn_state)
 {
 	chv_phy_post_pll_disable(encoder);
 }
 
-static void vlv_hdmi_post_disable(struct intel_encoder *encoder)
+static void vlv_hdmi_post_disable(struct intel_encoder *encoder,
+				  struct intel_crtc_state *old_crtc_state,
+				  struct drm_connector_state *old_conn_state)
 {
 	/* Reset lanes to avoid HDMI flicker (VLV w/a) */
 	vlv_phy_reset_lanes(encoder);
 }
 
-static void chv_hdmi_post_disable(struct intel_encoder *encoder)
+static void chv_hdmi_post_disable(struct intel_encoder *encoder,
+				  struct intel_crtc_state *old_crtc_state,
+				  struct drm_connector_state *old_conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -1714,7 +1729,9 @@ static void chv_hdmi_post_disable(struct intel_encoder *encoder)
 	mutex_unlock(&dev_priv->sb_lock);
 }
 
-static void chv_hdmi_pre_enable(struct intel_encoder *encoder)
+static void chv_hdmi_pre_enable(struct intel_encoder *encoder,
+				struct intel_crtc_state *pipe_config,
+				struct drm_connector_state *conn_state)
 {
 	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
 	struct intel_hdmi *intel_hdmi = &dport->hdmi;
@@ -1734,7 +1751,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder)
 				   intel_crtc->config->has_hdmi_sink,
 				   adjusted_mode);
 
-	g4x_enable_hdmi(encoder);
+	g4x_enable_hdmi(encoder, pipe_config, conn_state);
 
 	vlv_wait_port_ready(dev_priv, dport, 0x0);
 
diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c
index f48957ea100d..334d47b5811a 100644
--- a/drivers/gpu/drm/i915/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/intel_hotplug.c
@@ -477,7 +477,8 @@ void intel_hpd_init(struct drm_i915_private *dev_priv)
 	spin_unlock_irq(&dev_priv->irq_lock);
 }
 
-void i915_hpd_poll_init_work(struct work_struct *work) {
+static void i915_hpd_poll_init_work(struct work_struct *work)
+{
 	struct drm_i915_private *dev_priv =
 		container_of(work, struct drm_i915_private,
 			     hotplug.poll_init_work);
@@ -525,7 +526,6 @@ void i915_hpd_poll_init_work(struct work_struct *work) {
 /**
  * intel_hpd_poll_init - enables/disables polling for connectors with hpd
  * @dev_priv: i915 device instance
- * @enabled: Whether to enable or disable polling
  *
  * This function enables polling for all connectors, regardless of whether or
  * not they support hotplug detection. Under certain conditions HPD may not be
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 1f266d7df2ec..79aab9ad6faa 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -255,67 +255,59 @@ intel_gpio_setup(struct intel_gmbus *bus, unsigned int pin)
 	algo->data = bus;
 }
 
-static int
-gmbus_wait_hw_status(struct drm_i915_private *dev_priv,
-		     u32 gmbus2_status,
-		     u32 gmbus4_irq_en)
+static int gmbus_wait(struct drm_i915_private *dev_priv, u32 status, u32 irq_en)
 {
-	int i;
-	u32 gmbus2 = 0;
 	DEFINE_WAIT(wait);
-
-	if (!HAS_GMBUS_IRQ(dev_priv))
-		gmbus4_irq_en = 0;
+	u32 gmbus2;
+	int ret;
 
 	/* Important: The hw handles only the first bit, so set only one! Since
 	 * we also need to check for NAKs besides the hw ready/idle signal, we
-	 * need to wake up periodically and check that ourselves. */
-	I915_WRITE(GMBUS4, gmbus4_irq_en);
-
-	for (i = 0; i < msecs_to_jiffies_timeout(50); i++) {
-		prepare_to_wait(&dev_priv->gmbus_wait_queue, &wait,
-				TASK_UNINTERRUPTIBLE);
+	 * need to wake up periodically and check that ourselves.
+	 */
+	if (!HAS_GMBUS_IRQ(dev_priv))
+		irq_en = 0;
 
-		gmbus2 = I915_READ_NOTRACE(GMBUS2);
-		if (gmbus2 & (GMBUS_SATOER | gmbus2_status))
-			break;
+	add_wait_queue(&dev_priv->gmbus_wait_queue, &wait);
+	I915_WRITE_FW(GMBUS4, irq_en);
 
-		schedule_timeout(1);
-	}
-	finish_wait(&dev_priv->gmbus_wait_queue, &wait);
+	status |= GMBUS_SATOER;
+	ret = wait_for_us((gmbus2 = I915_READ_FW(GMBUS2)) & status, 2);
+	if (ret)
+		ret = wait_for((gmbus2 = I915_READ_FW(GMBUS2)) & status, 50);
 
-	I915_WRITE(GMBUS4, 0);
+	I915_WRITE_FW(GMBUS4, 0);
+	remove_wait_queue(&dev_priv->gmbus_wait_queue, &wait);
 
 	if (gmbus2 & GMBUS_SATOER)
 		return -ENXIO;
-	if (gmbus2 & gmbus2_status)
-		return 0;
-	return -ETIMEDOUT;
+
+	return ret;
 }
 
 static int
 gmbus_wait_idle(struct drm_i915_private *dev_priv)
 {
+	DEFINE_WAIT(wait);
+	u32 irq_enable;
 	int ret;
 
-	if (!HAS_GMBUS_IRQ(dev_priv))
-		return intel_wait_for_register(dev_priv,
-					       GMBUS2, GMBUS_ACTIVE, 0,
-					       10);
-
 	/* Important: The hw handles only the first bit, so set only one! */
-	I915_WRITE(GMBUS4, GMBUS_IDLE_EN);
+	irq_enable = 0;
+	if (HAS_GMBUS_IRQ(dev_priv))
+		irq_enable = GMBUS_IDLE_EN;
 
-	ret = wait_event_timeout(dev_priv->gmbus_wait_queue,
-				 (I915_READ_NOTRACE(GMBUS2) & GMBUS_ACTIVE) == 0,
-				 msecs_to_jiffies_timeout(10));
+	add_wait_queue(&dev_priv->gmbus_wait_queue, &wait);
+	I915_WRITE_FW(GMBUS4, irq_enable);
 
-	I915_WRITE(GMBUS4, 0);
+	ret = intel_wait_for_register_fw(dev_priv,
+					 GMBUS2, GMBUS_ACTIVE, 0,
+					 10);
 
-	if (ret)
-		return 0;
-	else
-		return -ETIMEDOUT;
+	I915_WRITE_FW(GMBUS4, 0);
+	remove_wait_queue(&dev_priv->gmbus_wait_queue, &wait);
+
+	return ret;
 }
 
 static int
@@ -323,22 +315,21 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv,
 		      unsigned short addr, u8 *buf, unsigned int len,
 		      u32 gmbus1_index)
 {
-	I915_WRITE(GMBUS1,
-		   gmbus1_index |
-		   GMBUS_CYCLE_WAIT |
-		   (len << GMBUS_BYTE_COUNT_SHIFT) |
-		   (addr << GMBUS_SLAVE_ADDR_SHIFT) |
-		   GMBUS_SLAVE_READ | GMBUS_SW_RDY);
+	I915_WRITE_FW(GMBUS1,
+		      gmbus1_index |
+		      GMBUS_CYCLE_WAIT |
+		      (len << GMBUS_BYTE_COUNT_SHIFT) |
+		      (addr << GMBUS_SLAVE_ADDR_SHIFT) |
+		      GMBUS_SLAVE_READ | GMBUS_SW_RDY);
 	while (len) {
 		int ret;
 		u32 val, loop = 0;
 
-		ret = gmbus_wait_hw_status(dev_priv, GMBUS_HW_RDY,
-					   GMBUS_HW_RDY_EN);
+		ret = gmbus_wait(dev_priv, GMBUS_HW_RDY, GMBUS_HW_RDY_EN);
 		if (ret)
 			return ret;
 
-		val = I915_READ(GMBUS3);
+		val = I915_READ_FW(GMBUS3);
 		do {
 			*buf++ = val & 0xff;
 			val >>= 8;
@@ -385,12 +376,12 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv,
 		len -= 1;
 	}
 
-	I915_WRITE(GMBUS3, val);
-	I915_WRITE(GMBUS1,
-		   GMBUS_CYCLE_WAIT |
-		   (chunk_size << GMBUS_BYTE_COUNT_SHIFT) |
-		   (addr << GMBUS_SLAVE_ADDR_SHIFT) |
-		   GMBUS_SLAVE_WRITE | GMBUS_SW_RDY);
+	I915_WRITE_FW(GMBUS3, val);
+	I915_WRITE_FW(GMBUS1,
+		      GMBUS_CYCLE_WAIT |
+		      (chunk_size << GMBUS_BYTE_COUNT_SHIFT) |
+		      (addr << GMBUS_SLAVE_ADDR_SHIFT) |
+		      GMBUS_SLAVE_WRITE | GMBUS_SW_RDY);
 	while (len) {
 		int ret;
 
@@ -399,10 +390,9 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv,
 			val |= *buf++ << (8 * loop);
 		} while (--len && ++loop < 4);
 
-		I915_WRITE(GMBUS3, val);
+		I915_WRITE_FW(GMBUS3, val);
 
-		ret = gmbus_wait_hw_status(dev_priv, GMBUS_HW_RDY,
-					   GMBUS_HW_RDY_EN);
+		ret = gmbus_wait(dev_priv, GMBUS_HW_RDY, GMBUS_HW_RDY_EN);
 		if (ret)
 			return ret;
 	}
@@ -460,13 +450,13 @@ gmbus_xfer_index_read(struct drm_i915_private *dev_priv, struct i2c_msg *msgs)
 
 	/* GMBUS5 holds 16-bit index */
 	if (gmbus5)
-		I915_WRITE(GMBUS5, gmbus5);
+		I915_WRITE_FW(GMBUS5, gmbus5);
 
 	ret = gmbus_xfer_read(dev_priv, &msgs[1], gmbus1_index);
 
 	/* Clear GMBUS5 after each index transfer */
 	if (gmbus5)
-		I915_WRITE(GMBUS5, 0);
+		I915_WRITE_FW(GMBUS5, 0);
 
 	return ret;
 }
@@ -478,11 +468,15 @@ do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 					       struct intel_gmbus,
 					       adapter);
 	struct drm_i915_private *dev_priv = bus->dev_priv;
+	const unsigned int fw =
+		intel_uncore_forcewake_for_reg(dev_priv, GMBUS0,
+					       FW_REG_READ | FW_REG_WRITE);
 	int i = 0, inc, try = 0;
 	int ret = 0;
 
+	intel_uncore_forcewake_get(dev_priv, fw);
 retry:
-	I915_WRITE(GMBUS0, bus->reg0);
+	I915_WRITE_FW(GMBUS0, bus->reg0);
 
 	for (; i < num; i += inc) {
 		inc = 1;
@@ -496,8 +490,8 @@ retry:
 		}
 
 		if (!ret)
-			ret = gmbus_wait_hw_status(dev_priv, GMBUS_HW_WAIT_PHASE,
-						   GMBUS_HW_WAIT_EN);
+			ret = gmbus_wait(dev_priv,
+					 GMBUS_HW_WAIT_PHASE, GMBUS_HW_WAIT_EN);
 		if (ret == -ETIMEDOUT)
 			goto timeout;
 		else if (ret)
@@ -508,7 +502,7 @@ retry:
 	 * a STOP on the very first cycle. To simplify the code we
 	 * unconditionally generate the STOP condition with an additional gmbus
 	 * cycle. */
-	I915_WRITE(GMBUS1, GMBUS_CYCLE_STOP | GMBUS_SW_RDY);
+	I915_WRITE_FW(GMBUS1, GMBUS_CYCLE_STOP | GMBUS_SW_RDY);
 
 	/* Mark the GMBUS interface as disabled after waiting for idle.
 	 * We will re-enable it at the start of the next xfer,
@@ -519,7 +513,7 @@ retry:
 			 adapter->name);
 		ret = -ETIMEDOUT;
 	}
-	I915_WRITE(GMBUS0, 0);
+	I915_WRITE_FW(GMBUS0, 0);
 	ret = ret ?: i;
 	goto out;
 
@@ -548,9 +542,9 @@ clear_err:
 	 * of resetting the GMBUS controller and so clearing the
 	 * BUS_ERROR raised by the slave's NAK.
 	 */
-	I915_WRITE(GMBUS1, GMBUS_SW_CLR_INT);
-	I915_WRITE(GMBUS1, 0);
-	I915_WRITE(GMBUS0, 0);
+	I915_WRITE_FW(GMBUS1, GMBUS_SW_CLR_INT);
+	I915_WRITE_FW(GMBUS1, 0);
+	I915_WRITE_FW(GMBUS0, 0);
 
 	DRM_DEBUG_KMS("GMBUS [%s] NAK for addr: %04x %c(%d)\n",
 			 adapter->name, msgs[i].addr,
@@ -573,7 +567,7 @@ clear_err:
 timeout:
 	DRM_DEBUG_KMS("GMBUS [%s] timed out, falling back to bit banging on pin %d\n",
 		      bus->adapter.name, bus->reg0 & 0xff);
-	I915_WRITE(GMBUS0, 0);
+	I915_WRITE_FW(GMBUS0, 0);
 
 	/*
 	 * Hardware may not support GMBUS over these pins? Try GPIO bitbanging
@@ -582,6 +576,7 @@ timeout:
 	ret = -EAGAIN;
 
 out:
+	intel_uncore_forcewake_put(dev_priv, fw);
 	return ret;
 }
 
@@ -633,6 +628,7 @@ static const struct i2c_algorithm gmbus_algorithm = {
 int intel_setup_gmbus(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct intel_gmbus *bus;
 	unsigned int pin;
 	int ret;
@@ -663,7 +659,7 @@ int intel_setup_gmbus(struct drm_device *dev)
 			 "i915 gmbus %s",
 			 get_gmbus_pin(dev_priv, pin)->name);
 
-		bus->adapter.dev.parent = &dev->pdev->dev;
+		bus->adapter.dev.parent = &pdev->dev;
 		bus->dev_priv = dev_priv;
 
 		bus->adapter.algo = &gmbus_algorithm;
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 414ddda43922..0adb879833ff 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -156,6 +156,11 @@
 #define GEN8_CTX_STATUS_COMPLETE	(1 << 4)
 #define GEN8_CTX_STATUS_LITE_RESTORE	(1 << 15)
 
+#define GEN8_CTX_STATUS_COMPLETED_MASK \
+	 (GEN8_CTX_STATUS_ACTIVE_IDLE | \
+	  GEN8_CTX_STATUS_PREEMPTED | \
+	  GEN8_CTX_STATUS_ELEMENT_SWITCH)
+
 #define CTX_LRI_HEADER_0		0x01
 #define CTX_CONTEXT_CONTROL		0x02
 #define CTX_RING_HEAD			0x04
@@ -221,10 +226,16 @@ enum {
 /* Typical size of the average request (2 pipecontrols and a MI_BB) */
 #define EXECLISTS_REQUEST_SIZE 64 /* bytes */
 
+#define WA_TAIL_DWORDS 2
+
 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 					    struct intel_engine_cs *engine);
 static int intel_lr_context_pin(struct i915_gem_context *ctx,
 				struct intel_engine_cs *engine);
+static void execlists_init_reg_state(u32 *reg_state,
+				     struct i915_gem_context *ctx,
+				     struct intel_engine_cs *engine,
+				     struct intel_ring *ring);
 
 /**
  * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
@@ -263,12 +274,10 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
-	if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv))
-		engine->idle_lite_restore_wa = ~0;
-
-	engine->disable_lite_restore_wa = (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
-					IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) &&
-					(engine->id == VCS || engine->id == VCS2);
+	engine->disable_lite_restore_wa =
+		(IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
+		 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) &&
+		(engine->id == VCS || engine->id == VCS2);
 
 	engine->ctx_desc_template = GEN8_CTX_VALID;
 	if (IS_GEN8(dev_priv))
@@ -288,7 +297,6 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
 /**
  * intel_lr_context_descriptor_update() - calculate & cache the descriptor
  * 					  descriptor for a pinned context
- *
  * @ctx: Context to work on
  * @engine: Engine the descriptor will be used with
  *
@@ -297,12 +305,13 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
  * expensive to calculate, we'll just do it once and cache the result,
  * which remains valid until the context is unpinned.
  *
- * This is what a descriptor looks like, from LSB to MSB:
- *    bits  0-11:    flags, GEN8_CTX_* (cached in ctx_desc_template)
- *    bits 12-31:    LRCA, GTT address of (the HWSP of) this context
- *    bits 32-52:    ctx ID, a globally unique tag
- *    bits 53-54:    mbz, reserved for use by hardware
- *    bits 55-63:    group ID, currently unused and set to 0
+ * This is what a descriptor looks like, from LSB to MSB::
+ *
+ *      bits  0-11:    flags, GEN8_CTX_* (cached in ctx_desc_template)
+ *      bits 12-31:    LRCA, GTT address of (the HWSP of) this context
+ *      bits 32-52:    ctx ID, a globally unique tag
+ *      bits 53-54:    mbz, reserved for use by hardware
+ *      bits 55-63:    group ID, currently unused and set to 0
  */
 static void
 intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
@@ -315,7 +324,7 @@ intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
 
 	desc = ctx->desc_template;				/* bits  3-4  */
 	desc |= engine->ctx_desc_template;			/* bits  0-11 */
-	desc |= ce->lrc_vma->node.start + LRC_PPHWSP_PN * PAGE_SIZE;
+	desc |= i915_ggtt_offset(ce->state) + LRC_PPHWSP_PN * PAGE_SIZE;
 								/* bits 12-31 */
 	desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT;		/* bits 32-52 */
 
@@ -328,34 +337,18 @@ uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
 	return ctx->engine[engine->id].lrc_desc;
 }
 
-static void execlists_elsp_write(struct drm_i915_gem_request *rq0,
-				 struct drm_i915_gem_request *rq1)
+static inline void
+execlists_context_status_change(struct drm_i915_gem_request *rq,
+				unsigned long status)
 {
+	/*
+	 * Only used when GVT-g is enabled now. When GVT-g is disabled,
+	 * The compiler should eliminate this function as dead-code.
+	 */
+	if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
+		return;
 
-	struct intel_engine_cs *engine = rq0->engine;
-	struct drm_i915_private *dev_priv = rq0->i915;
-	uint64_t desc[2];
-
-	if (rq1) {
-		desc[1] = intel_lr_context_descriptor(rq1->ctx, rq1->engine);
-		rq1->elsp_submitted++;
-	} else {
-		desc[1] = 0;
-	}
-
-	desc[0] = intel_lr_context_descriptor(rq0->ctx, rq0->engine);
-	rq0->elsp_submitted++;
-
-	/* You must always write both descriptors in the order below. */
-	I915_WRITE_FW(RING_ELSP(engine), upper_32_bits(desc[1]));
-	I915_WRITE_FW(RING_ELSP(engine), lower_32_bits(desc[1]));
-
-	I915_WRITE_FW(RING_ELSP(engine), upper_32_bits(desc[0]));
-	/* The context is automatically loaded after the following */
-	I915_WRITE_FW(RING_ELSP(engine), lower_32_bits(desc[0]));
-
-	/* ELSP is a wo register, use another nearby reg for posting */
-	POSTING_READ_FW(RING_EXECLIST_STATUS_LO(engine));
+	atomic_notifier_call_chain(&rq->ctx->status_notifier, status, rq);
 }
 
 static void
@@ -367,13 +360,13 @@ execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state)
 	ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
 }
 
-static void execlists_update_context(struct drm_i915_gem_request *rq)
+static u64 execlists_update_context(struct drm_i915_gem_request *rq)
 {
-	struct intel_engine_cs *engine = rq->engine;
+	struct intel_context *ce = &rq->ctx->engine[rq->engine->id];
 	struct i915_hw_ppgtt *ppgtt = rq->ctx->ppgtt;
-	uint32_t *reg_state = rq->ctx->engine[engine->id].lrc_reg_state;
+	u32 *reg_state = ce->lrc_reg_state;
 
-	reg_state[CTX_RING_TAIL+1] = rq->tail;
+	reg_state[CTX_RING_TAIL+1] = intel_ring_offset(rq->ring, rq->tail);
 
 	/* True 32b PPGTT with dynamic page allocation: update PDP
 	 * registers and point the unallocated PDPs to scratch page.
@@ -382,321 +375,236 @@ static void execlists_update_context(struct drm_i915_gem_request *rq)
 	 */
 	if (ppgtt && !USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
 		execlists_update_context_pdps(ppgtt, reg_state);
+
+	return ce->lrc_desc;
 }
 
-static void execlists_submit_requests(struct drm_i915_gem_request *rq0,
-				      struct drm_i915_gem_request *rq1)
+static void execlists_submit_ports(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = rq0->i915;
-	unsigned int fw_domains = rq0->engine->fw_domains;
-
-	execlists_update_context(rq0);
+	struct drm_i915_private *dev_priv = engine->i915;
+	struct execlist_port *port = engine->execlist_port;
+	u32 __iomem *elsp =
+		dev_priv->regs + i915_mmio_reg_offset(RING_ELSP(engine));
+	u64 desc[2];
 
-	if (rq1)
-		execlists_update_context(rq1);
+	if (!port[0].count)
+		execlists_context_status_change(port[0].request,
+						INTEL_CONTEXT_SCHEDULE_IN);
+	desc[0] = execlists_update_context(port[0].request);
+	engine->preempt_wa = port[0].count++; /* bdw only? fixed on skl? */
 
-	spin_lock_irq(&dev_priv->uncore.lock);
-	intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
+	if (port[1].request) {
+		GEM_BUG_ON(port[1].count);
+		execlists_context_status_change(port[1].request,
+						INTEL_CONTEXT_SCHEDULE_IN);
+		desc[1] = execlists_update_context(port[1].request);
+		port[1].count = 1;
+	} else {
+		desc[1] = 0;
+	}
+	GEM_BUG_ON(desc[0] == desc[1]);
 
-	execlists_elsp_write(rq0, rq1);
+	/* You must always write both descriptors in the order below. */
+	writel(upper_32_bits(desc[1]), elsp);
+	writel(lower_32_bits(desc[1]), elsp);
 
-	intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
-	spin_unlock_irq(&dev_priv->uncore.lock);
+	writel(upper_32_bits(desc[0]), elsp);
+	/* The context is automatically loaded after the following */
+	writel(lower_32_bits(desc[0]), elsp);
 }
 
-static inline void execlists_context_status_change(
-		struct drm_i915_gem_request *rq,
-		unsigned long status)
+static bool ctx_single_port_submission(const struct i915_gem_context *ctx)
 {
-	/*
-	 * Only used when GVT-g is enabled now. When GVT-g is disabled,
-	 * The compiler should eliminate this function as dead-code.
-	 */
-	if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
-		return;
-
-	atomic_notifier_call_chain(&rq->ctx->status_notifier, status, rq);
+	return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
+		ctx->execlists_force_single_submission);
 }
 
-static void execlists_context_unqueue(struct intel_engine_cs *engine)
+static bool can_merge_ctx(const struct i915_gem_context *prev,
+			  const struct i915_gem_context *next)
 {
-	struct drm_i915_gem_request *req0 = NULL, *req1 = NULL;
-	struct drm_i915_gem_request *cursor, *tmp;
+	if (prev != next)
+		return false;
 
-	assert_spin_locked(&engine->execlist_lock);
+	if (ctx_single_port_submission(prev))
+		return false;
 
-	/*
-	 * If irqs are not active generate a warning as batches that finish
-	 * without the irqs may get lost and a GPU Hang may occur.
-	 */
-	WARN_ON(!intel_irqs_enabled(engine->i915));
-
-	/* Try to read in pairs */
-	list_for_each_entry_safe(cursor, tmp, &engine->execlist_queue,
-				 execlist_link) {
-		if (!req0) {
-			req0 = cursor;
-		} else if (req0->ctx == cursor->ctx) {
-			/* Same ctx: ignore first request, as second request
-			 * will update tail past first request's workload */
-			cursor->elsp_submitted = req0->elsp_submitted;
-			list_del(&req0->execlist_link);
-			i915_gem_request_unreference(req0);
-			req0 = cursor;
-		} else {
-			if (IS_ENABLED(CONFIG_DRM_I915_GVT)) {
-				/*
-				 * req0 (after merged) ctx requires single
-				 * submission, stop picking
-				 */
-				if (req0->ctx->execlists_force_single_submission)
-					break;
-				/*
-				 * req0 ctx doesn't require single submission,
-				 * but next req ctx requires, stop picking
-				 */
-				if (cursor->ctx->execlists_force_single_submission)
-					break;
-			}
-			req1 = cursor;
-			WARN_ON(req1->elsp_submitted);
-			break;
-		}
-	}
+	return true;
+}
 
-	if (unlikely(!req0))
-		return;
+static void execlists_dequeue(struct intel_engine_cs *engine)
+{
+	struct drm_i915_gem_request *cursor, *last;
+	struct execlist_port *port = engine->execlist_port;
+	bool submit = false;
+
+	last = port->request;
+	if (last)
+		/* WaIdleLiteRestore:bdw,skl
+		 * Apply the wa NOOPs to prevent ring:HEAD == req:TAIL
+		 * as we resubmit the request. See gen8_emit_request()
+		 * for where we prepare the padding after the end of the
+		 * request.
+		 */
+		last->tail = last->wa_tail;
 
-	execlists_context_status_change(req0, INTEL_CONTEXT_SCHEDULE_IN);
+	GEM_BUG_ON(port[1].request);
 
-	if (req1)
-		execlists_context_status_change(req1,
-						INTEL_CONTEXT_SCHEDULE_IN);
+	/* Hardware submission is through 2 ports. Conceptually each port
+	 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
+	 * static for a context, and unique to each, so we only execute
+	 * requests belonging to a single context from each ring. RING_HEAD
+	 * is maintained by the CS in the context image, it marks the place
+	 * where it got up to last time, and through RING_TAIL we tell the CS
+	 * where we want to execute up to this time.
+	 *
+	 * In this list the requests are in order of execution. Consecutive
+	 * requests from the same context are adjacent in the ringbuffer. We
+	 * can combine these requests into a single RING_TAIL update:
+	 *
+	 *              RING_HEAD...req1...req2
+	 *                                    ^- RING_TAIL
+	 * since to execute req2 the CS must first execute req1.
+	 *
+	 * Our goal then is to point each port to the end of a consecutive
+	 * sequence of requests as being the most optimal (fewest wake ups
+	 * and context switches) submission.
+	 */
 
-	if (req0->elsp_submitted & engine->idle_lite_restore_wa) {
-		/*
-		 * WaIdleLiteRestore: make sure we never cause a lite restore
-		 * with HEAD==TAIL.
+	spin_lock(&engine->execlist_lock);
+	list_for_each_entry(cursor, &engine->execlist_queue, execlist_link) {
+		/* Can we combine this request with the current port? It has to
+		 * be the same context/ringbuffer and not have any exceptions
+		 * (e.g. GVT saying never to combine contexts).
 		 *
-		 * Apply the wa NOOPS to prevent ring:HEAD == req:TAIL as we
-		 * resubmit the request. See gen8_emit_request() for where we
-		 * prepare the padding after the end of the request.
+		 * If we can combine the requests, we can execute both by
+		 * updating the RING_TAIL to point to the end of the second
+		 * request, and so we never need to tell the hardware about
+		 * the first.
 		 */
-		struct intel_ringbuffer *ringbuf;
+		if (last && !can_merge_ctx(cursor->ctx, last->ctx)) {
+			/* If we are on the second port and cannot combine
+			 * this request with the last, then we are done.
+			 */
+			if (port != engine->execlist_port)
+				break;
+
+			/* If GVT overrides us we only ever submit port[0],
+			 * leaving port[1] empty. Note that we also have
+			 * to be careful that we don't queue the same
+			 * context (even though a different request) to
+			 * the second port.
+			 */
+			if (ctx_single_port_submission(cursor->ctx))
+				break;
+
+			GEM_BUG_ON(last->ctx == cursor->ctx);
+
+			i915_gem_request_assign(&port->request, last);
+			port++;
+		}
+		last = cursor;
+		submit = true;
+	}
+	if (submit) {
+		/* Decouple all the requests submitted from the queue */
+		engine->execlist_queue.next = &cursor->execlist_link;
+		cursor->execlist_link.prev = &engine->execlist_queue;
 
-		ringbuf = req0->ctx->engine[engine->id].ringbuf;
-		req0->tail += 8;
-		req0->tail &= ringbuf->size - 1;
+		i915_gem_request_assign(&port->request, last);
 	}
+	spin_unlock(&engine->execlist_lock);
 
-	execlists_submit_requests(req0, req1);
+	if (submit)
+		execlists_submit_ports(engine);
 }
 
-static unsigned int
-execlists_check_remove_request(struct intel_engine_cs *engine, u32 ctx_id)
+static bool execlists_elsp_idle(struct intel_engine_cs *engine)
 {
-	struct drm_i915_gem_request *head_req;
-
-	assert_spin_locked(&engine->execlist_lock);
-
-	head_req = list_first_entry_or_null(&engine->execlist_queue,
-					    struct drm_i915_gem_request,
-					    execlist_link);
-
-	if (WARN_ON(!head_req || (head_req->ctx_hw_id != ctx_id)))
-               return 0;
-
-	WARN(head_req->elsp_submitted == 0, "Never submitted head request\n");
-
-	if (--head_req->elsp_submitted > 0)
-		return 0;
-
-	execlists_context_status_change(head_req, INTEL_CONTEXT_SCHEDULE_OUT);
-
-	list_del(&head_req->execlist_link);
-	i915_gem_request_unreference(head_req);
-
-	return 1;
+	return !engine->execlist_port[0].request;
 }
 
-static u32
-get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer,
-		   u32 *context_id)
+static bool execlists_elsp_ready(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	u32 status;
+	int port;
 
-	read_pointer %= GEN8_CSB_ENTRIES;
+	port = 1; /* wait for a free slot */
+	if (engine->disable_lite_restore_wa || engine->preempt_wa)
+		port = 0; /* wait for GPU to be idle before continuing */
 
-	status = I915_READ_FW(RING_CONTEXT_STATUS_BUF_LO(engine, read_pointer));
-
-	if (status & GEN8_CTX_STATUS_IDLE_ACTIVE)
-		return 0;
-
-	*context_id = I915_READ_FW(RING_CONTEXT_STATUS_BUF_HI(engine,
-							      read_pointer));
-
-	return status;
+	return !engine->execlist_port[port].request;
 }
 
-/**
- * intel_lrc_irq_handler() - handle Context Switch interrupts
- * @data: tasklet handler passed in unsigned long
- *
+/*
  * Check the unread Context Status Buffers and manage the submission of new
  * contexts to the ELSP accordingly.
  */
 static void intel_lrc_irq_handler(unsigned long data)
 {
 	struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
+	struct execlist_port *port = engine->execlist_port;
 	struct drm_i915_private *dev_priv = engine->i915;
-	u32 status_pointer;
-	unsigned int read_pointer, write_pointer;
-	u32 csb[GEN8_CSB_ENTRIES][2];
-	unsigned int csb_read = 0, i;
-	unsigned int submit_contexts = 0;
 
 	intel_uncore_forcewake_get(dev_priv, engine->fw_domains);
 
-	status_pointer = I915_READ_FW(RING_CONTEXT_STATUS_PTR(engine));
-
-	read_pointer = engine->next_context_status_buffer;
-	write_pointer = GEN8_CSB_WRITE_PTR(status_pointer);
-	if (read_pointer > write_pointer)
-		write_pointer += GEN8_CSB_ENTRIES;
-
-	while (read_pointer < write_pointer) {
-		if (WARN_ON_ONCE(csb_read == GEN8_CSB_ENTRIES))
-			break;
-		csb[csb_read][0] = get_context_status(engine, ++read_pointer,
-						      &csb[csb_read][1]);
-		csb_read++;
-	}
-
-	engine->next_context_status_buffer = write_pointer % GEN8_CSB_ENTRIES;
-
-	/* Update the read pointer to the old write pointer. Manual ringbuffer
-	 * management ftw </sarcasm> */
-	I915_WRITE_FW(RING_CONTEXT_STATUS_PTR(engine),
-		      _MASKED_FIELD(GEN8_CSB_READ_PTR_MASK,
-				    engine->next_context_status_buffer << 8));
-
-	intel_uncore_forcewake_put(dev_priv, engine->fw_domains);
-
-	spin_lock(&engine->execlist_lock);
+	if (!execlists_elsp_idle(engine)) {
+		u32 __iomem *csb_mmio =
+			dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine));
+		u32 __iomem *buf =
+			dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0));
+		unsigned int csb, head, tail;
+
+		csb = readl(csb_mmio);
+		head = GEN8_CSB_READ_PTR(csb);
+		tail = GEN8_CSB_WRITE_PTR(csb);
+		if (tail < head)
+			tail += GEN8_CSB_ENTRIES;
+		while (head < tail) {
+			unsigned int idx = ++head % GEN8_CSB_ENTRIES;
+			unsigned int status = readl(buf + 2 * idx);
+
+			if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
+				continue;
+
+			GEM_BUG_ON(port[0].count == 0);
+			if (--port[0].count == 0) {
+				GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
+				execlists_context_status_change(port[0].request,
+								INTEL_CONTEXT_SCHEDULE_OUT);
+
+				i915_gem_request_put(port[0].request);
+				port[0] = port[1];
+				memset(&port[1], 0, sizeof(port[1]));
+
+				engine->preempt_wa = false;
+			}
 
-	for (i = 0; i < csb_read; i++) {
-		if (unlikely(csb[i][0] & GEN8_CTX_STATUS_PREEMPTED)) {
-			if (csb[i][0] & GEN8_CTX_STATUS_LITE_RESTORE) {
-				if (execlists_check_remove_request(engine, csb[i][1]))
-					WARN(1, "Lite Restored request removed from queue\n");
-			} else
-				WARN(1, "Preemption without Lite Restore\n");
+			GEM_BUG_ON(port[0].count == 0 &&
+				   !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
 		}
 
-		if (csb[i][0] & (GEN8_CTX_STATUS_ACTIVE_IDLE |
-		    GEN8_CTX_STATUS_ELEMENT_SWITCH))
-			submit_contexts +=
-				execlists_check_remove_request(engine, csb[i][1]);
-	}
-
-	if (submit_contexts) {
-		if (!engine->disable_lite_restore_wa ||
-		    (csb[i][0] & GEN8_CTX_STATUS_ACTIVE_IDLE))
-			execlists_context_unqueue(engine);
+		writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK,
+				     GEN8_CSB_WRITE_PTR(csb) << 8),
+		       csb_mmio);
 	}
 
-	spin_unlock(&engine->execlist_lock);
+	if (execlists_elsp_ready(engine))
+		execlists_dequeue(engine);
 
-	if (unlikely(submit_contexts > 2))
-		DRM_ERROR("More than two context complete events?\n");
+	intel_uncore_forcewake_put(dev_priv, engine->fw_domains);
 }
 
-static void execlists_context_queue(struct drm_i915_gem_request *request)
+static void execlists_submit_request(struct drm_i915_gem_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
-	struct drm_i915_gem_request *cursor;
-	int num_elements = 0;
-
-	spin_lock_bh(&engine->execlist_lock);
+	unsigned long flags;
 
-	list_for_each_entry(cursor, &engine->execlist_queue, execlist_link)
-		if (++num_elements > 2)
-			break;
+	spin_lock_irqsave(&engine->execlist_lock, flags);
 
-	if (num_elements > 2) {
-		struct drm_i915_gem_request *tail_req;
-
-		tail_req = list_last_entry(&engine->execlist_queue,
-					   struct drm_i915_gem_request,
-					   execlist_link);
-
-		if (request->ctx == tail_req->ctx) {
-			WARN(tail_req->elsp_submitted != 0,
-				"More than 2 already-submitted reqs queued\n");
-			list_del(&tail_req->execlist_link);
-			i915_gem_request_unreference(tail_req);
-		}
-	}
-
-	i915_gem_request_reference(request);
 	list_add_tail(&request->execlist_link, &engine->execlist_queue);
-	request->ctx_hw_id = request->ctx->hw_id;
-	if (num_elements == 0)
-		execlists_context_unqueue(engine);
+	if (execlists_elsp_idle(engine))
+		tasklet_hi_schedule(&engine->irq_tasklet);
 
-	spin_unlock_bh(&engine->execlist_lock);
-}
-
-static int logical_ring_invalidate_all_caches(struct drm_i915_gem_request *req)
-{
-	struct intel_engine_cs *engine = req->engine;
-	uint32_t flush_domains;
-	int ret;
-
-	flush_domains = 0;
-	if (engine->gpu_caches_dirty)
-		flush_domains = I915_GEM_GPU_DOMAINS;
-
-	ret = engine->emit_flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
-	if (ret)
-		return ret;
-
-	engine->gpu_caches_dirty = false;
-	return 0;
-}
-
-static int execlists_move_to_gpu(struct drm_i915_gem_request *req,
-				 struct list_head *vmas)
-{
-	const unsigned other_rings = ~intel_engine_flag(req->engine);
-	struct i915_vma *vma;
-	uint32_t flush_domains = 0;
-	bool flush_chipset = false;
-	int ret;
-
-	list_for_each_entry(vma, vmas, exec_list) {
-		struct drm_i915_gem_object *obj = vma->obj;
-
-		if (obj->active & other_rings) {
-			ret = i915_gem_object_sync(obj, req->engine, &req);
-			if (ret)
-				return ret;
-		}
-
-		if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
-			flush_chipset |= i915_gem_clflush_object(obj, false);
-
-		flush_domains |= obj->base.write_domain;
-	}
-
-	if (flush_domains & I915_GEM_DOMAIN_GTT)
-		wmb();
-
-	/* Unconditionally invalidate gpu caches and ensure that we do flush
-	 * any residual writes from the previous batch.
-	 */
-	return logical_ring_invalidate_all_caches(req);
+	spin_unlock_irqrestore(&engine->execlist_lock, flags);
 }
 
 int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request)
@@ -717,7 +625,11 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
 			return ret;
 	}
 
-	request->ringbuf = ce->ringbuf;
+	request->ring = ce->ring;
+
+	ret = intel_lr_context_pin(request->ctx, engine);
+	if (ret)
+		return ret;
 
 	if (i915.enable_guc_submission) {
 		/*
@@ -725,23 +637,19 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
 		 * going any further, as the i915_add_request() call
 		 * later on mustn't fail ...
 		 */
-		ret = i915_guc_wq_check_space(request);
+		ret = i915_guc_wq_reserve(request);
 		if (ret)
-			return ret;
+			goto err_unpin;
 	}
 
-	ret = intel_lr_context_pin(request->ctx, engine);
-	if (ret)
-		return ret;
-
 	ret = intel_ring_begin(request, 0);
 	if (ret)
-		goto err_unpin;
+		goto err_unreserve;
 
 	if (!ce->initialised) {
 		ret = engine->init_context(request);
 		if (ret)
-			goto err_unpin;
+			goto err_unreserve;
 
 		ce->initialised = true;
 	}
@@ -756,13 +664,16 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
 	request->reserved_space -= EXECLISTS_REQUEST_SIZE;
 	return 0;
 
+err_unreserve:
+	if (i915.enable_guc_submission)
+		i915_guc_wq_unreserve(request);
 err_unpin:
 	intel_lr_context_unpin(request->ctx, engine);
 	return ret;
 }
 
 /*
- * intel_logical_ring_advance_and_submit() - advance the tail and submit the workload
+ * intel_logical_ring_advance() - advance the tail and prepare for submission
  * @request: Request to advance the logical ringbuffer of.
  *
  * The tail is updated in our logical ringbuffer struct, not in the actual context. What
@@ -771,13 +682,13 @@ err_unpin:
  * point, the tail *inside* the context is updated and the ELSP written to.
  */
 static int
-intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
+intel_logical_ring_advance(struct drm_i915_gem_request *request)
 {
-	struct intel_ringbuffer *ringbuf = request->ringbuf;
+	struct intel_ring *ring = request->ring;
 	struct intel_engine_cs *engine = request->engine;
 
-	intel_logical_ring_advance(ringbuf);
-	request->tail = ringbuf->tail;
+	intel_ring_advance(ring);
+	request->tail = ring->tail;
 
 	/*
 	 * Here we add two extra NOOPs as padding to avoid
@@ -785,9 +696,10 @@ intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
 	 *
 	 * Caller must reserve WA_TAIL_DWORDS for us!
 	 */
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
-	intel_logical_ring_advance(ringbuf);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
+	request->wa_tail = ring->tail;
 
 	/* We keep the previous context alive until we retire the following
 	 * request. This ensures that any the context object is still pinned
@@ -797,168 +709,14 @@ intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
 	 */
 	request->previous_context = engine->last_context;
 	engine->last_context = request->ctx;
-
-	if (i915.enable_guc_submission)
-		i915_guc_submit(request);
-	else
-		execlists_context_queue(request);
-
-	return 0;
-}
-
-/**
- * execlists_submission() - submit a batchbuffer for execution, Execlists style
- * @params: execbuffer call parameters.
- * @args: execbuffer call arguments.
- * @vmas: list of vmas.
- *
- * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts
- * away the submission details of the execbuffer ioctl call.
- *
- * Return: non-zero if the submission fails.
- */
-int intel_execlists_submission(struct i915_execbuffer_params *params,
-			       struct drm_i915_gem_execbuffer2 *args,
-			       struct list_head *vmas)
-{
-	struct drm_device       *dev = params->dev;
-	struct intel_engine_cs *engine = params->engine;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_ringbuffer *ringbuf = params->ctx->engine[engine->id].ringbuf;
-	u64 exec_start;
-	int instp_mode;
-	u32 instp_mask;
-	int ret;
-
-	instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
-	instp_mask = I915_EXEC_CONSTANTS_MASK;
-	switch (instp_mode) {
-	case I915_EXEC_CONSTANTS_REL_GENERAL:
-	case I915_EXEC_CONSTANTS_ABSOLUTE:
-	case I915_EXEC_CONSTANTS_REL_SURFACE:
-		if (instp_mode != 0 && engine != &dev_priv->engine[RCS]) {
-			DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
-			return -EINVAL;
-		}
-
-		if (instp_mode != dev_priv->relative_constants_mode) {
-			if (instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
-				DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
-				return -EINVAL;
-			}
-
-			/* The HW changed the meaning on this bit on gen6 */
-			instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
-		}
-		break;
-	default:
-		DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
-		return -EINVAL;
-	}
-
-	if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
-		DRM_DEBUG("sol reset is gen7 only\n");
-		return -EINVAL;
-	}
-
-	ret = execlists_move_to_gpu(params->request, vmas);
-	if (ret)
-		return ret;
-
-	if (engine == &dev_priv->engine[RCS] &&
-	    instp_mode != dev_priv->relative_constants_mode) {
-		ret = intel_ring_begin(params->request, 4);
-		if (ret)
-			return ret;
-
-		intel_logical_ring_emit(ringbuf, MI_NOOP);
-		intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1));
-		intel_logical_ring_emit_reg(ringbuf, INSTPM);
-		intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode);
-		intel_logical_ring_advance(ringbuf);
-
-		dev_priv->relative_constants_mode = instp_mode;
-	}
-
-	exec_start = params->batch_obj_vm_offset +
-		     args->batch_start_offset;
-
-	ret = engine->emit_bb_start(params->request, exec_start, params->dispatch_flags);
-	if (ret)
-		return ret;
-
-	trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
-
-	i915_gem_execbuffer_move_to_active(vmas, params->request);
-
-	return 0;
-}
-
-void intel_execlists_cancel_requests(struct intel_engine_cs *engine)
-{
-	struct drm_i915_gem_request *req, *tmp;
-	LIST_HEAD(cancel_list);
-
-	WARN_ON(!mutex_is_locked(&engine->i915->drm.struct_mutex));
-
-	spin_lock_bh(&engine->execlist_lock);
-	list_replace_init(&engine->execlist_queue, &cancel_list);
-	spin_unlock_bh(&engine->execlist_lock);
-
-	list_for_each_entry_safe(req, tmp, &cancel_list, execlist_link) {
-		list_del(&req->execlist_link);
-		i915_gem_request_unreference(req);
-	}
-}
-
-void intel_logical_ring_stop(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	int ret;
-
-	if (!intel_engine_initialized(engine))
-		return;
-
-	ret = intel_engine_idle(engine);
-	if (ret)
-		DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
-			  engine->name, ret);
-
-	/* TODO: Is this correct with Execlists enabled? */
-	I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
-	if (intel_wait_for_register(dev_priv,
-				    RING_MI_MODE(engine->mmio_base),
-				    MODE_IDLE, MODE_IDLE,
-				    1000)) {
-		DRM_ERROR("%s :timed out trying to stop ring\n", engine->name);
-		return;
-	}
-	I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
-}
-
-int logical_ring_flush_all_caches(struct drm_i915_gem_request *req)
-{
-	struct intel_engine_cs *engine = req->engine;
-	int ret;
-
-	if (!engine->gpu_caches_dirty)
-		return 0;
-
-	ret = engine->emit_flush(req, 0, I915_GEM_GPU_DOMAINS);
-	if (ret)
-		return ret;
-
-	engine->gpu_caches_dirty = false;
 	return 0;
 }
 
 static int intel_lr_context_pin(struct i915_gem_context *ctx,
 				struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = ctx->i915;
 	struct intel_context *ce = &ctx->engine[engine->id];
 	void *vaddr;
-	u32 *lrc_reg_state;
 	int ret;
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
@@ -966,41 +724,42 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx,
 	if (ce->pin_count++)
 		return 0;
 
-	ret = i915_gem_obj_ggtt_pin(ce->state, GEN8_LR_CONTEXT_ALIGN,
-				    PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
+	ret = i915_vma_pin(ce->state, 0, GEN8_LR_CONTEXT_ALIGN,
+			   PIN_OFFSET_BIAS | GUC_WOPCM_TOP | PIN_GLOBAL);
 	if (ret)
 		goto err;
 
-	vaddr = i915_gem_object_pin_map(ce->state);
+	vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
 	if (IS_ERR(vaddr)) {
 		ret = PTR_ERR(vaddr);
-		goto unpin_ctx_obj;
+		goto unpin_vma;
 	}
 
-	lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
-
-	ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ce->ringbuf);
+	ret = intel_ring_pin(ce->ring);
 	if (ret)
 		goto unpin_map;
 
-	i915_gem_context_reference(ctx);
-	ce->lrc_vma = i915_gem_obj_to_ggtt(ce->state);
 	intel_lr_context_descriptor_update(ctx, engine);
 
-	lrc_reg_state[CTX_RING_BUFFER_START+1] = ce->ringbuf->vma->node.start;
-	ce->lrc_reg_state = lrc_reg_state;
-	ce->state->dirty = true;
+	ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
+	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
+		i915_ggtt_offset(ce->ring->vma);
+
+	ce->state->obj->dirty = true;
 
 	/* Invalidate GuC TLB. */
-	if (i915.enable_guc_submission)
+	if (i915.enable_guc_submission) {
+		struct drm_i915_private *dev_priv = ctx->i915;
 		I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
+	}
 
+	i915_gem_context_get(ctx);
 	return 0;
 
 unpin_map:
-	i915_gem_object_unpin_map(ce->state);
-unpin_ctx_obj:
-	i915_gem_object_ggtt_unpin(ce->state);
+	i915_gem_object_unpin_map(ce->state->obj);
+unpin_vma:
+	__i915_vma_unpin(ce->state);
 err:
 	ce->pin_count = 0;
 	return ret;
@@ -1017,30 +776,24 @@ void intel_lr_context_unpin(struct i915_gem_context *ctx,
 	if (--ce->pin_count)
 		return;
 
-	intel_unpin_ringbuffer_obj(ce->ringbuf);
+	intel_ring_unpin(ce->ring);
 
-	i915_gem_object_unpin_map(ce->state);
-	i915_gem_object_ggtt_unpin(ce->state);
+	i915_gem_object_unpin_map(ce->state->obj);
+	i915_vma_unpin(ce->state);
 
-	ce->lrc_vma = NULL;
-	ce->lrc_desc = 0;
-	ce->lrc_reg_state = NULL;
-
-	i915_gem_context_unreference(ctx);
+	i915_gem_context_put(ctx);
 }
 
 static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
 {
 	int ret, i;
-	struct intel_engine_cs *engine = req->engine;
-	struct intel_ringbuffer *ringbuf = req->ringbuf;
+	struct intel_ring *ring = req->ring;
 	struct i915_workarounds *w = &req->i915->workarounds;
 
 	if (w->count == 0)
 		return 0;
 
-	engine->gpu_caches_dirty = true;
-	ret = logical_ring_flush_all_caches(req);
+	ret = req->engine->emit_flush(req, EMIT_BARRIER);
 	if (ret)
 		return ret;
 
@@ -1048,17 +801,16 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
 	if (ret)
 		return ret;
 
-	intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(w->count));
+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
 	for (i = 0; i < w->count; i++) {
-		intel_logical_ring_emit_reg(ringbuf, w->reg[i].addr);
-		intel_logical_ring_emit(ringbuf, w->reg[i].value);
+		intel_ring_emit_reg(ring, w->reg[i].addr);
+		intel_ring_emit(ring, w->reg[i].value);
 	}
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
+	intel_ring_emit(ring, MI_NOOP);
 
-	intel_logical_ring_advance(ringbuf);
+	intel_ring_advance(ring);
 
-	engine->gpu_caches_dirty = true;
-	ret = logical_ring_flush_all_caches(req);
+	ret = req->engine->emit_flush(req, EMIT_BARRIER);
 	if (ret)
 		return ret;
 
@@ -1094,7 +846,7 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
  * code duplication.
  */
 static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine,
-						uint32_t *const batch,
+						uint32_t *batch,
 						uint32_t index)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
@@ -1113,7 +865,7 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine,
 	wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 |
 				   MI_SRM_LRM_GLOBAL_GTT));
 	wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4);
-	wa_ctx_emit(batch, index, engine->scratch.gtt_offset + 256);
+	wa_ctx_emit(batch, index, i915_ggtt_offset(engine->scratch) + 256);
 	wa_ctx_emit(batch, index, 0);
 
 	wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
@@ -1131,7 +883,7 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine,
 	wa_ctx_emit(batch, index, (MI_LOAD_REGISTER_MEM_GEN8 |
 				   MI_SRM_LRM_GLOBAL_GTT));
 	wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4);
-	wa_ctx_emit(batch, index, engine->scratch.gtt_offset + 256);
+	wa_ctx_emit(batch, index, i915_ggtt_offset(engine->scratch) + 256);
 	wa_ctx_emit(batch, index, 0);
 
 	return index;
@@ -1156,37 +908,24 @@ static inline int wa_ctx_end(struct i915_wa_ctx_bb *wa_ctx,
 	return 0;
 }
 
-/**
- * gen8_init_indirectctx_bb() - initialize indirect ctx batch with WA
- *
- * @engine: only applicable for RCS
- * @wa_ctx: structure representing wa_ctx
- *  offset: specifies start of the batch, should be cache-aligned. This is updated
- *    with the offset value received as input.
- *  size: size of the batch in DWORDS but HW expects in terms of cachelines
- * @batch: page in which WA are loaded
- * @offset: This field specifies the start of the batch, it should be
- *  cache-aligned otherwise it is adjusted accordingly.
- *  Typically we only have one indirect_ctx and per_ctx batch buffer which are
- *  initialized at the beginning and shared across all contexts but this field
- *  helps us to have multiple batches at different offsets and select them based
- *  on a criteria. At the moment this batch always start at the beginning of the page
- *  and at this point we don't have multiple wa_ctx batch buffers.
- *
- *  The number of WA applied are not known at the beginning; we use this field
- *  to return the no of DWORDS written.
+/*
+ * Typically we only have one indirect_ctx and per_ctx batch buffer which are
+ * initialized at the beginning and shared across all contexts but this field
+ * helps us to have multiple batches at different offsets and select them based
+ * on a criteria. At the moment this batch always start at the beginning of the page
+ * and at this point we don't have multiple wa_ctx batch buffers.
  *
- *  It is to be noted that this batch does not contain MI_BATCH_BUFFER_END
- *  so it adds NOOPs as padding to make it cacheline aligned.
- *  MI_BATCH_BUFFER_END will be added to perctx batch and both of them together
- *  makes a complete batch buffer.
+ * The number of WA applied are not known at the beginning; we use this field
+ * to return the no of DWORDS written.
  *
- * Return: non-zero if we exceed the PAGE_SIZE limit.
+ * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END
+ * so it adds NOOPs as padding to make it cacheline aligned.
+ * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together
+ * makes a complete batch buffer.
  */
-
 static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine,
 				    struct i915_wa_ctx_bb *wa_ctx,
-				    uint32_t *const batch,
+				    uint32_t *batch,
 				    uint32_t *offset)
 {
 	uint32_t scratch_addr;
@@ -1205,7 +944,7 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine,
 
 	/* WaClearSlmSpaceAtContextSwitch:bdw,chv */
 	/* Actual scratch location is at 128 bytes offset */
-	scratch_addr = engine->scratch.gtt_offset + 2*CACHELINE_BYTES;
+	scratch_addr = i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
 
 	wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6));
 	wa_ctx_emit(batch, index, (PIPE_CONTROL_FLUSH_L3 |
@@ -1230,26 +969,18 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine,
 	return wa_ctx_end(wa_ctx, *offset = index, CACHELINE_DWORDS);
 }
 
-/**
- * gen8_init_perctx_bb() - initialize per ctx batch with WA
- *
- * @engine: only applicable for RCS
- * @wa_ctx: structure representing wa_ctx
- *  offset: specifies start of the batch, should be cache-aligned.
- *  size: size of the batch in DWORDS but HW expects in terms of cachelines
- * @batch: page in which WA are loaded
- * @offset: This field specifies the start of this batch.
- *   This batch is started immediately after indirect_ctx batch. Since we ensure
- *   that indirect_ctx ends on a cacheline this batch is aligned automatically.
+/*
+ *  This batch is started immediately after indirect_ctx batch. Since we ensure
+ *  that indirect_ctx ends on a cacheline this batch is aligned automatically.
  *
- *   The number of DWORDS written are returned using this field.
+ *  The number of DWORDS written are returned using this field.
  *
  *  This batch is terminated with MI_BATCH_BUFFER_END and so we need not add padding
  *  to align it with cacheline as padding after MI_BATCH_BUFFER_END is redundant.
  */
 static int gen8_init_perctx_bb(struct intel_engine_cs *engine,
 			       struct i915_wa_ctx_bb *wa_ctx,
-			       uint32_t *const batch,
+			       uint32_t *batch,
 			       uint32_t *offset)
 {
 	uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
@@ -1264,7 +995,7 @@ static int gen8_init_perctx_bb(struct intel_engine_cs *engine,
 
 static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
 				    struct i915_wa_ctx_bb *wa_ctx,
-				    uint32_t *const batch,
+				    uint32_t *batch,
 				    uint32_t *offset)
 {
 	int ret;
@@ -1282,11 +1013,18 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
 		return ret;
 	index = ret;
 
+	/* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl */
+	wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
+	wa_ctx_emit_reg(batch, index, COMMON_SLICE_CHICKEN2);
+	wa_ctx_emit(batch, index, _MASKED_BIT_DISABLE(
+			    GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE));
+	wa_ctx_emit(batch, index, MI_NOOP);
+
 	/* WaClearSlmSpaceAtContextSwitch:kbl */
 	/* Actual scratch location is at 128 bytes offset */
 	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0)) {
-		uint32_t scratch_addr
-			= engine->scratch.gtt_offset + 2*CACHELINE_BYTES;
+		u32 scratch_addr =
+			i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
 
 		wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6));
 		wa_ctx_emit(batch, index, (PIPE_CONTROL_FLUSH_L3 |
@@ -1332,7 +1070,7 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
 
 static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
 			       struct i915_wa_ctx_bb *wa_ctx,
-			       uint32_t *const batch,
+			       uint32_t *batch,
 			       uint32_t *offset)
 {
 	uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
@@ -1378,44 +1116,44 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
 
 static int lrc_setup_wa_ctx_obj(struct intel_engine_cs *engine, u32 size)
 {
-	int ret;
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	int err;
 
-	engine->wa_ctx.obj = i915_gem_object_create(&engine->i915->drm,
-						    PAGE_ALIGN(size));
-	if (IS_ERR(engine->wa_ctx.obj)) {
-		DRM_DEBUG_DRIVER("alloc LRC WA ctx backing obj failed.\n");
-		ret = PTR_ERR(engine->wa_ctx.obj);
-		engine->wa_ctx.obj = NULL;
-		return ret;
-	}
+	obj = i915_gem_object_create(&engine->i915->drm, PAGE_ALIGN(size));
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
 
-	ret = i915_gem_obj_ggtt_pin(engine->wa_ctx.obj, PAGE_SIZE, 0);
-	if (ret) {
-		DRM_DEBUG_DRIVER("pin LRC WA ctx backing obj failed: %d\n",
-				 ret);
-		drm_gem_object_unreference(&engine->wa_ctx.obj->base);
-		return ret;
+	vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto err;
 	}
 
+	err = i915_vma_pin(vma, 0, PAGE_SIZE, PIN_GLOBAL | PIN_HIGH);
+	if (err)
+		goto err;
+
+	engine->wa_ctx.vma = vma;
 	return 0;
+
+err:
+	i915_gem_object_put(obj);
+	return err;
 }
 
 static void lrc_destroy_wa_ctx_obj(struct intel_engine_cs *engine)
 {
-	if (engine->wa_ctx.obj) {
-		i915_gem_object_ggtt_unpin(engine->wa_ctx.obj);
-		drm_gem_object_unreference(&engine->wa_ctx.obj->base);
-		engine->wa_ctx.obj = NULL;
-	}
+	i915_vma_unpin_and_release(&engine->wa_ctx.vma);
 }
 
 static int intel_init_workaround_bb(struct intel_engine_cs *engine)
 {
-	int ret;
+	struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
 	uint32_t *batch;
 	uint32_t offset;
 	struct page *page;
-	struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
+	int ret;
 
 	WARN_ON(engine->id != RCS);
 
@@ -1427,7 +1165,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
 	}
 
 	/* some WA perform writes to scratch page, ensure it is valid */
-	if (engine->scratch.obj == NULL) {
+	if (!engine->scratch) {
 		DRM_ERROR("scratch page not allocated for %s\n", engine->name);
 		return -EINVAL;
 	}
@@ -1438,7 +1176,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
 		return ret;
 	}
 
-	page = i915_gem_object_get_dirty_page(wa_ctx->obj, 0);
+	page = i915_gem_object_get_dirty_page(wa_ctx->vma->obj, 0);
 	batch = kmap_atomic(page);
 	offset = 0;
 
@@ -1485,55 +1223,37 @@ static void lrc_init_hws(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 
 	I915_WRITE(RING_HWS_PGA(engine->mmio_base),
-		   (u32)engine->status_page.gfx_addr);
+		   engine->status_page.ggtt_offset);
 	POSTING_READ(RING_HWS_PGA(engine->mmio_base));
 }
 
 static int gen8_init_common_ring(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
-	unsigned int next_context_status_buffer_hw;
+	int ret;
+
+	ret = intel_mocs_init_engine(engine);
+	if (ret)
+		return ret;
 
 	lrc_init_hws(engine);
 
-	I915_WRITE_IMR(engine,
-		       ~(engine->irq_enable_mask | engine->irq_keep_mask));
+	intel_engine_reset_breadcrumbs(engine);
+
 	I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
 
 	I915_WRITE(RING_MODE_GEN7(engine),
 		   _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
 		   _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
-	POSTING_READ(RING_MODE_GEN7(engine));
-
-	/*
-	 * Instead of resetting the Context Status Buffer (CSB) read pointer to
-	 * zero, we need to read the write pointer from hardware and use its
-	 * value because "this register is power context save restored".
-	 * Effectively, these states have been observed:
-	 *
-	 *      | Suspend-to-idle (freeze) | Suspend-to-RAM (mem) |
-	 * BDW  | CSB regs not reset       | CSB regs reset       |
-	 * CHT  | CSB regs not reset       | CSB regs not reset   |
-	 * SKL  |         ?                |         ?            |
-	 * BXT  |         ?                |         ?            |
-	 */
-	next_context_status_buffer_hw =
-		GEN8_CSB_WRITE_PTR(I915_READ(RING_CONTEXT_STATUS_PTR(engine)));
-
-	/*
-	 * When the CSB registers are reset (also after power-up / gpu reset),
-	 * CSB write pointer is set to all 1's, which is not valid, use '5' in
-	 * this special case, so the first element read is CSB[0].
-	 */
-	if (next_context_status_buffer_hw == GEN8_CSB_PTR_MASK)
-		next_context_status_buffer_hw = (GEN8_CSB_ENTRIES - 1);
 
-	engine->next_context_status_buffer = next_context_status_buffer_hw;
 	DRM_DEBUG_DRIVER("Execlists enabled for %s\n", engine->name);
 
 	intel_engine_init_hangcheck(engine);
 
-	return intel_mocs_init_engine(engine);
+	if (!execlists_elsp_idle(engine))
+		execlists_submit_ports(engine);
+
+	return 0;
 }
 
 static int gen8_init_render_ring(struct intel_engine_cs *engine)
@@ -1569,11 +1289,57 @@ static int gen9_init_render_ring(struct intel_engine_cs *engine)
 	return init_workarounds_ring(engine);
 }
 
+static void reset_common_ring(struct intel_engine_cs *engine,
+			      struct drm_i915_gem_request *request)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	struct execlist_port *port = engine->execlist_port;
+	struct intel_context *ce = &request->ctx->engine[engine->id];
+
+	/* We want a simple context + ring to execute the breadcrumb update.
+	 * We cannot rely on the context being intact across the GPU hang,
+	 * so clear it and rebuild just what we need for the breadcrumb.
+	 * All pending requests for this context will be zapped, and any
+	 * future request will be after userspace has had the opportunity
+	 * to recreate its own state.
+	 */
+	execlists_init_reg_state(ce->lrc_reg_state,
+				 request->ctx, engine, ce->ring);
+
+	/* Move the RING_HEAD onto the breadcrumb, past the hanging batch */
+	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
+		i915_ggtt_offset(ce->ring->vma);
+	ce->lrc_reg_state[CTX_RING_HEAD+1] = request->postfix;
+
+	request->ring->head = request->postfix;
+	request->ring->last_retired_head = -1;
+	intel_ring_update_space(request->ring);
+
+	if (i915.enable_guc_submission)
+		return;
+
+	/* Catch up with any missed context-switch interrupts */
+	I915_WRITE(RING_CONTEXT_STATUS_PTR(engine), _MASKED_FIELD(0xffff, 0));
+	if (request->ctx != port[0].request->ctx) {
+		i915_gem_request_put(port[0].request);
+		port[0] = port[1];
+		memset(&port[1], 0, sizeof(port[1]));
+	}
+
+	/* CS is stopped, and we will resubmit both ports on resume */
+	GEM_BUG_ON(request->ctx != port[0].request->ctx);
+	port[0].count = 0;
+	port[1].count = 0;
+
+	/* Reset WaIdleLiteRestore:bdw,skl as well */
+	request->tail = request->wa_tail - WA_TAIL_DWORDS * sizeof(u32);
+}
+
 static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
 {
 	struct i915_hw_ppgtt *ppgtt = req->ctx->ppgtt;
+	struct intel_ring *ring = req->ring;
 	struct intel_engine_cs *engine = req->engine;
-	struct intel_ringbuffer *ringbuf = req->ringbuf;
 	const int num_lri_cmds = GEN8_LEGACY_PDPES * 2;
 	int i, ret;
 
@@ -1581,28 +1347,27 @@ static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
 	if (ret)
 		return ret;
 
-	intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(num_lri_cmds));
+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(num_lri_cmds));
 	for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
 		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
 
-		intel_logical_ring_emit_reg(ringbuf,
-					    GEN8_RING_PDP_UDW(engine, i));
-		intel_logical_ring_emit(ringbuf, upper_32_bits(pd_daddr));
-		intel_logical_ring_emit_reg(ringbuf,
-					    GEN8_RING_PDP_LDW(engine, i));
-		intel_logical_ring_emit(ringbuf, lower_32_bits(pd_daddr));
+		intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(engine, i));
+		intel_ring_emit(ring, upper_32_bits(pd_daddr));
+		intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(engine, i));
+		intel_ring_emit(ring, lower_32_bits(pd_daddr));
 	}
 
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
-	intel_logical_ring_advance(ringbuf);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
-			      u64 offset, unsigned dispatch_flags)
+			      u64 offset, u32 len,
+			      unsigned int dispatch_flags)
 {
-	struct intel_ringbuffer *ringbuf = req->ringbuf;
+	struct intel_ring *ring = req->ring;
 	bool ppgtt = !(dispatch_flags & I915_DISPATCH_SECURE);
 	int ret;
 
@@ -1629,14 +1394,14 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
 		return ret;
 
 	/* FIXME(BDW): Address space and security selectors. */
-	intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 |
-				(ppgtt<<8) |
-				(dispatch_flags & I915_DISPATCH_RS ?
-				 MI_BATCH_RESOURCE_STREAMER : 0));
-	intel_logical_ring_emit(ringbuf, lower_32_bits(offset));
-	intel_logical_ring_emit(ringbuf, upper_32_bits(offset));
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
-	intel_logical_ring_advance(ringbuf);
+	intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 |
+			(ppgtt<<8) |
+			(dispatch_flags & I915_DISPATCH_RS ?
+			 MI_BATCH_RESOURCE_STREAMER : 0));
+	intel_ring_emit(ring, lower_32_bits(offset));
+	intel_ring_emit(ring, upper_32_bits(offset));
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -1655,14 +1420,10 @@ static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
 	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
 }
 
-static int gen8_emit_flush(struct drm_i915_gem_request *request,
-			   u32 invalidate_domains,
-			   u32 unused)
+static int gen8_emit_flush(struct drm_i915_gem_request *request, u32 mode)
 {
-	struct intel_ringbuffer *ringbuf = request->ringbuf;
-	struct intel_engine_cs *engine = ringbuf->engine;
-	struct drm_i915_private *dev_priv = request->i915;
-	uint32_t cmd;
+	struct intel_ring *ring = request->ring;
+	u32 cmd;
 	int ret;
 
 	ret = intel_ring_begin(request, 4);
@@ -1678,30 +1439,30 @@ static int gen8_emit_flush(struct drm_i915_gem_request *request,
 	 */
 	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
 
-	if (invalidate_domains & I915_GEM_GPU_DOMAINS) {
+	if (mode & EMIT_INVALIDATE) {
 		cmd |= MI_INVALIDATE_TLB;
-		if (engine == &dev_priv->engine[VCS])
+		if (request->engine->id == VCS)
 			cmd |= MI_INVALIDATE_BSD;
 	}
 
-	intel_logical_ring_emit(ringbuf, cmd);
-	intel_logical_ring_emit(ringbuf,
-				I915_GEM_HWS_SCRATCH_ADDR |
-				MI_FLUSH_DW_USE_GTT);
-	intel_logical_ring_emit(ringbuf, 0); /* upper addr */
-	intel_logical_ring_emit(ringbuf, 0); /* value */
-	intel_logical_ring_advance(ringbuf);
+	intel_ring_emit(ring, cmd);
+	intel_ring_emit(ring,
+			I915_GEM_HWS_SCRATCH_ADDR |
+			MI_FLUSH_DW_USE_GTT);
+	intel_ring_emit(ring, 0); /* upper addr */
+	intel_ring_emit(ring, 0); /* value */
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
-				  u32 invalidate_domains,
-				  u32 flush_domains)
+				  u32 mode)
 {
-	struct intel_ringbuffer *ringbuf = request->ringbuf;
-	struct intel_engine_cs *engine = ringbuf->engine;
-	u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
+	struct intel_ring *ring = request->ring;
+	struct intel_engine_cs *engine = request->engine;
+	u32 scratch_addr =
+		i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
 	bool vf_flush_wa = false, dc_flush_wa = false;
 	u32 flags = 0;
 	int ret;
@@ -1709,14 +1470,14 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
 
 	flags |= PIPE_CONTROL_CS_STALL;
 
-	if (flush_domains) {
+	if (mode & EMIT_FLUSH) {
 		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
 	}
 
-	if (invalidate_domains) {
+	if (mode & EMIT_INVALIDATE) {
 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
 		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
@@ -1751,40 +1512,40 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
 		return ret;
 
 	if (vf_flush_wa) {
-		intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
+		intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
 	}
 
 	if (dc_flush_wa) {
-		intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
-		intel_logical_ring_emit(ringbuf, PIPE_CONTROL_DC_FLUSH_ENABLE);
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
+		intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
+		intel_ring_emit(ring, PIPE_CONTROL_DC_FLUSH_ENABLE);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
 	}
 
-	intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
-	intel_logical_ring_emit(ringbuf, flags);
-	intel_logical_ring_emit(ringbuf, scratch_addr);
-	intel_logical_ring_emit(ringbuf, 0);
-	intel_logical_ring_emit(ringbuf, 0);
-	intel_logical_ring_emit(ringbuf, 0);
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
+	intel_ring_emit(ring, flags);
+	intel_ring_emit(ring, scratch_addr);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, 0);
 
 	if (dc_flush_wa) {
-		intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
-		intel_logical_ring_emit(ringbuf, PIPE_CONTROL_CS_STALL);
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
-		intel_logical_ring_emit(ringbuf, 0);
+		intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
+		intel_ring_emit(ring, PIPE_CONTROL_CS_STALL);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, 0);
 	}
 
-	intel_logical_ring_advance(ringbuf);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -1809,11 +1570,10 @@ static void bxt_a_seqno_barrier(struct intel_engine_cs *engine)
  * used as a workaround for not being allowed to do lite
  * restore with HEAD==TAIL (WaIdleLiteRestore).
  */
-#define WA_TAIL_DWORDS 2
 
 static int gen8_emit_request(struct drm_i915_gem_request *request)
 {
-	struct intel_ringbuffer *ringbuf = request->ringbuf;
+	struct intel_ring *ring = request->ring;
 	int ret;
 
 	ret = intel_ring_begin(request, 6 + WA_TAIL_DWORDS);
@@ -1823,21 +1583,20 @@ static int gen8_emit_request(struct drm_i915_gem_request *request)
 	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
 	BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
 
-	intel_logical_ring_emit(ringbuf,
-				(MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW);
-	intel_logical_ring_emit(ringbuf,
-				intel_hws_seqno_address(request->engine) |
-				MI_FLUSH_DW_USE_GTT);
-	intel_logical_ring_emit(ringbuf, 0);
-	intel_logical_ring_emit(ringbuf, request->seqno);
-	intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
-	return intel_logical_ring_advance_and_submit(request);
+	intel_ring_emit(ring, (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW);
+	intel_ring_emit(ring,
+			intel_hws_seqno_address(request->engine) |
+			MI_FLUSH_DW_USE_GTT);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, request->fence.seqno);
+	intel_ring_emit(ring, MI_USER_INTERRUPT);
+	intel_ring_emit(ring, MI_NOOP);
+	return intel_logical_ring_advance(request);
 }
 
 static int gen8_emit_request_render(struct drm_i915_gem_request *request)
 {
-	struct intel_ringbuffer *ringbuf = request->ringbuf;
+	struct intel_ring *ring = request->ring;
 	int ret;
 
 	ret = intel_ring_begin(request, 8 + WA_TAIL_DWORDS);
@@ -1851,50 +1610,19 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request)
 	 * need a prior CS_STALL, which is emitted by the flush
 	 * following the batch.
 	 */
-	intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
-	intel_logical_ring_emit(ringbuf,
-				(PIPE_CONTROL_GLOBAL_GTT_IVB |
-				 PIPE_CONTROL_CS_STALL |
-				 PIPE_CONTROL_QW_WRITE));
-	intel_logical_ring_emit(ringbuf,
-				intel_hws_seqno_address(request->engine));
-	intel_logical_ring_emit(ringbuf, 0);
-	intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request));
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
+	intel_ring_emit(ring,
+			(PIPE_CONTROL_GLOBAL_GTT_IVB |
+			 PIPE_CONTROL_CS_STALL |
+			 PIPE_CONTROL_QW_WRITE));
+	intel_ring_emit(ring, intel_hws_seqno_address(request->engine));
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, i915_gem_request_get_seqno(request));
 	/* We're thrashing one dword of HWS. */
-	intel_logical_ring_emit(ringbuf, 0);
-	intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
-	return intel_logical_ring_advance_and_submit(request);
-}
-
-static int intel_lr_context_render_state_init(struct drm_i915_gem_request *req)
-{
-	struct render_state so;
-	int ret;
-
-	ret = i915_gem_render_state_prepare(req->engine, &so);
-	if (ret)
-		return ret;
-
-	if (so.rodata == NULL)
-		return 0;
-
-	ret = req->engine->emit_bb_start(req, so.ggtt_offset,
-				       I915_DISPATCH_SECURE);
-	if (ret)
-		goto out;
-
-	ret = req->engine->emit_bb_start(req,
-				       (so.ggtt_offset + so.aux_batch_offset),
-				       I915_DISPATCH_SECURE);
-	if (ret)
-		goto out;
-
-	i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), req);
-
-out:
-	i915_gem_render_state_fini(&so);
-	return ret;
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, MI_USER_INTERRUPT);
+	intel_ring_emit(ring, MI_NOOP);
+	return intel_logical_ring_advance(request);
 }
 
 static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
@@ -1913,14 +1641,12 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
 	if (ret)
 		DRM_ERROR("MOCS failed to program: expect performance issues.\n");
 
-	return intel_lr_context_render_state_init(req);
+	return i915_gem_render_state_init(req);
 }
 
 /**
  * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
- *
  * @engine: Engine Command Streamer.
- *
  */
 void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
 {
@@ -1939,39 +1665,42 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
 	dev_priv = engine->i915;
 
 	if (engine->buffer) {
-		intel_logical_ring_stop(engine);
 		WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
 	}
 
 	if (engine->cleanup)
 		engine->cleanup(engine);
 
-	i915_cmd_parser_fini_ring(engine);
-	i915_gem_batch_pool_fini(&engine->batch_pool);
-
-	intel_engine_fini_breadcrumbs(engine);
+	intel_engine_cleanup_common(engine);
 
-	if (engine->status_page.obj) {
-		i915_gem_object_unpin_map(engine->status_page.obj);
-		engine->status_page.obj = NULL;
+	if (engine->status_page.vma) {
+		i915_gem_object_unpin_map(engine->status_page.vma->obj);
+		engine->status_page.vma = NULL;
 	}
 	intel_lr_context_unpin(dev_priv->kernel_context, engine);
 
-	engine->idle_lite_restore_wa = 0;
-	engine->disable_lite_restore_wa = false;
-	engine->ctx_desc_template = 0;
-
 	lrc_destroy_wa_ctx_obj(engine);
 	engine->i915 = NULL;
 }
 
+void intel_execlists_enable_submission(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+
+	for_each_engine(engine, dev_priv)
+		engine->submit_request = execlists_submit_request;
+}
+
 static void
 logical_ring_default_vfuncs(struct intel_engine_cs *engine)
 {
 	/* Default vfuncs which can be overriden by each engine. */
 	engine->init_hw = gen8_init_common_ring;
-	engine->emit_request = gen8_emit_request;
+	engine->reset_hw = reset_common_ring;
 	engine->emit_flush = gen8_emit_flush;
+	engine->emit_request = gen8_emit_request;
+	engine->submit_request = execlists_submit_request;
+
 	engine->irq_enable = gen8_logical_ring_enable_irq;
 	engine->irq_disable = gen8_logical_ring_disable_irq;
 	engine->emit_bb_start = gen8_emit_bb_start;
@@ -1980,41 +1709,71 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine)
 }
 
 static inline void
-logical_ring_default_irqs(struct intel_engine_cs *engine, unsigned shift)
+logical_ring_default_irqs(struct intel_engine_cs *engine)
 {
+	unsigned shift = engine->irq_shift;
 	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
 	engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
 }
 
 static int
-lrc_setup_hws(struct intel_engine_cs *engine,
-	      struct drm_i915_gem_object *dctx_obj)
+lrc_setup_hws(struct intel_engine_cs *engine, struct i915_vma *vma)
 {
+	const int hws_offset = LRC_PPHWSP_PN * PAGE_SIZE;
 	void *hws;
 
 	/* The HWSP is part of the default context object in LRC mode. */
-	engine->status_page.gfx_addr = i915_gem_obj_ggtt_offset(dctx_obj) +
-				       LRC_PPHWSP_PN * PAGE_SIZE;
-	hws = i915_gem_object_pin_map(dctx_obj);
+	hws = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
 	if (IS_ERR(hws))
 		return PTR_ERR(hws);
-	engine->status_page.page_addr = hws + LRC_PPHWSP_PN * PAGE_SIZE;
-	engine->status_page.obj = dctx_obj;
+
+	engine->status_page.page_addr = hws + hws_offset;
+	engine->status_page.ggtt_offset = i915_ggtt_offset(vma) + hws_offset;
+	engine->status_page.vma = vma;
 
 	return 0;
 }
 
+static void
+logical_ring_setup(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	enum forcewake_domains fw_domains;
+
+	intel_engine_setup_common(engine);
+
+	/* Intentionally left blank. */
+	engine->buffer = NULL;
+
+	fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
+						    RING_ELSP(engine),
+						    FW_REG_WRITE);
+
+	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+						     RING_CONTEXT_STATUS_PTR(engine),
+						     FW_REG_READ | FW_REG_WRITE);
+
+	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+						     RING_CONTEXT_STATUS_BUF_BASE(engine),
+						     FW_REG_READ);
+
+	engine->fw_domains = fw_domains;
+
+	tasklet_init(&engine->irq_tasklet,
+		     intel_lrc_irq_handler, (unsigned long)engine);
+
+	logical_ring_init_platform_invariants(engine);
+	logical_ring_default_vfuncs(engine);
+	logical_ring_default_irqs(engine);
+}
+
 static int
 logical_ring_init(struct intel_engine_cs *engine)
 {
 	struct i915_gem_context *dctx = engine->i915->kernel_context;
 	int ret;
 
-	ret = intel_engine_init_breadcrumbs(engine);
-	if (ret)
-		goto error;
-
-	ret = i915_cmd_parser_init_ring(engine);
+	ret = intel_engine_init_common(engine);
 	if (ret)
 		goto error;
 
@@ -2044,11 +1803,13 @@ error:
 	return ret;
 }
 
-static int logical_render_ring_init(struct intel_engine_cs *engine)
+int logical_render_ring_init(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 	int ret;
 
+	logical_ring_setup(engine);
+
 	if (HAS_L3_DPF(dev_priv))
 		engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 
@@ -2058,11 +1819,10 @@ static int logical_render_ring_init(struct intel_engine_cs *engine)
 	else
 		engine->init_hw = gen8_init_render_ring;
 	engine->init_context = gen8_init_rcs_context;
-	engine->cleanup = intel_fini_pipe_control;
 	engine->emit_flush = gen8_emit_flush_render;
 	engine->emit_request = gen8_emit_request_render;
 
-	ret = intel_init_pipe_control(engine, 4096);
+	ret = intel_engine_create_scratch(engine, 4096);
 	if (ret)
 		return ret;
 
@@ -2085,160 +1845,11 @@ static int logical_render_ring_init(struct intel_engine_cs *engine)
 	return ret;
 }
 
-static const struct logical_ring_info {
-	const char *name;
-	unsigned exec_id;
-	unsigned guc_id;
-	u32 mmio_base;
-	unsigned irq_shift;
-	int (*init)(struct intel_engine_cs *engine);
-} logical_rings[] = {
-	[RCS] = {
-		.name = "render ring",
-		.exec_id = I915_EXEC_RENDER,
-		.guc_id = GUC_RENDER_ENGINE,
-		.mmio_base = RENDER_RING_BASE,
-		.irq_shift = GEN8_RCS_IRQ_SHIFT,
-		.init = logical_render_ring_init,
-	},
-	[BCS] = {
-		.name = "blitter ring",
-		.exec_id = I915_EXEC_BLT,
-		.guc_id = GUC_BLITTER_ENGINE,
-		.mmio_base = BLT_RING_BASE,
-		.irq_shift = GEN8_BCS_IRQ_SHIFT,
-		.init = logical_ring_init,
-	},
-	[VCS] = {
-		.name = "bsd ring",
-		.exec_id = I915_EXEC_BSD,
-		.guc_id = GUC_VIDEO_ENGINE,
-		.mmio_base = GEN6_BSD_RING_BASE,
-		.irq_shift = GEN8_VCS1_IRQ_SHIFT,
-		.init = logical_ring_init,
-	},
-	[VCS2] = {
-		.name = "bsd2 ring",
-		.exec_id = I915_EXEC_BSD,
-		.guc_id = GUC_VIDEO_ENGINE2,
-		.mmio_base = GEN8_BSD2_RING_BASE,
-		.irq_shift = GEN8_VCS2_IRQ_SHIFT,
-		.init = logical_ring_init,
-	},
-	[VECS] = {
-		.name = "video enhancement ring",
-		.exec_id = I915_EXEC_VEBOX,
-		.guc_id = GUC_VIDEOENHANCE_ENGINE,
-		.mmio_base = VEBOX_RING_BASE,
-		.irq_shift = GEN8_VECS_IRQ_SHIFT,
-		.init = logical_ring_init,
-	},
-};
-
-static struct intel_engine_cs *
-logical_ring_setup(struct drm_i915_private *dev_priv, enum intel_engine_id id)
-{
-	const struct logical_ring_info *info = &logical_rings[id];
-	struct intel_engine_cs *engine = &dev_priv->engine[id];
-	enum forcewake_domains fw_domains;
-
-	engine->id = id;
-	engine->name = info->name;
-	engine->exec_id = info->exec_id;
-	engine->guc_id = info->guc_id;
-	engine->mmio_base = info->mmio_base;
-
-	engine->i915 = dev_priv;
-
-	/* Intentionally left blank. */
-	engine->buffer = NULL;
-
-	fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
-						    RING_ELSP(engine),
-						    FW_REG_WRITE);
-
-	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
-						     RING_CONTEXT_STATUS_PTR(engine),
-						     FW_REG_READ | FW_REG_WRITE);
-
-	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
-						     RING_CONTEXT_STATUS_BUF_BASE(engine),
-						     FW_REG_READ);
-
-	engine->fw_domains = fw_domains;
-
-	INIT_LIST_HEAD(&engine->active_list);
-	INIT_LIST_HEAD(&engine->request_list);
-	INIT_LIST_HEAD(&engine->buffers);
-	INIT_LIST_HEAD(&engine->execlist_queue);
-	spin_lock_init(&engine->execlist_lock);
-
-	tasklet_init(&engine->irq_tasklet,
-		     intel_lrc_irq_handler, (unsigned long)engine);
-
-	logical_ring_init_platform_invariants(engine);
-	logical_ring_default_vfuncs(engine);
-	logical_ring_default_irqs(engine, info->irq_shift);
-
-	intel_engine_init_hangcheck(engine);
-	i915_gem_batch_pool_init(&dev_priv->drm, &engine->batch_pool);
-
-	return engine;
-}
-
-/**
- * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers
- * @dev: DRM device.
- *
- * This function inits the engines for an Execlists submission style (the
- * equivalent in the legacy ringbuffer submission world would be
- * i915_gem_init_engines). It does it only for those engines that are present in
- * the hardware.
- *
- * Return: non-zero if the initialization failed.
- */
-int intel_logical_rings_init(struct drm_device *dev)
+int logical_xcs_ring_init(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	unsigned int mask = 0;
-	unsigned int i;
-	int ret;
-
-	WARN_ON(INTEL_INFO(dev_priv)->ring_mask &
-		GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
-
-	for (i = 0; i < ARRAY_SIZE(logical_rings); i++) {
-		if (!HAS_ENGINE(dev_priv, i))
-			continue;
-
-		if (!logical_rings[i].init)
-			continue;
+	logical_ring_setup(engine);
 
-		ret = logical_rings[i].init(logical_ring_setup(dev_priv, i));
-		if (ret)
-			goto cleanup;
-
-		mask |= ENGINE_MASK(i);
-	}
-
-	/*
-	 * Catch failures to update logical_rings table when the new engines
-	 * are added to the driver by a warning and disabling the forgotten
-	 * engines.
-	 */
-	if (WARN_ON(mask != INTEL_INFO(dev_priv)->ring_mask)) {
-		struct intel_device_info *info =
-			(struct intel_device_info *)&dev_priv->info;
-		info->ring_mask = mask;
-	}
-
-	return 0;
-
-cleanup:
-	for (i = 0; i < I915_NUM_ENGINES; i++)
-		intel_logical_ring_cleanup(&dev_priv->engine[i]);
-
-	return ret;
+	return logical_ring_init(engine);
 }
 
 static u32
@@ -2259,24 +1870,24 @@ make_rpcs(struct drm_i915_private *dev_priv)
 	 * must make an explicit request through RPCS for full
 	 * enablement.
 	*/
-	if (INTEL_INFO(dev_priv)->has_slice_pg) {
+	if (INTEL_INFO(dev_priv)->sseu.has_slice_pg) {
 		rpcs |= GEN8_RPCS_S_CNT_ENABLE;
-		rpcs |= INTEL_INFO(dev_priv)->slice_total <<
+		rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.slice_mask) <<
 			GEN8_RPCS_S_CNT_SHIFT;
 		rpcs |= GEN8_RPCS_ENABLE;
 	}
 
-	if (INTEL_INFO(dev_priv)->has_subslice_pg) {
+	if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) {
 		rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
-		rpcs |= INTEL_INFO(dev_priv)->subslice_per_slice <<
+		rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask) <<
 			GEN8_RPCS_SS_CNT_SHIFT;
 		rpcs |= GEN8_RPCS_ENABLE;
 	}
 
-	if (INTEL_INFO(dev_priv)->has_eu_pg) {
-		rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice <<
+	if (INTEL_INFO(dev_priv)->sseu.has_eu_pg) {
+		rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
 			GEN8_RPCS_EU_MIN_SHIFT;
-		rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice <<
+		rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
 			GEN8_RPCS_EU_MAX_SHIFT;
 		rpcs |= GEN8_RPCS_ENABLE;
 	}
@@ -2305,38 +1916,13 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
 	return indirect_ctx_offset;
 }
 
-static int
-populate_lr_context(struct i915_gem_context *ctx,
-		    struct drm_i915_gem_object *ctx_obj,
-		    struct intel_engine_cs *engine,
-		    struct intel_ringbuffer *ringbuf)
+static void execlists_init_reg_state(u32 *reg_state,
+				     struct i915_gem_context *ctx,
+				     struct intel_engine_cs *engine,
+				     struct intel_ring *ring)
 {
-	struct drm_i915_private *dev_priv = ctx->i915;
-	struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
-	void *vaddr;
-	u32 *reg_state;
-	int ret;
-
-	if (!ppgtt)
-		ppgtt = dev_priv->mm.aliasing_ppgtt;
-
-	ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
-	if (ret) {
-		DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
-		return ret;
-	}
-
-	vaddr = i915_gem_object_pin_map(ctx_obj);
-	if (IS_ERR(vaddr)) {
-		ret = PTR_ERR(vaddr);
-		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
-		return ret;
-	}
-	ctx_obj->dirty = true;
-
-	/* The second page of the context object contains some fields which must
-	 * be set up prior to the first execution. */
-	reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
+	struct drm_i915_private *dev_priv = engine->i915;
+	struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt;
 
 	/* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
 	 * commands followed by (reg, value) pairs. The values we are setting here are
@@ -2350,19 +1936,16 @@ populate_lr_context(struct i915_gem_context *ctx,
 		       _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
 					  CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
 					  (HAS_RESOURCE_STREAMER(dev_priv) ?
-					    CTX_CTRL_RS_CTX_ENABLE : 0)));
+					   CTX_CTRL_RS_CTX_ENABLE : 0)));
 	ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(engine->mmio_base),
 		       0);
 	ASSIGN_CTX_REG(reg_state, CTX_RING_TAIL, RING_TAIL(engine->mmio_base),
 		       0);
-	/* Ring buffer start address is not known until the buffer is pinned.
-	 * It is written to the context image in execlists_update_context()
-	 */
 	ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_START,
 		       RING_START(engine->mmio_base), 0);
 	ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_CONTROL,
 		       RING_CTL(engine->mmio_base),
-		       ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID);
+		       ((ring->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID);
 	ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_U,
 		       RING_BBADDR_UDW(engine->mmio_base), 0);
 	ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_L,
@@ -2383,9 +1966,9 @@ populate_lr_context(struct i915_gem_context *ctx,
 			       RING_INDIRECT_CTX(engine->mmio_base), 0);
 		ASSIGN_CTX_REG(reg_state, CTX_RCS_INDIRECT_CTX_OFFSET,
 			       RING_INDIRECT_CTX_OFFSET(engine->mmio_base), 0);
-		if (engine->wa_ctx.obj) {
+		if (engine->wa_ctx.vma) {
 			struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
-			uint32_t ggtt_offset = i915_gem_obj_ggtt_offset(wa_ctx->obj);
+			u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
 
 			reg_state[CTX_RCS_INDIRECT_CTX+1] =
 				(ggtt_offset + wa_ctx->indirect_ctx.offset * sizeof(uint32_t)) |
@@ -2440,6 +2023,36 @@ populate_lr_context(struct i915_gem_context *ctx,
 		ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
 			       make_rpcs(dev_priv));
 	}
+}
+
+static int
+populate_lr_context(struct i915_gem_context *ctx,
+		    struct drm_i915_gem_object *ctx_obj,
+		    struct intel_engine_cs *engine,
+		    struct intel_ring *ring)
+{
+	void *vaddr;
+	int ret;
+
+	ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
+	if (ret) {
+		DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
+		return ret;
+	}
+
+	vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
+		return ret;
+	}
+	ctx_obj->dirty = true;
+
+	/* The second page of the context object contains some fields which must
+	 * be set up prior to the first execution. */
+
+	execlists_init_reg_state(vaddr + LRC_STATE_PN * PAGE_SIZE,
+				 ctx, engine, ring);
 
 	i915_gem_object_unpin_map(ctx_obj);
 
@@ -2484,26 +2097,14 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
 	return ret;
 }
 
-/**
- * execlists_context_deferred_alloc() - create the LRC specific bits of a context
- * @ctx: LR context to create.
- * @engine: engine to be used with the context.
- *
- * This function can be called more than once, with different engines, if we plan
- * to use the context with them. The context backing objects and the ringbuffers
- * (specially the ringbuffer backing objects) suck a lot of memory up, and that's why
- * the creation is a deferred call: it's better to make sure first that we need to use
- * a given ring with the context.
- *
- * Return: non-zero on error.
- */
 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 					    struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_object *ctx_obj;
 	struct intel_context *ce = &ctx->engine[engine->id];
+	struct i915_vma *vma;
 	uint32_t context_size;
-	struct intel_ringbuffer *ringbuf;
+	struct intel_ring *ring;
 	int ret;
 
 	WARN_ON(ce->state);
@@ -2519,60 +2120,63 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 		return PTR_ERR(ctx_obj);
 	}
 
-	ringbuf = intel_engine_create_ringbuffer(engine, ctx->ring_size);
-	if (IS_ERR(ringbuf)) {
-		ret = PTR_ERR(ringbuf);
+	vma = i915_vma_create(ctx_obj, &ctx->i915->ggtt.base, NULL);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
+		goto error_deref_obj;
+	}
+
+	ring = intel_engine_create_ring(engine, ctx->ring_size);
+	if (IS_ERR(ring)) {
+		ret = PTR_ERR(ring);
 		goto error_deref_obj;
 	}
 
-	ret = populate_lr_context(ctx, ctx_obj, engine, ringbuf);
+	ret = populate_lr_context(ctx, ctx_obj, engine, ring);
 	if (ret) {
 		DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
-		goto error_ringbuf;
+		goto error_ring_free;
 	}
 
-	ce->ringbuf = ringbuf;
-	ce->state = ctx_obj;
+	ce->ring = ring;
+	ce->state = vma;
 	ce->initialised = engine->init_context == NULL;
 
 	return 0;
 
-error_ringbuf:
-	intel_ringbuffer_free(ringbuf);
+error_ring_free:
+	intel_ring_free(ring);
 error_deref_obj:
-	drm_gem_object_unreference(&ctx_obj->base);
-	ce->ringbuf = NULL;
-	ce->state = NULL;
+	i915_gem_object_put(ctx_obj);
 	return ret;
 }
 
-void intel_lr_context_reset(struct drm_i915_private *dev_priv,
-			    struct i915_gem_context *ctx)
+void intel_lr_context_resume(struct drm_i915_private *dev_priv)
 {
+	struct i915_gem_context *ctx = dev_priv->kernel_context;
 	struct intel_engine_cs *engine;
 
 	for_each_engine(engine, dev_priv) {
 		struct intel_context *ce = &ctx->engine[engine->id];
-		struct drm_i915_gem_object *ctx_obj = ce->state;
 		void *vaddr;
 		uint32_t *reg_state;
 
-		if (!ctx_obj)
+		if (!ce->state)
 			continue;
 
-		vaddr = i915_gem_object_pin_map(ctx_obj);
+		vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
 		if (WARN_ON(IS_ERR(vaddr)))
 			continue;
 
 		reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
-		ctx_obj->dirty = true;
 
 		reg_state[CTX_RING_HEAD+1] = 0;
 		reg_state[CTX_RING_TAIL+1] = 0;
 
-		i915_gem_object_unpin_map(ctx_obj);
+		ce->state->obj->dirty = true;
+		i915_gem_object_unpin_map(ce->state->obj);
 
-		ce->ringbuf->head = 0;
-		ce->ringbuf->tail = 0;
+		ce->ring->head = 0;
+		ce->ring->tail = 0;
 	}
 }
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 2b8255c19dcc..4fed8165f98a 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -29,17 +29,17 @@
 #define GEN8_LR_CONTEXT_ALIGN 4096
 
 /* Execlists regs */
-#define RING_ELSP(ring)				_MMIO((ring)->mmio_base + 0x230)
-#define RING_EXECLIST_STATUS_LO(ring)		_MMIO((ring)->mmio_base + 0x234)
-#define RING_EXECLIST_STATUS_HI(ring)		_MMIO((ring)->mmio_base + 0x234 + 4)
-#define RING_CONTEXT_CONTROL(ring)		_MMIO((ring)->mmio_base + 0x244)
+#define RING_ELSP(engine)			_MMIO((engine)->mmio_base + 0x230)
+#define RING_EXECLIST_STATUS_LO(engine)		_MMIO((engine)->mmio_base + 0x234)
+#define RING_EXECLIST_STATUS_HI(engine)		_MMIO((engine)->mmio_base + 0x234 + 4)
+#define RING_CONTEXT_CONTROL(engine)		_MMIO((engine)->mmio_base + 0x244)
 #define	  CTX_CTRL_INHIBIT_SYN_CTX_SWITCH	(1 << 3)
 #define	  CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT	(1 << 0)
 #define   CTX_CTRL_RS_CTX_ENABLE                (1 << 1)
-#define RING_CONTEXT_STATUS_BUF_BASE(ring)	_MMIO((ring)->mmio_base + 0x370)
-#define RING_CONTEXT_STATUS_BUF_LO(ring, i)	_MMIO((ring)->mmio_base + 0x370 + (i) * 8)
-#define RING_CONTEXT_STATUS_BUF_HI(ring, i)	_MMIO((ring)->mmio_base + 0x370 + (i) * 8 + 4)
-#define RING_CONTEXT_STATUS_PTR(ring)		_MMIO((ring)->mmio_base + 0x3a0)
+#define RING_CONTEXT_STATUS_BUF_BASE(engine)	_MMIO((engine)->mmio_base + 0x370)
+#define RING_CONTEXT_STATUS_BUF_LO(engine, i)	_MMIO((engine)->mmio_base + 0x370 + (i) * 8)
+#define RING_CONTEXT_STATUS_BUF_HI(engine, i)	_MMIO((engine)->mmio_base + 0x370 + (i) * 8 + 4)
+#define RING_CONTEXT_STATUS_PTR(engine)		_MMIO((engine)->mmio_base + 0x3a0)
 
 /* The docs specify that the write pointer wraps around after 5h, "After status
  * is written out to the last available status QW at offset 5h, this pointer
@@ -67,35 +67,10 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
 int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request);
 void intel_logical_ring_stop(struct intel_engine_cs *engine);
 void intel_logical_ring_cleanup(struct intel_engine_cs *engine);
-int intel_logical_rings_init(struct drm_device *dev);
+int logical_render_ring_init(struct intel_engine_cs *engine);
+int logical_xcs_ring_init(struct intel_engine_cs *engine);
 
-int logical_ring_flush_all_caches(struct drm_i915_gem_request *req);
-/**
- * intel_logical_ring_advance() - advance the ringbuffer tail
- * @ringbuf: Ringbuffer to advance.
- *
- * The tail is only updated in our logical ringbuffer struct.
- */
-static inline void intel_logical_ring_advance(struct intel_ringbuffer *ringbuf)
-{
-	ringbuf->tail &= ringbuf->size - 1;
-}
-/**
- * intel_logical_ring_emit() - write a DWORD to the ringbuffer.
- * @ringbuf: Ringbuffer to write to.
- * @data: DWORD to write.
- */
-static inline void intel_logical_ring_emit(struct intel_ringbuffer *ringbuf,
-					   u32 data)
-{
-	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
-	ringbuf->tail += 4;
-}
-static inline void intel_logical_ring_emit_reg(struct intel_ringbuffer *ringbuf,
-					       i915_reg_t reg)
-{
-	intel_logical_ring_emit(ringbuf, i915_mmio_reg_offset(reg));
-}
+int intel_engines_init(struct drm_device *dev);
 
 /* Logical Ring Contexts */
 
@@ -112,19 +87,13 @@ void intel_lr_context_unpin(struct i915_gem_context *ctx,
 
 struct drm_i915_private;
 
-void intel_lr_context_reset(struct drm_i915_private *dev_priv,
-			    struct i915_gem_context *ctx);
+void intel_lr_context_resume(struct drm_i915_private *dev_priv);
 uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
 				     struct intel_engine_cs *engine);
 
 /* Execlists */
 int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv,
 				    int enable_execlists);
-struct i915_execbuffer_params;
-int intel_execlists_submission(struct i915_execbuffer_params *params,
-			       struct drm_i915_gem_execbuffer2 *args,
-			       struct list_head *vmas);
-
-void intel_execlists_cancel_requests(struct intel_engine_cs *engine);
+void intel_execlists_enable_submission(struct drm_i915_private *dev_priv);
 
 #endif /* _INTEL_LRC_H_ */
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 49550470483e..e1d47d51ea47 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -48,6 +48,20 @@ struct intel_lvds_connector {
 	struct notifier_block lid_notifier;
 };
 
+struct intel_lvds_pps {
+	/* 100us units */
+	int t1_t2;
+	int t3;
+	int t4;
+	int t5;
+	int tx;
+
+	int divider;
+
+	int port;
+	bool powerdown_on_reset;
+};
+
 struct intel_lvds_encoder {
 	struct intel_encoder base;
 
@@ -55,6 +69,9 @@ struct intel_lvds_encoder {
 	i915_reg_t reg;
 	u32 a3_power;
 
+	struct intel_lvds_pps init_pps;
+	u32 init_lvds_val;
+
 	struct intel_lvds_connector *attached_connector;
 };
 
@@ -136,28 +153,108 @@ static void intel_lvds_get_config(struct intel_encoder *encoder,
 	pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock;
 }
 
-static void intel_pre_enable_lvds(struct intel_encoder *encoder)
+static void intel_lvds_pps_get_hw_state(struct drm_i915_private *dev_priv,
+					struct intel_lvds_pps *pps)
+{
+	u32 val;
+
+	pps->powerdown_on_reset = I915_READ(PP_CONTROL(0)) & PANEL_POWER_RESET;
+
+	val = I915_READ(PP_ON_DELAYS(0));
+	pps->port = (val & PANEL_PORT_SELECT_MASK) >>
+		    PANEL_PORT_SELECT_SHIFT;
+	pps->t1_t2 = (val & PANEL_POWER_UP_DELAY_MASK) >>
+		     PANEL_POWER_UP_DELAY_SHIFT;
+	pps->t5 = (val & PANEL_LIGHT_ON_DELAY_MASK) >>
+		  PANEL_LIGHT_ON_DELAY_SHIFT;
+
+	val = I915_READ(PP_OFF_DELAYS(0));
+	pps->t3 = (val & PANEL_POWER_DOWN_DELAY_MASK) >>
+		  PANEL_POWER_DOWN_DELAY_SHIFT;
+	pps->tx = (val & PANEL_LIGHT_OFF_DELAY_MASK) >>
+		  PANEL_LIGHT_OFF_DELAY_SHIFT;
+
+	val = I915_READ(PP_DIVISOR(0));
+	pps->divider = (val & PP_REFERENCE_DIVIDER_MASK) >>
+		       PP_REFERENCE_DIVIDER_SHIFT;
+	val = (val & PANEL_POWER_CYCLE_DELAY_MASK) >>
+	      PANEL_POWER_CYCLE_DELAY_SHIFT;
+	/*
+	 * Remove the BSpec specified +1 (100ms) offset that accounts for a
+	 * too short power-cycle delay due to the asynchronous programming of
+	 * the register.
+	 */
+	if (val)
+		val--;
+	/* Convert from 100ms to 100us units */
+	pps->t4 = val * 1000;
+
+	if (INTEL_INFO(dev_priv)->gen <= 4 &&
+	    pps->t1_t2 == 0 && pps->t5 == 0 && pps->t3 == 0 && pps->tx == 0) {
+		DRM_DEBUG_KMS("Panel power timings uninitialized, "
+			      "setting defaults\n");
+		/* Set T2 to 40ms and T5 to 200ms in 100 usec units */
+		pps->t1_t2 = 40 * 10;
+		pps->t5 = 200 * 10;
+		/* Set T3 to 35ms and Tx to 200ms in 100 usec units */
+		pps->t3 = 35 * 10;
+		pps->tx = 200 * 10;
+	}
+
+	DRM_DEBUG_DRIVER("LVDS PPS:t1+t2 %d t3 %d t4 %d t5 %d tx %d "
+			 "divider %d port %d powerdown_on_reset %d\n",
+			 pps->t1_t2, pps->t3, pps->t4, pps->t5, pps->tx,
+			 pps->divider, pps->port, pps->powerdown_on_reset);
+}
+
+static void intel_lvds_pps_init_hw(struct drm_i915_private *dev_priv,
+				   struct intel_lvds_pps *pps)
+{
+	u32 val;
+
+	val = I915_READ(PP_CONTROL(0));
+	WARN_ON((val & PANEL_UNLOCK_MASK) != PANEL_UNLOCK_REGS);
+	if (pps->powerdown_on_reset)
+		val |= PANEL_POWER_RESET;
+	I915_WRITE(PP_CONTROL(0), val);
+
+	I915_WRITE(PP_ON_DELAYS(0), (pps->port << PANEL_PORT_SELECT_SHIFT) |
+				    (pps->t1_t2 << PANEL_POWER_UP_DELAY_SHIFT) |
+				    (pps->t5 << PANEL_LIGHT_ON_DELAY_SHIFT));
+	I915_WRITE(PP_OFF_DELAYS(0), (pps->t3 << PANEL_POWER_DOWN_DELAY_SHIFT) |
+				     (pps->tx << PANEL_LIGHT_OFF_DELAY_SHIFT));
+
+	val = pps->divider << PP_REFERENCE_DIVIDER_SHIFT;
+	val |= (DIV_ROUND_UP(pps->t4, 1000) + 1) <<
+	       PANEL_POWER_CYCLE_DELAY_SHIFT;
+	I915_WRITE(PP_DIVISOR(0), val);
+}
+
+static void intel_pre_enable_lvds(struct intel_encoder *encoder,
+				  struct intel_crtc_state *pipe_config,
+				  struct drm_connector_state *conn_state)
 {
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-	const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
+	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	int pipe = crtc->pipe;
 	u32 temp;
 
-	if (HAS_PCH_SPLIT(dev)) {
+	if (HAS_PCH_SPLIT(dev_priv)) {
 		assert_fdi_rx_pll_disabled(dev_priv, pipe);
 		assert_shared_dpll_disabled(dev_priv,
-					    crtc->config->shared_dpll);
+					    pipe_config->shared_dpll);
 	} else {
 		assert_pll_disabled(dev_priv, pipe);
 	}
 
-	temp = I915_READ(lvds_encoder->reg);
+	intel_lvds_pps_init_hw(dev_priv, &lvds_encoder->init_pps);
+
+	temp = lvds_encoder->init_lvds_val;
 	temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
 
-	if (HAS_PCH_CPT(dev)) {
+	if (HAS_PCH_CPT(dev_priv)) {
 		temp &= ~PORT_TRANS_SEL_MASK;
 		temp |= PORT_TRANS_SEL_CPT(pipe);
 	} else {
@@ -170,7 +267,7 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder)
 
 	/* set the corresponsding LVDS_BORDER bit */
 	temp &= ~LVDS_BORDER_ENABLE;
-	temp |= crtc->config->gmch_pfit.lvds_border_bits;
+	temp |= pipe_config->gmch_pfit.lvds_border_bits;
 	/* Set the B0-B3 data pairs corresponding to whether we're going to
 	 * set the DPLLs for dual-channel mode or not.
 	 */
@@ -193,7 +290,7 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder)
 	if (IS_GEN4(dev_priv)) {
 		/* Bspec wording suggests that LVDS port dithering only exists
 		 * for 18bpp panels. */
-		if (crtc->config->dither && crtc->config->pipe_bpp == 18)
+		if (pipe_config->dither && pipe_config->pipe_bpp == 18)
 			temp |= LVDS_ENABLE_DITHER;
 		else
 			temp &= ~LVDS_ENABLE_DITHER;
@@ -210,57 +307,45 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder)
 /**
  * Sets the power state for the panel.
  */
-static void intel_enable_lvds(struct intel_encoder *encoder)
+static void intel_enable_lvds(struct intel_encoder *encoder,
+			      struct intel_crtc_state *pipe_config,
+			      struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
 	struct intel_connector *intel_connector =
 		&lvds_encoder->attached_connector->base;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	i915_reg_t ctl_reg, stat_reg;
-
-	if (HAS_PCH_SPLIT(dev)) {
-		ctl_reg = PCH_PP_CONTROL;
-		stat_reg = PCH_PP_STATUS;
-	} else {
-		ctl_reg = PP_CONTROL;
-		stat_reg = PP_STATUS;
-	}
 
 	I915_WRITE(lvds_encoder->reg, I915_READ(lvds_encoder->reg) | LVDS_PORT_EN);
 
-	I915_WRITE(ctl_reg, I915_READ(ctl_reg) | POWER_TARGET_ON);
+	I915_WRITE(PP_CONTROL(0), I915_READ(PP_CONTROL(0)) | PANEL_POWER_ON);
 	POSTING_READ(lvds_encoder->reg);
-	if (intel_wait_for_register(dev_priv, stat_reg, PP_ON, PP_ON, 1000))
+	if (intel_wait_for_register(dev_priv, PP_STATUS(0), PP_ON, PP_ON, 1000))
 		DRM_ERROR("timed out waiting for panel to power on\n");
 
 	intel_panel_enable_backlight(intel_connector);
 }
 
-static void intel_disable_lvds(struct intel_encoder *encoder)
+static void intel_disable_lvds(struct intel_encoder *encoder,
+			       struct intel_crtc_state *old_crtc_state,
+			       struct drm_connector_state *old_conn_state)
 {
-	struct drm_device *dev = encoder->base.dev;
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	i915_reg_t ctl_reg, stat_reg;
-
-	if (HAS_PCH_SPLIT(dev)) {
-		ctl_reg = PCH_PP_CONTROL;
-		stat_reg = PCH_PP_STATUS;
-	} else {
-		ctl_reg = PP_CONTROL;
-		stat_reg = PP_STATUS;
-	}
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON);
-	if (intel_wait_for_register(dev_priv, stat_reg, PP_ON, 0, 1000))
+	I915_WRITE(PP_CONTROL(0), I915_READ(PP_CONTROL(0)) & ~PANEL_POWER_ON);
+	if (intel_wait_for_register(dev_priv, PP_STATUS(0), PP_ON, 0, 1000))
 		DRM_ERROR("timed out waiting for panel to power off\n");
 
 	I915_WRITE(lvds_encoder->reg, I915_READ(lvds_encoder->reg) & ~LVDS_PORT_EN);
 	POSTING_READ(lvds_encoder->reg);
 }
 
-static void gmch_disable_lvds(struct intel_encoder *encoder)
+static void gmch_disable_lvds(struct intel_encoder *encoder,
+			      struct intel_crtc_state *old_crtc_state,
+			      struct drm_connector_state *old_conn_state)
+
 {
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
 	struct intel_connector *intel_connector =
@@ -268,10 +353,12 @@ static void gmch_disable_lvds(struct intel_encoder *encoder)
 
 	intel_panel_disable_backlight(intel_connector);
 
-	intel_disable_lvds(encoder);
+	intel_disable_lvds(encoder, old_crtc_state, old_conn_state);
 }
 
-static void pch_disable_lvds(struct intel_encoder *encoder)
+static void pch_disable_lvds(struct intel_encoder *encoder,
+			     struct intel_crtc_state *old_crtc_state,
+			     struct drm_connector_state *old_conn_state)
 {
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
 	struct intel_connector *intel_connector =
@@ -280,9 +367,11 @@ static void pch_disable_lvds(struct intel_encoder *encoder)
 	intel_panel_disable_backlight(intel_connector);
 }
 
-static void pch_post_disable_lvds(struct intel_encoder *encoder)
+static void pch_post_disable_lvds(struct intel_encoder *encoder,
+				  struct intel_crtc_state *old_crtc_state,
+				  struct drm_connector_state *old_conn_state)
 {
-	intel_disable_lvds(encoder);
+	intel_disable_lvds(encoder, old_crtc_state, old_conn_state);
 }
 
 static enum drm_mode_status
@@ -304,7 +393,8 @@ intel_lvds_mode_valid(struct drm_connector *connector,
 }
 
 static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
-				      struct intel_crtc_state *pipe_config)
+				      struct intel_crtc_state *pipe_config,
+				      struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = intel_encoder->base.dev;
 	struct intel_lvds_encoder *lvds_encoder =
@@ -900,17 +990,6 @@ void intel_lvds_init(struct drm_device *dev)
 	int pipe;
 	u8 pin;
 
-	/*
-	 * Unlock registers and just leave them unlocked. Do this before
-	 * checking quirk lists to avoid bogus WARNINGs.
-	 */
-	if (HAS_PCH_SPLIT(dev)) {
-		I915_WRITE(PCH_PP_CONTROL,
-			   I915_READ(PCH_PP_CONTROL) | PANEL_UNLOCK_REGS);
-	} else if (INTEL_INFO(dev_priv)->gen < 5) {
-		I915_WRITE(PP_CONTROL,
-			   I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS);
-	}
 	if (!intel_lvds_supported(dev))
 		return;
 
@@ -943,18 +1022,6 @@ void intel_lvds_init(struct drm_device *dev)
 		DRM_DEBUG_KMS("LVDS is not present in VBT, but enabled anyway\n");
 	}
 
-	 /* Set the Panel Power On/Off timings if uninitialized. */
-	if (INTEL_INFO(dev_priv)->gen < 5 &&
-	    I915_READ(PP_ON_DELAYS) == 0 && I915_READ(PP_OFF_DELAYS) == 0) {
-		/* Set T2 to 40ms and T5 to 200ms */
-		I915_WRITE(PP_ON_DELAYS, 0x019007d0);
-
-		/* Set T3 to 35ms and Tx to 200ms */
-		I915_WRITE(PP_OFF_DELAYS, 0x015e07d0);
-
-		DRM_DEBUG_KMS("Panel power timings uninitialized, setting defaults\n");
-	}
-
 	lvds_encoder = kzalloc(sizeof(*lvds_encoder), GFP_KERNEL);
 	if (!lvds_encoder)
 		return;
@@ -1020,6 +1087,10 @@ void intel_lvds_init(struct drm_device *dev)
 				      dev->mode_config.scaling_mode_property,
 				      DRM_MODE_SCALE_ASPECT);
 	intel_connector->panel.fitting_mode = DRM_MODE_SCALE_ASPECT;
+
+	intel_lvds_pps_get_hw_state(dev_priv, &lvds_encoder->init_pps);
+	lvds_encoder->init_lvds_val = lvds;
+
 	/*
 	 * LVDS discovery:
 	 * 1) check for EDID on DDC
@@ -1054,17 +1125,6 @@ void intel_lvds_init(struct drm_device *dev)
 	}
 	lvds_connector->base.edid = edid;
 
-	if (IS_ERR_OR_NULL(edid)) {
-		/* Didn't get an EDID, so
-		 * Set wide sync ranges so we get all modes
-		 * handed to valid_mode for checking
-		 */
-		connector->display_info.min_vfreq = 0;
-		connector->display_info.max_vfreq = 200;
-		connector->display_info.min_hfreq = 0;
-		connector->display_info.max_hfreq = 200;
-	}
-
 	list_for_each_entry(scan, &connector->probed_modes, head) {
 		if (scan->type & DRM_MODE_TYPE_PREFERRED) {
 			DRM_DEBUG_KMS("using preferred mode from EDID: ");
diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c
index 927825f5b284..80bb9247ce66 100644
--- a/drivers/gpu/drm/i915/intel_mocs.c
+++ b/drivers/gpu/drm/i915/intel_mocs.c
@@ -97,7 +97,8 @@ struct drm_i915_mocs_table {
  *       end.
  */
 static const struct drm_i915_mocs_entry skylake_mocs_table[] = {
-	{ /* 0x00000009 */
+	[I915_MOCS_UNCACHED] = {
+	  /* 0x00000009 */
 	  .control_value = LE_CACHEABILITY(LE_UC) |
 			   LE_TGT_CACHE(LE_TC_LLC_ELLC) |
 			   LE_LRUM(0) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) |
@@ -106,7 +107,7 @@ static const struct drm_i915_mocs_entry skylake_mocs_table[] = {
 	  /* 0x0010 */
 	  .l3cc_value =    L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC),
 	},
-	{
+	[I915_MOCS_PTE] = {
 	  /* 0x00000038 */
 	  .control_value = LE_CACHEABILITY(LE_PAGETABLE) |
 			   LE_TGT_CACHE(LE_TC_LLC_ELLC) |
@@ -115,7 +116,7 @@ static const struct drm_i915_mocs_entry skylake_mocs_table[] = {
 	  /* 0x0030 */
 	  .l3cc_value =    L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB),
 	},
-	{
+	[I915_MOCS_CACHED] = {
 	  /* 0x0000003b */
 	  .control_value = LE_CACHEABILITY(LE_WB) |
 			   LE_TGT_CACHE(LE_TC_LLC_ELLC) |
@@ -128,7 +129,7 @@ static const struct drm_i915_mocs_entry skylake_mocs_table[] = {
 
 /* NOTE: the LE_TGT_CACHE is not used on Broxton */
 static const struct drm_i915_mocs_entry broxton_mocs_table[] = {
-	{
+	[I915_MOCS_UNCACHED] = {
 	  /* 0x00000009 */
 	  .control_value = LE_CACHEABILITY(LE_UC) |
 			   LE_TGT_CACHE(LE_TC_LLC_ELLC) |
@@ -138,7 +139,7 @@ static const struct drm_i915_mocs_entry broxton_mocs_table[] = {
 	  /* 0x0010 */
 	  .l3cc_value =    L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC),
 	},
-	{
+	[I915_MOCS_PTE] = {
 	  /* 0x00000038 */
 	  .control_value = LE_CACHEABILITY(LE_PAGETABLE) |
 			   LE_TGT_CACHE(LE_TC_LLC_ELLC) |
@@ -148,7 +149,7 @@ static const struct drm_i915_mocs_entry broxton_mocs_table[] = {
 	  /* 0x0030 */
 	  .l3cc_value =    L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB),
 	},
-	{
+	[I915_MOCS_CACHED] = {
 	  /* 0x00000039 */
 	  .control_value = LE_CACHEABILITY(LE_UC) |
 			   LE_TGT_CACHE(LE_TC_LLC_ELLC) |
@@ -203,9 +204,9 @@ static bool get_mocs_settings(struct drm_i915_private *dev_priv,
 	return result;
 }
 
-static i915_reg_t mocs_register(enum intel_engine_id ring, int index)
+static i915_reg_t mocs_register(enum intel_engine_id engine_id, int index)
 {
-	switch (ring) {
+	switch (engine_id) {
 	case RCS:
 		return GEN9_GFX_MOCS(index);
 	case VCS:
@@ -217,7 +218,7 @@ static i915_reg_t mocs_register(enum intel_engine_id ring, int index)
 	case VCS2:
 		return GEN9_MFX1_MOCS(index);
 	default:
-		MISSING_CASE(ring);
+		MISSING_CASE(engine_id);
 		return INVALID_MMIO_REG;
 	}
 }
@@ -275,7 +276,7 @@ int intel_mocs_init_engine(struct intel_engine_cs *engine)
 static int emit_mocs_control_table(struct drm_i915_gem_request *req,
 				   const struct drm_i915_mocs_table *table)
 {
-	struct intel_ringbuffer *ringbuf = req->ringbuf;
+	struct intel_ring *ring = req->ring;
 	enum intel_engine_id engine = req->engine->id;
 	unsigned int index;
 	int ret;
@@ -287,14 +288,11 @@ static int emit_mocs_control_table(struct drm_i915_gem_request *req,
 	if (ret)
 		return ret;
 
-	intel_logical_ring_emit(ringbuf,
-				MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES));
+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES));
 
 	for (index = 0; index < table->size; index++) {
-		intel_logical_ring_emit_reg(ringbuf,
-					    mocs_register(engine, index));
-		intel_logical_ring_emit(ringbuf,
-					table->table[index].control_value);
+		intel_ring_emit_reg(ring, mocs_register(engine, index));
+		intel_ring_emit(ring, table->table[index].control_value);
 	}
 
 	/*
@@ -306,14 +304,12 @@ static int emit_mocs_control_table(struct drm_i915_gem_request *req,
 	 * that value to all the used entries.
 	 */
 	for (; index < GEN9_NUM_MOCS_ENTRIES; index++) {
-		intel_logical_ring_emit_reg(ringbuf,
-					    mocs_register(engine, index));
-		intel_logical_ring_emit(ringbuf,
-					table->table[0].control_value);
+		intel_ring_emit_reg(ring, mocs_register(engine, index));
+		intel_ring_emit(ring, table->table[0].control_value);
 	}
 
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
-	intel_logical_ring_advance(ringbuf);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -340,7 +336,7 @@ static inline u32 l3cc_combine(const struct drm_i915_mocs_table *table,
 static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req,
 				const struct drm_i915_mocs_table *table)
 {
-	struct intel_ringbuffer *ringbuf = req->ringbuf;
+	struct intel_ring *ring = req->ring;
 	unsigned int i;
 	int ret;
 
@@ -351,19 +347,18 @@ static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req,
 	if (ret)
 		return ret;
 
-	intel_logical_ring_emit(ringbuf,
+	intel_ring_emit(ring,
 			MI_LOAD_REGISTER_IMM(GEN9_NUM_MOCS_ENTRIES / 2));
 
 	for (i = 0; i < table->size/2; i++) {
-		intel_logical_ring_emit_reg(ringbuf, GEN9_LNCFCMOCS(i));
-		intel_logical_ring_emit(ringbuf,
-					l3cc_combine(table, 2*i, 2*i+1));
+		intel_ring_emit_reg(ring, GEN9_LNCFCMOCS(i));
+		intel_ring_emit(ring, l3cc_combine(table, 2*i, 2*i+1));
 	}
 
 	if (table->size & 0x01) {
 		/* Odd table size - 1 left over */
-		intel_logical_ring_emit_reg(ringbuf, GEN9_LNCFCMOCS(i));
-		intel_logical_ring_emit(ringbuf, l3cc_combine(table, 2*i, 0));
+		intel_ring_emit_reg(ring, GEN9_LNCFCMOCS(i));
+		intel_ring_emit(ring, l3cc_combine(table, 2*i, 0));
 		i++;
 	}
 
@@ -373,12 +368,12 @@ static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req,
 	 * they are reserved by the hardware.
 	 */
 	for (; i < GEN9_NUM_MOCS_ENTRIES / 2; i++) {
-		intel_logical_ring_emit_reg(ringbuf, GEN9_LNCFCMOCS(i));
-		intel_logical_ring_emit(ringbuf, l3cc_combine(table, 0, 0));
+		intel_ring_emit_reg(ring, GEN9_LNCFCMOCS(i));
+		intel_ring_emit(ring, l3cc_combine(table, 0, 0));
 	}
 
-	intel_logical_ring_emit(ringbuf, MI_NOOP);
-	intel_logical_ring_advance(ringbuf);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/intel_mocs.h b/drivers/gpu/drm/i915/intel_mocs.h
index 4640299e04ec..a8bd9f7bfece 100644
--- a/drivers/gpu/drm/i915/intel_mocs.h
+++ b/drivers/gpu/drm/i915/intel_mocs.h
@@ -54,6 +54,6 @@
 
 int intel_rcs_context_init_mocs(struct drm_i915_gem_request *req);
 void intel_mocs_init_l3cc_table(struct drm_device *dev);
-int intel_mocs_init_engine(struct intel_engine_cs *ring);
+int intel_mocs_init_engine(struct intel_engine_cs *engine);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_modes.c b/drivers/gpu/drm/i915/intel_modes.c
index f2584d0a01ab..951e834dd274 100644
--- a/drivers/gpu/drm/i915/intel_modes.c
+++ b/drivers/gpu/drm/i915/intel_modes.c
@@ -25,7 +25,6 @@
 
 #include <linux/slab.h>
 #include <linux/i2c.h>
-#include <linux/fb.h>
 #include <drm/drm_edid.h>
 #include <drm/drmP.h>
 #include "intel_drv.h"
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index adca262d591a..7acbbbf97833 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -1047,6 +1047,23 @@ err_out:
 	return err;
 }
 
+static int intel_use_opregion_panel_type_callback(const struct dmi_system_id *id)
+{
+	DRM_INFO("Using panel type from OpRegion on %s\n", id->ident);
+	return 1;
+}
+
+static const struct dmi_system_id intel_use_opregion_panel_type[] = {
+	{
+		.callback = intel_use_opregion_panel_type_callback,
+		.ident = "Conrac GmbH IX45GM2",
+		.matches = {DMI_MATCH(DMI_SYS_VENDOR, "Conrac GmbH"),
+			    DMI_MATCH(DMI_PRODUCT_NAME, "IX45GM2"),
+		},
+	},
+	{ }
+};
+
 int
 intel_opregion_get_panel_type(struct drm_i915_private *dev_priv)
 {
@@ -1073,6 +1090,16 @@ intel_opregion_get_panel_type(struct drm_i915_private *dev_priv)
 	}
 
 	/*
+	 * So far we know that some machined must use it, others must not use it.
+	 * There doesn't seem to be any way to determine which way to go, except
+	 * via a quirk list :(
+	 */
+	if (!dmi_check_system(intel_use_opregion_panel_type)) {
+		DRM_DEBUG_KMS("Ignoring OpRegion panel type (%d)\n", ret - 1);
+		return -ENODEV;
+	}
+
+	/*
 	 * FIXME On Dell XPS 13 9350 the OpRegion panel type (0) gives us
 	 * low vswing for eDP, whereas the VBT panel type (2) gives us normal
 	 * vswing instead. Low vswing results in some display flickers, so
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 3212d8806b5a..a24bc8c7889f 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -30,6 +30,7 @@
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "intel_drv.h"
+#include "intel_frontbuffer.h"
 
 /* Limits for overlay size. According to intel doc, the real limits are:
  * Y width: 4095, UV width (planar): 2047, Y height: 2047,
@@ -170,8 +171,8 @@ struct overlay_registers {
 struct intel_overlay {
 	struct drm_i915_private *i915;
 	struct intel_crtc *crtc;
-	struct drm_i915_gem_object *vid_bo;
-	struct drm_i915_gem_object *old_vid_bo;
+	struct i915_vma *vma;
+	struct i915_vma *old_vma;
 	bool active;
 	bool pfit_active;
 	u32 pfit_vscale_ratio; /* shifted-point number, (1<<12) == 1.0 */
@@ -183,8 +184,7 @@ struct intel_overlay {
 	u32 flip_addr;
 	struct drm_i915_gem_object *reg_bo;
 	/* flip handling */
-	struct drm_i915_gem_request *last_flip_req;
-	void (*flip_tail)(struct intel_overlay *);
+	struct i915_gem_active last_flip;
 };
 
 static struct overlay_registers __iomem *
@@ -196,7 +196,7 @@ intel_overlay_map_regs(struct intel_overlay *overlay)
 	if (OVERLAY_NEEDS_PHYSICAL(dev_priv))
 		regs = (struct overlay_registers __iomem *)overlay->reg_bo->phys_handle->vaddr;
 	else
-		regs = io_mapping_map_wc(dev_priv->ggtt.mappable,
+		regs = io_mapping_map_wc(&dev_priv->ggtt.mappable,
 					 overlay->flip_addr,
 					 PAGE_SIZE);
 
@@ -210,37 +210,46 @@ static void intel_overlay_unmap_regs(struct intel_overlay *overlay,
 		io_mapping_unmap(regs);
 }
 
-static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
+static void intel_overlay_submit_request(struct intel_overlay *overlay,
 					 struct drm_i915_gem_request *req,
-					 void (*tail)(struct intel_overlay *))
+					 i915_gem_retire_fn retire)
 {
-	int ret;
-
-	WARN_ON(overlay->last_flip_req);
-	i915_gem_request_assign(&overlay->last_flip_req, req);
+	GEM_BUG_ON(i915_gem_active_peek(&overlay->last_flip,
+					&overlay->i915->drm.struct_mutex));
+	overlay->last_flip.retire = retire;
+	i915_gem_active_set(&overlay->last_flip, req);
 	i915_add_request(req);
+}
 
-	overlay->flip_tail = tail;
-	ret = i915_wait_request(overlay->last_flip_req);
-	if (ret)
-		return ret;
+static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
+					 struct drm_i915_gem_request *req,
+					 i915_gem_retire_fn retire)
+{
+	intel_overlay_submit_request(overlay, req, retire);
+	return i915_gem_active_retire(&overlay->last_flip,
+				      &overlay->i915->drm.struct_mutex);
+}
 
-	i915_gem_request_assign(&overlay->last_flip_req, NULL);
-	return 0;
+static struct drm_i915_gem_request *alloc_request(struct intel_overlay *overlay)
+{
+	struct drm_i915_private *dev_priv = overlay->i915;
+	struct intel_engine_cs *engine = &dev_priv->engine[RCS];
+
+	return i915_gem_request_alloc(engine, dev_priv->kernel_context);
 }
 
 /* overlay needs to be disable in OCMD reg */
 static int intel_overlay_on(struct intel_overlay *overlay)
 {
 	struct drm_i915_private *dev_priv = overlay->i915;
-	struct intel_engine_cs *engine = &dev_priv->engine[RCS];
 	struct drm_i915_gem_request *req;
+	struct intel_ring *ring;
 	int ret;
 
 	WARN_ON(overlay->active);
 	WARN_ON(IS_I830(dev_priv) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE));
 
-	req = i915_gem_request_alloc(engine, NULL);
+	req = alloc_request(overlay);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
@@ -252,11 +261,12 @@ static int intel_overlay_on(struct intel_overlay *overlay)
 
 	overlay->active = true;
 
-	intel_ring_emit(engine, MI_OVERLAY_FLIP | MI_OVERLAY_ON);
-	intel_ring_emit(engine, overlay->flip_addr | OFC_UPDATE);
-	intel_ring_emit(engine, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	ring = req->ring;
+	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_ON);
+	intel_ring_emit(ring, overlay->flip_addr | OFC_UPDATE);
+	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return intel_overlay_do_wait_request(overlay, req, NULL);
 }
@@ -266,8 +276,8 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
 				  bool load_polyphase_filter)
 {
 	struct drm_i915_private *dev_priv = overlay->i915;
-	struct intel_engine_cs *engine = &dev_priv->engine[RCS];
 	struct drm_i915_gem_request *req;
+	struct intel_ring *ring;
 	u32 flip_addr = overlay->flip_addr;
 	u32 tmp;
 	int ret;
@@ -282,7 +292,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
 	if (tmp & (1 << 17))
 		DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp);
 
-	req = i915_gem_request_alloc(engine, NULL);
+	req = alloc_request(overlay);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
@@ -292,38 +302,48 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
 		return ret;
 	}
 
-	intel_ring_emit(engine, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
-	intel_ring_emit(engine, flip_addr);
-	intel_ring_advance(engine);
+	ring = req->ring;
+	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
+	intel_ring_emit(ring, flip_addr);
+	intel_ring_advance(ring);
 
-	WARN_ON(overlay->last_flip_req);
-	i915_gem_request_assign(&overlay->last_flip_req, req);
-	i915_add_request(req);
+	intel_overlay_submit_request(overlay, req, NULL);
 
 	return 0;
 }
 
-static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
+static void intel_overlay_release_old_vid_tail(struct i915_gem_active *active,
+					       struct drm_i915_gem_request *req)
 {
-	struct drm_i915_gem_object *obj = overlay->old_vid_bo;
+	struct intel_overlay *overlay =
+		container_of(active, typeof(*overlay), last_flip);
+	struct i915_vma *vma;
 
-	i915_gem_object_ggtt_unpin(obj);
-	drm_gem_object_unreference(&obj->base);
+	vma = fetch_and_zero(&overlay->old_vma);
+	if (WARN_ON(!vma))
+		return;
 
-	overlay->old_vid_bo = NULL;
+	i915_gem_track_fb(vma->obj, NULL,
+			  INTEL_FRONTBUFFER_OVERLAY(overlay->crtc->pipe));
+
+	i915_gem_object_unpin_from_display_plane(vma);
+	i915_vma_put(vma);
 }
 
-static void intel_overlay_off_tail(struct intel_overlay *overlay)
+static void intel_overlay_off_tail(struct i915_gem_active *active,
+				   struct drm_i915_gem_request *req)
 {
-	struct drm_i915_gem_object *obj = overlay->vid_bo;
+	struct intel_overlay *overlay =
+		container_of(active, typeof(*overlay), last_flip);
+	struct i915_vma *vma;
 
 	/* never have the overlay hw on without showing a frame */
-	if (WARN_ON(!obj))
+	vma = fetch_and_zero(&overlay->vma);
+	if (WARN_ON(!vma))
 		return;
 
-	i915_gem_object_ggtt_unpin(obj);
-	drm_gem_object_unreference(&obj->base);
-	overlay->vid_bo = NULL;
+	i915_gem_object_unpin_from_display_plane(vma);
+	i915_vma_put(vma);
 
 	overlay->crtc->overlay = NULL;
 	overlay->crtc = NULL;
@@ -334,8 +354,8 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay)
 static int intel_overlay_off(struct intel_overlay *overlay)
 {
 	struct drm_i915_private *dev_priv = overlay->i915;
-	struct intel_engine_cs *engine = &dev_priv->engine[RCS];
 	struct drm_i915_gem_request *req;
+	struct intel_ring *ring;
 	u32 flip_addr = overlay->flip_addr;
 	int ret;
 
@@ -347,7 +367,7 @@ static int intel_overlay_off(struct intel_overlay *overlay)
 	 * of the hw. Do it in both cases */
 	flip_addr |= OFC_UPDATE;
 
-	req = i915_gem_request_alloc(engine, NULL);
+	req = alloc_request(overlay);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
@@ -357,46 +377,36 @@ static int intel_overlay_off(struct intel_overlay *overlay)
 		return ret;
 	}
 
+	ring = req->ring;
 	/* wait for overlay to go idle */
-	intel_ring_emit(engine, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
-	intel_ring_emit(engine, flip_addr);
-	intel_ring_emit(engine, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
+	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
+	intel_ring_emit(ring, flip_addr);
+	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
 	/* turn overlay off */
 	if (IS_I830(dev_priv)) {
 		/* Workaround: Don't disable the overlay fully, since otherwise
 		 * it dies on the next OVERLAY_ON cmd. */
-		intel_ring_emit(engine, MI_NOOP);
-		intel_ring_emit(engine, MI_NOOP);
-		intel_ring_emit(engine, MI_NOOP);
+		intel_ring_emit(ring, MI_NOOP);
+		intel_ring_emit(ring, MI_NOOP);
+		intel_ring_emit(ring, MI_NOOP);
 	} else {
-		intel_ring_emit(engine, MI_OVERLAY_FLIP | MI_OVERLAY_OFF);
-		intel_ring_emit(engine, flip_addr);
-		intel_ring_emit(engine,
+		intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_OFF);
+		intel_ring_emit(ring, flip_addr);
+		intel_ring_emit(ring,
 				MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
 	}
-	intel_ring_advance(engine);
+	intel_ring_advance(ring);
 
-	return intel_overlay_do_wait_request(overlay, req, intel_overlay_off_tail);
+	return intel_overlay_do_wait_request(overlay, req,
+					     intel_overlay_off_tail);
 }
 
 /* recover from an interruption due to a signal
  * We have to be careful not to repeat work forever an make forward progess. */
 static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
 {
-	int ret;
-
-	if (overlay->last_flip_req == NULL)
-		return 0;
-
-	ret = i915_wait_request(overlay->last_flip_req);
-	if (ret)
-		return ret;
-
-	if (overlay->flip_tail)
-		overlay->flip_tail(overlay);
-
-	i915_gem_request_assign(&overlay->last_flip_req, NULL);
-	return 0;
+	return i915_gem_active_retire(&overlay->last_flip,
+				      &overlay->i915->drm.struct_mutex);
 }
 
 /* Wait for pending overlay flip and release old frame.
@@ -406,7 +416,6 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
 static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 {
 	struct drm_i915_private *dev_priv = overlay->i915;
-	struct intel_engine_cs *engine = &dev_priv->engine[RCS];
 	int ret;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
@@ -414,14 +423,15 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 	/* Only wait if there is actually an old frame to release to
 	 * guarantee forward progress.
 	 */
-	if (!overlay->old_vid_bo)
+	if (!overlay->old_vma)
 		return 0;
 
 	if (I915_READ(ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT) {
 		/* synchronous slowpath */
 		struct drm_i915_gem_request *req;
+		struct intel_ring *ring;
 
-		req = i915_gem_request_alloc(engine, NULL);
+		req = alloc_request(overlay);
 		if (IS_ERR(req))
 			return PTR_ERR(req);
 
@@ -431,22 +441,19 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 			return ret;
 		}
 
-		intel_ring_emit(engine,
+		ring = req->ring;
+		intel_ring_emit(ring,
 				MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
-		intel_ring_emit(engine, MI_NOOP);
-		intel_ring_advance(engine);
+		intel_ring_emit(ring, MI_NOOP);
+		intel_ring_advance(ring);
 
 		ret = intel_overlay_do_wait_request(overlay, req,
 						    intel_overlay_release_old_vid_tail);
 		if (ret)
 			return ret;
-	}
+	} else
+		intel_overlay_release_old_vid_tail(&overlay->last_flip, NULL);
 
-	intel_overlay_release_old_vid_tail(overlay);
-
-
-	i915_gem_track_fb(overlay->old_vid_bo, NULL,
-			  INTEL_FRONTBUFFER_OVERLAY(overlay->crtc->pipe));
 	return 0;
 }
 
@@ -459,7 +466,6 @@ void intel_overlay_reset(struct drm_i915_private *dev_priv)
 
 	intel_overlay_release_old_vid(overlay);
 
-	overlay->last_flip_req = NULL;
 	overlay->old_xscale = 0;
 	overlay->old_yscale = 0;
 	overlay->crtc = NULL;
@@ -740,6 +746,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	struct drm_i915_private *dev_priv = overlay->i915;
 	u32 swidth, swidthsw, sheight, ostride;
 	enum pipe pipe = overlay->crtc->pipe;
+	struct i915_vma *vma;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 	WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex));
@@ -748,12 +755,12 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	if (ret != 0)
 		return ret;
 
-	ret = i915_gem_object_pin_to_display_plane(new_bo, 0,
+	vma = i915_gem_object_pin_to_display_plane(new_bo, 0,
 						   &i915_ggtt_view_normal);
-	if (ret != 0)
-		return ret;
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
 
-	ret = i915_gem_object_put_fence(new_bo);
+	ret = i915_vma_put_fence(vma);
 	if (ret)
 		goto out_unpin;
 
@@ -794,7 +801,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	swidth = params->src_w;
 	swidthsw = calc_swidthsw(dev_priv, params->offset_Y, tmp_width);
 	sheight = params->src_h;
-	iowrite32(i915_gem_obj_ggtt_offset(new_bo) + params->offset_Y, &regs->OBUF_0Y);
+	iowrite32(i915_ggtt_offset(vma) + params->offset_Y, &regs->OBUF_0Y);
 	ostride = params->stride_Y;
 
 	if (params->format & I915_OVERLAY_YUV_PLANAR) {
@@ -808,8 +815,10 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 				      params->src_w/uv_hscale);
 		swidthsw |= max_t(u32, tmp_U, tmp_V) << 16;
 		sheight |= (params->src_h/uv_vscale) << 16;
-		iowrite32(i915_gem_obj_ggtt_offset(new_bo) + params->offset_U, &regs->OBUF_0U);
-		iowrite32(i915_gem_obj_ggtt_offset(new_bo) + params->offset_V, &regs->OBUF_0V);
+		iowrite32(i915_ggtt_offset(vma) + params->offset_U,
+			  &regs->OBUF_0U);
+		iowrite32(i915_ggtt_offset(vma) + params->offset_V,
+			  &regs->OBUF_0V);
 		ostride |= params->stride_UV << 16;
 	}
 
@@ -830,19 +839,18 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	if (ret)
 		goto out_unpin;
 
-	i915_gem_track_fb(overlay->vid_bo, new_bo,
+	i915_gem_track_fb(overlay->vma->obj, new_bo,
 			  INTEL_FRONTBUFFER_OVERLAY(pipe));
 
-	overlay->old_vid_bo = overlay->vid_bo;
-	overlay->vid_bo = new_bo;
+	overlay->old_vma = overlay->vma;
+	overlay->vma = vma;
 
-	intel_frontbuffer_flip(&dev_priv->drm,
-			       INTEL_FRONTBUFFER_OVERLAY(pipe));
+	intel_frontbuffer_flip(dev_priv, INTEL_FRONTBUFFER_OVERLAY(pipe));
 
 	return 0;
 
 out_unpin:
-	i915_gem_object_ggtt_unpin(new_bo);
+	i915_gem_object_unpin_from_display_plane(vma);
 	return ret;
 }
 
@@ -870,12 +878,7 @@ int intel_overlay_switch_off(struct intel_overlay *overlay)
 	iowrite32(0, &regs->OCMD);
 	intel_overlay_unmap_regs(overlay, regs);
 
-	ret = intel_overlay_off(overlay);
-	if (ret != 0)
-		return ret;
-
-	intel_overlay_off_tail(overlay);
-	return 0;
+	return intel_overlay_off(overlay);
 }
 
 static int check_overlay_possible_on_crtc(struct intel_overlay *overlay,
@@ -1122,9 +1125,8 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 	}
 	crtc = to_intel_crtc(drmmode_crtc);
 
-	new_bo = to_intel_bo(drm_gem_object_lookup(file_priv,
-						   put_image_rec->bo_handle));
-	if (&new_bo->base == NULL) {
+	new_bo = i915_gem_object_lookup(file_priv, put_image_rec->bo_handle);
+	if (!new_bo) {
 		ret = -ENOENT;
 		goto out_free;
 	}
@@ -1132,7 +1134,7 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 	drm_modeset_lock_all(dev);
 	mutex_lock(&dev->struct_mutex);
 
-	if (new_bo->tiling_mode) {
+	if (i915_gem_object_is_tiled(new_bo)) {
 		DRM_DEBUG_KMS("buffer used for overlay image can not be tiled\n");
 		ret = -EINVAL;
 		goto out_unlock;
@@ -1220,7 +1222,7 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 out_unlock:
 	mutex_unlock(&dev->struct_mutex);
 	drm_modeset_unlock_all(dev);
-	drm_gem_object_unreference_unlocked(&new_bo->base);
+	i915_gem_object_put_unlocked(new_bo);
 out_free:
 	kfree(params);
 
@@ -1371,6 +1373,7 @@ void intel_setup_overlay(struct drm_i915_private *dev_priv)
 	struct intel_overlay *overlay;
 	struct drm_i915_gem_object *reg_bo;
 	struct overlay_registers __iomem *regs;
+	struct i915_vma *vma = NULL;
 	int ret;
 
 	if (!HAS_OVERLAY(dev_priv))
@@ -1404,12 +1407,14 @@ void intel_setup_overlay(struct drm_i915_private *dev_priv)
 		}
 		overlay->flip_addr = reg_bo->phys_handle->busaddr;
 	} else {
-		ret = i915_gem_obj_ggtt_pin(reg_bo, PAGE_SIZE, PIN_MAPPABLE);
-		if (ret) {
+		vma = i915_gem_object_ggtt_pin(reg_bo, NULL,
+					       0, PAGE_SIZE, PIN_MAPPABLE);
+		if (IS_ERR(vma)) {
 			DRM_ERROR("failed to pin overlay register bo\n");
+			ret = PTR_ERR(vma);
 			goto out_free_bo;
 		}
-		overlay->flip_addr = i915_gem_obj_ggtt_offset(reg_bo);
+		overlay->flip_addr = i915_ggtt_offset(vma);
 
 		ret = i915_gem_object_set_to_gtt_domain(reg_bo, true);
 		if (ret) {
@@ -1441,10 +1446,10 @@ void intel_setup_overlay(struct drm_i915_private *dev_priv)
 	return;
 
 out_unpin_bo:
-	if (!OVERLAY_NEEDS_PHYSICAL(dev_priv))
-		i915_gem_object_ggtt_unpin(reg_bo);
+	if (vma)
+		i915_vma_unpin(vma);
 out_free_bo:
-	drm_gem_object_unreference(&reg_bo->base);
+	i915_gem_object_put(reg_bo);
 out_free:
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 	kfree(overlay);
@@ -1461,7 +1466,7 @@ void intel_cleanup_overlay(struct drm_i915_private *dev_priv)
 	 * hardware should be off already */
 	WARN_ON(dev_priv->overlay->active);
 
-	drm_gem_object_unreference_unlocked(&dev_priv->overlay->reg_bo->base);
+	i915_gem_object_put_unlocked(dev_priv->overlay->reg_bo);
 	kfree(dev_priv->overlay);
 }
 
@@ -1484,7 +1489,7 @@ intel_overlay_map_regs_atomic(struct intel_overlay *overlay)
 		regs = (struct overlay_registers __iomem *)
 			overlay->reg_bo->phys_handle->vaddr;
 	else
-		regs = io_mapping_map_atomic_wc(dev_priv->ggtt.mappable,
+		regs = io_mapping_map_atomic_wc(&dev_priv->ggtt.mappable,
 						overlay->flip_addr);
 
 	return regs;
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index 96c65d77e886..be4b4d546fd9 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -841,7 +841,7 @@ static void lpt_enable_backlight(struct intel_connector *connector)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
-	u32 pch_ctl1, pch_ctl2;
+	u32 pch_ctl1, pch_ctl2, schicken;
 
 	pch_ctl1 = I915_READ(BLC_PWM_PCH_CTL1);
 	if (pch_ctl1 & BLM_PCH_PWM_ENABLE) {
@@ -850,6 +850,22 @@ static void lpt_enable_backlight(struct intel_connector *connector)
 		I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1);
 	}
 
+	if (HAS_PCH_LPT(dev_priv)) {
+		schicken = I915_READ(SOUTH_CHICKEN2);
+		if (panel->backlight.alternate_pwm_increment)
+			schicken |= LPT_PWM_GRANULARITY;
+		else
+			schicken &= ~LPT_PWM_GRANULARITY;
+		I915_WRITE(SOUTH_CHICKEN2, schicken);
+	} else {
+		schicken = I915_READ(SOUTH_CHICKEN1);
+		if (panel->backlight.alternate_pwm_increment)
+			schicken |= SPT_PWM_GRANULARITY;
+		else
+			schicken &= ~SPT_PWM_GRANULARITY;
+		I915_WRITE(SOUTH_CHICKEN1, schicken);
+	}
+
 	pch_ctl2 = panel->backlight.max << 16;
 	I915_WRITE(BLC_PWM_PCH_CTL2, pch_ctl2);
 
@@ -1242,10 +1258,10 @@ static u32 bxt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
  */
 static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 {
-	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_panel *panel = &connector->panel;
 	u32 mul;
 
-	if (I915_READ(SOUTH_CHICKEN1) & SPT_PWM_GRANULARITY)
+	if (panel->backlight.alternate_pwm_increment)
 		mul = 128;
 	else
 		mul = 16;
@@ -1261,9 +1277,10 @@ static u32 spt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 static u32 lpt_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_panel *panel = &connector->panel;
 	u32 mul, clock;
 
-	if (I915_READ(SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY)
+	if (panel->backlight.alternate_pwm_increment)
 		mul = 16;
 	else
 		mul = 128;
@@ -1414,6 +1431,13 @@ static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unus
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
 	u32 pch_ctl1, pch_ctl2, val;
+	bool alt;
+
+	if (HAS_PCH_LPT(dev_priv))
+		alt = I915_READ(SOUTH_CHICKEN2) & LPT_PWM_GRANULARITY;
+	else
+		alt = I915_READ(SOUTH_CHICKEN1) & SPT_PWM_GRANULARITY;
+	panel->backlight.alternate_pwm_increment = alt;
 
 	pch_ctl1 = I915_READ(BLC_PWM_PCH_CTL1);
 	panel->backlight.active_low_pwm = pch_ctl1 & BLM_PCH_POLARITY;
@@ -1430,10 +1454,11 @@ static int lpt_setup_backlight(struct intel_connector *connector, enum pipe unus
 	panel->backlight.min = get_backlight_min_vbt(connector);
 
 	val = lpt_get_backlight(connector);
-	panel->backlight.level = intel_panel_compute_brightness(connector, val);
+	val = intel_panel_compute_brightness(connector, val);
+	panel->backlight.level = clamp(val, panel->backlight.min,
+				       panel->backlight.max);
 
-	panel->backlight.enabled = (pch_ctl1 & BLM_PCH_PWM_ENABLE) &&
-		panel->backlight.level != 0;
+	panel->backlight.enabled = pch_ctl1 & BLM_PCH_PWM_ENABLE;
 
 	return 0;
 }
@@ -1459,11 +1484,13 @@ static int pch_setup_backlight(struct intel_connector *connector, enum pipe unus
 	panel->backlight.min = get_backlight_min_vbt(connector);
 
 	val = pch_get_backlight(connector);
-	panel->backlight.level = intel_panel_compute_brightness(connector, val);
+	val = intel_panel_compute_brightness(connector, val);
+	panel->backlight.level = clamp(val, panel->backlight.min,
+				       panel->backlight.max);
 
 	cpu_ctl2 = I915_READ(BLC_PWM_CPU_CTL2);
 	panel->backlight.enabled = (cpu_ctl2 & BLM_PWM_ENABLE) &&
-		(pch_ctl1 & BLM_PCH_PWM_ENABLE) && panel->backlight.level != 0;
+		(pch_ctl1 & BLM_PCH_PWM_ENABLE);
 
 	return 0;
 }
@@ -1498,9 +1525,11 @@ static int i9xx_setup_backlight(struct intel_connector *connector, enum pipe unu
 	panel->backlight.min = get_backlight_min_vbt(connector);
 
 	val = i9xx_get_backlight(connector);
-	panel->backlight.level = intel_panel_compute_brightness(connector, val);
+	val = intel_panel_compute_brightness(connector, val);
+	panel->backlight.level = clamp(val, panel->backlight.min,
+				       panel->backlight.max);
 
-	panel->backlight.enabled = panel->backlight.level != 0;
+	panel->backlight.enabled = val != 0;
 
 	return 0;
 }
@@ -1530,10 +1559,11 @@ static int i965_setup_backlight(struct intel_connector *connector, enum pipe unu
 	panel->backlight.min = get_backlight_min_vbt(connector);
 
 	val = i9xx_get_backlight(connector);
-	panel->backlight.level = intel_panel_compute_brightness(connector, val);
+	val = intel_panel_compute_brightness(connector, val);
+	panel->backlight.level = clamp(val, panel->backlight.min,
+				       panel->backlight.max);
 
-	panel->backlight.enabled = (ctl2 & BLM_PWM_ENABLE) &&
-		panel->backlight.level != 0;
+	panel->backlight.enabled = ctl2 & BLM_PWM_ENABLE;
 
 	return 0;
 }
@@ -1562,10 +1592,11 @@ static int vlv_setup_backlight(struct intel_connector *connector, enum pipe pipe
 	panel->backlight.min = get_backlight_min_vbt(connector);
 
 	val = _vlv_get_backlight(dev_priv, pipe);
-	panel->backlight.level = intel_panel_compute_brightness(connector, val);
+	val = intel_panel_compute_brightness(connector, val);
+	panel->backlight.level = clamp(val, panel->backlight.min,
+				       panel->backlight.max);
 
-	panel->backlight.enabled = (ctl2 & BLM_PWM_ENABLE) &&
-		panel->backlight.level != 0;
+	panel->backlight.enabled = ctl2 & BLM_PWM_ENABLE;
 
 	return 0;
 }
@@ -1607,10 +1638,11 @@ bxt_setup_backlight(struct intel_connector *connector, enum pipe unused)
 		return -ENODEV;
 
 	val = bxt_get_backlight(connector);
-	panel->backlight.level = intel_panel_compute_brightness(connector, val);
+	val = intel_panel_compute_brightness(connector, val);
+	panel->backlight.level = clamp(val, panel->backlight.min,
+				       panel->backlight.max);
 
-	panel->backlight.enabled = (pwm_ctl & BXT_BLC_PWM_ENABLE) &&
-		panel->backlight.level != 0;
+	panel->backlight.enabled = pwm_ctl & BXT_BLC_PWM_ENABLE;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index f4f3fcc8b3be..a2f751cd187a 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -340,6 +340,11 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 		I915_WRITE(FW_BLC_SELF, val);
 		POSTING_READ(FW_BLC_SELF);
 	} else if (IS_I915GM(dev)) {
+		/*
+		 * FIXME can't find a bit like this for 915G, and
+		 * and yet it does have the related watermark in
+		 * FW_BLC_SELF. What's going on?
+		 */
 		val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 			       _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 		I915_WRITE(INSTPM, val);
@@ -960,7 +965,7 @@ static uint16_t vlv_compute_wm_level(struct intel_plane *plane,
 	if (dev_priv->wm.pri_latency[level] == 0)
 		return USHRT_MAX;
 
-	if (!state->visible)
+	if (!state->base.visible)
 		return 0;
 
 	cpp = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
@@ -1002,7 +1007,7 @@ static void vlv_compute_fifo(struct intel_crtc *crtc)
 		if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
 			continue;
 
-		if (state->visible) {
+		if (state->base.visible) {
 			wm_state->num_active_planes++;
 			total_rate += drm_format_plane_cpp(state->base.fb->pixel_format, 0);
 		}
@@ -1018,7 +1023,7 @@ static void vlv_compute_fifo(struct intel_crtc *crtc)
 			continue;
 		}
 
-		if (!state->visible) {
+		if (!state->base.visible) {
 			plane->wm.fifo_size = 0;
 			continue;
 		}
@@ -1118,7 +1123,7 @@ static void vlv_compute_wm(struct intel_crtc *crtc)
 		struct intel_plane_state *state =
 			to_intel_plane_state(plane->base.state);
 
-		if (!state->visible)
+		if (!state->base.visible)
 			continue;
 
 		/* normal watermarks */
@@ -1580,7 +1585,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 		obj = intel_fb_obj(enabled->primary->state->fb);
 
 		/* self-refresh seems busted with untiled */
-		if (obj->tiling_mode == I915_TILING_NONE)
+		if (!i915_gem_object_is_tiled(obj))
 			enabled = NULL;
 	}
 
@@ -1604,6 +1609,9 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 		unsigned long line_time_us;
 		int entries;
 
+		if (IS_I915GM(dev) || IS_I945GM(dev))
+			cpp = 4;
+
 		line_time_us = max(htotal * 1000 / clock, 1);
 
 		/* Use ns/us then divide to preserve precision */
@@ -1618,7 +1626,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 		if (IS_I945G(dev) || IS_I945GM(dev))
 			I915_WRITE(FW_BLC_SELF,
 				   FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
-		else if (IS_I915GM(dev))
+		else
 			I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
 	}
 
@@ -1767,7 +1775,7 @@ static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
 		drm_format_plane_cpp(pstate->base.fb->pixel_format, 0) : 0;
 	uint32_t method1, method2;
 
-	if (!cstate->base.active || !pstate->visible)
+	if (!cstate->base.active || !pstate->base.visible)
 		return 0;
 
 	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), cpp, mem_value);
@@ -1777,7 +1785,7 @@ static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
 
 	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
 				 cstate->base.adjusted_mode.crtc_htotal,
-				 drm_rect_width(&pstate->dst),
+				 drm_rect_width(&pstate->base.dst),
 				 cpp, mem_value);
 
 	return min(method1, method2);
@@ -1795,13 +1803,13 @@ static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
 		drm_format_plane_cpp(pstate->base.fb->pixel_format, 0) : 0;
 	uint32_t method1, method2;
 
-	if (!cstate->base.active || !pstate->visible)
+	if (!cstate->base.active || !pstate->base.visible)
 		return 0;
 
 	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), cpp, mem_value);
 	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
 				 cstate->base.adjusted_mode.crtc_htotal,
-				 drm_rect_width(&pstate->dst),
+				 drm_rect_width(&pstate->base.dst),
 				 cpp, mem_value);
 	return min(method1, method2);
 }
@@ -1820,7 +1828,7 @@ static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
 	 * this is necessary to avoid flickering.
 	 */
 	int cpp = 4;
-	int width = pstate->visible ? pstate->base.crtc_w : 64;
+	int width = pstate->base.visible ? pstate->base.crtc_w : 64;
 
 	if (!cstate->base.active)
 		return 0;
@@ -1838,10 +1846,10 @@ static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
 	int cpp = pstate->base.fb ?
 		drm_format_plane_cpp(pstate->base.fb->pixel_format, 0) : 0;
 
-	if (!cstate->base.active || !pstate->visible)
+	if (!cstate->base.active || !pstate->base.visible)
 		return 0;
 
-	return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->dst), cpp);
+	return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
 }
 
 static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
@@ -2119,32 +2127,34 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
 				GEN9_MEM_LATENCY_LEVEL_MASK;
 
 		/*
+		 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
+		 * need to be disabled. We make sure to sanitize the values out
+		 * of the punit to satisfy this requirement.
+		 */
+		for (level = 1; level <= max_level; level++) {
+			if (wm[level] == 0) {
+				for (i = level + 1; i <= max_level; i++)
+					wm[i] = 0;
+				break;
+			}
+		}
+
+		/*
 		 * WaWmMemoryReadLatency:skl
 		 *
 		 * punit doesn't take into account the read latency so we need
-		 * to add 2us to the various latency levels we retrieve from
-		 * the punit.
-		 *   - W0 is a bit special in that it's the only level that
-		 *   can't be disabled if we want to have display working, so
-		 *   we always add 2us there.
-		 *   - For levels >=1, punit returns 0us latency when they are
-		 *   disabled, so we respect that and don't add 2us then
-		 *
-		 * Additionally, if a level n (n > 1) has a 0us latency, all
-		 * levels m (m >= n) need to be disabled. We make sure to
-		 * sanitize the values out of the punit to satisfy this
-		 * requirement.
+		 * to add 2us to the various latency levels we retrieve from the
+		 * punit when level 0 response data us 0us.
 		 */
-		wm[0] += 2;
-		for (level = 1; level <= max_level; level++)
-			if (wm[level] != 0)
+		if (wm[0] == 0) {
+			wm[0] += 2;
+			for (level = 1; level <= max_level; level++) {
+				if (wm[level] == 0)
+					break;
 				wm[level] += 2;
-			else {
-				for (i = level + 1; i <= max_level; i++)
-					wm[i] = 0;
-
-				break;
 			}
+		}
+
 	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
 		uint64_t sskpd = I915_READ64(MCH_SSKPD);
 
@@ -2358,10 +2368,10 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
 
 	pipe_wm->pipe_enabled = cstate->base.active;
 	if (sprstate) {
-		pipe_wm->sprites_enabled = sprstate->visible;
-		pipe_wm->sprites_scaled = sprstate->visible &&
-			(drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 ||
-			 drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16);
+		pipe_wm->sprites_enabled = sprstate->base.visible;
+		pipe_wm->sprites_scaled = sprstate->base.visible &&
+			(drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
+			 drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
 	}
 
 	usable_level = max_level;
@@ -2845,13 +2855,7 @@ bool ilk_disable_lp_wm(struct drm_device *dev)
 	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
 }
 
-/*
- * On gen9, we need to allocate Display Data Buffer (DDB) portions to the
- * different active planes.
- */
-
-#define SKL_DDB_SIZE		896	/* in blocks */
-#define BXT_DDB_SIZE		512
+#define SKL_SAGV_BLOCK_TIME	30 /* µs */
 
 /*
  * Return the index of a plane in the SKL DDB and wm result arrays.  Primary
@@ -2875,6 +2879,173 @@ skl_wm_plane_id(const struct intel_plane *plane)
 	}
 }
 
+static bool
+intel_has_sagv(struct drm_i915_private *dev_priv)
+{
+	if (IS_KABYLAKE(dev_priv))
+		return true;
+
+	if (IS_SKYLAKE(dev_priv) &&
+	    dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED)
+		return true;
+
+	return false;
+}
+
+/*
+ * SAGV dynamically adjusts the system agent voltage and clock frequencies
+ * depending on power and performance requirements. The display engine access
+ * to system memory is blocked during the adjustment time. Because of the
+ * blocking time, having this enabled can cause full system hangs and/or pipe
+ * underruns if we don't meet all of the following requirements:
+ *
+ *  - <= 1 pipe enabled
+ *  - All planes can enable watermarks for latencies >= SAGV engine block time
+ *  - We're not using an interlaced display configuration
+ */
+int
+intel_enable_sagv(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	if (!intel_has_sagv(dev_priv))
+		return 0;
+
+	if (dev_priv->sagv_status == I915_SAGV_ENABLED)
+		return 0;
+
+	DRM_DEBUG_KMS("Enabling the SAGV\n");
+	mutex_lock(&dev_priv->rps.hw_lock);
+
+	ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
+				      GEN9_SAGV_ENABLE);
+
+	/* We don't need to wait for the SAGV when enabling */
+	mutex_unlock(&dev_priv->rps.hw_lock);
+
+	/*
+	 * Some skl systems, pre-release machines in particular,
+	 * don't actually have an SAGV.
+	 */
+	if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
+		DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
+		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
+		return 0;
+	} else if (ret < 0) {
+		DRM_ERROR("Failed to enable the SAGV\n");
+		return ret;
+	}
+
+	dev_priv->sagv_status = I915_SAGV_ENABLED;
+	return 0;
+}
+
+static int
+intel_do_sagv_disable(struct drm_i915_private *dev_priv)
+{
+	int ret;
+	uint32_t temp = GEN9_SAGV_DISABLE;
+
+	ret = sandybridge_pcode_read(dev_priv, GEN9_PCODE_SAGV_CONTROL,
+				     &temp);
+	if (ret)
+		return ret;
+	else
+		return temp & GEN9_SAGV_IS_DISABLED;
+}
+
+int
+intel_disable_sagv(struct drm_i915_private *dev_priv)
+{
+	int ret, result;
+
+	if (!intel_has_sagv(dev_priv))
+		return 0;
+
+	if (dev_priv->sagv_status == I915_SAGV_DISABLED)
+		return 0;
+
+	DRM_DEBUG_KMS("Disabling the SAGV\n");
+	mutex_lock(&dev_priv->rps.hw_lock);
+
+	/* bspec says to keep retrying for at least 1 ms */
+	ret = wait_for(result = intel_do_sagv_disable(dev_priv), 1);
+	mutex_unlock(&dev_priv->rps.hw_lock);
+
+	if (ret == -ETIMEDOUT) {
+		DRM_ERROR("Request to disable SAGV timed out\n");
+		return -ETIMEDOUT;
+	}
+
+	/*
+	 * Some skl systems, pre-release machines in particular,
+	 * don't actually have an SAGV.
+	 */
+	if (IS_SKYLAKE(dev_priv) && result == -ENXIO) {
+		DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
+		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
+		return 0;
+	} else if (result < 0) {
+		DRM_ERROR("Failed to disable the SAGV\n");
+		return result;
+	}
+
+	dev_priv->sagv_status = I915_SAGV_DISABLED;
+	return 0;
+}
+
+bool intel_can_enable_sagv(struct drm_atomic_state *state)
+{
+	struct drm_device *dev = state->dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+	struct drm_crtc *crtc;
+	enum pipe pipe;
+	int level, plane;
+
+	if (!intel_has_sagv(dev_priv))
+		return false;
+
+	/*
+	 * SKL workaround: bspec recommends we disable the SAGV when we have
+	 * more then one pipe enabled
+	 *
+	 * If there are no active CRTCs, no additional checks need be performed
+	 */
+	if (hweight32(intel_state->active_crtcs) == 0)
+		return true;
+	else if (hweight32(intel_state->active_crtcs) > 1)
+		return false;
+
+	/* Since we're now guaranteed to only have one active CRTC... */
+	pipe = ffs(intel_state->active_crtcs) - 1;
+	crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+
+	if (crtc->state->mode.flags & DRM_MODE_FLAG_INTERLACE)
+		return false;
+
+	for_each_plane(dev_priv, pipe, plane) {
+		/* Skip this plane if it's not enabled */
+		if (intel_state->wm_results.plane[pipe][plane][0] == 0)
+			continue;
+
+		/* Find the highest enabled wm level for this plane */
+		for (level = ilk_wm_max_level(dev);
+		     intel_state->wm_results.plane[pipe][plane][level] == 0; --level)
+		     { }
+
+		/*
+		 * If any of the planes on this pipe don't enable wm levels
+		 * that incur memory latencies higher then 30µs we can't enable
+		 * the SAGV
+		 */
+		if (dev_priv->wm.skl_latency[level] < SKL_SAGV_BLOCK_TIME)
+			return false;
+	}
+
+	return true;
+}
+
 static void
 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
 				   const struct intel_crtc_state *cstate,
@@ -2901,10 +3072,8 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
 	else
 		*num_active = hweight32(dev_priv->active_crtcs);
 
-	if (IS_BROXTON(dev))
-		ddb_size = BXT_DDB_SIZE;
-	else
-		ddb_size = SKL_DDB_SIZE;
+	ddb_size = INTEL_INFO(dev_priv)->ddb_size;
+	WARN_ON(ddb_size == 0);
 
 	ddb_size -= 4; /* 4 blocks for bypass path allocation */
 
@@ -2996,14 +3165,14 @@ skl_plane_downscale_amount(const struct intel_plane_state *pstate)
 	uint32_t downscale_h, downscale_w;
 	uint32_t src_w, src_h, dst_w, dst_h;
 
-	if (WARN_ON(!pstate->visible))
+	if (WARN_ON(!pstate->base.visible))
 		return DRM_PLANE_HELPER_NO_SCALING;
 
 	/* n.b., src is 16.16 fixed point, dst is whole integer */
-	src_w = drm_rect_width(&pstate->src);
-	src_h = drm_rect_height(&pstate->src);
-	dst_w = drm_rect_width(&pstate->dst);
-	dst_h = drm_rect_height(&pstate->dst);
+	src_w = drm_rect_width(&pstate->base.src);
+	src_h = drm_rect_height(&pstate->base.src);
+	dst_w = drm_rect_width(&pstate->base.dst);
+	dst_h = drm_rect_height(&pstate->base.dst);
 	if (intel_rotation_90_or_270(pstate->base.rotation))
 		swap(dst_w, dst_h);
 
@@ -3025,15 +3194,15 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 	uint32_t width = 0, height = 0;
 	unsigned format = fb ? fb->pixel_format : DRM_FORMAT_XRGB8888;
 
-	if (!intel_pstate->visible)
+	if (!intel_pstate->base.visible)
 		return 0;
 	if (pstate->plane->type == DRM_PLANE_TYPE_CURSOR)
 		return 0;
 	if (y && format != DRM_FORMAT_NV12)
 		return 0;
 
-	width = drm_rect_width(&intel_pstate->src) >> 16;
-	height = drm_rect_height(&intel_pstate->src) >> 16;
+	width = drm_rect_width(&intel_pstate->base.src) >> 16;
+	height = drm_rect_height(&intel_pstate->base.src) >> 16;
 
 	if (intel_rotation_90_or_270(pstate->rotation))
 		swap(width, height);
@@ -3107,8 +3276,6 @@ skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate)
 		total_data_rate += intel_cstate->wm.skl.plane_y_data_rate[id];
 	}
 
-	WARN_ON(cstate->plane_mask && total_data_rate == 0);
-
 	return total_data_rate;
 }
 
@@ -3134,8 +3301,8 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate,
 	    fb->modifier[0] != I915_FORMAT_MOD_Yf_TILED)
 		return 8;
 
-	src_w = drm_rect_width(&intel_pstate->src) >> 16;
-	src_h = drm_rect_height(&intel_pstate->src) >> 16;
+	src_w = drm_rect_width(&intel_pstate->base.src) >> 16;
+	src_h = drm_rect_height(&intel_pstate->base.src) >> 16;
 
 	if (intel_rotation_90_or_270(pstate->rotation))
 		swap(src_w, src_h);
@@ -3226,7 +3393,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 		if (intel_plane->pipe != pipe)
 			continue;
 
-		if (!to_intel_plane_state(pstate)->visible) {
+		if (!to_intel_plane_state(pstate)->base.visible) {
 			minimum[id] = 0;
 			y_minimum[id] = 0;
 			continue;
@@ -3327,27 +3494,14 @@ static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t cpp, uint32_t latenc
 }
 
 static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
-			       uint32_t horiz_pixels, uint8_t cpp,
-			       uint64_t tiling, uint32_t latency)
+			       uint32_t latency, uint32_t plane_blocks_per_line)
 {
 	uint32_t ret;
-	uint32_t plane_bytes_per_line, plane_blocks_per_line;
 	uint32_t wm_intermediate_val;
 
 	if (latency == 0)
 		return UINT_MAX;
 
-	plane_bytes_per_line = horiz_pixels * cpp;
-
-	if (tiling == I915_FORMAT_MOD_Y_TILED ||
-	    tiling == I915_FORMAT_MOD_Yf_TILED) {
-		plane_bytes_per_line *= 4;
-		plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
-		plane_blocks_per_line /= 4;
-	} else {
-		plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
-	}
-
 	wm_intermediate_val = latency * pixel_rate;
 	ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
 				plane_blocks_per_line;
@@ -3363,7 +3517,7 @@ static uint32_t skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cst
 	uint64_t pixel_rate;
 
 	/* Shouldn't reach here on disabled planes... */
-	if (WARN_ON(!pstate->visible))
+	if (WARN_ON(!pstate->base.visible))
 		return 0;
 
 	/*
@@ -3398,14 +3552,15 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	uint8_t cpp;
 	uint32_t width = 0, height = 0;
 	uint32_t plane_pixel_rate;
+	uint32_t y_tile_minimum, y_min_scanlines;
 
-	if (latency == 0 || !cstate->base.active || !intel_pstate->visible) {
+	if (latency == 0 || !cstate->base.active || !intel_pstate->base.visible) {
 		*enabled = false;
 		return 0;
 	}
 
-	width = drm_rect_width(&intel_pstate->src) >> 16;
-	height = drm_rect_height(&intel_pstate->src) >> 16;
+	width = drm_rect_width(&intel_pstate->base.src) >> 16;
+	height = drm_rect_height(&intel_pstate->base.src) >> 16;
 
 	if (intel_rotation_90_or_270(pstate->rotation))
 		swap(width, height);
@@ -3413,38 +3568,51 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 	plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate);
 
+	if (intel_rotation_90_or_270(pstate->rotation)) {
+		int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
+			drm_format_plane_cpp(fb->pixel_format, 1) :
+			drm_format_plane_cpp(fb->pixel_format, 0);
+
+		switch (cpp) {
+		case 1:
+			y_min_scanlines = 16;
+			break;
+		case 2:
+			y_min_scanlines = 8;
+			break;
+		default:
+			WARN(1, "Unsupported pixel depth for rotation");
+		case 4:
+			y_min_scanlines = 4;
+			break;
+		}
+	} else {
+		y_min_scanlines = 4;
+	}
+
+	plane_bytes_per_line = width * cpp;
+	if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
+	    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
+		plane_blocks_per_line =
+		      DIV_ROUND_UP(plane_bytes_per_line * y_min_scanlines, 512);
+		plane_blocks_per_line /= y_min_scanlines;
+	} else if (fb->modifier[0] == DRM_FORMAT_MOD_NONE) {
+		plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512)
+					+ 1;
+	} else {
+		plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
+	}
+
 	method1 = skl_wm_method1(plane_pixel_rate, cpp, latency);
 	method2 = skl_wm_method2(plane_pixel_rate,
 				 cstate->base.adjusted_mode.crtc_htotal,
-				 width,
-				 cpp,
-				 fb->modifier[0],
-				 latency);
+				 latency,
+				 plane_blocks_per_line);
 
-	plane_bytes_per_line = width * cpp;
-	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
+	y_tile_minimum = plane_blocks_per_line * y_min_scanlines;
 
 	if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
 	    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
-		uint32_t min_scanlines = 4;
-		uint32_t y_tile_minimum;
-		if (intel_rotation_90_or_270(pstate->rotation)) {
-			int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
-				drm_format_plane_cpp(fb->pixel_format, 1) :
-				drm_format_plane_cpp(fb->pixel_format, 0);
-
-			switch (cpp) {
-			case 1:
-				min_scanlines = 16;
-				break;
-			case 2:
-				min_scanlines = 8;
-				break;
-			case 8:
-				WARN(1, "Unsupported pixel depth for rotation");
-			}
-		}
-		y_tile_minimum = plane_blocks_per_line * min_scanlines;
 		selected_result = max(method2, y_tile_minimum);
 	} else {
 		if ((ddb_allocation / plane_blocks_per_line) >= 1)
@@ -3458,10 +3626,12 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 
 	if (level >= 1 && level <= 7) {
 		if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
-		    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)
-			res_lines += 4;
-		else
+		    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
+			res_blocks += y_tile_minimum;
+			res_lines += y_min_scanlines;
+		} else {
 			res_blocks++;
+		}
 	}
 
 	if (res_blocks >= ddb_allocation || res_lines > 31) {
@@ -3680,183 +3850,82 @@ static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
 		I915_WRITE(reg, 0);
 }
 
-static void skl_write_wm_values(struct drm_i915_private *dev_priv,
-				const struct skl_wm_values *new)
+void skl_write_plane_wm(struct intel_crtc *intel_crtc,
+			const struct skl_wm_values *wm,
+			int plane)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_crtc *crtc;
-
-	for_each_intel_crtc(dev, crtc) {
-		int i, level, max_level = ilk_wm_max_level(dev);
-		enum pipe pipe = crtc->pipe;
-
-		if ((new->dirty_pipes & drm_crtc_mask(&crtc->base)) == 0)
-			continue;
-		if (!crtc->active)
-			continue;
-
-		I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
-
-		for (level = 0; level <= max_level; level++) {
-			for (i = 0; i < intel_num_planes(crtc); i++)
-				I915_WRITE(PLANE_WM(pipe, i, level),
-					   new->plane[pipe][i][level]);
-			I915_WRITE(CUR_WM(pipe, level),
-				   new->plane[pipe][PLANE_CURSOR][level]);
-		}
-		for (i = 0; i < intel_num_planes(crtc); i++)
-			I915_WRITE(PLANE_WM_TRANS(pipe, i),
-				   new->plane_trans[pipe][i]);
-		I915_WRITE(CUR_WM_TRANS(pipe),
-			   new->plane_trans[pipe][PLANE_CURSOR]);
-
-		for (i = 0; i < intel_num_planes(crtc); i++) {
-			skl_ddb_entry_write(dev_priv,
-					    PLANE_BUF_CFG(pipe, i),
-					    &new->ddb.plane[pipe][i]);
-			skl_ddb_entry_write(dev_priv,
-					    PLANE_NV12_BUF_CFG(pipe, i),
-					    &new->ddb.y_plane[pipe][i]);
-		}
+	struct drm_crtc *crtc = &intel_crtc->base;
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	int level, max_level = ilk_wm_max_level(dev);
+	enum pipe pipe = intel_crtc->pipe;
 
-		skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
-				    &new->ddb.plane[pipe][PLANE_CURSOR]);
+	for (level = 0; level <= max_level; level++) {
+		I915_WRITE(PLANE_WM(pipe, plane, level),
+			   wm->plane[pipe][plane][level]);
 	}
-}
+	I915_WRITE(PLANE_WM_TRANS(pipe, plane), wm->plane_trans[pipe][plane]);
 
-/*
- * When setting up a new DDB allocation arrangement, we need to correctly
- * sequence the times at which the new allocations for the pipes are taken into
- * account or we'll have pipes fetching from space previously allocated to
- * another pipe.
- *
- * Roughly the sequence looks like:
- *  1. re-allocate the pipe(s) with the allocation being reduced and not
- *     overlapping with a previous light-up pipe (another way to put it is:
- *     pipes with their new allocation strickly included into their old ones).
- *  2. re-allocate the other pipes that get their allocation reduced
- *  3. allocate the pipes having their allocation increased
- *
- * Steps 1. and 2. are here to take care of the following case:
- * - Initially DDB looks like this:
- *     |   B    |   C    |
- * - enable pipe A.
- * - pipe B has a reduced DDB allocation that overlaps with the old pipe C
- *   allocation
- *     |  A  |  B  |  C  |
- *
- * We need to sequence the re-allocation: C, B, A (and not B, C, A).
- */
+	skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane),
+			    &wm->ddb.plane[pipe][plane]);
+	skl_ddb_entry_write(dev_priv, PLANE_NV12_BUF_CFG(pipe, plane),
+			    &wm->ddb.y_plane[pipe][plane]);
+}
 
-static void
-skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
+void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
+			 const struct skl_wm_values *wm)
 {
-	int plane;
-
-	DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
+	struct drm_crtc *crtc = &intel_crtc->base;
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	int level, max_level = ilk_wm_max_level(dev);
+	enum pipe pipe = intel_crtc->pipe;
 
-	for_each_plane(dev_priv, pipe, plane) {
-		I915_WRITE(PLANE_SURF(pipe, plane),
-			   I915_READ(PLANE_SURF(pipe, plane)));
+	for (level = 0; level <= max_level; level++) {
+		I915_WRITE(CUR_WM(pipe, level),
+			   wm->plane[pipe][PLANE_CURSOR][level]);
 	}
-	I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
+	I915_WRITE(CUR_WM_TRANS(pipe), wm->plane_trans[pipe][PLANE_CURSOR]);
+
+	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
+			    &wm->ddb.plane[pipe][PLANE_CURSOR]);
 }
 
-static bool
-skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
-			    const struct skl_ddb_allocation *new,
-			    enum pipe pipe)
+bool skl_ddb_allocation_equals(const struct skl_ddb_allocation *old,
+			       const struct skl_ddb_allocation *new,
+			       enum pipe pipe)
 {
-	uint16_t old_size, new_size;
-
-	old_size = skl_ddb_entry_size(&old->pipe[pipe]);
-	new_size = skl_ddb_entry_size(&new->pipe[pipe]);
-
-	return old_size != new_size &&
-	       new->pipe[pipe].start >= old->pipe[pipe].start &&
-	       new->pipe[pipe].end <= old->pipe[pipe].end;
+	return new->pipe[pipe].start == old->pipe[pipe].start &&
+	       new->pipe[pipe].end == old->pipe[pipe].end;
 }
 
-static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
-				struct skl_wm_values *new_values)
+static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
+					   const struct skl_ddb_entry *b)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct skl_ddb_allocation *cur_ddb, *new_ddb;
-	bool reallocated[I915_MAX_PIPES] = {};
-	struct intel_crtc *crtc;
-	enum pipe pipe;
-
-	new_ddb = &new_values->ddb;
-	cur_ddb = &dev_priv->wm.skl_hw.ddb;
-
-	/*
-	 * First pass: flush the pipes with the new allocation contained into
-	 * the old space.
-	 *
-	 * We'll wait for the vblank on those pipes to ensure we can safely
-	 * re-allocate the freed space without this pipe fetching from it.
-	 */
-	for_each_intel_crtc(dev, crtc) {
-		if (!crtc->active)
-			continue;
-
-		pipe = crtc->pipe;
-
-		if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
-			continue;
-
-		skl_wm_flush_pipe(dev_priv, pipe, 1);
-		intel_wait_for_vblank(dev, pipe);
-
-		reallocated[pipe] = true;
-	}
-
+	return a->start < b->end && b->start < a->end;
+}
 
-	/*
-	 * Second pass: flush the pipes that are having their allocation
-	 * reduced, but overlapping with a previous allocation.
-	 *
-	 * Here as well we need to wait for the vblank to make sure the freed
-	 * space is not used anymore.
-	 */
-	for_each_intel_crtc(dev, crtc) {
-		if (!crtc->active)
-			continue;
+bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state,
+				 const struct skl_ddb_allocation *old,
+				 const struct skl_ddb_allocation *new,
+				 enum pipe pipe)
+{
+	struct drm_device *dev = state->dev;
+	struct intel_crtc *intel_crtc;
+	enum pipe otherp;
 
-		pipe = crtc->pipe;
+	for_each_intel_crtc(dev, intel_crtc) {
+		otherp = intel_crtc->pipe;
 
-		if (reallocated[pipe])
+		if (otherp == pipe)
 			continue;
 
-		if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
-		    skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
-			skl_wm_flush_pipe(dev_priv, pipe, 2);
-			intel_wait_for_vblank(dev, pipe);
-			reallocated[pipe] = true;
-		}
+		if (skl_ddb_entries_overlap(&new->pipe[pipe],
+					    &old->pipe[otherp]))
+			return true;
 	}
 
-	/*
-	 * Third pass: flush the pipes that got more space allocated.
-	 *
-	 * We don't need to actively wait for the update here, next vblank
-	 * will just get more DDB space with the correct WM values.
-	 */
-	for_each_intel_crtc(dev, crtc) {
-		if (!crtc->active)
-			continue;
-
-		pipe = crtc->pipe;
-
-		/*
-		 * At this point, only the pipes more space than before are
-		 * left to re-allocate.
-		 */
-		if (reallocated[pipe])
-			continue;
-
-		skl_wm_flush_pipe(dev_priv, pipe, 3);
-	}
+	return false;
 }
 
 static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
@@ -3893,6 +3962,41 @@ pipes_modified(struct drm_atomic_state *state)
 	return ret;
 }
 
+int
+skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
+{
+	struct drm_atomic_state *state = cstate->base.state;
+	struct drm_device *dev = state->dev;
+	struct drm_crtc *crtc = cstate->base.crtc;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+	struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
+	struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
+	struct drm_plane_state *plane_state;
+	struct drm_plane *plane;
+	enum pipe pipe = intel_crtc->pipe;
+	int id;
+
+	WARN_ON(!drm_atomic_get_existing_crtc_state(state, crtc));
+
+	drm_for_each_plane_mask(plane, dev, crtc->state->plane_mask) {
+		id = skl_wm_plane_id(to_intel_plane(plane));
+
+		if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][id],
+					&new_ddb->plane[pipe][id]) &&
+		    skl_ddb_entry_equal(&cur_ddb->y_plane[pipe][id],
+					&new_ddb->y_plane[pipe][id]))
+			continue;
+
+		plane_state = drm_atomic_get_plane_state(state, plane);
+		if (IS_ERR(plane_state))
+			return PTR_ERR(plane_state);
+	}
+
+	return 0;
+}
+
 static int
 skl_compute_ddb(struct drm_atomic_state *state)
 {
@@ -3910,9 +4014,24 @@ skl_compute_ddb(struct drm_atomic_state *state)
 	 * pretend that all pipes switched active status so that we'll
 	 * ensure a full DDB recompute.
 	 */
-	if (dev_priv->wm.distrust_bios_wm)
+	if (dev_priv->wm.distrust_bios_wm) {
+		ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
+				       state->acquire_ctx);
+		if (ret)
+			return ret;
+
 		intel_state->active_pipe_changes = ~0;
 
+		/*
+		 * We usually only initialize intel_state->active_crtcs if we
+		 * we're doing a modeset; make sure this field is always
+		 * initialized during the sanitization process that happens
+		 * on the first commit too.
+		 */
+		if (!intel_state->modeset)
+			intel_state->active_crtcs = dev_priv->active_crtcs;
+	}
+
 	/*
 	 * If the modeset changes which CRTC's are active, we need to
 	 * recompute the DDB allocation for *all* active pipes, even
@@ -3941,11 +4060,33 @@ skl_compute_ddb(struct drm_atomic_state *state)
 		ret = skl_allocate_pipe_ddb(cstate, ddb);
 		if (ret)
 			return ret;
+
+		ret = skl_ddb_add_affected_planes(cstate);
+		if (ret)
+			return ret;
 	}
 
 	return 0;
 }
 
+static void
+skl_copy_wm_for_pipe(struct skl_wm_values *dst,
+		     struct skl_wm_values *src,
+		     enum pipe pipe)
+{
+	dst->wm_linetime[pipe] = src->wm_linetime[pipe];
+	memcpy(dst->plane[pipe], src->plane[pipe],
+	       sizeof(dst->plane[pipe]));
+	memcpy(dst->plane_trans[pipe], src->plane_trans[pipe],
+	       sizeof(dst->plane_trans[pipe]));
+
+	dst->ddb.pipe[pipe] = src->ddb.pipe[pipe];
+	memcpy(dst->ddb.y_plane[pipe], src->ddb.y_plane[pipe],
+	       sizeof(dst->ddb.y_plane[pipe]));
+	memcpy(dst->ddb.plane[pipe], src->ddb.plane[pipe],
+	       sizeof(dst->ddb.plane[pipe]));
+}
+
 static int
 skl_compute_wm(struct drm_atomic_state *state)
 {
@@ -4018,8 +4159,10 @@ static void skl_update_wm(struct drm_crtc *crtc)
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct skl_wm_values *results = &dev_priv->wm.skl_results;
+	struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw;
 	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
 	struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
+	enum pipe pipe = intel_crtc->pipe;
 
 	if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
 		return;
@@ -4028,11 +4171,22 @@ static void skl_update_wm(struct drm_crtc *crtc)
 
 	mutex_lock(&dev_priv->wm.wm_mutex);
 
-	skl_write_wm_values(dev_priv, results);
-	skl_flush_wm_values(dev_priv, results);
+	/*
+	 * If this pipe isn't active already, we're going to be enabling it
+	 * very soon. Since it's safe to update a pipe's ddb allocation while
+	 * the pipe's shut off, just do so here. Already active pipes will have
+	 * their watermarks updated once we update their planes.
+	 */
+	if (crtc->state->active_changed) {
+		int plane;
 
-	/* store the new configuration */
-	dev_priv->wm.skl_hw = *results;
+		for (plane = 0; plane < intel_num_planes(intel_crtc); plane++)
+			skl_write_plane_wm(intel_crtc, results, plane);
+
+		skl_write_cursor_wm(intel_crtc, results);
+	}
+
+	skl_copy_wm_for_pipe(hw_vals, results, pipe);
 
 	mutex_unlock(&dev_priv->wm.wm_mutex);
 }
@@ -4892,7 +5046,8 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv)
 		else
 			gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
 		dev_priv->rps.last_adj = 0;
-		I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
+		I915_WRITE(GEN6_PMINTRMSK,
+			   gen6_sanitize_rps_pm_mask(dev_priv, ~0));
 	}
 	mutex_unlock(&dev_priv->rps.hw_lock);
 
@@ -4911,7 +5066,7 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv,
 	 */
 	if (!(dev_priv->gt.awake &&
 	      dev_priv->rps.enabled &&
-	      dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
+	      dev_priv->rps.cur_freq < dev_priv->rps.boost_freq))
 		return;
 
 	/* Force a RPS boost (and don't count it against the client) if
@@ -5102,35 +5257,31 @@ int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6)
 
 static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
 {
-	uint32_t rp_state_cap;
-	u32 ddcc_status = 0;
-	int ret;
-
 	/* All of these values are in units of 50MHz */
-	dev_priv->rps.cur_freq		= 0;
+
 	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
 	if (IS_BROXTON(dev_priv)) {
-		rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
+		u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
 		dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
 		dev_priv->rps.rp1_freq = (rp_state_cap >>  8) & 0xff;
 		dev_priv->rps.min_freq = (rp_state_cap >>  0) & 0xff;
 	} else {
-		rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+		u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
 		dev_priv->rps.rp0_freq = (rp_state_cap >>  0) & 0xff;
 		dev_priv->rps.rp1_freq = (rp_state_cap >>  8) & 0xff;
 		dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
 	}
-
 	/* hw_max = RP0 until we check for overclocking */
-	dev_priv->rps.max_freq		= dev_priv->rps.rp0_freq;
+	dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
 
 	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
 	    IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
-		ret = sandybridge_pcode_read(dev_priv,
-					HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
-					&ddcc_status);
-		if (0 == ret)
+		u32 ddcc_status = 0;
+
+		if (sandybridge_pcode_read(dev_priv,
+					   HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
+					   &ddcc_status) == 0)
 			dev_priv->rps.efficient_freq =
 				clamp_t(u8,
 					((ddcc_status >> 8) & 0xff),
@@ -5140,29 +5291,26 @@ static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
 
 	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		/* Store the frequency values in 16.66 MHZ units, which is
-		   the natural hardware unit for SKL */
+		 * the natural hardware unit for SKL
+		 */
 		dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
 		dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
 		dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
 		dev_priv->rps.max_freq *= GEN9_FREQ_SCALER;
 		dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER;
 	}
+}
 
-	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
+static void reset_rps(struct drm_i915_private *dev_priv,
+		      void (*set)(struct drm_i915_private *, u8))
+{
+	u8 freq = dev_priv->rps.cur_freq;
 
-	/* Preserve min/max settings in case of re-init */
-	if (dev_priv->rps.max_freq_softlimit == 0)
-		dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
+	/* force a reset */
+	dev_priv->rps.power = -1;
+	dev_priv->rps.cur_freq = -1;
 
-	if (dev_priv->rps.min_freq_softlimit == 0) {
-		if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
-			dev_priv->rps.min_freq_softlimit =
-				max_t(int, dev_priv->rps.efficient_freq,
-				      intel_freq_opcode(dev_priv, 450));
-		else
-			dev_priv->rps.min_freq_softlimit =
-				dev_priv->rps.min_freq;
-	}
+	set(dev_priv, freq);
 }
 
 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
@@ -5170,8 +5318,6 @@ static void gen9_enable_rps(struct drm_i915_private *dev_priv)
 {
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-	gen6_init_rps_frequencies(dev_priv);
-
 	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
 	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
 		/*
@@ -5201,8 +5347,7 @@ static void gen9_enable_rps(struct drm_i915_private *dev_priv)
 	/* Leaning on the below call to gen6_set_rps to program/setup the
 	 * Up/Down EI & threshold registers, as well as the RP_CONTROL,
 	 * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
-	dev_priv->rps.power = HIGH_POWER; /* force a reset */
-	gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
+	reset_rps(dev_priv, gen6_set_rps);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
@@ -5289,9 +5434,6 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 	/* 2a: Disable RC states. */
 	I915_WRITE(GEN6_RC_CONTROL, 0);
 
-	/* Initialize rps frequencies */
-	gen6_init_rps_frequencies(dev_priv);
-
 	/* 2b: Program RC6 thresholds.*/
 	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
@@ -5348,8 +5490,7 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 
 	/* 6: Ring frequency + overclocking (our driver does this later */
 
-	dev_priv->rps.power = HIGH_POWER; /* force a reset */
-	gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
+	reset_rps(dev_priv, gen6_set_rps);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
@@ -5357,7 +5498,7 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
-	u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
+	u32 rc6vids, rc6_mask = 0;
 	u32 gtfifodbg;
 	int rc6_mode;
 	int ret;
@@ -5381,9 +5522,6 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-	/* Initialize rps frequencies */
-	gen6_init_rps_frequencies(dev_priv);
-
 	/* disable the counters and set deterministic thresholds */
 	I915_WRITE(GEN6_RC_CONTROL, 0);
 
@@ -5434,16 +5572,7 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 	if (ret)
 		DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
 
-	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
-	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
-		DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
-				 (dev_priv->rps.max_freq_softlimit & 0xff) * 50,
-				 (pcu_mbox & 0xff) * 50);
-		dev_priv->rps.max_freq = pcu_mbox & 0xff;
-	}
-
-	dev_priv->rps.power = HIGH_POWER; /* force a reset */
-	gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
+	reset_rps(dev_priv, gen6_set_rps);
 
 	rc6vids = 0;
 	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
@@ -5462,7 +5591,7 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
 
-static void __gen6_update_ring_freq(struct drm_i915_private *dev_priv)
+static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
 {
 	int min_freq = 15;
 	unsigned int gpu_freq;
@@ -5546,23 +5675,13 @@ static void __gen6_update_ring_freq(struct drm_i915_private *dev_priv)
 	}
 }
 
-void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
-{
-	if (!HAS_CORE_RING_FREQ(dev_priv))
-		return;
-
-	mutex_lock(&dev_priv->rps.hw_lock);
-	__gen6_update_ring_freq(dev_priv);
-	mutex_unlock(&dev_priv->rps.hw_lock);
-}
-
 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
 {
 	u32 val, rp0;
 
 	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
 
-	switch (INTEL_INFO(dev_priv)->eu_total) {
+	switch (INTEL_INFO(dev_priv)->sseu.eu_total) {
 	case 8:
 		/* (2 * 4) config */
 		rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
@@ -5700,8 +5819,6 @@ static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
 	u32 pcbr;
 	int pctx_size = 24*1024;
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
-
 	pcbr = I915_READ(VLV_PCBR);
 	if (pcbr) {
 		/* BIOS set it up already, grab the pre-alloc'd space */
@@ -5737,7 +5854,6 @@ static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
 out:
 	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
 	dev_priv->vlv_pctx = pctx;
-	mutex_unlock(&dev_priv->drm.struct_mutex);
 }
 
 static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
@@ -5745,7 +5861,7 @@ static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
 	if (WARN_ON(!dev_priv->vlv_pctx))
 		return;
 
-	drm_gem_object_unreference_unlocked(&dev_priv->vlv_pctx->base);
+	i915_gem_object_put_unlocked(dev_priv->vlv_pctx);
 	dev_priv->vlv_pctx = NULL;
 }
 
@@ -5768,8 +5884,6 @@ static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
 
 	vlv_init_gpll_ref_freq(dev_priv);
 
-	mutex_lock(&dev_priv->rps.hw_lock);
-
 	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
 	switch ((val >> 6) & 3) {
 	case 0:
@@ -5805,17 +5919,6 @@ static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
 	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
 			 intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
 			 dev_priv->rps.min_freq);
-
-	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
-
-	/* Preserve min/max settings in case of re-init */
-	if (dev_priv->rps.max_freq_softlimit == 0)
-		dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
-
-	if (dev_priv->rps.min_freq_softlimit == 0)
-		dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
-
-	mutex_unlock(&dev_priv->rps.hw_lock);
 }
 
 static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
@@ -5826,8 +5929,6 @@ static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
 
 	vlv_init_gpll_ref_freq(dev_priv);
 
-	mutex_lock(&dev_priv->rps.hw_lock);
-
 	mutex_lock(&dev_priv->sb_lock);
 	val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
 	mutex_unlock(&dev_priv->sb_lock);
@@ -5869,17 +5970,6 @@ static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
 		   dev_priv->rps.rp1_freq |
 		   dev_priv->rps.min_freq) & 1,
 		  "Odd GPU freq values\n");
-
-	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
-
-	/* Preserve min/max settings in case of re-init */
-	if (dev_priv->rps.max_freq_softlimit == 0)
-		dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
-
-	if (dev_priv->rps.min_freq_softlimit == 0)
-		dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
-
-	mutex_unlock(&dev_priv->rps.hw_lock);
 }
 
 static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
@@ -5970,16 +6060,7 @@ static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
 	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
 	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
 
-	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
-	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
-			 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
-			 dev_priv->rps.cur_freq);
-
-	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
-			 intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq),
-			 dev_priv->rps.idle_freq);
-
-	valleyview_set_rps(dev_priv, dev_priv->rps.idle_freq);
+	reset_rps(dev_priv, valleyview_set_rps);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
@@ -6059,16 +6140,7 @@ static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
 	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
 	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
 
-	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
-	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
-			 intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
-			 dev_priv->rps.cur_freq);
-
-	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
-			 intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq),
-			 dev_priv->rps.idle_freq);
-
-	valleyview_set_rps(dev_priv, dev_priv->rps.idle_freq);
+	reset_rps(dev_priv, valleyview_set_rps);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
@@ -6397,19 +6469,11 @@ EXPORT_SYMBOL_GPL(i915_gpu_lower);
  */
 bool i915_gpu_busy(void)
 {
-	struct drm_i915_private *dev_priv;
-	struct intel_engine_cs *engine;
 	bool ret = false;
 
 	spin_lock_irq(&mchdev_lock);
-	if (!i915_mch_dev)
-		goto out_unlock;
-	dev_priv = i915_mch_dev;
-
-	for_each_engine(engine, dev_priv)
-		ret |= !list_empty(&engine->request_list);
-
-out_unlock:
+	if (i915_mch_dev)
+		ret = i915_mch_dev->gt.awake;
 	spin_unlock_irq(&mchdev_lock);
 
 	return ret;
@@ -6565,30 +6629,62 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
 		intel_runtime_pm_get(dev_priv);
 	}
 
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	mutex_lock(&dev_priv->rps.hw_lock);
+
+	/* Initialize RPS limits (for userspace) */
 	if (IS_CHERRYVIEW(dev_priv))
 		cherryview_init_gt_powersave(dev_priv);
 	else if (IS_VALLEYVIEW(dev_priv))
 		valleyview_init_gt_powersave(dev_priv);
+	else if (INTEL_GEN(dev_priv) >= 6)
+		gen6_init_rps_frequencies(dev_priv);
+
+	/* Derive initial user preferences/limits from the hardware limits */
+	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
+	dev_priv->rps.cur_freq = dev_priv->rps.idle_freq;
+
+	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
+	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
+
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
+		dev_priv->rps.min_freq_softlimit =
+			max_t(int,
+			      dev_priv->rps.efficient_freq,
+			      intel_freq_opcode(dev_priv, 450));
+
+	/* After setting max-softlimit, find the overclock max freq */
+	if (IS_GEN6(dev_priv) ||
+	    IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
+		u32 params = 0;
+
+		sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &params);
+		if (params & BIT(31)) { /* OC supported */
+			DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
+					 (dev_priv->rps.max_freq & 0xff) * 50,
+					 (params & 0xff) * 50);
+			dev_priv->rps.max_freq = params & 0xff;
+		}
+	}
+
+	/* Finally allow us to boost to max by default */
+	dev_priv->rps.boost_freq = dev_priv->rps.max_freq;
+
+	mutex_unlock(&dev_priv->rps.hw_lock);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+
+	intel_autoenable_gt_powersave(dev_priv);
 }
 
 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
 {
-	if (IS_CHERRYVIEW(dev_priv))
-		return;
-	else if (IS_VALLEYVIEW(dev_priv))
+	if (IS_VALLEYVIEW(dev_priv))
 		valleyview_cleanup_gt_powersave(dev_priv);
 
 	if (!i915.enable_rc6)
 		intel_runtime_pm_put(dev_priv);
 }
 
-static void gen6_suspend_rps(struct drm_i915_private *dev_priv)
-{
-	flush_delayed_work(&dev_priv->rps.delayed_resume_work);
-
-	gen6_disable_rps_interrupts(dev_priv);
-}
-
 /**
  * intel_suspend_gt_powersave - suspend PM work and helper threads
  * @dev_priv: i915 device
@@ -6602,60 +6698,76 @@ void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
 	if (INTEL_GEN(dev_priv) < 6)
 		return;
 
-	gen6_suspend_rps(dev_priv);
+	if (cancel_delayed_work_sync(&dev_priv->rps.autoenable_work))
+		intel_runtime_pm_put(dev_priv);
+
+	/* gen6_rps_idle() will be called later to disable interrupts */
+}
+
+void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
+{
+	dev_priv->rps.enabled = true; /* force disabling */
+	intel_disable_gt_powersave(dev_priv);
 
-	/* Force GPU to min freq during suspend */
-	gen6_rps_idle(dev_priv);
+	gen6_reset_rps_interrupts(dev_priv);
 }
 
 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
 {
-	if (IS_IRONLAKE_M(dev_priv)) {
-		ironlake_disable_drps(dev_priv);
-	} else if (INTEL_INFO(dev_priv)->gen >= 6) {
-		intel_suspend_gt_powersave(dev_priv);
+	if (!READ_ONCE(dev_priv->rps.enabled))
+		return;
 
-		mutex_lock(&dev_priv->rps.hw_lock);
-		if (INTEL_INFO(dev_priv)->gen >= 9) {
-			gen9_disable_rc6(dev_priv);
-			gen9_disable_rps(dev_priv);
-		} else if (IS_CHERRYVIEW(dev_priv))
-			cherryview_disable_rps(dev_priv);
-		else if (IS_VALLEYVIEW(dev_priv))
-			valleyview_disable_rps(dev_priv);
-		else
-			gen6_disable_rps(dev_priv);
+	mutex_lock(&dev_priv->rps.hw_lock);
 
-		dev_priv->rps.enabled = false;
-		mutex_unlock(&dev_priv->rps.hw_lock);
+	if (INTEL_GEN(dev_priv) >= 9) {
+		gen9_disable_rc6(dev_priv);
+		gen9_disable_rps(dev_priv);
+	} else if (IS_CHERRYVIEW(dev_priv)) {
+		cherryview_disable_rps(dev_priv);
+	} else if (IS_VALLEYVIEW(dev_priv)) {
+		valleyview_disable_rps(dev_priv);
+	} else if (INTEL_GEN(dev_priv) >= 6) {
+		gen6_disable_rps(dev_priv);
+	}  else if (IS_IRONLAKE_M(dev_priv)) {
+		ironlake_disable_drps(dev_priv);
 	}
+
+	dev_priv->rps.enabled = false;
+	mutex_unlock(&dev_priv->rps.hw_lock);
 }
 
-static void intel_gen6_powersave_work(struct work_struct *work)
+void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv =
-		container_of(work, struct drm_i915_private,
-			     rps.delayed_resume_work.work);
+	/* We shouldn't be disabling as we submit, so this should be less
+	 * racy than it appears!
+	 */
+	if (READ_ONCE(dev_priv->rps.enabled))
+		return;
 
-	mutex_lock(&dev_priv->rps.hw_lock);
+	/* Powersaving is controlled by the host when inside a VM */
+	if (intel_vgpu_active(dev_priv))
+		return;
 
-	gen6_reset_rps_interrupts(dev_priv);
+	mutex_lock(&dev_priv->rps.hw_lock);
 
 	if (IS_CHERRYVIEW(dev_priv)) {
 		cherryview_enable_rps(dev_priv);
 	} else if (IS_VALLEYVIEW(dev_priv)) {
 		valleyview_enable_rps(dev_priv);
-	} else if (INTEL_INFO(dev_priv)->gen >= 9) {
+	} else if (INTEL_GEN(dev_priv) >= 9) {
 		gen9_enable_rc6(dev_priv);
 		gen9_enable_rps(dev_priv);
 		if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
-			__gen6_update_ring_freq(dev_priv);
+			gen6_update_ring_freq(dev_priv);
 	} else if (IS_BROADWELL(dev_priv)) {
 		gen8_enable_rps(dev_priv);
-		__gen6_update_ring_freq(dev_priv);
-	} else {
+		gen6_update_ring_freq(dev_priv);
+	} else if (INTEL_GEN(dev_priv) >= 6) {
 		gen6_enable_rps(dev_priv);
-		__gen6_update_ring_freq(dev_priv);
+		gen6_update_ring_freq(dev_priv);
+	} else if (IS_IRONLAKE_M(dev_priv)) {
+		ironlake_enable_drps(dev_priv);
+		intel_init_emon(dev_priv);
 	}
 
 	WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
@@ -6665,25 +6777,52 @@ static void intel_gen6_powersave_work(struct work_struct *work)
 	WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq);
 
 	dev_priv->rps.enabled = true;
+	mutex_unlock(&dev_priv->rps.hw_lock);
+}
 
-	gen6_enable_rps_interrupts(dev_priv);
+static void __intel_autoenable_gt_powersave(struct work_struct *work)
+{
+	struct drm_i915_private *dev_priv =
+		container_of(work, typeof(*dev_priv), rps.autoenable_work.work);
+	struct intel_engine_cs *rcs;
+	struct drm_i915_gem_request *req;
 
-	mutex_unlock(&dev_priv->rps.hw_lock);
+	if (READ_ONCE(dev_priv->rps.enabled))
+		goto out;
+
+	rcs = &dev_priv->engine[RCS];
+	if (rcs->last_context)
+		goto out;
+
+	if (!rcs->init_context)
+		goto out;
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+
+	req = i915_gem_request_alloc(rcs, dev_priv->kernel_context);
+	if (IS_ERR(req))
+		goto unlock;
 
+	if (!i915.enable_execlists && i915_switch_context(req) == 0)
+		rcs->init_context(req);
+
+	/* Mark the device busy, calling intel_enable_gt_powersave() */
+	i915_add_request_no_flush(req);
+
+unlock:
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+out:
 	intel_runtime_pm_put(dev_priv);
 }
 
-void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
+void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv)
 {
-	/* Powersaving is controlled by the host when inside a VM */
-	if (intel_vgpu_active(dev_priv))
+	if (READ_ONCE(dev_priv->rps.enabled))
 		return;
 
 	if (IS_IRONLAKE_M(dev_priv)) {
 		ironlake_enable_drps(dev_priv);
-		mutex_lock(&dev_priv->drm.struct_mutex);
 		intel_init_emon(dev_priv);
-		mutex_unlock(&dev_priv->drm.struct_mutex);
 	} else if (INTEL_INFO(dev_priv)->gen >= 6) {
 		/*
 		 * PCU communication is slow and this doesn't need to be
@@ -6697,21 +6836,13 @@ void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
 		 * paths, so the _noresume version is enough (and in case of
 		 * runtime resume it's necessary).
 		 */
-		if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
-					   round_jiffies_up_relative(HZ)))
+		if (queue_delayed_work(dev_priv->wq,
+				       &dev_priv->rps.autoenable_work,
+				       round_jiffies_up_relative(HZ)))
 			intel_runtime_pm_get_noresume(dev_priv);
 	}
 }
 
-void intel_reset_gt_powersave(struct drm_i915_private *dev_priv)
-{
-	if (INTEL_INFO(dev_priv)->gen < 6)
-		return;
-
-	gen6_suspend_rps(dev_priv);
-	dev_priv->rps.enabled = false;
-}
-
 static void ibx_init_clock_gating(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -7657,8 +7788,54 @@ void intel_init_pm(struct drm_device *dev)
 	}
 }
 
+static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv)
+{
+	uint32_t flags =
+		I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
+
+	switch (flags) {
+	case GEN6_PCODE_SUCCESS:
+		return 0;
+	case GEN6_PCODE_UNIMPLEMENTED_CMD:
+	case GEN6_PCODE_ILLEGAL_CMD:
+		return -ENXIO;
+	case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
+	case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
+		return -EOVERFLOW;
+	case GEN6_PCODE_TIMEOUT:
+		return -ETIMEDOUT;
+	default:
+		MISSING_CASE(flags)
+		return 0;
+	}
+}
+
+static inline int gen7_check_mailbox_status(struct drm_i915_private *dev_priv)
+{
+	uint32_t flags =
+		I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
+
+	switch (flags) {
+	case GEN6_PCODE_SUCCESS:
+		return 0;
+	case GEN6_PCODE_ILLEGAL_CMD:
+		return -ENXIO;
+	case GEN7_PCODE_TIMEOUT:
+		return -ETIMEDOUT;
+	case GEN7_PCODE_ILLEGAL_DATA:
+		return -EINVAL;
+	case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
+		return -EOVERFLOW;
+	default:
+		MISSING_CASE(flags);
+		return 0;
+	}
+}
+
 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
 {
+	int status;
+
 	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
 
 	/* GEN6_PCODE_* are outside of the forcewake domain, we can
@@ -7685,12 +7862,25 @@ int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val
 	*val = I915_READ_FW(GEN6_PCODE_DATA);
 	I915_WRITE_FW(GEN6_PCODE_DATA, 0);
 
+	if (INTEL_GEN(dev_priv) > 6)
+		status = gen7_check_mailbox_status(dev_priv);
+	else
+		status = gen6_check_mailbox_status(dev_priv);
+
+	if (status) {
+		DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed: %d\n",
+				 status);
+		return status;
+	}
+
 	return 0;
 }
 
 int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
-			       u32 mbox, u32 val)
+			    u32 mbox, u32 val)
 {
+	int status;
+
 	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
 
 	/* GEN6_PCODE_* are outside of the forcewake domain, we can
@@ -7715,6 +7905,17 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
 
 	I915_WRITE_FW(GEN6_PCODE_DATA, 0);
 
+	if (INTEL_GEN(dev_priv) > 6)
+		status = gen7_check_mailbox_status(dev_priv);
+	else
+		status = gen6_check_mailbox_status(dev_priv);
+
+	if (status) {
+		DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed: %d\n",
+				 status);
+		return status;
+	}
+
 	return 0;
 }
 
@@ -7786,7 +7987,7 @@ static void __intel_rps_boost_work(struct work_struct *work)
 	if (!i915_gem_request_completed(req))
 		gen6_rps_boost(req->i915, NULL, req->emitted_jiffies);
 
-	i915_gem_request_unreference(req);
+	i915_gem_request_put(req);
 	kfree(boost);
 }
 
@@ -7804,8 +8005,7 @@ void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req)
 	if (boost == NULL)
 		return;
 
-	i915_gem_request_reference(req);
-	boost->req = req;
+	boost->req = i915_gem_request_get(req);
 
 	INIT_WORK(&boost->work, __intel_rps_boost_work);
 	queue_work(req->i915->wq, &boost->work);
@@ -7818,11 +8018,9 @@ void intel_pm_setup(struct drm_device *dev)
 	mutex_init(&dev_priv->rps.hw_lock);
 	spin_lock_init(&dev_priv->rps.client_lock);
 
-	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
-			  intel_gen6_powersave_work);
+	INIT_DELAYED_WORK(&dev_priv->rps.autoenable_work,
+			  __intel_autoenable_gt_powersave);
 	INIT_LIST_HEAD(&dev_priv->rps.clients);
-	INIT_LIST_HEAD(&dev_priv->rps.semaphores.link);
-	INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link);
 
 	dev_priv->pm.suspended = false;
 	atomic_set(&dev_priv->pm.wakeref_count, 0);
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 2b0d1baf15b3..108ba1e5d658 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -255,14 +255,14 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
 	uint32_t max_sleep_time = 0x1f;
-	/* Lately it was identified that depending on panel idle frame count
-	 * calculated at HW can be off by 1. So let's use what came
-	 * from VBT + 1.
-	 * There are also other cases where panel demands at least 4
-	 * but VBT is not being set. To cover these 2 cases lets use
-	 * at least 5 when VBT isn't set to be on the safest side.
+	/*
+	 * Let's respect VBT in case VBT asks a higher idle_frame value.
+	 * Let's use 6 as the minimum to cover all known cases including
+	 * the off-by-one issue that HW has in some cases. Also there are
+	 * cases where sink should be able to train
+	 * with the 5 or 6 idle patterns.
 	 */
-	uint32_t idle_frames = dev_priv->vbt.psr.idle_frames + 1;
+	uint32_t idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
 	uint32_t val = EDP_PSR_ENABLE;
 
 	val |= max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT;
@@ -645,9 +645,8 @@ unlock:
 	mutex_unlock(&dev_priv->psr.lock);
 }
 
-static void intel_psr_exit(struct drm_device *dev)
+static void intel_psr_exit(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_dp *intel_dp = dev_priv->psr.enabled;
 	struct drm_crtc *crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
@@ -656,7 +655,7 @@ static void intel_psr_exit(struct drm_device *dev)
 	if (!dev_priv->psr.active)
 		return;
 
-	if (HAS_DDI(dev)) {
+	if (HAS_DDI(dev_priv)) {
 		val = I915_READ(EDP_PSR_CTL);
 
 		WARN_ON(!(val & EDP_PSR_ENABLE));
@@ -691,7 +690,7 @@ static void intel_psr_exit(struct drm_device *dev)
 
 /**
  * intel_psr_single_frame_update - Single Frame Update
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
  *
  * Some platforms support a single frame update feature that is used to
@@ -699,10 +698,9 @@ static void intel_psr_exit(struct drm_device *dev)
  * So far it is only implemented for Valleyview and Cherryview because
  * hardware requires this to be done before a page flip.
  */
-void intel_psr_single_frame_update(struct drm_device *dev,
+void intel_psr_single_frame_update(struct drm_i915_private *dev_priv,
 				   unsigned frontbuffer_bits)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_crtc *crtc;
 	enum pipe pipe;
 	u32 val;
@@ -711,7 +709,7 @@ void intel_psr_single_frame_update(struct drm_device *dev,
 	 * Single frame update is already supported on BDW+ but it requires
 	 * many W/A and it isn't really needed.
 	 */
-	if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))
+	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
 		return;
 
 	mutex_lock(&dev_priv->psr.lock);
@@ -737,7 +735,7 @@ void intel_psr_single_frame_update(struct drm_device *dev,
 
 /**
  * intel_psr_invalidate - Invalidade PSR
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
  *
  * Since the hardware frontbuffer tracking has gaps we need to integrate
@@ -747,10 +745,9 @@ void intel_psr_single_frame_update(struct drm_device *dev,
  *
  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits."
  */
-void intel_psr_invalidate(struct drm_device *dev,
+void intel_psr_invalidate(struct drm_i915_private *dev_priv,
 			  unsigned frontbuffer_bits)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_crtc *crtc;
 	enum pipe pipe;
 
@@ -767,14 +764,14 @@ void intel_psr_invalidate(struct drm_device *dev,
 	dev_priv->psr.busy_frontbuffer_bits |= frontbuffer_bits;
 
 	if (frontbuffer_bits)
-		intel_psr_exit(dev);
+		intel_psr_exit(dev_priv);
 
 	mutex_unlock(&dev_priv->psr.lock);
 }
 
 /**
  * intel_psr_flush - Flush PSR
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @frontbuffer_bits: frontbuffer plane tracking bits
  * @origin: which operation caused the flush
  *
@@ -785,10 +782,9 @@ void intel_psr_invalidate(struct drm_device *dev,
  *
  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits.
  */
-void intel_psr_flush(struct drm_device *dev,
+void intel_psr_flush(struct drm_i915_private *dev_priv,
 		     unsigned frontbuffer_bits, enum fb_op_origin origin)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_crtc *crtc;
 	enum pipe pipe;
 
@@ -806,7 +802,7 @@ void intel_psr_flush(struct drm_device *dev,
 
 	/* By definition flush = invalidate + flush */
 	if (frontbuffer_bits)
-		intel_psr_exit(dev);
+		intel_psr_exit(dev_priv);
 
 	if (!dev_priv->psr.active && !dev_priv->psr.busy_frontbuffer_bits)
 		if (!work_busy(&dev_priv->psr.work.work))
diff --git a/drivers/gpu/drm/i915/intel_renderstate.h b/drivers/gpu/drm/i915/intel_renderstate.h
index 5bd69852752c..08f6fea05a2c 100644
--- a/drivers/gpu/drm/i915/intel_renderstate.h
+++ b/drivers/gpu/drm/i915/intel_renderstate.h
@@ -24,12 +24,13 @@
 #ifndef _INTEL_RENDERSTATE_H
 #define _INTEL_RENDERSTATE_H
 
-#include "i915_drv.h"
+#include <linux/types.h>
 
-extern const struct intel_renderstate_rodata gen6_null_state;
-extern const struct intel_renderstate_rodata gen7_null_state;
-extern const struct intel_renderstate_rodata gen8_null_state;
-extern const struct intel_renderstate_rodata gen9_null_state;
+struct intel_renderstate_rodata {
+	const u32 *reloc;
+	const u32 *batch;
+	const u32 batch_items;
+};
 
 #define RO_RENDERSTATE(_g)						\
 	const struct intel_renderstate_rodata gen ## _g ## _null_state = { \
@@ -38,4 +39,9 @@ extern const struct intel_renderstate_rodata gen9_null_state;
 		.batch_items = sizeof(gen ## _g ## _null_state_batch)/4, \
 	}
 
+extern const struct intel_renderstate_rodata gen6_null_state;
+extern const struct intel_renderstate_rodata gen7_null_state;
+extern const struct intel_renderstate_rodata gen8_null_state;
+extern const struct intel_renderstate_rodata gen9_null_state;
+
 #endif /* INTEL_RENDERSTATE_H */
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index cca7792f26d5..ed9955dce156 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -47,57 +47,44 @@ int __intel_ring_space(int head, int tail, int size)
 	return space - I915_RING_FREE_SPACE;
 }
 
-void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
+void intel_ring_update_space(struct intel_ring *ring)
 {
-	if (ringbuf->last_retired_head != -1) {
-		ringbuf->head = ringbuf->last_retired_head;
-		ringbuf->last_retired_head = -1;
+	if (ring->last_retired_head != -1) {
+		ring->head = ring->last_retired_head;
+		ring->last_retired_head = -1;
 	}
 
-	ringbuf->space = __intel_ring_space(ringbuf->head & HEAD_ADDR,
-					    ringbuf->tail, ringbuf->size);
-}
-
-static void __intel_ring_advance(struct intel_engine_cs *engine)
-{
-	struct intel_ringbuffer *ringbuf = engine->buffer;
-	ringbuf->tail &= ringbuf->size - 1;
-	engine->write_tail(engine, ringbuf->tail);
+	ring->space = __intel_ring_space(ring->head & HEAD_ADDR,
+					 ring->tail, ring->size);
 }
 
 static int
-gen2_render_ring_flush(struct drm_i915_gem_request *req,
-		       u32	invalidate_domains,
-		       u32	flush_domains)
+gen2_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	u32 cmd;
 	int ret;
 
 	cmd = MI_FLUSH;
-	if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
-		cmd |= MI_NO_WRITE_FLUSH;
 
-	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
+	if (mode & EMIT_INVALIDATE)
 		cmd |= MI_READ_FLUSH;
 
 	ret = intel_ring_begin(req, 2);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, cmd);
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, cmd);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static int
-gen4_render_ring_flush(struct drm_i915_gem_request *req,
-		       u32	invalidate_domains,
-		       u32	flush_domains)
+gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	u32 cmd;
 	int ret;
 
@@ -129,23 +116,20 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req,
 	 * are flushed at any MI_FLUSH.
 	 */
 
-	cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
-	if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
-		cmd &= ~MI_NO_WRITE_FLUSH;
-	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
+	cmd = MI_FLUSH;
+	if (mode & EMIT_INVALIDATE) {
 		cmd |= MI_EXE_FLUSH;
-
-	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
-	    (IS_G4X(req->i915) || IS_GEN5(req->i915)))
-		cmd |= MI_INVALIDATE_ISP;
+		if (IS_G4X(req->i915) || IS_GEN5(req->i915))
+			cmd |= MI_INVALIDATE_ISP;
+	}
 
 	ret = intel_ring_begin(req, 2);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, cmd);
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, cmd);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -190,45 +174,46 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req,
 static int
 intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
 {
-	struct intel_engine_cs *engine = req->engine;
-	u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
+	struct intel_ring *ring = req->ring;
+	u32 scratch_addr =
+		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
 	int ret;
 
 	ret = intel_ring_begin(req, 6);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(5));
-	intel_ring_emit(engine, PIPE_CONTROL_CS_STALL |
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
+	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 			PIPE_CONTROL_STALL_AT_SCOREBOARD);
-	intel_ring_emit(engine, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
-	intel_ring_emit(engine, 0); /* low dword */
-	intel_ring_emit(engine, 0); /* high dword */
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
+	intel_ring_emit(ring, 0); /* low dword */
+	intel_ring_emit(ring, 0); /* high dword */
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	ret = intel_ring_begin(req, 6);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(5));
-	intel_ring_emit(engine, PIPE_CONTROL_QW_WRITE);
-	intel_ring_emit(engine, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
-	intel_ring_emit(engine, 0);
-	intel_ring_emit(engine, 0);
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
+	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
+	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static int
-gen6_render_ring_flush(struct drm_i915_gem_request *req,
-		       u32 invalidate_domains, u32 flush_domains)
+gen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
+	u32 scratch_addr =
+		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
 	u32 flags = 0;
-	u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 	int ret;
 
 	/* Force SNB workarounds for PIPE_CONTROL flushes */
@@ -240,7 +225,7 @@ gen6_render_ring_flush(struct drm_i915_gem_request *req,
 	 * number of bits based on the write domains has little performance
 	 * impact.
 	 */
-	if (flush_domains) {
+	if (mode & EMIT_FLUSH) {
 		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 		/*
@@ -249,7 +234,7 @@ gen6_render_ring_flush(struct drm_i915_gem_request *req,
 		 */
 		flags |= PIPE_CONTROL_CS_STALL;
 	}
-	if (invalidate_domains) {
+	if (mode & EMIT_INVALIDATE) {
 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
 		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
@@ -266,11 +251,11 @@ gen6_render_ring_flush(struct drm_i915_gem_request *req,
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(4));
-	intel_ring_emit(engine, flags);
-	intel_ring_emit(engine, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
-	intel_ring_emit(engine, 0);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
+	intel_ring_emit(ring, flags);
+	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
+	intel_ring_emit(ring, 0);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -278,30 +263,31 @@ gen6_render_ring_flush(struct drm_i915_gem_request *req,
 static int
 gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	int ret;
 
 	ret = intel_ring_begin(req, 4);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(4));
-	intel_ring_emit(engine, PIPE_CONTROL_CS_STALL |
-			      PIPE_CONTROL_STALL_AT_SCOREBOARD);
-	intel_ring_emit(engine, 0);
-	intel_ring_emit(engine, 0);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
+	intel_ring_emit(ring,
+			PIPE_CONTROL_CS_STALL |
+			PIPE_CONTROL_STALL_AT_SCOREBOARD);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, 0);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static int
-gen7_render_ring_flush(struct drm_i915_gem_request *req,
-		       u32 invalidate_domains, u32 flush_domains)
+gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
+	u32 scratch_addr =
+		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
 	u32 flags = 0;
-	u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 	int ret;
 
 	/*
@@ -318,13 +304,13 @@ gen7_render_ring_flush(struct drm_i915_gem_request *req,
 	 * number of bits based on the write domains has little performance
 	 * impact.
 	 */
-	if (flush_domains) {
+	if (mode & EMIT_FLUSH) {
 		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
 	}
-	if (invalidate_domains) {
+	if (mode & EMIT_INVALIDATE) {
 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
 		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
@@ -350,11 +336,11 @@ gen7_render_ring_flush(struct drm_i915_gem_request *req,
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(4));
-	intel_ring_emit(engine, flags);
-	intel_ring_emit(engine, scratch_addr);
-	intel_ring_emit(engine, 0);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
+	intel_ring_emit(ring, flags);
+	intel_ring_emit(ring, scratch_addr);
+	intel_ring_emit(ring, 0);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -363,41 +349,41 @@ static int
 gen8_emit_pipe_control(struct drm_i915_gem_request *req,
 		       u32 flags, u32 scratch_addr)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	int ret;
 
 	ret = intel_ring_begin(req, 6);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(6));
-	intel_ring_emit(engine, flags);
-	intel_ring_emit(engine, scratch_addr);
-	intel_ring_emit(engine, 0);
-	intel_ring_emit(engine, 0);
-	intel_ring_emit(engine, 0);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
+	intel_ring_emit(ring, flags);
+	intel_ring_emit(ring, scratch_addr);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, 0);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static int
-gen8_render_ring_flush(struct drm_i915_gem_request *req,
-		       u32 invalidate_domains, u32 flush_domains)
+gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 {
+	u32 scratch_addr =
+		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
 	u32 flags = 0;
-	u32 scratch_addr = req->engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 	int ret;
 
 	flags |= PIPE_CONTROL_CS_STALL;
 
-	if (flush_domains) {
+	if (mode & EMIT_FLUSH) {
 		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
 	}
-	if (invalidate_domains) {
+	if (mode & EMIT_INVALIDATE) {
 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
 		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
@@ -419,14 +405,7 @@ gen8_render_ring_flush(struct drm_i915_gem_request *req,
 	return gen8_emit_pipe_control(req, flags, scratch_addr);
 }
 
-static void ring_write_tail(struct intel_engine_cs *engine,
-			    u32 value)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	I915_WRITE_TAIL(engine, value);
-}
-
-u64 intel_ring_get_active_head(struct intel_engine_cs *engine)
+u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 	u64 acthd;
@@ -488,7 +467,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
 		mmio = RING_HWS_PGA(engine->mmio_base);
 	}
 
-	I915_WRITE(mmio, (u32)engine->status_page.gfx_addr);
+	I915_WRITE(mmio, engine->status_page.ggtt_offset);
 	POSTING_READ(mmio);
 
 	/*
@@ -519,7 +498,7 @@ static bool stop_ring(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
-	if (!IS_GEN2(dev_priv)) {
+	if (INTEL_GEN(dev_priv) > 2) {
 		I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
 		if (intel_wait_for_register(dev_priv,
 					    RING_MI_MODE(engine->mmio_base),
@@ -539,9 +518,9 @@ static bool stop_ring(struct intel_engine_cs *engine)
 
 	I915_WRITE_CTL(engine, 0);
 	I915_WRITE_HEAD(engine, 0);
-	engine->write_tail(engine, 0);
+	I915_WRITE_TAIL(engine, 0);
 
-	if (!IS_GEN2(dev_priv)) {
+	if (INTEL_GEN(dev_priv) > 2) {
 		(void)I915_READ_CTL(engine);
 		I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
 	}
@@ -549,16 +528,10 @@ static bool stop_ring(struct intel_engine_cs *engine)
 	return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
 }
 
-void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
-{
-	memset(&engine->hangcheck, 0, sizeof(engine->hangcheck));
-}
-
 static int init_ring_common(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
-	struct intel_ringbuffer *ringbuf = engine->buffer;
-	struct drm_i915_gem_object *obj = ringbuf->obj;
+	struct intel_ring *ring = engine->buffer;
 	int ret = 0;
 
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
@@ -586,10 +559,12 @@ static int init_ring_common(struct intel_engine_cs *engine)
 		}
 	}
 
-	if (I915_NEED_GFX_HWS(dev_priv))
-		intel_ring_setup_status_page(engine);
-	else
+	if (HWS_NEEDS_PHYSICAL(dev_priv))
 		ring_setup_phys_status_page(engine);
+	else
+		intel_ring_setup_status_page(engine);
+
+	intel_engine_reset_breadcrumbs(engine);
 
 	/* Enforce ordering by reading HEAD register back */
 	I915_READ_HEAD(engine);
@@ -598,40 +573,39 @@ static int init_ring_common(struct intel_engine_cs *engine)
 	 * registers with the above sequence (the readback of the HEAD registers
 	 * also enforces ordering), otherwise the hw might lose the new ring
 	 * register values. */
-	I915_WRITE_START(engine, i915_gem_obj_ggtt_offset(obj));
+	I915_WRITE_START(engine, i915_ggtt_offset(ring->vma));
 
 	/* WaClearRingBufHeadRegAtInit:ctg,elk */
 	if (I915_READ_HEAD(engine))
 		DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
 			  engine->name, I915_READ_HEAD(engine));
-	I915_WRITE_HEAD(engine, 0);
-	(void)I915_READ_HEAD(engine);
+
+	intel_ring_update_space(ring);
+	I915_WRITE_HEAD(engine, ring->head);
+	I915_WRITE_TAIL(engine, ring->tail);
+	(void)I915_READ_TAIL(engine);
 
 	I915_WRITE_CTL(engine,
-			((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
+			((ring->size - PAGE_SIZE) & RING_NR_PAGES)
 			| RING_VALID);
 
 	/* If the head is still not zero, the ring is dead */
-	if (wait_for((I915_READ_CTL(engine) & RING_VALID) != 0 &&
-		     I915_READ_START(engine) == i915_gem_obj_ggtt_offset(obj) &&
-		     (I915_READ_HEAD(engine) & HEAD_ADDR) == 0, 50)) {
+	if (intel_wait_for_register_fw(dev_priv, RING_CTL(engine->mmio_base),
+				       RING_VALID, RING_VALID,
+				       50)) {
 		DRM_ERROR("%s initialization failed "
-			  "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
+			  "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
 			  engine->name,
 			  I915_READ_CTL(engine),
 			  I915_READ_CTL(engine) & RING_VALID,
-			  I915_READ_HEAD(engine), I915_READ_TAIL(engine),
+			  I915_READ_HEAD(engine), ring->head,
+			  I915_READ_TAIL(engine), ring->tail,
 			  I915_READ_START(engine),
-			  (unsigned long)i915_gem_obj_ggtt_offset(obj));
+			  i915_ggtt_offset(ring->vma));
 		ret = -EIO;
 		goto out;
 	}
 
-	ringbuf->last_retired_head = -1;
-	ringbuf->head = I915_READ_HEAD(engine);
-	ringbuf->tail = I915_READ_TAIL(engine) & TAIL_ADDR;
-	intel_ring_update_space(ringbuf);
-
 	intel_engine_init_hangcheck(engine);
 
 out:
@@ -640,59 +614,25 @@ out:
 	return ret;
 }
 
-void intel_fini_pipe_control(struct intel_engine_cs *engine)
-{
-	if (engine->scratch.obj == NULL)
-		return;
-
-	i915_gem_object_ggtt_unpin(engine->scratch.obj);
-	drm_gem_object_unreference(&engine->scratch.obj->base);
-	engine->scratch.obj = NULL;
-}
-
-int intel_init_pipe_control(struct intel_engine_cs *engine, int size)
+static void reset_ring_common(struct intel_engine_cs *engine,
+			      struct drm_i915_gem_request *request)
 {
-	struct drm_i915_gem_object *obj;
-	int ret;
-
-	WARN_ON(engine->scratch.obj);
+	struct intel_ring *ring = request->ring;
 
-	obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
-	if (!obj)
-		obj = i915_gem_object_create(&engine->i915->drm, size);
-	if (IS_ERR(obj)) {
-		DRM_ERROR("Failed to allocate scratch page\n");
-		ret = PTR_ERR(obj);
-		goto err;
-	}
-
-	ret = i915_gem_obj_ggtt_pin(obj, 4096, PIN_HIGH);
-	if (ret)
-		goto err_unref;
-
-	engine->scratch.obj = obj;
-	engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
-	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
-			 engine->name, engine->scratch.gtt_offset);
-	return 0;
-
-err_unref:
-	drm_gem_object_unreference(&engine->scratch.obj->base);
-err:
-	return ret;
+	ring->head = request->postfix;
+	ring->last_retired_head = -1;
 }
 
 static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	struct i915_workarounds *w = &req->i915->workarounds;
 	int ret, i;
 
 	if (w->count == 0)
 		return 0;
 
-	engine->gpu_caches_dirty = true;
-	ret = intel_ring_flush_all_caches(req);
+	ret = req->engine->emit_flush(req, EMIT_BARRIER);
 	if (ret)
 		return ret;
 
@@ -700,17 +640,16 @@ static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(w->count));
+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
 	for (i = 0; i < w->count; i++) {
-		intel_ring_emit_reg(engine, w->reg[i].addr);
-		intel_ring_emit(engine, w->reg[i].value);
+		intel_ring_emit_reg(ring, w->reg[i].addr);
+		intel_ring_emit(ring, w->reg[i].value);
 	}
-	intel_ring_emit(engine, MI_NOOP);
+	intel_ring_emit(ring, MI_NOOP);
 
-	intel_ring_advance(engine);
+	intel_ring_advance(ring);
 
-	engine->gpu_caches_dirty = true;
-	ret = intel_ring_flush_all_caches(req);
+	ret = req->engine->emit_flush(req, EMIT_BARRIER);
 	if (ret)
 		return ret;
 
@@ -1022,7 +961,7 @@ static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
 		 * Only consider slices where one, and only one, subslice has 7
 		 * EUs
 		 */
-		if (!is_power_of_2(dev_priv->info.subslice_7eu[i]))
+		if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
 			continue;
 
 		/*
@@ -1031,7 +970,7 @@ static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
 		 *
 		 * ->    0 <= ss <= 3;
 		 */
-		ss = ffs(dev_priv->info.subslice_7eu[i]) - 1;
+		ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
 		vals[i] = 3 - ss;
 	}
 
@@ -1178,8 +1117,8 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
 		I915_WRITE(GEN8_L3SQCREG1, L3_GENERAL_PRIO_CREDITS(62) |
 					   L3_HIGH_PRIO_CREDITS(2));
 
-	/* WaInsertDummyPushConstPs:bxt */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
+	/* WaToEnableHwFixForPushConstHWBug:bxt */
+	if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
 		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 				  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 
@@ -1222,8 +1161,8 @@ static int kbl_init_workarounds(struct intel_engine_cs *engine)
 		I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
 			   GEN8_LQSC_RO_PERF_DIS);
 
-	/* WaInsertDummyPushConstPs:kbl */
-	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
+	/* WaToEnableHwFixForPushConstHWBug:kbl */
+	if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
 		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 				  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 
@@ -1329,191 +1268,194 @@ static void render_ring_cleanup(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
-	if (dev_priv->semaphore_obj) {
-		i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
-		drm_gem_object_unreference(&dev_priv->semaphore_obj->base);
-		dev_priv->semaphore_obj = NULL;
-	}
-
-	intel_fini_pipe_control(engine);
+	i915_vma_unpin_and_release(&dev_priv->semaphore);
 }
 
-static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
-			   unsigned int num_dwords)
+static int gen8_rcs_signal(struct drm_i915_gem_request *req)
 {
-#define MBOX_UPDATE_DWORDS 8
-	struct intel_engine_cs *signaller = signaller_req->engine;
-	struct drm_i915_private *dev_priv = signaller_req->i915;
+	struct intel_ring *ring = req->ring;
+	struct drm_i915_private *dev_priv = req->i915;
 	struct intel_engine_cs *waiter;
 	enum intel_engine_id id;
 	int ret, num_rings;
 
-	num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
-	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
-#undef MBOX_UPDATE_DWORDS
-
-	ret = intel_ring_begin(signaller_req, num_dwords);
+	num_rings = INTEL_INFO(dev_priv)->num_rings;
+	ret = intel_ring_begin(req, (num_rings-1) * 8);
 	if (ret)
 		return ret;
 
 	for_each_engine_id(waiter, dev_priv, id) {
-		u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
+		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
 		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
 			continue;
 
-		intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
-		intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
-					   PIPE_CONTROL_QW_WRITE |
-					   PIPE_CONTROL_CS_STALL);
-		intel_ring_emit(signaller, lower_32_bits(gtt_offset));
-		intel_ring_emit(signaller, upper_32_bits(gtt_offset));
-		intel_ring_emit(signaller, signaller_req->seqno);
-		intel_ring_emit(signaller, 0);
-		intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
-					   MI_SEMAPHORE_TARGET(waiter->hw_id));
-		intel_ring_emit(signaller, 0);
+		intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
+		intel_ring_emit(ring,
+				PIPE_CONTROL_GLOBAL_GTT_IVB |
+				PIPE_CONTROL_QW_WRITE |
+				PIPE_CONTROL_CS_STALL);
+		intel_ring_emit(ring, lower_32_bits(gtt_offset));
+		intel_ring_emit(ring, upper_32_bits(gtt_offset));
+		intel_ring_emit(ring, req->fence.seqno);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring,
+				MI_SEMAPHORE_SIGNAL |
+				MI_SEMAPHORE_TARGET(waiter->hw_id));
+		intel_ring_emit(ring, 0);
 	}
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
-static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
-			   unsigned int num_dwords)
+static int gen8_xcs_signal(struct drm_i915_gem_request *req)
 {
-#define MBOX_UPDATE_DWORDS 6
-	struct intel_engine_cs *signaller = signaller_req->engine;
-	struct drm_i915_private *dev_priv = signaller_req->i915;
+	struct intel_ring *ring = req->ring;
+	struct drm_i915_private *dev_priv = req->i915;
 	struct intel_engine_cs *waiter;
 	enum intel_engine_id id;
 	int ret, num_rings;
 
-	num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
-	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
-#undef MBOX_UPDATE_DWORDS
-
-	ret = intel_ring_begin(signaller_req, num_dwords);
+	num_rings = INTEL_INFO(dev_priv)->num_rings;
+	ret = intel_ring_begin(req, (num_rings-1) * 6);
 	if (ret)
 		return ret;
 
 	for_each_engine_id(waiter, dev_priv, id) {
-		u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
+		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
 		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
 			continue;
 
-		intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
-					   MI_FLUSH_DW_OP_STOREDW);
-		intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
-					   MI_FLUSH_DW_USE_GTT);
-		intel_ring_emit(signaller, upper_32_bits(gtt_offset));
-		intel_ring_emit(signaller, signaller_req->seqno);
-		intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
-					   MI_SEMAPHORE_TARGET(waiter->hw_id));
-		intel_ring_emit(signaller, 0);
+		intel_ring_emit(ring,
+				(MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW);
+		intel_ring_emit(ring,
+				lower_32_bits(gtt_offset) |
+				MI_FLUSH_DW_USE_GTT);
+		intel_ring_emit(ring, upper_32_bits(gtt_offset));
+		intel_ring_emit(ring, req->fence.seqno);
+		intel_ring_emit(ring,
+				MI_SEMAPHORE_SIGNAL |
+				MI_SEMAPHORE_TARGET(waiter->hw_id));
+		intel_ring_emit(ring, 0);
 	}
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
-static int gen6_signal(struct drm_i915_gem_request *signaller_req,
-		       unsigned int num_dwords)
+static int gen6_signal(struct drm_i915_gem_request *req)
 {
-	struct intel_engine_cs *signaller = signaller_req->engine;
-	struct drm_i915_private *dev_priv = signaller_req->i915;
-	struct intel_engine_cs *useless;
-	enum intel_engine_id id;
+	struct intel_ring *ring = req->ring;
+	struct drm_i915_private *dev_priv = req->i915;
+	struct intel_engine_cs *engine;
 	int ret, num_rings;
 
-#define MBOX_UPDATE_DWORDS 3
-	num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
-	num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
-#undef MBOX_UPDATE_DWORDS
-
-	ret = intel_ring_begin(signaller_req, num_dwords);
+	num_rings = INTEL_INFO(dev_priv)->num_rings;
+	ret = intel_ring_begin(req, round_up((num_rings-1) * 3, 2));
 	if (ret)
 		return ret;
 
-	for_each_engine_id(useless, dev_priv, id) {
-		i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[id];
+	for_each_engine(engine, dev_priv) {
+		i915_reg_t mbox_reg;
+
+		if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK))
+			continue;
 
+		mbox_reg = req->engine->semaphore.mbox.signal[engine->hw_id];
 		if (i915_mmio_reg_valid(mbox_reg)) {
-			intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
-			intel_ring_emit_reg(signaller, mbox_reg);
-			intel_ring_emit(signaller, signaller_req->seqno);
+			intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+			intel_ring_emit_reg(ring, mbox_reg);
+			intel_ring_emit(ring, req->fence.seqno);
 		}
 	}
 
 	/* If num_dwords was rounded, make sure the tail pointer is correct */
 	if (num_rings % 2 == 0)
-		intel_ring_emit(signaller, MI_NOOP);
+		intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
+
+	return 0;
+}
+
+static void i9xx_submit_request(struct drm_i915_gem_request *request)
+{
+	struct drm_i915_private *dev_priv = request->i915;
+
+	I915_WRITE_TAIL(request->engine,
+			intel_ring_offset(request->ring, request->tail));
+}
+
+static int i9xx_emit_request(struct drm_i915_gem_request *req)
+{
+	struct intel_ring *ring = req->ring;
+	int ret;
+
+	ret = intel_ring_begin(req, 4);
+	if (ret)
+		return ret;
+
+	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
+	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+	intel_ring_emit(ring, req->fence.seqno);
+	intel_ring_emit(ring, MI_USER_INTERRUPT);
+	intel_ring_advance(ring);
+
+	req->tail = ring->tail;
 
 	return 0;
 }
 
 /**
- * gen6_add_request - Update the semaphore mailbox registers
+ * gen6_sema_emit_request - Update the semaphore mailbox registers
  *
  * @request - request to write to the ring
  *
  * Update the mailbox registers in the *other* rings with the current seqno.
  * This acts like a signal in the canonical semaphore.
  */
-static int
-gen6_add_request(struct drm_i915_gem_request *req)
+static int gen6_sema_emit_request(struct drm_i915_gem_request *req)
 {
-	struct intel_engine_cs *engine = req->engine;
 	int ret;
 
-	if (engine->semaphore.signal)
-		ret = engine->semaphore.signal(req, 4);
-	else
-		ret = intel_ring_begin(req, 4);
-
+	ret = req->engine->semaphore.signal(req);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
-	intel_ring_emit(engine,
-			I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-	intel_ring_emit(engine, req->seqno);
-	intel_ring_emit(engine, MI_USER_INTERRUPT);
-	__intel_ring_advance(engine);
-
-	return 0;
+	return i9xx_emit_request(req);
 }
 
-static int
-gen8_render_add_request(struct drm_i915_gem_request *req)
+static int gen8_render_emit_request(struct drm_i915_gem_request *req)
 {
 	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	int ret;
 
-	if (engine->semaphore.signal)
-		ret = engine->semaphore.signal(req, 8);
-	else
-		ret = intel_ring_begin(req, 8);
+	if (engine->semaphore.signal) {
+		ret = engine->semaphore.signal(req);
+		if (ret)
+			return ret;
+	}
+
+	ret = intel_ring_begin(req, 8);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(6));
-	intel_ring_emit(engine, (PIPE_CONTROL_GLOBAL_GTT_IVB |
-				 PIPE_CONTROL_CS_STALL |
-				 PIPE_CONTROL_QW_WRITE));
-	intel_ring_emit(engine, intel_hws_seqno_address(req->engine));
-	intel_ring_emit(engine, 0);
-	intel_ring_emit(engine, i915_gem_request_get_seqno(req));
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
+	intel_ring_emit(ring, (PIPE_CONTROL_GLOBAL_GTT_IVB |
+			       PIPE_CONTROL_CS_STALL |
+			       PIPE_CONTROL_QW_WRITE));
+	intel_ring_emit(ring, intel_hws_seqno_address(engine));
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
 	/* We're thrashing one dword of HWS. */
-	intel_ring_emit(engine, 0);
-	intel_ring_emit(engine, MI_USER_INTERRUPT);
-	intel_ring_emit(engine, MI_NOOP);
-	__intel_ring_advance(engine);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, MI_USER_INTERRUPT);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
-	return 0;
-}
+	req->tail = ring->tail;
 
-static inline bool i915_gem_has_seqno_wrapped(struct drm_i915_private *dev_priv,
-					      u32 seqno)
-{
-	return dev_priv->last_seqno < seqno;
+	return 0;
 }
 
 /**
@@ -1525,82 +1467,71 @@ static inline bool i915_gem_has_seqno_wrapped(struct drm_i915_private *dev_priv,
  */
 
 static int
-gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
-	       struct intel_engine_cs *signaller,
-	       u32 seqno)
+gen8_ring_sync_to(struct drm_i915_gem_request *req,
+		  struct drm_i915_gem_request *signal)
 {
-	struct intel_engine_cs *waiter = waiter_req->engine;
-	struct drm_i915_private *dev_priv = waiter_req->i915;
-	u64 offset = GEN8_WAIT_OFFSET(waiter, signaller->id);
+	struct intel_ring *ring = req->ring;
+	struct drm_i915_private *dev_priv = req->i915;
+	u64 offset = GEN8_WAIT_OFFSET(req->engine, signal->engine->id);
 	struct i915_hw_ppgtt *ppgtt;
 	int ret;
 
-	ret = intel_ring_begin(waiter_req, 4);
+	ret = intel_ring_begin(req, 4);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
-				MI_SEMAPHORE_GLOBAL_GTT |
-				MI_SEMAPHORE_SAD_GTE_SDD);
-	intel_ring_emit(waiter, seqno);
-	intel_ring_emit(waiter, lower_32_bits(offset));
-	intel_ring_emit(waiter, upper_32_bits(offset));
-	intel_ring_advance(waiter);
+	intel_ring_emit(ring,
+			MI_SEMAPHORE_WAIT |
+			MI_SEMAPHORE_GLOBAL_GTT |
+			MI_SEMAPHORE_SAD_GTE_SDD);
+	intel_ring_emit(ring, signal->fence.seqno);
+	intel_ring_emit(ring, lower_32_bits(offset));
+	intel_ring_emit(ring, upper_32_bits(offset));
+	intel_ring_advance(ring);
 
 	/* When the !RCS engines idle waiting upon a semaphore, they lose their
 	 * pagetables and we must reload them before executing the batch.
 	 * We do this on the i915_switch_context() following the wait and
 	 * before the dispatch.
 	 */
-	ppgtt = waiter_req->ctx->ppgtt;
-	if (ppgtt && waiter_req->engine->id != RCS)
-		ppgtt->pd_dirty_rings |= intel_engine_flag(waiter_req->engine);
+	ppgtt = req->ctx->ppgtt;
+	if (ppgtt && req->engine->id != RCS)
+		ppgtt->pd_dirty_rings |= intel_engine_flag(req->engine);
 	return 0;
 }
 
 static int
-gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
-	       struct intel_engine_cs *signaller,
-	       u32 seqno)
+gen6_ring_sync_to(struct drm_i915_gem_request *req,
+		  struct drm_i915_gem_request *signal)
 {
-	struct intel_engine_cs *waiter = waiter_req->engine;
+	struct intel_ring *ring = req->ring;
 	u32 dw1 = MI_SEMAPHORE_MBOX |
 		  MI_SEMAPHORE_COMPARE |
 		  MI_SEMAPHORE_REGISTER;
-	u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
+	u32 wait_mbox = signal->engine->semaphore.mbox.wait[req->engine->hw_id];
 	int ret;
 
-	/* Throughout all of the GEM code, seqno passed implies our current
-	 * seqno is >= the last seqno executed. However for hardware the
-	 * comparison is strictly greater than.
-	 */
-	seqno -= 1;
-
 	WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
 
-	ret = intel_ring_begin(waiter_req, 4);
+	ret = intel_ring_begin(req, 4);
 	if (ret)
 		return ret;
 
-	/* If seqno wrap happened, omit the wait with no-ops */
-	if (likely(!i915_gem_has_seqno_wrapped(waiter_req->i915, seqno))) {
-		intel_ring_emit(waiter, dw1 | wait_mbox);
-		intel_ring_emit(waiter, seqno);
-		intel_ring_emit(waiter, 0);
-		intel_ring_emit(waiter, MI_NOOP);
-	} else {
-		intel_ring_emit(waiter, MI_NOOP);
-		intel_ring_emit(waiter, MI_NOOP);
-		intel_ring_emit(waiter, MI_NOOP);
-		intel_ring_emit(waiter, MI_NOOP);
-	}
-	intel_ring_advance(waiter);
+	intel_ring_emit(ring, dw1 | wait_mbox);
+	/* Throughout all of the GEM code, seqno passed implies our current
+	 * seqno is >= the last seqno executed. However for hardware the
+	 * comparison is strictly greater than.
+	 */
+	intel_ring_emit(ring, signal->fence.seqno - 1);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static void
-gen5_seqno_barrier(struct intel_engine_cs *ring)
+gen5_seqno_barrier(struct intel_engine_cs *engine)
 {
 	/* MI_STORE are internally buffered by the GPU and not flushed
 	 * either by MI_FLUSH or SyncFlush or any other combination of
@@ -1693,40 +1624,18 @@ i8xx_irq_disable(struct intel_engine_cs *engine)
 }
 
 static int
-bsd_ring_flush(struct drm_i915_gem_request *req,
-	       u32     invalidate_domains,
-	       u32     flush_domains)
+bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	int ret;
 
 	ret = intel_ring_begin(req, 2);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, MI_FLUSH);
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
-	return 0;
-}
-
-static int
-i9xx_add_request(struct drm_i915_gem_request *req)
-{
-	struct intel_engine_cs *engine = req->engine;
-	int ret;
-
-	ret = intel_ring_begin(req, 4);
-	if (ret)
-		return ret;
-
-	intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
-	intel_ring_emit(engine,
-			I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-	intel_ring_emit(engine, req->seqno);
-	intel_ring_emit(engine, MI_USER_INTERRUPT);
-	__intel_ring_advance(engine);
-
+	intel_ring_emit(ring, MI_FLUSH);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 	return 0;
 }
 
@@ -1788,24 +1697,24 @@ gen8_irq_disable(struct intel_engine_cs *engine)
 }
 
 static int
-i965_dispatch_execbuffer(struct drm_i915_gem_request *req,
-			 u64 offset, u32 length,
-			 unsigned dispatch_flags)
+i965_emit_bb_start(struct drm_i915_gem_request *req,
+		   u64 offset, u32 length,
+		   unsigned int dispatch_flags)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	int ret;
 
 	ret = intel_ring_begin(req, 2);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine,
+	intel_ring_emit(ring,
 			MI_BATCH_BUFFER_START |
 			MI_BATCH_GTT |
 			(dispatch_flags & I915_DISPATCH_SECURE ?
 			 0 : MI_BATCH_NON_SECURE_I965));
-	intel_ring_emit(engine, offset);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, offset);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -1815,12 +1724,12 @@ i965_dispatch_execbuffer(struct drm_i915_gem_request *req,
 #define I830_TLB_ENTRIES (2)
 #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
 static int
-i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
-			 u64 offset, u32 len,
-			 unsigned dispatch_flags)
+i830_emit_bb_start(struct drm_i915_gem_request *req,
+		   u64 offset, u32 len,
+		   unsigned int dispatch_flags)
 {
-	struct intel_engine_cs *engine = req->engine;
-	u32 cs_offset = engine->scratch.gtt_offset;
+	struct intel_ring *ring = req->ring;
+	u32 cs_offset = i915_ggtt_offset(req->engine->scratch);
 	int ret;
 
 	ret = intel_ring_begin(req, 6);
@@ -1828,13 +1737,13 @@ i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
 		return ret;
 
 	/* Evict the invalid PTE TLBs */
-	intel_ring_emit(engine, COLOR_BLT_CMD | BLT_WRITE_RGBA);
-	intel_ring_emit(engine, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
-	intel_ring_emit(engine, I830_TLB_ENTRIES << 16 | 4); /* load each page */
-	intel_ring_emit(engine, cs_offset);
-	intel_ring_emit(engine, 0xdeadbeef);
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA);
+	intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
+	intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */
+	intel_ring_emit(ring, cs_offset);
+	intel_ring_emit(ring, 0xdeadbeef);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
 		if (len > I830_BATCH_LIMIT)
@@ -1848,17 +1757,17 @@ i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
 		 * stable batch scratch bo area (so that the CS never
 		 * stumbles over its tlb invalidation bug) ...
 		 */
-		intel_ring_emit(engine, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
-		intel_ring_emit(engine,
+		intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
+		intel_ring_emit(ring,
 				BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096);
-		intel_ring_emit(engine, DIV_ROUND_UP(len, 4096) << 16 | 4096);
-		intel_ring_emit(engine, cs_offset);
-		intel_ring_emit(engine, 4096);
-		intel_ring_emit(engine, offset);
+		intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 4096);
+		intel_ring_emit(ring, cs_offset);
+		intel_ring_emit(ring, 4096);
+		intel_ring_emit(ring, offset);
 
-		intel_ring_emit(engine, MI_FLUSH);
-		intel_ring_emit(engine, MI_NOOP);
-		intel_ring_advance(engine);
+		intel_ring_emit(ring, MI_FLUSH);
+		intel_ring_emit(ring, MI_NOOP);
+		intel_ring_advance(ring);
 
 		/* ... and execute it. */
 		offset = cs_offset;
@@ -1868,30 +1777,30 @@ i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
-	intel_ring_emit(engine, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
-					  0 : MI_BATCH_NON_SECURE));
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
+	intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
+					0 : MI_BATCH_NON_SECURE));
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static int
-i915_dispatch_execbuffer(struct drm_i915_gem_request *req,
-			 u64 offset, u32 len,
-			 unsigned dispatch_flags)
+i915_emit_bb_start(struct drm_i915_gem_request *req,
+		   u64 offset, u32 len,
+		   unsigned int dispatch_flags)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	int ret;
 
 	ret = intel_ring_begin(req, 2);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
-	intel_ring_emit(engine, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
-					  0 : MI_BATCH_NON_SECURE));
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
+	intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
+					0 : MI_BATCH_NON_SECURE));
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -1909,79 +1818,79 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine)
 
 static void cleanup_status_page(struct intel_engine_cs *engine)
 {
-	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
 
-	obj = engine->status_page.obj;
-	if (obj == NULL)
+	vma = fetch_and_zero(&engine->status_page.vma);
+	if (!vma)
 		return;
 
-	kunmap(sg_page(obj->pages->sgl));
-	i915_gem_object_ggtt_unpin(obj);
-	drm_gem_object_unreference(&obj->base);
-	engine->status_page.obj = NULL;
+	i915_vma_unpin(vma);
+	i915_gem_object_unpin_map(vma->obj);
+	i915_vma_put(vma);
 }
 
 static int init_status_page(struct intel_engine_cs *engine)
 {
-	struct drm_i915_gem_object *obj = engine->status_page.obj;
-
-	if (obj == NULL) {
-		unsigned flags;
-		int ret;
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	unsigned int flags;
+	int ret;
 
-		obj = i915_gem_object_create(&engine->i915->drm, 4096);
-		if (IS_ERR(obj)) {
-			DRM_ERROR("Failed to allocate status page\n");
-			return PTR_ERR(obj);
-		}
+	obj = i915_gem_object_create(&engine->i915->drm, 4096);
+	if (IS_ERR(obj)) {
+		DRM_ERROR("Failed to allocate status page\n");
+		return PTR_ERR(obj);
+	}
 
-		ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
-		if (ret)
-			goto err_unref;
-
-		flags = 0;
-		if (!HAS_LLC(engine->i915))
-			/* On g33, we cannot place HWS above 256MiB, so
-			 * restrict its pinning to the low mappable arena.
-			 * Though this restriction is not documented for
-			 * gen4, gen5, or byt, they also behave similarly
-			 * and hang if the HWS is placed at the top of the
-			 * GTT. To generalise, it appears that all !llc
-			 * platforms have issues with us placing the HWS
-			 * above the mappable region (even though we never
-			 * actualy map it).
-			 */
-			flags |= PIN_MAPPABLE;
-		ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
-		if (ret) {
-err_unref:
-			drm_gem_object_unreference(&obj->base);
-			return ret;
-		}
+	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
+	if (ret)
+		goto err;
 
-		engine->status_page.obj = obj;
+	vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
+		goto err;
 	}
 
-	engine->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
-	engine->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
-	memset(engine->status_page.page_addr, 0, PAGE_SIZE);
+	flags = PIN_GLOBAL;
+	if (!HAS_LLC(engine->i915))
+		/* On g33, we cannot place HWS above 256MiB, so
+		 * restrict its pinning to the low mappable arena.
+		 * Though this restriction is not documented for
+		 * gen4, gen5, or byt, they also behave similarly
+		 * and hang if the HWS is placed at the top of the
+		 * GTT. To generalise, it appears that all !llc
+		 * platforms have issues with us placing the HWS
+		 * above the mappable region (even though we never
+		 * actualy map it).
+		 */
+		flags |= PIN_MAPPABLE;
+	ret = i915_vma_pin(vma, 0, 4096, flags);
+	if (ret)
+		goto err;
 
-	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
-			engine->name, engine->status_page.gfx_addr);
+	engine->status_page.vma = vma;
+	engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
+	engine->status_page.page_addr =
+		i915_gem_object_pin_map(obj, I915_MAP_WB);
 
+	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
+			 engine->name, i915_ggtt_offset(vma));
 	return 0;
+
+err:
+	i915_gem_object_put(obj);
+	return ret;
 }
 
 static int init_phys_status_page(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
-	if (!dev_priv->status_page_dmah) {
-		dev_priv->status_page_dmah =
-			drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
-		if (!dev_priv->status_page_dmah)
-			return -ENOMEM;
-	}
+	dev_priv->status_page_dmah =
+		drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
+	if (!dev_priv->status_page_dmah)
+		return -ENOMEM;
 
 	engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
 	memset(engine->status_page.page_addr, 0, PAGE_SIZE);
@@ -1989,115 +1898,105 @@ static int init_phys_status_page(struct intel_engine_cs *engine)
 	return 0;
 }
 
-void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
-{
-	GEM_BUG_ON(ringbuf->vma == NULL);
-	GEM_BUG_ON(ringbuf->virtual_start == NULL);
-
-	if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen)
-		i915_gem_object_unpin_map(ringbuf->obj);
-	else
-		i915_vma_unpin_iomap(ringbuf->vma);
-	ringbuf->virtual_start = NULL;
-
-	i915_gem_object_ggtt_unpin(ringbuf->obj);
-	ringbuf->vma = NULL;
-}
-
-int intel_pin_and_map_ringbuffer_obj(struct drm_i915_private *dev_priv,
-				     struct intel_ringbuffer *ringbuf)
+int intel_ring_pin(struct intel_ring *ring)
 {
-	struct drm_i915_gem_object *obj = ringbuf->obj;
 	/* Ring wraparound at offset 0 sometimes hangs. No idea why. */
-	unsigned flags = PIN_OFFSET_BIAS | 4096;
+	unsigned int flags = PIN_GLOBAL | PIN_OFFSET_BIAS | 4096;
+	enum i915_map_type map;
+	struct i915_vma *vma = ring->vma;
 	void *addr;
 	int ret;
 
-	if (HAS_LLC(dev_priv) && !obj->stolen) {
-		ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, flags);
-		if (ret)
-			return ret;
+	GEM_BUG_ON(ring->vaddr);
 
-		ret = i915_gem_object_set_to_cpu_domain(obj, true);
-		if (ret)
-			goto err_unpin;
+	map = HAS_LLC(ring->engine->i915) ? I915_MAP_WB : I915_MAP_WC;
 
-		addr = i915_gem_object_pin_map(obj);
-		if (IS_ERR(addr)) {
-			ret = PTR_ERR(addr);
-			goto err_unpin;
-		}
-	} else {
-		ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE,
-					    flags | PIN_MAPPABLE);
-		if (ret)
-			return ret;
+	if (vma->obj->stolen)
+		flags |= PIN_MAPPABLE;
 
-		ret = i915_gem_object_set_to_gtt_domain(obj, true);
-		if (ret)
-			goto err_unpin;
+	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
+		if (flags & PIN_MAPPABLE || map == I915_MAP_WC)
+			ret = i915_gem_object_set_to_gtt_domain(vma->obj, true);
+		else
+			ret = i915_gem_object_set_to_cpu_domain(vma->obj, true);
+		if (unlikely(ret))
+			return ret;
+	}
 
-		/* Access through the GTT requires the device to be awake. */
-		assert_rpm_wakelock_held(dev_priv);
+	ret = i915_vma_pin(vma, 0, PAGE_SIZE, flags);
+	if (unlikely(ret))
+		return ret;
 
-		addr = i915_vma_pin_iomap(i915_gem_obj_to_ggtt(obj));
-		if (IS_ERR(addr)) {
-			ret = PTR_ERR(addr);
-			goto err_unpin;
-		}
-	}
+	if (i915_vma_is_map_and_fenceable(vma))
+		addr = (void __force *)i915_vma_pin_iomap(vma);
+	else
+		addr = i915_gem_object_pin_map(vma->obj, map);
+	if (IS_ERR(addr))
+		goto err;
 
-	ringbuf->virtual_start = addr;
-	ringbuf->vma = i915_gem_obj_to_ggtt(obj);
+	ring->vaddr = addr;
 	return 0;
 
-err_unpin:
-	i915_gem_object_ggtt_unpin(obj);
-	return ret;
+err:
+	i915_vma_unpin(vma);
+	return PTR_ERR(addr);
 }
 
-static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
+void intel_ring_unpin(struct intel_ring *ring)
 {
-	drm_gem_object_unreference(&ringbuf->obj->base);
-	ringbuf->obj = NULL;
+	GEM_BUG_ON(!ring->vma);
+	GEM_BUG_ON(!ring->vaddr);
+
+	if (i915_vma_is_map_and_fenceable(ring->vma))
+		i915_vma_unpin_iomap(ring->vma);
+	else
+		i915_gem_object_unpin_map(ring->vma->obj);
+	ring->vaddr = NULL;
+
+	i915_vma_unpin(ring->vma);
 }
 
-static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
-				      struct intel_ringbuffer *ringbuf)
+static struct i915_vma *
+intel_ring_create_vma(struct drm_i915_private *dev_priv, int size)
 {
 	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
 
-	obj = NULL;
-	if (!HAS_LLC(dev))
-		obj = i915_gem_object_create_stolen(dev, ringbuf->size);
-	if (obj == NULL)
-		obj = i915_gem_object_create(dev, ringbuf->size);
+	obj = i915_gem_object_create_stolen(&dev_priv->drm, size);
+	if (!obj)
+		obj = i915_gem_object_create(&dev_priv->drm, size);
 	if (IS_ERR(obj))
-		return PTR_ERR(obj);
+		return ERR_CAST(obj);
 
 	/* mark ring buffers as read-only from GPU side by default */
 	obj->gt_ro = 1;
 
-	ringbuf->obj = obj;
+	vma = i915_vma_create(obj, &dev_priv->ggtt.base, NULL);
+	if (IS_ERR(vma))
+		goto err;
 
-	return 0;
+	return vma;
+
+err:
+	i915_gem_object_put(obj);
+	return vma;
 }
 
-struct intel_ringbuffer *
-intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
+struct intel_ring *
+intel_engine_create_ring(struct intel_engine_cs *engine, int size)
 {
-	struct intel_ringbuffer *ring;
-	int ret;
+	struct intel_ring *ring;
+	struct i915_vma *vma;
+
+	GEM_BUG_ON(!is_power_of_2(size));
 
 	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
-	if (ring == NULL) {
-		DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
-				 engine->name);
+	if (!ring)
 		return ERR_PTR(-ENOMEM);
-	}
 
 	ring->engine = engine;
-	list_add(&ring->link, &engine->buffers);
+
+	INIT_LIST_HEAD(&ring->request_list);
 
 	ring->size = size;
 	/* Workaround an erratum on the i830 which causes a hang if
@@ -2111,23 +2010,20 @@ intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
 	ring->last_retired_head = -1;
 	intel_ring_update_space(ring);
 
-	ret = intel_alloc_ringbuffer_obj(&engine->i915->drm, ring);
-	if (ret) {
-		DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n",
-				 engine->name, ret);
-		list_del(&ring->link);
+	vma = intel_ring_create_vma(engine->i915, size);
+	if (IS_ERR(vma)) {
 		kfree(ring);
-		return ERR_PTR(ret);
+		return ERR_CAST(vma);
 	}
+	ring->vma = vma;
 
 	return ring;
 }
 
 void
-intel_ringbuffer_free(struct intel_ringbuffer *ring)
+intel_ring_free(struct intel_ring *ring)
 {
-	intel_destroy_ringbuffer_obj(ring);
-	list_del(&ring->link);
+	i915_vma_put(ring->vma);
 	kfree(ring);
 }
 
@@ -2143,7 +2039,12 @@ static int intel_ring_context_pin(struct i915_gem_context *ctx,
 		return 0;
 
 	if (ce->state) {
-		ret = i915_gem_obj_ggtt_pin(ce->state, ctx->ggtt_alignment, 0);
+		ret = i915_gem_object_set_to_gtt_domain(ce->state->obj, false);
+		if (ret)
+			goto error;
+
+		ret = i915_vma_pin(ce->state, 0, ctx->ggtt_alignment,
+				   PIN_GLOBAL | PIN_HIGH);
 		if (ret)
 			goto error;
 	}
@@ -2158,7 +2059,7 @@ static int intel_ring_context_pin(struct i915_gem_context *ctx,
 	if (ctx == ctx->i915->kernel_context)
 		ce->initialised = true;
 
-	i915_gem_context_reference(ctx);
+	i915_gem_context_get(ctx);
 	return 0;
 
 error:
@@ -2177,30 +2078,25 @@ static void intel_ring_context_unpin(struct i915_gem_context *ctx,
 		return;
 
 	if (ce->state)
-		i915_gem_object_ggtt_unpin(ce->state);
+		i915_vma_unpin(ce->state);
 
-	i915_gem_context_unreference(ctx);
+	i915_gem_context_put(ctx);
 }
 
-static int intel_init_ring_buffer(struct drm_device *dev,
-				  struct intel_engine_cs *engine)
+static int intel_init_ring_buffer(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_ringbuffer *ringbuf;
+	struct drm_i915_private *dev_priv = engine->i915;
+	struct intel_ring *ring;
 	int ret;
 
 	WARN_ON(engine->buffer);
 
-	engine->i915 = dev_priv;
-	INIT_LIST_HEAD(&engine->active_list);
-	INIT_LIST_HEAD(&engine->request_list);
-	INIT_LIST_HEAD(&engine->execlist_queue);
-	INIT_LIST_HEAD(&engine->buffers);
-	i915_gem_batch_pool_init(dev, &engine->batch_pool);
+	intel_engine_setup_common(engine);
+
 	memset(engine->semaphore.sync_seqno, 0,
 	       sizeof(engine->semaphore.sync_seqno));
 
-	ret = intel_engine_init_breadcrumbs(engine);
+	ret = intel_engine_init_common(engine);
 	if (ret)
 		goto error;
 
@@ -2215,44 +2111,38 @@ static int intel_init_ring_buffer(struct drm_device *dev,
 	if (ret)
 		goto error;
 
-	ringbuf = intel_engine_create_ringbuffer(engine, 32 * PAGE_SIZE);
-	if (IS_ERR(ringbuf)) {
-		ret = PTR_ERR(ringbuf);
+	ring = intel_engine_create_ring(engine, 32 * PAGE_SIZE);
+	if (IS_ERR(ring)) {
+		ret = PTR_ERR(ring);
 		goto error;
 	}
-	engine->buffer = ringbuf;
 
-	if (I915_NEED_GFX_HWS(dev_priv)) {
-		ret = init_status_page(engine);
+	if (HWS_NEEDS_PHYSICAL(dev_priv)) {
+		WARN_ON(engine->id != RCS);
+		ret = init_phys_status_page(engine);
 		if (ret)
 			goto error;
 	} else {
-		WARN_ON(engine->id != RCS);
-		ret = init_phys_status_page(engine);
+		ret = init_status_page(engine);
 		if (ret)
 			goto error;
 	}
 
-	ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ringbuf);
+	ret = intel_ring_pin(ring);
 	if (ret) {
-		DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
-				engine->name, ret);
-		intel_destroy_ringbuffer_obj(ringbuf);
+		intel_ring_free(ring);
 		goto error;
 	}
-
-	ret = i915_cmd_parser_init_ring(engine);
-	if (ret)
-		goto error;
+	engine->buffer = ring;
 
 	return 0;
 
 error:
-	intel_cleanup_engine(engine);
+	intel_engine_cleanup(engine);
 	return ret;
 }
 
-void intel_cleanup_engine(struct intel_engine_cs *engine)
+void intel_engine_cleanup(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv;
 
@@ -2262,49 +2152,39 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
 	dev_priv = engine->i915;
 
 	if (engine->buffer) {
-		intel_stop_engine(engine);
-		WARN_ON(!IS_GEN2(dev_priv) && (I915_READ_MODE(engine) & MODE_IDLE) == 0);
+		WARN_ON(INTEL_GEN(dev_priv) > 2 &&
+			(I915_READ_MODE(engine) & MODE_IDLE) == 0);
 
-		intel_unpin_ringbuffer_obj(engine->buffer);
-		intel_ringbuffer_free(engine->buffer);
+		intel_ring_unpin(engine->buffer);
+		intel_ring_free(engine->buffer);
 		engine->buffer = NULL;
 	}
 
 	if (engine->cleanup)
 		engine->cleanup(engine);
 
-	if (I915_NEED_GFX_HWS(dev_priv)) {
-		cleanup_status_page(engine);
-	} else {
+	if (HWS_NEEDS_PHYSICAL(dev_priv)) {
 		WARN_ON(engine->id != RCS);
 		cleanup_phys_status_page(engine);
+	} else {
+		cleanup_status_page(engine);
 	}
 
-	i915_cmd_parser_fini_ring(engine);
-	i915_gem_batch_pool_fini(&engine->batch_pool);
-	intel_engine_fini_breadcrumbs(engine);
+	intel_engine_cleanup_common(engine);
 
 	intel_ring_context_unpin(dev_priv->kernel_context, engine);
 
 	engine->i915 = NULL;
 }
 
-int intel_engine_idle(struct intel_engine_cs *engine)
+void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_gem_request *req;
-
-	/* Wait upon the last request to be completed */
-	if (list_empty(&engine->request_list))
-		return 0;
+	struct intel_engine_cs *engine;
 
-	req = list_entry(engine->request_list.prev,
-			 struct drm_i915_gem_request,
-			 list);
-
-	/* Make sure we do not trigger any retires */
-	return __i915_wait_request(req,
-				   req->i915->mm.interruptible,
-				   NULL, NULL);
+	for_each_engine(engine, dev_priv) {
+		engine->buffer->head = engine->buffer->tail;
+		engine->buffer->last_retired_head = -1;
+	}
 }
 
 int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
@@ -2317,7 +2197,7 @@ int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
 	 */
 	request->reserved_space += LEGACY_REQUEST_SIZE;
 
-	request->ringbuf = request->engine->buffer;
+	request->ring = request->engine->buffer;
 
 	ret = intel_ring_begin(request, 0);
 	if (ret)
@@ -2329,12 +2209,12 @@ int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
 
 static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
 {
-	struct intel_ringbuffer *ringbuf = req->ringbuf;
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	struct drm_i915_gem_request *target;
+	int ret;
 
-	intel_ring_update_space(ringbuf);
-	if (ringbuf->space >= bytes)
+	intel_ring_update_space(ring);
+	if (ring->space >= bytes)
 		return 0;
 
 	/*
@@ -2348,35 +2228,37 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
 	 */
 	GEM_BUG_ON(!req->reserved_space);
 
-	list_for_each_entry(target, &engine->request_list, list) {
+	list_for_each_entry(target, &ring->request_list, ring_link) {
 		unsigned space;
 
-		/*
-		 * The request queue is per-engine, so can contain requests
-		 * from multiple ringbuffers. Here, we must ignore any that
-		 * aren't from the ringbuffer we're considering.
-		 */
-		if (target->ringbuf != ringbuf)
-			continue;
-
 		/* Would completion of this request free enough space? */
-		space = __intel_ring_space(target->postfix, ringbuf->tail,
-					   ringbuf->size);
+		space = __intel_ring_space(target->postfix, ring->tail,
+					   ring->size);
 		if (space >= bytes)
 			break;
 	}
 
-	if (WARN_ON(&target->list == &engine->request_list))
+	if (WARN_ON(&target->ring_link == &ring->request_list))
 		return -ENOSPC;
 
-	return i915_wait_request(target);
+	ret = i915_wait_request(target,
+				I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
+				NULL, NO_WAITBOOST);
+	if (ret)
+		return ret;
+
+	i915_gem_request_retire_upto(target);
+
+	intel_ring_update_space(ring);
+	GEM_BUG_ON(ring->space < bytes);
+	return 0;
 }
 
 int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
 {
-	struct intel_ringbuffer *ringbuf = req->ringbuf;
-	int remain_actual = ringbuf->size - ringbuf->tail;
-	int remain_usable = ringbuf->effective_size - ringbuf->tail;
+	struct intel_ring *ring = req->ring;
+	int remain_actual = ring->size - ring->tail;
+	int remain_usable = ring->effective_size - ring->tail;
 	int bytes = num_dwords * sizeof(u32);
 	int total_bytes, wait_bytes;
 	bool need_wrap = false;
@@ -2403,37 +2285,33 @@ int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
 		wait_bytes = total_bytes;
 	}
 
-	if (wait_bytes > ringbuf->space) {
+	if (wait_bytes > ring->space) {
 		int ret = wait_for_space(req, wait_bytes);
 		if (unlikely(ret))
 			return ret;
-
-		intel_ring_update_space(ringbuf);
-		if (unlikely(ringbuf->space < wait_bytes))
-			return -EAGAIN;
 	}
 
 	if (unlikely(need_wrap)) {
-		GEM_BUG_ON(remain_actual > ringbuf->space);
-		GEM_BUG_ON(ringbuf->tail + remain_actual > ringbuf->size);
+		GEM_BUG_ON(remain_actual > ring->space);
+		GEM_BUG_ON(ring->tail + remain_actual > ring->size);
 
 		/* Fill the tail with MI_NOOP */
-		memset(ringbuf->virtual_start + ringbuf->tail,
-		       0, remain_actual);
-		ringbuf->tail = 0;
-		ringbuf->space -= remain_actual;
+		memset(ring->vaddr + ring->tail, 0, remain_actual);
+		ring->tail = 0;
+		ring->space -= remain_actual;
 	}
 
-	ringbuf->space -= bytes;
-	GEM_BUG_ON(ringbuf->space < 0);
+	ring->space -= bytes;
+	GEM_BUG_ON(ring->space < 0);
 	return 0;
 }
 
 /* Align the ring tail to a cacheline boundary */
 int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
 {
-	struct intel_engine_cs *engine = req->engine;
-	int num_dwords = (engine->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
+	struct intel_ring *ring = req->ring;
+	int num_dwords =
+		(ring->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
 	int ret;
 
 	if (num_dwords == 0)
@@ -2445,61 +2323,16 @@ int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
 		return ret;
 
 	while (num_dwords--)
-		intel_ring_emit(engine, MI_NOOP);
+		intel_ring_emit(ring, MI_NOOP);
 
-	intel_ring_advance(engine);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
-void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	/* Our semaphore implementation is strictly monotonic (i.e. we proceed
-	 * so long as the semaphore value in the register/page is greater
-	 * than the sync value), so whenever we reset the seqno,
-	 * so long as we reset the tracking semaphore value to 0, it will
-	 * always be before the next request's seqno. If we don't reset
-	 * the semaphore value, then when the seqno moves backwards all
-	 * future waits will complete instantly (causing rendering corruption).
-	 */
-	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
-		I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
-		I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
-		if (HAS_VEBOX(dev_priv))
-			I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
-	}
-	if (dev_priv->semaphore_obj) {
-		struct drm_i915_gem_object *obj = dev_priv->semaphore_obj;
-		struct page *page = i915_gem_object_get_dirty_page(obj, 0);
-		void *semaphores = kmap(page);
-		memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
-		       0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
-		kunmap(page);
-	}
-	memset(engine->semaphore.sync_seqno, 0,
-	       sizeof(engine->semaphore.sync_seqno));
-
-	intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
-	if (engine->irq_seqno_barrier)
-		engine->irq_seqno_barrier(engine);
-	engine->last_submitted_seqno = seqno;
-
-	engine->hangcheck.seqno = seqno;
-
-	/* After manually advancing the seqno, fake the interrupt in case
-	 * there are any waiters for that seqno.
-	 */
-	rcu_read_lock();
-	intel_engine_wakeup(engine);
-	rcu_read_unlock();
-}
-
-static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
-				     u32 value)
+static void gen6_bsd_submit_request(struct drm_i915_gem_request *request)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
+	struct drm_i915_private *dev_priv = request->i915;
 
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
@@ -2523,8 +2356,7 @@ static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
 		DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
 
 	/* Now that the ring is fully powered up, update the tail */
-	I915_WRITE_FW(RING_TAIL(engine->mmio_base), value);
-	POSTING_READ_FW(RING_TAIL(engine->mmio_base));
+	i9xx_submit_request(request);
 
 	/* Let the ring send IDLE messages to the GT again,
 	 * and so let it sleep to conserve power when idle.
@@ -2535,10 +2367,9 @@ static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
 
-static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
-			       u32 invalidate, u32 flush)
+static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	uint32_t cmd;
 	int ret;
 
@@ -2563,30 +2394,29 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
 	 * operation is complete. This bit is only valid when the
 	 * Post-Sync Operation field is a value of 1h or 3h."
 	 */
-	if (invalidate & I915_GEM_GPU_DOMAINS)
+	if (mode & EMIT_INVALIDATE)
 		cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
 
-	intel_ring_emit(engine, cmd);
-	intel_ring_emit(engine,
-			I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
+	intel_ring_emit(ring, cmd);
+	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
 	if (INTEL_GEN(req->i915) >= 8) {
-		intel_ring_emit(engine, 0); /* upper addr */
-		intel_ring_emit(engine, 0); /* value */
+		intel_ring_emit(ring, 0); /* upper addr */
+		intel_ring_emit(ring, 0); /* value */
 	} else  {
-		intel_ring_emit(engine, 0);
-		intel_ring_emit(engine, MI_NOOP);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, MI_NOOP);
 	}
-	intel_ring_advance(engine);
+	intel_ring_advance(ring);
 	return 0;
 }
 
 static int
-gen8_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
-			      u64 offset, u32 len,
-			      unsigned dispatch_flags)
+gen8_emit_bb_start(struct drm_i915_gem_request *req,
+		   u64 offset, u32 len,
+		   unsigned int dispatch_flags)
 {
-	struct intel_engine_cs *engine = req->engine;
-	bool ppgtt = USES_PPGTT(engine->dev) &&
+	struct intel_ring *ring = req->ring;
+	bool ppgtt = USES_PPGTT(req->i915) &&
 			!(dispatch_flags & I915_DISPATCH_SECURE);
 	int ret;
 
@@ -2595,71 +2425,70 @@ gen8_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
 		return ret;
 
 	/* FIXME(BDW): Address space and security selectors. */
-	intel_ring_emit(engine, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) |
+	intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) |
 			(dispatch_flags & I915_DISPATCH_RS ?
 			 MI_BATCH_RESOURCE_STREAMER : 0));
-	intel_ring_emit(engine, lower_32_bits(offset));
-	intel_ring_emit(engine, upper_32_bits(offset));
-	intel_ring_emit(engine, MI_NOOP);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, lower_32_bits(offset));
+	intel_ring_emit(ring, upper_32_bits(offset));
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static int
-hsw_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
-			     u64 offset, u32 len,
-			     unsigned dispatch_flags)
+hsw_emit_bb_start(struct drm_i915_gem_request *req,
+		  u64 offset, u32 len,
+		  unsigned int dispatch_flags)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	int ret;
 
 	ret = intel_ring_begin(req, 2);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine,
+	intel_ring_emit(ring,
 			MI_BATCH_BUFFER_START |
 			(dispatch_flags & I915_DISPATCH_SECURE ?
 			 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) |
 			(dispatch_flags & I915_DISPATCH_RS ?
 			 MI_BATCH_RESOURCE_STREAMER : 0));
 	/* bit0-7 is the length on GEN6+ */
-	intel_ring_emit(engine, offset);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, offset);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 static int
-gen6_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
-			      u64 offset, u32 len,
-			      unsigned dispatch_flags)
+gen6_emit_bb_start(struct drm_i915_gem_request *req,
+		   u64 offset, u32 len,
+		   unsigned int dispatch_flags)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	int ret;
 
 	ret = intel_ring_begin(req, 2);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(engine,
+	intel_ring_emit(ring,
 			MI_BATCH_BUFFER_START |
 			(dispatch_flags & I915_DISPATCH_SECURE ?
 			 0 : MI_BATCH_NON_SECURE_I965));
 	/* bit0-7 is the length on GEN6+ */
-	intel_ring_emit(engine, offset);
-	intel_ring_advance(engine);
+	intel_ring_emit(ring, offset);
+	intel_ring_advance(ring);
 
 	return 0;
 }
 
 /* Blitter support (SandyBridge+) */
 
-static int gen6_ring_flush(struct drm_i915_gem_request *req,
-			   u32 invalidate, u32 flush)
+static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 {
-	struct intel_engine_cs *engine = req->engine;
+	struct intel_ring *ring = req->ring;
 	uint32_t cmd;
 	int ret;
 
@@ -2684,19 +2513,19 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
 	 * operation is complete. This bit is only valid when the
 	 * Post-Sync Operation field is a value of 1h or 3h."
 	 */
-	if (invalidate & I915_GEM_DOMAIN_RENDER)
+	if (mode & EMIT_INVALIDATE)
 		cmd |= MI_INVALIDATE_TLB;
-	intel_ring_emit(engine, cmd);
-	intel_ring_emit(engine,
+	intel_ring_emit(ring, cmd);
+	intel_ring_emit(ring,
 			I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
 	if (INTEL_GEN(req->i915) >= 8) {
-		intel_ring_emit(engine, 0); /* upper addr */
-		intel_ring_emit(engine, 0); /* value */
+		intel_ring_emit(ring, 0); /* upper addr */
+		intel_ring_emit(ring, 0); /* value */
 	} else  {
-		intel_ring_emit(engine, 0);
-		intel_ring_emit(engine, MI_NOOP);
+		intel_ring_emit(ring, 0);
+		intel_ring_emit(ring, MI_NOOP);
 	}
-	intel_ring_advance(engine);
+	intel_ring_advance(ring);
 
 	return 0;
 }
@@ -2707,38 +2536,39 @@ static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
 	struct drm_i915_gem_object *obj;
 	int ret, i;
 
-	if (!i915_semaphore_is_enabled(dev_priv))
+	if (!i915.semaphores)
 		return;
 
-	if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore_obj) {
+	if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore) {
+		struct i915_vma *vma;
+
 		obj = i915_gem_object_create(&dev_priv->drm, 4096);
-		if (IS_ERR(obj)) {
-			DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
-			i915.semaphores = 0;
-		} else {
-			i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
-			ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
-			if (ret != 0) {
-				drm_gem_object_unreference(&obj->base);
-				DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
-				i915.semaphores = 0;
-			} else {
-				dev_priv->semaphore_obj = obj;
-			}
-		}
-	}
+		if (IS_ERR(obj))
+			goto err;
 
-	if (!i915_semaphore_is_enabled(dev_priv))
-		return;
+		vma = i915_vma_create(obj, &dev_priv->ggtt.base, NULL);
+		if (IS_ERR(vma))
+			goto err_obj;
+
+		ret = i915_gem_object_set_to_gtt_domain(obj, false);
+		if (ret)
+			goto err_obj;
+
+		ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
+		if (ret)
+			goto err_obj;
+
+		dev_priv->semaphore = vma;
+	}
 
 	if (INTEL_GEN(dev_priv) >= 8) {
-		u64 offset = i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj);
+		u32 offset = i915_ggtt_offset(dev_priv->semaphore);
 
-		engine->semaphore.sync_to = gen8_ring_sync;
+		engine->semaphore.sync_to = gen8_ring_sync_to;
 		engine->semaphore.signal = gen8_xcs_signal;
 
 		for (i = 0; i < I915_NUM_ENGINES; i++) {
-			u64 ring_offset;
+			u32 ring_offset;
 
 			if (i != engine->id)
 				ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i);
@@ -2748,7 +2578,7 @@ static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
 			engine->semaphore.signal_ggtt[i] = ring_offset;
 		}
 	} else if (INTEL_GEN(dev_priv) >= 6) {
-		engine->semaphore.sync_to = gen6_ring_sync;
+		engine->semaphore.sync_to = gen6_ring_sync_to;
 		engine->semaphore.signal = gen6_signal;
 
 		/*
@@ -2758,52 +2588,62 @@ static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
 		 * initialized as INVALID.  Gen8 will initialize the
 		 * sema between VCS2 and RCS later.
 		 */
-		for (i = 0; i < I915_NUM_ENGINES; i++) {
+		for (i = 0; i < GEN6_NUM_SEMAPHORES; i++) {
 			static const struct {
 				u32 wait_mbox;
 				i915_reg_t mbox_reg;
-			} sem_data[I915_NUM_ENGINES][I915_NUM_ENGINES] = {
-				[RCS] = {
-					[VCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RV,  .mbox_reg = GEN6_VRSYNC },
-					[BCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RB,  .mbox_reg = GEN6_BRSYNC },
-					[VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
+			} sem_data[GEN6_NUM_SEMAPHORES][GEN6_NUM_SEMAPHORES] = {
+				[RCS_HW] = {
+					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RV,  .mbox_reg = GEN6_VRSYNC },
+					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RB,  .mbox_reg = GEN6_BRSYNC },
+					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
 				},
-				[VCS] = {
-					[RCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VR,  .mbox_reg = GEN6_RVSYNC },
-					[BCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VB,  .mbox_reg = GEN6_BVSYNC },
-					[VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
+				[VCS_HW] = {
+					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VR,  .mbox_reg = GEN6_RVSYNC },
+					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VB,  .mbox_reg = GEN6_BVSYNC },
+					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
 				},
-				[BCS] = {
-					[RCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BR,  .mbox_reg = GEN6_RBSYNC },
-					[VCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BV,  .mbox_reg = GEN6_VBSYNC },
-					[VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
+				[BCS_HW] = {
+					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BR,  .mbox_reg = GEN6_RBSYNC },
+					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BV,  .mbox_reg = GEN6_VBSYNC },
+					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
 				},
-				[VECS] = {
-					[RCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
-					[VCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
-					[BCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
+				[VECS_HW] = {
+					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
+					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
+					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
 				},
 			};
 			u32 wait_mbox;
 			i915_reg_t mbox_reg;
 
-			if (i == engine->id || i == VCS2) {
+			if (i == engine->hw_id) {
 				wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
 				mbox_reg = GEN6_NOSYNC;
 			} else {
-				wait_mbox = sem_data[engine->id][i].wait_mbox;
-				mbox_reg = sem_data[engine->id][i].mbox_reg;
+				wait_mbox = sem_data[engine->hw_id][i].wait_mbox;
+				mbox_reg = sem_data[engine->hw_id][i].mbox_reg;
 			}
 
 			engine->semaphore.mbox.wait[i] = wait_mbox;
 			engine->semaphore.mbox.signal[i] = mbox_reg;
 		}
 	}
+
+	return;
+
+err_obj:
+	i915_gem_object_put(obj);
+err:
+	DRM_DEBUG_DRIVER("Failed to allocate space for semaphores, disabling\n");
+	i915.semaphores = 0;
 }
 
 static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
 				struct intel_engine_cs *engine)
 {
+	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << engine->irq_shift;
+
 	if (INTEL_GEN(dev_priv) >= 8) {
 		engine->irq_enable = gen8_irq_enable;
 		engine->irq_disable = gen8_irq_disable;
@@ -2828,83 +2668,76 @@ static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
 static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
 				      struct intel_engine_cs *engine)
 {
+	intel_ring_init_irq(dev_priv, engine);
+	intel_ring_init_semaphores(dev_priv, engine);
+
 	engine->init_hw = init_ring_common;
-	engine->write_tail = ring_write_tail;
+	engine->reset_hw = reset_ring_common;
 
-	engine->add_request = i9xx_add_request;
-	if (INTEL_GEN(dev_priv) >= 6)
-		engine->add_request = gen6_add_request;
+	engine->emit_request = i9xx_emit_request;
+	if (i915.semaphores)
+		engine->emit_request = gen6_sema_emit_request;
+	engine->submit_request = i9xx_submit_request;
 
 	if (INTEL_GEN(dev_priv) >= 8)
-		engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
+		engine->emit_bb_start = gen8_emit_bb_start;
 	else if (INTEL_GEN(dev_priv) >= 6)
-		engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+		engine->emit_bb_start = gen6_emit_bb_start;
 	else if (INTEL_GEN(dev_priv) >= 4)
-		engine->dispatch_execbuffer = i965_dispatch_execbuffer;
+		engine->emit_bb_start = i965_emit_bb_start;
 	else if (IS_I830(dev_priv) || IS_845G(dev_priv))
-		engine->dispatch_execbuffer = i830_dispatch_execbuffer;
+		engine->emit_bb_start = i830_emit_bb_start;
 	else
-		engine->dispatch_execbuffer = i915_dispatch_execbuffer;
-
-	intel_ring_init_irq(dev_priv, engine);
-	intel_ring_init_semaphores(dev_priv, engine);
+		engine->emit_bb_start = i915_emit_bb_start;
 }
 
-int intel_init_render_ring_buffer(struct drm_device *dev)
+int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_engine_cs *engine = &dev_priv->engine[RCS];
+	struct drm_i915_private *dev_priv = engine->i915;
 	int ret;
 
-	engine->name = "render ring";
-	engine->id = RCS;
-	engine->exec_id = I915_EXEC_RENDER;
-	engine->hw_id = 0;
-	engine->mmio_base = RENDER_RING_BASE;
-
 	intel_ring_default_vfuncs(dev_priv, engine);
 
-	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
 	if (HAS_L3_DPF(dev_priv))
 		engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 
 	if (INTEL_GEN(dev_priv) >= 8) {
 		engine->init_context = intel_rcs_ctx_init;
-		engine->add_request = gen8_render_add_request;
-		engine->flush = gen8_render_ring_flush;
-		if (i915_semaphore_is_enabled(dev_priv))
+		engine->emit_request = gen8_render_emit_request;
+		engine->emit_flush = gen8_render_ring_flush;
+		if (i915.semaphores)
 			engine->semaphore.signal = gen8_rcs_signal;
 	} else if (INTEL_GEN(dev_priv) >= 6) {
 		engine->init_context = intel_rcs_ctx_init;
-		engine->flush = gen7_render_ring_flush;
+		engine->emit_flush = gen7_render_ring_flush;
 		if (IS_GEN6(dev_priv))
-			engine->flush = gen6_render_ring_flush;
+			engine->emit_flush = gen6_render_ring_flush;
 	} else if (IS_GEN5(dev_priv)) {
-		engine->flush = gen4_render_ring_flush;
+		engine->emit_flush = gen4_render_ring_flush;
 	} else {
 		if (INTEL_GEN(dev_priv) < 4)
-			engine->flush = gen2_render_ring_flush;
+			engine->emit_flush = gen2_render_ring_flush;
 		else
-			engine->flush = gen4_render_ring_flush;
+			engine->emit_flush = gen4_render_ring_flush;
 		engine->irq_enable_mask = I915_USER_INTERRUPT;
 	}
 
 	if (IS_HASWELL(dev_priv))
-		engine->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
+		engine->emit_bb_start = hsw_emit_bb_start;
 
 	engine->init_hw = init_render_ring;
 	engine->cleanup = render_ring_cleanup;
 
-	ret = intel_init_ring_buffer(dev, engine);
+	ret = intel_init_ring_buffer(engine);
 	if (ret)
 		return ret;
 
 	if (INTEL_GEN(dev_priv) >= 6) {
-		ret = intel_init_pipe_control(engine, 4096);
+		ret = intel_engine_create_scratch(engine, 4096);
 		if (ret)
 			return ret;
 	} else if (HAS_BROKEN_CS_TLB(dev_priv)) {
-		ret = intel_init_pipe_control(engine, I830_WA_SIZE);
+		ret = intel_engine_create_scratch(engine, I830_WA_SIZE);
 		if (ret)
 			return ret;
 	}
@@ -2912,166 +2745,71 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
 	return 0;
 }
 
-int intel_init_bsd_ring_buffer(struct drm_device *dev)
+int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_engine_cs *engine = &dev_priv->engine[VCS];
-
-	engine->name = "bsd ring";
-	engine->id = VCS;
-	engine->exec_id = I915_EXEC_BSD;
-	engine->hw_id = 1;
+	struct drm_i915_private *dev_priv = engine->i915;
 
 	intel_ring_default_vfuncs(dev_priv, engine);
 
 	if (INTEL_GEN(dev_priv) >= 6) {
-		engine->mmio_base = GEN6_BSD_RING_BASE;
 		/* gen6 bsd needs a special wa for tail updates */
 		if (IS_GEN6(dev_priv))
-			engine->write_tail = gen6_bsd_ring_write_tail;
-		engine->flush = gen6_bsd_ring_flush;
-		if (INTEL_GEN(dev_priv) >= 8)
-			engine->irq_enable_mask =
-				GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
-		else
+			engine->submit_request = gen6_bsd_submit_request;
+		engine->emit_flush = gen6_bsd_ring_flush;
+		if (INTEL_GEN(dev_priv) < 8)
 			engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
 	} else {
 		engine->mmio_base = BSD_RING_BASE;
-		engine->flush = bsd_ring_flush;
+		engine->emit_flush = bsd_ring_flush;
 		if (IS_GEN5(dev_priv))
 			engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
 		else
 			engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
 	}
 
-	return intel_init_ring_buffer(dev, engine);
+	return intel_init_ring_buffer(engine);
 }
 
 /**
  * Initialize the second BSD ring (eg. Broadwell GT3, Skylake GT3)
  */
-int intel_init_bsd2_ring_buffer(struct drm_device *dev)
+int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_engine_cs *engine = &dev_priv->engine[VCS2];
-
-	engine->name = "bsd2 ring";
-	engine->id = VCS2;
-	engine->exec_id = I915_EXEC_BSD;
-	engine->hw_id = 4;
-	engine->mmio_base = GEN8_BSD2_RING_BASE;
+	struct drm_i915_private *dev_priv = engine->i915;
 
 	intel_ring_default_vfuncs(dev_priv, engine);
 
-	engine->flush = gen6_bsd_ring_flush;
-	engine->irq_enable_mask =
-			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
+	engine->emit_flush = gen6_bsd_ring_flush;
 
-	return intel_init_ring_buffer(dev, engine);
+	return intel_init_ring_buffer(engine);
 }
 
-int intel_init_blt_ring_buffer(struct drm_device *dev)
+int intel_init_blt_ring_buffer(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_engine_cs *engine = &dev_priv->engine[BCS];
-
-	engine->name = "blitter ring";
-	engine->id = BCS;
-	engine->exec_id = I915_EXEC_BLT;
-	engine->hw_id = 2;
-	engine->mmio_base = BLT_RING_BASE;
+	struct drm_i915_private *dev_priv = engine->i915;
 
 	intel_ring_default_vfuncs(dev_priv, engine);
 
-	engine->flush = gen6_ring_flush;
-	if (INTEL_GEN(dev_priv) >= 8)
-		engine->irq_enable_mask =
-			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
-	else
+	engine->emit_flush = gen6_ring_flush;
+	if (INTEL_GEN(dev_priv) < 8)
 		engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
 
-	return intel_init_ring_buffer(dev, engine);
+	return intel_init_ring_buffer(engine);
 }
 
-int intel_init_vebox_ring_buffer(struct drm_device *dev)
+int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_engine_cs *engine = &dev_priv->engine[VECS];
-
-	engine->name = "video enhancement ring";
-	engine->id = VECS;
-	engine->exec_id = I915_EXEC_VEBOX;
-	engine->hw_id = 3;
-	engine->mmio_base = VEBOX_RING_BASE;
+	struct drm_i915_private *dev_priv = engine->i915;
 
 	intel_ring_default_vfuncs(dev_priv, engine);
 
-	engine->flush = gen6_ring_flush;
+	engine->emit_flush = gen6_ring_flush;
 
-	if (INTEL_GEN(dev_priv) >= 8) {
-		engine->irq_enable_mask =
-			GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
-	} else {
+	if (INTEL_GEN(dev_priv) < 8) {
 		engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
 		engine->irq_enable = hsw_vebox_irq_enable;
 		engine->irq_disable = hsw_vebox_irq_disable;
 	}
 
-	return intel_init_ring_buffer(dev, engine);
-}
-
-int
-intel_ring_flush_all_caches(struct drm_i915_gem_request *req)
-{
-	struct intel_engine_cs *engine = req->engine;
-	int ret;
-
-	if (!engine->gpu_caches_dirty)
-		return 0;
-
-	ret = engine->flush(req, 0, I915_GEM_GPU_DOMAINS);
-	if (ret)
-		return ret;
-
-	trace_i915_gem_ring_flush(req, 0, I915_GEM_GPU_DOMAINS);
-
-	engine->gpu_caches_dirty = false;
-	return 0;
-}
-
-int
-intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req)
-{
-	struct intel_engine_cs *engine = req->engine;
-	uint32_t flush_domains;
-	int ret;
-
-	flush_domains = 0;
-	if (engine->gpu_caches_dirty)
-		flush_domains = I915_GEM_GPU_DOMAINS;
-
-	ret = engine->flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
-	if (ret)
-		return ret;
-
-	trace_i915_gem_ring_flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
-
-	engine->gpu_caches_dirty = false;
-	return 0;
-}
-
-void
-intel_stop_engine(struct intel_engine_cs *engine)
-{
-	int ret;
-
-	if (!intel_engine_initialized(engine))
-		return;
-
-	ret = intel_engine_idle(engine);
-	if (ret)
-		DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
-			  engine->name, ret);
-
-	stop_ring(engine);
+	return intel_init_ring_buffer(engine);
 }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 12cb7ed90014..ec0b4a0c605d 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -3,6 +3,7 @@
 
 #include <linux/hashtable.h>
 #include "i915_gem_batch_pool.h"
+#include "i915_gem_request.h"
 
 #define I915_CMD_HASH_ORDER 9
 
@@ -25,29 +26,29 @@
  */
 #define I915_RING_FREE_SPACE 64
 
-struct  intel_hw_status_page {
-	u32		*page_addr;
-	unsigned int	gfx_addr;
-	struct		drm_i915_gem_object *obj;
+struct intel_hw_status_page {
+	struct i915_vma *vma;
+	u32 *page_addr;
+	u32 ggtt_offset;
 };
 
-#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
-#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
+#define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base))
+#define I915_WRITE_TAIL(engine, val) I915_WRITE(RING_TAIL((engine)->mmio_base), val)
 
-#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base))
-#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
+#define I915_READ_START(engine) I915_READ(RING_START((engine)->mmio_base))
+#define I915_WRITE_START(engine, val) I915_WRITE(RING_START((engine)->mmio_base), val)
 
-#define I915_READ_HEAD(ring)  I915_READ(RING_HEAD((ring)->mmio_base))
-#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
+#define I915_READ_HEAD(engine)  I915_READ(RING_HEAD((engine)->mmio_base))
+#define I915_WRITE_HEAD(engine, val) I915_WRITE(RING_HEAD((engine)->mmio_base), val)
 
-#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base))
-#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
+#define I915_READ_CTL(engine) I915_READ(RING_CTL((engine)->mmio_base))
+#define I915_WRITE_CTL(engine, val) I915_WRITE(RING_CTL((engine)->mmio_base), val)
 
-#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
-#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
+#define I915_READ_IMR(engine) I915_READ(RING_IMR((engine)->mmio_base))
+#define I915_WRITE_IMR(engine, val) I915_WRITE(RING_IMR((engine)->mmio_base), val)
 
-#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
-#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val)
+#define I915_READ_MODE(engine) I915_READ(RING_MI_MODE((engine)->mmio_base))
+#define I915_WRITE_MODE(engine, val) I915_WRITE(RING_MI_MODE((engine)->mmio_base), val)
 
 /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
  * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
@@ -56,13 +57,13 @@ struct  intel_hw_status_page {
 #define GEN8_SEMAPHORE_OFFSET(__from, __to)			     \
 	(((__from) * I915_NUM_ENGINES  + (__to)) * gen8_semaphore_seqno_size)
 #define GEN8_SIGNAL_OFFSET(__ring, to)			     \
-	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
+	(dev_priv->semaphore->node.start + \
 	 GEN8_SEMAPHORE_OFFSET((__ring)->id, (to)))
 #define GEN8_WAIT_OFFSET(__ring, from)			     \
-	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
+	(dev_priv->semaphore->node.start + \
 	 GEN8_SEMAPHORE_OFFSET(from, (__ring)->id))
 
-enum intel_ring_hangcheck_action {
+enum intel_engine_hangcheck_action {
 	HANGCHECK_IDLE = 0,
 	HANGCHECK_WAIT,
 	HANGCHECK_ACTIVE,
@@ -72,23 +73,22 @@ enum intel_ring_hangcheck_action {
 
 #define HANGCHECK_SCORE_RING_HUNG 31
 
-struct intel_ring_hangcheck {
+struct intel_engine_hangcheck {
 	u64 acthd;
-	unsigned long user_interrupts;
 	u32 seqno;
 	int score;
-	enum intel_ring_hangcheck_action action;
+	enum intel_engine_hangcheck_action action;
 	int deadlock;
 	u32 instdone[I915_NUM_INSTDONE_REG];
 };
 
-struct intel_ringbuffer {
-	struct drm_i915_gem_object *obj;
-	void __iomem *virtual_start;
+struct intel_ring {
 	struct i915_vma *vma;
+	void *vaddr;
 
 	struct intel_engine_cs *engine;
-	struct list_head link;
+
+	struct list_head request_list;
 
 	u32 head;
 	u32 tail;
@@ -121,12 +121,12 @@ struct drm_i915_reg_table;
  *    an option for future use.
  *  size: size of the batch in DWORDS
  */
-struct  i915_ctx_workarounds {
+struct i915_ctx_workarounds {
 	struct i915_wa_ctx_bb {
 		u32 offset;
 		u32 size;
 	} indirect_ctx, per_ctx;
-	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
 };
 
 struct drm_i915_gem_request;
@@ -144,11 +144,18 @@ struct intel_engine_cs {
 #define I915_NUM_ENGINES 5
 #define _VCS(n) (VCS + (n))
 	unsigned int exec_id;
-	unsigned int hw_id;
-	unsigned int guc_id; /* XXX same as hw_id? */
+	enum intel_engine_hw_id {
+		RCS_HW = 0,
+		VCS_HW,
+		BCS_HW,
+		VECS_HW,
+		VCS2_HW
+	} hw_id;
+	enum intel_engine_hw_id guc_id; /* XXX same as hw_id? */
+	u64 fence_context;
 	u32		mmio_base;
-	struct intel_ringbuffer *buffer;
-	struct list_head buffers;
+	unsigned int irq_shift;
+	struct intel_ring *buffer;
 
 	/* Rather than have every client wait upon all user interrupts,
 	 * with the herd waking after every interrupt and each doing the
@@ -167,8 +174,7 @@ struct intel_engine_cs {
 	 * the overhead of waking that client is much preferred.
 	 */
 	struct intel_breadcrumbs {
-		struct task_struct *irq_seqno_bh; /* bh for user interrupts */
-		unsigned long irq_wakeups;
+		struct task_struct __rcu *irq_seqno_bh; /* bh for interrupts */
 		bool irq_posted;
 
 		spinlock_t lock; /* protects the lists of requests */
@@ -178,6 +184,9 @@ struct intel_engine_cs {
 		struct task_struct *signaler; /* used for fence signalling */
 		struct drm_i915_gem_request *first_signal;
 		struct timer_list fake_irq; /* used after a missed interrupt */
+		struct timer_list hangcheck; /* detect missed interrupts */
+
+		unsigned long timeout;
 
 		bool irq_enabled : 1;
 		bool rpm_wakelock : 1;
@@ -192,36 +201,48 @@ struct intel_engine_cs {
 
 	struct intel_hw_status_page status_page;
 	struct i915_ctx_workarounds wa_ctx;
+	struct i915_vma *scratch;
 
 	u32             irq_keep_mask; /* always keep these interrupts */
 	u32		irq_enable_mask; /* bitmask to enable ring interrupt */
-	void		(*irq_enable)(struct intel_engine_cs *ring);
-	void		(*irq_disable)(struct intel_engine_cs *ring);
+	void		(*irq_enable)(struct intel_engine_cs *engine);
+	void		(*irq_disable)(struct intel_engine_cs *engine);
 
-	int		(*init_hw)(struct intel_engine_cs *ring);
+	int		(*init_hw)(struct intel_engine_cs *engine);
+	void		(*reset_hw)(struct intel_engine_cs *engine,
+				    struct drm_i915_gem_request *req);
 
 	int		(*init_context)(struct drm_i915_gem_request *req);
 
-	void		(*write_tail)(struct intel_engine_cs *ring,
-				      u32 value);
-	int __must_check (*flush)(struct drm_i915_gem_request *req,
-				  u32	invalidate_domains,
-				  u32	flush_domains);
-	int		(*add_request)(struct drm_i915_gem_request *req);
+	int		(*emit_flush)(struct drm_i915_gem_request *request,
+				      u32 mode);
+#define EMIT_INVALIDATE	BIT(0)
+#define EMIT_FLUSH	BIT(1)
+#define EMIT_BARRIER	(EMIT_INVALIDATE | EMIT_FLUSH)
+	int		(*emit_bb_start)(struct drm_i915_gem_request *req,
+					 u64 offset, u32 length,
+					 unsigned int dispatch_flags);
+#define I915_DISPATCH_SECURE BIT(0)
+#define I915_DISPATCH_PINNED BIT(1)
+#define I915_DISPATCH_RS     BIT(2)
+	int		(*emit_request)(struct drm_i915_gem_request *req);
+
+	/* Pass the request to the hardware queue (e.g. directly into
+	 * the legacy ringbuffer or to the end of an execlist).
+	 *
+	 * This is called from an atomic context with irqs disabled; must
+	 * be irq safe.
+	 */
+	void		(*submit_request)(struct drm_i915_gem_request *req);
+
 	/* Some chipsets are not quite as coherent as advertised and need
 	 * an expensive kick to force a true read of the up-to-date seqno.
 	 * However, the up-to-date seqno is not always required and the last
 	 * seen value is good enough. Note that the seqno will always be
 	 * monotonic, even if not coherent.
 	 */
-	void		(*irq_seqno_barrier)(struct intel_engine_cs *ring);
-	int		(*dispatch_execbuffer)(struct drm_i915_gem_request *req,
-					       u64 offset, u32 length,
-					       unsigned dispatch_flags);
-#define I915_DISPATCH_SECURE 0x1
-#define I915_DISPATCH_PINNED 0x2
-#define I915_DISPATCH_RS     0x4
-	void		(*cleanup)(struct intel_engine_cs *ring);
+	void		(*irq_seqno_barrier)(struct intel_engine_cs *engine);
+	void		(*cleanup)(struct intel_engine_cs *engine);
 
 	/* GEN8 signal/wait table - never trust comments!
 	 *	  signal to	signal to    signal to   signal to      signal to
@@ -264,51 +285,36 @@ struct intel_engine_cs {
 		u32	sync_seqno[I915_NUM_ENGINES-1];
 
 		union {
+#define GEN6_SEMAPHORE_LAST	VECS_HW
+#define GEN6_NUM_SEMAPHORES	(GEN6_SEMAPHORE_LAST + 1)
+#define GEN6_SEMAPHORES_MASK	GENMASK(GEN6_SEMAPHORE_LAST, 0)
 			struct {
 				/* our mbox written by others */
-				u32		wait[I915_NUM_ENGINES];
+				u32		wait[GEN6_NUM_SEMAPHORES];
 				/* mboxes this ring signals to */
-				i915_reg_t	signal[I915_NUM_ENGINES];
+				i915_reg_t	signal[GEN6_NUM_SEMAPHORES];
 			} mbox;
 			u64		signal_ggtt[I915_NUM_ENGINES];
 		};
 
 		/* AKA wait() */
-		int	(*sync_to)(struct drm_i915_gem_request *to_req,
-				   struct intel_engine_cs *from,
-				   u32 seqno);
-		int	(*signal)(struct drm_i915_gem_request *signaller_req,
-				  /* num_dwords needed by caller */
-				  unsigned int num_dwords);
+		int	(*sync_to)(struct drm_i915_gem_request *req,
+				   struct drm_i915_gem_request *signal);
+		int	(*signal)(struct drm_i915_gem_request *req);
 	} semaphore;
 
 	/* Execlists */
 	struct tasklet_struct irq_tasklet;
 	spinlock_t execlist_lock; /* used inside tasklet, use spin_lock_bh */
+	struct execlist_port {
+		struct drm_i915_gem_request *request;
+		unsigned int count;
+	} execlist_port[2];
 	struct list_head execlist_queue;
 	unsigned int fw_domains;
-	unsigned int next_context_status_buffer;
-	unsigned int idle_lite_restore_wa;
 	bool disable_lite_restore_wa;
+	bool preempt_wa;
 	u32 ctx_desc_template;
-	int		(*emit_request)(struct drm_i915_gem_request *request);
-	int		(*emit_flush)(struct drm_i915_gem_request *request,
-				      u32 invalidate_domains,
-				      u32 flush_domains);
-	int		(*emit_bb_start)(struct drm_i915_gem_request *req,
-					 u64 offset, unsigned dispatch_flags);
-
-	/**
-	 * List of objects currently involved in rendering from the
-	 * ringbuffer.
-	 *
-	 * Includes buffers having the contents of their GPU caches
-	 * flushed, not necessarily primitives.  last_read_req
-	 * represents when the rendering involved will be completed.
-	 *
-	 * A reference is held on the buffer while on this list.
-	 */
-	struct list_head active_list;
 
 	/**
 	 * List of breadcrumbs associated with GPU requests currently
@@ -322,23 +328,24 @@ struct intel_engine_cs {
 	 * inspecting request list.
 	 */
 	u32 last_submitted_seqno;
+	u32 last_pending_seqno;
 
-	bool gpu_caches_dirty;
+	/* An RCU guarded pointer to the last request. No reference is
+	 * held to the request, users must carefully acquire a reference to
+	 * the request using i915_gem_active_get_rcu(), or hold the
+	 * struct_mutex.
+	 */
+	struct i915_gem_active last_request;
 
 	struct i915_gem_context *last_context;
 
-	struct intel_ring_hangcheck hangcheck;
-
-	struct {
-		struct drm_i915_gem_object *obj;
-		u32 gtt_offset;
-	} scratch;
+	struct intel_engine_hangcheck hangcheck;
 
 	bool needs_cmd_parser;
 
 	/*
 	 * Table of commands the command parser needs to know about
-	 * for this ring.
+	 * for this engine.
 	 */
 	DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
 
@@ -352,11 +359,11 @@ struct intel_engine_cs {
 	 * Returns the bitmask for the length field of the specified command.
 	 * Return 0 for an unrecognized/invalid command.
 	 *
-	 * If the command parser finds an entry for a command in the ring's
+	 * If the command parser finds an entry for a command in the engine's
 	 * cmd_tables, it gets the command's length based on the table entry.
-	 * If not, it calls this function to determine the per-ring length field
-	 * encoding for the command (i.e. certain opcode ranges use certain bits
-	 * to encode the command length in the header).
+	 * If not, it calls this function to determine the per-engine length
+	 * field encoding for the command (i.e. different opcode ranges use
+	 * certain bits to encode the command length in the header).
 	 */
 	u32 (*get_cmd_length_mask)(u32 cmd_header);
 };
@@ -374,8 +381,8 @@ intel_engine_flag(const struct intel_engine_cs *engine)
 }
 
 static inline u32
-intel_ring_sync_index(struct intel_engine_cs *engine,
-		      struct intel_engine_cs *other)
+intel_engine_sync_index(struct intel_engine_cs *engine,
+			struct intel_engine_cs *other)
 {
 	int idx;
 
@@ -437,55 +444,76 @@ intel_write_status_page(struct intel_engine_cs *engine,
 #define I915_GEM_HWS_SCRATCH_INDEX	0x40
 #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
 
-struct intel_ringbuffer *
-intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size);
-int intel_pin_and_map_ringbuffer_obj(struct drm_i915_private *dev_priv,
-				     struct intel_ringbuffer *ringbuf);
-void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
-void intel_ringbuffer_free(struct intel_ringbuffer *ring);
+struct intel_ring *
+intel_engine_create_ring(struct intel_engine_cs *engine, int size);
+int intel_ring_pin(struct intel_ring *ring);
+void intel_ring_unpin(struct intel_ring *ring);
+void intel_ring_free(struct intel_ring *ring);
+
+void intel_engine_stop(struct intel_engine_cs *engine);
+void intel_engine_cleanup(struct intel_engine_cs *engine);
 
-void intel_stop_engine(struct intel_engine_cs *engine);
-void intel_cleanup_engine(struct intel_engine_cs *engine);
+void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
 
 int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request);
 
 int __must_check intel_ring_begin(struct drm_i915_gem_request *req, int n);
 int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
-static inline void intel_ring_emit(struct intel_engine_cs *engine,
-				   u32 data)
+
+static inline void intel_ring_emit(struct intel_ring *ring, u32 data)
 {
-	struct intel_ringbuffer *ringbuf = engine->buffer;
-	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
-	ringbuf->tail += 4;
+	*(uint32_t *)(ring->vaddr + ring->tail) = data;
+	ring->tail += 4;
 }
-static inline void intel_ring_emit_reg(struct intel_engine_cs *engine,
-				       i915_reg_t reg)
+
+static inline void intel_ring_emit_reg(struct intel_ring *ring, i915_reg_t reg)
 {
-	intel_ring_emit(engine, i915_mmio_reg_offset(reg));
+	intel_ring_emit(ring, i915_mmio_reg_offset(reg));
 }
-static inline void intel_ring_advance(struct intel_engine_cs *engine)
+
+static inline void intel_ring_advance(struct intel_ring *ring)
 {
-	struct intel_ringbuffer *ringbuf = engine->buffer;
-	ringbuf->tail &= ringbuf->size - 1;
+	/* Dummy function.
+	 *
+	 * This serves as a placeholder in the code so that the reader
+	 * can compare against the preceding intel_ring_begin() and
+	 * check that the number of dwords emitted matches the space
+	 * reserved for the command packet (i.e. the value passed to
+	 * intel_ring_begin()).
+	 */
+}
+
+static inline u32 intel_ring_offset(struct intel_ring *ring, u32 value)
+{
+	/* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
+	return value & (ring->size - 1);
 }
+
 int __intel_ring_space(int head, int tail, int size);
-void intel_ring_update_space(struct intel_ringbuffer *ringbuf);
+void intel_ring_update_space(struct intel_ring *ring);
 
-int __must_check intel_engine_idle(struct intel_engine_cs *engine);
-void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno);
-int intel_ring_flush_all_caches(struct drm_i915_gem_request *req);
-int intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req);
+void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno);
 
-int intel_init_pipe_control(struct intel_engine_cs *engine, int size);
-void intel_fini_pipe_control(struct intel_engine_cs *engine);
+void intel_engine_setup_common(struct intel_engine_cs *engine);
+int intel_engine_init_common(struct intel_engine_cs *engine);
+int intel_engine_create_scratch(struct intel_engine_cs *engine, int size);
+void intel_engine_cleanup_common(struct intel_engine_cs *engine);
 
-int intel_init_render_ring_buffer(struct drm_device *dev);
-int intel_init_bsd_ring_buffer(struct drm_device *dev);
-int intel_init_bsd2_ring_buffer(struct drm_device *dev);
-int intel_init_blt_ring_buffer(struct drm_device *dev);
-int intel_init_vebox_ring_buffer(struct drm_device *dev);
+static inline int intel_engine_idle(struct intel_engine_cs *engine,
+				    unsigned int flags)
+{
+	/* Wait upon the last request to be completed */
+	return i915_gem_active_wait_unlocked(&engine->last_request,
+					     flags, NULL, NULL);
+}
+
+int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
+int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine);
+int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine);
+int intel_init_blt_ring_buffer(struct intel_engine_cs *engine);
+int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine);
 
-u64 intel_ring_get_active_head(struct intel_engine_cs *engine);
+u64 intel_engine_get_active_head(struct intel_engine_cs *engine);
 static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
 {
 	return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
@@ -493,11 +521,6 @@ static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
 
 int init_workarounds_ring(struct intel_engine_cs *engine);
 
-static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf)
-{
-	return ringbuf->tail;
-}
-
 /*
  * Arbitrary size for largest possible 'add request' sequence. The code paths
  * are complex and variable. Empirical measurement shows that the worst case
@@ -509,21 +532,10 @@ static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf)
 
 static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
 {
-	return engine->status_page.gfx_addr + I915_GEM_HWS_INDEX_ADDR;
+	return engine->status_page.ggtt_offset + I915_GEM_HWS_INDEX_ADDR;
 }
 
 /* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */
-struct intel_wait {
-	struct rb_node node;
-	struct task_struct *tsk;
-	u32 seqno;
-};
-
-struct intel_signal_node {
-	struct rb_node node;
-	struct intel_wait wait;
-};
-
 int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
 
 static inline void intel_wait_init(struct intel_wait *wait, u32 seqno)
@@ -543,31 +555,43 @@ void intel_engine_remove_wait(struct intel_engine_cs *engine,
 			      struct intel_wait *wait);
 void intel_engine_enable_signaling(struct drm_i915_gem_request *request);
 
-static inline bool intel_engine_has_waiter(struct intel_engine_cs *engine)
+static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine)
 {
-	return READ_ONCE(engine->breadcrumbs.irq_seqno_bh);
+	return rcu_access_pointer(engine->breadcrumbs.irq_seqno_bh);
 }
 
-static inline bool intel_engine_wakeup(struct intel_engine_cs *engine)
+static inline bool intel_engine_wakeup(const struct intel_engine_cs *engine)
 {
 	bool wakeup = false;
-	struct task_struct *tsk = READ_ONCE(engine->breadcrumbs.irq_seqno_bh);
+
 	/* Note that for this not to dangerously chase a dangling pointer,
-	 * the caller is responsible for ensure that the task remain valid for
-	 * wake_up_process() i.e. that the RCU grace period cannot expire.
+	 * we must hold the rcu_read_lock here.
 	 *
 	 * Also note that tsk is likely to be in !TASK_RUNNING state so an
 	 * early test for tsk->state != TASK_RUNNING before wake_up_process()
 	 * is unlikely to be beneficial.
 	 */
-	if (tsk)
-		wakeup = wake_up_process(tsk);
+	if (intel_engine_has_waiter(engine)) {
+		struct task_struct *tsk;
+
+		rcu_read_lock();
+		tsk = rcu_dereference(engine->breadcrumbs.irq_seqno_bh);
+		if (tsk)
+			wakeup = wake_up_process(tsk);
+		rcu_read_unlock();
+	}
+
 	return wakeup;
 }
 
-void intel_engine_enable_fake_irq(struct intel_engine_cs *engine);
+void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
 unsigned int intel_kick_waiters(struct drm_i915_private *i915);
 unsigned int intel_kick_signalers(struct drm_i915_private *i915);
 
+static inline bool intel_engine_is_active(struct intel_engine_cs *engine)
+{
+	return i915_gem_active_isset(&engine->last_request);
+}
+
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 1c603bbe5784..6c11168facd6 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -287,6 +287,7 @@ void intel_display_set_init_power(struct drm_i915_private *dev_priv,
  */
 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 {
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct drm_device *dev = &dev_priv->drm;
 
 	/*
@@ -299,9 +300,9 @@ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 	 * sure vgacon can keep working normally without triggering interrupts
 	 * and error messages.
 	 */
-	vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
+	vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 	outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
-	vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
+	vga_put(pdev, VGA_RSRC_LEGACY_IO);
 
 	if (IS_BROADWELL(dev))
 		gen8_irq_power_well_post_enable(dev_priv,
@@ -318,7 +319,7 @@ static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv)
 static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
 				       struct i915_power_well *power_well)
 {
-	struct drm_device *dev = &dev_priv->drm;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 
 	/*
 	 * After we re-enable the power well, if we touch VGA register 0x3d5
@@ -331,9 +332,9 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
 	 * and error messages.
 	 */
 	if (power_well->data == SKL_DISP_PW_2) {
-		vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
+		vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 		outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
-		vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
+		vga_put(pdev, VGA_RSRC_LEGACY_IO);
 
 		gen8_irq_power_well_post_enable(dev_priv,
 						1 << PIPE_C | 1 << PIPE_B);
@@ -592,6 +593,8 @@ void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 	DRM_DEBUG_KMS("Disabling DC9\n");
 
 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
+
+	intel_pps_unlock_regs_wa(dev_priv);
 }
 
 static void assert_csr_loaded(struct drm_i915_private *dev_priv)
@@ -854,7 +857,7 @@ static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 					   struct i915_power_well *power_well)
 {
 	enum skl_disp_power_wells power_well_id = power_well->data;
-	struct i915_power_well *cmn_a_well;
+	struct i915_power_well *cmn_a_well = NULL;
 
 	if (power_well_id == BXT_DPIO_CMN_BC) {
 		/*
@@ -867,7 +870,7 @@ static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 
 	bxt_ddi_phy_init(dev_priv, bxt_power_well_to_phy(power_well));
 
-	if (power_well_id == BXT_DPIO_CMN_BC)
+	if (cmn_a_well)
 		intel_power_well_put(dev_priv, cmn_a_well);
 }
 
@@ -1121,6 +1124,8 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 	}
 
 	i915_redisable_vga_power_on(&dev_priv->drm);
+
+	intel_pps_unlock_regs_wa(dev_priv);
 }
 
 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
@@ -2284,7 +2289,7 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
  */
 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
 {
-	struct device *device = &dev_priv->drm.pdev->dev;
+	struct device *kdev = &dev_priv->drm.pdev->dev;
 
 	/*
 	 * The i915.ko module is still not prepared to be loaded when
@@ -2306,7 +2311,7 @@ void intel_power_domains_fini(struct drm_i915_private *dev_priv)
 	 * the platform doesn't support runtime PM.
 	 */
 	if (!HAS_RUNTIME_PM(dev_priv))
-		pm_runtime_put(device);
+		pm_runtime_put(kdev);
 }
 
 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
@@ -2647,10 +2652,10 @@ void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
  */
 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct device *device = &dev->pdev->dev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
+	struct device *kdev = &pdev->dev;
 
-	pm_runtime_get_sync(device);
+	pm_runtime_get_sync(kdev);
 
 	atomic_inc(&dev_priv->pm.wakeref_count);
 	assert_rpm_wakelock_held(dev_priv);
@@ -2668,11 +2673,11 @@ void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
  */
 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct device *device = &dev->pdev->dev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
+	struct device *kdev = &pdev->dev;
 
 	if (IS_ENABLED(CONFIG_PM)) {
-		int ret = pm_runtime_get_if_in_use(device);
+		int ret = pm_runtime_get_if_in_use(kdev);
 
 		/*
 		 * In cases runtime PM is disabled by the RPM core and we get
@@ -2710,11 +2715,11 @@ bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
  */
 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct device *device = &dev->pdev->dev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
+	struct device *kdev = &pdev->dev;
 
 	assert_rpm_wakelock_held(dev_priv);
-	pm_runtime_get_noresume(device);
+	pm_runtime_get_noresume(kdev);
 
 	atomic_inc(&dev_priv->pm.wakeref_count);
 }
@@ -2729,15 +2734,15 @@ void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
  */
 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct device *device = &dev->pdev->dev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
+	struct device *kdev = &pdev->dev;
 
 	assert_rpm_wakelock_held(dev_priv);
 	if (atomic_dec_and_test(&dev_priv->pm.wakeref_count))
 		atomic_inc(&dev_priv->pm.atomic_seq);
 
-	pm_runtime_mark_last_busy(device);
-	pm_runtime_put_autosuspend(device);
+	pm_runtime_mark_last_busy(kdev);
+	pm_runtime_put_autosuspend(kdev);
 }
 
 /**
@@ -2752,11 +2757,12 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
  */
 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
 {
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct drm_device *dev = &dev_priv->drm;
-	struct device *device = &dev->pdev->dev;
+	struct device *kdev = &pdev->dev;
 
-	pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */
-	pm_runtime_mark_last_busy(device);
+	pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
+	pm_runtime_mark_last_busy(kdev);
 
 	/*
 	 * Take a permanent reference to disable the RPM functionality and drop
@@ -2765,10 +2771,10 @@ void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
 	 * platforms without RPM support.
 	 */
 	if (!HAS_RUNTIME_PM(dev)) {
-		pm_runtime_dont_use_autosuspend(device);
-		pm_runtime_get_sync(device);
+		pm_runtime_dont_use_autosuspend(kdev);
+		pm_runtime_get_sync(kdev);
 	} else {
-		pm_runtime_use_autosuspend(device);
+		pm_runtime_use_autosuspend(kdev);
 	}
 
 	/*
@@ -2776,6 +2782,5 @@ void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
 	 * We drop that here and will reacquire it during unloading in
 	 * intel_power_domains_fini().
 	 */
-	pm_runtime_put_autosuspend(device);
+	pm_runtime_put_autosuspend(kdev);
 }
-
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index e378f35365a2..c551024d4871 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -1003,24 +1003,22 @@ static bool intel_sdvo_write_infoframe(struct intel_sdvo *intel_sdvo,
 }
 
 static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo,
-					 const struct drm_display_mode *adjusted_mode)
+					 struct intel_crtc_state *pipe_config)
 {
 	uint8_t sdvo_data[HDMI_INFOFRAME_SIZE(AVI)];
-	struct drm_crtc *crtc = intel_sdvo->base.base.crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	union hdmi_infoframe frame;
 	int ret;
 	ssize_t len;
 
 	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi,
-						       adjusted_mode);
+						       &pipe_config->base.adjusted_mode);
 	if (ret < 0) {
 		DRM_ERROR("couldn't fill AVI infoframe\n");
 		return false;
 	}
 
 	if (intel_sdvo->rgb_quant_range_selectable) {
-		if (intel_crtc->config->limited_color_range)
+		if (pipe_config->limited_color_range)
 			frame.avi.quantization_range =
 				HDMI_QUANTIZATION_RANGE_LIMITED;
 		else
@@ -1125,7 +1123,8 @@ static void i9xx_adjust_sdvo_tv_clock(struct intel_crtc_state *pipe_config)
 }
 
 static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
-				      struct intel_crtc_state *pipe_config)
+				      struct intel_crtc_state *pipe_config,
+				      struct drm_connector_state *conn_state)
 {
 	struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
 	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
@@ -1192,22 +1191,21 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
 	return true;
 }
 
-static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder)
+static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
+				  struct intel_crtc_state *crtc_state,
+				  struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = intel_encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *crtc = to_intel_crtc(intel_encoder->base.crtc);
-	const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
-	struct drm_display_mode *mode = &crtc->config->base.mode;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
+	struct drm_display_mode *mode = &crtc_state->base.mode;
 	struct intel_sdvo *intel_sdvo = to_sdvo(intel_encoder);
 	u32 sdvox;
 	struct intel_sdvo_in_out_map in_out;
 	struct intel_sdvo_dtd input_dtd, output_dtd;
 	int rate;
 
-	if (!mode)
-		return;
-
 	/* First, set the input mapping for the first input to our controlled
 	 * output. This is only correct if we're a single-input device, in
 	 * which case the first input is the output from the appropriate SDVO
@@ -1240,11 +1238,11 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder)
 	if (!intel_sdvo_set_target_input(intel_sdvo))
 		return;
 
-	if (crtc->config->has_hdmi_sink) {
+	if (crtc_state->has_hdmi_sink) {
 		intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI);
 		intel_sdvo_set_colorimetry(intel_sdvo,
 					   SDVO_COLORIMETRY_RGB256);
-		intel_sdvo_set_avi_infoframe(intel_sdvo, adjusted_mode);
+		intel_sdvo_set_avi_infoframe(intel_sdvo, crtc_state);
 	} else
 		intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_DVI);
 
@@ -1260,7 +1258,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder)
 		DRM_INFO("Setting input timings on %s failed\n",
 			 SDVO_NAME(intel_sdvo));
 
-	switch (crtc->config->pixel_multiplier) {
+	switch (crtc_state->pixel_multiplier) {
 	default:
 		WARN(1, "unknown pixel multiplier specified\n");
 	case 1: rate = SDVO_CLOCK_RATE_MULT_1X; break;
@@ -1275,7 +1273,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder)
 		/* The real mode polarity is set by the SDVO commands, using
 		 * struct intel_sdvo_dtd. */
 		sdvox = SDVO_VSYNC_ACTIVE_HIGH | SDVO_HSYNC_ACTIVE_HIGH;
-		if (!HAS_PCH_SPLIT(dev) && crtc->config->limited_color_range)
+		if (!HAS_PCH_SPLIT(dev) && crtc_state->limited_color_range)
 			sdvox |= HDMI_COLOR_RANGE_16_235;
 		if (INTEL_INFO(dev)->gen < 5)
 			sdvox |= SDVO_BORDER_ENABLE;
@@ -1301,7 +1299,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder)
 	} else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
 		/* done in crtc_mode_set as it lives inside the dpll register */
 	} else {
-		sdvox |= (crtc->config->pixel_multiplier - 1)
+		sdvox |= (crtc_state->pixel_multiplier - 1)
 			<< SDVO_PORT_MULTIPLY_SHIFT;
 	}
 
@@ -1434,7 +1432,9 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
 	     pipe_config->pixel_multiplier, encoder_pixel_multiplier);
 }
 
-static void intel_disable_sdvo(struct intel_encoder *encoder)
+static void intel_disable_sdvo(struct intel_encoder *encoder,
+			       struct intel_crtc_state *old_crtc_state,
+			       struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
@@ -1477,16 +1477,22 @@ static void intel_disable_sdvo(struct intel_encoder *encoder)
 	}
 }
 
-static void pch_disable_sdvo(struct intel_encoder *encoder)
+static void pch_disable_sdvo(struct intel_encoder *encoder,
+			     struct intel_crtc_state *old_crtc_state,
+			     struct drm_connector_state *old_conn_state)
 {
 }
 
-static void pch_post_disable_sdvo(struct intel_encoder *encoder)
+static void pch_post_disable_sdvo(struct intel_encoder *encoder,
+				  struct intel_crtc_state *old_crtc_state,
+				  struct drm_connector_state *old_conn_state)
 {
-	intel_disable_sdvo(encoder);
+	intel_disable_sdvo(encoder, old_crtc_state, old_conn_state);
 }
 
-static void intel_enable_sdvo(struct intel_encoder *encoder)
+static void intel_enable_sdvo(struct intel_encoder *encoder,
+			      struct intel_crtc_state *pipe_config,
+			      struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -2930,10 +2936,12 @@ static bool
 intel_sdvo_init_ddc_proxy(struct intel_sdvo *sdvo,
 			  struct drm_device *dev)
 {
+	struct pci_dev *pdev = dev->pdev;
+
 	sdvo->ddc.owner = THIS_MODULE;
 	sdvo->ddc.class = I2C_CLASS_DDC;
 	snprintf(sdvo->ddc.name, I2C_NAME_SIZE, "SDVO DDC proxy");
-	sdvo->ddc.dev.parent = &dev->pdev->dev;
+	sdvo->ddc.dev.parent = &pdev->dev;
 	sdvo->ddc.algo_data = sdvo;
 	sdvo->ddc.algo = &intel_sdvo_ddc_proxy;
 
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 7c08e4f29032..73a521fdf1bd 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -36,6 +36,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_plane_helper.h>
 #include "intel_drv.h"
+#include "intel_frontbuffer.h"
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 
@@ -202,23 +203,24 @@ skl_update_plane(struct drm_plane *drm_plane,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_plane *intel_plane = to_intel_plane(drm_plane);
 	struct drm_framebuffer *fb = plane_state->base.fb;
-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
+	const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
+	struct drm_crtc *crtc = crtc_state->base.crtc;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	const int pipe = intel_plane->pipe;
 	const int plane = intel_plane->plane + 1;
-	u32 plane_ctl, stride_div, stride;
+	u32 plane_ctl;
 	const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
-	u32 surf_addr;
-	u32 tile_height, plane_offset, plane_size;
+	u32 surf_addr = plane_state->main.offset;
 	unsigned int rotation = plane_state->base.rotation;
-	int x_offset, y_offset;
-	int crtc_x = plane_state->dst.x1;
-	int crtc_y = plane_state->dst.y1;
-	uint32_t crtc_w = drm_rect_width(&plane_state->dst);
-	uint32_t crtc_h = drm_rect_height(&plane_state->dst);
-	uint32_t x = plane_state->src.x1 >> 16;
-	uint32_t y = plane_state->src.y1 >> 16;
-	uint32_t src_w = drm_rect_width(&plane_state->src) >> 16;
-	uint32_t src_h = drm_rect_height(&plane_state->src) >> 16;
+	u32 stride = skl_plane_stride(fb, 0, rotation);
+	int crtc_x = plane_state->base.dst.x1;
+	int crtc_y = plane_state->base.dst.y1;
+	uint32_t crtc_w = drm_rect_width(&plane_state->base.dst);
+	uint32_t crtc_h = drm_rect_height(&plane_state->base.dst);
+	uint32_t x = plane_state->main.x;
+	uint32_t y = plane_state->main.y;
+	uint32_t src_w = drm_rect_width(&plane_state->base.src) >> 16;
+	uint32_t src_h = drm_rect_height(&plane_state->base.src) >> 16;
 
 	plane_ctl = PLANE_CTL_ENABLE |
 		PLANE_CTL_PIPE_GAMMA_ENABLE |
@@ -229,14 +231,8 @@ skl_update_plane(struct drm_plane *drm_plane,
 
 	plane_ctl |= skl_plane_ctl_rotation(rotation);
 
-	stride_div = intel_fb_stride_alignment(dev_priv, fb->modifier[0],
-					       fb->pixel_format);
-
-	/* Sizes are 0 based */
-	src_w--;
-	src_h--;
-	crtc_w--;
-	crtc_h--;
+	if (wm->dirty_pipes & drm_crtc_mask(crtc))
+		skl_write_plane_wm(intel_crtc, wm, plane);
 
 	if (key->flags) {
 		I915_WRITE(PLANE_KEYVAL(pipe, plane), key->min_value);
@@ -249,28 +245,15 @@ skl_update_plane(struct drm_plane *drm_plane,
 	else if (key->flags & I915_SET_COLORKEY_SOURCE)
 		plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
 
-	surf_addr = intel_plane_obj_offset(intel_plane, obj, 0);
-
-	if (intel_rotation_90_or_270(rotation)) {
-		int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
-
-		/* stride: Surface height in tiles */
-		tile_height = intel_tile_height(dev_priv, fb->modifier[0], cpp);
-		stride = DIV_ROUND_UP(fb->height, tile_height);
-		plane_size = (src_w << 16) | src_h;
-		x_offset = stride * tile_height - y - (src_h + 1);
-		y_offset = x;
-	} else {
-		stride = fb->pitches[0] / stride_div;
-		plane_size = (src_h << 16) | src_w;
-		x_offset = x;
-		y_offset = y;
-	}
-	plane_offset = y_offset << 16 | x_offset;
+	/* Sizes are 0 based */
+	src_w--;
+	src_h--;
+	crtc_w--;
+	crtc_h--;
 
-	I915_WRITE(PLANE_OFFSET(pipe, plane), plane_offset);
+	I915_WRITE(PLANE_OFFSET(pipe, plane), (y << 16) | x);
 	I915_WRITE(PLANE_STRIDE(pipe, plane), stride);
-	I915_WRITE(PLANE_SIZE(pipe, plane), plane_size);
+	I915_WRITE(PLANE_SIZE(pipe, plane), (src_h << 16) | src_w);
 
 	/* program plane scaler */
 	if (plane_state->scaler_id >= 0) {
@@ -295,7 +278,8 @@ skl_update_plane(struct drm_plane *drm_plane,
 	}
 
 	I915_WRITE(PLANE_CTL(pipe, plane), plane_ctl);
-	I915_WRITE(PLANE_SURF(pipe, plane), surf_addr);
+	I915_WRITE(PLANE_SURF(pipe, plane),
+		   intel_fb_gtt_offset(fb, rotation) + surf_addr);
 	POSTING_READ(PLANE_SURF(pipe, plane));
 }
 
@@ -308,6 +292,14 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
 	const int pipe = intel_plane->pipe;
 	const int plane = intel_plane->plane + 1;
 
+	/*
+	 * We only populate skl_results on watermark updates, and if the
+	 * plane's visiblity isn't actually changing neither is its watermarks.
+	 */
+	if (!dplane->state->visible)
+		skl_write_plane_wm(to_intel_crtc(crtc),
+				   &dev_priv->wm.skl_results, plane);
+
 	I915_WRITE(PLANE_CTL(pipe, plane), 0);
 
 	I915_WRITE(PLANE_SURF(pipe, plane), 0);
@@ -362,22 +354,20 @@ vlv_update_plane(struct drm_plane *dplane,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_plane *intel_plane = to_intel_plane(dplane);
 	struct drm_framebuffer *fb = plane_state->base.fb;
-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	int pipe = intel_plane->pipe;
 	int plane = intel_plane->plane;
 	u32 sprctl;
 	u32 sprsurf_offset, linear_offset;
 	unsigned int rotation = dplane->state->rotation;
-	int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 	const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
-	int crtc_x = plane_state->dst.x1;
-	int crtc_y = plane_state->dst.y1;
-	uint32_t crtc_w = drm_rect_width(&plane_state->dst);
-	uint32_t crtc_h = drm_rect_height(&plane_state->dst);
-	uint32_t x = plane_state->src.x1 >> 16;
-	uint32_t y = plane_state->src.y1 >> 16;
-	uint32_t src_w = drm_rect_width(&plane_state->src) >> 16;
-	uint32_t src_h = drm_rect_height(&plane_state->src) >> 16;
+	int crtc_x = plane_state->base.dst.x1;
+	int crtc_y = plane_state->base.dst.y1;
+	uint32_t crtc_w = drm_rect_width(&plane_state->base.dst);
+	uint32_t crtc_h = drm_rect_height(&plane_state->base.dst);
+	uint32_t x = plane_state->base.src.x1 >> 16;
+	uint32_t y = plane_state->base.src.y1 >> 16;
+	uint32_t src_w = drm_rect_width(&plane_state->base.src) >> 16;
+	uint32_t src_h = drm_rect_height(&plane_state->base.src) >> 16;
 
 	sprctl = SP_ENABLE;
 
@@ -430,7 +420,7 @@ vlv_update_plane(struct drm_plane *dplane,
 	 */
 	sprctl |= SP_GAMMA_ENABLE;
 
-	if (obj->tiling_mode != I915_TILING_NONE)
+	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
 		sprctl |= SP_TILED;
 
 	/* Sizes are 0 based */
@@ -439,19 +429,18 @@ vlv_update_plane(struct drm_plane *dplane,
 	crtc_w--;
 	crtc_h--;
 
-	linear_offset = y * fb->pitches[0] + x * cpp;
-	sprsurf_offset = intel_compute_tile_offset(&x, &y, fb, 0,
-						   fb->pitches[0], rotation);
-	linear_offset -= sprsurf_offset;
+	intel_add_fb_offsets(&x, &y, plane_state, 0);
+	sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	if (rotation == BIT(DRM_ROTATE_180)) {
+	if (rotation == DRM_ROTATE_180) {
 		sprctl |= SP_ROTATE_180;
 
 		x += src_w;
 		y += src_h;
-		linear_offset += src_h * fb->pitches[0] + src_w * cpp;
 	}
 
+	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
+
 	if (key->flags) {
 		I915_WRITE(SPKEYMINVAL(pipe, plane), key->min_value);
 		I915_WRITE(SPKEYMAXVAL(pipe, plane), key->max_value);
@@ -467,7 +456,7 @@ vlv_update_plane(struct drm_plane *dplane,
 	I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]);
 	I915_WRITE(SPPOS(pipe, plane), (crtc_y << 16) | crtc_x);
 
-	if (obj->tiling_mode != I915_TILING_NONE)
+	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
 		I915_WRITE(SPTILEOFF(pipe, plane), (y << 16) | x);
 	else
 		I915_WRITE(SPLINOFF(pipe, plane), linear_offset);
@@ -476,8 +465,8 @@ vlv_update_plane(struct drm_plane *dplane,
 
 	I915_WRITE(SPSIZE(pipe, plane), (crtc_h << 16) | crtc_w);
 	I915_WRITE(SPCNTR(pipe, plane), sprctl);
-	I915_WRITE(SPSURF(pipe, plane), i915_gem_obj_ggtt_offset(obj) +
-		   sprsurf_offset);
+	I915_WRITE(SPSURF(pipe, plane),
+		   intel_fb_gtt_offset(fb, rotation) + sprsurf_offset);
 	POSTING_READ(SPSURF(pipe, plane));
 }
 
@@ -505,21 +494,19 @@ ivb_update_plane(struct drm_plane *plane,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_plane *intel_plane = to_intel_plane(plane);
 	struct drm_framebuffer *fb = plane_state->base.fb;
-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	enum pipe pipe = intel_plane->pipe;
 	u32 sprctl, sprscale = 0;
 	u32 sprsurf_offset, linear_offset;
 	unsigned int rotation = plane_state->base.rotation;
-	int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 	const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
-	int crtc_x = plane_state->dst.x1;
-	int crtc_y = plane_state->dst.y1;
-	uint32_t crtc_w = drm_rect_width(&plane_state->dst);
-	uint32_t crtc_h = drm_rect_height(&plane_state->dst);
-	uint32_t x = plane_state->src.x1 >> 16;
-	uint32_t y = plane_state->src.y1 >> 16;
-	uint32_t src_w = drm_rect_width(&plane_state->src) >> 16;
-	uint32_t src_h = drm_rect_height(&plane_state->src) >> 16;
+	int crtc_x = plane_state->base.dst.x1;
+	int crtc_y = plane_state->base.dst.y1;
+	uint32_t crtc_w = drm_rect_width(&plane_state->base.dst);
+	uint32_t crtc_h = drm_rect_height(&plane_state->base.dst);
+	uint32_t x = plane_state->base.src.x1 >> 16;
+	uint32_t y = plane_state->base.src.y1 >> 16;
+	uint32_t src_w = drm_rect_width(&plane_state->base.src) >> 16;
+	uint32_t src_h = drm_rect_height(&plane_state->base.src) >> 16;
 
 	sprctl = SPRITE_ENABLE;
 
@@ -552,7 +539,7 @@ ivb_update_plane(struct drm_plane *plane,
 	 */
 	sprctl |= SPRITE_GAMMA_ENABLE;
 
-	if (obj->tiling_mode != I915_TILING_NONE)
+	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
 		sprctl |= SPRITE_TILED;
 
 	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
@@ -572,22 +559,21 @@ ivb_update_plane(struct drm_plane *plane,
 	if (crtc_w != src_w || crtc_h != src_h)
 		sprscale = SPRITE_SCALE_ENABLE | (src_w << 16) | src_h;
 
-	linear_offset = y * fb->pitches[0] + x * cpp;
-	sprsurf_offset = intel_compute_tile_offset(&x, &y, fb, 0,
-						   fb->pitches[0], rotation);
-	linear_offset -= sprsurf_offset;
+	intel_add_fb_offsets(&x, &y, plane_state, 0);
+	sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	if (rotation == BIT(DRM_ROTATE_180)) {
+	if (rotation == DRM_ROTATE_180) {
 		sprctl |= SPRITE_ROTATE_180;
 
 		/* HSW and BDW does this automagically in hardware */
 		if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
 			x += src_w;
 			y += src_h;
-			linear_offset += src_h * fb->pitches[0] + src_w * cpp;
 		}
 	}
 
+	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
+
 	if (key->flags) {
 		I915_WRITE(SPRKEYVAL(pipe), key->min_value);
 		I915_WRITE(SPRKEYMAX(pipe), key->max_value);
@@ -606,7 +592,7 @@ ivb_update_plane(struct drm_plane *plane,
 	 * register */
 	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
 		I915_WRITE(SPROFFSET(pipe), (y << 16) | x);
-	else if (obj->tiling_mode != I915_TILING_NONE)
+	else if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
 		I915_WRITE(SPRTILEOFF(pipe), (y << 16) | x);
 	else
 		I915_WRITE(SPRLINOFF(pipe), linear_offset);
@@ -616,7 +602,7 @@ ivb_update_plane(struct drm_plane *plane,
 		I915_WRITE(SPRSCALE(pipe), sprscale);
 	I915_WRITE(SPRCTL(pipe), sprctl);
 	I915_WRITE(SPRSURF(pipe),
-		   i915_gem_obj_ggtt_offset(obj) + sprsurf_offset);
+		   intel_fb_gtt_offset(fb, rotation) + sprsurf_offset);
 	POSTING_READ(SPRSURF(pipe));
 }
 
@@ -646,21 +632,19 @@ ilk_update_plane(struct drm_plane *plane,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_plane *intel_plane = to_intel_plane(plane);
 	struct drm_framebuffer *fb = plane_state->base.fb;
-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	int pipe = intel_plane->pipe;
 	u32 dvscntr, dvsscale;
 	u32 dvssurf_offset, linear_offset;
 	unsigned int rotation = plane_state->base.rotation;
-	int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 	const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
-	int crtc_x = plane_state->dst.x1;
-	int crtc_y = plane_state->dst.y1;
-	uint32_t crtc_w = drm_rect_width(&plane_state->dst);
-	uint32_t crtc_h = drm_rect_height(&plane_state->dst);
-	uint32_t x = plane_state->src.x1 >> 16;
-	uint32_t y = plane_state->src.y1 >> 16;
-	uint32_t src_w = drm_rect_width(&plane_state->src) >> 16;
-	uint32_t src_h = drm_rect_height(&plane_state->src) >> 16;
+	int crtc_x = plane_state->base.dst.x1;
+	int crtc_y = plane_state->base.dst.y1;
+	uint32_t crtc_w = drm_rect_width(&plane_state->base.dst);
+	uint32_t crtc_h = drm_rect_height(&plane_state->base.dst);
+	uint32_t x = plane_state->base.src.x1 >> 16;
+	uint32_t y = plane_state->base.src.y1 >> 16;
+	uint32_t src_w = drm_rect_width(&plane_state->base.src) >> 16;
+	uint32_t src_h = drm_rect_height(&plane_state->base.src) >> 16;
 
 	dvscntr = DVS_ENABLE;
 
@@ -693,7 +677,7 @@ ilk_update_plane(struct drm_plane *plane,
 	 */
 	dvscntr |= DVS_GAMMA_ENABLE;
 
-	if (obj->tiling_mode != I915_TILING_NONE)
+	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
 		dvscntr |= DVS_TILED;
 
 	if (IS_GEN6(dev))
@@ -709,19 +693,18 @@ ilk_update_plane(struct drm_plane *plane,
 	if (crtc_w != src_w || crtc_h != src_h)
 		dvsscale = DVS_SCALE_ENABLE | (src_w << 16) | src_h;
 
-	linear_offset = y * fb->pitches[0] + x * cpp;
-	dvssurf_offset = intel_compute_tile_offset(&x, &y, fb, 0,
-						   fb->pitches[0], rotation);
-	linear_offset -= dvssurf_offset;
+	intel_add_fb_offsets(&x, &y, plane_state, 0);
+	dvssurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	if (rotation == BIT(DRM_ROTATE_180)) {
+	if (rotation == DRM_ROTATE_180) {
 		dvscntr |= DVS_ROTATE_180;
 
 		x += src_w;
 		y += src_h;
-		linear_offset += src_h * fb->pitches[0] + src_w * cpp;
 	}
 
+	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
+
 	if (key->flags) {
 		I915_WRITE(DVSKEYVAL(pipe), key->min_value);
 		I915_WRITE(DVSKEYMAX(pipe), key->max_value);
@@ -736,7 +719,7 @@ ilk_update_plane(struct drm_plane *plane,
 	I915_WRITE(DVSSTRIDE(pipe), fb->pitches[0]);
 	I915_WRITE(DVSPOS(pipe), (crtc_y << 16) | crtc_x);
 
-	if (obj->tiling_mode != I915_TILING_NONE)
+	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
 		I915_WRITE(DVSTILEOFF(pipe), (y << 16) | x);
 	else
 		I915_WRITE(DVSLINOFF(pipe), linear_offset);
@@ -745,7 +728,7 @@ ilk_update_plane(struct drm_plane *plane,
 	I915_WRITE(DVSSCALE(pipe), dvsscale);
 	I915_WRITE(DVSCNTR(pipe), dvscntr);
 	I915_WRITE(DVSSURF(pipe),
-		   i915_gem_obj_ggtt_offset(obj) + dvssurf_offset);
+		   intel_fb_gtt_offset(fb, rotation) + dvssurf_offset);
 	POSTING_READ(DVSSURF(pipe));
 }
 
@@ -778,15 +761,26 @@ intel_check_sprite_plane(struct drm_plane *plane,
 	int crtc_x, crtc_y;
 	unsigned int crtc_w, crtc_h;
 	uint32_t src_x, src_y, src_w, src_h;
-	struct drm_rect *src = &state->src;
-	struct drm_rect *dst = &state->dst;
+	struct drm_rect *src = &state->base.src;
+	struct drm_rect *dst = &state->base.dst;
 	const struct drm_rect *clip = &state->clip;
 	int hscale, vscale;
 	int max_scale, min_scale;
 	bool can_scale;
+	int ret;
+
+	src->x1 = state->base.src_x;
+	src->y1 = state->base.src_y;
+	src->x2 = state->base.src_x + state->base.src_w;
+	src->y2 = state->base.src_y + state->base.src_h;
+
+	dst->x1 = state->base.crtc_x;
+	dst->y1 = state->base.crtc_y;
+	dst->x2 = state->base.crtc_x + state->base.crtc_w;
+	dst->y2 = state->base.crtc_y + state->base.crtc_h;
 
 	if (!fb) {
-		state->visible = false;
+		state->base.visible = false;
 		return 0;
 	}
 
@@ -834,14 +828,14 @@ intel_check_sprite_plane(struct drm_plane *plane,
 	vscale = drm_rect_calc_vscale_relaxed(src, dst, min_scale, max_scale);
 	BUG_ON(vscale < 0);
 
-	state->visible = drm_rect_clip_scaled(src, dst, clip, hscale, vscale);
+	state->base.visible = drm_rect_clip_scaled(src, dst, clip, hscale, vscale);
 
 	crtc_x = dst->x1;
 	crtc_y = dst->y1;
 	crtc_w = drm_rect_width(dst);
 	crtc_h = drm_rect_height(dst);
 
-	if (state->visible) {
+	if (state->base.visible) {
 		/* check again in case clipping clamped the results */
 		hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale);
 		if (hscale < 0) {
@@ -898,12 +892,12 @@ intel_check_sprite_plane(struct drm_plane *plane,
 				crtc_w &= ~1;
 
 			if (crtc_w == 0)
-				state->visible = false;
+				state->base.visible = false;
 		}
 	}
 
 	/* Check size restrictions when scaling */
-	if (state->visible && (src_w != crtc_w || src_h != crtc_h)) {
+	if (state->base.visible && (src_w != crtc_w || src_h != crtc_h)) {
 		unsigned int width_bytes;
 		int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 
@@ -912,10 +906,10 @@ intel_check_sprite_plane(struct drm_plane *plane,
 		/* FIXME interlacing min height is 6 */
 
 		if (crtc_w < 3 || crtc_h < 3)
-			state->visible = false;
+			state->base.visible = false;
 
 		if (src_w < 3 || src_h < 3)
-			state->visible = false;
+			state->base.visible = false;
 
 		width_bytes = ((src_x * cpp) & 63) + src_w * cpp;
 
@@ -926,7 +920,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 		}
 	}
 
-	if (state->visible) {
+	if (state->base.visible) {
 		src->x1 = src_x << 16;
 		src->x2 = (src_x + src_w) << 16;
 		src->y1 = src_y << 16;
@@ -938,6 +932,12 @@ intel_check_sprite_plane(struct drm_plane *plane,
 	dst->y1 = crtc_y;
 	dst->y2 = crtc_y + crtc_h;
 
+	if (INTEL_GEN(dev) >= 9) {
+		ret = skl_check_plane_surface(state);
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 49136ad5473e..d960e4866595 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -838,7 +838,9 @@ intel_tv_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe)
 }
 
 static void
-intel_enable_tv(struct intel_encoder *encoder)
+intel_enable_tv(struct intel_encoder *encoder,
+		struct intel_crtc_state *pipe_config,
+		struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -851,7 +853,9 @@ intel_enable_tv(struct intel_encoder *encoder)
 }
 
 static void
-intel_disable_tv(struct intel_encoder *encoder)
+intel_disable_tv(struct intel_encoder *encoder,
+		 struct intel_crtc_state *old_crtc_state,
+		 struct drm_connector_state *old_conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -908,7 +912,8 @@ intel_tv_get_config(struct intel_encoder *encoder,
 
 static bool
 intel_tv_compute_config(struct intel_encoder *encoder,
-			struct intel_crtc_state *pipe_config)
+			struct intel_crtc_state *pipe_config,
+			struct drm_connector_state *conn_state)
 {
 	struct intel_tv *intel_tv = enc_to_tv(encoder);
 	const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
@@ -1010,7 +1015,9 @@ static void set_color_conversion(struct drm_i915_private *dev_priv,
 		   color_conversion->av);
 }
 
-static void intel_tv_pre_enable(struct intel_encoder *encoder)
+static void intel_tv_pre_enable(struct intel_encoder *encoder,
+				struct intel_crtc_state *pipe_config,
+				struct drm_connector_state *conn_state)
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index ff80a81b1a84..ee2306a79747 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -435,7 +435,7 @@ void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
 	i915.enable_rc6 = sanitize_rc6_option(dev_priv, i915.enable_rc6);
 
 	/* BIOS often leaves RC6 enabled, but disable it for hw init */
-	intel_disable_gt_powersave(dev_priv);
+	intel_sanitize_gt_powersave(dev_priv);
 }
 
 static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
@@ -796,10 +796,9 @@ __unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 		      const bool read,
 		      const bool before)
 {
-	if (WARN(check_for_unclaimed_mmio(dev_priv),
-		 "Unclaimed register detected %s %s register 0x%x\n",
-		 before ? "before" : "after",
-		 read ? "reading" : "writing to",
+	if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
+		 "Unclaimed %s register 0x%x\n",
+		 read ? "read from" : "write to",
 		 i915_mmio_reg_offset(reg)))
 		i915.mmio_debug--; /* Only report the first N failures */
 }
@@ -1018,11 +1017,9 @@ gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool
 __gen5_write(8)
 __gen5_write(16)
 __gen5_write(32)
-__gen5_write(64)
 __gen2_write(8)
 __gen2_write(16)
 __gen2_write(32)
-__gen2_write(64)
 
 #undef __gen5_write
 #undef __gen2_write
@@ -1112,23 +1109,18 @@ gen9_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, \
 __gen9_write(8)
 __gen9_write(16)
 __gen9_write(32)
-__gen9_write(64)
 __chv_write(8)
 __chv_write(16)
 __chv_write(32)
-__chv_write(64)
 __gen8_write(8)
 __gen8_write(16)
 __gen8_write(32)
-__gen8_write(64)
 __hsw_write(8)
 __hsw_write(16)
 __hsw_write(32)
-__hsw_write(64)
 __gen6_write(8)
 __gen6_write(16)
 __gen6_write(32)
-__gen6_write(64)
 
 #undef __gen9_write
 #undef __chv_write
@@ -1158,7 +1150,6 @@ static void vgpu_write##x(struct drm_i915_private *dev_priv, \
 __vgpu_write(8)
 __vgpu_write(16)
 __vgpu_write(32)
-__vgpu_write(64)
 
 #undef __vgpu_write
 #undef VGPU_WRITE_FOOTER
@@ -1169,7 +1160,6 @@ do { \
 	dev_priv->uncore.funcs.mmio_writeb = x##_write8; \
 	dev_priv->uncore.funcs.mmio_writew = x##_write16; \
 	dev_priv->uncore.funcs.mmio_writel = x##_write32; \
-	dev_priv->uncore.funcs.mmio_writeq = x##_write64; \
 } while (0)
 
 #define ASSIGN_READ_MMIO_VFUNCS(x) \
@@ -1597,8 +1587,10 @@ static int gen6_reset_engines(struct drm_i915_private *dev_priv,
 	if (engine_mask == ALL_ENGINES) {
 		hw_mask = GEN6_GRDOM_FULL;
 	} else {
+		unsigned int tmp;
+
 		hw_mask = 0;
-		for_each_engine_masked(engine, dev_priv, engine_mask)
+		for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
 			hw_mask |= hw_engine_mask[engine->id];
 	}
 
@@ -1618,8 +1610,10 @@ static int gen6_reset_engines(struct drm_i915_private *dev_priv,
  * @timeout_ms: timeout in millisecond
  *
  * This routine waits until the target register @reg contains the expected
- * @value after applying the @mask, i.e. it waits until
- *   (I915_READ_FW(@reg) & @mask) == @value
+ * @value after applying the @mask, i.e. it waits until ::
+ *
+ *     (I915_READ_FW(reg) & mask) == value
+ *
  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
  *
  * Note that this routine assumes the caller holds forcewake asserted, it is
@@ -1652,8 +1646,10 @@ int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
  * @timeout_ms: timeout in millisecond
  *
  * This routine waits until the target register @reg contains the expected
- * @value after applying the @mask, i.e. it waits until
- *   (I915_READ(@reg) & @mask) == @value
+ * @value after applying the @mask, i.e. it waits until ::
+ *
+ *     (I915_READ(reg) & mask) == value
+ *
  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
  *
  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
@@ -1710,15 +1706,16 @@ static int gen8_reset_engines(struct drm_i915_private *dev_priv,
 			      unsigned engine_mask)
 {
 	struct intel_engine_cs *engine;
+	unsigned int tmp;
 
-	for_each_engine_masked(engine, dev_priv, engine_mask)
+	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
 		if (gen8_request_engine_reset(engine))
 			goto not_ready;
 
 	return gen6_reset_engines(dev_priv, engine_mask);
 
 not_ready:
-	for_each_engine_masked(engine, dev_priv, engine_mask)
+	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
 		gen8_unrequest_engine_reset(engine);
 
 	return -EIO;
diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
index 9f7dafce3a4c..98df09c2b388 100644
--- a/drivers/gpu/drm/imx/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/imx-drm-core.c
@@ -16,7 +16,6 @@
 #include <linux/component.h>
 #include <linux/device.h>
 #include <linux/dma-buf.h>
-#include <linux/fb.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/reservation.h>
@@ -58,12 +57,6 @@ static int legacyfb_depth = 16;
 module_param(legacyfb_depth, int, 0444);
 #endif
 
-unsigned int imx_drm_crtc_id(struct imx_drm_crtc *crtc)
-{
-	return drm_crtc_index(crtc->crtc);
-}
-EXPORT_SYMBOL_GPL(imx_drm_crtc_id);
-
 static void imx_drm_driver_lastclose(struct drm_device *drm)
 {
 	struct imx_drm_device *imxdrm = drm->dev_private;
@@ -71,43 +64,6 @@ static void imx_drm_driver_lastclose(struct drm_device *drm)
 	drm_fbdev_cma_restore_mode(imxdrm->fbhelper);
 }
 
-static int imx_drm_driver_unload(struct drm_device *drm)
-{
-	struct imx_drm_device *imxdrm = drm->dev_private;
-
-	drm_kms_helper_poll_fini(drm);
-
-	if (imxdrm->fbhelper)
-		drm_fbdev_cma_fini(imxdrm->fbhelper);
-
-	component_unbind_all(drm->dev, drm);
-
-	drm_vblank_cleanup(drm);
-	drm_mode_config_cleanup(drm);
-
-	platform_set_drvdata(drm->platformdev, NULL);
-
-	return 0;
-}
-
-int imx_drm_crtc_vblank_get(struct imx_drm_crtc *imx_drm_crtc)
-{
-	return drm_crtc_vblank_get(imx_drm_crtc->crtc);
-}
-EXPORT_SYMBOL_GPL(imx_drm_crtc_vblank_get);
-
-void imx_drm_crtc_vblank_put(struct imx_drm_crtc *imx_drm_crtc)
-{
-	drm_crtc_vblank_put(imx_drm_crtc->crtc);
-}
-EXPORT_SYMBOL_GPL(imx_drm_crtc_vblank_put);
-
-void imx_drm_handle_vblank(struct imx_drm_crtc *imx_drm_crtc)
-{
-	drm_crtc_handle_vblank(imx_drm_crtc->crtc);
-}
-EXPORT_SYMBOL_GPL(imx_drm_handle_vblank);
-
 static int imx_drm_enable_vblank(struct drm_device *drm, unsigned int pipe)
 {
 	struct imx_drm_device *imxdrm = drm->dev_private;
@@ -171,54 +127,73 @@ static void imx_drm_output_poll_changed(struct drm_device *drm)
 	drm_fbdev_cma_hotplug_event(imxdrm->fbhelper);
 }
 
+static int imx_drm_atomic_check(struct drm_device *dev,
+				struct drm_atomic_state *state)
+{
+	int ret;
+
+	ret = drm_atomic_helper_check_modeset(dev, state);
+	if (ret)
+		return ret;
+
+	ret = drm_atomic_helper_check_planes(dev, state);
+	if (ret)
+		return ret;
+
+	/*
+	 * Check modeset again in case crtc_state->mode_changed is
+	 * updated in plane's ->atomic_check callback.
+	 */
+	ret = drm_atomic_helper_check_modeset(dev, state);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static int imx_drm_atomic_commit(struct drm_device *dev,
+				 struct drm_atomic_state *state,
+				 bool nonblock)
+{
+	struct drm_plane_state *plane_state;
+	struct drm_plane *plane;
+	struct dma_buf *dma_buf;
+	int i;
+
+	/*
+	 * If the plane fb has an dma-buf attached, fish out the exclusive
+	 * fence for the atomic helper to wait on.
+	 */
+	for_each_plane_in_state(state, plane, plane_state, i) {
+		if ((plane->state->fb != plane_state->fb) && plane_state->fb) {
+			dma_buf = drm_fb_cma_get_gem_obj(plane_state->fb,
+							 0)->base.dma_buf;
+			if (!dma_buf)
+				continue;
+			plane_state->fence =
+				reservation_object_get_excl_rcu(dma_buf->resv);
+		}
+	}
+
+	return drm_atomic_helper_commit(dev, state, nonblock);
+}
+
 static const struct drm_mode_config_funcs imx_drm_mode_config_funcs = {
 	.fb_create = drm_fb_cma_create,
 	.output_poll_changed = imx_drm_output_poll_changed,
-	.atomic_check = drm_atomic_helper_check,
-	.atomic_commit = drm_atomic_helper_commit,
+	.atomic_check = imx_drm_atomic_check,
+	.atomic_commit = imx_drm_atomic_commit,
 };
 
 static void imx_drm_atomic_commit_tail(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *crtc_state;
-	struct drm_plane_state *plane_state;
-	struct drm_gem_cma_object *cma_obj;
-	struct fence *excl;
-	unsigned shared_count;
-	struct fence **shared;
-	unsigned int i, j;
-	int ret;
-
-	/* Wait for fences. */
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		plane_state = crtc->primary->state;
-		if (plane_state->fb) {
-			cma_obj = drm_fb_cma_get_gem_obj(plane_state->fb, 0);
-			if (cma_obj->base.dma_buf) {
-				ret = reservation_object_get_fences_rcu(
-					cma_obj->base.dma_buf->resv, &excl,
-					&shared_count, &shared);
-				if (unlikely(ret))
-					DRM_ERROR("failed to get fences "
-						  "for buffer\n");
-
-				if (excl) {
-					fence_wait(excl, false);
-					fence_put(excl);
-				}
-				for (j = 0; j < shared_count; i++) {
-					fence_wait(shared[j], false);
-					fence_put(shared[j]);
-				}
-			}
-		}
-	}
 
 	drm_atomic_helper_commit_modeset_disables(dev, state);
 
-	drm_atomic_helper_commit_planes(dev, state, true);
+	drm_atomic_helper_commit_planes(dev, state,
+				DRM_PLANE_COMMIT_ACTIVE_ONLY |
+				DRM_PLANE_COMMIT_NO_DISABLE_AFTER_MODESET);
 
 	drm_atomic_helper_commit_modeset_enables(dev, state);
 
@@ -234,111 +209,6 @@ static struct drm_mode_config_helper_funcs imx_drm_mode_config_helpers = {
 };
 
 /*
- * Main DRM initialisation. This binds, initialises and registers
- * with DRM the subcomponents of the driver.
- */
-static int imx_drm_driver_load(struct drm_device *drm, unsigned long flags)
-{
-	struct imx_drm_device *imxdrm;
-	struct drm_connector *connector;
-	int ret;
-
-	imxdrm = devm_kzalloc(drm->dev, sizeof(*imxdrm), GFP_KERNEL);
-	if (!imxdrm)
-		return -ENOMEM;
-
-	imxdrm->drm = drm;
-
-	drm->dev_private = imxdrm;
-
-	/*
-	 * enable drm irq mode.
-	 * - with irq_enabled = true, we can use the vblank feature.
-	 *
-	 * P.S. note that we wouldn't use drm irq handler but
-	 *      just specific driver own one instead because
-	 *      drm framework supports only one irq handler and
-	 *      drivers can well take care of their interrupts
-	 */
-	drm->irq_enabled = true;
-
-	/*
-	 * set max width and height as default value(4096x4096).
-	 * this value would be used to check framebuffer size limitation
-	 * at drm_mode_addfb().
-	 */
-	drm->mode_config.min_width = 64;
-	drm->mode_config.min_height = 64;
-	drm->mode_config.max_width = 4096;
-	drm->mode_config.max_height = 4096;
-	drm->mode_config.funcs = &imx_drm_mode_config_funcs;
-	drm->mode_config.helper_private = &imx_drm_mode_config_helpers;
-
-	drm_mode_config_init(drm);
-
-	ret = drm_vblank_init(drm, MAX_CRTC);
-	if (ret)
-		goto err_kms;
-
-	platform_set_drvdata(drm->platformdev, drm);
-
-	/* Now try and bind all our sub-components */
-	ret = component_bind_all(drm->dev, drm);
-	if (ret)
-		goto err_vblank;
-
-	/*
-	 * All components are now added, we can publish the connector sysfs
-	 * entries to userspace.  This will generate hotplug events and so
-	 * userspace will expect to be able to access DRM at this point.
-	 */
-	list_for_each_entry(connector, &drm->mode_config.connector_list, head) {
-		ret = drm_connector_register(connector);
-		if (ret) {
-			dev_err(drm->dev,
-				"[CONNECTOR:%d:%s] drm_connector_register failed: %d\n",
-				connector->base.id,
-				connector->name, ret);
-			goto err_unbind;
-		}
-	}
-
-	drm_mode_config_reset(drm);
-
-	/*
-	 * All components are now initialised, so setup the fb helper.
-	 * The fb helper takes copies of key hardware information, so the
-	 * crtcs/connectors/encoders must not change after this point.
-	 */
-#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
-	if (legacyfb_depth != 16 && legacyfb_depth != 32) {
-		dev_warn(drm->dev, "Invalid legacyfb_depth.  Defaulting to 16bpp\n");
-		legacyfb_depth = 16;
-	}
-	imxdrm->fbhelper = drm_fbdev_cma_init(drm, legacyfb_depth,
-				drm->mode_config.num_crtc, MAX_CRTC);
-	if (IS_ERR(imxdrm->fbhelper)) {
-		ret = PTR_ERR(imxdrm->fbhelper);
-		imxdrm->fbhelper = NULL;
-		goto err_unbind;
-	}
-#endif
-
-	drm_kms_helper_poll_init(drm);
-
-	return 0;
-
-err_unbind:
-	component_unbind_all(drm->dev, drm);
-err_vblank:
-	drm_vblank_cleanup(drm);
-err_kms:
-	drm_mode_config_cleanup(drm);
-
-	return ret;
-}
-
-/*
  * imx_drm_add_crtc - add a new crtc
  */
 int imx_drm_add_crtc(struct drm_device *drm, struct drm_crtc *crtc,
@@ -430,8 +300,6 @@ static const struct drm_ioctl_desc imx_drm_ioctls[] = {
 static struct drm_driver imx_drm_driver = {
 	.driver_features	= DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
 				  DRIVER_ATOMIC,
-	.load			= imx_drm_driver_load,
-	.unload			= imx_drm_driver_unload,
 	.lastclose		= imx_drm_driver_lastclose,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
@@ -484,12 +352,122 @@ static int compare_of(struct device *dev, void *data)
 
 static int imx_drm_bind(struct device *dev)
 {
-	return drm_platform_init(&imx_drm_driver, to_platform_device(dev));
+	struct drm_device *drm;
+	struct imx_drm_device *imxdrm;
+	int ret;
+
+	drm = drm_dev_alloc(&imx_drm_driver, dev);
+	if (!drm)
+		return -ENOMEM;
+
+	imxdrm = devm_kzalloc(dev, sizeof(*imxdrm), GFP_KERNEL);
+	if (!imxdrm) {
+		ret = -ENOMEM;
+		goto err_unref;
+	}
+
+	imxdrm->drm = drm;
+	drm->dev_private = imxdrm;
+
+	/*
+	 * enable drm irq mode.
+	 * - with irq_enabled = true, we can use the vblank feature.
+	 *
+	 * P.S. note that we wouldn't use drm irq handler but
+	 *      just specific driver own one instead because
+	 *      drm framework supports only one irq handler and
+	 *      drivers can well take care of their interrupts
+	 */
+	drm->irq_enabled = true;
+
+	/*
+	 * set max width and height as default value(4096x4096).
+	 * this value would be used to check framebuffer size limitation
+	 * at drm_mode_addfb().
+	 */
+	drm->mode_config.min_width = 64;
+	drm->mode_config.min_height = 64;
+	drm->mode_config.max_width = 4096;
+	drm->mode_config.max_height = 4096;
+	drm->mode_config.funcs = &imx_drm_mode_config_funcs;
+	drm->mode_config.helper_private = &imx_drm_mode_config_helpers;
+
+	drm_mode_config_init(drm);
+
+	ret = drm_vblank_init(drm, MAX_CRTC);
+	if (ret)
+		goto err_kms;
+
+	dev_set_drvdata(dev, drm);
+
+	/* Now try and bind all our sub-components */
+	ret = component_bind_all(dev, drm);
+	if (ret)
+		goto err_vblank;
+
+	drm_mode_config_reset(drm);
+
+	/*
+	 * All components are now initialised, so setup the fb helper.
+	 * The fb helper takes copies of key hardware information, so the
+	 * crtcs/connectors/encoders must not change after this point.
+	 */
+#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
+	if (legacyfb_depth != 16 && legacyfb_depth != 32) {
+		dev_warn(dev, "Invalid legacyfb_depth.  Defaulting to 16bpp\n");
+		legacyfb_depth = 16;
+	}
+	imxdrm->fbhelper = drm_fbdev_cma_init(drm, legacyfb_depth,
+				drm->mode_config.num_crtc, MAX_CRTC);
+	if (IS_ERR(imxdrm->fbhelper)) {
+		ret = PTR_ERR(imxdrm->fbhelper);
+		imxdrm->fbhelper = NULL;
+		goto err_unbind;
+	}
+#endif
+
+	drm_kms_helper_poll_init(drm);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto err_fbhelper;
+
+	return 0;
+
+err_fbhelper:
+	drm_kms_helper_poll_fini(drm);
+	if (imxdrm->fbhelper)
+		drm_fbdev_cma_fini(imxdrm->fbhelper);
+err_unbind:
+	component_unbind_all(drm->dev, drm);
+err_vblank:
+	drm_vblank_cleanup(drm);
+err_kms:
+	drm_mode_config_cleanup(drm);
+err_unref:
+	drm_dev_unref(drm);
+
+	return ret;
 }
 
 static void imx_drm_unbind(struct device *dev)
 {
-	drm_put_dev(dev_get_drvdata(dev));
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct imx_drm_device *imxdrm = drm->dev_private;
+
+	drm_dev_unregister(drm);
+
+	drm_kms_helper_poll_fini(drm);
+
+	if (imxdrm->fbhelper)
+		drm_fbdev_cma_fini(imxdrm->fbhelper);
+
+	drm_mode_config_cleanup(drm);
+
+	component_unbind_all(drm->dev, drm);
+	dev_set_drvdata(dev, NULL);
+
+	drm_dev_unref(drm);
 }
 
 static const struct component_master_ops imx_drm_ops = {
diff --git a/drivers/gpu/drm/imx/imx-drm.h b/drivers/gpu/drm/imx/imx-drm.h
index 07d33e45f90f..5a91cb16c8fa 100644
--- a/drivers/gpu/drm/imx/imx-drm.h
+++ b/drivers/gpu/drm/imx/imx-drm.h
@@ -13,8 +13,6 @@ struct drm_plane;
 struct imx_drm_crtc;
 struct platform_device;
 
-unsigned int imx_drm_crtc_id(struct imx_drm_crtc *crtc);
-
 struct imx_crtc_state {
 	struct drm_crtc_state			base;
 	u32					bus_format;
@@ -44,10 +42,6 @@ int imx_drm_init_drm(struct platform_device *pdev,
 		int preferred_bpp);
 int imx_drm_exit_drm(void);
 
-int imx_drm_crtc_vblank_get(struct imx_drm_crtc *imx_drm_crtc);
-void imx_drm_crtc_vblank_put(struct imx_drm_crtc *imx_drm_crtc);
-void imx_drm_handle_vblank(struct imx_drm_crtc *imx_drm_crtc);
-
 void imx_drm_mode_config_init(struct drm_device *drm);
 
 struct drm_gem_cma_object *imx_drm_fb_get_obj(struct drm_framebuffer *fb);
diff --git a/drivers/gpu/drm/imx/imx-ldb.c b/drivers/gpu/drm/imx/imx-ldb.c
index b03919ed60ba..3ce391c239b0 100644
--- a/drivers/gpu/drm/imx/imx-ldb.c
+++ b/drivers/gpu/drm/imx/imx-ldb.c
@@ -57,7 +57,11 @@ struct imx_ldb_channel {
 	struct imx_ldb *ldb;
 	struct drm_connector connector;
 	struct drm_encoder encoder;
+
+	/* Defines what is connected to the ldb, only one at a time */
 	struct drm_panel *panel;
+	struct drm_bridge *bridge;
+
 	struct device_node *child;
 	struct i2c_adapter *ddc;
 	int chno;
@@ -66,6 +70,7 @@ struct imx_ldb_channel {
 	struct drm_display_mode mode;
 	int mode_valid;
 	u32 bus_format;
+	u32 bus_flags;
 };
 
 static inline struct imx_ldb_channel *con_to_imx_ldb_ch(struct drm_connector *c)
@@ -251,11 +256,13 @@ static void imx_ldb_encoder_enable(struct drm_encoder *encoder)
 	drm_panel_enable(imx_ldb_ch->panel);
 }
 
-static void imx_ldb_encoder_mode_set(struct drm_encoder *encoder,
-			 struct drm_display_mode *orig_mode,
-			 struct drm_display_mode *mode)
+static void
+imx_ldb_encoder_atomic_mode_set(struct drm_encoder *encoder,
+				struct drm_crtc_state *crtc_state,
+				struct drm_connector_state *connector_state)
 {
 	struct imx_ldb_channel *imx_ldb_ch = enc_to_imx_ldb_ch(encoder);
+	struct drm_display_mode *mode = &crtc_state->adjusted_mode;
 	struct imx_ldb *ldb = imx_ldb_ch->ldb;
 	int dual = ldb->ldb_ctrl & LDB_SPLIT_MODE_EN;
 	unsigned long serial_clk;
@@ -297,17 +304,11 @@ static void imx_ldb_encoder_mode_set(struct drm_encoder *encoder,
 	}
 
 	if (!bus_format) {
-		struct drm_connector *connector;
-
-		drm_for_each_connector(connector, encoder->dev) {
-			struct drm_display_info *di = &connector->display_info;
+		struct drm_connector *connector = connector_state->connector;
+		struct drm_display_info *di = &connector->display_info;
 
-			if (connector->encoder == encoder &&
-			    di->num_bus_formats) {
-				bus_format = di->bus_formats[0];
-				break;
-			}
-		}
+		if (di->num_bus_formats)
+			bus_format = di->bus_formats[0];
 	}
 	imx_ldb_ch_set_bus_format(imx_ldb_ch, bus_format);
 }
@@ -379,8 +380,13 @@ static int imx_ldb_encoder_atomic_check(struct drm_encoder *encoder,
 	u32 bus_format = imx_ldb_ch->bus_format;
 
 	/* Bus format description in DT overrides connector display info. */
-	if (!bus_format && di->num_bus_formats)
+	if (!bus_format && di->num_bus_formats) {
 		bus_format = di->bus_formats[0];
+		imx_crtc_state->bus_flags = di->bus_flags;
+	} else {
+		bus_format = imx_ldb_ch->bus_format;
+		imx_crtc_state->bus_flags = imx_ldb_ch->bus_flags;
+	}
 	switch (bus_format) {
 	case MEDIA_BUS_FMT_RGB666_1X7X3_SPWG:
 		imx_crtc_state->bus_format = MEDIA_BUS_FMT_RGB666_1X18;
@@ -420,7 +426,7 @@ static const struct drm_encoder_funcs imx_ldb_encoder_funcs = {
 };
 
 static const struct drm_encoder_helper_funcs imx_ldb_encoder_helper_funcs = {
-	.mode_set = imx_ldb_encoder_mode_set,
+	.atomic_mode_set = imx_ldb_encoder_atomic_mode_set,
 	.enable = imx_ldb_encoder_enable,
 	.disable = imx_ldb_encoder_disable,
 	.atomic_check = imx_ldb_encoder_atomic_check,
@@ -466,10 +472,30 @@ static int imx_ldb_register(struct drm_device *drm,
 	drm_encoder_init(drm, encoder, &imx_ldb_encoder_funcs,
 			 DRM_MODE_ENCODER_LVDS, NULL);
 
-	drm_connector_helper_add(&imx_ldb_ch->connector,
-			&imx_ldb_connector_helper_funcs);
-	drm_connector_init(drm, &imx_ldb_ch->connector,
-			   &imx_ldb_connector_funcs, DRM_MODE_CONNECTOR_LVDS);
+	if (imx_ldb_ch->bridge) {
+		imx_ldb_ch->bridge->encoder = encoder;
+
+		imx_ldb_ch->encoder.bridge = imx_ldb_ch->bridge;
+		ret = drm_bridge_attach(drm, imx_ldb_ch->bridge);
+		if (ret) {
+			DRM_ERROR("Failed to initialize bridge with drm\n");
+			return ret;
+		}
+	} else {
+		/*
+		 * We want to add the connector whenever there is no bridge
+		 * that brings its own, not only when there is a panel. For
+		 * historical reasons, the ldb driver can also work without
+		 * a panel.
+		 */
+		drm_connector_helper_add(&imx_ldb_ch->connector,
+				&imx_ldb_connector_helper_funcs);
+		drm_connector_init(drm, &imx_ldb_ch->connector,
+				&imx_ldb_connector_funcs,
+				DRM_MODE_CONNECTOR_LVDS);
+		drm_mode_connector_attach_encoder(&imx_ldb_ch->connector,
+				encoder);
+	}
 
 	if (imx_ldb_ch->panel) {
 		ret = drm_panel_attach(imx_ldb_ch->panel,
@@ -478,8 +504,6 @@ static int imx_ldb_register(struct drm_device *drm,
 			return ret;
 	}
 
-	drm_mode_connector_attach_encoder(&imx_ldb_ch->connector, encoder);
-
 	return 0;
 }
 
@@ -548,6 +572,46 @@ static const struct of_device_id imx_ldb_dt_ids[] = {
 };
 MODULE_DEVICE_TABLE(of, imx_ldb_dt_ids);
 
+static int imx_ldb_panel_ddc(struct device *dev,
+		struct imx_ldb_channel *channel, struct device_node *child)
+{
+	struct device_node *ddc_node;
+	const u8 *edidp;
+	int ret;
+
+	ddc_node = of_parse_phandle(child, "ddc-i2c-bus", 0);
+	if (ddc_node) {
+		channel->ddc = of_find_i2c_adapter_by_node(ddc_node);
+		of_node_put(ddc_node);
+		if (!channel->ddc) {
+			dev_warn(dev, "failed to get ddc i2c adapter\n");
+			return -EPROBE_DEFER;
+		}
+	}
+
+	if (!channel->ddc) {
+		/* if no DDC available, fallback to hardcoded EDID */
+		dev_dbg(dev, "no ddc available\n");
+
+		edidp = of_get_property(child, "edid",
+					&channel->edid_len);
+		if (edidp) {
+			channel->edid = kmemdup(edidp,
+						channel->edid_len,
+						GFP_KERNEL);
+		} else if (!channel->panel) {
+			/* fallback to display-timings node */
+			ret = of_get_drm_display_mode(child,
+						      &channel->mode,
+						      &channel->bus_flags,
+						      OF_USE_NATIVE_MODE);
+			if (!ret)
+				channel->mode_valid = 1;
+		}
+	}
+	return 0;
+}
+
 static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
 {
 	struct drm_device *drm = data;
@@ -555,7 +619,6 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
 	const struct of_device_id *of_id =
 			of_match_device(imx_ldb_dt_ids, dev);
 	struct device_node *child;
-	const u8 *edidp;
 	struct imx_ldb *imx_ldb;
 	int dual;
 	int ret;
@@ -605,7 +668,6 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
 
 	for_each_child_of_node(np, child) {
 		struct imx_ldb_channel *channel;
-		struct device_node *ddc_node;
 		struct device_node *ep;
 		int bus_format;
 
@@ -638,46 +700,25 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
 
 			remote = of_graph_get_remote_port_parent(ep);
 			of_node_put(ep);
-			if (remote)
+			if (remote) {
 				channel->panel = of_drm_find_panel(remote);
-			else
+				channel->bridge = of_drm_find_bridge(remote);
+			} else
 				return -EPROBE_DEFER;
 			of_node_put(remote);
-			if (!channel->panel) {
-				dev_err(dev, "panel not found: %s\n",
+
+			if (!channel->panel && !channel->bridge) {
+				dev_err(dev, "panel/bridge not found: %s\n",
 					remote->full_name);
 				return -EPROBE_DEFER;
 			}
 		}
 
-		ddc_node = of_parse_phandle(child, "ddc-i2c-bus", 0);
-		if (ddc_node) {
-			channel->ddc = of_find_i2c_adapter_by_node(ddc_node);
-			of_node_put(ddc_node);
-			if (!channel->ddc) {
-				dev_warn(dev, "failed to get ddc i2c adapter\n");
-				return -EPROBE_DEFER;
-			}
-		}
-
-		if (!channel->ddc) {
-			/* if no DDC available, fallback to hardcoded EDID */
-			dev_dbg(dev, "no ddc available\n");
-
-			edidp = of_get_property(child, "edid",
-						&channel->edid_len);
-			if (edidp) {
-				channel->edid = kmemdup(edidp,
-							channel->edid_len,
-							GFP_KERNEL);
-			} else if (!channel->panel) {
-				/* fallback to display-timings node */
-				ret = of_get_drm_display_mode(child,
-							      &channel->mode,
-							      OF_USE_NATIVE_MODE);
-				if (!ret)
-					channel->mode_valid = 1;
-			}
+		/* panel ddc only if there is no bridge */
+		if (!channel->bridge) {
+			ret = imx_ldb_panel_ddc(dev, channel, child);
+			if (ret)
+				return ret;
 		}
 
 		bus_format = of_get_bus_format(dev, child);
@@ -716,11 +757,10 @@ static void imx_ldb_unbind(struct device *dev, struct device *master,
 	for (i = 0; i < 2; i++) {
 		struct imx_ldb_channel *channel = &imx_ldb->channel[i];
 
-		if (!channel->connector.funcs)
-			continue;
-
-		channel->connector.funcs->destroy(&channel->connector);
-		channel->encoder.funcs->destroy(&channel->encoder);
+		if (channel->bridge)
+			drm_bridge_detach(channel->bridge);
+		if (channel->panel)
+			drm_panel_detach(channel->panel);
 
 		kfree(channel->edid);
 		i2c_put_adapter(channel->ddc);
diff --git a/drivers/gpu/drm/imx/imx-tve.c b/drivers/gpu/drm/imx/imx-tve.c
index 5e875944ffa2..8fc088843e55 100644
--- a/drivers/gpu/drm/imx/imx-tve.c
+++ b/drivers/gpu/drm/imx/imx-tve.c
@@ -685,9 +685,6 @@ static void imx_tve_unbind(struct device *dev, struct device *master,
 {
 	struct imx_tve *tve = dev_get_drvdata(dev);
 
-	tve->connector.funcs->destroy(&tve->connector);
-	tve->encoder.funcs->destroy(&tve->encoder);
-
 	if (!IS_ERR(tve->dac_reg))
 		regulator_disable(tve->dac_reg);
 }
diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
index 08e188bc10fc..4e1ae3fc462d 100644
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c
@@ -21,7 +21,6 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
-#include <linux/fb.h>
 #include <linux/clk.h>
 #include <linux/errno.h>
 #include <drm/drm_gem_cma_helper.h>
@@ -61,7 +60,8 @@ static void ipu_crtc_enable(struct drm_crtc *crtc)
 	ipu_di_enable(ipu_crtc->di);
 }
 
-static void ipu_crtc_disable(struct drm_crtc *crtc)
+static void ipu_crtc_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_crtc_state)
 {
 	struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
 	struct ipu_soc *ipu = dev_get_drvdata(ipu_crtc->dev->parent);
@@ -76,6 +76,11 @@ static void ipu_crtc_disable(struct drm_crtc *crtc)
 		crtc->state->event = NULL;
 	}
 	spin_unlock_irq(&crtc->dev->event_lock);
+
+	/* always disable planes on the CRTC */
+	drm_atomic_helper_disable_planes_on_crtc(old_crtc_state, true);
+
+	drm_crtc_vblank_off(crtc);
 }
 
 static void imx_drm_crtc_reset(struct drm_crtc *crtc)
@@ -121,9 +126,14 @@ static void imx_drm_crtc_destroy_state(struct drm_crtc *crtc,
 	kfree(to_imx_crtc_state(state));
 }
 
+static void imx_drm_crtc_destroy(struct drm_crtc *crtc)
+{
+	imx_drm_remove_crtc(to_ipu_crtc(crtc)->imx_crtc);
+}
+
 static const struct drm_crtc_funcs ipu_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
-	.destroy = drm_crtc_cleanup,
+	.destroy = imx_drm_crtc_destroy,
 	.page_flip = drm_atomic_helper_page_flip,
 	.reset = imx_drm_crtc_reset,
 	.atomic_duplicate_state = imx_drm_crtc_duplicate_state,
@@ -134,7 +144,7 @@ static irqreturn_t ipu_irq_handler(int irq, void *dev_id)
 {
 	struct ipu_crtc *ipu_crtc = dev_id;
 
-	imx_drm_handle_vblank(ipu_crtc->imx_crtc);
+	drm_crtc_handle_vblank(&ipu_crtc->base);
 
 	return IRQ_HANDLED;
 }
@@ -175,6 +185,8 @@ static int ipu_crtc_atomic_check(struct drm_crtc *crtc,
 static void ipu_crtc_atomic_begin(struct drm_crtc *crtc,
 				  struct drm_crtc_state *old_crtc_state)
 {
+	drm_crtc_vblank_on(crtc);
+
 	spin_lock_irq(&crtc->dev->event_lock);
 	if (crtc->state->event) {
 		WARN_ON(drm_crtc_vblank_get(crtc));
@@ -242,7 +254,7 @@ static const struct drm_crtc_helper_funcs ipu_helper_funcs = {
 	.mode_set_nofb = ipu_crtc_mode_set_nofb,
 	.atomic_check = ipu_crtc_atomic_check,
 	.atomic_begin = ipu_crtc_atomic_begin,
-	.disable = ipu_crtc_disable,
+	.atomic_disable = ipu_crtc_atomic_disable,
 	.enable = ipu_crtc_enable,
 };
 
@@ -410,8 +422,6 @@ static void ipu_drm_unbind(struct device *dev, struct device *master,
 {
 	struct ipu_crtc *ipu_crtc = dev_get_drvdata(dev);
 
-	imx_drm_remove_crtc(ipu_crtc->imx_crtc);
-
 	ipu_put_resources(ipu_crtc);
 	if (ipu_crtc->plane[1])
 		ipu_plane_put_resources(ipu_crtc->plane[1]);
diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c
index 4ad67d015ec7..ce22d0a0ddc8 100644
--- a/drivers/gpu/drm/imx/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3-plane.c
@@ -213,8 +213,12 @@ static void ipu_plane_enable(struct ipu_plane *ipu_plane)
 		ipu_dp_enable_channel(ipu_plane->dp);
 }
 
-static void ipu_plane_disable(struct ipu_plane *ipu_plane)
+static int ipu_disable_plane(struct drm_plane *plane)
 {
+	struct ipu_plane *ipu_plane = to_ipu_plane(plane);
+
+	DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
+
 	ipu_idmac_wait_busy(ipu_plane->ipu_ch, 50);
 
 	if (ipu_plane->dp)
@@ -223,15 +227,6 @@ static void ipu_plane_disable(struct ipu_plane *ipu_plane)
 	ipu_dmfc_disable_channel(ipu_plane->dmfc);
 	if (ipu_plane->dp)
 		ipu_dp_disable(ipu_plane->ipu);
-}
-
-static int ipu_disable_plane(struct drm_plane *plane)
-{
-	struct ipu_plane *ipu_plane = to_ipu_plane(plane);
-
-	DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
-
-	ipu_plane_disable(ipu_plane);
 
 	return 0;
 }
@@ -242,7 +237,6 @@ static void ipu_plane_destroy(struct drm_plane *plane)
 
 	DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
 
-	ipu_disable_plane(plane);
 	drm_plane_cleanup(plane);
 	kfree(ipu_plane);
 }
@@ -319,13 +313,16 @@ static int ipu_plane_atomic_check(struct drm_plane *plane,
 		return -EINVAL;
 
 	/*
-	 * since we cannot touch active IDMAC channels, we do not support
-	 * resizing the enabled plane or changing its format
+	 * We support resizing active plane or changing its format by
+	 * forcing CRTC mode change in plane's ->atomic_check callback
+	 * and disabling all affected active planes in CRTC's ->atomic_disable
+	 * callback.  The planes will be reenabled in plane's ->atomic_update
+	 * callback.
 	 */
 	if (old_fb && (state->src_w != old_state->src_w ||
 			      state->src_h != old_state->src_h ||
 			      fb->pixel_format != old_fb->pixel_format))
-		return -EINVAL;
+		crtc_state->mode_changed = true;
 
 	eba = drm_plane_state_to_eba(state);
 
@@ -336,7 +333,7 @@ static int ipu_plane_atomic_check(struct drm_plane *plane,
 		return -EINVAL;
 
 	if (old_fb && fb->pitches[0] != old_fb->pitches[0])
-		return -EINVAL;
+		crtc_state->mode_changed = true;
 
 	switch (fb->pixel_format) {
 	case DRM_FORMAT_YUV420:
@@ -372,7 +369,7 @@ static int ipu_plane_atomic_check(struct drm_plane *plane,
 			return -EINVAL;
 
 		if (old_fb && old_fb->pitches[1] != fb->pitches[1])
-			return -EINVAL;
+			crtc_state->mode_changed = true;
 	}
 
 	return 0;
@@ -392,8 +389,12 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 	enum ipu_color_space ics;
 
 	if (old_state->fb) {
-		ipu_plane_atomic_set_base(ipu_plane, old_state);
-		return;
+		struct drm_crtc_state *crtc_state = state->crtc->state;
+
+		if (!drm_atomic_crtc_needs_modeset(crtc_state)) {
+			ipu_plane_atomic_set_base(ipu_plane, old_state);
+			return;
+		}
 	}
 
 	switch (ipu_plane->dp_flow) {
diff --git a/drivers/gpu/drm/imx/parallel-display.c b/drivers/gpu/drm/imx/parallel-display.c
index 1dad297b01fd..d796ada2a47a 100644
--- a/drivers/gpu/drm/imx/parallel-display.c
+++ b/drivers/gpu/drm/imx/parallel-display.c
@@ -33,6 +33,7 @@ struct imx_parallel_display {
 	void *edid;
 	int edid_len;
 	u32 bus_format;
+	u32 bus_flags;
 	struct drm_display_mode mode;
 	struct drm_panel *panel;
 	struct drm_bridge *bridge;
@@ -80,6 +81,7 @@ static int imx_pd_connector_get_modes(struct drm_connector *connector)
 			return -EINVAL;
 
 		ret = of_get_drm_display_mode(np, &imxpd->mode,
+					      &imxpd->bus_flags,
 					      OF_USE_NATIVE_MODE);
 		if (ret)
 			return ret;
@@ -125,11 +127,13 @@ static int imx_pd_encoder_atomic_check(struct drm_encoder *encoder,
 	struct drm_display_info *di = &conn_state->connector->display_info;
 	struct imx_parallel_display *imxpd = enc_to_imxpd(encoder);
 
-	imx_crtc_state->bus_flags = di->bus_flags;
-	if (!imxpd->bus_format && di->num_bus_formats)
+	if (!imxpd->bus_format && di->num_bus_formats) {
+		imx_crtc_state->bus_flags = di->bus_flags;
 		imx_crtc_state->bus_format = di->bus_formats[0];
-	else
+	} else {
+		imx_crtc_state->bus_flags = imxpd->bus_flags;
 		imx_crtc_state->bus_format = imxpd->bus_format;
+	}
 	imx_crtc_state->di_hsync_pin = 2;
 	imx_crtc_state->di_vsync_pin = 3;
 
@@ -289,8 +293,10 @@ static void imx_pd_unbind(struct device *dev, struct device *master,
 {
 	struct imx_parallel_display *imxpd = dev_get_drvdata(dev);
 
-	imxpd->encoder.funcs->destroy(&imxpd->encoder);
-	imxpd->connector.funcs->destroy(&imxpd->connector);
+	if (imxpd->bridge)
+		drm_bridge_detach(imxpd->bridge);
+	if (imxpd->panel)
+		drm_panel_detach(imxpd->panel);
 
 	kfree(imxpd->edid);
 }
diff --git a/drivers/gpu/drm/mediatek/Kconfig b/drivers/gpu/drm/mediatek/Kconfig
index 23ac8041c562..294de4549922 100644
--- a/drivers/gpu/drm/mediatek/Kconfig
+++ b/drivers/gpu/drm/mediatek/Kconfig
@@ -2,6 +2,9 @@ config DRM_MEDIATEK
 	tristate "DRM Support for Mediatek SoCs"
 	depends on DRM
 	depends on ARCH_MEDIATEK || (ARM && COMPILE_TEST)
+	depends on COMMON_CLK
+	depends on HAVE_ARM_SMCCC
+	depends on OF
 	select DRM_GEM_CMA_HELPER
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
index 8f62671fcfbf..019b7ca392d7 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
@@ -103,7 +103,8 @@ static void mtk_ovl_stop(struct mtk_ddp_comp *comp)
 }
 
 static void mtk_ovl_config(struct mtk_ddp_comp *comp, unsigned int w,
-			   unsigned int h, unsigned int vrefresh)
+			   unsigned int h, unsigned int vrefresh,
+			   unsigned int bpc)
 {
 	if (w != 0 && h != 0)
 		writel_relaxed(h << 16 | w, comp->regs + DISP_REG_OVL_ROI_SIZE);
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
index 5fb80cbe4c5b..0df05f95b916 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
@@ -106,7 +106,8 @@ static void mtk_rdma_stop(struct mtk_ddp_comp *comp)
 }
 
 static void mtk_rdma_config(struct mtk_ddp_comp *comp, unsigned int width,
-			    unsigned int height, unsigned int vrefresh)
+			    unsigned int height, unsigned int vrefresh,
+			    unsigned int bpc)
 {
 	unsigned int threshold;
 	unsigned int reg;
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
index 24aa3bad1e76..01a21dd835b5 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
@@ -31,7 +31,7 @@
  * struct mtk_drm_crtc - MediaTek specific crtc structure.
  * @base: crtc object.
  * @enabled: records whether crtc_enable succeeded
- * @planes: array of 4 mtk_drm_plane structures, one for each overlay plane
+ * @planes: array of 4 drm_plane structures, one for each overlay plane
  * @pending_planes: whether any plane has pending changes to be applied
  * @config_regs: memory mapped mmsys configuration register space
  * @mutex: handle to one of the ten disp_mutex streams
@@ -45,7 +45,7 @@ struct mtk_drm_crtc {
 	bool				pending_needs_vblank;
 	struct drm_pending_vblank_event	*event;
 
-	struct mtk_drm_plane		planes[OVL_LAYER_NR];
+	struct drm_plane		planes[OVL_LAYER_NR];
 	bool				pending_planes;
 
 	void __iomem			*config_regs;
@@ -112,8 +112,7 @@ static void mtk_drm_crtc_reset(struct drm_crtc *crtc)
 	struct mtk_crtc_state *state;
 
 	if (crtc->state) {
-		if (crtc->state->mode_blob)
-			drm_property_unreference_blob(crtc->state->mode_blob);
+		__drm_atomic_helper_crtc_destroy_state(crtc->state);
 
 		state = to_mtk_crtc_state(crtc->state);
 		memset(state, 0, sizeof(*state));
@@ -222,7 +221,9 @@ static void mtk_crtc_ddp_clk_disable(struct mtk_drm_crtc *mtk_crtc)
 static int mtk_crtc_ddp_hw_init(struct mtk_drm_crtc *mtk_crtc)
 {
 	struct drm_crtc *crtc = &mtk_crtc->base;
-	unsigned int width, height, vrefresh;
+	struct drm_connector *connector;
+	struct drm_encoder *encoder;
+	unsigned int width, height, vrefresh, bpc = MTK_MAX_BPC;
 	int ret;
 	int i;
 
@@ -234,6 +235,19 @@ static int mtk_crtc_ddp_hw_init(struct mtk_drm_crtc *mtk_crtc)
 	height = crtc->state->adjusted_mode.vdisplay;
 	vrefresh = crtc->state->adjusted_mode.vrefresh;
 
+	drm_for_each_encoder(encoder, crtc->dev) {
+		if (encoder->crtc != crtc)
+			continue;
+
+		drm_for_each_connector(connector, crtc->dev) {
+			if (connector->encoder != encoder)
+				continue;
+			if (connector->display_info.bpc != 0 &&
+			    bpc > connector->display_info.bpc)
+				bpc = connector->display_info.bpc;
+		}
+	}
+
 	ret = pm_runtime_get_sync(crtc->dev->dev);
 	if (ret < 0) {
 		DRM_ERROR("Failed to enable power domain: %d\n", ret);
@@ -266,13 +280,13 @@ static int mtk_crtc_ddp_hw_init(struct mtk_drm_crtc *mtk_crtc)
 	for (i = 0; i < mtk_crtc->ddp_comp_nr; i++) {
 		struct mtk_ddp_comp *comp = mtk_crtc->ddp_comp[i];
 
-		mtk_ddp_comp_config(comp, width, height, vrefresh);
+		mtk_ddp_comp_config(comp, width, height, vrefresh, bpc);
 		mtk_ddp_comp_start(comp);
 	}
 
 	/* Initially configure all planes */
 	for (i = 0; i < OVL_LAYER_NR; i++) {
-		struct drm_plane *plane = &mtk_crtc->planes[i].base;
+		struct drm_plane *plane = &mtk_crtc->planes[i];
 		struct mtk_plane_state *plane_state;
 
 		plane_state = to_mtk_plane_state(plane->state);
@@ -351,7 +365,7 @@ static void mtk_drm_crtc_disable(struct drm_crtc *crtc)
 
 	/* Set all pending plane state to disabled */
 	for (i = 0; i < OVL_LAYER_NR; i++) {
-		struct drm_plane *plane = &mtk_crtc->planes[i].base;
+		struct drm_plane *plane = &mtk_crtc->planes[i];
 		struct mtk_plane_state *plane_state;
 
 		plane_state = to_mtk_plane_state(plane->state);
@@ -397,7 +411,7 @@ static void mtk_drm_crtc_atomic_flush(struct drm_crtc *crtc,
 	if (mtk_crtc->event)
 		mtk_crtc->pending_needs_vblank = true;
 	for (i = 0; i < OVL_LAYER_NR; i++) {
-		struct drm_plane *plane = &mtk_crtc->planes[i].base;
+		struct drm_plane *plane = &mtk_crtc->planes[i];
 		struct mtk_plane_state *plane_state;
 
 		plane_state = to_mtk_plane_state(plane->state);
@@ -409,6 +423,9 @@ static void mtk_drm_crtc_atomic_flush(struct drm_crtc *crtc,
 	}
 	if (pending_planes)
 		mtk_crtc->pending_planes = true;
+	if (crtc->state->color_mgmt_changed)
+		for (i = 0; i < mtk_crtc->ddp_comp_nr; i++)
+			mtk_ddp_gamma_set(mtk_crtc->ddp_comp[i], crtc->state);
 }
 
 static const struct drm_crtc_funcs mtk_crtc_funcs = {
@@ -418,6 +435,7 @@ static const struct drm_crtc_funcs mtk_crtc_funcs = {
 	.reset			= mtk_drm_crtc_reset,
 	.atomic_duplicate_state	= mtk_drm_crtc_duplicate_state,
 	.atomic_destroy_state	= mtk_drm_crtc_destroy_state,
+	.gamma_set		= drm_atomic_helper_legacy_gamma_set,
 };
 
 static const struct drm_crtc_helper_funcs mtk_crtc_helper_funcs = {
@@ -464,14 +482,14 @@ void mtk_crtc_ddp_irq(struct drm_crtc *crtc, struct mtk_ddp_comp *ovl)
 	if (state->pending_config) {
 		mtk_ddp_comp_config(ovl, state->pending_width,
 				    state->pending_height,
-				    state->pending_vrefresh);
+				    state->pending_vrefresh, 0);
 
 		state->pending_config = false;
 	}
 
 	if (mtk_crtc->pending_planes) {
 		for (i = 0; i < OVL_LAYER_NR; i++) {
-			struct drm_plane *plane = &mtk_crtc->planes[i].base;
+			struct drm_plane *plane = &mtk_crtc->planes[i];
 			struct mtk_plane_state *plane_state;
 
 			plane_state = to_mtk_plane_state(plane->state);
@@ -559,16 +577,17 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 				(zpos == 1) ? DRM_PLANE_TYPE_CURSOR :
 						DRM_PLANE_TYPE_OVERLAY;
 		ret = mtk_plane_init(drm_dev, &mtk_crtc->planes[zpos],
-				     BIT(pipe), type, zpos);
+				     BIT(pipe), type);
 		if (ret)
 			goto unprepare;
 	}
 
-	ret = mtk_drm_crtc_init(drm_dev, mtk_crtc, &mtk_crtc->planes[0].base,
-				&mtk_crtc->planes[1].base, pipe);
+	ret = mtk_drm_crtc_init(drm_dev, mtk_crtc, &mtk_crtc->planes[0],
+				&mtk_crtc->planes[1], pipe);
 	if (ret < 0)
 		goto unprepare;
-
+	drm_mode_crtc_set_gamma_size(&mtk_crtc->base, MTK_LUT_SIZE);
+	drm_crtc_enable_color_mgmt(&mtk_crtc->base, 0, false, MTK_LUT_SIZE);
 	priv->crtc[pipe] = &mtk_crtc->base;
 	priv->num_pipes++;
 
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.h b/drivers/gpu/drm/mediatek/mtk_drm_crtc.h
index 81e5566ec82f..a1550fa3c9d2 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.h
@@ -19,10 +19,12 @@
 #include "mtk_drm_plane.h"
 
 #define OVL_LAYER_NR	4
+#define MTK_LUT_SIZE	512
+#define MTK_MAX_BPC	10
+#define MTK_MIN_BPC	3
 
 int mtk_drm_crtc_enable_vblank(struct drm_device *drm, unsigned int pipe);
 void mtk_drm_crtc_disable_vblank(struct drm_device *drm, unsigned int pipe);
-void mtk_drm_crtc_check_flush(struct drm_crtc *crtc);
 void mtk_drm_crtc_commit(struct drm_crtc *crtc);
 void mtk_crtc_ddp_irq(struct drm_crtc *crtc, struct mtk_ddp_comp *ovl);
 int mtk_drm_crtc_create(struct drm_device *drm_dev,
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
index 3970fcf0f05f..df33b3ca6ffd 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
@@ -24,12 +24,17 @@
 #include "mtk_drm_drv.h"
 #include "mtk_drm_plane.h"
 #include "mtk_drm_ddp_comp.h"
+#include "mtk_drm_crtc.h"
 
 #define DISP_OD_EN				0x0000
 #define DISP_OD_INTEN				0x0008
 #define DISP_OD_INTSTA				0x000c
 #define DISP_OD_CFG				0x0020
 #define DISP_OD_SIZE				0x0030
+#define DISP_DITHER_5				0x0114
+#define DISP_DITHER_7				0x011c
+#define DISP_DITHER_15				0x013c
+#define DISP_DITHER_16				0x0140
 
 #define DISP_REG_UFO_START			0x0000
 
@@ -38,15 +43,69 @@
 #define DISP_COLOR_WIDTH			0x0c50
 #define DISP_COLOR_HEIGHT			0x0c54
 
-#define	OD_RELAY_MODE		BIT(0)
+#define DISP_AAL_EN				0x0000
+#define DISP_AAL_SIZE				0x0030
 
-#define	UFO_BYPASS		BIT(2)
+#define DISP_GAMMA_EN				0x0000
+#define DISP_GAMMA_CFG				0x0020
+#define DISP_GAMMA_SIZE				0x0030
+#define DISP_GAMMA_LUT				0x0700
 
-#define	COLOR_BYPASS_ALL	BIT(7)
-#define	COLOR_SEQ_SEL		BIT(13)
+#define LUT_10BIT_MASK				0x03ff
+
+#define COLOR_BYPASS_ALL			BIT(7)
+#define COLOR_SEQ_SEL				BIT(13)
+
+#define OD_RELAYMODE				BIT(0)
+
+#define UFO_BYPASS				BIT(2)
+
+#define AAL_EN					BIT(0)
+
+#define GAMMA_EN				BIT(0)
+#define GAMMA_LUT_EN				BIT(1)
+
+#define DISP_DITHERING				BIT(2)
+#define DITHER_LSB_ERR_SHIFT_R(x)		(((x) & 0x7) << 28)
+#define DITHER_OVFLW_BIT_R(x)			(((x) & 0x7) << 24)
+#define DITHER_ADD_LSHIFT_R(x)			(((x) & 0x7) << 20)
+#define DITHER_ADD_RSHIFT_R(x)			(((x) & 0x7) << 16)
+#define DITHER_NEW_BIT_MODE			BIT(0)
+#define DITHER_LSB_ERR_SHIFT_B(x)		(((x) & 0x7) << 28)
+#define DITHER_OVFLW_BIT_B(x)			(((x) & 0x7) << 24)
+#define DITHER_ADD_LSHIFT_B(x)			(((x) & 0x7) << 20)
+#define DITHER_ADD_RSHIFT_B(x)			(((x) & 0x7) << 16)
+#define DITHER_LSB_ERR_SHIFT_G(x)		(((x) & 0x7) << 12)
+#define DITHER_OVFLW_BIT_G(x)			(((x) & 0x7) << 8)
+#define DITHER_ADD_LSHIFT_G(x)			(((x) & 0x7) << 4)
+#define DITHER_ADD_RSHIFT_G(x)			(((x) & 0x7) << 0)
+
+void mtk_dither_set(struct mtk_ddp_comp *comp, unsigned int bpc,
+		    unsigned int CFG)
+{
+	/* If bpc equal to 0, the dithering function didn't be enabled */
+	if (bpc == 0)
+		return;
+
+	if (bpc >= MTK_MIN_BPC) {
+		writel(0, comp->regs + DISP_DITHER_5);
+		writel(0, comp->regs + DISP_DITHER_7);
+		writel(DITHER_LSB_ERR_SHIFT_R(MTK_MAX_BPC - bpc) |
+		       DITHER_ADD_LSHIFT_R(MTK_MAX_BPC - bpc) |
+		       DITHER_NEW_BIT_MODE,
+		       comp->regs + DISP_DITHER_15);
+		writel(DITHER_LSB_ERR_SHIFT_B(MTK_MAX_BPC - bpc) |
+		       DITHER_ADD_LSHIFT_B(MTK_MAX_BPC - bpc) |
+		       DITHER_LSB_ERR_SHIFT_G(MTK_MAX_BPC - bpc) |
+		       DITHER_ADD_LSHIFT_G(MTK_MAX_BPC - bpc),
+		       comp->regs + DISP_DITHER_16);
+		writel(DISP_DITHERING, comp->regs + CFG);
+	}
+}
 
 static void mtk_color_config(struct mtk_ddp_comp *comp, unsigned int w,
-			     unsigned int h, unsigned int vrefresh)
+			     unsigned int h, unsigned int vrefresh,
+			     unsigned int bpc)
 {
 	writel(w, comp->regs + DISP_COLOR_WIDTH);
 	writel(h, comp->regs + DISP_COLOR_HEIGHT);
@@ -60,14 +119,16 @@ static void mtk_color_start(struct mtk_ddp_comp *comp)
 }
 
 static void mtk_od_config(struct mtk_ddp_comp *comp, unsigned int w,
-			  unsigned int h, unsigned int vrefresh)
+			  unsigned int h, unsigned int vrefresh,
+			  unsigned int bpc)
 {
 	writel(w << 16 | h, comp->regs + DISP_OD_SIZE);
+	writel(OD_RELAYMODE, comp->regs + OD_RELAYMODE);
+	mtk_dither_set(comp, bpc, DISP_OD_CFG);
 }
 
 static void mtk_od_start(struct mtk_ddp_comp *comp)
 {
-	writel(OD_RELAY_MODE, comp->regs + DISP_OD_CFG);
 	writel(1, comp->regs + DISP_OD_EN);
 }
 
@@ -76,6 +137,78 @@ static void mtk_ufoe_start(struct mtk_ddp_comp *comp)
 	writel(UFO_BYPASS, comp->regs + DISP_REG_UFO_START);
 }
 
+static void mtk_aal_config(struct mtk_ddp_comp *comp, unsigned int w,
+			   unsigned int h, unsigned int vrefresh,
+			   unsigned int bpc)
+{
+	writel(h << 16 | w, comp->regs + DISP_AAL_SIZE);
+}
+
+static void mtk_aal_start(struct mtk_ddp_comp *comp)
+{
+	writel(AAL_EN, comp->regs + DISP_AAL_EN);
+}
+
+static void mtk_aal_stop(struct mtk_ddp_comp *comp)
+{
+	writel_relaxed(0x0, comp->regs + DISP_AAL_EN);
+}
+
+static void mtk_gamma_config(struct mtk_ddp_comp *comp, unsigned int w,
+			     unsigned int h, unsigned int vrefresh,
+			     unsigned int bpc)
+{
+	writel(h << 16 | w, comp->regs + DISP_GAMMA_SIZE);
+	mtk_dither_set(comp, bpc, DISP_GAMMA_CFG);
+}
+
+static void mtk_gamma_start(struct mtk_ddp_comp *comp)
+{
+	writel(GAMMA_EN, comp->regs  + DISP_GAMMA_EN);
+}
+
+static void mtk_gamma_stop(struct mtk_ddp_comp *comp)
+{
+	writel_relaxed(0x0, comp->regs  + DISP_GAMMA_EN);
+}
+
+static void mtk_gamma_set(struct mtk_ddp_comp *comp,
+			  struct drm_crtc_state *state)
+{
+	unsigned int i, reg;
+	struct drm_color_lut *lut;
+	void __iomem *lut_base;
+	u32 word;
+
+	if (state->gamma_lut) {
+		reg = readl(comp->regs + DISP_GAMMA_CFG);
+		reg = reg | GAMMA_LUT_EN;
+		writel(reg, comp->regs + DISP_GAMMA_CFG);
+		lut_base = comp->regs + DISP_GAMMA_LUT;
+		lut = (struct drm_color_lut *)state->gamma_lut->data;
+		for (i = 0; i < MTK_LUT_SIZE; i++) {
+			word = (((lut[i].red >> 6) & LUT_10BIT_MASK) << 20) +
+				(((lut[i].green >> 6) & LUT_10BIT_MASK) << 10) +
+				((lut[i].blue >> 6) & LUT_10BIT_MASK);
+			writel(word, (lut_base + i * 4));
+		}
+	}
+}
+
+static const struct mtk_ddp_comp_funcs ddp_aal = {
+	.gamma_set = mtk_gamma_set,
+	.config = mtk_aal_config,
+	.start = mtk_aal_start,
+	.stop = mtk_aal_stop,
+};
+
+static const struct mtk_ddp_comp_funcs ddp_gamma = {
+	.gamma_set = mtk_gamma_set,
+	.config = mtk_gamma_config,
+	.start = mtk_gamma_start,
+	.stop = mtk_gamma_stop,
+};
+
 static const struct mtk_ddp_comp_funcs ddp_color = {
 	.config = mtk_color_config,
 	.start = mtk_color_start,
@@ -112,13 +245,13 @@ struct mtk_ddp_comp_match {
 };
 
 static const struct mtk_ddp_comp_match mtk_ddp_matches[DDP_COMPONENT_ID_MAX] = {
-	[DDP_COMPONENT_AAL]	= { MTK_DISP_AAL,	0, NULL },
+	[DDP_COMPONENT_AAL]	= { MTK_DISP_AAL,	0, &ddp_aal },
 	[DDP_COMPONENT_COLOR0]	= { MTK_DISP_COLOR,	0, &ddp_color },
 	[DDP_COMPONENT_COLOR1]	= { MTK_DISP_COLOR,	1, &ddp_color },
 	[DDP_COMPONENT_DPI0]	= { MTK_DPI,		0, NULL },
 	[DDP_COMPONENT_DSI0]	= { MTK_DSI,		0, NULL },
 	[DDP_COMPONENT_DSI1]	= { MTK_DSI,		1, NULL },
-	[DDP_COMPONENT_GAMMA]	= { MTK_DISP_GAMMA,	0, NULL },
+	[DDP_COMPONENT_GAMMA]	= { MTK_DISP_GAMMA,	0, &ddp_gamma },
 	[DDP_COMPONENT_OD]	= { MTK_DISP_OD,	0, &ddp_od },
 	[DDP_COMPONENT_OVL0]	= { MTK_DISP_OVL,	0, NULL },
 	[DDP_COMPONENT_OVL1]	= { MTK_DISP_OVL,	1, NULL },
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
index 6b13ba97094d..22a33ee451c4 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.h
@@ -21,6 +21,7 @@ struct device_node;
 struct drm_crtc;
 struct drm_device;
 struct mtk_plane_state;
+struct drm_crtc_state;
 
 enum mtk_ddp_comp_type {
 	MTK_DISP_OVL,
@@ -64,7 +65,7 @@ struct mtk_ddp_comp;
 
 struct mtk_ddp_comp_funcs {
 	void (*config)(struct mtk_ddp_comp *comp, unsigned int w,
-		       unsigned int h, unsigned int vrefresh);
+		       unsigned int h, unsigned int vrefresh, unsigned int bpc);
 	void (*start)(struct mtk_ddp_comp *comp);
 	void (*stop)(struct mtk_ddp_comp *comp);
 	void (*enable_vblank)(struct mtk_ddp_comp *comp, struct drm_crtc *crtc);
@@ -73,6 +74,8 @@ struct mtk_ddp_comp_funcs {
 	void (*layer_off)(struct mtk_ddp_comp *comp, unsigned int idx);
 	void (*layer_config)(struct mtk_ddp_comp *comp, unsigned int idx,
 			     struct mtk_plane_state *state);
+	void (*gamma_set)(struct mtk_ddp_comp *comp,
+			  struct drm_crtc_state *state);
 };
 
 struct mtk_ddp_comp {
@@ -86,10 +89,10 @@ struct mtk_ddp_comp {
 
 static inline void mtk_ddp_comp_config(struct mtk_ddp_comp *comp,
 				       unsigned int w, unsigned int h,
-				       unsigned int vrefresh)
+				       unsigned int vrefresh, unsigned int bpc)
 {
 	if (comp->funcs && comp->funcs->config)
-		comp->funcs->config(comp, w, h, vrefresh);
+		comp->funcs->config(comp, w, h, vrefresh, bpc);
 }
 
 static inline void mtk_ddp_comp_start(struct mtk_ddp_comp *comp)
@@ -139,6 +142,13 @@ static inline void mtk_ddp_comp_layer_config(struct mtk_ddp_comp *comp,
 		comp->funcs->layer_config(comp, idx, state);
 }
 
+static inline void mtk_ddp_gamma_set(struct mtk_ddp_comp *comp,
+				     struct drm_crtc_state *state)
+{
+	if (comp->funcs && comp->funcs->gamma_set)
+		comp->funcs->gamma_set(comp, state);
+}
+
 int mtk_ddp_comp_get_id(struct device_node *node,
 			enum mtk_ddp_comp_type comp_type);
 int mtk_ddp_comp_init(struct device *dev, struct device_node *comp_node,
@@ -146,5 +156,7 @@ int mtk_ddp_comp_init(struct device *dev, struct device_node *comp_node,
 		      const struct mtk_ddp_comp_funcs *funcs);
 int mtk_ddp_comp_register(struct drm_device *drm, struct mtk_ddp_comp *comp);
 void mtk_ddp_comp_unregister(struct drm_device *drm, struct mtk_ddp_comp *comp);
+void mtk_dither_set(struct mtk_ddp_comp *comp, unsigned int bpc,
+		    unsigned int CFG);
 
 #endif /* MTK_DRM_DDP_COMP_H */
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
index eebb7d881c2b..cf83f6507ec8 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
@@ -61,10 +61,27 @@ static void mtk_atomic_complete(struct mtk_drm_private *private,
 
 	mtk_atomic_wait_for_fences(state);
 
+	/*
+	 * Mediatek drm supports runtime PM, so plane registers cannot be
+	 * written when their crtc is disabled.
+	 *
+	 * The comment for drm_atomic_helper_commit states:
+	 *     For drivers supporting runtime PM the recommended sequence is
+	 *
+	 *     drm_atomic_helper_commit_modeset_disables(dev, state);
+	 *     drm_atomic_helper_commit_modeset_enables(dev, state);
+	 *     drm_atomic_helper_commit_planes(dev, state,
+	 *                                     DRM_PLANE_COMMIT_ACTIVE_ONLY);
+	 *
+	 * See the kerneldoc entries for these three functions for more details.
+	 */
 	drm_atomic_helper_commit_modeset_disables(drm, state);
-	drm_atomic_helper_commit_planes(drm, state, false);
 	drm_atomic_helper_commit_modeset_enables(drm, state);
+	drm_atomic_helper_commit_planes(drm, state,
+					DRM_PLANE_COMMIT_ACTIVE_ONLY);
+
 	drm_atomic_helper_wait_for_vblanks(drm, state);
+
 	drm_atomic_helper_cleanup_planes(drm, state);
 	drm_atomic_state_free(state);
 }
@@ -277,8 +294,8 @@ static int mtk_drm_bind(struct device *dev)
 	int ret;
 
 	drm = drm_dev_alloc(&mtk_drm_driver, dev);
-	if (!drm)
-		return -ENOMEM;
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
 
 	drm->dev_private = private;
 	private->drm = drm;
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.c b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
index 3995765a90dc..c461a232cbf5 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
@@ -30,57 +30,12 @@ static const u32 formats[] = {
 	DRM_FORMAT_RGB565,
 };
 
-static void mtk_plane_enable(struct mtk_drm_plane *mtk_plane, bool enable,
-			     dma_addr_t addr, struct drm_rect *dest)
-{
-	struct drm_plane *plane = &mtk_plane->base;
-	struct mtk_plane_state *state = to_mtk_plane_state(plane->state);
-	unsigned int pitch, format;
-	int x, y;
-
-	if (WARN_ON(!plane->state || (enable && !plane->state->fb)))
-		return;
-
-	if (plane->state->fb) {
-		pitch = plane->state->fb->pitches[0];
-		format = plane->state->fb->pixel_format;
-	} else {
-		pitch = 0;
-		format = DRM_FORMAT_RGBA8888;
-	}
-
-	x = plane->state->crtc_x;
-	y = plane->state->crtc_y;
-
-	if (x < 0) {
-		addr -= x * 4;
-		x = 0;
-	}
-
-	if (y < 0) {
-		addr -= y * pitch;
-		y = 0;
-	}
-
-	state->pending.enable = enable;
-	state->pending.pitch = pitch;
-	state->pending.format = format;
-	state->pending.addr = addr;
-	state->pending.x = x;
-	state->pending.y = y;
-	state->pending.width = dest->x2 - dest->x1;
-	state->pending.height = dest->y2 - dest->y1;
-	wmb(); /* Make sure the above parameters are set before update */
-	state->pending.dirty = true;
-}
-
 static void mtk_plane_reset(struct drm_plane *plane)
 {
 	struct mtk_plane_state *state;
 
 	if (plane->state) {
-		if (plane->state->fb)
-			drm_framebuffer_unreference(plane->state->fb);
+		__drm_atomic_helper_plane_destroy_state(plane->state);
 
 		state = to_mtk_plane_state(plane->state);
 		memset(state, 0, sizeof(*state));
@@ -134,20 +89,6 @@ static int mtk_plane_atomic_check(struct drm_plane *plane,
 {
 	struct drm_framebuffer *fb = state->fb;
 	struct drm_crtc_state *crtc_state;
-	bool visible;
-	struct drm_rect dest = {
-		.x1 = state->crtc_x,
-		.y1 = state->crtc_y,
-		.x2 = state->crtc_x + state->crtc_w,
-		.y2 = state->crtc_y + state->crtc_h,
-	};
-	struct drm_rect src = {
-		/* 16.16 fixed point */
-		.x1 = state->src_x,
-		.y1 = state->src_y,
-		.x2 = state->src_x + state->src_w,
-		.y2 = state->src_y + state->src_h,
-	};
 	struct drm_rect clip = { 0, };
 
 	if (!fb)
@@ -168,40 +109,45 @@ static int mtk_plane_atomic_check(struct drm_plane *plane,
 	clip.x2 = crtc_state->mode.hdisplay;
 	clip.y2 = crtc_state->mode.vdisplay;
 
-	return drm_plane_helper_check_update(plane, state->crtc, fb,
-					     &src, &dest, &clip,
-					     state->rotation,
-					     DRM_PLANE_HELPER_NO_SCALING,
-					     DRM_PLANE_HELPER_NO_SCALING,
-					     true, true, &visible);
+	return drm_plane_helper_check_state(state, &clip,
+					    DRM_PLANE_HELPER_NO_SCALING,
+					    DRM_PLANE_HELPER_NO_SCALING,
+					    true, true);
 }
 
 static void mtk_plane_atomic_update(struct drm_plane *plane,
 				    struct drm_plane_state *old_state)
 {
 	struct mtk_plane_state *state = to_mtk_plane_state(plane->state);
-	struct drm_crtc *crtc = state->base.crtc;
+	struct drm_crtc *crtc = plane->state->crtc;
+	struct drm_framebuffer *fb = plane->state->fb;
 	struct drm_gem_object *gem;
 	struct mtk_drm_gem_obj *mtk_gem;
-	struct mtk_drm_plane *mtk_plane = to_mtk_plane(plane);
-	struct drm_rect dest = {
-		.x1 = state->base.crtc_x,
-		.y1 = state->base.crtc_y,
-		.x2 = state->base.crtc_x + state->base.crtc_w,
-		.y2 = state->base.crtc_y + state->base.crtc_h,
-	};
-	struct drm_rect clip = { 0, };
+	unsigned int pitch, format;
+	dma_addr_t addr;
 
-	if (!crtc)
+	if (!crtc || WARN_ON(!fb))
 		return;
 
-	clip.x2 = state->base.crtc->state->mode.hdisplay;
-	clip.y2 = state->base.crtc->state->mode.vdisplay;
-	drm_rect_intersect(&dest, &clip);
-
-	gem = mtk_fb_get_gem_obj(state->base.fb);
+	gem = mtk_fb_get_gem_obj(fb);
 	mtk_gem = to_mtk_gem_obj(gem);
-	mtk_plane_enable(mtk_plane, true, mtk_gem->dma_addr, &dest);
+	addr = mtk_gem->dma_addr;
+	pitch = fb->pitches[0];
+	format = fb->pixel_format;
+
+	addr += (plane->state->src.x1 >> 16) * drm_format_plane_cpp(format, 0);
+	addr += (plane->state->src.y1 >> 16) * pitch;
+
+	state->pending.enable = true;
+	state->pending.pitch = pitch;
+	state->pending.format = format;
+	state->pending.addr = addr;
+	state->pending.x = plane->state->dst.x1;
+	state->pending.y = plane->state->dst.y1;
+	state->pending.width = drm_rect_width(&plane->state->dst);
+	state->pending.height = drm_rect_height(&plane->state->dst);
+	wmb(); /* Make sure the above parameters are set before update */
+	state->pending.dirty = true;
 }
 
 static void mtk_plane_atomic_disable(struct drm_plane *plane,
@@ -220,13 +166,12 @@ static const struct drm_plane_helper_funcs mtk_plane_helper_funcs = {
 	.atomic_disable = mtk_plane_atomic_disable,
 };
 
-int mtk_plane_init(struct drm_device *dev, struct mtk_drm_plane *mtk_plane,
-		   unsigned long possible_crtcs, enum drm_plane_type type,
-		   unsigned int zpos)
+int mtk_plane_init(struct drm_device *dev, struct drm_plane *plane,
+		   unsigned long possible_crtcs, enum drm_plane_type type)
 {
 	int err;
 
-	err = drm_universal_plane_init(dev, &mtk_plane->base, possible_crtcs,
+	err = drm_universal_plane_init(dev, plane, possible_crtcs,
 				       &mtk_plane_funcs, formats,
 				       ARRAY_SIZE(formats), type, NULL);
 	if (err) {
@@ -234,8 +179,7 @@ int mtk_plane_init(struct drm_device *dev, struct mtk_drm_plane *mtk_plane,
 		return err;
 	}
 
-	drm_plane_helper_add(&mtk_plane->base, &mtk_plane_helper_funcs);
-	mtk_plane->idx = zpos;
+	drm_plane_helper_add(plane, &mtk_plane_helper_funcs);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.h b/drivers/gpu/drm/mediatek/mtk_drm_plane.h
index 72a7b3e4c126..6a20b49e0f2f 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.h
@@ -18,11 +18,6 @@
 #include <drm/drm_crtc.h>
 #include <linux/types.h>
 
-struct mtk_drm_plane {
-	struct drm_plane		base;
-	unsigned int			idx;
-};
-
 struct mtk_plane_pending_state {
 	bool				config;
 	bool				enable;
@@ -41,19 +36,13 @@ struct mtk_plane_state {
 	struct mtk_plane_pending_state	pending;
 };
 
-static inline struct mtk_drm_plane *to_mtk_plane(struct drm_plane *plane)
-{
-	return container_of(plane, struct mtk_drm_plane, base);
-}
-
 static inline struct mtk_plane_state *
 to_mtk_plane_state(struct drm_plane_state *state)
 {
 	return container_of(state, struct mtk_plane_state, base);
 }
 
-int mtk_plane_init(struct drm_device *dev, struct mtk_drm_plane *mtk_plane,
-		   unsigned long possible_crtcs, enum drm_plane_type type,
-		   unsigned int zpos);
+int mtk_plane_init(struct drm_device *dev, struct drm_plane *plane,
+		   unsigned long possible_crtcs, enum drm_plane_type type);
 
 #endif
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
index 334562d06731..71227deef21b 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
@@ -1086,20 +1086,20 @@ static int mtk_hdmi_output_init(struct mtk_hdmi *hdmi)
 	return 0;
 }
 
-void mtk_hdmi_audio_enable(struct mtk_hdmi *hdmi)
+static void mtk_hdmi_audio_enable(struct mtk_hdmi *hdmi)
 {
 	mtk_hdmi_aud_enable_packet(hdmi, true);
 	hdmi->audio_enable = true;
 }
 
-void mtk_hdmi_audio_disable(struct mtk_hdmi *hdmi)
+static void mtk_hdmi_audio_disable(struct mtk_hdmi *hdmi)
 {
 	mtk_hdmi_aud_enable_packet(hdmi, false);
 	hdmi->audio_enable = false;
 }
 
-int mtk_hdmi_audio_set_param(struct mtk_hdmi *hdmi,
-			     struct hdmi_audio_param *param)
+static int mtk_hdmi_audio_set_param(struct mtk_hdmi *hdmi,
+				    struct hdmi_audio_param *param)
 {
 	if (!hdmi->audio_enable) {
 		dev_err(hdmi->dev, "hdmi audio is in disable state!\n");
@@ -1624,7 +1624,8 @@ static void mtk_hdmi_audio_shutdown(struct device *dev, void *data)
 	mtk_hdmi_audio_disable(hdmi);
 }
 
-int mtk_hdmi_audio_digital_mute(struct device *dev, void *data, bool enable)
+static int
+mtk_hdmi_audio_digital_mute(struct device *dev, void *data, bool enable)
 {
 	struct mtk_hdmi *hdmi = dev_get_drvdata(dev);
 
diff --git a/drivers/gpu/drm/mga/mga_drv.c b/drivers/gpu/drm/mga/mga_drv.c
index 5e2f131a6a72..25b2a1a424e6 100644
--- a/drivers/gpu/drm/mga/mga_drv.c
+++ b/drivers/gpu/drm/mga/mga_drv.c
@@ -58,7 +58,7 @@ static const struct file_operations mga_driver_fops = {
 
 static struct drm_driver driver = {
 	.driver_features =
-	    DRIVER_USE_AGP | DRIVER_PCI_DMA |
+	    DRIVER_USE_AGP | DRIVER_PCI_DMA | DRIVER_LEGACY |
 	    DRIVER_HAVE_DMA | DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED,
 	.dev_priv_size = sizeof(drm_mga_buf_priv_t),
 	.load = mga_driver_load,
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c
index 2b4b125eebc3..1443b3a34775 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.c
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.c
@@ -56,7 +56,7 @@ static void mgag200_kick_out_firmware_fb(struct pci_dev *pdev)
 #ifdef CONFIG_X86
 	primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
 #endif
-	remove_conflicting_framebuffers(ap, "mgag200drmfb", primary);
+	drm_fb_helper_remove_conflicting_framebuffers(ap, "mgag200drmfb", primary);
 	kfree(ap);
 }
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_fb.c b/drivers/gpu/drm/mgag200/mgag200_fb.c
index d9b04b008feb..88dd2214114d 100644
--- a/drivers/gpu/drm/mgag200/mgag200_fb.c
+++ b/drivers/gpu/drm/mgag200/mgag200_fb.c
@@ -15,8 +15,6 @@
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_crtc_helper.h>
 
-#include <linux/fb.h>
-
 #include "mgag200_drv.h"
 
 static void mga_dirty_update(struct mga_fbdev *mfbdev,
@@ -185,8 +183,10 @@ static int mgag200fb_create(struct drm_fb_helper *helper,
 	}
 
 	sysram = vmalloc(size);
-	if (!sysram)
+	if (!sysram) {
+		ret = -ENOMEM;
 		goto err_sysram;
+	}
 
 	info = drm_fb_helper_alloc_fbi(helper);
 	if (IS_ERR(info)) {
diff --git a/drivers/gpu/drm/mgag200/mgag200_main.c b/drivers/gpu/drm/mgag200/mgag200_main.c
index 13798b3e6beb..e79cbc25ae3c 100644
--- a/drivers/gpu/drm/mgag200/mgag200_main.c
+++ b/drivers/gpu/drm/mgag200/mgag200_main.c
@@ -135,7 +135,7 @@ static int mga_vram_init(struct mga_device *mdev)
 	aper->ranges[0].base = mdev->mc.vram_base;
 	aper->ranges[0].size = mdev->mc.vram_window;
 
-	remove_conflicting_framebuffers(aper, "mgafb", true);
+	drm_fb_helper_remove_conflicting_framebuffers(aper, "mgafb", true);
 	kfree(aper);
 
 	if (!devm_request_mem_region(mdev->dev->dev, mdev->mc.vram_base, mdev->mc.vram_window,
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index 68268e55d595..919b35f2ad24 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -150,7 +150,8 @@ static int mgag200_bo_verify_access(struct ttm_buffer_object *bo, struct file *f
 {
 	struct mgag200_bo *mgabo = mgag200_bo(bo);
 
-	return drm_vma_node_verify_access(&mgabo->gem.vma_node, filp);
+	return drm_vma_node_verify_access(&mgabo->gem.vma_node,
+					  filp->private_data);
 }
 
 static int mgag200_ttm_io_mem_reserve(struct ttm_bo_device *bdev,
diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index 7c7a0314a756..d96b2b6898a3 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -11,6 +11,7 @@ config DRM_MSM
 	select TMPFS
 	select QCOM_SCM
 	select SND_SOC_HDMI_CODEC if SND_SOC
+	select SYNC_FILE
 	default y
 	help
 	  DRM/KMS driver for MSM/snapdragon.
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c
index 973720792236..a968cad509c2 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.c
@@ -422,12 +422,29 @@ static const struct {
 
 static int msm_hdmi_get_gpio(struct device_node *of_node, const char *name)
 {
-	int gpio = of_get_named_gpio(of_node, name, 0);
+	int gpio;
+
+	/* try with the gpio names as in the table (downstream bindings) */
+	gpio = of_get_named_gpio(of_node, name, 0);
 	if (gpio < 0) {
 		char name2[32];
-		snprintf(name2, sizeof(name2), "%s-gpio", name);
+
+		/* try with the gpio names as in the upstream bindings */
+		snprintf(name2, sizeof(name2), "%s-gpios", name);
 		gpio = of_get_named_gpio(of_node, name2, 0);
 		if (gpio < 0) {
+			char name3[32];
+
+			/*
+			 * try again after stripping out the "qcom,hdmi-tx"
+			 * prefix. This is mainly to match "hpd-gpios" used
+			 * in the upstream bindings
+			 */
+			if (sscanf(name2, "qcom,hdmi-tx-%s", name3))
+				gpio = of_get_named_gpio(of_node, name3, 0);
+		}
+
+		if (gpio < 0) {
 			DBG("failed to get gpio: %s (%d)", name, gpio);
 			gpio = -1;
 		}
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_i2c.c b/drivers/gpu/drm/msm/hdmi/hdmi_i2c.c
index de9007e72f4e..73e20219d431 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_i2c.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_i2c.c
@@ -243,7 +243,6 @@ void msm_hdmi_i2c_destroy(struct i2c_adapter *i2c)
 
 struct i2c_adapter *msm_hdmi_i2c_init(struct hdmi *hdmi)
 {
-	struct drm_device *dev = hdmi->dev;
 	struct hdmi_i2c_adapter *hdmi_i2c;
 	struct i2c_adapter *i2c = NULL;
 	int ret;
@@ -267,10 +266,8 @@ struct i2c_adapter *msm_hdmi_i2c_init(struct hdmi *hdmi)
 	i2c->algo = &msm_hdmi_i2c_algorithm;
 
 	ret = i2c_add_adapter(i2c);
-	if (ret) {
-		dev_err(dev->dev, "failed to register hdmi i2c: %d\n", ret);
+	if (ret)
 		goto fail;
-	}
 
 	return i2c;
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
index 7b39e89fbc2b..571a91ee9607 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
@@ -228,18 +228,21 @@ static struct device_node *mdp4_detect_lcdc_panel(struct drm_device *dev)
 	struct device_node *endpoint, *panel_node;
 	struct device_node *np = dev->dev->of_node;
 
-	endpoint = of_graph_get_next_endpoint(np, NULL);
+	/*
+	 * LVDS/LCDC is the first port described in the list of ports in the
+	 * MDP4 DT node.
+	 */
+	endpoint = of_graph_get_endpoint_by_regs(np, 0, -1);
 	if (!endpoint) {
-		DBG("no endpoint in MDP4 to fetch LVDS panel\n");
+		DBG("no LVDS remote endpoint\n");
 		return NULL;
 	}
 
-	/* don't proceed if we have an endpoint but no panel_node tied to it */
 	panel_node = of_graph_get_remote_port_parent(endpoint);
 	if (!panel_node) {
-		dev_err(dev->dev, "no valid panel node\n");
+		DBG("no valid panel node in LVDS endpoint\n");
 		of_node_put(endpoint);
-		return ERR_PTR(-ENODEV);
+		return NULL;
 	}
 
 	of_node_put(endpoint);
@@ -262,14 +265,12 @@ static int mdp4_modeset_init_intf(struct mdp4_kms *mdp4_kms,
 	switch (intf_type) {
 	case DRM_MODE_ENCODER_LVDS:
 		/*
-		 * bail out early if:
-		 * - there is no panel node (no need to initialize lcdc
-		 *   encoder and lvds connector), or
-		 * - panel node is a bad pointer
+		 * bail out early if there is no panel node (no need to
+		 * initialize LCDC encoder and LVDS connector)
 		 */
 		panel_node = mdp4_detect_lcdc_panel(dev);
-		if (IS_ERR_OR_NULL(panel_node))
-			return PTR_ERR(panel_node);
+		if (!panel_node)
+			return 0;
 
 		encoder = mdp4_lcdc_encoder_init(dev, panel_node);
 		if (IS_ERR(encoder)) {
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lcdc_encoder.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lcdc_encoder.c
index bc3d8e719c6c..a06b064f86c1 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lcdc_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lcdc_encoder.c
@@ -93,7 +93,7 @@ static const struct drm_encoder_funcs mdp4_lcdc_encoder_funcs = {
 };
 
 /* this should probably be a helper: */
-struct drm_connector *get_connector(struct drm_encoder *encoder)
+static struct drm_connector *get_connector(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
 	struct drm_connector *connector;
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
index 9f96dfe67769..3903dbcda763 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
@@ -81,7 +81,7 @@ static void mdp4_plane_install_properties(struct drm_plane *plane,
 	// XXX
 }
 
-int mdp4_plane_set_property(struct drm_plane *plane,
+static int mdp4_plane_set_property(struct drm_plane *plane,
 		struct drm_property *property, uint64_t val)
 {
 	// XXX
@@ -99,7 +99,7 @@ static const struct drm_plane_funcs mdp4_plane_funcs = {
 };
 
 static int mdp4_plane_prepare_fb(struct drm_plane *plane,
-		const struct drm_plane_state *new_state)
+				 struct drm_plane_state *new_state)
 {
 	struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
 	struct mdp4_kms *mdp4_kms = get_kms(plane);
@@ -113,7 +113,7 @@ static int mdp4_plane_prepare_fb(struct drm_plane *plane,
 }
 
 static void mdp4_plane_cleanup_fb(struct drm_plane *plane,
-		const struct drm_plane_state *old_state)
+				  struct drm_plane_state *old_state)
 {
 	struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
 	struct mdp4_kms *mdp4_kms = get_kms(plane);
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
index 432c09836b0e..951c002b05df 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
@@ -78,12 +78,12 @@ static void mdp5_plane_install_rotation_property(struct drm_device *dev,
 	if (!dev->mode_config.rotation_property)
 		dev->mode_config.rotation_property =
 			drm_mode_create_rotation_property(dev,
-			BIT(DRM_REFLECT_X) | BIT(DRM_REFLECT_Y));
+				DRM_ROTATE_0 | DRM_REFLECT_X | DRM_REFLECT_Y);
 
 	if (dev->mode_config.rotation_property)
 		drm_object_attach_property(&plane->base,
 			dev->mode_config.rotation_property,
-			0);
+			DRM_ROTATE_0);
 }
 
 /* helper to install properties which are common to planes and crtcs */
@@ -250,7 +250,7 @@ static const struct drm_plane_funcs mdp5_plane_funcs = {
 };
 
 static int mdp5_plane_prepare_fb(struct drm_plane *plane,
-		const struct drm_plane_state *new_state)
+				 struct drm_plane_state *new_state)
 {
 	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
 	struct mdp5_kms *mdp5_kms = get_kms(plane);
@@ -264,7 +264,7 @@ static int mdp5_plane_prepare_fb(struct drm_plane *plane,
 }
 
 static void mdp5_plane_cleanup_fb(struct drm_plane *plane,
-		const struct drm_plane_state *old_state)
+				  struct drm_plane_state *old_state)
 {
 	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
 	struct mdp5_kms *mdp5_kms = get_kms(plane);
@@ -309,8 +309,8 @@ static int mdp5_plane_atomic_check(struct drm_plane *plane,
 			return -EINVAL;
 		}
 
-		hflip = !!(state->rotation & BIT(DRM_REFLECT_X));
-		vflip = !!(state->rotation & BIT(DRM_REFLECT_Y));
+		hflip = !!(state->rotation & DRM_REFLECT_X);
+		vflip = !!(state->rotation & DRM_REFLECT_Y);
 		if ((vflip && !(mdp5_plane->caps & MDP_PIPE_CAP_VFLIP)) ||
 			(hflip && !(mdp5_plane->caps & MDP_PIPE_CAP_HFLIP))) {
 			dev_err(plane->dev->dev,
@@ -743,8 +743,8 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 	config |= get_scale_config(format, src_h, crtc_h, false);
 	DBG("scale config = %x", config);
 
-	hflip = !!(pstate->rotation & BIT(DRM_REFLECT_X));
-	vflip = !!(pstate->rotation & BIT(DRM_REFLECT_Y));
+	hflip = !!(pstate->rotation & DRM_REFLECT_X);
+	vflip = !!(pstate->rotation & DRM_REFLECT_Y);
 
 	spin_lock_irqsave(&mdp5_plane->pipe_lock, flags);
 
diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c
index 4a8a6f1f1151..73bae382eac3 100644
--- a/drivers/gpu/drm/msm/msm_atomic.c
+++ b/drivers/gpu/drm/msm/msm_atomic.c
@@ -112,13 +112,13 @@ static void complete_commit(struct msm_commit *c, bool async)
 	struct msm_drm_private *priv = dev->dev_private;
 	struct msm_kms *kms = priv->kms;
 
-	drm_atomic_helper_wait_for_fences(dev, state);
+	drm_atomic_helper_wait_for_fences(dev, state, false);
 
 	kms->funcs->prepare_commit(kms, state);
 
 	drm_atomic_helper_commit_modeset_disables(dev, state);
 
-	drm_atomic_helper_commit_planes(dev, state, false);
+	drm_atomic_helper_commit_planes(dev, state, 0);
 
 	drm_atomic_helper_commit_modeset_enables(dev, state);
 
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 8a0237008f74..fb5c0b0a7594 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -26,9 +26,10 @@
  * MSM driver version:
  * - 1.0.0 - initial interface
  * - 1.1.0 - adds madvise, and support for submits with > 4 cmd buffers
+ * - 1.2.0 - adds explicit fence support for submit ioctl
  */
 #define MSM_VERSION_MAJOR	1
-#define MSM_VERSION_MINOR	1
+#define MSM_VERSION_MINOR	2
 #define MSM_VERSION_PATCHLEVEL	0
 
 static void msm_fb_output_poll_changed(struct drm_device *dev)
@@ -347,9 +348,9 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
 	int ret;
 
 	ddev = drm_dev_alloc(drv, dev);
-	if (!ddev) {
+	if (IS_ERR(ddev)) {
 		dev_err(dev, "failed to allocate drm_device\n");
-		return -ENOMEM;
+		return PTR_ERR(ddev);
 	}
 
 	platform_set_drvdata(pdev, ddev);
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index b4bc7f1ef717..d0da52f2a806 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -157,6 +157,12 @@ struct msm_drm_private {
 	struct shrinker shrinker;
 
 	struct msm_vblank_ctrl vblank_ctrl;
+
+	/* task holding struct_mutex.. currently only used in submit path
+	 * to detect and reject faults from copy_from_user() for submit
+	 * ioctl.
+	 */
+	struct task_struct *struct_mutex_task;
 };
 
 struct msm_format {
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 6cd4af443139..b6ac27e31929 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -196,11 +196,20 @@ int msm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct drm_gem_object *obj = vma->vm_private_data;
 	struct drm_device *dev = obj->dev;
+	struct msm_drm_private *priv = dev->dev_private;
 	struct page **pages;
 	unsigned long pfn;
 	pgoff_t pgoff;
 	int ret;
 
+	/* This should only happen if userspace tries to pass a mmap'd
+	 * but unfaulted gem bo vaddr into submit ioctl, triggering
+	 * a page fault while struct_mutex is already held.  This is
+	 * not a valid use-case so just bail.
+	 */
+	if (priv->struct_mutex_task == current)
+		return VM_FAULT_SIGBUS;
+
 	/* Make sure we don't parallel update on a fault, nor move or remove
 	 * something from beneath our feet
 	 */
@@ -584,18 +593,16 @@ int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	bool write = !!(op & MSM_PREP_WRITE);
-
-	if (op & MSM_PREP_NOSYNC) {
-		if (!reservation_object_test_signaled_rcu(msm_obj->resv, write))
-			return -EBUSY;
-	} else {
-		int ret;
-
-		ret = reservation_object_wait_timeout_rcu(msm_obj->resv, write,
-				true, timeout_to_jiffies(timeout));
-		if (ret <= 0)
-			return ret == 0 ? -ETIMEDOUT : ret;
-	}
+	unsigned long remain =
+		op & MSM_PREP_NOSYNC ? 0 : timeout_to_jiffies(timeout);
+	long ret;
+
+	ret = reservation_object_wait_timeout_rcu(msm_obj->resv, write,
+						  true,  remain);
+	if (ret == 0)
+		return remain == 0 ? -EBUSY : -ETIMEDOUT;
+	else if (ret < 0)
+		return ret;
 
 	/* TODO cache maintenance */
 
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index 9766f9ae4b7d..b6a0f37a65f3 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -15,6 +15,8 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/sync_file.h>
+
 #include "msm_drv.h"
 #include "msm_gpu.h"
 #include "msm_gem.h"
@@ -64,6 +66,14 @@ void msm_gem_submit_free(struct msm_gem_submit *submit)
 	kfree(submit);
 }
 
+static inline unsigned long __must_check
+copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
+{
+	if (access_ok(VERIFY_READ, from, n))
+		return __copy_from_user_inatomic(to, from, n);
+	return -EFAULT;
+}
+
 static int submit_lookup_objects(struct msm_gem_submit *submit,
 		struct drm_msm_gem_submit *args, struct drm_file *file)
 {
@@ -71,6 +81,7 @@ static int submit_lookup_objects(struct msm_gem_submit *submit,
 	int ret = 0;
 
 	spin_lock(&file->table_lock);
+	pagefault_disable();
 
 	for (i = 0; i < args->nr_bos; i++) {
 		struct drm_msm_gem_submit_bo submit_bo;
@@ -84,10 +95,15 @@ static int submit_lookup_objects(struct msm_gem_submit *submit,
 		 */
 		submit->bos[i].flags = 0;
 
-		ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
-		if (ret) {
-			ret = -EFAULT;
-			goto out_unlock;
+		ret = copy_from_user_inatomic(&submit_bo, userptr, sizeof(submit_bo));
+		if (unlikely(ret)) {
+			pagefault_enable();
+			spin_unlock(&file->table_lock);
+			ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
+			if (ret)
+				goto out;
+			spin_lock(&file->table_lock);
+			pagefault_disable();
 		}
 
 		if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
@@ -127,9 +143,12 @@ static int submit_lookup_objects(struct msm_gem_submit *submit,
 	}
 
 out_unlock:
-	submit->nr_bos = i;
+	pagefault_enable();
 	spin_unlock(&file->table_lock);
 
+out:
+	submit->nr_bos = i;
+
 	return ret;
 }
 
@@ -361,6 +380,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	struct msm_file_private *ctx = file->driver_priv;
 	struct msm_gem_submit *submit;
 	struct msm_gpu *gpu = priv->gpu;
+	struct fence *in_fence = NULL;
+	struct sync_file *sync_file = NULL;
+	int out_fence_fd = -1;
 	unsigned i;
 	int ret;
 
@@ -370,13 +392,25 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	/* for now, we just have 3d pipe.. eventually this would need to
 	 * be more clever to dispatch to appropriate gpu module:
 	 */
-	if (args->pipe != MSM_PIPE_3D0)
+	if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
+		return -EINVAL;
+
+	if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
 		return -EINVAL;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 
+	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
+		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
+		if (out_fence_fd < 0) {
+			ret = out_fence_fd;
+			goto out_unlock;
+		}
+	}
+	priv->struct_mutex_task = current;
+
 	submit = submit_create(dev, gpu, args->nr_bos, args->nr_cmds);
 	if (!submit) {
 		ret = -ENOMEM;
@@ -391,9 +425,32 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	if (ret)
 		goto out;
 
-	ret = submit_fence_sync(submit);
-	if (ret)
-		goto out;
+	if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
+		in_fence = sync_file_get_fence(args->fence_fd);
+
+		if (!in_fence) {
+			ret = -EINVAL;
+			goto out;
+		}
+
+		/* TODO if we get an array-fence due to userspace merging multiple
+		 * fences, we need a way to determine if all the backing fences
+		 * are from our own context..
+		 */
+
+		if (in_fence->context != gpu->fctx->context) {
+			ret = fence_wait(in_fence, true);
+			if (ret)
+				goto out;
+		}
+
+	}
+
+	if (!(args->fence & MSM_SUBMIT_NO_IMPLICIT)) {
+		ret = submit_fence_sync(submit);
+		if (ret)
+			goto out;
+	}
 
 	ret = submit_pin_objects(submit);
 	if (ret)
@@ -459,15 +516,40 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 
 	submit->nr_cmds = i;
 
-	ret = msm_gpu_submit(gpu, submit, ctx);
+	submit->fence = msm_fence_alloc(gpu->fctx);
+	if (IS_ERR(submit->fence)) {
+		ret = PTR_ERR(submit->fence);
+		submit->fence = NULL;
+		goto out;
+	}
+
+	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
+		sync_file = sync_file_create(submit->fence);
+		if (!sync_file) {
+			ret = -ENOMEM;
+			goto out;
+		}
+	}
+
+	msm_gpu_submit(gpu, submit, ctx);
 
 	args->fence = submit->fence->seqno;
 
+	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
+		fd_install(out_fence_fd, sync_file->file);
+		args->fence_fd = out_fence_fd;
+	}
+
 out:
+	if (in_fence)
+		fence_put(in_fence);
 	submit_cleanup(submit);
 	if (ret)
 		msm_gem_submit_free(submit);
 out_unlock:
+	if (ret && (out_fence_fd >= 0))
+		put_unused_fd(out_fence_fd);
+	priv->struct_mutex_task = NULL;
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
 }
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 36ed53e661fe..5bb09838b5ae 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -509,22 +509,15 @@ void msm_gpu_retire(struct msm_gpu *gpu)
 }
 
 /* add bo's to gpu's ring, and kick gpu: */
-int msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
+void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 		struct msm_file_private *ctx)
 {
 	struct drm_device *dev = gpu->dev;
 	struct msm_drm_private *priv = dev->dev_private;
-	int i, ret;
+	int i;
 
 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
-	submit->fence = msm_fence_alloc(gpu->fctx);
-	if (IS_ERR(submit->fence)) {
-		ret = PTR_ERR(submit->fence);
-		submit->fence = NULL;
-		return ret;
-	}
-
 	inactive_cancel(gpu);
 
 	list_add_tail(&submit->node, &gpu->submit_list);
@@ -557,8 +550,6 @@ int msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 	priv->lastctx = ctx;
 
 	hangcheck_timer_reset(gpu);
-
-	return 0;
 }
 
 /*
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index c9022837a1a4..d61d98a6e047 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -163,7 +163,7 @@ int msm_gpu_perfcntr_sample(struct msm_gpu *gpu, uint32_t *activetime,
 		uint32_t *totaltime, uint32_t ncntrs, uint32_t *cntrs);
 
 void msm_gpu_retire(struct msm_gpu *gpu);
-int msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
+void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 		struct msm_file_private *ctx);
 
 int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
diff --git a/drivers/gpu/drm/msm/msm_perf.c b/drivers/gpu/drm/msm/msm_perf.c
index 17fe4e53e0d1..1627294575cb 100644
--- a/drivers/gpu/drm/msm/msm_perf.c
+++ b/drivers/gpu/drm/msm/msm_perf.c
@@ -229,8 +229,8 @@ int msm_perf_debugfs_init(struct drm_minor *minor)
 	perf->ent = debugfs_create_file("perf", S_IFREG | S_IRUGO,
 			minor->debugfs_root, perf, &perf_debugfs_fops);
 	if (!perf->ent) {
-		DRM_ERROR("Cannot create /sys/kernel/debug/dri/%s/perf\n",
-				minor->debugfs_root->d_name.name);
+		DRM_ERROR("Cannot create /sys/kernel/debug/dri/%pd/perf\n",
+				minor->debugfs_root);
 		goto fail;
 	}
 
diff --git a/drivers/gpu/drm/msm/msm_rd.c b/drivers/gpu/drm/msm/msm_rd.c
index 3a5fdfcd67ae..8487f461f05f 100644
--- a/drivers/gpu/drm/msm/msm_rd.c
+++ b/drivers/gpu/drm/msm/msm_rd.c
@@ -243,8 +243,8 @@ int msm_rd_debugfs_init(struct drm_minor *minor)
 	rd->ent = debugfs_create_file("rd", S_IFREG | S_IRUGO,
 			minor->debugfs_root, rd, &rd_debugfs_fops);
 	if (!rd->ent) {
-		DRM_ERROR("Cannot create /sys/kernel/debug/dri/%s/rd\n",
-				minor->debugfs_root->d_name.name);
+		DRM_ERROR("Cannot create /sys/kernel/debug/dri/%pd/rd\n",
+				minor->debugfs_root);
 		goto fail;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
index 7ea8aa7ca408..6bc712f32c8b 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
@@ -175,6 +175,7 @@ struct nvkm_device_func {
 	void (*fini)(struct nvkm_device *, bool suspend);
 	resource_size_t (*resource_addr)(struct nvkm_device *, unsigned bar);
 	resource_size_t (*resource_size)(struct nvkm_device *, unsigned bar);
+	bool cpu_coherent;
 };
 
 struct nvkm_device_quirk {
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index f2ad17aa33f0..dc57b628e074 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -225,6 +225,17 @@ static bool nouveau_pr3_present(struct pci_dev *pdev)
 	if (!parent_pdev)
 		return false;
 
+	if (!parent_pdev->bridge_d3) {
+		/*
+		 * Parent PCI bridge is currently not power managed.
+		 * Since userspace can change these afterwards to be on
+		 * the safe side we stick with _DSM and prevent usage of
+		 * _PR3 from the bridge.
+		 */
+		pci_d3cold_disable(pdev);
+		return false;
+	}
+
 	parent_adev = ACPI_COMPANION(&parent_pdev->dev);
 	if (!parent_adev)
 		return false;
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 528bdeffb339..343b8659472c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -209,7 +209,8 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
 	nvbo->tile_flags = tile_flags;
 	nvbo->bo.bdev = &drm->ttm.bdev;
 
-	nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
+	if (!nvxx_device(&drm->device)->func->cpu_coherent)
+		nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
 
 	nvbo->page_shift = 12;
 	if (drm->client.vm) {
@@ -1151,7 +1152,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
 	if (ret)
 		goto out;
 
-	ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem);
+	ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, new_mem);
 out:
 	ttm_bo_mem_put(bo, &tmp_mem);
 	return ret;
@@ -1179,7 +1180,7 @@ nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
 	if (ret)
 		return ret;
 
-	ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);
+	ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, &tmp_mem);
 	if (ret)
 		goto out;
 
@@ -1297,7 +1298,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
 	/* Fallback to software copy. */
 	ret = ttm_bo_wait(bo, intr, no_wait_gpu);
 	if (ret == 0)
-		ret = ttm_bo_move_memcpy(bo, evict, intr, no_wait_gpu, new_mem);
+		ret = ttm_bo_move_memcpy(bo, intr, no_wait_gpu, new_mem);
 
 out:
 	if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
@@ -1315,7 +1316,8 @@ nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 {
 	struct nouveau_bo *nvbo = nouveau_bo(bo);
 
-	return drm_vma_node_verify_access(&nvbo->gem.vma_node, filp);
+	return drm_vma_node_verify_access(&nvbo->gem.vma_node,
+					  filp->private_data);
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 66c1280c0f1f..3100fd88a015 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -351,7 +351,7 @@ static int nouveau_drm_probe(struct pci_dev *pdev,
 	boot = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
 #endif
 	if (nouveau_modeset != 2)
-		remove_conflicting_framebuffers(aper, "nouveaufb", boot);
+		drm_fb_helper_remove_conflicting_framebuffers(aper, "nouveaufb", boot);
 	kfree(aper);
 
 	ret = nvkm_device_pci_new(pdev, nouveau_config, nouveau_debug,
@@ -1067,8 +1067,8 @@ nouveau_platform_device_create(const struct nvkm_device_tegra_func *func,
 		goto err_free;
 
 	drm = drm_dev_alloc(&driver_platform, &pdev->dev);
-	if (!drm) {
-		err = -ENOMEM;
+	if (IS_ERR(drm)) {
+		err = PTR_ERR(drm);
 		goto err_free;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index d1f248fd3506..9f5692726c16 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -32,7 +32,6 @@
 #include <linux/tty.h>
 #include <linux/sysrq.h>
 #include <linux/delay.h>
-#include <linux/fb.h>
 #include <linux/init.h>
 #include <linux/screen_info.h>
 #include <linux/vga_switcheroo.h>
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
index b1b693219db3..62ad0300cfa5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
@@ -1614,6 +1614,7 @@ nvkm_device_pci_func = {
 	.fini = nvkm_device_pci_fini,
 	.resource_addr = nvkm_device_pci_resource_addr,
 	.resource_size = nvkm_device_pci_resource_size,
+	.cpu_coherent = !IS_ENABLED(CONFIG_ARM),
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
index 939682f18788..9b638bd905ff 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
@@ -245,6 +245,7 @@ nvkm_device_tegra_func = {
 	.fini = nvkm_device_tegra_fini,
 	.resource_addr = nvkm_device_tegra_resource_addr,
 	.resource_size = nvkm_device_tegra_resource_size,
+	.cpu_coherent = false,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
index edec30fd3ecd..0a7b6ed5ed28 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
@@ -37,7 +37,10 @@ nv04_fifo_dma_object_dtor(struct nvkm_fifo_chan *base, int cookie)
 {
 	struct nv04_fifo_chan *chan = nv04_fifo_chan(base);
 	struct nvkm_instmem *imem = chan->fifo->base.engine.subdev.device->imem;
+
+	mutex_lock(&chan->fifo->base.engine.subdev.mutex);
 	nvkm_ramht_remove(imem->ramht, cookie);
+	mutex_unlock(&chan->fifo->base.engine.subdev.mutex);
 }
 
 static int
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
index 0eae8afaed90..b1f3b818edf4 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
@@ -13,7 +13,6 @@
 
 #include <linux/backlight.h>
 #include <linux/delay.h>
-#include <linux/fb.h>
 #include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/jiffies.h>
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
index fc4c238c9583..9f3d6f48f3e1 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
@@ -14,7 +14,6 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/spi/spi.h>
-#include <linux/fb.h>
 #include <linux/gpio/consumer.h>
 #include <linux/of_gpio.h>
 
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
index 157c512205d1..3557a4c7dd7b 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
@@ -28,7 +28,6 @@
 #include <linux/jiffies.h>
 #include <linux/sched.h>
 #include <linux/backlight.h>
-#include <linux/fb.h>
 #include <linux/gpio/consumer.h>
 #include <linux/of.h>
 #include <linux/of_gpio.h>
diff --git a/drivers/gpu/drm/omapdrm/dss/dss-of.c b/drivers/gpu/drm/omapdrm/dss/dss-of.c
index e256d879b25c..dfd4e9621e3b 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss-of.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss-of.c
@@ -125,16 +125,15 @@ u32 dss_of_port_get_port_number(struct device_node *port)
 
 static struct device_node *omapdss_of_get_remote_port(const struct device_node *node)
 {
-	struct device_node *np, *np_parent;
+	struct device_node *np;
 
 	np = of_parse_phandle(node, "remote-endpoint", 0);
 	if (!np)
 		return NULL;
 
-	np_parent = of_get_next_parent(np);
-	of_node_put(np);
+	np = of_get_next_parent(np);
 
-	return np_parent;
+	return np;
 }
 
 struct device_node *
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index 26c6134eb744..e1cfba51cff6 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -96,7 +96,7 @@ static void omap_atomic_complete(struct omap_atomic_state_commit *commit)
 	dispc_runtime_get();
 
 	drm_atomic_helper_commit_modeset_disables(dev, old_state);
-	drm_atomic_helper_commit_planes(dev, old_state, false);
+	drm_atomic_helper_commit_planes(dev, old_state, 0);
 	drm_atomic_helper_commit_modeset_enables(dev, old_state);
 
 	omap_atomic_wait_for_completion(dev, old_state);
@@ -295,9 +295,9 @@ static int omap_modeset_init_properties(struct drm_device *dev)
 	if (priv->has_dmm) {
 		dev->mode_config.rotation_property =
 			drm_mode_create_rotation_property(dev,
-				BIT(DRM_ROTATE_0) | BIT(DRM_ROTATE_90) |
-				BIT(DRM_ROTATE_180) | BIT(DRM_ROTATE_270) |
-				BIT(DRM_REFLECT_X) | BIT(DRM_REFLECT_Y));
+				DRM_ROTATE_0 | DRM_ROTATE_90 |
+				DRM_ROTATE_180 | DRM_ROTATE_270 |
+				DRM_REFLECT_X | DRM_REFLECT_Y);
 		if (!dev->mode_config.rotation_property)
 			return -ENOMEM;
 	}
diff --git a/drivers/gpu/drm/omapdrm/omap_fb.c b/drivers/gpu/drm/omapdrm/omap_fb.c
index 31f5178c22c7..5f3337f1e9aa 100644
--- a/drivers/gpu/drm/omapdrm/omap_fb.c
+++ b/drivers/gpu/drm/omapdrm/omap_fb.c
@@ -179,24 +179,24 @@ void omap_framebuffer_update_scanout(struct drm_framebuffer *fb,
 					(uint32_t)win->rotation);
 			/* fallthru to default to no rotation */
 		case 0:
-		case BIT(DRM_ROTATE_0):
+		case DRM_ROTATE_0:
 			orient = 0;
 			break;
-		case BIT(DRM_ROTATE_90):
+		case DRM_ROTATE_90:
 			orient = MASK_XY_FLIP | MASK_X_INVERT;
 			break;
-		case BIT(DRM_ROTATE_180):
+		case DRM_ROTATE_180:
 			orient = MASK_X_INVERT | MASK_Y_INVERT;
 			break;
-		case BIT(DRM_ROTATE_270):
+		case DRM_ROTATE_270:
 			orient = MASK_XY_FLIP | MASK_Y_INVERT;
 			break;
 		}
 
-		if (win->rotation & BIT(DRM_REFLECT_X))
+		if (win->rotation & DRM_REFLECT_X)
 			orient ^= MASK_X_INVERT;
 
-		if (win->rotation & BIT(DRM_REFLECT_Y))
+		if (win->rotation & DRM_REFLECT_Y)
 			orient ^= MASK_Y_INVERT;
 
 		/* adjust x,y offset for flip/invert: */
@@ -213,7 +213,7 @@ void omap_framebuffer_update_scanout(struct drm_framebuffer *fb,
 	} else {
 		switch (win->rotation & DRM_ROTATE_MASK) {
 		case 0:
-		case BIT(DRM_ROTATE_0):
+		case DRM_ROTATE_0:
 			/* OK */
 			break;
 
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
index 5252ab720e70..66ac8c40db26 100644
--- a/drivers/gpu/drm/omapdrm/omap_plane.c
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c
@@ -60,7 +60,7 @@ to_omap_plane_state(struct drm_plane_state *state)
 }
 
 static int omap_plane_prepare_fb(struct drm_plane *plane,
-				 const struct drm_plane_state *new_state)
+				 struct drm_plane_state *new_state)
 {
 	if (!new_state->fb)
 		return 0;
@@ -69,7 +69,7 @@ static int omap_plane_prepare_fb(struct drm_plane *plane,
 }
 
 static void omap_plane_cleanup_fb(struct drm_plane *plane,
-				  const struct drm_plane_state *old_state)
+				  struct drm_plane_state *old_state)
 {
 	if (old_state->fb)
 		omap_framebuffer_unpin(old_state->fb);
@@ -109,8 +109,8 @@ static void omap_plane_atomic_update(struct drm_plane *plane,
 	win.src_y = state->src_y >> 16;
 
 	switch (state->rotation & DRM_ROTATE_MASK) {
-	case BIT(DRM_ROTATE_90):
-	case BIT(DRM_ROTATE_270):
+	case DRM_ROTATE_90:
+	case DRM_ROTATE_270:
 		win.src_w = state->src_h >> 16;
 		win.src_h = state->src_w >> 16;
 		break;
@@ -149,7 +149,7 @@ static void omap_plane_atomic_disable(struct drm_plane *plane,
 	struct omap_plane_state *omap_state = to_omap_plane_state(plane->state);
 	struct omap_plane *omap_plane = to_omap_plane(plane);
 
-	plane->state->rotation = BIT(DRM_ROTATE_0);
+	plane->state->rotation = DRM_ROTATE_0;
 	omap_state->zorder = plane->type == DRM_PLANE_TYPE_PRIMARY
 			   ? 0 : omap_plane->id;
 
@@ -178,7 +178,7 @@ static int omap_plane_atomic_check(struct drm_plane *plane,
 		return -EINVAL;
 
 	if (state->fb) {
-		if (state->rotation != BIT(DRM_ROTATE_0) &&
+		if (state->rotation != DRM_ROTATE_0 &&
 		    !omap_framebuffer_supports_rotation(state->fb))
 			return -EINVAL;
 	}
@@ -269,7 +269,7 @@ static void omap_plane_reset(struct drm_plane *plane)
 	 */
 	omap_state->zorder = plane->type == DRM_PLANE_TYPE_PRIMARY
 			   ? 0 : omap_plane->id;
-	omap_state->base.rotation = BIT(DRM_ROTATE_0);
+	omap_state->base.rotation = DRM_ROTATE_0;
 
 	plane->state = &omap_state->base;
 	plane->state->plane = plane;
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index 1500ab99f548..62aba976e744 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -18,6 +18,17 @@ config DRM_PANEL_SIMPLE
 	  that it can be automatically turned off when the panel goes into a
 	  low power state.
 
+config DRM_PANEL_JDI_LT070ME05000
+	tristate "JDI LT070ME05000 WUXGA DSI panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for JDI DSI video mode
+	  panel as found in Google Nexus 7 (2013) devices.
+	  The panel has a 1200(RGB)×1920 (WUXGA) resolution and uses
+	  24 bit per pixel.
+
 config DRM_PANEL_SAMSUNG_LD9040
 	tristate "Samsung LD9040 RGB/SPI panel"
 	depends on OF && SPI
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index f277eed933d6..a5c7ec0236e0 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -1,4 +1,5 @@
 obj-$(CONFIG_DRM_PANEL_SIMPLE) += panel-simple.o
+obj-$(CONFIG_DRM_PANEL_JDI_LT070ME05000) += panel-jdi-lt070me05000.o
 obj-$(CONFIG_DRM_PANEL_LG_LG4573) += panel-lg-lg4573.o
 obj-$(CONFIG_DRM_PANEL_PANASONIC_VVX10F034N00) += panel-panasonic-vvx10f034n00.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_LD9040) += panel-samsung-ld9040.o
diff --git a/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
new file mode 100644
index 000000000000..5b2340ef74ed
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
@@ -0,0 +1,532 @@
+/*
+ * Copyright (C) 2016 InforceComputing
+ * Author: Vinay Simha BN <simhavcs@gmail.com>
+ *
+ * Copyright (C) 2016 Linaro Ltd
+ * Author: Sumit Semwal <sumit.semwal@linaro.org>
+ *
+ * From internet archives, the panel for Nexus 7 2nd Gen, 2013 model is a
+ * JDI model LT070ME05000, and its data sheet is at:
+ * http://panelone.net/en/7-0-inch/JDI_LT070ME05000_7.0_inch-datasheet
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/backlight.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+
+#include <video/mipi_display.h>
+
+static const char * const regulator_names[] = {
+	"vddp",
+	"iovcc"
+};
+
+struct jdi_panel {
+	struct drm_panel base;
+	struct mipi_dsi_device *dsi;
+
+	struct regulator_bulk_data supplies[ARRAY_SIZE(regulator_names)];
+
+	struct gpio_desc *enable_gpio;
+	struct gpio_desc *reset_gpio;
+	struct gpio_desc *dcdc_en_gpio;
+	struct backlight_device *backlight;
+
+	bool prepared;
+	bool enabled;
+
+	const struct drm_display_mode *mode;
+};
+
+static inline struct jdi_panel *to_jdi_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct jdi_panel, base);
+}
+
+static int jdi_panel_init(struct jdi_panel *jdi)
+{
+	struct mipi_dsi_device *dsi = jdi->dsi;
+	struct device *dev = &jdi->dsi->dev;
+	int ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_soft_reset(dsi);
+	if (ret < 0)
+		return ret;
+
+	usleep_range(10000, 20000);
+
+	ret = mipi_dsi_dcs_set_pixel_format(dsi, MIPI_DCS_PIXEL_FMT_24BIT << 4);
+	if (ret < 0) {
+		dev_err(dev, "failed to set pixel format: %d\n", ret);
+		return ret;
+	}
+
+	ret = mipi_dsi_dcs_set_column_address(dsi, 0, jdi->mode->hdisplay - 1);
+	if (ret < 0) {
+		dev_err(dev, "failed to set column address: %d\n", ret);
+		return ret;
+	}
+
+	ret = mipi_dsi_dcs_set_page_address(dsi, 0, jdi->mode->vdisplay - 1);
+	if (ret < 0) {
+		dev_err(dev, "failed to set page address: %d\n", ret);
+		return ret;
+	}
+
+	/*
+	 * BIT(5) BCTRL = 1 Backlight Control Block On, Brightness registers
+	 *                  are active
+	 * BIT(3) BL = 1    Backlight Control On
+	 * BIT(2) DD = 0    Display Dimming is Off
+	 */
+	ret = mipi_dsi_dcs_write(dsi, MIPI_DCS_WRITE_CONTROL_DISPLAY,
+				 (u8[]){ 0x24 }, 1);
+	if (ret < 0) {
+		dev_err(dev, "failed to write control display: %d\n", ret);
+		return ret;
+	}
+
+	/* CABC off */
+	ret = mipi_dsi_dcs_write(dsi, MIPI_DCS_WRITE_POWER_SAVE,
+				 (u8[]){ 0x00 }, 1);
+	if (ret < 0) {
+		dev_err(dev, "failed to set cabc off: %d\n", ret);
+		return ret;
+	}
+
+	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
+	if (ret < 0) {
+		dev_err(dev, "failed to set exit sleep mode: %d\n", ret);
+		return ret;
+	}
+
+	msleep(120);
+
+	ret = mipi_dsi_generic_write(dsi, (u8[]){0xB0, 0x00}, 2);
+	if (ret < 0) {
+		dev_err(dev, "failed to set mcap: %d\n", ret);
+		return ret;
+	}
+
+	mdelay(10);
+
+	/* Interface setting, video mode */
+	ret = mipi_dsi_generic_write(dsi, (u8[])
+				     {0xB3, 0x26, 0x08, 0x00, 0x20, 0x00}, 6);
+	if (ret < 0) {
+		dev_err(dev, "failed to set display interface setting: %d\n"
+			, ret);
+		return ret;
+	}
+
+	mdelay(20);
+
+	ret = mipi_dsi_generic_write(dsi, (u8[]){0xB0, 0x03}, 2);
+	if (ret < 0) {
+		dev_err(dev, "failed to set default values for mcap: %d\n"
+			, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int jdi_panel_on(struct jdi_panel *jdi)
+{
+	struct mipi_dsi_device *dsi = jdi->dsi;
+	struct device *dev = &jdi->dsi->dev;
+	int ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_on(dsi);
+	if (ret < 0)
+		dev_err(dev, "failed to set display on: %d\n", ret);
+
+	return ret;
+}
+
+static void jdi_panel_off(struct jdi_panel *jdi)
+{
+	struct mipi_dsi_device *dsi = jdi->dsi;
+	struct device *dev = &jdi->dsi->dev;
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_off(dsi);
+	if (ret < 0)
+		dev_err(dev, "failed to set display off: %d\n", ret);
+
+	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
+	if (ret < 0)
+		dev_err(dev, "failed to enter sleep mode: %d\n", ret);
+
+	msleep(100);
+}
+
+static int jdi_panel_disable(struct drm_panel *panel)
+{
+	struct jdi_panel *jdi = to_jdi_panel(panel);
+
+	if (!jdi->enabled)
+		return 0;
+
+	jdi->backlight->props.power = FB_BLANK_POWERDOWN;
+	backlight_update_status(jdi->backlight);
+
+	jdi->enabled = false;
+
+	return 0;
+}
+
+static int jdi_panel_unprepare(struct drm_panel *panel)
+{
+	struct jdi_panel *jdi = to_jdi_panel(panel);
+	struct device *dev = &jdi->dsi->dev;
+	int ret;
+
+	if (!jdi->prepared)
+		return 0;
+
+	jdi_panel_off(jdi);
+
+	ret = regulator_bulk_disable(ARRAY_SIZE(jdi->supplies), jdi->supplies);
+	if (ret < 0)
+		dev_err(dev, "regulator disable failed, %d\n", ret);
+
+	gpiod_set_value(jdi->enable_gpio, 0);
+
+	gpiod_set_value(jdi->reset_gpio, 1);
+
+	gpiod_set_value(jdi->dcdc_en_gpio, 0);
+
+	jdi->prepared = false;
+
+	return 0;
+}
+
+static int jdi_panel_prepare(struct drm_panel *panel)
+{
+	struct jdi_panel *jdi = to_jdi_panel(panel);
+	struct device *dev = &jdi->dsi->dev;
+	int ret;
+
+	if (jdi->prepared)
+		return 0;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(jdi->supplies), jdi->supplies);
+	if (ret < 0) {
+		dev_err(dev, "regulator enable failed, %d\n", ret);
+		return ret;
+	}
+
+	msleep(20);
+
+	gpiod_set_value(jdi->dcdc_en_gpio, 1);
+	usleep_range(10, 20);
+
+	gpiod_set_value(jdi->reset_gpio, 0);
+	usleep_range(10, 20);
+
+	gpiod_set_value(jdi->enable_gpio, 1);
+	usleep_range(10, 20);
+
+	ret = jdi_panel_init(jdi);
+	if (ret < 0) {
+		dev_err(dev, "failed to init panel: %d\n", ret);
+		goto poweroff;
+	}
+
+	ret = jdi_panel_on(jdi);
+	if (ret < 0) {
+		dev_err(dev, "failed to set panel on: %d\n", ret);
+		goto poweroff;
+	}
+
+	jdi->prepared = true;
+
+	return 0;
+
+poweroff:
+	ret = regulator_bulk_disable(ARRAY_SIZE(jdi->supplies), jdi->supplies);
+	if (ret < 0)
+		dev_err(dev, "regulator disable failed, %d\n", ret);
+
+	gpiod_set_value(jdi->enable_gpio, 0);
+
+	gpiod_set_value(jdi->reset_gpio, 1);
+
+	gpiod_set_value(jdi->dcdc_en_gpio, 0);
+
+	return ret;
+}
+
+static int jdi_panel_enable(struct drm_panel *panel)
+{
+	struct jdi_panel *jdi = to_jdi_panel(panel);
+
+	if (jdi->enabled)
+		return 0;
+
+	jdi->backlight->props.power = FB_BLANK_UNBLANK;
+	backlight_update_status(jdi->backlight);
+
+	jdi->enabled = true;
+
+	return 0;
+}
+
+static const struct drm_display_mode default_mode = {
+		.clock = 155493,
+		.hdisplay = 1200,
+		.hsync_start = 1200 + 48,
+		.hsync_end = 1200 + 48 + 32,
+		.htotal = 1200 + 48 + 32 + 60,
+		.vdisplay = 1920,
+		.vsync_start = 1920 + 3,
+		.vsync_end = 1920 + 3 + 5,
+		.vtotal = 1920 + 3 + 5 + 6,
+		.vrefresh = 60,
+		.flags = 0,
+};
+
+static int jdi_panel_get_modes(struct drm_panel *panel)
+{
+	struct drm_display_mode *mode;
+	struct jdi_panel *jdi = to_jdi_panel(panel);
+	struct device *dev = &jdi->dsi->dev;
+
+	mode = drm_mode_duplicate(panel->drm, &default_mode);
+	if (!mode) {
+		dev_err(dev, "failed to add mode %ux%ux@%u\n",
+			default_mode.hdisplay, default_mode.vdisplay,
+			default_mode.vrefresh);
+		return -ENOMEM;
+	}
+
+	drm_mode_set_name(mode);
+
+	drm_mode_probed_add(panel->connector, mode);
+
+	panel->connector->display_info.width_mm = 95;
+	panel->connector->display_info.height_mm = 151;
+
+	return 1;
+}
+
+static int dsi_dcs_bl_get_brightness(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	int ret;
+	u16 brightness = bl->props.brightness;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_get_display_brightness(dsi, &brightness);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return brightness & 0xff;
+}
+
+static int dsi_dcs_bl_update_status(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_brightness(dsi, bl->props.brightness);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return 0;
+}
+
+static const struct backlight_ops dsi_bl_ops = {
+	.update_status = dsi_dcs_bl_update_status,
+	.get_brightness = dsi_dcs_bl_get_brightness,
+};
+
+static struct backlight_device *
+drm_panel_create_dsi_backlight(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct backlight_properties props;
+
+	memset(&props, 0, sizeof(props));
+	props.type = BACKLIGHT_RAW;
+	props.brightness = 255;
+	props.max_brightness = 255;
+
+	return devm_backlight_device_register(dev, dev_name(dev), dev, dsi,
+					      &dsi_bl_ops, &props);
+}
+
+static const struct drm_panel_funcs jdi_panel_funcs = {
+	.disable = jdi_panel_disable,
+	.unprepare = jdi_panel_unprepare,
+	.prepare = jdi_panel_prepare,
+	.enable = jdi_panel_enable,
+	.get_modes = jdi_panel_get_modes,
+};
+
+static const struct of_device_id jdi_of_match[] = {
+	{ .compatible = "jdi,lt070me05000", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, jdi_of_match);
+
+static int jdi_panel_add(struct jdi_panel *jdi)
+{
+	struct device *dev = &jdi->dsi->dev;
+	int ret;
+	unsigned int i;
+
+	jdi->mode = &default_mode;
+
+	for (i = 0; i < ARRAY_SIZE(jdi->supplies); i++)
+		jdi->supplies[i].supply = regulator_names[i];
+
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(jdi->supplies),
+				      jdi->supplies);
+	if (ret < 0) {
+		dev_err(dev, "failed to init regulator, ret=%d\n", ret);
+		return ret;
+	}
+
+	jdi->enable_gpio = devm_gpiod_get(dev, "enable", GPIOD_OUT_LOW);
+	if (IS_ERR(jdi->enable_gpio)) {
+		ret = PTR_ERR(jdi->enable_gpio);
+		dev_err(dev, "cannot get enable-gpio %d\n", ret);
+		return ret;
+	}
+
+	jdi->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(jdi->reset_gpio)) {
+		ret = PTR_ERR(jdi->reset_gpio);
+		dev_err(dev, "cannot get reset-gpios %d\n", ret);
+		return ret;
+	}
+
+	jdi->dcdc_en_gpio = devm_gpiod_get(dev, "dcdc-en", GPIOD_OUT_LOW);
+	if (IS_ERR(jdi->dcdc_en_gpio)) {
+		ret = PTR_ERR(jdi->dcdc_en_gpio);
+		dev_err(dev, "cannot get dcdc-en-gpio %d\n", ret);
+		return ret;
+	}
+
+	jdi->backlight = drm_panel_create_dsi_backlight(jdi->dsi);
+	if (IS_ERR(jdi->backlight)) {
+		ret = PTR_ERR(jdi->backlight);
+		dev_err(dev, "failed to register backlight %d\n", ret);
+		return ret;
+	}
+
+	drm_panel_init(&jdi->base);
+	jdi->base.funcs = &jdi_panel_funcs;
+	jdi->base.dev = &jdi->dsi->dev;
+
+	ret = drm_panel_add(&jdi->base);
+
+	return ret;
+}
+
+static void jdi_panel_del(struct jdi_panel *jdi)
+{
+	if (jdi->base.dev)
+		drm_panel_remove(&jdi->base);
+}
+
+static int jdi_panel_probe(struct mipi_dsi_device *dsi)
+{
+	struct jdi_panel *jdi;
+	int ret;
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags =  MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+			   MIPI_DSI_CLOCK_NON_CONTINUOUS;
+
+	jdi = devm_kzalloc(&dsi->dev, sizeof(*jdi), GFP_KERNEL);
+	if (!jdi)
+		return -ENOMEM;
+
+	mipi_dsi_set_drvdata(dsi, jdi);
+
+	jdi->dsi = dsi;
+
+	ret = jdi_panel_add(jdi);
+	if (ret < 0)
+		return ret;
+
+	return mipi_dsi_attach(dsi);
+}
+
+static int jdi_panel_remove(struct mipi_dsi_device *dsi)
+{
+	struct jdi_panel *jdi = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = jdi_panel_disable(&jdi->base);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to disable panel: %d\n", ret);
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n",
+			ret);
+
+	drm_panel_detach(&jdi->base);
+	jdi_panel_del(jdi);
+
+	return 0;
+}
+
+static void jdi_panel_shutdown(struct mipi_dsi_device *dsi)
+{
+	struct jdi_panel *jdi = mipi_dsi_get_drvdata(dsi);
+
+	jdi_panel_disable(&jdi->base);
+}
+
+static struct mipi_dsi_driver jdi_panel_driver = {
+	.driver = {
+		.name = "panel-jdi-lt070me05000",
+		.of_match_table = jdi_of_match,
+	},
+	.probe = jdi_panel_probe,
+	.remove = jdi_panel_remove,
+	.shutdown = jdi_panel_shutdown,
+};
+module_mipi_dsi_driver(jdi_panel_driver);
+
+MODULE_AUTHOR("Sumit Semwal <sumit.semwal@linaro.org>");
+MODULE_AUTHOR("Vinay Simha BN <simhavcs@gmail.com>");
+MODULE_DESCRIPTION("JDI LT070ME05000 WUXGA");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 85143d1b9b31..113db3c4a633 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -849,6 +849,34 @@ static const struct panel_desc innolux_at070tn92 = {
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
 };
 
+static const struct display_timing innolux_g101ice_l01_timing = {
+	.pixelclock = { 60400000, 71100000, 74700000 },
+	.hactive = { 1280, 1280, 1280 },
+	.hfront_porch = { 41, 80, 100 },
+	.hback_porch = { 40, 79, 99 },
+	.hsync_len = { 1, 1, 1 },
+	.vactive = { 800, 800, 800 },
+	.vfront_porch = { 5, 11, 14 },
+	.vback_porch = { 4, 11, 14 },
+	.vsync_len = { 1, 1, 1 },
+	.flags = DISPLAY_FLAGS_DE_HIGH,
+};
+
+static const struct panel_desc innolux_g101ice_l01 = {
+	.timings = &innolux_g101ice_l01_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 217,
+		.height = 135,
+	},
+	.delay = {
+		.enable = 200,
+		.disable = 200,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+};
+
 static const struct drm_display_mode innolux_g121i1_l01_mode = {
 	.clock = 71000,
 	.hdisplay = 1280,
@@ -1186,7 +1214,7 @@ static const struct panel_desc olimex_lcd_olinuxino_43ts = {
 		.width = 105,
 		.height = 67,
 	},
-	.bus_format = MEDIA_BUS_FMT_RGB666_1X18,
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
 };
 
 /*
@@ -1245,6 +1273,7 @@ static const struct panel_desc ortustech_com43h4m85ulc = {
 		.height = 93,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
 };
 
 static const struct drm_display_mode qd43003c0_40_mode = {
@@ -1384,6 +1413,11 @@ static const struct panel_desc sharp_lq123p1jx31 = {
 		.width = 259,
 		.height = 173,
 	},
+	.delay = {
+		.prepare = 110,
+		.enable = 50,
+		.unprepare = 550,
+	},
 };
 
 static const struct drm_display_mode shelly_sca07010_bfn_lnn_mode = {
@@ -1430,6 +1464,11 @@ static const struct panel_desc starry_kr122ea0sra = {
 		.width = 263,
 		.height = 164,
 	},
+	.delay = {
+		.prepare = 10 + 200,
+		.enable = 50,
+		.unprepare = 10 + 500,
+	},
 };
 
 static const struct drm_display_mode tpk_f07a_0102_mode = {
@@ -1575,6 +1614,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "innolux,at070tn92",
 		.data = &innolux_at070tn92,
 	}, {
+		.compatible ="innolux,g101ice-l01",
+		.data = &innolux_g101ice_l01
+	}, {
 		.compatible ="innolux,g121i1-l01",
 		.data = &innolux_g121i1_l01
 	}, {
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 3aef12742a53..a61c0d460ec2 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -211,6 +211,7 @@ static void qxl_crtc_destroy(struct drm_crtc *crtc)
 	struct qxl_crtc *qxl_crtc = to_qxl_crtc(crtc);
 
 	drm_crtc_cleanup(crtc);
+	qxl_bo_unref(&qxl_crtc->cursor_bo);
 	kfree(qxl_crtc);
 }
 
@@ -296,6 +297,52 @@ qxl_hide_cursor(struct qxl_device *qdev)
 	return 0;
 }
 
+static int qxl_crtc_apply_cursor(struct drm_crtc *crtc)
+{
+	struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct qxl_device *qdev = dev->dev_private;
+	struct qxl_cursor_cmd *cmd;
+	struct qxl_release *release;
+	int ret = 0;
+
+	if (!qcrtc->cursor_bo)
+		return 0;
+
+	ret = qxl_alloc_release_reserved(qdev, sizeof(*cmd),
+					 QXL_RELEASE_CURSOR_CMD,
+					 &release, NULL);
+	if (ret)
+		return ret;
+
+	ret = qxl_release_list_add(release, qcrtc->cursor_bo);
+	if (ret)
+		goto out_free_release;
+
+	ret = qxl_release_reserve_list(release, false);
+	if (ret)
+		goto out_free_release;
+
+	cmd = (struct qxl_cursor_cmd *)qxl_release_map(qdev, release);
+	cmd->type = QXL_CURSOR_SET;
+	cmd->u.set.position.x = qcrtc->cur_x + qcrtc->hot_spot_x;
+	cmd->u.set.position.y = qcrtc->cur_y + qcrtc->hot_spot_y;
+
+	cmd->u.set.shape = qxl_bo_physical_address(qdev, qcrtc->cursor_bo, 0);
+
+	cmd->u.set.visible = 1;
+	qxl_release_unmap(qdev, release, &cmd->release_info);
+
+	qxl_push_cursor_ring_release(qdev, release, QXL_CMD_CURSOR, false);
+	qxl_release_fence_buffer_objects(release);
+
+	return ret;
+
+out_free_release:
+	qxl_release_free(qdev, release);
+	return ret;
+}
+
 static int qxl_crtc_cursor_set2(struct drm_crtc *crtc,
 				struct drm_file *file_priv,
 				uint32_t handle,
@@ -400,7 +447,8 @@ static int qxl_crtc_cursor_set2(struct drm_crtc *crtc,
 	}
 	drm_gem_object_unreference_unlocked(obj);
 
-	qxl_bo_unref(&cursor_bo);
+	qxl_bo_unref (&qcrtc->cursor_bo);
+	qcrtc->cursor_bo = cursor_bo;
 
 	return ret;
 
@@ -655,6 +703,12 @@ static int qxl_crtc_mode_set(struct drm_crtc *crtc,
 			   bo->surf.stride, bo->surf.format);
 		qxl_io_create_primary(qdev, 0, bo);
 		bo->is_primary = true;
+
+		ret = qxl_crtc_apply_cursor(crtc);
+		if (ret) {
+			DRM_ERROR("could not set cursor after modeset");
+			ret = 0;
+		}
 	}
 
 	if (bo->is_primary) {
diff --git a/drivers/gpu/drm/qxl/qxl_draw.c b/drivers/gpu/drm/qxl/qxl_draw.c
index ffe885395145..9b728edf1b49 100644
--- a/drivers/gpu/drm/qxl/qxl_draw.c
+++ b/drivers/gpu/drm/qxl/qxl_draw.c
@@ -57,11 +57,8 @@ static struct qxl_rect *drawable_set_clipping(struct qxl_device *qdev,
 static int
 alloc_drawable(struct qxl_device *qdev, struct qxl_release **release)
 {
-	int ret;
-	ret = qxl_alloc_release_reserved(qdev, sizeof(struct qxl_drawable),
-					 QXL_RELEASE_DRAWABLE, release,
-					 NULL);
-	return ret;
+	return qxl_alloc_release_reserved(qdev, sizeof(struct qxl_drawable),
+					  QXL_RELEASE_DRAWABLE, release, NULL);
 }
 
 static void
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 8e633caa4078..5f3e5ad99de7 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -137,6 +137,7 @@ struct qxl_crtc {
 	int cur_y;
 	int hot_spot_x;
 	int hot_spot_y;
+	struct qxl_bo *cursor_bo;
 };
 
 struct qxl_output {
diff --git a/drivers/gpu/drm/qxl/qxl_fb.c b/drivers/gpu/drm/qxl/qxl_fb.c
index df2657051afd..2cd879a4ae15 100644
--- a/drivers/gpu/drm/qxl/qxl_fb.c
+++ b/drivers/gpu/drm/qxl/qxl_fb.c
@@ -24,7 +24,6 @@
  *     David Airlie
  */
 #include <linux/module.h>
-#include <linux/fb.h>
 
 #include "drmP.h"
 #include "drm/drm.h"
@@ -73,10 +72,12 @@ static void qxl_fb_image_init(struct qxl_fb_image *qxl_fb_image,
 	}
 }
 
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 static struct fb_deferred_io qxl_defio = {
 	.delay		= QXL_DIRTY_DELAY,
 	.deferred_io	= drm_fb_helper_deferred_io,
 };
+#endif
 
 static struct fb_ops qxlfb_ops = {
 	.owner = THIS_MODULE,
@@ -313,8 +314,10 @@ static int qxlfb_create(struct qxl_fbdev *qfbdev,
 		goto out_destroy_fbi;
 	}
 
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	info->fbdefio = &qxl_defio;
 	fb_deferred_io_init(info);
+#endif
 
 	qdev->fbdev_info = info;
 	qdev->fbdev_qfb = &qfbdev->qfb;
diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c
index 5e1d7899dd72..fa5440dc9a19 100644
--- a/drivers/gpu/drm/qxl/qxl_object.c
+++ b/drivers/gpu/drm/qxl/qxl_object.c
@@ -61,7 +61,7 @@ void qxl_ttm_placement_from_domain(struct qxl_bo *qbo, u32 domain, bool pinned)
 	if (domain == QXL_GEM_DOMAIN_VRAM)
 		qbo->placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_VRAM | pflag;
 	if (domain == QXL_GEM_DOMAIN_SURFACE)
-		qbo->placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_PRIV0 | pflag;
+		qbo->placements[c++].flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_PRIV | pflag;
 	if (domain == QXL_GEM_DOMAIN_CPU)
 		qbo->placements[c++].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM | pflag;
 	if (!c)
@@ -151,7 +151,7 @@ void *qxl_bo_kmap_atomic_page(struct qxl_device *qdev,
 
 	if (bo->tbo.mem.mem_type == TTM_PL_VRAM)
 		map = qdev->vram_mapping;
-	else if (bo->tbo.mem.mem_type == TTM_PL_PRIV0)
+	else if (bo->tbo.mem.mem_type == TTM_PL_PRIV)
 		map = qdev->surface_mapping;
 	else
 		goto fallback;
@@ -191,7 +191,7 @@ void qxl_bo_kunmap_atomic_page(struct qxl_device *qdev,
 
 	if (bo->tbo.mem.mem_type == TTM_PL_VRAM)
 		map = qdev->vram_mapping;
-	else if (bo->tbo.mem.mem_type == TTM_PL_PRIV0)
+	else if (bo->tbo.mem.mem_type == TTM_PL_PRIV)
 		map = qdev->surface_mapping;
 	else
 		goto fallback;
@@ -311,7 +311,7 @@ int qxl_bo_check_id(struct qxl_device *qdev, struct qxl_bo *bo)
 
 int qxl_surf_evict(struct qxl_device *qdev)
 {
-	return ttm_bo_evict_mm(&qdev->mman.bdev, TTM_PL_PRIV0);
+	return ttm_bo_evict_mm(&qdev->mman.bdev, TTM_PL_PRIV);
 }
 
 int qxl_vram_evict(struct qxl_device *qdev)
diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c
index f599cd073b72..cd83f050cf3e 100644
--- a/drivers/gpu/drm/qxl/qxl_release.c
+++ b/drivers/gpu/drm/qxl/qxl_release.c
@@ -203,12 +203,9 @@ qxl_release_free(struct qxl_device *qdev,
 static int qxl_release_bo_alloc(struct qxl_device *qdev,
 				struct qxl_bo **bo)
 {
-	int ret;
 	/* pin releases bo's they are too messy to evict */
-	ret = qxl_bo_create(qdev, PAGE_SIZE, false, true,
-			    QXL_GEM_DOMAIN_VRAM, NULL,
-			    bo);
-	return ret;
+	return qxl_bo_create(qdev, PAGE_SIZE, false, true,
+			     QXL_GEM_DOMAIN_VRAM, NULL, bo);
 }
 
 int qxl_release_list_add(struct qxl_release *release, struct qxl_bo *bo)
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index d50c9679e631..e26c82db948b 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -168,7 +168,7 @@ static int qxl_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 		man->default_caching = TTM_PL_FLAG_CACHED;
 		break;
 	case TTM_PL_VRAM:
-	case TTM_PL_PRIV0:
+	case TTM_PL_PRIV:
 		/* "On-card" video ram */
 		man->func = &ttm_bo_manager_func;
 		man->gpu_offset = 0;
@@ -210,7 +210,8 @@ static int qxl_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 {
 	struct qxl_bo *qbo = to_qxl_bo(bo);
 
-	return drm_vma_node_verify_access(&qbo->gem_base.vma_node, filp);
+	return drm_vma_node_verify_access(&qbo->gem_base.vma_node,
+					  filp->private_data);
 }
 
 static int qxl_ttm_io_mem_reserve(struct ttm_bo_device *bdev,
@@ -235,7 +236,7 @@ static int qxl_ttm_io_mem_reserve(struct ttm_bo_device *bdev,
 		mem->bus.base = qdev->vram_base;
 		mem->bus.offset = mem->start << PAGE_SHIFT;
 		break;
-	case TTM_PL_PRIV0:
+	case TTM_PL_PRIV:
 		mem->bus.is_iomem = true;
 		mem->bus.base = qdev->surfaceram_base;
 		mem->bus.offset = mem->start << PAGE_SHIFT;
@@ -361,8 +362,8 @@ static int qxl_bo_move(struct ttm_buffer_object *bo,
 		qxl_move_null(bo, new_mem);
 		return 0;
 	}
-	return ttm_bo_move_memcpy(bo, evict, interruptible,
-				  no_wait_gpu, new_mem);
+	return ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu,
+				  new_mem);
 }
 
 static void qxl_bo_move_notify(struct ttm_buffer_object *bo,
@@ -376,7 +377,7 @@ static void qxl_bo_move_notify(struct ttm_buffer_object *bo,
 	qbo = to_qxl_bo(bo);
 	qdev = qbo->gem_base.dev->dev_private;
 
-	if (bo->mem.mem_type == TTM_PL_PRIV0 && qbo->surface_id)
+	if (bo->mem.mem_type == TTM_PL_PRIV && qbo->surface_id)
 		qxl_surface_evict(qdev, qbo, new_mem ? true : false);
 }
 
@@ -422,7 +423,7 @@ int qxl_ttm_init(struct qxl_device *qdev)
 		DRM_ERROR("Failed initializing VRAM heap.\n");
 		return r;
 	}
-	r = ttm_bo_init_mm(&qdev->mman.bdev, TTM_PL_PRIV0,
+	r = ttm_bo_init_mm(&qdev->mman.bdev, TTM_PL_PRIV,
 			   qdev->surfaceram_size / PAGE_SIZE);
 	if (r) {
 		DRM_ERROR("Failed initializing Surfaces heap.\n");
@@ -445,7 +446,7 @@ int qxl_ttm_init(struct qxl_device *qdev)
 void qxl_ttm_fini(struct qxl_device *qdev)
 {
 	ttm_bo_clean_mm(&qdev->mman.bdev, TTM_PL_VRAM);
-	ttm_bo_clean_mm(&qdev->mman.bdev, TTM_PL_PRIV0);
+	ttm_bo_clean_mm(&qdev->mman.bdev, TTM_PL_PRIV);
 	ttm_bo_device_release(&qdev->mman.bdev);
 	qxl_ttm_global_fini(qdev);
 	DRM_INFO("qxl: ttm finalized\n");
@@ -489,7 +490,7 @@ static int qxl_ttm_debugfs_init(struct qxl_device *qdev)
 		if (i == 0)
 			qxl_mem_types_list[i].data = qdev->mman.bdev.man[TTM_PL_VRAM].priv;
 		else
-			qxl_mem_types_list[i].data = qdev->mman.bdev.man[TTM_PL_PRIV0].priv;
+			qxl_mem_types_list[i].data = qdev->mman.bdev.man[TTM_PL_PRIV].priv;
 
 	}
 	return qxl_debugfs_add_files(qdev, qxl_mem_types_list, i);
diff --git a/drivers/gpu/drm/r128/r128_drv.c b/drivers/gpu/drm/r128/r128_drv.c
index c57b4de63caf..a982be57d1ef 100644
--- a/drivers/gpu/drm/r128/r128_drv.c
+++ b/drivers/gpu/drm/r128/r128_drv.c
@@ -56,7 +56,7 @@ static const struct file_operations r128_driver_fops = {
 
 static struct drm_driver driver = {
 	.driver_features =
-	    DRIVER_USE_AGP | DRIVER_PCI_DMA | DRIVER_SG |
+	    DRIVER_USE_AGP | DRIVER_PCI_DMA | DRIVER_SG | DRIVER_LEGACY |
 	    DRIVER_HAVE_DMA | DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED,
 	.dev_priv_size = sizeof(drm_r128_buf_priv_t),
 	.load = r128_driver_load,
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index a97abc8af657..74f99bac08b1 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -627,7 +627,9 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
 			if (radeon_crtc->ss.refdiv) {
 				radeon_crtc->pll_flags |= RADEON_PLL_USE_REF_DIV;
 				radeon_crtc->pll_reference_div = radeon_crtc->ss.refdiv;
-				if (rdev->family >= CHIP_RV770)
+				if (ASIC_IS_AVIVO(rdev) &&
+				    rdev->family != CHIP_RS780 &&
+				    rdev->family != CHIP_RS880)
 					radeon_crtc->pll_flags |= RADEON_PLL_USE_FRAC_FB_DIV;
 			}
 		}
@@ -1154,6 +1156,7 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 tmp, viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
+	char *format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -1257,8 +1260,9 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		DRM_ERROR("Unsupported screen format %s\n",
-			  drm_get_format_name(target_fb->pixel_format));
+		format_name = drm_get_format_name(target_fb->pixel_format);
+		DRM_ERROR("Unsupported screen format %s\n", format_name);
+		kfree(format_name);
 		return -EINVAL;
 	}
 
@@ -1433,8 +1437,8 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(EVERGREEN_VIEWPORT_SIZE + radeon_crtc->crtc_offset,
 	       (viewport_w << 16) | viewport_h);
 
-	/* set pageflip to happen only at start of vblank interval (front porch) */
-	WREG32(EVERGREEN_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 3);
+	/* set pageflip to happen anywhere in vblank interval */
+	WREG32(EVERGREEN_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
 		radeon_fb = to_radeon_framebuffer(fb);
@@ -1469,6 +1473,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
+	char *format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -1558,8 +1563,9 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		DRM_ERROR("Unsupported screen format %s\n",
-			  drm_get_format_name(target_fb->pixel_format));
+		format_name = drm_get_format_name(target_fb->pixel_format);
+		DRM_ERROR("Unsupported screen format %s\n", format_name);
+		kfree(format_name);
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/radeon/atombios_dp.c b/drivers/gpu/drm/radeon/atombios_dp.c
index cead089a9e7d..432cb46f6a34 100644
--- a/drivers/gpu/drm/radeon/atombios_dp.c
+++ b/drivers/gpu/drm/radeon/atombios_dp.c
@@ -389,22 +389,21 @@ bool radeon_dp_getdpcd(struct radeon_connector *radeon_connector)
 {
 	struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
 	u8 msg[DP_DPCD_SIZE];
-	int ret, i;
+	int ret;
 
-	for (i = 0; i < 7; i++) {
-		ret = drm_dp_dpcd_read(&radeon_connector->ddc_bus->aux, DP_DPCD_REV, msg,
-				       DP_DPCD_SIZE);
-		if (ret == DP_DPCD_SIZE) {
-			memcpy(dig_connector->dpcd, msg, DP_DPCD_SIZE);
+	ret = drm_dp_dpcd_read(&radeon_connector->ddc_bus->aux, DP_DPCD_REV, msg,
+			       DP_DPCD_SIZE);
+	if (ret == DP_DPCD_SIZE) {
+		memcpy(dig_connector->dpcd, msg, DP_DPCD_SIZE);
 
-			DRM_DEBUG_KMS("DPCD: %*ph\n", (int)sizeof(dig_connector->dpcd),
-				      dig_connector->dpcd);
+		DRM_DEBUG_KMS("DPCD: %*ph\n", (int)sizeof(dig_connector->dpcd),
+			      dig_connector->dpcd);
 
-			radeon_dp_probe_oui(radeon_connector);
+		radeon_dp_probe_oui(radeon_connector);
 
-			return true;
-		}
+		return true;
 	}
+
 	dig_connector->dpcd[0] = 0;
 	return false;
 }
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index 0c1b9ff433af..f6ff41a0eed6 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -1871,7 +1871,7 @@ int ci_mc_load_microcode(struct radeon_device *rdev)
 {
 	const __be32 *fw_data = NULL;
 	const __le32 *new_fw_data = NULL;
-	u32 running, blackout = 0, tmp;
+	u32 running, tmp;
 	u32 *io_mc_regs = NULL;
 	const __le32 *new_io_mc_regs = NULL;
 	int i, regs_size, ucode_size;
@@ -1912,11 +1912,6 @@ int ci_mc_load_microcode(struct radeon_device *rdev)
 	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 
 	if (running == 0) {
-		if (running) {
-			blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
-			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
-		}
-
 		/* reset the engine and set to writable */
 		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
@@ -1964,9 +1959,6 @@ int ci_mc_load_microcode(struct radeon_device *rdev)
 				break;
 			udelay(1);
 		}
-
-		if (running)
-			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 	}
 
 	return 0;
@@ -4201,11 +4193,7 @@ u32 cik_gfx_get_rptr(struct radeon_device *rdev,
 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
 		     struct radeon_ring *ring)
 {
-	u32 wptr;
-
-	wptr = RREG32(CP_RB0_WPTR);
-
-	return wptr;
+	return RREG32(CP_RB0_WPTR);
 }
 
 void cik_gfx_set_wptr(struct radeon_device *rdev,
@@ -8215,7 +8203,7 @@ static void cik_uvd_resume(struct radeon_device *rdev)
 		return;
 
 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
-	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
+	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
 	if (r) {
 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
 		return;
diff --git a/drivers/gpu/drm/radeon/cikd.h b/drivers/gpu/drm/radeon/cikd.h
index cead2284fd79..48db93577c1d 100644
--- a/drivers/gpu/drm/radeon/cikd.h
+++ b/drivers/gpu/drm/radeon/cikd.h
@@ -2069,6 +2069,7 @@
 #define UVD_UDEC_ADDR_CONFIG		0xef4c
 #define UVD_UDEC_DB_ADDR_CONFIG		0xef50
 #define UVD_UDEC_DBW_ADDR_CONFIG	0xef54
+#define UVD_NO_OP			0xeffc
 
 #define UVD_LMI_EXT40_ADDR		0xf498
 #define UVD_GP_SCRATCH4			0xf4e0
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index db275b7ed34a..0b6b5766216f 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -2878,9 +2878,8 @@ void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *s
 	for (i = 0; i < rdev->num_crtc; i++) {
 		if (save->crtc_enabled[i]) {
 			tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
-			if ((tmp & 0x7) != 3) {
+			if ((tmp & 0x7) != 0) {
 				tmp &= ~0x7;
-				tmp |= 0x3;
 				WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
 			}
 			tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
@@ -5580,7 +5579,7 @@ static void evergreen_uvd_resume(struct radeon_device *rdev)
 		return;
 
 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
-	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
+	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
 	if (r) {
 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
 		return;
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index c8e3d394cde7..f3d88ca2aa8f 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -1523,6 +1523,7 @@
 #define UVD_UDEC_ADDR_CONFIG				0xef4c
 #define UVD_UDEC_DB_ADDR_CONFIG				0xef50
 #define UVD_UDEC_DBW_ADDR_CONFIG			0xef54
+#define UVD_NO_OP					0xeffc
 #define UVD_RBC_RB_RPTR					0xf690
 #define UVD_RBC_RB_WPTR					0xf694
 #define UVD_STATUS					0xf6bc
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 4a3d7cab83f7..103fc8650197 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -2062,7 +2062,7 @@ static void cayman_uvd_resume(struct radeon_device *rdev)
 		return;
 
 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
-	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
+	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
 	if (r) {
 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
 		return;
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
index 47eb49b77d32..3c9fec88ea44 100644
--- a/drivers/gpu/drm/radeon/nid.h
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -1137,6 +1137,7 @@
 #define UVD_UDEC_ADDR_CONFIG				0xEF4C
 #define UVD_UDEC_DB_ADDR_CONFIG				0xEF50
 #define UVD_UDEC_DBW_ADDR_CONFIG			0xEF54
+#define UVD_NO_OP					0xEFFC
 #define UVD_RBC_RB_RPTR					0xF690
 #define UVD_RBC_RB_WPTR					0xF694
 #define UVD_STATUS					0xf6bc
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index f25994b3afa6..f5e84f4b58e6 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -1071,11 +1071,7 @@ u32 r100_gfx_get_rptr(struct radeon_device *rdev,
 u32 r100_gfx_get_wptr(struct radeon_device *rdev,
 		      struct radeon_ring *ring)
 {
-	u32 wptr;
-
-	wptr = RREG32(RADEON_CP_RB_WPTR);
-
-	return wptr;
+	return RREG32(RADEON_CP_RB_WPTR);
 }
 
 void r100_gfx_set_wptr(struct radeon_device *rdev,
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 9247e7d207fe..a951881c2a50 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -2631,11 +2631,7 @@ u32 r600_gfx_get_rptr(struct radeon_device *rdev,
 u32 r600_gfx_get_wptr(struct radeon_device *rdev,
 		      struct radeon_ring *ring)
 {
-	u32 wptr;
-
-	wptr = RREG32(R600_CP_RB_WPTR);
-
-	return wptr;
+	return RREG32(R600_CP_RB_WPTR);
 }
 
 void r600_gfx_set_wptr(struct radeon_device *rdev,
@@ -3097,7 +3093,7 @@ static void r600_uvd_resume(struct radeon_device *rdev)
 		return;
 
 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
-	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
+	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
 	if (r) {
 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
 		return;
diff --git a/drivers/gpu/drm/radeon/r600d.h b/drivers/gpu/drm/radeon/r600d.h
index 1e8495cca41e..2e00a5287bd2 100644
--- a/drivers/gpu/drm/radeon/r600d.h
+++ b/drivers/gpu/drm/radeon/r600d.h
@@ -1490,6 +1490,7 @@
 #define UVD_GPCOM_VCPU_DATA0				0xef10
 #define UVD_GPCOM_VCPU_DATA1				0xef14
 #define UVD_ENGINE_CNTL					0xef18
+#define UVD_NO_OP					0xeffc
 
 #define UVD_SEMA_CNTL					0xf400
 #define UVD_RB_ARB_CTRL					0xf480
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 5633ee3eb46e..1b0dcad916b0 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -742,6 +742,7 @@ struct radeon_flip_work {
 	struct work_struct		unpin_work;
 	struct radeon_device		*rdev;
 	int				crtc_id;
+	u32				target_vblank;
 	uint64_t			base;
 	struct drm_pending_vblank_event *event;
 	struct radeon_bo		*old_rbo;
diff --git a/drivers/gpu/drm/radeon/radeon_acpi.c b/drivers/gpu/drm/radeon/radeon_acpi.c
index 31c9a92d6a1b..6efbd65c929e 100644
--- a/drivers/gpu/drm/radeon/radeon_acpi.c
+++ b/drivers/gpu/drm/radeon/radeon_acpi.c
@@ -25,6 +25,7 @@
 #include <linux/acpi.h>
 #include <linux/slab.h>
 #include <linux/power_supply.h>
+#include <linux/pm_runtime.h>
 #include <acpi/video.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
@@ -32,6 +33,12 @@
 #include "radeon_acpi.h"
 #include "atom.h"
 
+#if defined(CONFIG_VGA_SWITCHEROO)
+bool radeon_atpx_dgpu_req_power_for_displays(void);
+#else
+static inline bool radeon_atpx_dgpu_req_power_for_displays(void) { return false; }
+#endif
+
 #define ACPI_AC_CLASS           "ac_adapter"
 
 extern void radeon_pm_acpi_event_handler(struct radeon_device *rdev);
@@ -394,6 +401,16 @@ int radeon_atif_handler(struct radeon_device *rdev,
 #endif
 		}
 	}
+	if (req.pending & ATIF_DGPU_DISPLAY_EVENT) {
+		if ((rdev->flags & RADEON_IS_PX) &&
+		    radeon_atpx_dgpu_req_power_for_displays()) {
+			pm_runtime_get_sync(rdev->ddev->dev);
+			/* Just fire off a uevent and let userspace tell us what to do */
+			drm_helper_hpd_irq_event(rdev->ddev);
+			pm_runtime_mark_last_busy(rdev->ddev->dev);
+			pm_runtime_put_autosuspend(rdev->ddev->dev);
+		}
+	}
 	/* TODO: check other events */
 
 	/* We've handled the event, stop the notifier chain. The ACPI interface
diff --git a/drivers/gpu/drm/radeon/radeon_atpx_handler.c b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
index 6de342861202..2fdcd04bc93f 100644
--- a/drivers/gpu/drm/radeon/radeon_atpx_handler.c
+++ b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
@@ -29,6 +29,7 @@ struct radeon_atpx {
 	acpi_handle handle;
 	struct radeon_atpx_functions functions;
 	bool is_hybrid;
+	bool dgpu_req_power_for_displays;
 };
 
 static struct radeon_atpx_priv {
@@ -72,6 +73,10 @@ bool radeon_is_atpx_hybrid(void) {
 	return radeon_atpx_priv.atpx.is_hybrid;
 }
 
+bool radeon_atpx_dgpu_req_power_for_displays(void) {
+	return radeon_atpx_priv.atpx.dgpu_req_power_for_displays;
+}
+
 /**
  * radeon_atpx_call - call an ATPX method
  *
@@ -198,16 +203,7 @@ static int radeon_atpx_validate(struct radeon_atpx *atpx)
 	atpx->is_hybrid = false;
 	if (valid_bits & ATPX_MS_HYBRID_GFX_SUPPORTED) {
 		printk("ATPX Hybrid Graphics\n");
-#if 1
-		/* This is a temporary hack until the D3 cold support
-		 * makes it upstream.  The ATPX power_control method seems
-		 * to still work on even if the system should be using
-		 * the new standardized hybrid D3 cold ACPI interface.
-		 */
-		atpx->functions.power_cntl = true;
-#else
 		atpx->functions.power_cntl = false;
-#endif
 		atpx->is_hybrid = true;
 	}
 
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index b79f3b002471..50e96d2c593d 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -198,12 +198,12 @@ int radeon_get_monitor_bpc(struct drm_connector *connector)
 		}
 
 		/* Any defined maximum tmds clock limit we must not exceed? */
-		if (connector->max_tmds_clock > 0) {
+		if (connector->display_info.max_tmds_clock > 0) {
 			/* mode_clock is clock in kHz for mode to be modeset on this connector */
 			mode_clock = radeon_connector->pixelclock_for_modeset;
 
 			/* Maximum allowable input clock in kHz */
-			max_tmds_clock = connector->max_tmds_clock * 1000;
+			max_tmds_clock = connector->display_info.max_tmds_clock;
 
 			DRM_DEBUG("%s: hdmi mode dotclock %d kHz, max tmds input clock %d kHz.\n",
 					  connector->name, mode_clock, max_tmds_clock);
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index a00dd2f74527..eb92aef46e3c 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -639,7 +639,7 @@ void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
  * Used at driver startup.
  * Returns true if virtual or false if not.
  */
-static bool radeon_device_is_virtual(void)
+bool radeon_device_is_virtual(void)
 {
 #ifdef CONFIG_X86
 	return boot_cpu_has(X86_FEATURE_HYPERVISOR);
@@ -661,8 +661,9 @@ bool radeon_card_posted(struct radeon_device *rdev)
 {
 	uint32_t reg;
 
-	/* for pass through, always force asic_init */
-	if (radeon_device_is_virtual())
+	/* for pass through, always force asic_init for CI */
+	if (rdev->family >= CHIP_BONAIRE &&
+	    radeon_device_is_virtual())
 		return false;
 
 	/* required for EFI mode on macbook2,1 which uses an r5xx asic */
@@ -1956,14 +1957,3 @@ static void radeon_debugfs_remove_files(struct radeon_device *rdev)
 	}
 #endif
 }
-
-#if defined(CONFIG_DEBUG_FS)
-int radeon_debugfs_init(struct drm_minor *minor)
-{
-	return 0;
-}
-
-void radeon_debugfs_cleanup(struct drm_minor *minor)
-{
-}
-#endif
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index c3206fb8f4cf..b8ab30a7dd6d 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -321,16 +321,30 @@ void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id)
 	update_pending = radeon_page_flip_pending(rdev, crtc_id);
 
 	/* Has the pageflip already completed in crtc, or is it certain
-	 * to complete in this vblank?
+	 * to complete in this vblank? GET_DISTANCE_TO_VBLANKSTART provides
+	 * distance to start of "fudged earlier" vblank in vpos, distance to
+	 * start of real vblank in hpos. vpos >= 0 && hpos < 0 means we are in
+	 * the last few scanlines before start of real vblank, where the vblank
+	 * irq can fire, so we have sampled update_pending a bit too early and
+	 * know the flip will complete at leading edge of the upcoming real
+	 * vblank. On pre-AVIVO hardware, flips also complete inside the real
+	 * vblank, not only at leading edge, so if update_pending for hpos >= 0
+	 *  == inside real vblank, the flip will complete almost immediately.
+	 * Note that this method of completion handling is still not 100% race
+	 * free, as we could execute before the radeon_flip_work_func managed
+	 * to run and set the RADEON_FLIP_SUBMITTED status, thereby we no-op,
+	 * but the flip still gets programmed into hw and completed during
+	 * vblank, leading to a delayed emission of the flip completion event.
+	 * This applies at least to pre-AVIVO hardware, where flips are always
+	 * completing inside vblank, not only at leading edge of vblank.
 	 */
 	if (update_pending &&
-	    (DRM_SCANOUTPOS_VALID & radeon_get_crtc_scanoutpos(rdev->ddev,
-							       crtc_id,
-							       USE_REAL_VBLANKSTART,
-							       &vpos, &hpos, NULL, NULL,
-							       &rdev->mode_info.crtcs[crtc_id]->base.hwmode)) &&
-	    ((vpos >= (99 * rdev->mode_info.crtcs[crtc_id]->base.hwmode.crtc_vdisplay)/100) ||
-	     (vpos < 0 && !ASIC_IS_AVIVO(rdev)))) {
+	    (DRM_SCANOUTPOS_VALID &
+	     radeon_get_crtc_scanoutpos(rdev->ddev, crtc_id,
+					GET_DISTANCE_TO_VBLANKSTART,
+					&vpos, &hpos, NULL, NULL,
+					&rdev->mode_info.crtcs[crtc_id]->base.hwmode)) &&
+	    ((vpos >= 0 && hpos < 0) || (hpos >= 0 && !ASIC_IS_AVIVO(rdev)))) {
 		/* crtc didn't flip in this target vblank interval,
 		 * but flip is pending in crtc. Based on the current
 		 * scanout position we know that the current frame is
@@ -400,14 +414,13 @@ static void radeon_flip_work_func(struct work_struct *__work)
 	struct radeon_flip_work *work =
 		container_of(__work, struct radeon_flip_work, flip_work);
 	struct radeon_device *rdev = work->rdev;
+	struct drm_device *dev = rdev->ddev;
 	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[work->crtc_id];
 
 	struct drm_crtc *crtc = &radeon_crtc->base;
 	unsigned long flags;
 	int r;
-	int vpos, hpos, stat, min_udelay = 0;
-	unsigned repcnt = 4;
-	struct drm_vblank_crtc *vblank = &crtc->dev->vblank[work->crtc_id];
+	int vpos, hpos;
 
 	down_read(&rdev->exclusive_lock);
 	if (work->fence) {
@@ -438,59 +451,28 @@ static void radeon_flip_work_func(struct work_struct *__work)
 		work->fence = NULL;
 	}
 
+	/* Wait until we're out of the vertical blank period before the one
+	 * targeted by the flip. Always wait on pre DCE4 to avoid races with
+	 * flip completion handling from vblank irq, as these old asics don't
+	 * have reliable pageflip completion interrupts.
+	 */
+	while (radeon_crtc->enabled &&
+		(radeon_get_crtc_scanoutpos(dev, work->crtc_id, 0,
+					    &vpos, &hpos, NULL, NULL,
+					    &crtc->hwmode)
+		& (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK)) ==
+		(DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK) &&
+		(!ASIC_IS_AVIVO(rdev) ||
+		((int) (work->target_vblank -
+		dev->driver->get_vblank_counter(dev, work->crtc_id)) > 0)))
+		usleep_range(1000, 2000);
+
 	/* We borrow the event spin lock for protecting flip_status */
 	spin_lock_irqsave(&crtc->dev->event_lock, flags);
 
 	/* set the proper interrupt */
 	radeon_irq_kms_pflip_irq_get(rdev, radeon_crtc->crtc_id);
 
-	/* If this happens to execute within the "virtually extended" vblank
-	 * interval before the start of the real vblank interval then it needs
-	 * to delay programming the mmio flip until the real vblank is entered.
-	 * This prevents completing a flip too early due to the way we fudge
-	 * our vblank counter and vblank timestamps in order to work around the
-	 * problem that the hw fires vblank interrupts before actual start of
-	 * vblank (when line buffer refilling is done for a frame). It
-	 * complements the fudging logic in radeon_get_crtc_scanoutpos() for
-	 * timestamping and radeon_get_vblank_counter_kms() for vblank counts.
-	 *
-	 * In practice this won't execute very often unless on very fast
-	 * machines because the time window for this to happen is very small.
-	 */
-	while (radeon_crtc->enabled && --repcnt) {
-		/* GET_DISTANCE_TO_VBLANKSTART returns distance to real vblank
-		 * start in hpos, and to the "fudged earlier" vblank start in
-		 * vpos.
-		 */
-		stat = radeon_get_crtc_scanoutpos(rdev->ddev, work->crtc_id,
-						  GET_DISTANCE_TO_VBLANKSTART,
-						  &vpos, &hpos, NULL, NULL,
-						  &crtc->hwmode);
-
-		if ((stat & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE)) !=
-		    (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE) ||
-		    !(vpos >= 0 && hpos <= 0))
-			break;
-
-		/* Sleep at least until estimated real start of hw vblank */
-		min_udelay = (-hpos + 1) * max(vblank->linedur_ns / 1000, 5);
-		if (min_udelay > vblank->framedur_ns / 2000) {
-			/* Don't wait ridiculously long - something is wrong */
-			repcnt = 0;
-			break;
-		}
-		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
-		usleep_range(min_udelay, 2 * min_udelay);
-		spin_lock_irqsave(&crtc->dev->event_lock, flags);
-	};
-
-	if (!repcnt)
-		DRM_DEBUG_DRIVER("Delay problem on crtc %d: min_udelay %d, "
-				 "framedur %d, linedur %d, stat %d, vpos %d, "
-				 "hpos %d\n", work->crtc_id, min_udelay,
-				 vblank->framedur_ns / 1000,
-				 vblank->linedur_ns / 1000, stat, vpos, hpos);
-
 	/* do the flip (mmio) */
 	radeon_page_flip(rdev, radeon_crtc->crtc_id, work->base, work->async);
 
@@ -499,10 +481,11 @@ static void radeon_flip_work_func(struct work_struct *__work)
 	up_read(&rdev->exclusive_lock);
 }
 
-static int radeon_crtc_page_flip(struct drm_crtc *crtc,
-				 struct drm_framebuffer *fb,
-				 struct drm_pending_vblank_event *event,
-				 uint32_t page_flip_flags)
+static int radeon_crtc_page_flip_target(struct drm_crtc *crtc,
+					struct drm_framebuffer *fb,
+					struct drm_pending_vblank_event *event,
+					uint32_t page_flip_flags,
+					uint32_t target)
 {
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
@@ -599,12 +582,8 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
 		base &= ~7;
 	}
 	work->base = base;
-
-	r = drm_crtc_vblank_get(crtc);
-	if (r) {
-		DRM_ERROR("failed to get vblank before flip\n");
-		goto pflip_cleanup;
-	}
+	work->target_vblank = target - drm_crtc_vblank_count(crtc) +
+		dev->driver->get_vblank_counter(dev, work->crtc_id);
 
 	/* We borrow the event spin lock for protecting flip_work */
 	spin_lock_irqsave(&crtc->dev->event_lock, flags);
@@ -613,7 +592,7 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
 		DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
 		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
 		r = -EBUSY;
-		goto vblank_cleanup;
+		goto pflip_cleanup;
 	}
 	radeon_crtc->flip_status = RADEON_FLIP_PENDING;
 	radeon_crtc->flip_work = work;
@@ -626,9 +605,6 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
 	queue_work(radeon_crtc->flip_queue, &work->flip_work);
 	return 0;
 
-vblank_cleanup:
-	drm_crtc_vblank_put(crtc);
-
 pflip_cleanup:
 	if (unlikely(radeon_bo_reserve(new_rbo, false) != 0)) {
 		DRM_ERROR("failed to reserve new rbo in error path\n");
@@ -697,7 +673,7 @@ static const struct drm_crtc_funcs radeon_crtc_funcs = {
 	.gamma_set = radeon_crtc_gamma_set,
 	.set_config = radeon_crtc_set_config,
 	.destroy = radeon_crtc_destroy,
-	.page_flip = radeon_crtc_page_flip,
+	.page_flip_target = radeon_crtc_page_flip_target,
 };
 
 static void radeon_crtc_init(struct drm_device *dev, int index)
diff --git a/drivers/gpu/drm/radeon/radeon_dp_auxch.c b/drivers/gpu/drm/radeon/radeon_dp_auxch.c
index db64e0062689..2d465648856a 100644
--- a/drivers/gpu/drm/radeon/radeon_dp_auxch.c
+++ b/drivers/gpu/drm/radeon/radeon_dp_auxch.c
@@ -164,7 +164,6 @@ radeon_dp_aux_transfer_native(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg
 	}
 
 	if (tmp & AUX_SW_RX_TIMEOUT) {
-		DRM_DEBUG_KMS("dp_aux_ch timed out\n");
 		ret = -ETIMEDOUT;
 		goto done;
 	}
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index c01a7c6abb49..91c8f4339566 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -39,6 +39,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/vga_switcheroo.h>
 #include <drm/drm_gem.h>
+#include <drm/drm_fb_helper.h>
 
 #include "drm_crtc_helper.h"
 #include "radeon_kfd.h"
@@ -94,9 +95,10 @@
  *   2.44.0 - SET_APPEND_CNT packet3 support
  *   2.45.0 - Allow setting shader registers using DMA/COPY packet3 on SI
  *   2.46.0 - Add PFP_SYNC_ME support on evergreen
+ *   2.47.0 - Add UVD_NO_OP register support
  */
 #define KMS_DRIVER_MAJOR	2
-#define KMS_DRIVER_MINOR	46
+#define KMS_DRIVER_MINOR	47
 #define KMS_DRIVER_PATCHLEVEL	0
 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
 int radeon_driver_unload_kms(struct drm_device *dev);
@@ -154,11 +156,6 @@ void radeon_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
 extern long radeon_kms_compat_ioctl(struct file *filp, unsigned int cmd,
 				    unsigned long arg);
 
-#if defined(CONFIG_DEBUG_FS)
-int radeon_debugfs_init(struct drm_minor *minor);
-void radeon_debugfs_cleanup(struct drm_minor *minor);
-#endif
-
 /* atpx handler */
 #if defined(CONFIG_VGA_SWITCHEROO)
 void radeon_register_atpx_handler(void);
@@ -309,6 +306,8 @@ MODULE_DEVICE_TABLE(pci, pciidlist);
 
 static struct drm_driver kms_driver;
 
+bool radeon_device_is_virtual(void);
+
 static int radeon_kick_out_firmware_fb(struct pci_dev *pdev)
 {
 	struct apertures_struct *ap;
@@ -324,7 +323,7 @@ static int radeon_kick_out_firmware_fb(struct pci_dev *pdev)
 #ifdef CONFIG_X86
 	primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
 #endif
-	remove_conflicting_framebuffers(ap, "radeondrmfb", primary);
+	drm_fb_helper_remove_conflicting_framebuffers(ap, "radeondrmfb", primary);
 	kfree(ap);
 
 	return 0;
@@ -362,6 +361,17 @@ radeon_pci_remove(struct pci_dev *pdev)
 	drm_put_dev(dev);
 }
 
+static void
+radeon_pci_shutdown(struct pci_dev *pdev)
+{
+	/* if we are running in a VM, make sure the device
+	 * torn down properly on reboot/shutdown.
+	 * unfortunately we can't detect certain
+	 * hypervisors so just do this all the time.
+	 */
+	radeon_pci_remove(pdev);
+}
+
 static int radeon_pmops_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
@@ -373,6 +383,14 @@ static int radeon_pmops_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+
+	/* GPU comes up enabled by the bios on resume */
+	if (radeon_is_px(drm_dev)) {
+		pm_runtime_disable(dev);
+		pm_runtime_set_active(dev);
+		pm_runtime_enable(dev);
+	}
+
 	return radeon_resume_kms(drm_dev, true, true);
 }
 
@@ -529,10 +547,6 @@ static struct drm_driver kms_driver = {
 	.disable_vblank = radeon_disable_vblank_kms,
 	.get_vblank_timestamp = radeon_get_vblank_timestamp_kms,
 	.get_scanout_position = radeon_get_crtc_scanoutpos,
-#if defined(CONFIG_DEBUG_FS)
-	.debugfs_init = radeon_debugfs_init,
-	.debugfs_cleanup = radeon_debugfs_cleanup,
-#endif
 	.irq_preinstall = radeon_driver_irq_preinstall_kms,
 	.irq_postinstall = radeon_driver_irq_postinstall_kms,
 	.irq_uninstall = radeon_driver_irq_uninstall_kms,
@@ -574,6 +588,7 @@ static struct pci_driver radeon_kms_pci_driver = {
 	.id_table = pciidlist,
 	.probe = radeon_pci_probe,
 	.remove = radeon_pci_remove,
+	.shutdown = radeon_pci_shutdown,
 	.driver.pm = &radeon_pm_ops,
 };
 
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 0e3143acb565..0daad446d2c7 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -25,7 +25,7 @@
  */
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/fb.h>
+#include <linux/pm_runtime.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_crtc.h>
@@ -47,8 +47,35 @@ struct radeon_fbdev {
 	struct radeon_device *rdev;
 };
 
+static int
+radeonfb_open(struct fb_info *info, int user)
+{
+	struct radeon_fbdev *rfbdev = info->par;
+	struct radeon_device *rdev = rfbdev->rdev;
+	int ret = pm_runtime_get_sync(rdev->ddev->dev);
+	if (ret < 0 && ret != -EACCES) {
+		pm_runtime_mark_last_busy(rdev->ddev->dev);
+		pm_runtime_put_autosuspend(rdev->ddev->dev);
+		return ret;
+	}
+	return 0;
+}
+
+static int
+radeonfb_release(struct fb_info *info, int user)
+{
+	struct radeon_fbdev *rfbdev = info->par;
+	struct radeon_device *rdev = rfbdev->rdev;
+
+	pm_runtime_mark_last_busy(rdev->ddev->dev);
+	pm_runtime_put_autosuspend(rdev->ddev->dev);
+	return 0;
+}
+
 static struct fb_ops radeonfb_ops = {
 	.owner = THIS_MODULE,
+	.fb_open = radeonfb_open,
+	.fb_release = radeonfb_release,
 	.fb_check_var = drm_fb_helper_check_var,
 	.fb_set_par = drm_fb_helper_set_par,
 	.fb_fillrect = drm_fb_helper_cfb_fillrect,
@@ -383,7 +410,7 @@ void radeon_fbdev_fini(struct radeon_device *rdev)
 void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state)
 {
 	if (rdev->mode_info.rfbdev)
-		fb_set_suspend(rdev->mode_info.rfbdev->helper.fbdev, state);
+		drm_fb_helper_set_suspend(&rdev->mode_info.rfbdev->helper, state);
 }
 
 bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj)
diff --git a/drivers/gpu/drm/radeon/radeon_i2c.c b/drivers/gpu/drm/radeon/radeon_i2c.c
index 9590bcd321c0..021aa005623f 100644
--- a/drivers/gpu/drm/radeon/radeon_i2c.c
+++ b/drivers/gpu/drm/radeon/radeon_i2c.c
@@ -938,10 +938,8 @@ struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev,
 			 "Radeon i2c hw bus %s", name);
 		i2c->adapter.algo = &radeon_i2c_algo;
 		ret = i2c_add_adapter(&i2c->adapter);
-		if (ret) {
-			DRM_ERROR("Failed to register hw i2c %s\n", name);
+		if (ret)
 			goto out_free;
-		}
 	} else if (rec->hw_capable &&
 		   radeon_hw_i2c &&
 		   ASIC_IS_DCE3(rdev)) {
@@ -950,10 +948,8 @@ struct radeon_i2c_chan *radeon_i2c_create(struct drm_device *dev,
 			 "Radeon i2c hw bus %s", name);
 		i2c->adapter.algo = &radeon_atom_i2c_algo;
 		ret = i2c_add_adapter(&i2c->adapter);
-		if (ret) {
-			DRM_ERROR("Failed to register hw i2c %s\n", name);
+		if (ret)
 			goto out_free;
-		}
 	} else {
 		/* set the radeon bit adapter */
 		snprintf(i2c->adapter.name, sizeof(i2c->adapter.name),
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 835563c1f0ed..4388ddeec8d2 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -641,11 +641,11 @@ int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 	if (rdev->family >= CHIP_CAYMAN) {
 		struct radeon_fpriv *fpriv;
 		struct radeon_vm *vm;
-		int r;
 
 		fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
 		if (unlikely(!fpriv)) {
-			return -ENOMEM;
+			r = -ENOMEM;
+			goto out_suspend;
 		}
 
 		if (rdev->accel_working) {
@@ -653,14 +653,14 @@ int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 			r = radeon_vm_init(rdev, vm);
 			if (r) {
 				kfree(fpriv);
-				return r;
+				goto out_suspend;
 			}
 
 			r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false);
 			if (r) {
 				radeon_vm_fini(rdev, vm);
 				kfree(fpriv);
-				return r;
+				goto out_suspend;
 			}
 
 			/* map the ib pool buffer read only into
@@ -674,15 +674,16 @@ int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 			if (r) {
 				radeon_vm_fini(rdev, vm);
 				kfree(fpriv);
-				return r;
+				goto out_suspend;
 			}
 		}
 		file_priv->driver_priv = fpriv;
 	}
 
+out_suspend:
 	pm_runtime_mark_last_busy(dev->dev);
 	pm_runtime_put_autosuspend(dev->dev);
-	return 0;
+	return r;
 }
 
 /**
@@ -717,6 +718,8 @@ void radeon_driver_postclose_kms(struct drm_device *dev,
 		kfree(fpriv);
 		file_priv->driver_priv = NULL;
 	}
+	pm_runtime_mark_last_busy(dev->dev);
+	pm_runtime_put_autosuspend(dev->dev);
 }
 
 /**
@@ -733,6 +736,8 @@ void radeon_driver_preclose_kms(struct drm_device *dev,
 {
 	struct radeon_device *rdev = dev->dev_private;
 
+	pm_runtime_get_sync(dev->dev);
+
 	mutex_lock(&rdev->gem.mutex);
 	if (rdev->hyperz_filp == file_priv)
 		rdev->hyperz_filp = NULL;
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index ffdad81ef964..455268214b89 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -237,7 +237,8 @@ static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 
 	if (radeon_ttm_tt_has_userptr(bo->ttm))
 		return -EPERM;
-	return drm_vma_node_verify_access(&rbo->gem_base.vma_node, filp);
+	return drm_vma_node_verify_access(&rbo->gem_base.vma_node,
+					  filp->private_data);
 }
 
 static void radeon_move_null(struct ttm_buffer_object *bo,
@@ -263,8 +264,8 @@ static int radeon_move_blit(struct ttm_buffer_object *bo,
 
 	rdev = radeon_get_rdev(bo->bdev);
 	ridx = radeon_copy_ring_index(rdev);
-	old_start = old_mem->start << PAGE_SHIFT;
-	new_start = new_mem->start << PAGE_SHIFT;
+	old_start = (u64)old_mem->start << PAGE_SHIFT;
+	new_start = (u64)new_mem->start << PAGE_SHIFT;
 
 	switch (old_mem->mem_type) {
 	case TTM_PL_VRAM:
@@ -346,7 +347,7 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
-	r = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem);
+	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem);
 out_cleanup:
 	ttm_bo_mem_put(bo, &tmp_mem);
 	return r;
@@ -379,7 +380,7 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
 	if (unlikely(r)) {
 		return r;
 	}
-	r = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);
+	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
@@ -444,8 +445,7 @@ static int radeon_bo_move(struct ttm_buffer_object *bo,
 
 	if (r) {
 memcpy:
-		r = ttm_bo_move_memcpy(bo, evict, interruptible,
-				       no_wait_gpu, new_mem);
+		r = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, new_mem);
 		if (r) {
 			return r;
 		}
diff --git a/drivers/gpu/drm/radeon/radeon_uvd.c b/drivers/gpu/drm/radeon/radeon_uvd.c
index 73dfe01435ea..0cd0e7bdee55 100644
--- a/drivers/gpu/drm/radeon/radeon_uvd.c
+++ b/drivers/gpu/drm/radeon/radeon_uvd.c
@@ -669,6 +669,7 @@ static int radeon_uvd_cs_reg(struct radeon_cs_parser *p,
 				return r;
 			break;
 		case UVD_ENGINE_CNTL:
+		case UVD_NO_OP:
 			break;
 		default:
 			DRM_ERROR("Invalid reg 0x%X!\n",
@@ -753,8 +754,10 @@ static int radeon_uvd_send_msg(struct radeon_device *rdev,
 	ib.ptr[3] = addr >> 32;
 	ib.ptr[4] = PACKET0(UVD_GPCOM_VCPU_CMD, 0);
 	ib.ptr[5] = 0;
-	for (i = 6; i < 16; ++i)
-		ib.ptr[i] = PACKET2(0);
+	for (i = 6; i < 16; i += 2) {
+		ib.ptr[i] = PACKET0(UVD_NO_OP, 0);
+		ib.ptr[i+1] = 0;
+	}
 	ib.length_dw = 16;
 
 	r = radeon_ib_schedule(rdev, &ib, NULL, false);
diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c
index 1c120a4c3c97..729ae588c970 100644
--- a/drivers/gpu/drm/radeon/rv770.c
+++ b/drivers/gpu/drm/radeon/rv770.c
@@ -1738,7 +1738,7 @@ static void rv770_uvd_resume(struct radeon_device *rdev)
 		return;
 
 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
-	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
+	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
 	if (r) {
 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
 		return;
diff --git a/drivers/gpu/drm/radeon/rv770d.h b/drivers/gpu/drm/radeon/rv770d.h
index 9ef2064b1c9c..0271f4c559ae 100644
--- a/drivers/gpu/drm/radeon/rv770d.h
+++ b/drivers/gpu/drm/radeon/rv770d.h
@@ -387,6 +387,7 @@
 #define UVD_UDEC_TILING_CONFIG                          0xef40
 #define UVD_UDEC_DB_TILING_CONFIG                       0xef44
 #define UVD_UDEC_DBW_TILING_CONFIG                      0xef48
+#define UVD_NO_OP					0xeffc
 
 #define	GC_USER_SHADER_PIPE_CONFIG			0x8954
 #define		INACTIVE_QD_PIPES(x)				((x) << 8)
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index 2523ca96c6c7..7ee9aafbdf74 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -1547,7 +1547,7 @@ int si_mc_load_microcode(struct radeon_device *rdev)
 {
 	const __be32 *fw_data = NULL;
 	const __le32 *new_fw_data = NULL;
-	u32 running, blackout = 0;
+	u32 running;
 	u32 *io_mc_regs = NULL;
 	const __le32 *new_io_mc_regs = NULL;
 	int i, regs_size, ucode_size;
@@ -1598,11 +1598,6 @@ int si_mc_load_microcode(struct radeon_device *rdev)
 	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 
 	if (running == 0) {
-		if (running) {
-			blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
-			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
-		}
-
 		/* reset the engine and set to writable */
 		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
@@ -1641,9 +1636,6 @@ int si_mc_load_microcode(struct radeon_device *rdev)
 				break;
 			udelay(1);
 		}
-
-		if (running)
-			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 	}
 
 	return 0;
@@ -6928,7 +6920,7 @@ static void si_uvd_resume(struct radeon_device *rdev)
 		return;
 
 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
-	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
+	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
 	if (r) {
 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
 		return;
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index e6abc09b67e3..89bdf20344ae 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -3015,6 +3015,12 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev,
 	if (rdev->pdev->device == 0x6811 &&
 	    rdev->pdev->revision == 0x81)
 		max_mclk = 120000;
+	/* limit sclk/mclk on Jet parts for stability */
+	if (rdev->pdev->device == 0x6665 &&
+	    rdev->pdev->revision == 0xc3) {
+		max_sclk = 75000;
+		max_mclk = 80000;
+	}
 
 	if (rps->vce_active) {
 		rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
@@ -4106,7 +4112,7 @@ static int si_populate_smc_voltage_tables(struct radeon_device *rdev,
 							      &rdev->pm.dpm.dyn_state.phase_shedding_limits_table)) {
 				si_populate_smc_voltage_table(rdev, &si_pi->vddc_phase_shed_table, table);
 
-				table->phaseMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDC] =
+				table->phaseMaskTable.lowMask[SISLANDS_SMC_VOLTAGEMASK_VDDC_PHASE_SHEDDING] =
 					cpu_to_be32(si_pi->vddc_phase_shed_table.mask_low);
 
 				si_write_smc_soft_register(rdev, SI_SMC_SOFT_REGISTER_phase_shedding_delay,
diff --git a/drivers/gpu/drm/radeon/sid.h b/drivers/gpu/drm/radeon/sid.h
index d1a7b58dd291..eb220eecba78 100644
--- a/drivers/gpu/drm/radeon/sid.h
+++ b/drivers/gpu/drm/radeon/sid.h
@@ -1559,6 +1559,7 @@
 #define UVD_UDEC_ADDR_CONFIG				0xEF4C
 #define UVD_UDEC_DB_ADDR_CONFIG				0xEF50
 #define UVD_UDEC_DBW_ADDR_CONFIG			0xEF54
+#define UVD_NO_OP					0xEFFC
 #define UVD_RBC_RB_RPTR					0xF690
 #define UVD_RBC_RB_WPTR					0xF694
 #define UVD_STATUS					0xf6bc
diff --git a/drivers/gpu/drm/radeon/sislands_smc.h b/drivers/gpu/drm/radeon/sislands_smc.h
index 3c779838d9ab..966e3a556011 100644
--- a/drivers/gpu/drm/radeon/sislands_smc.h
+++ b/drivers/gpu/drm/radeon/sislands_smc.h
@@ -194,6 +194,7 @@ typedef struct SISLANDS_SMC_SWSTATE SISLANDS_SMC_SWSTATE;
 #define SISLANDS_SMC_VOLTAGEMASK_VDDC  0
 #define SISLANDS_SMC_VOLTAGEMASK_MVDD  1
 #define SISLANDS_SMC_VOLTAGEMASK_VDDCI 2
+#define SISLANDS_SMC_VOLTAGEMASK_VDDC_PHASE_SHEDDING 3
 #define SISLANDS_SMC_VOLTAGEMASK_MAX   4
 
 struct SISLANDS_SMC_VOLTAGEMASKTABLE
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
index 899ef7a2a7b4..73c971e39b1c 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
@@ -316,8 +316,8 @@ static int rcar_du_probe(struct platform_device *pdev)
 	rcdu->info = of_match_device(rcar_du_of_table, rcdu->dev)->data;
 
 	ddev = drm_dev_alloc(&rcar_du_driver, &pdev->dev);
-	if (!ddev)
-		return -ENOMEM;
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
 
 	rcdu->ddev = ddev;
 	ddev->dev_private = rcdu;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c b/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
index 4de3ff0dbebd..e03004f4588d 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
@@ -125,6 +125,7 @@ int rcar_du_hdmienc_init(struct rcar_du_device *rcdu,
 
 	/* Link drm_bridge to encoder */
 	bridge->encoder = encoder;
+	encoder->bridge = bridge;
 
 	ret = drm_bridge_attach(rcdu->ddev, bridge);
 	if (ret) {
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
index f03eb55318c1..bd9c3bb9252c 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -257,7 +257,8 @@ static void rcar_du_atomic_complete(struct rcar_du_commit *commit)
 	/* Apply the atomic update. */
 	drm_atomic_helper_commit_modeset_disables(dev, old_state);
 	drm_atomic_helper_commit_modeset_enables(dev, old_state);
-	drm_atomic_helper_commit_planes(dev, old_state, true);
+	drm_atomic_helper_commit_planes(dev, old_state,
+					DRM_PLANE_COMMIT_ACTIVE_ONLY);
 
 	drm_atomic_helper_wait_for_vblanks(dev, old_state);
 
diff --git a/drivers/gpu/drm/rockchip/Makefile b/drivers/gpu/drm/rockchip/Makefile
index 05d07138a2b2..9746365694ba 100644
--- a/drivers/gpu/drm/rockchip/Makefile
+++ b/drivers/gpu/drm/rockchip/Makefile
@@ -3,7 +3,7 @@
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
 rockchipdrm-y := rockchip_drm_drv.o rockchip_drm_fb.o \
-		rockchip_drm_gem.o rockchip_drm_vop.o
+		rockchip_drm_gem.o rockchip_drm_psr.o rockchip_drm_vop.o
 rockchipdrm-$(CONFIG_DRM_FBDEV_EMULATION) += rockchip_drm_fbdev.o
 
 obj-$(CONFIG_ROCKCHIP_ANALOGIX_DP) += analogix_dp-rockchip.o
diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
index 89aadbf465f8..8548e8271639 100644
--- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
@@ -32,6 +32,7 @@
 #include <drm/bridge/analogix_dp.h>
 
 #include "rockchip_drm_drv.h"
+#include "rockchip_drm_psr.h"
 #include "rockchip_drm_vop.h"
 
 #define RK3288_GRF_SOC_CON6		0x25c
@@ -41,6 +42,8 @@
 
 #define HIWORD_UPDATE(val, mask)	(val | (mask) << 16)
 
+#define PSR_WAIT_LINE_FLAG_TIMEOUT_MS	100
+
 #define to_dp(nm)	container_of(nm, struct rockchip_dp_device, nm)
 
 /**
@@ -68,11 +71,65 @@ struct rockchip_dp_device {
 	struct regmap            *grf;
 	struct reset_control     *rst;
 
+	struct work_struct	 psr_work;
+	spinlock_t		 psr_lock;
+	unsigned int             psr_state;
+
 	const struct rockchip_dp_chip_data *data;
 
 	struct analogix_dp_plat_data plat_data;
 };
 
+static void analogix_dp_psr_set(struct drm_encoder *encoder, bool enabled)
+{
+	struct rockchip_dp_device *dp = to_dp(encoder);
+	unsigned long flags;
+
+	if (!analogix_dp_psr_supported(dp->dev))
+		return;
+
+	dev_dbg(dp->dev, "%s PSR...\n", enabled ? "Entry" : "Exit");
+
+	spin_lock_irqsave(&dp->psr_lock, flags);
+	if (enabled)
+		dp->psr_state = EDP_VSC_PSR_STATE_ACTIVE;
+	else
+		dp->psr_state = ~EDP_VSC_PSR_STATE_ACTIVE;
+
+	schedule_work(&dp->psr_work);
+	spin_unlock_irqrestore(&dp->psr_lock, flags);
+}
+
+static void analogix_dp_psr_work(struct work_struct *work)
+{
+	struct rockchip_dp_device *dp =
+				container_of(work, typeof(*dp), psr_work);
+	struct drm_crtc *crtc = dp->encoder.crtc;
+	int psr_state = dp->psr_state;
+	int vact_end;
+	int ret;
+	unsigned long flags;
+
+	if (!crtc)
+		return;
+
+	vact_end = crtc->mode.vtotal - crtc->mode.vsync_start + crtc->mode.vdisplay;
+
+	ret = rockchip_drm_wait_line_flag(dp->encoder.crtc, vact_end,
+					  PSR_WAIT_LINE_FLAG_TIMEOUT_MS);
+	if (ret) {
+		dev_err(dp->dev, "line flag interrupt did not arrive\n");
+		return;
+	}
+
+	spin_lock_irqsave(&dp->psr_lock, flags);
+	if (psr_state == EDP_VSC_PSR_STATE_ACTIVE)
+		analogix_dp_enable_psr(dp->dev);
+	else
+		analogix_dp_disable_psr(dp->dev);
+	spin_unlock_irqrestore(&dp->psr_lock, flags);
+}
+
 static int rockchip_dp_pre_init(struct rockchip_dp_device *dp)
 {
 	reset_control_assert(dp->rst);
@@ -87,6 +144,8 @@ static int rockchip_dp_poweron(struct analogix_dp_plat_data *plat_data)
 	struct rockchip_dp_device *dp = to_dp(plat_data);
 	int ret;
 
+	cancel_work_sync(&dp->psr_work);
+
 	ret = clk_prepare_enable(dp->pclk);
 	if (ret < 0) {
 		dev_err(dp->dev, "failed to enable pclk %d\n", ret);
@@ -342,12 +401,22 @@ static int rockchip_dp_bind(struct device *dev, struct device *master,
 	dp->plat_data.power_off = rockchip_dp_powerdown;
 	dp->plat_data.get_modes = rockchip_dp_get_modes;
 
+	spin_lock_init(&dp->psr_lock);
+	dp->psr_state = ~EDP_VSC_PSR_STATE_ACTIVE;
+	INIT_WORK(&dp->psr_work, analogix_dp_psr_work);
+
+	rockchip_drm_psr_register(&dp->encoder, analogix_dp_psr_set);
+
 	return analogix_dp_bind(dev, dp->drm_dev, &dp->plat_data);
 }
 
 static void rockchip_dp_unbind(struct device *dev, struct device *master,
 			       void *data)
 {
+	struct rockchip_dp_device *dp = dev_get_drvdata(dev);
+
+	rockchip_drm_psr_unregister(&dp->encoder);
+
 	return analogix_dp_unbind(dev, master, data);
 }
 
@@ -381,10 +450,8 @@ static int rockchip_dp_probe(struct platform_device *pdev)
 
 		panel = of_drm_find_panel(panel_node);
 		of_node_put(panel_node);
-		if (!panel) {
-			DRM_ERROR("failed to find panel\n");
+		if (!panel)
 			return -EPROBE_DEFER;
-		}
 	}
 
 	dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL);
@@ -445,7 +512,6 @@ static struct platform_driver rockchip_dp_driver = {
 	.remove = rockchip_dp_remove,
 	.driver = {
 		   .name = "rockchip-dp",
-		   .owner = THIS_MODULE,
 		   .pm = &rockchip_dp_pm_ops,
 		   .of_match_table = of_match_ptr(rockchip_dp_dt_ids),
 	},
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
index a822d49a255a..8c8cbe837e61 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
@@ -143,8 +143,8 @@ static int rockchip_drm_bind(struct device *dev)
 	int ret;
 
 	drm_dev = drm_dev_alloc(&rockchip_drm_driver, dev);
-	if (!drm_dev)
-		return -ENOMEM;
+	if (IS_ERR(drm_dev))
+		return PTR_ERR(drm_dev);
 
 	dev_set_drvdata(dev, drm_dev);
 
@@ -156,6 +156,9 @@ static int rockchip_drm_bind(struct device *dev)
 
 	drm_dev->dev_private = private;
 
+	INIT_LIST_HEAD(&private->psr_list);
+	spin_lock_init(&private->psr_list_lock);
+
 	drm_mode_config_init(drm_dev);
 
 	rockchip_drm_mode_config_init(drm_dev);
@@ -306,7 +309,7 @@ static struct drm_driver rockchip_drm_driver = {
 };
 
 #ifdef CONFIG_PM_SLEEP
-void rockchip_drm_fb_suspend(struct drm_device *drm)
+static void rockchip_drm_fb_suspend(struct drm_device *drm)
 {
 	struct rockchip_drm_private *priv = drm->dev_private;
 
@@ -315,7 +318,7 @@ void rockchip_drm_fb_suspend(struct drm_device *drm)
 	console_unlock();
 }
 
-void rockchip_drm_fb_resume(struct drm_device *drm)
+static void rockchip_drm_fb_resume(struct drm_device *drm)
 {
 	struct rockchip_drm_private *priv = drm->dev_private;
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
index ea3932940061..fb6226cf84b7 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
@@ -39,7 +39,6 @@ struct drm_connector;
 struct rockchip_crtc_funcs {
 	int (*enable_vblank)(struct drm_crtc *crtc);
 	void (*disable_vblank)(struct drm_crtc *crtc);
-	void (*wait_for_update)(struct drm_crtc *crtc);
 };
 
 struct rockchip_crtc_state {
@@ -61,6 +60,9 @@ struct rockchip_drm_private {
 	struct drm_gem_object *fbdev_bo;
 	const struct rockchip_crtc_funcs *crtc_funcs[ROCKCHIP_MAX_CRTC];
 	struct drm_atomic_state *state;
+
+	struct list_head psr_list;
+	spinlock_t psr_list_lock;
 };
 
 int rockchip_register_crtc_funcs(struct drm_crtc *crtc,
@@ -70,4 +72,7 @@ int rockchip_drm_dma_attach_device(struct drm_device *drm_dev,
 				   struct device *dev);
 void rockchip_drm_dma_detach_device(struct drm_device *drm_dev,
 				    struct device *dev);
+int rockchip_drm_wait_line_flag(struct drm_crtc *crtc, unsigned int line_num,
+				unsigned int mstimeout);
+
 #endif /* _ROCKCHIP_DRM_DRV_H_ */
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
index 55c52734c52d..0f6eda023bd0 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
@@ -22,6 +22,7 @@
 #include "rockchip_drm_drv.h"
 #include "rockchip_drm_fb.h"
 #include "rockchip_drm_gem.h"
+#include "rockchip_drm_psr.h"
 
 #define to_rockchip_fb(x) container_of(x, struct rockchip_drm_fb, fb)
 
@@ -63,9 +64,20 @@ static int rockchip_drm_fb_create_handle(struct drm_framebuffer *fb,
 				     rockchip_fb->obj[0], handle);
 }
 
+static int rockchip_drm_fb_dirty(struct drm_framebuffer *fb,
+				 struct drm_file *file,
+				 unsigned int flags, unsigned int color,
+				 struct drm_clip_rect *clips,
+				 unsigned int num_clips)
+{
+	rockchip_drm_psr_flush_all(fb->dev);
+	return 0;
+}
+
 static const struct drm_framebuffer_funcs rockchip_drm_fb_funcs = {
 	.destroy	= rockchip_drm_fb_destroy,
 	.create_handle	= rockchip_drm_fb_create_handle,
+	.dirty		= rockchip_drm_fb_dirty,
 };
 
 static struct rockchip_drm_fb *
@@ -162,68 +174,6 @@ static void rockchip_drm_output_poll_changed(struct drm_device *dev)
 		drm_fb_helper_hotplug_event(fb_helper);
 }
 
-static void rockchip_crtc_wait_for_update(struct drm_crtc *crtc)
-{
-	struct rockchip_drm_private *priv = crtc->dev->dev_private;
-	int pipe = drm_crtc_index(crtc);
-	const struct rockchip_crtc_funcs *crtc_funcs = priv->crtc_funcs[pipe];
-
-	if (crtc_funcs && crtc_funcs->wait_for_update)
-		crtc_funcs->wait_for_update(crtc);
-}
-
-/*
- * We can't use drm_atomic_helper_wait_for_vblanks() because rk3288 and rk3066
- * have hardware counters for neither vblanks nor scanlines, which results in
- * a race where:
- *				| <-- HW vsync irq and reg take effect
- *	       plane_commit --> |
- *	get_vblank and wait --> |
- *				| <-- handle_vblank, vblank->count + 1
- *		 cleanup_fb --> |
- *		iommu crash --> |
- *				| <-- HW vsync irq and reg take effect
- *
- * This function is equivalent but uses rockchip_crtc_wait_for_update() instead
- * of waiting for vblank_count to change.
- */
-static void
-rockchip_atomic_wait_for_complete(struct drm_device *dev, struct drm_atomic_state *old_state)
-{
-	struct drm_crtc_state *old_crtc_state;
-	struct drm_crtc *crtc;
-	int i, ret;
-
-	for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) {
-		/* No one cares about the old state, so abuse it for tracking
-		 * and store whether we hold a vblank reference (and should do a
-		 * vblank wait) in the ->enable boolean.
-		 */
-		old_crtc_state->enable = false;
-
-		if (!crtc->state->active)
-			continue;
-
-		if (!drm_atomic_helper_framebuffer_changed(dev,
-				old_state, crtc))
-			continue;
-
-		ret = drm_crtc_vblank_get(crtc);
-		if (ret != 0)
-			continue;
-
-		old_crtc_state->enable = true;
-	}
-
-	for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) {
-		if (!old_crtc_state->enable)
-			continue;
-
-		rockchip_crtc_wait_for_update(crtc);
-		drm_crtc_vblank_put(crtc);
-	}
-}
-
 static void
 rockchip_atomic_commit_tail(struct drm_atomic_state *state)
 {
@@ -233,11 +183,12 @@ rockchip_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_modeset_enables(dev, state);
 
-	drm_atomic_helper_commit_planes(dev, state, true);
+	drm_atomic_helper_commit_planes(dev, state,
+					DRM_PLANE_COMMIT_ACTIVE_ONLY);
 
 	drm_atomic_helper_commit_hw_done(state);
 
-	rockchip_atomic_wait_for_complete(dev, state);
+	drm_atomic_helper_wait_for_vblanks(dev, state);
 
 	drm_atomic_helper_cleanup_planes(dev, state);
 }
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
index 207e01de6e32..a16c69f96ed5 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
@@ -20,6 +20,7 @@
 #include "rockchip_drm_drv.h"
 #include "rockchip_drm_gem.h"
 #include "rockchip_drm_fb.h"
+#include "rockchip_drm_fbdev.h"
 
 #define PREFERRED_BPP		32
 #define to_drm_private(x) \
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_psr.c b/drivers/gpu/drm/rockchip/rockchip_drm_psr.c
new file mode 100644
index 000000000000..a553e182ff53
--- /dev/null
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_psr.c
@@ -0,0 +1,275 @@
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Yakir Yang <ykk@rock-chips.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "rockchip_drm_drv.h"
+#include "rockchip_drm_psr.h"
+
+#define PSR_FLUSH_TIMEOUT	msecs_to_jiffies(100)
+
+enum psr_state {
+	PSR_FLUSH,
+	PSR_ENABLE,
+	PSR_DISABLE,
+};
+
+struct psr_drv {
+	struct list_head	list;
+	struct drm_encoder	*encoder;
+
+	spinlock_t		lock;
+	bool			active;
+	enum psr_state		state;
+
+	struct timer_list	flush_timer;
+
+	void (*set)(struct drm_encoder *encoder, bool enable);
+};
+
+static struct psr_drv *find_psr_by_crtc(struct drm_crtc *crtc)
+{
+	struct rockchip_drm_private *drm_drv = crtc->dev->dev_private;
+	struct psr_drv *psr;
+	unsigned long flags;
+
+	spin_lock_irqsave(&drm_drv->psr_list_lock, flags);
+	list_for_each_entry(psr, &drm_drv->psr_list, list) {
+		if (psr->encoder->crtc == crtc)
+			goto out;
+	}
+	psr = ERR_PTR(-ENODEV);
+
+out:
+	spin_unlock_irqrestore(&drm_drv->psr_list_lock, flags);
+	return psr;
+}
+
+static void psr_set_state_locked(struct psr_drv *psr, enum psr_state state)
+{
+	/*
+	 * Allowed finite state machine:
+	 *
+	 *   PSR_ENABLE  < = = = = = >  PSR_FLUSH
+	 *       | ^                        |
+	 *       | |                        |
+	 *       v |                        |
+	 *   PSR_DISABLE < - - - - - - - - -
+	 */
+	if (state == psr->state || !psr->active)
+		return;
+
+	/* Already disabled in flush, change the state, but not the hardware */
+	if (state == PSR_DISABLE && psr->state == PSR_FLUSH) {
+		psr->state = state;
+		return;
+	}
+
+	psr->state = state;
+
+	/* Actually commit the state change to hardware */
+	switch (psr->state) {
+	case PSR_ENABLE:
+		psr->set(psr->encoder, true);
+		break;
+
+	case PSR_DISABLE:
+	case PSR_FLUSH:
+		psr->set(psr->encoder, false);
+		break;
+	}
+}
+
+static void psr_set_state(struct psr_drv *psr, enum psr_state state)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&psr->lock, flags);
+	psr_set_state_locked(psr, state);
+	spin_unlock_irqrestore(&psr->lock, flags);
+}
+
+static void psr_flush_handler(unsigned long data)
+{
+	struct psr_drv *psr = (struct psr_drv *)data;
+	unsigned long flags;
+
+	/* If the state has changed since we initiated the flush, do nothing */
+	spin_lock_irqsave(&psr->lock, flags);
+	if (psr->state == PSR_FLUSH)
+		psr_set_state_locked(psr, PSR_ENABLE);
+	spin_unlock_irqrestore(&psr->lock, flags);
+}
+
+/**
+ * rockchip_drm_psr_activate - activate PSR on the given pipe
+ * @crtc: CRTC to obtain the PSR encoder
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int rockchip_drm_psr_activate(struct drm_crtc *crtc)
+{
+	struct psr_drv *psr = find_psr_by_crtc(crtc);
+	unsigned long flags;
+
+	if (IS_ERR(psr))
+		return PTR_ERR(psr);
+
+	spin_lock_irqsave(&psr->lock, flags);
+	psr->active = true;
+	spin_unlock_irqrestore(&psr->lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL(rockchip_drm_psr_activate);
+
+/**
+ * rockchip_drm_psr_deactivate - deactivate PSR on the given pipe
+ * @crtc: CRTC to obtain the PSR encoder
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int rockchip_drm_psr_deactivate(struct drm_crtc *crtc)
+{
+	struct psr_drv *psr = find_psr_by_crtc(crtc);
+	unsigned long flags;
+
+	if (IS_ERR(psr))
+		return PTR_ERR(psr);
+
+	spin_lock_irqsave(&psr->lock, flags);
+	psr->active = false;
+	spin_unlock_irqrestore(&psr->lock, flags);
+	del_timer_sync(&psr->flush_timer);
+
+	return 0;
+}
+EXPORT_SYMBOL(rockchip_drm_psr_deactivate);
+
+static void rockchip_drm_do_flush(struct psr_drv *psr)
+{
+	mod_timer(&psr->flush_timer,
+		  round_jiffies_up(jiffies + PSR_FLUSH_TIMEOUT));
+	psr_set_state(psr, PSR_FLUSH);
+}
+
+/**
+ * rockchip_drm_psr_flush - flush a single pipe
+ * @crtc: CRTC of the pipe to flush
+ *
+ * Returns:
+ * 0 on success, -errno on fail
+ */
+int rockchip_drm_psr_flush(struct drm_crtc *crtc)
+{
+	struct psr_drv *psr = find_psr_by_crtc(crtc);
+	if (IS_ERR(psr))
+		return PTR_ERR(psr);
+
+	rockchip_drm_do_flush(psr);
+	return 0;
+}
+EXPORT_SYMBOL(rockchip_drm_psr_flush);
+
+/**
+ * rockchip_drm_psr_flush_all - force to flush all registered PSR encoders
+ * @dev: drm device
+ *
+ * Disable the PSR function for all registered encoders, and then enable the
+ * PSR function back after PSR_FLUSH_TIMEOUT. If encoder PSR state have been
+ * changed during flush time, then keep the state no change after flush
+ * timeout.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+void rockchip_drm_psr_flush_all(struct drm_device *dev)
+{
+	struct rockchip_drm_private *drm_drv = dev->dev_private;
+	struct psr_drv *psr;
+	unsigned long flags;
+
+	spin_lock_irqsave(&drm_drv->psr_list_lock, flags);
+	list_for_each_entry(psr, &drm_drv->psr_list, list)
+		rockchip_drm_do_flush(psr);
+	spin_unlock_irqrestore(&drm_drv->psr_list_lock, flags);
+}
+EXPORT_SYMBOL(rockchip_drm_psr_flush_all);
+
+/**
+ * rockchip_drm_psr_register - register encoder to psr driver
+ * @encoder: encoder that obtain the PSR function
+ * @psr_set: call back to set PSR state
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int rockchip_drm_psr_register(struct drm_encoder *encoder,
+			void (*psr_set)(struct drm_encoder *, bool enable))
+{
+	struct rockchip_drm_private *drm_drv = encoder->dev->dev_private;
+	struct psr_drv *psr;
+	unsigned long flags;
+
+	if (!encoder || !psr_set)
+		return -EINVAL;
+
+	psr = kzalloc(sizeof(struct psr_drv), GFP_KERNEL);
+	if (!psr)
+		return -ENOMEM;
+
+	setup_timer(&psr->flush_timer, psr_flush_handler, (unsigned long)psr);
+	spin_lock_init(&psr->lock);
+
+	psr->active = true;
+	psr->state = PSR_DISABLE;
+	psr->encoder = encoder;
+	psr->set = psr_set;
+
+	spin_lock_irqsave(&drm_drv->psr_list_lock, flags);
+	list_add_tail(&psr->list, &drm_drv->psr_list);
+	spin_unlock_irqrestore(&drm_drv->psr_list_lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL(rockchip_drm_psr_register);
+
+/**
+ * rockchip_drm_psr_unregister - unregister encoder to psr driver
+ * @encoder: encoder that obtain the PSR function
+ * @psr_set: call back to set PSR state
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+void rockchip_drm_psr_unregister(struct drm_encoder *encoder)
+{
+	struct rockchip_drm_private *drm_drv = encoder->dev->dev_private;
+	struct psr_drv *psr, *n;
+	unsigned long flags;
+
+	spin_lock_irqsave(&drm_drv->psr_list_lock, flags);
+	list_for_each_entry_safe(psr, n, &drm_drv->psr_list, list) {
+		if (psr->encoder == encoder) {
+			del_timer(&psr->flush_timer);
+			list_del(&psr->list);
+			kfree(psr);
+		}
+	}
+	spin_unlock_irqrestore(&drm_drv->psr_list_lock, flags);
+}
+EXPORT_SYMBOL(rockchip_drm_psr_unregister);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_psr.h b/drivers/gpu/drm/rockchip/rockchip_drm_psr.h
new file mode 100644
index 000000000000..b420cf1bf902
--- /dev/null
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_psr.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Yakir Yang <ykk@rock-chips.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __ROCKCHIP_DRM_PSR___
+#define __ROCKCHIP_DRM_PSR___
+
+void rockchip_drm_psr_flush_all(struct drm_device *dev);
+int rockchip_drm_psr_flush(struct drm_crtc *crtc);
+
+int rockchip_drm_psr_activate(struct drm_crtc *crtc);
+int rockchip_drm_psr_deactivate(struct drm_crtc *crtc);
+
+int rockchip_drm_psr_register(struct drm_encoder *encoder,
+			void (*psr_set)(struct drm_encoder *, bool enable));
+void rockchip_drm_psr_unregister(struct drm_encoder *encoder);
+
+#endif /* __ROCKCHIP_DRM_PSR__ */
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 91305eb7d312..c7eba305c488 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -17,12 +17,14 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_flip_work.h>
 #include <drm/drm_plane_helper.h>
 
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/clk.h>
+#include <linux/iopoll.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/pm_runtime.h>
@@ -34,17 +36,21 @@
 #include "rockchip_drm_drv.h"
 #include "rockchip_drm_gem.h"
 #include "rockchip_drm_fb.h"
+#include "rockchip_drm_psr.h"
 #include "rockchip_drm_vop.h"
 
-#define __REG_SET_RELAXED(x, off, mask, shift, v) \
-		vop_mask_write_relaxed(x, off, (mask) << shift, (v) << shift)
-#define __REG_SET_NORMAL(x, off, mask, shift, v) \
-		vop_mask_write(x, off, (mask) << shift, (v) << shift)
+#define __REG_SET_RELAXED(x, off, mask, shift, v, write_mask) \
+		vop_mask_write(x, off, mask, shift, v, write_mask, true)
+
+#define __REG_SET_NORMAL(x, off, mask, shift, v, write_mask) \
+		vop_mask_write(x, off, mask, shift, v, write_mask, false)
 
 #define REG_SET(x, base, reg, v, mode) \
-		__REG_SET_##mode(x, base + reg.offset, reg.mask, reg.shift, v)
+		__REG_SET_##mode(x, base + reg.offset, \
+				 reg.mask, reg.shift, v, reg.write_mask)
 #define REG_SET_MASK(x, base, reg, mask, v, mode) \
-		__REG_SET_##mode(x, base + reg.offset, mask, reg.shift, v)
+		__REG_SET_##mode(x, base + reg.offset, \
+				 mask, reg.shift, v, reg.write_mask)
 
 #define VOP_WIN_SET(x, win, name, v) \
 		REG_SET(x, win->base, win->phy->name, v, RELAXED)
@@ -82,25 +88,15 @@
 
 #define to_vop(x) container_of(x, struct vop, crtc)
 #define to_vop_win(x) container_of(x, struct vop_win, base)
-#define to_vop_plane_state(x) container_of(x, struct vop_plane_state, base)
 
-struct vop_plane_state {
-	struct drm_plane_state base;
-	int format;
-	struct drm_rect src;
-	struct drm_rect dest;
-	dma_addr_t yrgb_mst;
-	bool enable;
+enum vop_pending {
+	VOP_PENDING_FB_UNREF,
 };
 
 struct vop_win {
 	struct drm_plane base;
 	const struct vop_win_data *data;
 	struct vop *vop;
-
-	/* protected by dev->event_lock */
-	bool enable;
-	dma_addr_t yrgb_mst;
 };
 
 struct vop {
@@ -113,11 +109,15 @@ struct vop {
 	struct mutex vsync_mutex;
 	bool vsync_work_pending;
 	struct completion dsp_hold_completion;
-	struct completion wait_update_complete;
 
 	/* protected by dev->event_lock */
 	struct drm_pending_vblank_event *event;
 
+	struct drm_flip_work fb_unref_work;
+	unsigned long pending;
+
+	struct completion line_flag_completion;
+
 	const struct vop_data *data;
 
 	uint32_t *regsbak;
@@ -164,27 +164,25 @@ static inline uint32_t vop_read_reg(struct vop *vop, uint32_t base,
 }
 
 static inline void vop_mask_write(struct vop *vop, uint32_t offset,
-				  uint32_t mask, uint32_t v)
+				  uint32_t mask, uint32_t shift, uint32_t v,
+				  bool write_mask, bool relaxed)
 {
-	if (mask) {
-		uint32_t cached_val = vop->regsbak[offset >> 2];
-
-		cached_val = (cached_val & ~mask) | v;
-		writel(cached_val, vop->regs + offset);
-		vop->regsbak[offset >> 2] = cached_val;
-	}
-}
+	if (!mask)
+		return;
 
-static inline void vop_mask_write_relaxed(struct vop *vop, uint32_t offset,
-					  uint32_t mask, uint32_t v)
-{
-	if (mask) {
+	if (write_mask) {
+		v = ((v << shift) & 0xffff) | (mask << (shift + 16));
+	} else {
 		uint32_t cached_val = vop->regsbak[offset >> 2];
 
-		cached_val = (cached_val & ~mask) | v;
-		writel_relaxed(cached_val, vop->regs + offset);
-		vop->regsbak[offset >> 2] = cached_val;
+		v = (cached_val & ~(mask << shift)) | ((v & mask) << shift);
+		vop->regsbak[offset >> 2] = v;
 	}
+
+	if (relaxed)
+		writel_relaxed(v, vop->regs + offset);
+	else
+		writel(v, vop->regs + offset);
 }
 
 static inline uint32_t vop_get_intr_type(struct vop *vop,
@@ -240,7 +238,7 @@ static enum vop_data_format vop_convert_format(uint32_t format)
 	case DRM_FORMAT_NV24:
 		return VOP_FMT_YUV444SP;
 	default:
-		DRM_ERROR("unsupport format[%08x]\n", format);
+		DRM_ERROR("unsupported format[%08x]\n", format);
 		return -EINVAL;
 	}
 }
@@ -317,7 +315,7 @@ static void scl_vop_cal_scl_fac(struct vop *vop, const struct vop_win_data *win,
 	int vskiplines = 0;
 
 	if (dst_w > 3840) {
-		DRM_ERROR("Maximum destination width (3840) exceeded\n");
+		DRM_DEV_ERROR(vop->dev, "Maximum dst width (3840) exceeded\n");
 		return;
 	}
 
@@ -355,11 +353,11 @@ static void scl_vop_cal_scl_fac(struct vop *vop, const struct vop_win_data *win,
 	VOP_SCL_SET_EXT(vop, win, lb_mode, lb_mode);
 	if (lb_mode == LB_RGB_3840X2) {
 		if (yrgb_ver_scl_mode != SCALE_NONE) {
-			DRM_ERROR("ERROR : not allow yrgb ver scale\n");
+			DRM_DEV_ERROR(vop->dev, "not allow yrgb ver scale\n");
 			return;
 		}
 		if (cbcr_ver_scl_mode != SCALE_NONE) {
-			DRM_ERROR("ERROR : not allow cbcr ver scale\n");
+			DRM_DEV_ERROR(vop->dev, "not allow cbcr ver scale\n");
 			return;
 		}
 		vsu_mode = SCALE_UP_BIL;
@@ -411,6 +409,7 @@ static void vop_dsp_hold_valid_irq_enable(struct vop *vop)
 
 	spin_lock_irqsave(&vop->irq_lock, flags);
 
+	VOP_INTR_SET_TYPE(vop, clear, DSP_HOLD_VALID_INTR, 1);
 	VOP_INTR_SET_TYPE(vop, enable, DSP_HOLD_VALID_INTR, 1);
 
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
@@ -430,7 +429,73 @@ static void vop_dsp_hold_valid_irq_disable(struct vop *vop)
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
 }
 
-static void vop_enable(struct drm_crtc *crtc)
+/*
+ * (1) each frame starts at the start of the Vsync pulse which is signaled by
+ *     the "FRAME_SYNC" interrupt.
+ * (2) the active data region of each frame ends at dsp_vact_end
+ * (3) we should program this same number (dsp_vact_end) into dsp_line_frag_num,
+ *      to get "LINE_FLAG" interrupt at the end of the active on screen data.
+ *
+ * VOP_INTR_CTRL0.dsp_line_frag_num = VOP_DSP_VACT_ST_END.dsp_vact_end
+ * Interrupts
+ * LINE_FLAG -------------------------------+
+ * FRAME_SYNC ----+                         |
+ *                |                         |
+ *                v                         v
+ *                | Vsync | Vbp |  Vactive  | Vfp |
+ *                        ^     ^           ^     ^
+ *                        |     |           |     |
+ *                        |     |           |     |
+ * dsp_vs_end ------------+     |           |     |   VOP_DSP_VTOTAL_VS_END
+ * dsp_vact_start --------------+           |     |   VOP_DSP_VACT_ST_END
+ * dsp_vact_end ----------------------------+     |   VOP_DSP_VACT_ST_END
+ * dsp_total -------------------------------------+   VOP_DSP_VTOTAL_VS_END
+ */
+static bool vop_line_flag_irq_is_enabled(struct vop *vop)
+{
+	uint32_t line_flag_irq;
+	unsigned long flags;
+
+	spin_lock_irqsave(&vop->irq_lock, flags);
+
+	line_flag_irq = VOP_INTR_GET_TYPE(vop, enable, LINE_FLAG_INTR);
+
+	spin_unlock_irqrestore(&vop->irq_lock, flags);
+
+	return !!line_flag_irq;
+}
+
+static void vop_line_flag_irq_enable(struct vop *vop, int line_num)
+{
+	unsigned long flags;
+
+	if (WARN_ON(!vop->is_enabled))
+		return;
+
+	spin_lock_irqsave(&vop->irq_lock, flags);
+
+	VOP_CTRL_SET(vop, line_flag_num[0], line_num);
+	VOP_INTR_SET_TYPE(vop, clear, LINE_FLAG_INTR, 1);
+	VOP_INTR_SET_TYPE(vop, enable, LINE_FLAG_INTR, 1);
+
+	spin_unlock_irqrestore(&vop->irq_lock, flags);
+}
+
+static void vop_line_flag_irq_disable(struct vop *vop)
+{
+	unsigned long flags;
+
+	if (WARN_ON(!vop->is_enabled))
+		return;
+
+	spin_lock_irqsave(&vop->irq_lock, flags);
+
+	VOP_INTR_SET_TYPE(vop, enable, LINE_FLAG_INTR, 0);
+
+	spin_unlock_irqrestore(&vop->irq_lock, flags);
+}
+
+static int vop_enable(struct drm_crtc *crtc)
 {
 	struct vop *vop = to_vop(crtc);
 	int ret;
@@ -438,26 +503,20 @@ static void vop_enable(struct drm_crtc *crtc)
 	ret = pm_runtime_get_sync(vop->dev);
 	if (ret < 0) {
 		dev_err(vop->dev, "failed to get pm runtime: %d\n", ret);
-		return;
+		goto err_put_pm_runtime;
 	}
 
 	ret = clk_enable(vop->hclk);
-	if (ret < 0) {
-		dev_err(vop->dev, "failed to enable hclk - %d\n", ret);
-		return;
-	}
+	if (WARN_ON(ret < 0))
+		goto err_put_pm_runtime;
 
 	ret = clk_enable(vop->dclk);
-	if (ret < 0) {
-		dev_err(vop->dev, "failed to enable dclk - %d\n", ret);
+	if (WARN_ON(ret < 0))
 		goto err_disable_hclk;
-	}
 
 	ret = clk_enable(vop->aclk);
-	if (ret < 0) {
-		dev_err(vop->dev, "failed to enable aclk - %d\n", ret);
+	if (WARN_ON(ret < 0))
 		goto err_disable_dclk;
-	}
 
 	/*
 	 * Slave iommu shares power, irq and clock with vop.  It was associated
@@ -487,7 +546,7 @@ static void vop_enable(struct drm_crtc *crtc)
 
 	drm_crtc_vblank_on(crtc);
 
-	return;
+	return 0;
 
 err_disable_aclk:
 	clk_disable(vop->aclk);
@@ -495,6 +554,9 @@ err_disable_dclk:
 	clk_disable(vop->dclk);
 err_disable_hclk:
 	clk_disable(vop->hclk);
+err_put_pm_runtime:
+	pm_runtime_put_sync(vop->dev);
+	return ret;
 }
 
 static void vop_crtc_disable(struct drm_crtc *crtc)
@@ -504,6 +566,8 @@ static void vop_crtc_disable(struct drm_crtc *crtc)
 
 	WARN_ON(vop->event);
 
+	rockchip_drm_psr_deactivate(&vop->crtc);
+
 	/*
 	 * We need to make sure that all windows are disabled before we
 	 * disable that crtc. Otherwise we might try to scan from a destroyed
@@ -568,22 +632,6 @@ static void vop_plane_destroy(struct drm_plane *plane)
 	drm_plane_cleanup(plane);
 }
 
-static int vop_plane_prepare_fb(struct drm_plane *plane,
-				const struct drm_plane_state *new_state)
-{
-	if (plane->state->fb)
-		drm_framebuffer_reference(plane->state->fb);
-
-	return 0;
-}
-
-static void vop_plane_cleanup_fb(struct drm_plane *plane,
-				 const struct drm_plane_state *old_state)
-{
-	if (old_state->fb)
-		drm_framebuffer_unreference(old_state->fb);
-}
-
 static int vop_plane_atomic_check(struct drm_plane *plane,
 			   struct drm_plane_state *state)
 {
@@ -591,12 +639,8 @@ static int vop_plane_atomic_check(struct drm_plane *plane,
 	struct drm_crtc_state *crtc_state;
 	struct drm_framebuffer *fb = state->fb;
 	struct vop_win *vop_win = to_vop_win(plane);
-	struct vop_plane_state *vop_plane_state = to_vop_plane_state(state);
 	const struct vop_win_data *win = vop_win->data;
-	bool visible;
 	int ret;
-	struct drm_rect *dest = &vop_plane_state->dest;
-	struct drm_rect *src = &vop_plane_state->src;
 	struct drm_rect clip;
 	int min_scale = win->phy->scl ? FRAC_16_16(1, 8) :
 					DRM_PLANE_HELPER_NO_SCALING;
@@ -604,62 +648,43 @@ static int vop_plane_atomic_check(struct drm_plane *plane,
 					DRM_PLANE_HELPER_NO_SCALING;
 
 	if (!crtc || !fb)
-		goto out_disable;
+		return 0;
 
 	crtc_state = drm_atomic_get_existing_crtc_state(state->state, crtc);
 	if (WARN_ON(!crtc_state))
 		return -EINVAL;
 
-	src->x1 = state->src_x;
-	src->y1 = state->src_y;
-	src->x2 = state->src_x + state->src_w;
-	src->y2 = state->src_y + state->src_h;
-	dest->x1 = state->crtc_x;
-	dest->y1 = state->crtc_y;
-	dest->x2 = state->crtc_x + state->crtc_w;
-	dest->y2 = state->crtc_y + state->crtc_h;
-
 	clip.x1 = 0;
 	clip.y1 = 0;
 	clip.x2 = crtc_state->adjusted_mode.hdisplay;
 	clip.y2 = crtc_state->adjusted_mode.vdisplay;
 
-	ret = drm_plane_helper_check_update(plane, crtc, state->fb,
-					    src, dest, &clip,
-					    state->rotation,
-					    min_scale,
-					    max_scale,
-					    true, true, &visible);
+	ret = drm_plane_helper_check_state(state, &clip,
+					   min_scale, max_scale,
+					   true, true);
 	if (ret)
 		return ret;
 
-	if (!visible)
-		goto out_disable;
+	if (!state->visible)
+		return 0;
 
-	vop_plane_state->format = vop_convert_format(fb->pixel_format);
-	if (vop_plane_state->format < 0)
-		return vop_plane_state->format;
+	ret = vop_convert_format(fb->pixel_format);
+	if (ret < 0)
+		return ret;
 
 	/*
 	 * Src.x1 can be odd when do clip, but yuv plane start point
 	 * need align with 2 pixel.
 	 */
-	if (is_yuv_support(fb->pixel_format) && ((src->x1 >> 16) % 2))
+	if (is_yuv_support(fb->pixel_format) && ((state->src.x1 >> 16) % 2))
 		return -EINVAL;
 
-	vop_plane_state->enable = true;
-
-	return 0;
-
-out_disable:
-	vop_plane_state->enable = false;
 	return 0;
 }
 
 static void vop_plane_atomic_disable(struct drm_plane *plane,
 				     struct drm_plane_state *old_state)
 {
-	struct vop_plane_state *vop_plane_state = to_vop_plane_state(old_state);
 	struct vop_win *vop_win = to_vop_win(plane);
 	const struct vop_win_data *win = vop_win->data;
 	struct vop *vop = to_vop(old_state->crtc);
@@ -667,18 +692,11 @@ static void vop_plane_atomic_disable(struct drm_plane *plane,
 	if (!old_state->crtc)
 		return;
 
-	spin_lock_irq(&plane->dev->event_lock);
-	vop_win->enable = false;
-	vop_win->yrgb_mst = 0;
-	spin_unlock_irq(&plane->dev->event_lock);
-
 	spin_lock(&vop->reg_lock);
 
 	VOP_WIN_SET(vop, win, enable, 0);
 
 	spin_unlock(&vop->reg_lock);
-
-	vop_plane_state->enable = false;
 }
 
 static void vop_plane_atomic_update(struct drm_plane *plane,
@@ -687,21 +705,21 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
 	struct drm_plane_state *state = plane->state;
 	struct drm_crtc *crtc = state->crtc;
 	struct vop_win *vop_win = to_vop_win(plane);
-	struct vop_plane_state *vop_plane_state = to_vop_plane_state(state);
 	const struct vop_win_data *win = vop_win->data;
 	struct vop *vop = to_vop(state->crtc);
 	struct drm_framebuffer *fb = state->fb;
 	unsigned int actual_w, actual_h;
 	unsigned int dsp_stx, dsp_sty;
 	uint32_t act_info, dsp_info, dsp_st;
-	struct drm_rect *src = &vop_plane_state->src;
-	struct drm_rect *dest = &vop_plane_state->dest;
+	struct drm_rect *src = &state->src;
+	struct drm_rect *dest = &state->dst;
 	struct drm_gem_object *obj, *uv_obj;
 	struct rockchip_gem_object *rk_obj, *rk_uv_obj;
 	unsigned long offset;
 	dma_addr_t dma_addr;
 	uint32_t val;
 	bool rb_swap;
+	int format;
 
 	/*
 	 * can't update plane when vop is disabled.
@@ -712,7 +730,7 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
 	if (WARN_ON(!vop->is_enabled))
 		return;
 
-	if (!vop_plane_state->enable) {
+	if (!state->visible) {
 		vop_plane_atomic_disable(plane, old_state);
 		return;
 	}
@@ -733,18 +751,15 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
 
 	offset = (src->x1 >> 16) * drm_format_plane_cpp(fb->pixel_format, 0);
 	offset += (src->y1 >> 16) * fb->pitches[0];
-	vop_plane_state->yrgb_mst = rk_obj->dma_addr + offset + fb->offsets[0];
+	dma_addr = rk_obj->dma_addr + offset + fb->offsets[0];
 
-	spin_lock_irq(&plane->dev->event_lock);
-	vop_win->enable = true;
-	vop_win->yrgb_mst = vop_plane_state->yrgb_mst;
-	spin_unlock_irq(&plane->dev->event_lock);
+	format = vop_convert_format(fb->pixel_format);
 
 	spin_lock(&vop->reg_lock);
 
-	VOP_WIN_SET(vop, win, format, vop_plane_state->format);
+	VOP_WIN_SET(vop, win, format, format);
 	VOP_WIN_SET(vop, win, yrgb_vir, fb->pitches[0] >> 2);
-	VOP_WIN_SET(vop, win, yrgb_mst, vop_plane_state->yrgb_mst);
+	VOP_WIN_SET(vop, win, yrgb_mst, dma_addr);
 	if (is_yuv_support(fb->pixel_format)) {
 		int hsub = drm_format_horz_chroma_subsampling(fb->pixel_format);
 		int vsub = drm_format_vert_chroma_subsampling(fb->pixel_format);
@@ -791,68 +806,18 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
 }
 
 static const struct drm_plane_helper_funcs plane_helper_funcs = {
-	.prepare_fb = vop_plane_prepare_fb,
-	.cleanup_fb = vop_plane_cleanup_fb,
 	.atomic_check = vop_plane_atomic_check,
 	.atomic_update = vop_plane_atomic_update,
 	.atomic_disable = vop_plane_atomic_disable,
 };
 
-static void vop_atomic_plane_reset(struct drm_plane *plane)
-{
-	struct vop_plane_state *vop_plane_state =
-					to_vop_plane_state(plane->state);
-
-	if (plane->state && plane->state->fb)
-		drm_framebuffer_unreference(plane->state->fb);
-
-	kfree(vop_plane_state);
-	vop_plane_state = kzalloc(sizeof(*vop_plane_state), GFP_KERNEL);
-	if (!vop_plane_state)
-		return;
-
-	plane->state = &vop_plane_state->base;
-	plane->state->plane = plane;
-}
-
-static struct drm_plane_state *
-vop_atomic_plane_duplicate_state(struct drm_plane *plane)
-{
-	struct vop_plane_state *old_vop_plane_state;
-	struct vop_plane_state *vop_plane_state;
-
-	if (WARN_ON(!plane->state))
-		return NULL;
-
-	old_vop_plane_state = to_vop_plane_state(plane->state);
-	vop_plane_state = kmemdup(old_vop_plane_state,
-				  sizeof(*vop_plane_state), GFP_KERNEL);
-	if (!vop_plane_state)
-		return NULL;
-
-	__drm_atomic_helper_plane_duplicate_state(plane,
-						  &vop_plane_state->base);
-
-	return &vop_plane_state->base;
-}
-
-static void vop_atomic_plane_destroy_state(struct drm_plane *plane,
-					   struct drm_plane_state *state)
-{
-	struct vop_plane_state *vop_state = to_vop_plane_state(state);
-
-	__drm_atomic_helper_plane_destroy_state(state);
-
-	kfree(vop_state);
-}
-
 static const struct drm_plane_funcs vop_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy = vop_plane_destroy,
-	.reset = vop_atomic_plane_reset,
-	.atomic_duplicate_state = vop_atomic_plane_duplicate_state,
-	.atomic_destroy_state = vop_atomic_plane_destroy_state,
+	.reset = drm_atomic_helper_plane_reset,
+	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
 
 static int vop_crtc_enable_vblank(struct drm_crtc *crtc)
@@ -865,6 +830,7 @@ static int vop_crtc_enable_vblank(struct drm_crtc *crtc)
 
 	spin_lock_irqsave(&vop->irq_lock, flags);
 
+	VOP_INTR_SET_TYPE(vop, clear, FS_INTR, 1);
 	VOP_INTR_SET_TYPE(vop, enable, FS_INTR, 1);
 
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
@@ -887,18 +853,9 @@ static void vop_crtc_disable_vblank(struct drm_crtc *crtc)
 	spin_unlock_irqrestore(&vop->irq_lock, flags);
 }
 
-static void vop_crtc_wait_for_update(struct drm_crtc *crtc)
-{
-	struct vop *vop = to_vop(crtc);
-
-	reinit_completion(&vop->wait_update_complete);
-	WARN_ON(!wait_for_completion_timeout(&vop->wait_update_complete, 100));
-}
-
 static const struct rockchip_crtc_funcs private_crtc_funcs = {
 	.enable_vblank = vop_crtc_enable_vblank,
 	.disable_vblank = vop_crtc_disable_vblank,
-	.wait_for_update = vop_crtc_wait_for_update,
 };
 
 static bool vop_crtc_mode_fixup(struct drm_crtc *crtc,
@@ -928,11 +885,17 @@ static void vop_crtc_enable(struct drm_crtc *crtc)
 	u16 vsync_len = adjusted_mode->vsync_end - adjusted_mode->vsync_start;
 	u16 vact_st = adjusted_mode->vtotal - adjusted_mode->vsync_start;
 	u16 vact_end = vact_st + vdisplay;
-	uint32_t val;
+	uint32_t pin_pol, val;
+	int ret;
 
 	WARN_ON(vop->event);
 
-	vop_enable(crtc);
+	ret = vop_enable(crtc);
+	if (ret) {
+		DRM_DEV_ERROR(vop->dev, "Failed to enable vop (%d)\n", ret);
+		return;
+	}
+
 	/*
 	 * If dclk rate is zero, mean that scanout is stop,
 	 * we don't need wait any more.
@@ -969,25 +932,31 @@ static void vop_crtc_enable(struct drm_crtc *crtc)
 		vop_dsp_hold_valid_irq_disable(vop);
 	}
 
-	val = 0x8;
-	val |= (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC) ? 0 : 1;
-	val |= (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC) ? 0 : (1 << 1);
-	VOP_CTRL_SET(vop, pin_pol, val);
+	pin_pol = 0x8;
+	pin_pol |= (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC) ? 0 : 1;
+	pin_pol |= (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC) ? 0 : (1 << 1);
+	VOP_CTRL_SET(vop, pin_pol, pin_pol);
+
 	switch (s->output_type) {
 	case DRM_MODE_CONNECTOR_LVDS:
 		VOP_CTRL_SET(vop, rgb_en, 1);
+		VOP_CTRL_SET(vop, rgb_pin_pol, pin_pol);
 		break;
 	case DRM_MODE_CONNECTOR_eDP:
+		VOP_CTRL_SET(vop, edp_pin_pol, pin_pol);
 		VOP_CTRL_SET(vop, edp_en, 1);
 		break;
 	case DRM_MODE_CONNECTOR_HDMIA:
+		VOP_CTRL_SET(vop, hdmi_pin_pol, pin_pol);
 		VOP_CTRL_SET(vop, hdmi_en, 1);
 		break;
 	case DRM_MODE_CONNECTOR_DSI:
+		VOP_CTRL_SET(vop, mipi_pin_pol, pin_pol);
 		VOP_CTRL_SET(vop, mipi_en, 1);
 		break;
 	default:
-		DRM_ERROR("unsupport connector_type[%d]\n", s->output_type);
+		DRM_DEV_ERROR(vop->dev, "unsupported connector_type [%d]\n",
+			      s->output_type);
 	}
 	VOP_CTRL_SET(vop, out_mode, s->output_mode);
 
@@ -1006,12 +975,44 @@ static void vop_crtc_enable(struct drm_crtc *crtc)
 	clk_set_rate(vop->dclk, adjusted_mode->clock * 1000);
 
 	VOP_CTRL_SET(vop, standby, 0);
+
+	rockchip_drm_psr_activate(&vop->crtc);
+}
+
+static bool vop_fs_irq_is_pending(struct vop *vop)
+{
+	return VOP_INTR_GET_TYPE(vop, status, FS_INTR);
+}
+
+static void vop_wait_for_irq_handler(struct vop *vop)
+{
+	bool pending;
+	int ret;
+
+	/*
+	 * Spin until frame start interrupt status bit goes low, which means
+	 * that interrupt handler was invoked and cleared it. The timeout of
+	 * 10 msecs is really too long, but it is just a safety measure if
+	 * something goes really wrong. The wait will only happen in the very
+	 * unlikely case of a vblank happening exactly at the same time and
+	 * shouldn't exceed microseconds range.
+	 */
+	ret = readx_poll_timeout_atomic(vop_fs_irq_is_pending, vop, pending,
+					!pending, 0, 10 * 1000);
+	if (ret)
+		DRM_DEV_ERROR(vop->dev, "VOP vblank IRQ stuck for 10 ms\n");
+
+	synchronize_irq(vop->irq);
 }
 
 static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
 				  struct drm_crtc_state *old_crtc_state)
 {
+	struct drm_atomic_state *old_state = old_crtc_state->state;
+	struct drm_plane_state *old_plane_state;
 	struct vop *vop = to_vop(crtc);
+	struct drm_plane *plane;
+	int i;
 
 	if (WARN_ON(!vop->is_enabled))
 		return;
@@ -1021,12 +1022,13 @@ static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
 	vop_cfg_done(vop);
 
 	spin_unlock(&vop->reg_lock);
-}
 
-static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
-				  struct drm_crtc_state *old_crtc_state)
-{
-	struct vop *vop = to_vop(crtc);
+	/*
+	 * There is a (rather unlikely) possiblity that a vblank interrupt
+	 * fired before we set the cfg_done bit. To avoid spuriously
+	 * signalling flip completion we need to wait for it to finish.
+	 */
+	vop_wait_for_irq_handler(vop);
 
 	spin_lock_irq(&crtc->dev->event_lock);
 	if (crtc->state->event) {
@@ -1037,6 +1039,25 @@ static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
 		crtc->state->event = NULL;
 	}
 	spin_unlock_irq(&crtc->dev->event_lock);
+
+	for_each_plane_in_state(old_state, plane, old_plane_state, i) {
+		if (!old_plane_state->fb)
+			continue;
+
+		if (old_plane_state->fb == plane->state->fb)
+			continue;
+
+		drm_framebuffer_reference(old_plane_state->fb);
+		drm_flip_work_queue(&vop->fb_unref_work, old_plane_state->fb);
+		set_bit(VOP_PENDING_FB_UNREF, &vop->pending);
+		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+	}
+}
+
+static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
+				  struct drm_crtc_state *old_crtc_state)
+{
+	rockchip_drm_psr_flush(crtc);
 }
 
 static const struct drm_crtc_helper_funcs vop_crtc_helper_funcs = {
@@ -1093,16 +1114,13 @@ static const struct drm_crtc_funcs vop_crtc_funcs = {
 	.atomic_destroy_state = vop_crtc_destroy_state,
 };
 
-static bool vop_win_pending_is_complete(struct vop_win *vop_win)
+static void vop_fb_unref_worker(struct drm_flip_work *work, void *val)
 {
-	dma_addr_t yrgb_mst;
-
-	if (!vop_win->enable)
-		return VOP_WIN_GET(vop_win->vop, vop_win->data, enable) == 0;
-
-	yrgb_mst = VOP_WIN_GET_YRGBADDR(vop_win->vop, vop_win->data);
+	struct vop *vop = container_of(work, struct vop, fb_unref_work);
+	struct drm_framebuffer *fb = val;
 
-	return yrgb_mst == vop_win->yrgb_mst;
+	drm_crtc_vblank_put(&vop->crtc);
+	drm_framebuffer_unreference(fb);
 }
 
 static void vop_handle_vblank(struct vop *vop)
@@ -1110,25 +1128,17 @@ static void vop_handle_vblank(struct vop *vop)
 	struct drm_device *drm = vop->drm_dev;
 	struct drm_crtc *crtc = &vop->crtc;
 	unsigned long flags;
-	int i;
-
-	for (i = 0; i < vop->data->win_size; i++) {
-		if (!vop_win_pending_is_complete(&vop->win[i]))
-			return;
-	}
 
 	spin_lock_irqsave(&drm->event_lock, flags);
 	if (vop->event) {
-
 		drm_crtc_send_vblank_event(crtc, vop->event);
 		drm_crtc_vblank_put(crtc);
 		vop->event = NULL;
-
 	}
 	spin_unlock_irqrestore(&drm->event_lock, flags);
 
-	if (!completion_done(&vop->wait_update_complete))
-		complete(&vop->wait_update_complete);
+	if (test_and_clear_bit(VOP_PENDING_FB_UNREF, &vop->pending))
+		drm_flip_work_commit(&vop->fb_unref_work, system_unbound_wq);
 }
 
 static irqreturn_t vop_isr(int irq, void *data)
@@ -1162,6 +1172,12 @@ static irqreturn_t vop_isr(int irq, void *data)
 		ret = IRQ_HANDLED;
 	}
 
+	if (active_irqs & LINE_FLAG_INTR) {
+		complete(&vop->line_flag_completion);
+		active_irqs &= ~LINE_FLAG_INTR;
+		ret = IRQ_HANDLED;
+	}
+
 	if (active_irqs & FS_INTR) {
 		drm_crtc_handle_vblank(crtc);
 		vop_handle_vblank(vop);
@@ -1171,7 +1187,8 @@ static irqreturn_t vop_isr(int irq, void *data)
 
 	/* Unhandled irqs are spurious. */
 	if (active_irqs)
-		DRM_ERROR("Unknown VOP IRQs: %#02x\n", active_irqs);
+		DRM_DEV_ERROR(vop->dev, "Unknown VOP IRQs: %#02x\n",
+			      active_irqs);
 
 	return ret;
 }
@@ -1206,7 +1223,8 @@ static int vop_create_crtc(struct vop *vop)
 					       win_data->phy->nformats,
 					       win_data->type, NULL);
 		if (ret) {
-			DRM_ERROR("failed to initialize plane\n");
+			DRM_DEV_ERROR(vop->dev, "failed to init plane %d\n",
+				      ret);
 			goto err_cleanup_planes;
 		}
 
@@ -1244,7 +1262,8 @@ static int vop_create_crtc(struct vop *vop)
 					       win_data->phy->nformats,
 					       win_data->type, NULL);
 		if (ret) {
-			DRM_ERROR("failed to initialize overlay plane\n");
+			DRM_DEV_ERROR(vop->dev, "failed to init overlay %d\n",
+				      ret);
 			goto err_cleanup_crtc;
 		}
 		drm_plane_helper_add(&vop_win->base, &plane_helper_funcs);
@@ -1252,14 +1271,17 @@ static int vop_create_crtc(struct vop *vop)
 
 	port = of_get_child_by_name(dev->of_node, "port");
 	if (!port) {
-		DRM_ERROR("no port node found in %s\n",
-			  dev->of_node->full_name);
+		DRM_DEV_ERROR(vop->dev, "no port node found in %s\n",
+			      dev->of_node->full_name);
 		ret = -ENOENT;
 		goto err_cleanup_crtc;
 	}
 
+	drm_flip_work_init(&vop->fb_unref_work, "fb_unref",
+			   vop_fb_unref_worker);
+
 	init_completion(&vop->dsp_hold_completion);
-	init_completion(&vop->wait_update_complete);
+	init_completion(&vop->line_flag_completion);
 	crtc->port = port;
 	rockchip_register_crtc_funcs(crtc, &private_crtc_funcs);
 
@@ -1300,6 +1322,7 @@ static void vop_destroy_crtc(struct vop *vop)
 	 * references the CRTC.
 	 */
 	drm_crtc_cleanup(crtc);
+	drm_flip_work_cleanup(&vop->fb_unref_work);
 }
 
 static int vop_initial(struct vop *vop)
@@ -1416,6 +1439,49 @@ static void vop_win_init(struct vop *vop)
 	}
 }
 
+/**
+ * rockchip_drm_wait_line_flag - acqiure the give line flag event
+ * @crtc: CRTC to enable line flag
+ * @line_num: interested line number
+ * @mstimeout: millisecond for timeout
+ *
+ * Driver would hold here until the interested line flag interrupt have
+ * happened or timeout to wait.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int rockchip_drm_wait_line_flag(struct drm_crtc *crtc, unsigned int line_num,
+				unsigned int mstimeout)
+{
+	struct vop *vop = to_vop(crtc);
+	unsigned long jiffies_left;
+
+	if (!crtc || !vop->is_enabled)
+		return -ENODEV;
+
+	if (line_num > crtc->mode.vtotal || mstimeout <= 0)
+		return -EINVAL;
+
+	if (vop_line_flag_irq_is_enabled(vop))
+		return -EBUSY;
+
+	reinit_completion(&vop->line_flag_completion);
+	vop_line_flag_irq_enable(vop, line_num);
+
+	jiffies_left = wait_for_completion_timeout(&vop->line_flag_completion,
+						   msecs_to_jiffies(mstimeout));
+	vop_line_flag_irq_disable(vop);
+
+	if (jiffies_left == 0) {
+		dev_err(vop->dev, "Timeout waiting for IRQ\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(rockchip_drm_wait_line_flag);
+
 static int vop_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
@@ -1481,10 +1547,15 @@ static int vop_bind(struct device *dev, struct device *master, void *data)
 
 	ret = vop_create_crtc(vop);
 	if (ret)
-		return ret;
+		goto err_enable_irq;
 
 	pm_runtime_enable(&pdev->dev);
+
 	return 0;
+
+err_enable_irq:
+	enable_irq(vop->irq); /* To balance out the disable_irq above */
+	return ret;
 }
 
 static void vop_unbind(struct device *dev, struct device *master, void *data)
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
index 071ff0be7a95..1dbc52615257 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
@@ -33,6 +33,7 @@ struct vop_reg {
 	uint32_t offset;
 	uint32_t shift;
 	uint32_t mask;
+	bool write_mask;
 };
 
 struct vop_ctrl {
@@ -48,6 +49,10 @@ struct vop_ctrl {
 	struct vop_reg dither_down;
 	struct vop_reg dither_up;
 	struct vop_reg pin_pol;
+	struct vop_reg rgb_pin_pol;
+	struct vop_reg hdmi_pin_pol;
+	struct vop_reg edp_pin_pol;
+	struct vop_reg mipi_pin_pol;
 
 	struct vop_reg htotal_pw;
 	struct vop_reg hact_st_end;
@@ -56,6 +61,8 @@ struct vop_ctrl {
 	struct vop_reg hpost_st_end;
 	struct vop_reg vpost_st_end;
 
+	struct vop_reg line_flag_num[2];
+
 	struct vop_reg cfg_done;
 };
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
index 919992cdc97e..35c51f3402f2 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
@@ -23,7 +23,14 @@
 #define VOP_REG(off, _mask, s) \
 		{.offset = off, \
 		 .mask = _mask, \
-		 .shift = s,}
+		 .shift = s, \
+		 .write_mask = false,}
+
+#define VOP_REG_MASK(off, _mask, s) \
+		{.offset = off, \
+		 .mask = _mask, \
+		 .shift = s, \
+		 .write_mask = true,}
 
 static const uint32_t formats_win_full[] = {
 	DRM_FORMAT_XRGB8888,
@@ -50,6 +57,89 @@ static const uint32_t formats_win_lite[] = {
 	DRM_FORMAT_BGR565,
 };
 
+static const struct vop_scl_regs rk3036_win_scl = {
+	.scale_yrgb_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 0x0),
+	.scale_yrgb_y = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 16),
+	.scale_cbcr_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_CBR, 0xffff, 0x0),
+	.scale_cbcr_y = VOP_REG(RK3036_WIN0_SCL_FACTOR_CBR, 0xffff, 16),
+};
+
+static const struct vop_win_phy rk3036_win0_data = {
+	.scl = &rk3036_win_scl,
+	.data_formats = formats_win_full,
+	.nformats = ARRAY_SIZE(formats_win_full),
+	.enable = VOP_REG(RK3036_SYS_CTRL, 0x1, 0),
+	.format = VOP_REG(RK3036_SYS_CTRL, 0x7, 3),
+	.rb_swap = VOP_REG(RK3036_SYS_CTRL, 0x1, 15),
+	.act_info = VOP_REG(RK3036_WIN0_ACT_INFO, 0x1fff1fff, 0),
+	.dsp_info = VOP_REG(RK3036_WIN0_DSP_INFO, 0x0fff0fff, 0),
+	.dsp_st = VOP_REG(RK3036_WIN0_DSP_ST, 0x1fff1fff, 0),
+	.yrgb_mst = VOP_REG(RK3036_WIN0_YRGB_MST, 0xffffffff, 0),
+	.uv_mst = VOP_REG(RK3036_WIN0_CBR_MST, 0xffffffff, 0),
+	.yrgb_vir = VOP_REG(RK3036_WIN0_VIR, 0xffff, 0),
+	.uv_vir = VOP_REG(RK3036_WIN0_VIR, 0x1fff, 16),
+};
+
+static const struct vop_win_phy rk3036_win1_data = {
+	.data_formats = formats_win_lite,
+	.nformats = ARRAY_SIZE(formats_win_lite),
+	.enable = VOP_REG(RK3036_SYS_CTRL, 0x1, 1),
+	.format = VOP_REG(RK3036_SYS_CTRL, 0x7, 6),
+	.rb_swap = VOP_REG(RK3036_SYS_CTRL, 0x1, 19),
+	.act_info = VOP_REG(RK3036_WIN1_ACT_INFO, 0x1fff1fff, 0),
+	.dsp_info = VOP_REG(RK3036_WIN1_DSP_INFO, 0x0fff0fff, 0),
+	.dsp_st = VOP_REG(RK3036_WIN1_DSP_ST, 0x1fff1fff, 0),
+	.yrgb_mst = VOP_REG(RK3036_WIN1_MST, 0xffffffff, 0),
+	.yrgb_vir = VOP_REG(RK3036_WIN1_VIR, 0xffff, 0),
+};
+
+static const struct vop_win_data rk3036_vop_win_data[] = {
+	{ .base = 0x00, .phy = &rk3036_win0_data,
+	  .type = DRM_PLANE_TYPE_PRIMARY },
+	{ .base = 0x00, .phy = &rk3036_win1_data,
+	  .type = DRM_PLANE_TYPE_CURSOR },
+};
+
+static const int rk3036_vop_intrs[] = {
+	DSP_HOLD_VALID_INTR,
+	FS_INTR,
+	LINE_FLAG_INTR,
+	BUS_ERROR_INTR,
+};
+
+static const struct vop_intr rk3036_intr = {
+	.intrs = rk3036_vop_intrs,
+	.nintrs = ARRAY_SIZE(rk3036_vop_intrs),
+	.status = VOP_REG(RK3036_INT_STATUS, 0xf, 0),
+	.enable = VOP_REG(RK3036_INT_STATUS, 0xf, 4),
+	.clear = VOP_REG(RK3036_INT_STATUS, 0xf, 8),
+};
+
+static const struct vop_ctrl rk3036_ctrl_data = {
+	.standby = VOP_REG(RK3036_SYS_CTRL, 0x1, 30),
+	.out_mode = VOP_REG(RK3036_DSP_CTRL0, 0xf, 0),
+	.pin_pol = VOP_REG(RK3036_DSP_CTRL0, 0xf, 4),
+	.htotal_pw = VOP_REG(RK3036_DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
+	.hact_st_end = VOP_REG(RK3036_DSP_HACT_ST_END, 0x1fff1fff, 0),
+	.vtotal_pw = VOP_REG(RK3036_DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
+	.vact_st_end = VOP_REG(RK3036_DSP_VACT_ST_END, 0x1fff1fff, 0),
+	.line_flag_num[0] = VOP_REG(RK3036_INT_STATUS, 0xfff, 12),
+	.cfg_done = VOP_REG(RK3036_REG_CFG_DONE, 0x1, 0),
+};
+
+static const struct vop_reg_data rk3036_vop_init_reg_table[] = {
+	{RK3036_DSP_CTRL1, 0x00000000},
+};
+
+static const struct vop_data rk3036_vop = {
+	.init_table = rk3036_vop_init_reg_table,
+	.table_size = ARRAY_SIZE(rk3036_vop_init_reg_table),
+	.ctrl = &rk3036_ctrl_data,
+	.intr = &rk3036_intr,
+	.win = rk3036_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3036_vop_win_data),
+};
+
 static const struct vop_scl_extension rk3288_win_full_scl_ext = {
 	.cbcr_vsd_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 31),
 	.cbcr_vsu_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 30),
@@ -133,6 +223,7 @@ static const struct vop_ctrl rk3288_ctrl_data = {
 	.vact_st_end = VOP_REG(RK3288_DSP_VACT_ST_END, 0x1fff1fff, 0),
 	.hpost_st_end = VOP_REG(RK3288_POST_DSP_HACT_INFO, 0x1fff1fff, 0),
 	.vpost_st_end = VOP_REG(RK3288_POST_DSP_VACT_INFO, 0x1fff1fff, 0),
+	.line_flag_num[0] = VOP_REG(RK3288_INTR_CTRL0, 0x1fff, 12),
 	.cfg_done = VOP_REG(RK3288_REG_CFG_DONE, 0x1, 0),
 };
 
@@ -190,93 +281,104 @@ static const struct vop_data rk3288_vop = {
 	.win_size = ARRAY_SIZE(rk3288_vop_win_data),
 };
 
-static const struct vop_scl_regs rk3036_win_scl = {
-	.scale_yrgb_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 0x0),
-	.scale_yrgb_y = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 16),
-	.scale_cbcr_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_CBR, 0xffff, 0x0),
-	.scale_cbcr_y = VOP_REG(RK3036_WIN0_SCL_FACTOR_CBR, 0xffff, 16),
-};
-
-static const struct vop_win_phy rk3036_win0_data = {
-	.scl = &rk3036_win_scl,
-	.data_formats = formats_win_full,
-	.nformats = ARRAY_SIZE(formats_win_full),
-	.enable = VOP_REG(RK3036_SYS_CTRL, 0x1, 0),
-	.format = VOP_REG(RK3036_SYS_CTRL, 0x7, 3),
-	.rb_swap = VOP_REG(RK3036_SYS_CTRL, 0x1, 15),
-	.act_info = VOP_REG(RK3036_WIN0_ACT_INFO, 0x1fff1fff, 0),
-	.dsp_info = VOP_REG(RK3036_WIN0_DSP_INFO, 0x0fff0fff, 0),
-	.dsp_st = VOP_REG(RK3036_WIN0_DSP_ST, 0x1fff1fff, 0),
-	.yrgb_mst = VOP_REG(RK3036_WIN0_YRGB_MST, 0xffffffff, 0),
-	.uv_mst = VOP_REG(RK3036_WIN0_CBR_MST, 0xffffffff, 0),
-	.yrgb_vir = VOP_REG(RK3036_WIN0_VIR, 0xffff, 0),
-	.uv_vir = VOP_REG(RK3036_WIN0_VIR, 0x1fff, 16),
+static const struct vop_ctrl rk3399_ctrl_data = {
+	.standby = VOP_REG(RK3399_SYS_CTRL, 0x1, 22),
+	.gate_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 23),
+	.rgb_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 12),
+	.hdmi_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 13),
+	.edp_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 14),
+	.mipi_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 15),
+	.dither_down = VOP_REG(RK3399_DSP_CTRL1, 0xf, 1),
+	.dither_up = VOP_REG(RK3399_DSP_CTRL1, 0x1, 6),
+	.data_blank = VOP_REG(RK3399_DSP_CTRL0, 0x1, 19),
+	.out_mode = VOP_REG(RK3399_DSP_CTRL0, 0xf, 0),
+	.rgb_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 16),
+	.hdmi_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 20),
+	.edp_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 24),
+	.mipi_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 28),
+	.htotal_pw = VOP_REG(RK3399_DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
+	.hact_st_end = VOP_REG(RK3399_DSP_HACT_ST_END, 0x1fff1fff, 0),
+	.vtotal_pw = VOP_REG(RK3399_DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
+	.vact_st_end = VOP_REG(RK3399_DSP_VACT_ST_END, 0x1fff1fff, 0),
+	.hpost_st_end = VOP_REG(RK3399_POST_DSP_HACT_INFO, 0x1fff1fff, 0),
+	.vpost_st_end = VOP_REG(RK3399_POST_DSP_VACT_INFO, 0x1fff1fff, 0),
+	.line_flag_num[0] = VOP_REG(RK3399_LINE_FLAG, 0xffff, 0),
+	.line_flag_num[1] = VOP_REG(RK3399_LINE_FLAG, 0xffff, 16),
+	.cfg_done = VOP_REG_MASK(RK3399_REG_CFG_DONE, 0x1, 0),
 };
 
-static const struct vop_win_phy rk3036_win1_data = {
-	.data_formats = formats_win_lite,
-	.nformats = ARRAY_SIZE(formats_win_lite),
-	.enable = VOP_REG(RK3036_SYS_CTRL, 0x1, 1),
-	.format = VOP_REG(RK3036_SYS_CTRL, 0x7, 6),
-	.rb_swap = VOP_REG(RK3036_SYS_CTRL, 0x1, 19),
-	.act_info = VOP_REG(RK3036_WIN1_ACT_INFO, 0x1fff1fff, 0),
-	.dsp_info = VOP_REG(RK3036_WIN1_DSP_INFO, 0x0fff0fff, 0),
-	.dsp_st = VOP_REG(RK3036_WIN1_DSP_ST, 0x1fff1fff, 0),
-	.yrgb_mst = VOP_REG(RK3036_WIN1_MST, 0xffffffff, 0),
-	.yrgb_vir = VOP_REG(RK3036_WIN1_VIR, 0xffff, 0),
-};
-
-static const struct vop_win_data rk3036_vop_win_data[] = {
-	{ .base = 0x00, .phy = &rk3036_win0_data,
-	  .type = DRM_PLANE_TYPE_PRIMARY },
-	{ .base = 0x00, .phy = &rk3036_win1_data,
-	  .type = DRM_PLANE_TYPE_CURSOR },
-};
-
-static const int rk3036_vop_intrs[] = {
-	DSP_HOLD_VALID_INTR,
+static const int rk3399_vop_intrs[] = {
 	FS_INTR,
+	0, 0,
 	LINE_FLAG_INTR,
+	0,
 	BUS_ERROR_INTR,
+	0, 0, 0, 0, 0, 0, 0,
+	DSP_HOLD_VALID_INTR,
 };
 
-static const struct vop_intr rk3036_intr = {
-	.intrs = rk3036_vop_intrs,
-	.nintrs = ARRAY_SIZE(rk3036_vop_intrs),
-	.status = VOP_REG(RK3036_INT_STATUS, 0xf, 0),
-	.enable = VOP_REG(RK3036_INT_STATUS, 0xf, 4),
-	.clear = VOP_REG(RK3036_INT_STATUS, 0xf, 8),
+static const struct vop_intr rk3399_vop_intr = {
+	.intrs = rk3399_vop_intrs,
+	.nintrs = ARRAY_SIZE(rk3399_vop_intrs),
+	.status = VOP_REG_MASK(RK3399_INTR_STATUS0, 0xffff, 0),
+	.enable = VOP_REG_MASK(RK3399_INTR_EN0, 0xffff, 0),
+	.clear = VOP_REG_MASK(RK3399_INTR_CLEAR0, 0xffff, 0),
 };
 
-static const struct vop_ctrl rk3036_ctrl_data = {
-	.standby = VOP_REG(RK3036_SYS_CTRL, 0x1, 30),
-	.out_mode = VOP_REG(RK3036_DSP_CTRL0, 0xf, 0),
-	.pin_pol = VOP_REG(RK3036_DSP_CTRL0, 0xf, 4),
-	.htotal_pw = VOP_REG(RK3036_DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
-	.hact_st_end = VOP_REG(RK3036_DSP_HACT_ST_END, 0x1fff1fff, 0),
-	.vtotal_pw = VOP_REG(RK3036_DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
-	.vact_st_end = VOP_REG(RK3036_DSP_VACT_ST_END, 0x1fff1fff, 0),
-	.cfg_done = VOP_REG(RK3036_REG_CFG_DONE, 0x1, 0),
+static const struct vop_reg_data rk3399_init_reg_table[] = {
+	{RK3399_SYS_CTRL, 0x2000f800},
+	{RK3399_DSP_CTRL0, 0x00000000},
+	{RK3399_WIN0_CTRL0, 0x00000080},
+	{RK3399_WIN1_CTRL0, 0x00000080},
+	/* TODO: Win2/3 support multiple area function, but we haven't found
+	 * a suitable way to use it yet, so let's just use them as other windows
+	 * with only area 0 enabled.
+	 */
+	{RK3399_WIN2_CTRL0, 0x00000010},
+	{RK3399_WIN3_CTRL0, 0x00000010},
 };
 
-static const struct vop_reg_data rk3036_vop_init_reg_table[] = {
-	{RK3036_DSP_CTRL1, 0x00000000},
+static const struct vop_data rk3399_vop_big = {
+	.init_table = rk3399_init_reg_table,
+	.table_size = ARRAY_SIZE(rk3399_init_reg_table),
+	.intr = &rk3399_vop_intr,
+	.ctrl = &rk3399_ctrl_data,
+	/*
+	 * rk3399 vop big windows register layout is same as rk3288.
+	 */
+	.win = rk3288_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3288_vop_win_data),
 };
 
-static const struct vop_data rk3036_vop = {
-	.init_table = rk3036_vop_init_reg_table,
-	.table_size = ARRAY_SIZE(rk3036_vop_init_reg_table),
-	.ctrl = &rk3036_ctrl_data,
-	.intr = &rk3036_intr,
-	.win = rk3036_vop_win_data,
-	.win_size = ARRAY_SIZE(rk3036_vop_win_data),
+static const struct vop_win_data rk3399_vop_lit_win_data[] = {
+	{ .base = 0x00, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_PRIMARY },
+	{ .base = 0x00, .phy = &rk3288_win23_data,
+	  .type = DRM_PLANE_TYPE_CURSOR},
+};
+
+static const struct vop_data rk3399_vop_lit = {
+	.init_table = rk3399_init_reg_table,
+	.table_size = ARRAY_SIZE(rk3399_init_reg_table),
+	.intr = &rk3399_vop_intr,
+	.ctrl = &rk3399_ctrl_data,
+	/*
+	 * rk3399 vop lit windows register layout is same as rk3288,
+	 * but cut off the win1 and win3 windows.
+	 */
+	.win = rk3399_vop_lit_win_data,
+	.win_size = ARRAY_SIZE(rk3399_vop_lit_win_data),
 };
 
 static const struct of_device_id vop_driver_dt_match[] = {
-	{ .compatible = "rockchip,rk3288-vop",
-	  .data = &rk3288_vop },
 	{ .compatible = "rockchip,rk3036-vop",
 	  .data = &rk3036_vop },
+	{ .compatible = "rockchip,rk3288-vop",
+	  .data = &rk3288_vop },
+	{ .compatible = "rockchip,rk3399-vop-big",
+	  .data = &rk3399_vop_big },
+	{ .compatible = "rockchip,rk3399-vop-lit",
+	  .data = &rk3399_vop_lit },
 	{},
 };
 MODULE_DEVICE_TABLE(of, vop_driver_dt_match);
@@ -305,7 +407,6 @@ static struct platform_driver vop_platform_driver = {
 	.remove = vop_remove,
 	.driver = {
 		.name = "rockchip-vop",
-		.owner = THIS_MODULE,
 		.of_match_table = of_match_ptr(vop_driver_dt_match),
 	},
 };
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
index d4b46cba2f26..cd197260ece5 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
@@ -166,4 +166,197 @@
 #define RK3036_HWC_LUT_ADDR		0x800
 /* rk3036 register definition end */
 
+/* rk3399 register definition */
+#define RK3399_REG_CFG_DONE		0x00000
+#define RK3399_VERSION_INFO		0x00004
+#define RK3399_SYS_CTRL			0x00008
+#define RK3399_SYS_CTRL1		0x0000c
+#define RK3399_DSP_CTRL0		0x00010
+#define RK3399_DSP_CTRL1		0x00014
+#define RK3399_DSP_BG			0x00018
+#define RK3399_MCU_CTRL			0x0001c
+#define RK3399_WB_CTRL0			0x00020
+#define RK3399_WB_CTRL1			0x00024
+#define RK3399_WB_YRGB_MST		0x00028
+#define RK3399_WB_CBR_MST		0x0002c
+#define RK3399_WIN0_CTRL0		0x00030
+#define RK3399_WIN0_CTRL1		0x00034
+#define RK3399_WIN0_COLOR_KEY		0x00038
+#define RK3399_WIN0_VIR			0x0003c
+#define RK3399_WIN0_YRGB_MST		0x00040
+#define RK3399_WIN0_CBR_MST		0x00044
+#define RK3399_WIN0_ACT_INFO		0x00048
+#define RK3399_WIN0_DSP_INFO		0x0004c
+#define RK3399_WIN0_DSP_ST		0x00050
+#define RK3399_WIN0_SCL_FACTOR_YRGB	0x00054
+#define RK3399_WIN0_SCL_FACTOR_CBR	0x00058
+#define RK3399_WIN0_SCL_OFFSET		0x0005c
+#define RK3399_WIN0_SRC_ALPHA_CTRL	0x00060
+#define RK3399_WIN0_DST_ALPHA_CTRL	0x00064
+#define RK3399_WIN0_FADING_CTRL		0x00068
+#define RK3399_WIN0_CTRL2		0x0006c
+#define RK3399_WIN1_CTRL0		0x00070
+#define RK3399_WIN1_CTRL1		0x00074
+#define RK3399_WIN1_COLOR_KEY		0x00078
+#define RK3399_WIN1_VIR			0x0007c
+#define RK3399_WIN1_YRGB_MST		0x00080
+#define RK3399_WIN1_CBR_MST		0x00084
+#define RK3399_WIN1_ACT_INFO		0x00088
+#define RK3399_WIN1_DSP_INFO		0x0008c
+#define RK3399_WIN1_DSP_ST		0x00090
+#define RK3399_WIN1_SCL_FACTOR_YRGB	0x00094
+#define RK3399_WIN1_SCL_FACTOR_CBR	0x00098
+#define RK3399_WIN1_SCL_OFFSET		0x0009c
+#define RK3399_WIN1_SRC_ALPHA_CTRL	0x000a0
+#define RK3399_WIN1_DST_ALPHA_CTRL	0x000a4
+#define RK3399_WIN1_FADING_CTRL		0x000a8
+#define RK3399_WIN1_CTRL2		0x000ac
+#define RK3399_WIN2_CTRL0		0x000b0
+#define RK3399_WIN2_CTRL1		0x000b4
+#define RK3399_WIN2_VIR0_1		0x000b8
+#define RK3399_WIN2_VIR2_3		0x000bc
+#define RK3399_WIN2_MST0		0x000c0
+#define RK3399_WIN2_DSP_INFO0		0x000c4
+#define RK3399_WIN2_DSP_ST0		0x000c8
+#define RK3399_WIN2_COLOR_KEY		0x000cc
+#define RK3399_WIN2_MST1		0x000d0
+#define RK3399_WIN2_DSP_INFO1		0x000d4
+#define RK3399_WIN2_DSP_ST1		0x000d8
+#define RK3399_WIN2_SRC_ALPHA_CTRL	0x000dc
+#define RK3399_WIN2_MST2		0x000e0
+#define RK3399_WIN2_DSP_INFO2		0x000e4
+#define RK3399_WIN2_DSP_ST2		0x000e8
+#define RK3399_WIN2_DST_ALPHA_CTRL	0x000ec
+#define RK3399_WIN2_MST3		0x000f0
+#define RK3399_WIN2_DSP_INFO3		0x000f4
+#define RK3399_WIN2_DSP_ST3		0x000f8
+#define RK3399_WIN2_FADING_CTRL		0x000fc
+#define RK3399_WIN3_CTRL0		0x00100
+#define RK3399_WIN3_CTRL1		0x00104
+#define RK3399_WIN3_VIR0_1		0x00108
+#define RK3399_WIN3_VIR2_3		0x0010c
+#define RK3399_WIN3_MST0		0x00110
+#define RK3399_WIN3_DSP_INFO0		0x00114
+#define RK3399_WIN3_DSP_ST0		0x00118
+#define RK3399_WIN3_COLOR_KEY		0x0011c
+#define RK3399_WIN3_MST1		0x00120
+#define RK3399_WIN3_DSP_INFO1		0x00124
+#define RK3399_WIN3_DSP_ST1		0x00128
+#define RK3399_WIN3_SRC_ALPHA_CTRL	0x0012c
+#define RK3399_WIN3_MST2		0x00130
+#define RK3399_WIN3_DSP_INFO2		0x00134
+#define RK3399_WIN3_DSP_ST2		0x00138
+#define RK3399_WIN3_DST_ALPHA_CTRL	0x0013c
+#define RK3399_WIN3_MST3		0x00140
+#define RK3399_WIN3_DSP_INFO3		0x00144
+#define RK3399_WIN3_DSP_ST3		0x00148
+#define RK3399_WIN3_FADING_CTRL		0x0014c
+#define RK3399_HWC_CTRL0		0x00150
+#define RK3399_HWC_CTRL1		0x00154
+#define RK3399_HWC_MST			0x00158
+#define RK3399_HWC_DSP_ST		0x0015c
+#define RK3399_HWC_SRC_ALPHA_CTRL	0x00160
+#define RK3399_HWC_DST_ALPHA_CTRL	0x00164
+#define RK3399_HWC_FADING_CTRL		0x00168
+#define RK3399_HWC_RESERVED1		0x0016c
+#define RK3399_POST_DSP_HACT_INFO	0x00170
+#define RK3399_POST_DSP_VACT_INFO	0x00174
+#define RK3399_POST_SCL_FACTOR_YRGB	0x00178
+#define RK3399_POST_RESERVED		0x0017c
+#define RK3399_POST_SCL_CTRL		0x00180
+#define RK3399_POST_DSP_VACT_INFO_F1	0x00184
+#define RK3399_DSP_HTOTAL_HS_END	0x00188
+#define RK3399_DSP_HACT_ST_END		0x0018c
+#define RK3399_DSP_VTOTAL_VS_END	0x00190
+#define RK3399_DSP_VACT_ST_END		0x00194
+#define RK3399_DSP_VS_ST_END_F1		0x00198
+#define RK3399_DSP_VACT_ST_END_F1	0x0019c
+#define RK3399_PWM_CTRL			0x001a0
+#define RK3399_PWM_PERIOD_HPR		0x001a4
+#define RK3399_PWM_DUTY_LPR		0x001a8
+#define RK3399_PWM_CNT			0x001ac
+#define RK3399_BCSH_COLOR_BAR		0x001b0
+#define RK3399_BCSH_BCS			0x001b4
+#define RK3399_BCSH_H			0x001b8
+#define RK3399_BCSH_CTRL		0x001bc
+#define RK3399_CABC_CTRL0		0x001c0
+#define RK3399_CABC_CTRL1		0x001c4
+#define RK3399_CABC_CTRL2		0x001c8
+#define RK3399_CABC_CTRL3		0x001cc
+#define RK3399_CABC_GAUSS_LINE0_0	0x001d0
+#define RK3399_CABC_GAUSS_LINE0_1	0x001d4
+#define RK3399_CABC_GAUSS_LINE1_0	0x001d8
+#define RK3399_CABC_GAUSS_LINE1_1	0x001dc
+#define RK3399_CABC_GAUSS_LINE2_0	0x001e0
+#define RK3399_CABC_GAUSS_LINE2_1	0x001e4
+#define RK3399_FRC_LOWER01_0		0x001e8
+#define RK3399_FRC_LOWER01_1		0x001ec
+#define RK3399_FRC_LOWER10_0		0x001f0
+#define RK3399_FRC_LOWER10_1		0x001f4
+#define RK3399_FRC_LOWER11_0		0x001f8
+#define RK3399_FRC_LOWER11_1		0x001fc
+#define RK3399_AFBCD0_CTRL		0x00200
+#define RK3399_AFBCD0_HDR_PTR		0x00204
+#define RK3399_AFBCD0_PIC_SIZE		0x00208
+#define RK3399_AFBCD0_STATUS		0x0020c
+#define RK3399_AFBCD1_CTRL		0x00220
+#define RK3399_AFBCD1_HDR_PTR		0x00224
+#define RK3399_AFBCD1_PIC_SIZE		0x00228
+#define RK3399_AFBCD1_STATUS		0x0022c
+#define RK3399_AFBCD2_CTRL		0x00240
+#define RK3399_AFBCD2_HDR_PTR		0x00244
+#define RK3399_AFBCD2_PIC_SIZE		0x00248
+#define RK3399_AFBCD2_STATUS		0x0024c
+#define RK3399_AFBCD3_CTRL		0x00260
+#define RK3399_AFBCD3_HDR_PTR		0x00264
+#define RK3399_AFBCD3_PIC_SIZE		0x00268
+#define RK3399_AFBCD3_STATUS		0x0026c
+#define RK3399_INTR_EN0			0x00280
+#define RK3399_INTR_CLEAR0		0x00284
+#define RK3399_INTR_STATUS0		0x00288
+#define RK3399_INTR_RAW_STATUS0		0x0028c
+#define RK3399_INTR_EN1			0x00290
+#define RK3399_INTR_CLEAR1		0x00294
+#define RK3399_INTR_STATUS1		0x00298
+#define RK3399_INTR_RAW_STATUS1		0x0029c
+#define RK3399_LINE_FLAG		0x002a0
+#define RK3399_VOP_STATUS		0x002a4
+#define RK3399_BLANKING_VALUE		0x002a8
+#define RK3399_MCU_BYPASS_PORT		0x002ac
+#define RK3399_WIN0_DSP_BG		0x002b0
+#define RK3399_WIN1_DSP_BG		0x002b4
+#define RK3399_WIN2_DSP_BG		0x002b8
+#define RK3399_WIN3_DSP_BG		0x002bc
+#define RK3399_YUV2YUV_WIN		0x002c0
+#define RK3399_YUV2YUV_POST		0x002c4
+#define RK3399_AUTO_GATING_EN		0x002cc
+#define RK3399_WIN0_CSC_COE		0x003a0
+#define RK3399_WIN1_CSC_COE		0x003c0
+#define RK3399_WIN2_CSC_COE		0x003e0
+#define RK3399_WIN3_CSC_COE		0x00400
+#define RK3399_HWC_CSC_COE		0x00420
+#define RK3399_BCSH_R2Y_CSC_COE		0x00440
+#define RK3399_BCSH_Y2R_CSC_COE		0x00460
+#define RK3399_POST_YUV2YUV_Y2R_COE	0x00480
+#define RK3399_POST_YUV2YUV_3X3_COE	0x004a0
+#define RK3399_POST_YUV2YUV_R2Y_COE	0x004c0
+#define RK3399_WIN0_YUV2YUV_Y2R		0x004e0
+#define RK3399_WIN0_YUV2YUV_3X3		0x00500
+#define RK3399_WIN0_YUV2YUV_R2Y		0x00520
+#define RK3399_WIN1_YUV2YUV_Y2R		0x00540
+#define RK3399_WIN1_YUV2YUV_3X3		0x00560
+#define RK3399_WIN1_YUV2YUV_R2Y		0x00580
+#define RK3399_WIN2_YUV2YUV_Y2R		0x005a0
+#define RK3399_WIN2_YUV2YUV_3X3		0x005c0
+#define RK3399_WIN2_YUV2YUV_R2Y		0x005e0
+#define RK3399_WIN3_YUV2YUV_Y2R		0x00600
+#define RK3399_WIN3_YUV2YUV_3X3		0x00620
+#define RK3399_WIN3_YUV2YUV_R2Y		0x00640
+#define RK3399_WIN2_LUT_ADDR		0x01000
+#define RK3399_WIN3_LUT_ADDR		0x01400
+#define RK3399_HWC_LUT_ADDR		0x01800
+#define RK3399_CABC_GAMMA_LUT_ADDR	0x01c00
+#define RK3399_GAMMA_LUT_ADDR		0x02000
+/* rk3399 register definition end */
+
 #endif /* _ROCKCHIP_VOP_REG_H */
diff --git a/drivers/gpu/drm/savage/savage_drv.c b/drivers/gpu/drm/savage/savage_drv.c
index 21aed1febeb4..3b807135a5cd 100644
--- a/drivers/gpu/drm/savage/savage_drv.c
+++ b/drivers/gpu/drm/savage/savage_drv.c
@@ -50,7 +50,7 @@ static const struct file_operations savage_driver_fops = {
 
 static struct drm_driver driver = {
 	.driver_features =
-	    DRIVER_USE_AGP | DRIVER_HAVE_DMA | DRIVER_PCI_DMA,
+	    DRIVER_USE_AGP | DRIVER_HAVE_DMA | DRIVER_PCI_DMA | DRIVER_LEGACY,
 	.dev_priv_size = sizeof(drm_savage_buf_priv_t),
 	.load = savage_driver_load,
 	.firstopen = savage_driver_firstopen,
diff --git a/drivers/gpu/drm/savage/savage_state.c b/drivers/gpu/drm/savage/savage_state.c
index c01ad0aeaa58..3dc0d8ff95ec 100644
--- a/drivers/gpu/drm/savage/savage_state.c
+++ b/drivers/gpu/drm/savage/savage_state.c
@@ -1001,15 +1001,9 @@ int savage_bci_cmdbuf(struct drm_device *dev, void *data, struct drm_file *file_
 		cmdbuf->cmd_addr = kcmd_addr;
 	}
 	if (cmdbuf->vb_size) {
-		kvb_addr = kmalloc(cmdbuf->vb_size, GFP_KERNEL);
-		if (kvb_addr == NULL) {
-			ret = -ENOMEM;
-			goto done;
-		}
-
-		if (copy_from_user(kvb_addr, cmdbuf->vb_addr,
-				       cmdbuf->vb_size)) {
-			ret = -EFAULT;
+		kvb_addr = memdup_user(cmdbuf->vb_addr, cmdbuf->vb_size);
+		if (IS_ERR(kvb_addr)) {
+			ret = PTR_ERR(kvb_addr);
 			goto done;
 		}
 		cmdbuf->vb_addr = kvb_addr;
diff --git a/drivers/gpu/drm/sis/sis_drv.c b/drivers/gpu/drm/sis/sis_drv.c
index 79bce76cb8f7..ae9839886c4d 100644
--- a/drivers/gpu/drm/sis/sis_drv.c
+++ b/drivers/gpu/drm/sis/sis_drv.c
@@ -102,7 +102,7 @@ static void sis_driver_postclose(struct drm_device *dev, struct drm_file *file)
 }
 
 static struct drm_driver driver = {
-	.driver_features = DRIVER_USE_AGP,
+	.driver_features = DRIVER_USE_AGP | DRIVER_LEGACY,
 	.load = sis_driver_load,
 	.unload = sis_driver_unload,
 	.open = sis_driver_open,
diff --git a/drivers/gpu/drm/sti/Kconfig b/drivers/gpu/drm/sti/Kconfig
index 494ab257f77c..acd72865feac 100644
--- a/drivers/gpu/drm/sti/Kconfig
+++ b/drivers/gpu/drm/sti/Kconfig
@@ -1,6 +1,6 @@
 config DRM_STI
-	tristate "DRM Support for STMicroelectronics SoC stiH41x Series"
-	depends on DRM && (SOC_STIH415 || SOC_STIH416 || ARCH_MULTIPLATFORM)
+	tristate "DRM Support for STMicroelectronics SoC stiH4xx Series"
+	depends on DRM && (ARCH_STI || ARCH_MULTIPLATFORM)
 	select RESET_CONTROLLER
 	select DRM_KMS_HELPER
 	select DRM_GEM_CMA_HELPER
@@ -9,4 +9,4 @@ config DRM_STI
 	select FW_LOADER
 	select SND_SOC_HDMI_CODEC if SND_SOC
 	help
-	  Choose this option to enable DRM on STM stiH41x chipset
+	  Choose this option to enable DRM on STM stiH4xx chipset
diff --git a/drivers/gpu/drm/sti/Makefile b/drivers/gpu/drm/sti/Makefile
index b8057620b3b3..d20f7c0b4eac 100644
--- a/drivers/gpu/drm/sti/Makefile
+++ b/drivers/gpu/drm/sti/Makefile
@@ -9,7 +9,6 @@ sti-drm-y := \
 	sti_crtc.o \
 	sti_plane.o \
 	sti_hdmi.o \
-	sti_hdmi_tx3g0c55phy.o \
 	sti_hdmi_tx3g4c28phy.o \
 	sti_dvo.o \
 	sti_awg_utils.o \
diff --git a/drivers/gpu/drm/sti/sti_compositor.c b/drivers/gpu/drm/sti/sti_compositor.c
index 134201ecc6fd..f62041fe8412 100644
--- a/drivers/gpu/drm/sti/sti_compositor.c
+++ b/drivers/gpu/drm/sti/sti_compositor.c
@@ -25,7 +25,7 @@
 /*
  * stiH407 compositor properties
  */
-struct sti_compositor_data stih407_compositor_data = {
+static const struct sti_compositor_data stih407_compositor_data = {
 	.nb_subdev = 8,
 	.subdev_desc = {
 			{STI_CURSOR_SUBDEV, (int)STI_CURSOR, 0x000},
@@ -39,38 +39,18 @@ struct sti_compositor_data stih407_compositor_data = {
 	},
 };
 
-/*
- * stiH416 compositor properties
- * Note:
- * on stih416 MIXER_AUX has a different base address from MIXER_MAIN
- * Moreover, GDPx is different for Main and Aux Mixer. So this subdev map does
- * not fit for stiH416 if we want to enable the MIXER_AUX.
- */
-struct sti_compositor_data stih416_compositor_data = {
-	.nb_subdev = 3,
-	.subdev_desc = {
-			{STI_GPD_SUBDEV, (int)STI_GDP_0, 0x100},
-			{STI_GPD_SUBDEV, (int)STI_GDP_1, 0x200},
-			{STI_MIXER_MAIN_SUBDEV, STI_MIXER_MAIN, 0xC00}
-	},
-};
-
-int sti_compositor_debufs_init(struct sti_compositor *compo,
-			       struct drm_minor *minor)
+int sti_compositor_debugfs_init(struct sti_compositor *compo,
+				struct drm_minor *minor)
 {
-	int ret = 0, i;
+	unsigned int i;
 
-	for (i = 0; compo->vid[i]; i++) {
-		ret = vid_debugfs_init(compo->vid[i], minor);
-		if (ret)
-			return ret;
-	}
+	for (i = 0; i < STI_MAX_VID; i++)
+		if (compo->vid[i])
+			vid_debugfs_init(compo->vid[i], minor);
 
-	for (i = 0; compo->mixer[i]; i++) {
-		ret = sti_mixer_debugfs_init(compo->mixer[i], minor);
-		if (ret)
-			return ret;
-	}
+	for (i = 0; i < STI_MAX_MIXER; i++)
+		if (compo->mixer[i])
+			sti_mixer_debugfs_init(compo->mixer[i], minor);
 
 	return 0;
 }
@@ -183,9 +163,6 @@ static const struct component_ops sti_compositor_ops = {
 
 static const struct of_device_id compositor_of_match[] = {
 	{
-		.compatible = "st,stih416-compositor",
-		.data = &stih416_compositor_data,
-	}, {
 		.compatible = "st,stih407-compositor",
 		.data = &stih407_compositor_data,
 	}, {
@@ -201,6 +178,7 @@ static int sti_compositor_probe(struct platform_device *pdev)
 	struct device_node *vtg_np;
 	struct sti_compositor *compo;
 	struct resource *res;
+	unsigned int i;
 
 	compo = devm_kzalloc(dev, sizeof(*compo), GFP_KERNEL);
 	if (!compo) {
@@ -208,7 +186,8 @@ static int sti_compositor_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 	compo->dev = dev;
-	compo->vtg_vblank_nb.notifier_call = sti_crtc_vblank_cb;
+	for (i = 0; i < STI_MAX_MIXER; i++)
+		compo->vtg_vblank_nb[i].notifier_call = sti_crtc_vblank_cb;
 
 	/* populate data structure depending on compatibility */
 	BUG_ON(!of_match_node(compositor_of_match, np)->data);
@@ -266,12 +245,12 @@ static int sti_compositor_probe(struct platform_device *pdev)
 
 	vtg_np = of_parse_phandle(pdev->dev.of_node, "st,vtg", 0);
 	if (vtg_np)
-		compo->vtg_main = of_vtg_find(vtg_np);
+		compo->vtg[STI_MIXER_MAIN] = of_vtg_find(vtg_np);
 	of_node_put(vtg_np);
 
 	vtg_np = of_parse_phandle(pdev->dev.of_node, "st,vtg", 1);
 	if (vtg_np)
-		compo->vtg_aux = of_vtg_find(vtg_np);
+		compo->vtg[STI_MIXER_AUX] = of_vtg_find(vtg_np);
 	of_node_put(vtg_np);
 
 	platform_set_drvdata(pdev, compo);
diff --git a/drivers/gpu/drm/sti/sti_compositor.h b/drivers/gpu/drm/sti/sti_compositor.h
index 24444ef42a98..2952a2d25a52 100644
--- a/drivers/gpu/drm/sti/sti_compositor.h
+++ b/drivers/gpu/drm/sti/sti_compositor.h
@@ -60,9 +60,8 @@ struct sti_compositor_data {
  * @rst_aux: reset control of the aux path
  * @mixer: array of mixers
  * @vid: array of vids
- * @vtg_main: vtg for main data path
- * @vtg_aux: vtg for auxillary data path
- * @vtg_vblank_nb: callback for VTG VSYNC notification
+ * @vtg: array of vtgs
+ * @vtg_vblank_nb: array of callbacks for VTG VSYNC notification
  */
 struct sti_compositor {
 	struct device *dev;
@@ -76,12 +75,11 @@ struct sti_compositor {
 	struct reset_control *rst_aux;
 	struct sti_mixer *mixer[STI_MAX_MIXER];
 	struct sti_vid *vid[STI_MAX_VID];
-	struct sti_vtg *vtg_main;
-	struct sti_vtg *vtg_aux;
-	struct notifier_block vtg_vblank_nb;
+	struct sti_vtg *vtg[STI_MAX_MIXER];
+	struct notifier_block vtg_vblank_nb[STI_MAX_MIXER];
 };
 
-int sti_compositor_debufs_init(struct sti_compositor *compo,
-			       struct drm_minor *minor);
+int sti_compositor_debugfs_init(struct sti_compositor *compo,
+				struct drm_minor *minor);
 
 #endif
diff --git a/drivers/gpu/drm/sti/sti_crtc.c b/drivers/gpu/drm/sti/sti_crtc.c
index c7d734dc3cf4..e992bed98dcb 100644
--- a/drivers/gpu/drm/sti/sti_crtc.c
+++ b/drivers/gpu/drm/sti/sti_crtc.c
@@ -86,8 +86,7 @@ sti_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode)
 		goto pix_error;
 	}
 
-	sti_vtg_set_config(mixer->id == STI_MIXER_MAIN ?
-			compo->vtg_main : compo->vtg_aux, &crtc->mode);
+	sti_vtg_set_config(compo->vtg[mixer->id], &crtc->mode);
 
 	if (sti_mixer_active_video_area(mixer, &crtc->mode)) {
 		DRM_ERROR("Can't set active video area\n");
@@ -166,6 +165,10 @@ static void sti_crtc_atomic_flush(struct drm_crtc *crtc,
 
 		switch (plane->status) {
 		case STI_PLANE_UPDATED:
+			/* ignore update for other CRTC */
+			if (p->state->crtc != crtc)
+				continue;
+
 			/* update planes tag as updated */
 			DRM_DEBUG_DRIVER("update plane %s\n",
 					 sti_plane_to_str(plane));
@@ -244,8 +247,7 @@ static int sti_crtc_set_property(struct drm_crtc *crtc,
 int sti_crtc_vblank_cb(struct notifier_block *nb,
 		       unsigned long event, void *data)
 {
-	struct sti_compositor *compo =
-		container_of(nb, struct sti_compositor, vtg_vblank_nb);
+	struct sti_compositor *compo;
 	struct drm_crtc *crtc = data;
 	struct sti_mixer *mixer;
 	unsigned long flags;
@@ -254,6 +256,7 @@ int sti_crtc_vblank_cb(struct notifier_block *nb,
 
 	priv = crtc->dev->dev_private;
 	pipe = drm_crtc_index(crtc);
+	compo = container_of(nb, struct sti_compositor, vtg_vblank_nb[pipe]);
 	mixer = compo->mixer[pipe];
 
 	if ((event != VTG_TOP_FIELD_EVENT) &&
@@ -295,14 +298,13 @@ int sti_crtc_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	struct sti_private *dev_priv = dev->dev_private;
 	struct sti_compositor *compo = dev_priv->compo;
-	struct notifier_block *vtg_vblank_nb = &compo->vtg_vblank_nb;
+	struct notifier_block *vtg_vblank_nb = &compo->vtg_vblank_nb[pipe];
 	struct drm_crtc *crtc = &compo->mixer[pipe]->drm_crtc;
+	struct sti_vtg *vtg = compo->vtg[pipe];
 
 	DRM_DEBUG_DRIVER("\n");
 
-	if (sti_vtg_register_client(pipe == STI_MIXER_MAIN ?
-			compo->vtg_main : compo->vtg_aux,
-			vtg_vblank_nb, crtc)) {
+	if (sti_vtg_register_client(vtg, vtg_vblank_nb, crtc)) {
 		DRM_ERROR("Cannot register VTG notifier\n");
 		return -EINVAL;
 	}
@@ -314,13 +316,13 @@ void sti_crtc_disable_vblank(struct drm_device *drm_dev, unsigned int pipe)
 {
 	struct sti_private *priv = drm_dev->dev_private;
 	struct sti_compositor *compo = priv->compo;
-	struct notifier_block *vtg_vblank_nb = &compo->vtg_vblank_nb;
+	struct notifier_block *vtg_vblank_nb = &compo->vtg_vblank_nb[pipe];
 	struct drm_crtc *crtc = &compo->mixer[pipe]->drm_crtc;
+	struct sti_vtg *vtg = compo->vtg[pipe];
 
 	DRM_DEBUG_DRIVER("\n");
 
-	if (sti_vtg_unregister_client(pipe == STI_MIXER_MAIN ?
-			compo->vtg_main : compo->vtg_aux, vtg_vblank_nb))
+	if (sti_vtg_unregister_client(vtg, vtg_vblank_nb))
 		DRM_DEBUG_DRIVER("Warning: cannot unregister VTG notifier\n");
 
 	/* free the resources of the pending requests */
@@ -336,7 +338,7 @@ static int sti_crtc_late_register(struct drm_crtc *crtc)
 	struct sti_compositor *compo = dev_get_drvdata(mixer->dev);
 
 	if (drm_crtc_index(crtc) == 0)
-		return sti_compositor_debufs_init(compo, crtc->dev->primary);
+		return sti_compositor_debugfs_init(compo, crtc->dev->primary);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/sti/sti_cursor.c b/drivers/gpu/drm/sti/sti_cursor.c
index 3b53f7f2e3fc..cca75bddb9ad 100644
--- a/drivers/gpu/drm/sti/sti_cursor.c
+++ b/drivers/gpu/drm/sti/sti_cursor.c
@@ -309,15 +309,15 @@ static void sti_cursor_atomic_disable(struct drm_plane *drm_plane,
 {
 	struct sti_plane *plane = to_sti_plane(drm_plane);
 
-	if (!drm_plane->crtc) {
+	if (!oldstate->crtc) {
 		DRM_DEBUG_DRIVER("drm plane:%d not enabled\n",
 				 drm_plane->base.id);
 		return;
 	}
 
 	DRM_DEBUG_DRIVER("CRTC:%d (%s) drm plane:%d (%s)\n",
-			 drm_plane->crtc->base.id,
-			 sti_mixer_to_str(to_sti_mixer(drm_plane->crtc)),
+			 oldstate->crtc->base.id,
+			 sti_mixer_to_str(to_sti_mixer(oldstate->crtc)),
 			 drm_plane->base.id, sti_plane_to_str(plane));
 
 	plane->status = STI_PLANE_DISABLING;
@@ -345,7 +345,7 @@ static int sti_cursor_late_register(struct drm_plane *drm_plane)
 	return cursor_debugfs_init(cursor, drm_plane->dev->primary);
 }
 
-struct drm_plane_funcs sti_cursor_plane_helpers_funcs = {
+static const struct drm_plane_funcs sti_cursor_plane_helpers_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = sti_cursor_destroy,
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index 96bd3d08b2d4..2784919a7366 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -140,7 +140,7 @@ err:
 	return ret;
 }
 
-void sti_drm_dbg_cleanup(struct drm_minor *minor)
+static void sti_drm_dbg_cleanup(struct drm_minor *minor)
 {
 	drm_debugfs_remove_files(sti_drm_dbg_list,
 				 ARRAY_SIZE(sti_drm_dbg_list), minor);
@@ -178,7 +178,7 @@ static void sti_atomic_complete(struct sti_private *private,
 	 */
 
 	drm_atomic_helper_commit_modeset_disables(drm, state);
-	drm_atomic_helper_commit_planes(drm, state, false);
+	drm_atomic_helper_commit_planes(drm, state, 0);
 	drm_atomic_helper_commit_modeset_enables(drm, state);
 
 	drm_atomic_helper_wait_for_vblanks(drm, state);
@@ -282,7 +282,7 @@ static const struct file_operations sti_driver_fops = {
 };
 
 static struct drm_driver sti_driver = {
-	.driver_features = DRIVER_HAVE_IRQ | DRIVER_MODESET |
+	.driver_features = DRIVER_MODESET |
 	    DRIVER_GEM | DRIVER_PRIME | DRIVER_ATOMIC,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
@@ -365,8 +365,8 @@ static int sti_bind(struct device *dev)
 	int ret;
 
 	ddev = drm_dev_alloc(&sti_driver, dev);
-	if (!ddev)
-		return -ENOMEM;
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
 
 	ddev->platformdev = to_platform_device(dev);
 
diff --git a/drivers/gpu/drm/sti/sti_dvo.c b/drivers/gpu/drm/sti/sti_dvo.c
index 00881eb4536e..e8c1ed08a9f7 100644
--- a/drivers/gpu/drm/sti/sti_dvo.c
+++ b/drivers/gpu/drm/sti/sti_dvo.c
@@ -17,6 +17,7 @@
 #include <drm/drm_panel.h>
 
 #include "sti_awg_utils.h"
+#include "sti_drv.h"
 #include "sti_mixer.h"
 
 /* DVO registers */
@@ -106,7 +107,7 @@ struct sti_dvo_connector {
 	container_of(x, struct sti_dvo_connector, drm_connector)
 
 #define BLANKING_LEVEL 16
-int dvo_awg_generate_code(struct sti_dvo *dvo, u8 *ram_size, u32 *ram_code)
+static int dvo_awg_generate_code(struct sti_dvo *dvo, u8 *ram_size, u32 *ram_code)
 {
 	struct drm_display_mode *mode = &dvo->mode;
 	struct dvo_config *config = dvo->config;
diff --git a/drivers/gpu/drm/sti/sti_gdp.c b/drivers/gpu/drm/sti/sti_gdp.c
index b8d942ca45e8..81df3097b545 100644
--- a/drivers/gpu/drm/sti/sti_gdp.c
+++ b/drivers/gpu/drm/sti/sti_gdp.c
@@ -460,6 +460,7 @@ static void sti_gdp_disable(struct sti_gdp *gdp)
 		clk_disable_unprepare(gdp->clk_pix);
 
 	gdp->plane.status = STI_PLANE_DISABLED;
+	gdp->vtg = NULL;
 }
 
 /**
@@ -473,8 +474,8 @@ static void sti_gdp_disable(struct sti_gdp *gdp)
  * RETURNS:
  * 0 on success.
  */
-int sti_gdp_field_cb(struct notifier_block *nb,
-		unsigned long event, void *data)
+static int sti_gdp_field_cb(struct notifier_block *nb,
+			    unsigned long event, void *data)
 {
 	struct sti_gdp *gdp = container_of(nb, struct sti_gdp, vtg_field_nb);
 
@@ -611,7 +612,6 @@ static int sti_gdp_atomic_check(struct drm_plane *drm_plane,
 	struct drm_crtc *crtc = state->crtc;
 	struct sti_compositor *compo = dev_get_drvdata(gdp->dev);
 	struct drm_framebuffer *fb =  state->fb;
-	bool first_prepare = plane->status == STI_PLANE_DISABLED ? true : false;
 	struct drm_crtc_state *crtc_state;
 	struct sti_mixer *mixer;
 	struct drm_display_mode *mode;
@@ -628,8 +628,8 @@ static int sti_gdp_atomic_check(struct drm_plane *drm_plane,
 	mode = &crtc_state->mode;
 	dst_x = state->crtc_x;
 	dst_y = state->crtc_y;
-	dst_w = clamp_val(state->crtc_w, 0, mode->crtc_hdisplay - dst_x);
-	dst_h = clamp_val(state->crtc_h, 0, mode->crtc_vdisplay - dst_y);
+	dst_w = clamp_val(state->crtc_w, 0, mode->hdisplay - dst_x);
+	dst_h = clamp_val(state->crtc_h, 0, mode->vdisplay - dst_y);
 	/* src_x are in 16.16 format */
 	src_x = state->src_x >> 16;
 	src_y = state->src_y >> 16;
@@ -648,10 +648,9 @@ static int sti_gdp_atomic_check(struct drm_plane *drm_plane,
 		return -EINVAL;
 	}
 
-	if (first_prepare) {
+	if (!gdp->vtg) {
 		/* Register gdp callback */
-		gdp->vtg = mixer->id == STI_MIXER_MAIN ?
-					compo->vtg_main : compo->vtg_aux;
+		gdp->vtg = compo->vtg[mixer->id];
 		if (sti_vtg_register_client(gdp->vtg,
 					    &gdp->vtg_field_nb, crtc)) {
 			DRM_ERROR("Cannot register VTG notifier\n");
@@ -719,7 +718,7 @@ static void sti_gdp_atomic_update(struct drm_plane *drm_plane,
 	u32 dma_updated_top;
 	u32 dma_updated_btm;
 	int format;
-	unsigned int depth, bpp;
+	unsigned int bpp;
 	u32 ydo, xdo, yds, xds;
 
 	if (!crtc || !fb)
@@ -728,8 +727,8 @@ static void sti_gdp_atomic_update(struct drm_plane *drm_plane,
 	mode = &crtc->mode;
 	dst_x = state->crtc_x;
 	dst_y = state->crtc_y;
-	dst_w = clamp_val(state->crtc_w, 0, mode->crtc_hdisplay - dst_x);
-	dst_h = clamp_val(state->crtc_h, 0, mode->crtc_vdisplay - dst_y);
+	dst_w = clamp_val(state->crtc_w, 0, mode->hdisplay - dst_x);
+	dst_h = clamp_val(state->crtc_h, 0, mode->vdisplay - dst_y);
 	/* src_x are in 16.16 format */
 	src_x = state->src_x >> 16;
 	src_y = state->src_y >> 16;
@@ -758,9 +757,9 @@ static void sti_gdp_atomic_update(struct drm_plane *drm_plane,
 			 (unsigned long)cma_obj->paddr);
 
 	/* pixel memory location */
-	drm_fb_get_bpp_depth(fb->pixel_format, &depth, &bpp);
+	bpp = drm_format_plane_cpp(fb->pixel_format, 0);
 	top_field->gam_gdp_pml = (u32)cma_obj->paddr + fb->offsets[0];
-	top_field->gam_gdp_pml += src_x * (bpp >> 3);
+	top_field->gam_gdp_pml += src_x * bpp;
 	top_field->gam_gdp_pml += src_y * fb->pitches[0];
 
 	/* output parameters (clamped / cropped) */
@@ -810,7 +809,7 @@ static void sti_gdp_atomic_update(struct drm_plane *drm_plane,
 	if (!curr_list) {
 		/* First update or invalid node should directly write in the
 		 * hw register */
-		DRM_DEBUG_DRIVER("%s first update (or invalid node)",
+		DRM_DEBUG_DRIVER("%s first update (or invalid node)\n",
 				 sti_plane_to_str(plane));
 
 		writel(gdp->is_curr_top ?
@@ -846,15 +845,15 @@ static void sti_gdp_atomic_disable(struct drm_plane *drm_plane,
 {
 	struct sti_plane *plane = to_sti_plane(drm_plane);
 
-	if (!drm_plane->crtc) {
+	if (!oldstate->crtc) {
 		DRM_DEBUG_DRIVER("drm plane:%d not enabled\n",
 				 drm_plane->base.id);
 		return;
 	}
 
 	DRM_DEBUG_DRIVER("CRTC:%d (%s) drm plane:%d (%s)\n",
-			 drm_plane->crtc->base.id,
-			 sti_mixer_to_str(to_sti_mixer(drm_plane->crtc)),
+			 oldstate->crtc->base.id,
+			 sti_mixer_to_str(to_sti_mixer(oldstate->crtc)),
 			 drm_plane->base.id, sti_plane_to_str(plane));
 
 	plane->status = STI_PLANE_DISABLING;
@@ -882,7 +881,7 @@ static int sti_gdp_late_register(struct drm_plane *drm_plane)
 	return gdp_debugfs_init(gdp, drm_plane->dev->primary);
 }
 
-struct drm_plane_funcs sti_gdp_plane_helpers_funcs = {
+static const struct drm_plane_funcs sti_gdp_plane_helpers_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = sti_gdp_destroy,
diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c
index 8505569f75de..e7c243f70870 100644
--- a/drivers/gpu/drm/sti/sti_hda.c
+++ b/drivers/gpu/drm/sti/sti_hda.c
@@ -62,14 +62,8 @@
 #define SCALE_CTRL_CR_DFLT              0x00DB0249
 
 /* Video DACs control */
-#define VIDEO_DACS_CONTROL_MASK         0x0FFF
-#define VIDEO_DACS_CONTROL_SYSCFG2535   0x085C /* for stih416 */
-#define DAC_CFG_HD_OFF_SHIFT            5
-#define DAC_CFG_HD_OFF_MASK             (0x7 << DAC_CFG_HD_OFF_SHIFT)
-#define VIDEO_DACS_CONTROL_SYSCFG5072   0x0120 /* for stih407 */
 #define DAC_CFG_HD_HZUVW_OFF_MASK       BIT(1)
 
-
 /* Upsampler values for the alternative 2X Filter */
 #define SAMPLER_COEF_NB                 8
 #define HDA_ANA_SRC_Y_CFG_ALT_2X        0x01130000
@@ -300,28 +294,14 @@ static bool hda_get_mode_idx(struct drm_display_mode mode, int *idx)
  */
 static void hda_enable_hd_dacs(struct sti_hda *hda, bool enable)
 {
-	u32 mask;
-
 	if (hda->video_dacs_ctrl) {
 		u32 val;
 
-		switch ((u32)hda->video_dacs_ctrl & VIDEO_DACS_CONTROL_MASK) {
-		case VIDEO_DACS_CONTROL_SYSCFG2535:
-			mask = DAC_CFG_HD_OFF_MASK;
-			break;
-		case VIDEO_DACS_CONTROL_SYSCFG5072:
-			mask = DAC_CFG_HD_HZUVW_OFF_MASK;
-			break;
-		default:
-			DRM_INFO("Video DACS control register not supported!");
-			return;
-		}
-
 		val = readl(hda->video_dacs_ctrl);
 		if (enable)
-			val &= ~mask;
+			val &= ~DAC_CFG_HD_HZUVW_OFF_MASK;
 		else
-			val |= mask;
+			val |= DAC_CFG_HD_HZUVW_OFF_MASK;
 
 		writel(val, hda->video_dacs_ctrl);
 	}
@@ -352,24 +332,11 @@ static void hda_dbg_awg_microcode(struct seq_file *s, void __iomem *reg)
 static void hda_dbg_video_dacs_ctrl(struct seq_file *s, void __iomem *reg)
 {
 	u32 val = readl(reg);
-	u32 mask;
-
-	switch ((u32)reg & VIDEO_DACS_CONTROL_MASK) {
-	case VIDEO_DACS_CONTROL_SYSCFG2535:
-		mask = DAC_CFG_HD_OFF_MASK;
-		break;
-	case VIDEO_DACS_CONTROL_SYSCFG5072:
-		mask = DAC_CFG_HD_HZUVW_OFF_MASK;
-		break;
-	default:
-		DRM_DEBUG_DRIVER("Warning: DACS ctrl register not supported!");
-		return;
-	}
 
 	seq_puts(s, "\n");
 	seq_printf(s, "\n  %-25s 0x%08X", "VIDEO_DACS_CONTROL", val);
 	seq_puts(s, "\tHD DACs ");
-	seq_puts(s, val & mask ? "disabled" : "enabled");
+	seq_puts(s, val & DAC_CFG_HD_HZUVW_OFF_MASK ? "disabled" : "enabled");
 }
 
 static int hda_dbg_show(struct seq_file *s, void *data)
diff --git a/drivers/gpu/drm/sti/sti_hdmi.c b/drivers/gpu/drm/sti/sti_hdmi.c
index fedc17f98d9b..376b0763c874 100644
--- a/drivers/gpu/drm/sti/sti_hdmi.c
+++ b/drivers/gpu/drm/sti/sti_hdmi.c
@@ -22,7 +22,6 @@
 
 #include "sti_hdmi.h"
 #include "sti_hdmi_tx3g4c28phy.h"
-#include "sti_hdmi_tx3g0c55phy.h"
 #include "sti_vtg.h"
 
 #define HDMI_CFG                        0x0000
@@ -203,7 +202,7 @@ static irqreturn_t hdmi_irq_thread(int irq, void *arg)
 
 	/* Audio FIFO underrun IRQ */
 	if (hdmi->irq_status & HDMI_INT_AUDIO_FIFO_XRUN)
-		DRM_INFO("Warning: audio FIFO underrun occurs!");
+		DRM_INFO("Warning: audio FIFO underrun occurs!\n");
 
 	return IRQ_HANDLED;
 }
@@ -569,7 +568,7 @@ static void hdmi_swreset(struct sti_hdmi *hdmi)
 
 	/* Wait reset completed */
 	wait_event_interruptible_timeout(hdmi->wait_event,
-					 hdmi->event_received == true,
+					 hdmi->event_received,
 					 msecs_to_jiffies
 					 (HDMI_TIMEOUT_SWRESET));
 
@@ -1054,6 +1053,7 @@ static int sti_hdmi_late_register(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs sti_hdmi_connector_funcs = {
+	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = sti_hdmi_connector_detect,
 	.destroy = drm_connector_cleanup,
@@ -1181,7 +1181,7 @@ static void hdmi_audio_shutdown(struct device *dev, void *data)
 		    HDMI_AUD_CFG_ONE_BIT_INVALID;
 	hdmi_write(hdmi, audio_cfg, HDMI_AUDIO_CFG);
 
-	hdmi->audio.enabled = 0;
+	hdmi->audio.enabled = false;
 	hdmi_audio_infoframe_config(hdmi);
 }
 
@@ -1213,7 +1213,7 @@ static int hdmi_audio_hw_params(struct device *dev,
 		return -EINVAL;
 	}
 
-	audio.enabled = 1;
+	audio.enabled = true;
 
 	ret = hdmi_audio_configure(hdmi, &audio);
 	if (ret < 0)
@@ -1265,7 +1265,7 @@ static int sti_hdmi_register_audio_driver(struct device *dev,
 
 	DRM_DEBUG_DRIVER("\n");
 
-	hdmi->audio.enabled = 0;
+	hdmi->audio.enabled = false;
 
 	hdmi->audio_pdev = platform_device_register_data(
 		dev, HDMI_CODEC_DRV_NAME, PLATFORM_DEVID_AUTO,
@@ -1373,9 +1373,6 @@ static const struct component_ops sti_hdmi_ops = {
 
 static const struct of_device_id hdmi_of_match[] = {
 	{
-		.compatible = "st,stih416-hdmi",
-		.data = &tx3g0c55phy_ops,
-	}, {
 		.compatible = "st,stih407-hdmi",
 		.data = &tx3g4c28phy_ops,
 	}, {
@@ -1422,22 +1419,6 @@ static int sti_hdmi_probe(struct platform_device *pdev)
 		goto release_adapter;
 	}
 
-	if (of_device_is_compatible(np, "st,stih416-hdmi")) {
-		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
-						   "syscfg");
-		if (!res) {
-			DRM_ERROR("Invalid syscfg resource\n");
-			ret = -ENOMEM;
-			goto release_adapter;
-		}
-		hdmi->syscfg = devm_ioremap_nocache(dev, res->start,
-						    resource_size(res));
-		if (!hdmi->syscfg) {
-			ret = -ENOMEM;
-			goto release_adapter;
-		}
-	}
-
 	hdmi->phy_ops = (struct hdmi_phy_ops *)
 		of_match_node(hdmi_of_match, np)->data;
 
diff --git a/drivers/gpu/drm/sti/sti_hdmi_tx3g0c55phy.c b/drivers/gpu/drm/sti/sti_hdmi_tx3g0c55phy.c
deleted file mode 100644
index 49ae8e44b285..000000000000
--- a/drivers/gpu/drm/sti/sti_hdmi_tx3g0c55phy.c
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Copyright (C) STMicroelectronics SA 2014
- * Author: Vincent Abriou <vincent.abriou@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
- */
-
-#include "sti_hdmi_tx3g0c55phy.h"
-
-#define HDMI_SRZ_PLL_CFG                0x0504
-#define HDMI_SRZ_TAP_1                  0x0508
-#define HDMI_SRZ_TAP_2                  0x050C
-#define HDMI_SRZ_TAP_3                  0x0510
-#define HDMI_SRZ_CTRL                   0x0514
-
-#define HDMI_SRZ_PLL_CFG_POWER_DOWN     BIT(0)
-#define HDMI_SRZ_PLL_CFG_VCOR_SHIFT     1
-#define HDMI_SRZ_PLL_CFG_VCOR_425MHZ    0
-#define HDMI_SRZ_PLL_CFG_VCOR_850MHZ    1
-#define HDMI_SRZ_PLL_CFG_VCOR_1700MHZ   2
-#define HDMI_SRZ_PLL_CFG_VCOR_3000MHZ   3
-#define HDMI_SRZ_PLL_CFG_VCOR_MASK      3
-#define HDMI_SRZ_PLL_CFG_VCOR(x)        (x << HDMI_SRZ_PLL_CFG_VCOR_SHIFT)
-#define HDMI_SRZ_PLL_CFG_NDIV_SHIFT     8
-#define HDMI_SRZ_PLL_CFG_NDIV_MASK      (0x1F << HDMI_SRZ_PLL_CFG_NDIV_SHIFT)
-#define HDMI_SRZ_PLL_CFG_MODE_SHIFT     16
-#define HDMI_SRZ_PLL_CFG_MODE_13_5_MHZ  0x1
-#define HDMI_SRZ_PLL_CFG_MODE_25_2_MHZ  0x4
-#define HDMI_SRZ_PLL_CFG_MODE_27_MHZ    0x5
-#define HDMI_SRZ_PLL_CFG_MODE_33_75_MHZ 0x6
-#define HDMI_SRZ_PLL_CFG_MODE_40_5_MHZ  0x7
-#define HDMI_SRZ_PLL_CFG_MODE_54_MHZ    0x8
-#define HDMI_SRZ_PLL_CFG_MODE_67_5_MHZ  0x9
-#define HDMI_SRZ_PLL_CFG_MODE_74_25_MHZ 0xA
-#define HDMI_SRZ_PLL_CFG_MODE_81_MHZ    0xB
-#define HDMI_SRZ_PLL_CFG_MODE_82_5_MHZ  0xC
-#define HDMI_SRZ_PLL_CFG_MODE_108_MHZ   0xD
-#define HDMI_SRZ_PLL_CFG_MODE_148_5_MHZ 0xE
-#define HDMI_SRZ_PLL_CFG_MODE_165_MHZ   0xF
-#define HDMI_SRZ_PLL_CFG_MODE_MASK      0xF
-#define HDMI_SRZ_PLL_CFG_MODE(x)        (x << HDMI_SRZ_PLL_CFG_MODE_SHIFT)
-
-#define HDMI_SRZ_CTRL_POWER_DOWN        (1 << 0)
-#define HDMI_SRZ_CTRL_EXTERNAL_DATA_EN  (1 << 1)
-
-/* sysconf registers */
-#define HDMI_REJECTION_PLL_CONFIGURATION 0x0858	/* SYSTEM_CONFIG2534 */
-#define HDMI_REJECTION_PLL_STATUS        0x0948	/* SYSTEM_CONFIG2594 */
-
-#define REJECTION_PLL_HDMI_ENABLE_SHIFT 0
-#define REJECTION_PLL_HDMI_ENABLE_MASK  (0x1 << REJECTION_PLL_HDMI_ENABLE_SHIFT)
-#define REJECTION_PLL_HDMI_PDIV_SHIFT   24
-#define REJECTION_PLL_HDMI_PDIV_MASK    (0x7 << REJECTION_PLL_HDMI_PDIV_SHIFT)
-#define REJECTION_PLL_HDMI_NDIV_SHIFT   16
-#define REJECTION_PLL_HDMI_NDIV_MASK    (0xFF << REJECTION_PLL_HDMI_NDIV_SHIFT)
-#define REJECTION_PLL_HDMI_MDIV_SHIFT   8
-#define REJECTION_PLL_HDMI_MDIV_MASK    (0xFF << REJECTION_PLL_HDMI_MDIV_SHIFT)
-
-#define REJECTION_PLL_HDMI_REJ_PLL_LOCK BIT(0)
-
-#define HDMI_TIMEOUT_PLL_LOCK  50   /*milliseconds */
-
-/**
- * pll mode structure
- *
- * A pointer to an array of these structures is passed to a TMDS (HDMI) output
- * via the control interface to provide board and SoC specific
- * configurations of the HDMI PHY. Each entry in the array specifies a hardware
- * specific configuration for a given TMDS clock frequency range. The array
- * should be terminated with an entry that has all fields set to zero.
- *
- * @min: Lower bound of TMDS clock frequency this entry applies to
- * @max: Upper bound of TMDS clock frequency this entry applies to
- * @mode: SoC specific register configuration
- */
-struct pllmode {
-	u32 min;
-	u32 max;
-	u32 mode;
-};
-
-#define NB_PLL_MODE 7
-static struct pllmode pllmodes[NB_PLL_MODE] = {
-	{13500000, 13513500, HDMI_SRZ_PLL_CFG_MODE_13_5_MHZ},
-	{25174800, 25200000, HDMI_SRZ_PLL_CFG_MODE_25_2_MHZ},
-	{27000000, 27027000, HDMI_SRZ_PLL_CFG_MODE_27_MHZ},
-	{54000000, 54054000, HDMI_SRZ_PLL_CFG_MODE_54_MHZ},
-	{72000000, 74250000, HDMI_SRZ_PLL_CFG_MODE_74_25_MHZ},
-	{108000000, 108108000, HDMI_SRZ_PLL_CFG_MODE_108_MHZ},
-	{148351648, 297000000, HDMI_SRZ_PLL_CFG_MODE_148_5_MHZ}
-};
-
-#define NB_HDMI_PHY_CONFIG 5
-static struct hdmi_phy_config hdmiphy_config[NB_HDMI_PHY_CONFIG] = {
-	{0, 40000000, {0x00101010, 0x00101010, 0x00101010, 0x02} },
-	{40000000, 140000000, {0x00111111, 0x00111111, 0x00111111, 0x02} },
-	{140000000, 160000000, {0x00131313, 0x00101010, 0x00101010, 0x02} },
-	{160000000, 250000000, {0x00131313, 0x00111111, 0x00111111, 0x03FE} },
-	{250000000, 300000000, {0x00151515, 0x00101010, 0x00101010, 0x03FE} },
-};
-
-#define PLL_CHANGE_DELAY	1 /* ms */
-
-/**
- * Disable the pll rejection
- *
- * @hdmi: pointer on the hdmi internal structure
- *
- * return true if the pll has been disabled
- */
-static bool disable_pll_rejection(struct sti_hdmi *hdmi)
-{
-	u32 val;
-
-	DRM_DEBUG_DRIVER("\n");
-
-	val = readl(hdmi->syscfg + HDMI_REJECTION_PLL_CONFIGURATION);
-	val &= ~REJECTION_PLL_HDMI_ENABLE_MASK;
-	writel(val, hdmi->syscfg + HDMI_REJECTION_PLL_CONFIGURATION);
-
-	msleep(PLL_CHANGE_DELAY);
-	val = readl(hdmi->syscfg + HDMI_REJECTION_PLL_STATUS);
-
-	return !(val & REJECTION_PLL_HDMI_REJ_PLL_LOCK);
-}
-
-/**
- * Enable the old BCH/rejection PLL is now reused to provide the CLKPXPLL
- * clock input to the new PHY PLL that generates the serializer clock
- * (TMDS*10) and the TMDS clock which is now fed back into the HDMI
- * formatter instead of the TMDS clock line from ClockGenB.
- *
- * @hdmi: pointer on the hdmi internal structure
- *
- * return true if pll has been correctly set
- */
-static bool enable_pll_rejection(struct sti_hdmi *hdmi)
-{
-	unsigned int inputclock;
-	u32 mdiv, ndiv, pdiv, val;
-
-	DRM_DEBUG_DRIVER("\n");
-
-	if (!disable_pll_rejection(hdmi))
-		return false;
-
-	inputclock = hdmi->mode.clock * 1000;
-
-	DRM_DEBUG_DRIVER("hdmi rejection pll input clock = %dHz\n", inputclock);
-
-
-	/* Power up the HDMI rejection PLL
-	 * Note: On this SoC (stiH416) we are forced to have the input clock
-	 * be equal to the HDMI pixel clock.
-	 *
-	 * The values here have been suggested by validation however they are
-	 * still provisional and subject to change.
-	 *
-	 * PLLout = (Fin*Mdiv) / ((2 * Ndiv) / 2^Pdiv)
-	 */
-	if (inputclock < 50000000) {
-		/*
-		 * For slower clocks we need to multiply more to keep the
-		 * internal VCO frequency within the physical specification
-		 * of the PLL.
-		 */
-		pdiv = 4;
-		ndiv = 240;
-		mdiv = 30;
-	} else {
-		pdiv = 2;
-		ndiv = 60;
-		mdiv = 30;
-	}
-
-	val = readl(hdmi->syscfg + HDMI_REJECTION_PLL_CONFIGURATION);
-
-	val &= ~(REJECTION_PLL_HDMI_PDIV_MASK |
-		REJECTION_PLL_HDMI_NDIV_MASK |
-		REJECTION_PLL_HDMI_MDIV_MASK |
-		REJECTION_PLL_HDMI_ENABLE_MASK);
-
-	val |=	(pdiv << REJECTION_PLL_HDMI_PDIV_SHIFT) |
-		(ndiv << REJECTION_PLL_HDMI_NDIV_SHIFT) |
-		(mdiv << REJECTION_PLL_HDMI_MDIV_SHIFT) |
-		(0x1 << REJECTION_PLL_HDMI_ENABLE_SHIFT);
-
-	writel(val, hdmi->syscfg + HDMI_REJECTION_PLL_CONFIGURATION);
-
-	msleep(PLL_CHANGE_DELAY);
-	val = readl(hdmi->syscfg + HDMI_REJECTION_PLL_STATUS);
-
-	return (val & REJECTION_PLL_HDMI_REJ_PLL_LOCK);
-}
-
-/**
- * Start hdmi phy macro cell tx3g0c55
- *
- * @hdmi: pointer on the hdmi internal structure
- *
- * Return false if an error occur
- */
-static bool sti_hdmi_tx3g0c55phy_start(struct sti_hdmi *hdmi)
-{
-	u32 ckpxpll = hdmi->mode.clock * 1000;
-	u32 val, tmdsck, freqvco, pllctrl = 0;
-	unsigned int i;
-
-	if (!enable_pll_rejection(hdmi))
-		return false;
-
-	DRM_DEBUG_DRIVER("ckpxpll = %dHz\n", ckpxpll);
-
-	/* Assuming no pixel repetition and 24bits color */
-	tmdsck = ckpxpll;
-	pllctrl = 2 << HDMI_SRZ_PLL_CFG_NDIV_SHIFT;
-
-	/*
-	 * Setup the PLL mode parameter based on the ckpxpll. If we haven't got
-	 * a clock frequency supported by one of the specific PLL modes then we
-	 * will end up using the generic mode (0) which only supports a 10x
-	 * multiplier, hence only 24bit color.
-	 */
-	for (i = 0; i < NB_PLL_MODE; i++) {
-		if (ckpxpll >= pllmodes[i].min && ckpxpll <= pllmodes[i].max)
-			pllctrl |= HDMI_SRZ_PLL_CFG_MODE(pllmodes[i].mode);
-	}
-
-	freqvco = tmdsck * 10;
-	if (freqvco <= 425000000UL)
-		pllctrl |= HDMI_SRZ_PLL_CFG_VCOR(HDMI_SRZ_PLL_CFG_VCOR_425MHZ);
-	else if (freqvco <= 850000000UL)
-		pllctrl |= HDMI_SRZ_PLL_CFG_VCOR(HDMI_SRZ_PLL_CFG_VCOR_850MHZ);
-	else if (freqvco <= 1700000000UL)
-		pllctrl |= HDMI_SRZ_PLL_CFG_VCOR(HDMI_SRZ_PLL_CFG_VCOR_1700MHZ);
-	else if (freqvco <= 2970000000UL)
-		pllctrl |= HDMI_SRZ_PLL_CFG_VCOR(HDMI_SRZ_PLL_CFG_VCOR_3000MHZ);
-	else {
-		DRM_ERROR("PHY serializer clock out of range\n");
-		goto err;
-	}
-
-	/*
-	 * Configure and power up the PHY PLL
-	 */
-	hdmi->event_received = false;
-	DRM_DEBUG_DRIVER("pllctrl = 0x%x\n", pllctrl);
-	hdmi_write(hdmi, pllctrl, HDMI_SRZ_PLL_CFG);
-
-	/* wait PLL interrupt */
-	wait_event_interruptible_timeout(hdmi->wait_event,
-					 hdmi->event_received == true,
-					 msecs_to_jiffies
-					 (HDMI_TIMEOUT_PLL_LOCK));
-
-	if ((hdmi_read(hdmi, HDMI_STA) & HDMI_STA_DLL_LCK) == 0) {
-		DRM_ERROR("hdmi phy pll not locked\n");
-		goto err;
-	}
-
-	DRM_DEBUG_DRIVER("got PHY PLL Lock\n");
-
-	/*
-	 * To configure the source termination and pre-emphasis appropriately
-	 * for different high speed TMDS clock frequencies a phy configuration
-	 * table must be provided, tailored to the SoC and board combination.
-	 */
-	for (i = 0; i < NB_HDMI_PHY_CONFIG; i++) {
-		if ((hdmiphy_config[i].min_tmds_freq <= tmdsck) &&
-		    (hdmiphy_config[i].max_tmds_freq >= tmdsck)) {
-			val = hdmiphy_config[i].config[0];
-			hdmi_write(hdmi, val, HDMI_SRZ_TAP_1);
-			val = hdmiphy_config[i].config[1];
-			hdmi_write(hdmi, val, HDMI_SRZ_TAP_2);
-			val = hdmiphy_config[i].config[2];
-			hdmi_write(hdmi, val, HDMI_SRZ_TAP_3);
-			val = hdmiphy_config[i].config[3];
-			val |= HDMI_SRZ_CTRL_EXTERNAL_DATA_EN;
-			val &= ~HDMI_SRZ_CTRL_POWER_DOWN;
-			hdmi_write(hdmi, val, HDMI_SRZ_CTRL);
-
-			DRM_DEBUG_DRIVER("serializer cfg 0x%x 0x%x 0x%x 0x%x\n",
-					 hdmiphy_config[i].config[0],
-					 hdmiphy_config[i].config[1],
-					 hdmiphy_config[i].config[2],
-					 hdmiphy_config[i].config[3]);
-			return true;
-		}
-	}
-
-	/*
-	 * Default, power up the serializer with no pre-emphasis or source
-	 * termination.
-	 */
-	hdmi_write(hdmi, 0x0, HDMI_SRZ_TAP_1);
-	hdmi_write(hdmi, 0x0, HDMI_SRZ_TAP_2);
-	hdmi_write(hdmi, 0x0, HDMI_SRZ_TAP_3);
-	hdmi_write(hdmi, HDMI_SRZ_CTRL_EXTERNAL_DATA_EN, HDMI_SRZ_CTRL);
-
-	return true;
-
-err:
-	disable_pll_rejection(hdmi);
-
-	return false;
-}
-
-/**
- * Stop hdmi phy macro cell tx3g0c55
- *
- * @hdmi: pointer on the hdmi internal structure
- */
-static void sti_hdmi_tx3g0c55phy_stop(struct sti_hdmi *hdmi)
-{
-	DRM_DEBUG_DRIVER("\n");
-
-	hdmi->event_received = false;
-
-	hdmi_write(hdmi, HDMI_SRZ_CTRL_POWER_DOWN, HDMI_SRZ_CTRL);
-	hdmi_write(hdmi, HDMI_SRZ_PLL_CFG_POWER_DOWN, HDMI_SRZ_PLL_CFG);
-
-	/* wait PLL interrupt */
-	wait_event_interruptible_timeout(hdmi->wait_event,
-					 hdmi->event_received == true,
-					 msecs_to_jiffies
-					 (HDMI_TIMEOUT_PLL_LOCK));
-
-	if (hdmi_read(hdmi, HDMI_STA) & HDMI_STA_DLL_LCK)
-		DRM_ERROR("hdmi phy pll not well disabled\n");
-
-	disable_pll_rejection(hdmi);
-}
-
-struct hdmi_phy_ops tx3g0c55phy_ops = {
-	.start = sti_hdmi_tx3g0c55phy_start,
-	.stop = sti_hdmi_tx3g0c55phy_stop,
-};
diff --git a/drivers/gpu/drm/sti/sti_hdmi_tx3g0c55phy.h b/drivers/gpu/drm/sti/sti_hdmi_tx3g0c55phy.h
deleted file mode 100644
index 068237b3a303..000000000000
--- a/drivers/gpu/drm/sti/sti_hdmi_tx3g0c55phy.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (C) STMicroelectronics SA 2014
- * Author: Benjamin Gaignard <benjamin.gaignard@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
- */
-
-#ifndef _STI_HDMI_TX3G0C55PHY_H_
-#define _STI_HDMI_TX3G0C55PHY_H_
-
-#include "sti_hdmi.h"
-
-extern struct hdmi_phy_ops tx3g0c55phy_ops;
-
-#endif
diff --git a/drivers/gpu/drm/sti/sti_hqvdp.c b/drivers/gpu/drm/sti/sti_hqvdp.c
index b5ee783e3e7c..f88130f2eb48 100644
--- a/drivers/gpu/drm/sti/sti_hqvdp.c
+++ b/drivers/gpu/drm/sti/sti_hqvdp.c
@@ -17,6 +17,7 @@
 #include "sti_hqvdp_lut.h"
 #include "sti_plane.h"
 #include "sti_vtg.h"
+#include "sti_drv.h"
 
 /* Firmware name */
 #define HQVDP_FMW_NAME          "hqvdp-stih407.bin"
@@ -770,6 +771,7 @@ static void sti_hqvdp_disable(struct sti_hqvdp *hqvdp)
 		DRM_ERROR("XP70 could not revert to idle\n");
 
 	hqvdp->plane.status = STI_PLANE_DISABLED;
+	hqvdp->xp70_initialized = false;
 }
 
 /**
@@ -783,7 +785,7 @@ static void sti_hqvdp_disable(struct sti_hqvdp *hqvdp)
  * RETURNS:
  * 0 on success.
  */
-int sti_hqvdp_vtg_cb(struct notifier_block *nb, unsigned long evt, void *data)
+static int sti_hqvdp_vtg_cb(struct notifier_block *nb, unsigned long evt, void *data)
 {
 	struct sti_hqvdp *hqvdp = container_of(nb, struct sti_hqvdp, vtg_nb);
 	int btm_cmd_offset, top_cmd_offest;
@@ -1012,7 +1014,6 @@ static int sti_hqvdp_atomic_check(struct drm_plane *drm_plane,
 	struct sti_hqvdp *hqvdp = to_sti_hqvdp(plane);
 	struct drm_crtc *crtc = state->crtc;
 	struct drm_framebuffer *fb = state->fb;
-	bool first_prepare = plane->status == STI_PLANE_DISABLED ? true : false;
 	struct drm_crtc_state *crtc_state;
 	struct drm_display_mode *mode;
 	int dst_x, dst_y, dst_w, dst_h;
@@ -1026,8 +1027,8 @@ static int sti_hqvdp_atomic_check(struct drm_plane *drm_plane,
 	mode = &crtc_state->mode;
 	dst_x = state->crtc_x;
 	dst_y = state->crtc_y;
-	dst_w = clamp_val(state->crtc_w, 0, mode->crtc_hdisplay - dst_x);
-	dst_h = clamp_val(state->crtc_h, 0, mode->crtc_vdisplay - dst_y);
+	dst_w = clamp_val(state->crtc_w, 0, mode->hdisplay - dst_x);
+	dst_h = clamp_val(state->crtc_h, 0, mode->vdisplay - dst_y);
 	/* src_x are in 16.16 format */
 	src_x = state->src_x >> 16;
 	src_y = state->src_y >> 16;
@@ -1063,7 +1064,7 @@ static int sti_hqvdp_atomic_check(struct drm_plane *drm_plane,
 		return -EINVAL;
 	}
 
-	if (first_prepare) {
+	if (!hqvdp->xp70_initialized) {
 		/* Start HQVDP XP70 coprocessor */
 		sti_hqvdp_start_xp70(hqvdp);
 
@@ -1115,8 +1116,8 @@ static void sti_hqvdp_atomic_update(struct drm_plane *drm_plane,
 	mode = &crtc->mode;
 	dst_x = state->crtc_x;
 	dst_y = state->crtc_y;
-	dst_w = clamp_val(state->crtc_w, 0, mode->crtc_hdisplay - dst_x);
-	dst_h = clamp_val(state->crtc_h, 0, mode->crtc_vdisplay - dst_y);
+	dst_w = clamp_val(state->crtc_w, 0, mode->hdisplay - dst_x);
+	dst_h = clamp_val(state->crtc_h, 0, mode->vdisplay - dst_y);
 	/* src_x are in 16.16 format */
 	src_x = state->src_x >> 16;
 	src_y = state->src_y >> 16;
@@ -1214,15 +1215,15 @@ static void sti_hqvdp_atomic_disable(struct drm_plane *drm_plane,
 {
 	struct sti_plane *plane = to_sti_plane(drm_plane);
 
-	if (!drm_plane->crtc) {
+	if (!oldstate->crtc) {
 		DRM_DEBUG_DRIVER("drm plane:%d not enabled\n",
 				 drm_plane->base.id);
 		return;
 	}
 
 	DRM_DEBUG_DRIVER("CRTC:%d (%s) drm plane:%d (%s)\n",
-			 drm_plane->crtc->base.id,
-			 sti_mixer_to_str(to_sti_mixer(drm_plane->crtc)),
+			 oldstate->crtc->base.id,
+			 sti_mixer_to_str(to_sti_mixer(oldstate->crtc)),
 			 drm_plane->base.id, sti_plane_to_str(plane));
 
 	plane->status = STI_PLANE_DISABLING;
@@ -1250,7 +1251,7 @@ static int sti_hqvdp_late_register(struct drm_plane *drm_plane)
 	return hqvdp_debugfs_init(hqvdp, drm_plane->dev->primary);
 }
 
-struct drm_plane_funcs sti_hqvdp_plane_helpers_funcs = {
+static const struct drm_plane_funcs sti_hqvdp_plane_helpers_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = sti_hqvdp_destroy,
@@ -1289,7 +1290,7 @@ static struct drm_plane *sti_hqvdp_create(struct drm_device *drm_dev,
 	return &hqvdp->plane.drm_plane;
 }
 
-int sti_hqvdp_bind(struct device *dev, struct device *master, void *data)
+static int sti_hqvdp_bind(struct device *dev, struct device *master, void *data)
 {
 	struct sti_hqvdp *hqvdp = dev_get_drvdata(dev);
 	struct drm_device *drm_dev = data;
diff --git a/drivers/gpu/drm/sti/sti_mixer.c b/drivers/gpu/drm/sti/sti_mixer.c
index 7d9aea805eab..4ddc58f7fe2e 100644
--- a/drivers/gpu/drm/sti/sti_mixer.c
+++ b/drivers/gpu/drm/sti/sti_mixer.c
@@ -16,12 +16,6 @@ static unsigned int bkg_color = 0x000000;
 MODULE_PARM_DESC(bkgcolor, "Value of the background color 0xRRGGBB");
 module_param_named(bkgcolor, bkg_color, int, 0644);
 
-/* Identity: G=Y , B=Cb , R=Cr */
-static const u32 mixerColorSpaceMatIdentity[] = {
-	0x10000000, 0x00000000, 0x10000000, 0x00001000,
-	0x00000000, 0x00000000, 0x00000000, 0x00000000
-};
-
 /* regs offset */
 #define GAM_MIXER_CTL      0x00
 #define GAM_MIXER_BKC      0x04
@@ -358,22 +352,12 @@ int sti_mixer_set_plane_status(struct sti_mixer *mixer,
 	return 0;
 }
 
-void sti_mixer_set_matrix(struct sti_mixer *mixer)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(mixerColorSpaceMatIdentity); i++)
-		sti_mixer_reg_write(mixer, GAM_MIXER_MX0 + (i * 4),
-				    mixerColorSpaceMatIdentity[i]);
-}
-
 struct sti_mixer *sti_mixer_create(struct device *dev,
 				   struct drm_device *drm_dev,
 				   int id,
 				   void __iomem *baseaddr)
 {
 	struct sti_mixer *mixer = devm_kzalloc(dev, sizeof(*mixer), GFP_KERNEL);
-	struct device_node *np = dev->of_node;
 
 	dev_dbg(dev, "%s\n", __func__);
 	if (!mixer) {
@@ -384,9 +368,6 @@ struct sti_mixer *sti_mixer_create(struct device *dev,
 	mixer->dev = dev;
 	mixer->id = id;
 
-	if (of_device_is_compatible(np, "st,stih416-compositor"))
-		sti_mixer_set_matrix(mixer);
-
 	DRM_DEBUG_DRIVER("%s created. Regs=%p\n",
 			 sti_mixer_to_str(mixer), mixer->regs);
 
diff --git a/drivers/gpu/drm/sti/sti_tvout.c b/drivers/gpu/drm/sti/sti_tvout.c
index e25995b35715..ad46d3558d91 100644
--- a/drivers/gpu/drm/sti/sti_tvout.c
+++ b/drivers/gpu/drm/sti/sti_tvout.c
@@ -18,6 +18,7 @@
 #include <drm/drm_crtc_helper.h>
 
 #include "sti_crtc.h"
+#include "sti_drv.h"
 #include "sti_vtg.h"
 
 /* glue registers */
@@ -209,13 +210,11 @@ static void tvout_vip_set_rnd(struct sti_tvout *tvout, int reg, u32 rnd)
  * @tvout: tvout structure
  * @reg: register to set
  * @main_path: main or auxiliary path
- * @sel_input_logic_inverted: need to invert the logic
  * @sel_input: selected_input (main/aux + conv)
  */
 static void tvout_vip_set_sel_input(struct sti_tvout *tvout,
 				    int reg,
 				    bool main_path,
-				    bool sel_input_logic_inverted,
 				    enum sti_tvout_video_out_type video_out)
 {
 	u32 sel_input;
@@ -236,8 +235,7 @@ static void tvout_vip_set_sel_input(struct sti_tvout *tvout,
 	}
 
 	/* on stih407 chip the sel_input bypass mode logic is inverted */
-	if (sel_input_logic_inverted)
-		sel_input = sel_input ^ TVO_VIP_SEL_INPUT_BYPASS_MASK;
+	sel_input = sel_input ^ TVO_VIP_SEL_INPUT_BYPASS_MASK;
 
 	val &= ~TVO_VIP_SEL_INPUT_MASK;
 	val |= sel_input;
@@ -295,8 +293,6 @@ static void tvout_preformatter_set_matrix(struct sti_tvout *tvout,
  */
 static void tvout_dvo_start(struct sti_tvout *tvout, bool main_path)
 {
-	struct device_node *node = tvout->dev->of_node;
-	bool sel_input_logic_inverted = false;
 	u32 tvo_in_vid_format;
 	int val, tmp;
 
@@ -334,16 +330,11 @@ static void tvout_dvo_start(struct sti_tvout *tvout, bool main_path)
 	/* Set round mode (rounded to 8-bit per component) */
 	tvout_vip_set_rnd(tvout, TVO_VIP_DVO, TVO_VIP_RND_8BIT_ROUNDED);
 
-	if (of_device_is_compatible(node, "st,stih407-tvout")) {
-		/* Set input video format */
-		tvout_vip_set_in_vid_fmt(tvout, tvo_in_vid_format,
-					 TVO_IN_FMT_SIGNED);
-		sel_input_logic_inverted = true;
-	}
+	/* Set input video format */
+	tvout_vip_set_in_vid_fmt(tvout, tvo_in_vid_format, TVO_IN_FMT_SIGNED);
 
 	/* Input selection */
 	tvout_vip_set_sel_input(tvout, TVO_VIP_DVO, main_path,
-				sel_input_logic_inverted,
 				STI_TVOUT_VIDEO_OUT_RGB);
 }
 
@@ -356,8 +347,6 @@ static void tvout_dvo_start(struct sti_tvout *tvout, bool main_path)
  */
 static void tvout_hdmi_start(struct sti_tvout *tvout, bool main_path)
 {
-	struct device_node *node = tvout->dev->of_node;
-	bool sel_input_logic_inverted = false;
 	u32 tvo_in_vid_format;
 
 	dev_dbg(tvout->dev, "%s\n", __func__);
@@ -390,16 +379,12 @@ static void tvout_hdmi_start(struct sti_tvout *tvout, bool main_path)
 	/* set round mode (rounded to 8-bit per component) */
 	tvout_vip_set_rnd(tvout, TVO_VIP_HDMI, TVO_VIP_RND_8BIT_ROUNDED);
 
-	if (of_device_is_compatible(node, "st,stih407-tvout")) {
-		/* set input video format */
-		tvout_vip_set_in_vid_fmt(tvout, tvo_in_vid_format,
-					TVO_IN_FMT_SIGNED);
-		sel_input_logic_inverted = true;
-	}
+	/* set input video format */
+	tvout_vip_set_in_vid_fmt(tvout, tvo_in_vid_format, TVO_IN_FMT_SIGNED);
 
 	/* input selection */
 	tvout_vip_set_sel_input(tvout, TVO_VIP_HDMI, main_path,
-			sel_input_logic_inverted, STI_TVOUT_VIDEO_OUT_RGB);
+				STI_TVOUT_VIDEO_OUT_RGB);
 }
 
 /**
@@ -411,8 +396,6 @@ static void tvout_hdmi_start(struct sti_tvout *tvout, bool main_path)
  */
 static void tvout_hda_start(struct sti_tvout *tvout, bool main_path)
 {
-	struct device_node *node = tvout->dev->of_node;
-	bool sel_input_logic_inverted = false;
 	u32 tvo_in_vid_format;
 	int val;
 
@@ -448,16 +431,11 @@ static void tvout_hda_start(struct sti_tvout *tvout, bool main_path)
 	/* set round mode (rounded to 10-bit per component) */
 	tvout_vip_set_rnd(tvout, TVO_VIP_HDF, TVO_VIP_RND_10BIT_ROUNDED);
 
-	if (of_device_is_compatible(node, "st,stih407-tvout")) {
-		/* set input video format */
-		tvout_vip_set_in_vid_fmt(tvout,
-			tvo_in_vid_format, TVO_IN_FMT_SIGNED);
-		sel_input_logic_inverted = true;
-	}
+	/* Set input video format */
+	tvout_vip_set_in_vid_fmt(tvout, tvo_in_vid_format, TVO_IN_FMT_SIGNED);
 
 	/* Input selection */
 	tvout_vip_set_sel_input(tvout, TVO_VIP_HDF, main_path,
-				sel_input_logic_inverted,
 				STI_TVOUT_VIDEO_OUT_YUV);
 
 	/* power up HD DAC */
@@ -905,7 +883,6 @@ static int sti_tvout_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id tvout_of_match[] = {
-	{ .compatible = "st,stih416-tvout", },
 	{ .compatible = "st,stih407-tvout", },
 	{ /* end node */ }
 };
diff --git a/drivers/gpu/drm/sti/sti_vid.c b/drivers/gpu/drm/sti/sti_vid.c
index 47634a0251fc..2ad59892b57e 100644
--- a/drivers/gpu/drm/sti/sti_vid.c
+++ b/drivers/gpu/drm/sti/sti_vid.c
@@ -142,8 +142,8 @@ void sti_vid_commit(struct sti_vid *vid,
 	struct drm_display_mode *mode = &crtc->mode;
 	int dst_x = state->crtc_x;
 	int dst_y = state->crtc_y;
-	int dst_w = clamp_val(state->crtc_w, 0, mode->crtc_hdisplay - dst_x);
-	int dst_h = clamp_val(state->crtc_h, 0, mode->crtc_vdisplay - dst_y);
+	int dst_w = clamp_val(state->crtc_w, 0, mode->hdisplay - dst_x);
+	int dst_h = clamp_val(state->crtc_h, 0, mode->vdisplay - dst_y);
 	int src_h = state->src_h >> 16;
 	u32 val, ydo, xdo, yds, xds;
 
diff --git a/drivers/gpu/drm/sti/sti_vtac.c b/drivers/gpu/drm/sti/sti_vtac.c
index b1eb0d77630d..cf7fe8a1db42 100644
--- a/drivers/gpu/drm/sti/sti_vtac.c
+++ b/drivers/gpu/drm/sti/sti_vtac.c
@@ -12,6 +12,8 @@
 
 #include <drm/drmP.h>
 
+#include "sti_drv.h"
+
 /* registers offset */
 #define VTAC_CONFIG                     0x00
 #define VTAC_RX_FIFO_CONFIG             0x04
diff --git a/drivers/gpu/drm/sti/sti_vtg.c b/drivers/gpu/drm/sti/sti_vtg.c
index 0bdc385eec17..a8882bdd0f8b 100644
--- a/drivers/gpu/drm/sti/sti_vtg.c
+++ b/drivers/gpu/drm/sti/sti_vtg.c
@@ -13,6 +13,7 @@
 
 #include <drm/drmP.h>
 
+#include "sti_drv.h"
 #include "sti_vtg.h"
 
 #define VTG_MODE_MASTER         0
@@ -72,7 +73,7 @@
 #define AWG_DELAY_ED        (-8)
 #define AWG_DELAY_SD        (-7)
 
-LIST_HEAD(vtg_lookup);
+static LIST_HEAD(vtg_lookup);
 
 /*
  * STI VTG register offset structure
diff --git a/drivers/gpu/drm/sun4i/Makefile b/drivers/gpu/drm/sun4i/Makefile
index 58cd55149827..d625a82a6e5f 100644
--- a/drivers/gpu/drm/sun4i/Makefile
+++ b/drivers/gpu/drm/sun4i/Makefile
@@ -9,5 +9,5 @@ sun4i-tcon-y += sun4i_dotclock.o
 
 obj-$(CONFIG_DRM_SUN4I)		+= sun4i-drm.o sun4i-tcon.o
 obj-$(CONFIG_DRM_SUN4I)		+= sun4i_backend.o
-
+obj-$(CONFIG_DRM_SUN4I)		+= sun6i_drc.o
 obj-$(CONFIG_DRM_SUN4I)		+= sun4i_tv.o
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index 3ab560450a82..32c0584e3c35 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -83,8 +83,13 @@ void sun4i_backend_layer_enable(struct sun4i_backend *backend,
 }
 EXPORT_SYMBOL(sun4i_backend_layer_enable);
 
-static int sun4i_backend_drm_format_to_layer(u32 format, u32 *mode)
+static int sun4i_backend_drm_format_to_layer(struct drm_plane *plane,
+					     u32 format, u32 *mode)
 {
+	if ((plane->type == DRM_PLANE_TYPE_PRIMARY) &&
+	    (format == DRM_FORMAT_ARGB8888))
+		format = DRM_FORMAT_XRGB8888;
+
 	switch (format) {
 	case DRM_FORMAT_ARGB8888:
 		*mode = SUN4I_BACKEND_LAY_FBFMT_ARGB8888;
@@ -164,7 +169,7 @@ int sun4i_backend_update_layer_formats(struct sun4i_backend *backend,
 	DRM_DEBUG_DRIVER("Switching display backend interlaced mode %s\n",
 			 interlaced ? "on" : "off");
 
-	ret = sun4i_backend_drm_format_to_layer(fb->pixel_format, &val);
+	ret = sun4i_backend_drm_format_to_layer(plane, fb->pixel_format, &val);
 	if (ret) {
 		DRM_DEBUG_DRIVER("Invalid format\n");
 		return val;
@@ -217,6 +222,51 @@ int sun4i_backend_update_layer_buffer(struct sun4i_backend *backend,
 }
 EXPORT_SYMBOL(sun4i_backend_update_layer_buffer);
 
+static int sun4i_backend_init_sat(struct device *dev) {
+	struct sun4i_backend *backend = dev_get_drvdata(dev);
+	int ret;
+
+	backend->sat_reset = devm_reset_control_get(dev, "sat");
+	if (IS_ERR(backend->sat_reset)) {
+		dev_err(dev, "Couldn't get the SAT reset line\n");
+		return PTR_ERR(backend->sat_reset);
+	}
+
+	ret = reset_control_deassert(backend->sat_reset);
+	if (ret) {
+		dev_err(dev, "Couldn't deassert the SAT reset line\n");
+		return ret;
+	}
+
+	backend->sat_clk = devm_clk_get(dev, "sat");
+	if (IS_ERR(backend->sat_clk)) {
+		dev_err(dev, "Couldn't get our SAT clock\n");
+		ret = PTR_ERR(backend->sat_clk);
+		goto err_assert_reset;
+	}
+
+	ret = clk_prepare_enable(backend->sat_clk);
+	if (ret) {
+		dev_err(dev, "Couldn't enable the SAT clock\n");
+		return ret;
+	}
+
+	return 0;
+
+err_assert_reset:
+	reset_control_assert(backend->sat_reset);
+	return ret;
+}
+
+static int sun4i_backend_free_sat(struct device *dev) {
+	struct sun4i_backend *backend = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(backend->sat_clk);
+	reset_control_assert(backend->sat_reset);
+
+	return 0;
+}
+
 static struct regmap_config sun4i_backend_regmap_config = {
 	.reg_bits	= 32,
 	.val_bits	= 32,
@@ -243,10 +293,8 @@ static int sun4i_backend_bind(struct device *dev, struct device *master,
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	regs = devm_ioremap_resource(dev, res);
-	if (IS_ERR(regs)) {
-		dev_err(dev, "Couldn't map the backend registers\n");
+	if (IS_ERR(regs))
 		return PTR_ERR(regs);
-	}
 
 	backend->regs = devm_regmap_init_mmio(dev, regs,
 					      &sun4i_backend_regmap_config);
@@ -291,6 +339,15 @@ static int sun4i_backend_bind(struct device *dev, struct device *master,
 	}
 	clk_prepare_enable(backend->ram_clk);
 
+	if (of_device_is_compatible(dev->of_node,
+				    "allwinner,sun8i-a33-display-backend")) {
+		ret = sun4i_backend_init_sat(dev);
+		if (ret) {
+			dev_err(dev, "Couldn't init SAT resources\n");
+			goto err_disable_ram_clk;
+		}
+	}
+
 	/* Reset the registers */
 	for (i = 0x800; i < 0x1000; i += 4)
 		regmap_write(backend->regs, i, 0);
@@ -306,6 +363,8 @@ static int sun4i_backend_bind(struct device *dev, struct device *master,
 
 	return 0;
 
+err_disable_ram_clk:
+	clk_disable_unprepare(backend->ram_clk);
 err_disable_mod_clk:
 	clk_disable_unprepare(backend->mod_clk);
 err_disable_bus_clk:
@@ -320,6 +379,10 @@ static void sun4i_backend_unbind(struct device *dev, struct device *master,
 {
 	struct sun4i_backend *backend = dev_get_drvdata(dev);
 
+	if (of_device_is_compatible(dev->of_node,
+				    "allwinner,sun8i-a33-display-backend"))
+		sun4i_backend_free_sat(dev);
+
 	clk_disable_unprepare(backend->ram_clk);
 	clk_disable_unprepare(backend->mod_clk);
 	clk_disable_unprepare(backend->bus_clk);
@@ -345,6 +408,7 @@ static int sun4i_backend_remove(struct platform_device *pdev)
 
 static const struct of_device_id sun4i_backend_of_table[] = {
 	{ .compatible = "allwinner,sun5i-a13-display-backend" },
+	{ .compatible = "allwinner,sun8i-a33-display-backend" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sun4i_backend_of_table);
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.h b/drivers/gpu/drm/sun4i/sun4i_backend.h
index 7070bb3434e5..83e63cc702b4 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.h
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.h
@@ -52,8 +52,8 @@
 #define SUN4I_BACKEND_LAYFB_L32ADD_REG(l)	(0x850 + (0x4 * (l)))
 
 #define SUN4I_BACKEND_LAYFB_H4ADD_REG		0x860
-#define SUN4I_BACKEND_LAYFB_H4ADD_MSK(l)		GENMASK(3 + ((l) * 8), 0)
-#define SUN4I_BACKEND_LAYFB_H4ADD(l, val)			((val) << ((l) * 8))
+#define SUN4I_BACKEND_LAYFB_H4ADD_MSK(l)		GENMASK(3 + ((l) * 8), (l) * 8)
+#define SUN4I_BACKEND_LAYFB_H4ADD(l, val)		((val) << ((l) * 8))
 
 #define SUN4I_BACKEND_REGBUFFCTL_REG		0x870
 #define SUN4I_BACKEND_REGBUFFCTL_AUTOLOAD_DIS		BIT(1)
@@ -146,6 +146,9 @@ struct sun4i_backend {
 	struct clk		*bus_clk;
 	struct clk		*mod_clk;
 	struct clk		*ram_clk;
+
+	struct clk		*sat_clk;
+	struct reset_control	*sat_reset;
 };
 
 void sun4i_backend_apply_color_correction(struct sun4i_backend *backend);
diff --git a/drivers/gpu/drm/sun4i/sun4i_dotclock.c b/drivers/gpu/drm/sun4i/sun4i_dotclock.c
index 5b3463197c48..d401156490f3 100644
--- a/drivers/gpu/drm/sun4i/sun4i_dotclock.c
+++ b/drivers/gpu/drm/sun4i/sun4i_dotclock.c
@@ -14,6 +14,7 @@
 #include <linux/regmap.h>
 
 #include "sun4i_tcon.h"
+#include "sun4i_dotclock.h"
 
 struct sun4i_dclk {
 	struct clk_hw	hw;
@@ -61,7 +62,7 @@ static unsigned long sun4i_dclk_recalc_rate(struct clk_hw *hw,
 	regmap_read(dclk->regmap, SUN4I_TCON0_DCLK_REG, &val);
 
 	val >>= SUN4I_TCON0_DCLK_DIV_SHIFT;
-	val &= SUN4I_TCON0_DCLK_DIV_WIDTH;
+	val &= (1 << SUN4I_TCON0_DCLK_DIV_WIDTH) - 1;
 
 	if (!val)
 		val = 1;
@@ -76,7 +77,7 @@ static long sun4i_dclk_round_rate(struct clk_hw *hw, unsigned long rate,
 	u8 best_div = 1;
 	int i;
 
-	for (i = 6; i < 127; i++) {
+	for (i = 6; i <= 127; i++) {
 		unsigned long ideal = rate * i;
 		unsigned long rounded;
 
@@ -89,7 +90,8 @@ static long sun4i_dclk_round_rate(struct clk_hw *hw, unsigned long rate,
 			goto out;
 		}
 
-		if ((rounded < ideal) && (rounded > best_parent)) {
+		if (abs(rate - rounded / i) <
+		    abs(rate - best_parent / best_div)) {
 			best_parent = rounded;
 			best_div = i;
 		}
diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c
index 7092daaf6c43..0da9862ad8ed 100644
--- a/drivers/gpu/drm/sun4i/sun4i_drv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_drv.c
@@ -17,6 +17,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_fb_helper.h>
 
 #include "sun4i_crtc.h"
 #include "sun4i_drv.h"
@@ -109,7 +110,7 @@ static void sun4i_remove_framebuffers(void)
 	ap->ranges[0].base = 0;
 	ap->ranges[0].size = ~0;
 
-	remove_conflicting_framebuffers(ap, "sun4i-drm-fb", false);
+	drm_fb_helper_remove_conflicting_framebuffers(ap, "sun4i-drm-fb", false);
 	kfree(ap);
 }
 
@@ -120,8 +121,8 @@ static int sun4i_drv_bind(struct device *dev)
 	int ret;
 
 	drm = drm_dev_alloc(&sun4i_drv_driver, dev);
-	if (!drm)
-		return -ENOMEM;
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
 
 	drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);
 	if (!drv) {
@@ -199,13 +200,14 @@ static const struct component_master_ops sun4i_drv_master_ops = {
 
 static bool sun4i_drv_node_is_frontend(struct device_node *node)
 {
-	return of_device_is_compatible(node,
-				       "allwinner,sun5i-a13-display-frontend");
+	return of_device_is_compatible(node, "allwinner,sun5i-a13-display-frontend") ||
+		of_device_is_compatible(node, "allwinner,sun8i-a33-display-frontend");
 }
 
 static bool sun4i_drv_node_is_tcon(struct device_node *node)
 {
-	return of_device_is_compatible(node, "allwinner,sun5i-a13-tcon");
+	return of_device_is_compatible(node, "allwinner,sun5i-a13-tcon") ||
+		of_device_is_compatible(node, "allwinner,sun8i-a33-tcon");
 }
 
 static int compare_of(struct device *dev, void *data)
@@ -257,8 +259,8 @@ static int sun4i_drv_add_endpoints(struct device *dev,
 		}
 
 		/*
-		 * If the node is our TCON, the first port is used for our
-		 * panel, and will not be part of the
+		 * If the node is our TCON, the first port is used for
+		 * panel or bridges, and will not be part of the
 		 * component framework.
 		 */
 		if (sun4i_drv_node_is_tcon(node)) {
@@ -320,6 +322,7 @@ static int sun4i_drv_remove(struct platform_device *pdev)
 
 static const struct of_device_id sun4i_drv_of_table[] = {
 	{ .compatible = "allwinner,sun5i-a13-display-engine" },
+	{ .compatible = "allwinner,sun8i-a33-display-engine" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sun4i_drv_of_table);
diff --git a/drivers/gpu/drm/sun4i/sun4i_framebuffer.c b/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
index 70688febd7ac..8b6ce619ad81 100644
--- a/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
@@ -15,6 +15,7 @@
 #include <drm/drmP.h>
 
 #include "sun4i_drv.h"
+#include "sun4i_framebuffer.h"
 
 static void sun4i_de_output_poll_changed(struct drm_device *drm)
 {
diff --git a/drivers/gpu/drm/sun4i/sun4i_layer.c b/drivers/gpu/drm/sun4i/sun4i_layer.c
index 068ab806309b..f0035bf5efea 100644
--- a/drivers/gpu/drm/sun4i/sun4i_layer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_layer.c
@@ -19,7 +19,12 @@
 #include "sun4i_drv.h"
 #include "sun4i_layer.h"
 
-#define SUN4I_NUM_LAYERS	2
+struct sun4i_plane_desc {
+	       enum drm_plane_type     type;
+	       u8                      pipe;
+	       const uint32_t          *formats;
+	       uint32_t                nformats;
+};
 
 static int sun4i_backend_layer_atomic_check(struct drm_plane *plane,
 					    struct drm_plane_state *state)
@@ -65,14 +70,35 @@ static const struct drm_plane_funcs sun4i_backend_layer_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 };
 
-static const uint32_t sun4i_backend_layer_formats[] = {
+static const uint32_t sun4i_backend_layer_formats_primary[] = {
 	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_RGB888,
 	DRM_FORMAT_XRGB8888,
+};
+
+static const uint32_t sun4i_backend_layer_formats_overlay[] = {
+	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_RGB888,
+	DRM_FORMAT_XRGB8888,
+};
+
+static const struct sun4i_plane_desc sun4i_backend_planes[] = {
+	{
+		.type = DRM_PLANE_TYPE_PRIMARY,
+		.pipe = 0,
+		.formats = sun4i_backend_layer_formats_primary,
+		.nformats = ARRAY_SIZE(sun4i_backend_layer_formats_primary),
+	},
+	{
+		.type = DRM_PLANE_TYPE_OVERLAY,
+		.pipe = 1,
+		.formats = sun4i_backend_layer_formats_overlay,
+		.nformats = ARRAY_SIZE(sun4i_backend_layer_formats_overlay),
+	},
 };
 
 static struct sun4i_layer *sun4i_layer_init_one(struct drm_device *drm,
-						enum drm_plane_type type)
+						const struct sun4i_plane_desc *plane)
 {
 	struct sun4i_drv *drv = drm->dev_private;
 	struct sun4i_layer *layer;
@@ -84,10 +110,8 @@ static struct sun4i_layer *sun4i_layer_init_one(struct drm_device *drm,
 
 	ret = drm_universal_plane_init(drm, &layer->plane, BIT(0),
 				       &sun4i_backend_layer_funcs,
-				       sun4i_backend_layer_formats,
-				       ARRAY_SIZE(sun4i_backend_layer_formats),
-				       type,
-				       NULL);
+				       plane->formats, plane->nformats,
+				       plane->type, NULL);
 	if (ret) {
 		dev_err(drm->dev, "Couldn't initialize layer\n");
 		return ERR_PTR(ret);
@@ -97,7 +121,7 @@ static struct sun4i_layer *sun4i_layer_init_one(struct drm_device *drm,
 			     &sun4i_backend_layer_helper_funcs);
 	layer->drv = drv;
 
-	if (type == DRM_PLANE_TYPE_PRIMARY)
+	if (plane->type == DRM_PLANE_TYPE_PRIMARY)
 		drv->primary = &layer->plane;
 
 	return layer;
@@ -109,8 +133,8 @@ struct sun4i_layer **sun4i_layers_init(struct drm_device *drm)
 	struct sun4i_layer **layers;
 	int i;
 
-	layers = devm_kcalloc(drm->dev, SUN4I_NUM_LAYERS, sizeof(**layers),
-			      GFP_KERNEL);
+	layers = devm_kcalloc(drm->dev, ARRAY_SIZE(sun4i_backend_planes),
+			      sizeof(**layers), GFP_KERNEL);
 	if (!layers)
 		return ERR_PTR(-ENOMEM);
 
@@ -135,13 +159,11 @@ struct sun4i_layer **sun4i_layers_init(struct drm_device *drm)
 	 * SoCs that support it, sprites could fill the need for more
 	 * layers.
 	 */
-	for (i = 0; i < SUN4I_NUM_LAYERS; i++) {
-		enum drm_plane_type type = (i == 0)
-					 ? DRM_PLANE_TYPE_PRIMARY
-					 : DRM_PLANE_TYPE_OVERLAY;
+	for (i = 0; i < ARRAY_SIZE(sun4i_backend_planes); i++) {
+		const struct sun4i_plane_desc *plane = &sun4i_backend_planes[i];
 		struct sun4i_layer *layer = layers[i];
 
-		layer = sun4i_layer_init_one(drm, type);
+		layer = sun4i_layer_init_one(drm, plane);
 		if (IS_ERR(layer)) {
 			dev_err(drm->dev, "Couldn't initialize %s plane\n",
 				i ? "overlay" : "primary");
@@ -149,10 +171,10 @@ struct sun4i_layer **sun4i_layers_init(struct drm_device *drm)
 		};
 
 		DRM_DEBUG_DRIVER("Assigning %s plane to pipe %d\n",
-				 i ? "overlay" : "primary", i);
+				 i ? "overlay" : "primary", plane->pipe);
 		regmap_update_bits(drv->backend->regs, SUN4I_BACKEND_ATTCTL_REG0(i),
 				   SUN4I_BACKEND_ATTCTL_REG0_LAY_PIPESEL_MASK,
-				   SUN4I_BACKEND_ATTCTL_REG0_LAY_PIPESEL(i));
+				   SUN4I_BACKEND_ATTCTL_REG0_LAY_PIPESEL(plane->pipe));
 
 		layer->id = i;
 	};
diff --git a/drivers/gpu/drm/sun4i/sun4i_rgb.c b/drivers/gpu/drm/sun4i/sun4i_rgb.c
index f5bbac6efb4c..c3ff10f559cc 100644
--- a/drivers/gpu/drm/sun4i/sun4i_rgb.c
+++ b/drivers/gpu/drm/sun4i/sun4i_rgb.c
@@ -19,6 +19,7 @@
 
 #include "sun4i_drv.h"
 #include "sun4i_tcon.h"
+#include "sun4i_rgb.h"
 
 struct sun4i_rgb {
 	struct drm_connector	connector;
@@ -151,7 +152,14 @@ static void sun4i_rgb_encoder_enable(struct drm_encoder *encoder)
 
 	DRM_DEBUG_DRIVER("Enabling RGB output\n");
 
-	drm_panel_enable(tcon->panel);
+	if (!IS_ERR(tcon->panel)) {
+		drm_panel_prepare(tcon->panel);
+		drm_panel_enable(tcon->panel);
+	}
+
+	/* encoder->bridge can be NULL; drm_bridge_enable checks for it */
+	drm_bridge_enable(encoder->bridge);
+
 	sun4i_tcon_channel_enable(tcon, 0);
 }
 
@@ -164,7 +172,14 @@ static void sun4i_rgb_encoder_disable(struct drm_encoder *encoder)
 	DRM_DEBUG_DRIVER("Disabling RGB output\n");
 
 	sun4i_tcon_channel_disable(tcon, 0);
-	drm_panel_disable(tcon->panel);
+
+	/* encoder->bridge can be NULL; drm_bridge_disable checks for it */
+	drm_bridge_disable(encoder->bridge);
+
+	if (!IS_ERR(tcon->panel)) {
+		drm_panel_disable(tcon->panel);
+		drm_panel_unprepare(tcon->panel);
+	}
 }
 
 static void sun4i_rgb_encoder_mode_set(struct drm_encoder *encoder,
@@ -203,17 +218,22 @@ int sun4i_rgb_init(struct drm_device *drm)
 {
 	struct sun4i_drv *drv = drm->dev_private;
 	struct sun4i_tcon *tcon = drv->tcon;
+	struct drm_encoder *encoder;
 	struct sun4i_rgb *rgb;
 	int ret;
 
-	/* If we don't have a panel, there's no point in going on */
-	if (IS_ERR(tcon->panel))
-		return -ENODEV;
-
 	rgb = devm_kzalloc(drm->dev, sizeof(*rgb), GFP_KERNEL);
 	if (!rgb)
 		return -ENOMEM;
 	rgb->drv = drv;
+	encoder = &rgb->encoder;
+
+	tcon->panel = sun4i_tcon_find_panel(tcon->dev->of_node);
+	encoder->bridge = sun4i_tcon_find_bridge(tcon->dev->of_node);
+	if (IS_ERR(tcon->panel) && IS_ERR(encoder->bridge)) {
+		dev_info(drm->dev, "No panel or bridge found... RGB output disabled\n");
+		return 0;
+	}
 
 	drm_encoder_helper_add(&rgb->encoder,
 			       &sun4i_rgb_enc_helper_funcs);
@@ -230,19 +250,38 @@ int sun4i_rgb_init(struct drm_device *drm)
 	/* The RGB encoder can only work with the TCON channel 0 */
 	rgb->encoder.possible_crtcs = BIT(0);
 
-	drm_connector_helper_add(&rgb->connector,
-				 &sun4i_rgb_con_helper_funcs);
-	ret = drm_connector_init(drm, &rgb->connector,
-				 &sun4i_rgb_con_funcs,
-				 DRM_MODE_CONNECTOR_Unknown);
-	if (ret) {
-		dev_err(drm->dev, "Couldn't initialise the rgb connector\n");
-		goto err_cleanup_connector;
+	if (!IS_ERR(tcon->panel)) {
+		drm_connector_helper_add(&rgb->connector,
+					 &sun4i_rgb_con_helper_funcs);
+		ret = drm_connector_init(drm, &rgb->connector,
+					 &sun4i_rgb_con_funcs,
+					 DRM_MODE_CONNECTOR_Unknown);
+		if (ret) {
+			dev_err(drm->dev, "Couldn't initialise the rgb connector\n");
+			goto err_cleanup_connector;
+		}
+
+		drm_mode_connector_attach_encoder(&rgb->connector,
+						  &rgb->encoder);
+
+		ret = drm_panel_attach(tcon->panel, &rgb->connector);
+		if (ret) {
+			dev_err(drm->dev, "Couldn't attach our panel\n");
+			goto err_cleanup_connector;
+		}
 	}
 
-	drm_mode_connector_attach_encoder(&rgb->connector, &rgb->encoder);
+	if (!IS_ERR(encoder->bridge)) {
+		encoder->bridge->encoder = &rgb->encoder;
 
-	drm_panel_attach(tcon->panel, &rgb->connector);
+		ret = drm_bridge_attach(drm, encoder->bridge);
+		if (ret) {
+			dev_err(drm->dev, "Couldn't attach our bridge\n");
+			goto err_cleanup_connector;
+		}
+	} else {
+		encoder->bridge = NULL;
+	}
 
 	return 0;
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c
index 652385f09735..cadacb517f95 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c
@@ -59,11 +59,13 @@ void sun4i_tcon_channel_disable(struct sun4i_tcon *tcon, int channel)
 		regmap_update_bits(tcon->regs, SUN4I_TCON0_CTL_REG,
 				   SUN4I_TCON0_CTL_TCON_ENABLE, 0);
 		clk_disable_unprepare(tcon->dclk);
-	} else if (channel == 1) {
-		regmap_update_bits(tcon->regs, SUN4I_TCON1_CTL_REG,
-				   SUN4I_TCON1_CTL_TCON_ENABLE, 0);
-		clk_disable_unprepare(tcon->sclk1);
+		return;
 	}
+
+	WARN_ON(!tcon->has_channel_1);
+	regmap_update_bits(tcon->regs, SUN4I_TCON1_CTL_REG,
+			   SUN4I_TCON1_CTL_TCON_ENABLE, 0);
+	clk_disable_unprepare(tcon->sclk1);
 }
 EXPORT_SYMBOL(sun4i_tcon_channel_disable);
 
@@ -75,12 +77,14 @@ void sun4i_tcon_channel_enable(struct sun4i_tcon *tcon, int channel)
 				   SUN4I_TCON0_CTL_TCON_ENABLE,
 				   SUN4I_TCON0_CTL_TCON_ENABLE);
 		clk_prepare_enable(tcon->dclk);
-	} else if (channel == 1) {
-		regmap_update_bits(tcon->regs, SUN4I_TCON1_CTL_REG,
-				   SUN4I_TCON1_CTL_TCON_ENABLE,
-				   SUN4I_TCON1_CTL_TCON_ENABLE);
-		clk_prepare_enable(tcon->sclk1);
+		return;
 	}
+
+	WARN_ON(!tcon->has_channel_1);
+	regmap_update_bits(tcon->regs, SUN4I_TCON1_CTL_REG,
+			   SUN4I_TCON1_CTL_TCON_ENABLE,
+			   SUN4I_TCON1_CTL_TCON_ENABLE);
+	clk_prepare_enable(tcon->sclk1);
 }
 EXPORT_SYMBOL(sun4i_tcon_channel_enable);
 
@@ -198,6 +202,8 @@ void sun4i_tcon1_mode_set(struct sun4i_tcon *tcon,
 	u8 clk_delay;
 	u32 val;
 
+	WARN_ON(!tcon->has_channel_1);
+
 	/* Adjust clock delay */
 	clk_delay = sun4i_tcon_get_clk_delay(mode, 1);
 	regmap_update_bits(tcon->regs, SUN4I_TCON1_CTL_REG,
@@ -321,10 +327,12 @@ static int sun4i_tcon_init_clocks(struct device *dev,
 		return PTR_ERR(tcon->sclk0);
 	}
 
-	tcon->sclk1 = devm_clk_get(dev, "tcon-ch1");
-	if (IS_ERR(tcon->sclk1)) {
-		dev_err(dev, "Couldn't get the TCON channel 1 clock\n");
-		return PTR_ERR(tcon->sclk1);
+	if (tcon->has_channel_1) {
+		tcon->sclk1 = devm_clk_get(dev, "tcon-ch1");
+		if (IS_ERR(tcon->sclk1)) {
+			dev_err(dev, "Couldn't get the TCON channel 1 clock\n");
+			return PTR_ERR(tcon->sclk1);
+		}
 	}
 
 	return sun4i_dclk_create(dev, tcon);
@@ -374,10 +382,8 @@ static int sun4i_tcon_init_regmap(struct device *dev,
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	regs = devm_ioremap_resource(dev, res);
-	if (IS_ERR(regs)) {
-		dev_err(dev, "Couldn't map the TCON registers\n");
+	if (IS_ERR(regs))
 		return PTR_ERR(regs);
-	}
 
 	tcon->regs = devm_regmap_init_mmio(dev, regs,
 					   &sun4i_tcon_regmap_config);
@@ -398,7 +404,7 @@ static int sun4i_tcon_init_regmap(struct device *dev,
 	return 0;
 }
 
-static struct drm_panel *sun4i_tcon_find_panel(struct device_node *node)
+struct drm_panel *sun4i_tcon_find_panel(struct device_node *node)
 {
 	struct device_node *port, *remote, *child;
 	struct device_node *end_node = NULL;
@@ -432,6 +438,40 @@ static struct drm_panel *sun4i_tcon_find_panel(struct device_node *node)
 	return of_drm_find_panel(remote) ?: ERR_PTR(-EPROBE_DEFER);
 }
 
+struct drm_bridge *sun4i_tcon_find_bridge(struct device_node *node)
+{
+	struct device_node *port, *remote, *child;
+	struct device_node *end_node = NULL;
+
+	/* Inputs are listed first, then outputs */
+	port = of_graph_get_port_by_id(node, 1);
+
+	/*
+	 * Our first output is the RGB interface where the panel will
+	 * be connected.
+	 */
+	for_each_child_of_node(port, child) {
+		u32 reg;
+
+		of_property_read_u32(child, "reg", &reg);
+		if (reg == 0)
+			end_node = child;
+	}
+
+	if (!end_node) {
+		DRM_DEBUG_DRIVER("Missing bridge endpoint\n");
+		return ERR_PTR(-ENODEV);
+	}
+
+	remote = of_graph_get_remote_port_parent(end_node);
+	if (!remote) {
+		DRM_DEBUG_DRIVER("Enable to parse remote node\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	return of_drm_find_bridge(remote) ?: ERR_PTR(-EPROBE_DEFER);
+}
+
 static int sun4i_tcon_bind(struct device *dev, struct device *master,
 			   void *data)
 {
@@ -446,9 +486,15 @@ static int sun4i_tcon_bind(struct device *dev, struct device *master,
 	dev_set_drvdata(dev, tcon);
 	drv->tcon = tcon;
 	tcon->drm = drm;
+	tcon->dev = dev;
 
-	if (of_device_is_compatible(dev->of_node, "allwinner,sun5i-a13-tcon"))
+	if (of_device_is_compatible(dev->of_node, "allwinner,sun5i-a13-tcon")) {
 		tcon->has_mux = true;
+		tcon->has_channel_1 = true;
+	} else {
+		tcon->has_mux = false;
+		tcon->has_channel_1 = false;
+	}
 
 	tcon->lcd_rst = devm_reset_control_get(dev, "lcd");
 	if (IS_ERR(tcon->lcd_rst)) {
@@ -484,12 +530,6 @@ static int sun4i_tcon_bind(struct device *dev, struct device *master,
 		goto err_free_clocks;
 	}
 
-	tcon->panel = sun4i_tcon_find_panel(dev->of_node);
-	if (IS_ERR(tcon->panel)) {
-		dev_info(dev, "No panel found... RGB output disabled\n");
-		return 0;
-	}
-
 	ret = sun4i_rgb_init(drm);
 	if (ret < 0)
 		goto err_free_clocks;
@@ -519,19 +559,22 @@ static struct component_ops sun4i_tcon_ops = {
 static int sun4i_tcon_probe(struct platform_device *pdev)
 {
 	struct device_node *node = pdev->dev.of_node;
+	struct drm_bridge *bridge;
 	struct drm_panel *panel;
 
 	/*
-	 * The panel is not ready.
+	 * Neither the bridge or the panel is ready.
 	 * Defer the probe.
 	 */
 	panel = sun4i_tcon_find_panel(node);
+	bridge = sun4i_tcon_find_bridge(node);
 
 	/*
 	 * If we don't have a panel endpoint, just go on
 	 */
-	if (PTR_ERR(panel) == -EPROBE_DEFER) {
-		DRM_DEBUG_DRIVER("Still waiting for our panel. Deferring...\n");
+	if ((PTR_ERR(panel) == -EPROBE_DEFER) &&
+	    (PTR_ERR(bridge) == -EPROBE_DEFER)) {
+		DRM_DEBUG_DRIVER("Still waiting for our panel/bridge. Deferring...\n");
 		return -EPROBE_DEFER;
 	}
 
@@ -547,6 +590,7 @@ static int sun4i_tcon_remove(struct platform_device *pdev)
 
 static const struct of_device_id sun4i_tcon_of_table[] = {
 	{ .compatible = "allwinner,sun5i-a13-tcon" },
+	{ .compatible = "allwinner,sun8i-a33-tcon" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sun4i_tcon_of_table);
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.h b/drivers/gpu/drm/sun4i/sun4i_tcon.h
index 0e0b11db401b..12bd48925f4d 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.h
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.h
@@ -143,6 +143,7 @@
 #define SUN4I_TCON_MAX_CHANNELS		2
 
 struct sun4i_tcon {
+	struct device			*dev;
 	struct drm_device		*drm;
 	struct regmap			*regs;
 
@@ -163,8 +164,13 @@ struct sun4i_tcon {
 	bool				has_mux;
 
 	struct drm_panel		*panel;
+
+	bool				has_channel_1;
 };
 
+struct drm_bridge *sun4i_tcon_find_bridge(struct device_node *node);
+struct drm_panel *sun4i_tcon_find_panel(struct device_node *node);
+
 /* Global Control */
 void sun4i_tcon_disable(struct sun4i_tcon *tcon);
 void sun4i_tcon_enable(struct sun4i_tcon *tcon);
diff --git a/drivers/gpu/drm/sun4i/sun4i_tv.c b/drivers/gpu/drm/sun4i/sun4i_tv.c
index b84147896294..1dd3d9eabf2e 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tv.c
@@ -161,10 +161,10 @@ struct tv_mode {
 	bool		dac3_en;
 	bool		dac_bit25_en;
 
-	struct color_gains		*color_gains;
-	struct burst_levels		*burst_levels;
-	struct video_levels		*video_levels;
-	struct resync_parameters	*resync_params;
+	const struct color_gains	*color_gains;
+	const struct burst_levels	*burst_levels;
+	const struct video_levels	*video_levels;
+	const struct resync_parameters	*resync_params;
 };
 
 struct sun4i_tv {
@@ -178,39 +178,39 @@ struct sun4i_tv {
 	struct sun4i_drv	*drv;
 };
 
-struct video_levels ntsc_video_levels = {
+static const struct video_levels ntsc_video_levels = {
 	.black = 282,	.blank = 240,
 };
 
-struct video_levels pal_video_levels = {
+static const struct video_levels pal_video_levels = {
 	.black = 252,	.blank = 252,
 };
 
-struct burst_levels ntsc_burst_levels = {
+static const struct burst_levels ntsc_burst_levels = {
 	.cb = 79,	.cr = 0,
 };
 
-struct burst_levels pal_burst_levels = {
+static const struct burst_levels pal_burst_levels = {
 	.cb = 40,	.cr = 40,
 };
 
-struct color_gains ntsc_color_gains = {
+static const struct color_gains ntsc_color_gains = {
 	.cb = 160,	.cr = 160,
 };
 
-struct color_gains pal_color_gains = {
+static const struct color_gains pal_color_gains = {
 	.cb = 224,	.cr = 224,
 };
 
-struct resync_parameters ntsc_resync_parameters = {
+static const struct resync_parameters ntsc_resync_parameters = {
 	.field = false,	.line = 14,	.pixel = 12,
 };
 
-struct resync_parameters pal_resync_parameters = {
+static const struct resync_parameters pal_resync_parameters = {
 	.field = true,	.line = 13,	.pixel = 12,
 };
 
-struct tv_mode tv_modes[] = {
+static const struct tv_mode tv_modes[] = {
 	{
 		.name		= "NTSC",
 		.mode		= SUN4I_TVE_CFG0_RES_480i,
@@ -289,13 +289,13 @@ drm_connector_to_sun4i_tv(struct drm_connector *connector)
  * So far, it doesn't seem to be preserved when the mode is passed by
  * to mode_set for some reason.
  */
-static struct tv_mode *sun4i_tv_find_tv_by_mode(struct drm_display_mode *mode)
+static const struct tv_mode *sun4i_tv_find_tv_by_mode(const struct drm_display_mode *mode)
 {
 	int i;
 
 	/* First try to identify the mode by name */
 	for (i = 0; i < ARRAY_SIZE(tv_modes); i++) {
-		struct tv_mode *tv_mode = &tv_modes[i];
+		const struct tv_mode *tv_mode = &tv_modes[i];
 
 		DRM_DEBUG_DRIVER("Comparing mode %s vs %s",
 				 mode->name, tv_mode->name);
@@ -306,7 +306,7 @@ static struct tv_mode *sun4i_tv_find_tv_by_mode(struct drm_display_mode *mode)
 
 	/* Then by number of lines */
 	for (i = 0; i < ARRAY_SIZE(tv_modes); i++) {
-		struct tv_mode *tv_mode = &tv_modes[i];
+		const struct tv_mode *tv_mode = &tv_modes[i];
 
 		DRM_DEBUG_DRIVER("Comparing mode %s vs %s (X: %d vs %d)",
 				 mode->name, tv_mode->name,
@@ -319,7 +319,7 @@ static struct tv_mode *sun4i_tv_find_tv_by_mode(struct drm_display_mode *mode)
 	return NULL;
 }
 
-static void sun4i_tv_mode_to_drm_mode(struct tv_mode *tv_mode,
+static void sun4i_tv_mode_to_drm_mode(const struct tv_mode *tv_mode,
 				      struct drm_display_mode *mode)
 {
 	DRM_DEBUG_DRIVER("Creating mode %s\n", mode->name);
@@ -386,7 +386,7 @@ static void sun4i_tv_mode_set(struct drm_encoder *encoder,
 	struct sun4i_tv *tv = drm_encoder_to_sun4i_tv(encoder);
 	struct sun4i_drv *drv = tv->drv;
 	struct sun4i_tcon *tcon = drv->tcon;
-	struct tv_mode *tv_mode = sun4i_tv_find_tv_by_mode(mode);
+	const struct tv_mode *tv_mode = sun4i_tv_find_tv_by_mode(mode);
 
 	sun4i_tcon1_mode_set(tcon, mode);
 
@@ -507,8 +507,14 @@ static int sun4i_tv_comp_get_modes(struct drm_connector *connector)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(tv_modes); i++) {
-		struct drm_display_mode *mode = drm_mode_create(connector->dev);
-		struct tv_mode *tv_mode = &tv_modes[i];
+		struct drm_display_mode *mode;
+		const struct tv_mode *tv_mode = &tv_modes[i];
+
+		mode = drm_mode_create(connector->dev);
+		if (!mode) {
+			DRM_ERROR("Failed to create a new display mode\n");
+			return 0;
+		}
 
 		strcpy(mode->name, tv_mode->name);
 
diff --git a/drivers/gpu/drm/sun4i/sun6i_drc.c b/drivers/gpu/drm/sun4i/sun6i_drc.c
new file mode 100644
index 000000000000..bf6d846d8132
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun6i_drc.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2016 Free Electrons
+ *
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+struct sun6i_drc {
+	struct clk		*bus_clk;
+	struct clk		*mod_clk;
+	struct reset_control	*reset;
+};
+
+static int sun6i_drc_bind(struct device *dev, struct device *master,
+			 void *data)
+{
+	struct sun6i_drc *drc;
+	int ret;
+
+	drc = devm_kzalloc(dev, sizeof(*drc), GFP_KERNEL);
+	if (!drc)
+		return -ENOMEM;
+	dev_set_drvdata(dev, drc);
+
+	drc->reset = devm_reset_control_get(dev, NULL);
+	if (IS_ERR(drc->reset)) {
+		dev_err(dev, "Couldn't get our reset line\n");
+		return PTR_ERR(drc->reset);
+	}
+
+	ret = reset_control_deassert(drc->reset);
+	if (ret) {
+		dev_err(dev, "Couldn't deassert our reset line\n");
+		return ret;
+	}
+
+	drc->bus_clk = devm_clk_get(dev, "ahb");
+	if (IS_ERR(drc->bus_clk)) {
+		dev_err(dev, "Couldn't get our bus clock\n");
+		ret = PTR_ERR(drc->bus_clk);
+		goto err_assert_reset;
+	}
+	clk_prepare_enable(drc->bus_clk);
+
+	drc->mod_clk = devm_clk_get(dev, "mod");
+	if (IS_ERR(drc->mod_clk)) {
+		dev_err(dev, "Couldn't get our mod clock\n");
+		ret = PTR_ERR(drc->mod_clk);
+		goto err_disable_bus_clk;
+	}
+	clk_prepare_enable(drc->mod_clk);
+
+	return 0;
+
+err_disable_bus_clk:
+	clk_disable_unprepare(drc->bus_clk);
+err_assert_reset:
+	reset_control_assert(drc->reset);
+	return ret;
+}
+
+static void sun6i_drc_unbind(struct device *dev, struct device *master,
+			    void *data)
+{
+	struct sun6i_drc *drc = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(drc->mod_clk);
+	clk_disable_unprepare(drc->bus_clk);
+	reset_control_assert(drc->reset);
+}
+
+static struct component_ops sun6i_drc_ops = {
+	.bind	= sun6i_drc_bind,
+	.unbind	= sun6i_drc_unbind,
+};
+
+static int sun6i_drc_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &sun6i_drc_ops);
+}
+
+static int sun6i_drc_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &sun6i_drc_ops);
+
+	return 0;
+}
+
+static const struct of_device_id sun6i_drc_of_table[] = {
+	{ .compatible = "allwinner,sun8i-a33-drc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sun6i_drc_of_table);
+
+static struct platform_driver sun6i_drc_platform_driver = {
+	.probe		= sun6i_drc_probe,
+	.remove		= sun6i_drc_remove,
+	.driver		= {
+		.name		= "sun6i-drc",
+		.of_match_table	= sun6i_drc_of_table,
+	},
+};
+module_platform_driver(sun6i_drc_platform_driver);
+
+MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
+MODULE_DESCRIPTION("Allwinner A31 Dynamic Range Control (DRC) Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tdfx/tdfx_drv.c b/drivers/gpu/drm/tdfx/tdfx_drv.c
index fab5ebcb0fef..f418892b0c71 100644
--- a/drivers/gpu/drm/tdfx/tdfx_drv.c
+++ b/drivers/gpu/drm/tdfx/tdfx_drv.c
@@ -56,6 +56,7 @@ static const struct file_operations tdfx_driver_fops = {
 };
 
 static struct drm_driver driver = {
+	.driver_features = DRIVER_LEGACY,
 	.set_busid = drm_pci_set_busid,
 	.fops = &tdfx_driver_fops,
 	.name = DRIVER_NAME,
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index 8495bd01b544..4010d69cbd08 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -480,17 +480,6 @@ static const struct drm_plane_funcs tegra_primary_plane_funcs = {
 	.atomic_destroy_state = tegra_plane_atomic_destroy_state,
 };
 
-static int tegra_plane_prepare_fb(struct drm_plane *plane,
-				  const struct drm_plane_state *new_state)
-{
-	return 0;
-}
-
-static void tegra_plane_cleanup_fb(struct drm_plane *plane,
-				   const struct drm_plane_state *old_fb)
-{
-}
-
 static int tegra_plane_state_add(struct tegra_plane *plane,
 				 struct drm_plane_state *state)
 {
@@ -591,7 +580,14 @@ static void tegra_plane_atomic_update(struct drm_plane *plane,
 		struct tegra_bo *bo = tegra_fb_get_plane(fb, i);
 
 		window.base[i] = bo->paddr + fb->offsets[i];
-		window.stride[i] = fb->pitches[i];
+
+		/*
+		 * Tegra uses a shared stride for UV planes. Framebuffers are
+		 * already checked for this in the tegra_plane_atomic_check()
+		 * function, so it's safe to ignore the V-plane pitch here.
+		 */
+		if (i < 2)
+			window.stride[i] = fb->pitches[i];
 	}
 
 	tegra_dc_setup_window(dc, p->index, &window);
@@ -624,8 +620,6 @@ static void tegra_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static const struct drm_plane_helper_funcs tegra_primary_plane_helper_funcs = {
-	.prepare_fb = tegra_plane_prepare_fb,
-	.cleanup_fb = tegra_plane_cleanup_fb,
 	.atomic_check = tegra_plane_atomic_check,
 	.atomic_update = tegra_plane_atomic_update,
 	.atomic_disable = tegra_plane_atomic_disable,
@@ -796,8 +790,6 @@ static const struct drm_plane_funcs tegra_cursor_plane_funcs = {
 };
 
 static const struct drm_plane_helper_funcs tegra_cursor_plane_helper_funcs = {
-	.prepare_fb = tegra_plane_prepare_fb,
-	.cleanup_fb = tegra_plane_cleanup_fb,
 	.atomic_check = tegra_cursor_atomic_check,
 	.atomic_update = tegra_cursor_atomic_update,
 	.atomic_disable = tegra_cursor_atomic_disable,
@@ -866,8 +858,6 @@ static const uint32_t tegra_overlay_plane_formats[] = {
 };
 
 static const struct drm_plane_helper_funcs tegra_overlay_plane_helper_funcs = {
-	.prepare_fb = tegra_plane_prepare_fb,
-	.cleanup_fb = tegra_plane_cleanup_fb,
 	.atomic_check = tegra_plane_atomic_check,
 	.atomic_update = tegra_plane_atomic_update,
 	.atomic_disable = tegra_plane_atomic_disable,
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 755264d9db22..8ab47b502d83 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -57,7 +57,8 @@ static void tegra_atomic_complete(struct tegra_drm *tegra,
 
 	drm_atomic_helper_commit_modeset_disables(drm, state);
 	drm_atomic_helper_commit_modeset_enables(drm, state);
-	drm_atomic_helper_commit_planes(drm, state, true);
+	drm_atomic_helper_commit_planes(drm, state,
+					DRM_PLANE_COMMIT_ACTIVE_ONLY);
 
 	drm_atomic_helper_wait_for_vblanks(drm, state);
 
@@ -982,8 +983,8 @@ static int host1x_drm_probe(struct host1x_device *dev)
 	int err;
 
 	drm = drm_dev_alloc(driver, &dev->dev);
-	if (!drm)
-		return -ENOMEM;
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
 
 	dev_set_drvdata(&dev->dev, drm);
 
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index 3d228ad90e0f..3dea1216bafd 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -840,6 +840,21 @@ static const struct drm_encoder_funcs tegra_dsi_encoder_funcs = {
 	.destroy = tegra_output_encoder_destroy,
 };
 
+static void tegra_dsi_unprepare(struct tegra_dsi *dsi)
+{
+	int err;
+
+	if (dsi->slave)
+		tegra_dsi_unprepare(dsi->slave);
+
+	err = tegra_mipi_disable(dsi->mipi);
+	if (err < 0)
+		dev_err(dsi->dev, "failed to disable MIPI calibration: %d\n",
+			err);
+
+	pm_runtime_put(dsi->dev);
+}
+
 static void tegra_dsi_encoder_disable(struct drm_encoder *encoder)
 {
 	struct tegra_output *output = encoder_to_output(encoder);
@@ -876,7 +891,26 @@ static void tegra_dsi_encoder_disable(struct drm_encoder *encoder)
 
 	tegra_dsi_disable(dsi);
 
-	pm_runtime_put(dsi->dev);
+	tegra_dsi_unprepare(dsi);
+}
+
+static void tegra_dsi_prepare(struct tegra_dsi *dsi)
+{
+	int err;
+
+	pm_runtime_get_sync(dsi->dev);
+
+	err = tegra_mipi_enable(dsi->mipi);
+	if (err < 0)
+		dev_err(dsi->dev, "failed to enable MIPI calibration: %d\n",
+			err);
+
+	err = tegra_dsi_pad_calibrate(dsi);
+	if (err < 0)
+		dev_err(dsi->dev, "MIPI calibration failed: %d\n", err);
+
+	if (dsi->slave)
+		tegra_dsi_prepare(dsi->slave);
 }
 
 static void tegra_dsi_encoder_enable(struct drm_encoder *encoder)
@@ -887,13 +921,8 @@ static void tegra_dsi_encoder_enable(struct drm_encoder *encoder)
 	struct tegra_dsi *dsi = to_dsi(output);
 	struct tegra_dsi_state *state;
 	u32 value;
-	int err;
-
-	pm_runtime_get_sync(dsi->dev);
 
-	err = tegra_dsi_pad_calibrate(dsi);
-	if (err < 0)
-		dev_err(dsi->dev, "MIPI calibration failed: %d\n", err);
+	tegra_dsi_prepare(dsi);
 
 	state = tegra_dsi_get_state(dsi);
 
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index aa60d9909ea2..95e622e31931 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -613,7 +613,7 @@ struct dma_buf *tegra_gem_prime_export(struct drm_device *drm,
 	exp_info.flags = flags;
 	exp_info.priv = gem;
 
-	return dma_buf_export(&exp_info);
+	return drm_gem_dmabuf_export(drm, &exp_info);
 }
 
 struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm,
diff --git a/drivers/gpu/drm/tilcdc/Makefile b/drivers/gpu/drm/tilcdc/Makefile
index deeca4869d94..6f675175a9e5 100644
--- a/drivers/gpu/drm/tilcdc/Makefile
+++ b/drivers/gpu/drm/tilcdc/Makefile
@@ -7,6 +7,7 @@ obj-$(CONFIG_DRM_TILCDC_SLAVE_COMPAT) += tilcdc_slave_compat.o \
 					 tilcdc_slave_compat.dtb.o
 
 tilcdc-y := \
+	tilcdc_plane.o \
 	tilcdc_crtc.o \
 	tilcdc_tfp410.o \
 	tilcdc_panel.o \
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index 107c8bd04f6d..52ebe8fc1784 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -15,8 +15,12 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "drm_flip_work.h"
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_flip_work.h>
 #include <drm/drm_plane_helper.h>
+#include <linux/workqueue.h>
 
 #include "tilcdc_drv.h"
 #include "tilcdc_regs.h"
@@ -26,13 +30,16 @@
 struct tilcdc_crtc {
 	struct drm_crtc base;
 
+	struct drm_plane primary;
 	const struct tilcdc_panel_info *info;
 	struct drm_pending_vblank_event *event;
-	int dpms;
+	bool enabled;
 	wait_queue_head_t frame_done_wq;
 	bool frame_done;
 	spinlock_t irq_lock;
 
+	unsigned int lcd_fck_rate;
+
 	ktime_t last_vblank;
 
 	struct drm_framebuffer *curr_fb;
@@ -67,6 +74,7 @@ static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 	struct drm_gem_cma_object *gem;
 	unsigned int depth, bpp;
 	dma_addr_t start, end;
+	u64 dma_base_and_ceiling;
 
 	drm_fb_get_bpp_depth(fb->pixel_format, &depth, &bpp);
 	gem = drm_fb_cma_get_gem_obj(fb, 0);
@@ -77,8 +85,13 @@ static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 
 	end = start + (crtc->mode.vdisplay * fb->pitches[0]);
 
-	tilcdc_write(dev, LCDC_DMA_FB_BASE_ADDR_0_REG, start);
-	tilcdc_write(dev, LCDC_DMA_FB_CEILING_ADDR_0_REG, end);
+	/* Write LCDC_DMA_FB_BASE_ADDR_0_REG and LCDC_DMA_FB_CEILING_ADDR_0_REG
+	 * with a single insruction, if available. This should make it more
+	 * unlikely that LCDC would fetch the DMA addresses in the middle of
+	 * an update.
+	 */
+	dma_base_and_ceiling = (u64)(end - 1) << 32 | start;
+	tilcdc_write64(dev, LCDC_DMA_FB_BASE_ADDR_0_REG, dma_base_and_ceiling);
 
 	if (tilcdc_crtc->curr_fb)
 		drm_flip_work_queue(&tilcdc_crtc->unref_work,
@@ -87,6 +100,43 @@ static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 	tilcdc_crtc->curr_fb = fb;
 }
 
+static void tilcdc_crtc_enable_irqs(struct drm_device *dev)
+{
+	struct tilcdc_drm_private *priv = dev->dev_private;
+
+	tilcdc_clear_irqstatus(dev, 0xffffffff);
+
+	if (priv->rev == 1) {
+		tilcdc_set(dev, LCDC_RASTER_CTRL_REG,
+			LCDC_V1_UNDERFLOW_INT_ENA);
+		tilcdc_set(dev, LCDC_DMA_CTRL_REG,
+			LCDC_V1_END_OF_FRAME_INT_ENA);
+	} else {
+		tilcdc_write(dev, LCDC_INT_ENABLE_SET_REG,
+			LCDC_V2_UNDERFLOW_INT_ENA |
+			LCDC_V2_END_OF_FRAME0_INT_ENA |
+			LCDC_FRAME_DONE | LCDC_SYNC_LOST);
+	}
+}
+
+static void tilcdc_crtc_disable_irqs(struct drm_device *dev)
+{
+	struct tilcdc_drm_private *priv = dev->dev_private;
+
+	/* disable irqs that we might have enabled: */
+	if (priv->rev == 1) {
+		tilcdc_clear(dev, LCDC_RASTER_CTRL_REG,
+			LCDC_V1_UNDERFLOW_INT_ENA | LCDC_V1_PL_INT_ENA);
+		tilcdc_clear(dev, LCDC_DMA_CTRL_REG,
+			LCDC_V1_END_OF_FRAME_INT_ENA);
+	} else {
+		tilcdc_write(dev, LCDC_INT_ENABLE_CLR_REG,
+			LCDC_V2_UNDERFLOW_INT_ENA | LCDC_V2_PL_INT_ENA |
+			LCDC_V2_END_OF_FRAME0_INT_ENA |
+			LCDC_FRAME_DONE | LCDC_SYNC_LOST);
+	}
+}
+
 static void reset(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
@@ -100,66 +150,112 @@ static void reset(struct drm_crtc *crtc)
 	tilcdc_clear(dev, LCDC_CLK_RESET_REG, LCDC_CLK_MAIN_RESET);
 }
 
-static void start(struct drm_crtc *crtc)
+static void tilcdc_crtc_enable(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+
+	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
+
+	if (tilcdc_crtc->enabled)
+		return;
+
+	pm_runtime_get_sync(dev->dev);
 
 	reset(crtc);
 
+	tilcdc_crtc_enable_irqs(dev);
+
 	tilcdc_clear(dev, LCDC_DMA_CTRL_REG, LCDC_DUAL_FRAME_BUFFER_ENABLE);
 	tilcdc_set(dev, LCDC_RASTER_CTRL_REG, LCDC_PALETTE_LOAD_MODE(DATA_ONLY));
 	tilcdc_set(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ENABLE);
+
+	drm_crtc_vblank_on(crtc);
+
+	tilcdc_crtc->enabled = true;
 }
 
-static void stop(struct drm_crtc *crtc)
+void tilcdc_crtc_disable(struct drm_crtc *crtc)
 {
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
+	struct tilcdc_drm_private *priv = dev->dev_private;
+
+	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
+
+	if (!tilcdc_crtc->enabled)
+		return;
 
+	tilcdc_crtc->frame_done = false;
 	tilcdc_clear(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ENABLE);
+
+	/*
+	 * if necessary wait for framedone irq which will still come
+	 * before putting things to sleep..
+	 */
+	if (priv->rev == 2) {
+		int ret = wait_event_timeout(tilcdc_crtc->frame_done_wq,
+					     tilcdc_crtc->frame_done,
+					     msecs_to_jiffies(500));
+		if (ret == 0)
+			dev_err(dev->dev, "%s: timeout waiting for framedone\n",
+				__func__);
+	}
+
+	drm_crtc_vblank_off(crtc);
+
+	tilcdc_crtc_disable_irqs(dev);
+
+	pm_runtime_put_sync(dev->dev);
+
+	if (tilcdc_crtc->next_fb) {
+		drm_flip_work_queue(&tilcdc_crtc->unref_work,
+				    tilcdc_crtc->next_fb);
+		tilcdc_crtc->next_fb = NULL;
+	}
+
+	if (tilcdc_crtc->curr_fb) {
+		drm_flip_work_queue(&tilcdc_crtc->unref_work,
+				    tilcdc_crtc->curr_fb);
+		tilcdc_crtc->curr_fb = NULL;
+	}
+
+	drm_flip_work_commit(&tilcdc_crtc->unref_work, priv->wq);
+	tilcdc_crtc->last_vblank = ktime_set(0, 0);
+
+	tilcdc_crtc->enabled = false;
+}
+
+static bool tilcdc_crtc_is_on(struct drm_crtc *crtc)
+{
+	return crtc->state && crtc->state->enable && crtc->state->active;
 }
 
 static void tilcdc_crtc_destroy(struct drm_crtc *crtc)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+	struct tilcdc_drm_private *priv = crtc->dev->dev_private;
+
+	drm_modeset_lock_crtc(crtc, NULL);
+	tilcdc_crtc_disable(crtc);
+	drm_modeset_unlock_crtc(crtc);
 
-	tilcdc_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+	flush_workqueue(priv->wq);
 
 	of_node_put(crtc->port);
 	drm_crtc_cleanup(crtc);
 	drm_flip_work_cleanup(&tilcdc_crtc->unref_work);
 }
 
-static int tilcdc_verify_fb(struct drm_crtc *crtc, struct drm_framebuffer *fb)
-{
-	struct drm_device *dev = crtc->dev;
-	unsigned int depth, bpp;
-
-	drm_fb_get_bpp_depth(fb->pixel_format, &depth, &bpp);
-
-	if (fb->pitches[0] != crtc->mode.hdisplay * bpp / 8) {
-		dev_err(dev->dev,
-			"Invalid pitch: fb and crtc widths must be the same");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int tilcdc_crtc_page_flip(struct drm_crtc *crtc,
+int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
 		struct drm_framebuffer *fb,
-		struct drm_pending_vblank_event *event,
-		uint32_t page_flip_flags)
+		struct drm_pending_vblank_event *event)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
-	int r;
 	unsigned long flags;
-	s64 tdiff;
-	ktime_t next_vblank;
 
-	r = tilcdc_verify_fb(crtc, fb);
-	if (r)
-		return r;
+	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
 
 	if (tilcdc_crtc->event) {
 		dev_err(dev->dev, "already pending page flip!\n");
@@ -170,82 +266,31 @@ static int tilcdc_crtc_page_flip(struct drm_crtc *crtc,
 
 	crtc->primary->fb = fb;
 
-	pm_runtime_get_sync(dev->dev);
-
 	spin_lock_irqsave(&tilcdc_crtc->irq_lock, flags);
 
-	next_vblank = ktime_add_us(tilcdc_crtc->last_vblank,
-		1000000 / crtc->hwmode.vrefresh);
+	if (crtc->hwmode.vrefresh && ktime_to_ns(tilcdc_crtc->last_vblank)) {
+		ktime_t next_vblank;
+		s64 tdiff;
+
+		next_vblank = ktime_add_us(tilcdc_crtc->last_vblank,
+			1000000 / crtc->hwmode.vrefresh);
 
-	tdiff = ktime_to_us(ktime_sub(next_vblank, ktime_get()));
+		tdiff = ktime_to_us(ktime_sub(next_vblank, ktime_get()));
 
-	if (tdiff >= TILCDC_VBLANK_SAFETY_THRESHOLD_US)
+		if (tdiff < TILCDC_VBLANK_SAFETY_THRESHOLD_US)
+			tilcdc_crtc->next_fb = fb;
+	}
+
+	if (tilcdc_crtc->next_fb != fb)
 		set_scanout(crtc, fb);
-	else
-		tilcdc_crtc->next_fb = fb;
 
 	tilcdc_crtc->event = event;
 
 	spin_unlock_irqrestore(&tilcdc_crtc->irq_lock, flags);
 
-	pm_runtime_put_sync(dev->dev);
-
 	return 0;
 }
 
-void tilcdc_crtc_dpms(struct drm_crtc *crtc, int mode)
-{
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
-	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
-
-	/* we really only care about on or off: */
-	if (mode != DRM_MODE_DPMS_ON)
-		mode = DRM_MODE_DPMS_OFF;
-
-	if (tilcdc_crtc->dpms == mode)
-		return;
-
-	tilcdc_crtc->dpms = mode;
-
-	if (mode == DRM_MODE_DPMS_ON) {
-		pm_runtime_get_sync(dev->dev);
-		start(crtc);
-	} else {
-		tilcdc_crtc->frame_done = false;
-		stop(crtc);
-
-		/*
-		 * if necessary wait for framedone irq which will still come
-		 * before putting things to sleep..
-		 */
-		if (priv->rev == 2) {
-			int ret = wait_event_timeout(
-					tilcdc_crtc->frame_done_wq,
-					tilcdc_crtc->frame_done,
-					msecs_to_jiffies(50));
-			if (ret == 0)
-				dev_err(dev->dev, "timeout waiting for framedone\n");
-		}
-
-		pm_runtime_put_sync(dev->dev);
-
-		if (tilcdc_crtc->next_fb) {
-			drm_flip_work_queue(&tilcdc_crtc->unref_work,
-					    tilcdc_crtc->next_fb);
-			tilcdc_crtc->next_fb = NULL;
-		}
-
-		if (tilcdc_crtc->curr_fb) {
-			drm_flip_work_queue(&tilcdc_crtc->unref_work,
-					    tilcdc_crtc->curr_fb);
-			tilcdc_crtc->curr_fb = NULL;
-		}
-
-		drm_flip_work_commit(&tilcdc_crtc->unref_work, priv->wq);
-	}
-}
-
 static bool tilcdc_crtc_mode_fixup(struct drm_crtc *crtc,
 		const struct drm_display_mode *mode,
 		struct drm_display_mode *adjusted_mode)
@@ -275,41 +320,54 @@ static bool tilcdc_crtc_mode_fixup(struct drm_crtc *crtc,
 	return true;
 }
 
-static void tilcdc_crtc_prepare(struct drm_crtc *crtc)
+static void tilcdc_crtc_set_clk(struct drm_crtc *crtc)
 {
-	tilcdc_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
-}
+	struct drm_device *dev = crtc->dev;
+	struct tilcdc_drm_private *priv = dev->dev_private;
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+	const unsigned clkdiv = 2; /* using a fixed divider of 2 */
+	int ret;
 
-static void tilcdc_crtc_commit(struct drm_crtc *crtc)
-{
-	tilcdc_crtc_dpms(crtc, DRM_MODE_DPMS_ON);
+	/* mode.clock is in KHz, set_rate wants parameter in Hz */
+	ret = clk_set_rate(priv->clk, crtc->mode.clock * 1000 * clkdiv);
+	if (ret < 0) {
+		dev_err(dev->dev, "failed to set display clock rate to: %d\n",
+			crtc->mode.clock);
+		return;
+	}
+
+	tilcdc_crtc->lcd_fck_rate = clk_get_rate(priv->clk);
+
+	DBG("lcd_clk=%u, mode clock=%d, div=%u",
+	    tilcdc_crtc->lcd_fck_rate, crtc->mode.clock, clkdiv);
+
+	/* Configure the LCD clock divisor. */
+	tilcdc_write(dev, LCDC_CTRL_REG, LCDC_CLK_DIVISOR(clkdiv) |
+		     LCDC_RASTER_MODE);
+
+	if (priv->rev == 2)
+		tilcdc_set(dev, LCDC_CLK_ENABLE_REG,
+				LCDC_V2_DMA_CLK_EN | LCDC_V2_LIDD_CLK_EN |
+				LCDC_V2_CORE_CLK_EN);
 }
 
-static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
-		struct drm_display_mode *mode,
-		struct drm_display_mode *adjusted_mode,
-		int x, int y,
-		struct drm_framebuffer *old_fb)
+static void tilcdc_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct tilcdc_drm_private *priv = dev->dev_private;
 	const struct tilcdc_panel_info *info = tilcdc_crtc->info;
 	uint32_t reg, hbp, hfp, hsw, vbp, vfp, vsw;
-	int ret;
+	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	struct drm_framebuffer *fb = crtc->primary->state->fb;
 
-	ret = tilcdc_crtc_mode_valid(crtc, mode);
-	if (WARN_ON(ret))
-		return ret;
+	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
 
 	if (WARN_ON(!info))
-		return -EINVAL;
-
-	ret = tilcdc_verify_fb(crtc, crtc->primary->fb);
-	if (ret)
-		return ret;
+		return;
 
-	pm_runtime_get_sync(dev->dev);
+	if (WARN_ON(!fb))
+		return;
 
 	/* Configure the Burst Size and fifo threshold of DMA: */
 	reg = tilcdc_read(dev, LCDC_DMA_CTRL_REG) & ~0x00000770;
@@ -330,7 +388,8 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
 		reg |= LCDC_DMA_BURST_SIZE(LCDC_DMA_BURST_16);
 		break;
 	default:
-		return -EINVAL;
+		dev_err(dev->dev, "invalid burst size\n");
+		return;
 	}
 	reg |= (info->fifo_th << 8);
 	tilcdc_write(dev, LCDC_DMA_CTRL_REG, reg);
@@ -344,9 +403,9 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
 	vsw = mode->vsync_end - mode->vsync_start;
 
 	DBG("%dx%d, hbp=%u, hfp=%u, hsw=%u, vbp=%u, vfp=%u, vsw=%u",
-			mode->hdisplay, mode->vdisplay, hbp, hfp, hsw, vbp, vfp, vsw);
+	    mode->hdisplay, mode->vdisplay, hbp, hfp, hsw, vbp, vfp, vsw);
 
-	/* Configure the AC Bias Period and Number of Transitions per Interrupt: */
+	/* Set AC Bias Period and Number of Transitions per Interrupt: */
 	reg = tilcdc_read(dev, LCDC_RASTER_TIMING_2_REG) & ~0x000fff00;
 	reg |= LCDC_AC_BIAS_FREQUENCY(info->ac_bias) |
 		LCDC_AC_BIAS_TRANSITIONS_PER_INT(info->ac_bias_intrpt);
@@ -381,7 +440,7 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
 	/*
 	 * be sure to set Bit 10 for the V2 LCDC controller,
 	 * otherwise limited to 1024 pixels width, stopping
-	 * 1920x1080 being suppoted.
+	 * 1920x1080 being supported.
 	 */
 	if (priv->rev == 2) {
 		if ((mode->vdisplay - 1) & 0x400) {
@@ -396,14 +455,15 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
 	/* Configure display type: */
 	reg = tilcdc_read(dev, LCDC_RASTER_CTRL_REG) &
 		~(LCDC_TFT_MODE | LCDC_MONO_8BIT_MODE | LCDC_MONOCHROME_MODE |
-			LCDC_V2_TFT_24BPP_MODE | LCDC_V2_TFT_24BPP_UNPACK | 0x000ff000);
+		  LCDC_V2_TFT_24BPP_MODE | LCDC_V2_TFT_24BPP_UNPACK |
+		  0x000ff000 /* Palette Loading Delay bits */);
 	reg |= LCDC_TFT_MODE; /* no monochrome/passive support */
 	if (info->tft_alt_mode)
 		reg |= LCDC_TFT_ALT_ENABLE;
 	if (priv->rev == 2) {
 		unsigned int depth, bpp;
 
-		drm_fb_get_bpp_depth(crtc->primary->fb->pixel_format, &depth, &bpp);
+		drm_fb_get_bpp_depth(fb->pixel_format, &depth, &bpp);
 		switch (bpp) {
 		case 16:
 			break;
@@ -415,7 +475,7 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
 			break;
 		default:
 			dev_err(dev->dev, "invalid pixel format\n");
-			return -EINVAL;
+			return;
 		}
 	}
 	reg |= info->fdd < 12;
@@ -436,12 +496,7 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
 	else
 		tilcdc_clear(dev, LCDC_RASTER_TIMING_2_REG, LCDC_SYNC_EDGE);
 
-	/*
-	 * use value from adjusted_mode here as this might have been
-	 * changed as part of the fixup for slave encoders to solve the
-	 * issue where tilcdc timings are not VESA compliant
-	 */
-	if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
+	if (mode->flags & DRM_MODE_FLAG_NHSYNC)
 		tilcdc_set(dev, LCDC_RASTER_TIMING_2_REG, LCDC_INVERT_HSYNC);
 	else
 		tilcdc_clear(dev, LCDC_RASTER_TIMING_2_REG, LCDC_INVERT_HSYNC);
@@ -456,51 +511,56 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
 	else
 		tilcdc_clear(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ORDER);
 
-	drm_framebuffer_reference(crtc->primary->fb);
-
-	set_scanout(crtc, crtc->primary->fb);
+	drm_framebuffer_reference(fb);
 
-	tilcdc_crtc_update_clk(crtc);
+	set_scanout(crtc, fb);
 
-	pm_runtime_put_sync(dev->dev);
+	tilcdc_crtc_set_clk(crtc);
 
-	return 0;
+	crtc->hwmode = crtc->state->adjusted_mode;
 }
 
-static int tilcdc_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
-		struct drm_framebuffer *old_fb)
+static int tilcdc_crtc_atomic_check(struct drm_crtc *crtc,
+				    struct drm_crtc_state *state)
 {
-	struct drm_device *dev = crtc->dev;
-	int r;
-
-	r = tilcdc_verify_fb(crtc, crtc->primary->fb);
-	if (r)
-		return r;
-
-	drm_framebuffer_reference(crtc->primary->fb);
+	struct drm_display_mode *mode = &state->mode;
+	int ret;
 
-	pm_runtime_get_sync(dev->dev);
+	/* If we are not active we don't care */
+	if (!state->active)
+		return 0;
 
-	set_scanout(crtc, crtc->primary->fb);
+	if (state->state->planes[0].ptr != crtc->primary ||
+	    state->state->planes[0].state == NULL ||
+	    state->state->planes[0].state->crtc != crtc) {
+		dev_dbg(crtc->dev->dev, "CRTC primary plane must be present");
+		return -EINVAL;
+	}
 
-	pm_runtime_put_sync(dev->dev);
+	ret = tilcdc_crtc_mode_valid(crtc, mode);
+	if (ret) {
+		dev_dbg(crtc->dev->dev, "Mode \"%s\" not valid", mode->name);
+		return -EINVAL;
+	}
 
 	return 0;
 }
 
 static const struct drm_crtc_funcs tilcdc_crtc_funcs = {
-		.destroy        = tilcdc_crtc_destroy,
-		.set_config     = drm_crtc_helper_set_config,
-		.page_flip      = tilcdc_crtc_page_flip,
+	.destroy        = tilcdc_crtc_destroy,
+	.set_config     = drm_atomic_helper_set_config,
+	.page_flip      = drm_atomic_helper_page_flip,
+	.reset		= drm_atomic_helper_crtc_reset,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 };
 
 static const struct drm_crtc_helper_funcs tilcdc_crtc_helper_funcs = {
-		.dpms           = tilcdc_crtc_dpms,
 		.mode_fixup     = tilcdc_crtc_mode_fixup,
-		.prepare        = tilcdc_crtc_prepare,
-		.commit         = tilcdc_crtc_commit,
-		.mode_set       = tilcdc_crtc_mode_set,
-		.mode_set_base  = tilcdc_crtc_mode_set_base,
+		.enable		= tilcdc_crtc_enable,
+		.disable	= tilcdc_crtc_disable,
+		.atomic_check	= tilcdc_crtc_atomic_check,
+		.mode_set_nofb	= tilcdc_crtc_mode_set_nofb,
 };
 
 int tilcdc_crtc_max_width(struct drm_crtc *crtc)
@@ -622,46 +682,23 @@ void tilcdc_crtc_set_simulate_vesa_sync(struct drm_crtc *crtc,
 
 void tilcdc_crtc_update_clk(struct drm_crtc *crtc)
 {
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct tilcdc_drm_private *priv = dev->dev_private;
-	int dpms = tilcdc_crtc->dpms;
-	unsigned long lcd_clk;
-	const unsigned clkdiv = 2; /* using a fixed divider of 2 */
-	int ret;
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 
-	pm_runtime_get_sync(dev->dev);
+	drm_modeset_lock_crtc(crtc, NULL);
+	if (tilcdc_crtc->lcd_fck_rate != clk_get_rate(priv->clk)) {
+		if (tilcdc_crtc_is_on(crtc)) {
+			pm_runtime_get_sync(dev->dev);
+			tilcdc_crtc_disable(crtc);
 
-	if (dpms == DRM_MODE_DPMS_ON)
-		tilcdc_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+			tilcdc_crtc_set_clk(crtc);
 
-	/* mode.clock is in KHz, set_rate wants parameter in Hz */
-	ret = clk_set_rate(priv->clk, crtc->mode.clock * 1000 * clkdiv);
-	if (ret < 0) {
-		dev_err(dev->dev, "failed to set display clock rate to: %d\n",
-				crtc->mode.clock);
-		goto out;
+			tilcdc_crtc_enable(crtc);
+			pm_runtime_put_sync(dev->dev);
+		}
 	}
-
-	lcd_clk = clk_get_rate(priv->clk);
-
-	DBG("lcd_clk=%lu, mode clock=%d, div=%u",
-		lcd_clk, crtc->mode.clock, clkdiv);
-
-	/* Configure the LCD clock divisor. */
-	tilcdc_write(dev, LCDC_CTRL_REG, LCDC_CLK_DIVISOR(clkdiv) |
-			LCDC_RASTER_MODE);
-
-	if (priv->rev == 2)
-		tilcdc_set(dev, LCDC_CLK_ENABLE_REG,
-				LCDC_V2_DMA_CLK_EN | LCDC_V2_LIDD_CLK_EN |
-				LCDC_V2_CORE_CLK_EN);
-
-	if (dpms == DRM_MODE_DPMS_ON)
-		tilcdc_crtc_dpms(crtc, DRM_MODE_DPMS_ON);
-
-out:
-	pm_runtime_put_sync(dev->dev);
+	drm_modeset_unlock_crtc(crtc);
 }
 
 #define SYNC_LOST_COUNT_LIMIT 50
@@ -718,30 +755,34 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 			tilcdc_crtc->frame_intact = true;
 	}
 
+	if (stat & LCDC_FIFO_UNDERFLOW)
+		dev_err_ratelimited(dev->dev, "%s(0x%08x): FIFO underfow",
+				    __func__, stat);
+
+	/* For revision 2 only */
 	if (priv->rev == 2) {
 		if (stat & LCDC_FRAME_DONE) {
 			tilcdc_crtc->frame_done = true;
 			wake_up(&tilcdc_crtc->frame_done_wq);
 		}
-		tilcdc_write(dev, LCDC_END_OF_INT_IND_REG, 0);
-	}
 
-	if (stat & LCDC_SYNC_LOST) {
-		dev_err_ratelimited(dev->dev, "%s(0x%08x): Sync lost",
-				    __func__, stat);
-		tilcdc_crtc->frame_intact = false;
-		if (tilcdc_crtc->sync_lost_count++ > SYNC_LOST_COUNT_LIMIT) {
-			dev_err(dev->dev,
-				"%s(0x%08x): Sync lost flood detected, disabling the interrupt",
-				__func__, stat);
-			tilcdc_write(dev, LCDC_INT_ENABLE_CLR_REG,
-				     LCDC_SYNC_LOST);
+		if (stat & LCDC_SYNC_LOST) {
+			dev_err_ratelimited(dev->dev, "%s(0x%08x): Sync lost",
+					    __func__, stat);
+			tilcdc_crtc->frame_intact = false;
+			if (tilcdc_crtc->sync_lost_count++ >
+			    SYNC_LOST_COUNT_LIMIT) {
+				dev_err(dev->dev, "%s(0x%08x): Sync lost flood detected, disabling the interrupt", __func__, stat);
+				tilcdc_write(dev, LCDC_INT_ENABLE_CLR_REG,
+					     LCDC_SYNC_LOST);
+			}
 		}
-	}
 
-	if (stat & LCDC_FIFO_UNDERFLOW)
-		dev_err_ratelimited(dev->dev, "%s(0x%08x): FIFO underfow",
-				    __func__, stat);
+		/* Indicate to LCDC that the interrupt service routine has
+		 * completed, see 13.3.6.1.6 in AM335x TRM.
+		 */
+		tilcdc_write(dev, LCDC_END_OF_INT_IND_REG, 0);
+	}
 
 	return IRQ_HANDLED;
 }
@@ -761,7 +802,10 @@ struct drm_crtc *tilcdc_crtc_create(struct drm_device *dev)
 
 	crtc = &tilcdc_crtc->base;
 
-	tilcdc_crtc->dpms = DRM_MODE_DPMS_OFF;
+	ret = tilcdc_plane_init(dev, &tilcdc_crtc->primary);
+	if (ret < 0)
+		goto fail;
+
 	init_waitqueue_head(&tilcdc_crtc->frame_done_wq);
 
 	drm_flip_work_init(&tilcdc_crtc->unref_work,
@@ -769,7 +813,11 @@ struct drm_crtc *tilcdc_crtc_create(struct drm_device *dev)
 
 	spin_lock_init(&tilcdc_crtc->irq_lock);
 
-	ret = drm_crtc_init(dev, crtc, &tilcdc_crtc_funcs);
+	ret = drm_crtc_init_with_planes(dev, crtc,
+					&tilcdc_crtc->primary,
+					NULL,
+					&tilcdc_crtc_funcs,
+					"tilcdc crtc");
 	if (ret < 0)
 		goto fail;
 
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index d27809372d54..a694977c32f4 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -20,6 +20,8 @@
 #include <linux/component.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/suspend.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 
 #include "tilcdc_drv.h"
 #include "tilcdc_regs.h"
@@ -31,6 +33,20 @@
 
 static LIST_HEAD(module_list);
 
+static const u32 tilcdc_rev1_formats[] = { DRM_FORMAT_RGB565 };
+
+static const u32 tilcdc_straight_formats[] = { DRM_FORMAT_RGB565,
+					       DRM_FORMAT_BGR888,
+					       DRM_FORMAT_XBGR8888 };
+
+static const u32 tilcdc_crossed_formats[] = { DRM_FORMAT_BGR565,
+					      DRM_FORMAT_RGB888,
+					      DRM_FORMAT_XRGB8888 };
+
+static const u32 tilcdc_legacy_formats[] = { DRM_FORMAT_RGB565,
+					     DRM_FORMAT_RGB888,
+					     DRM_FORMAT_XRGB8888 };
+
 void tilcdc_module_init(struct tilcdc_module *mod, const char *name,
 		const struct tilcdc_module_ops *funcs)
 {
@@ -59,9 +75,84 @@ static void tilcdc_fb_output_poll_changed(struct drm_device *dev)
 	drm_fbdev_cma_hotplug_event(priv->fbdev);
 }
 
+static int tilcdc_atomic_check(struct drm_device *dev,
+			       struct drm_atomic_state *state)
+{
+	int ret;
+
+	ret = drm_atomic_helper_check_modeset(dev, state);
+	if (ret)
+		return ret;
+
+	ret = drm_atomic_helper_check_planes(dev, state);
+	if (ret)
+		return ret;
+
+	/*
+	 * tilcdc ->atomic_check can update ->mode_changed if pixel format
+	 * changes, hence will we check modeset changes again.
+	 */
+	ret = drm_atomic_helper_check_modeset(dev, state);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static int tilcdc_commit(struct drm_device *dev,
+		  struct drm_atomic_state *state,
+		  bool async)
+{
+	int ret;
+
+	ret = drm_atomic_helper_prepare_planes(dev, state);
+	if (ret)
+		return ret;
+
+	drm_atomic_helper_swap_state(state, true);
+
+	/*
+	 * Everything below can be run asynchronously without the need to grab
+	 * any modeset locks at all under one condition: It must be guaranteed
+	 * that the asynchronous work has either been cancelled (if the driver
+	 * supports it, which at least requires that the framebuffers get
+	 * cleaned up with drm_atomic_helper_cleanup_planes()) or completed
+	 * before the new state gets committed on the software side with
+	 * drm_atomic_helper_swap_state().
+	 *
+	 * This scheme allows new atomic state updates to be prepared and
+	 * checked in parallel to the asynchronous completion of the previous
+	 * update. Which is important since compositors need to figure out the
+	 * composition of the next frame right after having submitted the
+	 * current layout.
+	 */
+
+	/* Keep HW on while we commit the state. */
+	pm_runtime_get_sync(dev->dev);
+
+	drm_atomic_helper_commit_modeset_disables(dev, state);
+
+	drm_atomic_helper_commit_planes(dev, state, 0);
+
+	drm_atomic_helper_commit_modeset_enables(dev, state);
+
+	/* Now HW should remain on if need becase the crtc is enabled */
+	pm_runtime_put_sync(dev->dev);
+
+	drm_atomic_helper_wait_for_vblanks(dev, state);
+
+	drm_atomic_helper_cleanup_planes(dev, state);
+
+	drm_atomic_state_free(state);
+
+	return 0;
+}
+
 static const struct drm_mode_config_funcs mode_config_funcs = {
 	.fb_create = tilcdc_fb_create,
 	.output_poll_changed = tilcdc_fb_output_poll_changed,
+	.atomic_check = tilcdc_atomic_check,
+	.atomic_commit = tilcdc_commit,
 };
 
 static int modeset_init(struct drm_device *dev)
@@ -93,12 +184,9 @@ static int cpufreq_transition(struct notifier_block *nb,
 {
 	struct tilcdc_drm_private *priv = container_of(nb,
 			struct tilcdc_drm_private, freq_transition);
-	if (val == CPUFREQ_POSTCHANGE) {
-		if (priv->lcd_fck_rate != clk_get_rate(priv->clk)) {
-			priv->lcd_fck_rate = clk_get_rate(priv->clk);
-			tilcdc_crtc_update_clk(priv->crtc);
-		}
-	}
+
+	if (val == CPUFREQ_POSTCHANGE)
+		tilcdc_crtc_update_clk(priv->crtc);
 
 	return 0;
 }
@@ -112,8 +200,6 @@ static int tilcdc_unload(struct drm_device *dev)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
 
-	tilcdc_crtc_dpms(priv->crtc, DRM_MODE_DPMS_OFF);
-
 	tilcdc_remove_external_encoders(dev);
 
 	drm_fbdev_cma_fini(priv->fbdev);
@@ -121,9 +207,7 @@ static int tilcdc_unload(struct drm_device *dev)
 	drm_mode_config_cleanup(dev);
 	drm_vblank_cleanup(dev);
 
-	pm_runtime_get_sync(dev->dev);
 	drm_irq_uninstall(dev);
-	pm_runtime_put_sync(dev->dev);
 
 #ifdef CONFIG_CPU_FREQ
 	cpufreq_unregister_notifier(&priv->freq_transition,
@@ -146,24 +230,17 @@ static int tilcdc_unload(struct drm_device *dev)
 	return 0;
 }
 
-static size_t tilcdc_num_regs(void);
-
 static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 {
 	struct platform_device *pdev = dev->platformdev;
 	struct device_node *node = pdev->dev.of_node;
 	struct tilcdc_drm_private *priv;
-	struct tilcdc_module *mod;
 	struct resource *res;
 	u32 bpp = 0;
 	int ret;
 
 	priv = devm_kzalloc(dev->dev, sizeof(*priv), GFP_KERNEL);
-	if (priv)
-		priv->saved_register =
-			devm_kcalloc(dev->dev, tilcdc_num_regs(),
-				     sizeof(*priv->saved_register), GFP_KERNEL);
-	if (!priv || !priv->saved_register) {
+	if (!priv) {
 		dev_err(dev->dev, "failed to allocate private data\n");
 		return -ENOMEM;
 	}
@@ -201,7 +278,6 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 	}
 
 #ifdef CONFIG_CPU_FREQ
-	priv->lcd_fck_rate = clk_get_rate(priv->clk);
 	priv->freq_transition.notifier_call = cpufreq_transition;
 	ret = cpufreq_register_notifier(&priv->freq_transition,
 			CPUFREQ_TRANSITION_NOTIFIER);
@@ -249,6 +325,37 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 
 	pm_runtime_put_sync(dev->dev);
 
+	if (priv->rev == 1) {
+		DBG("Revision 1 LCDC supports only RGB565 format");
+		priv->pixelformats = tilcdc_rev1_formats;
+		priv->num_pixelformats = ARRAY_SIZE(tilcdc_rev1_formats);
+		bpp = 16;
+	} else {
+		const char *str = "\0";
+
+		of_property_read_string(node, "blue-and-red-wiring", &str);
+		if (0 == strcmp(str, "crossed")) {
+			DBG("Configured for crossed blue and red wires");
+			priv->pixelformats = tilcdc_crossed_formats;
+			priv->num_pixelformats =
+				ARRAY_SIZE(tilcdc_crossed_formats);
+			bpp = 32; /* Choose bpp with RGB support for fbdef */
+		} else if (0 == strcmp(str, "straight")) {
+			DBG("Configured for straight blue and red wires");
+			priv->pixelformats = tilcdc_straight_formats;
+			priv->num_pixelformats =
+				ARRAY_SIZE(tilcdc_straight_formats);
+			bpp = 16; /* Choose bpp with RGB support for fbdef */
+		} else {
+			DBG("Blue and red wiring '%s' unknown, use legacy mode",
+			    str);
+			priv->pixelformats = tilcdc_legacy_formats;
+			priv->num_pixelformats =
+				ARRAY_SIZE(tilcdc_legacy_formats);
+			bpp = 16; /* This is just a guess */
+		}
+	}
+
 	ret = modeset_init(dev);
 	if (ret < 0) {
 		dev_err(dev->dev, "failed to initialize mode setting\n");
@@ -262,7 +369,7 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 		if (ret < 0)
 			goto fail_mode_config_cleanup;
 
-		ret = tilcdc_add_external_encoders(dev, &bpp);
+		ret = tilcdc_add_external_encoders(dev);
 		if (ret < 0)
 			goto fail_component_cleanup;
 	}
@@ -279,22 +386,14 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 		goto fail_external_cleanup;
 	}
 
-	pm_runtime_get_sync(dev->dev);
 	ret = drm_irq_install(dev, platform_get_irq(dev->platformdev, 0));
-	pm_runtime_put_sync(dev->dev);
 	if (ret < 0) {
 		dev_err(dev->dev, "failed to install IRQ handler\n");
 		goto fail_vblank_cleanup;
 	}
 
-	list_for_each_entry(mod, &module_list, list) {
-		DBG("%s: preferred_bpp: %d", mod->name, mod->preferred_bpp);
-		bpp = mod->preferred_bpp;
-		if (bpp > 0)
-			break;
-	}
+	drm_mode_config_reset(dev);
 
-	drm_helper_disable_unused_functions(dev);
 	priv->fbdev = drm_fbdev_cma_init(dev, bpp,
 			dev->mode_config.num_crtc,
 			dev->mode_config.num_connector);
@@ -308,20 +407,18 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 	return 0;
 
 fail_irq_uninstall:
-	pm_runtime_get_sync(dev->dev);
 	drm_irq_uninstall(dev);
-	pm_runtime_put_sync(dev->dev);
 
 fail_vblank_cleanup:
 	drm_vblank_cleanup(dev);
 
-fail_mode_config_cleanup:
-	drm_mode_config_cleanup(dev);
-
 fail_component_cleanup:
 	if (priv->is_componentized)
 		component_unbind_all(dev->dev, dev);
 
+fail_mode_config_cleanup:
+	drm_mode_config_cleanup(dev);
+
 fail_external_cleanup:
 	tilcdc_remove_external_encoders(dev);
 
@@ -361,45 +458,6 @@ static irqreturn_t tilcdc_irq(int irq, void *arg)
 	return tilcdc_crtc_irq(priv->crtc);
 }
 
-static void tilcdc_irq_preinstall(struct drm_device *dev)
-{
-	tilcdc_clear_irqstatus(dev, 0xffffffff);
-}
-
-static int tilcdc_irq_postinstall(struct drm_device *dev)
-{
-	struct tilcdc_drm_private *priv = dev->dev_private;
-
-	/* enable FIFO underflow irq: */
-	if (priv->rev == 1) {
-		tilcdc_set(dev, LCDC_RASTER_CTRL_REG, LCDC_V1_UNDERFLOW_INT_ENA);
-	} else {
-		tilcdc_write(dev, LCDC_INT_ENABLE_SET_REG,
-			   LCDC_V2_UNDERFLOW_INT_ENA |
-			   LCDC_V2_END_OF_FRAME0_INT_ENA |
-			   LCDC_FRAME_DONE | LCDC_SYNC_LOST);
-	}
-
-	return 0;
-}
-
-static void tilcdc_irq_uninstall(struct drm_device *dev)
-{
-	struct tilcdc_drm_private *priv = dev->dev_private;
-
-	/* disable irqs that we might have enabled: */
-	if (priv->rev == 1) {
-		tilcdc_clear(dev, LCDC_RASTER_CTRL_REG,
-				LCDC_V1_UNDERFLOW_INT_ENA | LCDC_V1_PL_INT_ENA);
-		tilcdc_clear(dev, LCDC_DMA_CTRL_REG, LCDC_V1_END_OF_FRAME_INT_ENA);
-	} else {
-		tilcdc_write(dev, LCDC_INT_ENABLE_CLR_REG,
-			LCDC_V2_UNDERFLOW_INT_ENA | LCDC_V2_PL_INT_ENA |
-			LCDC_V2_END_OF_FRAME0_INT_ENA |
-			LCDC_FRAME_DONE | LCDC_SYNC_LOST);
-	}
-}
-
 static int tilcdc_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	return 0;
@@ -410,7 +468,7 @@ static void tilcdc_disable_vblank(struct drm_device *dev, unsigned int pipe)
 	return;
 }
 
-#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_PM_SLEEP)
+#if defined(CONFIG_DEBUG_FS)
 static const struct {
 	const char *name;
 	uint8_t  rev;
@@ -441,15 +499,6 @@ static const struct {
 #undef REG
 };
 
-static size_t tilcdc_num_regs(void)
-{
-	return ARRAY_SIZE(registers);
-}
-#else
-static size_t tilcdc_num_regs(void)
-{
-	return 0;
-}
 #endif
 
 #ifdef CONFIG_DEBUG_FS
@@ -537,14 +586,11 @@ static const struct file_operations fops = {
 
 static struct drm_driver tilcdc_driver = {
 	.driver_features    = (DRIVER_HAVE_IRQ | DRIVER_GEM | DRIVER_MODESET |
-			       DRIVER_PRIME),
+			       DRIVER_PRIME | DRIVER_ATOMIC),
 	.load               = tilcdc_load,
 	.unload             = tilcdc_unload,
 	.lastclose          = tilcdc_lastclose,
 	.irq_handler        = tilcdc_irq,
-	.irq_preinstall     = tilcdc_irq_preinstall,
-	.irq_postinstall    = tilcdc_irq_postinstall,
-	.irq_uninstall      = tilcdc_irq_uninstall,
 	.get_vblank_counter = drm_vblank_no_hw_counter,
 	.enable_vblank      = tilcdc_enable_vblank,
 	.disable_vblank     = tilcdc_disable_vblank,
@@ -584,28 +630,12 @@ static int tilcdc_pm_suspend(struct device *dev)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct tilcdc_drm_private *priv = ddev->dev_private;
-	unsigned i, n = 0;
 
-	drm_kms_helper_poll_disable(ddev);
+	priv->saved_state = drm_atomic_helper_suspend(ddev);
 
 	/* Select sleep pin state */
 	pinctrl_pm_select_sleep_state(dev);
 
-	if (pm_runtime_suspended(dev)) {
-		priv->ctx_valid = false;
-		return 0;
-	}
-
-	/* Disable the LCDC controller, to avoid locking up the PRCM */
-	tilcdc_crtc_dpms(priv->crtc, DRM_MODE_DPMS_OFF);
-
-	/* Save register state: */
-	for (i = 0; i < ARRAY_SIZE(registers); i++)
-		if (registers[i].save && (priv->rev >= registers[i].rev))
-			priv->saved_register[n++] = tilcdc_read(ddev, registers[i].reg);
-
-	priv->ctx_valid = true;
-
 	return 0;
 }
 
@@ -613,23 +643,15 @@ static int tilcdc_pm_resume(struct device *dev)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct tilcdc_drm_private *priv = ddev->dev_private;
-	unsigned i, n = 0;
+	int ret = 0;
 
 	/* Select default pin state */
 	pinctrl_pm_select_default_state(dev);
 
-	if (priv->ctx_valid == true) {
-		/* Restore register state: */
-		for (i = 0; i < ARRAY_SIZE(registers); i++)
-			if (registers[i].save &&
-			    (priv->rev >= registers[i].rev))
-				tilcdc_write(ddev, registers[i].reg,
-					     priv->saved_register[n++]);
-	}
-
-	drm_kms_helper_poll_enable(ddev);
+	if (priv->saved_state)
+		ret = drm_atomic_helper_resume(ddev, priv->saved_state);
 
-	return 0;
+	return ret;
 }
 #endif
 
@@ -648,6 +670,12 @@ static int tilcdc_bind(struct device *dev)
 
 static void tilcdc_unbind(struct device *dev)
 {
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	/* Check if a subcomponent has already triggered the unloading. */
+	if (!ddev->dev_private)
+		return;
+
 	drm_put_dev(dev_get_drvdata(dev));
 }
 
@@ -680,17 +708,15 @@ static int tilcdc_pdev_probe(struct platform_device *pdev)
 
 static int tilcdc_pdev_remove(struct platform_device *pdev)
 {
-	struct drm_device *ddev = dev_get_drvdata(&pdev->dev);
-	struct tilcdc_drm_private *priv = ddev->dev_private;
-
-	/* Check if a subcomponent has already triggered the unloading. */
-	if (!priv)
-		return 0;
+	int ret;
 
-	if (priv->is_componentized)
-		component_master_del(&pdev->dev, &tilcdc_comp_ops);
-	else
+	ret = tilcdc_get_external_components(&pdev->dev, NULL);
+	if (ret < 0)
+		return ret;
+	else if (ret == 0)
 		drm_put_dev(platform_get_drvdata(pdev));
+	else
+		component_master_del(&pdev->dev, &tilcdc_comp_ops);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.h b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
index c1de18bae415..9780c37ec4cd 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
@@ -65,13 +65,15 @@ struct tilcdc_drm_private {
 	 */
 	uint32_t max_width;
 
-	/* register contents saved across suspend/resume: */
-	u32 *saved_register;
-	bool ctx_valid;
+	/* Supported pixel formats */
+	const uint32_t *pixelformats;
+	uint32_t num_pixelformats;
+
+	/* The context for pm susped/resume cycle is stored here */
+	struct drm_atomic_state *saved_state;
 
 #ifdef CONFIG_CPU_FREQ
 	struct notifier_block freq_transition;
-	unsigned int lcd_fck_rate;
 #endif
 
 	struct workqueue_struct *wq;
@@ -113,7 +115,6 @@ struct tilcdc_module {
 	const char *name;
 	struct list_head list;
 	const struct tilcdc_module_ops *funcs;
-	unsigned int preferred_bpp;
 };
 
 void tilcdc_module_init(struct tilcdc_module *mod, const char *name,
@@ -171,6 +172,11 @@ void tilcdc_crtc_set_simulate_vesa_sync(struct drm_crtc *crtc,
 					bool simulate_vesa_sync);
 int tilcdc_crtc_mode_valid(struct drm_crtc *crtc, struct drm_display_mode *mode);
 int tilcdc_crtc_max_width(struct drm_crtc *crtc);
-void tilcdc_crtc_dpms(struct drm_crtc *crtc, int mode);
+void tilcdc_crtc_disable(struct drm_crtc *crtc);
+int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
+		struct drm_framebuffer *fb,
+		struct drm_pending_vblank_event *event);
+
+int tilcdc_plane_init(struct drm_device *dev, struct drm_plane *plane);
 
 #endif /* __TILCDC_DRV_H__ */
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_external.c b/drivers/gpu/drm/tilcdc/tilcdc_external.c
index 03acb4f99982..68e895021005 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_external.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_external.c
@@ -52,7 +52,7 @@ static int tilcdc_external_mode_valid(struct drm_connector *connector,
 	return MODE_OK;
 }
 
-static int tilcdc_add_external_encoder(struct drm_device *dev, int *bpp,
+static int tilcdc_add_external_encoder(struct drm_device *dev,
 				       struct drm_connector *connector)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
@@ -64,7 +64,6 @@ static int tilcdc_add_external_encoder(struct drm_device *dev, int *bpp,
 	/* Only tda998x is supported at the moment. */
 	tilcdc_crtc_set_simulate_vesa_sync(priv->crtc, true);
 	tilcdc_crtc_set_panel_info(priv->crtc, &panel_info_tda998x);
-	*bpp = panel_info_tda998x.bpp;
 
 	connector_funcs = devm_kzalloc(dev->dev, sizeof(*connector_funcs),
 				       GFP_KERNEL);
@@ -94,7 +93,7 @@ static int tilcdc_add_external_encoder(struct drm_device *dev, int *bpp,
 	return 0;
 }
 
-int tilcdc_add_external_encoders(struct drm_device *dev, int *bpp)
+int tilcdc_add_external_encoders(struct drm_device *dev)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
 	struct drm_connector *connector;
@@ -108,7 +107,7 @@ int tilcdc_add_external_encoders(struct drm_device *dev, int *bpp)
 			if (connector == priv->connectors[i])
 				found = true;
 		if (!found) {
-			ret = tilcdc_add_external_encoder(dev, bpp, connector);
+			ret = tilcdc_add_external_encoder(dev, connector);
 			if (ret)
 				return ret;
 		}
@@ -138,14 +137,23 @@ static int dev_match_of(struct device *dev, void *data)
 int tilcdc_get_external_components(struct device *dev,
 				   struct component_match **match)
 {
+	struct device_node *node;
 	struct device_node *ep = NULL;
 	int count = 0;
 
-	while ((ep = of_graph_get_next_endpoint(dev->of_node, ep))) {
-		struct device_node *node;
+	/* Avoid error print by of_graph_get_next_endpoint() if there
+	 * is no ports present.
+	 */
+	node = of_get_child_by_name(dev->of_node, "ports");
+	if (!node)
+		node = of_get_child_by_name(dev->of_node, "port");
+	if (!node)
+		return 0;
+	of_node_put(node);
 
+	while ((ep = of_graph_get_next_endpoint(dev->of_node, ep))) {
 		node = of_graph_get_remote_port_parent(ep);
-		if (!node && !of_device_is_available(node)) {
+		if (!node || !of_device_is_available(node)) {
 			of_node_put(node);
 			continue;
 		}
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_external.h b/drivers/gpu/drm/tilcdc/tilcdc_external.h
index 6aabe2788760..c700e0c1623e 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_external.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_external.h
@@ -18,7 +18,7 @@
 #ifndef __TILCDC_EXTERNAL_H__
 #define __TILCDC_EXTERNAL_H__
 
-int tilcdc_add_external_encoders(struct drm_device *dev, int *bpp);
+int tilcdc_add_external_encoders(struct drm_device *dev);
 void tilcdc_remove_external_encoders(struct drm_device *dev);
 int tilcdc_get_external_components(struct device *dev,
 				   struct component_match **match);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel.c b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
index ff7774c17d7c..2134bb20fbe9 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_panel.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
@@ -22,8 +22,10 @@
 #include <video/display_timing.h>
 #include <video/of_display_timing.h>
 #include <video/videomode.h>
+#include <drm/drm_atomic_helper.h>
 
 #include "tilcdc_drv.h"
+#include "tilcdc_panel.h"
 
 struct panel_module {
 	struct tilcdc_module base;
@@ -64,9 +66,7 @@ static void panel_encoder_dpms(struct drm_encoder *encoder, int mode)
 
 static void panel_encoder_prepare(struct drm_encoder *encoder)
 {
-	struct panel_encoder *panel_encoder = to_panel_encoder(encoder);
 	panel_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
-	tilcdc_crtc_set_panel_info(encoder->crtc, panel_encoder->mod->info);
 }
 
 static void panel_encoder_commit(struct drm_encoder *encoder)
@@ -196,9 +196,12 @@ static struct drm_encoder *panel_connector_best_encoder(
 
 static const struct drm_connector_funcs panel_connector_funcs = {
 	.destroy            = panel_connector_destroy,
-	.dpms               = drm_helper_connector_dpms,
+	.dpms               = drm_atomic_helper_connector_dpms,
 	.detect             = panel_connector_detect,
 	.fill_modes         = drm_helper_probe_single_connector_modes,
+	.reset              = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
 static const struct drm_connector_helper_funcs panel_connector_helper_funcs = {
@@ -268,6 +271,9 @@ static int panel_modeset_init(struct tilcdc_module *mod, struct drm_device *dev)
 	priv->encoders[priv->num_encoders++] = encoder;
 	priv->connectors[priv->num_connectors++] = connector;
 
+	tilcdc_crtc_set_panel_info(priv->crtc,
+				   to_panel_encoder(encoder)->mod->info);
+
 	return 0;
 }
 
@@ -392,8 +398,6 @@ static int panel_probe(struct platform_device *pdev)
 		goto fail_timings;
 	}
 
-	mod->preferred_bpp = panel_mod->info->bpp;
-
 	return 0;
 
 fail_timings:
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_plane.c b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
new file mode 100644
index 000000000000..74c65fa859b2
--- /dev/null
+++ b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2015 Texas Instruments
+ * Author: Jyri Sarha <jsarha@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <drm/drmP.h>
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_atomic_helper.h>
+#include <uapi/drm/drm_fourcc.h>
+
+#include "tilcdc_drv.h"
+
+static struct drm_plane_funcs tilcdc_plane_funcs = {
+	.update_plane	= drm_atomic_helper_update_plane,
+	.disable_plane	= drm_atomic_helper_disable_plane,
+	.destroy	= drm_plane_cleanup,
+	.set_property	= drm_atomic_helper_plane_set_property,
+	.reset		= drm_atomic_helper_plane_reset,
+	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+};
+
+static int tilcdc_plane_atomic_check(struct drm_plane *plane,
+				     struct drm_plane_state *state)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_plane_state *old_state = plane->state;
+	unsigned int depth, bpp;
+
+	if (!state->crtc)
+		return 0;
+
+	if (WARN_ON(!state->fb))
+		return -EINVAL;
+
+	if (state->crtc_x || state->crtc_y) {
+		dev_err(plane->dev->dev, "%s: crtc position must be zero.",
+			__func__);
+		return -EINVAL;
+	}
+
+	crtc_state = drm_atomic_get_existing_crtc_state(state->state,
+							state->crtc);
+	/* we should have a crtc state if the plane is attached to a crtc */
+	if (WARN_ON(!crtc_state))
+		return 0;
+
+	if (crtc_state->mode.hdisplay != state->crtc_w ||
+	    crtc_state->mode.vdisplay != state->crtc_h) {
+		dev_err(plane->dev->dev,
+			"%s: Size must match mode (%dx%d == %dx%d)", __func__,
+			crtc_state->mode.hdisplay, crtc_state->mode.vdisplay,
+			state->crtc_w, state->crtc_h);
+		return -EINVAL;
+	}
+
+	drm_fb_get_bpp_depth(state->fb->pixel_format, &depth, &bpp);
+	if (state->fb->pitches[0] != crtc_state->mode.hdisplay * bpp / 8) {
+		dev_err(plane->dev->dev,
+			"Invalid pitch: fb and crtc widths must be the same");
+		return -EINVAL;
+	}
+
+	if (state->fb && old_state->fb &&
+	    state->fb->pixel_format != old_state->fb->pixel_format) {
+		dev_dbg(plane->dev->dev,
+			"%s(): pixel format change requires mode_change\n",
+			__func__);
+		crtc_state->mode_changed = true;
+	}
+
+	return 0;
+}
+
+static void tilcdc_plane_atomic_update(struct drm_plane *plane,
+				       struct drm_plane_state *old_state)
+{
+	struct drm_plane_state *state = plane->state;
+
+	if (!state->crtc)
+		return;
+
+	if (WARN_ON(!state->fb || !state->crtc->state))
+		return;
+
+	tilcdc_crtc_update_fb(state->crtc,
+			      state->fb,
+			      state->crtc->state->event);
+}
+
+static const struct drm_plane_helper_funcs plane_helper_funcs = {
+	.atomic_check = tilcdc_plane_atomic_check,
+	.atomic_update = tilcdc_plane_atomic_update,
+};
+
+int tilcdc_plane_init(struct drm_device *dev,
+		      struct drm_plane *plane)
+{
+	struct tilcdc_drm_private *priv = dev->dev_private;
+	int ret;
+
+	ret = drm_plane_init(dev, plane, 1,
+			     &tilcdc_plane_funcs,
+			     priv->pixelformats,
+			     priv->num_pixelformats,
+			     true);
+	if (ret) {
+		dev_err(dev->dev, "Failed to initialize plane: %d\n", ret);
+		return ret;
+	}
+
+	drm_plane_helper_add(plane, &plane_helper_funcs);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_regs.h b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
index 1bf5e2553acc..f57c0d62c76a 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_regs.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
@@ -119,6 +119,20 @@ static inline void tilcdc_write(struct drm_device *dev, u32 reg, u32 data)
 	iowrite32(data, priv->mmio + reg);
 }
 
+static inline void tilcdc_write64(struct drm_device *dev, u32 reg, u64 data)
+{
+	struct tilcdc_drm_private *priv = dev->dev_private;
+	volatile void __iomem *addr = priv->mmio + reg;
+
+#ifdef iowrite64
+	iowrite64(data, addr);
+#else
+	__iowmb();
+	/* This compiles to strd (=64-bit write) on ARM7 */
+	*(volatile u64 __force *)addr = __cpu_to_le64(data);
+#endif
+}
+
 static inline u32 tilcdc_read(struct drm_device *dev, u32 reg)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_slave_compat.c b/drivers/gpu/drm/tilcdc/tilcdc_slave_compat.c
index f9c79dabce20..623a9140493c 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_slave_compat.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_slave_compat.c
@@ -139,7 +139,7 @@ static void __init tilcdc_node_disable(struct device_node *node)
 	of_update_property(node, prop);
 }
 
-struct device_node * __init tilcdc_get_overlay(struct kfree_table *kft)
+static struct device_node * __init tilcdc_get_overlay(struct kfree_table *kft)
 {
 	const int size = __dtb_tilcdc_slave_compat_end -
 		__dtb_tilcdc_slave_compat_begin;
@@ -195,7 +195,7 @@ static const char * const tilcdc_slave_props[] __initconst = {
 	NULL
 };
 
-void __init tilcdc_convert_slave_node(void)
+static void __init tilcdc_convert_slave_node(void)
 {
 	struct device_node *slave = NULL, *lcdc = NULL;
 	struct device_node *i2c = NULL, *fragment = NULL;
@@ -207,7 +207,7 @@ void __init tilcdc_convert_slave_node(void)
 	int ret;
 
 	if (kfree_table_init(&kft))
-		goto out;
+		return;
 
 	lcdc = of_find_matching_node(NULL, tilcdc_of_match);
 	slave = of_find_matching_node(NULL, tilcdc_slave_of_match);
@@ -261,7 +261,7 @@ out:
 	of_node_put(fragment);
 }
 
-int __init tilcdc_slave_compat_init(void)
+static int __init tilcdc_slave_compat_init(void)
 {
 	tilcdc_convert_slave_node();
 	return 0;
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
index 6b8c5b3bf588..458043a53995 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
@@ -20,8 +20,10 @@
 #include <linux/of_gpio.h>
 #include <linux/pinctrl/pinmux.h>
 #include <linux/pinctrl/consumer.h>
+#include <drm/drm_atomic_helper.h>
 
 #include "tilcdc_drv.h"
+#include "tilcdc_tfp410.h"
 
 struct tfp410_module {
 	struct tilcdc_module base;
@@ -75,7 +77,6 @@ static void tfp410_encoder_dpms(struct drm_encoder *encoder, int mode)
 static void tfp410_encoder_prepare(struct drm_encoder *encoder)
 {
 	tfp410_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
-	tilcdc_crtc_set_panel_info(encoder->crtc, &dvi_info);
 }
 
 static void tfp410_encoder_commit(struct drm_encoder *encoder)
@@ -201,9 +202,12 @@ static struct drm_encoder *tfp410_connector_best_encoder(
 
 static const struct drm_connector_funcs tfp410_connector_funcs = {
 	.destroy            = tfp410_connector_destroy,
-	.dpms               = drm_helper_connector_dpms,
+	.dpms               = drm_atomic_helper_connector_dpms,
 	.detect             = tfp410_connector_detect,
 	.fill_modes         = drm_helper_probe_single_connector_modes,
+	.reset              = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
 static const struct drm_connector_helper_funcs tfp410_connector_helper_funcs = {
@@ -276,6 +280,7 @@ static int tfp410_modeset_init(struct tilcdc_module *mod, struct drm_device *dev
 	priv->encoders[priv->num_encoders++] = encoder;
 	priv->connectors[priv->num_connectors++] = connector;
 
+	tilcdc_crtc_set_panel_info(priv->crtc, &dvi_info);
 	return 0;
 }
 
@@ -323,8 +328,6 @@ static int tfp410_probe(struct platform_device *pdev)
 		goto fail;
 	}
 
-	mod->preferred_bpp = dvi_info.bpp;
-
 	i2c_node = of_find_node_by_phandle(i2c_phandle);
 	if (!i2c_node) {
 		dev_err(&pdev->dev, "could not get i2c bus node\n");
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 4054d804fe06..fc6217dfe401 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -57,14 +57,14 @@ static struct attribute ttm_bo_count = {
 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
 					  uint32_t *mem_type)
 {
-	int i;
+	int pos;
 
-	for (i = 0; i <= TTM_PL_PRIV5; i++)
-		if (place->flags & (1 << i)) {
-			*mem_type = i;
-			return 0;
-		}
-	return -EINVAL;
+	pos = ffs(place->flags & TTM_PL_MASK_MEM);
+	if (unlikely(!pos))
+		return -EINVAL;
+
+	*mem_type = pos - 1;
+	return 0;
 }
 
 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
@@ -354,13 +354,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 
 	if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 	    !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
-		ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem);
+		ret = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, mem);
 	else if (bdev->driver->move)
 		ret = bdev->driver->move(bo, evict, interruptible,
 					 no_wait_gpu, mem);
 	else
-		ret = ttm_bo_move_memcpy(bo, evict, interruptible,
-					 no_wait_gpu, mem);
+		ret = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, mem);
 
 	if (ret) {
 		if (bdev->driver->move_notify) {
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 2df602a35f92..bf6e21655c57 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -45,14 +45,22 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
 }
 
 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
-		    bool evict,
-		    bool no_wait_gpu, struct ttm_mem_reg *new_mem)
+		    bool interruptible, bool no_wait_gpu,
+		    struct ttm_mem_reg *new_mem)
 {
 	struct ttm_tt *ttm = bo->ttm;
 	struct ttm_mem_reg *old_mem = &bo->mem;
 	int ret;
 
 	if (old_mem->mem_type != TTM_PL_SYSTEM) {
+		ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+
+		if (unlikely(ret != 0)) {
+			if (ret != -ERESTARTSYS)
+				pr_err("Failed to expire sync object before unbinding TTM\n");
+			return ret;
+		}
+
 		ttm_tt_unbind(ttm);
 		ttm_bo_free_old_node(bo);
 		ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
@@ -321,8 +329,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 }
 
 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
-		       bool evict, bool interruptible,
-		       bool no_wait_gpu,
+		       bool interruptible, bool no_wait_gpu,
 		       struct ttm_mem_reg *new_mem)
 {
 	struct ttm_bo_device *bdev = bo->bdev;
diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c
index a1803fbcc898..29855be96be0 100644
--- a/drivers/gpu/drm/ttm/ttm_memory.c
+++ b/drivers/gpu/drm/ttm/ttm_memory.c
@@ -600,3 +600,9 @@ size_t ttm_round_pot(size_t size)
 	return 0;
 }
 EXPORT_SYMBOL(ttm_round_pot);
+
+uint64_t ttm_get_kernel_zone_memory_size(struct ttm_mem_global *glob)
+{
+	return glob->zone_kernel->max_mem;
+}
+EXPORT_SYMBOL(ttm_get_kernel_zone_memory_size);
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
index bef9f6feb635..cec4b4baa179 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
@@ -858,7 +858,6 @@ static int ttm_dma_pool_get_pages(struct dma_pool *pool,
 	if (count) {
 		d_page = list_first_entry(&pool->free_list, struct dma_page, page_list);
 		ttm->pages[index] = d_page->p;
-		ttm_dma->cpu_address[index] = d_page->vaddr;
 		ttm_dma->dma_address[index] = d_page->dma;
 		list_move_tail(&d_page->page_list, &ttm_dma->pages_list);
 		r = 0;
@@ -989,7 +988,6 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 	INIT_LIST_HEAD(&ttm_dma->pages_list);
 	for (i = 0; i < ttm->num_pages; i++) {
 		ttm->pages[i] = NULL;
-		ttm_dma->cpu_address[i] = 0;
 		ttm_dma->dma_address[i] = 0;
 	}
 
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index bc5aa573f466..aee3c00f836e 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -57,10 +57,8 @@ static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
 {
 	ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages,
 					  sizeof(*ttm->ttm.pages) +
-					  sizeof(*ttm->dma_address) +
-					  sizeof(*ttm->cpu_address));
-	ttm->cpu_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages);
-	ttm->dma_address = (void *) (ttm->cpu_address + ttm->ttm.num_pages);
+					  sizeof(*ttm->dma_address));
+	ttm->dma_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages);
 }
 
 #ifdef CONFIG_X86
@@ -244,7 +242,6 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma)
 
 	drm_free_large(ttm->pages);
 	ttm->pages = NULL;
-	ttm_dma->cpu_address = NULL;
 	ttm_dma->dma_address = NULL;
 }
 EXPORT_SYMBOL(ttm_dma_tt_fini);
diff --git a/drivers/gpu/drm/udl/udl_connector.c b/drivers/gpu/drm/udl/udl_connector.c
index 4709b54c204c..d2f57c52f7db 100644
--- a/drivers/gpu/drm/udl/udl_connector.c
+++ b/drivers/gpu/drm/udl/udl_connector.c
@@ -150,8 +150,5 @@ int udl_connector_init(struct drm_device *dev, struct drm_encoder *encoder)
 	drm_connector_register(connector);
 	drm_mode_connector_attach_encoder(connector, encoder);
 
-	drm_object_attach_property(&connector->base,
-				      dev->mode_config.dirty_info_property,
-				      1);
 	return 0;
 }
diff --git a/drivers/gpu/drm/udl/udl_dmabuf.c b/drivers/gpu/drm/udl/udl_dmabuf.c
index e2243edd1ce3..ac90ffdb5912 100644
--- a/drivers/gpu/drm/udl/udl_dmabuf.c
+++ b/drivers/gpu/drm/udl/udl_dmabuf.c
@@ -209,7 +209,7 @@ struct dma_buf *udl_gem_prime_export(struct drm_device *dev,
 	exp_info.flags = flags;
 	exp_info.priv = obj;
 
-	return dma_buf_export(&exp_info);
+	return drm_gem_dmabuf_export(dev, &exp_info);
 }
 
 static int udl_prime_create(struct drm_device *dev,
diff --git a/drivers/gpu/drm/udl/udl_drv.c b/drivers/gpu/drm/udl/udl_drv.c
index 17d34e0edbdd..cc45d98f9bb5 100644
--- a/drivers/gpu/drm/udl/udl_drv.c
+++ b/drivers/gpu/drm/udl/udl_drv.c
@@ -16,6 +16,20 @@ static int udl_driver_set_busid(struct drm_device *d, struct drm_master *m)
 	return 0;
 }
 
+static int udl_usb_suspend(struct usb_interface *interface,
+			   pm_message_t message)
+{
+	return 0;
+}
+
+static int udl_usb_resume(struct usb_interface *interface)
+{
+	struct drm_device *dev = usb_get_intfdata(interface);
+
+	udl_modeset_restore(dev);
+	return 0;
+}
+
 static const struct vm_operations_struct udl_gem_vm_ops = {
 	.fault = udl_gem_fault,
 	.open = drm_gem_vm_open,
@@ -72,8 +86,8 @@ static int udl_usb_probe(struct usb_interface *interface,
 	int r;
 
 	dev = drm_dev_alloc(&driver, &interface->dev);
-	if (!dev)
-		return -ENOMEM;
+	if (IS_ERR(dev))
+		return PTR_ERR(dev);
 
 	r = drm_dev_register(dev, (unsigned long)udev);
 	if (r)
@@ -122,6 +136,8 @@ static struct usb_driver udl_driver = {
 	.name = "udl",
 	.probe = udl_usb_probe,
 	.disconnect = udl_usb_disconnect,
+	.suspend = udl_usb_suspend,
+	.resume = udl_usb_resume,
 	.id_table = id_table,
 };
 module_usb_driver(udl_driver);
diff --git a/drivers/gpu/drm/udl/udl_drv.h b/drivers/gpu/drm/udl/udl_drv.h
index 0b03d34ffdee..f338a576efc8 100644
--- a/drivers/gpu/drm/udl/udl_drv.h
+++ b/drivers/gpu/drm/udl/udl_drv.h
@@ -52,6 +52,7 @@ struct udl_device {
 	struct device *dev;
 	struct drm_device *ddev;
 	struct usb_device *udev;
+	struct drm_crtc *crtc;
 
 	int sku_pixel_limit;
 
@@ -87,6 +88,7 @@ struct udl_framebuffer {
 
 /* modeset */
 int udl_modeset_init(struct drm_device *dev);
+void udl_modeset_restore(struct drm_device *dev);
 void udl_modeset_cleanup(struct drm_device *dev);
 int udl_connector_init(struct drm_device *dev, struct drm_encoder *encoder);
 
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index d5df555aeba0..611b6b9bb3cb 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -122,7 +122,7 @@ int udl_handle_damage(struct udl_framebuffer *fb, int x, int y,
 		return 0;
 	cmd = urb->transfer_buffer;
 
-	for (i = y; i < height ; i++) {
+	for (i = y; i < y + height ; i++) {
 		const int line_offset = fb->base.pitches[0] * i;
 		const int byte_offset = line_offset + (x * bpp);
 		const int dev_byte_offset = (fb->base.width * bpp * i) + (x * bpp);
@@ -203,6 +203,7 @@ static int udl_fb_open(struct fb_info *info, int user)
 
 	ufbdev->fb_count++;
 
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	if (fb_defio && (info->fbdefio == NULL)) {
 		/* enable defio at last moment if not disabled by client */
 
@@ -218,6 +219,7 @@ static int udl_fb_open(struct fb_info *info, int user)
 		info->fbdefio = fbdefio;
 		fb_deferred_io_init(info);
 	}
+#endif
 
 	pr_notice("open /dev/fb%d user=%d fb_info=%p count=%d\n",
 		  info->node, user, info, ufbdev->fb_count);
@@ -235,12 +237,14 @@ static int udl_fb_release(struct fb_info *info, int user)
 
 	ufbdev->fb_count--;
 
+#ifdef CONFIG_DRM_FBDEV_EMULATION
 	if ((ufbdev->fb_count == 0) && (info->fbdefio)) {
 		fb_deferred_io_cleanup(info);
 		kfree(info->fbdefio);
 		info->fbdefio = NULL;
 		info->fbops->fb_mmap = udl_fb_mmap;
 	}
+#endif
 
 	pr_warn("released /dev/fb%d user=%d count=%d\n",
 		info->node, user, ufbdev->fb_count);
diff --git a/drivers/gpu/drm/udl/udl_main.c b/drivers/gpu/drm/udl/udl_main.c
index 33dbfb2c4748..29f0207fa677 100644
--- a/drivers/gpu/drm/udl/udl_main.c
+++ b/drivers/gpu/drm/udl/udl_main.c
@@ -16,6 +16,8 @@
 /* -BULK_SIZE as per usb-skeleton. Can we get full page and avoid overhead? */
 #define BULK_SIZE 512
 
+#define NR_USB_REQUEST_CHANNEL 0x12
+
 #define MAX_TRANSFER (PAGE_SIZE*16 - BULK_SIZE)
 #define WRITES_IN_FLIGHT (4)
 #define MAX_VENDOR_DESCRIPTOR_SIZE 256
@@ -90,6 +92,26 @@ success:
 	return true;
 }
 
+/*
+ * Need to ensure a channel is selected before submitting URBs
+ */
+static int udl_select_std_channel(struct udl_device *udl)
+{
+	int ret;
+	u8 set_def_chn[] = {0x57, 0xCD, 0xDC, 0xA7,
+			    0x1C, 0x88, 0x5E, 0x15,
+			    0x60, 0xFE, 0xC6, 0x97,
+			    0x16, 0x3D, 0x47, 0xF2};
+
+	ret = usb_control_msg(udl->udev,
+			      usb_sndctrlpipe(udl->udev, 0),
+			      NR_USB_REQUEST_CHANNEL,
+			      (USB_DIR_OUT | USB_TYPE_VENDOR), 0, 0,
+			      set_def_chn, sizeof(set_def_chn),
+			      USB_CTRL_SET_TIMEOUT);
+	return ret < 0 ? ret : 0;
+}
+
 static void udl_release_urb_work(struct work_struct *work)
 {
 	struct urb_node *unode = container_of(work, struct urb_node,
@@ -301,6 +323,9 @@ int udl_driver_load(struct drm_device *dev, unsigned long flags)
 		goto err;
 	}
 
+	if (udl_select_std_channel(udl))
+		DRM_ERROR("Selecting channel failed\n");
+
 	if (!udl_alloc_urb_list(dev, WRITES_IN_FLIGHT, MAX_TRANSFER)) {
 		DRM_ERROR("udl_alloc_urb_list failed\n");
 		goto err;
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index f92ea9579674..f2b2481cad52 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -309,6 +309,8 @@ static int udl_crtc_mode_set(struct drm_crtc *crtc,
 	char *wrptr;
 	int color_depth = 0;
 
+	udl->crtc = crtc;
+
 	buf = (char *)udl->mode_buf;
 
 	/* for now we just clip 24 -> 16 - if we fix that fix this */
@@ -441,8 +443,6 @@ int udl_modeset_init(struct drm_device *dev)
 
 	dev->mode_config.funcs = &udl_mode_funcs;
 
-	drm_mode_create_dirty_info_property(dev);
-
 	udl_crtc_init(dev);
 
 	encoder = udl_encoder_init(dev);
@@ -452,6 +452,18 @@ int udl_modeset_init(struct drm_device *dev)
 	return 0;
 }
 
+void udl_modeset_restore(struct drm_device *dev)
+{
+	struct udl_device *udl = dev->dev_private;
+	struct udl_framebuffer *ufb;
+
+	if (!udl->crtc || !udl->crtc->primary->fb)
+		return;
+	udl_crtc_commit(udl->crtc);
+	ufb = to_udl_fb(udl->crtc->primary->fb);
+	udl_handle_damage(ufb, 0, 0, ufb->base.width, ufb->base.height);
+}
+
 void udl_modeset_cleanup(struct drm_device *dev)
 {
 	drm_mode_config_cleanup(dev);
diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
index 59adcf8532dd..3f6704cf6608 100644
--- a/drivers/gpu/drm/vc4/vc4_bo.c
+++ b/drivers/gpu/drm/vc4/vc4_bo.c
@@ -144,7 +144,7 @@ static struct list_head *vc4_get_cache_list_for_size(struct drm_device *dev,
 	return &vc4->bo_cache.size_list[page_index];
 }
 
-void vc4_bo_cache_purge(struct drm_device *dev)
+static void vc4_bo_cache_purge(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index 8fc2b731b59a..7f08d681a74b 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -163,14 +163,6 @@ int vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
 	int vblank_lines;
 	int ret = 0;
 
-	/*
-	 * XXX Doesn't work well in interlaced mode yet, partially due
-	 * to problems in vc4 kms or drm core interlaced mode handling,
-	 * so disable for now in interlaced mode.
-	 */
-	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-		return ret;
-
 	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
 
 	/* Get optional system timestamp before query. */
@@ -191,10 +183,15 @@ int vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
 
 	/* Vertical position of hvs composed scanline. */
 	*vpos = VC4_GET_FIELD(val, SCALER_DISPSTATX_LINE);
+	*hpos = 0;
 
-	/* No hpos info available. */
-	if (hpos)
-		*hpos = 0;
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
+		*vpos /= 2;
+
+		/* Use hpos to correct for field offset in interlaced mode. */
+		if (VC4_GET_FIELD(val, SCALER_DISPSTATX_FRAME_COUNT) % 2)
+			*hpos += mode->crtc_htotal / 2;
+	}
 
 	/* This is the offset we need for translating hvs -> pv scanout pos. */
 	fifo_lines = vc4_crtc->cob_size / mode->crtc_hdisplay;
@@ -217,8 +214,6 @@ int vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
 		 * position of the PV.
 		 */
 		*vpos -= fifo_lines + 1;
-		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-			*vpos /= 2;
 
 		ret |= DRM_SCANOUTPOS_ACCURATE;
 		return ret;
@@ -234,7 +229,7 @@ int vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
 	 * and need to make things up in a approximative but consistent way.
 	 */
 	ret |= DRM_SCANOUTPOS_IN_VBLANK;
-	vblank_lines = mode->crtc_vtotal - mode->crtc_vdisplay;
+	vblank_lines = mode->vtotal - mode->vdisplay;
 
 	if (flags & DRM_CALLED_FROM_VBLIRQ) {
 		/*
@@ -383,7 +378,7 @@ static void vc4_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	struct drm_crtc_state *state = crtc->state;
 	struct drm_display_mode *mode = &state->adjusted_mode;
 	bool interlace = mode->flags & DRM_MODE_FLAG_INTERLACE;
-	u32 vactive = (mode->vdisplay >> (interlace ? 1 : 0));
+	u32 pixel_rep = (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 2 : 1;
 	u32 format = PV_CONTROL_FORMAT_24;
 	bool debug_dump_regs = false;
 	int clock_select = vc4_get_clock_select(crtc);
@@ -399,47 +394,65 @@ static void vc4_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	CRTC_WRITE(PV_CONTROL, 0);
 
 	CRTC_WRITE(PV_HORZA,
-		   VC4_SET_FIELD(mode->htotal - mode->hsync_end,
+		   VC4_SET_FIELD((mode->htotal -
+				  mode->hsync_end) * pixel_rep,
 				 PV_HORZA_HBP) |
-		   VC4_SET_FIELD(mode->hsync_end - mode->hsync_start,
+		   VC4_SET_FIELD((mode->hsync_end -
+				  mode->hsync_start) * pixel_rep,
 				 PV_HORZA_HSYNC));
 	CRTC_WRITE(PV_HORZB,
-		   VC4_SET_FIELD(mode->hsync_start - mode->hdisplay,
+		   VC4_SET_FIELD((mode->hsync_start -
+				  mode->hdisplay) * pixel_rep,
 				 PV_HORZB_HFP) |
-		   VC4_SET_FIELD(mode->hdisplay, PV_HORZB_HACTIVE));
+		   VC4_SET_FIELD(mode->hdisplay * pixel_rep, PV_HORZB_HACTIVE));
 
 	CRTC_WRITE(PV_VERTA,
-		   VC4_SET_FIELD(mode->vtotal - mode->vsync_end,
+		   VC4_SET_FIELD(mode->crtc_vtotal - mode->crtc_vsync_end,
 				 PV_VERTA_VBP) |
-		   VC4_SET_FIELD(mode->vsync_end - mode->vsync_start,
+		   VC4_SET_FIELD(mode->crtc_vsync_end - mode->crtc_vsync_start,
 				 PV_VERTA_VSYNC));
 	CRTC_WRITE(PV_VERTB,
-		   VC4_SET_FIELD(mode->vsync_start - mode->vdisplay,
+		   VC4_SET_FIELD(mode->crtc_vsync_start - mode->crtc_vdisplay,
 				 PV_VERTB_VFP) |
-		   VC4_SET_FIELD(vactive, PV_VERTB_VACTIVE));
+		   VC4_SET_FIELD(mode->crtc_vdisplay, PV_VERTB_VACTIVE));
 
 	if (interlace) {
 		CRTC_WRITE(PV_VERTA_EVEN,
-			   VC4_SET_FIELD(mode->vtotal - mode->vsync_end - 1,
+			   VC4_SET_FIELD(mode->crtc_vtotal -
+					 mode->crtc_vsync_end - 1,
 					 PV_VERTA_VBP) |
-			   VC4_SET_FIELD(mode->vsync_end - mode->vsync_start,
+			   VC4_SET_FIELD(mode->crtc_vsync_end -
+					 mode->crtc_vsync_start,
 					 PV_VERTA_VSYNC));
 		CRTC_WRITE(PV_VERTB_EVEN,
-			   VC4_SET_FIELD(mode->vsync_start - mode->vdisplay,
+			   VC4_SET_FIELD(mode->crtc_vsync_start -
+					 mode->crtc_vdisplay,
 					 PV_VERTB_VFP) |
-			   VC4_SET_FIELD(vactive, PV_VERTB_VACTIVE));
+			   VC4_SET_FIELD(mode->crtc_vdisplay, PV_VERTB_VACTIVE));
+
+		/* We set up first field even mode for HDMI.  VEC's
+		 * NTSC mode would want first field odd instead, once
+		 * we support it (to do so, set ODD_FIRST and put the
+		 * delay in VSYNCD_EVEN instead).
+		 */
+		CRTC_WRITE(PV_V_CONTROL,
+			   PV_VCONTROL_CONTINUOUS |
+			   PV_VCONTROL_INTERLACE |
+			   VC4_SET_FIELD(mode->htotal * pixel_rep / 2,
+					 PV_VCONTROL_ODD_DELAY));
+		CRTC_WRITE(PV_VSYNCD_EVEN, 0);
+	} else {
+		CRTC_WRITE(PV_V_CONTROL, PV_VCONTROL_CONTINUOUS);
 	}
 
-	CRTC_WRITE(PV_HACT_ACT, mode->hdisplay);
+	CRTC_WRITE(PV_HACT_ACT, mode->hdisplay * pixel_rep);
 
-	CRTC_WRITE(PV_V_CONTROL,
-		   PV_VCONTROL_CONTINUOUS |
-		   (interlace ? PV_VCONTROL_INTERLACE : 0));
 
 	CRTC_WRITE(PV_CONTROL,
 		   VC4_SET_FIELD(format, PV_CONTROL_FORMAT) |
 		   VC4_SET_FIELD(vc4_get_fifo_full_level(format),
 				 PV_CONTROL_FIFO_LEVEL) |
+		   VC4_SET_FIELD(pixel_rep - 1, PV_CONTROL_PIXEL_REP) |
 		   PV_CONTROL_CLR_AT_START |
 		   PV_CONTROL_TRIGGER_UNDERFLOW |
 		   PV_CONTROL_WAIT_HSTART |
@@ -480,6 +493,9 @@ static void vc4_crtc_disable(struct drm_crtc *crtc)
 	int ret;
 	require_hvs_enabled(dev);
 
+	/* Disable vblank irq handling before crtc is disabled. */
+	drm_crtc_vblank_off(crtc);
+
 	CRTC_WRITE(PV_V_CONTROL,
 		   CRTC_READ(PV_V_CONTROL) & ~PV_VCONTROL_VIDEN);
 	ret = wait_for(!(CRTC_READ(PV_V_CONTROL) & PV_VCONTROL_VIDEN), 1);
@@ -530,6 +546,23 @@ static void vc4_crtc_enable(struct drm_crtc *crtc)
 	/* Turn on the pixel valve, which will emit the vstart signal. */
 	CRTC_WRITE(PV_V_CONTROL,
 		   CRTC_READ(PV_V_CONTROL) | PV_VCONTROL_VIDEN);
+
+	/* Enable vblank irq handling after crtc is started. */
+	drm_crtc_vblank_on(crtc);
+}
+
+static bool vc4_crtc_mode_fixup(struct drm_crtc *crtc,
+				const struct drm_display_mode *mode,
+				struct drm_display_mode *adjusted_mode)
+{
+	/* Do not allow doublescan modes from user space */
+	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) {
+		DRM_DEBUG_KMS("[CRTC:%d] Doublescan mode rejected.\n",
+			      crtc->base.id);
+		return false;
+	}
+
+	return true;
 }
 
 static int vc4_crtc_atomic_check(struct drm_crtc *crtc,
@@ -819,6 +852,7 @@ static const struct drm_crtc_helper_funcs vc4_crtc_helper_funcs = {
 	.mode_set_nofb = vc4_crtc_mode_set_nofb,
 	.disable = vc4_crtc_disable,
 	.enable = vc4_crtc_enable,
+	.mode_fixup = vc4_crtc_mode_fixup,
 	.atomic_check = vc4_crtc_atomic_check,
 	.atomic_flush = vc4_crtc_atomic_flush,
 };
diff --git a/drivers/gpu/drm/vc4/vc4_dpi.c b/drivers/gpu/drm/vc4/vc4_dpi.c
index 275fedbdbd9e..1e1f6b8184d0 100644
--- a/drivers/gpu/drm/vc4/vc4_dpi.c
+++ b/drivers/gpu/drm/vc4/vc4_dpi.c
@@ -340,9 +340,20 @@ static void vc4_dpi_encoder_enable(struct drm_encoder *encoder)
 	}
 }
 
+static bool vc4_dpi_encoder_mode_fixup(struct drm_encoder *encoder,
+				       const struct drm_display_mode *mode,
+				       struct drm_display_mode *adjusted_mode)
+{
+	if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
+		return false;
+
+	return true;
+}
+
 static const struct drm_encoder_helper_funcs vc4_dpi_encoder_helper_funcs = {
 	.disable = vc4_dpi_encoder_disable,
 	.enable = vc4_dpi_encoder_enable,
+	.mode_fixup = vc4_dpi_encoder_mode_fixup,
 };
 
 static const struct of_device_id vc4_dpi_dt_match[] = {
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index 8b42d31a7f0e..8703f56b7947 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -16,6 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include "drm_fb_cma_helper.h"
+#include <drm/drm_fb_helper.h>
 
 #include "uapi/drm/vc4_drm.h"
 #include "vc4_drv.h"
@@ -57,21 +58,21 @@ static int vc4_get_param_ioctl(struct drm_device *dev, void *data,
 	switch (args->param) {
 	case DRM_VC4_PARAM_V3D_IDENT0:
 		ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
-		if (ret)
+		if (ret < 0)
 			return ret;
 		args->value = V3D_READ(V3D_IDENT0);
 		pm_runtime_put(&vc4->v3d->pdev->dev);
 		break;
 	case DRM_VC4_PARAM_V3D_IDENT1:
 		ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
-		if (ret)
+		if (ret < 0)
 			return ret;
 		args->value = V3D_READ(V3D_IDENT1);
 		pm_runtime_put(&vc4->v3d->pdev->dev);
 		break;
 	case DRM_VC4_PARAM_V3D_IDENT2:
 		ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
-		if (ret)
+		if (ret < 0)
 			return ret;
 		args->value = V3D_READ(V3D_IDENT2);
 		pm_runtime_put(&vc4->v3d->pdev->dev);
@@ -214,7 +215,7 @@ static void vc4_kick_out_firmware_fb(void)
 	ap->ranges[0].base = 0;
 	ap->ranges[0].size = ~0;
 
-	remove_conflicting_framebuffers(ap, "vc4drmfb", false);
+	drm_fb_helper_remove_conflicting_framebuffers(ap, "vc4drmfb", false);
 	kfree(ap);
 }
 
@@ -232,8 +233,8 @@ static int vc4_drm_bind(struct device *dev)
 		return -ENOMEM;
 
 	drm = drm_dev_alloc(&vc4_drm_driver, dev);
-	if (!drm)
-		return -ENOMEM;
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
 	platform_set_drvdata(pdev, drm);
 	vc4->dev = drm;
 	drm->dev_private = vc4;
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 489e3de0c050..7c1e4d97486f 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -122,9 +122,16 @@ to_vc4_dev(struct drm_device *dev)
 struct vc4_bo {
 	struct drm_gem_cma_object base;
 
-	/* seqno of the last job to render to this BO. */
+	/* seqno of the last job to render using this BO. */
 	uint64_t seqno;
 
+	/* seqno of the last job to use the RCL to write to this BO.
+	 *
+	 * Note that this doesn't include binner overflow memory
+	 * writes.
+	 */
+	uint64_t write_seqno;
+
 	/* List entry for the BO's position in either
 	 * vc4_exec_info->unref_list or vc4_dev->bo_cache.time_list
 	 */
@@ -216,6 +223,9 @@ struct vc4_exec_info {
 	/* Sequence number for this bin/render job. */
 	uint64_t seqno;
 
+	/* Latest write_seqno of any BO that binning depends on. */
+	uint64_t bin_dep_seqno;
+
 	/* Last current addresses the hardware was processing when the
 	 * hangcheck timer checked on us.
 	 */
@@ -230,6 +240,13 @@ struct vc4_exec_info {
 	struct drm_gem_cma_object **bo;
 	uint32_t bo_count;
 
+	/* List of BOs that are being written by the RCL.  Other than
+	 * the binner temporary storage, this is all the BOs written
+	 * by the job.
+	 */
+	struct drm_gem_cma_object *rcl_write_bo[4];
+	uint32_t rcl_write_bo_count;
+
 	/* Pointers for our position in vc4->job_list */
 	struct list_head head;
 
@@ -307,18 +324,24 @@ struct vc4_exec_info {
 static inline struct vc4_exec_info *
 vc4_first_bin_job(struct vc4_dev *vc4)
 {
-	if (list_empty(&vc4->bin_job_list))
-		return NULL;
-	return list_first_entry(&vc4->bin_job_list, struct vc4_exec_info, head);
+	return list_first_entry_or_null(&vc4->bin_job_list,
+					struct vc4_exec_info, head);
 }
 
 static inline struct vc4_exec_info *
 vc4_first_render_job(struct vc4_dev *vc4)
 {
+	return list_first_entry_or_null(&vc4->render_job_list,
+					struct vc4_exec_info, head);
+}
+
+static inline struct vc4_exec_info *
+vc4_last_render_job(struct vc4_dev *vc4)
+{
 	if (list_empty(&vc4->render_job_list))
 		return NULL;
-	return list_first_entry(&vc4->render_job_list,
-				struct vc4_exec_info, head);
+	return list_last_entry(&vc4->render_job_list,
+			       struct vc4_exec_info, head);
 }
 
 /**
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index 6155e8aca1c6..47a095f392f8 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -419,10 +419,6 @@ again:
 
 	vc4_flush_caches(dev);
 
-	/* Disable the binner's pre-loaded overflow memory address */
-	V3D_WRITE(V3D_BPOA, 0);
-	V3D_WRITE(V3D_BPOS, 0);
-
 	/* Either put the job in the binner if it uses the binner, or
 	 * immediately move it to the to-be-rendered queue.
 	 */
@@ -471,6 +467,11 @@ vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)
 	list_for_each_entry(bo, &exec->unref_list, unref_head) {
 		bo->seqno = seqno;
 	}
+
+	for (i = 0; i < exec->rcl_write_bo_count; i++) {
+		bo = to_vc4_bo(&exec->rcl_write_bo[i]->base);
+		bo->write_seqno = seqno;
+	}
 }
 
 /* Queues a struct vc4_exec_info for execution.  If no job is
@@ -534,8 +535,8 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	exec->bo = kcalloc(exec->bo_count, sizeof(struct drm_gem_cma_object *),
-			   GFP_KERNEL);
+	exec->bo = drm_calloc_large(exec->bo_count,
+				    sizeof(struct drm_gem_cma_object *));
 	if (!exec->bo) {
 		DRM_ERROR("Failed to allocate validated BO pointers\n");
 		return -ENOMEM;
@@ -572,8 +573,8 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 	spin_unlock(&file_priv->table_lock);
 
 fail:
-	kfree(handles);
-	return 0;
+	drm_free_large(handles);
+	return ret;
 }
 
 static int
@@ -608,7 +609,7 @@ vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
 	 * read the contents back for validation, and I think the
 	 * bo->vaddr is uncached access.
 	 */
-	temp = kmalloc(temp_size, GFP_KERNEL);
+	temp = drm_malloc_ab(temp_size, 1);
 	if (!temp) {
 		DRM_ERROR("Failed to allocate storage for copying "
 			  "in bin/render CLs.\n");
@@ -673,9 +674,17 @@ vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
 		goto fail;
 
 	ret = vc4_validate_shader_recs(dev, exec);
+	if (ret)
+		goto fail;
+
+	/* Block waiting on any previous rendering into the CS's VBO,
+	 * IB, or textures, so that pixels are actually written by the
+	 * time we try to read them.
+	 */
+	ret = vc4_wait_for_seqno(dev, exec->bin_dep_seqno, ~0ull, true);
 
 fail:
-	kfree(temp);
+	drm_free_large(temp);
 	return ret;
 }
 
@@ -688,7 +697,7 @@ vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
 	if (exec->bo) {
 		for (i = 0; i < exec->bo_count; i++)
 			drm_gem_object_unreference_unlocked(&exec->bo[i]->base);
-		kfree(exec->bo);
+		drm_free_large(exec->bo);
 	}
 
 	while (!list_empty(&exec->unref_list)) {
@@ -942,8 +951,8 @@ vc4_gem_destroy(struct drm_device *dev)
 		vc4->overflow_mem = NULL;
 	}
 
-	vc4_bo_cache_destroy(dev);
-
 	if (vc4->hang_state)
 		vc4_free_hang_state(dev, vc4->hang_state);
+
+	vc4_bo_cache_destroy(dev);
 }
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index 4452f3631cac..c4cb2e26de32 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -62,6 +62,8 @@ struct vc4_hdmi {
 struct vc4_hdmi_encoder {
 	struct vc4_encoder base;
 	bool hdmi_monitor;
+	bool limited_rgb_range;
+	bool rgb_range_selectable;
 };
 
 static inline struct vc4_hdmi_encoder *
@@ -174,6 +176,9 @@ vc4_hdmi_connector_detect(struct drm_connector *connector, bool force)
 			return connector_status_disconnected;
 	}
 
+	if (drm_probe_ddc(vc4->hdmi->ddc))
+		return connector_status_connected;
+
 	if (HDMI_READ(VC4_HDMI_HOTPLUG) & VC4_HDMI_HOTPLUG_CONNECTED)
 		return connector_status_connected;
 	else
@@ -202,6 +207,12 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
 		return -ENODEV;
 
 	vc4_encoder->hdmi_monitor = drm_detect_hdmi_monitor(edid);
+
+	if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) {
+		vc4_encoder->rgb_range_selectable =
+			drm_rgb_quant_range_selectable(edid);
+	}
+
 	drm_mode_connector_update_edid_property(connector, edid);
 	ret = drm_add_edid_modes(connector, edid);
 
@@ -246,7 +257,7 @@ static struct drm_connector *vc4_hdmi_connector_init(struct drm_device *dev,
 	connector->polled = (DRM_CONNECTOR_POLL_CONNECT |
 			     DRM_CONNECTOR_POLL_DISCONNECT);
 
-	connector->interlace_allowed = 0;
+	connector->interlace_allowed = 1;
 	connector->doublescan_allowed = 0;
 
 	drm_mode_connector_attach_encoder(connector, encoder);
@@ -269,25 +280,143 @@ static const struct drm_encoder_funcs vc4_hdmi_encoder_funcs = {
 	.destroy = vc4_hdmi_encoder_destroy,
 };
 
+static int vc4_hdmi_stop_packet(struct drm_encoder *encoder,
+				enum hdmi_infoframe_type type)
+{
+	struct drm_device *dev = encoder->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	u32 packet_id = type - 0x80;
+
+	HDMI_WRITE(VC4_HDMI_RAM_PACKET_CONFIG,
+		   HDMI_READ(VC4_HDMI_RAM_PACKET_CONFIG) & ~BIT(packet_id));
+
+	return wait_for(!(HDMI_READ(VC4_HDMI_RAM_PACKET_STATUS) &
+			  BIT(packet_id)), 100);
+}
+
+static void vc4_hdmi_write_infoframe(struct drm_encoder *encoder,
+				     union hdmi_infoframe *frame)
+{
+	struct drm_device *dev = encoder->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	u32 packet_id = frame->any.type - 0x80;
+	u32 packet_reg = VC4_HDMI_GCP_0 + VC4_HDMI_PACKET_STRIDE * packet_id;
+	uint8_t buffer[VC4_HDMI_PACKET_STRIDE];
+	ssize_t len, i;
+	int ret;
+
+	WARN_ONCE(!(HDMI_READ(VC4_HDMI_RAM_PACKET_CONFIG) &
+		    VC4_HDMI_RAM_PACKET_ENABLE),
+		  "Packet RAM has to be on to store the packet.");
+
+	len = hdmi_infoframe_pack(frame, buffer, sizeof(buffer));
+	if (len < 0)
+		return;
+
+	ret = vc4_hdmi_stop_packet(encoder, frame->any.type);
+	if (ret) {
+		DRM_ERROR("Failed to wait for infoframe to go idle: %d\n", ret);
+		return;
+	}
+
+	for (i = 0; i < len; i += 7) {
+		HDMI_WRITE(packet_reg,
+			   buffer[i + 0] << 0 |
+			   buffer[i + 1] << 8 |
+			   buffer[i + 2] << 16);
+		packet_reg += 4;
+
+		HDMI_WRITE(packet_reg,
+			   buffer[i + 3] << 0 |
+			   buffer[i + 4] << 8 |
+			   buffer[i + 5] << 16 |
+			   buffer[i + 6] << 24);
+		packet_reg += 4;
+	}
+
+	HDMI_WRITE(VC4_HDMI_RAM_PACKET_CONFIG,
+		   HDMI_READ(VC4_HDMI_RAM_PACKET_CONFIG) | BIT(packet_id));
+	ret = wait_for((HDMI_READ(VC4_HDMI_RAM_PACKET_STATUS) &
+			BIT(packet_id)), 100);
+	if (ret)
+		DRM_ERROR("Failed to wait for infoframe to start: %d\n", ret);
+}
+
+static void vc4_hdmi_set_avi_infoframe(struct drm_encoder *encoder)
+{
+	struct vc4_hdmi_encoder *vc4_encoder = to_vc4_hdmi_encoder(encoder);
+	struct drm_crtc *crtc = encoder->crtc;
+	const struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	union hdmi_infoframe frame;
+	int ret;
+
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode);
+	if (ret < 0) {
+		DRM_ERROR("couldn't fill AVI infoframe\n");
+		return;
+	}
+
+	if (vc4_encoder->rgb_range_selectable) {
+		if (vc4_encoder->limited_rgb_range) {
+			frame.avi.quantization_range =
+				HDMI_QUANTIZATION_RANGE_LIMITED;
+		} else {
+			frame.avi.quantization_range =
+				HDMI_QUANTIZATION_RANGE_FULL;
+		}
+	}
+
+	vc4_hdmi_write_infoframe(encoder, &frame);
+}
+
+static void vc4_hdmi_set_spd_infoframe(struct drm_encoder *encoder)
+{
+	union hdmi_infoframe frame;
+	int ret;
+
+	ret = hdmi_spd_infoframe_init(&frame.spd, "Broadcom", "Videocore");
+	if (ret < 0) {
+		DRM_ERROR("couldn't fill SPD infoframe\n");
+		return;
+	}
+
+	frame.spd.sdi = HDMI_SPD_SDI_PC;
+
+	vc4_hdmi_write_infoframe(encoder, &frame);
+}
+
+static void vc4_hdmi_set_infoframes(struct drm_encoder *encoder)
+{
+	vc4_hdmi_set_avi_infoframe(encoder);
+	vc4_hdmi_set_spd_infoframe(encoder);
+}
+
 static void vc4_hdmi_encoder_mode_set(struct drm_encoder *encoder,
 				      struct drm_display_mode *unadjusted_mode,
 				      struct drm_display_mode *mode)
 {
+	struct vc4_hdmi_encoder *vc4_encoder = to_vc4_hdmi_encoder(encoder);
 	struct drm_device *dev = encoder->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	bool debug_dump_regs = false;
 	bool hsync_pos = mode->flags & DRM_MODE_FLAG_PHSYNC;
 	bool vsync_pos = mode->flags & DRM_MODE_FLAG_PVSYNC;
-	u32 vactive = (mode->vdisplay >>
-		       ((mode->flags & DRM_MODE_FLAG_INTERLACE) ? 1 : 0));
-	u32 verta = (VC4_SET_FIELD(mode->vsync_end - mode->vsync_start,
+	bool interlaced = mode->flags & DRM_MODE_FLAG_INTERLACE;
+	u32 pixel_rep = (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 2 : 1;
+	u32 verta = (VC4_SET_FIELD(mode->crtc_vsync_end - mode->crtc_vsync_start,
 				   VC4_HDMI_VERTA_VSP) |
-		     VC4_SET_FIELD(mode->vsync_start - mode->vdisplay,
+		     VC4_SET_FIELD(mode->crtc_vsync_start - mode->crtc_vdisplay,
 				   VC4_HDMI_VERTA_VFP) |
-		     VC4_SET_FIELD(vactive, VC4_HDMI_VERTA_VAL));
+		     VC4_SET_FIELD(mode->crtc_vdisplay, VC4_HDMI_VERTA_VAL));
 	u32 vertb = (VC4_SET_FIELD(0, VC4_HDMI_VERTB_VSPO) |
-		     VC4_SET_FIELD(mode->vtotal - mode->vsync_end,
+		     VC4_SET_FIELD(mode->crtc_vtotal - mode->crtc_vsync_end,
 				   VC4_HDMI_VERTB_VBP));
+	u32 vertb_even = (VC4_SET_FIELD(0, VC4_HDMI_VERTB_VSPO) |
+			  VC4_SET_FIELD(mode->crtc_vtotal -
+					mode->crtc_vsync_end -
+					interlaced,
+					VC4_HDMI_VERTB_VBP));
+	u32 csc_ctl;
 
 	if (debug_dump_regs) {
 		DRM_INFO("HDMI regs before:\n");
@@ -296,7 +425,8 @@ static void vc4_hdmi_encoder_mode_set(struct drm_encoder *encoder,
 
 	HD_WRITE(VC4_HD_VID_CTL, 0);
 
-	clk_set_rate(vc4->hdmi->pixel_clock, mode->clock * 1000);
+	clk_set_rate(vc4->hdmi->pixel_clock, mode->clock * 1000 *
+		     ((mode->flags & DRM_MODE_FLAG_DBLCLK) ? 2 : 1));
 
 	HDMI_WRITE(VC4_HDMI_SCHEDULER_CONTROL,
 		   HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) |
@@ -306,29 +436,62 @@ static void vc4_hdmi_encoder_mode_set(struct drm_encoder *encoder,
 	HDMI_WRITE(VC4_HDMI_HORZA,
 		   (vsync_pos ? VC4_HDMI_HORZA_VPOS : 0) |
 		   (hsync_pos ? VC4_HDMI_HORZA_HPOS : 0) |
-		   VC4_SET_FIELD(mode->hdisplay, VC4_HDMI_HORZA_HAP));
+		   VC4_SET_FIELD(mode->hdisplay * pixel_rep,
+				 VC4_HDMI_HORZA_HAP));
 
 	HDMI_WRITE(VC4_HDMI_HORZB,
-		   VC4_SET_FIELD(mode->htotal - mode->hsync_end,
+		   VC4_SET_FIELD((mode->htotal -
+				  mode->hsync_end) * pixel_rep,
 				 VC4_HDMI_HORZB_HBP) |
-		   VC4_SET_FIELD(mode->hsync_end - mode->hsync_start,
+		   VC4_SET_FIELD((mode->hsync_end -
+				  mode->hsync_start) * pixel_rep,
 				 VC4_HDMI_HORZB_HSP) |
-		   VC4_SET_FIELD(mode->hsync_start - mode->hdisplay,
+		   VC4_SET_FIELD((mode->hsync_start -
+				  mode->hdisplay) * pixel_rep,
 				 VC4_HDMI_HORZB_HFP));
 
 	HDMI_WRITE(VC4_HDMI_VERTA0, verta);
 	HDMI_WRITE(VC4_HDMI_VERTA1, verta);
 
-	HDMI_WRITE(VC4_HDMI_VERTB0, vertb);
+	HDMI_WRITE(VC4_HDMI_VERTB0, vertb_even);
 	HDMI_WRITE(VC4_HDMI_VERTB1, vertb);
 
 	HD_WRITE(VC4_HD_VID_CTL,
 		 (vsync_pos ? 0 : VC4_HD_VID_CTL_VSYNC_LOW) |
 		 (hsync_pos ? 0 : VC4_HD_VID_CTL_HSYNC_LOW));
 
+	csc_ctl = VC4_SET_FIELD(VC4_HD_CSC_CTL_ORDER_BGR,
+				VC4_HD_CSC_CTL_ORDER);
+
+	if (vc4_encoder->hdmi_monitor && drm_match_cea_mode(mode) > 1) {
+		/* CEA VICs other than #1 requre limited range RGB
+		 * output unless overridden by an AVI infoframe.
+		 * Apply a colorspace conversion to squash 0-255 down
+		 * to 16-235.  The matrix here is:
+		 *
+		 * [ 0      0      0.8594 16]
+		 * [ 0      0.8594 0      16]
+		 * [ 0.8594 0      0      16]
+		 * [ 0      0      0       1]
+		 */
+		csc_ctl |= VC4_HD_CSC_CTL_ENABLE;
+		csc_ctl |= VC4_HD_CSC_CTL_RGB2YCC;
+		csc_ctl |= VC4_SET_FIELD(VC4_HD_CSC_CTL_MODE_CUSTOM,
+					 VC4_HD_CSC_CTL_MODE);
+
+		HD_WRITE(VC4_HD_CSC_12_11, (0x000 << 16) | 0x000);
+		HD_WRITE(VC4_HD_CSC_14_13, (0x100 << 16) | 0x6e0);
+		HD_WRITE(VC4_HD_CSC_22_21, (0x6e0 << 16) | 0x000);
+		HD_WRITE(VC4_HD_CSC_24_23, (0x100 << 16) | 0x000);
+		HD_WRITE(VC4_HD_CSC_32_31, (0x000 << 16) | 0x6e0);
+		HD_WRITE(VC4_HD_CSC_34_33, (0x100 << 16) | 0x000);
+		vc4_encoder->limited_rgb_range = true;
+	} else {
+		vc4_encoder->limited_rgb_range = false;
+	}
+
 	/* The RGB order applies even when CSC is disabled. */
-	HD_WRITE(VC4_HD_CSC_CTL, VC4_SET_FIELD(VC4_HD_CSC_CTL_ORDER_BGR,
-					       VC4_HD_CSC_CTL_ORDER));
+	HD_WRITE(VC4_HD_CSC_CTL, csc_ctl);
 
 	HDMI_WRITE(VC4_HDMI_FIFO_CTL, VC4_HDMI_FIFO_CTL_MASTER_SLAVE_N);
 
@@ -343,6 +506,8 @@ static void vc4_hdmi_encoder_disable(struct drm_encoder *encoder)
 	struct drm_device *dev = encoder->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 
+	HDMI_WRITE(VC4_HDMI_RAM_PACKET_CONFIG, 0);
+
 	HDMI_WRITE(VC4_HDMI_TX_PHY_RESET_CTL, 0xf << 16);
 	HD_WRITE(VC4_HD_VID_CTL,
 		 HD_READ(VC4_HD_VID_CTL) & ~VC4_HD_VID_CTL_ENABLE);
@@ -369,7 +534,7 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
 			   VC4_HDMI_SCHEDULER_CONTROL_MODE_HDMI);
 
 		ret = wait_for(HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) &
-			       VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE, 1);
+			       VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE, 1000);
 		WARN_ONCE(ret, "Timeout waiting for "
 			  "VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE\n");
 	} else {
@@ -381,7 +546,7 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
 			   ~VC4_HDMI_SCHEDULER_CONTROL_MODE_HDMI);
 
 		ret = wait_for(!(HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) &
-				 VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE), 1);
+				 VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE), 1000);
 		WARN_ONCE(ret, "Timeout waiting for "
 			  "!VC4_HDMI_SCHEDULER_CONTROL_HDMI_ACTIVE\n");
 	}
@@ -395,9 +560,10 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
 			   HDMI_READ(VC4_HDMI_SCHEDULER_CONTROL) |
 			   VC4_HDMI_SCHEDULER_CONTROL_VERT_ALWAYS_KEEPOUT);
 
-		/* XXX: Set HDMI_RAM_PACKET_CONFIG (1 << 16) and set
-		 * up the infoframe.
-		 */
+		HDMI_WRITE(VC4_HDMI_RAM_PACKET_CONFIG,
+			   VC4_HDMI_RAM_PACKET_ENABLE);
+
+		vc4_hdmi_set_infoframes(encoder);
 
 		drift = HDMI_READ(VC4_HDMI_FIFO_CTL);
 		drift &= VC4_HDMI_FIFO_VALID_WRITE_MASK;
diff --git a/drivers/gpu/drm/vc4/vc4_irq.c b/drivers/gpu/drm/vc4/vc4_irq.c
index b0104a346a74..094bc6a475c1 100644
--- a/drivers/gpu/drm/vc4/vc4_irq.c
+++ b/drivers/gpu/drm/vc4/vc4_irq.c
@@ -83,8 +83,10 @@ vc4_overflow_mem_work(struct work_struct *work)
 
 		spin_lock_irqsave(&vc4->job_lock, irqflags);
 		current_exec = vc4_first_bin_job(vc4);
+		if (!current_exec)
+			current_exec = vc4_last_render_job(vc4);
 		if (current_exec) {
-			vc4->overflow_mem->seqno = vc4->finished_seqno + 1;
+			vc4->overflow_mem->seqno = current_exec->seqno;
 			list_add_tail(&vc4->overflow_mem->unref_head,
 				      &current_exec->unref_list);
 			vc4->overflow_mem = NULL;
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index 4ac894d993cd..c1f65c6c8e60 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -44,7 +44,7 @@ vc4_atomic_complete_commit(struct vc4_commit *c)
 
 	drm_atomic_helper_commit_modeset_disables(dev, state);
 
-	drm_atomic_helper_commit_planes(dev, state, false);
+	drm_atomic_helper_commit_planes(dev, state, 0);
 
 	drm_atomic_helper_commit_modeset_enables(dev, state);
 
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index 29e4b400e25e..881bf489478b 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -735,8 +735,6 @@ void vc4_plane_async_set_fb(struct drm_plane *plane, struct drm_framebuffer *fb)
 }
 
 static const struct drm_plane_helper_funcs vc4_plane_helper_funcs = {
-	.prepare_fb = NULL,
-	.cleanup_fb = NULL,
 	.atomic_check = vc4_plane_atomic_check,
 	.atomic_update = vc4_plane_atomic_update,
 };
diff --git a/drivers/gpu/drm/vc4/vc4_regs.h b/drivers/gpu/drm/vc4/vc4_regs.h
index 160942a9180e..1aa44c2db556 100644
--- a/drivers/gpu/drm/vc4/vc4_regs.h
+++ b/drivers/gpu/drm/vc4/vc4_regs.h
@@ -175,6 +175,8 @@
 # define PV_CONTROL_CLR_AT_START		BIT(14)
 # define PV_CONTROL_TRIGGER_UNDERFLOW		BIT(13)
 # define PV_CONTROL_WAIT_HSTART			BIT(12)
+# define PV_CONTROL_PIXEL_REP_MASK		VC4_MASK(5, 4)
+# define PV_CONTROL_PIXEL_REP_SHIFT		4
 # define PV_CONTROL_CLK_SELECT_DSI_VEC		0
 # define PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI	1
 # define PV_CONTROL_CLK_SELECT_MASK		VC4_MASK(3, 2)
@@ -183,6 +185,9 @@
 # define PV_CONTROL_EN				BIT(0)
 
 #define PV_V_CONTROL				0x04
+# define PV_VCONTROL_ODD_DELAY_MASK		VC4_MASK(22, 6)
+# define PV_VCONTROL_ODD_DELAY_SHIFT		6
+# define PV_VCONTROL_ODD_FIRST			BIT(5)
 # define PV_VCONTROL_INTERLACE			BIT(4)
 # define PV_VCONTROL_CONTINUOUS			BIT(1)
 # define PV_VCONTROL_VIDEN			BIT(0)
@@ -438,6 +443,8 @@
 #define VC4_HDMI_RAM_PACKET_CONFIG		0x0a0
 # define VC4_HDMI_RAM_PACKET_ENABLE		BIT(16)
 
+#define VC4_HDMI_RAM_PACKET_STATUS		0x0a4
+
 #define VC4_HDMI_HORZA				0x0c4
 # define VC4_HDMI_HORZA_VPOS			BIT(14)
 # define VC4_HDMI_HORZA_HPOS			BIT(13)
@@ -499,6 +506,9 @@
 
 #define VC4_HDMI_TX_PHY_RESET_CTL		0x2c0
 
+#define VC4_HDMI_GCP_0				0x400
+#define VC4_HDMI_PACKET_STRIDE			0x24
+
 #define VC4_HD_M_CTL				0x00c
 # define VC4_HD_M_REGISTER_FILE_STANDBY		(3 << 6)
 # define VC4_HD_M_RAM_STANDBY			(3 << 4)
@@ -528,10 +538,17 @@
 # define VC4_HD_CSC_CTL_MODE_SHIFT		2
 # define VC4_HD_CSC_CTL_MODE_RGB_TO_SD_YPRPB	0
 # define VC4_HD_CSC_CTL_MODE_RGB_TO_HD_YPRPB	1
-# define VC4_HD_CSC_CTL_MODE_CUSTOM		2
+# define VC4_HD_CSC_CTL_MODE_CUSTOM		3
 # define VC4_HD_CSC_CTL_RGB2YCC			BIT(1)
 # define VC4_HD_CSC_CTL_ENABLE			BIT(0)
 
+#define VC4_HD_CSC_12_11			0x044
+#define VC4_HD_CSC_14_13			0x048
+#define VC4_HD_CSC_22_21			0x04c
+#define VC4_HD_CSC_24_23			0x050
+#define VC4_HD_CSC_32_31			0x054
+#define VC4_HD_CSC_34_33			0x058
+
 #define VC4_HD_FRAME_COUNT			0x068
 
 /* HVS display list information. */
diff --git a/drivers/gpu/drm/vc4/vc4_render_cl.c b/drivers/gpu/drm/vc4/vc4_render_cl.c
index 0f12418725e5..08886a309757 100644
--- a/drivers/gpu/drm/vc4/vc4_render_cl.c
+++ b/drivers/gpu/drm/vc4/vc4_render_cl.c
@@ -45,6 +45,8 @@ struct vc4_rcl_setup {
 
 	struct drm_gem_cma_object *rcl;
 	u32 next_offset;
+
+	u32 next_write_bo_index;
 };
 
 static inline void rcl_u8(struct vc4_rcl_setup *setup, u8 val)
@@ -407,6 +409,8 @@ static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
 	if (!*obj)
 		return -EINVAL;
 
+	exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
+
 	if (surf->offset & 0xf) {
 		DRM_ERROR("MSAA write must be 16b aligned.\n");
 		return -EINVAL;
@@ -417,7 +421,8 @@ static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
 
 static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
 				 struct drm_gem_cma_object **obj,
-				 struct drm_vc4_submit_rcl_surface *surf)
+				 struct drm_vc4_submit_rcl_surface *surf,
+				 bool is_write)
 {
 	uint8_t tiling = VC4_GET_FIELD(surf->bits,
 				       VC4_LOADSTORE_TILE_BUFFER_TILING);
@@ -440,6 +445,9 @@ static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
 	if (!*obj)
 		return -EINVAL;
 
+	if (is_write)
+		exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
+
 	if (surf->flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
 		if (surf == &exec->args->zs_write) {
 			DRM_ERROR("general zs write may not be a full-res.\n");
@@ -542,6 +550,8 @@ vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
 	if (!*obj)
 		return -EINVAL;
 
+	exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
+
 	if (tiling > VC4_TILING_FORMAT_LT) {
 		DRM_ERROR("Bad tiling format\n");
 		return -EINVAL;
@@ -599,15 +609,18 @@ int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
 	if (ret)
 		return ret;
 
-	ret = vc4_rcl_surface_setup(exec, &setup.color_read, &args->color_read);
+	ret = vc4_rcl_surface_setup(exec, &setup.color_read, &args->color_read,
+				    false);
 	if (ret)
 		return ret;
 
-	ret = vc4_rcl_surface_setup(exec, &setup.zs_read, &args->zs_read);
+	ret = vc4_rcl_surface_setup(exec, &setup.zs_read, &args->zs_read,
+				    false);
 	if (ret)
 		return ret;
 
-	ret = vc4_rcl_surface_setup(exec, &setup.zs_write, &args->zs_write);
+	ret = vc4_rcl_surface_setup(exec, &setup.zs_write, &args->zs_write,
+				    true);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c
index 9ce1d0adf882..26503e307438 100644
--- a/drivers/gpu/drm/vc4/vc4_validate.c
+++ b/drivers/gpu/drm/vc4/vc4_validate.c
@@ -267,6 +267,9 @@ validate_indexed_prim_list(VALIDATE_ARGS)
 	if (!ib)
 		return -EINVAL;
 
+	exec->bin_dep_seqno = max(exec->bin_dep_seqno,
+				  to_vc4_bo(&ib->base)->write_seqno);
+
 	if (offset > ib->base.size ||
 	    (ib->base.size - offset) / index_size < length) {
 		DRM_ERROR("IB access overflow (%d + %d*%d > %zd)\n",
@@ -555,8 +558,7 @@ static bool
 reloc_tex(struct vc4_exec_info *exec,
 	  void *uniform_data_u,
 	  struct vc4_texture_sample_info *sample,
-	  uint32_t texture_handle_index)
-
+	  uint32_t texture_handle_index, bool is_cs)
 {
 	struct drm_gem_cma_object *tex;
 	uint32_t p0 = *(uint32_t *)(uniform_data_u + sample->p_offset[0]);
@@ -714,6 +716,11 @@ reloc_tex(struct vc4_exec_info *exec,
 
 	*validated_p0 = tex->paddr + p0;
 
+	if (is_cs) {
+		exec->bin_dep_seqno = max(exec->bin_dep_seqno,
+					  to_vc4_bo(&tex->base)->write_seqno);
+	}
+
 	return true;
  fail:
 	DRM_INFO("Texture p0 at %d: 0x%08x\n", sample->p_offset[0], p0);
@@ -835,7 +842,8 @@ validate_gl_shader_rec(struct drm_device *dev,
 			if (!reloc_tex(exec,
 				       uniform_data_u,
 				       &validated_shader->texture_samples[tex],
-				       texture_handles_u[tex])) {
+				       texture_handles_u[tex],
+				       i == 2)) {
 				return -EINVAL;
 			}
 		}
@@ -867,6 +875,9 @@ validate_gl_shader_rec(struct drm_device *dev,
 		uint32_t stride = *(uint8_t *)(pkt_u + o + 5);
 		uint32_t max_index;
 
+		exec->bin_dep_seqno = max(exec->bin_dep_seqno,
+					  to_vc4_bo(&vbo->base)->write_seqno);
+
 		if (state->addr & 0x8)
 			stride |= (*(uint32_t *)(pkt_u + 100 + i * 4)) & ~0xff;
 
diff --git a/drivers/gpu/drm/vc4/vc4_validate_shaders.c b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
index 46527e989ce3..2543cf5b8b51 100644
--- a/drivers/gpu/drm/vc4/vc4_validate_shaders.c
+++ b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
@@ -309,8 +309,14 @@ validate_uniform_address_write(struct vc4_validated_shader_info *validated_shade
 	 * of uniforms on each side.  However, this scheme is easy to
 	 * validate so it's all we allow for now.
 	 */
-
-	if (QPU_GET_FIELD(inst, QPU_SIG) != QPU_SIG_NONE) {
+	switch (QPU_GET_FIELD(inst, QPU_SIG)) {
+	case QPU_SIG_NONE:
+	case QPU_SIG_SCOREBOARD_UNLOCK:
+	case QPU_SIG_COLOR_LOAD:
+	case QPU_SIG_LOAD_TMU0:
+	case QPU_SIG_LOAD_TMU1:
+		break;
+	default:
 		DRM_ERROR("uniforms address change must be "
 			  "normal math\n");
 		return false;
diff --git a/drivers/gpu/drm/vgem/vgem_drv.c b/drivers/gpu/drm/vgem/vgem_drv.c
index c15bafb06665..f36c14729b55 100644
--- a/drivers/gpu/drm/vgem/vgem_drv.c
+++ b/drivers/gpu/drm/vgem/vgem_drv.c
@@ -334,8 +334,8 @@ static int __init vgem_init(void)
 	int ret;
 
 	vgem_device = drm_dev_alloc(&vgem_driver, NULL);
-	if (!vgem_device) {
-		ret = -ENOMEM;
+	if (IS_ERR(vgem_device)) {
+		ret = PTR_ERR(vgem_device);
 		goto out;
 	}
 
diff --git a/drivers/gpu/drm/via/via_drv.c b/drivers/gpu/drm/via/via_drv.c
index ed8aa8ff861a..e5582bab7e3c 100644
--- a/drivers/gpu/drm/via/via_drv.c
+++ b/drivers/gpu/drm/via/via_drv.c
@@ -72,7 +72,7 @@ static const struct file_operations via_driver_fops = {
 
 static struct drm_driver driver = {
 	.driver_features =
-	    DRIVER_USE_AGP | DRIVER_HAVE_IRQ |
+	    DRIVER_USE_AGP | DRIVER_HAVE_IRQ | DRIVER_LEGACY |
 	    DRIVER_IRQ_SHARED,
 	.load = via_driver_load,
 	.unload = via_driver_unload,
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index 4e192aa2d021..7cf3678623c3 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -338,7 +338,8 @@ static void vgdev_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_modeset_disables(dev, state);
 	drm_atomic_helper_commit_modeset_enables(dev, state);
-	drm_atomic_helper_commit_planes(dev, state, true);
+	drm_atomic_helper_commit_planes(dev, state,
+					DRM_PLANE_COMMIT_ACTIVE_ONLY);
 
 	drm_atomic_helper_commit_hw_done(state);
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_drm_bus.c b/drivers/gpu/drm/virtio/virtgpu_drm_bus.c
index 7f0e93f87a55..49e5996cb9f2 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drm_bus.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drm_bus.c
@@ -24,9 +24,20 @@
  */
 
 #include <linux/pci.h>
+#include <drm/drm_fb_helper.h>
 
 #include "virtgpu_drv.h"
 
+int drm_virtio_set_busid(struct drm_device *dev, struct drm_master *master)
+{
+	struct pci_dev *pdev = dev->pdev;
+
+	if (pdev) {
+		return drm_pci_set_busid(dev, master);
+	}
+	return 0;
+}
+
 static void virtio_pci_kick_out_firmware_fb(struct pci_dev *pci_dev)
 {
 	struct apertures_struct *ap;
@@ -42,7 +53,7 @@ static void virtio_pci_kick_out_firmware_fb(struct pci_dev *pci_dev)
 	primary = pci_dev->resource[PCI_ROM_RESOURCE].flags
 		& IORESOURCE_ROM_SHADOW;
 
-	remove_conflicting_framebuffers(ap, "virtiodrmfb", primary);
+	drm_fb_helper_remove_conflicting_framebuffers(ap, "virtiodrmfb", primary);
 
 	kfree(ap);
 }
@@ -53,8 +64,8 @@ int drm_virtio_init(struct drm_driver *driver, struct virtio_device *vdev)
 	int ret;
 
 	dev = drm_dev_alloc(driver, &vdev->dev);
-	if (!dev)
-		return -ENOMEM;
+	if (IS_ERR(dev))
+		return PTR_ERR(dev);
 	dev->virtdev = vdev;
 	vdev->priv = dev;
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index c13f70cfc461..5820b7020ae5 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -117,6 +117,7 @@ static const struct file_operations virtio_gpu_driver_fops = {
 
 static struct drm_driver driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER | DRIVER_ATOMIC,
+	.set_busid = drm_virtio_set_busid,
 	.load = virtio_gpu_driver_load,
 	.unload = virtio_gpu_driver_unload,
 	.open = virtio_gpu_driver_open,
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index b18ef3111f0c..ae59080d63d1 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -49,6 +49,7 @@
 #define DRIVER_PATCHLEVEL 1
 
 /* virtgpu_drm_bus.c */
+int drm_virtio_set_busid(struct drm_device *dev, struct drm_master *master);
 int drm_virtio_init(struct drm_driver *driver, struct virtio_device *vdev);
 
 struct virtio_gpu_object {
@@ -75,6 +76,7 @@ typedef void (*virtio_gpu_resp_cb)(struct virtio_gpu_device *vgdev,
 struct virtio_gpu_fence_driver {
 	atomic64_t       last_seq;
 	uint64_t         sync_seq;
+	uint64_t         context;
 	struct list_head fences;
 	spinlock_t       lock;
 };
diff --git a/drivers/gpu/drm/virtio/virtgpu_fence.c b/drivers/gpu/drm/virtio/virtgpu_fence.c
index cf4418709e76..f3f70fa8a4c7 100644
--- a/drivers/gpu/drm/virtio/virtgpu_fence.c
+++ b/drivers/gpu/drm/virtio/virtgpu_fence.c
@@ -89,7 +89,7 @@ int virtio_gpu_fence_emit(struct virtio_gpu_device *vgdev,
 	(*fence)->drv = drv;
 	(*fence)->seq = ++drv->sync_seq;
 	fence_init(&(*fence)->f, &virtio_fence_ops, &drv->lock,
-		   0, (*fence)->seq);
+		   drv->context, (*fence)->seq);
 	fence_get(&(*fence)->f);
 	list_add_tail(&(*fence)->node, &drv->fences);
 	spin_unlock_irqrestore(&drv->lock, irq_flags);
diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
index c046903cb47b..818478b4c4f0 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
@@ -89,10 +89,16 @@ static void virtio_gpu_unref_list(struct list_head *head)
 	}
 }
 
-static int virtio_gpu_execbuffer(struct drm_device *dev,
-				 struct drm_virtgpu_execbuffer *exbuf,
+/*
+ * Usage of execbuffer:
+ * Relocations need to take into account the full VIRTIO_GPUDrawable size.
+ * However, the command as passed from user space must *not* contain the initial
+ * VIRTIO_GPUReleaseInfo struct (first XXX bytes)
+ */
+static int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
 				 struct drm_file *drm_file)
 {
+	struct drm_virtgpu_execbuffer *exbuf = data;
 	struct virtio_gpu_device *vgdev = dev->dev_private;
 	struct virtio_gpu_fpriv *vfpriv = drm_file->driver_priv;
 	struct drm_gem_object *gobj;
@@ -152,15 +158,10 @@ static int virtio_gpu_execbuffer(struct drm_device *dev,
 	if (ret)
 		goto out_free;
 
-	buf = kmalloc(exbuf->size, GFP_KERNEL);
-	if (!buf) {
-		ret = -ENOMEM;
-		goto out_unresv;
-	}
-	if (copy_from_user(buf, (void __user *)(uintptr_t)exbuf->command,
-			   exbuf->size)) {
-		kfree(buf);
-		ret = -EFAULT;
+	buf = memdup_user((void __user *)(uintptr_t)exbuf->command,
+			  exbuf->size);
+	if (IS_ERR(buf)) {
+		ret = PTR_ERR(buf);
 		goto out_unresv;
 	}
 	virtio_gpu_cmd_submit(vgdev, buf, exbuf->size,
@@ -182,20 +183,6 @@ out_free:
 	return ret;
 }
 
-/*
- * Usage of execbuffer:
- * Relocations need to take into account the full VIRTIO_GPUDrawable size.
- * However, the command as passed from user space must *not* contain the initial
- * VIRTIO_GPUReleaseInfo struct (first XXX bytes)
- */
-static int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
-				       struct drm_file *file_priv)
-{
-	struct drm_virtgpu_execbuffer *execbuffer = data;
-	return virtio_gpu_execbuffer(dev, execbuffer, file_priv);
-}
-
-
 static int virtio_gpu_getparam_ioctl(struct drm_device *dev, void *data,
 				     struct drm_file *file_priv)
 {
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
index 4150873d432e..036b0fbae0fb 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -159,6 +159,7 @@ int virtio_gpu_driver_load(struct drm_device *dev, unsigned long flags)
 	virtio_gpu_init_vq(&vgdev->ctrlq, virtio_gpu_dequeue_ctrl_func);
 	virtio_gpu_init_vq(&vgdev->cursorq, virtio_gpu_dequeue_cursor_func);
 
+	vgdev->fence_drv.context = fence_context_alloc(1);
 	spin_lock_init(&vgdev->fence_drv.lock);
 	INIT_LIST_HEAD(&vgdev->fence_drv.fences);
 	INIT_LIST_HEAD(&vgdev->cap_cache);
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index 925ca25209df..ba28c0f6f28a 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -76,7 +76,8 @@ static void virtio_gpu_primary_plane_update(struct drm_plane *plane,
 		output = drm_crtc_to_virtio_gpu_output(plane->state->crtc);
 	if (old_state->crtc)
 		output = drm_crtc_to_virtio_gpu_output(old_state->crtc);
-	WARN_ON(!output);
+	if (WARN_ON(!output))
+		return;
 
 	if (plane->state->fb) {
 		vgfb = to_virtio_gpu_framebuffer(plane->state->fb);
@@ -129,7 +130,8 @@ static void virtio_gpu_cursor_plane_update(struct drm_plane *plane,
 		output = drm_crtc_to_virtio_gpu_output(plane->state->crtc);
 	if (old_state->crtc)
 		output = drm_crtc_to_virtio_gpu_output(old_state->crtc);
-	WARN_ON(!output);
+	if (WARN_ON(!output))
+		return;
 
 	if (plane->state->fb) {
 		vgfb = to_virtio_gpu_framebuffer(plane->state->fb);
diff --git a/drivers/gpu/drm/vmwgfx/Kconfig b/drivers/gpu/drm/vmwgfx/Kconfig
index b49445df8a7e..fb7b82aad763 100644
--- a/drivers/gpu/drm/vmwgfx/Kconfig
+++ b/drivers/gpu/drm/vmwgfx/Kconfig
@@ -6,6 +6,7 @@ config DRM_VMWGFX
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
 	select DRM_TTM
+	select FB
 	# Only needed for the transitional use of drm_crtc_init - can be removed
 	# again once vmwgfx sets up the primary plane itself.
 	select DRM_KMS_HELPER
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 74304b03f9d4..070d750af16d 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -67,10 +67,10 @@
 			VMWGFX_NUM_GB_SURFACE +\
 			VMWGFX_NUM_GB_SCREEN_TARGET)
 
-#define VMW_PL_GMR TTM_PL_PRIV0
-#define VMW_PL_FLAG_GMR TTM_PL_FLAG_PRIV0
-#define VMW_PL_MOB TTM_PL_PRIV1
-#define VMW_PL_FLAG_MOB TTM_PL_FLAG_PRIV1
+#define VMW_PL_GMR (TTM_PL_PRIV + 0)
+#define VMW_PL_FLAG_GMR (TTM_PL_FLAG_PRIV << 0)
+#define VMW_PL_MOB (TTM_PL_PRIV + 1)
+#define VMW_PL_FLAG_MOB (TTM_PL_FLAG_PRIV << 1)
 
 #define VMW_RES_CONTEXT ttm_driver_type0
 #define VMW_RES_SURFACE ttm_driver_type1
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index 63ccd9871ec9..23ec673d5e16 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -377,9 +377,6 @@ static int vmw_ldu_init(struct vmw_private *dev_priv, unsigned unit)
 	drm_mode_crtc_set_gamma_size(crtc, 256);
 
 	drm_object_attach_property(&connector->base,
-				   dev->mode_config.dirty_info_property,
-				   1);
-	drm_object_attach_property(&connector->base,
 				   dev_priv->hotplug_mode_update_property, 1);
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.suggested_x_property, 0);
@@ -421,10 +418,6 @@ int vmw_kms_ldu_init_display(struct vmw_private *dev_priv)
 	if (ret != 0)
 		goto err_free;
 
-	ret = drm_mode_create_dirty_info_property(dev);
-	if (ret != 0)
-		goto err_vblank_cleanup;
-
 	vmw_kms_create_implicit_placement_property(dev_priv, true);
 
 	if (dev_priv->capabilities & SVGA_CAP_MULTIMON)
@@ -439,8 +432,6 @@ int vmw_kms_ldu_init_display(struct vmw_private *dev_priv)
 
 	return 0;
 
-err_vblank_cleanup:
-	drm_vblank_cleanup(dev);
 err_free:
 	kfree(dev_priv->ldu_priv);
 	dev_priv->ldu_priv = NULL;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index b74eae2b8594..f42359084adc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -538,9 +538,6 @@ static int vmw_sou_init(struct vmw_private *dev_priv, unsigned unit)
 	drm_mode_crtc_set_gamma_size(crtc, 256);
 
 	drm_object_attach_property(&connector->base,
-				   dev->mode_config.dirty_info_property,
-				   1);
-	drm_object_attach_property(&connector->base,
 				   dev_priv->hotplug_mode_update_property, 1);
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.suggested_x_property, 0);
@@ -574,10 +571,6 @@ int vmw_kms_sou_init_display(struct vmw_private *dev_priv)
 	if (unlikely(ret != 0))
 		return ret;
 
-	ret = drm_mode_create_dirty_info_property(dev);
-	if (unlikely(ret != 0))
-		goto err_vblank_cleanup;
-
 	vmw_kms_create_implicit_placement_property(dev_priv, false);
 
 	for (i = 0; i < VMWGFX_NUM_DISPLAY_UNITS; ++i)
@@ -588,10 +581,6 @@ int vmw_kms_sou_init_display(struct vmw_private *dev_priv)
 	DRM_INFO("Screen Objects Display Unit initialized\n");
 
 	return 0;
-
-err_vblank_cleanup:
-	drm_vblank_cleanup(dev);
-	return ret;
 }
 
 int vmw_kms_sou_close_display(struct vmw_private *dev_priv)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index 41932a7c4f79..94ad8d2acf9a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -1131,9 +1131,6 @@ static int vmw_stdu_init(struct vmw_private *dev_priv, unsigned unit)
 	drm_mode_crtc_set_gamma_size(crtc, 256);
 
 	drm_object_attach_property(&connector->base,
-				   dev->mode_config.dirty_info_property,
-				   1);
-	drm_object_attach_property(&connector->base,
 				   dev_priv->hotplug_mode_update_property, 1);
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.suggested_x_property, 0);
@@ -1202,10 +1199,6 @@ int vmw_kms_stdu_init_display(struct vmw_private *dev_priv)
 	if (unlikely(ret != 0))
 		return ret;
 
-	ret = drm_mode_create_dirty_info_property(dev);
-	if (unlikely(ret != 0))
-		goto err_vblank_cleanup;
-
 	dev_priv->active_display_unit = vmw_du_screen_target;
 
 	vmw_kms_create_implicit_placement_property(dev_priv, false);
diff --git a/drivers/gpu/host1x/mipi.c b/drivers/gpu/host1x/mipi.c
index 52a6fd224127..e00809d996a2 100644
--- a/drivers/gpu/host1x/mipi.c
+++ b/drivers/gpu/host1x/mipi.c
@@ -242,20 +242,6 @@ struct tegra_mipi_device *tegra_mipi_request(struct device *device)
 	dev->pads = args.args[0];
 	dev->device = device;
 
-	mutex_lock(&dev->mipi->lock);
-
-	if (dev->mipi->usage_count++ == 0) {
-		err = tegra_mipi_power_up(dev->mipi);
-		if (err < 0) {
-			dev_err(dev->mipi->dev,
-				"failed to power up MIPI bricks: %d\n",
-				err);
-			return ERR_PTR(err);
-		}
-	}
-
-	mutex_unlock(&dev->mipi->lock);
-
 	return dev;
 
 put:
@@ -270,29 +256,42 @@ EXPORT_SYMBOL(tegra_mipi_request);
 
 void tegra_mipi_free(struct tegra_mipi_device *device)
 {
-	int err;
+	platform_device_put(device->pdev);
+	kfree(device);
+}
+EXPORT_SYMBOL(tegra_mipi_free);
 
-	mutex_lock(&device->mipi->lock);
+int tegra_mipi_enable(struct tegra_mipi_device *dev)
+{
+	int err = 0;
 
-	if (--device->mipi->usage_count == 0) {
-		err = tegra_mipi_power_down(device->mipi);
-		if (err < 0) {
-			/*
-			 * Not much that can be done here, so an error message
-			 * will have to do.
-			 */
-			dev_err(device->mipi->dev,
-				"failed to power down MIPI bricks: %d\n",
-				err);
-		}
-	}
+	mutex_lock(&dev->mipi->lock);
 
-	mutex_unlock(&device->mipi->lock);
+	if (dev->mipi->usage_count++ == 0)
+		err = tegra_mipi_power_up(dev->mipi);
+
+	mutex_unlock(&dev->mipi->lock);
+
+	return err;
 
-	platform_device_put(device->pdev);
-	kfree(device);
 }
-EXPORT_SYMBOL(tegra_mipi_free);
+EXPORT_SYMBOL(tegra_mipi_enable);
+
+int tegra_mipi_disable(struct tegra_mipi_device *dev)
+{
+	int err = 0;
+
+	mutex_lock(&dev->mipi->lock);
+
+	if (--dev->mipi->usage_count == 0)
+		err = tegra_mipi_power_down(dev->mipi);
+
+	mutex_unlock(&dev->mipi->lock);
+
+	return err;
+
+}
+EXPORT_SYMBOL(tegra_mipi_disable);
 
 static int tegra_mipi_wait(struct tegra_mipi *mipi)
 {
diff --git a/drivers/gpu/ipu-v3/Makefile b/drivers/gpu/ipu-v3/Makefile
index 107ec236a4a6..5f961416c4ee 100644
--- a/drivers/gpu/ipu-v3/Makefile
+++ b/drivers/gpu/ipu-v3/Makefile
@@ -1,4 +1,5 @@
 obj-$(CONFIG_IMX_IPUV3_CORE) += imx-ipu-v3.o
 
 imx-ipu-v3-objs := ipu-common.o ipu-cpmem.o ipu-csi.o ipu-dc.o ipu-di.o \
-		ipu-dp.o ipu-dmfc.o ipu-ic.o ipu-smfc.o
+		ipu-dp.o ipu-dmfc.o ipu-ic.o ipu-image-convert.o \
+		ipu-smfc.o ipu-vdi.o
diff --git a/drivers/gpu/ipu-v3/ipu-common.c b/drivers/gpu/ipu-v3/ipu-common.c
index 99dcacf05b99..b9539f7c5e9a 100644
--- a/drivers/gpu/ipu-v3/ipu-common.c
+++ b/drivers/gpu/ipu-v3/ipu-common.c
@@ -45,6 +45,12 @@ static inline void ipu_cm_write(struct ipu_soc *ipu, u32 value, unsigned offset)
 	writel(value, ipu->cm_reg + offset);
 }
 
+int ipu_get_num(struct ipu_soc *ipu)
+{
+	return ipu->id;
+}
+EXPORT_SYMBOL_GPL(ipu_get_num);
+
 void ipu_srm_dp_sync_update(struct ipu_soc *ipu)
 {
 	u32 val;
@@ -724,6 +730,137 @@ void ipu_set_ic_src_mux(struct ipu_soc *ipu, int csi_id, bool vdi)
 }
 EXPORT_SYMBOL_GPL(ipu_set_ic_src_mux);
 
+
+/* Frame Synchronization Unit Channel Linking */
+
+struct fsu_link_reg_info {
+	int chno;
+	u32 reg;
+	u32 mask;
+	u32 val;
+};
+
+struct fsu_link_info {
+	struct fsu_link_reg_info src;
+	struct fsu_link_reg_info sink;
+};
+
+static const struct fsu_link_info fsu_link_info[] = {
+	{
+		.src  = { IPUV3_CHANNEL_IC_PRP_ENC_MEM, IPU_FS_PROC_FLOW2,
+			  FS_PRP_ENC_DEST_SEL_MASK, FS_PRP_ENC_DEST_SEL_IRT_ENC },
+		.sink = { IPUV3_CHANNEL_MEM_ROT_ENC, IPU_FS_PROC_FLOW1,
+			  FS_PRPENC_ROT_SRC_SEL_MASK, FS_PRPENC_ROT_SRC_SEL_ENC },
+	}, {
+		.src =  { IPUV3_CHANNEL_IC_PRP_VF_MEM, IPU_FS_PROC_FLOW2,
+			  FS_PRPVF_DEST_SEL_MASK, FS_PRPVF_DEST_SEL_IRT_VF },
+		.sink = { IPUV3_CHANNEL_MEM_ROT_VF, IPU_FS_PROC_FLOW1,
+			  FS_PRPVF_ROT_SRC_SEL_MASK, FS_PRPVF_ROT_SRC_SEL_VF },
+	}, {
+		.src =  { IPUV3_CHANNEL_IC_PP_MEM, IPU_FS_PROC_FLOW2,
+			  FS_PP_DEST_SEL_MASK, FS_PP_DEST_SEL_IRT_PP },
+		.sink = { IPUV3_CHANNEL_MEM_ROT_PP, IPU_FS_PROC_FLOW1,
+			  FS_PP_ROT_SRC_SEL_MASK, FS_PP_ROT_SRC_SEL_PP },
+	}, {
+		.src =  { IPUV3_CHANNEL_CSI_DIRECT, 0 },
+		.sink = { IPUV3_CHANNEL_CSI_VDI_PREV, IPU_FS_PROC_FLOW1,
+			  FS_VDI_SRC_SEL_MASK, FS_VDI_SRC_SEL_CSI_DIRECT },
+	},
+};
+
+static const struct fsu_link_info *find_fsu_link_info(int src, int sink)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(fsu_link_info); i++) {
+		if (src == fsu_link_info[i].src.chno &&
+		    sink == fsu_link_info[i].sink.chno)
+			return &fsu_link_info[i];
+	}
+
+	return NULL;
+}
+
+/*
+ * Links a source channel to a sink channel in the FSU.
+ */
+int ipu_fsu_link(struct ipu_soc *ipu, int src_ch, int sink_ch)
+{
+	const struct fsu_link_info *link;
+	u32 src_reg, sink_reg;
+	unsigned long flags;
+
+	link = find_fsu_link_info(src_ch, sink_ch);
+	if (!link)
+		return -EINVAL;
+
+	spin_lock_irqsave(&ipu->lock, flags);
+
+	if (link->src.mask) {
+		src_reg = ipu_cm_read(ipu, link->src.reg);
+		src_reg &= ~link->src.mask;
+		src_reg |= link->src.val;
+		ipu_cm_write(ipu, src_reg, link->src.reg);
+	}
+
+	if (link->sink.mask) {
+		sink_reg = ipu_cm_read(ipu, link->sink.reg);
+		sink_reg &= ~link->sink.mask;
+		sink_reg |= link->sink.val;
+		ipu_cm_write(ipu, sink_reg, link->sink.reg);
+	}
+
+	spin_unlock_irqrestore(&ipu->lock, flags);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipu_fsu_link);
+
+/*
+ * Unlinks source and sink channels in the FSU.
+ */
+int ipu_fsu_unlink(struct ipu_soc *ipu, int src_ch, int sink_ch)
+{
+	const struct fsu_link_info *link;
+	u32 src_reg, sink_reg;
+	unsigned long flags;
+
+	link = find_fsu_link_info(src_ch, sink_ch);
+	if (!link)
+		return -EINVAL;
+
+	spin_lock_irqsave(&ipu->lock, flags);
+
+	if (link->src.mask) {
+		src_reg = ipu_cm_read(ipu, link->src.reg);
+		src_reg &= ~link->src.mask;
+		ipu_cm_write(ipu, src_reg, link->src.reg);
+	}
+
+	if (link->sink.mask) {
+		sink_reg = ipu_cm_read(ipu, link->sink.reg);
+		sink_reg &= ~link->sink.mask;
+		ipu_cm_write(ipu, sink_reg, link->sink.reg);
+	}
+
+	spin_unlock_irqrestore(&ipu->lock, flags);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipu_fsu_unlink);
+
+/* Link IDMAC channels in the FSU */
+int ipu_idmac_link(struct ipuv3_channel *src, struct ipuv3_channel *sink)
+{
+	return ipu_fsu_link(src->ipu, src->num, sink->num);
+}
+EXPORT_SYMBOL_GPL(ipu_idmac_link);
+
+/* Unlink IDMAC channels in the FSU */
+int ipu_idmac_unlink(struct ipuv3_channel *src, struct ipuv3_channel *sink)
+{
+	return ipu_fsu_unlink(src->ipu, src->num, sink->num);
+}
+EXPORT_SYMBOL_GPL(ipu_idmac_unlink);
+
 struct ipu_devtype {
 	const char *name;
 	unsigned long cm_ofs;
@@ -833,6 +970,20 @@ static int ipu_submodules_init(struct ipu_soc *ipu,
 		goto err_ic;
 	}
 
+	ret = ipu_vdi_init(ipu, dev, ipu_base + devtype->vdi_ofs,
+			   IPU_CONF_VDI_EN | IPU_CONF_ISP_EN |
+			   IPU_CONF_IC_INPUT);
+	if (ret) {
+		unit = "vdi";
+		goto err_vdi;
+	}
+
+	ret = ipu_image_convert_init(ipu, dev);
+	if (ret) {
+		unit = "image_convert";
+		goto err_image_convert;
+	}
+
 	ret = ipu_di_init(ipu, dev, 0, ipu_base + devtype->disp0_ofs,
 			  IPU_CONF_DI0_EN, ipu_clk);
 	if (ret) {
@@ -887,6 +1038,10 @@ err_dc:
 err_di_1:
 	ipu_di_exit(ipu, 0);
 err_di_0:
+	ipu_image_convert_exit(ipu);
+err_image_convert:
+	ipu_vdi_exit(ipu);
+err_vdi:
 	ipu_ic_exit(ipu);
 err_ic:
 	ipu_csi_exit(ipu, 1);
@@ -971,6 +1126,8 @@ static void ipu_submodules_exit(struct ipu_soc *ipu)
 	ipu_dc_exit(ipu);
 	ipu_di_exit(ipu, 1);
 	ipu_di_exit(ipu, 0);
+	ipu_image_convert_exit(ipu);
+	ipu_vdi_exit(ipu);
 	ipu_ic_exit(ipu);
 	ipu_csi_exit(ipu, 1);
 	ipu_csi_exit(ipu, 0);
@@ -1004,14 +1161,14 @@ static struct ipu_platform_reg client_reg[] = {
 			.dma[0] = IPUV3_CHANNEL_CSI0,
 			.dma[1] = -EINVAL,
 		},
-		.name = "imx-ipuv3-camera",
+		.name = "imx-ipuv3-csi",
 	}, {
 		.pdata = {
 			.csi = 1,
 			.dma[0] = IPUV3_CHANNEL_CSI1,
 			.dma[1] = -EINVAL,
 		},
-		.name = "imx-ipuv3-camera",
+		.name = "imx-ipuv3-csi",
 	}, {
 		.pdata = {
 			.di = 0,
@@ -1207,15 +1364,16 @@ EXPORT_SYMBOL_GPL(ipu_dump);
 
 static int ipu_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *of_id =
-			of_match_device(imx_ipu_dt_ids, &pdev->dev);
+	struct device_node *np = pdev->dev.of_node;
 	struct ipu_soc *ipu;
 	struct resource *res;
 	unsigned long ipu_base;
 	int i, ret, irq_sync, irq_err;
 	const struct ipu_devtype *devtype;
 
-	devtype = of_id->data;
+	devtype = of_device_get_match_data(&pdev->dev);
+	if (!devtype)
+		return -EINVAL;
 
 	irq_sync = platform_get_irq(pdev, 0);
 	irq_err = platform_get_irq(pdev, 1);
@@ -1237,6 +1395,7 @@ static int ipu_probe(struct platform_device *pdev)
 		ipu->channel[i].ipu = ipu;
 	ipu->devtype = devtype;
 	ipu->ipu_type = devtype->type;
+	ipu->id = of_alias_get_id(np, "ipu");
 
 	spin_lock_init(&ipu->lock);
 	mutex_init(&ipu->channel_lock);
diff --git a/drivers/gpu/ipu-v3/ipu-cpmem.c b/drivers/gpu/ipu-v3/ipu-cpmem.c
index 6494a4d28171..fcb7dc86167b 100644
--- a/drivers/gpu/ipu-v3/ipu-cpmem.c
+++ b/drivers/gpu/ipu-v3/ipu-cpmem.c
@@ -253,6 +253,13 @@ void ipu_cpmem_set_buffer(struct ipuv3_channel *ch, int bufnum, dma_addr_t buf)
 }
 EXPORT_SYMBOL_GPL(ipu_cpmem_set_buffer);
 
+void ipu_cpmem_set_uv_offset(struct ipuv3_channel *ch, u32 u_off, u32 v_off)
+{
+	ipu_ch_param_write_field(ch, IPU_FIELD_UBO, u_off / 8);
+	ipu_ch_param_write_field(ch, IPU_FIELD_VBO, v_off / 8);
+}
+EXPORT_SYMBOL_GPL(ipu_cpmem_set_uv_offset);
+
 void ipu_cpmem_interlaced_scan(struct ipuv3_channel *ch, int stride)
 {
 	ipu_ch_param_write_field(ch, IPU_FIELD_SO, 1);
@@ -268,6 +275,12 @@ void ipu_cpmem_set_axi_id(struct ipuv3_channel *ch, u32 id)
 }
 EXPORT_SYMBOL_GPL(ipu_cpmem_set_axi_id);
 
+int ipu_cpmem_get_burstsize(struct ipuv3_channel *ch)
+{
+	return ipu_ch_param_read_field(ch, IPU_FIELD_NPB) + 1;
+}
+EXPORT_SYMBOL_GPL(ipu_cpmem_get_burstsize);
+
 void ipu_cpmem_set_burstsize(struct ipuv3_channel *ch, int burstsize)
 {
 	ipu_ch_param_write_field(ch, IPU_FIELD_NPB, burstsize - 1);
diff --git a/drivers/gpu/ipu-v3/ipu-csi.c b/drivers/gpu/ipu-v3/ipu-csi.c
index 06631ac61b04..d6e5ded24418 100644
--- a/drivers/gpu/ipu-v3/ipu-csi.c
+++ b/drivers/gpu/ipu-v3/ipu-csi.c
@@ -258,12 +258,8 @@ static int mbus_code_to_bus_cfg(struct ipu_csi_bus_config *cfg, u32 mbus_code)
 		cfg->data_width = IPU_CSI_DATA_WIDTH_8;
 		break;
 	case MEDIA_BUS_FMT_UYVY8_1X16:
-		cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_YUV422_UYVY;
-		cfg->mipi_dt = MIPI_DT_YUV422;
-		cfg->data_width = IPU_CSI_DATA_WIDTH_16;
-		break;
 	case MEDIA_BUS_FMT_YUYV8_1X16:
-		cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_YUV422_YUYV;
+		cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_BAYER;
 		cfg->mipi_dt = MIPI_DT_YUV422;
 		cfg->data_width = IPU_CSI_DATA_WIDTH_16;
 		break;
@@ -365,10 +361,14 @@ int ipu_csi_init_interface(struct ipu_csi *csi,
 {
 	struct ipu_csi_bus_config cfg;
 	unsigned long flags;
-	u32 data = 0;
+	u32 width, height, data = 0;
 
 	fill_csi_bus_cfg(&cfg, mbus_cfg, mbus_fmt);
 
+	/* set default sensor frame width and height */
+	width = mbus_fmt->width;
+	height = mbus_fmt->height;
+
 	/* Set the CSI_SENS_CONF register remaining fields */
 	data |= cfg.data_width << CSI_SENS_CONF_DATA_WIDTH_SHIFT |
 		cfg.data_fmt << CSI_SENS_CONF_DATA_FMT_SHIFT |
@@ -386,11 +386,6 @@ int ipu_csi_init_interface(struct ipu_csi *csi,
 
 	ipu_csi_write(csi, data, CSI_SENS_CONF);
 
-	/* Setup sensor frame size */
-	ipu_csi_write(csi,
-		      (mbus_fmt->width - 1) | ((mbus_fmt->height - 1) << 16),
-		      CSI_SENS_FRM_SIZE);
-
 	/* Set CCIR registers */
 
 	switch (cfg.clk_mode) {
@@ -408,11 +403,12 @@ int ipu_csi_init_interface(struct ipu_csi *csi,
 			 * Field1BlankEnd = 0x7, Field1BlankStart = 0x3,
 			 * Field1ActiveEnd = 0x5, Field1ActiveStart = 0x1
 			 */
+			height = 625; /* framelines for PAL */
+
 			ipu_csi_write(csi, 0x40596 | CSI_CCIR_ERR_DET_EN,
 					  CSI_CCIR_CODE_1);
 			ipu_csi_write(csi, 0xD07DF, CSI_CCIR_CODE_2);
 			ipu_csi_write(csi, 0xFF0000, CSI_CCIR_CODE_3);
-
 		} else if (mbus_fmt->width == 720 && mbus_fmt->height == 480) {
 			/*
 			 * NTSC case
@@ -422,6 +418,8 @@ int ipu_csi_init_interface(struct ipu_csi *csi,
 			 * Field1BlankEnd = 0x6, Field1BlankStart = 0x2,
 			 * Field1ActiveEnd = 0x4, Field1ActiveStart = 0
 			 */
+			height = 525; /* framelines for NTSC */
+
 			ipu_csi_write(csi, 0xD07DF | CSI_CCIR_ERR_DET_EN,
 					  CSI_CCIR_CODE_1);
 			ipu_csi_write(csi, 0x40596, CSI_CCIR_CODE_2);
@@ -447,6 +445,10 @@ int ipu_csi_init_interface(struct ipu_csi *csi,
 		break;
 	}
 
+	/* Setup sensor frame size */
+	ipu_csi_write(csi, (width - 1) | ((height - 1) << 16),
+		      CSI_SENS_FRM_SIZE);
+
 	dev_dbg(csi->ipu->dev, "CSI_SENS_CONF = 0x%08X\n",
 		ipu_csi_read(csi, CSI_SENS_CONF));
 	dev_dbg(csi->ipu->dev, "CSI_ACT_FRM_SIZE = 0x%08X\n",
diff --git a/drivers/gpu/ipu-v3/ipu-dmfc.c b/drivers/gpu/ipu-v3/ipu-dmfc.c
index 42705bb5aaa3..a40f211f382f 100644
--- a/drivers/gpu/ipu-v3/ipu-dmfc.c
+++ b/drivers/gpu/ipu-v3/ipu-dmfc.c
@@ -123,20 +123,6 @@ int ipu_dmfc_enable_channel(struct dmfc_channel *dmfc)
 }
 EXPORT_SYMBOL_GPL(ipu_dmfc_enable_channel);
 
-static void ipu_dmfc_wait_fifos(struct ipu_dmfc_priv *priv)
-{
-	unsigned long timeout = jiffies + msecs_to_jiffies(1000);
-
-	while ((readl(priv->base + DMFC_STAT) & 0x02fff000) != 0x02fff000) {
-		if (time_after(jiffies, timeout)) {
-			dev_warn(priv->dev,
-				 "Timeout waiting for DMFC FIFOs to clear\n");
-			break;
-		}
-		cpu_relax();
-	}
-}
-
 void ipu_dmfc_disable_channel(struct dmfc_channel *dmfc)
 {
 	struct ipu_dmfc_priv *priv = dmfc->priv;
@@ -145,10 +131,8 @@ void ipu_dmfc_disable_channel(struct dmfc_channel *dmfc)
 
 	priv->use_count--;
 
-	if (!priv->use_count) {
-		ipu_dmfc_wait_fifos(priv);
+	if (!priv->use_count)
 		ipu_module_disable(priv->ipu, IPU_CONF_DMFC_EN);
-	}
 
 	if (priv->use_count < 0)
 		priv->use_count = 0;
diff --git a/drivers/gpu/ipu-v3/ipu-ic.c b/drivers/gpu/ipu-v3/ipu-ic.c
index 1dcb96ccda66..321eb983c2f5 100644
--- a/drivers/gpu/ipu-v3/ipu-ic.c
+++ b/drivers/gpu/ipu-v3/ipu-ic.c
@@ -160,6 +160,7 @@ struct ipu_ic_priv {
 	spinlock_t lock;
 	struct ipu_soc *ipu;
 	int use_count;
+	int irt_use_count;
 	struct ipu_ic task[IC_NUM_TASKS];
 };
 
@@ -379,8 +380,6 @@ void ipu_ic_task_disable(struct ipu_ic *ic)
 
 	ipu_ic_write(ic, ic_conf, IC_CONF);
 
-	ic->rotation = ic->graphics = false;
-
 	spin_unlock_irqrestore(&priv->lock, flags);
 }
 EXPORT_SYMBOL_GPL(ipu_ic_task_disable);
@@ -620,7 +619,7 @@ int ipu_ic_task_idma_init(struct ipu_ic *ic, struct ipuv3_channel *channel,
 	ipu_ic_write(ic, ic_idmac_2, IC_IDMAC_2);
 	ipu_ic_write(ic, ic_idmac_3, IC_IDMAC_3);
 
-	if (rot >= IPU_ROTATE_90_RIGHT)
+	if (ipu_rot_mode_is_irt(rot))
 		ic->rotation = true;
 
 unlock:
@@ -629,22 +628,41 @@ unlock:
 }
 EXPORT_SYMBOL_GPL(ipu_ic_task_idma_init);
 
+static void ipu_irt_enable(struct ipu_ic *ic)
+{
+	struct ipu_ic_priv *priv = ic->priv;
+
+	if (!priv->irt_use_count)
+		ipu_module_enable(priv->ipu, IPU_CONF_ROT_EN);
+
+	priv->irt_use_count++;
+}
+
+static void ipu_irt_disable(struct ipu_ic *ic)
+{
+	struct ipu_ic_priv *priv = ic->priv;
+
+	if (priv->irt_use_count) {
+		if (!--priv->irt_use_count)
+			ipu_module_disable(priv->ipu, IPU_CONF_ROT_EN);
+	}
+}
+
 int ipu_ic_enable(struct ipu_ic *ic)
 {
 	struct ipu_ic_priv *priv = ic->priv;
 	unsigned long flags;
-	u32 module = IPU_CONF_IC_EN;
 
 	spin_lock_irqsave(&priv->lock, flags);
 
-	if (ic->rotation)
-		module |= IPU_CONF_ROT_EN;
-
 	if (!priv->use_count)
-		ipu_module_enable(priv->ipu, module);
+		ipu_module_enable(priv->ipu, IPU_CONF_IC_EN);
 
 	priv->use_count++;
 
+	if (ic->rotation)
+		ipu_irt_enable(ic);
+
 	spin_unlock_irqrestore(&priv->lock, flags);
 
 	return 0;
@@ -655,18 +673,22 @@ int ipu_ic_disable(struct ipu_ic *ic)
 {
 	struct ipu_ic_priv *priv = ic->priv;
 	unsigned long flags;
-	u32 module = IPU_CONF_IC_EN | IPU_CONF_ROT_EN;
 
 	spin_lock_irqsave(&priv->lock, flags);
 
 	priv->use_count--;
 
 	if (!priv->use_count)
-		ipu_module_disable(priv->ipu, module);
+		ipu_module_disable(priv->ipu, IPU_CONF_IC_EN);
 
 	if (priv->use_count < 0)
 		priv->use_count = 0;
 
+	if (ic->rotation)
+		ipu_irt_disable(ic);
+
+	ic->rotation = ic->graphics = false;
+
 	spin_unlock_irqrestore(&priv->lock, flags);
 
 	return 0;
diff --git a/drivers/gpu/ipu-v3/ipu-image-convert.c b/drivers/gpu/ipu-v3/ipu-image-convert.c
new file mode 100644
index 000000000000..2ba7d437a2af
--- /dev/null
+++ b/drivers/gpu/ipu-v3/ipu-image-convert.c
@@ -0,0 +1,1709 @@
+/*
+ * Copyright (C) 2012-2016 Mentor Graphics Inc.
+ *
+ * Queued image conversion support, with tiling and rotation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include <video/imx-ipu-image-convert.h>
+#include "ipu-prv.h"
+
+/*
+ * The IC Resizer has a restriction that the output frame from the
+ * resizer must be 1024 or less in both width (pixels) and height
+ * (lines).
+ *
+ * The image converter attempts to split up a conversion when
+ * the desired output (converted) frame resolution exceeds the
+ * IC resizer limit of 1024 in either dimension.
+ *
+ * If either dimension of the output frame exceeds the limit, the
+ * dimension is split into 1, 2, or 4 equal stripes, for a maximum
+ * of 4*4 or 16 tiles. A conversion is then carried out for each
+ * tile (but taking care to pass the full frame stride length to
+ * the DMA channel's parameter memory!). IDMA double-buffering is used
+ * to convert each tile back-to-back when possible (see note below
+ * when double_buffering boolean is set).
+ *
+ * Note that the input frame must be split up into the same number
+ * of tiles as the output frame.
+ *
+ * FIXME: at this point there is no attempt to deal with visible seams
+ * at the tile boundaries when upscaling. The seams are caused by a reset
+ * of the bilinear upscale interpolation when starting a new tile. The
+ * seams are barely visible for small upscale factors, but become
+ * increasingly visible as the upscale factor gets larger, since more
+ * interpolated pixels get thrown out at the tile boundaries. A possilble
+ * fix might be to overlap tiles of different sizes, but this must be done
+ * while also maintaining the IDMAC dma buffer address alignment and 8x8 IRT
+ * alignment restrictions of each tile.
+ */
+
+#define MAX_STRIPES_W    4
+#define MAX_STRIPES_H    4
+#define MAX_TILES (MAX_STRIPES_W * MAX_STRIPES_H)
+
+#define MIN_W     16
+#define MIN_H     8
+#define MAX_W     4096
+#define MAX_H     4096
+
+enum ipu_image_convert_type {
+	IMAGE_CONVERT_IN = 0,
+	IMAGE_CONVERT_OUT,
+};
+
+struct ipu_image_convert_dma_buf {
+	void          *virt;
+	dma_addr_t    phys;
+	unsigned long len;
+};
+
+struct ipu_image_convert_dma_chan {
+	int in;
+	int out;
+	int rot_in;
+	int rot_out;
+	int vdi_in_p;
+	int vdi_in;
+	int vdi_in_n;
+};
+
+/* dimensions of one tile */
+struct ipu_image_tile {
+	u32 width;
+	u32 height;
+	/* size and strides are in bytes */
+	u32 size;
+	u32 stride;
+	u32 rot_stride;
+	/* start Y or packed offset of this tile */
+	u32 offset;
+	/* offset from start to tile in U plane, for planar formats */
+	u32 u_off;
+	/* offset from start to tile in V plane, for planar formats */
+	u32 v_off;
+};
+
+struct ipu_image_convert_image {
+	struct ipu_image base;
+	enum ipu_image_convert_type type;
+
+	const struct ipu_image_pixfmt *fmt;
+	unsigned int stride;
+
+	/* # of rows (horizontal stripes) if dest height is > 1024 */
+	unsigned int num_rows;
+	/* # of columns (vertical stripes) if dest width is > 1024 */
+	unsigned int num_cols;
+
+	struct ipu_image_tile tile[MAX_TILES];
+};
+
+struct ipu_image_pixfmt {
+	u32	fourcc;        /* V4L2 fourcc */
+	int     bpp;           /* total bpp */
+	int     uv_width_dec;  /* decimation in width for U/V planes */
+	int     uv_height_dec; /* decimation in height for U/V planes */
+	bool    planar;        /* planar format */
+	bool    uv_swapped;    /* U and V planes are swapped */
+	bool    uv_packed;     /* partial planar (U and V in same plane) */
+};
+
+struct ipu_image_convert_ctx;
+struct ipu_image_convert_chan;
+struct ipu_image_convert_priv;
+
+struct ipu_image_convert_ctx {
+	struct ipu_image_convert_chan *chan;
+
+	ipu_image_convert_cb_t complete;
+	void *complete_context;
+
+	/* Source/destination image data and rotation mode */
+	struct ipu_image_convert_image in;
+	struct ipu_image_convert_image out;
+	enum ipu_rotate_mode rot_mode;
+
+	/* intermediate buffer for rotation */
+	struct ipu_image_convert_dma_buf rot_intermediate[2];
+
+	/* current buffer number for double buffering */
+	int cur_buf_num;
+
+	bool aborting;
+	struct completion aborted;
+
+	/* can we use double-buffering for this conversion operation? */
+	bool double_buffering;
+	/* num_rows * num_cols */
+	unsigned int num_tiles;
+	/* next tile to process */
+	unsigned int next_tile;
+	/* where to place converted tile in dest image */
+	unsigned int out_tile_map[MAX_TILES];
+
+	struct list_head list;
+};
+
+struct ipu_image_convert_chan {
+	struct ipu_image_convert_priv *priv;
+
+	enum ipu_ic_task ic_task;
+	const struct ipu_image_convert_dma_chan *dma_ch;
+
+	struct ipu_ic *ic;
+	struct ipuv3_channel *in_chan;
+	struct ipuv3_channel *out_chan;
+	struct ipuv3_channel *rotation_in_chan;
+	struct ipuv3_channel *rotation_out_chan;
+
+	/* the IPU end-of-frame irqs */
+	int out_eof_irq;
+	int rot_out_eof_irq;
+
+	spinlock_t irqlock;
+
+	/* list of convert contexts */
+	struct list_head ctx_list;
+	/* queue of conversion runs */
+	struct list_head pending_q;
+	/* queue of completed runs */
+	struct list_head done_q;
+
+	/* the current conversion run */
+	struct ipu_image_convert_run *current_run;
+};
+
+struct ipu_image_convert_priv {
+	struct ipu_image_convert_chan chan[IC_NUM_TASKS];
+	struct ipu_soc *ipu;
+};
+
+static const struct ipu_image_convert_dma_chan
+image_convert_dma_chan[IC_NUM_TASKS] = {
+	[IC_TASK_VIEWFINDER] = {
+		.in = IPUV3_CHANNEL_MEM_IC_PRP_VF,
+		.out = IPUV3_CHANNEL_IC_PRP_VF_MEM,
+		.rot_in = IPUV3_CHANNEL_MEM_ROT_VF,
+		.rot_out = IPUV3_CHANNEL_ROT_VF_MEM,
+		.vdi_in_p = IPUV3_CHANNEL_MEM_VDI_PREV,
+		.vdi_in = IPUV3_CHANNEL_MEM_VDI_CUR,
+		.vdi_in_n = IPUV3_CHANNEL_MEM_VDI_NEXT,
+	},
+	[IC_TASK_POST_PROCESSOR] = {
+		.in = IPUV3_CHANNEL_MEM_IC_PP,
+		.out = IPUV3_CHANNEL_IC_PP_MEM,
+		.rot_in = IPUV3_CHANNEL_MEM_ROT_PP,
+		.rot_out = IPUV3_CHANNEL_ROT_PP_MEM,
+	},
+};
+
+static const struct ipu_image_pixfmt image_convert_formats[] = {
+	{
+		.fourcc	= V4L2_PIX_FMT_RGB565,
+		.bpp    = 16,
+	}, {
+		.fourcc	= V4L2_PIX_FMT_RGB24,
+		.bpp    = 24,
+	}, {
+		.fourcc	= V4L2_PIX_FMT_BGR24,
+		.bpp    = 24,
+	}, {
+		.fourcc	= V4L2_PIX_FMT_RGB32,
+		.bpp    = 32,
+	}, {
+		.fourcc	= V4L2_PIX_FMT_BGR32,
+		.bpp    = 32,
+	}, {
+		.fourcc	= V4L2_PIX_FMT_YUYV,
+		.bpp    = 16,
+		.uv_width_dec = 2,
+		.uv_height_dec = 1,
+	}, {
+		.fourcc	= V4L2_PIX_FMT_UYVY,
+		.bpp    = 16,
+		.uv_width_dec = 2,
+		.uv_height_dec = 1,
+	}, {
+		.fourcc	= V4L2_PIX_FMT_YUV420,
+		.bpp    = 12,
+		.planar = true,
+		.uv_width_dec = 2,
+		.uv_height_dec = 2,
+	}, {
+		.fourcc	= V4L2_PIX_FMT_YVU420,
+		.bpp    = 12,
+		.planar = true,
+		.uv_width_dec = 2,
+		.uv_height_dec = 2,
+		.uv_swapped = true,
+	}, {
+		.fourcc = V4L2_PIX_FMT_NV12,
+		.bpp    = 12,
+		.planar = true,
+		.uv_width_dec = 2,
+		.uv_height_dec = 2,
+		.uv_packed = true,
+	}, {
+		.fourcc = V4L2_PIX_FMT_YUV422P,
+		.bpp    = 16,
+		.planar = true,
+		.uv_width_dec = 2,
+		.uv_height_dec = 1,
+	}, {
+		.fourcc = V4L2_PIX_FMT_NV16,
+		.bpp    = 16,
+		.planar = true,
+		.uv_width_dec = 2,
+		.uv_height_dec = 1,
+		.uv_packed = true,
+	},
+};
+
+static const struct ipu_image_pixfmt *get_format(u32 fourcc)
+{
+	const struct ipu_image_pixfmt *ret = NULL;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(image_convert_formats); i++) {
+		if (image_convert_formats[i].fourcc == fourcc) {
+			ret = &image_convert_formats[i];
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static void dump_format(struct ipu_image_convert_ctx *ctx,
+			struct ipu_image_convert_image *ic_image)
+{
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_convert_priv *priv = chan->priv;
+
+	dev_dbg(priv->ipu->dev,
+		"task %u: ctx %p: %s format: %dx%d (%dx%d tiles of size %dx%d), %c%c%c%c\n",
+		chan->ic_task, ctx,
+		ic_image->type == IMAGE_CONVERT_OUT ? "Output" : "Input",
+		ic_image->base.pix.width, ic_image->base.pix.height,
+		ic_image->num_cols, ic_image->num_rows,
+		ic_image->tile[0].width, ic_image->tile[0].height,
+		ic_image->fmt->fourcc & 0xff,
+		(ic_image->fmt->fourcc >> 8) & 0xff,
+		(ic_image->fmt->fourcc >> 16) & 0xff,
+		(ic_image->fmt->fourcc >> 24) & 0xff);
+}
+
+int ipu_image_convert_enum_format(int index, u32 *fourcc)
+{
+	const struct ipu_image_pixfmt *fmt;
+
+	if (index >= (int)ARRAY_SIZE(image_convert_formats))
+		return -EINVAL;
+
+	/* Format found */
+	fmt = &image_convert_formats[index];
+	*fourcc = fmt->fourcc;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipu_image_convert_enum_format);
+
+static void free_dma_buf(struct ipu_image_convert_priv *priv,
+			 struct ipu_image_convert_dma_buf *buf)
+{
+	if (buf->virt)
+		dma_free_coherent(priv->ipu->dev,
+				  buf->len, buf->virt, buf->phys);
+	buf->virt = NULL;
+	buf->phys = 0;
+}
+
+static int alloc_dma_buf(struct ipu_image_convert_priv *priv,
+			 struct ipu_image_convert_dma_buf *buf,
+			 int size)
+{
+	buf->len = PAGE_ALIGN(size);
+	buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys,
+				       GFP_DMA | GFP_KERNEL);
+	if (!buf->virt) {
+		dev_err(priv->ipu->dev, "failed to alloc dma buffer\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static inline int num_stripes(int dim)
+{
+	if (dim <= 1024)
+		return 1;
+	else if (dim <= 2048)
+		return 2;
+	else
+		return 4;
+}
+
+static void calc_tile_dimensions(struct ipu_image_convert_ctx *ctx,
+				 struct ipu_image_convert_image *image)
+{
+	int i;
+
+	for (i = 0; i < ctx->num_tiles; i++) {
+		struct ipu_image_tile *tile = &image->tile[i];
+
+		tile->height = image->base.pix.height / image->num_rows;
+		tile->width = image->base.pix.width / image->num_cols;
+		tile->size = ((tile->height * image->fmt->bpp) >> 3) *
+			tile->width;
+
+		if (image->fmt->planar) {
+			tile->stride = tile->width;
+			tile->rot_stride = tile->height;
+		} else {
+			tile->stride =
+				(image->fmt->bpp * tile->width) >> 3;
+			tile->rot_stride =
+				(image->fmt->bpp * tile->height) >> 3;
+		}
+	}
+}
+
+/*
+ * Use the rotation transformation to find the tile coordinates
+ * (row, col) of a tile in the destination frame that corresponds
+ * to the given tile coordinates of a source frame. The destination
+ * coordinate is then converted to a tile index.
+ */
+static int transform_tile_index(struct ipu_image_convert_ctx *ctx,
+				int src_row, int src_col)
+{
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_convert_priv *priv = chan->priv;
+	struct ipu_image_convert_image *s_image = &ctx->in;
+	struct ipu_image_convert_image *d_image = &ctx->out;
+	int dst_row, dst_col;
+
+	/* with no rotation it's a 1:1 mapping */
+	if (ctx->rot_mode == IPU_ROTATE_NONE)
+		return src_row * s_image->num_cols + src_col;
+
+	/*
+	 * before doing the transform, first we have to translate
+	 * source row,col for an origin in the center of s_image
+	 */
+	src_row = src_row * 2 - (s_image->num_rows - 1);
+	src_col = src_col * 2 - (s_image->num_cols - 1);
+
+	/* do the rotation transform */
+	if (ctx->rot_mode & IPU_ROT_BIT_90) {
+		dst_col = -src_row;
+		dst_row = src_col;
+	} else {
+		dst_col = src_col;
+		dst_row = src_row;
+	}
+
+	/* apply flip */
+	if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
+		dst_col = -dst_col;
+	if (ctx->rot_mode & IPU_ROT_BIT_VFLIP)
+		dst_row = -dst_row;
+
+	dev_dbg(priv->ipu->dev, "task %u: ctx %p: [%d,%d] --> [%d,%d]\n",
+		chan->ic_task, ctx, src_col, src_row, dst_col, dst_row);
+
+	/*
+	 * finally translate dest row,col using an origin in upper
+	 * left of d_image
+	 */
+	dst_row += d_image->num_rows - 1;
+	dst_col += d_image->num_cols - 1;
+	dst_row /= 2;
+	dst_col /= 2;
+
+	return dst_row * d_image->num_cols + dst_col;
+}
+
+/*
+ * Fill the out_tile_map[] with transformed destination tile indeces.
+ */
+static void calc_out_tile_map(struct ipu_image_convert_ctx *ctx)
+{
+	struct ipu_image_convert_image *s_image = &ctx->in;
+	unsigned int row, col, tile = 0;
+
+	for (row = 0; row < s_image->num_rows; row++) {
+		for (col = 0; col < s_image->num_cols; col++) {
+			ctx->out_tile_map[tile] =
+				transform_tile_index(ctx, row, col);
+			tile++;
+		}
+	}
+}
+
+static void calc_tile_offsets_planar(struct ipu_image_convert_ctx *ctx,
+				     struct ipu_image_convert_image *image)
+{
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_convert_priv *priv = chan->priv;
+	const struct ipu_image_pixfmt *fmt = image->fmt;
+	unsigned int row, col, tile = 0;
+	u32 H, w, h, y_stride, uv_stride;
+	u32 uv_row_off, uv_col_off, uv_off, u_off, v_off, tmp;
+	u32 y_row_off, y_col_off, y_off;
+	u32 y_size, uv_size;
+
+	/* setup some convenience vars */
+	H = image->base.pix.height;
+
+	y_stride = image->stride;
+	uv_stride = y_stride / fmt->uv_width_dec;
+	if (fmt->uv_packed)
+		uv_stride *= 2;
+
+	y_size = H * y_stride;
+	uv_size = y_size / (fmt->uv_width_dec * fmt->uv_height_dec);
+
+	for (row = 0; row < image->num_rows; row++) {
+		w = image->tile[tile].width;
+		h = image->tile[tile].height;
+		y_row_off = row * h * y_stride;
+		uv_row_off = (row * h * uv_stride) / fmt->uv_height_dec;
+
+		for (col = 0; col < image->num_cols; col++) {
+			y_col_off = col * w;
+			uv_col_off = y_col_off / fmt->uv_width_dec;
+			if (fmt->uv_packed)
+				uv_col_off *= 2;
+
+			y_off = y_row_off + y_col_off;
+			uv_off = uv_row_off + uv_col_off;
+
+			u_off = y_size - y_off + uv_off;
+			v_off = (fmt->uv_packed) ? 0 : u_off + uv_size;
+			if (fmt->uv_swapped) {
+				tmp = u_off;
+				u_off = v_off;
+				v_off = tmp;
+			}
+
+			image->tile[tile].offset = y_off;
+			image->tile[tile].u_off = u_off;
+			image->tile[tile++].v_off = v_off;
+
+			dev_dbg(priv->ipu->dev,
+				"task %u: ctx %p: %s@[%d,%d]: y_off %08x, u_off %08x, v_off %08x\n",
+				chan->ic_task, ctx,
+				image->type == IMAGE_CONVERT_IN ?
+				"Input" : "Output", row, col,
+				y_off, u_off, v_off);
+		}
+	}
+}
+
+static void calc_tile_offsets_packed(struct ipu_image_convert_ctx *ctx,
+				     struct ipu_image_convert_image *image)
+{
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_convert_priv *priv = chan->priv;
+	const struct ipu_image_pixfmt *fmt = image->fmt;
+	unsigned int row, col, tile = 0;
+	u32 w, h, bpp, stride;
+	u32 row_off, col_off;
+
+	/* setup some convenience vars */
+	stride = image->stride;
+	bpp = fmt->bpp;
+
+	for (row = 0; row < image->num_rows; row++) {
+		w = image->tile[tile].width;
+		h = image->tile[tile].height;
+		row_off = row * h * stride;
+
+		for (col = 0; col < image->num_cols; col++) {
+			col_off = (col * w * bpp) >> 3;
+
+			image->tile[tile].offset = row_off + col_off;
+			image->tile[tile].u_off = 0;
+			image->tile[tile++].v_off = 0;
+
+			dev_dbg(priv->ipu->dev,
+				"task %u: ctx %p: %s@[%d,%d]: phys %08x\n",
+				chan->ic_task, ctx,
+				image->type == IMAGE_CONVERT_IN ?
+				"Input" : "Output", row, col,
+				row_off + col_off);
+		}
+	}
+}
+
+static void calc_tile_offsets(struct ipu_image_convert_ctx *ctx,
+			      struct ipu_image_convert_image *image)
+{
+	if (image->fmt->planar)
+		calc_tile_offsets_planar(ctx, image);
+	else
+		calc_tile_offsets_packed(ctx, image);
+}
+
+/*
+ * return the number of runs in given queue (pending_q or done_q)
+ * for this context. hold irqlock when calling.
+ */
+static int get_run_count(struct ipu_image_convert_ctx *ctx,
+			 struct list_head *q)
+{
+	struct ipu_image_convert_run *run;
+	int count = 0;
+
+	lockdep_assert_held(&ctx->chan->irqlock);
+
+	list_for_each_entry(run, q, list) {
+		if (run->ctx == ctx)
+			count++;
+	}
+
+	return count;
+}
+
+static void convert_stop(struct ipu_image_convert_run *run)
+{
+	struct ipu_image_convert_ctx *ctx = run->ctx;
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_convert_priv *priv = chan->priv;
+
+	dev_dbg(priv->ipu->dev, "%s: task %u: stopping ctx %p run %p\n",
+		__func__, chan->ic_task, ctx, run);
+
+	/* disable IC tasks and the channels */
+	ipu_ic_task_disable(chan->ic);
+	ipu_idmac_disable_channel(chan->in_chan);
+	ipu_idmac_disable_channel(chan->out_chan);
+
+	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
+		ipu_idmac_disable_channel(chan->rotation_in_chan);
+		ipu_idmac_disable_channel(chan->rotation_out_chan);
+		ipu_idmac_unlink(chan->out_chan, chan->rotation_in_chan);
+	}
+
+	ipu_ic_disable(chan->ic);
+}
+
+static void init_idmac_channel(struct ipu_image_convert_ctx *ctx,
+			       struct ipuv3_channel *channel,
+			       struct ipu_image_convert_image *image,
+			       enum ipu_rotate_mode rot_mode,
+			       bool rot_swap_width_height)
+{
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	unsigned int burst_size;
+	u32 width, height, stride;
+	dma_addr_t addr0, addr1 = 0;
+	struct ipu_image tile_image;
+	unsigned int tile_idx[2];
+
+	if (image->type == IMAGE_CONVERT_OUT) {
+		tile_idx[0] = ctx->out_tile_map[0];
+		tile_idx[1] = ctx->out_tile_map[1];
+	} else {
+		tile_idx[0] = 0;
+		tile_idx[1] = 1;
+	}
+
+	if (rot_swap_width_height) {
+		width = image->tile[0].height;
+		height = image->tile[0].width;
+		stride = image->tile[0].rot_stride;
+		addr0 = ctx->rot_intermediate[0].phys;
+		if (ctx->double_buffering)
+			addr1 = ctx->rot_intermediate[1].phys;
+	} else {
+		width = image->tile[0].width;
+		height = image->tile[0].height;
+		stride = image->stride;
+		addr0 = image->base.phys0 +
+			image->tile[tile_idx[0]].offset;
+		if (ctx->double_buffering)
+			addr1 = image->base.phys0 +
+				image->tile[tile_idx[1]].offset;
+	}
+
+	ipu_cpmem_zero(channel);
+
+	memset(&tile_image, 0, sizeof(tile_image));
+	tile_image.pix.width = tile_image.rect.width = width;
+	tile_image.pix.height = tile_image.rect.height = height;
+	tile_image.pix.bytesperline = stride;
+	tile_image.pix.pixelformat =  image->fmt->fourcc;
+	tile_image.phys0 = addr0;
+	tile_image.phys1 = addr1;
+	ipu_cpmem_set_image(channel, &tile_image);
+
+	if (image->fmt->planar && !rot_swap_width_height)
+		ipu_cpmem_set_uv_offset(channel,
+					image->tile[tile_idx[0]].u_off,
+					image->tile[tile_idx[0]].v_off);
+
+	if (rot_mode)
+		ipu_cpmem_set_rotation(channel, rot_mode);
+
+	if (channel == chan->rotation_in_chan ||
+	    channel == chan->rotation_out_chan) {
+		burst_size = 8;
+		ipu_cpmem_set_block_mode(channel);
+	} else
+		burst_size = (width % 16) ? 8 : 16;
+
+	ipu_cpmem_set_burstsize(channel, burst_size);
+
+	ipu_ic_task_idma_init(chan->ic, channel, width, height,
+			      burst_size, rot_mode);
+
+	ipu_cpmem_set_axi_id(channel, 1);
+
+	ipu_idmac_set_double_buffer(channel, ctx->double_buffering);
+}
+
+static int convert_start(struct ipu_image_convert_run *run)
+{
+	struct ipu_image_convert_ctx *ctx = run->ctx;
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_convert_priv *priv = chan->priv;
+	struct ipu_image_convert_image *s_image = &ctx->in;
+	struct ipu_image_convert_image *d_image = &ctx->out;
+	enum ipu_color_space src_cs, dest_cs;
+	unsigned int dest_width, dest_height;
+	int ret;
+
+	dev_dbg(priv->ipu->dev, "%s: task %u: starting ctx %p run %p\n",
+		__func__, chan->ic_task, ctx, run);
+
+	src_cs = ipu_pixelformat_to_colorspace(s_image->fmt->fourcc);
+	dest_cs = ipu_pixelformat_to_colorspace(d_image->fmt->fourcc);
+
+	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
+		/* swap width/height for resizer */
+		dest_width = d_image->tile[0].height;
+		dest_height = d_image->tile[0].width;
+	} else {
+		dest_width = d_image->tile[0].width;
+		dest_height = d_image->tile[0].height;
+	}
+
+	/* setup the IC resizer and CSC */
+	ret = ipu_ic_task_init(chan->ic,
+			       s_image->tile[0].width,
+			       s_image->tile[0].height,
+			       dest_width,
+			       dest_height,
+			       src_cs, dest_cs);
+	if (ret) {
+		dev_err(priv->ipu->dev, "ipu_ic_task_init failed, %d\n", ret);
+		return ret;
+	}
+
+	/* init the source MEM-->IC PP IDMAC channel */
+	init_idmac_channel(ctx, chan->in_chan, s_image,
+			   IPU_ROTATE_NONE, false);
+
+	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
+		/* init the IC PP-->MEM IDMAC channel */
+		init_idmac_channel(ctx, chan->out_chan, d_image,
+				   IPU_ROTATE_NONE, true);
+
+		/* init the MEM-->IC PP ROT IDMAC channel */
+		init_idmac_channel(ctx, chan->rotation_in_chan, d_image,
+				   ctx->rot_mode, true);
+
+		/* init the destination IC PP ROT-->MEM IDMAC channel */
+		init_idmac_channel(ctx, chan->rotation_out_chan, d_image,
+				   IPU_ROTATE_NONE, false);
+
+		/* now link IC PP-->MEM to MEM-->IC PP ROT */
+		ipu_idmac_link(chan->out_chan, chan->rotation_in_chan);
+	} else {
+		/* init the destination IC PP-->MEM IDMAC channel */
+		init_idmac_channel(ctx, chan->out_chan, d_image,
+				   ctx->rot_mode, false);
+	}
+
+	/* enable the IC */
+	ipu_ic_enable(chan->ic);
+
+	/* set buffers ready */
+	ipu_idmac_select_buffer(chan->in_chan, 0);
+	ipu_idmac_select_buffer(chan->out_chan, 0);
+	if (ipu_rot_mode_is_irt(ctx->rot_mode))
+		ipu_idmac_select_buffer(chan->rotation_out_chan, 0);
+	if (ctx->double_buffering) {
+		ipu_idmac_select_buffer(chan->in_chan, 1);
+		ipu_idmac_select_buffer(chan->out_chan, 1);
+		if (ipu_rot_mode_is_irt(ctx->rot_mode))
+			ipu_idmac_select_buffer(chan->rotation_out_chan, 1);
+	}
+
+	/* enable the channels! */
+	ipu_idmac_enable_channel(chan->in_chan);
+	ipu_idmac_enable_channel(chan->out_chan);
+	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
+		ipu_idmac_enable_channel(chan->rotation_in_chan);
+		ipu_idmac_enable_channel(chan->rotation_out_chan);
+	}
+
+	ipu_ic_task_enable(chan->ic);
+
+	ipu_cpmem_dump(chan->in_chan);
+	ipu_cpmem_dump(chan->out_chan);
+	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
+		ipu_cpmem_dump(chan->rotation_in_chan);
+		ipu_cpmem_dump(chan->rotation_out_chan);
+	}
+
+	ipu_dump(priv->ipu);
+
+	return 0;
+}
+
+/* hold irqlock when calling */
+static int do_run(struct ipu_image_convert_run *run)
+{
+	struct ipu_image_convert_ctx *ctx = run->ctx;
+	struct ipu_image_convert_chan *chan = ctx->chan;
+
+	lockdep_assert_held(&chan->irqlock);
+
+	ctx->in.base.phys0 = run->in_phys;
+	ctx->out.base.phys0 = run->out_phys;
+
+	ctx->cur_buf_num = 0;
+	ctx->next_tile = 1;
+
+	/* remove run from pending_q and set as current */
+	list_del(&run->list);
+	chan->current_run = run;
+
+	return convert_start(run);
+}
+
+/* hold irqlock when calling */
+static void run_next(struct ipu_image_convert_chan *chan)
+{
+	struct ipu_image_convert_priv *priv = chan->priv;
+	struct ipu_image_convert_run *run, *tmp;
+	int ret;
+
+	lockdep_assert_held(&chan->irqlock);
+
+	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
+		/* skip contexts that are aborting */
+		if (run->ctx->aborting) {
+			dev_dbg(priv->ipu->dev,
+				"%s: task %u: skipping aborting ctx %p run %p\n",
+				__func__, chan->ic_task, run->ctx, run);
+			continue;
+		}
+
+		ret = do_run(run);
+		if (!ret)
+			break;
+
+		/*
+		 * something went wrong with start, add the run
+		 * to done q and continue to the next run in the
+		 * pending q.
+		 */
+		run->status = ret;
+		list_add_tail(&run->list, &chan->done_q);
+		chan->current_run = NULL;
+	}
+}
+
+static void empty_done_q(struct ipu_image_convert_chan *chan)
+{
+	struct ipu_image_convert_priv *priv = chan->priv;
+	struct ipu_image_convert_run *run;
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->irqlock, flags);
+
+	while (!list_empty(&chan->done_q)) {
+		run = list_entry(chan->done_q.next,
+				 struct ipu_image_convert_run,
+				 list);
+
+		list_del(&run->list);
+
+		dev_dbg(priv->ipu->dev,
+			"%s: task %u: completing ctx %p run %p with %d\n",
+			__func__, chan->ic_task, run->ctx, run, run->status);
+
+		/* call the completion callback and free the run */
+		spin_unlock_irqrestore(&chan->irqlock, flags);
+		run->ctx->complete(run, run->ctx->complete_context);
+		spin_lock_irqsave(&chan->irqlock, flags);
+	}
+
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+}
+
+/*
+ * the bottom half thread clears out the done_q, calling the
+ * completion handler for each.
+ */
+static irqreturn_t do_bh(int irq, void *dev_id)
+{
+	struct ipu_image_convert_chan *chan = dev_id;
+	struct ipu_image_convert_priv *priv = chan->priv;
+	struct ipu_image_convert_ctx *ctx;
+	unsigned long flags;
+
+	dev_dbg(priv->ipu->dev, "%s: task %u: enter\n", __func__,
+		chan->ic_task);
+
+	empty_done_q(chan);
+
+	spin_lock_irqsave(&chan->irqlock, flags);
+
+	/*
+	 * the done_q is cleared out, signal any contexts
+	 * that are aborting that abort can complete.
+	 */
+	list_for_each_entry(ctx, &chan->ctx_list, list) {
+		if (ctx->aborting) {
+			dev_dbg(priv->ipu->dev,
+				"%s: task %u: signaling abort for ctx %p\n",
+				__func__, chan->ic_task, ctx);
+			complete(&ctx->aborted);
+		}
+	}
+
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+
+	dev_dbg(priv->ipu->dev, "%s: task %u: exit\n", __func__,
+		chan->ic_task);
+
+	return IRQ_HANDLED;
+}
+
+/* hold irqlock when calling */
+static irqreturn_t do_irq(struct ipu_image_convert_run *run)
+{
+	struct ipu_image_convert_ctx *ctx = run->ctx;
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_tile *src_tile, *dst_tile;
+	struct ipu_image_convert_image *s_image = &ctx->in;
+	struct ipu_image_convert_image *d_image = &ctx->out;
+	struct ipuv3_channel *outch;
+	unsigned int dst_idx;
+
+	lockdep_assert_held(&chan->irqlock);
+
+	outch = ipu_rot_mode_is_irt(ctx->rot_mode) ?
+		chan->rotation_out_chan : chan->out_chan;
+
+	/*
+	 * It is difficult to stop the channel DMA before the channels
+	 * enter the paused state. Without double-buffering the channels
+	 * are always in a paused state when the EOF irq occurs, so it
+	 * is safe to stop the channels now. For double-buffering we
+	 * just ignore the abort until the operation completes, when it
+	 * is safe to shut down.
+	 */
+	if (ctx->aborting && !ctx->double_buffering) {
+		convert_stop(run);
+		run->status = -EIO;
+		goto done;
+	}
+
+	if (ctx->next_tile == ctx->num_tiles) {
+		/*
+		 * the conversion is complete
+		 */
+		convert_stop(run);
+		run->status = 0;
+		goto done;
+	}
+
+	/*
+	 * not done, place the next tile buffers.
+	 */
+	if (!ctx->double_buffering) {
+
+		src_tile = &s_image->tile[ctx->next_tile];
+		dst_idx = ctx->out_tile_map[ctx->next_tile];
+		dst_tile = &d_image->tile[dst_idx];
+
+		ipu_cpmem_set_buffer(chan->in_chan, 0,
+				     s_image->base.phys0 + src_tile->offset);
+		ipu_cpmem_set_buffer(outch, 0,
+				     d_image->base.phys0 + dst_tile->offset);
+		if (s_image->fmt->planar)
+			ipu_cpmem_set_uv_offset(chan->in_chan,
+						src_tile->u_off,
+						src_tile->v_off);
+		if (d_image->fmt->planar)
+			ipu_cpmem_set_uv_offset(outch,
+						dst_tile->u_off,
+						dst_tile->v_off);
+
+		ipu_idmac_select_buffer(chan->in_chan, 0);
+		ipu_idmac_select_buffer(outch, 0);
+
+	} else if (ctx->next_tile < ctx->num_tiles - 1) {
+
+		src_tile = &s_image->tile[ctx->next_tile + 1];
+		dst_idx = ctx->out_tile_map[ctx->next_tile + 1];
+		dst_tile = &d_image->tile[dst_idx];
+
+		ipu_cpmem_set_buffer(chan->in_chan, ctx->cur_buf_num,
+				     s_image->base.phys0 + src_tile->offset);
+		ipu_cpmem_set_buffer(outch, ctx->cur_buf_num,
+				     d_image->base.phys0 + dst_tile->offset);
+
+		ipu_idmac_select_buffer(chan->in_chan, ctx->cur_buf_num);
+		ipu_idmac_select_buffer(outch, ctx->cur_buf_num);
+
+		ctx->cur_buf_num ^= 1;
+	}
+
+	ctx->next_tile++;
+	return IRQ_HANDLED;
+done:
+	list_add_tail(&run->list, &chan->done_q);
+	chan->current_run = NULL;
+	run_next(chan);
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t norotate_irq(int irq, void *data)
+{
+	struct ipu_image_convert_chan *chan = data;
+	struct ipu_image_convert_ctx *ctx;
+	struct ipu_image_convert_run *run;
+	unsigned long flags;
+	irqreturn_t ret;
+
+	spin_lock_irqsave(&chan->irqlock, flags);
+
+	/* get current run and its context */
+	run = chan->current_run;
+	if (!run) {
+		ret = IRQ_NONE;
+		goto out;
+	}
+
+	ctx = run->ctx;
+
+	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
+		/* this is a rotation operation, just ignore */
+		spin_unlock_irqrestore(&chan->irqlock, flags);
+		return IRQ_HANDLED;
+	}
+
+	ret = do_irq(run);
+out:
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+	return ret;
+}
+
+static irqreturn_t rotate_irq(int irq, void *data)
+{
+	struct ipu_image_convert_chan *chan = data;
+	struct ipu_image_convert_priv *priv = chan->priv;
+	struct ipu_image_convert_ctx *ctx;
+	struct ipu_image_convert_run *run;
+	unsigned long flags;
+	irqreturn_t ret;
+
+	spin_lock_irqsave(&chan->irqlock, flags);
+
+	/* get current run and its context */
+	run = chan->current_run;
+	if (!run) {
+		ret = IRQ_NONE;
+		goto out;
+	}
+
+	ctx = run->ctx;
+
+	if (!ipu_rot_mode_is_irt(ctx->rot_mode)) {
+		/* this was NOT a rotation operation, shouldn't happen */
+		dev_err(priv->ipu->dev, "Unexpected rotation interrupt\n");
+		spin_unlock_irqrestore(&chan->irqlock, flags);
+		return IRQ_HANDLED;
+	}
+
+	ret = do_irq(run);
+out:
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+	return ret;
+}
+
+/*
+ * try to force the completion of runs for this ctx. Called when
+ * abort wait times out in ipu_image_convert_abort().
+ */
+static void force_abort(struct ipu_image_convert_ctx *ctx)
+{
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_convert_run *run;
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->irqlock, flags);
+
+	run = chan->current_run;
+	if (run && run->ctx == ctx) {
+		convert_stop(run);
+		run->status = -EIO;
+		list_add_tail(&run->list, &chan->done_q);
+		chan->current_run = NULL;
+		run_next(chan);
+	}
+
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+
+	empty_done_q(chan);
+}
+
+static void release_ipu_resources(struct ipu_image_convert_chan *chan)
+{
+	if (chan->out_eof_irq >= 0)
+		free_irq(chan->out_eof_irq, chan);
+	if (chan->rot_out_eof_irq >= 0)
+		free_irq(chan->rot_out_eof_irq, chan);
+
+	if (!IS_ERR_OR_NULL(chan->in_chan))
+		ipu_idmac_put(chan->in_chan);
+	if (!IS_ERR_OR_NULL(chan->out_chan))
+		ipu_idmac_put(chan->out_chan);
+	if (!IS_ERR_OR_NULL(chan->rotation_in_chan))
+		ipu_idmac_put(chan->rotation_in_chan);
+	if (!IS_ERR_OR_NULL(chan->rotation_out_chan))
+		ipu_idmac_put(chan->rotation_out_chan);
+	if (!IS_ERR_OR_NULL(chan->ic))
+		ipu_ic_put(chan->ic);
+
+	chan->in_chan = chan->out_chan = chan->rotation_in_chan =
+		chan->rotation_out_chan = NULL;
+	chan->out_eof_irq = chan->rot_out_eof_irq = -1;
+}
+
+static int get_ipu_resources(struct ipu_image_convert_chan *chan)
+{
+	const struct ipu_image_convert_dma_chan *dma = chan->dma_ch;
+	struct ipu_image_convert_priv *priv = chan->priv;
+	int ret;
+
+	/* get IC */
+	chan->ic = ipu_ic_get(priv->ipu, chan->ic_task);
+	if (IS_ERR(chan->ic)) {
+		dev_err(priv->ipu->dev, "could not acquire IC\n");
+		ret = PTR_ERR(chan->ic);
+		goto err;
+	}
+
+	/* get IDMAC channels */
+	chan->in_chan = ipu_idmac_get(priv->ipu, dma->in);
+	chan->out_chan = ipu_idmac_get(priv->ipu, dma->out);
+	if (IS_ERR(chan->in_chan) || IS_ERR(chan->out_chan)) {
+		dev_err(priv->ipu->dev, "could not acquire idmac channels\n");
+		ret = -EBUSY;
+		goto err;
+	}
+
+	chan->rotation_in_chan = ipu_idmac_get(priv->ipu, dma->rot_in);
+	chan->rotation_out_chan = ipu_idmac_get(priv->ipu, dma->rot_out);
+	if (IS_ERR(chan->rotation_in_chan) || IS_ERR(chan->rotation_out_chan)) {
+		dev_err(priv->ipu->dev,
+			"could not acquire idmac rotation channels\n");
+		ret = -EBUSY;
+		goto err;
+	}
+
+	/* acquire the EOF interrupts */
+	chan->out_eof_irq = ipu_idmac_channel_irq(priv->ipu,
+						  chan->out_chan,
+						  IPU_IRQ_EOF);
+
+	ret = request_threaded_irq(chan->out_eof_irq, norotate_irq, do_bh,
+				   0, "ipu-ic", chan);
+	if (ret < 0) {
+		dev_err(priv->ipu->dev, "could not acquire irq %d\n",
+			 chan->out_eof_irq);
+		chan->out_eof_irq = -1;
+		goto err;
+	}
+
+	chan->rot_out_eof_irq = ipu_idmac_channel_irq(priv->ipu,
+						     chan->rotation_out_chan,
+						     IPU_IRQ_EOF);
+
+	ret = request_threaded_irq(chan->rot_out_eof_irq, rotate_irq, do_bh,
+				   0, "ipu-ic", chan);
+	if (ret < 0) {
+		dev_err(priv->ipu->dev, "could not acquire irq %d\n",
+			chan->rot_out_eof_irq);
+		chan->rot_out_eof_irq = -1;
+		goto err;
+	}
+
+	return 0;
+err:
+	release_ipu_resources(chan);
+	return ret;
+}
+
+static int fill_image(struct ipu_image_convert_ctx *ctx,
+		      struct ipu_image_convert_image *ic_image,
+		      struct ipu_image *image,
+		      enum ipu_image_convert_type type)
+{
+	struct ipu_image_convert_priv *priv = ctx->chan->priv;
+
+	ic_image->base = *image;
+	ic_image->type = type;
+
+	ic_image->fmt = get_format(image->pix.pixelformat);
+	if (!ic_image->fmt) {
+		dev_err(priv->ipu->dev, "pixelformat not supported for %s\n",
+			type == IMAGE_CONVERT_OUT ? "Output" : "Input");
+		return -EINVAL;
+	}
+
+	if (ic_image->fmt->planar)
+		ic_image->stride = ic_image->base.pix.width;
+	else
+		ic_image->stride  = ic_image->base.pix.bytesperline;
+
+	calc_tile_dimensions(ctx, ic_image);
+	calc_tile_offsets(ctx, ic_image);
+
+	return 0;
+}
+
+/* borrowed from drivers/media/v4l2-core/v4l2-common.c */
+static unsigned int clamp_align(unsigned int x, unsigned int min,
+				unsigned int max, unsigned int align)
+{
+	/* Bits that must be zero to be aligned */
+	unsigned int mask = ~((1 << align) - 1);
+
+	/* Clamp to aligned min and max */
+	x = clamp(x, (min + ~mask) & mask, max & mask);
+
+	/* Round to nearest aligned value */
+	if (align)
+		x = (x + (1 << (align - 1))) & mask;
+
+	return x;
+}
+
+/*
+ * We have to adjust the tile width such that the tile physaddrs and
+ * U and V plane offsets are multiples of 8 bytes as required by
+ * the IPU DMA Controller. For the planar formats, this corresponds
+ * to a pixel alignment of 16 (but use a more formal equation since
+ * the variables are available). For all the packed formats, 8 is
+ * good enough.
+ */
+static inline u32 tile_width_align(const struct ipu_image_pixfmt *fmt)
+{
+	return fmt->planar ? 8 * fmt->uv_width_dec : 8;
+}
+
+/*
+ * For tile height alignment, we have to ensure that the output tile
+ * heights are multiples of 8 lines if the IRT is required by the
+ * given rotation mode (the IRT performs rotations on 8x8 blocks
+ * at a time). If the IRT is not used, or for input image tiles,
+ * 2 lines are good enough.
+ */
+static inline u32 tile_height_align(enum ipu_image_convert_type type,
+				    enum ipu_rotate_mode rot_mode)
+{
+	return (type == IMAGE_CONVERT_OUT &&
+		ipu_rot_mode_is_irt(rot_mode)) ? 8 : 2;
+}
+
+/* Adjusts input/output images to IPU restrictions */
+void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out,
+			      enum ipu_rotate_mode rot_mode)
+{
+	const struct ipu_image_pixfmt *infmt, *outfmt;
+	unsigned int num_in_rows, num_in_cols;
+	unsigned int num_out_rows, num_out_cols;
+	u32 w_align, h_align;
+
+	infmt = get_format(in->pix.pixelformat);
+	outfmt = get_format(out->pix.pixelformat);
+
+	/* set some default pixel formats if needed */
+	if (!infmt) {
+		in->pix.pixelformat = V4L2_PIX_FMT_RGB24;
+		infmt = get_format(V4L2_PIX_FMT_RGB24);
+	}
+	if (!outfmt) {
+		out->pix.pixelformat = V4L2_PIX_FMT_RGB24;
+		outfmt = get_format(V4L2_PIX_FMT_RGB24);
+	}
+
+	/* image converter does not handle fields */
+	in->pix.field = out->pix.field = V4L2_FIELD_NONE;
+
+	/* resizer cannot downsize more than 4:1 */
+	if (ipu_rot_mode_is_irt(rot_mode)) {
+		out->pix.height = max_t(__u32, out->pix.height,
+					in->pix.width / 4);
+		out->pix.width = max_t(__u32, out->pix.width,
+				       in->pix.height / 4);
+	} else {
+		out->pix.width = max_t(__u32, out->pix.width,
+				       in->pix.width / 4);
+		out->pix.height = max_t(__u32, out->pix.height,
+					in->pix.height / 4);
+	}
+
+	/* get tiling rows/cols from output format */
+	num_out_rows = num_stripes(out->pix.height);
+	num_out_cols = num_stripes(out->pix.width);
+	if (ipu_rot_mode_is_irt(rot_mode)) {
+		num_in_rows = num_out_cols;
+		num_in_cols = num_out_rows;
+	} else {
+		num_in_rows = num_out_rows;
+		num_in_cols = num_out_cols;
+	}
+
+	/* align input width/height */
+	w_align = ilog2(tile_width_align(infmt) * num_in_cols);
+	h_align = ilog2(tile_height_align(IMAGE_CONVERT_IN, rot_mode) *
+			num_in_rows);
+	in->pix.width = clamp_align(in->pix.width, MIN_W, MAX_W, w_align);
+	in->pix.height = clamp_align(in->pix.height, MIN_H, MAX_H, h_align);
+
+	/* align output width/height */
+	w_align = ilog2(tile_width_align(outfmt) * num_out_cols);
+	h_align = ilog2(tile_height_align(IMAGE_CONVERT_OUT, rot_mode) *
+			num_out_rows);
+	out->pix.width = clamp_align(out->pix.width, MIN_W, MAX_W, w_align);
+	out->pix.height = clamp_align(out->pix.height, MIN_H, MAX_H, h_align);
+
+	/* set input/output strides and image sizes */
+	in->pix.bytesperline = (in->pix.width * infmt->bpp) >> 3;
+	in->pix.sizeimage = in->pix.height * in->pix.bytesperline;
+	out->pix.bytesperline = (out->pix.width * outfmt->bpp) >> 3;
+	out->pix.sizeimage = out->pix.height * out->pix.bytesperline;
+}
+EXPORT_SYMBOL_GPL(ipu_image_convert_adjust);
+
+/*
+ * this is used by ipu_image_convert_prepare() to verify set input and
+ * output images are valid before starting the conversion. Clients can
+ * also call it before calling ipu_image_convert_prepare().
+ */
+int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out,
+			     enum ipu_rotate_mode rot_mode)
+{
+	struct ipu_image testin, testout;
+
+	testin = *in;
+	testout = *out;
+
+	ipu_image_convert_adjust(&testin, &testout, rot_mode);
+
+	if (testin.pix.width != in->pix.width ||
+	    testin.pix.height != in->pix.height ||
+	    testout.pix.width != out->pix.width ||
+	    testout.pix.height != out->pix.height)
+		return -EINVAL;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipu_image_convert_verify);
+
+/*
+ * Call ipu_image_convert_prepare() to prepare for the conversion of
+ * given images and rotation mode. Returns a new conversion context.
+ */
+struct ipu_image_convert_ctx *
+ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
+			  struct ipu_image *in, struct ipu_image *out,
+			  enum ipu_rotate_mode rot_mode,
+			  ipu_image_convert_cb_t complete,
+			  void *complete_context)
+{
+	struct ipu_image_convert_priv *priv = ipu->image_convert_priv;
+	struct ipu_image_convert_image *s_image, *d_image;
+	struct ipu_image_convert_chan *chan;
+	struct ipu_image_convert_ctx *ctx;
+	unsigned long flags;
+	bool get_res;
+	int ret;
+
+	if (!in || !out || !complete ||
+	    (ic_task != IC_TASK_VIEWFINDER &&
+	     ic_task != IC_TASK_POST_PROCESSOR))
+		return ERR_PTR(-EINVAL);
+
+	/* verify the in/out images before continuing */
+	ret = ipu_image_convert_verify(in, out, rot_mode);
+	if (ret) {
+		dev_err(priv->ipu->dev, "%s: in/out formats invalid\n",
+			__func__);
+		return ERR_PTR(ret);
+	}
+
+	chan = &priv->chan[ic_task];
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p\n", __func__,
+		chan->ic_task, ctx);
+
+	ctx->chan = chan;
+	init_completion(&ctx->aborted);
+
+	s_image = &ctx->in;
+	d_image = &ctx->out;
+
+	/* set tiling and rotation */
+	d_image->num_rows = num_stripes(out->pix.height);
+	d_image->num_cols = num_stripes(out->pix.width);
+	if (ipu_rot_mode_is_irt(rot_mode)) {
+		s_image->num_rows = d_image->num_cols;
+		s_image->num_cols = d_image->num_rows;
+	} else {
+		s_image->num_rows = d_image->num_rows;
+		s_image->num_cols = d_image->num_cols;
+	}
+
+	ctx->num_tiles = d_image->num_cols * d_image->num_rows;
+	ctx->rot_mode = rot_mode;
+
+	ret = fill_image(ctx, s_image, in, IMAGE_CONVERT_IN);
+	if (ret)
+		goto out_free;
+	ret = fill_image(ctx, d_image, out, IMAGE_CONVERT_OUT);
+	if (ret)
+		goto out_free;
+
+	calc_out_tile_map(ctx);
+
+	dump_format(ctx, s_image);
+	dump_format(ctx, d_image);
+
+	ctx->complete = complete;
+	ctx->complete_context = complete_context;
+
+	/*
+	 * Can we use double-buffering for this operation? If there is
+	 * only one tile (the whole image can be converted in a single
+	 * operation) there's no point in using double-buffering. Also,
+	 * the IPU's IDMAC channels allow only a single U and V plane
+	 * offset shared between both buffers, but these offsets change
+	 * for every tile, and therefore would have to be updated for
+	 * each buffer which is not possible. So double-buffering is
+	 * impossible when either the source or destination images are
+	 * a planar format (YUV420, YUV422P, etc.).
+	 */
+	ctx->double_buffering = (ctx->num_tiles > 1 &&
+				 !s_image->fmt->planar &&
+				 !d_image->fmt->planar);
+
+	if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
+		ret = alloc_dma_buf(priv, &ctx->rot_intermediate[0],
+				    d_image->tile[0].size);
+		if (ret)
+			goto out_free;
+		if (ctx->double_buffering) {
+			ret = alloc_dma_buf(priv,
+					    &ctx->rot_intermediate[1],
+					    d_image->tile[0].size);
+			if (ret)
+				goto out_free_dmabuf0;
+		}
+	}
+
+	spin_lock_irqsave(&chan->irqlock, flags);
+
+	get_res = list_empty(&chan->ctx_list);
+
+	list_add_tail(&ctx->list, &chan->ctx_list);
+
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+
+	if (get_res) {
+		ret = get_ipu_resources(chan);
+		if (ret)
+			goto out_free_dmabuf1;
+	}
+
+	return ctx;
+
+out_free_dmabuf1:
+	free_dma_buf(priv, &ctx->rot_intermediate[1]);
+	spin_lock_irqsave(&chan->irqlock, flags);
+	list_del(&ctx->list);
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+out_free_dmabuf0:
+	free_dma_buf(priv, &ctx->rot_intermediate[0]);
+out_free:
+	kfree(ctx);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(ipu_image_convert_prepare);
+
+/*
+ * Carry out a single image conversion run. Only the physaddr's of the input
+ * and output image buffers are needed. The conversion context must have
+ * been created previously with ipu_image_convert_prepare().
+ */
+int ipu_image_convert_queue(struct ipu_image_convert_run *run)
+{
+	struct ipu_image_convert_chan *chan;
+	struct ipu_image_convert_priv *priv;
+	struct ipu_image_convert_ctx *ctx;
+	unsigned long flags;
+	int ret = 0;
+
+	if (!run || !run->ctx || !run->in_phys || !run->out_phys)
+		return -EINVAL;
+
+	ctx = run->ctx;
+	chan = ctx->chan;
+	priv = chan->priv;
+
+	dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p run %p\n", __func__,
+		chan->ic_task, ctx, run);
+
+	INIT_LIST_HEAD(&run->list);
+
+	spin_lock_irqsave(&chan->irqlock, flags);
+
+	if (ctx->aborting) {
+		ret = -EIO;
+		goto unlock;
+	}
+
+	list_add_tail(&run->list, &chan->pending_q);
+
+	if (!chan->current_run) {
+		ret = do_run(run);
+		if (ret)
+			chan->current_run = NULL;
+	}
+unlock:
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ipu_image_convert_queue);
+
+/* Abort any active or pending conversions for this context */
+void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
+{
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_convert_priv *priv = chan->priv;
+	struct ipu_image_convert_run *run, *active_run, *tmp;
+	unsigned long flags;
+	int run_count, ret;
+	bool need_abort;
+
+	reinit_completion(&ctx->aborted);
+
+	spin_lock_irqsave(&chan->irqlock, flags);
+
+	/* move all remaining pending runs in this context to done_q */
+	list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
+		if (run->ctx != ctx)
+			continue;
+		run->status = -EIO;
+		list_move_tail(&run->list, &chan->done_q);
+	}
+
+	run_count = get_run_count(ctx, &chan->done_q);
+	active_run = (chan->current_run && chan->current_run->ctx == ctx) ?
+		chan->current_run : NULL;
+
+	need_abort = (run_count || active_run);
+
+	ctx->aborting = need_abort;
+
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+
+	if (!need_abort) {
+		dev_dbg(priv->ipu->dev,
+			"%s: task %u: no abort needed for ctx %p\n",
+			__func__, chan->ic_task, ctx);
+		return;
+	}
+
+	dev_dbg(priv->ipu->dev,
+		"%s: task %u: wait for completion: %d runs, active run %p\n",
+		__func__, chan->ic_task, run_count, active_run);
+
+	ret = wait_for_completion_timeout(&ctx->aborted,
+					  msecs_to_jiffies(10000));
+	if (ret == 0) {
+		dev_warn(priv->ipu->dev, "%s: timeout\n", __func__);
+		force_abort(ctx);
+	}
+
+	ctx->aborting = false;
+}
+EXPORT_SYMBOL_GPL(ipu_image_convert_abort);
+
+/* Unprepare image conversion context */
+void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx)
+{
+	struct ipu_image_convert_chan *chan = ctx->chan;
+	struct ipu_image_convert_priv *priv = chan->priv;
+	unsigned long flags;
+	bool put_res;
+
+	/* make sure no runs are hanging around */
+	ipu_image_convert_abort(ctx);
+
+	dev_dbg(priv->ipu->dev, "%s: task %u: removing ctx %p\n", __func__,
+		chan->ic_task, ctx);
+
+	spin_lock_irqsave(&chan->irqlock, flags);
+
+	list_del(&ctx->list);
+
+	put_res = list_empty(&chan->ctx_list);
+
+	spin_unlock_irqrestore(&chan->irqlock, flags);
+
+	if (put_res)
+		release_ipu_resources(chan);
+
+	free_dma_buf(priv, &ctx->rot_intermediate[1]);
+	free_dma_buf(priv, &ctx->rot_intermediate[0]);
+
+	kfree(ctx);
+}
+EXPORT_SYMBOL_GPL(ipu_image_convert_unprepare);
+
+/*
+ * "Canned" asynchronous single image conversion. Allocates and returns
+ * a new conversion run.  On successful return the caller must free the
+ * run and call ipu_image_convert_unprepare() after conversion completes.
+ */
+struct ipu_image_convert_run *
+ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
+		  struct ipu_image *in, struct ipu_image *out,
+		  enum ipu_rotate_mode rot_mode,
+		  ipu_image_convert_cb_t complete,
+		  void *complete_context)
+{
+	struct ipu_image_convert_ctx *ctx;
+	struct ipu_image_convert_run *run;
+	int ret;
+
+	ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode,
+					complete, complete_context);
+	if (IS_ERR(ctx))
+		return ERR_PTR(PTR_ERR(ctx));
+
+	run = kzalloc(sizeof(*run), GFP_KERNEL);
+	if (!run) {
+		ipu_image_convert_unprepare(ctx);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	run->ctx = ctx;
+	run->in_phys = in->phys0;
+	run->out_phys = out->phys0;
+
+	ret = ipu_image_convert_queue(run);
+	if (ret) {
+		ipu_image_convert_unprepare(ctx);
+		kfree(run);
+		return ERR_PTR(ret);
+	}
+
+	return run;
+}
+EXPORT_SYMBOL_GPL(ipu_image_convert);
+
+/* "Canned" synchronous single image conversion */
+static void image_convert_sync_complete(struct ipu_image_convert_run *run,
+					void *data)
+{
+	struct completion *comp = data;
+
+	complete(comp);
+}
+
+int ipu_image_convert_sync(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
+			   struct ipu_image *in, struct ipu_image *out,
+			   enum ipu_rotate_mode rot_mode)
+{
+	struct ipu_image_convert_run *run;
+	struct completion comp;
+	int ret;
+
+	init_completion(&comp);
+
+	run = ipu_image_convert(ipu, ic_task, in, out, rot_mode,
+				image_convert_sync_complete, &comp);
+	if (IS_ERR(run))
+		return PTR_ERR(run);
+
+	ret = wait_for_completion_timeout(&comp, msecs_to_jiffies(10000));
+	ret = (ret == 0) ? -ETIMEDOUT : 0;
+
+	ipu_image_convert_unprepare(run->ctx);
+	kfree(run);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ipu_image_convert_sync);
+
+int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev)
+{
+	struct ipu_image_convert_priv *priv;
+	int i;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	ipu->image_convert_priv = priv;
+	priv->ipu = ipu;
+
+	for (i = 0; i < IC_NUM_TASKS; i++) {
+		struct ipu_image_convert_chan *chan = &priv->chan[i];
+
+		chan->ic_task = i;
+		chan->priv = priv;
+		chan->dma_ch = &image_convert_dma_chan[i];
+		chan->out_eof_irq = -1;
+		chan->rot_out_eof_irq = -1;
+
+		spin_lock_init(&chan->irqlock);
+		INIT_LIST_HEAD(&chan->ctx_list);
+		INIT_LIST_HEAD(&chan->pending_q);
+		INIT_LIST_HEAD(&chan->done_q);
+	}
+
+	return 0;
+}
+
+void ipu_image_convert_exit(struct ipu_soc *ipu)
+{
+}
diff --git a/drivers/gpu/ipu-v3/ipu-prv.h b/drivers/gpu/ipu-v3/ipu-prv.h
index bfb1e8a4483f..22e47b68b14a 100644
--- a/drivers/gpu/ipu-v3/ipu-prv.h
+++ b/drivers/gpu/ipu-v3/ipu-prv.h
@@ -75,6 +75,33 @@ struct ipu_soc;
 #define IPU_INT_CTRL(n)		IPU_CM_REG(0x003C + 4 * (n))
 #define IPU_INT_STAT(n)		IPU_CM_REG(0x0200 + 4 * (n))
 
+/* FS_PROC_FLOW1 */
+#define FS_PRPENC_ROT_SRC_SEL_MASK	(0xf << 0)
+#define FS_PRPENC_ROT_SRC_SEL_ENC		(0x7 << 0)
+#define FS_PRPVF_ROT_SRC_SEL_MASK	(0xf << 8)
+#define FS_PRPVF_ROT_SRC_SEL_VF			(0x8 << 8)
+#define FS_PP_SRC_SEL_MASK		(0xf << 12)
+#define FS_PP_ROT_SRC_SEL_MASK		(0xf << 16)
+#define FS_PP_ROT_SRC_SEL_PP			(0x5 << 16)
+#define FS_VDI1_SRC_SEL_MASK		(0x3 << 20)
+#define FS_VDI3_SRC_SEL_MASK		(0x3 << 20)
+#define FS_PRP_SRC_SEL_MASK		(0xf << 24)
+#define FS_VDI_SRC_SEL_MASK		(0x3 << 28)
+#define FS_VDI_SRC_SEL_CSI_DIRECT		(0x1 << 28)
+#define FS_VDI_SRC_SEL_VDOA			(0x2 << 28)
+
+/* FS_PROC_FLOW2 */
+#define FS_PRP_ENC_DEST_SEL_MASK	(0xf << 0)
+#define FS_PRP_ENC_DEST_SEL_IRT_ENC		(0x1 << 0)
+#define FS_PRPVF_DEST_SEL_MASK		(0xf << 4)
+#define FS_PRPVF_DEST_SEL_IRT_VF		(0x1 << 4)
+#define FS_PRPVF_ROT_DEST_SEL_MASK	(0xf << 8)
+#define FS_PP_DEST_SEL_MASK		(0xf << 12)
+#define FS_PP_DEST_SEL_IRT_PP			(0x3 << 12)
+#define FS_PP_ROT_DEST_SEL_MASK		(0xf << 16)
+#define FS_PRPENC_ROT_DEST_SEL_MASK	(0xf << 20)
+#define FS_PRP_DEST_SEL_MASK		(0xf << 24)
+
 #define IPU_DI0_COUNTER_RELEASE			(1 << 24)
 #define IPU_DI1_COUNTER_RELEASE			(1 << 25)
 
@@ -138,6 +165,8 @@ struct ipu_dc_priv;
 struct ipu_dmfc_priv;
 struct ipu_di;
 struct ipu_ic_priv;
+struct ipu_vdi;
+struct ipu_image_convert_priv;
 struct ipu_smfc_priv;
 
 struct ipu_devtype;
@@ -152,6 +181,7 @@ struct ipu_soc {
 	void __iomem		*cm_reg;
 	void __iomem		*idmac_reg;
 
+	int			id;
 	int			usecount;
 
 	struct clk		*clk;
@@ -169,6 +199,8 @@ struct ipu_soc {
 	struct ipu_di		*di_priv[2];
 	struct ipu_csi		*csi_priv[2];
 	struct ipu_ic_priv	*ic_priv;
+	struct ipu_vdi          *vdi_priv;
+	struct ipu_image_convert_priv *image_convert_priv;
 	struct ipu_smfc_priv	*smfc_priv;
 };
 
@@ -199,6 +231,13 @@ int ipu_ic_init(struct ipu_soc *ipu, struct device *dev,
 		unsigned long base, unsigned long tpmem_base);
 void ipu_ic_exit(struct ipu_soc *ipu);
 
+int ipu_vdi_init(struct ipu_soc *ipu, struct device *dev,
+		 unsigned long base, u32 module);
+void ipu_vdi_exit(struct ipu_soc *ipu);
+
+int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev);
+void ipu_image_convert_exit(struct ipu_soc *ipu);
+
 int ipu_di_init(struct ipu_soc *ipu, struct device *dev, int id,
 		unsigned long base, u32 module, struct clk *ipu_clk);
 void ipu_di_exit(struct ipu_soc *ipu, int id);
diff --git a/drivers/gpu/ipu-v3/ipu-vdi.c b/drivers/gpu/ipu-v3/ipu-vdi.c
new file mode 100644
index 000000000000..f27bf5a12ebc
--- /dev/null
+++ b/drivers/gpu/ipu-v3/ipu-vdi.c
@@ -0,0 +1,243 @@
+/*
+ * Copyright (C) 2012-2016 Mentor Graphics Inc.
+ * Copyright (C) 2005-2009 Freescale Semiconductor, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ */
+#include <linux/io.h>
+#include "ipu-prv.h"
+
+struct ipu_vdi {
+	void __iomem *base;
+	u32 module;
+	spinlock_t lock;
+	int use_count;
+	struct ipu_soc *ipu;
+};
+
+
+/* VDI Register Offsets */
+#define VDI_FSIZE 0x0000
+#define VDI_C     0x0004
+
+/* VDI Register Fields */
+#define VDI_C_CH_420             (0 << 1)
+#define VDI_C_CH_422             (1 << 1)
+#define VDI_C_MOT_SEL_MASK       (0x3 << 2)
+#define VDI_C_MOT_SEL_FULL       (2 << 2)
+#define VDI_C_MOT_SEL_LOW        (1 << 2)
+#define VDI_C_MOT_SEL_MED        (0 << 2)
+#define VDI_C_BURST_SIZE1_4      (3 << 4)
+#define VDI_C_BURST_SIZE2_4      (3 << 8)
+#define VDI_C_BURST_SIZE3_4      (3 << 12)
+#define VDI_C_BURST_SIZE_MASK    0xF
+#define VDI_C_BURST_SIZE1_OFFSET 4
+#define VDI_C_BURST_SIZE2_OFFSET 8
+#define VDI_C_BURST_SIZE3_OFFSET 12
+#define VDI_C_VWM1_SET_1         (0 << 16)
+#define VDI_C_VWM1_SET_2         (1 << 16)
+#define VDI_C_VWM1_CLR_2         (1 << 19)
+#define VDI_C_VWM3_SET_1         (0 << 22)
+#define VDI_C_VWM3_SET_2         (1 << 22)
+#define VDI_C_VWM3_CLR_2         (1 << 25)
+#define VDI_C_TOP_FIELD_MAN_1    (1 << 30)
+#define VDI_C_TOP_FIELD_AUTO_1   (1 << 31)
+
+static inline u32 ipu_vdi_read(struct ipu_vdi *vdi, unsigned int offset)
+{
+	return readl(vdi->base + offset);
+}
+
+static inline void ipu_vdi_write(struct ipu_vdi *vdi, u32 value,
+				 unsigned int offset)
+{
+	writel(value, vdi->base + offset);
+}
+
+void ipu_vdi_set_field_order(struct ipu_vdi *vdi, v4l2_std_id std, u32 field)
+{
+	bool top_field_0 = false;
+	unsigned long flags;
+	u32 reg;
+
+	switch (field) {
+	case V4L2_FIELD_INTERLACED_TB:
+	case V4L2_FIELD_SEQ_TB:
+	case V4L2_FIELD_TOP:
+		top_field_0 = true;
+		break;
+	case V4L2_FIELD_INTERLACED_BT:
+	case V4L2_FIELD_SEQ_BT:
+	case V4L2_FIELD_BOTTOM:
+		top_field_0 = false;
+		break;
+	default:
+		top_field_0 = (std & V4L2_STD_525_60) ? true : false;
+		break;
+	}
+
+	spin_lock_irqsave(&vdi->lock, flags);
+
+	reg = ipu_vdi_read(vdi, VDI_C);
+	if (top_field_0)
+		reg &= ~VDI_C_TOP_FIELD_MAN_1;
+	else
+		reg |= VDI_C_TOP_FIELD_MAN_1;
+	ipu_vdi_write(vdi, reg, VDI_C);
+
+	spin_unlock_irqrestore(&vdi->lock, flags);
+}
+EXPORT_SYMBOL_GPL(ipu_vdi_set_field_order);
+
+void ipu_vdi_set_motion(struct ipu_vdi *vdi, enum ipu_motion_sel motion_sel)
+{
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&vdi->lock, flags);
+
+	reg = ipu_vdi_read(vdi, VDI_C);
+
+	reg &= ~VDI_C_MOT_SEL_MASK;
+
+	switch (motion_sel) {
+	case MED_MOTION:
+		reg |= VDI_C_MOT_SEL_MED;
+		break;
+	case HIGH_MOTION:
+		reg |= VDI_C_MOT_SEL_FULL;
+		break;
+	default:
+		reg |= VDI_C_MOT_SEL_LOW;
+		break;
+	}
+
+	ipu_vdi_write(vdi, reg, VDI_C);
+
+	spin_unlock_irqrestore(&vdi->lock, flags);
+}
+EXPORT_SYMBOL_GPL(ipu_vdi_set_motion);
+
+void ipu_vdi_setup(struct ipu_vdi *vdi, u32 code, int xres, int yres)
+{
+	unsigned long flags;
+	u32 pixel_fmt, reg;
+
+	spin_lock_irqsave(&vdi->lock, flags);
+
+	reg = ((yres - 1) << 16) | (xres - 1);
+	ipu_vdi_write(vdi, reg, VDI_FSIZE);
+
+	/*
+	 * Full motion, only vertical filter is used.
+	 * Burst size is 4 accesses
+	 */
+	if (code == MEDIA_BUS_FMT_UYVY8_2X8 ||
+	    code == MEDIA_BUS_FMT_UYVY8_1X16 ||
+	    code == MEDIA_BUS_FMT_YUYV8_2X8 ||
+	    code == MEDIA_BUS_FMT_YUYV8_1X16)
+		pixel_fmt = VDI_C_CH_422;
+	else
+		pixel_fmt = VDI_C_CH_420;
+
+	reg = ipu_vdi_read(vdi, VDI_C);
+	reg |= pixel_fmt;
+	reg |= VDI_C_BURST_SIZE2_4;
+	reg |= VDI_C_BURST_SIZE1_4 | VDI_C_VWM1_CLR_2;
+	reg |= VDI_C_BURST_SIZE3_4 | VDI_C_VWM3_CLR_2;
+	ipu_vdi_write(vdi, reg, VDI_C);
+
+	spin_unlock_irqrestore(&vdi->lock, flags);
+}
+EXPORT_SYMBOL_GPL(ipu_vdi_setup);
+
+void ipu_vdi_unsetup(struct ipu_vdi *vdi)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&vdi->lock, flags);
+	ipu_vdi_write(vdi, 0, VDI_FSIZE);
+	ipu_vdi_write(vdi, 0, VDI_C);
+	spin_unlock_irqrestore(&vdi->lock, flags);
+}
+EXPORT_SYMBOL_GPL(ipu_vdi_unsetup);
+
+int ipu_vdi_enable(struct ipu_vdi *vdi)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&vdi->lock, flags);
+
+	if (!vdi->use_count)
+		ipu_module_enable(vdi->ipu, vdi->module);
+
+	vdi->use_count++;
+
+	spin_unlock_irqrestore(&vdi->lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipu_vdi_enable);
+
+int ipu_vdi_disable(struct ipu_vdi *vdi)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&vdi->lock, flags);
+
+	if (vdi->use_count) {
+		if (!--vdi->use_count)
+			ipu_module_disable(vdi->ipu, vdi->module);
+	}
+
+	spin_unlock_irqrestore(&vdi->lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ipu_vdi_disable);
+
+struct ipu_vdi *ipu_vdi_get(struct ipu_soc *ipu)
+{
+	return ipu->vdi_priv;
+}
+EXPORT_SYMBOL_GPL(ipu_vdi_get);
+
+void ipu_vdi_put(struct ipu_vdi *vdi)
+{
+}
+EXPORT_SYMBOL_GPL(ipu_vdi_put);
+
+int ipu_vdi_init(struct ipu_soc *ipu, struct device *dev,
+		 unsigned long base, u32 module)
+{
+	struct ipu_vdi *vdi;
+
+	vdi = devm_kzalloc(dev, sizeof(*vdi), GFP_KERNEL);
+	if (!vdi)
+		return -ENOMEM;
+
+	ipu->vdi_priv = vdi;
+
+	spin_lock_init(&vdi->lock);
+	vdi->module = module;
+	vdi->base = devm_ioremap(dev, base, PAGE_SIZE);
+	if (!vdi->base)
+		return -ENOMEM;
+
+	dev_dbg(dev, "VDI base: 0x%08lx remapped to %p\n", base, vdi->base);
+	vdi->ipu = ipu;
+
+	return 0;
+}
+
+void ipu_vdi_exit(struct ipu_soc *ipu)
+{
+}
diff --git a/drivers/gpu/vga/vgaarb.c b/drivers/gpu/vga/vgaarb.c
index f17cb0431833..1887f199ccb7 100644
--- a/drivers/gpu/vga/vgaarb.c
+++ b/drivers/gpu/vga/vgaarb.c
@@ -131,7 +131,24 @@ static struct vga_device *vgadev_find(struct pci_dev *pdev)
 	return NULL;
 }
 
-/* Returns the default VGA device (vgacon's babe) */
+/**
+ * vga_default_device - return the default VGA device, for vgacon
+ *
+ * This can be defined by the platform. The default implementation
+ * is rather dumb and will probably only work properly on single
+ * vga card setups and/or x86 platforms.
+ *
+ * If your VGA default device is not PCI, you'll have to return
+ * NULL here. In this case, I assume it will not conflict with
+ * any PCI card. If this is not true, I'll have to define two archs
+ * hooks for enabling/disabling the VGA default device if that is
+ * possible. This may be a problem with real _ISA_ VGA cards, in
+ * addition to a PCI one. I don't know at this point how to deal
+ * with that card. Can theirs IOs be disabled at all ? If not, then
+ * I suppose it's a matter of having the proper arch hook telling
+ * us about it, so we basically never allow anybody to succeed a
+ * vga_get()...
+ */
 struct pci_dev *vga_default_device(void)
 {
 	return vga_default;
@@ -356,6 +373,40 @@ static void __vga_put(struct vga_device *vgadev, unsigned int rsrc)
 		wake_up_all(&vga_wait_queue);
 }
 
+/**
+ * vga_get - acquire & locks VGA resources
+ * @pdev: pci device of the VGA card or NULL for the system default
+ * @rsrc: bit mask of resources to acquire and lock
+ * @interruptible: blocking should be interruptible by signals ?
+ *
+ * This function acquires VGA resources for the given card and mark those
+ * resources locked. If the resource requested are "normal" (and not legacy)
+ * resources, the arbiter will first check whether the card is doing legacy
+ * decoding for that type of resource. If yes, the lock is "converted" into a
+ * legacy resource lock.
+ *
+ * The arbiter will first look for all VGA cards that might conflict and disable
+ * their IOs and/or Memory access, including VGA forwarding on P2P bridges if
+ * necessary, so that the requested resources can be used. Then, the card is
+ * marked as locking these resources and the IO and/or Memory accesses are
+ * enabled on the card (including VGA forwarding on parent P2P bridges if any).
+ *
+ * This function will block if some conflicting card is already locking one of
+ * the required resources (or any resource on a different bus segment, since P2P
+ * bridges don't differentiate VGA memory and IO afaik). You can indicate
+ * whether this blocking should be interruptible by a signal (for userland
+ * interface) or not.
+ *
+ * Must not be called at interrupt time or in atomic context.  If the card
+ * already owns the resources, the function succeeds.  Nested calls are
+ * supported (a per-resource counter is maintained)
+ *
+ * On success, release the VGA resource again with vga_put().
+ *
+ * Returns:
+ *
+ * 0 on success, negative error code on failure.
+ */
 int vga_get(struct pci_dev *pdev, unsigned int rsrc, int interruptible)
 {
 	struct vga_device *vgadev, *conflict;
@@ -408,6 +459,21 @@ int vga_get(struct pci_dev *pdev, unsigned int rsrc, int interruptible)
 }
 EXPORT_SYMBOL(vga_get);
 
+/**
+ * vga_tryget - try to acquire & lock legacy VGA resources
+ * @pdev: pci devivce of VGA card or NULL for system default
+ * @rsrc: bit mask of resources to acquire and lock
+ *
+ * This function performs the same operation as vga_get(), but will return an
+ * error (-EBUSY) instead of blocking if the resources are already locked by
+ * another card. It can be called in any context
+ *
+ * On success, release the VGA resource again with vga_put().
+ *
+ * Returns:
+ *
+ * 0 on success, negative error code on failure.
+ */
 int vga_tryget(struct pci_dev *pdev, unsigned int rsrc)
 {
 	struct vga_device *vgadev;
@@ -435,6 +501,16 @@ bail:
 }
 EXPORT_SYMBOL(vga_tryget);
 
+/**
+ * vga_put - release lock on legacy VGA resources
+ * @pdev: pci device of VGA card or NULL for system default
+ * @rsrc: but mask of resource to release
+ *
+ * This fuction releases resources previously locked by vga_get() or
+ * vga_tryget(). The resources aren't disabled right away, so that a subsequence
+ * vga_get() on the same card will succeed immediately. Resources have a
+ * counter, so locks are only released if the counter reaches 0.
+ */
 void vga_put(struct pci_dev *pdev, unsigned int rsrc)
 {
 	struct vga_device *vgadev;
@@ -716,7 +792,37 @@ void vga_set_legacy_decoding(struct pci_dev *pdev, unsigned int decodes)
 }
 EXPORT_SYMBOL(vga_set_legacy_decoding);
 
-/* call with NULL to unregister */
+/**
+ * vga_client_register - register or unregister a VGA arbitration client
+ * @pdev: pci device of the VGA client
+ * @cookie: client cookie to be used in callbacks
+ * @irq_set_state: irq state change callback
+ * @set_vga_decode: vga decode change callback
+ *
+ * Clients have two callback mechanisms they can use.
+ *
+ * @irq_set_state callback: If a client can't disable its GPUs VGA
+ * resources, then we need to be able to ask it to turn off its irqs when we
+ * turn off its mem and io decoding.
+ *
+ * @set_vga_decode callback: If a client can disable its GPU VGA resource, it
+ * will get a callback from this to set the encode/decode state.
+ *
+ * Rationale: we cannot disable VGA decode resources unconditionally some single
+ * GPU laptops seem to require ACPI or BIOS access to the VGA registers to
+ * control things like backlights etc.  Hopefully newer multi-GPU laptops do
+ * something saner, and desktops won't have any special ACPI for this. The
+ * driver will get a callback when VGA arbitration is first used by userspace
+ * since some older X servers have issues.
+ *
+ * This function does not check whether a client for @pdev has been registered
+ * already.
+ *
+ * To unregister just call this function with @irq_set_state and @set_vga_decode
+ * both set to NULL for the same @pdev as originally used to register them.
+ *
+ * Returns: 0 on success, -1 on failure
+ */
 int vga_client_register(struct pci_dev *pdev, void *cookie,
 			void (*irq_set_state)(void *cookie, bool state),
 			unsigned int (*set_vga_decode)(void *cookie,
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index 78ac4811bd3c..cd4599c0523b 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -457,8 +457,6 @@ config LOGITECH_FF
 	  - Logitech WingMan Cordless RumblePad
 	  - Logitech WingMan Cordless RumblePad 2
 	  - Logitech WingMan Force 3D
-	  - Logitech Formula Force EX
-	  - Logitech WingMan Formula Force GP
 
 	  and if you want to enable force feedback for them.
 	  Note: if you say N here, this device will still be supported, but without
@@ -488,15 +486,22 @@ config LOGIWHEELS_FF
 	select INPUT_FF_MEMLESS
 	default LOGITECH_FF
 	help
-	  Say Y here if you want to enable force feedback and range setting
+	  Say Y here if you want to enable force feedback and range setting(*)
 	  support for following Logitech wheels:
+	  - Logitech G25 (*)
+	  - Logitech G27 (*)
+	  - Logitech G29 (*)
 	  - Logitech Driving Force
-	  - Logitech Driving Force Pro
-	  - Logitech Driving Force GT
-	  - Logitech G25
-	  - Logitech G27
-	  - Logitech MOMO/MOMO 2
-	  - Logitech Formula Force EX
+	  - Logitech Driving Force Pro (*)
+	  - Logitech Driving Force GT (*)
+	  - Logitech Driving Force EX/RX
+	  - Logitech Driving Force Wireless
+	  - Logitech Speed Force Wireless
+	  - Logitech MOMO Force
+	  - Logitech MOMO Racing Force
+	  - Logitech Formula Force GP
+	  - Logitech Formula Force EX/RX
+	  - Logitech Wingman Formula Force GP
 
 config HID_MAGICMOUSE
 	tristate "Apple Magic Mouse/Trackpad multi-touch support"
@@ -862,6 +867,7 @@ config HID_WACOM
 	select POWER_SUPPLY
 	select NEW_LEDS
 	select LEDS_CLASS
+	select LEDS_TRIGGERS
 	help
 	  Say Y here if you want to use the USB or BT version of the Wacom Intuos
 	  or Graphire tablet.
@@ -967,4 +973,6 @@ source "drivers/hid/usbhid/Kconfig"
 
 source "drivers/hid/i2c-hid/Kconfig"
 
+source "drivers/hid/intel-ish-hid/Kconfig"
+
 endmenu
diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile
index fc4b2aa47f2e..86b2b5785fd2 100644
--- a/drivers/hid/Makefile
+++ b/drivers/hid/Makefile
@@ -113,3 +113,5 @@ obj-$(CONFIG_USB_MOUSE)		+= usbhid/
 obj-$(CONFIG_USB_KBD)		+= usbhid/
 
 obj-$(CONFIG_I2C_HID)		+= i2c-hid/
+
+obj-$(CONFIG_INTEL_ISH_HID)	+= intel-ish-hid/
diff --git a/drivers/hid/hid-alps.c b/drivers/hid/hid-alps.c
index 048befde295a..ed9c0ea5b026 100644
--- a/drivers/hid/hid-alps.c
+++ b/drivers/hid/hid-alps.c
@@ -139,8 +139,8 @@ static int u1_read_write_register(struct hid_device *hdev, u32 address,
 	if (read_flag) {
 		readbuf = kzalloc(U1_FEATURE_REPORT_LEN, GFP_KERNEL);
 		if (!readbuf) {
-			kfree(input);
-			return -ENOMEM;
+			ret = -ENOMEM;
+			goto exit;
 		}
 
 		ret = hid_hw_raw_request(hdev, U1_FEATURE_REPORT_ID, readbuf,
@@ -149,6 +149,7 @@ static int u1_read_write_register(struct hid_device *hdev, u32 address,
 
 		if (ret < 0) {
 			dev_err(&hdev->dev, "failed read register (%d)\n", ret);
+			kfree(readbuf);
 			goto exit;
 		}
 
@@ -190,16 +191,16 @@ static int alps_raw_event(struct hid_device *hdev,
 			if (z != 0) {
 				input_mt_report_slot_state(hdata->input,
 					MT_TOOL_FINGER, 1);
+				input_report_abs(hdata->input,
+					ABS_MT_POSITION_X, x);
+				input_report_abs(hdata->input,
+					ABS_MT_POSITION_Y, y);
+				input_report_abs(hdata->input,
+					ABS_MT_PRESSURE, z);
 			} else {
 				input_mt_report_slot_state(hdata->input,
 					MT_TOOL_FINGER, 0);
-				break;
 			}
-
-			input_report_abs(hdata->input, ABS_MT_POSITION_X, x);
-			input_report_abs(hdata->input, ABS_MT_POSITION_Y, y);
-			input_report_abs(hdata->input, ABS_MT_PRESSURE, z);
-
 		}
 
 		input_mt_sync_frame(hdata->input);
@@ -244,13 +245,13 @@ static int alps_raw_event(struct hid_device *hdev,
 static int alps_post_reset(struct hid_device *hdev)
 {
 	return u1_read_write_register(hdev, ADDRESS_U1_DEV_CTRL_1,
-				NULL, U1_TP_ABS_MODE, false);
+				NULL, U1_TP_ABS_MODE | U1_SP_ABS_MODE, false);
 }
 
 static int alps_post_resume(struct hid_device *hdev)
 {
 	return u1_read_write_register(hdev, ADDRESS_U1_DEV_CTRL_1,
-				NULL, U1_TP_ABS_MODE, false);
+				NULL, U1_TP_ABS_MODE | U1_SP_ABS_MODE, false);
 }
 #endif /* CONFIG_PM */
 
@@ -383,7 +384,7 @@ static int alps_input_configured(struct hid_device *hdev, struct hid_input *hi)
 
 		input2 = input_allocate_device();
 		if (!input2) {
-			input_free_device(input2);
+			ret = -ENOMEM;
 			goto exit;
 		}
 
@@ -425,7 +426,8 @@ static int alps_input_configured(struct hid_device *hdev, struct hid_input *hi)
 		__set_bit(INPUT_PROP_POINTER, input2->propbit);
 		__set_bit(INPUT_PROP_POINTING_STICK, input2->propbit);
 
-		if (input_register_device(data->input2)) {
+		ret = input_register_device(data->input2);
+		if (ret) {
 			input_free_device(input2);
 			goto exit;
 		}
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index 08f53c7fd513..2b89c701076f 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -727,6 +727,7 @@ static void hid_scan_collection(struct hid_parser *parser, unsigned type)
 	    (hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_3 ||
 	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_2 ||
 	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP ||
+	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP ||
 	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_3 ||
 	     hid->product == USB_DEVICE_ID_MS_POWER_COVER) &&
 	    hid->group == HID_GROUP_MULTITOUCH)
@@ -1916,7 +1917,7 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_ERGO_525V) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_I405X) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M610X) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_PENSKETCH_M912) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LABTEC, USB_DEVICE_ID_LABTEC_WIRELESS_KEYBOARD) },
@@ -1982,6 +1983,7 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_2) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_3) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_7K) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_600) },
@@ -2037,6 +2039,7 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_PS1000) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT7_OLD) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT7) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT9) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_MMO7) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MADCATZ, USB_DEVICE_ID_MADCATZ_RAT5) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MADCATZ, USB_DEVICE_ID_MADCATZ_RAT9) },
@@ -2083,6 +2086,11 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP1062) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_WIRELESS_TABLET_TWHL850) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_TWHA60) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_YIYNOVA_TABLET) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_81) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_45) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_DRAWIMAGE_G3) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UGTIZER, USB_DEVICE_ID_UGTIZER_TABLET_GP0610) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_SMARTJOY_PLUS) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_SUPER_JOY_BOX_3) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_DUAL_USB_JOYPAD) },
@@ -2480,7 +2488,7 @@ static const struct hid_device_id hid_ignore_list[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_PANJIT, 0x0004) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_PHILIPS, USB_DEVICE_ID_PHILIPS_IEEE802154_DONGLE) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_POWERCOM, USB_DEVICE_ID_POWERCOM_UPS) },
-#if defined(CONFIG_MOUSE_SYNAPTICS_USB) || defined(CONFIG_MOUSE_SYNAPTICS_USB_MODULE)
+#if IS_ENABLED(CONFIG_MOUSE_SYNAPTICS_USB)
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_TP) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_INT_TP) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_CPAD) },
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 4ed9a4fdfea7..cd59c79eebdd 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -268,6 +268,7 @@
 #define USB_DEVICE_ID_CORSAIR_K95RGB    0x1b11
 #define USB_DEVICE_ID_CORSAIR_M65RGB    0x1b12
 #define USB_DEVICE_ID_CORSAIR_K70RGB    0x1b13
+#define USB_DEVICE_ID_CORSAIR_STRAFE    0x1b15
 #define USB_DEVICE_ID_CORSAIR_K65RGB    0x1b17
 
 #define USB_VENDOR_ID_CREATIVELABS	0x041e
@@ -565,7 +566,7 @@
 #define USB_DEVICE_ID_KYE_GPEN_560	0x5003
 #define USB_DEVICE_ID_KYE_EASYPEN_I405X	0x5010
 #define USB_DEVICE_ID_KYE_MOUSEPEN_I608X	0x5011
-#define USB_DEVICE_ID_KYE_MOUSEPEN_I608X_2	0x501a
+#define USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2	0x501a
 #define USB_DEVICE_ID_KYE_EASYPEN_M610X	0x5013
 #define USB_DEVICE_ID_KYE_PENSKETCH_M912	0x5015
 
@@ -713,6 +714,7 @@
 #define USB_DEVICE_ID_MS_TYPE_COVER_PRO_3    0x07dc
 #define USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_2  0x07e2
 #define USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP 0x07dd
+#define USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP 0x07e9
 #define USB_DEVICE_ID_MS_TYPE_COVER_3    0x07de
 #define USB_DEVICE_ID_MS_POWER_COVER     0x07da
 
@@ -859,6 +861,7 @@
 #define USB_DEVICE_ID_SAITEK_PS1000	0x0621
 #define USB_DEVICE_ID_SAITEK_RAT7_OLD	0x0ccb
 #define USB_DEVICE_ID_SAITEK_RAT7	0x0cd7
+#define USB_DEVICE_ID_SAITEK_RAT9	0x0cfa
 #define USB_DEVICE_ID_SAITEK_MMO7	0x0cd0
 
 #define USB_VENDOR_ID_SAMSUNG		0x0419
@@ -996,6 +999,10 @@
 #define USB_DEVICE_ID_UCLOGIC_TABLET_WP1062	0x0064
 #define USB_DEVICE_ID_UCLOGIC_WIRELESS_TABLET_TWHL850	0x0522
 #define USB_DEVICE_ID_UCLOGIC_TABLET_TWHA60	0x0781
+#define USB_DEVICE_ID_UCLOGIC_DRAWIMAGE_G3	0x3031
+#define USB_DEVICE_ID_UGEE_TABLET_81		0x0081
+#define USB_DEVICE_ID_UGEE_TABLET_45		0x0045
+#define USB_DEVICE_ID_YIYNOVA_TABLET		0x004d
 
 #define USB_VENDOR_ID_UNITEC	0x227d
 #define USB_DEVICE_ID_UNITEC_USB_TOUCH_0709	0x0709
@@ -1085,4 +1092,7 @@
 #define USB_DEVICE_ID_RAPHNET_2NES2SNES	0x0002
 #define USB_DEVICE_ID_RAPHNET_4NES4SNES	0x0003
 
+#define USB_VENDOR_ID_UGTIZER			0x2179
+#define USB_DEVICE_ID_UGTIZER_TABLET_GP0610	0x0053
+
 #endif
diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c
index bcfaf32d9e5e..fb9ace1cef8b 100644
--- a/drivers/hid/hid-input.c
+++ b/drivers/hid/hid-input.c
@@ -604,6 +604,15 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
 			break;
 		}
 
+		/*
+		 * Some lazy vendors declare 255 usages for System Control,
+		 * leading to the creation of ABS_X|Y axis and too many others.
+		 * It wouldn't be a problem if joydev doesn't consider the
+		 * device as a joystick then.
+		 */
+		if (field->application == HID_GD_SYSTEM_CONTROL)
+			goto ignore;
+
 		if ((usage->hid & 0xf0) == 0x90) {	/* D-pad */
 			switch (usage->hid) {
 			case HID_GD_UP:	   usage->hat_dir = 1; break;
@@ -953,6 +962,7 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
 	case HID_UP_HPVENDOR2:
 		set_bit(EV_REP, input->evbit);
 		switch (usage->hid & HID_USAGE) {
+		case 0x001: map_key_clear(KEY_MICMUTE);		break;
 		case 0x003: map_key_clear(KEY_BRIGHTNESSDOWN);	break;
 		case 0x004: map_key_clear(KEY_BRIGHTNESSUP);	break;
 		default:    goto ignore;
diff --git a/drivers/hid/hid-kye.c b/drivers/hid/hid-kye.c
index 32e6d8d9ded0..0dd1167b2c9b 100644
--- a/drivers/hid/hid-kye.c
+++ b/drivers/hid/hid-kye.c
@@ -19,11 +19,6 @@
 
 #include "hid-ids.h"
 
-/*
- * See EasyPen i405X description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=KYE_EasyPen_i405X
- */
-
 /* Original EasyPen i405X report descriptor size */
 #define EASYPEN_I405X_RDESC_ORIG_SIZE	476
 
@@ -82,11 +77,6 @@ static __u8 easypen_i405x_rdesc_fixed[] = {
 	0xC0              /*  End Collection                  */
 };
 
-/*
- * See MousePen i608X description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=KYE_MousePen_i608X
- */
-
 /* Original MousePen i608X report descriptor size */
 #define MOUSEPEN_I608X_RDESC_ORIG_SIZE	476
 
@@ -186,10 +176,104 @@ static __u8 mousepen_i608x_rdesc_fixed[] = {
 	0xC0              /*  End Collection                  */
 };
 
-/*
- * See EasyPen M610X description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=KYE_EasyPen_M610X
- */
+/* Original MousePen i608X v2 report descriptor size */
+#define MOUSEPEN_I608X_V2_RDESC_ORIG_SIZE	482
+
+/* Fixed MousePen i608X v2 report descriptor */
+static __u8 mousepen_i608x_v2_rdesc_fixed[] = {
+	0x06, 0x00, 0xFF,             /*  Usage Page (FF00h),             */
+	0x09, 0x01,                   /*  Usage (01h),                    */
+	0xA1, 0x01,                   /*  Collection (Application),       */
+	0x85, 0x05,                   /*    Report ID (5),                */
+	0x09, 0x01,                   /*    Usage (01h),                  */
+	0x15, 0x80,                   /*    Logical Minimum (-128),       */
+	0x25, 0x7F,                   /*    Logical Maximum (127),        */
+	0x75, 0x08,                   /*    Report Size (8),              */
+	0x95, 0x07,                   /*    Report Count (7),             */
+	0xB1, 0x02,                   /*    Feature (Variable),           */
+	0xC0,                         /*  End Collection,                 */
+	0x05, 0x0D,                   /*  Usage Page (Digitizer),         */
+	0x09, 0x02,                   /*  Usage (Pen),                    */
+	0xA1, 0x01,                   /*  Collection (Application),       */
+	0x85, 0x10,                   /*    Report ID (16),               */
+	0x09, 0x20,                   /*    Usage (Stylus),               */
+	0xA0,                         /*    Collection (Physical),        */
+	0x14,                         /*      Logical Minimum (0),        */
+	0x25, 0x01,                   /*      Logical Maximum (1),        */
+	0x75, 0x01,                   /*      Report Size (1),            */
+	0x09, 0x42,                   /*      Usage (Tip Switch),         */
+	0x09, 0x44,                   /*      Usage (Barrel Switch),      */
+	0x09, 0x46,                   /*      Usage (Tablet Pick),        */
+	0x95, 0x03,                   /*      Report Count (3),           */
+	0x81, 0x02,                   /*      Input (Variable),           */
+	0x95, 0x04,                   /*      Report Count (4),           */
+	0x81, 0x03,                   /*      Input (Constant, Variable), */
+	0x09, 0x32,                   /*      Usage (In Range),           */
+	0x95, 0x01,                   /*      Report Count (1),           */
+	0x81, 0x02,                   /*      Input (Variable),           */
+	0x75, 0x10,                   /*      Report Size (16),           */
+	0x95, 0x01,                   /*      Report Count (1),           */
+	0xA4,                         /*      Push,                       */
+	0x05, 0x01,                   /*      Usage Page (Desktop),       */
+	0x55, 0xFD,                   /*      Unit Exponent (-3),         */
+	0x65, 0x13,                   /*      Unit (Inch),                */
+	0x34,                         /*      Physical Minimum (0),       */
+	0x09, 0x30,                   /*      Usage (X),                  */
+	0x46, 0x40, 0x1F,             /*      Physical Maximum (8000),    */
+	0x27, 0x00, 0xA0, 0x00, 0x00, /*      Logical Maximum (40960),    */
+	0x81, 0x02,                   /*      Input (Variable),           */
+	0x09, 0x31,                   /*      Usage (Y),                  */
+	0x46, 0x70, 0x17,             /*      Physical Maximum (6000),    */
+	0x26, 0x00, 0x78,             /*      Logical Maximum (30720),    */
+	0x81, 0x02,                   /*      Input (Variable),           */
+	0xB4,                         /*      Pop,                        */
+	0x09, 0x30,                   /*      Usage (Tip Pressure),       */
+	0x26, 0xFF, 0x07,             /*      Logical Maximum (2047),     */
+	0x81, 0x02,                   /*      Input (Variable),           */
+	0xC0,                         /*    End Collection,               */
+	0xC0,                         /*  End Collection,                 */
+	0x05, 0x01,                   /*  Usage Page (Desktop),           */
+	0x09, 0x02,                   /*  Usage (Mouse),                  */
+	0xA1, 0x01,                   /*  Collection (Application),       */
+	0x85, 0x11,                   /*    Report ID (17),               */
+	0x09, 0x01,                   /*    Usage (Pointer),              */
+	0xA0,                         /*    Collection (Physical),        */
+	0x14,                         /*      Logical Minimum (0),        */
+	0xA4,                         /*      Push,                       */
+	0x05, 0x09,                   /*      Usage Page (Button),        */
+	0x75, 0x01,                   /*      Report Size (1),            */
+	0x19, 0x01,                   /*      Usage Minimum (01h),        */
+	0x29, 0x03,                   /*      Usage Maximum (03h),        */
+	0x25, 0x01,                   /*      Logical Maximum (1),        */
+	0x95, 0x03,                   /*      Report Count (3),           */
+	0x81, 0x02,                   /*      Input (Variable),           */
+	0x95, 0x05,                   /*      Report Count (5),           */
+	0x81, 0x01,                   /*      Input (Constant),           */
+	0xB4,                         /*      Pop,                        */
+	0x95, 0x01,                   /*      Report Count (1),           */
+	0xA4,                         /*      Push,                       */
+	0x55, 0xFD,                   /*      Unit Exponent (-3),         */
+	0x65, 0x13,                   /*      Unit (Inch),                */
+	0x34,                         /*      Physical Minimum (0),       */
+	0x75, 0x10,                   /*      Report Size (16),           */
+	0x09, 0x30,                   /*      Usage (X),                  */
+	0x46, 0x40, 0x1F,             /*      Physical Maximum (8000),    */
+	0x27, 0x00, 0xA0, 0x00, 0x00, /*      Logical Maximum (40960),    */
+	0x81, 0x02,                   /*      Input (Variable),           */
+	0x09, 0x31,                   /*      Usage (Y),                  */
+	0x46, 0x70, 0x17,             /*      Physical Maximum (6000),    */
+	0x26, 0x00, 0x78,             /*      Logical Maximum (30720),    */
+	0x81, 0x02,                   /*      Input (Variable),           */
+	0xB4,                         /*      Pop,                        */
+	0x75, 0x08,                   /*      Report Size (8),            */
+	0x09, 0x38,                   /*      Usage (Wheel),              */
+	0x15, 0xFF,                   /*      Logical Minimum (-1),       */
+	0x25, 0x01,                   /*      Logical Maximum (1),        */
+	0x81, 0x06,                   /*      Input (Variable, Relative), */
+	0x81, 0x01,                   /*      Input (Constant),           */
+	0xC0,                         /*    End Collection,               */
+	0xC0                          /*  End Collection                  */
+};
 
 /* Original EasyPen M610X report descriptor size */
 #define EASYPEN_M610X_RDESC_ORIG_SIZE	476
@@ -454,12 +538,17 @@ static __u8 *kye_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		}
 		break;
 	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X:
-	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X_2:
 		if (*rsize == MOUSEPEN_I608X_RDESC_ORIG_SIZE) {
 			rdesc = mousepen_i608x_rdesc_fixed;
 			*rsize = sizeof(mousepen_i608x_rdesc_fixed);
 		}
 		break;
+	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2:
+		if (*rsize == MOUSEPEN_I608X_V2_RDESC_ORIG_SIZE) {
+			rdesc = mousepen_i608x_v2_rdesc_fixed;
+			*rsize = sizeof(mousepen_i608x_v2_rdesc_fixed);
+		}
+		break;
 	case USB_DEVICE_ID_KYE_EASYPEN_M610X:
 		if (*rsize == EASYPEN_M610X_RDESC_ORIG_SIZE) {
 			rdesc = easypen_m610x_rdesc_fixed;
@@ -553,7 +642,7 @@ static int kye_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	switch (id->product) {
 	case USB_DEVICE_ID_KYE_EASYPEN_I405X:
 	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X:
-	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X_2:
+	case USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2:
 	case USB_DEVICE_ID_KYE_EASYPEN_M610X:
 	case USB_DEVICE_ID_KYE_PENSKETCH_M912:
 		ret = kye_tablet_enable(hdev);
@@ -586,7 +675,7 @@ static const struct hid_device_id kye_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
 				USB_DEVICE_ID_KYE_MOUSEPEN_I608X) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
-				USB_DEVICE_ID_KYE_MOUSEPEN_I608X_2) },
+				USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
 				USB_DEVICE_ID_KYE_EASYPEN_M610X) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE,
diff --git a/drivers/hid/hid-lg.c b/drivers/hid/hid-lg.c
index feb2be71f77c..76f644deb0a7 100644
--- a/drivers/hid/hid-lg.c
+++ b/drivers/hid/hid-lg.c
@@ -49,6 +49,7 @@
 #define FV_RDESC_ORIG_SIZE	130
 #define MOMO_RDESC_ORIG_SIZE	87
 #define MOMO2_RDESC_ORIG_SIZE	87
+#define FFG_RDESC_ORIG_SIZE	85
 
 /* Fixed report descriptors for Logitech Driving Force (and Pro)
  * wheel controllers
@@ -334,6 +335,52 @@ static __u8 momo2_rdesc_fixed[] = {
 0xC0                /*  End Collection                      */
 };
 
+static __u8 ffg_rdesc_fixed[] = {
+0x05, 0x01,         /*  Usage Page (Desktop),               */
+0x09, 0x04,         /*  Usage (Joystik),                    */
+0xA1, 0x01,         /*  Collection (Application),           */
+0xA1, 0x02,         /*      Collection (Logical),           */
+0x95, 0x01,         /*          Report Count (1),           */
+0x75, 0x0A,         /*          Report Size (10),           */
+0x15, 0x00,         /*          Logical Minimum (0),        */
+0x26, 0xFF, 0x03,   /*          Logical Maximum (1023),     */
+0x35, 0x00,         /*          Physical Minimum (0),       */
+0x46, 0xFF, 0x03,   /*          Physical Maximum (1023),    */
+0x09, 0x30,         /*          Usage (X),                  */
+0x81, 0x02,         /*          Input (Variable),           */
+0x95, 0x06,         /*          Report Count (6),           */
+0x75, 0x01,         /*          Report Size (1),            */
+0x25, 0x01,         /*          Logical Maximum (1),        */
+0x45, 0x01,         /*          Physical Maximum (1),       */
+0x05, 0x09,         /*          Usage Page (Button),        */
+0x19, 0x01,         /*          Usage Minimum (01h),        */
+0x29, 0x06,         /*          Usage Maximum (06h),        */
+0x81, 0x02,         /*          Input (Variable),           */
+0x95, 0x01,         /*          Report Count (1),           */
+0x75, 0x08,         /*          Report Size (8),            */
+0x26, 0xFF, 0x00,   /*          Logical Maximum (255),      */
+0x46, 0xFF, 0x00,   /*          Physical Maximum (255),     */
+0x06, 0x00, 0xFF,   /*          Usage Page (FF00h),         */
+0x09, 0x01,         /*          Usage (01h),                */
+0x81, 0x02,         /*          Input (Variable),           */
+0x05, 0x01,         /*          Usage Page (Desktop),       */
+0x81, 0x01,         /*          Input (Constant),           */
+0x09, 0x31,         /*          Usage (Y),                  */
+0x81, 0x02,         /*          Input (Variable),           */
+0x09, 0x32,         /*          Usage (Z),                  */
+0x81, 0x02,         /*          Input (Variable),           */
+0x06, 0x00, 0xFF,   /*          Usage Page (FF00h),         */
+0x09, 0x01,         /*          Usage (01h),                */
+0x81, 0x02,         /*          Input (Variable),           */
+0xC0,               /*      End Collection,                 */
+0xA1, 0x02,         /*      Collection (Logical),           */
+0x09, 0x02,         /*          Usage (02h),                */
+0x95, 0x07,         /*          Report Count (7),           */
+0x91, 0x02,         /*          Output (Variable),          */
+0xC0,               /*      End Collection,                 */
+0xC0                /*  End Collection                      */
+};
+
 /*
  * Certain Logitech keyboards send in report #3 keys which are far
  * above the logical maximum described in descriptor. This extends
@@ -343,8 +390,6 @@ static __u8 *lg_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		unsigned int *rsize)
 {
 	struct lg_drv_data *drv_data = hid_get_drvdata(hdev);
-	struct usb_device_descriptor *udesc;
-	__u16 bcdDevice, rev_maj, rev_min;
 
 	if ((drv_data->quirks & LG_RDESC) && *rsize >= 91 && rdesc[83] == 0x26 &&
 			rdesc[84] == 0x8c && rdesc[85] == 0x02) {
@@ -363,20 +408,18 @@ static __u8 *lg_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 
 	switch (hdev->product) {
 
-	/* Several wheels report as this id when operating in emulation mode. */
-	case USB_DEVICE_ID_LOGITECH_WHEEL:
-		udesc = &(hid_to_usb_dev(hdev)->descriptor);
-		if (!udesc) {
-			hid_err(hdev, "NULL USB device descriptor\n");
-			break;
+	case USB_DEVICE_ID_LOGITECH_WINGMAN_FFG:
+		if (*rsize == FFG_RDESC_ORIG_SIZE) {
+			hid_info(hdev,
+				"fixing up Logitech Wingman Formula Force GP report descriptor\n");
+			rdesc = ffg_rdesc_fixed;
+			*rsize = sizeof(ffg_rdesc_fixed);
 		}
-		bcdDevice = le16_to_cpu(udesc->bcdDevice);
-		rev_maj = bcdDevice >> 8;
-		rev_min = bcdDevice & 0xff;
+		break;
 
-		/* Update the report descriptor for only the Driving Force wheel */
-		if (rev_maj == 1 && rev_min == 2 &&
-				*rsize == DF_RDESC_ORIG_SIZE) {
+	/* Several wheels report as this id when operating in emulation mode. */
+	case USB_DEVICE_ID_LOGITECH_WHEEL:
+		if (*rsize == DF_RDESC_ORIG_SIZE) {
 			hid_info(hdev,
 				"fixing up Logitech Driving Force report descriptor\n");
 			rdesc = df_rdesc_fixed;
@@ -621,6 +664,7 @@ static int lg_input_mapped(struct hid_device *hdev, struct hid_input *hi,
 			usage->code == ABS_RZ)) {
 		switch (hdev->product) {
 		case USB_DEVICE_ID_LOGITECH_G29_WHEEL:
+		case USB_DEVICE_ID_LOGITECH_WINGMAN_FFG:
 		case USB_DEVICE_ID_LOGITECH_WHEEL:
 		case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL:
 		case USB_DEVICE_ID_LOGITECH_DFP_WHEEL:
@@ -657,6 +701,17 @@ static int lg_event(struct hid_device *hdev, struct hid_field *field,
 	return 0;
 }
 
+static int lg_raw_event(struct hid_device *hdev, struct hid_report *report,
+		u8 *rd, int size)
+{
+	struct lg_drv_data *drv_data = hid_get_drvdata(hdev);
+
+	if (drv_data->quirks & LG_FF4)
+		return lg4ff_raw_event(hdev, report, rd, size, drv_data);
+
+	return 0;
+}
+
 static int lg_probe(struct hid_device *hdev, const struct hid_device_id *id)
 {
 	struct usb_interface *iface = to_usb_interface(hdev->dev.parent);
@@ -809,7 +864,7 @@ static const struct hid_device_id lg_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WII_WHEEL),
 		.driver_data = LG_FF4 },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WINGMAN_FFG),
-		.driver_data = LG_FF },
+		.driver_data = LG_NOGET | LG_FF4 },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD2),
 		.driver_data = LG_FF2 },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_FLIGHT_SYSTEM_G940),
@@ -830,6 +885,7 @@ static struct hid_driver lg_driver = {
 	.input_mapping = lg_input_mapping,
 	.input_mapped = lg_input_mapped,
 	.event = lg_event,
+	.raw_event = lg_raw_event,
 	.probe = lg_probe,
 	.remove = lg_remove,
 };
diff --git a/drivers/hid/hid-lg4ff.c b/drivers/hid/hid-lg4ff.c
index af3a8ec8a746..1fc12e357035 100644
--- a/drivers/hid/hid-lg4ff.c
+++ b/drivers/hid/hid-lg4ff.c
@@ -75,6 +75,7 @@ static void lg4ff_set_range_g25(struct hid_device *hid, u16 range);
 
 struct lg4ff_wheel_data {
 	const u32 product_id;
+	u16 combine;
 	u16 range;
 	const u16 min_range;
 	const u16 max_range;
@@ -136,6 +137,7 @@ struct lg4ff_alternate_mode {
 };
 
 static const struct lg4ff_wheel lg4ff_devices[] = {
+	{USB_DEVICE_ID_LOGITECH_WINGMAN_FFG, lg4ff_wheel_effects, 40, 180, NULL},
 	{USB_DEVICE_ID_LOGITECH_WHEEL,       lg4ff_wheel_effects, 40, 270, NULL},
 	{USB_DEVICE_ID_LOGITECH_MOMO_WHEEL,  lg4ff_wheel_effects, 40, 270, NULL},
 	{USB_DEVICE_ID_LOGITECH_DFP_WHEEL,   lg4ff_wheel_effects, 40, 900, lg4ff_set_range_dfp},
@@ -328,6 +330,56 @@ int lg4ff_adjust_input_event(struct hid_device *hid, struct hid_field *field,
 	}
 }
 
+int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report,
+		u8 *rd, int size, struct lg_drv_data *drv_data)
+{
+	int offset;
+	struct lg4ff_device_entry *entry = drv_data->device_props;
+
+	if (!entry)
+		return 0;
+
+	/* adjust HID report present combined pedals data */
+	if (entry->wdata.combine) {
+		switch (entry->wdata.product_id) {
+		case USB_DEVICE_ID_LOGITECH_WHEEL:
+			rd[5] = rd[3];
+			rd[6] = 0x7F;
+			return 1;
+		case USB_DEVICE_ID_LOGITECH_WINGMAN_FFG:
+		case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL:
+		case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL2:
+			rd[4] = rd[3];
+			rd[5] = 0x7F;
+			return 1;
+		case USB_DEVICE_ID_LOGITECH_DFP_WHEEL:
+			rd[5] = rd[4];
+			rd[6] = 0x7F;
+			return 1;
+		case USB_DEVICE_ID_LOGITECH_G25_WHEEL:
+		case USB_DEVICE_ID_LOGITECH_G27_WHEEL:
+			offset = 5;
+			break;
+		case USB_DEVICE_ID_LOGITECH_DFGT_WHEEL:
+		case USB_DEVICE_ID_LOGITECH_G29_WHEEL:
+			offset = 6;
+			break;
+		case USB_DEVICE_ID_LOGITECH_WII_WHEEL:
+			offset = 3;
+			break;
+		default:
+			return 0;
+		}
+
+		/* Compute a combined axis when wheel does not supply it */
+		rd[offset] = (0xFF + rd[offset] - rd[offset+1]) >> 1;
+		rd[offset+1] = 0x7F;
+		return 1;
+	}
+
+	return 0;
+}
+
 static void lg4ff_init_wheel_data(struct lg4ff_wheel_data * const wdata, const struct lg4ff_wheel *wheel,
 				  const struct lg4ff_multimode_wheel *mmode_wheel,
 				  const u16 real_product_id)
@@ -345,6 +397,7 @@ static void lg4ff_init_wheel_data(struct lg4ff_wheel_data * const wdata, const s
 	{
 		struct lg4ff_wheel_data t_wdata =  { .product_id = wheel->product_id,
 						     .real_product_id = real_product_id,
+						     .combine = 0,
 						     .min_range = wheel->min_range,
 						     .max_range = wheel->max_range,
 						     .set_range = wheel->set_range,
@@ -885,6 +938,58 @@ static ssize_t lg4ff_alternate_modes_store(struct device *dev, struct device_att
 }
 static DEVICE_ATTR(alternate_modes, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH, lg4ff_alternate_modes_show, lg4ff_alternate_modes_store);
 
+static ssize_t lg4ff_combine_show(struct device *dev, struct device_attribute *attr,
+				char *buf)
+{
+	struct hid_device *hid = to_hid_device(dev);
+	struct lg4ff_device_entry *entry;
+	struct lg_drv_data *drv_data;
+	size_t count;
+
+	drv_data = hid_get_drvdata(hid);
+	if (!drv_data) {
+		hid_err(hid, "Private driver data not found!\n");
+		return 0;
+	}
+
+	entry = drv_data->device_props;
+	if (!entry) {
+		hid_err(hid, "Device properties not found!\n");
+		return 0;
+	}
+
+	count = scnprintf(buf, PAGE_SIZE, "%u\n", entry->wdata.combine);
+	return count;
+}
+
+static ssize_t lg4ff_combine_store(struct device *dev, struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	struct hid_device *hid = to_hid_device(dev);
+	struct lg4ff_device_entry *entry;
+	struct lg_drv_data *drv_data;
+	u16 combine = simple_strtoul(buf, NULL, 10);
+
+	drv_data = hid_get_drvdata(hid);
+	if (!drv_data) {
+		hid_err(hid, "Private driver data not found!\n");
+		return -EINVAL;
+	}
+
+	entry = drv_data->device_props;
+	if (!entry) {
+		hid_err(hid, "Device properties not found!\n");
+		return -EINVAL;
+	}
+
+	if (combine > 1)
+		combine = 1;
+
+	entry->wdata.combine = combine;
+	return count;
+}
+static DEVICE_ATTR(combine_pedals, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH, lg4ff_combine_show, lg4ff_combine_store);
+
 /* Export the currently set range of the wheel */
 static ssize_t lg4ff_range_show(struct device *dev, struct device_attribute *attr,
 				char *buf)
@@ -1259,6 +1364,9 @@ int lg4ff_init(struct hid_device *hid)
 	}
 
 	/* Create sysfs interface */
+	error = device_create_file(&hid->dev, &dev_attr_combine_pedals);
+	if (error)
+		hid_warn(hid, "Unable to create sysfs interface for \"combine\", errno %d\n", error);
 	error = device_create_file(&hid->dev, &dev_attr_range);
 	if (error)
 		hid_warn(hid, "Unable to create sysfs interface for \"range\", errno %d\n", error);
@@ -1358,6 +1466,7 @@ int lg4ff_deinit(struct hid_device *hid)
 		device_remove_file(&hid->dev, &dev_attr_alternate_modes);
 	}
 
+	device_remove_file(&hid->dev, &dev_attr_combine_pedals);
 	device_remove_file(&hid->dev, &dev_attr_range);
 #ifdef CONFIG_LEDS_CLASS
 	{
diff --git a/drivers/hid/hid-lg4ff.h b/drivers/hid/hid-lg4ff.h
index 66201af44da3..de1f350e0bd3 100644
--- a/drivers/hid/hid-lg4ff.h
+++ b/drivers/hid/hid-lg4ff.h
@@ -6,11 +6,15 @@ extern int lg4ff_no_autoswitch; /* From hid-lg.c */
 
 int lg4ff_adjust_input_event(struct hid_device *hid, struct hid_field *field,
 			     struct hid_usage *usage, s32 value, struct lg_drv_data *drv_data);
+int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report,
+		u8 *rd, int size, struct lg_drv_data *drv_data);
 int lg4ff_init(struct hid_device *hdev);
 int lg4ff_deinit(struct hid_device *hdev);
 #else
 static inline int lg4ff_adjust_input_event(struct hid_device *hid, struct hid_field *field,
 					   struct hid_usage *usage, s32 value, struct lg_drv_data *drv_data) { return 0; }
+static inline int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report,
+		u8 *rd, int size, struct lg_drv_data *drv_data) { return 0; }
 static inline int lg4ff_init(struct hid_device *hdev) { return -1; }
 static inline int lg4ff_deinit(struct hid_device *hdev) { return -1; }
 #endif
diff --git a/drivers/hid/hid-microsoft.c b/drivers/hid/hid-microsoft.c
index e924d555536c..c6cd392e9f99 100644
--- a/drivers/hid/hid-microsoft.c
+++ b/drivers/hid/hid-microsoft.c
@@ -28,7 +28,6 @@
 #define MS_RDESC		0x08
 #define MS_NOGET		0x10
 #define MS_DUPLICATE_USAGES	0x20
-#define MS_RDESC_3K		0x40
 
 static __u8 *ms_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		unsigned int *rsize)
@@ -45,13 +44,6 @@ static __u8 *ms_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		rdesc[557] = 0x35;
 		rdesc[559] = 0x45;
 	}
-	/* the same as above (s/usage/physical/) */
-	if ((quirks & MS_RDESC_3K) && *rsize == 106 && rdesc[94] == 0x19 &&
-			rdesc[95] == 0x00 && rdesc[96] == 0x29 &&
-			rdesc[97] == 0xff) {
-		rdesc[94] = 0x35;
-		rdesc[96] = 0x45;
-	}
 	return rdesc;
 }
 
@@ -271,7 +263,7 @@ static const struct hid_device_id ms_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_PRESENTER_8K_USB),
 		.driver_data = MS_PRESENTER },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_3K),
-		.driver_data = MS_ERGONOMY | MS_RDESC_3K },
+		.driver_data = MS_ERGONOMY },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_7K),
 		.driver_data = MS_ERGONOMY },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_600),
@@ -288,6 +280,8 @@ static const struct hid_device_id ms_devices[] = {
 		.driver_data = MS_HIDINPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP),
 		.driver_data = MS_HIDINPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP),
+		.driver_data = MS_HIDINPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_3),
 		.driver_data = MS_HIDINPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER),
diff --git a/drivers/hid/hid-saitek.c b/drivers/hid/hid-saitek.c
index 2f84b26f1167..39e642686ff0 100644
--- a/drivers/hid/hid-saitek.c
+++ b/drivers/hid/hid-saitek.c
@@ -183,6 +183,8 @@ static const struct hid_device_id saitek_devices[] = {
 		.driver_data = SAITEK_RELEASE_MODE_RAT7 },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT7),
 		.driver_data = SAITEK_RELEASE_MODE_RAT7 },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT9),
+		.driver_data = SAITEK_RELEASE_MODE_RAT7 },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MADCATZ, USB_DEVICE_ID_MADCATZ_RAT9),
 		.driver_data = SAITEK_RELEASE_MODE_RAT7 },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_MMO7),
diff --git a/drivers/hid/hid-sensor-hub.c b/drivers/hid/hid-sensor-hub.c
index 3d5ba5b51af3..658a607dc6d9 100644
--- a/drivers/hid/hid-sensor-hub.c
+++ b/drivers/hid/hid-sensor-hub.c
@@ -16,6 +16,7 @@
  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  *
  */
+
 #include <linux/device.h>
 #include <linux/hid.h>
 #include <linux/module.h>
@@ -798,6 +799,9 @@ static const struct hid_device_id sensor_hub_devices[] = {
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_ITE,
 			USB_DEVICE_ID_ITE_LENOVO_YOGA900),
 			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
+	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_INTEL_0,
+			0x22D8),
+			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, HID_ANY_ID,
 		     HID_ANY_ID) },
 	{ }
diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c
index 310436a54a3f..b0bb99a821bd 100644
--- a/drivers/hid/hid-sony.c
+++ b/drivers/hid/hid-sony.c
@@ -8,7 +8,7 @@
  *  Copyright (c) 2012 David Dillow <dave@thedillows.org>
  *  Copyright (c) 2006-2013 Jiri Kosina
  *  Copyright (c) 2013 Colin Leitner <colin.leitner@gmail.com>
- *  Copyright (c) 2014 Frank Praznik <frank.praznik@gmail.com>
+ *  Copyright (c) 2014-2016 Frank Praznik <frank.praznik@gmail.com>
  */
 
 /*
@@ -51,6 +51,7 @@
 #define NAVIGATION_CONTROLLER_USB BIT(9)
 #define NAVIGATION_CONTROLLER_BT  BIT(10)
 #define SINO_LITE_CONTROLLER      BIT(11)
+#define FUTUREMAX_DANCE_MAT       BIT(12)
 
 #define SIXAXIS_CONTROLLER (SIXAXIS_CONTROLLER_USB | SIXAXIS_CONTROLLER_BT)
 #define MOTION_CONTROLLER (MOTION_CONTROLLER_USB | MOTION_CONTROLLER_BT)
@@ -65,6 +66,8 @@
 				MOTION_CONTROLLER_BT | NAVIGATION_CONTROLLER)
 #define SONY_FF_SUPPORT (SIXAXIS_CONTROLLER | DUALSHOCK4_CONTROLLER |\
 				MOTION_CONTROLLER)
+#define SONY_BT_DEVICE (SIXAXIS_CONTROLLER_BT | DUALSHOCK4_CONTROLLER_BT |\
+			MOTION_CONTROLLER_BT | NAVIGATION_CONTROLLER_BT)
 
 #define MAX_LEDS 4
 
@@ -1048,6 +1051,7 @@ struct sony_sc {
 
 	u8 mac_address[6];
 	u8 worker_initialized;
+	u8 defer_initialization;
 	u8 cable_state;
 	u8 battery_charging;
 	u8 battery_capacity;
@@ -1058,6 +1062,12 @@ struct sony_sc {
 	u8 led_count;
 };
 
+static inline void sony_schedule_work(struct sony_sc *sc)
+{
+	if (!sc->defer_initialization)
+		schedule_work(&sc->state_worker);
+}
+
 static u8 *sixaxis_fixup(struct hid_device *hdev, u8 *rdesc,
 			     unsigned int *rsize)
 {
@@ -1125,7 +1135,7 @@ static u8 *sony_report_fixup(struct hid_device *hdev, u8 *rdesc,
 {
 	struct sony_sc *sc = hid_get_drvdata(hdev);
 
-	if (sc->quirks & SINO_LITE_CONTROLLER)
+	if (sc->quirks & (SINO_LITE_CONTROLLER | FUTUREMAX_DANCE_MAT))
 		return rdesc;
 
 	/*
@@ -1317,6 +1327,11 @@ static int sony_raw_event(struct hid_device *hdev, struct hid_report *report,
 		dualshock4_parse_report(sc, rd, size);
 	}
 
+	if (sc->defer_initialization) {
+		sc->defer_initialization = 0;
+		sony_schedule_work(sc);
+	}
+
 	return 0;
 }
 
@@ -1554,7 +1569,7 @@ static void buzz_set_leds(struct sony_sc *sc)
 static void sony_set_leds(struct sony_sc *sc)
 {
 	if (!(sc->quirks & BUZZ_CONTROLLER))
-		schedule_work(&sc->state_worker);
+		sony_schedule_work(sc);
 	else
 		buzz_set_leds(sc);
 }
@@ -1665,7 +1680,7 @@ static int sony_led_blink_set(struct led_classdev *led, unsigned long *delay_on,
 		new_off != drv_data->led_delay_off[n]) {
 		drv_data->led_delay_on[n] = new_on;
 		drv_data->led_delay_off[n] = new_off;
-		schedule_work(&drv_data->state_worker);
+		sony_schedule_work(drv_data);
 	}
 
 	return 0;
@@ -1865,6 +1880,17 @@ static void dualshock4_send_output_report(struct sony_sc *sc)
 	u8 *buf = sc->output_report_dmabuf;
 	int offset;
 
+	/*
+	 * NOTE: The buf[1] field of the Bluetooth report controls
+	 * the Dualshock 4 reporting rate.
+	 *
+	 * Known values include:
+	 *
+	 * 0x80 - 1000hz (full speed)
+	 * 0xA0 - 31hz
+	 * 0xB0 - 20hz
+	 * 0xD0 - 66hz
+	 */
 	if (sc->quirks & DUALSHOCK4_CONTROLLER_USB) {
 		memset(buf, 0, DS4_REPORT_0x05_SIZE);
 		buf[0] = 0x05;
@@ -1976,7 +2002,7 @@ static int sony_play_effect(struct input_dev *dev, void *data,
 	sc->left = effect->u.rumble.strong_magnitude / 256;
 	sc->right = effect->u.rumble.weak_magnitude / 256;
 
-	schedule_work(&sc->state_worker);
+	sony_schedule_work(sc);
 	return 0;
 }
 
@@ -2039,8 +2065,11 @@ static int sony_battery_get_property(struct power_supply *psy,
 	return ret;
 }
 
-static int sony_battery_probe(struct sony_sc *sc)
+static int sony_battery_probe(struct sony_sc *sc, int append_dev_id)
 {
+	const char *battery_str_fmt = append_dev_id ?
+		"sony_controller_battery_%pMR_%i" :
+		"sony_controller_battery_%pMR";
 	struct power_supply_config psy_cfg = { .drv_data = sc, };
 	struct hid_device *hdev = sc->hdev;
 	int ret;
@@ -2056,9 +2085,8 @@ static int sony_battery_probe(struct sony_sc *sc)
 	sc->battery_desc.get_property = sony_battery_get_property;
 	sc->battery_desc.type = POWER_SUPPLY_TYPE_BATTERY;
 	sc->battery_desc.use_for_apm = 0;
-	sc->battery_desc.name = kasprintf(GFP_KERNEL,
-					  "sony_controller_battery_%pMR",
-					  sc->mac_address);
+	sc->battery_desc.name = kasprintf(GFP_KERNEL, battery_str_fmt,
+					  sc->mac_address, sc->device_id);
 	if (!sc->battery_desc.name)
 		return -ENOMEM;
 
@@ -2094,7 +2122,21 @@ static void sony_battery_remove(struct sony_sc *sc)
  * it will show up as two devices. A global list of connected controllers and
  * their MAC addresses is maintained to ensure that a device is only connected
  * once.
+ *
+ * Some USB-only devices masquerade as Sixaxis controllers and all have the
+ * same dummy Bluetooth address, so a comparison of the connection type is
+ * required.  Devices are only rejected in the case where two devices have
+ * matching Bluetooth addresses on different bus types.
  */
+static inline int sony_compare_connection_type(struct sony_sc *sc0,
+						struct sony_sc *sc1)
+{
+	const int sc0_not_bt = !(sc0->quirks & SONY_BT_DEVICE);
+	const int sc1_not_bt = !(sc1->quirks & SONY_BT_DEVICE);
+
+	return sc0_not_bt == sc1_not_bt;
+}
+
 static int sony_check_add_dev_list(struct sony_sc *sc)
 {
 	struct sony_sc *entry;
@@ -2107,9 +2149,14 @@ static int sony_check_add_dev_list(struct sony_sc *sc)
 		ret = memcmp(sc->mac_address, entry->mac_address,
 				sizeof(sc->mac_address));
 		if (!ret) {
-			ret = -EEXIST;
-			hid_info(sc->hdev, "controller with MAC address %pMR already connected\n",
+			if (sony_compare_connection_type(sc, entry)) {
+				ret = 1;
+			} else {
+				ret = -EEXIST;
+				hid_info(sc->hdev,
+				"controller with MAC address %pMR already connected\n",
 				sc->mac_address);
+			}
 			goto unlock;
 		}
 	}
@@ -2285,10 +2332,14 @@ static inline void sony_cancel_work_sync(struct sony_sc *sc)
 static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
 {
 	int ret;
+	int append_dev_id;
 	unsigned long quirks = id->driver_data;
 	struct sony_sc *sc;
 	unsigned int connect_mask = HID_CONNECT_DEFAULT;
 
+	if (!strcmp(hdev->name, "FutureMax Dance Mat"))
+		quirks |= FUTUREMAX_DANCE_MAT;
+
 	sc = devm_kzalloc(&hdev->dev, sizeof(*sc), GFP_KERNEL);
 	if (sc == NULL) {
 		hid_err(hdev, "can't alloc sony descriptor\n");
@@ -2341,9 +2392,16 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		 * the Sixaxis does not want the report_id as part of the data
 		 * packet, so we have to discard buf[0] when sending the actual
 		 * control message, even for numbered reports, humpf!
+		 *
+		 * Additionally, the Sixaxis on USB isn't properly initialized
+		 * until the PS logo button is pressed and as such won't retain
+		 * any state set by an output report, so the initial
+		 * configuration report is deferred until the first input
+		 * report arrives.
 		 */
 		hdev->quirks |= HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP;
 		hdev->quirks |= HID_QUIRK_SKIP_OUTPUT_REPORT_ID;
+		sc->defer_initialization = 1;
 		ret = sixaxis_set_operational_usb(hdev);
 		sony_init_output_report(sc, sixaxis_send_output_report);
 	} else if ((sc->quirks & SIXAXIS_CONTROLLER_BT) ||
@@ -2379,7 +2437,7 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	if (ret < 0)
 		goto err_stop;
 
-	ret = sony_check_add(sc);
+	ret = append_dev_id = sony_check_add(sc);
 	if (ret < 0)
 		goto err_stop;
 
@@ -2390,7 +2448,7 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	}
 
 	if (sc->quirks & SONY_BATTERY_SUPPORT) {
-		ret = sony_battery_probe(sc);
+		ret = sony_battery_probe(sc, append_dev_id);
 		if (ret < 0)
 			goto err_stop;
 
@@ -2486,8 +2544,10 @@ static int sony_resume(struct hid_device *hdev)
 		 * reinitialized on resume or they won't behave properly.
 		 */
 		if ((sc->quirks & SIXAXIS_CONTROLLER_USB) ||
-			(sc->quirks & NAVIGATION_CONTROLLER_USB))
+			(sc->quirks & NAVIGATION_CONTROLLER_USB)) {
 			sixaxis_set_operational_usb(sc->hdev);
+			sc->defer_initialization = 1;
+		}
 
 		sony_set_leds(sc);
 	}
diff --git a/drivers/hid/hid-uclogic.c b/drivers/hid/hid-uclogic.c
index 85ac43517e3f..1509d7287ff3 100644
--- a/drivers/hid/hid-uclogic.c
+++ b/drivers/hid/hid-uclogic.c
@@ -21,13 +21,6 @@
 
 #include "hid-ids.h"
 
-/*
- * See WPXXXXU model descriptions, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=UC-Logic_Tablet_WP4030U
- * http://sf.net/apps/mediawiki/digimend/?title=UC-Logic_Tablet_WP5540U
- * http://sf.net/apps/mediawiki/digimend/?title=UC-Logic_Tablet_WP8060U
- */
-
 /* Size of the original descriptor of WPXXXXU tablets */
 #define WPXXXXU_RDESC_ORIG_SIZE	212
 
@@ -221,11 +214,6 @@ static __u8 wp8060u_rdesc_fixed[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See WP1062 description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=UC-Logic_Tablet_WP1062
- */
-
 /* Size of the original descriptor of WP1062 tablet */
 #define WP1062_RDESC_ORIG_SIZE	254
 
@@ -274,11 +262,6 @@ static __u8 wp1062_rdesc_fixed[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See PF1209 description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=UC-Logic_Tablet_PF1209
- */
-
 /* Size of the original descriptor of PF1209 tablet */
 #define PF1209_RDESC_ORIG_SIZE	234
 
@@ -356,11 +339,6 @@ static __u8 pf1209_rdesc_fixed[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See TWHL850 description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=UC-Logic_Wireless_Tablet_TWHL850
- */
-
 /* Size of the original descriptors of TWHL850 tablet */
 #define TWHL850_RDESC_ORIG_SIZE0	182
 #define TWHL850_RDESC_ORIG_SIZE1	161
@@ -469,11 +447,6 @@ static __u8 twhl850_rdesc_fixed2[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See TWHA60 description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=UC-Logic_Tablet_TWHA60
- */
-
 /* Size of the original descriptors of TWHA60 tablet */
 #define TWHA60_RDESC_ORIG_SIZE0 254
 #define TWHA60_RDESC_ORIG_SIZE1 139
@@ -613,6 +586,27 @@ static const __u8 uclogic_tablet_rdesc_template[] = {
 	0xC0                    /*  End Collection                          */
 };
 
+/* Fixed virtual pad report descriptor */
+static const __u8 uclogic_buttonpad_rdesc[] = {
+	0x05, 0x01,             /*  Usage Page (Desktop),                   */
+	0x09, 0x07,             /*  Usage (Keypad),                         */
+	0xA1, 0x01,             /*  Collection (Application),               */
+	0x85, 0xF7,             /*      Report ID (247),                    */
+	0x05, 0x0D,             /*      Usage Page (Digitizers),            */
+	0x09, 0x39,             /*      Usage (Tablet Function Keys),       */
+	0xA0,                   /*      Collection (Physical),              */
+	0x05, 0x09,             /*          Usage Page (Button),            */
+	0x75, 0x01,             /*          Report Size (1),                */
+	0x95, 0x18,             /*          Report Count (24),              */
+	0x81, 0x03,             /*          Input (Constant, Variable),     */
+	0x19, 0x01,             /*          Usage Minimum (01h),            */
+	0x29, 0x08,             /*          Usage Maximum (08h),            */
+	0x95, 0x08,             /*          Report Count (8),               */
+	0x81, 0x02,             /*          Input (Variable),               */
+	0xC0,                   /*      End Collection                      */
+	0xC0                    /*  End Collection                          */
+};
+
 /* Parameter indices */
 enum uclogic_prm {
 	UCLOGIC_PRM_X_LM	= 1,
@@ -628,6 +622,7 @@ struct uclogic_drvdata {
 	unsigned int rsize;
 	bool invert_pen_inrange;
 	bool ignore_pen_usage;
+	bool has_virtual_pad_interface;
 };
 
 static __u8 *uclogic_report_fixup(struct hid_device *hdev, __u8 *rdesc,
@@ -637,6 +632,12 @@ static __u8 *uclogic_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 	__u8 iface_num = iface->cur_altsetting->desc.bInterfaceNumber;
 	struct uclogic_drvdata *drvdata = hid_get_drvdata(hdev);
 
+	if (drvdata->rdesc != NULL) {
+		rdesc = drvdata->rdesc;
+		*rsize = drvdata->rsize;
+		return rdesc;
+	}
+
 	switch (hdev->product) {
 	case USB_DEVICE_ID_UCLOGIC_TABLET_PF1209:
 		if (*rsize == PF1209_RDESC_ORIG_SIZE) {
@@ -706,11 +707,6 @@ static __u8 *uclogic_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 			break;
 		}
 		break;
-	default:
-		if (drvdata->rdesc != NULL) {
-			rdesc = drvdata->rdesc;
-			*rsize = drvdata->rsize;
-		}
 	}
 
 	return rdesc;
@@ -804,7 +800,6 @@ static int uclogic_tablet_enable(struct hid_device *hdev)
 	len = UCLOGIC_PRM_NUM * sizeof(*buf);
 	buf = kmalloc(len, GFP_KERNEL);
 	if (buf == NULL) {
-		hid_err(hdev, "failed to allocate parameter buffer\n");
 		rc = -ENOMEM;
 		goto cleanup;
 	}
@@ -848,7 +843,6 @@ static int uclogic_tablet_enable(struct hid_device *hdev)
 				sizeof(uclogic_tablet_rdesc_template),
 				GFP_KERNEL);
 	if (drvdata->rdesc == NULL) {
-		hid_err(hdev, "failed to allocate fixed rdesc\n");
 		rc = -ENOMEM;
 		goto cleanup;
 	}
@@ -876,11 +870,75 @@ cleanup:
 	return rc;
 }
 
+/**
+ * Enable actual button mode.
+ *
+ * @hdev:	HID device
+ */
+static int uclogic_button_enable(struct hid_device *hdev)
+{
+	int rc;
+	struct usb_device *usb_dev = hid_to_usb_dev(hdev);
+	struct uclogic_drvdata *drvdata = hid_get_drvdata(hdev);
+	char *str_buf;
+	size_t str_len = 16;
+	unsigned char *rdesc;
+	size_t rdesc_len;
+
+	str_buf = kzalloc(str_len, GFP_KERNEL);
+	if (str_buf == NULL) {
+		rc = -ENOMEM;
+		goto cleanup;
+	}
+
+	/* Enable abstract keyboard mode */
+	rc = usb_string(usb_dev, 0x7b, str_buf, str_len);
+	if (rc == -EPIPE) {
+		hid_info(hdev, "button mode setting not found\n");
+		rc = 0;
+		goto cleanup;
+	} else if (rc < 0) {
+		hid_err(hdev, "failed to enable abstract keyboard\n");
+		goto cleanup;
+	} else if (strncmp(str_buf, "HK On", rc)) {
+		hid_info(hdev, "invalid answer when requesting buttons: '%s'\n",
+			str_buf);
+		rc = -EINVAL;
+		goto cleanup;
+	}
+
+	/* Re-allocate fixed report descriptor */
+	rdesc_len = drvdata->rsize + sizeof(uclogic_buttonpad_rdesc);
+	rdesc = devm_kzalloc(&hdev->dev, rdesc_len, GFP_KERNEL);
+	if (!rdesc) {
+		rc = -ENOMEM;
+		goto cleanup;
+	}
+
+	memcpy(rdesc, drvdata->rdesc, drvdata->rsize);
+
+	/* Append the buttonpad descriptor */
+	memcpy(rdesc + drvdata->rsize, uclogic_buttonpad_rdesc,
+	       sizeof(uclogic_buttonpad_rdesc));
+
+	/* clean up old rdesc and use the new one */
+	drvdata->rsize = rdesc_len;
+	devm_kfree(&hdev->dev, drvdata->rdesc);
+	drvdata->rdesc = rdesc;
+
+	rc = 0;
+
+cleanup:
+	kfree(str_buf);
+	return rc;
+}
+
 static int uclogic_probe(struct hid_device *hdev,
 		const struct hid_device_id *id)
 {
 	int rc;
 	struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
+	struct usb_device *udev = hid_to_usb_dev(hdev);
 	struct uclogic_drvdata *drvdata;
 
 	/*
@@ -899,6 +957,10 @@ static int uclogic_probe(struct hid_device *hdev,
 
 	switch (id->product) {
 	case USB_DEVICE_ID_HUION_TABLET:
+	case USB_DEVICE_ID_YIYNOVA_TABLET:
+	case USB_DEVICE_ID_UGEE_TABLET_81:
+	case USB_DEVICE_ID_UCLOGIC_DRAWIMAGE_G3:
+	case USB_DEVICE_ID_UGEE_TABLET_45:
 		/* If this is the pen interface */
 		if (intf->cur_altsetting->desc.bInterfaceNumber == 0) {
 			rc = uclogic_tablet_enable(hdev);
@@ -907,10 +969,48 @@ static int uclogic_probe(struct hid_device *hdev,
 				return rc;
 			}
 			drvdata->invert_pen_inrange = true;
+
+			rc = uclogic_button_enable(hdev);
+			drvdata->has_virtual_pad_interface = !rc;
 		} else {
 			drvdata->ignore_pen_usage = true;
 		}
 		break;
+	case USB_DEVICE_ID_UGTIZER_TABLET_GP0610:
+		/* If this is the pen interface */
+		if (intf->cur_altsetting->desc.bInterfaceNumber == 1) {
+			rc = uclogic_tablet_enable(hdev);
+			if (rc) {
+				hid_err(hdev, "tablet enabling failed\n");
+				return rc;
+			}
+			drvdata->invert_pen_inrange = true;
+		} else {
+			drvdata->ignore_pen_usage = true;
+		}
+		break;
+	case USB_DEVICE_ID_UCLOGIC_TABLET_TWHA60:
+		/*
+		 * If it is the three-interface version, which is known to
+		 * respond to initialization.
+		 */
+		if (udev->config->desc.bNumInterfaces == 3) {
+			/* If it is the pen interface */
+			if (intf->cur_altsetting->desc.bInterfaceNumber == 0) {
+				rc = uclogic_tablet_enable(hdev);
+				if (rc) {
+					hid_err(hdev, "tablet enabling failed\n");
+					return rc;
+				}
+				drvdata->invert_pen_inrange = true;
+
+				rc = uclogic_button_enable(hdev);
+				drvdata->has_virtual_pad_interface = !rc;
+			} else {
+				drvdata->ignore_pen_usage = true;
+			}
+		}
+		break;
 	}
 
 	rc = hid_parse(hdev);
@@ -933,12 +1033,16 @@ static int uclogic_raw_event(struct hid_device *hdev, struct hid_report *report,
 {
 	struct uclogic_drvdata *drvdata = hid_get_drvdata(hdev);
 
-	if ((drvdata->invert_pen_inrange) &&
-	    (report->type == HID_INPUT_REPORT) &&
+	if ((report->type == HID_INPUT_REPORT) &&
 	    (report->id == UCLOGIC_PEN_REPORT_ID) &&
-	    (size >= 2))
-		/* Invert the in-range bit */
-		data[1] ^= 0x40;
+	    (size >= 2)) {
+		if (drvdata->has_virtual_pad_interface && (data[1] & 0x20))
+			/* Change to virtual frame button report ID */
+			data[0] = 0xf7;
+		else if (drvdata->invert_pen_inrange)
+			/* Invert the in-range bit */
+			data[1] ^= 0x40;
+	}
 
 	return 0;
 }
@@ -960,6 +1064,11 @@ static const struct hid_device_id uclogic_devices[] = {
 				USB_DEVICE_ID_UCLOGIC_TABLET_TWHA60) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_HUION, USB_DEVICE_ID_HUION_TABLET) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_HUION_TABLET) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_YIYNOVA_TABLET) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_81) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_45) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_DRAWIMAGE_G3) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UGTIZER, USB_DEVICE_ID_UGTIZER_TABLET_GP0610) },
 	{ }
 };
 MODULE_DEVICE_TABLE(hid, uclogic_devices);
diff --git a/drivers/hid/hid-waltop.c b/drivers/hid/hid-waltop.c
index 059931d7b392..a91aabe4a70a 100644
--- a/drivers/hid/hid-waltop.c
+++ b/drivers/hid/hid-waltop.c
@@ -42,11 +42,6 @@
  * 02 16 02     ink
  */
 
-/*
- * See Slim Tablet 5.8 inch description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=Waltop_Slim_Tablet_5.8%22
- */
-
 /* Size of the original report descriptor of Slim Tablet 5.8 inch */
 #define SLIM_TABLET_5_8_INCH_RDESC_ORIG_SIZE	222
 
@@ -98,11 +93,6 @@ static __u8 slim_tablet_5_8_inch_rdesc_fixed[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See Slim Tablet 12.1 inch description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=Waltop_Slim_Tablet_12.1%22
- */
-
 /* Size of the original report descriptor of Slim Tablet 12.1 inch */
 #define SLIM_TABLET_12_1_INCH_RDESC_ORIG_SIZE	269
 
@@ -154,11 +144,6 @@ static __u8 slim_tablet_12_1_inch_rdesc_fixed[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See Q Pad description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=Waltop_Q_Pad
- */
-
 /* Size of the original report descriptor of Q Pad */
 #define Q_PAD_RDESC_ORIG_SIZE	241
 
@@ -210,11 +195,6 @@ static __u8 q_pad_rdesc_fixed[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See description, device and HID report descriptors of tablet with PID 0038 at
- * http://sf.net/apps/mediawiki/digimend/?title=Waltop_PID_0038
- */
-
 /* Size of the original report descriptor of tablet with PID 0038 */
 #define PID_0038_RDESC_ORIG_SIZE	241
 
@@ -268,11 +248,6 @@ static __u8 pid_0038_rdesc_fixed[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See Media Tablet 10.6 inch description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=Waltop_Media_Tablet_10.6%22
- */
-
 /* Size of the original report descriptor of Media Tablet 10.6 inch */
 #define MEDIA_TABLET_10_6_INCH_RDESC_ORIG_SIZE	300
 
@@ -386,11 +361,6 @@ static __u8 media_tablet_10_6_inch_rdesc_fixed[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See Media Tablet 14.1 inch description, device and HID report descriptors at
- * http://sf.net/apps/mediawiki/digimend/?title=Waltop_Media_Tablet_14.1%22
- */
-
 /* Size of the original report descriptor of Media Tablet 14.1 inch */
 #define MEDIA_TABLET_14_1_INCH_RDESC_ORIG_SIZE	309
 
@@ -502,12 +472,6 @@ static __u8 media_tablet_14_1_inch_rdesc_fixed[] = {
 	0xC0                /*  End Collection                      */
 };
 
-/*
- * See Sirius Battery Free Tablet description, device and HID report descriptors
- * at
- * http://sf.net/apps/mediawiki/digimend/?title=Waltop_Sirius_Battery_Free_Tablet
- */
-
 /* Size of the original report descriptor of Sirius Battery Free Tablet */
 #define SIRIUS_BATTERY_FREE_TABLET_RDESC_ORIG_SIZE	335
 
diff --git a/drivers/hid/intel-ish-hid/Kconfig b/drivers/hid/intel-ish-hid/Kconfig
new file mode 100644
index 000000000000..ea065b3684a2
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/Kconfig
@@ -0,0 +1,17 @@
+menu "Intel ISH HID support"
+	depends on X86_64 && PCI
+
+config INTEL_ISH_HID
+	tristate "Intel Integrated Sensor Hub"
+	default n
+	select HID
+	help
+	  The Integrated Sensor Hub (ISH) enables the ability to offload
+	  sensor polling and algorithm processing to a dedicated low power
+	  processor in the chipset. This allows the core processor to go into
+	  low power modes more often, resulting in the increased battery life.
+	  The current processors that support ISH are: Cherrytrail, Skylake,
+	  Broxton and Kaby Lake.
+
+	  Say Y here if you want to support Intel ISH. If unsure, say N.
+endmenu
diff --git a/drivers/hid/intel-ish-hid/Makefile b/drivers/hid/intel-ish-hid/Makefile
new file mode 100644
index 000000000000..8c08b0b358b1
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/Makefile
@@ -0,0 +1,22 @@
+#
+# Makefile - Intel ISH HID drivers
+# Copyright (c) 2014-2016, Intel Corporation.
+#
+#
+obj-$(CONFIG_INTEL_ISH_HID) += intel-ishtp.o
+intel-ishtp-objs := ishtp/init.o
+intel-ishtp-objs += ishtp/hbm.o
+intel-ishtp-objs += ishtp/client.o
+intel-ishtp-objs += ishtp/bus.o
+intel-ishtp-objs += ishtp/dma-if.o
+intel-ishtp-objs += ishtp/client-buffers.o
+
+obj-$(CONFIG_INTEL_ISH_HID) += intel-ish-ipc.o
+intel-ish-ipc-objs := ipc/ipc.o
+intel-ish-ipc-objs += ipc/pci-ish.o
+
+obj-$(CONFIG_INTEL_ISH_HID) += intel-ishtp-hid.o
+intel-ishtp-hid-objs := ishtp-hid.o
+intel-ishtp-hid-objs += ishtp-hid-client.o
+
+ccflags-y += -Idrivers/hid/intel-ish-hid/ishtp
diff --git a/drivers/hid/intel-ish-hid/ipc/hw-ish-regs.h b/drivers/hid/intel-ish-hid/ipc/hw-ish-regs.h
new file mode 100644
index 000000000000..ab68afcba2a2
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ipc/hw-ish-regs.h
@@ -0,0 +1,220 @@
+/*
+ * ISH registers definitions
+ *
+ * Copyright (c) 2012-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _ISHTP_ISH_REGS_H_
+#define _ISHTP_ISH_REGS_H_
+
+
+/*** IPC PCI Offsets and sizes ***/
+/* ISH IPC Base Address */
+#define IPC_REG_BASE		0x0000
+/* Peripheral Interrupt Status Register */
+#define IPC_REG_PISR_CHV_AB      (IPC_REG_BASE + 0x00)
+/* Peripheral Interrupt Mask Register */
+#define IPC_REG_PIMR_CHV_AB      (IPC_REG_BASE + 0x04)
+/*BXT, CHV_K0*/
+/*Peripheral Interrupt Status Register */
+#define IPC_REG_PISR_BXT	 (IPC_REG_BASE + 0x0C)
+/*Peripheral Interrupt Mask Register */
+#define IPC_REG_PIMR_BXT	 (IPC_REG_BASE + 0x08)
+/***********************************/
+/* ISH Host Firmware status Register */
+#define IPC_REG_ISH_HOST_FWSTS	(IPC_REG_BASE + 0x34)
+/* Host Communication Register */
+#define IPC_REG_HOST_COMM	(IPC_REG_BASE + 0x38)
+/* Reset register */
+#define IPC_REG_ISH_RST		(IPC_REG_BASE + 0x44)
+
+/* Inbound doorbell register Host to ISH */
+#define IPC_REG_HOST2ISH_DRBL	(IPC_REG_BASE + 0x48)
+/* Outbound doorbell register ISH to Host */
+#define IPC_REG_ISH2HOST_DRBL	(IPC_REG_BASE + 0x54)
+/* ISH to HOST message registers */
+#define IPC_REG_ISH2HOST_MSG	(IPC_REG_BASE + 0x60)
+/* HOST to ISH message registers */
+#define IPC_REG_HOST2ISH_MSG	(IPC_REG_BASE + 0xE0)
+/* REMAP2 to enable DMA (D3 RCR) */
+#define	IPC_REG_ISH_RMP2	(IPC_REG_BASE + 0x368)
+
+#define	IPC_REG_MAX		(IPC_REG_BASE + 0x400)
+
+/*** register bits - HISR ***/
+/* bit corresponds HOST2ISH interrupt in PISR and PIMR registers */
+#define IPC_INT_HOST2ISH_BIT            (1<<0)
+/***********************************/
+/*CHV_A0, CHV_B0*/
+/* bit corresponds ISH2HOST interrupt in PISR and PIMR registers */
+#define IPC_INT_ISH2HOST_BIT_CHV_AB	(1<<3)
+/*BXT, CHV_K0*/
+/* bit corresponds ISH2HOST interrupt in PISR and PIMR registers */
+#define IPC_INT_ISH2HOST_BIT_BXT	(1<<0)
+/***********************************/
+
+/* bit corresponds ISH2HOST busy clear interrupt in PIMR register */
+#define IPC_INT_ISH2HOST_CLR_MASK_BIT	(1<<11)
+
+/* offset of ISH2HOST busy clear interrupt in IPC_BUSY_CLR register */
+#define IPC_INT_ISH2HOST_CLR_OFFS	(0)
+
+/* bit corresponds ISH2HOST busy clear interrupt in IPC_BUSY_CLR register */
+#define IPC_INT_ISH2HOST_CLR_BIT	(1<<IPC_INT_ISH2HOST_CLR_OFFS)
+
+/* bit corresponds busy bit in doorbell registers */
+#define IPC_DRBL_BUSY_OFFS		(31)
+#define IPC_DRBL_BUSY_BIT		(1<<IPC_DRBL_BUSY_OFFS)
+
+#define	IPC_HOST_OWNS_MSG_OFFS		(30)
+
+/*
+ * A0: bit means that host owns MSGnn registers and is reading them.
+ * ISH FW may not write to them
+ */
+#define	IPC_HOST_OWNS_MSG_BIT		(1<<IPC_HOST_OWNS_MSG_OFFS)
+
+/*
+ * Host status bits (HOSTCOMM)
+ */
+/* bit corresponds host ready bit in Host Status Register (HOST_COMM) */
+#define IPC_HOSTCOMM_READY_OFFS		(7)
+#define IPC_HOSTCOMM_READY_BIT		(1<<IPC_HOSTCOMM_READY_OFFS)
+
+/***********************************/
+/*CHV_A0, CHV_B0*/
+#define	IPC_HOSTCOMM_INT_EN_OFFS_CHV_AB	(31)
+#define	IPC_HOSTCOMM_INT_EN_BIT_CHV_AB		\
+	(1<<IPC_HOSTCOMM_INT_EN_OFFS_CHV_AB)
+/*BXT, CHV_K0*/
+#define IPC_PIMR_INT_EN_OFFS_BXT	(0)
+#define IPC_PIMR_INT_EN_BIT_BXT		(1<<IPC_PIMR_INT_EN_OFFS_BXT)
+
+#define IPC_HOST2ISH_BUSYCLEAR_MASK_OFFS_BXT	(8)
+#define IPC_HOST2ISH_BUSYCLEAR_MASK_BIT		\
+	(1<<IPC_HOST2ISH_BUSYCLEAR_MASK_OFFS_BXT)
+/***********************************/
+/*
+ * both Host and ISH have ILUP at bit 0
+ * bit corresponds host ready bit in both status registers
+ */
+#define IPC_ILUP_OFFS			(0)
+#define IPC_ILUP_BIT			(1<<IPC_ILUP_OFFS)
+
+/*
+ * FW status bits (relevant)
+ */
+#define	IPC_FWSTS_ILUP		0x1
+#define	IPC_FWSTS_ISHTP_UP	(1<<1)
+#define	IPC_FWSTS_DMA0		(1<<16)
+#define	IPC_FWSTS_DMA1		(1<<17)
+#define	IPC_FWSTS_DMA2		(1<<18)
+#define	IPC_FWSTS_DMA3		(1<<19)
+
+#define	IPC_ISH_IN_DMA		\
+	(IPC_FWSTS_DMA0 | IPC_FWSTS_DMA1 | IPC_FWSTS_DMA2 | IPC_FWSTS_DMA3)
+
+/* bit corresponds host ready bit in ISH FW Status Register */
+#define IPC_ISH_ISHTP_READY_OFFS		(1)
+#define IPC_ISH_ISHTP_READY_BIT		(1<<IPC_ISH_ISHTP_READY_OFFS)
+
+#define	IPC_RMP2_DMA_ENABLED	0x1	/* Value to enable DMA, per D3 RCR */
+
+#define IPC_MSG_MAX_SIZE	0x80
+
+
+#define IPC_HEADER_LENGTH_MASK		0x03FF
+#define IPC_HEADER_PROTOCOL_MASK	0x0F
+#define IPC_HEADER_MNG_CMD_MASK		0x0F
+
+#define IPC_HEADER_LENGTH_OFFSET	0
+#define IPC_HEADER_PROTOCOL_OFFSET	10
+#define IPC_HEADER_MNG_CMD_OFFSET	16
+
+#define IPC_HEADER_GET_LENGTH(drbl_reg)		\
+	(((drbl_reg) >> IPC_HEADER_LENGTH_OFFSET)&IPC_HEADER_LENGTH_MASK)
+#define IPC_HEADER_GET_PROTOCOL(drbl_reg)	\
+	(((drbl_reg) >> IPC_HEADER_PROTOCOL_OFFSET)&IPC_HEADER_PROTOCOL_MASK)
+#define IPC_HEADER_GET_MNG_CMD(drbl_reg)	\
+	(((drbl_reg) >> IPC_HEADER_MNG_CMD_OFFSET)&IPC_HEADER_MNG_CMD_MASK)
+
+#define IPC_IS_BUSY(drbl_reg)			\
+	(((drbl_reg)&IPC_DRBL_BUSY_BIT) == ((uint32_t)IPC_DRBL_BUSY_BIT))
+
+/***********************************/
+/*CHV_A0, CHV_B0*/
+#define IPC_INT_FROM_ISH_TO_HOST_CHV_AB(drbl_reg) \
+	(((drbl_reg)&IPC_INT_ISH2HOST_BIT_CHV_AB) == \
+	((u32)IPC_INT_ISH2HOST_BIT_CHV_AB))
+/*BXT, CHV_K0*/
+#define IPC_INT_FROM_ISH_TO_HOST_BXT(drbl_reg) \
+	(((drbl_reg)&IPC_INT_ISH2HOST_BIT_BXT) == \
+	((u32)IPC_INT_ISH2HOST_BIT_BXT))
+/***********************************/
+
+#define IPC_BUILD_HEADER(length, protocol, busy)		\
+	(((busy)<<IPC_DRBL_BUSY_OFFS) |				\
+	((protocol) << IPC_HEADER_PROTOCOL_OFFSET) |		\
+	((length)<<IPC_HEADER_LENGTH_OFFSET))
+
+#define IPC_BUILD_MNG_MSG(cmd, length)				\
+	(((1)<<IPC_DRBL_BUSY_OFFS)|				\
+	((IPC_PROTOCOL_MNG)<<IPC_HEADER_PROTOCOL_OFFSET)|	\
+	((cmd)<<IPC_HEADER_MNG_CMD_OFFSET)|			\
+	 ((length)<<IPC_HEADER_LENGTH_OFFSET))
+
+
+#define IPC_SET_HOST_READY(host_status)		\
+				((host_status) |= (IPC_HOSTCOMM_READY_BIT))
+
+#define IPC_SET_HOST_ILUP(host_status)		\
+				((host_status) |= (IPC_ILUP_BIT))
+
+#define IPC_CLEAR_HOST_READY(host_status)	\
+				((host_status) ^= (IPC_HOSTCOMM_READY_BIT))
+
+#define IPC_CLEAR_HOST_ILUP(host_status)	\
+				((host_status) ^= (IPC_ILUP_BIT))
+
+/* todo - temp until PIMR HW ready */
+#define IPC_HOST_BUSY_READING_OFFS	6
+
+/* bit corresponds host ready bit in Host Status Register (HOST_COMM) */
+#define IPC_HOST_BUSY_READING_BIT	(1<<IPC_HOST_BUSY_READING_OFFS)
+
+#define IPC_SET_HOST_BUSY_READING(host_status)	\
+				((host_status) |= (IPC_HOST_BUSY_READING_BIT))
+
+#define IPC_CLEAR_HOST_BUSY_READING(host_status)\
+				((host_status) ^= (IPC_HOST_BUSY_READING_BIT))
+
+
+#define IPC_IS_ISH_ISHTP_READY(ish_status)	\
+		(((ish_status) & IPC_ISH_ISHTP_READY_BIT) ==	\
+			((uint32_t)IPC_ISH_ISHTP_READY_BIT))
+
+#define IPC_IS_ISH_ILUP(ish_status)		\
+		(((ish_status) & IPC_ILUP_BIT) == ((uint32_t)IPC_ILUP_BIT))
+
+
+#define IPC_PROTOCOL_ISHTP		1
+#define IPC_PROTOCOL_MNG		3
+
+#define MNG_RX_CMPL_ENABLE		0
+#define MNG_RX_CMPL_DISABLE		1
+#define MNG_RX_CMPL_INDICATION		2
+#define MNG_RESET_NOTIFY		3
+#define MNG_RESET_NOTIFY_ACK		4
+#define MNG_SYNC_FW_CLOCK		5
+#define MNG_ILLEGAL_CMD			0xFF
+
+#endif /* _ISHTP_ISH_REGS_H_ */
diff --git a/drivers/hid/intel-ish-hid/ipc/hw-ish.h b/drivers/hid/intel-ish-hid/ipc/hw-ish.h
new file mode 100644
index 000000000000..46615a03e78f
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ipc/hw-ish.h
@@ -0,0 +1,71 @@
+/*
+ * H/W layer of ISHTP provider device (ISH)
+ *
+ * Copyright (c) 2014-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _ISHTP_HW_ISH_H_
+#define _ISHTP_HW_ISH_H_
+
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include "hw-ish-regs.h"
+#include "ishtp-dev.h"
+
+#define CHV_DEVICE_ID		0x22D8
+#define BXT_Ax_DEVICE_ID	0x0AA2
+#define BXT_Bx_DEVICE_ID	0x1AA2
+#define APL_Ax_DEVICE_ID	0x5AA2
+#define SPT_Ax_DEVICE_ID	0x9D35
+
+#define	REVISION_ID_CHT_A0	0x6
+#define	REVISION_ID_CHT_Ax_SI	0x0
+#define	REVISION_ID_CHT_Bx_SI	0x10
+#define	REVISION_ID_CHT_Kx_SI	0x20
+#define	REVISION_ID_CHT_Dx_SI	0x30
+#define	REVISION_ID_CHT_B0	0xB0
+#define	REVISION_ID_SI_MASK	0x70
+
+struct ipc_rst_payload_type {
+	uint16_t	reset_id;
+	uint16_t	reserved;
+};
+
+struct time_sync_format {
+	uint8_t ts1_source;
+	uint8_t ts2_source;
+	uint16_t reserved;
+} __packed;
+
+struct ipc_time_update_msg {
+	uint64_t primary_host_time;
+	struct time_sync_format sync_info;
+	uint64_t secondary_host_time;
+} __packed;
+
+enum {
+	HOST_UTC_TIME_USEC = 0,
+	HOST_SYSTEM_TIME_USEC = 1
+};
+
+struct ish_hw {
+	void __iomem *mem_addr;
+};
+
+#define to_ish_hw(dev) (struct ish_hw *)((dev)->hw)
+
+irqreturn_t ish_irq_handler(int irq, void *dev_id);
+struct ishtp_device *ish_dev_init(struct pci_dev *pdev);
+int ish_hw_start(struct ishtp_device *dev);
+void ish_device_disable(struct ishtp_device *dev);
+
+#endif /* _ISHTP_HW_ISH_H_ */
diff --git a/drivers/hid/intel-ish-hid/ipc/ipc.c b/drivers/hid/intel-ish-hid/ipc/ipc.c
new file mode 100644
index 000000000000..e2517c11e0ee
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ipc/ipc.c
@@ -0,0 +1,881 @@
+/*
+ * H/W layer of ISHTP provider device (ISH)
+ *
+ * Copyright (c) 2014-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/sched.h>
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#include <linux/jiffies.h>
+#include "client.h"
+#include "hw-ish.h"
+#include "utils.h"
+#include "hbm.h"
+
+/* For FW reset flow */
+static struct work_struct fw_reset_work;
+static struct ishtp_device *ishtp_dev;
+
+/**
+ * ish_reg_read() - Read register
+ * @dev: ISHTP device pointer
+ * @offset: Register offset
+ *
+ * Read 32 bit register at a given offset
+ *
+ * Return: Read register value
+ */
+static inline uint32_t ish_reg_read(const struct ishtp_device *dev,
+	unsigned long offset)
+{
+	struct ish_hw *hw = to_ish_hw(dev);
+
+	return readl(hw->mem_addr + offset);
+}
+
+/**
+ * ish_reg_write() - Write register
+ * @dev: ISHTP device pointer
+ * @offset: Register offset
+ * @value: Value to write
+ *
+ * Writes 32 bit register at a give offset
+ */
+static inline void ish_reg_write(struct ishtp_device *dev,
+				 unsigned long offset,
+				 uint32_t value)
+{
+	struct ish_hw *hw = to_ish_hw(dev);
+
+	writel(value, hw->mem_addr + offset);
+}
+
+/**
+ * _ish_read_fw_sts_reg() - Read FW status register
+ * @dev: ISHTP device pointer
+ *
+ * Read FW status register
+ *
+ * Return: Read register value
+ */
+static inline uint32_t _ish_read_fw_sts_reg(struct ishtp_device *dev)
+{
+	return ish_reg_read(dev, IPC_REG_ISH_HOST_FWSTS);
+}
+
+/**
+ * check_generated_interrupt() - Check if ISH interrupt
+ * @dev: ISHTP device pointer
+ *
+ * Check if an interrupt was generated for ISH
+ *
+ * Return: Read true or false
+ */
+static bool check_generated_interrupt(struct ishtp_device *dev)
+{
+	bool interrupt_generated = true;
+	uint32_t pisr_val = 0;
+
+	if (dev->pdev->device == CHV_DEVICE_ID) {
+		pisr_val = ish_reg_read(dev, IPC_REG_PISR_CHV_AB);
+		interrupt_generated =
+			IPC_INT_FROM_ISH_TO_HOST_CHV_AB(pisr_val);
+	} else {
+		pisr_val = ish_reg_read(dev, IPC_REG_PISR_BXT);
+		interrupt_generated = IPC_INT_FROM_ISH_TO_HOST_BXT(pisr_val);
+	}
+
+	return interrupt_generated;
+}
+
+/**
+ * ish_is_input_ready() - Check if FW ready for RX
+ * @dev: ISHTP device pointer
+ *
+ * Check if ISH FW is ready for receiving data
+ *
+ * Return: Read true or false
+ */
+static bool ish_is_input_ready(struct ishtp_device *dev)
+{
+	uint32_t doorbell_val;
+
+	doorbell_val = ish_reg_read(dev, IPC_REG_HOST2ISH_DRBL);
+	return !IPC_IS_BUSY(doorbell_val);
+}
+
+/**
+ * set_host_ready() - Indicate host ready
+ * @dev: ISHTP device pointer
+ *
+ * Set host ready indication to FW
+ */
+static void set_host_ready(struct ishtp_device *dev)
+{
+	if (dev->pdev->device == CHV_DEVICE_ID) {
+		if (dev->pdev->revision == REVISION_ID_CHT_A0 ||
+				(dev->pdev->revision & REVISION_ID_SI_MASK) ==
+				REVISION_ID_CHT_Ax_SI)
+			ish_reg_write(dev, IPC_REG_HOST_COMM, 0x81);
+		else if (dev->pdev->revision == REVISION_ID_CHT_B0 ||
+				(dev->pdev->revision & REVISION_ID_SI_MASK) ==
+				REVISION_ID_CHT_Bx_SI ||
+				(dev->pdev->revision & REVISION_ID_SI_MASK) ==
+				REVISION_ID_CHT_Kx_SI ||
+				(dev->pdev->revision & REVISION_ID_SI_MASK) ==
+				REVISION_ID_CHT_Dx_SI) {
+			uint32_t host_comm_val;
+
+			host_comm_val = ish_reg_read(dev, IPC_REG_HOST_COMM);
+			host_comm_val |= IPC_HOSTCOMM_INT_EN_BIT_CHV_AB | 0x81;
+			ish_reg_write(dev, IPC_REG_HOST_COMM, host_comm_val);
+		}
+	} else {
+			uint32_t host_pimr_val;
+
+			host_pimr_val = ish_reg_read(dev, IPC_REG_PIMR_BXT);
+			host_pimr_val |= IPC_PIMR_INT_EN_BIT_BXT;
+			/*
+			 * disable interrupt generated instead of
+			 * RX_complete_msg
+			 */
+			host_pimr_val &= ~IPC_HOST2ISH_BUSYCLEAR_MASK_BIT;
+
+			ish_reg_write(dev, IPC_REG_PIMR_BXT, host_pimr_val);
+	}
+}
+
+/**
+ * ishtp_fw_is_ready() - Check if FW ready
+ * @dev: ISHTP device pointer
+ *
+ * Check if ISH FW is ready
+ *
+ * Return: Read true or false
+ */
+static bool ishtp_fw_is_ready(struct ishtp_device *dev)
+{
+	uint32_t ish_status = _ish_read_fw_sts_reg(dev);
+
+	return IPC_IS_ISH_ILUP(ish_status) &&
+		IPC_IS_ISH_ISHTP_READY(ish_status);
+}
+
+/**
+ * ish_set_host_rdy() - Indicate host ready
+ * @dev: ISHTP device pointer
+ *
+ * Set host ready indication to FW
+ */
+static void ish_set_host_rdy(struct ishtp_device *dev)
+{
+	uint32_t host_status = ish_reg_read(dev, IPC_REG_HOST_COMM);
+
+	IPC_SET_HOST_READY(host_status);
+	ish_reg_write(dev, IPC_REG_HOST_COMM, host_status);
+}
+
+/**
+ * ish_clr_host_rdy() - Indicate host not ready
+ * @dev: ISHTP device pointer
+ *
+ * Send host not ready indication to FW
+ */
+static void ish_clr_host_rdy(struct ishtp_device *dev)
+{
+	uint32_t host_status = ish_reg_read(dev, IPC_REG_HOST_COMM);
+
+	IPC_CLEAR_HOST_READY(host_status);
+	ish_reg_write(dev, IPC_REG_HOST_COMM, host_status);
+}
+
+/**
+ * _ishtp_read_hdr() - Read message header
+ * @dev: ISHTP device pointer
+ *
+ * Read header of 32bit length
+ *
+ * Return: Read register value
+ */
+static uint32_t _ishtp_read_hdr(const struct ishtp_device *dev)
+{
+	return ish_reg_read(dev, IPC_REG_ISH2HOST_MSG);
+}
+
+/**
+ * _ishtp_read - Read message
+ * @dev: ISHTP device pointer
+ * @buffer: message buffer
+ * @buffer_length: length of message buffer
+ *
+ * Read message from FW
+ *
+ * Return: Always 0
+ */
+static int _ishtp_read(struct ishtp_device *dev, unsigned char *buffer,
+	unsigned long buffer_length)
+{
+	uint32_t	i;
+	uint32_t	*r_buf = (uint32_t *)buffer;
+	uint32_t	msg_offs;
+
+	msg_offs = IPC_REG_ISH2HOST_MSG + sizeof(struct ishtp_msg_hdr);
+	for (i = 0; i < buffer_length; i += sizeof(uint32_t))
+		*r_buf++ = ish_reg_read(dev, msg_offs + i);
+
+	return 0;
+}
+
+/**
+ * write_ipc_from_queue() - try to write ipc msg from Tx queue to device
+ * @dev: ishtp device pointer
+ *
+ * Check if DRBL is cleared. if it is - write the first IPC msg,  then call
+ * the callback function (unless it's NULL)
+ *
+ * Return: 0 for success else failure code
+ */
+static int write_ipc_from_queue(struct ishtp_device *dev)
+{
+	struct wr_msg_ctl_info	*ipc_link;
+	unsigned long	length;
+	unsigned long	rem;
+	unsigned long	flags;
+	uint32_t	doorbell_val;
+	uint32_t	*r_buf;
+	uint32_t	reg_addr;
+	int	i;
+	void	(*ipc_send_compl)(void *);
+	void	*ipc_send_compl_prm;
+	static int	out_ipc_locked;
+	unsigned long	out_ipc_flags;
+
+	if (dev->dev_state == ISHTP_DEV_DISABLED)
+		return	-EINVAL;
+
+	spin_lock_irqsave(&dev->out_ipc_spinlock, out_ipc_flags);
+	if (out_ipc_locked) {
+		spin_unlock_irqrestore(&dev->out_ipc_spinlock, out_ipc_flags);
+		return -EBUSY;
+	}
+	out_ipc_locked = 1;
+	if (!ish_is_input_ready(dev)) {
+		out_ipc_locked = 0;
+		spin_unlock_irqrestore(&dev->out_ipc_spinlock, out_ipc_flags);
+		return -EBUSY;
+	}
+	spin_unlock_irqrestore(&dev->out_ipc_spinlock, out_ipc_flags);
+
+	spin_lock_irqsave(&dev->wr_processing_spinlock, flags);
+	/*
+	 * if tx send list is empty - return 0;
+	 * may happen, as RX_COMPLETE handler doesn't check list emptiness.
+	 */
+	if (list_empty(&dev->wr_processing_list_head.link)) {
+		spin_unlock_irqrestore(&dev->wr_processing_spinlock, flags);
+		out_ipc_locked = 0;
+		return	0;
+	}
+
+	ipc_link = list_entry(dev->wr_processing_list_head.link.next,
+			      struct wr_msg_ctl_info, link);
+	/* first 4 bytes of the data is the doorbell value (IPC header) */
+	length = ipc_link->length - sizeof(uint32_t);
+	doorbell_val = *(uint32_t *)ipc_link->inline_data;
+	r_buf = (uint32_t *)(ipc_link->inline_data + sizeof(uint32_t));
+
+	/* If sending MNG_SYNC_FW_CLOCK, update clock again */
+	if (IPC_HEADER_GET_PROTOCOL(doorbell_val) == IPC_PROTOCOL_MNG &&
+		IPC_HEADER_GET_MNG_CMD(doorbell_val) == MNG_SYNC_FW_CLOCK) {
+		struct timespec ts_system;
+		struct timeval tv_utc;
+		uint64_t        usec_system, usec_utc;
+		struct ipc_time_update_msg time_update;
+		struct time_sync_format ts_format;
+
+		get_monotonic_boottime(&ts_system);
+		do_gettimeofday(&tv_utc);
+		usec_system = (timespec_to_ns(&ts_system)) / NSEC_PER_USEC;
+		usec_utc = (uint64_t)tv_utc.tv_sec * 1000000 +
+						((uint32_t)tv_utc.tv_usec);
+		ts_format.ts1_source = HOST_SYSTEM_TIME_USEC;
+		ts_format.ts2_source = HOST_UTC_TIME_USEC;
+
+		time_update.primary_host_time = usec_system;
+		time_update.secondary_host_time = usec_utc;
+		time_update.sync_info = ts_format;
+
+		memcpy(r_buf, &time_update,
+		       sizeof(struct ipc_time_update_msg));
+	}
+
+	for (i = 0, reg_addr = IPC_REG_HOST2ISH_MSG; i < length >> 2; i++,
+			reg_addr += 4)
+		ish_reg_write(dev, reg_addr, r_buf[i]);
+
+	rem = length & 0x3;
+	if (rem > 0) {
+		uint32_t reg = 0;
+
+		memcpy(&reg, &r_buf[length >> 2], rem);
+		ish_reg_write(dev, reg_addr, reg);
+	}
+	/* Flush writes to msg registers and doorbell */
+	ish_reg_read(dev, IPC_REG_ISH_HOST_FWSTS);
+
+	/* Update IPC counters */
+	++dev->ipc_tx_cnt;
+	dev->ipc_tx_bytes_cnt += IPC_HEADER_GET_LENGTH(doorbell_val);
+
+	ish_reg_write(dev, IPC_REG_HOST2ISH_DRBL, doorbell_val);
+	out_ipc_locked = 0;
+
+	ipc_send_compl = ipc_link->ipc_send_compl;
+	ipc_send_compl_prm = ipc_link->ipc_send_compl_prm;
+	list_del_init(&ipc_link->link);
+	list_add_tail(&ipc_link->link, &dev->wr_free_list_head.link);
+	spin_unlock_irqrestore(&dev->wr_processing_spinlock, flags);
+
+	/*
+	 * callback will be called out of spinlock,
+	 * after ipc_link returned to free list
+	 */
+	if (ipc_send_compl)
+		ipc_send_compl(ipc_send_compl_prm);
+
+	return 0;
+}
+
+/**
+ * write_ipc_to_queue() - write ipc msg to Tx queue
+ * @dev: ishtp device instance
+ * @ipc_send_compl: Send complete callback
+ * @ipc_send_compl_prm:	Parameter to send in complete callback
+ * @msg: Pointer to message
+ * @length: Length of message
+ *
+ * Recived msg with IPC (and upper protocol) header  and add it to the device
+ *  Tx-to-write list then try to send the first IPC waiting msg
+ *  (if DRBL is cleared)
+ * This function returns negative value for failure (means free list
+ *  is empty, or msg too long) and 0 for success.
+ *
+ * Return: 0 for success else failure code
+ */
+static int write_ipc_to_queue(struct ishtp_device *dev,
+	void (*ipc_send_compl)(void *), void *ipc_send_compl_prm,
+	unsigned char *msg, int length)
+{
+	struct wr_msg_ctl_info *ipc_link;
+	unsigned long	flags;
+
+	if (length > IPC_FULL_MSG_SIZE)
+		return -EMSGSIZE;
+
+	spin_lock_irqsave(&dev->wr_processing_spinlock, flags);
+	if (list_empty(&dev->wr_free_list_head.link)) {
+		spin_unlock_irqrestore(&dev->wr_processing_spinlock, flags);
+		return -ENOMEM;
+	}
+	ipc_link = list_entry(dev->wr_free_list_head.link.next,
+		struct wr_msg_ctl_info, link);
+	list_del_init(&ipc_link->link);
+
+	ipc_link->ipc_send_compl = ipc_send_compl;
+	ipc_link->ipc_send_compl_prm = ipc_send_compl_prm;
+	ipc_link->length = length;
+	memcpy(ipc_link->inline_data, msg, length);
+
+	list_add_tail(&ipc_link->link, &dev->wr_processing_list_head.link);
+	spin_unlock_irqrestore(&dev->wr_processing_spinlock, flags);
+
+	write_ipc_from_queue(dev);
+
+	return 0;
+}
+
+/**
+ * ipc_send_mng_msg() - Send management message
+ * @dev: ishtp device instance
+ * @msg_code: Message code
+ * @msg: Pointer to message
+ * @size: Length of message
+ *
+ * Send management message to FW
+ *
+ * Return: 0 for success else failure code
+ */
+static int ipc_send_mng_msg(struct ishtp_device *dev, uint32_t msg_code,
+	void *msg, size_t size)
+{
+	unsigned char	ipc_msg[IPC_FULL_MSG_SIZE];
+	uint32_t	drbl_val = IPC_BUILD_MNG_MSG(msg_code, size);
+
+	memcpy(ipc_msg, &drbl_val, sizeof(uint32_t));
+	memcpy(ipc_msg + sizeof(uint32_t), msg, size);
+	return	write_ipc_to_queue(dev, NULL, NULL, ipc_msg,
+		sizeof(uint32_t) + size);
+}
+
+/**
+ * ish_fw_reset_handler() - FW reset handler
+ * @dev: ishtp device pointer
+ *
+ * Handle FW reset
+ *
+ * Return: 0 for success else failure code
+ */
+static int ish_fw_reset_handler(struct ishtp_device *dev)
+{
+	uint32_t	reset_id;
+	unsigned long	flags;
+	struct wr_msg_ctl_info *processing, *next;
+
+	/* Read reset ID */
+	reset_id = ish_reg_read(dev, IPC_REG_ISH2HOST_MSG) & 0xFFFF;
+
+	/* Clear IPC output queue */
+	spin_lock_irqsave(&dev->wr_processing_spinlock, flags);
+	list_for_each_entry_safe(processing, next,
+			&dev->wr_processing_list_head.link, link) {
+		list_move_tail(&processing->link, &dev->wr_free_list_head.link);
+	}
+	spin_unlock_irqrestore(&dev->wr_processing_spinlock, flags);
+
+	/* ISHTP notification in IPC_RESET */
+	ishtp_reset_handler(dev);
+
+	if (!ish_is_input_ready(dev))
+		timed_wait_for_timeout(WAIT_FOR_SEND_SLICE,
+			ish_is_input_ready(dev), (2 * HZ));
+
+	/* ISH FW is dead */
+	if (!ish_is_input_ready(dev))
+		return	-EPIPE;
+	/*
+	 * Set HOST2ISH.ILUP. Apparently we need this BEFORE sending
+	 * RESET_NOTIFY_ACK - FW will be checking for it
+	 */
+	ish_set_host_rdy(dev);
+	/* Send RESET_NOTIFY_ACK (with reset_id) */
+	ipc_send_mng_msg(dev, MNG_RESET_NOTIFY_ACK, &reset_id,
+			 sizeof(uint32_t));
+
+	/* Wait for ISH FW'es ILUP and ISHTP_READY */
+	timed_wait_for_timeout(WAIT_FOR_SEND_SLICE, ishtp_fw_is_ready(dev),
+		(2 * HZ));
+	if (!ishtp_fw_is_ready(dev)) {
+		/* ISH FW is dead */
+		uint32_t	ish_status;
+
+		ish_status = _ish_read_fw_sts_reg(dev);
+		dev_err(dev->devc,
+			"[ishtp-ish]: completed reset, ISH is dead (FWSTS = %08X)\n",
+			ish_status);
+		return -ENODEV;
+	}
+	return	0;
+}
+
+/**
+ * ish_fw_reset_work_fn() - FW reset worker function
+ * @unused: not used
+ *
+ * Call ish_fw_reset_handler to complete FW reset
+ */
+static void fw_reset_work_fn(struct work_struct *unused)
+{
+	int	rv;
+
+	rv = ish_fw_reset_handler(ishtp_dev);
+	if (!rv) {
+		/* ISH is ILUP & ISHTP-ready. Restart ISHTP */
+		schedule_timeout(HZ / 3);
+		ishtp_dev->recvd_hw_ready = 1;
+		wake_up_interruptible(&ishtp_dev->wait_hw_ready);
+
+		/* ISHTP notification in IPC_RESET sequence completion */
+		ishtp_reset_compl_handler(ishtp_dev);
+	} else
+		dev_err(ishtp_dev->devc, "[ishtp-ish]: FW reset failed (%d)\n",
+			rv);
+}
+
+/**
+ * _ish_sync_fw_clock() -Sync FW clock with the OS clock
+ * @dev: ishtp device pointer
+ *
+ * Sync FW and OS time
+ */
+static void _ish_sync_fw_clock(struct ishtp_device *dev)
+{
+	static unsigned long	prev_sync;
+	struct timespec	ts;
+	uint64_t	usec;
+
+	if (prev_sync && jiffies - prev_sync < 20 * HZ)
+		return;
+
+	prev_sync = jiffies;
+	get_monotonic_boottime(&ts);
+	usec = (timespec_to_ns(&ts)) / NSEC_PER_USEC;
+	ipc_send_mng_msg(dev, MNG_SYNC_FW_CLOCK, &usec, sizeof(uint64_t));
+}
+
+/**
+ * recv_ipc() - Receive and process IPC management messages
+ * @dev: ishtp device instance
+ * @doorbell_val: doorbell value
+ *
+ * This function runs in ISR context.
+ * NOTE: Any other mng command than reset_notify and reset_notify_ack
+ * won't wake BH handler
+ */
+static void	recv_ipc(struct ishtp_device *dev, uint32_t doorbell_val)
+{
+	uint32_t	mng_cmd;
+
+	mng_cmd = IPC_HEADER_GET_MNG_CMD(doorbell_val);
+
+	switch (mng_cmd) {
+	default:
+		break;
+
+	case MNG_RX_CMPL_INDICATION:
+		if (dev->suspend_flag) {
+			dev->suspend_flag = 0;
+			wake_up_interruptible(&dev->suspend_wait);
+		}
+		if (dev->resume_flag) {
+			dev->resume_flag = 0;
+			wake_up_interruptible(&dev->resume_wait);
+		}
+
+		write_ipc_from_queue(dev);
+		break;
+
+	case MNG_RESET_NOTIFY:
+		if (!ishtp_dev) {
+			ishtp_dev = dev;
+			INIT_WORK(&fw_reset_work, fw_reset_work_fn);
+		}
+		schedule_work(&fw_reset_work);
+		break;
+
+	case MNG_RESET_NOTIFY_ACK:
+		dev->recvd_hw_ready = 1;
+		wake_up_interruptible(&dev->wait_hw_ready);
+		break;
+	}
+}
+
+/**
+ * ish_irq_handler() - ISH IRQ handler
+ * @irq: irq number
+ * @dev_id: ishtp device pointer
+ *
+ * ISH IRQ handler. If interrupt is generated and is for ISH it will process
+ * the interrupt.
+ */
+irqreturn_t ish_irq_handler(int irq, void *dev_id)
+{
+	struct ishtp_device	*dev = dev_id;
+	uint32_t	doorbell_val;
+	bool	interrupt_generated;
+
+	/* Check that it's interrupt from ISH (may be shared) */
+	interrupt_generated = check_generated_interrupt(dev);
+
+	if (!interrupt_generated)
+		return IRQ_NONE;
+
+	doorbell_val = ish_reg_read(dev, IPC_REG_ISH2HOST_DRBL);
+	if (!IPC_IS_BUSY(doorbell_val))
+		return IRQ_HANDLED;
+
+	if (dev->dev_state == ISHTP_DEV_DISABLED)
+		return	IRQ_HANDLED;
+
+	/* Sanity check: IPC dgram length in header */
+	if (IPC_HEADER_GET_LENGTH(doorbell_val) > IPC_PAYLOAD_SIZE) {
+		dev_err(dev->devc,
+			"IPC hdr - bad length: %u; dropped\n",
+			(unsigned int)IPC_HEADER_GET_LENGTH(doorbell_val));
+		goto	eoi;
+	}
+
+	switch (IPC_HEADER_GET_PROTOCOL(doorbell_val)) {
+	default:
+		break;
+	case IPC_PROTOCOL_MNG:
+		recv_ipc(dev, doorbell_val);
+		break;
+	case IPC_PROTOCOL_ISHTP:
+		ishtp_recv(dev);
+		break;
+	}
+
+eoi:
+	/* Update IPC counters */
+	++dev->ipc_rx_cnt;
+	dev->ipc_rx_bytes_cnt += IPC_HEADER_GET_LENGTH(doorbell_val);
+
+	ish_reg_write(dev, IPC_REG_ISH2HOST_DRBL, 0);
+	/* Flush write to doorbell */
+	ish_reg_read(dev, IPC_REG_ISH_HOST_FWSTS);
+
+	return	IRQ_HANDLED;
+}
+
+/**
+ * _ish_hw_reset() - HW reset
+ * @dev: ishtp device pointer
+ *
+ * Reset ISH HW to recover if any error
+ *
+ * Return: 0 for success else error fault code
+ */
+static int _ish_hw_reset(struct ishtp_device *dev)
+{
+	struct pci_dev *pdev = dev->pdev;
+	int	rv;
+	unsigned int	dma_delay;
+	uint16_t csr;
+
+	if (!pdev)
+		return	-ENODEV;
+
+	rv = pci_reset_function(pdev);
+	if (!rv)
+		dev->dev_state = ISHTP_DEV_RESETTING;
+
+	if (!pdev->pm_cap) {
+		dev_err(&pdev->dev, "Can't reset - no PM caps\n");
+		return	-EINVAL;
+	}
+
+	/* Now trigger reset to FW */
+	ish_reg_write(dev, IPC_REG_ISH_RMP2, 0);
+
+	for (dma_delay = 0; dma_delay < MAX_DMA_DELAY &&
+		_ish_read_fw_sts_reg(dev) & (IPC_ISH_IN_DMA);
+		dma_delay += 5)
+		mdelay(5);
+
+	if (dma_delay >= MAX_DMA_DELAY) {
+		dev_err(&pdev->dev,
+			"Can't reset - stuck with DMA in-progress\n");
+		return	-EBUSY;
+	}
+
+	pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &csr);
+
+	csr &= ~PCI_PM_CTRL_STATE_MASK;
+	csr |= PCI_D3hot;
+	pci_write_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, csr);
+
+	mdelay(pdev->d3_delay);
+
+	csr &= ~PCI_PM_CTRL_STATE_MASK;
+	csr |= PCI_D0;
+	pci_write_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, csr);
+
+	ish_reg_write(dev, IPC_REG_ISH_RMP2, IPC_RMP2_DMA_ENABLED);
+
+	/*
+	 * Send 0 IPC message so that ISH FW wakes up if it was already
+	 * asleep
+	 */
+	ish_reg_write(dev, IPC_REG_HOST2ISH_DRBL, IPC_DRBL_BUSY_BIT);
+
+	/* Flush writes to doorbell and REMAP2 */
+	ish_reg_read(dev, IPC_REG_ISH_HOST_FWSTS);
+
+	return	0;
+}
+
+/**
+ * _ish_ipc_reset() - IPC reset
+ * @dev: ishtp device pointer
+ *
+ * Resets host and fw IPC and upper layers
+ *
+ * Return: 0 for success else error fault code
+ */
+static int _ish_ipc_reset(struct ishtp_device *dev)
+{
+	struct ipc_rst_payload_type ipc_mng_msg;
+	int	rv = 0;
+
+	ipc_mng_msg.reset_id = 1;
+	ipc_mng_msg.reserved = 0;
+
+	set_host_ready(dev);
+
+	/* Clear the incoming doorbell */
+	ish_reg_write(dev, IPC_REG_ISH2HOST_DRBL, 0);
+	/* Flush write to doorbell */
+	ish_reg_read(dev, IPC_REG_ISH_HOST_FWSTS);
+
+	dev->recvd_hw_ready = 0;
+
+	/* send message */
+	rv = ipc_send_mng_msg(dev, MNG_RESET_NOTIFY, &ipc_mng_msg,
+		sizeof(struct ipc_rst_payload_type));
+	if (rv) {
+		dev_err(dev->devc, "Failed to send IPC MNG_RESET_NOTIFY\n");
+		return	rv;
+	}
+
+	wait_event_interruptible_timeout(dev->wait_hw_ready,
+					 dev->recvd_hw_ready, 2 * HZ);
+	if (!dev->recvd_hw_ready) {
+		dev_err(dev->devc, "Timed out waiting for HW ready\n");
+		rv = -ENODEV;
+	}
+
+	return rv;
+}
+
+/**
+ * ish_hw_start() -Start ISH HW
+ * @dev: ishtp device pointer
+ *
+ * Set host to ready state and wait for FW reset
+ *
+ * Return: 0 for success else error fault code
+ */
+int ish_hw_start(struct ishtp_device *dev)
+{
+	ish_set_host_rdy(dev);
+	/* After that we can enable ISH DMA operation */
+	ish_reg_write(dev, IPC_REG_ISH_RMP2, IPC_RMP2_DMA_ENABLED);
+
+	/*
+	 * Send 0 IPC message so that ISH FW wakes up if it was already
+	 * asleep
+	 */
+	ish_reg_write(dev, IPC_REG_HOST2ISH_DRBL, IPC_DRBL_BUSY_BIT);
+	/* Flush write to doorbell */
+	ish_reg_read(dev, IPC_REG_ISH_HOST_FWSTS);
+
+	set_host_ready(dev);
+
+	/* wait for FW-initiated reset flow */
+	if (!dev->recvd_hw_ready)
+		wait_event_interruptible_timeout(dev->wait_hw_ready,
+						 dev->recvd_hw_ready,
+						 10 * HZ);
+
+	if (!dev->recvd_hw_ready) {
+		dev_err(dev->devc,
+			"[ishtp-ish]: Timed out waiting for FW-initiated reset\n");
+		return	-ENODEV;
+	}
+
+	return 0;
+}
+
+/**
+ * ish_ipc_get_header() -Get doorbell value
+ * @dev: ishtp device pointer
+ * @length: length of message
+ * @busy: busy status
+ *
+ * Get door bell value from message header
+ *
+ * Return: door bell value
+ */
+static uint32_t ish_ipc_get_header(struct ishtp_device *dev, int length,
+				   int busy)
+{
+	uint32_t drbl_val;
+
+	drbl_val = IPC_BUILD_HEADER(length, IPC_PROTOCOL_ISHTP, busy);
+
+	return drbl_val;
+}
+
+static const struct ishtp_hw_ops ish_hw_ops = {
+	.hw_reset = _ish_hw_reset,
+	.ipc_reset = _ish_ipc_reset,
+	.ipc_get_header = ish_ipc_get_header,
+	.ishtp_read = _ishtp_read,
+	.write = write_ipc_to_queue,
+	.get_fw_status = _ish_read_fw_sts_reg,
+	.sync_fw_clock = _ish_sync_fw_clock,
+	.ishtp_read_hdr = _ishtp_read_hdr
+};
+
+/**
+ * ish_dev_init() -Initialize ISH devoce
+ * @pdev: PCI device
+ *
+ * Allocate ISHTP device and initialize IPC processing
+ *
+ * Return: ISHTP device instance on success else NULL
+ */
+struct ishtp_device *ish_dev_init(struct pci_dev *pdev)
+{
+	struct ishtp_device *dev;
+	int	i;
+
+	dev = kzalloc(sizeof(struct ishtp_device) + sizeof(struct ish_hw),
+		GFP_KERNEL);
+	if (!dev)
+		return NULL;
+
+	ishtp_device_init(dev);
+
+	init_waitqueue_head(&dev->wait_hw_ready);
+
+	spin_lock_init(&dev->wr_processing_spinlock);
+	spin_lock_init(&dev->out_ipc_spinlock);
+
+	/* Init IPC processing and free lists */
+	INIT_LIST_HEAD(&dev->wr_processing_list_head.link);
+	INIT_LIST_HEAD(&dev->wr_free_list_head.link);
+	for (i = 0; i < IPC_TX_FIFO_SIZE; ++i) {
+		struct wr_msg_ctl_info	*tx_buf;
+
+		tx_buf = kzalloc(sizeof(struct wr_msg_ctl_info), GFP_KERNEL);
+		if (!tx_buf) {
+			/*
+			 * IPC buffers may be limited or not available
+			 * at all - although this shouldn't happen
+			 */
+			dev_err(dev->devc,
+				"[ishtp-ish]: failure in Tx FIFO allocations (%d)\n",
+				i);
+			break;
+		}
+		list_add_tail(&tx_buf->link, &dev->wr_free_list_head.link);
+	}
+
+	dev->ops = &ish_hw_ops;
+	dev->devc = &pdev->dev;
+	dev->mtu = IPC_PAYLOAD_SIZE - sizeof(struct ishtp_msg_hdr);
+	return dev;
+}
+
+/**
+ * ish_device_disable() - Disable ISH device
+ * @dev: ISHTP device pointer
+ *
+ * Disable ISH by clearing host ready to inform firmware.
+ */
+void	ish_device_disable(struct ishtp_device *dev)
+{
+	dev->dev_state = ISHTP_DEV_DISABLED;
+	ish_clr_host_rdy(dev);
+}
diff --git a/drivers/hid/intel-ish-hid/ipc/pci-ish.c b/drivers/hid/intel-ish-hid/ipc/pci-ish.c
new file mode 100644
index 000000000000..42f0beeb09fd
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ipc/pci-ish.c
@@ -0,0 +1,322 @@
+/*
+ * PCI glue for ISHTP provider device (ISH) driver
+ *
+ * Copyright (c) 2014-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/fs.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/sched.h>
+#include <linux/interrupt.h>
+#include <linux/workqueue.h>
+#include <linux/miscdevice.h>
+#define CREATE_TRACE_POINTS
+#include <trace/events/intel_ish.h>
+#include "ishtp-dev.h"
+#include "hw-ish.h"
+
+static const struct pci_device_id ish_pci_tbl[] = {
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, CHV_DEVICE_ID)},
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, BXT_Ax_DEVICE_ID)},
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, BXT_Bx_DEVICE_ID)},
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, APL_Ax_DEVICE_ID)},
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, SPT_Ax_DEVICE_ID)},
+	{0, }
+};
+MODULE_DEVICE_TABLE(pci, ish_pci_tbl);
+
+/**
+ * ish_event_tracer() - Callback function to dump trace messages
+ * @dev:	ishtp device
+ * @format:	printf style format
+ *
+ * Callback to direct log messages to Linux trace buffers
+ */
+static void ish_event_tracer(struct ishtp_device *dev, char *format, ...)
+{
+	if (trace_ishtp_dump_enabled()) {
+		va_list args;
+		char tmp_buf[100];
+
+		va_start(args, format);
+		vsnprintf(tmp_buf, sizeof(tmp_buf), format, args);
+		va_end(args);
+
+		trace_ishtp_dump(tmp_buf);
+	}
+}
+
+/**
+ * ish_init() - Init function
+ * @dev:	ishtp device
+ *
+ * This function initialize wait queues for suspend/resume and call
+ * calls hadware initialization function. This will initiate
+ * startup sequence
+ *
+ * Return: 0 for success or error code for failure
+ */
+static int ish_init(struct ishtp_device *dev)
+{
+	int ret;
+
+	/* Set the state of ISH HW to start */
+	ret = ish_hw_start(dev);
+	if (ret) {
+		dev_err(dev->devc, "ISH: hw start failed.\n");
+		return ret;
+	}
+
+	/* Start the inter process communication to ISH processor */
+	ret = ishtp_start(dev);
+	if (ret) {
+		dev_err(dev->devc, "ISHTP: Protocol init failed.\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * ish_probe() - PCI driver probe callback
+ * @pdev:	pci device
+ * @ent:	pci device id
+ *
+ * Initialize PCI function, setup interrupt and call for ISH initialization
+ *
+ * Return: 0 for success or error code for failure
+ */
+static int ish_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	struct ishtp_device *dev;
+	struct ish_hw *hw;
+	int	ret;
+
+	/* enable pci dev */
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		dev_err(&pdev->dev, "ISH: Failed to enable PCI device\n");
+		return ret;
+	}
+
+	/* set PCI host mastering */
+	pci_set_master(pdev);
+
+	/* pci request regions for ISH driver */
+	ret = pci_request_regions(pdev, KBUILD_MODNAME);
+	if (ret) {
+		dev_err(&pdev->dev, "ISH: Failed to get PCI regions\n");
+		goto disable_device;
+	}
+
+	/* allocates and initializes the ISH dev structure */
+	dev = ish_dev_init(pdev);
+	if (!dev) {
+		ret = -ENOMEM;
+		goto release_regions;
+	}
+	hw = to_ish_hw(dev);
+	dev->print_log = ish_event_tracer;
+
+	/* mapping IO device memory */
+	hw->mem_addr = pci_iomap(pdev, 0, 0);
+	if (!hw->mem_addr) {
+		dev_err(&pdev->dev, "ISH: mapping I/O range failure\n");
+		ret = -ENOMEM;
+		goto free_device;
+	}
+
+	dev->pdev = pdev;
+
+	pdev->dev_flags |= PCI_DEV_FLAGS_NO_D3;
+
+	/* request and enable interrupt */
+	ret = request_irq(pdev->irq, ish_irq_handler, IRQF_NO_SUSPEND,
+			  KBUILD_MODNAME, dev);
+	if (ret) {
+		dev_err(&pdev->dev, "ISH: request IRQ failure (%d)\n",
+			pdev->irq);
+		goto free_device;
+	}
+
+	dev_set_drvdata(dev->devc, dev);
+
+	init_waitqueue_head(&dev->suspend_wait);
+	init_waitqueue_head(&dev->resume_wait);
+
+	ret = ish_init(dev);
+	if (ret)
+		goto free_irq;
+
+	return 0;
+
+free_irq:
+	free_irq(pdev->irq, dev);
+free_device:
+	pci_iounmap(pdev, hw->mem_addr);
+	kfree(dev);
+release_regions:
+	pci_release_regions(pdev);
+disable_device:
+	pci_clear_master(pdev);
+	pci_disable_device(pdev);
+	dev_err(&pdev->dev, "ISH: PCI driver initialization failed.\n");
+
+	return ret;
+}
+
+/**
+ * ish_remove() - PCI driver remove callback
+ * @pdev:	pci device
+ *
+ * This function does cleanup of ISH on pci remove callback
+ */
+static void ish_remove(struct pci_dev *pdev)
+{
+	struct ishtp_device *ishtp_dev = pci_get_drvdata(pdev);
+	struct ish_hw *hw = to_ish_hw(ishtp_dev);
+
+	ishtp_bus_remove_all_clients(ishtp_dev, false);
+	ish_device_disable(ishtp_dev);
+
+	free_irq(pdev->irq, ishtp_dev);
+	pci_iounmap(pdev, hw->mem_addr);
+	pci_release_regions(pdev);
+	pci_clear_master(pdev);
+	pci_disable_device(pdev);
+	kfree(ishtp_dev);
+}
+
+static struct device *ish_resume_device;
+
+/**
+ * ish_resume_handler() - Work function to complete resume
+ * @work:	work struct
+ *
+ * The resume work function to complete resume function asynchronously.
+ * There are two types of platforms, one where ISH is not powered off,
+ * in that case a simple resume message is enough, others we need
+ * a reset sequence.
+ */
+static void ish_resume_handler(struct work_struct *work)
+{
+	struct pci_dev *pdev = to_pci_dev(ish_resume_device);
+	struct ishtp_device *dev = pci_get_drvdata(pdev);
+	int ret;
+
+	ishtp_send_resume(dev);
+
+	/* 50 ms to get resume response */
+	if (dev->resume_flag)
+		ret = wait_event_interruptible_timeout(dev->resume_wait,
+						       !dev->resume_flag,
+						       msecs_to_jiffies(50));
+
+	/*
+	 * If no resume response. This platform  is not S0ix compatible
+	 * So on resume full reboot of ISH processor will happen, so
+	 * need to go through init sequence again
+	 */
+	if (dev->resume_flag)
+		ish_init(dev);
+}
+
+/**
+ * ish_suspend() - ISH suspend callback
+ * @device:	device pointer
+ *
+ * ISH suspend callback
+ *
+ * Return: 0 to the pm core
+ */
+static int ish_suspend(struct device *device)
+{
+	struct pci_dev *pdev = to_pci_dev(device);
+	struct ishtp_device *dev = pci_get_drvdata(pdev);
+
+	enable_irq_wake(pdev->irq);
+	/*
+	 * If previous suspend hasn't been asnwered then ISH is likely dead,
+	 * don't attempt nested notification
+	 */
+	if (dev->suspend_flag)
+		return	0;
+
+	dev->resume_flag = 0;
+	dev->suspend_flag = 1;
+	ishtp_send_suspend(dev);
+
+	/* 25 ms should be enough for live ISH to flush all IPC buf */
+	if (dev->suspend_flag)
+		wait_event_interruptible_timeout(dev->suspend_wait,
+						 !dev->suspend_flag,
+						  msecs_to_jiffies(25));
+
+	return 0;
+}
+
+static DECLARE_WORK(resume_work, ish_resume_handler);
+/**
+ * ish_resume() - ISH resume callback
+ * @device:	device pointer
+ *
+ * ISH resume callback
+ *
+ * Return: 0 to the pm core
+ */
+static int ish_resume(struct device *device)
+{
+	struct pci_dev *pdev = to_pci_dev(device);
+	struct ishtp_device *dev = pci_get_drvdata(pdev);
+
+	ish_resume_device = device;
+	dev->resume_flag = 1;
+
+	disable_irq_wake(pdev->irq);
+	schedule_work(&resume_work);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static const struct dev_pm_ops ish_pm_ops = {
+	.suspend = ish_suspend,
+	.resume = ish_resume,
+};
+#define ISHTP_ISH_PM_OPS	(&ish_pm_ops)
+#else
+#define ISHTP_ISH_PM_OPS	NULL
+#endif
+
+static struct pci_driver ish_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = ish_pci_tbl,
+	.probe = ish_probe,
+	.remove = ish_remove,
+	.driver.pm = ISHTP_ISH_PM_OPS,
+};
+
+module_pci_driver(ish_driver);
+
+/* Original author */
+MODULE_AUTHOR("Daniel Drubin <daniel.drubin@intel.com>");
+/* Adoption to upstream Linux kernel */
+MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
+
+MODULE_DESCRIPTION("Intel(R) Integrated Sensor Hub PCI Device Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hid/intel-ish-hid/ipc/utils.h b/drivers/hid/intel-ish-hid/ipc/utils.h
new file mode 100644
index 000000000000..5a82123dc7b4
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ipc/utils.h
@@ -0,0 +1,64 @@
+/*
+ * Utility macros of ISH
+ *
+ * Copyright (c) 2014-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+#ifndef UTILS__H
+#define UTILS__H
+
+#define	WAIT_FOR_SEND_SLICE	(HZ / 10)
+#define	WAIT_FOR_CONNECT_SLICE	(HZ / 10)
+
+/*
+ * Waits for specified event when a thread that triggers event can't signal
+ * Also, waits *at_least* `timeinc` after condition is satisfied
+ */
+#define	timed_wait_for(timeinc, condition)			\
+	do {							\
+		int completed = 0;				\
+		do {						\
+			unsigned long	j;			\
+			int	done = 0;			\
+								\
+			completed = (condition);		\
+			for (j = jiffies, done = 0; !done; ) {	\
+				schedule_timeout(timeinc);	\
+				if (time_is_before_eq_jiffies(j + timeinc)) \
+					done = 1;		\
+			}					\
+		} while (!(completed));				\
+	} while (0)
+
+
+/*
+ * Waits for specified event when a thread that triggers event
+ * can't signal with timeout (use whenever we may hang)
+ */
+#define	timed_wait_for_timeout(timeinc, condition, timeout)	\
+	do {							\
+		int	t = timeout;				\
+		do {						\
+			unsigned long	j;			\
+			int	done = 0;			\
+								\
+			for (j = jiffies, done = 0; !done; ) {	\
+				schedule_timeout(timeinc);	\
+				if (time_is_before_eq_jiffies(j + timeinc)) \
+					done = 1;		\
+			} \
+			t -= timeinc;				\
+			if (t <= 0)				\
+				break;				\
+		} while (!(condition));				\
+	} while (0)
+
+#endif /* UTILS__H */
diff --git a/drivers/hid/intel-ish-hid/ishtp-hid-client.c b/drivers/hid/intel-ish-hid/ishtp-hid-client.c
new file mode 100644
index 000000000000..5c643d7a07b2
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp-hid-client.c
@@ -0,0 +1,978 @@
+/*
+ * ISHTP client driver for HID (ISH)
+ *
+ * Copyright (c) 2014-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/module.h>
+#include <linux/hid.h>
+#include <linux/sched.h>
+#include "ishtp/ishtp-dev.h"
+#include "ishtp/client.h"
+#include "ishtp-hid.h"
+
+/* Rx ring buffer pool size */
+#define HID_CL_RX_RING_SIZE	32
+#define HID_CL_TX_RING_SIZE	16
+
+/**
+ * report_bad_packets() - Report bad packets
+ * @hid_ishtp_cl:	Client instance to get stats
+ * @recv_buf:		Raw received host interface message
+ * @cur_pos:		Current position index in payload
+ * @payload_len:	Length of payload expected
+ *
+ * Dumps error in case bad packet is received
+ */
+static void report_bad_packet(struct ishtp_cl *hid_ishtp_cl, void *recv_buf,
+			      size_t cur_pos,  size_t payload_len)
+{
+	struct hostif_msg *recv_msg = recv_buf;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+
+	dev_err(&client_data->cl_device->dev, "[hid-ish]: BAD packet %02X\n"
+		"total_bad=%u cur_pos=%u\n"
+		"[%02X %02X %02X %02X]\n"
+		"payload_len=%u\n"
+		"multi_packet_cnt=%u\n"
+		"is_response=%02X\n",
+		recv_msg->hdr.command, client_data->bad_recv_cnt,
+		(unsigned int)cur_pos,
+		((unsigned char *)recv_msg)[0], ((unsigned char *)recv_msg)[1],
+		((unsigned char *)recv_msg)[2], ((unsigned char *)recv_msg)[3],
+		(unsigned int)payload_len, client_data->multi_packet_cnt,
+		recv_msg->hdr.command & ~CMD_MASK);
+}
+
+/**
+ * process_recv() - Received and parse incoming packet
+ * @hid_ishtp_cl:	Client instance to get stats
+ * @recv_buf:		Raw received host interface message
+ * @data_len:		length of the message
+ *
+ * Parse the incoming packet. If it is a response packet then it will update
+ * per instance flags and wake up the caller waiting to for the response.
+ */
+static void process_recv(struct ishtp_cl *hid_ishtp_cl, void *recv_buf,
+			 size_t data_len)
+{
+	struct hostif_msg *recv_msg;
+	unsigned char *payload;
+	struct device_info *dev_info;
+	int i, j;
+	size_t	payload_len, total_len, cur_pos;
+	int report_type;
+	struct report_list *reports_list;
+	char *reports;
+	size_t report_len;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+	int curr_hid_dev = client_data->cur_hid_dev;
+
+	if (data_len < sizeof(struct hostif_msg_hdr)) {
+		dev_err(&client_data->cl_device->dev,
+			"[hid-ish]: error, received %u which is less than data header %u\n",
+			(unsigned int)data_len,
+			(unsigned int)sizeof(struct hostif_msg_hdr));
+		++client_data->bad_recv_cnt;
+		ish_hw_reset(hid_ishtp_cl->dev);
+		return;
+	}
+
+	payload = recv_buf + sizeof(struct hostif_msg_hdr);
+	total_len = data_len;
+	cur_pos = 0;
+
+	do {
+		recv_msg = (struct hostif_msg *)(recv_buf + cur_pos);
+		payload_len = recv_msg->hdr.size;
+
+		/* Sanity checks */
+		if (cur_pos + payload_len + sizeof(struct hostif_msg) >
+				total_len) {
+			++client_data->bad_recv_cnt;
+			report_bad_packet(hid_ishtp_cl, recv_msg, cur_pos,
+					  payload_len);
+			ish_hw_reset(hid_ishtp_cl->dev);
+			break;
+		}
+
+		hid_ishtp_trace(client_data,  "%s %d\n",
+				__func__, recv_msg->hdr.command & CMD_MASK);
+
+		switch (recv_msg->hdr.command & CMD_MASK) {
+		case HOSTIF_DM_ENUM_DEVICES:
+			if ((!(recv_msg->hdr.command & ~CMD_MASK) ||
+					client_data->init_done)) {
+				++client_data->bad_recv_cnt;
+				report_bad_packet(hid_ishtp_cl, recv_msg,
+						  cur_pos,
+						  payload_len);
+				ish_hw_reset(hid_ishtp_cl->dev);
+				break;
+			}
+			client_data->hid_dev_count = (unsigned int)*payload;
+			if (!client_data->hid_devices)
+				client_data->hid_devices = devm_kzalloc(
+						&client_data->cl_device->dev,
+						client_data->hid_dev_count *
+						sizeof(struct device_info),
+						GFP_KERNEL);
+			if (!client_data->hid_devices) {
+				dev_err(&client_data->cl_device->dev,
+				"Mem alloc failed for hid device info\n");
+				wake_up_interruptible(&client_data->init_wait);
+				break;
+			}
+			for (i = 0; i < client_data->hid_dev_count; ++i) {
+				if (1 + sizeof(struct device_info) * i >=
+						payload_len) {
+					dev_err(&client_data->cl_device->dev,
+						"[hid-ish]: [ENUM_DEVICES]: content size %lu is bigger than payload_len %u\n",
+						1 + sizeof(struct device_info)
+						* i,
+						(unsigned int)payload_len);
+				}
+
+				if (1 + sizeof(struct device_info) * i >=
+						data_len)
+					break;
+
+				dev_info = (struct device_info *)(payload + 1 +
+					sizeof(struct device_info) * i);
+				if (client_data->hid_devices)
+					memcpy(client_data->hid_devices + i,
+					       dev_info,
+					       sizeof(struct device_info));
+			}
+
+			client_data->enum_devices_done = true;
+			wake_up_interruptible(&client_data->init_wait);
+
+			break;
+
+		case HOSTIF_GET_HID_DESCRIPTOR:
+			if ((!(recv_msg->hdr.command & ~CMD_MASK) ||
+					client_data->init_done)) {
+				++client_data->bad_recv_cnt;
+				report_bad_packet(hid_ishtp_cl, recv_msg,
+						  cur_pos,
+						  payload_len);
+				ish_hw_reset(hid_ishtp_cl->dev);
+				break;
+			}
+			if (!client_data->hid_descr[curr_hid_dev])
+				client_data->hid_descr[curr_hid_dev] =
+				devm_kmalloc(&client_data->cl_device->dev,
+					     payload_len, GFP_KERNEL);
+			if (client_data->hid_descr[curr_hid_dev]) {
+				memcpy(client_data->hid_descr[curr_hid_dev],
+				       payload, payload_len);
+				client_data->hid_descr_size[curr_hid_dev] =
+					payload_len;
+				client_data->hid_descr_done = true;
+			}
+			wake_up_interruptible(&client_data->init_wait);
+
+			break;
+
+		case HOSTIF_GET_REPORT_DESCRIPTOR:
+			if ((!(recv_msg->hdr.command & ~CMD_MASK) ||
+					client_data->init_done)) {
+				++client_data->bad_recv_cnt;
+				report_bad_packet(hid_ishtp_cl, recv_msg,
+						  cur_pos,
+						  payload_len);
+				ish_hw_reset(hid_ishtp_cl->dev);
+				break;
+			}
+			if (!client_data->report_descr[curr_hid_dev])
+				client_data->report_descr[curr_hid_dev] =
+				devm_kmalloc(&client_data->cl_device->dev,
+					     payload_len, GFP_KERNEL);
+			if (client_data->report_descr[curr_hid_dev])  {
+				memcpy(client_data->report_descr[curr_hid_dev],
+				       payload,
+				       payload_len);
+				client_data->report_descr_size[curr_hid_dev] =
+					payload_len;
+				client_data->report_descr_done = true;
+			}
+			wake_up_interruptible(&client_data->init_wait);
+
+			break;
+
+		case HOSTIF_GET_FEATURE_REPORT:
+			report_type = HID_FEATURE_REPORT;
+			goto	do_get_report;
+
+		case HOSTIF_GET_INPUT_REPORT:
+			report_type = HID_INPUT_REPORT;
+do_get_report:
+			/* Get index of device that matches this id */
+			for (i = 0; i < client_data->num_hid_devices; ++i) {
+				if (recv_msg->hdr.device_id ==
+					client_data->hid_devices[i].dev_id)
+					if (client_data->hid_sensor_hubs[i]) {
+						hid_input_report(
+						client_data->hid_sensor_hubs[
+									i],
+						report_type, payload,
+						payload_len, 0);
+						ishtp_hid_wakeup(
+						client_data->hid_sensor_hubs[
+							i]);
+						break;
+					}
+			}
+			break;
+
+		case HOSTIF_SET_FEATURE_REPORT:
+			/* Get index of device that matches this id */
+			for (i = 0; i < client_data->num_hid_devices; ++i) {
+				if (recv_msg->hdr.device_id ==
+					client_data->hid_devices[i].dev_id)
+					if (client_data->hid_sensor_hubs[i]) {
+						ishtp_hid_wakeup(
+						client_data->hid_sensor_hubs[
+							i]);
+						break;
+					}
+			}
+			break;
+
+		case HOSTIF_PUBLISH_INPUT_REPORT:
+			report_type = HID_INPUT_REPORT;
+			for (i = 0; i < client_data->num_hid_devices; ++i)
+				if (recv_msg->hdr.device_id ==
+					client_data->hid_devices[i].dev_id)
+					if (client_data->hid_sensor_hubs[i])
+						hid_input_report(
+						client_data->hid_sensor_hubs[
+									i],
+						report_type, payload,
+						payload_len, 0);
+			break;
+
+		case HOSTIF_PUBLISH_INPUT_REPORT_LIST:
+			report_type = HID_INPUT_REPORT;
+			reports_list = (struct report_list *)payload;
+			reports = (char *)reports_list->reports;
+
+			for (j = 0; j < reports_list->num_of_reports; j++) {
+				recv_msg = (struct hostif_msg *)(reports +
+					sizeof(uint16_t));
+				report_len = *(uint16_t *)reports;
+				payload = reports + sizeof(uint16_t) +
+					sizeof(struct hostif_msg_hdr);
+				payload_len = report_len -
+					sizeof(struct hostif_msg_hdr);
+
+				for (i = 0; i < client_data->num_hid_devices;
+				     ++i)
+					if (recv_msg->hdr.device_id ==
+					client_data->hid_devices[i].dev_id &&
+					client_data->hid_sensor_hubs[i]) {
+						hid_input_report(
+						client_data->hid_sensor_hubs[
+									i],
+						report_type,
+						payload, payload_len,
+						0);
+					}
+
+				reports += sizeof(uint16_t) + report_len;
+			}
+			break;
+		default:
+			++client_data->bad_recv_cnt;
+			report_bad_packet(hid_ishtp_cl, recv_msg, cur_pos,
+					  payload_len);
+			ish_hw_reset(hid_ishtp_cl->dev);
+			break;
+
+		}
+
+		if (!cur_pos && cur_pos + payload_len +
+				sizeof(struct hostif_msg) < total_len)
+			++client_data->multi_packet_cnt;
+
+		cur_pos += payload_len + sizeof(struct hostif_msg);
+		payload += payload_len + sizeof(struct hostif_msg);
+
+	} while (cur_pos < total_len);
+}
+
+/**
+ * ish_cl_event_cb() - bus driver callback for incoming message/packet
+ * @device:	Pointer to the the ishtp client device for which this message
+ *		is targeted
+ *
+ * Remove the packet from the list and process the message by calling
+ * process_recv
+ */
+static void ish_cl_event_cb(struct ishtp_cl_device *device)
+{
+	struct ishtp_cl	*hid_ishtp_cl = device->driver_data;
+	struct ishtp_cl_rb *rb_in_proc;
+	size_t r_length;
+	unsigned long flags;
+
+	if (!hid_ishtp_cl)
+		return;
+
+	spin_lock_irqsave(&hid_ishtp_cl->in_process_spinlock, flags);
+	while (!list_empty(&hid_ishtp_cl->in_process_list.list)) {
+		rb_in_proc = list_entry(
+			hid_ishtp_cl->in_process_list.list.next,
+			struct ishtp_cl_rb, list);
+		list_del_init(&rb_in_proc->list);
+		spin_unlock_irqrestore(&hid_ishtp_cl->in_process_spinlock,
+			flags);
+
+		if (!rb_in_proc->buffer.data)
+			return;
+
+		r_length = rb_in_proc->buf_idx;
+
+		/* decide what to do with received data */
+		process_recv(hid_ishtp_cl, rb_in_proc->buffer.data, r_length);
+
+		ishtp_cl_io_rb_recycle(rb_in_proc);
+		spin_lock_irqsave(&hid_ishtp_cl->in_process_spinlock, flags);
+	}
+	spin_unlock_irqrestore(&hid_ishtp_cl->in_process_spinlock, flags);
+}
+
+/**
+ * hid_ishtp_set_feature() - send request to ISH FW to set a feature request
+ * @hid:	hid device instance for this request
+ * @buf:	feature buffer
+ * @len:	Length of feature buffer
+ * @report_id:	Report id for the feature set request
+ *
+ * This is called from hid core .request() callback. This function doesn't wait
+ * for response.
+ */
+void hid_ishtp_set_feature(struct hid_device *hid, char *buf, unsigned int len,
+			   int report_id)
+{
+	struct ishtp_hid_data *hid_data =  hid->driver_data;
+	struct ishtp_cl_data *client_data = hid_data->client_data;
+	struct hostif_msg *msg = (struct hostif_msg *)buf;
+	int	rv;
+	int	i;
+
+	hid_ishtp_trace(client_data,  "%s hid %p\n", __func__, hid);
+
+	rv = ishtp_hid_link_ready_wait(client_data);
+	if (rv) {
+		hid_ishtp_trace(client_data,  "%s hid %p link not ready\n",
+				__func__, hid);
+		return;
+	}
+
+	memset(msg, 0, sizeof(struct hostif_msg));
+	msg->hdr.command = HOSTIF_SET_FEATURE_REPORT;
+	for (i = 0; i < client_data->num_hid_devices; ++i) {
+		if (hid == client_data->hid_sensor_hubs[i]) {
+			msg->hdr.device_id =
+				client_data->hid_devices[i].dev_id;
+			break;
+		}
+	}
+
+	if (i == client_data->num_hid_devices)
+		return;
+
+	rv = ishtp_cl_send(client_data->hid_ishtp_cl, buf, len);
+	if (rv)
+		hid_ishtp_trace(client_data,  "%s hid %p send failed\n",
+				__func__, hid);
+}
+
+/**
+ * hid_ishtp_get_report() - request to get feature/input report
+ * @hid:	hid device instance for this request
+ * @report_id:	Report id for the get request
+ * @report_type:	Report type for the this request
+ *
+ * This is called from hid core .request() callback. This function will send
+ * request to FW and return without waiting for response.
+ */
+void hid_ishtp_get_report(struct hid_device *hid, int report_id,
+			  int report_type)
+{
+	struct ishtp_hid_data *hid_data =  hid->driver_data;
+	struct ishtp_cl_data *client_data = hid_data->client_data;
+	static unsigned char	buf[10];
+	unsigned int	len;
+	struct hostif_msg_to_sensor *msg = (struct hostif_msg_to_sensor *)buf;
+	int	rv;
+	int	i;
+
+	hid_ishtp_trace(client_data,  "%s hid %p\n", __func__, hid);
+	rv = ishtp_hid_link_ready_wait(client_data);
+	if (rv) {
+		hid_ishtp_trace(client_data,  "%s hid %p link not ready\n",
+				__func__, hid);
+		return;
+	}
+
+	len = sizeof(struct hostif_msg_to_sensor);
+
+	memset(msg, 0, sizeof(struct hostif_msg_to_sensor));
+	msg->hdr.command = (report_type == HID_FEATURE_REPORT) ?
+		HOSTIF_GET_FEATURE_REPORT : HOSTIF_GET_INPUT_REPORT;
+	for (i = 0; i < client_data->num_hid_devices; ++i) {
+		if (hid == client_data->hid_sensor_hubs[i]) {
+			msg->hdr.device_id =
+				client_data->hid_devices[i].dev_id;
+			break;
+		}
+	}
+
+	if (i == client_data->num_hid_devices)
+		return;
+
+	msg->report_id = report_id;
+	rv = ishtp_cl_send(client_data->hid_ishtp_cl, buf, len);
+	if (rv)
+		hid_ishtp_trace(client_data,  "%s hid %p send failed\n",
+				__func__, hid);
+}
+
+/**
+ * ishtp_hid_link_ready_wait() - Wait for link ready
+ * @client_data:	client data instance
+ *
+ * If the transport link started suspend process, then wait, till either
+ * resumed or timeout
+ *
+ * Return: 0 on success, non zero on error
+ */
+int ishtp_hid_link_ready_wait(struct ishtp_cl_data *client_data)
+{
+	int rc;
+
+	if (client_data->suspended) {
+		hid_ishtp_trace(client_data,  "wait for link ready\n");
+		rc = wait_event_interruptible_timeout(
+					client_data->ishtp_resume_wait,
+					!client_data->suspended,
+					5 * HZ);
+
+		if (rc == 0) {
+			hid_ishtp_trace(client_data,  "link not ready\n");
+			return -EIO;
+		}
+		hid_ishtp_trace(client_data,  "link ready\n");
+	}
+
+	return 0;
+}
+
+/**
+ * ishtp_enum_enum_devices() - Enumerate hid devices
+ * @hid_ishtp_cl:	client instance
+ *
+ * Helper function to send request to firmware to enumerate HID devices
+ *
+ * Return: 0 on success, non zero on error
+ */
+static int ishtp_enum_enum_devices(struct ishtp_cl *hid_ishtp_cl)
+{
+	struct hostif_msg msg;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+	int retry_count;
+	int rv;
+
+	/* Send HOSTIF_DM_ENUM_DEVICES */
+	memset(&msg, 0, sizeof(struct hostif_msg));
+	msg.hdr.command = HOSTIF_DM_ENUM_DEVICES;
+	rv = ishtp_cl_send(hid_ishtp_cl, (unsigned char *)&msg,
+			   sizeof(struct hostif_msg));
+	if (rv)
+		return rv;
+
+	retry_count = 0;
+	while (!client_data->enum_devices_done &&
+	       retry_count < 10) {
+		wait_event_interruptible_timeout(client_data->init_wait,
+					 client_data->enum_devices_done,
+					 3 * HZ);
+		++retry_count;
+		if (!client_data->enum_devices_done)
+			/* Send HOSTIF_DM_ENUM_DEVICES */
+			rv = ishtp_cl_send(hid_ishtp_cl,
+					   (unsigned char *) &msg,
+					   sizeof(struct hostif_msg));
+	}
+	if (!client_data->enum_devices_done) {
+		dev_err(&client_data->cl_device->dev,
+			"[hid-ish]: timed out waiting for enum_devices\n");
+		return -ETIMEDOUT;
+	}
+	if (!client_data->hid_devices) {
+		dev_err(&client_data->cl_device->dev,
+			"[hid-ish]: failed to allocate HID dev structures\n");
+		return -ENOMEM;
+	}
+
+	client_data->num_hid_devices = client_data->hid_dev_count;
+	dev_info(&hid_ishtp_cl->device->dev,
+		"[hid-ish]: enum_devices_done OK, num_hid_devices=%d\n",
+		client_data->num_hid_devices);
+
+	return	0;
+}
+
+/**
+ * ishtp_get_hid_descriptor() - Get hid descriptor
+ * @hid_ishtp_cl:	client instance
+ * @index:		Index into the hid_descr array
+ *
+ * Helper function to send request to firmware get HID descriptor of a device
+ *
+ * Return: 0 on success, non zero on error
+ */
+static int ishtp_get_hid_descriptor(struct ishtp_cl *hid_ishtp_cl, int index)
+{
+	struct hostif_msg msg;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+	int rv;
+
+	/* Get HID descriptor */
+	client_data->hid_descr_done = false;
+	memset(&msg, 0, sizeof(struct hostif_msg));
+	msg.hdr.command = HOSTIF_GET_HID_DESCRIPTOR;
+	msg.hdr.device_id = client_data->hid_devices[index].dev_id;
+	rv = ishtp_cl_send(hid_ishtp_cl, (unsigned char *) &msg,
+			   sizeof(struct hostif_msg));
+	if (rv)
+		return rv;
+
+	if (!client_data->hid_descr_done) {
+		wait_event_interruptible_timeout(client_data->init_wait,
+						 client_data->hid_descr_done,
+						 3 * HZ);
+		if (!client_data->hid_descr_done) {
+			dev_err(&client_data->cl_device->dev,
+				"[hid-ish]: timed out for hid_descr_done\n");
+			return -EIO;
+		}
+
+		if (!client_data->hid_descr[index]) {
+			dev_err(&client_data->cl_device->dev,
+				"[hid-ish]: allocation HID desc fail\n");
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * ishtp_get_report_descriptor() - Get report descriptor
+ * @hid_ishtp_cl:	client instance
+ * @index:		Index into the hid_descr array
+ *
+ * Helper function to send request to firmware get HID report descriptor of
+ * a device
+ *
+ * Return: 0 on success, non zero on error
+ */
+static int ishtp_get_report_descriptor(struct ishtp_cl *hid_ishtp_cl,
+				       int index)
+{
+	struct hostif_msg msg;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+	int rv;
+
+	/* Get report descriptor */
+	client_data->report_descr_done = false;
+	memset(&msg, 0, sizeof(struct hostif_msg));
+	msg.hdr.command = HOSTIF_GET_REPORT_DESCRIPTOR;
+	msg.hdr.device_id = client_data->hid_devices[index].dev_id;
+	rv = ishtp_cl_send(hid_ishtp_cl, (unsigned char *) &msg,
+			   sizeof(struct hostif_msg));
+	if (rv)
+		return rv;
+
+	if (!client_data->report_descr_done)
+		wait_event_interruptible_timeout(client_data->init_wait,
+					 client_data->report_descr_done,
+					 3 * HZ);
+	if (!client_data->report_descr_done) {
+		dev_err(&client_data->cl_device->dev,
+				"[hid-ish]: timed out for report descr\n");
+		return -EIO;
+	}
+	if (!client_data->report_descr[index]) {
+		dev_err(&client_data->cl_device->dev,
+			"[hid-ish]: failed to alloc report descr\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * hid_ishtp_cl_init() - Init function for ISHTP client
+ * @hid_ishtp_cl:	ISHTP client instance
+ * @reset:		true if called for init after reset
+ *
+ * This function complete the initializtion of the client. The summary of
+ * processing:
+ * - Send request to enumerate the hid clients
+ *	Get the HID descriptor for each enumearated device
+ *	Get report description of each device
+ *	Register each device wik hid core by calling ishtp_hid_probe
+ *
+ * Return: 0 on success, non zero on error
+ */
+static int hid_ishtp_cl_init(struct ishtp_cl *hid_ishtp_cl, int reset)
+{
+	struct ishtp_device *dev;
+	unsigned long flags;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+	int i;
+	int rv;
+
+	dev_dbg(&client_data->cl_device->dev, "%s\n", __func__);
+	hid_ishtp_trace(client_data,  "%s reset flag: %d\n", __func__, reset);
+
+	rv = ishtp_cl_link(hid_ishtp_cl, ISHTP_HOST_CLIENT_ID_ANY);
+	if (rv) {
+		dev_err(&client_data->cl_device->dev,
+			"ishtp_cl_link failed\n");
+		return	-ENOMEM;
+	}
+
+	client_data->init_done = 0;
+
+	dev = hid_ishtp_cl->dev;
+
+	/* Connect to FW client */
+	hid_ishtp_cl->rx_ring_size = HID_CL_RX_RING_SIZE;
+	hid_ishtp_cl->tx_ring_size = HID_CL_TX_RING_SIZE;
+
+	spin_lock_irqsave(&dev->fw_clients_lock, flags);
+	i = ishtp_fw_cl_by_uuid(dev, &hid_ishtp_guid);
+	if (i < 0) {
+		spin_unlock_irqrestore(&dev->fw_clients_lock, flags);
+		dev_err(&client_data->cl_device->dev,
+			"ish client uuid not found\n");
+		return i;
+	}
+	hid_ishtp_cl->fw_client_id = dev->fw_clients[i].client_id;
+	spin_unlock_irqrestore(&dev->fw_clients_lock, flags);
+	hid_ishtp_cl->state = ISHTP_CL_CONNECTING;
+
+	rv = ishtp_cl_connect(hid_ishtp_cl);
+	if (rv) {
+		dev_err(&client_data->cl_device->dev,
+			"client connect fail\n");
+		goto err_cl_unlink;
+	}
+
+	hid_ishtp_trace(client_data,  "%s client connected\n", __func__);
+
+	/* Register read callback */
+	ishtp_register_event_cb(hid_ishtp_cl->device, ish_cl_event_cb);
+
+	rv = ishtp_enum_enum_devices(hid_ishtp_cl);
+	if (rv)
+		goto err_cl_disconnect;
+
+	hid_ishtp_trace(client_data,  "%s enumerated device count %d\n",
+			__func__, client_data->num_hid_devices);
+
+	for (i = 0; i < client_data->num_hid_devices; ++i) {
+		client_data->cur_hid_dev = i;
+
+		rv = ishtp_get_hid_descriptor(hid_ishtp_cl, i);
+		if (rv)
+			goto err_cl_disconnect;
+
+		rv = ishtp_get_report_descriptor(hid_ishtp_cl, i);
+		if (rv)
+			goto err_cl_disconnect;
+
+		if (!reset) {
+			rv = ishtp_hid_probe(i, client_data);
+			if (rv) {
+				dev_err(&client_data->cl_device->dev,
+				"[hid-ish]: HID probe for #%u failed: %d\n",
+				i, rv);
+				goto err_cl_disconnect;
+			}
+		}
+	} /* for() on all hid devices */
+
+	client_data->init_done = 1;
+	client_data->suspended = false;
+	wake_up_interruptible(&client_data->ishtp_resume_wait);
+	hid_ishtp_trace(client_data,  "%s successful init\n", __func__);
+	return 0;
+
+err_cl_disconnect:
+	hid_ishtp_cl->state = ISHTP_CL_DISCONNECTING;
+	ishtp_cl_disconnect(hid_ishtp_cl);
+err_cl_unlink:
+	ishtp_cl_unlink(hid_ishtp_cl);
+	return rv;
+}
+
+/**
+ * hid_ishtp_cl_deinit() - Deinit function for ISHTP client
+ * @hid_ishtp_cl:	ISHTP client instance
+ *
+ * Unlink and free hid client
+ */
+static void hid_ishtp_cl_deinit(struct ishtp_cl *hid_ishtp_cl)
+{
+	ishtp_cl_unlink(hid_ishtp_cl);
+	ishtp_cl_flush_queues(hid_ishtp_cl);
+
+	/* disband and free all Tx and Rx client-level rings */
+	ishtp_cl_free(hid_ishtp_cl);
+}
+
+static void hid_ishtp_cl_reset_handler(struct work_struct *work)
+{
+	struct ishtp_cl_data *client_data;
+	struct ishtp_cl *hid_ishtp_cl;
+	struct ishtp_cl_device *cl_device;
+	int retry;
+	int rv;
+
+	client_data = container_of(work, struct ishtp_cl_data, work);
+
+	hid_ishtp_cl = client_data->hid_ishtp_cl;
+	cl_device = client_data->cl_device;
+
+	hid_ishtp_trace(client_data, "%s hid_ishtp_cl %p\n", __func__,
+			hid_ishtp_cl);
+	dev_dbg(&cl_device->dev, "%s\n", __func__);
+
+	hid_ishtp_cl_deinit(hid_ishtp_cl);
+
+	hid_ishtp_cl = ishtp_cl_allocate(cl_device->ishtp_dev);
+	if (!hid_ishtp_cl)
+		return;
+
+	cl_device->driver_data = hid_ishtp_cl;
+	hid_ishtp_cl->client_data = client_data;
+	client_data->hid_ishtp_cl = hid_ishtp_cl;
+
+	client_data->num_hid_devices = 0;
+
+	for (retry = 0; retry < 3; ++retry) {
+		rv = hid_ishtp_cl_init(hid_ishtp_cl, 1);
+		if (!rv)
+			break;
+		dev_err(&client_data->cl_device->dev, "Retry reset init\n");
+	}
+	if (rv) {
+		dev_err(&client_data->cl_device->dev, "Reset Failed\n");
+		hid_ishtp_trace(client_data, "%s Failed hid_ishtp_cl %p\n",
+				__func__, hid_ishtp_cl);
+	}
+}
+
+/**
+ * hid_ishtp_cl_probe() - ISHTP client driver probe
+ * @cl_device:		ISHTP client device instance
+ *
+ * This function gets called on device create on ISHTP bus
+ *
+ * Return: 0 on success, non zero on error
+ */
+static int hid_ishtp_cl_probe(struct ishtp_cl_device *cl_device)
+{
+	struct ishtp_cl *hid_ishtp_cl;
+	struct ishtp_cl_data *client_data;
+	int rv;
+
+	if (!cl_device)
+		return	-ENODEV;
+
+	if (uuid_le_cmp(hid_ishtp_guid,
+			cl_device->fw_client->props.protocol_name) != 0)
+		return	-ENODEV;
+
+	client_data = devm_kzalloc(&cl_device->dev, sizeof(*client_data),
+				   GFP_KERNEL);
+	if (!client_data)
+		return -ENOMEM;
+
+	hid_ishtp_cl = ishtp_cl_allocate(cl_device->ishtp_dev);
+	if (!hid_ishtp_cl)
+		return -ENOMEM;
+
+	cl_device->driver_data = hid_ishtp_cl;
+	hid_ishtp_cl->client_data = client_data;
+	client_data->hid_ishtp_cl = hid_ishtp_cl;
+	client_data->cl_device = cl_device;
+
+	init_waitqueue_head(&client_data->init_wait);
+	init_waitqueue_head(&client_data->ishtp_resume_wait);
+
+	INIT_WORK(&client_data->work, hid_ishtp_cl_reset_handler);
+
+	rv = hid_ishtp_cl_init(hid_ishtp_cl, 0);
+	if (rv) {
+		ishtp_cl_free(hid_ishtp_cl);
+		return rv;
+	}
+	ishtp_get_device(cl_device);
+
+	return 0;
+}
+
+/**
+ * hid_ishtp_cl_remove() - ISHTP client driver remove
+ * @cl_device:		ISHTP client device instance
+ *
+ * This function gets called on device remove on ISHTP bus
+ *
+ * Return: 0
+ */
+static int hid_ishtp_cl_remove(struct ishtp_cl_device *cl_device)
+{
+	struct ishtp_cl *hid_ishtp_cl = cl_device->driver_data;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+
+	hid_ishtp_trace(client_data, "%s hid_ishtp_cl %p\n", __func__,
+			hid_ishtp_cl);
+
+	dev_dbg(&cl_device->dev, "%s\n", __func__);
+	hid_ishtp_cl->state = ISHTP_CL_DISCONNECTING;
+	ishtp_cl_disconnect(hid_ishtp_cl);
+	ishtp_put_device(cl_device);
+	ishtp_hid_remove(client_data);
+	hid_ishtp_cl_deinit(hid_ishtp_cl);
+
+	hid_ishtp_cl = NULL;
+
+	client_data->num_hid_devices = 0;
+
+	return 0;
+}
+
+/**
+ * hid_ishtp_cl_reset() - ISHTP client driver reset
+ * @cl_device:		ISHTP client device instance
+ *
+ * This function gets called on device reset on ISHTP bus
+ *
+ * Return: 0
+ */
+static int hid_ishtp_cl_reset(struct ishtp_cl_device *cl_device)
+{
+	struct ishtp_cl *hid_ishtp_cl = cl_device->driver_data;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+
+	hid_ishtp_trace(client_data, "%s hid_ishtp_cl %p\n", __func__,
+			hid_ishtp_cl);
+
+	schedule_work(&client_data->work);
+
+	return 0;
+}
+
+#define to_ishtp_cl_device(d) container_of(d, struct ishtp_cl_device, dev)
+
+/**
+ * hid_ishtp_cl_suspend() - ISHTP client driver suspend
+ * @device:	device instance
+ *
+ * This function gets called on system suspend
+ *
+ * Return: 0
+ */
+static int hid_ishtp_cl_suspend(struct device *device)
+{
+	struct ishtp_cl_device *cl_device = to_ishtp_cl_device(device);
+	struct ishtp_cl *hid_ishtp_cl = cl_device->driver_data;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+
+	hid_ishtp_trace(client_data, "%s hid_ishtp_cl %p\n", __func__,
+			hid_ishtp_cl);
+	client_data->suspended = true;
+
+	return 0;
+}
+
+/**
+ * hid_ishtp_cl_resume() - ISHTP client driver resume
+ * @device:	device instance
+ *
+ * This function gets called on system resume
+ *
+ * Return: 0
+ */
+static int hid_ishtp_cl_resume(struct device *device)
+{
+	struct ishtp_cl_device *cl_device = to_ishtp_cl_device(device);
+	struct ishtp_cl *hid_ishtp_cl = cl_device->driver_data;
+	struct ishtp_cl_data *client_data = hid_ishtp_cl->client_data;
+
+	hid_ishtp_trace(client_data, "%s hid_ishtp_cl %p\n", __func__,
+			hid_ishtp_cl);
+	client_data->suspended = false;
+	return 0;
+}
+
+static const struct dev_pm_ops hid_ishtp_pm_ops = {
+	.suspend = hid_ishtp_cl_suspend,
+	.resume = hid_ishtp_cl_resume,
+};
+
+static struct ishtp_cl_driver	hid_ishtp_cl_driver = {
+	.name = "ish-hid",
+	.probe = hid_ishtp_cl_probe,
+	.remove = hid_ishtp_cl_remove,
+	.reset = hid_ishtp_cl_reset,
+	.driver.pm = &hid_ishtp_pm_ops,
+};
+
+static int __init ish_hid_init(void)
+{
+	int	rv;
+
+	/* Register ISHTP client device driver with ISHTP Bus */
+	rv = ishtp_cl_driver_register(&hid_ishtp_cl_driver);
+
+	return rv;
+
+}
+
+static void __exit ish_hid_exit(void)
+{
+	ishtp_cl_driver_unregister(&hid_ishtp_cl_driver);
+}
+
+late_initcall(ish_hid_init);
+module_exit(ish_hid_exit);
+
+MODULE_DESCRIPTION("ISH ISHTP HID client driver");
+/* Primary author */
+MODULE_AUTHOR("Daniel Drubin <daniel.drubin@intel.com>");
+/*
+ * Several modification for multi instance support
+ * suspend/resume and clean up
+ */
+MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
+
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("ishtp:*");
diff --git a/drivers/hid/intel-ish-hid/ishtp-hid.c b/drivers/hid/intel-ish-hid/ishtp-hid.c
new file mode 100644
index 000000000000..277983aa1d90
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp-hid.c
@@ -0,0 +1,246 @@
+/*
+ * ISHTP-HID glue driver.
+ *
+ * Copyright (c) 2012-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/hid.h>
+#include <uapi/linux/input.h>
+#include "ishtp/client.h"
+#include "ishtp-hid.h"
+
+/**
+ * ishtp_hid_parse() - hid-core .parse() callback
+ * @hid:	hid device instance
+ *
+ * This function gets called during call to hid_add_device
+ *
+ * Return: 0 on success and non zero on error
+ */
+static int ishtp_hid_parse(struct hid_device *hid)
+{
+	struct ishtp_hid_data *hid_data =  hid->driver_data;
+	struct ishtp_cl_data *client_data = hid_data->client_data;
+	int rv;
+
+	rv = hid_parse_report(hid, client_data->report_descr[hid_data->index],
+			      client_data->report_descr_size[hid_data->index]);
+	if (rv)
+		return	rv;
+
+	return 0;
+}
+
+/* Empty callbacks with success return code */
+static int ishtp_hid_start(struct hid_device *hid)
+{
+	return 0;
+}
+
+static void ishtp_hid_stop(struct hid_device *hid)
+{
+}
+
+static int ishtp_hid_open(struct hid_device *hid)
+{
+	return 0;
+}
+
+static void ishtp_hid_close(struct hid_device *hid)
+{
+}
+
+static int ishtp_raw_request(struct hid_device *hdev, unsigned char reportnum,
+	__u8 *buf, size_t len, unsigned char rtype, int reqtype)
+{
+	return 0;
+}
+
+/**
+ * ishtp_hid_request() - hid-core .request() callback
+ * @hid:	hid device instance
+ * @rep:	pointer to hid_report
+ * @reqtype:	type of req. [GET|SET]_REPORT
+ *
+ * This function is used to set/get feaure/input report.
+ */
+static void ishtp_hid_request(struct hid_device *hid, struct hid_report *rep,
+	int reqtype)
+{
+	struct ishtp_hid_data *hid_data =  hid->driver_data;
+	/* the specific report length, just HID part of it */
+	unsigned int len = ((rep->size - 1) >> 3) + 1 + (rep->id > 0);
+	char *buf;
+	unsigned int header_size = sizeof(struct hostif_msg);
+
+	len += header_size;
+
+	hid_data->request_done = false;
+	switch (reqtype) {
+	case HID_REQ_GET_REPORT:
+		hid_ishtp_get_report(hid, rep->id, rep->type);
+		break;
+	case HID_REQ_SET_REPORT:
+		/*
+		 * Spare 7 bytes for 64b accesses through
+		 * get/put_unaligned_le64()
+		 */
+		buf = kzalloc(len + 7, GFP_KERNEL);
+		if (!buf)
+			return;
+
+		hid_output_report(rep, buf + header_size);
+		hid_ishtp_set_feature(hid, buf, len, rep->id);
+		kfree(buf);
+		break;
+	}
+}
+
+/**
+ * ishtp_wait_for_response() - hid-core .wait() callback
+ * @hid:	hid device instance
+ *
+ * This function is used to wait after get feaure/input report.
+ *
+ * Return: 0 on success and non zero on error
+ */
+static int ishtp_wait_for_response(struct hid_device *hid)
+{
+	struct ishtp_hid_data *hid_data =  hid->driver_data;
+	struct ishtp_cl_data *client_data = hid_data->client_data;
+	int rv;
+
+	hid_ishtp_trace(client_data,  "%s hid %p\n", __func__, hid);
+
+	rv = ishtp_hid_link_ready_wait(hid_data->client_data);
+	if (rv)
+		return rv;
+
+	if (!hid_data->request_done)
+		wait_event_interruptible_timeout(hid_data->hid_wait,
+					hid_data->request_done, 3 * HZ);
+
+	if (!hid_data->request_done) {
+		hid_err(hid,
+			"timeout waiting for response from ISHTP device\n");
+		return -ETIMEDOUT;
+	}
+	hid_ishtp_trace(client_data,  "%s hid %p done\n", __func__, hid);
+
+	hid_data->request_done = false;
+
+	return 0;
+}
+
+/**
+ * ishtp_hid_wakeup() - Wakeup caller
+ * @hid:	hid device instance
+ *
+ * This function will wakeup caller waiting for Get/Set feature report
+ */
+void ishtp_hid_wakeup(struct hid_device *hid)
+{
+	struct ishtp_hid_data *hid_data = hid->driver_data;
+
+	hid_data->request_done = true;
+	wake_up_interruptible(&hid_data->hid_wait);
+}
+
+static struct hid_ll_driver ishtp_hid_ll_driver = {
+	.parse = ishtp_hid_parse,
+	.start = ishtp_hid_start,
+	.stop = ishtp_hid_stop,
+	.open = ishtp_hid_open,
+	.close = ishtp_hid_close,
+	.request = ishtp_hid_request,
+	.wait = ishtp_wait_for_response,
+	.raw_request = ishtp_raw_request
+};
+
+/**
+ * ishtp_hid_probe() - hid register ll driver
+ * @cur_hid_dev:	Index of hid device calling to register
+ * @client_data:	Client data pointer
+ *
+ * This function is used to allocate and add HID device.
+ *
+ * Return: 0 on success, non zero on error
+ */
+int ishtp_hid_probe(unsigned int cur_hid_dev,
+		    struct ishtp_cl_data *client_data)
+{
+	int rv;
+	struct hid_device *hid;
+	struct ishtp_hid_data *hid_data;
+
+	hid = hid_allocate_device();
+	if (IS_ERR(hid)) {
+		rv = PTR_ERR(hid);
+		return	-ENOMEM;
+	}
+
+	hid_data = kzalloc(sizeof(*hid_data), GFP_KERNEL);
+	if (!hid_data) {
+		rv = -ENOMEM;
+		goto err_hid_data;
+	}
+
+	hid_data->index = cur_hid_dev;
+	hid_data->client_data = client_data;
+	init_waitqueue_head(&hid_data->hid_wait);
+
+	hid->driver_data = hid_data;
+
+	client_data->hid_sensor_hubs[cur_hid_dev] = hid;
+
+	hid->ll_driver = &ishtp_hid_ll_driver;
+	hid->bus = BUS_INTEL_ISHTP;
+	hid->dev.parent = &client_data->cl_device->dev;
+	hid->version = le16_to_cpu(ISH_HID_VERSION);
+	hid->vendor = le16_to_cpu(ISH_HID_VENDOR);
+	hid->product = le16_to_cpu(ISH_HID_PRODUCT);
+	snprintf(hid->name, sizeof(hid->name), "%s %04hX:%04hX", "hid-ishtp",
+		hid->vendor, hid->product);
+
+	rv = hid_add_device(hid);
+	if (rv)
+		goto err_hid_device;
+
+	hid_ishtp_trace(client_data,  "%s allocated hid %p\n", __func__, hid);
+
+	return 0;
+
+err_hid_device:
+	kfree(hid_data);
+err_hid_data:
+	kfree(hid);
+	return rv;
+}
+
+/**
+ * ishtp_hid_probe() - Remove registered hid device
+ * @client_data:	client data pointer
+ *
+ * This function is used to destroy allocatd HID device.
+ */
+void ishtp_hid_remove(struct ishtp_cl_data *client_data)
+{
+	int i;
+
+	for (i = 0; i < client_data->num_hid_devices; ++i) {
+		if (client_data->hid_sensor_hubs[i]) {
+			kfree(client_data->hid_sensor_hubs[i]->driver_data);
+			hid_destroy_device(client_data->hid_sensor_hubs[i]);
+			client_data->hid_sensor_hubs[i] = NULL;
+		}
+	}
+}
diff --git a/drivers/hid/intel-ish-hid/ishtp-hid.h b/drivers/hid/intel-ish-hid/ishtp-hid.h
new file mode 100644
index 000000000000..f5c7eb79b7b5
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp-hid.h
@@ -0,0 +1,182 @@
+/*
+ * ISHTP-HID glue driver's definitions.
+ *
+ * Copyright (c) 2014-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+#ifndef ISHTP_HID__H
+#define	ISHTP_HID__H
+
+/* The fixed ISH product and vendor id */
+#define	ISH_HID_VENDOR	0x8086
+#define	ISH_HID_PRODUCT	0x22D8
+#define	ISH_HID_VERSION	0x0200
+
+#define	CMD_MASK	0x7F
+#define	IS_RESPONSE	0x80
+
+/* Used to dump to Linux trace buffer, if enabled */
+#define hid_ishtp_trace(client, ...)	\
+	client->cl_device->ishtp_dev->print_log(\
+		client->cl_device->ishtp_dev, __VA_ARGS__)
+
+/* ISH Transport protocol (ISHTP in short) GUID */
+static const uuid_le hid_ishtp_guid = UUID_LE(0x33AECD58, 0xB679, 0x4E54,
+					      0x9B, 0xD9, 0xA0, 0x4D, 0x34,
+					      0xF0, 0xC2, 0x26);
+
+/* ISH HID message structure */
+struct hostif_msg_hdr {
+	uint8_t	command; /* Bit 7: is_response */
+	uint8_t	device_id;
+	uint8_t	status;
+	uint8_t	flags;
+	uint16_t size;
+} __packed;
+
+struct hostif_msg {
+	struct hostif_msg_hdr	hdr;
+} __packed;
+
+struct hostif_msg_to_sensor {
+	struct hostif_msg_hdr	hdr;
+	uint8_t	report_id;
+} __packed;
+
+struct device_info {
+	uint32_t dev_id;
+	uint8_t dev_class;
+	uint16_t pid;
+	uint16_t vid;
+} __packed;
+
+struct ishtp_version {
+	uint8_t	major;
+	uint8_t	minor;
+	uint8_t	hotfix;
+	uint16_t build;
+} __packed;
+
+/* struct for ISHTP aggregated input data */
+struct report_list {
+	uint16_t total_size;
+	uint8_t	num_of_reports;
+	uint8_t	flags;
+	struct {
+		uint16_t	size_of_report;
+		uint8_t report[1];
+	} __packed reports[1];
+} __packed;
+
+/* HOSTIF commands */
+#define	HOSTIF_HID_COMMAND_BASE			0
+#define	HOSTIF_GET_HID_DESCRIPTOR		0
+#define	HOSTIF_GET_REPORT_DESCRIPTOR		1
+#define HOSTIF_GET_FEATURE_REPORT		2
+#define	HOSTIF_SET_FEATURE_REPORT		3
+#define	HOSTIF_GET_INPUT_REPORT			4
+#define	HOSTIF_PUBLISH_INPUT_REPORT		5
+#define	HOSTIF_PUBLISH_INPUT_REPORT_LIST	6
+#define	HOSTIF_DM_COMMAND_BASE			32
+#define	HOSTIF_DM_ENUM_DEVICES			33
+#define	HOSTIF_DM_ADD_DEVICE			34
+
+#define	MAX_HID_DEVICES				32
+
+/**
+ * struct ishtp_cl_data - Encapsulate per ISH TP HID Client
+ * @enum_device_done:	Enum devices response complete flag
+ * @hid_descr_done:	HID descriptor complete flag
+ * @report_descr_done:	Get report descriptor complete flag
+ * @init_done:		Init process completed successfully
+ * @suspended:		System is under suspend state or in progress
+ * @num_hid_devices:	Number of HID devices enumerated in this client
+ * @cur_hid_dev:	This keeps track of the device index for which
+ *			initialization and registration with HID core
+ *			in progress.
+ * @hid_devices:	Store vid/pid/devid for each enumerated HID device
+ * @report_descr:	Stores the raw report descriptors for each HID device
+ * @report_descr_size:	Report description of size of above repo_descr[]
+ * @hid_sensor_hubs:	Pointer to hid_device for all HID device, so that
+ *			when clients are removed, they can be freed
+ * @hid_descr:		Pointer to hid descriptor for each enumerated hid
+ *			device
+ * @hid_descr_size:	Size of each above report descriptor
+ * @init_wait:		Wait queue to wait during initialization, where the
+ *			client send message to ISH FW and wait for response
+ * @ishtp_hid_wait:	The wait for get report during wait callback from hid
+ *			core
+ * @bad_recv_cnt:	Running count of packets received with error
+ * @multi_packet_cnt:	Count of fragmented packet count
+ *
+ * This structure is used to store completion flags and per client data like
+ * like report description, number of HID devices etc.
+ */
+struct ishtp_cl_data {
+	/* completion flags */
+	bool enum_devices_done;
+	bool hid_descr_done;
+	bool report_descr_done;
+	bool init_done;
+	bool suspended;
+
+	unsigned int num_hid_devices;
+	unsigned int cur_hid_dev;
+	unsigned int hid_dev_count;
+
+	struct device_info *hid_devices;
+	unsigned char *report_descr[MAX_HID_DEVICES];
+	int report_descr_size[MAX_HID_DEVICES];
+	struct hid_device *hid_sensor_hubs[MAX_HID_DEVICES];
+	unsigned char *hid_descr[MAX_HID_DEVICES];
+	int hid_descr_size[MAX_HID_DEVICES];
+
+	wait_queue_head_t init_wait;
+	wait_queue_head_t ishtp_resume_wait;
+	struct ishtp_cl *hid_ishtp_cl;
+
+	/* Statistics */
+	unsigned int bad_recv_cnt;
+	int multi_packet_cnt;
+
+	struct work_struct work;
+	struct ishtp_cl_device *cl_device;
+};
+
+/**
+ * struct ishtp_hid_data - Per instance HID data
+ * @index:		Device index in the order of enumeration
+ * @request_done:	Get Feature/Input report complete flag
+ *			used during get/set request from hid core
+ * @client_data:	Link to the client instance
+ * @hid_wait:		Completion waitq
+ *
+ * Used to tie hid hid->driver data to driver client instance
+ */
+struct ishtp_hid_data {
+	int index;
+	bool request_done;
+	struct ishtp_cl_data *client_data;
+	wait_queue_head_t hid_wait;
+};
+
+/* Interface functions between HID LL driver and ISH TP client */
+void hid_ishtp_set_feature(struct hid_device *hid, char *buf, unsigned int len,
+			   int report_id);
+void hid_ishtp_get_report(struct hid_device *hid, int report_id,
+			  int report_type);
+int ishtp_hid_probe(unsigned int cur_hid_dev,
+		    struct ishtp_cl_data *client_data);
+void ishtp_hid_remove(struct ishtp_cl_data *client_data);
+int ishtp_hid_link_ready_wait(struct ishtp_cl_data *client_data);
+void ishtp_hid_wakeup(struct hid_device *hid);
+
+#endif	/* ISHTP_HID__H */
diff --git a/drivers/hid/intel-ish-hid/ishtp/bus.c b/drivers/hid/intel-ish-hid/ishtp/bus.c
new file mode 100644
index 000000000000..256521509d20
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/bus.c
@@ -0,0 +1,788 @@
+/*
+ * ISHTP bus driver
+ *
+ * Copyright (c) 2012-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include "bus.h"
+#include "ishtp-dev.h"
+#include "client.h"
+#include "hbm.h"
+
+static int ishtp_use_dma;
+module_param_named(ishtp_use_dma, ishtp_use_dma, int, 0600);
+MODULE_PARM_DESC(ishtp_use_dma, "Use DMA to send messages");
+
+#define to_ishtp_cl_driver(d) container_of(d, struct ishtp_cl_driver, driver)
+#define to_ishtp_cl_device(d) container_of(d, struct ishtp_cl_device, dev)
+static bool ishtp_device_ready;
+
+/**
+ * ishtp_recv() - process ishtp message
+ * @dev: ishtp device
+ *
+ * If a message with valid header and size is received, then
+ * this function calls appropriate handler. The host or firmware
+ * address is zero, then they are host bus management message,
+ * otherwise they are message fo clients.
+ */
+void ishtp_recv(struct ishtp_device *dev)
+{
+	uint32_t	msg_hdr;
+	struct ishtp_msg_hdr	*ishtp_hdr;
+
+	/* Read ISHTP header dword */
+	msg_hdr = dev->ops->ishtp_read_hdr(dev);
+	if (!msg_hdr)
+		return;
+
+	dev->ops->sync_fw_clock(dev);
+
+	ishtp_hdr = (struct ishtp_msg_hdr *)&msg_hdr;
+	dev->ishtp_msg_hdr = msg_hdr;
+
+	/* Sanity check: ISHTP frag. length in header */
+	if (ishtp_hdr->length > dev->mtu) {
+		dev_err(dev->devc,
+			"ISHTP hdr - bad length: %u; dropped [%08X]\n",
+			(unsigned int)ishtp_hdr->length, msg_hdr);
+		return;
+	}
+
+	/* ISHTP bus message */
+	if (!ishtp_hdr->host_addr && !ishtp_hdr->fw_addr)
+		recv_hbm(dev, ishtp_hdr);
+	/* ISHTP fixed-client message */
+	else if (!ishtp_hdr->host_addr)
+		recv_fixed_cl_msg(dev, ishtp_hdr);
+	else
+		/* ISHTP client message */
+		recv_ishtp_cl_msg(dev, ishtp_hdr);
+}
+EXPORT_SYMBOL(ishtp_recv);
+
+/**
+ * ishtp_send_msg() - Send ishtp message
+ * @dev: ishtp device
+ * @hdr: Message header
+ * @msg: Message contents
+ * @ipc_send_compl: completion callback
+ * @ipc_send_compl_prm: completion callback parameter
+ *
+ * Send a multi fragment message via IPC. After sending the first fragment
+ * the completion callback is called to schedule transmit of next fragment.
+ *
+ * Return: This returns IPC send message status.
+ */
+int ishtp_send_msg(struct ishtp_device *dev, struct ishtp_msg_hdr *hdr,
+		       void *msg, void(*ipc_send_compl)(void *),
+		       void *ipc_send_compl_prm)
+{
+	unsigned char	ipc_msg[IPC_FULL_MSG_SIZE];
+	uint32_t	drbl_val;
+
+	drbl_val = dev->ops->ipc_get_header(dev, hdr->length +
+					    sizeof(struct ishtp_msg_hdr),
+					    1);
+
+	memcpy(ipc_msg, &drbl_val, sizeof(uint32_t));
+	memcpy(ipc_msg + sizeof(uint32_t), hdr, sizeof(uint32_t));
+	memcpy(ipc_msg + 2 * sizeof(uint32_t), msg, hdr->length);
+	return	dev->ops->write(dev, ipc_send_compl, ipc_send_compl_prm,
+				ipc_msg, 2 * sizeof(uint32_t) + hdr->length);
+}
+
+/**
+ * ishtp_write_message() - Send ishtp single fragment message
+ * @dev: ishtp device
+ * @hdr: Message header
+ * @buf: message data
+ *
+ * Send a single fragment message via IPC.  This returns IPC send message
+ * status.
+ *
+ * Return: This returns IPC send message status.
+ */
+int ishtp_write_message(struct ishtp_device *dev, struct ishtp_msg_hdr *hdr,
+			unsigned char *buf)
+{
+	return ishtp_send_msg(dev, hdr, buf, NULL, NULL);
+}
+
+/**
+ * ishtp_fw_cl_by_uuid() - locate index of fw client
+ * @dev: ishtp device
+ * @uuid: uuid of the client to search
+ *
+ * Search firmware client using UUID.
+ *
+ * Return: fw client index or -ENOENT if not found
+ */
+int ishtp_fw_cl_by_uuid(struct ishtp_device *dev, const uuid_le *uuid)
+{
+	int i, res = -ENOENT;
+
+	for (i = 0; i < dev->fw_clients_num; ++i) {
+		if (uuid_le_cmp(*uuid, dev->fw_clients[i].props.protocol_name)
+				== 0) {
+			res = i;
+			break;
+		}
+	}
+	return res;
+}
+EXPORT_SYMBOL(ishtp_fw_cl_by_uuid);
+
+/**
+ * ishtp_fw_cl_by_id() - return index to fw_clients for client_id
+ * @dev: the ishtp device structure
+ * @client_id: fw client id to search
+ *
+ * Search firmware client using client id.
+ *
+ * Return: index on success, -ENOENT on failure.
+ */
+int ishtp_fw_cl_by_id(struct ishtp_device *dev, uint8_t client_id)
+{
+	int i, res = -ENOENT;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&dev->fw_clients_lock, flags);
+	for (i = 0; i < dev->fw_clients_num; i++) {
+		if (dev->fw_clients[i].client_id == client_id) {
+			res = i;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&dev->fw_clients_lock, flags);
+
+	return res;
+}
+
+/**
+ * ishtp_cl_device_probe() - Bus probe() callback
+ * @dev: the device structure
+ *
+ * This is a bus probe callback and calls the drive probe function.
+ *
+ * Return: Return value from driver probe() call.
+ */
+static int ishtp_cl_device_probe(struct device *dev)
+{
+	struct ishtp_cl_device *device = to_ishtp_cl_device(dev);
+	struct ishtp_cl_driver *driver;
+
+	if (!device)
+		return 0;
+
+	driver = to_ishtp_cl_driver(dev->driver);
+	if (!driver || !driver->probe)
+		return -ENODEV;
+
+	return driver->probe(device);
+}
+
+/**
+ * ishtp_cl_device_remove() - Bus remove() callback
+ * @dev: the device structure
+ *
+ * This is a bus remove callback and calls the drive remove function.
+ * Since the ISH driver model supports only built in, this is
+ * primarily can be called during pci driver init failure.
+ *
+ * Return: Return value from driver remove() call.
+ */
+static int ishtp_cl_device_remove(struct device *dev)
+{
+	struct ishtp_cl_device *device = to_ishtp_cl_device(dev);
+	struct ishtp_cl_driver *driver;
+
+	if (!device || !dev->driver)
+		return 0;
+
+	if (device->event_cb) {
+		device->event_cb = NULL;
+		cancel_work_sync(&device->event_work);
+	}
+
+	driver = to_ishtp_cl_driver(dev->driver);
+	if (!driver->remove) {
+		dev->driver = NULL;
+
+		return 0;
+	}
+
+	return driver->remove(device);
+}
+
+/**
+ * ishtp_cl_device_suspend() - Bus suspend callback
+ * @dev:	device
+ *
+ * Called during device suspend process.
+ *
+ * Return: Return value from driver suspend() call.
+ */
+static int ishtp_cl_device_suspend(struct device *dev)
+{
+	struct ishtp_cl_device *device = to_ishtp_cl_device(dev);
+	struct ishtp_cl_driver *driver;
+	int ret = 0;
+
+	if (!device)
+		return 0;
+
+	driver = to_ishtp_cl_driver(dev->driver);
+	if (driver && driver->driver.pm) {
+		if (driver->driver.pm->suspend)
+			ret = driver->driver.pm->suspend(dev);
+	}
+
+	return ret;
+}
+
+/**
+ * ishtp_cl_device_resume() - Bus resume callback
+ * @dev:	device
+ *
+ * Called during device resume process.
+ *
+ * Return: Return value from driver resume() call.
+ */
+static int ishtp_cl_device_resume(struct device *dev)
+{
+	struct ishtp_cl_device *device = to_ishtp_cl_device(dev);
+	struct ishtp_cl_driver *driver;
+	int ret = 0;
+
+	if (!device)
+		return 0;
+
+	/*
+	 * When ISH needs hard reset, it is done asynchrnously, hence bus
+	 * resume will  be called before full ISH resume
+	 */
+	if (device->ishtp_dev->resume_flag)
+		return 0;
+
+	driver = to_ishtp_cl_driver(dev->driver);
+	if (driver && driver->driver.pm) {
+		if (driver->driver.pm->resume)
+			ret = driver->driver.pm->resume(dev);
+	}
+
+	return ret;
+}
+
+/**
+ * ishtp_cl_device_reset() - Reset callback
+ * @device:	ishtp client device instance
+ *
+ * This is a callback when HW reset is done and the device need
+ * reinit.
+ *
+ * Return: Return value from driver reset() call.
+ */
+static int ishtp_cl_device_reset(struct ishtp_cl_device *device)
+{
+	struct ishtp_cl_driver *driver;
+	int ret = 0;
+
+	device->event_cb = NULL;
+	cancel_work_sync(&device->event_work);
+
+	driver = to_ishtp_cl_driver(device->dev.driver);
+	if (driver && driver->reset)
+		ret = driver->reset(device);
+
+	return ret;
+}
+
+static ssize_t modalias_show(struct device *dev, struct device_attribute *a,
+	char *buf)
+{
+	int len;
+
+	len = snprintf(buf, PAGE_SIZE, "ishtp:%s\n", dev_name(dev));
+	return (len >= PAGE_SIZE) ? (PAGE_SIZE - 1) : len;
+}
+
+static struct device_attribute ishtp_cl_dev_attrs[] = {
+	__ATTR_RO(modalias),
+	__ATTR_NULL,
+};
+
+static int ishtp_cl_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+	if (add_uevent_var(env, "MODALIAS=ishtp:%s", dev_name(dev)))
+		return -ENOMEM;
+	return 0;
+}
+
+static const struct dev_pm_ops ishtp_cl_bus_dev_pm_ops = {
+	/* Suspend callbacks */
+	.suspend = ishtp_cl_device_suspend,
+	.resume = ishtp_cl_device_resume,
+	/* Hibernate callbacks */
+	.freeze = ishtp_cl_device_suspend,
+	.thaw = ishtp_cl_device_resume,
+	.restore = ishtp_cl_device_resume,
+};
+
+static struct bus_type ishtp_cl_bus_type = {
+	.name		= "ishtp",
+	.dev_attrs	= ishtp_cl_dev_attrs,
+	.probe		= ishtp_cl_device_probe,
+	.remove		= ishtp_cl_device_remove,
+	.pm		= &ishtp_cl_bus_dev_pm_ops,
+	.uevent		= ishtp_cl_uevent,
+};
+
+static void ishtp_cl_dev_release(struct device *dev)
+{
+	kfree(to_ishtp_cl_device(dev));
+}
+
+static struct device_type ishtp_cl_device_type = {
+	.release	= ishtp_cl_dev_release,
+};
+
+/**
+ * ishtp_bus_add_device() - Function to create device on bus
+ * @dev:	ishtp device
+ * @uuid:	uuid of the client
+ * @name:	Name of the client
+ *
+ * Allocate ISHTP bus client device, attach it to uuid
+ * and register with ISHTP bus.
+ *
+ * Return: ishtp_cl_device pointer or NULL on failure
+ */
+static struct ishtp_cl_device *ishtp_bus_add_device(struct ishtp_device *dev,
+						    uuid_le uuid, char *name)
+{
+	struct ishtp_cl_device *device;
+	int status;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->device_list_lock, flags);
+	list_for_each_entry(device, &dev->device_list, device_link) {
+		if (!strcmp(name, dev_name(&device->dev))) {
+			device->fw_client = &dev->fw_clients[
+				dev->fw_client_presentation_num - 1];
+			spin_unlock_irqrestore(&dev->device_list_lock, flags);
+			ishtp_cl_device_reset(device);
+			return device;
+		}
+	}
+	spin_unlock_irqrestore(&dev->device_list_lock, flags);
+
+	device = kzalloc(sizeof(struct ishtp_cl_device), GFP_KERNEL);
+	if (!device)
+		return NULL;
+
+	device->dev.parent = dev->devc;
+	device->dev.bus = &ishtp_cl_bus_type;
+	device->dev.type = &ishtp_cl_device_type;
+	device->ishtp_dev = dev;
+
+	device->fw_client =
+		&dev->fw_clients[dev->fw_client_presentation_num - 1];
+
+	dev_set_name(&device->dev, "%s", name);
+
+	spin_lock_irqsave(&dev->device_list_lock, flags);
+	list_add_tail(&device->device_link, &dev->device_list);
+	spin_unlock_irqrestore(&dev->device_list_lock, flags);
+
+	status = device_register(&device->dev);
+	if (status) {
+		spin_lock_irqsave(&dev->device_list_lock, flags);
+		list_del(&device->device_link);
+		spin_unlock_irqrestore(&dev->device_list_lock, flags);
+		dev_err(dev->devc, "Failed to register ISHTP client device\n");
+		kfree(device);
+		return NULL;
+	}
+
+	ishtp_device_ready = true;
+
+	return device;
+}
+
+/**
+ * ishtp_bus_remove_device() - Function to relase device on bus
+ * @device:	client device instance
+ *
+ * This is a counterpart of ishtp_bus_add_device.
+ * Device is unregistered.
+ * the device structure is freed in 'ishtp_cl_dev_release' function
+ * Called only during error in pci driver init path.
+ */
+static void ishtp_bus_remove_device(struct ishtp_cl_device *device)
+{
+	device_unregister(&device->dev);
+}
+
+/**
+ * __ishtp_cl_driver_register() - Client driver register
+ * @driver:	the client driver instance
+ * @owner:	Owner of this driver module
+ *
+ * Once a client driver is probed, it created a client
+ * instance and registers with the bus.
+ *
+ * Return: Return value of driver_register or -ENODEV if not ready
+ */
+int __ishtp_cl_driver_register(struct ishtp_cl_driver *driver,
+			       struct module *owner)
+{
+	int err;
+
+	if (!ishtp_device_ready)
+		return -ENODEV;
+
+	driver->driver.name = driver->name;
+	driver->driver.owner = owner;
+	driver->driver.bus = &ishtp_cl_bus_type;
+
+	err = driver_register(&driver->driver);
+	if (err)
+		return err;
+
+	return 0;
+}
+EXPORT_SYMBOL(__ishtp_cl_driver_register);
+
+/**
+ * ishtp_cl_driver_unregister() - Client driver unregister
+ * @driver:	the client driver instance
+ *
+ * Unregister client during device removal process.
+ */
+void ishtp_cl_driver_unregister(struct ishtp_cl_driver *driver)
+{
+	driver_unregister(&driver->driver);
+}
+EXPORT_SYMBOL(ishtp_cl_driver_unregister);
+
+/**
+ * ishtp_bus_event_work() - event work function
+ * @work:	work struct pointer
+ *
+ * Once an event is received for a client this work
+ * function is called. If the device has registered a
+ * callback then the callback is called.
+ */
+static void ishtp_bus_event_work(struct work_struct *work)
+{
+	struct ishtp_cl_device *device;
+
+	device = container_of(work, struct ishtp_cl_device, event_work);
+
+	if (device->event_cb)
+		device->event_cb(device);
+}
+
+/**
+ * ishtp_cl_bus_rx_event() - schedule event work
+ * @device:	client device instance
+ *
+ * Once an event is received for a client this schedules
+ * a work function to process.
+ */
+void ishtp_cl_bus_rx_event(struct ishtp_cl_device *device)
+{
+	if (!device || !device->event_cb)
+		return;
+
+	if (device->event_cb)
+		schedule_work(&device->event_work);
+}
+
+/**
+ * ishtp_register_event_cb() - Register callback
+ * @device:	client device instance
+ * @event_cb:	Event processor for an client
+ *
+ * Register a callback for events, called from client driver
+ *
+ * Return: Return 0 or -EALREADY if already registered
+ */
+int ishtp_register_event_cb(struct ishtp_cl_device *device,
+	void (*event_cb)(struct ishtp_cl_device *))
+{
+	if (device->event_cb)
+		return -EALREADY;
+
+	device->event_cb = event_cb;
+	INIT_WORK(&device->event_work, ishtp_bus_event_work);
+
+	return 0;
+}
+EXPORT_SYMBOL(ishtp_register_event_cb);
+
+/**
+ * ishtp_get_device() - update usage count for the device
+ * @cl_device:	client device instance
+ *
+ * Increment the usage count. The device can't be deleted
+ */
+void ishtp_get_device(struct ishtp_cl_device *cl_device)
+{
+	cl_device->reference_count++;
+}
+EXPORT_SYMBOL(ishtp_get_device);
+
+/**
+ * ishtp_put_device() - decrement usage count for the device
+ * @cl_device:	client device instance
+ *
+ * Decrement the usage count. The device can be deleted is count = 0
+ */
+void ishtp_put_device(struct ishtp_cl_device *cl_device)
+{
+	cl_device->reference_count--;
+}
+EXPORT_SYMBOL(ishtp_put_device);
+
+/**
+ * ishtp_bus_new_client() - Create a new client
+ * @dev:	ISHTP device instance
+ *
+ * Once bus protocol enumerates a client, this is called
+ * to add a device for the client.
+ *
+ * Return: 0 on success or error code on failure
+ */
+int ishtp_bus_new_client(struct ishtp_device *dev)
+{
+	int	i;
+	char	*dev_name;
+	struct ishtp_cl_device	*cl_device;
+	uuid_le	device_uuid;
+
+	/*
+	 * For all reported clients, create an unconnected client and add its
+	 * device to ISHTP bus.
+	 * If appropriate driver has loaded, this will trigger its probe().
+	 * Otherwise, probe() will be called when driver is loaded
+	 */
+	i = dev->fw_client_presentation_num - 1;
+	device_uuid = dev->fw_clients[i].props.protocol_name;
+	dev_name = kasprintf(GFP_KERNEL,
+		"{%02X%02X%02X%02X-%02X%02X-%02X%02X-%02X%02X-%02X%02X%02X%02X%02X%02X}",
+		device_uuid.b[3], device_uuid.b[2], device_uuid.b[1],
+		device_uuid.b[0], device_uuid.b[5], device_uuid.b[4],
+		device_uuid.b[7], device_uuid.b[6], device_uuid.b[8],
+		device_uuid.b[9], device_uuid.b[10], device_uuid.b[11],
+		device_uuid.b[12], device_uuid.b[13], device_uuid.b[14],
+		device_uuid.b[15]);
+	if (!dev_name)
+		return	-ENOMEM;
+
+	cl_device = ishtp_bus_add_device(dev, device_uuid, dev_name);
+	if (!cl_device) {
+		kfree(dev_name);
+		return	-ENOENT;
+	}
+
+	kfree(dev_name);
+
+	return	0;
+}
+
+/**
+ * ishtp_cl_device_bind() - bind a device
+ * @cl:		ishtp client device
+ *
+ * Binds connected ishtp_cl to ISHTP bus device
+ *
+ * Return: 0 on success or fault code
+ */
+int ishtp_cl_device_bind(struct ishtp_cl *cl)
+{
+	struct ishtp_cl_device	*cl_device;
+	unsigned long flags;
+	int	rv;
+
+	if (!cl->fw_client_id || cl->state != ISHTP_CL_CONNECTED)
+		return	-EFAULT;
+
+	rv = -ENOENT;
+	spin_lock_irqsave(&cl->dev->device_list_lock, flags);
+	list_for_each_entry(cl_device, &cl->dev->device_list,
+			device_link) {
+		if (cl_device->fw_client->client_id == cl->fw_client_id) {
+			cl->device = cl_device;
+			rv = 0;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&cl->dev->device_list_lock, flags);
+	return	rv;
+}
+
+/**
+ * ishtp_bus_remove_all_clients() - Remove all clients
+ * @ishtp_dev:		ishtp device
+ * @warm_reset:		Reset due to FW reset dure to errors or S3 suspend
+ *
+ * This is part of reset/remove flow. This function the main processing
+ * only targets error processing, if the FW has forced reset or
+ * error to remove connected clients. When warm reset the client devices are
+ * not removed.
+ */
+void ishtp_bus_remove_all_clients(struct ishtp_device *ishtp_dev,
+				  bool warm_reset)
+{
+	struct ishtp_cl_device	*cl_device, *n;
+	struct ishtp_cl	*cl;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&ishtp_dev->cl_list_lock, flags);
+	list_for_each_entry(cl, &ishtp_dev->cl_list, link) {
+		cl->state = ISHTP_CL_DISCONNECTED;
+
+		/*
+		 * Wake any pending process. The waiter would check dev->state
+		 * and determine that it's not enabled already,
+		 * and will return error to its caller
+		 */
+		wake_up_interruptible(&cl->wait_ctrl_res);
+
+		/* Disband any pending read/write requests and free rb */
+		ishtp_cl_flush_queues(cl);
+
+		/* Remove all free and in_process rings, both Rx and Tx */
+		ishtp_cl_free_rx_ring(cl);
+		ishtp_cl_free_tx_ring(cl);
+
+		/*
+		 * Free client and ISHTP bus client device structures
+		 * don't free host client because it is part of the OS fd
+		 * structure
+		 */
+	}
+	spin_unlock_irqrestore(&ishtp_dev->cl_list_lock, flags);
+
+	/* Release DMA buffers for client messages */
+	ishtp_cl_free_dma_buf(ishtp_dev);
+
+	/* remove bus clients */
+	spin_lock_irqsave(&ishtp_dev->device_list_lock, flags);
+	list_for_each_entry_safe(cl_device, n, &ishtp_dev->device_list,
+				 device_link) {
+		if (warm_reset && cl_device->reference_count)
+			continue;
+
+		list_del(&cl_device->device_link);
+		spin_unlock_irqrestore(&ishtp_dev->device_list_lock, flags);
+		ishtp_bus_remove_device(cl_device);
+		spin_lock_irqsave(&ishtp_dev->device_list_lock, flags);
+	}
+	spin_unlock_irqrestore(&ishtp_dev->device_list_lock, flags);
+
+	/* Free all client structures */
+	spin_lock_irqsave(&ishtp_dev->fw_clients_lock, flags);
+	kfree(ishtp_dev->fw_clients);
+	ishtp_dev->fw_clients = NULL;
+	ishtp_dev->fw_clients_num = 0;
+	ishtp_dev->fw_client_presentation_num = 0;
+	ishtp_dev->fw_client_index = 0;
+	bitmap_zero(ishtp_dev->fw_clients_map, ISHTP_CLIENTS_MAX);
+	spin_unlock_irqrestore(&ishtp_dev->fw_clients_lock, flags);
+}
+EXPORT_SYMBOL(ishtp_bus_remove_all_clients);
+
+/**
+ * ishtp_reset_handler() - IPC reset handler
+ * @dev:	ishtp device
+ *
+ * ISHTP Handler for IPC_RESET notification
+ */
+void ishtp_reset_handler(struct ishtp_device *dev)
+{
+	unsigned long	flags;
+
+	/* Handle FW-initiated reset */
+	dev->dev_state = ISHTP_DEV_RESETTING;
+
+	/* Clear BH processing queue - no further HBMs */
+	spin_lock_irqsave(&dev->rd_msg_spinlock, flags);
+	dev->rd_msg_fifo_head = dev->rd_msg_fifo_tail = 0;
+	spin_unlock_irqrestore(&dev->rd_msg_spinlock, flags);
+
+	/* Handle ISH FW reset against upper layers */
+	ishtp_bus_remove_all_clients(dev, true);
+}
+EXPORT_SYMBOL(ishtp_reset_handler);
+
+/**
+ * ishtp_reset_compl_handler() - Reset completion handler
+ * @dev:	ishtp device
+ *
+ * ISHTP handler for IPC_RESET sequence completion to start
+ * host message bus start protocol sequence.
+ */
+void ishtp_reset_compl_handler(struct ishtp_device *dev)
+{
+	dev->dev_state = ISHTP_DEV_INIT_CLIENTS;
+	dev->hbm_state = ISHTP_HBM_START;
+	ishtp_hbm_start_req(dev);
+}
+EXPORT_SYMBOL(ishtp_reset_compl_handler);
+
+/**
+ * ishtp_use_dma_transfer() - Function to use DMA
+ *
+ * This interface is used to enable usage of DMA
+ *
+ * Return non zero if DMA can be enabled
+ */
+int ishtp_use_dma_transfer(void)
+{
+	return ishtp_use_dma;
+}
+
+/**
+ * ishtp_bus_register() - Function to register bus
+ *
+ * This register ishtp bus
+ *
+ * Return: Return output of bus_register
+ */
+static int  __init ishtp_bus_register(void)
+{
+	return bus_register(&ishtp_cl_bus_type);
+}
+
+/**
+ * ishtp_bus_unregister() - Function to unregister bus
+ *
+ * This unregister ishtp bus
+ */
+static void __exit ishtp_bus_unregister(void)
+{
+	bus_unregister(&ishtp_cl_bus_type);
+}
+
+module_init(ishtp_bus_register);
+module_exit(ishtp_bus_unregister);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/hid/intel-ish-hid/ishtp/bus.h b/drivers/hid/intel-ish-hid/ishtp/bus.h
new file mode 100644
index 000000000000..a1ffae7f26ad
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/bus.h
@@ -0,0 +1,114 @@
+/*
+ * ISHTP bus definitions
+ *
+ * Copyright (c) 2014-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+#ifndef _LINUX_ISHTP_CL_BUS_H
+#define _LINUX_ISHTP_CL_BUS_H
+
+#include <linux/device.h>
+#include <linux/mod_devicetable.h>
+
+struct ishtp_cl;
+struct ishtp_cl_device;
+struct ishtp_device;
+struct ishtp_msg_hdr;
+
+/**
+ * struct ishtp_cl_device - ISHTP device handle
+ * @dev:	device pointer
+ * @ishtp_dev:	pointer to ishtp device structure to primarily to access
+ *		hw device operation callbacks and properties
+ * @fw_client:	fw_client pointer to get fw information like protocol name
+ *		max message length etc.
+ * @device_link: Link to next client in the list on a bus
+ * @event_work:	Used to schedule rx event for client
+ * @driver_data: Storage driver private data
+ * @reference_count:	Used for get/put device
+ * @event_cb:	Callback to driver to send events
+ *
+ * An ishtp_cl_device pointer is returned from ishtp_add_device()
+ * and links ISHTP bus clients to their actual host client pointer.
+ * Drivers for ISHTP devices will get an ishtp_cl_device pointer
+ * when being probed and shall use it for doing bus I/O.
+ */
+struct ishtp_cl_device {
+	struct device		dev;
+	struct ishtp_device	*ishtp_dev;
+	struct ishtp_fw_client	*fw_client;
+	struct list_head	device_link;
+	struct work_struct	event_work;
+	void			*driver_data;
+	int			reference_count;
+	void (*event_cb)(struct ishtp_cl_device *device);
+};
+
+/**
+ * struct ishtp_cl_device - ISHTP device handle
+ * @driver:	driver instance on a bus
+ * @name:	Name of the device for probe
+ * @probe:	driver callback for device probe
+ * @remove:	driver callback on device removal
+ *
+ * Client drivers defines to get probed/removed for ISHTP client device.
+ */
+struct ishtp_cl_driver {
+	struct device_driver driver;
+	const char *name;
+	int (*probe)(struct ishtp_cl_device *dev);
+	int (*remove)(struct ishtp_cl_device *dev);
+	int (*reset)(struct ishtp_cl_device *dev);
+	const struct dev_pm_ops *pm;
+};
+
+
+int	ishtp_bus_new_client(struct ishtp_device *dev);
+void	ishtp_remove_all_clients(struct ishtp_device *dev);
+int	ishtp_cl_device_bind(struct ishtp_cl *cl);
+void	ishtp_cl_bus_rx_event(struct ishtp_cl_device *device);
+
+/* Write a multi-fragment message */
+int	ishtp_send_msg(struct ishtp_device *dev,
+		       struct ishtp_msg_hdr *hdr, void *msg,
+		       void (*ipc_send_compl)(void *),
+		       void *ipc_send_compl_prm);
+
+/* Write a single-fragment message */
+int	ishtp_write_message(struct ishtp_device *dev,
+			    struct ishtp_msg_hdr *hdr,
+			    unsigned char *buf);
+
+/* Use DMA to send/receive messages */
+int ishtp_use_dma_transfer(void);
+
+/* Exported functions */
+void	ishtp_bus_remove_all_clients(struct ishtp_device *ishtp_dev,
+				     bool warm_reset);
+
+void	ishtp_recv(struct ishtp_device *dev);
+void	ishtp_reset_handler(struct ishtp_device *dev);
+void	ishtp_reset_compl_handler(struct ishtp_device *dev);
+
+void	ishtp_put_device(struct ishtp_cl_device *);
+void	ishtp_get_device(struct ishtp_cl_device *);
+
+int	__ishtp_cl_driver_register(struct ishtp_cl_driver *driver,
+				   struct module *owner);
+#define ishtp_cl_driver_register(driver)		\
+	__ishtp_cl_driver_register(driver, THIS_MODULE)
+void	ishtp_cl_driver_unregister(struct ishtp_cl_driver *driver);
+
+int	ishtp_register_event_cb(struct ishtp_cl_device *device,
+				void (*read_cb)(struct ishtp_cl_device *));
+int	ishtp_fw_cl_by_uuid(struct ishtp_device *dev, const uuid_le *cuuid);
+
+#endif /* _LINUX_ISHTP_CL_BUS_H */
diff --git a/drivers/hid/intel-ish-hid/ishtp/client-buffers.c b/drivers/hid/intel-ish-hid/ishtp/client-buffers.c
new file mode 100644
index 000000000000..b9b917d2d50d
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/client-buffers.c
@@ -0,0 +1,257 @@
+/*
+ * ISHTP Ring Buffers
+ *
+ * Copyright (c) 2003-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#include <linux/slab.h>
+#include "client.h"
+
+/**
+ * ishtp_cl_alloc_rx_ring() - Allocate RX ring buffers
+ * @cl: client device instance
+ *
+ * Allocate and initialize RX ring buffers
+ *
+ * Return: 0 on success else -ENOMEM
+ */
+int ishtp_cl_alloc_rx_ring(struct ishtp_cl *cl)
+{
+	size_t	len = cl->device->fw_client->props.max_msg_length;
+	int	j;
+	struct ishtp_cl_rb *rb;
+	int	ret = 0;
+	unsigned long	flags;
+
+	for (j = 0; j < cl->rx_ring_size; ++j) {
+		rb = ishtp_io_rb_init(cl);
+		if (!rb) {
+			ret = -ENOMEM;
+			goto out;
+		}
+		ret = ishtp_io_rb_alloc_buf(rb, len);
+		if (ret)
+			goto out;
+		spin_lock_irqsave(&cl->free_list_spinlock, flags);
+		list_add_tail(&rb->list, &cl->free_rb_list.list);
+		spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
+	}
+
+	return	0;
+
+out:
+	dev_err(&cl->device->dev, "error in allocating Rx buffers\n");
+	ishtp_cl_free_rx_ring(cl);
+	return	ret;
+}
+
+/**
+ * ishtp_cl_alloc_tx_ring() - Allocate TX ring buffers
+ * @cl: client device instance
+ *
+ * Allocate and initialize TX ring buffers
+ *
+ * Return: 0 on success else -ENOMEM
+ */
+int ishtp_cl_alloc_tx_ring(struct ishtp_cl *cl)
+{
+	size_t	len = cl->device->fw_client->props.max_msg_length;
+	int	j;
+	unsigned long	flags;
+
+	/* Allocate pool to free Tx bufs */
+	for (j = 0; j < cl->tx_ring_size; ++j) {
+		struct ishtp_cl_tx_ring	*tx_buf;
+
+		tx_buf = kzalloc(sizeof(struct ishtp_cl_tx_ring), GFP_KERNEL);
+		if (!tx_buf)
+			goto	out;
+
+		tx_buf->send_buf.data = kmalloc(len, GFP_KERNEL);
+		if (!tx_buf->send_buf.data) {
+			kfree(tx_buf);
+			goto	out;
+		}
+
+		spin_lock_irqsave(&cl->tx_free_list_spinlock, flags);
+		list_add_tail(&tx_buf->list, &cl->tx_free_list.list);
+		spin_unlock_irqrestore(&cl->tx_free_list_spinlock, flags);
+	}
+	return	0;
+out:
+	dev_err(&cl->device->dev, "error in allocating Tx pool\n");
+	ishtp_cl_free_rx_ring(cl);
+	return	-ENOMEM;
+}
+
+/**
+ * ishtp_cl_free_rx_ring() - Free RX ring buffers
+ * @cl: client device instance
+ *
+ * Free RX ring buffers
+ */
+void ishtp_cl_free_rx_ring(struct ishtp_cl *cl)
+{
+	struct ishtp_cl_rb *rb;
+	unsigned long	flags;
+
+	/* release allocated memory - pass over free_rb_list */
+	spin_lock_irqsave(&cl->free_list_spinlock, flags);
+	while (!list_empty(&cl->free_rb_list.list)) {
+		rb = list_entry(cl->free_rb_list.list.next, struct ishtp_cl_rb,
+				list);
+		list_del(&rb->list);
+		kfree(rb->buffer.data);
+		kfree(rb);
+	}
+	spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
+	/* release allocated memory - pass over in_process_list */
+	spin_lock_irqsave(&cl->in_process_spinlock, flags);
+	while (!list_empty(&cl->in_process_list.list)) {
+		rb = list_entry(cl->in_process_list.list.next,
+				struct ishtp_cl_rb, list);
+		list_del(&rb->list);
+		kfree(rb->buffer.data);
+		kfree(rb);
+	}
+	spin_unlock_irqrestore(&cl->in_process_spinlock, flags);
+}
+
+/**
+ * ishtp_cl_free_tx_ring() - Free TX ring buffers
+ * @cl: client device instance
+ *
+ * Free TX ring buffers
+ */
+void ishtp_cl_free_tx_ring(struct ishtp_cl *cl)
+{
+	struct ishtp_cl_tx_ring	*tx_buf;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&cl->tx_free_list_spinlock, flags);
+	/* release allocated memory - pass over tx_free_list */
+	while (!list_empty(&cl->tx_free_list.list)) {
+		tx_buf = list_entry(cl->tx_free_list.list.next,
+				    struct ishtp_cl_tx_ring, list);
+		list_del(&tx_buf->list);
+		kfree(tx_buf->send_buf.data);
+		kfree(tx_buf);
+	}
+	spin_unlock_irqrestore(&cl->tx_free_list_spinlock, flags);
+
+	spin_lock_irqsave(&cl->tx_list_spinlock, flags);
+	/* release allocated memory - pass over tx_list */
+	while (!list_empty(&cl->tx_list.list)) {
+		tx_buf = list_entry(cl->tx_list.list.next,
+				    struct ishtp_cl_tx_ring, list);
+		list_del(&tx_buf->list);
+		kfree(tx_buf->send_buf.data);
+		kfree(tx_buf);
+	}
+	spin_unlock_irqrestore(&cl->tx_list_spinlock, flags);
+}
+
+/**
+ * ishtp_io_rb_free() - Free IO request block
+ * @rb: IO request block
+ *
+ * Free io request block memory
+ */
+void ishtp_io_rb_free(struct ishtp_cl_rb *rb)
+{
+	if (rb == NULL)
+		return;
+
+	kfree(rb->buffer.data);
+	kfree(rb);
+}
+
+/**
+ * ishtp_io_rb_init() - Allocate and init IO request block
+ * @cl: client device instance
+ *
+ * Allocate and initialize request block
+ *
+ * Return: Allocted IO request block pointer
+ */
+struct ishtp_cl_rb *ishtp_io_rb_init(struct ishtp_cl *cl)
+{
+	struct ishtp_cl_rb *rb;
+
+	rb = kzalloc(sizeof(struct ishtp_cl_rb), GFP_KERNEL);
+	if (!rb)
+		return NULL;
+
+	INIT_LIST_HEAD(&rb->list);
+	rb->cl = cl;
+	rb->buf_idx = 0;
+	return rb;
+}
+
+/**
+ * ishtp_io_rb_alloc_buf() - Allocate and init response buffer
+ * @rb: IO request block
+ * @length: length of response buffer
+ *
+ * Allocate respose buffer
+ *
+ * Return: 0 on success else -ENOMEM
+ */
+int ishtp_io_rb_alloc_buf(struct ishtp_cl_rb *rb, size_t length)
+{
+	if (!rb)
+		return -EINVAL;
+
+	if (length == 0)
+		return 0;
+
+	rb->buffer.data = kmalloc(length, GFP_KERNEL);
+	if (!rb->buffer.data)
+		return -ENOMEM;
+
+	rb->buffer.size = length;
+	return 0;
+}
+
+/**
+ * ishtp_cl_io_rb_recycle() - Recycle IO request blocks
+ * @rb: IO request block
+ *
+ * Re-append rb to its client's free list and send flow control if needed
+ *
+ * Return: 0 on success else -EFAULT
+ */
+int ishtp_cl_io_rb_recycle(struct ishtp_cl_rb *rb)
+{
+	struct ishtp_cl *cl;
+	int	rets = 0;
+	unsigned long	flags;
+
+	if (!rb || !rb->cl)
+		return	-EFAULT;
+
+	cl = rb->cl;
+	spin_lock_irqsave(&cl->free_list_spinlock, flags);
+	list_add_tail(&rb->list, &cl->free_rb_list.list);
+	spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
+
+	/*
+	 * If we returned the first buffer to empty 'free' list,
+	 * send flow control
+	 */
+	if (!cl->out_flow_ctrl_creds)
+		rets = ishtp_cl_read_start(cl);
+
+	return	rets;
+}
+EXPORT_SYMBOL(ishtp_cl_io_rb_recycle);
diff --git a/drivers/hid/intel-ish-hid/ishtp/client.c b/drivers/hid/intel-ish-hid/ishtp/client.c
new file mode 100644
index 000000000000..aad61328f282
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/client.c
@@ -0,0 +1,1054 @@
+/*
+ * ISHTP client logic
+ *
+ * Copyright (c) 2003-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include "hbm.h"
+#include "client.h"
+
+/**
+ * ishtp_read_list_flush() - Flush read queue
+ * @cl: ishtp client instance
+ *
+ * Used to remove all entries from read queue for a client
+ */
+static void ishtp_read_list_flush(struct ishtp_cl *cl)
+{
+	struct ishtp_cl_rb *rb;
+	struct ishtp_cl_rb *next;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&cl->dev->read_list_spinlock, flags);
+	list_for_each_entry_safe(rb, next, &cl->dev->read_list.list, list)
+		if (rb->cl && ishtp_cl_cmp_id(cl, rb->cl)) {
+			list_del(&rb->list);
+			ishtp_io_rb_free(rb);
+		}
+	spin_unlock_irqrestore(&cl->dev->read_list_spinlock, flags);
+}
+
+/**
+ * ishtp_cl_flush_queues() - Flush all queues for a client
+ * @cl: ishtp client instance
+ *
+ * Used to remove all queues for a client. This is called when a client device
+ * needs reset due to error, S3 resume or during module removal
+ *
+ * Return: 0 on success else -EINVAL if device is NULL
+ */
+int ishtp_cl_flush_queues(struct ishtp_cl *cl)
+{
+	if (WARN_ON(!cl || !cl->dev))
+		return -EINVAL;
+
+	ishtp_read_list_flush(cl);
+
+	return 0;
+}
+EXPORT_SYMBOL(ishtp_cl_flush_queues);
+
+/**
+ * ishtp_cl_init() - Initialize all fields of a client device
+ * @cl: ishtp client instance
+ * @dev: ishtp device
+ *
+ * Initializes a client device fields: Init spinlocks, init queues etc.
+ * This function is called during new client creation
+ */
+static void ishtp_cl_init(struct ishtp_cl *cl, struct ishtp_device *dev)
+{
+	memset(cl, 0, sizeof(struct ishtp_cl));
+	init_waitqueue_head(&cl->wait_ctrl_res);
+	spin_lock_init(&cl->free_list_spinlock);
+	spin_lock_init(&cl->in_process_spinlock);
+	spin_lock_init(&cl->tx_list_spinlock);
+	spin_lock_init(&cl->tx_free_list_spinlock);
+	spin_lock_init(&cl->fc_spinlock);
+	INIT_LIST_HEAD(&cl->link);
+	cl->dev = dev;
+
+	INIT_LIST_HEAD(&cl->free_rb_list.list);
+	INIT_LIST_HEAD(&cl->tx_list.list);
+	INIT_LIST_HEAD(&cl->tx_free_list.list);
+	INIT_LIST_HEAD(&cl->in_process_list.list);
+
+	cl->rx_ring_size = CL_DEF_RX_RING_SIZE;
+	cl->tx_ring_size = CL_DEF_TX_RING_SIZE;
+
+	/* dma */
+	cl->last_tx_path = CL_TX_PATH_IPC;
+	cl->last_dma_acked = 1;
+	cl->last_dma_addr = NULL;
+	cl->last_ipc_acked = 1;
+}
+
+/**
+ * ishtp_cl_allocate() - allocates client structure and sets it up.
+ * @dev: ishtp device
+ *
+ * Allocate memory for new client device and call to initialize each field.
+ *
+ * Return: The allocated client instance or NULL on failure
+ */
+struct ishtp_cl *ishtp_cl_allocate(struct ishtp_device *dev)
+{
+	struct ishtp_cl *cl;
+
+	cl = kmalloc(sizeof(struct ishtp_cl), GFP_KERNEL);
+	if (!cl)
+		return NULL;
+
+	ishtp_cl_init(cl, dev);
+	return cl;
+}
+EXPORT_SYMBOL(ishtp_cl_allocate);
+
+/**
+ * ishtp_cl_free() - Frees a client device
+ * @cl: client device instance
+ *
+ * Frees a client device
+ */
+void	ishtp_cl_free(struct ishtp_cl *cl)
+{
+	struct ishtp_device *dev;
+	unsigned long flags;
+
+	if (!cl)
+		return;
+
+	dev = cl->dev;
+	if (!dev)
+		return;
+
+	spin_lock_irqsave(&dev->cl_list_lock, flags);
+	ishtp_cl_free_rx_ring(cl);
+	ishtp_cl_free_tx_ring(cl);
+	kfree(cl);
+	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
+}
+EXPORT_SYMBOL(ishtp_cl_free);
+
+/**
+ * ishtp_cl_link() - Reserve a host id and link the client instance
+ * @cl: client device instance
+ * @id: host client id to use. It can be ISHTP_HOST_CLIENT_ID_ANY if any
+ *	id from the available can be used
+ *
+ *
+ * This allocates a single bit in the hostmap. This function will make sure
+ * that not many client sessions are opened at the same time. Once allocated
+ * the client device instance is added to the ishtp device in the current
+ * client list
+ *
+ * Return: 0 or error code on failure
+ */
+int ishtp_cl_link(struct ishtp_cl *cl, int id)
+{
+	struct ishtp_device *dev;
+	unsigned long	flags, flags_cl;
+	int	ret = 0;
+
+	if (WARN_ON(!cl || !cl->dev))
+		return -EINVAL;
+
+	dev = cl->dev;
+
+	spin_lock_irqsave(&dev->device_lock, flags);
+
+	if (dev->open_handle_count >= ISHTP_MAX_OPEN_HANDLE_COUNT) {
+		ret = -EMFILE;
+		goto unlock_dev;
+	}
+
+	/* If Id is not assigned get one*/
+	if (id == ISHTP_HOST_CLIENT_ID_ANY)
+		id = find_first_zero_bit(dev->host_clients_map,
+			ISHTP_CLIENTS_MAX);
+
+	if (id >= ISHTP_CLIENTS_MAX) {
+		spin_unlock_irqrestore(&dev->device_lock, flags);
+		dev_err(&cl->device->dev, "id exceeded %d", ISHTP_CLIENTS_MAX);
+		return -ENOENT;
+	}
+
+	dev->open_handle_count++;
+	cl->host_client_id = id;
+	spin_lock_irqsave(&dev->cl_list_lock, flags_cl);
+	if (dev->dev_state != ISHTP_DEV_ENABLED) {
+		ret = -ENODEV;
+		goto unlock_cl;
+	}
+	list_add_tail(&cl->link, &dev->cl_list);
+	set_bit(id, dev->host_clients_map);
+	cl->state = ISHTP_CL_INITIALIZING;
+
+unlock_cl:
+	spin_unlock_irqrestore(&dev->cl_list_lock, flags_cl);
+unlock_dev:
+	spin_unlock_irqrestore(&dev->device_lock, flags);
+	return ret;
+}
+EXPORT_SYMBOL(ishtp_cl_link);
+
+/**
+ * ishtp_cl_unlink() - remove fw_cl from the client device list
+ * @cl: client device instance
+ *
+ * Remove a previously linked device to a ishtp device
+ */
+void ishtp_cl_unlink(struct ishtp_cl *cl)
+{
+	struct ishtp_device *dev;
+	struct ishtp_cl *pos;
+	unsigned long	flags;
+
+	/* don't shout on error exit path */
+	if (!cl || !cl->dev)
+		return;
+
+	dev = cl->dev;
+
+	spin_lock_irqsave(&dev->device_lock, flags);
+	if (dev->open_handle_count > 0) {
+		clear_bit(cl->host_client_id, dev->host_clients_map);
+		dev->open_handle_count--;
+	}
+	spin_unlock_irqrestore(&dev->device_lock, flags);
+
+	/*
+	 * This checks that 'cl' is actually linked into device's structure,
+	 * before attempting 'list_del'
+	 */
+	spin_lock_irqsave(&dev->cl_list_lock, flags);
+	list_for_each_entry(pos, &dev->cl_list, link)
+		if (cl->host_client_id == pos->host_client_id) {
+			list_del_init(&pos->link);
+			break;
+		}
+	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
+}
+EXPORT_SYMBOL(ishtp_cl_unlink);
+
+/**
+ * ishtp_cl_disconnect() - Send disconnect request to firmware
+ * @cl: client device instance
+ *
+ * Send a disconnect request for a client to firmware.
+ *
+ * Return: 0 if successful disconnect response from the firmware or error
+ * code on failure
+ */
+int ishtp_cl_disconnect(struct ishtp_cl *cl)
+{
+	struct ishtp_device *dev;
+	int err;
+
+	if (WARN_ON(!cl || !cl->dev))
+		return -ENODEV;
+
+	dev = cl->dev;
+
+	dev->print_log(dev, "%s() state %d\n", __func__, cl->state);
+
+	if (cl->state != ISHTP_CL_DISCONNECTING) {
+		dev->print_log(dev, "%s() Disconnect in progress\n", __func__);
+		return 0;
+	}
+
+	if (ishtp_hbm_cl_disconnect_req(dev, cl)) {
+		dev->print_log(dev, "%s() Failed to disconnect\n", __func__);
+		dev_err(&cl->device->dev, "failed to disconnect.\n");
+		return -ENODEV;
+	}
+
+	err = wait_event_interruptible_timeout(cl->wait_ctrl_res,
+			(dev->dev_state != ISHTP_DEV_ENABLED ||
+			cl->state == ISHTP_CL_DISCONNECTED),
+			ishtp_secs_to_jiffies(ISHTP_CL_CONNECT_TIMEOUT));
+
+	/*
+	 * If FW reset arrived, this will happen. Don't check cl->,
+	 * as 'cl' may be freed already
+	 */
+	if (dev->dev_state != ISHTP_DEV_ENABLED) {
+		dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n",
+			       __func__);
+		return -ENODEV;
+	}
+
+	if (cl->state == ISHTP_CL_DISCONNECTED) {
+		dev->print_log(dev, "%s() successful\n", __func__);
+		return 0;
+	}
+
+	return -ENODEV;
+}
+EXPORT_SYMBOL(ishtp_cl_disconnect);
+
+/**
+ * ishtp_cl_is_other_connecting() - Check other client is connecting
+ * @cl: client device instance
+ *
+ * Checks if other client with the same fw client id is connecting
+ *
+ * Return: true if other client is connected else false
+ */
+static bool ishtp_cl_is_other_connecting(struct ishtp_cl *cl)
+{
+	struct ishtp_device *dev;
+	struct ishtp_cl *pos;
+	unsigned long	flags;
+
+	if (WARN_ON(!cl || !cl->dev))
+		return false;
+
+	dev = cl->dev;
+	spin_lock_irqsave(&dev->cl_list_lock, flags);
+	list_for_each_entry(pos, &dev->cl_list, link) {
+		if ((pos->state == ISHTP_CL_CONNECTING) && (pos != cl) &&
+				cl->fw_client_id == pos->fw_client_id) {
+			spin_unlock_irqrestore(&dev->cl_list_lock, flags);
+			return true;
+		}
+	}
+	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
+
+	return false;
+}
+
+/**
+ * ishtp_cl_connect() - Send connect request to firmware
+ * @cl: client device instance
+ *
+ * Send a connect request for a client to firmware. If successful it will
+ * RX and TX ring buffers
+ *
+ * Return: 0 if successful connect response from the firmware and able
+ * to bind and allocate ring buffers or error code on failure
+ */
+int ishtp_cl_connect(struct ishtp_cl *cl)
+{
+	struct ishtp_device *dev;
+	int rets;
+
+	if (WARN_ON(!cl || !cl->dev))
+		return -ENODEV;
+
+	dev = cl->dev;
+
+	dev->print_log(dev, "%s() current_state = %d\n", __func__, cl->state);
+
+	if (ishtp_cl_is_other_connecting(cl)) {
+		dev->print_log(dev, "%s() Busy\n", __func__);
+		return	-EBUSY;
+	}
+
+	if (ishtp_hbm_cl_connect_req(dev, cl)) {
+		dev->print_log(dev, "%s() HBM connect req fail\n", __func__);
+		return -ENODEV;
+	}
+
+	rets = wait_event_interruptible_timeout(cl->wait_ctrl_res,
+				(dev->dev_state == ISHTP_DEV_ENABLED &&
+				(cl->state == ISHTP_CL_CONNECTED ||
+				 cl->state == ISHTP_CL_DISCONNECTED)),
+				ishtp_secs_to_jiffies(
+					ISHTP_CL_CONNECT_TIMEOUT));
+	/*
+	 * If FW reset arrived, this will happen. Don't check cl->,
+	 * as 'cl' may be freed already
+	 */
+	if (dev->dev_state != ISHTP_DEV_ENABLED) {
+		dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n",
+			       __func__);
+		return -EFAULT;
+	}
+
+	if (cl->state != ISHTP_CL_CONNECTED) {
+		dev->print_log(dev, "%s() state != ISHTP_CL_CONNECTED\n",
+			       __func__);
+		return -EFAULT;
+	}
+
+	rets = cl->status;
+	if (rets) {
+		dev->print_log(dev, "%s() Invalid status\n", __func__);
+		return rets;
+	}
+
+	rets = ishtp_cl_device_bind(cl);
+	if (rets) {
+		dev->print_log(dev, "%s() Bind error\n", __func__);
+		ishtp_cl_disconnect(cl);
+		return rets;
+	}
+
+	rets = ishtp_cl_alloc_rx_ring(cl);
+	if (rets) {
+		dev->print_log(dev, "%s() Alloc RX ring failed\n", __func__);
+		/* if failed allocation, disconnect */
+		ishtp_cl_disconnect(cl);
+		return rets;
+	}
+
+	rets = ishtp_cl_alloc_tx_ring(cl);
+	if (rets) {
+		dev->print_log(dev, "%s() Alloc TX ring failed\n", __func__);
+		/* if failed allocation, disconnect */
+		ishtp_cl_free_rx_ring(cl);
+		ishtp_cl_disconnect(cl);
+		return rets;
+	}
+
+	/* Upon successful connection and allocation, emit flow-control */
+	rets = ishtp_cl_read_start(cl);
+
+	dev->print_log(dev, "%s() successful\n", __func__);
+
+	return rets;
+}
+EXPORT_SYMBOL(ishtp_cl_connect);
+
+/**
+ * ishtp_cl_read_start() - Prepare to read client message
+ * @cl: client device instance
+ *
+ * Get a free buffer from pool of free read buffers and add to read buffer
+ * pool to add contents. Send a flow control request to firmware to be able
+ * send next message.
+ *
+ * Return: 0 if successful or error code on failure
+ */
+int ishtp_cl_read_start(struct ishtp_cl *cl)
+{
+	struct ishtp_device *dev;
+	struct ishtp_cl_rb *rb;
+	int rets;
+	int i;
+	unsigned long	flags;
+	unsigned long	dev_flags;
+
+	if (WARN_ON(!cl || !cl->dev))
+		return -ENODEV;
+
+	dev = cl->dev;
+
+	if (cl->state != ISHTP_CL_CONNECTED)
+		return -ENODEV;
+
+	if (dev->dev_state != ISHTP_DEV_ENABLED)
+		return -ENODEV;
+
+	i = ishtp_fw_cl_by_id(dev, cl->fw_client_id);
+	if (i < 0) {
+		dev_err(&cl->device->dev, "no such fw client %d\n",
+			cl->fw_client_id);
+		return -ENODEV;
+	}
+
+	/* The current rb is the head of the free rb list */
+	spin_lock_irqsave(&cl->free_list_spinlock, flags);
+	if (list_empty(&cl->free_rb_list.list)) {
+		dev_warn(&cl->device->dev,
+			 "[ishtp-ish] Rx buffers pool is empty\n");
+		rets = -ENOMEM;
+		rb = NULL;
+		spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
+		goto out;
+	}
+	rb = list_entry(cl->free_rb_list.list.next, struct ishtp_cl_rb, list);
+	list_del_init(&rb->list);
+	spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
+
+	rb->cl = cl;
+	rb->buf_idx = 0;
+
+	INIT_LIST_HEAD(&rb->list);
+	rets = 0;
+
+	/*
+	 * This must be BEFORE sending flow control -
+	 * response in ISR may come too fast...
+	 */
+	spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);
+	list_add_tail(&rb->list, &dev->read_list.list);
+	spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);
+	if (ishtp_hbm_cl_flow_control_req(dev, cl)) {
+		rets = -ENODEV;
+		goto out;
+	}
+out:
+	/* if ishtp_hbm_cl_flow_control_req failed, return rb to free list */
+	if (rets && rb) {
+		spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);
+		list_del(&rb->list);
+		spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);
+
+		spin_lock_irqsave(&cl->free_list_spinlock, flags);
+		list_add_tail(&rb->list, &cl->free_rb_list.list);
+		spin_unlock_irqrestore(&cl->free_list_spinlock, flags);
+	}
+	return rets;
+}
+
+/**
+ * ishtp_cl_send() - Send a message to firmware
+ * @cl: client device instance
+ * @buf: message buffer
+ * @length: length of message
+ *
+ * If the client is correct state to send message, this function gets a buffer
+ * from tx ring buffers, copy the message data and call to send the message
+ * using ishtp_cl_send_msg()
+ *
+ * Return: 0 if successful or error code on failure
+ */
+int ishtp_cl_send(struct ishtp_cl *cl, uint8_t *buf, size_t length)
+{
+	struct ishtp_device	*dev;
+	int	id;
+	struct ishtp_cl_tx_ring	*cl_msg;
+	int	have_msg_to_send = 0;
+	unsigned long	tx_flags, tx_free_flags;
+
+	if (WARN_ON(!cl || !cl->dev))
+		return -ENODEV;
+
+	dev = cl->dev;
+
+	if (cl->state != ISHTP_CL_CONNECTED) {
+		++cl->err_send_msg;
+		return -EPIPE;
+	}
+
+	if (dev->dev_state != ISHTP_DEV_ENABLED) {
+		++cl->err_send_msg;
+		return -ENODEV;
+	}
+
+	/* Check if we have fw client device */
+	id = ishtp_fw_cl_by_id(dev, cl->fw_client_id);
+	if (id < 0) {
+		++cl->err_send_msg;
+		return -ENOENT;
+	}
+
+	if (length > dev->fw_clients[id].props.max_msg_length) {
+		++cl->err_send_msg;
+		return -EMSGSIZE;
+	}
+
+	/* No free bufs */
+	spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
+	if (list_empty(&cl->tx_free_list.list)) {
+		spin_unlock_irqrestore(&cl->tx_free_list_spinlock,
+			tx_free_flags);
+		++cl->err_send_msg;
+		return	-ENOMEM;
+	}
+
+	cl_msg = list_first_entry(&cl->tx_free_list.list,
+		struct ishtp_cl_tx_ring, list);
+	if (!cl_msg->send_buf.data) {
+		spin_unlock_irqrestore(&cl->tx_free_list_spinlock,
+			tx_free_flags);
+		return	-EIO;
+		/* Should not happen, as free list is pre-allocated */
+	}
+	/*
+	 * This is safe, as 'length' is already checked for not exceeding
+	 * max ISHTP message size per client
+	 */
+	list_del_init(&cl_msg->list);
+	spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);
+	memcpy(cl_msg->send_buf.data, buf, length);
+	cl_msg->send_buf.size = length;
+	spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);
+	have_msg_to_send = !list_empty(&cl->tx_list.list);
+	list_add_tail(&cl_msg->list, &cl->tx_list.list);
+	spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
+
+	if (!have_msg_to_send && cl->ishtp_flow_ctrl_creds > 0)
+		ishtp_cl_send_msg(dev, cl);
+
+	return	0;
+}
+EXPORT_SYMBOL(ishtp_cl_send);
+
+/**
+ * ishtp_cl_read_complete() - read complete
+ * @rb: Pointer to client request block
+ *
+ * If the message is completely received call ishtp_cl_bus_rx_event()
+ * to process message
+ */
+static void ishtp_cl_read_complete(struct ishtp_cl_rb *rb)
+{
+	unsigned long	flags;
+	int	schedule_work_flag = 0;
+	struct ishtp_cl	*cl = rb->cl;
+
+	spin_lock_irqsave(&cl->in_process_spinlock, flags);
+	/*
+	 * if in-process list is empty, then need to schedule
+	 * the processing thread
+	 */
+	schedule_work_flag = list_empty(&cl->in_process_list.list);
+	list_add_tail(&rb->list, &cl->in_process_list.list);
+	spin_unlock_irqrestore(&cl->in_process_spinlock, flags);
+
+	if (schedule_work_flag)
+		ishtp_cl_bus_rx_event(cl->device);
+}
+
+/**
+ * ipc_tx_callback() - IPC tx callback function
+ * @prm: Pointer to client device instance
+ *
+ * Send message over IPC either first time or on callback on previous message
+ * completion
+ */
+static void ipc_tx_callback(void *prm)
+{
+	struct ishtp_cl	*cl = prm;
+	struct ishtp_cl_tx_ring	*cl_msg;
+	size_t	rem;
+	struct ishtp_device	*dev = (cl ? cl->dev : NULL);
+	struct ishtp_msg_hdr	ishtp_hdr;
+	unsigned long	tx_flags, tx_free_flags;
+	unsigned char	*pmsg;
+
+	if (!dev)
+		return;
+
+	/*
+	 * Other conditions if some critical error has
+	 * occurred before this callback is called
+	 */
+	if (dev->dev_state != ISHTP_DEV_ENABLED)
+		return;
+
+	if (cl->state != ISHTP_CL_CONNECTED)
+		return;
+
+	spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);
+	if (list_empty(&cl->tx_list.list)) {
+		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
+		return;
+	}
+
+	if (cl->ishtp_flow_ctrl_creds != 1 && !cl->sending) {
+		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
+		return;
+	}
+
+	if (!cl->sending) {
+		--cl->ishtp_flow_ctrl_creds;
+		cl->last_ipc_acked = 0;
+		cl->last_tx_path = CL_TX_PATH_IPC;
+		cl->sending = 1;
+	}
+
+	cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring,
+			    list);
+	rem = cl_msg->send_buf.size - cl->tx_offs;
+
+	ishtp_hdr.host_addr = cl->host_client_id;
+	ishtp_hdr.fw_addr = cl->fw_client_id;
+	ishtp_hdr.reserved = 0;
+	pmsg = cl_msg->send_buf.data + cl->tx_offs;
+
+	if (rem <= dev->mtu) {
+		ishtp_hdr.length = rem;
+		ishtp_hdr.msg_complete = 1;
+		cl->sending = 0;
+		list_del_init(&cl_msg->list);	/* Must be before write */
+		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
+		/* Submit to IPC queue with no callback */
+		ishtp_write_message(dev, &ishtp_hdr, pmsg);
+		spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
+		list_add_tail(&cl_msg->list, &cl->tx_free_list.list);
+		spin_unlock_irqrestore(&cl->tx_free_list_spinlock,
+			tx_free_flags);
+	} else {
+		/* Send IPC fragment */
+		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
+		cl->tx_offs += dev->mtu;
+		ishtp_hdr.length = dev->mtu;
+		ishtp_hdr.msg_complete = 0;
+		ishtp_send_msg(dev, &ishtp_hdr, pmsg, ipc_tx_callback, cl);
+	}
+}
+
+/**
+ * ishtp_cl_send_msg_ipc() -Send message using IPC
+ * @dev: ISHTP device instance
+ * @cl: Pointer to client device instance
+ *
+ * Send message over IPC not using DMA
+ */
+static void ishtp_cl_send_msg_ipc(struct ishtp_device *dev,
+				  struct ishtp_cl *cl)
+{
+	/* If last DMA message wasn't acked yet, leave this one in Tx queue */
+	if (cl->last_tx_path == CL_TX_PATH_DMA && cl->last_dma_acked == 0)
+		return;
+
+	cl->tx_offs = 0;
+	ipc_tx_callback(cl);
+	++cl->send_msg_cnt_ipc;
+}
+
+/**
+ * ishtp_cl_send_msg_dma() -Send message using DMA
+ * @dev: ISHTP device instance
+ * @cl: Pointer to client device instance
+ *
+ * Send message using DMA
+ */
+static void ishtp_cl_send_msg_dma(struct ishtp_device *dev,
+	struct ishtp_cl *cl)
+{
+	struct ishtp_msg_hdr	hdr;
+	struct dma_xfer_hbm	dma_xfer;
+	unsigned char	*msg_addr;
+	int off;
+	struct ishtp_cl_tx_ring	*cl_msg;
+	unsigned long tx_flags, tx_free_flags;
+
+	/* If last IPC message wasn't acked yet, leave this one in Tx queue */
+	if (cl->last_tx_path == CL_TX_PATH_IPC && cl->last_ipc_acked == 0)
+		return;
+
+	spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);
+	if (list_empty(&cl->tx_list.list)) {
+		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
+		return;
+	}
+
+	cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring,
+		list);
+
+	msg_addr = ishtp_cl_get_dma_send_buf(dev, cl_msg->send_buf.size);
+	if (!msg_addr) {
+		spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
+		if (dev->transfer_path == CL_TX_PATH_DEFAULT)
+			ishtp_cl_send_msg_ipc(dev, cl);
+		return;
+	}
+
+	list_del_init(&cl_msg->list);	/* Must be before write */
+	spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);
+
+	--cl->ishtp_flow_ctrl_creds;
+	cl->last_dma_acked = 0;
+	cl->last_dma_addr = msg_addr;
+	cl->last_tx_path = CL_TX_PATH_DMA;
+
+	/* write msg to dma buf */
+	memcpy(msg_addr, cl_msg->send_buf.data, cl_msg->send_buf.size);
+
+	/* send dma_xfer hbm msg */
+	off = msg_addr - (unsigned char *)dev->ishtp_host_dma_tx_buf;
+	ishtp_hbm_hdr(&hdr, sizeof(struct dma_xfer_hbm));
+	dma_xfer.hbm = DMA_XFER;
+	dma_xfer.fw_client_id = cl->fw_client_id;
+	dma_xfer.host_client_id = cl->host_client_id;
+	dma_xfer.reserved = 0;
+	dma_xfer.msg_addr = dev->ishtp_host_dma_tx_buf_phys + off;
+	dma_xfer.msg_length = cl_msg->send_buf.size;
+	dma_xfer.reserved2 = 0;
+	ishtp_write_message(dev, &hdr, (unsigned char *)&dma_xfer);
+	spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);
+	list_add_tail(&cl_msg->list, &cl->tx_free_list.list);
+	spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);
+	++cl->send_msg_cnt_dma;
+}
+
+/**
+ * ishtp_cl_send_msg() -Send message using DMA or IPC
+ * @dev: ISHTP device instance
+ * @cl: Pointer to client device instance
+ *
+ * Send message using DMA or IPC based on transfer_path
+ */
+void ishtp_cl_send_msg(struct ishtp_device *dev, struct ishtp_cl *cl)
+{
+	if (dev->transfer_path == CL_TX_PATH_DMA)
+		ishtp_cl_send_msg_dma(dev, cl);
+	else
+		ishtp_cl_send_msg_ipc(dev, cl);
+}
+
+/**
+ * recv_ishtp_cl_msg() -Receive client message
+ * @dev: ISHTP device instance
+ * @ishtp_hdr: Pointer to message header
+ *
+ * Receive and dispatch ISHTP client messages. This function executes in ISR
+ * context
+ */
+void recv_ishtp_cl_msg(struct ishtp_device *dev,
+		       struct ishtp_msg_hdr *ishtp_hdr)
+{
+	struct ishtp_cl *cl;
+	struct ishtp_cl_rb *rb;
+	struct ishtp_cl_rb *new_rb;
+	unsigned char *buffer = NULL;
+	struct ishtp_cl_rb *complete_rb = NULL;
+	unsigned long	dev_flags;
+	unsigned long	flags;
+	int	rb_count;
+
+	if (ishtp_hdr->reserved) {
+		dev_err(dev->devc, "corrupted message header.\n");
+		goto	eoi;
+	}
+
+	if (ishtp_hdr->length > IPC_PAYLOAD_SIZE) {
+		dev_err(dev->devc,
+			"ISHTP message length in hdr exceeds IPC MTU\n");
+		goto	eoi;
+	}
+
+	spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);
+	rb_count = -1;
+	list_for_each_entry(rb, &dev->read_list.list, list) {
+		++rb_count;
+		cl = rb->cl;
+		if (!cl || !(cl->host_client_id == ishtp_hdr->host_addr &&
+				cl->fw_client_id == ishtp_hdr->fw_addr) ||
+				!(cl->state == ISHTP_CL_CONNECTED))
+			continue;
+
+		 /* If no Rx buffer is allocated, disband the rb */
+		if (rb->buffer.size == 0 || rb->buffer.data == NULL) {
+			spin_unlock_irqrestore(&dev->read_list_spinlock,
+				dev_flags);
+			dev_err(&cl->device->dev,
+				"Rx buffer is not allocated.\n");
+			list_del(&rb->list);
+			ishtp_io_rb_free(rb);
+			cl->status = -ENOMEM;
+			goto	eoi;
+		}
+
+		/*
+		 * If message buffer overflown (exceeds max. client msg
+		 * size, drop message and return to free buffer.
+		 * Do we need to disconnect such a client? (We don't send
+		 * back FC, so communication will be stuck anyway)
+		 */
+		if (rb->buffer.size < ishtp_hdr->length + rb->buf_idx) {
+			spin_unlock_irqrestore(&dev->read_list_spinlock,
+				dev_flags);
+			dev_err(&cl->device->dev,
+				"message overflow. size %d len %d idx %ld\n",
+				rb->buffer.size, ishtp_hdr->length,
+				rb->buf_idx);
+			list_del(&rb->list);
+			ishtp_cl_io_rb_recycle(rb);
+			cl->status = -EIO;
+			goto	eoi;
+		}
+
+		buffer = rb->buffer.data + rb->buf_idx;
+		dev->ops->ishtp_read(dev, buffer, ishtp_hdr->length);
+
+		rb->buf_idx += ishtp_hdr->length;
+		if (ishtp_hdr->msg_complete) {
+			/* Last fragment in message - it's complete */
+			cl->status = 0;
+			list_del(&rb->list);
+			complete_rb = rb;
+
+			--cl->out_flow_ctrl_creds;
+			/*
+			 * the whole msg arrived, send a new FC, and add a new
+			 * rb buffer for the next coming msg
+			 */
+			spin_lock_irqsave(&cl->free_list_spinlock, flags);
+
+			if (!list_empty(&cl->free_rb_list.list)) {
+				new_rb = list_entry(cl->free_rb_list.list.next,
+					struct ishtp_cl_rb, list);
+				list_del_init(&new_rb->list);
+				spin_unlock_irqrestore(&cl->free_list_spinlock,
+					flags);
+				new_rb->cl = cl;
+				new_rb->buf_idx = 0;
+				INIT_LIST_HEAD(&new_rb->list);
+				list_add_tail(&new_rb->list,
+					&dev->read_list.list);
+
+				ishtp_hbm_cl_flow_control_req(dev, cl);
+			} else {
+				spin_unlock_irqrestore(&cl->free_list_spinlock,
+					flags);
+			}
+		}
+		/* One more fragment in message (even if this was last) */
+		++cl->recv_msg_num_frags;
+
+		/*
+		 * We can safely break here (and in BH too),
+		 * a single input message can go only to a single request!
+		 */
+		break;
+	}
+
+	spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);
+	/* If it's nobody's message, just read and discard it */
+	if (!buffer) {
+		uint8_t	rd_msg_buf[ISHTP_RD_MSG_BUF_SIZE];
+
+		dev_err(dev->devc, "Dropped Rx msg - no request\n");
+		dev->ops->ishtp_read(dev, rd_msg_buf, ishtp_hdr->length);
+		goto	eoi;
+	}
+
+	if (complete_rb) {
+		getnstimeofday(&cl->ts_rx);
+		++cl->recv_msg_cnt_ipc;
+		ishtp_cl_read_complete(complete_rb);
+	}
+eoi:
+	return;
+}
+
+/**
+ * recv_ishtp_cl_msg_dma() -Receive client message
+ * @dev: ISHTP device instance
+ * @msg: message pointer
+ * @hbm: hbm buffer
+ *
+ * Receive and dispatch ISHTP client messages using DMA. This function executes
+ * in ISR context
+ */
+void recv_ishtp_cl_msg_dma(struct ishtp_device *dev, void *msg,
+			   struct dma_xfer_hbm *hbm)
+{
+	struct ishtp_cl *cl;
+	struct ishtp_cl_rb *rb;
+	struct ishtp_cl_rb *new_rb;
+	unsigned char *buffer = NULL;
+	struct ishtp_cl_rb *complete_rb = NULL;
+	unsigned long	dev_flags;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);
+	list_for_each_entry(rb, &dev->read_list.list, list) {
+		cl = rb->cl;
+		if (!cl || !(cl->host_client_id == hbm->host_client_id &&
+				cl->fw_client_id == hbm->fw_client_id) ||
+				!(cl->state == ISHTP_CL_CONNECTED))
+			continue;
+
+		/*
+		 * If no Rx buffer is allocated, disband the rb
+		 */
+		if (rb->buffer.size == 0 || rb->buffer.data == NULL) {
+			spin_unlock_irqrestore(&dev->read_list_spinlock,
+				dev_flags);
+			dev_err(&cl->device->dev,
+				"response buffer is not allocated.\n");
+			list_del(&rb->list);
+			ishtp_io_rb_free(rb);
+			cl->status = -ENOMEM;
+			goto	eoi;
+		}
+
+		/*
+		 * If message buffer overflown (exceeds max. client msg
+		 * size, drop message and return to free buffer.
+		 * Do we need to disconnect such a client? (We don't send
+		 * back FC, so communication will be stuck anyway)
+		 */
+		if (rb->buffer.size < hbm->msg_length) {
+			spin_unlock_irqrestore(&dev->read_list_spinlock,
+				dev_flags);
+			dev_err(&cl->device->dev,
+				"message overflow. size %d len %d idx %ld\n",
+				rb->buffer.size, hbm->msg_length, rb->buf_idx);
+			list_del(&rb->list);
+			ishtp_cl_io_rb_recycle(rb);
+			cl->status = -EIO;
+			goto	eoi;
+		}
+
+		buffer = rb->buffer.data;
+		memcpy(buffer, msg, hbm->msg_length);
+		rb->buf_idx = hbm->msg_length;
+
+		/* Last fragment in message - it's complete */
+		cl->status = 0;
+		list_del(&rb->list);
+		complete_rb = rb;
+
+		--cl->out_flow_ctrl_creds;
+		/*
+		 * the whole msg arrived, send a new FC, and add a new
+		 * rb buffer for the next coming msg
+		 */
+		spin_lock_irqsave(&cl->free_list_spinlock, flags);
+
+		if (!list_empty(&cl->free_rb_list.list)) {
+			new_rb = list_entry(cl->free_rb_list.list.next,
+				struct ishtp_cl_rb, list);
+			list_del_init(&new_rb->list);
+			spin_unlock_irqrestore(&cl->free_list_spinlock,
+				flags);
+			new_rb->cl = cl;
+			new_rb->buf_idx = 0;
+			INIT_LIST_HEAD(&new_rb->list);
+			list_add_tail(&new_rb->list,
+				&dev->read_list.list);
+
+			ishtp_hbm_cl_flow_control_req(dev, cl);
+		} else {
+			spin_unlock_irqrestore(&cl->free_list_spinlock,
+				flags);
+		}
+
+		/* One more fragment in message (this is always last) */
+		++cl->recv_msg_num_frags;
+
+		/*
+		 * We can safely break here (and in BH too),
+		 * a single input message can go only to a single request!
+		 */
+		break;
+	}
+
+	spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);
+	/* If it's nobody's message, just read and discard it */
+	if (!buffer) {
+		dev_err(dev->devc, "Dropped Rx (DMA) msg - no request\n");
+		goto	eoi;
+	}
+
+	if (complete_rb) {
+		getnstimeofday(&cl->ts_rx);
+		++cl->recv_msg_cnt_dma;
+		ishtp_cl_read_complete(complete_rb);
+	}
+eoi:
+	return;
+}
diff --git a/drivers/hid/intel-ish-hid/ishtp/client.h b/drivers/hid/intel-ish-hid/ishtp/client.h
new file mode 100644
index 000000000000..444d069c2ed4
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/client.h
@@ -0,0 +1,182 @@
+/*
+ * ISHTP client logic
+ *
+ * Copyright (c) 2003-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _ISHTP_CLIENT_H_
+#define _ISHTP_CLIENT_H_
+
+#include <linux/types.h>
+#include "ishtp-dev.h"
+
+/* Client state */
+enum cl_state {
+	ISHTP_CL_INITIALIZING = 0,
+	ISHTP_CL_CONNECTING,
+	ISHTP_CL_CONNECTED,
+	ISHTP_CL_DISCONNECTING,
+	ISHTP_CL_DISCONNECTED
+};
+
+/* Tx and Rx ring size */
+#define	CL_DEF_RX_RING_SIZE	2
+#define	CL_DEF_TX_RING_SIZE	2
+#define	CL_MAX_RX_RING_SIZE	32
+#define	CL_MAX_TX_RING_SIZE	32
+
+#define DMA_SLOT_SIZE		4096
+/* Number of IPC fragments after which it's worth sending via DMA */
+#define	DMA_WORTH_THRESHOLD	3
+
+/* DMA/IPC Tx paths. Other the default means enforcement */
+#define	CL_TX_PATH_DEFAULT	0
+#define	CL_TX_PATH_IPC		1
+#define	CL_TX_PATH_DMA		2
+
+/* Client Tx buffer list entry */
+struct ishtp_cl_tx_ring {
+	struct list_head	list;
+	struct ishtp_msg_data	send_buf;
+};
+
+/* ISHTP client instance */
+struct ishtp_cl {
+	struct list_head	link;
+	struct ishtp_device	*dev;
+	enum cl_state		state;
+	int			status;
+
+	/* Link to ISHTP bus device */
+	struct ishtp_cl_device	*device;
+
+	/* ID of client connected */
+	uint8_t	host_client_id;
+	uint8_t	fw_client_id;
+	uint8_t	ishtp_flow_ctrl_creds;
+	uint8_t	out_flow_ctrl_creds;
+
+	/* dma */
+	int	last_tx_path;
+	/* 0: ack wasn't received,1:ack was received */
+	int	last_dma_acked;
+	unsigned char	*last_dma_addr;
+	/* 0: ack wasn't received,1:ack was received */
+	int	last_ipc_acked;
+
+	/* Rx ring buffer pool */
+	unsigned int	rx_ring_size;
+	struct ishtp_cl_rb	free_rb_list;
+	spinlock_t	free_list_spinlock;
+	/* Rx in-process list */
+	struct ishtp_cl_rb	in_process_list;
+	spinlock_t	in_process_spinlock;
+
+	/* Client Tx buffers list */
+	unsigned int	tx_ring_size;
+	struct ishtp_cl_tx_ring	tx_list, tx_free_list;
+	spinlock_t	tx_list_spinlock;
+	spinlock_t	tx_free_list_spinlock;
+	size_t	tx_offs;	/* Offset in buffer at head of 'tx_list' */
+
+	/**
+	 * if we get a FC, and the list is not empty, we must know whether we
+	 * are at the middle of sending.
+	 * if so -need to increase FC counter, otherwise, need to start sending
+	 * the first msg in list
+	 * (!)This is for counting-FC implementation only. Within single-FC the
+	 * other party may NOT send FC until it receives complete message
+	 */
+	int	sending;
+
+	/* Send FC spinlock */
+	spinlock_t	fc_spinlock;
+
+	/* wait queue for connect and disconnect response from FW */
+	wait_queue_head_t	wait_ctrl_res;
+
+	/* Error stats */
+	unsigned int	err_send_msg;
+	unsigned int	err_send_fc;
+
+	/* Send/recv stats */
+	unsigned int	send_msg_cnt_ipc;
+	unsigned int	send_msg_cnt_dma;
+	unsigned int	recv_msg_cnt_ipc;
+	unsigned int	recv_msg_cnt_dma;
+	unsigned int	recv_msg_num_frags;
+	unsigned int	ishtp_flow_ctrl_cnt;
+	unsigned int	out_flow_ctrl_cnt;
+
+	/* Rx msg ... out FC timing */
+	struct timespec ts_rx;
+	struct timespec ts_out_fc;
+	struct timespec ts_max_fc_delay;
+	void *client_data;
+};
+
+/* Client connection managenment internal functions */
+int ishtp_can_client_connect(struct ishtp_device *ishtp_dev, uuid_le *uuid);
+int ishtp_fw_cl_by_id(struct ishtp_device *dev, uint8_t client_id);
+void ishtp_cl_send_msg(struct ishtp_device *dev, struct ishtp_cl *cl);
+void recv_ishtp_cl_msg(struct ishtp_device *dev,
+		       struct ishtp_msg_hdr *ishtp_hdr);
+int ishtp_cl_read_start(struct ishtp_cl *cl);
+
+/* Ring Buffer I/F */
+int ishtp_cl_alloc_rx_ring(struct ishtp_cl *cl);
+int ishtp_cl_alloc_tx_ring(struct ishtp_cl *cl);
+void ishtp_cl_free_rx_ring(struct ishtp_cl *cl);
+void ishtp_cl_free_tx_ring(struct ishtp_cl *cl);
+
+/* DMA I/F functions */
+void recv_ishtp_cl_msg_dma(struct ishtp_device *dev, void *msg,
+			   struct dma_xfer_hbm *hbm);
+void ishtp_cl_alloc_dma_buf(struct ishtp_device *dev);
+void ishtp_cl_free_dma_buf(struct ishtp_device *dev);
+void *ishtp_cl_get_dma_send_buf(struct ishtp_device *dev,
+				uint32_t size);
+void ishtp_cl_release_dma_acked_mem(struct ishtp_device *dev,
+				    void *msg_addr,
+				    uint8_t size);
+
+/* Request blocks alloc/free I/F */
+struct ishtp_cl_rb *ishtp_io_rb_init(struct ishtp_cl *cl);
+void ishtp_io_rb_free(struct ishtp_cl_rb *priv_rb);
+int ishtp_io_rb_alloc_buf(struct ishtp_cl_rb *rb, size_t length);
+
+/**
+ * ishtp_cl_cmp_id - tells if file private data have same id
+ * returns true  - if ids are the same and not NULL
+ */
+static inline bool ishtp_cl_cmp_id(const struct ishtp_cl *cl1,
+				   const struct ishtp_cl *cl2)
+{
+	return cl1 && cl2 &&
+		(cl1->host_client_id == cl2->host_client_id) &&
+		(cl1->fw_client_id == cl2->fw_client_id);
+}
+
+/* exported functions from ISHTP under client management scope */
+struct ishtp_cl	*ishtp_cl_allocate(struct ishtp_device *dev);
+void ishtp_cl_free(struct ishtp_cl *cl);
+int ishtp_cl_link(struct ishtp_cl *cl, int id);
+void ishtp_cl_unlink(struct ishtp_cl *cl);
+int ishtp_cl_disconnect(struct ishtp_cl *cl);
+int ishtp_cl_connect(struct ishtp_cl *cl);
+int ishtp_cl_send(struct ishtp_cl *cl, uint8_t *buf, size_t length);
+int ishtp_cl_flush_queues(struct ishtp_cl *cl);
+
+/* exported functions from ISHTP client buffer management scope */
+int ishtp_cl_io_rb_recycle(struct ishtp_cl_rb *rb);
+
+#endif /* _ISHTP_CLIENT_H_ */
diff --git a/drivers/hid/intel-ish-hid/ishtp/dma-if.c b/drivers/hid/intel-ish-hid/ishtp/dma-if.c
new file mode 100644
index 000000000000..2783f3666114
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/dma-if.c
@@ -0,0 +1,175 @@
+/*
+ * ISHTP DMA I/F functions
+ *
+ * Copyright (c) 2003-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include "ishtp-dev.h"
+#include "client.h"
+
+/**
+ * ishtp_cl_alloc_dma_buf() - Allocate DMA RX and TX buffer
+ * @dev: ishtp device
+ *
+ * Allocate RX and TX DMA buffer once during bus setup.
+ * It allocates 1MB, RX and TX DMA buffer, which are divided
+ * into slots.
+ */
+void	ishtp_cl_alloc_dma_buf(struct ishtp_device *dev)
+{
+	dma_addr_t	h;
+
+	if (dev->ishtp_host_dma_tx_buf)
+		return;
+
+	dev->ishtp_host_dma_tx_buf_size = 1024*1024;
+	dev->ishtp_host_dma_rx_buf_size = 1024*1024;
+
+	/* Allocate Tx buffer and init usage bitmap */
+	dev->ishtp_host_dma_tx_buf = dma_alloc_coherent(dev->devc,
+					dev->ishtp_host_dma_tx_buf_size,
+					&h, GFP_KERNEL);
+	if (dev->ishtp_host_dma_tx_buf)
+		dev->ishtp_host_dma_tx_buf_phys = h;
+
+	dev->ishtp_dma_num_slots = dev->ishtp_host_dma_tx_buf_size /
+						DMA_SLOT_SIZE;
+
+	dev->ishtp_dma_tx_map = kcalloc(dev->ishtp_dma_num_slots,
+					sizeof(uint8_t),
+					GFP_KERNEL);
+	spin_lock_init(&dev->ishtp_dma_tx_lock);
+
+	/* Allocate Rx buffer */
+	dev->ishtp_host_dma_rx_buf = dma_alloc_coherent(dev->devc,
+					dev->ishtp_host_dma_rx_buf_size,
+					 &h, GFP_KERNEL);
+
+	if (dev->ishtp_host_dma_rx_buf)
+		dev->ishtp_host_dma_rx_buf_phys = h;
+}
+
+/**
+ * ishtp_cl_free_dma_buf() - Free DMA RX and TX buffer
+ * @dev: ishtp device
+ *
+ * Free DMA buffer when all clients are released. This is
+ * only happens during error path in ISH built in driver
+ * model
+ */
+void	ishtp_cl_free_dma_buf(struct ishtp_device *dev)
+{
+	dma_addr_t	h;
+
+	if (dev->ishtp_host_dma_tx_buf) {
+		h = dev->ishtp_host_dma_tx_buf_phys;
+		dma_free_coherent(dev->devc, dev->ishtp_host_dma_tx_buf_size,
+				  dev->ishtp_host_dma_tx_buf, h);
+	}
+
+	if (dev->ishtp_host_dma_rx_buf) {
+		h = dev->ishtp_host_dma_rx_buf_phys;
+		dma_free_coherent(dev->devc, dev->ishtp_host_dma_rx_buf_size,
+				  dev->ishtp_host_dma_rx_buf, h);
+	}
+
+	kfree(dev->ishtp_dma_tx_map);
+	dev->ishtp_host_dma_tx_buf = NULL;
+	dev->ishtp_host_dma_rx_buf = NULL;
+	dev->ishtp_dma_tx_map = NULL;
+}
+
+/*
+ * ishtp_cl_get_dma_send_buf() - Get a DMA memory slot
+ * @dev:	ishtp device
+ * @size:	Size of memory to get
+ *
+ * Find and return free address of "size" bytes in dma tx buffer.
+ * the function will mark this address as "in-used" memory.
+ *
+ * Return: NULL when no free buffer else a buffer to copy
+ */
+void *ishtp_cl_get_dma_send_buf(struct ishtp_device *dev,
+				uint32_t size)
+{
+	unsigned long	flags;
+	int i, j, free;
+	/* additional slot is needed if there is rem */
+	int required_slots = (size / DMA_SLOT_SIZE)
+		+ 1 * (size % DMA_SLOT_SIZE != 0);
+
+	spin_lock_irqsave(&dev->ishtp_dma_tx_lock, flags);
+	for (i = 0; i <= (dev->ishtp_dma_num_slots - required_slots); i++) {
+		free = 1;
+		for (j = 0; j < required_slots; j++)
+			if (dev->ishtp_dma_tx_map[i+j]) {
+				free = 0;
+				i += j;
+				break;
+			}
+		if (free) {
+			/* mark memory as "caught" */
+			for (j = 0; j < required_slots; j++)
+				dev->ishtp_dma_tx_map[i+j] = 1;
+			spin_unlock_irqrestore(&dev->ishtp_dma_tx_lock, flags);
+			return (i * DMA_SLOT_SIZE) +
+				(unsigned char *)dev->ishtp_host_dma_tx_buf;
+		}
+	}
+	spin_unlock_irqrestore(&dev->ishtp_dma_tx_lock, flags);
+	dev_err(dev->devc, "No free DMA buffer to send msg\n");
+	return NULL;
+}
+
+/*
+ * ishtp_cl_release_dma_acked_mem() - Release DMA memory slot
+ * @dev:	ishtp device
+ * @msg_addr:	message address of slot
+ * @size:	Size of memory to get
+ *
+ * Release_dma_acked_mem - returnes the acked memory to free list.
+ * (from msg_addr, size bytes long)
+ */
+void ishtp_cl_release_dma_acked_mem(struct ishtp_device *dev,
+				    void *msg_addr,
+				    uint8_t size)
+{
+	unsigned long	flags;
+	int acked_slots = (size / DMA_SLOT_SIZE)
+		+ 1 * (size % DMA_SLOT_SIZE != 0);
+	int i, j;
+
+	if ((msg_addr - dev->ishtp_host_dma_tx_buf) % DMA_SLOT_SIZE) {
+		dev_err(dev->devc, "Bad DMA Tx ack address\n");
+		return;
+	}
+
+	i = (msg_addr - dev->ishtp_host_dma_tx_buf) / DMA_SLOT_SIZE;
+	spin_lock_irqsave(&dev->ishtp_dma_tx_lock, flags);
+	for (j = 0; j < acked_slots; j++) {
+		if ((i + j) >= dev->ishtp_dma_num_slots ||
+					!dev->ishtp_dma_tx_map[i+j]) {
+			/* no such slot, or memory is already free */
+			spin_unlock_irqrestore(&dev->ishtp_dma_tx_lock, flags);
+			dev_err(dev->devc, "Bad DMA Tx ack address\n");
+			return;
+		}
+		dev->ishtp_dma_tx_map[i+j] = 0;
+	}
+	spin_unlock_irqrestore(&dev->ishtp_dma_tx_lock, flags);
+}
diff --git a/drivers/hid/intel-ish-hid/ishtp/hbm.c b/drivers/hid/intel-ish-hid/ishtp/hbm.c
new file mode 100644
index 000000000000..74bffee60774
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/hbm.c
@@ -0,0 +1,1032 @@
+/*
+ * ISHTP bus layer messages handling
+ *
+ * Copyright (c) 2003-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/spinlock.h>
+#include <linux/miscdevice.h>
+#include "ishtp-dev.h"
+#include "hbm.h"
+#include "client.h"
+
+/**
+ * ishtp_hbm_fw_cl_allocate() - Allocate FW clients
+ * @dev: ISHTP device instance
+ *
+ * Allocates storage for fw clients
+ */
+static void ishtp_hbm_fw_cl_allocate(struct ishtp_device *dev)
+{
+	struct ishtp_fw_client *clients;
+	int b;
+
+	/* count how many ISH clients we have */
+	for_each_set_bit(b, dev->fw_clients_map, ISHTP_CLIENTS_MAX)
+		dev->fw_clients_num++;
+
+	if (dev->fw_clients_num <= 0)
+		return;
+
+	/* allocate storage for fw clients representation */
+	clients = kcalloc(dev->fw_clients_num, sizeof(struct ishtp_fw_client),
+			  GFP_KERNEL);
+	if (!clients) {
+		dev->dev_state = ISHTP_DEV_RESETTING;
+		ish_hw_reset(dev);
+		return;
+	}
+	dev->fw_clients = clients;
+}
+
+/**
+ * ishtp_hbm_cl_hdr() - construct client hbm header
+ * @cl: client
+ * @hbm_cmd: host bus message command
+ * @buf: buffer for cl header
+ * @len: buffer length
+ *
+ * Initialize HBM buffer
+ */
+static inline void ishtp_hbm_cl_hdr(struct ishtp_cl *cl, uint8_t hbm_cmd,
+	void *buf, size_t len)
+{
+	struct ishtp_hbm_cl_cmd *cmd = buf;
+
+	memset(cmd, 0, len);
+
+	cmd->hbm_cmd = hbm_cmd;
+	cmd->host_addr = cl->host_client_id;
+	cmd->fw_addr = cl->fw_client_id;
+}
+
+/**
+ * ishtp_hbm_cl_addr_equal() - Compare client address
+ * @cl: client
+ * @buf: Client command buffer
+ *
+ * Compare client address with the address in command buffer
+ *
+ * Return: True if they have the same address
+ */
+static inline bool ishtp_hbm_cl_addr_equal(struct ishtp_cl *cl, void *buf)
+{
+	struct ishtp_hbm_cl_cmd *cmd = buf;
+
+	return cl->host_client_id == cmd->host_addr &&
+		cl->fw_client_id == cmd->fw_addr;
+}
+
+/**
+ * ishtp_hbm_start_wait() - Wait for HBM start message
+ * @dev: ISHTP device instance
+ *
+ * Wait for HBM start message from firmware
+ *
+ * Return: 0 if HBM start is/was received else timeout error
+ */
+int ishtp_hbm_start_wait(struct ishtp_device *dev)
+{
+	int ret;
+
+	if (dev->hbm_state > ISHTP_HBM_START)
+		return 0;
+
+	dev_dbg(dev->devc, "Going to wait for ishtp start. hbm_state=%08X\n",
+		dev->hbm_state);
+	ret = wait_event_interruptible_timeout(dev->wait_hbm_recvd_msg,
+					dev->hbm_state >= ISHTP_HBM_STARTED,
+					(ISHTP_INTEROP_TIMEOUT * HZ));
+
+	dev_dbg(dev->devc,
+		"Woke up from waiting for ishtp start. hbm_state=%08X\n",
+		dev->hbm_state);
+
+	if (ret <= 0 && (dev->hbm_state <= ISHTP_HBM_START)) {
+		dev->hbm_state = ISHTP_HBM_IDLE;
+		dev_err(dev->devc,
+		"waiting for ishtp start failed. ret=%d hbm_state=%08X\n",
+			ret, dev->hbm_state);
+		return -ETIMEDOUT;
+	}
+	return 0;
+}
+
+/**
+ * ishtp_hbm_start_req() - Send HBM start message
+ * @dev: ISHTP device instance
+ *
+ * Send HBM start message to firmware
+ *
+ * Return: 0 if success else error code
+ */
+int ishtp_hbm_start_req(struct ishtp_device *dev)
+{
+	struct ishtp_msg_hdr hdr;
+	unsigned char data[128];
+	struct ishtp_msg_hdr *ishtp_hdr = &hdr;
+	struct hbm_host_version_request *start_req;
+	const size_t len = sizeof(struct hbm_host_version_request);
+
+	ishtp_hbm_hdr(ishtp_hdr, len);
+
+	/* host start message */
+	start_req = (struct hbm_host_version_request *)data;
+	memset(start_req, 0, len);
+	start_req->hbm_cmd = HOST_START_REQ_CMD;
+	start_req->host_version.major_version = HBM_MAJOR_VERSION;
+	start_req->host_version.minor_version = HBM_MINOR_VERSION;
+
+	/*
+	 * (!) Response to HBM start may be so quick that this thread would get
+	 * preempted BEFORE managing to set hbm_state = ISHTP_HBM_START.
+	 * So set it at first, change back to ISHTP_HBM_IDLE upon failure
+	 */
+	dev->hbm_state = ISHTP_HBM_START;
+	if (ishtp_write_message(dev, ishtp_hdr, data)) {
+		dev_err(dev->devc, "version message send failed\n");
+		dev->dev_state = ISHTP_DEV_RESETTING;
+		dev->hbm_state = ISHTP_HBM_IDLE;
+		ish_hw_reset(dev);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+/**
+ * ishtp_hbm_enum_clients_req() - Send client enum req
+ * @dev: ISHTP device instance
+ *
+ * Send enumeration client request message
+ *
+ * Return: 0 if success else error code
+ */
+void ishtp_hbm_enum_clients_req(struct ishtp_device *dev)
+{
+	struct ishtp_msg_hdr hdr;
+	unsigned char data[128];
+	struct ishtp_msg_hdr *ishtp_hdr = &hdr;
+	struct hbm_host_enum_request *enum_req;
+	const size_t len = sizeof(struct hbm_host_enum_request);
+
+	/* enumerate clients */
+	ishtp_hbm_hdr(ishtp_hdr, len);
+
+	enum_req = (struct hbm_host_enum_request *)data;
+	memset(enum_req, 0, len);
+	enum_req->hbm_cmd = HOST_ENUM_REQ_CMD;
+
+	if (ishtp_write_message(dev, ishtp_hdr, data)) {
+		dev->dev_state = ISHTP_DEV_RESETTING;
+		dev_err(dev->devc, "enumeration request send failed\n");
+		ish_hw_reset(dev);
+	}
+	dev->hbm_state = ISHTP_HBM_ENUM_CLIENTS;
+}
+
+/**
+ * ishtp_hbm_prop_req() - Request property
+ * @dev: ISHTP device instance
+ *
+ * Request property for a single client
+ *
+ * Return: 0 if success else error code
+ */
+static int ishtp_hbm_prop_req(struct ishtp_device *dev)
+{
+
+	struct ishtp_msg_hdr hdr;
+	unsigned char data[128];
+	struct ishtp_msg_hdr *ishtp_hdr = &hdr;
+	struct hbm_props_request *prop_req;
+	const size_t len = sizeof(struct hbm_props_request);
+	unsigned long next_client_index;
+	uint8_t client_num;
+
+	client_num = dev->fw_client_presentation_num;
+
+	next_client_index = find_next_bit(dev->fw_clients_map,
+		ISHTP_CLIENTS_MAX, dev->fw_client_index);
+
+	/* We got all client properties */
+	if (next_client_index == ISHTP_CLIENTS_MAX) {
+		dev->hbm_state = ISHTP_HBM_WORKING;
+		dev->dev_state = ISHTP_DEV_ENABLED;
+
+		for (dev->fw_client_presentation_num = 1;
+			dev->fw_client_presentation_num < client_num + 1;
+				++dev->fw_client_presentation_num)
+			/* Add new client device */
+			ishtp_bus_new_client(dev);
+		return 0;
+	}
+
+	dev->fw_clients[client_num].client_id = next_client_index;
+
+	ishtp_hbm_hdr(ishtp_hdr, len);
+	prop_req = (struct hbm_props_request *)data;
+
+	memset(prop_req, 0, sizeof(struct hbm_props_request));
+
+	prop_req->hbm_cmd = HOST_CLIENT_PROPERTIES_REQ_CMD;
+	prop_req->address = next_client_index;
+
+	if (ishtp_write_message(dev, ishtp_hdr, data)) {
+		dev->dev_state = ISHTP_DEV_RESETTING;
+		dev_err(dev->devc, "properties request send failed\n");
+		ish_hw_reset(dev);
+		return -EIO;
+	}
+
+	dev->fw_client_index = next_client_index;
+
+	return 0;
+}
+
+/**
+ * ishtp_hbm_stop_req() - Send HBM stop
+ * @dev: ISHTP device instance
+ *
+ * Send stop request message
+ */
+static void ishtp_hbm_stop_req(struct ishtp_device *dev)
+{
+	struct ishtp_msg_hdr hdr;
+	unsigned char data[128];
+	struct ishtp_msg_hdr *ishtp_hdr = &hdr;
+	struct hbm_host_stop_request *req;
+	const size_t len = sizeof(struct hbm_host_stop_request);
+
+	ishtp_hbm_hdr(ishtp_hdr, len);
+	req = (struct hbm_host_stop_request *)data;
+
+	memset(req, 0, sizeof(struct hbm_host_stop_request));
+	req->hbm_cmd = HOST_STOP_REQ_CMD;
+	req->reason = DRIVER_STOP_REQUEST;
+
+	ishtp_write_message(dev, ishtp_hdr, data);
+}
+
+/**
+ * ishtp_hbm_cl_flow_control_req() - Send flow control request
+ * @dev: ISHTP device instance
+ * @cl: ISHTP client instance
+ *
+ * Send flow control request
+ *
+ * Return: 0 if success else error code
+ */
+int ishtp_hbm_cl_flow_control_req(struct ishtp_device *dev,
+				  struct ishtp_cl *cl)
+{
+	struct ishtp_msg_hdr hdr;
+	unsigned char data[128];
+	struct ishtp_msg_hdr *ishtp_hdr = &hdr;
+	const size_t len = sizeof(struct hbm_flow_control);
+	int	rv;
+	unsigned int	num_frags;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&cl->fc_spinlock, flags);
+	ishtp_hbm_hdr(ishtp_hdr, len);
+	ishtp_hbm_cl_hdr(cl, ISHTP_FLOW_CONTROL_CMD, data, len);
+
+	/*
+	 * Sync possible race when RB recycle and packet receive paths
+	 * both try to send an out FC
+	 */
+	if (cl->out_flow_ctrl_creds) {
+		spin_unlock_irqrestore(&cl->fc_spinlock, flags);
+		return	0;
+	}
+
+	num_frags = cl->recv_msg_num_frags;
+	cl->recv_msg_num_frags = 0;
+
+	rv = ishtp_write_message(dev, ishtp_hdr, data);
+	if (!rv) {
+		++cl->out_flow_ctrl_creds;
+		++cl->out_flow_ctrl_cnt;
+		getnstimeofday(&cl->ts_out_fc);
+		if (cl->ts_rx.tv_sec && cl->ts_rx.tv_nsec) {
+			struct timespec ts_diff;
+
+			ts_diff = timespec_sub(cl->ts_out_fc, cl->ts_rx);
+			if (timespec_compare(&ts_diff, &cl->ts_max_fc_delay)
+					> 0)
+				cl->ts_max_fc_delay = ts_diff;
+		}
+	} else {
+		++cl->err_send_fc;
+	}
+
+	spin_unlock_irqrestore(&cl->fc_spinlock, flags);
+	return	rv;
+}
+
+/**
+ * ishtp_hbm_cl_disconnect_req() - Send disconnect request
+ * @dev: ISHTP device instance
+ * @cl: ISHTP client instance
+ *
+ * Send disconnect message to fw
+ *
+ * Return: 0 if success else error code
+ */
+int ishtp_hbm_cl_disconnect_req(struct ishtp_device *dev, struct ishtp_cl *cl)
+{
+	struct ishtp_msg_hdr hdr;
+	unsigned char data[128];
+	struct ishtp_msg_hdr *ishtp_hdr = &hdr;
+	const size_t len = sizeof(struct hbm_client_connect_request);
+
+	ishtp_hbm_hdr(ishtp_hdr, len);
+	ishtp_hbm_cl_hdr(cl, CLIENT_DISCONNECT_REQ_CMD, data, len);
+
+	return ishtp_write_message(dev, ishtp_hdr, data);
+}
+
+/**
+ * ishtp_hbm_cl_disconnect_res() - Get disconnect response
+ * @dev: ISHTP device instance
+ * @rs: Response message
+ *
+ * Received disconnect response from fw
+ */
+static void ishtp_hbm_cl_disconnect_res(struct ishtp_device *dev,
+	struct hbm_client_connect_response *rs)
+{
+	struct ishtp_cl *cl = NULL;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&dev->cl_list_lock, flags);
+	list_for_each_entry(cl, &dev->cl_list, link) {
+		if (!rs->status && ishtp_hbm_cl_addr_equal(cl, rs)) {
+			cl->state = ISHTP_CL_DISCONNECTED;
+			break;
+		}
+	}
+	if (cl)
+		wake_up_interruptible(&cl->wait_ctrl_res);
+	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
+}
+
+/**
+ * ishtp_hbm_cl_connect_req() - Send connect request
+ * @dev: ISHTP device instance
+ * @cl: client device instance
+ *
+ * Send connection request to specific fw client
+ *
+ * Return: 0 if success else error code
+ */
+int ishtp_hbm_cl_connect_req(struct ishtp_device *dev, struct ishtp_cl *cl)
+{
+	struct ishtp_msg_hdr hdr;
+	unsigned char data[128];
+	struct ishtp_msg_hdr *ishtp_hdr = &hdr;
+	const size_t len = sizeof(struct hbm_client_connect_request);
+
+	ishtp_hbm_hdr(ishtp_hdr, len);
+	ishtp_hbm_cl_hdr(cl, CLIENT_CONNECT_REQ_CMD, data, len);
+
+	return ishtp_write_message(dev, ishtp_hdr, data);
+}
+
+/**
+ * ishtp_hbm_cl_connect_res() - Get connect response
+ * @dev: ISHTP device instance
+ * @rs: Response message
+ *
+ * Received connect response from fw
+ */
+static void ishtp_hbm_cl_connect_res(struct ishtp_device *dev,
+	struct hbm_client_connect_response *rs)
+{
+	struct ishtp_cl *cl = NULL;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&dev->cl_list_lock, flags);
+	list_for_each_entry(cl, &dev->cl_list, link) {
+		if (ishtp_hbm_cl_addr_equal(cl, rs)) {
+			if (!rs->status) {
+				cl->state = ISHTP_CL_CONNECTED;
+				cl->status = 0;
+			} else {
+				cl->state = ISHTP_CL_DISCONNECTED;
+				cl->status = -ENODEV;
+			}
+			break;
+		}
+	}
+	if (cl)
+		wake_up_interruptible(&cl->wait_ctrl_res);
+	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
+}
+
+/**
+ * ishtp_client_disconnect_request() - Receive disconnect request
+ * @dev: ISHTP device instance
+ * @disconnect_req: disconnect request structure
+ *
+ * Disconnect request bus message from the fw. Send diconnect response.
+ */
+static void ishtp_hbm_fw_disconnect_req(struct ishtp_device *dev,
+	struct hbm_client_connect_request *disconnect_req)
+{
+	struct ishtp_cl *cl;
+	const size_t len = sizeof(struct hbm_client_connect_response);
+	unsigned long	flags;
+	struct ishtp_msg_hdr hdr;
+	unsigned char data[4];	/* All HBM messages are 4 bytes */
+
+	spin_lock_irqsave(&dev->cl_list_lock, flags);
+	list_for_each_entry(cl, &dev->cl_list, link) {
+		if (ishtp_hbm_cl_addr_equal(cl, disconnect_req)) {
+			cl->state = ISHTP_CL_DISCONNECTED;
+
+			/* send disconnect response */
+			ishtp_hbm_hdr(&hdr, len);
+			ishtp_hbm_cl_hdr(cl, CLIENT_DISCONNECT_RES_CMD, data,
+				len);
+			ishtp_write_message(dev, &hdr, data);
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
+}
+
+/**
+ * ishtp_hbm_dma_xfer_ack(() - Receive transfer ACK
+ * @dev: ISHTP device instance
+ * @dma_xfer: HBM transfer message
+ *
+ * Receive ack for ISHTP-over-DMA client message
+ */
+static void ishtp_hbm_dma_xfer_ack(struct ishtp_device *dev,
+				   struct dma_xfer_hbm *dma_xfer)
+{
+	void	*msg;
+	uint64_t	offs;
+	struct ishtp_msg_hdr	*ishtp_hdr =
+		(struct ishtp_msg_hdr *)&dev->ishtp_msg_hdr;
+	unsigned int	msg_offs;
+	struct ishtp_cl *cl;
+
+	for (msg_offs = 0; msg_offs < ishtp_hdr->length;
+		msg_offs += sizeof(struct dma_xfer_hbm)) {
+		offs = dma_xfer->msg_addr - dev->ishtp_host_dma_tx_buf_phys;
+		if (offs > dev->ishtp_host_dma_tx_buf_size) {
+			dev_err(dev->devc, "Bad DMA Tx ack message address\n");
+			return;
+		}
+		if (dma_xfer->msg_length >
+				dev->ishtp_host_dma_tx_buf_size - offs) {
+			dev_err(dev->devc, "Bad DMA Tx ack message size\n");
+			return;
+		}
+
+		/* logical address of the acked mem */
+		msg = (unsigned char *)dev->ishtp_host_dma_tx_buf + offs;
+		ishtp_cl_release_dma_acked_mem(dev, msg, dma_xfer->msg_length);
+
+		list_for_each_entry(cl, &dev->cl_list, link) {
+			if (cl->fw_client_id == dma_xfer->fw_client_id &&
+			    cl->host_client_id == dma_xfer->host_client_id)
+				/*
+				 * in case that a single ack may be sent
+				 * over several dma transfers, and the last msg
+				 * addr was inside the acked memory, but not in
+				 * its start
+				 */
+				if (cl->last_dma_addr >=
+							(unsigned char *)msg &&
+						cl->last_dma_addr <
+						(unsigned char *)msg +
+						dma_xfer->msg_length) {
+					cl->last_dma_acked = 1;
+
+					if (!list_empty(&cl->tx_list.list) &&
+						cl->ishtp_flow_ctrl_creds) {
+						/*
+						 * start sending the first msg
+						 */
+						ishtp_cl_send_msg(dev, cl);
+					}
+				}
+		}
+		++dma_xfer;
+	}
+}
+
+/**
+ * ishtp_hbm_dma_xfer() - Receive DMA transfer message
+ * @dev: ISHTP device instance
+ * @dma_xfer: HBM transfer message
+ *
+ * Receive ISHTP-over-DMA client message
+ */
+static void ishtp_hbm_dma_xfer(struct ishtp_device *dev,
+			       struct dma_xfer_hbm *dma_xfer)
+{
+	void	*msg;
+	uint64_t	offs;
+	struct ishtp_msg_hdr	hdr;
+	struct ishtp_msg_hdr	*ishtp_hdr =
+		(struct ishtp_msg_hdr *) &dev->ishtp_msg_hdr;
+	struct dma_xfer_hbm	*prm = dma_xfer;
+	unsigned int	msg_offs;
+
+	for (msg_offs = 0; msg_offs < ishtp_hdr->length;
+		msg_offs += sizeof(struct dma_xfer_hbm)) {
+
+		offs = dma_xfer->msg_addr - dev->ishtp_host_dma_rx_buf_phys;
+		if (offs > dev->ishtp_host_dma_rx_buf_size) {
+			dev_err(dev->devc, "Bad DMA Rx message address\n");
+			return;
+		}
+		if (dma_xfer->msg_length >
+				dev->ishtp_host_dma_rx_buf_size - offs) {
+			dev_err(dev->devc, "Bad DMA Rx message size\n");
+			return;
+		}
+		msg = dev->ishtp_host_dma_rx_buf + offs;
+		recv_ishtp_cl_msg_dma(dev, msg, dma_xfer);
+		dma_xfer->hbm = DMA_XFER_ACK;	/* Prepare for response */
+		++dma_xfer;
+	}
+
+	/* Send DMA_XFER_ACK [...] */
+	ishtp_hbm_hdr(&hdr, ishtp_hdr->length);
+	ishtp_write_message(dev, &hdr, (unsigned char *)prm);
+}
+
+/**
+ * ishtp_hbm_dispatch() - HBM dispatch function
+ * @dev: ISHTP device instance
+ * @hdr: bus message
+ *
+ * Bottom half read routine after ISR to handle the read bus message cmd
+ * processing
+ */
+void ishtp_hbm_dispatch(struct ishtp_device *dev,
+			struct ishtp_bus_message *hdr)
+{
+	struct ishtp_bus_message *ishtp_msg;
+	struct ishtp_fw_client *fw_client;
+	struct hbm_host_version_response *version_res;
+	struct hbm_client_connect_response *connect_res;
+	struct hbm_client_connect_response *disconnect_res;
+	struct hbm_client_connect_request *disconnect_req;
+	struct hbm_props_response *props_res;
+	struct hbm_host_enum_response *enum_res;
+	struct ishtp_msg_hdr ishtp_hdr;
+	struct dma_alloc_notify	dma_alloc_notify;
+	struct dma_xfer_hbm	*dma_xfer;
+
+	ishtp_msg = hdr;
+
+	switch (ishtp_msg->hbm_cmd) {
+	case HOST_START_RES_CMD:
+		version_res = (struct hbm_host_version_response *)ishtp_msg;
+		if (!version_res->host_version_supported) {
+			dev->version = version_res->fw_max_version;
+
+			dev->hbm_state = ISHTP_HBM_STOPPED;
+			ishtp_hbm_stop_req(dev);
+			return;
+		}
+
+		dev->version.major_version = HBM_MAJOR_VERSION;
+		dev->version.minor_version = HBM_MINOR_VERSION;
+		if (dev->dev_state == ISHTP_DEV_INIT_CLIENTS &&
+				dev->hbm_state == ISHTP_HBM_START) {
+			dev->hbm_state = ISHTP_HBM_STARTED;
+			ishtp_hbm_enum_clients_req(dev);
+		} else {
+			dev_err(dev->devc,
+				"reset: wrong host start response\n");
+			/* BUG: why do we arrive here? */
+			ish_hw_reset(dev);
+			return;
+		}
+
+		wake_up_interruptible(&dev->wait_hbm_recvd_msg);
+		break;
+
+	case CLIENT_CONNECT_RES_CMD:
+		connect_res = (struct hbm_client_connect_response *)ishtp_msg;
+		ishtp_hbm_cl_connect_res(dev, connect_res);
+		break;
+
+	case CLIENT_DISCONNECT_RES_CMD:
+		disconnect_res =
+			(struct hbm_client_connect_response *)ishtp_msg;
+		ishtp_hbm_cl_disconnect_res(dev, disconnect_res);
+		break;
+
+	case HOST_CLIENT_PROPERTIES_RES_CMD:
+		props_res = (struct hbm_props_response *)ishtp_msg;
+		fw_client = &dev->fw_clients[dev->fw_client_presentation_num];
+
+		if (props_res->status || !dev->fw_clients) {
+			dev_err(dev->devc,
+			"reset: properties response hbm wrong status\n");
+			ish_hw_reset(dev);
+			return;
+		}
+
+		if (fw_client->client_id != props_res->address) {
+			dev_err(dev->devc,
+				"reset: host properties response address mismatch [%02X %02X]\n",
+				fw_client->client_id, props_res->address);
+			ish_hw_reset(dev);
+			return;
+		}
+
+		if (dev->dev_state != ISHTP_DEV_INIT_CLIENTS ||
+			dev->hbm_state != ISHTP_HBM_CLIENT_PROPERTIES) {
+			dev_err(dev->devc,
+				"reset: unexpected properties response\n");
+			ish_hw_reset(dev);
+			return;
+		}
+
+		fw_client->props = props_res->client_properties;
+		dev->fw_client_index++;
+		dev->fw_client_presentation_num++;
+
+		/* request property for the next client */
+		ishtp_hbm_prop_req(dev);
+
+		if (dev->dev_state != ISHTP_DEV_ENABLED)
+			break;
+
+		if (!ishtp_use_dma_transfer())
+			break;
+
+		dev_dbg(dev->devc, "Requesting to use DMA\n");
+		ishtp_cl_alloc_dma_buf(dev);
+		if (dev->ishtp_host_dma_rx_buf) {
+			const size_t len = sizeof(dma_alloc_notify);
+
+			memset(&dma_alloc_notify, 0, sizeof(dma_alloc_notify));
+			dma_alloc_notify.hbm = DMA_BUFFER_ALLOC_NOTIFY;
+			dma_alloc_notify.buf_size =
+					dev->ishtp_host_dma_rx_buf_size;
+			dma_alloc_notify.buf_address =
+					dev->ishtp_host_dma_rx_buf_phys;
+			ishtp_hbm_hdr(&ishtp_hdr, len);
+			ishtp_write_message(dev, &ishtp_hdr,
+				(unsigned char *)&dma_alloc_notify);
+		}
+
+		break;
+
+	case HOST_ENUM_RES_CMD:
+		enum_res = (struct hbm_host_enum_response *) ishtp_msg;
+		memcpy(dev->fw_clients_map, enum_res->valid_addresses, 32);
+		if (dev->dev_state == ISHTP_DEV_INIT_CLIENTS &&
+			dev->hbm_state == ISHTP_HBM_ENUM_CLIENTS) {
+			dev->fw_client_presentation_num = 0;
+			dev->fw_client_index = 0;
+
+			ishtp_hbm_fw_cl_allocate(dev);
+			dev->hbm_state = ISHTP_HBM_CLIENT_PROPERTIES;
+
+			/* first property request */
+			ishtp_hbm_prop_req(dev);
+		} else {
+			dev_err(dev->devc,
+			      "reset: unexpected enumeration response hbm\n");
+			ish_hw_reset(dev);
+			return;
+		}
+		break;
+
+	case HOST_STOP_RES_CMD:
+		if (dev->hbm_state != ISHTP_HBM_STOPPED)
+			dev_err(dev->devc, "unexpected stop response\n");
+
+		dev->dev_state = ISHTP_DEV_DISABLED;
+		dev_info(dev->devc, "reset: FW stop response\n");
+		ish_hw_reset(dev);
+		break;
+
+	case CLIENT_DISCONNECT_REQ_CMD:
+		/* search for client */
+		disconnect_req =
+			(struct hbm_client_connect_request *)ishtp_msg;
+		ishtp_hbm_fw_disconnect_req(dev, disconnect_req);
+		break;
+
+	case FW_STOP_REQ_CMD:
+		dev->hbm_state = ISHTP_HBM_STOPPED;
+		break;
+
+	case DMA_BUFFER_ALLOC_RESPONSE:
+		dev->ishtp_host_dma_enabled = 1;
+		break;
+
+	case DMA_XFER:
+		dma_xfer = (struct dma_xfer_hbm *)ishtp_msg;
+		if (!dev->ishtp_host_dma_enabled) {
+			dev_err(dev->devc,
+				"DMA XFER requested but DMA is not enabled\n");
+			break;
+		}
+		ishtp_hbm_dma_xfer(dev, dma_xfer);
+		break;
+
+	case DMA_XFER_ACK:
+		dma_xfer = (struct dma_xfer_hbm *)ishtp_msg;
+		if (!dev->ishtp_host_dma_enabled ||
+		    !dev->ishtp_host_dma_tx_buf) {
+			dev_err(dev->devc,
+				"DMA XFER acked but DMA Tx is not enabled\n");
+			break;
+		}
+		ishtp_hbm_dma_xfer_ack(dev, dma_xfer);
+		break;
+
+	default:
+		dev_err(dev->devc, "unknown HBM: %u\n",
+			(unsigned int)ishtp_msg->hbm_cmd);
+
+		break;
+	}
+}
+
+/**
+ * bh_hbm_work_fn() - HBM work function
+ * @work: work struct
+ *
+ * Bottom half processing work function (instead of thread handler)
+ * for processing hbm messages
+ */
+void	bh_hbm_work_fn(struct work_struct *work)
+{
+	unsigned long	flags;
+	struct ishtp_device	*dev;
+	unsigned char	hbm[IPC_PAYLOAD_SIZE];
+
+	dev = container_of(work, struct ishtp_device, bh_hbm_work);
+	spin_lock_irqsave(&dev->rd_msg_spinlock, flags);
+	if (dev->rd_msg_fifo_head != dev->rd_msg_fifo_tail) {
+		memcpy(hbm, dev->rd_msg_fifo + dev->rd_msg_fifo_head,
+			IPC_PAYLOAD_SIZE);
+		dev->rd_msg_fifo_head =
+			(dev->rd_msg_fifo_head + IPC_PAYLOAD_SIZE) %
+			(RD_INT_FIFO_SIZE * IPC_PAYLOAD_SIZE);
+		spin_unlock_irqrestore(&dev->rd_msg_spinlock, flags);
+		ishtp_hbm_dispatch(dev, (struct ishtp_bus_message *)hbm);
+	} else {
+		spin_unlock_irqrestore(&dev->rd_msg_spinlock, flags);
+	}
+}
+
+/**
+ * recv_hbm() - Receive HBM message
+ * @dev: ISHTP device instance
+ * @ishtp_hdr: received bus message
+ *
+ * Receive and process ISHTP bus messages in ISR context. This will schedule
+ * work function to process message
+ */
+void	recv_hbm(struct ishtp_device *dev, struct ishtp_msg_hdr *ishtp_hdr)
+{
+	uint8_t	rd_msg_buf[ISHTP_RD_MSG_BUF_SIZE];
+	struct ishtp_bus_message	*ishtp_msg =
+		(struct ishtp_bus_message *)rd_msg_buf;
+	unsigned long	flags;
+
+	dev->ops->ishtp_read(dev, rd_msg_buf, ishtp_hdr->length);
+
+	/* Flow control - handle in place */
+	if (ishtp_msg->hbm_cmd == ISHTP_FLOW_CONTROL_CMD) {
+		struct hbm_flow_control *flow_control =
+			(struct hbm_flow_control *)ishtp_msg;
+		struct ishtp_cl *cl = NULL;
+		unsigned long	flags, tx_flags;
+
+		spin_lock_irqsave(&dev->cl_list_lock, flags);
+		list_for_each_entry(cl, &dev->cl_list, link) {
+			if (cl->host_client_id == flow_control->host_addr &&
+					cl->fw_client_id ==
+					flow_control->fw_addr) {
+				/*
+				 * NOTE: It's valid only for counting
+				 * flow-control implementation to receive a
+				 * FC in the middle of sending. Meanwhile not
+				 * supported
+				 */
+				if (cl->ishtp_flow_ctrl_creds)
+					dev_err(dev->devc,
+					 "recv extra FC from FW client %u (host client %u) (FC count was %d)\n",
+					 (unsigned int)cl->fw_client_id,
+					 (unsigned int)cl->host_client_id,
+					 cl->ishtp_flow_ctrl_creds);
+				else {
+					++cl->ishtp_flow_ctrl_creds;
+					++cl->ishtp_flow_ctrl_cnt;
+					cl->last_ipc_acked = 1;
+					spin_lock_irqsave(
+							&cl->tx_list_spinlock,
+							tx_flags);
+					if (!list_empty(&cl->tx_list.list)) {
+						/*
+						 * start sending the first msg
+						 *	= the callback function
+						 */
+						spin_unlock_irqrestore(
+							&cl->tx_list_spinlock,
+							tx_flags);
+						ishtp_cl_send_msg(dev, cl);
+					} else {
+						spin_unlock_irqrestore(
+							&cl->tx_list_spinlock,
+							tx_flags);
+					}
+				}
+				break;
+			}
+		}
+		spin_unlock_irqrestore(&dev->cl_list_lock, flags);
+		goto	eoi;
+	}
+
+	/*
+	 * Some messages that are safe for ISR processing and important
+	 * to be done "quickly" and in-order, go here
+	 */
+	if (ishtp_msg->hbm_cmd == CLIENT_CONNECT_RES_CMD ||
+			ishtp_msg->hbm_cmd == CLIENT_DISCONNECT_RES_CMD ||
+			ishtp_msg->hbm_cmd == CLIENT_DISCONNECT_REQ_CMD ||
+			ishtp_msg->hbm_cmd == DMA_XFER) {
+		ishtp_hbm_dispatch(dev, ishtp_msg);
+		goto	eoi;
+	}
+
+	/*
+	 * All other HBMs go here.
+	 * We schedule HBMs for processing serially by using system wq,
+	 * possibly there will be multiple HBMs scheduled at the same time.
+	 */
+	spin_lock_irqsave(&dev->rd_msg_spinlock, flags);
+	if ((dev->rd_msg_fifo_tail + IPC_PAYLOAD_SIZE) %
+			(RD_INT_FIFO_SIZE * IPC_PAYLOAD_SIZE) ==
+			dev->rd_msg_fifo_head) {
+		spin_unlock_irqrestore(&dev->rd_msg_spinlock, flags);
+		dev_err(dev->devc, "BH buffer overflow, dropping HBM %u\n",
+			(unsigned int)ishtp_msg->hbm_cmd);
+		goto	eoi;
+	}
+	memcpy(dev->rd_msg_fifo + dev->rd_msg_fifo_tail, ishtp_msg,
+		ishtp_hdr->length);
+	dev->rd_msg_fifo_tail = (dev->rd_msg_fifo_tail + IPC_PAYLOAD_SIZE) %
+		(RD_INT_FIFO_SIZE * IPC_PAYLOAD_SIZE);
+	spin_unlock_irqrestore(&dev->rd_msg_spinlock, flags);
+	schedule_work(&dev->bh_hbm_work);
+eoi:
+	return;
+}
+
+/**
+ * recv_fixed_cl_msg() - Receive fixed client message
+ * @dev: ISHTP device instance
+ * @ishtp_hdr: received bus message
+ *
+ * Receive and process ISHTP fixed client messages (address == 0)
+ * in ISR context
+ */
+void recv_fixed_cl_msg(struct ishtp_device *dev,
+	struct ishtp_msg_hdr *ishtp_hdr)
+{
+	uint8_t rd_msg_buf[ISHTP_RD_MSG_BUF_SIZE];
+
+	dev->print_log(dev,
+		"%s() got fixed client msg from client #%d\n",
+		__func__, ishtp_hdr->fw_addr);
+	dev->ops->ishtp_read(dev, rd_msg_buf, ishtp_hdr->length);
+	if (ishtp_hdr->fw_addr == ISHTP_SYSTEM_STATE_CLIENT_ADDR) {
+		struct ish_system_states_header *msg_hdr =
+			(struct ish_system_states_header *)rd_msg_buf;
+		if (msg_hdr->cmd == SYSTEM_STATE_SUBSCRIBE)
+			ishtp_send_resume(dev);
+		/* if FW request arrived here, the system is not suspended */
+		else
+			dev_err(dev->devc, "unknown fixed client msg [%02X]\n",
+				msg_hdr->cmd);
+	}
+}
+
+/**
+ * fix_cl_hdr() - Initialize fixed client header
+ * @hdr: message header
+ * @length: length of message
+ * @cl_addr: Client address
+ *
+ * Initialize message header for fixed client
+ */
+static inline void fix_cl_hdr(struct ishtp_msg_hdr *hdr, size_t length,
+	uint8_t cl_addr)
+{
+	hdr->host_addr = 0;
+	hdr->fw_addr = cl_addr;
+	hdr->length = length;
+	hdr->msg_complete = 1;
+	hdr->reserved = 0;
+}
+
+/*** Suspend and resume notification ***/
+
+static uint32_t current_state;
+static uint32_t supported_states = 0 | SUSPEND_STATE_BIT;
+
+/**
+ * ishtp_send_suspend() - Send suspend message to FW
+ * @dev: ISHTP device instance
+ *
+ * Send suspend message to FW. This is useful for system freeze (non S3) case
+ */
+void ishtp_send_suspend(struct ishtp_device *dev)
+{
+	struct ishtp_msg_hdr	ishtp_hdr;
+	struct ish_system_states_status state_status_msg;
+	const size_t len = sizeof(struct ish_system_states_status);
+
+	fix_cl_hdr(&ishtp_hdr, len, ISHTP_SYSTEM_STATE_CLIENT_ADDR);
+
+	memset(&state_status_msg, 0, len);
+	state_status_msg.hdr.cmd = SYSTEM_STATE_STATUS;
+	state_status_msg.supported_states = supported_states;
+	current_state |= SUSPEND_STATE_BIT;
+	dev->print_log(dev, "%s() sends SUSPEND notification\n", __func__);
+	state_status_msg.states_status = current_state;
+
+	ishtp_write_message(dev, &ishtp_hdr,
+		(unsigned char *)&state_status_msg);
+}
+EXPORT_SYMBOL(ishtp_send_suspend);
+
+/**
+ * ishtp_send_resume() - Send resume message to FW
+ * @dev: ISHTP device instance
+ *
+ * Send resume message to FW. This is useful for system freeze (non S3) case
+ */
+void ishtp_send_resume(struct ishtp_device *dev)
+{
+	struct ishtp_msg_hdr	ishtp_hdr;
+	struct ish_system_states_status state_status_msg;
+	const size_t len = sizeof(struct ish_system_states_status);
+
+	fix_cl_hdr(&ishtp_hdr, len, ISHTP_SYSTEM_STATE_CLIENT_ADDR);
+
+	memset(&state_status_msg, 0, len);
+	state_status_msg.hdr.cmd = SYSTEM_STATE_STATUS;
+	state_status_msg.supported_states = supported_states;
+	current_state &= ~SUSPEND_STATE_BIT;
+	dev->print_log(dev, "%s() sends RESUME notification\n", __func__);
+	state_status_msg.states_status = current_state;
+
+	ishtp_write_message(dev, &ishtp_hdr,
+		(unsigned char *)&state_status_msg);
+}
+EXPORT_SYMBOL(ishtp_send_resume);
+
+/**
+ * ishtp_query_subscribers() - Send query subscribers message
+ * @dev: ISHTP device instance
+ *
+ * Send message to query subscribers
+ */
+void ishtp_query_subscribers(struct ishtp_device *dev)
+{
+	struct ishtp_msg_hdr	ishtp_hdr;
+	struct ish_system_states_query_subscribers query_subscribers_msg;
+	const size_t len = sizeof(struct ish_system_states_query_subscribers);
+
+	fix_cl_hdr(&ishtp_hdr, len, ISHTP_SYSTEM_STATE_CLIENT_ADDR);
+
+	memset(&query_subscribers_msg, 0, len);
+	query_subscribers_msg.hdr.cmd = SYSTEM_STATE_QUERY_SUBSCRIBERS;
+
+	ishtp_write_message(dev, &ishtp_hdr,
+		(unsigned char *)&query_subscribers_msg);
+}
diff --git a/drivers/hid/intel-ish-hid/ishtp/hbm.h b/drivers/hid/intel-ish-hid/ishtp/hbm.h
new file mode 100644
index 000000000000..d96111cef7f8
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/hbm.h
@@ -0,0 +1,321 @@
+/*
+ * ISHTP bus layer messages handling
+ *
+ * Copyright (c) 2003-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _ISHTP_HBM_H_
+#define _ISHTP_HBM_H_
+
+#include <linux/uuid.h>
+
+struct ishtp_device;
+struct ishtp_msg_hdr;
+struct ishtp_cl;
+
+/*
+ * Timeouts in Seconds
+ */
+#define ISHTP_INTEROP_TIMEOUT		7 /* Timeout on ready message */
+
+#define ISHTP_CL_CONNECT_TIMEOUT	15 /* HPS: Client Connect Timeout */
+
+/*
+ * ISHTP Version
+ */
+#define HBM_MINOR_VERSION		0
+#define HBM_MAJOR_VERSION		1
+
+/* Host bus message command opcode */
+#define ISHTP_HBM_CMD_OP_MSK		0x7f
+/* Host bus message command RESPONSE */
+#define ISHTP_HBM_CMD_RES_MSK		0x80
+
+/*
+ * ISHTP Bus Message Command IDs
+ */
+#define HOST_START_REQ_CMD		0x01
+#define HOST_START_RES_CMD		0x81
+
+#define HOST_STOP_REQ_CMD		0x02
+#define HOST_STOP_RES_CMD		0x82
+
+#define FW_STOP_REQ_CMD			0x03
+
+#define HOST_ENUM_REQ_CMD		0x04
+#define HOST_ENUM_RES_CMD		0x84
+
+#define HOST_CLIENT_PROPERTIES_REQ_CMD	0x05
+#define HOST_CLIENT_PROPERTIES_RES_CMD	0x85
+
+#define CLIENT_CONNECT_REQ_CMD		0x06
+#define CLIENT_CONNECT_RES_CMD		0x86
+
+#define CLIENT_DISCONNECT_REQ_CMD	0x07
+#define CLIENT_DISCONNECT_RES_CMD	0x87
+
+#define ISHTP_FLOW_CONTROL_CMD		0x08
+
+#define DMA_BUFFER_ALLOC_NOTIFY		0x11
+#define DMA_BUFFER_ALLOC_RESPONSE	0x91
+
+#define DMA_XFER			0x12
+#define DMA_XFER_ACK			0x92
+
+/*
+ * ISHTP Stop Reason
+ * used by hbm_host_stop_request.reason
+ */
+#define	DRIVER_STOP_REQUEST		0x00
+
+/*
+ * ISHTP BUS Interface Section
+ */
+struct ishtp_msg_hdr {
+	uint32_t fw_addr:8;
+	uint32_t host_addr:8;
+	uint32_t length:9;
+	uint32_t reserved:6;
+	uint32_t msg_complete:1;
+} __packed;
+
+struct ishtp_bus_message {
+	uint8_t hbm_cmd;
+	uint8_t data[0];
+} __packed;
+
+/**
+ * struct hbm_cl_cmd - client specific host bus command
+ *	CONNECT, DISCONNECT, and FlOW CONTROL
+ *
+ * @hbm_cmd - bus message command header
+ * @fw_addr - address of the fw client
+ * @host_addr - address of the client in the driver
+ * @data
+ */
+struct ishtp_hbm_cl_cmd {
+	uint8_t hbm_cmd;
+	uint8_t fw_addr;
+	uint8_t host_addr;
+	uint8_t data;
+};
+
+struct hbm_version {
+	uint8_t minor_version;
+	uint8_t major_version;
+} __packed;
+
+struct hbm_host_version_request {
+	uint8_t hbm_cmd;
+	uint8_t reserved;
+	struct hbm_version host_version;
+} __packed;
+
+struct hbm_host_version_response {
+	uint8_t hbm_cmd;
+	uint8_t host_version_supported;
+	struct hbm_version fw_max_version;
+} __packed;
+
+struct hbm_host_stop_request {
+	uint8_t hbm_cmd;
+	uint8_t reason;
+	uint8_t reserved[2];
+} __packed;
+
+struct hbm_host_stop_response {
+	uint8_t hbm_cmd;
+	uint8_t reserved[3];
+} __packed;
+
+struct hbm_host_enum_request {
+	uint8_t hbm_cmd;
+	uint8_t reserved[3];
+} __packed;
+
+struct hbm_host_enum_response {
+	uint8_t hbm_cmd;
+	uint8_t reserved[3];
+	uint8_t valid_addresses[32];
+} __packed;
+
+struct ishtp_client_properties {
+	uuid_le protocol_name;
+	uint8_t protocol_version;
+	uint8_t max_number_of_connections;
+	uint8_t fixed_address;
+	uint8_t single_recv_buf;
+	uint32_t max_msg_length;
+	uint8_t dma_hdr_len;
+#define	ISHTP_CLIENT_DMA_ENABLED	0x80
+	uint8_t reserved4;
+	uint8_t reserved5;
+	uint8_t reserved6;
+} __packed;
+
+struct hbm_props_request {
+	uint8_t hbm_cmd;
+	uint8_t address;
+	uint8_t reserved[2];
+} __packed;
+
+struct hbm_props_response {
+	uint8_t hbm_cmd;
+	uint8_t address;
+	uint8_t status;
+	uint8_t reserved[1];
+	struct ishtp_client_properties client_properties;
+} __packed;
+
+/**
+ * struct hbm_client_connect_request - connect/disconnect request
+ *
+ * @hbm_cmd - bus message command header
+ * @fw_addr - address of the fw client
+ * @host_addr - address of the client in the driver
+ * @reserved
+ */
+struct hbm_client_connect_request {
+	uint8_t hbm_cmd;
+	uint8_t fw_addr;
+	uint8_t host_addr;
+	uint8_t reserved;
+} __packed;
+
+/**
+ * struct hbm_client_connect_response - connect/disconnect response
+ *
+ * @hbm_cmd - bus message command header
+ * @fw_addr - address of the fw client
+ * @host_addr - address of the client in the driver
+ * @status - status of the request
+ */
+struct hbm_client_connect_response {
+	uint8_t hbm_cmd;
+	uint8_t fw_addr;
+	uint8_t host_addr;
+	uint8_t status;
+} __packed;
+
+
+#define ISHTP_FC_MESSAGE_RESERVED_LENGTH		5
+
+struct hbm_flow_control {
+	uint8_t hbm_cmd;
+	uint8_t fw_addr;
+	uint8_t host_addr;
+	uint8_t reserved[ISHTP_FC_MESSAGE_RESERVED_LENGTH];
+} __packed;
+
+struct dma_alloc_notify {
+	uint8_t hbm;
+	uint8_t status;
+	uint8_t reserved[2];
+	uint32_t buf_size;
+	uint64_t buf_address;
+	/* [...] May come more size/address pairs */
+} __packed;
+
+struct dma_xfer_hbm {
+	uint8_t hbm;
+	uint8_t fw_client_id;
+	uint8_t host_client_id;
+	uint8_t reserved;
+	uint64_t msg_addr;
+	uint32_t msg_length;
+	uint32_t reserved2;
+} __packed;
+
+/* System state */
+#define ISHTP_SYSTEM_STATE_CLIENT_ADDR		13
+
+#define SYSTEM_STATE_SUBSCRIBE			0x1
+#define SYSTEM_STATE_STATUS			0x2
+#define SYSTEM_STATE_QUERY_SUBSCRIBERS		0x3
+#define SYSTEM_STATE_STATE_CHANGE_REQ		0x4
+/*indicates suspend and resume states*/
+#define SUSPEND_STATE_BIT			(1<<1)
+
+struct ish_system_states_header {
+	uint32_t cmd;
+	uint32_t cmd_status;	/*responses will have this set*/
+} __packed;
+
+struct ish_system_states_subscribe {
+	struct ish_system_states_header hdr;
+	uint32_t states;
+} __packed;
+
+struct ish_system_states_status {
+	struct ish_system_states_header hdr;
+	uint32_t supported_states;
+	uint32_t states_status;
+} __packed;
+
+struct ish_system_states_query_subscribers {
+	struct ish_system_states_header hdr;
+} __packed;
+
+struct ish_system_states_state_change_req {
+	struct ish_system_states_header hdr;
+	uint32_t requested_states;
+	uint32_t states_status;
+} __packed;
+
+/**
+ * enum ishtp_hbm_state - host bus message protocol state
+ *
+ * @ISHTP_HBM_IDLE : protocol not started
+ * @ISHTP_HBM_START : start request message was sent
+ * @ISHTP_HBM_ENUM_CLIENTS : enumeration request was sent
+ * @ISHTP_HBM_CLIENT_PROPERTIES : acquiring clients properties
+ */
+enum ishtp_hbm_state {
+	ISHTP_HBM_IDLE = 0,
+	ISHTP_HBM_START,
+	ISHTP_HBM_STARTED,
+	ISHTP_HBM_ENUM_CLIENTS,
+	ISHTP_HBM_CLIENT_PROPERTIES,
+	ISHTP_HBM_WORKING,
+	ISHTP_HBM_STOPPED,
+};
+
+static inline void ishtp_hbm_hdr(struct ishtp_msg_hdr *hdr, size_t length)
+{
+	hdr->host_addr = 0;
+	hdr->fw_addr = 0;
+	hdr->length = length;
+	hdr->msg_complete = 1;
+	hdr->reserved = 0;
+}
+
+int ishtp_hbm_start_req(struct ishtp_device *dev);
+int ishtp_hbm_start_wait(struct ishtp_device *dev);
+int ishtp_hbm_cl_flow_control_req(struct ishtp_device *dev,
+				  struct ishtp_cl *cl);
+int ishtp_hbm_cl_disconnect_req(struct ishtp_device *dev, struct ishtp_cl *cl);
+int ishtp_hbm_cl_connect_req(struct ishtp_device *dev, struct ishtp_cl *cl);
+void ishtp_hbm_enum_clients_req(struct ishtp_device *dev);
+void bh_hbm_work_fn(struct work_struct *work);
+void recv_hbm(struct ishtp_device *dev, struct ishtp_msg_hdr *ishtp_hdr);
+void recv_fixed_cl_msg(struct ishtp_device *dev,
+	struct ishtp_msg_hdr *ishtp_hdr);
+void ishtp_hbm_dispatch(struct ishtp_device *dev,
+	struct ishtp_bus_message *hdr);
+
+void ishtp_query_subscribers(struct ishtp_device *dev);
+
+/* Exported I/F */
+void ishtp_send_suspend(struct ishtp_device *dev);
+void ishtp_send_resume(struct ishtp_device *dev);
+
+#endif /* _ISHTP_HBM_H_ */
diff --git a/drivers/hid/intel-ish-hid/ishtp/init.c b/drivers/hid/intel-ish-hid/ishtp/init.c
new file mode 100644
index 000000000000..ac364418e17c
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/init.c
@@ -0,0 +1,115 @@
+/*
+ * Initialization protocol for ISHTP driver
+ *
+ * Copyright (c) 2003-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/miscdevice.h>
+#include "ishtp-dev.h"
+#include "hbm.h"
+#include "client.h"
+
+/**
+ * ishtp_dev_state_str() -Convert to string format
+ * @state: state to convert
+ *
+ * Convert state to string for prints
+ *
+ * Return: character pointer to converted string
+ */
+const char *ishtp_dev_state_str(int state)
+{
+	switch (state) {
+	case ISHTP_DEV_INITIALIZING:
+		return	"INITIALIZING";
+	case ISHTP_DEV_INIT_CLIENTS:
+		return	"INIT_CLIENTS";
+	case ISHTP_DEV_ENABLED:
+		return	"ENABLED";
+	case ISHTP_DEV_RESETTING:
+		return	"RESETTING";
+	case ISHTP_DEV_DISABLED:
+		return	"DISABLED";
+	case ISHTP_DEV_POWER_DOWN:
+		return	"POWER_DOWN";
+	case ISHTP_DEV_POWER_UP:
+		return	"POWER_UP";
+	default:
+		return "unknown";
+	}
+}
+
+/**
+ * ishtp_device_init() - ishtp device init
+ * @dev: ISHTP device instance
+ *
+ * After ISHTP device is alloacted, this function is used to initialize
+ * each field which includes spin lock, work struct and lists
+ */
+void ishtp_device_init(struct ishtp_device *dev)
+{
+	dev->dev_state = ISHTP_DEV_INITIALIZING;
+	INIT_LIST_HEAD(&dev->cl_list);
+	INIT_LIST_HEAD(&dev->device_list);
+	dev->rd_msg_fifo_head = 0;
+	dev->rd_msg_fifo_tail = 0;
+	spin_lock_init(&dev->rd_msg_spinlock);
+
+	init_waitqueue_head(&dev->wait_hbm_recvd_msg);
+	spin_lock_init(&dev->read_list_spinlock);
+	spin_lock_init(&dev->device_lock);
+	spin_lock_init(&dev->device_list_lock);
+	spin_lock_init(&dev->cl_list_lock);
+	spin_lock_init(&dev->fw_clients_lock);
+	INIT_WORK(&dev->bh_hbm_work, bh_hbm_work_fn);
+
+	bitmap_zero(dev->host_clients_map, ISHTP_CLIENTS_MAX);
+	dev->open_handle_count = 0;
+
+	/*
+	 * Reserving client ID 0 for ISHTP Bus Message communications
+	 */
+	bitmap_set(dev->host_clients_map, 0, 1);
+
+	INIT_LIST_HEAD(&dev->read_list.list);
+
+}
+EXPORT_SYMBOL(ishtp_device_init);
+
+/**
+ * ishtp_start() - Start ISH processing
+ * @dev: ISHTP device instance
+ *
+ * Start ISHTP processing by sending query subscriber message
+ *
+ * Return: 0 on success else -ENODEV
+ */
+int ishtp_start(struct ishtp_device *dev)
+{
+	if (ishtp_hbm_start_wait(dev)) {
+		dev_err(dev->devc, "HBM haven't started");
+		goto err;
+	}
+
+	/* suspend & resume notification - send QUERY_SUBSCRIBERS msg */
+	ishtp_query_subscribers(dev);
+
+	return 0;
+err:
+	dev_err(dev->devc, "link layer initialization failed.\n");
+	dev->dev_state = ISHTP_DEV_DISABLED;
+	return -ENODEV;
+}
+EXPORT_SYMBOL(ishtp_start);
diff --git a/drivers/hid/intel-ish-hid/ishtp/ishtp-dev.h b/drivers/hid/intel-ish-hid/ishtp/ishtp-dev.h
new file mode 100644
index 000000000000..a94f9a8a96a0
--- /dev/null
+++ b/drivers/hid/intel-ish-hid/ishtp/ishtp-dev.h
@@ -0,0 +1,277 @@
+/*
+ * Most ISHTP provider device and ISHTP logic declarations
+ *
+ * Copyright (c) 2003-2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _ISHTP_DEV_H_
+#define _ISHTP_DEV_H_
+
+#include <linux/types.h>
+#include <linux/spinlock.h>
+#include "bus.h"
+#include "hbm.h"
+
+#define	IPC_PAYLOAD_SIZE	128
+#define ISHTP_RD_MSG_BUF_SIZE	IPC_PAYLOAD_SIZE
+#define	IPC_FULL_MSG_SIZE	132
+
+/* Number of messages to be held in ISR->BH FIFO */
+#define	RD_INT_FIFO_SIZE	64
+
+/*
+ * Number of IPC messages to be held in Tx FIFO, to be sent by ISR -
+ * Tx complete interrupt or RX_COMPLETE handler
+ */
+#define	IPC_TX_FIFO_SIZE	512
+
+/*
+ * Number of Maximum ISHTP Clients
+ */
+#define ISHTP_CLIENTS_MAX 256
+
+/*
+ * Number of File descriptors/handles
+ * that can be opened to the driver.
+ *
+ * Limit to 255: 256 Total Clients
+ * minus internal client for ISHTP Bus Messages
+ */
+#define ISHTP_MAX_OPEN_HANDLE_COUNT (ISHTP_CLIENTS_MAX - 1)
+
+/* Internal Clients Number */
+#define ISHTP_HOST_CLIENT_ID_ANY		(-1)
+#define ISHTP_HBM_HOST_CLIENT_ID		0
+
+#define	MAX_DMA_DELAY	20
+
+/* ISHTP device states */
+enum ishtp_dev_state {
+	ISHTP_DEV_INITIALIZING = 0,
+	ISHTP_DEV_INIT_CLIENTS,
+	ISHTP_DEV_ENABLED,
+	ISHTP_DEV_RESETTING,
+	ISHTP_DEV_DISABLED,
+	ISHTP_DEV_POWER_DOWN,
+	ISHTP_DEV_POWER_UP
+};
+const char *ishtp_dev_state_str(int state);
+
+struct ishtp_cl;
+
+/**
+ * struct ishtp_fw_client - representation of fw client
+ *
+ * @props - client properties
+ * @client_id - fw client id
+ */
+struct ishtp_fw_client {
+	struct ishtp_client_properties props;
+	uint8_t client_id;
+};
+
+/**
+ * struct ishtp_msg_data - ISHTP message data struct
+ * @size:	Size of data in the *data
+ * @data:	Pointer to data
+ */
+struct ishtp_msg_data {
+	uint32_t size;
+	unsigned char *data;
+};
+
+/*
+ * struct ishtp_cl_rb - request block structure
+ * @list:	Link to list members
+ * @cl:		ISHTP client instance
+ * @buffer:	message header
+ * @buf_idx:	Index into buffer
+ * @read_time:	 unused at this time
+ */
+struct ishtp_cl_rb {
+	struct list_head list;
+	struct ishtp_cl *cl;
+	struct ishtp_msg_data buffer;
+	unsigned long buf_idx;
+	unsigned long read_time;
+};
+
+/*
+ * Control info for IPC messages ISHTP/IPC sending FIFO -
+ * list with inline data buffer
+ * This structure will be filled with parameters submitted
+ * by the caller glue layer
+ * 'buf' may be pointing to the external buffer or to 'inline_data'
+ * 'offset' will be initialized to 0 by submitting
+ *
+ * 'ipc_send_compl' is intended for use by clients that send fragmented
+ * messages. When a fragment is sent down to IPC msg regs,
+ * it will be called.
+ * If it has more fragments to send, it will do it. With last fragment
+ * it will send appropriate ISHTP "message-complete" flag.
+ * It will remove the outstanding message
+ * (mark outstanding buffer as available).
+ * If counting flow control is in work and there are more flow control
+ * credits, it can put the next client message queued in cl.
+ * structure for IPC processing.
+ *
+ */
+struct wr_msg_ctl_info {
+	/* Will be called with 'ipc_send_compl_prm' as parameter */
+	void (*ipc_send_compl)(void *);
+
+	void *ipc_send_compl_prm;
+	size_t length;
+	struct list_head	link;
+	unsigned char	inline_data[IPC_FULL_MSG_SIZE];
+};
+
+/*
+ * The ISHTP layer talks to hardware IPC message using the following
+ * callbacks
+ */
+struct ishtp_hw_ops {
+	int	(*hw_reset)(struct ishtp_device *dev);
+	int	(*ipc_reset)(struct ishtp_device *dev);
+	uint32_t (*ipc_get_header)(struct ishtp_device *dev, int length,
+				   int busy);
+	int	(*write)(struct ishtp_device *dev,
+		void (*ipc_send_compl)(void *), void *ipc_send_compl_prm,
+		unsigned char *msg, int length);
+	uint32_t	(*ishtp_read_hdr)(const struct ishtp_device *dev);
+	int	(*ishtp_read)(struct ishtp_device *dev, unsigned char *buffer,
+			unsigned long buffer_length);
+	uint32_t	(*get_fw_status)(struct ishtp_device *dev);
+	void	(*sync_fw_clock)(struct ishtp_device *dev);
+};
+
+/**
+ * struct ishtp_device - ISHTP private device struct
+ */
+struct ishtp_device {
+	struct device *devc;	/* pointer to lowest device */
+	struct pci_dev *pdev;	/* PCI device to get device ids */
+
+	/* waitq for waiting for suspend response */
+	wait_queue_head_t suspend_wait;
+	bool suspend_flag;	/* Suspend is active */
+
+	/* waitq for waiting for resume response */
+	wait_queue_head_t resume_wait;
+	bool resume_flag;	/*Resume is active */
+
+	/*
+	 * lock for the device, for everything that doesn't have
+	 * a dedicated spinlock
+	 */
+	spinlock_t device_lock;
+
+	bool recvd_hw_ready;
+	struct hbm_version version;
+	int transfer_path; /* Choice of transfer path: IPC or DMA */
+
+	/* ishtp device states */
+	enum ishtp_dev_state dev_state;
+	enum ishtp_hbm_state hbm_state;
+
+	/* driver read queue */
+	struct ishtp_cl_rb read_list;
+	spinlock_t read_list_spinlock;
+
+	/* list of ishtp_cl's */
+	struct list_head cl_list;
+	spinlock_t cl_list_lock;
+	long open_handle_count;
+
+	/* List of bus devices */
+	struct list_head device_list;
+	spinlock_t device_list_lock;
+
+	/* waiting queues for receive message from FW */
+	wait_queue_head_t wait_hw_ready;
+	wait_queue_head_t wait_hbm_recvd_msg;
+
+	/* FIFO for input messages for BH processing */
+	unsigned char rd_msg_fifo[RD_INT_FIFO_SIZE * IPC_PAYLOAD_SIZE];
+	unsigned int rd_msg_fifo_head, rd_msg_fifo_tail;
+	spinlock_t rd_msg_spinlock;
+	struct work_struct bh_hbm_work;
+
+	/* IPC write queue */
+	struct wr_msg_ctl_info wr_processing_list_head, wr_free_list_head;
+	/* For both processing list  and free list */
+	spinlock_t wr_processing_spinlock;
+
+	spinlock_t out_ipc_spinlock;
+
+	struct ishtp_fw_client *fw_clients; /*Note:memory has to be allocated*/
+	DECLARE_BITMAP(fw_clients_map, ISHTP_CLIENTS_MAX);
+	DECLARE_BITMAP(host_clients_map, ISHTP_CLIENTS_MAX);
+	uint8_t fw_clients_num;
+	uint8_t fw_client_presentation_num;
+	uint8_t fw_client_index;
+	spinlock_t fw_clients_lock;
+
+	/* TX DMA buffers and slots */
+	int ishtp_host_dma_enabled;
+	void *ishtp_host_dma_tx_buf;
+	unsigned int ishtp_host_dma_tx_buf_size;
+	uint64_t ishtp_host_dma_tx_buf_phys;
+	int ishtp_dma_num_slots;
+
+	/* map of 4k blocks in Tx dma buf: 0-free, 1-used */
+	uint8_t *ishtp_dma_tx_map;
+	spinlock_t ishtp_dma_tx_lock;
+
+	/* RX DMA buffers and slots */
+	void *ishtp_host_dma_rx_buf;
+	unsigned int ishtp_host_dma_rx_buf_size;
+	uint64_t ishtp_host_dma_rx_buf_phys;
+
+	/* Dump to trace buffers if enabled*/
+	void (*print_log)(struct ishtp_device *dev, char *format, ...);
+
+	/* Debug stats */
+	unsigned int	ipc_rx_cnt;
+	unsigned long long	ipc_rx_bytes_cnt;
+	unsigned int	ipc_tx_cnt;
+	unsigned long long	ipc_tx_bytes_cnt;
+
+	const struct ishtp_hw_ops *ops;
+	size_t	mtu;
+	uint32_t	ishtp_msg_hdr;
+	char hw[0] __aligned(sizeof(void *));
+};
+
+static inline unsigned long ishtp_secs_to_jiffies(unsigned long sec)
+{
+	return msecs_to_jiffies(sec * MSEC_PER_SEC);
+}
+
+/*
+ * Register Access Function
+ */
+static inline int ish_ipc_reset(struct ishtp_device *dev)
+{
+	return dev->ops->ipc_reset(dev);
+}
+
+static inline int ish_hw_reset(struct ishtp_device *dev)
+{
+	return dev->ops->hw_reset(dev);
+}
+
+/* Exported function */
+void	ishtp_device_init(struct ishtp_device *dev);
+int	ishtp_start(struct ishtp_device *dev);
+
+#endif /*_ISHTP_DEV_H_*/
diff --git a/drivers/hid/uhid.c b/drivers/hid/uhid.c
index 99ec3ff7563b..7f8ff39ed44b 100644
--- a/drivers/hid/uhid.c
+++ b/drivers/hid/uhid.c
@@ -779,19 +779,8 @@ static struct miscdevice uhid_misc = {
 	.minor		= UHID_MINOR,
 	.name		= UHID_NAME,
 };
+module_misc_device(uhid_misc);
 
-static int __init uhid_init(void)
-{
-	return misc_register(&uhid_misc);
-}
-
-static void __exit uhid_exit(void)
-{
-	misc_deregister(&uhid_misc);
-}
-
-module_init(uhid_init);
-module_exit(uhid_exit);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("David Herrmann <dh.herrmann@gmail.com>");
 MODULE_DESCRIPTION("User-space I/O driver support for HID subsystem");
diff --git a/drivers/hid/usbhid/hid-quirks.c b/drivers/hid/usbhid/hid-quirks.c
index b4b8c6abb03e..0a0eca5da47d 100644
--- a/drivers/hid/usbhid/hid-quirks.c
+++ b/drivers/hid/usbhid/hid-quirks.c
@@ -76,6 +76,7 @@ static const struct hid_blacklist {
 	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K95RGB, HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
 	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K70RGB, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K65RGB, HID_QUIRK_NO_INIT_REPORTS },
+	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_STRAFE, HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
 	{ USB_VENDOR_ID_CREATIVELABS, USB_DEVICE_ID_CREATIVE_SB_OMNI_SURROUND_51, HID_QUIRK_NOGET },
 	{ USB_VENDOR_ID_DMI, USB_DEVICE_ID_DMI_ENC, HID_QUIRK_NOGET },
 	{ USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_WIIU, HID_QUIRK_MULTI_INPUT },
@@ -98,6 +99,7 @@ static const struct hid_blacklist {
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_2, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP, HID_QUIRK_NO_INIT_REPORTS },
+	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_3, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MSI, USB_DEVICE_ID_MSI_GT683R_LED_PANEL, HID_QUIRK_NO_INIT_REPORTS },
@@ -143,7 +145,7 @@ static const struct hid_blacklist {
 	{ USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_WIRELESS, HID_QUIRK_MULTI_INPUT },
 	{ USB_VENDOR_ID_SIGMA_MICRO, USB_DEVICE_ID_SIGMA_MICRO_KEYBOARD, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_2, HID_QUIRK_MULTI_INPUT },
+	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2, HID_QUIRK_MULTI_INPUT },
 	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M610X, HID_QUIRK_MULTI_INPUT },
 	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_PENSKETCH_M912, HID_QUIRK_MULTI_INPUT },
 	{ USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_DUOSENSE, HID_QUIRK_NO_INIT_REPORTS },
diff --git a/drivers/hid/wacom.h b/drivers/hid/wacom.h
index 4681a65a4579..b4800ea891cb 100644
--- a/drivers/hid/wacom.h
+++ b/drivers/hid/wacom.h
@@ -90,6 +90,8 @@
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
 #include <linux/hid.h>
+#include <linux/kfifo.h>
+#include <linux/leds.h>
 #include <linux/usb/input.h>
 #include <linux/power_supply.h>
 #include <asm/unaligned.h>
@@ -105,32 +107,95 @@
 #define USB_VENDOR_ID_WACOM	0x056a
 #define USB_VENDOR_ID_LENOVO	0x17ef
 
+enum wacom_worker {
+	WACOM_WORKER_WIRELESS,
+	WACOM_WORKER_BATTERY,
+	WACOM_WORKER_REMOTE,
+};
+
+struct wacom;
+
+struct wacom_led {
+	struct led_classdev cdev;
+	struct led_trigger trigger;
+	struct wacom *wacom;
+	unsigned int group;
+	unsigned int id;
+	u8 llv;
+	u8 hlv;
+	bool held;
+};
+
+struct wacom_group_leds {
+	u8 select; /* status led selector (0..3) */
+	struct wacom_led *leds;
+	unsigned int count;
+	struct device *dev;
+};
+
+struct wacom_battery {
+	struct wacom *wacom;
+	struct power_supply_desc bat_desc;
+	struct power_supply *battery;
+	char bat_name[WACOM_NAME_MAX];
+	int battery_capacity;
+	int bat_charging;
+	int bat_connected;
+	int ps_connected;
+};
+
+struct wacom_remote {
+	spinlock_t remote_lock;
+	struct kfifo remote_fifo;
+	struct kobject *remote_dir;
+	struct {
+		struct attribute_group group;
+		u32 serial;
+		struct input_dev *input;
+		bool registered;
+		struct wacom_battery battery;
+	} remotes[WACOM_MAX_REMOTES];
+};
+
 struct wacom {
 	struct usb_device *usbdev;
 	struct usb_interface *intf;
 	struct wacom_wac wacom_wac;
 	struct hid_device *hdev;
 	struct mutex lock;
-	struct work_struct work;
-	struct wacom_led {
-		u8 select[5]; /* status led selector (0..3) */
+	struct work_struct wireless_work;
+	struct work_struct battery_work;
+	struct work_struct remote_work;
+	struct wacom_remote *remote;
+	struct wacom_leds {
+		struct wacom_group_leds *groups;
+		unsigned int count;
 		u8 llv;       /* status led brightness no button (1..127) */
 		u8 hlv;       /* status led brightness button pressed (1..127) */
 		u8 img_lum;   /* OLED matrix display brightness */
+		u8 max_llv;   /* maximum brightness of LED (llv) */
+		u8 max_hlv;   /* maximum brightness of LED (hlv) */
 	} led;
-	bool led_initialized;
-	struct power_supply *battery;
-	struct power_supply *ac;
-	struct power_supply_desc battery_desc;
-	struct power_supply_desc ac_desc;
-	struct kobject *remote_dir;
-	struct attribute_group remote_group[5];
+	struct wacom_battery battery;
+	bool resources;
 };
 
-static inline void wacom_schedule_work(struct wacom_wac *wacom_wac)
+static inline void wacom_schedule_work(struct wacom_wac *wacom_wac,
+				       enum wacom_worker which)
 {
 	struct wacom *wacom = container_of(wacom_wac, struct wacom, wacom_wac);
-	schedule_work(&wacom->work);
+
+	switch (which) {
+	case WACOM_WORKER_WIRELESS:
+		schedule_work(&wacom->wireless_work);
+		break;
+	case WACOM_WORKER_BATTERY:
+		schedule_work(&wacom->battery_work);
+		break;
+	case WACOM_WORKER_REMOTE:
+		schedule_work(&wacom->remote_work);
+		break;
+	}
 }
 
 extern const struct hid_device_id wacom_ids[];
@@ -149,7 +214,8 @@ int wacom_wac_event(struct hid_device *hdev, struct hid_field *field,
 		struct hid_usage *usage, __s32 value);
 void wacom_wac_report(struct hid_device *hdev, struct hid_report *report);
 void wacom_battery_work(struct work_struct *work);
-int wacom_remote_create_attr_group(struct wacom *wacom, __u32 serial,
-				   int index);
-void wacom_remote_destroy_attr_group(struct wacom *wacom, __u32 serial);
+enum led_brightness wacom_leds_brightness_get(struct wacom_led *led);
+struct wacom_led *wacom_led_find(struct wacom *wacom, unsigned int group,
+				 unsigned int id);
+struct wacom_led *wacom_led_next(struct wacom *wacom, struct wacom_led *cur);
 #endif
diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
index 499cc8213cfe..5e7a5648e708 100644
--- a/drivers/hid/wacom_sys.c
+++ b/drivers/hid/wacom_sys.c
@@ -25,7 +25,6 @@
 #define WAC_CMD_RETRIES		10
 #define WAC_CMD_DELETE_PAIRING	0x20
 #define WAC_CMD_UNPAIR_ALL	0xFF
-#define WAC_REMOTE_SERIAL_MAX_STRLEN	9
 
 #define DEV_ATTR_RW_PERM (S_IRUGO | S_IWUSR | S_IWGRP)
 #define DEV_ATTR_WO_PERM (S_IWUSR | S_IWGRP)
@@ -91,7 +90,12 @@ static void wacom_close(struct input_dev *dev)
 {
 	struct wacom *wacom = input_get_drvdata(dev);
 
-	hid_hw_close(wacom->hdev);
+	/*
+	 * wacom->hdev should never be null, but surprisingly, I had the case
+	 * once while unplugging the Wacom Wireless Receiver.
+	 */
+	if (wacom->hdev)
+		hid_hw_close(wacom->hdev);
 }
 
 /*
@@ -523,36 +527,95 @@ struct wacom_hdev_data {
 static LIST_HEAD(wacom_udev_list);
 static DEFINE_MUTEX(wacom_udev_list_lock);
 
+static bool compare_device_paths(struct hid_device *hdev_a,
+		struct hid_device *hdev_b, char separator)
+{
+	int n1 = strrchr(hdev_a->phys, separator) - hdev_a->phys;
+	int n2 = strrchr(hdev_b->phys, separator) - hdev_b->phys;
+
+	if (n1 != n2 || n1 <= 0 || n2 <= 0)
+		return false;
+
+	return !strncmp(hdev_a->phys, hdev_b->phys, n1);
+}
+
 static bool wacom_are_sibling(struct hid_device *hdev,
 		struct hid_device *sibling)
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	struct wacom_features *features = &wacom->wacom_wac.features;
-	int vid = features->oVid;
-	int pid = features->oPid;
-	int n1,n2;
+	struct wacom *sibling_wacom = hid_get_drvdata(sibling);
+	struct wacom_features *sibling_features = &sibling_wacom->wacom_wac.features;
+	__u32 oVid = features->oVid ? features->oVid : hdev->vendor;
+	__u32 oPid = features->oPid ? features->oPid : hdev->product;
+
+	/* The defined oVid/oPid must match that of the sibling */
+	if (features->oVid != HID_ANY_ID && sibling->vendor != oVid)
+		return false;
+	if (features->oPid != HID_ANY_ID && sibling->product != oPid)
+		return false;
 
-	if (vid == 0 && pid == 0) {
-		vid = hdev->vendor;
-		pid = hdev->product;
+	/*
+	 * Devices with the same VID/PID must share the same physical
+	 * device path, while those with different VID/PID must share
+	 * the same physical parent device path.
+	 */
+	if (hdev->vendor == sibling->vendor && hdev->product == sibling->product) {
+		if (!compare_device_paths(hdev, sibling, '/'))
+			return false;
+	} else {
+		if (!compare_device_paths(hdev, sibling, '.'))
+			return false;
 	}
 
-	if (vid != sibling->vendor || pid != sibling->product)
+	/* Skip the remaining heuristics unless you are a HID_GENERIC device */
+	if (features->type != HID_GENERIC)
+		return true;
+
+	/*
+	 * Direct-input devices may not be siblings of indirect-input
+	 * devices.
+	 */
+	if ((features->device_type & WACOM_DEVICETYPE_DIRECT) &&
+	    !(sibling_features->device_type & WACOM_DEVICETYPE_DIRECT))
 		return false;
 
-	/* Compare the physical path. */
-	n1 = strrchr(hdev->phys, '.') - hdev->phys;
-	n2 = strrchr(sibling->phys, '.') - sibling->phys;
-	if (n1 != n2 || n1 <= 0 || n2 <= 0)
+	/*
+	 * Indirect-input devices may not be siblings of direct-input
+	 * devices.
+	 */
+	if (!(features->device_type & WACOM_DEVICETYPE_DIRECT) &&
+	    (sibling_features->device_type & WACOM_DEVICETYPE_DIRECT))
+		return false;
+
+	/* Pen devices may only be siblings of touch devices */
+	if ((features->device_type & WACOM_DEVICETYPE_PEN) &&
+	    !(sibling_features->device_type & WACOM_DEVICETYPE_TOUCH))
 		return false;
 
-	return !strncmp(hdev->phys, sibling->phys, n1);
+	/* Touch devices may only be siblings of pen devices */
+	if ((features->device_type & WACOM_DEVICETYPE_TOUCH) &&
+	    !(sibling_features->device_type & WACOM_DEVICETYPE_PEN))
+		return false;
+
+	/*
+	 * No reason could be found for these two devices to NOT be
+	 * siblings, so there's a good chance they ARE siblings
+	 */
+	return true;
 }
 
 static struct wacom_hdev_data *wacom_get_hdev_data(struct hid_device *hdev)
 {
 	struct wacom_hdev_data *data;
 
+	/* Try to find an already-probed interface from the same device */
+	list_for_each_entry(data, &wacom_udev_list, list) {
+		if (compare_device_paths(hdev, data->dev, '/'))
+			return data;
+	}
+
+	/* Fallback to finding devices that appear to be "siblings" */
 	list_for_each_entry(data, &wacom_udev_list, list) {
 		if (wacom_are_sibling(hdev, data->dev)) {
 			kref_get(&data->kref);
@@ -563,6 +626,38 @@ static struct wacom_hdev_data *wacom_get_hdev_data(struct hid_device *hdev)
 	return NULL;
 }
 
+static void wacom_release_shared_data(struct kref *kref)
+{
+	struct wacom_hdev_data *data =
+		container_of(kref, struct wacom_hdev_data, kref);
+
+	mutex_lock(&wacom_udev_list_lock);
+	list_del(&data->list);
+	mutex_unlock(&wacom_udev_list_lock);
+
+	kfree(data);
+}
+
+static void wacom_remove_shared_data(void *res)
+{
+	struct wacom *wacom = res;
+	struct wacom_hdev_data *data;
+	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+
+	if (wacom_wac->shared) {
+		data = container_of(wacom_wac->shared, struct wacom_hdev_data,
+				    shared);
+
+		if (wacom_wac->shared->touch == wacom->hdev)
+			wacom_wac->shared->touch = NULL;
+		else if (wacom_wac->shared->pen == wacom->hdev)
+			wacom_wac->shared->pen = NULL;
+
+		kref_put(&data->kref, wacom_release_shared_data);
+		wacom_wac->shared = NULL;
+	}
+}
+
 static int wacom_add_shared_data(struct hid_device *hdev)
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
@@ -587,6 +682,13 @@ static int wacom_add_shared_data(struct hid_device *hdev)
 
 	wacom_wac->shared = &data->shared;
 
+	retval = devm_add_action(&hdev->dev, wacom_remove_shared_data, wacom);
+	if (retval) {
+		mutex_unlock(&wacom_udev_list_lock);
+		wacom_remove_shared_data(wacom);
+		return retval;
+	}
+
 	if (wacom_wac->features.device_type & WACOM_DEVICETYPE_TOUCH)
 		wacom_wac->shared->touch = hdev;
 	else if (wacom_wac->features.device_type & WACOM_DEVICETYPE_PEN)
@@ -597,37 +699,6 @@ out:
 	return retval;
 }
 
-static void wacom_release_shared_data(struct kref *kref)
-{
-	struct wacom_hdev_data *data =
-		container_of(kref, struct wacom_hdev_data, kref);
-
-	mutex_lock(&wacom_udev_list_lock);
-	list_del(&data->list);
-	mutex_unlock(&wacom_udev_list_lock);
-
-	kfree(data);
-}
-
-static void wacom_remove_shared_data(struct wacom *wacom)
-{
-	struct wacom_hdev_data *data;
-	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
-
-	if (wacom_wac->shared) {
-		data = container_of(wacom_wac->shared, struct wacom_hdev_data,
-				    shared);
-
-		if (wacom_wac->shared->touch == wacom->hdev)
-			wacom_wac->shared->touch = NULL;
-		else if (wacom_wac->shared->pen == wacom->hdev)
-			wacom_wac->shared->pen = NULL;
-
-		kref_put(&data->kref, wacom_release_shared_data);
-		wacom_wac->shared = NULL;
-	}
-}
-
 static int wacom_led_control(struct wacom *wacom)
 {
 	unsigned char *buf;
@@ -635,6 +706,12 @@ static int wacom_led_control(struct wacom *wacom)
 	unsigned char report_id = WAC_CMD_LED_CONTROL;
 	int buf_size = 9;
 
+	if (!hid_get_drvdata(wacom->hdev))
+		return -ENODEV;
+
+	if (!wacom->led.groups)
+		return -ENOTSUPP;
+
 	if (wacom->wacom_wac.pid) { /* wireless connected */
 		report_id = WAC_CMD_WL_LED_CONTROL;
 		buf_size = 13;
@@ -650,7 +727,7 @@ static int wacom_led_control(struct wacom *wacom)
 		 * one of four values:
 		 *    0 = Low; 1 = Medium; 2 = High; 3 = Off
 		 */
-		int ring_led = wacom->led.select[0] & 0x03;
+		int ring_led = wacom->led.groups[0].select & 0x03;
 		int ring_lum = (((wacom->led.llv & 0x60) >> 5) - 1) & 0x03;
 		int crop_lum = 0;
 		unsigned char led_bits = (crop_lum << 4) | (ring_lum << 2) | (ring_led);
@@ -665,11 +742,11 @@ static int wacom_led_control(struct wacom *wacom)
 			buf[1] = led_bits;
 	}
 	else {
-		int led = wacom->led.select[0] | 0x4;
+		int led = wacom->led.groups[0].select | 0x4;
 
 		if (wacom->wacom_wac.features.type == WACOM_21UX2 ||
 		    wacom->wacom_wac.features.type == WACOM_24HD)
-			led |= (wacom->led.select[1] << 4) | 0x40;
+			led |= (wacom->led.groups[1].select << 4) | 0x40;
 
 		buf[0] = report_id;
 		buf[1] = led;
@@ -741,7 +818,7 @@ static ssize_t wacom_led_select_store(struct device *dev, int set_id,
 
 	mutex_lock(&wacom->lock);
 
-	wacom->led.select[set_id] = id & 0x3;
+	wacom->led.groups[set_id].select = id & 0x3;
 	err = wacom_led_control(wacom);
 
 	mutex_unlock(&wacom->lock);
@@ -761,7 +838,7 @@ static ssize_t wacom_led##SET_ID##_select_show(struct device *dev,	\
 	struct hid_device *hdev = to_hid_device(dev);\
 	struct wacom *wacom = hid_get_drvdata(hdev);			\
 	return scnprintf(buf, PAGE_SIZE, "%d\n",			\
-			 wacom->led.select[SET_ID]);			\
+			 wacom->led.groups[SET_ID].select);		\
 }									\
 static DEVICE_ATTR(status_led##SET_ID##_select, DEV_ATTR_RW_PERM,	\
 		    wacom_led##SET_ID##_select_show,			\
@@ -904,6 +981,327 @@ static struct attribute_group intuos5_led_attr_group = {
 	.attrs = intuos5_led_attrs,
 };
 
+struct wacom_sysfs_group_devres {
+	struct attribute_group *group;
+	struct kobject *root;
+};
+
+static void wacom_devm_sysfs_group_release(struct device *dev, void *res)
+{
+	struct wacom_sysfs_group_devres *devres = res;
+	struct kobject *kobj = devres->root;
+
+	dev_dbg(dev, "%s: dropping reference to %s\n",
+		__func__, devres->group->name);
+	sysfs_remove_group(kobj, devres->group);
+}
+
+static int __wacom_devm_sysfs_create_group(struct wacom *wacom,
+					   struct kobject *root,
+					   struct attribute_group *group)
+{
+	struct wacom_sysfs_group_devres *devres;
+	int error;
+
+	devres = devres_alloc(wacom_devm_sysfs_group_release,
+			      sizeof(struct wacom_sysfs_group_devres),
+			      GFP_KERNEL);
+	if (!devres)
+		return -ENOMEM;
+
+	devres->group = group;
+	devres->root = root;
+
+	error = sysfs_create_group(devres->root, group);
+	if (error)
+		return error;
+
+	devres_add(&wacom->hdev->dev, devres);
+
+	return 0;
+}
+
+static int wacom_devm_sysfs_create_group(struct wacom *wacom,
+					 struct attribute_group *group)
+{
+	return __wacom_devm_sysfs_create_group(wacom, &wacom->hdev->dev.kobj,
+					       group);
+}
+
+enum led_brightness wacom_leds_brightness_get(struct wacom_led *led)
+{
+	struct wacom *wacom = led->wacom;
+
+	if (wacom->led.max_hlv)
+		return led->hlv * LED_FULL / wacom->led.max_hlv;
+
+	if (wacom->led.max_llv)
+		return led->llv * LED_FULL / wacom->led.max_llv;
+
+	/* device doesn't support brightness tuning */
+	return LED_FULL;
+}
+
+static enum led_brightness __wacom_led_brightness_get(struct led_classdev *cdev)
+{
+	struct wacom_led *led = container_of(cdev, struct wacom_led, cdev);
+	struct wacom *wacom = led->wacom;
+
+	if (wacom->led.groups[led->group].select != led->id)
+		return LED_OFF;
+
+	return wacom_leds_brightness_get(led);
+}
+
+static int wacom_led_brightness_set(struct led_classdev *cdev,
+				    enum led_brightness brightness)
+{
+	struct wacom_led *led = container_of(cdev, struct wacom_led, cdev);
+	struct wacom *wacom = led->wacom;
+	int error;
+
+	mutex_lock(&wacom->lock);
+
+	if (!wacom->led.groups || (brightness == LED_OFF &&
+	    wacom->led.groups[led->group].select != led->id)) {
+		error = 0;
+		goto out;
+	}
+
+	led->llv = wacom->led.llv = wacom->led.max_llv * brightness / LED_FULL;
+	led->hlv = wacom->led.hlv = wacom->led.max_hlv * brightness / LED_FULL;
+
+	wacom->led.groups[led->group].select = led->id;
+
+	error = wacom_led_control(wacom);
+
+out:
+	mutex_unlock(&wacom->lock);
+
+	return error;
+}
+
+static void wacom_led_readonly_brightness_set(struct led_classdev *cdev,
+					       enum led_brightness brightness)
+{
+}
+
+static int wacom_led_register_one(struct device *dev, struct wacom *wacom,
+				  struct wacom_led *led, unsigned int group,
+				  unsigned int id, bool read_only)
+{
+	int error;
+	char *name;
+
+	name = devm_kasprintf(dev, GFP_KERNEL,
+			      "%s::wacom-%d.%d",
+			      dev_name(dev),
+			      group,
+			      id);
+	if (!name)
+		return -ENOMEM;
+
+	if (!read_only) {
+		led->trigger.name = name;
+		error = devm_led_trigger_register(dev, &led->trigger);
+		if (error) {
+			hid_err(wacom->hdev,
+				"failed to register LED trigger %s: %d\n",
+				led->cdev.name, error);
+			return error;
+		}
+	}
+
+	led->group = group;
+	led->id = id;
+	led->wacom = wacom;
+	led->llv = wacom->led.llv;
+	led->hlv = wacom->led.hlv;
+	led->cdev.name = name;
+	led->cdev.max_brightness = LED_FULL;
+	led->cdev.flags = LED_HW_PLUGGABLE;
+	led->cdev.brightness_get = __wacom_led_brightness_get;
+	if (!read_only) {
+		led->cdev.brightness_set_blocking = wacom_led_brightness_set;
+		led->cdev.default_trigger = led->cdev.name;
+	} else {
+		led->cdev.brightness_set = wacom_led_readonly_brightness_set;
+	}
+
+	error = devm_led_classdev_register(dev, &led->cdev);
+	if (error) {
+		hid_err(wacom->hdev,
+			"failed to register LED %s: %d\n",
+			led->cdev.name, error);
+		led->cdev.name = NULL;
+		return error;
+	}
+
+	return 0;
+}
+
+static void wacom_led_groups_release_one(void *data)
+{
+	struct wacom_group_leds *group = data;
+
+	devres_release_group(group->dev, group);
+}
+
+static int wacom_led_groups_alloc_and_register_one(struct device *dev,
+						   struct wacom *wacom,
+						   int group_id, int count,
+						   bool read_only)
+{
+	struct wacom_led *leds;
+	int i, error;
+
+	if (group_id >= wacom->led.count || count <= 0)
+		return -EINVAL;
+
+	if (!devres_open_group(dev, &wacom->led.groups[group_id], GFP_KERNEL))
+		return -ENOMEM;
+
+	leds = devm_kzalloc(dev, sizeof(struct wacom_led) * count, GFP_KERNEL);
+	if (!leds) {
+		error = -ENOMEM;
+		goto err;
+	}
+
+	wacom->led.groups[group_id].leds = leds;
+	wacom->led.groups[group_id].count = count;
+
+	for (i = 0; i < count; i++) {
+		error = wacom_led_register_one(dev, wacom, &leds[i],
+					       group_id, i, read_only);
+		if (error)
+			goto err;
+	}
+
+	wacom->led.groups[group_id].dev = dev;
+
+	devres_close_group(dev, &wacom->led.groups[group_id]);
+
+	/*
+	 * There is a bug (?) in devm_led_classdev_register() in which its
+	 * increments the refcount of the parent. If the parent is an input
+	 * device, that means the ref count never reaches 0 when
+	 * devm_input_device_release() gets called.
+	 * This means that the LEDs are still there after disconnect.
+	 * Manually force the release of the group so that the leds are released
+	 * once we are done using them.
+	 */
+	error = devm_add_action_or_reset(&wacom->hdev->dev,
+					 wacom_led_groups_release_one,
+					 &wacom->led.groups[group_id]);
+	if (error)
+		return error;
+
+	return 0;
+
+err:
+	devres_release_group(dev, &wacom->led.groups[group_id]);
+	return error;
+}
+
+struct wacom_led *wacom_led_find(struct wacom *wacom, unsigned int group_id,
+				 unsigned int id)
+{
+	struct wacom_group_leds *group;
+
+	if (group_id >= wacom->led.count)
+		return NULL;
+
+	group = &wacom->led.groups[group_id];
+
+	if (!group->leds)
+		return NULL;
+
+	id %= group->count;
+
+	return &group->leds[id];
+}
+
+/**
+ * wacom_led_next: gives the next available led with a wacom trigger.
+ *
+ * returns the next available struct wacom_led which has its default trigger
+ * or the current one if none is available.
+ */
+struct wacom_led *wacom_led_next(struct wacom *wacom, struct wacom_led *cur)
+{
+	struct wacom_led *next_led;
+	int group, next;
+
+	if (!wacom || !cur)
+		return NULL;
+
+	group = cur->group;
+	next = cur->id;
+
+	do {
+		next_led = wacom_led_find(wacom, group, ++next);
+		if (!next_led || next_led == cur)
+			return next_led;
+	} while (next_led->cdev.trigger != &next_led->trigger);
+
+	return next_led;
+}
+
+static void wacom_led_groups_release(void *data)
+{
+	struct wacom *wacom = data;
+
+	wacom->led.groups = NULL;
+	wacom->led.count = 0;
+}
+
+static int wacom_led_groups_allocate(struct wacom *wacom, int count)
+{
+	struct device *dev = &wacom->hdev->dev;
+	struct wacom_group_leds *groups;
+	int error;
+
+	groups = devm_kzalloc(dev, sizeof(struct wacom_group_leds) * count,
+			      GFP_KERNEL);
+	if (!groups)
+		return -ENOMEM;
+
+	error = devm_add_action_or_reset(dev, wacom_led_groups_release, wacom);
+	if (error)
+		return error;
+
+	wacom->led.groups = groups;
+	wacom->led.count = count;
+
+	return 0;
+}
+
+static int wacom_leds_alloc_and_register(struct wacom *wacom, int group_count,
+					 int led_per_group, bool read_only)
+{
+	struct device *dev;
+	int i, error;
+
+	if (!wacom->wacom_wac.pad_input)
+		return -EINVAL;
+
+	dev = &wacom->wacom_wac.pad_input->dev;
+
+	error = wacom_led_groups_allocate(wacom, group_count);
+	if (error)
+		return error;
+
+	for (i = 0; i < group_count; i++) {
+		error = wacom_led_groups_alloc_and_register_one(dev, wacom, i,
+								led_per_group,
+								read_only);
+		if (error)
+			return error;
+	}
+
+	return 0;
+}
+
 static int wacom_initialize_leds(struct wacom *wacom)
 {
 	int error;
@@ -917,25 +1315,38 @@ static int wacom_initialize_leds(struct wacom *wacom)
 	case INTUOS4:
 	case INTUOS4WL:
 	case INTUOS4L:
-		wacom->led.select[0] = 0;
-		wacom->led.select[1] = 0;
 		wacom->led.llv = 10;
 		wacom->led.hlv = 20;
+		wacom->led.max_llv = 127;
+		wacom->led.max_hlv = 127;
 		wacom->led.img_lum = 10;
-		error = sysfs_create_group(&wacom->hdev->dev.kobj,
-					   &intuos4_led_attr_group);
+
+		error = wacom_leds_alloc_and_register(wacom, 1, 4, false);
+		if (error) {
+			hid_err(wacom->hdev,
+				"cannot create leds err: %d\n", error);
+			return error;
+		}
+
+		error = wacom_devm_sysfs_create_group(wacom,
+						      &intuos4_led_attr_group);
 		break;
 
 	case WACOM_24HD:
 	case WACOM_21UX2:
-		wacom->led.select[0] = 0;
-		wacom->led.select[1] = 0;
 		wacom->led.llv = 0;
 		wacom->led.hlv = 0;
 		wacom->led.img_lum = 0;
 
-		error = sysfs_create_group(&wacom->hdev->dev.kobj,
-					   &cintiq_led_attr_group);
+		error = wacom_leds_alloc_and_register(wacom, 2, 4, false);
+		if (error) {
+			hid_err(wacom->hdev,
+				"cannot create leds err: %d\n", error);
+			return error;
+		}
+
+		error = wacom_devm_sysfs_create_group(wacom,
+						      &cintiq_led_attr_group);
 		break;
 
 	case INTUOS5S:
@@ -944,16 +1355,31 @@ static int wacom_initialize_leds(struct wacom *wacom)
 	case INTUOSPS:
 	case INTUOSPM:
 	case INTUOSPL:
-		wacom->led.select[0] = 0;
-		wacom->led.select[1] = 0;
 		wacom->led.llv = 32;
-		wacom->led.hlv = 0;
-		wacom->led.img_lum = 0;
+		wacom->led.max_llv = 96;
+
+		error = wacom_leds_alloc_and_register(wacom, 1, 4, false);
+		if (error) {
+			hid_err(wacom->hdev,
+				"cannot create leds err: %d\n", error);
+			return error;
+		}
 
-		error = sysfs_create_group(&wacom->hdev->dev.kobj,
-					  &intuos5_led_attr_group);
+		error = wacom_devm_sysfs_create_group(wacom,
+						      &intuos5_led_attr_group);
 		break;
 
+	case REMOTE:
+		wacom->led.llv = 255;
+		wacom->led.max_llv = 255;
+		error = wacom_led_groups_allocate(wacom, 5);
+		if (error) {
+			hid_err(wacom->hdev,
+				"cannot create leds err: %d\n", error);
+			return error;
+		}
+		return 0;
+
 	default:
 		return 0;
 	}
@@ -964,86 +1390,45 @@ static int wacom_initialize_leds(struct wacom *wacom)
 		return error;
 	}
 	wacom_led_control(wacom);
-	wacom->led_initialized = true;
 
 	return 0;
 }
 
-static void wacom_destroy_leds(struct wacom *wacom)
-{
-	if (!wacom->led_initialized)
-		return;
-
-	if (!(wacom->wacom_wac.features.device_type & WACOM_DEVICETYPE_PAD))
-		return;
-
-	wacom->led_initialized = false;
-
-	switch (wacom->wacom_wac.features.type) {
-	case INTUOS4S:
-	case INTUOS4:
-	case INTUOS4WL:
-	case INTUOS4L:
-		sysfs_remove_group(&wacom->hdev->dev.kobj,
-				   &intuos4_led_attr_group);
-		break;
-
-	case WACOM_24HD:
-	case WACOM_21UX2:
-		sysfs_remove_group(&wacom->hdev->dev.kobj,
-				   &cintiq_led_attr_group);
-		break;
-
-	case INTUOS5S:
-	case INTUOS5:
-	case INTUOS5L:
-	case INTUOSPS:
-	case INTUOSPM:
-	case INTUOSPL:
-		sysfs_remove_group(&wacom->hdev->dev.kobj,
-				   &intuos5_led_attr_group);
-		break;
-	}
-}
-
 static enum power_supply_property wacom_battery_props[] = {
+	POWER_SUPPLY_PROP_MODEL_NAME,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_SCOPE,
 	POWER_SUPPLY_PROP_CAPACITY
 };
 
-static enum power_supply_property wacom_ac_props[] = {
-	POWER_SUPPLY_PROP_PRESENT,
-	POWER_SUPPLY_PROP_ONLINE,
-	POWER_SUPPLY_PROP_SCOPE,
-};
-
 static int wacom_battery_get_property(struct power_supply *psy,
 				      enum power_supply_property psp,
 				      union power_supply_propval *val)
 {
-	struct wacom *wacom = power_supply_get_drvdata(psy);
+	struct wacom_battery *battery = power_supply_get_drvdata(psy);
 	int ret = 0;
 
 	switch (psp) {
+		case POWER_SUPPLY_PROP_MODEL_NAME:
+			val->strval = battery->wacom->wacom_wac.name;
+			break;
 		case POWER_SUPPLY_PROP_PRESENT:
-			val->intval = wacom->wacom_wac.bat_connected;
+			val->intval = battery->bat_connected;
 			break;
 		case POWER_SUPPLY_PROP_SCOPE:
 			val->intval = POWER_SUPPLY_SCOPE_DEVICE;
 			break;
 		case POWER_SUPPLY_PROP_CAPACITY:
-			val->intval =
-				wacom->wacom_wac.battery_capacity;
+			val->intval = battery->battery_capacity;
 			break;
 		case POWER_SUPPLY_PROP_STATUS:
-			if (wacom->wacom_wac.bat_charging)
+			if (battery->bat_charging)
 				val->intval = POWER_SUPPLY_STATUS_CHARGING;
-			else if (wacom->wacom_wac.battery_capacity == 100 &&
-				    wacom->wacom_wac.ps_connected)
+			else if (battery->battery_capacity == 100 &&
+				    battery->ps_connected)
 				val->intval = POWER_SUPPLY_STATUS_FULL;
-			else if (wacom->wacom_wac.ps_connected)
+			else if (battery->ps_connected)
 				val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING;
 			else
 				val->intval = POWER_SUPPLY_STATUS_DISCHARGING;
@@ -1056,84 +1441,64 @@ static int wacom_battery_get_property(struct power_supply *psy,
 	return ret;
 }
 
-static int wacom_ac_get_property(struct power_supply *psy,
-				enum power_supply_property psp,
-				union power_supply_propval *val)
+static int __wacom_initialize_battery(struct wacom *wacom,
+				      struct wacom_battery *battery)
 {
-	struct wacom *wacom = power_supply_get_drvdata(psy);
-	int ret = 0;
+	static atomic_t battery_no = ATOMIC_INIT(0);
+	struct device *dev = &wacom->hdev->dev;
+	struct power_supply_config psy_cfg = { .drv_data = battery, };
+	struct power_supply *ps_bat;
+	struct power_supply_desc *bat_desc = &battery->bat_desc;
+	unsigned long n;
+	int error;
 
-	switch (psp) {
-	case POWER_SUPPLY_PROP_PRESENT:
-		/* fall through */
-	case POWER_SUPPLY_PROP_ONLINE:
-		val->intval = wacom->wacom_wac.ps_connected;
-		break;
-	case POWER_SUPPLY_PROP_SCOPE:
-		val->intval = POWER_SUPPLY_SCOPE_DEVICE;
-		break;
-	default:
-		ret = -EINVAL;
-		break;
+	if (!devres_open_group(dev, bat_desc, GFP_KERNEL))
+		return -ENOMEM;
+
+	battery->wacom = wacom;
+
+	n = atomic_inc_return(&battery_no) - 1;
+
+	bat_desc->properties = wacom_battery_props;
+	bat_desc->num_properties = ARRAY_SIZE(wacom_battery_props);
+	bat_desc->get_property = wacom_battery_get_property;
+	sprintf(battery->bat_name, "wacom_battery_%ld", n);
+	bat_desc->name = battery->bat_name;
+	bat_desc->type = POWER_SUPPLY_TYPE_USB;
+	bat_desc->use_for_apm = 0;
+
+	ps_bat = devm_power_supply_register(dev, bat_desc, &psy_cfg);
+	if (IS_ERR(ps_bat)) {
+		error = PTR_ERR(ps_bat);
+		goto err;
 	}
-	return ret;
+
+	power_supply_powers(ps_bat, &wacom->hdev->dev);
+
+	battery->battery = ps_bat;
+
+	devres_close_group(dev, bat_desc);
+	return 0;
+
+err:
+	devres_release_group(dev, bat_desc);
+	return error;
 }
 
 static int wacom_initialize_battery(struct wacom *wacom)
 {
-	static atomic_t battery_no = ATOMIC_INIT(0);
-	struct power_supply_config psy_cfg = { .drv_data = wacom, };
-	unsigned long n;
-
-	if (wacom->wacom_wac.features.quirks & WACOM_QUIRK_BATTERY) {
-		struct power_supply_desc *bat_desc = &wacom->battery_desc;
-		struct power_supply_desc *ac_desc = &wacom->ac_desc;
-		n = atomic_inc_return(&battery_no) - 1;
-
-		bat_desc->properties = wacom_battery_props;
-		bat_desc->num_properties = ARRAY_SIZE(wacom_battery_props);
-		bat_desc->get_property = wacom_battery_get_property;
-		sprintf(wacom->wacom_wac.bat_name, "wacom_battery_%ld", n);
-		bat_desc->name = wacom->wacom_wac.bat_name;
-		bat_desc->type = POWER_SUPPLY_TYPE_BATTERY;
-		bat_desc->use_for_apm = 0;
-
-		ac_desc->properties = wacom_ac_props;
-		ac_desc->num_properties = ARRAY_SIZE(wacom_ac_props);
-		ac_desc->get_property = wacom_ac_get_property;
-		sprintf(wacom->wacom_wac.ac_name, "wacom_ac_%ld", n);
-		ac_desc->name = wacom->wacom_wac.ac_name;
-		ac_desc->type = POWER_SUPPLY_TYPE_MAINS;
-		ac_desc->use_for_apm = 0;
-
-		wacom->battery = power_supply_register(&wacom->hdev->dev,
-					      &wacom->battery_desc, &psy_cfg);
-		if (IS_ERR(wacom->battery))
-			return PTR_ERR(wacom->battery);
-
-		power_supply_powers(wacom->battery, &wacom->hdev->dev);
-
-		wacom->ac = power_supply_register(&wacom->hdev->dev,
-						  &wacom->ac_desc,
-						  &psy_cfg);
-		if (IS_ERR(wacom->ac)) {
-			power_supply_unregister(wacom->battery);
-			return PTR_ERR(wacom->ac);
-		}
-
-		power_supply_powers(wacom->ac, &wacom->hdev->dev);
-	}
+	if (wacom->wacom_wac.features.quirks & WACOM_QUIRK_BATTERY)
+		return __wacom_initialize_battery(wacom, &wacom->battery);
 
 	return 0;
 }
 
 static void wacom_destroy_battery(struct wacom *wacom)
 {
-	if (wacom->battery) {
-		power_supply_unregister(wacom->battery);
-		wacom->battery = NULL;
-		power_supply_unregister(wacom->ac);
-		wacom->ac = NULL;
+	if (wacom->battery.battery) {
+		devres_release_group(&wacom->hdev->dev,
+				     &wacom->battery.bat_desc);
+		wacom->battery.battery = NULL;
 	}
 }
 
@@ -1179,7 +1544,7 @@ static ssize_t wacom_show_remote_mode(struct kobject *kobj,
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	u8 mode;
 
-	mode = wacom->led.select[index];
+	mode = wacom->led.groups[index].select;
 	if (mode >= 0 && mode < 3)
 		return snprintf(buf, PAGE_SIZE, "%d\n", mode);
 	else
@@ -1212,54 +1577,30 @@ DEVICE_EKR_ATTR_GROUP(2);
 DEVICE_EKR_ATTR_GROUP(3);
 DEVICE_EKR_ATTR_GROUP(4);
 
-int wacom_remote_create_attr_group(struct wacom *wacom, __u32 serial, int index)
+static int wacom_remote_create_attr_group(struct wacom *wacom, __u32 serial,
+					  int index)
 {
 	int error = 0;
-	char *buf;
-	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_remote *remote = wacom->remote;
 
-	wacom_wac->serial[index] = serial;
-
-	buf = kzalloc(WAC_REMOTE_SERIAL_MAX_STRLEN, GFP_KERNEL);
-	if (!buf)
+	remote->remotes[index].group.name = devm_kasprintf(&wacom->hdev->dev,
+							  GFP_KERNEL,
+							  "%d", serial);
+	if (!remote->remotes[index].group.name)
 		return -ENOMEM;
-	snprintf(buf, WAC_REMOTE_SERIAL_MAX_STRLEN, "%d", serial);
-	wacom->remote_group[index].name = buf;
 
-	error = sysfs_create_group(wacom->remote_dir,
-				   &wacom->remote_group[index]);
+	error = __wacom_devm_sysfs_create_group(wacom, remote->remote_dir,
+						&remote->remotes[index].group);
 	if (error) {
+		remote->remotes[index].group.name = NULL;
 		hid_err(wacom->hdev,
 			"cannot create sysfs group err: %d\n", error);
-		kobject_put(wacom->remote_dir);
 		return error;
 	}
 
 	return 0;
 }
 
-void wacom_remote_destroy_attr_group(struct wacom *wacom, __u32 serial)
-{
-	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
-	int i;
-
-	if (!serial)
-		return;
-
-	for (i = 0; i < WACOM_MAX_REMOTES; i++) {
-		if (wacom_wac->serial[i] == serial) {
-			wacom_wac->serial[i] = 0;
-			wacom->led.select[i] = WACOM_STATUS_UNKNOWN;
-			if (wacom->remote_group[i].name) {
-				sysfs_remove_group(wacom->remote_dir,
-						   &wacom->remote_group[i]);
-				kfree(wacom->remote_group[i].name);
-				wacom->remote_group[i].name = NULL;
-			}
-		}
-	}
-}
-
 static int wacom_cmd_unpair_remote(struct wacom *wacom, unsigned char selector)
 {
 	const size_t buf_size = 2;
@@ -1316,27 +1657,57 @@ static const struct attribute *remote_unpair_attrs[] = {
 	NULL
 };
 
-static int wacom_initialize_remote(struct wacom *wacom)
+static void wacom_remotes_destroy(void *data)
+{
+	struct wacom *wacom = data;
+	struct wacom_remote *remote = wacom->remote;
+
+	if (!remote)
+		return;
+
+	kobject_put(remote->remote_dir);
+	kfifo_free(&remote->remote_fifo);
+	wacom->remote = NULL;
+}
+
+static int wacom_initialize_remotes(struct wacom *wacom)
 {
 	int error = 0;
-	struct wacom_wac *wacom_wac = &(wacom->wacom_wac);
+	struct wacom_remote *remote;
 	int i;
 
 	if (wacom->wacom_wac.features.type != REMOTE)
 		return 0;
 
-	wacom->remote_group[0] = remote0_serial_group;
-	wacom->remote_group[1] = remote1_serial_group;
-	wacom->remote_group[2] = remote2_serial_group;
-	wacom->remote_group[3] = remote3_serial_group;
-	wacom->remote_group[4] = remote4_serial_group;
+	remote = devm_kzalloc(&wacom->hdev->dev, sizeof(*wacom->remote),
+			      GFP_KERNEL);
+	if (!remote)
+		return -ENOMEM;
+
+	wacom->remote = remote;
 
-	wacom->remote_dir = kobject_create_and_add("wacom_remote",
-						   &wacom->hdev->dev.kobj);
-	if (!wacom->remote_dir)
+	spin_lock_init(&remote->remote_lock);
+
+	error = kfifo_alloc(&remote->remote_fifo,
+			5 * sizeof(struct wacom_remote_data),
+			GFP_KERNEL);
+	if (error) {
+		hid_err(wacom->hdev, "failed allocating remote_fifo\n");
 		return -ENOMEM;
+	}
 
-	error = sysfs_create_files(wacom->remote_dir, remote_unpair_attrs);
+	remote->remotes[0].group = remote0_serial_group;
+	remote->remotes[1].group = remote1_serial_group;
+	remote->remotes[2].group = remote2_serial_group;
+	remote->remotes[3].group = remote3_serial_group;
+	remote->remotes[4].group = remote4_serial_group;
+
+	remote->remote_dir = kobject_create_and_add("wacom_remote",
+						    &wacom->hdev->dev.kobj);
+	if (!remote->remote_dir)
+		return -ENOMEM;
+
+	error = sysfs_create_files(remote->remote_dir, remote_unpair_attrs);
 
 	if (error) {
 		hid_err(wacom->hdev,
@@ -1345,10 +1716,15 @@ static int wacom_initialize_remote(struct wacom *wacom)
 	}
 
 	for (i = 0; i < WACOM_MAX_REMOTES; i++) {
-		wacom->led.select[i] = WACOM_STATUS_UNKNOWN;
-		wacom_wac->serial[i] = 0;
+		wacom->led.groups[i].select = WACOM_STATUS_UNKNOWN;
+		remote->remotes[i].serial = 0;
 	}
 
+	error = devm_add_action_or_reset(&wacom->hdev->dev,
+					 wacom_remotes_destroy, wacom);
+	if (error)
+		return error;
+
 	return 0;
 }
 
@@ -1358,7 +1734,7 @@ static struct input_dev *wacom_allocate_input(struct wacom *wacom)
 	struct hid_device *hdev = wacom->hdev;
 	struct wacom_wac *wacom_wac = &(wacom->wacom_wac);
 
-	input_dev = input_allocate_device();
+	input_dev = devm_input_allocate_device(&hdev->dev);
 	if (!input_dev)
 		return NULL;
 
@@ -1377,36 +1753,6 @@ static struct input_dev *wacom_allocate_input(struct wacom *wacom)
 	return input_dev;
 }
 
-static void wacom_clean_inputs(struct wacom *wacom)
-{
-	if (wacom->wacom_wac.pen_input) {
-		if (wacom->wacom_wac.pen_registered)
-			input_unregister_device(wacom->wacom_wac.pen_input);
-		else
-			input_free_device(wacom->wacom_wac.pen_input);
-	}
-	if (wacom->wacom_wac.touch_input) {
-		if (wacom->wacom_wac.touch_registered)
-			input_unregister_device(wacom->wacom_wac.touch_input);
-		else
-			input_free_device(wacom->wacom_wac.touch_input);
-	}
-	if (wacom->wacom_wac.pad_input) {
-		if (wacom->wacom_wac.pad_registered)
-			input_unregister_device(wacom->wacom_wac.pad_input);
-		else
-			input_free_device(wacom->wacom_wac.pad_input);
-	}
-	kobject_put(wacom->remote_dir);
-	wacom->wacom_wac.pen_input = NULL;
-	wacom->wacom_wac.touch_input = NULL;
-	wacom->wacom_wac.pad_input = NULL;
-	wacom->wacom_wac.pen_registered = false;
-	wacom->wacom_wac.touch_registered = false;
-	wacom->wacom_wac.pad_registered = false;
-	wacom_destroy_leds(wacom);
-}
-
 static int wacom_allocate_inputs(struct wacom *wacom)
 {
 	struct wacom_wac *wacom_wac = &(wacom->wacom_wac);
@@ -1414,10 +1760,10 @@ static int wacom_allocate_inputs(struct wacom *wacom)
 	wacom_wac->pen_input = wacom_allocate_input(wacom);
 	wacom_wac->touch_input = wacom_allocate_input(wacom);
 	wacom_wac->pad_input = wacom_allocate_input(wacom);
-	if (!wacom_wac->pen_input || !wacom_wac->touch_input || !wacom_wac->pad_input) {
-		wacom_clean_inputs(wacom);
+	if (!wacom_wac->pen_input ||
+	    !wacom_wac->touch_input ||
+	    !wacom_wac->pad_input)
 		return -ENOMEM;
-	}
 
 	wacom_wac->pen_input->name = wacom_wac->pen_name;
 	wacom_wac->touch_input->name = wacom_wac->touch_name;
@@ -1448,8 +1794,7 @@ static int wacom_register_inputs(struct wacom *wacom)
 	} else {
 		error = input_register_device(pen_input_dev);
 		if (error)
-			goto fail_register_pen_input;
-		wacom_wac->pen_registered = true;
+			goto fail;
 	}
 
 	error = wacom_setup_touch_input_capabilities(touch_input_dev, wacom_wac);
@@ -1461,8 +1806,7 @@ static int wacom_register_inputs(struct wacom *wacom)
 	} else {
 		error = input_register_device(touch_input_dev);
 		if (error)
-			goto fail_register_touch_input;
-		wacom_wac->touch_registered = true;
+			goto fail;
 	}
 
 	error = wacom_setup_pad_input_capabilities(pad_input_dev, wacom_wac);
@@ -1474,37 +1818,15 @@ static int wacom_register_inputs(struct wacom *wacom)
 	} else {
 		error = input_register_device(pad_input_dev);
 		if (error)
-			goto fail_register_pad_input;
-		wacom_wac->pad_registered = true;
-
-		error = wacom_initialize_leds(wacom);
-		if (error)
-			goto fail_leds;
-
-		error = wacom_initialize_remote(wacom);
-		if (error)
-			goto fail_remote;
+			goto fail;
 	}
 
 	return 0;
 
-fail_remote:
-	wacom_destroy_leds(wacom);
-fail_leds:
-	input_unregister_device(pad_input_dev);
-	pad_input_dev = NULL;
-	wacom_wac->pad_registered = false;
-fail_register_pad_input:
-	if (touch_input_dev)
-		input_unregister_device(touch_input_dev);
+fail:
+	wacom_wac->pad_input = NULL;
 	wacom_wac->touch_input = NULL;
-	wacom_wac->touch_registered = false;
-fail_register_touch_input:
-	if (pen_input_dev)
-		input_unregister_device(pen_input_dev);
 	wacom_wac->pen_input = NULL;
-	wacom_wac->pen_registered = false;
-fail_register_pen_input:
 	return error;
 }
 
@@ -1543,14 +1865,14 @@ static void wacom_calculate_res(struct wacom_features *features)
 
 void wacom_battery_work(struct work_struct *work)
 {
-	struct wacom *wacom = container_of(work, struct wacom, work);
+	struct wacom *wacom = container_of(work, struct wacom, battery_work);
 
 	if ((wacom->wacom_wac.features.quirks & WACOM_QUIRK_BATTERY) &&
-	     !wacom->battery) {
+	     !wacom->battery.battery) {
 		wacom_initialize_battery(wacom);
 	}
 	else if (!(wacom->wacom_wac.features.quirks & WACOM_QUIRK_BATTERY) &&
-		 wacom->battery) {
+		 wacom->battery.battery) {
 		wacom_destroy_battery(wacom);
 	}
 }
@@ -1606,6 +1928,9 @@ static void wacom_update_name(struct wacom *wacom, const char *suffix)
 		strlcpy(name, features->name, sizeof(name));
 	}
 
+	snprintf(wacom_wac->name, sizeof(wacom_wac->name), "%s%s",
+		 name, suffix);
+
 	/* Append the device type to the name */
 	snprintf(wacom_wac->pen_name, sizeof(wacom_wac->pen_name),
 		"%s%s Pen", name, suffix);
@@ -1615,6 +1940,22 @@ static void wacom_update_name(struct wacom *wacom, const char *suffix)
 		"%s%s Pad", name, suffix);
 }
 
+static void wacom_release_resources(struct wacom *wacom)
+{
+	struct hid_device *hdev = wacom->hdev;
+
+	if (!wacom->resources)
+		return;
+
+	devres_release_group(&hdev->dev, wacom);
+
+	wacom->resources = false;
+
+	wacom->wacom_wac.pen_input = NULL;
+	wacom->wacom_wac.touch_input = NULL;
+	wacom->wacom_wac.pad_input = NULL;
+}
+
 static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 {
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
@@ -1627,9 +1968,14 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 	if (features->pktlen > WACOM_PKGLEN_MAX)
 		return -EINVAL;
 
+	if (!devres_open_group(&hdev->dev, wacom, GFP_KERNEL))
+		return -ENOMEM;
+
+	wacom->resources = true;
+
 	error = wacom_allocate_inputs(wacom);
 	if (error)
-		return error;
+		goto fail;
 
 	/*
 	 * Bamboo Pad has a generic hid handling for the Pen, and we switch it
@@ -1642,7 +1988,7 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 		} else if ((features->pktlen != WACOM_PKGLEN_BPAD_TOUCH) &&
 			   (features->pktlen != WACOM_PKGLEN_BPAD_TOUCH_USB)) {
 			error = -ENODEV;
-			goto fail_allocate_inputs;
+			goto fail;
 		}
 	}
 
@@ -1662,7 +2008,7 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 			 error ? "Ignoring" : "Assuming pen");
 
 		if (error)
-			goto fail_parsed;
+			goto fail;
 
 		features->device_type |= WACOM_DEVICETYPE_PEN;
 	}
@@ -1673,18 +2019,28 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 
 	error = wacom_add_shared_data(hdev);
 	if (error)
-		goto fail_shared_data;
+		goto fail;
 
 	if (!(features->device_type & WACOM_DEVICETYPE_WL_MONITOR) &&
 	     (features->quirks & WACOM_QUIRK_BATTERY)) {
 		error = wacom_initialize_battery(wacom);
 		if (error)
-			goto fail_battery;
+			goto fail;
 	}
 
 	error = wacom_register_inputs(wacom);
 	if (error)
-		goto fail_register_inputs;
+		goto fail;
+
+	if (wacom->wacom_wac.features.device_type & WACOM_DEVICETYPE_PAD) {
+		error = wacom_initialize_leds(wacom);
+		if (error)
+			goto fail;
+
+		error = wacom_initialize_remotes(wacom);
+		if (error)
+			goto fail;
+	}
 
 	if (features->type == HID_GENERIC)
 		connect_mask |= HID_CONNECT_DRIVER;
@@ -1693,7 +2049,7 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 	error = hid_hw_start(hdev, connect_mask);
 	if (error) {
 		hid_err(hdev, "hw start failed\n");
-		goto fail_hw_start;
+		goto fail;
 	}
 
 	if (!wireless) {
@@ -1705,7 +2061,7 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 	if ((features->type == BAMBOO_TOUCH) &&
 	    (features->device_type & WACOM_DEVICETYPE_PEN)) {
 		error = -ENODEV;
-		goto fail_hw_start;
+		goto fail_quirks;
 	}
 
 	/* pen only Bamboo neither support touch nor pad */
@@ -1713,37 +2069,33 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 	    ((features->device_type & WACOM_DEVICETYPE_TOUCH) ||
 	    (features->device_type & WACOM_DEVICETYPE_PAD))) {
 		error = -ENODEV;
-		goto fail_hw_start;
+		goto fail_quirks;
 	}
 
 	if (features->device_type & WACOM_DEVICETYPE_WL_MONITOR)
 		error = hid_hw_open(hdev);
 
 	if ((wacom_wac->features.type == INTUOSHT ||
-	    wacom_wac->features.type == INTUOSHT2) &&
+	     wacom_wac->features.type == INTUOSHT2) &&
 	    (wacom_wac->features.device_type & WACOM_DEVICETYPE_TOUCH)) {
-			wacom_wac->shared->touch_input = wacom_wac->touch_input;
+		wacom_wac->shared->type = wacom_wac->features.type;
+		wacom_wac->shared->touch_input = wacom_wac->touch_input;
 	}
 
+	devres_close_group(&hdev->dev, wacom);
+
 	return 0;
 
-fail_hw_start:
+fail_quirks:
 	hid_hw_stop(hdev);
-fail_register_inputs:
-	wacom_clean_inputs(wacom);
-	wacom_destroy_battery(wacom);
-fail_battery:
-	wacom_remove_shared_data(wacom);
-fail_shared_data:
-fail_parsed:
-fail_allocate_inputs:
-	wacom_clean_inputs(wacom);
+fail:
+	wacom_release_resources(wacom);
 	return error;
 }
 
 static void wacom_wireless_work(struct work_struct *work)
 {
-	struct wacom *wacom = container_of(work, struct wacom, work);
+	struct wacom *wacom = container_of(work, struct wacom, wireless_work);
 	struct usb_device *usbdev = wacom->usbdev;
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
 	struct hid_device *hdev1, *hdev2;
@@ -1762,17 +2114,16 @@ static void wacom_wireless_work(struct work_struct *work)
 	hdev1 = usb_get_intfdata(usbdev->config->interface[1]);
 	wacom1 = hid_get_drvdata(hdev1);
 	wacom_wac1 = &(wacom1->wacom_wac);
-	wacom_clean_inputs(wacom1);
+	wacom_release_resources(wacom1);
 
 	/* Touch interface */
 	hdev2 = usb_get_intfdata(usbdev->config->interface[2]);
 	wacom2 = hid_get_drvdata(hdev2);
 	wacom_wac2 = &(wacom2->wacom_wac);
-	wacom_clean_inputs(wacom2);
+	wacom_release_resources(wacom2);
 
 	if (wacom_wac->pid == 0) {
 		hid_info(wacom->hdev, "wireless tablet disconnected\n");
-		wacom_wac1->shared->type = 0;
 	} else {
 		const struct hid_device_id *id = wacom_ids;
 
@@ -1814,6 +2165,8 @@ static void wacom_wireless_work(struct work_struct *work)
 				goto fail;
 		}
 
+		strlcpy(wacom_wac->name, wacom_wac1->name,
+			sizeof(wacom_wac->name));
 		error = wacom_initialize_battery(wacom);
 		if (error)
 			goto fail;
@@ -1822,11 +2175,177 @@ static void wacom_wireless_work(struct work_struct *work)
 	return;
 
 fail:
-	wacom_clean_inputs(wacom1);
-	wacom_clean_inputs(wacom2);
+	wacom_release_resources(wacom1);
+	wacom_release_resources(wacom2);
 	return;
 }
 
+static void wacom_remote_destroy_one(struct wacom *wacom, unsigned int index)
+{
+	struct wacom_remote *remote = wacom->remote;
+	u32 serial = remote->remotes[index].serial;
+	int i;
+	unsigned long flags;
+
+	spin_lock_irqsave(&remote->remote_lock, flags);
+	remote->remotes[index].registered = false;
+	spin_unlock_irqrestore(&remote->remote_lock, flags);
+
+	if (remote->remotes[index].battery.battery)
+		devres_release_group(&wacom->hdev->dev,
+				     &remote->remotes[index].battery.bat_desc);
+
+	if (remote->remotes[index].group.name)
+		devres_release_group(&wacom->hdev->dev,
+				     &remote->remotes[index]);
+
+	for (i = 0; i < WACOM_MAX_REMOTES; i++) {
+		if (remote->remotes[i].serial == serial) {
+			remote->remotes[i].serial = 0;
+			remote->remotes[i].group.name = NULL;
+			remote->remotes[i].registered = false;
+			remote->remotes[i].battery.battery = NULL;
+			wacom->led.groups[i].select = WACOM_STATUS_UNKNOWN;
+		}
+	}
+}
+
+static int wacom_remote_create_one(struct wacom *wacom, u32 serial,
+				   unsigned int index)
+{
+	struct wacom_remote *remote = wacom->remote;
+	struct device *dev = &wacom->hdev->dev;
+	int error, k;
+
+	/* A remote can pair more than once with an EKR,
+	 * check to make sure this serial isn't already paired.
+	 */
+	for (k = 0; k < WACOM_MAX_REMOTES; k++) {
+		if (remote->remotes[k].serial == serial)
+			break;
+	}
+
+	if (k < WACOM_MAX_REMOTES) {
+		remote->remotes[index].serial = serial;
+		return 0;
+	}
+
+	if (!devres_open_group(dev, &remote->remotes[index], GFP_KERNEL))
+		return -ENOMEM;
+
+	error = wacom_remote_create_attr_group(wacom, serial, index);
+	if (error)
+		goto fail;
+
+	remote->remotes[index].input = wacom_allocate_input(wacom);
+	if (!remote->remotes[index].input) {
+		error = -ENOMEM;
+		goto fail;
+	}
+	remote->remotes[index].input->uniq = remote->remotes[index].group.name;
+	remote->remotes[index].input->name = wacom->wacom_wac.pad_name;
+
+	if (!remote->remotes[index].input->name) {
+		error = -EINVAL;
+		goto fail;
+	}
+
+	error = wacom_setup_pad_input_capabilities(remote->remotes[index].input,
+						   &wacom->wacom_wac);
+	if (error)
+		goto fail;
+
+	remote->remotes[index].serial = serial;
+
+	error = input_register_device(remote->remotes[index].input);
+	if (error)
+		goto fail;
+
+	error = wacom_led_groups_alloc_and_register_one(
+					&remote->remotes[index].input->dev,
+					wacom, index, 3, true);
+	if (error)
+		goto fail;
+
+	remote->remotes[index].registered = true;
+
+	devres_close_group(dev, &remote->remotes[index]);
+	return 0;
+
+fail:
+	devres_release_group(dev, &remote->remotes[index]);
+	remote->remotes[index].serial = 0;
+	return error;
+}
+
+static int wacom_remote_attach_battery(struct wacom *wacom, int index)
+{
+	struct wacom_remote *remote = wacom->remote;
+	int error;
+
+	if (!remote->remotes[index].registered)
+		return 0;
+
+	if (remote->remotes[index].battery.battery)
+		return 0;
+
+	if (wacom->led.groups[index].select == WACOM_STATUS_UNKNOWN)
+		return 0;
+
+	error = __wacom_initialize_battery(wacom,
+					&wacom->remote->remotes[index].battery);
+	if (error)
+		return error;
+
+	return 0;
+}
+
+static void wacom_remote_work(struct work_struct *work)
+{
+	struct wacom *wacom = container_of(work, struct wacom, remote_work);
+	struct wacom_remote *remote = wacom->remote;
+	struct wacom_remote_data data;
+	unsigned long flags;
+	unsigned int count;
+	u32 serial;
+	int i;
+
+	spin_lock_irqsave(&remote->remote_lock, flags);
+
+	count = kfifo_out(&remote->remote_fifo, &data, sizeof(data));
+
+	if (count != sizeof(data)) {
+		hid_err(wacom->hdev,
+			"workitem triggered without status available\n");
+		spin_unlock_irqrestore(&remote->remote_lock, flags);
+		return;
+	}
+
+	if (!kfifo_is_empty(&remote->remote_fifo))
+		wacom_schedule_work(&wacom->wacom_wac, WACOM_WORKER_REMOTE);
+
+	spin_unlock_irqrestore(&remote->remote_lock, flags);
+
+	for (i = 0; i < WACOM_MAX_REMOTES; i++) {
+		serial = data.remote[i].serial;
+		if (data.remote[i].connected) {
+
+			if (remote->remotes[i].serial == serial) {
+				wacom_remote_attach_battery(wacom, i);
+				continue;
+			}
+
+			if (remote->remotes[i].serial)
+				wacom_remote_destroy_one(wacom, i);
+
+			wacom_remote_create_one(wacom, serial, i);
+
+		} else if (remote->remotes[i].serial) {
+			wacom_remote_destroy_one(wacom, i);
+		}
+	}
+}
+
 static int wacom_probe(struct hid_device *hdev,
 		const struct hid_device_id *id)
 {
@@ -1845,7 +2364,7 @@ static int wacom_probe(struct hid_device *hdev,
 	/* hid-core sets this quirk for the boot interface */
 	hdev->quirks &= ~HID_QUIRK_NOGET;
 
-	wacom = kzalloc(sizeof(struct wacom), GFP_KERNEL);
+	wacom = devm_kzalloc(&hdev->dev, sizeof(struct wacom), GFP_KERNEL);
 	if (!wacom)
 		return -ENOMEM;
 
@@ -1858,7 +2377,7 @@ static int wacom_probe(struct hid_device *hdev,
 
 	if (features->check_for_hid_type && features->hid_type != hdev->type) {
 		error = -ENODEV;
-		goto fail_type;
+		goto fail;
 	}
 
 	wacom_wac->hid_data.inputmode = -1;
@@ -1867,18 +2386,20 @@ static int wacom_probe(struct hid_device *hdev,
 	wacom->usbdev = dev;
 	wacom->intf = intf;
 	mutex_init(&wacom->lock);
-	INIT_WORK(&wacom->work, wacom_wireless_work);
+	INIT_WORK(&wacom->wireless_work, wacom_wireless_work);
+	INIT_WORK(&wacom->battery_work, wacom_battery_work);
+	INIT_WORK(&wacom->remote_work, wacom_remote_work);
 
 	/* ask for the report descriptor to be loaded by HID */
 	error = hid_parse(hdev);
 	if (error) {
 		hid_err(hdev, "parse failed\n");
-		goto fail_parse;
+		goto fail;
 	}
 
 	error = wacom_parse_and_register(wacom, false);
 	if (error)
-		goto fail_parse;
+		goto fail;
 
 	if (hdev->bus == BUS_BLUETOOTH) {
 		error = device_create_file(&hdev->dev, &dev_attr_speed);
@@ -1890,9 +2411,7 @@ static int wacom_probe(struct hid_device *hdev,
 
 	return 0;
 
-fail_type:
-fail_parse:
-	kfree(wacom);
+fail:
 	hid_set_drvdata(hdev, NULL);
 	return error;
 }
@@ -1908,15 +2427,13 @@ static void wacom_remove(struct hid_device *hdev)
 
 	hid_hw_stop(hdev);
 
-	cancel_work_sync(&wacom->work);
-	wacom_clean_inputs(wacom);
+	cancel_work_sync(&wacom->wireless_work);
+	cancel_work_sync(&wacom->battery_work);
+	cancel_work_sync(&wacom->remote_work);
 	if (hdev->bus == BUS_BLUETOOTH)
 		device_remove_file(&hdev->dev, &dev_attr_speed);
-	wacom_destroy_battery(wacom);
-	wacom_remove_shared_data(wacom);
 
 	hid_set_drvdata(hdev, NULL);
-	kfree(wacom);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index 1eae13cdc502..1cb79925730d 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -34,6 +34,10 @@
  */
 #define WACOM_CONTACT_AREA_SCALE 2607
 
+static bool touch_arbitration = 1;
+module_param(touch_arbitration, bool, 0644);
+MODULE_PARM_DESC(touch_arbitration, " on (Y) off (N)");
+
 static void wacom_report_numbered_buttons(struct input_dev *input_dev,
 				int button_count, int mask);
 
@@ -48,25 +52,34 @@ static unsigned short batcap_gr[8] = { 1, 15, 25, 35, 50, 70, 100, 100 };
  */
 static unsigned short batcap_i4[8] = { 1, 15, 30, 45, 60, 70, 85, 100 };
 
+static void __wacom_notify_battery(struct wacom_battery *battery,
+				   int bat_capacity, bool bat_charging,
+				   bool bat_connected, bool ps_connected)
+{
+	bool changed = battery->battery_capacity != bat_capacity  ||
+		       battery->bat_charging     != bat_charging  ||
+		       battery->bat_connected    != bat_connected ||
+		       battery->ps_connected     != ps_connected;
+
+	if (changed) {
+		battery->battery_capacity = bat_capacity;
+		battery->bat_charging = bat_charging;
+		battery->bat_connected = bat_connected;
+		battery->ps_connected = ps_connected;
+
+		if (battery->battery)
+			power_supply_changed(battery->battery);
+	}
+}
+
 static void wacom_notify_battery(struct wacom_wac *wacom_wac,
 	int bat_capacity, bool bat_charging, bool bat_connected,
 	bool ps_connected)
 {
 	struct wacom *wacom = container_of(wacom_wac, struct wacom, wacom_wac);
-	bool changed = wacom_wac->battery_capacity != bat_capacity  ||
-		       wacom_wac->bat_charging     != bat_charging  ||
-		       wacom_wac->bat_connected    != bat_connected ||
-		       wacom_wac->ps_connected     != ps_connected;
 
-	if (changed) {
-		wacom_wac->battery_capacity = bat_capacity;
-		wacom_wac->bat_charging = bat_charging;
-		wacom_wac->bat_connected = bat_connected;
-		wacom_wac->ps_connected = ps_connected;
-
-		if (wacom->battery)
-			power_supply_changed(wacom->battery);
-	}
+	__wacom_notify_battery(&wacom->battery, bat_capacity, bat_charging,
+			       bat_connected, ps_connected);
 }
 
 static int wacom_penpartner_irq(struct wacom_wac *wacom)
@@ -751,22 +764,37 @@ static int wacom_intuos_inout(struct wacom_wac *wacom)
 static int wacom_remote_irq(struct wacom_wac *wacom_wac, size_t len)
 {
 	unsigned char *data = wacom_wac->data;
-	struct input_dev *input = wacom_wac->pad_input;
+	struct input_dev *input;
 	struct wacom *wacom = container_of(wacom_wac, struct wacom, wacom_wac);
-	struct wacom_features *features = &wacom_wac->features;
+	struct wacom_remote *remote = wacom->remote;
 	int bat_charging, bat_percent, touch_ring_mode;
 	__u32 serial;
-	int i;
+	int i, index = -1;
+	unsigned long flags;
 
 	if (data[0] != WACOM_REPORT_REMOTE) {
-		dev_dbg(input->dev.parent,
-			"%s: received unknown report #%d", __func__, data[0]);
+		hid_dbg(wacom->hdev, "%s: received unknown report #%d",
+			__func__, data[0]);
 		return 0;
 	}
 
 	serial = data[3] + (data[4] << 8) + (data[5] << 16);
 	wacom_wac->id[0] = PAD_DEVICE_ID;
 
+	spin_lock_irqsave(&remote->remote_lock, flags);
+
+	for (i = 0; i < WACOM_MAX_REMOTES; i++) {
+		if (remote->remotes[i].serial == serial) {
+			index = i;
+			break;
+		}
+	}
+
+	if (index < 0 || !remote->remotes[index].registered)
+		goto out;
+
+	input = remote->remotes[index].input;
+
 	input_report_key(input, BTN_0, (data[9] & 0x01));
 	input_report_key(input, BTN_1, (data[9] & 0x02));
 	input_report_key(input, BTN_2, (data[9] & 0x04));
@@ -803,73 +831,69 @@ static int wacom_remote_irq(struct wacom_wac *wacom_wac, size_t len)
 
 	input_event(input, EV_MSC, MSC_SERIAL, serial);
 
+	input_sync(input);
+
 	/*Which mode select (LED light) is currently on?*/
 	touch_ring_mode = (data[11] & 0xC0) >> 6;
 
 	for (i = 0; i < WACOM_MAX_REMOTES; i++) {
-		if (wacom_wac->serial[i] == serial)
-			wacom->led.select[i] = touch_ring_mode;
-	}
-
-	if (!wacom->battery &&
-	    !(features->quirks & WACOM_QUIRK_BATTERY)) {
-		features->quirks |= WACOM_QUIRK_BATTERY;
-		INIT_WORK(&wacom->work, wacom_battery_work);
-		wacom_schedule_work(wacom_wac);
+		if (remote->remotes[i].serial == serial)
+			wacom->led.groups[i].select = touch_ring_mode;
 	}
 
-	wacom_notify_battery(wacom_wac, bat_percent, bat_charging, 1,
-			     bat_charging);
+	__wacom_notify_battery(&remote->remotes[index].battery, bat_percent,
+				bat_charging, 1, bat_charging);
 
-	return 1;
+out:
+	spin_unlock_irqrestore(&remote->remote_lock, flags);
+	return 0;
 }
 
-static int wacom_remote_status_irq(struct wacom_wac *wacom_wac, size_t len)
+static void wacom_remote_status_irq(struct wacom_wac *wacom_wac, size_t len)
 {
 	struct wacom *wacom = container_of(wacom_wac, struct wacom, wacom_wac);
 	unsigned char *data = wacom_wac->data;
-	int i;
+	struct wacom_remote *remote = wacom->remote;
+	struct wacom_remote_data remote_data;
+	unsigned long flags;
+	int i, ret;
 
 	if (data[0] != WACOM_REPORT_DEVICE_LIST)
-		return 0;
+		return;
+
+	memset(&remote_data, 0, sizeof(struct wacom_remote_data));
 
 	for (i = 0; i < WACOM_MAX_REMOTES; i++) {
 		int j = i * 6;
 		int serial = (data[j+6] << 16) + (data[j+5] << 8) + data[j+4];
 		bool connected = data[j+2];
 
-		if (connected) {
-			int k;
+		remote_data.remote[i].serial = serial;
+		remote_data.remote[i].connected = connected;
+	}
 
-			if (wacom_wac->serial[i] == serial)
-				continue;
+	spin_lock_irqsave(&remote->remote_lock, flags);
 
-			if (wacom_wac->serial[i]) {
-				wacom_remote_destroy_attr_group(wacom,
-							wacom_wac->serial[i]);
-			}
+	ret = kfifo_in(&remote->remote_fifo, &remote_data, sizeof(remote_data));
+	if (ret != sizeof(remote_data)) {
+		spin_unlock_irqrestore(&remote->remote_lock, flags);
+		hid_err(wacom->hdev, "Can't queue Remote status event.\n");
+		return;
+	}
 
-			/* A remote can pair more than once with an EKR,
-			 * check to make sure this serial isn't already paired.
-			 */
-			for (k = 0; k < WACOM_MAX_REMOTES; k++) {
-				if (wacom_wac->serial[k] == serial)
-					break;
-			}
+	spin_unlock_irqrestore(&remote->remote_lock, flags);
 
-			if (k < WACOM_MAX_REMOTES) {
-				wacom_wac->serial[i] = serial;
-				continue;
-			}
-			wacom_remote_create_attr_group(wacom, serial, i);
+	wacom_schedule_work(wacom_wac, WACOM_WORKER_REMOTE);
+}
 
-		} else if (wacom_wac->serial[i]) {
-			wacom_remote_destroy_attr_group(wacom,
-							wacom_wac->serial[i]);
-		}
-	}
+static inline bool report_touch_events(struct wacom_wac *wacom)
+{
+	return (touch_arbitration ? !wacom->shared->stylus_in_proximity : 1);
+}
 
-	return 0;
+static inline bool delay_pen_events(struct wacom_wac *wacom)
+{
+	return (wacom->shared->touch_down && touch_arbitration);
 }
 
 static int wacom_intuos_general(struct wacom_wac *wacom)
@@ -885,7 +909,7 @@ static int wacom_intuos_general(struct wacom_wac *wacom)
 		data[0] != WACOM_REPORT_INTUOS_PEN)
 		return 0;
 
-	if (wacom->shared->touch_down)
+	if (delay_pen_events(wacom))
 		return 1;
 
 	/* don't report events if we don't know the tool ID */
@@ -1145,7 +1169,7 @@ static int wacom_wac_finger_count_touches(struct wacom_wac *wacom)
 
 	if (touch_max == 1)
 		return test_bit(BTN_TOUCH, input->key) &&
-		       !wacom->shared->stylus_in_proximity;
+			report_touch_events(wacom);
 
 	for (i = 0; i < input->mt->num_slots; i++) {
 		struct input_mt_slot *ps = &input->mt->slots[i];
@@ -1186,7 +1210,7 @@ static int wacom_24hdt_irq(struct wacom_wac *wacom)
 
 	for (i = 0; i < contacts_to_send; i++) {
 		int offset = (byte_per_packet * i) + 1;
-		bool touch = (data[offset] & 0x1) && !wacom->shared->stylus_in_proximity;
+		bool touch = (data[offset] & 0x1) && report_touch_events(wacom);
 		int slot = input_mt_get_slot_by_key(input, data[offset + 1]);
 
 		if (slot < 0)
@@ -1250,7 +1274,7 @@ static int wacom_mt_touch(struct wacom_wac *wacom)
 
 	for (i = 0; i < contacts_to_send; i++) {
 		int offset = (WACOM_BYTES_PER_MT_PACKET + x_offset) * i + 3;
-		bool touch = (data[offset] & 0x1) && !wacom->shared->stylus_in_proximity;
+		bool touch = (data[offset] & 0x1) && report_touch_events(wacom);
 		int id = get_unaligned_le16(&data[offset + 1]);
 		int slot = input_mt_get_slot_by_key(input, id);
 
@@ -1284,7 +1308,7 @@ static int wacom_tpc_mt_touch(struct wacom_wac *wacom)
 
 	for (i = 0; i < 2; i++) {
 		int p = data[1] & (1 << i);
-		bool touch = p && !wacom->shared->stylus_in_proximity;
+		bool touch = p && report_touch_events(wacom);
 
 		input_mt_slot(input, i);
 		input_mt_report_slot_state(input, MT_TOOL_FINGER, touch);
@@ -1308,7 +1332,7 @@ static int wacom_tpc_single_touch(struct wacom_wac *wacom, size_t len)
 {
 	unsigned char *data = wacom->data;
 	struct input_dev *input = wacom->touch_input;
-	bool prox = !wacom->shared->stylus_in_proximity;
+	bool prox = report_touch_events(wacom);
 	int x = 0, y = 0;
 
 	if (wacom->features.touch_max > 1 || len > WACOM_PKGLEN_TPC2FG)
@@ -1353,8 +1377,10 @@ static int wacom_tpc_pen(struct wacom_wac *wacom)
 	/* keep pen state for touch events */
 	wacom->shared->stylus_in_proximity = prox;
 
-	/* send pen events only when touch is up or forced out */
-	if (!wacom->shared->touch_down) {
+	/* send pen events only when touch is up or forced out
+	 * or touch arbitration is off
+	 */
+	if (!delay_pen_events(wacom)) {
 		input_report_key(input, BTN_STYLUS, data[1] & 0x02);
 		input_report_key(input, BTN_STYLUS2, data[1] & 0x10);
 		input_report_abs(input, ABS_X, le16_to_cpup((__le16 *)&data[2]));
@@ -1496,8 +1522,10 @@ static int wacom_wac_pen_event(struct hid_device *hdev, struct hid_field *field,
 		return 0;
 	}
 
-	/* send pen events only when touch is up or forced out */
-	if (!usage->type || wacom_wac->shared->touch_down)
+	/* send pen events only when touch is up or forced out
+	 * or touch arbitration is off
+	 */
+	if (!usage->type || delay_pen_events(wacom_wac))
 		return 0;
 
 	input_event(input, usage->type, usage->code, value);
@@ -1527,8 +1555,7 @@ static void wacom_wac_pen_report(struct hid_device *hdev,
 	/* keep pen state for touch events */
 	wacom_wac->shared->stylus_in_proximity = prox;
 
-	/* send pen events only when touch is up or forced out */
-	if (!wacom_wac->shared->touch_down) {
+	if (!delay_pen_events(wacom_wac)) {
 		input_report_key(input, BTN_TOUCH,
 				wacom_wac->hid_data.tipswitch);
 		input_report_key(input, wacom_wac->tool[0], prox);
@@ -1544,13 +1571,11 @@ static void wacom_wac_finger_usage_mapping(struct hid_device *hdev,
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
-	struct wacom_features *features = &wacom_wac->features;
 	struct input_dev *input = wacom_wac->touch_input;
 	unsigned touch_max = wacom_wac->features.touch_max;
 
 	switch (usage->hid) {
 	case HID_GD_X:
-		features->last_slot_field = usage->hid;
 		if (touch_max == 1)
 			wacom_map_usage(input, usage, field, EV_ABS, ABS_X, 4);
 		else
@@ -1558,7 +1583,6 @@ static void wacom_wac_finger_usage_mapping(struct hid_device *hdev,
 					ABS_MT_POSITION_X, 4);
 		break;
 	case HID_GD_Y:
-		features->last_slot_field = usage->hid;
 		if (touch_max == 1)
 			wacom_map_usage(input, usage, field, EV_ABS, ABS_Y, 4);
 		else
@@ -1567,22 +1591,11 @@ static void wacom_wac_finger_usage_mapping(struct hid_device *hdev,
 		break;
 	case HID_DG_WIDTH:
 	case HID_DG_HEIGHT:
-		features->last_slot_field = usage->hid;
 		wacom_map_usage(input, usage, field, EV_ABS, ABS_MT_TOUCH_MAJOR, 0);
 		wacom_map_usage(input, usage, field, EV_ABS, ABS_MT_TOUCH_MINOR, 0);
 		input_set_abs_params(input, ABS_MT_ORIENTATION, 0, 1, 0, 0);
 		break;
-	case HID_DG_CONTACTID:
-		features->last_slot_field = usage->hid;
-		break;
-	case HID_DG_INRANGE:
-		features->last_slot_field = usage->hid;
-		break;
-	case HID_DG_INVERT:
-		features->last_slot_field = usage->hid;
-		break;
 	case HID_DG_TIPSWITCH:
-		features->last_slot_field = usage->hid;
 		wacom_map_usage(input, usage, field, EV_KEY, BTN_TOUCH, 0);
 		break;
 	case HID_DG_CONTACTCOUNT:
@@ -1599,7 +1612,7 @@ static void wacom_wac_finger_slot(struct wacom_wac *wacom_wac,
 	struct hid_data *hid_data = &wacom_wac->hid_data;
 	bool mt = wacom_wac->features.touch_max > 1;
 	bool prox = hid_data->tipswitch &&
-		    !wacom_wac->shared->stylus_in_proximity;
+		    report_touch_events(wacom_wac);
 
 	wacom_wac->hid_data.num_received++;
 	if (wacom_wac->hid_data.num_received > wacom_wac->hid_data.num_expected)
@@ -1660,7 +1673,7 @@ static int wacom_wac_finger_event(struct hid_device *hdev,
 
 
 	if (usage->usage_index + 1 == field->report_count) {
-		if (usage->hid == wacom_wac->features.last_slot_field)
+		if (usage->hid == wacom_wac->hid_data.last_slot_field)
 			wacom_wac_finger_slot(wacom_wac, wacom_wac->touch_input);
 	}
 
@@ -1673,31 +1686,35 @@ static void wacom_wac_finger_pre_report(struct hid_device *hdev,
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
 	struct hid_data* hid_data = &wacom_wac->hid_data;
+	int i;
 
-	if (hid_data->cc_report != 0 &&
-	    hid_data->cc_report != report->id) {
-		int i;
-
-		hid_data->cc_report = report->id;
-		hid_data->cc_index = -1;
-		hid_data->cc_value_index = -1;
-
-		for (i = 0; i < report->maxfield; i++) {
-			struct hid_field *field = report->field[i];
-			int j;
-
-			for (j = 0; j < field->maxusage; j++) {
-				if (field->usage[j].hid == HID_DG_CONTACTCOUNT) {
-					hid_data->cc_index = i;
-					hid_data->cc_value_index = j;
-
-					/* break */
-					i = report->maxfield;
-					j = field->maxusage;
-				}
+	for (i = 0; i < report->maxfield; i++) {
+		struct hid_field *field = report->field[i];
+		int j;
+
+		for (j = 0; j < field->maxusage; j++) {
+			struct hid_usage *usage = &field->usage[j];
+
+			switch (usage->hid) {
+			case HID_GD_X:
+			case HID_GD_Y:
+			case HID_DG_WIDTH:
+			case HID_DG_HEIGHT:
+			case HID_DG_CONTACTID:
+			case HID_DG_INRANGE:
+			case HID_DG_INVERT:
+			case HID_DG_TIPSWITCH:
+				hid_data->last_slot_field = usage->hid;
+				break;
+			case HID_DG_CONTACTCOUNT:
+				hid_data->cc_report = report->id;
+				hid_data->cc_index = i;
+				hid_data->cc_value_index = j;
+				break;
 			}
 		}
 	}
+
 	if (hid_data->cc_report != 0 &&
 	    hid_data->cc_index >= 0) {
 		struct hid_field *field = report->field[hid_data->cc_index];
@@ -1740,10 +1757,10 @@ void wacom_wac_usage_mapping(struct hid_device *hdev,
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_features *features = &wacom_wac->features;
 
 	/* currently, only direct devices have proper hid report descriptors */
-	__set_bit(INPUT_PROP_DIRECT, wacom_wac->pen_input->propbit);
-	__set_bit(INPUT_PROP_DIRECT, wacom_wac->touch_input->propbit);
+	features->device_type |= WACOM_DEVICETYPE_DIRECT;
 
 	if (WACOM_PEN_FIELD(field))
 		return wacom_wac_pen_usage_mapping(hdev, field, usage);
@@ -1825,15 +1842,8 @@ static int wacom_bpt_touch(struct wacom_wac *wacom)
 
 	for (i = 0; i < 2; i++) {
 		int offset = (data[1] & 0x80) ? (8 * i) : (9 * i);
-		bool touch = data[offset + 3] & 0x80;
-
-		/*
-		 * Touch events need to be disabled while stylus is
-		 * in proximity because user's hand is resting on touchpad
-		 * and sending unwanted events.  User expects tablet buttons
-		 * to continue working though.
-		 */
-		touch = touch && !wacom->shared->stylus_in_proximity;
+		bool touch = report_touch_events(wacom)
+			   && (data[offset + 3] & 0x80);
 
 		input_mt_slot(input, i);
 		input_mt_report_slot_state(input, MT_TOOL_FINGER, touch);
@@ -1870,7 +1880,7 @@ static void wacom_bpt3_touch_msg(struct wacom_wac *wacom, unsigned char *data)
 	if (slot < 0)
 		return;
 
-	touch = touch && !wacom->shared->stylus_in_proximity;
+	touch = touch && report_touch_events(wacom);
 
 	input_mt_slot(input, slot);
 	input_mt_report_slot_state(input, MT_TOOL_FINGER, touch);
@@ -1942,7 +1952,7 @@ static int wacom_bpt3_touch(struct wacom_wac *wacom)
 	}
 
 	/* only update touch if we actually have a touchpad and touch data changed */
-	if (wacom->touch_registered && touch_changed) {
+	if (wacom->touch_input && touch_changed) {
 		input_mt_sync_frame(wacom->touch_input);
 		wacom->shared->touch_down = wacom_wac_finger_count_touches(wacom);
 	}
@@ -1983,7 +1993,7 @@ static int wacom_bpt_pen(struct wacom_wac *wacom)
 	}
 
 	wacom->shared->stylus_in_proximity = prox;
-	if (wacom->shared->touch_down)
+	if (delay_pen_events(wacom))
 		return 0;
 
 	if (prox) {
@@ -2077,7 +2087,7 @@ static int wacom_bamboo_pad_touch_event(struct wacom_wac *wacom,
 
 	for (id = 0; id < wacom->features.touch_max; id++) {
 		valid = !!(prefix & BIT(id)) &&
-			!wacom->shared->stylus_in_proximity;
+			report_touch_events(wacom);
 
 		input_mt_slot(input, id);
 		input_mt_report_slot_state(input, MT_TOOL_FINGER, valid);
@@ -2099,8 +2109,7 @@ static int wacom_bamboo_pad_touch_event(struct wacom_wac *wacom,
 	input_report_key(input, BTN_RIGHT, prefix & 0x80);
 
 	/* keep touch state for pen event */
-	wacom->shared->touch_down = !!prefix &&
-				    !wacom->shared->stylus_in_proximity;
+	wacom->shared->touch_down = !!prefix && report_touch_events(wacom);
 
 	return 1;
 }
@@ -2149,16 +2158,15 @@ static int wacom_wireless_irq(struct wacom_wac *wacom, size_t len)
 		charging = !!(data[5] & 0x80);
 		if (wacom->pid != pid) {
 			wacom->pid = pid;
-			wacom_schedule_work(wacom);
+			wacom_schedule_work(wacom, WACOM_WORKER_WIRELESS);
 		}
 
-		if (wacom->shared->type)
-			wacom_notify_battery(wacom, battery, charging, 1, 0);
+		wacom_notify_battery(wacom, battery, charging, 1, 0);
 
 	} else if (wacom->pid != 0) {
 		/* disconnected while previously connected */
 		wacom->pid = 0;
-		wacom_schedule_work(wacom);
+		wacom_schedule_work(wacom, WACOM_WORKER_WIRELESS);
 		wacom_notify_battery(wacom, 0, 0, 0, 0);
 	}
 
@@ -2190,18 +2198,16 @@ static int wacom_status_irq(struct wacom_wac *wacom_wac, size_t len)
 		wacom_notify_battery(wacom_wac, battery, charging,
 				     battery || charging, 1);
 
-		if (!wacom->battery &&
+		if (!wacom->battery.battery &&
 		    !(features->quirks & WACOM_QUIRK_BATTERY)) {
 			features->quirks |= WACOM_QUIRK_BATTERY;
-			INIT_WORK(&wacom->work, wacom_battery_work);
-			wacom_schedule_work(wacom_wac);
+			wacom_schedule_work(wacom_wac, WACOM_WORKER_BATTERY);
 		}
 	}
 	else if ((features->quirks & WACOM_QUIRK_BATTERY) &&
-		 wacom->battery) {
+		 wacom->battery.battery) {
 		features->quirks &= ~WACOM_QUIRK_BATTERY;
-		INIT_WORK(&wacom->work, wacom_battery_work);
-		wacom_schedule_work(wacom_wac);
+		wacom_schedule_work(wacom_wac, WACOM_WORKER_BATTERY);
 		wacom_notify_battery(wacom_wac, 0, 0, 0, 0);
 	}
 	return 0;
@@ -2312,8 +2318,9 @@ void wacom_wac_irq(struct wacom_wac *wacom_wac, size_t len)
 		break;
 
 	case REMOTE:
+		sync = false;
 		if (wacom_wac->data[0] == WACOM_REPORT_DEVICE_LIST)
-			sync = wacom_remote_status_irq(wacom_wac, len);
+			wacom_remote_status_irq(wacom_wac, len);
 		else
 			sync = wacom_remote_irq(wacom_wac, len);
 		break;
@@ -2451,6 +2458,33 @@ void wacom_setup_device_quirks(struct wacom *wacom)
 	if (features->type == REMOTE)
 		features->device_type = WACOM_DEVICETYPE_PAD;
 
+	switch (features->type) {
+	case PL:
+	case DTU:
+	case DTUS:
+	case DTUSX:
+	case WACOM_21UX2:
+	case WACOM_22HD:
+	case DTK:
+	case WACOM_24HD:
+	case WACOM_27QHD:
+	case CINTIQ_HYBRID:
+	case CINTIQ_COMPANION_2:
+	case CINTIQ:
+	case WACOM_BEE:
+	case WACOM_13HD:
+	case WACOM_24HDT:
+	case WACOM_27QHDT:
+	case TABLETPC:
+	case TABLETPCE:
+	case TABLETPC2FG:
+	case MTSCREEN:
+	case MTTPC:
+	case MTTPC_B:
+		features->device_type |= WACOM_DEVICETYPE_DIRECT;
+		break;
+	}
+
 	if (wacom->hdev->bus == BUS_BLUETOOTH)
 		features->quirks |= WACOM_QUIRK_BATTERY;
 
@@ -2469,6 +2503,9 @@ void wacom_setup_device_quirks(struct wacom *wacom)
 			features->quirks |= WACOM_QUIRK_BATTERY;
 		}
 	}
+
+	if (features->type == REMOTE)
+		features->device_type |= WACOM_DEVICETYPE_WL_MONITOR;
 }
 
 int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
@@ -2481,6 +2518,11 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 	if (!(features->device_type & WACOM_DEVICETYPE_PEN))
 		return -ENODEV;
 
+	if (features->device_type & WACOM_DEVICETYPE_DIRECT)
+		__set_bit(INPUT_PROP_DIRECT, input_dev->propbit);
+	else
+		__set_bit(INPUT_PROP_POINTER, input_dev->propbit);
+
 	if (features->type == HID_GENERIC)
 		/* setup has already been done */
 		return 0;
@@ -2499,7 +2541,6 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 	input_abs_set_res(input_dev, ABS_X, features->x_resolution);
 	input_abs_set_res(input_dev, ABS_Y, features->y_resolution);
 
-
 	switch (features->type) {
 	case GRAPHIRE_BT:
 		__clear_bit(ABS_MISC, input_dev->absbit);
@@ -2523,8 +2564,6 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 		__set_bit(BTN_TOOL_MOUSE, input_dev->keybit);
 		__set_bit(BTN_STYLUS, input_dev->keybit);
 		__set_bit(BTN_STYLUS2, input_dev->keybit);
-
-		__set_bit(INPUT_PROP_POINTER, input_dev->propbit);
 		break;
 
 	case WACOM_27QHD:
@@ -2539,7 +2578,6 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 	case CINTIQ_COMPANION_2:
 		input_set_abs_params(input_dev, ABS_Z, -900, 899, 0, 0);
 		input_abs_set_res(input_dev, ABS_Z, 287);
-		__set_bit(INPUT_PROP_DIRECT, input_dev->propbit);
 		wacom_setup_cintiq(wacom_wac);
 		break;
 
@@ -2555,8 +2593,6 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 		/* fall through */
 
 	case INTUOS:
-		__set_bit(INPUT_PROP_POINTER, input_dev->propbit);
-
 		wacom_setup_intuos(wacom_wac);
 		break;
 
@@ -2566,8 +2602,6 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 	case INTUOSPL:
 	case INTUOS5S:
 	case INTUOSPS:
-		__set_bit(INPUT_PROP_POINTER, input_dev->propbit);
-
 		input_set_abs_params(input_dev, ABS_DISTANCE, 0,
 				      features->distance_max,
 				      features->distance_fuzz, 0);
@@ -2597,8 +2631,6 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 		__set_bit(BTN_TOOL_RUBBER, input_dev->keybit);
 		__set_bit(BTN_STYLUS, input_dev->keybit);
 		__set_bit(BTN_STYLUS2, input_dev->keybit);
-
-		__set_bit(INPUT_PROP_DIRECT, input_dev->propbit);
 		break;
 
 	case PTU:
@@ -2609,16 +2641,12 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 		__set_bit(BTN_TOOL_PEN, input_dev->keybit);
 		__set_bit(BTN_TOOL_RUBBER, input_dev->keybit);
 		__set_bit(BTN_STYLUS, input_dev->keybit);
-
-		__set_bit(INPUT_PROP_POINTER, input_dev->propbit);
 		break;
 
 	case INTUOSHT:
 	case BAMBOO_PT:
 	case BAMBOO_PEN:
 	case INTUOSHT2:
-		__set_bit(INPUT_PROP_POINTER, input_dev->propbit);
-
 		if (features->type == INTUOSHT2) {
 			wacom_setup_basic_pro_pen(wacom_wac);
 		} else {
@@ -2649,6 +2677,11 @@ int wacom_setup_touch_input_capabilities(struct input_dev *input_dev,
 	if (!(features->device_type & WACOM_DEVICETYPE_TOUCH))
 		return -ENODEV;
 
+	if (features->device_type & WACOM_DEVICETYPE_DIRECT)
+		__set_bit(INPUT_PROP_DIRECT, input_dev->propbit);
+	else
+		__set_bit(INPUT_PROP_POINTER, input_dev->propbit);
+
 	if (features->type == HID_GENERIC)
 		/* setup has already been done */
 		return 0;
@@ -2683,8 +2716,6 @@ int wacom_setup_touch_input_capabilities(struct input_dev *input_dev,
 	case INTUOSPL:
 	case INTUOS5S:
 	case INTUOSPS:
-		__set_bit(INPUT_PROP_POINTER, input_dev->propbit);
-
 		input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR, 0, features->x_max, 0, 0);
 		input_set_abs_params(input_dev, ABS_MT_TOUCH_MINOR, 0, features->y_max, 0, 0);
 		input_mt_init_slots(input_dev, features->touch_max, INPUT_MT_POINTER);
@@ -2707,7 +2738,6 @@ int wacom_setup_touch_input_capabilities(struct input_dev *input_dev,
 
 	case TABLETPC:
 	case TABLETPCE:
-		__set_bit(INPUT_PROP_DIRECT, input_dev->propbit);
 		break;
 
 	case INTUOSHT:
@@ -2752,11 +2782,105 @@ static void wacom_setup_numbered_buttons(struct input_dev *input_dev,
 		__set_bit(BTN_BASE + (i-16), input_dev->keybit);
 }
 
+static void wacom_24hd_update_leds(struct wacom *wacom, int mask, int group)
+{
+	struct wacom_led *led;
+	int i;
+	bool updated = false;
+
+	/*
+	 * 24HD has LED group 1 to the left and LED group 0 to the right.
+	 * So group 0 matches the second half of the buttons and thus the mask
+	 * needs to be shifted.
+	 */
+	if (group == 0)
+		mask >>= 8;
+
+	for (i = 0; i < 3; i++) {
+		led = wacom_led_find(wacom, group, i);
+		if (!led) {
+			hid_err(wacom->hdev, "can't find LED %d in group %d\n",
+				i, group);
+			continue;
+		}
+		if (!updated && mask & BIT(i)) {
+			led->held = true;
+			led_trigger_event(&led->trigger, LED_FULL);
+		} else {
+			led->held = false;
+		}
+	}
+}
+
+static bool wacom_is_led_toggled(struct wacom *wacom, int button_count,
+				 int mask, int group)
+{
+	int button_per_group;
+
+	/*
+	 * 21UX2 has LED group 1 to the left and LED group 0
+	 * to the right. We need to reverse the group to match this
+	 * historical behavior.
+	 */
+	if (wacom->wacom_wac.features.type == WACOM_21UX2)
+		group = 1 - group;
+
+	button_per_group = button_count/wacom->led.count;
+
+	return mask & (1 << (group * button_per_group));
+}
+
+static void wacom_update_led(struct wacom *wacom, int button_count, int mask,
+			     int group)
+{
+	struct wacom_led *led, *next_led;
+	int cur;
+	bool pressed;
+
+	if (wacom->wacom_wac.features.type == WACOM_24HD)
+		return wacom_24hd_update_leds(wacom, mask, group);
+
+	pressed = wacom_is_led_toggled(wacom, button_count, mask, group);
+	cur = wacom->led.groups[group].select;
+
+	led = wacom_led_find(wacom, group, cur);
+	if (!led) {
+		hid_err(wacom->hdev, "can't find current LED %d in group %d\n",
+			cur, group);
+		return;
+	}
+
+	if (!pressed) {
+		led->held = false;
+		return;
+	}
+
+	if (led->held && pressed)
+		return;
+
+	next_led = wacom_led_next(wacom, led);
+	if (!next_led) {
+		hid_err(wacom->hdev, "can't find next LED in group %d\n",
+			group);
+		return;
+	}
+	if (next_led == led)
+		return;
+
+	next_led->held = true;
+	led_trigger_event(&next_led->trigger,
+			  wacom_leds_brightness_get(next_led));
+}
+
 static void wacom_report_numbered_buttons(struct input_dev *input_dev,
 				int button_count, int mask)
 {
+	struct wacom *wacom = input_get_drvdata(input_dev);
 	int i;
 
+	for (i = 0; i < wacom->led.count; i++)
+		wacom_update_led(wacom,  button_count, mask, i);
+
 	for (i = 0; i < button_count && i < 10; i++)
 		input_report_key(input_dev, BTN_0 + i, mask & (1 << i));
 	for (i = 10; i < button_count && i < 16; i++)
@@ -2773,6 +2897,9 @@ int wacom_setup_pad_input_capabilities(struct input_dev *input_dev,
 	if (!(features->device_type & WACOM_DEVICETYPE_PAD))
 		return -ENODEV;
 
+	if (features->type == REMOTE && input_dev == wacom_wac->pad_input)
+		return -ENODEV;
+
 	input_dev->evbit[0] |= BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS);
 
 	/* kept for making legacy xf86-input-wacom working with the wheels */
@@ -3403,7 +3530,7 @@ static const struct wacom_features wacom_features_0x343 =
 	  WACOM_DTU_OFFSET, WACOM_DTU_OFFSET };
 
 static const struct wacom_features wacom_features_HID_ANY_ID =
-	{ "Wacom HID", .type = HID_GENERIC };
+	{ "Wacom HID", .type = HID_GENERIC, .oVid = HID_ANY_ID, .oPid = HID_ANY_ID };
 
 #define USB_DEVICE_WACOM(prod)						\
 	HID_DEVICE(BUS_USB, HID_GROUP_WACOM, USB_VENDOR_ID_WACOM, prod),\
diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h
index 53d16537fd2a..324c40b0c119 100644
--- a/drivers/hid/wacom_wac.h
+++ b/drivers/hid/wacom_wac.h
@@ -82,6 +82,7 @@
 #define WACOM_DEVICETYPE_TOUCH          0x0002
 #define WACOM_DEVICETYPE_PAD            0x0004
 #define WACOM_DEVICETYPE_WL_MONITOR     0x0008
+#define WACOM_DEVICETYPE_DIRECT         0x0010
 
 #define WACOM_VENDORDEFINED_PEN		0xff0d0001
 #define WACOM_G9_PAGE			0xff090000
@@ -185,7 +186,6 @@ struct wacom_features {
 	int pktlen;
 	bool check_for_hid_type;
 	int hid_type;
-	int last_slot_field;
 };
 
 struct wacom_shared {
@@ -214,35 +214,35 @@ struct hid_data {
 	int cc_report;
 	int cc_index;
 	int cc_value_index;
+	int last_slot_field;
 	int num_expected;
 	int num_received;
 };
 
+struct wacom_remote_data {
+	struct {
+		u32 serial;
+		bool connected;
+	} remote[WACOM_MAX_REMOTES];
+};
+
 struct wacom_wac {
+	char name[WACOM_NAME_MAX];
 	char pen_name[WACOM_NAME_MAX];
 	char touch_name[WACOM_NAME_MAX];
 	char pad_name[WACOM_NAME_MAX];
-	char bat_name[WACOM_NAME_MAX];
-	char ac_name[WACOM_NAME_MAX];
 	unsigned char data[WACOM_PKGLEN_MAX];
 	int tool[2];
 	int id[2];
-	__u32 serial[5];
+	__u32 serial[2];
 	bool reporting_data;
 	struct wacom_features features;
 	struct wacom_shared *shared;
 	struct input_dev *pen_input;
 	struct input_dev *touch_input;
 	struct input_dev *pad_input;
-	bool pen_registered;
-	bool touch_registered;
-	bool pad_registered;
 	int pid;
-	int battery_capacity;
 	int num_contacts_left;
-	int bat_charging;
-	int bat_connected;
-	int ps_connected;
 	u8 bt_features;
 	u8 bt_high_speed;
 	int mode_report;
diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c
index 56dd261f7142..16f91c8490fe 100644
--- a/drivers/hv/channel.c
+++ b/drivers/hv/channel.c
@@ -43,7 +43,12 @@ static void vmbus_setevent(struct vmbus_channel *channel)
 {
 	struct hv_monitor_page *monitorpage;
 
-	if (channel->offermsg.monitor_allocated) {
+	/*
+	 * For channels marked as in "low latency" mode
+	 * bypass the monitor page mechanism.
+	 */
+	if ((channel->offermsg.monitor_allocated) &&
+	    (!channel->low_latency)) {
 		/* Each u32 represents 32 channels */
 		sync_set_bit(channel->offermsg.child_relid & 31,
 			(unsigned long *) vmbus_connection.send_int_page +
@@ -70,12 +75,14 @@ int vmbus_open(struct vmbus_channel *newchannel, u32 send_ringbuffer_size,
 {
 	struct vmbus_channel_open_channel *open_msg;
 	struct vmbus_channel_msginfo *open_info = NULL;
-	void *in, *out;
 	unsigned long flags;
 	int ret, err = 0;
-	unsigned long t;
 	struct page *page;
 
+	if (send_ringbuffer_size % PAGE_SIZE ||
+	    recv_ringbuffer_size % PAGE_SIZE)
+		return -EINVAL;
+
 	spin_lock_irqsave(&newchannel->lock, flags);
 	if (newchannel->state == CHANNEL_OPEN_STATE) {
 		newchannel->state = CHANNEL_OPENING_STATE;
@@ -95,36 +102,33 @@ int vmbus_open(struct vmbus_channel *newchannel, u32 send_ringbuffer_size,
 				recv_ringbuffer_size));
 
 	if (!page)
-		out = (void *)__get_free_pages(GFP_KERNEL|__GFP_ZERO,
-					       get_order(send_ringbuffer_size +
-					       recv_ringbuffer_size));
-	else
-		out = (void *)page_address(page);
+		page = alloc_pages(GFP_KERNEL|__GFP_ZERO,
+				   get_order(send_ringbuffer_size +
+					     recv_ringbuffer_size));
 
-	if (!out) {
+	if (!page) {
 		err = -ENOMEM;
-		goto error0;
+		goto error_set_chnstate;
 	}
 
-	in = (void *)((unsigned long)out + send_ringbuffer_size);
-
-	newchannel->ringbuffer_pages = out;
+	newchannel->ringbuffer_pages = page_address(page);
 	newchannel->ringbuffer_pagecount = (send_ringbuffer_size +
 					   recv_ringbuffer_size) >> PAGE_SHIFT;
 
-	ret = hv_ringbuffer_init(
-		&newchannel->outbound, out, send_ringbuffer_size);
+	ret = hv_ringbuffer_init(&newchannel->outbound, page,
+				 send_ringbuffer_size >> PAGE_SHIFT);
 
 	if (ret != 0) {
 		err = ret;
-		goto error0;
+		goto error_free_pages;
 	}
 
-	ret = hv_ringbuffer_init(
-		&newchannel->inbound, in, recv_ringbuffer_size);
+	ret = hv_ringbuffer_init(&newchannel->inbound,
+				 &page[send_ringbuffer_size >> PAGE_SHIFT],
+				 recv_ringbuffer_size >> PAGE_SHIFT);
 	if (ret != 0) {
 		err = ret;
-		goto error0;
+		goto error_free_pages;
 	}
 
 
@@ -132,14 +136,14 @@ int vmbus_open(struct vmbus_channel *newchannel, u32 send_ringbuffer_size,
 	newchannel->ringbuffer_gpadlhandle = 0;
 
 	ret = vmbus_establish_gpadl(newchannel,
-					 newchannel->outbound.ring_buffer,
-					 send_ringbuffer_size +
-					 recv_ringbuffer_size,
-					 &newchannel->ringbuffer_gpadlhandle);
+				    page_address(page),
+				    send_ringbuffer_size +
+				    recv_ringbuffer_size,
+				    &newchannel->ringbuffer_gpadlhandle);
 
 	if (ret != 0) {
 		err = ret;
-		goto error0;
+		goto error_free_pages;
 	}
 
 	/* Create and init the channel open message */
@@ -148,7 +152,7 @@ int vmbus_open(struct vmbus_channel *newchannel, u32 send_ringbuffer_size,
 			   GFP_KERNEL);
 	if (!open_info) {
 		err = -ENOMEM;
-		goto error_gpadl;
+		goto error_free_gpadl;
 	}
 
 	init_completion(&open_info->waitevent);
@@ -164,7 +168,7 @@ int vmbus_open(struct vmbus_channel *newchannel, u32 send_ringbuffer_size,
 
 	if (userdatalen > MAX_USER_DEFINED_BYTES) {
 		err = -EINVAL;
-		goto error_gpadl;
+		goto error_free_gpadl;
 	}
 
 	if (userdatalen)
@@ -180,14 +184,10 @@ int vmbus_open(struct vmbus_channel *newchannel, u32 send_ringbuffer_size,
 
 	if (ret != 0) {
 		err = ret;
-		goto error1;
+		goto error_clean_msglist;
 	}
 
-	t = wait_for_completion_timeout(&open_info->waitevent, 5*HZ);
-	if (t == 0) {
-		err = -ETIMEDOUT;
-		goto error1;
-	}
+	wait_for_completion(&open_info->waitevent);
 
 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 	list_del(&open_info->msglistentry);
@@ -195,25 +195,27 @@ int vmbus_open(struct vmbus_channel *newchannel, u32 send_ringbuffer_size,
 
 	if (open_info->response.open_result.status) {
 		err = -EAGAIN;
-		goto error_gpadl;
+		goto error_free_gpadl;
 	}
 
 	newchannel->state = CHANNEL_OPENED_STATE;
 	kfree(open_info);
 	return 0;
 
-error1:
+error_clean_msglist:
 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 	list_del(&open_info->msglistentry);
 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 
-error_gpadl:
+error_free_gpadl:
 	vmbus_teardown_gpadl(newchannel, newchannel->ringbuffer_gpadlhandle);
-
-error0:
-	free_pages((unsigned long)out,
-		get_order(send_ringbuffer_size + recv_ringbuffer_size));
 	kfree(open_info);
+error_free_pages:
+	hv_ringbuffer_cleanup(&newchannel->outbound);
+	hv_ringbuffer_cleanup(&newchannel->inbound);
+	__free_pages(page,
+		     get_order(send_ringbuffer_size + recv_ringbuffer_size));
+error_set_chnstate:
 	newchannel->state = CHANNEL_OPEN_STATE;
 	return err;
 }
@@ -238,8 +240,7 @@ EXPORT_SYMBOL_GPL(vmbus_send_tl_connect_request);
  * create_gpadl_header - Creates a gpadl for the specified buffer
  */
 static int create_gpadl_header(void *kbuffer, u32 size,
-					 struct vmbus_channel_msginfo **msginfo,
-					 u32 *messagecount)
+			       struct vmbus_channel_msginfo **msginfo)
 {
 	int i;
 	int pagecount;
@@ -283,7 +284,6 @@ static int create_gpadl_header(void *kbuffer, u32 size,
 			gpadl_header->range[0].pfn_array[i] = slow_virt_to_phys(
 				kbuffer + PAGE_SIZE * i) >> PAGE_SHIFT;
 		*msginfo = msgheader;
-		*messagecount = 1;
 
 		pfnsum = pfncount;
 		pfnleft = pagecount - pfncount;
@@ -323,7 +323,6 @@ static int create_gpadl_header(void *kbuffer, u32 size,
 			}
 
 			msgbody->msgsize = msgsize;
-			(*messagecount)++;
 			gpadl_body =
 				(struct vmbus_channel_gpadl_body *)msgbody->msg;
 
@@ -352,6 +351,8 @@ static int create_gpadl_header(void *kbuffer, u32 size,
 		msgheader = kzalloc(msgsize, GFP_KERNEL);
 		if (msgheader == NULL)
 			goto nomem;
+
+		INIT_LIST_HEAD(&msgheader->submsglist);
 		msgheader->msgsize = msgsize;
 
 		gpadl_header = (struct vmbus_channel_gpadl_header *)
@@ -366,7 +367,6 @@ static int create_gpadl_header(void *kbuffer, u32 size,
 				kbuffer + PAGE_SIZE * i) >> PAGE_SHIFT;
 
 		*msginfo = msgheader;
-		*messagecount = 1;
 	}
 
 	return 0;
@@ -390,8 +390,7 @@ int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer,
 	struct vmbus_channel_gpadl_header *gpadlmsg;
 	struct vmbus_channel_gpadl_body *gpadl_body;
 	struct vmbus_channel_msginfo *msginfo = NULL;
-	struct vmbus_channel_msginfo *submsginfo;
-	u32 msgcount;
+	struct vmbus_channel_msginfo *submsginfo, *tmp;
 	struct list_head *curr;
 	u32 next_gpadl_handle;
 	unsigned long flags;
@@ -400,7 +399,7 @@ int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer,
 	next_gpadl_handle =
 		(atomic_inc_return(&vmbus_connection.next_gpadl_handle) - 1);
 
-	ret = create_gpadl_header(kbuffer, size, &msginfo, &msgcount);
+	ret = create_gpadl_header(kbuffer, size, &msginfo);
 	if (ret)
 		return ret;
 
@@ -423,24 +422,21 @@ int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer,
 	if (ret != 0)
 		goto cleanup;
 
-	if (msgcount > 1) {
-		list_for_each(curr, &msginfo->submsglist) {
-
-			submsginfo = (struct vmbus_channel_msginfo *)curr;
-			gpadl_body =
-			     (struct vmbus_channel_gpadl_body *)submsginfo->msg;
+	list_for_each(curr, &msginfo->submsglist) {
+		submsginfo = (struct vmbus_channel_msginfo *)curr;
+		gpadl_body =
+			(struct vmbus_channel_gpadl_body *)submsginfo->msg;
 
-			gpadl_body->header.msgtype =
-				CHANNELMSG_GPADL_BODY;
-			gpadl_body->gpadl = next_gpadl_handle;
+		gpadl_body->header.msgtype =
+			CHANNELMSG_GPADL_BODY;
+		gpadl_body->gpadl = next_gpadl_handle;
 
-			ret = vmbus_post_msg(gpadl_body,
-					       submsginfo->msgsize -
-					       sizeof(*submsginfo));
-			if (ret != 0)
-				goto cleanup;
+		ret = vmbus_post_msg(gpadl_body,
+				     submsginfo->msgsize -
+				     sizeof(*submsginfo));
+		if (ret != 0)
+			goto cleanup;
 
-		}
 	}
 	wait_for_completion(&msginfo->waitevent);
 
@@ -451,6 +447,10 @@ cleanup:
 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
 	list_del(&msginfo->msglistentry);
 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
+	list_for_each_entry_safe(submsginfo, tmp, &msginfo->submsglist,
+				 msglistentry) {
+		kfree(submsginfo);
+	}
 
 	kfree(msginfo);
 	return ret;
@@ -512,7 +512,6 @@ static void reset_channel_cb(void *arg)
 static int vmbus_close_internal(struct vmbus_channel *channel)
 {
 	struct vmbus_channel_close_channel *msg;
-	struct tasklet_struct *tasklet;
 	int ret;
 
 	/*
@@ -524,8 +523,7 @@ static int vmbus_close_internal(struct vmbus_channel *channel)
 	 * To resolve the race, we can serialize them by disabling the
 	 * tasklet when the latter is running here.
 	 */
-	tasklet = hv_context.event_dpc[channel->target_cpu];
-	tasklet_disable(tasklet);
+	hv_event_tasklet_disable(channel);
 
 	/*
 	 * In case a device driver's probe() fails (e.g.,
@@ -591,7 +589,7 @@ static int vmbus_close_internal(struct vmbus_channel *channel)
 		get_order(channel->ringbuffer_pagecount * PAGE_SIZE));
 
 out:
-	tasklet_enable(tasklet);
+	hv_event_tasklet_enable(channel);
 
 	return ret;
 }
@@ -659,7 +657,7 @@ int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer,
 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 
 	ret = hv_ringbuffer_write(&channel->outbound, bufferlist, num_vecs,
-				  &signal, lock);
+				  &signal, lock, channel->signal_policy);
 
 	/*
 	 * Signalling the host is conditional on many factors:
@@ -680,11 +678,6 @@ int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer,
 	 * mechanism which can hurt the performance otherwise.
 	 */
 
-	if (channel->signal_policy)
-		signal = true;
-	else
-		kick_q = true;
-
 	if (((ret == 0) && kick_q && signal) ||
 	    (ret && !is_hvsock_channel(channel)))
 		vmbus_setevent(channel);
@@ -777,7 +770,7 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel,
 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 
 	ret = hv_ringbuffer_write(&channel->outbound, bufferlist, 3,
-				  &signal, lock);
+				  &signal, lock, channel->signal_policy);
 
 	/*
 	 * Signalling the host is conditional on many factors:
@@ -795,11 +788,6 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel,
 	 * enough condition that it should not matter.
 	 */
 
-	if (channel->signal_policy)
-		signal = true;
-	else
-		kick_q = true;
-
 	if (((ret == 0) && kick_q && signal) || (ret))
 		vmbus_setevent(channel);
 
@@ -861,7 +849,7 @@ int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel,
 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 
 	ret = hv_ringbuffer_write(&channel->outbound, bufferlist, 3,
-				  &signal, lock);
+				  &signal, lock, channel->signal_policy);
 
 	if (ret == 0 && signal)
 		vmbus_setevent(channel);
@@ -926,7 +914,7 @@ int vmbus_sendpacket_multipagebuffer(struct vmbus_channel *channel,
 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 
 	ret = hv_ringbuffer_write(&channel->outbound, bufferlist, 3,
-				  &signal, lock);
+				  &signal, lock, channel->signal_policy);
 
 	if (ret == 0 && signal)
 		vmbus_setevent(channel);
diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c
index b6c1211b4df7..96a85cd39580 100644
--- a/drivers/hv/channel_mgmt.c
+++ b/drivers/hv/channel_mgmt.c
@@ -21,6 +21,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/kernel.h>
+#include <linux/interrupt.h>
 #include <linux/sched.h>
 #include <linux/wait.h>
 #include <linux/mm.h>
@@ -138,10 +139,32 @@ static const struct vmbus_device vmbus_devs[] = {
 	},
 };
 
-static u16 hv_get_dev_type(const uuid_le *guid)
+static const struct {
+	uuid_le guid;
+} vmbus_unsupported_devs[] = {
+	{ HV_AVMA1_GUID },
+	{ HV_AVMA2_GUID },
+	{ HV_RDV_GUID	},
+};
+
+static bool is_unsupported_vmbus_devs(const uuid_le *guid)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)
+		if (!uuid_le_cmp(*guid, vmbus_unsupported_devs[i].guid))
+			return true;
+	return false;
+}
+
+static u16 hv_get_dev_type(const struct vmbus_channel *channel)
 {
+	const uuid_le *guid = &channel->offermsg.offer.if_type;
 	u16 i;
 
+	if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
+		return HV_UNKOWN;
+
 	for (i = HV_IDE; i < HV_UNKOWN; i++) {
 		if (!uuid_le_cmp(*guid, vmbus_devs[i].guid))
 			return i;
@@ -251,14 +274,12 @@ EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
  */
 static struct vmbus_channel *alloc_channel(void)
 {
-	static atomic_t chan_num = ATOMIC_INIT(0);
 	struct vmbus_channel *channel;
 
 	channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
 	if (!channel)
 		return NULL;
 
-	channel->id = atomic_inc_return(&chan_num);
 	channel->acquire_ring_lock = true;
 	spin_lock_init(&channel->inbound_lock);
 	spin_lock_init(&channel->lock);
@@ -303,16 +324,32 @@ static void vmbus_release_relid(u32 relid)
 	vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released));
 }
 
+void hv_event_tasklet_disable(struct vmbus_channel *channel)
+{
+	struct tasklet_struct *tasklet;
+	tasklet = hv_context.event_dpc[channel->target_cpu];
+	tasklet_disable(tasklet);
+}
+
+void hv_event_tasklet_enable(struct vmbus_channel *channel)
+{
+	struct tasklet_struct *tasklet;
+	tasklet = hv_context.event_dpc[channel->target_cpu];
+	tasklet_enable(tasklet);
+
+	/* In case there is any pending event */
+	tasklet_schedule(tasklet);
+}
+
 void hv_process_channel_removal(struct vmbus_channel *channel, u32 relid)
 {
 	unsigned long flags;
 	struct vmbus_channel *primary_channel;
 
-	vmbus_release_relid(relid);
-
 	BUG_ON(!channel->rescind);
 	BUG_ON(!mutex_is_locked(&vmbus_connection.channel_mutex));
 
+	hv_event_tasklet_disable(channel);
 	if (channel->target_cpu != get_cpu()) {
 		put_cpu();
 		smp_call_function_single(channel->target_cpu,
@@ -321,6 +358,7 @@ void hv_process_channel_removal(struct vmbus_channel *channel, u32 relid)
 		percpu_channel_deq(channel);
 		put_cpu();
 	}
+	hv_event_tasklet_enable(channel);
 
 	if (channel->primary_channel == NULL) {
 		list_del(&channel->listentry);
@@ -338,8 +376,11 @@ void hv_process_channel_removal(struct vmbus_channel *channel, u32 relid)
 	 * We need to free the bit for init_vp_index() to work in the case
 	 * of sub-channel, when we reload drivers like hv_netvsc.
 	 */
-	cpumask_clear_cpu(channel->target_cpu,
-			  &primary_channel->alloced_cpus_in_node);
+	if (channel->affinity_policy == HV_LOCALIZED)
+		cpumask_clear_cpu(channel->target_cpu,
+				  &primary_channel->alloced_cpus_in_node);
+
+	vmbus_release_relid(relid);
 
 	free_channel(channel);
 }
@@ -405,10 +446,13 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel)
 			goto err_free_chan;
 	}
 
-	dev_type = hv_get_dev_type(&newchannel->offermsg.offer.if_type);
+	dev_type = hv_get_dev_type(newchannel);
+	if (dev_type == HV_NIC)
+		set_channel_signal_state(newchannel, HV_SIGNAL_POLICY_EXPLICIT);
 
 	init_vp_index(newchannel, dev_type);
 
+	hv_event_tasklet_disable(newchannel);
 	if (newchannel->target_cpu != get_cpu()) {
 		put_cpu();
 		smp_call_function_single(newchannel->target_cpu,
@@ -418,6 +462,7 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel)
 		percpu_channel_enq(newchannel);
 		put_cpu();
 	}
+	hv_event_tasklet_enable(newchannel);
 
 	/*
 	 * This state is used to indicate a successful open
@@ -463,12 +508,11 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel)
 	return;
 
 err_deq_chan:
-	vmbus_release_relid(newchannel->offermsg.child_relid);
-
 	mutex_lock(&vmbus_connection.channel_mutex);
 	list_del(&newchannel->listentry);
 	mutex_unlock(&vmbus_connection.channel_mutex);
 
+	hv_event_tasklet_disable(newchannel);
 	if (newchannel->target_cpu != get_cpu()) {
 		put_cpu();
 		smp_call_function_single(newchannel->target_cpu,
@@ -477,6 +521,9 @@ err_deq_chan:
 		percpu_channel_deq(newchannel);
 		put_cpu();
 	}
+	hv_event_tasklet_enable(newchannel);
+
+	vmbus_release_relid(newchannel->offermsg.child_relid);
 
 err_free_chan:
 	free_channel(newchannel);
@@ -522,17 +569,17 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type)
 	}
 
 	/*
-	 * We distribute primary channels evenly across all the available
-	 * NUMA nodes and within the assigned NUMA node we will assign the
-	 * first available CPU to the primary channel.
-	 * The sub-channels will be assigned to the CPUs available in the
-	 * NUMA node evenly.
+	 * Based on the channel affinity policy, we will assign the NUMA
+	 * nodes.
 	 */
-	if (!primary) {
+
+	if ((channel->affinity_policy == HV_BALANCED) || (!primary)) {
 		while (true) {
 			next_node = next_numa_node_id++;
-			if (next_node == nr_node_ids)
+			if (next_node == nr_node_ids) {
 				next_node = next_numa_node_id = 0;
+				continue;
+			}
 			if (cpumask_empty(cpumask_of_node(next_node)))
 				continue;
 			break;
@@ -556,15 +603,17 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type)
 
 	cur_cpu = -1;
 
-	/*
-	 * Normally Hyper-V host doesn't create more subchannels than there
-	 * are VCPUs on the node but it is possible when not all present VCPUs
-	 * on the node are initialized by guest. Clear the alloced_cpus_in_node
-	 * to start over.
-	 */
-	if (cpumask_equal(&primary->alloced_cpus_in_node,
-			  cpumask_of_node(primary->numa_node)))
-		cpumask_clear(&primary->alloced_cpus_in_node);
+	if (primary->affinity_policy == HV_LOCALIZED) {
+		/*
+		 * Normally Hyper-V host doesn't create more subchannels
+		 * than there are VCPUs on the node but it is possible when not
+		 * all present VCPUs on the node are initialized by guest.
+		 * Clear the alloced_cpus_in_node to start over.
+		 */
+		if (cpumask_equal(&primary->alloced_cpus_in_node,
+				  cpumask_of_node(primary->numa_node)))
+			cpumask_clear(&primary->alloced_cpus_in_node);
+	}
 
 	while (true) {
 		cur_cpu = cpumask_next(cur_cpu, &available_mask);
@@ -575,17 +624,24 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type)
 			continue;
 		}
 
-		/*
-		 * NOTE: in the case of sub-channel, we clear the sub-channel
-		 * related bit(s) in primary->alloced_cpus_in_node in
-		 * hv_process_channel_removal(), so when we reload drivers
-		 * like hv_netvsc in SMP guest, here we're able to re-allocate
-		 * bit from primary->alloced_cpus_in_node.
-		 */
-		if (!cpumask_test_cpu(cur_cpu,
-				&primary->alloced_cpus_in_node)) {
-			cpumask_set_cpu(cur_cpu,
-					&primary->alloced_cpus_in_node);
+		if (primary->affinity_policy == HV_LOCALIZED) {
+			/*
+			 * NOTE: in the case of sub-channel, we clear the
+			 * sub-channel related bit(s) in
+			 * primary->alloced_cpus_in_node in
+			 * hv_process_channel_removal(), so when we
+			 * reload drivers like hv_netvsc in SMP guest, here
+			 * we're able to re-allocate
+			 * bit from primary->alloced_cpus_in_node.
+			 */
+			if (!cpumask_test_cpu(cur_cpu,
+					      &primary->alloced_cpus_in_node)) {
+				cpumask_set_cpu(cur_cpu,
+						&primary->alloced_cpus_in_node);
+				cpumask_set_cpu(cur_cpu, alloced_mask);
+				break;
+			}
+		} else {
 			cpumask_set_cpu(cur_cpu, alloced_mask);
 			break;
 		}
diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c
index fcf8a02dc0ea..78e6368a4423 100644
--- a/drivers/hv/connection.c
+++ b/drivers/hv/connection.c
@@ -439,7 +439,7 @@ int vmbus_post_msg(void *buffer, size_t buflen)
 	union hv_connection_id conn_id;
 	int ret = 0;
 	int retries = 0;
-	u32 msec = 1;
+	u32 usec = 1;
 
 	conn_id.asu32 = 0;
 	conn_id.u.id = VMBUS_MESSAGE_CONNECTION_ID;
@@ -472,9 +472,9 @@ int vmbus_post_msg(void *buffer, size_t buflen)
 		}
 
 		retries++;
-		msleep(msec);
-		if (msec < 2048)
-			msec *= 2;
+		udelay(usec);
+		if (usec < 2048)
+			usec *= 2;
 	}
 	return ret;
 }
diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c
index a1c086ba3b9a..60dbd6cb4640 100644
--- a/drivers/hv/hv.c
+++ b/drivers/hv/hv.c
@@ -278,7 +278,7 @@ cleanup:
  *
  * This routine is called normally during driver unloading or exiting.
  */
-void hv_cleanup(void)
+void hv_cleanup(bool crash)
 {
 	union hv_x64_msr_hypercall_contents hypercall_msr;
 
@@ -288,7 +288,8 @@ void hv_cleanup(void)
 	if (hv_context.hypercall_page) {
 		hypercall_msr.as_uint64 = 0;
 		wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
-		vfree(hv_context.hypercall_page);
+		if (!crash)
+			vfree(hv_context.hypercall_page);
 		hv_context.hypercall_page = NULL;
 	}
 
@@ -308,7 +309,8 @@ void hv_cleanup(void)
 
 		hypercall_msr.as_uint64 = 0;
 		wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64);
-		vfree(hv_context.tsc_page);
+		if (!crash)
+			vfree(hv_context.tsc_page);
 		hv_context.tsc_page = NULL;
 	}
 #endif
diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c
index df35fb7ed5df..fdf8da929cbe 100644
--- a/drivers/hv/hv_balloon.c
+++ b/drivers/hv/hv_balloon.c
@@ -430,16 +430,27 @@ struct dm_info_msg {
  * currently hot added. We hot add in multiples of 128M
  * chunks; it is possible that we may not be able to bring
  * online all the pages in the region. The range
- * covered_end_pfn defines the pages that can
+ * covered_start_pfn:covered_end_pfn defines the pages that can
  * be brough online.
  */
 
 struct hv_hotadd_state {
 	struct list_head list;
 	unsigned long start_pfn;
+	unsigned long covered_start_pfn;
 	unsigned long covered_end_pfn;
 	unsigned long ha_end_pfn;
 	unsigned long end_pfn;
+	/*
+	 * A list of gaps.
+	 */
+	struct list_head gap_list;
+};
+
+struct hv_hotadd_gap {
+	struct list_head list;
+	unsigned long start_pfn;
+	unsigned long end_pfn;
 };
 
 struct balloon_state {
@@ -536,7 +547,11 @@ struct hv_dynmem_device {
 	 */
 	struct task_struct *thread;
 
-	struct mutex ha_region_mutex;
+	/*
+	 * Protects ha_region_list, num_pages_onlined counter and individual
+	 * regions from ha_region_list.
+	 */
+	spinlock_t ha_lock;
 
 	/*
 	 * A list of hot-add regions.
@@ -560,18 +575,14 @@ static int hv_memory_notifier(struct notifier_block *nb, unsigned long val,
 			      void *v)
 {
 	struct memory_notify *mem = (struct memory_notify *)v;
+	unsigned long flags;
 
 	switch (val) {
-	case MEM_GOING_ONLINE:
-		mutex_lock(&dm_device.ha_region_mutex);
-		break;
-
 	case MEM_ONLINE:
+		spin_lock_irqsave(&dm_device.ha_lock, flags);
 		dm_device.num_pages_onlined += mem->nr_pages;
+		spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 	case MEM_CANCEL_ONLINE:
-		if (val == MEM_ONLINE ||
-		    mutex_is_locked(&dm_device.ha_region_mutex))
-			mutex_unlock(&dm_device.ha_region_mutex);
 		if (dm_device.ha_waiting) {
 			dm_device.ha_waiting = false;
 			complete(&dm_device.ol_waitevent);
@@ -579,10 +590,11 @@ static int hv_memory_notifier(struct notifier_block *nb, unsigned long val,
 		break;
 
 	case MEM_OFFLINE:
-		mutex_lock(&dm_device.ha_region_mutex);
+		spin_lock_irqsave(&dm_device.ha_lock, flags);
 		dm_device.num_pages_onlined -= mem->nr_pages;
-		mutex_unlock(&dm_device.ha_region_mutex);
+		spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 		break;
+	case MEM_GOING_ONLINE:
 	case MEM_GOING_OFFLINE:
 	case MEM_CANCEL_OFFLINE:
 		break;
@@ -595,18 +607,46 @@ static struct notifier_block hv_memory_nb = {
 	.priority = 0
 };
 
+/* Check if the particular page is backed and can be onlined and online it. */
+static void hv_page_online_one(struct hv_hotadd_state *has, struct page *pg)
+{
+	unsigned long cur_start_pgp;
+	unsigned long cur_end_pgp;
+	struct hv_hotadd_gap *gap;
+
+	cur_start_pgp = (unsigned long)pfn_to_page(has->covered_start_pfn);
+	cur_end_pgp = (unsigned long)pfn_to_page(has->covered_end_pfn);
+
+	/* The page is not backed. */
+	if (((unsigned long)pg < cur_start_pgp) ||
+	    ((unsigned long)pg >= cur_end_pgp))
+		return;
+
+	/* Check for gaps. */
+	list_for_each_entry(gap, &has->gap_list, list) {
+		cur_start_pgp = (unsigned long)
+			pfn_to_page(gap->start_pfn);
+		cur_end_pgp = (unsigned long)
+			pfn_to_page(gap->end_pfn);
+		if (((unsigned long)pg >= cur_start_pgp) &&
+		    ((unsigned long)pg < cur_end_pgp)) {
+			return;
+		}
+	}
+
+	/* This frame is currently backed; online the page. */
+	__online_page_set_limits(pg);
+	__online_page_increment_counters(pg);
+	__online_page_free(pg);
+}
 
-static void hv_bring_pgs_online(unsigned long start_pfn, unsigned long size)
+static void hv_bring_pgs_online(struct hv_hotadd_state *has,
+				unsigned long start_pfn, unsigned long size)
 {
 	int i;
 
-	for (i = 0; i < size; i++) {
-		struct page *pg;
-		pg = pfn_to_page(start_pfn + i);
-		__online_page_set_limits(pg);
-		__online_page_increment_counters(pg);
-		__online_page_free(pg);
-	}
+	for (i = 0; i < size; i++)
+		hv_page_online_one(has, pfn_to_page(start_pfn + i));
 }
 
 static void hv_mem_hot_add(unsigned long start, unsigned long size,
@@ -618,9 +658,12 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size,
 	unsigned long start_pfn;
 	unsigned long processed_pfn;
 	unsigned long total_pfn = pfn_count;
+	unsigned long flags;
 
 	for (i = 0; i < (size/HA_CHUNK); i++) {
 		start_pfn = start + (i * HA_CHUNK);
+
+		spin_lock_irqsave(&dm_device.ha_lock, flags);
 		has->ha_end_pfn +=  HA_CHUNK;
 
 		if (total_pfn > HA_CHUNK) {
@@ -632,11 +675,11 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size,
 		}
 
 		has->covered_end_pfn +=  processed_pfn;
+		spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 
 		init_completion(&dm_device.ol_waitevent);
-		dm_device.ha_waiting = true;
+		dm_device.ha_waiting = !memhp_auto_online;
 
-		mutex_unlock(&dm_device.ha_region_mutex);
 		nid = memory_add_physaddr_to_nid(PFN_PHYS(start_pfn));
 		ret = add_memory(nid, PFN_PHYS((start_pfn)),
 				(HA_CHUNK << PAGE_SHIFT));
@@ -653,20 +696,23 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size,
 				 */
 				do_hot_add = false;
 			}
+			spin_lock_irqsave(&dm_device.ha_lock, flags);
 			has->ha_end_pfn -= HA_CHUNK;
 			has->covered_end_pfn -=  processed_pfn;
-			mutex_lock(&dm_device.ha_region_mutex);
+			spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 			break;
 		}
 
 		/*
-		 * Wait for the memory block to be onlined.
-		 * Since the hot add has succeeded, it is ok to
-		 * proceed even if the pages in the hot added region
-		 * have not been "onlined" within the allowed time.
+		 * Wait for the memory block to be onlined when memory onlining
+		 * is done outside of kernel (memhp_auto_online). Since the hot
+		 * add has succeeded, it is ok to proceed even if the pages in
+		 * the hot added region have not been "onlined" within the
+		 * allowed time.
 		 */
-		wait_for_completion_timeout(&dm_device.ol_waitevent, 5*HZ);
-		mutex_lock(&dm_device.ha_region_mutex);
+		if (dm_device.ha_waiting)
+			wait_for_completion_timeout(&dm_device.ol_waitevent,
+						    5*HZ);
 		post_status(&dm_device);
 	}
 
@@ -675,47 +721,64 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size,
 
 static void hv_online_page(struct page *pg)
 {
-	struct list_head *cur;
 	struct hv_hotadd_state *has;
 	unsigned long cur_start_pgp;
 	unsigned long cur_end_pgp;
+	unsigned long flags;
 
-	list_for_each(cur, &dm_device.ha_region_list) {
-		has = list_entry(cur, struct hv_hotadd_state, list);
-		cur_start_pgp = (unsigned long)pfn_to_page(has->start_pfn);
-		cur_end_pgp = (unsigned long)pfn_to_page(has->covered_end_pfn);
+	spin_lock_irqsave(&dm_device.ha_lock, flags);
+	list_for_each_entry(has, &dm_device.ha_region_list, list) {
+		cur_start_pgp = (unsigned long)
+			pfn_to_page(has->start_pfn);
+		cur_end_pgp = (unsigned long)pfn_to_page(has->end_pfn);
 
-		if (((unsigned long)pg >= cur_start_pgp) &&
-			((unsigned long)pg < cur_end_pgp)) {
-			/*
-			 * This frame is currently backed; online the
-			 * page.
-			 */
-			__online_page_set_limits(pg);
-			__online_page_increment_counters(pg);
-			__online_page_free(pg);
-		}
+		/* The page belongs to a different HAS. */
+		if (((unsigned long)pg < cur_start_pgp) ||
+		    ((unsigned long)pg >= cur_end_pgp))
+			continue;
+
+		hv_page_online_one(has, pg);
+		break;
 	}
+	spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 }
 
-static bool pfn_covered(unsigned long start_pfn, unsigned long pfn_cnt)
+static int pfn_covered(unsigned long start_pfn, unsigned long pfn_cnt)
 {
-	struct list_head *cur;
 	struct hv_hotadd_state *has;
+	struct hv_hotadd_gap *gap;
 	unsigned long residual, new_inc;
+	int ret = 0;
+	unsigned long flags;
 
-	if (list_empty(&dm_device.ha_region_list))
-		return false;
-
-	list_for_each(cur, &dm_device.ha_region_list) {
-		has = list_entry(cur, struct hv_hotadd_state, list);
-
+	spin_lock_irqsave(&dm_device.ha_lock, flags);
+	list_for_each_entry(has, &dm_device.ha_region_list, list) {
 		/*
 		 * If the pfn range we are dealing with is not in the current
 		 * "hot add block", move on.
 		 */
 		if (start_pfn < has->start_pfn || start_pfn >= has->end_pfn)
 			continue;
+
+		/*
+		 * If the current start pfn is not where the covered_end
+		 * is, create a gap and update covered_end_pfn.
+		 */
+		if (has->covered_end_pfn != start_pfn) {
+			gap = kzalloc(sizeof(struct hv_hotadd_gap), GFP_ATOMIC);
+			if (!gap) {
+				ret = -ENOMEM;
+				break;
+			}
+
+			INIT_LIST_HEAD(&gap->list);
+			gap->start_pfn = has->covered_end_pfn;
+			gap->end_pfn = start_pfn;
+			list_add_tail(&gap->list, &has->gap_list);
+
+			has->covered_end_pfn = start_pfn;
+		}
+
 		/*
 		 * If the current hot add-request extends beyond
 		 * our current limit; extend it.
@@ -732,19 +795,12 @@ static bool pfn_covered(unsigned long start_pfn, unsigned long pfn_cnt)
 			has->end_pfn += new_inc;
 		}
 
-		/*
-		 * If the current start pfn is not where the covered_end
-		 * is, update it.
-		 */
-
-		if (has->covered_end_pfn != start_pfn)
-			has->covered_end_pfn = start_pfn;
-
-		return true;
-
+		ret = 1;
+		break;
 	}
+	spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 
-	return false;
+	return ret;
 }
 
 static unsigned long handle_pg_range(unsigned long pg_start,
@@ -753,17 +809,13 @@ static unsigned long handle_pg_range(unsigned long pg_start,
 	unsigned long start_pfn = pg_start;
 	unsigned long pfn_cnt = pg_count;
 	unsigned long size;
-	struct list_head *cur;
 	struct hv_hotadd_state *has;
 	unsigned long pgs_ol = 0;
 	unsigned long old_covered_state;
+	unsigned long res = 0, flags;
 
-	if (list_empty(&dm_device.ha_region_list))
-		return 0;
-
-	list_for_each(cur, &dm_device.ha_region_list) {
-		has = list_entry(cur, struct hv_hotadd_state, list);
-
+	spin_lock_irqsave(&dm_device.ha_lock, flags);
+	list_for_each_entry(has, &dm_device.ha_region_list, list) {
 		/*
 		 * If the pfn range we are dealing with is not in the current
 		 * "hot add block", move on.
@@ -783,6 +835,8 @@ static unsigned long handle_pg_range(unsigned long pg_start,
 			if (pgs_ol > pfn_cnt)
 				pgs_ol = pfn_cnt;
 
+			has->covered_end_pfn +=  pgs_ol;
+			pfn_cnt -= pgs_ol;
 			/*
 			 * Check if the corresponding memory block is already
 			 * online by checking its last previously backed page.
@@ -791,10 +845,8 @@ static unsigned long handle_pg_range(unsigned long pg_start,
 			 */
 			if (start_pfn > has->start_pfn &&
 			    !PageReserved(pfn_to_page(start_pfn - 1)))
-				hv_bring_pgs_online(start_pfn, pgs_ol);
+				hv_bring_pgs_online(has, start_pfn, pgs_ol);
 
-			has->covered_end_pfn +=  pgs_ol;
-			pfn_cnt -= pgs_ol;
 		}
 
 		if ((has->ha_end_pfn < has->end_pfn) && (pfn_cnt > 0)) {
@@ -813,17 +865,20 @@ static unsigned long handle_pg_range(unsigned long pg_start,
 			} else {
 				pfn_cnt = size;
 			}
+			spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 			hv_mem_hot_add(has->ha_end_pfn, size, pfn_cnt, has);
+			spin_lock_irqsave(&dm_device.ha_lock, flags);
 		}
 		/*
 		 * If we managed to online any pages that were given to us,
 		 * we declare success.
 		 */
-		return has->covered_end_pfn - old_covered_state;
-
+		res = has->covered_end_pfn - old_covered_state;
+		break;
 	}
+	spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 
-	return 0;
+	return res;
 }
 
 static unsigned long process_hot_add(unsigned long pg_start,
@@ -832,13 +887,20 @@ static unsigned long process_hot_add(unsigned long pg_start,
 					unsigned long rg_size)
 {
 	struct hv_hotadd_state *ha_region = NULL;
+	int covered;
+	unsigned long flags;
 
 	if (pfn_cnt == 0)
 		return 0;
 
-	if (!dm_device.host_specified_ha_region)
-		if (pfn_covered(pg_start, pfn_cnt))
+	if (!dm_device.host_specified_ha_region) {
+		covered = pfn_covered(pg_start, pfn_cnt);
+		if (covered < 0)
+			return 0;
+
+		if (covered)
 			goto do_pg_range;
+	}
 
 	/*
 	 * If the host has specified a hot-add range; deal with it first.
@@ -850,12 +912,17 @@ static unsigned long process_hot_add(unsigned long pg_start,
 			return 0;
 
 		INIT_LIST_HEAD(&ha_region->list);
+		INIT_LIST_HEAD(&ha_region->gap_list);
 
-		list_add_tail(&ha_region->list, &dm_device.ha_region_list);
 		ha_region->start_pfn = rg_start;
 		ha_region->ha_end_pfn = rg_start;
+		ha_region->covered_start_pfn = pg_start;
 		ha_region->covered_end_pfn = pg_start;
 		ha_region->end_pfn = rg_start + rg_size;
+
+		spin_lock_irqsave(&dm_device.ha_lock, flags);
+		list_add_tail(&ha_region->list, &dm_device.ha_region_list);
+		spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 	}
 
 do_pg_range:
@@ -882,7 +949,6 @@ static void hot_add_req(struct work_struct *dummy)
 	resp.hdr.size = sizeof(struct dm_hot_add_response);
 
 #ifdef CONFIG_MEMORY_HOTPLUG
-	mutex_lock(&dm_device.ha_region_mutex);
 	pg_start = dm->ha_wrk.ha_page_range.finfo.start_page;
 	pfn_cnt = dm->ha_wrk.ha_page_range.finfo.page_cnt;
 
@@ -916,7 +982,6 @@ static void hot_add_req(struct work_struct *dummy)
 						rg_start, rg_sz);
 
 	dm->num_pages_added += resp.page_count;
-	mutex_unlock(&dm_device.ha_region_mutex);
 #endif
 	/*
 	 * The result field of the response structure has the
@@ -1010,7 +1075,6 @@ static unsigned long compute_balloon_floor(void)
 static void post_status(struct hv_dynmem_device *dm)
 {
 	struct dm_status status;
-	struct sysinfo val;
 	unsigned long now = jiffies;
 	unsigned long last_post = last_post_time;
 
@@ -1022,7 +1086,6 @@ static void post_status(struct hv_dynmem_device *dm)
 	if (!time_after(now, (last_post_time + HZ)))
 		return;
 
-	si_meminfo(&val);
 	memset(&status, 0, sizeof(struct dm_status));
 	status.hdr.type = DM_STATUS_REPORT;
 	status.hdr.size = sizeof(struct dm_status);
@@ -1038,7 +1101,7 @@ static void post_status(struct hv_dynmem_device *dm)
 	 * num_pages_onlined) as committed to the host, otherwise it can try
 	 * asking us to balloon them out.
 	 */
-	status.num_avail = val.freeram;
+	status.num_avail = si_mem_available();
 	status.num_committed = vm_memory_committed() +
 		dm->num_pages_ballooned +
 		(dm->num_pages_added > dm->num_pages_onlined ?
@@ -1144,7 +1207,7 @@ static void balloon_up(struct work_struct *dummy)
 	int ret;
 	bool done = false;
 	int i;
-	struct sysinfo val;
+	long avail_pages;
 	unsigned long floor;
 
 	/* The host balloons pages in 2M granularity. */
@@ -1156,12 +1219,12 @@ static void balloon_up(struct work_struct *dummy)
 	 */
 	alloc_unit = 512;
 
-	si_meminfo(&val);
+	avail_pages = si_mem_available();
 	floor = compute_balloon_floor();
 
 	/* Refuse to balloon below the floor, keep the 2M granularity. */
-	if (val.freeram < num_pages || val.freeram - num_pages < floor) {
-		num_pages = val.freeram > floor ? (val.freeram - floor) : 0;
+	if (avail_pages < num_pages || avail_pages - num_pages < floor) {
+		num_pages = avail_pages > floor ? (avail_pages - floor) : 0;
 		num_pages -= num_pages % PAGES_IN_2M;
 	}
 
@@ -1172,7 +1235,6 @@ static void balloon_up(struct work_struct *dummy)
 		bl_resp->hdr.size = sizeof(struct dm_balloon_response);
 		bl_resp->more_pages = 1;
 
-
 		num_pages -= num_ballooned;
 		num_ballooned = alloc_balloon_pages(&dm_device, num_pages,
 						    bl_resp, alloc_unit);
@@ -1461,7 +1523,7 @@ static int balloon_probe(struct hv_device *dev,
 	init_completion(&dm_device.host_event);
 	init_completion(&dm_device.config_event);
 	INIT_LIST_HEAD(&dm_device.ha_region_list);
-	mutex_init(&dm_device.ha_region_mutex);
+	spin_lock_init(&dm_device.ha_lock);
 	INIT_WORK(&dm_device.balloon_wrk.wrk, balloon_up);
 	INIT_WORK(&dm_device.ha_wrk.wrk, hot_add_req);
 	dm_device.host_specified_ha_region = false;
@@ -1580,8 +1642,9 @@ probe_error0:
 static int balloon_remove(struct hv_device *dev)
 {
 	struct hv_dynmem_device *dm = hv_get_drvdata(dev);
-	struct list_head *cur, *tmp;
-	struct hv_hotadd_state *has;
+	struct hv_hotadd_state *has, *tmp;
+	struct hv_hotadd_gap *gap, *tmp_gap;
+	unsigned long flags;
 
 	if (dm->num_pages_ballooned != 0)
 		pr_warn("Ballooned pages: %d\n", dm->num_pages_ballooned);
@@ -1596,11 +1659,16 @@ static int balloon_remove(struct hv_device *dev)
 	restore_online_page_callback(&hv_online_page);
 	unregister_memory_notifier(&hv_memory_nb);
 #endif
-	list_for_each_safe(cur, tmp, &dm->ha_region_list) {
-		has = list_entry(cur, struct hv_hotadd_state, list);
+	spin_lock_irqsave(&dm_device.ha_lock, flags);
+	list_for_each_entry_safe(has, tmp, &dm->ha_region_list, list) {
+		list_for_each_entry_safe(gap, tmp_gap, &has->gap_list, list) {
+			list_del(&gap->list);
+			kfree(gap);
+		}
 		list_del(&has->list);
 		kfree(has);
 	}
+	spin_unlock_irqrestore(&dm_device.ha_lock, flags);
 
 	return 0;
 }
diff --git a/drivers/hv/hv_fcopy.c b/drivers/hv/hv_fcopy.c
index 23c70799ad8a..8b2ba98831ec 100644
--- a/drivers/hv/hv_fcopy.c
+++ b/drivers/hv/hv_fcopy.c
@@ -83,6 +83,12 @@ static void fcopy_timeout_func(struct work_struct *dummy)
 	hv_poll_channel(fcopy_transaction.recv_channel, fcopy_poll_wrapper);
 }
 
+static void fcopy_register_done(void)
+{
+	pr_debug("FCP: userspace daemon registered\n");
+	hv_poll_channel(fcopy_transaction.recv_channel, fcopy_poll_wrapper);
+}
+
 static int fcopy_handle_handshake(u32 version)
 {
 	u32 our_ver = FCOPY_CURRENT_VERSION;
@@ -94,7 +100,8 @@ static int fcopy_handle_handshake(u32 version)
 		break;
 	case FCOPY_VERSION_1:
 		/* Daemon expects us to reply with our own version */
-		if (hvutil_transport_send(hvt, &our_ver, sizeof(our_ver)))
+		if (hvutil_transport_send(hvt, &our_ver, sizeof(our_ver),
+		    fcopy_register_done))
 			return -EFAULT;
 		dm_reg_value = version;
 		break;
@@ -107,8 +114,7 @@ static int fcopy_handle_handshake(u32 version)
 		 */
 		return -EINVAL;
 	}
-	pr_debug("FCP: userspace daemon ver. %d registered\n", version);
-	hv_poll_channel(fcopy_transaction.recv_channel, fcopy_poll_wrapper);
+	pr_debug("FCP: userspace daemon ver. %d connected\n", version);
 	return 0;
 }
 
@@ -161,7 +167,7 @@ static void fcopy_send_data(struct work_struct *dummy)
 	}
 
 	fcopy_transaction.state = HVUTIL_USERSPACE_REQ;
-	rc = hvutil_transport_send(hvt, out_src, out_len);
+	rc = hvutil_transport_send(hvt, out_src, out_len, NULL);
 	if (rc) {
 		pr_debug("FCP: failed to communicate to the daemon: %d\n", rc);
 		if (cancel_delayed_work_sync(&fcopy_timeout_work)) {
diff --git a/drivers/hv/hv_kvp.c b/drivers/hv/hv_kvp.c
index cb1a9160aab1..5e1fdc8d32ab 100644
--- a/drivers/hv/hv_kvp.c
+++ b/drivers/hv/hv_kvp.c
@@ -102,6 +102,17 @@ static void kvp_poll_wrapper(void *channel)
 	hv_kvp_onchannelcallback(channel);
 }
 
+static void kvp_register_done(void)
+{
+	/*
+	 * If we're still negotiating with the host cancel the timeout
+	 * work to not poll the channel twice.
+	 */
+	pr_debug("KVP: userspace daemon registered\n");
+	cancel_delayed_work_sync(&kvp_host_handshake_work);
+	hv_poll_channel(kvp_transaction.recv_channel, kvp_poll_wrapper);
+}
+
 static void
 kvp_register(int reg_value)
 {
@@ -116,7 +127,8 @@ kvp_register(int reg_value)
 		kvp_msg->kvp_hdr.operation = reg_value;
 		strcpy(version, HV_DRV_VERSION);
 
-		hvutil_transport_send(hvt, kvp_msg, sizeof(*kvp_msg));
+		hvutil_transport_send(hvt, kvp_msg, sizeof(*kvp_msg),
+				      kvp_register_done);
 		kfree(kvp_msg);
 	}
 }
@@ -158,17 +170,10 @@ static int kvp_handle_handshake(struct hv_kvp_msg *msg)
 	/*
 	 * We have a compatible daemon; complete the handshake.
 	 */
-	pr_debug("KVP: userspace daemon ver. %d registered\n",
-		 KVP_OP_REGISTER);
+	pr_debug("KVP: userspace daemon ver. %d connected\n",
+		 msg->kvp_hdr.operation);
 	kvp_register(dm_reg_value);
 
-	/*
-	 * If we're still negotiating with the host cancel the timeout
-	 * work to not poll the channel twice.
-	 */
-	cancel_delayed_work_sync(&kvp_host_handshake_work);
-	hv_poll_channel(kvp_transaction.recv_channel, kvp_poll_wrapper);
-
 	return 0;
 }
 
@@ -455,7 +460,7 @@ kvp_send_key(struct work_struct *dummy)
 	}
 
 	kvp_transaction.state = HVUTIL_USERSPACE_REQ;
-	rc = hvutil_transport_send(hvt, message, sizeof(*message));
+	rc = hvutil_transport_send(hvt, message, sizeof(*message), NULL);
 	if (rc) {
 		pr_debug("KVP: failed to communicate to the daemon: %d\n", rc);
 		if (cancel_delayed_work_sync(&kvp_timeout_work)) {
diff --git a/drivers/hv/hv_snapshot.c b/drivers/hv/hv_snapshot.c
index 3fba14e88f03..a6707133c297 100644
--- a/drivers/hv/hv_snapshot.c
+++ b/drivers/hv/hv_snapshot.c
@@ -67,11 +67,11 @@ static const char vss_devname[] = "vmbus/hv_vss";
 static __u8 *recv_buffer;
 static struct hvutil_transport *hvt;
 
-static void vss_send_op(struct work_struct *dummy);
 static void vss_timeout_func(struct work_struct *dummy);
+static void vss_handle_request(struct work_struct *dummy);
 
 static DECLARE_DELAYED_WORK(vss_timeout_work, vss_timeout_func);
-static DECLARE_WORK(vss_send_op_work, vss_send_op);
+static DECLARE_WORK(vss_handle_request_work, vss_handle_request);
 
 static void vss_poll_wrapper(void *channel)
 {
@@ -95,6 +95,12 @@ static void vss_timeout_func(struct work_struct *dummy)
 	hv_poll_channel(vss_transaction.recv_channel, vss_poll_wrapper);
 }
 
+static void vss_register_done(void)
+{
+	hv_poll_channel(vss_transaction.recv_channel, vss_poll_wrapper);
+	pr_debug("VSS: userspace daemon registered\n");
+}
+
 static int vss_handle_handshake(struct hv_vss_msg *vss_msg)
 {
 	u32 our_ver = VSS_OP_REGISTER1;
@@ -105,16 +111,16 @@ static int vss_handle_handshake(struct hv_vss_msg *vss_msg)
 		dm_reg_value = VSS_OP_REGISTER;
 		break;
 	case VSS_OP_REGISTER1:
-		/* Daemon expects us to reply with our own version*/
-		if (hvutil_transport_send(hvt, &our_ver, sizeof(our_ver)))
+		/* Daemon expects us to reply with our own version */
+		if (hvutil_transport_send(hvt, &our_ver, sizeof(our_ver),
+					  vss_register_done))
 			return -EFAULT;
 		dm_reg_value = VSS_OP_REGISTER1;
 		break;
 	default:
 		return -EINVAL;
 	}
-	hv_poll_channel(vss_transaction.recv_channel, vss_poll_wrapper);
-	pr_debug("VSS: userspace daemon ver. %d registered\n", dm_reg_value);
+	pr_debug("VSS: userspace daemon ver. %d connected\n", dm_reg_value);
 	return 0;
 }
 
@@ -136,6 +142,11 @@ static int vss_on_msg(void *msg, int len)
 		return vss_handle_handshake(vss_msg);
 	} else if (vss_transaction.state == HVUTIL_USERSPACE_REQ) {
 		vss_transaction.state = HVUTIL_USERSPACE_RECV;
+
+		if (vss_msg->vss_hdr.operation == VSS_OP_HOT_BACKUP)
+			vss_transaction.msg->vss_cf.flags =
+				VSS_HBU_NO_AUTO_RECOVERY;
+
 		if (cancel_delayed_work_sync(&vss_timeout_work)) {
 			vss_respond_to_host(vss_msg->error);
 			/* Transaction is finished, reset the state. */
@@ -150,8 +161,7 @@ static int vss_on_msg(void *msg, int len)
 	return 0;
 }
 
-
-static void vss_send_op(struct work_struct *dummy)
+static void vss_send_op(void)
 {
 	int op = vss_transaction.msg->vss_hdr.operation;
 	int rc;
@@ -168,7 +178,10 @@ static void vss_send_op(struct work_struct *dummy)
 	vss_msg->vss_hdr.operation = op;
 
 	vss_transaction.state = HVUTIL_USERSPACE_REQ;
-	rc = hvutil_transport_send(hvt, vss_msg, sizeof(*vss_msg));
+
+	schedule_delayed_work(&vss_timeout_work, VSS_USERSPACE_TIMEOUT);
+
+	rc = hvutil_transport_send(hvt, vss_msg, sizeof(*vss_msg), NULL);
 	if (rc) {
 		pr_warn("VSS: failed to communicate to the daemon: %d\n", rc);
 		if (cancel_delayed_work_sync(&vss_timeout_work)) {
@@ -182,6 +195,38 @@ static void vss_send_op(struct work_struct *dummy)
 	return;
 }
 
+static void vss_handle_request(struct work_struct *dummy)
+{
+	switch (vss_transaction.msg->vss_hdr.operation) {
+	/*
+	 * Initiate a "freeze/thaw" operation in the guest.
+	 * We respond to the host once the operation is complete.
+	 *
+	 * We send the message to the user space daemon and the operation is
+	 * performed in the daemon.
+	 */
+	case VSS_OP_THAW:
+	case VSS_OP_FREEZE:
+	case VSS_OP_HOT_BACKUP:
+		if (vss_transaction.state < HVUTIL_READY) {
+			/* Userspace is not registered yet */
+			vss_respond_to_host(HV_E_FAIL);
+			return;
+		}
+		vss_transaction.state = HVUTIL_HOSTMSG_RECEIVED;
+		vss_send_op();
+		return;
+	case VSS_OP_GET_DM_INFO:
+		vss_transaction.msg->dm_info.flags = 0;
+		break;
+	default:
+		break;
+	}
+
+	vss_respond_to_host(0);
+	hv_poll_channel(vss_transaction.recv_channel, vss_poll_wrapper);
+}
+
 /*
  * Send a response back to the host.
  */
@@ -266,48 +311,8 @@ void hv_vss_onchannelcallback(void *context)
 			vss_transaction.recv_req_id = requestid;
 			vss_transaction.msg = (struct hv_vss_msg *)vss_msg;
 
-			switch (vss_msg->vss_hdr.operation) {
-				/*
-				 * Initiate a "freeze/thaw"
-				 * operation in the guest.
-				 * We respond to the host once
-				 * the operation is complete.
-				 *
-				 * We send the message to the
-				 * user space daemon and the
-				 * operation is performed in
-				 * the daemon.
-				 */
-			case VSS_OP_FREEZE:
-			case VSS_OP_THAW:
-				if (vss_transaction.state < HVUTIL_READY) {
-					/* Userspace is not registered yet */
-					vss_respond_to_host(HV_E_FAIL);
-					return;
-				}
-				vss_transaction.state = HVUTIL_HOSTMSG_RECEIVED;
-				schedule_work(&vss_send_op_work);
-				schedule_delayed_work(&vss_timeout_work,
-						      VSS_USERSPACE_TIMEOUT);
-				return;
-
-			case VSS_OP_HOT_BACKUP:
-				vss_msg->vss_cf.flags =
-					 VSS_HBU_NO_AUTO_RECOVERY;
-				vss_respond_to_host(0);
-				return;
-
-			case VSS_OP_GET_DM_INFO:
-				vss_msg->dm_info.flags = 0;
-				vss_respond_to_host(0);
-				return;
-
-			default:
-				vss_respond_to_host(0);
-				return;
-
-			}
-
+			schedule_work(&vss_handle_request_work);
+			return;
 		}
 
 		icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION
@@ -358,6 +363,6 @@ void hv_vss_deinit(void)
 {
 	vss_transaction.state = HVUTIL_DEVICE_DYING;
 	cancel_delayed_work_sync(&vss_timeout_work);
-	cancel_work_sync(&vss_send_op_work);
+	cancel_work_sync(&vss_handle_request_work);
 	hvutil_transport_destroy(hvt);
 }
diff --git a/drivers/hv/hv_util.c b/drivers/hv/hv_util.c
index d5acaa2d8e61..4aa3cb63fd41 100644
--- a/drivers/hv/hv_util.c
+++ b/drivers/hv/hv_util.c
@@ -34,22 +34,25 @@
 #define SD_MINOR	0
 #define SD_VERSION	(SD_MAJOR << 16 | SD_MINOR)
 
-#define SD_WS2008_MAJOR		1
-#define SD_WS2008_VERSION	(SD_WS2008_MAJOR << 16 | SD_MINOR)
+#define SD_MAJOR_1	1
+#define SD_VERSION_1	(SD_MAJOR_1 << 16 | SD_MINOR)
 
-#define TS_MAJOR	3
+#define TS_MAJOR	4
 #define TS_MINOR	0
 #define TS_VERSION	(TS_MAJOR << 16 | TS_MINOR)
 
-#define TS_WS2008_MAJOR		1
-#define TS_WS2008_VERSION	(TS_WS2008_MAJOR << 16 | TS_MINOR)
+#define TS_MAJOR_1	1
+#define TS_VERSION_1	(TS_MAJOR_1 << 16 | TS_MINOR)
+
+#define TS_MAJOR_3	3
+#define TS_VERSION_3	(TS_MAJOR_3 << 16 | TS_MINOR)
 
 #define HB_MAJOR	3
-#define HB_MINOR 0
+#define HB_MINOR	0
 #define HB_VERSION	(HB_MAJOR << 16 | HB_MINOR)
 
-#define HB_WS2008_MAJOR	1
-#define HB_WS2008_VERSION	(HB_WS2008_MAJOR << 16 | HB_MINOR)
+#define HB_MAJOR_1	1
+#define HB_VERSION_1	(HB_MAJOR_1 << 16 | HB_MINOR)
 
 static int sd_srv_version;
 static int ts_srv_version;
@@ -61,9 +64,14 @@ static struct hv_util_service util_shutdown = {
 	.util_cb = shutdown_onchannelcallback,
 };
 
+static int hv_timesync_init(struct hv_util_service *srv);
+static void hv_timesync_deinit(void);
+
 static void timesync_onchannelcallback(void *context);
 static struct hv_util_service util_timesynch = {
 	.util_cb = timesync_onchannelcallback,
+	.util_init = hv_timesync_init,
+	.util_deinit = hv_timesync_deinit,
 };
 
 static void heartbeat_onchannelcallback(void *context);
@@ -161,35 +169,43 @@ static void shutdown_onchannelcallback(void *context)
 }
 
 /*
- * Set guest time to host UTC time.
- */
-static inline void do_adj_guesttime(u64 hosttime)
-{
-	s64 host_tns;
-	struct timespec host_ts;
-
-	host_tns = (hosttime - WLTIMEDELTA) * 100;
-	host_ts = ns_to_timespec(host_tns);
-
-	do_settimeofday(&host_ts);
-}
-
-/*
  * Set the host time in a process context.
  */
 
 struct adj_time_work {
 	struct work_struct work;
 	u64	host_time;
+	u64	ref_time;
+	u8	flags;
 };
 
 static void hv_set_host_time(struct work_struct *work)
 {
 	struct adj_time_work	*wrk;
+	s64 host_tns;
+	u64 newtime;
+	struct timespec host_ts;
 
 	wrk = container_of(work, struct adj_time_work, work);
-	do_adj_guesttime(wrk->host_time);
-	kfree(wrk);
+
+	newtime = wrk->host_time;
+	if (ts_srv_version > TS_VERSION_3) {
+		/*
+		 * Some latency has been introduced since Hyper-V generated
+		 * its time sample. Take that latency into account before
+		 * using TSC reference time sample from Hyper-V.
+		 *
+		 * This sample is given by TimeSync v4 and above hosts.
+		 */
+		u64 current_tick;
+
+		rdmsrl(HV_X64_MSR_TIME_REF_COUNT, current_tick);
+		newtime += (current_tick - wrk->ref_time);
+	}
+	host_tns = (newtime - WLTIMEDELTA) * 100;
+	host_ts = ns_to_timespec(host_tns);
+
+	do_settimeofday(&host_ts);
 }
 
 /*
@@ -198,33 +214,31 @@ static void hv_set_host_time(struct work_struct *work)
  * ICTIMESYNCFLAG_SYNC flag bit indicates reboot, restore events of the VM.
  * After reboot the flag ICTIMESYNCFLAG_SYNC is included in the first time
  * message after the timesync channel is opened. Since the hv_utils module is
- * loaded after hv_vmbus, the first message is usually missed. The other
- * thing is, systime is automatically set to emulated hardware clock which may
- * not be UTC time or in the same time zone. So, to override these effects, we
- * use the first 50 time samples for initial system time setting.
+ * loaded after hv_vmbus, the first message is usually missed. This bit is
+ * considered a hard request to discipline the clock.
+ *
+ * ICTIMESYNCFLAG_SAMPLE bit indicates a time sample from host. This is
+ * typically used as a hint to the guest. The guest is under no obligation
+ * to discipline the clock.
  */
-static inline void adj_guesttime(u64 hosttime, u8 flags)
+static struct adj_time_work  wrk;
+static inline void adj_guesttime(u64 hosttime, u64 reftime, u8 flags)
 {
-	struct adj_time_work    *wrk;
-	static s32 scnt = 50;
 
-	wrk = kmalloc(sizeof(struct adj_time_work), GFP_ATOMIC);
-	if (wrk == NULL)
+	/*
+	 * This check is safe since we are executing in the
+	 * interrupt context and time synch messages arre always
+	 * delivered on the same CPU.
+	 */
+	if (work_pending(&wrk.work))
 		return;
 
-	wrk->host_time = hosttime;
-	if ((flags & ICTIMESYNCFLAG_SYNC) != 0) {
-		INIT_WORK(&wrk->work, hv_set_host_time);
-		schedule_work(&wrk->work);
-		return;
+	wrk.host_time = hosttime;
+	wrk.ref_time = reftime;
+	wrk.flags = flags;
+	if ((flags & (ICTIMESYNCFLAG_SYNC | ICTIMESYNCFLAG_SAMPLE)) != 0) {
+		schedule_work(&wrk.work);
 	}
-
-	if ((flags & ICTIMESYNCFLAG_SAMPLE) != 0 && scnt > 0) {
-		scnt--;
-		INIT_WORK(&wrk->work, hv_set_host_time);
-		schedule_work(&wrk->work);
-	} else
-		kfree(wrk);
 }
 
 /*
@@ -237,6 +251,7 @@ static void timesync_onchannelcallback(void *context)
 	u64 requestid;
 	struct icmsg_hdr *icmsghdrp;
 	struct ictimesync_data *timedatap;
+	struct ictimesync_ref_data *refdata;
 	u8 *time_txf_buf = util_timesynch.recv_buffer;
 	struct icmsg_negotiate *negop = NULL;
 
@@ -252,11 +267,27 @@ static void timesync_onchannelcallback(void *context)
 						time_txf_buf,
 						util_fw_version,
 						ts_srv_version);
+			pr_info("Using TimeSync version %d.%d\n",
+				ts_srv_version >> 16, ts_srv_version & 0xFFFF);
 		} else {
-			timedatap = (struct ictimesync_data *)&time_txf_buf[
-				sizeof(struct vmbuspipe_hdr) +
-				sizeof(struct icmsg_hdr)];
-			adj_guesttime(timedatap->parenttime, timedatap->flags);
+			if (ts_srv_version > TS_VERSION_3) {
+				refdata = (struct ictimesync_ref_data *)
+					&time_txf_buf[
+					sizeof(struct vmbuspipe_hdr) +
+					sizeof(struct icmsg_hdr)];
+
+				adj_guesttime(refdata->parenttime,
+						refdata->vmreferencetime,
+						refdata->flags);
+			} else {
+				timedatap = (struct ictimesync_data *)
+					&time_txf_buf[
+					sizeof(struct vmbuspipe_hdr) +
+					sizeof(struct icmsg_hdr)];
+				adj_guesttime(timedatap->parenttime,
+						0,
+						timedatap->flags);
+			}
 		}
 
 		icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION
@@ -350,16 +381,21 @@ static int util_probe(struct hv_device *dev,
 	switch (vmbus_proto_version) {
 	case (VERSION_WS2008):
 		util_fw_version = UTIL_WS2K8_FW_VERSION;
-		sd_srv_version = SD_WS2008_VERSION;
-		ts_srv_version = TS_WS2008_VERSION;
-		hb_srv_version = HB_WS2008_VERSION;
+		sd_srv_version = SD_VERSION_1;
+		ts_srv_version = TS_VERSION_1;
+		hb_srv_version = HB_VERSION_1;
 		break;
-
-	default:
+	case(VERSION_WIN10):
 		util_fw_version = UTIL_FW_VERSION;
 		sd_srv_version = SD_VERSION;
 		ts_srv_version = TS_VERSION;
 		hb_srv_version = HB_VERSION;
+		break;
+	default:
+		util_fw_version = UTIL_FW_VERSION;
+		sd_srv_version = SD_VERSION;
+		ts_srv_version = TS_VERSION_3;
+		hb_srv_version = HB_VERSION;
 	}
 
 	ret = vmbus_open(dev->channel, 4 * PAGE_SIZE, 4 * PAGE_SIZE, NULL, 0,
@@ -427,6 +463,17 @@ static  struct hv_driver util_drv = {
 	.remove =  util_remove,
 };
 
+static int hv_timesync_init(struct hv_util_service *srv)
+{
+	INIT_WORK(&wrk.work, hv_set_host_time);
+	return 0;
+}
+
+static void hv_timesync_deinit(void)
+{
+	cancel_work_sync(&wrk.work);
+}
+
 static int __init init_hyperv_utils(void)
 {
 	pr_info("Registering HyperV Utility Driver\n");
diff --git a/drivers/hv/hv_utils_transport.c b/drivers/hv/hv_utils_transport.c
index 9a9983fa4531..c235a9515267 100644
--- a/drivers/hv/hv_utils_transport.c
+++ b/drivers/hv/hv_utils_transport.c
@@ -72,6 +72,10 @@ static ssize_t hvt_op_read(struct file *file, char __user *buf,
 	hvt->outmsg = NULL;
 	hvt->outmsg_len = 0;
 
+	if (hvt->on_read)
+		hvt->on_read();
+	hvt->on_read = NULL;
+
 out_unlock:
 	mutex_unlock(&hvt->lock);
 	return ret;
@@ -219,7 +223,8 @@ static void hvt_cn_callback(struct cn_msg *msg, struct netlink_skb_parms *nsp)
 	mutex_unlock(&hvt->lock);
 }
 
-int hvutil_transport_send(struct hvutil_transport *hvt, void *msg, int len)
+int hvutil_transport_send(struct hvutil_transport *hvt, void *msg, int len,
+			  void (*on_read_cb)(void))
 {
 	struct cn_msg *cn_msg;
 	int ret = 0;
@@ -237,6 +242,13 @@ int hvutil_transport_send(struct hvutil_transport *hvt, void *msg, int len)
 		memcpy(cn_msg->data, msg, len);
 		ret = cn_netlink_send(cn_msg, 0, 0, GFP_ATOMIC);
 		kfree(cn_msg);
+		/*
+		 * We don't know when netlink messages are delivered but unlike
+		 * in CHARDEV mode we're not blocked and we can send next
+		 * messages right away.
+		 */
+		if (on_read_cb)
+			on_read_cb();
 		return ret;
 	}
 	/* HVUTIL_TRANSPORT_CHARDEV */
@@ -255,6 +267,7 @@ int hvutil_transport_send(struct hvutil_transport *hvt, void *msg, int len)
 	if (hvt->outmsg) {
 		memcpy(hvt->outmsg, msg, len);
 		hvt->outmsg_len = len;
+		hvt->on_read = on_read_cb;
 		wake_up_interruptible(&hvt->outmsg_q);
 	} else
 		ret = -ENOMEM;
diff --git a/drivers/hv/hv_utils_transport.h b/drivers/hv/hv_utils_transport.h
index 06254a165a18..d98f5225c3e6 100644
--- a/drivers/hv/hv_utils_transport.h
+++ b/drivers/hv/hv_utils_transport.h
@@ -36,6 +36,7 @@ struct hvutil_transport {
 	struct list_head list;              /* hvt_list */
 	int (*on_msg)(void *, int);         /* callback on new user message */
 	void (*on_reset)(void);             /* callback when userspace drops */
+	void (*on_read)(void);              /* callback on message read */
 	u8 *outmsg;                         /* message to the userspace */
 	int outmsg_len;                     /* its length */
 	wait_queue_head_t outmsg_q;         /* poll/read wait queue */
@@ -46,7 +47,8 @@ struct hvutil_transport *hvutil_transport_init(const char *name,
 					       u32 cn_idx, u32 cn_val,
 					       int (*on_msg)(void *, int),
 					       void (*on_reset)(void));
-int hvutil_transport_send(struct hvutil_transport *hvt, void *msg, int len);
+int hvutil_transport_send(struct hvutil_transport *hvt, void *msg, int len,
+			  void (*on_read_cb)(void));
 void hvutil_transport_destroy(struct hvutil_transport *hvt);
 
 #endif /* _HV_UTILS_TRANSPORT_H */
diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h
index 718b5c72f0c8..a5b4442433c8 100644
--- a/drivers/hv/hyperv_vmbus.h
+++ b/drivers/hv/hyperv_vmbus.h
@@ -495,7 +495,7 @@ struct hv_ring_buffer_debug_info {
 
 extern int hv_init(void);
 
-extern void hv_cleanup(void);
+extern void hv_cleanup(bool crash);
 
 extern int hv_post_message(union hv_connection_id connection_id,
 			 enum hv_message_type message_type,
@@ -522,14 +522,15 @@ extern unsigned int host_info_edx;
 /* Interface */
 
 
-int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, void *buffer,
-		   u32 buflen);
+int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
+		       struct page *pages, u32 pagecnt);
 
 void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info);
 
 int hv_ringbuffer_write(struct hv_ring_buffer_info *ring_info,
 		    struct kvec *kv_list,
-		    u32 kv_count, bool *signal, bool lock);
+		    u32 kv_count, bool *signal, bool lock,
+		    enum hv_signal_policy policy);
 
 int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info,
 		       void *buffer, u32 buflen, u32 *buffer_actual_len,
diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c
index fe586bf74e17..08043da1a61c 100644
--- a/drivers/hv/ring_buffer.c
+++ b/drivers/hv/ring_buffer.c
@@ -27,6 +27,8 @@
 #include <linux/mm.h>
 #include <linux/hyperv.h>
 #include <linux/uio.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
 
 #include "hyperv_vmbus.h"
 
@@ -66,12 +68,20 @@ u32 hv_end_read(struct hv_ring_buffer_info *rbi)
  *	   arrived.
  */
 
-static bool hv_need_to_signal(u32 old_write, struct hv_ring_buffer_info *rbi)
+static bool hv_need_to_signal(u32 old_write, struct hv_ring_buffer_info *rbi,
+			      enum hv_signal_policy policy)
 {
 	virt_mb();
 	if (READ_ONCE(rbi->ring_buffer->interrupt_mask))
 		return false;
 
+	/*
+	 * When the client wants to control signaling,
+	 * we only honour the host interrupt mask.
+	 */
+	if (policy == HV_SIGNAL_POLICY_EXPLICIT)
+		return true;
+
 	/* check interrupt_mask before read_index */
 	virt_rmb();
 	/*
@@ -162,18 +172,7 @@ static u32 hv_copyfrom_ringbuffer(
 	void *ring_buffer = hv_get_ring_buffer(ring_info);
 	u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
 
-	u32 frag_len;
-
-	/* wrap-around detected at the src */
-	if (destlen > ring_buffer_size - start_read_offset) {
-		frag_len = ring_buffer_size - start_read_offset;
-
-		memcpy(dest, ring_buffer + start_read_offset, frag_len);
-		memcpy(dest + frag_len, ring_buffer, destlen - frag_len);
-	} else
-
-		memcpy(dest, ring_buffer + start_read_offset, destlen);
-
+	memcpy(dest, ring_buffer + start_read_offset, destlen);
 
 	start_read_offset += destlen;
 	start_read_offset %= ring_buffer_size;
@@ -194,15 +193,8 @@ static u32 hv_copyto_ringbuffer(
 {
 	void *ring_buffer = hv_get_ring_buffer(ring_info);
 	u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
-	u32 frag_len;
 
-	/* wrap-around detected! */
-	if (srclen > ring_buffer_size - start_write_offset) {
-		frag_len = ring_buffer_size - start_write_offset;
-		memcpy(ring_buffer + start_write_offset, src, frag_len);
-		memcpy(ring_buffer, src + frag_len, srclen - frag_len);
-	} else
-		memcpy(ring_buffer + start_write_offset, src, srclen);
+	memcpy(ring_buffer + start_write_offset, src, srclen);
 
 	start_write_offset += srclen;
 	start_write_offset %= ring_buffer_size;
@@ -235,22 +227,46 @@ void hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
 
 /* Initialize the ring buffer. */
 int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
-		   void *buffer, u32 buflen)
+		       struct page *pages, u32 page_cnt)
 {
-	if (sizeof(struct hv_ring_buffer) != PAGE_SIZE)
-		return -EINVAL;
+	int i;
+	struct page **pages_wraparound;
+
+	BUILD_BUG_ON((sizeof(struct hv_ring_buffer) != PAGE_SIZE));
 
 	memset(ring_info, 0, sizeof(struct hv_ring_buffer_info));
 
-	ring_info->ring_buffer = (struct hv_ring_buffer *)buffer;
+	/*
+	 * First page holds struct hv_ring_buffer, do wraparound mapping for
+	 * the rest.
+	 */
+	pages_wraparound = kzalloc(sizeof(struct page *) * (page_cnt * 2 - 1),
+				   GFP_KERNEL);
+	if (!pages_wraparound)
+		return -ENOMEM;
+
+	pages_wraparound[0] = pages;
+	for (i = 0; i < 2 * (page_cnt - 1); i++)
+		pages_wraparound[i + 1] = &pages[i % (page_cnt - 1) + 1];
+
+	ring_info->ring_buffer = (struct hv_ring_buffer *)
+		vmap(pages_wraparound, page_cnt * 2 - 1, VM_MAP, PAGE_KERNEL);
+
+	kfree(pages_wraparound);
+
+
+	if (!ring_info->ring_buffer)
+		return -ENOMEM;
+
 	ring_info->ring_buffer->read_index =
 		ring_info->ring_buffer->write_index = 0;
 
 	/* Set the feature bit for enabling flow control. */
 	ring_info->ring_buffer->feature_bits.value = 1;
 
-	ring_info->ring_size = buflen;
-	ring_info->ring_datasize = buflen - sizeof(struct hv_ring_buffer);
+	ring_info->ring_size = page_cnt << PAGE_SHIFT;
+	ring_info->ring_datasize = ring_info->ring_size -
+		sizeof(struct hv_ring_buffer);
 
 	spin_lock_init(&ring_info->ring_lock);
 
@@ -260,11 +276,13 @@ int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
 /* Cleanup the ring buffer. */
 void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
 {
+	vunmap(ring_info->ring_buffer);
 }
 
 /* Write to the ring buffer. */
 int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info,
-		    struct kvec *kv_list, u32 kv_count, bool *signal, bool lock)
+		    struct kvec *kv_list, u32 kv_count, bool *signal, bool lock,
+		    enum hv_signal_policy policy)
 {
 	int i = 0;
 	u32 bytes_avail_towrite;
@@ -326,7 +344,7 @@ int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info,
 	if (lock)
 		spin_unlock_irqrestore(&outring_info->ring_lock, flags);
 
-	*signal = hv_need_to_signal(old_write, outring_info);
+	*signal = hv_need_to_signal(old_write, outring_info, policy);
 	return 0;
 }
 
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index e82f7e1c217c..a259e18d22d5 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -105,8 +105,8 @@ static struct notifier_block hyperv_panic_block = {
 
 static const char *fb_mmio_name = "fb_range";
 static struct resource *fb_mmio;
-struct resource *hyperv_mmio;
-DEFINE_SEMAPHORE(hyperv_mmio_lock);
+static struct resource *hyperv_mmio;
+static DEFINE_SEMAPHORE(hyperv_mmio_lock);
 
 static int vmbus_exists(void)
 {
@@ -874,7 +874,7 @@ err_alloc:
 	bus_unregister(&hv_bus);
 
 err_cleanup:
-	hv_cleanup();
+	hv_cleanup(false);
 
 	return ret;
 }
@@ -961,8 +961,8 @@ int vmbus_device_register(struct hv_device *child_device_obj)
 {
 	int ret = 0;
 
-	dev_set_name(&child_device_obj->device, "vmbus_%d",
-		     child_device_obj->channel->id);
+	dev_set_name(&child_device_obj->device, "vmbus-%pUl",
+		     child_device_obj->channel->offermsg.offer.if_instance.b);
 
 	child_device_obj->device.bus = &hv_bus;
 	child_device_obj->device.parent = &hv_acpi_dev->dev;
@@ -1326,7 +1326,7 @@ static void hv_kexec_handler(void)
 	vmbus_initiate_unload(false);
 	for_each_online_cpu(cpu)
 		smp_call_function_single(cpu, hv_synic_cleanup, NULL, 1);
-	hv_cleanup();
+	hv_cleanup(false);
 };
 
 static void hv_crash_handler(struct pt_regs *regs)
@@ -1338,7 +1338,7 @@ static void hv_crash_handler(struct pt_regs *regs)
 	 * for kdump.
 	 */
 	hv_synic_cleanup(NULL);
-	hv_cleanup();
+	hv_cleanup(true);
 };
 
 static int __init hv_acpi_init(void)
@@ -1398,7 +1398,7 @@ static void __exit vmbus_exit(void)
 						 &hyperv_panic_block);
 	}
 	bus_unregister(&hv_bus);
-	hv_cleanup();
+	hv_cleanup(false);
 	for_each_online_cpu(cpu) {
 		tasklet_kill(hv_context.event_dpc[cpu]);
 		smp_call_function_single(cpu, hv_synic_cleanup, NULL, 1);
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index eaf2f916d48c..45cef3d2c75c 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -969,7 +969,6 @@ config SENSORS_LM73
 config SENSORS_LM75
 	tristate "National Semiconductor LM75 and compatibles"
 	depends on I2C
-	depends on THERMAL || !THERMAL_OF
 	select REGMAP_I2C
 	help
 	  If you say yes here you get support for one common type of
@@ -1119,6 +1118,7 @@ config SENSORS_LM95241
 config SENSORS_LM95245
 	tristate "National Semiconductor LM95245 and compatibles"
 	depends on I2C
+	select REGMAP_I2C
 	help
 	  If you say yes here you get support for LM95235 and LM95245
 	  temperature sensor chips.
@@ -1572,7 +1572,6 @@ config SENSORS_THMC50
 config SENSORS_TMP102
 	tristate "Texas Instruments TMP102"
 	depends on I2C
-	depends on THERMAL || !THERMAL_OF
 	select REGMAP_I2C
 	help
 	  If you say yes here you get support for Texas Instruments TMP102
@@ -1823,6 +1822,13 @@ config SENSORS_ULTRA45
 	  This driver provides support for the Ultra45 workstation environmental
 	  sensors.
 
+config SENSORS_XGENE
+	tristate "APM X-Gene SoC hardware monitoring driver"
+	depends on XGENE_SLIMPRO_MBOX || PCC
+	help
+	  If you say yes here you get support for the temperature
+	  and power sensors for APM X-Gene SoC.
+
 if ACPI
 
 comment "ACPI drivers"
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index fe87d2895a97..aecf4ba17460 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -165,6 +165,7 @@ obj-$(CONFIG_SENSORS_W83L785TS)	+= w83l785ts.o
 obj-$(CONFIG_SENSORS_W83L786NG)	+= w83l786ng.o
 obj-$(CONFIG_SENSORS_WM831X)	+= wm831x-hwmon.o
 obj-$(CONFIG_SENSORS_WM8350)	+= wm8350-hwmon.o
+obj-$(CONFIG_SENSORS_XGENE)	+= xgene-hwmon.o
 
 obj-$(CONFIG_PMBUS)		+= pmbus/
 
diff --git a/drivers/hwmon/adt7411.c b/drivers/hwmon/adt7411.c
index fc1e65a263a4..812fbc00f693 100644
--- a/drivers/hwmon/adt7411.c
+++ b/drivers/hwmon/adt7411.c
@@ -7,8 +7,7 @@
  *  it under the terms of the GNU General Public License version 2 as
  *  published by the Free Software Foundation.
  *
- *  TODO: SPI, support for external temperature sensor
- *	  use power-down mode for suspend?, interrupt handling?
+ *  TODO: SPI, use power-down mode for suspend?, interrupt handling?
  */
 
 #include <linux/kernel.h>
@@ -31,6 +30,7 @@
 #define ADT7411_REG_CFG1			0x18
 #define ADT7411_CFG1_START_MONITOR		(1 << 0)
 #define ADT7411_CFG1_RESERVED_BIT1		(1 << 1)
+#define ADT7411_CFG1_EXT_TDM			(1 << 2)
 #define ADT7411_CFG1_RESERVED_BIT3		(1 << 3)
 
 #define ADT7411_REG_CFG2			0x19
@@ -57,6 +57,7 @@ struct adt7411_data {
 	unsigned long next_update;
 	int vref_cached;
 	struct i2c_client *client;
+	bool use_ext_temp;
 };
 
 /*
@@ -127,11 +128,20 @@ static ssize_t adt7411_show_vdd(struct device *dev,
 static ssize_t adt7411_show_temp(struct device *dev,
 			struct device_attribute *attr, char *buf)
 {
+	int nr = to_sensor_dev_attr(attr)->index;
 	struct adt7411_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
-	int val = adt7411_read_10_bit(client, ADT7411_REG_INT_TEMP_VDD_LSB,
-			ADT7411_REG_INT_TEMP_MSB, 0);
-
+	int val;
+	struct {
+		u8 low;
+		u8 high;
+	} reg[2] = {
+		{ ADT7411_REG_INT_TEMP_VDD_LSB, ADT7411_REG_INT_TEMP_MSB },
+		{ ADT7411_REG_EXT_TEMP_AIN14_LSB,
+		  ADT7411_REG_EXT_TEMP_AIN1_MSB },
+	};
+
+	val = adt7411_read_10_bit(client, reg[nr].low, reg[nr].high, 0);
 	if (val < 0)
 		return val;
 
@@ -218,11 +228,13 @@ static ssize_t adt7411_set_bit(struct device *dev,
 	return ret < 0 ? ret : count;
 }
 
+
 #define ADT7411_BIT_ATTR(__name, __reg, __bit) \
 	SENSOR_DEVICE_ATTR_2(__name, S_IRUGO | S_IWUSR, adt7411_show_bit, \
 	adt7411_set_bit, __bit, __reg)
 
-static DEVICE_ATTR(temp1_input, S_IRUGO, adt7411_show_temp, NULL);
+static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, adt7411_show_temp, NULL, 0);
+static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, adt7411_show_temp, NULL, 1);
 static DEVICE_ATTR(in0_input, S_IRUGO, adt7411_show_vdd, NULL);
 static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, adt7411_show_input, NULL, 0);
 static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, adt7411_show_input, NULL, 1);
@@ -237,7 +249,8 @@ static ADT7411_BIT_ATTR(fast_sampling, ADT7411_REG_CFG3, ADT7411_CFG3_ADC_CLK_22
 static ADT7411_BIT_ATTR(adc_ref_vdd, ADT7411_REG_CFG3, ADT7411_CFG3_REF_VDD);
 
 static struct attribute *adt7411_attrs[] = {
-	&dev_attr_temp1_input.attr,
+	&sensor_dev_attr_temp1_input.dev_attr.attr,
+	&sensor_dev_attr_temp2_input.dev_attr.attr,
 	&dev_attr_in0_input.attr,
 	&sensor_dev_attr_in1_input.dev_attr.attr,
 	&sensor_dev_attr_in2_input.dev_attr.attr,
@@ -253,7 +266,27 @@ static struct attribute *adt7411_attrs[] = {
 	NULL
 };
 
-ATTRIBUTE_GROUPS(adt7411);
+static umode_t adt7411_attrs_visible(struct kobject *kobj,
+				     struct attribute *attr, int index)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct adt7411_data *data = dev_get_drvdata(dev);
+	bool visible = true;
+
+	if (attr == &sensor_dev_attr_temp2_input.dev_attr.attr)
+		visible = data->use_ext_temp;
+	else if (attr == &sensor_dev_attr_in1_input.dev_attr.attr ||
+		 attr == &sensor_dev_attr_in2_input.dev_attr.attr)
+		visible = !data->use_ext_temp;
+
+	return visible ? attr->mode : 0;
+}
+
+static const struct attribute_group adt7411_group = {
+	.attrs = adt7411_attrs,
+	.is_visible = adt7411_attrs_visible,
+};
+__ATTRIBUTE_GROUPS(adt7411);
 
 static int adt7411_detect(struct i2c_client *client,
 			  struct i2c_board_info *info)
@@ -309,6 +342,8 @@ static int adt7411_init_device(struct adt7411_data *data)
 	if (ret < 0)
 		return ret;
 
+	data->use_ext_temp = ret & ADT7411_CFG1_EXT_TDM;
+
 	/*
 	 * We must only write zero to bit 1 and only one to bit 3 according to
 	 * the datasheet.
diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c
index f5da39a68929..6e60ca53406e 100644
--- a/drivers/hwmon/adt7470.c
+++ b/drivers/hwmon/adt7470.c
@@ -32,6 +32,7 @@
 #include <linux/log2.h>
 #include <linux/kthread.h>
 #include <linux/slab.h>
+#include <linux/util_macros.h>
 
 /* Addresses to scan */
 static const unsigned short normal_i2c[] = { 0x2C, 0x2E, 0x2F, I2C_CLIENT_END };
@@ -83,6 +84,7 @@ static const unsigned short normal_i2c[] = { 0x2C, 0x2E, 0x2F, I2C_CLIENT_END };
 #define ADT7470_REG_PWM_MIN_MAX_ADDR		0x6D
 #define ADT7470_REG_PWM_TEMP_MIN_BASE_ADDR	0x6E
 #define ADT7470_REG_PWM_TEMP_MIN_MAX_ADDR	0x71
+#define ADT7470_REG_CFG_2			0x74
 #define ADT7470_REG_ACOUSTICS12			0x75
 #define ADT7470_REG_ACOUSTICS34			0x76
 #define ADT7470_REG_DEVICE			0x3D
@@ -142,6 +144,11 @@ static const unsigned short normal_i2c[] = { 0x2C, 0x2E, 0x2F, I2C_CLIENT_END };
 #define FAN_PERIOD_INVALID	65535
 #define FAN_DATA_VALID(x)	((x) && (x) != FAN_PERIOD_INVALID)
 
+/* Config registers 1 and 2 include fields for selecting the PWM frequency */
+#define ADT7470_CFG_LF		0x40
+#define ADT7470_FREQ_MASK	0x70
+#define ADT7470_FREQ_SHIFT	4
+
 struct adt7470_data {
 	struct i2c_client	*client;
 	struct mutex		lock;
@@ -170,7 +177,6 @@ struct adt7470_data {
 	u8			pwm_auto_temp[ADT7470_PWM_COUNT];
 
 	struct task_struct	*auto_update;
-	struct completion	auto_update_stop;
 	unsigned int		auto_update_interval;
 };
 
@@ -266,12 +272,14 @@ static int adt7470_update_thread(void *p)
 		mutex_lock(&data->lock);
 		adt7470_read_temperatures(client, data);
 		mutex_unlock(&data->lock);
+
+		set_current_state(TASK_INTERRUPTIBLE);
 		if (kthread_should_stop())
 			break;
-		msleep_interruptible(data->auto_update_interval);
+
+		schedule_timeout(msecs_to_jiffies(data->auto_update_interval));
 	}
 
-	complete_all(&data->auto_update_stop);
 	return 0;
 }
 
@@ -538,6 +546,28 @@ static ssize_t show_alarm_mask(struct device *dev,
 	return sprintf(buf, "%x\n", data->alarms_mask);
 }
 
+static ssize_t set_alarm_mask(struct device *dev,
+			      struct device_attribute *devattr,
+			      const char *buf,
+			      size_t count)
+{
+	struct adt7470_data *data = dev_get_drvdata(dev);
+	long mask;
+
+	if (kstrtoul(buf, 0, &mask))
+		return -EINVAL;
+
+	if (mask & ~0xffff)
+		return -EINVAL;
+
+	mutex_lock(&data->lock);
+	data->alarms_mask = mask;
+	adt7470_write_word_data(data->client, ADT7470_REG_ALARM1_MASK, mask);
+	mutex_unlock(&data->lock);
+
+	return count;
+}
+
 static ssize_t show_fan_max(struct device *dev,
 			    struct device_attribute *devattr,
 			    char *buf)
@@ -688,6 +718,70 @@ static ssize_t set_pwm(struct device *dev, struct device_attribute *devattr,
 	return count;
 }
 
+/* These are the valid PWM frequencies to the nearest Hz */
+static const int adt7470_freq_map[] = {
+	11, 15, 22, 29, 35, 44, 59, 88, 1400, 22500
+};
+
+static ssize_t show_pwm_freq(struct device *dev,
+			     struct device_attribute *devattr, char *buf)
+{
+	struct adt7470_data *data = adt7470_update_device(dev);
+	unsigned char cfg_reg_1;
+	unsigned char cfg_reg_2;
+	int index;
+
+	mutex_lock(&data->lock);
+	cfg_reg_1 = i2c_smbus_read_byte_data(data->client, ADT7470_REG_CFG);
+	cfg_reg_2 = i2c_smbus_read_byte_data(data->client, ADT7470_REG_CFG_2);
+	mutex_unlock(&data->lock);
+
+	index = (cfg_reg_2 & ADT7470_FREQ_MASK) >> ADT7470_FREQ_SHIFT;
+	if (!(cfg_reg_1 & ADT7470_CFG_LF))
+		index += 8;
+	if (index >= ARRAY_SIZE(adt7470_freq_map))
+		index = ARRAY_SIZE(adt7470_freq_map) - 1;
+
+	return scnprintf(buf, PAGE_SIZE, "%d\n", adt7470_freq_map[index]);
+}
+
+static ssize_t set_pwm_freq(struct device *dev,
+			    struct device_attribute *devattr,
+			    const char *buf, size_t count)
+{
+	struct adt7470_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	long freq;
+	int index;
+	int low_freq = ADT7470_CFG_LF;
+	unsigned char val;
+
+	if (kstrtol(buf, 10, &freq))
+		return -EINVAL;
+
+	/* Round the user value given to the closest available frequency */
+	index = find_closest(freq, adt7470_freq_map,
+			     ARRAY_SIZE(adt7470_freq_map));
+
+	if (index >= 8) {
+		index -= 8;
+		low_freq = 0;
+	}
+
+	mutex_lock(&data->lock);
+	/* Configuration Register 1 */
+	val = i2c_smbus_read_byte_data(client, ADT7470_REG_CFG);
+	i2c_smbus_write_byte_data(client, ADT7470_REG_CFG,
+				  (val & ~ADT7470_CFG_LF) | low_freq);
+	/* Configuration Register 2 */
+	val = i2c_smbus_read_byte_data(client, ADT7470_REG_CFG_2);
+	i2c_smbus_write_byte_data(client, ADT7470_REG_CFG_2,
+		(val & ~ADT7470_FREQ_MASK) | (index << ADT7470_FREQ_SHIFT));
+	mutex_unlock(&data->lock);
+
+	return count;
+}
+
 static ssize_t show_pwm_max(struct device *dev,
 			    struct device_attribute *devattr,
 			    char *buf)
@@ -918,7 +1012,8 @@ static ssize_t show_alarm(struct device *dev,
 		return sprintf(buf, "0\n");
 }
 
-static DEVICE_ATTR(alarm_mask, S_IRUGO, show_alarm_mask, NULL);
+static DEVICE_ATTR(alarm_mask, S_IWUSR | S_IRUGO, show_alarm_mask,
+		   set_alarm_mask);
 static DEVICE_ATTR(num_temp_sensors, S_IWUSR | S_IRUGO, show_num_temp_sensors,
 		   set_num_temp_sensors);
 static DEVICE_ATTR(auto_update_interval, S_IWUSR | S_IRUGO,
@@ -1038,6 +1133,8 @@ static SENSOR_DEVICE_ATTR(pwm2, S_IWUSR | S_IRUGO, show_pwm, set_pwm, 1);
 static SENSOR_DEVICE_ATTR(pwm3, S_IWUSR | S_IRUGO, show_pwm, set_pwm, 2);
 static SENSOR_DEVICE_ATTR(pwm4, S_IWUSR | S_IRUGO, show_pwm, set_pwm, 3);
 
+static DEVICE_ATTR(pwm1_freq, S_IWUSR | S_IRUGO, show_pwm_freq, set_pwm_freq);
+
 static SENSOR_DEVICE_ATTR(pwm1_auto_point1_pwm, S_IWUSR | S_IRUGO,
 		    show_pwm_min, set_pwm_min, 0);
 static SENSOR_DEVICE_ATTR(pwm2_auto_point1_pwm, S_IWUSR | S_IRUGO,
@@ -1154,6 +1251,7 @@ static struct attribute *adt7470_attrs[] = {
 	&sensor_dev_attr_fan4_alarm.dev_attr.attr,
 	&sensor_dev_attr_force_pwm_max.dev_attr.attr,
 	&sensor_dev_attr_pwm1.dev_attr.attr,
+	&dev_attr_pwm1_freq.attr,
 	&sensor_dev_attr_pwm2.dev_attr.attr,
 	&sensor_dev_attr_pwm3.dev_attr.attr,
 	&sensor_dev_attr_pwm4.dev_attr.attr,
@@ -1256,7 +1354,6 @@ static int adt7470_probe(struct i2c_client *client,
 	if (IS_ERR(hwmon_dev))
 		return PTR_ERR(hwmon_dev);
 
-	init_completion(&data->auto_update_stop);
 	data->auto_update = kthread_run(adt7470_update_thread, client, "%s",
 					dev_name(hwmon_dev));
 	if (IS_ERR(data->auto_update)) {
@@ -1271,7 +1368,6 @@ static int adt7470_remove(struct i2c_client *client)
 	struct adt7470_data *data = i2c_get_clientdata(client);
 
 	kthread_stop(data->auto_update);
-	wait_for_completion(&data->auto_update_stop);
 	return 0;
 }
 
diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c
index acf9c0361d9f..34704b0451b4 100644
--- a/drivers/hwmon/dell-smm-hwmon.c
+++ b/drivers/hwmon/dell-smm-hwmon.c
@@ -21,6 +21,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/cpu.h>
 #include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/types.h>
@@ -36,6 +37,7 @@
 #include <linux/io.h>
 #include <linux/sched.h>
 #include <linux/ctype.h>
+#include <linux/smp.h>
 
 #include <linux/i8k.h>
 
@@ -134,11 +136,11 @@ static inline const char *i8k_get_dmi_data(int field)
 /*
  * Call the System Management Mode BIOS. Code provided by Jonathan Buzzard.
  */
-static int i8k_smm(struct smm_regs *regs)
+static int i8k_smm_func(void *par)
 {
 	int rc;
+	struct smm_regs *regs = par;
 	int eax = regs->eax;
-	cpumask_var_t old_mask;
 
 #ifdef DEBUG
 	int ebx = regs->ebx;
@@ -149,16 +151,8 @@ static int i8k_smm(struct smm_regs *regs)
 #endif
 
 	/* SMM requires CPU 0 */
-	if (!alloc_cpumask_var(&old_mask, GFP_KERNEL))
-		return -ENOMEM;
-	cpumask_copy(old_mask, &current->cpus_allowed);
-	rc = set_cpus_allowed_ptr(current, cpumask_of(0));
-	if (rc)
-		goto out;
-	if (smp_processor_id() != 0) {
-		rc = -EBUSY;
-		goto out;
-	}
+	if (smp_processor_id() != 0)
+		return -EBUSY;
 
 #if defined(CONFIG_X86_64)
 	asm volatile("pushq %%rax\n\t"
@@ -216,10 +210,6 @@ static int i8k_smm(struct smm_regs *regs)
 	if (rc != 0 || (regs->eax & 0xffff) == 0xffff || regs->eax == eax)
 		rc = -EINVAL;
 
-out:
-	set_cpus_allowed_ptr(current, old_mask);
-	free_cpumask_var(old_mask);
-
 #ifdef DEBUG
 	rettime = ktime_get();
 	delta = ktime_sub(rettime, calltime);
@@ -232,6 +222,20 @@ out:
 }
 
 /*
+ * Call the System Management Mode BIOS.
+ */
+static int i8k_smm(struct smm_regs *regs)
+{
+	int ret;
+
+	get_online_cpus();
+	ret = smp_call_on_cpu(0, i8k_smm_func, regs, true);
+	put_online_cpus();
+
+	return ret;
+}
+
+/*
  * Read the fan status.
  */
 static int i8k_get_fan_status(int fan)
diff --git a/drivers/hwmon/ftsteutates.c b/drivers/hwmon/ftsteutates.c
index 48633e541dc3..0f0277e7aae5 100644
--- a/drivers/hwmon/ftsteutates.c
+++ b/drivers/hwmon/ftsteutates.c
@@ -36,6 +36,10 @@
 #define FTS_EVENT_STATUS_REG		0x0006
 #define FTS_GLOBAL_CONTROL_REG		0x0007
 
+#define FTS_DEVICE_DETECT_REG_1		0x0C
+#define FTS_DEVICE_DETECT_REG_2		0x0D
+#define FTS_DEVICE_DETECT_REG_3		0x0E
+
 #define FTS_SENSOR_EVENT_REG		0x0010
 
 #define FTS_FAN_EVENT_REG		0x0014
@@ -54,6 +58,8 @@
 #define FTS_NO_TEMP_SENSORS		0x10
 #define FTS_NO_VOLT_SENSORS		0x04
 
+static const unsigned short normal_i2c[] = { 0x73, I2C_CLIENT_END };
+
 static struct i2c_device_id fts_id[] = {
 	{ "ftsteutates", 0 },
 	{ }
@@ -734,6 +740,42 @@ static const struct attribute_group *fts_attr_groups[] = {
 /*****************************************************************************/
 /* Module initialization / remove functions				     */
 /*****************************************************************************/
+static int fts_detect(struct i2c_client *client,
+		      struct i2c_board_info *info)
+{
+	int val;
+
+	/* detection works with revsion greater or equal to 0x2b */
+	val = i2c_smbus_read_byte_data(client, FTS_DEVICE_REVISION_REG);
+	if (val < 0x2b)
+		return -ENODEV;
+
+	/* Device Detect Regs must have 0x17 0x34 and 0x54 */
+	val = i2c_smbus_read_byte_data(client, FTS_DEVICE_DETECT_REG_1);
+	if (val != 0x17)
+		return -ENODEV;
+
+	val = i2c_smbus_read_byte_data(client, FTS_DEVICE_DETECT_REG_2);
+	if (val != 0x34)
+		return -ENODEV;
+
+	val = i2c_smbus_read_byte_data(client, FTS_DEVICE_DETECT_REG_3);
+	if (val != 0x54)
+		return -ENODEV;
+
+	/*
+	 * 0x10 == Baseboard Management Controller, 0x01 == Teutates
+	 * Device ID Reg needs to be 0x11
+	 */
+	val = i2c_smbus_read_byte_data(client, FTS_DEVICE_ID_REG);
+	if (val != 0x11)
+		return -ENODEV;
+
+	strlcpy(info->type, fts_id[0].name, I2C_NAME_SIZE);
+	info->flags = 0;
+	return 0;
+}
+
 static int fts_remove(struct i2c_client *client)
 {
 	struct fts_data *data = dev_get_drvdata(&client->dev);
@@ -804,12 +846,15 @@ static int fts_probe(struct i2c_client *client, const struct i2c_device_id *id)
 /* Module Details							     */
 /*****************************************************************************/
 static struct i2c_driver fts_driver = {
+	.class = I2C_CLASS_HWMON,
 	.driver = {
 		.name = "ftsteutates",
 	},
 	.id_table = fts_id,
 	.probe = fts_probe,
 	.remove = fts_remove,
+	.detect = fts_detect,
+	.address_list = normal_i2c,
 };
 
 module_i2c_driver(fts_driver);
diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
index a26c385a435b..adae6848ffb2 100644
--- a/drivers/hwmon/hwmon.c
+++ b/drivers/hwmon/hwmon.c
@@ -12,17 +12,17 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/module.h>
+#include <linux/bitops.h>
 #include <linux/device.h>
 #include <linux/err.h>
-#include <linux/slab.h>
-#include <linux/kdev_t.h>
-#include <linux/idr.h>
-#include <linux/hwmon.h>
 #include <linux/gfp.h>
-#include <linux/spinlock.h>
+#include <linux/hwmon.h>
+#include <linux/idr.h>
+#include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/slab.h>
 #include <linux/string.h>
+#include <linux/thermal.h>
 
 #define HWMON_ID_PREFIX "hwmon"
 #define HWMON_ID_FORMAT HWMON_ID_PREFIX "%d"
@@ -30,9 +30,35 @@
 struct hwmon_device {
 	const char *name;
 	struct device dev;
+	const struct hwmon_chip_info *chip;
+
+	struct attribute_group group;
+	const struct attribute_group **groups;
 };
+
 #define to_hwmon_device(d) container_of(d, struct hwmon_device, dev)
 
+struct hwmon_device_attribute {
+	struct device_attribute dev_attr;
+	const struct hwmon_ops *ops;
+	enum hwmon_sensor_types type;
+	u32 attr;
+	int index;
+};
+
+#define to_hwmon_attr(d) \
+	container_of(d, struct hwmon_device_attribute, dev_attr)
+
+/*
+ * Thermal zone information
+ * In addition to the reference to the hwmon device,
+ * also provides the sensor index.
+ */
+struct hwmon_thermal_data {
+	struct hwmon_device *hwdev;	/* Reference to hwmon device */
+	int index;			/* sensor index */
+};
+
 static ssize_t
 show_name(struct device *dev, struct device_attribute *attr, char *buf)
 {
@@ -80,25 +106,409 @@ static struct class hwmon_class = {
 
 static DEFINE_IDA(hwmon_ida);
 
-/**
- * hwmon_device_register_with_groups - register w/ hwmon
- * @dev: the parent device
- * @name: hwmon name attribute
- * @drvdata: driver data to attach to created device
- * @groups: List of attribute groups to create
- *
- * hwmon_device_unregister() must be called when the device is no
- * longer needed.
- *
- * Returns the pointer to the new device.
+/* Thermal zone handling */
+
+/*
+ * The complex conditional is necessary to avoid a cyclic dependency
+ * between hwmon and thermal_sys modules.
  */
-struct device *
-hwmon_device_register_with_groups(struct device *dev, const char *name,
-				  void *drvdata,
-				  const struct attribute_group **groups)
+#if IS_REACHABLE(CONFIG_THERMAL) && defined(CONFIG_THERMAL_OF) && \
+	(!defined(CONFIG_THERMAL_HWMON) || \
+	 !(defined(MODULE) && IS_MODULE(CONFIG_THERMAL)))
+static int hwmon_thermal_get_temp(void *data, int *temp)
+{
+	struct hwmon_thermal_data *tdata = data;
+	struct hwmon_device *hwdev = tdata->hwdev;
+	int ret;
+	long t;
+
+	ret = hwdev->chip->ops->read(&hwdev->dev, hwmon_temp, hwmon_temp_input,
+				     tdata->index, &t);
+	if (ret < 0)
+		return ret;
+
+	*temp = t;
+
+	return 0;
+}
+
+static struct thermal_zone_of_device_ops hwmon_thermal_ops = {
+	.get_temp = hwmon_thermal_get_temp,
+};
+
+static int hwmon_thermal_add_sensor(struct device *dev,
+				    struct hwmon_device *hwdev, int index)
+{
+	struct hwmon_thermal_data *tdata;
+
+	tdata = devm_kzalloc(dev, sizeof(*tdata), GFP_KERNEL);
+	if (!tdata)
+		return -ENOMEM;
+
+	tdata->hwdev = hwdev;
+	tdata->index = index;
+
+	devm_thermal_zone_of_sensor_register(&hwdev->dev, index, tdata,
+					     &hwmon_thermal_ops);
+
+	return 0;
+}
+#else
+static int hwmon_thermal_add_sensor(struct device *dev,
+				    struct hwmon_device *hwdev, int index)
+{
+	return 0;
+}
+#endif /* IS_REACHABLE(CONFIG_THERMAL) && ... */
+
+/* sysfs attribute management */
+
+static ssize_t hwmon_attr_show(struct device *dev,
+			       struct device_attribute *devattr, char *buf)
+{
+	struct hwmon_device_attribute *hattr = to_hwmon_attr(devattr);
+	long val;
+	int ret;
+
+	ret = hattr->ops->read(dev, hattr->type, hattr->attr, hattr->index,
+			       &val);
+	if (ret < 0)
+		return ret;
+
+	return sprintf(buf, "%ld\n", val);
+}
+
+static ssize_t hwmon_attr_store(struct device *dev,
+				struct device_attribute *devattr,
+				const char *buf, size_t count)
+{
+	struct hwmon_device_attribute *hattr = to_hwmon_attr(devattr);
+	long val;
+	int ret;
+
+	ret = kstrtol(buf, 10, &val);
+	if (ret < 0)
+		return ret;
+
+	ret = hattr->ops->write(dev, hattr->type, hattr->attr, hattr->index,
+				val);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static int hwmon_attr_base(enum hwmon_sensor_types type)
+{
+	if (type == hwmon_in)
+		return 0;
+	return 1;
+}
+
+static struct attribute *hwmon_genattr(struct device *dev,
+				       const void *drvdata,
+				       enum hwmon_sensor_types type,
+				       u32 attr,
+				       int index,
+				       const char *template,
+				       const struct hwmon_ops *ops)
+{
+	struct hwmon_device_attribute *hattr;
+	struct device_attribute *dattr;
+	struct attribute *a;
+	umode_t mode;
+	char *name;
+
+	/* The attribute is invisible if there is no template string */
+	if (!template)
+		return ERR_PTR(-ENOENT);
+
+	mode = ops->is_visible(drvdata, type, attr, index);
+	if (!mode)
+		return ERR_PTR(-ENOENT);
+
+	if ((mode & S_IRUGO) && !ops->read)
+		return ERR_PTR(-EINVAL);
+	if ((mode & S_IWUGO) && !ops->write)
+		return ERR_PTR(-EINVAL);
+
+	if (type == hwmon_chip) {
+		name = (char *)template;
+	} else {
+		name = devm_kzalloc(dev, strlen(template) + 16, GFP_KERNEL);
+		if (!name)
+			return ERR_PTR(-ENOMEM);
+		scnprintf(name, strlen(template) + 16, template,
+			  index + hwmon_attr_base(type));
+	}
+
+	hattr = devm_kzalloc(dev, sizeof(*hattr), GFP_KERNEL);
+	if (!hattr)
+		return ERR_PTR(-ENOMEM);
+
+	hattr->type = type;
+	hattr->attr = attr;
+	hattr->index = index;
+	hattr->ops = ops;
+
+	dattr = &hattr->dev_attr;
+	dattr->show = hwmon_attr_show;
+	dattr->store = hwmon_attr_store;
+
+	a = &dattr->attr;
+	sysfs_attr_init(a);
+	a->name = name;
+	a->mode = mode;
+
+	return a;
+}
+
+static const char * const hwmon_chip_attr_templates[] = {
+	[hwmon_chip_temp_reset_history] = "temp_reset_history",
+	[hwmon_chip_in_reset_history] = "in_reset_history",
+	[hwmon_chip_curr_reset_history] = "curr_reset_history",
+	[hwmon_chip_power_reset_history] = "power_reset_history",
+	[hwmon_chip_update_interval] = "update_interval",
+	[hwmon_chip_alarms] = "alarms",
+};
+
+static const char * const hwmon_temp_attr_templates[] = {
+	[hwmon_temp_input] = "temp%d_input",
+	[hwmon_temp_type] = "temp%d_type",
+	[hwmon_temp_lcrit] = "temp%d_lcrit",
+	[hwmon_temp_lcrit_hyst] = "temp%d_lcrit_hyst",
+	[hwmon_temp_min] = "temp%d_min",
+	[hwmon_temp_min_hyst] = "temp%d_min_hyst",
+	[hwmon_temp_max] = "temp%d_max",
+	[hwmon_temp_max_hyst] = "temp%d_max_hyst",
+	[hwmon_temp_crit] = "temp%d_crit",
+	[hwmon_temp_crit_hyst] = "temp%d_crit_hyst",
+	[hwmon_temp_emergency] = "temp%d_emergency",
+	[hwmon_temp_emergency_hyst] = "temp%d_emergency_hyst",
+	[hwmon_temp_alarm] = "temp%d_alarm",
+	[hwmon_temp_lcrit_alarm] = "temp%d_lcrit_alarm",
+	[hwmon_temp_min_alarm] = "temp%d_min_alarm",
+	[hwmon_temp_max_alarm] = "temp%d_max_alarm",
+	[hwmon_temp_crit_alarm] = "temp%d_crit_alarm",
+	[hwmon_temp_emergency_alarm] = "temp%d_emergency_alarm",
+	[hwmon_temp_fault] = "temp%d_fault",
+	[hwmon_temp_offset] = "temp%d_offset",
+	[hwmon_temp_label] = "temp%d_label",
+	[hwmon_temp_lowest] = "temp%d_lowest",
+	[hwmon_temp_highest] = "temp%d_highest",
+	[hwmon_temp_reset_history] = "temp%d_reset_history",
+};
+
+static const char * const hwmon_in_attr_templates[] = {
+	[hwmon_in_input] = "in%d_input",
+	[hwmon_in_min] = "in%d_min",
+	[hwmon_in_max] = "in%d_max",
+	[hwmon_in_lcrit] = "in%d_lcrit",
+	[hwmon_in_crit] = "in%d_crit",
+	[hwmon_in_average] = "in%d_average",
+	[hwmon_in_lowest] = "in%d_lowest",
+	[hwmon_in_highest] = "in%d_highest",
+	[hwmon_in_reset_history] = "in%d_reset_history",
+	[hwmon_in_label] = "in%d_label",
+	[hwmon_in_alarm] = "in%d_alarm",
+	[hwmon_in_min_alarm] = "in%d_min_alarm",
+	[hwmon_in_max_alarm] = "in%d_max_alarm",
+	[hwmon_in_lcrit_alarm] = "in%d_lcrit_alarm",
+	[hwmon_in_crit_alarm] = "in%d_crit_alarm",
+};
+
+static const char * const hwmon_curr_attr_templates[] = {
+	[hwmon_curr_input] = "curr%d_input",
+	[hwmon_curr_min] = "curr%d_min",
+	[hwmon_curr_max] = "curr%d_max",
+	[hwmon_curr_lcrit] = "curr%d_lcrit",
+	[hwmon_curr_crit] = "curr%d_crit",
+	[hwmon_curr_average] = "curr%d_average",
+	[hwmon_curr_lowest] = "curr%d_lowest",
+	[hwmon_curr_highest] = "curr%d_highest",
+	[hwmon_curr_reset_history] = "curr%d_reset_history",
+	[hwmon_curr_label] = "curr%d_label",
+	[hwmon_curr_alarm] = "curr%d_alarm",
+	[hwmon_curr_min_alarm] = "curr%d_min_alarm",
+	[hwmon_curr_max_alarm] = "curr%d_max_alarm",
+	[hwmon_curr_lcrit_alarm] = "curr%d_lcrit_alarm",
+	[hwmon_curr_crit_alarm] = "curr%d_crit_alarm",
+};
+
+static const char * const hwmon_power_attr_templates[] = {
+	[hwmon_power_average] = "power%d_average",
+	[hwmon_power_average_interval] = "power%d_average_interval",
+	[hwmon_power_average_interval_max] = "power%d_interval_max",
+	[hwmon_power_average_interval_min] = "power%d_interval_min",
+	[hwmon_power_average_highest] = "power%d_average_highest",
+	[hwmon_power_average_lowest] = "power%d_average_lowest",
+	[hwmon_power_average_max] = "power%d_average_max",
+	[hwmon_power_average_min] = "power%d_average_min",
+	[hwmon_power_input] = "power%d_input",
+	[hwmon_power_input_highest] = "power%d_input_highest",
+	[hwmon_power_input_lowest] = "power%d_input_lowest",
+	[hwmon_power_reset_history] = "power%d_reset_history",
+	[hwmon_power_accuracy] = "power%d_accuracy",
+	[hwmon_power_cap] = "power%d_cap",
+	[hwmon_power_cap_hyst] = "power%d_cap_hyst",
+	[hwmon_power_cap_max] = "power%d_cap_max",
+	[hwmon_power_cap_min] = "power%d_cap_min",
+	[hwmon_power_max] = "power%d_max",
+	[hwmon_power_crit] = "power%d_crit",
+	[hwmon_power_label] = "power%d_label",
+	[hwmon_power_alarm] = "power%d_alarm",
+	[hwmon_power_cap_alarm] = "power%d_cap_alarm",
+	[hwmon_power_max_alarm] = "power%d_max_alarm",
+	[hwmon_power_crit_alarm] = "power%d_crit_alarm",
+};
+
+static const char * const hwmon_energy_attr_templates[] = {
+	[hwmon_energy_input] = "energy%d_input",
+	[hwmon_energy_label] = "energy%d_label",
+};
+
+static const char * const hwmon_humidity_attr_templates[] = {
+	[hwmon_humidity_input] = "humidity%d_input",
+	[hwmon_humidity_label] = "humidity%d_label",
+	[hwmon_humidity_min] = "humidity%d_min",
+	[hwmon_humidity_min_hyst] = "humidity%d_min_hyst",
+	[hwmon_humidity_max] = "humidity%d_max",
+	[hwmon_humidity_max_hyst] = "humidity%d_max_hyst",
+	[hwmon_humidity_alarm] = "humidity%d_alarm",
+	[hwmon_humidity_fault] = "humidity%d_fault",
+};
+
+static const char * const hwmon_fan_attr_templates[] = {
+	[hwmon_fan_input] = "fan%d_input",
+	[hwmon_fan_label] = "fan%d_label",
+	[hwmon_fan_min] = "fan%d_min",
+	[hwmon_fan_max] = "fan%d_max",
+	[hwmon_fan_div] = "fan%d_div",
+	[hwmon_fan_pulses] = "fan%d_pulses",
+	[hwmon_fan_target] = "fan%d_target",
+	[hwmon_fan_alarm] = "fan%d_alarm",
+	[hwmon_fan_min_alarm] = "fan%d_min_alarm",
+	[hwmon_fan_max_alarm] = "fan%d_max_alarm",
+	[hwmon_fan_fault] = "fan%d_fault",
+};
+
+static const char * const hwmon_pwm_attr_templates[] = {
+	[hwmon_pwm_input] = "pwm%d",
+	[hwmon_pwm_enable] = "pwm%d_enable",
+	[hwmon_pwm_mode] = "pwm%d_mode",
+	[hwmon_pwm_freq] = "pwm%d_freq",
+};
+
+static const char * const *__templates[] = {
+	[hwmon_chip] = hwmon_chip_attr_templates,
+	[hwmon_temp] = hwmon_temp_attr_templates,
+	[hwmon_in] = hwmon_in_attr_templates,
+	[hwmon_curr] = hwmon_curr_attr_templates,
+	[hwmon_power] = hwmon_power_attr_templates,
+	[hwmon_energy] = hwmon_energy_attr_templates,
+	[hwmon_humidity] = hwmon_humidity_attr_templates,
+	[hwmon_fan] = hwmon_fan_attr_templates,
+	[hwmon_pwm] = hwmon_pwm_attr_templates,
+};
+
+static const int __templates_size[] = {
+	[hwmon_chip] = ARRAY_SIZE(hwmon_chip_attr_templates),
+	[hwmon_temp] = ARRAY_SIZE(hwmon_temp_attr_templates),
+	[hwmon_in] = ARRAY_SIZE(hwmon_in_attr_templates),
+	[hwmon_curr] = ARRAY_SIZE(hwmon_curr_attr_templates),
+	[hwmon_power] = ARRAY_SIZE(hwmon_power_attr_templates),
+	[hwmon_energy] = ARRAY_SIZE(hwmon_energy_attr_templates),
+	[hwmon_humidity] = ARRAY_SIZE(hwmon_humidity_attr_templates),
+	[hwmon_fan] = ARRAY_SIZE(hwmon_fan_attr_templates),
+	[hwmon_pwm] = ARRAY_SIZE(hwmon_pwm_attr_templates),
+};
+
+static int hwmon_num_channel_attrs(const struct hwmon_channel_info *info)
+{
+	int i, n;
+
+	for (i = n = 0; info->config[i]; i++)
+		n += hweight32(info->config[i]);
+
+	return n;
+}
+
+static int hwmon_genattrs(struct device *dev,
+			  const void *drvdata,
+			  struct attribute **attrs,
+			  const struct hwmon_ops *ops,
+			  const struct hwmon_channel_info *info)
+{
+	const char * const *templates;
+	int template_size;
+	int i, aindex = 0;
+
+	if (info->type >= ARRAY_SIZE(__templates))
+		return -EINVAL;
+
+	templates = __templates[info->type];
+	template_size = __templates_size[info->type];
+
+	for (i = 0; info->config[i]; i++) {
+		u32 attr_mask = info->config[i];
+		u32 attr;
+
+		while (attr_mask) {
+			struct attribute *a;
+
+			attr = __ffs(attr_mask);
+			attr_mask &= ~BIT(attr);
+			if (attr >= template_size)
+				return -EINVAL;
+			a = hwmon_genattr(dev, drvdata, info->type, attr, i,
+					  templates[attr], ops);
+			if (IS_ERR(a)) {
+				if (PTR_ERR(a) != -ENOENT)
+					return PTR_ERR(a);
+				continue;
+			}
+			attrs[aindex++] = a;
+		}
+	}
+	return aindex;
+}
+
+static struct attribute **
+__hwmon_create_attrs(struct device *dev, const void *drvdata,
+		     const struct hwmon_chip_info *chip)
+{
+	int ret, i, aindex = 0, nattrs = 0;
+	struct attribute **attrs;
+
+	for (i = 0; chip->info[i]; i++)
+		nattrs += hwmon_num_channel_attrs(chip->info[i]);
+
+	if (nattrs == 0)
+		return ERR_PTR(-EINVAL);
+
+	attrs = devm_kcalloc(dev, nattrs + 1, sizeof(*attrs), GFP_KERNEL);
+	if (!attrs)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; chip->info[i]; i++) {
+		ret = hwmon_genattrs(dev, drvdata, &attrs[aindex], chip->ops,
+				     chip->info[i]);
+		if (ret < 0)
+			return ERR_PTR(ret);
+		aindex += ret;
+	}
+
+	return attrs;
+}
+
+static struct device *
+__hwmon_device_register(struct device *dev, const char *name, void *drvdata,
+			const struct hwmon_chip_info *chip,
+			const struct attribute_group **groups)
 {
 	struct hwmon_device *hwdev;
-	int err, id;
+	struct device *hdev;
+	int i, j, err, id;
 
 	/* Do not accept invalid characters in hwmon name attribute */
 	if (name && (!strlen(name) || strpbrk(name, "-* \t\n")))
@@ -114,28 +524,128 @@ hwmon_device_register_with_groups(struct device *dev, const char *name,
 		goto ida_remove;
 	}
 
+	hdev = &hwdev->dev;
+
+	if (chip && chip->ops->is_visible) {
+		struct attribute **attrs;
+		int ngroups = 2;
+
+		if (groups)
+			for (i = 0; groups[i]; i++)
+				ngroups++;
+
+		hwdev->groups = devm_kcalloc(dev, ngroups, sizeof(*groups),
+					     GFP_KERNEL);
+		if (!hwdev->groups)
+			return ERR_PTR(-ENOMEM);
+
+		attrs = __hwmon_create_attrs(dev, drvdata, chip);
+		if (IS_ERR(attrs)) {
+			err = PTR_ERR(attrs);
+			goto free_hwmon;
+		}
+
+		hwdev->group.attrs = attrs;
+		ngroups = 0;
+		hwdev->groups[ngroups++] = &hwdev->group;
+
+		if (groups) {
+			for (i = 0; groups[i]; i++)
+				hwdev->groups[ngroups++] = groups[i];
+		}
+
+		hdev->groups = hwdev->groups;
+	} else {
+		hdev->groups = groups;
+	}
+
 	hwdev->name = name;
-	hwdev->dev.class = &hwmon_class;
-	hwdev->dev.parent = dev;
-	hwdev->dev.groups = groups;
-	hwdev->dev.of_node = dev ? dev->of_node : NULL;
-	dev_set_drvdata(&hwdev->dev, drvdata);
-	dev_set_name(&hwdev->dev, HWMON_ID_FORMAT, id);
-	err = device_register(&hwdev->dev);
+	hdev->class = &hwmon_class;
+	hdev->parent = dev;
+	hdev->of_node = dev ? dev->of_node : NULL;
+	hwdev->chip = chip;
+	dev_set_drvdata(hdev, drvdata);
+	dev_set_name(hdev, HWMON_ID_FORMAT, id);
+	err = device_register(hdev);
 	if (err)
-		goto free;
+		goto free_hwmon;
+
+	if (chip && chip->ops->is_visible && chip->ops->read &&
+	    chip->info[0]->type == hwmon_chip &&
+	    (chip->info[0]->config[0] & HWMON_C_REGISTER_TZ)) {
+		const struct hwmon_channel_info **info = chip->info;
+
+		for (i = 1; info[i]; i++) {
+			if (info[i]->type != hwmon_temp)
+				continue;
+
+			for (j = 0; info[i]->config[j]; j++) {
+				if (!chip->ops->is_visible(drvdata, hwmon_temp,
+							   hwmon_temp_input, j))
+					continue;
+				if (info[i]->config[j] & HWMON_T_INPUT)
+					hwmon_thermal_add_sensor(dev, hwdev, j);
+			}
+		}
+	}
 
-	return &hwdev->dev;
+	return hdev;
 
-free:
+free_hwmon:
 	kfree(hwdev);
 ida_remove:
 	ida_simple_remove(&hwmon_ida, id);
 	return ERR_PTR(err);
 }
+
+/**
+ * hwmon_device_register_with_groups - register w/ hwmon
+ * @dev: the parent device
+ * @name: hwmon name attribute
+ * @drvdata: driver data to attach to created device
+ * @groups: List of attribute groups to create
+ *
+ * hwmon_device_unregister() must be called when the device is no
+ * longer needed.
+ *
+ * Returns the pointer to the new device.
+ */
+struct device *
+hwmon_device_register_with_groups(struct device *dev, const char *name,
+				  void *drvdata,
+				  const struct attribute_group **groups)
+{
+	return __hwmon_device_register(dev, name, drvdata, NULL, groups);
+}
 EXPORT_SYMBOL_GPL(hwmon_device_register_with_groups);
 
 /**
+ * hwmon_device_register_with_info - register w/ hwmon
+ * @dev: the parent device
+ * @name: hwmon name attribute
+ * @drvdata: driver data to attach to created device
+ * @info: Pointer to hwmon chip information
+ * @groups - pointer to list of driver specific attribute groups
+ *
+ * hwmon_device_unregister() must be called when the device is no
+ * longer needed.
+ *
+ * Returns the pointer to the new device.
+ */
+struct device *
+hwmon_device_register_with_info(struct device *dev, const char *name,
+				void *drvdata,
+				const struct hwmon_chip_info *chip,
+				const struct attribute_group **groups)
+{
+	if (chip && (!chip->ops || !chip->info))
+		return ERR_PTR(-EINVAL);
+
+	return __hwmon_device_register(dev, name, drvdata, chip, groups);
+}
+EXPORT_SYMBOL_GPL(hwmon_device_register_with_info);
+
+/**
  * hwmon_device_register - register w/ hwmon
  * @dev: the device to register
  *
@@ -213,6 +723,48 @@ error:
 }
 EXPORT_SYMBOL_GPL(devm_hwmon_device_register_with_groups);
 
+/**
+ * devm_hwmon_device_register_with_info - register w/ hwmon
+ * @dev: the parent device
+ * @name: hwmon name attribute
+ * @drvdata: driver data to attach to created device
+ * @info: Pointer to hwmon chip information
+ * @groups - pointer to list of driver specific attribute groups
+ *
+ * Returns the pointer to the new device. The new device is automatically
+ * unregistered with the parent device.
+ */
+struct device *
+devm_hwmon_device_register_with_info(struct device *dev, const char *name,
+				     void *drvdata,
+				     const struct hwmon_chip_info *chip,
+				     const struct attribute_group **groups)
+{
+	struct device **ptr, *hwdev;
+
+	if (!dev)
+		return ERR_PTR(-EINVAL);
+
+	ptr = devres_alloc(devm_hwmon_release, sizeof(*ptr), GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+
+	hwdev = hwmon_device_register_with_info(dev, name, drvdata, chip,
+						groups);
+	if (IS_ERR(hwdev))
+		goto error;
+
+	*ptr = hwdev;
+	devres_add(dev, ptr);
+
+	return hwdev;
+
+error:
+	devres_free(ptr);
+	return hwdev;
+}
+EXPORT_SYMBOL_GPL(devm_hwmon_device_register_with_info);
+
 static int devm_hwmon_match(struct device *dev, void *res, void *data)
 {
 	struct device **hwdev = res;
diff --git a/drivers/hwmon/ibmpowernv.c b/drivers/hwmon/ibmpowernv.c
index 55b5a8ff1cfe..6d2e6605751c 100644
--- a/drivers/hwmon/ibmpowernv.c
+++ b/drivers/hwmon/ibmpowernv.c
@@ -143,13 +143,11 @@ static void __init make_sensor_label(struct device_node *np,
 		if (cpuid >= 0)
 			/*
 			 * The digital thermal sensors are associated
-			 * with a core. Let's print out the range of
-			 * cpu ids corresponding to the hardware
-			 * threads of the core.
+			 * with a core.
 			 */
 			n += snprintf(sdata->label + n,
-				      sizeof(sdata->label) - n, " %d-%d",
-				      cpuid, cpuid + threads_per_core - 1);
+				      sizeof(sdata->label) - n, " %d",
+				      cpuid);
 		else
 			n += snprintf(sdata->label + n,
 				      sizeof(sdata->label) - n, " phy%d", id);
diff --git a/drivers/hwmon/iio_hwmon.c b/drivers/hwmon/iio_hwmon.c
index 89449871bca7..f6a76679c650 100644
--- a/drivers/hwmon/iio_hwmon.c
+++ b/drivers/hwmon/iio_hwmon.c
@@ -73,8 +73,11 @@ static int iio_hwmon_probe(struct platform_device *pdev)
 		name = dev->of_node->name;
 
 	channels = iio_channel_get_all(dev);
-	if (IS_ERR(channels))
+	if (IS_ERR(channels)) {
+		if (PTR_ERR(channels) == -ENODEV)
+			return -EPROBE_DEFER;
 		return PTR_ERR(channels);
+	}
 
 	st = devm_kzalloc(dev, sizeof(*st), GFP_KERNEL);
 	if (st == NULL) {
diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c
index 730d84028260..ad82cb28d87a 100644
--- a/drivers/hwmon/it87.c
+++ b/drivers/hwmon/it87.c
@@ -491,7 +491,7 @@ struct it87_sio_data {
 struct it87_data {
 	const struct attribute_group *groups[7];
 	enum chips type;
-	u16 features;
+	u32 features;
 	u8 peci_mask;
 	u8 old_peci_mask;
 
@@ -2011,10 +2011,11 @@ static struct attribute *it87_attributes_in[] = {
 	&sensor_dev_attr_in7_beep.dev_attr.attr,	/* 39 */
 
 	&sensor_dev_attr_in8_input.dev_attr.attr,	/* 40 */
-	&sensor_dev_attr_in9_input.dev_attr.attr,	/* 41 */
-	&sensor_dev_attr_in10_input.dev_attr.attr,	/* 41 */
-	&sensor_dev_attr_in11_input.dev_attr.attr,	/* 41 */
-	&sensor_dev_attr_in12_input.dev_attr.attr,	/* 41 */
+	&sensor_dev_attr_in9_input.dev_attr.attr,
+	&sensor_dev_attr_in10_input.dev_attr.attr,
+	&sensor_dev_attr_in11_input.dev_attr.attr,
+	&sensor_dev_attr_in12_input.dev_attr.attr,
+	NULL
 };
 
 static const struct attribute_group it87_group_in = {
diff --git a/drivers/hwmon/jc42.c b/drivers/hwmon/jc42.c
index 9d5f85f3384f..1bf22eff0b08 100644
--- a/drivers/hwmon/jc42.c
+++ b/drivers/hwmon/jc42.c
@@ -28,7 +28,6 @@
 #include <linux/jiffies.h>
 #include <linux/i2c.h>
 #include <linux/hwmon.h>
-#include <linux/hwmon-sysfs.h>
 #include <linux/err.h>
 #include <linux/mutex.h>
 #include <linux/of.h>
@@ -254,170 +253,148 @@ abort:
 	return ret;
 }
 
-/* sysfs functions */
-
-static ssize_t show_temp(struct device *dev, struct device_attribute *devattr,
-			 char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct jc42_data *data = jc42_update_device(dev);
-	if (IS_ERR(data))
-		return PTR_ERR(data);
-	return sprintf(buf, "%d\n",
-		       jc42_temp_from_reg(data->temp[attr->index]));
-}
-
-static ssize_t show_temp_hyst(struct device *dev,
-			      struct device_attribute *devattr, char *buf)
+static int jc42_read(struct device *dev, enum hwmon_sensor_types type,
+		     u32 attr, int channel, long *val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct jc42_data *data = jc42_update_device(dev);
 	int temp, hyst;
 
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
-	temp = jc42_temp_from_reg(data->temp[attr->index]);
-	hyst = jc42_hysteresis[(data->config & JC42_CFG_HYST_MASK)
-			       >> JC42_CFG_HYST_SHIFT];
-	return sprintf(buf, "%d\n", temp - hyst);
+	switch (attr) {
+	case hwmon_temp_input:
+		*val = jc42_temp_from_reg(data->temp[t_input]);
+		return 0;
+	case hwmon_temp_min:
+		*val = jc42_temp_from_reg(data->temp[t_min]);
+		return 0;
+	case hwmon_temp_max:
+		*val = jc42_temp_from_reg(data->temp[t_max]);
+		return 0;
+	case hwmon_temp_crit:
+		*val = jc42_temp_from_reg(data->temp[t_crit]);
+		return 0;
+	case hwmon_temp_max_hyst:
+		temp = jc42_temp_from_reg(data->temp[t_max]);
+		hyst = jc42_hysteresis[(data->config & JC42_CFG_HYST_MASK)
+						>> JC42_CFG_HYST_SHIFT];
+		*val = temp - hyst;
+		return 0;
+	case hwmon_temp_crit_hyst:
+		temp = jc42_temp_from_reg(data->temp[t_crit]);
+		hyst = jc42_hysteresis[(data->config & JC42_CFG_HYST_MASK)
+						>> JC42_CFG_HYST_SHIFT];
+		*val = temp - hyst;
+		return 0;
+	case hwmon_temp_min_alarm:
+		*val = (data->temp[t_input] >> JC42_ALARM_MIN_BIT) & 1;
+		return 0;
+	case hwmon_temp_max_alarm:
+		*val = (data->temp[t_input] >> JC42_ALARM_MAX_BIT) & 1;
+		return 0;
+	case hwmon_temp_crit_alarm:
+		*val = (data->temp[t_input] >> JC42_ALARM_CRIT_BIT) & 1;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t set_temp(struct device *dev, struct device_attribute *devattr,
-			const char *buf, size_t count)
+static int jc42_write(struct device *dev, enum hwmon_sensor_types type,
+		      u32 attr, int channel, long val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct jc42_data *data = dev_get_drvdata(dev);
-	int err, ret = count;
-	int nr = attr->index;
-	long val;
+	struct i2c_client *client = data->client;
+	int diff, hyst;
+	int ret;
 
-	if (kstrtol(buf, 10, &val) < 0)
-		return -EINVAL;
 	mutex_lock(&data->update_lock);
-	data->temp[nr] = jc42_temp_to_reg(val, data->extended);
-	err = i2c_smbus_write_word_swapped(data->client, temp_regs[nr],
-					   data->temp[nr]);
-	if (err < 0)
-		ret = err;
-	mutex_unlock(&data->update_lock);
-	return ret;
-}
 
-/*
- * JC42.4 compliant chips only support four hysteresis values.
- * Pick best choice and go from there.
- */
-static ssize_t set_temp_crit_hyst(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t count)
-{
-	struct jc42_data *data = dev_get_drvdata(dev);
-	long val;
-	int diff, hyst;
-	int err;
-	int ret = count;
-
-	if (kstrtol(buf, 10, &val) < 0)
-		return -EINVAL;
-
-	val = clamp_val(val, (data->extended ? JC42_TEMP_MIN_EXTENDED :
-			      JC42_TEMP_MIN) - 6000, JC42_TEMP_MAX);
-	diff = jc42_temp_from_reg(data->temp[t_crit]) - val;
-
-	hyst = 0;
-	if (diff > 0) {
-		if (diff < 2250)
-			hyst = 1;	/* 1.5 degrees C */
-		else if (diff < 4500)
-			hyst = 2;	/* 3.0 degrees C */
-		else
-			hyst = 3;	/* 6.0 degrees C */
+	switch (attr) {
+	case hwmon_temp_min:
+		data->temp[t_min] = jc42_temp_to_reg(val, data->extended);
+		ret = i2c_smbus_write_word_swapped(client, temp_regs[t_min],
+						   data->temp[t_min]);
+		break;
+	case hwmon_temp_max:
+		data->temp[t_max] = jc42_temp_to_reg(val, data->extended);
+		ret = i2c_smbus_write_word_swapped(client, temp_regs[t_max],
+						   data->temp[t_max]);
+		break;
+	case hwmon_temp_crit:
+		data->temp[t_crit] = jc42_temp_to_reg(val, data->extended);
+		ret = i2c_smbus_write_word_swapped(client, temp_regs[t_crit],
+						   data->temp[t_crit]);
+		break;
+	case hwmon_temp_crit_hyst:
+		/*
+		 * JC42.4 compliant chips only support four hysteresis values.
+		 * Pick best choice and go from there.
+		 */
+		val = clamp_val(val, (data->extended ? JC42_TEMP_MIN_EXTENDED
+						     : JC42_TEMP_MIN) - 6000,
+				JC42_TEMP_MAX);
+		diff = jc42_temp_from_reg(data->temp[t_crit]) - val;
+		hyst = 0;
+		if (diff > 0) {
+			if (diff < 2250)
+				hyst = 1;	/* 1.5 degrees C */
+			else if (diff < 4500)
+				hyst = 2;	/* 3.0 degrees C */
+			else
+				hyst = 3;	/* 6.0 degrees C */
+		}
+		data->config = (data->config & ~JC42_CFG_HYST_MASK) |
+				(hyst << JC42_CFG_HYST_SHIFT);
+		ret = i2c_smbus_write_word_swapped(data->client,
+						   JC42_REG_CONFIG,
+						   data->config);
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
 	}
 
-	mutex_lock(&data->update_lock);
-	data->config = (data->config & ~JC42_CFG_HYST_MASK)
-	  | (hyst << JC42_CFG_HYST_SHIFT);
-	err = i2c_smbus_write_word_swapped(data->client, JC42_REG_CONFIG,
-					   data->config);
-	if (err < 0)
-		ret = err;
 	mutex_unlock(&data->update_lock);
-	return ret;
-}
 
-static ssize_t show_alarm(struct device *dev,
-			  struct device_attribute *attr, char *buf)
-{
-	u16 bit = to_sensor_dev_attr(attr)->index;
-	struct jc42_data *data = jc42_update_device(dev);
-	u16 val;
-
-	if (IS_ERR(data))
-		return PTR_ERR(data);
-
-	val = data->temp[t_input];
-	if (bit != JC42_ALARM_CRIT_BIT && (data->config & JC42_CFG_CRIT_ONLY))
-		val = 0;
-	return sprintf(buf, "%u\n", (val >> bit) & 1);
+	return ret;
 }
 
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, show_temp, NULL, t_input);
-static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO, show_temp, set_temp, t_crit);
-static SENSOR_DEVICE_ATTR(temp1_min, S_IRUGO, show_temp, set_temp, t_min);
-static SENSOR_DEVICE_ATTR(temp1_max, S_IRUGO, show_temp, set_temp, t_max);
-
-static SENSOR_DEVICE_ATTR(temp1_crit_hyst, S_IRUGO, show_temp_hyst,
-			  set_temp_crit_hyst, t_crit);
-static SENSOR_DEVICE_ATTR(temp1_max_hyst, S_IRUGO, show_temp_hyst, NULL, t_max);
-
-static SENSOR_DEVICE_ATTR(temp1_crit_alarm, S_IRUGO, show_alarm, NULL,
-			  JC42_ALARM_CRIT_BIT);
-static SENSOR_DEVICE_ATTR(temp1_min_alarm, S_IRUGO, show_alarm, NULL,
-			  JC42_ALARM_MIN_BIT);
-static SENSOR_DEVICE_ATTR(temp1_max_alarm, S_IRUGO, show_alarm, NULL,
-			  JC42_ALARM_MAX_BIT);
-
-static struct attribute *jc42_attributes[] = {
-	&sensor_dev_attr_temp1_input.dev_attr.attr,
-	&sensor_dev_attr_temp1_crit.dev_attr.attr,
-	&sensor_dev_attr_temp1_min.dev_attr.attr,
-	&sensor_dev_attr_temp1_max.dev_attr.attr,
-	&sensor_dev_attr_temp1_crit_hyst.dev_attr.attr,
-	&sensor_dev_attr_temp1_max_hyst.dev_attr.attr,
-	&sensor_dev_attr_temp1_crit_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp1_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp1_max_alarm.dev_attr.attr,
-	NULL
-};
-
-static umode_t jc42_attribute_mode(struct kobject *kobj,
-				  struct attribute *attr, int index)
+static umode_t jc42_is_visible(const void *_data, enum hwmon_sensor_types type,
+			       u32 attr, int channel)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
-	struct jc42_data *data = dev_get_drvdata(dev);
+	const struct jc42_data *data = _data;
 	unsigned int config = data->config;
-	bool readonly;
-
-	if (attr == &sensor_dev_attr_temp1_crit.dev_attr.attr)
-		readonly = config & JC42_CFG_TCRIT_LOCK;
-	else if (attr == &sensor_dev_attr_temp1_min.dev_attr.attr ||
-		 attr == &sensor_dev_attr_temp1_max.dev_attr.attr)
-		readonly = config & JC42_CFG_EVENT_LOCK;
-	else if (attr == &sensor_dev_attr_temp1_crit_hyst.dev_attr.attr)
-		readonly = config & (JC42_CFG_EVENT_LOCK | JC42_CFG_TCRIT_LOCK);
-	else
-		readonly = true;
-
-	return S_IRUGO | (readonly ? 0 : S_IWUSR);
+	umode_t mode = S_IRUGO;
+
+	switch (attr) {
+	case hwmon_temp_min:
+	case hwmon_temp_max:
+		if (!(config & JC42_CFG_EVENT_LOCK))
+			mode |= S_IWUSR;
+		break;
+	case hwmon_temp_crit:
+		if (!(config & JC42_CFG_TCRIT_LOCK))
+			mode |= S_IWUSR;
+		break;
+	case hwmon_temp_crit_hyst:
+		if (!(config & (JC42_CFG_EVENT_LOCK | JC42_CFG_TCRIT_LOCK)))
+			mode |= S_IWUSR;
+		break;
+	case hwmon_temp_input:
+	case hwmon_temp_max_hyst:
+	case hwmon_temp_min_alarm:
+	case hwmon_temp_max_alarm:
+	case hwmon_temp_crit_alarm:
+		break;
+	default:
+		mode = 0;
+		break;
+	}
+	return mode;
 }
 
-static const struct attribute_group jc42_group = {
-	.attrs = jc42_attributes,
-	.is_visible = jc42_attribute_mode,
-};
-__ATTRIBUTE_GROUPS(jc42);
-
 /* Return 0 if detection is successful, -ENODEV otherwise */
 static int jc42_detect(struct i2c_client *client, struct i2c_board_info *info)
 {
@@ -450,6 +427,34 @@ static int jc42_detect(struct i2c_client *client, struct i2c_board_info *info)
 	return -ENODEV;
 }
 
+static const u32 jc42_temp_config[] = {
+	HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | HWMON_T_CRIT |
+	HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST |
+	HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM,
+	0
+};
+
+static const struct hwmon_channel_info jc42_temp = {
+	.type = hwmon_temp,
+	.config = jc42_temp_config,
+};
+
+static const struct hwmon_channel_info *jc42_info[] = {
+	&jc42_temp,
+	NULL
+};
+
+static const struct hwmon_ops jc42_hwmon_ops = {
+	.is_visible = jc42_is_visible,
+	.read = jc42_read,
+	.write = jc42_write,
+};
+
+static const struct hwmon_chip_info jc42_chip_info = {
+	.ops = &jc42_hwmon_ops,
+	.info = jc42_info,
+};
+
 static int jc42_probe(struct i2c_client *client, const struct i2c_device_id *id)
 {
 	struct device *dev = &client->dev;
@@ -482,9 +487,9 @@ static int jc42_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	}
 	data->config = config;
 
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
-							   data,
-							   jc42_groups);
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 data, &jc42_chip_info,
+							 NULL);
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c
index 92f9d4bbf597..eff3b24d8473 100644
--- a/drivers/hwmon/lm75.c
+++ b/drivers/hwmon/lm75.c
@@ -28,7 +28,6 @@
 #include <linux/err.h>
 #include <linux/of.h>
 #include <linux/regmap.h>
-#include <linux/thermal.h>
 #include "lm75.h"
 
 
@@ -88,56 +87,75 @@ static inline long lm75_reg_to_mc(s16 temp, u8 resolution)
 	return ((temp >> (16 - resolution)) * 1000) >> (resolution - 8);
 }
 
-/* sysfs attributes for hwmon */
-
-static int lm75_read_temp(void *dev, int *temp)
+static int lm75_read(struct device *dev, enum hwmon_sensor_types type,
+		     u32 attr, int channel, long *val)
 {
 	struct lm75_data *data = dev_get_drvdata(dev);
-	unsigned int _temp;
-	int err;
-
-	err = regmap_read(data->regmap, LM75_REG_TEMP, &_temp);
-	if (err < 0)
-		return err;
-
-	*temp = lm75_reg_to_mc(_temp, data->resolution);
-
+	unsigned int regval;
+	int err, reg;
+
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			*val = data->sample_time;
+			break;;
+		default:
+			return -EINVAL;
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+			reg = LM75_REG_TEMP;
+			break;
+		case hwmon_temp_max:
+			reg = LM75_REG_MAX;
+			break;
+		case hwmon_temp_max_hyst:
+			reg = LM75_REG_HYST;
+			break;
+		default:
+			return -EINVAL;
+		}
+		err = regmap_read(data->regmap, reg, &regval);
+		if (err < 0)
+			return err;
+
+		*val = lm75_reg_to_mc(regval, data->resolution);
+		break;
+	default:
+		return -EINVAL;
+	}
 	return 0;
 }
 
-static ssize_t show_temp(struct device *dev, struct device_attribute *da,
-			 char *buf)
+static int lm75_write(struct device *dev, enum hwmon_sensor_types type,
+		      u32 attr, int channel, long temp)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
 	struct lm75_data *data = dev_get_drvdata(dev);
-	unsigned int temp = 0;
-	int err;
-
-	err = regmap_read(data->regmap, attr->index, &temp);
-	if (err < 0)
-		return err;
-
-	return sprintf(buf, "%ld\n", lm75_reg_to_mc(temp, data->resolution));
-}
-
-static ssize_t set_temp(struct device *dev, struct device_attribute *da,
-			const char *buf, size_t count)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
-	struct lm75_data *data = dev_get_drvdata(dev);
-	long temp;
-	int error;
 	u8 resolution;
+	int reg;
+
+	if (type != hwmon_temp)
+		return -EINVAL;
 
-	error = kstrtol(buf, 10, &temp);
-	if (error)
-		return error;
+	switch (attr) {
+	case hwmon_temp_max:
+		reg = LM75_REG_MAX;
+		break;
+	case hwmon_temp_max_hyst:
+		reg = LM75_REG_HYST;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	/*
 	 * Resolution of limit registers is assumed to be the same as the
 	 * temperature input register resolution unless given explicitly.
 	 */
-	if (attr->index && data->resolution_limits)
+	if (data->resolution_limits)
 		resolution = data->resolution_limits;
 	else
 		resolution = data->resolution;
@@ -145,45 +163,77 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *da,
 	temp = clamp_val(temp, LM75_TEMP_MIN, LM75_TEMP_MAX);
 	temp = DIV_ROUND_CLOSEST(temp  << (resolution - 8),
 				 1000) << (16 - resolution);
-	error = regmap_write(data->regmap, attr->index, temp);
-	if (error < 0)
-		return error;
 
-	return count;
+	return regmap_write(data->regmap, reg, temp);
 }
 
-static ssize_t show_update_interval(struct device *dev,
-				    struct device_attribute *da, char *buf)
+static umode_t lm75_is_visible(const void *data, enum hwmon_sensor_types type,
+			       u32 attr, int channel)
 {
-	struct lm75_data *data = dev_get_drvdata(dev);
-
-	return sprintf(buf, "%u\n", data->sample_time);
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return S_IRUGO;
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+			return S_IRUGO;
+		case hwmon_temp_max:
+		case hwmon_temp_max_hyst:
+			return S_IRUGO | S_IWUSR;
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
 }
 
-static SENSOR_DEVICE_ATTR(temp1_max, S_IWUSR | S_IRUGO,
-			show_temp, set_temp, LM75_REG_MAX);
-static SENSOR_DEVICE_ATTR(temp1_max_hyst, S_IWUSR | S_IRUGO,
-			show_temp, set_temp, LM75_REG_HYST);
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, show_temp, NULL, LM75_REG_TEMP);
-static DEVICE_ATTR(update_interval, S_IRUGO, show_update_interval, NULL);
+/*-----------------------------------------------------------------------*/
 
-static struct attribute *lm75_attrs[] = {
-	&sensor_dev_attr_temp1_input.dev_attr.attr,
-	&sensor_dev_attr_temp1_max.dev_attr.attr,
-	&sensor_dev_attr_temp1_max_hyst.dev_attr.attr,
-	&dev_attr_update_interval.attr,
+/* device probe and removal */
 
-	NULL
+/* chip configuration */
+
+static const u32 lm75_chip_config[] = {
+	HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL,
+	0
 };
-ATTRIBUTE_GROUPS(lm75);
 
-static const struct thermal_zone_of_device_ops lm75_of_thermal_ops = {
-	.get_temp = lm75_read_temp,
+static const struct hwmon_channel_info lm75_chip = {
+	.type = hwmon_chip,
+	.config = lm75_chip_config,
 };
 
-/*-----------------------------------------------------------------------*/
+static const u32 lm75_temp_config[] = {
+	HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MAX_HYST,
+	0
+};
 
-/* device probe and removal */
+static const struct hwmon_channel_info lm75_temp = {
+	.type = hwmon_temp,
+	.config = lm75_temp_config,
+};
+
+static const struct hwmon_channel_info *lm75_info[] = {
+	&lm75_chip,
+	&lm75_temp,
+	NULL
+};
+
+static const struct hwmon_ops lm75_hwmon_ops = {
+	.is_visible = lm75_is_visible,
+	.read = lm75_read,
+	.write = lm75_write,
+};
+
+static const struct hwmon_chip_info lm75_chip_info = {
+	.ops = &lm75_hwmon_ops,
+	.info = lm75_info,
+};
 
 static bool lm75_is_writeable_reg(struct device *dev, unsigned int reg)
 {
@@ -337,15 +387,12 @@ lm75_probe(struct i2c_client *client, const struct i2c_device_id *id)
 
 	dev_dbg(dev, "Config %02x\n", new);
 
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
-							   data, lm75_groups);
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 data, &lm75_chip_info,
+							 NULL);
 	if (IS_ERR(hwmon_dev))
 		return PTR_ERR(hwmon_dev);
 
-	devm_thermal_zone_of_sensor_register(hwmon_dev, 0,
-					     hwmon_dev,
-					     &lm75_of_thermal_ops);
-
 	dev_info(dev, "%s: sensor '%s'\n", dev_name(hwmon_dev), client->name);
 
 	return 0;
diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c
index 496e771b363f..322ed9272811 100644
--- a/drivers/hwmon/lm90.c
+++ b/drivers/hwmon/lm90.c
@@ -89,7 +89,6 @@
 #include <linux/slab.h>
 #include <linux/jiffies.h>
 #include <linux/i2c.h>
-#include <linux/hwmon-sysfs.h>
 #include <linux/hwmon.h>
 #include <linux/err.h>
 #include <linux/mutex.h>
@@ -326,7 +325,7 @@ static const struct lm90_params lm90_params[] = {
 		.alert_alarms = 0x7c,
 		.max_convrate = 9,
 		.reg_local_ext = TMP451_REG_R_LOCAL_TEMPL,
-	}
+	},
 };
 
 /*
@@ -365,7 +364,10 @@ enum lm90_temp11_reg_index {
 
 struct lm90_data {
 	struct i2c_client *client;
-	const struct attribute_group *groups[6];
+	u32 channel_config[4];
+	struct hwmon_channel_info temp_info;
+	const struct hwmon_channel_info *info[3];
+	struct hwmon_chip_info chip;
 	struct mutex update_lock;
 	bool valid;		/* true if register values are valid */
 	unsigned long last_updated; /* in jiffies */
@@ -489,11 +491,11 @@ static inline int lm90_select_remote_channel(struct i2c_client *client,
  * client->update_lock must be held when calling this function (unless we are
  * in detection or initialization steps).
  */
-static void lm90_set_convrate(struct i2c_client *client, struct lm90_data *data,
-			      unsigned int interval)
+static int lm90_set_convrate(struct i2c_client *client, struct lm90_data *data,
+			     unsigned int interval)
 {
-	int i;
 	unsigned int update_interval;
+	int i, err;
 
 	/* Shift calculations to avoid rounding errors */
 	interval <<= 6;
@@ -504,8 +506,9 @@ static void lm90_set_convrate(struct i2c_client *client, struct lm90_data *data,
 		if (interval >= update_interval * 3 / 4)
 			break;
 
-	i2c_smbus_write_byte_data(client, LM90_REG_W_CONVRATE, i);
+	err = i2c_smbus_write_byte_data(client, LM90_REG_W_CONVRATE, i);
 	data->update_interval = DIV_ROUND_CLOSEST(update_interval, 64);
+	return err;
 }
 
 static int lm90_update_limits(struct device *dev)
@@ -604,19 +607,17 @@ static int lm90_update_limits(struct device *dev)
 	return 0;
 }
 
-static struct lm90_data *lm90_update_device(struct device *dev)
+static int lm90_update_device(struct device *dev)
 {
 	struct lm90_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
 	unsigned long next_update;
-	int val = 0;
-
-	mutex_lock(&data->update_lock);
+	int val;
 
 	if (!data->valid) {
 		val = lm90_update_limits(dev);
 		if (val < 0)
-			goto error;
+			return val;
 	}
 
 	next_update = data->last_updated +
@@ -628,53 +629,55 @@ static struct lm90_data *lm90_update_device(struct device *dev)
 
 		val = lm90_read_reg(client, LM90_REG_R_LOCAL_LOW);
 		if (val < 0)
-			goto error;
+			return val;
 		data->temp8[LOCAL_LOW] = val;
 
 		val = lm90_read_reg(client, LM90_REG_R_LOCAL_HIGH);
 		if (val < 0)
-			goto error;
+			return val;
 		data->temp8[LOCAL_HIGH] = val;
 
 		if (data->reg_local_ext) {
 			val = lm90_read16(client, LM90_REG_R_LOCAL_TEMP,
 					  data->reg_local_ext);
 			if (val < 0)
-				goto error;
+				return val;
 			data->temp11[LOCAL_TEMP] = val;
 		} else {
 			val = lm90_read_reg(client, LM90_REG_R_LOCAL_TEMP);
 			if (val < 0)
-				goto error;
+				return val;
 			data->temp11[LOCAL_TEMP] = val << 8;
 		}
 		val = lm90_read16(client, LM90_REG_R_REMOTE_TEMPH,
 				  LM90_REG_R_REMOTE_TEMPL);
 		if (val < 0)
-			goto error;
+			return val;
 		data->temp11[REMOTE_TEMP] = val;
 
 		val = lm90_read_reg(client, LM90_REG_R_STATUS);
 		if (val < 0)
-			goto error;
+			return val;
 		data->alarms = val;	/* lower 8 bit of alarms */
 
 		if (data->kind == max6696) {
 			val = lm90_select_remote_channel(client, data, 1);
 			if (val < 0)
-				goto error;
+				return val;
 
 			val = lm90_read16(client, LM90_REG_R_REMOTE_TEMPH,
 					  LM90_REG_R_REMOTE_TEMPL);
-			if (val < 0)
-				goto error;
+			if (val < 0) {
+				lm90_select_remote_channel(client, data, 0);
+				return val;
+			}
 			data->temp11[REMOTE2_TEMP] = val;
 
 			lm90_select_remote_channel(client, data, 0);
 
 			val = lm90_read_reg(client, MAX6696_REG_R_STATUS2);
 			if (val < 0)
-				goto error;
+				return val;
 			data->alarms |= val << 8;
 		}
 
@@ -686,7 +689,7 @@ static struct lm90_data *lm90_update_device(struct device *dev)
 		    !(data->alarms & data->alert_alarms)) {
 			val = lm90_read_reg(client, LM90_REG_R_CONFIG1);
 			if (val < 0)
-				goto error;
+				return val;
 
 			if (val & 0x80) {
 				dev_dbg(&client->dev, "Re-enabling ALERT#\n");
@@ -700,13 +703,7 @@ static struct lm90_data *lm90_update_device(struct device *dev)
 		data->valid = true;
 	}
 
-error:
-	mutex_unlock(&data->update_lock);
-
-	if (val < 0)
-		return ERR_PTR(val);
-
-	return data;
+	return 0;
 }
 
 /*
@@ -832,52 +829,19 @@ static u16 temp_to_u16_adt7461(struct lm90_data *data, long val)
 	return (val + 125) / 250 * 64;
 }
 
-/*
- * Sysfs stuff
- */
-
-static ssize_t show_temp8(struct device *dev, struct device_attribute *devattr,
-			  char *buf)
+/* pec used for ADM1032 only */
+static ssize_t show_pec(struct device *dev, struct device_attribute *dummy,
+			char *buf)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct lm90_data *data = lm90_update_device(dev);
-	int temp;
-
-	if (IS_ERR(data))
-		return PTR_ERR(data);
-
-	if (data->kind == adt7461 || data->kind == tmp451)
-		temp = temp_from_u8_adt7461(data, data->temp8[attr->index]);
-	else if (data->kind == max6646)
-		temp = temp_from_u8(data->temp8[attr->index]);
-	else
-		temp = temp_from_s8(data->temp8[attr->index]);
-
-	/* +16 degrees offset for temp2 for the LM99 */
-	if (data->kind == lm99 && attr->index == 3)
-		temp += 16000;
+	struct i2c_client *client = to_i2c_client(dev);
 
-	return sprintf(buf, "%d\n", temp);
+	return sprintf(buf, "%d\n", !!(client->flags & I2C_CLIENT_PEC));
 }
 
-static ssize_t set_temp8(struct device *dev, struct device_attribute *devattr,
-			 const char *buf, size_t count)
+static ssize_t set_pec(struct device *dev, struct device_attribute *dummy,
+		       const char *buf, size_t count)
 {
-	static const u8 reg[TEMP8_REG_NUM] = {
-		LM90_REG_W_LOCAL_LOW,
-		LM90_REG_W_LOCAL_HIGH,
-		LM90_REG_W_LOCAL_CRIT,
-		LM90_REG_W_REMOTE_CRIT,
-		MAX6659_REG_W_LOCAL_EMERG,
-		MAX6659_REG_W_REMOTE_EMERG,
-		LM90_REG_W_REMOTE_CRIT,
-		MAX6659_REG_W_REMOTE_EMERG,
-	};
-
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct lm90_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
-	int nr = attr->index;
+	struct i2c_client *client = to_i2c_client(dev);
 	long val;
 	int err;
 
@@ -885,82 +849,61 @@ static ssize_t set_temp8(struct device *dev, struct device_attribute *devattr,
 	if (err < 0)
 		return err;
 
-	/* +16 degrees offset for temp2 for the LM99 */
-	if (data->kind == lm99 && attr->index == 3)
-		val -= 16000;
-
-	mutex_lock(&data->update_lock);
-	if (data->kind == adt7461 || data->kind == tmp451)
-		data->temp8[nr] = temp_to_u8_adt7461(data, val);
-	else if (data->kind == max6646)
-		data->temp8[nr] = temp_to_u8(val);
-	else
-		data->temp8[nr] = temp_to_s8(val);
-
-	lm90_select_remote_channel(client, data, nr >= 6);
-	i2c_smbus_write_byte_data(client, reg[nr], data->temp8[nr]);
-	lm90_select_remote_channel(client, data, 0);
+	switch (val) {
+	case 0:
+		client->flags &= ~I2C_CLIENT_PEC;
+		break;
+	case 1:
+		client->flags |= I2C_CLIENT_PEC;
+		break;
+	default:
+		return -EINVAL;
+	}
 
-	mutex_unlock(&data->update_lock);
 	return count;
 }
 
-static ssize_t show_temp11(struct device *dev, struct device_attribute *devattr,
-			   char *buf)
+static DEVICE_ATTR(pec, S_IWUSR | S_IRUGO, show_pec, set_pec);
+
+static int lm90_get_temp11(struct lm90_data *data, int index)
 {
-	struct sensor_device_attribute_2 *attr = to_sensor_dev_attr_2(devattr);
-	struct lm90_data *data = lm90_update_device(dev);
+	s16 temp11 = data->temp11[index];
 	int temp;
 
-	if (IS_ERR(data))
-		return PTR_ERR(data);
-
 	if (data->kind == adt7461 || data->kind == tmp451)
-		temp = temp_from_u16_adt7461(data, data->temp11[attr->index]);
+		temp = temp_from_u16_adt7461(data, temp11);
 	else if (data->kind == max6646)
-		temp = temp_from_u16(data->temp11[attr->index]);
+		temp = temp_from_u16(temp11);
 	else
-		temp = temp_from_s16(data->temp11[attr->index]);
+		temp = temp_from_s16(temp11);
 
 	/* +16 degrees offset for temp2 for the LM99 */
-	if (data->kind == lm99 &&  attr->index <= 2)
+	if (data->kind == lm99 && index <= 2)
 		temp += 16000;
 
-	return sprintf(buf, "%d\n", temp);
+	return temp;
 }
 
-static ssize_t set_temp11(struct device *dev, struct device_attribute *devattr,
-			  const char *buf, size_t count)
+static int lm90_set_temp11(struct lm90_data *data, int index, long val)
 {
-	struct {
+	static struct reg {
 		u8 high;
 		u8 low;
-		int channel;
-	} reg[5] = {
-		{ LM90_REG_W_REMOTE_LOWH, LM90_REG_W_REMOTE_LOWL, 0 },
-		{ LM90_REG_W_REMOTE_HIGHH, LM90_REG_W_REMOTE_HIGHL, 0 },
-		{ LM90_REG_W_REMOTE_OFFSH, LM90_REG_W_REMOTE_OFFSL, 0 },
-		{ LM90_REG_W_REMOTE_LOWH, LM90_REG_W_REMOTE_LOWL, 1 },
-		{ LM90_REG_W_REMOTE_HIGHH, LM90_REG_W_REMOTE_HIGHL, 1 }
+	} reg[] = {
+	[REMOTE_LOW] = { LM90_REG_W_REMOTE_LOWH, LM90_REG_W_REMOTE_LOWL },
+	[REMOTE_HIGH] = { LM90_REG_W_REMOTE_HIGHH, LM90_REG_W_REMOTE_HIGHL },
+	[REMOTE_OFFSET] = { LM90_REG_W_REMOTE_OFFSH, LM90_REG_W_REMOTE_OFFSL },
+	[REMOTE2_LOW] = { LM90_REG_W_REMOTE_LOWH, LM90_REG_W_REMOTE_LOWL },
+	[REMOTE2_HIGH] = { LM90_REG_W_REMOTE_HIGHH, LM90_REG_W_REMOTE_HIGHL }
 	};
-
-	struct sensor_device_attribute_2 *attr = to_sensor_dev_attr_2(devattr);
-	struct lm90_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
-	int nr = attr->nr;
-	int index = attr->index;
-	long val;
+	struct reg *regp = &reg[index];
 	int err;
 
-	err = kstrtol(buf, 10, &val);
-	if (err < 0)
-		return err;
-
 	/* +16 degrees offset for temp2 for the LM99 */
 	if (data->kind == lm99 && index <= 2)
 		val -= 16000;
 
-	mutex_lock(&data->update_lock);
 	if (data->kind == adt7461 || data->kind == tmp451)
 		data->temp11[index] = temp_to_u16_adt7461(data, val);
 	else if (data->kind == max6646)
@@ -970,317 +913,383 @@ static ssize_t set_temp11(struct device *dev, struct device_attribute *devattr,
 	else
 		data->temp11[index] = temp_to_s8(val) << 8;
 
-	lm90_select_remote_channel(client, data, reg[nr].channel);
-	i2c_smbus_write_byte_data(client, reg[nr].high,
+	lm90_select_remote_channel(client, data, index >= 3);
+	err = i2c_smbus_write_byte_data(client, regp->high,
 				  data->temp11[index] >> 8);
+	if (err < 0)
+		return err;
 	if (data->flags & LM90_HAVE_REM_LIMIT_EXT)
-		i2c_smbus_write_byte_data(client, reg[nr].low,
-					  data->temp11[index] & 0xff);
-	lm90_select_remote_channel(client, data, 0);
+		err = i2c_smbus_write_byte_data(client, regp->low,
+						data->temp11[index] & 0xff);
 
-	mutex_unlock(&data->update_lock);
-	return count;
+	lm90_select_remote_channel(client, data, 0);
+	return err;
 }
 
-static ssize_t show_temphyst(struct device *dev,
-			     struct device_attribute *devattr,
-			     char *buf)
+static int lm90_get_temp8(struct lm90_data *data, int index)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct lm90_data *data = lm90_update_device(dev);
+	s8 temp8 = data->temp8[index];
 	int temp;
 
-	if (IS_ERR(data))
-		return PTR_ERR(data);
-
 	if (data->kind == adt7461 || data->kind == tmp451)
-		temp = temp_from_u8_adt7461(data, data->temp8[attr->index]);
+		temp = temp_from_u8_adt7461(data, temp8);
 	else if (data->kind == max6646)
-		temp = temp_from_u8(data->temp8[attr->index]);
+		temp = temp_from_u8(temp8);
 	else
-		temp = temp_from_s8(data->temp8[attr->index]);
+		temp = temp_from_s8(temp8);
 
 	/* +16 degrees offset for temp2 for the LM99 */
-	if (data->kind == lm99 && attr->index == 3)
+	if (data->kind == lm99 && index == 3)
 		temp += 16000;
 
-	return sprintf(buf, "%d\n", temp - temp_from_s8(data->temp_hyst));
+	return temp;
 }
 
-static ssize_t set_temphyst(struct device *dev, struct device_attribute *dummy,
-			    const char *buf, size_t count)
+static int lm90_set_temp8(struct lm90_data *data, int index, long val)
 {
-	struct lm90_data *data = dev_get_drvdata(dev);
+	static const u8 reg[TEMP8_REG_NUM] = {
+		LM90_REG_W_LOCAL_LOW,
+		LM90_REG_W_LOCAL_HIGH,
+		LM90_REG_W_LOCAL_CRIT,
+		LM90_REG_W_REMOTE_CRIT,
+		MAX6659_REG_W_LOCAL_EMERG,
+		MAX6659_REG_W_REMOTE_EMERG,
+		LM90_REG_W_REMOTE_CRIT,
+		MAX6659_REG_W_REMOTE_EMERG,
+	};
 	struct i2c_client *client = data->client;
-	long val;
 	int err;
-	int temp;
 
-	err = kstrtol(buf, 10, &val);
-	if (err < 0)
-		return err;
+	/* +16 degrees offset for temp2 for the LM99 */
+	if (data->kind == lm99 && index == 3)
+		val -= 16000;
 
-	mutex_lock(&data->update_lock);
 	if (data->kind == adt7461 || data->kind == tmp451)
-		temp = temp_from_u8_adt7461(data, data->temp8[LOCAL_CRIT]);
+		data->temp8[index] = temp_to_u8_adt7461(data, val);
 	else if (data->kind == max6646)
-		temp = temp_from_u8(data->temp8[LOCAL_CRIT]);
+		data->temp8[index] = temp_to_u8(val);
 	else
-		temp = temp_from_s8(data->temp8[LOCAL_CRIT]);
+		data->temp8[index] = temp_to_s8(val);
 
-	data->temp_hyst = hyst_to_reg(temp - val);
-	i2c_smbus_write_byte_data(client, LM90_REG_W_TCRIT_HYST,
-				  data->temp_hyst);
-	mutex_unlock(&data->update_lock);
-	return count;
-}
-
-static ssize_t show_alarms(struct device *dev, struct device_attribute *dummy,
-			   char *buf)
-{
-	struct lm90_data *data = lm90_update_device(dev);
-
-	if (IS_ERR(data))
-		return PTR_ERR(data);
+	lm90_select_remote_channel(client, data, index >= 6);
+	err = i2c_smbus_write_byte_data(client, reg[index], data->temp8[index]);
+	lm90_select_remote_channel(client, data, 0);
 
-	return sprintf(buf, "%d\n", data->alarms);
+	return err;
 }
 
-static ssize_t show_alarm(struct device *dev, struct device_attribute
-			  *devattr, char *buf)
+static int lm90_get_temphyst(struct lm90_data *data, int index)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct lm90_data *data = lm90_update_device(dev);
-	int bitnr = attr->index;
-
-	if (IS_ERR(data))
-		return PTR_ERR(data);
+	int temp;
 
-	return sprintf(buf, "%d\n", (data->alarms >> bitnr) & 1);
-}
+	if (data->kind == adt7461 || data->kind == tmp451)
+		temp = temp_from_u8_adt7461(data, data->temp8[index]);
+	else if (data->kind == max6646)
+		temp = temp_from_u8(data->temp8[index]);
+	else
+		temp = temp_from_s8(data->temp8[index]);
 
-static ssize_t show_update_interval(struct device *dev,
-				    struct device_attribute *attr, char *buf)
-{
-	struct lm90_data *data = dev_get_drvdata(dev);
+	/* +16 degrees offset for temp2 for the LM99 */
+	if (data->kind == lm99 && index == 3)
+		temp += 16000;
 
-	return sprintf(buf, "%u\n", data->update_interval);
+	return temp - temp_from_s8(data->temp_hyst);
 }
 
-static ssize_t set_update_interval(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf, size_t count)
+static int lm90_set_temphyst(struct lm90_data *data, long val)
 {
-	struct lm90_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
-	unsigned long val;
+	int temp;
 	int err;
 
-	err = kstrtoul(buf, 10, &val);
-	if (err)
-		return err;
-
-	mutex_lock(&data->update_lock);
-	lm90_set_convrate(client, data, clamp_val(val, 0, 100000));
-	mutex_unlock(&data->update_lock);
+	if (data->kind == adt7461 || data->kind == tmp451)
+		temp = temp_from_u8_adt7461(data, data->temp8[LOCAL_CRIT]);
+	else if (data->kind == max6646)
+		temp = temp_from_u8(data->temp8[LOCAL_CRIT]);
+	else
+		temp = temp_from_s8(data->temp8[LOCAL_CRIT]);
 
-	return count;
+	data->temp_hyst = hyst_to_reg(temp - val);
+	err = i2c_smbus_write_byte_data(client, LM90_REG_W_TCRIT_HYST,
+					data->temp_hyst);
+	return err;
 }
 
-static SENSOR_DEVICE_ATTR_2(temp1_input, S_IRUGO, show_temp11, NULL,
-	0, LOCAL_TEMP);
-static SENSOR_DEVICE_ATTR_2(temp2_input, S_IRUGO, show_temp11, NULL,
-	0, REMOTE_TEMP);
-static SENSOR_DEVICE_ATTR(temp1_min, S_IWUSR | S_IRUGO, show_temp8,
-	set_temp8, LOCAL_LOW);
-static SENSOR_DEVICE_ATTR_2(temp2_min, S_IWUSR | S_IRUGO, show_temp11,
-	set_temp11, 0, REMOTE_LOW);
-static SENSOR_DEVICE_ATTR(temp1_max, S_IWUSR | S_IRUGO, show_temp8,
-	set_temp8, LOCAL_HIGH);
-static SENSOR_DEVICE_ATTR_2(temp2_max, S_IWUSR | S_IRUGO, show_temp11,
-	set_temp11, 1, REMOTE_HIGH);
-static SENSOR_DEVICE_ATTR(temp1_crit, S_IWUSR | S_IRUGO, show_temp8,
-	set_temp8, LOCAL_CRIT);
-static SENSOR_DEVICE_ATTR(temp2_crit, S_IWUSR | S_IRUGO, show_temp8,
-	set_temp8, REMOTE_CRIT);
-static SENSOR_DEVICE_ATTR(temp1_crit_hyst, S_IWUSR | S_IRUGO, show_temphyst,
-	set_temphyst, LOCAL_CRIT);
-static SENSOR_DEVICE_ATTR(temp2_crit_hyst, S_IRUGO, show_temphyst, NULL,
-	REMOTE_CRIT);
-static SENSOR_DEVICE_ATTR_2(temp2_offset, S_IWUSR | S_IRUGO, show_temp11,
-	set_temp11, 2, REMOTE_OFFSET);
-
-/* Individual alarm files */
-static SENSOR_DEVICE_ATTR(temp1_crit_alarm, S_IRUGO, show_alarm, NULL, 0);
-static SENSOR_DEVICE_ATTR(temp2_crit_alarm, S_IRUGO, show_alarm, NULL, 1);
-static SENSOR_DEVICE_ATTR(temp2_fault, S_IRUGO, show_alarm, NULL, 2);
-static SENSOR_DEVICE_ATTR(temp2_min_alarm, S_IRUGO, show_alarm, NULL, 3);
-static SENSOR_DEVICE_ATTR(temp2_max_alarm, S_IRUGO, show_alarm, NULL, 4);
-static SENSOR_DEVICE_ATTR(temp1_min_alarm, S_IRUGO, show_alarm, NULL, 5);
-static SENSOR_DEVICE_ATTR(temp1_max_alarm, S_IRUGO, show_alarm, NULL, 6);
-/* Raw alarm file for compatibility */
-static DEVICE_ATTR(alarms, S_IRUGO, show_alarms, NULL);
-
-static DEVICE_ATTR(update_interval, S_IRUGO | S_IWUSR, show_update_interval,
-		   set_update_interval);
-
-static struct attribute *lm90_attributes[] = {
-	&sensor_dev_attr_temp1_input.dev_attr.attr,
-	&sensor_dev_attr_temp2_input.dev_attr.attr,
-	&sensor_dev_attr_temp1_min.dev_attr.attr,
-	&sensor_dev_attr_temp2_min.dev_attr.attr,
-	&sensor_dev_attr_temp1_max.dev_attr.attr,
-	&sensor_dev_attr_temp2_max.dev_attr.attr,
-	&sensor_dev_attr_temp1_crit.dev_attr.attr,
-	&sensor_dev_attr_temp2_crit.dev_attr.attr,
-	&sensor_dev_attr_temp1_crit_hyst.dev_attr.attr,
-	&sensor_dev_attr_temp2_crit_hyst.dev_attr.attr,
-
-	&sensor_dev_attr_temp1_crit_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp2_crit_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp2_fault.dev_attr.attr,
-	&sensor_dev_attr_temp2_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp2_max_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp1_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp1_max_alarm.dev_attr.attr,
-	&dev_attr_alarms.attr,
-	&dev_attr_update_interval.attr,
-	NULL
+static const u8 lm90_temp_index[3] = {
+	LOCAL_TEMP, REMOTE_TEMP, REMOTE2_TEMP
 };
 
-static const struct attribute_group lm90_group = {
-	.attrs = lm90_attributes,
+static const u8 lm90_temp_min_index[3] = {
+	LOCAL_LOW, REMOTE_LOW, REMOTE2_LOW
 };
 
-static struct attribute *lm90_temp2_offset_attributes[] = {
-	&sensor_dev_attr_temp2_offset.dev_attr.attr,
-	NULL
+static const u8 lm90_temp_max_index[3] = {
+	LOCAL_HIGH, REMOTE_HIGH, REMOTE2_HIGH
 };
 
-static const struct attribute_group lm90_temp2_offset_group = {
-	.attrs = lm90_temp2_offset_attributes,
+static const u8 lm90_temp_crit_index[3] = {
+	LOCAL_CRIT, REMOTE_CRIT, REMOTE2_CRIT
 };
 
-/*
- * Additional attributes for devices with emergency sensors
- */
-static SENSOR_DEVICE_ATTR(temp1_emergency, S_IWUSR | S_IRUGO, show_temp8,
-	set_temp8, LOCAL_EMERG);
-static SENSOR_DEVICE_ATTR(temp2_emergency, S_IWUSR | S_IRUGO, show_temp8,
-	set_temp8, REMOTE_EMERG);
-static SENSOR_DEVICE_ATTR(temp1_emergency_hyst, S_IRUGO, show_temphyst,
-			  NULL, LOCAL_EMERG);
-static SENSOR_DEVICE_ATTR(temp2_emergency_hyst, S_IRUGO, show_temphyst,
-			  NULL, REMOTE_EMERG);
-
-static struct attribute *lm90_emergency_attributes[] = {
-	&sensor_dev_attr_temp1_emergency.dev_attr.attr,
-	&sensor_dev_attr_temp2_emergency.dev_attr.attr,
-	&sensor_dev_attr_temp1_emergency_hyst.dev_attr.attr,
-	&sensor_dev_attr_temp2_emergency_hyst.dev_attr.attr,
-	NULL
+static const u8 lm90_temp_emerg_index[3] = {
+	LOCAL_EMERG, REMOTE_EMERG, REMOTE2_EMERG
 };
 
-static const struct attribute_group lm90_emergency_group = {
-	.attrs = lm90_emergency_attributes,
-};
+static const u8 lm90_min_alarm_bits[3] = { 5, 3, 11 };
+static const u8 lm90_max_alarm_bits[3] = { 0, 4, 12 };
+static const u8 lm90_crit_alarm_bits[3] = { 0, 1, 9 };
+static const u8 lm90_emergency_alarm_bits[3] = { 15, 13, 14 };
+static const u8 lm90_fault_bits[3] = { 0, 2, 10 };
 
-static SENSOR_DEVICE_ATTR(temp1_emergency_alarm, S_IRUGO, show_alarm, NULL, 15);
-static SENSOR_DEVICE_ATTR(temp2_emergency_alarm, S_IRUGO, show_alarm, NULL, 13);
+static int lm90_temp_read(struct device *dev, u32 attr, int channel, long *val)
+{
+	struct lm90_data *data = dev_get_drvdata(dev);
+	int err;
 
-static struct attribute *lm90_emergency_alarm_attributes[] = {
-	&sensor_dev_attr_temp1_emergency_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp2_emergency_alarm.dev_attr.attr,
-	NULL
-};
+	mutex_lock(&data->update_lock);
+	err = lm90_update_device(dev);
+	mutex_unlock(&data->update_lock);
+	if (err)
+		return err;
 
-static const struct attribute_group lm90_emergency_alarm_group = {
-	.attrs = lm90_emergency_alarm_attributes,
-};
+	switch (attr) {
+	case hwmon_temp_input:
+		*val = lm90_get_temp11(data, lm90_temp_index[channel]);
+		break;
+	case hwmon_temp_min_alarm:
+		*val = (data->alarms >> lm90_min_alarm_bits[channel]) & 1;
+		break;
+	case hwmon_temp_max_alarm:
+		*val = (data->alarms >> lm90_max_alarm_bits[channel]) & 1;
+		break;
+	case hwmon_temp_crit_alarm:
+		*val = (data->alarms >> lm90_crit_alarm_bits[channel]) & 1;
+		break;
+	case hwmon_temp_emergency_alarm:
+		*val = (data->alarms >> lm90_emergency_alarm_bits[channel]) & 1;
+		break;
+	case hwmon_temp_fault:
+		*val = (data->alarms >> lm90_fault_bits[channel]) & 1;
+		break;
+	case hwmon_temp_min:
+		if (channel == 0)
+			*val = lm90_get_temp8(data,
+					      lm90_temp_min_index[channel]);
+		else
+			*val = lm90_get_temp11(data,
+					       lm90_temp_min_index[channel]);
+		break;
+	case hwmon_temp_max:
+		if (channel == 0)
+			*val = lm90_get_temp8(data,
+					      lm90_temp_max_index[channel]);
+		else
+			*val = lm90_get_temp11(data,
+					       lm90_temp_max_index[channel]);
+		break;
+	case hwmon_temp_crit:
+		*val = lm90_get_temp8(data, lm90_temp_crit_index[channel]);
+		break;
+	case hwmon_temp_crit_hyst:
+		*val = lm90_get_temphyst(data, lm90_temp_crit_index[channel]);
+		break;
+	case hwmon_temp_emergency:
+		*val = lm90_get_temp8(data, lm90_temp_emerg_index[channel]);
+		break;
+	case hwmon_temp_emergency_hyst:
+		*val = lm90_get_temphyst(data, lm90_temp_emerg_index[channel]);
+		break;
+	case hwmon_temp_offset:
+		*val = lm90_get_temp11(data, REMOTE_OFFSET);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+	return 0;
+}
 
-/*
- * Additional attributes for devices with 3 temperature sensors
- */
-static SENSOR_DEVICE_ATTR_2(temp3_input, S_IRUGO, show_temp11, NULL,
-	0, REMOTE2_TEMP);
-static SENSOR_DEVICE_ATTR_2(temp3_min, S_IWUSR | S_IRUGO, show_temp11,
-	set_temp11, 3, REMOTE2_LOW);
-static SENSOR_DEVICE_ATTR_2(temp3_max, S_IWUSR | S_IRUGO, show_temp11,
-	set_temp11, 4, REMOTE2_HIGH);
-static SENSOR_DEVICE_ATTR(temp3_crit, S_IWUSR | S_IRUGO, show_temp8,
-	set_temp8, REMOTE2_CRIT);
-static SENSOR_DEVICE_ATTR(temp3_crit_hyst, S_IRUGO, show_temphyst, NULL,
-	REMOTE2_CRIT);
-static SENSOR_DEVICE_ATTR(temp3_emergency, S_IWUSR | S_IRUGO, show_temp8,
-	set_temp8, REMOTE2_EMERG);
-static SENSOR_DEVICE_ATTR(temp3_emergency_hyst, S_IRUGO, show_temphyst,
-			  NULL, REMOTE2_EMERG);
-
-static SENSOR_DEVICE_ATTR(temp3_crit_alarm, S_IRUGO, show_alarm, NULL, 9);
-static SENSOR_DEVICE_ATTR(temp3_fault, S_IRUGO, show_alarm, NULL, 10);
-static SENSOR_DEVICE_ATTR(temp3_min_alarm, S_IRUGO, show_alarm, NULL, 11);
-static SENSOR_DEVICE_ATTR(temp3_max_alarm, S_IRUGO, show_alarm, NULL, 12);
-static SENSOR_DEVICE_ATTR(temp3_emergency_alarm, S_IRUGO, show_alarm, NULL, 14);
-
-static struct attribute *lm90_temp3_attributes[] = {
-	&sensor_dev_attr_temp3_input.dev_attr.attr,
-	&sensor_dev_attr_temp3_min.dev_attr.attr,
-	&sensor_dev_attr_temp3_max.dev_attr.attr,
-	&sensor_dev_attr_temp3_crit.dev_attr.attr,
-	&sensor_dev_attr_temp3_crit_hyst.dev_attr.attr,
-	&sensor_dev_attr_temp3_emergency.dev_attr.attr,
-	&sensor_dev_attr_temp3_emergency_hyst.dev_attr.attr,
-
-	&sensor_dev_attr_temp3_fault.dev_attr.attr,
-	&sensor_dev_attr_temp3_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp3_max_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp3_crit_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp3_emergency_alarm.dev_attr.attr,
-	NULL
-};
+static int lm90_temp_write(struct device *dev, u32 attr, int channel, long val)
+{
+	struct lm90_data *data = dev_get_drvdata(dev);
+	int err;
 
-static const struct attribute_group lm90_temp3_group = {
-	.attrs = lm90_temp3_attributes,
-};
+	mutex_lock(&data->update_lock);
 
-/* pec used for ADM1032 only */
-static ssize_t show_pec(struct device *dev, struct device_attribute *dummy,
-			char *buf)
+	err = lm90_update_device(dev);
+	if (err)
+		goto error;
+
+	switch (attr) {
+	case hwmon_temp_min:
+		if (channel == 0)
+			err = lm90_set_temp8(data,
+					      lm90_temp_min_index[channel],
+					      val);
+		else
+			err = lm90_set_temp11(data,
+					      lm90_temp_min_index[channel],
+					      val);
+		break;
+	case hwmon_temp_max:
+		if (channel == 0)
+			err = lm90_set_temp8(data,
+					     lm90_temp_max_index[channel],
+					     val);
+		else
+			err = lm90_set_temp11(data,
+					      lm90_temp_max_index[channel],
+					      val);
+		break;
+	case hwmon_temp_crit:
+		err = lm90_set_temp8(data, lm90_temp_crit_index[channel], val);
+		break;
+	case hwmon_temp_crit_hyst:
+		err = lm90_set_temphyst(data, val);
+		break;
+	case hwmon_temp_emergency:
+		err = lm90_set_temp8(data, lm90_temp_emerg_index[channel], val);
+		break;
+	case hwmon_temp_offset:
+		err = lm90_set_temp11(data, REMOTE_OFFSET, val);
+		break;
+	default:
+		err = -EOPNOTSUPP;
+		break;
+	}
+error:
+	mutex_unlock(&data->update_lock);
+
+	return err;
+}
+
+static umode_t lm90_temp_is_visible(const void *data, u32 attr, int channel)
 {
-	struct i2c_client *client = to_i2c_client(dev);
-	return sprintf(buf, "%d\n", !!(client->flags & I2C_CLIENT_PEC));
+	switch (attr) {
+	case hwmon_temp_input:
+	case hwmon_temp_min_alarm:
+	case hwmon_temp_max_alarm:
+	case hwmon_temp_crit_alarm:
+	case hwmon_temp_emergency_alarm:
+	case hwmon_temp_emergency_hyst:
+	case hwmon_temp_fault:
+		return S_IRUGO;
+	case hwmon_temp_min:
+	case hwmon_temp_max:
+	case hwmon_temp_crit:
+	case hwmon_temp_emergency:
+	case hwmon_temp_offset:
+		return S_IRUGO | S_IWUSR;
+	case hwmon_temp_crit_hyst:
+		if (channel == 0)
+			return S_IRUGO | S_IWUSR;
+		return S_IRUGO;
+	default:
+		return 0;
+	}
 }
 
-static ssize_t set_pec(struct device *dev, struct device_attribute *dummy,
-		       const char *buf, size_t count)
+static int lm90_chip_read(struct device *dev, u32 attr, int channel, long *val)
 {
-	struct i2c_client *client = to_i2c_client(dev);
-	long val;
+	struct lm90_data *data = dev_get_drvdata(dev);
 	int err;
 
-	err = kstrtol(buf, 10, &val);
-	if (err < 0)
+	mutex_lock(&data->update_lock);
+	err = lm90_update_device(dev);
+	mutex_unlock(&data->update_lock);
+	if (err)
 		return err;
 
-	switch (val) {
-	case 0:
-		client->flags &= ~I2C_CLIENT_PEC;
+	switch (attr) {
+	case hwmon_chip_update_interval:
+		*val = data->update_interval;
 		break;
-	case 1:
-		client->flags |= I2C_CLIENT_PEC;
+	case hwmon_chip_alarms:
+		*val = data->alarms;
 		break;
 	default:
-		return -EINVAL;
+		return -EOPNOTSUPP;
 	}
 
-	return count;
+	return 0;
 }
 
-static DEVICE_ATTR(pec, S_IWUSR | S_IRUGO, show_pec, set_pec);
+static int lm90_chip_write(struct device *dev, u32 attr, int channel, long val)
+{
+	struct lm90_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	int err;
 
-/*
- * Real code
- */
+	mutex_lock(&data->update_lock);
+
+	err = lm90_update_device(dev);
+	if (err)
+		goto error;
+
+	switch (attr) {
+	case hwmon_chip_update_interval:
+		err = lm90_set_convrate(client, data,
+					clamp_val(val, 0, 100000));
+		break;
+	default:
+		err = -EOPNOTSUPP;
+		break;
+	}
+error:
+	mutex_unlock(&data->update_lock);
+
+	return err;
+}
+
+static umode_t lm90_chip_is_visible(const void *data, u32 attr, int channel)
+{
+	switch (attr) {
+	case hwmon_chip_update_interval:
+		return S_IRUGO | S_IWUSR;
+	case hwmon_chip_alarms:
+		return S_IRUGO;
+	default:
+		return 0;
+	}
+}
+
+static int lm90_read(struct device *dev, enum hwmon_sensor_types type,
+		     u32 attr, int channel, long *val)
+{
+	switch (type) {
+	case hwmon_chip:
+		return lm90_chip_read(dev, attr, channel, val);
+	case hwmon_temp:
+		return lm90_temp_read(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int lm90_write(struct device *dev, enum hwmon_sensor_types type,
+		      u32 attr, int channel, long val)
+{
+	switch (type) {
+	case hwmon_chip:
+		return lm90_chip_write(dev, attr, channel, val);
+	case hwmon_temp:
+		return lm90_temp_write(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static umode_t lm90_is_visible(const void *data, enum hwmon_sensor_types type,
+			       u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_chip:
+		return lm90_chip_is_visible(data, attr, channel);
+	case hwmon_temp:
+		return lm90_temp_is_visible(data, attr, channel);
+	default:
+		return 0;
+	}
+}
 
 /* Return 0 if detection is successful, -ENODEV otherwise */
 static int lm90_detect(struct i2c_client *client,
@@ -1617,15 +1626,32 @@ static void lm90_regulator_disable(void *regulator)
 	regulator_disable(regulator);
 }
 
+static const u32 lm90_chip_config[] = {
+	HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL | HWMON_C_ALARMS,
+	0
+};
+
+static const struct hwmon_channel_info lm90_chip_info = {
+	.type = hwmon_chip,
+	.config = lm90_chip_config,
+};
+
+
+static const struct hwmon_ops lm90_ops = {
+	.is_visible = lm90_is_visible,
+	.read = lm90_read,
+	.write = lm90_write,
+};
+
 static int lm90_probe(struct i2c_client *client,
 		      const struct i2c_device_id *id)
 {
 	struct device *dev = &client->dev;
 	struct i2c_adapter *adapter = to_i2c_adapter(dev->parent);
-	struct lm90_data *data;
+	struct hwmon_channel_info *info;
 	struct regulator *regulator;
 	struct device *hwmon_dev;
-	int groups = 0;
+	struct lm90_data *data;
 	int err;
 
 	regulator = devm_regulator_get(dev, "vcc");
@@ -1665,6 +1691,49 @@ static int lm90_probe(struct i2c_client *client,
 
 	/* Set chip capabilities */
 	data->flags = lm90_params[data->kind].flags;
+
+	data->chip.ops = &lm90_ops;
+	data->chip.info = data->info;
+
+	data->info[0] = &lm90_chip_info;
+	data->info[1] = &data->temp_info;
+
+	info = &data->temp_info;
+	info->type = hwmon_temp;
+	info->config = data->channel_config;
+
+	data->channel_config[0] = HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX |
+		HWMON_T_CRIT | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM |
+		HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM;
+	data->channel_config[1] = HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX |
+		HWMON_T_CRIT | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM |
+		HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT;
+
+	if (data->flags & LM90_HAVE_OFFSET)
+		data->channel_config[1] |= HWMON_T_OFFSET;
+
+	if (data->flags & LM90_HAVE_EMERGENCY) {
+		data->channel_config[0] |= HWMON_T_EMERGENCY |
+			HWMON_T_EMERGENCY_HYST;
+		data->channel_config[1] |= HWMON_T_EMERGENCY |
+			HWMON_T_EMERGENCY_HYST;
+	}
+
+	if (data->flags & LM90_HAVE_EMERGENCY_ALARM) {
+		data->channel_config[0] |= HWMON_T_EMERGENCY_ALARM;
+		data->channel_config[1] |= HWMON_T_EMERGENCY_ALARM;
+	}
+
+	if (data->flags & LM90_HAVE_TEMP3) {
+		data->channel_config[2] = HWMON_T_INPUT |
+			HWMON_T_MIN | HWMON_T_MAX |
+			HWMON_T_CRIT | HWMON_T_CRIT_HYST |
+			HWMON_T_EMERGENCY | HWMON_T_EMERGENCY_HYST |
+			HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM |
+			HWMON_T_CRIT_ALARM | HWMON_T_EMERGENCY_ALARM |
+			HWMON_T_FAULT;
+	}
+
 	data->reg_local_ext = lm90_params[data->kind].reg_local_ext;
 
 	/* Set maximum conversion rate */
@@ -1677,21 +1746,10 @@ static int lm90_probe(struct i2c_client *client,
 		return err;
 	}
 
-	/* Register sysfs hooks */
-	data->groups[groups++] = &lm90_group;
-
-	if (data->flags & LM90_HAVE_OFFSET)
-		data->groups[groups++] = &lm90_temp2_offset_group;
-
-	if (data->flags & LM90_HAVE_EMERGENCY)
-		data->groups[groups++] = &lm90_emergency_group;
-
-	if (data->flags & LM90_HAVE_EMERGENCY_ALARM)
-		data->groups[groups++] = &lm90_emergency_alarm_group;
-
-	if (data->flags & LM90_HAVE_TEMP3)
-		data->groups[groups++] = &lm90_temp3_group;
-
+	/*
+	 * The 'pec' attribute is attached to the i2c device and thus created
+	 * separately.
+	 */
 	if (client->flags & I2C_CLIENT_PEC) {
 		err = device_create_file(dev, &dev_attr_pec);
 		if (err)
@@ -1701,8 +1759,9 @@ static int lm90_probe(struct i2c_client *client,
 			return err;
 	}
 
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
-							   data, data->groups);
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 data, &data->chip,
+							 NULL);
 	if (IS_ERR(hwmon_dev))
 		return PTR_ERR(hwmon_dev);
 
diff --git a/drivers/hwmon/lm95241.c b/drivers/hwmon/lm95241.c
index cdf19adaec79..8c573e6e9726 100644
--- a/drivers/hwmon/lm95241.c
+++ b/drivers/hwmon/lm95241.c
@@ -15,22 +15,17 @@
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#include <linux/module.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
 #include <linux/init.h>
-#include <linux/slab.h>
 #include <linux/jiffies.h>
-#include <linux/i2c.h>
 #include <linux/hwmon.h>
-#include <linux/hwmon-sysfs.h>
-#include <linux/err.h>
+#include <linux/module.h>
 #include <linux/mutex.h>
-#include <linux/sysfs.h>
+#include <linux/slab.h>
 
 #define DEVNAME "lm95241"
 
@@ -54,26 +49,25 @@ static const unsigned short normal_i2c[] = {
 #define LM95241_REG_RW_REMOTE_MODEL	0x30
 
 /* LM95241 specific bitfields */
-#define CFG_STOP 0x40
-#define CFG_CR0076 0x00
-#define CFG_CR0182 0x10
-#define CFG_CR1000 0x20
-#define CFG_CR2700 0x30
-#define R1MS_SHIFT 0
-#define R2MS_SHIFT 2
-#define R1MS_MASK (0x01 << (R1MS_SHIFT))
-#define R2MS_MASK (0x01 << (R2MS_SHIFT))
-#define R1DF_SHIFT 1
-#define R2DF_SHIFT 2
-#define R1DF_MASK (0x01 << (R1DF_SHIFT))
-#define R2DF_MASK (0x01 << (R2DF_SHIFT))
-#define R1FE_MASK 0x01
-#define R2FE_MASK 0x05
-#define TT1_SHIFT 0
-#define TT2_SHIFT 4
-#define TT_OFF 0
-#define TT_ON 1
-#define TT_MASK 7
+#define CFG_STOP	BIT(6)
+#define CFG_CR0076	0x00
+#define CFG_CR0182	BIT(4)
+#define CFG_CR1000	BIT(5)
+#define CFG_CR2700	(BIT(4) | BIT(5))
+#define CFG_CRMASK	(BIT(4) | BIT(5))
+#define R1MS_MASK	BIT(0)
+#define R2MS_MASK	BIT(2)
+#define R1DF_MASK	BIT(1)
+#define R2DF_MASK	BIT(2)
+#define R1FE_MASK	BIT(0)
+#define R2FE_MASK	BIT(2)
+#define R1DM		BIT(0)
+#define R2DM		BIT(1)
+#define TT1_SHIFT	0
+#define TT2_SHIFT	4
+#define TT_OFF		0
+#define TT_ON		1
+#define TT_MASK		7
 #define NATSEMI_MAN_ID	0x01
 #define LM95231_CHIP_ID	0xA1
 #define LM95241_CHIP_ID	0xA4
@@ -91,11 +85,12 @@ static const u8 lm95241_reg_address[] = {
 struct lm95241_data {
 	struct i2c_client *client;
 	struct mutex update_lock;
-	unsigned long last_updated, interval;	/* in jiffies */
+	unsigned long last_updated;	/* in jiffies */
+	unsigned long interval;		/* in milli-seconds */
 	char valid;		/* zero until following fields are valid */
 	/* registers values */
 	u8 temp[ARRAY_SIZE(lm95241_reg_address)];
-	u8 config, model, trutherm;
+	u8 status, config, model, trutherm;
 };
 
 /* Conversions */
@@ -118,7 +113,8 @@ static struct lm95241_data *lm95241_update_device(struct device *dev)
 
 	mutex_lock(&data->update_lock);
 
-	if (time_after(jiffies, data->last_updated + data->interval) ||
+	if (time_after(jiffies, data->last_updated
+		       + msecs_to_jiffies(data->interval)) ||
 	    !data->valid) {
 		int i;
 
@@ -127,6 +123,9 @@ static struct lm95241_data *lm95241_update_device(struct device *dev)
 			data->temp[i]
 			  = i2c_smbus_read_byte_data(client,
 						     lm95241_reg_address[i]);
+
+		data->status = i2c_smbus_read_byte_data(client,
+							LM95241_REG_R_STATUS);
 		data->last_updated = jiffies;
 		data->valid = 1;
 	}
@@ -136,197 +135,241 @@ static struct lm95241_data *lm95241_update_device(struct device *dev)
 	return data;
 }
 
-/* Sysfs stuff */
-static ssize_t show_input(struct device *dev, struct device_attribute *attr,
-			  char *buf)
-{
-	struct lm95241_data *data = lm95241_update_device(dev);
-	int index = to_sensor_dev_attr(attr)->index;
-
-	return snprintf(buf, PAGE_SIZE - 1, "%d\n",
-			index == 0 || (data->config & (1 << (index / 2))) ?
-		temp_from_reg_signed(data->temp[index], data->temp[index + 1]) :
-		temp_from_reg_unsigned(data->temp[index],
-				       data->temp[index + 1]));
-}
-
-static ssize_t show_type(struct device *dev, struct device_attribute *attr,
-			 char *buf)
+static int lm95241_read_chip(struct device *dev, u32 attr, int channel,
+			     long *val)
 {
 	struct lm95241_data *data = dev_get_drvdata(dev);
 
-	return snprintf(buf, PAGE_SIZE - 1,
-		data->model & to_sensor_dev_attr(attr)->index ? "1\n" : "2\n");
+	switch (attr) {
+	case hwmon_chip_update_interval:
+		*val = data->interval;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t set_type(struct device *dev, struct device_attribute *attr,
-			const char *buf, size_t count)
+static int lm95241_read_temp(struct device *dev, u32 attr, int channel,
+			     long *val)
 {
-	struct lm95241_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
-	unsigned long val;
-	int shift;
-	u8 mask = to_sensor_dev_attr(attr)->index;
-
-	if (kstrtoul(buf, 10, &val) < 0)
-		return -EINVAL;
-	if (val != 1 && val != 2)
-		return -EINVAL;
-
-	shift = mask == R1MS_MASK ? TT1_SHIFT : TT2_SHIFT;
-
-	mutex_lock(&data->update_lock);
+	struct lm95241_data *data = lm95241_update_device(dev);
 
-	data->trutherm &= ~(TT_MASK << shift);
-	if (val == 1) {
-		data->model |= mask;
-		data->trutherm |= (TT_ON << shift);
-	} else {
-		data->model &= ~mask;
-		data->trutherm |= (TT_OFF << shift);
+	switch (attr) {
+	case hwmon_temp_input:
+		if (!channel || (data->config & BIT(channel - 1)))
+			*val = temp_from_reg_signed(data->temp[channel * 2],
+						data->temp[channel * 2 + 1]);
+		else
+			*val = temp_from_reg_unsigned(data->temp[channel * 2],
+						data->temp[channel * 2 + 1]);
+		return 0;
+	case hwmon_temp_min:
+		if (channel == 1)
+			*val = (data->config & R1DF_MASK) ? -128000 : 0;
+		else
+			*val = (data->config & R2DF_MASK) ? -128000 : 0;
+		return 0;
+	case hwmon_temp_max:
+		if (channel == 1)
+			*val = (data->config & R1DF_MASK) ? 127875 : 255875;
+		else
+			*val = (data->config & R2DF_MASK) ? 127875 : 255875;
+		return 0;
+	case hwmon_temp_type:
+		if (channel == 1)
+			*val = (data->model & R1MS_MASK) ? 1 : 2;
+		else
+			*val = (data->model & R2MS_MASK) ? 1 : 2;
+		return 0;
+	case hwmon_temp_fault:
+		if (channel == 1)
+			*val = !!(data->status & R1DM);
+		else
+			*val = !!(data->status & R2DM);
+		return 0;
+	default:
+		return -EOPNOTSUPP;
 	}
-	data->valid = 0;
-
-	i2c_smbus_write_byte_data(client, LM95241_REG_RW_REMOTE_MODEL,
-				  data->model);
-	i2c_smbus_write_byte_data(client, LM95241_REG_RW_TRUTHERM,
-				  data->trutherm);
-
-	mutex_unlock(&data->update_lock);
-
-	return count;
 }
 
-static ssize_t show_min(struct device *dev, struct device_attribute *attr,
-			char *buf)
+static int lm95241_read(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long *val)
 {
-	struct lm95241_data *data = dev_get_drvdata(dev);
-
-	return snprintf(buf, PAGE_SIZE - 1,
-			data->config & to_sensor_dev_attr(attr)->index ?
-			"-127000\n" : "0\n");
+	switch (type) {
+	case hwmon_chip:
+		return lm95241_read_chip(dev, attr, channel, val);
+	case hwmon_temp:
+		return lm95241_read_temp(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t set_min(struct device *dev, struct device_attribute *attr,
-		       const char *buf, size_t count)
+static int lm95241_write_chip(struct device *dev, u32 attr, int channel,
+			      long val)
 {
 	struct lm95241_data *data = dev_get_drvdata(dev);
-	long val;
-
-	if (kstrtol(buf, 10, &val) < 0)
-		return -EINVAL;
-	if (val < -128000)
-		return -EINVAL;
+	int convrate;
+	u8 config;
+	int ret;
 
 	mutex_lock(&data->update_lock);
 
-	if (val < 0)
-		data->config |= to_sensor_dev_attr(attr)->index;
-	else
-		data->config &= ~to_sensor_dev_attr(attr)->index;
-	data->valid = 0;
-
-	i2c_smbus_write_byte_data(data->client, LM95241_REG_RW_CONFIG,
-				  data->config);
-
+	switch (attr) {
+	case hwmon_chip_update_interval:
+		config = data->config & ~CFG_CRMASK;
+		if (val < 130) {
+			convrate = 76;
+			config |= CFG_CR0076;
+		} else if (val < 590) {
+			convrate = 182;
+			config |= CFG_CR0182;
+		} else if (val < 1850) {
+			convrate = 1000;
+			config |= CFG_CR1000;
+		} else {
+			convrate = 2700;
+			config |= CFG_CR2700;
+		}
+		data->interval = convrate;
+		data->config = config;
+		ret = i2c_smbus_write_byte_data(data->client,
+						LM95241_REG_RW_CONFIG, config);
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
 	mutex_unlock(&data->update_lock);
-
-	return count;
+	return ret;
 }
 
-static ssize_t show_max(struct device *dev, struct device_attribute *attr,
-			char *buf)
+static int lm95241_write_temp(struct device *dev, u32 attr, int channel,
+			      long val)
 {
 	struct lm95241_data *data = dev_get_drvdata(dev);
-
-	return snprintf(buf, PAGE_SIZE - 1,
-			data->config & to_sensor_dev_attr(attr)->index ?
-			"127000\n" : "255000\n");
-}
-
-static ssize_t set_max(struct device *dev, struct device_attribute *attr,
-		       const char *buf, size_t count)
-{
-	struct lm95241_data *data = dev_get_drvdata(dev);
-	long val;
-
-	if (kstrtol(buf, 10, &val) < 0)
-		return -EINVAL;
-	if (val >= 256000)
-		return -EINVAL;
+	struct i2c_client *client = data->client;
+	int ret;
 
 	mutex_lock(&data->update_lock);
 
-	if (val <= 127000)
-		data->config |= to_sensor_dev_attr(attr)->index;
-	else
-		data->config &= ~to_sensor_dev_attr(attr)->index;
-	data->valid = 0;
-
-	i2c_smbus_write_byte_data(data->client, LM95241_REG_RW_CONFIG,
-				  data->config);
+	switch (attr) {
+	case hwmon_temp_min:
+		if (channel == 1) {
+			if (val < 0)
+				data->config |= R1DF_MASK;
+			else
+				data->config &= ~R1DF_MASK;
+		} else {
+			if (val < 0)
+				data->config |= R2DF_MASK;
+			else
+				data->config &= ~R2DF_MASK;
+		}
+		data->valid = 0;
+		ret = i2c_smbus_write_byte_data(client, LM95241_REG_RW_CONFIG,
+						data->config);
+		break;
+	case hwmon_temp_max:
+		if (channel == 1) {
+			if (val <= 127875)
+				data->config |= R1DF_MASK;
+			else
+				data->config &= ~R1DF_MASK;
+		} else {
+			if (val <= 127875)
+				data->config |= R2DF_MASK;
+			else
+				data->config &= ~R2DF_MASK;
+		}
+		data->valid = 0;
+		ret = i2c_smbus_write_byte_data(client, LM95241_REG_RW_CONFIG,
+						data->config);
+		break;
+	case hwmon_temp_type:
+		if (val != 1 && val != 2) {
+			ret = -EINVAL;
+			break;
+		}
+		if (channel == 1) {
+			data->trutherm &= ~(TT_MASK << TT1_SHIFT);
+			if (val == 1) {
+				data->model |= R1MS_MASK;
+				data->trutherm |= (TT_ON << TT1_SHIFT);
+			} else {
+				data->model &= ~R1MS_MASK;
+				data->trutherm |= (TT_OFF << TT1_SHIFT);
+			}
+		} else {
+			data->trutherm &= ~(TT_MASK << TT2_SHIFT);
+			if (val == 1) {
+				data->model |= R2MS_MASK;
+				data->trutherm |= (TT_ON << TT2_SHIFT);
+			} else {
+				data->model &= ~R2MS_MASK;
+				data->trutherm |= (TT_OFF << TT2_SHIFT);
+			}
+		}
+		ret = i2c_smbus_write_byte_data(client,
+						LM95241_REG_RW_REMOTE_MODEL,
+						data->model);
+		if (ret < 0)
+			break;
+		ret = i2c_smbus_write_byte_data(client, LM95241_REG_RW_TRUTHERM,
+						data->trutherm);
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
 
 	mutex_unlock(&data->update_lock);
 
-	return count;
+	return ret;
 }
 
-static ssize_t show_interval(struct device *dev, struct device_attribute *attr,
-			     char *buf)
+static int lm95241_write(struct device *dev, enum hwmon_sensor_types type,
+			 u32 attr, int channel, long val)
 {
-	struct lm95241_data *data = lm95241_update_device(dev);
-
-	return snprintf(buf, PAGE_SIZE - 1, "%lu\n", 1000 * data->interval
-			/ HZ);
+	switch (type) {
+	case hwmon_chip:
+		return lm95241_write_chip(dev, attr, channel, val);
+	case hwmon_temp:
+		return lm95241_write_temp(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t set_interval(struct device *dev, struct device_attribute *attr,
-			    const char *buf, size_t count)
+static umode_t lm95241_is_visible(const void *data,
+				  enum hwmon_sensor_types type,
+				  u32 attr, int channel)
 {
-	struct lm95241_data *data = dev_get_drvdata(dev);
-	unsigned long val;
-
-	if (kstrtoul(buf, 10, &val) < 0)
-		return -EINVAL;
-
-	data->interval = val * HZ / 1000;
-
-	return count;
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return S_IRUGO | S_IWUSR;
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+			return S_IRUGO;
+		case hwmon_temp_fault:
+			return S_IRUGO;
+		case hwmon_temp_min:
+		case hwmon_temp_max:
+		case hwmon_temp_type:
+			return S_IRUGO | S_IWUSR;
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
 }
 
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, show_input, NULL, 0);
-static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, show_input, NULL, 2);
-static SENSOR_DEVICE_ATTR(temp3_input, S_IRUGO, show_input, NULL, 4);
-static SENSOR_DEVICE_ATTR(temp2_type, S_IWUSR | S_IRUGO, show_type, set_type,
-			  R1MS_MASK);
-static SENSOR_DEVICE_ATTR(temp3_type, S_IWUSR | S_IRUGO, show_type, set_type,
-			  R2MS_MASK);
-static SENSOR_DEVICE_ATTR(temp2_min, S_IWUSR | S_IRUGO, show_min, set_min,
-			  R1DF_MASK);
-static SENSOR_DEVICE_ATTR(temp3_min, S_IWUSR | S_IRUGO, show_min, set_min,
-			  R2DF_MASK);
-static SENSOR_DEVICE_ATTR(temp2_max, S_IWUSR | S_IRUGO, show_max, set_max,
-			  R1DF_MASK);
-static SENSOR_DEVICE_ATTR(temp3_max, S_IWUSR | S_IRUGO, show_max, set_max,
-			  R2DF_MASK);
-static DEVICE_ATTR(update_interval, S_IWUSR | S_IRUGO, show_interval,
-		   set_interval);
-
-static struct attribute *lm95241_attrs[] = {
-	&sensor_dev_attr_temp1_input.dev_attr.attr,
-	&sensor_dev_attr_temp2_input.dev_attr.attr,
-	&sensor_dev_attr_temp3_input.dev_attr.attr,
-	&sensor_dev_attr_temp2_type.dev_attr.attr,
-	&sensor_dev_attr_temp3_type.dev_attr.attr,
-	&sensor_dev_attr_temp2_min.dev_attr.attr,
-	&sensor_dev_attr_temp3_min.dev_attr.attr,
-	&sensor_dev_attr_temp2_max.dev_attr.attr,
-	&sensor_dev_attr_temp3_max.dev_attr.attr,
-	&dev_attr_update_interval.attr,
-	NULL
-};
-ATTRIBUTE_GROUPS(lm95241);
-
 /* Return 0 if detection is successful, -ENODEV otherwise */
 static int lm95241_detect(struct i2c_client *new_client,
 			  struct i2c_board_info *info)
@@ -362,8 +405,8 @@ static int lm95241_detect(struct i2c_client *new_client,
 static void lm95241_init_client(struct i2c_client *client,
 				struct lm95241_data *data)
 {
-	data->interval = HZ;	/* 1 sec default */
-	data->config = CFG_CR0076;
+	data->interval = 1000;
+	data->config = CFG_CR1000;
 	data->trutherm = (TT_OFF << TT1_SHIFT) | (TT_OFF << TT2_SHIFT);
 
 	i2c_smbus_write_byte_data(client, LM95241_REG_RW_CONFIG, data->config);
@@ -375,6 +418,47 @@ static void lm95241_init_client(struct i2c_client *client,
 				  data->model);
 }
 
+static const u32 lm95241_chip_config[] = {
+	HWMON_C_UPDATE_INTERVAL,
+	0
+};
+
+static const struct hwmon_channel_info lm95241_chip = {
+	.type = hwmon_chip,
+	.config = lm95241_chip_config,
+};
+
+static const u32 lm95241_temp_config[] = {
+	HWMON_T_INPUT,
+	HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MIN | HWMON_T_TYPE |
+		HWMON_T_FAULT,
+	HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MIN | HWMON_T_TYPE |
+		HWMON_T_FAULT,
+	0
+};
+
+static const struct hwmon_channel_info lm95241_temp = {
+	.type = hwmon_temp,
+	.config = lm95241_temp_config,
+};
+
+static const struct hwmon_channel_info *lm95241_info[] = {
+	&lm95241_chip,
+	&lm95241_temp,
+	NULL
+};
+
+static const struct hwmon_ops lm95241_hwmon_ops = {
+	.is_visible = lm95241_is_visible,
+	.read = lm95241_read,
+	.write = lm95241_write,
+};
+
+static const struct hwmon_chip_info lm95241_chip_info = {
+	.ops = &lm95241_hwmon_ops,
+	.info = lm95241_info,
+};
+
 static int lm95241_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
@@ -392,9 +476,10 @@ static int lm95241_probe(struct i2c_client *client,
 	/* Initialize the LM95241 chip */
 	lm95241_init_client(client, data);
 
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
 							   data,
-							   lm95241_groups);
+							   &lm95241_chip_info,
+							   NULL);
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
@@ -420,5 +505,5 @@ static struct i2c_driver lm95241_driver = {
 module_i2c_driver(lm95241_driver);
 
 MODULE_AUTHOR("Davide Rizzo <elpa.rizzo@gmail.com>");
-MODULE_DESCRIPTION("LM95241 sensor driver");
+MODULE_DESCRIPTION("LM95231/LM95241 sensor driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/lm95245.c b/drivers/hwmon/lm95245.c
index e7aef4561c83..a3bfd88752ca 100644
--- a/drivers/hwmon/lm95245.c
+++ b/drivers/hwmon/lm95245.c
@@ -15,22 +15,16 @@
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#include <linux/module.h>
+#include <linux/err.h>
 #include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/jiffies.h>
-#include <linux/i2c.h>
 #include <linux/hwmon.h>
-#include <linux/hwmon-sysfs.h>
-#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
 #include <linux/mutex.h>
-#include <linux/sysfs.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
 
 static const unsigned short normal_i2c[] = {
 	0x18, 0x19, 0x29, 0x4c, 0x4d, I2C_CLIENT_END };
@@ -89,6 +83,7 @@ static const unsigned short normal_i2c[] = {
 #define RATE_CR1000	0x02
 #define RATE_CR2500	0x03
 
+#define STATUS1_ROS		0x10
 #define STATUS1_DIODE_FAULT	0x04
 #define STATUS1_RTCRIT		0x02
 #define STATUS1_LOC		0x01
@@ -112,14 +107,9 @@ static const u8 lm95245_reg_address[] = {
 
 /* Client data (each client gets its own) */
 struct lm95245_data {
-	struct i2c_client *client;
+	struct regmap *regmap;
 	struct mutex update_lock;
-	unsigned long last_updated;	/* in jiffies */
-	unsigned long interval;	/* in msecs */
-	bool valid;		/* zero until following fields are valid */
-	/* registers values */
-	u8 regs[ARRAY_SIZE(lm95245_reg_address)];
-	u8 config1, config2;
+	int interval;	/* in msecs */
 };
 
 /* Conversions */
@@ -135,60 +125,36 @@ static int temp_from_reg_signed(u8 val_h, u8 val_l)
 	return temp_from_reg_unsigned(val_h, val_l);
 }
 
-static struct lm95245_data *lm95245_update_device(struct device *dev)
+static int lm95245_read_conversion_rate(struct lm95245_data *data)
 {
-	struct lm95245_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
+	unsigned int rate;
+	int ret;
 
-	mutex_lock(&data->update_lock);
-
-	if (time_after(jiffies, data->last_updated
-		+ msecs_to_jiffies(data->interval)) || !data->valid) {
-		int i;
-
-		for (i = 0; i < ARRAY_SIZE(lm95245_reg_address); i++)
-			data->regs[i]
-			  = i2c_smbus_read_byte_data(client,
-						     lm95245_reg_address[i]);
-		data->last_updated = jiffies;
-		data->valid = 1;
-	}
-
-	mutex_unlock(&data->update_lock);
-
-	return data;
-}
-
-static unsigned long lm95245_read_conversion_rate(struct i2c_client *client)
-{
-	int rate;
-	unsigned long interval;
-
-	rate = i2c_smbus_read_byte_data(client, LM95245_REG_RW_CONVERS_RATE);
+	ret = regmap_read(data->regmap, LM95245_REG_RW_CONVERS_RATE, &rate);
+	if (ret < 0)
+		return ret;
 
 	switch (rate) {
 	case RATE_CR0063:
-		interval = 63;
+		data->interval = 63;
 		break;
 	case RATE_CR0364:
-		interval = 364;
+		data->interval = 364;
 		break;
 	case RATE_CR1000:
-		interval = 1000;
+		data->interval = 1000;
 		break;
 	case RATE_CR2500:
 	default:
-		interval = 2500;
+		data->interval = 2500;
 		break;
 	}
-
-	return interval;
+	return 0;
 }
 
-static unsigned long lm95245_set_conversion_rate(struct i2c_client *client,
-			unsigned long interval)
+static int lm95245_set_conversion_rate(struct lm95245_data *data, long interval)
 {
-	int rate;
+	int ret, rate;
 
 	if (interval <= 63) {
 		interval = 63;
@@ -204,221 +170,289 @@ static unsigned long lm95245_set_conversion_rate(struct i2c_client *client,
 		rate = RATE_CR2500;
 	}
 
-	i2c_smbus_write_byte_data(client, LM95245_REG_RW_CONVERS_RATE, rate);
+	ret = regmap_write(data->regmap, LM95245_REG_RW_CONVERS_RATE, rate);
+	if (ret < 0)
+		return ret;
 
-	return interval;
-}
-
-/* Sysfs stuff */
-static ssize_t show_input(struct device *dev, struct device_attribute *attr,
-			  char *buf)
-{
-	struct lm95245_data *data = lm95245_update_device(dev);
-	int temp;
-	int index = to_sensor_dev_attr(attr)->index;
-
-	/*
-	 * Index 0 (Local temp) is always signed
-	 * Index 2 (Remote temp) has both signed and unsigned data
-	 * use signed calculation for remote if signed bit is set
-	 */
-	if (index == 0 || data->regs[index] & 0x80)
-		temp = temp_from_reg_signed(data->regs[index],
-			    data->regs[index + 1]);
-	else
-		temp = temp_from_reg_unsigned(data->regs[index + 2],
-			    data->regs[index + 3]);
-
-	return snprintf(buf, PAGE_SIZE - 1, "%d\n", temp);
-}
-
-static ssize_t show_limit(struct device *dev, struct device_attribute *attr,
-			 char *buf)
-{
-	struct lm95245_data *data = lm95245_update_device(dev);
-	int index = to_sensor_dev_attr(attr)->index;
-
-	return snprintf(buf, PAGE_SIZE - 1, "%d\n",
-			data->regs[index] * 1000);
+	data->interval = interval;
+	return 0;
 }
 
-static ssize_t set_limit(struct device *dev, struct device_attribute *attr,
-			const char *buf, size_t count)
+static int lm95245_read_temp(struct device *dev, u32 attr, int channel,
+			     long *val)
 {
 	struct lm95245_data *data = dev_get_drvdata(dev);
-	int index = to_sensor_dev_attr(attr)->index;
-	struct i2c_client *client = data->client;
-	unsigned long val;
-
-	if (kstrtoul(buf, 10, &val) < 0)
-		return -EINVAL;
-
-	val /= 1000;
-
-	val = clamp_val(val, 0, (index == 6 ? 127 : 255));
-
-	mutex_lock(&data->update_lock);
-
-	data->valid = 0;
-
-	i2c_smbus_write_byte_data(client, lm95245_reg_address[index], val);
-
-	mutex_unlock(&data->update_lock);
-
-	return count;
+	struct regmap *regmap = data->regmap;
+	int ret, regl, regh, regvall, regvalh;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		regl = channel ? LM95245_REG_R_REMOTE_TEMPL_S :
+				 LM95245_REG_R_LOCAL_TEMPL_S;
+		regh = channel ? LM95245_REG_R_REMOTE_TEMPH_S :
+				 LM95245_REG_R_LOCAL_TEMPH_S;
+		ret = regmap_read(regmap, regl, &regvall);
+		if (ret < 0)
+			return ret;
+		ret = regmap_read(regmap, regh, &regvalh);
+		if (ret < 0)
+			return ret;
+		/*
+		 * Local temp is always signed.
+		 * Remote temp has both signed and unsigned data.
+		 * Use signed calculation for remote if signed bit is set
+		 * or if reported temperature is below signed limit.
+		 */
+		if (!channel || (regvalh & 0x80) || regvalh < 0x7f) {
+			*val = temp_from_reg_signed(regvalh, regvall);
+			return 0;
+		}
+		ret = regmap_read(regmap, LM95245_REG_R_REMOTE_TEMPL_U,
+				  &regvall);
+		if (ret < 0)
+			return ret;
+		ret = regmap_read(regmap, LM95245_REG_R_REMOTE_TEMPH_U,
+				  &regvalh);
+		if (ret < 0)
+			return ret;
+		*val = temp_from_reg_unsigned(regvalh, regvall);
+		return 0;
+	case hwmon_temp_max:
+		ret = regmap_read(regmap, LM95245_REG_RW_REMOTE_OS_LIMIT,
+				  &regvalh);
+		if (ret < 0)
+			return ret;
+		*val = regvalh * 1000;
+		return 0;
+	case hwmon_temp_crit:
+		regh = channel ? LM95245_REG_RW_REMOTE_TCRIT_LIMIT :
+				 LM95245_REG_RW_LOCAL_OS_TCRIT_LIMIT;
+		ret = regmap_read(regmap, regh, &regvalh);
+		if (ret < 0)
+			return ret;
+		*val = regvalh * 1000;
+		return 0;
+	case hwmon_temp_max_hyst:
+		ret = regmap_read(regmap, LM95245_REG_RW_REMOTE_OS_LIMIT,
+				  &regvalh);
+		if (ret < 0)
+			return ret;
+		ret = regmap_read(regmap, LM95245_REG_RW_COMMON_HYSTERESIS,
+				  &regvall);
+		if (ret < 0)
+			return ret;
+		*val = (regvalh - regvall) * 1000;
+		return 0;
+	case hwmon_temp_crit_hyst:
+		regh = channel ? LM95245_REG_RW_REMOTE_TCRIT_LIMIT :
+				 LM95245_REG_RW_LOCAL_OS_TCRIT_LIMIT;
+		ret = regmap_read(regmap, regh, &regvalh);
+		if (ret < 0)
+			return ret;
+		ret = regmap_read(regmap, LM95245_REG_RW_COMMON_HYSTERESIS,
+				  &regvall);
+		if (ret < 0)
+			return ret;
+		*val = (regvalh - regvall) * 1000;
+		return 0;
+	case hwmon_temp_type:
+		ret = regmap_read(regmap, LM95245_REG_RW_CONFIG2, &regvalh);
+		if (ret < 0)
+			return ret;
+		*val = (regvalh & CFG2_REMOTE_TT) ? 1 : 2;
+		return 0;
+	case hwmon_temp_offset:
+		ret = regmap_read(regmap, LM95245_REG_RW_REMOTE_OFFL,
+				  &regvall);
+		if (ret < 0)
+			return ret;
+		ret = regmap_read(regmap, LM95245_REG_RW_REMOTE_OFFH,
+				  &regvalh);
+		if (ret < 0)
+			return ret;
+		*val = temp_from_reg_signed(regvalh, regvall);
+		return 0;
+	case hwmon_temp_max_alarm:
+		ret = regmap_read(regmap, LM95245_REG_R_STATUS1, &regvalh);
+		if (ret < 0)
+			return ret;
+		*val = !!(regvalh & STATUS1_ROS);
+		return 0;
+	case hwmon_temp_crit_alarm:
+		ret = regmap_read(regmap, LM95245_REG_R_STATUS1, &regvalh);
+		if (ret < 0)
+			return ret;
+		*val = !!(regvalh & (channel ? STATUS1_RTCRIT : STATUS1_LOC));
+		return 0;
+	case hwmon_temp_fault:
+		ret = regmap_read(regmap, LM95245_REG_R_STATUS1, &regvalh);
+		if (ret < 0)
+			return ret;
+		*val = !!(regvalh & STATUS1_DIODE_FAULT);
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t show_crit_hyst(struct device *dev, struct device_attribute *attr,
-			char *buf)
+static int lm95245_write_temp(struct device *dev, u32 attr, int channel,
+			      long val)
 {
-	struct lm95245_data *data = lm95245_update_device(dev);
-	int index = to_sensor_dev_attr(attr)->index;
-	int hyst = data->regs[index] - data->regs[8];
-
-	return snprintf(buf, PAGE_SIZE - 1, "%d\n", hyst * 1000);
+	struct lm95245_data *data = dev_get_drvdata(dev);
+	struct regmap *regmap = data->regmap;
+	unsigned int regval;
+	int ret, reg;
+
+	switch (attr) {
+	case hwmon_temp_max:
+		val = clamp_val(val / 1000, 0, 255);
+		ret = regmap_write(regmap, LM95245_REG_RW_REMOTE_OS_LIMIT, val);
+		return ret;
+	case hwmon_temp_crit:
+		reg = channel ? LM95245_REG_RW_REMOTE_TCRIT_LIMIT :
+				LM95245_REG_RW_LOCAL_OS_TCRIT_LIMIT;
+		val = clamp_val(val / 1000, 0, channel ? 255 : 127);
+		ret = regmap_write(regmap, reg, val);
+		return ret;
+	case hwmon_temp_crit_hyst:
+		mutex_lock(&data->update_lock);
+		ret = regmap_read(regmap, LM95245_REG_RW_LOCAL_OS_TCRIT_LIMIT,
+				  &regval);
+		if (ret < 0) {
+			mutex_unlock(&data->update_lock);
+			return ret;
+		}
+		/* Clamp to reasonable range to prevent overflow */
+		val = clamp_val(val, -1000000, 1000000);
+		val = regval - val / 1000;
+		val = clamp_val(val, 0, 31);
+		ret = regmap_write(regmap, LM95245_REG_RW_COMMON_HYSTERESIS,
+				   val);
+		mutex_unlock(&data->update_lock);
+		return ret;
+	case hwmon_temp_offset:
+		val = clamp_val(val, -128000, 127875);
+		val = val * 256 / 1000;
+		mutex_lock(&data->update_lock);
+		ret = regmap_write(regmap, LM95245_REG_RW_REMOTE_OFFL,
+				   val & 0xe0);
+		if (ret < 0) {
+			mutex_unlock(&data->update_lock);
+			return ret;
+		}
+		ret = regmap_write(regmap, LM95245_REG_RW_REMOTE_OFFH,
+				   (val >> 8) & 0xff);
+		mutex_unlock(&data->update_lock);
+		return ret;
+	case hwmon_temp_type:
+		if (val != 1 && val != 2)
+			return -EINVAL;
+		ret = regmap_update_bits(regmap, LM95245_REG_RW_CONFIG2,
+					 CFG2_REMOTE_TT,
+					 val == 1 ? CFG2_REMOTE_TT : 0);
+		return ret;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t set_crit_hyst(struct device *dev, struct device_attribute *attr,
-			const char *buf, size_t count)
+static int lm95245_read_chip(struct device *dev, u32 attr, int channel,
+			     long *val)
 {
 	struct lm95245_data *data = dev_get_drvdata(dev);
-	int index = to_sensor_dev_attr(attr)->index;
-	struct i2c_client *client = data->client;
-	unsigned long val;
-	int hyst, limit;
-
-	if (kstrtoul(buf, 10, &val) < 0)
-		return -EINVAL;
-
-	mutex_lock(&data->update_lock);
-
-	limit = i2c_smbus_read_byte_data(client, lm95245_reg_address[index]);
-	hyst = limit - val / 1000;
-	hyst = clamp_val(hyst, 0, 31);
-	data->regs[8] = hyst;
 
-	/* shared crit hysteresis */
-	i2c_smbus_write_byte_data(client, LM95245_REG_RW_COMMON_HYSTERESIS,
-		hyst);
-
-	mutex_unlock(&data->update_lock);
-
-	return count;
+	switch (attr) {
+	case hwmon_chip_update_interval:
+		*val = data->interval;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t show_type(struct device *dev, struct device_attribute *attr,
-			 char *buf)
+static int lm95245_write_chip(struct device *dev, u32 attr, int channel,
+			      long val)
 {
 	struct lm95245_data *data = dev_get_drvdata(dev);
-
-	return snprintf(buf, PAGE_SIZE - 1,
-		data->config2 & CFG2_REMOTE_TT ? "1\n" : "2\n");
+	int ret;
+
+	switch (attr) {
+	case hwmon_chip_update_interval:
+		mutex_lock(&data->update_lock);
+		ret = lm95245_set_conversion_rate(data, val);
+		mutex_unlock(&data->update_lock);
+		return ret;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t set_type(struct device *dev, struct device_attribute *attr,
-			const char *buf, size_t count)
+static int lm95245_read(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long *val)
 {
-	struct lm95245_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
-	unsigned long val;
-
-	if (kstrtoul(buf, 10, &val) < 0)
-		return -EINVAL;
-	if (val != 1 && val != 2)
-		return -EINVAL;
-
-	mutex_lock(&data->update_lock);
-
-	if (val == 1)
-		data->config2 |= CFG2_REMOTE_TT;
-	else
-		data->config2 &= ~CFG2_REMOTE_TT;
-
-	data->valid = 0;
-
-	i2c_smbus_write_byte_data(client, LM95245_REG_RW_CONFIG2,
-				  data->config2);
-
-	mutex_unlock(&data->update_lock);
-
-	return count;
+	switch (type) {
+	case hwmon_chip:
+		return lm95245_read_chip(dev, attr, channel, val);
+	case hwmon_temp:
+		return lm95245_read_temp(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t show_alarm(struct device *dev, struct device_attribute *attr,
-			 char *buf)
+static int lm95245_write(struct device *dev, enum hwmon_sensor_types type,
+			 u32 attr, int channel, long val)
 {
-	struct lm95245_data *data = lm95245_update_device(dev);
-	int index = to_sensor_dev_attr(attr)->index;
-
-	return snprintf(buf, PAGE_SIZE - 1, "%d\n",
-			!!(data->regs[9] & index));
+	switch (type) {
+	case hwmon_chip:
+		return lm95245_write_chip(dev, attr, channel, val);
+	case hwmon_temp:
+		return lm95245_write_temp(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t show_interval(struct device *dev, struct device_attribute *attr,
-			     char *buf)
+static umode_t lm95245_temp_is_visible(const void *data, u32 attr, int channel)
 {
-	struct lm95245_data *data = lm95245_update_device(dev);
-
-	return snprintf(buf, PAGE_SIZE - 1, "%lu\n", data->interval);
+	switch (attr) {
+	case hwmon_temp_input:
+	case hwmon_temp_max_alarm:
+	case hwmon_temp_max_hyst:
+	case hwmon_temp_crit_alarm:
+	case hwmon_temp_fault:
+		return S_IRUGO;
+	case hwmon_temp_type:
+	case hwmon_temp_max:
+	case hwmon_temp_crit:
+	case hwmon_temp_offset:
+		return S_IRUGO | S_IWUSR;
+	case hwmon_temp_crit_hyst:
+		return (channel == 0) ? S_IRUGO | S_IWUSR : S_IRUGO;
+	default:
+		return 0;
+	}
 }
 
-static ssize_t set_interval(struct device *dev, struct device_attribute *attr,
-			    const char *buf, size_t count)
+static umode_t lm95245_is_visible(const void *data,
+				  enum hwmon_sensor_types type,
+				  u32 attr, int channel)
 {
-	struct lm95245_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
-	unsigned long val;
-
-	if (kstrtoul(buf, 10, &val) < 0)
-		return -EINVAL;
-
-	mutex_lock(&data->update_lock);
-
-	data->interval = lm95245_set_conversion_rate(client, val);
-
-	mutex_unlock(&data->update_lock);
-
-	return count;
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return S_IRUGO | S_IWUSR;
+		default:
+			return 0;
+		}
+	case hwmon_temp:
+		return lm95245_temp_is_visible(data, attr, channel);
+	default:
+		return 0;
+	}
 }
 
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, show_input, NULL, 0);
-static SENSOR_DEVICE_ATTR(temp1_crit, S_IWUSR | S_IRUGO, show_limit,
-		set_limit, 6);
-static SENSOR_DEVICE_ATTR(temp1_crit_hyst, S_IWUSR | S_IRUGO, show_crit_hyst,
-		set_crit_hyst, 6);
-static SENSOR_DEVICE_ATTR(temp1_crit_alarm, S_IRUGO, show_alarm, NULL,
-		STATUS1_LOC);
-
-static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, show_input, NULL, 2);
-static SENSOR_DEVICE_ATTR(temp2_crit, S_IWUSR | S_IRUGO, show_limit,
-		set_limit, 7);
-static SENSOR_DEVICE_ATTR(temp2_crit_hyst, S_IRUGO, show_crit_hyst, NULL, 7);
-static SENSOR_DEVICE_ATTR(temp2_crit_alarm, S_IRUGO, show_alarm, NULL,
-		STATUS1_RTCRIT);
-static SENSOR_DEVICE_ATTR(temp2_type, S_IWUSR | S_IRUGO, show_type,
-		set_type, 0);
-static SENSOR_DEVICE_ATTR(temp2_fault, S_IRUGO, show_alarm, NULL,
-		STATUS1_DIODE_FAULT);
-
-static DEVICE_ATTR(update_interval, S_IWUSR | S_IRUGO, show_interval,
-		set_interval);
-
-static struct attribute *lm95245_attrs[] = {
-	&sensor_dev_attr_temp1_input.dev_attr.attr,
-	&sensor_dev_attr_temp1_crit.dev_attr.attr,
-	&sensor_dev_attr_temp1_crit_hyst.dev_attr.attr,
-	&sensor_dev_attr_temp1_crit_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp2_input.dev_attr.attr,
-	&sensor_dev_attr_temp2_crit.dev_attr.attr,
-	&sensor_dev_attr_temp2_crit_hyst.dev_attr.attr,
-	&sensor_dev_attr_temp2_crit_alarm.dev_attr.attr,
-	&sensor_dev_attr_temp2_type.dev_attr.attr,
-	&sensor_dev_attr_temp2_fault.dev_attr.attr,
-	&dev_attr_update_interval.attr,
-	NULL
-};
-ATTRIBUTE_GROUPS(lm95245);
-
 /* Return 0 if detection is successful, -ENODEV otherwise */
 static int lm95245_detect(struct i2c_client *new_client,
 			  struct i2c_board_info *info)
@@ -453,44 +487,130 @@ static int lm95245_detect(struct i2c_client *new_client,
 	return 0;
 }
 
-static void lm95245_init_client(struct i2c_client *client,
-				struct lm95245_data *data)
+static int lm95245_init_client(struct lm95245_data *data)
 {
-	data->interval = lm95245_read_conversion_rate(client);
-
-	data->config1 = i2c_smbus_read_byte_data(client,
-		LM95245_REG_RW_CONFIG1);
-	data->config2 = i2c_smbus_read_byte_data(client,
-		LM95245_REG_RW_CONFIG2);
-
-	if (data->config1 & CFG_STOP) {
-		/* Clear the standby bit */
-		data->config1 &= ~CFG_STOP;
-		i2c_smbus_write_byte_data(client, LM95245_REG_RW_CONFIG1,
-			data->config1);
+	int ret;
+
+	ret = lm95245_read_conversion_rate(data);
+	if (ret < 0)
+		return ret;
+
+	return regmap_update_bits(data->regmap, LM95245_REG_RW_CONFIG1,
+				  CFG_STOP, 0);
+}
+
+static bool lm95245_is_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case LM95245_REG_RW_CONFIG1:
+	case LM95245_REG_RW_CONVERS_RATE:
+	case LM95245_REG_W_ONE_SHOT:
+	case LM95245_REG_RW_CONFIG2:
+	case LM95245_REG_RW_REMOTE_OFFH:
+	case LM95245_REG_RW_REMOTE_OFFL:
+	case LM95245_REG_RW_REMOTE_OS_LIMIT:
+	case LM95245_REG_RW_LOCAL_OS_TCRIT_LIMIT:
+	case LM95245_REG_RW_REMOTE_TCRIT_LIMIT:
+	case LM95245_REG_RW_COMMON_HYSTERESIS:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool lm95245_is_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case LM95245_REG_R_STATUS1:
+	case LM95245_REG_R_STATUS2:
+	case LM95245_REG_R_LOCAL_TEMPH_S:
+	case LM95245_REG_R_LOCAL_TEMPL_S:
+	case LM95245_REG_R_REMOTE_TEMPH_S:
+	case LM95245_REG_R_REMOTE_TEMPL_S:
+	case LM95245_REG_R_REMOTE_TEMPH_U:
+	case LM95245_REG_R_REMOTE_TEMPL_U:
+		return true;
+	default:
+		return false;
 	}
 }
 
+static const struct regmap_config lm95245_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.writeable_reg = lm95245_is_writeable_reg,
+	.volatile_reg = lm95245_is_volatile_reg,
+	.cache_type = REGCACHE_RBTREE,
+	.use_single_rw = true,
+};
+
+static const u32 lm95245_chip_config[] = {
+	HWMON_C_UPDATE_INTERVAL,
+	0
+};
+
+static const struct hwmon_channel_info lm95245_chip = {
+	.type = hwmon_chip,
+	.config = lm95245_chip_config,
+};
+
+static const u32 lm95245_temp_config[] = {
+	HWMON_T_INPUT | HWMON_T_CRIT | HWMON_T_CRIT_HYST | HWMON_T_CRIT_ALARM,
+	HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MAX_HYST | HWMON_T_CRIT |
+		HWMON_T_CRIT_HYST | HWMON_T_FAULT | HWMON_T_MAX_ALARM |
+		HWMON_T_CRIT_ALARM | HWMON_T_TYPE | HWMON_T_OFFSET,
+	0
+};
+
+static const struct hwmon_channel_info lm95245_temp = {
+	.type = hwmon_temp,
+	.config = lm95245_temp_config,
+};
+
+static const struct hwmon_channel_info *lm95245_info[] = {
+	&lm95245_chip,
+	&lm95245_temp,
+	NULL
+};
+
+static const struct hwmon_ops lm95245_hwmon_ops = {
+	.is_visible = lm95245_is_visible,
+	.read = lm95245_read,
+	.write = lm95245_write,
+};
+
+static const struct hwmon_chip_info lm95245_chip_info = {
+	.ops = &lm95245_hwmon_ops,
+	.info = lm95245_info,
+};
+
 static int lm95245_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
 	struct device *dev = &client->dev;
 	struct lm95245_data *data;
 	struct device *hwmon_dev;
+	int ret;
 
 	data = devm_kzalloc(dev, sizeof(struct lm95245_data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
-	data->client = client;
+	data->regmap = devm_regmap_init_i2c(client, &lm95245_regmap_config);
+	if (IS_ERR(data->regmap))
+		return PTR_ERR(data->regmap);
+
 	mutex_init(&data->update_lock);
 
 	/* Initialize the LM95245 chip */
-	lm95245_init_client(client, data);
-
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
-							   data,
-							   lm95245_groups);
+	ret = lm95245_init_client(data);
+	if (ret < 0)
+		return ret;
+
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 data,
+							 &lm95245_chip_info,
+							 NULL);
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
diff --git a/drivers/hwmon/ltc4151.c b/drivers/hwmon/ltc4151.c
index c86a18402496..8445c9fd946b 100644
--- a/drivers/hwmon/ltc4151.c
+++ b/drivers/hwmon/ltc4151.c
@@ -30,6 +30,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/init.h>
 #include <linux/err.h>
 #include <linux/slab.h>
@@ -52,6 +53,7 @@ struct ltc4151_data {
 	struct mutex update_lock;
 	bool valid;
 	unsigned long last_updated; /* in jiffies */
+	unsigned int shunt; /* in micro ohms */
 
 	/* Registers */
 	u8 regs[6];
@@ -111,9 +113,9 @@ static int ltc4151_get_value(struct ltc4151_data *data, u8 reg)
 	case LTC4151_SENSE_H:
 		/*
 		 * 20uV resolution. Convert to current as measured with
-		 * an 1 mOhm sense resistor, in mA.
+		 * a given sense resistor, in mA.
 		 */
-		val = val * 20;
+		val = val * 20 * 1000 / data->shunt;
 		break;
 	case LTC4151_VIN_H:
 		/* 25 mV per increment */
@@ -176,6 +178,7 @@ static int ltc4151_probe(struct i2c_client *client,
 	struct device *dev = &client->dev;
 	struct ltc4151_data *data;
 	struct device *hwmon_dev;
+	u32 shunt;
 
 	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA))
 		return -ENODEV;
@@ -184,6 +187,15 @@ static int ltc4151_probe(struct i2c_client *client,
 	if (!data)
 		return -ENOMEM;
 
+	if (of_property_read_u32(client->dev.of_node,
+				 "shunt-resistor-micro-ohms", &shunt))
+		shunt = 1000; /* 1 mOhm if not set via DT */
+
+	if (shunt == 0)
+		return -EINVAL;
+
+	data->shunt = shunt;
+
 	data->client = client;
 	mutex_init(&data->update_lock);
 
@@ -199,10 +211,16 @@ static const struct i2c_device_id ltc4151_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, ltc4151_id);
 
+static const struct of_device_id ltc4151_match[] = {
+	{ .compatible = "lltc,ltc4151" },
+	{},
+};
+
 /* This is the driver that will be inserted */
 static struct i2c_driver ltc4151_driver = {
 	.driver = {
 		.name	= "ltc4151",
+		.of_match_table = of_match_ptr(ltc4151_match),
 	},
 	.probe		= ltc4151_probe,
 	.id_table	= ltc4151_id,
diff --git a/drivers/hwmon/ltc4245.c b/drivers/hwmon/ltc4245.c
index 681b5b7b3c3b..4680d89556ce 100644
--- a/drivers/hwmon/ltc4245.c
+++ b/drivers/hwmon/ltc4245.c
@@ -16,6 +16,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/init.h>
+#include <linux/bitops.h>
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/i2c.h>
@@ -53,8 +54,6 @@ enum ltc4245_cmd {
 struct ltc4245_data {
 	struct i2c_client *client;
 
-	const struct attribute_group *groups[3];
-
 	struct mutex update_lock;
 	bool valid;
 	unsigned long last_updated; /* in jiffies */
@@ -162,7 +161,7 @@ static struct ltc4245_data *ltc4245_update_device(struct device *dev)
 		ltc4245_update_gpios(dev);
 
 		data->last_updated = jiffies;
-		data->valid = 1;
+		data->valid = true;
 	}
 
 	mutex_unlock(&data->update_lock);
@@ -256,213 +255,204 @@ static unsigned int ltc4245_get_current(struct device *dev, u8 reg)
 	return curr;
 }
 
-static ssize_t ltc4245_show_voltage(struct device *dev,
-				    struct device_attribute *da,
-				    char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
-	const int voltage = ltc4245_get_voltage(dev, attr->index);
+/* Map from voltage channel index to voltage register */
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", voltage);
-}
+static const s8 ltc4245_in_regs[] = {
+	LTC4245_12VIN, LTC4245_5VIN, LTC4245_3VIN, LTC4245_VEEIN,
+	LTC4245_12VOUT, LTC4245_5VOUT, LTC4245_3VOUT, LTC4245_VEEOUT,
+};
+
+/* Map from current channel index to current register */
 
-static ssize_t ltc4245_show_current(struct device *dev,
-				    struct device_attribute *da,
-				    char *buf)
+static const s8 ltc4245_curr_regs[] = {
+	LTC4245_12VSENSE, LTC4245_5VSENSE, LTC4245_3VSENSE, LTC4245_VEESENSE,
+};
+
+static int ltc4245_read_curr(struct device *dev, u32 attr, int channel,
+			     long *val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
-	const unsigned int curr = ltc4245_get_current(dev, attr->index);
+	struct ltc4245_data *data = ltc4245_update_device(dev);
 
-	return snprintf(buf, PAGE_SIZE, "%u\n", curr);
+	switch (attr) {
+	case hwmon_curr_input:
+		*val = ltc4245_get_current(dev, ltc4245_curr_regs[channel]);
+		return 0;
+	case hwmon_curr_max_alarm:
+		*val = !!(data->cregs[LTC4245_FAULT1] & BIT(channel + 4));
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t ltc4245_show_power(struct device *dev,
-				  struct device_attribute *da,
-				  char *buf)
+static int ltc4245_read_in(struct device *dev, u32 attr, int channel, long *val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
-	const unsigned int curr = ltc4245_get_current(dev, attr->index);
-	const int output_voltage = ltc4245_get_voltage(dev, attr->index+1);
+	struct ltc4245_data *data = ltc4245_update_device(dev);
 
-	/* current in mA * voltage in mV == power in uW */
-	const unsigned int power = abs(output_voltage * curr);
+	switch (attr) {
+	case hwmon_in_input:
+		if (channel < 8) {
+			*val = ltc4245_get_voltage(dev,
+						ltc4245_in_regs[channel]);
+		} else {
+			int regval = data->gpios[channel - 8];
+
+			if (regval < 0)
+				return regval;
+			*val = regval * 10;
+		}
+		return 0;
+	case hwmon_in_min_alarm:
+		if (channel < 4)
+			*val = !!(data->cregs[LTC4245_FAULT1] & BIT(channel));
+		else
+			*val = !!(data->cregs[LTC4245_FAULT2] &
+				  BIT(channel - 4));
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
 
-	return snprintf(buf, PAGE_SIZE, "%u\n", power);
+static int ltc4245_read_power(struct device *dev, u32 attr, int channel,
+			      long *val)
+{
+	unsigned long curr;
+	long voltage;
+
+	switch (attr) {
+	case hwmon_power_input:
+		(void)ltc4245_update_device(dev);
+		curr = ltc4245_get_current(dev, ltc4245_curr_regs[channel]);
+		voltage = ltc4245_get_voltage(dev, ltc4245_in_regs[channel]);
+		*val = abs(curr * voltage);
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t ltc4245_show_alarm(struct device *dev,
-					  struct device_attribute *da,
-					  char *buf)
+static int ltc4245_read(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long *val)
 {
-	struct sensor_device_attribute_2 *attr = to_sensor_dev_attr_2(da);
-	struct ltc4245_data *data = ltc4245_update_device(dev);
-	const u8 reg = data->cregs[attr->index];
-	const u32 mask = attr->nr;
 
-	return snprintf(buf, PAGE_SIZE, "%u\n", (reg & mask) ? 1 : 0);
+	switch (type) {
+	case hwmon_curr:
+		return ltc4245_read_curr(dev, attr, channel, val);
+	case hwmon_power:
+		return ltc4245_read_power(dev, attr, channel, val);
+	case hwmon_in:
+		return ltc4245_read_in(dev, attr, channel - 1, val);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t ltc4245_show_gpio(struct device *dev,
-				 struct device_attribute *da,
-				 char *buf)
+static umode_t ltc4245_is_visible(const void *_data,
+				  enum hwmon_sensor_types type,
+				  u32 attr, int channel)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
-	struct ltc4245_data *data = ltc4245_update_device(dev);
-	int val = data->gpios[attr->index];
+	const struct ltc4245_data *data = _data;
+
+	switch (type) {
+	case hwmon_in:
+		if (channel == 0)
+			return 0;
+		switch (attr) {
+		case hwmon_in_input:
+			if (channel > 9 && !data->use_extra_gpios)
+				return 0;
+			return S_IRUGO;
+		case hwmon_in_min_alarm:
+			if (channel > 8)
+				return 0;
+			return S_IRUGO;
+		default:
+			return 0;
+		}
+	case hwmon_curr:
+		switch (attr) {
+		case hwmon_curr_input:
+		case hwmon_curr_max_alarm:
+			return S_IRUGO;
+		default:
+			return 0;
+		}
+	case hwmon_power:
+		switch (attr) {
+		case hwmon_power_input:
+			return S_IRUGO;
+		default:
+			return 0;
+		}
+	default:
+		return 0;
+	}
+}
 
-	/* handle stale GPIO's */
-	if (val < 0)
-		return val;
+static const u32 ltc4245_in_config[] = {
+	HWMON_I_INPUT,			/* dummy, skipped in is_visible */
+	HWMON_I_INPUT | HWMON_I_MIN_ALARM,
+	HWMON_I_INPUT | HWMON_I_MIN_ALARM,
+	HWMON_I_INPUT | HWMON_I_MIN_ALARM,
+	HWMON_I_INPUT | HWMON_I_MIN_ALARM,
+	HWMON_I_INPUT | HWMON_I_MIN_ALARM,
+	HWMON_I_INPUT | HWMON_I_MIN_ALARM,
+	HWMON_I_INPUT | HWMON_I_MIN_ALARM,
+	HWMON_I_INPUT | HWMON_I_MIN_ALARM,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	0
+};
 
-	/* Convert to millivolts and print */
-	return snprintf(buf, PAGE_SIZE, "%u\n", val * 10);
-}
+static const struct hwmon_channel_info ltc4245_in = {
+	.type = hwmon_in,
+	.config = ltc4245_in_config,
+};
 
-/* Construct a sensor_device_attribute structure for each register */
-
-/* Input voltages */
-static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, ltc4245_show_voltage, NULL,
-			  LTC4245_12VIN);
-static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, ltc4245_show_voltage, NULL,
-			  LTC4245_5VIN);
-static SENSOR_DEVICE_ATTR(in3_input, S_IRUGO, ltc4245_show_voltage, NULL,
-			  LTC4245_3VIN);
-static SENSOR_DEVICE_ATTR(in4_input, S_IRUGO, ltc4245_show_voltage, NULL,
-			  LTC4245_VEEIN);
-
-/* Input undervoltage alarms */
-static SENSOR_DEVICE_ATTR_2(in1_min_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 0, LTC4245_FAULT1);
-static SENSOR_DEVICE_ATTR_2(in2_min_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 1, LTC4245_FAULT1);
-static SENSOR_DEVICE_ATTR_2(in3_min_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 2, LTC4245_FAULT1);
-static SENSOR_DEVICE_ATTR_2(in4_min_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 3, LTC4245_FAULT1);
-
-/* Currents (via sense resistor) */
-static SENSOR_DEVICE_ATTR(curr1_input, S_IRUGO, ltc4245_show_current, NULL,
-			  LTC4245_12VSENSE);
-static SENSOR_DEVICE_ATTR(curr2_input, S_IRUGO, ltc4245_show_current, NULL,
-			  LTC4245_5VSENSE);
-static SENSOR_DEVICE_ATTR(curr3_input, S_IRUGO, ltc4245_show_current, NULL,
-			  LTC4245_3VSENSE);
-static SENSOR_DEVICE_ATTR(curr4_input, S_IRUGO, ltc4245_show_current, NULL,
-			  LTC4245_VEESENSE);
-
-/* Overcurrent alarms */
-static SENSOR_DEVICE_ATTR_2(curr1_max_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 4, LTC4245_FAULT1);
-static SENSOR_DEVICE_ATTR_2(curr2_max_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 5, LTC4245_FAULT1);
-static SENSOR_DEVICE_ATTR_2(curr3_max_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 6, LTC4245_FAULT1);
-static SENSOR_DEVICE_ATTR_2(curr4_max_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 7, LTC4245_FAULT1);
-
-/* Output voltages */
-static SENSOR_DEVICE_ATTR(in5_input, S_IRUGO, ltc4245_show_voltage, NULL,
-			  LTC4245_12VOUT);
-static SENSOR_DEVICE_ATTR(in6_input, S_IRUGO, ltc4245_show_voltage, NULL,
-			  LTC4245_5VOUT);
-static SENSOR_DEVICE_ATTR(in7_input, S_IRUGO, ltc4245_show_voltage, NULL,
-			  LTC4245_3VOUT);
-static SENSOR_DEVICE_ATTR(in8_input, S_IRUGO, ltc4245_show_voltage, NULL,
-			  LTC4245_VEEOUT);
-
-/* Power Bad alarms */
-static SENSOR_DEVICE_ATTR_2(in5_min_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 0, LTC4245_FAULT2);
-static SENSOR_DEVICE_ATTR_2(in6_min_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 1, LTC4245_FAULT2);
-static SENSOR_DEVICE_ATTR_2(in7_min_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 2, LTC4245_FAULT2);
-static SENSOR_DEVICE_ATTR_2(in8_min_alarm, S_IRUGO, ltc4245_show_alarm, NULL,
-			    1 << 3, LTC4245_FAULT2);
-
-/* GPIO voltages */
-static SENSOR_DEVICE_ATTR(in9_input, S_IRUGO, ltc4245_show_gpio, NULL, 0);
-static SENSOR_DEVICE_ATTR(in10_input, S_IRUGO, ltc4245_show_gpio, NULL, 1);
-static SENSOR_DEVICE_ATTR(in11_input, S_IRUGO, ltc4245_show_gpio, NULL, 2);
-
-/* Power Consumption (virtual) */
-static SENSOR_DEVICE_ATTR(power1_input, S_IRUGO, ltc4245_show_power, NULL,
-			  LTC4245_12VSENSE);
-static SENSOR_DEVICE_ATTR(power2_input, S_IRUGO, ltc4245_show_power, NULL,
-			  LTC4245_5VSENSE);
-static SENSOR_DEVICE_ATTR(power3_input, S_IRUGO, ltc4245_show_power, NULL,
-			  LTC4245_3VSENSE);
-static SENSOR_DEVICE_ATTR(power4_input, S_IRUGO, ltc4245_show_power, NULL,
-			  LTC4245_VEESENSE);
+static const u32 ltc4245_curr_config[] = {
+	HWMON_C_INPUT | HWMON_C_MAX_ALARM,
+	HWMON_C_INPUT | HWMON_C_MAX_ALARM,
+	HWMON_C_INPUT | HWMON_C_MAX_ALARM,
+	HWMON_C_INPUT | HWMON_C_MAX_ALARM,
+	0
+};
 
-/*
- * Finally, construct an array of pointers to members of the above objects,
- * as required for sysfs_create_group()
- */
-static struct attribute *ltc4245_std_attributes[] = {
-	&sensor_dev_attr_in1_input.dev_attr.attr,
-	&sensor_dev_attr_in2_input.dev_attr.attr,
-	&sensor_dev_attr_in3_input.dev_attr.attr,
-	&sensor_dev_attr_in4_input.dev_attr.attr,
-
-	&sensor_dev_attr_in1_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_in2_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_in3_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_in4_min_alarm.dev_attr.attr,
-
-	&sensor_dev_attr_curr1_input.dev_attr.attr,
-	&sensor_dev_attr_curr2_input.dev_attr.attr,
-	&sensor_dev_attr_curr3_input.dev_attr.attr,
-	&sensor_dev_attr_curr4_input.dev_attr.attr,
-
-	&sensor_dev_attr_curr1_max_alarm.dev_attr.attr,
-	&sensor_dev_attr_curr2_max_alarm.dev_attr.attr,
-	&sensor_dev_attr_curr3_max_alarm.dev_attr.attr,
-	&sensor_dev_attr_curr4_max_alarm.dev_attr.attr,
-
-	&sensor_dev_attr_in5_input.dev_attr.attr,
-	&sensor_dev_attr_in6_input.dev_attr.attr,
-	&sensor_dev_attr_in7_input.dev_attr.attr,
-	&sensor_dev_attr_in8_input.dev_attr.attr,
-
-	&sensor_dev_attr_in5_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_in6_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_in7_min_alarm.dev_attr.attr,
-	&sensor_dev_attr_in8_min_alarm.dev_attr.attr,
-
-	&sensor_dev_attr_in9_input.dev_attr.attr,
-
-	&sensor_dev_attr_power1_input.dev_attr.attr,
-	&sensor_dev_attr_power2_input.dev_attr.attr,
-	&sensor_dev_attr_power3_input.dev_attr.attr,
-	&sensor_dev_attr_power4_input.dev_attr.attr,
-
-	NULL,
+static const struct hwmon_channel_info ltc4245_curr = {
+	.type = hwmon_curr,
+	.config = ltc4245_curr_config,
 };
 
-static struct attribute *ltc4245_gpio_attributes[] = {
-	&sensor_dev_attr_in10_input.dev_attr.attr,
-	&sensor_dev_attr_in11_input.dev_attr.attr,
-	NULL,
+static const u32 ltc4245_power_config[] = {
+	HWMON_P_INPUT,
+	HWMON_P_INPUT,
+	HWMON_P_INPUT,
+	HWMON_P_INPUT,
+	0
 };
 
-static const struct attribute_group ltc4245_std_group = {
-	.attrs = ltc4245_std_attributes,
+static const struct hwmon_channel_info ltc4245_power = {
+	.type = hwmon_power,
+	.config = ltc4245_power_config,
 };
 
-static const struct attribute_group ltc4245_gpio_group = {
-	.attrs = ltc4245_gpio_attributes,
+static const struct hwmon_channel_info *ltc4245_info[] = {
+	&ltc4245_in,
+	&ltc4245_curr,
+	&ltc4245_power,
+	NULL
 };
 
-static void ltc4245_sysfs_add_groups(struct ltc4245_data *data)
-{
-	/* standard sysfs attributes */
-	data->groups[0] = &ltc4245_std_group;
+static const struct hwmon_ops ltc4245_hwmon_ops = {
+	.is_visible = ltc4245_is_visible,
+	.read = ltc4245_read,
+};
 
-	/* if we're using the extra gpio support, register it's attributes */
-	if (data->use_extra_gpios)
-		data->groups[1] = &ltc4245_gpio_group;
-}
+static const struct hwmon_chip_info ltc4245_chip_info = {
+	.ops = &ltc4245_hwmon_ops,
+	.info = ltc4245_info,
+};
 
 static bool ltc4245_use_extra_gpios(struct i2c_client *client)
 {
@@ -502,12 +492,10 @@ static int ltc4245_probe(struct i2c_client *client,
 	i2c_smbus_write_byte_data(client, LTC4245_FAULT1, 0x00);
 	i2c_smbus_write_byte_data(client, LTC4245_FAULT2, 0x00);
 
-	/* Add sysfs hooks */
-	ltc4245_sysfs_add_groups(data);
-
-	hwmon_dev = devm_hwmon_device_register_with_groups(&client->dev,
-							   client->name, data,
-							   data->groups);
+	hwmon_dev = devm_hwmon_device_register_with_info(&client->dev,
+							 client->name, data,
+							 &ltc4245_chip_info,
+							 NULL);
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
diff --git a/drivers/hwmon/max31790.c b/drivers/hwmon/max31790.c
index 69c0ac80a946..bef84e085973 100644
--- a/drivers/hwmon/max31790.c
+++ b/drivers/hwmon/max31790.c
@@ -17,7 +17,6 @@
 
 #include <linux/err.h>
 #include <linux/hwmon.h>
-#include <linux/hwmon-sysfs.h>
 #include <linux/i2c.h>
 #include <linux/init.h>
 #include <linux/jiffies.h>
@@ -169,362 +168,288 @@ static u8 bits_for_tach_period(int rpm)
 	return bits;
 }
 
-static ssize_t get_fan(struct device *dev,
-		       struct device_attribute *devattr, char *buf)
+static int max31790_read_fan(struct device *dev, u32 attr, int channel,
+			     long *val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct max31790_data *data = max31790_update_device(dev);
 	int sr, rpm;
 
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
-	sr = get_tach_period(data->fan_dynamics[attr->index]);
-	rpm = RPM_FROM_REG(data->tach[attr->index], sr);
-
-	return sprintf(buf, "%d\n", rpm);
-}
-
-static ssize_t get_fan_target(struct device *dev,
-			      struct device_attribute *devattr, char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct max31790_data *data = max31790_update_device(dev);
-	int sr, rpm;
-
-	if (IS_ERR(data))
-		return PTR_ERR(data);
-
-	sr = get_tach_period(data->fan_dynamics[attr->index]);
-	rpm = RPM_FROM_REG(data->target_count[attr->index], sr);
-
-	return sprintf(buf, "%d\n", rpm);
+	switch (attr) {
+	case hwmon_fan_input:
+		sr = get_tach_period(data->fan_dynamics[channel]);
+		rpm = RPM_FROM_REG(data->tach[channel], sr);
+		*val = rpm;
+		return 0;
+	case hwmon_fan_target:
+		sr = get_tach_period(data->fan_dynamics[channel]);
+		rpm = RPM_FROM_REG(data->target_count[channel], sr);
+		*val = rpm;
+		return 0;
+	case hwmon_fan_fault:
+		*val = !!(data->fault_status & (1 << channel));
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t set_fan_target(struct device *dev,
-			      struct device_attribute *devattr,
-			      const char *buf, size_t count)
+static int max31790_write_fan(struct device *dev, u32 attr, int channel,
+			      long val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct max31790_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
+	int target_count;
+	int err = 0;
 	u8 bits;
 	int sr;
-	int target_count;
-	unsigned long rpm;
-	int err;
-
-	err = kstrtoul(buf, 10, &rpm);
-	if (err)
-		return err;
 
 	mutex_lock(&data->update_lock);
 
-	rpm = clamp_val(rpm, FAN_RPM_MIN, FAN_RPM_MAX);
-	bits = bits_for_tach_period(rpm);
-	data->fan_dynamics[attr->index] =
-			((data->fan_dynamics[attr->index]
-			  & ~MAX31790_FAN_DYN_SR_MASK)
-			 | (bits << MAX31790_FAN_DYN_SR_SHIFT));
-	err = i2c_smbus_write_byte_data(client,
-			MAX31790_REG_FAN_DYNAMICS(attr->index),
-			data->fan_dynamics[attr->index]);
-
-	if (err < 0) {
-		mutex_unlock(&data->update_lock);
-		return err;
+	switch (attr) {
+	case hwmon_fan_target:
+		val = clamp_val(val, FAN_RPM_MIN, FAN_RPM_MAX);
+		bits = bits_for_tach_period(val);
+		data->fan_dynamics[channel] =
+			((data->fan_dynamics[channel] &
+			  ~MAX31790_FAN_DYN_SR_MASK) |
+			 (bits << MAX31790_FAN_DYN_SR_SHIFT));
+		err = i2c_smbus_write_byte_data(client,
+					MAX31790_REG_FAN_DYNAMICS(channel),
+					data->fan_dynamics[channel]);
+		if (err < 0)
+			break;
+
+		sr = get_tach_period(data->fan_dynamics[channel]);
+		target_count = RPM_TO_REG(val, sr);
+		target_count = clamp_val(target_count, 0x1, 0x7FF);
+
+		data->target_count[channel] = target_count << 5;
+
+		err = i2c_smbus_write_word_swapped(client,
+					MAX31790_REG_TARGET_COUNT(channel),
+					data->target_count[channel]);
+		break;
+	default:
+		err = -EOPNOTSUPP;
+		break;
 	}
 
-	sr = get_tach_period(data->fan_dynamics[attr->index]);
-	target_count = RPM_TO_REG(rpm, sr);
-	target_count = clamp_val(target_count, 0x1, 0x7FF);
-
-	data->target_count[attr->index] = target_count << 5;
-
-	err = i2c_smbus_write_word_swapped(client,
-			MAX31790_REG_TARGET_COUNT(attr->index),
-			data->target_count[attr->index]);
-
 	mutex_unlock(&data->update_lock);
 
-	if (err < 0)
-		return err;
+	return err;
+}
 
-	return count;
+static umode_t max31790_fan_is_visible(const void *_data, u32 attr, int channel)
+{
+	const struct max31790_data *data = _data;
+	u8 fan_config = data->fan_config[channel % NR_CHANNEL];
+
+	switch (attr) {
+	case hwmon_fan_input:
+	case hwmon_fan_fault:
+		if (channel < NR_CHANNEL ||
+		    (fan_config & MAX31790_FAN_CFG_TACH_INPUT))
+			return S_IRUGO;
+		return 0;
+	case hwmon_fan_target:
+		if (channel < NR_CHANNEL &&
+		    !(fan_config & MAX31790_FAN_CFG_TACH_INPUT))
+			return S_IRUGO | S_IWUSR;
+		return 0;
+	default:
+		return 0;
+	}
 }
 
-static ssize_t get_pwm(struct device *dev,
-		       struct device_attribute *devattr, char *buf)
+static int max31790_read_pwm(struct device *dev, u32 attr, int channel,
+			     long *val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct max31790_data *data = max31790_update_device(dev);
-	int pwm;
+	u8 fan_config = data->fan_config[channel];
 
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
-	pwm = data->pwm[attr->index] >> 8;
-
-	return sprintf(buf, "%d\n", pwm);
+	switch (attr) {
+	case hwmon_pwm_input:
+		*val = data->pwm[channel] >> 8;
+		return 0;
+	case hwmon_pwm_enable:
+		if (fan_config & MAX31790_FAN_CFG_RPM_MODE)
+			*val = 2;
+		else if (fan_config & MAX31790_FAN_CFG_TACH_INPUT_EN)
+			*val = 1;
+		else
+			*val = 0;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t set_pwm(struct device *dev,
-		       struct device_attribute *devattr,
-		       const char *buf, size_t count)
+static int max31790_write_pwm(struct device *dev, u32 attr, int channel,
+			      long val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct max31790_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
-	unsigned long pwm;
-	int err;
-
-	err = kstrtoul(buf, 10, &pwm);
-	if (err)
-		return err;
-
-	if (pwm > 255)
-		return -EINVAL;
+	u8 fan_config;
+	int err = 0;
 
 	mutex_lock(&data->update_lock);
 
-	data->pwm[attr->index] = pwm << 8;
-	err = i2c_smbus_write_word_swapped(client,
-			MAX31790_REG_PWMOUT(attr->index),
-			data->pwm[attr->index]);
+	switch (attr) {
+	case hwmon_pwm_input:
+		if (val < 0 || val > 255) {
+			err = -EINVAL;
+			break;
+		}
+		data->pwm[channel] = val << 8;
+		err = i2c_smbus_write_word_swapped(client,
+						   MAX31790_REG_PWMOUT(channel),
+						   val);
+		break;
+	case hwmon_pwm_enable:
+		fan_config = data->fan_config[channel];
+		if (val == 0) {
+			fan_config &= ~(MAX31790_FAN_CFG_TACH_INPUT_EN |
+					MAX31790_FAN_CFG_RPM_MODE);
+		} else if (val == 1) {
+			fan_config = (fan_config |
+				      MAX31790_FAN_CFG_TACH_INPUT_EN) &
+				     ~MAX31790_FAN_CFG_RPM_MODE;
+		} else if (val == 2) {
+			fan_config |= MAX31790_FAN_CFG_TACH_INPUT_EN |
+				      MAX31790_FAN_CFG_RPM_MODE;
+		} else {
+			err = -EINVAL;
+			break;
+		}
+		data->fan_config[channel] = fan_config;
+		err = i2c_smbus_write_byte_data(client,
+					MAX31790_REG_FAN_CONFIG(channel),
+					fan_config);
+		break;
+	default:
+		err = -EOPNOTSUPP;
+		break;
+	}
 
 	mutex_unlock(&data->update_lock);
 
-	if (err < 0)
-		return err;
-
-	return count;
+	return err;
 }
 
-static ssize_t get_pwm_enable(struct device *dev,
-			      struct device_attribute *devattr, char *buf)
+static umode_t max31790_pwm_is_visible(const void *_data, u32 attr, int channel)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct max31790_data *data = max31790_update_device(dev);
-	int mode;
-
-	if (IS_ERR(data))
-		return PTR_ERR(data);
-
-	if (data->fan_config[attr->index] & MAX31790_FAN_CFG_RPM_MODE)
-		mode = 2;
-	else if (data->fan_config[attr->index] & MAX31790_FAN_CFG_TACH_INPUT_EN)
-		mode = 1;
-	else
-		mode = 0;
-
-	return sprintf(buf, "%d\n", mode);
+	const struct max31790_data *data = _data;
+	u8 fan_config = data->fan_config[channel];
+
+	switch (attr) {
+	case hwmon_pwm_input:
+	case hwmon_pwm_enable:
+		if (!(fan_config & MAX31790_FAN_CFG_TACH_INPUT))
+			return S_IRUGO | S_IWUSR;
+		return 0;
+	default:
+		return 0;
+	}
 }
 
-static ssize_t set_pwm_enable(struct device *dev,
-			      struct device_attribute *devattr,
-			      const char *buf, size_t count)
+static int max31790_read(struct device *dev, enum hwmon_sensor_types type,
+			 u32 attr, int channel, long *val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct max31790_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
-	unsigned long mode;
-	int err;
-
-	err = kstrtoul(buf, 10, &mode);
-	if (err)
-		return err;
-
-	switch (mode) {
-	case 0:
-		data->fan_config[attr->index] =
-			data->fan_config[attr->index]
-			& ~(MAX31790_FAN_CFG_TACH_INPUT_EN
-			    | MAX31790_FAN_CFG_RPM_MODE);
-		break;
-	case 1:
-		data->fan_config[attr->index] =
-			(data->fan_config[attr->index]
-			 | MAX31790_FAN_CFG_TACH_INPUT_EN)
-			& ~MAX31790_FAN_CFG_RPM_MODE;
-		break;
-	case 2:
-		data->fan_config[attr->index] =
-			data->fan_config[attr->index]
-			| MAX31790_FAN_CFG_TACH_INPUT_EN
-			| MAX31790_FAN_CFG_RPM_MODE;
-		break;
+	switch (type) {
+	case hwmon_fan:
+		return max31790_read_fan(dev, attr, channel, val);
+	case hwmon_pwm:
+		return max31790_read_pwm(dev, attr, channel, val);
 	default:
-		return -EINVAL;
+		return -EOPNOTSUPP;
 	}
-
-	mutex_lock(&data->update_lock);
-
-	err = i2c_smbus_write_byte_data(client,
-			MAX31790_REG_FAN_CONFIG(attr->index),
-			data->fan_config[attr->index]);
-
-	mutex_unlock(&data->update_lock);
-
-	if (err < 0)
-		return err;
-
-	return count;
 }
 
-static ssize_t get_fan_fault(struct device *dev,
-			     struct device_attribute *devattr, char *buf)
+static int max31790_write(struct device *dev, enum hwmon_sensor_types type,
+			  u32 attr, int channel, long val)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct max31790_data *data = max31790_update_device(dev);
-	int fault;
-
-	if (IS_ERR(data))
-		return PTR_ERR(data);
-
-	fault = !!(data->fault_status & (1 << attr->index));
+	switch (type) {
+	case hwmon_fan:
+		return max31790_write_fan(dev, attr, channel, val);
+	case hwmon_pwm:
+		return max31790_write_pwm(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
 
-	return sprintf(buf, "%d\n", fault);
+static umode_t max31790_is_visible(const void *data,
+				   enum hwmon_sensor_types type,
+				   u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_fan:
+		return max31790_fan_is_visible(data, attr, channel);
+	case hwmon_pwm:
+		return max31790_pwm_is_visible(data, attr, channel);
+	default:
+		return 0;
+	}
 }
 
-static SENSOR_DEVICE_ATTR(fan1_input, S_IRUGO, get_fan, NULL, 0);
-static SENSOR_DEVICE_ATTR(fan2_input, S_IRUGO, get_fan, NULL, 1);
-static SENSOR_DEVICE_ATTR(fan3_input, S_IRUGO, get_fan, NULL, 2);
-static SENSOR_DEVICE_ATTR(fan4_input, S_IRUGO, get_fan, NULL, 3);
-static SENSOR_DEVICE_ATTR(fan5_input, S_IRUGO, get_fan, NULL, 4);
-static SENSOR_DEVICE_ATTR(fan6_input, S_IRUGO, get_fan, NULL, 5);
-
-static SENSOR_DEVICE_ATTR(fan1_fault, S_IRUGO, get_fan_fault, NULL, 0);
-static SENSOR_DEVICE_ATTR(fan2_fault, S_IRUGO, get_fan_fault, NULL, 1);
-static SENSOR_DEVICE_ATTR(fan3_fault, S_IRUGO, get_fan_fault, NULL, 2);
-static SENSOR_DEVICE_ATTR(fan4_fault, S_IRUGO, get_fan_fault, NULL, 3);
-static SENSOR_DEVICE_ATTR(fan5_fault, S_IRUGO, get_fan_fault, NULL, 4);
-static SENSOR_DEVICE_ATTR(fan6_fault, S_IRUGO, get_fan_fault, NULL, 5);
-
-static SENSOR_DEVICE_ATTR(fan7_input, S_IRUGO, get_fan, NULL, 6);
-static SENSOR_DEVICE_ATTR(fan8_input, S_IRUGO, get_fan, NULL, 7);
-static SENSOR_DEVICE_ATTR(fan9_input, S_IRUGO, get_fan, NULL, 8);
-static SENSOR_DEVICE_ATTR(fan10_input, S_IRUGO, get_fan, NULL, 9);
-static SENSOR_DEVICE_ATTR(fan11_input, S_IRUGO, get_fan, NULL, 10);
-static SENSOR_DEVICE_ATTR(fan12_input, S_IRUGO, get_fan, NULL, 11);
-
-static SENSOR_DEVICE_ATTR(fan7_fault, S_IRUGO, get_fan_fault, NULL, 6);
-static SENSOR_DEVICE_ATTR(fan8_fault, S_IRUGO, get_fan_fault, NULL, 7);
-static SENSOR_DEVICE_ATTR(fan9_fault, S_IRUGO, get_fan_fault, NULL, 8);
-static SENSOR_DEVICE_ATTR(fan10_fault, S_IRUGO, get_fan_fault, NULL, 9);
-static SENSOR_DEVICE_ATTR(fan11_fault, S_IRUGO, get_fan_fault, NULL, 10);
-static SENSOR_DEVICE_ATTR(fan12_fault, S_IRUGO, get_fan_fault, NULL, 11);
-
-static SENSOR_DEVICE_ATTR(fan1_target, S_IWUSR | S_IRUGO,
-		get_fan_target, set_fan_target, 0);
-static SENSOR_DEVICE_ATTR(fan2_target, S_IWUSR | S_IRUGO,
-		get_fan_target, set_fan_target, 1);
-static SENSOR_DEVICE_ATTR(fan3_target, S_IWUSR | S_IRUGO,
-		get_fan_target, set_fan_target, 2);
-static SENSOR_DEVICE_ATTR(fan4_target, S_IWUSR | S_IRUGO,
-		get_fan_target, set_fan_target, 3);
-static SENSOR_DEVICE_ATTR(fan5_target, S_IWUSR | S_IRUGO,
-		get_fan_target, set_fan_target, 4);
-static SENSOR_DEVICE_ATTR(fan6_target, S_IWUSR | S_IRUGO,
-		get_fan_target, set_fan_target, 5);
-
-static SENSOR_DEVICE_ATTR(pwm1, S_IWUSR | S_IRUGO, get_pwm, set_pwm, 0);
-static SENSOR_DEVICE_ATTR(pwm2, S_IWUSR | S_IRUGO, get_pwm, set_pwm, 1);
-static SENSOR_DEVICE_ATTR(pwm3, S_IWUSR | S_IRUGO, get_pwm, set_pwm, 2);
-static SENSOR_DEVICE_ATTR(pwm4, S_IWUSR | S_IRUGO, get_pwm, set_pwm, 3);
-static SENSOR_DEVICE_ATTR(pwm5, S_IWUSR | S_IRUGO, get_pwm, set_pwm, 4);
-static SENSOR_DEVICE_ATTR(pwm6, S_IWUSR | S_IRUGO, get_pwm, set_pwm, 5);
-
-static SENSOR_DEVICE_ATTR(pwm1_enable, S_IWUSR | S_IRUGO,
-		get_pwm_enable, set_pwm_enable, 0);
-static SENSOR_DEVICE_ATTR(pwm2_enable, S_IWUSR | S_IRUGO,
-		get_pwm_enable, set_pwm_enable, 1);
-static SENSOR_DEVICE_ATTR(pwm3_enable, S_IWUSR | S_IRUGO,
-		get_pwm_enable, set_pwm_enable, 2);
-static SENSOR_DEVICE_ATTR(pwm4_enable, S_IWUSR | S_IRUGO,
-		get_pwm_enable, set_pwm_enable, 3);
-static SENSOR_DEVICE_ATTR(pwm5_enable, S_IWUSR | S_IRUGO,
-		get_pwm_enable, set_pwm_enable, 4);
-static SENSOR_DEVICE_ATTR(pwm6_enable, S_IWUSR | S_IRUGO,
-		get_pwm_enable, set_pwm_enable, 5);
-
-static struct attribute *max31790_attrs[] = {
-	&sensor_dev_attr_fan1_input.dev_attr.attr,
-	&sensor_dev_attr_fan2_input.dev_attr.attr,
-	&sensor_dev_attr_fan3_input.dev_attr.attr,
-	&sensor_dev_attr_fan4_input.dev_attr.attr,
-	&sensor_dev_attr_fan5_input.dev_attr.attr,
-	&sensor_dev_attr_fan6_input.dev_attr.attr,
-
-	&sensor_dev_attr_fan1_fault.dev_attr.attr,
-	&sensor_dev_attr_fan2_fault.dev_attr.attr,
-	&sensor_dev_attr_fan3_fault.dev_attr.attr,
-	&sensor_dev_attr_fan4_fault.dev_attr.attr,
-	&sensor_dev_attr_fan5_fault.dev_attr.attr,
-	&sensor_dev_attr_fan6_fault.dev_attr.attr,
-
-	&sensor_dev_attr_fan7_input.dev_attr.attr,
-	&sensor_dev_attr_fan8_input.dev_attr.attr,
-	&sensor_dev_attr_fan9_input.dev_attr.attr,
-	&sensor_dev_attr_fan10_input.dev_attr.attr,
-	&sensor_dev_attr_fan11_input.dev_attr.attr,
-	&sensor_dev_attr_fan12_input.dev_attr.attr,
-
-	&sensor_dev_attr_fan7_fault.dev_attr.attr,
-	&sensor_dev_attr_fan8_fault.dev_attr.attr,
-	&sensor_dev_attr_fan9_fault.dev_attr.attr,
-	&sensor_dev_attr_fan10_fault.dev_attr.attr,
-	&sensor_dev_attr_fan11_fault.dev_attr.attr,
-	&sensor_dev_attr_fan12_fault.dev_attr.attr,
-
-	&sensor_dev_attr_fan1_target.dev_attr.attr,
-	&sensor_dev_attr_fan2_target.dev_attr.attr,
-	&sensor_dev_attr_fan3_target.dev_attr.attr,
-	&sensor_dev_attr_fan4_target.dev_attr.attr,
-	&sensor_dev_attr_fan5_target.dev_attr.attr,
-	&sensor_dev_attr_fan6_target.dev_attr.attr,
-
-	&sensor_dev_attr_pwm1.dev_attr.attr,
-	&sensor_dev_attr_pwm2.dev_attr.attr,
-	&sensor_dev_attr_pwm3.dev_attr.attr,
-	&sensor_dev_attr_pwm4.dev_attr.attr,
-	&sensor_dev_attr_pwm5.dev_attr.attr,
-	&sensor_dev_attr_pwm6.dev_attr.attr,
-
-	&sensor_dev_attr_pwm1_enable.dev_attr.attr,
-	&sensor_dev_attr_pwm2_enable.dev_attr.attr,
-	&sensor_dev_attr_pwm3_enable.dev_attr.attr,
-	&sensor_dev_attr_pwm4_enable.dev_attr.attr,
-	&sensor_dev_attr_pwm5_enable.dev_attr.attr,
-	&sensor_dev_attr_pwm6_enable.dev_attr.attr,
-	NULL
+static const u32 max31790_fan_config[] = {
+	HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_FAULT,
+	HWMON_F_INPUT | HWMON_F_FAULT,
+	0
 };
 
-static umode_t max31790_attrs_visible(struct kobject *kobj,
-				     struct attribute *a, int n)
-{
-	struct device *dev = container_of(kobj, struct device, kobj);
-	struct max31790_data *data = dev_get_drvdata(dev);
-	struct device_attribute *devattr =
-			container_of(a, struct device_attribute, attr);
-	int index = to_sensor_dev_attr(devattr)->index % NR_CHANNEL;
-	u8 fan_config;
+static const struct hwmon_channel_info max31790_fan = {
+	.type = hwmon_fan,
+	.config = max31790_fan_config,
+};
 
-	fan_config = data->fan_config[index];
+static const u32 max31790_pwm_config[] = {
+	HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+	HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+	HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+	HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+	HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+	HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+	0
+};
 
-	if (n >= NR_CHANNEL * 2 && n < NR_CHANNEL * 4 &&
-	    !(fan_config & MAX31790_FAN_CFG_TACH_INPUT))
-		return 0;
-	if (n >= NR_CHANNEL * 4 && (fan_config & MAX31790_FAN_CFG_TACH_INPUT))
-		return 0;
+static const struct hwmon_channel_info max31790_pwm = {
+	.type = hwmon_pwm,
+	.config = max31790_pwm_config,
+};
 
-	return a->mode;
-}
+static const struct hwmon_channel_info *max31790_info[] = {
+	&max31790_fan,
+	&max31790_pwm,
+	NULL
+};
+
+static const struct hwmon_ops max31790_hwmon_ops = {
+	.is_visible = max31790_is_visible,
+	.read = max31790_read,
+	.write = max31790_write,
+};
 
-static const struct attribute_group max31790_group = {
-	.attrs = max31790_attrs,
-	.is_visible = max31790_attrs_visible,
+static const struct hwmon_chip_info max31790_chip_info = {
+	.ops = &max31790_hwmon_ops,
+	.info = max31790_info,
 };
-__ATTRIBUTE_GROUPS(max31790);
 
 static int max31790_init_client(struct i2c_client *client,
 				struct max31790_data *data)
@@ -575,8 +500,10 @@ static int max31790_probe(struct i2c_client *client,
 	if (err)
 		return err;
 
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev,
-			client->name, data, max31790_groups);
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 data,
+							 &max31790_chip_info,
+							 NULL);
 
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
diff --git a/drivers/hwmon/max6650.c b/drivers/hwmon/max6650.c
index 162a520f4bd6..a993b44ed538 100644
--- a/drivers/hwmon/max6650.c
+++ b/drivers/hwmon/max6650.c
@@ -39,6 +39,7 @@
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
 #include <linux/err.h>
+#include <linux/of_device.h>
 
 /*
  * Insmod parameters
@@ -48,7 +49,7 @@
 static int fan_voltage;
 /* prescaler: Possible values are 1, 2, 4, 8, 16 or 0 for don't change */
 static int prescaler;
-/* clock: The clock frequency of the chip the driver should assume */
+/* clock: The clock frequency of the chip (max6651 can be clocked externally) */
 static int clock = 254000;
 
 module_param(fan_voltage, int, S_IRUGO);
@@ -133,6 +134,19 @@ static const u8 tach_reg[] = {
 	MAX6650_REG_TACH3,
 };
 
+static const struct of_device_id max6650_dt_match[] = {
+	{
+		.compatible = "maxim,max6650",
+		.data = (void *)1
+	},
+	{
+		.compatible = "maxim,max6651",
+		.data = (void *)4
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, max6650_dt_match);
+
 static struct max6650_data *max6650_update_device(struct device *dev)
 {
 	struct max6650_data *data = dev_get_drvdata(dev);
@@ -171,6 +185,30 @@ static struct max6650_data *max6650_update_device(struct device *dev)
 	return data;
 }
 
+/*
+ * Change the operating mode of the chip (if needed).
+ * mode is one of the MAX6650_CFG_MODE_* values.
+ */
+static int max6650_set_operating_mode(struct max6650_data *data, u8 mode)
+{
+	int result;
+	u8 config = data->config;
+
+	if (mode == (config & MAX6650_CFG_MODE_MASK))
+		return 0;
+
+	config = (config & ~MAX6650_CFG_MODE_MASK) | mode;
+
+	result = i2c_smbus_write_byte_data(data->client, MAX6650_REG_CONFIG,
+					   config);
+	if (result < 0)
+		return result;
+
+	data->config = config;
+
+	return 0;
+}
+
 static ssize_t get_fan(struct device *dev, struct device_attribute *devattr,
 		       char *buf)
 {
@@ -252,18 +290,12 @@ static ssize_t get_target(struct device *dev, struct device_attribute *devattr,
 	return sprintf(buf, "%d\n", rpm);
 }
 
-static ssize_t set_target(struct device *dev, struct device_attribute *devattr,
-			 const char *buf, size_t count)
+static int max6650_set_target(struct max6650_data *data, unsigned long rpm)
 {
-	struct max6650_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
 	int kscale, ktach;
-	unsigned long rpm;
-	int err;
 
-	err = kstrtoul(buf, 10, &rpm);
-	if (err)
-		return err;
+	if (rpm == 0)
+		return max6650_set_operating_mode(data, MAX6650_CFG_MODE_OFF);
 
 	rpm = clamp_val(rpm, FAN_RPM_MIN, FAN_RPM_MAX);
 
@@ -274,8 +306,6 @@ static ssize_t set_target(struct device *dev, struct device_attribute *devattr,
 	 *     KTACH = [(fCLK x KSCALE) / (256 x FanSpeed)] - 1
 	 */
 
-	mutex_lock(&data->update_lock);
-
 	kscale = DIV_FROM_REG(data->config);
 	ktach = ((clock * kscale) / (256 * rpm / 60)) - 1;
 	if (ktach < 0)
@@ -284,10 +314,30 @@ static ssize_t set_target(struct device *dev, struct device_attribute *devattr,
 		ktach = 255;
 	data->speed = ktach;
 
-	i2c_smbus_write_byte_data(client, MAX6650_REG_SPEED, data->speed);
+	return i2c_smbus_write_byte_data(data->client, MAX6650_REG_SPEED,
+					 data->speed);
+}
+
+static ssize_t set_target(struct device *dev, struct device_attribute *devattr,
+			 const char *buf, size_t count)
+{
+	struct max6650_data *data = dev_get_drvdata(dev);
+	unsigned long rpm;
+	int err;
+
+	err = kstrtoul(buf, 10, &rpm);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+
+	err = max6650_set_target(data, rpm);
 
 	mutex_unlock(&data->update_lock);
 
+	if (err < 0)
+		return err;
+
 	return count;
 }
 
@@ -341,12 +391,11 @@ static ssize_t set_pwm(struct device *dev, struct device_attribute *devattr,
 		data->dac = 180 - (180 * pwm)/255;
 	else
 		data->dac = 76 - (76 * pwm)/255;
-
-	i2c_smbus_write_byte_data(client, MAX6650_REG_DAC, data->dac);
+	err = i2c_smbus_write_byte_data(client, MAX6650_REG_DAC, data->dac);
 
 	mutex_unlock(&data->update_lock);
 
-	return count;
+	return err < 0 ? err : count;
 }
 
 /*
@@ -355,14 +404,14 @@ static ssize_t set_pwm(struct device *dev, struct device_attribute *devattr,
  * 0 = Fan always on
  * 1 = Open loop, Voltage is set according to speed, not regulated.
  * 2 = Closed loop, RPM for all fans regulated by fan1 tachometer
+ * 3 = Fan off
  */
-
 static ssize_t get_enable(struct device *dev, struct device_attribute *devattr,
 			  char *buf)
 {
 	struct max6650_data *data = max6650_update_device(dev);
 	int mode = (data->config & MAX6650_CFG_MODE_MASK) >> 4;
-	int sysfs_modes[4] = {0, 1, 2, 1};
+	int sysfs_modes[4] = {0, 3, 2, 1};
 
 	return sprintf(buf, "%d\n", sysfs_modes[mode]);
 }
@@ -371,25 +420,25 @@ static ssize_t set_enable(struct device *dev, struct device_attribute *devattr,
 			  const char *buf, size_t count)
 {
 	struct max6650_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
-	int max6650_modes[3] = {0, 3, 2};
 	unsigned long mode;
 	int err;
+	const u8 max6650_modes[] = {
+		MAX6650_CFG_MODE_ON,
+		MAX6650_CFG_MODE_OPEN_LOOP,
+		MAX6650_CFG_MODE_CLOSED_LOOP,
+		MAX6650_CFG_MODE_OFF,
+		};
 
 	err = kstrtoul(buf, 10, &mode);
 	if (err)
 		return err;
 
-	if (mode > 2)
+	if (mode >= ARRAY_SIZE(max6650_modes))
 		return -EINVAL;
 
 	mutex_lock(&data->update_lock);
 
-	data->config = i2c_smbus_read_byte_data(client, MAX6650_REG_CONFIG);
-	data->config = (data->config & ~MAX6650_CFG_MODE_MASK)
-		       | (max6650_modes[mode] << 4);
-
-	i2c_smbus_write_byte_data(client, MAX6650_REG_CONFIG, data->config);
+	max6650_set_operating_mode(data, max6650_modes[mode]);
 
 	mutex_unlock(&data->update_lock);
 
@@ -566,6 +615,18 @@ static int max6650_init_client(struct max6650_data *data,
 	struct device *dev = &client->dev;
 	int config;
 	int err = -EIO;
+	u32 voltage;
+	u32 prescale;
+	u32 target_rpm;
+
+	if (of_property_read_u32(dev->of_node, "maxim,fan-microvolt",
+				 &voltage))
+		voltage = fan_voltage;
+	else
+		voltage /= 1000000; /* Microvolts to volts */
+	if (of_property_read_u32(dev->of_node, "maxim,fan-prescale",
+				 &prescale))
+		prescale = prescaler;
 
 	config = i2c_smbus_read_byte_data(client, MAX6650_REG_CONFIG);
 
@@ -574,7 +635,7 @@ static int max6650_init_client(struct max6650_data *data,
 		return err;
 	}
 
-	switch (fan_voltage) {
+	switch (voltage) {
 	case 0:
 		break;
 	case 5:
@@ -584,14 +645,10 @@ static int max6650_init_client(struct max6650_data *data,
 		config |= MAX6650_CFG_V12;
 		break;
 	default:
-		dev_err(dev, "illegal value for fan_voltage (%d)\n",
-			fan_voltage);
+		dev_err(dev, "illegal value for fan_voltage (%d)\n", voltage);
 	}
 
-	dev_info(dev, "Fan voltage is set to %dV.\n",
-		 (config & MAX6650_CFG_V12) ? 12 : 5);
-
-	switch (prescaler) {
+	switch (prescale) {
 	case 0:
 		break;
 	case 1:
@@ -614,28 +671,13 @@ static int max6650_init_client(struct max6650_data *data,
 			 | MAX6650_CFG_PRESCALER_16;
 		break;
 	default:
-		dev_err(dev, "illegal value for prescaler (%d)\n", prescaler);
+		dev_err(dev, "illegal value for prescaler (%d)\n", prescale);
 	}
 
-	dev_info(dev, "Prescaler is set to %d.\n",
+	dev_info(dev, "Fan voltage: %dV, prescaler: %d.\n",
+		 (config & MAX6650_CFG_V12) ? 12 : 5,
 		 1 << (config & MAX6650_CFG_PRESCALER_MASK));
 
-	/*
-	 * If mode is set to "full off", we change it to "open loop" and
-	 * set DAC to 255, which has the same effect. We do this because
-	 * there's no "full off" mode defined in hwmon specifications.
-	 */
-
-	if ((config & MAX6650_CFG_MODE_MASK) == MAX6650_CFG_MODE_OFF) {
-		dev_dbg(dev, "Change mode to open loop, full off.\n");
-		config = (config & ~MAX6650_CFG_MODE_MASK)
-			 | MAX6650_CFG_MODE_OPEN_LOOP;
-		if (i2c_smbus_write_byte_data(client, MAX6650_REG_DAC, 255)) {
-			dev_err(dev, "DAC write error, aborting.\n");
-			return err;
-		}
-	}
-
 	if (i2c_smbus_write_byte_data(client, MAX6650_REG_CONFIG, config)) {
 		dev_err(dev, "Config write error, aborting.\n");
 		return err;
@@ -644,6 +686,12 @@ static int max6650_init_client(struct max6650_data *data,
 	data->config = config;
 	data->count = i2c_smbus_read_byte_data(client, MAX6650_REG_COUNT);
 
+	if (!of_property_read_u32(client->dev.of_node, "maxim,fan-target-rpm",
+				  &target_rpm)) {
+		max6650_set_target(data, target_rpm);
+		max6650_set_operating_mode(data, MAX6650_CFG_MODE_CLOSED_LOOP);
+	}
+
 	return 0;
 }
 
@@ -651,6 +699,8 @@ static int max6650_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
 	struct device *dev = &client->dev;
+	const struct of_device_id *of_id =
+		of_match_device(of_match_ptr(max6650_dt_match), dev);
 	struct max6650_data *data;
 	struct device *hwmon_dev;
 	int err;
@@ -661,7 +711,7 @@ static int max6650_probe(struct i2c_client *client,
 
 	data->client = client;
 	mutex_init(&data->update_lock);
-	data->nr_fans = id->driver_data;
+	data->nr_fans = of_id ? (int)(uintptr_t)of_id->data : id->driver_data;
 
 	/*
 	 * Initialize the max6650 chip
@@ -691,6 +741,7 @@ MODULE_DEVICE_TABLE(i2c, max6650_id);
 static struct i2c_driver max6650_driver = {
 	.driver = {
 		.name	= "max6650",
+		.of_match_table = of_match_ptr(max6650_dt_match),
 	},
 	.probe		= max6650_probe,
 	.id_table	= max6650_id,
diff --git a/drivers/hwmon/nct6775.c b/drivers/hwmon/nct6775.c
index d087a8e00cf5..ce75dd4db7eb 100644
--- a/drivers/hwmon/nct6775.c
+++ b/drivers/hwmon/nct6775.c
@@ -195,6 +195,8 @@ superio_exit(int ioreg)
 
 #define NUM_FAN		6
 
+#define TEMP_SOURCE_VIRTUAL	0x1f
+
 /* Common and NCT6775 specific data */
 
 /* Voltage min/max registers for nr=7..14 are in bank 5 */
@@ -3940,7 +3942,7 @@ static int nct6775_probe(struct platform_device *pdev)
 			continue;
 
 		src = nct6775_read_value(data, data->REG_TEMP_SEL[i]) & 0x1f;
-		if (!src || (mask & (1 << src)))
+		if (!src)
 			continue;
 
 		if (src >= data->temp_label_num ||
@@ -3952,7 +3954,16 @@ static int nct6775_probe(struct platform_device *pdev)
 			continue;
 		}
 
-		mask |= 1 << src;
+		/*
+		 * For virtual temperature sources, the 'virtual' temperature
+		 * for each fan reflects a different temperature, and there
+		 * are no duplicates.
+		 */
+		if (src != TEMP_SOURCE_VIRTUAL) {
+			if (mask & (1 << src))
+				continue;
+			mask |= 1 << src;
+		}
 
 		/* Use fixed index for SYSTIN(1), CPUTIN(2), AUXTIN(3) */
 		if (src <= data->temp_fixed_num) {
@@ -4232,11 +4243,11 @@ static int __init nct6775_find(int sioaddr, struct nct6775_sio_data *sio_data)
 	if (err)
 		return err;
 
-	if (force_id)
+	val = (superio_inb(sioaddr, SIO_REG_DEVID) << 8) |
+		superio_inb(sioaddr, SIO_REG_DEVID + 1);
+	if (force_id && val != 0xffff)
 		val = force_id;
-	else
-		val = (superio_inb(sioaddr, SIO_REG_DEVID) << 8)
-		    | superio_inb(sioaddr, SIO_REG_DEVID + 1);
+
 	switch (val & SIO_ID_MASK) {
 	case SIO_NCT6106_ID:
 		sio_data->kind = nct6106;
diff --git a/drivers/hwmon/nct7904.c b/drivers/hwmon/nct7904.c
index 08ff89d222e5..95a68ab175c7 100644
--- a/drivers/hwmon/nct7904.c
+++ b/drivers/hwmon/nct7904.c
@@ -21,7 +21,6 @@
 #include <linux/i2c.h>
 #include <linux/mutex.h>
 #include <linux/hwmon.h>
-#include <linux/hwmon-sysfs.h>
 
 #define VENDOR_ID_REG		0x7A	/* Any bank */
 #define NUVOTON_ID		0x50
@@ -153,341 +152,230 @@ static int nct7904_write_reg(struct nct7904_data *data,
 	return ret;
 }
 
-/* FANIN ATTR */
-static ssize_t show_fan(struct device *dev,
-			struct device_attribute *devattr, char *buf)
+static int nct7904_read_fan(struct device *dev, u32 attr, int channel,
+			    long *val)
 {
-	int index = to_sensor_dev_attr(devattr)->index;
 	struct nct7904_data *data = dev_get_drvdata(dev);
+	unsigned int cnt, rpm;
 	int ret;
-	unsigned cnt, rpm;
 
-	ret = nct7904_read_reg16(data, BANK_0, FANIN1_HV_REG + index * 2);
-	if (ret < 0)
-		return ret;
-	cnt = ((ret & 0xff00) >> 3) | (ret & 0x1f);
-	if (cnt == 0x1fff)
-		rpm = 0;
-	else
-		rpm = 1350000 / cnt;
-	return sprintf(buf, "%u\n", rpm);
+	switch(attr) {
+	case hwmon_fan_input:
+		ret = nct7904_read_reg16(data, BANK_0,
+					 FANIN1_HV_REG + channel * 2);
+		if (ret < 0)
+			return ret;
+		cnt = ((ret & 0xff00) >> 3) | (ret & 0x1f);
+		if (cnt == 0x1fff)
+			rpm = 0;
+		else
+			rpm = 1350000 / cnt;
+		*val = rpm;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static umode_t nct7904_fanin_is_visible(struct kobject *kobj,
-					struct attribute *a, int n)
+static umode_t nct7904_fan_is_visible(const void *_data, u32 attr, int channel)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
-	struct nct7904_data *data = dev_get_drvdata(dev);
+	const struct nct7904_data *data = _data;
 
-	if (data->fanin_mask & (1 << n))
-		return a->mode;
+	if (attr == hwmon_fan_input && data->fanin_mask & (1 << channel))
+		return S_IRUGO;
 	return 0;
 }
 
-static SENSOR_DEVICE_ATTR(fan1_input, S_IRUGO, show_fan, NULL, 0);
-static SENSOR_DEVICE_ATTR(fan2_input, S_IRUGO, show_fan, NULL, 1);
-static SENSOR_DEVICE_ATTR(fan3_input, S_IRUGO, show_fan, NULL, 2);
-static SENSOR_DEVICE_ATTR(fan4_input, S_IRUGO, show_fan, NULL, 3);
-static SENSOR_DEVICE_ATTR(fan5_input, S_IRUGO, show_fan, NULL, 4);
-static SENSOR_DEVICE_ATTR(fan6_input, S_IRUGO, show_fan, NULL, 5);
-static SENSOR_DEVICE_ATTR(fan7_input, S_IRUGO, show_fan, NULL, 6);
-static SENSOR_DEVICE_ATTR(fan8_input, S_IRUGO, show_fan, NULL, 7);
-static SENSOR_DEVICE_ATTR(fan9_input, S_IRUGO, show_fan, NULL, 8);
-static SENSOR_DEVICE_ATTR(fan10_input, S_IRUGO, show_fan, NULL, 9);
-static SENSOR_DEVICE_ATTR(fan11_input, S_IRUGO, show_fan, NULL, 10);
-static SENSOR_DEVICE_ATTR(fan12_input, S_IRUGO, show_fan, NULL, 11);
-
-static struct attribute *nct7904_fanin_attrs[] = {
-	&sensor_dev_attr_fan1_input.dev_attr.attr,
-	&sensor_dev_attr_fan2_input.dev_attr.attr,
-	&sensor_dev_attr_fan3_input.dev_attr.attr,
-	&sensor_dev_attr_fan4_input.dev_attr.attr,
-	&sensor_dev_attr_fan5_input.dev_attr.attr,
-	&sensor_dev_attr_fan6_input.dev_attr.attr,
-	&sensor_dev_attr_fan7_input.dev_attr.attr,
-	&sensor_dev_attr_fan8_input.dev_attr.attr,
-	&sensor_dev_attr_fan9_input.dev_attr.attr,
-	&sensor_dev_attr_fan10_input.dev_attr.attr,
-	&sensor_dev_attr_fan11_input.dev_attr.attr,
-	&sensor_dev_attr_fan12_input.dev_attr.attr,
-	NULL
-};
-
-static const struct attribute_group nct7904_fanin_group = {
-	.attrs = nct7904_fanin_attrs,
-	.is_visible = nct7904_fanin_is_visible,
+static u8 nct7904_chan_to_index[] = {
+	0,	/* Not used */
+	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+	18, 19, 20, 16
 };
 
-/* VSEN ATTR */
-static ssize_t show_voltage(struct device *dev,
-			    struct device_attribute *devattr, char *buf)
+static int nct7904_read_in(struct device *dev, u32 attr, int channel,
+			   long *val)
 {
-	int index = to_sensor_dev_attr(devattr)->index;
 	struct nct7904_data *data = dev_get_drvdata(dev);
-	int ret;
-	int volt;
+	int ret, volt, index;
 
-	ret = nct7904_read_reg16(data, BANK_0, VSEN1_HV_REG + index * 2);
-	if (ret < 0)
-		return ret;
-	volt = ((ret & 0xff00) >> 5) | (ret & 0x7);
-	if (index < 14)
-		volt *= 2; /* 0.002V scale */
-	else
-		volt *= 6; /* 0.006V scale */
+	index = nct7904_chan_to_index[channel];
 
-	return sprintf(buf, "%d\n", volt);
+	switch(attr) {
+	case hwmon_in_input:
+		ret = nct7904_read_reg16(data, BANK_0,
+					 VSEN1_HV_REG + index * 2);
+		if (ret < 0)
+			return ret;
+		volt = ((ret & 0xff00) >> 5) | (ret & 0x7);
+		if (index < 14)
+			volt *= 2; /* 0.002V scale */
+		else
+			volt *= 6; /* 0.006V scale */
+		*val = volt;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t show_ltemp(struct device *dev,
-			  struct device_attribute *devattr, char *buf)
+static umode_t nct7904_in_is_visible(const void *_data, u32 attr, int channel)
 {
-	struct nct7904_data *data = dev_get_drvdata(dev);
-	int ret;
-	int temp;
+	const struct nct7904_data *data = _data;
+	int index = nct7904_chan_to_index[channel];
 
-	ret = nct7904_read_reg16(data, BANK_0, LTD_HV_REG);
-	if (ret < 0)
-		return ret;
-	temp = ((ret & 0xff00) >> 5) | (ret & 0x7);
-	temp = sign_extend32(temp, 10) * 125;
-
-	return sprintf(buf, "%d\n", temp);
-}
-
-static umode_t nct7904_vsen_is_visible(struct kobject *kobj,
-				       struct attribute *a, int n)
-{
-	struct device *dev = container_of(kobj, struct device, kobj);
-	struct nct7904_data *data = dev_get_drvdata(dev);
+	if (channel > 0 && attr == hwmon_in_input &&
+	    (data->vsen_mask & BIT(index)))
+		return S_IRUGO;
 
-	if (data->vsen_mask & (1 << n))
-		return a->mode;
 	return 0;
 }
 
-static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, show_voltage, NULL, 0);
-static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, show_voltage, NULL, 1);
-static SENSOR_DEVICE_ATTR(in3_input, S_IRUGO, show_voltage, NULL, 2);
-static SENSOR_DEVICE_ATTR(in4_input, S_IRUGO, show_voltage, NULL, 3);
-static SENSOR_DEVICE_ATTR(in5_input, S_IRUGO, show_voltage, NULL, 4);
-static SENSOR_DEVICE_ATTR(in6_input, S_IRUGO, show_voltage, NULL, 5);
-static SENSOR_DEVICE_ATTR(in7_input, S_IRUGO, show_voltage, NULL, 6);
-static SENSOR_DEVICE_ATTR(in8_input, S_IRUGO, show_voltage, NULL, 7);
-static SENSOR_DEVICE_ATTR(in9_input, S_IRUGO, show_voltage, NULL, 8);
-static SENSOR_DEVICE_ATTR(in10_input, S_IRUGO, show_voltage, NULL, 9);
-static SENSOR_DEVICE_ATTR(in11_input, S_IRUGO, show_voltage, NULL, 10);
-static SENSOR_DEVICE_ATTR(in12_input, S_IRUGO, show_voltage, NULL, 11);
-static SENSOR_DEVICE_ATTR(in13_input, S_IRUGO, show_voltage, NULL, 12);
-static SENSOR_DEVICE_ATTR(in14_input, S_IRUGO, show_voltage, NULL, 13);
-/*
- * Next 3 voltage sensors have specific names in the Nuvoton doc
- * (3VDD, VBAT, 3VSB) but we use vacant numbers for them.
- */
-static SENSOR_DEVICE_ATTR(in15_input, S_IRUGO, show_voltage, NULL, 14);
-static SENSOR_DEVICE_ATTR(in16_input, S_IRUGO, show_voltage, NULL, 15);
-static SENSOR_DEVICE_ATTR(in20_input, S_IRUGO, show_voltage, NULL, 16);
-/* This is not a voltage, but a local temperature sensor. */
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, show_ltemp, NULL, 0);
-static SENSOR_DEVICE_ATTR(in17_input, S_IRUGO, show_voltage, NULL, 18);
-static SENSOR_DEVICE_ATTR(in18_input, S_IRUGO, show_voltage, NULL, 19);
-static SENSOR_DEVICE_ATTR(in19_input, S_IRUGO, show_voltage, NULL, 20);
-
-static struct attribute *nct7904_vsen_attrs[] = {
-	&sensor_dev_attr_in1_input.dev_attr.attr,
-	&sensor_dev_attr_in2_input.dev_attr.attr,
-	&sensor_dev_attr_in3_input.dev_attr.attr,
-	&sensor_dev_attr_in4_input.dev_attr.attr,
-	&sensor_dev_attr_in5_input.dev_attr.attr,
-	&sensor_dev_attr_in6_input.dev_attr.attr,
-	&sensor_dev_attr_in7_input.dev_attr.attr,
-	&sensor_dev_attr_in8_input.dev_attr.attr,
-	&sensor_dev_attr_in9_input.dev_attr.attr,
-	&sensor_dev_attr_in10_input.dev_attr.attr,
-	&sensor_dev_attr_in11_input.dev_attr.attr,
-	&sensor_dev_attr_in12_input.dev_attr.attr,
-	&sensor_dev_attr_in13_input.dev_attr.attr,
-	&sensor_dev_attr_in14_input.dev_attr.attr,
-	&sensor_dev_attr_in15_input.dev_attr.attr,
-	&sensor_dev_attr_in16_input.dev_attr.attr,
-	&sensor_dev_attr_in20_input.dev_attr.attr,
-	&sensor_dev_attr_temp1_input.dev_attr.attr,
-	&sensor_dev_attr_in17_input.dev_attr.attr,
-	&sensor_dev_attr_in18_input.dev_attr.attr,
-	&sensor_dev_attr_in19_input.dev_attr.attr,
-	NULL
-};
-
-static const struct attribute_group nct7904_vsen_group = {
-	.attrs = nct7904_vsen_attrs,
-	.is_visible = nct7904_vsen_is_visible,
-};
-
-/* CPU_TEMP ATTR */
-static ssize_t show_tcpu(struct device *dev,
-			 struct device_attribute *devattr, char *buf)
+static int nct7904_read_temp(struct device *dev, u32 attr, int channel,
+			     long *val)
 {
-	int index = to_sensor_dev_attr(devattr)->index;
 	struct nct7904_data *data = dev_get_drvdata(dev);
-	int ret;
-	int temp;
-
-	ret = nct7904_read_reg16(data, BANK_0, T_CPU1_HV_REG + index * 2);
-	if (ret < 0)
-		return ret;
-
-	temp = ((ret & 0xff00) >> 5) | (ret & 0x7);
-	temp = sign_extend32(temp, 10) * 125;
-	return sprintf(buf, "%d\n", temp);
+	int ret, temp;
+
+	switch(attr) {
+	case hwmon_temp_input:
+		if (channel == 0)
+			ret = nct7904_read_reg16(data, BANK_0, LTD_HV_REG);
+		else
+			ret = nct7904_read_reg16(data, BANK_0,
+					T_CPU1_HV_REG + (channel - 1) * 2);
+		if (ret < 0)
+			return ret;
+		temp = ((ret & 0xff00) >> 5) | (ret & 0x7);
+		*val = sign_extend32(temp, 10) * 125;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static umode_t nct7904_tcpu_is_visible(struct kobject *kobj,
-				       struct attribute *a, int n)
+static umode_t nct7904_temp_is_visible(const void *_data, u32 attr, int channel)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
-	struct nct7904_data *data = dev_get_drvdata(dev);
+	const struct nct7904_data *data = _data;
+
+	if (attr == hwmon_temp_input) {
+		if (channel == 0) {
+			if (data->vsen_mask & BIT(17))
+				return S_IRUGO;
+		} else {
+			if (data->tcpu_mask & BIT(channel - 1))
+				return S_IRUGO;
+		}
+	}
 
-	if (data->tcpu_mask & (1 << n))
-		return a->mode;
 	return 0;
 }
 
-/* "temp1_input" reserved for local temp */
-static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, show_tcpu, NULL, 0);
-static SENSOR_DEVICE_ATTR(temp3_input, S_IRUGO, show_tcpu, NULL, 1);
-static SENSOR_DEVICE_ATTR(temp4_input, S_IRUGO, show_tcpu, NULL, 2);
-static SENSOR_DEVICE_ATTR(temp5_input, S_IRUGO, show_tcpu, NULL, 3);
-static SENSOR_DEVICE_ATTR(temp6_input, S_IRUGO, show_tcpu, NULL, 4);
-static SENSOR_DEVICE_ATTR(temp7_input, S_IRUGO, show_tcpu, NULL, 5);
-static SENSOR_DEVICE_ATTR(temp8_input, S_IRUGO, show_tcpu, NULL, 6);
-static SENSOR_DEVICE_ATTR(temp9_input, S_IRUGO, show_tcpu, NULL, 7);
-
-static struct attribute *nct7904_tcpu_attrs[] = {
-	&sensor_dev_attr_temp2_input.dev_attr.attr,
-	&sensor_dev_attr_temp3_input.dev_attr.attr,
-	&sensor_dev_attr_temp4_input.dev_attr.attr,
-	&sensor_dev_attr_temp5_input.dev_attr.attr,
-	&sensor_dev_attr_temp6_input.dev_attr.attr,
-	&sensor_dev_attr_temp7_input.dev_attr.attr,
-	&sensor_dev_attr_temp8_input.dev_attr.attr,
-	&sensor_dev_attr_temp9_input.dev_attr.attr,
-	NULL
-};
-
-static const struct attribute_group nct7904_tcpu_group = {
-	.attrs = nct7904_tcpu_attrs,
-	.is_visible = nct7904_tcpu_is_visible,
-};
-
-/* PWM ATTR */
-static ssize_t store_pwm(struct device *dev, struct device_attribute *devattr,
-			 const char *buf, size_t count)
+static int nct7904_read_pwm(struct device *dev, u32 attr, int channel,
+			    long *val)
 {
-	int index = to_sensor_dev_attr(devattr)->index;
 	struct nct7904_data *data = dev_get_drvdata(dev);
-	unsigned long val;
 	int ret;
 
-	if (kstrtoul(buf, 10, &val) < 0)
-		return -EINVAL;
-	if (val > 255)
-		return -EINVAL;
-
-	ret = nct7904_write_reg(data, BANK_3, FANCTL1_OUT_REG + index, val);
+	switch(attr) {
+	case hwmon_pwm_input:
+		ret = nct7904_read_reg(data, BANK_3, FANCTL1_OUT_REG + channel);
+		if (ret < 0)
+			return ret;
+		*val = ret;
+		return 0;
+	case hwmon_pwm_enable:
+		ret = nct7904_read_reg(data, BANK_3, FANCTL1_FMR_REG + channel);
+		if (ret < 0)
+			return ret;
 
-	return ret ? ret : count;
+		*val = ret ? 2 : 1;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t show_pwm(struct device *dev,
-			struct device_attribute *devattr, char *buf)
+static int nct7904_write_pwm(struct device *dev, u32 attr, int channel,
+			     long val)
 {
-	int index = to_sensor_dev_attr(devattr)->index;
 	struct nct7904_data *data = dev_get_drvdata(dev);
-	int val;
-
-	val = nct7904_read_reg(data, BANK_3, FANCTL1_OUT_REG + index);
-	if (val < 0)
-		return val;
+	int ret;
 
-	return sprintf(buf, "%d\n", val);
+	switch(attr) {
+	case hwmon_pwm_input:
+		if (val < 0 || val > 255)
+			return -EINVAL;
+		ret = nct7904_write_reg(data, BANK_3, FANCTL1_OUT_REG + channel,
+					val);
+		return ret;
+	case hwmon_pwm_enable:
+		if (val < 1 || val > 2 ||
+		    (val == 2 && !data->fan_mode[channel]))
+			return -EINVAL;
+		ret = nct7904_write_reg(data, BANK_3, FANCTL1_FMR_REG + channel,
+					val == 2 ? data->fan_mode[channel] : 0);
+		return ret;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-static ssize_t store_enable(struct device *dev,
-			    struct device_attribute *devattr,
-			    const char *buf, size_t count)
+static umode_t nct7904_pwm_is_visible(const void *_data, u32 attr, int channel)
 {
-	int index = to_sensor_dev_attr(devattr)->index;
-	struct nct7904_data *data = dev_get_drvdata(dev);
-	unsigned long val;
-	int ret;
-
-	if (kstrtoul(buf, 10, &val) < 0)
-		return -EINVAL;
-	if (val < 1 || val > 2 || (val == 2 && !data->fan_mode[index]))
-		return -EINVAL;
-
-	ret = nct7904_write_reg(data, BANK_3, FANCTL1_FMR_REG + index,
-				val == 2 ? data->fan_mode[index] : 0);
-
-	return ret ? ret : count;
+	switch(attr) {
+	case hwmon_pwm_input:
+	case hwmon_pwm_enable:
+		return S_IRUGO | S_IWUSR;
+	default:
+		return 0;
+	}
 }
 
-/* Return 1 for manual mode or 2 for SmartFan mode */
-static ssize_t show_enable(struct device *dev,
-			   struct device_attribute *devattr, char *buf)
+static int nct7904_read(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long *val)
 {
-	int index = to_sensor_dev_attr(devattr)->index;
-	struct nct7904_data *data = dev_get_drvdata(dev);
-	int val;
-
-	val = nct7904_read_reg(data, BANK_3, FANCTL1_FMR_REG + index);
-	if (val < 0)
-		return val;
-
-	return sprintf(buf, "%d\n", val ? 2 : 1);
+	switch (type) {
+	case hwmon_in:
+		return nct7904_read_in(dev, attr, channel, val);
+	case hwmon_fan:
+		return nct7904_read_fan(dev, attr, channel, val);
+	case hwmon_pwm:
+		return nct7904_read_pwm(dev, attr, channel, val);
+	case hwmon_temp:
+		return nct7904_read_temp(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
-/* 2 attributes per channel: pwm and mode */
-static SENSOR_DEVICE_ATTR(pwm1, S_IRUGO | S_IWUSR,
-			show_pwm, store_pwm, 0);
-static SENSOR_DEVICE_ATTR(pwm1_enable, S_IRUGO | S_IWUSR,
-			show_enable, store_enable, 0);
-static SENSOR_DEVICE_ATTR(pwm2, S_IRUGO | S_IWUSR,
-			show_pwm, store_pwm, 1);
-static SENSOR_DEVICE_ATTR(pwm2_enable, S_IRUGO | S_IWUSR,
-			show_enable, store_enable, 1);
-static SENSOR_DEVICE_ATTR(pwm3, S_IRUGO | S_IWUSR,
-			show_pwm, store_pwm, 2);
-static SENSOR_DEVICE_ATTR(pwm3_enable, S_IRUGO | S_IWUSR,
-			show_enable, store_enable, 2);
-static SENSOR_DEVICE_ATTR(pwm4, S_IRUGO | S_IWUSR,
-			show_pwm, store_pwm, 3);
-static SENSOR_DEVICE_ATTR(pwm4_enable, S_IRUGO | S_IWUSR,
-			show_enable, store_enable, 3);
-
-static struct attribute *nct7904_fanctl_attrs[] = {
-	&sensor_dev_attr_pwm1.dev_attr.attr,
-	&sensor_dev_attr_pwm1_enable.dev_attr.attr,
-	&sensor_dev_attr_pwm2.dev_attr.attr,
-	&sensor_dev_attr_pwm2_enable.dev_attr.attr,
-	&sensor_dev_attr_pwm3.dev_attr.attr,
-	&sensor_dev_attr_pwm3_enable.dev_attr.attr,
-	&sensor_dev_attr_pwm4.dev_attr.attr,
-	&sensor_dev_attr_pwm4_enable.dev_attr.attr,
-	NULL
-};
-
-static const struct attribute_group nct7904_fanctl_group = {
-	.attrs = nct7904_fanctl_attrs,
-};
+static int nct7904_write(struct device *dev, enum hwmon_sensor_types type,
+			 u32 attr, int channel, long val)
+{
+	switch (type) {
+	case hwmon_pwm:
+		return nct7904_write_pwm(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
 
-static const struct attribute_group *nct7904_groups[] = {
-	&nct7904_fanin_group,
-	&nct7904_vsen_group,
-	&nct7904_tcpu_group,
-	&nct7904_fanctl_group,
-	NULL
-};
+static umode_t nct7904_is_visible(const void *data,
+				  enum hwmon_sensor_types type,
+				  u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_in:
+		return nct7904_in_is_visible(data, attr, channel);
+	case hwmon_fan:
+		return nct7904_fan_is_visible(data, attr, channel);
+	case hwmon_pwm:
+		return nct7904_pwm_is_visible(data, attr, channel);
+	case hwmon_temp:
+		return nct7904_temp_is_visible(data, attr, channel);
+	default:
+		return 0;
+	}
+}
 
 /* Return 0 if detection is successful, -ENODEV otherwise */
 static int nct7904_detect(struct i2c_client *client,
@@ -512,6 +400,103 @@ static int nct7904_detect(struct i2c_client *client,
 	return 0;
 }
 
+static const u32 nct7904_in_config[] = {
+	HWMON_I_INPUT,                  /* dummy, skipped in is_visible */
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	0
+};
+
+static const struct hwmon_channel_info nct7904_in = {
+	.type = hwmon_in,
+	.config = nct7904_in_config,
+};
+
+static const u32 nct7904_fan_config[] = {
+            HWMON_F_INPUT,
+            HWMON_F_INPUT,
+            HWMON_F_INPUT,
+            HWMON_F_INPUT,
+            HWMON_F_INPUT,
+            HWMON_F_INPUT,
+            HWMON_F_INPUT,
+            HWMON_F_INPUT,
+	    0
+};
+
+static const struct hwmon_channel_info nct7904_fan = {
+	.type = hwmon_fan,
+	.config = nct7904_fan_config,
+};
+
+static const u32 nct7904_pwm_config[] = {
+            HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+            HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+            HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+            HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
+	    0
+};
+
+static const struct hwmon_channel_info nct7904_pwm = {
+	.type = hwmon_pwm,
+	.config = nct7904_pwm_config,
+};
+
+static const u32 nct7904_temp_config[] = {
+            HWMON_T_INPUT,
+            HWMON_T_INPUT,
+            HWMON_T_INPUT,
+            HWMON_T_INPUT,
+            HWMON_T_INPUT,
+            HWMON_T_INPUT,
+            HWMON_T_INPUT,
+            HWMON_T_INPUT,
+            HWMON_T_INPUT,
+	    0
+};
+
+static const struct hwmon_channel_info nct7904_temp = {
+	.type = hwmon_temp,
+	.config = nct7904_temp_config,
+};
+
+static const struct hwmon_channel_info *nct7904_info[] = {
+	&nct7904_in,
+	&nct7904_fan,
+	&nct7904_pwm,
+	&nct7904_temp,
+	NULL
+};
+
+static const struct hwmon_ops nct7904_hwmon_ops = {
+	.is_visible = nct7904_is_visible,
+	.read = nct7904_read,
+	.write = nct7904_write,
+};
+
+static const struct hwmon_chip_info nct7904_chip_info = {
+	.ops = &nct7904_hwmon_ops,
+	.info = nct7904_info,
+};
+
 static int nct7904_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
@@ -566,8 +551,8 @@ static int nct7904_probe(struct i2c_client *client,
 	}
 
 	hwmon_dev =
-		devm_hwmon_device_register_with_groups(dev, client->name, data,
-						       nct7904_groups);
+		devm_hwmon_device_register_with_info(dev, client->name, data,
+						     &nct7904_chip_info, NULL);
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
diff --git a/drivers/hwmon/ntc_thermistor.c b/drivers/hwmon/ntc_thermistor.c
index 8ef7b713cb1a..c52d07c6b49f 100644
--- a/drivers/hwmon/ntc_thermistor.c
+++ b/drivers/hwmon/ntc_thermistor.c
@@ -253,12 +253,9 @@ static const struct ntc_compensation b57330v2103[] = {
 };
 
 struct ntc_data {
-	struct device *hwmon_dev;
 	struct ntc_thermistor_platform_data *pdata;
 	const struct ntc_compensation *comp;
-	struct device *dev;
 	int n_comp;
-	char name[PLATFORM_NAME_SIZE];
 };
 
 #if defined(CONFIG_OF) && IS_ENABLED(CONFIG_IIO)
@@ -316,22 +313,22 @@ static const struct of_device_id ntc_match[] = {
 MODULE_DEVICE_TABLE(of, ntc_match);
 
 static struct ntc_thermistor_platform_data *
-ntc_thermistor_parse_dt(struct platform_device *pdev)
+ntc_thermistor_parse_dt(struct device *dev)
 {
 	struct iio_channel *chan;
 	enum iio_chan_type type;
-	struct device_node *np = pdev->dev.of_node;
+	struct device_node *np = dev->of_node;
 	struct ntc_thermistor_platform_data *pdata;
 	int ret;
 
 	if (!np)
 		return NULL;
 
-	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata)
 		return ERR_PTR(-ENOMEM);
 
-	chan = iio_channel_get(&pdev->dev, NULL);
+	chan = devm_iio_channel_get(dev, NULL);
 	if (IS_ERR(chan))
 		return ERR_CAST(chan);
 
@@ -359,22 +356,15 @@ ntc_thermistor_parse_dt(struct platform_device *pdev)
 
 	return pdata;
 }
-static void ntc_iio_channel_release(struct ntc_thermistor_platform_data *pdata)
-{
-	if (pdata->chan)
-		iio_channel_release(pdata->chan);
-}
 #else
 static struct ntc_thermistor_platform_data *
-ntc_thermistor_parse_dt(struct platform_device *pdev)
+ntc_thermistor_parse_dt(struct device *dev)
 {
 	return NULL;
 }
 
 #define ntc_match	NULL
 
-static void ntc_iio_channel_release(struct ntc_thermistor_platform_data *pdata)
-{ }
 #endif
 
 static inline u64 div64_u64_safe(u64 dividend, u64 divisor)
@@ -516,9 +506,8 @@ static int ntc_thermistor_get_ohm(struct ntc_data *data)
 	return -EINVAL;
 }
 
-static int ntc_read_temp(void *dev, int *temp)
+static int ntc_read_temp(void *data, int *temp)
 {
-	struct ntc_data *data = dev_get_drvdata(dev);
 	int ohm;
 
 	ohm = ntc_thermistor_get_ohm(data);
@@ -530,14 +519,6 @@ static int ntc_read_temp(void *dev, int *temp)
 	return 0;
 }
 
-static ssize_t ntc_show_name(struct device *dev,
-		struct device_attribute *attr, char *buf)
-{
-	struct ntc_data *data = dev_get_drvdata(dev);
-
-	return sprintf(buf, "%s\n", data->name);
-}
-
 static ssize_t ntc_show_type(struct device *dev,
 		struct device_attribute *attr, char *buf)
 {
@@ -559,18 +540,13 @@ static ssize_t ntc_show_temp(struct device *dev,
 
 static SENSOR_DEVICE_ATTR(temp1_type, S_IRUGO, ntc_show_type, NULL, 0);
 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, ntc_show_temp, NULL, 0);
-static DEVICE_ATTR(name, S_IRUGO, ntc_show_name, NULL);
 
-static struct attribute *ntc_attributes[] = {
-	&dev_attr_name.attr,
+static struct attribute *ntc_attrs[] = {
 	&sensor_dev_attr_temp1_type.dev_attr.attr,
 	&sensor_dev_attr_temp1_input.dev_attr.attr,
 	NULL,
 };
-
-static const struct attribute_group ntc_attr_group = {
-	.attrs = ntc_attributes,
-};
+ATTRIBUTE_GROUPS(ntc);
 
 static const struct thermal_zone_of_device_ops ntc_of_thermal_ops = {
 	.get_temp = ntc_read_temp,
@@ -579,33 +555,34 @@ static const struct thermal_zone_of_device_ops ntc_of_thermal_ops = {
 static int ntc_thermistor_probe(struct platform_device *pdev)
 {
 	struct thermal_zone_device *tz;
+	struct device *dev = &pdev->dev;
 	const struct of_device_id *of_id =
-			of_match_device(of_match_ptr(ntc_match), &pdev->dev);
+			of_match_device(of_match_ptr(ntc_match), dev);
 	const struct platform_device_id *pdev_id;
 	struct ntc_thermistor_platform_data *pdata;
+	struct device *hwmon_dev;
 	struct ntc_data *data;
-	int ret;
 
-	pdata = ntc_thermistor_parse_dt(pdev);
+	pdata = ntc_thermistor_parse_dt(dev);
 	if (IS_ERR(pdata))
 		return PTR_ERR(pdata);
 	else if (pdata == NULL)
-		pdata = dev_get_platdata(&pdev->dev);
+		pdata = dev_get_platdata(dev);
 
 	if (!pdata) {
-		dev_err(&pdev->dev, "No platform init data supplied.\n");
+		dev_err(dev, "No platform init data supplied.\n");
 		return -ENODEV;
 	}
 
 	/* Either one of the two is required. */
 	if (!pdata->read_uv && !pdata->read_ohm) {
-		dev_err(&pdev->dev,
+		dev_err(dev,
 			"Both read_uv and read_ohm missing. Need either one of the two.\n");
 		return -EINVAL;
 	}
 
 	if (pdata->read_uv && pdata->read_ohm) {
-		dev_warn(&pdev->dev,
+		dev_warn(dev,
 			 "Only one of read_uv and read_ohm is needed; ignoring read_uv.\n");
 		pdata->read_uv = NULL;
 	}
@@ -617,20 +594,17 @@ static int ntc_thermistor_probe(struct platform_device *pdev)
 				 NTC_CONNECTED_POSITIVE) ||
 				(pdata->connect != NTC_CONNECTED_POSITIVE &&
 				 pdata->connect != NTC_CONNECTED_GROUND))) {
-		dev_err(&pdev->dev,
-			"Required data to use read_uv not supplied.\n");
+		dev_err(dev, "Required data to use read_uv not supplied.\n");
 		return -EINVAL;
 	}
 
-	data = devm_kzalloc(&pdev->dev, sizeof(struct ntc_data), GFP_KERNEL);
+	data = devm_kzalloc(dev, sizeof(struct ntc_data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
 	pdev_id = of_id ? of_id->data : platform_get_device_id(pdev);
 
-	data->dev = &pdev->dev;
 	data->pdata = pdata;
-	strlcpy(data->name, pdev_id->name, sizeof(data->name));
 
 	switch (pdev_id->driver_data) {
 	case TYPE_NCPXXWB473:
@@ -654,49 +628,25 @@ static int ntc_thermistor_probe(struct platform_device *pdev)
 		data->n_comp = ARRAY_SIZE(ncpXXxh103);
 		break;
 	default:
-		dev_err(&pdev->dev, "Unknown device type: %lu(%s)\n",
+		dev_err(dev, "Unknown device type: %lu(%s)\n",
 				pdev_id->driver_data, pdev_id->name);
 		return -EINVAL;
 	}
 
-	platform_set_drvdata(pdev, data);
-
-	ret = sysfs_create_group(&data->dev->kobj, &ntc_attr_group);
-	if (ret) {
-		dev_err(data->dev, "unable to create sysfs files\n");
-		return ret;
-	}
-
-	data->hwmon_dev = hwmon_device_register(data->dev);
-	if (IS_ERR(data->hwmon_dev)) {
-		dev_err(data->dev, "unable to register as hwmon device.\n");
-		ret = PTR_ERR(data->hwmon_dev);
-		goto err_after_sysfs;
+	hwmon_dev = devm_hwmon_device_register_with_groups(dev, pdev_id->name,
+							   data, ntc_groups);
+	if (IS_ERR(hwmon_dev)) {
+		dev_err(dev, "unable to register as hwmon device.\n");
+		return PTR_ERR(hwmon_dev);
 	}
 
-	dev_info(&pdev->dev, "Thermistor type: %s successfully probed.\n",
-								pdev_id->name);
+	dev_info(dev, "Thermistor type: %s successfully probed.\n",
+		 pdev_id->name);
 
-	tz = devm_thermal_zone_of_sensor_register(data->dev, 0, data->dev,
+	tz = devm_thermal_zone_of_sensor_register(dev, 0, data,
 						  &ntc_of_thermal_ops);
 	if (IS_ERR(tz))
-		dev_dbg(&pdev->dev, "Failed to register to thermal fw.\n");
-
-	return 0;
-err_after_sysfs:
-	sysfs_remove_group(&data->dev->kobj, &ntc_attr_group);
-	ntc_iio_channel_release(pdata);
-	return ret;
-}
-
-static int ntc_thermistor_remove(struct platform_device *pdev)
-{
-	struct ntc_data *data = platform_get_drvdata(pdev);
-	struct ntc_thermistor_platform_data *pdata = data->pdata;
-
-	hwmon_device_unregister(data->hwmon_dev);
-	sysfs_remove_group(&data->dev->kobj, &ntc_attr_group);
-	ntc_iio_channel_release(pdata);
+		dev_dbg(dev, "Failed to register to thermal fw.\n");
 
 	return 0;
 }
@@ -707,7 +657,6 @@ static struct platform_driver ntc_thermistor_driver = {
 		.of_match_table = of_match_ptr(ntc_match),
 	},
 	.probe = ntc_thermistor_probe,
-	.remove = ntc_thermistor_remove,
 	.id_table = ntc_thermistor_id,
 };
 
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index 054d3d863802..cad1229b7e17 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -126,12 +126,12 @@ config SENSORS_TPS40422
 	  be called tps40422.
 
 config SENSORS_UCD9000
-	tristate "TI UCD90120, UCD90124, UCD9090, UCD90910"
+	tristate "TI UCD90120, UCD90124, UCD90160, UCD9090, UCD90910"
 	default n
 	help
 	  If you say yes here you get hardware monitoring support for TI
-	  UCD90120, UCD90124, UCD9090, UCD90910 Sequencer and System Health
-	  Controllers.
+	  UCD90120, UCD90124, UCD90160, UCD9090, UCD90910, Sequencer and System
+	  Health Controllers.
 
 	  This driver can also be built as a module. If so, the module will
 	  be called ucd9000.
diff --git a/drivers/hwmon/pmbus/pmbus.c b/drivers/hwmon/pmbus/pmbus.c
index 0a74991a60f0..44ca8a94873d 100644
--- a/drivers/hwmon/pmbus/pmbus.c
+++ b/drivers/hwmon/pmbus/pmbus.c
@@ -25,6 +25,7 @@
 #include <linux/slab.h>
 #include <linux/mutex.h>
 #include <linux/i2c.h>
+#include <linux/i2c/pmbus.h>
 #include "pmbus.h"
 
 /*
@@ -167,14 +168,26 @@ static int pmbus_probe(struct i2c_client *client,
 		       const struct i2c_device_id *id)
 {
 	struct pmbus_driver_info *info;
+	struct pmbus_platform_data *pdata = NULL;
+	struct device *dev = &client->dev;
 
-	info = devm_kzalloc(&client->dev, sizeof(struct pmbus_driver_info),
-			    GFP_KERNEL);
+	info = devm_kzalloc(dev, sizeof(struct pmbus_driver_info), GFP_KERNEL);
 	if (!info)
 		return -ENOMEM;
 
+	if (!strcmp(id->name, "dps460") || !strcmp(id->name, "dps800") ||
+	    !strcmp(id->name, "sgd009")) {
+		pdata = devm_kzalloc(dev, sizeof(struct pmbus_platform_data),
+				     GFP_KERNEL);
+		if (!pdata)
+			return -ENOMEM;
+
+		pdata->flags = PMBUS_SKIP_STATUS_CHECK;
+	}
+
 	info->pages = id->driver_data;
 	info->identify = pmbus_identify;
+	dev->platform_data = pdata;
 
 	return pmbus_do_probe(client, id, info);
 }
@@ -186,6 +199,8 @@ static const struct i2c_device_id pmbus_id[] = {
 	{"adp4000", 1},
 	{"bmr453", 1},
 	{"bmr454", 1},
+	{"dps460", 1},
+	{"dps800", 1},
 	{"mdt040", 1},
 	{"ncp4200", 1},
 	{"ncp4208", 1},
@@ -193,6 +208,7 @@ static const struct i2c_device_id pmbus_id[] = {
 	{"pdt006", 1},
 	{"pdt012", 1},
 	{"pmbus", 0},
+	{"sgd009", 1},
 	{"tps40400", 1},
 	{"tps544b20", 1},
 	{"tps544b25", 1},
diff --git a/drivers/hwmon/pmbus/ucd9000.c b/drivers/hwmon/pmbus/ucd9000.c
index fbb1479d3ad4..3e3aa950277f 100644
--- a/drivers/hwmon/pmbus/ucd9000.c
+++ b/drivers/hwmon/pmbus/ucd9000.c
@@ -28,7 +28,7 @@
 #include <linux/i2c/pmbus.h>
 #include "pmbus.h"
 
-enum chips { ucd9000, ucd90120, ucd90124, ucd9090, ucd90910 };
+enum chips { ucd9000, ucd90120, ucd90124, ucd90160, ucd9090, ucd90910 };
 
 #define UCD9000_MONITOR_CONFIG		0xd5
 #define UCD9000_NUM_PAGES		0xd6
@@ -112,6 +112,7 @@ static const struct i2c_device_id ucd9000_id[] = {
 	{"ucd9000", ucd9000},
 	{"ucd90120", ucd90120},
 	{"ucd90124", ucd90124},
+	{"ucd90160", ucd90160},
 	{"ucd9090", ucd9090},
 	{"ucd90910", ucd90910},
 	{}
diff --git a/drivers/hwmon/scpi-hwmon.c b/drivers/hwmon/scpi-hwmon.c
index 25b44e68926d..559a3dcd64d8 100644
--- a/drivers/hwmon/scpi-hwmon.c
+++ b/drivers/hwmon/scpi-hwmon.c
@@ -255,7 +255,6 @@ static const struct of_device_id scpi_of_match[] = {
 static struct platform_driver scpi_hwmon_platdrv = {
 	.driver = {
 		.name	= "scpi-hwmon",
-		.owner	= THIS_MODULE,
 		.of_match_table = scpi_of_match,
 	},
 	.probe		= scpi_hwmon_probe,
diff --git a/drivers/hwmon/tmp102.c b/drivers/hwmon/tmp102.c
index 8479ac5eb853..36bba2a816a4 100644
--- a/drivers/hwmon/tmp102.c
+++ b/drivers/hwmon/tmp102.c
@@ -25,7 +25,6 @@
 #include <linux/device.h>
 #include <linux/jiffies.h>
 #include <linux/regmap.h>
-#include <linux/thermal.h>
 #include <linux/of.h>
 
 #define	DRIVER_NAME "tmp102"
@@ -79,84 +78,113 @@ static inline u16 tmp102_mC_to_reg(int val)
 	return (val * 128) / 1000;
 }
 
-static int tmp102_read_temp(void *dev, int *temp)
+static int tmp102_read(struct device *dev, enum hwmon_sensor_types type,
+		       u32 attr, int channel, long *temp)
 {
 	struct tmp102 *tmp102 = dev_get_drvdata(dev);
-	unsigned int reg;
-	int ret;
-
-	if (time_before(jiffies, tmp102->ready_time)) {
-		dev_dbg(dev, "%s: Conversion not ready yet..\n", __func__);
-		return -EAGAIN;
+	unsigned int regval;
+	int err, reg;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		/* Is it too early to return a conversion ? */
+		if (time_before(jiffies, tmp102->ready_time)) {
+			dev_dbg(dev, "%s: Conversion not ready yet..\n", __func__);
+			return -EAGAIN;
+		}
+		reg = TMP102_TEMP_REG;
+		break;
+	case hwmon_temp_max_hyst:
+		reg = TMP102_TLOW_REG;
+		break;
+	case hwmon_temp_max:
+		reg = TMP102_THIGH_REG;
+		break;
+	default:
+		return -EOPNOTSUPP;
 	}
 
-	ret = regmap_read(tmp102->regmap, TMP102_TEMP_REG, &reg);
-	if (ret < 0)
-		return ret;
-
-	*temp = tmp102_reg_to_mC(reg);
+	err = regmap_read(tmp102->regmap, reg, &regval);
+	if (err < 0)
+		return err;
+	*temp = tmp102_reg_to_mC(regval);
 
 	return 0;
 }
 
-static ssize_t tmp102_show_temp(struct device *dev,
-				struct device_attribute *attr,
-				char *buf)
+static int tmp102_write(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long temp)
 {
-	struct sensor_device_attribute *sda = to_sensor_dev_attr(attr);
 	struct tmp102 *tmp102 = dev_get_drvdata(dev);
-	int regaddr = sda->index;
-	unsigned int reg;
-	int err;
-
-	if (regaddr == TMP102_TEMP_REG &&
-	    time_before(jiffies, tmp102->ready_time))
-		return -EAGAIN;
-
-	err = regmap_read(tmp102->regmap, regaddr, &reg);
-	if (err < 0)
-		return err;
+	int reg;
+
+	switch (attr) {
+	case hwmon_temp_max_hyst:
+		reg = TMP102_TLOW_REG;
+		break;
+	case hwmon_temp_max:
+		reg = TMP102_THIGH_REG;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
 
-	return sprintf(buf, "%d\n", tmp102_reg_to_mC(reg));
+	temp = clamp_val(temp, -256000, 255000);
+	return regmap_write(tmp102->regmap, reg, tmp102_mC_to_reg(temp));
 }
 
-static ssize_t tmp102_set_temp(struct device *dev,
-			       struct device_attribute *attr,
-			       const char *buf, size_t count)
+static umode_t tmp102_is_visible(const void *data, enum hwmon_sensor_types type,
+				 u32 attr, int channel)
 {
-	struct sensor_device_attribute *sda = to_sensor_dev_attr(attr);
-	struct tmp102 *tmp102 = dev_get_drvdata(dev);
-	int reg = sda->index;
-	long val;
-	int err;
-
-	if (kstrtol(buf, 10, &val) < 0)
-		return -EINVAL;
-	val = clamp_val(val, -256000, 255000);
-
-	err = regmap_write(tmp102->regmap, reg, tmp102_mC_to_reg(val));
-	return err ? : count;
+	if (type != hwmon_temp)
+		return 0;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		return S_IRUGO;
+	case hwmon_temp_max_hyst:
+	case hwmon_temp_max:
+		return S_IRUGO | S_IWUSR;
+	default:
+		return 0;
+	}
 }
 
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, tmp102_show_temp, NULL,
-			  TMP102_TEMP_REG);
+static u32 tmp102_chip_config[] = {
+	HWMON_C_REGISTER_TZ,
+	0
+};
+
+static const struct hwmon_channel_info tmp102_chip = {
+	.type = hwmon_chip,
+	.config = tmp102_chip_config,
+};
 
-static SENSOR_DEVICE_ATTR(temp1_max_hyst, S_IWUSR | S_IRUGO, tmp102_show_temp,
-			  tmp102_set_temp, TMP102_TLOW_REG);
+static u32 tmp102_temp_config[] = {
+	HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MAX_HYST,
+	0
+};
 
-static SENSOR_DEVICE_ATTR(temp1_max, S_IWUSR | S_IRUGO, tmp102_show_temp,
-			  tmp102_set_temp, TMP102_THIGH_REG);
+static const struct hwmon_channel_info tmp102_temp = {
+	.type = hwmon_temp,
+	.config = tmp102_temp_config,
+};
 
-static struct attribute *tmp102_attrs[] = {
-	&sensor_dev_attr_temp1_input.dev_attr.attr,
-	&sensor_dev_attr_temp1_max_hyst.dev_attr.attr,
-	&sensor_dev_attr_temp1_max.dev_attr.attr,
+static const struct hwmon_channel_info *tmp102_info[] = {
+	&tmp102_chip,
+	&tmp102_temp,
 	NULL
 };
-ATTRIBUTE_GROUPS(tmp102);
 
-static const struct thermal_zone_of_device_ops tmp102_of_thermal_ops = {
-	.get_temp = tmp102_read_temp,
+static const struct hwmon_ops tmp102_hwmon_ops = {
+	.is_visible = tmp102_is_visible,
+	.read = tmp102_read,
+	.write = tmp102_write,
+};
+
+static const struct hwmon_chip_info tmp102_chip_info = {
+	.ops = &tmp102_hwmon_ops,
+	.info = tmp102_info,
 };
 
 static void tmp102_restore_config(void *data)
@@ -188,7 +216,7 @@ static const struct regmap_config tmp102_regmap_config = {
 };
 
 static int tmp102_probe(struct i2c_client *client,
-				  const struct i2c_device_id *id)
+			const struct i2c_device_id *id)
 {
 	struct device *dev = &client->dev;
 	struct device *hwmon_dev;
@@ -249,16 +277,14 @@ static int tmp102_probe(struct i2c_client *client,
 		tmp102->ready_time += msecs_to_jiffies(CONVERSION_TIME_MS);
 	}
 
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
-							   tmp102,
-							   tmp102_groups);
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 tmp102,
+							 &tmp102_chip_info,
+							 NULL);
 	if (IS_ERR(hwmon_dev)) {
 		dev_dbg(dev, "unable to register hwmon device\n");
 		return PTR_ERR(hwmon_dev);
 	}
-	devm_thermal_zone_of_sensor_register(hwmon_dev, 0, hwmon_dev,
-					     &tmp102_of_thermal_ops);
-
 	dev_info(dev, "initialized\n");
 
 	return 0;
diff --git a/drivers/hwmon/tmp421.c b/drivers/hwmon/tmp421.c
index 85d48d80822a..bfb98b96c781 100644
--- a/drivers/hwmon/tmp421.c
+++ b/drivers/hwmon/tmp421.c
@@ -72,6 +72,10 @@ MODULE_DEVICE_TABLE(i2c, tmp421_id);
 struct tmp421_data {
 	struct i2c_client *client;
 	struct mutex update_lock;
+	u32 temp_config[5];
+	struct hwmon_channel_info temp_info;
+	const struct hwmon_channel_info *info[2];
+	struct hwmon_chip_info chip;
 	char valid;
 	unsigned long last_updated;
 	int channels;
@@ -125,85 +129,46 @@ static struct tmp421_data *tmp421_update_device(struct device *dev)
 	return data;
 }
 
-static ssize_t show_temp_value(struct device *dev,
-			       struct device_attribute *devattr, char *buf)
+static int tmp421_read(struct device *dev, enum hwmon_sensor_types type,
+		       u32 attr, int channel, long *val)
 {
-	int index = to_sensor_dev_attr(devattr)->index;
-	struct tmp421_data *data = tmp421_update_device(dev);
-	int temp;
-
-	mutex_lock(&data->update_lock);
-	if (data->config & TMP421_CONFIG_RANGE)
-		temp = temp_from_u16(data->temp[index]);
-	else
-		temp = temp_from_s16(data->temp[index]);
-	mutex_unlock(&data->update_lock);
-
-	return sprintf(buf, "%d\n", temp);
-}
+	struct tmp421_data *tmp421 = tmp421_update_device(dev);
+
+	switch (attr) {
+	case hwmon_temp_input:
+		if (tmp421->config & TMP421_CONFIG_RANGE)
+			*val = temp_from_u16(tmp421->temp[channel]);
+		else
+			*val = temp_from_s16(tmp421->temp[channel]);
+		return 0;
+	case hwmon_temp_fault:
+		/*
+		 * The OPEN bit signals a fault. This is bit 0 of the temperature
+		 * register (low byte).
+		 */
+		*val = tmp421->temp[channel] & 0x01;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
 
-static ssize_t show_fault(struct device *dev,
-			  struct device_attribute *devattr, char *buf)
-{
-	int index = to_sensor_dev_attr(devattr)->index;
-	struct tmp421_data *data = tmp421_update_device(dev);
-
-	/*
-	 * The OPEN bit signals a fault. This is bit 0 of the temperature
-	 * register (low byte).
-	 */
-	if (data->temp[index] & 0x01)
-		return sprintf(buf, "1\n");
-	else
-		return sprintf(buf, "0\n");
 }
 
-static umode_t tmp421_is_visible(struct kobject *kobj, struct attribute *a,
-				int n)
+static umode_t tmp421_is_visible(const void *data, enum hwmon_sensor_types type,
+				 u32 attr, int channel)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
-	struct tmp421_data *data = dev_get_drvdata(dev);
-	struct device_attribute *devattr;
-	unsigned int index;
-
-	devattr = container_of(a, struct device_attribute, attr);
-	index = to_sensor_dev_attr(devattr)->index;
-
-	if (index < data->channels)
-		return a->mode;
-
-	return 0;
+	switch (attr) {
+	case hwmon_temp_fault:
+		if (channel == 0)
+			return 0;
+		return S_IRUGO;
+	case hwmon_temp_input:
+		return S_IRUGO;
+	default:
+		return 0;
+	}
 }
 
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, show_temp_value, NULL, 0);
-static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, show_temp_value, NULL, 1);
-static SENSOR_DEVICE_ATTR(temp2_fault, S_IRUGO, show_fault, NULL, 1);
-static SENSOR_DEVICE_ATTR(temp3_input, S_IRUGO, show_temp_value, NULL, 2);
-static SENSOR_DEVICE_ATTR(temp3_fault, S_IRUGO, show_fault, NULL, 2);
-static SENSOR_DEVICE_ATTR(temp4_input, S_IRUGO, show_temp_value, NULL, 3);
-static SENSOR_DEVICE_ATTR(temp4_fault, S_IRUGO, show_fault, NULL, 3);
-
-static struct attribute *tmp421_attr[] = {
-	&sensor_dev_attr_temp1_input.dev_attr.attr,
-	&sensor_dev_attr_temp2_input.dev_attr.attr,
-	&sensor_dev_attr_temp2_fault.dev_attr.attr,
-	&sensor_dev_attr_temp3_input.dev_attr.attr,
-	&sensor_dev_attr_temp3_fault.dev_attr.attr,
-	&sensor_dev_attr_temp4_input.dev_attr.attr,
-	&sensor_dev_attr_temp4_fault.dev_attr.attr,
-	NULL
-};
-
-static const struct attribute_group tmp421_group = {
-	.attrs = tmp421_attr,
-	.is_visible = tmp421_is_visible,
-};
-
-static const struct attribute_group *tmp421_groups[] = {
-	&tmp421_group,
-	NULL
-};
-
 static int tmp421_init_client(struct i2c_client *client)
 {
 	int config, config_orig;
@@ -289,13 +254,18 @@ static int tmp421_detect(struct i2c_client *client,
 	return 0;
 }
 
+static const struct hwmon_ops tmp421_ops = {
+	.is_visible = tmp421_is_visible,
+	.read = tmp421_read,
+};
+
 static int tmp421_probe(struct i2c_client *client,
 			const struct i2c_device_id *id)
 {
 	struct device *dev = &client->dev;
 	struct device *hwmon_dev;
 	struct tmp421_data *data;
-	int err;
+	int i, err;
 
 	data = devm_kzalloc(dev, sizeof(struct tmp421_data), GFP_KERNEL);
 	if (!data)
@@ -309,8 +279,21 @@ static int tmp421_probe(struct i2c_client *client,
 	if (err)
 		return err;
 
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
-							   data, tmp421_groups);
+	for (i = 0; i < data->channels; i++)
+		data->temp_config[i] = HWMON_T_INPUT | HWMON_T_FAULT;
+
+	data->chip.ops = &tmp421_ops;
+	data->chip.info = data->info;
+
+	data->info[0] = &data->temp_info;
+
+	data->temp_info.type = hwmon_temp;
+	data->temp_info.config = data->temp_config;
+
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 data,
+							 &data->chip,
+							 NULL);
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
diff --git a/drivers/hwmon/xgene-hwmon.c b/drivers/hwmon/xgene-hwmon.c
new file mode 100644
index 000000000000..9c0dbb8191ad
--- /dev/null
+++ b/drivers/hwmon/xgene-hwmon.c
@@ -0,0 +1,787 @@
+/*
+ * APM X-Gene SoC Hardware Monitoring Driver
+ *
+ * Copyright (c) 2016, Applied Micro Circuits Corporation
+ * Author: Loc Ho <lho@apm.com>
+ *         Hoan Tran <hotran@apm.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * This driver provides the following features:
+ *  - Retrieve CPU total power (uW)
+ *  - Retrieve IO total power (uW)
+ *  - Retrieve SoC temperature (milli-degree C) and alarm
+ */
+#include <linux/acpi.h>
+#include <linux/dma-mapping.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/kfifo.h>
+#include <linux/mailbox_controller.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include <acpi/pcc.h>
+
+/* SLIMpro message defines */
+#define MSG_TYPE_DBG			0
+#define MSG_TYPE_ERR			7
+#define MSG_TYPE_PWRMGMT		9
+
+#define MSG_TYPE(v)			(((v) & 0xF0000000) >> 28)
+#define MSG_TYPE_SET(v)			(((v) << 28) & 0xF0000000)
+#define MSG_SUBTYPE(v)			(((v) & 0x0F000000) >> 24)
+#define MSG_SUBTYPE_SET(v)		(((v) << 24) & 0x0F000000)
+
+#define DBG_SUBTYPE_SENSOR_READ		4
+#define SENSOR_RD_MSG			0x04FFE902
+#define SENSOR_RD_EN_ADDR(a)		((a) & 0x000FFFFF)
+#define PMD_PWR_REG			0x20
+#define PMD_PWR_MW_REG			0x26
+#define SOC_PWR_REG			0x21
+#define SOC_PWR_MW_REG			0x27
+#define SOC_TEMP_REG			0x10
+
+#define TEMP_NEGATIVE_BIT		8
+#define SENSOR_INVALID_DATA		BIT(15)
+
+#define PWRMGMT_SUBTYPE_TPC		1
+#define TPC_ALARM			2
+#define TPC_GET_ALARM			3
+#define TPC_CMD(v)			(((v) & 0x00FF0000) >> 16)
+#define TPC_CMD_SET(v)			(((v) << 16) & 0x00FF0000)
+#define TPC_EN_MSG(hndl, cmd, type) \
+	(MSG_TYPE_SET(MSG_TYPE_PWRMGMT) | \
+	MSG_SUBTYPE_SET(hndl) | TPC_CMD_SET(cmd) | type)
+
+/* PCC defines */
+#define PCC_SIGNATURE_MASK		0x50424300
+#define PCCC_GENERATE_DB_INT		BIT(15)
+#define PCCS_CMD_COMPLETE		BIT(0)
+#define PCCS_SCI_DOORBEL		BIT(1)
+#define PCCS_PLATFORM_NOTIFICATION	BIT(3)
+/*
+ * Arbitrary retries in case the remote processor is slow to respond
+ * to PCC commands
+ */
+#define PCC_NUM_RETRIES			500
+
+#define ASYNC_MSG_FIFO_SIZE		16
+#define MBOX_OP_TIMEOUTMS		1000
+
+#define WATT_TO_mWATT(x)		((x) * 1000)
+#define mWATT_TO_uWATT(x)		((x) * 1000)
+#define CELSIUS_TO_mCELSIUS(x)		((x) * 1000)
+
+#define to_xgene_hwmon_dev(cl)		\
+	container_of(cl, struct xgene_hwmon_dev, mbox_client)
+
+struct slimpro_resp_msg {
+	u32 msg;
+	u32 param1;
+	u32 param2;
+} __packed;
+
+struct xgene_hwmon_dev {
+	struct device		*dev;
+	struct mbox_chan	*mbox_chan;
+	struct mbox_client	mbox_client;
+	int			mbox_idx;
+
+	spinlock_t		kfifo_lock;
+	struct mutex		rd_mutex;
+	struct completion	rd_complete;
+	int			resp_pending;
+	struct slimpro_resp_msg sync_msg;
+
+	struct work_struct	workq;
+	struct kfifo_rec_ptr_1	async_msg_fifo;
+
+	struct device		*hwmon_dev;
+	bool			temp_critical_alarm;
+
+	phys_addr_t		comm_base_addr;
+	void			*pcc_comm_addr;
+	u64			usecs_lat;
+};
+
+/*
+ * This function tests and clears a bitmask then returns its old value
+ */
+static u16 xgene_word_tst_and_clr(u16 *addr, u16 mask)
+{
+	u16 ret, val;
+
+	val = le16_to_cpu(READ_ONCE(*addr));
+	ret = val & mask;
+	val &= ~mask;
+	WRITE_ONCE(*addr, cpu_to_le16(val));
+
+	return ret;
+}
+
+static int xgene_hwmon_pcc_rd(struct xgene_hwmon_dev *ctx, u32 *msg)
+{
+	struct acpi_pcct_shared_memory *generic_comm_base = ctx->pcc_comm_addr;
+	u32 *ptr = (void *)(generic_comm_base + 1);
+	int rc, i;
+	u16 val;
+
+	mutex_lock(&ctx->rd_mutex);
+	init_completion(&ctx->rd_complete);
+	ctx->resp_pending = true;
+
+	/* Write signature for subspace */
+	WRITE_ONCE(generic_comm_base->signature,
+		   cpu_to_le32(PCC_SIGNATURE_MASK | ctx->mbox_idx));
+
+	/* Write to the shared command region */
+	WRITE_ONCE(generic_comm_base->command,
+		   cpu_to_le16(MSG_TYPE(msg[0]) | PCCC_GENERATE_DB_INT));
+
+	/* Flip CMD COMPLETE bit */
+	val = le16_to_cpu(READ_ONCE(generic_comm_base->status));
+	val &= ~PCCS_CMD_COMPLETE;
+	WRITE_ONCE(generic_comm_base->status, cpu_to_le16(val));
+
+	/* Copy the message to the PCC comm space */
+	for (i = 0; i < sizeof(struct slimpro_resp_msg) / 4; i++)
+		WRITE_ONCE(ptr[i], cpu_to_le32(msg[i]));
+
+	/* Ring the doorbell */
+	rc = mbox_send_message(ctx->mbox_chan, msg);
+	if (rc < 0) {
+		dev_err(ctx->dev, "Mailbox send error %d\n", rc);
+		goto err;
+	}
+	if (!wait_for_completion_timeout(&ctx->rd_complete,
+					 usecs_to_jiffies(ctx->usecs_lat))) {
+		dev_err(ctx->dev, "Mailbox operation timed out\n");
+		rc = -ETIMEDOUT;
+		goto err;
+	}
+
+	/* Check for error message */
+	if (MSG_TYPE(ctx->sync_msg.msg) == MSG_TYPE_ERR) {
+		rc = -EINVAL;
+		goto err;
+	}
+
+	msg[0] = ctx->sync_msg.msg;
+	msg[1] = ctx->sync_msg.param1;
+	msg[2] = ctx->sync_msg.param2;
+
+err:
+	mbox_chan_txdone(ctx->mbox_chan, 0);
+	ctx->resp_pending = false;
+	mutex_unlock(&ctx->rd_mutex);
+	return rc;
+}
+
+static int xgene_hwmon_rd(struct xgene_hwmon_dev *ctx, u32 *msg)
+{
+	int rc;
+
+	mutex_lock(&ctx->rd_mutex);
+	init_completion(&ctx->rd_complete);
+	ctx->resp_pending = true;
+
+	rc = mbox_send_message(ctx->mbox_chan, msg);
+	if (rc < 0) {
+		dev_err(ctx->dev, "Mailbox send error %d\n", rc);
+		goto err;
+	}
+
+	if (!wait_for_completion_timeout(&ctx->rd_complete,
+					 msecs_to_jiffies(MBOX_OP_TIMEOUTMS))) {
+		dev_err(ctx->dev, "Mailbox operation timed out\n");
+		rc = -ETIMEDOUT;
+		goto err;
+	}
+
+	/* Check for error message */
+	if (MSG_TYPE(ctx->sync_msg.msg) == MSG_TYPE_ERR) {
+		rc = -EINVAL;
+		goto err;
+	}
+
+	msg[0] = ctx->sync_msg.msg;
+	msg[1] = ctx->sync_msg.param1;
+	msg[2] = ctx->sync_msg.param2;
+
+err:
+	ctx->resp_pending = false;
+	mutex_unlock(&ctx->rd_mutex);
+	return rc;
+}
+
+static int xgene_hwmon_reg_map_rd(struct xgene_hwmon_dev *ctx, u32 addr,
+				  u32 *data)
+{
+	u32 msg[3];
+	int rc;
+
+	msg[0] = SENSOR_RD_MSG;
+	msg[1] = SENSOR_RD_EN_ADDR(addr);
+	msg[2] = 0;
+
+	if (acpi_disabled)
+		rc = xgene_hwmon_rd(ctx, msg);
+	else
+		rc = xgene_hwmon_pcc_rd(ctx, msg);
+
+	if (rc < 0)
+		return rc;
+
+	/*
+	 * Check if sensor data is valid.
+	 */
+	if (msg[1] & SENSOR_INVALID_DATA)
+		return -ENODATA;
+
+	*data = msg[1];
+
+	return rc;
+}
+
+static int xgene_hwmon_get_notification_msg(struct xgene_hwmon_dev *ctx,
+					    u32 *amsg)
+{
+	u32 msg[3];
+	int rc;
+
+	msg[0] = TPC_EN_MSG(PWRMGMT_SUBTYPE_TPC, TPC_GET_ALARM, 0);
+	msg[1] = 0;
+	msg[2] = 0;
+
+	rc = xgene_hwmon_pcc_rd(ctx, msg);
+	if (rc < 0)
+		return rc;
+
+	amsg[0] = msg[0];
+	amsg[1] = msg[1];
+	amsg[2] = msg[2];
+
+	return rc;
+}
+
+static int xgene_hwmon_get_cpu_pwr(struct xgene_hwmon_dev *ctx, u32 *val)
+{
+	u32 watt, mwatt;
+	int rc;
+
+	rc = xgene_hwmon_reg_map_rd(ctx, PMD_PWR_REG, &watt);
+	if (rc < 0)
+		return rc;
+
+	rc = xgene_hwmon_reg_map_rd(ctx, PMD_PWR_MW_REG, &mwatt);
+	if (rc < 0)
+		return rc;
+
+	*val = WATT_TO_mWATT(watt) + mwatt;
+	return 0;
+}
+
+static int xgene_hwmon_get_io_pwr(struct xgene_hwmon_dev *ctx, u32 *val)
+{
+	u32 watt, mwatt;
+	int rc;
+
+	rc = xgene_hwmon_reg_map_rd(ctx, SOC_PWR_REG, &watt);
+	if (rc < 0)
+		return rc;
+
+	rc = xgene_hwmon_reg_map_rd(ctx, SOC_PWR_MW_REG, &mwatt);
+	if (rc < 0)
+		return rc;
+
+	*val = WATT_TO_mWATT(watt) + mwatt;
+	return 0;
+}
+
+static int xgene_hwmon_get_temp(struct xgene_hwmon_dev *ctx, u32 *val)
+{
+	return xgene_hwmon_reg_map_rd(ctx, SOC_TEMP_REG, val);
+}
+
+/*
+ * Sensor temperature/power functions
+ */
+static ssize_t temp1_input_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct xgene_hwmon_dev *ctx = dev_get_drvdata(dev);
+	int rc, temp;
+	u32 val;
+
+	rc = xgene_hwmon_get_temp(ctx, &val);
+	if (rc < 0)
+		return rc;
+
+	temp = sign_extend32(val, TEMP_NEGATIVE_BIT);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", CELSIUS_TO_mCELSIUS(temp));
+}
+
+static ssize_t temp1_label_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "SoC Temperature\n");
+}
+
+static ssize_t temp1_critical_alarm_show(struct device *dev,
+					 struct device_attribute *devattr,
+					 char *buf)
+{
+	struct xgene_hwmon_dev *ctx = dev_get_drvdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", ctx->temp_critical_alarm);
+}
+
+static ssize_t power1_label_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "CPU power\n");
+}
+
+static ssize_t power2_label_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "IO power\n");
+}
+
+static ssize_t power1_input_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct xgene_hwmon_dev *ctx = dev_get_drvdata(dev);
+	u32 val;
+	int rc;
+
+	rc = xgene_hwmon_get_cpu_pwr(ctx, &val);
+	if (rc < 0)
+		return rc;
+
+	return snprintf(buf, PAGE_SIZE, "%u\n", mWATT_TO_uWATT(val));
+}
+
+static ssize_t power2_input_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct xgene_hwmon_dev *ctx = dev_get_drvdata(dev);
+	u32 val;
+	int rc;
+
+	rc = xgene_hwmon_get_io_pwr(ctx, &val);
+	if (rc < 0)
+		return rc;
+
+	return snprintf(buf, PAGE_SIZE, "%u\n", mWATT_TO_uWATT(val));
+}
+
+static DEVICE_ATTR_RO(temp1_label);
+static DEVICE_ATTR_RO(temp1_input);
+static DEVICE_ATTR_RO(temp1_critical_alarm);
+static DEVICE_ATTR_RO(power1_label);
+static DEVICE_ATTR_RO(power1_input);
+static DEVICE_ATTR_RO(power2_label);
+static DEVICE_ATTR_RO(power2_input);
+
+static struct attribute *xgene_hwmon_attrs[] = {
+	&dev_attr_temp1_label.attr,
+	&dev_attr_temp1_input.attr,
+	&dev_attr_temp1_critical_alarm.attr,
+	&dev_attr_power1_label.attr,
+	&dev_attr_power1_input.attr,
+	&dev_attr_power2_label.attr,
+	&dev_attr_power2_input.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(xgene_hwmon);
+
+static int xgene_hwmon_tpc_alarm(struct xgene_hwmon_dev *ctx,
+				 struct slimpro_resp_msg *amsg)
+{
+	ctx->temp_critical_alarm = !!amsg->param2;
+	sysfs_notify(&ctx->dev->kobj, NULL, "temp1_critical_alarm");
+
+	return 0;
+}
+
+static void xgene_hwmon_process_pwrmsg(struct xgene_hwmon_dev *ctx,
+				       struct slimpro_resp_msg *amsg)
+{
+	if ((MSG_SUBTYPE(amsg->msg) == PWRMGMT_SUBTYPE_TPC) &&
+	    (TPC_CMD(amsg->msg) == TPC_ALARM))
+		xgene_hwmon_tpc_alarm(ctx, amsg);
+}
+
+/*
+ * This function is called to process async work queue
+ */
+static void xgene_hwmon_evt_work(struct work_struct *work)
+{
+	struct slimpro_resp_msg amsg;
+	struct xgene_hwmon_dev *ctx;
+	int ret;
+
+	ctx = container_of(work, struct xgene_hwmon_dev, workq);
+	while (kfifo_out_spinlocked(&ctx->async_msg_fifo, &amsg,
+				    sizeof(struct slimpro_resp_msg),
+				    &ctx->kfifo_lock)) {
+		/*
+		 * If PCC, send a consumer command to Platform to get info
+		 * If Slimpro Mailbox, get message from specific FIFO
+		 */
+		if (!acpi_disabled) {
+			ret = xgene_hwmon_get_notification_msg(ctx,
+							       (u32 *)&amsg);
+			if (ret < 0)
+				continue;
+		}
+
+		if (MSG_TYPE(amsg.msg) == MSG_TYPE_PWRMGMT)
+			xgene_hwmon_process_pwrmsg(ctx, &amsg);
+	}
+}
+
+static int xgene_hwmon_rx_ready(struct xgene_hwmon_dev *ctx, void *msg)
+{
+	if (IS_ERR_OR_NULL(ctx->hwmon_dev) && !ctx->resp_pending) {
+		/* Enqueue to the FIFO */
+		kfifo_in_spinlocked(&ctx->async_msg_fifo, msg,
+				    sizeof(struct slimpro_resp_msg),
+				    &ctx->kfifo_lock);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+/*
+ * This function is called when the SLIMpro Mailbox received a message
+ */
+static void xgene_hwmon_rx_cb(struct mbox_client *cl, void *msg)
+{
+	struct xgene_hwmon_dev *ctx = to_xgene_hwmon_dev(cl);
+
+	/*
+	 * While the driver registers with the mailbox framework, an interrupt
+	 * can be pending before the probe function completes its
+	 * initialization. If such condition occurs, just queue up the message
+	 * as the driver is not ready for servicing the callback.
+	 */
+	if (xgene_hwmon_rx_ready(ctx, msg) < 0)
+		return;
+
+	/*
+	 * Response message format:
+	 * msg[0] is the return code of the operation
+	 * msg[1] is the first parameter word
+	 * msg[2] is the second parameter word
+	 *
+	 * As message only supports dword size, just assign it.
+	 */
+
+	/* Check for sync query */
+	if (ctx->resp_pending &&
+	    ((MSG_TYPE(((u32 *)msg)[0]) == MSG_TYPE_ERR) ||
+	     (MSG_TYPE(((u32 *)msg)[0]) == MSG_TYPE_DBG &&
+	      MSG_SUBTYPE(((u32 *)msg)[0]) == DBG_SUBTYPE_SENSOR_READ) ||
+	     (MSG_TYPE(((u32 *)msg)[0]) == MSG_TYPE_PWRMGMT &&
+	      MSG_SUBTYPE(((u32 *)msg)[0]) == PWRMGMT_SUBTYPE_TPC &&
+	      TPC_CMD(((u32 *)msg)[0]) == TPC_ALARM))) {
+		ctx->sync_msg.msg = ((u32 *)msg)[0];
+		ctx->sync_msg.param1 = ((u32 *)msg)[1];
+		ctx->sync_msg.param2 = ((u32 *)msg)[2];
+
+		/* Operation waiting for response */
+		complete(&ctx->rd_complete);
+
+		return;
+	}
+
+	/* Enqueue to the FIFO */
+	kfifo_in_spinlocked(&ctx->async_msg_fifo, msg,
+			    sizeof(struct slimpro_resp_msg), &ctx->kfifo_lock);
+	/* Schedule the bottom handler */
+	schedule_work(&ctx->workq);
+}
+
+/*
+ * This function is called when the PCC Mailbox received a message
+ */
+static void xgene_hwmon_pcc_rx_cb(struct mbox_client *cl, void *msg)
+{
+	struct xgene_hwmon_dev *ctx = to_xgene_hwmon_dev(cl);
+	struct acpi_pcct_shared_memory *generic_comm_base = ctx->pcc_comm_addr;
+	struct slimpro_resp_msg amsg;
+
+	/*
+	 * While the driver registers with the mailbox framework, an interrupt
+	 * can be pending before the probe function completes its
+	 * initialization. If such condition occurs, just queue up the message
+	 * as the driver is not ready for servicing the callback.
+	 */
+	if (xgene_hwmon_rx_ready(ctx, &amsg) < 0)
+		return;
+
+	msg = generic_comm_base + 1;
+	/* Check if platform sends interrupt */
+	if (!xgene_word_tst_and_clr(&generic_comm_base->status,
+				    PCCS_SCI_DOORBEL))
+		return;
+
+	/*
+	 * Response message format:
+	 * msg[0] is the return code of the operation
+	 * msg[1] is the first parameter word
+	 * msg[2] is the second parameter word
+	 *
+	 * As message only supports dword size, just assign it.
+	 */
+
+	/* Check for sync query */
+	if (ctx->resp_pending &&
+	    ((MSG_TYPE(((u32 *)msg)[0]) == MSG_TYPE_ERR) ||
+	     (MSG_TYPE(((u32 *)msg)[0]) == MSG_TYPE_DBG &&
+	      MSG_SUBTYPE(((u32 *)msg)[0]) == DBG_SUBTYPE_SENSOR_READ) ||
+	     (MSG_TYPE(((u32 *)msg)[0]) == MSG_TYPE_PWRMGMT &&
+	      MSG_SUBTYPE(((u32 *)msg)[0]) == PWRMGMT_SUBTYPE_TPC &&
+	      TPC_CMD(((u32 *)msg)[0]) == TPC_ALARM))) {
+		/* Check if platform completes command */
+		if (xgene_word_tst_and_clr(&generic_comm_base->status,
+					   PCCS_CMD_COMPLETE)) {
+			ctx->sync_msg.msg = ((u32 *)msg)[0];
+			ctx->sync_msg.param1 = ((u32 *)msg)[1];
+			ctx->sync_msg.param2 = ((u32 *)msg)[2];
+
+			/* Operation waiting for response */
+			complete(&ctx->rd_complete);
+
+			return;
+		}
+	}
+
+	/*
+	 * Platform notifies interrupt to OSPM.
+	 * OPSM schedules a consumer command to get this information
+	 * in a workqueue. Platform must wait until OSPM has issued
+	 * a consumer command that serves this notification.
+	 */
+
+	/* Enqueue to the FIFO */
+	kfifo_in_spinlocked(&ctx->async_msg_fifo, &amsg,
+			    sizeof(struct slimpro_resp_msg), &ctx->kfifo_lock);
+	/* Schedule the bottom handler */
+	schedule_work(&ctx->workq);
+}
+
+static void xgene_hwmon_tx_done(struct mbox_client *cl, void *msg, int ret)
+{
+	if (ret) {
+		dev_dbg(cl->dev, "TX did not complete: CMD sent:%x, ret:%d\n",
+			*(u16 *)msg, ret);
+	} else {
+		dev_dbg(cl->dev, "TX completed. CMD sent:%x, ret:%d\n",
+			*(u16 *)msg, ret);
+	}
+}
+
+static int xgene_hwmon_probe(struct platform_device *pdev)
+{
+	struct xgene_hwmon_dev *ctx;
+	struct mbox_client *cl;
+	int rc;
+
+	ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->dev = &pdev->dev;
+	platform_set_drvdata(pdev, ctx);
+	cl = &ctx->mbox_client;
+
+	spin_lock_init(&ctx->kfifo_lock);
+	mutex_init(&ctx->rd_mutex);
+
+	rc = kfifo_alloc(&ctx->async_msg_fifo,
+			 sizeof(struct slimpro_resp_msg) * ASYNC_MSG_FIFO_SIZE,
+			 GFP_KERNEL);
+	if (rc)
+		goto out_mbox_free;
+
+	INIT_WORK(&ctx->workq, xgene_hwmon_evt_work);
+
+	/* Request mailbox channel */
+	cl->dev = &pdev->dev;
+	cl->tx_done = xgene_hwmon_tx_done;
+	cl->tx_block = false;
+	cl->tx_tout = MBOX_OP_TIMEOUTMS;
+	cl->knows_txdone = false;
+	if (acpi_disabled) {
+		cl->rx_callback = xgene_hwmon_rx_cb;
+		ctx->mbox_chan = mbox_request_channel(cl, 0);
+		if (IS_ERR(ctx->mbox_chan)) {
+			dev_err(&pdev->dev,
+				"SLIMpro mailbox channel request failed\n");
+			return -ENODEV;
+		}
+	} else {
+		struct acpi_pcct_hw_reduced *cppc_ss;
+
+		if (device_property_read_u32(&pdev->dev, "pcc-channel",
+					     &ctx->mbox_idx)) {
+			dev_err(&pdev->dev, "no pcc-channel property\n");
+			return -ENODEV;
+		}
+
+		cl->rx_callback = xgene_hwmon_pcc_rx_cb;
+		ctx->mbox_chan = pcc_mbox_request_channel(cl, ctx->mbox_idx);
+		if (IS_ERR(ctx->mbox_chan)) {
+			dev_err(&pdev->dev,
+				"PPC channel request failed\n");
+			return -ENODEV;
+		}
+
+		/*
+		 * The PCC mailbox controller driver should
+		 * have parsed the PCCT (global table of all
+		 * PCC channels) and stored pointers to the
+		 * subspace communication region in con_priv.
+		 */
+		cppc_ss = ctx->mbox_chan->con_priv;
+		if (!cppc_ss) {
+			dev_err(&pdev->dev, "PPC subspace not found\n");
+			rc = -ENODEV;
+			goto out_mbox_free;
+		}
+
+		if (!ctx->mbox_chan->mbox->txdone_irq) {
+			dev_err(&pdev->dev, "PCC IRQ not supported\n");
+			rc = -ENODEV;
+			goto out_mbox_free;
+		}
+
+		/*
+		 * This is the shared communication region
+		 * for the OS and Platform to communicate over.
+		 */
+		ctx->comm_base_addr = cppc_ss->base_address;
+		if (ctx->comm_base_addr) {
+			ctx->pcc_comm_addr = memremap(ctx->comm_base_addr,
+							cppc_ss->length,
+							MEMREMAP_WB);
+		} else {
+			dev_err(&pdev->dev, "Failed to get PCC comm region\n");
+			rc = -ENODEV;
+			goto out_mbox_free;
+		}
+
+		if (!ctx->pcc_comm_addr) {
+			dev_err(&pdev->dev,
+				"Failed to ioremap PCC comm region\n");
+			rc = -ENOMEM;
+			goto out_mbox_free;
+		}
+
+		/*
+		 * cppc_ss->latency is just a Nominal value. In reality
+		 * the remote processor could be much slower to reply.
+		 * So add an arbitrary amount of wait on top of Nominal.
+		 */
+		ctx->usecs_lat = PCC_NUM_RETRIES * cppc_ss->latency;
+	}
+
+	ctx->hwmon_dev = hwmon_device_register_with_groups(ctx->dev,
+							   "apm_xgene",
+							   ctx,
+							   xgene_hwmon_groups);
+	if (IS_ERR(ctx->hwmon_dev)) {
+		dev_err(&pdev->dev, "Failed to register HW monitor device\n");
+		rc = PTR_ERR(ctx->hwmon_dev);
+		goto out;
+	}
+
+	/*
+	 * Schedule the bottom handler if there is a pending message.
+	 */
+	schedule_work(&ctx->workq);
+
+	dev_info(&pdev->dev, "APM X-Gene SoC HW monitor driver registered\n");
+
+	return 0;
+
+out:
+	if (acpi_disabled)
+		mbox_free_channel(ctx->mbox_chan);
+	else
+		pcc_mbox_free_channel(ctx->mbox_chan);
+out_mbox_free:
+	kfifo_free(&ctx->async_msg_fifo);
+
+	return rc;
+}
+
+static int xgene_hwmon_remove(struct platform_device *pdev)
+{
+	struct xgene_hwmon_dev *ctx = platform_get_drvdata(pdev);
+
+	hwmon_device_unregister(ctx->hwmon_dev);
+	kfifo_free(&ctx->async_msg_fifo);
+	if (acpi_disabled)
+		mbox_free_channel(ctx->mbox_chan);
+	else
+		pcc_mbox_free_channel(ctx->mbox_chan);
+
+	return 0;
+}
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id xgene_hwmon_acpi_match[] = {
+	{"APMC0D29", 0},
+	{},
+};
+MODULE_DEVICE_TABLE(acpi, xgene_hwmon_acpi_match);
+#endif
+
+static const struct of_device_id xgene_hwmon_of_match[] = {
+	{.compatible = "apm,xgene-slimpro-hwmon"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, xgene_hwmon_of_match);
+
+static struct platform_driver xgene_hwmon_driver __refdata = {
+	.probe = xgene_hwmon_probe,
+	.remove = xgene_hwmon_remove,
+	.driver = {
+		.name = "xgene-slimpro-hwmon",
+		.of_match_table = xgene_hwmon_of_match,
+		.acpi_match_table = ACPI_PTR(xgene_hwmon_acpi_match),
+	},
+};
+module_platform_driver(xgene_hwmon_driver);
+
+MODULE_DESCRIPTION("APM X-Gene SoC hardware monitor");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c
index 4d20b0be0c0b..d7325c6534ad 100644
--- a/drivers/hwtracing/coresight/coresight-etb10.c
+++ b/drivers/hwtracing/coresight/coresight-etb10.c
@@ -184,8 +184,7 @@ static void etb_disable_hw(struct etb_drvdata *drvdata)
 
 	if (coresight_timeout(drvdata->base, ETB_FFCR, ETB_FFCR_BIT, 0)) {
 		dev_err(drvdata->dev,
-			"timeout observed when probing at offset %#x\n",
-			ETB_FFCR);
+		"timeout while waiting for completion of Manual Flush\n");
 	}
 
 	/* disable trace capture */
@@ -193,8 +192,7 @@ static void etb_disable_hw(struct etb_drvdata *drvdata)
 
 	if (coresight_timeout(drvdata->base, ETB_FFSR, ETB_FFSR_BIT, 1)) {
 		dev_err(drvdata->dev,
-			"timeout observed when probing at offset %#x\n",
-			ETB_FFCR);
+			"timeout while waiting for Formatter to Stop\n");
 	}
 
 	CS_LOCK(drvdata->base);
@@ -561,7 +559,7 @@ static const struct file_operations etb_fops = {
 };
 
 #define coresight_etb10_simple_func(name, offset)                       \
-	coresight_simple_func(struct etb_drvdata, name, offset)
+	coresight_simple_func(struct etb_drvdata, NULL, name, offset)
 
 coresight_etb10_simple_func(rdp, ETB_RAM_DEPTH_REG);
 coresight_etb10_simple_func(sts, ETB_STATUS_REG);
@@ -638,7 +636,7 @@ static int etb_probe(struct amba_device *adev, const struct amba_id *id)
 	struct coresight_platform_data *pdata = NULL;
 	struct etb_drvdata *drvdata;
 	struct resource *res = &adev->res;
-	struct coresight_desc *desc;
+	struct coresight_desc desc = { 0 };
 	struct device_node *np = adev->dev.of_node;
 
 	if (np) {
@@ -684,17 +682,13 @@ static int etb_probe(struct amba_device *adev, const struct amba_id *id)
 		return -ENOMEM;
 	}
 
-	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
-	desc->type = CORESIGHT_DEV_TYPE_SINK;
-	desc->subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
-	desc->ops = &etb_cs_ops;
-	desc->pdata = pdata;
-	desc->dev = dev;
-	desc->groups = coresight_etb_groups;
-	drvdata->csdev = coresight_register(desc);
+	desc.type = CORESIGHT_DEV_TYPE_SINK;
+	desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
+	desc.ops = &etb_cs_ops;
+	desc.pdata = pdata;
+	desc.dev = dev;
+	desc.groups = coresight_etb_groups;
+	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev))
 		return PTR_ERR(drvdata->csdev);
 
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c
index 755125f7917f..2cd7c718198a 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.c
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.c
@@ -27,6 +27,7 @@
 #include <linux/types.h>
 #include <linux/workqueue.h>
 
+#include "coresight-etm-perf.h"
 #include "coresight-priv.h"
 
 static struct pmu etm_pmu;
@@ -71,14 +72,48 @@ static const struct attribute_group *etm_pmu_attr_groups[] = {
 
 static void etm_event_read(struct perf_event *event) {}
 
-static int etm_event_init(struct perf_event *event)
+static int etm_addr_filters_alloc(struct perf_event *event)
 {
-	if (event->attr.type != etm_pmu.type)
-		return -ENOENT;
+	struct etm_filters *filters;
+	int node = event->cpu == -1 ? -1 : cpu_to_node(event->cpu);
+
+	filters = kzalloc_node(sizeof(struct etm_filters), GFP_KERNEL, node);
+	if (!filters)
+		return -ENOMEM;
+
+	if (event->parent)
+		memcpy(filters, event->parent->hw.addr_filters,
+		       sizeof(*filters));
+
+	event->hw.addr_filters = filters;
 
 	return 0;
 }
 
+static void etm_event_destroy(struct perf_event *event)
+{
+	kfree(event->hw.addr_filters);
+	event->hw.addr_filters = NULL;
+}
+
+static int etm_event_init(struct perf_event *event)
+{
+	int ret = 0;
+
+	if (event->attr.type != etm_pmu.type) {
+		ret = -ENOENT;
+		goto out;
+	}
+
+	ret = etm_addr_filters_alloc(event);
+	if (ret)
+		goto out;
+
+	event->destroy = etm_event_destroy;
+out:
+	return ret;
+}
+
 static void free_event_data(struct work_struct *work)
 {
 	int cpu;
@@ -100,7 +135,7 @@ static void free_event_data(struct work_struct *work)
 	}
 
 	for_each_cpu(cpu, mask) {
-		if (event_data->path[cpu])
+		if (!(IS_ERR_OR_NULL(event_data->path[cpu])))
 			coresight_release_path(event_data->path[cpu]);
 	}
 
@@ -185,7 +220,7 @@ static void *etm_setup_aux(int event_cpu, void **pages,
 		 * referenced later when the path is actually needed.
 		 */
 		event_data->path[cpu] = coresight_build_path(csdev);
-		if (!event_data->path[cpu])
+		if (IS_ERR(event_data->path[cpu]))
 			goto err;
 	}
 
@@ -258,7 +293,7 @@ static void etm_event_start(struct perf_event *event, int flags)
 	event->hw.state = 0;
 
 	/* Finally enable the tracer */
-	if (source_ops(csdev)->enable(csdev, &event->attr, CS_MODE_PERF))
+	if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF))
 		goto fail_end_stop;
 
 out:
@@ -291,7 +326,7 @@ static void etm_event_stop(struct perf_event *event, int mode)
 		return;
 
 	/* stop tracer */
-	source_ops(csdev)->disable(csdev);
+	source_ops(csdev)->disable(csdev, event);
 
 	/* tell the core */
 	event->hw.state = PERF_HES_STOPPED;
@@ -342,6 +377,87 @@ static void etm_event_del(struct perf_event *event, int mode)
 	etm_event_stop(event, PERF_EF_UPDATE);
 }
 
+static int etm_addr_filters_validate(struct list_head *filters)
+{
+	bool range = false, address = false;
+	int index = 0;
+	struct perf_addr_filter *filter;
+
+	list_for_each_entry(filter, filters, entry) {
+		/*
+		 * No need to go further if there's no more
+		 * room for filters.
+		 */
+		if (++index > ETM_ADDR_CMP_MAX)
+			return -EOPNOTSUPP;
+
+		/*
+		 * As taken from the struct perf_addr_filter documentation:
+		 *	@range:	1: range, 0: address
+		 *
+		 * At this time we don't allow range and start/stop filtering
+		 * to cohabitate, they have to be mutually exclusive.
+		 */
+		if ((filter->range == 1) && address)
+			return -EOPNOTSUPP;
+
+		if ((filter->range == 0) && range)
+			return -EOPNOTSUPP;
+
+		/*
+		 * For range filtering, the second address in the address
+		 * range comparator needs to be higher than the first.
+		 * Invalid otherwise.
+		 */
+		if (filter->range && filter->size == 0)
+			return -EINVAL;
+
+		/*
+		 * Everything checks out with this filter, record what we've
+		 * received before moving on to the next one.
+		 */
+		if (filter->range)
+			range = true;
+		else
+			address = true;
+	}
+
+	return 0;
+}
+
+static void etm_addr_filters_sync(struct perf_event *event)
+{
+	struct perf_addr_filters_head *head = perf_event_addr_filters(event);
+	unsigned long start, stop, *offs = event->addr_filters_offs;
+	struct etm_filters *filters = event->hw.addr_filters;
+	struct etm_filter *etm_filter;
+	struct perf_addr_filter *filter;
+	int i = 0;
+
+	list_for_each_entry(filter, &head->list, entry) {
+		start = filter->offset + offs[i];
+		stop = start + filter->size;
+		etm_filter = &filters->etm_filter[i];
+
+		if (filter->range == 1) {
+			etm_filter->start_addr = start;
+			etm_filter->stop_addr = stop;
+			etm_filter->type = ETM_ADDR_TYPE_RANGE;
+		} else {
+			if (filter->filter == 1) {
+				etm_filter->start_addr = start;
+				etm_filter->type = ETM_ADDR_TYPE_START;
+			} else {
+				etm_filter->stop_addr = stop;
+				etm_filter->type = ETM_ADDR_TYPE_STOP;
+			}
+		}
+		i++;
+	}
+
+	filters->nr_filters = i;
+}
+
 int etm_perf_symlink(struct coresight_device *csdev, bool link)
 {
 	char entry[sizeof("cpu9999999")];
@@ -371,18 +487,21 @@ static int __init etm_perf_init(void)
 {
 	int ret;
 
-	etm_pmu.capabilities	= PERF_PMU_CAP_EXCLUSIVE;
-
-	etm_pmu.attr_groups	= etm_pmu_attr_groups;
-	etm_pmu.task_ctx_nr	= perf_sw_context;
-	etm_pmu.read		= etm_event_read;
-	etm_pmu.event_init	= etm_event_init;
-	etm_pmu.setup_aux	= etm_setup_aux;
-	etm_pmu.free_aux	= etm_free_aux;
-	etm_pmu.start		= etm_event_start;
-	etm_pmu.stop		= etm_event_stop;
-	etm_pmu.add		= etm_event_add;
-	etm_pmu.del		= etm_event_del;
+	etm_pmu.capabilities		= PERF_PMU_CAP_EXCLUSIVE;
+
+	etm_pmu.attr_groups		= etm_pmu_attr_groups;
+	etm_pmu.task_ctx_nr		= perf_sw_context;
+	etm_pmu.read			= etm_event_read;
+	etm_pmu.event_init		= etm_event_init;
+	etm_pmu.setup_aux		= etm_setup_aux;
+	etm_pmu.free_aux		= etm_free_aux;
+	etm_pmu.start			= etm_event_start;
+	etm_pmu.stop			= etm_event_stop;
+	etm_pmu.add			= etm_event_add;
+	etm_pmu.del			= etm_event_del;
+	etm_pmu.addr_filters_sync	= etm_addr_filters_sync;
+	etm_pmu.addr_filters_validate	= etm_addr_filters_validate;
+	etm_pmu.nr_addr_filters		= ETM_ADDR_CMP_MAX;
 
 	ret = perf_pmu_register(&etm_pmu, CORESIGHT_ETM_PMU_NAME, -1);
 	if (ret == 0)
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.h b/drivers/hwtracing/coresight/coresight-etm-perf.h
index 87f5a134eb6f..3ffc9feb2d64 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.h
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.h
@@ -18,8 +18,42 @@
 #ifndef _CORESIGHT_ETM_PERF_H
 #define _CORESIGHT_ETM_PERF_H
 
+#include "coresight-priv.h"
+
 struct coresight_device;
 
+/*
+ * In both ETMv3 and v4 the maximum number of address comparator implentable
+ * is 8.  The actual number is implementation specific and will be checked
+ * when filters are applied.
+ */
+#define ETM_ADDR_CMP_MAX	8
+
+/**
+ * struct etm_filter - single instruction range or start/stop configuration.
+ * @start_addr:	The address to start tracing on.
+ * @stop_addr:	The address to stop tracing on.
+ * @type:	Is this a range or start/stop filter.
+ */
+struct etm_filter {
+	unsigned long start_addr;
+	unsigned long stop_addr;
+	enum etm_addr_type type;
+};
+
+/**
+ * struct etm_filters - set of filters for a session
+ * @etm_filter:	All the filters for this session.
+ * @nr_filters:	Number of filters
+ * @ssstatus:	Status of the start/stop logic.
+ */
+struct etm_filters {
+	struct etm_filter	etm_filter[ETM_ADDR_CMP_MAX];
+	unsigned int		nr_filters;
+	bool			ssstatus;
+};
+
+
 #ifdef CONFIG_CORESIGHT
 int etm_perf_symlink(struct coresight_device *csdev, bool link);
 
diff --git a/drivers/hwtracing/coresight/coresight-etm.h b/drivers/hwtracing/coresight/coresight-etm.h
index 51597cb2c08a..4a18ee499965 100644
--- a/drivers/hwtracing/coresight/coresight-etm.h
+++ b/drivers/hwtracing/coresight/coresight-etm.h
@@ -259,14 +259,6 @@ struct etm_drvdata {
 	struct etm_config		config;
 };
 
-enum etm_addr_type {
-	ETM_ADDR_TYPE_NONE,
-	ETM_ADDR_TYPE_SINGLE,
-	ETM_ADDR_TYPE_RANGE,
-	ETM_ADDR_TYPE_START,
-	ETM_ADDR_TYPE_STOP,
-};
-
 static inline void etm_writel(struct etm_drvdata *drvdata,
 			      u32 val, u32 off)
 {
diff --git a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
index 02d4b629891f..e9b071953f80 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
@@ -18,6 +18,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/sysfs.h>
 #include "coresight-etm.h"
+#include "coresight-priv.h"
 
 static ssize_t nr_addr_cmp_show(struct device *dev,
 				struct device_attribute *attr, char *buf)
@@ -1222,7 +1223,7 @@ static struct attribute *coresight_etm_attrs[] = {
 };
 
 #define coresight_etm3x_simple_func(name, offset)			\
-	coresight_simple_func(struct etm_drvdata, name, offset)
+	coresight_simple_func(struct etm_drvdata, NULL, name, offset)
 
 coresight_etm3x_simple_func(etmccr, ETMCCR);
 coresight_etm3x_simple_func(etmccer, ETMCCER);
diff --git a/drivers/hwtracing/coresight/coresight-etm3x.c b/drivers/hwtracing/coresight/coresight-etm3x.c
index 2de4cad9c5ed..3fe368b23d15 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x.c
@@ -311,9 +311,10 @@ void etm_config_trace_mode(struct etm_config *config)
 #define ETM3X_SUPPORTED_OPTIONS (ETMCR_CYC_ACC | ETMCR_TIMESTAMP_EN)
 
 static int etm_parse_event_config(struct etm_drvdata *drvdata,
-				  struct perf_event_attr *attr)
+				  struct perf_event *event)
 {
 	struct etm_config *config = &drvdata->config;
+	struct perf_event_attr *attr = &event->attr;
 
 	if (!attr)
 		return -EINVAL;
@@ -459,7 +460,7 @@ static int etm_trace_id(struct coresight_device *csdev)
 }
 
 static int etm_enable_perf(struct coresight_device *csdev,
-			   struct perf_event_attr *attr)
+			   struct perf_event *event)
 {
 	struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
@@ -467,7 +468,7 @@ static int etm_enable_perf(struct coresight_device *csdev,
 		return -EINVAL;
 
 	/* Configure the tracer based on the session's specifics */
-	etm_parse_event_config(drvdata, attr);
+	etm_parse_event_config(drvdata, event);
 	/* And enable it */
 	etm_enable_hw(drvdata);
 
@@ -504,7 +505,7 @@ err:
 }
 
 static int etm_enable(struct coresight_device *csdev,
-		      struct perf_event_attr *attr, u32 mode)
+		      struct perf_event *event, u32 mode)
 {
 	int ret;
 	u32 val;
@@ -521,7 +522,7 @@ static int etm_enable(struct coresight_device *csdev,
 		ret = etm_enable_sysfs(csdev);
 		break;
 	case CS_MODE_PERF:
-		ret = etm_enable_perf(csdev, attr);
+		ret = etm_enable_perf(csdev, event);
 		break;
 	default:
 		ret = -EINVAL;
@@ -601,7 +602,8 @@ static void etm_disable_sysfs(struct coresight_device *csdev)
 	dev_info(drvdata->dev, "ETM tracing disabled\n");
 }
 
-static void etm_disable(struct coresight_device *csdev)
+static void etm_disable(struct coresight_device *csdev,
+			struct perf_event *event)
 {
 	u32 mode;
 	struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
@@ -756,13 +758,9 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id)
 	struct coresight_platform_data *pdata = NULL;
 	struct etm_drvdata *drvdata;
 	struct resource *res = &adev->res;
-	struct coresight_desc *desc;
+	struct coresight_desc desc = { 0 };
 	struct device_node *np = adev->dev.of_node;
 
-	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
 	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
 	if (!drvdata)
 		return -ENOMEM;
@@ -825,13 +823,13 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id)
 	etm_init_trace_id(drvdata);
 	etm_set_default(&drvdata->config);
 
-	desc->type = CORESIGHT_DEV_TYPE_SOURCE;
-	desc->subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
-	desc->ops = &etm_cs_ops;
-	desc->pdata = pdata;
-	desc->dev = dev;
-	desc->groups = coresight_etm_groups;
-	drvdata->csdev = coresight_register(desc);
+	desc.type = CORESIGHT_DEV_TYPE_SOURCE;
+	desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
+	desc.ops = &etm_cs_ops;
+	desc.pdata = pdata;
+	desc.dev = dev;
+	desc.groups = coresight_etm_groups;
+	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev)) {
 		ret = PTR_ERR(drvdata->csdev);
 		goto err_arch_supported;
@@ -893,6 +891,11 @@ static struct amba_id etm_ids[] = {
 		.mask	= 0x0003ffff,
 		.data	= "ETM 3.3",
 	},
+	{	/* ETM 3.5 - Cortex-A5 */
+		.id	= 0x0003b955,
+		.mask	= 0x0003ffff,
+		.data	= "ETM 3.5",
+	},
 	{	/* ETM 3.5 */
 		.id	= 0x0003b956,
 		.mask	= 0x0003ffff,
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
index 7c84308c5564..b9b1e9c8f4c4 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
@@ -18,6 +18,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/sysfs.h>
 #include "coresight-etm4x.h"
+#include "coresight-priv.h"
 
 static int etm4_set_mode_exclude(struct etmv4_drvdata *drvdata, bool exclude)
 {
@@ -2039,15 +2040,42 @@ static struct attribute *coresight_etmv4_attrs[] = {
 	NULL,
 };
 
+struct etmv4_reg {
+	void __iomem *addr;
+	u32 data;
+};
+
+static void do_smp_cross_read(void *data)
+{
+	struct etmv4_reg *reg = data;
+
+	reg->data = readl_relaxed(reg->addr);
+}
+
+static u32 etmv4_cross_read(const struct device *dev, u32 offset)
+{
+	struct etmv4_drvdata *drvdata = dev_get_drvdata(dev);
+	struct etmv4_reg reg;
+
+	reg.addr = drvdata->base + offset;
+	/*
+	 * smp cross call ensures the CPU will be powered up before
+	 * accessing the ETMv4 trace core registers
+	 */
+	smp_call_function_single(drvdata->cpu, do_smp_cross_read, &reg, 1);
+	return reg.data;
+}
+
 #define coresight_etm4x_simple_func(name, offset)			\
-	coresight_simple_func(struct etmv4_drvdata, name, offset)
+	coresight_simple_func(struct etmv4_drvdata, NULL, name, offset)
+
+#define coresight_etm4x_cross_read(name, offset)			\
+	coresight_simple_func(struct etmv4_drvdata, etmv4_cross_read,	\
+			      name, offset)
 
-coresight_etm4x_simple_func(trcoslsr, TRCOSLSR);
 coresight_etm4x_simple_func(trcpdcr, TRCPDCR);
 coresight_etm4x_simple_func(trcpdsr, TRCPDSR);
 coresight_etm4x_simple_func(trclsr, TRCLSR);
-coresight_etm4x_simple_func(trcconfig, TRCCONFIGR);
-coresight_etm4x_simple_func(trctraceid, TRCTRACEIDR);
 coresight_etm4x_simple_func(trcauthstatus, TRCAUTHSTATUS);
 coresight_etm4x_simple_func(trcdevid, TRCDEVID);
 coresight_etm4x_simple_func(trcdevtype, TRCDEVTYPE);
@@ -2055,6 +2083,9 @@ coresight_etm4x_simple_func(trcpidr0, TRCPIDR0);
 coresight_etm4x_simple_func(trcpidr1, TRCPIDR1);
 coresight_etm4x_simple_func(trcpidr2, TRCPIDR2);
 coresight_etm4x_simple_func(trcpidr3, TRCPIDR3);
+coresight_etm4x_cross_read(trcoslsr, TRCOSLSR);
+coresight_etm4x_cross_read(trcconfig, TRCCONFIGR);
+coresight_etm4x_cross_read(trctraceid, TRCTRACEIDR);
 
 static struct attribute *coresight_etmv4_mgmt_attrs[] = {
 	&dev_attr_trcoslsr.attr,
@@ -2073,19 +2104,19 @@ static struct attribute *coresight_etmv4_mgmt_attrs[] = {
 	NULL,
 };
 
-coresight_etm4x_simple_func(trcidr0, TRCIDR0);
-coresight_etm4x_simple_func(trcidr1, TRCIDR1);
-coresight_etm4x_simple_func(trcidr2, TRCIDR2);
-coresight_etm4x_simple_func(trcidr3, TRCIDR3);
-coresight_etm4x_simple_func(trcidr4, TRCIDR4);
-coresight_etm4x_simple_func(trcidr5, TRCIDR5);
+coresight_etm4x_cross_read(trcidr0, TRCIDR0);
+coresight_etm4x_cross_read(trcidr1, TRCIDR1);
+coresight_etm4x_cross_read(trcidr2, TRCIDR2);
+coresight_etm4x_cross_read(trcidr3, TRCIDR3);
+coresight_etm4x_cross_read(trcidr4, TRCIDR4);
+coresight_etm4x_cross_read(trcidr5, TRCIDR5);
 /* trcidr[6,7] are reserved */
-coresight_etm4x_simple_func(trcidr8, TRCIDR8);
-coresight_etm4x_simple_func(trcidr9, TRCIDR9);
-coresight_etm4x_simple_func(trcidr10, TRCIDR10);
-coresight_etm4x_simple_func(trcidr11, TRCIDR11);
-coresight_etm4x_simple_func(trcidr12, TRCIDR12);
-coresight_etm4x_simple_func(trcidr13, TRCIDR13);
+coresight_etm4x_cross_read(trcidr8, TRCIDR8);
+coresight_etm4x_cross_read(trcidr9, TRCIDR9);
+coresight_etm4x_cross_read(trcidr10, TRCIDR10);
+coresight_etm4x_cross_read(trcidr11, TRCIDR11);
+coresight_etm4x_cross_read(trcidr12, TRCIDR12);
+coresight_etm4x_cross_read(trcidr13, TRCIDR13);
 
 static struct attribute *coresight_etmv4_trcidr_attrs[] = {
 	&dev_attr_trcidr0.attr,
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
index 1a5e0d14c1dd..4db8d6a4d0cb 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x.c
@@ -33,7 +33,6 @@
 #include <linux/uaccess.h>
 #include <linux/perf_event.h>
 #include <linux/pm_runtime.h>
-#include <linux/perf_event.h>
 #include <asm/sections.h>
 #include <asm/local.h>
 
@@ -46,7 +45,9 @@ module_param_named(boot_enable, boot_enable, int, S_IRUGO);
 /* The number of ETMv4 currently registered */
 static int etm4_count;
 static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
-static void etm4_set_default(struct etmv4_config *config);
+static void etm4_set_default_config(struct etmv4_config *config);
+static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
+				  struct perf_event *event);
 
 static enum cpuhp_state hp_online;
 
@@ -79,22 +80,8 @@ static int etm4_cpu_id(struct coresight_device *csdev)
 static int etm4_trace_id(struct coresight_device *csdev)
 {
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
-	unsigned long flags;
-	int trace_id = -1;
-
-	if (!local_read(&drvdata->mode))
-		return drvdata->trcid;
-
-	spin_lock_irqsave(&drvdata->spinlock, flags);
-
-	CS_UNLOCK(drvdata->base);
-	trace_id = readl_relaxed(drvdata->base + TRCTRACEIDR);
-	trace_id &= ETM_TRACEID_MASK;
-	CS_LOCK(drvdata->base);
 
-	spin_unlock_irqrestore(&drvdata->spinlock, flags);
-
-	return trace_id;
+	return drvdata->trcid;
 }
 
 static void etm4_enable_hw(void *info)
@@ -113,8 +100,7 @@ static void etm4_enable_hw(void *info)
 	/* wait for TRCSTATR.IDLE to go up */
 	if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1))
 		dev_err(drvdata->dev,
-			"timeout observed when probing at offset %#x\n",
-			TRCSTATR);
+			"timeout while waiting for Idle Trace Status\n");
 
 	writel_relaxed(config->pe_sel, drvdata->base + TRCPROCSELR);
 	writel_relaxed(config->cfg, drvdata->base + TRCCONFIGR);
@@ -180,14 +166,20 @@ static void etm4_enable_hw(void *info)
 	writel_relaxed(config->vmid_mask0, drvdata->base + TRCVMIDCCTLR0);
 	writel_relaxed(config->vmid_mask1, drvdata->base + TRCVMIDCCTLR1);
 
+	/*
+	 * Request to keep the trace unit powered and also
+	 * emulation of powerdown
+	 */
+	writel_relaxed(readl_relaxed(drvdata->base + TRCPDCR) | TRCPDCR_PU,
+		       drvdata->base + TRCPDCR);
+
 	/* Enable the trace unit */
 	writel_relaxed(1, drvdata->base + TRCPRGCTLR);
 
 	/* wait for TRCSTATR.IDLE to go back down to '0' */
 	if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 0))
 		dev_err(drvdata->dev,
-			"timeout observed when probing at offset %#x\n",
-			TRCSTATR);
+			"timeout while waiting for Idle Trace Status\n");
 
 	CS_LOCK(drvdata->base);
 
@@ -195,12 +187,16 @@ static void etm4_enable_hw(void *info)
 }
 
 static int etm4_parse_event_config(struct etmv4_drvdata *drvdata,
-				   struct perf_event_attr *attr)
+				   struct perf_event *event)
 {
+	int ret = 0;
 	struct etmv4_config *config = &drvdata->config;
+	struct perf_event_attr *attr = &event->attr;
 
-	if (!attr)
-		return -EINVAL;
+	if (!attr) {
+		ret = -EINVAL;
+		goto out;
+	}
 
 	/* Clear configuration from previous run */
 	memset(config, 0, sizeof(struct etmv4_config));
@@ -212,14 +208,12 @@ static int etm4_parse_event_config(struct etmv4_drvdata *drvdata,
 		config->mode = ETM_MODE_EXCL_USER;
 
 	/* Always start from the default config */
-	etm4_set_default(config);
+	etm4_set_default_config(config);
 
-	/*
-	 * By default the tracers are configured to trace the whole address
-	 * range.  Narrow the field only if requested by user space.
-	 */
-	if (config->mode)
-		etm4_config_trace_mode(config);
+	/* Configure filters specified on the perf cmd line, if any. */
+	ret = etm4_set_event_filters(drvdata, event);
+	if (ret)
+		goto out;
 
 	/* Go from generic option to ETMv4 specifics */
 	if (attr->config & BIT(ETM_OPT_CYCACC))
@@ -227,23 +221,30 @@ static int etm4_parse_event_config(struct etmv4_drvdata *drvdata,
 	if (attr->config & BIT(ETM_OPT_TS))
 		config->cfg |= ETMv4_MODE_TIMESTAMP;
 
-	return 0;
+out:
+	return ret;
 }
 
 static int etm4_enable_perf(struct coresight_device *csdev,
-			    struct perf_event_attr *attr)
+			    struct perf_event *event)
 {
+	int ret = 0;
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
-	if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))
-		return -EINVAL;
+	if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) {
+		ret = -EINVAL;
+		goto out;
+	}
 
 	/* Configure the tracer based on the session's specifics */
-	etm4_parse_event_config(drvdata, attr);
+	ret = etm4_parse_event_config(drvdata, event);
+	if (ret)
+		goto out;
 	/* And enable it */
 	etm4_enable_hw(drvdata);
 
-	return 0;
+out:
+	return ret;
 }
 
 static int etm4_enable_sysfs(struct coresight_device *csdev)
@@ -274,7 +275,7 @@ err:
 }
 
 static int etm4_enable(struct coresight_device *csdev,
-		       struct perf_event_attr *attr, u32 mode)
+		       struct perf_event *event, u32 mode)
 {
 	int ret;
 	u32 val;
@@ -291,7 +292,7 @@ static int etm4_enable(struct coresight_device *csdev,
 		ret = etm4_enable_sysfs(csdev);
 		break;
 	case CS_MODE_PERF:
-		ret = etm4_enable_perf(csdev, attr);
+		ret = etm4_enable_perf(csdev, event);
 		break;
 	default:
 		ret = -EINVAL;
@@ -311,6 +312,11 @@ static void etm4_disable_hw(void *info)
 
 	CS_UNLOCK(drvdata->base);
 
+	/* power can be removed from the trace unit now */
+	control = readl_relaxed(drvdata->base + TRCPDCR);
+	control &= ~TRCPDCR_PU;
+	writel_relaxed(control, drvdata->base + TRCPDCR);
+
 	control = readl_relaxed(drvdata->base + TRCPRGCTLR);
 
 	/* EN, bit[0] Trace unit enable bit */
@@ -326,14 +332,28 @@ static void etm4_disable_hw(void *info)
 	dev_dbg(drvdata->dev, "cpu: %d disable smp call done\n", drvdata->cpu);
 }
 
-static int etm4_disable_perf(struct coresight_device *csdev)
+static int etm4_disable_perf(struct coresight_device *csdev,
+			     struct perf_event *event)
 {
+	u32 control;
+	struct etm_filters *filters = event->hw.addr_filters;
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
 	if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))
 		return -EINVAL;
 
 	etm4_disable_hw(drvdata);
+
+	/*
+	 * Check if the start/stop logic was active when the unit was stopped.
+	 * That way we can re-enable the start/stop logic when the process is
+	 * scheduled again.  Configuration of the start/stop logic happens in
+	 * function etm4_set_event_filters().
+	 */
+	control = readl_relaxed(drvdata->base + TRCVICTLR);
+	/* TRCVICTLR::SSSTATUS, bit[9] */
+	filters->ssstatus = (control & BIT(9));
+
 	return 0;
 }
 
@@ -362,7 +382,8 @@ static void etm4_disable_sysfs(struct coresight_device *csdev)
 	dev_info(drvdata->dev, "ETM tracing disabled\n");
 }
 
-static void etm4_disable(struct coresight_device *csdev)
+static void etm4_disable(struct coresight_device *csdev,
+			 struct perf_event *event)
 {
 	u32 mode;
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
@@ -381,7 +402,7 @@ static void etm4_disable(struct coresight_device *csdev)
 		etm4_disable_sysfs(csdev);
 		break;
 	case CS_MODE_PERF:
-		etm4_disable_perf(csdev);
+		etm4_disable_perf(csdev, event);
 		break;
 	}
 
@@ -564,21 +585,8 @@ static void etm4_init_arch_data(void *info)
 	CS_LOCK(drvdata->base);
 }
 
-static void etm4_set_default(struct etmv4_config *config)
+static void etm4_set_default_config(struct etmv4_config *config)
 {
-	if (WARN_ON_ONCE(!config))
-		return;
-
-	/*
-	 * Make default initialisation trace everything
-	 *
-	 * Select the "always true" resource selector on the
-	 * "Enablign Event" line and configure address range comparator
-	 * '0' to trace all the possible address range.  From there
-	 * configure the "include/exclude" engine to include address
-	 * range comparator '0'.
-	 */
-
 	/* disable all events tracing */
 	config->eventctrl0 = 0x0;
 	config->eventctrl1 = 0x0;
@@ -594,6 +602,108 @@ static void etm4_set_default(struct etmv4_config *config)
 
 	/* TRCVICTLR::EVENT = 0x01, select the always on logic */
 	config->vinst_ctrl |= BIT(0);
+}
+
+static u64 etm4_get_access_type(struct etmv4_config *config)
+{
+	u64 access_type = 0;
+
+	/*
+	 * EXLEVEL_NS, bits[15:12]
+	 * The Exception levels are:
+	 *   Bit[12] Exception level 0 - Application
+	 *   Bit[13] Exception level 1 - OS
+	 *   Bit[14] Exception level 2 - Hypervisor
+	 *   Bit[15] Never implemented
+	 *
+	 * Always stay away from hypervisor mode.
+	 */
+	access_type = ETM_EXLEVEL_NS_HYP;
+
+	if (config->mode & ETM_MODE_EXCL_KERN)
+		access_type |= ETM_EXLEVEL_NS_OS;
+
+	if (config->mode & ETM_MODE_EXCL_USER)
+		access_type |= ETM_EXLEVEL_NS_APP;
+
+	/*
+	 * EXLEVEL_S, bits[11:8], don't trace anything happening
+	 * in secure state.
+	 */
+	access_type |= (ETM_EXLEVEL_S_APP	|
+			ETM_EXLEVEL_S_OS	|
+			ETM_EXLEVEL_S_HYP);
+
+	return access_type;
+}
+
+static void etm4_set_comparator_filter(struct etmv4_config *config,
+				       u64 start, u64 stop, int comparator)
+{
+	u64 access_type = etm4_get_access_type(config);
+
+	/* First half of default address comparator */
+	config->addr_val[comparator] = start;
+	config->addr_acc[comparator] = access_type;
+	config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE;
+
+	/* Second half of default address comparator */
+	config->addr_val[comparator + 1] = stop;
+	config->addr_acc[comparator + 1] = access_type;
+	config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE;
+
+	/*
+	 * Configure the ViewInst function to include this address range
+	 * comparator.
+	 *
+	 * @comparator is divided by two since it is the index in the
+	 * etmv4_config::addr_val array but register TRCVIIECTLR deals with
+	 * address range comparator _pairs_.
+	 *
+	 * Therefore:
+	 *	index 0 -> compatator pair 0
+	 *	index 2 -> comparator pair 1
+	 *	index 4 -> comparator pair 2
+	 *	...
+	 *	index 14 -> comparator pair 7
+	 */
+	config->viiectlr |= BIT(comparator / 2);
+}
+
+static void etm4_set_start_stop_filter(struct etmv4_config *config,
+				       u64 address, int comparator,
+				       enum etm_addr_type type)
+{
+	int shift;
+	u64 access_type = etm4_get_access_type(config);
+
+	/* Configure the comparator */
+	config->addr_val[comparator] = address;
+	config->addr_acc[comparator] = access_type;
+	config->addr_type[comparator] = type;
+
+	/*
+	 * Configure ViewInst Start-Stop control register.
+	 * Addresses configured to start tracing go from bit 0 to n-1,
+	 * while those configured to stop tracing from 16 to 16 + n-1.
+	 */
+	shift = (type == ETM_ADDR_TYPE_START ? 0 : 16);
+	config->vissctlr |= BIT(shift + comparator);
+}
+
+static void etm4_set_default_filter(struct etmv4_config *config)
+{
+	u64 start, stop;
+
+	/*
+	 * Configure address range comparator '0' to encompass all
+	 * possible addresses.
+	 */
+	start = 0x0;
+	stop = ~0x0;
+
+	etm4_set_comparator_filter(config, start, stop,
+				   ETM_DEFAULT_ADDR_COMP);
 
 	/*
 	 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
@@ -601,43 +711,156 @@ static void etm4_set_default(struct etmv4_config *config)
 	 */
 	config->vinst_ctrl |= BIT(9);
 
+	/* No start-stop filtering for ViewInst */
+	config->vissctlr = 0x0;
+}
+
+static void etm4_set_default(struct etmv4_config *config)
+{
+	if (WARN_ON_ONCE(!config))
+		return;
+
 	/*
-	 * Configure address range comparator '0' to encompass all
-	 * possible addresses.
+	 * Make default initialisation trace everything
+	 *
+	 * Select the "always true" resource selector on the
+	 * "Enablign Event" line and configure address range comparator
+	 * '0' to trace all the possible address range.  From there
+	 * configure the "include/exclude" engine to include address
+	 * range comparator '0'.
 	 */
+	etm4_set_default_config(config);
+	etm4_set_default_filter(config);
+}
 
-	/* First half of default address comparator: start at address 0 */
-	config->addr_val[ETM_DEFAULT_ADDR_COMP] = 0x0;
-	/* trace instruction addresses */
-	config->addr_acc[ETM_DEFAULT_ADDR_COMP] &= ~(BIT(0) | BIT(1));
-	/* EXLEVEL_NS, bits[12:15], only trace application and kernel space */
-	config->addr_acc[ETM_DEFAULT_ADDR_COMP] |= ETM_EXLEVEL_NS_HYP;
-	/* EXLEVEL_S, bits[11:8], don't trace anything in secure state */
-	config->addr_acc[ETM_DEFAULT_ADDR_COMP] |= (ETM_EXLEVEL_S_APP |
-						    ETM_EXLEVEL_S_OS |
-						    ETM_EXLEVEL_S_HYP);
-	config->addr_type[ETM_DEFAULT_ADDR_COMP] = ETM_ADDR_TYPE_RANGE;
+static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type)
+{
+	int nr_comparator, index = 0;
+	struct etmv4_config *config = &drvdata->config;
 
 	/*
-	 * Second half of default address comparator: go all
-	 * the way to the top.
-	*/
-	config->addr_val[ETM_DEFAULT_ADDR_COMP + 1] = ~0x0;
-	/* trace instruction addresses */
-	config->addr_acc[ETM_DEFAULT_ADDR_COMP + 1] &= ~(BIT(0) | BIT(1));
-	/* Address comparator type must be equal for both halves */
-	config->addr_acc[ETM_DEFAULT_ADDR_COMP + 1] =
-					config->addr_acc[ETM_DEFAULT_ADDR_COMP];
-	config->addr_type[ETM_DEFAULT_ADDR_COMP + 1] = ETM_ADDR_TYPE_RANGE;
+	 * nr_addr_cmp holds the number of comparator _pair_, so time 2
+	 * for the total number of comparators.
+	 */
+	nr_comparator = drvdata->nr_addr_cmp * 2;
+
+	/* Go through the tally of comparators looking for a free one. */
+	while (index < nr_comparator) {
+		switch (type) {
+		case ETM_ADDR_TYPE_RANGE:
+			if (config->addr_type[index] == ETM_ADDR_TYPE_NONE &&
+			    config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE)
+				return index;
+
+			/* Address range comparators go in pairs */
+			index += 2;
+			break;
+		case ETM_ADDR_TYPE_START:
+		case ETM_ADDR_TYPE_STOP:
+			if (config->addr_type[index] == ETM_ADDR_TYPE_NONE)
+				return index;
+
+			/* Start/stop address can have odd indexes */
+			index += 1;
+			break;
+		default:
+			return -EINVAL;
+		}
+	}
+
+	/* If we are here all the comparators have been used. */
+	return -ENOSPC;
+}
+
+static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
+				  struct perf_event *event)
+{
+	int i, comparator, ret = 0;
+	u64 address;
+	struct etmv4_config *config = &drvdata->config;
+	struct etm_filters *filters = event->hw.addr_filters;
+
+	if (!filters)
+		goto default_filter;
+
+	/* Sync events with what Perf got */
+	perf_event_addr_filters_sync(event);
 
 	/*
-	 * Configure the ViewInst function to filter on address range
-	 * comparator '0'.
+	 * If there are no filters to deal with simply go ahead with
+	 * the default filter, i.e the entire address range.
 	 */
-	config->viiectlr = BIT(0);
+	if (!filters->nr_filters)
+		goto default_filter;
+
+	for (i = 0; i < filters->nr_filters; i++) {
+		struct etm_filter *filter = &filters->etm_filter[i];
+		enum etm_addr_type type = filter->type;
+
+		/* See if a comparator is free. */
+		comparator = etm4_get_next_comparator(drvdata, type);
+		if (comparator < 0) {
+			ret = comparator;
+			goto out;
+		}
+
+		switch (type) {
+		case ETM_ADDR_TYPE_RANGE:
+			etm4_set_comparator_filter(config,
+						   filter->start_addr,
+						   filter->stop_addr,
+						   comparator);
+			/*
+			 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
+			 * in the started state
+			 */
+			config->vinst_ctrl |= BIT(9);
+
+			/* No start-stop filtering for ViewInst */
+			config->vissctlr = 0x0;
+			break;
+		case ETM_ADDR_TYPE_START:
+		case ETM_ADDR_TYPE_STOP:
+			/* Get the right start or stop address */
+			address = (type == ETM_ADDR_TYPE_START ?
+				   filter->start_addr :
+				   filter->stop_addr);
+
+			/* Configure comparator */
+			etm4_set_start_stop_filter(config, address,
+						   comparator, type);
+
+			/*
+			 * If filters::ssstatus == 1, trace acquisition was
+			 * started but the process was yanked away before the
+			 * the stop address was hit.  As such the start/stop
+			 * logic needs to be re-started so that tracing can
+			 * resume where it left.
+			 *
+			 * The start/stop logic status when a process is
+			 * scheduled out is checked in function
+			 * etm4_disable_perf().
+			 */
+			if (filters->ssstatus)
+				config->vinst_ctrl |= BIT(9);
+
+			/* No include/exclude filtering for ViewInst */
+			config->viiectlr = 0x0;
+			break;
+		default:
+			ret = -EINVAL;
+			goto out;
+		}
+	}
 
-	/* no start-stop filtering for ViewInst */
-	config->vissctlr = 0x0;
+	goto out;
+
+
+default_filter:
+	etm4_set_default_filter(config);
+
+out:
+	return ret;
 }
 
 void etm4_config_trace_mode(struct etmv4_config *config)
@@ -727,13 +950,9 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id)
 	struct coresight_platform_data *pdata = NULL;
 	struct etmv4_drvdata *drvdata;
 	struct resource *res = &adev->res;
-	struct coresight_desc *desc;
+	struct coresight_desc desc = { 0 };
 	struct device_node *np = adev->dev.of_node;
 
-	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
 	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
 	if (!drvdata)
 		return -ENOMEM;
@@ -788,13 +1007,13 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id)
 	etm4_init_trace_id(drvdata);
 	etm4_set_default(&drvdata->config);
 
-	desc->type = CORESIGHT_DEV_TYPE_SOURCE;
-	desc->subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
-	desc->ops = &etm4_cs_ops;
-	desc->pdata = pdata;
-	desc->dev = dev;
-	desc->groups = coresight_etmv4_groups;
-	drvdata->csdev = coresight_register(desc);
+	desc.type = CORESIGHT_DEV_TYPE_SOURCE;
+	desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
+	desc.ops = &etm4_cs_ops;
+	desc.pdata = pdata;
+	desc.dev = dev;
+	desc.groups = coresight_etmv4_groups;
+	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev)) {
 		ret = PTR_ERR(drvdata->csdev);
 		goto err_arch_supported;
@@ -826,12 +1045,12 @@ err_arch_supported:
 }
 
 static struct amba_id etm4_ids[] = {
-	{       /* ETM 4.0 - Qualcomm */
-		.id	= 0x0003b95d,
-		.mask	= 0x0003ffff,
+	{       /* ETM 4.0 - Cortex-A53  */
+		.id	= 0x000bb95d,
+		.mask	= 0x000fffff,
 		.data	= "ETM 4.0",
 	},
-	{       /* ETM 4.0 - Juno board */
+	{       /* ETM 4.0 - Cortex-A57 */
 		.id	= 0x000bb95e,
 		.mask	= 0x000fffff,
 		.data	= "ETM 4.0",
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h
index 5359c5197c1d..ba8d3f86de21 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.h
+++ b/drivers/hwtracing/coresight/coresight-etm4x.h
@@ -183,6 +183,9 @@
 #define TRCSTATR_IDLE_BIT		0
 #define ETM_DEFAULT_ADDR_COMP		0
 
+/* PowerDown Control Register bits */
+#define TRCPDCR_PU			BIT(3)
+
 /* secure state access levels */
 #define ETM_EXLEVEL_S_APP		BIT(8)
 #define ETM_EXLEVEL_S_OS		BIT(9)
@@ -407,14 +410,6 @@ enum etm_addr_ctxtype {
 	ETM_CTX_CTXID_VMID,
 };
 
-enum etm_addr_type {
-	ETM_ADDR_TYPE_NONE,
-	ETM_ADDR_TYPE_SINGLE,
-	ETM_ADDR_TYPE_RANGE,
-	ETM_ADDR_TYPE_START,
-	ETM_ADDR_TYPE_STOP,
-};
-
 extern const struct attribute_group *coresight_etmv4_groups[];
 void etm4_config_trace_mode(struct etmv4_config *config);
 #endif
diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c
index 05df789056cc..860fe6ef5632 100644
--- a/drivers/hwtracing/coresight/coresight-funnel.c
+++ b/drivers/hwtracing/coresight/coresight-funnel.c
@@ -176,7 +176,7 @@ static int funnel_probe(struct amba_device *adev, const struct amba_id *id)
 	struct coresight_platform_data *pdata = NULL;
 	struct funnel_drvdata *drvdata;
 	struct resource *res = &adev->res;
-	struct coresight_desc *desc;
+	struct coresight_desc desc = { 0 };
 	struct device_node *np = adev->dev.of_node;
 
 	if (np) {
@@ -207,17 +207,13 @@ static int funnel_probe(struct amba_device *adev, const struct amba_id *id)
 	drvdata->base = base;
 	pm_runtime_put(&adev->dev);
 
-	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
-	desc->type = CORESIGHT_DEV_TYPE_LINK;
-	desc->subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_MERG;
-	desc->ops = &funnel_cs_ops;
-	desc->pdata = pdata;
-	desc->dev = dev;
-	desc->groups = coresight_funnel_groups;
-	drvdata->csdev = coresight_register(desc);
+	desc.type = CORESIGHT_DEV_TYPE_LINK;
+	desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_MERG;
+	desc.ops = &funnel_cs_ops;
+	desc.pdata = pdata;
+	desc.dev = dev;
+	desc.groups = coresight_funnel_groups;
+	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev))
 		return PTR_ERR(drvdata->csdev);
 
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index ad975c58080d..196a14be4b3d 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -16,6 +16,7 @@
 #include <linux/bitops.h>
 #include <linux/io.h>
 #include <linux/coresight.h>
+#include <linux/pm_runtime.h>
 
 /*
  * Coresight management registers (0xf00-0xfcc)
@@ -37,16 +38,32 @@
 #define ETM_MODE_EXCL_KERN	BIT(30)
 #define ETM_MODE_EXCL_USER	BIT(31)
 
-#define coresight_simple_func(type, name, offset)			\
+typedef u32 (*coresight_read_fn)(const struct device *, u32 offset);
+#define coresight_simple_func(type, func, name, offset)			\
 static ssize_t name##_show(struct device *_dev,				\
 			   struct device_attribute *attr, char *buf)	\
 {									\
 	type *drvdata = dev_get_drvdata(_dev->parent);			\
-	return scnprintf(buf, PAGE_SIZE, "0x%x\n",			\
-			 readl_relaxed(drvdata->base + offset));	\
+	coresight_read_fn fn = func;					\
+	u32 val;							\
+	pm_runtime_get_sync(_dev->parent);				\
+	if (fn)								\
+		val = fn(_dev->parent, offset);				\
+	else								\
+		val = readl_relaxed(drvdata->base + offset);		\
+	pm_runtime_put_sync(_dev->parent);				\
+	return scnprintf(buf, PAGE_SIZE, "0x%x\n", val);		\
 }									\
 static DEVICE_ATTR_RO(name)
 
+enum etm_addr_type {
+	ETM_ADDR_TYPE_NONE,
+	ETM_ADDR_TYPE_SINGLE,
+	ETM_ADDR_TYPE_RANGE,
+	ETM_ADDR_TYPE_START,
+	ETM_ADDR_TYPE_STOP,
+};
+
 enum cs_mode {
 	CS_MODE_DISABLED,
 	CS_MODE_SYSFS,
diff --git a/drivers/hwtracing/coresight/coresight-replicator-qcom.c b/drivers/hwtracing/coresight/coresight-replicator-qcom.c
index 700f710e4bfa..0a3d15f0b009 100644
--- a/drivers/hwtracing/coresight/coresight-replicator-qcom.c
+++ b/drivers/hwtracing/coresight/coresight-replicator-qcom.c
@@ -102,7 +102,7 @@ static int replicator_probe(struct amba_device *adev, const struct amba_id *id)
 	struct resource *res = &adev->res;
 	struct coresight_platform_data *pdata = NULL;
 	struct replicator_state *drvdata;
-	struct coresight_desc *desc;
+	struct coresight_desc desc = { 0 };
 	struct device_node *np = adev->dev.of_node;
 	void __iomem *base;
 
@@ -134,16 +134,12 @@ static int replicator_probe(struct amba_device *adev, const struct amba_id *id)
 	dev_set_drvdata(dev, drvdata);
 	pm_runtime_put(&adev->dev);
 
-	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
-	desc->type = CORESIGHT_DEV_TYPE_LINK;
-	desc->subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_SPLIT;
-	desc->ops = &replicator_cs_ops;
-	desc->pdata = adev->dev.platform_data;
-	desc->dev = &adev->dev;
-	drvdata->csdev = coresight_register(desc);
+	desc.type = CORESIGHT_DEV_TYPE_LINK;
+	desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_SPLIT;
+	desc.ops = &replicator_cs_ops;
+	desc.pdata = adev->dev.platform_data;
+	desc.dev = &adev->dev;
+	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev))
 		return PTR_ERR(drvdata->csdev);
 
diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c
index c6982e312e15..3756e71cb8f5 100644
--- a/drivers/hwtracing/coresight/coresight-replicator.c
+++ b/drivers/hwtracing/coresight/coresight-replicator.c
@@ -69,7 +69,7 @@ static int replicator_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct coresight_platform_data *pdata = NULL;
 	struct replicator_drvdata *drvdata;
-	struct coresight_desc *desc;
+	struct coresight_desc desc = { 0 };
 	struct device_node *np = pdev->dev.of_node;
 
 	if (np) {
@@ -95,18 +95,12 @@ static int replicator_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 	platform_set_drvdata(pdev, drvdata);
 
-	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
-	if (!desc) {
-		ret = -ENOMEM;
-		goto out_disable_pm;
-	}
-
-	desc->type = CORESIGHT_DEV_TYPE_LINK;
-	desc->subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_SPLIT;
-	desc->ops = &replicator_cs_ops;
-	desc->pdata = pdev->dev.platform_data;
-	desc->dev = &pdev->dev;
-	drvdata->csdev = coresight_register(desc);
+	desc.type = CORESIGHT_DEV_TYPE_LINK;
+	desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_SPLIT;
+	desc.ops = &replicator_cs_ops;
+	desc.pdata = pdev->dev.platform_data;
+	desc.dev = &pdev->dev;
+	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev)) {
 		ret = PTR_ERR(drvdata->csdev);
 		goto out_disable_pm;
diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c
index 73be58a11e4f..49e0f1b925a5 100644
--- a/drivers/hwtracing/coresight/coresight-stm.c
+++ b/drivers/hwtracing/coresight/coresight-stm.c
@@ -105,10 +105,12 @@ module_param_named(
 /**
  * struct channel_space - central management entity for extended ports
  * @base:		memory mapped base address where channels start.
+ * @phys:		physical base address of channel region.
  * @guaraneed:		is the channel delivery guaranteed.
  */
 struct channel_space {
 	void __iomem		*base;
+	phys_addr_t		phys;
 	unsigned long		*guaranteed;
 };
 
@@ -196,7 +198,7 @@ static void stm_enable_hw(struct stm_drvdata *drvdata)
 }
 
 static int stm_enable(struct coresight_device *csdev,
-		      struct perf_event_attr *attr, u32 mode)
+		      struct perf_event *event, u32 mode)
 {
 	u32 val;
 	struct stm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
@@ -258,7 +260,8 @@ static void stm_disable_hw(struct stm_drvdata *drvdata)
 		stm_hwevent_disable_hw(drvdata);
 }
 
-static void stm_disable(struct coresight_device *csdev)
+static void stm_disable(struct coresight_device *csdev,
+			struct perf_event *event)
 {
 	struct stm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
@@ -353,7 +356,24 @@ static void stm_generic_unlink(struct stm_data *stm_data,
 	if (!drvdata || !drvdata->csdev)
 		return;
 
-	stm_disable(drvdata->csdev);
+	stm_disable(drvdata->csdev, NULL);
+}
+
+static phys_addr_t
+stm_mmio_addr(struct stm_data *stm_data, unsigned int master,
+	      unsigned int channel, unsigned int nr_chans)
+{
+	struct stm_drvdata *drvdata = container_of(stm_data,
+						   struct stm_drvdata, stm);
+	phys_addr_t addr;
+
+	addr = drvdata->chs.phys + channel * BYTES_PER_CHANNEL;
+
+	if (offset_in_page(addr) ||
+	    offset_in_page(nr_chans * BYTES_PER_CHANNEL))
+		return 0;
+
+	return addr;
 }
 
 static long stm_generic_set_options(struct stm_data *stm_data,
@@ -616,7 +636,7 @@ static ssize_t traceid_store(struct device *dev,
 static DEVICE_ATTR_RW(traceid);
 
 #define coresight_stm_simple_func(name, offset)	\
-	coresight_simple_func(struct stm_drvdata, name, offset)
+	coresight_simple_func(struct stm_drvdata, NULL, name, offset)
 
 coresight_stm_simple_func(tcsr, STMTCSR);
 coresight_stm_simple_func(tsfreqr, STMTSFREQR);
@@ -761,7 +781,9 @@ static void stm_init_generic_data(struct stm_drvdata *drvdata)
 	drvdata->stm.sw_end = 1;
 	drvdata->stm.hw_override = true;
 	drvdata->stm.sw_nchannels = drvdata->numsp;
+	drvdata->stm.sw_mmiosz = BYTES_PER_CHANNEL;
 	drvdata->stm.packet = stm_generic_packet;
+	drvdata->stm.mmio_addr = stm_mmio_addr;
 	drvdata->stm.link = stm_generic_link;
 	drvdata->stm.unlink = stm_generic_unlink;
 	drvdata->stm.set_options = stm_generic_set_options;
@@ -778,7 +800,7 @@ static int stm_probe(struct amba_device *adev, const struct amba_id *id)
 	struct resource *res = &adev->res;
 	struct resource ch_res;
 	size_t res_size, bitmap_size;
-	struct coresight_desc *desc;
+	struct coresight_desc desc = { 0 };
 	struct device_node *np = adev->dev.of_node;
 
 	if (np) {
@@ -808,6 +830,7 @@ static int stm_probe(struct amba_device *adev, const struct amba_id *id)
 	ret = stm_get_resource_byname(np, "stm-stimulus-base", &ch_res);
 	if (ret)
 		return ret;
+	drvdata->chs.phys = ch_res.start;
 
 	base = devm_ioremap_resource(dev, &ch_res);
 	if (IS_ERR(base))
@@ -843,19 +866,13 @@ static int stm_probe(struct amba_device *adev, const struct amba_id *id)
 		return -EPROBE_DEFER;
 	}
 
-	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
-	if (!desc) {
-		ret = -ENOMEM;
-		goto stm_unregister;
-	}
-
-	desc->type = CORESIGHT_DEV_TYPE_SOURCE;
-	desc->subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE;
-	desc->ops = &stm_cs_ops;
-	desc->pdata = pdata;
-	desc->dev = dev;
-	desc->groups = coresight_stm_groups;
-	drvdata->csdev = coresight_register(desc);
+	desc.type = CORESIGHT_DEV_TYPE_SOURCE;
+	desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE;
+	desc.ops = &stm_cs_ops;
+	desc.pdata = pdata;
+	desc.dev = dev;
+	desc.groups = coresight_stm_groups;
+	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev)) {
 		ret = PTR_ERR(drvdata->csdev);
 		goto stm_unregister;
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c
index 466af86fd76f..d6941ea24d8d 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etf.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c
@@ -22,7 +22,7 @@
 #include "coresight-priv.h"
 #include "coresight-tmc.h"
 
-void tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
+static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
 {
 	CS_UNLOCK(drvdata->base);
 
@@ -48,6 +48,7 @@ static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
 	int i;
 
 	bufp = drvdata->buf;
+	drvdata->len = 0;
 	while (1) {
 		for (i = 0; i < drvdata->memwidth; i++) {
 			read_data = readl_relaxed(drvdata->base + TMC_RRD);
@@ -55,6 +56,7 @@ static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
 				return;
 			memcpy(bufp, &read_data, 4);
 			bufp += 4;
+			drvdata->len += 4;
 		}
 	}
 }
@@ -166,7 +168,7 @@ out:
 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
 
 	/* Free memory outside the spinlock if need be */
-	if (!used && buf)
+	if (!used)
 		kfree(buf);
 
 	if (!ret)
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c
index 688be9e060fc..886ea83c68e0 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etr.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c
@@ -20,7 +20,7 @@
 #include "coresight-priv.h"
 #include "coresight-tmc.h"
 
-void tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
+static void tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
 {
 	u32 axictl;
 
@@ -64,11 +64,17 @@ static void tmc_etr_dump_hw(struct tmc_drvdata *drvdata)
 	rwp = readl_relaxed(drvdata->base + TMC_RWP);
 	val = readl_relaxed(drvdata->base + TMC_STS);
 
-	/* How much memory do we still have */
-	if (val & BIT(0))
+	/*
+	 * Adjust the buffer to point to the beginning of the trace data
+	 * and update the available trace data.
+	 */
+	if (val & TMC_STS_FULL) {
 		drvdata->buf = drvdata->vaddr + rwp - drvdata->paddr;
-	else
+		drvdata->len = drvdata->size;
+	} else {
 		drvdata->buf = drvdata->vaddr;
+		drvdata->len = rwp - drvdata->paddr;
+	}
 }
 
 static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
diff --git a/drivers/hwtracing/coresight/coresight-tmc.c b/drivers/hwtracing/coresight/coresight-tmc.c
index 9e02ac963cd0..d8517d2a968c 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.c
+++ b/drivers/hwtracing/coresight/coresight-tmc.c
@@ -38,8 +38,7 @@ void tmc_wait_for_tmcready(struct tmc_drvdata *drvdata)
 	if (coresight_timeout(drvdata->base,
 			      TMC_STS, TMC_STS_TMCREADY_BIT, 1)) {
 		dev_err(drvdata->dev,
-			"timeout observed when probing at offset %#x\n",
-			TMC_STS);
+			"timeout while waiting for TMC to be Ready\n");
 	}
 }
 
@@ -56,8 +55,7 @@ void tmc_flush_and_stop(struct tmc_drvdata *drvdata)
 	if (coresight_timeout(drvdata->base,
 			      TMC_FFCR, TMC_FFCR_FLUSHMAN_BIT, 0)) {
 		dev_err(drvdata->dev,
-			"timeout observed when probing at offset %#x\n",
-			TMC_FFCR);
+		"timeout while waiting for completion of Manual Flush\n");
 	}
 
 	tmc_wait_for_tmcready(drvdata);
@@ -140,8 +138,8 @@ static ssize_t tmc_read(struct file *file, char __user *data, size_t len,
 						   struct tmc_drvdata, miscdev);
 	char *bufp = drvdata->buf + *ppos;
 
-	if (*ppos + len > drvdata->size)
-		len = drvdata->size - *ppos;
+	if (*ppos + len > drvdata->len)
+		len = drvdata->len - *ppos;
 
 	if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) {
 		if (bufp == (char *)(drvdata->vaddr + drvdata->size))
@@ -160,7 +158,7 @@ static ssize_t tmc_read(struct file *file, char __user *data, size_t len,
 	*ppos += len;
 
 	dev_dbg(drvdata->dev, "%s: %zu bytes copied, %d bytes left\n",
-		__func__, len, (int)(drvdata->size - *ppos));
+		__func__, len, (int)(drvdata->len - *ppos));
 	return len;
 }
 
@@ -220,7 +218,7 @@ static enum tmc_mem_intf_width tmc_get_memwidth(u32 devid)
 }
 
 #define coresight_tmc_simple_func(name, offset)			\
-	coresight_simple_func(struct tmc_drvdata, name, offset)
+	coresight_simple_func(struct tmc_drvdata, NULL, name, offset)
 
 coresight_tmc_simple_func(rsz, TMC_RSZ);
 coresight_tmc_simple_func(sts, TMC_STS);
@@ -249,8 +247,8 @@ static struct attribute *coresight_tmc_mgmt_attrs[] = {
 	NULL,
 };
 
-ssize_t trigger_cntr_show(struct device *dev,
-			  struct device_attribute *attr, char *buf)
+static ssize_t trigger_cntr_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
 {
 	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
 	unsigned long val = drvdata->trigger_cntr;
@@ -304,27 +302,32 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id)
 	struct coresight_platform_data *pdata = NULL;
 	struct tmc_drvdata *drvdata;
 	struct resource *res = &adev->res;
-	struct coresight_desc *desc;
+	struct coresight_desc desc = { 0 };
 	struct device_node *np = adev->dev.of_node;
 
 	if (np) {
 		pdata = of_get_coresight_platform_data(dev, np);
-		if (IS_ERR(pdata))
-			return PTR_ERR(pdata);
+		if (IS_ERR(pdata)) {
+			ret = PTR_ERR(pdata);
+			goto out;
+		}
 		adev->dev.platform_data = pdata;
 	}
 
+	ret = -ENOMEM;
 	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
 	if (!drvdata)
-		return -ENOMEM;
+		goto out;
 
 	drvdata->dev = &adev->dev;
 	dev_set_drvdata(dev, drvdata);
 
 	/* Validity for the resource is already checked by the AMBA core */
 	base = devm_ioremap_resource(dev, res);
-	if (IS_ERR(base))
-		return PTR_ERR(base);
+	if (IS_ERR(base)) {
+		ret = PTR_ERR(base);
+		goto out;
+	}
 
 	drvdata->base = base;
 
@@ -347,33 +350,28 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id)
 
 	pm_runtime_put(&adev->dev);
 
-	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
-	if (!desc) {
-		ret = -ENOMEM;
-		goto err_devm_kzalloc;
-	}
-
-	desc->pdata = pdata;
-	desc->dev = dev;
-	desc->subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
-	desc->groups = coresight_tmc_groups;
+	desc.pdata = pdata;
+	desc.dev = dev;
+	desc.groups = coresight_tmc_groups;
 
 	if (drvdata->config_type == TMC_CONFIG_TYPE_ETB) {
-		desc->type = CORESIGHT_DEV_TYPE_SINK;
-		desc->ops = &tmc_etb_cs_ops;
+		desc.type = CORESIGHT_DEV_TYPE_SINK;
+		desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
+		desc.ops = &tmc_etb_cs_ops;
 	} else if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) {
-		desc->type = CORESIGHT_DEV_TYPE_SINK;
-		desc->ops = &tmc_etr_cs_ops;
+		desc.type = CORESIGHT_DEV_TYPE_SINK;
+		desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
+		desc.ops = &tmc_etr_cs_ops;
 	} else {
-		desc->type = CORESIGHT_DEV_TYPE_LINKSINK;
-		desc->subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_FIFO;
-		desc->ops = &tmc_etf_cs_ops;
+		desc.type = CORESIGHT_DEV_TYPE_LINKSINK;
+		desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_FIFO;
+		desc.ops = &tmc_etf_cs_ops;
 	}
 
-	drvdata->csdev = coresight_register(desc);
+	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev)) {
 		ret = PTR_ERR(drvdata->csdev);
-		goto err_devm_kzalloc;
+		goto out;
 	}
 
 	drvdata->miscdev.name = pdata->name;
@@ -381,16 +379,8 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id)
 	drvdata->miscdev.fops = &tmc_fops;
 	ret = misc_register(&drvdata->miscdev);
 	if (ret)
-		goto err_misc_register;
-
-	return 0;
-
-err_misc_register:
-	coresight_unregister(drvdata->csdev);
-err_devm_kzalloc:
-	if (drvdata->config_type == TMC_CONFIG_TYPE_ETR)
-		dma_free_coherent(dev, drvdata->size,
-				drvdata->vaddr, drvdata->paddr);
+		coresight_unregister(drvdata->csdev);
+out:
 	return ret;
 }
 
diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h
index 5c5fe2ad2ca7..44b3ae346118 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.h
+++ b/drivers/hwtracing/coresight/coresight-tmc.h
@@ -98,7 +98,8 @@ enum tmc_mem_intf_width {
  * @buf:	area of memory where trace data get sent.
  * @paddr:	DMA start location in RAM.
  * @vaddr:	virtual representation of @paddr.
- * @size:	@buf size.
+ * @size:	trace buffer size.
+ * @len:	size of the available trace.
  * @mode:	how this TMC is being used.
  * @config_type: TMC variant, must be of type @tmc_config_type.
  * @memwidth:	width of the memory interface databus, in bytes.
@@ -115,6 +116,7 @@ struct tmc_drvdata {
 	dma_addr_t		paddr;
 	void __iomem		*vaddr;
 	u32			size;
+	u32			len;
 	local_t			mode;
 	enum tmc_config_type	config_type;
 	enum tmc_mem_intf_width	memwidth;
diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c
index 4e471e2e9d89..0673baf0f2f5 100644
--- a/drivers/hwtracing/coresight/coresight-tpiu.c
+++ b/drivers/hwtracing/coresight/coresight-tpiu.c
@@ -119,7 +119,7 @@ static int tpiu_probe(struct amba_device *adev, const struct amba_id *id)
 	struct coresight_platform_data *pdata = NULL;
 	struct tpiu_drvdata *drvdata;
 	struct resource *res = &adev->res;
-	struct coresight_desc *desc;
+	struct coresight_desc desc = { 0 };
 	struct device_node *np = adev->dev.of_node;
 
 	if (np) {
@@ -154,16 +154,12 @@ static int tpiu_probe(struct amba_device *adev, const struct amba_id *id)
 
 	pm_runtime_put(&adev->dev);
 
-	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
-	desc->type = CORESIGHT_DEV_TYPE_SINK;
-	desc->subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PORT;
-	desc->ops = &tpiu_cs_ops;
-	desc->pdata = pdata;
-	desc->dev = dev;
-	drvdata->csdev = coresight_register(desc);
+	desc.type = CORESIGHT_DEV_TYPE_SINK;
+	desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PORT;
+	desc.ops = &tpiu_cs_ops;
+	desc.pdata = pdata;
+	desc.dev = dev;
+	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev))
 		return PTR_ERR(drvdata->csdev);
 
diff --git a/drivers/hwtracing/coresight/coresight.c b/drivers/hwtracing/coresight/coresight.c
index d08d1ab9bba5..7bf00a0beb6f 100644
--- a/drivers/hwtracing/coresight/coresight.c
+++ b/drivers/hwtracing/coresight/coresight.c
@@ -257,7 +257,7 @@ static void coresight_disable_source(struct coresight_device *csdev)
 {
 	if (atomic_dec_return(csdev->refcnt) == 0) {
 		if (source_ops(csdev)->disable) {
-			source_ops(csdev)->disable(csdev);
+			source_ops(csdev)->disable(csdev, NULL);
 			csdev->enable = false;
 		}
 	}
@@ -429,7 +429,7 @@ struct list_head *coresight_build_path(struct coresight_device *csdev)
 
 	path = kzalloc(sizeof(struct list_head), GFP_KERNEL);
 	if (!path)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	INIT_LIST_HEAD(path);
 
@@ -725,7 +725,8 @@ static int coresight_orphan_match(struct device *dev, void *data)
 		/* We have found at least one orphan connection */
 		if (conn->child_dev == NULL) {
 			/* Does it match this newly added device? */
-			if (!strcmp(dev_name(&csdev->dev), conn->child_name)) {
+			if (conn->child_name &&
+			    !strcmp(dev_name(&csdev->dev), conn->child_name)) {
 				conn->child_dev = csdev;
 			} else {
 				/* This component still has an orphan */
@@ -893,7 +894,7 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 	int nr_refcnts = 1;
 	atomic_t *refcnts = NULL;
 	struct coresight_device *csdev;
-	struct coresight_connection *conns;
+	struct coresight_connection *conns = NULL;
 
 	csdev = kzalloc(sizeof(*csdev), GFP_KERNEL);
 	if (!csdev) {
@@ -921,16 +922,20 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 
 	csdev->nr_inport = desc->pdata->nr_inport;
 	csdev->nr_outport = desc->pdata->nr_outport;
-	conns = kcalloc(csdev->nr_outport, sizeof(*conns), GFP_KERNEL);
-	if (!conns) {
-		ret = -ENOMEM;
-		goto err_kzalloc_conns;
-	}
 
-	for (i = 0; i < csdev->nr_outport; i++) {
-		conns[i].outport = desc->pdata->outports[i];
-		conns[i].child_name = desc->pdata->child_names[i];
-		conns[i].child_port = desc->pdata->child_ports[i];
+	/* Initialise connections if there is at least one outport */
+	if (csdev->nr_outport) {
+		conns = kcalloc(csdev->nr_outport, sizeof(*conns), GFP_KERNEL);
+		if (!conns) {
+			ret = -ENOMEM;
+			goto err_kzalloc_conns;
+		}
+
+		for (i = 0; i < csdev->nr_outport; i++) {
+			conns[i].outport = desc->pdata->outports[i];
+			conns[i].child_name = desc->pdata->child_names[i];
+			conns[i].child_port = desc->pdata->child_ports[i];
+		}
 	}
 
 	csdev->conns = conns;
diff --git a/drivers/hwtracing/coresight/of_coresight.c b/drivers/hwtracing/coresight/of_coresight.c
index b68da1888fd5..629e031b7456 100644
--- a/drivers/hwtracing/coresight/of_coresight.c
+++ b/drivers/hwtracing/coresight/of_coresight.c
@@ -166,7 +166,7 @@ struct coresight_platform_data *of_get_coresight_platform_data(
 
 			rdev = of_coresight_get_endpoint_device(rparent);
 			if (!rdev)
-				continue;
+				return ERR_PTR(-EPROBE_DEFER);
 
 			pdata->child_names[i] = dev_name(rdev);
 			pdata->child_ports[i] = rendpoint.id;
@@ -184,6 +184,7 @@ struct coresight_platform_data *of_get_coresight_platform_data(
 			break;
 		}
 	}
+	of_node_put(dn);
 
 	return pdata;
 }
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 5c3993b26129..6d94e2ec5b4f 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -836,7 +836,7 @@ config I2C_SH7760
 config I2C_SH_MOBILE
 	tristate "SuperH Mobile I2C Controller"
 	depends on HAS_DMA
-	depends on SUPERH || ARCH_RENESAS || COMPILE_TEST
+	depends on ARCH_SHMOBILE || ARCH_RENESAS || COMPILE_TEST
 	help
 	  If you say yes to this option, support will be included for the
 	  built-in I2C interface on the Renesas SH-Mobile processor.
@@ -956,6 +956,17 @@ config I2C_OCTEON
 	  This driver can also be built as a module.  If so, the module
 	  will be called i2c-octeon.
 
+config I2C_THUNDERX
+	tristate "Cavium ThunderX I2C bus support"
+	depends on 64BIT && PCI && (ARM64 || COMPILE_TEST)
+	select I2C_SMBUS
+	help
+	  Say yes if you want to support the I2C serial bus on Cavium
+	  ThunderX SOC.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called i2c-thunderx.
+
 config I2C_XILINX
 	tristate "Xilinx I2C Controller"
 	depends on HAS_IOMEM
diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
index 37f2819b4560..29764cc20a44 100644
--- a/drivers/i2c/busses/Makefile
+++ b/drivers/i2c/busses/Makefile
@@ -91,7 +91,10 @@ obj-$(CONFIG_I2C_UNIPHIER)	+= i2c-uniphier.o
 obj-$(CONFIG_I2C_UNIPHIER_F)	+= i2c-uniphier-f.o
 obj-$(CONFIG_I2C_VERSATILE)	+= i2c-versatile.o
 obj-$(CONFIG_I2C_WMT)		+= i2c-wmt.o
+i2c-octeon-objs := i2c-octeon-core.o i2c-octeon-platdrv.o
 obj-$(CONFIG_I2C_OCTEON)	+= i2c-octeon.o
+i2c-thunderx-objs := i2c-octeon-core.o i2c-thunderx-pcidrv.o
+obj-$(CONFIG_I2C_THUNDERX)	+= i2c-thunderx.o
 obj-$(CONFIG_I2C_XILINX)	+= i2c-xiic.o
 obj-$(CONFIG_I2C_XLR)		+= i2c-xlr.o
 obj-$(CONFIG_I2C_XLP9XX)	+= i2c-xlp9xx.o
diff --git a/drivers/i2c/busses/i2c-amd756.c b/drivers/i2c/busses/i2c-amd756.c
index 6c7113d990f8..274908cd1fde 100644
--- a/drivers/i2c/busses/i2c-amd756.c
+++ b/drivers/i2c/busses/i2c-amd756.c
@@ -378,11 +378,8 @@ static int amd756_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		 amd756_ioport);
 
 	error = i2c_add_adapter(&amd756_smbus);
-	if (error) {
-		dev_err(&pdev->dev,
-			"Adapter registration failed, module not inserted\n");
+	if (error)
 		goto out_err;
-	}
 
 	return 0;
 
diff --git a/drivers/i2c/busses/i2c-at91.c b/drivers/i2c/busses/i2c-at91.c
index f23372669f77..0b86c6173e07 100644
--- a/drivers/i2c/busses/i2c-at91.c
+++ b/drivers/i2c/busses/i2c-at91.c
@@ -38,6 +38,7 @@
 #define AT91_I2C_TIMEOUT	msecs_to_jiffies(100)	/* transfer timeout */
 #define AT91_I2C_DMA_THRESHOLD	8			/* enable DMA if transfer size is bigger than this threshold */
 #define AUTOSUSPEND_TIMEOUT		2000
+#define AT91_I2C_MAX_ALT_CMD_DATA_SIZE	256
 
 /* AT91 TWI register definitions */
 #define	AT91_TWI_CR		0x0000	/* Control Register */
@@ -141,6 +142,7 @@ struct at91_twi_dev {
 	unsigned twi_cwgr_reg;
 	struct at91_twi_pdata *pdata;
 	bool use_dma;
+	bool use_alt_cmd;
 	bool recv_len_abort;
 	u32 fifo_size;
 	struct at91_twi_dma dma;
@@ -269,7 +271,7 @@ static void at91_twi_write_next_byte(struct at91_twi_dev *dev)
 
 	/* send stop when last byte has been written */
 	if (--dev->buf_len == 0)
-		if (!dev->pdata->has_alt_cmd)
+		if (!dev->use_alt_cmd)
 			at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_STOP);
 
 	dev_dbg(dev->dev, "wrote 0x%x, to go %d\n", *dev->buf, dev->buf_len);
@@ -292,7 +294,7 @@ static void at91_twi_write_data_dma_callback(void *data)
 	 * we just have to enable TXCOMP one.
 	 */
 	at91_twi_write(dev, AT91_TWI_IER, AT91_TWI_TXCOMP);
-	if (!dev->pdata->has_alt_cmd)
+	if (!dev->use_alt_cmd)
 		at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_STOP);
 }
 
@@ -410,7 +412,7 @@ static void at91_twi_read_next_byte(struct at91_twi_dev *dev)
 	}
 
 	/* send stop if second but last byte has been read */
-	if (!dev->pdata->has_alt_cmd && dev->buf_len == 1)
+	if (!dev->use_alt_cmd && dev->buf_len == 1)
 		at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_STOP);
 
 	dev_dbg(dev->dev, "read 0x%x, to go %d\n", *dev->buf, dev->buf_len);
@@ -426,7 +428,7 @@ static void at91_twi_read_data_dma_callback(void *data)
 	dma_unmap_single(dev->dev, sg_dma_address(&dev->dma.sg[0]),
 			 dev->buf_len, DMA_FROM_DEVICE);
 
-	if (!dev->pdata->has_alt_cmd) {
+	if (!dev->use_alt_cmd) {
 		/* The last two bytes have to be read without using dma */
 		dev->buf += dev->buf_len - 2;
 		dev->buf_len = 2;
@@ -443,7 +445,7 @@ static void at91_twi_read_data_dma(struct at91_twi_dev *dev)
 	struct dma_chan *chan_rx = dma->chan_rx;
 	size_t buf_len;
 
-	buf_len = (dev->pdata->has_alt_cmd) ? dev->buf_len : dev->buf_len - 2;
+	buf_len = (dev->use_alt_cmd) ? dev->buf_len : dev->buf_len - 2;
 	dma->direction = DMA_FROM_DEVICE;
 
 	/* Keep in mind that we won't use dma to read the last two bytes */
@@ -651,7 +653,7 @@ static int at91_do_twi_transfer(struct at91_twi_dev *dev)
 		unsigned start_flags = AT91_TWI_START;
 
 		/* if only one byte is to be read, immediately stop transfer */
-		if (!has_alt_cmd && dev->buf_len <= 1 &&
+		if (!dev->use_alt_cmd && dev->buf_len <= 1 &&
 		    !(dev->msg->flags & I2C_M_RECV_LEN))
 			start_flags |= AT91_TWI_STOP;
 		at91_twi_write(dev, AT91_TWI_CR, start_flags);
@@ -745,7 +747,7 @@ static int at91_twi_xfer(struct i2c_adapter *adap, struct i2c_msg *msg, int num)
 	int ret;
 	unsigned int_addr_flag = 0;
 	struct i2c_msg *m_start = msg;
-	bool is_read, use_alt_cmd = false;
+	bool is_read;
 
 	dev_dbg(&adap->dev, "at91_xfer: processing %d messages:\n", num);
 
@@ -768,14 +770,16 @@ static int at91_twi_xfer(struct i2c_adapter *adap, struct i2c_msg *msg, int num)
 		at91_twi_write(dev, AT91_TWI_IADR, internal_address);
 	}
 
+	dev->use_alt_cmd = false;
 	is_read = (m_start->flags & I2C_M_RD);
 	if (dev->pdata->has_alt_cmd) {
-		if (m_start->len > 0) {
+		if (m_start->len > 0 &&
+		    m_start->len < AT91_I2C_MAX_ALT_CMD_DATA_SIZE) {
 			at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_ACMEN);
 			at91_twi_write(dev, AT91_TWI_ACR,
 				       AT91_TWI_ACR_DATAL(m_start->len) |
 				       ((is_read) ? AT91_TWI_ACR_DIR : 0));
-			use_alt_cmd = true;
+			dev->use_alt_cmd = true;
 		} else {
 			at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_ACMDIS);
 		}
@@ -784,7 +788,7 @@ static int at91_twi_xfer(struct i2c_adapter *adap, struct i2c_msg *msg, int num)
 	at91_twi_write(dev, AT91_TWI_MMR,
 		       (m_start->addr << 16) |
 		       int_addr_flag |
-		       ((!use_alt_cmd && is_read) ? AT91_TWI_MREAD : 0));
+		       ((!dev->use_alt_cmd && is_read) ? AT91_TWI_MREAD : 0));
 
 	dev->buf_len = m_start->len;
 	dev->buf = m_start->buf;
@@ -1118,8 +1122,6 @@ static int at91_twi_probe(struct platform_device *pdev)
 
 	rc = i2c_add_numbered_adapter(&dev->adapter);
 	if (rc) {
-		dev_err(dev->dev, "Adapter %s registration failed\n",
-			dev->adapter.name);
 		clk_disable_unprepare(dev->clk);
 
 		pm_runtime_disable(dev->dev);
diff --git a/drivers/i2c/busses/i2c-axxia.c b/drivers/i2c/busses/i2c-axxia.c
index c335cc7852f9..4351a9343058 100644
--- a/drivers/i2c/busses/i2c-axxia.c
+++ b/drivers/i2c/busses/i2c-axxia.c
@@ -545,7 +545,11 @@ static int axxia_i2c_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	clk_prepare_enable(idev->i2c_clk);
+	ret = clk_prepare_enable(idev->i2c_clk);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to enable clock\n");
+		return ret;
+	}
 
 	i2c_set_adapdata(&idev->adapter, idev);
 	strlcpy(idev->adapter.name, pdev->name, sizeof(idev->adapter.name));
@@ -560,7 +564,7 @@ static int axxia_i2c_probe(struct platform_device *pdev)
 
 	ret = i2c_add_adapter(&idev->adapter);
 	if (ret) {
-		dev_err(&pdev->dev, "failed to add adapter\n");
+		clk_disable_unprepare(idev->i2c_clk);
 		return ret;
 	}
 
diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c
index 19c843828fe2..326b3db02c48 100644
--- a/drivers/i2c/busses/i2c-bcm-iproc.c
+++ b/drivers/i2c/busses/i2c-bcm-iproc.c
@@ -158,7 +158,7 @@ static irqreturn_t bcm_iproc_i2c_isr(int irq, void *data)
 
 	if (status & BIT(IS_M_START_BUSY_SHIFT)) {
 		iproc_i2c->xfer_is_done = 1;
-		complete_all(&iproc_i2c->done);
+		complete(&iproc_i2c->done);
 	}
 
 	writel(status, iproc_i2c->base + IS_OFFSET);
@@ -488,13 +488,7 @@ static int bcm_iproc_i2c_probe(struct platform_device *pdev)
 	adap->dev.parent = &pdev->dev;
 	adap->dev.of_node = pdev->dev.of_node;
 
-	ret = i2c_add_adapter(adap);
-	if (ret) {
-		dev_err(iproc_i2c->device, "failed to add adapter\n");
-		return ret;
-	}
-
-	return 0;
+	return i2c_add_adapter(adap);
 }
 
 static int bcm_iproc_i2c_remove(struct platform_device *pdev)
diff --git a/drivers/i2c/busses/i2c-bcm-kona.c b/drivers/i2c/busses/i2c-bcm-kona.c
index ac9f47679c3a..4e489a9d16fb 100644
--- a/drivers/i2c/busses/i2c-bcm-kona.c
+++ b/drivers/i2c/busses/i2c-bcm-kona.c
@@ -229,7 +229,7 @@ static irqreturn_t bcm_kona_i2c_isr(int irq, void *devid)
 		       dev->base + TXFCR_OFFSET);
 
 	writel(status & ~ISR_RESERVED_MASK, dev->base + ISR_OFFSET);
-	complete_all(&dev->done);
+	complete(&dev->done);
 
 	return IRQ_HANDLED;
 }
@@ -643,7 +643,7 @@ static int bcm_kona_i2c_xfer(struct i2c_adapter *adapter,
 			if (rc < 0) {
 				dev_err(dev->device,
 					"restart cmd failed rc = %d\n", rc);
-					goto xfer_send_stop;
+				goto xfer_send_stop;
 			}
 		}
 
@@ -858,10 +858,8 @@ static int bcm_kona_i2c_probe(struct platform_device *pdev)
 	adap->dev.of_node = pdev->dev.of_node;
 
 	rc = i2c_add_adapter(adap);
-	if (rc) {
-		dev_err(dev->device, "failed to add adapter\n");
+	if (rc)
 		return rc;
-	}
 
 	dev_info(dev->device, "device registered successfully\n");
 
diff --git a/drivers/i2c/busses/i2c-bfin-twi.c b/drivers/i2c/busses/i2c-bfin-twi.c
index 025686d41640..29d00c4f7824 100644
--- a/drivers/i2c/busses/i2c-bfin-twi.c
+++ b/drivers/i2c/busses/i2c-bfin-twi.c
@@ -685,10 +685,8 @@ static int i2c_bfin_twi_probe(struct platform_device *pdev)
 	write_CONTROL(iface, read_CONTROL(iface) | TWI_ENA);
 
 	rc = i2c_add_numbered_adapter(p_adap);
-	if (rc < 0) {
-		dev_err(&pdev->dev, "Can't add i2c adapter!\n");
+	if (rc < 0)
 		goto out_error;
-	}
 
 	platform_set_drvdata(pdev, iface);
 
diff --git a/drivers/i2c/busses/i2c-brcmstb.c b/drivers/i2c/busses/i2c-brcmstb.c
index 3f5a4d71d3bf..0652281662a8 100644
--- a/drivers/i2c/busses/i2c-brcmstb.c
+++ b/drivers/i2c/busses/i2c-brcmstb.c
@@ -228,7 +228,7 @@ static irqreturn_t brcmstb_i2c_isr(int irq, void *devid)
 		return IRQ_NONE;
 
 	brcmstb_i2c_enable_disable_irq(dev, INT_DISABLE);
-	complete_all(&dev->done);
+	complete(&dev->done);
 
 	dev_dbg(dev->device, "isr handled");
 	return IRQ_HANDLED;
@@ -648,10 +648,8 @@ static int brcmstb_i2c_probe(struct platform_device *pdev)
 	adap->dev.parent = &pdev->dev;
 	adap->dev.of_node = pdev->dev.of_node;
 	rc = i2c_add_adapter(adap);
-	if (rc) {
-		dev_err(dev->device, "failed to add adapter\n");
+	if (rc)
 		goto probe_errorout;
-	}
 
 	dev_info(dev->device, "%s@%dhz registered in %s mode\n",
 		 int_name ? int_name : " ", dev->clk_freq_hz,
diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c
index 90bbd9f9dd8f..686971263bef 100644
--- a/drivers/i2c/busses/i2c-cadence.c
+++ b/drivers/i2c/busses/i2c-cadence.c
@@ -767,7 +767,7 @@ static int cdns_i2c_setclk(unsigned long clk_in, struct cdns_i2c *id)
  * depending on the scaling direction.
  *
  * Return:	NOTIFY_STOP if the rate change should be aborted, NOTIFY_OK
- *		to acknowedge the change, NOTIFY_DONE if the notification is
+ *		to acknowledge the change, NOTIFY_DONE if the notification is
  *		considered irrelevant.
  */
 static int cdns_i2c_clk_notifier_cb(struct notifier_block *nb, unsigned long
@@ -963,10 +963,8 @@ static int cdns_i2c_probe(struct platform_device *pdev)
 	}
 
 	ret = i2c_add_adapter(&id->adap);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "reg adap failed: %d\n", ret);
+	if (ret < 0)
 		goto err_clk_dis;
-	}
 
 	/*
 	 * Cadence I2C controller has a bug wherein it generates
diff --git a/drivers/i2c/busses/i2c-cpm.c b/drivers/i2c/busses/i2c-cpm.c
index ee57c1e865e2..d89bde2c5da2 100644
--- a/drivers/i2c/busses/i2c-cpm.c
+++ b/drivers/i2c/busses/i2c-cpm.c
@@ -665,10 +665,8 @@ static int cpm_i2c_probe(struct platform_device *ofdev)
 	cpm->adap.nr = (data && len == 4) ? be32_to_cpup(data) : -1;
 	result = i2c_add_numbered_adapter(&cpm->adap);
 
-	if (result < 0) {
-		dev_err(&ofdev->dev, "Unable to register with I2C\n");
+	if (result < 0)
 		goto out_shut;
-	}
 
 	dev_dbg(&ofdev->dev, "hw routines for %s registered.\n",
 		cpm->adap.name);
diff --git a/drivers/i2c/busses/i2c-cros-ec-tunnel.c b/drivers/i2c/busses/i2c-cros-ec-tunnel.c
index a0d95ff682ae..9b36a7b3befd 100644
--- a/drivers/i2c/busses/i2c-cros-ec-tunnel.c
+++ b/drivers/i2c/busses/i2c-cros-ec-tunnel.c
@@ -215,7 +215,7 @@ static int ec_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg i2c_msgs[],
 	msg->outsize = request_len;
 	msg->insize = response_len;
 
-	result = cros_ec_cmd_xfer(bus->ec, msg);
+	result = cros_ec_cmd_xfer_status(bus->ec, msg);
 	if (result < 0) {
 		dev_err(dev, "Error transferring EC i2c message %d\n", result);
 		goto exit;
@@ -281,10 +281,8 @@ static int ec_i2c_probe(struct platform_device *pdev)
 	bus->adap.retries = I2C_MAX_RETRIES;
 
 	err = i2c_add_adapter(&bus->adap);
-	if (err) {
-		dev_err(dev, "cannot register i2c adapter\n");
+	if (err)
 		return err;
-	}
 	platform_set_drvdata(pdev, bus);
 
 	return err;
diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
index a8bdcb5292f5..9e7ef5cf5d49 100644
--- a/drivers/i2c/busses/i2c-davinci.c
+++ b/drivers/i2c/busses/i2c-davinci.c
@@ -846,10 +846,8 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 
 	adap->nr = pdev->id;
 	r = i2c_add_numbered_adapter(adap);
-	if (r) {
-		dev_err(&pdev->dev, "failure adding adapter\n");
+	if (r)
 		goto err_unuse_clocks;
-	}
 
 	return 0;
 
diff --git a/drivers/i2c/busses/i2c-designware-core.c b/drivers/i2c/busses/i2c-designware-core.c
index c6922b806fb7..1fe93c43215c 100644
--- a/drivers/i2c/busses/i2c-designware-core.c
+++ b/drivers/i2c/busses/i2c-designware-core.c
@@ -42,6 +42,8 @@
 #define DW_IC_SS_SCL_LCNT	0x18
 #define DW_IC_FS_SCL_HCNT	0x1c
 #define DW_IC_FS_SCL_LCNT	0x20
+#define DW_IC_HS_SCL_HCNT	0x24
+#define DW_IC_HS_SCL_LCNT	0x28
 #define DW_IC_INTR_STAT		0x2c
 #define DW_IC_INTR_MASK		0x30
 #define DW_IC_RAW_INTR_STAT	0x34
@@ -89,12 +91,17 @@
 					 DW_IC_INTR_TX_ABRT | \
 					 DW_IC_INTR_STOP_DET)
 
-#define DW_IC_STATUS_ACTIVITY	0x1
+#define DW_IC_STATUS_ACTIVITY		0x1
+#define DW_IC_STATUS_TFE		BIT(2)
+#define DW_IC_STATUS_MST_ACTIVITY	BIT(5)
 
 #define DW_IC_ERR_TX_ABRT	0x1
 
 #define DW_IC_TAR_10BITADDR_MASTER BIT(12)
 
+#define DW_IC_COMP_PARAM_1_SPEED_MODE_HIGH	(BIT(2) | BIT(3))
+#define DW_IC_COMP_PARAM_1_SPEED_MODE_MASK	GENMASK(3, 2)
+
 /*
  * status codes
  */
@@ -252,10 +259,15 @@ static u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset)
 
 static void __i2c_dw_enable(struct dw_i2c_dev *dev, bool enable)
 {
+	dw_writel(dev, enable, DW_IC_ENABLE);
+}
+
+static void __i2c_dw_enable_and_wait(struct dw_i2c_dev *dev, bool enable)
+{
 	int timeout = 100;
 
 	do {
-		dw_writel(dev, enable, DW_IC_ENABLE);
+		__i2c_dw_enable(dev, enable);
 		if ((dw_readl(dev, DW_IC_ENABLE_STATUS) & 1) == enable)
 			return;
 
@@ -282,6 +294,28 @@ static unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev)
 	return dev->get_clk_rate_khz(dev);
 }
 
+static int i2c_dw_acquire_lock(struct dw_i2c_dev *dev)
+{
+	int ret;
+
+	if (!dev->acquire_lock)
+		return 0;
+
+	ret = dev->acquire_lock(dev);
+	if (!ret)
+		return 0;
+
+	dev_err(dev->dev, "couldn't acquire bus ownership\n");
+
+	return ret;
+}
+
+static void i2c_dw_release_lock(struct dw_i2c_dev *dev)
+{
+	if (dev->release_lock)
+		dev->release_lock(dev);
+}
+
 /**
  * i2c_dw_init() - initialize the designware i2c master hardware
  * @dev: device private data
@@ -293,17 +327,13 @@ static unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev)
 int i2c_dw_init(struct dw_i2c_dev *dev)
 {
 	u32 hcnt, lcnt;
-	u32 reg;
+	u32 reg, comp_param1;
 	u32 sda_falling_time, scl_falling_time;
 	int ret;
 
-	if (dev->acquire_lock) {
-		ret = dev->acquire_lock(dev);
-		if (ret) {
-			dev_err(dev->dev, "couldn't acquire bus ownership\n");
-			return ret;
-		}
-	}
+	ret = i2c_dw_acquire_lock(dev);
+	if (ret)
+		return ret;
 
 	reg = dw_readl(dev, DW_IC_COMP_TYPE);
 	if (reg == ___constant_swab32(DW_IC_COMP_TYPE_VALUE)) {
@@ -315,13 +345,14 @@ int i2c_dw_init(struct dw_i2c_dev *dev)
 	} else if (reg != DW_IC_COMP_TYPE_VALUE) {
 		dev_err(dev->dev, "Unknown Synopsys component type: "
 			"0x%08x\n", reg);
-		if (dev->release_lock)
-			dev->release_lock(dev);
+		i2c_dw_release_lock(dev);
 		return -ENODEV;
 	}
 
+	comp_param1 = dw_readl(dev, DW_IC_COMP_PARAM_1);
+
 	/* Disable the adapter */
-	__i2c_dw_enable(dev, false);
+	__i2c_dw_enable_and_wait(dev, false);
 
 	/* set standard and fast speed deviders for high/low periods */
 
@@ -347,8 +378,11 @@ int i2c_dw_init(struct dw_i2c_dev *dev)
 	dw_writel(dev, lcnt, DW_IC_SS_SCL_LCNT);
 	dev_dbg(dev->dev, "Standard-mode HCNT:LCNT = %d:%d\n", hcnt, lcnt);
 
-	/* Set SCL timing parameters for fast-mode */
-	if (dev->fs_hcnt && dev->fs_lcnt) {
+	/* Set SCL timing parameters for fast-mode or fast-mode plus */
+	if ((dev->clk_freq == 1000000) && dev->fp_hcnt && dev->fp_lcnt) {
+		hcnt = dev->fp_hcnt;
+		lcnt = dev->fp_lcnt;
+	} else if (dev->fs_hcnt && dev->fs_lcnt) {
 		hcnt = dev->fs_hcnt;
 		lcnt = dev->fs_lcnt;
 	} else {
@@ -366,14 +400,35 @@ int i2c_dw_init(struct dw_i2c_dev *dev)
 	dw_writel(dev, lcnt, DW_IC_FS_SCL_LCNT);
 	dev_dbg(dev->dev, "Fast-mode HCNT:LCNT = %d:%d\n", hcnt, lcnt);
 
+	if ((dev->master_cfg & DW_IC_CON_SPEED_MASK) ==
+		DW_IC_CON_SPEED_HIGH) {
+		if ((comp_param1 & DW_IC_COMP_PARAM_1_SPEED_MODE_MASK)
+			!= DW_IC_COMP_PARAM_1_SPEED_MODE_HIGH) {
+			dev_err(dev->dev, "High Speed not supported!\n");
+			dev->master_cfg &= ~DW_IC_CON_SPEED_MASK;
+			dev->master_cfg |= DW_IC_CON_SPEED_FAST;
+		} else if (dev->hs_hcnt && dev->hs_lcnt) {
+			hcnt = dev->hs_hcnt;
+			lcnt = dev->hs_lcnt;
+			dw_writel(dev, hcnt, DW_IC_HS_SCL_HCNT);
+			dw_writel(dev, lcnt, DW_IC_HS_SCL_LCNT);
+			dev_dbg(dev->dev, "HighSpeed-mode HCNT:LCNT = %d:%d\n",
+				hcnt, lcnt);
+		}
+	}
+
 	/* Configure SDA Hold Time if required */
-	if (dev->sda_hold_time) {
-		reg = dw_readl(dev, DW_IC_COMP_VERSION);
-		if (reg >= DW_IC_SDA_HOLD_MIN_VERS)
+	reg = dw_readl(dev, DW_IC_COMP_VERSION);
+	if (reg >= DW_IC_SDA_HOLD_MIN_VERS) {
+		if (dev->sda_hold_time) {
 			dw_writel(dev, dev->sda_hold_time, DW_IC_SDA_HOLD);
-		else
-			dev_warn(dev->dev,
-				"Hardware too old to adjust SDA hold time.");
+		} else {
+			/* Keep previous hold time setting if no one set it */
+			dev->sda_hold_time = dw_readl(dev, DW_IC_SDA_HOLD);
+		}
+	} else {
+		dev_warn(dev->dev,
+			"Hardware too old to adjust SDA hold time.\n");
 	}
 
 	/* Configure Tx/Rx FIFO threshold levels */
@@ -383,8 +438,8 @@ int i2c_dw_init(struct dw_i2c_dev *dev)
 	/* configure the i2c master */
 	dw_writel(dev, dev->master_cfg , DW_IC_CON);
 
-	if (dev->release_lock)
-		dev->release_lock(dev);
+	i2c_dw_release_lock(dev);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(i2c_dw_init);
@@ -411,27 +466,45 @@ static int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev)
 static void i2c_dw_xfer_init(struct dw_i2c_dev *dev)
 {
 	struct i2c_msg *msgs = dev->msgs;
-	u32 ic_con, ic_tar = 0;
+	u32 ic_tar = 0;
+	bool enabled;
 
-	/* Disable the adapter */
-	__i2c_dw_enable(dev, false);
+	enabled = dw_readl(dev, DW_IC_ENABLE_STATUS) & 1;
+
+	if (enabled) {
+		u32 ic_status;
+
+		/*
+		 * Only disable adapter if ic_tar and ic_con can't be
+		 * dynamically updated
+		 */
+		ic_status = dw_readl(dev, DW_IC_STATUS);
+		if (!dev->dynamic_tar_update_enabled ||
+		    (ic_status & DW_IC_STATUS_MST_ACTIVITY) ||
+		    !(ic_status & DW_IC_STATUS_TFE)) {
+			__i2c_dw_enable_and_wait(dev, false);
+			enabled = false;
+		}
+	}
 
 	/* if the slave address is ten bit address, enable 10BITADDR */
-	ic_con = dw_readl(dev, DW_IC_CON);
-	if (msgs[dev->msg_write_idx].flags & I2C_M_TEN) {
-		ic_con |= DW_IC_CON_10BITADDR_MASTER;
+	if (dev->dynamic_tar_update_enabled) {
 		/*
 		 * If I2C_DYNAMIC_TAR_UPDATE is set, the 10-bit addressing
-		 * mode has to be enabled via bit 12 of IC_TAR register.
-		 * We set it always as I2C_DYNAMIC_TAR_UPDATE can't be
-		 * detected from registers.
+		 * mode has to be enabled via bit 12 of IC_TAR register,
+		 * otherwise bit 4 of IC_CON is used.
 		 */
-		ic_tar = DW_IC_TAR_10BITADDR_MASTER;
+		if (msgs[dev->msg_write_idx].flags & I2C_M_TEN)
+			ic_tar = DW_IC_TAR_10BITADDR_MASTER;
 	} else {
-		ic_con &= ~DW_IC_CON_10BITADDR_MASTER;
-	}
+		u32 ic_con = dw_readl(dev, DW_IC_CON);
 
-	dw_writel(dev, ic_con, DW_IC_CON);
+		if (msgs[dev->msg_write_idx].flags & I2C_M_TEN)
+			ic_con |= DW_IC_CON_10BITADDR_MASTER;
+		else
+			ic_con &= ~DW_IC_CON_10BITADDR_MASTER;
+		dw_writel(dev, ic_con, DW_IC_CON);
+	}
 
 	/*
 	 * Set the slave (target) address and enable 10-bit addressing mode
@@ -442,8 +515,8 @@ static void i2c_dw_xfer_init(struct dw_i2c_dev *dev)
 	/* enforce disabled interrupts (due to HW issues) */
 	i2c_dw_disable_int(dev);
 
-	/* Enable the adapter */
-	__i2c_dw_enable(dev, true);
+	if (!enabled)
+		__i2c_dw_enable(dev, true);
 
 	/* Clear and enable interrupts */
 	dw_readl(dev, DW_IC_CLR_INTR);
@@ -624,7 +697,8 @@ static int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev)
 }
 
 /*
- * Prepare controller for a transaction and call i2c_dw_xfer_msg
+ * Prepare controller for a transaction and start transfer by calling
+ * i2c_dw_xfer_init()
  */
 static int
 i2c_dw_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
@@ -647,13 +721,9 @@ i2c_dw_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 	dev->abort_source = 0;
 	dev->rx_outstanding = 0;
 
-	if (dev->acquire_lock) {
-		ret = dev->acquire_lock(dev);
-		if (ret) {
-			dev_err(dev->dev, "couldn't acquire bus ownership\n");
-			goto done_nolock;
-		}
-	}
+	ret = i2c_dw_acquire_lock(dev);
+	if (ret)
+		goto done_nolock;
 
 	ret = i2c_dw_wait_bus_not_busy(dev);
 	if (ret < 0)
@@ -671,16 +741,6 @@ i2c_dw_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 		goto done;
 	}
 
-	/*
-	 * We must disable the adapter before returning and signaling the end
-	 * of the current transfer. Otherwise the hardware might continue
-	 * generating interrupts which in turn causes a race condition with
-	 * the following transfer.  Needs some more investigation if the
-	 * additional interrupts are a hardware bug or this driver doesn't
-	 * handle them correctly yet.
-	 */
-	__i2c_dw_enable(dev, false);
-
 	if (dev->msg_err) {
 		ret = dev->msg_err;
 		goto done;
@@ -700,8 +760,7 @@ i2c_dw_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 	ret = -EIO;
 
 done:
-	if (dev->release_lock)
-		dev->release_lock(dev);
+	i2c_dw_release_lock(dev);
 
 done_nolock:
 	pm_runtime_mark_last_busy(dev->dev);
@@ -818,9 +877,19 @@ static irqreturn_t i2c_dw_isr(int this_irq, void *dev_id)
 	 */
 
 tx_aborted:
-	if ((stat & (DW_IC_INTR_TX_ABRT | DW_IC_INTR_STOP_DET)) || dev->msg_err)
+	if ((stat & (DW_IC_INTR_TX_ABRT | DW_IC_INTR_STOP_DET))
+			|| dev->msg_err) {
+		/*
+		 * We must disable interruts before returning and signaling
+		 * the end of the current transfer. Otherwise the hardware
+		 * might continue generating interrupts for non-existent
+		 * transfers.
+		 */
+		i2c_dw_disable_int(dev);
+		dw_readl(dev, DW_IC_CLR_INTR);
+
 		complete(&dev->cmd_complete);
-	else if (unlikely(dev->accessor_flags & ACCESS_INTR_MASK)) {
+	} else if (unlikely(dev->accessor_flags & ACCESS_INTR_MASK)) {
 		/* workaround to trigger pending interrupt */
 		stat = dw_readl(dev, DW_IC_INTR_MASK);
 		i2c_dw_disable_int(dev);
@@ -833,7 +902,7 @@ tx_aborted:
 void i2c_dw_disable(struct dw_i2c_dev *dev)
 {
 	/* Disable controller */
-	__i2c_dw_enable(dev, false);
+	__i2c_dw_enable_and_wait(dev, false);
 
 	/* Disable all interupts */
 	dw_writel(dev, 0, DW_IC_INTR_MASK);
@@ -857,6 +926,7 @@ int i2c_dw_probe(struct dw_i2c_dev *dev)
 {
 	struct i2c_adapter *adap = &dev->adapter;
 	int r;
+	u32 reg;
 
 	init_completion(&dev->cmd_complete);
 
@@ -864,6 +934,26 @@ int i2c_dw_probe(struct dw_i2c_dev *dev)
 	if (r)
 		return r;
 
+	r = i2c_dw_acquire_lock(dev);
+	if (r)
+		return r;
+
+	/*
+	 * Test if dynamic TAR update is enabled in this controller by writing
+	 * to IC_10BITADDR_MASTER field in IC_CON: when it is enabled this
+	 * field is read-only so it should not succeed
+	 */
+	reg = dw_readl(dev, DW_IC_CON);
+	dw_writel(dev, reg ^ DW_IC_CON_10BITADDR_MASTER, DW_IC_CON);
+
+	if ((dw_readl(dev, DW_IC_CON) & DW_IC_CON_10BITADDR_MASTER) ==
+	    (reg & DW_IC_CON_10BITADDR_MASTER)) {
+		dev->dynamic_tar_update_enabled = true;
+		dev_dbg(dev->dev, "Dynamic TAR update enabled");
+	}
+
+	i2c_dw_release_lock(dev);
+
 	snprintf(adap->name, sizeof(adap->name),
 		 "Synopsys DesignWare I2C adapter");
 	adap->retries = 3;
diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h
index 38493a7142ad..0d44d2ae7d4c 100644
--- a/drivers/i2c/busses/i2c-designware-core.h
+++ b/drivers/i2c/busses/i2c-designware-core.h
@@ -26,6 +26,7 @@
 #define DW_IC_CON_MASTER		0x1
 #define DW_IC_CON_SPEED_STD		0x2
 #define DW_IC_CON_SPEED_FAST		0x4
+#define DW_IC_CON_SPEED_HIGH		0x6
 #define DW_IC_CON_SPEED_MASK		0x6
 #define DW_IC_CON_10BITADDR_MASTER	0x10
 #define DW_IC_CON_RESTART_EN		0x20
@@ -57,10 +58,15 @@
  * @tx_fifo_depth: depth of the hardware tx fifo
  * @rx_fifo_depth: depth of the hardware rx fifo
  * @rx_outstanding: current master-rx elements in tx fifo
+ * @clk_freq: bus clock frequency
  * @ss_hcnt: standard speed HCNT value
  * @ss_lcnt: standard speed LCNT value
  * @fs_hcnt: fast speed HCNT value
  * @fs_lcnt: fast speed LCNT value
+ * @fp_hcnt: fast plus HCNT value
+ * @fp_lcnt: fast plus LCNT value
+ * @hs_hcnt: high speed HCNT value
+ * @hs_lcnt: high speed LCNT value
  * @acquire_lock: function to acquire a hardware lock on the bus
  * @release_lock: function to release a hardware lock on the bus
  * @pm_runtime_disabled: true if pm runtime is disabled
@@ -96,6 +102,7 @@ struct dw_i2c_dev {
 	unsigned int		tx_fifo_depth;
 	unsigned int		rx_fifo_depth;
 	int			rx_outstanding;
+	u32			clk_freq;
 	u32			sda_hold_time;
 	u32			sda_falling_time;
 	u32			scl_falling_time;
@@ -103,9 +110,14 @@ struct dw_i2c_dev {
 	u16			ss_lcnt;
 	u16			fs_hcnt;
 	u16			fs_lcnt;
+	u16			fp_hcnt;
+	u16			fp_lcnt;
+	u16			hs_hcnt;
+	u16			hs_lcnt;
 	int			(*acquire_lock)(struct dw_i2c_dev *dev);
 	void			(*release_lock)(struct dw_i2c_dev *dev);
 	bool			pm_runtime_disabled;
+	bool			dynamic_tar_update_enabled;
 };
 
 #define ACCESS_SWAP		0x00000001
diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c
index d656657b805c..0b42a12171f3 100644
--- a/drivers/i2c/busses/i2c-designware-platdrv.c
+++ b/drivers/i2c/busses/i2c-designware-platdrv.c
@@ -107,6 +107,8 @@ static int dw_i2c_acpi_configure(struct platform_device *pdev)
 	dw_i2c_acpi_params(pdev, "SSCN", &dev->ss_hcnt, &dev->ss_lcnt, NULL);
 	dw_i2c_acpi_params(pdev, "FMCN", &dev->fs_hcnt, &dev->fs_lcnt,
 			   &dev->sda_hold_time);
+	dw_i2c_acpi_params(pdev, "FPCN", &dev->fp_hcnt, &dev->fp_lcnt, NULL);
+	dw_i2c_acpi_params(pdev, "HSCN", &dev->hs_hcnt, &dev->hs_lcnt, NULL);
 
 	id = acpi_match_device(pdev->dev.driver->acpi_match_table, &pdev->dev);
 	if (id && id->driver_data)
@@ -155,7 +157,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	struct i2c_adapter *adap;
 	struct resource *mem;
 	int irq, r;
-	u32 clk_freq, ht = 0;
+	u32 acpi_speed, ht = 0;
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
@@ -175,10 +177,10 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, dev);
 
 	/* fast mode by default because of legacy reasons */
-	clk_freq = 400000;
+	dev->clk_freq = 400000;
 
 	if (pdata) {
-		clk_freq = pdata->i2c_scl_freq;
+		dev->clk_freq = pdata->i2c_scl_freq;
 	} else {
 		device_property_read_u32(&pdev->dev, "i2c-sda-hold-time-ns",
 					 &ht);
@@ -187,17 +189,24 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 		device_property_read_u32(&pdev->dev, "i2c-scl-falling-time-ns",
 					 &dev->scl_falling_time);
 		device_property_read_u32(&pdev->dev, "clock-frequency",
-					 &clk_freq);
+					 &dev->clk_freq);
 	}
 
+	acpi_speed = i2c_acpi_find_bus_speed(&pdev->dev);
+	if (acpi_speed)
+		dev->clk_freq = acpi_speed;
+
 	if (has_acpi_companion(&pdev->dev))
 		dw_i2c_acpi_configure(pdev);
 
 	/*
-	 * Only standard mode at 100kHz and fast mode at 400kHz are supported.
+	 * Only standard mode at 100kHz, fast mode at 400kHz,
+	 * fast mode plus at 1MHz and high speed mode at 3.4MHz are supported.
 	 */
-	if (clk_freq != 100000 && clk_freq != 400000) {
-		dev_err(&pdev->dev, "Only 100kHz and 400kHz supported");
+	if (dev->clk_freq != 100000 && dev->clk_freq != 400000
+	    && dev->clk_freq != 1000000 && dev->clk_freq != 3400000) {
+		dev_err(&pdev->dev,
+			"Only 100kHz, 400kHz, 1MHz and 3.4MHz supported");
 		return -EINVAL;
 	}
 
@@ -212,12 +221,20 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 		I2C_FUNC_SMBUS_BYTE_DATA |
 		I2C_FUNC_SMBUS_WORD_DATA |
 		I2C_FUNC_SMBUS_I2C_BLOCK;
-	if (clk_freq == 100000)
-		dev->master_cfg =  DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE |
-			DW_IC_CON_RESTART_EN | DW_IC_CON_SPEED_STD;
-	else
-		dev->master_cfg =  DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE |
-			DW_IC_CON_RESTART_EN | DW_IC_CON_SPEED_FAST;
+
+	dev->master_cfg = DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE |
+			  DW_IC_CON_RESTART_EN;
+
+	switch (dev->clk_freq) {
+	case 100000:
+		dev->master_cfg |= DW_IC_CON_SPEED_STD;
+		break;
+	case 3400000:
+		dev->master_cfg |= DW_IC_CON_SPEED_HIGH;
+		break;
+	default:
+		dev->master_cfg |= DW_IC_CON_SPEED_FAST;
+	}
 
 	dev->clk = devm_clk_get(&pdev->dev, NULL);
 	if (!i2c_dw_plat_prepare_clk(dev, true)) {
diff --git a/drivers/i2c/busses/i2c-diolan-u2c.c b/drivers/i2c/busses/i2c-diolan-u2c.c
index b19a310bf9b3..f718ee4e3332 100644
--- a/drivers/i2c/busses/i2c-diolan-u2c.c
+++ b/drivers/i2c/busses/i2c-diolan-u2c.c
@@ -487,10 +487,8 @@ static int diolan_u2c_probe(struct usb_interface *interface,
 
 	/* and finally attach to i2c layer */
 	ret = i2c_add_adapter(&dev->adapter);
-	if (ret < 0) {
-		dev_err(&interface->dev, "failed to add I2C adapter\n");
+	if (ret < 0)
 		goto error_free;
-	}
 
 	dev_dbg(&interface->dev, "connected " DRIVER_NAME "\n");
 
diff --git a/drivers/i2c/busses/i2c-dln2.c b/drivers/i2c/busses/i2c-dln2.c
index f2eb4f76591f..8acda2aa1558 100644
--- a/drivers/i2c/busses/i2c-dln2.c
+++ b/drivers/i2c/busses/i2c-dln2.c
@@ -228,10 +228,8 @@ static int dln2_i2c_probe(struct platform_device *pdev)
 
 	/* and finally attach to i2c layer */
 	ret = i2c_add_adapter(&dln2->adapter);
-	if (ret < 0) {
-		dev_err(dev, "failed to add I2C adapter: %d\n", ret);
+	if (ret < 0)
 		goto out_disable;
-	}
 
 	return 0;
 
diff --git a/drivers/i2c/busses/i2c-efm32.c b/drivers/i2c/busses/i2c-efm32.c
index e253598d764c..aa336ba89aa3 100644
--- a/drivers/i2c/busses/i2c-efm32.c
+++ b/drivers/i2c/busses/i2c-efm32.c
@@ -438,7 +438,6 @@ static int efm32_i2c_probe(struct platform_device *pdev)
 
 	ret = i2c_add_adapter(&ddata->adapter);
 	if (ret) {
-		dev_err(&pdev->dev, "failed to add i2c adapter (%d)\n", ret);
 		free_irq(ddata->irq, ddata);
 
 err_disable_clk:
diff --git a/drivers/i2c/busses/i2c-eg20t.c b/drivers/i2c/busses/i2c-eg20t.c
index 137125b5eae7..5ce71ce7b6c4 100644
--- a/drivers/i2c/busses/i2c-eg20t.c
+++ b/drivers/i2c/busses/i2c-eg20t.c
@@ -773,13 +773,6 @@ static int pch_i2c_probe(struct pci_dev *pdev,
 	/* Set the number of I2C channel instance */
 	adap_info->ch_num = id->driver_data;
 
-	ret = request_irq(pdev->irq, pch_i2c_handler, IRQF_SHARED,
-		  KBUILD_MODNAME, adap_info);
-	if (ret) {
-		pch_pci_err(pdev, "request_irq FAILED\n");
-		goto err_request_irq;
-	}
-
 	for (i = 0; i < adap_info->ch_num; i++) {
 		pch_adap = &adap_info->pch_data[i].pch_adapter;
 		adap_info->pch_i2c_suspended = false;
@@ -797,6 +790,17 @@ static int pch_i2c_probe(struct pci_dev *pdev,
 
 		pch_adap->dev.of_node = pdev->dev.of_node;
 		pch_adap->dev.parent = &pdev->dev;
+	}
+
+	ret = request_irq(pdev->irq, pch_i2c_handler, IRQF_SHARED,
+		  KBUILD_MODNAME, adap_info);
+	if (ret) {
+		pch_pci_err(pdev, "request_irq FAILED\n");
+		goto err_request_irq;
+	}
+
+	for (i = 0; i < adap_info->ch_num; i++) {
+		pch_adap = &adap_info->pch_data[i].pch_adapter;
 
 		pch_i2c_init(&adap_info->pch_data[i]);
 
diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c
index c0e3ada02876..bea607149972 100644
--- a/drivers/i2c/busses/i2c-exynos5.c
+++ b/drivers/i2c/busses/i2c-exynos5.c
@@ -796,10 +796,8 @@ static int exynos5_i2c_probe(struct platform_device *pdev)
 	exynos5_i2c_reset(i2c);
 
 	ret = i2c_add_adapter(&i2c->adap);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to add bus to i2c core\n");
+	if (ret < 0)
 		goto err_clk;
-	}
 
 	platform_set_drvdata(pdev, i2c);
 
diff --git a/drivers/i2c/busses/i2c-hix5hd2.c b/drivers/i2c/busses/i2c-hix5hd2.c
index 7c6966434ee7..ae7f3180f7e8 100644
--- a/drivers/i2c/busses/i2c-hix5hd2.c
+++ b/drivers/i2c/busses/i2c-hix5hd2.c
@@ -478,10 +478,8 @@ static int hix5hd2_i2c_probe(struct platform_device *pdev)
 	pm_runtime_enable(priv->dev);
 
 	ret = i2c_add_adapter(&priv->adap);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to add bus to i2c core\n");
+	if (ret < 0)
 		goto err_runtime;
-	}
 
 	return ret;
 
diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
index 5ef9b733d153..08847e8b8998 100644
--- a/drivers/i2c/busses/i2c-i801.c
+++ b/drivers/i2c/busses/i2c-i801.c
@@ -64,6 +64,7 @@
  * Broxton (SOC)		0x5ad4	32	hard	yes	yes	yes
  * Lewisburg (PCH)		0xa1a3	32	hard	yes	yes	yes
  * Lewisburg Supersku (PCH)	0xa223	32	hard	yes	yes	yes
+ * Kaby Lake PCH-H (PCH)	0xa2a3	32	hard	yes	yes	yes
  *
  * Features supported by this driver:
  * Software PEC				no
@@ -226,6 +227,7 @@
 #define PCI_DEVICE_ID_INTEL_SUNRISEPOINT_H_SMBUS	0xa123
 #define PCI_DEVICE_ID_INTEL_LEWISBURG_SMBUS		0xa1a3
 #define PCI_DEVICE_ID_INTEL_LEWISBURG_SSKU_SMBUS	0xa223
+#define PCI_DEVICE_ID_INTEL_KABYLAKE_PCH_H_SMBUS	0xa2a3
 
 struct i801_mux_config {
 	char *gpio_chip;
@@ -1006,6 +1008,7 @@ static const struct pci_device_id i801_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BROXTON_SMBUS) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LEWISBURG_SMBUS) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LEWISBURG_SSKU_SMBUS) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KABYLAKE_PCH_H_SMBUS) },
 	{ 0, }
 };
 
@@ -1482,11 +1485,14 @@ static int i801_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	case PCI_DEVICE_ID_INTEL_LEWISBURG_SMBUS:
 	case PCI_DEVICE_ID_INTEL_LEWISBURG_SSKU_SMBUS:
 	case PCI_DEVICE_ID_INTEL_DNV_SMBUS:
+	case PCI_DEVICE_ID_INTEL_KABYLAKE_PCH_H_SMBUS:
 		priv->features |= FEATURE_I2C_BLOCK_READ;
 		priv->features |= FEATURE_IRQ;
 		priv->features |= FEATURE_SMBUS_PEC;
 		priv->features |= FEATURE_BLOCK_BUFFER;
-		priv->features |= FEATURE_TCO;
+		/* If we have ACPI based watchdog use that instead */
+		if (!acpi_has_watchdog())
+			priv->features |= FEATURE_TCO;
 		priv->features |= FEATURE_HOST_NOTIFY;
 		break;
 
@@ -1613,7 +1619,6 @@ static int i801_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		"SMBus I801 adapter at %04lx", priv->smba);
 	err = i2c_add_adapter(&priv->adapter);
 	if (err) {
-		dev_err(&dev->dev, "Failed to add SMBus adapter\n");
 		i801_acpi_remove(priv);
 		return err;
 	}
diff --git a/drivers/i2c/busses/i2c-ibm_iic.c b/drivers/i2c/busses/i2c-ibm_iic.c
index cdaa7be2cd1b..412b91d255ad 100644
--- a/drivers/i2c/busses/i2c-ibm_iic.c
+++ b/drivers/i2c/busses/i2c-ibm_iic.c
@@ -751,10 +751,8 @@ static int iic_probe(struct platform_device *ofdev)
 	adap->timeout = HZ;
 
 	ret = i2c_add_adapter(adap);
-	if (ret  < 0) {
-		dev_err(&ofdev->dev, "failed to register i2c adapter\n");
+	if (ret  < 0)
 		goto error_cleanup;
-	}
 
 	dev_info(&ofdev->dev, "using %s mode\n",
 		 dev->fast_mode ? "fast (400 kHz)" : "standard (100 kHz)");
diff --git a/drivers/i2c/busses/i2c-img-scb.c b/drivers/i2c/busses/i2c-img-scb.c
index ea20425b6972..db8e8b40569d 100644
--- a/drivers/i2c/busses/i2c-img-scb.c
+++ b/drivers/i2c/busses/i2c-img-scb.c
@@ -1394,10 +1394,8 @@ static int img_i2c_probe(struct platform_device *pdev)
 		goto disable_clk;
 
 	ret = i2c_add_numbered_adapter(&i2c->adap);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to add adapter\n");
+	if (ret < 0)
 		goto disable_clk;
-	}
 
 	return 0;
 
diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c
index 1844bc9f7cd5..592a8f26a708 100644
--- a/drivers/i2c/busses/i2c-imx.c
+++ b/drivers/i2c/busses/i2c-imx.c
@@ -984,11 +984,24 @@ static void i2c_imx_unprepare_recovery(struct i2c_adapter *adap)
 	pinctrl_select_state(i2c_imx->pinctrl, i2c_imx->pinctrl_pins_default);
 }
 
-static void i2c_imx_init_recovery_info(struct imx_i2c_struct *i2c_imx,
+/*
+ * We switch SCL and SDA to their GPIO function and do some bitbanging
+ * for bus recovery. These alternative pinmux settings can be
+ * described in the device tree by a separate pinctrl state "gpio". If
+ * this is missing this is not a big problem, the only implication is
+ * that we can't do bus recovery.
+ */
+static int i2c_imx_init_recovery_info(struct imx_i2c_struct *i2c_imx,
 		struct platform_device *pdev)
 {
 	struct i2c_bus_recovery_info *rinfo = &i2c_imx->rinfo;
 
+	i2c_imx->pinctrl = devm_pinctrl_get(&pdev->dev);
+	if (!i2c_imx->pinctrl || IS_ERR(i2c_imx->pinctrl)) {
+		dev_info(&pdev->dev, "can't get pinctrl, bus recovery not supported\n");
+		return PTR_ERR(i2c_imx->pinctrl);
+	}
+
 	i2c_imx->pinctrl_pins_default = pinctrl_lookup_state(i2c_imx->pinctrl,
 			PINCTRL_STATE_DEFAULT);
 	i2c_imx->pinctrl_pins_gpio = pinctrl_lookup_state(i2c_imx->pinctrl,
@@ -1001,7 +1014,7 @@ static void i2c_imx_init_recovery_info(struct imx_i2c_struct *i2c_imx,
 	    IS_ERR(i2c_imx->pinctrl_pins_default) ||
 	    IS_ERR(i2c_imx->pinctrl_pins_gpio)) {
 		dev_dbg(&pdev->dev, "recovery information incomplete\n");
-		return;
+		return 0;
 	}
 
 	dev_dbg(&pdev->dev, "using scl-gpio %d and sda-gpio %d for recovery\n",
@@ -1011,6 +1024,8 @@ static void i2c_imx_init_recovery_info(struct imx_i2c_struct *i2c_imx,
 	rinfo->unprepare_recovery = i2c_imx_unprepare_recovery;
 	rinfo->recover_bus = i2c_generic_gpio_recovery;
 	i2c_imx->adapter.bus_recovery_info = rinfo;
+
+	return 0;
 }
 
 static u32 i2c_imx_func(struct i2c_adapter *adapter)
@@ -1081,12 +1096,6 @@ static int i2c_imx_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	i2c_imx->pinctrl = devm_pinctrl_get(&pdev->dev);
-	if (IS_ERR(i2c_imx->pinctrl)) {
-		ret = PTR_ERR(i2c_imx->pinctrl);
-		goto clk_disable;
-	}
-
 	/* Request IRQ */
 	ret = devm_request_irq(&pdev->dev, irq, i2c_imx_isr, 0,
 				pdev->name, i2c_imx);
@@ -1125,14 +1134,16 @@ static int i2c_imx_probe(struct platform_device *pdev)
 			i2c_imx, IMX_I2C_I2CR);
 	imx_i2c_write_reg(i2c_imx->hwdata->i2sr_clr_opcode, i2c_imx, IMX_I2C_I2SR);
 
-	i2c_imx_init_recovery_info(i2c_imx, pdev);
+	/* Init optional bus recovery function */
+	ret = i2c_imx_init_recovery_info(i2c_imx, pdev);
+	/* Give it another chance if pinctrl used is not ready yet */
+	if (ret == -EPROBE_DEFER)
+		goto rpm_disable;
 
 	/* Add I2C adapter */
 	ret = i2c_add_numbered_adapter(&i2c_imx->adapter);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "registration failed\n");
+	if (ret < 0)
 		goto rpm_disable;
-	}
 
 	pm_runtime_mark_last_busy(&pdev->dev);
 	pm_runtime_put_autosuspend(&pdev->dev);
diff --git a/drivers/i2c/busses/i2c-isch.c b/drivers/i2c/busses/i2c-isch.c
index c2f25f19d76f..0cf1379f4e80 100644
--- a/drivers/i2c/busses/i2c-isch.c
+++ b/drivers/i2c/busses/i2c-isch.c
@@ -288,10 +288,8 @@ static int smbus_sch_probe(struct platform_device *dev)
 		"SMBus SCH adapter at %04x", sch_smba);
 
 	retval = i2c_add_adapter(&sch_adapter);
-	if (retval) {
-		dev_err(&dev->dev, "Couldn't register adapter!\n");
+	if (retval)
 		sch_smba = 0;
-	}
 
 	return retval;
 }
diff --git a/drivers/i2c/busses/i2c-ismt.c b/drivers/i2c/busses/i2c-ismt.c
index 1c8707710098..f573448d2132 100644
--- a/drivers/i2c/busses/i2c-ismt.c
+++ b/drivers/i2c/busses/i2c-ismt.c
@@ -922,10 +922,8 @@ ismt_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		return err;
 
 	err = i2c_add_adapter(&priv->adapter);
-	if (err) {
-		dev_err(&pdev->dev, "Failed to add SMBus iSMT adapter\n");
+	if (err)
 		return -ENODEV;
-	}
 	return 0;
 }
 
diff --git a/drivers/i2c/busses/i2c-jz4780.c b/drivers/i2c/busses/i2c-jz4780.c
index cd9872594fe2..b8ea62105f42 100644
--- a/drivers/i2c/busses/i2c-jz4780.c
+++ b/drivers/i2c/busses/i2c-jz4780.c
@@ -798,10 +798,8 @@ static int jz4780_i2c_probe(struct platform_device *pdev)
 		goto err;
 
 	ret = i2c_add_adapter(&i2c->adap);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "Failed to add bus\n");
+	if (ret < 0)
 		goto err;
-	}
 
 	return 0;
 
diff --git a/drivers/i2c/busses/i2c-lpc2k.c b/drivers/i2c/busses/i2c-lpc2k.c
index 586a15205e61..9b1fef455a89 100644
--- a/drivers/i2c/busses/i2c-lpc2k.c
+++ b/drivers/i2c/busses/i2c-lpc2k.c
@@ -432,10 +432,8 @@ static int i2c_lpc2k_probe(struct platform_device *pdev)
 	i2c->adap.dev.of_node = pdev->dev.of_node;
 
 	ret = i2c_add_adapter(&i2c->adap);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to add adapter!\n");
+	if (ret < 0)
 		goto fail_clk;
-	}
 
 	dev_info(&pdev->dev, "LPC2K I2C adapter\n");
 
diff --git a/drivers/i2c/busses/i2c-meson.c b/drivers/i2c/busses/i2c-meson.c
index 71d3929adf54..2aa61bbbd307 100644
--- a/drivers/i2c/busses/i2c-meson.c
+++ b/drivers/i2c/busses/i2c-meson.c
@@ -211,7 +211,7 @@ static void meson_i2c_stop(struct meson_i2c *i2c)
 		meson_i2c_add_token(i2c, TOKEN_STOP);
 	} else {
 		i2c->state = STATE_IDLE;
-		complete_all(&i2c->done);
+		complete(&i2c->done);
 	}
 }
 
@@ -238,7 +238,7 @@ static irqreturn_t meson_i2c_irq(int irqno, void *dev_id)
 		dev_dbg(i2c->dev, "error bit set\n");
 		i2c->error = -ENXIO;
 		i2c->state = STATE_IDLE;
-		complete_all(&i2c->done);
+		complete(&i2c->done);
 		goto out;
 	}
 
@@ -269,7 +269,7 @@ static irqreturn_t meson_i2c_irq(int irqno, void *dev_id)
 		break;
 	case STATE_STOP:
 		i2c->state = STATE_IDLE;
-		complete_all(&i2c->done);
+		complete(&i2c->done);
 		break;
 	case STATE_IDLE:
 		break;
@@ -453,7 +453,6 @@ static int meson_i2c_probe(struct platform_device *pdev)
 
 	ret = i2c_add_adapter(&i2c->adap);
 	if (ret < 0) {
-		dev_err(&pdev->dev, "can't register adapter\n");
 		clk_unprepare(i2c->clk);
 		return ret;
 	}
@@ -473,6 +472,7 @@ static int meson_i2c_remove(struct platform_device *pdev)
 
 static const struct of_device_id meson_i2c_match[] = {
 	{ .compatible = "amlogic,meson6-i2c" },
+	{ .compatible = "amlogic,meson-gxbb-i2c" },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, meson_i2c_match);
diff --git a/drivers/i2c/busses/i2c-mpc.c b/drivers/i2c/busses/i2c-mpc.c
index 48ecffecc0ed..565a49a0c564 100644
--- a/drivers/i2c/busses/i2c-mpc.c
+++ b/drivers/i2c/busses/i2c-mpc.c
@@ -737,10 +737,8 @@ static int fsl_i2c_probe(struct platform_device *op)
 	i2c->adap.dev.of_node = of_node_get(op->dev.of_node);
 
 	result = i2c_add_adapter(&i2c->adap);
-	if (result < 0) {
-		dev_err(i2c->dev, "failed to add adapter\n");
+	if (result < 0)
 		goto fail_add;
-	}
 
 	return result;
 
diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c
index d9373e60be8a..4a7d9bc2142b 100644
--- a/drivers/i2c/busses/i2c-mt65xx.c
+++ b/drivers/i2c/busses/i2c-mt65xx.c
@@ -786,10 +786,8 @@ static int mtk_i2c_probe(struct platform_device *pdev)
 
 	i2c_set_adapdata(&i2c->adap, i2c);
 	ret = i2c_add_adapter(&i2c->adap);
-	if (ret) {
-		dev_err(&pdev->dev, "Failed to add i2c bus to i2c core\n");
+	if (ret)
 		return ret;
-	}
 
 	platform_set_drvdata(pdev, i2c);
 
diff --git a/drivers/i2c/busses/i2c-mxs.c b/drivers/i2c/busses/i2c-mxs.c
index 033846cdf266..5738556b6aac 100644
--- a/drivers/i2c/busses/i2c-mxs.c
+++ b/drivers/i2c/busses/i2c-mxs.c
@@ -868,7 +868,6 @@ static int mxs_i2c_probe(struct platform_device *pdev)
 	i2c_set_adapdata(adap, i2c);
 	err = i2c_add_numbered_adapter(adap);
 	if (err) {
-		dev_err(dev, "Failed to add adapter (%d)\n", err);
 		writel(MXS_I2C_CTRL0_SFTRST,
 				i2c->regs + MXS_I2C_CTRL0_SET);
 		return err;
diff --git a/drivers/i2c/busses/i2c-nforce2.c b/drivers/i2c/busses/i2c-nforce2.c
index 42fcc9458432..374b35e7e450 100644
--- a/drivers/i2c/busses/i2c-nforce2.c
+++ b/drivers/i2c/busses/i2c-nforce2.c
@@ -366,7 +366,6 @@ static int nforce2_probe_smb(struct pci_dev *dev, int bar, int alt_reg,
 
 	error = i2c_add_adapter(&smbus->adapter);
 	if (error) {
-		dev_err(&smbus->adapter.dev, "Failed to register adapter.\n");
 		release_region(smbus->base, smbus->size);
 		return error;
 	}
diff --git a/drivers/i2c/busses/i2c-nomadik.c b/drivers/i2c/busses/i2c-nomadik.c
index bcd17e8cbcb4..da6609d62848 100644
--- a/drivers/i2c/busses/i2c-nomadik.c
+++ b/drivers/i2c/busses/i2c-nomadik.c
@@ -1046,10 +1046,8 @@ static int nmk_i2c_probe(struct amba_device *adev, const struct amba_id *id)
 		 adap->name, dev->virtbase);
 
 	ret = i2c_add_adapter(adap);
-	if (ret) {
-		dev_err(&adev->dev, "failed to add adapter\n");
+	if (ret)
 		goto err_no_adap;
-	}
 
 	pm_runtime_put(&adev->dev);
 
diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c
index dfa7a4b4a91d..34f1889a4073 100644
--- a/drivers/i2c/busses/i2c-ocores.c
+++ b/drivers/i2c/busses/i2c-ocores.c
@@ -379,6 +379,7 @@ static int ocores_i2c_of_probe(struct platform_device *pdev,
 			if (!clock_frequency_present) {
 				dev_err(&pdev->dev,
 					"Missing required parameter 'opencores,ip-clock-frequency'\n");
+				clk_disable_unprepare(i2c->clk);
 				return -ENODEV;
 			}
 			i2c->ip_clock_khz = clock_frequency / 1000;
@@ -467,20 +468,21 @@ static int ocores_i2c_probe(struct platform_device *pdev)
 		default:
 			dev_err(&pdev->dev, "Unsupported I/O width (%d)\n",
 				i2c->reg_io_width);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto err_clk;
 		}
 	}
 
 	ret = ocores_init(&pdev->dev, i2c);
 	if (ret)
-		return ret;
+		goto err_clk;
 
 	init_waitqueue_head(&i2c->wait);
 	ret = devm_request_irq(&pdev->dev, irq, ocores_isr, 0,
 			       pdev->name, i2c);
 	if (ret) {
 		dev_err(&pdev->dev, "Cannot claim IRQ\n");
-		return ret;
+		goto err_clk;
 	}
 
 	/* hook up driver to tree */
@@ -492,10 +494,8 @@ static int ocores_i2c_probe(struct platform_device *pdev)
 
 	/* add i2c adapter to i2c tree */
 	ret = i2c_add_adapter(&i2c->adap);
-	if (ret) {
-		dev_err(&pdev->dev, "Failed to add adapter\n");
-		return ret;
-	}
+	if (ret)
+		goto err_clk;
 
 	/* add in known devices to the bus */
 	if (pdata) {
@@ -504,6 +504,10 @@ static int ocores_i2c_probe(struct platform_device *pdev)
 	}
 
 	return 0;
+
+err_clk:
+	clk_disable_unprepare(i2c->clk);
+	return ret;
 }
 
 static int ocores_i2c_remove(struct platform_device *pdev)
diff --git a/drivers/i2c/busses/i2c-octeon.c b/drivers/i2c/busses/i2c-octeon-core.c
index 30ae35146723..419b54bfc7c7 100644
--- a/drivers/i2c/busses/i2c-octeon.c
+++ b/drivers/i2c/busses/i2c-octeon-core.c
@@ -4,280 +4,127 @@
  *
  * Portions Copyright (C) 2010 - 2016 Cavium, Inc.
  *
- * This is a driver for the i2c adapter in Cavium Networks' OCTEON processors.
+ * This file contains the shared part of the driver for the i2c adapter in
+ * Cavium Networks' OCTEON processors and ThunderX SOCs.
  *
  * This file is licensed under the terms of the GNU General Public
  * License version 2. This program is licensed "as is" without any
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/atomic.h>
-#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/delay.h>
-#include <linux/sched.h>
-#include <linux/slab.h>
-#include <linux/i2c.h>
-#include <linux/io.h>
-#include <linux/of.h>
-
-#include <asm/octeon/octeon.h>
-
-#define DRV_NAME "i2c-octeon"
-
-/* Register offsets */
-#define SW_TWSI			0x00
-#define TWSI_INT		0x10
-#define SW_TWSI_EXT		0x18
-
-/* Controller command patterns */
-#define SW_TWSI_V		BIT_ULL(63)	/* Valid bit */
-#define SW_TWSI_EIA		BIT_ULL(61)	/* Extended internal address */
-#define SW_TWSI_R		BIT_ULL(56)	/* Result or read bit */
-#define SW_TWSI_SOVR		BIT_ULL(55)	/* Size override */
-#define SW_TWSI_SIZE_SHIFT	52
-#define SW_TWSI_ADDR_SHIFT	40
-#define SW_TWSI_IA_SHIFT	32		/* Internal address */
-
-/* Controller opcode word (bits 60:57) */
-#define SW_TWSI_OP_SHIFT	57
-#define SW_TWSI_OP_7		(0ULL << SW_TWSI_OP_SHIFT)
-#define SW_TWSI_OP_7_IA		(1ULL << SW_TWSI_OP_SHIFT)
-#define SW_TWSI_OP_10		(2ULL << SW_TWSI_OP_SHIFT)
-#define SW_TWSI_OP_10_IA	(3ULL << SW_TWSI_OP_SHIFT)
-#define SW_TWSI_OP_TWSI_CLK	(4ULL << SW_TWSI_OP_SHIFT)
-#define SW_TWSI_OP_EOP		(6ULL << SW_TWSI_OP_SHIFT) /* Extended opcode */
-
-/* Controller extended opcode word (bits 34:32) */
-#define SW_TWSI_EOP_SHIFT	32
-#define SW_TWSI_EOP_TWSI_DATA	(SW_TWSI_OP_EOP | 1ULL << SW_TWSI_EOP_SHIFT)
-#define SW_TWSI_EOP_TWSI_CTL	(SW_TWSI_OP_EOP | 2ULL << SW_TWSI_EOP_SHIFT)
-#define SW_TWSI_EOP_TWSI_CLKCTL	(SW_TWSI_OP_EOP | 3ULL << SW_TWSI_EOP_SHIFT)
-#define SW_TWSI_EOP_TWSI_STAT	(SW_TWSI_OP_EOP | 3ULL << SW_TWSI_EOP_SHIFT)
-#define SW_TWSI_EOP_TWSI_RST	(SW_TWSI_OP_EOP | 7ULL << SW_TWSI_EOP_SHIFT)
-
-/* Controller command and status bits */
-#define TWSI_CTL_CE		0x80	/* High level controller enable */
-#define TWSI_CTL_ENAB		0x40	/* Bus enable */
-#define TWSI_CTL_STA		0x20	/* Master-mode start, HW clears when done */
-#define TWSI_CTL_STP		0x10	/* Master-mode stop, HW clears when done */
-#define TWSI_CTL_IFLG		0x08	/* HW event, SW writes 0 to ACK */
-#define TWSI_CTL_AAK		0x04	/* Assert ACK */
-
-/* Status values */
-#define STAT_ERROR		0x00
-#define STAT_START		0x08
-#define STAT_REP_START		0x10
-#define STAT_TXADDR_ACK		0x18
-#define STAT_TXADDR_NAK		0x20
-#define STAT_TXDATA_ACK		0x28
-#define STAT_TXDATA_NAK		0x30
-#define STAT_LOST_ARB_38	0x38
-#define STAT_RXADDR_ACK		0x40
-#define STAT_RXADDR_NAK		0x48
-#define STAT_RXDATA_ACK		0x50
-#define STAT_RXDATA_NAK		0x58
-#define STAT_SLAVE_60		0x60
-#define STAT_LOST_ARB_68	0x68
-#define STAT_SLAVE_70		0x70
-#define STAT_LOST_ARB_78	0x78
-#define STAT_SLAVE_80		0x80
-#define STAT_SLAVE_88		0x88
-#define STAT_GENDATA_ACK	0x90
-#define STAT_GENDATA_NAK	0x98
-#define STAT_SLAVE_A0		0xA0
-#define STAT_SLAVE_A8		0xA8
-#define STAT_LOST_ARB_B0	0xB0
-#define STAT_SLAVE_LOST		0xB8
-#define STAT_SLAVE_NAK		0xC0
-#define STAT_SLAVE_ACK		0xC8
-#define STAT_AD2W_ACK		0xD0
-#define STAT_AD2W_NAK		0xD8
-#define STAT_IDLE		0xF8
-
-/* TWSI_INT values */
-#define TWSI_INT_ST_INT		BIT_ULL(0)
-#define TWSI_INT_TS_INT		BIT_ULL(1)
-#define TWSI_INT_CORE_INT	BIT_ULL(2)
-#define TWSI_INT_ST_EN		BIT_ULL(4)
-#define TWSI_INT_TS_EN		BIT_ULL(5)
-#define TWSI_INT_CORE_EN	BIT_ULL(6)
-#define TWSI_INT_SDA_OVR	BIT_ULL(8)
-#define TWSI_INT_SCL_OVR	BIT_ULL(9)
-#define TWSI_INT_SDA		BIT_ULL(10)
-#define TWSI_INT_SCL		BIT_ULL(11)
-
-#define I2C_OCTEON_EVENT_WAIT 80 /* microseconds */
-
-struct octeon_i2c {
-	wait_queue_head_t queue;
-	struct i2c_adapter adap;
-	int irq;
-	int hlc_irq;		/* For cn7890 only */
-	u32 twsi_freq;
-	int sys_freq;
-	void __iomem *twsi_base;
-	struct device *dev;
-	bool hlc_enabled;
-	bool broken_irq_mode;
-	bool broken_irq_check;
-	void (*int_enable)(struct octeon_i2c *);
-	void (*int_disable)(struct octeon_i2c *);
-	void (*hlc_int_enable)(struct octeon_i2c *);
-	void (*hlc_int_disable)(struct octeon_i2c *);
-	atomic_t int_enable_cnt;
-	atomic_t hlc_int_enable_cnt;
-};
-
-static void octeon_i2c_writeq_flush(u64 val, void __iomem *addr)
-{
-	__raw_writeq(val, addr);
-	__raw_readq(addr);	/* wait for write to land */
-}
-
-/**
- * octeon_i2c_reg_write - write an I2C core register
- * @i2c: The struct octeon_i2c
- * @eop_reg: Register selector
- * @data: Value to be written
- *
- * The I2C core registers are accessed indirectly via the SW_TWSI CSR.
- */
-static void octeon_i2c_reg_write(struct octeon_i2c *i2c, u64 eop_reg, u8 data)
-{
-	u64 tmp;
 
-	__raw_writeq(SW_TWSI_V | eop_reg | data, i2c->twsi_base + SW_TWSI);
-	do {
-		tmp = __raw_readq(i2c->twsi_base + SW_TWSI);
-	} while ((tmp & SW_TWSI_V) != 0);
-}
-
-#define octeon_i2c_ctl_write(i2c, val)					\
-	octeon_i2c_reg_write(i2c, SW_TWSI_EOP_TWSI_CTL, val)
-#define octeon_i2c_data_write(i2c, val)					\
-	octeon_i2c_reg_write(i2c, SW_TWSI_EOP_TWSI_DATA, val)
+#include "i2c-octeon-core.h"
 
-/**
- * octeon_i2c_reg_read - read lower bits of an I2C core register
- * @i2c: The struct octeon_i2c
- * @eop_reg: Register selector
- *
- * Returns the data.
- *
- * The I2C core registers are accessed indirectly via the SW_TWSI CSR.
- */
-static u8 octeon_i2c_reg_read(struct octeon_i2c *i2c, u64 eop_reg)
+/* interrupt service routine */
+irqreturn_t octeon_i2c_isr(int irq, void *dev_id)
 {
-	u64 tmp;
+	struct octeon_i2c *i2c = dev_id;
 
-	__raw_writeq(SW_TWSI_V | eop_reg | SW_TWSI_R, i2c->twsi_base + SW_TWSI);
-	do {
-		tmp = __raw_readq(i2c->twsi_base + SW_TWSI);
-	} while ((tmp & SW_TWSI_V) != 0);
+	i2c->int_disable(i2c);
+	wake_up(&i2c->queue);
 
-	return tmp & 0xFF;
+	return IRQ_HANDLED;
 }
 
-#define octeon_i2c_ctl_read(i2c)					\
-	octeon_i2c_reg_read(i2c, SW_TWSI_EOP_TWSI_CTL)
-#define octeon_i2c_data_read(i2c)					\
-	octeon_i2c_reg_read(i2c, SW_TWSI_EOP_TWSI_DATA)
-#define octeon_i2c_stat_read(i2c)					\
-	octeon_i2c_reg_read(i2c, SW_TWSI_EOP_TWSI_STAT)
-
-/**
- * octeon_i2c_read_int - read the TWSI_INT register
- * @i2c: The struct octeon_i2c
- *
- * Returns the value of the register.
- */
-static u64 octeon_i2c_read_int(struct octeon_i2c *i2c)
+static bool octeon_i2c_test_iflg(struct octeon_i2c *i2c)
 {
-	return __raw_readq(i2c->twsi_base + TWSI_INT);
+	return (octeon_i2c_ctl_read(i2c) & TWSI_CTL_IFLG);
 }
 
-/**
- * octeon_i2c_write_int - write the TWSI_INT register
- * @i2c: The struct octeon_i2c
- * @data: Value to be written
- */
-static void octeon_i2c_write_int(struct octeon_i2c *i2c, u64 data)
+static bool octeon_i2c_test_ready(struct octeon_i2c *i2c, bool *first)
 {
-	octeon_i2c_writeq_flush(data, i2c->twsi_base + TWSI_INT);
-}
+	if (octeon_i2c_test_iflg(i2c))
+		return true;
 
-/**
- * octeon_i2c_int_enable - enable the CORE interrupt
- * @i2c: The struct octeon_i2c
- *
- * The interrupt will be asserted when there is non-STAT_IDLE state in
- * the SW_TWSI_EOP_TWSI_STAT register.
- */
-static void octeon_i2c_int_enable(struct octeon_i2c *i2c)
-{
-	octeon_i2c_write_int(i2c, TWSI_INT_CORE_EN);
-}
+	if (*first) {
+		*first = false;
+		return false;
+	}
 
-/* disable the CORE interrupt */
-static void octeon_i2c_int_disable(struct octeon_i2c *i2c)
-{
-	/* clear TS/ST/IFLG events */
-	octeon_i2c_write_int(i2c, 0);
+	/*
+	 * IRQ has signaled an event but IFLG hasn't changed.
+	 * Sleep and retry once.
+	 */
+	usleep_range(I2C_OCTEON_EVENT_WAIT, 2 * I2C_OCTEON_EVENT_WAIT);
+	return octeon_i2c_test_iflg(i2c);
 }
 
 /**
- * octeon_i2c_int_enable78 - enable the CORE interrupt
+ * octeon_i2c_wait - wait for the IFLG to be set
  * @i2c: The struct octeon_i2c
  *
- * The interrupt will be asserted when there is non-STAT_IDLE state in the
- * SW_TWSI_EOP_TWSI_STAT register.
+ * Returns 0 on success, otherwise a negative errno.
  */
-static void octeon_i2c_int_enable78(struct octeon_i2c *i2c)
-{
-	atomic_inc_return(&i2c->int_enable_cnt);
-	enable_irq(i2c->irq);
-}
-
-static void __octeon_i2c_irq_disable(atomic_t *cnt, int irq)
+static int octeon_i2c_wait(struct octeon_i2c *i2c)
 {
-	int count;
+	long time_left;
+	bool first = true;
 
 	/*
-	 * The interrupt can be disabled in two places, but we only
-	 * want to make the disable_irq_nosync() call once, so keep
-	 * track with the atomic variable.
+	 * Some chip revisions don't assert the irq in the interrupt
+	 * controller. So we must poll for the IFLG change.
 	 */
-	count = atomic_dec_if_positive(cnt);
-	if (count >= 0)
-		disable_irq_nosync(irq);
+	if (i2c->broken_irq_mode) {
+		u64 end = get_jiffies_64() + i2c->adap.timeout;
+
+		while (!octeon_i2c_test_iflg(i2c) &&
+		       time_before64(get_jiffies_64(), end))
+			usleep_range(I2C_OCTEON_EVENT_WAIT / 2, I2C_OCTEON_EVENT_WAIT);
+
+		return octeon_i2c_test_iflg(i2c) ? 0 : -ETIMEDOUT;
+	}
+
+	i2c->int_enable(i2c);
+	time_left = wait_event_timeout(i2c->queue, octeon_i2c_test_ready(i2c, &first),
+				       i2c->adap.timeout);
+	i2c->int_disable(i2c);
+
+	if (i2c->broken_irq_check && !time_left &&
+	    octeon_i2c_test_iflg(i2c)) {
+		dev_err(i2c->dev, "broken irq connection detected, switching to polling mode.\n");
+		i2c->broken_irq_mode = true;
+		return 0;
+	}
+
+	if (!time_left)
+		return -ETIMEDOUT;
+
+	return 0;
 }
 
-/* disable the CORE interrupt */
-static void octeon_i2c_int_disable78(struct octeon_i2c *i2c)
+static bool octeon_i2c_hlc_test_valid(struct octeon_i2c *i2c)
 {
-	__octeon_i2c_irq_disable(&i2c->int_enable_cnt, i2c->irq);
+	return (__raw_readq(i2c->twsi_base + SW_TWSI(i2c)) & SW_TWSI_V) == 0;
 }
 
-/**
- * octeon_i2c_hlc_int_enable78 - enable the ST interrupt
- * @i2c: The struct octeon_i2c
- *
- * The interrupt will be asserted when there is non-STAT_IDLE state in
- * the SW_TWSI_EOP_TWSI_STAT register.
- */
-static void octeon_i2c_hlc_int_enable78(struct octeon_i2c *i2c)
+static bool octeon_i2c_hlc_test_ready(struct octeon_i2c *i2c, bool *first)
 {
-	atomic_inc_return(&i2c->hlc_int_enable_cnt);
-	enable_irq(i2c->hlc_irq);
+	/* check if valid bit is cleared */
+	if (octeon_i2c_hlc_test_valid(i2c))
+		return true;
+
+	if (*first) {
+		*first = false;
+		return false;
+	}
+
+	/*
+	 * IRQ has signaled an event but valid bit isn't cleared.
+	 * Sleep and retry once.
+	 */
+	usleep_range(I2C_OCTEON_EVENT_WAIT, 2 * I2C_OCTEON_EVENT_WAIT);
+	return octeon_i2c_hlc_test_valid(i2c);
 }
 
-/* disable the ST interrupt */
-static void octeon_i2c_hlc_int_disable78(struct octeon_i2c *i2c)
+static void octeon_i2c_hlc_int_clear(struct octeon_i2c *i2c)
 {
-	__octeon_i2c_irq_disable(&i2c->hlc_int_enable_cnt, i2c->hlc_irq);
+	/* clear ST/TS events, listen for neither */
+	octeon_i2c_write_int(i2c, TWSI_INT_ST_INT | TWSI_INT_TS_INT);
 }
 
 /*
@@ -321,83 +168,41 @@ static void octeon_i2c_hlc_disable(struct octeon_i2c *i2c)
 	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
 }
 
-/* interrupt service routine */
-static irqreturn_t octeon_i2c_isr(int irq, void *dev_id)
-{
-	struct octeon_i2c *i2c = dev_id;
-
-	i2c->int_disable(i2c);
-	wake_up(&i2c->queue);
-
-	return IRQ_HANDLED;
-}
-
-/* HLC interrupt service routine */
-static irqreturn_t octeon_i2c_hlc_isr78(int irq, void *dev_id)
-{
-	struct octeon_i2c *i2c = dev_id;
-
-	i2c->hlc_int_disable(i2c);
-	wake_up(&i2c->queue);
-
-	return IRQ_HANDLED;
-}
-
-static bool octeon_i2c_test_iflg(struct octeon_i2c *i2c)
-{
-	return (octeon_i2c_ctl_read(i2c) & TWSI_CTL_IFLG);
-}
-
-static bool octeon_i2c_test_ready(struct octeon_i2c *i2c, bool *first)
-{
-	if (octeon_i2c_test_iflg(i2c))
-		return true;
-
-	if (*first) {
-		*first = false;
-		return false;
-	}
-
-	/*
-	 * IRQ has signaled an event but IFLG hasn't changed.
-	 * Sleep and retry once.
-	 */
-	usleep_range(I2C_OCTEON_EVENT_WAIT, 2 * I2C_OCTEON_EVENT_WAIT);
-	return octeon_i2c_test_iflg(i2c);
-}
-
 /**
- * octeon_i2c_wait - wait for the IFLG to be set
+ * octeon_i2c_hlc_wait - wait for an HLC operation to complete
  * @i2c: The struct octeon_i2c
  *
- * Returns 0 on success, otherwise a negative errno.
+ * Returns 0 on success, otherwise -ETIMEDOUT.
  */
-static int octeon_i2c_wait(struct octeon_i2c *i2c)
+static int octeon_i2c_hlc_wait(struct octeon_i2c *i2c)
 {
-	long time_left;
-	bool first = 1;
+	bool first = true;
+	int time_left;
 
 	/*
-	 * Some chip revisions don't assert the irq in the interrupt
-	 * controller. So we must poll for the IFLG change.
+	 * Some cn38xx boards don't assert the irq in the interrupt
+	 * controller. So we must poll for the valid bit change.
 	 */
 	if (i2c->broken_irq_mode) {
 		u64 end = get_jiffies_64() + i2c->adap.timeout;
 
-		while (!octeon_i2c_test_iflg(i2c) &&
+		while (!octeon_i2c_hlc_test_valid(i2c) &&
 		       time_before64(get_jiffies_64(), end))
 			usleep_range(I2C_OCTEON_EVENT_WAIT / 2, I2C_OCTEON_EVENT_WAIT);
 
-		return octeon_i2c_test_iflg(i2c) ? 0 : -ETIMEDOUT;
+		return octeon_i2c_hlc_test_valid(i2c) ? 0 : -ETIMEDOUT;
 	}
 
-	i2c->int_enable(i2c);
-	time_left = wait_event_timeout(i2c->queue, octeon_i2c_test_ready(i2c, &first),
+	i2c->hlc_int_enable(i2c);
+	time_left = wait_event_timeout(i2c->queue,
+				       octeon_i2c_hlc_test_ready(i2c, &first),
 				       i2c->adap.timeout);
-	i2c->int_disable(i2c);
+	i2c->hlc_int_disable(i2c);
+	if (!time_left)
+		octeon_i2c_hlc_int_clear(i2c);
 
 	if (i2c->broken_irq_check && !time_left &&
-	    octeon_i2c_test_iflg(i2c)) {
+	    octeon_i2c_hlc_test_valid(i2c)) {
 		dev_err(i2c->dev, "broken irq connection detected, switching to polling mode.\n");
 		i2c->broken_irq_mode = true;
 		return 0;
@@ -405,13 +210,21 @@ static int octeon_i2c_wait(struct octeon_i2c *i2c)
 
 	if (!time_left)
 		return -ETIMEDOUT;
-
 	return 0;
 }
 
 static int octeon_i2c_check_status(struct octeon_i2c *i2c, int final_read)
 {
-	u8 stat = octeon_i2c_stat_read(i2c);
+	u8 stat;
+
+	/*
+	 * This is ugly... in HLC mode the status is not in the status register
+	 * but in the lower 8 bits of SW_TWSI.
+	 */
+	if (i2c->hlc_enabled)
+		stat = __raw_readq(i2c->twsi_base + SW_TWSI(i2c));
+	else
+		stat = octeon_i2c_stat_read(i2c);
 
 	switch (stat) {
 	/* Everything is fine */
@@ -470,83 +283,157 @@ static int octeon_i2c_check_status(struct octeon_i2c *i2c, int final_read)
 	}
 }
 
-static bool octeon_i2c_hlc_test_valid(struct octeon_i2c *i2c)
+static int octeon_i2c_recovery(struct octeon_i2c *i2c)
 {
-	return (__raw_readq(i2c->twsi_base + SW_TWSI) & SW_TWSI_V) == 0;
+	int ret;
+
+	ret = i2c_recover_bus(&i2c->adap);
+	if (ret)
+		/* recover failed, try hardware re-init */
+		ret = octeon_i2c_init_lowlevel(i2c);
+	return ret;
 }
 
-static bool octeon_i2c_hlc_test_ready(struct octeon_i2c *i2c, bool *first)
+/**
+ * octeon_i2c_start - send START to the bus
+ * @i2c: The struct octeon_i2c
+ *
+ * Returns 0 on success, otherwise a negative errno.
+ */
+static int octeon_i2c_start(struct octeon_i2c *i2c)
 {
-	/* check if valid bit is cleared */
-	if (octeon_i2c_hlc_test_valid(i2c))
-		return true;
+	int ret;
+	u8 stat;
 
-	if (*first) {
-		*first = false;
-		return false;
-	}
+	octeon_i2c_hlc_disable(i2c);
 
-	/*
-	 * IRQ has signaled an event but valid bit isn't cleared.
-	 * Sleep and retry once.
-	 */
-	usleep_range(I2C_OCTEON_EVENT_WAIT, 2 * I2C_OCTEON_EVENT_WAIT);
-	return octeon_i2c_hlc_test_valid(i2c);
+	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB | TWSI_CTL_STA);
+	ret = octeon_i2c_wait(i2c);
+	if (ret)
+		goto error;
+
+	stat = octeon_i2c_stat_read(i2c);
+	if (stat == STAT_START || stat == STAT_REP_START)
+		/* START successful, bail out */
+		return 0;
+
+error:
+	/* START failed, try to recover */
+	ret = octeon_i2c_recovery(i2c);
+	return (ret) ? ret : -EAGAIN;
 }
 
-static void octeon_i2c_hlc_int_enable(struct octeon_i2c *i2c)
+/* send STOP to the bus */
+static void octeon_i2c_stop(struct octeon_i2c *i2c)
 {
-	octeon_i2c_write_int(i2c, TWSI_INT_ST_EN);
+	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB | TWSI_CTL_STP);
 }
 
-static void octeon_i2c_hlc_int_clear(struct octeon_i2c *i2c)
+/**
+ * octeon_i2c_read - receive data from the bus via low-level controller
+ * @i2c: The struct octeon_i2c
+ * @target: Target address
+ * @data: Pointer to the location to store the data
+ * @rlength: Length of the data
+ * @recv_len: flag for length byte
+ *
+ * The address is sent over the bus, then the data is read.
+ *
+ * Returns 0 on success, otherwise a negative errno.
+ */
+static int octeon_i2c_read(struct octeon_i2c *i2c, int target,
+			   u8 *data, u16 *rlength, bool recv_len)
 {
-	/* clear ST/TS events, listen for neither */
-	octeon_i2c_write_int(i2c, TWSI_INT_ST_INT | TWSI_INT_TS_INT);
+	int i, result, length = *rlength;
+	bool final_read = false;
+
+	octeon_i2c_data_write(i2c, (target << 1) | 1);
+	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
+
+	result = octeon_i2c_wait(i2c);
+	if (result)
+		return result;
+
+	/* address OK ? */
+	result = octeon_i2c_check_status(i2c, false);
+	if (result)
+		return result;
+
+	for (i = 0; i < length; i++) {
+		/*
+		 * For the last byte to receive TWSI_CTL_AAK must not be set.
+		 *
+		 * A special case is I2C_M_RECV_LEN where we don't know the
+		 * additional length yet. If recv_len is set we assume we're
+		 * not reading the final byte and therefore need to set
+		 * TWSI_CTL_AAK.
+		 */
+		if ((i + 1 == length) && !(recv_len && i == 0))
+			final_read = true;
+
+		/* clear iflg to allow next event */
+		if (final_read)
+			octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
+		else
+			octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB | TWSI_CTL_AAK);
+
+		result = octeon_i2c_wait(i2c);
+		if (result)
+			return result;
+
+		data[i] = octeon_i2c_data_read(i2c, &result);
+		if (result)
+			return result;
+		if (recv_len && i == 0) {
+			if (data[i] > I2C_SMBUS_BLOCK_MAX + 1)
+				return -EPROTO;
+			length += data[i];
+		}
+
+		result = octeon_i2c_check_status(i2c, final_read);
+		if (result)
+			return result;
+	}
+	*rlength = length;
+	return 0;
 }
 
 /**
- * octeon_i2c_hlc_wait - wait for an HLC operation to complete
+ * octeon_i2c_write - send data to the bus via low-level controller
  * @i2c: The struct octeon_i2c
+ * @target: Target address
+ * @data: Pointer to the data to be sent
+ * @length: Length of the data
  *
- * Returns 0 on success, otherwise -ETIMEDOUT.
+ * The address is sent over the bus, then the data.
+ *
+ * Returns 0 on success, otherwise a negative errno.
  */
-static int octeon_i2c_hlc_wait(struct octeon_i2c *i2c)
+static int octeon_i2c_write(struct octeon_i2c *i2c, int target,
+			    const u8 *data, int length)
 {
-	bool first = 1;
-	int time_left;
+	int i, result;
 
-	/*
-	 * Some cn38xx boards don't assert the irq in the interrupt
-	 * controller. So we must poll for the valid bit change.
-	 */
-	if (i2c->broken_irq_mode) {
-		u64 end = get_jiffies_64() + i2c->adap.timeout;
+	octeon_i2c_data_write(i2c, target << 1);
+	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
 
-		while (!octeon_i2c_hlc_test_valid(i2c) &&
-		       time_before64(get_jiffies_64(), end))
-			usleep_range(I2C_OCTEON_EVENT_WAIT / 2, I2C_OCTEON_EVENT_WAIT);
+	result = octeon_i2c_wait(i2c);
+	if (result)
+		return result;
 
-		return octeon_i2c_hlc_test_valid(i2c) ? 0 : -ETIMEDOUT;
-	}
+	for (i = 0; i < length; i++) {
+		result = octeon_i2c_check_status(i2c, false);
+		if (result)
+			return result;
 
-	i2c->hlc_int_enable(i2c);
-	time_left = wait_event_timeout(i2c->queue,
-				       octeon_i2c_hlc_test_ready(i2c, &first),
-				       i2c->adap.timeout);
-	i2c->hlc_int_disable(i2c);
-	if (!time_left)
-		octeon_i2c_hlc_int_clear(i2c);
+		octeon_i2c_data_write(i2c, data[i]);
+		octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
 
-	if (i2c->broken_irq_check && !time_left &&
-	    octeon_i2c_hlc_test_valid(i2c)) {
-		dev_err(i2c->dev, "broken irq connection detected, switching to polling mode.\n");
-		i2c->broken_irq_mode = true;
-		return 0;
+		result = octeon_i2c_wait(i2c);
+		if (result)
+			return result;
 	}
 
-	if (!time_left)
-		return -ETIMEDOUT;
 	return 0;
 }
 
@@ -570,20 +457,20 @@ static int octeon_i2c_hlc_read(struct octeon_i2c *i2c, struct i2c_msg *msgs)
 	else
 		cmd |= SW_TWSI_OP_7;
 
-	octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI);
+	octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c));
 	ret = octeon_i2c_hlc_wait(i2c);
 	if (ret)
 		goto err;
 
-	cmd = __raw_readq(i2c->twsi_base + SW_TWSI);
+	cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c));
 	if ((cmd & SW_TWSI_R) == 0)
-		return -EAGAIN;
+		return octeon_i2c_check_status(i2c, false);
 
 	for (i = 0, j = msgs[0].len - 1; i  < msgs[0].len && i < 4; i++, j--)
 		msgs[0].buf[j] = (cmd >> (8 * i)) & 0xff;
 
 	if (msgs[0].len > 4) {
-		cmd = __raw_readq(i2c->twsi_base + SW_TWSI_EXT);
+		cmd = __raw_readq(i2c->twsi_base + SW_TWSI_EXT(i2c));
 		for (i = 0; i  < msgs[0].len - 4 && i < 4; i++, j--)
 			msgs[0].buf[j] = (cmd >> (8 * i)) & 0xff;
 	}
@@ -620,19 +507,17 @@ static int octeon_i2c_hlc_write(struct octeon_i2c *i2c, struct i2c_msg *msgs)
 
 		for (i = 0; i < msgs[0].len - 4 && i < 4; i++, j--)
 			ext |= (u64)msgs[0].buf[j] << (8 * i);
-		octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT);
+		octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT(i2c));
 	}
 
-	octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI);
+	octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c));
 	ret = octeon_i2c_hlc_wait(i2c);
 	if (ret)
 		goto err;
 
-	cmd = __raw_readq(i2c->twsi_base + SW_TWSI);
+	cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c));
 	if ((cmd & SW_TWSI_R) == 0)
-		return -EAGAIN;
-
-	ret = octeon_i2c_check_status(i2c, false);
+		return octeon_i2c_check_status(i2c, false);
 
 err:
 	return ret;
@@ -663,27 +548,27 @@ static int octeon_i2c_hlc_comp_read(struct octeon_i2c *i2c, struct i2c_msg *msgs
 		cmd |= SW_TWSI_EIA;
 		ext = (u64)msgs[0].buf[0] << SW_TWSI_IA_SHIFT;
 		cmd |= (u64)msgs[0].buf[1] << SW_TWSI_IA_SHIFT;
-		octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT);
+		octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT(i2c));
 	} else {
 		cmd |= (u64)msgs[0].buf[0] << SW_TWSI_IA_SHIFT;
 	}
 
 	octeon_i2c_hlc_int_clear(i2c);
-	octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI);
+	octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c));
 
 	ret = octeon_i2c_hlc_wait(i2c);
 	if (ret)
 		goto err;
 
-	cmd = __raw_readq(i2c->twsi_base + SW_TWSI);
+	cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c));
 	if ((cmd & SW_TWSI_R) == 0)
-		return -EAGAIN;
+		return octeon_i2c_check_status(i2c, false);
 
 	for (i = 0, j = msgs[1].len - 1; i  < msgs[1].len && i < 4; i++, j--)
 		msgs[1].buf[j] = (cmd >> (8 * i)) & 0xff;
 
 	if (msgs[1].len > 4) {
-		cmd = __raw_readq(i2c->twsi_base + SW_TWSI_EXT);
+		cmd = __raw_readq(i2c->twsi_base + SW_TWSI_EXT(i2c));
 		for (i = 0; i  < msgs[1].len - 4 && i < 4; i++, j--)
 			msgs[1].buf[j] = (cmd >> (8 * i)) & 0xff;
 	}
@@ -730,27 +615,85 @@ static int octeon_i2c_hlc_comp_write(struct octeon_i2c *i2c, struct i2c_msg *msg
 		set_ext = true;
 	}
 	if (set_ext)
-		octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT);
+		octeon_i2c_writeq_flush(ext, i2c->twsi_base + SW_TWSI_EXT(i2c));
 
 	octeon_i2c_hlc_int_clear(i2c);
-	octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI);
+	octeon_i2c_writeq_flush(cmd, i2c->twsi_base + SW_TWSI(i2c));
 
 	ret = octeon_i2c_hlc_wait(i2c);
 	if (ret)
 		goto err;
 
-	cmd = __raw_readq(i2c->twsi_base + SW_TWSI);
+	cmd = __raw_readq(i2c->twsi_base + SW_TWSI(i2c));
 	if ((cmd & SW_TWSI_R) == 0)
-		return -EAGAIN;
-
-	ret = octeon_i2c_check_status(i2c, false);
+		return octeon_i2c_check_status(i2c, false);
 
 err:
 	return ret;
 }
 
+/**
+ * octeon_i2c_xfer - The driver's master_xfer function
+ * @adap: Pointer to the i2c_adapter structure
+ * @msgs: Pointer to the messages to be processed
+ * @num: Length of the MSGS array
+ *
+ * Returns the number of messages processed, or a negative errno on failure.
+ */
+int octeon_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
+{
+	struct octeon_i2c *i2c = i2c_get_adapdata(adap);
+	int i, ret = 0;
+
+	if (num == 1) {
+		if (msgs[0].len > 0 && msgs[0].len <= 8) {
+			if (msgs[0].flags & I2C_M_RD)
+				ret = octeon_i2c_hlc_read(i2c, msgs);
+			else
+				ret = octeon_i2c_hlc_write(i2c, msgs);
+			goto out;
+		}
+	} else if (num == 2) {
+		if ((msgs[0].flags & I2C_M_RD) == 0 &&
+		    (msgs[1].flags & I2C_M_RECV_LEN) == 0 &&
+		    msgs[0].len > 0 && msgs[0].len <= 2 &&
+		    msgs[1].len > 0 && msgs[1].len <= 8 &&
+		    msgs[0].addr == msgs[1].addr) {
+			if (msgs[1].flags & I2C_M_RD)
+				ret = octeon_i2c_hlc_comp_read(i2c, msgs);
+			else
+				ret = octeon_i2c_hlc_comp_write(i2c, msgs);
+			goto out;
+		}
+	}
+
+	for (i = 0; ret == 0 && i < num; i++) {
+		struct i2c_msg *pmsg = &msgs[i];
+
+		/* zero-length messages are not supported */
+		if (!pmsg->len) {
+			ret = -EOPNOTSUPP;
+			break;
+		}
+
+		ret = octeon_i2c_start(i2c);
+		if (ret)
+			return ret;
+
+		if (pmsg->flags & I2C_M_RD)
+			ret = octeon_i2c_read(i2c, pmsg->addr, pmsg->buf,
+					      &pmsg->len, pmsg->flags & I2C_M_RECV_LEN);
+		else
+			ret = octeon_i2c_write(i2c, pmsg->addr, pmsg->buf,
+					       pmsg->len);
+	}
+	octeon_i2c_stop(i2c);
+out:
+	return (ret != 0) ? ret : num;
+}
+
 /* calculate and set clock divisors */
-static void octeon_i2c_set_clock(struct octeon_i2c *i2c)
+void octeon_i2c_set_clock(struct octeon_i2c *i2c)
 {
 	int tclk, thp_base, inc, thp_idx, mdiv_idx, ndiv_idx, foscl, diff;
 	int thp = 0x18, mdiv = 2, ndiv = 0, delta_hz = 1000000;
@@ -791,7 +734,7 @@ static void octeon_i2c_set_clock(struct octeon_i2c *i2c)
 	octeon_i2c_reg_write(i2c, SW_TWSI_EOP_TWSI_CLKCTL, (mdiv << 3) | ndiv);
 }
 
-static int octeon_i2c_init_lowlevel(struct octeon_i2c *i2c)
+int octeon_i2c_init_lowlevel(struct octeon_i2c *i2c)
 {
 	u8 status = 0;
 	int tries;
@@ -818,219 +761,6 @@ static int octeon_i2c_init_lowlevel(struct octeon_i2c *i2c)
 	return 0;
 }
 
-static int octeon_i2c_recovery(struct octeon_i2c *i2c)
-{
-	int ret;
-
-	ret = i2c_recover_bus(&i2c->adap);
-	if (ret)
-		/* recover failed, try hardware re-init */
-		ret = octeon_i2c_init_lowlevel(i2c);
-	return ret;
-}
-
-/**
- * octeon_i2c_start - send START to the bus
- * @i2c: The struct octeon_i2c
- *
- * Returns 0 on success, otherwise a negative errno.
- */
-static int octeon_i2c_start(struct octeon_i2c *i2c)
-{
-	int ret;
-	u8 stat;
-
-	octeon_i2c_hlc_disable(i2c);
-
-	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB | TWSI_CTL_STA);
-	ret = octeon_i2c_wait(i2c);
-	if (ret)
-		goto error;
-
-	stat = octeon_i2c_stat_read(i2c);
-	if (stat == STAT_START || stat == STAT_REP_START)
-		/* START successful, bail out */
-		return 0;
-
-error:
-	/* START failed, try to recover */
-	ret = octeon_i2c_recovery(i2c);
-	return (ret) ? ret : -EAGAIN;
-}
-
-/* send STOP to the bus */
-static void octeon_i2c_stop(struct octeon_i2c *i2c)
-{
-	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB | TWSI_CTL_STP);
-}
-
-/**
- * octeon_i2c_write - send data to the bus via low-level controller
- * @i2c: The struct octeon_i2c
- * @target: Target address
- * @data: Pointer to the data to be sent
- * @length: Length of the data
- *
- * The address is sent over the bus, then the data.
- *
- * Returns 0 on success, otherwise a negative errno.
- */
-static int octeon_i2c_write(struct octeon_i2c *i2c, int target,
-			    const u8 *data, int length)
-{
-	int i, result;
-
-	octeon_i2c_data_write(i2c, target << 1);
-	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
-
-	result = octeon_i2c_wait(i2c);
-	if (result)
-		return result;
-
-	for (i = 0; i < length; i++) {
-		result = octeon_i2c_check_status(i2c, false);
-		if (result)
-			return result;
-
-		octeon_i2c_data_write(i2c, data[i]);
-		octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
-
-		result = octeon_i2c_wait(i2c);
-		if (result)
-			return result;
-	}
-
-	return 0;
-}
-
-/**
- * octeon_i2c_read - receive data from the bus via low-level controller
- * @i2c: The struct octeon_i2c
- * @target: Target address
- * @data: Pointer to the location to store the data
- * @rlength: Length of the data
- * @recv_len: flag for length byte
- *
- * The address is sent over the bus, then the data is read.
- *
- * Returns 0 on success, otherwise a negative errno.
- */
-static int octeon_i2c_read(struct octeon_i2c *i2c, int target,
-			   u8 *data, u16 *rlength, bool recv_len)
-{
-	int i, result, length = *rlength;
-	bool final_read = false;
-
-	octeon_i2c_data_write(i2c, (target << 1) | 1);
-	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
-
-	result = octeon_i2c_wait(i2c);
-	if (result)
-		return result;
-
-	/* address OK ? */
-	result = octeon_i2c_check_status(i2c, false);
-	if (result)
-		return result;
-
-	for (i = 0; i < length; i++) {
-		/*
-		 * For the last byte to receive TWSI_CTL_AAK must not be set.
-		 *
-		 * A special case is I2C_M_RECV_LEN where we don't know the
-		 * additional length yet. If recv_len is set we assume we're
-		 * not reading the final byte and therefore need to set
-		 * TWSI_CTL_AAK.
-		 */
-		if ((i + 1 == length) && !(recv_len && i == 0))
-			final_read = true;
-
-		/* clear iflg to allow next event */
-		if (final_read)
-			octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
-		else
-			octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB | TWSI_CTL_AAK);
-
-		result = octeon_i2c_wait(i2c);
-		if (result)
-			return result;
-
-		data[i] = octeon_i2c_data_read(i2c);
-		if (recv_len && i == 0) {
-			if (data[i] > I2C_SMBUS_BLOCK_MAX + 1)
-				return -EPROTO;
-			length += data[i];
-		}
-
-		result = octeon_i2c_check_status(i2c, final_read);
-		if (result)
-			return result;
-	}
-	*rlength = length;
-	return 0;
-}
-
-/**
- * octeon_i2c_xfer - The driver's master_xfer function
- * @adap: Pointer to the i2c_adapter structure
- * @msgs: Pointer to the messages to be processed
- * @num: Length of the MSGS array
- *
- * Returns the number of messages processed, or a negative errno on failure.
- */
-static int octeon_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
-			   int num)
-{
-	struct octeon_i2c *i2c = i2c_get_adapdata(adap);
-	int i, ret = 0;
-
-	if (num == 1) {
-		if (msgs[0].len > 0 && msgs[0].len <= 8) {
-			if (msgs[0].flags & I2C_M_RD)
-				ret = octeon_i2c_hlc_read(i2c, msgs);
-			else
-				ret = octeon_i2c_hlc_write(i2c, msgs);
-			goto out;
-		}
-	} else if (num == 2) {
-		if ((msgs[0].flags & I2C_M_RD) == 0 &&
-		    (msgs[1].flags & I2C_M_RECV_LEN) == 0 &&
-		    msgs[0].len > 0 && msgs[0].len <= 2 &&
-		    msgs[1].len > 0 && msgs[1].len <= 8 &&
-		    msgs[0].addr == msgs[1].addr) {
-			if (msgs[1].flags & I2C_M_RD)
-				ret = octeon_i2c_hlc_comp_read(i2c, msgs);
-			else
-				ret = octeon_i2c_hlc_comp_write(i2c, msgs);
-			goto out;
-		}
-	}
-
-	for (i = 0; ret == 0 && i < num; i++) {
-		struct i2c_msg *pmsg = &msgs[i];
-
-		/* zero-length messages are not supported */
-		if (!pmsg->len) {
-			ret = -EOPNOTSUPP;
-			break;
-		}
-
-		ret = octeon_i2c_start(i2c);
-		if (ret)
-			return ret;
-
-		if (pmsg->flags & I2C_M_RD)
-			ret = octeon_i2c_read(i2c, pmsg->addr, pmsg->buf,
-					      &pmsg->len, pmsg->flags & I2C_M_RECV_LEN);
-		else
-			ret = octeon_i2c_write(i2c, pmsg->addr, pmsg->buf,
-					       pmsg->len);
-	}
-	octeon_i2c_stop(i2c);
-out:
-	return (ret != 0) ? ret : num;
-}
-
 static int octeon_i2c_get_scl(struct i2c_adapter *adap)
 {
 	struct octeon_i2c *i2c = i2c_get_adapdata(adap);
@@ -1044,7 +774,7 @@ static void octeon_i2c_set_scl(struct i2c_adapter *adap, int val)
 {
 	struct octeon_i2c *i2c = i2c_get_adapdata(adap);
 
-	octeon_i2c_write_int(i2c, TWSI_INT_SCL_OVR);
+	octeon_i2c_write_int(i2c, val ? 0 : TWSI_INT_SCL_OVR);
 }
 
 static int octeon_i2c_get_sda(struct i2c_adapter *adap)
@@ -1060,13 +790,14 @@ static void octeon_i2c_prepare_recovery(struct i2c_adapter *adap)
 {
 	struct octeon_i2c *i2c = i2c_get_adapdata(adap);
 
+	octeon_i2c_hlc_disable(i2c);
+
 	/*
-	 * The stop resets the state machine, does not _transmit_ STOP unless
-	 * engine was active.
+	 * Bring control register to a good state regardless
+	 * of HLC state.
 	 */
-	octeon_i2c_stop(i2c);
+	octeon_i2c_ctl_write(i2c, TWSI_CTL_ENAB);
 
-	octeon_i2c_hlc_disable(i2c);
 	octeon_i2c_write_int(i2c, 0);
 }
 
@@ -1074,10 +805,19 @@ static void octeon_i2c_unprepare_recovery(struct i2c_adapter *adap)
 {
 	struct octeon_i2c *i2c = i2c_get_adapdata(adap);
 
+	/*
+	 * Generate STOP to finish the unfinished transaction.
+	 * Can't generate STOP via the TWSI CTL register
+	 * since it could bring the TWSI controller into an inoperable state.
+	 */
+	octeon_i2c_write_int(i2c, TWSI_INT_SDA_OVR | TWSI_INT_SCL_OVR);
+	udelay(5);
+	octeon_i2c_write_int(i2c, TWSI_INT_SDA_OVR);
+	udelay(5);
 	octeon_i2c_write_int(i2c, 0);
 }
 
-static struct i2c_bus_recovery_info octeon_i2c_recovery_info = {
+struct i2c_bus_recovery_info octeon_i2c_recovery_info = {
 	.recover_bus = i2c_generic_scl_recovery,
 	.get_scl = octeon_i2c_get_scl,
 	.set_scl = octeon_i2c_set_scl,
@@ -1085,171 +825,3 @@ static struct i2c_bus_recovery_info octeon_i2c_recovery_info = {
 	.prepare_recovery = octeon_i2c_prepare_recovery,
 	.unprepare_recovery = octeon_i2c_unprepare_recovery,
 };
-
-static u32 octeon_i2c_functionality(struct i2c_adapter *adap)
-{
-	return I2C_FUNC_I2C | (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK) |
-	       I2C_FUNC_SMBUS_READ_BLOCK_DATA | I2C_SMBUS_BLOCK_PROC_CALL;
-}
-
-static const struct i2c_algorithm octeon_i2c_algo = {
-	.master_xfer = octeon_i2c_xfer,
-	.functionality = octeon_i2c_functionality,
-};
-
-static struct i2c_adapter octeon_i2c_ops = {
-	.owner = THIS_MODULE,
-	.name = "OCTEON adapter",
-	.algo = &octeon_i2c_algo,
-};
-
-static int octeon_i2c_probe(struct platform_device *pdev)
-{
-	struct device_node *node = pdev->dev.of_node;
-	int irq, result = 0, hlc_irq = 0;
-	struct resource *res_mem;
-	struct octeon_i2c *i2c;
-	bool cn78xx_style;
-
-	cn78xx_style = of_device_is_compatible(node, "cavium,octeon-7890-twsi");
-	if (cn78xx_style) {
-		hlc_irq = platform_get_irq(pdev, 0);
-		if (hlc_irq < 0)
-			return hlc_irq;
-
-		irq = platform_get_irq(pdev, 2);
-		if (irq < 0)
-			return irq;
-	} else {
-		/* All adaptors have an irq.  */
-		irq = platform_get_irq(pdev, 0);
-		if (irq < 0)
-			return irq;
-	}
-
-	i2c = devm_kzalloc(&pdev->dev, sizeof(*i2c), GFP_KERNEL);
-	if (!i2c) {
-		result = -ENOMEM;
-		goto out;
-	}
-	i2c->dev = &pdev->dev;
-
-	res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->twsi_base = devm_ioremap_resource(&pdev->dev, res_mem);
-	if (IS_ERR(i2c->twsi_base)) {
-		result = PTR_ERR(i2c->twsi_base);
-		goto out;
-	}
-
-	/*
-	 * "clock-rate" is a legacy binding, the official binding is
-	 * "clock-frequency".  Try the official one first and then
-	 * fall back if it doesn't exist.
-	 */
-	if (of_property_read_u32(node, "clock-frequency", &i2c->twsi_freq) &&
-	    of_property_read_u32(node, "clock-rate", &i2c->twsi_freq)) {
-		dev_err(i2c->dev,
-			"no I2C 'clock-rate' or 'clock-frequency' property\n");
-		result = -ENXIO;
-		goto out;
-	}
-
-	i2c->sys_freq = octeon_get_io_clock_rate();
-
-	init_waitqueue_head(&i2c->queue);
-
-	i2c->irq = irq;
-
-	if (cn78xx_style) {
-		i2c->hlc_irq = hlc_irq;
-
-		i2c->int_enable = octeon_i2c_int_enable78;
-		i2c->int_disable = octeon_i2c_int_disable78;
-		i2c->hlc_int_enable = octeon_i2c_hlc_int_enable78;
-		i2c->hlc_int_disable = octeon_i2c_hlc_int_disable78;
-
-		irq_set_status_flags(i2c->irq, IRQ_NOAUTOEN);
-		irq_set_status_flags(i2c->hlc_irq, IRQ_NOAUTOEN);
-
-		result = devm_request_irq(&pdev->dev, i2c->hlc_irq,
-					  octeon_i2c_hlc_isr78, 0,
-					  DRV_NAME, i2c);
-		if (result < 0) {
-			dev_err(i2c->dev, "failed to attach interrupt\n");
-			goto out;
-		}
-	} else {
-		i2c->int_enable = octeon_i2c_int_enable;
-		i2c->int_disable = octeon_i2c_int_disable;
-		i2c->hlc_int_enable = octeon_i2c_hlc_int_enable;
-		i2c->hlc_int_disable = octeon_i2c_int_disable;
-	}
-
-	result = devm_request_irq(&pdev->dev, i2c->irq,
-				  octeon_i2c_isr, 0, DRV_NAME, i2c);
-	if (result < 0) {
-		dev_err(i2c->dev, "failed to attach interrupt\n");
-		goto out;
-	}
-
-	if (OCTEON_IS_MODEL(OCTEON_CN38XX))
-		i2c->broken_irq_check = true;
-
-	result = octeon_i2c_init_lowlevel(i2c);
-	if (result) {
-		dev_err(i2c->dev, "init low level failed\n");
-		goto  out;
-	}
-
-	octeon_i2c_set_clock(i2c);
-
-	i2c->adap = octeon_i2c_ops;
-	i2c->adap.timeout = msecs_to_jiffies(2);
-	i2c->adap.retries = 5;
-	i2c->adap.bus_recovery_info = &octeon_i2c_recovery_info;
-	i2c->adap.dev.parent = &pdev->dev;
-	i2c->adap.dev.of_node = node;
-	i2c_set_adapdata(&i2c->adap, i2c);
-	platform_set_drvdata(pdev, i2c);
-
-	result = i2c_add_adapter(&i2c->adap);
-	if (result < 0) {
-		dev_err(i2c->dev, "failed to add adapter\n");
-		goto out;
-	}
-	dev_info(i2c->dev, "probed\n");
-	return 0;
-
-out:
-	return result;
-};
-
-static int octeon_i2c_remove(struct platform_device *pdev)
-{
-	struct octeon_i2c *i2c = platform_get_drvdata(pdev);
-
-	i2c_del_adapter(&i2c->adap);
-	return 0;
-};
-
-static const struct of_device_id octeon_i2c_match[] = {
-	{ .compatible = "cavium,octeon-3860-twsi", },
-	{ .compatible = "cavium,octeon-7890-twsi", },
-	{},
-};
-MODULE_DEVICE_TABLE(of, octeon_i2c_match);
-
-static struct platform_driver octeon_i2c_driver = {
-	.probe		= octeon_i2c_probe,
-	.remove		= octeon_i2c_remove,
-	.driver		= {
-		.name	= DRV_NAME,
-		.of_match_table = octeon_i2c_match,
-	},
-};
-
-module_platform_driver(octeon_i2c_driver);
-
-MODULE_AUTHOR("Michael Lawnick <michael.lawnick.ext@nsn.com>");
-MODULE_DESCRIPTION("I2C-Bus adapter for Cavium OCTEON processors");
-MODULE_LICENSE("GPL");
diff --git a/drivers/i2c/busses/i2c-octeon-core.h b/drivers/i2c/busses/i2c-octeon-core.h
new file mode 100644
index 000000000000..1db7c835a454
--- /dev/null
+++ b/drivers/i2c/busses/i2c-octeon-core.h
@@ -0,0 +1,216 @@
+#include <linux/atomic.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/i2c.h>
+#include <linux/i2c-smbus.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+
+/* Controller command patterns */
+#define SW_TWSI_V		BIT_ULL(63)	/* Valid bit */
+#define SW_TWSI_EIA		BIT_ULL(61)	/* Extended internal address */
+#define SW_TWSI_R		BIT_ULL(56)	/* Result or read bit */
+#define SW_TWSI_SOVR		BIT_ULL(55)	/* Size override */
+#define SW_TWSI_SIZE_SHIFT	52
+#define SW_TWSI_ADDR_SHIFT	40
+#define SW_TWSI_IA_SHIFT	32		/* Internal address */
+
+/* Controller opcode word (bits 60:57) */
+#define SW_TWSI_OP_SHIFT	57
+#define SW_TWSI_OP_7		(0ULL << SW_TWSI_OP_SHIFT)
+#define SW_TWSI_OP_7_IA		(1ULL << SW_TWSI_OP_SHIFT)
+#define SW_TWSI_OP_10		(2ULL << SW_TWSI_OP_SHIFT)
+#define SW_TWSI_OP_10_IA	(3ULL << SW_TWSI_OP_SHIFT)
+#define SW_TWSI_OP_TWSI_CLK	(4ULL << SW_TWSI_OP_SHIFT)
+#define SW_TWSI_OP_EOP		(6ULL << SW_TWSI_OP_SHIFT) /* Extended opcode */
+
+/* Controller extended opcode word (bits 34:32) */
+#define SW_TWSI_EOP_SHIFT	32
+#define SW_TWSI_EOP_TWSI_DATA	(SW_TWSI_OP_EOP | 1ULL << SW_TWSI_EOP_SHIFT)
+#define SW_TWSI_EOP_TWSI_CTL	(SW_TWSI_OP_EOP | 2ULL << SW_TWSI_EOP_SHIFT)
+#define SW_TWSI_EOP_TWSI_CLKCTL	(SW_TWSI_OP_EOP | 3ULL << SW_TWSI_EOP_SHIFT)
+#define SW_TWSI_EOP_TWSI_STAT	(SW_TWSI_OP_EOP | 3ULL << SW_TWSI_EOP_SHIFT)
+#define SW_TWSI_EOP_TWSI_RST	(SW_TWSI_OP_EOP | 7ULL << SW_TWSI_EOP_SHIFT)
+
+/* Controller command and status bits */
+#define TWSI_CTL_CE		0x80	/* High level controller enable */
+#define TWSI_CTL_ENAB		0x40	/* Bus enable */
+#define TWSI_CTL_STA		0x20	/* Master-mode start, HW clears when done */
+#define TWSI_CTL_STP		0x10	/* Master-mode stop, HW clears when done */
+#define TWSI_CTL_IFLG		0x08	/* HW event, SW writes 0 to ACK */
+#define TWSI_CTL_AAK		0x04	/* Assert ACK */
+
+/* Status values */
+#define STAT_ERROR		0x00
+#define STAT_START		0x08
+#define STAT_REP_START		0x10
+#define STAT_TXADDR_ACK		0x18
+#define STAT_TXADDR_NAK		0x20
+#define STAT_TXDATA_ACK		0x28
+#define STAT_TXDATA_NAK		0x30
+#define STAT_LOST_ARB_38	0x38
+#define STAT_RXADDR_ACK		0x40
+#define STAT_RXADDR_NAK		0x48
+#define STAT_RXDATA_ACK		0x50
+#define STAT_RXDATA_NAK		0x58
+#define STAT_SLAVE_60		0x60
+#define STAT_LOST_ARB_68	0x68
+#define STAT_SLAVE_70		0x70
+#define STAT_LOST_ARB_78	0x78
+#define STAT_SLAVE_80		0x80
+#define STAT_SLAVE_88		0x88
+#define STAT_GENDATA_ACK	0x90
+#define STAT_GENDATA_NAK	0x98
+#define STAT_SLAVE_A0		0xA0
+#define STAT_SLAVE_A8		0xA8
+#define STAT_LOST_ARB_B0	0xB0
+#define STAT_SLAVE_LOST		0xB8
+#define STAT_SLAVE_NAK		0xC0
+#define STAT_SLAVE_ACK		0xC8
+#define STAT_AD2W_ACK		0xD0
+#define STAT_AD2W_NAK		0xD8
+#define STAT_IDLE		0xF8
+
+/* TWSI_INT values */
+#define TWSI_INT_ST_INT		BIT_ULL(0)
+#define TWSI_INT_TS_INT		BIT_ULL(1)
+#define TWSI_INT_CORE_INT	BIT_ULL(2)
+#define TWSI_INT_ST_EN		BIT_ULL(4)
+#define TWSI_INT_TS_EN		BIT_ULL(5)
+#define TWSI_INT_CORE_EN	BIT_ULL(6)
+#define TWSI_INT_SDA_OVR	BIT_ULL(8)
+#define TWSI_INT_SCL_OVR	BIT_ULL(9)
+#define TWSI_INT_SDA		BIT_ULL(10)
+#define TWSI_INT_SCL		BIT_ULL(11)
+
+#define I2C_OCTEON_EVENT_WAIT 80 /* microseconds */
+
+/* Register offsets */
+struct octeon_i2c_reg_offset {
+	unsigned int sw_twsi;
+	unsigned int twsi_int;
+	unsigned int sw_twsi_ext;
+};
+
+#define SW_TWSI(x)	(x->roff.sw_twsi)
+#define TWSI_INT(x)	(x->roff.twsi_int)
+#define SW_TWSI_EXT(x)	(x->roff.sw_twsi_ext)
+
+struct octeon_i2c {
+	wait_queue_head_t queue;
+	struct i2c_adapter adap;
+	struct octeon_i2c_reg_offset roff;
+	struct clk *clk;
+	int irq;
+	int hlc_irq;		/* For cn7890 only */
+	u32 twsi_freq;
+	int sys_freq;
+	void __iomem *twsi_base;
+	struct device *dev;
+	bool hlc_enabled;
+	bool broken_irq_mode;
+	bool broken_irq_check;
+	void (*int_enable)(struct octeon_i2c *);
+	void (*int_disable)(struct octeon_i2c *);
+	void (*hlc_int_enable)(struct octeon_i2c *);
+	void (*hlc_int_disable)(struct octeon_i2c *);
+	atomic_t int_enable_cnt;
+	atomic_t hlc_int_enable_cnt;
+#if IS_ENABLED(CONFIG_I2C_THUNDERX)
+	struct msix_entry i2c_msix;
+#endif
+	struct i2c_smbus_alert_setup alert_data;
+	struct i2c_client *ara;
+};
+
+static inline void octeon_i2c_writeq_flush(u64 val, void __iomem *addr)
+{
+	__raw_writeq(val, addr);
+	__raw_readq(addr);	/* wait for write to land */
+}
+
+/**
+ * octeon_i2c_reg_write - write an I2C core register
+ * @i2c: The struct octeon_i2c
+ * @eop_reg: Register selector
+ * @data: Value to be written
+ *
+ * The I2C core registers are accessed indirectly via the SW_TWSI CSR.
+ */
+static inline void octeon_i2c_reg_write(struct octeon_i2c *i2c, u64 eop_reg, u8 data)
+{
+	u64 tmp;
+
+	__raw_writeq(SW_TWSI_V | eop_reg | data, i2c->twsi_base + SW_TWSI(i2c));
+
+	readq_poll_timeout(i2c->twsi_base + SW_TWSI(i2c), tmp, tmp & SW_TWSI_V,
+			   I2C_OCTEON_EVENT_WAIT, i2c->adap.timeout);
+}
+
+#define octeon_i2c_ctl_write(i2c, val)					\
+	octeon_i2c_reg_write(i2c, SW_TWSI_EOP_TWSI_CTL, val)
+#define octeon_i2c_data_write(i2c, val)					\
+	octeon_i2c_reg_write(i2c, SW_TWSI_EOP_TWSI_DATA, val)
+
+/**
+ * octeon_i2c_reg_read - read lower bits of an I2C core register
+ * @i2c: The struct octeon_i2c
+ * @eop_reg: Register selector
+ *
+ * Returns the data.
+ *
+ * The I2C core registers are accessed indirectly via the SW_TWSI CSR.
+ */
+static inline int octeon_i2c_reg_read(struct octeon_i2c *i2c, u64 eop_reg,
+				      int *error)
+{
+	u64 tmp;
+	int ret;
+
+	__raw_writeq(SW_TWSI_V | eop_reg | SW_TWSI_R, i2c->twsi_base + SW_TWSI(i2c));
+
+	ret = readq_poll_timeout(i2c->twsi_base + SW_TWSI(i2c), tmp,
+				 tmp & SW_TWSI_V, I2C_OCTEON_EVENT_WAIT,
+				 i2c->adap.timeout);
+	if (error)
+		*error = ret;
+	return tmp & 0xFF;
+}
+
+#define octeon_i2c_ctl_read(i2c)					\
+	octeon_i2c_reg_read(i2c, SW_TWSI_EOP_TWSI_CTL, NULL)
+#define octeon_i2c_data_read(i2c, error)				\
+	octeon_i2c_reg_read(i2c, SW_TWSI_EOP_TWSI_DATA, error)
+#define octeon_i2c_stat_read(i2c)					\
+	octeon_i2c_reg_read(i2c, SW_TWSI_EOP_TWSI_STAT, NULL)
+
+/**
+ * octeon_i2c_read_int - read the TWSI_INT register
+ * @i2c: The struct octeon_i2c
+ *
+ * Returns the value of the register.
+ */
+static inline u64 octeon_i2c_read_int(struct octeon_i2c *i2c)
+{
+	return __raw_readq(i2c->twsi_base + TWSI_INT(i2c));
+}
+
+/**
+ * octeon_i2c_write_int - write the TWSI_INT register
+ * @i2c: The struct octeon_i2c
+ * @data: Value to be written
+ */
+static inline void octeon_i2c_write_int(struct octeon_i2c *i2c, u64 data)
+{
+	octeon_i2c_writeq_flush(data, i2c->twsi_base + TWSI_INT(i2c));
+}
+
+/* Prototypes */
+irqreturn_t octeon_i2c_isr(int irq, void *dev_id);
+int octeon_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num);
+int octeon_i2c_init_lowlevel(struct octeon_i2c *i2c);
+void octeon_i2c_set_clock(struct octeon_i2c *i2c);
+extern struct i2c_bus_recovery_info octeon_i2c_recovery_info;
diff --git a/drivers/i2c/busses/i2c-octeon-platdrv.c b/drivers/i2c/busses/i2c-octeon-platdrv.c
new file mode 100644
index 000000000000..917524ce6890
--- /dev/null
+++ b/drivers/i2c/busses/i2c-octeon-platdrv.c
@@ -0,0 +1,286 @@
+/*
+ * (C) Copyright 2009-2010
+ * Nokia Siemens Networks, michael.lawnick.ext@nsn.com
+ *
+ * Portions Copyright (C) 2010 - 2016 Cavium, Inc.
+ *
+ * This is a driver for the i2c adapter in Cavium Networks' OCTEON processors.
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/atomic.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#include <asm/octeon/octeon.h>
+#include "i2c-octeon-core.h"
+
+#define DRV_NAME "i2c-octeon"
+
+/**
+ * octeon_i2c_int_enable - enable the CORE interrupt
+ * @i2c: The struct octeon_i2c
+ *
+ * The interrupt will be asserted when there is non-STAT_IDLE state in
+ * the SW_TWSI_EOP_TWSI_STAT register.
+ */
+static void octeon_i2c_int_enable(struct octeon_i2c *i2c)
+{
+	octeon_i2c_write_int(i2c, TWSI_INT_CORE_EN);
+}
+
+/* disable the CORE interrupt */
+static void octeon_i2c_int_disable(struct octeon_i2c *i2c)
+{
+	/* clear TS/ST/IFLG events */
+	octeon_i2c_write_int(i2c, 0);
+}
+
+/**
+ * octeon_i2c_int_enable78 - enable the CORE interrupt
+ * @i2c: The struct octeon_i2c
+ *
+ * The interrupt will be asserted when there is non-STAT_IDLE state in the
+ * SW_TWSI_EOP_TWSI_STAT register.
+ */
+static void octeon_i2c_int_enable78(struct octeon_i2c *i2c)
+{
+	atomic_inc_return(&i2c->int_enable_cnt);
+	enable_irq(i2c->irq);
+}
+
+static void __octeon_i2c_irq_disable(atomic_t *cnt, int irq)
+{
+	int count;
+
+	/*
+	 * The interrupt can be disabled in two places, but we only
+	 * want to make the disable_irq_nosync() call once, so keep
+	 * track with the atomic variable.
+	 */
+	count = atomic_dec_if_positive(cnt);
+	if (count >= 0)
+		disable_irq_nosync(irq);
+}
+
+/* disable the CORE interrupt */
+static void octeon_i2c_int_disable78(struct octeon_i2c *i2c)
+{
+	__octeon_i2c_irq_disable(&i2c->int_enable_cnt, i2c->irq);
+}
+
+/**
+ * octeon_i2c_hlc_int_enable78 - enable the ST interrupt
+ * @i2c: The struct octeon_i2c
+ *
+ * The interrupt will be asserted when there is non-STAT_IDLE state in
+ * the SW_TWSI_EOP_TWSI_STAT register.
+ */
+static void octeon_i2c_hlc_int_enable78(struct octeon_i2c *i2c)
+{
+	atomic_inc_return(&i2c->hlc_int_enable_cnt);
+	enable_irq(i2c->hlc_irq);
+}
+
+/* disable the ST interrupt */
+static void octeon_i2c_hlc_int_disable78(struct octeon_i2c *i2c)
+{
+	__octeon_i2c_irq_disable(&i2c->hlc_int_enable_cnt, i2c->hlc_irq);
+}
+
+/* HLC interrupt service routine */
+static irqreturn_t octeon_i2c_hlc_isr78(int irq, void *dev_id)
+{
+	struct octeon_i2c *i2c = dev_id;
+
+	i2c->hlc_int_disable(i2c);
+	wake_up(&i2c->queue);
+
+	return IRQ_HANDLED;
+}
+
+static void octeon_i2c_hlc_int_enable(struct octeon_i2c *i2c)
+{
+	octeon_i2c_write_int(i2c, TWSI_INT_ST_EN);
+}
+
+static u32 octeon_i2c_functionality(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK) |
+	       I2C_FUNC_SMBUS_READ_BLOCK_DATA | I2C_SMBUS_BLOCK_PROC_CALL;
+}
+
+static const struct i2c_algorithm octeon_i2c_algo = {
+	.master_xfer = octeon_i2c_xfer,
+	.functionality = octeon_i2c_functionality,
+};
+
+static struct i2c_adapter octeon_i2c_ops = {
+	.owner = THIS_MODULE,
+	.name = "OCTEON adapter",
+	.algo = &octeon_i2c_algo,
+};
+
+static int octeon_i2c_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	int irq, result = 0, hlc_irq = 0;
+	struct resource *res_mem;
+	struct octeon_i2c *i2c;
+	bool cn78xx_style;
+
+	cn78xx_style = of_device_is_compatible(node, "cavium,octeon-7890-twsi");
+	if (cn78xx_style) {
+		hlc_irq = platform_get_irq(pdev, 0);
+		if (hlc_irq < 0)
+			return hlc_irq;
+
+		irq = platform_get_irq(pdev, 2);
+		if (irq < 0)
+			return irq;
+	} else {
+		/* All adaptors have an irq.  */
+		irq = platform_get_irq(pdev, 0);
+		if (irq < 0)
+			return irq;
+	}
+
+	i2c = devm_kzalloc(&pdev->dev, sizeof(*i2c), GFP_KERNEL);
+	if (!i2c) {
+		result = -ENOMEM;
+		goto out;
+	}
+	i2c->dev = &pdev->dev;
+
+	i2c->roff.sw_twsi = 0x00;
+	i2c->roff.twsi_int = 0x10;
+	i2c->roff.sw_twsi_ext = 0x18;
+
+	res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	i2c->twsi_base = devm_ioremap_resource(&pdev->dev, res_mem);
+	if (IS_ERR(i2c->twsi_base)) {
+		result = PTR_ERR(i2c->twsi_base);
+		goto out;
+	}
+
+	/*
+	 * "clock-rate" is a legacy binding, the official binding is
+	 * "clock-frequency".  Try the official one first and then
+	 * fall back if it doesn't exist.
+	 */
+	if (of_property_read_u32(node, "clock-frequency", &i2c->twsi_freq) &&
+	    of_property_read_u32(node, "clock-rate", &i2c->twsi_freq)) {
+		dev_err(i2c->dev,
+			"no I2C 'clock-rate' or 'clock-frequency' property\n");
+		result = -ENXIO;
+		goto out;
+	}
+
+	i2c->sys_freq = octeon_get_io_clock_rate();
+
+	init_waitqueue_head(&i2c->queue);
+
+	i2c->irq = irq;
+
+	if (cn78xx_style) {
+		i2c->hlc_irq = hlc_irq;
+
+		i2c->int_enable = octeon_i2c_int_enable78;
+		i2c->int_disable = octeon_i2c_int_disable78;
+		i2c->hlc_int_enable = octeon_i2c_hlc_int_enable78;
+		i2c->hlc_int_disable = octeon_i2c_hlc_int_disable78;
+
+		irq_set_status_flags(i2c->irq, IRQ_NOAUTOEN);
+		irq_set_status_flags(i2c->hlc_irq, IRQ_NOAUTOEN);
+
+		result = devm_request_irq(&pdev->dev, i2c->hlc_irq,
+					  octeon_i2c_hlc_isr78, 0,
+					  DRV_NAME, i2c);
+		if (result < 0) {
+			dev_err(i2c->dev, "failed to attach interrupt\n");
+			goto out;
+		}
+	} else {
+		i2c->int_enable = octeon_i2c_int_enable;
+		i2c->int_disable = octeon_i2c_int_disable;
+		i2c->hlc_int_enable = octeon_i2c_hlc_int_enable;
+		i2c->hlc_int_disable = octeon_i2c_int_disable;
+	}
+
+	result = devm_request_irq(&pdev->dev, i2c->irq,
+				  octeon_i2c_isr, 0, DRV_NAME, i2c);
+	if (result < 0) {
+		dev_err(i2c->dev, "failed to attach interrupt\n");
+		goto out;
+	}
+
+	if (OCTEON_IS_MODEL(OCTEON_CN38XX))
+		i2c->broken_irq_check = true;
+
+	result = octeon_i2c_init_lowlevel(i2c);
+	if (result) {
+		dev_err(i2c->dev, "init low level failed\n");
+		goto  out;
+	}
+
+	octeon_i2c_set_clock(i2c);
+
+	i2c->adap = octeon_i2c_ops;
+	i2c->adap.timeout = msecs_to_jiffies(2);
+	i2c->adap.retries = 5;
+	i2c->adap.bus_recovery_info = &octeon_i2c_recovery_info;
+	i2c->adap.dev.parent = &pdev->dev;
+	i2c->adap.dev.of_node = node;
+	i2c_set_adapdata(&i2c->adap, i2c);
+	platform_set_drvdata(pdev, i2c);
+
+	result = i2c_add_adapter(&i2c->adap);
+	if (result < 0)
+		goto out;
+	dev_info(i2c->dev, "probed\n");
+	return 0;
+
+out:
+	return result;
+};
+
+static int octeon_i2c_remove(struct platform_device *pdev)
+{
+	struct octeon_i2c *i2c = platform_get_drvdata(pdev);
+
+	i2c_del_adapter(&i2c->adap);
+	return 0;
+};
+
+static const struct of_device_id octeon_i2c_match[] = {
+	{ .compatible = "cavium,octeon-3860-twsi", },
+	{ .compatible = "cavium,octeon-7890-twsi", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, octeon_i2c_match);
+
+static struct platform_driver octeon_i2c_driver = {
+	.probe		= octeon_i2c_probe,
+	.remove		= octeon_i2c_remove,
+	.driver		= {
+		.name	= DRV_NAME,
+		.of_match_table = octeon_i2c_match,
+	},
+};
+
+module_platform_driver(octeon_i2c_driver);
+
+MODULE_AUTHOR("Michael Lawnick <michael.lawnick.ext@nsn.com>");
+MODULE_DESCRIPTION("I2C-Bus adapter for Cavium OCTEON processors");
+MODULE_LICENSE("GPL");
diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c
index ab1279b8e240..c7da0c42baee 100644
--- a/drivers/i2c/busses/i2c-omap.c
+++ b/drivers/i2c/busses/i2c-omap.c
@@ -1425,10 +1425,8 @@ omap_i2c_probe(struct platform_device *pdev)
 	/* i2c device drivers may be active on return from add_adapter() */
 	adap->nr = pdev->id;
 	r = i2c_add_numbered_adapter(adap);
-	if (r) {
-		dev_err(omap->dev, "failure adding adapter\n");
+	if (r)
 		goto err_unuse_clocks;
-	}
 
 	dev_info(omap->dev, "bus %d rev%d.%d at %d kHz\n", adap->nr,
 		 major, minor, omap->speed);
diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c
index 23d1c167b5d7..c2268cdf38e8 100644
--- a/drivers/i2c/busses/i2c-piix4.c
+++ b/drivers/i2c/busses/i2c-piix4.c
@@ -694,7 +694,6 @@ static int piix4_add_adapter(struct pci_dev *dev, unsigned short smba,
 
 	retval = i2c_add_adapter(adap);
 	if (retval) {
-		dev_err(&dev->dev, "Couldn't register adapter!\n");
 		kfree(adapdata);
 		kfree(adap);
 		release_region(smba, SMBIOSIZE);
diff --git a/drivers/i2c/busses/i2c-pmcmsp.c b/drivers/i2c/busses/i2c-pmcmsp.c
index 2c40edbf6224..217c78711d65 100644
--- a/drivers/i2c/busses/i2c-pmcmsp.c
+++ b/drivers/i2c/busses/i2c-pmcmsp.c
@@ -329,10 +329,8 @@ static int pmcmsptwi_probe(struct platform_device *pldev)
 	i2c_set_adapdata(&pmcmsptwi_adapter, &pmcmsptwi_data);
 
 	rc = i2c_add_adapter(&pmcmsptwi_adapter);
-	if (rc) {
-		dev_err(&pldev->dev, "Unable to register I2C adapter\n");
+	if (rc)
 		goto ret_unmap;
-	}
 
 	return 0;
 
diff --git a/drivers/i2c/busses/i2c-pnx.c b/drivers/i2c/busses/i2c-pnx.c
index 7ea67aa46fb7..fd5f9d2bf6d9 100644
--- a/drivers/i2c/busses/i2c-pnx.c
+++ b/drivers/i2c/busses/i2c-pnx.c
@@ -714,10 +714,8 @@ static int i2c_pnx_probe(struct platform_device *pdev)
 
 	/* Register this adapter with the I2C subsystem */
 	ret = i2c_add_numbered_adapter(&alg_data->adapter);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "I2C: Failed to add bus\n");
+	if (ret < 0)
 		goto out_clock;
-	}
 
 	dev_dbg(&pdev->dev, "%s: Master at %#8x, irq %d.\n",
 		alg_data->adapter.name, res->start, alg_data->irq);
diff --git a/drivers/i2c/busses/i2c-puv3.c b/drivers/i2c/busses/i2c-puv3.c
index 82b6f02544da..0c8b1571886d 100644
--- a/drivers/i2c/busses/i2c-puv3.c
+++ b/drivers/i2c/busses/i2c-puv3.c
@@ -212,11 +212,8 @@ static int puv3_i2c_probe(struct platform_device *pdev)
 
 	adapter->nr = pdev->id;
 	rc = i2c_add_numbered_adapter(adapter);
-	if (rc) {
-		dev_err(&pdev->dev, "Adapter '%s' registration failed\n",
-				adapter->name);
+	if (rc)
 		goto fail_add_adapter;
-	}
 
 	dev_info(&pdev->dev, "PKUnity v3 i2c bus adapter.\n");
 	return 0;
diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c
index 0d351954db02..e28b825b0433 100644
--- a/drivers/i2c/busses/i2c-pxa.c
+++ b/drivers/i2c/busses/i2c-pxa.c
@@ -1292,10 +1292,8 @@ static int i2c_pxa_probe(struct platform_device *dev)
 #endif
 
 	ret = i2c_add_numbered_adapter(&i2c->adap);
-	if (ret < 0) {
-		dev_err(&dev->dev, "failed to add bus: %d\n", ret);
+	if (ret < 0)
 		goto ereqirq;
-	}
 
 	platform_set_drvdata(dev, i2c);
 
diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c
index 501bd15cb78e..a8497cfdae6f 100644
--- a/drivers/i2c/busses/i2c-qup.c
+++ b/drivers/i2c/busses/i2c-qup.c
@@ -1599,7 +1599,8 @@ static int qup_i2c_pm_resume_runtime(struct device *device)
 #ifdef CONFIG_PM_SLEEP
 static int qup_i2c_suspend(struct device *device)
 {
-	qup_i2c_pm_suspend_runtime(device);
+	if (!pm_runtime_suspended(device))
+		return qup_i2c_pm_suspend_runtime(device);
 	return 0;
 }
 
diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c
index 52407f3c9e1c..726615e54f2a 100644
--- a/drivers/i2c/busses/i2c-rcar.c
+++ b/drivers/i2c/busses/i2c-rcar.c
@@ -378,7 +378,7 @@ static void rcar_i2c_dma(struct rcar_i2c_priv *priv)
 	}
 
 	dma_addr = dma_map_single(chan->device->dev, buf, len, dir);
-	if (dma_mapping_error(dev, dma_addr)) {
+	if (dma_mapping_error(chan->device->dev, dma_addr)) {
 		dev_dbg(dev, "dma map failed, using PIO\n");
 		return;
 	}
@@ -802,6 +802,7 @@ static const struct of_device_id rcar_i2c_dt_ids[] = {
 	{ .compatible = "renesas,i2c-r8a7793", .data = (void *)I2C_RCAR_GEN2 },
 	{ .compatible = "renesas,i2c-r8a7794", .data = (void *)I2C_RCAR_GEN2 },
 	{ .compatible = "renesas,i2c-r8a7795", .data = (void *)I2C_RCAR_GEN3 },
+	{ .compatible = "renesas,i2c-r8a7796", .data = (void *)I2C_RCAR_GEN3 },
 	{},
 };
 MODULE_DEVICE_TABLE(of, rcar_i2c_dt_ids);
@@ -875,10 +876,8 @@ static int rcar_i2c_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, priv);
 
 	ret = i2c_add_numbered_adapter(adap);
-	if (ret < 0) {
-		dev_err(dev, "reg adap failed: %d\n", ret);
+	if (ret < 0)
 		goto out_pm_disable;
-	}
 
 	dev_info(dev, "probed\n");
 
diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c
index d7e3af671543..6263ea82d6ac 100644
--- a/drivers/i2c/busses/i2c-riic.c
+++ b/drivers/i2c/busses/i2c-riic.c
@@ -383,10 +383,8 @@ static int riic_i2c_probe(struct platform_device *pdev)
 
 
 	ret = i2c_add_adapter(adap);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to add adapter\n");
+	if (ret)
 		return ret;
-	}
 
 	platform_set_drvdata(pdev, riic);
 
diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c
index 2bc8b01153d6..50702c7bb244 100644
--- a/drivers/i2c/busses/i2c-rk3x.c
+++ b/drivers/i2c/busses/i2c-rk3x.c
@@ -58,7 +58,7 @@ enum {
 #define REG_CON_LASTACK   BIT(5) /* 1: send NACK after last received byte */
 #define REG_CON_ACTACK    BIT(6) /* 1: stop if NACK is received */
 
-#define REG_CON_TUNING_MASK GENMASK(15, 8)
+#define REG_CON_TUNING_MASK GENMASK_ULL(15, 8)
 
 #define REG_CON_SDA_CFG(cfg) ((cfg) << 8)
 #define REG_CON_STA_CFG(cfg) ((cfg) << 12)
@@ -742,7 +742,7 @@ static int rk3x_i2c_v1_calc_timings(unsigned long clk_rate,
 				    struct i2c_timings *t,
 				    struct rk3x_i2c_calced_timings *t_calc)
 {
-	unsigned long min_low_ns, min_high_ns, min_total_ns;
+	unsigned long min_low_ns, min_high_ns;
 	unsigned long min_setup_start_ns, min_setup_data_ns;
 	unsigned long min_setup_stop_ns, max_hold_data_ns;
 
@@ -793,7 +793,6 @@ static int rk3x_i2c_v1_calc_timings(unsigned long clk_rate,
 
 	/* These are the min dividers needed for min hold times. */
 	min_div_for_hold = (min_low_div + min_high_div);
-	min_total_ns = min_low_ns + min_high_ns;
 
 	/*
 	 * This is the maximum divider so we don't go over the maximum.
@@ -918,7 +917,7 @@ static void rk3x_i2c_adapt_div(struct rk3x_i2c *i2c, unsigned long clk_rate)
  * Code adapted from i2c-cadence.c.
  *
  * Return:	NOTIFY_STOP if the rate change should be aborted, NOTIFY_OK
- *		to acknowedge the change, NOTIFY_DONE if the notification is
+ *		to acknowledge the change, NOTIFY_DONE if the notification is
  *		considered irrelevant.
  */
 static int rk3x_i2c_clk_notifier_cb(struct notifier_block *nb, unsigned long
@@ -1111,6 +1110,15 @@ static int rk3x_i2c_xfer(struct i2c_adapter *adap,
 	return ret < 0 ? ret : num;
 }
 
+static __maybe_unused int rk3x_i2c_resume(struct device *dev)
+{
+	struct rk3x_i2c *i2c = dev_get_drvdata(dev);
+
+	rk3x_i2c_adapt_div(i2c, clk_get_rate(i2c->clk));
+
+	return 0;
+}
+
 static u32 rk3x_i2c_func(struct i2c_adapter *adap)
 {
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_PROTOCOL_MANGLING;
@@ -1303,10 +1311,8 @@ static int rk3x_i2c_probe(struct platform_device *pdev)
 	rk3x_i2c_adapt_div(i2c, clk_rate);
 
 	ret = i2c_add_adapter(&i2c->adap);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "Could not register adapter\n");
+	if (ret < 0)
 		goto err_clk_notifier;
-	}
 
 	dev_info(&pdev->dev, "Initialized RK3xxx I2C bus at %p\n", i2c->regs);
 
@@ -1334,12 +1340,15 @@ static int rk3x_i2c_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static SIMPLE_DEV_PM_OPS(rk3x_i2c_pm_ops, NULL, rk3x_i2c_resume);
+
 static struct platform_driver rk3x_i2c_driver = {
 	.probe   = rk3x_i2c_probe,
 	.remove  = rk3x_i2c_remove,
 	.driver  = {
 		.name  = "rk3x-i2c",
 		.of_match_table = rk3x_i2c_match,
+		.pm = &rk3x_i2c_pm_ops,
 	},
 };
 
diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c
index 38dc1cacfd8b..499af26e736e 100644
--- a/drivers/i2c/busses/i2c-s3c2410.c
+++ b/drivers/i2c/busses/i2c-s3c2410.c
@@ -1215,7 +1215,6 @@ static int s3c24xx_i2c_probe(struct platform_device *pdev)
 
 	ret = i2c_add_numbered_adapter(&i2c->adap);
 	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to add bus to i2c core\n");
 		pm_runtime_disable(&pdev->dev);
 		s3c24xx_i2c_deregister_cpufreq(i2c);
 		clk_unprepare(i2c->clk);
diff --git a/drivers/i2c/busses/i2c-sh7760.c b/drivers/i2c/busses/i2c-sh7760.c
index 24968384b401..c2005c789d2b 100644
--- a/drivers/i2c/busses/i2c-sh7760.c
+++ b/drivers/i2c/busses/i2c-sh7760.c
@@ -510,10 +510,8 @@ static int sh7760_i2c_probe(struct platform_device *pdev)
 	}
 
 	ret = i2c_add_numbered_adapter(&id->adap);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "reg adap failed: %d\n", ret);
+	if (ret < 0)
 		goto out4;
-	}
 
 	platform_set_drvdata(pdev, id);
 
diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c
index 6fb3e2645992..192f36f00e4d 100644
--- a/drivers/i2c/busses/i2c-sh_mobile.c
+++ b/drivers/i2c/busses/i2c-sh_mobile.c
@@ -610,7 +610,7 @@ static void sh_mobile_i2c_xfer_dma(struct sh_mobile_i2c_data *pd)
 		return;
 
 	dma_addr = dma_map_single(chan->device->dev, pd->msg->buf, pd->msg->len, dir);
-	if (dma_mapping_error(pd->dev, dma_addr)) {
+	if (dma_mapping_error(chan->device->dev, dma_addr)) {
 		dev_dbg(pd->dev, "dma map failed, using PIO\n");
 		return;
 	}
@@ -981,7 +981,6 @@ static int sh_mobile_i2c_probe(struct platform_device *dev)
 	ret = i2c_add_numbered_adapter(adap);
 	if (ret < 0) {
 		sh_mobile_i2c_release_dma(pd);
-		dev_err(&dev->dev, "cannot add numbered adapter\n");
 		return ret;
 	}
 
diff --git a/drivers/i2c/busses/i2c-sirf.c b/drivers/i2c/busses/i2c-sirf.c
index 792a42bdd335..95e81d0f72b4 100644
--- a/drivers/i2c/busses/i2c-sirf.c
+++ b/drivers/i2c/busses/i2c-sirf.c
@@ -387,10 +387,8 @@ static int i2c_sirfsoc_probe(struct platform_device *pdev)
 		writel(regval, siic->base + SIRFSOC_I2C_SDA_DELAY);
 
 	err = i2c_add_numbered_adapter(adap);
-	if (err < 0) {
-		dev_err(&pdev->dev, "Can't add new i2c adapter\n");
+	if (err < 0)
 		goto out;
-	}
 
 	clk_disable(clk);
 
diff --git a/drivers/i2c/busses/i2c-st.c b/drivers/i2c/busses/i2c-st.c
index 944ec4205084..1371547ce1a3 100644
--- a/drivers/i2c/busses/i2c-st.c
+++ b/drivers/i2c/busses/i2c-st.c
@@ -874,10 +874,8 @@ static int st_i2c_probe(struct platform_device *pdev)
 	init_completion(&i2c_dev->complete);
 
 	ret = i2c_add_adapter(adap);
-	if (ret) {
-		dev_err(&pdev->dev, "Failed to add adapter\n");
+	if (ret)
 		return ret;
-	}
 
 	platform_set_drvdata(pdev, i2c_dev);
 
diff --git a/drivers/i2c/busses/i2c-stu300.c b/drivers/i2c/busses/i2c-stu300.c
index 460c134832ac..dc63236b45b2 100644
--- a/drivers/i2c/busses/i2c-stu300.c
+++ b/drivers/i2c/busses/i2c-stu300.c
@@ -920,11 +920,8 @@ static int stu300_probe(struct platform_device *pdev)
 
 	/* i2c device drivers may be active on return from add_adapter() */
 	ret = i2c_add_numbered_adapter(adap);
-	if (ret) {
-		dev_err(&pdev->dev, "failure adding ST Micro DDC "
-		       "I2C adapter\n");
+	if (ret)
 		return ret;
-	}
 
 	platform_set_drvdata(pdev, dev);
 	dev_info(&pdev->dev, "ST DDC I2C @ %p, irq %d\n",
diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c
index b126dbaa47e3..4af9bbae20df 100644
--- a/drivers/i2c/busses/i2c-tegra.c
+++ b/drivers/i2c/busses/i2c-tegra.c
@@ -28,6 +28,9 @@
 #include <linux/of_device.h>
 #include <linux/module.h>
 #include <linux/reset.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/pm_runtime.h>
+#include <linux/iopoll.h>
 
 #include <asm/unaligned.h>
 
@@ -36,21 +39,21 @@
 
 #define I2C_CNFG				0x000
 #define I2C_CNFG_DEBOUNCE_CNT_SHIFT		12
-#define I2C_CNFG_PACKET_MODE_EN			(1<<10)
-#define I2C_CNFG_NEW_MASTER_FSM			(1<<11)
-#define I2C_CNFG_MULTI_MASTER_MODE		(1<<17)
+#define I2C_CNFG_PACKET_MODE_EN			BIT(10)
+#define I2C_CNFG_NEW_MASTER_FSM			BIT(11)
+#define I2C_CNFG_MULTI_MASTER_MODE		BIT(17)
 #define I2C_STATUS				0x01C
 #define I2C_SL_CNFG				0x020
-#define I2C_SL_CNFG_NACK			(1<<1)
-#define I2C_SL_CNFG_NEWSL			(1<<2)
+#define I2C_SL_CNFG_NACK			BIT(1)
+#define I2C_SL_CNFG_NEWSL			BIT(2)
 #define I2C_SL_ADDR1				0x02c
 #define I2C_SL_ADDR2				0x030
 #define I2C_TX_FIFO				0x050
 #define I2C_RX_FIFO				0x054
 #define I2C_PACKET_TRANSFER_STATUS		0x058
 #define I2C_FIFO_CONTROL			0x05c
-#define I2C_FIFO_CONTROL_TX_FLUSH		(1<<1)
-#define I2C_FIFO_CONTROL_RX_FLUSH		(1<<0)
+#define I2C_FIFO_CONTROL_TX_FLUSH		BIT(1)
+#define I2C_FIFO_CONTROL_RX_FLUSH		BIT(0)
 #define I2C_FIFO_CONTROL_TX_TRIG_SHIFT		5
 #define I2C_FIFO_CONTROL_RX_TRIG_SHIFT		2
 #define I2C_FIFO_STATUS				0x060
@@ -60,26 +63,26 @@
 #define I2C_FIFO_STATUS_RX_SHIFT		0
 #define I2C_INT_MASK				0x064
 #define I2C_INT_STATUS				0x068
-#define I2C_INT_PACKET_XFER_COMPLETE		(1<<7)
-#define I2C_INT_ALL_PACKETS_XFER_COMPLETE	(1<<6)
-#define I2C_INT_TX_FIFO_OVERFLOW		(1<<5)
-#define I2C_INT_RX_FIFO_UNDERFLOW		(1<<4)
-#define I2C_INT_NO_ACK				(1<<3)
-#define I2C_INT_ARBITRATION_LOST		(1<<2)
-#define I2C_INT_TX_FIFO_DATA_REQ		(1<<1)
-#define I2C_INT_RX_FIFO_DATA_REQ		(1<<0)
+#define I2C_INT_PACKET_XFER_COMPLETE		BIT(7)
+#define I2C_INT_ALL_PACKETS_XFER_COMPLETE	BIT(6)
+#define I2C_INT_TX_FIFO_OVERFLOW		BIT(5)
+#define I2C_INT_RX_FIFO_UNDERFLOW		BIT(4)
+#define I2C_INT_NO_ACK				BIT(3)
+#define I2C_INT_ARBITRATION_LOST		BIT(2)
+#define I2C_INT_TX_FIFO_DATA_REQ		BIT(1)
+#define I2C_INT_RX_FIFO_DATA_REQ		BIT(0)
 #define I2C_CLK_DIVISOR				0x06c
 #define I2C_CLK_DIVISOR_STD_FAST_MODE_SHIFT	16
 #define I2C_CLK_MULTIPLIER_STD_FAST_MODE	8
 
 #define DVC_CTRL_REG1				0x000
-#define DVC_CTRL_REG1_INTR_EN			(1<<10)
+#define DVC_CTRL_REG1_INTR_EN			BIT(10)
 #define DVC_CTRL_REG2				0x004
 #define DVC_CTRL_REG3				0x008
-#define DVC_CTRL_REG3_SW_PROG			(1<<26)
-#define DVC_CTRL_REG3_I2C_DONE_INTR_EN		(1<<30)
+#define DVC_CTRL_REG3_SW_PROG			BIT(26)
+#define DVC_CTRL_REG3_I2C_DONE_INTR_EN		BIT(30)
 #define DVC_STATUS				0x00c
-#define DVC_STATUS_I2C_DONE_INTR		(1<<30)
+#define DVC_STATUS_I2C_DONE_INTR		BIT(30)
 
 #define I2C_ERR_NONE				0x00
 #define I2C_ERR_NO_ACK				0x01
@@ -89,26 +92,28 @@
 #define PACKET_HEADER0_HEADER_SIZE_SHIFT	28
 #define PACKET_HEADER0_PACKET_ID_SHIFT		16
 #define PACKET_HEADER0_CONT_ID_SHIFT		12
-#define PACKET_HEADER0_PROTOCOL_I2C		(1<<4)
-
-#define I2C_HEADER_HIGHSPEED_MODE		(1<<22)
-#define I2C_HEADER_CONT_ON_NAK			(1<<21)
-#define I2C_HEADER_SEND_START_BYTE		(1<<20)
-#define I2C_HEADER_READ				(1<<19)
-#define I2C_HEADER_10BIT_ADDR			(1<<18)
-#define I2C_HEADER_IE_ENABLE			(1<<17)
-#define I2C_HEADER_REPEAT_START			(1<<16)
-#define I2C_HEADER_CONTINUE_XFER		(1<<15)
+#define PACKET_HEADER0_PROTOCOL_I2C		BIT(4)
+
+#define I2C_HEADER_HIGHSPEED_MODE		BIT(22)
+#define I2C_HEADER_CONT_ON_NAK			BIT(21)
+#define I2C_HEADER_SEND_START_BYTE		BIT(20)
+#define I2C_HEADER_READ				BIT(19)
+#define I2C_HEADER_10BIT_ADDR			BIT(18)
+#define I2C_HEADER_IE_ENABLE			BIT(17)
+#define I2C_HEADER_REPEAT_START			BIT(16)
+#define I2C_HEADER_CONTINUE_XFER		BIT(15)
 #define I2C_HEADER_MASTER_ADDR_SHIFT		12
 #define I2C_HEADER_SLAVE_ADDR_SHIFT		1
 
 #define I2C_CONFIG_LOAD				0x08C
-#define I2C_MSTR_CONFIG_LOAD			(1 << 0)
-#define I2C_SLV_CONFIG_LOAD			(1 << 1)
-#define I2C_TIMEOUT_CONFIG_LOAD			(1 << 2)
+#define I2C_MSTR_CONFIG_LOAD			BIT(0)
+#define I2C_SLV_CONFIG_LOAD			BIT(1)
+#define I2C_TIMEOUT_CONFIG_LOAD			BIT(2)
 
 #define I2C_CLKEN_OVERRIDE			0x090
-#define I2C_MST_CORE_CLKEN_OVR			(1 << 0)
+#define I2C_MST_CORE_CLKEN_OVR			BIT(0)
+
+#define I2C_CONFIG_LOAD_TIMEOUT			1000000
 
 /*
  * msg_end_type: The bus control which need to be send at end of transfer.
@@ -191,9 +196,11 @@ struct tegra_i2c_dev {
 	u16 clk_divisor_non_hs_mode;
 	bool is_suspended;
 	bool is_multimaster_mode;
+	spinlock_t xfer_lock;
 };
 
-static void dvc_writel(struct tegra_i2c_dev *i2c_dev, u32 val, unsigned long reg)
+static void dvc_writel(struct tegra_i2c_dev *i2c_dev, u32 val,
+		       unsigned long reg)
 {
 	writel(val, i2c_dev->base + reg);
 }
@@ -244,15 +251,17 @@ static void i2c_readsl(struct tegra_i2c_dev *i2c_dev, void *data,
 
 static void tegra_i2c_mask_irq(struct tegra_i2c_dev *i2c_dev, u32 mask)
 {
-	u32 int_mask = i2c_readl(i2c_dev, I2C_INT_MASK);
-	int_mask &= ~mask;
+	u32 int_mask;
+
+	int_mask = i2c_readl(i2c_dev, I2C_INT_MASK) & ~mask;
 	i2c_writel(i2c_dev, int_mask, I2C_INT_MASK);
 }
 
 static void tegra_i2c_unmask_irq(struct tegra_i2c_dev *i2c_dev, u32 mask)
 {
-	u32 int_mask = i2c_readl(i2c_dev, I2C_INT_MASK);
-	int_mask |= mask;
+	u32 int_mask;
+
+	int_mask = i2c_readl(i2c_dev, I2C_INT_MASK) | mask;
 	i2c_writel(i2c_dev, int_mask, I2C_INT_MASK);
 }
 
@@ -260,6 +269,7 @@ static int tegra_i2c_flush_fifos(struct tegra_i2c_dev *i2c_dev)
 {
 	unsigned long timeout = jiffies + HZ;
 	u32 val = i2c_readl(i2c_dev, I2C_FIFO_CONTROL);
+
 	val |= I2C_FIFO_CONTROL_TX_FLUSH | I2C_FIFO_CONTROL_RX_FLUSH;
 	i2c_writel(i2c_dev, val, I2C_FIFO_CONTROL);
 
@@ -385,7 +395,8 @@ static int tegra_i2c_fill_tx_fifo(struct tegra_i2c_dev *i2c_dev)
  */
 static void tegra_dvc_init(struct tegra_i2c_dev *i2c_dev)
 {
-	u32 val = 0;
+	u32 val;
+
 	val = dvc_readl(i2c_dev, DVC_CTRL_REG3);
 	val |= DVC_CTRL_REG3_SW_PROG;
 	val |= DVC_CTRL_REG3_I2C_DONE_INTR_EN;
@@ -396,9 +407,15 @@ static void tegra_dvc_init(struct tegra_i2c_dev *i2c_dev)
 	dvc_writel(i2c_dev, val, DVC_CTRL_REG1);
 }
 
-static inline int tegra_i2c_clock_enable(struct tegra_i2c_dev *i2c_dev)
+static int tegra_i2c_runtime_resume(struct device *dev)
 {
+	struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev);
 	int ret;
+
+	ret = pinctrl_pm_select_default_state(i2c_dev->dev);
+	if (ret)
+		return ret;
+
 	if (!i2c_dev->hw->has_single_clk_source) {
 		ret = clk_enable(i2c_dev->fast_clk);
 		if (ret < 0) {
@@ -407,32 +424,66 @@ static inline int tegra_i2c_clock_enable(struct tegra_i2c_dev *i2c_dev)
 			return ret;
 		}
 	}
+
 	ret = clk_enable(i2c_dev->div_clk);
 	if (ret < 0) {
 		dev_err(i2c_dev->dev,
 			"Enabling div clk failed, err %d\n", ret);
 		clk_disable(i2c_dev->fast_clk);
+		return ret;
 	}
-	return ret;
+
+	return 0;
 }
 
-static inline void tegra_i2c_clock_disable(struct tegra_i2c_dev *i2c_dev)
+static int tegra_i2c_runtime_suspend(struct device *dev)
 {
+	struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev);
+
 	clk_disable(i2c_dev->div_clk);
 	if (!i2c_dev->hw->has_single_clk_source)
 		clk_disable(i2c_dev->fast_clk);
+
+	return pinctrl_pm_select_idle_state(i2c_dev->dev);
+}
+
+static int tegra_i2c_wait_for_config_load(struct tegra_i2c_dev *i2c_dev)
+{
+	unsigned long reg_offset;
+	void __iomem *addr;
+	u32 val;
+	int err;
+
+	if (i2c_dev->hw->has_config_load_reg) {
+		reg_offset = tegra_i2c_reg_addr(i2c_dev, I2C_CONFIG_LOAD);
+		addr = i2c_dev->base + reg_offset;
+		i2c_writel(i2c_dev, I2C_MSTR_CONFIG_LOAD, I2C_CONFIG_LOAD);
+		if (in_interrupt())
+			err = readl_poll_timeout_atomic(addr, val, val == 0,
+					1000, I2C_CONFIG_LOAD_TIMEOUT);
+		else
+			err = readl_poll_timeout(addr, val, val == 0,
+					1000, I2C_CONFIG_LOAD_TIMEOUT);
+
+		if (err) {
+			dev_warn(i2c_dev->dev,
+				 "timeout waiting for config load\n");
+			return err;
+		}
+	}
+
+	return 0;
 }
 
 static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev)
 {
 	u32 val;
-	int err = 0;
+	int err;
 	u32 clk_divisor;
-	unsigned long timeout = jiffies + HZ;
 
-	err = tegra_i2c_clock_enable(i2c_dev);
+	err = pm_runtime_get_sync(i2c_dev->dev);
 	if (err < 0) {
-		dev_err(i2c_dev->dev, "Clock enable failed %d\n", err);
+		dev_err(i2c_dev->dev, "runtime resume failed %d\n", err);
 		return err;
 	}
 
@@ -460,54 +511,59 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev)
 
 	if (!i2c_dev->is_dvc) {
 		u32 sl_cfg = i2c_readl(i2c_dev, I2C_SL_CNFG);
+
 		sl_cfg |= I2C_SL_CNFG_NACK | I2C_SL_CNFG_NEWSL;
 		i2c_writel(i2c_dev, sl_cfg, I2C_SL_CNFG);
 		i2c_writel(i2c_dev, 0xfc, I2C_SL_ADDR1);
 		i2c_writel(i2c_dev, 0x00, I2C_SL_ADDR2);
-
 	}
 
 	val = 7 << I2C_FIFO_CONTROL_TX_TRIG_SHIFT |
 		0 << I2C_FIFO_CONTROL_RX_TRIG_SHIFT;
 	i2c_writel(i2c_dev, val, I2C_FIFO_CONTROL);
 
-	if (tegra_i2c_flush_fifos(i2c_dev))
-		err = -ETIMEDOUT;
+	err = tegra_i2c_flush_fifos(i2c_dev);
+	if (err)
+		goto err;
 
 	if (i2c_dev->is_multimaster_mode && i2c_dev->hw->has_slcg_override_reg)
 		i2c_writel(i2c_dev, I2C_MST_CORE_CLKEN_OVR, I2C_CLKEN_OVERRIDE);
 
-	if (i2c_dev->hw->has_config_load_reg) {
-		i2c_writel(i2c_dev, I2C_MSTR_CONFIG_LOAD, I2C_CONFIG_LOAD);
-		while (i2c_readl(i2c_dev, I2C_CONFIG_LOAD) != 0) {
-			if (time_after(jiffies, timeout)) {
-				dev_warn(i2c_dev->dev,
-					"timeout waiting for config load\n");
-				err = -ETIMEDOUT;
-				goto err;
-			}
-			msleep(1);
-		}
-	}
+	err = tegra_i2c_wait_for_config_load(i2c_dev);
+	if (err)
+		goto err;
 
 	if (i2c_dev->irq_disabled) {
-		i2c_dev->irq_disabled = 0;
+		i2c_dev->irq_disabled = false;
 		enable_irq(i2c_dev->irq);
 	}
 
 err:
-	tegra_i2c_clock_disable(i2c_dev);
+	pm_runtime_put(i2c_dev->dev);
 	return err;
 }
 
+static int tegra_i2c_disable_packet_mode(struct tegra_i2c_dev *i2c_dev)
+{
+	u32 cnfg;
+
+	cnfg = i2c_readl(i2c_dev, I2C_CNFG);
+	if (cnfg & I2C_CNFG_PACKET_MODE_EN)
+		i2c_writel(i2c_dev, cnfg & ~I2C_CNFG_PACKET_MODE_EN, I2C_CNFG);
+
+	return tegra_i2c_wait_for_config_load(i2c_dev);
+}
+
 static irqreturn_t tegra_i2c_isr(int irq, void *dev_id)
 {
 	u32 status;
 	const u32 status_err = I2C_INT_NO_ACK | I2C_INT_ARBITRATION_LOST;
 	struct tegra_i2c_dev *i2c_dev = dev_id;
+	unsigned long flags;
 
 	status = i2c_readl(i2c_dev, I2C_INT_STATUS);
 
+	spin_lock_irqsave(&i2c_dev->xfer_lock, flags);
 	if (status == 0) {
 		dev_warn(i2c_dev->dev, "irq status 0 %08x %08x %08x\n",
 			 i2c_readl(i2c_dev, I2C_PACKET_TRANSFER_STATUS),
@@ -517,12 +573,13 @@ static irqreturn_t tegra_i2c_isr(int irq, void *dev_id)
 
 		if (!i2c_dev->irq_disabled) {
 			disable_irq_nosync(i2c_dev->irq);
-			i2c_dev->irq_disabled = 1;
+			i2c_dev->irq_disabled = true;
 		}
 		goto err;
 	}
 
 	if (unlikely(status & status_err)) {
+		tegra_i2c_disable_packet_mode(i2c_dev);
 		if (status & I2C_INT_NO_ACK)
 			i2c_dev->msg_err |= I2C_ERR_NO_ACK;
 		if (status & I2C_INT_ARBITRATION_LOST)
@@ -552,7 +609,7 @@ static irqreturn_t tegra_i2c_isr(int irq, void *dev_id)
 		BUG_ON(i2c_dev->msg_buf_remaining);
 		complete(&i2c_dev->msg_complete);
 	}
-	return IRQ_HANDLED;
+	goto done;
 err:
 	/* An error occurred, mask all interrupts */
 	tegra_i2c_mask_irq(i2c_dev, I2C_INT_NO_ACK | I2C_INT_ARBITRATION_LOST |
@@ -563,6 +620,8 @@ err:
 		dvc_writel(i2c_dev, DVC_STATUS_I2C_DONE_INTR, DVC_STATUS);
 
 	complete(&i2c_dev->msg_complete);
+done:
+	spin_unlock_irqrestore(&i2c_dev->xfer_lock, flags);
 	return IRQ_HANDLED;
 }
 
@@ -572,6 +631,7 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev,
 	u32 packet_header;
 	u32 int_mask;
 	unsigned long time_left;
+	unsigned long flags;
 
 	tegra_i2c_flush_fifos(i2c_dev);
 
@@ -584,6 +644,11 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev,
 	i2c_dev->msg_read = (msg->flags & I2C_M_RD);
 	reinit_completion(&i2c_dev->msg_complete);
 
+	spin_lock_irqsave(&i2c_dev->xfer_lock, flags);
+
+	int_mask = I2C_INT_NO_ACK | I2C_INT_ARBITRATION_LOST;
+	tegra_i2c_unmask_irq(i2c_dev, int_mask);
+
 	packet_header = (0 << PACKET_HEADER0_HEADER_SIZE_SHIFT) |
 			PACKET_HEADER0_PROTOCOL_I2C |
 			(i2c_dev->cont_id << PACKET_HEADER0_CONT_ID_SHIFT) |
@@ -613,14 +678,15 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev,
 	if (!(msg->flags & I2C_M_RD))
 		tegra_i2c_fill_tx_fifo(i2c_dev);
 
-	int_mask = I2C_INT_NO_ACK | I2C_INT_ARBITRATION_LOST;
 	if (i2c_dev->hw->has_per_pkt_xfer_complete_irq)
 		int_mask |= I2C_INT_PACKET_XFER_COMPLETE;
 	if (msg->flags & I2C_M_RD)
 		int_mask |= I2C_INT_RX_FIFO_DATA_REQ;
 	else if (i2c_dev->msg_buf_remaining)
 		int_mask |= I2C_INT_TX_FIFO_DATA_REQ;
+
 	tegra_i2c_unmask_irq(i2c_dev, int_mask);
+	spin_unlock_irqrestore(&i2c_dev->xfer_lock, flags);
 	dev_dbg(i2c_dev->dev, "unmasked irq: %02x\n",
 		i2c_readl(i2c_dev, I2C_INT_MASK));
 
@@ -643,9 +709,10 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev,
 		return 0;
 
 	/*
-	 * NACK interrupt is generated before the I2C controller generates the
-	 * STOP condition on the bus. So wait for 2 clock periods before resetting
-	 * the controller so that STOP condition has been delivered properly.
+	 * NACK interrupt is generated before the I2C controller generates
+	 * the STOP condition on the bus. So wait for 2 clock periods
+	 * before resetting the controller so that the STOP condition has
+	 * been delivered properly.
 	 */
 	if (i2c_dev->msg_err == I2C_ERR_NO_ACK)
 		udelay(DIV_ROUND_UP(2 * 1000000, i2c_dev->bus_clk_rate));
@@ -670,14 +737,15 @@ static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[],
 	if (i2c_dev->is_suspended)
 		return -EBUSY;
 
-	ret = tegra_i2c_clock_enable(i2c_dev);
+	ret = pm_runtime_get_sync(i2c_dev->dev);
 	if (ret < 0) {
-		dev_err(i2c_dev->dev, "Clock enable failed %d\n", ret);
+		dev_err(i2c_dev->dev, "runtime resume failed %d\n", ret);
 		return ret;
 	}
 
 	for (i = 0; i < num; i++) {
 		enum msg_end_type end_type = MSG_END_STOP;
+
 		if (i < (num - 1)) {
 			if (msgs[i + 1].flags & I2C_M_NOSTART)
 				end_type = MSG_END_CONTINUE;
@@ -688,7 +756,9 @@ static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[],
 		if (ret)
 			break;
 	}
-	tegra_i2c_clock_disable(i2c_dev);
+
+	pm_runtime_put(i2c_dev->dev);
+
 	return ret ?: i;
 }
 
@@ -825,7 +895,7 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 
 	div_clk = devm_clk_get(&pdev->dev, "div-clk");
 	if (IS_ERR(div_clk)) {
-		dev_err(&pdev->dev, "missing controller clock");
+		dev_err(&pdev->dev, "missing controller clock\n");
 		return PTR_ERR(div_clk);
 	}
 
@@ -843,27 +913,22 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 
 	i2c_dev->rst = devm_reset_control_get(&pdev->dev, "i2c");
 	if (IS_ERR(i2c_dev->rst)) {
-		dev_err(&pdev->dev, "missing controller reset");
+		dev_err(&pdev->dev, "missing controller reset\n");
 		return PTR_ERR(i2c_dev->rst);
 	}
 
 	tegra_i2c_parse_dt(i2c_dev);
 
-	i2c_dev->hw = &tegra20_i2c_hw;
-
-	if (pdev->dev.of_node) {
-		i2c_dev->hw = of_device_get_match_data(&pdev->dev);
-		i2c_dev->is_dvc = of_device_is_compatible(pdev->dev.of_node,
-						"nvidia,tegra20-i2c-dvc");
-	} else if (pdev->id == 3) {
-		i2c_dev->is_dvc = 1;
-	}
+	i2c_dev->hw = of_device_get_match_data(&pdev->dev);
+	i2c_dev->is_dvc = of_device_is_compatible(pdev->dev.of_node,
+						  "nvidia,tegra20-i2c-dvc");
 	init_completion(&i2c_dev->msg_complete);
+	spin_lock_init(&i2c_dev->xfer_lock);
 
 	if (!i2c_dev->hw->has_single_clk_source) {
 		fast_clk = devm_clk_get(&pdev->dev, "fast-clk");
 		if (IS_ERR(fast_clk)) {
-			dev_err(&pdev->dev, "missing fast clock");
+			dev_err(&pdev->dev, "missing fast clock\n");
 			return PTR_ERR(fast_clk);
 		}
 		i2c_dev->fast_clk = fast_clk;
@@ -900,18 +965,27 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 		goto unprepare_fast_clk;
 	}
 
+	pm_runtime_enable(&pdev->dev);
+	if (!pm_runtime_enabled(&pdev->dev)) {
+		ret = tegra_i2c_runtime_resume(&pdev->dev);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "runtime resume failed\n");
+			goto unprepare_div_clk;
+		}
+	}
+
 	if (i2c_dev->is_multimaster_mode) {
 		ret = clk_enable(i2c_dev->div_clk);
 		if (ret < 0) {
 			dev_err(i2c_dev->dev, "div_clk enable failed %d\n",
 				ret);
-			goto unprepare_div_clk;
+			goto disable_rpm;
 		}
 	}
 
 	ret = tegra_i2c_init(i2c_dev);
 	if (ret) {
-		dev_err(&pdev->dev, "Failed to initialize i2c controller");
+		dev_err(&pdev->dev, "Failed to initialize i2c controller\n");
 		goto disable_div_clk;
 	}
 
@@ -925,17 +999,15 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 	i2c_set_adapdata(&i2c_dev->adapter, i2c_dev);
 	i2c_dev->adapter.owner = THIS_MODULE;
 	i2c_dev->adapter.class = I2C_CLASS_DEPRECATED;
-	strlcpy(i2c_dev->adapter.name, "Tegra I2C adapter",
+	strlcpy(i2c_dev->adapter.name, dev_name(&pdev->dev),
 		sizeof(i2c_dev->adapter.name));
 	i2c_dev->adapter.dev.parent = &pdev->dev;
 	i2c_dev->adapter.nr = pdev->id;
 	i2c_dev->adapter.dev.of_node = pdev->dev.of_node;
 
 	ret = i2c_add_numbered_adapter(&i2c_dev->adapter);
-	if (ret) {
-		dev_err(&pdev->dev, "Failed to add I2C adapter\n");
+	if (ret)
 		goto disable_div_clk;
-	}
 
 	return 0;
 
@@ -943,6 +1015,11 @@ disable_div_clk:
 	if (i2c_dev->is_multimaster_mode)
 		clk_disable(i2c_dev->div_clk);
 
+disable_rpm:
+	pm_runtime_disable(&pdev->dev);
+	if (!pm_runtime_status_suspended(&pdev->dev))
+		tegra_i2c_runtime_suspend(&pdev->dev);
+
 unprepare_div_clk:
 	clk_unprepare(i2c_dev->div_clk);
 
@@ -956,11 +1033,16 @@ unprepare_fast_clk:
 static int tegra_i2c_remove(struct platform_device *pdev)
 {
 	struct tegra_i2c_dev *i2c_dev = platform_get_drvdata(pdev);
+
 	i2c_del_adapter(&i2c_dev->adapter);
 
 	if (i2c_dev->is_multimaster_mode)
 		clk_disable(i2c_dev->div_clk);
 
+	pm_runtime_disable(&pdev->dev);
+	if (!pm_runtime_status_suspended(&pdev->dev))
+		tegra_i2c_runtime_suspend(&pdev->dev);
+
 	clk_unprepare(i2c_dev->div_clk);
 	if (!i2c_dev->hw->has_single_clk_source)
 		clk_unprepare(i2c_dev->fast_clk);
@@ -988,20 +1070,19 @@ static int tegra_i2c_resume(struct device *dev)
 	i2c_lock_adapter(&i2c_dev->adapter);
 
 	ret = tegra_i2c_init(i2c_dev);
-
-	if (ret) {
-		i2c_unlock_adapter(&i2c_dev->adapter);
-		return ret;
-	}
-
-	i2c_dev->is_suspended = false;
+	if (!ret)
+		i2c_dev->is_suspended = false;
 
 	i2c_unlock_adapter(&i2c_dev->adapter);
 
-	return 0;
+	return ret;
 }
 
-static SIMPLE_DEV_PM_OPS(tegra_i2c_pm, tegra_i2c_suspend, tegra_i2c_resume);
+static const struct dev_pm_ops tegra_i2c_pm = {
+	SET_RUNTIME_PM_OPS(tegra_i2c_runtime_suspend, tegra_i2c_runtime_resume,
+			   NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(tegra_i2c_suspend, tegra_i2c_resume)
+};
 #define TEGRA_I2C_PM	(&tegra_i2c_pm)
 #else
 #define TEGRA_I2C_PM	NULL
diff --git a/drivers/i2c/busses/i2c-thunderx-pcidrv.c b/drivers/i2c/busses/i2c-thunderx-pcidrv.c
new file mode 100644
index 000000000000..bba5b429f69c
--- /dev/null
+++ b/drivers/i2c/busses/i2c-thunderx-pcidrv.c
@@ -0,0 +1,259 @@
+/*
+ * Cavium ThunderX i2c driver.
+ *
+ * Copyright (C) 2015,2016 Cavium Inc.
+ * Authors: Fred Martin <fmartin@caviumnetworks.com>
+ *	    Jan Glauber <jglauber@cavium.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/acpi.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/i2c-smbus.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_irq.h>
+#include <linux/pci.h>
+
+#include "i2c-octeon-core.h"
+
+#define DRV_NAME "i2c-thunderx"
+
+#define PCI_DEVICE_ID_THUNDER_TWSI	0xa012
+
+#define SYS_FREQ_DEFAULT		700000000
+
+#define TWSI_INT_ENA_W1C		0x1028
+#define TWSI_INT_ENA_W1S		0x1030
+
+/*
+ * Enable the CORE interrupt.
+ * The interrupt will be asserted when there is non-STAT_IDLE state in the
+ * SW_TWSI_EOP_TWSI_STAT register.
+ */
+static void thunder_i2c_int_enable(struct octeon_i2c *i2c)
+{
+	octeon_i2c_writeq_flush(TWSI_INT_CORE_INT,
+				i2c->twsi_base + TWSI_INT_ENA_W1S);
+}
+
+/*
+ * Disable the CORE interrupt.
+ */
+static void thunder_i2c_int_disable(struct octeon_i2c *i2c)
+{
+	octeon_i2c_writeq_flush(TWSI_INT_CORE_INT,
+				i2c->twsi_base + TWSI_INT_ENA_W1C);
+}
+
+static void thunder_i2c_hlc_int_enable(struct octeon_i2c *i2c)
+{
+	octeon_i2c_writeq_flush(TWSI_INT_ST_INT | TWSI_INT_TS_INT,
+				i2c->twsi_base + TWSI_INT_ENA_W1S);
+}
+
+static void thunder_i2c_hlc_int_disable(struct octeon_i2c *i2c)
+{
+	octeon_i2c_writeq_flush(TWSI_INT_ST_INT | TWSI_INT_TS_INT,
+				i2c->twsi_base + TWSI_INT_ENA_W1C);
+}
+
+static u32 thunderx_i2c_functionality(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK) |
+	       I2C_FUNC_SMBUS_READ_BLOCK_DATA | I2C_SMBUS_BLOCK_PROC_CALL;
+}
+
+static const struct i2c_algorithm thunderx_i2c_algo = {
+	.master_xfer = octeon_i2c_xfer,
+	.functionality = thunderx_i2c_functionality,
+};
+
+static struct i2c_adapter thunderx_i2c_ops = {
+	.owner	= THIS_MODULE,
+	.name	= "ThunderX adapter",
+	.algo	= &thunderx_i2c_algo,
+};
+
+static void thunder_i2c_clock_enable(struct device *dev, struct octeon_i2c *i2c)
+{
+	int ret;
+
+	i2c->clk = clk_get(dev, NULL);
+	if (IS_ERR(i2c->clk)) {
+		i2c->clk = NULL;
+		goto skip;
+	}
+
+	ret = clk_prepare_enable(i2c->clk);
+	if (ret)
+		goto skip;
+	i2c->sys_freq = clk_get_rate(i2c->clk);
+
+skip:
+	if (!i2c->sys_freq)
+		i2c->sys_freq = SYS_FREQ_DEFAULT;
+}
+
+static void thunder_i2c_clock_disable(struct device *dev, struct clk *clk)
+{
+	if (!clk)
+		return;
+	clk_disable_unprepare(clk);
+	clk_put(clk);
+}
+
+static int thunder_i2c_smbus_setup_of(struct octeon_i2c *i2c,
+				      struct device_node *node)
+{
+	u32 type;
+
+	if (!node)
+		return -EINVAL;
+
+	i2c->alert_data.irq = irq_of_parse_and_map(node, 0);
+	if (!i2c->alert_data.irq)
+		return -EINVAL;
+
+	type = irqd_get_trigger_type(irq_get_irq_data(i2c->alert_data.irq));
+	i2c->alert_data.alert_edge_triggered =
+		(type & IRQ_TYPE_LEVEL_MASK) ? 1 : 0;
+
+	i2c->ara = i2c_setup_smbus_alert(&i2c->adap, &i2c->alert_data);
+	if (!i2c->ara)
+		return -ENODEV;
+	return 0;
+}
+
+static int thunder_i2c_smbus_setup(struct octeon_i2c *i2c,
+				   struct device_node *node)
+{
+	/* TODO: ACPI support */
+	if (!acpi_disabled)
+		return -EOPNOTSUPP;
+
+	return thunder_i2c_smbus_setup_of(i2c, node);
+}
+
+static void thunder_i2c_smbus_remove(struct octeon_i2c *i2c)
+{
+	if (i2c->ara)
+		i2c_unregister_device(i2c->ara);
+}
+
+static int thunder_i2c_probe_pci(struct pci_dev *pdev,
+				 const struct pci_device_id *ent)
+{
+	struct device *dev = &pdev->dev;
+	struct octeon_i2c *i2c;
+	int ret;
+
+	i2c = devm_kzalloc(dev, sizeof(*i2c), GFP_KERNEL);
+	if (!i2c)
+		return -ENOMEM;
+
+	i2c->roff.sw_twsi = 0x1000;
+	i2c->roff.twsi_int = 0x1010;
+	i2c->roff.sw_twsi_ext = 0x1018;
+
+	i2c->dev = dev;
+	pci_set_drvdata(pdev, i2c);
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return ret;
+
+	ret = pci_request_regions(pdev, DRV_NAME);
+	if (ret)
+		return ret;
+
+	i2c->twsi_base = pcim_iomap(pdev, 0, pci_resource_len(pdev, 0));
+	if (!i2c->twsi_base)
+		return -EINVAL;
+
+	thunder_i2c_clock_enable(dev, i2c);
+	ret = device_property_read_u32(dev, "clock-frequency", &i2c->twsi_freq);
+	if (ret)
+		i2c->twsi_freq = 100000;
+
+	init_waitqueue_head(&i2c->queue);
+
+	i2c->int_enable = thunder_i2c_int_enable;
+	i2c->int_disable = thunder_i2c_int_disable;
+	i2c->hlc_int_enable = thunder_i2c_hlc_int_enable;
+	i2c->hlc_int_disable = thunder_i2c_hlc_int_disable;
+
+	ret = pci_enable_msix(pdev, &i2c->i2c_msix, 1);
+	if (ret)
+		goto error;
+
+	ret = devm_request_irq(dev, i2c->i2c_msix.vector, octeon_i2c_isr, 0,
+			       DRV_NAME, i2c);
+	if (ret)
+		goto error;
+
+	ret = octeon_i2c_init_lowlevel(i2c);
+	if (ret)
+		goto error;
+
+	octeon_i2c_set_clock(i2c);
+
+	i2c->adap = thunderx_i2c_ops;
+	i2c->adap.retries = 5;
+	i2c->adap.bus_recovery_info = &octeon_i2c_recovery_info;
+	i2c->adap.dev.parent = dev;
+	i2c->adap.dev.of_node = pdev->dev.of_node;
+	snprintf(i2c->adap.name, sizeof(i2c->adap.name),
+		 "Cavium ThunderX i2c adapter at %s", dev_name(dev));
+	i2c_set_adapdata(&i2c->adap, i2c);
+
+	ret = i2c_add_adapter(&i2c->adap);
+	if (ret)
+		goto error;
+
+	dev_info(i2c->dev, "Probed. Set system clock to %u\n", i2c->sys_freq);
+
+	ret = thunder_i2c_smbus_setup(i2c, pdev->dev.of_node);
+	if (ret)
+		dev_info(dev, "SMBUS alert not active on this bus\n");
+
+	return 0;
+
+error:
+	thunder_i2c_clock_disable(dev, i2c->clk);
+	return ret;
+}
+
+static void thunder_i2c_remove_pci(struct pci_dev *pdev)
+{
+	struct octeon_i2c *i2c = pci_get_drvdata(pdev);
+
+	thunder_i2c_smbus_remove(i2c);
+	thunder_i2c_clock_disable(&pdev->dev, i2c->clk);
+	i2c_del_adapter(&i2c->adap);
+}
+
+static const struct pci_device_id thunder_i2c_pci_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_TWSI) },
+	{ 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, thunder_i2c_pci_id_table);
+
+static struct pci_driver thunder_i2c_pci_driver = {
+	.name		= DRV_NAME,
+	.id_table	= thunder_i2c_pci_id_table,
+	.probe		= thunder_i2c_probe_pci,
+	.remove		= thunder_i2c_remove_pci,
+};
+
+module_pci_driver(thunder_i2c_pci_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Fred Martin <fmartin@caviumnetworks.com>");
+MODULE_DESCRIPTION("I2C-Bus adapter for Cavium ThunderX SOC");
diff --git a/drivers/i2c/busses/i2c-uniphier-f.c b/drivers/i2c/busses/i2c-uniphier-f.c
index aeead0d27d10..db9105e52c79 100644
--- a/drivers/i2c/busses/i2c-uniphier-f.c
+++ b/drivers/i2c/busses/i2c-uniphier-f.c
@@ -14,6 +14,7 @@
 
 #include <linux/clk.h>
 #include <linux/i2c.h>
+#include <linux/iopoll.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/module.h>
@@ -348,14 +349,19 @@ static int uniphier_fi2c_master_xfer_one(struct i2c_adapter *adap,
 	dev_dbg(&adap->dev, "complete\n");
 
 	if (unlikely(priv->flags & UNIPHIER_FI2C_DEFER_STOP_COMP)) {
-		u32 status = readl(priv->membase + UNIPHIER_FI2C_SR);
-
-		if (!(status & UNIPHIER_FI2C_SR_STS) ||
-		    status & UNIPHIER_FI2C_SR_BB) {
+		u32 status;
+		int ret;
+
+		ret = readl_poll_timeout(priv->membase + UNIPHIER_FI2C_SR,
+					 status,
+					 (status & UNIPHIER_FI2C_SR_STS) &&
+					 !(status & UNIPHIER_FI2C_SR_BB),
+					 1, 20);
+		if (ret) {
 			dev_err(&adap->dev,
 				"stop condition was not completed.\n");
 			uniphier_fi2c_recover(priv);
-			return -EBUSY;
+			return ret;
 		}
 	}
 
@@ -455,54 +461,25 @@ static struct i2c_bus_recovery_info uniphier_fi2c_bus_recovery_info = {
 	.unprepare_recovery = uniphier_fi2c_unprepare_recovery,
 };
 
-static int uniphier_fi2c_clk_init(struct device *dev,
-				  struct uniphier_fi2c_priv *priv)
+static void uniphier_fi2c_hw_init(struct uniphier_fi2c_priv *priv,
+				  u32 bus_speed, unsigned long clk_rate)
 {
-	struct device_node *np = dev->of_node;
-	unsigned long clk_rate;
-	u32 bus_speed, clk_count;
-	int ret;
-
-	if (of_property_read_u32(np, "clock-frequency", &bus_speed))
-		bus_speed = UNIPHIER_FI2C_DEFAULT_SPEED;
-
-	if (!bus_speed) {
-		dev_err(dev, "clock-frequency should not be zero\n");
-		return -EINVAL;
-	}
-
-	if (bus_speed > UNIPHIER_FI2C_MAX_SPEED)
-		bus_speed = UNIPHIER_FI2C_MAX_SPEED;
-
-	/* Get input clk rate through clk driver */
-	priv->clk = devm_clk_get(dev, NULL);
-	if (IS_ERR(priv->clk)) {
-		dev_err(dev, "failed to get clock\n");
-		return PTR_ERR(priv->clk);
-	}
+	u32 tmp;
 
-	ret = clk_prepare_enable(priv->clk);
-	if (ret)
-		return ret;
-
-	clk_rate = clk_get_rate(priv->clk);
-	if (!clk_rate) {
-		dev_err(dev, "input clock rate should not be zero\n");
-		return -EINVAL;
-	}
+	tmp = readl(priv->membase + UNIPHIER_FI2C_CR);
+	tmp |= UNIPHIER_FI2C_CR_MST;
+	writel(tmp, priv->membase + UNIPHIER_FI2C_CR);
 
 	uniphier_fi2c_reset(priv);
 
-	clk_count = clk_rate / bus_speed;
+	tmp = clk_rate / bus_speed;
 
-	writel(clk_count, priv->membase + UNIPHIER_FI2C_CYC);
-	writel(clk_count / 2, priv->membase + UNIPHIER_FI2C_LCTL);
-	writel(clk_count / 2, priv->membase + UNIPHIER_FI2C_SSUT);
-	writel(clk_count / 16, priv->membase + UNIPHIER_FI2C_DSUT);
+	writel(tmp, priv->membase + UNIPHIER_FI2C_CYC);
+	writel(tmp / 2, priv->membase + UNIPHIER_FI2C_LCTL);
+	writel(tmp / 2, priv->membase + UNIPHIER_FI2C_SSUT);
+	writel(tmp / 16, priv->membase + UNIPHIER_FI2C_DSUT);
 
 	uniphier_fi2c_prepare_operation(priv);
-
-	return 0;
 }
 
 static int uniphier_fi2c_probe(struct platform_device *pdev)
@@ -510,8 +487,9 @@ static int uniphier_fi2c_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct uniphier_fi2c_priv *priv;
 	struct resource *regs;
-	int irq;
-	int ret;
+	u32 bus_speed;
+	unsigned long clk_rate;
+	int irq, ret;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -528,6 +506,31 @@ static int uniphier_fi2c_probe(struct platform_device *pdev)
 		return irq;
 	}
 
+	if (of_property_read_u32(dev->of_node, "clock-frequency", &bus_speed))
+		bus_speed = UNIPHIER_FI2C_DEFAULT_SPEED;
+
+	if (!bus_speed || bus_speed > UNIPHIER_FI2C_MAX_SPEED) {
+		dev_err(dev, "invalid clock-frequency %d\n", bus_speed);
+		return -EINVAL;
+	}
+
+	priv->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		dev_err(dev, "failed to get clock\n");
+		return PTR_ERR(priv->clk);
+	}
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret)
+		return ret;
+
+	clk_rate = clk_get_rate(priv->clk);
+	if (!clk_rate) {
+		dev_err(dev, "input clock rate should not be zero\n");
+		ret = -EINVAL;
+		goto err;
+	}
+
 	init_completion(&priv->comp);
 	priv->adap.owner = THIS_MODULE;
 	priv->adap.algo = &uniphier_fi2c_algo;
@@ -538,9 +541,7 @@ static int uniphier_fi2c_probe(struct platform_device *pdev)
 	i2c_set_adapdata(&priv->adap, priv);
 	platform_set_drvdata(pdev, priv);
 
-	ret = uniphier_fi2c_clk_init(dev, priv);
-	if (ret)
-		goto err;
+	uniphier_fi2c_hw_init(priv, bus_speed, clk_rate);
 
 	ret = devm_request_irq(dev, irq, uniphier_fi2c_interrupt, 0,
 			       pdev->name, priv);
@@ -550,11 +551,6 @@ static int uniphier_fi2c_probe(struct platform_device *pdev)
 	}
 
 	ret = i2c_add_adapter(&priv->adap);
-	if (ret) {
-		dev_err(dev, "failed to add I2C adapter\n");
-		goto err;
-	}
-
 err:
 	if (ret)
 		clk_disable_unprepare(priv->clk);
diff --git a/drivers/i2c/busses/i2c-uniphier.c b/drivers/i2c/busses/i2c-uniphier.c
index 475a5eb514e2..56e92af46ddc 100644
--- a/drivers/i2c/busses/i2c-uniphier.c
+++ b/drivers/i2c/busses/i2c-uniphier.c
@@ -316,50 +316,15 @@ static struct i2c_bus_recovery_info uniphier_i2c_bus_recovery_info = {
 	.unprepare_recovery = uniphier_i2c_unprepare_recovery,
 };
 
-static int uniphier_i2c_clk_init(struct device *dev,
-				 struct uniphier_i2c_priv *priv)
+static void uniphier_i2c_hw_init(struct uniphier_i2c_priv *priv,
+				 u32 bus_speed, unsigned long clk_rate)
 {
-	struct device_node *np = dev->of_node;
-	unsigned long clk_rate;
-	u32 bus_speed;
-	int ret;
-
-	if (of_property_read_u32(np, "clock-frequency", &bus_speed))
-		bus_speed = UNIPHIER_I2C_DEFAULT_SPEED;
-
-	if (!bus_speed) {
-		dev_err(dev, "clock-frequency should not be zero\n");
-		return -EINVAL;
-	}
-
-	if (bus_speed > UNIPHIER_I2C_MAX_SPEED)
-		bus_speed = UNIPHIER_I2C_MAX_SPEED;
-
-	/* Get input clk rate through clk driver */
-	priv->clk = devm_clk_get(dev, NULL);
-	if (IS_ERR(priv->clk)) {
-		dev_err(dev, "failed to get clock\n");
-		return PTR_ERR(priv->clk);
-	}
-
-	ret = clk_prepare_enable(priv->clk);
-	if (ret)
-		return ret;
-
-	clk_rate = clk_get_rate(priv->clk);
-	if (!clk_rate) {
-		dev_err(dev, "input clock rate should not be zero\n");
-		return -EINVAL;
-	}
-
 	uniphier_i2c_reset(priv, true);
 
 	writel((clk_rate / bus_speed / 2 << 16) | (clk_rate / bus_speed),
 	       priv->membase + UNIPHIER_I2C_CLK);
 
 	uniphier_i2c_reset(priv, false);
-
-	return 0;
 }
 
 static int uniphier_i2c_probe(struct platform_device *pdev)
@@ -367,8 +332,9 @@ static int uniphier_i2c_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct uniphier_i2c_priv *priv;
 	struct resource *regs;
-	int irq;
-	int ret;
+	u32 bus_speed;
+	unsigned long clk_rate;
+	int irq, ret;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -385,6 +351,31 @@ static int uniphier_i2c_probe(struct platform_device *pdev)
 		return irq;
 	}
 
+	if (of_property_read_u32(dev->of_node, "clock-frequency", &bus_speed))
+		bus_speed = UNIPHIER_I2C_DEFAULT_SPEED;
+
+	if (!bus_speed || bus_speed > UNIPHIER_I2C_MAX_SPEED) {
+		dev_err(dev, "invalid clock-frequency %d\n", bus_speed);
+		return -EINVAL;
+	}
+
+	priv->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		dev_err(dev, "failed to get clock\n");
+		return PTR_ERR(priv->clk);
+	}
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret)
+		return ret;
+
+	clk_rate = clk_get_rate(priv->clk);
+	if (!clk_rate) {
+		dev_err(dev, "input clock rate should not be zero\n");
+		ret = -EINVAL;
+		goto err;
+	}
+
 	init_completion(&priv->comp);
 	priv->adap.owner = THIS_MODULE;
 	priv->adap.algo = &uniphier_i2c_algo;
@@ -395,9 +386,7 @@ static int uniphier_i2c_probe(struct platform_device *pdev)
 	i2c_set_adapdata(&priv->adap, priv);
 	platform_set_drvdata(pdev, priv);
 
-	ret = uniphier_i2c_clk_init(dev, priv);
-	if (ret)
-		goto err;
+	uniphier_i2c_hw_init(priv, bus_speed, clk_rate);
 
 	ret = devm_request_irq(dev, irq, uniphier_i2c_interrupt, 0, pdev->name,
 			       priv);
@@ -407,11 +396,6 @@ static int uniphier_i2c_probe(struct platform_device *pdev)
 	}
 
 	ret = i2c_add_adapter(&priv->adap);
-	if (ret) {
-		dev_err(dev, "failed to add I2C adapter\n");
-		goto err;
-	}
-
 err:
 	if (ret)
 		clk_disable_unprepare(priv->clk);
diff --git a/drivers/i2c/busses/i2c-wmt.c b/drivers/i2c/busses/i2c-wmt.c
index e1e3a85596c5..fbd0fd59f312 100644
--- a/drivers/i2c/busses/i2c-wmt.c
+++ b/drivers/i2c/busses/i2c-wmt.c
@@ -432,10 +432,8 @@ static int wmt_i2c_probe(struct platform_device *pdev)
 	}
 
 	err = i2c_add_adapter(adap);
-	if (err) {
-		dev_err(&pdev->dev, "failed to add adapter\n");
+	if (err)
 		return err;
-	}
 
 	platform_set_drvdata(pdev, i2c_dev);
 
diff --git a/drivers/i2c/busses/i2c-xgene-slimpro.c b/drivers/i2c/busses/i2c-xgene-slimpro.c
index 4233f5695352..263685c7a512 100644
--- a/drivers/i2c/busses/i2c-xgene-slimpro.c
+++ b/drivers/i2c/busses/i2c-xgene-slimpro.c
@@ -418,7 +418,6 @@ static int xgene_slimpro_i2c_probe(struct platform_device *pdev)
 	i2c_set_adapdata(adapter, ctx);
 	rc = i2c_add_adapter(adapter);
 	if (rc) {
-		dev_err(&pdev->dev, "Adapter registeration failed\n");
 		mbox_free_channel(ctx->mbox_chan);
 		return rc;
 	}
diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c
index 74f54f2f471f..66bce3b311a1 100644
--- a/drivers/i2c/busses/i2c-xiic.c
+++ b/drivers/i2c/busses/i2c-xiic.c
@@ -804,7 +804,6 @@ static int xiic_i2c_probe(struct platform_device *pdev)
 	/* add i2c adapter to i2c tree */
 	ret = i2c_add_adapter(&i2c->adap);
 	if (ret) {
-		dev_err(&pdev->dev, "Failed to add adapter\n");
 		xiic_deinit(i2c);
 		goto err_clk_dis;
 	}
diff --git a/drivers/i2c/busses/i2c-xlp9xx.c b/drivers/i2c/busses/i2c-xlp9xx.c
index 55a7bef1b2e1..2a972ed7aa0d 100644
--- a/drivers/i2c/busses/i2c-xlp9xx.c
+++ b/drivers/i2c/busses/i2c-xlp9xx.c
@@ -400,10 +400,8 @@ static int xlp9xx_i2c_probe(struct platform_device *pdev)
 	i2c_set_adapdata(&priv->adapter, priv);
 
 	err = i2c_add_adapter(&priv->adapter);
-	if (err) {
-		dev_err(&pdev->dev, "failed to add I2C adapter!\n");
+	if (err)
 		return err;
-	}
 
 	platform_set_drvdata(pdev, priv);
 	dev_dbg(&pdev->dev, "I2C bus:%d added\n", priv->adapter.nr);
diff --git a/drivers/i2c/busses/i2c-xlr.c b/drivers/i2c/busses/i2c-xlr.c
index 613c3a4f2c51..0968f59b6df5 100644
--- a/drivers/i2c/busses/i2c-xlr.c
+++ b/drivers/i2c/busses/i2c-xlr.c
@@ -432,10 +432,8 @@ static int xlr_i2c_probe(struct platform_device *pdev)
 
 	i2c_set_adapdata(&priv->adap, priv);
 	ret = i2c_add_numbered_adapter(&priv->adap);
-	if (ret < 0) {
-		dev_err(&priv->adap.dev, "Failed to add i2c bus.\n");
+	if (ret < 0)
 		return ret;
-	}
 
 	platform_set_drvdata(pdev, priv);
 	dev_info(&priv->adap.dev, "Added I2C Bus.\n");
diff --git a/drivers/i2c/i2c-core.c b/drivers/i2c/i2c-core.c
index da3a02ef4a31..98fffa3a09f7 100644
--- a/drivers/i2c/i2c-core.c
+++ b/drivers/i2c/i2c-core.c
@@ -88,7 +88,7 @@ void i2c_transfer_trace_unreg(void)
 }
 
 #if defined(CONFIG_ACPI)
-struct acpi_i2c_handler_data {
+struct i2c_acpi_handler_data {
 	struct acpi_connection_info info;
 	struct i2c_adapter *adapter;
 };
@@ -103,15 +103,18 @@ struct gsb_buffer {
 	};
 } __packed;
 
-struct acpi_i2c_lookup {
+struct i2c_acpi_lookup {
 	struct i2c_board_info *info;
 	acpi_handle adapter_handle;
 	acpi_handle device_handle;
+	acpi_handle search_handle;
+	u32 speed;
+	u32 min_speed;
 };
 
-static int acpi_i2c_fill_info(struct acpi_resource *ares, void *data)
+static int i2c_acpi_fill_info(struct acpi_resource *ares, void *data)
 {
-	struct acpi_i2c_lookup *lookup = data;
+	struct i2c_acpi_lookup *lookup = data;
 	struct i2c_board_info *info = lookup->info;
 	struct acpi_resource_i2c_serialbus *sb;
 	acpi_status status;
@@ -130,19 +133,18 @@ static int acpi_i2c_fill_info(struct acpi_resource *ares, void *data)
 		return 1;
 
 	info->addr = sb->slave_address;
+	lookup->speed = sb->connection_speed;
 	if (sb->access_mode == ACPI_I2C_10BIT_MODE)
 		info->flags |= I2C_CLIENT_TEN;
 
 	return 1;
 }
 
-static int acpi_i2c_get_info(struct acpi_device *adev,
-			     struct i2c_board_info *info,
-			     acpi_handle *adapter_handle)
+static int i2c_acpi_do_lookup(struct acpi_device *adev,
+			      struct i2c_acpi_lookup *lookup)
 {
+	struct i2c_board_info *info = lookup->info;
 	struct list_head resource_list;
-	struct resource_entry *entry;
-	struct acpi_i2c_lookup lookup;
 	int ret;
 
 	if (acpi_bus_get_status(adev) || !adev->status.present ||
@@ -150,24 +152,58 @@ static int acpi_i2c_get_info(struct acpi_device *adev,
 		return -EINVAL;
 
 	memset(info, 0, sizeof(*info));
-	info->fwnode = acpi_fwnode_handle(adev);
-
-	memset(&lookup, 0, sizeof(lookup));
-	lookup.device_handle = acpi_device_handle(adev);
-	lookup.info = info;
+	lookup->device_handle = acpi_device_handle(adev);
 
 	/* Look up for I2cSerialBus resource */
 	INIT_LIST_HEAD(&resource_list);
 	ret = acpi_dev_get_resources(adev, &resource_list,
-				     acpi_i2c_fill_info, &lookup);
+				     i2c_acpi_fill_info, lookup);
 	acpi_dev_free_resource_list(&resource_list);
 
 	if (ret < 0 || !info->addr)
 		return -EINVAL;
 
-	*adapter_handle = lookup.adapter_handle;
+	return 0;
+}
+
+static int i2c_acpi_get_info(struct acpi_device *adev,
+			     struct i2c_board_info *info,
+			     struct i2c_adapter *adapter,
+			     acpi_handle *adapter_handle)
+{
+	struct list_head resource_list;
+	struct resource_entry *entry;
+	struct i2c_acpi_lookup lookup;
+	int ret;
+
+	memset(&lookup, 0, sizeof(lookup));
+	lookup.info = info;
+
+	ret = i2c_acpi_do_lookup(adev, &lookup);
+	if (ret)
+		return ret;
+
+	if (adapter) {
+		/* The adapter must match the one in I2cSerialBus() connector */
+		if (ACPI_HANDLE(&adapter->dev) != lookup.adapter_handle)
+			return -ENODEV;
+	} else {
+		struct acpi_device *adapter_adev;
+
+		/* The adapter must be present */
+		if (acpi_bus_get_device(lookup.adapter_handle, &adapter_adev))
+			return -ENODEV;
+		if (acpi_bus_get_status(adapter_adev) ||
+		    !adapter_adev->status.present)
+			return -ENODEV;
+	}
+
+	info->fwnode = acpi_fwnode_handle(adev);
+	if (adapter_handle)
+		*adapter_handle = lookup.adapter_handle;
 
 	/* Then fill IRQ number if any */
+	INIT_LIST_HEAD(&resource_list);
 	ret = acpi_dev_get_resources(adev, &resource_list, NULL, NULL);
 	if (ret < 0)
 		return -EINVAL;
@@ -186,7 +222,7 @@ static int acpi_i2c_get_info(struct acpi_device *adev,
 	return 0;
 }
 
-static void acpi_i2c_register_device(struct i2c_adapter *adapter,
+static void i2c_acpi_register_device(struct i2c_adapter *adapter,
 				     struct acpi_device *adev,
 				     struct i2c_board_info *info)
 {
@@ -201,39 +237,35 @@ static void acpi_i2c_register_device(struct i2c_adapter *adapter,
 	}
 }
 
-static acpi_status acpi_i2c_add_device(acpi_handle handle, u32 level,
+static acpi_status i2c_acpi_add_device(acpi_handle handle, u32 level,
 				       void *data, void **return_value)
 {
 	struct i2c_adapter *adapter = data;
 	struct acpi_device *adev;
-	acpi_handle adapter_handle;
 	struct i2c_board_info info;
 
 	if (acpi_bus_get_device(handle, &adev))
 		return AE_OK;
 
-	if (acpi_i2c_get_info(adev, &info, &adapter_handle))
+	if (i2c_acpi_get_info(adev, &info, adapter, NULL))
 		return AE_OK;
 
-	if (adapter_handle != ACPI_HANDLE(&adapter->dev))
-		return AE_OK;
-
-	acpi_i2c_register_device(adapter, adev, &info);
+	i2c_acpi_register_device(adapter, adev, &info);
 
 	return AE_OK;
 }
 
-#define ACPI_I2C_MAX_SCAN_DEPTH 32
+#define I2C_ACPI_MAX_SCAN_DEPTH 32
 
 /**
- * acpi_i2c_register_devices - enumerate I2C slave devices behind adapter
+ * i2c_acpi_register_devices - enumerate I2C slave devices behind adapter
  * @adap: pointer to adapter
  *
  * Enumerate all I2C slave devices behind this adapter by walking the ACPI
  * namespace. When a device is found it will be added to the Linux device
  * model and bound to the corresponding ACPI handle.
  */
-static void acpi_i2c_register_devices(struct i2c_adapter *adap)
+static void i2c_acpi_register_devices(struct i2c_adapter *adap)
 {
 	acpi_status status;
 
@@ -241,14 +273,72 @@ static void acpi_i2c_register_devices(struct i2c_adapter *adap)
 		return;
 
 	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
-				     ACPI_I2C_MAX_SCAN_DEPTH,
-				     acpi_i2c_add_device, NULL,
+				     I2C_ACPI_MAX_SCAN_DEPTH,
+				     i2c_acpi_add_device, NULL,
 				     adap, NULL);
 	if (ACPI_FAILURE(status))
 		dev_warn(&adap->dev, "failed to enumerate I2C slaves\n");
 }
 
-static int acpi_i2c_match_adapter(struct device *dev, void *data)
+static acpi_status i2c_acpi_lookup_speed(acpi_handle handle, u32 level,
+					   void *data, void **return_value)
+{
+	struct i2c_acpi_lookup *lookup = data;
+	struct acpi_device *adev;
+
+	if (acpi_bus_get_device(handle, &adev))
+		return AE_OK;
+
+	if (i2c_acpi_do_lookup(adev, lookup))
+		return AE_OK;
+
+	if (lookup->search_handle != lookup->adapter_handle)
+		return AE_OK;
+
+	if (lookup->speed <= lookup->min_speed)
+		lookup->min_speed = lookup->speed;
+
+	return AE_OK;
+}
+
+/**
+ * i2c_acpi_find_bus_speed - find I2C bus speed from ACPI
+ * @dev: The device owning the bus
+ *
+ * Find the I2C bus speed by walking the ACPI namespace for all I2C slaves
+ * devices connected to this bus and use the speed of slowest device.
+ *
+ * Returns the speed in Hz or zero
+ */
+u32 i2c_acpi_find_bus_speed(struct device *dev)
+{
+	struct i2c_acpi_lookup lookup;
+	struct i2c_board_info dummy;
+	acpi_status status;
+
+	if (!has_acpi_companion(dev))
+		return 0;
+
+	memset(&lookup, 0, sizeof(lookup));
+	lookup.search_handle = ACPI_HANDLE(dev);
+	lookup.min_speed = UINT_MAX;
+	lookup.info = &dummy;
+
+	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
+				     I2C_ACPI_MAX_SCAN_DEPTH,
+				     i2c_acpi_lookup_speed, NULL,
+				     &lookup, NULL);
+
+	if (ACPI_FAILURE(status)) {
+		dev_warn(dev, "unable to find I2C bus speed from ACPI\n");
+		return 0;
+	}
+
+	return lookup.min_speed != UINT_MAX ? lookup.min_speed : 0;
+}
+EXPORT_SYMBOL_GPL(i2c_acpi_find_bus_speed);
+
+static int i2c_acpi_match_adapter(struct device *dev, void *data)
 {
 	struct i2c_adapter *adapter = i2c_verify_adapter(dev);
 
@@ -258,29 +348,29 @@ static int acpi_i2c_match_adapter(struct device *dev, void *data)
 	return ACPI_HANDLE(dev) == (acpi_handle)data;
 }
 
-static int acpi_i2c_match_device(struct device *dev, void *data)
+static int i2c_acpi_match_device(struct device *dev, void *data)
 {
 	return ACPI_COMPANION(dev) == data;
 }
 
-static struct i2c_adapter *acpi_i2c_find_adapter_by_handle(acpi_handle handle)
+static struct i2c_adapter *i2c_acpi_find_adapter_by_handle(acpi_handle handle)
 {
 	struct device *dev;
 
 	dev = bus_find_device(&i2c_bus_type, NULL, handle,
-			      acpi_i2c_match_adapter);
+			      i2c_acpi_match_adapter);
 	return dev ? i2c_verify_adapter(dev) : NULL;
 }
 
-static struct i2c_client *acpi_i2c_find_client_by_adev(struct acpi_device *adev)
+static struct i2c_client *i2c_acpi_find_client_by_adev(struct acpi_device *adev)
 {
 	struct device *dev;
 
-	dev = bus_find_device(&i2c_bus_type, NULL, adev, acpi_i2c_match_device);
+	dev = bus_find_device(&i2c_bus_type, NULL, adev, i2c_acpi_match_device);
 	return dev ? i2c_verify_client(dev) : NULL;
 }
 
-static int acpi_i2c_notify(struct notifier_block *nb, unsigned long value,
+static int i2c_acpi_notify(struct notifier_block *nb, unsigned long value,
 			   void *arg)
 {
 	struct acpi_device *adev = arg;
@@ -291,20 +381,20 @@ static int acpi_i2c_notify(struct notifier_block *nb, unsigned long value,
 
 	switch (value) {
 	case ACPI_RECONFIG_DEVICE_ADD:
-		if (acpi_i2c_get_info(adev, &info, &adapter_handle))
+		if (i2c_acpi_get_info(adev, &info, NULL, &adapter_handle))
 			break;
 
-		adapter = acpi_i2c_find_adapter_by_handle(adapter_handle);
+		adapter = i2c_acpi_find_adapter_by_handle(adapter_handle);
 		if (!adapter)
 			break;
 
-		acpi_i2c_register_device(adapter, adev, &info);
+		i2c_acpi_register_device(adapter, adev, &info);
 		break;
 	case ACPI_RECONFIG_DEVICE_REMOVE:
 		if (!acpi_device_enumerated(adev))
 			break;
 
-		client = acpi_i2c_find_client_by_adev(adev);
+		client = i2c_acpi_find_client_by_adev(adev);
 		if (!client)
 			break;
 
@@ -317,10 +407,10 @@ static int acpi_i2c_notify(struct notifier_block *nb, unsigned long value,
 }
 
 static struct notifier_block i2c_acpi_notifier = {
-	.notifier_call = acpi_i2c_notify,
+	.notifier_call = i2c_acpi_notify,
 };
 #else /* CONFIG_ACPI */
-static inline void acpi_i2c_register_devices(struct i2c_adapter *adap) { }
+static inline void i2c_acpi_register_devices(struct i2c_adapter *adap) { }
 extern struct notifier_block i2c_acpi_notifier;
 #endif /* CONFIG_ACPI */
 
@@ -386,12 +476,12 @@ static int acpi_gsb_i2c_write_bytes(struct i2c_client *client,
 }
 
 static acpi_status
-acpi_i2c_space_handler(u32 function, acpi_physical_address command,
+i2c_acpi_space_handler(u32 function, acpi_physical_address command,
 			u32 bits, u64 *value64,
 			void *handler_context, void *region_context)
 {
 	struct gsb_buffer *gsb = (struct gsb_buffer *)value64;
-	struct acpi_i2c_handler_data *data = handler_context;
+	struct i2c_acpi_handler_data *data = handler_context;
 	struct acpi_connection_info *info = &data->info;
 	struct acpi_resource_i2c_serialbus *sb;
 	struct i2c_adapter *adapter = data->adapter;
@@ -510,10 +600,10 @@ acpi_i2c_space_handler(u32 function, acpi_physical_address command,
 }
 
 
-static int acpi_i2c_install_space_handler(struct i2c_adapter *adapter)
+static int i2c_acpi_install_space_handler(struct i2c_adapter *adapter)
 {
 	acpi_handle handle;
-	struct acpi_i2c_handler_data *data;
+	struct i2c_acpi_handler_data *data;
 	acpi_status status;
 
 	if (!adapter->dev.parent)
@@ -524,7 +614,7 @@ static int acpi_i2c_install_space_handler(struct i2c_adapter *adapter)
 	if (!handle)
 		return -ENODEV;
 
-	data = kzalloc(sizeof(struct acpi_i2c_handler_data),
+	data = kzalloc(sizeof(struct i2c_acpi_handler_data),
 			    GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
@@ -538,7 +628,7 @@ static int acpi_i2c_install_space_handler(struct i2c_adapter *adapter)
 
 	status = acpi_install_address_space_handler(handle,
 				ACPI_ADR_SPACE_GSBUS,
-				&acpi_i2c_space_handler,
+				&i2c_acpi_space_handler,
 				NULL,
 				data);
 	if (ACPI_FAILURE(status)) {
@@ -552,10 +642,10 @@ static int acpi_i2c_install_space_handler(struct i2c_adapter *adapter)
 	return 0;
 }
 
-static void acpi_i2c_remove_space_handler(struct i2c_adapter *adapter)
+static void i2c_acpi_remove_space_handler(struct i2c_adapter *adapter)
 {
 	acpi_handle handle;
-	struct acpi_i2c_handler_data *data;
+	struct i2c_acpi_handler_data *data;
 	acpi_status status;
 
 	if (!adapter->dev.parent)
@@ -568,7 +658,7 @@ static void acpi_i2c_remove_space_handler(struct i2c_adapter *adapter)
 
 	acpi_remove_address_space_handler(handle,
 				ACPI_ADR_SPACE_GSBUS,
-				&acpi_i2c_space_handler);
+				&i2c_acpi_space_handler);
 
 	status = acpi_bus_get_private_data(handle, (void **)&data);
 	if (ACPI_SUCCESS(status))
@@ -577,10 +667,10 @@ static void acpi_i2c_remove_space_handler(struct i2c_adapter *adapter)
 	acpi_bus_detach_private_data(handle);
 }
 #else /* CONFIG_ACPI_I2C_OPREGION */
-static inline void acpi_i2c_remove_space_handler(struct i2c_adapter *adapter)
+static inline void i2c_acpi_remove_space_handler(struct i2c_adapter *adapter)
 { }
 
-static inline int acpi_i2c_install_space_handler(struct i2c_adapter *adapter)
+static inline int i2c_acpi_install_space_handler(struct i2c_adapter *adapter)
 { return 0; }
 #endif /* CONFIG_ACPI_I2C_OPREGION */
 
@@ -853,7 +943,7 @@ static int i2c_device_probe(struct device *dev)
 			status = 0;
 
 		if (status)
-			dev_warn(&client->dev, "failed to set up wakeup irq");
+			dev_warn(&client->dev, "failed to set up wakeup irq\n");
 	}
 
 	dev_dbg(dev, "probe\n");
@@ -1208,8 +1298,9 @@ i2c_new_device(struct i2c_adapter *adap, struct i2c_board_info const *info)
 	return client;
 
 out_err:
-	dev_err(&adap->dev, "Failed to register i2c client %s at 0x%02x "
-		"(%d)\n", client->name, client->addr, status);
+	dev_err(&adap->dev,
+		"Failed to register i2c client %s at 0x%02x (%d)\n",
+		client->name, client->addr, status);
 out_err_silent:
 	kfree(client);
 	return NULL;
@@ -1335,21 +1426,19 @@ static void i2c_adapter_dev_release(struct device *dev)
 	complete(&adap->dev_released);
 }
 
-/*
- * This function is only needed for mutex_lock_nested, so it is never
- * called unless locking correctness checking is enabled. Thus we
- * make it inline to avoid a compiler warning. That's what gcc ends up
- * doing anyway.
- */
-static inline unsigned int i2c_adapter_depth(struct i2c_adapter *adapter)
+unsigned int i2c_adapter_depth(struct i2c_adapter *adapter)
 {
 	unsigned int depth = 0;
 
 	while ((adapter = i2c_parent_is_i2c_adapter(adapter)))
 		depth++;
 
+	WARN_ONCE(depth >= MAX_LOCKDEP_SUBCLASSES,
+		  "adapter depth exceeds lockdep subclass limit\n");
+
 	return depth;
 }
+EXPORT_SYMBOL_GPL(i2c_adapter_depth);
 
 /*
  * Let users instantiate I2C devices through sysfs. This can be used when
@@ -1678,8 +1767,8 @@ static int i2c_do_add_adapter(struct i2c_driver *driver,
 	if (driver->attach_adapter) {
 		dev_warn(&adap->dev, "%s: attach_adapter method is deprecated\n",
 			 driver->driver.name);
-		dev_warn(&adap->dev, "Please use another way to instantiate "
-			 "your i2c_client\n");
+		dev_warn(&adap->dev,
+			 "Please use another way to instantiate your i2c_client\n");
 		/* We ignore the return code; if it fails, too bad */
 		driver->attach_adapter(adap);
 	}
@@ -1691,6 +1780,12 @@ static int __process_new_adapter(struct device_driver *d, void *data)
 	return i2c_do_add_adapter(to_i2c_driver(d), data);
 }
 
+static const struct i2c_lock_operations i2c_adapter_lock_ops = {
+	.lock_bus =    i2c_adapter_lock_bus,
+	.trylock_bus = i2c_adapter_trylock_bus,
+	.unlock_bus =  i2c_adapter_unlock_bus,
+};
+
 static int i2c_register_adapter(struct i2c_adapter *adap)
 {
 	int res = -EINVAL;
@@ -1710,11 +1805,8 @@ static int i2c_register_adapter(struct i2c_adapter *adap)
 		goto out_list;
 	}
 
-	if (!adap->lock_bus) {
-		adap->lock_bus = i2c_adapter_lock_bus;
-		adap->trylock_bus = i2c_adapter_trylock_bus;
-		adap->unlock_bus = i2c_adapter_unlock_bus;
-	}
+	if (!adap->lock_ops)
+		adap->lock_ops = &i2c_adapter_lock_ops;
 
 	rt_mutex_init(&adap->bus_lock);
 	rt_mutex_init(&adap->mux_lock);
@@ -1752,8 +1844,8 @@ static int i2c_register_adapter(struct i2c_adapter *adap)
 
 	/* create pre-declared device nodes */
 	of_i2c_register_devices(adap);
-	acpi_i2c_register_devices(adap);
-	acpi_i2c_install_space_handler(adap);
+	i2c_acpi_register_devices(adap);
+	i2c_acpi_install_space_handler(adap);
 
 	if (adap->nr < __i2c_first_dynamic_bus_num)
 		i2c_scan_static_board_info(adap);
@@ -1925,7 +2017,7 @@ void i2c_del_adapter(struct i2c_adapter *adap)
 		return;
 	}
 
-	acpi_i2c_remove_space_handler(adap);
+	i2c_acpi_remove_space_handler(adap);
 	/* Tell drivers about this removal */
 	mutex_lock(&core_lock);
 	bus_for_each_drv(&i2c_bus_type, NULL, adap,
@@ -2451,15 +2543,16 @@ int i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
 	if (adap->algo->master_xfer) {
 #ifdef DEBUG
 		for (ret = 0; ret < num; ret++) {
-			dev_dbg(&adap->dev, "master_xfer[%d] %c, addr=0x%02x, "
-				"len=%d%s\n", ret, (msgs[ret].flags & I2C_M_RD)
-				? 'R' : 'W', msgs[ret].addr, msgs[ret].len,
+			dev_dbg(&adap->dev,
+				"master_xfer[%d] %c, addr=0x%02x, len=%d%s\n",
+				ret, (msgs[ret].flags & I2C_M_RD) ? 'R' : 'W',
+				msgs[ret].addr, msgs[ret].len,
 				(msgs[ret].flags & I2C_M_RECV_LEN) ? "+" : "");
 		}
 #endif
 
 		if (in_atomic() || irqs_disabled()) {
-			ret = adap->trylock_bus(adap, I2C_LOCK_SEGMENT);
+			ret = i2c_trylock_bus(adap, I2C_LOCK_SEGMENT);
 			if (!ret)
 				/* I2C activity is ongoing. */
 				return -EAGAIN;
@@ -2619,9 +2712,9 @@ static int i2c_detect_address(struct i2c_client *temp_client,
 
 	/* Consistency check */
 	if (info.type[0] == '\0') {
-		dev_err(&adapter->dev, "%s detection function provided "
-			"no name for 0x%x\n", driver->driver.name,
-			addr);
+		dev_err(&adapter->dev,
+			"%s detection function provided no name for 0x%x\n",
+			driver->driver.name, addr);
 	} else {
 		struct i2c_client *client;
 
@@ -2659,9 +2752,8 @@ static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver)
 	/* Warn that the adapter lost class based instantiation */
 	if (adapter->class == I2C_CLASS_DEPRECATED) {
 		dev_dbg(&adapter->dev,
-			"This adapter dropped support for I2C classes and "
-			"won't auto-detect %s devices anymore. If you need it, check "
-			"'Documentation/i2c/instantiating-devices' for alternatives.\n",
+			"This adapter dropped support for I2C classes and won't auto-detect %s devices anymore. "
+			"If you need it, check 'Documentation/i2c/instantiating-devices' for alternatives.\n",
 			driver->driver.name);
 		return 0;
 	}
@@ -2677,8 +2769,9 @@ static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver)
 	temp_client->adapter = adapter;
 
 	for (i = 0; address_list[i] != I2C_CLIENT_END; i += 1) {
-		dev_dbg(&adapter->dev, "found normal entry for adapter %d, "
-			"addr 0x%02x\n", adap_id, address_list[i]);
+		dev_dbg(&adapter->dev,
+			"found normal entry for adapter %d, addr 0x%02x\n",
+			adap_id, address_list[i]);
 		temp_client->addr = address_list[i];
 		err = i2c_detect_address(temp_client, driver);
 		if (unlikely(err))
@@ -2710,15 +2803,16 @@ i2c_new_probed_device(struct i2c_adapter *adap,
 	for (i = 0; addr_list[i] != I2C_CLIENT_END; i++) {
 		/* Check address validity */
 		if (i2c_check_7bit_addr_validity_strict(addr_list[i]) < 0) {
-			dev_warn(&adap->dev, "Invalid 7-bit address "
-				 "0x%02x\n", addr_list[i]);
+			dev_warn(&adap->dev, "Invalid 7-bit address 0x%02x\n",
+				 addr_list[i]);
 			continue;
 		}
 
 		/* Check address availability (7 bit, no need to encode flags) */
 		if (i2c_check_addr_busy(adap, addr_list[i])) {
-			dev_dbg(&adap->dev, "Address 0x%02x already in "
-				"use, not probing\n", addr_list[i]);
+			dev_dbg(&adap->dev,
+				"Address 0x%02x already in use, not probing\n",
+				addr_list[i]);
 			continue;
 		}
 
diff --git a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c
index 8eee98634cda..83768e85a919 100644
--- a/drivers/i2c/i2c-mux.c
+++ b/drivers/i2c/i2c-mux.c
@@ -159,7 +159,7 @@ static int i2c_mux_trylock_bus(struct i2c_adapter *adapter, unsigned int flags)
 		return 0;	/* mux_lock not locked, failure */
 	if (!(flags & I2C_LOCK_ROOT_ADAPTER))
 		return 1;	/* we only want mux_lock, success */
-	if (parent->trylock_bus(parent, flags))
+	if (i2c_trylock_bus(parent, flags))
 		return 1;	/* parent locked too, success */
 	rt_mutex_unlock(&parent->mux_lock);
 	return 0;		/* parent not locked, failure */
@@ -193,7 +193,7 @@ static int i2c_parent_trylock_bus(struct i2c_adapter *adapter,
 
 	if (!rt_mutex_trylock(&parent->mux_lock))
 		return 0;	/* mux_lock not locked, failure */
-	if (parent->trylock_bus(parent, flags))
+	if (i2c_trylock_bus(parent, flags))
 		return 1;	/* parent locked too, success */
 	rt_mutex_unlock(&parent->mux_lock);
 	return 0;		/* parent not locked, failure */
@@ -255,6 +255,10 @@ struct i2c_mux_core *i2c_mux_alloc(struct i2c_adapter *parent,
 	muxc->dev = dev;
 	if (flags & I2C_MUX_LOCKED)
 		muxc->mux_locked = true;
+	if (flags & I2C_MUX_ARBITRATOR)
+		muxc->arbitrator = true;
+	if (flags & I2C_MUX_GATE)
+		muxc->gate = true;
 	muxc->select = select;
 	muxc->deselect = deselect;
 	muxc->max_adapters = max_adapters;
@@ -263,6 +267,18 @@ struct i2c_mux_core *i2c_mux_alloc(struct i2c_adapter *parent,
 }
 EXPORT_SYMBOL_GPL(i2c_mux_alloc);
 
+static const struct i2c_lock_operations i2c_mux_lock_ops = {
+	.lock_bus =    i2c_mux_lock_bus,
+	.trylock_bus = i2c_mux_trylock_bus,
+	.unlock_bus =  i2c_mux_unlock_bus,
+};
+
+static const struct i2c_lock_operations i2c_parent_lock_ops = {
+	.lock_bus =    i2c_parent_lock_bus,
+	.trylock_bus = i2c_parent_trylock_bus,
+	.unlock_bus =  i2c_parent_unlock_bus,
+};
+
 int i2c_mux_add_adapter(struct i2c_mux_core *muxc,
 			u32 force_nr, u32 chan_id,
 			unsigned int class)
@@ -312,15 +328,10 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc,
 	priv->adap.retries = parent->retries;
 	priv->adap.timeout = parent->timeout;
 	priv->adap.quirks = parent->quirks;
-	if (muxc->mux_locked) {
-		priv->adap.lock_bus = i2c_mux_lock_bus;
-		priv->adap.trylock_bus = i2c_mux_trylock_bus;
-		priv->adap.unlock_bus = i2c_mux_unlock_bus;
-	} else {
-		priv->adap.lock_bus = i2c_parent_lock_bus;
-		priv->adap.trylock_bus = i2c_parent_trylock_bus;
-		priv->adap.unlock_bus = i2c_parent_unlock_bus;
-	}
+	if (muxc->mux_locked)
+		priv->adap.lock_ops = &i2c_mux_lock_ops;
+	else
+		priv->adap.lock_ops = &i2c_parent_lock_ops;
 
 	/* Sanity check on class */
 	if (i2c_mux_parent_classes(parent) & class)
@@ -335,18 +346,42 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc,
 	 * nothing if !CONFIG_OF.
 	 */
 	if (muxc->dev->of_node) {
-		struct device_node *child;
+		struct device_node *dev_node = muxc->dev->of_node;
+		struct device_node *mux_node, *child = NULL;
 		u32 reg;
 
-		for_each_child_of_node(muxc->dev->of_node, child) {
-			ret = of_property_read_u32(child, "reg", &reg);
-			if (ret)
-				continue;
-			if (chan_id == reg) {
-				priv->adap.dev.of_node = child;
-				break;
+		if (muxc->arbitrator)
+			mux_node = of_get_child_by_name(dev_node, "i2c-arb");
+		else if (muxc->gate)
+			mux_node = of_get_child_by_name(dev_node, "i2c-gate");
+		else
+			mux_node = of_get_child_by_name(dev_node, "i2c-mux");
+
+		if (mux_node) {
+			/* A "reg" property indicates an old-style DT entry */
+			if (!of_property_read_u32(mux_node, "reg", &reg)) {
+				of_node_put(mux_node);
+				mux_node = NULL;
+			}
+		}
+
+		if (!mux_node)
+			mux_node = of_node_get(dev_node);
+		else if (muxc->arbitrator || muxc->gate)
+			child = of_node_get(mux_node);
+
+		if (!child) {
+			for_each_child_of_node(mux_node, child) {
+				ret = of_property_read_u32(child, "reg", &reg);
+				if (ret)
+					continue;
+				if (chan_id == reg)
+					break;
 			}
 		}
+
+		priv->adap.dev.of_node = child;
+		of_node_put(mux_node);
 	}
 
 	/*
diff --git a/drivers/i2c/muxes/i2c-arb-gpio-challenge.c b/drivers/i2c/muxes/i2c-arb-gpio-challenge.c
index a90bbc4037dd..86fc2d4c081b 100644
--- a/drivers/i2c/muxes/i2c-arb-gpio-challenge.c
+++ b/drivers/i2c/muxes/i2c-arb-gpio-challenge.c
@@ -130,7 +130,7 @@ static int i2c_arbitrator_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	muxc = i2c_mux_alloc(NULL, dev, 1, sizeof(*arb), 0,
+	muxc = i2c_mux_alloc(NULL, dev, 1, sizeof(*arb), I2C_MUX_ARBITRATOR,
 			     i2c_arbitrator_select, i2c_arbitrator_deselect);
 	if (!muxc)
 		return -ENOMEM;
diff --git a/drivers/i2c/muxes/i2c-demux-pinctrl.c b/drivers/i2c/muxes/i2c-demux-pinctrl.c
index 8de073aed001..b3893f6282ba 100644
--- a/drivers/i2c/muxes/i2c-demux-pinctrl.c
+++ b/drivers/i2c/muxes/i2c-demux-pinctrl.c
@@ -37,8 +37,6 @@ struct i2c_demux_pinctrl_priv {
 	struct i2c_demux_pinctrl_chan chan[];
 };
 
-static struct property status_okay = { .name = "status", .length = 3, .value = "ok" };
-
 static int i2c_demux_master_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 {
 	struct i2c_demux_pinctrl_priv *priv = adap->algo_data;
@@ -68,7 +66,7 @@ static int i2c_demux_activate_master(struct i2c_demux_pinctrl_priv *priv, u32 ne
 	adap = of_find_i2c_adapter_by_node(priv->chan[new_chan].parent_np);
 	if (!adap) {
 		ret = -ENODEV;
-		goto err;
+		goto err_with_revert;
 	}
 
 	p = devm_pinctrl_get_select(adap->dev.parent, priv->bus_name);
@@ -103,8 +101,11 @@ static int i2c_demux_activate_master(struct i2c_demux_pinctrl_priv *priv, u32 ne
 
  err_with_put:
 	i2c_put_adapter(adap);
+ err_with_revert:
+	of_changeset_revert(&priv->chan[new_chan].chgset);
  err:
 	dev_err(priv->dev, "failed to setup demux-adapter %d (%d)\n", new_chan, ret);
+	priv->cur_chan = -EINVAL;
 	return ret;
 }
 
@@ -190,6 +191,7 @@ static int i2c_demux_pinctrl_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	struct i2c_demux_pinctrl_priv *priv;
+	struct property *props;
 	int num_chan, i, j, err;
 
 	num_chan = of_count_phandle_with_args(np, "i2c-parent", NULL);
@@ -200,7 +202,10 @@ static int i2c_demux_pinctrl_probe(struct platform_device *pdev)
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv)
 			   + num_chan * sizeof(struct i2c_demux_pinctrl_chan), GFP_KERNEL);
-	if (!priv)
+
+	props = devm_kcalloc(&pdev->dev, num_chan, sizeof(*props), GFP_KERNEL);
+
+	if (!priv || !props)
 		return -ENOMEM;
 
 	err = of_property_read_string(np, "i2c-bus-name", &priv->bus_name);
@@ -218,8 +223,12 @@ static int i2c_demux_pinctrl_probe(struct platform_device *pdev)
 		}
 		priv->chan[i].parent_np = adap_np;
 
+		props[i].name = devm_kstrdup(&pdev->dev, "status", GFP_KERNEL);
+		props[i].value = devm_kstrdup(&pdev->dev, "ok", GFP_KERNEL);
+		props[i].length = 3;
+
 		of_changeset_init(&priv->chan[i].chgset);
-		of_changeset_update_property(&priv->chan[i].chgset, adap_np, &status_okay);
+		of_changeset_update_property(&priv->chan[i].chgset, adap_np, &props[i]);
 	}
 
 	priv->num_chan = num_chan;
diff --git a/drivers/i2c/muxes/i2c-mux-pca9541.c b/drivers/i2c/muxes/i2c-mux-pca9541.c
index 3cb8af635db5..4ea7e691afc7 100644
--- a/drivers/i2c/muxes/i2c-mux-pca9541.c
+++ b/drivers/i2c/muxes/i2c-mux-pca9541.c
@@ -85,6 +85,13 @@ static const struct i2c_device_id pca9541_id[] = {
 
 MODULE_DEVICE_TABLE(i2c, pca9541_id);
 
+#ifdef CONFIG_OF
+static const struct of_device_id pca9541_of_match[] = {
+	{ .compatible = "nxp,pca9541" },
+	{}
+};
+#endif
+
 /*
  * Write to chip register. Don't use i2c_transfer()/i2c_smbus_xfer()
  * as they will try to lock the adapter a second time.
@@ -349,7 +356,8 @@ static int pca9541_probe(struct i2c_client *client,
 	force = 0;
 	if (pdata)
 		force = pdata->modes[0].adap_id;
-	muxc = i2c_mux_alloc(adap, &client->dev, 1, sizeof(*data), 0,
+	muxc = i2c_mux_alloc(adap, &client->dev, 1, sizeof(*data),
+			     I2C_MUX_ARBITRATOR,
 			     pca9541_select_chan, pca9541_release_chan);
 	if (!muxc)
 		return -ENOMEM;
@@ -382,6 +390,7 @@ static int pca9541_remove(struct i2c_client *client)
 static struct i2c_driver pca9541_driver = {
 	.driver = {
 		   .name = "pca9541",
+		   .of_match_table = of_match_ptr(pca9541_of_match),
 		   },
 	.probe = pca9541_probe,
 	.remove = pca9541_remove,
diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c
index 528e755c468f..1091346f2480 100644
--- a/drivers/i2c/muxes/i2c-mux-pca954x.c
+++ b/drivers/i2c/muxes/i2c-mux-pca954x.c
@@ -42,6 +42,7 @@
 #include <linux/i2c/pca954x.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/pm.h>
 #include <linux/slab.h>
 
@@ -58,14 +59,6 @@ enum pca_type {
 	pca_9548,
 };
 
-struct pca954x {
-	enum pca_type type;
-
-	u8 last_chan;		/* last register value */
-	u8 deselect;
-	struct i2c_client *client;
-};
-
 struct chip_desc {
 	u8 nchans;
 	u8 enable;	/* used for muxes only */
@@ -75,6 +68,14 @@ struct chip_desc {
 	} muxtype;
 };
 
+struct pca954x {
+	const struct chip_desc *chip;
+
+	u8 last_chan;		/* last register value */
+	u8 deselect;
+	struct i2c_client *client;
+};
+
 /* Provide specs for the PCA954x types we know about */
 static const struct chip_desc chips[] = {
 	[pca_9540] = {
@@ -119,6 +120,20 @@ static const struct i2c_device_id pca954x_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, pca954x_id);
 
+#ifdef CONFIG_OF
+static const struct of_device_id pca954x_of_match[] = {
+	{ .compatible = "nxp,pca9540", .data = &chips[pca_9540] },
+	{ .compatible = "nxp,pca9542", .data = &chips[pca_9542] },
+	{ .compatible = "nxp,pca9543", .data = &chips[pca_9543] },
+	{ .compatible = "nxp,pca9544", .data = &chips[pca_9544] },
+	{ .compatible = "nxp,pca9545", .data = &chips[pca_9545] },
+	{ .compatible = "nxp,pca9546", .data = &chips[pca_9546] },
+	{ .compatible = "nxp,pca9547", .data = &chips[pca_9547] },
+	{ .compatible = "nxp,pca9548", .data = &chips[pca_9548] },
+	{}
+};
+#endif
+
 /* Write to mux register. Don't use i2c_transfer()/i2c_smbus_xfer()
    for this as they will try to lock adapter a second time */
 static int pca954x_reg_write(struct i2c_adapter *adap,
@@ -151,7 +166,7 @@ static int pca954x_select_chan(struct i2c_mux_core *muxc, u32 chan)
 {
 	struct pca954x *data = i2c_mux_priv(muxc);
 	struct i2c_client *client = data->client;
-	const struct chip_desc *chip = &chips[data->type];
+	const struct chip_desc *chip = data->chip;
 	u8 regval;
 	int ret = 0;
 
@@ -164,7 +179,7 @@ static int pca954x_select_chan(struct i2c_mux_core *muxc, u32 chan)
 	/* Only select the channel if its different from the last channel */
 	if (data->last_chan != regval) {
 		ret = pca954x_reg_write(muxc->parent, client, regval);
-		data->last_chan = regval;
+		data->last_chan = ret ? 0 : regval;
 	}
 
 	return ret;
@@ -197,6 +212,7 @@ static int pca954x_probe(struct i2c_client *client,
 	int num, force, class;
 	struct i2c_mux_core *muxc;
 	struct pca954x *data;
+	const struct of_device_id *match;
 	int ret;
 
 	if (!i2c_check_functionality(adap, I2C_FUNC_SMBUS_BYTE))
@@ -226,14 +242,19 @@ static int pca954x_probe(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	data->type = id->driver_data;
+	match = of_match_device(of_match_ptr(pca954x_of_match), &client->dev);
+	if (match)
+		data->chip = of_device_get_match_data(&client->dev);
+	else
+		data->chip = &chips[id->driver_data];
+
 	data->last_chan = 0;		   /* force the first selection */
 
 	idle_disconnect_dt = of_node &&
 		of_property_read_bool(of_node, "i2c-mux-idle-disconnect");
 
 	/* Now create an adapter for each channel */
-	for (num = 0; num < chips[data->type].nchans; num++) {
+	for (num = 0; num < data->chip->nchans; num++) {
 		bool idle_disconnect_pd = false;
 
 		force = 0;			  /* dynamic adap number */
@@ -263,7 +284,7 @@ static int pca954x_probe(struct i2c_client *client,
 
 	dev_info(&client->dev,
 		 "registered %d multiplexed busses for I2C %s %s\n",
-		 num, chips[data->type].muxtype == pca954x_ismux
+		 num, data->chip->muxtype == pca954x_ismux
 				? "mux" : "switch", client->name);
 
 	return 0;
@@ -299,6 +320,7 @@ static struct i2c_driver pca954x_driver = {
 	.driver		= {
 		.name	= "pca954x",
 		.pm	= &pca954x_pm,
+		.of_match_table = of_match_ptr(pca954x_of_match),
 	},
 	.probe		= pca954x_probe,
 	.remove		= pca954x_remove,
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index 67ec58f9ef99..4466a2f969d7 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -863,8 +863,8 @@ static struct cpuidle_state dnv_cstates[] = {
  *
  * Must be called under local_irq_disable().
  */
-static int intel_idle(struct cpuidle_device *dev,
-		struct cpuidle_driver *drv, int index)
+static __cpuidle int intel_idle(struct cpuidle_device *dev,
+				struct cpuidle_driver *drv, int index)
 {
 	unsigned long ecx = 1; /* break on interrupt flag */
 	struct cpuidle_state *state = &drv->states[index];
diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig
index 89d78208de3f..2b791fe1e2bc 100644
--- a/drivers/iio/accel/Kconfig
+++ b/drivers/iio/accel/Kconfig
@@ -20,6 +20,8 @@ config BMA180
 config BMA220
     tristate "Bosch BMA220 3-Axis Accelerometer Driver"
 	depends on SPI
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
     help
       Say yes here to add support for the Bosch BMA220 triaxial
       acceleration sensor.
@@ -50,6 +52,27 @@ config BMC150_ACCEL_SPI
 	tristate
 	select REGMAP_SPI
 
+config DMARD06
+	tristate "Domintech DMARD06 Digital Accelerometer Driver"
+	depends on OF || COMPILE_TEST
+	depends on I2C
+	help
+	  Say yes here to build support for the Domintech low-g tri-axial
+	  digital accelerometers: DMARD05, DMARD06, DMARD07.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called dmard06.
+
+config DMARD09
+	tristate "Domintech DMARD09 3-axis Accelerometer Driver"
+	depends on I2C
+	help
+	  Say yes here to get support for the Domintech DMARD09 3-axis
+	  accelerometer.
+
+	  Choosing M will build the driver as a module. If so, the module
+	  will be called dmard09.
+
 config HID_SENSOR_ACCEL_3D
 	depends on HID_SENSOR_HUB
 	select IIO_BUFFER
@@ -96,14 +119,35 @@ config IIO_ST_ACCEL_SPI_3AXIS
 
 config KXSD9
 	tristate "Kionix KXSD9 Accelerometer Driver"
-	depends on SPI
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
 	help
 	  Say yes here to build support for the Kionix KXSD9 accelerometer.
-	  Currently this only supports the device via an SPI interface.
+	  It can be accessed using an (optional) SPI or I2C interface.
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called kxsd9.
 
+config KXSD9_SPI
+	tristate "Kionix KXSD9 SPI transport"
+	depends on KXSD9
+	depends on SPI
+	default KXSD9
+	select REGMAP_SPI
+	help
+	  Say yes here to enable the Kionix KXSD9 accelerometer
+	  SPI transport channel.
+
+config KXSD9_I2C
+	tristate "Kionix KXSD9 I2C transport"
+	depends on KXSD9
+	depends on I2C
+	default KXSD9
+	select REGMAP_I2C
+	help
+	  Say yes here to enable the Kionix KXSD9 accelerometer
+	  I2C transport channel.
+
 config KXCJK1013
 	tristate "Kionix 3-Axis Accelerometer Driver"
 	depends on I2C
@@ -117,6 +161,16 @@ config KXCJK1013
 	  To compile this driver as a module, choose M here: the module will
 	  be called kxcjk-1013.
 
+config MC3230
+	tristate "mCube MC3230 Digital Accelerometer Driver"
+	depends on I2C
+	help
+	  Say yes here to build support for the mCube MC3230 low-g tri-axial
+	  digital accelerometer.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called mc3230.
+
 config MMA7455
 	tristate
 	select IIO_BUFFER
@@ -234,7 +288,8 @@ config STK8312
 config STK8BA50
 	tristate "Sensortek STK8BA50 3-Axis Accelerometer Driver"
 	depends on I2C
-	depends on IIO_TRIGGER
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
 	help
 	  Say yes here to get support for the Sensortek STK8BA50 3-axis
 	  accelerometer.
diff --git a/drivers/iio/accel/Makefile b/drivers/iio/accel/Makefile
index 6cedbecca2ee..f5d3ddee619e 100644
--- a/drivers/iio/accel/Makefile
+++ b/drivers/iio/accel/Makefile
@@ -8,9 +8,14 @@ obj-$(CONFIG_BMA220) += bma220_spi.o
 obj-$(CONFIG_BMC150_ACCEL) += bmc150-accel-core.o
 obj-$(CONFIG_BMC150_ACCEL_I2C) += bmc150-accel-i2c.o
 obj-$(CONFIG_BMC150_ACCEL_SPI) += bmc150-accel-spi.o
+obj-$(CONFIG_DMARD06)	+= dmard06.o
+obj-$(CONFIG_DMARD09)	+= dmard09.o
 obj-$(CONFIG_HID_SENSOR_ACCEL_3D) += hid-sensor-accel-3d.o
 obj-$(CONFIG_KXCJK1013) += kxcjk-1013.o
 obj-$(CONFIG_KXSD9)	+= kxsd9.o
+obj-$(CONFIG_KXSD9_SPI)	+= kxsd9-spi.o
+obj-$(CONFIG_KXSD9_I2C)	+= kxsd9-i2c.o
+obj-$(CONFIG_MC3230)	+= mc3230.o
 
 obj-$(CONFIG_MMA7455)		+= mma7455_core.o
 obj-$(CONFIG_MMA7455_I2C)	+= mma7455_i2c.o
diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c
index e3f88ba5faf3..0890934ef66f 100644
--- a/drivers/iio/accel/bma180.c
+++ b/drivers/iio/accel/bma180.c
@@ -469,13 +469,14 @@ static int bma180_read_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
 		mutex_lock(&data->mutex);
-		if (iio_buffer_enabled(indio_dev)) {
-			mutex_unlock(&data->mutex);
-			return -EBUSY;
-		}
 		ret = bma180_get_data_reg(data, chan->scan_index);
 		mutex_unlock(&data->mutex);
+		iio_device_release_direct_mode(indio_dev);
 		if (ret < 0)
 			return ret;
 		*val = sign_extend32(ret >> chan->scan_type.shift,
diff --git a/drivers/iio/accel/bma220_spi.c b/drivers/iio/accel/bma220_spi.c
index 1098d10df8e8..5099f295dd37 100644
--- a/drivers/iio/accel/bma220_spi.c
+++ b/drivers/iio/accel/bma220_spi.c
@@ -253,7 +253,7 @@ static int bma220_probe(struct spi_device *spi)
 	if (ret < 0)
 		return ret;
 
-	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+	ret = iio_triggered_buffer_setup(indio_dev, iio_pollfunc_store_time,
 					 bma220_trigger_handler, NULL);
 	if (ret < 0) {
 		dev_err(&spi->dev, "iio triggered buffer setup failed\n");
diff --git a/drivers/iio/accel/bmc150-accel-core.c b/drivers/iio/accel/bmc150-accel-core.c
index bf17aae66145..59b380dbf27f 100644
--- a/drivers/iio/accel/bmc150-accel-core.c
+++ b/drivers/iio/accel/bmc150-accel-core.c
@@ -67,6 +67,9 @@
 #define BMC150_ACCEL_REG_PMU_BW		0x10
 #define BMC150_ACCEL_DEF_BW			125
 
+#define BMC150_ACCEL_REG_RESET			0x14
+#define BMC150_ACCEL_RESET_VAL			0xB6
+
 #define BMC150_ACCEL_REG_INT_MAP_0		0x19
 #define BMC150_ACCEL_INT_MAP_0_BIT_SLOPE	BIT(2)
 
@@ -1497,6 +1500,14 @@ static int bmc150_accel_chip_init(struct bmc150_accel_data *data)
 	int ret, i;
 	unsigned int val;
 
+	/*
+	 * Reset chip to get it in a known good state. A delay of 1.8ms after
+	 * reset is required according to the data sheets of supported chips.
+	 */
+	regmap_write(data->regmap, BMC150_ACCEL_REG_RESET,
+		     BMC150_ACCEL_RESET_VAL);
+	usleep_range(1800, 2500);
+
 	ret = regmap_read(data->regmap, BMC150_ACCEL_REG_CHIP_ID, &val);
 	if (ret < 0) {
 		dev_err(dev, "Error: Reading chip id\n");
diff --git a/drivers/iio/accel/dmard06.c b/drivers/iio/accel/dmard06.c
new file mode 100644
index 000000000000..656ca8e1927f
--- /dev/null
+++ b/drivers/iio/accel/dmard06.c
@@ -0,0 +1,241 @@
+/*
+ * IIO driver for Domintech DMARD06 accelerometer
+ *
+ * Copyright (C) 2016 Aleksei Mamlin <mamlinav@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/iio/iio.h>
+
+#define DMARD06_DRV_NAME		"dmard06"
+
+/* Device data registers */
+#define DMARD06_CHIP_ID_REG		0x0f
+#define DMARD06_TOUT_REG		0x40
+#define DMARD06_XOUT_REG		0x41
+#define DMARD06_YOUT_REG		0x42
+#define DMARD06_ZOUT_REG		0x43
+#define DMARD06_CTRL1_REG		0x44
+
+/* Device ID value */
+#define DMARD05_CHIP_ID			0x05
+#define DMARD06_CHIP_ID			0x06
+#define DMARD07_CHIP_ID			0x07
+
+/* Device values */
+#define DMARD05_AXIS_SCALE_VAL		15625
+#define DMARD06_AXIS_SCALE_VAL		31250
+#define DMARD06_TEMP_CENTER_VAL		25
+#define DMARD06_SIGN_BIT		7
+
+/* Device power modes */
+#define DMARD06_MODE_NORMAL		0x27
+#define DMARD06_MODE_POWERDOWN		0x00
+
+/* Device channels */
+#define DMARD06_ACCEL_CHANNEL(_axis, _reg) {			\
+	.type = IIO_ACCEL,					\
+	.address = _reg,					\
+	.channel2 = IIO_MOD_##_axis,				\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+	.modified = 1,						\
+}
+
+#define DMARD06_TEMP_CHANNEL(_reg) {				\
+	.type = IIO_TEMP,					\
+	.address = _reg,					\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |		\
+			      BIT(IIO_CHAN_INFO_OFFSET),	\
+}
+
+struct dmard06_data {
+	struct i2c_client *client;
+	u8 chip_id;
+};
+
+static const struct iio_chan_spec dmard06_channels[] = {
+	DMARD06_ACCEL_CHANNEL(X, DMARD06_XOUT_REG),
+	DMARD06_ACCEL_CHANNEL(Y, DMARD06_YOUT_REG),
+	DMARD06_ACCEL_CHANNEL(Z, DMARD06_ZOUT_REG),
+	DMARD06_TEMP_CHANNEL(DMARD06_TOUT_REG),
+};
+
+static int dmard06_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
+{
+	struct dmard06_data *dmard06 = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = i2c_smbus_read_byte_data(dmard06->client,
+					       chan->address);
+		if (ret < 0) {
+			dev_err(&dmard06->client->dev,
+				"Error reading data: %d\n", ret);
+			return ret;
+		}
+
+		*val = sign_extend32(ret, DMARD06_SIGN_BIT);
+
+		if (dmard06->chip_id == DMARD06_CHIP_ID)
+			*val = *val >> 1;
+
+		switch (chan->type) {
+		case IIO_ACCEL:
+			return IIO_VAL_INT;
+		case IIO_TEMP:
+			if (dmard06->chip_id != DMARD06_CHIP_ID)
+				*val = *val / 2;
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_OFFSET:
+		switch (chan->type) {
+		case IIO_TEMP:
+			*val = DMARD06_TEMP_CENTER_VAL;
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_ACCEL:
+			*val = 0;
+			if (dmard06->chip_id == DMARD06_CHIP_ID)
+				*val2 = DMARD06_AXIS_SCALE_VAL;
+			else
+				*val2 = DMARD05_AXIS_SCALE_VAL;
+			return IIO_VAL_INT_PLUS_MICRO;
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info dmard06_info = {
+	.driver_module	= THIS_MODULE,
+	.read_raw	= dmard06_read_raw,
+};
+
+static int dmard06_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	int ret;
+	struct iio_dev *indio_dev;
+	struct dmard06_data *dmard06;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+		dev_err(&client->dev, "I2C check functionality failed\n");
+		return -ENXIO;
+	}
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*dmard06));
+	if (!indio_dev) {
+		dev_err(&client->dev, "Failed to allocate iio device\n");
+		return -ENOMEM;
+	}
+
+	dmard06 = iio_priv(indio_dev);
+	dmard06->client = client;
+
+	ret = i2c_smbus_read_byte_data(dmard06->client, DMARD06_CHIP_ID_REG);
+	if (ret < 0) {
+		dev_err(&client->dev, "Error reading chip id: %d\n", ret);
+		return ret;
+	}
+
+	if (ret != DMARD05_CHIP_ID && ret != DMARD06_CHIP_ID &&
+	    ret != DMARD07_CHIP_ID) {
+		dev_err(&client->dev, "Invalid chip id: %02d\n", ret);
+		return -ENODEV;
+	}
+
+	dmard06->chip_id = ret;
+
+	i2c_set_clientdata(client, indio_dev);
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->name = DMARD06_DRV_NAME;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = dmard06_channels;
+	indio_dev->num_channels = ARRAY_SIZE(dmard06_channels);
+	indio_dev->info = &dmard06_info;
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int dmard06_suspend(struct device *dev)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
+	struct dmard06_data *dmard06 = iio_priv(indio_dev);
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(dmard06->client, DMARD06_CTRL1_REG,
+					DMARD06_MODE_POWERDOWN);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dmard06_resume(struct device *dev)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
+	struct dmard06_data *dmard06 = iio_priv(indio_dev);
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(dmard06->client, DMARD06_CTRL1_REG,
+					DMARD06_MODE_NORMAL);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(dmard06_pm_ops, dmard06_suspend, dmard06_resume);
+#define DMARD06_PM_OPS (&dmard06_pm_ops)
+#else
+#define DMARD06_PM_OPS NULL
+#endif
+
+static const struct i2c_device_id dmard06_id[] = {
+	{ "dmard05", 0 },
+	{ "dmard06", 0 },
+	{ "dmard07", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, dmard06_id);
+
+static const struct of_device_id dmard06_of_match[] = {
+	{ .compatible = "domintech,dmard05" },
+	{ .compatible = "domintech,dmard06" },
+	{ .compatible = "domintech,dmard07" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, dmard06_of_match);
+
+static struct i2c_driver dmard06_driver = {
+	.probe = dmard06_probe,
+	.id_table = dmard06_id,
+	.driver = {
+		.name = DMARD06_DRV_NAME,
+		.of_match_table = of_match_ptr(dmard06_of_match),
+		.pm = DMARD06_PM_OPS,
+	},
+};
+module_i2c_driver(dmard06_driver);
+
+MODULE_AUTHOR("Aleksei Mamlin <mamlinav@gmail.com>");
+MODULE_DESCRIPTION("Domintech DMARD06 accelerometer driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/accel/dmard09.c b/drivers/iio/accel/dmard09.c
new file mode 100644
index 000000000000..d3a28f96565c
--- /dev/null
+++ b/drivers/iio/accel/dmard09.c
@@ -0,0 +1,157 @@
+/*
+ * IIO driver for the 3-axis accelerometer Domintech DMARD09.
+ *
+ * Copyright (c) 2016, Jelle van der Waa <jelle@vdwaa.nl>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <asm/unaligned.h>
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/iio/iio.h>
+
+#define DMARD09_DRV_NAME	"dmard09"
+
+#define DMARD09_REG_CHIPID      0x18
+#define DMARD09_REG_STAT	0x0A
+#define DMARD09_REG_X		0x0C
+#define DMARD09_REG_Y		0x0E
+#define DMARD09_REG_Z		0x10
+#define DMARD09_CHIPID		0x95
+
+#define DMARD09_BUF_LEN 8
+#define DMARD09_AXIS_X 0
+#define DMARD09_AXIS_Y 1
+#define DMARD09_AXIS_Z 2
+#define DMARD09_AXIS_X_OFFSET ((DMARD09_AXIS_X + 1) * 2)
+#define DMARD09_AXIS_Y_OFFSET ((DMARD09_AXIS_Y + 1 )* 2)
+#define DMARD09_AXIS_Z_OFFSET ((DMARD09_AXIS_Z + 1) * 2)
+
+struct dmard09_data {
+	struct i2c_client *client;
+};
+
+#define DMARD09_CHANNEL(_axis, offset) {			\
+	.type = IIO_ACCEL,					\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+	.modified = 1,						\
+	.address = offset,					\
+	.channel2 = IIO_MOD_##_axis,				\
+}
+
+static const struct iio_chan_spec dmard09_channels[] = {
+	DMARD09_CHANNEL(X, DMARD09_AXIS_X_OFFSET),
+	DMARD09_CHANNEL(Y, DMARD09_AXIS_Y_OFFSET),
+	DMARD09_CHANNEL(Z, DMARD09_AXIS_Z_OFFSET),
+};
+
+static int dmard09_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
+{
+	struct dmard09_data *data = iio_priv(indio_dev);
+	u8 buf[DMARD09_BUF_LEN];
+	int ret;
+	s16 accel;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		/*
+		 * Read from the DMAR09_REG_STAT register, since the chip
+		 * caches reads from the individual X, Y, Z registers.
+		 */
+		ret = i2c_smbus_read_i2c_block_data(data->client,
+						    DMARD09_REG_STAT,
+						    DMARD09_BUF_LEN, buf);
+		if (ret < 0) {
+			dev_err(&data->client->dev, "Error reading reg %d\n",
+				DMARD09_REG_STAT);
+			return ret;
+		}
+
+		accel = get_unaligned_le16(&buf[chan->address]);
+
+		/* Remove lower 3 bits and sign extend */
+		accel <<= 4;
+		accel >>= 7;
+
+		*val = accel;
+
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info dmard09_info = {
+	.driver_module	= THIS_MODULE,
+	.read_raw	= dmard09_read_raw,
+};
+
+static int dmard09_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	int ret;
+	struct iio_dev *indio_dev;
+	struct dmard09_data *data;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev) {
+		dev_err(&client->dev, "iio allocation failed\n");
+		return -ENOMEM;
+	}
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+
+	ret = i2c_smbus_read_byte_data(data->client, DMARD09_REG_CHIPID);
+	if (ret < 0) {
+		dev_err(&client->dev, "Error reading chip id %d\n", ret);
+		return ret;
+	}
+
+	if (ret != DMARD09_CHIPID) {
+		dev_err(&client->dev, "Invalid chip id %d\n", ret);
+		return -ENODEV;
+	}
+
+	i2c_set_clientdata(client, indio_dev);
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->name = DMARD09_DRV_NAME;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = dmard09_channels;
+	indio_dev->num_channels = ARRAY_SIZE(dmard09_channels);
+	indio_dev->info = &dmard09_info;
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+static const struct i2c_device_id dmard09_id[] = {
+	{ "dmard09", 0},
+	{ },
+};
+
+MODULE_DEVICE_TABLE(i2c, dmard09_id);
+
+static struct i2c_driver dmard09_driver = {
+	.driver = {
+		.name = DMARD09_DRV_NAME
+	},
+	.probe = dmard09_probe,
+	.id_table = dmard09_id,
+};
+
+module_i2c_driver(dmard09_driver);
+
+MODULE_AUTHOR("Jelle van der Waa <jelle@vdwaa.nl>");
+MODULE_DESCRIPTION("DMARD09 3-axis accelerometer driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/accel/kxcjk-1013.c b/drivers/iio/accel/kxcjk-1013.c
index 765a72362dc6..3f968c46e667 100644
--- a/drivers/iio/accel/kxcjk-1013.c
+++ b/drivers/iio/accel/kxcjk-1013.c
@@ -1392,6 +1392,7 @@ static const struct acpi_device_id kx_acpi_match[] = {
 	{"KXCJ1013", KXCJK1013},
 	{"KXCJ1008", KXCJ91008},
 	{"KXCJ9000", KXCJ91008},
+	{"KIOX000A", KXCJ91008},
 	{"KXTJ1009", KXTJ21009},
 	{"SMO8500",  KXCJ91008},
 	{ },
diff --git a/drivers/iio/accel/kxsd9-i2c.c b/drivers/iio/accel/kxsd9-i2c.c
new file mode 100644
index 000000000000..95e20855d2ef
--- /dev/null
+++ b/drivers/iio/accel/kxsd9-i2c.c
@@ -0,0 +1,64 @@
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/i2c.h>
+#include <linux/delay.h>
+#include <linux/regmap.h>
+
+#include "kxsd9.h"
+
+static int kxsd9_i2c_probe(struct i2c_client *i2c,
+			   const struct i2c_device_id *id)
+{
+	static const struct regmap_config config = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = 0x0e,
+	};
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init_i2c(i2c, &config);
+	if (IS_ERR(regmap)) {
+		dev_err(&i2c->dev, "Failed to register i2c regmap %d\n",
+			(int)PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
+	return kxsd9_common_probe(&i2c->dev,
+				  regmap,
+				  i2c->name);
+}
+
+static int kxsd9_i2c_remove(struct i2c_client *client)
+{
+	return kxsd9_common_remove(&client->dev);
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id kxsd9_of_match[] = {
+	{ .compatible = "kionix,kxsd9", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, kxsd9_of_match);
+#else
+#define kxsd9_of_match NULL
+#endif
+
+static const struct i2c_device_id kxsd9_i2c_id[] = {
+	{"kxsd9", 0},
+	{ },
+};
+MODULE_DEVICE_TABLE(i2c, kxsd9_i2c_id);
+
+static struct i2c_driver kxsd9_i2c_driver = {
+	.driver = {
+		.name	= "kxsd9",
+		.of_match_table = of_match_ptr(kxsd9_of_match),
+		.pm = &kxsd9_dev_pm_ops,
+	},
+	.probe		= kxsd9_i2c_probe,
+	.remove		= kxsd9_i2c_remove,
+	.id_table	= kxsd9_i2c_id,
+};
+module_i2c_driver(kxsd9_i2c_driver);
diff --git a/drivers/iio/accel/kxsd9-spi.c b/drivers/iio/accel/kxsd9-spi.c
new file mode 100644
index 000000000000..b7d0078fd00e
--- /dev/null
+++ b/drivers/iio/accel/kxsd9-spi.c
@@ -0,0 +1,56 @@
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/spi/spi.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/regmap.h>
+
+#include "kxsd9.h"
+
+static int kxsd9_spi_probe(struct spi_device *spi)
+{
+	static const struct regmap_config config = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = 0x0e,
+	};
+	struct regmap *regmap;
+
+	spi->mode = SPI_MODE_0;
+	regmap = devm_regmap_init_spi(spi, &config);
+	if (IS_ERR(regmap)) {
+		dev_err(&spi->dev, "%s: regmap allocation failed: %ld\n",
+			__func__, PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
+	return kxsd9_common_probe(&spi->dev,
+				  regmap,
+				  spi_get_device_id(spi)->name);
+}
+
+static int kxsd9_spi_remove(struct spi_device *spi)
+{
+	return kxsd9_common_remove(&spi->dev);
+}
+
+static const struct spi_device_id kxsd9_spi_id[] = {
+	{"kxsd9", 0},
+	{ },
+};
+MODULE_DEVICE_TABLE(spi, kxsd9_spi_id);
+
+static struct spi_driver kxsd9_spi_driver = {
+	.driver = {
+		.name = "kxsd9",
+		.pm = &kxsd9_dev_pm_ops,
+	},
+	.probe = kxsd9_spi_probe,
+	.remove = kxsd9_spi_remove,
+	.id_table = kxsd9_spi_id,
+};
+module_spi_driver(kxsd9_spi_driver);
+
+MODULE_AUTHOR("Jonathan Cameron <jic23@kernel.org>");
+MODULE_DESCRIPTION("Kionix KXSD9 SPI driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/accel/kxsd9.c b/drivers/iio/accel/kxsd9.c
index 3a9f106787d2..9af60ac70738 100644
--- a/drivers/iio/accel/kxsd9.c
+++ b/drivers/iio/accel/kxsd9.c
@@ -12,19 +12,25 @@
  * I have a suitable wire made up.
  *
  * TODO:	Support the motion detector
- *		Uses register address incrementing so could have a
- *		heavily optimized ring buffer access function.
  */
 
 #include <linux/device.h>
 #include <linux/kernel.h>
-#include <linux/spi/spi.h>
 #include <linux/sysfs.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-
+#include <linux/regmap.h>
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/regulator/consumer.h>
+#include <linux/pm_runtime.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/trigger_consumer.h>
+
+#include "kxsd9.h"
 
 #define KXSD9_REG_X		0x00
 #define KXSD9_REG_Y		0x02
@@ -33,28 +39,45 @@
 #define KXSD9_REG_RESET		0x0a
 #define KXSD9_REG_CTRL_C	0x0c
 
-#define KXSD9_FS_MASK		0x03
+#define KXSD9_CTRL_C_FS_MASK	0x03
+#define KXSD9_CTRL_C_FS_8G	0x00
+#define KXSD9_CTRL_C_FS_6G	0x01
+#define KXSD9_CTRL_C_FS_4G	0x02
+#define KXSD9_CTRL_C_FS_2G	0x03
+#define KXSD9_CTRL_C_MOT_LAT	BIT(3)
+#define KXSD9_CTRL_C_MOT_LEV	BIT(4)
+#define KXSD9_CTRL_C_LP_MASK	0xe0
+#define KXSD9_CTRL_C_LP_NONE	0x00
+#define KXSD9_CTRL_C_LP_2000HZC	BIT(5)
+#define KXSD9_CTRL_C_LP_2000HZB	BIT(6)
+#define KXSD9_CTRL_C_LP_2000HZA	(BIT(5)|BIT(6))
+#define KXSD9_CTRL_C_LP_1000HZ	BIT(7)
+#define KXSD9_CTRL_C_LP_500HZ	(BIT(7)|BIT(5))
+#define KXSD9_CTRL_C_LP_100HZ	(BIT(7)|BIT(6))
+#define KXSD9_CTRL_C_LP_50HZ	(BIT(7)|BIT(6)|BIT(5))
 
 #define KXSD9_REG_CTRL_B	0x0d
-#define KXSD9_REG_CTRL_A	0x0e
 
-#define KXSD9_READ(a) (0x80 | (a))
-#define KXSD9_WRITE(a) (a)
+#define KXSD9_CTRL_B_CLK_HLD	BIT(7)
+#define KXSD9_CTRL_B_ENABLE	BIT(6)
+#define KXSD9_CTRL_B_ST		BIT(5) /* Self-test */
+
+#define KXSD9_REG_CTRL_A	0x0e
 
-#define KXSD9_STATE_RX_SIZE 2
-#define KXSD9_STATE_TX_SIZE 2
 /**
  * struct kxsd9_state - device related storage
- * @buf_lock:	protect the rx and tx buffers.
- * @us:		spi device
- * @rx:		single rx buffer storage
- * @tx:		single tx buffer storage
- **/
+ * @dev: pointer to the parent device
+ * @map: regmap to the device
+ * @orientation: mounting matrix, flipped axis etc
+ * @regs: regulators for this device, VDD and IOVDD
+ * @scale: the current scaling setting
+ */
 struct kxsd9_state {
-	struct mutex buf_lock;
-	struct spi_device *us;
-	u8 rx[KXSD9_STATE_RX_SIZE] ____cacheline_aligned;
-	u8 tx[KXSD9_STATE_TX_SIZE];
+	struct device *dev;
+	struct regmap *map;
+	struct iio_mount_matrix orientation;
+	struct regulator_bulk_data regs[2];
+	u8 scale;
 };
 
 #define KXSD9_SCALE_2G "0.011978"
@@ -65,6 +88,14 @@ struct kxsd9_state {
 /* reverse order */
 static const int kxsd9_micro_scales[4] = { 47853, 35934, 23927, 11978 };
 
+#define KXSD9_ZERO_G_OFFSET -2048
+
+/*
+ * Regulator names
+ */
+static const char kxsd9_reg_vdd[] = "vdd";
+static const char kxsd9_reg_iovdd[] = "iovdd";
+
 static int kxsd9_write_scale(struct iio_dev *indio_dev, int micro)
 {
 	int ret, i;
@@ -79,42 +110,17 @@ static int kxsd9_write_scale(struct iio_dev *indio_dev, int micro)
 	if (!foundit)
 		return -EINVAL;
 
-	mutex_lock(&st->buf_lock);
-	ret = spi_w8r8(st->us, KXSD9_READ(KXSD9_REG_CTRL_C));
+	ret = regmap_update_bits(st->map,
+				 KXSD9_REG_CTRL_C,
+				 KXSD9_CTRL_C_FS_MASK,
+				 i);
 	if (ret < 0)
 		goto error_ret;
-	st->tx[0] = KXSD9_WRITE(KXSD9_REG_CTRL_C);
-	st->tx[1] = (ret & ~KXSD9_FS_MASK) | i;
 
-	ret = spi_write(st->us, st->tx, 2);
-error_ret:
-	mutex_unlock(&st->buf_lock);
-	return ret;
-}
+	/* Cached scale when the sensor is powered down */
+	st->scale = i;
 
-static int kxsd9_read(struct iio_dev *indio_dev, u8 address)
-{
-	int ret;
-	struct kxsd9_state *st = iio_priv(indio_dev);
-	struct spi_transfer xfers[] = {
-		{
-			.bits_per_word = 8,
-			.len = 1,
-			.delay_usecs = 200,
-			.tx_buf = st->tx,
-		}, {
-			.bits_per_word = 8,
-			.len = 2,
-			.rx_buf = st->rx,
-		},
-	};
-
-	mutex_lock(&st->buf_lock);
-	st->tx[0] = KXSD9_READ(address);
-	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
-	if (!ret)
-		ret = (((u16)(st->rx[0])) << 8) | (st->rx[1] & 0xF0);
-	mutex_unlock(&st->buf_lock);
+error_ret:
 	return ret;
 }
 
@@ -136,6 +142,9 @@ static int kxsd9_write_raw(struct iio_dev *indio_dev,
 			   long mask)
 {
 	int ret = -EINVAL;
+	struct kxsd9_state *st = iio_priv(indio_dev);
+
+	pm_runtime_get_sync(st->dev);
 
 	if (mask == IIO_CHAN_INFO_SCALE) {
 		/* Check no integer component */
@@ -144,6 +153,9 @@ static int kxsd9_write_raw(struct iio_dev *indio_dev,
 		ret = kxsd9_write_scale(indio_dev, val2);
 	}
 
+	pm_runtime_mark_last_busy(st->dev);
+	pm_runtime_put_autosuspend(st->dev);
+
 	return ret;
 }
 
@@ -153,44 +165,154 @@ static int kxsd9_read_raw(struct iio_dev *indio_dev,
 {
 	int ret = -EINVAL;
 	struct kxsd9_state *st = iio_priv(indio_dev);
+	unsigned int regval;
+	__be16 raw_val;
+	u16 nval;
+
+	pm_runtime_get_sync(st->dev);
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		ret = kxsd9_read(indio_dev, chan->address);
-		if (ret < 0)
+		ret = regmap_bulk_read(st->map, chan->address, &raw_val,
+				       sizeof(raw_val));
+		if (ret)
 			goto error_ret;
-		*val = ret;
+		nval = be16_to_cpu(raw_val);
+		/* Only 12 bits are valid */
+		nval >>= 4;
+		*val = nval;
+		ret = IIO_VAL_INT;
+		break;
+	case IIO_CHAN_INFO_OFFSET:
+		/* This has a bias of -2048 */
+		*val = KXSD9_ZERO_G_OFFSET;
+		ret = IIO_VAL_INT;
 		break;
 	case IIO_CHAN_INFO_SCALE:
-		ret = spi_w8r8(st->us, KXSD9_READ(KXSD9_REG_CTRL_C));
+		ret = regmap_read(st->map,
+				  KXSD9_REG_CTRL_C,
+				  &regval);
 		if (ret < 0)
 			goto error_ret;
-		*val2 = kxsd9_micro_scales[ret & KXSD9_FS_MASK];
+		*val = 0;
+		*val2 = kxsd9_micro_scales[regval & KXSD9_CTRL_C_FS_MASK];
 		ret = IIO_VAL_INT_PLUS_MICRO;
 		break;
 	}
 
 error_ret:
+	pm_runtime_mark_last_busy(st->dev);
+	pm_runtime_put_autosuspend(st->dev);
+
 	return ret;
 };
-#define KXSD9_ACCEL_CHAN(axis)						\
+
+static irqreturn_t kxsd9_trigger_handler(int irq, void *p)
+{
+	const struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct kxsd9_state *st = iio_priv(indio_dev);
+	int ret;
+	/* 4 * 16bit values AND timestamp */
+	__be16 hw_values[8];
+
+	ret = regmap_bulk_read(st->map,
+			       KXSD9_REG_X,
+			       &hw_values,
+			       8);
+	if (ret) {
+		dev_err(st->dev,
+			"error reading data\n");
+		return ret;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev,
+					   hw_values,
+					   iio_get_time_ns(indio_dev));
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int kxsd9_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct kxsd9_state *st = iio_priv(indio_dev);
+
+	pm_runtime_get_sync(st->dev);
+
+	return 0;
+}
+
+static int kxsd9_buffer_postdisable(struct iio_dev *indio_dev)
+{
+	struct kxsd9_state *st = iio_priv(indio_dev);
+
+	pm_runtime_mark_last_busy(st->dev);
+	pm_runtime_put_autosuspend(st->dev);
+
+	return 0;
+}
+
+static const struct iio_buffer_setup_ops kxsd9_buffer_setup_ops = {
+	.preenable = kxsd9_buffer_preenable,
+	.postenable = iio_triggered_buffer_postenable,
+	.predisable = iio_triggered_buffer_predisable,
+	.postdisable = kxsd9_buffer_postdisable,
+};
+
+static const struct iio_mount_matrix *
+kxsd9_get_mount_matrix(const struct iio_dev *indio_dev,
+		       const struct iio_chan_spec *chan)
+{
+	struct kxsd9_state *st = iio_priv(indio_dev);
+
+	return &st->orientation;
+}
+
+static const struct iio_chan_spec_ext_info kxsd9_ext_info[] = {
+	IIO_MOUNT_MATRIX(IIO_SHARED_BY_TYPE, kxsd9_get_mount_matrix),
+	{ },
+};
+
+#define KXSD9_ACCEL_CHAN(axis, index)						\
 	{								\
 		.type = IIO_ACCEL,					\
 		.modified = 1,						\
 		.channel2 = IIO_MOD_##axis,				\
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
-		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |	\
+					BIT(IIO_CHAN_INFO_OFFSET),	\
+		.ext_info = kxsd9_ext_info,				\
 		.address = KXSD9_REG_##axis,				\
+		.scan_index = index,					\
+		.scan_type = {                                          \
+			.sign = 'u',					\
+			.realbits = 12,					\
+			.storagebits = 16,				\
+			.shift = 4,					\
+			.endianness = IIO_BE,				\
+		},							\
 	}
 
 static const struct iio_chan_spec kxsd9_channels[] = {
-	KXSD9_ACCEL_CHAN(X), KXSD9_ACCEL_CHAN(Y), KXSD9_ACCEL_CHAN(Z),
+	KXSD9_ACCEL_CHAN(X, 0),
+	KXSD9_ACCEL_CHAN(Y, 1),
+	KXSD9_ACCEL_CHAN(Z, 2),
 	{
 		.type = IIO_VOLTAGE,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
 		.indexed = 1,
 		.address = KXSD9_REG_AUX,
-	}
+		.scan_index = 3,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 12,
+			.storagebits = 16,
+			.shift = 4,
+			.endianness = IIO_BE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(4),
 };
 
 static const struct attribute_group kxsd9_attribute_group = {
@@ -201,17 +323,69 @@ static int kxsd9_power_up(struct kxsd9_state *st)
 {
 	int ret;
 
-	st->tx[0] = 0x0d;
-	st->tx[1] = 0x40;
-	ret = spi_write(st->us, st->tx, 2);
+	/* Enable the regulators */
+	ret = regulator_bulk_enable(ARRAY_SIZE(st->regs), st->regs);
+	if (ret) {
+		dev_err(st->dev, "Cannot enable regulators\n");
+		return ret;
+	}
+
+	/* Power up */
+	ret = regmap_write(st->map,
+			   KXSD9_REG_CTRL_B,
+			   KXSD9_CTRL_B_ENABLE);
 	if (ret)
 		return ret;
 
-	st->tx[0] = 0x0c;
-	st->tx[1] = 0x9b;
-	return spi_write(st->us, st->tx, 2);
+	/*
+	 * Set 1000Hz LPF, 2g fullscale, motion wakeup threshold 1g,
+	 * latched wakeup
+	 */
+	ret = regmap_write(st->map,
+			   KXSD9_REG_CTRL_C,
+			   KXSD9_CTRL_C_LP_1000HZ |
+			   KXSD9_CTRL_C_MOT_LEV	|
+			   KXSD9_CTRL_C_MOT_LAT |
+			   st->scale);
+	if (ret)
+		return ret;
+
+	/*
+	 * Power-up time depends on the LPF setting, but typ 15.9 ms, let's
+	 * set 20 ms to allow for some slack.
+	 */
+	msleep(20);
+
+	return 0;
 };
 
+static int kxsd9_power_down(struct kxsd9_state *st)
+{
+	int ret;
+
+	/*
+	 * Set into low power mode - since there may be more users of the
+	 * regulators this is the first step of the power saving: it will
+	 * make sure we conserve power even if there are others users on the
+	 * regulators.
+	 */
+	ret = regmap_update_bits(st->map,
+				 KXSD9_REG_CTRL_B,
+				 KXSD9_CTRL_B_ENABLE,
+				 0);
+	if (ret)
+		return ret;
+
+	/* Disable the regulators */
+	ret = regulator_bulk_disable(ARRAY_SIZE(st->regs), st->regs);
+	if (ret) {
+		dev_err(st->dev, "Cannot disable regulators\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 static const struct iio_info kxsd9_info = {
 	.read_raw = &kxsd9_read_raw,
 	.write_raw = &kxsd9_write_raw,
@@ -219,57 +393,136 @@ static const struct iio_info kxsd9_info = {
 	.driver_module = THIS_MODULE,
 };
 
-static int kxsd9_probe(struct spi_device *spi)
+/* Four channels apart from timestamp, scan mask = 0x0f */
+static const unsigned long kxsd9_scan_masks[] = { 0xf, 0 };
+
+int kxsd9_common_probe(struct device *dev,
+		       struct regmap *map,
+		       const char *name)
 {
 	struct iio_dev *indio_dev;
 	struct kxsd9_state *st;
+	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
 	st = iio_priv(indio_dev);
-	spi_set_drvdata(spi, indio_dev);
+	st->dev = dev;
+	st->map = map;
 
-	st->us = spi;
-	mutex_init(&st->buf_lock);
 	indio_dev->channels = kxsd9_channels;
 	indio_dev->num_channels = ARRAY_SIZE(kxsd9_channels);
-	indio_dev->name = spi_get_device_id(spi)->name;
-	indio_dev->dev.parent = &spi->dev;
+	indio_dev->name = name;
+	indio_dev->dev.parent = dev;
 	indio_dev->info = &kxsd9_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->available_scan_masks = kxsd9_scan_masks;
+
+	/* Read the mounting matrix, if present */
+	ret = of_iio_read_mount_matrix(dev,
+				       "mount-matrix",
+				       &st->orientation);
+	if (ret)
+		return ret;
+
+	/* Fetch and turn on regulators */
+	st->regs[0].supply = kxsd9_reg_vdd;
+	st->regs[1].supply = kxsd9_reg_iovdd;
+	ret = devm_regulator_bulk_get(dev,
+				      ARRAY_SIZE(st->regs),
+				      st->regs);
+	if (ret) {
+		dev_err(dev, "Cannot get regulators\n");
+		return ret;
+	}
+	/* Default scaling */
+	st->scale = KXSD9_CTRL_C_FS_2G;
 
-	spi->mode = SPI_MODE_0;
-	spi_setup(spi);
 	kxsd9_power_up(st);
 
-	return iio_device_register(indio_dev);
+	ret = iio_triggered_buffer_setup(indio_dev,
+					 iio_pollfunc_store_time,
+					 kxsd9_trigger_handler,
+					 &kxsd9_buffer_setup_ops);
+	if (ret) {
+		dev_err(dev, "triggered buffer setup failed\n");
+		goto err_power_down;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret)
+		goto err_cleanup_buffer;
+
+	dev_set_drvdata(dev, indio_dev);
+
+	/* Enable runtime PM */
+	pm_runtime_get_noresume(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+	/*
+	 * Set autosuspend to two orders of magnitude larger than the
+	 * start-up time. 20ms start-up time means 2000ms autosuspend,
+	 * i.e. 2 seconds.
+	 */
+	pm_runtime_set_autosuspend_delay(dev, 2000);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_put(dev);
+
+	return 0;
+
+err_cleanup_buffer:
+	iio_triggered_buffer_cleanup(indio_dev);
+err_power_down:
+	kxsd9_power_down(st);
+
+	return ret;
 }
+EXPORT_SYMBOL(kxsd9_common_probe);
 
-static int kxsd9_remove(struct spi_device *spi)
+int kxsd9_common_remove(struct device *dev)
 {
-	iio_device_unregister(spi_get_drvdata(spi));
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct kxsd9_state *st = iio_priv(indio_dev);
+
+	iio_triggered_buffer_cleanup(indio_dev);
+	iio_device_unregister(indio_dev);
+	pm_runtime_get_sync(dev);
+	pm_runtime_put_noidle(dev);
+	pm_runtime_disable(dev);
+	kxsd9_power_down(st);
 
 	return 0;
 }
+EXPORT_SYMBOL(kxsd9_common_remove);
 
-static const struct spi_device_id kxsd9_id[] = {
-	{"kxsd9", 0},
-	{ },
-};
-MODULE_DEVICE_TABLE(spi, kxsd9_id);
+#ifdef CONFIG_PM
+static int kxsd9_runtime_suspend(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct kxsd9_state *st = iio_priv(indio_dev);
 
-static struct spi_driver kxsd9_driver = {
-	.driver = {
-		.name = "kxsd9",
-	},
-	.probe = kxsd9_probe,
-	.remove = kxsd9_remove,
-	.id_table = kxsd9_id,
+	return kxsd9_power_down(st);
+}
+
+static int kxsd9_runtime_resume(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct kxsd9_state *st = iio_priv(indio_dev);
+
+	return kxsd9_power_up(st);
+}
+#endif /* CONFIG_PM */
+
+const struct dev_pm_ops kxsd9_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(kxsd9_runtime_suspend,
+			   kxsd9_runtime_resume, NULL)
 };
-module_spi_driver(kxsd9_driver);
+EXPORT_SYMBOL(kxsd9_dev_pm_ops);
 
 MODULE_AUTHOR("Jonathan Cameron <jic23@kernel.org>");
-MODULE_DESCRIPTION("Kionix KXSD9 SPI driver");
+MODULE_DESCRIPTION("Kionix KXSD9 driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/accel/kxsd9.h b/drivers/iio/accel/kxsd9.h
new file mode 100644
index 000000000000..7e8a28168310
--- /dev/null
+++ b/drivers/iio/accel/kxsd9.h
@@ -0,0 +1,12 @@
+#include <linux/device.h>
+#include <linux/kernel.h>
+
+#define KXSD9_STATE_RX_SIZE 2
+#define KXSD9_STATE_TX_SIZE 2
+
+int kxsd9_common_probe(struct device *dev,
+		       struct regmap *map,
+		       const char *name);
+int kxsd9_common_remove(struct device *dev);
+
+extern const struct dev_pm_ops kxsd9_dev_pm_ops;
diff --git a/drivers/iio/accel/mc3230.c b/drivers/iio/accel/mc3230.c
new file mode 100644
index 000000000000..4ea2ff623a6d
--- /dev/null
+++ b/drivers/iio/accel/mc3230.c
@@ -0,0 +1,211 @@
+/**
+ * mCube MC3230 3-Axis Accelerometer
+ *
+ * Copyright (c) 2016 Hans de Goede <hdegoede@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * IIO driver for mCube MC3230; 7-bit I2C address: 0x4c.
+ */
+
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+#define MC3230_REG_XOUT			0x00
+#define MC3230_REG_YOUT			0x01
+#define MC3230_REG_ZOUT			0x02
+
+#define MC3230_REG_MODE			0x07
+#define MC3230_MODE_OPCON_MASK		0x03
+#define MC3230_MODE_OPCON_WAKE		0x01
+#define MC3230_MODE_OPCON_STANDBY	0x03
+
+#define MC3230_REG_CHIP_ID		0x18
+#define MC3230_CHIP_ID			0x01
+
+#define MC3230_REG_PRODUCT_CODE		0x3b
+#define MC3230_PRODUCT_CODE		0x19
+
+/*
+ * The accelerometer has one measurement range:
+ *
+ * -1.5g - +1.5g (8-bit, signed)
+ *
+ * scale = (1.5 + 1.5) * 9.81 / (2^8 - 1)	= 0.115411765
+ */
+
+static const int mc3230_nscale = 115411765;
+
+#define MC3230_CHANNEL(reg, axis) {	\
+	.type = IIO_ACCEL,	\
+	.address = reg,	\
+	.modified = 1,	\
+	.channel2 = IIO_MOD_##axis,	\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+}
+
+static const struct iio_chan_spec mc3230_channels[] = {
+	MC3230_CHANNEL(MC3230_REG_XOUT, X),
+	MC3230_CHANNEL(MC3230_REG_YOUT, Y),
+	MC3230_CHANNEL(MC3230_REG_ZOUT, Z),
+};
+
+struct mc3230_data {
+	struct i2c_client *client;
+};
+
+static int mc3230_set_opcon(struct mc3230_data *data, int opcon)
+{
+	int ret;
+	struct i2c_client *client = data->client;
+
+	ret = i2c_smbus_read_byte_data(client, MC3230_REG_MODE);
+	if (ret < 0) {
+		dev_err(&client->dev, "failed to read mode reg: %d\n", ret);
+		return ret;
+	}
+
+	ret &= ~MC3230_MODE_OPCON_MASK;
+	ret |= opcon;
+
+	ret = i2c_smbus_write_byte_data(client, MC3230_REG_MODE, ret);
+	if (ret < 0) {
+		dev_err(&client->dev, "failed to write mode reg: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int mc3230_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan,
+				int *val, int *val2, long mask)
+{
+	struct mc3230_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = i2c_smbus_read_byte_data(data->client, chan->address);
+		if (ret < 0)
+			return ret;
+		*val = sign_extend32(ret, 7);
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = 0;
+		*val2 = mc3230_nscale;
+		return IIO_VAL_INT_PLUS_NANO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info mc3230_info = {
+	.driver_module	= THIS_MODULE,
+	.read_raw	= mc3230_read_raw,
+};
+
+static int mc3230_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	int ret;
+	struct iio_dev *indio_dev;
+	struct mc3230_data *data;
+
+	/* First check chip-id and product-id */
+	ret = i2c_smbus_read_byte_data(client, MC3230_REG_CHIP_ID);
+	if (ret != MC3230_CHIP_ID)
+		return (ret < 0) ? ret : -ENODEV;
+
+	ret = i2c_smbus_read_byte_data(client, MC3230_REG_PRODUCT_CODE);
+	if (ret != MC3230_PRODUCT_CODE)
+		return (ret < 0) ? ret : -ENODEV;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev) {
+		dev_err(&client->dev, "iio allocation failed!\n");
+		return -ENOMEM;
+	}
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+	i2c_set_clientdata(client, indio_dev);
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->info = &mc3230_info;
+	indio_dev->name = "mc3230";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = mc3230_channels;
+	indio_dev->num_channels = ARRAY_SIZE(mc3230_channels);
+
+	ret = mc3230_set_opcon(data, MC3230_MODE_OPCON_WAKE);
+	if (ret < 0)
+		return ret;
+
+	ret = iio_device_register(indio_dev);
+	if (ret < 0) {
+		dev_err(&client->dev, "device_register failed\n");
+		mc3230_set_opcon(data, MC3230_MODE_OPCON_STANDBY);
+	}
+
+	return ret;
+}
+
+static int mc3230_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+	iio_device_unregister(indio_dev);
+
+	return mc3230_set_opcon(iio_priv(indio_dev), MC3230_MODE_OPCON_STANDBY);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int mc3230_suspend(struct device *dev)
+{
+	struct mc3230_data *data;
+
+	data = iio_priv(i2c_get_clientdata(to_i2c_client(dev)));
+
+	return mc3230_set_opcon(data, MC3230_MODE_OPCON_STANDBY);
+}
+
+static int mc3230_resume(struct device *dev)
+{
+	struct mc3230_data *data;
+
+	data = iio_priv(i2c_get_clientdata(to_i2c_client(dev)));
+
+	return mc3230_set_opcon(data, MC3230_MODE_OPCON_WAKE);
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(mc3230_pm_ops, mc3230_suspend, mc3230_resume);
+
+static const struct i2c_device_id mc3230_i2c_id[] = {
+	{"mc3230", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, mc3230_i2c_id);
+
+static struct i2c_driver mc3230_driver = {
+	.driver = {
+		.name = "mc3230",
+		.pm = &mc3230_pm_ops,
+	},
+	.probe		= mc3230_probe,
+	.remove		= mc3230_remove,
+	.id_table	= mc3230_i2c_id,
+};
+
+module_i2c_driver(mc3230_driver);
+
+MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
+MODULE_DESCRIPTION("mCube MC3230 3-Axis Accelerometer driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/accel/mma7660.c b/drivers/iio/accel/mma7660.c
index 0acdee516973..03beadf14ad3 100644
--- a/drivers/iio/accel/mma7660.c
+++ b/drivers/iio/accel/mma7660.c
@@ -251,6 +251,7 @@ static const struct i2c_device_id mma7660_i2c_id[] = {
 	{"mma7660", 0},
 	{}
 };
+MODULE_DEVICE_TABLE(i2c, mma7660_i2c_id);
 
 static const struct acpi_device_id mma7660_acpi_id[] = {
 	{"MMA7660", 0},
diff --git a/drivers/iio/accel/mxc6255.c b/drivers/iio/accel/mxc6255.c
index 97ccde722e7b..0abad6948201 100644
--- a/drivers/iio/accel/mxc6255.c
+++ b/drivers/iio/accel/mxc6255.c
@@ -154,7 +154,7 @@ static int mxc6255_probe(struct i2c_client *client,
 		return ret;
 	}
 
-	if (chip_id != MXC6255_CHIP_ID) {
+	if ((chip_id & 0x1f) != MXC6255_CHIP_ID) {
 		dev_err(&client->dev, "Invalid chip id %x\n", chip_id);
 		return -ENODEV;
 	}
@@ -171,12 +171,14 @@ static int mxc6255_probe(struct i2c_client *client,
 }
 
 static const struct acpi_device_id mxc6255_acpi_match[] = {
+	{"MXC6225",	0},
 	{"MXC6255",	0},
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, mxc6255_acpi_match);
 
 static const struct i2c_device_id mxc6255_id[] = {
+	{"mxc6225",	0},
 	{"mxc6255",	0},
 	{ }
 };
diff --git a/drivers/iio/accel/ssp_accel_sensor.c b/drivers/iio/accel/ssp_accel_sensor.c
index 4ae05fce9f24..31db00970fa0 100644
--- a/drivers/iio/accel/ssp_accel_sensor.c
+++ b/drivers/iio/accel/ssp_accel_sensor.c
@@ -74,7 +74,7 @@ static int ssp_accel_write_raw(struct iio_dev *indio_dev,
 	return -EINVAL;
 }
 
-static struct iio_info ssp_accel_iio_info = {
+static const struct iio_info ssp_accel_iio_info = {
 	.read_raw = &ssp_accel_read_raw,
 	.write_raw = &ssp_accel_write_raw,
 };
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 1de31bdd4ce4..7edcf3238620 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -264,6 +264,15 @@ config LPC18XX_ADC
 	  To compile this driver as a module, choose M here: the module will be
 	  called lpc18xx_adc.
 
+config LTC2485
+	tristate "Linear Technology LTC2485 ADC driver"
+	depends on I2C
+	help
+	  Say yes here to build support for Linear Technology LTC2485 ADC.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called ltc2485.
+
 config MAX1027
 	tristate "Maxim max1027 ADC driver"
 	depends on SPI
@@ -317,6 +326,19 @@ config MCP3422
 	  This driver can also be built as a module. If so, the module will be
 	  called mcp3422.
 
+config MEDIATEK_MT6577_AUXADC
+        tristate "MediaTek AUXADC driver"
+        depends on ARCH_MEDIATEK || COMPILE_TEST
+        depends on HAS_IOMEM
+        help
+          Say yes here to enable support for MediaTek mt65xx AUXADC.
+
+          The driver supports immediate mode operation to read from one of sixteen
+          channels (external or internal).
+
+          This driver can also be built as a module. If so, the module will be
+          called mt6577_auxadc.
+
 config MEN_Z188_ADC
 	tristate "MEN 16z188 ADC IP Core support"
 	depends on MCB
@@ -389,6 +411,7 @@ config QCOM_SPMI_VADC
 config ROCKCHIP_SARADC
 	tristate "Rockchip SARADC driver"
 	depends on ARCH_ROCKCHIP || (ARM && COMPILE_TEST)
+	depends on RESET_CONTROLLER
 	help
 	  Say yes here to build support for the SARADC found in SoCs from
 	  Rockchip.
@@ -396,6 +419,21 @@ config ROCKCHIP_SARADC
 	  To compile this driver as a module, choose M here: the
 	  module will be called rockchip_saradc.
 
+config STX104
+	tristate "Apex Embedded Systems STX104 driver"
+	depends on X86 && ISA_BUS_API
+	select GPIOLIB
+	help
+	  Say yes here to build support for the Apex Embedded Systems STX104
+	  integrated analog PC/104 card.
+
+	  This driver supports the 16 channels of single-ended (8 channels of
+	  differential) analog inputs, 2 channels of analog output, 4 digital
+	  inputs, and 4 digital outputs provided by the STX104.
+
+	  The base port addresses for the devices may be configured via the base
+	  array module parameter.
+
 config TI_ADC081C
 	tristate "Texas Instruments ADC081C/ADC101C/ADC121C family"
 	depends on I2C
@@ -416,6 +454,18 @@ config TI_ADC0832
 	  This driver can also be built as a module. If so, the module will be
 	  called ti-adc0832.
 
+config TI_ADC12138
+	tristate "Texas Instruments ADC12130/ADC12132/ADC12138"
+	depends on SPI
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	help
+	  If you say yes here you get support for Texas Instruments ADC12130,
+	  ADC12132 and ADC12138 chips.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called ti-adc12138.
+
 config TI_ADC128S052
 	tristate "Texas Instruments ADC128S052/ADC122S021/ADC124S021"
 	depends on SPI
@@ -426,6 +476,18 @@ config TI_ADC128S052
 	  This driver can also be built as a module. If so, the module will be
 	  called ti-adc128s052.
 
+config TI_ADC161S626
+	tristate "Texas Instruments ADC161S626 1-channel differential ADC"
+	depends on SPI
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	help
+	  If you say yes here you get support for Texas Instruments ADC141S626,
+	  and ADC161S626 chips.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called ti-adc161s626.
+
 config TI_ADS1015
 	tristate "Texas Instruments ADS1015 ADC"
 	depends on I2C && !SENSORS_ADS1015
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index 0ba0d500eedb..7a40c04c311f 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -27,10 +27,12 @@ obj-$(CONFIG_IMX7D_ADC) += imx7d_adc.o
 obj-$(CONFIG_INA2XX_ADC) += ina2xx-adc.o
 obj-$(CONFIG_LP8788_ADC) += lp8788_adc.o
 obj-$(CONFIG_LPC18XX_ADC) += lpc18xx_adc.o
+obj-$(CONFIG_LTC2485) += ltc2485.o
 obj-$(CONFIG_MAX1027) += max1027.o
 obj-$(CONFIG_MAX1363) += max1363.o
 obj-$(CONFIG_MCP320X) += mcp320x.o
 obj-$(CONFIG_MCP3422) += mcp3422.o
+obj-$(CONFIG_MEDIATEK_MT6577_AUXADC) += mt6577_auxadc.o
 obj-$(CONFIG_MEN_Z188_ADC) += men_z188_adc.o
 obj-$(CONFIG_MXS_LRADC) += mxs-lradc.o
 obj-$(CONFIG_NAU7802) += nau7802.o
@@ -38,9 +40,12 @@ obj-$(CONFIG_PALMAS_GPADC) += palmas_gpadc.o
 obj-$(CONFIG_QCOM_SPMI_IADC) += qcom-spmi-iadc.o
 obj-$(CONFIG_QCOM_SPMI_VADC) += qcom-spmi-vadc.o
 obj-$(CONFIG_ROCKCHIP_SARADC) += rockchip_saradc.o
+obj-$(CONFIG_STX104) += stx104.o
 obj-$(CONFIG_TI_ADC081C) += ti-adc081c.o
 obj-$(CONFIG_TI_ADC0832) += ti-adc0832.o
+obj-$(CONFIG_TI_ADC12138) += ti-adc12138.o
 obj-$(CONFIG_TI_ADC128S052) += ti-adc128s052.o
+obj-$(CONFIG_TI_ADC161S626) += ti-adc161s626.o
 obj-$(CONFIG_TI_ADS1015) += ti-ads1015.o
 obj-$(CONFIG_TI_ADS8688) += ti-ads8688.o
 obj-$(CONFIG_TI_AM335X_ADC) += ti_am335x_adc.o
diff --git a/drivers/iio/adc/ad7266.c b/drivers/iio/adc/ad7266.c
index c0f6a98fd9bd..b8d5cfd57ec4 100644
--- a/drivers/iio/adc/ad7266.c
+++ b/drivers/iio/adc/ad7266.c
@@ -481,7 +481,7 @@ error_free_gpios:
 	if (!st->fixed_addr)
 		gpio_free_array(st->gpios, ARRAY_SIZE(st->gpios));
 error_disable_reg:
-	if (!IS_ERR_OR_NULL(st->reg))
+	if (!IS_ERR(st->reg))
 		regulator_disable(st->reg);
 
 	return ret;
@@ -496,7 +496,7 @@ static int ad7266_remove(struct spi_device *spi)
 	iio_triggered_buffer_cleanup(indio_dev);
 	if (!st->fixed_addr)
 		gpio_free_array(st->gpios, ARRAY_SIZE(st->gpios));
-	if (!IS_ERR_OR_NULL(st->reg))
+	if (!IS_ERR(st->reg))
 		regulator_disable(st->reg);
 
 	return 0;
diff --git a/drivers/iio/adc/ad7298.c b/drivers/iio/adc/ad7298.c
index 10ec8fce395f..e399bf04c73a 100644
--- a/drivers/iio/adc/ad7298.c
+++ b/drivers/iio/adc/ad7298.c
@@ -239,16 +239,16 @@ static int ad7298_read_raw(struct iio_dev *indio_dev,
 
 	switch (m) {
 	case IIO_CHAN_INFO_RAW:
-		mutex_lock(&indio_dev->mlock);
-		if (indio_dev->currentmode == INDIO_BUFFER_TRIGGERED) {
-			ret = -EBUSY;
-		} else {
-			if (chan->address == AD7298_CH_TEMP)
-				ret = ad7298_scan_temp(st, val);
-			else
-				ret = ad7298_scan_direct(st, chan->address);
-		}
-		mutex_unlock(&indio_dev->mlock);
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		if (chan->address == AD7298_CH_TEMP)
+			ret = ad7298_scan_temp(st, val);
+		else
+			ret = ad7298_scan_direct(st, chan->address);
+
+		iio_device_release_direct_mode(indio_dev);
 
 		if (ret < 0)
 			return ret;
diff --git a/drivers/iio/adc/ad7793.c b/drivers/iio/adc/ad7793.c
index 847789bae821..e6706a09e100 100644
--- a/drivers/iio/adc/ad7793.c
+++ b/drivers/iio/adc/ad7793.c
@@ -519,11 +519,9 @@ static int ad7793_write_raw(struct iio_dev *indio_dev,
 	int ret, i;
 	unsigned int tmp;
 
-	mutex_lock(&indio_dev->mlock);
-	if (iio_buffer_enabled(indio_dev)) {
-		mutex_unlock(&indio_dev->mlock);
-		return -EBUSY;
-	}
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret)
+		return ret;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_SCALE:
@@ -548,7 +546,7 @@ static int ad7793_write_raw(struct iio_dev *indio_dev,
 		ret = -EINVAL;
 	}
 
-	mutex_unlock(&indio_dev->mlock);
+	iio_device_release_direct_mode(indio_dev);
 	return ret;
 }
 
diff --git a/drivers/iio/adc/ad799x.c b/drivers/iio/adc/ad799x.c
index b6163764489c..9704090b7908 100644
--- a/drivers/iio/adc/ad799x.c
+++ b/drivers/iio/adc/ad799x.c
@@ -527,6 +527,7 @@ static struct attribute_group ad799x_event_attrs_group = {
 static const struct iio_info ad7991_info = {
 	.read_raw = &ad799x_read_raw,
 	.driver_module = THIS_MODULE,
+	.update_scan_mode = ad799x_update_scan_mode,
 };
 
 static const struct iio_info ad7993_4_7_8_noirq_info = {
diff --git a/drivers/iio/adc/at91_adc.c b/drivers/iio/adc/at91_adc.c
index 52430ba171f3..bbdac07f4aaa 100644
--- a/drivers/iio/adc/at91_adc.c
+++ b/drivers/iio/adc/at91_adc.c
@@ -113,6 +113,7 @@
 #define		AT91_ADC_TSMR_TSAV	(3 << 4)	/* Averages samples */
 #define			AT91_ADC_TSMR_TSAV_(x)		((x) << 4)
 #define		AT91_ADC_TSMR_SCTIM	(0x0f << 16)	/* Switch closure time */
+#define			AT91_ADC_TSMR_SCTIM_(x)		((x) << 16)
 #define		AT91_ADC_TSMR_PENDBC	(0x0f << 28)	/* Pen Debounce time */
 #define			AT91_ADC_TSMR_PENDBC_(x)	((x) << 28)
 #define		AT91_ADC_TSMR_NOTSDMA	(1 << 22)	/* No Touchscreen DMA */
@@ -150,6 +151,7 @@
 #define MAX_RLPOS_BITS         10
 #define TOUCH_SAMPLE_PERIOD_US_RL      10000   /* 10ms, the SoC can't keep up with 2ms */
 #define TOUCH_SHTIM                    0xa
+#define TOUCH_SCTIM_US		10		/* 10us for the Touchscreen Switches Closure Time */
 
 /**
  * struct at91_adc_reg_desc - Various informations relative to registers
@@ -381,8 +383,8 @@ static irqreturn_t at91_adc_rl_interrupt(int irq, void *private)
 		st->ts_bufferedmeasure = false;
 		input_report_key(st->ts_input, BTN_TOUCH, 0);
 		input_sync(st->ts_input);
-	} else if (status & AT91_ADC_EOC(3)) {
-		/* Conversion finished */
+	} else if (status & AT91_ADC_EOC(3) && st->ts_input) {
+		/* Conversion finished and we've a touchscreen */
 		if (st->ts_bufferedmeasure) {
 			/*
 			 * Last measurement is always discarded, since it can
@@ -1001,7 +1003,9 @@ static void atmel_ts_close(struct input_dev *dev)
 
 static int at91_ts_hw_init(struct at91_adc_state *st, u32 adc_clk_khz)
 {
+	struct iio_dev *idev = iio_priv_to_dev(st);
 	u32 reg = 0;
+	u32 tssctim = 0;
 	int i = 0;
 
 	/* a Pen Detect Debounce Time is necessary for the ADC Touch to avoid
@@ -1034,11 +1038,20 @@ static int at91_ts_hw_init(struct at91_adc_state *st, u32 adc_clk_khz)
 		return 0;
 	}
 
+	/* Touchscreen Switches Closure time needed for allowing the value to
+	 * stabilize.
+	 * Switch Closure Time = (TSSCTIM * 4) ADCClock periods
+	 */
+	tssctim = DIV_ROUND_UP(TOUCH_SCTIM_US * adc_clk_khz / 1000, 4);
+	dev_dbg(&idev->dev, "adc_clk at: %d KHz, tssctim at: %d\n",
+		adc_clk_khz, tssctim);
+
 	if (st->touchscreen_type == ATMEL_ADC_TOUCHSCREEN_4WIRE)
 		reg = AT91_ADC_TSMR_TSMODE_4WIRE_PRESS;
 	else
 		reg = AT91_ADC_TSMR_TSMODE_5WIRE;
 
+	reg |= AT91_ADC_TSMR_SCTIM_(tssctim) & AT91_ADC_TSMR_SCTIM;
 	reg |= AT91_ADC_TSMR_TSAV_(st->caps->ts_filter_average)
 	       & AT91_ADC_TSMR_TSAV;
 	reg |= AT91_ADC_TSMR_PENDBC_(st->ts_pendbc) & AT91_ADC_TSMR_PENDBC;
diff --git a/drivers/iio/adc/ina2xx-adc.c b/drivers/iio/adc/ina2xx-adc.c
index 955f3fdaf519..59b7d76e1ad2 100644
--- a/drivers/iio/adc/ina2xx-adc.c
+++ b/drivers/iio/adc/ina2xx-adc.c
@@ -114,7 +114,6 @@ struct ina2xx_chip_info {
 	struct mutex state_lock;
 	unsigned int shunt_resistor;
 	int avg;
-	s64 prev_ns; /* track buffer capture time, check for underruns */
 	int int_time_vbus; /* Bus voltage integration time uS */
 	int int_time_vshunt; /* Shunt voltage integration time uS */
 	bool allow_async_readout;
@@ -509,8 +508,6 @@ static int ina2xx_work_buffer(struct iio_dev *indio_dev)
 	iio_push_to_buffers_with_timestamp(indio_dev,
 					   (unsigned int *)data, time_a);
 
-	chip->prev_ns = time_a;
-
 	return (unsigned long)(time_b - time_a) / 1000;
 };
 
@@ -554,8 +551,6 @@ static int ina2xx_buffer_enable(struct iio_dev *indio_dev)
 	dev_dbg(&indio_dev->dev, "Async readout mode: %d\n",
 		chip->allow_async_readout);
 
-	chip->prev_ns = iio_get_time_ns(indio_dev);
-
 	chip->task = kthread_run(ina2xx_capture_thread, (void *)indio_dev,
 				 "%s:%d-%uus", indio_dev->name, indio_dev->id,
 				 sampling_us);
diff --git a/drivers/iio/adc/ltc2485.c b/drivers/iio/adc/ltc2485.c
new file mode 100644
index 000000000000..eab91f12454a
--- /dev/null
+++ b/drivers/iio/adc/ltc2485.c
@@ -0,0 +1,148 @@
+/*
+ * ltc2485.c - Driver for Linear Technology LTC2485 ADC
+ *
+ * Copyright (C) 2016 Alison Schofield <amsfield22@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Datasheet: http://cds.linear.com/docs/en/datasheet/2485fd.pdf
+ */
+
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+/* Power-on configuration: rejects both 50/60Hz, operates at 1x speed */
+#define LTC2485_CONFIG_DEFAULT		0
+
+struct ltc2485_data {
+	struct i2c_client	*client;
+	ktime_t			time_prev;	/* last conversion */
+};
+
+static void ltc2485_wait_conv(struct ltc2485_data *data)
+{
+	const unsigned int conv_time = 147;	/* conversion time ms */
+	unsigned int time_elapsed;
+
+	/* delay if conversion time not passed since last read or write */
+	time_elapsed = ktime_ms_delta(ktime_get(), data->time_prev);
+
+	if (time_elapsed < conv_time)
+		msleep(conv_time - time_elapsed);
+}
+
+static int ltc2485_read(struct ltc2485_data *data, int *val)
+{
+	struct i2c_client *client = data->client;
+	__be32 buf = 0;
+	int ret;
+
+	ltc2485_wait_conv(data);
+
+	ret = i2c_master_recv(client, (char *)&buf, 4);
+	if (ret < 0)  {
+		dev_err(&client->dev, "i2c_master_recv failed\n");
+		return ret;
+	}
+	data->time_prev = ktime_get();
+	*val = sign_extend32(be32_to_cpu(buf) >> 6, 24);
+
+	return ret;
+}
+
+static int ltc2485_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
+{
+	struct ltc2485_data *data = iio_priv(indio_dev);
+	int ret;
+
+	if (mask == IIO_CHAN_INFO_RAW) {
+		ret = ltc2485_read(data, val);
+		if (ret < 0)
+			return ret;
+
+		return IIO_VAL_INT;
+
+	} else if (mask == IIO_CHAN_INFO_SCALE) {
+		*val = 5000;			/* on board vref millivolts */
+		*val2 = 25;			/* 25 (24 + sign) data bits */
+		return IIO_VAL_FRACTIONAL_LOG2;
+
+	} else {
+		return -EINVAL;
+	}
+}
+
+static const struct iio_chan_spec ltc2485_channel[] = {
+	{
+		.type = IIO_VOLTAGE,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE)
+	},
+};
+
+static const struct iio_info ltc2485_info = {
+	.read_raw = ltc2485_read_raw,
+	.driver_module = THIS_MODULE,
+};
+
+static int ltc2485_probe(struct i2c_client *client,
+			 const struct i2c_device_id *id)
+{
+	struct iio_dev *indio_dev;
+	struct ltc2485_data *data;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C |
+				     I2C_FUNC_SMBUS_WRITE_BYTE))
+		return -EOPNOTSUPP;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	i2c_set_clientdata(client, indio_dev);
+	data->client = client;
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->name = id->name;
+	indio_dev->info = &ltc2485_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = ltc2485_channel;
+	indio_dev->num_channels = ARRAY_SIZE(ltc2485_channel);
+
+	ret = i2c_smbus_write_byte(data->client, LTC2485_CONFIG_DEFAULT);
+	if (ret < 0)
+		return ret;
+
+	data->time_prev = ktime_get();
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+static const struct i2c_device_id ltc2485_id[] = {
+	{ "ltc2485", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, ltc2485_id);
+
+static struct i2c_driver ltc2485_driver = {
+	.driver = {
+		.name = "ltc2485",
+	},
+	.probe = ltc2485_probe,
+	.id_table = ltc2485_id,
+};
+module_i2c_driver(ltc2485_driver);
+
+MODULE_AUTHOR("Alison Schofield <amsfield22@gmail.com>");
+MODULE_DESCRIPTION("Linear Technology LTC2485 ADC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/men_z188_adc.c b/drivers/iio/adc/men_z188_adc.c
index d095efe1ba14..8f3606de4eaf 100644
--- a/drivers/iio/adc/men_z188_adc.c
+++ b/drivers/iio/adc/men_z188_adc.c
@@ -78,7 +78,7 @@ static int z188_iio_read_raw(struct iio_dev *iio_dev,
 	return ret;
 }
 
-static struct iio_info z188_adc_info = {
+static const struct iio_info z188_adc_info = {
 	.read_raw = &z188_iio_read_raw,
 	.driver_module = THIS_MODULE,
 };
diff --git a/drivers/iio/adc/mt6577_auxadc.c b/drivers/iio/adc/mt6577_auxadc.c
new file mode 100644
index 000000000000..2d104c828041
--- /dev/null
+++ b/drivers/iio/adc/mt6577_auxadc.c
@@ -0,0 +1,291 @@
+/*
+ * Copyright (c) 2016 MediaTek Inc.
+ * Author: Zhiyong Tao <zhiyong.tao@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/iopoll.h>
+#include <linux/io.h>
+#include <linux/iio/iio.h>
+
+/* Register definitions */
+#define MT6577_AUXADC_CON0                    0x00
+#define MT6577_AUXADC_CON1                    0x04
+#define MT6577_AUXADC_CON2                    0x10
+#define MT6577_AUXADC_STA                     BIT(0)
+
+#define MT6577_AUXADC_DAT0                    0x14
+#define MT6577_AUXADC_RDY0                    BIT(12)
+
+#define MT6577_AUXADC_MISC                    0x94
+#define MT6577_AUXADC_PDN_EN                  BIT(14)
+
+#define MT6577_AUXADC_DAT_MASK                0xfff
+#define MT6577_AUXADC_SLEEP_US                1000
+#define MT6577_AUXADC_TIMEOUT_US              10000
+#define MT6577_AUXADC_POWER_READY_MS          1
+#define MT6577_AUXADC_SAMPLE_READY_US         25
+
+struct mt6577_auxadc_device {
+	void __iomem *reg_base;
+	struct clk *adc_clk;
+	struct mutex lock;
+};
+
+#define MT6577_AUXADC_CHANNEL(idx) {				    \
+		.type = IIO_VOLTAGE,				    \
+		.indexed = 1,					    \
+		.channel = (idx),				    \
+		.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED), \
+}
+
+static const struct iio_chan_spec mt6577_auxadc_iio_channels[] = {
+	MT6577_AUXADC_CHANNEL(0),
+	MT6577_AUXADC_CHANNEL(1),
+	MT6577_AUXADC_CHANNEL(2),
+	MT6577_AUXADC_CHANNEL(3),
+	MT6577_AUXADC_CHANNEL(4),
+	MT6577_AUXADC_CHANNEL(5),
+	MT6577_AUXADC_CHANNEL(6),
+	MT6577_AUXADC_CHANNEL(7),
+	MT6577_AUXADC_CHANNEL(8),
+	MT6577_AUXADC_CHANNEL(9),
+	MT6577_AUXADC_CHANNEL(10),
+	MT6577_AUXADC_CHANNEL(11),
+	MT6577_AUXADC_CHANNEL(12),
+	MT6577_AUXADC_CHANNEL(13),
+	MT6577_AUXADC_CHANNEL(14),
+	MT6577_AUXADC_CHANNEL(15),
+};
+
+static inline void mt6577_auxadc_mod_reg(void __iomem *reg,
+					 u32 or_mask, u32 and_mask)
+{
+	u32 val;
+
+	val = readl(reg);
+	val |= or_mask;
+	val &= ~and_mask;
+	writel(val, reg);
+}
+
+static int mt6577_auxadc_read(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan)
+{
+	u32 val;
+	void __iomem *reg_channel;
+	int ret;
+	struct mt6577_auxadc_device *adc_dev = iio_priv(indio_dev);
+
+	reg_channel = adc_dev->reg_base + MT6577_AUXADC_DAT0 +
+		      chan->channel * 0x04;
+
+	mutex_lock(&adc_dev->lock);
+
+	mt6577_auxadc_mod_reg(adc_dev->reg_base + MT6577_AUXADC_CON1,
+			      0, 1 << chan->channel);
+
+	/* read channel and make sure old ready bit == 0 */
+	ret = readl_poll_timeout(reg_channel, val,
+				 ((val & MT6577_AUXADC_RDY0) == 0),
+				 MT6577_AUXADC_SLEEP_US,
+				 MT6577_AUXADC_TIMEOUT_US);
+	if (ret < 0) {
+		dev_err(indio_dev->dev.parent,
+			"wait for channel[%d] ready bit clear time out\n",
+			chan->channel);
+		goto err_timeout;
+	}
+
+	/* set bit to trigger sample */
+	mt6577_auxadc_mod_reg(adc_dev->reg_base + MT6577_AUXADC_CON1,
+			      1 << chan->channel, 0);
+
+	/* we must delay here for hardware sample channel data */
+	udelay(MT6577_AUXADC_SAMPLE_READY_US);
+
+	/* check MTK_AUXADC_CON2 if auxadc is idle */
+	ret = readl_poll_timeout(adc_dev->reg_base + MT6577_AUXADC_CON2, val,
+				 ((val & MT6577_AUXADC_STA) == 0),
+				 MT6577_AUXADC_SLEEP_US,
+				 MT6577_AUXADC_TIMEOUT_US);
+	if (ret < 0) {
+		dev_err(indio_dev->dev.parent,
+			"wait for auxadc idle time out\n");
+		goto err_timeout;
+	}
+
+	/* read channel and make sure ready bit == 1 */
+	ret = readl_poll_timeout(reg_channel, val,
+				 ((val & MT6577_AUXADC_RDY0) != 0),
+				 MT6577_AUXADC_SLEEP_US,
+				 MT6577_AUXADC_TIMEOUT_US);
+	if (ret < 0) {
+		dev_err(indio_dev->dev.parent,
+			"wait for channel[%d] data ready time out\n",
+			chan->channel);
+		goto err_timeout;
+	}
+
+	/* read data */
+	val = readl(reg_channel) & MT6577_AUXADC_DAT_MASK;
+
+	mutex_unlock(&adc_dev->lock);
+
+	return val;
+
+err_timeout:
+
+	mutex_unlock(&adc_dev->lock);
+
+	return -ETIMEDOUT;
+}
+
+static int mt6577_auxadc_read_raw(struct iio_dev *indio_dev,
+				  struct iio_chan_spec const *chan,
+				  int *val,
+				  int *val2,
+				  long info)
+{
+	switch (info) {
+	case IIO_CHAN_INFO_PROCESSED:
+		*val = mt6577_auxadc_read(indio_dev, chan);
+		if (*val < 0) {
+			dev_err(indio_dev->dev.parent,
+				"failed to sample data on channel[%d]\n",
+				chan->channel);
+			return *val;
+		}
+		return IIO_VAL_INT;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info mt6577_auxadc_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = &mt6577_auxadc_read_raw,
+};
+
+static int mt6577_auxadc_probe(struct platform_device *pdev)
+{
+	struct mt6577_auxadc_device *adc_dev;
+	unsigned long adc_clk_rate;
+	struct resource *res;
+	struct iio_dev *indio_dev;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*adc_dev));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	adc_dev = iio_priv(indio_dev);
+	indio_dev->dev.parent = &pdev->dev;
+	indio_dev->name = dev_name(&pdev->dev);
+	indio_dev->info = &mt6577_auxadc_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = mt6577_auxadc_iio_channels;
+	indio_dev->num_channels = ARRAY_SIZE(mt6577_auxadc_iio_channels);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	adc_dev->reg_base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(adc_dev->reg_base)) {
+		dev_err(&pdev->dev, "failed to get auxadc base address\n");
+		return PTR_ERR(adc_dev->reg_base);
+	}
+
+	adc_dev->adc_clk = devm_clk_get(&pdev->dev, "main");
+	if (IS_ERR(adc_dev->adc_clk)) {
+		dev_err(&pdev->dev, "failed to get auxadc clock\n");
+		return PTR_ERR(adc_dev->adc_clk);
+	}
+
+	ret = clk_prepare_enable(adc_dev->adc_clk);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to enable auxadc clock\n");
+		return ret;
+	}
+
+	adc_clk_rate = clk_get_rate(adc_dev->adc_clk);
+	if (!adc_clk_rate) {
+		ret = -EINVAL;
+		dev_err(&pdev->dev, "null clock rate\n");
+		goto err_disable_clk;
+	}
+
+	mutex_init(&adc_dev->lock);
+
+	mt6577_auxadc_mod_reg(adc_dev->reg_base + MT6577_AUXADC_MISC,
+			      MT6577_AUXADC_PDN_EN, 0);
+	mdelay(MT6577_AUXADC_POWER_READY_MS);
+
+	platform_set_drvdata(pdev, indio_dev);
+
+	ret = iio_device_register(indio_dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to register iio device\n");
+		goto err_power_off;
+	}
+
+	return 0;
+
+err_power_off:
+	mt6577_auxadc_mod_reg(adc_dev->reg_base + MT6577_AUXADC_MISC,
+			      0, MT6577_AUXADC_PDN_EN);
+err_disable_clk:
+	clk_disable_unprepare(adc_dev->adc_clk);
+	return ret;
+}
+
+static int mt6577_auxadc_remove(struct platform_device *pdev)
+{
+	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct mt6577_auxadc_device *adc_dev = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+
+	mt6577_auxadc_mod_reg(adc_dev->reg_base + MT6577_AUXADC_MISC,
+			      0, MT6577_AUXADC_PDN_EN);
+
+	clk_disable_unprepare(adc_dev->adc_clk);
+
+	return 0;
+}
+
+static const struct of_device_id mt6577_auxadc_of_match[] = {
+	{ .compatible = "mediatek,mt2701-auxadc", },
+	{ .compatible = "mediatek,mt8173-auxadc", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mt6577_auxadc_of_match);
+
+static struct platform_driver mt6577_auxadc_driver = {
+	.driver = {
+		.name   = "mt6577-auxadc",
+		.of_match_table = mt6577_auxadc_of_match,
+	},
+	.probe	= mt6577_auxadc_probe,
+	.remove	= mt6577_auxadc_remove,
+};
+module_platform_driver(mt6577_auxadc_driver);
+
+MODULE_AUTHOR("Zhiyong Tao <zhiyong.tao@mediatek.com>");
+MODULE_DESCRIPTION("MTK AUXADC Device Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/nau7802.c b/drivers/iio/adc/nau7802.c
index db9b829ccf0d..08f446695f97 100644
--- a/drivers/iio/adc/nau7802.c
+++ b/drivers/iio/adc/nau7802.c
@@ -197,7 +197,7 @@ static irqreturn_t nau7802_eoc_trigger(int irq, void *private)
 	if (st->conversion_count < NAU7802_MIN_CONVERSIONS)
 		st->conversion_count++;
 	if (st->conversion_count >= NAU7802_MIN_CONVERSIONS)
-		complete_all(&st->value_ok);
+		complete(&st->value_ok);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/iio/adc/rockchip_saradc.c b/drivers/iio/adc/rockchip_saradc.c
index f9ad6c2d6821..85d701291654 100644
--- a/drivers/iio/adc/rockchip_saradc.c
+++ b/drivers/iio/adc/rockchip_saradc.c
@@ -21,6 +21,8 @@
 #include <linux/of_device.h>
 #include <linux/clk.h>
 #include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/reset.h>
 #include <linux/regulator/consumer.h>
 #include <linux/iio/iio.h>
 
@@ -53,6 +55,7 @@ struct rockchip_saradc {
 	struct clk		*clk;
 	struct completion	completion;
 	struct regulator	*vref;
+	struct reset_control	*reset;
 	const struct rockchip_saradc_data *data;
 	u16			last_val;
 };
@@ -190,6 +193,16 @@ static const struct of_device_id rockchip_saradc_match[] = {
 };
 MODULE_DEVICE_TABLE(of, rockchip_saradc_match);
 
+/**
+ * Reset SARADC Controller.
+ */
+static void rockchip_saradc_reset_controller(struct reset_control *reset)
+{
+	reset_control_assert(reset);
+	usleep_range(10, 20);
+	reset_control_deassert(reset);
+}
+
 static int rockchip_saradc_probe(struct platform_device *pdev)
 {
 	struct rockchip_saradc *info = NULL;
@@ -218,6 +231,20 @@ static int rockchip_saradc_probe(struct platform_device *pdev)
 	if (IS_ERR(info->regs))
 		return PTR_ERR(info->regs);
 
+	/*
+	 * The reset should be an optional property, as it should work
+	 * with old devicetrees as well
+	 */
+	info->reset = devm_reset_control_get(&pdev->dev, "saradc-apb");
+	if (IS_ERR(info->reset)) {
+		ret = PTR_ERR(info->reset);
+		if (ret != -ENOENT)
+			return ret;
+
+		dev_dbg(&pdev->dev, "no reset control found\n");
+		info->reset = NULL;
+	}
+
 	init_completion(&info->completion);
 
 	irq = platform_get_irq(pdev, 0);
@@ -252,6 +279,9 @@ static int rockchip_saradc_probe(struct platform_device *pdev)
 		return PTR_ERR(info->vref);
 	}
 
+	if (info->reset)
+		rockchip_saradc_reset_controller(info->reset);
+
 	/*
 	 * Use a default value for the converter clock.
 	 * This may become user-configurable in the future.
diff --git a/drivers/iio/dac/stx104.c b/drivers/iio/adc/stx104.c
index 792a97164cb2..7e3645749eaf 100644
--- a/drivers/iio/dac/stx104.c
+++ b/drivers/iio/adc/stx104.c
@@ -1,5 +1,5 @@
 /*
- * DAC driver for the Apex Embedded Systems STX104
+ * IIO driver for the Apex Embedded Systems STX104
  * Copyright (C) 2016 William Breathitt Gray
  *
  * This program is free software; you can redistribute it and/or modify
@@ -20,19 +20,30 @@
 #include <linux/io.h>
 #include <linux/ioport.h>
 #include <linux/isa.h>
+#include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/spinlock.h>
 
-#define STX104_NUM_CHAN 2
-
-#define STX104_CHAN(chan) {				\
+#define STX104_OUT_CHAN(chan) {				\
 	.type = IIO_VOLTAGE,				\
 	.channel = chan,				\
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
 	.indexed = 1,					\
 	.output = 1					\
 }
+#define STX104_IN_CHAN(chan, diff) {					\
+	.type = IIO_VOLTAGE,						\
+	.channel = chan,						\
+	.channel2 = chan,						\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_HARDWAREGAIN) |	\
+		BIT(IIO_CHAN_INFO_OFFSET) | BIT(IIO_CHAN_INFO_SCALE),	\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),			\
+	.indexed = 1,							\
+	.differential = diff						\
+}
+
+#define STX104_NUM_OUT_CHAN 2
 
 #define STX104_EXTENT 16
 
@@ -47,8 +58,8 @@ MODULE_PARM_DESC(base, "Apex Embedded Systems STX104 base addresses");
  * @base:		base port address of the IIO device
  */
 struct stx104_iio {
-	unsigned chan_out_states[STX104_NUM_CHAN];
-	unsigned base;
+	unsigned int chan_out_states[STX104_NUM_OUT_CHAN];
+	unsigned int base;
 };
 
 /**
@@ -65,32 +76,109 @@ struct stx104_gpio {
 	unsigned int out_state;
 };
 
+/**
+ * struct stx104_dev - STX104 device private data structure
+ * @indio_dev:	IIO device
+ * @chip:	instance of the gpio_chip
+ */
+struct stx104_dev {
+	struct iio_dev *indio_dev;
+	struct gpio_chip *chip;
+};
+
 static int stx104_read_raw(struct iio_dev *indio_dev,
 	struct iio_chan_spec const *chan, int *val, int *val2, long mask)
 {
 	struct stx104_iio *const priv = iio_priv(indio_dev);
+	unsigned int adc_config;
+	int adbu;
+	int gain;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_HARDWAREGAIN:
+		/* get gain configuration */
+		adc_config = inb(priv->base + 11);
+		gain = adc_config & 0x3;
+
+		*val = 1 << gain;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_RAW:
+		if (chan->output) {
+			*val = priv->chan_out_states[chan->channel];
+			return IIO_VAL_INT;
+		}
+
+		/* select ADC channel */
+		outb(chan->channel | (chan->channel << 4), priv->base + 2);
+
+		/* trigger ADC sample capture and wait for completion */
+		outb(0, priv->base);
+		while (inb(priv->base + 8) & BIT(7));
+
+		*val = inw(priv->base);
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_OFFSET:
+		/* get ADC bipolar/unipolar configuration */
+		adc_config = inb(priv->base + 11);
+		adbu = !(adc_config & BIT(2));
+
+		*val = -32768 * adbu;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		/* get ADC bipolar/unipolar and gain configuration */
+		adc_config = inb(priv->base + 11);
+		adbu = !(adc_config & BIT(2));
+		gain = adc_config & 0x3;
+
+		*val = 5;
+		*val2 = 15 - adbu + gain;
+		return IIO_VAL_FRACTIONAL_LOG2;
+	}
 
-	if (mask != IIO_CHAN_INFO_RAW)
-		return -EINVAL;
-
-	*val = priv->chan_out_states[chan->channel];
-
-	return IIO_VAL_INT;
+	return -EINVAL;
 }
 
 static int stx104_write_raw(struct iio_dev *indio_dev,
 	struct iio_chan_spec const *chan, int val, int val2, long mask)
 {
 	struct stx104_iio *const priv = iio_priv(indio_dev);
-	const unsigned chan_addr_offset = 2 * chan->channel;
 
-	if (mask != IIO_CHAN_INFO_RAW)
+	switch (mask) {
+	case IIO_CHAN_INFO_HARDWAREGAIN:
+		/* Only four gain states (x1, x2, x4, x8) */
+		switch (val) {
+		case 1:
+			outb(0, priv->base + 11);
+			break;
+		case 2:
+			outb(1, priv->base + 11);
+			break;
+		case 4:
+			outb(2, priv->base + 11);
+			break;
+		case 8:
+			outb(3, priv->base + 11);
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		return 0;
+	case IIO_CHAN_INFO_RAW:
+		if (chan->output) {
+			/* DAC can only accept up to a 16-bit value */
+			if ((unsigned int)val > 65535)
+				return -EINVAL;
+
+			priv->chan_out_states[chan->channel] = val;
+			outw(val, priv->base + 4 + 2 * chan->channel);
+
+			return 0;
+		}
 		return -EINVAL;
+	}
 
-	priv->chan_out_states[chan->channel] = val;
-	outw(val, priv->base + 4 + chan_addr_offset);
-
-	return 0;
+	return -EINVAL;
 }
 
 static const struct iio_info stx104_info = {
@@ -99,14 +187,28 @@ static const struct iio_info stx104_info = {
 	.write_raw = stx104_write_raw
 };
 
-static const struct iio_chan_spec stx104_channels[STX104_NUM_CHAN] = {
-	STX104_CHAN(0),
-	STX104_CHAN(1)
+/* single-ended input channels configuration */
+static const struct iio_chan_spec stx104_channels_sing[] = {
+	STX104_OUT_CHAN(0), STX104_OUT_CHAN(1),
+	STX104_IN_CHAN(0, 0), STX104_IN_CHAN(1, 0), STX104_IN_CHAN(2, 0),
+	STX104_IN_CHAN(3, 0), STX104_IN_CHAN(4, 0), STX104_IN_CHAN(5, 0),
+	STX104_IN_CHAN(6, 0), STX104_IN_CHAN(7, 0), STX104_IN_CHAN(8, 0),
+	STX104_IN_CHAN(9, 0), STX104_IN_CHAN(10, 0), STX104_IN_CHAN(11, 0),
+	STX104_IN_CHAN(12, 0), STX104_IN_CHAN(13, 0), STX104_IN_CHAN(14, 0),
+	STX104_IN_CHAN(15, 0)
+};
+/* differential input channels configuration */
+static const struct iio_chan_spec stx104_channels_diff[] = {
+	STX104_OUT_CHAN(0), STX104_OUT_CHAN(1),
+	STX104_IN_CHAN(0, 1), STX104_IN_CHAN(1, 1), STX104_IN_CHAN(2, 1),
+	STX104_IN_CHAN(3, 1), STX104_IN_CHAN(4, 1), STX104_IN_CHAN(5, 1),
+	STX104_IN_CHAN(6, 1), STX104_IN_CHAN(7, 1)
 };
 
 static int stx104_gpio_get_direction(struct gpio_chip *chip,
 	unsigned int offset)
 {
+	/* GPIO 0-3 are input only, while the rest are output only */
 	if (offset < 4)
 		return 1;
 
@@ -169,6 +271,7 @@ static int stx104_probe(struct device *dev, unsigned int id)
 	struct iio_dev *indio_dev;
 	struct stx104_iio *priv;
 	struct stx104_gpio *stx104gpio;
+	struct stx104_dev *stx104dev;
 	int err;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*priv));
@@ -179,6 +282,10 @@ static int stx104_probe(struct device *dev, unsigned int id)
 	if (!stx104gpio)
 		return -ENOMEM;
 
+	stx104dev = devm_kzalloc(dev, sizeof(*stx104dev), GFP_KERNEL);
+	if (!stx104dev)
+		return -ENOMEM;
+
 	if (!devm_request_region(dev, base[id], STX104_EXTENT,
 		dev_name(dev))) {
 		dev_err(dev, "Unable to lock port addresses (0x%X-0x%X)\n",
@@ -188,23 +295,31 @@ static int stx104_probe(struct device *dev, unsigned int id)
 
 	indio_dev->info = &stx104_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->channels = stx104_channels;
-	indio_dev->num_channels = STX104_NUM_CHAN;
+
+	/* determine if differential inputs */
+	if (inb(base[id] + 8) & BIT(5)) {
+		indio_dev->num_channels = ARRAY_SIZE(stx104_channels_diff);
+		indio_dev->channels = stx104_channels_diff;
+	} else {
+		indio_dev->num_channels = ARRAY_SIZE(stx104_channels_sing);
+		indio_dev->channels = stx104_channels_sing;
+	}
+
 	indio_dev->name = dev_name(dev);
 
 	priv = iio_priv(indio_dev);
 	priv->base = base[id];
 
+	/* configure device for software trigger operation */
+	outb(0, base[id] + 9);
+
+	/* initialize gain setting to x1 */
+	outb(0, base[id] + 11);
+
 	/* initialize DAC output to 0V */
 	outw(0, base[id] + 4);
 	outw(0, base[id] + 6);
 
-	err = devm_iio_device_register(dev, indio_dev);
-	if (err) {
-		dev_err(dev, "IIO device registering failed (%d)\n", err);
-		return err;
-	}
-
 	stx104gpio->chip.label = dev_name(dev);
 	stx104gpio->chip.parent = dev;
 	stx104gpio->chip.owner = THIS_MODULE;
@@ -220,7 +335,9 @@ static int stx104_probe(struct device *dev, unsigned int id)
 
 	spin_lock_init(&stx104gpio->lock);
 
-	dev_set_drvdata(dev, stx104gpio);
+	stx104dev->indio_dev = indio_dev;
+	stx104dev->chip = &stx104gpio->chip;
+	dev_set_drvdata(dev, stx104dev);
 
 	err = gpiochip_add_data(&stx104gpio->chip, stx104gpio);
 	if (err) {
@@ -228,14 +345,22 @@ static int stx104_probe(struct device *dev, unsigned int id)
 		return err;
 	}
 
+	err = iio_device_register(indio_dev);
+	if (err) {
+		dev_err(dev, "IIO device registering failed (%d)\n", err);
+		gpiochip_remove(&stx104gpio->chip);
+		return err;
+	}
+
 	return 0;
 }
 
 static int stx104_remove(struct device *dev, unsigned int id)
 {
-	struct stx104_gpio *const stx104gpio = dev_get_drvdata(dev);
+	struct stx104_dev *const stx104dev = dev_get_drvdata(dev);
 
-	gpiochip_remove(&stx104gpio->chip);
+	iio_device_unregister(stx104dev->indio_dev);
+	gpiochip_remove(stx104dev->chip);
 
 	return 0;
 }
@@ -251,5 +376,5 @@ static struct isa_driver stx104_driver = {
 module_isa_driver(stx104_driver, num_stx104);
 
 MODULE_AUTHOR("William Breathitt Gray <vilhelm.gray@gmail.com>");
-MODULE_DESCRIPTION("Apex Embedded Systems STX104 DAC driver");
+MODULE_DESCRIPTION("Apex Embedded Systems STX104 IIO driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/ti-adc12138.c b/drivers/iio/adc/ti-adc12138.c
new file mode 100644
index 000000000000..072f03bfe6a0
--- /dev/null
+++ b/drivers/iio/adc/ti-adc12138.c
@@ -0,0 +1,552 @@
+/*
+ * ADC12130/ADC12132/ADC12138 12-bit plus sign ADC driver
+ *
+ * Copyright (c) 2016 Akinobu Mita <akinobu.mita@gmail.com>
+ *
+ * This file is subject to the terms and conditions of version 2 of
+ * the GNU General Public License.  See the file COPYING in the main
+ * directory of this archive for more details.
+ *
+ * Datasheet: http://www.ti.com/lit/ds/symlink/adc12138.pdf
+ */
+
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/completion.h>
+#include <linux/clk.h>
+#include <linux/spi/spi.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/regulator/consumer.h>
+
+#define ADC12138_MODE_AUTO_CAL			0x08
+#define ADC12138_MODE_READ_STATUS		0x0c
+#define ADC12138_MODE_ACQUISITION_TIME_6	0x0e
+#define ADC12138_MODE_ACQUISITION_TIME_10	0x4e
+#define ADC12138_MODE_ACQUISITION_TIME_18	0x8e
+#define ADC12138_MODE_ACQUISITION_TIME_34	0xce
+
+#define ADC12138_STATUS_CAL			BIT(6)
+
+enum {
+	adc12130,
+	adc12132,
+	adc12138,
+};
+
+struct adc12138 {
+	struct spi_device *spi;
+	unsigned int id;
+	/* conversion clock */
+	struct clk *cclk;
+	/* positive analog voltage reference */
+	struct regulator *vref_p;
+	/* negative analog voltage reference */
+	struct regulator *vref_n;
+	struct mutex lock;
+	struct completion complete;
+	/* The number of cclk periods for the S/H's acquisition time */
+	unsigned int acquisition_time;
+
+	u8 tx_buf[2] ____cacheline_aligned;
+	u8 rx_buf[2];
+};
+
+#define ADC12138_VOLTAGE_CHANNEL(chan)					\
+	{								\
+		.type = IIO_VOLTAGE,					\
+		.indexed = 1,						\
+		.channel = chan,					\
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE)	\
+					| BIT(IIO_CHAN_INFO_OFFSET),	\
+		.scan_index = chan,					\
+		.scan_type = {						\
+			.sign = 's',					\
+			.realbits = 13,					\
+			.storagebits = 16,				\
+			.shift = 3,					\
+			.endianness = IIO_BE,				\
+		},							\
+	}
+
+#define ADC12138_VOLTAGE_CHANNEL_DIFF(chan1, chan2, si)			\
+	{								\
+		.type = IIO_VOLTAGE,					\
+		.indexed = 1,						\
+		.channel = (chan1),					\
+		.channel2 = (chan2),					\
+		.differential = 1,					\
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE)	\
+					| BIT(IIO_CHAN_INFO_OFFSET),	\
+		.scan_index = si,					\
+		.scan_type = {						\
+			.sign = 's',					\
+			.realbits = 13,					\
+			.storagebits = 16,				\
+			.shift = 3,					\
+			.endianness = IIO_BE,				\
+		},							\
+	}
+
+static const struct iio_chan_spec adc12132_channels[] = {
+	ADC12138_VOLTAGE_CHANNEL(0),
+	ADC12138_VOLTAGE_CHANNEL(1),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(0, 1, 2),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(1, 0, 3),
+	IIO_CHAN_SOFT_TIMESTAMP(4),
+};
+
+static const struct iio_chan_spec adc12138_channels[] = {
+	ADC12138_VOLTAGE_CHANNEL(0),
+	ADC12138_VOLTAGE_CHANNEL(1),
+	ADC12138_VOLTAGE_CHANNEL(2),
+	ADC12138_VOLTAGE_CHANNEL(3),
+	ADC12138_VOLTAGE_CHANNEL(4),
+	ADC12138_VOLTAGE_CHANNEL(5),
+	ADC12138_VOLTAGE_CHANNEL(6),
+	ADC12138_VOLTAGE_CHANNEL(7),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(0, 1, 8),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(1, 0, 9),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(2, 3, 10),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(3, 2, 11),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(4, 5, 12),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(5, 4, 13),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(6, 7, 14),
+	ADC12138_VOLTAGE_CHANNEL_DIFF(7, 6, 15),
+	IIO_CHAN_SOFT_TIMESTAMP(16),
+};
+
+static int adc12138_mode_programming(struct adc12138 *adc, u8 mode,
+				     void *rx_buf, int len)
+{
+	struct spi_transfer xfer = {
+		.tx_buf = adc->tx_buf,
+		.rx_buf = adc->rx_buf,
+		.len = len,
+	};
+	int ret;
+
+	/* Skip unused bits for ADC12130 and ADC12132 */
+	if (adc->id != adc12138)
+		mode = (mode & 0xc0) | ((mode & 0x0f) << 2);
+
+	adc->tx_buf[0] = mode;
+
+	ret = spi_sync_transfer(adc->spi, &xfer, 1);
+	if (ret)
+		return ret;
+
+	memcpy(rx_buf, adc->rx_buf, len);
+
+	return 0;
+}
+
+static int adc12138_read_status(struct adc12138 *adc)
+{
+	u8 rx_buf[2];
+	int ret;
+
+	ret = adc12138_mode_programming(adc, ADC12138_MODE_READ_STATUS,
+					rx_buf, 2);
+	if (ret)
+		return ret;
+
+	return (rx_buf[0] << 1) | (rx_buf[1] >> 7);
+}
+
+static int __adc12138_start_conv(struct adc12138 *adc,
+				 struct iio_chan_spec const *channel,
+				 void *data, int len)
+
+{
+	const u8 ch_to_mux[] = { 0, 4, 1, 5, 2, 6, 3, 7 };
+	u8 mode = (ch_to_mux[channel->channel] << 4) |
+		  (channel->differential ? 0 : 0x80);
+
+	return adc12138_mode_programming(adc, mode, data, len);
+}
+
+static int adc12138_start_conv(struct adc12138 *adc,
+			       struct iio_chan_spec const *channel)
+{
+	u8 trash;
+
+	return __adc12138_start_conv(adc, channel, &trash, 1);
+}
+
+static int adc12138_start_and_read_conv(struct adc12138 *adc,
+					struct iio_chan_spec const *channel,
+					__be16 *data)
+{
+	return __adc12138_start_conv(adc, channel, data, 2);
+}
+
+static int adc12138_read_conv_data(struct adc12138 *adc, __be16 *value)
+{
+	/* Issue a read status instruction and read previous conversion data */
+	return adc12138_mode_programming(adc, ADC12138_MODE_READ_STATUS,
+					 value, sizeof(*value));
+}
+
+static int adc12138_wait_eoc(struct adc12138 *adc, unsigned long timeout)
+{
+	if (!wait_for_completion_timeout(&adc->complete, timeout))
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static int adc12138_adc_conversion(struct adc12138 *adc,
+				   struct iio_chan_spec const *channel,
+				   __be16 *value)
+{
+	int ret;
+
+	reinit_completion(&adc->complete);
+
+	ret = adc12138_start_conv(adc, channel);
+	if (ret)
+		return ret;
+
+	ret = adc12138_wait_eoc(adc, msecs_to_jiffies(100));
+	if (ret)
+		return ret;
+
+	return adc12138_read_conv_data(adc, value);
+}
+
+static int adc12138_read_raw(struct iio_dev *iio,
+			     struct iio_chan_spec const *channel, int *value,
+			     int *shift, long mask)
+{
+	struct adc12138 *adc = iio_priv(iio);
+	int ret;
+	__be16 data;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		mutex_lock(&adc->lock);
+		ret = adc12138_adc_conversion(adc, channel, &data);
+		mutex_unlock(&adc->lock);
+		if (ret)
+			return ret;
+
+		*value = sign_extend32(be16_to_cpu(data) >> 3, 12);
+
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		ret = regulator_get_voltage(adc->vref_p);
+		if (ret < 0)
+			return ret;
+		*value = ret;
+
+		if (!IS_ERR(adc->vref_n)) {
+			ret = regulator_get_voltage(adc->vref_n);
+			if (ret < 0)
+				return ret;
+			*value -= ret;
+		}
+
+		/* convert regulator output voltage to mV */
+		*value /= 1000;
+		*shift = channel->scan_type.realbits - 1;
+
+		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_OFFSET:
+		if (!IS_ERR(adc->vref_n)) {
+			*value = regulator_get_voltage(adc->vref_n);
+			if (*value < 0)
+				return *value;
+		} else {
+			*value = 0;
+		}
+
+		/* convert regulator output voltage to mV */
+		*value /= 1000;
+
+		return IIO_VAL_INT;
+	}
+
+	return -EINVAL;
+}
+
+static const struct iio_info adc12138_info = {
+	.read_raw = adc12138_read_raw,
+	.driver_module = THIS_MODULE,
+};
+
+static int adc12138_init(struct adc12138 *adc)
+{
+	int ret;
+	int status;
+	u8 mode;
+	u8 trash;
+
+	reinit_completion(&adc->complete);
+
+	ret = adc12138_mode_programming(adc, ADC12138_MODE_AUTO_CAL, &trash, 1);
+	if (ret)
+		return ret;
+
+	/* data output at this time has no significance */
+	status = adc12138_read_status(adc);
+	if (status < 0)
+		return status;
+
+	adc12138_wait_eoc(adc, msecs_to_jiffies(100));
+
+	status = adc12138_read_status(adc);
+	if (status & ADC12138_STATUS_CAL) {
+		dev_warn(&adc->spi->dev,
+			"Auto Cal sequence is still in progress: %#x\n",
+			status);
+		return -EIO;
+	}
+
+	switch (adc->acquisition_time) {
+	case 6:
+		mode = ADC12138_MODE_ACQUISITION_TIME_6;
+		break;
+	case 10:
+		mode = ADC12138_MODE_ACQUISITION_TIME_10;
+		break;
+	case 18:
+		mode = ADC12138_MODE_ACQUISITION_TIME_18;
+		break;
+	case 34:
+		mode = ADC12138_MODE_ACQUISITION_TIME_34;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return adc12138_mode_programming(adc, mode, &trash, 1);
+}
+
+static irqreturn_t adc12138_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct adc12138 *adc = iio_priv(indio_dev);
+	__be16 data[20] = { }; /* 16x 2 bytes ADC data + 8 bytes timestamp */
+	__be16 trash;
+	int ret;
+	int scan_index;
+	int i = 0;
+
+	mutex_lock(&adc->lock);
+
+	for_each_set_bit(scan_index, indio_dev->active_scan_mask,
+			 indio_dev->masklength) {
+		const struct iio_chan_spec *scan_chan =
+				&indio_dev->channels[scan_index];
+
+		reinit_completion(&adc->complete);
+
+		ret = adc12138_start_and_read_conv(adc, scan_chan,
+						   i ? &data[i - 1] : &trash);
+		if (ret) {
+			dev_warn(&adc->spi->dev,
+				 "failed to start conversion\n");
+			goto out;
+		}
+
+		ret = adc12138_wait_eoc(adc, msecs_to_jiffies(100));
+		if (ret) {
+			dev_warn(&adc->spi->dev, "wait eoc timeout\n");
+			goto out;
+		}
+
+		i++;
+	}
+
+	if (i) {
+		ret = adc12138_read_conv_data(adc, &data[i - 1]);
+		if (ret) {
+			dev_warn(&adc->spi->dev,
+				 "failed to get conversion data\n");
+			goto out;
+		}
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, data,
+					   iio_get_time_ns(indio_dev));
+out:
+	mutex_unlock(&adc->lock);
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t adc12138_eoc_handler(int irq, void *p)
+{
+	struct iio_dev *indio_dev = p;
+	struct adc12138 *adc = iio_priv(indio_dev);
+
+	complete(&adc->complete);
+
+	return IRQ_HANDLED;
+}
+
+static int adc12138_probe(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev;
+	struct adc12138 *adc;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*adc));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	adc = iio_priv(indio_dev);
+	adc->spi = spi;
+	adc->id = spi_get_device_id(spi)->driver_data;
+	mutex_init(&adc->lock);
+	init_completion(&adc->complete);
+
+	indio_dev->name = spi_get_device_id(spi)->name;
+	indio_dev->dev.parent = &spi->dev;
+	indio_dev->info = &adc12138_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	switch (adc->id) {
+	case adc12130:
+	case adc12132:
+		indio_dev->channels = adc12132_channels;
+		indio_dev->num_channels = ARRAY_SIZE(adc12132_channels);
+		break;
+	case adc12138:
+		indio_dev->channels = adc12138_channels;
+		indio_dev->num_channels = ARRAY_SIZE(adc12138_channels);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32(spi->dev.of_node, "ti,acquisition-time",
+				   &adc->acquisition_time);
+	if (ret)
+		adc->acquisition_time = 10;
+
+	adc->cclk = devm_clk_get(&spi->dev, NULL);
+	if (IS_ERR(adc->cclk))
+		return PTR_ERR(adc->cclk);
+
+	adc->vref_p = devm_regulator_get(&spi->dev, "vref-p");
+	if (IS_ERR(adc->vref_p))
+		return PTR_ERR(adc->vref_p);
+
+	adc->vref_n = devm_regulator_get_optional(&spi->dev, "vref-n");
+	if (IS_ERR(adc->vref_n)) {
+		/*
+		 * Assume vref_n is 0V if an optional regulator is not
+		 * specified, otherwise return the error code.
+		 */
+		ret = PTR_ERR(adc->vref_n);
+		if (ret != -ENODEV)
+			return ret;
+	}
+
+	ret = devm_request_irq(&spi->dev, spi->irq, adc12138_eoc_handler,
+			       IRQF_TRIGGER_RISING, indio_dev->name, indio_dev);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(adc->cclk);
+	if (ret)
+		return ret;
+
+	ret = regulator_enable(adc->vref_p);
+	if (ret)
+		goto err_clk_disable;
+
+	if (!IS_ERR(adc->vref_n)) {
+		ret = regulator_enable(adc->vref_n);
+		if (ret)
+			goto err_vref_p_disable;
+	}
+
+	ret = adc12138_init(adc);
+	if (ret)
+		goto err_vref_n_disable;
+
+	spi_set_drvdata(spi, indio_dev);
+
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+					 adc12138_trigger_handler, NULL);
+	if (ret)
+		goto err_vref_n_disable;
+
+	ret = iio_device_register(indio_dev);
+	if (ret)
+		goto err_buffer_cleanup;
+
+	return 0;
+err_buffer_cleanup:
+	iio_triggered_buffer_cleanup(indio_dev);
+err_vref_n_disable:
+	if (!IS_ERR(adc->vref_n))
+		regulator_disable(adc->vref_n);
+err_vref_p_disable:
+	regulator_disable(adc->vref_p);
+err_clk_disable:
+	clk_disable_unprepare(adc->cclk);
+
+	return ret;
+}
+
+static int adc12138_remove(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+	struct adc12138 *adc = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
+	if (!IS_ERR(adc->vref_n))
+		regulator_disable(adc->vref_n);
+	regulator_disable(adc->vref_p);
+	clk_disable_unprepare(adc->cclk);
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+
+static const struct of_device_id adc12138_dt_ids[] = {
+	{ .compatible = "ti,adc12130", },
+	{ .compatible = "ti,adc12132", },
+	{ .compatible = "ti,adc12138", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, adc12138_dt_ids);
+
+#endif
+
+static const struct spi_device_id adc12138_id[] = {
+	{ "adc12130", adc12130 },
+	{ "adc12132", adc12132 },
+	{ "adc12138", adc12138 },
+	{}
+};
+MODULE_DEVICE_TABLE(spi, adc12138_id);
+
+static struct spi_driver adc12138_driver = {
+	.driver = {
+		.name = "adc12138",
+		.of_match_table = of_match_ptr(adc12138_dt_ids),
+	},
+	.probe = adc12138_probe,
+	.remove = adc12138_remove,
+	.id_table = adc12138_id,
+};
+module_spi_driver(adc12138_driver);
+
+MODULE_AUTHOR("Akinobu Mita <akinobu.mita@gmail.com>");
+MODULE_DESCRIPTION("ADC12130/ADC12132/ADC12138 driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/ti-adc161s626.c b/drivers/iio/adc/ti-adc161s626.c
new file mode 100644
index 000000000000..f94b69f9c288
--- /dev/null
+++ b/drivers/iio/adc/ti-adc161s626.c
@@ -0,0 +1,248 @@
+/*
+ * ti-adc161s626.c - Texas Instruments ADC161S626 1-channel differential ADC
+ *
+ * ADC Devices Supported:
+ *  adc141s626 - 14-bit ADC
+ *  adc161s626 - 16-bit ADC
+ *
+ * Copyright (C) 2016 Matt Ranostay <mranostay@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/spi/spi.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+
+#define TI_ADC_DRV_NAME	"ti-adc161s626"
+
+enum {
+	TI_ADC141S626,
+	TI_ADC161S626,
+};
+
+static const struct iio_chan_spec ti_adc141s626_channels[] = {
+	{
+		.type = IIO_VOLTAGE,
+		.channel = 0,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 14,
+			.storagebits = 16,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+};
+
+static const struct iio_chan_spec ti_adc161s626_channels[] = {
+	{
+		.type = IIO_VOLTAGE,
+		.channel = 0,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 16,
+			.storagebits = 16,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+};
+
+struct ti_adc_data {
+	struct iio_dev *indio_dev;
+	struct spi_device *spi;
+	u8 read_size;
+	u8 shift;
+
+	u8 buffer[16] ____cacheline_aligned;
+};
+
+static int ti_adc_read_measurement(struct ti_adc_data *data,
+				   struct iio_chan_spec const *chan, int *val)
+{
+	int ret;
+
+	switch (data->read_size) {
+	case 2: {
+		__be16 buf;
+
+		ret = spi_read(data->spi, (void *) &buf, 2);
+		if (ret)
+			return ret;
+
+		*val = be16_to_cpu(buf);
+		break;
+	}
+	case 3: {
+		__be32 buf;
+
+		ret = spi_read(data->spi, (void *) &buf, 3);
+		if (ret)
+			return ret;
+
+		*val = be32_to_cpu(buf) >> 8;
+		break;
+	}
+	default:
+		return -EINVAL;
+	}
+
+	*val = sign_extend32(*val >> data->shift, chan->scan_type.realbits - 1);
+
+	return 0;
+}
+
+static irqreturn_t ti_adc_trigger_handler(int irq, void *private)
+{
+	struct iio_poll_func *pf = private;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct ti_adc_data *data = iio_priv(indio_dev);
+	int ret;
+
+	ret = ti_adc_read_measurement(data, &indio_dev->channels[0],
+				     (int *) &data->buffer);
+	if (!ret)
+		iio_push_to_buffers_with_timestamp(indio_dev,
+					data->buffer,
+					iio_get_time_ns(indio_dev));
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int ti_adc_read_raw(struct iio_dev *indio_dev,
+			   struct iio_chan_spec const *chan,
+			   int *val, int *val2, long mask)
+{
+	struct ti_adc_data *data = iio_priv(indio_dev);
+	int ret;
+
+	if (mask != IIO_CHAN_INFO_RAW)
+		return -EINVAL;
+
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret)
+		return ret;
+
+	ret = ti_adc_read_measurement(data, chan, val);
+	iio_device_release_direct_mode(indio_dev);
+
+	if (!ret)
+		return IIO_VAL_INT;
+
+	return 0;
+}
+
+static const struct iio_info ti_adc_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = ti_adc_read_raw,
+};
+
+static int ti_adc_probe(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev;
+	struct ti_adc_data *data;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	indio_dev->info = &ti_adc_info;
+	indio_dev->dev.parent = &spi->dev;
+	indio_dev->dev.of_node = spi->dev.of_node;
+	indio_dev->name = TI_ADC_DRV_NAME;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	spi_set_drvdata(spi, indio_dev);
+
+	data = iio_priv(indio_dev);
+	data->spi = spi;
+
+	switch (spi_get_device_id(spi)->driver_data) {
+	case TI_ADC141S626:
+		indio_dev->channels = ti_adc141s626_channels;
+		indio_dev->num_channels = ARRAY_SIZE(ti_adc141s626_channels);
+		data->shift = 0;
+		data->read_size = 2;
+		break;
+	case TI_ADC161S626:
+		indio_dev->channels = ti_adc161s626_channels;
+		indio_dev->num_channels = ARRAY_SIZE(ti_adc161s626_channels);
+		data->shift = 6;
+		data->read_size = 3;
+		break;
+	}
+
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+					 ti_adc_trigger_handler, NULL);
+	if (ret)
+		return ret;
+
+	ret = iio_device_register(indio_dev);
+	if (ret)
+		goto error_unreg_buffer;
+
+	return 0;
+
+error_unreg_buffer:
+	iio_triggered_buffer_cleanup(indio_dev);
+
+	return ret;
+}
+
+static int ti_adc_remove(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+
+	iio_device_unregister(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
+
+	return 0;
+}
+
+static const struct of_device_id ti_adc_dt_ids[] = {
+	{ .compatible = "ti,adc141s626", },
+	{ .compatible = "ti,adc161s626", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ti_adc_dt_ids);
+
+static const struct spi_device_id ti_adc_id[] = {
+	{"adc141s626", TI_ADC141S626},
+	{"adc161s626", TI_ADC161S626},
+	{},
+};
+MODULE_DEVICE_TABLE(spi, ti_adc_id);
+
+static struct spi_driver ti_adc_driver = {
+	.driver = {
+		.name	= TI_ADC_DRV_NAME,
+		.of_match_table = of_match_ptr(ti_adc_dt_ids),
+	},
+	.probe		= ti_adc_probe,
+	.remove		= ti_adc_remove,
+	.id_table	= ti_adc_id,
+};
+module_spi_driver(ti_adc_driver);
+
+MODULE_AUTHOR("Matt Ranostay <mranostay@gmail.com>");
+MODULE_DESCRIPTION("Texas Instruments ADC1x1S 1-channel differential ADC");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/adc/ti-ads1015.c b/drivers/iio/adc/ti-ads1015.c
index 1ef398770a1f..cde6f130a99a 100644
--- a/drivers/iio/adc/ti-ads1015.c
+++ b/drivers/iio/adc/ti-ads1015.c
@@ -489,7 +489,8 @@ static struct iio_info ads1115_info = {
 #ifdef CONFIG_OF
 static int ads1015_get_channels_config_of(struct i2c_client *client)
 {
-	struct ads1015_data *data = i2c_get_clientdata(client);
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+	struct ads1015_data *data = iio_priv(indio_dev);
 	struct device_node *node;
 
 	if (!client->dev.of_node ||
@@ -521,6 +522,7 @@ static int ads1015_get_channels_config_of(struct i2c_client *client)
 			if (pga > 6) {
 				dev_err(&client->dev, "invalid gain on %s\n",
 					node->full_name);
+				of_node_put(node);
 				return -EINVAL;
 			}
 		}
@@ -531,6 +533,7 @@ static int ads1015_get_channels_config_of(struct i2c_client *client)
 				dev_err(&client->dev,
 					"invalid data_rate on %s\n",
 					node->full_name);
+				of_node_put(node);
 				return -EINVAL;
 			}
 		}
diff --git a/drivers/iio/adc/ti-ads8688.c b/drivers/iio/adc/ti-ads8688.c
index c400439900af..4a163496d9e4 100644
--- a/drivers/iio/adc/ti-ads8688.c
+++ b/drivers/iio/adc/ti-ads8688.c
@@ -438,7 +438,7 @@ static int ads8688_probe(struct spi_device *spi)
 	return 0;
 
 error_out:
-	if (!IS_ERR_OR_NULL(st->reg))
+	if (!IS_ERR(st->reg))
 		regulator_disable(st->reg);
 
 	return ret;
@@ -451,7 +451,7 @@ static int ads8688_remove(struct spi_device *spi)
 
 	iio_device_unregister(indio_dev);
 
-	if (!IS_ERR_OR_NULL(st->reg))
+	if (!IS_ERR(st->reg))
 		regulator_disable(st->reg);
 
 	return 0;
diff --git a/drivers/iio/adc/ti_am335x_adc.c b/drivers/iio/adc/ti_am335x_adc.c
index 8a368756881b..c3cfacca2541 100644
--- a/drivers/iio/adc/ti_am335x_adc.c
+++ b/drivers/iio/adc/ti_am335x_adc.c
@@ -32,6 +32,7 @@
 
 struct tiadc_device {
 	struct ti_tscadc_dev *mfd_tscadc;
+	struct mutex fifo1_lock; /* to protect fifo access */
 	int channels;
 	u8 channel_line[8];
 	u8 channel_step[8];
@@ -359,6 +360,7 @@ static int tiadc_read_raw(struct iio_dev *indio_dev,
 		int *val, int *val2, long mask)
 {
 	struct tiadc_device *adc_dev = iio_priv(indio_dev);
+	int ret = IIO_VAL_INT;
 	int i, map_val;
 	unsigned int fifo1count, read, stepid;
 	bool found = false;
@@ -372,13 +374,14 @@ static int tiadc_read_raw(struct iio_dev *indio_dev,
 	if (!step_en)
 		return -EINVAL;
 
+	mutex_lock(&adc_dev->fifo1_lock);
 	fifo1count = tiadc_readl(adc_dev, REG_FIFO1CNT);
 	while (fifo1count--)
 		tiadc_readl(adc_dev, REG_FIFO1);
 
 	am335x_tsc_se_set_once(adc_dev->mfd_tscadc, step_en);
 
-	timeout = jiffies + usecs_to_jiffies
+	timeout = jiffies + msecs_to_jiffies
 				(IDLE_TIMEOUT * adc_dev->channels);
 	/* Wait for Fifo threshold interrupt */
 	while (1) {
@@ -388,7 +391,8 @@ static int tiadc_read_raw(struct iio_dev *indio_dev,
 
 		if (time_after(jiffies, timeout)) {
 			am335x_tsc_se_adc_done(adc_dev->mfd_tscadc);
-			return -EAGAIN;
+			ret = -EAGAIN;
+			goto err_unlock;
 		}
 	}
 	map_val = adc_dev->channel_step[chan->scan_index];
@@ -414,8 +418,11 @@ static int tiadc_read_raw(struct iio_dev *indio_dev,
 	am335x_tsc_se_adc_done(adc_dev->mfd_tscadc);
 
 	if (found == false)
-		return -EBUSY;
-	return IIO_VAL_INT;
+		ret =  -EBUSY;
+
+err_unlock:
+	mutex_unlock(&adc_dev->fifo1_lock);
+	return ret;
 }
 
 static const struct iio_info tiadc_info = {
@@ -483,6 +490,7 @@ static int tiadc_probe(struct platform_device *pdev)
 
 	tiadc_step_config(indio_dev);
 	tiadc_writel(adc_dev, REG_FIFO1THR, FIFO1_THRESHOLD);
+	mutex_init(&adc_dev->fifo1_lock);
 
 	err = tiadc_channel_init(indio_dev, adc_dev->channels);
 	if (err < 0)
diff --git a/drivers/iio/buffer/industrialio-buffer-cb.c b/drivers/iio/buffer/industrialio-buffer-cb.c
index 323079c3ccce..b8f550e47d3d 100644
--- a/drivers/iio/buffer/industrialio-buffer-cb.c
+++ b/drivers/iio/buffer/industrialio-buffer-cb.c
@@ -18,6 +18,7 @@ struct iio_cb_buffer {
 	int (*cb)(const void *data, void *private);
 	void *private;
 	struct iio_channel *channels;
+	struct iio_dev *indio_dev;
 };
 
 static struct iio_cb_buffer *buffer_to_cb_buffer(struct iio_buffer *buffer)
@@ -52,7 +53,6 @@ struct iio_cb_buffer *iio_channel_get_all_cb(struct device *dev,
 {
 	int ret;
 	struct iio_cb_buffer *cb_buff;
-	struct iio_dev *indio_dev;
 	struct iio_channel *chan;
 
 	cb_buff = kzalloc(sizeof(*cb_buff), GFP_KERNEL);
@@ -72,17 +72,17 @@ struct iio_cb_buffer *iio_channel_get_all_cb(struct device *dev,
 		goto error_free_cb_buff;
 	}
 
-	indio_dev = cb_buff->channels[0].indio_dev;
+	cb_buff->indio_dev = cb_buff->channels[0].indio_dev;
 	cb_buff->buffer.scan_mask
-		= kcalloc(BITS_TO_LONGS(indio_dev->masklength), sizeof(long),
-			  GFP_KERNEL);
+		= kcalloc(BITS_TO_LONGS(cb_buff->indio_dev->masklength),
+			  sizeof(long), GFP_KERNEL);
 	if (cb_buff->buffer.scan_mask == NULL) {
 		ret = -ENOMEM;
 		goto error_release_channels;
 	}
 	chan = &cb_buff->channels[0];
 	while (chan->indio_dev) {
-		if (chan->indio_dev != indio_dev) {
+		if (chan->indio_dev != cb_buff->indio_dev) {
 			ret = -EINVAL;
 			goto error_free_scan_mask;
 		}
@@ -105,17 +105,14 @@ EXPORT_SYMBOL_GPL(iio_channel_get_all_cb);
 
 int iio_channel_start_all_cb(struct iio_cb_buffer *cb_buff)
 {
-	return iio_update_buffers(cb_buff->channels[0].indio_dev,
-				  &cb_buff->buffer,
+	return iio_update_buffers(cb_buff->indio_dev, &cb_buff->buffer,
 				  NULL);
 }
 EXPORT_SYMBOL_GPL(iio_channel_start_all_cb);
 
 void iio_channel_stop_all_cb(struct iio_cb_buffer *cb_buff)
 {
-	iio_update_buffers(cb_buff->channels[0].indio_dev,
-			   NULL,
-			   &cb_buff->buffer);
+	iio_update_buffers(cb_buff->indio_dev, NULL, &cb_buff->buffer);
 }
 EXPORT_SYMBOL_GPL(iio_channel_stop_all_cb);
 
@@ -133,6 +130,13 @@ struct iio_channel
 }
 EXPORT_SYMBOL_GPL(iio_channel_cb_get_channels);
 
+struct iio_dev
+*iio_channel_cb_get_iio_dev(const struct iio_cb_buffer *cb_buffer)
+{
+	return cb_buffer->indio_dev;
+}
+EXPORT_SYMBOL_GPL(iio_channel_cb_get_iio_dev);
+
 MODULE_AUTHOR("Jonathan Cameron <jic23@kernel.org>");
 MODULE_DESCRIPTION("Industrial I/O callback buffer");
 MODULE_LICENSE("GPL");
diff --git a/drivers/iio/buffer/industrialio-triggered-buffer.c b/drivers/iio/buffer/industrialio-triggered-buffer.c
index 4b2858ba1fd6..d3db1fce54d2 100644
--- a/drivers/iio/buffer/industrialio-triggered-buffer.c
+++ b/drivers/iio/buffer/industrialio-triggered-buffer.c
@@ -98,6 +98,48 @@ void iio_triggered_buffer_cleanup(struct iio_dev *indio_dev)
 }
 EXPORT_SYMBOL(iio_triggered_buffer_cleanup);
 
+static void devm_iio_triggered_buffer_clean(struct device *dev, void *res)
+{
+	iio_triggered_buffer_cleanup(*(struct iio_dev **)res);
+}
+
+int devm_iio_triggered_buffer_setup(struct device *dev,
+				    struct iio_dev *indio_dev,
+				    irqreturn_t (*h)(int irq, void *p),
+				    irqreturn_t (*thread)(int irq, void *p),
+				    const struct iio_buffer_setup_ops *ops)
+{
+	struct iio_dev **ptr;
+	int ret;
+
+	ptr = devres_alloc(devm_iio_triggered_buffer_clean, sizeof(*ptr),
+			   GFP_KERNEL);
+	if (!ptr)
+		return -ENOMEM;
+
+	*ptr = indio_dev;
+
+	ret = iio_triggered_buffer_setup(indio_dev, h, thread, ops);
+	if (!ret)
+		devres_add(dev, ptr);
+	else
+		devres_free(ptr);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(devm_iio_triggered_buffer_setup);
+
+void devm_iio_triggered_buffer_cleanup(struct device *dev,
+				       struct iio_dev *indio_dev)
+{
+	int rc;
+
+	rc = devres_release(dev, devm_iio_triggered_buffer_clean,
+			    devm_iio_device_match, indio_dev);
+	WARN_ON(rc);
+}
+EXPORT_SYMBOL_GPL(devm_iio_triggered_buffer_cleanup);
+
 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
 MODULE_DESCRIPTION("IIO helper functions for setting up triggered buffers");
 MODULE_LICENSE("GPL");
diff --git a/drivers/iio/chemical/Kconfig b/drivers/iio/chemical/Kconfig
index 4bcc025e8c8a..cea7f9857a1f 100644
--- a/drivers/iio/chemical/Kconfig
+++ b/drivers/iio/chemical/Kconfig
@@ -16,6 +16,7 @@ config ATLAS_PH_SENSOR
 	 Atlas Scientific OEM SM sensors:
 	    * pH SM sensor
 	    * EC SM sensor
+	    * ORP SM sensor
 
 	 To compile this driver as module, choose M here: the
 	 module will be called atlas-ph-sensor.
diff --git a/drivers/iio/chemical/atlas-ph-sensor.c b/drivers/iio/chemical/atlas-ph-sensor.c
index ae038a59d256..bd321b305a0a 100644
--- a/drivers/iio/chemical/atlas-ph-sensor.c
+++ b/drivers/iio/chemical/atlas-ph-sensor.c
@@ -66,12 +66,17 @@
 #define ATLAS_REG_TDS_DATA		0x1c
 #define ATLAS_REG_PSS_DATA		0x20
 
+#define ATLAS_REG_ORP_CALIB_STATUS	0x0d
+#define ATLAS_REG_ORP_DATA		0x0e
+
 #define ATLAS_PH_INT_TIME_IN_US		450000
 #define ATLAS_EC_INT_TIME_IN_US		650000
+#define ATLAS_ORP_INT_TIME_IN_US	450000
 
 enum {
 	ATLAS_PH_SM,
 	ATLAS_EC_SM,
+	ATLAS_ORP_SM,
 };
 
 struct atlas_data {
@@ -84,26 +89,10 @@ struct atlas_data {
 	__be32 buffer[6]; /* 96-bit data + 32-bit pad + 64-bit timestamp */
 };
 
-static const struct regmap_range atlas_volatile_ranges[] = {
-	regmap_reg_range(ATLAS_REG_INT_CONTROL, ATLAS_REG_INT_CONTROL),
-	regmap_reg_range(ATLAS_REG_PH_DATA, ATLAS_REG_PH_DATA + 4),
-	regmap_reg_range(ATLAS_REG_EC_DATA, ATLAS_REG_PSS_DATA + 4),
-};
-
-static const struct regmap_access_table atlas_volatile_table = {
-	.yes_ranges	= atlas_volatile_ranges,
-	.n_yes_ranges	= ARRAY_SIZE(atlas_volatile_ranges),
-};
-
 static const struct regmap_config atlas_regmap_config = {
 	.name = ATLAS_REGMAP_NAME,
-
 	.reg_bits = 8,
 	.val_bits = 8,
-
-	.volatile_table = &atlas_volatile_table,
-	.max_register = ATLAS_REG_PSS_DATA + 4,
-	.cache_type = REGCACHE_RBTREE,
 };
 
 static const struct iio_chan_spec atlas_ph_channels[] = {
@@ -175,6 +164,23 @@ static const struct iio_chan_spec atlas_ec_channels[] = {
 	},
 };
 
+static const struct iio_chan_spec atlas_orp_channels[] = {
+	{
+		.type = IIO_VOLTAGE,
+		.address = ATLAS_REG_ORP_DATA,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 32,
+			.storagebits = 32,
+			.endianness = IIO_BE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+};
+
 static int atlas_check_ph_calibration(struct atlas_data *data)
 {
 	struct device *dev = &data->client->dev;
@@ -240,6 +246,22 @@ static int atlas_check_ec_calibration(struct atlas_data *data)
 	return 0;
 }
 
+static int atlas_check_orp_calibration(struct atlas_data *data)
+{
+	struct device *dev = &data->client->dev;
+	int ret;
+	unsigned int val;
+
+	ret = regmap_read(data->regmap, ATLAS_REG_ORP_CALIB_STATUS, &val);
+	if (ret)
+		return ret;
+
+	if (!val)
+		dev_warn(dev, "device has not been calibrated\n");
+
+	return 0;
+};
+
 struct atlas_device {
 	const struct iio_chan_spec *channels;
 	int num_channels;
@@ -264,7 +286,13 @@ static struct atlas_device atlas_devices[] = {
 				.calibration = &atlas_check_ec_calibration,
 				.delay = ATLAS_EC_INT_TIME_IN_US,
 	},
-
+	[ATLAS_ORP_SM] = {
+				.channels = atlas_orp_channels,
+				.num_channels = 2,
+				.data_reg = ATLAS_REG_ORP_DATA,
+				.calibration = &atlas_check_orp_calibration,
+				.delay = ATLAS_ORP_INT_TIME_IN_US,
+	},
 };
 
 static int atlas_set_powermode(struct atlas_data *data, int on)
@@ -402,15 +430,14 @@ static int atlas_read_raw(struct iio_dev *indio_dev,
 		case IIO_PH:
 		case IIO_CONCENTRATION:
 		case IIO_ELECTRICALCONDUCTIVITY:
-			mutex_lock(&indio_dev->mlock);
+		case IIO_VOLTAGE:
+			ret = iio_device_claim_direct_mode(indio_dev);
+			if (ret)
+				return ret;
 
-			if (iio_buffer_enabled(indio_dev))
-				ret = -EBUSY;
-			else
-				ret = atlas_read_measurement(data,
-							chan->address, &reg);
+			ret = atlas_read_measurement(data, chan->address, &reg);
 
-			mutex_unlock(&indio_dev->mlock);
+			iio_device_release_direct_mode(indio_dev);
 			break;
 		default:
 			ret = -EINVAL;
@@ -434,12 +461,16 @@ static int atlas_read_raw(struct iio_dev *indio_dev,
 			break;
 		case IIO_ELECTRICALCONDUCTIVITY:
 			*val = 1; /* 0.00001 */
-			*val = 100000;
+			*val2 = 100000;
 			break;
 		case IIO_CONCENTRATION:
 			*val = 0; /* 0.000000001 */
 			*val2 = 1000;
 			return IIO_VAL_INT_PLUS_NANO;
+		case IIO_VOLTAGE:
+			*val = 1; /* 0.1 */
+			*val2 = 10;
+			break;
 		default:
 			return -EINVAL;
 		}
@@ -475,6 +506,7 @@ static const struct iio_info atlas_info = {
 static const struct i2c_device_id atlas_id[] = {
 	{ "atlas-ph-sm", ATLAS_PH_SM},
 	{ "atlas-ec-sm", ATLAS_EC_SM},
+	{ "atlas-orp-sm", ATLAS_ORP_SM},
 	{}
 };
 MODULE_DEVICE_TABLE(i2c, atlas_id);
@@ -482,6 +514,7 @@ MODULE_DEVICE_TABLE(i2c, atlas_id);
 static const struct of_device_id atlas_dt_ids[] = {
 	{ .compatible = "atlas,ph-sm", .data = (void *)ATLAS_PH_SM, },
 	{ .compatible = "atlas,ec-sm", .data = (void *)ATLAS_EC_SM, },
+	{ .compatible = "atlas,orp-sm", .data = (void *)ATLAS_ORP_SM, },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, atlas_dt_ids);
diff --git a/drivers/iio/chemical/vz89x.c b/drivers/iio/chemical/vz89x.c
index 652649da500f..8e0e4415c161 100644
--- a/drivers/iio/chemical/vz89x.c
+++ b/drivers/iio/chemical/vz89x.c
@@ -19,25 +19,55 @@
 #include <linux/mutex.h>
 #include <linux/init.h>
 #include <linux/i2c.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
 #define VZ89X_REG_MEASUREMENT		0x09
-#define VZ89X_REG_MEASUREMENT_SIZE	6
+#define VZ89X_REG_MEASUREMENT_RD_SIZE	6
+#define VZ89X_REG_MEASUREMENT_WR_SIZE	3
 
 #define VZ89X_VOC_CO2_IDX		0
 #define VZ89X_VOC_SHORT_IDX		1
 #define VZ89X_VOC_TVOC_IDX		2
 #define VZ89X_VOC_RESISTANCE_IDX	3
 
+#define VZ89TE_REG_MEASUREMENT		0x0c
+#define VZ89TE_REG_MEASUREMENT_RD_SIZE	7
+#define VZ89TE_REG_MEASUREMENT_WR_SIZE	6
+
+#define VZ89TE_VOC_TVOC_IDX		0
+#define VZ89TE_VOC_CO2_IDX		1
+#define VZ89TE_VOC_RESISTANCE_IDX	2
+
+enum {
+	VZ89X,
+	VZ89TE,
+};
+
+struct vz89x_chip_data;
+
 struct vz89x_data {
 	struct i2c_client *client;
+	const struct vz89x_chip_data *chip;
 	struct mutex lock;
 	int (*xfer)(struct vz89x_data *data, u8 cmd);
 
+	bool is_valid;
 	unsigned long last_update;
-	u8 buffer[VZ89X_REG_MEASUREMENT_SIZE];
+	u8 buffer[VZ89TE_REG_MEASUREMENT_RD_SIZE];
+};
+
+struct vz89x_chip_data {
+	bool (*valid)(struct vz89x_data *data);
+	const struct iio_chan_spec *channels;
+	u8 num_channels;
+
+	u8 cmd;
+	u8 read_size;
+	u8 write_size;
 };
 
 static const struct iio_chan_spec vz89x_channels[] = {
@@ -70,6 +100,40 @@ static const struct iio_chan_spec vz89x_channels[] = {
 		.info_mask_separate =
 			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
 		.address = VZ89X_VOC_RESISTANCE_IDX,
+		.scan_index = -1,
+		.scan_type = {
+			.endianness = IIO_LE,
+		},
+	},
+};
+
+static const struct iio_chan_spec vz89te_channels[] = {
+	{
+		.type = IIO_CONCENTRATION,
+		.channel2 = IIO_MOD_VOC,
+		.modified = 1,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_OFFSET) | BIT(IIO_CHAN_INFO_RAW),
+		.address = VZ89TE_VOC_TVOC_IDX,
+	},
+
+	{
+		.type = IIO_CONCENTRATION,
+		.channel2 = IIO_MOD_CO2,
+		.modified = 1,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_OFFSET) | BIT(IIO_CHAN_INFO_RAW),
+		.address = VZ89TE_VOC_CO2_IDX,
+	},
+	{
+		.type = IIO_RESISTANCE,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+		.address = VZ89TE_VOC_RESISTANCE_IDX,
+		.scan_index = -1,
+		.scan_type = {
+			.endianness = IIO_BE,
+		},
 	},
 };
 
@@ -93,29 +157,45 @@ static const struct attribute_group vz89x_attrs_group = {
  * always zero, and by also confirming the VOC_short isn't zero.
  */
 
-static int vz89x_measurement_is_valid(struct vz89x_data *data)
+static bool vz89x_measurement_is_valid(struct vz89x_data *data)
 {
 	if (data->buffer[VZ89X_VOC_SHORT_IDX] == 0)
-		return 1;
+		return true;
 
-	return !!(data->buffer[VZ89X_REG_MEASUREMENT_SIZE - 1] > 0);
+	return !!(data->buffer[data->chip->read_size - 1] > 0);
+}
+
+/* VZ89TE device has a modified CRC-8 two complement check */
+static bool vz89te_measurement_is_valid(struct vz89x_data *data)
+{
+	u8 crc = 0;
+	int i, sum = 0;
+
+	for (i = 0; i < (data->chip->read_size - 1); i++) {
+		sum = crc + data->buffer[i];
+		crc = sum;
+		crc += sum / 256;
+	}
+
+	return !((0xff - crc) == data->buffer[data->chip->read_size - 1]);
 }
 
 static int vz89x_i2c_xfer(struct vz89x_data *data, u8 cmd)
 {
+	const struct vz89x_chip_data *chip = data->chip;
 	struct i2c_client *client = data->client;
 	struct i2c_msg msg[2];
 	int ret;
-	u8 buf[3] = { cmd, 0, 0};
+	u8 buf[6] = { cmd, 0, 0, 0, 0, 0xf3 };
 
 	msg[0].addr = client->addr;
 	msg[0].flags = client->flags;
-	msg[0].len = 3;
+	msg[0].len = chip->write_size;
 	msg[0].buf  = (char *) &buf;
 
 	msg[1].addr = client->addr;
 	msg[1].flags = client->flags | I2C_M_RD;
-	msg[1].len = VZ89X_REG_MEASUREMENT_SIZE;
+	msg[1].len = chip->read_size;
 	msg[1].buf = (char *) &data->buffer;
 
 	ret = i2c_transfer(client->adapter, msg, 2);
@@ -133,7 +213,7 @@ static int vz89x_smbus_xfer(struct vz89x_data *data, u8 cmd)
 	if (ret < 0)
 		return ret;
 
-	for (i = 0; i < VZ89X_REG_MEASUREMENT_SIZE; i++) {
+	for (i = 0; i < data->chip->read_size; i++) {
 		ret = i2c_smbus_read_byte(client);
 		if (ret < 0)
 			return ret;
@@ -145,30 +225,47 @@ static int vz89x_smbus_xfer(struct vz89x_data *data, u8 cmd)
 
 static int vz89x_get_measurement(struct vz89x_data *data)
 {
+	const struct vz89x_chip_data *chip = data->chip;
 	int ret;
 
 	/* sensor can only be polled once a second max per datasheet */
 	if (!time_after(jiffies, data->last_update + HZ))
-		return 0;
+		return data->is_valid ? 0 : -EAGAIN;
+
+	data->is_valid = false;
+	data->last_update = jiffies;
 
-	ret = data->xfer(data, VZ89X_REG_MEASUREMENT);
+	ret = data->xfer(data, chip->cmd);
 	if (ret < 0)
 		return ret;
 
-	ret = vz89x_measurement_is_valid(data);
+	ret = chip->valid(data);
 	if (ret)
 		return -EAGAIN;
 
-	data->last_update = jiffies;
+	data->is_valid = true;
 
 	return 0;
 }
 
-static int vz89x_get_resistance_reading(struct vz89x_data *data)
+static int vz89x_get_resistance_reading(struct vz89x_data *data,
+					struct iio_chan_spec const *chan,
+					int *val)
 {
-	u8 *buf = &data->buffer[VZ89X_VOC_RESISTANCE_IDX];
+	u8 *tmp = (u8 *) &data->buffer[chan->address];
 
-	return buf[0] | (buf[1] << 8);
+	switch (chan->scan_type.endianness) {
+	case IIO_LE:
+		*val = le32_to_cpup((__le32 *) tmp) & GENMASK(23, 0);
+		break;
+	case IIO_BE:
+		*val = be32_to_cpup((__be32 *) tmp) >> 8;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
 }
 
 static int vz89x_read_raw(struct iio_dev *indio_dev,
@@ -187,15 +284,15 @@ static int vz89x_read_raw(struct iio_dev *indio_dev,
 		if (ret)
 			return ret;
 
-		switch (chan->address) {
-		case VZ89X_VOC_CO2_IDX:
-		case VZ89X_VOC_SHORT_IDX:
-		case VZ89X_VOC_TVOC_IDX:
+		switch (chan->type) {
+		case IIO_CONCENTRATION:
 			*val = data->buffer[chan->address];
 			return IIO_VAL_INT;
-		case VZ89X_VOC_RESISTANCE_IDX:
-			*val = vz89x_get_resistance_reading(data);
-			return IIO_VAL_INT;
+		case IIO_RESISTANCE:
+			ret = vz89x_get_resistance_reading(data, chan, val);
+			if (!ret)
+				return IIO_VAL_INT;
+			break;
 		default:
 			return -EINVAL;
 		}
@@ -210,12 +307,12 @@ static int vz89x_read_raw(struct iio_dev *indio_dev,
 		}
 		break;
 	case IIO_CHAN_INFO_OFFSET:
-		switch (chan->address) {
-		case VZ89X_VOC_CO2_IDX:
+		switch (chan->channel2) {
+		case IIO_MOD_CO2:
 			*val = 44;
 			*val2 = 250000;
 			return IIO_VAL_INT_PLUS_MICRO;
-		case VZ89X_VOC_TVOC_IDX:
+		case IIO_MOD_VOC:
 			*val = -13;
 			return IIO_VAL_INT;
 		default:
@@ -232,11 +329,43 @@ static const struct iio_info vz89x_info = {
 	.driver_module	= THIS_MODULE,
 };
 
+static const struct vz89x_chip_data vz89x_chips[] = {
+	{
+		.valid = vz89x_measurement_is_valid,
+
+		.cmd = VZ89X_REG_MEASUREMENT,
+		.read_size = VZ89X_REG_MEASUREMENT_RD_SIZE,
+		.write_size = VZ89X_REG_MEASUREMENT_WR_SIZE,
+
+		.channels = vz89x_channels,
+		.num_channels = ARRAY_SIZE(vz89x_channels),
+	},
+	{
+		.valid = vz89te_measurement_is_valid,
+
+		.cmd = VZ89TE_REG_MEASUREMENT,
+		.read_size = VZ89TE_REG_MEASUREMENT_RD_SIZE,
+		.write_size = VZ89TE_REG_MEASUREMENT_WR_SIZE,
+
+		.channels = vz89te_channels,
+		.num_channels = ARRAY_SIZE(vz89te_channels),
+	},
+};
+
+static const struct of_device_id vz89x_dt_ids[] = {
+	{ .compatible = "sgx,vz89x", .data = (void *) VZ89X },
+	{ .compatible = "sgx,vz89te", .data = (void *) VZ89TE },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, vz89x_dt_ids);
+
 static int vz89x_probe(struct i2c_client *client,
 		       const struct i2c_device_id *id)
 {
 	struct iio_dev *indio_dev;
 	struct vz89x_data *data;
+	const struct of_device_id *of_id;
+	int chip_id;
 
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
 	if (!indio_dev)
@@ -251,8 +380,15 @@ static int vz89x_probe(struct i2c_client *client,
 	else
 		return -EOPNOTSUPP;
 
+	of_id = of_match_device(vz89x_dt_ids, &client->dev);
+	if (!of_id)
+		chip_id = id->driver_data;
+	else
+		chip_id = (unsigned long)of_id->data;
+
 	i2c_set_clientdata(client, indio_dev);
 	data->client = client;
+	data->chip = &vz89x_chips[chip_id];
 	data->last_update = jiffies - HZ;
 	mutex_init(&data->lock);
 
@@ -261,24 +397,19 @@ static int vz89x_probe(struct i2c_client *client,
 	indio_dev->name = dev_name(&client->dev);
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	indio_dev->channels = vz89x_channels;
-	indio_dev->num_channels = ARRAY_SIZE(vz89x_channels);
+	indio_dev->channels = data->chip->channels;
+	indio_dev->num_channels = data->chip->num_channels;
 
 	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
 static const struct i2c_device_id vz89x_id[] = {
-	{ "vz89x", 0 },
+	{ "vz89x", VZ89X },
+	{ "vz89te", VZ89TE },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, vz89x_id);
 
-static const struct of_device_id vz89x_dt_ids[] = {
-	{ .compatible = "sgx,vz89x" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, vz89x_dt_ids);
-
 static struct i2c_driver vz89x_driver = {
 	.driver = {
 		.name	= "vz89x",
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-attributes.c b/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
index e81f434760f4..dc33c1dd5191 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
@@ -56,8 +56,8 @@ static struct {
 	{HID_USAGE_SENSOR_ALS, 0, 1, 0},
 	{HID_USAGE_SENSOR_ALS, HID_USAGE_SENSOR_UNITS_LUX, 1, 0},
 
-	{HID_USAGE_SENSOR_PRESSURE, 0, 100000, 0},
-	{HID_USAGE_SENSOR_PRESSURE, HID_USAGE_SENSOR_UNITS_PASCAL, 1, 0},
+	{HID_USAGE_SENSOR_PRESSURE, 0, 100, 0},
+	{HID_USAGE_SENSOR_PRESSURE, HID_USAGE_SENSOR_UNITS_PASCAL, 0, 1000},
 };
 
 static int pow_10(unsigned power)
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
index 5b41f9d0d4f3..a3cce3a38300 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
@@ -122,6 +122,14 @@ int hid_sensor_power_state(struct hid_sensor_common *st, bool state)
 #endif
 }
 
+static void hid_sensor_set_power_work(struct work_struct *work)
+{
+	struct hid_sensor_common *attrb = container_of(work,
+						       struct hid_sensor_common,
+						       work);
+	_hid_sensor_power_state(attrb, true);
+}
+
 static int hid_sensor_data_rdy_trigger_set_state(struct iio_trigger *trig,
 						bool state)
 {
@@ -130,6 +138,7 @@ static int hid_sensor_data_rdy_trigger_set_state(struct iio_trigger *trig,
 
 void hid_sensor_remove_trigger(struct hid_sensor_common *attrb)
 {
+	cancel_work_sync(&attrb->work);
 	iio_trigger_unregister(attrb->trigger);
 	iio_trigger_free(attrb->trigger);
 }
@@ -170,6 +179,9 @@ int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
 		goto error_unreg_trigger;
 
 	iio_device_set_drvdata(indio_dev, attrb);
+
+	INIT_WORK(&attrb->work, hid_sensor_set_power_work);
+
 	pm_suspend_ignore_children(&attrb->pdev->dev, true);
 	pm_runtime_enable(&attrb->pdev->dev);
 	/* Default to 3 seconds, but can be changed from sysfs */
@@ -187,8 +199,7 @@ error_ret:
 }
 EXPORT_SYMBOL(hid_sensor_setup_trigger);
 
-#ifdef CONFIG_PM
-static int hid_sensor_suspend(struct device *dev)
+static int __maybe_unused hid_sensor_suspend(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
@@ -197,21 +208,27 @@ static int hid_sensor_suspend(struct device *dev)
 	return _hid_sensor_power_state(attrb, false);
 }
 
-static int hid_sensor_resume(struct device *dev)
+static int __maybe_unused hid_sensor_resume(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct hid_sensor_common *attrb = iio_device_get_drvdata(indio_dev);
+	schedule_work(&attrb->work);
+	return 0;
+}
 
+static int __maybe_unused hid_sensor_runtime_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct hid_sensor_common *attrb = iio_device_get_drvdata(indio_dev);
 	return _hid_sensor_power_state(attrb, true);
 }
 
-#endif
-
 const struct dev_pm_ops hid_sensor_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(hid_sensor_suspend, hid_sensor_resume)
 	SET_RUNTIME_PM_OPS(hid_sensor_suspend,
-			   hid_sensor_resume, NULL)
+			   hid_sensor_runtime_resume, NULL)
 };
 EXPORT_SYMBOL(hid_sensor_pm_ops);
 
diff --git a/drivers/iio/common/st_sensors/st_sensors_buffer.c b/drivers/iio/common/st_sensors/st_sensors_buffer.c
index d06e728cea37..fe7775bb3740 100644
--- a/drivers/iio/common/st_sensors/st_sensors_buffer.c
+++ b/drivers/iio/common/st_sensors/st_sensors_buffer.c
@@ -63,7 +63,7 @@ irqreturn_t st_sensors_trigger_handler(int irq, void *p)
 	 * the hardware trigger) and the hw_timestamp may get updated.
 	 * By storing it in a local variable first, we are safe.
 	 */
-	if (sdata->hw_irq_trigger)
+	if (iio_trigger_using_own(indio_dev))
 		timestamp = sdata->hw_timestamp;
 	else
 		timestamp = iio_get_time_ns(indio_dev);
diff --git a/drivers/iio/common/st_sensors/st_sensors_core.c b/drivers/iio/common/st_sensors/st_sensors_core.c
index 2d5282e05482..285a64a589d7 100644
--- a/drivers/iio/common/st_sensors/st_sensors_core.c
+++ b/drivers/iio/common/st_sensors/st_sensors_core.c
@@ -234,39 +234,35 @@ int st_sensors_power_enable(struct iio_dev *indio_dev)
 	int err;
 
 	/* Regulators not mandatory, but if requested we should enable them. */
-	pdata->vdd = devm_regulator_get_optional(indio_dev->dev.parent, "vdd");
-	if (!IS_ERR(pdata->vdd)) {
-		err = regulator_enable(pdata->vdd);
-		if (err != 0) {
-			dev_warn(&indio_dev->dev,
-				 "Failed to enable specified Vdd supply\n");
-			return err;
-		}
-	} else {
-		err = PTR_ERR(pdata->vdd);
-		if (err != -ENODEV)
-			return err;
+	pdata->vdd = devm_regulator_get(indio_dev->dev.parent, "vdd");
+	if (IS_ERR(pdata->vdd)) {
+		dev_err(&indio_dev->dev, "unable to get Vdd supply\n");
+		return PTR_ERR(pdata->vdd);
+	}
+	err = regulator_enable(pdata->vdd);
+	if (err != 0) {
+		dev_warn(&indio_dev->dev,
+			 "Failed to enable specified Vdd supply\n");
+		return err;
 	}
 
-	pdata->vdd_io = devm_regulator_get_optional(indio_dev->dev.parent, "vddio");
-	if (!IS_ERR(pdata->vdd_io)) {
-		err = regulator_enable(pdata->vdd_io);
-		if (err != 0) {
-			dev_warn(&indio_dev->dev,
-				 "Failed to enable specified Vdd_IO supply\n");
-			goto st_sensors_disable_vdd;
-		}
-	} else {
+	pdata->vdd_io = devm_regulator_get(indio_dev->dev.parent, "vddio");
+	if (IS_ERR(pdata->vdd_io)) {
+		dev_err(&indio_dev->dev, "unable to get Vdd_IO supply\n");
 		err = PTR_ERR(pdata->vdd_io);
-		if (err != -ENODEV)
-			goto st_sensors_disable_vdd;
+		goto st_sensors_disable_vdd;
+	}
+	err = regulator_enable(pdata->vdd_io);
+	if (err != 0) {
+		dev_warn(&indio_dev->dev,
+			 "Failed to enable specified Vdd_IO supply\n");
+		goto st_sensors_disable_vdd;
 	}
 
 	return 0;
 
 st_sensors_disable_vdd:
-	if (!IS_ERR_OR_NULL(pdata->vdd))
-		regulator_disable(pdata->vdd);
+	regulator_disable(pdata->vdd);
 	return err;
 }
 EXPORT_SYMBOL(st_sensors_power_enable);
@@ -275,11 +271,8 @@ void st_sensors_power_disable(struct iio_dev *indio_dev)
 {
 	struct st_sensor_data *pdata = iio_priv(indio_dev);
 
-	if (!IS_ERR_OR_NULL(pdata->vdd))
-		regulator_disable(pdata->vdd);
-
-	if (!IS_ERR_OR_NULL(pdata->vdd_io))
-		regulator_disable(pdata->vdd_io);
+	regulator_disable(pdata->vdd);
+	regulator_disable(pdata->vdd_io);
 }
 EXPORT_SYMBOL(st_sensors_power_disable);
 
diff --git a/drivers/iio/common/st_sensors/st_sensors_trigger.c b/drivers/iio/common/st_sensors/st_sensors_trigger.c
index e66f12ee8a55..fa73e6795359 100644
--- a/drivers/iio/common/st_sensors/st_sensors_trigger.c
+++ b/drivers/iio/common/st_sensors/st_sensors_trigger.c
@@ -66,7 +66,7 @@ static int st_sensors_new_samples_available(struct iio_dev *indio_dev,
  * @irq: irq number
  * @p: private handler data
  */
-irqreturn_t st_sensors_irq_handler(int irq, void *p)
+static irqreturn_t st_sensors_irq_handler(int irq, void *p)
 {
 	struct iio_trigger *trig = p;
 	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
@@ -82,7 +82,7 @@ irqreturn_t st_sensors_irq_handler(int irq, void *p)
  * @irq: irq number
  * @p: private handler data
  */
-irqreturn_t st_sensors_irq_thread(int irq, void *p)
+static irqreturn_t st_sensors_irq_thread(int irq, void *p)
 {
 	struct iio_trigger *trig = p;
 	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig
index ca814479fadf..120b24478469 100644
--- a/drivers/iio/dac/Kconfig
+++ b/drivers/iio/dac/Kconfig
@@ -181,6 +181,25 @@ config AD7303
 	  To compile this driver as module choose M here: the module will be called
 	  ad7303.
 
+config CIO_DAC
+	tristate "Measurement Computing CIO-DAC IIO driver"
+	depends on X86 && ISA_BUS_API
+	help
+	  Say yes here to build support for the Measurement Computing CIO-DAC
+	  analog output device family (CIO-DAC16, CIO-DAC08, PC104-DAC06). The
+	  base port addresses for the devices may be configured via the base
+	  array module parameter.
+
+config AD8801
+	tristate "Analog Devices AD8801/AD8803 DAC driver"
+	depends on SPI_MASTER
+	help
+	  Say yes here to build support for Analog Devices AD8801, AD8803 Digital to
+	  Analog Converters (DAC).
+
+	  To compile this driver as a module choose M here: the module will be called
+	  ad8801.
+
 config LPC18XX_DAC
 	tristate "NXP LPC18xx DAC driver"
 	depends on ARCH_LPC18XX || COMPILE_TEST
@@ -245,16 +264,6 @@ config MCP4922
 	  To compile this driver as a module, choose M here: the module
 	  will be called mcp4922.
 
-config STX104
-	tristate "Apex Embedded Systems STX104 DAC driver"
-	depends on X86 && ISA_BUS_API
-	select GPIOLIB
-	help
-	  Say yes here to build support for the 2-channel DAC and GPIO on the
-	  Apex Embedded Systems STX104 integrated analog PC/104 card. The base
-	  port addresses for the devices may be configured via the base array
-	  module parameter.
-
 config VF610_DAC
 	tristate "Vybrid vf610 DAC driver"
 	depends on OF
diff --git a/drivers/iio/dac/Makefile b/drivers/iio/dac/Makefile
index 8b78d5ca9b11..27642bbf75f2 100644
--- a/drivers/iio/dac/Makefile
+++ b/drivers/iio/dac/Makefile
@@ -20,11 +20,12 @@ obj-$(CONFIG_AD5764) += ad5764.o
 obj-$(CONFIG_AD5791) += ad5791.o
 obj-$(CONFIG_AD5686) += ad5686.o
 obj-$(CONFIG_AD7303) += ad7303.o
+obj-$(CONFIG_AD8801) += ad8801.o
+obj-$(CONFIG_CIO_DAC) += cio-dac.o
 obj-$(CONFIG_LPC18XX_DAC) += lpc18xx_dac.o
 obj-$(CONFIG_M62332) += m62332.o
 obj-$(CONFIG_MAX517) += max517.o
 obj-$(CONFIG_MAX5821) += max5821.o
 obj-$(CONFIG_MCP4725) += mcp4725.o
 obj-$(CONFIG_MCP4922) += mcp4922.o
-obj-$(CONFIG_STX104) += stx104.o
 obj-$(CONFIG_VF610_DAC) += vf610_dac.o
diff --git a/drivers/iio/dac/ad5755.c b/drivers/iio/dac/ad5755.c
index 0fde593ec0d9..5f7968232564 100644
--- a/drivers/iio/dac/ad5755.c
+++ b/drivers/iio/dac/ad5755.c
@@ -655,7 +655,7 @@ static struct ad5755_platform_data *ad5755_parse_dt(struct device *dev)
 
 	devnr = 0;
 	for_each_child_of_node(np, pp) {
-		if (devnr > AD5755_NUM_CHANNELS) {
+		if (devnr >= AD5755_NUM_CHANNELS) {
 			dev_err(dev,
 				"There is to many channels defined in DT\n");
 			goto error_out;
diff --git a/drivers/iio/dac/ad8801.c b/drivers/iio/dac/ad8801.c
new file mode 100644
index 000000000000..f06faa1aec09
--- /dev/null
+++ b/drivers/iio/dac/ad8801.c
@@ -0,0 +1,239 @@
+/*
+ * IIO DAC driver for Analog Devices AD8801 DAC
+ *
+ * Copyright (C) 2016 Gwenhael Goavec-Merou
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ */
+
+#include <linux/iio/iio.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
+#include <linux/spi/spi.h>
+#include <linux/sysfs.h>
+
+#define AD8801_CFG_ADDR_OFFSET 8
+
+enum ad8801_device_ids {
+	ID_AD8801,
+	ID_AD8803,
+};
+
+struct ad8801_state {
+	struct spi_device *spi;
+	unsigned char dac_cache[8]; /* Value write on each channel */
+	unsigned int vrefh_mv;
+	unsigned int vrefl_mv;
+	struct regulator *vrefh_reg;
+	struct regulator *vrefl_reg;
+
+	__be16 data ____cacheline_aligned;
+};
+
+static int ad8801_spi_write(struct ad8801_state *state,
+	u8 channel, unsigned char value)
+{
+	state->data = cpu_to_be16((channel << AD8801_CFG_ADDR_OFFSET) | value);
+	return spi_write(state->spi, &state->data, sizeof(state->data));
+}
+
+static int ad8801_write_raw(struct iio_dev *indio_dev,
+	struct iio_chan_spec const *chan, int val, int val2, long mask)
+{
+	struct ad8801_state *state = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		if (val >= 256 || val < 0)
+			return -EINVAL;
+
+		ret = ad8801_spi_write(state, chan->channel, val);
+		if (ret == 0)
+			state->dac_cache[chan->channel] = val;
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int ad8801_read_raw(struct iio_dev *indio_dev,
+	struct iio_chan_spec const *chan, int *val, int *val2, long info)
+{
+	struct ad8801_state *state = iio_priv(indio_dev);
+
+	switch (info) {
+	case IIO_CHAN_INFO_RAW:
+		*val = state->dac_cache[chan->channel];
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = state->vrefh_mv - state->vrefl_mv;
+		*val2 = 8;
+		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_OFFSET:
+		*val = state->vrefl_mv;
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+
+	return -EINVAL;
+}
+
+static const struct iio_info ad8801_info = {
+	.read_raw = ad8801_read_raw,
+	.write_raw = ad8801_write_raw,
+	.driver_module = THIS_MODULE,
+};
+
+#define AD8801_CHANNEL(chan) {		\
+	.type = IIO_VOLTAGE,			\
+	.indexed = 1,				\
+	.output = 1,				\
+	.channel = chan,			\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |	\
+		BIT(IIO_CHAN_INFO_OFFSET), \
+}
+
+static const struct iio_chan_spec ad8801_channels[] = {
+	AD8801_CHANNEL(0),
+	AD8801_CHANNEL(1),
+	AD8801_CHANNEL(2),
+	AD8801_CHANNEL(3),
+	AD8801_CHANNEL(4),
+	AD8801_CHANNEL(5),
+	AD8801_CHANNEL(6),
+	AD8801_CHANNEL(7),
+};
+
+static int ad8801_probe(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev;
+	struct ad8801_state *state;
+	const struct spi_device_id *id;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*state));
+	if (indio_dev == NULL)
+		return -ENOMEM;
+
+	state = iio_priv(indio_dev);
+	state->spi = spi;
+	id = spi_get_device_id(spi);
+
+	state->vrefh_reg = devm_regulator_get(&spi->dev, "vrefh");
+	if (IS_ERR(state->vrefh_reg)) {
+		dev_err(&spi->dev, "Vrefh regulator not specified\n");
+		return PTR_ERR(state->vrefh_reg);
+	}
+
+	ret = regulator_enable(state->vrefh_reg);
+	if (ret) {
+		dev_err(&spi->dev, "Failed to enable vrefh regulator: %d\n",
+				ret);
+		return ret;
+	}
+
+	ret = regulator_get_voltage(state->vrefh_reg);
+	if (ret < 0) {
+		dev_err(&spi->dev, "Failed to read vrefh regulator: %d\n",
+				ret);
+		goto error_disable_vrefh_reg;
+	}
+	state->vrefh_mv = ret / 1000;
+
+	if (id->driver_data == ID_AD8803) {
+		state->vrefl_reg = devm_regulator_get(&spi->dev, "vrefl");
+		if (IS_ERR(state->vrefl_reg)) {
+			dev_err(&spi->dev, "Vrefl regulator not specified\n");
+			ret = PTR_ERR(state->vrefl_reg);
+			goto error_disable_vrefh_reg;
+		}
+
+		ret = regulator_enable(state->vrefl_reg);
+		if (ret) {
+			dev_err(&spi->dev, "Failed to enable vrefl regulator: %d\n",
+					ret);
+			goto error_disable_vrefh_reg;
+		}
+
+		ret = regulator_get_voltage(state->vrefl_reg);
+		if (ret < 0) {
+			dev_err(&spi->dev, "Failed to read vrefl regulator: %d\n",
+					ret);
+			goto error_disable_vrefl_reg;
+		}
+		state->vrefl_mv = ret / 1000;
+	} else {
+		state->vrefl_mv = 0;
+		state->vrefl_reg = NULL;
+	}
+
+	spi_set_drvdata(spi, indio_dev);
+	indio_dev->dev.parent = &spi->dev;
+	indio_dev->info = &ad8801_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = ad8801_channels;
+	indio_dev->num_channels = ARRAY_SIZE(ad8801_channels);
+	indio_dev->name = id->name;
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&spi->dev, "Failed to register iio device: %d\n",
+				ret);
+		goto error_disable_vrefl_reg;
+	}
+
+	return 0;
+
+error_disable_vrefl_reg:
+	if (state->vrefl_reg)
+		regulator_disable(state->vrefl_reg);
+error_disable_vrefh_reg:
+	regulator_disable(state->vrefh_reg);
+	return ret;
+}
+
+static int ad8801_remove(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+	struct ad8801_state *state = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+	if (state->vrefl_reg)
+		regulator_disable(state->vrefl_reg);
+	regulator_disable(state->vrefh_reg);
+
+	return 0;
+}
+
+static const struct spi_device_id ad8801_ids[] = {
+	{"ad8801", ID_AD8801},
+	{"ad8803", ID_AD8803},
+	{}
+};
+MODULE_DEVICE_TABLE(spi, ad8801_ids);
+
+static struct spi_driver ad8801_driver = {
+	.driver = {
+		.name	= "ad8801",
+	},
+	.probe		= ad8801_probe,
+	.remove		= ad8801_remove,
+	.id_table	= ad8801_ids,
+};
+module_spi_driver(ad8801_driver);
+
+MODULE_AUTHOR("Gwenhael Goavec-Merou <gwenhael.goavec-merou@trabucayre.com>");
+MODULE_DESCRIPTION("Analog Devices AD8801/AD8803 DAC");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/dac/cio-dac.c b/drivers/iio/dac/cio-dac.c
new file mode 100644
index 000000000000..5a743e2a779d
--- /dev/null
+++ b/drivers/iio/dac/cio-dac.c
@@ -0,0 +1,144 @@
+/*
+ * IIO driver for the Measurement Computing CIO-DAC
+ * Copyright (C) 2016 William Breathitt Gray
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * This driver supports the following Measurement Computing devices: CIO-DAC16,
+ * CIO-DAC06, and PC104-DAC06.
+ */
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/types.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/isa.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+
+#define CIO_DAC_NUM_CHAN 16
+
+#define CIO_DAC_CHAN(chan) {				\
+	.type = IIO_VOLTAGE,				\
+	.channel = chan,				\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
+	.indexed = 1,					\
+	.output = 1					\
+}
+
+#define CIO_DAC_EXTENT 32
+
+static unsigned int base[max_num_isa_dev(CIO_DAC_EXTENT)];
+static unsigned int num_cio_dac;
+module_param_array(base, uint, &num_cio_dac, 0);
+MODULE_PARM_DESC(base, "Measurement Computing CIO-DAC base addresses");
+
+/**
+ * struct cio_dac_iio - IIO device private data structure
+ * @chan_out_states:	channels' output states
+ * @base:		base port address of the IIO device
+ */
+struct cio_dac_iio {
+	int chan_out_states[CIO_DAC_NUM_CHAN];
+	unsigned int base;
+};
+
+static int cio_dac_read_raw(struct iio_dev *indio_dev,
+	struct iio_chan_spec const *chan, int *val, int *val2, long mask)
+{
+	struct cio_dac_iio *const priv = iio_priv(indio_dev);
+
+	if (mask != IIO_CHAN_INFO_RAW)
+		return -EINVAL;
+
+	*val = priv->chan_out_states[chan->channel];
+
+	return IIO_VAL_INT;
+}
+
+static int cio_dac_write_raw(struct iio_dev *indio_dev,
+	struct iio_chan_spec const *chan, int val, int val2, long mask)
+{
+	struct cio_dac_iio *const priv = iio_priv(indio_dev);
+	const unsigned int chan_addr_offset = 2 * chan->channel;
+
+	if (mask != IIO_CHAN_INFO_RAW)
+		return -EINVAL;
+
+	/* DAC can only accept up to a 16-bit value */
+	if ((unsigned int)val > 65535)
+		return -EINVAL;
+
+	priv->chan_out_states[chan->channel] = val;
+	outw(val, priv->base + chan_addr_offset);
+
+	return 0;
+}
+
+static const struct iio_info cio_dac_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = cio_dac_read_raw,
+	.write_raw = cio_dac_write_raw
+};
+
+static const struct iio_chan_spec cio_dac_channels[CIO_DAC_NUM_CHAN] = {
+	CIO_DAC_CHAN(0), CIO_DAC_CHAN(1), CIO_DAC_CHAN(2), CIO_DAC_CHAN(3),
+	CIO_DAC_CHAN(4), CIO_DAC_CHAN(5), CIO_DAC_CHAN(6), CIO_DAC_CHAN(7),
+	CIO_DAC_CHAN(8), CIO_DAC_CHAN(9), CIO_DAC_CHAN(10), CIO_DAC_CHAN(11),
+	CIO_DAC_CHAN(12), CIO_DAC_CHAN(13), CIO_DAC_CHAN(14), CIO_DAC_CHAN(15)
+};
+
+static int cio_dac_probe(struct device *dev, unsigned int id)
+{
+	struct iio_dev *indio_dev;
+	struct cio_dac_iio *priv;
+	unsigned int i;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*priv));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	if (!devm_request_region(dev, base[id], CIO_DAC_EXTENT,
+		dev_name(dev))) {
+		dev_err(dev, "Unable to request port addresses (0x%X-0x%X)\n",
+			base[id], base[id] + CIO_DAC_EXTENT);
+		return -EBUSY;
+	}
+
+	indio_dev->info = &cio_dac_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = cio_dac_channels;
+	indio_dev->num_channels = CIO_DAC_NUM_CHAN;
+	indio_dev->name = dev_name(dev);
+
+	priv = iio_priv(indio_dev);
+	priv->base = base[id];
+
+	/* initialize DAC outputs to 0V */
+	for (i = 0; i < 32; i += 2)
+		outw(0, base[id] + i);
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static struct isa_driver cio_dac_driver = {
+	.probe = cio_dac_probe,
+	.driver = {
+		.name = "cio-dac"
+	}
+};
+
+module_isa_driver(cio_dac_driver, num_cio_dac);
+
+MODULE_AUTHOR("William Breathitt Gray <vilhelm.gray@gmail.com>");
+MODULE_DESCRIPTION("Measurement Computing CIO-DAC IIO driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/gyro/ssp_gyro_sensor.c b/drivers/iio/gyro/ssp_gyro_sensor.c
index 0a8afdd21728..1f25f406c545 100644
--- a/drivers/iio/gyro/ssp_gyro_sensor.c
+++ b/drivers/iio/gyro/ssp_gyro_sensor.c
@@ -74,7 +74,7 @@ static int ssp_gyro_write_raw(struct iio_dev *indio_dev,
 	return -EINVAL;
 }
 
-static struct iio_info ssp_gyro_iio_info = {
+static const struct iio_info ssp_gyro_iio_info = {
 	.read_raw = &ssp_gyro_read_raw,
 	.write_raw = &ssp_gyro_write_raw,
 };
diff --git a/drivers/iio/humidity/Kconfig b/drivers/iio/humidity/Kconfig
index 738a86d9e4a9..b17e2e2bd4f5 100644
--- a/drivers/iio/humidity/Kconfig
+++ b/drivers/iio/humidity/Kconfig
@@ -6,6 +6,8 @@ menu "Humidity sensors"
 config AM2315
     tristate "Aosong AM2315 relative humidity and temperature sensor"
     depends on I2C
+    select IIO_BUFFER
+    select IIO_TRIGGERED_BUFFER
     help
       If you say yes here you get support for the Aosong AM2315
       relative humidity and ambient temperature sensor.
@@ -26,11 +28,11 @@ config HDC100X
 	tristate "TI HDC100x relative humidity and temperature sensor"
 	depends on I2C
 	help
-	 Say yes here to build support for the TI HDC100x series of
-	 relative humidity and temperature sensors.
+	  Say yes here to build support for the Texas Instruments
+	  HDC1000 and HDC1008 relative humidity and temperature sensors.
 
-	 To compile this driver as a module, choose M here: the module
-	 will be called hdc100x.
+	  To compile this driver as a module, choose M here: the module
+	  will be called hdc100x.
 
 config HTU21
 	tristate "Measurement Specialties HTU21 humidity & temperature sensor"
diff --git a/drivers/iio/humidity/am2315.c b/drivers/iio/humidity/am2315.c
index 3e200f69e886..ff96b6d0fdae 100644
--- a/drivers/iio/humidity/am2315.c
+++ b/drivers/iio/humidity/am2315.c
@@ -244,7 +244,7 @@ static int am2315_probe(struct i2c_client *client,
 	indio_dev->channels = am2315_channels;
 	indio_dev->num_channels = ARRAY_SIZE(am2315_channels);
 
-	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+	ret = iio_triggered_buffer_setup(indio_dev, iio_pollfunc_store_time,
 					 am2315_trigger_handler, NULL);
 	if (ret < 0) {
 		dev_err(&client->dev, "iio triggered buffer setup failed\n");
diff --git a/drivers/iio/humidity/hdc100x.c b/drivers/iio/humidity/hdc100x.c
index a03832a5fc95..e0c9c70c2a4a 100644
--- a/drivers/iio/humidity/hdc100x.c
+++ b/drivers/iio/humidity/hdc100x.c
@@ -142,7 +142,7 @@ static int hdc100x_get_measurement(struct hdc100x_data *data,
 	struct i2c_client *client = data->client;
 	int delay = data->adc_int_us[chan->address];
 	int ret;
-	int val;
+	__be16 val;
 
 	/* start measurement */
 	ret = i2c_smbus_write_byte(client, chan->address);
@@ -154,26 +154,13 @@ static int hdc100x_get_measurement(struct hdc100x_data *data,
 	/* wait for integration time to pass */
 	usleep_range(delay, delay + 1000);
 
-	/*
-	 * i2c_smbus_read_word_data cannot() be used here due to the command
-	 * value not being understood and causes NAKs preventing any reading
-	 * from being accessed.
-	 */
-	ret = i2c_smbus_read_byte(client);
+	/* read measurement */
+	ret = i2c_master_recv(data->client, (char *)&val, sizeof(val));
 	if (ret < 0) {
-		dev_err(&client->dev, "cannot read high byte measurement");
+		dev_err(&client->dev, "cannot read sensor data\n");
 		return ret;
 	}
-	val = ret << 8;
-
-	ret = i2c_smbus_read_byte(client);
-	if (ret < 0) {
-		dev_err(&client->dev, "cannot read low byte measurement");
-		return ret;
-	}
-	val |= ret;
-
-	return val;
+	return be16_to_cpu(val);
 }
 
 static int hdc100x_get_heater_status(struct hdc100x_data *data)
@@ -272,8 +259,8 @@ static int hdc100x_probe(struct i2c_client *client,
 	struct iio_dev *indio_dev;
 	struct hdc100x_data *data;
 
-	if (!i2c_check_functionality(client->adapter,
-				I2C_FUNC_SMBUS_WORD_DATA | I2C_FUNC_SMBUS_BYTE))
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_WORD_DATA |
+				     I2C_FUNC_SMBUS_BYTE | I2C_FUNC_I2C))
 		return -EOPNOTSUPP;
 
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c
index 90462fcf5436..158aaf44dd95 100644
--- a/drivers/iio/industrialio-buffer.c
+++ b/drivers/iio/industrialio-buffer.c
@@ -107,9 +107,10 @@ ssize_t iio_buffer_read_first_n_outer(struct file *filp, char __user *buf,
 {
 	struct iio_dev *indio_dev = filp->private_data;
 	struct iio_buffer *rb = indio_dev->buffer;
+	DEFINE_WAIT_FUNC(wait, woken_wake_function);
 	size_t datum_size;
 	size_t to_wait;
-	int ret;
+	int ret = 0;
 
 	if (!indio_dev->info)
 		return -ENODEV;
@@ -131,19 +132,29 @@ ssize_t iio_buffer_read_first_n_outer(struct file *filp, char __user *buf,
 	else
 		to_wait = min_t(size_t, n / datum_size, rb->watermark);
 
+	add_wait_queue(&rb->pollq, &wait);
 	do {
-		ret = wait_event_interruptible(rb->pollq,
-		      iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size));
-		if (ret)
-			return ret;
+		if (!indio_dev->info) {
+			ret = -ENODEV;
+			break;
+		}
 
-		if (!indio_dev->info)
-			return -ENODEV;
+		if (!iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size)) {
+			if (signal_pending(current)) {
+				ret = -ERESTARTSYS;
+				break;
+			}
+
+			wait_woken(&wait, TASK_INTERRUPTIBLE,
+				   MAX_SCHEDULE_TIMEOUT);
+			continue;
+		}
 
 		ret = rb->access->read_first_n(rb, n, buf);
 		if (ret == 0 && (filp->f_flags & O_NONBLOCK))
 			ret = -EAGAIN;
-	 } while (ret == 0);
+	} while (ret == 0);
+	remove_wait_queue(&rb->pollq, &wait);
 
 	return ret;
 }
diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
index f914d5d140e4..fc340ed3dca1 100644
--- a/drivers/iio/industrialio-core.c
+++ b/drivers/iio/industrialio-core.c
@@ -613,9 +613,8 @@ ssize_t iio_format_value(char *buf, unsigned int type, int size, int *vals)
 			return sprintf(buf, "%d.%09u\n", vals[0], vals[1]);
 	case IIO_VAL_FRACTIONAL:
 		tmp = div_s64((s64)vals[0] * 1000000000LL, vals[1]);
-		vals[1] = do_div(tmp, 1000000000LL);
-		vals[0] = tmp;
-		return sprintf(buf, "%d.%09u\n", vals[0], vals[1]);
+		vals[0] = (int)div_s64_rem(tmp, 1000000000, &vals[1]);
+		return sprintf(buf, "%d.%09u\n", vals[0], abs(vals[1]));
 	case IIO_VAL_FRACTIONAL_LOG2:
 		tmp = (s64)vals[0] * 1000000000LL >> vals[1];
 		vals[1] = do_div(tmp, 1000000000LL);
@@ -1309,7 +1308,7 @@ static void devm_iio_device_release(struct device *dev, void *res)
 	iio_device_free(*(struct iio_dev **)res);
 }
 
-static int devm_iio_device_match(struct device *dev, void *res, void *data)
+int devm_iio_device_match(struct device *dev, void *res, void *data)
 {
 	struct iio_dev **r = res;
 	if (!r || !*r) {
@@ -1318,6 +1317,7 @@ static int devm_iio_device_match(struct device *dev, void *res, void *data)
 	}
 	return *r == data;
 }
+EXPORT_SYMBOL_GPL(devm_iio_device_match);
 
 /**
  * devm_iio_device_alloc - Resource-managed iio_device_alloc()
diff --git a/drivers/iio/industrialio-event.c b/drivers/iio/industrialio-event.c
index 0ebfc923a997..90fac8ec63c9 100644
--- a/drivers/iio/industrialio-event.c
+++ b/drivers/iio/industrialio-event.c
@@ -57,6 +57,11 @@ bool iio_event_enabled(const struct iio_event_interface *ev_int)
  *
  * Note: The caller must make sure that this function is not running
  * concurrently for the same indio_dev more than once.
+ *
+ * This function may be safely used as soon as a valid reference to iio_dev has
+ * been obtained via iio_device_alloc(), but any events that are submitted
+ * before iio_device_register() has successfully completed will be silently
+ * discarded.
  **/
 int iio_push_event(struct iio_dev *indio_dev, u64 ev_code, s64 timestamp)
 {
@@ -64,6 +69,9 @@ int iio_push_event(struct iio_dev *indio_dev, u64 ev_code, s64 timestamp)
 	struct iio_event_data ev;
 	int copied;
 
+	if (!ev_int)
+		return 0;
+
 	/* Does anyone care? */
 	if (iio_event_enabled(ev_int)) {
 
diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c
index 7ad82fdd3e5b..e1e104845e38 100644
--- a/drivers/iio/industrialio-trigger.c
+++ b/drivers/iio/industrialio-trigger.c
@@ -119,6 +119,22 @@ void iio_trigger_unregister(struct iio_trigger *trig_info)
 }
 EXPORT_SYMBOL(iio_trigger_unregister);
 
+int iio_trigger_set_immutable(struct iio_dev *indio_dev, struct iio_trigger *trig)
+{
+	if (!indio_dev || !trig)
+		return -EINVAL;
+
+	mutex_lock(&indio_dev->mlock);
+	WARN_ON(indio_dev->trig_readonly);
+
+	indio_dev->trig = iio_trigger_get(trig);
+	indio_dev->trig_readonly = true;
+	mutex_unlock(&indio_dev->mlock);
+
+	return 0;
+}
+EXPORT_SYMBOL(iio_trigger_set_immutable);
+
 /* Search for trigger by name, assuming iio_trigger_list_lock held */
 static struct iio_trigger *__iio_trigger_find_by_name(const char *name)
 {
@@ -255,6 +271,14 @@ static int iio_trigger_attach_poll_func(struct iio_trigger *trig,
 			goto out_free_irq;
 	}
 
+	/*
+	 * Check if we just registered to our own trigger: we determine that
+	 * this is the case if the IIO device and the trigger device share the
+	 * same parent device.
+	 */
+	if (pf->indio_dev->dev.parent == trig->dev.parent)
+		trig->attached_own_device = true;
+
 	return ret;
 
 out_free_irq:
@@ -279,6 +303,8 @@ static int iio_trigger_detach_poll_func(struct iio_trigger *trig,
 		if (ret)
 			return ret;
 	}
+	if (pf->indio_dev->dev.parent == trig->dev.parent)
+		trig->attached_own_device = false;
 	iio_trigger_put_irq(trig, pf->irq);
 	free_irq(pf->irq, pf);
 	module_put(pf->indio_dev->info->driver_module);
@@ -384,6 +410,10 @@ static ssize_t iio_trigger_write_current(struct device *dev,
 		mutex_unlock(&indio_dev->mlock);
 		return -EBUSY;
 	}
+	if (indio_dev->trig_readonly) {
+		mutex_unlock(&indio_dev->mlock);
+		return -EPERM;
+	}
 	mutex_unlock(&indio_dev->mlock);
 
 	trig = iio_trigger_find_by_name(buf, len);
@@ -622,6 +652,71 @@ void devm_iio_trigger_free(struct device *dev, struct iio_trigger *iio_trig)
 }
 EXPORT_SYMBOL_GPL(devm_iio_trigger_free);
 
+static void devm_iio_trigger_unreg(struct device *dev, void *res)
+{
+	iio_trigger_unregister(*(struct iio_trigger **)res);
+}
+
+/**
+ * devm_iio_trigger_register - Resource-managed iio_trigger_register()
+ * @dev:	device this trigger was allocated for
+ * @trig_info:	trigger to register
+ *
+ * Managed iio_trigger_register().  The IIO trigger registered with this
+ * function is automatically unregistered on driver detach. This function
+ * calls iio_trigger_register() internally. Refer to that function for more
+ * information.
+ *
+ * If an iio_trigger registered with this function needs to be unregistered
+ * separately, devm_iio_trigger_unregister() must be used.
+ *
+ * RETURNS:
+ * 0 on success, negative error number on failure.
+ */
+int devm_iio_trigger_register(struct device *dev, struct iio_trigger *trig_info)
+{
+	struct iio_trigger **ptr;
+	int ret;
+
+	ptr = devres_alloc(devm_iio_trigger_unreg, sizeof(*ptr), GFP_KERNEL);
+	if (!ptr)
+		return -ENOMEM;
+
+	*ptr = trig_info;
+	ret = iio_trigger_register(trig_info);
+	if (!ret)
+		devres_add(dev, ptr);
+	else
+		devres_free(ptr);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(devm_iio_trigger_register);
+
+/**
+ * devm_iio_trigger_unregister - Resource-managed iio_trigger_unregister()
+ * @dev:	device this iio_trigger belongs to
+ * @trig_info:	the trigger associated with the device
+ *
+ * Unregister trigger registered with devm_iio_trigger_register().
+ */
+void devm_iio_trigger_unregister(struct device *dev,
+				 struct iio_trigger *trig_info)
+{
+	int rc;
+
+	rc = devres_release(dev, devm_iio_trigger_unreg, devm_iio_trigger_match,
+			    trig_info);
+	WARN_ON(rc);
+}
+EXPORT_SYMBOL_GPL(devm_iio_trigger_unregister);
+
+bool iio_trigger_using_own(struct iio_dev *indio_dev)
+{
+	return indio_dev->trig->attached_own_device;
+}
+EXPORT_SYMBOL(iio_trigger_using_own);
+
 void iio_device_register_trigger_consumer(struct iio_dev *indio_dev)
 {
 	indio_dev->groups[indio_dev->groupcounter++] =
diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig
index 7c566f516572..ba2e64d7ee58 100644
--- a/drivers/iio/light/Kconfig
+++ b/drivers/iio/light/Kconfig
@@ -76,7 +76,6 @@ config BH1750
 config BH1780
 	tristate "ROHM BH1780 ambient light sensor"
 	depends on I2C
-	depends on !SENSORS_BH1780
 	help
 	 Say Y here to build support for the ROHM BH1780GLI ambient
 	 light sensor.
@@ -238,6 +237,8 @@ config MAX44000
 	tristate "MAX44000 Ambient and Infrared Proximity Sensor"
 	depends on I2C
 	select REGMAP_I2C
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
 	help
 	 Say Y here if you want to build support for Maxim Integrated's
 	 MAX44000 ambient and infrared proximity sensor device.
@@ -266,6 +267,19 @@ config PA12203001
          This driver can also be built as a module.  If so, the module
          will be called pa12203001.
 
+config SI1145
+	tristate "SI1132 and SI1141/2/3/5/6/7 combined ALS, UV index and proximity sensor"
+	depends on I2C
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	help
+	  Say Y here if you want to build a driver for the Silicon Labs SI1132 or
+	  SI1141/2/3/5/6/7 combined ambient light, UV index and proximity sensor
+	  chips.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called si1145.
+
 config STK3310
 	tristate "STK3310 ALS and proximity sensor"
 	depends on I2C
@@ -333,11 +347,11 @@ config US5182D
 	 will be called us5182d.
 
 config VCNL4000
-	tristate "VCNL4000 combined ALS and proximity sensor"
+	tristate "VCNL4000/4010/4020 combined ALS and proximity sensor"
 	depends on I2C
 	help
-	 Say Y here if you want to build a driver for the Vishay VCNL4000
-	 combined ambient light and proximity sensor.
+	 Say Y here if you want to build a driver for the Vishay VCNL4000,
+	 VCNL4010, VCNL4020 combined ambient light and proximity sensor.
 
 	 To compile this driver as a module, choose M here: the
 	 module will be called vcnl4000.
diff --git a/drivers/iio/light/Makefile b/drivers/iio/light/Makefile
index 6f2a3c62de27..c5768df87a17 100644
--- a/drivers/iio/light/Makefile
+++ b/drivers/iio/light/Makefile
@@ -26,6 +26,7 @@ obj-$(CONFIG_OPT3001)		+= opt3001.o
 obj-$(CONFIG_PA12203001)	+= pa12203001.o
 obj-$(CONFIG_RPR0521)		+= rpr0521.o
 obj-$(CONFIG_SENSORS_TSL2563)	+= tsl2563.o
+obj-$(CONFIG_SI1145)		+= si1145.o
 obj-$(CONFIG_STK3310)          += stk3310.o
 obj-$(CONFIG_TCS3414)		+= tcs3414.o
 obj-$(CONFIG_TCS3472)		+= tcs3472.o
diff --git a/drivers/iio/light/si1145.c b/drivers/iio/light/si1145.c
new file mode 100644
index 000000000000..096034c126a4
--- /dev/null
+++ b/drivers/iio/light/si1145.c
@@ -0,0 +1,1404 @@
+/*
+ * si1145.c - Support for Silabs SI1132 and SI1141/2/3/5/6/7 combined ambient
+ * light, UV index and proximity sensors
+ *
+ * Copyright 2014-16 Peter Meerwald-Stadler <pmeerw@pmeerw.net>
+ * Copyright 2016 Crestez Dan Leonard <leonard.crestez@intel.com>
+ *
+ * This file is subject to the terms and conditions of version 2 of
+ * the GNU General Public License.  See the file COPYING in the main
+ * directory of this archive for more details.
+ *
+ * SI1132 (7-bit I2C slave address 0x60)
+ * SI1141/2/3 (7-bit I2C slave address 0x5a)
+ * SI1145/6/6 (7-bit I2C slave address 0x60)
+ */
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/irq.h>
+#include <linux/gpio.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/buffer.h>
+#include <linux/util_macros.h>
+
+#define SI1145_REG_PART_ID		0x00
+#define SI1145_REG_REV_ID		0x01
+#define SI1145_REG_SEQ_ID		0x02
+#define SI1145_REG_INT_CFG		0x03
+#define SI1145_REG_IRQ_ENABLE		0x04
+#define SI1145_REG_IRQ_MODE		0x05
+#define SI1145_REG_HW_KEY		0x07
+#define SI1145_REG_MEAS_RATE		0x08
+#define SI1145_REG_PS_LED21		0x0f
+#define SI1145_REG_PS_LED3		0x10
+#define SI1145_REG_UCOEF1		0x13
+#define SI1145_REG_UCOEF2		0x14
+#define SI1145_REG_UCOEF3		0x15
+#define SI1145_REG_UCOEF4		0x16
+#define SI1145_REG_PARAM_WR		0x17
+#define SI1145_REG_COMMAND		0x18
+#define SI1145_REG_RESPONSE		0x20
+#define SI1145_REG_IRQ_STATUS		0x21
+#define SI1145_REG_ALSVIS_DATA		0x22
+#define SI1145_REG_ALSIR_DATA		0x24
+#define SI1145_REG_PS1_DATA		0x26
+#define SI1145_REG_PS2_DATA		0x28
+#define SI1145_REG_PS3_DATA		0x2a
+#define SI1145_REG_AUX_DATA		0x2c
+#define SI1145_REG_PARAM_RD		0x2e
+#define SI1145_REG_CHIP_STAT		0x30
+
+#define SI1145_UCOEF1_DEFAULT		0x7b
+#define SI1145_UCOEF2_DEFAULT		0x6b
+#define SI1145_UCOEF3_DEFAULT		0x01
+#define SI1145_UCOEF4_DEFAULT		0x00
+
+/* Helper to figure out PS_LED register / shift per channel */
+#define SI1145_PS_LED_REG(ch) \
+	(((ch) == 2) ? SI1145_REG_PS_LED3 : SI1145_REG_PS_LED21)
+#define SI1145_PS_LED_SHIFT(ch) \
+	(((ch) == 1) ? 4 : 0)
+
+/* Parameter offsets */
+#define SI1145_PARAM_CHLIST		0x01
+#define SI1145_PARAM_PSLED12_SELECT	0x02
+#define SI1145_PARAM_PSLED3_SELECT	0x03
+#define SI1145_PARAM_PS_ENCODING	0x05
+#define SI1145_PARAM_ALS_ENCODING	0x06
+#define SI1145_PARAM_PS1_ADC_MUX	0x07
+#define SI1145_PARAM_PS2_ADC_MUX	0x08
+#define SI1145_PARAM_PS3_ADC_MUX	0x09
+#define SI1145_PARAM_PS_ADC_COUNTER	0x0a
+#define SI1145_PARAM_PS_ADC_GAIN	0x0b
+#define SI1145_PARAM_PS_ADC_MISC	0x0c
+#define SI1145_PARAM_ALS_ADC_MUX	0x0d
+#define SI1145_PARAM_ALSIR_ADC_MUX	0x0e
+#define SI1145_PARAM_AUX_ADC_MUX	0x0f
+#define SI1145_PARAM_ALSVIS_ADC_COUNTER	0x10
+#define SI1145_PARAM_ALSVIS_ADC_GAIN	0x11
+#define SI1145_PARAM_ALSVIS_ADC_MISC	0x12
+#define SI1145_PARAM_LED_RECOVERY	0x1c
+#define SI1145_PARAM_ALSIR_ADC_COUNTER	0x1d
+#define SI1145_PARAM_ALSIR_ADC_GAIN	0x1e
+#define SI1145_PARAM_ALSIR_ADC_MISC	0x1f
+#define SI1145_PARAM_ADC_OFFSET		0x1a
+
+/* Channel enable masks for CHLIST parameter */
+#define SI1145_CHLIST_EN_PS1		BIT(0)
+#define SI1145_CHLIST_EN_PS2		BIT(1)
+#define SI1145_CHLIST_EN_PS3		BIT(2)
+#define SI1145_CHLIST_EN_ALSVIS		BIT(4)
+#define SI1145_CHLIST_EN_ALSIR		BIT(5)
+#define SI1145_CHLIST_EN_AUX		BIT(6)
+#define SI1145_CHLIST_EN_UV		BIT(7)
+
+/* Proximity measurement mode for ADC_MISC parameter */
+#define SI1145_PS_ADC_MODE_NORMAL	BIT(2)
+/* Signal range mask for ADC_MISC parameter */
+#define SI1145_ADC_MISC_RANGE		BIT(5)
+
+/* Commands for REG_COMMAND */
+#define SI1145_CMD_NOP			0x00
+#define SI1145_CMD_RESET		0x01
+#define SI1145_CMD_PS_FORCE		0x05
+#define SI1145_CMD_ALS_FORCE		0x06
+#define SI1145_CMD_PSALS_FORCE		0x07
+#define SI1145_CMD_PS_PAUSE		0x09
+#define SI1145_CMD_ALS_PAUSE		0x0a
+#define SI1145_CMD_PSALS_PAUSE		0x0b
+#define SI1145_CMD_PS_AUTO		0x0d
+#define SI1145_CMD_ALS_AUTO		0x0e
+#define SI1145_CMD_PSALS_AUTO		0x0f
+#define SI1145_CMD_PARAM_QUERY		0x80
+#define SI1145_CMD_PARAM_SET		0xa0
+
+#define SI1145_RSP_INVALID_SETTING	0x80
+#define SI1145_RSP_COUNTER_MASK		0x0F
+
+/* Minimum sleep after each command to ensure it's received */
+#define SI1145_COMMAND_MINSLEEP_MS	5
+/* Return -ETIMEDOUT after this long */
+#define SI1145_COMMAND_TIMEOUT_MS	25
+
+/* Interrupt configuration masks for INT_CFG register */
+#define SI1145_INT_CFG_OE		BIT(0) /* enable interrupt */
+#define SI1145_INT_CFG_MODE		BIT(1) /* auto reset interrupt pin */
+
+/* Interrupt enable masks for IRQ_ENABLE register */
+#define SI1145_MASK_ALL_IE		(BIT(4) | BIT(3) | BIT(2) | BIT(0))
+
+#define SI1145_MUX_TEMP			0x65
+#define SI1145_MUX_VDD			0x75
+
+/* Proximity LED current; see Table 2 in datasheet */
+#define SI1145_LED_CURRENT_45mA		0x04
+
+enum {
+	SI1132,
+	SI1141,
+	SI1142,
+	SI1143,
+	SI1145,
+	SI1146,
+	SI1147,
+};
+
+struct si1145_part_info {
+	u8 part;
+	const struct iio_info *iio_info;
+	const struct iio_chan_spec *channels;
+	unsigned int num_channels;
+	unsigned int num_leds;
+	bool uncompressed_meas_rate;
+};
+
+/**
+ * struct si1145_data - si1145 chip state data
+ * @client:	I2C client
+ * @lock:	mutex to protect shared state.
+ * @cmdlock:	Low-level mutex to protect command execution only
+ * @rsp_seq:	Next expected response number or -1 if counter reset required
+ * @scan_mask:	Saved scan mask to avoid duplicate set_chlist
+ * @autonomous: If automatic measurements are active (for buffer support)
+ * @part_info:	Part information
+ * @trig:	Pointer to iio trigger
+ * @meas_rate:	Value of MEAS_RATE register. Only set in HW in auto mode
+ */
+struct si1145_data {
+	struct i2c_client *client;
+	struct mutex lock;
+	struct mutex cmdlock;
+	int rsp_seq;
+	const struct si1145_part_info *part_info;
+	unsigned long scan_mask;
+	bool autonomous;
+	struct iio_trigger *trig;
+	int meas_rate;
+};
+
+/**
+ * __si1145_command_reset() - Send CMD_NOP and wait for response 0
+ *
+ * Does not modify data->rsp_seq
+ *
+ * Return: 0 on success and -errno on error.
+ */
+static int __si1145_command_reset(struct si1145_data *data)
+{
+	struct device *dev = &data->client->dev;
+	unsigned long stop_jiffies;
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(data->client, SI1145_REG_COMMAND,
+						      SI1145_CMD_NOP);
+	if (ret < 0)
+		return ret;
+	msleep(SI1145_COMMAND_MINSLEEP_MS);
+
+	stop_jiffies = jiffies + SI1145_COMMAND_TIMEOUT_MS * HZ / 1000;
+	while (true) {
+		ret = i2c_smbus_read_byte_data(data->client,
+					       SI1145_REG_RESPONSE);
+		if (ret <= 0)
+			return ret;
+		if (time_after(jiffies, stop_jiffies)) {
+			dev_warn(dev, "timeout on reset\n");
+			return -ETIMEDOUT;
+		}
+		msleep(SI1145_COMMAND_MINSLEEP_MS);
+		continue;
+	}
+}
+
+/**
+ * si1145_command() - Execute a command and poll the response register
+ *
+ * All conversion overflows are reported as -EOVERFLOW
+ * INVALID_SETTING is reported as -EINVAL
+ * Timeouts are reported as -ETIMEDOUT
+ *
+ * Return: 0 on success or -errno on failure
+ */
+static int si1145_command(struct si1145_data *data, u8 cmd)
+{
+	struct device *dev = &data->client->dev;
+	unsigned long stop_jiffies;
+	int ret;
+
+	mutex_lock(&data->cmdlock);
+
+	if (data->rsp_seq < 0) {
+		ret = __si1145_command_reset(data);
+		if (ret < 0) {
+			dev_err(dev, "failed to reset command counter, ret=%d\n",
+				ret);
+			goto out;
+		}
+		data->rsp_seq = 0;
+	}
+
+	ret = i2c_smbus_write_byte_data(data->client, SI1145_REG_COMMAND, cmd);
+	if (ret) {
+		dev_warn(dev, "failed to write command, ret=%d\n", ret);
+		goto out;
+	}
+	/* Sleep a little to ensure the command is received */
+	msleep(SI1145_COMMAND_MINSLEEP_MS);
+
+	stop_jiffies = jiffies + SI1145_COMMAND_TIMEOUT_MS * HZ / 1000;
+	while (true) {
+		ret = i2c_smbus_read_byte_data(data->client,
+					       SI1145_REG_RESPONSE);
+		if (ret < 0) {
+			dev_warn(dev, "failed to read response, ret=%d\n", ret);
+			break;
+		}
+
+		if ((ret & ~SI1145_RSP_COUNTER_MASK) == 0) {
+			if (ret == data->rsp_seq) {
+				if (time_after(jiffies, stop_jiffies)) {
+					dev_warn(dev, "timeout on command %#02hhx\n",
+						 cmd);
+					ret = -ETIMEDOUT;
+					break;
+				}
+				msleep(SI1145_COMMAND_MINSLEEP_MS);
+				continue;
+			}
+			if (ret == ((data->rsp_seq + 1) &
+				SI1145_RSP_COUNTER_MASK)) {
+				data->rsp_seq = ret;
+				ret = 0;
+				break;
+			}
+			dev_warn(dev, "unexpected response counter %d instead of %d\n",
+				 ret, (data->rsp_seq + 1) &
+					SI1145_RSP_COUNTER_MASK);
+			ret = -EIO;
+		} else {
+			if (ret == SI1145_RSP_INVALID_SETTING) {
+				dev_warn(dev, "INVALID_SETTING error on command %#02hhx\n",
+					 cmd);
+				ret = -EINVAL;
+			} else {
+				/* All overflows are treated identically */
+				dev_dbg(dev, "overflow, ret=%d, cmd=%#02hhx\n",
+					ret, cmd);
+				ret = -EOVERFLOW;
+			}
+		}
+
+		/* Force a counter reset next time */
+		data->rsp_seq = -1;
+		break;
+	}
+
+out:
+	mutex_unlock(&data->cmdlock);
+
+	return ret;
+}
+
+static int si1145_param_update(struct si1145_data *data, u8 op, u8 param,
+			       u8 value)
+{
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(data->client,
+		SI1145_REG_PARAM_WR, value);
+	if (ret < 0)
+		return ret;
+
+	return si1145_command(data, op | (param & 0x1F));
+}
+
+static int si1145_param_set(struct si1145_data *data, u8 param, u8 value)
+{
+	return si1145_param_update(data, SI1145_CMD_PARAM_SET, param, value);
+}
+
+/* Set param. Returns negative errno or current value */
+static int si1145_param_query(struct si1145_data *data, u8 param)
+{
+	int ret;
+
+	ret = si1145_command(data, SI1145_CMD_PARAM_QUERY | (param & 0x1F));
+	if (ret < 0)
+		return ret;
+
+	return i2c_smbus_read_byte_data(data->client, SI1145_REG_PARAM_RD);
+}
+
+/* Expand 8 bit compressed value to 16 bit, see Silabs AN498 */
+static u16 si1145_uncompress(u8 x)
+{
+	u16 result = 0;
+	u8 exponent = 0;
+
+	if (x < 8)
+		return 0;
+
+	exponent = (x & 0xf0) >> 4;
+	result = 0x10 | (x & 0x0f);
+
+	if (exponent >= 4)
+		return result << (exponent - 4);
+	return result >> (4 - exponent);
+}
+
+/* Compress 16 bit value to 8 bit, see Silabs AN498 */
+static u8 si1145_compress(u16 x)
+{
+	u32 exponent = 0;
+	u32 significand = 0;
+	u32 tmp = x;
+
+	if (x == 0x0000)
+		return 0x00;
+	if (x == 0x0001)
+		return 0x08;
+
+	while (1) {
+		tmp >>= 1;
+		exponent += 1;
+		if (tmp == 1)
+			break;
+	}
+
+	if (exponent < 5) {
+		significand = x << (4 - exponent);
+		return (exponent << 4) | (significand & 0xF);
+	}
+
+	significand = x >> (exponent - 5);
+	if (significand & 1) {
+		significand += 2;
+		if (significand & 0x0040) {
+			exponent += 1;
+			significand >>= 1;
+		}
+	}
+
+	return (exponent << 4) | ((significand >> 1) & 0xF);
+}
+
+/* Write meas_rate in hardware */
+static int si1145_set_meas_rate(struct si1145_data *data, int interval)
+{
+	if (data->part_info->uncompressed_meas_rate)
+		return i2c_smbus_write_word_data(data->client,
+			SI1145_REG_MEAS_RATE, interval);
+	else
+		return i2c_smbus_write_byte_data(data->client,
+			SI1145_REG_MEAS_RATE, interval);
+}
+
+static int si1145_read_samp_freq(struct si1145_data *data, int *val, int *val2)
+{
+	*val = 32000;
+	if (data->part_info->uncompressed_meas_rate)
+		*val2 = data->meas_rate;
+	else
+		*val2 = si1145_uncompress(data->meas_rate);
+	return IIO_VAL_FRACTIONAL;
+}
+
+/* Set the samp freq in driver private data */
+static int si1145_store_samp_freq(struct si1145_data *data, int val)
+{
+	int ret = 0;
+	int meas_rate;
+
+	if (val <= 0 || val > 32000)
+		return -ERANGE;
+	meas_rate = 32000 / val;
+
+	mutex_lock(&data->lock);
+	if (data->autonomous) {
+		ret = si1145_set_meas_rate(data, meas_rate);
+		if (ret)
+			goto out;
+	}
+	if (data->part_info->uncompressed_meas_rate)
+		data->meas_rate = meas_rate;
+	else
+		data->meas_rate = si1145_compress(meas_rate);
+
+out:
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static irqreturn_t si1145_trigger_handler(int irq, void *private)
+{
+	struct iio_poll_func *pf = private;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct si1145_data *data = iio_priv(indio_dev);
+	/*
+	 * Maximum buffer size:
+	 *   6*2 bytes channels data + 4 bytes alignment +
+	 *   8 bytes timestamp
+	 */
+	u8 buffer[24];
+	int i, j = 0;
+	int ret;
+	u8 irq_status = 0;
+
+	if (!data->autonomous) {
+		ret = si1145_command(data, SI1145_CMD_PSALS_FORCE);
+		if (ret < 0 && ret != -EOVERFLOW)
+			goto done;
+	} else {
+		irq_status = ret = i2c_smbus_read_byte_data(data->client,
+				SI1145_REG_IRQ_STATUS);
+		if (ret < 0)
+			goto done;
+		if (!(irq_status & SI1145_MASK_ALL_IE))
+			goto done;
+	}
+
+	for_each_set_bit(i, indio_dev->active_scan_mask,
+		indio_dev->masklength) {
+		int run = 1;
+
+		while (i + run < indio_dev->masklength) {
+			if (!test_bit(i + run, indio_dev->active_scan_mask))
+				break;
+			if (indio_dev->channels[i + run].address !=
+				indio_dev->channels[i].address + 2 * run)
+				break;
+			run++;
+		}
+
+		ret = i2c_smbus_read_i2c_block_data_or_emulated(
+				data->client, indio_dev->channels[i].address,
+				sizeof(u16) * run, &buffer[j]);
+		if (ret < 0)
+			goto done;
+		j += run * sizeof(u16);
+		i += run - 1;
+	}
+
+	if (data->autonomous) {
+		ret = i2c_smbus_write_byte_data(data->client,
+				SI1145_REG_IRQ_STATUS,
+				irq_status & SI1145_MASK_ALL_IE);
+		if (ret < 0)
+			goto done;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, buffer,
+		iio_get_time_ns(indio_dev));
+
+done:
+	iio_trigger_notify_done(indio_dev->trig);
+	return IRQ_HANDLED;
+}
+
+static int si1145_set_chlist(struct iio_dev *indio_dev, unsigned long scan_mask)
+{
+	struct si1145_data *data = iio_priv(indio_dev);
+	u8 reg = 0, mux;
+	int ret;
+	int i;
+
+	/* channel list already set, no need to reprogram */
+	if (data->scan_mask == scan_mask)
+		return 0;
+
+	for_each_set_bit(i, &scan_mask, indio_dev->masklength) {
+		switch (indio_dev->channels[i].address) {
+		case SI1145_REG_ALSVIS_DATA:
+			reg |= SI1145_CHLIST_EN_ALSVIS;
+			break;
+		case SI1145_REG_ALSIR_DATA:
+			reg |= SI1145_CHLIST_EN_ALSIR;
+			break;
+		case SI1145_REG_PS1_DATA:
+			reg |= SI1145_CHLIST_EN_PS1;
+			break;
+		case SI1145_REG_PS2_DATA:
+			reg |= SI1145_CHLIST_EN_PS2;
+			break;
+		case SI1145_REG_PS3_DATA:
+			reg |= SI1145_CHLIST_EN_PS3;
+			break;
+		case SI1145_REG_AUX_DATA:
+			switch (indio_dev->channels[i].type) {
+			case IIO_UVINDEX:
+				reg |= SI1145_CHLIST_EN_UV;
+				break;
+			default:
+				reg |= SI1145_CHLIST_EN_AUX;
+				if (indio_dev->channels[i].type == IIO_TEMP)
+					mux = SI1145_MUX_TEMP;
+				else
+					mux = SI1145_MUX_VDD;
+				ret = si1145_param_set(data,
+					SI1145_PARAM_AUX_ADC_MUX, mux);
+				if (ret < 0)
+					return ret;
+
+				break;
+			}
+		}
+	}
+
+	data->scan_mask = scan_mask;
+	ret = si1145_param_set(data, SI1145_PARAM_CHLIST, reg);
+
+	return ret < 0 ? ret : 0;
+}
+
+static int si1145_measure(struct iio_dev *indio_dev,
+			  struct iio_chan_spec const *chan)
+{
+	struct si1145_data *data = iio_priv(indio_dev);
+	u8 cmd;
+	int ret;
+
+	ret = si1145_set_chlist(indio_dev, BIT(chan->scan_index));
+	if (ret < 0)
+		return ret;
+
+	cmd = (chan->type == IIO_PROXIMITY) ? SI1145_CMD_PS_FORCE :
+		SI1145_CMD_ALS_FORCE;
+	ret = si1145_command(data, cmd);
+	if (ret < 0 && ret != -EOVERFLOW)
+		return ret;
+
+	return i2c_smbus_read_word_data(data->client, chan->address);
+}
+
+/*
+ * Conversion between iio scale and ADC_GAIN values
+ * These could be further adjusted but proximity/intensity are dimensionless
+ */
+static const int si1145_proximity_scale_available[] = {
+	128, 64, 32, 16, 8, 4};
+static const int si1145_intensity_scale_available[] = {
+	128, 64, 32, 16, 8, 4, 2, 1};
+static IIO_CONST_ATTR(in_proximity_scale_available,
+	"128 64 32 16 8 4");
+static IIO_CONST_ATTR(in_intensity_scale_available,
+	"128 64 32 16 8 4 2 1");
+static IIO_CONST_ATTR(in_intensity_ir_scale_available,
+	"128 64 32 16 8 4 2 1");
+
+static int si1145_scale_from_adcgain(int regval)
+{
+	return 128 >> regval;
+}
+
+static int si1145_proximity_adcgain_from_scale(int val, int val2)
+{
+	val = find_closest_descending(val, si1145_proximity_scale_available,
+				ARRAY_SIZE(si1145_proximity_scale_available));
+	if (val < 0 || val > 5 || val2 != 0)
+		return -EINVAL;
+
+	return val;
+}
+
+static int si1145_intensity_adcgain_from_scale(int val, int val2)
+{
+	val = find_closest_descending(val, si1145_intensity_scale_available,
+				ARRAY_SIZE(si1145_intensity_scale_available));
+	if (val < 0 || val > 7 || val2 != 0)
+		return -EINVAL;
+
+	return val;
+}
+
+static int si1145_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan,
+				int *val, int *val2, long mask)
+{
+	struct si1145_data *data = iio_priv(indio_dev);
+	int ret;
+	u8 reg;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		switch (chan->type) {
+		case IIO_INTENSITY:
+		case IIO_PROXIMITY:
+		case IIO_VOLTAGE:
+		case IIO_TEMP:
+		case IIO_UVINDEX:
+			ret = iio_device_claim_direct_mode(indio_dev);
+			if (ret)
+				return ret;
+			ret = si1145_measure(indio_dev, chan);
+			iio_device_release_direct_mode(indio_dev);
+
+			if (ret < 0)
+				return ret;
+
+			*val = ret;
+
+			return IIO_VAL_INT;
+		case IIO_CURRENT:
+			ret = i2c_smbus_read_byte_data(data->client,
+				SI1145_PS_LED_REG(chan->channel));
+			if (ret < 0)
+				return ret;
+
+			*val = (ret >> SI1145_PS_LED_SHIFT(chan->channel))
+				& 0x0f;
+
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_PROXIMITY:
+			reg = SI1145_PARAM_PS_ADC_GAIN;
+			break;
+		case IIO_INTENSITY:
+			if (chan->channel2 == IIO_MOD_LIGHT_IR)
+				reg = SI1145_PARAM_ALSIR_ADC_GAIN;
+			else
+				reg = SI1145_PARAM_ALSVIS_ADC_GAIN;
+			break;
+		case IIO_TEMP:
+			*val = 28;
+			*val2 = 571429;
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_UVINDEX:
+			*val = 0;
+			*val2 = 10000;
+			return IIO_VAL_INT_PLUS_MICRO;
+		default:
+			return -EINVAL;
+		}
+
+		ret = si1145_param_query(data, reg);
+		if (ret < 0)
+			return ret;
+
+		*val = si1145_scale_from_adcgain(ret & 0x07);
+
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_OFFSET:
+		switch (chan->type) {
+		case IIO_TEMP:
+			/*
+			 * -ADC offset - ADC counts @ 25°C -
+			 *   35 * ADC counts / °C
+			 */
+			*val = -256 - 11136 + 25 * 35;
+			return IIO_VAL_INT;
+		default:
+			/*
+			 * All ADC measurements have are by default offset
+			 * by -256
+			 * See AN498 5.6.3
+			 */
+			ret = si1145_param_query(data, SI1145_PARAM_ADC_OFFSET);
+			if (ret < 0)
+				return ret;
+			*val = -si1145_uncompress(ret);
+			return IIO_VAL_INT;
+		}
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return si1145_read_samp_freq(data, val, val2);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int si1145_write_raw(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan,
+			       int val, int val2, long mask)
+{
+	struct si1145_data *data = iio_priv(indio_dev);
+	u8 reg1, reg2, shift;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_PROXIMITY:
+			val = si1145_proximity_adcgain_from_scale(val, val2);
+			if (val < 0)
+				return val;
+			reg1 = SI1145_PARAM_PS_ADC_GAIN;
+			reg2 = SI1145_PARAM_PS_ADC_COUNTER;
+			break;
+		case IIO_INTENSITY:
+			val = si1145_intensity_adcgain_from_scale(val, val2);
+			if (val < 0)
+				return val;
+			if (chan->channel2 == IIO_MOD_LIGHT_IR) {
+				reg1 = SI1145_PARAM_ALSIR_ADC_GAIN;
+				reg2 = SI1145_PARAM_ALSIR_ADC_COUNTER;
+			} else {
+				reg1 = SI1145_PARAM_ALSVIS_ADC_GAIN;
+				reg2 = SI1145_PARAM_ALSVIS_ADC_COUNTER;
+			}
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		ret = si1145_param_set(data, reg1, val);
+		if (ret < 0) {
+			iio_device_release_direct_mode(indio_dev);
+			return ret;
+		}
+		/* Set recovery period to one's complement of gain */
+		ret = si1145_param_set(data, reg2, (~val & 0x07) << 4);
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+	case IIO_CHAN_INFO_RAW:
+		if (chan->type != IIO_CURRENT)
+			return -EINVAL;
+
+		if (val < 0 || val > 15 || val2 != 0)
+			return -EINVAL;
+
+		reg1 = SI1145_PS_LED_REG(chan->channel);
+		shift = SI1145_PS_LED_SHIFT(chan->channel);
+
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		ret = i2c_smbus_read_byte_data(data->client, reg1);
+		if (ret < 0) {
+			iio_device_release_direct_mode(indio_dev);
+			return ret;
+		}
+		ret = i2c_smbus_write_byte_data(data->client, reg1,
+			(ret & ~(0x0f << shift)) |
+			((val & 0x0f) << shift));
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return si1145_store_samp_freq(data, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+#define SI1145_ST { \
+	.sign = 'u', \
+	.realbits = 16, \
+	.storagebits = 16, \
+	.endianness = IIO_LE, \
+}
+
+#define SI1145_INTENSITY_CHANNEL(_si) { \
+	.type = IIO_INTENSITY, \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
+			      BIT(IIO_CHAN_INFO_OFFSET) | \
+			      BIT(IIO_CHAN_INFO_SCALE), \
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
+	.scan_type = SI1145_ST, \
+	.scan_index = _si, \
+	.address = SI1145_REG_ALSVIS_DATA, \
+}
+
+#define SI1145_INTENSITY_IR_CHANNEL(_si) { \
+	.type = IIO_INTENSITY, \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
+			      BIT(IIO_CHAN_INFO_OFFSET) | \
+			      BIT(IIO_CHAN_INFO_SCALE), \
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
+	.modified = 1, \
+	.channel2 = IIO_MOD_LIGHT_IR, \
+	.scan_type = SI1145_ST, \
+	.scan_index = _si, \
+	.address = SI1145_REG_ALSIR_DATA, \
+}
+
+#define SI1145_TEMP_CHANNEL(_si) { \
+	.type = IIO_TEMP, \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
+			      BIT(IIO_CHAN_INFO_OFFSET) | \
+			      BIT(IIO_CHAN_INFO_SCALE), \
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
+	.scan_type = SI1145_ST, \
+	.scan_index = _si, \
+	.address = SI1145_REG_AUX_DATA, \
+}
+
+#define SI1145_UV_CHANNEL(_si) { \
+	.type = IIO_UVINDEX, \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
+			      BIT(IIO_CHAN_INFO_SCALE), \
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
+	.scan_type = SI1145_ST, \
+	.scan_index = _si, \
+	.address = SI1145_REG_AUX_DATA, \
+}
+
+#define SI1145_PROXIMITY_CHANNEL(_si, _ch) { \
+	.type = IIO_PROXIMITY, \
+	.indexed = 1, \
+	.channel = _ch, \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) | \
+				    BIT(IIO_CHAN_INFO_OFFSET), \
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
+	.scan_type = SI1145_ST, \
+	.scan_index = _si, \
+	.address = SI1145_REG_PS1_DATA + _ch * 2, \
+}
+
+#define SI1145_VOLTAGE_CHANNEL(_si) { \
+	.type = IIO_VOLTAGE, \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
+	.scan_type = SI1145_ST, \
+	.scan_index = _si, \
+	.address = SI1145_REG_AUX_DATA, \
+}
+
+#define SI1145_CURRENT_CHANNEL(_ch) { \
+	.type = IIO_CURRENT, \
+	.indexed = 1, \
+	.channel = _ch, \
+	.output = 1, \
+	.scan_index = -1, \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
+}
+
+static const struct iio_chan_spec si1132_channels[] = {
+	SI1145_INTENSITY_CHANNEL(0),
+	SI1145_INTENSITY_IR_CHANNEL(1),
+	SI1145_TEMP_CHANNEL(2),
+	SI1145_VOLTAGE_CHANNEL(3),
+	SI1145_UV_CHANNEL(4),
+	IIO_CHAN_SOFT_TIMESTAMP(6),
+};
+
+static const struct iio_chan_spec si1141_channels[] = {
+	SI1145_INTENSITY_CHANNEL(0),
+	SI1145_INTENSITY_IR_CHANNEL(1),
+	SI1145_PROXIMITY_CHANNEL(2, 0),
+	SI1145_TEMP_CHANNEL(3),
+	SI1145_VOLTAGE_CHANNEL(4),
+	IIO_CHAN_SOFT_TIMESTAMP(5),
+	SI1145_CURRENT_CHANNEL(0),
+};
+
+static const struct iio_chan_spec si1142_channels[] = {
+	SI1145_INTENSITY_CHANNEL(0),
+	SI1145_INTENSITY_IR_CHANNEL(1),
+	SI1145_PROXIMITY_CHANNEL(2, 0),
+	SI1145_PROXIMITY_CHANNEL(3, 1),
+	SI1145_TEMP_CHANNEL(4),
+	SI1145_VOLTAGE_CHANNEL(5),
+	IIO_CHAN_SOFT_TIMESTAMP(6),
+	SI1145_CURRENT_CHANNEL(0),
+	SI1145_CURRENT_CHANNEL(1),
+};
+
+static const struct iio_chan_spec si1143_channels[] = {
+	SI1145_INTENSITY_CHANNEL(0),
+	SI1145_INTENSITY_IR_CHANNEL(1),
+	SI1145_PROXIMITY_CHANNEL(2, 0),
+	SI1145_PROXIMITY_CHANNEL(3, 1),
+	SI1145_PROXIMITY_CHANNEL(4, 2),
+	SI1145_TEMP_CHANNEL(5),
+	SI1145_VOLTAGE_CHANNEL(6),
+	IIO_CHAN_SOFT_TIMESTAMP(7),
+	SI1145_CURRENT_CHANNEL(0),
+	SI1145_CURRENT_CHANNEL(1),
+	SI1145_CURRENT_CHANNEL(2),
+};
+
+static const struct iio_chan_spec si1145_channels[] = {
+	SI1145_INTENSITY_CHANNEL(0),
+	SI1145_INTENSITY_IR_CHANNEL(1),
+	SI1145_PROXIMITY_CHANNEL(2, 0),
+	SI1145_TEMP_CHANNEL(3),
+	SI1145_VOLTAGE_CHANNEL(4),
+	SI1145_UV_CHANNEL(5),
+	IIO_CHAN_SOFT_TIMESTAMP(6),
+	SI1145_CURRENT_CHANNEL(0),
+};
+
+static const struct iio_chan_spec si1146_channels[] = {
+	SI1145_INTENSITY_CHANNEL(0),
+	SI1145_INTENSITY_IR_CHANNEL(1),
+	SI1145_TEMP_CHANNEL(2),
+	SI1145_VOLTAGE_CHANNEL(3),
+	SI1145_UV_CHANNEL(4),
+	SI1145_PROXIMITY_CHANNEL(5, 0),
+	SI1145_PROXIMITY_CHANNEL(6, 1),
+	IIO_CHAN_SOFT_TIMESTAMP(7),
+	SI1145_CURRENT_CHANNEL(0),
+	SI1145_CURRENT_CHANNEL(1),
+};
+
+static const struct iio_chan_spec si1147_channels[] = {
+	SI1145_INTENSITY_CHANNEL(0),
+	SI1145_INTENSITY_IR_CHANNEL(1),
+	SI1145_PROXIMITY_CHANNEL(2, 0),
+	SI1145_PROXIMITY_CHANNEL(3, 1),
+	SI1145_PROXIMITY_CHANNEL(4, 2),
+	SI1145_TEMP_CHANNEL(5),
+	SI1145_VOLTAGE_CHANNEL(6),
+	SI1145_UV_CHANNEL(7),
+	IIO_CHAN_SOFT_TIMESTAMP(8),
+	SI1145_CURRENT_CHANNEL(0),
+	SI1145_CURRENT_CHANNEL(1),
+	SI1145_CURRENT_CHANNEL(2),
+};
+
+static struct attribute *si1132_attributes[] = {
+	&iio_const_attr_in_intensity_scale_available.dev_attr.attr,
+	&iio_const_attr_in_intensity_ir_scale_available.dev_attr.attr,
+	NULL,
+};
+
+static struct attribute *si114x_attributes[] = {
+	&iio_const_attr_in_intensity_scale_available.dev_attr.attr,
+	&iio_const_attr_in_intensity_ir_scale_available.dev_attr.attr,
+	&iio_const_attr_in_proximity_scale_available.dev_attr.attr,
+	NULL,
+};
+
+static const struct attribute_group si1132_attribute_group = {
+	.attrs = si1132_attributes,
+};
+
+static const struct attribute_group si114x_attribute_group = {
+	.attrs = si114x_attributes,
+};
+
+
+static const struct iio_info si1132_info = {
+	.read_raw = si1145_read_raw,
+	.write_raw = si1145_write_raw,
+	.driver_module = THIS_MODULE,
+	.attrs = &si1132_attribute_group,
+};
+
+static const struct iio_info si114x_info = {
+	.read_raw = si1145_read_raw,
+	.write_raw = si1145_write_raw,
+	.driver_module = THIS_MODULE,
+	.attrs = &si114x_attribute_group,
+};
+
+#define SI1145_PART(id, iio_info, chans, leds, uncompressed_meas_rate) \
+	{id, iio_info, chans, ARRAY_SIZE(chans), leds, uncompressed_meas_rate}
+
+static const struct si1145_part_info si1145_part_info[] = {
+	[SI1132] = SI1145_PART(0x32, &si1132_info, si1132_channels, 0, true),
+	[SI1141] = SI1145_PART(0x41, &si114x_info, si1141_channels, 1, false),
+	[SI1142] = SI1145_PART(0x42, &si114x_info, si1142_channels, 2, false),
+	[SI1143] = SI1145_PART(0x43, &si114x_info, si1143_channels, 3, false),
+	[SI1145] = SI1145_PART(0x45, &si114x_info, si1145_channels, 1, true),
+	[SI1146] = SI1145_PART(0x46, &si114x_info, si1146_channels, 2, true),
+	[SI1147] = SI1145_PART(0x47, &si114x_info, si1147_channels, 3, true),
+};
+
+static int si1145_initialize(struct si1145_data *data)
+{
+	struct i2c_client *client = data->client;
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(client, SI1145_REG_COMMAND,
+					SI1145_CMD_RESET);
+	if (ret < 0)
+		return ret;
+	msleep(SI1145_COMMAND_TIMEOUT_MS);
+
+	/* Hardware key, magic value */
+	ret = i2c_smbus_write_byte_data(client, SI1145_REG_HW_KEY, 0x17);
+	if (ret < 0)
+		return ret;
+	msleep(SI1145_COMMAND_TIMEOUT_MS);
+
+	/* Turn off autonomous mode */
+	ret = si1145_set_meas_rate(data, 0);
+	if (ret < 0)
+		return ret;
+
+	/* Initialize sampling freq to 10 Hz */
+	ret = si1145_store_samp_freq(data, 10);
+	if (ret < 0)
+		return ret;
+
+	/* Set LED currents to 45 mA; have 4 bits, see Table 2 in datasheet */
+	switch (data->part_info->num_leds) {
+	case 3:
+		ret = i2c_smbus_write_byte_data(client,
+						SI1145_REG_PS_LED3,
+						SI1145_LED_CURRENT_45mA);
+		if (ret < 0)
+			return ret;
+		/* fallthrough */
+	case 2:
+		ret = i2c_smbus_write_byte_data(client,
+						SI1145_REG_PS_LED21,
+						(SI1145_LED_CURRENT_45mA << 4) |
+						SI1145_LED_CURRENT_45mA);
+		break;
+	case 1:
+		ret = i2c_smbus_write_byte_data(client,
+						SI1145_REG_PS_LED21,
+						SI1145_LED_CURRENT_45mA);
+		break;
+	default:
+		ret = 0;
+		break;
+	}
+	if (ret < 0)
+		return ret;
+
+	/* Set normal proximity measurement mode */
+	ret = si1145_param_set(data, SI1145_PARAM_PS_ADC_MISC,
+			       SI1145_PS_ADC_MODE_NORMAL);
+	if (ret < 0)
+		return ret;
+
+	ret = si1145_param_set(data, SI1145_PARAM_PS_ADC_GAIN, 0x01);
+	if (ret < 0)
+		return ret;
+
+	/* ADC_COUNTER should be one complement of ADC_GAIN */
+	ret = si1145_param_set(data, SI1145_PARAM_PS_ADC_COUNTER, 0x06 << 4);
+	if (ret < 0)
+		return ret;
+
+	/* Set ALS visible measurement mode */
+	ret = si1145_param_set(data, SI1145_PARAM_ALSVIS_ADC_MISC,
+			       SI1145_ADC_MISC_RANGE);
+	if (ret < 0)
+		return ret;
+
+	ret = si1145_param_set(data, SI1145_PARAM_ALSVIS_ADC_GAIN, 0x03);
+	if (ret < 0)
+		return ret;
+
+	ret = si1145_param_set(data, SI1145_PARAM_ALSVIS_ADC_COUNTER,
+			       0x04 << 4);
+	if (ret < 0)
+		return ret;
+
+	/* Set ALS IR measurement mode */
+	ret = si1145_param_set(data, SI1145_PARAM_ALSIR_ADC_MISC,
+			       SI1145_ADC_MISC_RANGE);
+	if (ret < 0)
+		return ret;
+
+	ret = si1145_param_set(data, SI1145_PARAM_ALSIR_ADC_GAIN, 0x01);
+	if (ret < 0)
+		return ret;
+
+	ret = si1145_param_set(data, SI1145_PARAM_ALSIR_ADC_COUNTER,
+			       0x06 << 4);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Initialize UCOEF to default values in datasheet
+	 * These registers are normally zero on reset
+	 */
+	if (data->part_info == &si1145_part_info[SI1132] ||
+		data->part_info == &si1145_part_info[SI1145] ||
+		data->part_info == &si1145_part_info[SI1146] ||
+		data->part_info == &si1145_part_info[SI1147]) {
+		ret = i2c_smbus_write_byte_data(data->client,
+						SI1145_REG_UCOEF1,
+						SI1145_UCOEF1_DEFAULT);
+		if (ret < 0)
+			return ret;
+		ret = i2c_smbus_write_byte_data(data->client,
+				SI1145_REG_UCOEF2, SI1145_UCOEF2_DEFAULT);
+		if (ret < 0)
+			return ret;
+		ret = i2c_smbus_write_byte_data(data->client,
+				SI1145_REG_UCOEF3, SI1145_UCOEF3_DEFAULT);
+		if (ret < 0)
+			return ret;
+		ret = i2c_smbus_write_byte_data(data->client,
+				SI1145_REG_UCOEF4, SI1145_UCOEF4_DEFAULT);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * Program the channels we want to measure with CMD_PSALS_AUTO. No need for
+ * _postdisable as we stop with CMD_PSALS_PAUSE; single measurement (direct)
+ * mode reprograms the channels list anyway...
+ */
+static int si1145_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct si1145_data *data = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&data->lock);
+	ret = si1145_set_chlist(indio_dev, *indio_dev->active_scan_mask);
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static bool si1145_validate_scan_mask(struct iio_dev *indio_dev,
+			       const unsigned long *scan_mask)
+{
+	struct si1145_data *data = iio_priv(indio_dev);
+	unsigned int count = 0;
+	int i;
+
+	/* Check that at most one AUX channel is enabled */
+	for_each_set_bit(i, scan_mask, data->part_info->num_channels) {
+		if (indio_dev->channels[i].address == SI1145_REG_AUX_DATA)
+			count++;
+	}
+
+	return count <= 1;
+}
+
+static const struct iio_buffer_setup_ops si1145_buffer_setup_ops = {
+	.preenable = si1145_buffer_preenable,
+	.postenable = iio_triggered_buffer_postenable,
+	.predisable = iio_triggered_buffer_predisable,
+	.validate_scan_mask = si1145_validate_scan_mask,
+};
+
+/**
+ * si1145_trigger_set_state() - Set trigger state
+ *
+ * When not using triggers interrupts are disabled and measurement rate is
+ * set to zero in order to minimize power consumption.
+ */
+static int si1145_trigger_set_state(struct iio_trigger *trig, bool state)
+{
+	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
+	struct si1145_data *data = iio_priv(indio_dev);
+	int err = 0, ret;
+
+	mutex_lock(&data->lock);
+
+	if (state) {
+		data->autonomous = true;
+		err = i2c_smbus_write_byte_data(data->client,
+				SI1145_REG_INT_CFG, SI1145_INT_CFG_OE);
+		if (err < 0)
+			goto disable;
+		err = i2c_smbus_write_byte_data(data->client,
+				SI1145_REG_IRQ_ENABLE, SI1145_MASK_ALL_IE);
+		if (err < 0)
+			goto disable;
+		err = si1145_set_meas_rate(data, data->meas_rate);
+		if (err < 0)
+			goto disable;
+		err = si1145_command(data, SI1145_CMD_PSALS_AUTO);
+		if (err < 0)
+			goto disable;
+	} else {
+disable:
+		/* Disable as much as possible skipping errors */
+		ret = si1145_command(data, SI1145_CMD_PSALS_PAUSE);
+		if (ret < 0 && !err)
+			err = ret;
+		ret = si1145_set_meas_rate(data, 0);
+		if (ret < 0 && !err)
+			err = ret;
+		ret = i2c_smbus_write_byte_data(data->client,
+						SI1145_REG_IRQ_ENABLE, 0);
+		if (ret < 0 && !err)
+			err = ret;
+		ret = i2c_smbus_write_byte_data(data->client,
+						SI1145_REG_INT_CFG, 0);
+		if (ret < 0 && !err)
+			err = ret;
+		data->autonomous = false;
+	}
+
+	mutex_unlock(&data->lock);
+	return err;
+}
+
+static const struct iio_trigger_ops si1145_trigger_ops = {
+	.owner = THIS_MODULE,
+	.set_trigger_state = si1145_trigger_set_state,
+};
+
+static int si1145_probe_trigger(struct iio_dev *indio_dev)
+{
+	struct si1145_data *data = iio_priv(indio_dev);
+	struct i2c_client *client = data->client;
+	struct iio_trigger *trig;
+	int ret;
+
+	trig = devm_iio_trigger_alloc(&client->dev,
+			"%s-dev%d", indio_dev->name, indio_dev->id);
+	if (!trig)
+		return -ENOMEM;
+
+	trig->dev.parent = &client->dev;
+	trig->ops = &si1145_trigger_ops;
+	iio_trigger_set_drvdata(trig, indio_dev);
+
+	ret = devm_request_irq(&client->dev, client->irq,
+			  iio_trigger_generic_data_rdy_poll,
+			  IRQF_TRIGGER_FALLING,
+			  "si1145_irq",
+			  trig);
+	if (ret < 0) {
+		dev_err(&client->dev, "irq request failed\n");
+		return ret;
+	}
+
+	ret = iio_trigger_register(trig);
+	if (ret)
+		return ret;
+
+	data->trig = trig;
+	indio_dev->trig = iio_trigger_get(data->trig);
+
+	return 0;
+}
+
+static void si1145_remove_trigger(struct iio_dev *indio_dev)
+{
+	struct si1145_data *data = iio_priv(indio_dev);
+
+	if (data->trig) {
+		iio_trigger_unregister(data->trig);
+		data->trig = NULL;
+	}
+}
+
+static int si1145_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	struct si1145_data *data;
+	struct iio_dev *indio_dev;
+	u8 part_id, rev_id, seq_id;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	i2c_set_clientdata(client, indio_dev);
+	data->client = client;
+	data->part_info = &si1145_part_info[id->driver_data];
+
+	part_id = ret = i2c_smbus_read_byte_data(data->client,
+						 SI1145_REG_PART_ID);
+	if (ret < 0)
+		return ret;
+	rev_id = ret = i2c_smbus_read_byte_data(data->client,
+						SI1145_REG_REV_ID);
+	if (ret < 0)
+		return ret;
+	seq_id = ret = i2c_smbus_read_byte_data(data->client,
+						SI1145_REG_SEQ_ID);
+	if (ret < 0)
+		return ret;
+	dev_info(&client->dev, "device ID part %#02hhx rev %#02hhx seq %#02hhx\n",
+			part_id, rev_id, seq_id);
+	if (part_id != data->part_info->part) {
+		dev_err(&client->dev, "part ID mismatch got %#02hhx, expected %#02x\n",
+				part_id, data->part_info->part);
+		return -ENODEV;
+	}
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->name = id->name;
+	indio_dev->channels = data->part_info->channels;
+	indio_dev->num_channels = data->part_info->num_channels;
+	indio_dev->info = data->part_info->iio_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	mutex_init(&data->lock);
+	mutex_init(&data->cmdlock);
+
+	ret = si1145_initialize(data);
+	if (ret < 0)
+		return ret;
+
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+		si1145_trigger_handler, &si1145_buffer_setup_ops);
+	if (ret < 0)
+		return ret;
+
+	if (client->irq) {
+		ret = si1145_probe_trigger(indio_dev);
+		if (ret < 0)
+			goto error_free_buffer;
+	} else {
+		dev_info(&client->dev, "no irq, using polling\n");
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret < 0)
+		goto error_free_trigger;
+
+	return 0;
+
+error_free_trigger:
+	si1145_remove_trigger(indio_dev);
+error_free_buffer:
+	iio_triggered_buffer_cleanup(indio_dev);
+
+	return ret;
+}
+
+static const struct i2c_device_id si1145_ids[] = {
+	{ "si1132", SI1132 },
+	{ "si1141", SI1141 },
+	{ "si1142", SI1142 },
+	{ "si1143", SI1143 },
+	{ "si1145", SI1145 },
+	{ "si1146", SI1146 },
+	{ "si1147", SI1147 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, si1145_ids);
+
+static int si1145_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+	iio_device_unregister(indio_dev);
+	si1145_remove_trigger(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
+
+	return 0;
+}
+
+static struct i2c_driver si1145_driver = {
+	.driver = {
+		.name   = "si1145",
+	},
+	.probe  = si1145_probe,
+	.remove = si1145_remove,
+	.id_table = si1145_ids,
+};
+
+module_i2c_driver(si1145_driver);
+
+MODULE_AUTHOR("Peter Meerwald-Stadler <pmeerw@pmeerw.net>");
+MODULE_DESCRIPTION("Silabs SI1132 and SI1141/2/3/5/6/7 proximity, ambient light and UV index sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/light/us5182d.c b/drivers/iio/light/us5182d.c
index 20c40f780964..18cf2e29e4d5 100644
--- a/drivers/iio/light/us5182d.c
+++ b/drivers/iio/light/us5182d.c
@@ -894,7 +894,7 @@ static int us5182d_probe(struct i2c_client *client,
 		goto out_err;
 
 	if (data->default_continuous) {
-		pm_runtime_set_active(&client->dev);
+		ret = pm_runtime_set_active(&client->dev);
 		if (ret < 0)
 			goto out_err;
 	}
diff --git a/drivers/iio/light/vcnl4000.c b/drivers/iio/light/vcnl4000.c
index c9d85bbc9230..360b6e98137a 100644
--- a/drivers/iio/light/vcnl4000.c
+++ b/drivers/iio/light/vcnl4000.c
@@ -1,6 +1,6 @@
 /*
- * vcnl4000.c - Support for Vishay VCNL4000 combined ambient light and
- * proximity sensor
+ * vcnl4000.c - Support for Vishay VCNL4000/4010/4020 combined ambient
+ * light and proximity sensor
  *
  * Copyright 2012 Peter Meerwald <pmeerw@pmeerw.net>
  *
@@ -13,6 +13,8 @@
  * TODO:
  *   allow to adjust IR current
  *   proximity threshold and event handling
+ *   periodic ALS/proximity measurement (VCNL4010/20)
+ *   interrupts (VCNL4010/20)
  */
 
 #include <linux/module.h>
@@ -24,6 +26,8 @@
 #include <linux/iio/sysfs.h>
 
 #define VCNL4000_DRV_NAME "vcnl4000"
+#define VCNL4000_ID		0x01
+#define VCNL4010_ID		0x02 /* for VCNL4020, VCNL4010 */
 
 #define VCNL4000_COMMAND	0x80 /* Command register */
 #define VCNL4000_PROD_REV	0x81 /* Product ID and Revision ID */
@@ -37,13 +41,14 @@
 #define VCNL4000_PS_MOD_ADJ	0x8a /* Proximity modulator timing adjustment */
 
 /* Bit masks for COMMAND register */
-#define VCNL4000_AL_RDY		0x40 /* ALS data ready? */
-#define VCNL4000_PS_RDY		0x20 /* proximity data ready? */
-#define VCNL4000_AL_OD		0x10 /* start on-demand ALS measurement */
-#define VCNL4000_PS_OD		0x08 /* start on-demand proximity measurement */
+#define VCNL4000_AL_RDY		BIT(6) /* ALS data ready? */
+#define VCNL4000_PS_RDY		BIT(5) /* proximity data ready? */
+#define VCNL4000_AL_OD		BIT(4) /* start on-demand ALS measurement */
+#define VCNL4000_PS_OD		BIT(3) /* start on-demand proximity measurement */
 
 struct vcnl4000_data {
 	struct i2c_client *client;
+	struct mutex lock;
 };
 
 static const struct i2c_device_id vcnl4000_id[] = {
@@ -59,16 +64,18 @@ static int vcnl4000_measure(struct vcnl4000_data *data, u8 req_mask,
 	__be16 buf;
 	int ret;
 
+	mutex_lock(&data->lock);
+
 	ret = i2c_smbus_write_byte_data(data->client, VCNL4000_COMMAND,
 					req_mask);
 	if (ret < 0)
-		return ret;
+		goto fail;
 
 	/* wait for data to become ready */
 	while (tries--) {
 		ret = i2c_smbus_read_byte_data(data->client, VCNL4000_COMMAND);
 		if (ret < 0)
-			return ret;
+			goto fail;
 		if (ret & rdy_mask)
 			break;
 		msleep(20); /* measurement takes up to 100 ms */
@@ -77,17 +84,23 @@ static int vcnl4000_measure(struct vcnl4000_data *data, u8 req_mask,
 	if (tries < 0) {
 		dev_err(&data->client->dev,
 			"vcnl4000_measure() failed, data not ready\n");
-		return -EIO;
+		ret = -EIO;
+		goto fail;
 	}
 
 	ret = i2c_smbus_read_i2c_block_data(data->client,
 		data_reg, sizeof(buf), (u8 *) &buf);
 	if (ret < 0)
-		return ret;
+		goto fail;
 
+	mutex_unlock(&data->lock);
 	*val = be16_to_cpu(buf);
 
 	return 0;
+
+fail:
+	mutex_unlock(&data->lock);
+	return ret;
 }
 
 static const struct iio_chan_spec vcnl4000_channels[] = {
@@ -105,7 +118,7 @@ static int vcnl4000_read_raw(struct iio_dev *indio_dev,
 				struct iio_chan_spec const *chan,
 				int *val, int *val2, long mask)
 {
-	int ret = -EINVAL;
+	int ret;
 	struct vcnl4000_data *data = iio_priv(indio_dev);
 
 	switch (mask) {
@@ -117,32 +130,27 @@ static int vcnl4000_read_raw(struct iio_dev *indio_dev,
 				VCNL4000_AL_RESULT_HI, val);
 			if (ret < 0)
 				return ret;
-			ret = IIO_VAL_INT;
-			break;
+			return IIO_VAL_INT;
 		case IIO_PROXIMITY:
 			ret = vcnl4000_measure(data,
 				VCNL4000_PS_OD, VCNL4000_PS_RDY,
 				VCNL4000_PS_RESULT_HI, val);
 			if (ret < 0)
 				return ret;
-			ret = IIO_VAL_INT;
-			break;
+			return IIO_VAL_INT;
 		default:
-			break;
+			return -EINVAL;
 		}
-		break;
 	case IIO_CHAN_INFO_SCALE:
-		if (chan->type == IIO_LIGHT) {
-			*val = 0;
-			*val2 = 250000;
-			ret = IIO_VAL_INT_PLUS_MICRO;
-		}
-		break;
+		if (chan->type != IIO_LIGHT)
+			return -EINVAL;
+
+		*val = 0;
+		*val2 = 250000;
+		return IIO_VAL_INT_PLUS_MICRO;
 	default:
-		break;
+		return -EINVAL;
 	}
-
-	return ret;
 }
 
 static const struct iio_info vcnl4000_info = {
@@ -155,7 +163,7 @@ static int vcnl4000_probe(struct i2c_client *client,
 {
 	struct vcnl4000_data *data;
 	struct iio_dev *indio_dev;
-	int ret;
+	int ret, prod_id;
 
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
 	if (!indio_dev)
@@ -164,13 +172,19 @@ static int vcnl4000_probe(struct i2c_client *client,
 	data = iio_priv(indio_dev);
 	i2c_set_clientdata(client, indio_dev);
 	data->client = client;
+	mutex_init(&data->lock);
 
 	ret = i2c_smbus_read_byte_data(data->client, VCNL4000_PROD_REV);
 	if (ret < 0)
 		return ret;
 
-	dev_info(&client->dev, "VCNL4000 Ambient light/proximity sensor, Prod %02x, Rev: %02x\n",
-		ret >> 4, ret & 0xf);
+	prod_id = ret >> 4;
+	if (prod_id != VCNL4010_ID && prod_id != VCNL4000_ID)
+		return -ENODEV;
+
+	dev_dbg(&client->dev, "%s Ambient light/proximity sensor, Rev: %02x\n",
+		(prod_id == VCNL4010_ID) ? "VCNL4010/4020" : "VCNL4000",
+		ret & 0xf);
 
 	indio_dev->dev.parent = &client->dev;
 	indio_dev->info = &vcnl4000_info;
diff --git a/drivers/iio/magnetometer/Kconfig b/drivers/iio/magnetometer/Kconfig
index 1f842abcb4a4..421ad90a5fbe 100644
--- a/drivers/iio/magnetometer/Kconfig
+++ b/drivers/iio/magnetometer/Kconfig
@@ -5,8 +5,22 @@
 
 menu "Magnetometer sensors"
 
+config AK8974
+	tristate "Asahi Kasei AK8974 3-Axis Magnetometer"
+	depends on I2C
+	depends on OF
+	select REGMAP_I2C
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	help
+	  Say yes here to build support for Asahi Kasei AK8974 or
+	  AMI305 I2C-based 3-axis magnetometer chips.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called ak8974.
+
 config AK8975
-	tristate "Asahi Kasei AK 3-Axis Magnetometer"
+	tristate "Asahi Kasei AK8975 3-Axis Magnetometer"
 	depends on I2C
 	depends on GPIOLIB || COMPILE_TEST
 	select IIO_BUFFER
diff --git a/drivers/iio/magnetometer/Makefile b/drivers/iio/magnetometer/Makefile
index 92a745c9a6e8..b86d6cb7f285 100644
--- a/drivers/iio/magnetometer/Makefile
+++ b/drivers/iio/magnetometer/Makefile
@@ -3,6 +3,7 @@
 #
 
 # When adding new entries keep the list in alphabetical order
+obj-$(CONFIG_AK8974)	+= ak8974.o
 obj-$(CONFIG_AK8975)	+= ak8975.o
 obj-$(CONFIG_BMC150_MAGN) += bmc150_magn.o
 obj-$(CONFIG_BMC150_MAGN_I2C) += bmc150_magn_i2c.o
diff --git a/drivers/iio/magnetometer/ak8974.c b/drivers/iio/magnetometer/ak8974.c
new file mode 100644
index 000000000000..217353145676
--- /dev/null
+++ b/drivers/iio/magnetometer/ak8974.c
@@ -0,0 +1,860 @@
+/*
+ * Driver for the Asahi Kasei EMD Corporation AK8974
+ * and Aichi Steel AMI305 magnetometer chips.
+ * Based on a patch from Samu Onkalo and the AK8975 IIO driver.
+ *
+ * Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
+ * Copyright (c) 2010 NVIDIA Corporation.
+ * Copyright (C) 2016 Linaro Ltd.
+ *
+ * Author: Samu Onkalo <samu.p.onkalo@nokia.com>
+ * Author: Linus Walleij <linus.walleij@linaro.org>
+ */
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h> /* For irq_get_irq_data() */
+#include <linux/completion.h>
+#include <linux/err.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+#include <linux/bitops.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/pm_runtime.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+
+/*
+ * 16-bit registers are little-endian. LSB is at the address defined below
+ * and MSB is at the next higher address.
+ */
+
+/* These registers are common for AK8974 and AMI305 */
+#define AK8974_SELFTEST		0x0C
+#define AK8974_SELFTEST_IDLE	0x55
+#define AK8974_SELFTEST_OK	0xAA
+
+#define AK8974_INFO		0x0D
+
+#define AK8974_WHOAMI		0x0F
+#define AK8974_WHOAMI_VALUE_AMI305 0x47
+#define AK8974_WHOAMI_VALUE_AK8974 0x48
+
+#define AK8974_DATA_X		0x10
+#define AK8974_DATA_Y		0x12
+#define AK8974_DATA_Z		0x14
+#define AK8974_INT_SRC		0x16
+#define AK8974_STATUS		0x18
+#define AK8974_INT_CLEAR	0x1A
+#define AK8974_CTRL1		0x1B
+#define AK8974_CTRL2		0x1C
+#define AK8974_CTRL3		0x1D
+#define AK8974_INT_CTRL		0x1E
+#define AK8974_INT_THRES	0x26  /* Absolute any axis value threshold */
+#define AK8974_PRESET		0x30
+
+/* AK8974-specific offsets */
+#define AK8974_OFFSET_X		0x20
+#define AK8974_OFFSET_Y		0x22
+#define AK8974_OFFSET_Z		0x24
+/* AMI305-specific offsets */
+#define AMI305_OFFSET_X		0x6C
+#define AMI305_OFFSET_Y		0x72
+#define AMI305_OFFSET_Z		0x78
+
+/* Different temperature registers */
+#define AK8974_TEMP		0x31
+#define AMI305_TEMP		0x60
+
+#define AK8974_INT_X_HIGH	BIT(7) /* Axis over +threshold  */
+#define AK8974_INT_Y_HIGH	BIT(6)
+#define AK8974_INT_Z_HIGH	BIT(5)
+#define AK8974_INT_X_LOW	BIT(4) /* Axis below -threshold	*/
+#define AK8974_INT_Y_LOW	BIT(3)
+#define AK8974_INT_Z_LOW	BIT(2)
+#define AK8974_INT_RANGE	BIT(1) /* Range overflow (any axis) */
+
+#define AK8974_STATUS_DRDY	BIT(6) /* Data ready */
+#define AK8974_STATUS_OVERRUN	BIT(5) /* Data overrun */
+#define AK8974_STATUS_INT	BIT(4) /* Interrupt occurred */
+
+#define AK8974_CTRL1_POWER	BIT(7) /* 0 = standby; 1 = active */
+#define AK8974_CTRL1_RATE	BIT(4) /* 0 = 10 Hz; 1 = 20 Hz	 */
+#define AK8974_CTRL1_FORCE_EN	BIT(1) /* 0 = normal; 1 = force	 */
+#define AK8974_CTRL1_MODE2	BIT(0) /* 0 */
+
+#define AK8974_CTRL2_INT_EN	BIT(4)  /* 1 = enable interrupts	      */
+#define AK8974_CTRL2_DRDY_EN	BIT(3)  /* 1 = enable data ready signal */
+#define AK8974_CTRL2_DRDY_POL	BIT(2)  /* 1 = data ready active high   */
+#define AK8974_CTRL2_RESDEF	(AK8974_CTRL2_DRDY_POL)
+
+#define AK8974_CTRL3_RESET	BIT(7) /* Software reset		  */
+#define AK8974_CTRL3_FORCE	BIT(6) /* Start forced measurement */
+#define AK8974_CTRL3_SELFTEST	BIT(4) /* Set selftest register	  */
+#define AK8974_CTRL3_RESDEF	0x00
+
+#define AK8974_INT_CTRL_XEN	BIT(7) /* Enable interrupt for this axis */
+#define AK8974_INT_CTRL_YEN	BIT(6)
+#define AK8974_INT_CTRL_ZEN	BIT(5)
+#define AK8974_INT_CTRL_XYZEN	(BIT(7)|BIT(6)|BIT(5))
+#define AK8974_INT_CTRL_POL	BIT(3) /* 0 = active low; 1 = active high */
+#define AK8974_INT_CTRL_PULSE	BIT(1) /* 0 = latched; 1 = pulse (50 usec) */
+#define AK8974_INT_CTRL_RESDEF	(AK8974_INT_CTRL_XYZEN | AK8974_INT_CTRL_POL)
+
+/* The AMI305 has elaborate FW version and serial number registers */
+#define AMI305_VER		0xE8
+#define AMI305_SN		0xEA
+
+#define AK8974_MAX_RANGE	2048
+
+#define AK8974_POWERON_DELAY	50
+#define AK8974_ACTIVATE_DELAY	1
+#define AK8974_SELFTEST_DELAY	1
+/*
+ * Set the autosuspend to two orders of magnitude larger than the poweron
+ * delay to make sane reasonable power tradeoff savings (5 seconds in
+ * this case).
+ */
+#define AK8974_AUTOSUSPEND_DELAY 5000
+
+#define AK8974_MEASTIME		3
+
+#define AK8974_PWR_ON		1
+#define AK8974_PWR_OFF		0
+
+/**
+ * struct ak8974 - state container for the AK8974 driver
+ * @i2c: parent I2C client
+ * @orientation: mounting matrix, flipped axis etc
+ * @map: regmap to access the AK8974 registers over I2C
+ * @regs: the avdd and dvdd power regulators
+ * @name: the name of the part
+ * @variant: the whoami ID value (for selecting code paths)
+ * @lock: locks the magnetometer for exclusive use during a measurement
+ * @drdy_irq: uses the DRDY IRQ line
+ * @drdy_complete: completion for DRDY
+ * @drdy_active_low: the DRDY IRQ is active low
+ */
+struct ak8974 {
+	struct i2c_client *i2c;
+	struct iio_mount_matrix orientation;
+	struct regmap *map;
+	struct regulator_bulk_data regs[2];
+	const char *name;
+	u8 variant;
+	struct mutex lock;
+	bool drdy_irq;
+	struct completion drdy_complete;
+	bool drdy_active_low;
+};
+
+static const char ak8974_reg_avdd[] = "avdd";
+static const char ak8974_reg_dvdd[] = "dvdd";
+
+static int ak8974_set_power(struct ak8974 *ak8974, bool mode)
+{
+	int ret;
+	u8 val;
+
+	val = mode ? AK8974_CTRL1_POWER : 0;
+	val |= AK8974_CTRL1_FORCE_EN;
+	ret = regmap_write(ak8974->map, AK8974_CTRL1, val);
+	if (ret < 0)
+		return ret;
+
+	if (mode)
+		msleep(AK8974_ACTIVATE_DELAY);
+
+	return 0;
+}
+
+static int ak8974_reset(struct ak8974 *ak8974)
+{
+	int ret;
+
+	/* Power on to get register access. Sets CTRL1 reg to reset state */
+	ret = ak8974_set_power(ak8974, AK8974_PWR_ON);
+	if (ret)
+		return ret;
+	ret = regmap_write(ak8974->map, AK8974_CTRL2, AK8974_CTRL2_RESDEF);
+	if (ret)
+		return ret;
+	ret = regmap_write(ak8974->map, AK8974_CTRL3, AK8974_CTRL3_RESDEF);
+	if (ret)
+		return ret;
+	ret = regmap_write(ak8974->map, AK8974_INT_CTRL,
+			   AK8974_INT_CTRL_RESDEF);
+	if (ret)
+		return ret;
+
+	/* After reset, power off is default state */
+	return ak8974_set_power(ak8974, AK8974_PWR_OFF);
+}
+
+static int ak8974_configure(struct ak8974 *ak8974)
+{
+	int ret;
+
+	ret = regmap_write(ak8974->map, AK8974_CTRL2, AK8974_CTRL2_DRDY_EN |
+			   AK8974_CTRL2_INT_EN);
+	if (ret)
+		return ret;
+	ret = regmap_write(ak8974->map, AK8974_CTRL3, 0);
+	if (ret)
+		return ret;
+	ret = regmap_write(ak8974->map, AK8974_INT_CTRL, AK8974_INT_CTRL_POL);
+	if (ret)
+		return ret;
+
+	return regmap_write(ak8974->map, AK8974_PRESET, 0);
+}
+
+static int ak8974_trigmeas(struct ak8974 *ak8974)
+{
+	unsigned int clear;
+	u8 mask;
+	u8 val;
+	int ret;
+
+	/* Clear any previous measurement overflow status */
+	ret = regmap_read(ak8974->map, AK8974_INT_CLEAR, &clear);
+	if (ret)
+		return ret;
+
+	/* If we have a DRDY IRQ line, use it */
+	if (ak8974->drdy_irq) {
+		mask = AK8974_CTRL2_INT_EN |
+			AK8974_CTRL2_DRDY_EN |
+			AK8974_CTRL2_DRDY_POL;
+		val = AK8974_CTRL2_DRDY_EN;
+
+		if (!ak8974->drdy_active_low)
+			val |= AK8974_CTRL2_DRDY_POL;
+
+		init_completion(&ak8974->drdy_complete);
+		ret = regmap_update_bits(ak8974->map, AK8974_CTRL2,
+					 mask, val);
+		if (ret)
+			return ret;
+	}
+
+	/* Force a measurement */
+	return regmap_update_bits(ak8974->map,
+				  AK8974_CTRL3,
+				  AK8974_CTRL3_FORCE,
+				  AK8974_CTRL3_FORCE);
+}
+
+static int ak8974_await_drdy(struct ak8974 *ak8974)
+{
+	int timeout = 2;
+	unsigned int val;
+	int ret;
+
+	if (ak8974->drdy_irq) {
+		ret = wait_for_completion_timeout(&ak8974->drdy_complete,
+					1 + msecs_to_jiffies(1000));
+		if (!ret) {
+			dev_err(&ak8974->i2c->dev,
+				"timeout waiting for DRDY IRQ\n");
+			return -ETIMEDOUT;
+		}
+		return 0;
+	}
+
+	/* Default delay-based poll loop */
+	do {
+		msleep(AK8974_MEASTIME);
+		ret = regmap_read(ak8974->map, AK8974_STATUS, &val);
+		if (ret < 0)
+			return ret;
+		if (val & AK8974_STATUS_DRDY)
+			return 0;
+	} while (--timeout);
+	if (!timeout) {
+		dev_err(&ak8974->i2c->dev,
+			"timeout waiting for DRDY\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int ak8974_getresult(struct ak8974 *ak8974, s16 *result)
+{
+	unsigned int src;
+	int ret;
+
+	ret = ak8974_await_drdy(ak8974);
+	if (ret)
+		return ret;
+	ret = regmap_read(ak8974->map, AK8974_INT_SRC, &src);
+	if (ret < 0)
+		return ret;
+
+	/* Out of range overflow! Strong magnet close? */
+	if (src & AK8974_INT_RANGE) {
+		dev_err(&ak8974->i2c->dev,
+			"range overflow in sensor\n");
+		return -ERANGE;
+	}
+
+	ret = regmap_bulk_read(ak8974->map, AK8974_DATA_X, result, 6);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static irqreturn_t ak8974_drdy_irq(int irq, void *d)
+{
+	struct ak8974 *ak8974 = d;
+
+	if (!ak8974->drdy_irq)
+		return IRQ_NONE;
+
+	/* TODO: timestamp here to get good measurement stamps */
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t ak8974_drdy_irq_thread(int irq, void *d)
+{
+	struct ak8974 *ak8974 = d;
+	unsigned int val;
+	int ret;
+
+	/* Check if this was a DRDY from us */
+	ret = regmap_read(ak8974->map, AK8974_STATUS, &val);
+	if (ret < 0) {
+		dev_err(&ak8974->i2c->dev, "error reading DRDY status\n");
+		return IRQ_HANDLED;
+	}
+	if (val & AK8974_STATUS_DRDY) {
+		/* Yes this was our IRQ */
+		complete(&ak8974->drdy_complete);
+		return IRQ_HANDLED;
+	}
+
+	/* We may be on a shared IRQ, let the next client check */
+	return IRQ_NONE;
+}
+
+static int ak8974_selftest(struct ak8974 *ak8974)
+{
+	struct device *dev = &ak8974->i2c->dev;
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(ak8974->map, AK8974_SELFTEST, &val);
+	if (ret)
+		return ret;
+	if (val != AK8974_SELFTEST_IDLE) {
+		dev_err(dev, "selftest not idle before test\n");
+		return -EIO;
+	}
+
+	/* Trigger self-test */
+	ret = regmap_update_bits(ak8974->map,
+			AK8974_CTRL3,
+			AK8974_CTRL3_SELFTEST,
+			AK8974_CTRL3_SELFTEST);
+	if (ret) {
+		dev_err(dev, "could not write CTRL3\n");
+		return ret;
+	}
+
+	msleep(AK8974_SELFTEST_DELAY);
+
+	ret = regmap_read(ak8974->map, AK8974_SELFTEST, &val);
+	if (ret)
+		return ret;
+	if (val != AK8974_SELFTEST_OK) {
+		dev_err(dev, "selftest result NOT OK (%02x)\n", val);
+		return -EIO;
+	}
+
+	ret = regmap_read(ak8974->map, AK8974_SELFTEST, &val);
+	if (ret)
+		return ret;
+	if (val != AK8974_SELFTEST_IDLE) {
+		dev_err(dev, "selftest not idle after test (%02x)\n", val);
+		return -EIO;
+	}
+	dev_dbg(dev, "passed self-test\n");
+
+	return 0;
+}
+
+static int ak8974_get_u16_val(struct ak8974 *ak8974, u8 reg, u16 *val)
+{
+	int ret;
+	u16 bulk;
+
+	ret = regmap_bulk_read(ak8974->map, reg, &bulk, 2);
+	if (ret)
+		return ret;
+	*val = le16_to_cpu(bulk);
+
+	return 0;
+}
+
+static int ak8974_detect(struct ak8974 *ak8974)
+{
+	unsigned int whoami;
+	const char *name;
+	int ret;
+	unsigned int fw;
+	u16 sn;
+
+	ret = regmap_read(ak8974->map, AK8974_WHOAMI, &whoami);
+	if (ret)
+		return ret;
+
+	switch (whoami) {
+	case AK8974_WHOAMI_VALUE_AMI305:
+		name = "ami305";
+		ret = regmap_read(ak8974->map, AMI305_VER, &fw);
+		if (ret)
+			return ret;
+		fw &= 0x7f; /* only bits 0 thru 6 valid */
+		ret = ak8974_get_u16_val(ak8974, AMI305_SN, &sn);
+		if (ret)
+			return ret;
+		dev_info(&ak8974->i2c->dev,
+			 "detected %s, FW ver %02x, S/N: %04x\n",
+			 name, fw, sn);
+		break;
+	case AK8974_WHOAMI_VALUE_AK8974:
+		name = "ak8974";
+		dev_info(&ak8974->i2c->dev, "detected AK8974\n");
+		break;
+	default:
+		dev_err(&ak8974->i2c->dev, "unsupported device (%02x) ",
+			whoami);
+		return -ENODEV;
+	}
+
+	ak8974->name = name;
+	ak8974->variant = whoami;
+
+	return 0;
+}
+
+static int ak8974_read_raw(struct iio_dev *indio_dev,
+			   struct iio_chan_spec const *chan,
+			   int *val, int *val2,
+			   long mask)
+{
+	struct ak8974 *ak8974 = iio_priv(indio_dev);
+	s16 hw_values[3];
+	int ret = -EINVAL;
+
+	pm_runtime_get_sync(&ak8974->i2c->dev);
+	mutex_lock(&ak8974->lock);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		if (chan->address > 2) {
+			dev_err(&ak8974->i2c->dev, "faulty channel address\n");
+			ret = -EIO;
+			goto out_unlock;
+		}
+		ret = ak8974_trigmeas(ak8974);
+		if (ret)
+			goto out_unlock;
+		ret = ak8974_getresult(ak8974, hw_values);
+		if (ret)
+			goto out_unlock;
+
+		/*
+		 * We read all axes and discard all but one, for optimized
+		 * reading, use the triggered buffer.
+		 */
+		*val = le16_to_cpu(hw_values[chan->address]);
+
+		ret = IIO_VAL_INT;
+	}
+
+ out_unlock:
+	mutex_unlock(&ak8974->lock);
+	pm_runtime_mark_last_busy(&ak8974->i2c->dev);
+	pm_runtime_put_autosuspend(&ak8974->i2c->dev);
+
+	return ret;
+}
+
+static void ak8974_fill_buffer(struct iio_dev *indio_dev)
+{
+	struct ak8974 *ak8974 = iio_priv(indio_dev);
+	int ret;
+	s16 hw_values[8]; /* Three axes + 64bit padding */
+
+	pm_runtime_get_sync(&ak8974->i2c->dev);
+	mutex_lock(&ak8974->lock);
+
+	ret = ak8974_trigmeas(ak8974);
+	if (ret) {
+		dev_err(&ak8974->i2c->dev, "error triggering measure\n");
+		goto out_unlock;
+	}
+	ret = ak8974_getresult(ak8974, hw_values);
+	if (ret) {
+		dev_err(&ak8974->i2c->dev, "error getting measures\n");
+		goto out_unlock;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, hw_values,
+					   iio_get_time_ns(indio_dev));
+
+ out_unlock:
+	mutex_unlock(&ak8974->lock);
+	pm_runtime_mark_last_busy(&ak8974->i2c->dev);
+	pm_runtime_put_autosuspend(&ak8974->i2c->dev);
+}
+
+static irqreturn_t ak8974_handle_trigger(int irq, void *p)
+{
+	const struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+
+	ak8974_fill_buffer(indio_dev);
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static const struct iio_mount_matrix *
+ak8974_get_mount_matrix(const struct iio_dev *indio_dev,
+			const struct iio_chan_spec *chan)
+{
+	struct ak8974 *ak8974 = iio_priv(indio_dev);
+
+	return &ak8974->orientation;
+}
+
+static const struct iio_chan_spec_ext_info ak8974_ext_info[] = {
+	IIO_MOUNT_MATRIX(IIO_SHARED_BY_DIR, ak8974_get_mount_matrix),
+	{ },
+};
+
+#define AK8974_AXIS_CHANNEL(axis, index)				\
+	{								\
+		.type = IIO_MAGN,					\
+		.modified = 1,						\
+		.channel2 = IIO_MOD_##axis,				\
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+		.ext_info = ak8974_ext_info,				\
+		.address = index,					\
+		.scan_index = index,					\
+		.scan_type = {						\
+			.sign = 's',					\
+			.realbits = 16,					\
+			.storagebits = 16,				\
+			.endianness = IIO_LE				\
+		},							\
+	}
+
+static const struct iio_chan_spec ak8974_channels[] = {
+	AK8974_AXIS_CHANNEL(X, 0),
+	AK8974_AXIS_CHANNEL(Y, 1),
+	AK8974_AXIS_CHANNEL(Z, 2),
+	IIO_CHAN_SOFT_TIMESTAMP(3),
+};
+
+static const unsigned long ak8974_scan_masks[] = { 0x7, 0 };
+
+static const struct iio_info ak8974_info = {
+	.read_raw = &ak8974_read_raw,
+	.driver_module = THIS_MODULE,
+};
+
+static bool ak8974_writeable_reg(struct device *dev, unsigned int reg)
+{
+	struct i2c_client *i2c = to_i2c_client(dev);
+	struct iio_dev *indio_dev = i2c_get_clientdata(i2c);
+	struct ak8974 *ak8974 = iio_priv(indio_dev);
+
+	switch (reg) {
+	case AK8974_CTRL1:
+	case AK8974_CTRL2:
+	case AK8974_CTRL3:
+	case AK8974_INT_CTRL:
+	case AK8974_INT_THRES:
+	case AK8974_INT_THRES + 1:
+	case AK8974_PRESET:
+	case AK8974_PRESET + 1:
+		return true;
+	case AK8974_OFFSET_X:
+	case AK8974_OFFSET_X + 1:
+	case AK8974_OFFSET_Y:
+	case AK8974_OFFSET_Y + 1:
+	case AK8974_OFFSET_Z:
+	case AK8974_OFFSET_Z + 1:
+		if (ak8974->variant == AK8974_WHOAMI_VALUE_AK8974)
+			return true;
+		return false;
+	case AMI305_OFFSET_X:
+	case AMI305_OFFSET_X + 1:
+	case AMI305_OFFSET_Y:
+	case AMI305_OFFSET_Y + 1:
+	case AMI305_OFFSET_Z:
+	case AMI305_OFFSET_Z + 1:
+		if (ak8974->variant == AK8974_WHOAMI_VALUE_AMI305)
+			return true;
+		return false;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_config ak8974_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = 0xff,
+	.writeable_reg = ak8974_writeable_reg,
+};
+
+static int ak8974_probe(struct i2c_client *i2c,
+			const struct i2c_device_id *id)
+{
+	struct iio_dev *indio_dev;
+	struct ak8974 *ak8974;
+	unsigned long irq_trig;
+	int irq = i2c->irq;
+	int ret;
+
+	/* Register with IIO */
+	indio_dev = devm_iio_device_alloc(&i2c->dev, sizeof(*ak8974));
+	if (indio_dev == NULL)
+		return -ENOMEM;
+
+	ak8974 = iio_priv(indio_dev);
+	i2c_set_clientdata(i2c, indio_dev);
+	ak8974->i2c = i2c;
+	mutex_init(&ak8974->lock);
+
+	ret = of_iio_read_mount_matrix(&i2c->dev,
+				       "mount-matrix",
+				       &ak8974->orientation);
+	if (ret)
+		return ret;
+
+	ak8974->regs[0].supply = ak8974_reg_avdd;
+	ak8974->regs[1].supply = ak8974_reg_dvdd;
+
+	ret = devm_regulator_bulk_get(&i2c->dev,
+				      ARRAY_SIZE(ak8974->regs),
+				      ak8974->regs);
+	if (ret < 0) {
+		dev_err(&i2c->dev, "cannot get regulators\n");
+		return ret;
+	}
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ak8974->regs), ak8974->regs);
+	if (ret < 0) {
+		dev_err(&i2c->dev, "cannot enable regulators\n");
+		return ret;
+	}
+
+	/* Take runtime PM online */
+	pm_runtime_get_noresume(&i2c->dev);
+	pm_runtime_set_active(&i2c->dev);
+	pm_runtime_enable(&i2c->dev);
+
+	ak8974->map = devm_regmap_init_i2c(i2c, &ak8974_regmap_config);
+	if (IS_ERR(ak8974->map)) {
+		dev_err(&i2c->dev, "failed to allocate register map\n");
+		return PTR_ERR(ak8974->map);
+	}
+
+	ret = ak8974_set_power(ak8974, AK8974_PWR_ON);
+	if (ret) {
+		dev_err(&i2c->dev, "could not power on\n");
+		goto power_off;
+	}
+
+	ret = ak8974_detect(ak8974);
+	if (ret) {
+		dev_err(&i2c->dev, "neither AK8974 nor AMI305 found\n");
+		goto power_off;
+	}
+
+	ret = ak8974_selftest(ak8974);
+	if (ret)
+		dev_err(&i2c->dev, "selftest failed (continuing anyway)\n");
+
+	ret = ak8974_reset(ak8974);
+	if (ret) {
+		dev_err(&i2c->dev, "AK8974 reset failed\n");
+		goto power_off;
+	}
+
+	pm_runtime_set_autosuspend_delay(&i2c->dev,
+					 AK8974_AUTOSUSPEND_DELAY);
+	pm_runtime_use_autosuspend(&i2c->dev);
+	pm_runtime_put(&i2c->dev);
+
+	indio_dev->dev.parent = &i2c->dev;
+	indio_dev->channels = ak8974_channels;
+	indio_dev->num_channels = ARRAY_SIZE(ak8974_channels);
+	indio_dev->info = &ak8974_info;
+	indio_dev->available_scan_masks = ak8974_scan_masks;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->name = ak8974->name;
+
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+					 ak8974_handle_trigger,
+					 NULL);
+	if (ret) {
+		dev_err(&i2c->dev, "triggered buffer setup failed\n");
+		goto disable_pm;
+	}
+
+	/* If we have a valid DRDY IRQ, make use of it */
+	if (irq > 0) {
+		irq_trig = irqd_get_trigger_type(irq_get_irq_data(irq));
+		if (irq_trig == IRQF_TRIGGER_RISING) {
+			dev_info(&i2c->dev, "enable rising edge DRDY IRQ\n");
+		} else if (irq_trig == IRQF_TRIGGER_FALLING) {
+			ak8974->drdy_active_low = true;
+			dev_info(&i2c->dev, "enable falling edge DRDY IRQ\n");
+		} else {
+			irq_trig = IRQF_TRIGGER_RISING;
+		}
+		irq_trig |= IRQF_ONESHOT;
+		irq_trig |= IRQF_SHARED;
+
+		ret = devm_request_threaded_irq(&i2c->dev,
+						irq,
+						ak8974_drdy_irq,
+						ak8974_drdy_irq_thread,
+						irq_trig,
+						ak8974->name,
+						ak8974);
+		if (ret) {
+			dev_err(&i2c->dev, "unable to request DRDY IRQ "
+				"- proceeding without IRQ\n");
+			goto no_irq;
+		}
+		ak8974->drdy_irq = true;
+	}
+
+no_irq:
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&i2c->dev, "device register failed\n");
+		goto cleanup_buffer;
+	}
+
+	return 0;
+
+cleanup_buffer:
+	iio_triggered_buffer_cleanup(indio_dev);
+disable_pm:
+	pm_runtime_put_noidle(&i2c->dev);
+	pm_runtime_disable(&i2c->dev);
+	ak8974_set_power(ak8974, AK8974_PWR_OFF);
+power_off:
+	regulator_bulk_disable(ARRAY_SIZE(ak8974->regs), ak8974->regs);
+
+	return ret;
+}
+
+static int __exit ak8974_remove(struct i2c_client *i2c)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(i2c);
+	struct ak8974 *ak8974 = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
+	pm_runtime_get_sync(&i2c->dev);
+	pm_runtime_put_noidle(&i2c->dev);
+	pm_runtime_disable(&i2c->dev);
+	ak8974_set_power(ak8974, AK8974_PWR_OFF);
+	regulator_bulk_disable(ARRAY_SIZE(ak8974->regs), ak8974->regs);
+
+	return 0;
+}
+
+static int __maybe_unused ak8974_runtime_suspend(struct device *dev)
+{
+	struct ak8974 *ak8974 =
+		iio_priv(i2c_get_clientdata(to_i2c_client(dev)));
+
+	ak8974_set_power(ak8974, AK8974_PWR_OFF);
+	regulator_bulk_disable(ARRAY_SIZE(ak8974->regs), ak8974->regs);
+
+	return 0;
+}
+
+static int __maybe_unused ak8974_runtime_resume(struct device *dev)
+{
+	struct ak8974 *ak8974 =
+		iio_priv(i2c_get_clientdata(to_i2c_client(dev)));
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ak8974->regs), ak8974->regs);
+	if (ret)
+		return ret;
+	msleep(AK8974_POWERON_DELAY);
+	ret = ak8974_set_power(ak8974, AK8974_PWR_ON);
+	if (ret)
+		goto out_regulator_disable;
+
+	ret = ak8974_configure(ak8974);
+	if (ret)
+		goto out_disable_power;
+
+	return 0;
+
+out_disable_power:
+	ak8974_set_power(ak8974, AK8974_PWR_OFF);
+out_regulator_disable:
+	regulator_bulk_disable(ARRAY_SIZE(ak8974->regs), ak8974->regs);
+
+	return ret;
+}
+
+static const struct dev_pm_ops ak8974_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(ak8974_runtime_suspend,
+			   ak8974_runtime_resume, NULL)
+};
+
+static const struct i2c_device_id ak8974_id[] = {
+	{"ami305", 0 },
+	{"ak8974", 0 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, ak8974_id);
+
+static const struct of_device_id ak8974_of_match[] = {
+	{ .compatible = "asahi-kasei,ak8974", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ak8974_of_match);
+
+static struct i2c_driver ak8974_driver = {
+	.driver	 = {
+		.name	= "ak8974",
+		.pm = &ak8974_dev_pm_ops,
+		.of_match_table = of_match_ptr(ak8974_of_match),
+	},
+	.probe	  = ak8974_probe,
+	.remove	  = __exit_p(ak8974_remove),
+	.id_table = ak8974_id,
+};
+module_i2c_driver(ak8974_driver);
+
+MODULE_DESCRIPTION("AK8974 and AMI305 3-axis magnetometer driver");
+MODULE_AUTHOR("Samu Onkalo");
+MODULE_AUTHOR("Linus Walleij");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/magnetometer/mag3110.c b/drivers/iio/magnetometer/mag3110.c
index f2be4a049056..f2b3bd7bf862 100644
--- a/drivers/iio/magnetometer/mag3110.c
+++ b/drivers/iio/magnetometer/mag3110.c
@@ -154,34 +154,41 @@ static int mag3110_read_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		if (iio_buffer_enabled(indio_dev))
-			return -EBUSY;
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
 
 		switch (chan->type) {
 		case IIO_MAGN: /* in 0.1 uT / LSB */
 			ret = mag3110_read(data, buffer);
 			if (ret < 0)
-				return ret;
+				goto release;
 			*val = sign_extend32(
 				be16_to_cpu(buffer[chan->scan_index]), 15);
-			return IIO_VAL_INT;
+			ret = IIO_VAL_INT;
+			break;
 		case IIO_TEMP: /* in 1 C / LSB */
 			mutex_lock(&data->lock);
 			ret = mag3110_request(data);
 			if (ret < 0) {
 				mutex_unlock(&data->lock);
-				return ret;
+				goto release;
 			}
 			ret = i2c_smbus_read_byte_data(data->client,
 				MAG3110_DIE_TEMP);
 			mutex_unlock(&data->lock);
 			if (ret < 0)
-				return ret;
+				goto release;
 			*val = sign_extend32(ret, 7);
-			return IIO_VAL_INT;
+			ret = IIO_VAL_INT;
+			break;
 		default:
-			return -EINVAL;
+			ret = -EINVAL;
 		}
+release:
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+
 	case IIO_CHAN_INFO_SCALE:
 		switch (chan->type) {
 		case IIO_MAGN:
diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig
index d130cdc78f43..15cd416365c1 100644
--- a/drivers/iio/pressure/Kconfig
+++ b/drivers/iio/pressure/Kconfig
@@ -8,8 +8,6 @@ menu "Pressure sensors"
 config BMP280
 	tristate "Bosch Sensortec BMP180/BMP280 pressure sensor I2C driver"
 	depends on (I2C || SPI_MASTER)
-	depends on !(BMP085_I2C=y || BMP085_I2C=m)
-	depends on !(BMP085_SPI=y || BMP085_SPI=m)
 	select REGMAP
 	select BMP280_I2C if (I2C)
 	select BMP280_SPI if (SPI_MASTER)
@@ -187,4 +185,26 @@ config HP206C
 	  This driver can also be built as a module. If so, the module will
 	  be called hp206c.
 
+config ZPA2326
+	tristate "Murata ZPA2326 pressure sensor driver"
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	select REGMAP
+	select ZPA2326_I2C if I2C
+	select ZPA2326_SPI if SPI_MASTER
+	help
+	  Say Y here to build support for the Murata ZPA2326 pressure and
+	  temperature sensor.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called zpa2326.
+
+config ZPA2326_I2C
+	tristate
+	select REGMAP_I2C
+
+config ZPA2326_SPI
+	tristate
+	select REGMAP_SPI
+
 endmenu
diff --git a/drivers/iio/pressure/Makefile b/drivers/iio/pressure/Makefile
index 7f395bed5e88..fff77185a5cc 100644
--- a/drivers/iio/pressure/Makefile
+++ b/drivers/iio/pressure/Makefile
@@ -22,6 +22,9 @@ st_pressure-y := st_pressure_core.o
 st_pressure-$(CONFIG_IIO_BUFFER) += st_pressure_buffer.o
 obj-$(CONFIG_T5403) += t5403.o
 obj-$(CONFIG_HP206C) += hp206c.o
+obj-$(CONFIG_ZPA2326) += zpa2326.o
+obj-$(CONFIG_ZPA2326_I2C) += zpa2326_i2c.o
+obj-$(CONFIG_ZPA2326_SPI) += zpa2326_spi.o
 
 obj-$(CONFIG_IIO_ST_PRESS_I2C) += st_pressure_i2c.o
 obj-$(CONFIG_IIO_ST_PRESS_SPI) += st_pressure_spi.o
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index 6943688e66df..e5a533cbd53f 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -970,7 +970,7 @@ int bmp280_common_probe(struct device *dev,
 	data->vdda = devm_regulator_get(dev, "vdda");
 	if (IS_ERR(data->vdda)) {
 		dev_err(dev, "failed to get VDDA regulator\n");
-		ret = PTR_ERR(data->vddd);
+		ret = PTR_ERR(data->vdda);
 		goto out_disable_vddd;
 	}
 	ret = regulator_enable(data->vdda);
@@ -1079,7 +1079,8 @@ EXPORT_SYMBOL(bmp280_common_remove);
 #ifdef CONFIG_PM
 static int bmp280_runtime_suspend(struct device *dev)
 {
-	struct bmp280_data *data = dev_get_drvdata(dev);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct bmp280_data *data = iio_priv(indio_dev);
 	int ret;
 
 	ret = regulator_disable(data->vdda);
@@ -1090,7 +1091,8 @@ static int bmp280_runtime_suspend(struct device *dev)
 
 static int bmp280_runtime_resume(struct device *dev)
 {
-	struct bmp280_data *data = dev_get_drvdata(dev);
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct bmp280_data *data = iio_priv(indio_dev);
 	int ret;
 
 	ret = regulator_enable(data->vddd);
diff --git a/drivers/iio/pressure/ms5611_core.c b/drivers/iio/pressure/ms5611_core.c
index feb41f82c64a..a74ed1f0c880 100644
--- a/drivers/iio/pressure/ms5611_core.c
+++ b/drivers/iio/pressure/ms5611_core.c
@@ -416,8 +416,7 @@ static int ms5611_init(struct iio_dev *indio_dev)
 	return 0;
 
 err_regulator_disable:
-	if (!IS_ERR_OR_NULL(st->vdd))
-		regulator_disable(st->vdd);
+	regulator_disable(st->vdd);
 	return ret;
 }
 
@@ -425,8 +424,7 @@ static void ms5611_fini(const struct iio_dev *indio_dev)
 {
 	const struct ms5611_state *st = iio_priv(indio_dev);
 
-	if (!IS_ERR_OR_NULL(st->vdd))
-		regulator_disable(st->vdd);
+	regulator_disable(st->vdd);
 }
 
 int ms5611_probe(struct iio_dev *indio_dev, struct device *dev,
diff --git a/drivers/iio/pressure/zpa2326.c b/drivers/iio/pressure/zpa2326.c
new file mode 100644
index 000000000000..19d2eb46fda6
--- /dev/null
+++ b/drivers/iio/pressure/zpa2326.c
@@ -0,0 +1,1735 @@
+/*
+ * Murata ZPA2326 pressure and temperature sensor IIO driver
+ *
+ * Copyright (c) 2016 Parrot S.A.
+ *
+ * Author: Gregor Boirie <gregor.boirie@parrot.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+/**
+ * DOC: ZPA2326 theory of operations
+ *
+ * This driver supports %INDIO_DIRECT_MODE and %INDIO_BUFFER_TRIGGERED IIO
+ * modes.
+ * A internal hardware trigger is also implemented to dispatch registered IIO
+ * trigger consumers upon "sample ready" interrupts.
+ *
+ * ZPA2326 hardware supports 2 sampling mode: one shot and continuous.
+ *
+ * A complete one shot sampling cycle gets device out of low power mode,
+ * performs pressure and temperature measurements, then automatically switches
+ * back to low power mode. It is meant for on demand sampling with optimal power
+ * saving at the cost of lower sampling rate and higher software overhead.
+ * This is a natural candidate for IIO read_raw hook implementation
+ * (%INDIO_DIRECT_MODE). It is also used for triggered buffering support to
+ * ensure explicit synchronization with external trigger events
+ * (%INDIO_BUFFER_TRIGGERED).
+ *
+ * The continuous mode works according to a periodic hardware measurement
+ * process continuously pushing samples into an internal hardware FIFO (for
+ * pressure samples only). Measurement cycle completion may be signaled by a
+ * "sample ready" interrupt.
+ * Typical software sequence of operations :
+ * - get device out of low power mode,
+ * - setup hardware sampling period,
+ * - at end of period, upon data ready interrupt: pop pressure samples out of
+ *   hardware FIFO and fetch temperature sample
+ * - when no longer needed, stop sampling process by putting device into
+ *   low power mode.
+ * This mode is used to implement %INDIO_BUFFER_TRIGGERED mode if device tree
+ * declares a valid interrupt line. In this case, the internal hardware trigger
+ * drives acquisition.
+ *
+ * Note that hardware sampling frequency is taken into account only when
+ * internal hardware trigger is attached as the highest sampling rate seems to
+ * be the most energy efficient.
+ *
+ * TODO:
+ *   preset pressure threshold crossing / IIO events ;
+ *   differential pressure sampling ;
+ *   hardware samples averaging.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/regulator/consumer.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+#include "zpa2326.h"
+
+/* 200 ms should be enough for the longest conversion time in one-shot mode. */
+#define ZPA2326_CONVERSION_JIFFIES (HZ / 5)
+
+/* There should be a 1 ms delay (Tpup) after getting out of reset. */
+#define ZPA2326_TPUP_USEC_MIN      (1000)
+#define ZPA2326_TPUP_USEC_MAX      (2000)
+
+/**
+ * struct zpa2326_frequency - Hardware sampling frequency descriptor
+ * @hz : Frequency in Hertz.
+ * @odr: Output Data Rate word as expected by %ZPA2326_CTRL_REG3_REG.
+ */
+struct zpa2326_frequency {
+	int hz;
+	u16 odr;
+};
+
+/*
+ * Keep these in strict ascending order: last array entry is expected to
+ * correspond to the highest sampling frequency.
+ */
+static const struct zpa2326_frequency zpa2326_sampling_frequencies[] = {
+	{ .hz = 1,  .odr = 1 << ZPA2326_CTRL_REG3_ODR_SHIFT },
+	{ .hz = 5,  .odr = 5 << ZPA2326_CTRL_REG3_ODR_SHIFT },
+	{ .hz = 11, .odr = 6 << ZPA2326_CTRL_REG3_ODR_SHIFT },
+	{ .hz = 23, .odr = 7 << ZPA2326_CTRL_REG3_ODR_SHIFT },
+};
+
+/* Return the highest hardware sampling frequency available. */
+static const struct zpa2326_frequency *zpa2326_highest_frequency(void)
+{
+	return &zpa2326_sampling_frequencies[
+		ARRAY_SIZE(zpa2326_sampling_frequencies) - 1];
+}
+
+/**
+ * struct zpa_private - Per-device internal private state
+ * @timestamp:  Buffered samples ready datum.
+ * @regmap:     Underlying I2C / SPI bus adapter used to abstract slave register
+ *              accesses.
+ * @result:     Allows sampling logic to get completion status of operations
+ *              that interrupt handlers perform asynchronously.
+ * @data_ready: Interrupt handler uses this to wake user context up at sampling
+ *              operation completion.
+ * @trigger:    Optional hardware / interrupt driven trigger used to notify
+ *              external devices a new sample is ready.
+ * @waken:      Flag indicating whether or not device has just been powered on.
+ * @irq:        Optional interrupt line: negative or zero if not declared into
+ *              DT, in which case sampling logic keeps polling status register
+ *              to detect completion.
+ * @frequency:  Current hardware sampling frequency.
+ * @vref:       Power / voltage reference.
+ * @vdd:        Power supply.
+ */
+struct zpa2326_private {
+	s64                             timestamp;
+	struct regmap                  *regmap;
+	int                             result;
+	struct completion               data_ready;
+	struct iio_trigger             *trigger;
+	bool                            waken;
+	int                             irq;
+	const struct zpa2326_frequency *frequency;
+	struct regulator               *vref;
+	struct regulator               *vdd;
+};
+
+#define zpa2326_err(_idev, _format, _arg...) \
+	dev_err(_idev->dev.parent, _format, ##_arg)
+
+#define zpa2326_warn(_idev, _format, _arg...) \
+	dev_warn(_idev->dev.parent, _format, ##_arg)
+
+#ifdef DEBUG
+#define zpa2326_dbg(_idev, _format, _arg...) \
+	dev_dbg(_idev->dev.parent, _format, ##_arg)
+#else
+#define zpa2326_dbg(_idev, _format, _arg...)
+#endif
+
+bool zpa2326_isreg_writeable(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case ZPA2326_REF_P_XL_REG:
+	case ZPA2326_REF_P_L_REG:
+	case ZPA2326_REF_P_H_REG:
+	case ZPA2326_RES_CONF_REG:
+	case ZPA2326_CTRL_REG0_REG:
+	case ZPA2326_CTRL_REG1_REG:
+	case ZPA2326_CTRL_REG2_REG:
+	case ZPA2326_CTRL_REG3_REG:
+	case ZPA2326_THS_P_LOW_REG:
+	case ZPA2326_THS_P_HIGH_REG:
+		return true;
+
+	default:
+		return false;
+	}
+}
+EXPORT_SYMBOL_GPL(zpa2326_isreg_writeable);
+
+bool zpa2326_isreg_readable(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case ZPA2326_REF_P_XL_REG:
+	case ZPA2326_REF_P_L_REG:
+	case ZPA2326_REF_P_H_REG:
+	case ZPA2326_DEVICE_ID_REG:
+	case ZPA2326_RES_CONF_REG:
+	case ZPA2326_CTRL_REG0_REG:
+	case ZPA2326_CTRL_REG1_REG:
+	case ZPA2326_CTRL_REG2_REG:
+	case ZPA2326_CTRL_REG3_REG:
+	case ZPA2326_INT_SOURCE_REG:
+	case ZPA2326_THS_P_LOW_REG:
+	case ZPA2326_THS_P_HIGH_REG:
+	case ZPA2326_STATUS_REG:
+	case ZPA2326_PRESS_OUT_XL_REG:
+	case ZPA2326_PRESS_OUT_L_REG:
+	case ZPA2326_PRESS_OUT_H_REG:
+	case ZPA2326_TEMP_OUT_L_REG:
+	case ZPA2326_TEMP_OUT_H_REG:
+		return true;
+
+	default:
+		return false;
+	}
+}
+EXPORT_SYMBOL_GPL(zpa2326_isreg_readable);
+
+bool zpa2326_isreg_precious(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case ZPA2326_INT_SOURCE_REG:
+	case ZPA2326_PRESS_OUT_H_REG:
+		return true;
+
+	default:
+		return false;
+	}
+}
+EXPORT_SYMBOL_GPL(zpa2326_isreg_precious);
+
+/**
+ * zpa2326_enable_device() - Enable device, i.e. get out of low power mode.
+ * @indio_dev: The IIO device associated with the hardware to enable.
+ *
+ * Required to access complete register space and to perform any sampling
+ * or control operations.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_enable_device(const struct iio_dev *indio_dev)
+{
+	int err;
+
+	err = regmap_write(((struct zpa2326_private *)
+			    iio_priv(indio_dev))->regmap,
+			    ZPA2326_CTRL_REG0_REG, ZPA2326_CTRL_REG0_ENABLE);
+	if (err) {
+		zpa2326_err(indio_dev, "failed to enable device (%d)", err);
+		return err;
+	}
+
+	zpa2326_dbg(indio_dev, "enabled");
+
+	return 0;
+}
+
+/**
+ * zpa2326_sleep() - Disable device, i.e. switch to low power mode.
+ * @indio_dev: The IIO device associated with the hardware to disable.
+ *
+ * Only %ZPA2326_DEVICE_ID_REG and %ZPA2326_CTRL_REG0_REG registers may be
+ * accessed once device is in the disabled state.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_sleep(const struct iio_dev *indio_dev)
+{
+	int err;
+
+	err = regmap_write(((struct zpa2326_private *)
+			    iio_priv(indio_dev))->regmap,
+			    ZPA2326_CTRL_REG0_REG, 0);
+	if (err) {
+		zpa2326_err(indio_dev, "failed to sleep (%d)", err);
+		return err;
+	}
+
+	zpa2326_dbg(indio_dev, "sleeping");
+
+	return 0;
+}
+
+/**
+ * zpa2326_reset_device() - Reset device to default hardware state.
+ * @indio_dev: The IIO device associated with the hardware to reset.
+ *
+ * Disable sampling and empty hardware FIFO.
+ * Device must be enabled before reset, i.e. not in low power mode.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_reset_device(const struct iio_dev *indio_dev)
+{
+	int err;
+
+	err = regmap_write(((struct zpa2326_private *)
+			    iio_priv(indio_dev))->regmap,
+			    ZPA2326_CTRL_REG2_REG, ZPA2326_CTRL_REG2_SWRESET);
+	if (err) {
+		zpa2326_err(indio_dev, "failed to reset device (%d)", err);
+		return err;
+	}
+
+	usleep_range(ZPA2326_TPUP_USEC_MIN, ZPA2326_TPUP_USEC_MAX);
+
+	zpa2326_dbg(indio_dev, "reset");
+
+	return 0;
+}
+
+/**
+ * zpa2326_start_oneshot() - Start a single sampling cycle, i.e. in one shot
+ *                           mode.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ *
+ * Device must have been previously enabled and configured for one shot mode.
+ * Device will be switched back to low power mode at end of cycle.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_start_oneshot(const struct iio_dev *indio_dev)
+{
+	int err;
+
+	err = regmap_write(((struct zpa2326_private *)
+			    iio_priv(indio_dev))->regmap,
+			    ZPA2326_CTRL_REG0_REG,
+			    ZPA2326_CTRL_REG0_ENABLE |
+			    ZPA2326_CTRL_REG0_ONE_SHOT);
+	if (err) {
+		zpa2326_err(indio_dev, "failed to start one shot cycle (%d)",
+			    err);
+		return err;
+	}
+
+	zpa2326_dbg(indio_dev, "one shot cycle started");
+
+	return 0;
+}
+
+/**
+ * zpa2326_power_on() - Power on device to allow subsequent configuration.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @private:   Internal private state related to @indio_dev.
+ *
+ * Sampling will be disabled, preventing strange things from happening in our
+ * back. Hardware FIFO content will be cleared.
+ * When successful, device will be left in the enabled state to allow further
+ * configuration.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_power_on(const struct iio_dev         *indio_dev,
+			    const struct zpa2326_private *private)
+{
+	int err;
+
+	err = regulator_enable(private->vref);
+	if (err)
+		return err;
+
+	err = regulator_enable(private->vdd);
+	if (err)
+		goto vref;
+
+	zpa2326_dbg(indio_dev, "powered on");
+
+	err = zpa2326_enable_device(indio_dev);
+	if (err)
+		goto vdd;
+
+	err = zpa2326_reset_device(indio_dev);
+	if (err)
+		goto sleep;
+
+	return 0;
+
+sleep:
+	zpa2326_sleep(indio_dev);
+vdd:
+	regulator_disable(private->vdd);
+vref:
+	regulator_disable(private->vref);
+
+	zpa2326_dbg(indio_dev, "powered off");
+
+	return err;
+}
+
+/**
+ * zpa2326_power_off() - Power off device, i.e. disable attached power
+ *                       regulators.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @private:   Internal private state related to @indio_dev.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static void zpa2326_power_off(const struct iio_dev         *indio_dev,
+			      const struct zpa2326_private *private)
+{
+	regulator_disable(private->vdd);
+	regulator_disable(private->vref);
+
+	zpa2326_dbg(indio_dev, "powered off");
+}
+
+/**
+ * zpa2326_config_oneshot() - Setup device for one shot / on demand mode.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @irq:       Optional interrupt line the hardware uses to notify new data
+ *             samples are ready. Negative or zero values indicate no interrupts
+ *             are available, meaning polling is required.
+ *
+ * Output Data Rate is configured for the highest possible rate so that
+ * conversion time and power consumption are reduced to a minimum.
+ * Note that hardware internal averaging machinery (not implemented in this
+ * driver) is not applicable in this mode.
+ *
+ * Device must have been previously enabled before calling
+ * zpa2326_config_oneshot().
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_config_oneshot(const struct iio_dev *indio_dev,
+				  int                   irq)
+{
+	struct regmap                  *regs = ((struct zpa2326_private *)
+						iio_priv(indio_dev))->regmap;
+	const struct zpa2326_frequency *freq = zpa2326_highest_frequency();
+	int                             err;
+
+	/* Setup highest available Output Data Rate for one shot mode. */
+	err = regmap_write(regs, ZPA2326_CTRL_REG3_REG, freq->odr);
+	if (err)
+		return err;
+
+	if (irq > 0) {
+		/* Request interrupt when new sample is available. */
+		err = regmap_write(regs, ZPA2326_CTRL_REG1_REG,
+				   (u8)~ZPA2326_CTRL_REG1_MASK_DATA_READY);
+
+		if (err) {
+			dev_err(indio_dev->dev.parent,
+				"failed to setup one shot mode (%d)", err);
+			return err;
+		}
+	}
+
+	zpa2326_dbg(indio_dev, "one shot mode setup @%dHz", freq->hz);
+
+	return 0;
+}
+
+/**
+ * zpa2326_clear_fifo() - Clear remaining entries in hardware FIFO.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @min_count: Number of samples present within hardware FIFO.
+ *
+ * @min_count argument is a hint corresponding to the known minimum number of
+ * samples currently living in the FIFO. This allows to reduce the number of bus
+ * accesses by skipping status register read operation as long as we know for
+ * sure there are still entries left.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_clear_fifo(const struct iio_dev *indio_dev,
+			      unsigned int          min_count)
+{
+	struct regmap *regs = ((struct zpa2326_private *)
+			       iio_priv(indio_dev))->regmap;
+	int            err;
+	unsigned int   val;
+
+	if (!min_count) {
+		/*
+		 * No hint: read status register to determine whether FIFO is
+		 * empty or not.
+		 */
+		err = regmap_read(regs, ZPA2326_STATUS_REG, &val);
+
+		if (err < 0)
+			goto err;
+
+		if (val & ZPA2326_STATUS_FIFO_E)
+			/* Fifo is empty: nothing to trash. */
+			return 0;
+	}
+
+	/* Clear FIFO. */
+	do {
+		/*
+		 * A single fetch from pressure MSB register is enough to pop
+		 * values out of FIFO.
+		 */
+		err = regmap_read(regs, ZPA2326_PRESS_OUT_H_REG, &val);
+		if (err < 0)
+			goto err;
+
+		if (min_count) {
+			/*
+			 * We know for sure there are at least min_count entries
+			 * left in FIFO. Skip status register read.
+			 */
+			min_count--;
+			continue;
+		}
+
+		err = regmap_read(regs, ZPA2326_STATUS_REG, &val);
+		if (err < 0)
+			goto err;
+
+	} while (!(val & ZPA2326_STATUS_FIFO_E));
+
+	zpa2326_dbg(indio_dev, "FIFO cleared");
+
+	return 0;
+
+err:
+	zpa2326_err(indio_dev, "failed to clear FIFO (%d)", err);
+
+	return err;
+}
+
+/**
+ * zpa2326_dequeue_pressure() - Retrieve the most recent pressure sample from
+ *                              hardware FIFO.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @pressure:  Sampled pressure output.
+ *
+ * Note that ZPA2326 hardware FIFO stores pressure samples only.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_dequeue_pressure(const struct iio_dev *indio_dev,
+				    u32                  *pressure)
+{
+	struct regmap *regs = ((struct zpa2326_private *)
+			       iio_priv(indio_dev))->regmap;
+	unsigned int   val;
+	int            err;
+	int            cleared = -1;
+
+	err = regmap_read(regs, ZPA2326_STATUS_REG, &val);
+	if (err < 0)
+		return err;
+
+	*pressure = 0;
+
+	if (val & ZPA2326_STATUS_P_OR) {
+		/*
+		 * Fifo overrun : first sample dequeued from FIFO is the
+		 * newest.
+		 */
+		zpa2326_warn(indio_dev, "FIFO overflow");
+
+		err = regmap_bulk_read(regs, ZPA2326_PRESS_OUT_XL_REG, pressure,
+				       3);
+		if (err)
+			return err;
+
+#define ZPA2326_FIFO_DEPTH (16U)
+		/* Hardware FIFO may hold no more than 16 pressure samples. */
+		return zpa2326_clear_fifo(indio_dev, ZPA2326_FIFO_DEPTH - 1);
+	}
+
+	/*
+	 * Fifo has not overflown : retrieve newest sample. We need to pop
+	 * values out until FIFO is empty : last fetched pressure is the newest.
+	 * In nominal cases, we should find a single queued sample only.
+	 */
+	do {
+		err = regmap_bulk_read(regs, ZPA2326_PRESS_OUT_XL_REG, pressure,
+				       3);
+		if (err)
+			return err;
+
+		err = regmap_read(regs, ZPA2326_STATUS_REG, &val);
+		if (err < 0)
+			return err;
+
+		cleared++;
+	} while (!(val & ZPA2326_STATUS_FIFO_E));
+
+	if (cleared)
+		/*
+		 * Samples were pushed by hardware during previous rounds but we
+		 * didn't consume them fast enough: inform user.
+		 */
+		zpa2326_dbg(indio_dev, "cleared %d FIFO entries", cleared);
+
+	return 0;
+}
+
+/**
+ * zpa2326_fill_sample_buffer() - Enqueue new channel samples to IIO buffer.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @private:   Internal private state related to @indio_dev.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_fill_sample_buffer(struct iio_dev               *indio_dev,
+				      const struct zpa2326_private *private)
+{
+	struct {
+		u32 pressure;
+		u16 temperature;
+		u64 timestamp;
+	}   sample;
+	int err;
+
+	if (test_bit(0, indio_dev->active_scan_mask)) {
+		/* Get current pressure from hardware FIFO. */
+		err = zpa2326_dequeue_pressure(indio_dev, &sample.pressure);
+		if (err) {
+			zpa2326_warn(indio_dev, "failed to fetch pressure (%d)",
+				     err);
+			return err;
+		}
+	}
+
+	if (test_bit(1, indio_dev->active_scan_mask)) {
+		/* Get current temperature. */
+		err = regmap_bulk_read(private->regmap, ZPA2326_TEMP_OUT_L_REG,
+				       &sample.temperature, 2);
+		if (err) {
+			zpa2326_warn(indio_dev,
+				     "failed to fetch temperature (%d)", err);
+			return err;
+		}
+	}
+
+	/*
+	 * Now push samples using timestamp stored either :
+	 *   - by hardware interrupt handler if interrupt is available: see
+	 *     zpa2326_handle_irq(),
+	 *   - or oneshot completion polling machinery : see
+	 *     zpa2326_trigger_handler().
+	 */
+	zpa2326_dbg(indio_dev, "filling raw samples buffer");
+
+	iio_push_to_buffers_with_timestamp(indio_dev, &sample,
+					   private->timestamp);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int zpa2326_runtime_suspend(struct device *parent)
+{
+	const struct iio_dev *indio_dev = dev_get_drvdata(parent);
+
+	if (pm_runtime_autosuspend_expiration(parent))
+		/* Userspace changed autosuspend delay. */
+		return -EAGAIN;
+
+	zpa2326_power_off(indio_dev, iio_priv(indio_dev));
+
+	return 0;
+}
+
+static int zpa2326_runtime_resume(struct device *parent)
+{
+	const struct iio_dev *indio_dev = dev_get_drvdata(parent);
+
+	return zpa2326_power_on(indio_dev, iio_priv(indio_dev));
+}
+
+const struct dev_pm_ops zpa2326_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(zpa2326_runtime_suspend, zpa2326_runtime_resume,
+			   NULL)
+};
+EXPORT_SYMBOL_GPL(zpa2326_pm_ops);
+
+/**
+ * zpa2326_resume() - Request the PM layer to power supply the device.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ *
+ * Return:
+ *  < 0 - a negative error code meaning failure ;
+ *    0 - success, device has just been powered up ;
+ *    1 - success, device was already powered.
+ */
+static int zpa2326_resume(const struct iio_dev *indio_dev)
+{
+	int err;
+
+	err = pm_runtime_get_sync(indio_dev->dev.parent);
+	if (err < 0)
+		return err;
+
+	if (err > 0) {
+		/*
+		 * Device was already power supplied: get it out of low power
+		 * mode and inform caller.
+		 */
+		zpa2326_enable_device(indio_dev);
+		return 1;
+	}
+
+	/* Inform caller device has just been brought back to life. */
+	return 0;
+}
+
+/**
+ * zpa2326_suspend() - Schedule a power down using autosuspend feature of PM
+ *                     layer.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ *
+ * Device is switched to low power mode at first to save power even when
+ * attached regulator is a "dummy" one.
+ */
+static void zpa2326_suspend(struct iio_dev *indio_dev)
+{
+	struct device *parent = indio_dev->dev.parent;
+
+	zpa2326_sleep(indio_dev);
+
+	pm_runtime_mark_last_busy(parent);
+	pm_runtime_put_autosuspend(parent);
+}
+
+static void zpa2326_init_runtime(struct device *parent)
+{
+	pm_runtime_get_noresume(parent);
+	pm_runtime_set_active(parent);
+	pm_runtime_enable(parent);
+	pm_runtime_set_autosuspend_delay(parent, 1000);
+	pm_runtime_use_autosuspend(parent);
+	pm_runtime_mark_last_busy(parent);
+	pm_runtime_put_autosuspend(parent);
+}
+
+static void zpa2326_fini_runtime(struct device *parent)
+{
+	pm_runtime_disable(parent);
+	pm_runtime_set_suspended(parent);
+}
+#else /* !CONFIG_PM */
+static int zpa2326_resume(const struct iio_dev *indio_dev)
+{
+	zpa2326_enable_device(indio_dev);
+
+	return 0;
+}
+
+static void zpa2326_suspend(struct iio_dev *indio_dev)
+{
+	zpa2326_sleep(indio_dev);
+}
+
+#define zpa2326_init_runtime(_parent)
+#define zpa2326_fini_runtime(_parent)
+#endif /* !CONFIG_PM */
+
+/**
+ * zpa2326_handle_irq() - Process hardware interrupts.
+ * @irq:  Interrupt line the hardware uses to notify new data has arrived.
+ * @data: The IIO device associated with the sampling hardware.
+ *
+ * Timestamp buffered samples as soon as possible then schedule threaded bottom
+ * half.
+ *
+ * Return: Always successful.
+ */
+static irqreturn_t zpa2326_handle_irq(int irq, void *data)
+{
+	struct iio_dev *indio_dev = (struct iio_dev *)data;
+
+	if (iio_buffer_enabled(indio_dev)) {
+		/* Timestamping needed for buffered sampling only. */
+		((struct zpa2326_private *)
+		 iio_priv(indio_dev))->timestamp = iio_get_time_ns(indio_dev);
+	}
+
+	return IRQ_WAKE_THREAD;
+}
+
+/**
+ * zpa2326_handle_threaded_irq() - Interrupt bottom-half handler.
+ * @irq:  Interrupt line the hardware uses to notify new data has arrived.
+ * @data: The IIO device associated with the sampling hardware.
+ *
+ * Mainly ensures interrupt is caused by a real "new sample available"
+ * condition. This relies upon the ability to perform blocking / sleeping bus
+ * accesses to slave's registers. This is why zpa2326_handle_threaded_irq() is
+ * called from within a thread, i.e. not called from hard interrupt context.
+ *
+ * When device is using its own internal hardware trigger in continuous sampling
+ * mode, data are available into hardware FIFO once interrupt has occurred. All
+ * we have to do is to dispatch the trigger, which in turn will fetch data and
+ * fill IIO buffer.
+ *
+ * When not using its own internal hardware trigger, the device has been
+ * configured in one-shot mode either by an external trigger or the IIO read_raw
+ * hook. This means one of the latter is currently waiting for sampling
+ * completion, in which case we must simply wake it up.
+ *
+ * See zpa2326_trigger_handler().
+ *
+ * Return:
+ *   %IRQ_NONE - no consistent interrupt happened ;
+ *   %IRQ_HANDLED - there was new samples available.
+ */
+static irqreturn_t zpa2326_handle_threaded_irq(int irq, void *data)
+{
+	struct iio_dev         *indio_dev = (struct iio_dev *)data;
+	struct zpa2326_private *priv = iio_priv(indio_dev);
+	unsigned int            val;
+	bool                    cont;
+	irqreturn_t             ret = IRQ_NONE;
+
+	/*
+	 * Are we using our own internal trigger in triggered buffer mode, i.e.,
+	 * currently working in continuous sampling mode ?
+	 */
+	cont = (iio_buffer_enabled(indio_dev) &&
+		iio_trigger_using_own(indio_dev));
+
+	/*
+	 * Device works according to a level interrupt scheme: reading interrupt
+	 * status de-asserts interrupt line.
+	 */
+	priv->result = regmap_read(priv->regmap, ZPA2326_INT_SOURCE_REG, &val);
+	if (priv->result < 0) {
+		if (cont)
+			return IRQ_NONE;
+
+		goto complete;
+	}
+
+	/* Data ready is the only interrupt source we requested. */
+	if (!(val & ZPA2326_INT_SOURCE_DATA_READY)) {
+		/*
+		 * Interrupt happened but no new sample available: likely caused
+		 * by spurious interrupts, in which case, returning IRQ_NONE
+		 * allows to benefit from the generic spurious interrupts
+		 * handling.
+		 */
+		zpa2326_warn(indio_dev, "unexpected interrupt status %02x",
+			     val);
+
+		if (cont)
+			return IRQ_NONE;
+
+		priv->result = -ENODATA;
+		goto complete;
+	}
+
+	/* New sample available: dispatch internal trigger consumers. */
+	iio_trigger_poll_chained(priv->trigger);
+
+	if (cont)
+		/*
+		 * Internal hardware trigger has been scheduled above : it will
+		 * fetch data on its own.
+		 */
+		return IRQ_HANDLED;
+
+	ret = IRQ_HANDLED;
+
+complete:
+	/*
+	 * Wake up direct or externaly triggered buffer mode waiters: see
+	 * zpa2326_sample_oneshot() and zpa2326_trigger_handler().
+	 */
+	complete(&priv->data_ready);
+
+	return ret;
+}
+
+/**
+ * zpa2326_wait_oneshot_completion() - Wait for oneshot data ready interrupt.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @private:   Internal private state related to @indio_dev.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_wait_oneshot_completion(const struct iio_dev   *indio_dev,
+					   struct zpa2326_private *private)
+{
+	int          ret;
+	unsigned int val;
+
+	zpa2326_dbg(indio_dev, "waiting for one shot completion interrupt");
+
+	ret = wait_for_completion_interruptible_timeout(
+		&private->data_ready, ZPA2326_CONVERSION_JIFFIES);
+	if (ret > 0)
+		/*
+		 * Interrupt handler completed before timeout: return operation
+		 * status.
+		 */
+		return private->result;
+
+	/* Clear all interrupts just to be sure. */
+	regmap_read(private->regmap, ZPA2326_INT_SOURCE_REG, &val);
+
+	if (!ret)
+		/* Timed out. */
+		ret = -ETIME;
+
+	if (ret != -ERESTARTSYS)
+		zpa2326_warn(indio_dev, "no one shot interrupt occurred (%d)",
+			     ret);
+
+	return ret;
+}
+
+static int zpa2326_init_managed_irq(struct device          *parent,
+				    struct iio_dev         *indio_dev,
+				    struct zpa2326_private *private,
+				    int                     irq)
+{
+	int err;
+
+	private->irq = irq;
+
+	if (irq <= 0) {
+		/*
+		 * Platform declared no interrupt line: device will be polled
+		 * for data availability.
+		 */
+		dev_info(parent, "no interrupt found, running in polling mode");
+		return 0;
+	}
+
+	init_completion(&private->data_ready);
+
+	/* Request handler to be scheduled into threaded interrupt context. */
+	err = devm_request_threaded_irq(parent, irq, zpa2326_handle_irq,
+					zpa2326_handle_threaded_irq,
+					IRQF_TRIGGER_RISING | IRQF_ONESHOT,
+					dev_name(parent), indio_dev);
+	if (err) {
+		dev_err(parent, "failed to request interrupt %d (%d)", irq,
+			err);
+		return err;
+	}
+
+	dev_info(parent, "using interrupt %d", irq);
+
+	return 0;
+}
+
+/**
+ * zpa2326_poll_oneshot_completion() - Actively poll for one shot data ready.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ *
+ * Loop over registers content to detect end of sampling cycle. Used when DT
+ * declared no valid interrupt lines.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_poll_oneshot_completion(const struct iio_dev *indio_dev)
+{
+	unsigned long  tmout = jiffies + ZPA2326_CONVERSION_JIFFIES;
+	struct regmap *regs = ((struct zpa2326_private *)
+			       iio_priv(indio_dev))->regmap;
+	unsigned int   val;
+	int            err;
+
+	zpa2326_dbg(indio_dev, "polling for one shot completion");
+
+	/*
+	 * At least, 100 ms is needed for the device to complete its one-shot
+	 * cycle.
+	 */
+	if (msleep_interruptible(100))
+		return -ERESTARTSYS;
+
+	/* Poll for conversion completion in hardware. */
+	while (true) {
+		err = regmap_read(regs, ZPA2326_CTRL_REG0_REG, &val);
+		if (err < 0)
+			goto err;
+
+		if (!(val & ZPA2326_CTRL_REG0_ONE_SHOT))
+			/* One-shot bit self clears at conversion end. */
+			break;
+
+		if (time_after(jiffies, tmout)) {
+			/* Prevent from waiting forever : let's time out. */
+			err = -ETIME;
+			goto err;
+		}
+
+		usleep_range(10000, 20000);
+	}
+
+	/*
+	 * In oneshot mode, pressure sample availability guarantees that
+	 * temperature conversion has also completed : just check pressure
+	 * status bit to keep things simple.
+	 */
+	err = regmap_read(regs, ZPA2326_STATUS_REG, &val);
+	if (err < 0)
+		goto err;
+
+	if (!(val & ZPA2326_STATUS_P_DA)) {
+		/* No sample available. */
+		err = -ENODATA;
+		goto err;
+	}
+
+	return 0;
+
+err:
+	zpa2326_warn(indio_dev, "failed to poll one shot completion (%d)", err);
+
+	return err;
+}
+
+/**
+ * zpa2326_fetch_raw_sample() - Retrieve a raw sample and convert it to CPU
+ *                              endianness.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @type:      Type of measurement / channel to fetch from.
+ * @value:     Sample output.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_fetch_raw_sample(const struct iio_dev *indio_dev,
+				    enum iio_chan_type    type,
+				    int                  *value)
+{
+	struct regmap *regs = ((struct zpa2326_private *)
+			       iio_priv(indio_dev))->regmap;
+	int            err;
+
+	switch (type) {
+	case IIO_PRESSURE:
+		zpa2326_dbg(indio_dev, "fetching raw pressure sample");
+
+		err = regmap_bulk_read(regs, ZPA2326_PRESS_OUT_XL_REG, value,
+				       3);
+		if (err) {
+			zpa2326_warn(indio_dev, "failed to fetch pressure (%d)",
+				     err);
+			return err;
+		}
+
+		/* Pressure is a 24 bits wide little-endian unsigned int. */
+		*value = (((u8 *)value)[2] << 16) | (((u8 *)value)[1] << 8) |
+			 ((u8 *)value)[0];
+
+		return IIO_VAL_INT;
+
+	case IIO_TEMP:
+		zpa2326_dbg(indio_dev, "fetching raw temperature sample");
+
+		err = regmap_bulk_read(regs, ZPA2326_TEMP_OUT_L_REG, value, 2);
+		if (err) {
+			zpa2326_warn(indio_dev,
+				     "failed to fetch temperature (%d)", err);
+			return err;
+		}
+
+		/* Temperature is a 16 bits wide little-endian signed int. */
+		*value = (int)le16_to_cpup((__le16 *)value);
+
+		return IIO_VAL_INT;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+/**
+ * zpa2326_sample_oneshot() - Perform a complete one shot sampling cycle.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @type:      Type of measurement / channel to fetch from.
+ * @value:     Sample output.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_sample_oneshot(struct iio_dev     *indio_dev,
+				  enum iio_chan_type  type,
+				  int                *value)
+{
+	int                     ret;
+	struct zpa2326_private *priv;
+
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret)
+		return ret;
+
+	ret = zpa2326_resume(indio_dev);
+	if (ret < 0)
+		goto release;
+
+	priv = iio_priv(indio_dev);
+
+	if (ret > 0) {
+		/*
+		 * We were already power supplied. Just clear hardware FIFO to
+		 * get rid of samples acquired during previous rounds (if any).
+		 * Sampling operation always generates both temperature and
+		 * pressure samples. The latter are always enqueued into
+		 * hardware FIFO. This may lead to situations were pressure
+		 * samples still sit into FIFO when previous cycle(s) fetched
+		 * temperature data only.
+		 * Hence, we need to clear hardware FIFO content to prevent from
+		 * getting outdated values at the end of current cycle.
+		 */
+		if (type == IIO_PRESSURE) {
+			ret = zpa2326_clear_fifo(indio_dev, 0);
+			if (ret)
+				goto suspend;
+		}
+	} else {
+		/*
+		 * We have just been power supplied, i.e. device is in default
+		 * "out of reset" state, meaning we need to reconfigure it
+		 * entirely.
+		 */
+		ret = zpa2326_config_oneshot(indio_dev, priv->irq);
+		if (ret)
+			goto suspend;
+	}
+
+	/* Start a sampling cycle in oneshot mode. */
+	ret = zpa2326_start_oneshot(indio_dev);
+	if (ret)
+		goto suspend;
+
+	/* Wait for sampling cycle to complete. */
+	if (priv->irq > 0)
+		ret = zpa2326_wait_oneshot_completion(indio_dev, priv);
+	else
+		ret = zpa2326_poll_oneshot_completion(indio_dev);
+
+	if (ret)
+		goto suspend;
+
+	/* Retrieve raw sample value and convert it to CPU endianness. */
+	ret = zpa2326_fetch_raw_sample(indio_dev, type, value);
+
+suspend:
+	zpa2326_suspend(indio_dev);
+release:
+	iio_device_release_direct_mode(indio_dev);
+
+	return ret;
+}
+
+/**
+ * zpa2326_trigger_handler() - Perform an IIO buffered sampling round in one
+ *                             shot mode.
+ * @irq:  The software interrupt assigned to @data
+ * @data: The IIO poll function dispatched by external trigger our device is
+ *        attached to.
+ *
+ * Bottom-half handler called by the IIO trigger to which our device is
+ * currently attached. Allows us to synchronize this device buffered sampling
+ * either with external events (such as timer expiration, external device sample
+ * ready, etc...) or with its own interrupt (internal hardware trigger).
+ *
+ * When using an external trigger, basically run the same sequence of operations
+ * as for zpa2326_sample_oneshot() with the following hereafter. Hardware FIFO
+ * is not cleared since already done at buffering enable time and samples
+ * dequeueing always retrieves the most recent value.
+ *
+ * Otherwise, when internal hardware trigger has dispatched us, just fetch data
+ * from hardware FIFO.
+ *
+ * Fetched data will pushed unprocessed to IIO buffer since samples conversion
+ * is delegated to userspace in buffered mode (endianness, etc...).
+ *
+ * Return:
+ *   %IRQ_NONE - no consistent interrupt happened ;
+ *   %IRQ_HANDLED - there was new samples available.
+ */
+static irqreturn_t zpa2326_trigger_handler(int irq, void *data)
+{
+	struct iio_dev         *indio_dev = ((struct iio_poll_func *)
+					     data)->indio_dev;
+	struct zpa2326_private *priv = iio_priv(indio_dev);
+	bool                    cont;
+
+	/*
+	 * We have been dispatched, meaning we are in triggered buffer mode.
+	 * Using our own internal trigger implies we are currently in continuous
+	 * hardware sampling mode.
+	 */
+	cont = iio_trigger_using_own(indio_dev);
+
+	if (!cont) {
+		/* On demand sampling : start a one shot cycle. */
+		if (zpa2326_start_oneshot(indio_dev))
+			goto out;
+
+		/* Wait for sampling cycle to complete. */
+		if (priv->irq <= 0) {
+			/* No interrupt available: poll for completion. */
+			if (zpa2326_poll_oneshot_completion(indio_dev))
+				goto out;
+
+			/* Only timestamp sample once it is ready. */
+			priv->timestamp = iio_get_time_ns(indio_dev);
+		} else {
+			/* Interrupt handlers will timestamp for us. */
+			if (zpa2326_wait_oneshot_completion(indio_dev, priv))
+				goto out;
+		}
+	}
+
+	/* Enqueue to IIO buffer / userspace. */
+	zpa2326_fill_sample_buffer(indio_dev, priv);
+
+out:
+	if (!cont)
+		/* Don't switch to low power if sampling continuously. */
+		zpa2326_sleep(indio_dev);
+
+	/* Inform attached trigger we are done. */
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * zpa2326_preenable_buffer() - Prepare device for configuring triggered
+ *                              sampling
+ * modes.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ *
+ * Basically power up device.
+ * Called with IIO device's lock held.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_preenable_buffer(struct iio_dev *indio_dev)
+{
+	int ret = zpa2326_resume(indio_dev);
+
+	if (ret < 0)
+		return ret;
+
+	/* Tell zpa2326_postenable_buffer() if we have just been powered on. */
+	((struct zpa2326_private *)
+	 iio_priv(indio_dev))->waken = iio_priv(indio_dev);
+
+	return 0;
+}
+
+/**
+ * zpa2326_postenable_buffer() - Configure device for triggered sampling.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ *
+ * Basically setup one-shot mode if plugging external trigger.
+ * Otherwise, let internal trigger configure continuous sampling :
+ * see zpa2326_set_trigger_state().
+ *
+ * If an error is returned, IIO layer will call our postdisable hook for us,
+ * i.e. no need to explicitly power device off here.
+ * Called with IIO device's lock held.
+ *
+ * Called with IIO device's lock held.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_postenable_buffer(struct iio_dev *indio_dev)
+{
+	const struct zpa2326_private *priv = iio_priv(indio_dev);
+	int                           err;
+
+	if (!priv->waken) {
+		/*
+		 * We were already power supplied. Just clear hardware FIFO to
+		 * get rid of samples acquired during previous rounds (if any).
+		 */
+		err = zpa2326_clear_fifo(indio_dev, 0);
+		if (err)
+			goto err;
+	}
+
+	if (!iio_trigger_using_own(indio_dev) && priv->waken) {
+		/*
+		 * We are using an external trigger and we have just been
+		 * powered up: reconfigure one-shot mode.
+		 */
+		err = zpa2326_config_oneshot(indio_dev, priv->irq);
+		if (err)
+			goto err;
+	}
+
+	/* Plug our own trigger event handler. */
+	err = iio_triggered_buffer_postenable(indio_dev);
+	if (err)
+		goto err;
+
+	return 0;
+
+err:
+	zpa2326_err(indio_dev, "failed to enable buffering (%d)", err);
+
+	return err;
+}
+
+static int zpa2326_postdisable_buffer(struct iio_dev *indio_dev)
+{
+	zpa2326_suspend(indio_dev);
+
+	return 0;
+}
+
+static const struct iio_buffer_setup_ops zpa2326_buffer_setup_ops = {
+	.preenable   = zpa2326_preenable_buffer,
+	.postenable  = zpa2326_postenable_buffer,
+	.predisable  = iio_triggered_buffer_predisable,
+	.postdisable = zpa2326_postdisable_buffer
+};
+
+/**
+ * zpa2326_set_trigger_state() - Start / stop continuous sampling.
+ * @trig:  The trigger being attached to IIO device associated with the sampling
+ *         hardware.
+ * @state: Tell whether to start (true) or stop (false)
+ *
+ * Basically enable / disable hardware continuous sampling mode.
+ *
+ * Called with IIO device's lock held at postenable() or predisable() time.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_set_trigger_state(struct iio_trigger *trig, bool state)
+{
+	const struct iio_dev         *indio_dev = dev_get_drvdata(
+							trig->dev.parent);
+	const struct zpa2326_private *priv = iio_priv(indio_dev);
+	int                           err;
+
+	if (!state) {
+		/*
+		 * Switch trigger off : in case of failure, interrupt is left
+		 * disabled in order to prevent handler from accessing released
+		 * resources.
+		 */
+		unsigned int val;
+
+		/*
+		 * As device is working in continuous mode, handlers may be
+		 * accessing resources we are currently freeing...
+		 * Prevent this by disabling interrupt handlers and ensure
+		 * the device will generate no more interrupts unless explicitly
+		 * required to, i.e. by restoring back to default one shot mode.
+		 */
+		disable_irq(priv->irq);
+
+		/*
+		 * Disable continuous sampling mode to restore settings for
+		 * one shot / direct sampling operations.
+		 */
+		err = regmap_write(priv->regmap, ZPA2326_CTRL_REG3_REG,
+				   zpa2326_highest_frequency()->odr);
+		if (err)
+			return err;
+
+		/*
+		 * Now that device won't generate interrupts on its own,
+		 * acknowledge any currently active interrupts (may happen on
+		 * rare occasions while stopping continuous mode).
+		 */
+		err = regmap_read(priv->regmap, ZPA2326_INT_SOURCE_REG, &val);
+		if (err < 0)
+			return err;
+
+		/*
+		 * Re-enable interrupts only if we can guarantee the device will
+		 * generate no more interrupts to prevent handlers from
+		 * accessing released resources.
+		 */
+		enable_irq(priv->irq);
+
+		zpa2326_dbg(indio_dev, "continuous mode stopped");
+	} else {
+		/*
+		 * Switch trigger on : start continuous sampling at required
+		 * frequency.
+		 */
+
+		if (priv->waken) {
+			/* Enable interrupt if getting out of reset. */
+			err = regmap_write(priv->regmap, ZPA2326_CTRL_REG1_REG,
+					   (u8)
+					   ~ZPA2326_CTRL_REG1_MASK_DATA_READY);
+			if (err)
+				return err;
+		}
+
+		/* Enable continuous sampling at specified frequency. */
+		err = regmap_write(priv->regmap, ZPA2326_CTRL_REG3_REG,
+				   ZPA2326_CTRL_REG3_ENABLE_MEAS |
+				   priv->frequency->odr);
+		if (err)
+			return err;
+
+		zpa2326_dbg(indio_dev, "continuous mode setup @%dHz",
+			    priv->frequency->hz);
+	}
+
+	return 0;
+}
+
+static const struct iio_trigger_ops zpa2326_trigger_ops = {
+	.owner             = THIS_MODULE,
+	.set_trigger_state = zpa2326_set_trigger_state,
+};
+
+/**
+ * zpa2326_init_trigger() - Create an interrupt driven / hardware trigger
+ *                          allowing to notify external devices a new sample is
+ *                          ready.
+ * @parent:    Hardware sampling device @indio_dev is a child of.
+ * @indio_dev: The IIO device associated with the sampling hardware.
+ * @private:   Internal private state related to @indio_dev.
+ * @irq:       Optional interrupt line the hardware uses to notify new data
+ *             samples are ready. Negative or zero values indicate no interrupts
+ *             are available, meaning polling is required.
+ *
+ * Only relevant when DT declares a valid interrupt line.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+static int zpa2326_init_managed_trigger(struct device          *parent,
+					struct iio_dev         *indio_dev,
+					struct zpa2326_private *private,
+					int                     irq)
+{
+	struct iio_trigger *trigger;
+	int                 ret;
+
+	if (irq <= 0)
+		return 0;
+
+	trigger = devm_iio_trigger_alloc(parent, "%s-dev%d",
+					 indio_dev->name, indio_dev->id);
+	if (!trigger)
+		return -ENOMEM;
+
+	/* Basic setup. */
+	trigger->dev.parent = parent;
+	trigger->ops = &zpa2326_trigger_ops;
+
+	private->trigger = trigger;
+
+	/* Register to triggers space. */
+	ret = devm_iio_trigger_register(parent, trigger);
+	if (ret)
+		dev_err(parent, "failed to register hardware trigger (%d)",
+			ret);
+
+	return ret;
+}
+
+static int zpa2326_get_frequency(const struct iio_dev *indio_dev)
+{
+	return ((struct zpa2326_private *)iio_priv(indio_dev))->frequency->hz;
+}
+
+static int zpa2326_set_frequency(struct iio_dev *indio_dev, int hz)
+{
+	struct zpa2326_private *priv = iio_priv(indio_dev);
+	int                     freq;
+	int                     err;
+
+	/* Check if requested frequency is supported. */
+	for (freq = 0; freq < ARRAY_SIZE(zpa2326_sampling_frequencies); freq++)
+		if (zpa2326_sampling_frequencies[freq].hz == hz)
+			break;
+	if (freq == ARRAY_SIZE(zpa2326_sampling_frequencies))
+		return -EINVAL;
+
+	/* Don't allow changing frequency if buffered sampling is ongoing. */
+	err = iio_device_claim_direct_mode(indio_dev);
+	if (err)
+		return err;
+
+	priv->frequency = &zpa2326_sampling_frequencies[freq];
+
+	iio_device_release_direct_mode(indio_dev);
+
+	return 0;
+}
+
+/* Expose supported hardware sampling frequencies (Hz) through sysfs. */
+static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("1 5 11 23");
+
+static struct attribute *zpa2326_attributes[] = {
+	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
+	NULL
+};
+
+static const struct attribute_group zpa2326_attribute_group = {
+	.attrs = zpa2326_attributes,
+};
+
+static int zpa2326_read_raw(struct iio_dev             *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int                        *val,
+			    int                        *val2,
+			    long                        mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		return zpa2326_sample_oneshot(indio_dev, chan->type, val);
+
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_PRESSURE:
+			/*
+			 * Pressure resolution is 1/64 Pascal. Scale to kPascal
+			 * as required by IIO ABI.
+			 */
+			*val = 1;
+			*val2 = 64000;
+			return IIO_VAL_FRACTIONAL;
+
+		case IIO_TEMP:
+			/*
+			 * Temperature follows the equation:
+			 *     Temp[degC] = Tempcode * 0.00649 - 176.83
+			 * where:
+			 *     Tempcode is composed the raw sampled 16 bits.
+			 *
+			 * Hence, to produce a temperature in milli-degrees
+			 * Celsius according to IIO ABI, we need to apply the
+			 * following equation to raw samples:
+			 *     Temp[milli degC] = (Tempcode + Offset) * Scale
+			 * where:
+			 *     Offset = -176.83 / 0.00649
+			 *     Scale = 0.00649 * 1000
+			 */
+			*val = 6;
+			*val2 = 490000;
+			return IIO_VAL_INT_PLUS_MICRO;
+
+		default:
+			return -EINVAL;
+		}
+
+	case IIO_CHAN_INFO_OFFSET:
+		switch (chan->type) {
+		case IIO_TEMP:
+			*val = -17683000;
+			*val2 = 649;
+			return IIO_VAL_FRACTIONAL;
+
+		default:
+			return -EINVAL;
+		}
+
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = zpa2326_get_frequency(indio_dev);
+		return IIO_VAL_INT;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int zpa2326_write_raw(struct iio_dev             *indio_dev,
+			     const struct iio_chan_spec *chan,
+			     int                         val,
+			     int                         val2,
+			     long                        mask)
+{
+	if ((mask != IIO_CHAN_INFO_SAMP_FREQ) || val2)
+		return -EINVAL;
+
+	return zpa2326_set_frequency(indio_dev, val);
+}
+
+static const struct iio_chan_spec zpa2326_channels[] = {
+	[0] = {
+		.type                    = IIO_PRESSURE,
+		.scan_index              = 0,
+		.scan_type               = {
+			.sign                   = 'u',
+			.realbits               = 24,
+			.storagebits            = 32,
+			.endianness             = IIO_LE,
+		},
+		.info_mask_separate      = BIT(IIO_CHAN_INFO_RAW) |
+					   BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+	},
+	[1] = {
+		.type                    = IIO_TEMP,
+		.scan_index              = 1,
+		.scan_type               = {
+			.sign                   = 's',
+			.realbits               = 16,
+			.storagebits            = 16,
+			.endianness             = IIO_LE,
+		},
+		.info_mask_separate      = BIT(IIO_CHAN_INFO_RAW) |
+					   BIT(IIO_CHAN_INFO_SCALE) |
+					   BIT(IIO_CHAN_INFO_OFFSET),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+	},
+	[2] = IIO_CHAN_SOFT_TIMESTAMP(2),
+};
+
+static const struct iio_info zpa2326_info = {
+	.driver_module = THIS_MODULE,
+	.attrs         = &zpa2326_attribute_group,
+	.read_raw      = zpa2326_read_raw,
+	.write_raw     = zpa2326_write_raw,
+};
+
+static struct iio_dev *zpa2326_create_managed_iiodev(struct device *device,
+						     const char    *name,
+						     struct regmap *regmap)
+{
+	struct iio_dev *indio_dev;
+
+	/* Allocate space to hold IIO device internal state. */
+	indio_dev = devm_iio_device_alloc(device,
+					  sizeof(struct zpa2326_private));
+	if (!indio_dev)
+		return NULL;
+
+	/* Setup for userspace synchronous on demand sampling. */
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->dev.parent = device;
+	indio_dev->channels = zpa2326_channels;
+	indio_dev->num_channels = ARRAY_SIZE(zpa2326_channels);
+	indio_dev->name = name;
+	indio_dev->info = &zpa2326_info;
+
+	return indio_dev;
+}
+
+int zpa2326_probe(struct device *parent,
+		  const char    *name,
+		  int            irq,
+		  unsigned int   hwid,
+		  struct regmap *regmap)
+{
+	struct iio_dev         *indio_dev;
+	struct zpa2326_private *priv;
+	int                     err;
+	unsigned int            id;
+
+	indio_dev = zpa2326_create_managed_iiodev(parent, name, regmap);
+	if (!indio_dev)
+		return -ENOMEM;
+
+	priv = iio_priv(indio_dev);
+
+	priv->vref = devm_regulator_get(parent, "vref");
+	if (IS_ERR(priv->vref))
+		return PTR_ERR(priv->vref);
+
+	priv->vdd = devm_regulator_get(parent, "vdd");
+	if (IS_ERR(priv->vdd))
+		return PTR_ERR(priv->vdd);
+
+	/* Set default hardware sampling frequency to highest rate supported. */
+	priv->frequency = zpa2326_highest_frequency();
+
+	/*
+	 * Plug device's underlying bus abstraction : this MUST be set before
+	 * registering interrupt handlers since an interrupt might happen if
+	 * power up sequence is not properly applied.
+	 */
+	priv->regmap = regmap;
+
+	err = devm_iio_triggered_buffer_setup(parent, indio_dev, NULL,
+					      zpa2326_trigger_handler,
+					      &zpa2326_buffer_setup_ops);
+	if (err)
+		return err;
+
+	err = zpa2326_init_managed_trigger(parent, indio_dev, priv, irq);
+	if (err)
+		return err;
+
+	err = zpa2326_init_managed_irq(parent, indio_dev, priv, irq);
+	if (err)
+		return err;
+
+	/* Power up to check device ID and perform initial hardware setup. */
+	err = zpa2326_power_on(indio_dev, priv);
+	if (err)
+		return err;
+
+	/* Read id register to check we are talking to the right slave. */
+	err = regmap_read(regmap, ZPA2326_DEVICE_ID_REG, &id);
+	if (err)
+		goto sleep;
+
+	if (id != hwid) {
+		dev_err(parent, "found device with unexpected id %02x", id);
+		err = -ENODEV;
+		goto sleep;
+	}
+
+	err = zpa2326_config_oneshot(indio_dev, irq);
+	if (err)
+		goto sleep;
+
+	/* Setup done : go sleeping. Device will be awaken upon user request. */
+	err = zpa2326_sleep(indio_dev);
+	if (err)
+		goto poweroff;
+
+	dev_set_drvdata(parent, indio_dev);
+
+	zpa2326_init_runtime(parent);
+
+	err = iio_device_register(indio_dev);
+	if (err) {
+		zpa2326_fini_runtime(parent);
+		goto poweroff;
+	}
+
+	return 0;
+
+sleep:
+	/* Put to sleep just in case power regulators are "dummy" ones. */
+	zpa2326_sleep(indio_dev);
+poweroff:
+	zpa2326_power_off(indio_dev, priv);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(zpa2326_probe);
+
+void zpa2326_remove(const struct device *parent)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(parent);
+
+	iio_device_unregister(indio_dev);
+	zpa2326_fini_runtime(indio_dev->dev.parent);
+	zpa2326_sleep(indio_dev);
+	zpa2326_power_off(indio_dev, iio_priv(indio_dev));
+}
+EXPORT_SYMBOL_GPL(zpa2326_remove);
+
+MODULE_AUTHOR("Gregor Boirie <gregor.boirie@parrot.com>");
+MODULE_DESCRIPTION("Core driver for Murata ZPA2326 pressure sensor");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/pressure/zpa2326.h b/drivers/iio/pressure/zpa2326.h
new file mode 100644
index 000000000000..05d3e1e3a449
--- /dev/null
+++ b/drivers/iio/pressure/zpa2326.h
@@ -0,0 +1,89 @@
+/*
+ * Murata ZPA2326 pressure and temperature sensor IIO driver
+ *
+ * Copyright (c) 2016 Parrot S.A.
+ *
+ * Author: Gregor Boirie <gregor.boirie@parrot.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _ZPA2326_H
+#define _ZPA2326_H
+
+/* Register map. */
+#define ZPA2326_REF_P_XL_REG              (0x8)
+#define ZPA2326_REF_P_L_REG               (0x9)
+#define ZPA2326_REF_P_H_REG               (0xa)
+#define ZPA2326_DEVICE_ID_REG             (0xf)
+#define ZPA2326_DEVICE_ID                 (0xb9)
+#define ZPA2326_RES_CONF_REG              (0x10)
+#define ZPA2326_CTRL_REG0_REG             (0x20)
+#define ZPA2326_CTRL_REG0_ONE_SHOT        BIT(0)
+#define ZPA2326_CTRL_REG0_ENABLE          BIT(1)
+#define ZPA2326_CTRL_REG1_REG             (0x21)
+#define ZPA2326_CTRL_REG1_MASK_DATA_READY BIT(2)
+#define ZPA2326_CTRL_REG2_REG             (0x22)
+#define ZPA2326_CTRL_REG2_SWRESET         BIT(2)
+#define ZPA2326_CTRL_REG3_REG             (0x23)
+#define ZPA2326_CTRL_REG3_ODR_SHIFT       (4)
+#define ZPA2326_CTRL_REG3_ENABLE_MEAS     BIT(7)
+#define ZPA2326_INT_SOURCE_REG            (0x24)
+#define ZPA2326_INT_SOURCE_DATA_READY     BIT(2)
+#define ZPA2326_THS_P_LOW_REG             (0x25)
+#define ZPA2326_THS_P_HIGH_REG            (0x26)
+#define ZPA2326_STATUS_REG                (0x27)
+#define ZPA2326_STATUS_P_DA               BIT(1)
+#define ZPA2326_STATUS_FIFO_E             BIT(2)
+#define ZPA2326_STATUS_P_OR               BIT(5)
+#define ZPA2326_PRESS_OUT_XL_REG          (0x28)
+#define ZPA2326_PRESS_OUT_L_REG           (0x29)
+#define ZPA2326_PRESS_OUT_H_REG           (0x2a)
+#define ZPA2326_TEMP_OUT_L_REG            (0x2b)
+#define ZPA2326_TEMP_OUT_H_REG            (0x2c)
+
+struct device;
+struct regmap;
+
+bool zpa2326_isreg_writeable(struct device *dev, unsigned int reg);
+bool zpa2326_isreg_readable(struct device *dev, unsigned int reg);
+bool zpa2326_isreg_precious(struct device *dev, unsigned int reg);
+
+/**
+ * zpa2326_probe() - Instantiate and register core ZPA2326 IIO device
+ * @parent: Hardware sampling device the created IIO device will be a child of.
+ * @name:   Arbitrary name to identify the device.
+ * @irq:    Interrupt line, negative if none.
+ * @hwid:   Expected device hardware id.
+ * @regmap: Registers map used to abstract underlying bus accesses.
+ *
+ * Return: Zero when successful, a negative error code otherwise.
+ */
+int zpa2326_probe(struct device        *parent,
+		  const char           *name,
+		  int                   irq,
+		  unsigned int          hwid,
+		  struct regmap        *regmap);
+
+/**
+ * zpa2326_remove() - Unregister and destroy core ZPA2326 IIO device.
+ * @parent: Hardware sampling device the IIO device to remove is a child of.
+ */
+void zpa2326_remove(const struct device *parent);
+
+#ifdef CONFIG_PM
+#include <linux/pm.h>
+extern const struct dev_pm_ops zpa2326_pm_ops;
+#define ZPA2326_PM_OPS (&zpa2326_pm_ops)
+#else
+#define ZPA2326_PM_OPS (NULL)
+#endif
+
+#endif
diff --git a/drivers/iio/pressure/zpa2326_i2c.c b/drivers/iio/pressure/zpa2326_i2c.c
new file mode 100644
index 000000000000..e4d27dd4493a
--- /dev/null
+++ b/drivers/iio/pressure/zpa2326_i2c.c
@@ -0,0 +1,99 @@
+/*
+ * Murata ZPA2326 I2C pressure and temperature sensor driver
+ *
+ * Copyright (c) 2016 Parrot S.A.
+ *
+ * Author: Gregor Boirie <gregor.boirie@parrot.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/i2c.h>
+#include <linux/of_device.h>
+#include "zpa2326.h"
+
+/*
+ * read_flag_mask:
+ *   - address bit 7 must be set to request a register read operation
+ */
+static const struct regmap_config zpa2326_regmap_i2c_config = {
+	.reg_bits       = 8,
+	.val_bits       = 8,
+	.writeable_reg  = zpa2326_isreg_writeable,
+	.readable_reg   = zpa2326_isreg_readable,
+	.precious_reg   = zpa2326_isreg_precious,
+	.max_register   = ZPA2326_TEMP_OUT_H_REG,
+	.read_flag_mask = BIT(7),
+	.cache_type     = REGCACHE_NONE,
+};
+
+static unsigned int zpa2326_i2c_hwid(const struct i2c_client *client)
+{
+#define ZPA2326_SA0(_addr)          (_addr & BIT(0))
+#define ZPA2326_DEVICE_ID_SA0_SHIFT (1)
+
+	/* Identification register bit 1 mirrors device address bit 0. */
+	return (ZPA2326_DEVICE_ID |
+		(ZPA2326_SA0(client->addr) << ZPA2326_DEVICE_ID_SA0_SHIFT));
+}
+
+static int zpa2326_probe_i2c(struct i2c_client          *client,
+			     const struct i2c_device_id *i2c_id)
+{
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init_i2c(client, &zpa2326_regmap_i2c_config);
+	if (IS_ERR(regmap)) {
+		dev_err(&client->dev, "failed to init registers map");
+		return PTR_ERR(regmap);
+	}
+
+	return zpa2326_probe(&client->dev, i2c_id->name, client->irq,
+			     zpa2326_i2c_hwid(client), regmap);
+}
+
+static int zpa2326_remove_i2c(struct i2c_client *client)
+{
+	zpa2326_remove(&client->dev);
+
+	return 0;
+}
+
+static const struct i2c_device_id zpa2326_i2c_ids[] = {
+	{ "zpa2326", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(i2c, zpa2326_i2c_ids);
+
+#if defined(CONFIG_OF)
+static const struct of_device_id zpa2326_i2c_matches[] = {
+	{ .compatible = "murata,zpa2326" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, zpa2326_i2c_matches);
+#endif
+
+static struct i2c_driver zpa2326_i2c_driver = {
+	.driver = {
+		.name           = "zpa2326-i2c",
+		.of_match_table = of_match_ptr(zpa2326_i2c_matches),
+		.pm             = ZPA2326_PM_OPS,
+	},
+	.probe    = zpa2326_probe_i2c,
+	.remove   = zpa2326_remove_i2c,
+	.id_table = zpa2326_i2c_ids,
+};
+module_i2c_driver(zpa2326_i2c_driver);
+
+MODULE_AUTHOR("Gregor Boirie <gregor.boirie@parrot.com>");
+MODULE_DESCRIPTION("I2C driver for Murata ZPA2326 pressure sensor");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/pressure/zpa2326_spi.c b/drivers/iio/pressure/zpa2326_spi.c
new file mode 100644
index 000000000000..bd2c1c319fca
--- /dev/null
+++ b/drivers/iio/pressure/zpa2326_spi.c
@@ -0,0 +1,103 @@
+/*
+ * Murata ZPA2326 SPI pressure and temperature sensor driver
+ *
+ * Copyright (c) 2016 Parrot S.A.
+ *
+ * Author: Gregor Boirie <gregor.boirie@parrot.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+#include <linux/of_device.h>
+#include "zpa2326.h"
+
+/*
+ * read_flag_mask:
+ *   - address bit 7 must be set to request a register read operation
+ *   - address bit 6 must be set to request register address auto increment
+ */
+static const struct regmap_config zpa2326_regmap_spi_config = {
+	.reg_bits       = 8,
+	.val_bits       = 8,
+	.writeable_reg  = zpa2326_isreg_writeable,
+	.readable_reg   = zpa2326_isreg_readable,
+	.precious_reg   = zpa2326_isreg_precious,
+	.max_register   = ZPA2326_TEMP_OUT_H_REG,
+	.read_flag_mask = BIT(7) | BIT(6),
+	.cache_type     = REGCACHE_NONE,
+};
+
+static int zpa2326_probe_spi(struct spi_device *spi)
+{
+	struct regmap *regmap;
+	int            err;
+
+	regmap = devm_regmap_init_spi(spi, &zpa2326_regmap_spi_config);
+	if (IS_ERR(regmap)) {
+		dev_err(&spi->dev, "failed to init registers map");
+		return PTR_ERR(regmap);
+	}
+
+	/*
+	 * Enforce SPI slave settings to prevent from DT misconfiguration.
+	 *
+	 * Clock is idle high. Sampling happens on trailing edge, i.e., rising
+	 * edge. Maximum bus frequency is 1 MHz. Registers are 8 bits wide.
+	 */
+	spi->mode = SPI_MODE_3;
+	spi->max_speed_hz = min(spi->max_speed_hz, 1000000U);
+	spi->bits_per_word = 8;
+	err = spi_setup(spi);
+	if (err < 0)
+		return err;
+
+	return zpa2326_probe(&spi->dev, spi_get_device_id(spi)->name,
+			     spi->irq, ZPA2326_DEVICE_ID, regmap);
+}
+
+static int zpa2326_remove_spi(struct spi_device *spi)
+{
+	zpa2326_remove(&spi->dev);
+
+	return 0;
+}
+
+static const struct spi_device_id zpa2326_spi_ids[] = {
+	{ "zpa2326", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(spi, zpa2326_spi_ids);
+
+#if defined(CONFIG_OF)
+static const struct of_device_id zpa2326_spi_matches[] = {
+	{ .compatible = "murata,zpa2326" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, zpa2326_spi_matches);
+#endif
+
+static struct spi_driver zpa2326_spi_driver = {
+	.driver = {
+		.name           = "zpa2326-spi",
+		.of_match_table = of_match_ptr(zpa2326_spi_matches),
+		.pm             = ZPA2326_PM_OPS,
+	},
+	.probe    = zpa2326_probe_spi,
+	.remove   = zpa2326_remove_spi,
+	.id_table = zpa2326_spi_ids,
+};
+module_spi_driver(zpa2326_spi_driver);
+
+MODULE_AUTHOR("Gregor Boirie <gregor.boirie@parrot.com>");
+MODULE_DESCRIPTION("SPI driver for Murata ZPA2326 pressure sensor");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/proximity/as3935.c b/drivers/iio/proximity/as3935.c
index 2e3a70e1b245..5656deb17261 100644
--- a/drivers/iio/proximity/as3935.c
+++ b/drivers/iio/proximity/as3935.c
@@ -397,7 +397,7 @@ static int as3935_probe(struct spi_device *spi)
 		return ret;
 	}
 
-	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+	ret = iio_triggered_buffer_setup(indio_dev, iio_pollfunc_store_time,
 		&as3935_trigger_handler, NULL);
 
 	if (ret) {
diff --git a/drivers/iio/proximity/sx9500.c b/drivers/iio/proximity/sx9500.c
index 1d74b3aafeed..1f06282ec793 100644
--- a/drivers/iio/proximity/sx9500.c
+++ b/drivers/iio/proximity/sx9500.c
@@ -516,7 +516,7 @@ static irqreturn_t sx9500_irq_thread_handler(int irq, void *private)
 		sx9500_push_events(indio_dev);
 
 	if (val & SX9500_CONVDONE_IRQ)
-		complete_all(&data->completion);
+		complete(&data->completion);
 
 out:
 	mutex_unlock(&data->mutex);
@@ -1025,6 +1025,12 @@ static const struct acpi_device_id sx9500_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, sx9500_acpi_match);
 
+static const struct of_device_id sx9500_of_match[] = {
+	{ .compatible = "semtech,sx9500", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sx9500_of_match);
+
 static const struct i2c_device_id sx9500_id[] = {
 	{"sx9500", 0},
 	{ },
@@ -1035,6 +1041,7 @@ static struct i2c_driver sx9500_driver = {
 	.driver = {
 		.name	= SX9500_DRIVER_NAME,
 		.acpi_match_table = ACPI_PTR(sx9500_acpi_match),
+		.of_match_table = of_match_ptr(sx9500_of_match),
 		.pm = &sx9500_pm_ops,
 	},
 	.probe		= sx9500_probe,
diff --git a/drivers/iio/temperature/Kconfig b/drivers/iio/temperature/Kconfig
index c4664e5de791..5ea77a7e261d 100644
--- a/drivers/iio/temperature/Kconfig
+++ b/drivers/iio/temperature/Kconfig
@@ -3,6 +3,22 @@
 #
 menu "Temperature sensors"
 
+config MAXIM_THERMOCOUPLE
+	tristate "Maxim thermocouple sensors"
+	depends on SPI
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	help
+	  If you say yes here you get support for the Maxim series of
+	  thermocouple sensors connected via SPI.
+
+	  Supported sensors:
+	   * MAX6675
+	   * MAX31855
+
+	  This driver can also be built as a module. If so, the module will
+	  be called maxim_thermocouple.
+
 config MLX90614
 	tristate "MLX90614 contact-less infrared sensor"
 	depends on I2C
diff --git a/drivers/iio/temperature/Makefile b/drivers/iio/temperature/Makefile
index 02bc79d49b24..78c3de0dc3f0 100644
--- a/drivers/iio/temperature/Makefile
+++ b/drivers/iio/temperature/Makefile
@@ -2,6 +2,7 @@
 # Makefile for industrial I/O temperature drivers
 #
 
+obj-$(CONFIG_MAXIM_THERMOCOUPLE) += maxim_thermocouple.o
 obj-$(CONFIG_MLX90614) += mlx90614.o
 obj-$(CONFIG_TMP006) += tmp006.o
 obj-$(CONFIG_TSYS01) += tsys01.o
diff --git a/drivers/iio/temperature/maxim_thermocouple.c b/drivers/iio/temperature/maxim_thermocouple.c
new file mode 100644
index 000000000000..39dd2026ccc9
--- /dev/null
+++ b/drivers/iio/temperature/maxim_thermocouple.c
@@ -0,0 +1,281 @@
+/*
+ * maxim_thermocouple.c  - Support for Maxim thermocouple chips
+ *
+ * Copyright (C) 2016 Matt Ranostay <mranostay@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/mutex.h>
+#include <linux/err.h>
+#include <linux/spi/spi.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/trigger_consumer.h>
+
+#define MAXIM_THERMOCOUPLE_DRV_NAME	"maxim_thermocouple"
+
+enum {
+	MAX6675,
+	MAX31855,
+};
+
+static const struct iio_chan_spec max6675_channels[] = {
+	{	/* thermocouple temperature */
+		.type = IIO_TEMP,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 13,
+			.storagebits = 16,
+			.shift = 3,
+			.endianness = IIO_BE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+};
+
+static const struct iio_chan_spec max31855_channels[] = {
+	{	/* thermocouple temperature */
+		.type = IIO_TEMP,
+		.address = 2,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 14,
+			.storagebits = 16,
+			.shift = 2,
+			.endianness = IIO_BE,
+		},
+	},
+	{	/* cold junction temperature */
+		.type = IIO_TEMP,
+		.address = 0,
+		.channel2 = IIO_MOD_TEMP_AMBIENT,
+		.modified = 1,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 1,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 12,
+			.storagebits = 16,
+			.shift = 4,
+			.endianness = IIO_BE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(2),
+};
+
+static const unsigned long max31855_scan_masks[] = {0x3, 0};
+
+struct maxim_thermocouple_chip {
+	const struct iio_chan_spec *channels;
+	const unsigned long *scan_masks;
+	u8 num_channels;
+	u8 read_size;
+
+	/* bit-check for valid input */
+	u32 status_bit;
+};
+
+static const struct maxim_thermocouple_chip maxim_thermocouple_chips[] = {
+	[MAX6675] = {
+			.channels = max6675_channels,
+			.num_channels = ARRAY_SIZE(max6675_channels),
+			.read_size = 2,
+			.status_bit = BIT(2),
+		},
+	[MAX31855] = {
+			.channels = max31855_channels,
+			.num_channels = ARRAY_SIZE(max31855_channels),
+			.read_size = 4,
+			.scan_masks = max31855_scan_masks,
+			.status_bit = BIT(16),
+		},
+};
+
+struct maxim_thermocouple_data {
+	struct spi_device *spi;
+	const struct maxim_thermocouple_chip *chip;
+
+	u8 buffer[16] ____cacheline_aligned;
+};
+
+static int maxim_thermocouple_read(struct maxim_thermocouple_data *data,
+				   struct iio_chan_spec const *chan, int *val)
+{
+	unsigned int storage_bytes = data->chip->read_size;
+	unsigned int shift = chan->scan_type.shift + (chan->address * 8);
+	unsigned int buf;
+	int ret;
+
+	ret = spi_read(data->spi, (void *) &buf, storage_bytes);
+	if (ret)
+		return ret;
+
+	switch (storage_bytes) {
+	case 2:
+		*val = be16_to_cpu(buf);
+		break;
+	case 4:
+		*val = be32_to_cpu(buf);
+		break;
+	}
+
+	/* check to be sure this is a valid reading */
+	if (*val & data->chip->status_bit)
+		return -EINVAL;
+
+	*val = sign_extend32(*val >> shift, chan->scan_type.realbits - 1);
+
+	return 0;
+}
+
+static irqreturn_t maxim_thermocouple_trigger_handler(int irq, void *private)
+{
+	struct iio_poll_func *pf = private;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct maxim_thermocouple_data *data = iio_priv(indio_dev);
+	int ret;
+
+	ret = spi_read(data->spi, data->buffer, data->chip->read_size);
+	if (!ret) {
+		iio_push_to_buffers_with_timestamp(indio_dev, data->buffer,
+						   iio_get_time_ns(indio_dev));
+	}
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int maxim_thermocouple_read_raw(struct iio_dev *indio_dev,
+				       struct iio_chan_spec const *chan,
+				       int *val, int *val2, long mask)
+{
+	struct maxim_thermocouple_data *data = iio_priv(indio_dev);
+	int ret = -EINVAL;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		ret = maxim_thermocouple_read(data, chan, val);
+		iio_device_release_direct_mode(indio_dev);
+
+		if (!ret)
+			return IIO_VAL_INT;
+
+		break;
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->channel2) {
+		case IIO_MOD_TEMP_AMBIENT:
+			*val = 62;
+			*val2 = 500000; /* 1000 * 0.0625 */
+			ret = IIO_VAL_INT_PLUS_MICRO;
+			break;
+		default:
+			*val = 250; /* 1000 * 0.25 */
+			ret = IIO_VAL_INT;
+		};
+		break;
+	}
+
+	return ret;
+}
+
+static const struct iio_info maxim_thermocouple_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = maxim_thermocouple_read_raw,
+};
+
+static int maxim_thermocouple_probe(struct spi_device *spi)
+{
+	const struct spi_device_id *id = spi_get_device_id(spi);
+	struct iio_dev *indio_dev;
+	struct maxim_thermocouple_data *data;
+	const struct maxim_thermocouple_chip *chip =
+			&maxim_thermocouple_chips[id->driver_data];
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	indio_dev->info = &maxim_thermocouple_info;
+	indio_dev->name = MAXIM_THERMOCOUPLE_DRV_NAME;
+	indio_dev->channels = chip->channels;
+	indio_dev->available_scan_masks = chip->scan_masks;
+	indio_dev->num_channels = chip->num_channels;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	data = iio_priv(indio_dev);
+	data->spi = spi;
+	data->chip = chip;
+
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+				maxim_thermocouple_trigger_handler, NULL);
+	if (ret)
+		return ret;
+
+	ret = iio_device_register(indio_dev);
+	if (ret)
+		goto error_unreg_buffer;
+
+	return 0;
+
+error_unreg_buffer:
+	iio_triggered_buffer_cleanup(indio_dev);
+
+	return ret;
+}
+
+static int maxim_thermocouple_remove(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+
+	iio_device_unregister(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
+
+	return 0;
+}
+
+static const struct spi_device_id maxim_thermocouple_id[] = {
+	{"max6675", MAX6675},
+	{"max31855", MAX31855},
+	{},
+};
+MODULE_DEVICE_TABLE(spi, maxim_thermocouple_id);
+
+static struct spi_driver maxim_thermocouple_driver = {
+	.driver = {
+		.name	= MAXIM_THERMOCOUPLE_DRV_NAME,
+	},
+	.probe		= maxim_thermocouple_probe,
+	.remove		= maxim_thermocouple_remove,
+	.id_table	= maxim_thermocouple_id,
+};
+module_spi_driver(maxim_thermocouple_driver);
+
+MODULE_AUTHOR("Matt Ranostay <mranostay@gmail.com>");
+MODULE_DESCRIPTION("Maxim thermocouple sensors");
+MODULE_LICENSE("GPL");
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index e9b7dc037ff8..19a418a1b631 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -74,6 +74,7 @@ source "drivers/infiniband/hw/mlx5/Kconfig"
 source "drivers/infiniband/hw/nes/Kconfig"
 source "drivers/infiniband/hw/ocrdma/Kconfig"
 source "drivers/infiniband/hw/usnic/Kconfig"
+source "drivers/infiniband/hw/hns/Kconfig"
 
 source "drivers/infiniband/ulp/ipoib/Kconfig"
 
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 1374541a4528..b136d3acc5bd 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -800,7 +800,7 @@ static struct notifier_block nb = {
 
 int addr_init(void)
 {
-	addr_wq = create_singlethread_workqueue("ib_addr");
+	addr_wq = alloc_workqueue("ib_addr", WQ_MEM_RECLAIM, 0);
 	if (!addr_wq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index e6dfa1bd3def..36bf50ebb187 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -2462,18 +2462,24 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
 
 	if (addr->dev_addr.bound_dev_if) {
 		ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
-		if (!ndev)
-			return -ENODEV;
+		if (!ndev) {
+			ret = -ENODEV;
+			goto err2;
+		}
 
 		if (ndev->flags & IFF_LOOPBACK) {
 			dev_put(ndev);
-			if (!id_priv->id.device->get_netdev)
-				return -EOPNOTSUPP;
+			if (!id_priv->id.device->get_netdev) {
+				ret = -EOPNOTSUPP;
+				goto err2;
+			}
 
 			ndev = id_priv->id.device->get_netdev(id_priv->id.device,
 							      id_priv->id.port_num);
-			if (!ndev)
-				return -ENODEV;
+			if (!ndev) {
+				ret = -ENODEV;
+				goto err2;
+			}
 		}
 
 		route->path_rec->net = &init_net;
@@ -4363,7 +4369,7 @@ static int __init cma_init(void)
 {
 	int ret;
 
-	cma_wq = create_singlethread_workqueue("rdma_cm");
+	cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM);
 	if (!cma_wq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
index 357624f8b9d3..5495e22839a7 100644
--- a/drivers/infiniband/core/iwcm.c
+++ b/drivers/infiniband/core/iwcm.c
@@ -1160,7 +1160,7 @@ static int __init iw_cm_init(void)
 	if (ret)
 		pr_err("iw_cm: couldn't register netlink callbacks\n");
 
-	iwcm_wq = create_singlethread_workqueue("iw_cm_wq");
+	iwcm_wq = alloc_ordered_workqueue("iw_cm_wq", WQ_MEM_RECLAIM);
 	if (!iwcm_wq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index 2d49228f28b2..40cbd6bdb73b 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -3160,7 +3160,7 @@ static int ib_mad_port_open(struct ib_device *device,
 		goto error3;
 	}
 
-	port_priv->pd = ib_alloc_pd(device);
+	port_priv->pd = ib_alloc_pd(device, 0);
 	if (IS_ERR(port_priv->pd)) {
 		dev_err(&device->dev, "Couldn't create ib_mad PD\n");
 		ret = PTR_ERR(port_priv->pd);
@@ -3177,7 +3177,7 @@ static int ib_mad_port_open(struct ib_device *device,
 		goto error7;
 
 	snprintf(name, sizeof name, "ib_mad%d", port_num);
-	port_priv->wq = create_singlethread_workqueue(name);
+	port_priv->wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM);
 	if (!port_priv->wq) {
 		ret = -ENOMEM;
 		goto error8;
diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c
index 3a3c5d73bbfc..e51b739f6ea3 100644
--- a/drivers/infiniband/core/multicast.c
+++ b/drivers/infiniband/core/multicast.c
@@ -106,7 +106,6 @@ struct mcast_group {
 	atomic_t		refcount;
 	enum mcast_group_state	state;
 	struct ib_sa_query	*query;
-	int			query_id;
 	u16			pkey_index;
 	u8			leave_state;
 	int			retries;
@@ -340,11 +339,7 @@ static int send_join(struct mcast_group *group, struct mcast_member *member)
 				       member->multicast.comp_mask,
 				       3000, GFP_KERNEL, join_handler, group,
 				       &group->query);
-	if (ret >= 0) {
-		group->query_id = ret;
-		ret = 0;
-	}
-	return ret;
+	return (ret > 0) ? 0 : ret;
 }
 
 static int send_leave(struct mcast_group *group, u8 leave_state)
@@ -364,11 +359,7 @@ static int send_leave(struct mcast_group *group, u8 leave_state)
 				       IB_SA_MCMEMBER_REC_JOIN_STATE,
 				       3000, GFP_KERNEL, leave_handler,
 				       group, &group->query);
-	if (ret >= 0) {
-		group->query_id = ret;
-		ret = 0;
-	}
-	return ret;
+	return (ret > 0) ? 0 : ret;
 }
 
 static void join_group(struct mcast_group *group, struct mcast_member *member,
@@ -882,7 +873,7 @@ int mcast_init(void)
 {
 	int ret;
 
-	mcast_wq = create_singlethread_workqueue("ib_mcast");
+	mcast_wq = alloc_ordered_workqueue("ib_mcast", WQ_MEM_RECLAIM);
 	if (!mcast_wq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index b9bf7aa055e7..81b742ca1639 100644
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -2015,7 +2015,7 @@ int ib_sa_init(void)
 		goto err2;
 	}
 
-	ib_nl_wq = create_singlethread_workqueue("ib_nl_sa_wq");
+	ib_nl_wq = alloc_ordered_workqueue("ib_nl_sa_wq", WQ_MEM_RECLAIM);
 	if (!ib_nl_wq) {
 		ret = -ENOMEM;
 		goto err3;
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index 15defefecb4f..c1fb545e8d78 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -1193,7 +1193,7 @@ static ssize_t set_node_desc(struct device *device,
 	if (!dev->modify_device)
 		return -EIO;
 
-	memcpy(desc.node_desc, buf, min_t(int, count, 64));
+	memcpy(desc.node_desc, buf, min_t(int, count, IB_DEVICE_NODE_DESC_MAX));
 	ret = ib_modify_device(dev, IB_DEVICE_MODIFY_NODE_DESC, &desc);
 	if (ret)
 		return ret;
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index 2825ece91d3c..9520154f1d7c 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -1638,7 +1638,8 @@ static int ucma_open(struct inode *inode, struct file *filp)
 	if (!file)
 		return -ENOMEM;
 
-	file->close_wq = create_singlethread_workqueue("ucma_close_id");
+	file->close_wq = alloc_ordered_workqueue("ucma_close_id",
+						 WQ_MEM_RECLAIM);
 	if (!file->close_wq) {
 		kfree(file);
 		return -ENOMEM;
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index f6647318138d..cb3f515a2285 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -571,7 +571,7 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
 
 	pd->device  = ib_dev;
 	pd->uobject = uobj;
-	pd->local_mr = NULL;
+	pd->__internal_mr = NULL;
 	atomic_set(&pd->usecnt, 0);
 
 	uobj->object = pd;
@@ -3078,51 +3078,102 @@ out_put:
 	return ret ? ret : in_len;
 }
 
+static size_t kern_spec_filter_sz(struct ib_uverbs_flow_spec_hdr *spec)
+{
+	/* Returns user space filter size, includes padding */
+	return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2;
+}
+
+static ssize_t spec_filter_size(void *kern_spec_filter, u16 kern_filter_size,
+				u16 ib_real_filter_sz)
+{
+	/*
+	 * User space filter structures must be 64 bit aligned, otherwise this
+	 * may pass, but we won't handle additional new attributes.
+	 */
+
+	if (kern_filter_size > ib_real_filter_sz) {
+		if (memchr_inv(kern_spec_filter +
+			       ib_real_filter_sz, 0,
+			       kern_filter_size - ib_real_filter_sz))
+			return -EINVAL;
+		return ib_real_filter_sz;
+	}
+	return kern_filter_size;
+}
+
 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec,
 				union ib_flow_spec *ib_spec)
 {
+	ssize_t actual_filter_sz;
+	ssize_t kern_filter_sz;
+	ssize_t ib_filter_sz;
+	void *kern_spec_mask;
+	void *kern_spec_val;
+
 	if (kern_spec->reserved)
 		return -EINVAL;
 
 	ib_spec->type = kern_spec->type;
 
+	kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr);
+	/* User flow spec size must be aligned to 4 bytes */
+	if (kern_filter_sz != ALIGN(kern_filter_sz, 4))
+		return -EINVAL;
+
+	kern_spec_val = (void *)kern_spec +
+		sizeof(struct ib_uverbs_flow_spec_hdr);
+	kern_spec_mask = kern_spec_val + kern_filter_sz;
+
 	switch (ib_spec->type) {
 	case IB_FLOW_SPEC_ETH:
-		ib_spec->eth.size = sizeof(struct ib_flow_spec_eth);
-		if (ib_spec->eth.size != kern_spec->eth.size)
+		ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz);
+		actual_filter_sz = spec_filter_size(kern_spec_mask,
+						    kern_filter_sz,
+						    ib_filter_sz);
+		if (actual_filter_sz <= 0)
 			return -EINVAL;
-		memcpy(&ib_spec->eth.val, &kern_spec->eth.val,
-		       sizeof(struct ib_flow_eth_filter));
-		memcpy(&ib_spec->eth.mask, &kern_spec->eth.mask,
-		       sizeof(struct ib_flow_eth_filter));
+		ib_spec->size = sizeof(struct ib_flow_spec_eth);
+		memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);
+		memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);
 		break;
 	case IB_FLOW_SPEC_IPV4:
-		ib_spec->ipv4.size = sizeof(struct ib_flow_spec_ipv4);
-		if (ib_spec->ipv4.size != kern_spec->ipv4.size)
+		ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz);
+		actual_filter_sz = spec_filter_size(kern_spec_mask,
+						    kern_filter_sz,
+						    ib_filter_sz);
+		if (actual_filter_sz <= 0)
 			return -EINVAL;
-		memcpy(&ib_spec->ipv4.val, &kern_spec->ipv4.val,
-		       sizeof(struct ib_flow_ipv4_filter));
-		memcpy(&ib_spec->ipv4.mask, &kern_spec->ipv4.mask,
-		       sizeof(struct ib_flow_ipv4_filter));
+		ib_spec->size = sizeof(struct ib_flow_spec_ipv4);
+		memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);
+		memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);
 		break;
 	case IB_FLOW_SPEC_IPV6:
-		ib_spec->ipv6.size = sizeof(struct ib_flow_spec_ipv6);
-		if (ib_spec->ipv6.size != kern_spec->ipv6.size)
+		ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz);
+		actual_filter_sz = spec_filter_size(kern_spec_mask,
+						    kern_filter_sz,
+						    ib_filter_sz);
+		if (actual_filter_sz <= 0)
+			return -EINVAL;
+		ib_spec->size = sizeof(struct ib_flow_spec_ipv6);
+		memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);
+		memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);
+
+		if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||
+		    (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))
 			return -EINVAL;
-		memcpy(&ib_spec->ipv6.val, &kern_spec->ipv6.val,
-		       sizeof(struct ib_flow_ipv6_filter));
-		memcpy(&ib_spec->ipv6.mask, &kern_spec->ipv6.mask,
-		       sizeof(struct ib_flow_ipv6_filter));
 		break;
 	case IB_FLOW_SPEC_TCP:
 	case IB_FLOW_SPEC_UDP:
-		ib_spec->tcp_udp.size = sizeof(struct ib_flow_spec_tcp_udp);
-		if (ib_spec->tcp_udp.size != kern_spec->tcp_udp.size)
+		ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz);
+		actual_filter_sz = spec_filter_size(kern_spec_mask,
+						    kern_filter_sz,
+						    ib_filter_sz);
+		if (actual_filter_sz <= 0)
 			return -EINVAL;
-		memcpy(&ib_spec->tcp_udp.val, &kern_spec->tcp_udp.val,
-		       sizeof(struct ib_flow_tcp_udp_filter));
-		memcpy(&ib_spec->tcp_udp.mask, &kern_spec->tcp_udp.mask,
-		       sizeof(struct ib_flow_tcp_udp_filter));
+		ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);
+		memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
+		memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
 		break;
 	default:
 		return -EINVAL;
@@ -3654,7 +3705,8 @@ int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
 		goto err_uobj;
 	}
 
-	flow_attr = kmalloc(sizeof(*flow_attr) + cmd.flow_attr.size, GFP_KERNEL);
+	flow_attr = kzalloc(sizeof(*flow_attr) + cmd.flow_attr.num_of_specs *
+			    sizeof(union ib_flow_spec), GFP_KERNEL);
 	if (!flow_attr) {
 		err = -ENOMEM;
 		goto err_put;
@@ -4173,6 +4225,23 @@ int ib_uverbs_ex_query_device(struct ib_uverbs_file *file,
 
 	resp.device_cap_flags_ex = attr.device_cap_flags;
 	resp.response_length += sizeof(resp.device_cap_flags_ex);
+
+	if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps))
+		goto end;
+
+	resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
+	resp.rss_caps.max_rwq_indirection_tables =
+		attr.rss_caps.max_rwq_indirection_tables;
+	resp.rss_caps.max_rwq_indirection_table_size =
+		attr.rss_caps.max_rwq_indirection_table_size;
+
+	resp.response_length += sizeof(resp.rss_caps);
+
+	if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq))
+		goto end;
+
+	resp.max_wq_type_rq = attr.max_wq_type_rq;
+	resp.response_length += sizeof(resp.max_wq_type_rq);
 end:
 	err = ib_copy_to_udata(ucore, &resp, resp.response_length);
 	return err;
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index f2b776efab3a..83687646da68 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -227,9 +227,11 @@ EXPORT_SYMBOL(rdma_port_get_link_layer);
  * Every PD has a local_dma_lkey which can be used as the lkey value for local
  * memory operations.
  */
-struct ib_pd *ib_alloc_pd(struct ib_device *device)
+struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
+		const char *caller)
 {
 	struct ib_pd *pd;
+	int mr_access_flags = 0;
 
 	pd = device->alloc_pd(device, NULL, NULL);
 	if (IS_ERR(pd))
@@ -237,26 +239,46 @@ struct ib_pd *ib_alloc_pd(struct ib_device *device)
 
 	pd->device = device;
 	pd->uobject = NULL;
-	pd->local_mr = NULL;
+	pd->__internal_mr = NULL;
 	atomic_set(&pd->usecnt, 0);
+	pd->flags = flags;
 
 	if (device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
 		pd->local_dma_lkey = device->local_dma_lkey;
-	else {
+	else
+		mr_access_flags |= IB_ACCESS_LOCAL_WRITE;
+
+	if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
+		pr_warn("%s: enabling unsafe global rkey\n", caller);
+		mr_access_flags |= IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE;
+	}
+
+	if (mr_access_flags) {
 		struct ib_mr *mr;
 
-		mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE);
+		mr = pd->device->get_dma_mr(pd, mr_access_flags);
 		if (IS_ERR(mr)) {
 			ib_dealloc_pd(pd);
-			return (struct ib_pd *)mr;
+			return ERR_CAST(mr);
 		}
 
-		pd->local_mr = mr;
-		pd->local_dma_lkey = pd->local_mr->lkey;
+		mr->device	= pd->device;
+		mr->pd		= pd;
+		mr->uobject	= NULL;
+		mr->need_inval	= false;
+
+		pd->__internal_mr = mr;
+
+		if (!(device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY))
+			pd->local_dma_lkey = pd->__internal_mr->lkey;
+
+		if (flags & IB_PD_UNSAFE_GLOBAL_RKEY)
+			pd->unsafe_global_rkey = pd->__internal_mr->rkey;
 	}
+
 	return pd;
 }
-EXPORT_SYMBOL(ib_alloc_pd);
+EXPORT_SYMBOL(__ib_alloc_pd);
 
 /**
  * ib_dealloc_pd - Deallocates a protection domain.
@@ -270,10 +292,10 @@ void ib_dealloc_pd(struct ib_pd *pd)
 {
 	int ret;
 
-	if (pd->local_mr) {
-		ret = ib_dereg_mr(pd->local_mr);
+	if (pd->__internal_mr) {
+		ret = pd->device->dereg_mr(pd->__internal_mr);
 		WARN_ON(ret);
-		pd->local_mr = NULL;
+		pd->__internal_mr = NULL;
 	}
 
 	/* uverbs manipulates usecnt with proper locking, while the kabi
@@ -821,7 +843,7 @@ struct ib_qp *ib_create_qp(struct ib_pd *pd,
 		if (ret) {
 			pr_err("failed to init MR pool ret= %d\n", ret);
 			ib_destroy_qp(qp);
-			qp = ERR_PTR(ret);
+			return ERR_PTR(ret);
 		}
 	}
 
@@ -1391,29 +1413,6 @@ EXPORT_SYMBOL(ib_resize_cq);
 
 /* Memory regions */
 
-struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
-{
-	struct ib_mr *mr;
-	int err;
-
-	err = ib_check_mr_access(mr_access_flags);
-	if (err)
-		return ERR_PTR(err);
-
-	mr = pd->device->get_dma_mr(pd, mr_access_flags);
-
-	if (!IS_ERR(mr)) {
-		mr->device  = pd->device;
-		mr->pd      = pd;
-		mr->uobject = NULL;
-		atomic_inc(&pd->usecnt);
-		mr->need_inval = false;
-	}
-
-	return mr;
-}
-EXPORT_SYMBOL(ib_get_dma_mr);
-
 int ib_dereg_mr(struct ib_mr *mr)
 {
 	struct ib_pd *pd = mr->pd;
@@ -1812,13 +1811,13 @@ EXPORT_SYMBOL(ib_set_vf_guid);
  *
  * Constraints:
  * - The first sg element is allowed to have an offset.
- * - Each sg element must be aligned to page_size (or physically
- *   contiguous to the previous element). In case an sg element has a
- *   non contiguous offset, the mapping prefix will not include it.
+ * - Each sg element must either be aligned to page_size or virtually
+ *   contiguous to the previous element. In case an sg element has a
+ *   non-contiguous offset, the mapping prefix will not include it.
  * - The last sg element is allowed to have length less than page_size.
  * - If sg_nents total byte length exceeds the mr max_num_sge * page_size
  *   then only max_num_sg entries will be mapped.
- * - If the MR was allocated with type IB_MR_TYPE_SG_GAPS_REG, non of these
+ * - If the MR was allocated with type IB_MR_TYPE_SG_GAPS, none of these
  *   constraints holds and the page_size argument is ignored.
  *
  * Returns the number of sg elements that were mapped to the memory region.
diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile
index c0c7cf8af3f4..21fe401ff178 100644
--- a/drivers/infiniband/hw/Makefile
+++ b/drivers/infiniband/hw/Makefile
@@ -9,3 +9,4 @@ obj-$(CONFIG_INFINIBAND_NES)		+= nes/
 obj-$(CONFIG_INFINIBAND_OCRDMA)		+= ocrdma/
 obj-$(CONFIG_INFINIBAND_USNIC)		+= usnic/
 obj-$(CONFIG_INFINIBAND_HFI1)		+= hfi1/
+obj-$(CONFIG_INFINIBAND_HNS)		+= hns/
diff --git a/drivers/infiniband/hw/cxgb3/iwch.c b/drivers/infiniband/hw/cxgb3/iwch.c
index 8e77dc543dd1..b3e11329801d 100644
--- a/drivers/infiniband/hw/cxgb3/iwch.c
+++ b/drivers/infiniband/hw/cxgb3/iwch.c
@@ -36,7 +36,7 @@
 
 #include "cxgb3_offload.h"
 #include "iwch_provider.h"
-#include "iwch_user.h"
+#include <rdma/cxgb3-abi.h>
 #include "iwch.h"
 #include "iwch_cm.h"
 
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c
index 04bbf172abde..65ee64400deb 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_cm.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c
@@ -2258,7 +2258,7 @@ int __init iwch_cm_init(void)
 {
 	skb_queue_head_init(&rxq);
 
-	workq = create_singlethread_workqueue("iw_cxgb3");
+	workq = alloc_ordered_workqueue("iw_cxgb3", WQ_MEM_RECLAIM);
 	if (!workq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 3edb80644b53..cba57bb53dba 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -58,7 +58,7 @@
 #include "iwch.h"
 #include "iwch_provider.h"
 #include "iwch_cm.h"
-#include "iwch_user.h"
+#include <rdma/cxgb3-abi.h>
 #include "common.h"
 
 static struct ib_ah *iwch_ah_create(struct ib_pd *pd,
@@ -1396,6 +1396,7 @@ int iwch_register_device(struct iwch_dev *dev)
 	    (1ull << IB_USER_VERBS_CMD_POST_SEND) |
 	    (1ull << IB_USER_VERBS_CMD_POST_RECV);
 	dev->ibdev.node_type = RDMA_NODE_RNIC;
+	BUILD_BUG_ON(sizeof(IWCH_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
 	memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
 	dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports;
 	dev->ibdev.num_comp_vectors = 1;
diff --git a/drivers/infiniband/hw/cxgb4/Kconfig b/drivers/infiniband/hw/cxgb4/Kconfig
index 23f38cf2c5cd..afe8b28e0878 100644
--- a/drivers/infiniband/hw/cxgb4/Kconfig
+++ b/drivers/infiniband/hw/cxgb4/Kconfig
@@ -1,6 +1,7 @@
 config INFINIBAND_CXGB4
 	tristate "Chelsio T4/T5 RDMA Driver"
 	depends on CHELSIO_T4 && INET && (IPV6 || IPV6=n)
+	select CHELSIO_LIB
 	select GENERIC_ALLOCATOR
 	---help---
 	  This is an iWARP/RDMA driver for the Chelsio T4 and T5
diff --git a/drivers/infiniband/hw/cxgb4/Makefile b/drivers/infiniband/hw/cxgb4/Makefile
index e11cf7299945..fa40b685831b 100644
--- a/drivers/infiniband/hw/cxgb4/Makefile
+++ b/drivers/infiniband/hw/cxgb4/Makefile
@@ -1,4 +1,5 @@
 ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
+ccflags-y += -Idrivers/net/ethernet/chelsio/libcxgb
 
 obj-$(CONFIG_INFINIBAND_CXGB4) += iw_cxgb4.o
 
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index 3aca7f6171b4..f1510cc76d2d 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -49,6 +49,7 @@
 
 #include <rdma/ib_addr.h>
 
+#include <libcxgb_cm.h>
 #include "iw_cxgb4.h"
 #include "clip_tbl.h"
 
@@ -239,15 +240,13 @@ int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb)
 
 static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb)
 {
-	struct cpl_tid_release *req;
+	u32 len = roundup(sizeof(struct cpl_tid_release), 16);
 
-	skb = get_skb(skb, sizeof *req, GFP_KERNEL);
+	skb = get_skb(skb, len, GFP_KERNEL);
 	if (!skb)
 		return;
-	req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req));
-	INIT_TP_WR(req, hwtid);
-	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid));
-	set_wr_txq(skb, CPL_PRIORITY_SETUP, 0);
+
+	cxgb_mk_tid_release(skb, len, hwtid, 0);
 	c4iw_ofld_send(rdev, skb);
 	return;
 }
@@ -333,6 +332,8 @@ static void remove_ep_tid(struct c4iw_ep *ep)
 
 	spin_lock_irqsave(&ep->com.dev->lock, flags);
 	_remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid, 0);
+	if (idr_is_empty(&ep->com.dev->hwtid_idr))
+		wake_up(&ep->com.dev->wait);
 	spin_unlock_irqrestore(&ep->com.dev->lock, flags);
 }
 
@@ -464,72 +465,6 @@ static struct net_device *get_real_dev(struct net_device *egress_dev)
 	return rdma_vlan_dev_real_dev(egress_dev) ? : egress_dev;
 }
 
-static int our_interface(struct c4iw_dev *dev, struct net_device *egress_dev)
-{
-	int i;
-
-	egress_dev = get_real_dev(egress_dev);
-	for (i = 0; i < dev->rdev.lldi.nports; i++)
-		if (dev->rdev.lldi.ports[i] == egress_dev)
-			return 1;
-	return 0;
-}
-
-static struct dst_entry *find_route6(struct c4iw_dev *dev, __u8 *local_ip,
-				     __u8 *peer_ip, __be16 local_port,
-				     __be16 peer_port, u8 tos,
-				     __u32 sin6_scope_id)
-{
-	struct dst_entry *dst = NULL;
-
-	if (IS_ENABLED(CONFIG_IPV6)) {
-		struct flowi6 fl6;
-
-		memset(&fl6, 0, sizeof(fl6));
-		memcpy(&fl6.daddr, peer_ip, 16);
-		memcpy(&fl6.saddr, local_ip, 16);
-		if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
-			fl6.flowi6_oif = sin6_scope_id;
-		dst = ip6_route_output(&init_net, NULL, &fl6);
-		if (!dst)
-			goto out;
-		if (!our_interface(dev, ip6_dst_idev(dst)->dev) &&
-		    !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK)) {
-			dst_release(dst);
-			dst = NULL;
-		}
-	}
-
-out:
-	return dst;
-}
-
-static struct dst_entry *find_route(struct c4iw_dev *dev, __be32 local_ip,
-				 __be32 peer_ip, __be16 local_port,
-				 __be16 peer_port, u8 tos)
-{
-	struct rtable *rt;
-	struct flowi4 fl4;
-	struct neighbour *n;
-
-	rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip,
-				   peer_port, local_port, IPPROTO_TCP,
-				   tos, 0);
-	if (IS_ERR(rt))
-		return NULL;
-	n = dst_neigh_lookup(&rt->dst, &peer_ip);
-	if (!n)
-		return NULL;
-	if (!our_interface(dev, n->dev) &&
-	    !(n->dev->flags & IFF_LOOPBACK)) {
-		neigh_release(n);
-		dst_release(&rt->dst);
-		return NULL;
-	}
-	neigh_release(n);
-	return &rt->dst;
-}
-
 static void arp_failure_discard(void *handle, struct sk_buff *skb)
 {
 	pr_err(MOD "ARP failure\n");
@@ -704,56 +639,32 @@ static int send_flowc(struct c4iw_ep *ep)
 
 static int send_halfclose(struct c4iw_ep *ep)
 {
-	struct cpl_close_con_req *req;
 	struct sk_buff *skb = skb_dequeue(&ep->com.ep_skb_list);
-	int wrlen = roundup(sizeof *req, 16);
+	u32 wrlen = roundup(sizeof(struct cpl_close_con_req), 16);
 
 	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
 	if (WARN_ON(!skb))
 		return -ENOMEM;
 
-	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
-	t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
-	req = (struct cpl_close_con_req *) skb_put(skb, wrlen);
-	memset(req, 0, wrlen);
-	INIT_TP_WR(req, ep->hwtid);
-	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ,
-						    ep->hwtid));
+	cxgb_mk_close_con_req(skb, wrlen, ep->hwtid, ep->txq_idx,
+			      NULL, arp_failure_discard);
+
 	return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
 }
 
 static int send_abort(struct c4iw_ep *ep)
 {
-	struct cpl_abort_req *req;
-	int wrlen = roundup(sizeof *req, 16);
+	u32 wrlen = roundup(sizeof(struct cpl_abort_req), 16);
 	struct sk_buff *req_skb = skb_dequeue(&ep->com.ep_skb_list);
 
 	PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
 	if (WARN_ON(!req_skb))
 		return -ENOMEM;
 
-	set_wr_txq(req_skb, CPL_PRIORITY_DATA, ep->txq_idx);
-	t4_set_arp_err_handler(req_skb, ep, abort_arp_failure);
-	req = (struct cpl_abort_req *)skb_put(req_skb, wrlen);
-	memset(req, 0, wrlen);
-	INIT_TP_WR(req, ep->hwtid);
-	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid));
-	req->cmd = CPL_ABORT_SEND_RST;
-	return c4iw_l2t_send(&ep->com.dev->rdev, req_skb, ep->l2t);
-}
+	cxgb_mk_abort_req(req_skb, wrlen, ep->hwtid, ep->txq_idx,
+			  ep, abort_arp_failure);
 
-static void best_mtu(const unsigned short *mtus, unsigned short mtu,
-		     unsigned int *idx, int use_ts, int ipv6)
-{
-	unsigned short hdr_size = (ipv6 ?
-				   sizeof(struct ipv6hdr) :
-				   sizeof(struct iphdr)) +
-				  sizeof(struct tcphdr) +
-				  (use_ts ?
-				   round_up(TCPOLEN_TIMESTAMP, 4) : 0);
-	unsigned short data_size = mtu - hdr_size;
-
-	cxgb4_best_aligned_mtu(mtus, hdr_size, data_size, 8, idx);
+	return c4iw_l2t_send(&ep->com.dev->rdev, req_skb, ep->l2t);
 }
 
 static int send_connect(struct c4iw_ep *ep)
@@ -768,7 +679,7 @@ static int send_connect(struct c4iw_ep *ep)
 	u64 opt0;
 	u32 opt2;
 	unsigned int mtu_idx;
-	int wscale;
+	u32 wscale;
 	int win, sizev4, sizev6, wrlen;
 	struct sockaddr_in *la = (struct sockaddr_in *)
 				 &ep->com.local_addr;
@@ -815,10 +726,10 @@ static int send_connect(struct c4iw_ep *ep)
 	}
 	set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx);
 
-	best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
-		 enable_tcp_timestamps,
-		 (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1);
-	wscale = compute_wscale(rcv_win);
+	cxgb_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
+		      enable_tcp_timestamps,
+		      (ep->com.remote_addr.ss_family == AF_INET) ? 0 : 1);
+	wscale = cxgb_compute_wscale(rcv_win);
 
 	/*
 	 * Specify the largest window that will fit in opt0. The
@@ -1445,9 +1356,9 @@ static void established_upcall(struct c4iw_ep *ep)
 
 static int update_rx_credits(struct c4iw_ep *ep, u32 credits)
 {
-	struct cpl_rx_data_ack *req;
 	struct sk_buff *skb;
-	int wrlen = roundup(sizeof *req, 16);
+	u32 wrlen = roundup(sizeof(struct cpl_rx_data_ack), 16);
+	u32 credit_dack;
 
 	PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits);
 	skb = get_skb(NULL, wrlen, GFP_KERNEL);
@@ -1464,15 +1375,12 @@ static int update_rx_credits(struct c4iw_ep *ep, u32 credits)
 	if (ep->rcv_win > RCV_BUFSIZ_M * 1024)
 		credits += ep->rcv_win - RCV_BUFSIZ_M * 1024;
 
-	req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen);
-	memset(req, 0, wrlen);
-	INIT_TP_WR(req, ep->hwtid);
-	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
-						    ep->hwtid));
-	req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK_F |
-				       RX_DACK_CHANGE_F |
-				       RX_DACK_MODE_V(dack_mode));
-	set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx);
+	credit_dack = credits | RX_FORCE_ACK_F | RX_DACK_CHANGE_F |
+		      RX_DACK_MODE_V(dack_mode);
+
+	cxgb_mk_rx_data_ack(skb, wrlen, ep->hwtid, ep->ctrlq_idx,
+			    credit_dack);
+
 	c4iw_ofld_send(&ep->com.dev->rdev, skb);
 	return credits;
 }
@@ -1827,8 +1735,12 @@ static int process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb)
 				(ep->mpa_pkt + sizeof(*mpa));
 			ep->ird = ntohs(mpa_v2_params->ird) &
 				MPA_V2_IRD_ORD_MASK;
+			ep->ird = min_t(u32, ep->ird,
+					cur_max_read_depth(ep->com.dev));
 			ep->ord = ntohs(mpa_v2_params->ord) &
 				MPA_V2_IRD_ORD_MASK;
+			ep->ord = min_t(u32, ep->ord,
+					cur_max_read_depth(ep->com.dev));
 			PDBG("%s initiator ird %u ord %u\n", __func__, ep->ird,
 			     ep->ord);
 			if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL)
@@ -1966,7 +1878,7 @@ static int send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid)
 	struct sk_buff *skb;
 	struct fw_ofld_connection_wr *req;
 	unsigned int mtu_idx;
-	int wscale;
+	u32 wscale;
 	struct sockaddr_in *sin;
 	int win;
 
@@ -1991,10 +1903,10 @@ static int send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid)
 			htons(FW_OFLD_CONNECTION_WR_CPLRXDATAACK_F);
 	req->tcb.tx_max = (__force __be32) jiffies;
 	req->tcb.rcv_adv = htons(1);
-	best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
-		 enable_tcp_timestamps,
-		 (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1);
-	wscale = compute_wscale(rcv_win);
+	cxgb_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
+		      enable_tcp_timestamps,
+		      (ep->com.remote_addr.ss_family == AF_INET) ? 0 : 1);
+	wscale = cxgb_compute_wscale(rcv_win);
 
 	/*
 	 * Specify the largest window that will fit in opt0. The
@@ -2048,15 +1960,6 @@ static inline int act_open_has_tid(int status)
 		status != CPL_ERR_CONN_EXIST);
 }
 
-/* Returns whether a CPL status conveys negative advice.
- */
-static int is_neg_adv(unsigned int status)
-{
-	return status == CPL_ERR_RTX_NEG_ADVICE ||
-	       status == CPL_ERR_PERSIST_NEG_ADVICE ||
-	       status == CPL_ERR_KEEPALV_NEG_ADVICE;
-}
-
 static char *neg_adv_str(unsigned int status)
 {
 	switch (status) {
@@ -2113,8 +2016,10 @@ static int import_ep(struct c4iw_ep *ep, int iptype, __u8 *peer_ip,
 		}
 		ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t,
 					n, pdev, rt_tos2priority(tos));
-		if (!ep->l2t)
+		if (!ep->l2t) {
+			dev_put(pdev);
 			goto out;
+		}
 		ep->mtu = pdev->mtu;
 		ep->tx_chan = cxgb4_port_chan(pdev);
 		ep->smac_idx = cxgb4_tp_smt_idx(adapter_type,
@@ -2210,16 +2115,21 @@ static int c4iw_reconnect(struct c4iw_ep *ep)
 
 	/* find a route */
 	if (ep->com.cm_id->m_local_addr.ss_family == AF_INET) {
-		ep->dst = find_route(ep->com.dev, laddr->sin_addr.s_addr,
-				     raddr->sin_addr.s_addr, laddr->sin_port,
-				     raddr->sin_port, ep->com.cm_id->tos);
+		ep->dst = cxgb_find_route(&ep->com.dev->rdev.lldi, get_real_dev,
+					  laddr->sin_addr.s_addr,
+					  raddr->sin_addr.s_addr,
+					  laddr->sin_port,
+					  raddr->sin_port, ep->com.cm_id->tos);
 		iptype = 4;
 		ra = (__u8 *)&raddr->sin_addr;
 	} else {
-		ep->dst = find_route6(ep->com.dev, laddr6->sin6_addr.s6_addr,
-				      raddr6->sin6_addr.s6_addr,
-				      laddr6->sin6_port, raddr6->sin6_port, 0,
-				      raddr6->sin6_scope_id);
+		ep->dst = cxgb_find_route6(&ep->com.dev->rdev.lldi,
+					   get_real_dev,
+					   laddr6->sin6_addr.s6_addr,
+					   raddr6->sin6_addr.s6_addr,
+					   laddr6->sin6_port,
+					   raddr6->sin6_port, 0,
+					   raddr6->sin6_scope_id);
 		iptype = 6;
 		ra = (__u8 *)&raddr6->sin6_addr;
 	}
@@ -2291,7 +2201,7 @@ static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
 	PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid,
 	     status, status2errno(status));
 
-	if (is_neg_adv(status)) {
+	if (cxgb_is_neg_adv(status)) {
 		PDBG("%s Connection problems for atid %u status %u (%s)\n",
 		     __func__, atid, status, neg_adv_str(status));
 		ep->stats.connect_neg_adv++;
@@ -2418,7 +2328,7 @@ static int accept_cr(struct c4iw_ep *ep, struct sk_buff *skb,
 	unsigned int mtu_idx;
 	u64 opt0;
 	u32 opt2;
-	int wscale;
+	u32 wscale;
 	struct cpl_t5_pass_accept_rpl *rpl5 = NULL;
 	int win;
 	enum chip_type adapter_type = ep->com.dev->rdev.lldi.adapter_type;
@@ -2439,10 +2349,10 @@ static int accept_cr(struct c4iw_ep *ep, struct sk_buff *skb,
 	OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
 						    ep->hwtid));
 
-	best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
-		 enable_tcp_timestamps && req->tcpopt.tstamp,
-		 (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1);
-	wscale = compute_wscale(rcv_win);
+	cxgb_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
+		      enable_tcp_timestamps && req->tcpopt.tstamp,
+		      (ep->com.remote_addr.ss_family == AF_INET) ? 0 : 1);
+	wscale = cxgb_compute_wscale(rcv_win);
 
 	/*
 	 * Specify the largest window that will fit in opt0. The
@@ -2514,42 +2424,6 @@ static void reject_cr(struct c4iw_dev *dev, u32 hwtid, struct sk_buff *skb)
 	return;
 }
 
-static void get_4tuple(struct cpl_pass_accept_req *req, enum chip_type type,
-		       int *iptype, __u8 *local_ip, __u8 *peer_ip,
-		       __be16 *local_port, __be16 *peer_port)
-{
-	int eth_len = (CHELSIO_CHIP_VERSION(type) <= CHELSIO_T5) ?
-		      ETH_HDR_LEN_G(be32_to_cpu(req->hdr_len)) :
-		      T6_ETH_HDR_LEN_G(be32_to_cpu(req->hdr_len));
-	int ip_len = (CHELSIO_CHIP_VERSION(type) <= CHELSIO_T5) ?
-		     IP_HDR_LEN_G(be32_to_cpu(req->hdr_len)) :
-		     T6_IP_HDR_LEN_G(be32_to_cpu(req->hdr_len));
-	struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len);
-	struct ipv6hdr *ip6 = (struct ipv6hdr *)((u8 *)(req + 1) + eth_len);
-	struct tcphdr *tcp = (struct tcphdr *)
-			     ((u8 *)(req + 1) + eth_len + ip_len);
-
-	if (ip->version == 4) {
-		PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__,
-		     ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source),
-		     ntohs(tcp->dest));
-		*iptype = 4;
-		memcpy(peer_ip, &ip->saddr, 4);
-		memcpy(local_ip, &ip->daddr, 4);
-	} else {
-		PDBG("%s saddr %pI6 daddr %pI6 sport %u dport %u\n", __func__,
-		     ip6->saddr.s6_addr, ip6->daddr.s6_addr, ntohs(tcp->source),
-		     ntohs(tcp->dest));
-		*iptype = 6;
-		memcpy(peer_ip, ip6->saddr.s6_addr, 16);
-		memcpy(local_ip, ip6->daddr.s6_addr, 16);
-	}
-	*peer_port = tcp->source;
-	*local_port = tcp->dest;
-
-	return;
-}
-
 static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
 {
 	struct c4iw_ep *child_ep = NULL, *parent_ep;
@@ -2578,8 +2452,8 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
 		goto reject;
 	}
 
-	get_4tuple(req, parent_ep->com.dev->rdev.lldi.adapter_type, &iptype,
-		   local_ip, peer_ip, &local_port, &peer_port);
+	cxgb_get_4tuple(req, parent_ep->com.dev->rdev.lldi.adapter_type,
+			&iptype, local_ip, peer_ip, &local_port, &peer_port);
 
 	/* Find output route */
 	if (iptype == 4)  {
@@ -2587,18 +2461,19 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
 		     , __func__, parent_ep, hwtid,
 		     local_ip, peer_ip, ntohs(local_port),
 		     ntohs(peer_port), peer_mss);
-		dst = find_route(dev, *(__be32 *)local_ip, *(__be32 *)peer_ip,
-				 local_port, peer_port,
-				 tos);
+		dst = cxgb_find_route(&dev->rdev.lldi, get_real_dev,
+				      *(__be32 *)local_ip, *(__be32 *)peer_ip,
+				      local_port, peer_port, tos);
 	} else {
 		PDBG("%s parent ep %p hwtid %u laddr %pI6 raddr %pI6 lport %d rport %d peer_mss %d\n"
 		     , __func__, parent_ep, hwtid,
 		     local_ip, peer_ip, ntohs(local_port),
 		     ntohs(peer_port), peer_mss);
-		dst = find_route6(dev, local_ip, peer_ip, local_port, peer_port,
-				  PASS_OPEN_TOS_G(ntohl(req->tos_stid)),
-				  ((struct sockaddr_in6 *)
-				  &parent_ep->com.local_addr)->sin6_scope_id);
+		dst = cxgb_find_route6(&dev->rdev.lldi, get_real_dev,
+				local_ip, peer_ip, local_port, peer_port,
+				PASS_OPEN_TOS_G(ntohl(req->tos_stid)),
+				((struct sockaddr_in6 *)
+				 &parent_ep->com.local_addr)->sin6_scope_id);
 	}
 	if (!dst) {
 		printk(KERN_ERR MOD "%s - failed to find dst entry!\n",
@@ -2831,18 +2706,18 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
 {
 	struct cpl_abort_req_rss *req = cplhdr(skb);
 	struct c4iw_ep *ep;
-	struct cpl_abort_rpl *rpl;
 	struct sk_buff *rpl_skb;
 	struct c4iw_qp_attributes attrs;
 	int ret;
 	int release = 0;
 	unsigned int tid = GET_TID(req);
+	u32 len = roundup(sizeof(struct cpl_abort_rpl), 16);
 
 	ep = get_ep_from_tid(dev, tid);
 	if (!ep)
 		return 0;
 
-	if (is_neg_adv(req->status)) {
+	if (cxgb_is_neg_adv(req->status)) {
 		PDBG("%s Negative advice on abort- tid %u status %d (%s)\n",
 		     __func__, ep->hwtid, req->status,
 		     neg_adv_str(req->status));
@@ -2935,11 +2810,9 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
 		release = 1;
 		goto out;
 	}
-	set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
-	rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl));
-	INIT_TP_WR(rpl, ep->hwtid);
-	OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid));
-	rpl->cmd = CPL_ABORT_NO_RST;
+
+	cxgb_mk_abort_rpl(rpl_skb, len, ep->hwtid, ep->txq_idx);
+
 	c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb);
 out:
 	if (release)
@@ -3136,7 +3009,7 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
 		if (conn_param->ord > ep->ird) {
 			if (RELAXED_IRD_NEGOTIATION) {
-				ep->ord = ep->ird;
+				conn_param->ord = ep->ird;
 			} else {
 				ep->ird = conn_param->ird;
 				ep->ord = conn_param->ord;
@@ -3371,9 +3244,11 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		PDBG("%s saddr %pI4 sport 0x%x raddr %pI4 rport 0x%x\n",
 		     __func__, &laddr->sin_addr, ntohs(laddr->sin_port),
 		     ra, ntohs(raddr->sin_port));
-		ep->dst = find_route(dev, laddr->sin_addr.s_addr,
-				     raddr->sin_addr.s_addr, laddr->sin_port,
-				     raddr->sin_port, cm_id->tos);
+		ep->dst = cxgb_find_route(&dev->rdev.lldi, get_real_dev,
+					  laddr->sin_addr.s_addr,
+					  raddr->sin_addr.s_addr,
+					  laddr->sin_port,
+					  raddr->sin_port, cm_id->tos);
 	} else {
 		iptype = 6;
 		ra = (__u8 *)&raddr6->sin6_addr;
@@ -3392,10 +3267,12 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		     __func__, laddr6->sin6_addr.s6_addr,
 		     ntohs(laddr6->sin6_port),
 		     raddr6->sin6_addr.s6_addr, ntohs(raddr6->sin6_port));
-		ep->dst = find_route6(dev, laddr6->sin6_addr.s6_addr,
-				      raddr6->sin6_addr.s6_addr,
-				      laddr6->sin6_port, raddr6->sin6_port, 0,
-				      raddr6->sin6_scope_id);
+		ep->dst = cxgb_find_route6(&dev->rdev.lldi, get_real_dev,
+					   laddr6->sin6_addr.s6_addr,
+					   raddr6->sin6_addr.s6_addr,
+					   laddr6->sin6_port,
+					   raddr6->sin6_port, 0,
+					   raddr6->sin6_scope_id);
 	}
 	if (!ep->dst) {
 		printk(KERN_ERR MOD "%s - cannot find route.\n", __func__);
@@ -4037,8 +3914,9 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb)
 	     ntohl(iph->daddr), ntohs(tcph->dest), ntohl(iph->saddr),
 	     ntohs(tcph->source), iph->tos);
 
-	dst = find_route(dev, iph->daddr, iph->saddr, tcph->dest, tcph->source,
-			 iph->tos);
+	dst = cxgb_find_route(&dev->rdev.lldi, get_real_dev,
+			      iph->daddr, iph->saddr, tcph->dest,
+			      tcph->source, iph->tos);
 	if (!dst) {
 		pr_err("%s - failed to find dst entry!\n",
 		       __func__);
@@ -4313,7 +4191,7 @@ static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb)
 		kfree_skb(skb);
 		return 0;
 	}
-	if (is_neg_adv(req->status)) {
+	if (cxgb_is_neg_adv(req->status)) {
 		PDBG("%s Negative advice on abort- tid %u status %d (%s)\n",
 		     __func__, ep->hwtid, req->status,
 		     neg_adv_str(req->status));
@@ -4357,7 +4235,7 @@ int __init c4iw_cm_init(void)
 	spin_lock_init(&timeout_lock);
 	skb_queue_head_init(&rxq);
 
-	workq = create_singlethread_workqueue("iw_cxgb4");
+	workq = alloc_ordered_workqueue("iw_cxgb4", WQ_MEM_RECLAIM);
 	if (!workq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index 812ab7278b8e..867b8cf82be8 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -666,6 +666,18 @@ skip_cqe:
 	return ret;
 }
 
+static void invalidate_mr(struct c4iw_dev *rhp, u32 rkey)
+{
+	struct c4iw_mr *mhp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&rhp->lock, flags);
+	mhp = get_mhp(rhp, rkey >> 8);
+	if (mhp)
+		mhp->attr.state = 0;
+	spin_unlock_irqrestore(&rhp->lock, flags);
+}
+
 /*
  * Get one cq entry from c4iw and map it to openib.
  *
@@ -721,6 +733,7 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
 		    CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_SE_INV) {
 			wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe);
 			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+			invalidate_mr(qhp->rhp, wc->ex.invalidate_rkey);
 		}
 	} else {
 		switch (CQE_OPCODE(&cqe)) {
@@ -746,6 +759,10 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
 			break;
 		case FW_RI_FAST_REGISTER:
 			wc->opcode = IB_WC_REG_MR;
+
+			/* Invalidate the MR if the fastreg failed */
+			if (CQE_STATUS(&cqe) != T4_ERR_SUCCESS)
+				invalidate_mr(qhp->rhp, CQE_WRID_FR_STAG(&cqe));
 			break;
 		default:
 			printk(KERN_ERR MOD "Unexpected opcode %d "
@@ -1016,15 +1033,15 @@ int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
 int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 {
 	struct c4iw_cq *chp;
-	int ret;
+	int ret = 0;
 	unsigned long flag;
 
 	chp = to_c4iw_cq(ibcq);
 	spin_lock_irqsave(&chp->lock, flag);
-	ret = t4_arm_cq(&chp->cq,
-			(flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);
+	t4_arm_cq(&chp->cq,
+		  (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);
+	if (flags & IB_CQ_REPORT_MISSED_EVENTS)
+		ret = t4_cq_notempty(&chp->cq);
 	spin_unlock_irqrestore(&chp->lock, flag);
-	if (ret && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
-		ret = 0;
 	return ret;
 }
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index 071d7332ec06..93e3d270a98a 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -872,9 +872,13 @@ static void c4iw_rdev_close(struct c4iw_rdev *rdev)
 static void c4iw_dealloc(struct uld_ctx *ctx)
 {
 	c4iw_rdev_close(&ctx->dev->rdev);
+	WARN_ON_ONCE(!idr_is_empty(&ctx->dev->cqidr));
 	idr_destroy(&ctx->dev->cqidr);
+	WARN_ON_ONCE(!idr_is_empty(&ctx->dev->qpidr));
 	idr_destroy(&ctx->dev->qpidr);
+	WARN_ON_ONCE(!idr_is_empty(&ctx->dev->mmidr));
 	idr_destroy(&ctx->dev->mmidr);
+	wait_event(ctx->dev->wait, idr_is_empty(&ctx->dev->hwtid_idr));
 	idr_destroy(&ctx->dev->hwtid_idr);
 	idr_destroy(&ctx->dev->stid_idr);
 	idr_destroy(&ctx->dev->atid_idr);
@@ -992,6 +996,7 @@ static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop)
 	mutex_init(&devp->rdev.stats.lock);
 	mutex_init(&devp->db_mutex);
 	INIT_LIST_HEAD(&devp->db_fc_list);
+	init_waitqueue_head(&devp->wait);
 	devp->avail_ird = devp->rdev.lldi.max_ird_adapter;
 
 	if (c4iw_debugfs_root) {
@@ -1475,6 +1480,10 @@ static int c4iw_uld_control(void *handle, enum cxgb4_control control, ...)
 
 static struct cxgb4_uld_info c4iw_uld_info = {
 	.name = DRV_NAME,
+	.nrxq = MAX_ULD_QSETS,
+	.rxq_size = 511,
+	.ciq = true,
+	.lro = false,
 	.add = c4iw_uld_add,
 	.rx_handler = c4iw_uld_rx_handler,
 	.state_change = c4iw_uld_state_change,
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index aa47e0ae80bc..7e7f79e55006 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -58,7 +58,7 @@
 #include "cxgb4.h"
 #include "cxgb4_uld.h"
 #include "l2t.h"
-#include "user.h"
+#include <rdma/cxgb4-abi.h>
 
 #define DRV_NAME "iw_cxgb4"
 #define MOD DRV_NAME ":"
@@ -263,6 +263,7 @@ struct c4iw_dev {
 	struct idr stid_idr;
 	struct list_head db_fc_list;
 	u32 avail_ird;
+	wait_queue_head_t wait;
 };
 
 static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)
@@ -881,15 +882,6 @@ static inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id)
 	return cm_id->provider_data;
 }
 
-static inline int compute_wscale(int win)
-{
-	int wscale = 0;
-
-	while (wscale < 14 && (65535<<wscale) < win)
-		wscale++;
-	return wscale;
-}
-
 static inline int ocqp_supported(const struct cxgb4_lld_info *infop)
 {
 #if defined(__i386__) || defined(__x86_64__) || defined(CONFIG_PPC64)
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index 0b91b0f4df71..80e27749420a 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -695,7 +695,7 @@ struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
 	mhp->attr.pdid = php->pdid;
 	mhp->attr.type = FW_RI_STAG_NSMR;
 	mhp->attr.stag = stag;
-	mhp->attr.state = 1;
+	mhp->attr.state = 0;
 	mmid = (stag) >> 8;
 	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
 	if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index df127ce6b6ec..645e606a17c5 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -563,6 +563,7 @@ int c4iw_register_device(struct c4iw_dev *dev)
 	    (1ull << IB_USER_VERBS_CMD_POST_SEND) |
 	    (1ull << IB_USER_VERBS_CMD_POST_RECV);
 	dev->ibdev.node_type = RDMA_NODE_RNIC;
+	BUILD_BUG_ON(sizeof(C4IW_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
 	memcpy(dev->ibdev.node_desc, C4IW_NODE_DESC, sizeof(C4IW_NODE_DESC));
 	dev->ibdev.phys_port_cnt = dev->rdev.lldi.nports;
 	dev->ibdev.num_comp_vectors =  dev->rdev.lldi.nciq;
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index edb1172b6f54..f57deba6717c 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -609,10 +609,42 @@ static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe,
 	return 0;
 }
 
+static void build_tpte_memreg(struct fw_ri_fr_nsmr_tpte_wr *fr,
+			      struct ib_reg_wr *wr, struct c4iw_mr *mhp,
+			      u8 *len16)
+{
+	__be64 *p = (__be64 *)fr->pbl;
+
+	fr->r2 = cpu_to_be32(0);
+	fr->stag = cpu_to_be32(mhp->ibmr.rkey);
+
+	fr->tpte.valid_to_pdid = cpu_to_be32(FW_RI_TPTE_VALID_F |
+		FW_RI_TPTE_STAGKEY_V((mhp->ibmr.rkey & FW_RI_TPTE_STAGKEY_M)) |
+		FW_RI_TPTE_STAGSTATE_V(1) |
+		FW_RI_TPTE_STAGTYPE_V(FW_RI_STAG_NSMR) |
+		FW_RI_TPTE_PDID_V(mhp->attr.pdid));
+	fr->tpte.locread_to_qpid = cpu_to_be32(
+		FW_RI_TPTE_PERM_V(c4iw_ib_to_tpt_access(wr->access)) |
+		FW_RI_TPTE_ADDRTYPE_V(FW_RI_VA_BASED_TO) |
+		FW_RI_TPTE_PS_V(ilog2(wr->mr->page_size) - 12));
+	fr->tpte.nosnoop_pbladdr = cpu_to_be32(FW_RI_TPTE_PBLADDR_V(
+		PBL_OFF(&mhp->rhp->rdev, mhp->attr.pbl_addr)>>3));
+	fr->tpte.dca_mwbcnt_pstag = cpu_to_be32(0);
+	fr->tpte.len_hi = cpu_to_be32(0);
+	fr->tpte.len_lo = cpu_to_be32(mhp->ibmr.length);
+	fr->tpte.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32);
+	fr->tpte.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & 0xffffffff);
+
+	p[0] = cpu_to_be64((u64)mhp->mpl[0]);
+	p[1] = cpu_to_be64((u64)mhp->mpl[1]);
+
+	*len16 = DIV_ROUND_UP(sizeof(*fr), 16);
+}
+
 static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
-			struct ib_reg_wr *wr, u8 *len16, bool dsgl_supported)
+			struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16,
+			bool dsgl_supported)
 {
-	struct c4iw_mr *mhp = to_c4iw_mr(wr->mr);
 	struct fw_ri_immd *imdp;
 	__be64 *p;
 	int i;
@@ -674,16 +706,19 @@ static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
 	return 0;
 }
 
-static int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr,
-			  u8 *len16)
+static int build_inv_stag(struct c4iw_dev *dev, union t4_wr *wqe,
+			  struct ib_send_wr *wr, u8 *len16)
 {
+	struct c4iw_mr *mhp = get_mhp(dev, wr->ex.invalidate_rkey >> 8);
+
+	mhp->attr.state = 0;
 	wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
 	wqe->inv.r2 = 0;
 	*len16 = DIV_ROUND_UP(sizeof wqe->inv, 16);
 	return 0;
 }
 
-void _free_qp(struct kref *kref)
+static void _free_qp(struct kref *kref)
 {
 	struct c4iw_qp *qhp;
 
@@ -816,18 +851,32 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 			if (!qhp->wq.sq.oldest_read)
 				qhp->wq.sq.oldest_read = swsqe;
 			break;
-		case IB_WR_REG_MR:
-			fw_opcode = FW_RI_FR_NSMR_WR;
+		case IB_WR_REG_MR: {
+			struct c4iw_mr *mhp = to_c4iw_mr(reg_wr(wr)->mr);
+
 			swsqe->opcode = FW_RI_FAST_REGISTER;
-			err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr), &len16,
-				qhp->rhp->rdev.lldi.ulptx_memwrite_dsgl);
+			if (qhp->rhp->rdev.lldi.fr_nsmr_tpte_wr_support &&
+			    !mhp->attr.state && mhp->mpl_len <= 2) {
+				fw_opcode = FW_RI_FR_NSMR_TPTE_WR;
+				build_tpte_memreg(&wqe->fr_tpte, reg_wr(wr),
+						  mhp, &len16);
+			} else {
+				fw_opcode = FW_RI_FR_NSMR_WR;
+				err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr),
+				       mhp, &len16,
+				       qhp->rhp->rdev.lldi.ulptx_memwrite_dsgl);
+				if (err)
+					break;
+			}
+			mhp->attr.state = 1;
 			break;
+		}
 		case IB_WR_LOCAL_INV:
 			if (wr->send_flags & IB_SEND_FENCE)
 				fw_flags |= FW_RI_LOCAL_FENCE_FLAG;
 			fw_opcode = FW_RI_INV_LSTAG_WR;
 			swsqe->opcode = FW_RI_LOCAL_INV;
-			err = build_inv_stag(wqe, wr, &len16);
+			err = build_inv_stag(qhp->rhp, wqe, wr, &len16);
 			break;
 		default:
 			PDBG("%s post of type=%d TBD!\n", __func__,
diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h
index 6126bbe36095..862381aa83c8 100644
--- a/drivers/infiniband/hw/cxgb4/t4.h
+++ b/drivers/infiniband/hw/cxgb4/t4.h
@@ -95,6 +95,7 @@ union t4_wr {
 	struct fw_ri_rdma_read_wr read;
 	struct fw_ri_bind_mw_wr bind;
 	struct fw_ri_fr_nsmr_wr fr;
+	struct fw_ri_fr_nsmr_tpte_wr fr_tpte;
 	struct fw_ri_inv_lstag_wr inv;
 	struct t4_status_page status;
 	__be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_SQ_NUM_SLOTS];
@@ -170,7 +171,7 @@ struct t4_cqe {
 			__be32 msn;
 		} rcqe;
 		struct {
-			u32 nada1;
+			u32 stag;
 			u16 nada2;
 			u16 cidx;
 		} scqe;
@@ -232,6 +233,7 @@ struct t4_cqe {
 
 /* used for SQ completion processing */
 #define CQE_WRID_SQ_IDX(x)	((x)->u.scqe.cidx)
+#define CQE_WRID_FR_STAG(x)     (be32_to_cpu((x)->u.scqe.stag))
 
 /* generic accessor macros */
 #define CQE_WRID_HI(x)		(be32_to_cpu((x)->u.gen.wrid_hi))
@@ -634,6 +636,11 @@ static inline int t4_valid_cqe(struct t4_cq *cq, struct t4_cqe *cqe)
 	return (CQE_GENBIT(cqe) == cq->gen);
 }
 
+static inline int t4_cq_notempty(struct t4_cq *cq)
+{
+	return cq->sw_in_use || t4_valid_cqe(cq, &cq->queue[cq->cidx]);
+}
+
 static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
 {
 	int ret;
diff --git a/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h b/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h
index 1e26669793c3..010c709ba3bb 100644
--- a/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h
+++ b/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h
@@ -669,6 +669,18 @@ struct fw_ri_fr_nsmr_wr {
 #define FW_RI_FR_NSMR_WR_DCACPU_G(x)	\
 	(((x) >> FW_RI_FR_NSMR_WR_DCACPU_S) & FW_RI_FR_NSMR_WR_DCACPU_M)
 
+struct fw_ri_fr_nsmr_tpte_wr {
+	__u8	opcode;
+	__u8   flags;
+	__u16  wrid;
+	__u8   r1[3];
+	__u8   len16;
+	__u32  r2;
+	__u32  stag;
+	struct fw_ri_tpte tpte;
+	__u64  pbl[2];
+};
+
 struct fw_ri_inv_lstag_wr {
 	__u8   opcode;
 	__u8   flags;
diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c
index 79575ee873f2..a26a9a0bfc41 100644
--- a/drivers/infiniband/hw/hfi1/affinity.c
+++ b/drivers/infiniband/hw/hfi1/affinity.c
@@ -47,7 +47,7 @@
 #include <linux/topology.h>
 #include <linux/cpumask.h>
 #include <linux/module.h>
-#include <linux/cpumask.h>
+#include <linux/interrupt.h>
 
 #include "hfi.h"
 #include "affinity.h"
@@ -56,7 +56,7 @@
 
 struct hfi1_affinity_node_list node_affinity = {
 	.list = LIST_HEAD_INIT(node_affinity.list),
-	.lock = __SPIN_LOCK_UNLOCKED(&node_affinity.lock),
+	.lock = __MUTEX_INITIALIZER(node_affinity.lock)
 };
 
 /* Name of IRQ types, indexed by enum irq_type */
@@ -160,14 +160,14 @@ void node_affinity_destroy(void)
 	struct list_head *pos, *q;
 	struct hfi1_affinity_node *entry;
 
-	spin_lock(&node_affinity.lock);
+	mutex_lock(&node_affinity.lock);
 	list_for_each_safe(pos, q, &node_affinity.list) {
 		entry = list_entry(pos, struct hfi1_affinity_node,
 				   list);
 		list_del(pos);
 		kfree(entry);
 	}
-	spin_unlock(&node_affinity.lock);
+	mutex_unlock(&node_affinity.lock);
 	kfree(hfi1_per_node_cntr);
 }
 
@@ -234,9 +234,8 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
 	if (cpumask_first(local_mask) >= nr_cpu_ids)
 		local_mask = topology_core_cpumask(0);
 
-	spin_lock(&node_affinity.lock);
+	mutex_lock(&node_affinity.lock);
 	entry = node_affinity_lookup(dd->node);
-	spin_unlock(&node_affinity.lock);
 
 	/*
 	 * If this is the first time this NUMA node's affinity is used,
@@ -247,6 +246,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
 		if (!entry) {
 			dd_dev_err(dd,
 				   "Unable to allocate global affinity node\n");
+			mutex_unlock(&node_affinity.lock);
 			return -ENOMEM;
 		}
 		init_cpu_mask_set(&entry->def_intr);
@@ -303,15 +303,113 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
 					     &entry->general_intr_mask);
 		}
 
-		spin_lock(&node_affinity.lock);
 		node_affinity_add_tail(entry);
-		spin_unlock(&node_affinity.lock);
 	}
-
+	mutex_unlock(&node_affinity.lock);
 	return 0;
 }
 
-int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
+/*
+ * Function updates the irq affinity hint for msix after it has been changed
+ * by the user using the /proc/irq interface. This function only accepts
+ * one cpu in the mask.
+ */
+static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu)
+{
+	struct sdma_engine *sde = msix->arg;
+	struct hfi1_devdata *dd = sde->dd;
+	struct hfi1_affinity_node *entry;
+	struct cpu_mask_set *set;
+	int i, old_cpu;
+
+	if (cpu > num_online_cpus() || cpu == sde->cpu)
+		return;
+
+	mutex_lock(&node_affinity.lock);
+	entry = node_affinity_lookup(dd->node);
+	if (!entry)
+		goto unlock;
+
+	old_cpu = sde->cpu;
+	sde->cpu = cpu;
+	cpumask_clear(&msix->mask);
+	cpumask_set_cpu(cpu, &msix->mask);
+	dd_dev_dbg(dd, "IRQ vector: %u, type %s engine %u -> cpu: %d\n",
+		   msix->msix.vector, irq_type_names[msix->type],
+		   sde->this_idx, cpu);
+	irq_set_affinity_hint(msix->msix.vector, &msix->mask);
+
+	/*
+	 * Set the new cpu in the hfi1_affinity_node and clean
+	 * the old cpu if it is not used by any other IRQ
+	 */
+	set = &entry->def_intr;
+	cpumask_set_cpu(cpu, &set->mask);
+	cpumask_set_cpu(cpu, &set->used);
+	for (i = 0; i < dd->num_msix_entries; i++) {
+		struct hfi1_msix_entry *other_msix;
+
+		other_msix = &dd->msix_entries[i];
+		if (other_msix->type != IRQ_SDMA || other_msix == msix)
+			continue;
+
+		if (cpumask_test_cpu(old_cpu, &other_msix->mask))
+			goto unlock;
+	}
+	cpumask_clear_cpu(old_cpu, &set->mask);
+	cpumask_clear_cpu(old_cpu, &set->used);
+unlock:
+	mutex_unlock(&node_affinity.lock);
+}
+
+static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify,
+				     const cpumask_t *mask)
+{
+	int cpu = cpumask_first(mask);
+	struct hfi1_msix_entry *msix = container_of(notify,
+						    struct hfi1_msix_entry,
+						    notify);
+
+	/* Only one CPU configuration supported currently */
+	hfi1_update_sdma_affinity(msix, cpu);
+}
+
+static void hfi1_irq_notifier_release(struct kref *ref)
+{
+	/*
+	 * This is required by affinity notifier. We don't have anything to
+	 * free here.
+	 */
+}
+
+static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix)
+{
+	struct irq_affinity_notify *notify = &msix->notify;
+
+	notify->irq = msix->msix.vector;
+	notify->notify = hfi1_irq_notifier_notify;
+	notify->release = hfi1_irq_notifier_release;
+
+	if (irq_set_affinity_notifier(notify->irq, notify))
+		pr_err("Failed to register sdma irq affinity notifier for irq %d\n",
+		       notify->irq);
+}
+
+static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix)
+{
+	struct irq_affinity_notify *notify = &msix->notify;
+
+	if (irq_set_affinity_notifier(notify->irq, NULL))
+		pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n",
+		       notify->irq);
+}
+
+/*
+ * Function sets the irq affinity for msix.
+ * It *must* be called with node_affinity.lock held.
+ */
+static int get_irq_affinity(struct hfi1_devdata *dd,
+			    struct hfi1_msix_entry *msix)
 {
 	int ret;
 	cpumask_var_t diff;
@@ -329,9 +427,7 @@ int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
 	if (!ret)
 		return -ENOMEM;
 
-	spin_lock(&node_affinity.lock);
 	entry = node_affinity_lookup(dd->node);
-	spin_unlock(&node_affinity.lock);
 
 	switch (msix->type) {
 	case IRQ_SDMA:
@@ -361,7 +457,6 @@ int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
 	 * finds its CPU here.
 	 */
 	if (cpu == -1 && set) {
-		spin_lock(&node_affinity.lock);
 		if (cpumask_equal(&set->mask, &set->used)) {
 			/*
 			 * We've used up all the CPUs, bump up the generation
@@ -373,17 +468,6 @@ int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
 		cpumask_andnot(diff, &set->mask, &set->used);
 		cpu = cpumask_first(diff);
 		cpumask_set_cpu(cpu, &set->used);
-		spin_unlock(&node_affinity.lock);
-	}
-
-	switch (msix->type) {
-	case IRQ_SDMA:
-		sde->cpu = cpu;
-		break;
-	case IRQ_GENERAL:
-	case IRQ_RCVCTXT:
-	case IRQ_OTHER:
-		break;
 	}
 
 	cpumask_set_cpu(cpu, &msix->mask);
@@ -392,10 +476,25 @@ int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
 		    extra, cpu);
 	irq_set_affinity_hint(msix->msix.vector, &msix->mask);
 
+	if (msix->type == IRQ_SDMA) {
+		sde->cpu = cpu;
+		hfi1_setup_sdma_notifier(msix);
+	}
+
 	free_cpumask_var(diff);
 	return 0;
 }
 
+int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
+{
+	int ret;
+
+	mutex_lock(&node_affinity.lock);
+	ret = get_irq_affinity(dd, msix);
+	mutex_unlock(&node_affinity.lock);
+	return ret;
+}
+
 void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
 			   struct hfi1_msix_entry *msix)
 {
@@ -403,13 +502,13 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
 	struct hfi1_ctxtdata *rcd;
 	struct hfi1_affinity_node *entry;
 
-	spin_lock(&node_affinity.lock);
+	mutex_lock(&node_affinity.lock);
 	entry = node_affinity_lookup(dd->node);
-	spin_unlock(&node_affinity.lock);
 
 	switch (msix->type) {
 	case IRQ_SDMA:
 		set = &entry->def_intr;
+		hfi1_cleanup_sdma_notifier(msix);
 		break;
 	case IRQ_GENERAL:
 		/* Don't do accounting for general contexts */
@@ -421,21 +520,21 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
 			set = &entry->rcv_intr;
 		break;
 	default:
+		mutex_unlock(&node_affinity.lock);
 		return;
 	}
 
 	if (set) {
-		spin_lock(&node_affinity.lock);
 		cpumask_andnot(&set->used, &set->used, &msix->mask);
 		if (cpumask_empty(&set->used) && set->gen) {
 			set->gen--;
 			cpumask_copy(&set->used, &set->mask);
 		}
-		spin_unlock(&node_affinity.lock);
 	}
 
 	irq_set_affinity_hint(msix->msix.vector, NULL);
 	cpumask_clear(&msix->mask);
+	mutex_unlock(&node_affinity.lock);
 }
 
 /* This should be called with node_affinity.lock held */
@@ -536,7 +635,7 @@ int hfi1_get_proc_affinity(int node)
 	if (!ret)
 		goto free_available_mask;
 
-	spin_lock(&affinity->lock);
+	mutex_lock(&affinity->lock);
 	/*
 	 * If we've used all available HW threads, clear the mask and start
 	 * overloading.
@@ -644,7 +743,8 @@ int hfi1_get_proc_affinity(int node)
 		cpu = -1;
 	else
 		cpumask_set_cpu(cpu, &set->used);
-	spin_unlock(&affinity->lock);
+
+	mutex_unlock(&affinity->lock);
 	hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
 
 	free_cpumask_var(intrs_mask);
@@ -665,49 +765,53 @@ void hfi1_put_proc_affinity(int cpu)
 
 	if (cpu < 0)
 		return;
-	spin_lock(&affinity->lock);
+
+	mutex_lock(&affinity->lock);
 	cpumask_clear_cpu(cpu, &set->used);
 	hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
 	if (cpumask_empty(&set->used) && set->gen) {
 		set->gen--;
 		cpumask_copy(&set->used, &set->mask);
 	}
-	spin_unlock(&affinity->lock);
+	mutex_unlock(&affinity->lock);
 }
 
-/* Prevents concurrent reads and writes of the sdma_affinity attrib */
-static DEFINE_MUTEX(sdma_affinity_mutex);
-
 int hfi1_set_sdma_affinity(struct hfi1_devdata *dd, const char *buf,
 			   size_t count)
 {
 	struct hfi1_affinity_node *entry;
-	struct cpumask mask;
+	cpumask_var_t mask;
 	int ret, i;
 
-	spin_lock(&node_affinity.lock);
+	mutex_lock(&node_affinity.lock);
 	entry = node_affinity_lookup(dd->node);
-	spin_unlock(&node_affinity.lock);
 
-	if (!entry)
-		return -EINVAL;
+	if (!entry) {
+		ret = -EINVAL;
+		goto unlock;
+	}
 
-	ret = cpulist_parse(buf, &mask);
+	ret = zalloc_cpumask_var(&mask, GFP_KERNEL);
+	if (!ret) {
+		ret = -ENOMEM;
+		goto unlock;
+	}
+
+	ret = cpulist_parse(buf, mask);
 	if (ret)
-		return ret;
+		goto out;
 
-	if (!cpumask_subset(&mask, cpu_online_mask) || cpumask_empty(&mask)) {
+	if (!cpumask_subset(mask, cpu_online_mask) || cpumask_empty(mask)) {
 		dd_dev_warn(dd, "Invalid CPU mask\n");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 
-	mutex_lock(&sdma_affinity_mutex);
 	/* reset the SDMA interrupt affinity details */
 	init_cpu_mask_set(&entry->def_intr);
-	cpumask_copy(&entry->def_intr.mask, &mask);
-	/*
-	 * Reassign the affinity for each SDMA interrupt.
-	 */
+	cpumask_copy(&entry->def_intr.mask, mask);
+
+	/* Reassign the affinity for each SDMA interrupt. */
 	for (i = 0; i < dd->num_msix_entries; i++) {
 		struct hfi1_msix_entry *msix;
 
@@ -715,13 +819,15 @@ int hfi1_set_sdma_affinity(struct hfi1_devdata *dd, const char *buf,
 		if (msix->type != IRQ_SDMA)
 			continue;
 
-		ret = hfi1_get_irq_affinity(dd, msix);
+		ret = get_irq_affinity(dd, msix);
 
 		if (ret)
 			break;
 	}
-
-	mutex_unlock(&sdma_affinity_mutex);
+out:
+	free_cpumask_var(mask);
+unlock:
+	mutex_unlock(&node_affinity.lock);
 	return ret ? ret : strnlen(buf, PAGE_SIZE);
 }
 
@@ -729,15 +835,15 @@ int hfi1_get_sdma_affinity(struct hfi1_devdata *dd, char *buf)
 {
 	struct hfi1_affinity_node *entry;
 
-	spin_lock(&node_affinity.lock);
+	mutex_lock(&node_affinity.lock);
 	entry = node_affinity_lookup(dd->node);
-	spin_unlock(&node_affinity.lock);
 
-	if (!entry)
+	if (!entry) {
+		mutex_unlock(&node_affinity.lock);
 		return -EINVAL;
+	}
 
-	mutex_lock(&sdma_affinity_mutex);
 	cpumap_print_to_pagebuf(true, buf, &entry->def_intr.mask);
-	mutex_unlock(&sdma_affinity_mutex);
+	mutex_unlock(&node_affinity.lock);
 	return strnlen(buf, PAGE_SIZE);
 }
diff --git a/drivers/infiniband/hw/hfi1/affinity.h b/drivers/infiniband/hw/hfi1/affinity.h
index 8879cf7a8cac..b89ea3c0ee1a 100644
--- a/drivers/infiniband/hw/hfi1/affinity.h
+++ b/drivers/infiniband/hw/hfi1/affinity.h
@@ -121,8 +121,7 @@ struct hfi1_affinity_node_list {
 	int num_core_siblings;
 	int num_online_nodes;
 	int num_online_cpus;
-	/* protect affinity node list */
-	spinlock_t lock;
+	struct mutex lock; /* protects affinity nodes */
 };
 
 int node_affinity_init(void);
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index b32638d58ae8..9bf5f23544d4 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -971,7 +971,9 @@ static struct flag_table dc8051_info_err_flags[] = {
 	FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
 	FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
 	FLAG_ENTRY0("Failed LNI(ConfigLT)",    FAILED_LNI_CONFIGLT),
-	FLAG_ENTRY0("Host Handshake Timeout",  HOST_HANDSHAKE_TIMEOUT)
+	FLAG_ENTRY0("Host Handshake Timeout",  HOST_HANDSHAKE_TIMEOUT),
+	FLAG_ENTRY0("External Device Request Timeout",
+		    EXTERNAL_DEVICE_REQ_TIMEOUT),
 };
 
 /*
@@ -6825,7 +6827,6 @@ void handle_link_up(struct work_struct *work)
 		set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
 				     OPA_LINKDOWN_REASON_SPEED_POLICY);
 		set_link_state(ppd, HLS_DN_OFFLINE);
-		tune_serdes(ppd);
 		start_link(ppd);
 	}
 }
@@ -6998,12 +6999,10 @@ void handle_link_down(struct work_struct *work)
 	 * If there is no cable attached, turn the DC off. Otherwise,
 	 * start the link bring up.
 	 */
-	if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd)) {
+	if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd))
 		dc_shutdown(ppd->dd);
-	} else {
-		tune_serdes(ppd);
+	else
 		start_link(ppd);
-	}
 }
 
 void handle_link_bounce(struct work_struct *work)
@@ -7016,7 +7015,6 @@ void handle_link_bounce(struct work_struct *work)
 	 */
 	if (ppd->host_link_state & HLS_UP) {
 		set_link_state(ppd, HLS_DN_OFFLINE);
-		tune_serdes(ppd);
 		start_link(ppd);
 	} else {
 		dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
@@ -7531,7 +7529,6 @@ done:
 		set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
 				     OPA_LINKDOWN_REASON_WIDTH_POLICY);
 		set_link_state(ppd, HLS_DN_OFFLINE);
-		tune_serdes(ppd);
 		start_link(ppd);
 	}
 }
@@ -9161,6 +9158,12 @@ set_local_link_attributes_fail:
  */
 int start_link(struct hfi1_pportdata *ppd)
 {
+	/*
+	 * Tune the SerDes to a ballpark setting for optimal signal and bit
+	 * error rate.  Needs to be done before starting the link.
+	 */
+	tune_serdes(ppd);
+
 	if (!ppd->link_enabled) {
 		dd_dev_info(ppd->dd,
 			    "%s: stopping link start because link is disabled\n",
@@ -9401,8 +9404,6 @@ void qsfp_event(struct work_struct *work)
 		 */
 		set_qsfp_int_n(ppd, 1);
 
-		tune_serdes(ppd);
-
 		start_link(ppd);
 	}
 
@@ -9490,6 +9491,73 @@ static void init_lcb(struct hfi1_devdata *dd)
 	write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
 }
 
+/*
+ * Perform a test read on the QSFP.  Return 0 on success, -ERRNO
+ * on error.
+ */
+static int test_qsfp_read(struct hfi1_pportdata *ppd)
+{
+	int ret;
+	u8 status;
+
+	/* report success if not a QSFP */
+	if (ppd->port_type != PORT_TYPE_QSFP)
+		return 0;
+
+	/* read byte 2, the status byte */
+	ret = one_qsfp_read(ppd, ppd->dd->hfi1_id, 2, &status, 1);
+	if (ret < 0)
+		return ret;
+	if (ret != 1)
+		return -EIO;
+
+	return 0; /* success */
+}
+
+/*
+ * Values for QSFP retry.
+ *
+ * Give up after 10s (20 x 500ms).  The overall timeout was empirically
+ * arrived at from experience on a large cluster.
+ */
+#define MAX_QSFP_RETRIES 20
+#define QSFP_RETRY_WAIT 500 /* msec */
+
+/*
+ * Try a QSFP read.  If it fails, schedule a retry for later.
+ * Called on first link activation after driver load.
+ */
+static void try_start_link(struct hfi1_pportdata *ppd)
+{
+	if (test_qsfp_read(ppd)) {
+		/* read failed */
+		if (ppd->qsfp_retry_count >= MAX_QSFP_RETRIES) {
+			dd_dev_err(ppd->dd, "QSFP not responding, giving up\n");
+			return;
+		}
+		dd_dev_info(ppd->dd,
+			    "QSFP not responding, waiting and retrying %d\n",
+			    (int)ppd->qsfp_retry_count);
+		ppd->qsfp_retry_count++;
+		queue_delayed_work(ppd->hfi1_wq, &ppd->start_link_work,
+				   msecs_to_jiffies(QSFP_RETRY_WAIT));
+		return;
+	}
+	ppd->qsfp_retry_count = 0;
+
+	start_link(ppd);
+}
+
+/*
+ * Workqueue function to start the link after a delay.
+ */
+void handle_start_link(struct work_struct *work)
+{
+	struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
+						  start_link_work.work);
+	try_start_link(ppd);
+}
+
 int bringup_serdes(struct hfi1_pportdata *ppd)
 {
 	struct hfi1_devdata *dd = ppd->dd;
@@ -9525,14 +9593,8 @@ int bringup_serdes(struct hfi1_pportdata *ppd)
 		set_qsfp_int_n(ppd, 1);
 	}
 
-	/*
-	 * Tune the SerDes to a ballpark setting for
-	 * optimal signal and bit error rate
-	 * Needs to be done before starting the link
-	 */
-	tune_serdes(ppd);
-
-	return start_link(ppd);
+	try_start_link(ppd);
+	return 0;
 }
 
 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
@@ -9549,6 +9611,10 @@ void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
 	ppd->driver_link_ready = 0;
 	ppd->link_enabled = 0;
 
+	ppd->qsfp_retry_count = MAX_QSFP_RETRIES; /* prevent more retries */
+	flush_delayed_work(&ppd->start_link_work);
+	cancel_delayed_work_sync(&ppd->start_link_work);
+
 	ppd->offline_disabled_reason =
 			HFI1_ODR_MASK(OPA_LINKDOWN_REASON_SMA_DISABLED);
 	set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SMA_DISABLED, 0,
@@ -9648,12 +9714,12 @@ void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
 		hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
 }
 
-struct hfi1_message_header *hfi1_get_msgheader(
-				struct hfi1_devdata *dd, __le32 *rhf_addr)
+struct ib_header *hfi1_get_msgheader(
+	struct hfi1_devdata *dd, __le32 *rhf_addr)
 {
 	u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
 
-	return (struct hfi1_message_header *)
+	return (struct ib_header *)
 		(rhf_addr - dd->rhf_offset + offset);
 }
 
@@ -11489,10 +11555,10 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
 	    !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
 		/* reset the tail and hdr addresses, and sequence count */
 		write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
-				rcd->rcvhdrq_phys);
+				rcd->rcvhdrq_dma);
 		if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
 			write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
-					rcd->rcvhdrqtailaddr_phys);
+					rcd->rcvhdrqtailaddr_dma);
 		rcd->seq_cnt = 1;
 
 		/* reset the cached receive header queue head value */
@@ -11557,9 +11623,9 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
 		 * update with a dummy tail address and then disable
 		 * receive context.
 		 */
-		if (dd->rcvhdrtail_dummy_physaddr) {
+		if (dd->rcvhdrtail_dummy_dma) {
 			write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
-					dd->rcvhdrtail_dummy_physaddr);
+					dd->rcvhdrtail_dummy_dma);
 			/* Enabling RcvCtxtCtrl.TailUpd is intentional. */
 			rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
 		}
@@ -11570,7 +11636,7 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
 		rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
 	if (op & HFI1_RCVCTRL_INTRAVAIL_DIS)
 		rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
-	if (op & HFI1_RCVCTRL_TAILUPD_ENB && rcd->rcvhdrqtailaddr_phys)
+	if (op & HFI1_RCVCTRL_TAILUPD_ENB && rcd->rcvhdrqtailaddr_dma)
 		rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
 	if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
 		/* See comment on RcvCtxtCtrl.TailUpd above */
@@ -11642,7 +11708,7 @@ void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
 		 * so it doesn't contain an address that is invalid.
 		 */
 		write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
-				dd->rcvhdrtail_dummy_physaddr);
+				dd->rcvhdrtail_dummy_dma);
 }
 
 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
@@ -12865,7 +12931,7 @@ fail:
  */
 static int set_up_context_variables(struct hfi1_devdata *dd)
 {
-	int num_kernel_contexts;
+	unsigned long num_kernel_contexts;
 	int total_contexts;
 	int ret;
 	unsigned ngroups;
@@ -12894,9 +12960,9 @@ static int set_up_context_variables(struct hfi1_devdata *dd)
 	 */
 	if (num_kernel_contexts > (dd->chip_send_contexts - num_vls - 1)) {
 		dd_dev_err(dd,
-			   "Reducing # kernel rcv contexts to: %d, from %d\n",
+			   "Reducing # kernel rcv contexts to: %d, from %lu\n",
 			   (int)(dd->chip_send_contexts - num_vls - 1),
-			   (int)num_kernel_contexts);
+			   num_kernel_contexts);
 		num_kernel_contexts = dd->chip_send_contexts - num_vls - 1;
 	}
 	/*
@@ -13319,9 +13385,9 @@ static void init_rbufs(struct hfi1_devdata *dd)
 		/*
 		 * Give up after 1ms - maximum wait time.
 		 *
-		 * RBuf size is 148KiB.  Slowest possible is PCIe Gen1 x1 at
+		 * RBuf size is 136KiB.  Slowest possible is PCIe Gen1 x1 at
 		 * 250MB/s bandwidth.  Lower rate to 66% for overhead to get:
-		 *	148 KB / (66% * 250MB/s) = 920us
+		 *	136 KB / (66% * 250MB/s) = 844us
 		 */
 		if (count++ > 500) {
 			dd_dev_err(dd,
@@ -14500,6 +14566,11 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 	if (ret)
 		goto bail_cleanup;
 
+	/* call before get_platform_config(), after init_chip_resources() */
+	ret = eprom_init(dd);
+	if (ret)
+		goto bail_free_rcverr;
+
 	/* Needs to be called before hfi1_firmware_init */
 	get_platform_config(dd);
 
@@ -14620,10 +14691,6 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 	if (ret)
 		goto bail_free_cntrs;
 
-	ret = eprom_init(dd);
-	if (ret)
-		goto bail_free_rcverr;
-
 	goto bail;
 
 bail_free_rcverr:
diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h
index ed11107c50fe..92345259a8f4 100644
--- a/drivers/infiniband/hw/hfi1/chip.h
+++ b/drivers/infiniband/hw/hfi1/chip.h
@@ -82,7 +82,7 @@
  */
 #define CM_VAU 3
 /* HFI link credit count, AKA receive buffer depth (RBUF_DEPTH) */
-#define CM_GLOBAL_CREDITS 0x940
+#define CM_GLOBAL_CREDITS 0x880
 /* Number of PKey entries in the HW */
 #define MAX_PKEY_VALUES 16
 
@@ -254,12 +254,14 @@
 #define FAILED_LNI_VERIFY_CAP2		BIT(10)
 #define FAILED_LNI_CONFIGLT		BIT(11)
 #define HOST_HANDSHAKE_TIMEOUT		BIT(12)
+#define EXTERNAL_DEVICE_REQ_TIMEOUT	BIT(13)
 
 #define FAILED_LNI (FAILED_LNI_POLLING | FAILED_LNI_DEBOUNCE \
 			| FAILED_LNI_ESTBCOMM | FAILED_LNI_OPTEQ \
 			| FAILED_LNI_VERIFY_CAP1 \
 			| FAILED_LNI_VERIFY_CAP2 \
-			| FAILED_LNI_CONFIGLT | HOST_HANDSHAKE_TIMEOUT)
+			| FAILED_LNI_CONFIGLT | HOST_HANDSHAKE_TIMEOUT \
+			| EXTERNAL_DEVICE_REQ_TIMEOUT)
 
 /* DC_DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG - host message flags */
 #define HOST_REQ_DONE		BIT(0)
@@ -706,6 +708,7 @@ void handle_link_up(struct work_struct *work);
 void handle_link_down(struct work_struct *work);
 void handle_link_downgrade(struct work_struct *work);
 void handle_link_bounce(struct work_struct *work);
+void handle_start_link(struct work_struct *work);
 void handle_sma_message(struct work_struct *work);
 void reset_qsfp(struct hfi1_pportdata *ppd);
 void qsfp_event(struct work_struct *work);
@@ -1335,7 +1338,7 @@ enum {
 u64 get_all_cpu_total(u64 __percpu *cntr);
 void hfi1_start_cleanup(struct hfi1_devdata *dd);
 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd);
-struct hfi1_message_header *hfi1_get_msgheader(
+struct ib_header *hfi1_get_msgheader(
 				struct hfi1_devdata *dd, __le32 *rhf_addr);
 int hfi1_init_ctxt(struct send_context *sc);
 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
diff --git a/drivers/infiniband/hw/hfi1/common.h b/drivers/infiniband/hw/hfi1/common.h
index fcc9c217a97a..da7be21bedb4 100644
--- a/drivers/infiniband/hw/hfi1/common.h
+++ b/drivers/infiniband/hw/hfi1/common.h
@@ -320,14 +320,6 @@ struct diag_pkt {
 /* RHF receive type error - bypass packet errors */
 #define RHF_RTE_BYPASS_NO_ERR		0x0
 
-/*
- * This structure contains the first field common to all protocols
- * that employ this chip.
- */
-struct hfi1_message_header {
-	__be16 lrh[4];
-};
-
 /* IB - LRH header constants */
 #define HFI1_LRH_GRH 0x0003      /* 1. word of IB LRH - next header: GRH */
 #define HFI1_LRH_BTH 0x0002      /* 1. word of IB LRH - next header: BTH */
diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c
index dbab9d9cc288..632ba21759ab 100644
--- a/drivers/infiniband/hw/hfi1/debugfs.c
+++ b/drivers/infiniband/hw/hfi1/debugfs.c
@@ -59,6 +59,40 @@
 
 static struct dentry *hfi1_dbg_root;
 
+/* wrappers to enforce srcu in seq file */
+static ssize_t hfi1_seq_read(
+	struct file *file,
+	char __user *buf,
+	size_t size,
+	loff_t *ppos)
+{
+	struct dentry *d = file->f_path.dentry;
+	int srcu_idx;
+	ssize_t r;
+
+	r = debugfs_use_file_start(d, &srcu_idx);
+	if (likely(!r))
+		r = seq_read(file, buf, size, ppos);
+	debugfs_use_file_finish(srcu_idx);
+	return r;
+}
+
+static loff_t hfi1_seq_lseek(
+	struct file *file,
+	loff_t offset,
+	int whence)
+{
+	struct dentry *d = file->f_path.dentry;
+	int srcu_idx;
+	loff_t r;
+
+	r = debugfs_use_file_start(d, &srcu_idx);
+	if (likely(!r))
+		r = seq_lseek(file, offset, whence);
+	debugfs_use_file_finish(srcu_idx);
+	return r;
+}
+
 #define private2dd(file) (file_inode(file)->i_private)
 #define private2ppd(file) (file_inode(file)->i_private)
 
@@ -87,8 +121,8 @@ static int _##name##_open(struct inode *inode, struct file *s) \
 static const struct file_operations _##name##_file_ops = { \
 	.owner   = THIS_MODULE, \
 	.open    = _##name##_open, \
-	.read    = seq_read, \
-	.llseek  = seq_lseek, \
+	.read    = hfi1_seq_read, \
+	.llseek  = hfi1_seq_lseek, \
 	.release = seq_release \
 }
 
@@ -105,11 +139,9 @@ do { \
 	DEBUGFS_FILE_CREATE(#name, parent, data, &_##name##_file_ops, S_IRUGO)
 
 static void *_opcode_stats_seq_start(struct seq_file *s, loff_t *pos)
-__acquires(RCU)
 {
 	struct hfi1_opcode_stats_perctx *opstats;
 
-	rcu_read_lock();
 	if (*pos >= ARRAY_SIZE(opstats->stats))
 		return NULL;
 	return pos;
@@ -126,9 +158,7 @@ static void *_opcode_stats_seq_next(struct seq_file *s, void *v, loff_t *pos)
 }
 
 static void _opcode_stats_seq_stop(struct seq_file *s, void *v)
-__releases(RCU)
 {
-	rcu_read_unlock();
 }
 
 static int _opcode_stats_seq_show(struct seq_file *s, void *v)
@@ -223,28 +253,32 @@ DEBUGFS_SEQ_FILE_OPEN(ctx_stats)
 DEBUGFS_FILE_OPS(ctx_stats);
 
 static void *_qp_stats_seq_start(struct seq_file *s, loff_t *pos)
-__acquires(RCU)
+	__acquires(RCU)
 {
 	struct qp_iter *iter;
 	loff_t n = *pos;
 
-	rcu_read_lock();
 	iter = qp_iter_init(s->private);
+
+	/* stop calls rcu_read_unlock */
+	rcu_read_lock();
+
 	if (!iter)
 		return NULL;
 
-	while (n--) {
+	do {
 		if (qp_iter_next(iter)) {
 			kfree(iter);
 			return NULL;
 		}
-	}
+	} while (n--);
 
 	return iter;
 }
 
 static void *_qp_stats_seq_next(struct seq_file *s, void *iter_ptr,
 				loff_t *pos)
+	__must_hold(RCU)
 {
 	struct qp_iter *iter = iter_ptr;
 
@@ -259,7 +293,7 @@ static void *_qp_stats_seq_next(struct seq_file *s, void *iter_ptr,
 }
 
 static void _qp_stats_seq_stop(struct seq_file *s, void *iter_ptr)
-__releases(RCU)
+	__releases(RCU)
 {
 	rcu_read_unlock();
 }
@@ -281,12 +315,10 @@ DEBUGFS_SEQ_FILE_OPEN(qp_stats)
 DEBUGFS_FILE_OPS(qp_stats);
 
 static void *_sdes_seq_start(struct seq_file *s, loff_t *pos)
-__acquires(RCU)
 {
 	struct hfi1_ibdev *ibd;
 	struct hfi1_devdata *dd;
 
-	rcu_read_lock();
 	ibd = (struct hfi1_ibdev *)s->private;
 	dd = dd_from_dev(ibd);
 	if (!dd->per_sdma || *pos >= dd->num_sdma)
@@ -306,9 +338,7 @@ static void *_sdes_seq_next(struct seq_file *s, void *v, loff_t *pos)
 }
 
 static void _sdes_seq_stop(struct seq_file *s, void *v)
-__releases(RCU)
 {
-	rcu_read_unlock();
 }
 
 static int _sdes_seq_show(struct seq_file *s, void *v)
@@ -335,11 +365,9 @@ static ssize_t dev_counters_read(struct file *file, char __user *buf,
 	struct hfi1_devdata *dd;
 	ssize_t rval;
 
-	rcu_read_lock();
 	dd = private2dd(file);
 	avail = hfi1_read_cntrs(dd, NULL, &counters);
 	rval =  simple_read_from_buffer(buf, count, ppos, counters, avail);
-	rcu_read_unlock();
 	return rval;
 }
 
@@ -352,11 +380,9 @@ static ssize_t dev_names_read(struct file *file, char __user *buf,
 	struct hfi1_devdata *dd;
 	ssize_t rval;
 
-	rcu_read_lock();
 	dd = private2dd(file);
 	avail = hfi1_read_cntrs(dd, &names, NULL);
 	rval =  simple_read_from_buffer(buf, count, ppos, names, avail);
-	rcu_read_unlock();
 	return rval;
 }
 
@@ -379,11 +405,9 @@ static ssize_t portnames_read(struct file *file, char __user *buf,
 	struct hfi1_devdata *dd;
 	ssize_t rval;
 
-	rcu_read_lock();
 	dd = private2dd(file);
 	avail = hfi1_read_portcntrs(dd->pport, &names, NULL);
 	rval = simple_read_from_buffer(buf, count, ppos, names, avail);
-	rcu_read_unlock();
 	return rval;
 }
 
@@ -396,11 +420,9 @@ static ssize_t portcntrs_debugfs_read(struct file *file, char __user *buf,
 	struct hfi1_pportdata *ppd;
 	ssize_t rval;
 
-	rcu_read_lock();
 	ppd = private2ppd(file);
 	avail = hfi1_read_portcntrs(ppd, NULL, &counters);
 	rval = simple_read_from_buffer(buf, count, ppos, counters, avail);
-	rcu_read_unlock();
 	return rval;
 }
 
@@ -430,16 +452,13 @@ static ssize_t asic_flags_read(struct file *file, char __user *buf,
 	int used;
 	int i;
 
-	rcu_read_lock();
 	ppd = private2ppd(file);
 	dd = ppd->dd;
 	size = PAGE_SIZE;
 	used = 0;
 	tmp = kmalloc(size, GFP_KERNEL);
-	if (!tmp) {
-		rcu_read_unlock();
+	if (!tmp)
 		return -ENOMEM;
-	}
 
 	scratch0 = read_csr(dd, ASIC_CFG_SCRATCH);
 	used += scnprintf(tmp + used, size - used,
@@ -466,7 +485,6 @@ static ssize_t asic_flags_read(struct file *file, char __user *buf,
 	used += scnprintf(tmp + used, size - used, "Write bits to clear\n");
 
 	ret = simple_read_from_buffer(buf, count, ppos, tmp, used);
-	rcu_read_unlock();
 	kfree(tmp);
 	return ret;
 }
@@ -482,15 +500,12 @@ static ssize_t asic_flags_write(struct file *file, const char __user *buf,
 	u64 scratch0;
 	u64 clear;
 
-	rcu_read_lock();
 	ppd = private2ppd(file);
 	dd = ppd->dd;
 
 	buff = kmalloc(count + 1, GFP_KERNEL);
-	if (!buff) {
-		ret = -ENOMEM;
-		goto do_return;
-	}
+	if (!buff)
+		return -ENOMEM;
 
 	ret = copy_from_user(buff, buf, count);
 	if (ret > 0) {
@@ -523,8 +538,6 @@ static ssize_t asic_flags_write(struct file *file, const char __user *buf,
 
  do_free:
 	kfree(buff);
- do_return:
-	rcu_read_unlock();
 	return ret;
 }
 
@@ -538,18 +551,14 @@ static ssize_t qsfp_debugfs_dump(struct file *file, char __user *buf,
 	char *tmp;
 	int ret;
 
-	rcu_read_lock();
 	ppd = private2ppd(file);
 	tmp = kmalloc(PAGE_SIZE, GFP_KERNEL);
-	if (!tmp) {
-		rcu_read_unlock();
+	if (!tmp)
 		return -ENOMEM;
-	}
 
 	ret = qsfp_dump(ppd, tmp, PAGE_SIZE);
 	if (ret > 0)
 		ret = simple_read_from_buffer(buf, count, ppos, tmp, ret);
-	rcu_read_unlock();
 	kfree(tmp);
 	return ret;
 }
@@ -565,7 +574,6 @@ static ssize_t __i2c_debugfs_write(struct file *file, const char __user *buf,
 	int offset;
 	int total_written;
 
-	rcu_read_lock();
 	ppd = private2ppd(file);
 
 	/* byte offset format: [offsetSize][i2cAddr][offsetHigh][offsetLow] */
@@ -573,16 +581,12 @@ static ssize_t __i2c_debugfs_write(struct file *file, const char __user *buf,
 	offset = *ppos & 0xffff;
 
 	/* explicitly reject invalid address 0 to catch cp and cat */
-	if (i2c_addr == 0) {
-		ret = -EINVAL;
-		goto _return;
-	}
+	if (i2c_addr == 0)
+		return -EINVAL;
 
 	buff = kmalloc(count, GFP_KERNEL);
-	if (!buff) {
-		ret = -ENOMEM;
-		goto _return;
-	}
+	if (!buff)
+		return -ENOMEM;
 
 	ret = copy_from_user(buff, buf, count);
 	if (ret > 0) {
@@ -602,8 +606,6 @@ static ssize_t __i2c_debugfs_write(struct file *file, const char __user *buf,
 
  _free:
 	kfree(buff);
- _return:
-	rcu_read_unlock();
 	return ret;
 }
 
@@ -632,7 +634,6 @@ static ssize_t __i2c_debugfs_read(struct file *file, char __user *buf,
 	int offset;
 	int total_read;
 
-	rcu_read_lock();
 	ppd = private2ppd(file);
 
 	/* byte offset format: [offsetSize][i2cAddr][offsetHigh][offsetLow] */
@@ -640,16 +641,12 @@ static ssize_t __i2c_debugfs_read(struct file *file, char __user *buf,
 	offset = *ppos & 0xffff;
 
 	/* explicitly reject invalid address 0 to catch cp and cat */
-	if (i2c_addr == 0) {
-		ret = -EINVAL;
-		goto _return;
-	}
+	if (i2c_addr == 0)
+		return -EINVAL;
 
 	buff = kmalloc(count, GFP_KERNEL);
-	if (!buff) {
-		ret = -ENOMEM;
-		goto _return;
-	}
+	if (!buff)
+		return -ENOMEM;
 
 	total_read = i2c_read(ppd, target, i2c_addr, offset, buff, count);
 	if (total_read < 0) {
@@ -669,8 +666,6 @@ static ssize_t __i2c_debugfs_read(struct file *file, char __user *buf,
 
  _free:
 	kfree(buff);
- _return:
-	rcu_read_unlock();
 	return ret;
 }
 
@@ -697,26 +692,20 @@ static ssize_t __qsfp_debugfs_write(struct file *file, const char __user *buf,
 	int ret;
 	int total_written;
 
-	rcu_read_lock();
-	if (*ppos + count > QSFP_PAGESIZE * 4) { /* base page + page00-page03 */
-		ret = -EINVAL;
-		goto _return;
-	}
+	if (*ppos + count > QSFP_PAGESIZE * 4) /* base page + page00-page03 */
+		return -EINVAL;
 
 	ppd = private2ppd(file);
 
 	buff = kmalloc(count, GFP_KERNEL);
-	if (!buff) {
-		ret = -ENOMEM;
-		goto _return;
-	}
+	if (!buff)
+		return -ENOMEM;
 
 	ret = copy_from_user(buff, buf, count);
 	if (ret > 0) {
 		ret = -EFAULT;
 		goto _free;
 	}
-
 	total_written = qsfp_write(ppd, target, *ppos, buff, count);
 	if (total_written < 0) {
 		ret = total_written;
@@ -729,8 +718,6 @@ static ssize_t __qsfp_debugfs_write(struct file *file, const char __user *buf,
 
  _free:
 	kfree(buff);
- _return:
-	rcu_read_unlock();
 	return ret;
 }
 
@@ -757,7 +744,6 @@ static ssize_t __qsfp_debugfs_read(struct file *file, char __user *buf,
 	int ret;
 	int total_read;
 
-	rcu_read_lock();
 	if (*ppos + count > QSFP_PAGESIZE * 4) { /* base page + page00-page03 */
 		ret = -EINVAL;
 		goto _return;
@@ -790,7 +776,6 @@ static ssize_t __qsfp_debugfs_read(struct file *file, char __user *buf,
  _free:
 	kfree(buff);
  _return:
-	rcu_read_unlock();
 	return ret;
 }
 
@@ -948,6 +933,43 @@ static const struct counter_info port_cntr_ops[] = {
 	DEBUGFS_OPS("asic_flags", asic_flags_read, asic_flags_write),
 };
 
+static void *_sdma_cpu_list_seq_start(struct seq_file *s, loff_t *pos)
+{
+	if (*pos >= num_online_cpus())
+		return NULL;
+
+	return pos;
+}
+
+static void *_sdma_cpu_list_seq_next(struct seq_file *s, void *v, loff_t *pos)
+{
+	++*pos;
+	if (*pos >= num_online_cpus())
+		return NULL;
+
+	return pos;
+}
+
+static void _sdma_cpu_list_seq_stop(struct seq_file *s, void *v)
+{
+	/* nothing allocated */
+}
+
+static int _sdma_cpu_list_seq_show(struct seq_file *s, void *v)
+{
+	struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
+	struct hfi1_devdata *dd = dd_from_dev(ibd);
+	loff_t *spos = v;
+	loff_t i = *spos;
+
+	sdma_seqfile_dump_cpu_list(s, dd, (unsigned long)i);
+	return 0;
+}
+
+DEBUGFS_SEQ_FILE_OPS(sdma_cpu_list);
+DEBUGFS_SEQ_FILE_OPEN(sdma_cpu_list)
+DEBUGFS_FILE_OPS(sdma_cpu_list);
+
 void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd)
 {
 	char name[sizeof("port0counters") + 1];
@@ -976,6 +998,7 @@ void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd)
 	DEBUGFS_SEQ_FILE_CREATE(ctx_stats, ibd->hfi1_ibdev_dbg, ibd);
 	DEBUGFS_SEQ_FILE_CREATE(qp_stats, ibd->hfi1_ibdev_dbg, ibd);
 	DEBUGFS_SEQ_FILE_CREATE(sdes, ibd->hfi1_ibdev_dbg, ibd);
+	DEBUGFS_SEQ_FILE_CREATE(sdma_cpu_list, ibd->hfi1_ibdev_dbg, ibd);
 	/* dev counter files */
 	for (i = 0; i < ARRAY_SIZE(cntr_ops); i++)
 		DEBUGFS_FILE_CREATE(cntr_ops[i].name,
@@ -1006,7 +1029,6 @@ void hfi1_dbg_ibdev_exit(struct hfi1_ibdev *ibd)
 	debugfs_remove_recursive(ibd->hfi1_ibdev_dbg);
 out:
 	ibd->hfi1_ibdev_dbg = NULL;
-	synchronize_rcu();
 }
 
 /*
@@ -1031,9 +1053,7 @@ static const char * const hfi1_statnames[] = {
 };
 
 static void *_driver_stats_names_seq_start(struct seq_file *s, loff_t *pos)
-__acquires(RCU)
 {
-	rcu_read_lock();
 	if (*pos >= ARRAY_SIZE(hfi1_statnames))
 		return NULL;
 	return pos;
@@ -1051,9 +1071,7 @@ static void *_driver_stats_names_seq_next(
 }
 
 static void _driver_stats_names_seq_stop(struct seq_file *s, void *v)
-__releases(RCU)
 {
-	rcu_read_unlock();
 }
 
 static int _driver_stats_names_seq_show(struct seq_file *s, void *v)
@@ -1069,9 +1087,7 @@ DEBUGFS_SEQ_FILE_OPEN(driver_stats_names)
 DEBUGFS_FILE_OPS(driver_stats_names);
 
 static void *_driver_stats_seq_start(struct seq_file *s, loff_t *pos)
-__acquires(RCU)
 {
-	rcu_read_lock();
 	if (*pos >= ARRAY_SIZE(hfi1_statnames))
 		return NULL;
 	return pos;
@@ -1086,9 +1102,7 @@ static void *_driver_stats_seq_next(struct seq_file *s, void *v, loff_t *pos)
 }
 
 static void _driver_stats_seq_stop(struct seq_file *s, void *v)
-__releases(RCU)
 {
-	rcu_read_unlock();
 }
 
 static u64 hfi1_sps_ints(void)
diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index 8246dc7d0573..6563e4d38b80 100644
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -276,7 +276,7 @@ inline int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded)
 static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 		       struct hfi1_packet *packet)
 {
-	struct hfi1_message_header *rhdr = packet->hdr;
+	struct ib_header *rhdr = packet->hdr;
 	u32 rte = rhf_rcv_type_err(packet->rhf);
 	int lnh = be16_to_cpu(rhdr->lrh[0]) & 3;
 	struct hfi1_ibport *ibp = &ppd->ibport_data;
@@ -288,10 +288,9 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 
 	if (packet->rhf & RHF_TID_ERR) {
 		/* For TIDERR and RC QPs preemptively schedule a NAK */
-		struct hfi1_ib_header *hdr = (struct hfi1_ib_header *)rhdr;
-		struct hfi1_other_headers *ohdr = NULL;
+		struct ib_other_headers *ohdr = NULL;
 		u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */
-		u16 lid  = be16_to_cpu(hdr->lrh[1]);
+		u16 lid  = be16_to_cpu(rhdr->lrh[1]);
 		u32 qp_num;
 		u32 rcv_flags = 0;
 
@@ -301,14 +300,14 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 
 		/* Check for GRH */
 		if (lnh == HFI1_LRH_BTH) {
-			ohdr = &hdr->u.oth;
+			ohdr = &rhdr->u.oth;
 		} else if (lnh == HFI1_LRH_GRH) {
 			u32 vtf;
 
-			ohdr = &hdr->u.l.oth;
-			if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
+			ohdr = &rhdr->u.l.oth;
+			if (rhdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
 				goto drop;
-			vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow);
+			vtf = be32_to_cpu(rhdr->u.l.grh.version_tclass_flow);
 			if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
 				goto drop;
 			rcv_flags |= HFI1_HAS_GRH;
@@ -344,7 +343,7 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 			case IB_QPT_RC:
 				hfi1_rc_hdrerr(
 					rcd,
-					hdr,
+					rhdr,
 					rcv_flags,
 					qp);
 				break;
@@ -452,8 +451,8 @@ void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
 			       bool do_cnp)
 {
 	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
-	struct hfi1_ib_header *hdr = pkt->hdr;
-	struct hfi1_other_headers *ohdr = pkt->ohdr;
+	struct ib_header *hdr = pkt->hdr;
+	struct ib_other_headers *ohdr = pkt->ohdr;
 	struct ib_grh *grh = NULL;
 	u32 rqpn = 0, bth1;
 	u16 rlid, dlid = be16_to_cpu(hdr->lrh[1]);
@@ -487,7 +486,7 @@ void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
 		return;
 	}
 
-	sc = hdr2sc((struct hfi1_message_header *)hdr, pkt->rhf);
+	sc = hdr2sc(hdr, pkt->rhf);
 
 	bth1 = be32_to_cpu(ohdr->bth[1]);
 	if (do_cnp && (bth1 & HFI1_FECN_SMASK)) {
@@ -599,8 +598,8 @@ static void __prescan_rxq(struct hfi1_packet *packet)
 		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
 					 dd->rhf_offset;
 		struct rvt_qp *qp;
-		struct hfi1_ib_header *hdr;
-		struct hfi1_other_headers *ohdr;
+		struct ib_header *hdr;
+		struct ib_other_headers *ohdr;
 		struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
 		u64 rhf = rhf_to_cpu(rhf_addr);
 		u32 etype = rhf_rcv_type(rhf), qpn, bth1;
@@ -616,8 +615,8 @@ static void __prescan_rxq(struct hfi1_packet *packet)
 		if (etype != RHF_RCV_TYPE_IB)
 			goto next;
 
-		hdr = (struct hfi1_ib_header *)
-			hfi1_get_msgheader(dd, rhf_addr);
+		hdr = hfi1_get_msgheader(dd, rhf_addr);
+
 		lnh = be16_to_cpu(hdr->lrh[0]) & 3;
 
 		if (lnh == HFI1_LRH_BTH)
@@ -888,14 +887,15 @@ void set_all_slowpath(struct hfi1_devdata *dd)
 }
 
 static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
-				      struct hfi1_packet packet,
+				      struct hfi1_packet *packet,
 				      struct hfi1_devdata *dd)
 {
 	struct work_struct *lsaw = &rcd->ppd->linkstate_active_work;
-	struct hfi1_message_header *hdr = hfi1_get_msgheader(packet.rcd->dd,
-							     packet.rhf_addr);
+	struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd,
+						   packet->rhf_addr);
+	u8 etype = rhf_rcv_type(packet->rhf);
 
-	if (hdr2sc(hdr, packet.rhf) != 0xf) {
+	if (etype == RHF_RCV_TYPE_IB && hdr2sc(hdr, packet->rhf) != 0xf) {
 		int hwstate = read_logical_state(dd);
 
 		if (hwstate != LSTATE_ACTIVE) {
@@ -979,7 +979,7 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
 			/* Auto activate link on non-SC15 packet receive */
 			if (unlikely(rcd->ppd->host_link_state ==
 				     HLS_UP_ARMED) &&
-			    set_armed_to_active(rcd, packet, dd))
+			    set_armed_to_active(rcd, &packet, dd))
 				goto bail;
 			last = process_rcv_packet(&packet, thread);
 		}
diff --git a/drivers/infiniband/hw/hfi1/eprom.c b/drivers/infiniband/hw/hfi1/eprom.c
index 36b77943cbfd..e70c223801b4 100644
--- a/drivers/infiniband/hw/hfi1/eprom.c
+++ b/drivers/infiniband/hw/hfi1/eprom.c
@@ -49,7 +49,26 @@
 #include "common.h"
 #include "eprom.h"
 
+/*
+ * The EPROM is logically divided into three partitions:
+ *	partition 0: the first 128K, visible from PCI ROM BAR
+ *	partition 1: 4K config file (sector size)
+ *	partition 2: the rest
+ */
+#define P0_SIZE (128 * 1024)
+#define P1_SIZE   (4 * 1024)
+#define P1_START P0_SIZE
+#define P2_START (P0_SIZE + P1_SIZE)
+
+/* controller page size, in bytes */
+#define EP_PAGE_SIZE 256
+#define EP_PAGE_MASK (EP_PAGE_SIZE - 1)
+#define EP_PAGE_DWORDS (EP_PAGE_SIZE / sizeof(u32))
+
+/* controller commands */
 #define CMD_SHIFT 24
+#define CMD_NOP			    (0)
+#define CMD_READ_DATA(addr)	    ((0x03 << CMD_SHIFT) | addr)
 #define CMD_RELEASE_POWERDOWN_NOID  ((0xab << CMD_SHIFT))
 
 /* controller interface speeds */
@@ -61,6 +80,90 @@
  * Double it for safety.
  */
 #define EPROM_TIMEOUT 80000 /* ms */
+
+/*
+ * Read a 256 byte (64 dword) EPROM page.
+ * All callers have verified the offset is at a page boundary.
+ */
+static void read_page(struct hfi1_devdata *dd, u32 offset, u32 *result)
+{
+	int i;
+
+	write_csr(dd, ASIC_EEP_ADDR_CMD, CMD_READ_DATA(offset));
+	for (i = 0; i < EP_PAGE_DWORDS; i++)
+		result[i] = (u32)read_csr(dd, ASIC_EEP_DATA);
+	write_csr(dd, ASIC_EEP_ADDR_CMD, CMD_NOP); /* close open page */
+}
+
+/*
+ * Read length bytes starting at offset from the start of the EPROM.
+ */
+static int read_length(struct hfi1_devdata *dd, u32 start, u32 len, void *dest)
+{
+	u32 buffer[EP_PAGE_DWORDS];
+	u32 end;
+	u32 start_offset;
+	u32 read_start;
+	u32 bytes;
+
+	if (len == 0)
+		return 0;
+
+	end = start + len;
+
+	/*
+	 * Make sure the read range is not outside of the controller read
+	 * command address range.  Note that '>' is correct below - the end
+	 * of the range is OK if it stops at the limit, but no higher.
+	 */
+	if (end > (1 << CMD_SHIFT))
+		return -EINVAL;
+
+	/* read the first partial page */
+	start_offset = start & EP_PAGE_MASK;
+	if (start_offset) {
+		/* partial starting page */
+
+		/* align and read the page that contains the start */
+		read_start = start & ~EP_PAGE_MASK;
+		read_page(dd, read_start, buffer);
+
+		/* the rest of the page is available data */
+		bytes = EP_PAGE_SIZE - start_offset;
+
+		if (len <= bytes) {
+			/* end is within this page */
+			memcpy(dest, (u8 *)buffer + start_offset, len);
+			return 0;
+		}
+
+		memcpy(dest, (u8 *)buffer + start_offset, bytes);
+
+		start += bytes;
+		len -= bytes;
+		dest += bytes;
+	}
+	/* start is now page aligned */
+
+	/* read whole pages */
+	while (len >= EP_PAGE_SIZE) {
+		read_page(dd, start, buffer);
+		memcpy(dest, buffer, EP_PAGE_SIZE);
+
+		start += EP_PAGE_SIZE;
+		len -= EP_PAGE_SIZE;
+		dest += EP_PAGE_SIZE;
+	}
+
+	/* read the last partial page */
+	if (len) {
+		read_page(dd, start, buffer);
+		memcpy(dest, buffer, len);
+	}
+
+	return 0;
+}
+
 /*
  * Initialize the EPROM handler.
  */
@@ -100,3 +203,85 @@ int eprom_init(struct hfi1_devdata *dd)
 done_asic:
 	return ret;
 }
+
+/* magic character sequence that trails an image */
+#define IMAGE_TRAIL_MAGIC "egamiAPO"
+
+/*
+ * Read all of partition 1.  The actual file is at the front.  Adjust
+ * the returned size if a trailing image magic is found.
+ */
+static int read_partition_platform_config(struct hfi1_devdata *dd, void **data,
+					  u32 *size)
+{
+	void *buffer;
+	void *p;
+	u32 length;
+	int ret;
+
+	buffer = kmalloc(P1_SIZE, GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	ret = read_length(dd, P1_START, P1_SIZE, buffer);
+	if (ret) {
+		kfree(buffer);
+		return ret;
+	}
+
+	/* scan for image magic that may trail the actual data */
+	p = strnstr(buffer, IMAGE_TRAIL_MAGIC, P1_SIZE);
+	if (p)
+		length = p - buffer;
+	else
+		length = P1_SIZE;
+
+	*data = buffer;
+	*size = length;
+	return 0;
+}
+
+/*
+ * Read the platform configuration file from the EPROM.
+ *
+ * On success, an allocated buffer containing the data and its size are
+ * returned.  It is up to the caller to free this buffer.
+ *
+ * Return value:
+ *   0	      - success
+ *   -ENXIO   - no EPROM is available
+ *   -EBUSY   - not able to acquire access to the EPROM
+ *   -ENOENT  - no recognizable file written
+ *   -ENOMEM  - buffer could not be allocated
+ */
+int eprom_read_platform_config(struct hfi1_devdata *dd, void **data, u32 *size)
+{
+	u32 directory[EP_PAGE_DWORDS]; /* aligned buffer */
+	int ret;
+
+	if (!dd->eprom_available)
+		return -ENXIO;
+
+	ret = acquire_chip_resource(dd, CR_EPROM, EPROM_TIMEOUT);
+	if (ret)
+		return -EBUSY;
+
+	/* read the last page of P0 for the EPROM format magic */
+	ret = read_length(dd, P1_START - EP_PAGE_SIZE, EP_PAGE_SIZE, directory);
+	if (ret)
+		goto done;
+
+	/* last dword of P0 contains a magic indicator */
+	if (directory[EP_PAGE_DWORDS - 1] == 0) {
+		/* partition format */
+		ret = read_partition_platform_config(dd, data, size);
+		goto done;
+	}
+
+	/* nothing recognized */
+	ret = -ENOENT;
+
+done:
+	release_chip_resource(dd, CR_EPROM);
+	return ret;
+}
diff --git a/drivers/infiniband/hw/hfi1/eprom.h b/drivers/infiniband/hw/hfi1/eprom.h
index d41f0b1afb15..e774184f1643 100644
--- a/drivers/infiniband/hw/hfi1/eprom.h
+++ b/drivers/infiniband/hw/hfi1/eprom.h
@@ -45,8 +45,8 @@
  *
  */
 
-struct hfi1_cmd;
 struct hfi1_devdata;
 
 int eprom_init(struct hfi1_devdata *dd);
-int handle_eprom_command(struct file *fp, const struct hfi1_cmd *cmd);
+int eprom_read_platform_config(struct hfi1_devdata *dd, void **buf_ret,
+			       u32 *size_ret);
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index 1ecbec192358..677efa0e8cd6 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -58,7 +58,6 @@
 #include "trace.h"
 #include "user_sdma.h"
 #include "user_exp_rcv.h"
-#include "eprom.h"
 #include "aspm.h"
 #include "mmu_rb.h"
 
@@ -183,6 +182,7 @@ static int hfi1_file_open(struct inode *inode, struct file *fp)
 	if (fd) {
 		fd->rec_cpu_num = -1; /* no cpu affinity by default */
 		fd->mm = current->mm;
+		atomic_inc(&fd->mm->mm_count);
 	}
 
 	fp->private_data = fd;
@@ -222,7 +222,7 @@ static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
 		ret = assign_ctxt(fp, &uinfo);
 		if (ret < 0)
 			return ret;
-		setup_ctxt(fp);
+		ret = setup_ctxt(fp);
 		if (ret)
 			return ret;
 		ret = user_init(fp);
@@ -439,9 +439,10 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 	struct hfi1_filedata *fd = fp->private_data;
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd;
-	unsigned long flags, pfn;
+	unsigned long flags;
 	u64 token = vma->vm_pgoff << PAGE_SHIFT,
 		memaddr = 0;
+	void *memvirt = NULL;
 	u8 subctxt, mapio = 0, vmf = 0, type;
 	ssize_t memlen = 0;
 	int ret = 0;
@@ -492,7 +493,8 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		 * second or third page allocated for credit returns (if number
 		 * of enabled contexts > 64 and 128 respectively).
 		 */
-		memaddr = dd->cr_base[uctxt->numa_id].pa +
+		memvirt = dd->cr_base[uctxt->numa_id].va;
+		memaddr = virt_to_phys(memvirt) +
 			(((u64)uctxt->sc->hw_free -
 			  (u64)dd->cr_base[uctxt->numa_id].va) & PAGE_MASK);
 		memlen = PAGE_SIZE;
@@ -507,8 +509,8 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		mapio = 1;
 		break;
 	case RCV_HDRQ:
-		memaddr = uctxt->rcvhdrq_phys;
 		memlen = uctxt->rcvhdrq_size;
+		memvirt = uctxt->rcvhdrq;
 		break;
 	case RCV_EGRBUF: {
 		unsigned long addr;
@@ -532,14 +534,21 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		vma->vm_flags &= ~VM_MAYWRITE;
 		addr = vma->vm_start;
 		for (i = 0 ; i < uctxt->egrbufs.numbufs; i++) {
+			memlen = uctxt->egrbufs.buffers[i].len;
+			memvirt = uctxt->egrbufs.buffers[i].addr;
 			ret = remap_pfn_range(
 				vma, addr,
-				uctxt->egrbufs.buffers[i].phys >> PAGE_SHIFT,
-				uctxt->egrbufs.buffers[i].len,
+				/*
+				 * virt_to_pfn() does the same, but
+				 * it's not available on x86_64
+				 * when CONFIG_MMU is enabled.
+				 */
+				PFN_DOWN(__pa(memvirt)),
+				memlen,
 				vma->vm_page_prot);
 			if (ret < 0)
 				goto done;
-			addr += uctxt->egrbufs.buffers[i].len;
+			addr += memlen;
 		}
 		ret = 0;
 		goto done;
@@ -595,8 +604,8 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 			ret = -EPERM;
 			goto done;
 		}
-		memaddr = uctxt->rcvhdrqtailaddr_phys;
 		memlen = PAGE_SIZE;
+		memvirt = (void *)uctxt->rcvhdrtail_kvaddr;
 		flags &= ~VM_MAYWRITE;
 		break;
 	case SUBCTXT_UREGS:
@@ -649,16 +658,24 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		  "%u:%u type:%u io/vf:%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx\n",
 		    ctxt, subctxt, type, mapio, vmf, memaddr, memlen,
 		    vma->vm_end - vma->vm_start, vma->vm_flags);
-	pfn = (unsigned long)(memaddr >> PAGE_SHIFT);
 	if (vmf) {
-		vma->vm_pgoff = pfn;
+		vma->vm_pgoff = PFN_DOWN(memaddr);
 		vma->vm_ops = &vm_ops;
 		ret = 0;
 	} else if (mapio) {
-		ret = io_remap_pfn_range(vma, vma->vm_start, pfn, memlen,
+		ret = io_remap_pfn_range(vma, vma->vm_start,
+					 PFN_DOWN(memaddr),
+					 memlen,
 					 vma->vm_page_prot);
+	} else if (memvirt) {
+		ret = remap_pfn_range(vma, vma->vm_start,
+				      PFN_DOWN(__pa(memvirt)),
+				      memlen,
+				      vma->vm_page_prot);
 	} else {
-		ret = remap_pfn_range(vma, vma->vm_start, pfn, memlen,
+		ret = remap_pfn_range(vma, vma->vm_start,
+				      PFN_DOWN(memaddr),
+				      memlen,
 				      vma->vm_page_prot);
 	}
 done:
@@ -779,6 +796,7 @@ static int hfi1_file_close(struct inode *inode, struct file *fp)
 	mutex_unlock(&hfi1_mutex);
 	hfi1_free_ctxtdata(dd, uctxt);
 done:
+	mmdrop(fdata->mm);
 	kobject_put(&dd->kobj);
 	kfree(fdata);
 	return 0;
@@ -959,14 +977,16 @@ static int allocate_ctxt(struct file *fp, struct hfi1_devdata *dd,
 	 */
 	uctxt->sc = sc_alloc(dd, SC_USER, uctxt->rcvhdrqentsize,
 			     uctxt->dd->node);
-	if (!uctxt->sc)
-		return -ENOMEM;
-
+	if (!uctxt->sc) {
+		ret = -ENOMEM;
+		goto ctxdata_free;
+	}
 	hfi1_cdbg(PROC, "allocated send context %u(%u)\n", uctxt->sc->sw_index,
 		  uctxt->sc->hw_context);
 	ret = sc_enable(uctxt->sc);
 	if (ret)
-		return ret;
+		goto ctxdata_free;
+
 	/*
 	 * Setup shared context resources if the user-level has requested
 	 * shared contexts and this is the 'master' process.
@@ -980,7 +1000,7 @@ static int allocate_ctxt(struct file *fp, struct hfi1_devdata *dd,
 		 * send context because it will be done during file close
 		 */
 		if (ret)
-			return ret;
+			goto ctxdata_free;
 	}
 	uctxt->userversion = uinfo->userversion;
 	uctxt->flags = hfi1_cap_mask; /* save current flag state */
@@ -1000,6 +1020,11 @@ static int allocate_ctxt(struct file *fp, struct hfi1_devdata *dd,
 	fd->uctxt = uctxt;
 
 	return 0;
+
+ctxdata_free:
+	dd->rcd[ctxt] = NULL;
+	hfi1_free_ctxtdata(dd, uctxt);
+	return ret;
 }
 
 static int init_subctxts(struct hfi1_ctxtdata *uctxt,
@@ -1258,7 +1283,7 @@ static int get_base_info(struct file *fp, void __user *ubase, __u32 len)
 					       uctxt->rcvhdrq);
 	binfo.rcvegr_bufbase = HFI1_MMAP_TOKEN(RCV_EGRBUF, uctxt->ctxt,
 					       fd->subctxt,
-					       uctxt->egrbufs.rcvtids[0].phys);
+					       uctxt->egrbufs.rcvtids[0].dma);
 	binfo.sdma_comp_bufbase = HFI1_MMAP_TOKEN(SDMA_COMP, uctxt->ctxt,
 						 fd->subctxt, 0);
 	/*
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index 1000e0fd96d9..7eef11b316ff 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -64,6 +64,8 @@
 #include <linux/kthread.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
+#include <rdma/ib_hdrs.h>
+#include <linux/rhashtable.h>
 #include <rdma/rdma_vt.h>
 
 #include "chip_registers.h"
@@ -171,12 +173,12 @@ struct ctxt_eager_bufs {
 	u32 threshold;           /* head update threshold */
 	struct eager_buffer {
 		void *addr;
-		dma_addr_t phys;
+		dma_addr_t dma;
 		ssize_t len;
 	} *buffers;
 	struct {
 		void *addr;
-		dma_addr_t phys;
+		dma_addr_t dma;
 	} *rcvtids;
 };
 
@@ -207,8 +209,8 @@ struct hfi1_ctxtdata {
 	/* size of each of the rcvhdrq entries */
 	u16 rcvhdrqentsize;
 	/* mmap of hdrq, must fit in 44 bits */
-	dma_addr_t rcvhdrq_phys;
-	dma_addr_t rcvhdrqtailaddr_phys;
+	dma_addr_t rcvhdrq_dma;
+	dma_addr_t rcvhdrqtailaddr_dma;
 	struct ctxt_eager_bufs egrbufs;
 	/* this receive context's assigned PIO ACK send context */
 	struct send_context *sc;
@@ -350,7 +352,7 @@ struct hfi1_packet {
 	struct hfi1_ctxtdata *rcd;
 	__le32 *rhf_addr;
 	struct rvt_qp *qp;
-	struct hfi1_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	u64 rhf;
 	u32 maxcnt;
 	u32 rhqoff;
@@ -529,6 +531,7 @@ struct hfi1_msix_entry {
 	void *arg;
 	char name[MAX_NAME_SIZE];
 	cpumask_t mask;
+	struct irq_affinity_notify notify;
 };
 
 /* per-SL CCA information */
@@ -605,6 +608,7 @@ struct hfi1_pportdata {
 	struct work_struct freeze_work;
 	struct work_struct link_downgrade_work;
 	struct work_struct link_bounce_work;
+	struct delayed_work start_link_work;
 	/* host link state variables */
 	struct mutex hls_lock;
 	u32 host_link_state;
@@ -659,6 +663,7 @@ struct hfi1_pportdata {
 	u8 linkinit_reason;
 	u8 local_tx_rate;	/* rate given to 8051 firmware */
 	u8 last_pstate;		/* info only */
+	u8 qsfp_retry_count;
 
 	/* placeholders for IB MAD packet settings */
 	u8 overrun_threshold;
@@ -1058,8 +1063,6 @@ struct hfi1_devdata {
 	u8 psxmitwait_supported;
 	/* cycle length of PS* counters in HW (in picoseconds) */
 	u16 psxmitwait_check_rate;
-	/* high volume overflow errors deferred to tasklet */
-	struct tasklet_struct error_tasklet;
 
 	/* MSI-X information */
 	struct hfi1_msix_entry *msix_entries;
@@ -1162,7 +1165,7 @@ struct hfi1_devdata {
 
 	/* receive context tail dummy address */
 	__le64 *rcvhdrtail_dummy_kvaddr;
-	dma_addr_t rcvhdrtail_dummy_physaddr;
+	dma_addr_t rcvhdrtail_dummy_dma;
 
 	bool eprom_available;	/* true if EPROM is available for this device */
 	bool aspm_supported;	/* Does HW support ASPM */
@@ -1173,6 +1176,7 @@ struct hfi1_devdata {
 	atomic_t aspm_disabled_cnt;
 
 	struct hfi1_affinity *affinity;
+	struct rhashtable sdma_rht;
 	struct kobject kobj;
 };
 
@@ -1266,15 +1270,32 @@ static inline u32 driver_lstate(struct hfi1_pportdata *ppd)
 void receive_interrupt_work(struct work_struct *work);
 
 /* extract service channel from header and rhf */
-static inline int hdr2sc(struct hfi1_message_header *hdr, u64 rhf)
+static inline int hdr2sc(struct ib_header *hdr, u64 rhf)
 {
 	return ((be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf) |
 	       ((!!(rhf_dc_info(rhf))) << 4);
 }
 
+#define HFI1_JKEY_WIDTH       16
+#define HFI1_JKEY_MASK        (BIT(16) - 1)
+#define HFI1_ADMIN_JKEY_RANGE 32
+
+/*
+ * J_KEYs are split and allocated in the following groups:
+ *   0 - 31    - users with administrator privileges
+ *  32 - 63    - kernel protocols using KDETH packets
+ *  64 - 65535 - all other users using KDETH packets
+ */
 static inline u16 generate_jkey(kuid_t uid)
 {
-	return from_kuid(current_user_ns(), uid) & 0xffff;
+	u16 jkey = from_kuid(current_user_ns(), uid) & HFI1_JKEY_MASK;
+
+	if (capable(CAP_SYS_ADMIN))
+		jkey &= HFI1_ADMIN_JKEY_RANGE - 1;
+	else if (jkey < 64)
+		jkey |= BIT(HFI1_JKEY_WIDTH - 1);
+
+	return jkey;
 }
 
 /*
@@ -1584,7 +1605,7 @@ void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
 static inline bool process_ecn(struct rvt_qp *qp, struct hfi1_packet *pkt,
 			       bool do_cnp)
 {
-	struct hfi1_other_headers *ohdr = pkt->ohdr;
+	struct ib_other_headers *ohdr = pkt->ohdr;
 	u32 bth1;
 
 	bth1 = be32_to_cpu(ohdr->bth[1]);
@@ -1656,7 +1677,6 @@ struct cc_state *get_cc_state_protected(struct hfi1_pportdata *ppd)
 struct hfi1_devdata *hfi1_init_dd(struct pci_dev *,
 				  const struct pci_device_id *);
 void hfi1_free_devdata(struct hfi1_devdata *);
-void cc_state_reclaim(struct rcu_head *rcu);
 struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra);
 
 /* LED beaconing functions */
@@ -1788,7 +1808,7 @@ extern unsigned int hfi1_max_mtu;
 extern unsigned int hfi1_cu;
 extern unsigned int user_credit_return_threshold;
 extern int num_user_contexts;
-extern unsigned n_krcvqs;
+extern unsigned long n_krcvqs;
 extern uint krcvqs[];
 extern int krcvqsset;
 extern uint kdeth_qp;
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index a358d23ecd54..60db61536fed 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -94,7 +94,7 @@ module_param_array(krcvqs, uint, &krcvqsset, S_IRUGO);
 MODULE_PARM_DESC(krcvqs, "Array of the number of non-control kernel receive queues by VL");
 
 /* computed based on above array */
-unsigned n_krcvqs;
+unsigned long n_krcvqs;
 
 static unsigned hfi1_rcvarr_split = 25;
 module_param_named(rcvarr_split, hfi1_rcvarr_split, uint, S_IRUGO);
@@ -336,6 +336,7 @@ struct hfi1_ctxtdata *hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, u32 ctxt,
 	}
 	return rcd;
 bail:
+	dd->rcd[ctxt] = NULL;
 	kfree(rcd->egrbufs.rcvtids);
 	kfree(rcd->egrbufs.buffers);
 	kfree(rcd);
@@ -500,6 +501,7 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
 	INIT_WORK(&ppd->link_downgrade_work, handle_link_downgrade);
 	INIT_WORK(&ppd->sma_message_work, handle_sma_message);
 	INIT_WORK(&ppd->link_bounce_work, handle_link_bounce);
+	INIT_DELAYED_WORK(&ppd->start_link_work, handle_start_link);
 	INIT_WORK(&ppd->linkstate_active_work, receive_interrupt_work);
 	INIT_WORK(&ppd->qsfp_info.qsfp_work, qsfp_event);
 
@@ -708,7 +710,7 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit)
 	/* allocate dummy tail memory for all receive contexts */
 	dd->rcvhdrtail_dummy_kvaddr = dma_zalloc_coherent(
 		&dd->pcidev->dev, sizeof(u64),
-		&dd->rcvhdrtail_dummy_physaddr,
+		&dd->rcvhdrtail_dummy_dma,
 		GFP_KERNEL);
 
 	if (!dd->rcvhdrtail_dummy_kvaddr) {
@@ -941,12 +943,12 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 
 	if (rcd->rcvhdrq) {
 		dma_free_coherent(&dd->pcidev->dev, rcd->rcvhdrq_size,
-				  rcd->rcvhdrq, rcd->rcvhdrq_phys);
+				  rcd->rcvhdrq, rcd->rcvhdrq_dma);
 		rcd->rcvhdrq = NULL;
 		if (rcd->rcvhdrtail_kvaddr) {
 			dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
 					  (void *)rcd->rcvhdrtail_kvaddr,
-					  rcd->rcvhdrqtailaddr_phys);
+					  rcd->rcvhdrqtailaddr_dma);
 			rcd->rcvhdrtail_kvaddr = NULL;
 		}
 	}
@@ -955,11 +957,11 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 	kfree(rcd->egrbufs.rcvtids);
 
 	for (e = 0; e < rcd->egrbufs.alloced; e++) {
-		if (rcd->egrbufs.buffers[e].phys)
+		if (rcd->egrbufs.buffers[e].dma)
 			dma_free_coherent(&dd->pcidev->dev,
 					  rcd->egrbufs.buffers[e].len,
 					  rcd->egrbufs.buffers[e].addr,
-					  rcd->egrbufs.buffers[e].phys);
+					  rcd->egrbufs.buffers[e].dma);
 	}
 	kfree(rcd->egrbufs.buffers);
 
@@ -1333,7 +1335,7 @@ static void cleanup_device_data(struct hfi1_devdata *dd)
 		spin_unlock(&ppd->cc_state_lock);
 
 		if (cc_state)
-			call_rcu(&cc_state->rcu, cc_state_reclaim);
+			kfree_rcu(cc_state, rcu);
 	}
 
 	free_credit_return(dd);
@@ -1353,7 +1355,7 @@ static void cleanup_device_data(struct hfi1_devdata *dd)
 	if (dd->rcvhdrtail_dummy_kvaddr) {
 		dma_free_coherent(&dd->pcidev->dev, sizeof(u64),
 				  (void *)dd->rcvhdrtail_dummy_kvaddr,
-				  dd->rcvhdrtail_dummy_physaddr);
+				  dd->rcvhdrtail_dummy_dma);
 		dd->rcvhdrtail_dummy_kvaddr = NULL;
 	}
 
@@ -1576,7 +1578,7 @@ int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 	u64 reg;
 
 	if (!rcd->rcvhdrq) {
-		dma_addr_t phys_hdrqtail;
+		dma_addr_t dma_hdrqtail;
 		gfp_t gfp_flags;
 
 		/*
@@ -1589,7 +1591,7 @@ int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 		gfp_flags = (rcd->ctxt >= dd->first_user_ctxt) ?
 			GFP_USER : GFP_KERNEL;
 		rcd->rcvhdrq = dma_zalloc_coherent(
-			&dd->pcidev->dev, amt, &rcd->rcvhdrq_phys,
+			&dd->pcidev->dev, amt, &rcd->rcvhdrq_dma,
 			gfp_flags | __GFP_COMP);
 
 		if (!rcd->rcvhdrq) {
@@ -1601,11 +1603,11 @@ int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 
 		if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
 			rcd->rcvhdrtail_kvaddr = dma_zalloc_coherent(
-				&dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail,
+				&dd->pcidev->dev, PAGE_SIZE, &dma_hdrqtail,
 				gfp_flags);
 			if (!rcd->rcvhdrtail_kvaddr)
 				goto bail_free;
-			rcd->rcvhdrqtailaddr_phys = phys_hdrqtail;
+			rcd->rcvhdrqtailaddr_dma = dma_hdrqtail;
 		}
 
 		rcd->rcvhdrq_size = amt;
@@ -1633,7 +1635,7 @@ int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 	 * before enabling any receive context
 	 */
 	write_kctxt_csr(dd, rcd->ctxt, RCV_HDR_TAIL_ADDR,
-			dd->rcvhdrtail_dummy_physaddr);
+			dd->rcvhdrtail_dummy_dma);
 
 	return 0;
 
@@ -1644,7 +1646,7 @@ bail_free:
 	vfree(rcd->user_event_mask);
 	rcd->user_event_mask = NULL;
 	dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,
-			  rcd->rcvhdrq_phys);
+			  rcd->rcvhdrq_dma);
 	rcd->rcvhdrq = NULL;
 bail:
 	return -ENOMEM;
@@ -1705,15 +1707,15 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)
 		rcd->egrbufs.buffers[idx].addr =
 			dma_zalloc_coherent(&dd->pcidev->dev,
 					    rcd->egrbufs.rcvtid_size,
-					    &rcd->egrbufs.buffers[idx].phys,
+					    &rcd->egrbufs.buffers[idx].dma,
 					    gfp_flags);
 		if (rcd->egrbufs.buffers[idx].addr) {
 			rcd->egrbufs.buffers[idx].len =
 				rcd->egrbufs.rcvtid_size;
 			rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].addr =
 				rcd->egrbufs.buffers[idx].addr;
-			rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].phys =
-				rcd->egrbufs.buffers[idx].phys;
+			rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].dma =
+				rcd->egrbufs.buffers[idx].dma;
 			rcd->egrbufs.alloced++;
 			alloced_bytes += rcd->egrbufs.rcvtid_size;
 			idx++;
@@ -1754,14 +1756,14 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)
 			for (i = 0, j = 0, offset = 0; j < idx; i++) {
 				if (i >= rcd->egrbufs.count)
 					break;
-				rcd->egrbufs.rcvtids[i].phys =
-					rcd->egrbufs.buffers[j].phys + offset;
+				rcd->egrbufs.rcvtids[i].dma =
+					rcd->egrbufs.buffers[j].dma + offset;
 				rcd->egrbufs.rcvtids[i].addr =
 					rcd->egrbufs.buffers[j].addr + offset;
 				rcd->egrbufs.alloced++;
-				if ((rcd->egrbufs.buffers[j].phys + offset +
+				if ((rcd->egrbufs.buffers[j].dma + offset +
 				     new_size) ==
-				    (rcd->egrbufs.buffers[j].phys +
+				    (rcd->egrbufs.buffers[j].dma +
 				     rcd->egrbufs.buffers[j].len)) {
 					j++;
 					offset = 0;
@@ -1813,7 +1815,7 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)
 
 	for (idx = 0; idx < rcd->egrbufs.alloced; idx++) {
 		hfi1_put_tid(dd, rcd->eager_base + idx, PT_EAGER,
-			     rcd->egrbufs.rcvtids[idx].phys, order);
+			     rcd->egrbufs.rcvtids[idx].dma, order);
 		cond_resched();
 	}
 	goto bail;
@@ -1825,9 +1827,9 @@ bail_rcvegrbuf_phys:
 		dma_free_coherent(&dd->pcidev->dev,
 				  rcd->egrbufs.buffers[idx].len,
 				  rcd->egrbufs.buffers[idx].addr,
-				  rcd->egrbufs.buffers[idx].phys);
+				  rcd->egrbufs.buffers[idx].dma);
 		rcd->egrbufs.buffers[idx].addr = NULL;
-		rcd->egrbufs.buffers[idx].phys = 0;
+		rcd->egrbufs.buffers[idx].dma = 0;
 		rcd->egrbufs.buffers[idx].len = 0;
 	}
 bail:
diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c
index 1263abe01999..9487c9bb8920 100644
--- a/drivers/infiniband/hw/hfi1/mad.c
+++ b/drivers/infiniband/hw/hfi1/mad.c
@@ -1013,7 +1013,6 @@ static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp,
 			 * offline.
 			 */
 			set_link_state(ppd, HLS_DN_OFFLINE);
-			tune_serdes(ppd);
 			start_link(ppd);
 		} else {
 			set_link_state(ppd, link_state);
@@ -1407,12 +1406,6 @@ static int set_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys)
 		if (key == okey)
 			continue;
 		/*
-		 * Don't update pkeys[2], if an HFI port without MgmtAllowed
-		 * by neighbor is a switch.
-		 */
-		if (i == 2 && !ppd->mgmt_allowed && ppd->neighbor_type == 1)
-			continue;
-		/*
 		 * The SM gives us the complete PKey table. We have
 		 * to ensure that we put the PKeys in the matching
 		 * slots.
@@ -1819,6 +1812,11 @@ static int __subn_get_opa_cable_info(struct opa_smp *smp, u32 am, u8 *data,
 	u32 len = OPA_AM_CI_LEN(am) + 1;
 	int ret;
 
+	if (dd->pport->port_type != PORT_TYPE_QSFP) {
+		smp->status |= IB_SMP_INVALID_FIELD;
+		return reply((struct ib_mad_hdr *)smp);
+	}
+
 #define __CI_PAGE_SIZE BIT(7) /* 128 bytes */
 #define __CI_PAGE_MASK ~(__CI_PAGE_SIZE - 1)
 #define __CI_PAGE_NUM(a) ((a) & __CI_PAGE_MASK)
@@ -2599,7 +2597,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,
 	u8 lq, num_vls;
 	u8 res_lli, res_ler;
 	u64 port_mask;
-	unsigned long port_num;
+	u8 port_num;
 	unsigned long vl;
 	u32 vl_select_mask;
 	int vfi;
@@ -2633,9 +2631,9 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,
 	 */
 	port_mask = be64_to_cpu(req->port_select_mask[3]);
 	port_num = find_first_bit((unsigned long *)&port_mask,
-				  sizeof(port_mask));
+				  sizeof(port_mask) * 8);
 
-	if ((u8)port_num != port) {
+	if (port_num != port) {
 		pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)pmp);
 	}
@@ -2837,7 +2835,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp,
 	 */
 	port_mask = be64_to_cpu(req->port_select_mask[3]);
 	port_num = find_first_bit((unsigned long *)&port_mask,
-				  sizeof(port_mask));
+				  sizeof(port_mask) * 8);
 
 	if (port_num != port) {
 		pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD;
@@ -3010,7 +3008,7 @@ static int pma_get_opa_errorinfo(struct opa_pma_mad *pmp,
 	 */
 	port_mask = be64_to_cpu(req->port_select_mask[3]);
 	port_num = find_first_bit((unsigned long *)&port_mask,
-				  sizeof(port_mask));
+				  sizeof(port_mask) * 8);
 
 	if (port_num != port) {
 		pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD;
@@ -3247,7 +3245,7 @@ static int pma_set_opa_errorinfo(struct opa_pma_mad *pmp,
 	 */
 	port_mask = be64_to_cpu(req->port_select_mask[3]);
 	port_num = find_first_bit((unsigned long *)&port_mask,
-				  sizeof(port_mask));
+				  sizeof(port_mask) * 8);
 
 	if (port_num != port) {
 		pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD;
@@ -3398,7 +3396,7 @@ static void apply_cc_state(struct hfi1_pportdata *ppd)
 
 	spin_unlock(&ppd->cc_state_lock);
 
-	call_rcu(&old_cc_state->rcu, cc_state_reclaim);
+	kfree_rcu(old_cc_state, rcu);
 }
 
 static int __subn_set_opa_cong_setting(struct opa_smp *smp, u32 am, u8 *data,
@@ -3553,13 +3551,6 @@ static int __subn_get_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,
 	return reply((struct ib_mad_hdr *)smp);
 }
 
-void cc_state_reclaim(struct rcu_head *rcu)
-{
-	struct cc_state *cc_state = container_of(rcu, struct cc_state, rcu);
-
-	kfree(cc_state);
-}
-
 static int __subn_set_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
 				   u32 *resp_len)
diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c
index ac1bf4a73571..50a3a36d9363 100644
--- a/drivers/infiniband/hw/hfi1/pio.c
+++ b/drivers/infiniband/hw/hfi1/pio.c
@@ -551,11 +551,11 @@ static inline u32 group_size(u32 group)
 }
 
 /*
- * Obtain the credit return addresses, kernel virtual and physical, for the
+ * Obtain the credit return addresses, kernel virtual and bus, for the
  * given sc.
  *
  * To understand this routine:
- * o va and pa are arrays of struct credit_return.  One for each physical
+ * o va and dma are arrays of struct credit_return.  One for each physical
  *   send context, per NUMA.
  * o Each send context always looks in its relative location in a struct
  *   credit_return for its credit return.
@@ -563,14 +563,14 @@ static inline u32 group_size(u32 group)
  *   with the same value.  Use the address of the first send context in the
  *   group.
  */
-static void cr_group_addresses(struct send_context *sc, dma_addr_t *pa)
+static void cr_group_addresses(struct send_context *sc, dma_addr_t *dma)
 {
 	u32 gc = group_context(sc->hw_context, sc->group);
 	u32 index = sc->hw_context & 0x7;
 
 	sc->hw_free = &sc->dd->cr_base[sc->node].va[gc].cr[index];
-	*pa = (unsigned long)
-	       &((struct credit_return *)sc->dd->cr_base[sc->node].pa)[gc];
+	*dma = (unsigned long)
+	       &((struct credit_return *)sc->dd->cr_base[sc->node].dma)[gc];
 }
 
 /*
@@ -710,7 +710,7 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,
 {
 	struct send_context_info *sci;
 	struct send_context *sc = NULL;
-	dma_addr_t pa;
+	dma_addr_t dma;
 	unsigned long flags;
 	u64 reg;
 	u32 thresh;
@@ -763,7 +763,7 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,
 
 	sc->sw_index = sw_index;
 	sc->hw_context = hw_context;
-	cr_group_addresses(sc, &pa);
+	cr_group_addresses(sc, &dma);
 	sc->credits = sci->credits;
 
 /* PIO Send Memory Address details */
@@ -805,7 +805,7 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,
 			((u64)opval << SC(CHECK_OPCODE_VALUE_SHIFT)));
 
 	/* set up credit return */
-	reg = pa & SC(CREDIT_RETURN_ADDR_ADDRESS_SMASK);
+	reg = dma & SC(CREDIT_RETURN_ADDR_ADDRESS_SMASK);
 	write_kctxt_csr(dd, hw_context, SC(CREDIT_RETURN_ADDR), reg);
 
 	/*
@@ -2064,7 +2064,7 @@ int init_credit_return(struct hfi1_devdata *dd)
 		dd->cr_base[i].va = dma_zalloc_coherent(
 					&dd->pcidev->dev,
 					bytes,
-					&dd->cr_base[i].pa,
+					&dd->cr_base[i].dma,
 					GFP_KERNEL);
 		if (!dd->cr_base[i].va) {
 			set_dev_node(&dd->pcidev->dev, dd->node);
@@ -2097,7 +2097,7 @@ void free_credit_return(struct hfi1_devdata *dd)
 					  TXE_NUM_CONTEXTS *
 					  sizeof(struct credit_return),
 					  dd->cr_base[i].va,
-					  dd->cr_base[i].pa);
+					  dd->cr_base[i].dma);
 		}
 	}
 	kfree(dd->cr_base);
diff --git a/drivers/infiniband/hw/hfi1/pio.h b/drivers/infiniband/hw/hfi1/pio.h
index 464cbd27b975..e709eaf743b5 100644
--- a/drivers/infiniband/hw/hfi1/pio.h
+++ b/drivers/infiniband/hw/hfi1/pio.h
@@ -154,7 +154,7 @@ struct credit_return {
 /* NUMA indexed credit return array */
 struct credit_return_base {
 	struct credit_return *va;
-	dma_addr_t pa;
+	dma_addr_t dma;
 };
 
 /* send context configuration sizes (one per type) */
diff --git a/drivers/infiniband/hw/hfi1/pio_copy.c b/drivers/infiniband/hw/hfi1/pio_copy.c
index 8c25e1b58849..aa7773643107 100644
--- a/drivers/infiniband/hw/hfi1/pio_copy.c
+++ b/drivers/infiniband/hw/hfi1/pio_copy.c
@@ -165,9 +165,6 @@ void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
 	preempt_enable();
 }
 
-/* USE_SHIFTS is faster in user-space tests on a Xeon X5570 @ 2.93GHz */
-#define USE_SHIFTS 1
-#ifdef USE_SHIFTS
 /*
  * Handle carry bytes using shifts and masks.
  *
@@ -187,150 +184,6 @@ void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
 #define mshift(x) (8 * (x))
 
 /*
- * Read nbytes bytes from "from" and return them in the LSB bytes
- * of pbuf->carry.  Other bytes are zeroed.  Any previous value
- * pbuf->carry is lost.
- *
- * NOTES:
- * o do not read from from if nbytes is zero
- * o from may _not_ be u64 aligned
- * o nbytes must not span a QW boundary
- */
-static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
-				  unsigned int nbytes)
-{
-	unsigned long off;
-
-	if (nbytes == 0) {
-		pbuf->carry.val64 = 0;
-	} else {
-		/* align our pointer */
-		off = (unsigned long)from & 0x7;
-		from = (void *)((unsigned long)from & ~0x7l);
-		pbuf->carry.val64 = ((*(u64 *)from)
-				<< zshift(nbytes + off))/* zero upper bytes */
-				>> zshift(nbytes);	/* place at bottom */
-	}
-	pbuf->carry_bytes = nbytes;
-}
-
-/*
- * Read nbytes bytes from "from" and put them at the next significant bytes
- * of pbuf->carry.  Unused bytes are zeroed.  It is expected that the extra
- * read does not overfill carry.
- *
- * NOTES:
- * o from may _not_ be u64 aligned
- * o nbytes may span a QW boundary
- */
-static inline void read_extra_bytes(struct pio_buf *pbuf,
-				    const void *from, unsigned int nbytes)
-{
-	unsigned long off = (unsigned long)from & 0x7;
-	unsigned int room, xbytes;
-
-	/* align our pointer */
-	from = (void *)((unsigned long)from & ~0x7l);
-
-	/* check count first - don't read anything if count is zero */
-	while (nbytes) {
-		/* find the number of bytes in this u64 */
-		room = 8 - off;	/* this u64 has room for this many bytes */
-		xbytes = min(room, nbytes);
-
-		/*
-		 * shift down to zero lower bytes, shift up to zero upper
-		 * bytes, shift back down to move into place
-		 */
-		pbuf->carry.val64 |= (((*(u64 *)from)
-					>> mshift(off))
-					<< zshift(xbytes))
-					>> zshift(xbytes + pbuf->carry_bytes);
-		off = 0;
-		pbuf->carry_bytes += xbytes;
-		nbytes -= xbytes;
-		from += sizeof(u64);
-	}
-}
-
-/*
- * Zero extra bytes from the end of pbuf->carry.
- *
- * NOTES:
- * o zbytes <= old_bytes
- */
-static inline void zero_extra_bytes(struct pio_buf *pbuf, unsigned int zbytes)
-{
-	unsigned int remaining;
-
-	if (zbytes == 0)	/* nothing to do */
-		return;
-
-	remaining = pbuf->carry_bytes - zbytes;	/* remaining bytes */
-
-	/* NOTE: zshift only guaranteed to work if remaining != 0 */
-	if (remaining)
-		pbuf->carry.val64 = (pbuf->carry.val64 << zshift(remaining))
-					>> zshift(remaining);
-	else
-		pbuf->carry.val64 = 0;
-	pbuf->carry_bytes = remaining;
-}
-
-/*
- * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
- * Put the unused part of the next 8 bytes of src into the LSB bytes of
- * pbuf->carry with the upper bytes zeroed..
- *
- * NOTES:
- * o result must keep unused bytes zeroed
- * o src must be u64 aligned
- */
-static inline void merge_write8(
-	struct pio_buf *pbuf,
-	void __iomem *dest,
-	const void *src)
-{
-	u64 new, temp;
-
-	new = *(u64 *)src;
-	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));
-	writeq(temp, dest);
-	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);
-}
-
-/*
- * Write a quad word using all bytes of carry.
- */
-static inline void carry8_write8(union mix carry, void __iomem *dest)
-{
-	writeq(carry.val64, dest);
-}
-
-/*
- * Write a quad word using all the valid bytes of carry.  If carry
- * has zero valid bytes, nothing is written.
- * Returns 0 on nothing written, non-zero on quad word written.
- */
-static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)
-{
-	if (pbuf->carry_bytes) {
-		/* unused bytes are always kept zeroed, so just write */
-		writeq(pbuf->carry.val64, dest);
-		return 1;
-	}
-
-	return 0;
-}
-
-#else /* USE_SHIFTS */
-/*
- * Handle carry bytes using byte copies.
- *
- * NOTE: the value the unused portion of carry is left uninitialized.
- */
-
-/*
  * Jump copy - no-loop copy for < 8 bytes.
  */
 static inline void jcopy(u8 *dest, const u8 *src, u32 n)
@@ -338,18 +191,25 @@ static inline void jcopy(u8 *dest, const u8 *src, u32 n)
 	switch (n) {
 	case 7:
 		*dest++ = *src++;
+		/* fall through */
 	case 6:
 		*dest++ = *src++;
+		/* fall through */
 	case 5:
 		*dest++ = *src++;
+		/* fall through */
 	case 4:
 		*dest++ = *src++;
+		/* fall through */
 	case 3:
 		*dest++ = *src++;
+		/* fall through */
 	case 2:
 		*dest++ = *src++;
+		/* fall through */
 	case 1:
 		*dest++ = *src++;
+		/* fall through */
 	}
 }
 
@@ -365,6 +225,7 @@ static inline void jcopy(u8 *dest, const u8 *src, u32 n)
 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
 				  unsigned int nbytes)
 {
+	pbuf->carry.val64 = 0;
 	jcopy(&pbuf->carry.val8[0], from, nbytes);
 	pbuf->carry_bytes = nbytes;
 }
@@ -385,40 +246,31 @@ static inline void read_extra_bytes(struct pio_buf *pbuf,
 }
 
 /*
- * Zero extra bytes from the end of pbuf->carry.
- *
- * We do not care about the value of unused bytes in carry, so just
- * reduce the byte count.
+ * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
+ * Put the unused part of the next 8 bytes of src into the LSB bytes of
+ * pbuf->carry with the upper bytes zeroed..
  *
  * NOTES:
- * o zbytes <= old_bytes
- */
-static inline void zero_extra_bytes(struct pio_buf *pbuf, unsigned int zbytes)
-{
-	pbuf->carry_bytes -= zbytes;
-}
-
-/*
- * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
- * Put the unused part of the next 8 bytes of src into the low bytes of
- * pbuf->carry.
+ * o result must keep unused bytes zeroed
+ * o src must be u64 aligned
  */
 static inline void merge_write8(
 	struct pio_buf *pbuf,
-	void *dest,
+	void __iomem *dest,
 	const void *src)
 {
-	u32 remainder = 8 - pbuf->carry_bytes;
+	u64 new, temp;
 
-	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], src, remainder);
-	writeq(pbuf->carry.val64, dest);
-	jcopy(&pbuf->carry.val8[0], src + remainder, pbuf->carry_bytes);
+	new = *(u64 *)src;
+	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));
+	writeq(temp, dest);
+	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);
 }
 
 /*
  * Write a quad word using all bytes of carry.
  */
-static inline void carry8_write8(union mix carry, void *dest)
+static inline void carry8_write8(union mix carry, void __iomem *dest)
 {
 	writeq(carry.val64, dest);
 }
@@ -428,20 +280,16 @@ static inline void carry8_write8(union mix carry, void *dest)
  * has zero valid bytes, nothing is written.
  * Returns 0 on nothing written, non-zero on quad word written.
  */
-static inline int carry_write8(struct pio_buf *pbuf, void *dest)
+static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)
 {
 	if (pbuf->carry_bytes) {
-		u64 zero = 0;
-
-		jcopy(&pbuf->carry.val8[pbuf->carry_bytes], (u8 *)&zero,
-		      8 - pbuf->carry_bytes);
+		/* unused bytes are always kept zeroed, so just write */
 		writeq(pbuf->carry.val64, dest);
 		return 1;
 	}
 
 	return 0;
 }
-#endif /* USE_SHIFTS */
 
 /*
  * Segmented PIO Copy - start
@@ -550,8 +398,8 @@ static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
 {
 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
 	void __iomem *dend;			/* 8-byte data end */
-	unsigned long qw_to_write = (pbuf->carry_bytes + nbytes) >> 3;
-	unsigned long bytes_left = (pbuf->carry_bytes + nbytes) & 0x7;
+	unsigned long qw_to_write = nbytes >> 3;
+	unsigned long bytes_left = nbytes & 0x7;
 
 	/* calculate 8-byte data end */
 	dend = dest + (qw_to_write * sizeof(u64));
@@ -621,16 +469,46 @@ static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
 		dest += sizeof(u64);
 	}
 
-	/* adjust carry */
-	if (pbuf->carry_bytes < bytes_left) {
-		/* need to read more */
-		read_extra_bytes(pbuf, from, bytes_left - pbuf->carry_bytes);
+	pbuf->qw_written += qw_to_write;
+
+	/* handle carry and left-over bytes */
+	if (pbuf->carry_bytes + bytes_left >= 8) {
+		unsigned long nread;
+
+		/* there is enough to fill another qw - fill carry */
+		nread = 8 - pbuf->carry_bytes;
+		read_extra_bytes(pbuf, from, nread);
+
+		/*
+		 * One more write - but need to make sure dest is correct.
+		 * Check for wrap and the possibility the write
+		 * should be in SOP space.
+		 *
+		 * The two checks immediately below cannot both be true, hence
+		 * the else. If we have wrapped, we cannot still be within the
+		 * first block. Conversely, if we are still in the first block,
+		 * we cannot have wrapped. We do the wrap check first as that
+		 * is more likely.
+		 */
+		/* adjust if we have wrapped */
+		if (dest >= pbuf->end)
+			dest -= pbuf->size;
+		/* jump to the SOP range if within the first block */
+		else if (pbuf->qw_written < PIO_BLOCK_QWS)
+			dest += SOP_DISTANCE;
+
+		/* flush out full carry */
+		carry8_write8(pbuf->carry, dest);
+		pbuf->qw_written++;
+
+		/* now adjust and read the rest of the bytes into carry */
+		bytes_left -= nread;
+		from += nread; /* from is now not aligned */
+		read_low_bytes(pbuf, from, bytes_left);
 	} else {
-		/* remove invalid bytes */
-		zero_extra_bytes(pbuf, pbuf->carry_bytes - bytes_left);
+		/* not enough to fill another qw, append the rest to carry */
+		read_extra_bytes(pbuf, from, bytes_left);
 	}
-
-	pbuf->qw_written += qw_to_write;
 }
 
 /*
@@ -771,6 +649,9 @@ void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
 			read_extra_bytes(pbuf, from, to_fill);
 			from += to_fill;
 			nbytes -= to_fill;
+			/* may not be enough valid bytes left to align */
+			if (extra > nbytes)
+				extra = nbytes;
 
 			/* ...now write carry */
 			dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
@@ -798,6 +679,15 @@ void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
 			read_low_bytes(pbuf, from, extra);
 			from += extra;
 			nbytes -= extra;
+			/*
+			 * If no bytes are left, return early - we are done.
+			 * NOTE: This short-circuit is *required* because
+			 * "extra" may have been reduced in size and "from"
+			 * is not aligned, as required when leaving this
+			 * if block.
+			 */
+			if (nbytes == 0)
+				return;
 		}
 
 		/* at this point, from is QW aligned */
diff --git a/drivers/infiniband/hw/hfi1/platform.c b/drivers/infiniband/hw/hfi1/platform.c
index 965c8aef0c60..202433178864 100644
--- a/drivers/infiniband/hw/hfi1/platform.c
+++ b/drivers/infiniband/hw/hfi1/platform.c
@@ -47,29 +47,39 @@
 
 #include "hfi.h"
 #include "efivar.h"
+#include "eprom.h"
 
 void get_platform_config(struct hfi1_devdata *dd)
 {
 	int ret = 0;
 	unsigned long size = 0;
 	u8 *temp_platform_config = NULL;
+	u32 esize;
+
+	ret = eprom_read_platform_config(dd, (void **)&temp_platform_config,
+					 &esize);
+	if (!ret) {
+		/* success */
+		size = esize;
+		goto success;
+	}
+	/* fail, try EFI variable */
 
 	ret = read_hfi1_efi_var(dd, "configuration", &size,
 				(void **)&temp_platform_config);
-	if (ret) {
-		dd_dev_info(dd,
-			    "%s: Failed to get platform config from UEFI, falling back to request firmware\n",
-			    __func__);
-		/* fall back to request firmware */
-		platform_config_load = 1;
-		goto bail;
-	}
+	if (!ret)
+		goto success;
+
+	dd_dev_info(dd,
+		    "%s: Failed to get platform config from UEFI, falling back to request firmware\n",
+		    __func__);
+	/* fall back to request firmware */
+	platform_config_load = 1;
+	return;
 
+success:
 	dd->platform_config.data = temp_platform_config;
 	dd->platform_config.size = size;
-
-bail:
-	/* exit */;
 }
 
 void free_platform_config(struct hfi1_devdata *dd)
diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c
index a5aa3517e7d5..9fc75e7e8781 100644
--- a/drivers/infiniband/hw/hfi1/qp.c
+++ b/drivers/infiniband/hw/hfi1/qp.c
@@ -202,8 +202,7 @@ static void flush_iowait(struct rvt_qp *qp)
 	write_seqlock_irqsave(&dev->iowait_lock, flags);
 	if (!list_empty(&priv->s_iowait.list)) {
 		list_del_init(&priv->s_iowait.list);
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
+		rvt_put_qp(qp);
 	}
 	write_sequnlock_irqrestore(&dev->iowait_lock, flags);
 }
@@ -450,13 +449,14 @@ static void qp_pio_drain(struct rvt_qp *qp)
  */
 void hfi1_schedule_send(struct rvt_qp *qp)
 {
+	lockdep_assert_held(&qp->s_lock);
 	if (hfi1_send_ok(qp))
 		_hfi1_schedule_send(qp);
 }
 
 /**
- * hfi1_get_credit - flush the send work queue of a QP
- * @qp: the qp who's send work queue to flush
+ * hfi1_get_credit - handle credit in aeth
+ * @qp: the qp
  * @aeth: the Acknowledge Extended Transport Header
  *
  * The QP s_lock should be held.
@@ -465,6 +465,7 @@ void hfi1_get_credit(struct rvt_qp *qp, u32 aeth)
 {
 	u32 credit = (aeth >> HFI1_AETH_CREDIT_SHIFT) & HFI1_AETH_CREDIT_MASK;
 
+	lockdep_assert_held(&qp->s_lock);
 	/*
 	 * If the credit is invalid, we can send
 	 * as many packets as we like.  Otherwise, we have to
@@ -503,8 +504,7 @@ void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag)
 	}
 	spin_unlock_irqrestore(&qp->s_lock, flags);
 	/* Notify hfi1_destroy_qp() if it is waiting. */
-	if (atomic_dec_and_test(&qp->refcount))
-		wake_up(&qp->wait);
+	rvt_put_qp(qp);
 }
 
 static int iowait_sleep(
@@ -544,7 +544,7 @@ static int iowait_sleep(
 			qp->s_flags |= RVT_S_WAIT_DMA_DESC;
 			list_add_tail(&priv->s_iowait.list, &sde->dmawait);
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC);
-			atomic_inc(&qp->refcount);
+			rvt_get_qp(qp);
 		}
 		write_sequnlock(&dev->iowait_lock);
 		qp->s_flags &= ~RVT_S_BUSY;
@@ -656,10 +656,6 @@ struct qp_iter *qp_iter_init(struct hfi1_ibdev *dev)
 
 	iter->dev = dev;
 	iter->specials = dev->rdi.ibdev.phys_port_cnt * 2;
-	if (qp_iter_next(iter)) {
-		kfree(iter);
-		return NULL;
-	}
 
 	return iter;
 }
@@ -812,6 +808,13 @@ void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp,
 		kfree(priv);
 		return ERR_PTR(-ENOMEM);
 	}
+	iowait_init(
+		&priv->s_iowait,
+		1,
+		_hfi1_do_send,
+		iowait_sleep,
+		iowait_wakeup,
+		iowait_sdma_drained);
 	setup_timer(&priv->s_rnr_timer, hfi1_rc_rnr_retry, (unsigned long)qp);
 	qp->s_timer.function = hfi1_rc_timeout;
 	return priv;
@@ -852,6 +855,7 @@ unsigned free_all_qps(struct rvt_dev_info *rdi)
 
 void flush_qp_waiters(struct rvt_qp *qp)
 {
+	lockdep_assert_held(&qp->s_lock);
 	flush_iowait(qp);
 	hfi1_stop_rc_timers(qp);
 }
@@ -877,13 +881,6 @@ void notify_qp_reset(struct rvt_qp *qp)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
 
-	iowait_init(
-		&priv->s_iowait,
-		1,
-		_hfi1_do_send,
-		iowait_sleep,
-		iowait_wakeup,
-		iowait_sdma_drained);
 	priv->r_adefered = 0;
 	clear_ahg(qp);
 }
@@ -967,8 +964,7 @@ void notify_error_qp(struct rvt_qp *qp)
 	if (!list_empty(&priv->s_iowait.list) && !(qp->s_flags & RVT_S_BUSY)) {
 		qp->s_flags &= ~RVT_S_ANY_WAIT_IO;
 		list_del_init(&priv->s_iowait.list);
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
+		rvt_put_qp(qp);
 	}
 	write_sequnlock(&dev->iowait_lock);
 
diff --git a/drivers/infiniband/hw/hfi1/qsfp.c b/drivers/infiniband/hw/hfi1/qsfp.c
index a207717ade2a..1869f639c3ae 100644
--- a/drivers/infiniband/hw/hfi1/qsfp.c
+++ b/drivers/infiniband/hw/hfi1/qsfp.c
@@ -161,7 +161,7 @@ static struct hfi1_i2c_bus *init_i2c_bus(struct hfi1_devdata *dd,
 	bus->algo.getsda = hfi1_getsda;
 	bus->algo.getscl = hfi1_getscl;
 	bus->algo.udelay = 5;
-	bus->algo.timeout = usecs_to_jiffies(50);
+	bus->algo.timeout = usecs_to_jiffies(100000);
 	bus->algo.data = bus;
 
 	bus->adapter.owner = THIS_MODULE;
@@ -706,8 +706,8 @@ int get_cable_info(struct hfi1_devdata *dd, u32 port_num, u32 addr, u32 len,
 		   u8 *data)
 {
 	struct hfi1_pportdata *ppd;
-	u32 excess_len = 0;
-	int ret = 0;
+	u32 excess_len = len;
+	int ret = 0, offset = 0;
 
 	if (port_num > dd->num_pports || port_num < 1) {
 		dd_dev_info(dd, "%s: Invalid port number %d\n",
@@ -740,6 +740,34 @@ int get_cable_info(struct hfi1_devdata *dd, u32 port_num, u32 addr, u32 len,
 	}
 
 	memcpy(data, &ppd->qsfp_info.cache[addr], len);
+
+	if (addr <= QSFP_MONITOR_VAL_END &&
+	    (addr + len) >= QSFP_MONITOR_VAL_START) {
+		/* Overlap with the dynamic channel monitor range */
+		if (addr < QSFP_MONITOR_VAL_START) {
+			if (addr + len <= QSFP_MONITOR_VAL_END)
+				len = addr + len - QSFP_MONITOR_VAL_START;
+			else
+				len = QSFP_MONITOR_RANGE;
+			offset = QSFP_MONITOR_VAL_START - addr;
+			addr = QSFP_MONITOR_VAL_START;
+		} else if (addr == QSFP_MONITOR_VAL_START) {
+			offset = 0;
+			if (addr + len > QSFP_MONITOR_VAL_END)
+				len = QSFP_MONITOR_RANGE;
+		} else {
+			offset = 0;
+			if (addr + len > QSFP_MONITOR_VAL_END)
+				len = QSFP_MONITOR_VAL_END - addr + 1;
+		}
+		/* Refresh the values of the dynamic monitors from the cable */
+		ret = one_qsfp_read(ppd, dd->hfi1_id, addr, data + offset, len);
+		if (ret != len) {
+			ret = -EAGAIN;
+			goto set_zeroes;
+		}
+	}
+
 	return 0;
 
 set_zeroes:
diff --git a/drivers/infiniband/hw/hfi1/qsfp.h b/drivers/infiniband/hw/hfi1/qsfp.h
index 69275ebd9597..36cf52359848 100644
--- a/drivers/infiniband/hw/hfi1/qsfp.h
+++ b/drivers/infiniband/hw/hfi1/qsfp.h
@@ -74,6 +74,9 @@
 /* Defined fields that Intel requires of qualified cables */
 /* Byte 0 is Identifier, not checked */
 /* Byte 1 is reserved "status MSB" */
+#define QSFP_MONITOR_VAL_START 22
+#define QSFP_MONITOR_VAL_END 81
+#define QSFP_MONITOR_RANGE (QSFP_MONITOR_VAL_END - QSFP_MONITOR_VAL_START + 1)
 #define QSFP_TX_CTRL_BYTE_OFFS 86
 #define QSFP_PWR_CTRL_BYTE_OFFS 93
 #define QSFP_CDR_CTRL_BYTE_OFFS 98
diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index 5da190e6011b..8bc5013f39a1 100644
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -55,7 +55,7 @@
 #include "trace.h"
 
 /* cut down ridiculously long IB macro names */
-#define OP(x) IB_OPCODE_RC_##x
+#define OP(x) RC_OP(x)
 
 /**
  * hfi1_add_retry_timer - add/start a retry timer
@@ -68,6 +68,7 @@ static inline void hfi1_add_retry_timer(struct rvt_qp *qp)
 	struct ib_qp *ibqp = &qp->ibqp;
 	struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
 
+	lockdep_assert_held(&qp->s_lock);
 	qp->s_flags |= RVT_S_TIMER;
 	/* 4.096 usec. * (1 << qp->timeout) */
 	qp->s_timer.expires = jiffies + qp->timeout_jiffies +
@@ -86,6 +87,7 @@ void hfi1_add_rnr_timer(struct rvt_qp *qp, u32 to)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
 
+	lockdep_assert_held(&qp->s_lock);
 	qp->s_flags |= RVT_S_WAIT_RNR;
 	qp->s_timer.expires = jiffies + usecs_to_jiffies(to);
 	add_timer(&priv->s_rnr_timer);
@@ -103,6 +105,7 @@ static inline void hfi1_mod_retry_timer(struct rvt_qp *qp)
 	struct ib_qp *ibqp = &qp->ibqp;
 	struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
 
+	lockdep_assert_held(&qp->s_lock);
 	qp->s_flags |= RVT_S_TIMER;
 	/* 4.096 usec. * (1 << qp->timeout) */
 	mod_timer(&qp->s_timer, jiffies + qp->timeout_jiffies +
@@ -120,6 +123,7 @@ static inline int hfi1_stop_retry_timer(struct rvt_qp *qp)
 {
 	int rval = 0;
 
+	lockdep_assert_held(&qp->s_lock);
 	/* Remove QP from retry */
 	if (qp->s_flags & RVT_S_TIMER) {
 		qp->s_flags &= ~RVT_S_TIMER;
@@ -138,6 +142,7 @@ void hfi1_stop_rc_timers(struct rvt_qp *qp)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
 
+	lockdep_assert_held(&qp->s_lock);
 	/* Remove QP from all timers */
 	if (qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR)) {
 		qp->s_flags &= ~(RVT_S_TIMER | RVT_S_WAIT_RNR);
@@ -158,6 +163,7 @@ static inline int hfi1_stop_rnr_timer(struct rvt_qp *qp)
 	int rval = 0;
 	struct hfi1_qp_priv *priv = qp->priv;
 
+	lockdep_assert_held(&qp->s_lock);
 	/* Remove QP from rnr timer */
 	if (qp->s_flags & RVT_S_WAIT_RNR) {
 		qp->s_flags &= ~RVT_S_WAIT_RNR;
@@ -178,18 +184,6 @@ void hfi1_del_timers_sync(struct rvt_qp *qp)
 	del_timer_sync(&priv->s_rnr_timer);
 }
 
-/* only opcode mask for adaptive pio */
-const u32 rc_only_opcode =
-	BIT(OP(SEND_ONLY) & 0x1f) |
-	BIT(OP(SEND_ONLY_WITH_IMMEDIATE & 0x1f)) |
-	BIT(OP(RDMA_WRITE_ONLY & 0x1f)) |
-	BIT(OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE & 0x1f)) |
-	BIT(OP(RDMA_READ_REQUEST & 0x1f)) |
-	BIT(OP(ACKNOWLEDGE & 0x1f)) |
-	BIT(OP(ATOMIC_ACKNOWLEDGE & 0x1f)) |
-	BIT(OP(COMPARE_SWAP & 0x1f)) |
-	BIT(OP(FETCH_ADD & 0x1f));
-
 static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe,
 		       u32 psn, u32 pmtu)
 {
@@ -216,7 +210,7 @@ static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe,
  * Note the QP s_lock must be held.
  */
 static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
-		       struct hfi1_other_headers *ohdr,
+		       struct ib_other_headers *ohdr,
 		       struct hfi1_pkt_state *ps)
 {
 	struct rvt_ack_entry *e;
@@ -228,6 +222,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
 	u32 pmtu = qp->pmtu;
 	struct hfi1_qp_priv *priv = qp->priv;
 
+	lockdep_assert_held(&qp->s_lock);
 	/* Don't send an ACK if we aren't supposed to. */
 	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
 		goto bail;
@@ -299,10 +294,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
 			len = 0;
 			qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
 			ohdr->u.at.aeth = hfi1_compute_aeth(qp);
-			ohdr->u.at.atomic_ack_eth[0] =
-				cpu_to_be32(e->atomic_data >> 32);
-			ohdr->u.at.atomic_ack_eth[1] =
-				cpu_to_be32(e->atomic_data);
+			ib_u64_put(e->atomic_data, &ohdr->u.at.atomic_ack_eth);
 			hwords += sizeof(ohdr->u.at) / sizeof(u32);
 			bth2 = mask_psn(e->psn);
 			e->sent = 1;
@@ -390,7 +382,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
 	struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
-	struct hfi1_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct rvt_sge_state *ss;
 	struct rvt_swqe *wqe;
 	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
@@ -403,6 +395,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	int middle = 0;
 	int delta;
 
+	lockdep_assert_held(&qp->s_lock);
 	ps->s_txreq = get_txreq(ps->dev, qp);
 	if (IS_ERR(ps->s_txreq))
 		goto bail_no_tx;
@@ -566,8 +559,9 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 				qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
 				goto bail;
 			}
-			ohdr->u.rc.reth.vaddr =
-				cpu_to_be64(wqe->rdma_wr.remote_addr);
+			put_ib_reth_vaddr(
+				wqe->rdma_wr.remote_addr,
+				&ohdr->u.rc.reth);
 			ohdr->u.rc.reth.rkey =
 				cpu_to_be32(wqe->rdma_wr.rkey);
 			ohdr->u.rc.reth.length = cpu_to_be32(len);
@@ -608,8 +602,9 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 				if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
 					qp->s_lsn++;
 			}
-			ohdr->u.rc.reth.vaddr =
-				cpu_to_be64(wqe->rdma_wr.remote_addr);
+			put_ib_reth_vaddr(
+				wqe->rdma_wr.remote_addr,
+				&ohdr->u.rc.reth);
 			ohdr->u.rc.reth.rkey =
 				cpu_to_be32(wqe->rdma_wr.rkey);
 			ohdr->u.rc.reth.length = cpu_to_be32(len);
@@ -640,20 +635,18 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 			}
 			if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
 				qp->s_state = OP(COMPARE_SWAP);
-				ohdr->u.atomic_eth.swap_data = cpu_to_be64(
-					wqe->atomic_wr.swap);
-				ohdr->u.atomic_eth.compare_data = cpu_to_be64(
-					wqe->atomic_wr.compare_add);
+				put_ib_ateth_swap(wqe->atomic_wr.swap,
+						  &ohdr->u.atomic_eth);
+				put_ib_ateth_compare(wqe->atomic_wr.compare_add,
+						     &ohdr->u.atomic_eth);
 			} else {
 				qp->s_state = OP(FETCH_ADD);
-				ohdr->u.atomic_eth.swap_data = cpu_to_be64(
-					wqe->atomic_wr.compare_add);
-				ohdr->u.atomic_eth.compare_data = 0;
+				put_ib_ateth_swap(wqe->atomic_wr.compare_add,
+						  &ohdr->u.atomic_eth);
+				put_ib_ateth_compare(0, &ohdr->u.atomic_eth);
 			}
-			ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32(
-				wqe->atomic_wr.remote_addr >> 32);
-			ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32(
-				wqe->atomic_wr.remote_addr);
+			put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr,
+					   &ohdr->u.atomic_eth);
 			ohdr->u.atomic_eth.rkey = cpu_to_be32(
 				wqe->atomic_wr.rkey);
 			hwords += sizeof(struct ib_atomic_eth) / sizeof(u32);
@@ -779,8 +772,9 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		 * See restart_rc().
 		 */
 		len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu;
-		ohdr->u.rc.reth.vaddr =
-			cpu_to_be64(wqe->rdma_wr.remote_addr + len);
+		put_ib_reth_vaddr(
+			wqe->rdma_wr.remote_addr + len,
+			&ohdr->u.rc.reth);
 		ohdr->u.rc.reth.rkey =
 			cpu_to_be32(wqe->rdma_wr.rkey);
 		ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);
@@ -841,7 +835,7 @@ bail_no_tx:
  *
  * This is called from hfi1_rc_rcv() and handle_receive_interrupt().
  * Note that RDMA reads and atomics are handled in the
- * send side QP state and tasklet.
+ * send side QP state and send engine.
  */
 void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
 		      int is_fecn)
@@ -856,8 +850,8 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
 	u32 vl, plen;
 	struct send_context *sc;
 	struct pio_buf *pbuf;
-	struct hfi1_ib_header hdr;
-	struct hfi1_other_headers *ohdr;
+	struct ib_header hdr;
+	struct ib_other_headers *ohdr;
 	unsigned long flags;
 
 	/* Don't send ACK or NAK if a RDMA read or atomic is pending. */
@@ -917,7 +911,7 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
 	if (!pbuf) {
 		/*
 		 * We have no room to send at the moment.  Pass
-		 * responsibility for sending the ACK to the send tasklet
+		 * responsibility for sending the ACK to the send engine
 		 * so that when enough buffer space becomes available,
 		 * the ACK is sent ahead of other outgoing packets.
 		 */
@@ -932,16 +926,19 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
 	return;
 
 queue_ack:
-	this_cpu_inc(*ibp->rvp.rc_qacks);
 	spin_lock_irqsave(&qp->s_lock, flags);
+	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
+		goto unlock;
+	this_cpu_inc(*ibp->rvp.rc_qacks);
 	qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
 	qp->s_nak_state = qp->r_nak_state;
 	qp->s_ack_psn = qp->r_ack_psn;
 	if (is_fecn)
 		qp->s_flags |= RVT_S_ECN;
 
-	/* Schedule the send tasklet. */
+	/* Schedule the send engine. */
 	hfi1_schedule_send(qp);
+unlock:
 	spin_unlock_irqrestore(&qp->s_lock, flags);
 }
 
@@ -960,6 +957,7 @@ static void reset_psn(struct rvt_qp *qp, u32 psn)
 	struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
 	u32 opcode;
 
+	lockdep_assert_held(&qp->s_lock);
 	qp->s_cur = n;
 
 	/*
@@ -1027,7 +1025,7 @@ done:
 	qp->s_psn = psn;
 	/*
 	 * Set RVT_S_WAIT_PSN as rc_complete() may start the timer
-	 * asynchronously before the send tasklet can get scheduled.
+	 * asynchronously before the send engine can get scheduled.
 	 * Doing it in hfi1_make_rc_req() is too late.
 	 */
 	if ((cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) &&
@@ -1045,6 +1043,8 @@ static void restart_rc(struct rvt_qp *qp, u32 psn, int wait)
 	struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
 	struct hfi1_ibport *ibp;
 
+	lockdep_assert_held(&qp->r_lock);
+	lockdep_assert_held(&qp->s_lock);
 	if (qp->s_retry == 0) {
 		if (qp->s_mig_state == IB_MIG_ARMED) {
 			hfi1_migrate_qp(qp);
@@ -1121,6 +1121,7 @@ static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
 	struct rvt_swqe *wqe;
 	u32 n = qp->s_last;
 
+	lockdep_assert_held(&qp->s_lock);
 	/* Find the work request corresponding to the given PSN. */
 	for (;;) {
 		wqe = rvt_get_swqe_ptr(qp, n);
@@ -1141,15 +1142,16 @@ static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
 /*
  * This should be called with the QP s_lock held and interrupts disabled.
  */
-void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_ib_header *hdr)
+void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 {
-	struct hfi1_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct rvt_swqe *wqe;
 	struct ib_wc wc;
 	unsigned i;
 	u32 opcode;
 	u32 psn;
 
+	lockdep_assert_held(&qp->s_lock);
 	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND))
 		return;
 
@@ -1241,6 +1243,7 @@ static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
 	struct ib_wc wc;
 	unsigned i;
 
+	lockdep_assert_held(&qp->s_lock);
 	/*
 	 * Don't decrement refcount and don't generate a
 	 * completion if the SWQE is being resent until the send
@@ -1340,6 +1343,7 @@ static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
 	int diff;
 	unsigned long to;
 
+	lockdep_assert_held(&qp->s_lock);
 	/*
 	 * Note that NAKs implicitly ACK outstanding SEND and RDMA write
 	 * requests and implicitly NAK RDMA read and atomic requests issued
@@ -1389,7 +1393,7 @@ static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
 				restart_rc(qp, qp->s_last_psn + 1, 0);
 				if (list_empty(&qp->rspwait)) {
 					qp->r_flags |= RVT_R_RSP_SEND;
-					atomic_inc(&qp->refcount);
+					rvt_get_qp(qp);
 					list_add_tail(&qp->rspwait,
 						      &rcd->qp_wait_list);
 				}
@@ -1555,6 +1559,7 @@ static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn,
 {
 	struct rvt_swqe *wqe;
 
+	lockdep_assert_held(&qp->s_lock);
 	/* Remove QP from retry timer */
 	hfi1_stop_rc_timers(qp);
 
@@ -1573,7 +1578,7 @@ static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn,
 	restart_rc(qp, qp->s_last_psn + 1, 0);
 	if (list_empty(&qp->rspwait)) {
 		qp->r_flags |= RVT_R_RSP_SEND;
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
 	}
 }
@@ -1595,7 +1600,7 @@ static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn,
  * Called at interrupt level.
  */
 static void rc_rcv_resp(struct hfi1_ibport *ibp,
-			struct hfi1_other_headers *ohdr,
+			struct ib_other_headers *ohdr,
 			void *data, u32 tlen, struct rvt_qp *qp,
 			u32 opcode, u32 psn, u32 hdrsize, u32 pmtu,
 			struct hfi1_ctxtdata *rcd)
@@ -1649,14 +1654,10 @@ static void rc_rcv_resp(struct hfi1_ibport *ibp,
 	case OP(ATOMIC_ACKNOWLEDGE):
 	case OP(RDMA_READ_RESPONSE_FIRST):
 		aeth = be32_to_cpu(ohdr->u.aeth);
-		if (opcode == OP(ATOMIC_ACKNOWLEDGE)) {
-			__be32 *p = ohdr->u.at.atomic_ack_eth;
-
-			val = ((u64)be32_to_cpu(p[0]) << 32) |
-				be32_to_cpu(p[1]);
-		} else {
+		if (opcode == OP(ATOMIC_ACKNOWLEDGE))
+			val = ib_u64_get(&ohdr->u.at.atomic_ack_eth);
+		else
 			val = 0;
-		}
 		if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
 		    opcode != OP(RDMA_READ_RESPONSE_FIRST))
 			goto ack_done;
@@ -1782,7 +1783,7 @@ static inline void rc_defered_ack(struct hfi1_ctxtdata *rcd,
 {
 	if (list_empty(&qp->rspwait)) {
 		qp->r_flags |= RVT_R_RSP_NAK;
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
 	}
 }
@@ -1796,8 +1797,7 @@ static inline void rc_cancel_ack(struct rvt_qp *qp)
 		return;
 	list_del_init(&qp->rspwait);
 	qp->r_flags &= ~RVT_R_RSP_NAK;
-	if (atomic_dec_and_test(&qp->refcount))
-		wake_up(&qp->wait);
+	rvt_put_qp(qp);
 }
 
 /**
@@ -1815,7 +1815,7 @@ static inline void rc_cancel_ack(struct rvt_qp *qp)
  * Return 1 if no more processing is needed; otherwise return 0 to
  * schedule a response to be sent.
  */
-static noinline int rc_rcv_error(struct hfi1_other_headers *ohdr, void *data,
+static noinline int rc_rcv_error(struct ib_other_headers *ohdr, void *data,
 				 struct rvt_qp *qp, u32 opcode, u32 psn,
 				 int diff, struct hfi1_ctxtdata *rcd)
 {
@@ -1923,7 +1923,7 @@ static noinline int rc_rcv_error(struct hfi1_other_headers *ohdr, void *data,
 		}
 		if (len != 0) {
 			u32 rkey = be32_to_cpu(reth->rkey);
-			u64 vaddr = be64_to_cpu(reth->vaddr);
+			u64 vaddr = get_ib_reth_vaddr(reth);
 			int ok;
 
 			ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
@@ -1946,7 +1946,7 @@ static noinline int rc_rcv_error(struct hfi1_other_headers *ohdr, void *data,
 	case OP(FETCH_ADD): {
 		/*
 		 * If we didn't find the atomic request in the ack queue
-		 * or the send tasklet is already backed up to send an
+		 * or the send engine is already backed up to send an
 		 * earlier entry, we can ignore this request.
 		 */
 		if (!e || e->opcode != (u8)opcode || old_req)
@@ -2123,13 +2123,13 @@ void process_becn(struct hfi1_pportdata *ppd, u8 sl, u16 rlid, u32 lqpn,
 void hfi1_rc_rcv(struct hfi1_packet *packet)
 {
 	struct hfi1_ctxtdata *rcd = packet->rcd;
-	struct hfi1_ib_header *hdr = packet->hdr;
+	struct ib_header *hdr = packet->hdr;
 	u32 rcv_flags = packet->rcv_flags;
 	void *data = packet->ebuf;
 	u32 tlen = packet->tlen;
 	struct rvt_qp *qp = packet->qp;
 	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
-	struct hfi1_other_headers *ohdr = packet->ohdr;
+	struct ib_other_headers *ohdr = packet->ohdr;
 	u32 bth0, opcode;
 	u32 hdrsize = packet->hlen;
 	u32 psn;
@@ -2143,6 +2143,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 	int copy_last = 0;
 	u32 rkey;
 
+	lockdep_assert_held(&qp->r_lock);
 	bth0 = be32_to_cpu(ohdr->bth[0]);
 	if (hfi1_ruc_check_hdr(ibp, hdr, rcv_flags & HFI1_HAS_GRH, qp, bth0))
 		return;
@@ -2342,7 +2343,7 @@ send_last:
 		qp->r_sge.sg_list = NULL;
 		if (qp->r_len != 0) {
 			u32 rkey = be32_to_cpu(reth->rkey);
-			u64 vaddr = be64_to_cpu(reth->vaddr);
+			u64 vaddr = get_ib_reth_vaddr(reth);
 			int ok;
 
 			/* Check rkey & NAK */
@@ -2397,7 +2398,7 @@ send_last:
 		len = be32_to_cpu(reth->length);
 		if (len) {
 			u32 rkey = be32_to_cpu(reth->rkey);
-			u64 vaddr = be64_to_cpu(reth->vaddr);
+			u64 vaddr = get_ib_reth_vaddr(reth);
 			int ok;
 
 			/* Check rkey & NAK */
@@ -2432,7 +2433,7 @@ send_last:
 		qp->r_nak_state = 0;
 		qp->r_head_ack_queue = next;
 
-		/* Schedule the send tasklet. */
+		/* Schedule the send engine. */
 		qp->s_flags |= RVT_S_RESP_PENDING;
 		hfi1_schedule_send(qp);
 
@@ -2469,8 +2470,7 @@ send_last:
 			e->rdma_sge.mr = NULL;
 		}
 		ateth = &ohdr->u.atomic_eth;
-		vaddr = ((u64)be32_to_cpu(ateth->vaddr[0]) << 32) |
-			be32_to_cpu(ateth->vaddr[1]);
+		vaddr = get_ib_ateth_vaddr(ateth);
 		if (unlikely(vaddr & (sizeof(u64) - 1)))
 			goto nack_inv_unlck;
 		rkey = be32_to_cpu(ateth->rkey);
@@ -2481,11 +2481,11 @@ send_last:
 			goto nack_acc_unlck;
 		/* Perform atomic OP and save result. */
 		maddr = (atomic64_t *)qp->r_sge.sge.vaddr;
-		sdata = be64_to_cpu(ateth->swap_data);
+		sdata = get_ib_ateth_swap(ateth);
 		e->atomic_data = (opcode == OP(FETCH_ADD)) ?
 			(u64)atomic64_add_return(sdata, maddr) - sdata :
 			(u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr,
-				      be64_to_cpu(ateth->compare_data),
+				      get_ib_ateth_compare(ateth),
 				      sdata);
 		rvt_put_mr(qp->r_sge.sge.mr);
 		qp->r_sge.num_sge = 0;
@@ -2499,7 +2499,7 @@ send_last:
 		qp->r_nak_state = 0;
 		qp->r_head_ack_queue = next;
 
-		/* Schedule the send tasklet. */
+		/* Schedule the send engine. */
 		qp->s_flags |= RVT_S_RESP_PENDING;
 		hfi1_schedule_send(qp);
 
@@ -2575,12 +2575,12 @@ send_ack:
 
 void hfi1_rc_hdrerr(
 	struct hfi1_ctxtdata *rcd,
-	struct hfi1_ib_header *hdr,
+	struct ib_header *hdr,
 	u32 rcv_flags,
 	struct rvt_qp *qp)
 {
 	int has_grh = rcv_flags & HFI1_HAS_GRH;
-	struct hfi1_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
 	int diff;
 	u32 opcode;
diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c
index 48d5094f98e2..a1576aea4756 100644
--- a/drivers/infiniband/hw/hfi1/ruc.c
+++ b/drivers/infiniband/hw/hfi1/ruc.c
@@ -262,7 +262,7 @@ static int gid_ok(union ib_gid *gid, __be64 gid_prefix, __be64 id)
  *
  * The s_lock will be acquired around the hfi1_migrate_qp() call.
  */
-int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_ib_header *hdr,
+int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct ib_header *hdr,
 		       int has_grh, struct rvt_qp *qp, u32 bth0)
 {
 	__be64 guid;
@@ -352,7 +352,7 @@ err:
  *
  * This is called from hfi1_do_send() to
  * forward a WQE addressed to the same HFI.
- * Note that although we are single threaded due to the tasklet, we still
+ * Note that although we are single threaded due to the send engine, we still
  * have to protect against post_send().  We don't have to worry about
  * receive interrupts since this is a connected protocol and all packets
  * will pass through here.
@@ -765,7 +765,7 @@ static inline void build_ahg(struct rvt_qp *qp, u32 npsn)
 	}
 }
 
-void hfi1_make_ruc_header(struct rvt_qp *qp, struct hfi1_other_headers *ohdr,
+void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
 			  u32 bth0, u32 bth2, int middle,
 			  struct hfi1_pkt_state *ps)
 {
@@ -846,7 +846,7 @@ void _hfi1_do_send(struct work_struct *work)
  * @work: contains a pointer to the QP
  *
  * Process entries in the send work queue until credit or queue is
- * exhausted.  Only allow one CPU to send a packet per QP (tasklet).
+ * exhausted.  Only allow one CPU to send a packet per QP.
  * Otherwise, two threads could send packets out of order.
  */
 void hfi1_do_send(struct rvt_qp *qp)
@@ -909,7 +909,7 @@ void hfi1_do_send(struct rvt_qp *qp)
 			spin_unlock_irqrestore(&qp->s_lock, ps.flags);
 			/*
 			 * If the packet cannot be sent now, return and
-			 * the send tasklet will be woken up later.
+			 * the send engine will be woken up later.
 			 */
 			if (hfi1_verbs_send(qp, &ps))
 				return;
diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
index f9befc05b349..fd39bcaa062d 100644
--- a/drivers/infiniband/hw/hfi1/sdma.c
+++ b/drivers/infiniband/hw/hfi1/sdma.c
@@ -726,6 +726,34 @@ u16 sdma_get_descq_cnt(void)
 }
 
 /**
+ * sdma_engine_get_vl() - return vl for a given sdma engine
+ * @sde: sdma engine
+ *
+ * This function returns the vl mapped to a given engine, or an error if
+ * the mapping can't be found. The mapping fields are protected by RCU.
+ */
+int sdma_engine_get_vl(struct sdma_engine *sde)
+{
+	struct hfi1_devdata *dd = sde->dd;
+	struct sdma_vl_map *m;
+	u8 vl;
+
+	if (sde->this_idx >= TXE_NUM_SDMA_ENGINES)
+		return -EINVAL;
+
+	rcu_read_lock();
+	m = rcu_dereference(dd->sdma_map);
+	if (unlikely(!m)) {
+		rcu_read_unlock();
+		return -EINVAL;
+	}
+	vl = m->engine_to_vl[sde->this_idx];
+	rcu_read_unlock();
+
+	return vl;
+}
+
+/**
  * sdma_select_engine_vl() - select sdma engine
  * @dd: devdata
  * @selector: a spreading factor
@@ -788,6 +816,326 @@ struct sdma_engine *sdma_select_engine_sc(
 	return sdma_select_engine_vl(dd, selector, vl);
 }
 
+struct sdma_rht_map_elem {
+	u32 mask;
+	u8 ctr;
+	struct sdma_engine *sde[0];
+};
+
+struct sdma_rht_node {
+	unsigned long cpu_id;
+	struct sdma_rht_map_elem *map[HFI1_MAX_VLS_SUPPORTED];
+	struct rhash_head node;
+};
+
+#define NR_CPUS_HINT 192
+
+static const struct rhashtable_params sdma_rht_params = {
+	.nelem_hint = NR_CPUS_HINT,
+	.head_offset = offsetof(struct sdma_rht_node, node),
+	.key_offset = offsetof(struct sdma_rht_node, cpu_id),
+	.key_len = FIELD_SIZEOF(struct sdma_rht_node, cpu_id),
+	.max_size = NR_CPUS,
+	.min_size = 8,
+	.automatic_shrinking = true,
+};
+
+/*
+ * sdma_select_user_engine() - select sdma engine based on user setup
+ * @dd: devdata
+ * @selector: a spreading factor
+ * @vl: this vl
+ *
+ * This function returns an sdma engine for a user sdma request.
+ * User defined sdma engine affinity setting is honored when applicable,
+ * otherwise system default sdma engine mapping is used. To ensure correct
+ * ordering, the mapping from <selector, vl> to sde must remain unchanged.
+ */
+struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd,
+					    u32 selector, u8 vl)
+{
+	struct sdma_rht_node *rht_node;
+	struct sdma_engine *sde = NULL;
+	const struct cpumask *current_mask = tsk_cpus_allowed(current);
+	unsigned long cpu_id;
+
+	/*
+	 * To ensure that always the same sdma engine(s) will be
+	 * selected make sure the process is pinned to this CPU only.
+	 */
+	if (cpumask_weight(current_mask) != 1)
+		goto out;
+
+	cpu_id = smp_processor_id();
+	rcu_read_lock();
+	rht_node = rhashtable_lookup_fast(&dd->sdma_rht, &cpu_id,
+					  sdma_rht_params);
+
+	if (rht_node && rht_node->map[vl]) {
+		struct sdma_rht_map_elem *map = rht_node->map[vl];
+
+		sde = map->sde[selector & map->mask];
+	}
+	rcu_read_unlock();
+
+	if (sde)
+		return sde;
+
+out:
+	return sdma_select_engine_vl(dd, selector, vl);
+}
+
+static void sdma_populate_sde_map(struct sdma_rht_map_elem *map)
+{
+	int i;
+
+	for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++)
+		map->sde[map->ctr + i] = map->sde[i];
+}
+
+static void sdma_cleanup_sde_map(struct sdma_rht_map_elem *map,
+				 struct sdma_engine *sde)
+{
+	unsigned int i, pow;
+
+	/* only need to check the first ctr entries for a match */
+	for (i = 0; i < map->ctr; i++) {
+		if (map->sde[i] == sde) {
+			memmove(&map->sde[i], &map->sde[i + 1],
+				(map->ctr - i - 1) * sizeof(map->sde[0]));
+			map->ctr--;
+			pow = roundup_pow_of_two(map->ctr ? : 1);
+			map->mask = pow - 1;
+			sdma_populate_sde_map(map);
+			break;
+		}
+	}
+}
+
+/*
+ * Prevents concurrent reads and writes of the sdma engine cpu_mask
+ */
+static DEFINE_MUTEX(process_to_sde_mutex);
+
+ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf,
+				size_t count)
+{
+	struct hfi1_devdata *dd = sde->dd;
+	cpumask_var_t mask, new_mask;
+	unsigned long cpu;
+	int ret, vl, sz;
+
+	vl = sdma_engine_get_vl(sde);
+	if (unlikely(vl < 0))
+		return -EINVAL;
+
+	ret = zalloc_cpumask_var(&mask, GFP_KERNEL);
+	if (!ret)
+		return -ENOMEM;
+
+	ret = zalloc_cpumask_var(&new_mask, GFP_KERNEL);
+	if (!ret) {
+		free_cpumask_var(mask);
+		return -ENOMEM;
+	}
+	ret = cpulist_parse(buf, mask);
+	if (ret)
+		goto out_free;
+
+	if (!cpumask_subset(mask, cpu_online_mask)) {
+		dd_dev_warn(sde->dd, "Invalid CPU mask\n");
+		ret = -EINVAL;
+		goto out_free;
+	}
+
+	sz = sizeof(struct sdma_rht_map_elem) +
+			(TXE_NUM_SDMA_ENGINES * sizeof(struct sdma_engine *));
+
+	mutex_lock(&process_to_sde_mutex);
+
+	for_each_cpu(cpu, mask) {
+		struct sdma_rht_node *rht_node;
+
+		/* Check if we have this already mapped */
+		if (cpumask_test_cpu(cpu, &sde->cpu_mask)) {
+			cpumask_set_cpu(cpu, new_mask);
+			continue;
+		}
+
+		rht_node = rhashtable_lookup_fast(&dd->sdma_rht, &cpu,
+						  sdma_rht_params);
+		if (!rht_node) {
+			rht_node = kzalloc(sizeof(*rht_node), GFP_KERNEL);
+			if (!rht_node) {
+				ret = -ENOMEM;
+				goto out;
+			}
+
+			rht_node->map[vl] = kzalloc(sz, GFP_KERNEL);
+			if (!rht_node->map[vl]) {
+				kfree(rht_node);
+				ret = -ENOMEM;
+				goto out;
+			}
+			rht_node->cpu_id = cpu;
+			rht_node->map[vl]->mask = 0;
+			rht_node->map[vl]->ctr = 1;
+			rht_node->map[vl]->sde[0] = sde;
+
+			ret = rhashtable_insert_fast(&dd->sdma_rht,
+						     &rht_node->node,
+						     sdma_rht_params);
+			if (ret) {
+				kfree(rht_node->map[vl]);
+				kfree(rht_node);
+				dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n",
+					   cpu);
+				goto out;
+			}
+
+		} else {
+			int ctr, pow;
+
+			/* Add new user mappings */
+			if (!rht_node->map[vl])
+				rht_node->map[vl] = kzalloc(sz, GFP_KERNEL);
+
+			if (!rht_node->map[vl]) {
+				ret = -ENOMEM;
+				goto out;
+			}
+
+			rht_node->map[vl]->ctr++;
+			ctr = rht_node->map[vl]->ctr;
+			rht_node->map[vl]->sde[ctr - 1] = sde;
+			pow = roundup_pow_of_two(ctr);
+			rht_node->map[vl]->mask = pow - 1;
+
+			/* Populate the sde map table */
+			sdma_populate_sde_map(rht_node->map[vl]);
+		}
+		cpumask_set_cpu(cpu, new_mask);
+	}
+
+	/* Clean up old mappings */
+	for_each_cpu(cpu, cpu_online_mask) {
+		struct sdma_rht_node *rht_node;
+
+		/* Don't cleanup sdes that are set in the new mask */
+		if (cpumask_test_cpu(cpu, mask))
+			continue;
+
+		rht_node = rhashtable_lookup_fast(&dd->sdma_rht, &cpu,
+						  sdma_rht_params);
+		if (rht_node) {
+			bool empty = true;
+			int i;
+
+			/* Remove mappings for old sde */
+			for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
+				if (rht_node->map[i])
+					sdma_cleanup_sde_map(rht_node->map[i],
+							     sde);
+
+			/* Free empty hash table entries */
+			for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) {
+				if (!rht_node->map[i])
+					continue;
+
+				if (rht_node->map[i]->ctr) {
+					empty = false;
+					break;
+				}
+			}
+
+			if (empty) {
+				ret = rhashtable_remove_fast(&dd->sdma_rht,
+							     &rht_node->node,
+							     sdma_rht_params);
+				WARN_ON(ret);
+
+				for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
+					kfree(rht_node->map[i]);
+
+				kfree(rht_node);
+			}
+		}
+	}
+
+	cpumask_copy(&sde->cpu_mask, new_mask);
+out:
+	mutex_unlock(&process_to_sde_mutex);
+out_free:
+	free_cpumask_var(mask);
+	free_cpumask_var(new_mask);
+	return ret ? : strnlen(buf, PAGE_SIZE);
+}
+
+ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf)
+{
+	mutex_lock(&process_to_sde_mutex);
+	if (cpumask_empty(&sde->cpu_mask))
+		snprintf(buf, PAGE_SIZE, "%s\n", "empty");
+	else
+		cpumap_print_to_pagebuf(true, buf, &sde->cpu_mask);
+	mutex_unlock(&process_to_sde_mutex);
+	return strnlen(buf, PAGE_SIZE);
+}
+
+static void sdma_rht_free(void *ptr, void *arg)
+{
+	struct sdma_rht_node *rht_node = ptr;
+	int i;
+
+	for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
+		kfree(rht_node->map[i]);
+
+	kfree(rht_node);
+}
+
+/**
+ * sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings
+ * @s: seq file
+ * @dd: hfi1_devdata
+ * @cpuid: cpu id
+ *
+ * This routine dumps the process to sde mappings per cpu
+ */
+void sdma_seqfile_dump_cpu_list(struct seq_file *s,
+				struct hfi1_devdata *dd,
+				unsigned long cpuid)
+{
+	struct sdma_rht_node *rht_node;
+	int i, j;
+
+	rht_node = rhashtable_lookup_fast(&dd->sdma_rht, &cpuid,
+					  sdma_rht_params);
+	if (!rht_node)
+		return;
+
+	seq_printf(s, "cpu%3lu: ", cpuid);
+	for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) {
+		if (!rht_node->map[i] || !rht_node->map[i]->ctr)
+			continue;
+
+		seq_printf(s, " vl%d: [", i);
+
+		for (j = 0; j < rht_node->map[i]->ctr; j++) {
+			if (!rht_node->map[i]->sde[j])
+				continue;
+
+			if (j > 0)
+				seq_puts(s, ",");
+
+			seq_printf(s, " sdma%2d",
+				   rht_node->map[i]->sde[j]->this_idx);
+		}
+		seq_puts(s, " ]");
+	}
+
+	seq_puts(s, "\n");
+}
+
 /*
  * Free the indicated map struct
  */
@@ -1161,6 +1509,10 @@ int sdma_init(struct hfi1_devdata *dd, u8 port)
 	dd->num_sdma = num_engines;
 	if (sdma_map_init(dd, port, ppd->vls_operational, NULL))
 		goto bail;
+
+	if (rhashtable_init(&dd->sdma_rht, &sdma_rht_params))
+		goto bail;
+
 	dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma);
 	return 0;
 
@@ -1252,6 +1604,7 @@ void sdma_exit(struct hfi1_devdata *dd)
 		sdma_finalput(&sde->state);
 	}
 	sdma_clean(dd, dd->num_sdma);
+	rhashtable_free_and_destroy(&dd->sdma_rht, sdma_rht_free, NULL);
 }
 
 /*
@@ -2086,6 +2439,11 @@ nodesc:
  * @sde: sdma engine to use
  * @wait: wait structure to use when full (may be NULL)
  * @tx_list: list of sdma_txreqs to submit
+ * @count: pointer to a u32 which, after return will contain the total number of
+ *         sdma_txreqs removed from the tx_list. This will include sdma_txreqs
+ *         whose SDMA descriptors are submitted to the ring and the sdma_txreqs
+ *         which are added to SDMA engine flush list if the SDMA engine state is
+ *         not running.
  *
  * The call submits the list into the ring.
  *
@@ -2100,18 +2458,18 @@ nodesc:
  * side locking.
  *
  * Return:
- * > 0 - Success (value is number of sdma_txreq's submitted),
+ * 0 - Success,
  * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL)
  * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
  */
 int sdma_send_txlist(struct sdma_engine *sde, struct iowait *wait,
-		     struct list_head *tx_list)
+		     struct list_head *tx_list, u32 *count_out)
 {
 	struct sdma_txreq *tx, *tx_next;
 	int ret = 0;
 	unsigned long flags;
 	u16 tail = INVALID_TAIL;
-	int count = 0;
+	u32 submit_count = 0, flush_count = 0, total_count;
 
 	spin_lock_irqsave(&sde->tail_lock, flags);
 retry:
@@ -2127,33 +2485,34 @@ retry:
 		}
 		list_del_init(&tx->list);
 		tail = submit_tx(sde, tx);
-		count++;
+		submit_count++;
 		if (tail != INVALID_TAIL &&
-		    (count & SDMA_TAIL_UPDATE_THRESH) == 0) {
+		    (submit_count & SDMA_TAIL_UPDATE_THRESH) == 0) {
 			sdma_update_tail(sde, tail);
 			tail = INVALID_TAIL;
 		}
 	}
 update_tail:
+	total_count = submit_count + flush_count;
 	if (wait)
-		iowait_sdma_add(wait, count);
+		iowait_sdma_add(wait, total_count);
 	if (tail != INVALID_TAIL)
 		sdma_update_tail(sde, tail);
 	spin_unlock_irqrestore(&sde->tail_lock, flags);
-	return ret == 0 ? count : ret;
+	*count_out = total_count;
+	return ret;
 unlock_noconn:
 	spin_lock(&sde->flushlist_lock);
 	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
 		tx->wait = wait;
 		list_del_init(&tx->list);
-		if (wait)
-			iowait_sdma_inc(wait);
 		tx->next_descq_idx = 0;
 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
 		tx->sn = sde->tail_sn++;
 		trace_hfi1_sdma_in_sn(sde, tx->sn);
 #endif
 		list_add_tail(&tx->list, &sde->flushlist);
+		flush_count++;
 		if (wait) {
 			wait->tx_count++;
 			wait->count += tx->num_desc;
diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h
index 8f50c99fe711..56257ea3598f 100644
--- a/drivers/infiniband/hw/hfi1/sdma.h
+++ b/drivers/infiniband/hw/hfi1/sdma.h
@@ -413,6 +413,8 @@ struct sdma_engine {
 	spinlock_t flushlist_lock;
 	/* private: */
 	struct list_head flushlist;
+	struct cpumask cpu_mask;
+	struct kobject kobj;
 };
 
 int sdma_init(struct hfi1_devdata *dd, u8 port);
@@ -847,7 +849,8 @@ int sdma_send_txreq(struct sdma_engine *sde,
 		    struct sdma_txreq *tx);
 int sdma_send_txlist(struct sdma_engine *sde,
 		     struct iowait *wait,
-		     struct list_head *tx_list);
+		     struct list_head *tx_list,
+		     u32 *count);
 
 int sdma_ahg_alloc(struct sdma_engine *sde);
 void sdma_ahg_free(struct sdma_engine *sde, int ahg_index);
@@ -1058,7 +1061,15 @@ struct sdma_engine *sdma_select_engine_vl(
 	u32 selector,
 	u8 vl);
 
+struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd,
+					    u32 selector, u8 vl);
+ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf);
+ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf,
+				size_t count);
+int sdma_engine_get_vl(struct sdma_engine *sde);
 void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *);
+void sdma_seqfile_dump_cpu_list(struct seq_file *s, struct hfi1_devdata *dd,
+				unsigned long cpuid);
 
 #ifdef CONFIG_SDMA_VERBOSITY
 void sdma_dumpstate(struct sdma_engine *);
diff --git a/drivers/infiniband/hw/hfi1/sysfs.c b/drivers/infiniband/hw/hfi1/sysfs.c
index 74c84c655f7e..edba22461a9c 100644
--- a/drivers/infiniband/hw/hfi1/sysfs.c
+++ b/drivers/infiniband/hw/hfi1/sysfs.c
@@ -766,13 +766,95 @@ bail:
 	return ret;
 }
 
+struct sde_attribute {
+	struct attribute attr;
+	ssize_t (*show)(struct sdma_engine *sde, char *buf);
+	ssize_t (*store)(struct sdma_engine *sde, const char *buf, size_t cnt);
+};
+
+static ssize_t sde_show(struct kobject *kobj, struct attribute *attr, char *buf)
+{
+	struct sde_attribute *sde_attr =
+		container_of(attr, struct sde_attribute, attr);
+	struct sdma_engine *sde =
+		container_of(kobj, struct sdma_engine, kobj);
+
+	if (!sde_attr->show)
+		return -EINVAL;
+
+	return sde_attr->show(sde, buf);
+}
+
+static ssize_t sde_store(struct kobject *kobj, struct attribute *attr,
+			 const char *buf, size_t count)
+{
+	struct sde_attribute *sde_attr =
+		container_of(attr, struct sde_attribute, attr);
+	struct sdma_engine *sde =
+		container_of(kobj, struct sdma_engine, kobj);
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	if (!sde_attr->store)
+		return -EINVAL;
+
+	return sde_attr->store(sde, buf, count);
+}
+
+static const struct sysfs_ops sde_sysfs_ops = {
+	.show = sde_show,
+	.store = sde_store,
+};
+
+static struct kobj_type sde_ktype = {
+	.sysfs_ops = &sde_sysfs_ops,
+};
+
+#define SDE_ATTR(_name, _mode, _show, _store) \
+	struct sde_attribute sde_attr_##_name = \
+		__ATTR(_name, _mode, _show, _store)
+
+static ssize_t sde_show_cpu_to_sde_map(struct sdma_engine *sde, char *buf)
+{
+	return sdma_get_cpu_to_sde_map(sde, buf);
+}
+
+static ssize_t sde_store_cpu_to_sde_map(struct sdma_engine *sde,
+					const char *buf, size_t count)
+{
+	return sdma_set_cpu_to_sde_map(sde, buf, count);
+}
+
+static ssize_t sde_show_vl(struct sdma_engine *sde, char *buf)
+{
+	int vl;
+
+	vl = sdma_engine_get_vl(sde);
+	if (vl < 0)
+		return vl;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", vl);
+}
+
+static SDE_ATTR(cpu_list, S_IWUSR | S_IRUGO,
+		sde_show_cpu_to_sde_map,
+		sde_store_cpu_to_sde_map);
+static SDE_ATTR(vl, S_IRUGO, sde_show_vl, NULL);
+
+static struct sde_attribute *sde_attribs[] = {
+	&sde_attr_cpu_list,
+	&sde_attr_vl
+};
+
 /*
  * Register and create our files in /sys/class/infiniband.
  */
 int hfi1_verbs_register_sysfs(struct hfi1_devdata *dd)
 {
 	struct ib_device *dev = &dd->verbs_dev.rdi.ibdev;
-	int i, ret;
+	struct device *class_dev = &dev->dev;
+	int i, j, ret;
 
 	for (i = 0; i < ARRAY_SIZE(hfi1_attributes); ++i) {
 		ret = device_create_file(&dev->dev, hfi1_attributes[i]);
@@ -780,10 +862,29 @@ int hfi1_verbs_register_sysfs(struct hfi1_devdata *dd)
 			goto bail;
 	}
 
+	for (i = 0; i < dd->num_sdma; i++) {
+		ret = kobject_init_and_add(&dd->per_sdma[i].kobj,
+					   &sde_ktype, &class_dev->kobj,
+					   "sdma%d", i);
+		if (ret)
+			goto bail;
+
+		for (j = 0; j < ARRAY_SIZE(sde_attribs); j++) {
+			ret = sysfs_create_file(&dd->per_sdma[i].kobj,
+						&sde_attribs[j]->attr);
+			if (ret)
+				goto bail;
+		}
+	}
+
 	return 0;
 bail:
 	for (i = 0; i < ARRAY_SIZE(hfi1_attributes); ++i)
 		device_remove_file(&dev->dev, hfi1_attributes[i]);
+
+	for (i = 0; i < dd->num_sdma; i++)
+		kobject_del(&dd->per_sdma[i].kobj);
+
 	return ret;
 }
 
diff --git a/drivers/infiniband/hw/hfi1/trace.c b/drivers/infiniband/hw/hfi1/trace.c
index 4cfb13771897..01f525cd985a 100644
--- a/drivers/infiniband/hw/hfi1/trace.c
+++ b/drivers/infiniband/hw/hfi1/trace.c
@@ -47,9 +47,9 @@
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 
-u8 ibhdr_exhdr_len(struct hfi1_ib_header *hdr)
+u8 ibhdr_exhdr_len(struct ib_header *hdr)
 {
-	struct hfi1_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	u8 opcode;
 	u8 lnh = (u8)(be16_to_cpu(hdr->lrh[0]) & 3);
 
@@ -67,16 +67,11 @@ u8 ibhdr_exhdr_len(struct hfi1_ib_header *hdr)
 #define AETH_PRN "aeth syn 0x%.2x %s msn 0x%.8x"
 #define DETH_PRN "deth qkey 0x%.8x sqpn 0x%.6x"
 #define IETH_PRN "ieth rkey 0x%.8x"
-#define ATOMICACKETH_PRN "origdata %lld"
-#define ATOMICETH_PRN "vaddr 0x%llx rkey 0x%.8x sdata %lld cdata %lld"
+#define ATOMICACKETH_PRN "origdata %llx"
+#define ATOMICETH_PRN "vaddr 0x%llx rkey 0x%.8x sdata %llx cdata %llx"
 
 #define OP(transport, op) IB_OPCODE_## transport ## _ ## op
 
-static u64 ib_u64_get(__be32 *p)
-{
-	return ((u64)be32_to_cpu(p[0]) << 32) | be32_to_cpu(p[1]);
-}
-
 static const char *parse_syndrome(u8 syndrome)
 {
 	switch (syndrome >> 5) {
@@ -113,8 +108,7 @@ const char *parse_everbs_hdrs(
 	case OP(RC, RDMA_WRITE_ONLY_WITH_IMMEDIATE):
 	case OP(UC, RDMA_WRITE_ONLY_WITH_IMMEDIATE):
 		trace_seq_printf(p, RETH_PRN " " IMM_PRN,
-				 (unsigned long long)ib_u64_get(
-				 (__be32 *)&eh->rc.reth.vaddr),
+				 get_ib_reth_vaddr(&eh->rc.reth),
 				 be32_to_cpu(eh->rc.reth.rkey),
 				 be32_to_cpu(eh->rc.reth.length),
 				 be32_to_cpu(eh->rc.imm_data));
@@ -126,8 +120,7 @@ const char *parse_everbs_hdrs(
 	case OP(RC, RDMA_WRITE_ONLY):
 	case OP(UC, RDMA_WRITE_ONLY):
 		trace_seq_printf(p, RETH_PRN,
-				 (unsigned long long)ib_u64_get(
-				 (__be32 *)&eh->rc.reth.vaddr),
+				 get_ib_reth_vaddr(&eh->rc.reth),
 				 be32_to_cpu(eh->rc.reth.rkey),
 				 be32_to_cpu(eh->rc.reth.length));
 		break;
@@ -145,20 +138,16 @@ const char *parse_everbs_hdrs(
 				 be32_to_cpu(eh->at.aeth) >> 24,
 				 parse_syndrome(be32_to_cpu(eh->at.aeth) >> 24),
 				 be32_to_cpu(eh->at.aeth) & HFI1_MSN_MASK,
-				 (unsigned long long)
-				 ib_u64_get(eh->at.atomic_ack_eth));
+				 ib_u64_get(&eh->at.atomic_ack_eth));
 		break;
 	/* atomiceth */
 	case OP(RC, COMPARE_SWAP):
 	case OP(RC, FETCH_ADD):
 		trace_seq_printf(p, ATOMICETH_PRN,
-				 (unsigned long long)ib_u64_get(
-				 eh->atomic_eth.vaddr),
+				 get_ib_ateth_vaddr(&eh->atomic_eth),
 				 eh->atomic_eth.rkey,
-				 (unsigned long long)ib_u64_get(
-				 (__be32 *)&eh->atomic_eth.swap_data),
-				 (unsigned long long)ib_u64_get(
-				 (__be32 *)&eh->atomic_eth.compare_data));
+				 get_ib_ateth_swap(&eh->atomic_eth),
+				 get_ib_ateth_compare(&eh->atomic_eth));
 		break;
 	/* deth */
 	case OP(UD, SEND_ONLY):
diff --git a/drivers/infiniband/hw/hfi1/trace_ctxts.h b/drivers/infiniband/hw/hfi1/trace_ctxts.h
index 31654bbac1cf..26ae789e47cf 100644
--- a/drivers/infiniband/hw/hfi1/trace_ctxts.h
+++ b/drivers/infiniband/hw/hfi1/trace_ctxts.h
@@ -67,9 +67,9 @@ TRACE_EVENT(hfi1_uctxtdata,
 			     __field(u64, hw_free)
 			     __field(void __iomem *, piobase)
 			     __field(u16, rcvhdrq_cnt)
-			     __field(u64, rcvhdrq_phys)
+			     __field(u64, rcvhdrq_dma)
 			     __field(u32, eager_cnt)
-			     __field(u64, rcvegr_phys)
+			     __field(u64, rcvegr_dma)
 			     ),
 	    TP_fast_assign(DD_DEV_ASSIGN(dd);
 			   __entry->ctxt = uctxt->ctxt;
@@ -77,10 +77,9 @@ TRACE_EVENT(hfi1_uctxtdata,
 			   __entry->hw_free = le64_to_cpu(*uctxt->sc->hw_free);
 			   __entry->piobase = uctxt->sc->base_addr;
 			   __entry->rcvhdrq_cnt = uctxt->rcvhdrq_cnt;
-			   __entry->rcvhdrq_phys = uctxt->rcvhdrq_phys;
+			   __entry->rcvhdrq_dma = uctxt->rcvhdrq_dma;
 			   __entry->eager_cnt = uctxt->egrbufs.alloced;
-			   __entry->rcvegr_phys =
-			   uctxt->egrbufs.rcvtids[0].phys;
+			   __entry->rcvegr_dma = uctxt->egrbufs.rcvtids[0].dma;
 			   ),
 	    TP_printk("[%s] ctxt %u " UCTXT_FMT,
 		      __get_str(dev),
@@ -89,9 +88,9 @@ TRACE_EVENT(hfi1_uctxtdata,
 		      __entry->hw_free,
 		      __entry->piobase,
 		      __entry->rcvhdrq_cnt,
-		      __entry->rcvhdrq_phys,
+		      __entry->rcvhdrq_dma,
 		      __entry->eager_cnt,
-		      __entry->rcvegr_phys
+		      __entry->rcvegr_dma
 		      )
 );
 
diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
index c3e41aed0034..382fcda3a5f6 100644
--- a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
+++ b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
@@ -55,7 +55,7 @@
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM hfi1_ibhdrs
 
-u8 ibhdr_exhdr_len(struct hfi1_ib_header *hdr);
+u8 ibhdr_exhdr_len(struct ib_header *hdr);
 const char *parse_everbs_hdrs(struct trace_seq *p, u8 opcode, void *ehdrs);
 
 #define __parse_ib_ehdrs(op, ehdrs) parse_everbs_hdrs(p, op, ehdrs)
@@ -74,7 +74,7 @@ __print_symbolic(lrh,                    \
 
 DECLARE_EVENT_CLASS(hfi1_ibhdr_template,
 		    TP_PROTO(struct hfi1_devdata *dd,
-			     struct hfi1_ib_header *hdr),
+			     struct ib_header *hdr),
 		    TP_ARGS(dd, hdr),
 		    TP_STRUCT__entry(
 			DD_DEV_ENTRY(dd)
@@ -102,7 +102,7 @@ DECLARE_EVENT_CLASS(hfi1_ibhdr_template,
 			__dynamic_array(u8, ehdrs, ibhdr_exhdr_len(hdr))
 			),
 		      TP_fast_assign(
-			struct hfi1_other_headers *ohdr;
+			struct ib_other_headers *ohdr;
 
 			DD_DEV_ASSIGN(dd);
 			/* LRH */
@@ -185,19 +185,19 @@ DECLARE_EVENT_CLASS(hfi1_ibhdr_template,
 );
 
 DEFINE_EVENT(hfi1_ibhdr_template, input_ibhdr,
-	     TP_PROTO(struct hfi1_devdata *dd, struct hfi1_ib_header *hdr),
+	     TP_PROTO(struct hfi1_devdata *dd, struct ib_header *hdr),
 	     TP_ARGS(dd, hdr));
 
 DEFINE_EVENT(hfi1_ibhdr_template, pio_output_ibhdr,
-	     TP_PROTO(struct hfi1_devdata *dd, struct hfi1_ib_header *hdr),
+	     TP_PROTO(struct hfi1_devdata *dd, struct ib_header *hdr),
 	     TP_ARGS(dd, hdr));
 
 DEFINE_EVENT(hfi1_ibhdr_template, ack_output_ibhdr,
-	     TP_PROTO(struct hfi1_devdata *dd, struct hfi1_ib_header *hdr),
+	     TP_PROTO(struct hfi1_devdata *dd, struct ib_header *hdr),
 	     TP_ARGS(dd, hdr));
 
 DEFINE_EVENT(hfi1_ibhdr_template, sdma_output_ibhdr,
-	     TP_PROTO(struct hfi1_devdata *dd, struct hfi1_ib_header *hdr),
+	     TP_PROTO(struct hfi1_devdata *dd, struct ib_header *hdr),
 	     TP_ARGS(dd, hdr));
 
 #endif /* __HFI1_TRACE_IBHDRS_H */
diff --git a/drivers/infiniband/hw/hfi1/trace_rx.h b/drivers/infiniband/hw/hfi1/trace_rx.h
index 9ba1f615ec95..11e02b228922 100644
--- a/drivers/infiniband/hw/hfi1/trace_rx.h
+++ b/drivers/infiniband/hw/hfi1/trace_rx.h
@@ -260,7 +260,7 @@ TRACE_EVENT(hfi1_mmu_invalidate,
 TRACE_EVENT(snoop_capture,
 	    TP_PROTO(struct hfi1_devdata *dd,
 		     int hdr_len,
-		     struct hfi1_ib_header *hdr,
+		     struct ib_header *hdr,
 		     int data_len,
 		     void *data),
 	    TP_ARGS(dd, hdr_len, hdr, data_len, data),
@@ -279,7 +279,7 @@ TRACE_EVENT(snoop_capture,
 			     __dynamic_array(u8, raw_pkt, data_len)
 			     ),
 	    TP_fast_assign(
-		struct hfi1_other_headers *ohdr;
+		struct ib_other_headers *ohdr;
 
 		__entry->lnh = (u8)(be16_to_cpu(hdr->lrh[0]) & 3);
 		if (__entry->lnh == HFI1_LRH_BTH)
diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c
index a726d96d185f..5e6d1bac4914 100644
--- a/drivers/infiniband/hw/hfi1/uc.c
+++ b/drivers/infiniband/hw/hfi1/uc.c
@@ -50,14 +50,7 @@
 #include "qp.h"
 
 /* cut down ridiculously long IB macro names */
-#define OP(x) IB_OPCODE_UC_##x
-
-/* only opcode mask for adaptive pio */
-const u32 uc_only_opcode =
-	BIT(OP(SEND_ONLY) & 0x1f) |
-	BIT(OP(SEND_ONLY_WITH_IMMEDIATE & 0x1f)) |
-	BIT(OP(RDMA_WRITE_ONLY & 0x1f)) |
-	BIT(OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE & 0x1f));
+#define OP(x) UC_OP(x)
 
 /**
  * hfi1_make_uc_req - construct a request packet (SEND, RDMA write)
@@ -70,7 +63,7 @@ const u32 uc_only_opcode =
 int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
-	struct hfi1_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct rvt_swqe *wqe;
 	u32 hwords = 5;
 	u32 bth0 = 0;
@@ -304,12 +297,12 @@ bail_no_tx:
 void hfi1_uc_rcv(struct hfi1_packet *packet)
 {
 	struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data;
-	struct hfi1_ib_header *hdr = packet->hdr;
+	struct ib_header *hdr = packet->hdr;
 	u32 rcv_flags = packet->rcv_flags;
 	void *data = packet->ebuf;
 	u32 tlen = packet->tlen;
 	struct rvt_qp *qp = packet->qp;
-	struct hfi1_other_headers *ohdr = packet->ohdr;
+	struct ib_other_headers *ohdr = packet->ohdr;
 	u32 bth0, opcode;
 	u32 hdrsize = packet->hlen;
 	u32 psn;
diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c
index f01e8e1d62d3..97ae24b6314c 100644
--- a/drivers/infiniband/hw/hfi1/ud.c
+++ b/drivers/infiniband/hw/hfi1/ud.c
@@ -271,7 +271,7 @@ drop:
 int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
-	struct hfi1_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct ib_ah_attr *ah_attr;
 	struct hfi1_pportdata *ppd;
 	struct hfi1_ibport *ibp;
@@ -510,8 +510,8 @@ void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
 	u32 bth0, plen, vl, hwords = 5;
 	u16 lrh0;
 	u8 sl = ibp->sc_to_sl[sc5];
-	struct hfi1_ib_header hdr;
-	struct hfi1_other_headers *ohdr;
+	struct ib_header hdr;
+	struct ib_other_headers *ohdr;
 	struct pio_buf *pbuf;
 	struct send_context *ctxt = qp_to_send_context(qp, sc5);
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
@@ -559,8 +559,8 @@ void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
 
 /*
  * opa_smp_check() - Do the regular pkey checking, and the additional
- * checks for SMPs specified in OPAv1 rev 0.90, section 9.10.26
- * ("SMA Packet Checks").
+ * checks for SMPs specified in OPAv1 rev 1.0, 9/19/2016 update, section
+ * 9.10.25 ("SMA Packet Checks").
  *
  * Note that:
  *   - Checks are done using the pkey directly from the packet's BTH,
@@ -603,23 +603,28 @@ static int opa_smp_check(struct hfi1_ibport *ibp, u16 pkey, u8 sc5,
 
 	/*
 	 * SMPs fall into one of four (disjoint) categories:
-	 * SMA request, SMA response, trap, or trap repress.
-	 * Our response depends, in part, on which type of
-	 * SMP we're processing.
+	 * SMA request, SMA response, SMA trap, or SMA trap repress.
+	 * Our response depends, in part, on which type of SMP we're
+	 * processing.
 	 *
-	 * If this is not an SMA request, or trap repress:
-	 *   - accept MAD if the port is running an SM
-	 *   - pkey == FULL_MGMT_P_KEY =>
-	 *       reply with unsupported method (i.e., just mark
-	 *       the smp's status field here, and let it be
-	 *       processed normally)
-	 *   - pkey != LIM_MGMT_P_KEY =>
-	 *       increment port recv constraint errors, drop MAD
-	 * If this is an SMA request or trap repress:
+	 * If this is an SMA response, skip the check here.
+	 *
+	 * If this is an SMA request or SMA trap repress:
 	 *   - pkey != FULL_MGMT_P_KEY =>
 	 *       increment port recv constraint errors, drop MAD
+	 *
+	 * Otherwise:
+	 *    - accept if the port is running an SM
+	 *    - drop MAD if it's an SMA trap
+	 *    - pkey == FULL_MGMT_P_KEY =>
+	 *        reply with unsupported method
+	 *    - pkey != FULL_MGMT_P_KEY =>
+	 *	  increment port recv constraint errors, drop MAD
 	 */
 	switch (smp->method) {
+	case IB_MGMT_METHOD_GET_RESP:
+	case IB_MGMT_METHOD_REPORT_RESP:
+		break;
 	case IB_MGMT_METHOD_GET:
 	case IB_MGMT_METHOD_SET:
 	case IB_MGMT_METHOD_REPORT:
@@ -629,23 +634,17 @@ static int opa_smp_check(struct hfi1_ibport *ibp, u16 pkey, u8 sc5,
 			return 1;
 		}
 		break;
-	case IB_MGMT_METHOD_SEND:
-	case IB_MGMT_METHOD_TRAP:
-	case IB_MGMT_METHOD_GET_RESP:
-	case IB_MGMT_METHOD_REPORT_RESP:
+	default:
 		if (ibp->rvp.port_cap_flags & IB_PORT_SM)
 			return 0;
+		if (smp->method == IB_MGMT_METHOD_TRAP)
+			return 1;
 		if (pkey == FULL_MGMT_P_KEY) {
 			smp->status |= IB_SMP_UNSUP_METHOD;
 			return 0;
 		}
-		if (pkey != LIM_MGMT_P_KEY) {
-			ingress_pkey_table_fail(ppd, pkey, slid);
-			return 1;
-		}
-		break;
-	default:
-		break;
+		ingress_pkey_table_fail(ppd, pkey, slid);
+		return 1;
 	}
 	return 0;
 }
@@ -665,7 +664,7 @@ static int opa_smp_check(struct hfi1_ibport *ibp, u16 pkey, u8 sc5,
  */
 void hfi1_ud_rcv(struct hfi1_packet *packet)
 {
-	struct hfi1_other_headers *ohdr = packet->ohdr;
+	struct ib_other_headers *ohdr = packet->ohdr;
 	int opcode;
 	u32 hdrsize = packet->hlen;
 	struct ib_wc wc;
@@ -675,13 +674,13 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 	int mgmt_pkey_idx = -1;
 	struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data;
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
-	struct hfi1_ib_header *hdr = packet->hdr;
+	struct ib_header *hdr = packet->hdr;
 	u32 rcv_flags = packet->rcv_flags;
 	void *data = packet->ebuf;
 	u32 tlen = packet->tlen;
 	struct rvt_qp *qp = packet->qp;
 	bool has_grh = rcv_flags & HFI1_HAS_GRH;
-	u8 sc5 = hdr2sc((struct hfi1_message_header *)hdr, packet->rhf);
+	u8 sc5 = hdr2sc(hdr, packet->rhf);
 	u32 bth1;
 	u8 sl_from_sc, sl;
 	u16 slid;
diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c
index 0ecf27903dc2..a761f804111e 100644
--- a/drivers/infiniband/hw/hfi1/user_sdma.c
+++ b/drivers/infiniband/hw/hfi1/user_sdma.c
@@ -114,6 +114,8 @@ MODULE_PARM_DESC(sdma_comp_size, "Size of User SDMA completion ring. Default: 12
 #define KDETH_HCRC_LOWER_SHIFT    24
 #define KDETH_HCRC_LOWER_MASK     0xff
 
+#define AHG_KDETH_INTR_SHIFT 12
+
 #define PBC2LRH(x) ((((x) & 0xfff) << 2) - 4)
 #define LRH2PBC(x) ((((x) >> 2) + 1) & 0xfff)
 
@@ -546,7 +548,7 @@ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec,
 	u8 opcode, sc, vl;
 	int req_queued = 0;
 	u16 dlid;
-	u8 selector;
+	u32 selector;
 
 	if (iovec[idx].iov_len < sizeof(info) + sizeof(req->hdr)) {
 		hfi1_cdbg(
@@ -751,12 +753,9 @@ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec,
 
 	dlid = be16_to_cpu(req->hdr.lrh[1]);
 	selector = dlid_to_selector(dlid);
+	selector += uctxt->ctxt + fd->subctxt;
+	req->sde = sdma_select_user_engine(dd, selector, vl);
 
-	/* Have to select the engine */
-	req->sde = sdma_select_engine_vl(dd,
-					 (u32)(uctxt->ctxt + fd->subctxt +
-					       selector),
-					 vl);
 	if (!req->sde || !sdma_running(req->sde)) {
 		ret = -ECOMM;
 		goto free_req;
@@ -892,7 +891,7 @@ static inline u32 get_lrh_len(struct hfi1_pkt_header hdr, u32 len)
 
 static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 {
-	int ret = 0;
+	int ret = 0, count;
 	unsigned npkts = 0;
 	struct user_sdma_txreq *tx = NULL;
 	struct hfi1_user_sdma_pkt_q *pq = NULL;
@@ -1088,23 +1087,18 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 		npkts++;
 	}
 dosend:
-	ret = sdma_send_txlist(req->sde, &pq->busy, &req->txps);
-	if (list_empty(&req->txps)) {
-		req->seqsubmitted = req->seqnum;
-		if (req->seqnum == req->info.npkts) {
-			set_bit(SDMA_REQ_SEND_DONE, &req->flags);
-			/*
-			 * The txreq has already been submitted to the HW queue
-			 * so we can free the AHG entry now. Corruption will not
-			 * happen due to the sequential manner in which
-			 * descriptors are processed.
-			 */
-			if (test_bit(SDMA_REQ_HAVE_AHG, &req->flags))
-				sdma_ahg_free(req->sde, req->ahg_idx);
-		}
-	} else if (ret > 0) {
-		req->seqsubmitted += ret;
-		ret = 0;
+	ret = sdma_send_txlist(req->sde, &pq->busy, &req->txps, &count);
+	req->seqsubmitted += count;
+	if (req->seqsubmitted == req->info.npkts) {
+		set_bit(SDMA_REQ_SEND_DONE, &req->flags);
+		/*
+		 * The txreq has already been submitted to the HW queue
+		 * so we can free the AHG entry now. Corruption will not
+		 * happen due to the sequential manner in which
+		 * descriptors are processed.
+		 */
+		if (test_bit(SDMA_REQ_HAVE_AHG, &req->flags))
+			sdma_ahg_free(req->sde, req->ahg_idx);
 	}
 	return ret;
 
@@ -1480,7 +1474,8 @@ static int set_txreq_header_ahg(struct user_sdma_request *req,
 		/* Clear KDETH.SH on last packet */
 		if (unlikely(tx->flags & TXREQ_FLAGS_REQ_LAST_PKT)) {
 			val |= cpu_to_le16(KDETH_GET(hdr->kdeth.ver_tid_offset,
-								INTR) >> 16);
+						     INTR) <<
+					   AHG_KDETH_INTR_SHIFT);
 			val &= cpu_to_le16(~(1U << 13));
 			AHG_HEADER_SET(req->ahg, diff, 7, 16, 14, val);
 		} else {
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 2b359540901d..4b7a16ceb362 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -76,7 +76,7 @@ static unsigned int hfi1_max_ahs = 0xFFFF;
 module_param_named(max_ahs, hfi1_max_ahs, uint, S_IRUGO);
 MODULE_PARM_DESC(max_ahs, "Maximum number of address handles to support");
 
-unsigned int hfi1_max_cqes = 0x2FFFF;
+unsigned int hfi1_max_cqes = 0x2FFFFF;
 module_param_named(max_cqes, hfi1_max_cqes, uint, S_IRUGO);
 MODULE_PARM_DESC(max_cqes,
 		 "Maximum number of completion queue entries to support");
@@ -89,7 +89,7 @@ unsigned int hfi1_max_qp_wrs = 0x3FFF;
 module_param_named(max_qp_wrs, hfi1_max_qp_wrs, uint, S_IRUGO);
 MODULE_PARM_DESC(max_qp_wrs, "Maximum number of QP WRs to support");
 
-unsigned int hfi1_max_qps = 16384;
+unsigned int hfi1_max_qps = 32768;
 module_param_named(max_qps, hfi1_max_qps, uint, S_IRUGO);
 MODULE_PARM_DESC(max_qps, "Maximum number of QPs to support");
 
@@ -335,7 +335,7 @@ const u8 hdr_len_by_opcode[256] = {
 	[IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST]        = 12 + 8 + 4,
 	[IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY]        = 12 + 8 + 4,
 	[IB_OPCODE_RC_ACKNOWLEDGE]                    = 12 + 8 + 4,
-	[IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE]             = 12 + 8 + 4,
+	[IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE]             = 12 + 8 + 4 + 8,
 	[IB_OPCODE_RC_COMPARE_SWAP]                   = 12 + 8 + 28,
 	[IB_OPCODE_RC_FETCH_ADD]                      = 12 + 8 + 28,
 	[IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE]      = 12 + 8 + 4,
@@ -403,6 +403,28 @@ static const opcode_handler opcode_handler_tbl[256] = {
 	[IB_OPCODE_CNP]				      = &hfi1_cnp_rcv
 };
 
+#define OPMASK 0x1f
+
+static const u32 pio_opmask[BIT(3)] = {
+	/* RC */
+	[IB_OPCODE_RC >> 5] =
+		BIT(RC_OP(SEND_ONLY) & OPMASK) |
+		BIT(RC_OP(SEND_ONLY_WITH_IMMEDIATE) & OPMASK) |
+		BIT(RC_OP(RDMA_WRITE_ONLY) & OPMASK) |
+		BIT(RC_OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE) & OPMASK) |
+		BIT(RC_OP(RDMA_READ_REQUEST) & OPMASK) |
+		BIT(RC_OP(ACKNOWLEDGE) & OPMASK) |
+		BIT(RC_OP(ATOMIC_ACKNOWLEDGE) & OPMASK) |
+		BIT(RC_OP(COMPARE_SWAP) & OPMASK) |
+		BIT(RC_OP(FETCH_ADD) & OPMASK),
+	/* UC */
+	[IB_OPCODE_UC >> 5] =
+		BIT(UC_OP(SEND_ONLY) & OPMASK) |
+		BIT(UC_OP(SEND_ONLY_WITH_IMMEDIATE) & OPMASK) |
+		BIT(UC_OP(RDMA_WRITE_ONLY) & OPMASK) |
+		BIT(UC_OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE) & OPMASK),
+};
+
 /*
  * System image GUID.
  */
@@ -567,7 +589,7 @@ static inline opcode_handler qp_ok(int opcode, struct hfi1_packet *packet)
 void hfi1_ib_rcv(struct hfi1_packet *packet)
 {
 	struct hfi1_ctxtdata *rcd = packet->rcd;
-	struct hfi1_ib_header *hdr = packet->hdr;
+	struct ib_header *hdr = packet->hdr;
 	u32 tlen = packet->tlen;
 	struct hfi1_pportdata *ppd = rcd->ppd;
 	struct hfi1_ibport *ibp = &ppd->ibport_data;
@@ -719,7 +741,7 @@ static void verbs_sdma_complete(
 	if (tx->wqe) {
 		hfi1_send_complete(qp, tx->wqe, IB_WC_SUCCESS);
 	} else if (qp->ibqp.qp_type == IB_QPT_RC) {
-		struct hfi1_ib_header *hdr;
+		struct ib_header *hdr;
 
 		hdr = &tx->phdr.hdr;
 		hfi1_rc_send_complete(qp, hdr);
@@ -748,7 +770,7 @@ static int wait_kmem(struct hfi1_ibdev *dev,
 			qp->s_flags |= RVT_S_WAIT_KMEM;
 			list_add_tail(&priv->s_iowait.list, &dev->memwait);
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_KMEM);
-			atomic_inc(&qp->refcount);
+			rvt_get_qp(qp);
 		}
 		write_sequnlock(&dev->iowait_lock);
 		qp->s_flags &= ~RVT_S_BUSY;
@@ -959,7 +981,7 @@ static int pio_wait(struct rvt_qp *qp,
 			was_empty = list_empty(&sc->piowait);
 			list_add_tail(&priv->s_iowait.list, &sc->piowait);
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_PIO);
-			atomic_inc(&qp->refcount);
+			rvt_get_qp(qp);
 			/* counting: only call wantpiobuf_intr if first user */
 			if (was_empty)
 				hfi1_sc_wantpiobuf_intr(sc, 1);
@@ -1200,7 +1222,7 @@ static inline send_routine get_send_routine(struct rvt_qp *qp,
 {
 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
 	struct hfi1_qp_priv *priv = qp->priv;
-	struct hfi1_ib_header *h = &tx->phdr.hdr;
+	struct ib_header *h = &tx->phdr.hdr;
 
 	if (unlikely(!(dd->flags & HFI1_HAS_SEND_DMA)))
 		return dd->process_pio_send;
@@ -1210,22 +1232,18 @@ static inline send_routine get_send_routine(struct rvt_qp *qp,
 	case IB_QPT_GSI:
 	case IB_QPT_UD:
 		break;
-	case IB_QPT_RC:
-		if (piothreshold &&
-		    qp->s_cur_size <= min(piothreshold, qp->pmtu) &&
-		    (BIT(get_opcode(h) & 0x1f) & rc_only_opcode) &&
-		    iowait_sdma_pending(&priv->s_iowait) == 0 &&
-		    !sdma_txreq_built(&tx->txreq))
-			return dd->process_pio_send;
-		break;
 	case IB_QPT_UC:
+	case IB_QPT_RC: {
+		u8 op = get_opcode(h);
+
 		if (piothreshold &&
 		    qp->s_cur_size <= min(piothreshold, qp->pmtu) &&
-		    (BIT(get_opcode(h) & 0x1f) & uc_only_opcode) &&
+		    (BIT(op & OPMASK) & pio_opmask[op >> 5]) &&
 		    iowait_sdma_pending(&priv->s_iowait) == 0 &&
 		    !sdma_txreq_built(&tx->txreq))
 			return dd->process_pio_send;
 		break;
+	}
 	default:
 		break;
 	}
@@ -1244,8 +1262,8 @@ int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 {
 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
 	struct hfi1_qp_priv *priv = qp->priv;
-	struct hfi1_other_headers *ohdr;
-	struct hfi1_ib_header *hdr;
+	struct ib_other_headers *ohdr;
+	struct ib_header *hdr;
 	send_routine sr;
 	int ret;
 	u8 lnh;
@@ -1423,7 +1441,8 @@ static int modify_device(struct ib_device *device,
 	}
 
 	if (device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC) {
-		memcpy(device->node_desc, device_modify->node_desc, 64);
+		memcpy(device->node_desc, device_modify->node_desc,
+		       IB_DEVICE_NODE_DESC_MAX);
 		for (i = 0; i < dd->num_pports; i++) {
 			struct hfi1_ibport *ibp = &dd->pport[i].ibport_data;
 
@@ -1754,7 +1773,7 @@ void hfi1_cnp_rcv(struct hfi1_packet *packet)
 {
 	struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data;
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
-	struct hfi1_ib_header *hdr = packet->hdr;
+	struct ib_header *hdr = packet->hdr;
 	struct rvt_qp *qp = packet->qp;
 	u32 lqpn, rqpn = 0;
 	u16 rlid = 0;
@@ -1781,7 +1800,7 @@ void hfi1_cnp_rcv(struct hfi1_packet *packet)
 		return;
 	}
 
-	sc5 = hdr2sc((struct hfi1_message_header *)hdr, packet->rhf);
+	sc5 = hdr2sc(hdr, packet->rhf);
 	sl = ibp->sc_to_sl[sc5];
 	lqpn = qp->ibqp.qp_num;
 
diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h
index d1b101c54828..1c3815d89eb7 100644
--- a/drivers/infiniband/hw/hfi1/verbs.h
+++ b/drivers/infiniband/hw/hfi1/verbs.h
@@ -60,6 +60,7 @@
 #include <rdma/ib_pack.h>
 #include <rdma/ib_user_verbs.h>
 #include <rdma/ib_mad.h>
+#include <rdma/ib_hdrs.h>
 #include <rdma/rdma_vt.h>
 #include <rdma/rdmavt_qp.h>
 #include <rdma/rdmavt_cq.h>
@@ -80,16 +81,6 @@ struct hfi1_packet;
  */
 #define HFI1_UVERBS_ABI_VERSION       2
 
-#define IB_SEQ_NAK	(3 << 29)
-
-/* AETH NAK opcode values */
-#define IB_RNR_NAK                      0x20
-#define IB_NAK_PSN_ERROR                0x60
-#define IB_NAK_INVALID_REQUEST          0x61
-#define IB_NAK_REMOTE_ACCESS_ERROR      0x62
-#define IB_NAK_REMOTE_OPERATIONAL_ERROR 0x63
-#define IB_NAK_INVALID_RD_REQUEST       0x64
-
 /* IB Performance Manager status values */
 #define IB_PMA_SAMPLE_STATUS_DONE       0x00
 #define IB_PMA_SAMPLE_STATUS_STARTED    0x01
@@ -104,80 +95,16 @@ struct hfi1_packet;
 
 #define HFI1_VENDOR_IPG		cpu_to_be16(0xFFA0)
 
-#define IB_BTH_REQ_ACK		BIT(31)
-#define IB_BTH_SOLICITED	BIT(23)
-#define IB_BTH_MIG_REQ		BIT(22)
-
-#define IB_GRH_VERSION		6
-#define IB_GRH_VERSION_MASK	0xF
-#define IB_GRH_VERSION_SHIFT	28
-#define IB_GRH_TCLASS_MASK	0xFF
-#define IB_GRH_TCLASS_SHIFT	20
-#define IB_GRH_FLOW_MASK	0xFFFFF
-#define IB_GRH_FLOW_SHIFT	0
-#define IB_GRH_NEXT_HDR		0x1B
-
 #define IB_DEFAULT_GID_PREFIX	cpu_to_be64(0xfe80000000000000ULL)
 
+#define RC_OP(x) IB_OPCODE_RC_##x
+#define UC_OP(x) IB_OPCODE_UC_##x
+
 /* flags passed by hfi1_ib_rcv() */
 enum {
 	HFI1_HAS_GRH = (1 << 0),
 };
 
-struct ib_reth {
-	__be64 vaddr;
-	__be32 rkey;
-	__be32 length;
-} __packed;
-
-struct ib_atomic_eth {
-	__be32 vaddr[2];        /* unaligned so access as 2 32-bit words */
-	__be32 rkey;
-	__be64 swap_data;
-	__be64 compare_data;
-} __packed;
-
-union ib_ehdrs {
-	struct {
-		__be32 deth[2];
-		__be32 imm_data;
-	} ud;
-	struct {
-		struct ib_reth reth;
-		__be32 imm_data;
-	} rc;
-	struct {
-		__be32 aeth;
-		__be32 atomic_ack_eth[2];
-	} at;
-	__be32 imm_data;
-	__be32 aeth;
-	__be32 ieth;
-	struct ib_atomic_eth atomic_eth;
-}  __packed;
-
-struct hfi1_other_headers {
-	__be32 bth[3];
-	union ib_ehdrs u;
-} __packed;
-
-/*
- * Note that UD packets with a GRH header are 8+40+12+8 = 68 bytes
- * long (72 w/ imm_data).  Only the first 56 bytes of the IB header
- * will be in the eager header buffer.  The remaining 12 or 16 bytes
- * are in the data buffer.
- */
-struct hfi1_ib_header {
-	__be16 lrh[4];
-	union {
-		struct {
-			struct ib_grh grh;
-			struct hfi1_other_headers oth;
-		} l;
-		struct hfi1_other_headers oth;
-	} u;
-} __packed;
-
 struct hfi1_ahg_info {
 	u32 ahgdesc[2];
 	u16 tx_flags;
@@ -187,7 +114,7 @@ struct hfi1_ahg_info {
 
 struct hfi1_sdma_header {
 	__le64 pbc;
-	struct hfi1_ib_header hdr;
+	struct ib_header hdr;
 } __packed;
 
 /*
@@ -386,7 +313,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet);
 
 void hfi1_rc_hdrerr(
 	struct hfi1_ctxtdata *rcd,
-	struct hfi1_ib_header *hdr,
+	struct ib_header *hdr,
 	u32 rcv_flags,
 	struct rvt_qp *qp);
 
@@ -400,7 +327,7 @@ void hfi1_rc_timeout(unsigned long arg);
 void hfi1_del_timers_sync(struct rvt_qp *qp);
 void hfi1_stop_rc_timers(struct rvt_qp *qp);
 
-void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_ib_header *hdr);
+void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr);
 
 void hfi1_rc_error(struct rvt_qp *qp, enum ib_wc_status err);
 
@@ -423,7 +350,7 @@ int hfi1_check_send_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe);
 extern const u32 rc_only_opcode;
 extern const u32 uc_only_opcode;
 
-static inline u8 get_opcode(struct hfi1_ib_header *h)
+static inline u8 get_opcode(struct ib_header *h)
 {
 	u16 lnh = be16_to_cpu(h->lrh[0]) & 3;
 
@@ -433,13 +360,13 @@ static inline u8 get_opcode(struct hfi1_ib_header *h)
 		return be32_to_cpu(h->u.l.oth.bth[0]) >> 24;
 }
 
-int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_ib_header *hdr,
+int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct ib_header *hdr,
 		       int has_grh, struct rvt_qp *qp, u32 bth0);
 
 u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,
 		  struct ib_global_route *grh, u32 hwords, u32 nwords);
 
-void hfi1_make_ruc_header(struct rvt_qp *qp, struct hfi1_other_headers *ohdr,
+void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
 			  u32 bth0, u32 bth2, int middle,
 			  struct hfi1_pkt_state *ps);
 
diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.c b/drivers/infiniband/hw/hfi1/verbs_txreq.c
index d8fb056526f8..094ab829ec42 100644
--- a/drivers/infiniband/hw/hfi1/verbs_txreq.c
+++ b/drivers/infiniband/hw/hfi1/verbs_txreq.c
@@ -109,7 +109,7 @@ struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev,
 			qp->s_flags |= RVT_S_WAIT_TX;
 			list_add_tail(&priv->s_iowait.list, &dev->txwait);
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_TX);
-			atomic_inc(&qp->refcount);
+			rvt_get_qp(qp);
 		}
 		qp->s_flags &= ~RVT_S_BUSY;
 	}
diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig
new file mode 100644
index 000000000000..e1a6e055cd60
--- /dev/null
+++ b/drivers/infiniband/hw/hns/Kconfig
@@ -0,0 +1,10 @@
+config INFINIBAND_HNS
+	tristate "HNS RoCE Driver"
+	depends on NET_VENDOR_HISILICON
+	depends on ARM64 && HNS && HNS_DSAF && HNS_ENET
+	---help---
+	  This is a RoCE/RDMA driver for the Hisilicon RoCE engine. The engine
+	  is used in Hisilicon Hi1610 and more further ICT SoC.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called hns-roce.
diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile
new file mode 100644
index 000000000000..7e8ebd24dcae
--- /dev/null
+++ b/drivers/infiniband/hw/hns/Makefile
@@ -0,0 +1,8 @@
+#
+# Makefile for the Hisilicon RoCE drivers.
+#
+
+obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o
+hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_eq.o hns_roce_pd.o \
+	hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
+	hns_roce_cq.o hns_roce_alloc.o hns_roce_hw_v1.o
diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c
new file mode 100644
index 000000000000..24f79ee39fdf
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_ah.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/platform_device.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+#include "hns_roce_device.h"
+
+#define HNS_ROCE_PORT_NUM_SHIFT		24
+#define HNS_ROCE_VLAN_SL_BIT_MASK	7
+#define HNS_ROCE_VLAN_SL_SHIFT		13
+
+struct ib_ah *hns_roce_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *ah_attr)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibpd->device);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct ib_gid_attr gid_attr;
+	struct hns_roce_ah *ah;
+	u16 vlan_tag = 0xffff;
+	struct in6_addr in6;
+	union ib_gid sgid;
+	int ret;
+
+	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
+	if (!ah)
+		return ERR_PTR(-ENOMEM);
+
+	/* Get mac address */
+	memcpy(&in6, ah_attr->grh.dgid.raw, sizeof(ah_attr->grh.dgid.raw));
+	if (rdma_is_multicast_addr(&in6))
+		rdma_get_mcast_mac(&in6, ah->av.mac);
+	else
+		memcpy(ah->av.mac, ah_attr->dmac, sizeof(ah_attr->dmac));
+
+	/* Get source gid */
+	ret = ib_get_cached_gid(ibpd->device, ah_attr->port_num,
+				ah_attr->grh.sgid_index, &sgid, &gid_attr);
+	if (ret) {
+		dev_err(dev, "get sgid failed! ret = %d\n", ret);
+		kfree(ah);
+		return ERR_PTR(ret);
+	}
+
+	if (gid_attr.ndev) {
+		if (is_vlan_dev(gid_attr.ndev))
+			vlan_tag = vlan_dev_vlan_id(gid_attr.ndev);
+		dev_put(gid_attr.ndev);
+	}
+
+	if (vlan_tag < 0x1000)
+		vlan_tag |= (ah_attr->sl & HNS_ROCE_VLAN_SL_BIT_MASK) <<
+			     HNS_ROCE_VLAN_SL_SHIFT;
+
+	ah->av.port_pd = cpu_to_be32(to_hr_pd(ibpd)->pdn | (ah_attr->port_num <<
+				     HNS_ROCE_PORT_NUM_SHIFT));
+	ah->av.gid_index = ah_attr->grh.sgid_index;
+	ah->av.vlan = cpu_to_le16(vlan_tag);
+	dev_dbg(dev, "gid_index = 0x%x,vlan = 0x%x\n", ah->av.gid_index,
+		ah->av.vlan);
+
+	if (ah_attr->static_rate)
+		ah->av.stat_rate = IB_RATE_10_GBPS;
+
+	memcpy(ah->av.dgid, ah_attr->grh.dgid.raw, HNS_ROCE_GID_SIZE);
+	ah->av.sl_tclass_flowlabel = cpu_to_le32(ah_attr->sl <<
+						 HNS_ROCE_SL_SHIFT);
+
+	return &ah->ibah;
+}
+
+int hns_roce_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
+{
+	struct hns_roce_ah *ah = to_hr_ah(ibah);
+
+	memset(ah_attr, 0, sizeof(*ah_attr));
+
+	ah_attr->sl = le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
+				  HNS_ROCE_SL_SHIFT;
+	ah_attr->port_num = le32_to_cpu(ah->av.port_pd) >>
+					HNS_ROCE_PORT_NUM_SHIFT;
+	ah_attr->static_rate = ah->av.stat_rate;
+	ah_attr->ah_flags = IB_AH_GRH;
+	ah_attr->grh.traffic_class = le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
+						 HNS_ROCE_TCLASS_SHIFT;
+	ah_attr->grh.flow_label = le32_to_cpu(ah->av.sl_tclass_flowlabel) &
+					      HNS_ROCE_FLOW_LABLE_MASK;
+	ah_attr->grh.hop_limit = ah->av.hop_limit;
+	ah_attr->grh.sgid_index = ah->av.gid_index;
+	memcpy(ah_attr->grh.dgid.raw, ah->av.dgid, HNS_ROCE_GID_SIZE);
+
+	return 0;
+}
+
+int hns_roce_destroy_ah(struct ib_ah *ah)
+{
+	kfree(to_hr_ah(ah));
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c
new file mode 100644
index 000000000000..863a17a2de40
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c
@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/platform_device.h>
+#include "hns_roce_device.h"
+
+int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj)
+{
+	int ret = 0;
+
+	spin_lock(&bitmap->lock);
+	*obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);
+	if (*obj >= bitmap->max) {
+		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
+			       & bitmap->mask;
+		*obj = find_first_zero_bit(bitmap->table, bitmap->max);
+	}
+
+	if (*obj < bitmap->max) {
+		set_bit(*obj, bitmap->table);
+		bitmap->last = (*obj + 1);
+		if (bitmap->last == bitmap->max)
+			bitmap->last = 0;
+		*obj |= bitmap->top;
+	} else {
+		ret = -1;
+	}
+
+	spin_unlock(&bitmap->lock);
+
+	return ret;
+}
+
+void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj)
+{
+	hns_roce_bitmap_free_range(bitmap, obj, 1);
+}
+
+int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,
+				int align, unsigned long *obj)
+{
+	int ret = 0;
+	int i;
+
+	if (likely(cnt == 1 && align == 1))
+		return hns_roce_bitmap_alloc(bitmap, obj);
+
+	spin_lock(&bitmap->lock);
+
+	*obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
+					  bitmap->last, cnt, align - 1);
+	if (*obj >= bitmap->max) {
+		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
+			       & bitmap->mask;
+		*obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, 0,
+						  cnt, align - 1);
+	}
+
+	if (*obj < bitmap->max) {
+		for (i = 0; i < cnt; i++)
+			set_bit(*obj + i, bitmap->table);
+
+		if (*obj == bitmap->last) {
+			bitmap->last = (*obj + cnt);
+			if (bitmap->last >= bitmap->max)
+				bitmap->last = 0;
+		}
+		*obj |= bitmap->top;
+	} else {
+		ret = -1;
+	}
+
+	spin_unlock(&bitmap->lock);
+
+	return ret;
+}
+
+void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,
+				unsigned long obj, int cnt)
+{
+	int i;
+
+	obj &= bitmap->max + bitmap->reserved_top - 1;
+
+	spin_lock(&bitmap->lock);
+	for (i = 0; i < cnt; i++)
+		clear_bit(obj + i, bitmap->table);
+
+	bitmap->last = min(bitmap->last, obj);
+	bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
+		       & bitmap->mask;
+	spin_unlock(&bitmap->lock);
+}
+
+int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask,
+			 u32 reserved_bot, u32 reserved_top)
+{
+	u32 i;
+
+	if (num != roundup_pow_of_two(num))
+		return -EINVAL;
+
+	bitmap->last = 0;
+	bitmap->top = 0;
+	bitmap->max = num - reserved_top;
+	bitmap->mask = mask;
+	bitmap->reserved_top = reserved_top;
+	spin_lock_init(&bitmap->lock);
+	bitmap->table = kcalloc(BITS_TO_LONGS(bitmap->max), sizeof(long),
+				GFP_KERNEL);
+	if (!bitmap->table)
+		return -ENOMEM;
+
+	for (i = 0; i < reserved_bot; ++i)
+		set_bit(i, bitmap->table);
+
+	return 0;
+}
+
+void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap)
+{
+	kfree(bitmap->table);
+}
+
+void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,
+		       struct hns_roce_buf *buf)
+{
+	int i;
+	struct device *dev = &hr_dev->pdev->dev;
+	u32 bits_per_long = BITS_PER_LONG;
+
+	if (buf->nbufs == 1) {
+		dma_free_coherent(dev, size, buf->direct.buf, buf->direct.map);
+	} else {
+		if (bits_per_long == 64)
+			vunmap(buf->direct.buf);
+
+		for (i = 0; i < buf->nbufs; ++i)
+			if (buf->page_list[i].buf)
+				dma_free_coherent(&hr_dev->pdev->dev, PAGE_SIZE,
+						  buf->page_list[i].buf,
+						  buf->page_list[i].map);
+		kfree(buf->page_list);
+	}
+}
+
+int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
+		       struct hns_roce_buf *buf)
+{
+	int i = 0;
+	dma_addr_t t;
+	struct page **pages;
+	struct device *dev = &hr_dev->pdev->dev;
+	u32 bits_per_long = BITS_PER_LONG;
+
+	/* SQ/RQ buf lease than one page, SQ + RQ = 8K */
+	if (size <= max_direct) {
+		buf->nbufs = 1;
+		/* Npages calculated by page_size */
+		buf->npages = 1 << get_order(size);
+		buf->page_shift = PAGE_SHIFT;
+		/* MTT PA must be recorded in 4k alignment, t is 4k aligned */
+		buf->direct.buf = dma_alloc_coherent(dev, size, &t, GFP_KERNEL);
+		if (!buf->direct.buf)
+			return -ENOMEM;
+
+		buf->direct.map = t;
+
+		while (t & ((1 << buf->page_shift) - 1)) {
+			--buf->page_shift;
+			buf->npages *= 2;
+		}
+
+		memset(buf->direct.buf, 0, size);
+	} else {
+		buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE;
+		buf->npages = buf->nbufs;
+		buf->page_shift = PAGE_SHIFT;
+		buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list),
+					 GFP_KERNEL);
+
+		if (!buf->page_list)
+			return -ENOMEM;
+
+		for (i = 0; i < buf->nbufs; ++i) {
+			buf->page_list[i].buf = dma_alloc_coherent(dev,
+								  PAGE_SIZE, &t,
+								  GFP_KERNEL);
+
+			if (!buf->page_list[i].buf)
+				goto err_free;
+
+			buf->page_list[i].map = t;
+			memset(buf->page_list[i].buf, 0, PAGE_SIZE);
+		}
+		if (bits_per_long == 64) {
+			pages = kmalloc_array(buf->nbufs, sizeof(*pages),
+					      GFP_KERNEL);
+			if (!pages)
+				goto err_free;
+
+			for (i = 0; i < buf->nbufs; ++i)
+				pages[i] = virt_to_page(buf->page_list[i].buf);
+
+			buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP,
+					       PAGE_KERNEL);
+			kfree(pages);
+			if (!buf->direct.buf)
+				goto err_free;
+		}
+	}
+
+	return 0;
+
+err_free:
+	hns_roce_buf_free(hr_dev, size, buf);
+	return -ENOMEM;
+}
+
+void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev)
+{
+	hns_roce_cleanup_qp_table(hr_dev);
+	hns_roce_cleanup_cq_table(hr_dev);
+	hns_roce_cleanup_mr_table(hr_dev);
+	hns_roce_cleanup_pd_table(hr_dev);
+	hns_roce_cleanup_uar_table(hr_dev);
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c
new file mode 100644
index 000000000000..2a0b6c05da5f
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c
@@ -0,0 +1,368 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/dmapool.h>
+#include <linux/platform_device.h>
+#include "hns_roce_common.h"
+#include "hns_roce_device.h"
+#include "hns_roce_cmd.h"
+
+#define CMD_POLL_TOKEN		0xffff
+#define CMD_MAX_NUM		32
+#define STATUS_MASK		0xff
+#define CMD_TOKEN_MASK		0x1f
+#define GO_BIT_TIMEOUT_MSECS	10000
+
+enum {
+	HCR_TOKEN_OFFSET	= 0x14,
+	HCR_STATUS_OFFSET	= 0x18,
+	HCR_GO_BIT		= 15,
+};
+
+static int cmd_pending(struct hns_roce_dev *hr_dev)
+{
+	u32 status = readl(hr_dev->cmd.hcr + HCR_TOKEN_OFFSET);
+
+	return (!!(status & (1 << HCR_GO_BIT)));
+}
+
+/* this function should be serialized with "hcr_mutex" */
+static int __hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev,
+				       u64 in_param, u64 out_param,
+				       u32 in_modifier, u8 op_modifier, u16 op,
+				       u16 token, int event)
+{
+	struct hns_roce_cmdq *cmd = &hr_dev->cmd;
+	struct device *dev = &hr_dev->pdev->dev;
+	u32 __iomem *hcr = (u32 *)cmd->hcr;
+	int ret = -EAGAIN;
+	unsigned long end;
+	u32 val = 0;
+
+	end = msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS) + jiffies;
+	while (cmd_pending(hr_dev)) {
+		if (time_after(jiffies, end)) {
+			dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
+				(int)end);
+			goto out;
+		}
+		cond_resched();
+	}
+
+	roce_set_field(val, ROCEE_MB6_ROCEE_MB_CMD_M, ROCEE_MB6_ROCEE_MB_CMD_S,
+		       op);
+	roce_set_field(val, ROCEE_MB6_ROCEE_MB_CMD_MDF_M,
+		       ROCEE_MB6_ROCEE_MB_CMD_MDF_S, op_modifier);
+	roce_set_bit(val, ROCEE_MB6_ROCEE_MB_EVENT_S, event);
+	roce_set_bit(val, ROCEE_MB6_ROCEE_MB_HW_RUN_S, 1);
+	roce_set_field(val, ROCEE_MB6_ROCEE_MB_TOKEN_M,
+		       ROCEE_MB6_ROCEE_MB_TOKEN_S, token);
+
+	__raw_writeq(cpu_to_le64(in_param), hcr + 0);
+	__raw_writeq(cpu_to_le64(out_param), hcr + 2);
+	__raw_writel(cpu_to_le32(in_modifier), hcr + 4);
+	/* Memory barrier */
+	wmb();
+
+	__raw_writel(cpu_to_le32(val), hcr + 5);
+
+	mmiowb();
+	ret = 0;
+
+out:
+	return ret;
+}
+
+static int hns_roce_cmd_mbox_post_hw(struct hns_roce_dev *hr_dev, u64 in_param,
+				     u64 out_param, u32 in_modifier,
+				     u8 op_modifier, u16 op, u16 token,
+				     int event)
+{
+	struct hns_roce_cmdq *cmd = &hr_dev->cmd;
+	int ret = -EAGAIN;
+
+	mutex_lock(&cmd->hcr_mutex);
+	ret = __hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param,
+					  in_modifier, op_modifier, op, token,
+					  event);
+	mutex_unlock(&cmd->hcr_mutex);
+
+	return ret;
+}
+
+/* this should be called with "poll_sem" */
+static int __hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param,
+				    u64 out_param, unsigned long in_modifier,
+				    u8 op_modifier, u16 op,
+				    unsigned long timeout)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	u8 __iomem *hcr = hr_dev->cmd.hcr;
+	unsigned long end = 0;
+	u32 status = 0;
+	int ret;
+
+	ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param,
+					in_modifier, op_modifier, op,
+					CMD_POLL_TOKEN, 0);
+	if (ret) {
+		dev_err(dev, "[cmd_poll]hns_roce_cmd_mbox_post_hw failed\n");
+		goto out;
+	}
+
+	end = msecs_to_jiffies(timeout) + jiffies;
+	while (cmd_pending(hr_dev) && time_before(jiffies, end))
+		cond_resched();
+
+	if (cmd_pending(hr_dev)) {
+		dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	status = le32_to_cpu((__force __be32)
+			      __raw_readl(hcr + HCR_STATUS_OFFSET));
+	if ((status & STATUS_MASK) != 0x1) {
+		dev_err(dev, "mailbox status 0x%x!\n", status);
+		ret = -EBUSY;
+		goto out;
+	}
+
+out:
+	return ret;
+}
+
+static int hns_roce_cmd_mbox_poll(struct hns_roce_dev *hr_dev, u64 in_param,
+				  u64 out_param, unsigned long in_modifier,
+				  u8 op_modifier, u16 op, unsigned long timeout)
+{
+	int ret;
+
+	down(&hr_dev->cmd.poll_sem);
+	ret = __hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param, in_modifier,
+				       op_modifier, op, timeout);
+	up(&hr_dev->cmd.poll_sem);
+
+	return ret;
+}
+
+void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,
+			u64 out_param)
+{
+	struct hns_roce_cmd_context
+		*context = &hr_dev->cmd.context[token & hr_dev->cmd.token_mask];
+
+	if (token != context->token)
+		return;
+
+	context->result = (status == HNS_ROCE_CMD_SUCCESS) ? 0 : (-EIO);
+	context->out_param = out_param;
+	complete(&context->done);
+}
+
+/* this should be called with "use_events" */
+static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
+				    u64 out_param, unsigned long in_modifier,
+				    u8 op_modifier, u16 op,
+				    unsigned long timeout)
+{
+	struct hns_roce_cmdq *cmd = &hr_dev->cmd;
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_cmd_context *context;
+	int ret = 0;
+
+	spin_lock(&cmd->context_lock);
+	WARN_ON(cmd->free_head < 0);
+	context = &cmd->context[cmd->free_head];
+	context->token += cmd->token_mask + 1;
+	cmd->free_head = context->next;
+	spin_unlock(&cmd->context_lock);
+
+	init_completion(&context->done);
+
+	ret = hns_roce_cmd_mbox_post_hw(hr_dev, in_param, out_param,
+					in_modifier, op_modifier, op,
+					context->token, 1);
+	if (ret)
+		goto out;
+
+	/*
+	* It is timeout when wait_for_completion_timeout return 0
+	* The return value is the time limit set in advance
+	* how many seconds showing
+	*/
+	if (!wait_for_completion_timeout(&context->done,
+					 msecs_to_jiffies(timeout))) {
+		dev_err(dev, "[cmd]wait_for_completion_timeout timeout\n");
+		ret = -EBUSY;
+		goto out;
+	}
+
+	ret = context->result;
+	if (ret) {
+		dev_err(dev, "[cmd]event mod cmd process error!err=%d\n", ret);
+		goto out;
+	}
+
+out:
+	spin_lock(&cmd->context_lock);
+	context->next = cmd->free_head;
+	cmd->free_head = context - cmd->context;
+	spin_unlock(&cmd->context_lock);
+
+	return ret;
+}
+
+static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
+				  u64 out_param, unsigned long in_modifier,
+				  u8 op_modifier, u16 op, unsigned long timeout)
+{
+	int ret = 0;
+
+	down(&hr_dev->cmd.event_sem);
+	ret = __hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param,
+				       in_modifier, op_modifier, op, timeout);
+	up(&hr_dev->cmd.event_sem);
+
+	return ret;
+}
+
+int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,
+		      unsigned long in_modifier, u8 op_modifier, u16 op,
+		      unsigned long timeout)
+{
+	if (hr_dev->cmd.use_events)
+		return hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param,
+					      in_modifier, op_modifier, op,
+					      timeout);
+	else
+		return hns_roce_cmd_mbox_poll(hr_dev, in_param, out_param,
+					      in_modifier, op_modifier, op,
+					      timeout);
+}
+
+int hns_roce_cmd_init(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+
+	mutex_init(&hr_dev->cmd.hcr_mutex);
+	sema_init(&hr_dev->cmd.poll_sem, 1);
+	hr_dev->cmd.use_events = 0;
+	hr_dev->cmd.toggle = 1;
+	hr_dev->cmd.max_cmds = CMD_MAX_NUM;
+	hr_dev->cmd.hcr = hr_dev->reg_base + ROCEE_MB1_REG;
+	hr_dev->cmd.pool = dma_pool_create("hns_roce_cmd", dev,
+					   HNS_ROCE_MAILBOX_SIZE,
+					   HNS_ROCE_MAILBOX_SIZE, 0);
+	if (!hr_dev->cmd.pool)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev)
+{
+	dma_pool_destroy(hr_dev->cmd.pool);
+}
+
+int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_cmdq *hr_cmd = &hr_dev->cmd;
+	int i;
+
+	hr_cmd->context = kmalloc(hr_cmd->max_cmds *
+				  sizeof(struct hns_roce_cmd_context),
+				  GFP_KERNEL);
+	if (!hr_cmd->context)
+		return -ENOMEM;
+
+	for (i = 0; i < hr_cmd->max_cmds; ++i) {
+		hr_cmd->context[i].token = i;
+		hr_cmd->context[i].next = i + 1;
+	}
+
+	hr_cmd->context[hr_cmd->max_cmds - 1].next = -1;
+	hr_cmd->free_head = 0;
+
+	sema_init(&hr_cmd->event_sem, hr_cmd->max_cmds);
+	spin_lock_init(&hr_cmd->context_lock);
+
+	hr_cmd->token_mask = CMD_TOKEN_MASK;
+	hr_cmd->use_events = 1;
+
+	down(&hr_cmd->poll_sem);
+
+	return 0;
+}
+
+void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_cmdq *hr_cmd = &hr_dev->cmd;
+	int i;
+
+	hr_cmd->use_events = 0;
+
+	for (i = 0; i < hr_cmd->max_cmds; ++i)
+		down(&hr_cmd->event_sem);
+
+	kfree(hr_cmd->context);
+	up(&hr_cmd->poll_sem);
+}
+
+struct hns_roce_cmd_mailbox
+	*hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_cmd_mailbox *mailbox;
+
+	mailbox = kmalloc(sizeof(*mailbox), GFP_KERNEL);
+	if (!mailbox)
+		return ERR_PTR(-ENOMEM);
+
+	mailbox->buf = dma_pool_alloc(hr_dev->cmd.pool, GFP_KERNEL,
+				      &mailbox->dma);
+	if (!mailbox->buf) {
+		kfree(mailbox);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return mailbox;
+}
+
+void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,
+			       struct hns_roce_cmd_mailbox *mailbox)
+{
+	if (!mailbox)
+		return;
+
+	dma_pool_free(hr_dev->cmd.pool, mailbox->buf, mailbox->dma);
+	kfree(mailbox);
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h
new file mode 100644
index 000000000000..e3997d312c55
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _HNS_ROCE_CMD_H
+#define _HNS_ROCE_CMD_H
+
+#define HNS_ROCE_MAILBOX_SIZE		4096
+
+enum {
+	/* TPT commands */
+	HNS_ROCE_CMD_SW2HW_MPT		= 0xd,
+	HNS_ROCE_CMD_HW2SW_MPT		= 0xf,
+
+	/* CQ commands */
+	HNS_ROCE_CMD_SW2HW_CQ		= 0x16,
+	HNS_ROCE_CMD_HW2SW_CQ		= 0x17,
+
+	/* QP/EE commands */
+	HNS_ROCE_CMD_RST2INIT_QP	= 0x19,
+	HNS_ROCE_CMD_INIT2RTR_QP	= 0x1a,
+	HNS_ROCE_CMD_RTR2RTS_QP		= 0x1b,
+	HNS_ROCE_CMD_RTS2RTS_QP		= 0x1c,
+	HNS_ROCE_CMD_2ERR_QP		= 0x1e,
+	HNS_ROCE_CMD_RTS2SQD_QP		= 0x1f,
+	HNS_ROCE_CMD_SQD2SQD_QP		= 0x38,
+	HNS_ROCE_CMD_SQD2RTS_QP		= 0x20,
+	HNS_ROCE_CMD_2RST_QP		= 0x21,
+	HNS_ROCE_CMD_QUERY_QP		= 0x22,
+};
+
+enum {
+	HNS_ROCE_CMD_TIME_CLASS_A	= 10000,
+	HNS_ROCE_CMD_TIME_CLASS_B	= 10000,
+	HNS_ROCE_CMD_TIME_CLASS_C	= 10000,
+};
+
+struct hns_roce_cmd_mailbox {
+	void		       *buf;
+	dma_addr_t		dma;
+};
+
+int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,
+		      unsigned long in_modifier, u8 op_modifier, u16 op,
+		      unsigned long timeout);
+
+struct hns_roce_cmd_mailbox
+	*hns_roce_alloc_cmd_mailbox(struct hns_roce_dev *hr_dev);
+void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev,
+			       struct hns_roce_cmd_mailbox *mailbox);
+
+#endif /* _HNS_ROCE_CMD_H */
diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h
new file mode 100644
index 000000000000..297016103aa7
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_common.h
@@ -0,0 +1,325 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _HNS_ROCE_COMMON_H
+#define _HNS_ROCE_COMMON_H
+
+#ifndef assert
+#define assert(cond)
+#endif
+
+#define roce_write(dev, reg, val)	writel((val), (dev)->reg_base + (reg))
+#define roce_read(dev, reg)		readl((dev)->reg_base + (reg))
+#define roce_raw_write(value, addr) \
+	__raw_writel((__force u32)cpu_to_le32(value), (addr))
+
+#define roce_get_field(origin, mask, shift) \
+	(((origin) & (mask)) >> (shift))
+
+#define roce_get_bit(origin, shift) \
+	roce_get_field((origin), (1ul << (shift)), (shift))
+
+#define roce_set_field(origin, mask, shift, val) \
+	do { \
+		(origin) &= (~(mask)); \
+		(origin) |= (((u32)(val) << (shift)) & (mask)); \
+	} while (0)
+
+#define roce_set_bit(origin, shift, val) \
+	roce_set_field((origin), (1ul << (shift)), (shift), (val))
+
+#define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3
+#define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4
+
+#define ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S 5
+
+#define ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S 6
+
+#define ROCEE_GLB_CFG_ROCEE_PORT_ST_S 10
+#define ROCEE_GLB_CFG_ROCEE_PORT_ST_M  \
+	(((1UL << 6) - 1) << ROCEE_GLB_CFG_ROCEE_PORT_ST_S)
+
+#define ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S 16
+
+#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S 0
+#define ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M  \
+	(((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S)
+
+#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S 24
+#define ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M  \
+	(((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S)
+
+#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S 0
+#define ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M   \
+	(((1UL << 24) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S)
+
+#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S 24
+#define ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M   \
+	(((1UL << 4) - 1) << ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S)
+
+#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S 0
+#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M   \
+	(((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S)
+
+#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S 16
+#define ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M   \
+	(((1UL << 16) - 1) << ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S)
+
+#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S 0
+#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M   \
+	(((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S)
+
+#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S 16
+#define ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M   \
+	(((1UL << 16) - 1) << ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S)
+
+#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_S 0
+#define ROCEE_RAQ_WL_ROCEE_RAQ_WL_M   \
+	(((1UL << 8) - 1) << ROCEE_RAQ_WL_ROCEE_RAQ_WL_S)
+
+#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S 0
+#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M   \
+	(((1UL << 15) - 1) << \
+	ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S)
+
+#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S 16
+#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M   \
+	(((1UL << 4) - 1) << \
+	ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S)
+
+#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S 20
+
+#define ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE 21
+
+#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S 0
+#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M   \
+	(((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S)
+
+#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S 5
+#define ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M   \
+	(((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S)
+
+#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S 0
+#define ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M   \
+	(((1UL << 5) - 1) << ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S)
+
+#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S 5
+#define ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M   \
+	(((1UL << 5) - 1) << ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S)
+
+#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S 0
+#define ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M   \
+	(((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S)
+
+#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S 8
+#define ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M   \
+	(((1UL << 5) - 1) << ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S)
+
+#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S 0
+#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M   \
+	(((1UL << 19) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S)
+
+#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_S 19
+
+#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S 20
+#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M   \
+	(((1UL << 2) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S)
+
+#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S 22
+#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M   \
+	(((1UL << 5) - 1) << ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S)
+
+#define ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S 31
+
+#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S 0
+#define ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M   \
+	(((1UL << 3) - 1) << ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S)
+
+#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S 0
+#define ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M   \
+	(((1UL << 15) - 1) << ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S)
+
+#define ROCEE_MB6_ROCEE_MB_CMD_S 0
+#define ROCEE_MB6_ROCEE_MB_CMD_M   \
+	(((1UL << 8) - 1) << ROCEE_MB6_ROCEE_MB_CMD_S)
+
+#define ROCEE_MB6_ROCEE_MB_CMD_MDF_S 8
+#define ROCEE_MB6_ROCEE_MB_CMD_MDF_M   \
+	(((1UL << 4) - 1) << ROCEE_MB6_ROCEE_MB_CMD_MDF_S)
+
+#define ROCEE_MB6_ROCEE_MB_EVENT_S 14
+
+#define ROCEE_MB6_ROCEE_MB_HW_RUN_S 15
+
+#define ROCEE_MB6_ROCEE_MB_TOKEN_S 16
+#define ROCEE_MB6_ROCEE_MB_TOKEN_M   \
+	(((1UL << 16) - 1) << ROCEE_MB6_ROCEE_MB_TOKEN_S)
+
+#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S 0
+#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M   \
+	(((1UL << 24) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S)
+
+#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S 24
+#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M   \
+	(((1UL << 4) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S)
+
+#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S 28
+#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M   \
+	(((1UL << 3) - 1) << ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S)
+
+#define ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S 31
+
+#define ROCEE_SMAC_H_ROCEE_SMAC_H_S 0
+#define ROCEE_SMAC_H_ROCEE_SMAC_H_M   \
+	(((1UL << 16) - 1) << ROCEE_SMAC_H_ROCEE_SMAC_H_S)
+
+#define ROCEE_SMAC_H_ROCEE_PORT_MTU_S 16
+#define ROCEE_SMAC_H_ROCEE_PORT_MTU_M   \
+	(((1UL << 4) - 1) << ROCEE_SMAC_H_ROCEE_PORT_MTU_S)
+
+#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S 0
+#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M   \
+	(((1UL << 2) - 1) << ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S)
+
+#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S 8
+#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M   \
+	(((1UL << 4) - 1) << ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S)
+
+#define ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S 17
+
+#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S 0
+#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M   \
+	(((1UL << 5) - 1) << ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S)
+
+#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S 16
+#define ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M   \
+	(((1UL << 16) - 1) << ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S)
+
+#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S 0
+#define ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M   \
+	(((1UL << 16) - 1) << ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S)
+
+#define ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S 16
+#define ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S 1
+#define ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S 0
+
+#define ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S 0
+#define ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S 1
+
+#define ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S 0
+
+#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S 0
+#define ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M   \
+	(((1UL << 28) - 1) << ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S)
+
+#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S 0
+#define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M   \
+	(((1UL << 28) - 1) << ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S)
+
+#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_S 0
+#define ROCEE_SDB_INV_CNT_SDB_INV_CNT_M   \
+	(((1UL << 16) - 1) << ROCEE_SDB_INV_CNT_SDB_INV_CNT_S)
+
+/*************ROCEE_REG DEFINITION****************/
+#define ROCEE_VENDOR_ID_REG			0x0
+#define ROCEE_VENDOR_PART_ID_REG		0x4
+
+#define ROCEE_HW_VERSION_REG			0x8
+
+#define ROCEE_SYS_IMAGE_GUID_L_REG		0xC
+#define ROCEE_SYS_IMAGE_GUID_H_REG		0x10
+
+#define ROCEE_PORT_GID_L_0_REG			0x50
+#define ROCEE_PORT_GID_ML_0_REG			0x54
+#define ROCEE_PORT_GID_MH_0_REG			0x58
+#define ROCEE_PORT_GID_H_0_REG			0x5C
+
+#define ROCEE_BT_CMD_H_REG			0x204
+
+#define ROCEE_SMAC_L_0_REG			0x240
+#define ROCEE_SMAC_H_0_REG			0x244
+
+#define ROCEE_QP1C_CFG3_0_REG			0x27C
+
+#define ROCEE_CAEP_AEQE_CONS_IDX_REG		0x3AC
+#define ROCEE_CAEP_CEQC_CONS_IDX_0_REG		0x3BC
+
+#define ROCEE_ECC_UCERR_ALM1_REG		0xB38
+#define ROCEE_ECC_UCERR_ALM2_REG		0xB3C
+#define ROCEE_ECC_CERR_ALM1_REG			0xB44
+#define ROCEE_ECC_CERR_ALM2_REG			0xB48
+
+#define ROCEE_ACK_DELAY_REG			0x14
+#define ROCEE_GLB_CFG_REG			0x18
+
+#define ROCEE_DMAE_USER_CFG1_REG		0x40
+#define ROCEE_DMAE_USER_CFG2_REG		0x44
+
+#define ROCEE_DB_SQ_WL_REG			0x154
+#define ROCEE_DB_OTHERS_WL_REG			0x158
+#define ROCEE_RAQ_WL_REG			0x15C
+#define ROCEE_WRMS_POL_TIME_INTERVAL_REG	0x160
+#define ROCEE_EXT_DB_SQ_REG			0x164
+#define ROCEE_EXT_DB_SQ_H_REG			0x168
+#define ROCEE_EXT_DB_OTH_REG			0x16C
+
+#define ROCEE_EXT_DB_OTH_H_REG			0x170
+#define ROCEE_EXT_DB_SQ_WL_EMPTY_REG		0x174
+#define ROCEE_EXT_DB_SQ_WL_REG			0x178
+#define ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG	0x17C
+#define ROCEE_EXT_DB_OTHERS_WL_REG		0x180
+#define ROCEE_EXT_RAQ_REG			0x184
+#define ROCEE_EXT_RAQ_H_REG			0x188
+
+#define ROCEE_CAEP_CE_INTERVAL_CFG_REG		0x190
+#define ROCEE_CAEP_CE_BURST_NUM_CFG_REG		0x194
+#define ROCEE_BT_CMD_L_REG			0x200
+
+#define ROCEE_MB1_REG				0x210
+#define ROCEE_DB_SQ_L_0_REG			0x230
+#define ROCEE_DB_OTHERS_L_0_REG			0x238
+#define ROCEE_QP1C_CFG0_0_REG			0x270
+
+#define ROCEE_CAEP_AEQC_AEQE_SHIFT_REG		0x3A0
+#define ROCEE_CAEP_CEQC_SHIFT_0_REG		0x3B0
+#define ROCEE_CAEP_CE_IRQ_MASK_0_REG		0x3C0
+#define ROCEE_CAEP_CEQ_ALM_OVF_0_REG		0x3C4
+#define ROCEE_CAEP_AE_MASK_REG			0x6C8
+#define ROCEE_CAEP_AE_ST_REG			0x6CC
+
+#define ROCEE_SDB_ISSUE_PTR_REG			0x758
+#define ROCEE_SDB_SEND_PTR_REG			0x75C
+#define ROCEE_SDB_INV_CNT_REG			0x9A4
+#define ROCEE_ECC_UCERR_ALM0_REG		0xB34
+#define ROCEE_ECC_CERR_ALM0_REG			0xB40
+
+#endif /* _HNS_ROCE_COMMON_H */
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c
new file mode 100644
index 000000000000..875597b0e69c
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_cq.c
@@ -0,0 +1,446 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/platform_device.h>
+#include <rdma/ib_umem.h>
+#include "hns_roce_device.h"
+#include "hns_roce_cmd.h"
+#include "hns_roce_hem.h"
+#include "hns_roce_user.h"
+#include "hns_roce_common.h"
+
+static void hns_roce_ib_cq_comp(struct hns_roce_cq *hr_cq)
+{
+	struct ib_cq *ibcq = &hr_cq->ib_cq;
+
+	ibcq->comp_handler(ibcq, ibcq->cq_context);
+}
+
+static void hns_roce_ib_cq_event(struct hns_roce_cq *hr_cq,
+				 enum hns_roce_event event_type)
+{
+	struct hns_roce_dev *hr_dev;
+	struct ib_event event;
+	struct ib_cq *ibcq;
+
+	ibcq = &hr_cq->ib_cq;
+	hr_dev = to_hr_dev(ibcq->device);
+
+	if (event_type != HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID &&
+	    event_type != HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR &&
+	    event_type != HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW) {
+		dev_err(&hr_dev->pdev->dev,
+			"hns_roce_ib: Unexpected event type 0x%x on CQ %06lx\n",
+			event_type, hr_cq->cqn);
+		return;
+	}
+
+	if (ibcq->event_handler) {
+		event.device = ibcq->device;
+		event.event = IB_EVENT_CQ_ERR;
+		event.element.cq = ibcq;
+		ibcq->event_handler(&event, ibcq->cq_context);
+	}
+}
+
+static int hns_roce_sw2hw_cq(struct hns_roce_dev *dev,
+			     struct hns_roce_cmd_mailbox *mailbox,
+			     unsigned long cq_num)
+{
+	return hns_roce_cmd_mbox(dev, mailbox->dma, 0, cq_num, 0,
+			    HNS_ROCE_CMD_SW2HW_CQ, HNS_ROCE_CMD_TIME_CLASS_A);
+}
+
+static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent,
+			     struct hns_roce_mtt *hr_mtt,
+			     struct hns_roce_uar *hr_uar,
+			     struct hns_roce_cq *hr_cq, int vector,
+			     int collapsed)
+{
+	struct hns_roce_cmd_mailbox *mailbox = NULL;
+	struct hns_roce_cq_table *cq_table = NULL;
+	struct device *dev = &hr_dev->pdev->dev;
+	dma_addr_t dma_handle;
+	u64 *mtts = NULL;
+	int ret = 0;
+
+	cq_table = &hr_dev->cq_table;
+
+	/* Get the physical address of cq buf */
+	mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
+				   hr_mtt->first_seg, &dma_handle);
+	if (!mtts) {
+		dev_err(dev, "CQ alloc.Failed to find cq buf addr.\n");
+		return -EINVAL;
+	}
+
+	if (vector >= hr_dev->caps.num_comp_vectors) {
+		dev_err(dev, "CQ alloc.Invalid vector.\n");
+		return -EINVAL;
+	}
+	hr_cq->vector = vector;
+
+	ret = hns_roce_bitmap_alloc(&cq_table->bitmap, &hr_cq->cqn);
+	if (ret == -1) {
+		dev_err(dev, "CQ alloc.Failed to alloc index.\n");
+		return -ENOMEM;
+	}
+
+	/* Get CQC memory HEM(Hardware Entry Memory) table */
+	ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn);
+	if (ret) {
+		dev_err(dev, "CQ alloc.Failed to get context mem.\n");
+		goto err_out;
+	}
+
+	/* The cq insert radix tree */
+	spin_lock_irq(&cq_table->lock);
+	/* Radix_tree: The associated pointer and long integer key value like */
+	ret = radix_tree_insert(&cq_table->tree, hr_cq->cqn, hr_cq);
+	spin_unlock_irq(&cq_table->lock);
+	if (ret) {
+		dev_err(dev, "CQ alloc.Failed to radix_tree_insert.\n");
+		goto err_put;
+	}
+
+	/* Allocate mailbox memory */
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox)) {
+		ret = PTR_ERR(mailbox);
+		goto err_radix;
+	}
+
+	hr_dev->hw->write_cqc(hr_dev, hr_cq, mailbox->buf, mtts, dma_handle,
+			      nent, vector);
+
+	/* Send mailbox to hw */
+	ret = hns_roce_sw2hw_cq(hr_dev, mailbox, hr_cq->cqn);
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+	if (ret) {
+		dev_err(dev, "CQ alloc.Failed to cmd mailbox.\n");
+		goto err_radix;
+	}
+
+	hr_cq->cons_index = 0;
+	hr_cq->uar = hr_uar;
+
+	return 0;
+
+err_radix:
+	spin_lock_irq(&cq_table->lock);
+	radix_tree_delete(&cq_table->tree, hr_cq->cqn);
+	spin_unlock_irq(&cq_table->lock);
+
+err_put:
+	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);
+
+err_out:
+	hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn);
+	return ret;
+}
+
+static int hns_roce_hw2sw_cq(struct hns_roce_dev *dev,
+			     struct hns_roce_cmd_mailbox *mailbox,
+			     unsigned long cq_num)
+{
+	return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox->dma : 0, cq_num,
+				 mailbox ? 0 : 1, HNS_ROCE_CMD_HW2SW_CQ,
+				 HNS_ROCE_CMD_TIME_CLASS_A);
+}
+
+static void hns_roce_free_cq(struct hns_roce_dev *hr_dev,
+			     struct hns_roce_cq *hr_cq)
+{
+	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;
+	struct device *dev = &hr_dev->pdev->dev;
+	int ret;
+
+	ret = hns_roce_hw2sw_cq(hr_dev, NULL, hr_cq->cqn);
+	if (ret)
+		dev_err(dev, "HW2SW_CQ failed (%d) for CQN %06lx\n", ret,
+			hr_cq->cqn);
+
+	/* Waiting interrupt process procedure carried out */
+	synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq);
+
+	spin_lock_irq(&cq_table->lock);
+	radix_tree_delete(&cq_table->tree, hr_cq->cqn);
+	spin_unlock_irq(&cq_table->lock);
+
+	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);
+	hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn);
+}
+
+static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev,
+				   struct ib_ucontext *context,
+				   struct hns_roce_cq_buf *buf,
+				   struct ib_umem **umem, u64 buf_addr, int cqe)
+{
+	int ret;
+
+	*umem = ib_umem_get(context, buf_addr, cqe * hr_dev->caps.cq_entry_sz,
+			    IB_ACCESS_LOCAL_WRITE, 1);
+	if (IS_ERR(*umem))
+		return PTR_ERR(*umem);
+
+	ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(*umem),
+				ilog2((unsigned int)(*umem)->page_size),
+				&buf->hr_mtt);
+	if (ret)
+		goto err_buf;
+
+	ret = hns_roce_ib_umem_write_mtt(hr_dev, &buf->hr_mtt, *umem);
+	if (ret)
+		goto err_mtt;
+
+	return 0;
+
+err_mtt:
+	hns_roce_mtt_cleanup(hr_dev, &buf->hr_mtt);
+
+err_buf:
+	ib_umem_release(*umem);
+	return ret;
+}
+
+static int hns_roce_ib_alloc_cq_buf(struct hns_roce_dev *hr_dev,
+				    struct hns_roce_cq_buf *buf, u32 nent)
+{
+	int ret;
+
+	ret = hns_roce_buf_alloc(hr_dev, nent * hr_dev->caps.cq_entry_sz,
+				 PAGE_SIZE * 2, &buf->hr_buf);
+	if (ret)
+		goto out;
+
+	ret = hns_roce_mtt_init(hr_dev, buf->hr_buf.npages,
+				buf->hr_buf.page_shift, &buf->hr_mtt);
+	if (ret)
+		goto err_buf;
+
+	ret = hns_roce_buf_write_mtt(hr_dev, &buf->hr_mtt, &buf->hr_buf);
+	if (ret)
+		goto err_mtt;
+
+	return 0;
+
+err_mtt:
+	hns_roce_mtt_cleanup(hr_dev, &buf->hr_mtt);
+
+err_buf:
+	hns_roce_buf_free(hr_dev, nent * hr_dev->caps.cq_entry_sz,
+			  &buf->hr_buf);
+out:
+	return ret;
+}
+
+static void hns_roce_ib_free_cq_buf(struct hns_roce_dev *hr_dev,
+				    struct hns_roce_cq_buf *buf, int cqe)
+{
+	hns_roce_buf_free(hr_dev, (cqe + 1) * hr_dev->caps.cq_entry_sz,
+			  &buf->hr_buf);
+}
+
+struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
+				    const struct ib_cq_init_attr *attr,
+				    struct ib_ucontext *context,
+				    struct ib_udata *udata)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_ib_create_cq ucmd;
+	struct hns_roce_cq *hr_cq = NULL;
+	struct hns_roce_uar *uar = NULL;
+	int vector = attr->comp_vector;
+	int cq_entries = attr->cqe;
+	int ret = 0;
+
+	if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) {
+		dev_err(dev, "Creat CQ failed. entries=%d, max=%d\n",
+			cq_entries, hr_dev->caps.max_cqes);
+		return ERR_PTR(-EINVAL);
+	}
+
+	hr_cq = kmalloc(sizeof(*hr_cq), GFP_KERNEL);
+	if (!hr_cq)
+		return ERR_PTR(-ENOMEM);
+
+	/* In v1 engine, parameter verification */
+	if (cq_entries < HNS_ROCE_MIN_CQE_NUM)
+		cq_entries = HNS_ROCE_MIN_CQE_NUM;
+
+	cq_entries = roundup_pow_of_two((unsigned int)cq_entries);
+	hr_cq->ib_cq.cqe = cq_entries - 1;
+	mutex_init(&hr_cq->resize_mutex);
+	spin_lock_init(&hr_cq->lock);
+	hr_cq->hr_resize_buf = NULL;
+	hr_cq->resize_umem = NULL;
+
+	if (context) {
+		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
+			dev_err(dev, "Failed to copy_from_udata.\n");
+			ret = -EFAULT;
+			goto err_cq;
+		}
+
+		/* Get user space address, write it into mtt table */
+		ret = hns_roce_ib_get_cq_umem(hr_dev, context, &hr_cq->hr_buf,
+					      &hr_cq->umem, ucmd.buf_addr,
+					      cq_entries);
+		if (ret) {
+			dev_err(dev, "Failed to get_cq_umem.\n");
+			goto err_cq;
+		}
+
+		/* Get user space parameters */
+		uar = &to_hr_ucontext(context)->uar;
+	} else {
+		/* Init mmt table and write buff address to mtt table */
+		ret = hns_roce_ib_alloc_cq_buf(hr_dev, &hr_cq->hr_buf,
+					       cq_entries);
+		if (ret) {
+			dev_err(dev, "Failed to alloc_cq_buf.\n");
+			goto err_cq;
+		}
+
+		uar = &hr_dev->priv_uar;
+		hr_cq->cq_db_l = hr_dev->reg_base + ROCEE_DB_OTHERS_L_0_REG +
+				 0x1000 * uar->index;
+	}
+
+	/* Allocate cq index, fill cq_context */
+	ret = hns_roce_cq_alloc(hr_dev, cq_entries, &hr_cq->hr_buf.hr_mtt,
+				uar, hr_cq, vector, 0);
+	if (ret) {
+		dev_err(dev, "Creat CQ .Failed to cq_alloc.\n");
+		goto err_mtt;
+	}
+
+	/* Get created cq handler and carry out event */
+	hr_cq->comp = hns_roce_ib_cq_comp;
+	hr_cq->event = hns_roce_ib_cq_event;
+	hr_cq->cq_depth = cq_entries;
+
+	if (context) {
+		if (ib_copy_to_udata(udata, &hr_cq->cqn, sizeof(u64))) {
+			ret = -EFAULT;
+			goto err_mtt;
+		}
+	}
+
+	return &hr_cq->ib_cq;
+
+err_mtt:
+	hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
+	if (context)
+		ib_umem_release(hr_cq->umem);
+	else
+		hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf,
+					hr_cq->ib_cq.cqe);
+
+err_cq:
+	kfree(hr_cq);
+	return ERR_PTR(ret);
+}
+
+int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
+	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
+
+	hns_roce_free_cq(hr_dev, hr_cq);
+	hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
+
+	if (ib_cq->uobject)
+		ib_umem_release(hr_cq->umem);
+	else
+		/* Free the buff of stored cq */
+		hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, ib_cq->cqe);
+
+	kfree(hr_cq);
+
+	return 0;
+}
+
+void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_cq *cq;
+
+	cq = radix_tree_lookup(&hr_dev->cq_table.tree,
+			       cqn & (hr_dev->caps.num_cqs - 1));
+	if (!cq) {
+		dev_warn(dev, "Completion event for bogus CQ 0x%08x\n", cqn);
+		return;
+	}
+
+	cq->comp(cq);
+}
+
+void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type)
+{
+	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_cq *cq;
+
+	cq = radix_tree_lookup(&cq_table->tree,
+			       cqn & (hr_dev->caps.num_cqs - 1));
+	if (cq)
+		atomic_inc(&cq->refcount);
+
+	if (!cq) {
+		dev_warn(dev, "Async event for bogus CQ %08x\n", cqn);
+		return;
+	}
+
+	cq->event(cq, (enum hns_roce_event)event_type);
+
+	if (atomic_dec_and_test(&cq->refcount))
+		complete(&cq->free);
+}
+
+int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;
+
+	spin_lock_init(&cq_table->lock);
+	INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
+
+	return hns_roce_bitmap_init(&cq_table->bitmap, hr_dev->caps.num_cqs,
+				    hr_dev->caps.num_cqs - 1,
+				    hr_dev->caps.reserved_cqs, 0);
+}
+
+void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev)
+{
+	hns_roce_bitmap_cleanup(&hr_dev->cq_table.bitmap);
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
new file mode 100644
index 000000000000..ea735800eb18
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -0,0 +1,734 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _HNS_ROCE_DEVICE_H
+#define _HNS_ROCE_DEVICE_H
+
+#include <rdma/ib_verbs.h>
+
+#define DRV_NAME "hns_roce"
+
+#define MAC_ADDR_OCTET_NUM			6
+#define HNS_ROCE_MAX_MSG_LEN			0x80000000
+
+#define HNS_ROCE_ALOGN_UP(a, b) ((((a) + (b) - 1) / (b)) * (b))
+
+#define HNS_ROCE_IB_MIN_SQ_STRIDE		6
+
+#define HNS_ROCE_BA_SIZE			(32 * 4096)
+
+/* Hardware specification only for v1 engine */
+#define HNS_ROCE_MIN_CQE_NUM			0x40
+#define HNS_ROCE_MIN_WQE_NUM			0x20
+
+/* Hardware specification only for v1 engine */
+#define HNS_ROCE_MAX_INNER_MTPT_NUM		0x7
+#define HNS_ROCE_MAX_MTPT_PBL_NUM		0x100000
+
+#define HNS_ROCE_MAX_IRQ_NUM			34
+
+#define HNS_ROCE_COMP_VEC_NUM			32
+
+#define HNS_ROCE_AEQE_VEC_NUM			1
+#define HNS_ROCE_AEQE_OF_VEC_NUM		1
+
+/* 4G/4K = 1M */
+#define HNS_ROCE_SL_SHIFT			29
+#define HNS_ROCE_TCLASS_SHIFT			20
+#define HNS_ROCE_FLOW_LABLE_MASK		0xfffff
+
+#define HNS_ROCE_MAX_PORTS			6
+#define HNS_ROCE_MAX_GID_NUM			16
+#define HNS_ROCE_GID_SIZE			16
+
+#define MR_TYPE_MR				0x00
+#define MR_TYPE_DMA				0x03
+
+#define PKEY_ID					0xffff
+#define NODE_DESC_SIZE				64
+
+#define SERV_TYPE_RC				0
+#define SERV_TYPE_RD				1
+#define SERV_TYPE_UC				2
+#define SERV_TYPE_UD				3
+
+#define PAGES_SHIFT_8				8
+#define PAGES_SHIFT_16				16
+#define PAGES_SHIFT_24				24
+#define PAGES_SHIFT_32				32
+
+enum hns_roce_qp_state {
+	HNS_ROCE_QP_STATE_RST,
+	HNS_ROCE_QP_STATE_INIT,
+	HNS_ROCE_QP_STATE_RTR,
+	HNS_ROCE_QP_STATE_RTS,
+	HNS_ROCE_QP_STATE_SQD,
+	HNS_ROCE_QP_STATE_ERR,
+	HNS_ROCE_QP_NUM_STATE,
+};
+
+enum hns_roce_event {
+	HNS_ROCE_EVENT_TYPE_PATH_MIG                  = 0x01,
+	HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED           = 0x02,
+	HNS_ROCE_EVENT_TYPE_COMM_EST                  = 0x03,
+	HNS_ROCE_EVENT_TYPE_SQ_DRAINED                = 0x04,
+	HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR            = 0x05,
+	HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR    = 0x06,
+	HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR     = 0x07,
+	HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH           = 0x08,
+	HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH        = 0x09,
+	HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR           = 0x0a,
+	HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR           = 0x0b,
+	HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW               = 0x0c,
+	HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID             = 0x0d,
+	HNS_ROCE_EVENT_TYPE_PORT_CHANGE               = 0x0f,
+	/* 0x10 and 0x11 is unused in currently application case */
+	HNS_ROCE_EVENT_TYPE_DB_OVERFLOW               = 0x12,
+	HNS_ROCE_EVENT_TYPE_MB                        = 0x13,
+	HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW              = 0x14,
+};
+
+/* Local Work Queue Catastrophic Error,SUBTYPE 0x5 */
+enum {
+	HNS_ROCE_LWQCE_QPC_ERROR		= 1,
+	HNS_ROCE_LWQCE_MTU_ERROR		= 2,
+	HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR	= 3,
+	HNS_ROCE_LWQCE_WQE_ADDR_ERROR		= 4,
+	HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR	= 5,
+	HNS_ROCE_LWQCE_SL_ERROR			= 6,
+	HNS_ROCE_LWQCE_PORT_ERROR		= 7,
+};
+
+/* Local Access Violation Work Queue Error,SUBTYPE 0x7 */
+enum {
+	HNS_ROCE_LAVWQE_R_KEY_VIOLATION		= 1,
+	HNS_ROCE_LAVWQE_LENGTH_ERROR		= 2,
+	HNS_ROCE_LAVWQE_VA_ERROR		= 3,
+	HNS_ROCE_LAVWQE_PD_ERROR		= 4,
+	HNS_ROCE_LAVWQE_RW_ACC_ERROR		= 5,
+	HNS_ROCE_LAVWQE_KEY_STATE_ERROR		= 6,
+	HNS_ROCE_LAVWQE_MR_OPERATION_ERROR	= 7,
+};
+
+/* DOORBELL overflow subtype */
+enum {
+	HNS_ROCE_DB_SUBTYPE_SDB_OVF		= 1,
+	HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF		= 2,
+	HNS_ROCE_DB_SUBTYPE_ODB_OVF		= 3,
+	HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF		= 4,
+	HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP		= 5,
+	HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP		= 6,
+};
+
+enum {
+	/* RQ&SRQ related operations */
+	HNS_ROCE_OPCODE_SEND_DATA_RECEIVE	= 0x06,
+	HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE	= 0x07,
+};
+
+#define HNS_ROCE_CMD_SUCCESS			1
+
+#define HNS_ROCE_PORT_DOWN			0
+#define HNS_ROCE_PORT_UP			1
+
+#define HNS_ROCE_MTT_ENTRY_PER_SEG		8
+
+#define PAGE_ADDR_SHIFT				12
+
+struct hns_roce_uar {
+	u64		pfn;
+	unsigned long	index;
+};
+
+struct hns_roce_ucontext {
+	struct ib_ucontext	ibucontext;
+	struct hns_roce_uar	uar;
+};
+
+struct hns_roce_pd {
+	struct ib_pd		ibpd;
+	unsigned long		pdn;
+};
+
+struct hns_roce_bitmap {
+	/* Bitmap Traversal last a bit which is 1 */
+	unsigned long		last;
+	unsigned long		top;
+	unsigned long		max;
+	unsigned long		reserved_top;
+	unsigned long		mask;
+	spinlock_t		lock;
+	unsigned long		*table;
+};
+
+/* Order bitmap length -- bit num compute formula: 1 << (max_order - order) */
+/* Order = 0: bitmap is biggest, order = max bitmap is least (only a bit) */
+/* Every bit repesent to a partner free/used status in bitmap */
+/*
+* Initial, bits of other bitmap are all 0 except that a bit of max_order is 1
+* Bit = 1 represent to idle and available; bit = 0: not available
+*/
+struct hns_roce_buddy {
+	/* Members point to every order level bitmap */
+	unsigned long **bits;
+	/* Represent to avail bits of the order level bitmap */
+	u32            *num_free;
+	int             max_order;
+	spinlock_t      lock;
+};
+
+/* For Hardware Entry Memory */
+struct hns_roce_hem_table {
+	/* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */
+	u32		type;
+	/* HEM array elment num */
+	unsigned long	num_hem;
+	/* HEM entry record obj total num */
+	unsigned long	num_obj;
+	/*Single obj size */
+	unsigned long	obj_size;
+	int		lowmem;
+	struct mutex	mutex;
+	struct hns_roce_hem **hem;
+};
+
+struct hns_roce_mtt {
+	unsigned long	first_seg;
+	int		order;
+	int		page_shift;
+};
+
+/* Only support 4K page size for mr register */
+#define MR_SIZE_4K 0
+
+struct hns_roce_mr {
+	struct ib_mr		ibmr;
+	struct ib_umem		*umem;
+	u64			iova; /* MR's virtual orignal addr */
+	u64			size; /* Address range of MR */
+	u32			key; /* Key of MR */
+	u32			pd;   /* PD num of MR */
+	u32			access;/* Access permission of MR */
+	int			enabled; /* MR's active status */
+	int			type;	/* MR's register type */
+	u64			*pbl_buf;/* MR's PBL space */
+	dma_addr_t		pbl_dma_addr;	/* MR's PBL space PA */
+};
+
+struct hns_roce_mr_table {
+	struct hns_roce_bitmap		mtpt_bitmap;
+	struct hns_roce_buddy		mtt_buddy;
+	struct hns_roce_hem_table	mtt_table;
+	struct hns_roce_hem_table	mtpt_table;
+};
+
+struct hns_roce_wq {
+	u64		*wrid;     /* Work request ID */
+	spinlock_t	lock;
+	int		wqe_cnt;  /* WQE num */
+	u32		max_post;
+	int		max_gs;
+	int		offset;
+	int		wqe_shift;/* WQE size */
+	u32		head;
+	u32		tail;
+	void __iomem	*db_reg_l;
+};
+
+struct hns_roce_buf_list {
+	void		*buf;
+	dma_addr_t	map;
+};
+
+struct hns_roce_buf {
+	struct hns_roce_buf_list	direct;
+	struct hns_roce_buf_list	*page_list;
+	int				nbufs;
+	u32				npages;
+	int				page_shift;
+};
+
+struct hns_roce_cq_buf {
+	struct hns_roce_buf hr_buf;
+	struct hns_roce_mtt hr_mtt;
+};
+
+struct hns_roce_cq_resize {
+	struct hns_roce_cq_buf	hr_buf;
+	int			cqe;
+};
+
+struct hns_roce_cq {
+	struct ib_cq			ib_cq;
+	struct hns_roce_cq_buf		hr_buf;
+	/* pointer to store information after resize*/
+	struct hns_roce_cq_resize	*hr_resize_buf;
+	spinlock_t			lock;
+	struct mutex			resize_mutex;
+	struct ib_umem			*umem;
+	struct ib_umem			*resize_umem;
+	void (*comp)(struct hns_roce_cq *);
+	void (*event)(struct hns_roce_cq *, enum hns_roce_event);
+
+	struct hns_roce_uar		*uar;
+	u32				cq_depth;
+	u32				cons_index;
+	void __iomem			*cq_db_l;
+	void __iomem			*tptr_addr;
+	unsigned long			cqn;
+	u32				vector;
+	atomic_t			refcount;
+	struct completion		free;
+};
+
+struct hns_roce_srq {
+	struct ib_srq		ibsrq;
+	int			srqn;
+};
+
+struct hns_roce_uar_table {
+	struct hns_roce_bitmap bitmap;
+};
+
+struct hns_roce_qp_table {
+	struct hns_roce_bitmap		bitmap;
+	spinlock_t			lock;
+	struct hns_roce_hem_table	qp_table;
+	struct hns_roce_hem_table	irrl_table;
+};
+
+struct hns_roce_cq_table {
+	struct hns_roce_bitmap		bitmap;
+	spinlock_t			lock;
+	struct radix_tree_root		tree;
+	struct hns_roce_hem_table	table;
+};
+
+struct hns_roce_raq_table {
+	struct hns_roce_buf_list	*e_raq_buf;
+};
+
+struct hns_roce_av {
+	__le32      port_pd;
+	u8          gid_index;
+	u8          stat_rate;
+	u8          hop_limit;
+	__le32      sl_tclass_flowlabel;
+	u8          dgid[HNS_ROCE_GID_SIZE];
+	u8          mac[6];
+	__le16      vlan;
+};
+
+struct hns_roce_ah {
+	struct ib_ah		ibah;
+	struct hns_roce_av	av;
+};
+
+struct hns_roce_cmd_context {
+	struct completion	done;
+	int			result;
+	int			next;
+	u64			out_param;
+	u16			token;
+};
+
+struct hns_roce_cmdq {
+	struct dma_pool		*pool;
+	u8 __iomem		*hcr;
+	struct mutex		hcr_mutex;
+	struct semaphore	poll_sem;
+	/*
+	* Event mode: cmd register mutex protection,
+	* ensure to not exceed max_cmds and user use limit region
+	*/
+	struct semaphore	event_sem;
+	int			max_cmds;
+	spinlock_t		context_lock;
+	int			free_head;
+	struct hns_roce_cmd_context *context;
+	/*
+	* Result of get integer part
+	* which max_comds compute according a power of 2
+	*/
+	u16			token_mask;
+	/*
+	* Process whether use event mode, init default non-zero
+	* After the event queue of cmd event ready,
+	* can switch into event mode
+	* close device, switch into poll mode(non event mode)
+	*/
+	u8			use_events;
+	u8			toggle;
+};
+
+struct hns_roce_dev;
+
+struct hns_roce_qp {
+	struct ib_qp		ibqp;
+	struct hns_roce_buf	hr_buf;
+	struct hns_roce_wq	rq;
+	__le64			doorbell_qpn;
+	__le32			sq_signal_bits;
+	u32			sq_next_wqe;
+	int			sq_max_wqes_per_wr;
+	int			sq_spare_wqes;
+	struct hns_roce_wq	sq;
+
+	struct ib_umem		*umem;
+	struct hns_roce_mtt	mtt;
+	u32			buff_size;
+	struct mutex		mutex;
+	u8			port;
+	u8			sl;
+	u8			resp_depth;
+	u8			state;
+	u32			access_flags;
+	u32			pkey_index;
+	void			(*event)(struct hns_roce_qp *,
+					 enum hns_roce_event);
+	unsigned long		qpn;
+
+	atomic_t		refcount;
+	struct completion	free;
+};
+
+struct hns_roce_sqp {
+	struct hns_roce_qp	hr_qp;
+};
+
+struct hns_roce_ib_iboe {
+	spinlock_t		lock;
+	struct net_device      *netdevs[HNS_ROCE_MAX_PORTS];
+	struct notifier_block	nb;
+	struct notifier_block	nb_inet;
+	/* 16 GID is shared by 6 port in v1 engine. */
+	union ib_gid		gid_table[HNS_ROCE_MAX_GID_NUM];
+	u8			phy_port[HNS_ROCE_MAX_PORTS];
+};
+
+struct hns_roce_eq {
+	struct hns_roce_dev		*hr_dev;
+	void __iomem			*doorbell;
+
+	int				type_flag;/* Aeq:1 ceq:0 */
+	int				eqn;
+	u32				entries;
+	int				log_entries;
+	int				eqe_size;
+	int				irq;
+	int				log_page_size;
+	int				cons_index;
+	struct hns_roce_buf_list	*buf_list;
+};
+
+struct hns_roce_eq_table {
+	struct hns_roce_eq	*eq;
+	void __iomem		**eqc_base;
+};
+
+struct hns_roce_caps {
+	u8		num_ports;
+	int		gid_table_len[HNS_ROCE_MAX_PORTS];
+	int		pkey_table_len[HNS_ROCE_MAX_PORTS];
+	int		local_ca_ack_delay;
+	int		num_uars;
+	u32		phy_num_uars;
+	u32		max_sq_sg;	/* 2 */
+	u32		max_sq_inline;	/* 32 */
+	u32		max_rq_sg;	/* 2 */
+	int		num_qps;	/* 256k */
+	u32		max_wqes;	/* 16k */
+	u32		max_sq_desc_sz;	/* 64 */
+	u32		max_rq_desc_sz;	/* 64 */
+	int		max_qp_init_rdma;
+	int		max_qp_dest_rdma;
+	int		sqp_start;
+	int		num_cqs;
+	int		max_cqes;
+	int		reserved_cqs;
+	int		num_aeq_vectors;	/* 1 */
+	int		num_comp_vectors;	/* 32 ceq */
+	int		num_other_vectors;
+	int		num_mtpts;
+	u32		num_mtt_segs;
+	int		reserved_mrws;
+	int		reserved_uars;
+	int		num_pds;
+	int		reserved_pds;
+	u32		mtt_entry_sz;
+	u32		cq_entry_sz;
+	u32		page_size_cap;
+	u32		reserved_lkey;
+	int		mtpt_entry_sz;
+	int		qpc_entry_sz;
+	int		irrl_entry_sz;
+	int		cqc_entry_sz;
+	int		aeqe_depth;
+	int		ceqe_depth[HNS_ROCE_COMP_VEC_NUM];
+	enum ib_mtu	max_mtu;
+};
+
+struct hns_roce_hw {
+	int (*reset)(struct hns_roce_dev *hr_dev, bool enable);
+	void (*hw_profile)(struct hns_roce_dev *hr_dev);
+	int (*hw_init)(struct hns_roce_dev *hr_dev);
+	void (*hw_exit)(struct hns_roce_dev *hr_dev);
+	void (*set_gid)(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
+			union ib_gid *gid);
+	void (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr);
+	void (*set_mtu)(struct hns_roce_dev *hr_dev, u8 phy_port,
+			enum ib_mtu mtu);
+	int (*write_mtpt)(void *mb_buf, struct hns_roce_mr *mr,
+			  unsigned long mtpt_idx);
+	void (*write_cqc)(struct hns_roce_dev *hr_dev,
+			  struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
+			  dma_addr_t dma_handle, int nent, u32 vector);
+	int (*query_qp)(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+			int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
+	int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
+			 int attr_mask, enum ib_qp_state cur_state,
+			 enum ib_qp_state new_state);
+	int (*destroy_qp)(struct ib_qp *ibqp);
+	int (*post_send)(struct ib_qp *ibqp, struct ib_send_wr *wr,
+			 struct ib_send_wr **bad_wr);
+	int (*post_recv)(struct ib_qp *qp, struct ib_recv_wr *recv_wr,
+			 struct ib_recv_wr **bad_recv_wr);
+	int (*req_notify_cq)(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
+	int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
+	void	*priv;
+};
+
+struct hns_roce_dev {
+	struct ib_device	ib_dev;
+	struct platform_device  *pdev;
+	struct hns_roce_uar     priv_uar;
+	const char		*irq_names[HNS_ROCE_MAX_IRQ_NUM];
+	spinlock_t		sm_lock;
+	spinlock_t		cq_db_lock;
+	spinlock_t		bt_cmd_lock;
+	struct hns_roce_ib_iboe iboe;
+
+	int			irq[HNS_ROCE_MAX_IRQ_NUM];
+	u8 __iomem		*reg_base;
+	struct hns_roce_caps	caps;
+	struct radix_tree_root  qp_table_tree;
+
+	unsigned char	dev_addr[HNS_ROCE_MAX_PORTS][MAC_ADDR_OCTET_NUM];
+	u64			sys_image_guid;
+	u32                     vendor_id;
+	u32                     vendor_part_id;
+	u32                     hw_rev;
+	void __iomem            *priv_addr;
+
+	struct hns_roce_cmdq	cmd;
+	struct hns_roce_bitmap    pd_bitmap;
+	struct hns_roce_uar_table uar_table;
+	struct hns_roce_mr_table  mr_table;
+	struct hns_roce_cq_table  cq_table;
+	struct hns_roce_qp_table  qp_table;
+	struct hns_roce_eq_table  eq_table;
+
+	int			cmd_mod;
+	int			loop_idc;
+	struct hns_roce_hw	*hw;
+};
+
+static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)
+{
+	return container_of(ib_dev, struct hns_roce_dev, ib_dev);
+}
+
+static inline struct hns_roce_ucontext
+			*to_hr_ucontext(struct ib_ucontext *ibucontext)
+{
+	return container_of(ibucontext, struct hns_roce_ucontext, ibucontext);
+}
+
+static inline struct hns_roce_pd *to_hr_pd(struct ib_pd *ibpd)
+{
+	return container_of(ibpd, struct hns_roce_pd, ibpd);
+}
+
+static inline struct hns_roce_ah *to_hr_ah(struct ib_ah *ibah)
+{
+	return container_of(ibah, struct hns_roce_ah, ibah);
+}
+
+static inline struct hns_roce_mr *to_hr_mr(struct ib_mr *ibmr)
+{
+	return container_of(ibmr, struct hns_roce_mr, ibmr);
+}
+
+static inline struct hns_roce_qp *to_hr_qp(struct ib_qp *ibqp)
+{
+	return container_of(ibqp, struct hns_roce_qp, ibqp);
+}
+
+static inline struct hns_roce_cq *to_hr_cq(struct ib_cq *ib_cq)
+{
+	return container_of(ib_cq, struct hns_roce_cq, ib_cq);
+}
+
+static inline struct hns_roce_srq *to_hr_srq(struct ib_srq *ibsrq)
+{
+	return container_of(ibsrq, struct hns_roce_srq, ibsrq);
+}
+
+static inline struct hns_roce_sqp *hr_to_hr_sqp(struct hns_roce_qp *hr_qp)
+{
+	return container_of(hr_qp, struct hns_roce_sqp, hr_qp);
+}
+
+static inline void hns_roce_write64_k(__be32 val[2], void __iomem *dest)
+{
+	__raw_writeq(*(u64 *) val, dest);
+}
+
+static inline struct hns_roce_qp
+	*__hns_roce_qp_lookup(struct hns_roce_dev *hr_dev, u32 qpn)
+{
+	return radix_tree_lookup(&hr_dev->qp_table_tree,
+				 qpn & (hr_dev->caps.num_qps - 1));
+}
+
+static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset)
+{
+	u32 bits_per_long_val = BITS_PER_LONG;
+
+	if (bits_per_long_val == 64 || buf->nbufs == 1)
+		return (char *)(buf->direct.buf) + offset;
+	else
+		return (char *)(buf->page_list[offset >> PAGE_SHIFT].buf) +
+		       (offset & (PAGE_SIZE - 1));
+}
+
+int hns_roce_init_uar_table(struct hns_roce_dev *dev);
+int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
+void hns_roce_uar_free(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
+void hns_roce_cleanup_uar_table(struct hns_roce_dev *dev);
+
+int hns_roce_cmd_init(struct hns_roce_dev *hr_dev);
+void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev);
+void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,
+			u64 out_param);
+int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev);
+void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev);
+
+int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
+		      struct hns_roce_mtt *mtt);
+void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev,
+			  struct hns_roce_mtt *mtt);
+int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
+			   struct hns_roce_mtt *mtt, struct hns_roce_buf *buf);
+
+int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);
+int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);
+int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev);
+int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev);
+int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev);
+
+void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev);
+void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev);
+void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev);
+void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev);
+void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev);
+
+int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj);
+void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj);
+int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask,
+			 u32 reserved_bot, u32 resetrved_top);
+void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap);
+void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev);
+int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,
+				int align, unsigned long *obj);
+void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,
+				unsigned long obj, int cnt);
+
+struct ib_ah *hns_roce_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
+int hns_roce_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
+int hns_roce_destroy_ah(struct ib_ah *ah);
+
+struct ib_pd *hns_roce_alloc_pd(struct ib_device *ib_dev,
+				struct ib_ucontext *context,
+				struct ib_udata *udata);
+int hns_roce_dealloc_pd(struct ib_pd *pd);
+
+struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc);
+struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+				   u64 virt_addr, int access_flags,
+				   struct ib_udata *udata);
+int hns_roce_dereg_mr(struct ib_mr *ibmr);
+
+void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,
+		       struct hns_roce_buf *buf);
+int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
+		       struct hns_roce_buf *buf);
+
+int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
+			       struct hns_roce_mtt *mtt, struct ib_umem *umem);
+
+struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd,
+				 struct ib_qp_init_attr *init_attr,
+				 struct ib_udata *udata);
+int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+		       int attr_mask, struct ib_udata *udata);
+void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n);
+void *get_send_wqe(struct hns_roce_qp *hr_qp, int n);
+bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
+			  struct ib_cq *ib_cq);
+enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state);
+void hns_roce_lock_cqs(struct hns_roce_cq *send_cq,
+		       struct hns_roce_cq *recv_cq);
+void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
+			 struct hns_roce_cq *recv_cq);
+void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
+void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
+void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
+			       int cnt);
+__be32 send_ieth(struct ib_send_wr *wr);
+int to_hr_qp_type(int qp_type);
+
+struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
+				    const struct ib_cq_init_attr *attr,
+				    struct ib_ucontext *context,
+				    struct ib_udata *udata);
+
+int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq);
+
+void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);
+void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);
+void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);
+int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
+
+extern struct hns_roce_hw hns_roce_hw_v1;
+
+#endif /* _HNS_ROCE_DEVICE_H */
diff --git a/drivers/infiniband/hw/hns/hns_roce_eq.c b/drivers/infiniband/hw/hns/hns_roce_eq.c
new file mode 100644
index 000000000000..98af7fecf2f1
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_eq.c
@@ -0,0 +1,762 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/platform_device.h>
+#include "hns_roce_common.h"
+#include "hns_roce_device.h"
+#include "hns_roce_eq.h"
+
+static void eq_set_cons_index(struct hns_roce_eq *eq, int req_not)
+{
+	roce_raw_write((eq->cons_index & CONS_INDEX_MASK) |
+		      (req_not << eq->log_entries), eq->doorbell);
+	/* Memory barrier */
+	mb();
+}
+
+static struct hns_roce_aeqe *get_aeqe(struct hns_roce_eq *eq, u32 entry)
+{
+	unsigned long off = (entry & (eq->entries - 1)) *
+			     HNS_ROCE_AEQ_ENTRY_SIZE;
+
+	return (struct hns_roce_aeqe *)((u8 *)
+		(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
+		off % HNS_ROCE_BA_SIZE);
+}
+
+static struct hns_roce_aeqe *next_aeqe_sw(struct hns_roce_eq *eq)
+{
+	struct hns_roce_aeqe *aeqe = get_aeqe(eq, eq->cons_index);
+
+	return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^
+		!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
+}
+
+static void hns_roce_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
+					 struct hns_roce_aeqe *aeqe, int qpn)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+
+	qpn = roce_get_field(aeqe->event.qp_event.qp,
+			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
+			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S);
+	dev_warn(dev, "Local Work Queue Catastrophic Error.\n");
+	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
+			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
+	case HNS_ROCE_LWQCE_QPC_ERROR:
+		dev_warn(dev, "QP %d, QPC error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_MTU_ERROR:
+		dev_warn(dev, "QP %d, MTU error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
+		dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
+		dev_warn(dev, "QP %d, WQE addr error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
+		dev_warn(dev, "QP %d, WQE shift error\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_SL_ERROR:
+		dev_warn(dev, "QP %d, SL error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_PORT_ERROR:
+		dev_warn(dev, "QP %d, port error.\n", qpn);
+		break;
+	default:
+		break;
+	}
+
+	hns_roce_qp_event(hr_dev, roce_get_field(aeqe->event.qp_event.qp,
+					HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
+					HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S),
+			  roce_get_field(aeqe->asyn,
+					HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+					HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
+}
+
+static void hns_roce_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,
+						struct hns_roce_aeqe *aeqe,
+						int qpn)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+
+	qpn = roce_get_field(aeqe->event.qp_event.qp,
+			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
+			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S);
+	dev_warn(dev, "Local Access Violation Work Queue Error.\n");
+	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
+			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
+	case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
+		dev_warn(dev, "QP %d, R_key violation.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_LENGTH_ERROR:
+		dev_warn(dev, "QP %d, length error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_VA_ERROR:
+		dev_warn(dev, "QP %d, VA error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_PD_ERROR:
+		dev_err(dev, "QP %d, PD error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
+		dev_warn(dev, "QP %d, rw acc error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
+		dev_warn(dev, "QP %d, key state error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
+		dev_warn(dev, "QP %d, MR operation error.\n", qpn);
+		break;
+	default:
+		break;
+	}
+
+	hns_roce_qp_event(hr_dev, roce_get_field(aeqe->event.qp_event.qp,
+					 HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
+					 HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S),
+			  roce_get_field(aeqe->asyn,
+					 HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+					 HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
+}
+
+static void hns_roce_db_overflow_handle(struct hns_roce_dev *hr_dev,
+					struct hns_roce_aeqe *aeqe)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+
+	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
+			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
+	case HNS_ROCE_DB_SUBTYPE_SDB_OVF:
+		dev_warn(dev, "SDB overflow.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF:
+		dev_warn(dev, "SDB almost overflow.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP:
+		dev_warn(dev, "SDB almost empty.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_ODB_OVF:
+		dev_warn(dev, "ODB overflow.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF:
+		dev_warn(dev, "ODB almost overflow.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP:
+		dev_warn(dev, "SDB almost empty.\n");
+		break;
+	default:
+		break;
+	}
+}
+
+static int hns_roce_aeq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_aeqe *aeqe;
+	int aeqes_found = 0;
+	int qpn = 0;
+
+	while ((aeqe = next_aeqe_sw(eq))) {
+		dev_dbg(dev, "aeqe = %p, aeqe->asyn.event_type = 0x%lx\n", aeqe,
+			roce_get_field(aeqe->asyn,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
+		/* Memory barrier */
+		rmb();
+
+		switch (roce_get_field(aeqe->asyn,
+			HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+			HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S)) {
+		case HNS_ROCE_EVENT_TYPE_PATH_MIG:
+			dev_warn(dev, "PATH MIG not supported\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_COMM_EST:
+			dev_warn(dev, "COMMUNICATION established\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
+			dev_warn(dev, "SQ DRAINED not supported\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
+			dev_warn(dev, "PATH MIG failed\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
+			dev_warn(dev, "qpn = 0x%lx\n",
+			roce_get_field(aeqe->event.qp_event.qp,
+				       HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
+				       HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S));
+			hns_roce_qp_event(hr_dev,
+				roce_get_field(aeqe->event.qp_event.qp,
+					HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
+					HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S),
+				roce_get_field(aeqe->asyn,
+					HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+					HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
+			break;
+		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+			hns_roce_wq_catas_err_handle(hr_dev, aeqe, qpn);
+			break;
+		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
+			hns_roce_local_wq_access_err_handle(hr_dev, aeqe, qpn);
+			break;
+		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
+		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
+		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
+			dev_warn(dev, "SRQ not support!\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
+			dev_warn(dev, "CQ 0x%lx access err.\n",
+			roce_get_field(aeqe->event.cq_event.cq,
+				       HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
+				       HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S));
+			hns_roce_cq_event(hr_dev,
+			le32_to_cpu(roce_get_field(aeqe->event.cq_event.cq,
+				    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
+				    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S)),
+			roce_get_field(aeqe->asyn,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
+			break;
+		case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
+			dev_warn(dev, "CQ 0x%lx overflow\n",
+			roce_get_field(aeqe->event.cq_event.cq,
+				       HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
+				       HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S));
+			hns_roce_cq_event(hr_dev,
+			le32_to_cpu(roce_get_field(aeqe->event.cq_event.cq,
+				    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
+				    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S)),
+			roce_get_field(aeqe->asyn,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
+			break;
+		case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
+			dev_warn(dev, "CQ ID invalid.\n");
+			hns_roce_cq_event(hr_dev,
+			le32_to_cpu(roce_get_field(aeqe->event.cq_event.cq,
+				    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
+				    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S)),
+			roce_get_field(aeqe->asyn,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
+			break;
+		case HNS_ROCE_EVENT_TYPE_PORT_CHANGE:
+			dev_warn(dev, "port change.\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_MB:
+			hns_roce_cmd_event(hr_dev,
+					   le16_to_cpu(aeqe->event.cmd.token),
+					   aeqe->event.cmd.status,
+					   le64_to_cpu(aeqe->event.cmd.out_param
+					   ));
+			break;
+		case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
+			hns_roce_db_overflow_handle(hr_dev, aeqe);
+			break;
+		case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
+			dev_warn(dev, "CEQ 0x%lx overflow.\n",
+			roce_get_field(aeqe->event.ce_event.ceqe,
+				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M,
+				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S));
+			break;
+		default:
+			dev_warn(dev, "Unhandled event 0x%lx on EQ %d at index %u\n",
+				 roce_get_field(aeqe->asyn,
+					      HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+					      HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S),
+				 eq->eqn, eq->cons_index);
+			break;
+		};
+
+		eq->cons_index++;
+		aeqes_found = 1;
+
+		if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) {
+			dev_warn(dev, "cons_index overflow, set back to zero\n"
+				);
+			eq->cons_index = 0;
+		}
+	}
+
+	eq_set_cons_index(eq, 0);
+
+	return aeqes_found;
+}
+
+static struct hns_roce_ceqe *get_ceqe(struct hns_roce_eq *eq, u32 entry)
+{
+	unsigned long off = (entry & (eq->entries - 1)) *
+			     HNS_ROCE_CEQ_ENTRY_SIZE;
+
+	return (struct hns_roce_ceqe *)((u8 *)
+			(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
+			off % HNS_ROCE_BA_SIZE);
+}
+
+static struct hns_roce_ceqe *next_ceqe_sw(struct hns_roce_eq *eq)
+{
+	struct hns_roce_ceqe *ceqe = get_ceqe(eq, eq->cons_index);
+
+	return (!!(roce_get_bit(ceqe->ceqe.comp,
+		 HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^
+		 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
+}
+
+static int hns_roce_ceq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
+{
+	struct hns_roce_ceqe *ceqe;
+	int ceqes_found = 0;
+	u32 cqn;
+
+	while ((ceqe = next_ceqe_sw(eq))) {
+		/* Memory barrier */
+		rmb();
+		cqn = roce_get_field(ceqe->ceqe.comp,
+				     HNS_ROCE_CEQE_CEQE_COMP_CQN_M,
+				     HNS_ROCE_CEQE_CEQE_COMP_CQN_S);
+		hns_roce_cq_completion(hr_dev, cqn);
+
+		++eq->cons_index;
+		ceqes_found = 1;
+
+		if (eq->cons_index > 2 * hr_dev->caps.ceqe_depth[eq->eqn] - 1) {
+			dev_warn(&eq->hr_dev->pdev->dev,
+				"cons_index overflow, set back to zero\n");
+			eq->cons_index = 0;
+		}
+	}
+
+	eq_set_cons_index(eq, 0);
+
+	return ceqes_found;
+}
+
+static int hns_roce_aeq_ovf_int(struct hns_roce_dev *hr_dev,
+				struct hns_roce_eq *eq)
+{
+	struct device *dev = &eq->hr_dev->pdev->dev;
+	int eqovf_found = 0;
+	u32 caepaemask_val;
+	u32 cealmovf_val;
+	u32 caepaest_val;
+	u32 aeshift_val;
+	u32 ceshift_val;
+	u32 cemask_val;
+	int i = 0;
+
+	/**
+	* AEQ overflow ECC mult bit err CEQ overflow alarm
+	* must clear interrupt, mask irq, clear irq, cancel mask operation
+	*/
+	aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG);
+
+	if (roce_get_bit(aeshift_val,
+		ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) {
+		dev_warn(dev, "AEQ overflow!\n");
+
+		/* Set mask */
+		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
+		roce_set_bit(caepaemask_val,
+			     ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
+			     HNS_ROCE_INT_MASK_ENABLE);
+		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
+
+		/* Clear int state(INT_WC : write 1 clear) */
+		caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG);
+		roce_set_bit(caepaest_val,
+			     ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1);
+		roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val);
+
+		/* Clear mask */
+		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
+		roce_set_bit(caepaemask_val,
+			     ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
+			     HNS_ROCE_INT_MASK_DISABLE);
+		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
+	}
+
+	/* CEQ almost overflow */
+	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
+		ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG +
+					i * CEQ_REG_OFFSET);
+
+		if (roce_get_bit(ceshift_val,
+		ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) {
+			dev_warn(dev, "CEQ[%d] almost overflow!\n", i);
+			eqovf_found++;
+
+			/* Set mask */
+			cemask_val = roce_read(hr_dev,
+					       ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+					       i * CEQ_REG_OFFSET);
+			roce_set_bit(cemask_val,
+				ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
+				HNS_ROCE_INT_MASK_ENABLE);
+			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+				   i * CEQ_REG_OFFSET, cemask_val);
+
+			/* Clear int state(INT_WC : write 1 clear) */
+			cealmovf_val = roce_read(hr_dev,
+				       ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
+				       i * CEQ_REG_OFFSET);
+			roce_set_bit(cealmovf_val,
+				     ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S,
+				     1);
+			roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
+				    i * CEQ_REG_OFFSET, cealmovf_val);
+
+			/* Clear mask */
+			cemask_val = roce_read(hr_dev,
+				     ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+				     i * CEQ_REG_OFFSET);
+			roce_set_bit(cemask_val,
+			       ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
+			       HNS_ROCE_INT_MASK_DISABLE);
+			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+				   i * CEQ_REG_OFFSET, cemask_val);
+		}
+	}
+
+	/* ECC multi-bit error alarm */
+	dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n",
+		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG),
+		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG),
+		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG));
+
+	dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n",
+		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG),
+		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG),
+		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG));
+
+	return eqovf_found;
+}
+
+static int hns_roce_eq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
+{
+	int eqes_found = 0;
+
+	if (likely(eq->type_flag == HNS_ROCE_CEQ))
+		/* CEQ irq routine, CEQ is pulse irq, not clear */
+		eqes_found = hns_roce_ceq_int(hr_dev, eq);
+	else if (likely(eq->type_flag == HNS_ROCE_AEQ))
+		/* AEQ irq routine, AEQ is pulse irq, not clear */
+		eqes_found = hns_roce_aeq_int(hr_dev, eq);
+	else
+		/* AEQ queue overflow irq */
+		eqes_found = hns_roce_aeq_ovf_int(hr_dev, eq);
+
+	return eqes_found;
+}
+
+static irqreturn_t hns_roce_msi_x_interrupt(int irq, void *eq_ptr)
+{
+	int int_work = 0;
+	struct hns_roce_eq  *eq  = eq_ptr;
+	struct hns_roce_dev *hr_dev = eq->hr_dev;
+
+	int_work = hns_roce_eq_int(hr_dev, eq);
+
+	return IRQ_RETVAL(int_work);
+}
+
+static void hns_roce_enable_eq(struct hns_roce_dev *hr_dev, int eq_num,
+			       int enable_flag)
+{
+	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num];
+	u32 val;
+
+	val = readl(eqc);
+
+	if (enable_flag)
+		roce_set_field(val,
+			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
+			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
+			       HNS_ROCE_EQ_STAT_VALID);
+	else
+		roce_set_field(val,
+			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
+			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
+			       HNS_ROCE_EQ_STAT_INVALID);
+	writel(val, eqc);
+}
+
+static int hns_roce_create_eq(struct hns_roce_dev *hr_dev,
+			      struct hns_roce_eq *eq)
+{
+	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn];
+	struct device *dev = &hr_dev->pdev->dev;
+	dma_addr_t tmp_dma_addr;
+	u32 eqconsindx_val = 0;
+	u32 eqcuridx_val = 0;
+	u32 eqshift_val = 0;
+	int num_bas = 0;
+	int ret;
+	int i;
+
+	num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) +
+		   HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;
+
+	if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) {
+		dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n",
+			(eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE,
+			num_bas);
+		return -EINVAL;
+	}
+
+	eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL);
+	if (!eq->buf_list)
+		return -ENOMEM;
+
+	for (i = 0; i < num_bas; ++i) {
+		eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE,
+							 &tmp_dma_addr,
+							 GFP_KERNEL);
+		if (!eq->buf_list[i].buf) {
+			ret = -ENOMEM;
+			goto err_out_free_pages;
+		}
+
+		eq->buf_list[i].map = tmp_dma_addr;
+		memset(eq->buf_list[i].buf, 0, HNS_ROCE_BA_SIZE);
+	}
+	eq->cons_index = 0;
+	roce_set_field(eqshift_val,
+		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
+		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
+		       HNS_ROCE_EQ_STAT_INVALID);
+	roce_set_field(eqshift_val,
+		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M,
+		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S,
+		       eq->log_entries);
+	writel(eqshift_val, eqc);
+
+	/* Configure eq extended address 12~44bit */
+	writel((u32)(eq->buf_list[0].map >> 12), (u8 *)eqc + 4);
+
+	/*
+	 * Configure eq extended address 45~49 bit.
+	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
+	 * using 4K page, and shift more 32 because of
+	 * caculating the high 32 bit value evaluated to hardware.
+	 */
+	roce_set_field(eqcuridx_val, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M,
+		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S,
+		       eq->buf_list[0].map >> 44);
+	roce_set_field(eqcuridx_val,
+		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M,
+		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0);
+	writel(eqcuridx_val, (u8 *)eqc + 8);
+
+	/* Configure eq consumer index */
+	roce_set_field(eqconsindx_val,
+		       ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M,
+		       ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0);
+	writel(eqconsindx_val, (u8 *)eqc + 0xc);
+
+	return 0;
+
+err_out_free_pages:
+	for (i = i - 1; i >= 0; i--)
+		dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf,
+				  eq->buf_list[i].map);
+
+	kfree(eq->buf_list);
+	return ret;
+}
+
+static void hns_roce_free_eq(struct hns_roce_dev *hr_dev,
+			     struct hns_roce_eq *eq)
+{
+	int i = 0;
+	int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) +
+		      HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;
+
+	if (!eq->buf_list)
+		return;
+
+	for (i = 0; i < npages; ++i)
+		dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE,
+				  eq->buf_list[i].buf, eq->buf_list[i].map);
+
+	kfree(eq->buf_list);
+}
+
+static void hns_roce_int_mask_en(struct hns_roce_dev *hr_dev)
+{
+	int i = 0;
+	u32 aemask_val;
+	int masken = 0;
+
+	/* AEQ INT */
+	aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
+	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
+		     masken);
+	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken);
+	roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val);
+
+	/* CEQ INT */
+	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
+		/* IRQ mask */
+		roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+			   i * CEQ_REG_OFFSET, masken);
+	}
+}
+
+static void hns_roce_ce_int_default_cfg(struct hns_roce_dev *hr_dev)
+{
+	/* Configure ce int interval */
+	roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG,
+		   HNS_ROCE_CEQ_DEFAULT_INTERVAL);
+
+	/* Configure ce int burst num */
+	roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG,
+		   HNS_ROCE_CEQ_DEFAULT_BURST_NUM);
+}
+
+int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_eq *eq = NULL;
+	int eq_num = 0;
+	int ret = 0;
+	int i = 0;
+	int j = 0;
+
+	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
+	eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
+	if (!eq_table->eq)
+		return -ENOMEM;
+
+	eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base),
+				     GFP_KERNEL);
+	if (!eq_table->eqc_base) {
+		ret = -ENOMEM;
+		goto err_eqc_base_alloc_fail;
+	}
+
+	for (i = 0; i < eq_num; i++) {
+		eq = &eq_table->eq[i];
+		eq->hr_dev = hr_dev;
+		eq->eqn = i;
+		eq->irq = hr_dev->irq[i];
+		eq->log_page_size = PAGE_SHIFT;
+
+		if (i < hr_dev->caps.num_comp_vectors) {
+			/* CEQ */
+			eq_table->eqc_base[i] = hr_dev->reg_base +
+						ROCEE_CAEP_CEQC_SHIFT_0_REG +
+						HNS_ROCE_CEQC_REG_OFFSET * i;
+			eq->type_flag = HNS_ROCE_CEQ;
+			eq->doorbell = hr_dev->reg_base +
+				       ROCEE_CAEP_CEQC_CONS_IDX_0_REG +
+				       HNS_ROCE_CEQC_REG_OFFSET * i;
+			eq->entries = hr_dev->caps.ceqe_depth[i];
+			eq->log_entries = ilog2(eq->entries);
+			eq->eqe_size = sizeof(struct hns_roce_ceqe);
+		} else {
+			/* AEQ */
+			eq_table->eqc_base[i] = hr_dev->reg_base +
+						ROCEE_CAEP_AEQC_AEQE_SHIFT_REG;
+			eq->type_flag = HNS_ROCE_AEQ;
+			eq->doorbell = hr_dev->reg_base +
+				       ROCEE_CAEP_AEQE_CONS_IDX_REG;
+			eq->entries = hr_dev->caps.aeqe_depth;
+			eq->log_entries = ilog2(eq->entries);
+			eq->eqe_size = sizeof(struct hns_roce_aeqe);
+		}
+	}
+
+	/* Disable irq */
+	hns_roce_int_mask_en(hr_dev);
+
+	/* Configure CE irq interval and burst num */
+	hns_roce_ce_int_default_cfg(hr_dev);
+
+	for (i = 0; i < eq_num; i++) {
+		ret = hns_roce_create_eq(hr_dev, &eq_table->eq[i]);
+		if (ret) {
+			dev_err(dev, "eq create failed\n");
+			goto err_create_eq_fail;
+		}
+	}
+
+	for (j = 0; j < eq_num; j++) {
+		ret = request_irq(eq_table->eq[j].irq, hns_roce_msi_x_interrupt,
+				  0, hr_dev->irq_names[j], eq_table->eq + j);
+		if (ret) {
+			dev_err(dev, "request irq error!\n");
+			goto err_request_irq_fail;
+		}
+	}
+
+	for (i = 0; i < eq_num; i++)
+		hns_roce_enable_eq(hr_dev, i, EQ_ENABLE);
+
+	return 0;
+
+err_request_irq_fail:
+	for (j = j - 1; j >= 0; j--)
+		free_irq(eq_table->eq[j].irq, eq_table->eq + j);
+
+err_create_eq_fail:
+	for (i = i - 1; i >= 0; i--)
+		hns_roce_free_eq(hr_dev, &eq_table->eq[i]);
+
+	kfree(eq_table->eqc_base);
+
+err_eqc_base_alloc_fail:
+	kfree(eq_table->eq);
+
+	return ret;
+}
+
+void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev)
+{
+	int i;
+	int eq_num;
+	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
+
+	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
+	for (i = 0; i < eq_num; i++) {
+		/* Disable EQ */
+		hns_roce_enable_eq(hr_dev, i, EQ_DISABLE);
+
+		free_irq(eq_table->eq[i].irq, eq_table->eq + i);
+
+		hns_roce_free_eq(hr_dev, &eq_table->eq[i]);
+	}
+
+	kfree(eq_table->eqc_base);
+	kfree(eq_table->eq);
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_eq.h b/drivers/infiniband/hw/hns/hns_roce_eq.h
new file mode 100644
index 000000000000..fe4388191a3c
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_eq.h
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _HNS_ROCE_EQ_H
+#define _HNS_ROCE_EQ_H
+
+#define HNS_ROCE_CEQ			1
+#define HNS_ROCE_AEQ			2
+
+#define HNS_ROCE_CEQ_ENTRY_SIZE		0x4
+#define HNS_ROCE_AEQ_ENTRY_SIZE		0x10
+#define HNS_ROCE_CEQC_REG_OFFSET	0x18
+
+#define HNS_ROCE_CEQ_DEFAULT_INTERVAL	0x10
+#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM	0x10
+
+#define HNS_ROCE_INT_MASK_DISABLE	0
+#define HNS_ROCE_INT_MASK_ENABLE	1
+
+#define EQ_ENABLE			1
+#define EQ_DISABLE			0
+#define CONS_INDEX_MASK			0xffff
+
+#define CEQ_REG_OFFSET			0x18
+
+enum {
+	HNS_ROCE_EQ_STAT_INVALID  = 0,
+	HNS_ROCE_EQ_STAT_VALID    = 2,
+};
+
+struct hns_roce_aeqe {
+	u32 asyn;
+	union {
+		struct {
+			u32 qp;
+			u32 rsv0;
+			u32 rsv1;
+		} qp_event;
+
+		struct {
+			u32 cq;
+			u32 rsv0;
+			u32 rsv1;
+		} cq_event;
+
+		struct {
+			u32 port;
+			u32 rsv0;
+			u32 rsv1;
+		} port_event;
+
+		struct {
+			u32 ceqe;
+			u32 rsv0;
+			u32 rsv1;
+		} ce_event;
+
+		struct {
+			__le64  out_param;
+			__le16  token;
+			u8	status;
+			u8	rsv0;
+		} __packed cmd;
+	 } event;
+};
+
+#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S 16
+#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M   \
+	(((1UL << 8) - 1) << HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S)
+
+#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S 24
+#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M   \
+	(((1UL << 7) - 1) << HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)
+
+#define HNS_ROCE_AEQE_U32_4_OWNER_S 31
+
+#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S 0
+#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M   \
+	(((1UL << 24) - 1) << HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S)
+
+#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S 0
+#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M   \
+	(((1UL << 16) - 1) << HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S)
+
+#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S 0
+#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M   \
+	(((1UL << 5) - 1) << HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S)
+
+struct hns_roce_ceqe {
+	union {
+		int		comp;
+	} ceqe;
+};
+
+#define HNS_ROCE_CEQE_CEQE_COMP_OWNER_S	0
+
+#define HNS_ROCE_CEQE_CEQE_COMP_CQN_S 16
+#define HNS_ROCE_CEQE_CEQE_COMP_CQN_M   \
+	(((1UL << 16) - 1) << HNS_ROCE_CEQE_CEQE_COMP_CQN_S)
+
+#endif /* _HNS_ROCE_EQ_H */
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c
new file mode 100644
index 000000000000..d53d64362389
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.c
@@ -0,0 +1,476 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/platform_device.h>
+#include "hns_roce_device.h"
+#include "hns_roce_hem.h"
+#include "hns_roce_common.h"
+
+#define HW_SYNC_TIMEOUT_MSECS		500
+#define HW_SYNC_SLEEP_TIME_INTERVAL	20
+
+#define HNS_ROCE_HEM_ALLOC_SIZE		(1 << 17)
+#define HNS_ROCE_TABLE_CHUNK_SIZE	(1 << 17)
+
+#define DMA_ADDR_T_SHIFT		12
+#define BT_CMD_SYNC_SHIFT		31
+#define BT_BA_SHIFT			32
+
+struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages,
+					gfp_t gfp_mask)
+{
+	struct hns_roce_hem_chunk *chunk = NULL;
+	struct hns_roce_hem *hem;
+	struct scatterlist *mem;
+	int order;
+	void *buf;
+
+	WARN_ON(gfp_mask & __GFP_HIGHMEM);
+
+	hem = kmalloc(sizeof(*hem),
+		      gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
+	if (!hem)
+		return NULL;
+
+	hem->refcount = 0;
+	INIT_LIST_HEAD(&hem->chunk_list);
+
+	order = get_order(HNS_ROCE_HEM_ALLOC_SIZE);
+
+	while (npages > 0) {
+		if (!chunk) {
+			chunk = kmalloc(sizeof(*chunk),
+				gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
+			if (!chunk)
+				goto fail;
+
+			sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
+			chunk->npages = 0;
+			chunk->nsg = 0;
+			list_add_tail(&chunk->list, &hem->chunk_list);
+		}
+
+		while (1 << order > npages)
+			--order;
+
+		/*
+		* Alloc memory one time. If failed, don't alloc small block
+		* memory, directly return fail.
+		*/
+		mem = &chunk->mem[chunk->npages];
+		buf = dma_alloc_coherent(&hr_dev->pdev->dev, PAGE_SIZE << order,
+				&sg_dma_address(mem), gfp_mask);
+		if (!buf)
+			goto fail;
+
+		sg_set_buf(mem, buf, PAGE_SIZE << order);
+		WARN_ON(mem->offset);
+		sg_dma_len(mem) = PAGE_SIZE << order;
+
+		++chunk->npages;
+		++chunk->nsg;
+		npages -= 1 << order;
+	}
+
+	return hem;
+
+fail:
+	hns_roce_free_hem(hr_dev, hem);
+	return NULL;
+}
+
+void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
+{
+	struct hns_roce_hem_chunk *chunk, *tmp;
+	int i;
+
+	if (!hem)
+		return;
+
+	list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
+		for (i = 0; i < chunk->npages; ++i)
+			dma_free_coherent(&hr_dev->pdev->dev,
+				   chunk->mem[i].length,
+				   lowmem_page_address(sg_page(&chunk->mem[i])),
+				   sg_dma_address(&chunk->mem[i]));
+		kfree(chunk);
+	}
+
+	kfree(hem);
+}
+
+static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
+			    struct hns_roce_hem_table *table, unsigned long obj)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	spinlock_t *lock = &hr_dev->bt_cmd_lock;
+	unsigned long end = 0;
+	unsigned long flags;
+	struct hns_roce_hem_iter iter;
+	void __iomem *bt_cmd;
+	u32 bt_cmd_h_val = 0;
+	u32 bt_cmd_val[2];
+	u32 bt_cmd_l = 0;
+	u64 bt_ba = 0;
+	int ret = 0;
+
+	/* Find the HEM(Hardware Entry Memory) entry */
+	unsigned long i = (obj & (table->num_obj - 1)) /
+			  (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size);
+
+	switch (table->type) {
+	case HEM_TYPE_QPC:
+		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
+			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
+		break;
+	case HEM_TYPE_MTPT:
+		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
+			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
+			       HEM_TYPE_MTPT);
+		break;
+	case HEM_TYPE_CQC:
+		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
+			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
+		break;
+	case HEM_TYPE_SRQC:
+		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
+			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
+			       HEM_TYPE_SRQC);
+		break;
+	default:
+		return ret;
+	}
+	roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
+		       ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
+	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
+	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
+
+	/* Currently iter only a chunk */
+	for (hns_roce_hem_first(table->hem[i], &iter);
+	     !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
+		bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
+
+		spin_lock_irqsave(lock, flags);
+
+		bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
+
+		end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
+		while (1) {
+			if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
+				if (!(time_before(jiffies, end))) {
+					dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
+					spin_unlock_irqrestore(lock, flags);
+					return -EBUSY;
+				}
+			} else {
+				break;
+			}
+			msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
+		}
+
+		bt_cmd_l = (u32)bt_ba;
+		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
+			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
+			       bt_ba >> BT_BA_SHIFT);
+
+		bt_cmd_val[0] = bt_cmd_l;
+		bt_cmd_val[1] = bt_cmd_h_val;
+		hns_roce_write64_k(bt_cmd_val,
+				   hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
+		spin_unlock_irqrestore(lock, flags);
+	}
+
+	return ret;
+}
+
+static int hns_roce_clear_hem(struct hns_roce_dev *hr_dev,
+			      struct hns_roce_hem_table *table,
+			      unsigned long obj)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	unsigned long end = 0;
+	unsigned long flags;
+	void __iomem *bt_cmd;
+	uint32_t bt_cmd_val[2];
+	u32 bt_cmd_h_val = 0;
+	int ret = 0;
+
+	switch (table->type) {
+	case HEM_TYPE_QPC:
+		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
+			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
+		break;
+	case HEM_TYPE_MTPT:
+		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
+			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
+			       HEM_TYPE_MTPT);
+		break;
+	case HEM_TYPE_CQC:
+		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
+			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
+		break;
+	case HEM_TYPE_SRQC:
+		roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
+			       ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
+			       HEM_TYPE_SRQC);
+		break;
+	default:
+		return ret;
+	}
+	roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
+		       ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
+	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
+	roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
+	roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
+		       ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, 0);
+
+	spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags);
+
+	bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
+
+	end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
+	while (1) {
+		if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
+			if (!(time_before(jiffies, end))) {
+				dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
+				spin_unlock_irqrestore(&hr_dev->bt_cmd_lock,
+						       flags);
+				return -EBUSY;
+			}
+		} else {
+			break;
+		}
+		msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
+	}
+
+	bt_cmd_val[0] = 0;
+	bt_cmd_val[1] = bt_cmd_h_val;
+	hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
+	spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags);
+
+	return ret;
+}
+
+int hns_roce_table_get(struct hns_roce_dev *hr_dev,
+		       struct hns_roce_hem_table *table, unsigned long obj)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	int ret = 0;
+	unsigned long i;
+
+	i = (obj & (table->num_obj - 1)) / (HNS_ROCE_TABLE_CHUNK_SIZE /
+	     table->obj_size);
+
+	mutex_lock(&table->mutex);
+
+	if (table->hem[i]) {
+		++table->hem[i]->refcount;
+		goto out;
+	}
+
+	table->hem[i] = hns_roce_alloc_hem(hr_dev,
+				       HNS_ROCE_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
+				       (table->lowmem ? GFP_KERNEL :
+					GFP_HIGHUSER) | __GFP_NOWARN);
+	if (!table->hem[i]) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	/* Set HEM base address(128K/page, pa) to Hardware */
+	if (hns_roce_set_hem(hr_dev, table, obj)) {
+		ret = -ENODEV;
+		dev_err(dev, "set HEM base address to HW failed.\n");
+		goto out;
+	}
+
+	++table->hem[i]->refcount;
+out:
+	mutex_unlock(&table->mutex);
+	return ret;
+}
+
+void hns_roce_table_put(struct hns_roce_dev *hr_dev,
+			struct hns_roce_hem_table *table, unsigned long obj)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	unsigned long i;
+
+	i = (obj & (table->num_obj - 1)) /
+	    (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size);
+
+	mutex_lock(&table->mutex);
+
+	if (--table->hem[i]->refcount == 0) {
+		/* Clear HEM base address */
+		if (hns_roce_clear_hem(hr_dev, table, obj))
+			dev_warn(dev, "Clear HEM base address failed.\n");
+
+		hns_roce_free_hem(hr_dev, table->hem[i]);
+		table->hem[i] = NULL;
+	}
+
+	mutex_unlock(&table->mutex);
+}
+
+void *hns_roce_table_find(struct hns_roce_hem_table *table, unsigned long obj,
+			  dma_addr_t *dma_handle)
+{
+	struct hns_roce_hem_chunk *chunk;
+	unsigned long idx;
+	int i;
+	int offset, dma_offset;
+	struct hns_roce_hem *hem;
+	struct page *page = NULL;
+
+	if (!table->lowmem)
+		return NULL;
+
+	mutex_lock(&table->mutex);
+	idx = (obj & (table->num_obj - 1)) * table->obj_size;
+	hem = table->hem[idx / HNS_ROCE_TABLE_CHUNK_SIZE];
+	dma_offset = offset = idx % HNS_ROCE_TABLE_CHUNK_SIZE;
+
+	if (!hem)
+		goto out;
+
+	list_for_each_entry(chunk, &hem->chunk_list, list) {
+		for (i = 0; i < chunk->npages; ++i) {
+			if (dma_handle && dma_offset >= 0) {
+				if (sg_dma_len(&chunk->mem[i]) >
+				    (u32)dma_offset)
+					*dma_handle = sg_dma_address(
+						&chunk->mem[i]) + dma_offset;
+				dma_offset -= sg_dma_len(&chunk->mem[i]);
+			}
+
+			if (chunk->mem[i].length > (u32)offset) {
+				page = sg_page(&chunk->mem[i]);
+				goto out;
+			}
+			offset -= chunk->mem[i].length;
+		}
+	}
+
+out:
+	mutex_unlock(&table->mutex);
+	return page ? lowmem_page_address(page) + offset : NULL;
+}
+
+int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
+			     struct hns_roce_hem_table *table,
+			     unsigned long start, unsigned long end)
+{
+	unsigned long inc = HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size;
+	unsigned long i = 0;
+	int ret = 0;
+
+	/* Allocate MTT entry memory according to chunk(128K) */
+	for (i = start; i <= end; i += inc) {
+		ret = hns_roce_table_get(hr_dev, table, i);
+		if (ret)
+			goto fail;
+	}
+
+	return 0;
+
+fail:
+	while (i > start) {
+		i -= inc;
+		hns_roce_table_put(hr_dev, table, i);
+	}
+	return ret;
+}
+
+void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
+			      struct hns_roce_hem_table *table,
+			      unsigned long start, unsigned long end)
+{
+	unsigned long i;
+
+	for (i = start; i <= end;
+		i += HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size)
+		hns_roce_table_put(hr_dev, table, i);
+}
+
+int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
+			    struct hns_roce_hem_table *table, u32 type,
+			    unsigned long obj_size, unsigned long nobj,
+			    int use_lowmem)
+{
+	unsigned long obj_per_chunk;
+	unsigned long num_hem;
+
+	obj_per_chunk = HNS_ROCE_TABLE_CHUNK_SIZE / obj_size;
+	num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
+
+	table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
+	if (!table->hem)
+		return -ENOMEM;
+
+	table->type = type;
+	table->num_hem = num_hem;
+	table->num_obj = nobj;
+	table->obj_size = obj_size;
+	table->lowmem = use_lowmem;
+	mutex_init(&table->mutex);
+
+	return 0;
+}
+
+void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
+				struct hns_roce_hem_table *table)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	unsigned long i;
+
+	for (i = 0; i < table->num_hem; ++i)
+		if (table->hem[i]) {
+			if (hns_roce_clear_hem(hr_dev, table,
+			    i * HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size))
+				dev_err(dev, "Clear HEM base address failed.\n");
+
+			hns_roce_free_hem(hr_dev, table->hem[i]);
+		}
+
+	kfree(table->hem);
+}
+
+void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
+{
+	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
+	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
+	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
+	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
+	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h
new file mode 100644
index 000000000000..ad6617588fba
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _HNS_ROCE_HEM_H
+#define _HNS_ROCE_HEM_H
+
+enum {
+	/* MAP HEM(Hardware Entry Memory) */
+	HEM_TYPE_QPC = 0,
+	HEM_TYPE_MTPT,
+	HEM_TYPE_CQC,
+	HEM_TYPE_SRQC,
+
+	 /* UNMAP HEM */
+	HEM_TYPE_MTT,
+	HEM_TYPE_IRRL,
+};
+
+#define HNS_ROCE_HEM_CHUNK_LEN	\
+	 ((256 - sizeof(struct list_head) - 2 * sizeof(int)) /	 \
+	 (sizeof(struct scatterlist)))
+
+enum {
+	 HNS_ROCE_HEM_PAGE_SHIFT = 12,
+	 HNS_ROCE_HEM_PAGE_SIZE  = 1 << HNS_ROCE_HEM_PAGE_SHIFT,
+};
+
+struct hns_roce_hem_chunk {
+	struct list_head	 list;
+	int			 npages;
+	int			 nsg;
+	struct scatterlist	 mem[HNS_ROCE_HEM_CHUNK_LEN];
+};
+
+struct hns_roce_hem {
+	struct list_head	 chunk_list;
+	int			 refcount;
+};
+
+struct hns_roce_hem_iter {
+	struct hns_roce_hem		 *hem;
+	struct hns_roce_hem_chunk	 *chunk;
+	int				 page_idx;
+};
+
+void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem);
+int hns_roce_table_get(struct hns_roce_dev *hr_dev,
+		       struct hns_roce_hem_table *table, unsigned long obj);
+void hns_roce_table_put(struct hns_roce_dev *hr_dev,
+			struct hns_roce_hem_table *table, unsigned long obj);
+void *hns_roce_table_find(struct hns_roce_hem_table *table, unsigned long obj,
+			  dma_addr_t *dma_handle);
+int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
+			     struct hns_roce_hem_table *table,
+			     unsigned long start, unsigned long end);
+void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
+			      struct hns_roce_hem_table *table,
+			      unsigned long start, unsigned long end);
+int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
+			    struct hns_roce_hem_table *table, u32 type,
+			    unsigned long obj_size, unsigned long nobj,
+			    int use_lowmem);
+void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
+				struct hns_roce_hem_table *table);
+void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev);
+
+static inline void hns_roce_hem_first(struct hns_roce_hem *hem,
+				      struct hns_roce_hem_iter *iter)
+{
+	iter->hem = hem;
+	iter->chunk = list_empty(&hem->chunk_list) ? NULL :
+				 list_entry(hem->chunk_list.next,
+					    struct hns_roce_hem_chunk, list);
+	 iter->page_idx = 0;
+}
+
+static inline int hns_roce_hem_last(struct hns_roce_hem_iter *iter)
+{
+	return !iter->chunk;
+}
+
+static inline void hns_roce_hem_next(struct hns_roce_hem_iter *iter)
+{
+	if (++iter->page_idx >= iter->chunk->nsg) {
+		if (iter->chunk->list.next == &iter->hem->chunk_list) {
+			iter->chunk = NULL;
+			return;
+		}
+
+		iter->chunk = list_entry(iter->chunk->list.next,
+					 struct hns_roce_hem_chunk, list);
+		iter->page_idx = 0;
+	}
+}
+
+static inline dma_addr_t hns_roce_hem_addr(struct hns_roce_hem_iter *iter)
+{
+	return sg_dma_address(&iter->chunk->mem[iter->page_idx]);
+}
+
+#endif /*_HNS_ROCE_HEM_H*/
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
new file mode 100644
index 000000000000..399f5dedaf2d
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
@@ -0,0 +1,2800 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/acpi.h>
+#include <rdma/ib_umem.h>
+#include "hns_roce_common.h"
+#include "hns_roce_device.h"
+#include "hns_roce_cmd.h"
+#include "hns_roce_hem.h"
+#include "hns_roce_hw_v1.h"
+
+static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg)
+{
+	dseg->lkey = cpu_to_le32(sg->lkey);
+	dseg->addr = cpu_to_le64(sg->addr);
+	dseg->len  = cpu_to_le32(sg->length);
+}
+
+static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr,
+			  u32 rkey)
+{
+	rseg->raddr = cpu_to_le64(remote_addr);
+	rseg->rkey  = cpu_to_le32(rkey);
+	rseg->len   = 0;
+}
+
+int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
+			  struct ib_send_wr **bad_wr)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
+	struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL;
+	struct hns_roce_wqe_ctrl_seg *ctrl = NULL;
+	struct hns_roce_wqe_data_seg *dseg = NULL;
+	struct hns_roce_qp *qp = to_hr_qp(ibqp);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_sq_db sq_db;
+	int ps_opcode = 0, i = 0;
+	unsigned long flags = 0;
+	void *wqe = NULL;
+	u32 doorbell[2];
+	int nreq = 0;
+	u32 ind = 0;
+	int ret = 0;
+
+	spin_lock_irqsave(&qp->sq.lock, flags);
+
+	ind = qp->sq_next_wqe;
+	for (nreq = 0; wr; ++nreq, wr = wr->next) {
+		if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
+			ret = -ENOMEM;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		if (unlikely(wr->num_sge > qp->sq.max_gs)) {
+			dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
+				wr->num_sge, qp->sq.max_gs);
+			ret = -EINVAL;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
+		qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
+								      wr->wr_id;
+
+		/* Corresponding to the RC and RD type wqe process separately */
+		if (ibqp->qp_type == IB_QPT_GSI) {
+			ud_sq_wqe = wqe;
+			roce_set_field(ud_sq_wqe->dmac_h,
+				       UD_SEND_WQE_U32_4_DMAC_0_M,
+				       UD_SEND_WQE_U32_4_DMAC_0_S,
+				       ah->av.mac[0]);
+			roce_set_field(ud_sq_wqe->dmac_h,
+				       UD_SEND_WQE_U32_4_DMAC_1_M,
+				       UD_SEND_WQE_U32_4_DMAC_1_S,
+				       ah->av.mac[1]);
+			roce_set_field(ud_sq_wqe->dmac_h,
+				       UD_SEND_WQE_U32_4_DMAC_2_M,
+				       UD_SEND_WQE_U32_4_DMAC_2_S,
+				       ah->av.mac[2]);
+			roce_set_field(ud_sq_wqe->dmac_h,
+				       UD_SEND_WQE_U32_4_DMAC_3_M,
+				       UD_SEND_WQE_U32_4_DMAC_3_S,
+				       ah->av.mac[3]);
+
+			roce_set_field(ud_sq_wqe->u32_8,
+				       UD_SEND_WQE_U32_8_DMAC_4_M,
+				       UD_SEND_WQE_U32_8_DMAC_4_S,
+				       ah->av.mac[4]);
+			roce_set_field(ud_sq_wqe->u32_8,
+				       UD_SEND_WQE_U32_8_DMAC_5_M,
+				       UD_SEND_WQE_U32_8_DMAC_5_S,
+				       ah->av.mac[5]);
+			roce_set_field(ud_sq_wqe->u32_8,
+				       UD_SEND_WQE_U32_8_OPERATION_TYPE_M,
+				       UD_SEND_WQE_U32_8_OPERATION_TYPE_S,
+				       HNS_ROCE_WQE_OPCODE_SEND);
+			roce_set_field(ud_sq_wqe->u32_8,
+				       UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M,
+				       UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S,
+				       2);
+			roce_set_bit(ud_sq_wqe->u32_8,
+				UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S,
+				1);
+
+			ud_sq_wqe->u32_8 |= (wr->send_flags & IB_SEND_SIGNALED ?
+				cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
+				(wr->send_flags & IB_SEND_SOLICITED ?
+				cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
+				((wr->opcode == IB_WR_SEND_WITH_IMM) ?
+				cpu_to_le32(HNS_ROCE_WQE_IMM) : 0);
+
+			roce_set_field(ud_sq_wqe->u32_16,
+				       UD_SEND_WQE_U32_16_DEST_QP_M,
+				       UD_SEND_WQE_U32_16_DEST_QP_S,
+				       ud_wr(wr)->remote_qpn);
+			roce_set_field(ud_sq_wqe->u32_16,
+				       UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M,
+				       UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S,
+				       ah->av.stat_rate);
+
+			roce_set_field(ud_sq_wqe->u32_36,
+				       UD_SEND_WQE_U32_36_FLOW_LABEL_M,
+				       UD_SEND_WQE_U32_36_FLOW_LABEL_S, 0);
+			roce_set_field(ud_sq_wqe->u32_36,
+				       UD_SEND_WQE_U32_36_PRIORITY_M,
+				       UD_SEND_WQE_U32_36_PRIORITY_S,
+				       ah->av.sl_tclass_flowlabel >>
+				       HNS_ROCE_SL_SHIFT);
+			roce_set_field(ud_sq_wqe->u32_36,
+				       UD_SEND_WQE_U32_36_SGID_INDEX_M,
+				       UD_SEND_WQE_U32_36_SGID_INDEX_S,
+				       hns_get_gid_index(hr_dev, qp->port,
+							 ah->av.gid_index));
+
+			roce_set_field(ud_sq_wqe->u32_40,
+				       UD_SEND_WQE_U32_40_HOP_LIMIT_M,
+				       UD_SEND_WQE_U32_40_HOP_LIMIT_S,
+				       ah->av.hop_limit);
+			roce_set_field(ud_sq_wqe->u32_40,
+				       UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M,
+				       UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S, 0);
+
+			memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN);
+
+			ud_sq_wqe->va0_l = (u32)wr->sg_list[0].addr;
+			ud_sq_wqe->va0_h = (wr->sg_list[0].addr) >> 32;
+			ud_sq_wqe->l_key0 = wr->sg_list[0].lkey;
+
+			ud_sq_wqe->va1_l = (u32)wr->sg_list[1].addr;
+			ud_sq_wqe->va1_h = (wr->sg_list[1].addr) >> 32;
+			ud_sq_wqe->l_key1 = wr->sg_list[1].lkey;
+			ind++;
+		} else if (ibqp->qp_type == IB_QPT_RC) {
+			ctrl = wqe;
+			memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
+			for (i = 0; i < wr->num_sge; i++)
+				ctrl->msg_length += wr->sg_list[i].length;
+
+			ctrl->sgl_pa_h = 0;
+			ctrl->flag = 0;
+			ctrl->imm_data = send_ieth(wr);
+
+			/*Ctrl field, ctrl set type: sig, solic, imm, fence */
+			/* SO wait for conforming application scenarios */
+			ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ?
+				      cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
+				      (wr->send_flags & IB_SEND_SOLICITED ?
+				      cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
+				      ((wr->opcode == IB_WR_SEND_WITH_IMM ||
+				      wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) ?
+				      cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) |
+				      (wr->send_flags & IB_SEND_FENCE ?
+				      (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0);
+
+			wqe = (struct hns_roce_wqe_ctrl_seg *)wqe +
+			       sizeof(struct hns_roce_wqe_ctrl_seg);
+
+			switch (wr->opcode) {
+			case IB_WR_RDMA_READ:
+				ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;
+				set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
+					      atomic_wr(wr)->rkey);
+				break;
+			case IB_WR_RDMA_WRITE:
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;
+				set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
+					      atomic_wr(wr)->rkey);
+				break;
+			case IB_WR_SEND:
+			case IB_WR_SEND_WITH_INV:
+			case IB_WR_SEND_WITH_IMM:
+				ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;
+				break;
+			case IB_WR_LOCAL_INV:
+				break;
+			case IB_WR_ATOMIC_CMP_AND_SWP:
+			case IB_WR_ATOMIC_FETCH_AND_ADD:
+			case IB_WR_LSO:
+			default:
+				ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;
+				break;
+			}
+			ctrl->flag |= cpu_to_le32(ps_opcode);
+			wqe = (struct hns_roce_wqe_raddr_seg *)wqe +
+			       sizeof(struct hns_roce_wqe_raddr_seg);
+
+			dseg = wqe;
+			if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
+				if (ctrl->msg_length >
+					hr_dev->caps.max_sq_inline) {
+					ret = -EINVAL;
+					*bad_wr = wr;
+					dev_err(dev, "inline len(1-%d)=%d, illegal",
+						ctrl->msg_length,
+						hr_dev->caps.max_sq_inline);
+					goto out;
+				}
+				for (i = 0; i < wr->num_sge; i++) {
+					memcpy(wqe, ((void *) (uintptr_t)
+					       wr->sg_list[i].addr),
+					       wr->sg_list[i].length);
+					wqe = (struct hns_roce_wqe_raddr_seg *)
+					       wqe + wr->sg_list[i].length;
+				}
+				ctrl->flag |= HNS_ROCE_WQE_INLINE;
+			} else {
+				/*sqe num is two */
+				for (i = 0; i < wr->num_sge; i++)
+					set_data_seg(dseg + i, wr->sg_list + i);
+
+				ctrl->flag |= cpu_to_le32(wr->num_sge <<
+					      HNS_ROCE_WQE_SGE_NUM_BIT);
+			}
+			ind++;
+		} else {
+			dev_dbg(dev, "unSupported QP type\n");
+			break;
+		}
+	}
+
+out:
+	/* Set DB return */
+	if (likely(nreq)) {
+		qp->sq.head += nreq;
+		/* Memory barrier */
+		wmb();
+
+		sq_db.u32_4 = 0;
+		sq_db.u32_8 = 0;
+		roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,
+			       SQ_DOORBELL_U32_4_SQ_HEAD_S,
+			      (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)));
+		roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M,
+			       SQ_DOORBELL_U32_4_PORT_S, qp->port);
+		roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M,
+			       SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn);
+		roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1);
+
+		doorbell[0] = sq_db.u32_4;
+		doorbell[1] = sq_db.u32_8;
+
+		hns_roce_write64_k(doorbell, qp->sq.db_reg_l);
+		qp->sq_next_wqe = ind;
+	}
+
+	spin_unlock_irqrestore(&qp->sq.lock, flags);
+
+	return ret;
+}
+
+int hns_roce_v1_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+			  struct ib_recv_wr **bad_wr)
+{
+	int ret = 0;
+	int nreq = 0;
+	int ind = 0;
+	int i = 0;
+	u32 reg_val = 0;
+	unsigned long flags = 0;
+	struct hns_roce_rq_wqe_ctrl *ctrl = NULL;
+	struct hns_roce_wqe_data_seg *scat = NULL;
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_rq_db rq_db;
+	uint32_t doorbell[2] = {0};
+
+	spin_lock_irqsave(&hr_qp->rq.lock, flags);
+	ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
+
+	for (nreq = 0; wr; ++nreq, wr = wr->next) {
+		if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
+			hr_qp->ibqp.recv_cq)) {
+			ret = -ENOMEM;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
+			dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
+				wr->num_sge, hr_qp->rq.max_gs);
+			ret = -EINVAL;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		ctrl = get_recv_wqe(hr_qp, ind);
+
+		roce_set_field(ctrl->rwqe_byte_12,
+			       RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M,
+			       RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S,
+			       wr->num_sge);
+
+		scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1);
+
+		for (i = 0; i < wr->num_sge; i++)
+			set_data_seg(scat + i, wr->sg_list + i);
+
+		hr_qp->rq.wrid[ind] = wr->wr_id;
+
+		ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
+	}
+
+out:
+	if (likely(nreq)) {
+		hr_qp->rq.head += nreq;
+		/* Memory barrier */
+		wmb();
+
+		if (ibqp->qp_type == IB_QPT_GSI) {
+			/* SW update GSI rq header */
+			reg_val = roce_read(to_hr_dev(ibqp->device),
+					    ROCEE_QP1C_CFG3_0_REG +
+					    QP1C_CFGN_OFFSET * hr_qp->port);
+			roce_set_field(reg_val,
+				       ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,
+				       ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,
+				       hr_qp->rq.head);
+			roce_write(to_hr_dev(ibqp->device),
+				   ROCEE_QP1C_CFG3_0_REG +
+				   QP1C_CFGN_OFFSET * hr_qp->port, reg_val);
+		} else {
+			rq_db.u32_4 = 0;
+			rq_db.u32_8 = 0;
+
+			roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
+				       RQ_DOORBELL_U32_4_RQ_HEAD_S,
+				       hr_qp->rq.head);
+			roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,
+				       RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
+			roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,
+				       RQ_DOORBELL_U32_8_CMD_S, 1);
+			roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
+				     1);
+
+			doorbell[0] = rq_db.u32_4;
+			doorbell[1] = rq_db.u32_8;
+
+			hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
+		}
+	}
+	spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
+
+	return ret;
+}
+
+static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev,
+				       int sdb_mode, int odb_mode)
+{
+	u32 val;
+
+	val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
+	roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode);
+	roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode);
+	roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
+}
+
+static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode,
+				     u32 odb_mode)
+{
+	u32 val;
+
+	/* Configure SDB/ODB extend mode */
+	val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
+	roce_set_bit(val, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode);
+	roce_set_bit(val, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode);
+	roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
+}
+
+static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept,
+			     u32 sdb_alful)
+{
+	u32 val;
+
+	/* Configure SDB */
+	val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG);
+	roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M,
+		       ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful);
+	roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M,
+		       ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept);
+	roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val);
+}
+
+static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept,
+			     u32 odb_alful)
+{
+	u32 val;
+
+	/* Configure ODB */
+	val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG);
+	roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M,
+		       ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful);
+	roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M,
+		       ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept);
+	roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val);
+}
+
+static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
+				 u32 ext_sdb_alful)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_db_table *db;
+	dma_addr_t sdb_dma_addr;
+	u32 val;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	db = &priv->db_table;
+
+	/* Configure extend SDB threshold */
+	roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept);
+	roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful);
+
+	/* Configure extend SDB base addr */
+	sdb_dma_addr = db->ext_db->sdb_buf_list->map;
+	roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12));
+
+	/* Configure extend SDB depth */
+	val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG);
+	roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M,
+		       ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S,
+		       db->ext_db->esdb_dep);
+	/*
+	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
+	 * using 4K page, and shift more 32 because of
+	 * caculating the high 32 bit value evaluated to hardware.
+	 */
+	roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M,
+		       ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44);
+	roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val);
+
+	dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep);
+	dev_dbg(dev, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
+		ext_sdb_alept, ext_sdb_alful);
+}
+
+static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
+				 u32 ext_odb_alful)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_db_table *db;
+	dma_addr_t odb_dma_addr;
+	u32 val;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	db = &priv->db_table;
+
+	/* Configure extend ODB threshold */
+	roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept);
+	roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful);
+
+	/* Configure extend ODB base addr */
+	odb_dma_addr = db->ext_db->odb_buf_list->map;
+	roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12));
+
+	/* Configure extend ODB depth */
+	val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG);
+	roce_set_field(val, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M,
+		       ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S,
+		       db->ext_db->eodb_dep);
+	roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M,
+		       ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S,
+		       db->ext_db->eodb_dep);
+	roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val);
+
+	dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep);
+	dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
+		ext_odb_alept, ext_odb_alful);
+}
+
+static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,
+				u32 odb_ext_mod)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_db_table *db;
+	dma_addr_t sdb_dma_addr;
+	dma_addr_t odb_dma_addr;
+	int ret = 0;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	db = &priv->db_table;
+
+	db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL);
+	if (!db->ext_db)
+		return -ENOMEM;
+
+	if (sdb_ext_mod) {
+		db->ext_db->sdb_buf_list = kmalloc(
+				sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL);
+		if (!db->ext_db->sdb_buf_list) {
+			ret = -ENOMEM;
+			goto ext_sdb_buf_fail_out;
+		}
+
+		db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev,
+						     HNS_ROCE_V1_EXT_SDB_SIZE,
+						     &sdb_dma_addr, GFP_KERNEL);
+		if (!db->ext_db->sdb_buf_list->buf) {
+			ret = -ENOMEM;
+			goto alloc_sq_db_buf_fail;
+		}
+		db->ext_db->sdb_buf_list->map = sdb_dma_addr;
+
+		db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH);
+		hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT,
+				     HNS_ROCE_V1_EXT_SDB_ALFUL);
+	} else
+		hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT,
+				 HNS_ROCE_V1_SDB_ALFUL);
+
+	if (odb_ext_mod) {
+		db->ext_db->odb_buf_list = kmalloc(
+				sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL);
+		if (!db->ext_db->odb_buf_list) {
+			ret = -ENOMEM;
+			goto ext_odb_buf_fail_out;
+		}
+
+		db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev,
+						     HNS_ROCE_V1_EXT_ODB_SIZE,
+						     &odb_dma_addr, GFP_KERNEL);
+		if (!db->ext_db->odb_buf_list->buf) {
+			ret = -ENOMEM;
+			goto alloc_otr_db_buf_fail;
+		}
+		db->ext_db->odb_buf_list->map = odb_dma_addr;
+
+		db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH);
+		hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT,
+				     HNS_ROCE_V1_EXT_ODB_ALFUL);
+	} else
+		hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT,
+				 HNS_ROCE_V1_ODB_ALFUL);
+
+	hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod);
+
+	return 0;
+
+alloc_otr_db_buf_fail:
+	kfree(db->ext_db->odb_buf_list);
+
+ext_odb_buf_fail_out:
+	if (sdb_ext_mod) {
+		dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
+				  db->ext_db->sdb_buf_list->buf,
+				  db->ext_db->sdb_buf_list->map);
+	}
+
+alloc_sq_db_buf_fail:
+	if (sdb_ext_mod)
+		kfree(db->ext_db->sdb_buf_list);
+
+ext_sdb_buf_fail_out:
+	kfree(db->ext_db);
+	return ret;
+}
+
+static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_db_table *db;
+	u32 sdb_ext_mod;
+	u32 odb_ext_mod;
+	u32 sdb_evt_mod;
+	u32 odb_evt_mod;
+	int ret = 0;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	db = &priv->db_table;
+
+	memset(db, 0, sizeof(*db));
+
+	/* Default DB mode */
+	sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE;
+	odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE;
+	sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE;
+	odb_evt_mod = HNS_ROCE_ODB_POLL_MODE;
+
+	db->sdb_ext_mod = sdb_ext_mod;
+	db->odb_ext_mod = odb_ext_mod;
+
+	/* Init extend DB */
+	ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod);
+	if (ret) {
+		dev_err(dev, "Failed in extend DB configuration.\n");
+		return ret;
+	}
+
+	hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod);
+
+	return 0;
+}
+
+static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_db_table *db;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	db = &priv->db_table;
+
+	if (db->sdb_ext_mod) {
+		dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
+				  db->ext_db->sdb_buf_list->buf,
+				  db->ext_db->sdb_buf_list->map);
+		kfree(db->ext_db->sdb_buf_list);
+	}
+
+	if (db->odb_ext_mod) {
+		dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE,
+				  db->ext_db->odb_buf_list->buf,
+				  db->ext_db->odb_buf_list->map);
+		kfree(db->ext_db->odb_buf_list);
+	}
+
+	kfree(db->ext_db);
+}
+
+static int hns_roce_raq_init(struct hns_roce_dev *hr_dev)
+{
+	int ret;
+	int raq_shift = 0;
+	dma_addr_t addr;
+	u32 val;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_raq_table *raq;
+	struct device *dev = &hr_dev->pdev->dev;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	raq = &priv->raq_table;
+
+	raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL);
+	if (!raq->e_raq_buf)
+		return -ENOMEM;
+
+	raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE,
+						 &addr, GFP_KERNEL);
+	if (!raq->e_raq_buf->buf) {
+		ret = -ENOMEM;
+		goto err_dma_alloc_raq;
+	}
+	raq->e_raq_buf->map = addr;
+
+	/* Configure raq extended address. 48bit 4K align*/
+	roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12);
+
+	/* Configure raq_shift */
+	raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY);
+	val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG);
+	roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M,
+		       ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift);
+	/*
+	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
+	 * using 4K page, and shift more 32 because of
+	 * caculating the high 32 bit value evaluated to hardware.
+	 */
+	roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M,
+		       ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S,
+		       raq->e_raq_buf->map >> 44);
+	roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val);
+	dev_dbg(dev, "Configure raq_shift 0x%x.\n", val);
+
+	/* Configure raq threshold */
+	val = roce_read(hr_dev, ROCEE_RAQ_WL_REG);
+	roce_set_field(val, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M,
+		       ROCEE_RAQ_WL_ROCEE_RAQ_WL_S,
+		       HNS_ROCE_V1_EXT_RAQ_WF);
+	roce_write(hr_dev, ROCEE_RAQ_WL_REG, val);
+	dev_dbg(dev, "Configure raq_wl 0x%x.\n", val);
+
+	/* Enable extend raq */
+	val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG);
+	roce_set_field(val,
+		       ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M,
+		       ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S,
+		       POL_TIME_INTERVAL_VAL);
+	roce_set_bit(val, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1);
+	roce_set_field(val,
+		       ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M,
+		       ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S,
+		       2);
+	roce_set_bit(val,
+		     ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1);
+	roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val);
+	dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val);
+
+	/* Enable raq drop */
+	val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
+	roce_set_bit(val, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1);
+	roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
+	dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val);
+
+	return 0;
+
+err_dma_alloc_raq:
+	kfree(raq->e_raq_buf);
+	return ret;
+}
+
+static void hns_roce_raq_free(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_raq_table *raq;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	raq = &priv->raq_table;
+
+	dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf,
+			  raq->e_raq_buf->map);
+	kfree(raq->e_raq_buf);
+}
+
+static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)
+{
+	u32 val;
+
+	if (enable_flag) {
+		val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
+		 /* Open all ports */
+		roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
+			       ROCEE_GLB_CFG_ROCEE_PORT_ST_S,
+			       ALL_PORT_VAL_OPEN);
+		roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
+	} else {
+		val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
+		/* Close all ports */
+		roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
+			       ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0);
+		roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
+	}
+}
+
+/**
+ * hns_roce_v1_reset - reset RoCE
+ * @hr_dev: RoCE device struct pointer
+ * @enable: true -- drop reset, false -- reset
+ * return 0 - success , negative --fail
+ */
+int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)
+{
+	struct device_node *dsaf_node;
+	struct device *dev = &hr_dev->pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct fwnode_handle *fwnode;
+	int ret;
+
+	/* check if this is DT/ACPI case */
+	if (dev_of_node(dev)) {
+		dsaf_node = of_parse_phandle(np, "dsaf-handle", 0);
+		if (!dsaf_node) {
+			dev_err(dev, "could not find dsaf-handle\n");
+			return -EINVAL;
+		}
+		fwnode = &dsaf_node->fwnode;
+	} else if (is_acpi_device_node(dev->fwnode)) {
+		struct acpi_reference_args args;
+
+		ret = acpi_node_get_property_reference(dev->fwnode,
+						       "dsaf-handle", 0, &args);
+		if (ret) {
+			dev_err(dev, "could not find dsaf-handle\n");
+			return ret;
+		}
+		fwnode = acpi_fwnode_handle(args.adev);
+	} else {
+		dev_err(dev, "cannot read data from DT or ACPI\n");
+		return -ENXIO;
+	}
+
+	ret = hns_dsaf_roce_reset(fwnode, false);
+	if (ret)
+		return ret;
+
+	if (dereset) {
+		msleep(SLEEP_TIME_INTERVAL);
+		ret = hns_dsaf_roce_reset(fwnode, true);
+	}
+
+	return ret;
+}
+
+void hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
+{
+	int i = 0;
+	struct hns_roce_caps *caps = &hr_dev->caps;
+
+	hr_dev->vendor_id = le32_to_cpu(roce_read(hr_dev, ROCEE_VENDOR_ID_REG));
+	hr_dev->vendor_part_id = le32_to_cpu(roce_read(hr_dev,
+					     ROCEE_VENDOR_PART_ID_REG));
+	hr_dev->hw_rev = le32_to_cpu(roce_read(hr_dev, ROCEE_HW_VERSION_REG));
+
+	hr_dev->sys_image_guid = le32_to_cpu(roce_read(hr_dev,
+					     ROCEE_SYS_IMAGE_GUID_L_REG)) |
+				((u64)le32_to_cpu(roce_read(hr_dev,
+					    ROCEE_SYS_IMAGE_GUID_H_REG)) << 32);
+
+	caps->num_qps		= HNS_ROCE_V1_MAX_QP_NUM;
+	caps->max_wqes		= HNS_ROCE_V1_MAX_WQE_NUM;
+	caps->num_cqs		= HNS_ROCE_V1_MAX_CQ_NUM;
+	caps->max_cqes		= HNS_ROCE_V1_MAX_CQE_NUM;
+	caps->max_sq_sg		= HNS_ROCE_V1_SG_NUM;
+	caps->max_rq_sg		= HNS_ROCE_V1_SG_NUM;
+	caps->max_sq_inline	= HNS_ROCE_V1_INLINE_SIZE;
+	caps->num_uars		= HNS_ROCE_V1_UAR_NUM;
+	caps->phy_num_uars	= HNS_ROCE_V1_PHY_UAR_NUM;
+	caps->num_aeq_vectors	= HNS_ROCE_AEQE_VEC_NUM;
+	caps->num_comp_vectors	= HNS_ROCE_COMP_VEC_NUM;
+	caps->num_other_vectors	= HNS_ROCE_AEQE_OF_VEC_NUM;
+	caps->num_mtpts		= HNS_ROCE_V1_MAX_MTPT_NUM;
+	caps->num_mtt_segs	= HNS_ROCE_V1_MAX_MTT_SEGS;
+	caps->num_pds		= HNS_ROCE_V1_MAX_PD_NUM;
+	caps->max_qp_init_rdma	= HNS_ROCE_V1_MAX_QP_INIT_RDMA;
+	caps->max_qp_dest_rdma	= HNS_ROCE_V1_MAX_QP_DEST_RDMA;
+	caps->max_sq_desc_sz	= HNS_ROCE_V1_MAX_SQ_DESC_SZ;
+	caps->max_rq_desc_sz	= HNS_ROCE_V1_MAX_RQ_DESC_SZ;
+	caps->qpc_entry_sz	= HNS_ROCE_V1_QPC_ENTRY_SIZE;
+	caps->irrl_entry_sz	= HNS_ROCE_V1_IRRL_ENTRY_SIZE;
+	caps->cqc_entry_sz	= HNS_ROCE_V1_CQC_ENTRY_SIZE;
+	caps->mtpt_entry_sz	= HNS_ROCE_V1_MTPT_ENTRY_SIZE;
+	caps->mtt_entry_sz	= HNS_ROCE_V1_MTT_ENTRY_SIZE;
+	caps->cq_entry_sz	= HNS_ROCE_V1_CQE_ENTRY_SIZE;
+	caps->page_size_cap	= HNS_ROCE_V1_PAGE_SIZE_SUPPORT;
+	caps->sqp_start		= 0;
+	caps->reserved_lkey	= 0;
+	caps->reserved_pds	= 0;
+	caps->reserved_mrws	= 1;
+	caps->reserved_uars	= 0;
+	caps->reserved_cqs	= 0;
+
+	for (i = 0; i < caps->num_ports; i++)
+		caps->pkey_table_len[i] = 1;
+
+	for (i = 0; i < caps->num_ports; i++) {
+		/* Six ports shared 16 GID in v1 engine */
+		if (i >= (HNS_ROCE_V1_GID_NUM % caps->num_ports))
+			caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
+						 caps->num_ports;
+		else
+			caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
+						 caps->num_ports + 1;
+	}
+
+	for (i = 0; i < caps->num_comp_vectors; i++)
+		caps->ceqe_depth[i] = HNS_ROCE_V1_NUM_COMP_EQE;
+
+	caps->aeqe_depth = HNS_ROCE_V1_NUM_ASYNC_EQE;
+	caps->local_ca_ack_delay = le32_to_cpu(roce_read(hr_dev,
+							 ROCEE_ACK_DELAY_REG));
+	caps->max_mtu = IB_MTU_2048;
+}
+
+int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
+{
+	int ret;
+	u32 val;
+	struct device *dev = &hr_dev->pdev->dev;
+
+	/* DMAE user config */
+	val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG);
+	roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M,
+		       ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf);
+	roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M,
+		       ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S,
+		       1 << PAGES_SHIFT_16);
+	roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val);
+
+	val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG);
+	roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M,
+		       ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf);
+	roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M,
+		       ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S,
+		       1 << PAGES_SHIFT_16);
+
+	ret = hns_roce_db_init(hr_dev);
+	if (ret) {
+		dev_err(dev, "doorbell init failed!\n");
+		return ret;
+	}
+
+	ret = hns_roce_raq_init(hr_dev);
+	if (ret) {
+		dev_err(dev, "raq init failed!\n");
+		goto error_failed_raq_init;
+	}
+
+	hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
+
+	return 0;
+
+error_failed_raq_init:
+	hns_roce_db_free(hr_dev);
+	return ret;
+}
+
+void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)
+{
+	hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
+	hns_roce_raq_free(hr_dev);
+	hns_roce_db_free(hr_dev);
+}
+
+void hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
+			 union ib_gid *gid)
+{
+	u32 *p = NULL;
+	u8 gid_idx = 0;
+
+	gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
+
+	p = (u32 *)&gid->raw[0];
+	roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG +
+		       (HNS_ROCE_V1_GID_NUM * gid_idx));
+
+	p = (u32 *)&gid->raw[4];
+	roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG +
+		       (HNS_ROCE_V1_GID_NUM * gid_idx));
+
+	p = (u32 *)&gid->raw[8];
+	roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG +
+		       (HNS_ROCE_V1_GID_NUM * gid_idx));
+
+	p = (u32 *)&gid->raw[0xc];
+	roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG +
+		       (HNS_ROCE_V1_GID_NUM * gid_idx));
+}
+
+void hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr)
+{
+	u32 reg_smac_l;
+	u16 reg_smac_h;
+	u16 *p_h;
+	u32 *p;
+	u32 val;
+
+	p = (u32 *)(&addr[0]);
+	reg_smac_l = *p;
+	roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG +
+		       PHY_PORT_OFFSET * phy_port);
+
+	val = roce_read(hr_dev,
+			ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
+	p_h = (u16 *)(&addr[4]);
+	reg_smac_h  = *p_h;
+	roce_set_field(val, ROCEE_SMAC_H_ROCEE_SMAC_H_M,
+		       ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h);
+	roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
+		   val);
+}
+
+void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port,
+			 enum ib_mtu mtu)
+{
+	u32 val;
+
+	val = roce_read(hr_dev,
+			ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
+	roce_set_field(val, ROCEE_SMAC_H_ROCEE_PORT_MTU_M,
+		       ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu);
+	roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
+		   val);
+}
+
+int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
+			   unsigned long mtpt_idx)
+{
+	struct hns_roce_v1_mpt_entry *mpt_entry;
+	struct scatterlist *sg;
+	u64 *pages;
+	int entry;
+	int i;
+
+	/* MPT filled into mailbox buf */
+	mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf;
+	memset(mpt_entry, 0, sizeof(*mpt_entry));
+
+	roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_STATE_M,
+		       MPT_BYTE_4_KEY_STATE_S, KEY_VALID);
+	roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_M,
+		       MPT_BYTE_4_KEY_S, mr->key);
+	roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M,
+		       MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K);
+	roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0);
+	roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S,
+		     (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
+	roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_OWN_S, 0);
+	roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M,
+		       MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr->type);
+	roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0);
+	roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S,
+		     (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
+	roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S,
+		     (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
+	roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S,
+		     (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
+	roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S,
+		     0);
+	roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0);
+
+	roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
+		       MPT_BYTE_12_PBL_ADDR_H_S, 0);
+	roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M,
+		       MPT_BYTE_12_MW_BIND_COUNTER_S, 0);
+
+	mpt_entry->virt_addr_l = (u32)mr->iova;
+	mpt_entry->virt_addr_h = (u32)(mr->iova >> 32);
+	mpt_entry->length = (u32)mr->size;
+
+	roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_PD_M,
+		       MPT_BYTE_28_PD_S, mr->pd);
+	roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M,
+		       MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx);
+	roce_set_field(mpt_entry->mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M,
+		       MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx >> MTPT_IDX_SHIFT);
+
+	/* DMA momery regsiter */
+	if (mr->type == MR_TYPE_DMA)
+		return 0;
+
+	pages = (u64 *) __get_free_page(GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
+
+	i = 0;
+	for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
+		pages[i] = ((u64)sg_dma_address(sg)) >> 12;
+
+		/* Directly record to MTPT table firstly 7 entry */
+		if (i >= HNS_ROCE_MAX_INNER_MTPT_NUM)
+			break;
+		i++;
+	}
+
+	/* Register user mr */
+	for (i = 0; i < HNS_ROCE_MAX_INNER_MTPT_NUM; i++) {
+		switch (i) {
+		case 0:
+			mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i]));
+			roce_set_field(mpt_entry->mpt_byte_36,
+				MPT_BYTE_36_PA0_H_M,
+				MPT_BYTE_36_PA0_H_S,
+				cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
+			break;
+		case 1:
+			roce_set_field(mpt_entry->mpt_byte_36,
+				       MPT_BYTE_36_PA1_L_M,
+				       MPT_BYTE_36_PA1_L_S,
+				       cpu_to_le32((u32)(pages[i])));
+			roce_set_field(mpt_entry->mpt_byte_40,
+				MPT_BYTE_40_PA1_H_M,
+				MPT_BYTE_40_PA1_H_S,
+				cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
+			break;
+		case 2:
+			roce_set_field(mpt_entry->mpt_byte_40,
+				       MPT_BYTE_40_PA2_L_M,
+				       MPT_BYTE_40_PA2_L_S,
+				       cpu_to_le32((u32)(pages[i])));
+			roce_set_field(mpt_entry->mpt_byte_44,
+				MPT_BYTE_44_PA2_H_M,
+				MPT_BYTE_44_PA2_H_S,
+				cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
+			break;
+		case 3:
+			roce_set_field(mpt_entry->mpt_byte_44,
+				       MPT_BYTE_44_PA3_L_M,
+				       MPT_BYTE_44_PA3_L_S,
+				       cpu_to_le32((u32)(pages[i])));
+			roce_set_field(mpt_entry->mpt_byte_48,
+				MPT_BYTE_48_PA3_H_M,
+				MPT_BYTE_48_PA3_H_S,
+				cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_8)));
+			break;
+		case 4:
+			mpt_entry->pa4_l = cpu_to_le32((u32)(pages[i]));
+			roce_set_field(mpt_entry->mpt_byte_56,
+				MPT_BYTE_56_PA4_H_M,
+				MPT_BYTE_56_PA4_H_S,
+				cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
+			break;
+		case 5:
+			roce_set_field(mpt_entry->mpt_byte_56,
+				       MPT_BYTE_56_PA5_L_M,
+				       MPT_BYTE_56_PA5_L_S,
+				       cpu_to_le32((u32)(pages[i])));
+			roce_set_field(mpt_entry->mpt_byte_60,
+				MPT_BYTE_60_PA5_H_M,
+				MPT_BYTE_60_PA5_H_S,
+				cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
+			break;
+		case 6:
+			roce_set_field(mpt_entry->mpt_byte_60,
+				       MPT_BYTE_60_PA6_L_M,
+				       MPT_BYTE_60_PA6_L_S,
+				       cpu_to_le32((u32)(pages[i])));
+			roce_set_field(mpt_entry->mpt_byte_64,
+				MPT_BYTE_64_PA6_H_M,
+				MPT_BYTE_64_PA6_H_S,
+				cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
+			break;
+		default:
+			break;
+		}
+	}
+
+	free_page((unsigned long) pages);
+
+	mpt_entry->pbl_addr_l = (u32)(mr->pbl_dma_addr);
+
+	roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
+		       MPT_BYTE_12_PBL_ADDR_H_S,
+		       ((u32)(mr->pbl_dma_addr >> 32)));
+
+	return 0;
+}
+
+static void *get_cqe(struct hns_roce_cq *hr_cq, int n)
+{
+	return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
+				   n * HNS_ROCE_V1_CQE_ENTRY_SIZE);
+}
+
+static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n)
+{
+	struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n & hr_cq->ib_cq.cqe);
+
+	/* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
+	return (roce_get_bit(hr_cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S) ^
+		!!(n & (hr_cq->ib_cq.cqe + 1))) ? hr_cqe : NULL;
+}
+
+static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq)
+{
+	return get_sw_cqe(hr_cq, hr_cq->cons_index);
+}
+
+void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index,
+			   spinlock_t *doorbell_lock)
+
+{
+	u32 doorbell[2];
+
+	doorbell[0] = cons_index & ((hr_cq->cq_depth << 1) - 1);
+	roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
+	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
+		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
+	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
+		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0);
+	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
+		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn);
+
+	hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
+}
+
+static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
+				   struct hns_roce_srq *srq)
+{
+	struct hns_roce_cqe *cqe, *dest;
+	u32 prod_index;
+	int nfreed = 0;
+	u8 owner_bit;
+
+	for (prod_index = hr_cq->cons_index; get_sw_cqe(hr_cq, prod_index);
+	     ++prod_index) {
+		if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
+			break;
+	}
+
+	/*
+	* Now backwards through the CQ, removing CQ entries
+	* that match our QP by overwriting them with next entries.
+	*/
+	while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
+		cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe);
+		if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
+				     CQE_BYTE_16_LOCAL_QPN_S) &
+				     HNS_ROCE_CQE_QPN_MASK) == qpn) {
+			/* In v1 engine, not support SRQ */
+			++nfreed;
+		} else if (nfreed) {
+			dest = get_cqe(hr_cq, (prod_index + nfreed) &
+				       hr_cq->ib_cq.cqe);
+			owner_bit = roce_get_bit(dest->cqe_byte_4,
+						 CQE_BYTE_4_OWNER_S);
+			memcpy(dest, cqe, sizeof(*cqe));
+			roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S,
+				     owner_bit);
+		}
+	}
+
+	if (nfreed) {
+		hr_cq->cons_index += nfreed;
+		/*
+		* Make sure update of buffer contents is done before
+		* updating consumer index.
+		*/
+		wmb();
+
+		hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index,
+				   &to_hr_dev(hr_cq->ib_cq.device)->cq_db_lock);
+	}
+}
+
+static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
+				 struct hns_roce_srq *srq)
+{
+	spin_lock_irq(&hr_cq->lock);
+	__hns_roce_v1_cq_clean(hr_cq, qpn, srq);
+	spin_unlock_irq(&hr_cq->lock);
+}
+
+void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
+			   struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
+			   dma_addr_t dma_handle, int nent, u32 vector)
+{
+	struct hns_roce_cq_context *cq_context = NULL;
+	void __iomem *tptr_addr;
+
+	cq_context = mb_buf;
+	memset(cq_context, 0, sizeof(*cq_context));
+
+	tptr_addr = 0;
+	hr_dev->priv_addr = tptr_addr;
+	hr_cq->tptr_addr = tptr_addr;
+
+	/* Register cq_context members */
+	roce_set_field(cq_context->cqc_byte_4,
+		       CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M,
+		       CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID);
+	roce_set_field(cq_context->cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M,
+		       CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq->cqn);
+	cq_context->cqc_byte_4 = cpu_to_le32(cq_context->cqc_byte_4);
+
+	cq_context->cq_bt_l = (u32)dma_handle;
+	cq_context->cq_bt_l = cpu_to_le32(cq_context->cq_bt_l);
+
+	roce_set_field(cq_context->cqc_byte_12,
+		       CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M,
+		       CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S,
+		       ((u64)dma_handle >> 32));
+	roce_set_field(cq_context->cqc_byte_12,
+		       CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M,
+		       CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S,
+		       ilog2((unsigned int)nent));
+	roce_set_field(cq_context->cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M,
+		       CQ_CONTEXT_CQC_BYTE_12_CEQN_S, vector);
+	cq_context->cqc_byte_12 = cpu_to_le32(cq_context->cqc_byte_12);
+
+	cq_context->cur_cqe_ba0_l = (u32)(mtts[0]);
+	cq_context->cur_cqe_ba0_l = cpu_to_le32(cq_context->cur_cqe_ba0_l);
+
+	roce_set_field(cq_context->cqc_byte_20,
+		       CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M,
+		       CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S,
+		       cpu_to_le32((mtts[0]) >> 32));
+	/* Dedicated hardware, directly set 0 */
+	roce_set_field(cq_context->cqc_byte_20,
+		       CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M,
+		       CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0);
+	/**
+	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
+	 * using 4K page, and shift more 32 because of
+	 * caculating the high 32 bit value evaluated to hardware.
+	 */
+	roce_set_field(cq_context->cqc_byte_20,
+		       CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M,
+		       CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S,
+		       (u64)tptr_addr >> 44);
+	cq_context->cqc_byte_20 = cpu_to_le32(cq_context->cqc_byte_20);
+
+	cq_context->cqe_tptr_addr_l = (u32)((u64)tptr_addr >> 12);
+
+	roce_set_field(cq_context->cqc_byte_32,
+		       CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M,
+		       CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0);
+	roce_set_bit(cq_context->cqc_byte_32,
+		     CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0);
+	roce_set_bit(cq_context->cqc_byte_32,
+		     CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0);
+	roce_set_bit(cq_context->cqc_byte_32,
+		     CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0);
+	roce_set_bit(cq_context->cqc_byte_32,
+		     CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S,
+		     0);
+	/*The initial value of cq's ci is 0 */
+	roce_set_field(cq_context->cqc_byte_32,
+		       CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M,
+		       CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);
+	cq_context->cqc_byte_32 = cpu_to_le32(cq_context->cqc_byte_32);
+}
+
+int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
+{
+	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
+	u32 notification_flag;
+	u32 doorbell[2];
+	int ret = 0;
+
+	notification_flag = (flags & IB_CQ_SOLICITED_MASK) ==
+			    IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;
+	/*
+	* flags = 0; Notification Flag = 1, next
+	* flags = 1; Notification Flag = 0, solocited
+	*/
+	doorbell[0] = hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1);
+	roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
+	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
+		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
+	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
+		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1);
+	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
+		       ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S,
+		       hr_cq->cqn | notification_flag);
+
+	hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
+
+	return ret;
+}
+
+static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
+				struct hns_roce_qp **cur_qp, struct ib_wc *wc)
+{
+	int qpn;
+	int is_send;
+	u16 wqe_ctr;
+	u32 status;
+	u32 opcode;
+	struct hns_roce_cqe *cqe;
+	struct hns_roce_qp *hr_qp;
+	struct hns_roce_wq *wq;
+	struct hns_roce_wqe_ctrl_seg *sq_wqe;
+	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
+	struct device *dev = &hr_dev->pdev->dev;
+
+	/* Find cqe according consumer index */
+	cqe = next_cqe_sw(hr_cq);
+	if (!cqe)
+		return -EAGAIN;
+
+	++hr_cq->cons_index;
+	/* Memory barrier */
+	rmb();
+	/* 0->SQ, 1->RQ */
+	is_send  = !(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S));
+
+	/* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
+	if (roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
+			   CQE_BYTE_16_LOCAL_QPN_S) <= 1) {
+		qpn = roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_PORT_NUM_M,
+				     CQE_BYTE_20_PORT_NUM_S) +
+		      roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
+				     CQE_BYTE_16_LOCAL_QPN_S) *
+				     HNS_ROCE_MAX_PORTS;
+	} else {
+		qpn = roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
+				     CQE_BYTE_16_LOCAL_QPN_S);
+	}
+
+	if (!*cur_qp || (qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->qpn) {
+		hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
+		if (unlikely(!hr_qp)) {
+			dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n",
+				hr_cq->cqn, (qpn & HNS_ROCE_CQE_QPN_MASK));
+			return -EINVAL;
+		}
+
+		*cur_qp = hr_qp;
+	}
+
+	wc->qp = &(*cur_qp)->ibqp;
+	wc->vendor_err = 0;
+
+	status = roce_get_field(cqe->cqe_byte_4,
+				CQE_BYTE_4_STATUS_OF_THE_OPERATION_M,
+				CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) &
+				HNS_ROCE_CQE_STATUS_MASK;
+	switch (status) {
+	case HNS_ROCE_CQE_SUCCESS:
+		wc->status = IB_WC_SUCCESS;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR:
+		wc->status = IB_WC_LOC_LEN_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR:
+		wc->status = IB_WC_LOC_QP_OP_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR:
+		wc->status = IB_WC_LOC_PROT_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR:
+		wc->status = IB_WC_WR_FLUSH_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR:
+		wc->status = IB_WC_MW_BIND_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR:
+		wc->status = IB_WC_BAD_RESP_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR:
+		wc->status = IB_WC_LOC_ACCESS_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
+		wc->status = IB_WC_REM_INV_REQ_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR:
+		wc->status = IB_WC_REM_ACCESS_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR:
+		wc->status = IB_WC_REM_OP_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
+		wc->status = IB_WC_RETRY_EXC_ERR;
+		break;
+	case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
+		wc->status = IB_WC_RNR_RETRY_EXC_ERR;
+		break;
+	default:
+		wc->status = IB_WC_GENERAL_ERR;
+		break;
+	}
+
+	/* CQE status error, directly return */
+	if (wc->status != IB_WC_SUCCESS)
+		return 0;
+
+	if (is_send) {
+		/* SQ conrespond to CQE */
+		sq_wqe = get_send_wqe(*cur_qp, roce_get_field(cqe->cqe_byte_4,
+						CQE_BYTE_4_WQE_INDEX_M,
+						CQE_BYTE_4_WQE_INDEX_S));
+		switch (sq_wqe->flag & HNS_ROCE_WQE_OPCODE_MASK) {
+		case HNS_ROCE_WQE_OPCODE_SEND:
+			wc->opcode = IB_WC_SEND;
+			break;
+		case HNS_ROCE_WQE_OPCODE_RDMA_READ:
+			wc->opcode = IB_WC_RDMA_READ;
+			wc->byte_len = le32_to_cpu(cqe->byte_cnt);
+			break;
+		case HNS_ROCE_WQE_OPCODE_RDMA_WRITE:
+			wc->opcode = IB_WC_RDMA_WRITE;
+			break;
+		case HNS_ROCE_WQE_OPCODE_LOCAL_INV:
+			wc->opcode = IB_WC_LOCAL_INV;
+			break;
+		case HNS_ROCE_WQE_OPCODE_UD_SEND:
+			wc->opcode = IB_WC_SEND;
+			break;
+		default:
+			wc->status = IB_WC_GENERAL_ERR;
+			break;
+		}
+		wc->wc_flags = (sq_wqe->flag & HNS_ROCE_WQE_IMM ?
+				IB_WC_WITH_IMM : 0);
+
+		wq = &(*cur_qp)->sq;
+		if ((*cur_qp)->sq_signal_bits) {
+			/*
+			* If sg_signal_bit is 1,
+			* firstly tail pointer updated to wqe
+			* which current cqe correspond to
+			*/
+			wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4,
+						      CQE_BYTE_4_WQE_INDEX_M,
+						      CQE_BYTE_4_WQE_INDEX_S);
+			wq->tail += (wqe_ctr - (u16)wq->tail) &
+				    (wq->wqe_cnt - 1);
+		}
+		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
+		++wq->tail;
+		} else {
+		/* RQ conrespond to CQE */
+		wc->byte_len = le32_to_cpu(cqe->byte_cnt);
+		opcode = roce_get_field(cqe->cqe_byte_4,
+					CQE_BYTE_4_OPERATION_TYPE_M,
+					CQE_BYTE_4_OPERATION_TYPE_S) &
+					HNS_ROCE_CQE_OPCODE_MASK;
+		switch (opcode) {
+		case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE:
+			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+			wc->wc_flags = IB_WC_WITH_IMM;
+			wc->ex.imm_data = le32_to_cpu(cqe->immediate_data);
+			break;
+		case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE:
+			if (roce_get_bit(cqe->cqe_byte_4,
+					 CQE_BYTE_4_IMM_INDICATOR_S)) {
+				wc->opcode = IB_WC_RECV;
+				wc->wc_flags = IB_WC_WITH_IMM;
+				wc->ex.imm_data = le32_to_cpu(
+						  cqe->immediate_data);
+			} else {
+				wc->opcode = IB_WC_RECV;
+				wc->wc_flags = 0;
+			}
+			break;
+		default:
+			wc->status = IB_WC_GENERAL_ERR;
+			break;
+		}
+
+		/* Update tail pointer, record wr_id */
+		wq = &(*cur_qp)->rq;
+		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
+		++wq->tail;
+		wc->sl = (u8)roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_SL_M,
+					    CQE_BYTE_20_SL_S);
+		wc->src_qp = (u8)roce_get_field(cqe->cqe_byte_20,
+						CQE_BYTE_20_REMOTE_QPN_M,
+						CQE_BYTE_20_REMOTE_QPN_S);
+		wc->wc_flags |= (roce_get_bit(cqe->cqe_byte_20,
+					      CQE_BYTE_20_GRH_PRESENT_S) ?
+					      IB_WC_GRH : 0);
+		wc->pkey_index = (u16)roce_get_field(cqe->cqe_byte_28,
+						     CQE_BYTE_28_P_KEY_IDX_M,
+						     CQE_BYTE_28_P_KEY_IDX_S);
+	}
+
+	return 0;
+}
+
+int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
+{
+	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
+	struct hns_roce_qp *cur_qp = NULL;
+	unsigned long flags;
+	int npolled;
+	int ret = 0;
+
+	spin_lock_irqsave(&hr_cq->lock, flags);
+
+	for (npolled = 0; npolled < num_entries; ++npolled) {
+		ret = hns_roce_v1_poll_one(hr_cq, &cur_qp, wc + npolled);
+		if (ret)
+			break;
+	}
+
+	if (npolled) {
+		hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index,
+				      &to_hr_dev(ibcq->device)->cq_db_lock);
+	}
+
+	spin_unlock_irqrestore(&hr_cq->lock, flags);
+
+	if (ret == 0 || ret == -EAGAIN)
+		return npolled;
+	else
+		return ret;
+}
+
+static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
+				 struct hns_roce_mtt *mtt,
+				 enum hns_roce_qp_state cur_state,
+				 enum hns_roce_qp_state new_state,
+				 struct hns_roce_qp_context *context,
+				 struct hns_roce_qp *hr_qp)
+{
+	static const u16
+	op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = {
+		[HNS_ROCE_QP_STATE_RST] = {
+		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
+		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
+		[HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
+		},
+		[HNS_ROCE_QP_STATE_INIT] = {
+		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
+		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
+		/* Note: In v1 engine, HW doesn't support RST2INIT.
+		 * We use RST2INIT cmd instead of INIT2INIT.
+		 */
+		[HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
+		[HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP,
+		},
+		[HNS_ROCE_QP_STATE_RTR] = {
+		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
+		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
+		[HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP,
+		},
+		[HNS_ROCE_QP_STATE_RTS] = {
+		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
+		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
+		[HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP,
+		[HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP,
+		},
+		[HNS_ROCE_QP_STATE_SQD] = {
+		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
+		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
+		[HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP,
+		[HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP,
+		},
+		[HNS_ROCE_QP_STATE_ERR] = {
+		[HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
+		[HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
+		}
+	};
+
+	struct hns_roce_cmd_mailbox *mailbox;
+	struct device *dev = &hr_dev->pdev->dev;
+	int ret = 0;
+
+	if (cur_state >= HNS_ROCE_QP_NUM_STATE ||
+	    new_state >= HNS_ROCE_QP_NUM_STATE ||
+	    !op[cur_state][new_state]) {
+		dev_err(dev, "[modify_qp]not support state %d to %d\n",
+			cur_state, new_state);
+		return -EINVAL;
+	}
+
+	if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP)
+		return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
+					 HNS_ROCE_CMD_2RST_QP,
+					 HNS_ROCE_CMD_TIME_CLASS_A);
+
+	if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP)
+		return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
+					 HNS_ROCE_CMD_2ERR_QP,
+					 HNS_ROCE_CMD_TIME_CLASS_A);
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	memcpy(mailbox->buf, context, sizeof(*context));
+
+	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
+				op[cur_state][new_state],
+				HNS_ROCE_CMD_TIME_CLASS_C);
+
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+	return ret;
+}
+
+static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
+			     int attr_mask, enum ib_qp_state cur_state,
+			     enum ib_qp_state new_state)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	struct hns_roce_sqp_context *context;
+	struct device *dev = &hr_dev->pdev->dev;
+	dma_addr_t dma_handle = 0;
+	int rq_pa_start;
+	u32 reg_val;
+	u64 *mtts;
+	u32 *addr;
+
+	context = kzalloc(sizeof(*context), GFP_KERNEL);
+	if (!context)
+		return -ENOMEM;
+
+	/* Search QP buf's MTTs */
+	mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
+				   hr_qp->mtt.first_seg, &dma_handle);
+	if (!mtts) {
+		dev_err(dev, "qp buf pa find failed\n");
+		goto out;
+	}
+
+	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
+		roce_set_field(context->qp1c_bytes_4,
+			       QP1C_BYTES_4_SQ_WQE_SHIFT_M,
+			       QP1C_BYTES_4_SQ_WQE_SHIFT_S,
+			       ilog2((unsigned int)hr_qp->sq.wqe_cnt));
+		roce_set_field(context->qp1c_bytes_4,
+			       QP1C_BYTES_4_RQ_WQE_SHIFT_M,
+			       QP1C_BYTES_4_RQ_WQE_SHIFT_S,
+			       ilog2((unsigned int)hr_qp->rq.wqe_cnt));
+		roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M,
+			       QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn);
+
+		context->sq_rq_bt_l = (u32)(dma_handle);
+		roce_set_field(context->qp1c_bytes_12,
+			       QP1C_BYTES_12_SQ_RQ_BT_H_M,
+			       QP1C_BYTES_12_SQ_RQ_BT_H_S,
+			       ((u32)(dma_handle >> 32)));
+
+		roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M,
+			       QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head);
+		roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M,
+			       QP1C_BYTES_16_PORT_NUM_S, hr_qp->port);
+		roce_set_bit(context->qp1c_bytes_16,
+			     QP1C_BYTES_16_SIGNALING_TYPE_S,
+			     hr_qp->sq_signal_bits);
+		roce_set_bit(context->qp1c_bytes_16,
+			     QP1C_BYTES_16_LOCAL_ENABLE_E2E_CREDIT_S,
+			     hr_qp->sq_signal_bits);
+		roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S,
+			     1);
+		roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S,
+			     1);
+		roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S,
+			     0);
+
+		roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M,
+			       QP1C_BYTES_20_SQ_HEAD_S, hr_qp->sq.head);
+		roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M,
+			       QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index);
+
+		rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
+		context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
+
+		roce_set_field(context->qp1c_bytes_28,
+			       QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M,
+			       QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S,
+			       (mtts[rq_pa_start]) >> 32);
+		roce_set_field(context->qp1c_bytes_28,
+			       QP1C_BYTES_28_RQ_CUR_IDX_M,
+			       QP1C_BYTES_28_RQ_CUR_IDX_S, 0);
+
+		roce_set_field(context->qp1c_bytes_32,
+			       QP1C_BYTES_32_RX_CQ_NUM_M,
+			       QP1C_BYTES_32_RX_CQ_NUM_S,
+			       to_hr_cq(ibqp->recv_cq)->cqn);
+		roce_set_field(context->qp1c_bytes_32,
+			       QP1C_BYTES_32_TX_CQ_NUM_M,
+			       QP1C_BYTES_32_TX_CQ_NUM_S,
+			       to_hr_cq(ibqp->send_cq)->cqn);
+
+		context->cur_sq_wqe_ba_l  = (u32)mtts[0];
+
+		roce_set_field(context->qp1c_bytes_40,
+			       QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M,
+			       QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S,
+			       (mtts[0]) >> 32);
+		roce_set_field(context->qp1c_bytes_40,
+			       QP1C_BYTES_40_SQ_CUR_IDX_M,
+			       QP1C_BYTES_40_SQ_CUR_IDX_S, 0);
+
+		/* Copy context to QP1C register */
+		addr = (u32 *)(hr_dev->reg_base + ROCEE_QP1C_CFG0_0_REG +
+			hr_qp->port * sizeof(*context));
+
+		writel(context->qp1c_bytes_4, addr);
+		writel(context->sq_rq_bt_l, addr + 1);
+		writel(context->qp1c_bytes_12, addr + 2);
+		writel(context->qp1c_bytes_16, addr + 3);
+		writel(context->qp1c_bytes_20, addr + 4);
+		writel(context->cur_rq_wqe_ba_l, addr + 5);
+		writel(context->qp1c_bytes_28, addr + 6);
+		writel(context->qp1c_bytes_32, addr + 7);
+		writel(context->cur_sq_wqe_ba_l, addr + 8);
+	}
+
+	/* Modify QP1C status */
+	reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG +
+			    hr_qp->port * sizeof(*context));
+	roce_set_field(reg_val, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M,
+		       ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state);
+	roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG +
+		    hr_qp->port * sizeof(*context), reg_val);
+
+	hr_qp->state = new_state;
+	if (new_state == IB_QPS_RESET) {
+		hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
+				     ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
+		if (ibqp->send_cq != ibqp->recv_cq)
+			hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
+					     hr_qp->qpn, NULL);
+
+		hr_qp->rq.head = 0;
+		hr_qp->rq.tail = 0;
+		hr_qp->sq.head = 0;
+		hr_qp->sq.tail = 0;
+		hr_qp->sq_next_wqe = 0;
+	}
+
+	kfree(context);
+	return 0;
+
+out:
+	kfree(context);
+	return -EINVAL;
+}
+
+static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
+			    int attr_mask, enum ib_qp_state cur_state,
+			    enum ib_qp_state new_state)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_qp_context *context;
+	struct hns_roce_rq_db rq_db;
+	dma_addr_t dma_handle_2 = 0;
+	dma_addr_t dma_handle = 0;
+	uint32_t doorbell[2] = {0};
+	int rq_pa_start = 0;
+	u32 reg_val = 0;
+	u64 *mtts_2 = NULL;
+	int ret = -EINVAL;
+	u64 *mtts = NULL;
+	int port;
+	u8 *dmac;
+	u8 *smac;
+
+	context = kzalloc(sizeof(*context), GFP_KERNEL);
+	if (!context)
+		return -ENOMEM;
+
+	/* Search qp buf's mtts */
+	mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
+				   hr_qp->mtt.first_seg, &dma_handle);
+	if (mtts == NULL) {
+		dev_err(dev, "qp buf pa find failed\n");
+		goto out;
+	}
+
+	/* Search IRRL's mtts */
+	mtts_2 = hns_roce_table_find(&hr_dev->qp_table.irrl_table, hr_qp->qpn,
+				     &dma_handle_2);
+	if (mtts_2 == NULL) {
+		dev_err(dev, "qp irrl_table find failed\n");
+		goto out;
+	}
+
+	/*
+	*Reset to init
+	*	Mandatory param:
+	*	IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
+	*	Optional param: NA
+	*/
+	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
+		roce_set_field(context->qpc_bytes_4,
+			       QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
+			       QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
+			       to_hr_qp_type(hr_qp->ibqp.qp_type));
+
+		roce_set_bit(context->qpc_bytes_4,
+			     QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
+		roce_set_bit(context->qpc_bytes_4,
+			     QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
+			     !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
+		roce_set_bit(context->qpc_bytes_4,
+			     QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
+			     !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
+			     );
+		roce_set_bit(context->qpc_bytes_4,
+			     QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S,
+			     !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)
+			     );
+		roce_set_bit(context->qpc_bytes_4,
+			     QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
+		roce_set_field(context->qpc_bytes_4,
+			       QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
+			       QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
+			       ilog2((unsigned int)hr_qp->sq.wqe_cnt));
+		roce_set_field(context->qpc_bytes_4,
+			       QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
+			       QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
+			       ilog2((unsigned int)hr_qp->rq.wqe_cnt));
+		roce_set_field(context->qpc_bytes_4,
+			       QP_CONTEXT_QPC_BYTES_4_PD_M,
+			       QP_CONTEXT_QPC_BYTES_4_PD_S,
+			       to_hr_pd(ibqp->pd)->pdn);
+		hr_qp->access_flags = attr->qp_access_flags;
+		roce_set_field(context->qpc_bytes_8,
+			       QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
+			       QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
+			       to_hr_cq(ibqp->send_cq)->cqn);
+		roce_set_field(context->qpc_bytes_8,
+			       QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
+			       QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
+			       to_hr_cq(ibqp->recv_cq)->cqn);
+
+		if (ibqp->srq)
+			roce_set_field(context->qpc_bytes_12,
+				       QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
+				       QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
+				       to_hr_srq(ibqp->srq)->srqn);
+
+		roce_set_field(context->qpc_bytes_12,
+			       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
+			       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
+			       attr->pkey_index);
+		hr_qp->pkey_index = attr->pkey_index;
+		roce_set_field(context->qpc_bytes_16,
+			       QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
+			       QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
+
+	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
+		roce_set_field(context->qpc_bytes_4,
+			       QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
+			       QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
+			       to_hr_qp_type(hr_qp->ibqp.qp_type));
+		roce_set_bit(context->qpc_bytes_4,
+			     QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
+		if (attr_mask & IB_QP_ACCESS_FLAGS) {
+			roce_set_bit(context->qpc_bytes_4,
+				     QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
+				     !!(attr->qp_access_flags &
+				     IB_ACCESS_REMOTE_READ));
+			roce_set_bit(context->qpc_bytes_4,
+				     QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
+				     !!(attr->qp_access_flags &
+				     IB_ACCESS_REMOTE_WRITE));
+		} else {
+			roce_set_bit(context->qpc_bytes_4,
+				     QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
+				     !!(hr_qp->access_flags &
+				     IB_ACCESS_REMOTE_READ));
+			roce_set_bit(context->qpc_bytes_4,
+				     QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
+				     !!(hr_qp->access_flags &
+				     IB_ACCESS_REMOTE_WRITE));
+		}
+
+		roce_set_bit(context->qpc_bytes_4,
+			     QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
+		roce_set_field(context->qpc_bytes_4,
+			       QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
+			       QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
+			       ilog2((unsigned int)hr_qp->sq.wqe_cnt));
+		roce_set_field(context->qpc_bytes_4,
+			       QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
+			       QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
+			       ilog2((unsigned int)hr_qp->rq.wqe_cnt));
+		roce_set_field(context->qpc_bytes_4,
+			       QP_CONTEXT_QPC_BYTES_4_PD_M,
+			       QP_CONTEXT_QPC_BYTES_4_PD_S,
+			       to_hr_pd(ibqp->pd)->pdn);
+
+		roce_set_field(context->qpc_bytes_8,
+			       QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
+			       QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
+			       to_hr_cq(ibqp->send_cq)->cqn);
+		roce_set_field(context->qpc_bytes_8,
+			       QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
+			       QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
+			       to_hr_cq(ibqp->recv_cq)->cqn);
+
+		if (ibqp->srq)
+			roce_set_field(context->qpc_bytes_12,
+				       QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
+				       QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
+				       to_hr_srq(ibqp->srq)->srqn);
+		if (attr_mask & IB_QP_PKEY_INDEX)
+			roce_set_field(context->qpc_bytes_12,
+				       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
+				       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
+				       attr->pkey_index);
+		else
+			roce_set_field(context->qpc_bytes_12,
+				       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
+				       QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
+				       hr_qp->pkey_index);
+
+		roce_set_field(context->qpc_bytes_16,
+			       QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
+			       QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
+	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
+		if ((attr_mask & IB_QP_ALT_PATH) ||
+		    (attr_mask & IB_QP_ACCESS_FLAGS) ||
+		    (attr_mask & IB_QP_PKEY_INDEX) ||
+		    (attr_mask & IB_QP_QKEY)) {
+			dev_err(dev, "INIT2RTR attr_mask error\n");
+			goto out;
+		}
+
+		dmac = (u8 *)attr->ah_attr.dmac;
+
+		context->sq_rq_bt_l = (u32)(dma_handle);
+		roce_set_field(context->qpc_bytes_24,
+			       QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M,
+			       QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S,
+			       ((u32)(dma_handle >> 32)));
+		roce_set_bit(context->qpc_bytes_24,
+			     QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S,
+			     1);
+		roce_set_field(context->qpc_bytes_24,
+			       QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
+			       QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S,
+			       attr->min_rnr_timer);
+		context->irrl_ba_l = (u32)(dma_handle_2);
+		roce_set_field(context->qpc_bytes_32,
+			       QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M,
+			       QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S,
+			       ((u32)(dma_handle_2 >> 32)) &
+				QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M);
+		roce_set_field(context->qpc_bytes_32,
+			       QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M,
+			       QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S, 0);
+		roce_set_bit(context->qpc_bytes_32,
+			     QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S,
+			     1);
+		roce_set_bit(context->qpc_bytes_32,
+			     QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S,
+			     hr_qp->sq_signal_bits);
+
+		for (port = 0; port < hr_dev->caps.num_ports; port++) {
+			smac = (u8 *)hr_dev->dev_addr[port];
+			dev_dbg(dev, "smac: %2x: %2x: %2x: %2x: %2x: %2x\n",
+				smac[0], smac[1], smac[2], smac[3], smac[4],
+				smac[5]);
+			if ((dmac[0] == smac[0]) && (dmac[1] == smac[1]) &&
+			    (dmac[2] == smac[2]) && (dmac[3] == smac[3]) &&
+			    (dmac[4] == smac[4]) && (dmac[5] == smac[5])) {
+				roce_set_bit(context->qpc_bytes_32,
+				    QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S,
+				    1);
+				break;
+			}
+		}
+
+		if (hr_dev->loop_idc == 0x1)
+			roce_set_bit(context->qpc_bytes_32,
+				QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1);
+
+		roce_set_bit(context->qpc_bytes_32,
+			     QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S,
+			     attr->ah_attr.ah_flags);
+		roce_set_field(context->qpc_bytes_32,
+			       QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
+			       QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S,
+			       ilog2((unsigned int)attr->max_dest_rd_atomic));
+
+		roce_set_field(context->qpc_bytes_36,
+			       QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
+			       QP_CONTEXT_QPC_BYTES_36_DEST_QP_S,
+			       attr->dest_qp_num);
+
+		/* Configure GID index */
+		roce_set_field(context->qpc_bytes_36,
+			       QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
+			       QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S,
+			       hns_get_gid_index(hr_dev,
+						 attr->ah_attr.port_num - 1,
+						 attr->ah_attr.grh.sgid_index));
+
+		memcpy(&(context->dmac_l), dmac, 4);
+
+		roce_set_field(context->qpc_bytes_44,
+			       QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
+			       QP_CONTEXT_QPC_BYTES_44_DMAC_H_S,
+			       *((u16 *)(&dmac[4])));
+		roce_set_field(context->qpc_bytes_44,
+			       QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M,
+			       QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S,
+			       attr->ah_attr.static_rate);
+		roce_set_field(context->qpc_bytes_44,
+			       QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
+			       QP_CONTEXT_QPC_BYTES_44_HOPLMT_S,
+			       attr->ah_attr.grh.hop_limit);
+
+		roce_set_field(context->qpc_bytes_48,
+			       QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
+			       QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S,
+			       attr->ah_attr.grh.flow_label);
+		roce_set_field(context->qpc_bytes_48,
+			       QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
+			       QP_CONTEXT_QPC_BYTES_48_TCLASS_S,
+			       attr->ah_attr.grh.traffic_class);
+		roce_set_field(context->qpc_bytes_48,
+			       QP_CONTEXT_QPC_BYTES_48_MTU_M,
+			       QP_CONTEXT_QPC_BYTES_48_MTU_S, attr->path_mtu);
+
+		memcpy(context->dgid, attr->ah_attr.grh.dgid.raw,
+		       sizeof(attr->ah_attr.grh.dgid.raw));
+
+		dev_dbg(dev, "dmac:%x :%lx\n", context->dmac_l,
+			roce_get_field(context->qpc_bytes_44,
+				       QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
+				       QP_CONTEXT_QPC_BYTES_44_DMAC_H_S));
+
+		roce_set_field(context->qpc_bytes_68,
+			       QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M,
+			       QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S, 0);
+		roce_set_field(context->qpc_bytes_68,
+			       QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M,
+			       QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0);
+
+		rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
+		context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
+
+		roce_set_field(context->qpc_bytes_76,
+			QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M,
+			QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S,
+			mtts[rq_pa_start] >> 32);
+		roce_set_field(context->qpc_bytes_76,
+			       QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M,
+			       QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0);
+
+		context->rx_rnr_time = 0;
+
+		roce_set_field(context->qpc_bytes_84,
+			       QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M,
+			       QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S,
+			       attr->rq_psn - 1);
+		roce_set_field(context->qpc_bytes_84,
+			       QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M,
+			       QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S, 0);
+
+		roce_set_field(context->qpc_bytes_88,
+			       QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
+			       QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S,
+			       attr->rq_psn);
+		roce_set_bit(context->qpc_bytes_88,
+			     QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S, 0);
+		roce_set_bit(context->qpc_bytes_88,
+			     QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S, 0);
+		roce_set_field(context->qpc_bytes_88,
+			QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M,
+			QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S,
+			0);
+		roce_set_field(context->qpc_bytes_88,
+			       QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M,
+			       QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S,
+			       0);
+
+		context->dma_length = 0;
+		context->r_key = 0;
+		context->va_l = 0;
+		context->va_h = 0;
+
+		roce_set_field(context->qpc_bytes_108,
+			       QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M,
+			       QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S, 0);
+		roce_set_bit(context->qpc_bytes_108,
+			     QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S, 0);
+		roce_set_bit(context->qpc_bytes_108,
+			     QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S, 0);
+
+		roce_set_field(context->qpc_bytes_112,
+			       QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M,
+			       QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S, 0);
+		roce_set_field(context->qpc_bytes_112,
+			       QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M,
+			       QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S, 0);
+
+		/* For chip resp ack */
+		roce_set_field(context->qpc_bytes_156,
+			       QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
+			       QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
+			       hr_qp->port);
+		roce_set_field(context->qpc_bytes_156,
+			       QP_CONTEXT_QPC_BYTES_156_SL_M,
+			       QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
+		hr_qp->sl = attr->ah_attr.sl;
+	} else if (cur_state == IB_QPS_RTR &&
+		new_state == IB_QPS_RTS) {
+		/* If exist optional param, return error */
+		if ((attr_mask & IB_QP_ALT_PATH) ||
+		    (attr_mask & IB_QP_ACCESS_FLAGS) ||
+		    (attr_mask & IB_QP_QKEY) ||
+		    (attr_mask & IB_QP_PATH_MIG_STATE) ||
+		    (attr_mask & IB_QP_CUR_STATE) ||
+		    (attr_mask & IB_QP_MIN_RNR_TIMER)) {
+			dev_err(dev, "RTR2RTS attr_mask error\n");
+			goto out;
+		}
+
+		context->rx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
+
+		roce_set_field(context->qpc_bytes_120,
+			       QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M,
+			       QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S,
+			       (mtts[0]) >> 32);
+
+		roce_set_field(context->qpc_bytes_124,
+			       QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M,
+			       QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S, 0);
+		roce_set_field(context->qpc_bytes_124,
+			       QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M,
+			       QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S, 0);
+
+		roce_set_field(context->qpc_bytes_128,
+			       QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M,
+			       QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S,
+			       attr->sq_psn);
+		roce_set_bit(context->qpc_bytes_128,
+			     QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S, 0);
+		roce_set_field(context->qpc_bytes_128,
+			     QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M,
+			     QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S,
+			     0);
+		roce_set_bit(context->qpc_bytes_128,
+			     QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S, 0);
+
+		roce_set_field(context->qpc_bytes_132,
+			       QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M,
+			       QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S, 0);
+		roce_set_field(context->qpc_bytes_132,
+			       QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M,
+			       QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S, 0);
+
+		roce_set_field(context->qpc_bytes_136,
+			       QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M,
+			       QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S,
+			       attr->sq_psn);
+		roce_set_field(context->qpc_bytes_136,
+			       QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M,
+			       QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S,
+			       attr->sq_psn);
+
+		roce_set_field(context->qpc_bytes_140,
+			       QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M,
+			       QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S,
+			       (attr->sq_psn >> SQ_PSN_SHIFT));
+		roce_set_field(context->qpc_bytes_140,
+			       QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M,
+			       QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S, 0);
+		roce_set_bit(context->qpc_bytes_140,
+			     QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S, 0);
+
+		roce_set_field(context->qpc_bytes_144,
+			       QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
+			       QP_CONTEXT_QPC_BYTES_144_QP_STATE_S,
+			       attr->qp_state);
+
+		roce_set_field(context->qpc_bytes_148,
+			       QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M,
+			       QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S, 0);
+		roce_set_field(context->qpc_bytes_148,
+			       QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
+			       QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S, 0);
+		roce_set_field(context->qpc_bytes_148,
+			       QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M,
+			       QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S, 0);
+		roce_set_field(context->qpc_bytes_148,
+			       QP_CONTEXT_QPC_BYTES_148_LSN_M,
+			       QP_CONTEXT_QPC_BYTES_148_LSN_S, 0x100);
+
+		context->rnr_retry = 0;
+
+		roce_set_field(context->qpc_bytes_156,
+			       QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M,
+			       QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S,
+			       attr->retry_cnt);
+		roce_set_field(context->qpc_bytes_156,
+			       QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
+			       QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
+			       attr->timeout);
+		roce_set_field(context->qpc_bytes_156,
+			       QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M,
+			       QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S,
+			       attr->rnr_retry);
+		roce_set_field(context->qpc_bytes_156,
+			       QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
+			       QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
+			       hr_qp->port);
+		roce_set_field(context->qpc_bytes_156,
+			       QP_CONTEXT_QPC_BYTES_156_SL_M,
+			       QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
+		hr_qp->sl = attr->ah_attr.sl;
+		roce_set_field(context->qpc_bytes_156,
+			       QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
+			       QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S,
+			       ilog2((unsigned int)attr->max_rd_atomic));
+		roce_set_field(context->qpc_bytes_156,
+			       QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M,
+			       QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S, 0);
+		context->pkt_use_len = 0;
+
+		roce_set_field(context->qpc_bytes_164,
+			       QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
+			       QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S, attr->sq_psn);
+		roce_set_field(context->qpc_bytes_164,
+			       QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M,
+			       QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S, 0);
+
+		roce_set_field(context->qpc_bytes_168,
+			       QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M,
+			       QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S,
+			       attr->sq_psn);
+		roce_set_field(context->qpc_bytes_168,
+			       QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M,
+			       QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S, 0);
+		roce_set_field(context->qpc_bytes_168,
+			       QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M,
+			       QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S, 0);
+		roce_set_bit(context->qpc_bytes_168,
+			     QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S, 0);
+		roce_set_bit(context->qpc_bytes_168,
+			     QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S, 0);
+		roce_set_bit(context->qpc_bytes_168,
+			     QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S, 0);
+		context->sge_use_len = 0;
+
+		roce_set_field(context->qpc_bytes_176,
+			       QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M,
+			       QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S, 0);
+		roce_set_field(context->qpc_bytes_176,
+			       QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M,
+			       QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S,
+			       0);
+		roce_set_field(context->qpc_bytes_180,
+			       QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M,
+			       QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S, 0);
+		roce_set_field(context->qpc_bytes_180,
+			       QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M,
+			       QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0);
+
+		context->tx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
+
+		roce_set_field(context->qpc_bytes_188,
+			       QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M,
+			       QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S,
+			       (mtts[0]) >> 32);
+		roce_set_bit(context->qpc_bytes_188,
+			     QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0);
+		roce_set_field(context->qpc_bytes_188,
+			       QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M,
+			       QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S,
+			       0);
+	} else if ((cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) ||
+		   (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) ||
+		   (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) ||
+		   (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) ||
+		   (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) ||
+		   (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) ||
+		   (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) ||
+		   (cur_state == IB_QPS_ERR && new_state == IB_QPS_ERR)) {
+		roce_set_field(context->qpc_bytes_144,
+			       QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
+			       QP_CONTEXT_QPC_BYTES_144_QP_STATE_S,
+			       attr->qp_state);
+
+	} else {
+		dev_err(dev, "not support this modify\n");
+		goto out;
+	}
+
+	/* Every status migrate must change state */
+	roce_set_field(context->qpc_bytes_144,
+		       QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
+		       QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, attr->qp_state);
+
+	/* SW pass context to HW */
+	ret = hns_roce_v1_qp_modify(hr_dev, &hr_qp->mtt,
+				    to_hns_roce_state(cur_state),
+				    to_hns_roce_state(new_state), context,
+				    hr_qp);
+	if (ret) {
+		dev_err(dev, "hns_roce_qp_modify failed\n");
+		goto out;
+	}
+
+	/*
+	* Use rst2init to instead of init2init with drv,
+	* need to hw to flash RQ HEAD by DB again
+	*/
+	if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
+		/* Memory barrier */
+		wmb();
+		if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {
+			/* SW update GSI rq header */
+			reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG3_0_REG +
+					    QP1C_CFGN_OFFSET * hr_qp->port);
+			roce_set_field(reg_val,
+				       ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,
+				       ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,
+				       hr_qp->rq.head);
+			roce_write(hr_dev, ROCEE_QP1C_CFG3_0_REG +
+				    QP1C_CFGN_OFFSET * hr_qp->port, reg_val);
+		} else {
+			rq_db.u32_4 = 0;
+			rq_db.u32_8 = 0;
+
+			roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
+				       RQ_DOORBELL_U32_4_RQ_HEAD_S,
+				       hr_qp->rq.head);
+			roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,
+				       RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
+			roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,
+				       RQ_DOORBELL_U32_8_CMD_S, 1);
+			roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
+				     1);
+
+			doorbell[0] = rq_db.u32_4;
+			doorbell[1] = rq_db.u32_8;
+
+			hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
+		}
+	}
+
+	hr_qp->state = new_state;
+
+	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
+		hr_qp->resp_depth = attr->max_dest_rd_atomic;
+	if (attr_mask & IB_QP_PORT)
+		hr_qp->port = (attr->port_num - 1);
+
+	if (new_state == IB_QPS_RESET && !ibqp->uobject) {
+		hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
+				     ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
+		if (ibqp->send_cq != ibqp->recv_cq)
+			hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
+					     hr_qp->qpn, NULL);
+
+		hr_qp->rq.head = 0;
+		hr_qp->rq.tail = 0;
+		hr_qp->sq.head = 0;
+		hr_qp->sq.tail = 0;
+		hr_qp->sq_next_wqe = 0;
+	}
+out:
+	kfree(context);
+	return ret;
+}
+
+int hns_roce_v1_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
+			  int attr_mask, enum ib_qp_state cur_state,
+			  enum ib_qp_state new_state)
+{
+
+	if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
+		return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state,
+					 new_state);
+	else
+		return hns_roce_v1_m_qp(ibqp, attr, attr_mask, cur_state,
+					new_state);
+}
+
+static enum ib_qp_state to_ib_qp_state(enum hns_roce_qp_state state)
+{
+	switch (state) {
+	case HNS_ROCE_QP_STATE_RST:
+		return IB_QPS_RESET;
+	case HNS_ROCE_QP_STATE_INIT:
+		return IB_QPS_INIT;
+	case HNS_ROCE_QP_STATE_RTR:
+		return IB_QPS_RTR;
+	case HNS_ROCE_QP_STATE_RTS:
+		return IB_QPS_RTS;
+	case HNS_ROCE_QP_STATE_SQD:
+		return IB_QPS_SQD;
+	case HNS_ROCE_QP_STATE_ERR:
+		return IB_QPS_ERR;
+	default:
+		return IB_QPS_ERR;
+	}
+}
+
+static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev,
+				 struct hns_roce_qp *hr_qp,
+				 struct hns_roce_qp_context *hr_context)
+{
+	struct hns_roce_cmd_mailbox *mailbox;
+	int ret;
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
+				HNS_ROCE_CMD_QUERY_QP,
+				HNS_ROCE_CMD_TIME_CLASS_A);
+	if (!ret)
+		memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
+	else
+		dev_err(&hr_dev->pdev->dev, "QUERY QP cmd process error\n");
+
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return ret;
+}
+
+int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+			 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_qp_context *context;
+	int tmp_qp_state = 0;
+	int ret = 0;
+	int state;
+
+	context = kzalloc(sizeof(*context), GFP_KERNEL);
+	if (!context)
+		return -ENOMEM;
+
+	memset(qp_attr, 0, sizeof(*qp_attr));
+	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
+
+	mutex_lock(&hr_qp->mutex);
+
+	if (hr_qp->state == IB_QPS_RESET) {
+		qp_attr->qp_state = IB_QPS_RESET;
+		goto done;
+	}
+
+	ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context);
+	if (ret) {
+		dev_err(dev, "query qpc error\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	state = roce_get_field(context->qpc_bytes_144,
+			       QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
+			       QP_CONTEXT_QPC_BYTES_144_QP_STATE_S);
+	tmp_qp_state = (int)to_ib_qp_state((enum hns_roce_qp_state)state);
+	if (tmp_qp_state == -1) {
+		dev_err(dev, "to_ib_qp_state error\n");
+		ret = -EINVAL;
+		goto out;
+	}
+	hr_qp->state = (u8)tmp_qp_state;
+	qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
+	qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->qpc_bytes_48,
+					       QP_CONTEXT_QPC_BYTES_48_MTU_M,
+					       QP_CONTEXT_QPC_BYTES_48_MTU_S);
+	qp_attr->path_mig_state = IB_MIG_ARMED;
+	if (hr_qp->ibqp.qp_type == IB_QPT_UD)
+		qp_attr->qkey = QKEY_VAL;
+
+	qp_attr->rq_psn = roce_get_field(context->qpc_bytes_88,
+					 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
+					 QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S);
+	qp_attr->sq_psn = (u32)roce_get_field(context->qpc_bytes_164,
+					     QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
+					     QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S);
+	qp_attr->dest_qp_num = (u8)roce_get_field(context->qpc_bytes_36,
+					QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
+					QP_CONTEXT_QPC_BYTES_36_DEST_QP_S);
+	qp_attr->qp_access_flags = ((roce_get_bit(context->qpc_bytes_4,
+			QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S)) << 2) |
+				   ((roce_get_bit(context->qpc_bytes_4,
+			QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S)) << 1) |
+				   ((roce_get_bit(context->qpc_bytes_4,
+			QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) << 3);
+
+	if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
+	    hr_qp->ibqp.qp_type == IB_QPT_UC) {
+		qp_attr->ah_attr.sl = roce_get_field(context->qpc_bytes_156,
+						QP_CONTEXT_QPC_BYTES_156_SL_M,
+						QP_CONTEXT_QPC_BYTES_156_SL_S);
+		qp_attr->ah_attr.grh.flow_label = roce_get_field(
+					context->qpc_bytes_48,
+					QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
+					QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S);
+		qp_attr->ah_attr.grh.sgid_index = roce_get_field(
+					context->qpc_bytes_36,
+					QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
+					QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S);
+		qp_attr->ah_attr.grh.hop_limit = roce_get_field(
+					context->qpc_bytes_44,
+					QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
+					QP_CONTEXT_QPC_BYTES_44_HOPLMT_S);
+		qp_attr->ah_attr.grh.traffic_class = roce_get_field(
+					context->qpc_bytes_48,
+					QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
+					QP_CONTEXT_QPC_BYTES_48_TCLASS_S);
+
+		memcpy(qp_attr->ah_attr.grh.dgid.raw, context->dgid,
+		       sizeof(qp_attr->ah_attr.grh.dgid.raw));
+	}
+
+	qp_attr->pkey_index = roce_get_field(context->qpc_bytes_12,
+			      QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
+			      QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S);
+	qp_attr->port_num = (u8)roce_get_field(context->qpc_bytes_156,
+			     QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
+			     QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S) + 1;
+	qp_attr->sq_draining = 0;
+	qp_attr->max_rd_atomic = roce_get_field(context->qpc_bytes_156,
+				 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
+				 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S);
+	qp_attr->max_dest_rd_atomic = roce_get_field(context->qpc_bytes_32,
+				 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
+				 QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S);
+	qp_attr->min_rnr_timer = (u8)(roce_get_field(context->qpc_bytes_24,
+			QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
+			QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S));
+	qp_attr->timeout = (u8)(roce_get_field(context->qpc_bytes_156,
+			    QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
+			    QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S));
+	qp_attr->retry_cnt = roce_get_field(context->qpc_bytes_148,
+			     QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
+			     QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S);
+	qp_attr->rnr_retry = context->rnr_retry;
+
+done:
+	qp_attr->cur_qp_state = qp_attr->qp_state;
+	qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
+	qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
+
+	if (!ibqp->uobject) {
+		qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
+		qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
+	} else {
+		qp_attr->cap.max_send_wr = 0;
+		qp_attr->cap.max_send_sge = 0;
+	}
+
+	qp_init_attr->cap = qp_attr->cap;
+
+out:
+	mutex_unlock(&hr_qp->mutex);
+	kfree(context);
+	return ret;
+}
+
+static void hns_roce_v1_destroy_qp_common(struct hns_roce_dev *hr_dev,
+					  struct hns_roce_qp *hr_qp,
+					  int is_user)
+{
+	u32 sdbinvcnt;
+	unsigned long end = 0;
+	u32 sdbinvcnt_val;
+	u32 sdbsendptr_val;
+	u32 sdbisusepr_val;
+	struct hns_roce_cq *send_cq, *recv_cq;
+	struct device *dev = &hr_dev->pdev->dev;
+
+	if (hr_qp->ibqp.qp_type == IB_QPT_RC) {
+		if (hr_qp->state != IB_QPS_RESET) {
+			/*
+			* Set qp to ERR,
+			* waiting for hw complete processing all dbs
+			*/
+			if (hns_roce_v1_qp_modify(hr_dev, NULL,
+					to_hns_roce_state(
+						(enum ib_qp_state)hr_qp->state),
+						HNS_ROCE_QP_STATE_ERR, NULL,
+						hr_qp))
+				dev_err(dev, "modify QP %06lx to ERR failed.\n",
+					hr_qp->qpn);
+
+			/* Record issued doorbell */
+			sdbisusepr_val = roce_read(hr_dev,
+					 ROCEE_SDB_ISSUE_PTR_REG);
+			/*
+			* Query db process status,
+			* until hw process completely
+			*/
+			end = msecs_to_jiffies(
+			      HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS) + jiffies;
+			do {
+				sdbsendptr_val = roce_read(hr_dev,
+						 ROCEE_SDB_SEND_PTR_REG);
+				if (!time_before(jiffies, end)) {
+					dev_err(dev, "destroy qp(0x%lx) timeout!!!",
+						hr_qp->qpn);
+					break;
+				}
+			} while ((short)(roce_get_field(sdbsendptr_val,
+					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
+					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) -
+				roce_get_field(sdbisusepr_val,
+					ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M,
+					ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S)
+				) < 0);
+
+			/* Get list pointer */
+			sdbinvcnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
+
+			/* Query db's list status, until hw reversal */
+			do {
+				sdbinvcnt_val = roce_read(hr_dev,
+						ROCEE_SDB_INV_CNT_REG);
+				if (!time_before(jiffies, end)) {
+					dev_err(dev, "destroy qp(0x%lx) timeout!!!",
+						hr_qp->qpn);
+					dev_err(dev, "SdbInvCnt = 0x%x\n",
+						sdbinvcnt_val);
+					break;
+				}
+			} while ((short)(roce_get_field(sdbinvcnt_val,
+				  ROCEE_SDB_INV_CNT_SDB_INV_CNT_M,
+				  ROCEE_SDB_INV_CNT_SDB_INV_CNT_S) -
+				  (sdbinvcnt + SDB_INV_CNT_OFFSET)) < 0);
+
+			/* Modify qp to reset before destroying qp */
+			if (hns_roce_v1_qp_modify(hr_dev, NULL,
+					to_hns_roce_state(
+					(enum ib_qp_state)hr_qp->state),
+					HNS_ROCE_QP_STATE_RST, NULL, hr_qp))
+				dev_err(dev, "modify QP %06lx to RESET failed.\n",
+					hr_qp->qpn);
+		}
+	}
+
+	send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
+	recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
+
+	hns_roce_lock_cqs(send_cq, recv_cq);
+
+	if (!is_user) {
+		__hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
+				       to_hr_srq(hr_qp->ibqp.srq) : NULL);
+		if (send_cq != recv_cq)
+			__hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL);
+	}
+
+	hns_roce_qp_remove(hr_dev, hr_qp);
+
+	hns_roce_unlock_cqs(send_cq, recv_cq);
+
+	hns_roce_qp_free(hr_dev, hr_qp);
+
+	/* Not special_QP, free their QPN */
+	if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
+	    (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
+	    (hr_qp->ibqp.qp_type == IB_QPT_UD))
+		hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
+
+	hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
+
+	if (is_user) {
+		ib_umem_release(hr_qp->umem);
+	} else {
+		kfree(hr_qp->sq.wrid);
+		kfree(hr_qp->rq.wrid);
+		hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
+	}
+}
+
+int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+
+	hns_roce_v1_destroy_qp_common(hr_dev, hr_qp, !!ibqp->pd->uobject);
+
+	if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
+		kfree(hr_to_hr_sqp(hr_qp));
+	else
+		kfree(hr_qp);
+
+	return 0;
+}
+
+struct hns_roce_v1_priv hr_v1_priv;
+
+struct hns_roce_hw hns_roce_hw_v1 = {
+	.reset = hns_roce_v1_reset,
+	.hw_profile = hns_roce_v1_profile,
+	.hw_init = hns_roce_v1_init,
+	.hw_exit = hns_roce_v1_exit,
+	.set_gid = hns_roce_v1_set_gid,
+	.set_mac = hns_roce_v1_set_mac,
+	.set_mtu = hns_roce_v1_set_mtu,
+	.write_mtpt = hns_roce_v1_write_mtpt,
+	.write_cqc = hns_roce_v1_write_cqc,
+	.modify_qp = hns_roce_v1_modify_qp,
+	.query_qp = hns_roce_v1_query_qp,
+	.destroy_qp = hns_roce_v1_destroy_qp,
+	.post_send = hns_roce_v1_post_send,
+	.post_recv = hns_roce_v1_post_recv,
+	.req_notify_cq = hns_roce_v1_req_notify_cq,
+	.poll_cq = hns_roce_v1_poll_cq,
+	.priv = &hr_v1_priv,
+};
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
new file mode 100644
index 000000000000..316b592b1636
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
@@ -0,0 +1,981 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _HNS_ROCE_HW_V1_H
+#define _HNS_ROCE_HW_V1_H
+
+#define CQ_STATE_VALID					2
+
+#define HNS_ROCE_V1_MAX_PD_NUM				0x8000
+#define HNS_ROCE_V1_MAX_CQ_NUM				0x10000
+#define HNS_ROCE_V1_MAX_CQE_NUM				0x8000
+
+#define HNS_ROCE_V1_MAX_QP_NUM				0x40000
+#define HNS_ROCE_V1_MAX_WQE_NUM				0x4000
+
+#define HNS_ROCE_V1_MAX_MTPT_NUM			0x80000
+
+#define HNS_ROCE_V1_MAX_MTT_SEGS			0x100000
+
+#define HNS_ROCE_V1_MAX_QP_INIT_RDMA			128
+#define HNS_ROCE_V1_MAX_QP_DEST_RDMA			128
+
+#define HNS_ROCE_V1_MAX_SQ_DESC_SZ			64
+#define HNS_ROCE_V1_MAX_RQ_DESC_SZ			64
+#define HNS_ROCE_V1_SG_NUM				2
+#define HNS_ROCE_V1_INLINE_SIZE				32
+
+#define HNS_ROCE_V1_UAR_NUM				256
+#define HNS_ROCE_V1_PHY_UAR_NUM				8
+
+#define HNS_ROCE_V1_GID_NUM				16
+
+#define HNS_ROCE_V1_NUM_COMP_EQE			0x8000
+#define HNS_ROCE_V1_NUM_ASYNC_EQE			0x400
+
+#define HNS_ROCE_V1_QPC_ENTRY_SIZE			256
+#define HNS_ROCE_V1_IRRL_ENTRY_SIZE			8
+#define HNS_ROCE_V1_CQC_ENTRY_SIZE			64
+#define HNS_ROCE_V1_MTPT_ENTRY_SIZE			64
+#define HNS_ROCE_V1_MTT_ENTRY_SIZE			64
+
+#define HNS_ROCE_V1_CQE_ENTRY_SIZE			32
+#define HNS_ROCE_V1_PAGE_SIZE_SUPPORT			0xFFFFF000
+
+#define HNS_ROCE_V1_EXT_RAQ_WF				8
+#define HNS_ROCE_V1_RAQ_ENTRY				64
+#define HNS_ROCE_V1_RAQ_DEPTH				32768
+#define HNS_ROCE_V1_RAQ_SIZE	(HNS_ROCE_V1_RAQ_ENTRY * HNS_ROCE_V1_RAQ_DEPTH)
+
+#define HNS_ROCE_V1_SDB_DEPTH				0x400
+#define HNS_ROCE_V1_ODB_DEPTH				0x400
+
+#define HNS_ROCE_V1_DB_RSVD				0x80
+
+#define HNS_ROCE_V1_SDB_ALEPT				HNS_ROCE_V1_DB_RSVD
+#define HNS_ROCE_V1_SDB_ALFUL	(HNS_ROCE_V1_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD)
+#define HNS_ROCE_V1_ODB_ALEPT				HNS_ROCE_V1_DB_RSVD
+#define HNS_ROCE_V1_ODB_ALFUL	(HNS_ROCE_V1_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)
+
+#define HNS_ROCE_V1_EXT_SDB_DEPTH			0x4000
+#define HNS_ROCE_V1_EXT_ODB_DEPTH			0x4000
+#define HNS_ROCE_V1_EXT_SDB_ENTRY			16
+#define HNS_ROCE_V1_EXT_ODB_ENTRY			16
+#define HNS_ROCE_V1_EXT_SDB_SIZE  \
+	(HNS_ROCE_V1_EXT_SDB_DEPTH * HNS_ROCE_V1_EXT_SDB_ENTRY)
+#define HNS_ROCE_V1_EXT_ODB_SIZE  \
+	(HNS_ROCE_V1_EXT_ODB_DEPTH * HNS_ROCE_V1_EXT_ODB_ENTRY)
+
+#define HNS_ROCE_V1_EXT_SDB_ALEPT			HNS_ROCE_V1_DB_RSVD
+#define HNS_ROCE_V1_EXT_SDB_ALFUL  \
+	(HNS_ROCE_V1_EXT_SDB_DEPTH - HNS_ROCE_V1_DB_RSVD)
+#define HNS_ROCE_V1_EXT_ODB_ALEPT			HNS_ROCE_V1_DB_RSVD
+#define HNS_ROCE_V1_EXT_ODB_ALFUL	\
+	(HNS_ROCE_V1_EXT_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)
+
+#define HNS_ROCE_ODB_POLL_MODE				0
+
+#define HNS_ROCE_SDB_NORMAL_MODE			0
+#define HNS_ROCE_SDB_EXTEND_MODE			1
+
+#define HNS_ROCE_ODB_EXTEND_MODE			1
+
+#define KEY_VALID					0x02
+
+#define HNS_ROCE_CQE_QPN_MASK				0x3ffff
+#define HNS_ROCE_CQE_STATUS_MASK			0x1f
+#define HNS_ROCE_CQE_OPCODE_MASK			0xf
+
+#define HNS_ROCE_CQE_SUCCESS				0x00
+#define HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR		0x01
+#define HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR		0x02
+#define HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR		0x03
+#define HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR		0x04
+#define HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR	0x05
+#define HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR		0x06
+#define HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR		0x07
+#define HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR	0x08
+#define HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR		0x09
+#define HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR		0x0a
+#define HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR	0x0b
+#define HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR		0x0c
+
+#define QP1C_CFGN_OFFSET				0x28
+#define PHY_PORT_OFFSET					0x8
+#define MTPT_IDX_SHIFT					16
+#define ALL_PORT_VAL_OPEN				0x3f
+#define POL_TIME_INTERVAL_VAL				0x80
+#define SLEEP_TIME_INTERVAL				20
+#define SQ_PSN_SHIFT					8
+#define QKEY_VAL					0x80010000
+#define SDB_INV_CNT_OFFSET				8
+
+struct hns_roce_cq_context {
+	u32 cqc_byte_4;
+	u32 cq_bt_l;
+	u32 cqc_byte_12;
+	u32 cur_cqe_ba0_l;
+	u32 cqc_byte_20;
+	u32 cqe_tptr_addr_l;
+	u32 cur_cqe_ba1_l;
+	u32 cqc_byte_32;
+};
+
+#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S 0
+#define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M   \
+	(((1UL << 2) - 1) << CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S)
+
+#define CQ_CONTEXT_CQC_BYTE_4_CQN_S 16
+#define CQ_CONTEXT_CQC_BYTE_4_CQN_M   \
+	(((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_4_CQN_S)
+
+#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S 0
+#define CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M   \
+	(((1UL << 17) - 1) << CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S)
+
+#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S 20
+#define CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M   \
+	(((1UL << 4) - 1) << CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S)
+
+#define CQ_CONTEXT_CQC_BYTE_12_CEQN_S 24
+#define CQ_CONTEXT_CQC_BYTE_12_CEQN_M   \
+	(((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_12_CEQN_S)
+
+#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S 0
+#define CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M   \
+	(((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S)
+
+#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S 16
+#define CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M   \
+	(((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S)
+
+#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S 8
+#define CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M   \
+	(((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S)
+
+#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S 0
+#define CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M   \
+	(((1UL << 5) - 1) << CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S)
+
+#define CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S 9
+
+#define CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S 8
+#define CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S 14
+#define CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S 15
+
+#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S 16
+#define CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M   \
+	(((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S)
+
+struct hns_roce_cqe {
+	u32 cqe_byte_4;
+	union {
+		u32 r_key;
+		u32 immediate_data;
+	};
+	u32 byte_cnt;
+	u32 cqe_byte_16;
+	u32 cqe_byte_20;
+	u32 s_mac_l;
+	u32 cqe_byte_28;
+	u32 reserved;
+};
+
+#define CQE_BYTE_4_OWNER_S 7
+#define CQE_BYTE_4_SQ_RQ_FLAG_S 14
+
+#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_S 8
+#define CQE_BYTE_4_STATUS_OF_THE_OPERATION_M   \
+	(((1UL << 5) - 1) << CQE_BYTE_4_STATUS_OF_THE_OPERATION_S)
+
+#define CQE_BYTE_4_WQE_INDEX_S 16
+#define CQE_BYTE_4_WQE_INDEX_M	(((1UL << 14) - 1) << CQE_BYTE_4_WQE_INDEX_S)
+
+#define CQE_BYTE_4_OPERATION_TYPE_S 0
+#define CQE_BYTE_4_OPERATION_TYPE_M   \
+	(((1UL << 4) - 1) << CQE_BYTE_4_OPERATION_TYPE_S)
+
+#define CQE_BYTE_4_IMM_INDICATOR_S 15
+
+#define CQE_BYTE_16_LOCAL_QPN_S 0
+#define CQE_BYTE_16_LOCAL_QPN_M	(((1UL << 24) - 1) << CQE_BYTE_16_LOCAL_QPN_S)
+
+#define CQE_BYTE_20_PORT_NUM_S 26
+#define CQE_BYTE_20_PORT_NUM_M	(((1UL << 3) - 1) << CQE_BYTE_20_PORT_NUM_S)
+
+#define CQE_BYTE_20_SL_S 24
+#define CQE_BYTE_20_SL_M	(((1UL << 2) - 1) << CQE_BYTE_20_SL_S)
+
+#define CQE_BYTE_20_REMOTE_QPN_S 0
+#define CQE_BYTE_20_REMOTE_QPN_M   \
+	(((1UL << 24) - 1) << CQE_BYTE_20_REMOTE_QPN_S)
+
+#define CQE_BYTE_20_GRH_PRESENT_S 29
+
+#define CQE_BYTE_28_P_KEY_IDX_S 16
+#define CQE_BYTE_28_P_KEY_IDX_M	(((1UL << 16) - 1) << CQE_BYTE_28_P_KEY_IDX_S)
+
+#define CQ_DB_REQ_NOT_SOL	0
+#define CQ_DB_REQ_NOT		(1 << 16)
+
+struct hns_roce_v1_mpt_entry {
+	u32  mpt_byte_4;
+	u32  pbl_addr_l;
+	u32  mpt_byte_12;
+	u32  virt_addr_l;
+	u32  virt_addr_h;
+	u32  length;
+	u32  mpt_byte_28;
+	u32  pa0_l;
+	u32  mpt_byte_36;
+	u32  mpt_byte_40;
+	u32  mpt_byte_44;
+	u32  mpt_byte_48;
+	u32  pa4_l;
+	u32  mpt_byte_56;
+	u32  mpt_byte_60;
+	u32  mpt_byte_64;
+};
+
+#define MPT_BYTE_4_KEY_STATE_S 0
+#define MPT_BYTE_4_KEY_STATE_M	(((1UL << 2) - 1) << MPT_BYTE_4_KEY_STATE_S)
+
+#define MPT_BYTE_4_KEY_S 8
+#define MPT_BYTE_4_KEY_M	(((1UL << 8) - 1) << MPT_BYTE_4_KEY_S)
+
+#define MPT_BYTE_4_PAGE_SIZE_S 16
+#define MPT_BYTE_4_PAGE_SIZE_M	(((1UL << 2) - 1) << MPT_BYTE_4_PAGE_SIZE_S)
+
+#define MPT_BYTE_4_MW_TYPE_S 20
+
+#define MPT_BYTE_4_MW_BIND_ENABLE_S 21
+
+#define MPT_BYTE_4_OWN_S 22
+
+#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_S 24
+#define MPT_BYTE_4_MEMORY_LOCATION_TYPE_M   \
+	(((1UL << 2) - 1) << MPT_BYTE_4_MEMORY_LOCATION_TYPE_S)
+
+#define MPT_BYTE_4_REMOTE_ATOMIC_S 26
+#define MPT_BYTE_4_LOCAL_WRITE_S 27
+#define MPT_BYTE_4_REMOTE_WRITE_S 28
+#define MPT_BYTE_4_REMOTE_READ_S 29
+#define MPT_BYTE_4_REMOTE_INVAL_ENABLE_S 30
+#define MPT_BYTE_4_ADDRESS_TYPE_S 31
+
+#define MPT_BYTE_12_PBL_ADDR_H_S 0
+#define MPT_BYTE_12_PBL_ADDR_H_M   \
+	(((1UL << 17) - 1) << MPT_BYTE_12_PBL_ADDR_H_S)
+
+#define MPT_BYTE_12_MW_BIND_COUNTER_S 17
+#define MPT_BYTE_12_MW_BIND_COUNTER_M   \
+	(((1UL << 15) - 1) << MPT_BYTE_12_MW_BIND_COUNTER_S)
+
+#define MPT_BYTE_28_PD_S 0
+#define MPT_BYTE_28_PD_M	(((1UL << 16) - 1) << MPT_BYTE_28_PD_S)
+
+#define MPT_BYTE_28_L_KEY_IDX_L_S 16
+#define MPT_BYTE_28_L_KEY_IDX_L_M   \
+	(((1UL << 16) - 1) << MPT_BYTE_28_L_KEY_IDX_L_S)
+
+#define MPT_BYTE_36_PA0_H_S 0
+#define MPT_BYTE_36_PA0_H_M	(((1UL << 5) - 1) << MPT_BYTE_36_PA0_H_S)
+
+#define MPT_BYTE_36_PA1_L_S 8
+#define MPT_BYTE_36_PA1_L_M	(((1UL << 24) - 1) << MPT_BYTE_36_PA1_L_S)
+
+#define MPT_BYTE_40_PA1_H_S 0
+#define MPT_BYTE_40_PA1_H_M	(((1UL << 13) - 1) << MPT_BYTE_40_PA1_H_S)
+
+#define MPT_BYTE_40_PA2_L_S 16
+#define MPT_BYTE_40_PA2_L_M	(((1UL << 16) - 1) << MPT_BYTE_40_PA2_L_S)
+
+#define MPT_BYTE_44_PA2_H_S 0
+#define MPT_BYTE_44_PA2_H_M	(((1UL << 21) - 1) << MPT_BYTE_44_PA2_H_S)
+
+#define MPT_BYTE_44_PA3_L_S 24
+#define MPT_BYTE_44_PA3_L_M	(((1UL << 8) - 1) << MPT_BYTE_44_PA3_L_S)
+
+#define MPT_BYTE_48_PA3_H_S 0
+#define MPT_BYTE_48_PA3_H_M	(((1UL << 29) - 1) << MPT_BYTE_48_PA3_H_S)
+
+#define MPT_BYTE_56_PA4_H_S 0
+#define MPT_BYTE_56_PA4_H_M	(((1UL << 5) - 1) << MPT_BYTE_56_PA4_H_S)
+
+#define MPT_BYTE_56_PA5_L_S 8
+#define MPT_BYTE_56_PA5_L_M	(((1UL << 24) - 1) << MPT_BYTE_56_PA5_L_S)
+
+#define MPT_BYTE_60_PA5_H_S 0
+#define MPT_BYTE_60_PA5_H_M	(((1UL << 13) - 1) << MPT_BYTE_60_PA5_H_S)
+
+#define MPT_BYTE_60_PA6_L_S 16
+#define MPT_BYTE_60_PA6_L_M	(((1UL << 16) - 1) << MPT_BYTE_60_PA6_L_S)
+
+#define MPT_BYTE_64_PA6_H_S 0
+#define MPT_BYTE_64_PA6_H_M	(((1UL << 21) - 1) << MPT_BYTE_64_PA6_H_S)
+
+#define MPT_BYTE_64_L_KEY_IDX_H_S 24
+#define MPT_BYTE_64_L_KEY_IDX_H_M   \
+	(((1UL << 8) - 1) << MPT_BYTE_64_L_KEY_IDX_H_S)
+
+struct hns_roce_wqe_ctrl_seg {
+	__be32 sgl_pa_h;
+	__be32 flag;
+	__be32 imm_data;
+	__be32 msg_length;
+};
+
+struct hns_roce_wqe_data_seg {
+	__be64    addr;
+	__be32    lkey;
+	__be32    len;
+};
+
+struct hns_roce_wqe_raddr_seg {
+	__be32 rkey;
+	__be32 len;/* reserved */
+	__be64 raddr;
+};
+
+struct hns_roce_rq_wqe_ctrl {
+
+	u32 rwqe_byte_4;
+	u32 rocee_sgl_ba_l;
+	u32 rwqe_byte_12;
+	u32 reserved[5];
+};
+
+#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S 16
+#define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M   \
+	(((1UL << 6) - 1) << RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S)
+
+#define HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS	10000
+
+#define GID_LEN					16
+
+struct hns_roce_ud_send_wqe {
+	u32 dmac_h;
+	u32 u32_8;
+	u32 immediate_data;
+
+	u32 u32_16;
+	union {
+		unsigned char dgid[GID_LEN];
+		struct {
+			u32 u32_20;
+			u32 u32_24;
+			u32 u32_28;
+			u32 u32_32;
+		};
+	};
+
+	u32 u32_36;
+	u32 u32_40;
+
+	u32 va0_l;
+	u32 va0_h;
+	u32 l_key0;
+
+	u32 va1_l;
+	u32 va1_h;
+	u32 l_key1;
+};
+
+#define UD_SEND_WQE_U32_4_DMAC_0_S 0
+#define UD_SEND_WQE_U32_4_DMAC_0_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_0_S)
+
+#define UD_SEND_WQE_U32_4_DMAC_1_S 8
+#define UD_SEND_WQE_U32_4_DMAC_1_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_1_S)
+
+#define UD_SEND_WQE_U32_4_DMAC_2_S 16
+#define UD_SEND_WQE_U32_4_DMAC_2_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_2_S)
+
+#define UD_SEND_WQE_U32_4_DMAC_3_S 24
+#define UD_SEND_WQE_U32_4_DMAC_3_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_4_DMAC_3_S)
+
+#define UD_SEND_WQE_U32_8_DMAC_4_S 0
+#define UD_SEND_WQE_U32_8_DMAC_4_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_8_DMAC_4_S)
+
+#define UD_SEND_WQE_U32_8_DMAC_5_S 8
+#define UD_SEND_WQE_U32_8_DMAC_5_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_8_DMAC_5_S)
+
+#define UD_SEND_WQE_U32_8_OPERATION_TYPE_S 16
+#define UD_SEND_WQE_U32_8_OPERATION_TYPE_M   \
+	(((1UL << 4) - 1) << UD_SEND_WQE_U32_8_OPERATION_TYPE_S)
+
+#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S 24
+#define UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M   \
+	(((1UL << 6) - 1) << UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S)
+
+#define UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S 31
+
+#define UD_SEND_WQE_U32_16_DEST_QP_S 0
+#define UD_SEND_WQE_U32_16_DEST_QP_M   \
+	(((1UL << 24) - 1) << UD_SEND_WQE_U32_16_DEST_QP_S)
+
+#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S 24
+#define UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S)
+
+#define UD_SEND_WQE_U32_36_FLOW_LABEL_S 0
+#define UD_SEND_WQE_U32_36_FLOW_LABEL_M   \
+	(((1UL << 20) - 1) << UD_SEND_WQE_U32_36_FLOW_LABEL_S)
+
+#define UD_SEND_WQE_U32_36_PRIORITY_S 20
+#define UD_SEND_WQE_U32_36_PRIORITY_M   \
+	(((1UL << 4) - 1) << UD_SEND_WQE_U32_36_PRIORITY_S)
+
+#define UD_SEND_WQE_U32_36_SGID_INDEX_S 24
+#define UD_SEND_WQE_U32_36_SGID_INDEX_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_36_SGID_INDEX_S)
+
+#define UD_SEND_WQE_U32_40_HOP_LIMIT_S 0
+#define UD_SEND_WQE_U32_40_HOP_LIMIT_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_40_HOP_LIMIT_S)
+
+#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S 8
+#define UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M   \
+	(((1UL << 8) - 1) << UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S)
+
+struct hns_roce_sqp_context {
+	u32 qp1c_bytes_4;
+	u32 sq_rq_bt_l;
+	u32 qp1c_bytes_12;
+	u32 qp1c_bytes_16;
+	u32 qp1c_bytes_20;
+	u32 qp1c_bytes_28;
+	u32 cur_rq_wqe_ba_l;
+	u32 qp1c_bytes_32;
+	u32 cur_sq_wqe_ba_l;
+	u32 qp1c_bytes_40;
+};
+
+#define QP1C_BYTES_4_SQ_WQE_SHIFT_S 8
+#define QP1C_BYTES_4_SQ_WQE_SHIFT_M   \
+	(((1UL << 4) - 1) << QP1C_BYTES_4_SQ_WQE_SHIFT_S)
+
+#define QP1C_BYTES_4_RQ_WQE_SHIFT_S 12
+#define QP1C_BYTES_4_RQ_WQE_SHIFT_M   \
+	(((1UL << 4) - 1) << QP1C_BYTES_4_RQ_WQE_SHIFT_S)
+
+#define QP1C_BYTES_4_PD_S 16
+#define QP1C_BYTES_4_PD_M	(((1UL << 16) - 1) << QP1C_BYTES_4_PD_S)
+
+#define QP1C_BYTES_12_SQ_RQ_BT_H_S 0
+#define QP1C_BYTES_12_SQ_RQ_BT_H_M   \
+	(((1UL << 17) - 1) << QP1C_BYTES_12_SQ_RQ_BT_H_S)
+
+#define QP1C_BYTES_16_RQ_HEAD_S 0
+#define QP1C_BYTES_16_RQ_HEAD_M	(((1UL << 15) - 1) << QP1C_BYTES_16_RQ_HEAD_S)
+
+#define QP1C_BYTES_16_PORT_NUM_S 16
+#define QP1C_BYTES_16_PORT_NUM_M   \
+	(((1UL << 3) - 1) << QP1C_BYTES_16_PORT_NUM_S)
+
+#define QP1C_BYTES_16_SIGNALING_TYPE_S 27
+#define QP1C_BYTES_16_LOCAL_ENABLE_E2E_CREDIT_S 28
+#define QP1C_BYTES_16_RQ_BA_FLG_S 29
+#define QP1C_BYTES_16_SQ_BA_FLG_S 30
+#define QP1C_BYTES_16_QP1_ERR_S 31
+
+#define QP1C_BYTES_20_SQ_HEAD_S 0
+#define QP1C_BYTES_20_SQ_HEAD_M	(((1UL << 15) - 1) << QP1C_BYTES_20_SQ_HEAD_S)
+
+#define QP1C_BYTES_20_PKEY_IDX_S 16
+#define QP1C_BYTES_20_PKEY_IDX_M   \
+	(((1UL << 16) - 1) << QP1C_BYTES_20_PKEY_IDX_S)
+
+#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S 0
+#define QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M   \
+	(((1UL << 5) - 1) << QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S)
+
+#define QP1C_BYTES_28_RQ_CUR_IDX_S 16
+#define QP1C_BYTES_28_RQ_CUR_IDX_M   \
+	(((1UL << 15) - 1) << QP1C_BYTES_28_RQ_CUR_IDX_S)
+
+#define QP1C_BYTES_32_TX_CQ_NUM_S 0
+#define QP1C_BYTES_32_TX_CQ_NUM_M   \
+	(((1UL << 16) - 1) << QP1C_BYTES_32_TX_CQ_NUM_S)
+
+#define QP1C_BYTES_32_RX_CQ_NUM_S 16
+#define QP1C_BYTES_32_RX_CQ_NUM_M   \
+	(((1UL << 16) - 1) << QP1C_BYTES_32_RX_CQ_NUM_S)
+
+#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S 0
+#define QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M   \
+	(((1UL << 5) - 1) << QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S)
+
+#define QP1C_BYTES_40_SQ_CUR_IDX_S 16
+#define QP1C_BYTES_40_SQ_CUR_IDX_M   \
+	(((1UL << 15) - 1) << QP1C_BYTES_40_SQ_CUR_IDX_S)
+
+#define HNS_ROCE_WQE_INLINE		(1UL<<31)
+#define HNS_ROCE_WQE_SE			(1UL<<30)
+
+#define HNS_ROCE_WQE_SGE_NUM_BIT	24
+#define HNS_ROCE_WQE_IMM		(1UL<<23)
+#define HNS_ROCE_WQE_FENCE		(1UL<<21)
+#define HNS_ROCE_WQE_CQ_NOTIFY		(1UL<<20)
+
+#define HNS_ROCE_WQE_OPCODE_SEND	(0<<16)
+#define HNS_ROCE_WQE_OPCODE_RDMA_READ	(1<<16)
+#define HNS_ROCE_WQE_OPCODE_RDMA_WRITE	(2<<16)
+#define HNS_ROCE_WQE_OPCODE_LOCAL_INV	(4<<16)
+#define HNS_ROCE_WQE_OPCODE_UD_SEND	(7<<16)
+#define HNS_ROCE_WQE_OPCODE_MASK	(15<<16)
+
+struct hns_roce_qp_context {
+	u32 qpc_bytes_4;
+	u32 qpc_bytes_8;
+	u32 qpc_bytes_12;
+	u32 qpc_bytes_16;
+	u32 sq_rq_bt_l;
+	u32 qpc_bytes_24;
+	u32 irrl_ba_l;
+	u32 qpc_bytes_32;
+	u32 qpc_bytes_36;
+	u32 dmac_l;
+	u32 qpc_bytes_44;
+	u32 qpc_bytes_48;
+	u8 dgid[16];
+	u32 qpc_bytes_68;
+	u32 cur_rq_wqe_ba_l;
+	u32 qpc_bytes_76;
+	u32 rx_rnr_time;
+	u32 qpc_bytes_84;
+	u32 qpc_bytes_88;
+	union {
+		u32 rx_sge_len;
+		u32 dma_length;
+	};
+	union {
+		u32 rx_sge_num;
+		u32 rx_send_pktn;
+		u32 r_key;
+	};
+	u32 va_l;
+	u32 va_h;
+	u32 qpc_bytes_108;
+	u32 qpc_bytes_112;
+	u32 rx_cur_sq_wqe_ba_l;
+	u32 qpc_bytes_120;
+	u32 qpc_bytes_124;
+	u32 qpc_bytes_128;
+	u32 qpc_bytes_132;
+	u32 qpc_bytes_136;
+	u32 qpc_bytes_140;
+	u32 qpc_bytes_144;
+	u32 qpc_bytes_148;
+	union {
+		u32 rnr_retry;
+		u32 ack_time;
+	};
+	u32 qpc_bytes_156;
+	u32 pkt_use_len;
+	u32 qpc_bytes_164;
+	u32 qpc_bytes_168;
+	union {
+		u32 sge_use_len;
+		u32 pa_use_len;
+	};
+	u32 qpc_bytes_176;
+	u32 qpc_bytes_180;
+	u32 tx_cur_sq_wqe_ba_l;
+	u32 qpc_bytes_188;
+	u32 rvd21;
+};
+
+#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S 0
+#define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M   \
+	(((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S)
+
+#define QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S 3
+#define QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S 4
+#define QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S 5
+#define QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S 6
+#define QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S 7
+
+#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S 8
+#define QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M   \
+	(((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S)
+
+#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S 12
+#define QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M   \
+	(((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S)
+
+#define QP_CONTEXT_QPC_BYTES_4_PD_S 16
+#define QP_CONTEXT_QPC_BYTES_4_PD_M   \
+	(((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_4_PD_S)
+
+#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S 0
+#define QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M   \
+	(((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S)
+
+#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S 16
+#define QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M   \
+	(((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S)
+
+#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S 0
+#define QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M   \
+	(((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S)
+
+#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S 16
+#define QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M   \
+	(((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S)
+
+#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_S 0
+#define QP_CONTEXT_QPC_BYTES_16_QP_NUM_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_16_QP_NUM_S)
+
+#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S 0
+#define QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M   \
+	(((1UL << 17) - 1) << QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S)
+
+#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S 18
+#define QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M   \
+	(((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S)
+
+#define QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S 23
+
+#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S 0
+#define QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M   \
+	(((1UL << 17) - 1) << QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S)
+
+#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S 18
+#define QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M   \
+	(((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S)
+
+#define QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S 20
+#define QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S 21
+#define QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S 22
+#define QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S 23
+
+#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S 24
+#define QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S)
+
+#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_S 0
+#define QP_CONTEXT_QPC_BYTES_36_DEST_QP_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_36_DEST_QP_S)
+
+#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S 24
+#define QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S)
+
+#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_S 0
+#define QP_CONTEXT_QPC_BYTES_44_DMAC_H_M   \
+	(((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_44_DMAC_H_S)
+
+#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S 16
+#define QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S)
+
+#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_S 24
+#define QP_CONTEXT_QPC_BYTES_44_HOPLMT_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_44_HOPLMT_S)
+
+#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S 0
+#define QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M   \
+	(((1UL << 20) - 1) << QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S)
+
+#define QP_CONTEXT_QPC_BYTES_48_TCLASS_S 20
+#define QP_CONTEXT_QPC_BYTES_48_TCLASS_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_48_TCLASS_S)
+
+#define QP_CONTEXT_QPC_BYTES_48_MTU_S 28
+#define QP_CONTEXT_QPC_BYTES_48_MTU_M   \
+	(((1UL << 4) - 1) << QP_CONTEXT_QPC_BYTES_48_MTU_S)
+
+#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S 0
+#define QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S)
+
+#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S 16
+#define QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S)
+
+#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S 0
+#define QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M   \
+	(((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S)
+
+#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S 8
+#define QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S 0
+#define QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S 24
+#define QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S)
+
+#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S 0
+#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S 24
+#define QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S 25
+
+#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S 26
+#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M   \
+	(((1UL << 2) - 1) << \
+	QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S)
+
+#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S 29
+#define QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M   \
+	(((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S)
+
+#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S 0
+#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S 24
+#define QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S 25
+
+#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S 0
+#define QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S 24
+#define QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S)
+
+#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S 0
+#define QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M   \
+	(((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S)
+
+#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S 0
+#define QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S 16
+#define QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S)
+
+#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S 0
+#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S 24
+
+#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S 25
+#define QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M   \
+	(((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S)
+
+#define QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S 27
+
+#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S 0
+#define QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S 24
+#define QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S)
+
+#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S 0
+#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S 24
+#define QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S)
+
+#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S 0
+#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M   \
+	(((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S)
+
+#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S 16
+#define QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S 31
+
+#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_S 0
+#define QP_CONTEXT_QPC_BYTES_144_QP_STATE_M   \
+	(((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_144_QP_STATE_S)
+
+#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S 0
+#define QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M   \
+	(((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S)
+
+#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S 2
+#define QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M   \
+	(((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S)
+
+#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S 5
+#define QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M   \
+	(((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S)
+
+#define QP_CONTEXT_QPC_BYTES_148_LSN_S 8
+#define QP_CONTEXT_QPC_BYTES_148_LSN_M   \
+	(((1UL << 16) - 1) << QP_CONTEXT_QPC_BYTES_148_LSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S 0
+#define QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M   \
+	(((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S)
+
+#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S 3
+#define QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M   \
+	(((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S)
+
+#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S 8
+#define QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M   \
+	(((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S)
+
+#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S 11
+#define QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M   \
+	(((1UL << 3) - 1) << QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S)
+
+#define QP_CONTEXT_QPC_BYTES_156_SL_S 14
+#define QP_CONTEXT_QPC_BYTES_156_SL_M   \
+	(((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_156_SL_S)
+
+#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S 16
+#define QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S)
+
+#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S 24
+#define QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M   \
+	(((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S)
+
+#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S 0
+#define QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S 24
+#define QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M   \
+	(((1UL << 8) - 1) << QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S)
+
+#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S 0
+#define QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M   \
+	(((1UL << 24) - 1) << QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S)
+
+#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S 24
+#define QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M   \
+	(((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S)
+
+#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S 26
+#define QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M   \
+	(((1UL << 2) - 1) << QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S)
+
+#define QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S 28
+#define QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S 29
+#define QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S 30
+
+#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S 0
+#define QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S)
+
+#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S 16
+#define QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S)
+
+#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S 0
+#define QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S)
+
+#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S 16
+#define QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S)
+
+#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S 0
+#define QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M   \
+	(((1UL << 5) - 1) << QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S)
+
+#define QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S 8
+
+#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S 16
+#define QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M   \
+	(((1UL << 15) - 1) << QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S)
+
+struct hns_roce_rq_db {
+	u32    u32_4;
+	u32    u32_8;
+};
+
+#define RQ_DOORBELL_U32_4_RQ_HEAD_S 0
+#define RQ_DOORBELL_U32_4_RQ_HEAD_M   \
+	(((1UL << 15) - 1) << RQ_DOORBELL_U32_4_RQ_HEAD_S)
+
+#define RQ_DOORBELL_U32_8_QPN_S 0
+#define RQ_DOORBELL_U32_8_QPN_M   (((1UL << 24) - 1) << RQ_DOORBELL_U32_8_QPN_S)
+
+#define RQ_DOORBELL_U32_8_CMD_S 28
+#define RQ_DOORBELL_U32_8_CMD_M   (((1UL << 3) - 1) << RQ_DOORBELL_U32_8_CMD_S)
+
+#define RQ_DOORBELL_U32_8_HW_SYNC_S 31
+
+struct hns_roce_sq_db {
+	u32    u32_4;
+	u32    u32_8;
+};
+
+#define SQ_DOORBELL_U32_4_SQ_HEAD_S 0
+#define SQ_DOORBELL_U32_4_SQ_HEAD_M   \
+	(((1UL << 15) - 1) << SQ_DOORBELL_U32_4_SQ_HEAD_S)
+
+#define SQ_DOORBELL_U32_4_PORT_S 18
+#define SQ_DOORBELL_U32_4_PORT_M  (((1UL << 3) - 1) << SQ_DOORBELL_U32_4_PORT_S)
+
+#define SQ_DOORBELL_U32_8_QPN_S 0
+#define SQ_DOORBELL_U32_8_QPN_M   (((1UL << 24) - 1) << SQ_DOORBELL_U32_8_QPN_S)
+
+#define SQ_DOORBELL_HW_SYNC_S 31
+
+struct hns_roce_ext_db {
+	int esdb_dep;
+	int eodb_dep;
+	struct hns_roce_buf_list *sdb_buf_list;
+	struct hns_roce_buf_list *odb_buf_list;
+};
+
+struct hns_roce_db_table {
+	int  sdb_ext_mod;
+	int  odb_ext_mod;
+	struct hns_roce_ext_db *ext_db;
+};
+
+struct hns_roce_v1_priv {
+	struct hns_roce_db_table  db_table;
+	struct hns_roce_raq_table raq_table;
+};
+
+int hns_dsaf_roce_reset(struct fwnode_handle *dsaf_fwnode, bool dereset);
+
+#endif
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
new file mode 100644
index 000000000000..f64f0dde9a88
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -0,0 +1,1142 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <linux/acpi.h>
+#include <linux/of_platform.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_user_verbs.h>
+#include "hns_roce_common.h"
+#include "hns_roce_device.h"
+#include "hns_roce_user.h"
+#include "hns_roce_hem.h"
+
+/**
+ * hns_roce_addrconf_ifid_eui48 - Get default gid.
+ * @eui: eui.
+ * @vlan_id:  gid
+ * @dev:  net device
+ * Description:
+ *    MAC convert to GID
+ *        gid[0..7] = fe80 0000 0000 0000
+ *        gid[8] = mac[0] ^ 2
+ *        gid[9] = mac[1]
+ *        gid[10] = mac[2]
+ *        gid[11] = ff        (VLAN ID high byte (4 MS bits))
+ *        gid[12] = fe        (VLAN ID low byte)
+ *        gid[13] = mac[3]
+ *        gid[14] = mac[4]
+ *        gid[15] = mac[5]
+ */
+static void hns_roce_addrconf_ifid_eui48(u8 *eui, u16 vlan_id,
+					 struct net_device *dev)
+{
+	memcpy(eui, dev->dev_addr, 3);
+	memcpy(eui + 5, dev->dev_addr + 3, 3);
+	if (vlan_id < 0x1000) {
+		eui[3] = vlan_id >> 8;
+		eui[4] = vlan_id & 0xff;
+	} else {
+		eui[3] = 0xff;
+		eui[4] = 0xfe;
+	}
+	eui[0] ^= 2;
+}
+
+static void hns_roce_make_default_gid(struct net_device *dev, union ib_gid *gid)
+{
+	memset(gid, 0, sizeof(*gid));
+	gid->raw[0] = 0xFE;
+	gid->raw[1] = 0x80;
+	hns_roce_addrconf_ifid_eui48(&gid->raw[8], 0xffff, dev);
+}
+
+/**
+ * hns_get_gid_index - Get gid index.
+ * @hr_dev: pointer to structure hns_roce_dev.
+ * @port:  port, value range: 0 ~ MAX
+ * @gid_index:  gid_index, value range: 0 ~ MAX
+ * Description:
+ *    N ports shared gids, allocation method as follow:
+ *		GID[0][0], GID[1][0],.....GID[N - 1][0],
+ *		GID[0][0], GID[1][0],.....GID[N - 1][0],
+ *		And so on
+ */
+int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index)
+{
+	return gid_index * hr_dev->caps.num_ports + port;
+}
+
+static int hns_roce_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
+		     union ib_gid *gid)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	u8 gid_idx = 0;
+
+	if (gid_index >= hr_dev->caps.gid_table_len[port]) {
+		dev_err(dev, "gid_index %d illegal, port %d gid range: 0~%d\n",
+			gid_index, port, hr_dev->caps.gid_table_len[port] - 1);
+		return -EINVAL;
+	}
+
+	gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
+
+	if (!memcmp(gid, &hr_dev->iboe.gid_table[gid_idx], sizeof(*gid)))
+		return -EINVAL;
+
+	memcpy(&hr_dev->iboe.gid_table[gid_idx], gid, sizeof(*gid));
+
+	hr_dev->hw->set_gid(hr_dev, port, gid_index, gid);
+
+	return 0;
+}
+
+static void hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr)
+{
+	u8 phy_port;
+	u32 i = 0;
+
+	if (!memcmp(hr_dev->dev_addr[port], addr, MAC_ADDR_OCTET_NUM))
+		return;
+
+	for (i = 0; i < MAC_ADDR_OCTET_NUM; i++)
+		hr_dev->dev_addr[port][i] = addr[i];
+
+	phy_port = hr_dev->iboe.phy_port[port];
+	hr_dev->hw->set_mac(hr_dev, phy_port, addr);
+}
+
+static void hns_roce_set_mtu(struct hns_roce_dev *hr_dev, u8 port, int mtu)
+{
+	u8 phy_port = hr_dev->iboe.phy_port[port];
+	enum ib_mtu tmp;
+
+	tmp = iboe_get_mtu(mtu);
+	if (!tmp)
+		tmp = IB_MTU_256;
+
+	hr_dev->hw->set_mtu(hr_dev, phy_port, tmp);
+}
+
+static void hns_roce_update_gids(struct hns_roce_dev *hr_dev, int port)
+{
+	struct ib_event event;
+
+	/* Refresh gid in ib_cache */
+	event.device = &hr_dev->ib_dev;
+	event.element.port_num = port + 1;
+	event.event = IB_EVENT_GID_CHANGE;
+	ib_dispatch_event(&event);
+}
+
+static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
+			   unsigned long event)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct net_device *netdev;
+	unsigned long flags;
+	union ib_gid gid;
+	int ret = 0;
+
+	netdev = hr_dev->iboe.netdevs[port];
+	if (!netdev) {
+		dev_err(dev, "port(%d) can't find netdev\n", port);
+		return -ENODEV;
+	}
+
+	spin_lock_irqsave(&hr_dev->iboe.lock, flags);
+
+	switch (event) {
+	case NETDEV_UP:
+	case NETDEV_CHANGE:
+	case NETDEV_REGISTER:
+	case NETDEV_CHANGEADDR:
+		hns_roce_set_mac(hr_dev, port, netdev->dev_addr);
+		hns_roce_make_default_gid(netdev, &gid);
+		ret = hns_roce_set_gid(hr_dev, port, 0, &gid);
+		if (!ret)
+			hns_roce_update_gids(hr_dev, port);
+		break;
+	case NETDEV_DOWN:
+		/*
+		* In v1 engine, only support all ports closed together.
+		*/
+		break;
+	default:
+		dev_dbg(dev, "NETDEV event = 0x%x!\n", (u32)(event));
+		break;
+	}
+
+	spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+	return ret;
+}
+
+static int hns_roce_netdev_event(struct notifier_block *self,
+				 unsigned long event, void *ptr)
+{
+	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+	struct hns_roce_ib_iboe *iboe = NULL;
+	struct hns_roce_dev *hr_dev = NULL;
+	u8 port = 0;
+	int ret = 0;
+
+	hr_dev = container_of(self, struct hns_roce_dev, iboe.nb);
+	iboe = &hr_dev->iboe;
+
+	for (port = 0; port < hr_dev->caps.num_ports; port++) {
+		if (dev == iboe->netdevs[port]) {
+			ret = handle_en_event(hr_dev, port, event);
+			if (ret)
+				return NOTIFY_DONE;
+			break;
+		}
+	}
+
+	return NOTIFY_DONE;
+}
+
+static void hns_roce_addr_event(int event, struct net_device *event_netdev,
+				struct hns_roce_dev *hr_dev, union ib_gid *gid)
+{
+	struct hns_roce_ib_iboe *iboe = NULL;
+	int gid_table_len = 0;
+	unsigned long flags;
+	union ib_gid zgid;
+	u8 gid_idx = 0;
+	u8 port = 0;
+	int i = 0;
+	int free;
+	struct net_device *real_dev = rdma_vlan_dev_real_dev(event_netdev) ?
+				      rdma_vlan_dev_real_dev(event_netdev) :
+				      event_netdev;
+
+	if (event != NETDEV_UP && event != NETDEV_DOWN)
+		return;
+
+	iboe = &hr_dev->iboe;
+	while (port < hr_dev->caps.num_ports) {
+		if (real_dev == iboe->netdevs[port])
+			break;
+		port++;
+	}
+
+	if (port >= hr_dev->caps.num_ports) {
+		dev_dbg(&hr_dev->pdev->dev, "can't find netdev\n");
+		return;
+	}
+
+	memset(zgid.raw, 0, sizeof(zgid.raw));
+	free = -1;
+	gid_table_len = hr_dev->caps.gid_table_len[port];
+
+	spin_lock_irqsave(&hr_dev->iboe.lock, flags);
+
+	for (i = 0; i < gid_table_len; i++) {
+		gid_idx = hns_get_gid_index(hr_dev, port, i);
+		if (!memcmp(gid->raw, iboe->gid_table[gid_idx].raw,
+			    sizeof(gid->raw)))
+			break;
+		if (free < 0 && !memcmp(zgid.raw,
+			iboe->gid_table[gid_idx].raw, sizeof(zgid.raw)))
+			free = i;
+	}
+
+	if (i >= gid_table_len) {
+		if (free < 0) {
+			spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+			dev_dbg(&hr_dev->pdev->dev,
+				"gid_index overflow, port(%d)\n", port);
+			return;
+		}
+		if (!hns_roce_set_gid(hr_dev, port, free, gid))
+			hns_roce_update_gids(hr_dev, port);
+	} else if (event == NETDEV_DOWN) {
+		if (!hns_roce_set_gid(hr_dev, port, i, &zgid))
+			hns_roce_update_gids(hr_dev, port);
+	}
+
+	spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+}
+
+static int hns_roce_inet_event(struct notifier_block *self, unsigned long event,
+			       void *ptr)
+{
+	struct in_ifaddr *ifa = ptr;
+	struct hns_roce_dev *hr_dev;
+	struct net_device *dev = ifa->ifa_dev->dev;
+	union ib_gid gid;
+
+	ipv6_addr_set_v4mapped(ifa->ifa_address, (struct in6_addr *)&gid);
+
+	hr_dev = container_of(self, struct hns_roce_dev, iboe.nb_inet);
+
+	hns_roce_addr_event(event, dev, hr_dev, &gid);
+
+	return NOTIFY_DONE;
+}
+
+static int hns_roce_setup_mtu_gids(struct hns_roce_dev *hr_dev)
+{
+	struct in_ifaddr *ifa_list = NULL;
+	union ib_gid gid = {{0} };
+	u32 ipaddr = 0;
+	int index = 0;
+	int ret = 0;
+	u8 i = 0;
+
+	for (i = 0; i < hr_dev->caps.num_ports; i++) {
+		hns_roce_set_mtu(hr_dev, i,
+				 ib_mtu_enum_to_int(hr_dev->caps.max_mtu));
+		hns_roce_set_mac(hr_dev, i, hr_dev->iboe.netdevs[i]->dev_addr);
+
+		if (hr_dev->iboe.netdevs[i]->ip_ptr) {
+			ifa_list = hr_dev->iboe.netdevs[i]->ip_ptr->ifa_list;
+			index = 1;
+			while (ifa_list) {
+				ipaddr = ifa_list->ifa_address;
+				ipv6_addr_set_v4mapped(ipaddr,
+						       (struct in6_addr *)&gid);
+				ret = hns_roce_set_gid(hr_dev, i, index, &gid);
+				if (ret)
+					break;
+				index++;
+				ifa_list = ifa_list->ifa_next;
+			}
+			hns_roce_update_gids(hr_dev, i);
+		}
+	}
+
+	return ret;
+}
+
+static int hns_roce_query_device(struct ib_device *ib_dev,
+				 struct ib_device_attr *props,
+				 struct ib_udata *uhw)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
+
+	memset(props, 0, sizeof(*props));
+
+	props->sys_image_guid = hr_dev->sys_image_guid;
+	props->max_mr_size = (u64)(~(0ULL));
+	props->page_size_cap = hr_dev->caps.page_size_cap;
+	props->vendor_id = hr_dev->vendor_id;
+	props->vendor_part_id = hr_dev->vendor_part_id;
+	props->hw_ver = hr_dev->hw_rev;
+	props->max_qp = hr_dev->caps.num_qps;
+	props->max_qp_wr = hr_dev->caps.max_wqes;
+	props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT |
+				  IB_DEVICE_RC_RNR_NAK_GEN |
+				  IB_DEVICE_LOCAL_DMA_LKEY;
+	props->max_sge = hr_dev->caps.max_sq_sg;
+	props->max_sge_rd = 1;
+	props->max_cq = hr_dev->caps.num_cqs;
+	props->max_cqe = hr_dev->caps.max_cqes;
+	props->max_mr = hr_dev->caps.num_mtpts;
+	props->max_pd = hr_dev->caps.num_pds;
+	props->max_qp_rd_atom = hr_dev->caps.max_qp_dest_rdma;
+	props->max_qp_init_rd_atom = hr_dev->caps.max_qp_init_rdma;
+	props->atomic_cap = IB_ATOMIC_NONE;
+	props->max_pkeys = 1;
+	props->local_ca_ack_delay = hr_dev->caps.local_ca_ack_delay;
+
+	return 0;
+}
+
+static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,
+			       struct ib_port_attr *props)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct net_device *net_dev;
+	unsigned long flags;
+	enum ib_mtu mtu;
+	u8 port;
+
+	assert(port_num > 0);
+	port = port_num - 1;
+
+	memset(props, 0, sizeof(*props));
+
+	props->max_mtu = hr_dev->caps.max_mtu;
+	props->gid_tbl_len = hr_dev->caps.gid_table_len[port];
+	props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
+				IB_PORT_VENDOR_CLASS_SUP |
+				IB_PORT_BOOT_MGMT_SUP;
+	props->max_msg_sz = HNS_ROCE_MAX_MSG_LEN;
+	props->pkey_tbl_len = 1;
+	props->active_width = IB_WIDTH_4X;
+	props->active_speed = 1;
+
+	spin_lock_irqsave(&hr_dev->iboe.lock, flags);
+
+	net_dev = hr_dev->iboe.netdevs[port];
+	if (!net_dev) {
+		spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+		dev_err(dev, "find netdev %d failed!\r\n", port);
+		return -EINVAL;
+	}
+
+	mtu = iboe_get_mtu(net_dev->mtu);
+	props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256;
+	props->state = (netif_running(net_dev) && netif_carrier_ok(net_dev)) ?
+			IB_PORT_ACTIVE : IB_PORT_DOWN;
+	props->phys_state = (props->state == IB_PORT_ACTIVE) ? 5 : 3;
+
+	spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+
+	return 0;
+}
+
+static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device,
+						    u8 port_num)
+{
+	return IB_LINK_LAYER_ETHERNET;
+}
+
+static int hns_roce_query_gid(struct ib_device *ib_dev, u8 port_num, int index,
+			      union ib_gid *gid)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
+	struct device *dev = &hr_dev->pdev->dev;
+	u8 gid_idx = 0;
+	u8 port;
+
+	if (port_num < 1 || port_num > hr_dev->caps.num_ports ||
+	    index >= hr_dev->caps.gid_table_len[port_num - 1]) {
+		dev_err(dev,
+			"port_num %d index %d illegal! correct range: port_num 1~%d index 0~%d!\n",
+			port_num, index, hr_dev->caps.num_ports,
+			hr_dev->caps.gid_table_len[port_num - 1] - 1);
+		return -EINVAL;
+	}
+
+	port = port_num - 1;
+	gid_idx = hns_get_gid_index(hr_dev, port, index);
+	if (gid_idx >= HNS_ROCE_MAX_GID_NUM) {
+		dev_err(dev, "port_num %d index %d illegal! total gid num %d!\n",
+			port_num, index, HNS_ROCE_MAX_GID_NUM);
+		return -EINVAL;
+	}
+
+	memcpy(gid->raw, hr_dev->iboe.gid_table[gid_idx].raw,
+	       HNS_ROCE_GID_SIZE);
+
+	return 0;
+}
+
+static int hns_roce_query_pkey(struct ib_device *ib_dev, u8 port, u16 index,
+			       u16 *pkey)
+{
+	*pkey = PKEY_ID;
+
+	return 0;
+}
+
+static int hns_roce_modify_device(struct ib_device *ib_dev, int mask,
+				  struct ib_device_modify *props)
+{
+	unsigned long flags;
+
+	if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
+		return -EOPNOTSUPP;
+
+	if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
+		spin_lock_irqsave(&to_hr_dev(ib_dev)->sm_lock, flags);
+		memcpy(ib_dev->node_desc, props->node_desc, NODE_DESC_SIZE);
+		spin_unlock_irqrestore(&to_hr_dev(ib_dev)->sm_lock, flags);
+	}
+
+	return 0;
+}
+
+static int hns_roce_modify_port(struct ib_device *ib_dev, u8 port_num, int mask,
+				struct ib_port_modify *props)
+{
+	return 0;
+}
+
+static struct ib_ucontext *hns_roce_alloc_ucontext(struct ib_device *ib_dev,
+						   struct ib_udata *udata)
+{
+	int ret = 0;
+	struct hns_roce_ucontext *context;
+	struct hns_roce_ib_alloc_ucontext_resp resp;
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
+
+	resp.qp_tab_size = hr_dev->caps.num_qps;
+
+	context = kmalloc(sizeof(*context), GFP_KERNEL);
+	if (!context)
+		return ERR_PTR(-ENOMEM);
+
+	ret = hns_roce_uar_alloc(hr_dev, &context->uar);
+	if (ret)
+		goto error_fail_uar_alloc;
+
+	ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
+	if (ret)
+		goto error_fail_copy_to_udata;
+
+	return &context->ibucontext;
+
+error_fail_copy_to_udata:
+	hns_roce_uar_free(hr_dev, &context->uar);
+
+error_fail_uar_alloc:
+	kfree(context);
+
+	return ERR_PTR(ret);
+}
+
+static int hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext)
+{
+	struct hns_roce_ucontext *context = to_hr_ucontext(ibcontext);
+
+	hns_roce_uar_free(to_hr_dev(ibcontext->device), &context->uar);
+	kfree(context);
+
+	return 0;
+}
+
+static int hns_roce_mmap(struct ib_ucontext *context,
+			 struct vm_area_struct *vma)
+{
+	if (((vma->vm_end - vma->vm_start) % PAGE_SIZE) != 0)
+		return -EINVAL;
+
+	if (vma->vm_pgoff == 0) {
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+		if (io_remap_pfn_range(vma, vma->vm_start,
+				       to_hr_ucontext(context)->uar.pfn,
+				       PAGE_SIZE, vma->vm_page_prot))
+			return -EAGAIN;
+
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int hns_roce_port_immutable(struct ib_device *ib_dev, u8 port_num,
+				   struct ib_port_immutable *immutable)
+{
+	struct ib_port_attr attr;
+	int ret;
+
+	ret = hns_roce_query_port(ib_dev, port_num, &attr);
+	if (ret)
+		return ret;
+
+	immutable->pkey_tbl_len = attr.pkey_tbl_len;
+	immutable->gid_tbl_len = attr.gid_tbl_len;
+
+	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+
+	return 0;
+}
+
+static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_ib_iboe *iboe = &hr_dev->iboe;
+
+	unregister_inetaddr_notifier(&iboe->nb_inet);
+	unregister_netdevice_notifier(&iboe->nb);
+	ib_unregister_device(&hr_dev->ib_dev);
+}
+
+static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
+{
+	int ret;
+	struct hns_roce_ib_iboe *iboe = NULL;
+	struct ib_device *ib_dev = NULL;
+	struct device *dev = &hr_dev->pdev->dev;
+
+	iboe = &hr_dev->iboe;
+
+	ib_dev = &hr_dev->ib_dev;
+	strlcpy(ib_dev->name, "hisi_%d", IB_DEVICE_NAME_MAX);
+
+	ib_dev->owner			= THIS_MODULE;
+	ib_dev->node_type		= RDMA_NODE_IB_CA;
+	ib_dev->dma_device		= dev;
+
+	ib_dev->phys_port_cnt		= hr_dev->caps.num_ports;
+	ib_dev->local_dma_lkey		= hr_dev->caps.reserved_lkey;
+	ib_dev->num_comp_vectors	= hr_dev->caps.num_comp_vectors;
+	ib_dev->uverbs_abi_ver		= 1;
+	ib_dev->uverbs_cmd_mask		=
+		(1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) |
+		(1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) |
+		(1ULL << IB_USER_VERBS_CMD_QUERY_PORT) |
+		(1ULL << IB_USER_VERBS_CMD_ALLOC_PD) |
+		(1ULL << IB_USER_VERBS_CMD_DEALLOC_PD) |
+		(1ULL << IB_USER_VERBS_CMD_REG_MR) |
+		(1ULL << IB_USER_VERBS_CMD_DEREG_MR) |
+		(1ULL << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+		(1ULL << IB_USER_VERBS_CMD_CREATE_CQ) |
+		(1ULL << IB_USER_VERBS_CMD_DESTROY_CQ) |
+		(1ULL << IB_USER_VERBS_CMD_CREATE_QP) |
+		(1ULL << IB_USER_VERBS_CMD_MODIFY_QP) |
+		(1ULL << IB_USER_VERBS_CMD_QUERY_QP) |
+		(1ULL << IB_USER_VERBS_CMD_DESTROY_QP);
+
+	/* HCA||device||port */
+	ib_dev->modify_device		= hns_roce_modify_device;
+	ib_dev->query_device		= hns_roce_query_device;
+	ib_dev->query_port		= hns_roce_query_port;
+	ib_dev->modify_port		= hns_roce_modify_port;
+	ib_dev->get_link_layer		= hns_roce_get_link_layer;
+	ib_dev->query_gid		= hns_roce_query_gid;
+	ib_dev->query_pkey		= hns_roce_query_pkey;
+	ib_dev->alloc_ucontext		= hns_roce_alloc_ucontext;
+	ib_dev->dealloc_ucontext	= hns_roce_dealloc_ucontext;
+	ib_dev->mmap			= hns_roce_mmap;
+
+	/* PD */
+	ib_dev->alloc_pd		= hns_roce_alloc_pd;
+	ib_dev->dealloc_pd		= hns_roce_dealloc_pd;
+
+	/* AH */
+	ib_dev->create_ah		= hns_roce_create_ah;
+	ib_dev->query_ah		= hns_roce_query_ah;
+	ib_dev->destroy_ah		= hns_roce_destroy_ah;
+
+	/* QP */
+	ib_dev->create_qp		= hns_roce_create_qp;
+	ib_dev->modify_qp		= hns_roce_modify_qp;
+	ib_dev->query_qp		= hr_dev->hw->query_qp;
+	ib_dev->destroy_qp		= hr_dev->hw->destroy_qp;
+	ib_dev->post_send		= hr_dev->hw->post_send;
+	ib_dev->post_recv		= hr_dev->hw->post_recv;
+
+	/* CQ */
+	ib_dev->create_cq		= hns_roce_ib_create_cq;
+	ib_dev->destroy_cq		= hns_roce_ib_destroy_cq;
+	ib_dev->req_notify_cq		= hr_dev->hw->req_notify_cq;
+	ib_dev->poll_cq			= hr_dev->hw->poll_cq;
+
+	/* MR */
+	ib_dev->get_dma_mr		= hns_roce_get_dma_mr;
+	ib_dev->reg_user_mr		= hns_roce_reg_user_mr;
+	ib_dev->dereg_mr		= hns_roce_dereg_mr;
+
+	/* OTHERS */
+	ib_dev->get_port_immutable	= hns_roce_port_immutable;
+
+	ret = ib_register_device(ib_dev, NULL);
+	if (ret) {
+		dev_err(dev, "ib_register_device failed!\n");
+		return ret;
+	}
+
+	ret = hns_roce_setup_mtu_gids(hr_dev);
+	if (ret) {
+		dev_err(dev, "roce_setup_mtu_gids failed!\n");
+		goto error_failed_setup_mtu_gids;
+	}
+
+	spin_lock_init(&iboe->lock);
+
+	iboe->nb.notifier_call = hns_roce_netdev_event;
+	ret = register_netdevice_notifier(&iboe->nb);
+	if (ret) {
+		dev_err(dev, "register_netdevice_notifier failed!\n");
+		goto error_failed_setup_mtu_gids;
+	}
+
+	iboe->nb_inet.notifier_call = hns_roce_inet_event;
+	ret = register_inetaddr_notifier(&iboe->nb_inet);
+	if (ret) {
+		dev_err(dev, "register inet addr notifier failed!\n");
+		goto error_failed_register_inetaddr_notifier;
+	}
+
+	return 0;
+
+error_failed_register_inetaddr_notifier:
+	unregister_netdevice_notifier(&iboe->nb);
+
+error_failed_setup_mtu_gids:
+	ib_unregister_device(ib_dev);
+
+	return ret;
+}
+
+static const struct of_device_id hns_roce_of_match[] = {
+	{ .compatible = "hisilicon,hns-roce-v1", .data = &hns_roce_hw_v1, },
+	{},
+};
+MODULE_DEVICE_TABLE(of, hns_roce_of_match);
+
+static const struct acpi_device_id hns_roce_acpi_match[] = {
+	{ "HISI00D1", (kernel_ulong_t)&hns_roce_hw_v1 },
+	{},
+};
+MODULE_DEVICE_TABLE(acpi, hns_roce_acpi_match);
+
+static int hns_roce_node_match(struct device *dev, void *fwnode)
+{
+	return dev->fwnode == fwnode;
+}
+
+static struct
+platform_device *hns_roce_find_pdev(struct fwnode_handle *fwnode)
+{
+	struct device *dev;
+
+	/* get the 'device'corresponding to matching 'fwnode' */
+	dev = bus_find_device(&platform_bus_type, NULL,
+			      fwnode, hns_roce_node_match);
+	/* get the platform device */
+	return dev ? to_platform_device(dev) : NULL;
+}
+
+static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev)
+{
+	int i;
+	int ret;
+	u8 phy_port;
+	int port_cnt = 0;
+	struct device *dev = &hr_dev->pdev->dev;
+	struct device_node *net_node;
+	struct net_device *netdev = NULL;
+	struct platform_device *pdev = NULL;
+	struct resource *res;
+
+	/* check if we are compatible with the underlying SoC */
+	if (dev_of_node(dev)) {
+		const struct of_device_id *of_id;
+
+		of_id = of_match_node(hns_roce_of_match, dev->of_node);
+		if (!of_id) {
+			dev_err(dev, "device is not compatible!\n");
+			return -ENXIO;
+		}
+		hr_dev->hw = (struct hns_roce_hw *)of_id->data;
+		if (!hr_dev->hw) {
+			dev_err(dev, "couldn't get H/W specific DT data!\n");
+			return -ENXIO;
+		}
+	} else if (is_acpi_device_node(dev->fwnode)) {
+		const struct acpi_device_id *acpi_id;
+
+		acpi_id = acpi_match_device(hns_roce_acpi_match, dev);
+		if (!acpi_id) {
+			dev_err(dev, "device is not compatible!\n");
+			return -ENXIO;
+		}
+		hr_dev->hw = (struct hns_roce_hw *) acpi_id->driver_data;
+		if (!hr_dev->hw) {
+			dev_err(dev, "couldn't get H/W specific ACPI data!\n");
+			return -ENXIO;
+		}
+	} else {
+		dev_err(dev, "can't read compatibility data from DT or ACPI\n");
+		return -ENXIO;
+	}
+
+	/* get the mapped register base address */
+	res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(dev, "memory resource not found!\n");
+		return -EINVAL;
+	}
+	hr_dev->reg_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(hr_dev->reg_base))
+		return PTR_ERR(hr_dev->reg_base);
+
+	/* get the RoCE associated ethernet ports or netdevices */
+	for (i = 0; i < HNS_ROCE_MAX_PORTS; i++) {
+		if (dev_of_node(dev)) {
+			net_node = of_parse_phandle(dev->of_node, "eth-handle",
+						    i);
+			if (!net_node)
+				continue;
+			pdev = of_find_device_by_node(net_node);
+		} else if (is_acpi_device_node(dev->fwnode)) {
+			struct acpi_reference_args args;
+			struct fwnode_handle *fwnode;
+
+			ret = acpi_node_get_property_reference(dev->fwnode,
+							       "eth-handle",
+							       i, &args);
+			if (ret)
+				continue;
+			fwnode = acpi_fwnode_handle(args.adev);
+			pdev = hns_roce_find_pdev(fwnode);
+		} else {
+			dev_err(dev, "cannot read data from DT or ACPI\n");
+			return -ENXIO;
+		}
+
+		if (pdev) {
+			netdev = platform_get_drvdata(pdev);
+			phy_port = (u8)i;
+			if (netdev) {
+				hr_dev->iboe.netdevs[port_cnt] = netdev;
+				hr_dev->iboe.phy_port[port_cnt] = phy_port;
+			} else {
+				dev_err(dev, "no netdev found with pdev %s\n",
+					pdev->name);
+				return -ENODEV;
+			}
+			port_cnt++;
+		}
+	}
+
+	if (port_cnt == 0) {
+		dev_err(dev, "unable to get eth-handle for available ports!\n");
+		return -EINVAL;
+	}
+
+	hr_dev->caps.num_ports = port_cnt;
+
+	/* cmd issue mode: 0 is poll, 1 is event */
+	hr_dev->cmd_mod = 1;
+	hr_dev->loop_idc = 0;
+
+	/* read the interrupt names from the DT or ACPI */
+	ret = device_property_read_string_array(dev, "interrupt-names",
+						hr_dev->irq_names,
+						HNS_ROCE_MAX_IRQ_NUM);
+	if (ret < 0) {
+		dev_err(dev, "couldn't get interrupt names from DT or ACPI!\n");
+		return ret;
+	}
+
+	/* fetch the interrupt numbers */
+	for (i = 0; i < HNS_ROCE_MAX_IRQ_NUM; i++) {
+		hr_dev->irq[i] = platform_get_irq(hr_dev->pdev, i);
+		if (hr_dev->irq[i] <= 0) {
+			dev_err(dev, "platform get of irq[=%d] failed!\n", i);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)
+{
+	int ret;
+	struct device *dev = &hr_dev->pdev->dev;
+
+	ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtt_table,
+				      HEM_TYPE_MTT, hr_dev->caps.mtt_entry_sz,
+				      hr_dev->caps.num_mtt_segs, 1);
+	if (ret) {
+		dev_err(dev, "Failed to init MTT context memory, aborting.\n");
+		return ret;
+	}
+
+	ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table,
+				      HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz,
+				      hr_dev->caps.num_mtpts, 1);
+	if (ret) {
+		dev_err(dev, "Failed to init MTPT context memory, aborting.\n");
+		goto err_unmap_mtt;
+	}
+
+	ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.qp_table,
+				      HEM_TYPE_QPC, hr_dev->caps.qpc_entry_sz,
+				      hr_dev->caps.num_qps, 1);
+	if (ret) {
+		dev_err(dev, "Failed to init QP context memory, aborting.\n");
+		goto err_unmap_dmpt;
+	}
+
+	ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.irrl_table,
+				      HEM_TYPE_IRRL,
+				      hr_dev->caps.irrl_entry_sz *
+				      hr_dev->caps.max_qp_init_rdma,
+				      hr_dev->caps.num_qps, 1);
+	if (ret) {
+		dev_err(dev, "Failed to init irrl_table memory, aborting.\n");
+		goto err_unmap_qp;
+	}
+
+	ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cq_table.table,
+				      HEM_TYPE_CQC, hr_dev->caps.cqc_entry_sz,
+				      hr_dev->caps.num_cqs, 1);
+	if (ret) {
+		dev_err(dev, "Failed to init CQ context memory, aborting.\n");
+		goto err_unmap_irrl;
+	}
+
+	return 0;
+
+err_unmap_irrl:
+	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
+
+err_unmap_qp:
+	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
+
+err_unmap_dmpt:
+	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
+
+err_unmap_mtt:
+	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
+
+	return ret;
+}
+
+/**
+* hns_roce_setup_hca - setup host channel adapter
+* @hr_dev: pointer to hns roce device
+* Return : int
+*/
+static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev)
+{
+	int ret;
+	struct device *dev = &hr_dev->pdev->dev;
+
+	spin_lock_init(&hr_dev->sm_lock);
+	spin_lock_init(&hr_dev->cq_db_lock);
+	spin_lock_init(&hr_dev->bt_cmd_lock);
+
+	ret = hns_roce_init_uar_table(hr_dev);
+	if (ret) {
+		dev_err(dev, "Failed to initialize uar table. aborting\n");
+		return ret;
+	}
+
+	ret = hns_roce_uar_alloc(hr_dev, &hr_dev->priv_uar);
+	if (ret) {
+		dev_err(dev, "Failed to allocate priv_uar.\n");
+		goto err_uar_table_free;
+	}
+
+	ret = hns_roce_init_pd_table(hr_dev);
+	if (ret) {
+		dev_err(dev, "Failed to init protected domain table.\n");
+		goto err_uar_alloc_free;
+	}
+
+	ret = hns_roce_init_mr_table(hr_dev);
+	if (ret) {
+		dev_err(dev, "Failed to init memory region table.\n");
+		goto err_pd_table_free;
+	}
+
+	ret = hns_roce_init_cq_table(hr_dev);
+	if (ret) {
+		dev_err(dev, "Failed to init completion queue table.\n");
+		goto err_mr_table_free;
+	}
+
+	ret = hns_roce_init_qp_table(hr_dev);
+	if (ret) {
+		dev_err(dev, "Failed to init queue pair table.\n");
+		goto err_cq_table_free;
+	}
+
+	return 0;
+
+err_cq_table_free:
+	hns_roce_cleanup_cq_table(hr_dev);
+
+err_mr_table_free:
+	hns_roce_cleanup_mr_table(hr_dev);
+
+err_pd_table_free:
+	hns_roce_cleanup_pd_table(hr_dev);
+
+err_uar_alloc_free:
+	hns_roce_uar_free(hr_dev, &hr_dev->priv_uar);
+
+err_uar_table_free:
+	hns_roce_cleanup_uar_table(hr_dev);
+	return ret;
+}
+
+/**
+* hns_roce_probe - RoCE driver entrance
+* @pdev: pointer to platform device
+* Return : int
+*
+*/
+static int hns_roce_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct hns_roce_dev *hr_dev;
+	struct device *dev = &pdev->dev;
+
+	hr_dev = (struct hns_roce_dev *)ib_alloc_device(sizeof(*hr_dev));
+	if (!hr_dev)
+		return -ENOMEM;
+
+	memset((u8 *)hr_dev + sizeof(struct ib_device), 0,
+		sizeof(struct hns_roce_dev) - sizeof(struct ib_device));
+
+	hr_dev->pdev = pdev;
+	platform_set_drvdata(pdev, hr_dev);
+
+	if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64ULL)) &&
+	    dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32ULL))) {
+		dev_err(dev, "Not usable DMA addressing mode\n");
+		ret = -EIO;
+		goto error_failed_get_cfg;
+	}
+
+	ret = hns_roce_get_cfg(hr_dev);
+	if (ret) {
+		dev_err(dev, "Get Configuration failed!\n");
+		goto error_failed_get_cfg;
+	}
+
+	ret = hr_dev->hw->reset(hr_dev, true);
+	if (ret) {
+		dev_err(dev, "Reset RoCE engine failed!\n");
+		goto error_failed_get_cfg;
+	}
+
+	hr_dev->hw->hw_profile(hr_dev);
+
+	ret = hns_roce_cmd_init(hr_dev);
+	if (ret) {
+		dev_err(dev, "cmd init failed!\n");
+		goto error_failed_cmd_init;
+	}
+
+	ret = hns_roce_init_eq_table(hr_dev);
+	if (ret) {
+		dev_err(dev, "eq init failed!\n");
+		goto error_failed_eq_table;
+	}
+
+	if (hr_dev->cmd_mod) {
+		ret = hns_roce_cmd_use_events(hr_dev);
+		if (ret) {
+			dev_err(dev, "Switch to event-driven cmd failed!\n");
+			goto error_failed_use_event;
+		}
+	}
+
+	ret = hns_roce_init_hem(hr_dev);
+	if (ret) {
+		dev_err(dev, "init HEM(Hardware Entry Memory) failed!\n");
+		goto error_failed_init_hem;
+	}
+
+	ret = hns_roce_setup_hca(hr_dev);
+	if (ret) {
+		dev_err(dev, "setup hca failed!\n");
+		goto error_failed_setup_hca;
+	}
+
+	ret = hr_dev->hw->hw_init(hr_dev);
+	if (ret) {
+		dev_err(dev, "hw_init failed!\n");
+		goto error_failed_engine_init;
+	}
+
+	ret = hns_roce_register_device(hr_dev);
+	if (ret)
+		goto error_failed_register_device;
+
+	return 0;
+
+error_failed_register_device:
+	hr_dev->hw->hw_exit(hr_dev);
+
+error_failed_engine_init:
+	hns_roce_cleanup_bitmap(hr_dev);
+
+error_failed_setup_hca:
+	hns_roce_cleanup_hem(hr_dev);
+
+error_failed_init_hem:
+	if (hr_dev->cmd_mod)
+		hns_roce_cmd_use_polling(hr_dev);
+
+error_failed_use_event:
+	hns_roce_cleanup_eq_table(hr_dev);
+
+error_failed_eq_table:
+	hns_roce_cmd_cleanup(hr_dev);
+
+error_failed_cmd_init:
+	ret = hr_dev->hw->reset(hr_dev, false);
+	if (ret)
+		dev_err(&hr_dev->pdev->dev, "roce_engine reset fail\n");
+
+error_failed_get_cfg:
+	ib_dealloc_device(&hr_dev->ib_dev);
+
+	return ret;
+}
+
+/**
+* hns_roce_remove - remove RoCE device
+* @pdev: pointer to platform device
+*/
+static int hns_roce_remove(struct platform_device *pdev)
+{
+	struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev);
+
+	hns_roce_unregister_device(hr_dev);
+	hr_dev->hw->hw_exit(hr_dev);
+	hns_roce_cleanup_bitmap(hr_dev);
+	hns_roce_cleanup_hem(hr_dev);
+
+	if (hr_dev->cmd_mod)
+		hns_roce_cmd_use_polling(hr_dev);
+
+	hns_roce_cleanup_eq_table(hr_dev);
+	hns_roce_cmd_cleanup(hr_dev);
+	hr_dev->hw->reset(hr_dev, false);
+
+	ib_dealloc_device(&hr_dev->ib_dev);
+
+	return 0;
+}
+
+static struct platform_driver hns_roce_driver = {
+	.probe = hns_roce_probe,
+	.remove = hns_roce_remove,
+	.driver = {
+		.name = DRV_NAME,
+		.of_match_table = hns_roce_of_match,
+		.acpi_match_table = ACPI_PTR(hns_roce_acpi_match),
+	},
+};
+
+module_platform_driver(hns_roce_driver);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
+MODULE_AUTHOR("Nenglong Zhao <zhaonenglong@hisilicon.com>");
+MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
+MODULE_DESCRIPTION("HNS RoCE Driver");
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
new file mode 100644
index 000000000000..59f5e2be046b
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -0,0 +1,614 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/platform_device.h>
+#include <rdma/ib_umem.h>
+#include "hns_roce_device.h"
+#include "hns_roce_cmd.h"
+#include "hns_roce_hem.h"
+
+static u32 hw_index_to_key(unsigned long ind)
+{
+	return (u32)(ind >> 24) | (ind << 8);
+}
+
+static unsigned long key_to_hw_index(u32 key)
+{
+	return (key << 24) | (key >> 8);
+}
+
+static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev,
+			      struct hns_roce_cmd_mailbox *mailbox,
+			      unsigned long mpt_index)
+{
+	return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0,
+				 HNS_ROCE_CMD_SW2HW_MPT,
+				 HNS_ROCE_CMD_TIME_CLASS_B);
+}
+
+static int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
+			      struct hns_roce_cmd_mailbox *mailbox,
+			      unsigned long mpt_index)
+{
+	return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0,
+				 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT,
+				 HNS_ROCE_CMD_TIME_CLASS_B);
+}
+
+static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order,
+				unsigned long *seg)
+{
+	int o;
+	u32 m;
+
+	spin_lock(&buddy->lock);
+
+	for (o = order; o <= buddy->max_order; ++o) {
+		if (buddy->num_free[o]) {
+			m = 1 << (buddy->max_order - o);
+			*seg = find_first_bit(buddy->bits[o], m);
+			if (*seg < m)
+				goto found;
+		}
+	}
+	spin_unlock(&buddy->lock);
+	return -1;
+
+ found:
+	clear_bit(*seg, buddy->bits[o]);
+	--buddy->num_free[o];
+
+	while (o > order) {
+		--o;
+		*seg <<= 1;
+		set_bit(*seg ^ 1, buddy->bits[o]);
+		++buddy->num_free[o];
+	}
+
+	spin_unlock(&buddy->lock);
+
+	*seg <<= order;
+	return 0;
+}
+
+static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg,
+				int order)
+{
+	seg >>= order;
+
+	spin_lock(&buddy->lock);
+
+	while (test_bit(seg ^ 1, buddy->bits[order])) {
+		clear_bit(seg ^ 1, buddy->bits[order]);
+		--buddy->num_free[order];
+		seg >>= 1;
+		++order;
+	}
+
+	set_bit(seg, buddy->bits[order]);
+	++buddy->num_free[order];
+
+	spin_unlock(&buddy->lock);
+}
+
+static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order)
+{
+	int i, s;
+
+	buddy->max_order = max_order;
+	spin_lock_init(&buddy->lock);
+
+	buddy->bits = kzalloc((buddy->max_order + 1) * sizeof(long *),
+			       GFP_KERNEL);
+	buddy->num_free = kzalloc((buddy->max_order + 1) * sizeof(int *),
+				   GFP_KERNEL);
+	if (!buddy->bits || !buddy->num_free)
+		goto err_out;
+
+	for (i = 0; i <= buddy->max_order; ++i) {
+		s = BITS_TO_LONGS(1 << (buddy->max_order - i));
+		buddy->bits[i] = kmalloc_array(s, sizeof(long), GFP_KERNEL);
+		if (!buddy->bits[i])
+			goto err_out_free;
+
+		bitmap_zero(buddy->bits[i], 1 << (buddy->max_order - i));
+	}
+
+	set_bit(0, buddy->bits[buddy->max_order]);
+	buddy->num_free[buddy->max_order] = 1;
+
+	return 0;
+
+err_out_free:
+	for (i = 0; i <= buddy->max_order; ++i)
+		kfree(buddy->bits[i]);
+
+err_out:
+	kfree(buddy->bits);
+	kfree(buddy->num_free);
+	return -ENOMEM;
+}
+
+static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy)
+{
+	int i;
+
+	for (i = 0; i <= buddy->max_order; ++i)
+		kfree(buddy->bits[i]);
+
+	kfree(buddy->bits);
+	kfree(buddy->num_free);
+}
+
+static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order,
+				    unsigned long *seg)
+{
+	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
+	int ret = 0;
+
+	ret = hns_roce_buddy_alloc(&mr_table->mtt_buddy, order, seg);
+	if (ret == -1)
+		return -1;
+
+	if (hns_roce_table_get_range(hr_dev, &mr_table->mtt_table, *seg,
+				     *seg + (1 << order) - 1)) {
+		hns_roce_buddy_free(&mr_table->mtt_buddy, *seg, order);
+		return -1;
+	}
+
+	return 0;
+}
+
+int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
+		      struct hns_roce_mtt *mtt)
+{
+	int ret = 0;
+	int i;
+
+	/* Page num is zero, correspond to DMA memory register */
+	if (!npages) {
+		mtt->order = -1;
+		mtt->page_shift = HNS_ROCE_HEM_PAGE_SHIFT;
+		return 0;
+	}
+
+	/* Note: if page_shift is zero, FAST memory regsiter */
+	mtt->page_shift = page_shift;
+
+	/* Compute MTT entry necessary */
+	for (mtt->order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i < npages;
+	     i <<= 1)
+		++mtt->order;
+
+	/* Allocate MTT entry */
+	ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg);
+	if (ret == -1)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt)
+{
+	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
+
+	if (mtt->order < 0)
+		return;
+
+	hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg, mtt->order);
+	hns_roce_table_put_range(hr_dev, &mr_table->mtt_table, mtt->first_seg,
+				 mtt->first_seg + (1 << mtt->order) - 1);
+}
+
+static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova,
+			     u64 size, u32 access, int npages,
+			     struct hns_roce_mr *mr)
+{
+	unsigned long index = 0;
+	int ret = 0;
+	struct device *dev = &hr_dev->pdev->dev;
+
+	/* Allocate a key for mr from mr_table */
+	ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index);
+	if (ret == -1)
+		return -ENOMEM;
+
+	mr->iova = iova;			/* MR va starting addr */
+	mr->size = size;			/* MR addr range */
+	mr->pd = pd;				/* MR num */
+	mr->access = access;			/* MR access permit */
+	mr->enabled = 0;			/* MR active status */
+	mr->key = hw_index_to_key(index);	/* MR key */
+
+	if (size == ~0ull) {
+		mr->type = MR_TYPE_DMA;
+		mr->pbl_buf = NULL;
+		mr->pbl_dma_addr = 0;
+	} else {
+		mr->type = MR_TYPE_MR;
+		mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
+						 &(mr->pbl_dma_addr),
+						 GFP_KERNEL);
+		if (!mr->pbl_buf)
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
+			     struct hns_roce_mr *mr)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	int npages = 0;
+	int ret;
+
+	if (mr->enabled) {
+		ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key)
+					 & (hr_dev->caps.num_mtpts - 1));
+		if (ret)
+			dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);
+	}
+
+	if (mr->size != ~0ULL) {
+		npages = ib_umem_page_count(mr->umem);
+		dma_free_coherent(dev, (unsigned int)(npages * 8), mr->pbl_buf,
+				  mr->pbl_dma_addr);
+	}
+
+	hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
+			     key_to_hw_index(mr->key));
+}
+
+static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
+			      struct hns_roce_mr *mr)
+{
+	int ret;
+	unsigned long mtpt_idx = key_to_hw_index(mr->key);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_cmd_mailbox *mailbox;
+	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
+
+	/* Prepare HEM entry memory */
+	ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);
+	if (ret)
+		return ret;
+
+	/* Allocate mailbox memory */
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox)) {
+		ret = PTR_ERR(mailbox);
+		goto err_table;
+	}
+
+	ret = hr_dev->hw->write_mtpt(mailbox->buf, mr, mtpt_idx);
+	if (ret) {
+		dev_err(dev, "Write mtpt fail!\n");
+		goto err_page;
+	}
+
+	ret = hns_roce_sw2hw_mpt(hr_dev, mailbox,
+				 mtpt_idx & (hr_dev->caps.num_mtpts - 1));
+	if (ret) {
+		dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);
+		goto err_page;
+	}
+
+	mr->enabled = 1;
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return 0;
+
+err_page:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+err_table:
+	hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);
+	return ret;
+}
+
+static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev,
+				    struct hns_roce_mtt *mtt, u32 start_index,
+				    u32 npages, u64 *page_list)
+{
+	u32 i = 0;
+	__le64 *mtts = NULL;
+	dma_addr_t dma_handle;
+	u32 s = start_index * sizeof(u64);
+
+	/* All MTTs must fit in the same page */
+	if (start_index / (PAGE_SIZE / sizeof(u64)) !=
+		(start_index + npages - 1) / (PAGE_SIZE / sizeof(u64)))
+		return -EINVAL;
+
+	if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))
+		return -EINVAL;
+
+	mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
+				mtt->first_seg + s / hr_dev->caps.mtt_entry_sz,
+				&dma_handle);
+	if (!mtts)
+		return -ENOMEM;
+
+	/* Save page addr, low 12 bits : 0 */
+	for (i = 0; i < npages; ++i)
+		mtts[i] = (cpu_to_le64(page_list[i])) >> PAGE_ADDR_SHIFT;
+
+	return 0;
+}
+
+static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev,
+			      struct hns_roce_mtt *mtt, u32 start_index,
+			      u32 npages, u64 *page_list)
+{
+	int chunk;
+	int ret;
+
+	if (mtt->order < 0)
+		return -EINVAL;
+
+	while (npages > 0) {
+		chunk = min_t(int, PAGE_SIZE / sizeof(u64), npages);
+
+		ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk,
+					       page_list);
+		if (ret)
+			return ret;
+
+		npages -= chunk;
+		start_index += chunk;
+		page_list += chunk;
+	}
+
+	return 0;
+}
+
+int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
+			   struct hns_roce_mtt *mtt, struct hns_roce_buf *buf)
+{
+	u32 i = 0;
+	int ret = 0;
+	u64 *page_list = NULL;
+
+	page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL);
+	if (!page_list)
+		return -ENOMEM;
+
+	for (i = 0; i < buf->npages; ++i) {
+		if (buf->nbufs == 1)
+			page_list[i] = buf->direct.map + (i << buf->page_shift);
+		else
+			page_list[i] = buf->page_list[i].map;
+
+	}
+	ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list);
+
+	kfree(page_list);
+
+	return ret;
+}
+
+int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
+	int ret = 0;
+
+	ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap,
+				   hr_dev->caps.num_mtpts,
+				   hr_dev->caps.num_mtpts - 1,
+				   hr_dev->caps.reserved_mrws, 0);
+	if (ret)
+		return ret;
+
+	ret = hns_roce_buddy_init(&mr_table->mtt_buddy,
+				  ilog2(hr_dev->caps.num_mtt_segs));
+	if (ret)
+		goto err_buddy;
+
+	return 0;
+
+err_buddy:
+	hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
+	return ret;
+}
+
+void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
+
+	hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
+	hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
+}
+
+struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc)
+{
+	int ret = 0;
+	struct hns_roce_mr *mr = NULL;
+
+	mr = kmalloc(sizeof(*mr), GFP_KERNEL);
+	if (mr == NULL)
+		return  ERR_PTR(-ENOMEM);
+
+	/* Allocate memory region key */
+	ret = hns_roce_mr_alloc(to_hr_dev(pd->device), to_hr_pd(pd)->pdn, 0,
+				~0ULL, acc, 0, mr);
+	if (ret)
+		goto err_free;
+
+	ret = hns_roce_mr_enable(to_hr_dev(pd->device), mr);
+	if (ret)
+		goto err_mr;
+
+	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
+	mr->umem = NULL;
+
+	return &mr->ibmr;
+
+err_mr:
+	hns_roce_mr_free(to_hr_dev(pd->device), mr);
+
+err_free:
+	kfree(mr);
+	return ERR_PTR(ret);
+}
+
+int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
+			       struct hns_roce_mtt *mtt, struct ib_umem *umem)
+{
+	struct scatterlist *sg;
+	int i, k, entry;
+	int ret = 0;
+	u64 *pages;
+	u32 n;
+	int len;
+
+	pages = (u64 *) __get_free_page(GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
+
+	i = n = 0;
+
+	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
+		len = sg_dma_len(sg) >> mtt->page_shift;
+		for (k = 0; k < len; ++k) {
+			pages[i++] = sg_dma_address(sg) + umem->page_size * k;
+			if (i == PAGE_SIZE / sizeof(u64)) {
+				ret = hns_roce_write_mtt(hr_dev, mtt, n, i,
+							 pages);
+				if (ret)
+					goto out;
+				n += i;
+				i = 0;
+			}
+		}
+	}
+
+	if (i)
+		ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);
+
+out:
+	free_page((unsigned long) pages);
+	return ret;
+}
+
+static int hns_roce_ib_umem_write_mr(struct hns_roce_mr *mr,
+				     struct ib_umem *umem)
+{
+	int i = 0;
+	int entry;
+	struct scatterlist *sg;
+
+	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
+		mr->pbl_buf[i] = ((u64)sg_dma_address(sg)) >> 12;
+		i++;
+	}
+
+	/* Memory barrier */
+	mb();
+
+	return 0;
+}
+
+struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+				   u64 virt_addr, int access_flags,
+				   struct ib_udata *udata)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_mr *mr = NULL;
+	int ret = 0;
+	int n = 0;
+
+	mr = kmalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
+
+	mr->umem = ib_umem_get(pd->uobject->context, start, length,
+			       access_flags, 0);
+	if (IS_ERR(mr->umem)) {
+		ret = PTR_ERR(mr->umem);
+		goto err_free;
+	}
+
+	n = ib_umem_page_count(mr->umem);
+	if (mr->umem->page_size != HNS_ROCE_HEM_PAGE_SIZE) {
+		dev_err(dev, "Just support 4K page size but is 0x%x now!\n",
+			mr->umem->page_size);
+	}
+
+	if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) {
+		dev_err(dev, " MR len %lld err. MR is limited to 4G at most!\n",
+			length);
+		goto err_umem;
+	}
+
+	ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length,
+				access_flags, n, mr);
+	if (ret)
+		goto err_umem;
+
+	ret = hns_roce_ib_umem_write_mr(mr, mr->umem);
+	if (ret)
+		goto err_mr;
+
+	ret = hns_roce_mr_enable(hr_dev, mr);
+	if (ret)
+		goto err_mr;
+
+	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
+
+	return &mr->ibmr;
+
+err_mr:
+	hns_roce_mr_free(hr_dev, mr);
+
+err_umem:
+	ib_umem_release(mr->umem);
+
+err_free:
+	kfree(mr);
+	return ERR_PTR(ret);
+}
+
+int hns_roce_dereg_mr(struct ib_mr *ibmr)
+{
+	struct hns_roce_mr *mr = to_hr_mr(ibmr);
+
+	hns_roce_mr_free(to_hr_dev(ibmr->device), mr);
+	if (mr->umem)
+		ib_umem_release(mr->umem);
+
+	kfree(mr);
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c
new file mode 100644
index 000000000000..16271b5bd170
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_pd.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/platform_device.h>
+#include "hns_roce_device.h"
+
+static int hns_roce_pd_alloc(struct hns_roce_dev *hr_dev, unsigned long *pdn)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	unsigned long pd_number;
+	int ret = 0;
+
+	ret = hns_roce_bitmap_alloc(&hr_dev->pd_bitmap, &pd_number);
+	if (ret == -1) {
+		dev_err(dev, "alloc pdn from pdbitmap failed\n");
+		return -ENOMEM;
+	}
+
+	*pdn = pd_number;
+
+	return 0;
+}
+
+static void hns_roce_pd_free(struct hns_roce_dev *hr_dev, unsigned long pdn)
+{
+	hns_roce_bitmap_free(&hr_dev->pd_bitmap, pdn);
+}
+
+int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev)
+{
+	return hns_roce_bitmap_init(&hr_dev->pd_bitmap, hr_dev->caps.num_pds,
+				    hr_dev->caps.num_pds - 1,
+				    hr_dev->caps.reserved_pds, 0);
+}
+
+void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev)
+{
+	hns_roce_bitmap_cleanup(&hr_dev->pd_bitmap);
+}
+
+struct ib_pd *hns_roce_alloc_pd(struct ib_device *ib_dev,
+				struct ib_ucontext *context,
+				struct ib_udata *udata)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_pd *pd;
+	int ret;
+
+	pd = kmalloc(sizeof(*pd), GFP_KERNEL);
+	if (!pd)
+		return ERR_PTR(-ENOMEM);
+
+	ret = hns_roce_pd_alloc(to_hr_dev(ib_dev), &pd->pdn);
+	if (ret) {
+		kfree(pd);
+		dev_err(dev, "[alloc_pd]hns_roce_pd_alloc failed!\n");
+		return ERR_PTR(ret);
+	}
+
+	if (context) {
+		if (ib_copy_to_udata(udata, &pd->pdn, sizeof(u64))) {
+			hns_roce_pd_free(to_hr_dev(ib_dev), pd->pdn);
+			dev_err(dev, "[alloc_pd]ib_copy_to_udata failed!\n");
+			kfree(pd);
+			return ERR_PTR(-EFAULT);
+		}
+	}
+
+	return &pd->ibpd;
+}
+
+int hns_roce_dealloc_pd(struct ib_pd *pd)
+{
+	hns_roce_pd_free(to_hr_dev(pd->device), to_hr_pd(pd)->pdn);
+	kfree(to_hr_pd(pd));
+
+	return 0;
+}
+
+int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
+{
+	struct resource *res;
+	int ret = 0;
+
+	/* Using bitmap to manager UAR index */
+	ret = hns_roce_bitmap_alloc(&hr_dev->uar_table.bitmap, &uar->index);
+	if (ret == -1)
+		return -ENOMEM;
+
+	uar->index = (uar->index - 1) % hr_dev->caps.phy_num_uars + 1;
+
+	res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0);
+	uar->pfn = ((res->start) >> PAGE_SHIFT) + uar->index;
+
+	return 0;
+}
+
+void hns_roce_uar_free(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
+{
+	hns_roce_bitmap_free(&hr_dev->uar_table.bitmap, uar->index);
+}
+
+int hns_roce_init_uar_table(struct hns_roce_dev *hr_dev)
+{
+	return hns_roce_bitmap_init(&hr_dev->uar_table.bitmap,
+				    hr_dev->caps.num_uars,
+				    hr_dev->caps.num_uars - 1,
+				    hr_dev->caps.reserved_uars, 0);
+}
+
+void hns_roce_cleanup_uar_table(struct hns_roce_dev *hr_dev)
+{
+	hns_roce_bitmap_cleanup(&hr_dev->uar_table.bitmap);
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
new file mode 100644
index 000000000000..645c18d809a5
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -0,0 +1,855 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/platform_device.h>
+#include <rdma/ib_umem.h>
+#include "hns_roce_common.h"
+#include "hns_roce_device.h"
+#include "hns_roce_hem.h"
+#include "hns_roce_user.h"
+
+#define DB_REG_OFFSET			0x1000
+#define SQP_NUM				12
+
+void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
+{
+	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_qp *qp;
+
+	spin_lock(&qp_table->lock);
+
+	qp = __hns_roce_qp_lookup(hr_dev, qpn);
+	if (qp)
+		atomic_inc(&qp->refcount);
+
+	spin_unlock(&qp_table->lock);
+
+	if (!qp) {
+		dev_warn(dev, "Async event for bogus QP %08x\n", qpn);
+		return;
+	}
+
+	qp->event(qp, (enum hns_roce_event)event_type);
+
+	if (atomic_dec_and_test(&qp->refcount))
+		complete(&qp->free);
+}
+
+static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
+				 enum hns_roce_event type)
+{
+	struct ib_event event;
+	struct ib_qp *ibqp = &hr_qp->ibqp;
+
+	if (ibqp->event_handler) {
+		event.device = ibqp->device;
+		event.element.qp = ibqp;
+		switch (type) {
+		case HNS_ROCE_EVENT_TYPE_PATH_MIG:
+			event.event = IB_EVENT_PATH_MIG;
+			break;
+		case HNS_ROCE_EVENT_TYPE_COMM_EST:
+			event.event = IB_EVENT_COMM_EST;
+			break;
+		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
+			event.event = IB_EVENT_SQ_DRAINED;
+			break;
+		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
+			event.event = IB_EVENT_QP_LAST_WQE_REACHED;
+			break;
+		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+			event.event = IB_EVENT_QP_FATAL;
+			break;
+		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
+			event.event = IB_EVENT_PATH_MIG_ERR;
+			break;
+		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
+			event.event = IB_EVENT_QP_REQ_ERR;
+			break;
+		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
+			event.event = IB_EVENT_QP_ACCESS_ERR;
+			break;
+		default:
+			dev_dbg(ibqp->device->dma_device, "roce_ib: Unexpected event type %d on QP %06lx\n",
+				type, hr_qp->qpn);
+			return;
+		}
+		ibqp->event_handler(&event, ibqp->qp_context);
+	}
+}
+
+static int hns_roce_reserve_range_qp(struct hns_roce_dev *hr_dev, int cnt,
+				     int align, unsigned long *base)
+{
+	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+	int ret = 0;
+	unsigned long qpn;
+
+	ret = hns_roce_bitmap_alloc_range(&qp_table->bitmap, cnt, align, &qpn);
+	if (ret == -1)
+		return -ENOMEM;
+
+	*base = qpn;
+
+	return 0;
+}
+
+enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state)
+{
+	switch (state) {
+	case IB_QPS_RESET:
+		return HNS_ROCE_QP_STATE_RST;
+	case IB_QPS_INIT:
+		return HNS_ROCE_QP_STATE_INIT;
+	case IB_QPS_RTR:
+		return HNS_ROCE_QP_STATE_RTR;
+	case IB_QPS_RTS:
+		return HNS_ROCE_QP_STATE_RTS;
+	case IB_QPS_SQD:
+		return HNS_ROCE_QP_STATE_SQD;
+	case IB_QPS_ERR:
+		return HNS_ROCE_QP_STATE_ERR;
+	default:
+		return HNS_ROCE_QP_NUM_STATE;
+	}
+}
+
+static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
+				 struct hns_roce_qp *hr_qp)
+{
+	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+	int ret;
+
+	if (!qpn)
+		return -EINVAL;
+
+	hr_qp->qpn = qpn;
+
+	spin_lock_irq(&qp_table->lock);
+	ret = radix_tree_insert(&hr_dev->qp_table_tree,
+				hr_qp->qpn & (hr_dev->caps.num_qps - 1), hr_qp);
+	spin_unlock_irq(&qp_table->lock);
+	if (ret) {
+		dev_err(&hr_dev->pdev->dev, "QPC radix_tree_insert failed\n");
+		goto err_put_irrl;
+	}
+
+	atomic_set(&hr_qp->refcount, 1);
+	init_completion(&hr_qp->free);
+
+	return 0;
+
+err_put_irrl:
+
+	return ret;
+}
+
+static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
+			     struct hns_roce_qp *hr_qp)
+{
+	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+	struct device *dev = &hr_dev->pdev->dev;
+	int ret;
+
+	if (!qpn)
+		return -EINVAL;
+
+	hr_qp->qpn = qpn;
+
+	/* Alloc memory for QPC */
+	ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);
+	if (ret) {
+		dev_err(dev, "QPC table get failed\n");
+		goto err_out;
+	}
+
+	/* Alloc memory for IRRL */
+	ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
+	if (ret) {
+		dev_err(dev, "IRRL table get failed\n");
+		goto err_put_qp;
+	}
+
+	spin_lock_irq(&qp_table->lock);
+	ret = radix_tree_insert(&hr_dev->qp_table_tree,
+				hr_qp->qpn & (hr_dev->caps.num_qps - 1), hr_qp);
+	spin_unlock_irq(&qp_table->lock);
+	if (ret) {
+		dev_err(dev, "QPC radix_tree_insert failed\n");
+		goto err_put_irrl;
+	}
+
+	atomic_set(&hr_qp->refcount, 1);
+	init_completion(&hr_qp->free);
+
+	return 0;
+
+err_put_irrl:
+	hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
+
+err_put_qp:
+	hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
+
+err_out:
+	return ret;
+}
+
+void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
+{
+	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+	unsigned long flags;
+
+	spin_lock_irqsave(&qp_table->lock, flags);
+	radix_tree_delete(&hr_dev->qp_table_tree,
+			  hr_qp->qpn & (hr_dev->caps.num_qps - 1));
+	spin_unlock_irqrestore(&qp_table->lock, flags);
+}
+
+void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
+{
+	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+
+	if (atomic_dec_and_test(&hr_qp->refcount))
+		complete(&hr_qp->free);
+	wait_for_completion(&hr_qp->free);
+
+	if ((hr_qp->ibqp.qp_type) != IB_QPT_GSI) {
+		hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
+		hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
+	}
+}
+
+void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
+			       int cnt)
+{
+	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+
+	if (base_qpn < (hr_dev->caps.sqp_start + 2 * hr_dev->caps.num_ports))
+		return;
+
+	hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt);
+}
+
+static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,
+				struct ib_qp_cap *cap, int is_user, int has_srq,
+				struct hns_roce_qp *hr_qp)
+{
+	u32 max_cnt;
+	struct device *dev = &hr_dev->pdev->dev;
+
+	/* Check the validity of QP support capacity */
+	if (cap->max_recv_wr > hr_dev->caps.max_wqes ||
+	    cap->max_recv_sge > hr_dev->caps.max_rq_sg) {
+		dev_err(dev, "RQ WR or sge error!max_recv_wr=%d max_recv_sge=%d\n",
+			cap->max_recv_wr, cap->max_recv_sge);
+		return -EINVAL;
+	}
+
+	/* If srq exit, set zero for relative number of rq */
+	if (has_srq) {
+		if (cap->max_recv_wr) {
+			dev_dbg(dev, "srq no need config max_recv_wr\n");
+			return -EINVAL;
+		}
+
+		hr_qp->rq.wqe_cnt = hr_qp->rq.max_gs = 0;
+	} else {
+		if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) {
+			dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n");
+			return -EINVAL;
+		}
+
+		/* In v1 engine, parameter verification procession */
+		max_cnt = cap->max_recv_wr > HNS_ROCE_MIN_WQE_NUM ?
+			  cap->max_recv_wr : HNS_ROCE_MIN_WQE_NUM;
+		hr_qp->rq.wqe_cnt = roundup_pow_of_two(max_cnt);
+
+		if ((u32)hr_qp->rq.wqe_cnt > hr_dev->caps.max_wqes) {
+			dev_err(dev, "hns_roce_set_rq_size rq.wqe_cnt too large\n");
+			return -EINVAL;
+		}
+
+		max_cnt = max(1U, cap->max_recv_sge);
+		hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt);
+		/* WQE is fixed for 64B */
+		hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz);
+	}
+
+	cap->max_recv_wr = hr_qp->rq.max_post = hr_qp->rq.wqe_cnt;
+	cap->max_recv_sge = hr_qp->rq.max_gs;
+
+	return 0;
+}
+
+static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev,
+				     struct hns_roce_qp *hr_qp,
+				     struct hns_roce_ib_create_qp *ucmd)
+{
+	u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz);
+	u8 max_sq_stride = ilog2(roundup_sq_stride);
+
+	/* Sanity check SQ size before proceeding */
+	if ((u32)(1 << ucmd->log_sq_bb_count) > hr_dev->caps.max_wqes ||
+	     ucmd->log_sq_stride > max_sq_stride ||
+	     ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) {
+		dev_err(&hr_dev->pdev->dev, "check SQ size error!\n");
+		return -EINVAL;
+	}
+
+	hr_qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;
+	hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
+
+	/* Get buf size, SQ and RQ  are aligned to page_szie */
+	hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt <<
+					     hr_qp->rq.wqe_shift), PAGE_SIZE) +
+			   HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
+					     hr_qp->sq.wqe_shift), PAGE_SIZE);
+
+	hr_qp->sq.offset = 0;
+	hr_qp->rq.offset = HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
+					     hr_qp->sq.wqe_shift), PAGE_SIZE);
+
+	return 0;
+}
+
+static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev,
+				       struct ib_qp_cap *cap,
+				       enum ib_qp_type type,
+				       struct hns_roce_qp *hr_qp)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	u32 max_cnt;
+	(void)type;
+
+	if (cap->max_send_wr  > hr_dev->caps.max_wqes  ||
+	    cap->max_send_sge > hr_dev->caps.max_sq_sg ||
+	    cap->max_inline_data > hr_dev->caps.max_sq_inline) {
+		dev_err(dev, "hns_roce_set_kernel_sq_size error1\n");
+		return -EINVAL;
+	}
+
+	hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
+	hr_qp->sq_max_wqes_per_wr = 1;
+	hr_qp->sq_spare_wqes = 0;
+
+	/* In v1 engine, parameter verification procession */
+	max_cnt = cap->max_send_wr > HNS_ROCE_MIN_WQE_NUM ?
+		  cap->max_send_wr : HNS_ROCE_MIN_WQE_NUM;
+	hr_qp->sq.wqe_cnt = roundup_pow_of_two(max_cnt);
+	if ((u32)hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) {
+		dev_err(dev, "hns_roce_set_kernel_sq_size sq.wqe_cnt too large\n");
+		return -EINVAL;
+	}
+
+	/* Get data_seg numbers */
+	max_cnt = max(1U, cap->max_send_sge);
+	hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt);
+
+	/* Get buf size, SQ and RQ  are aligned to page_szie */
+	hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt <<
+					     hr_qp->rq.wqe_shift), PAGE_SIZE) +
+			   HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
+					     hr_qp->sq.wqe_shift), PAGE_SIZE);
+	hr_qp->sq.offset = 0;
+	hr_qp->rq.offset = HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
+					      hr_qp->sq.wqe_shift), PAGE_SIZE);
+
+	/* Get wr and sge number which send */
+	cap->max_send_wr = hr_qp->sq.max_post = hr_qp->sq.wqe_cnt;
+	cap->max_send_sge = hr_qp->sq.max_gs;
+
+	/* We don't support inline sends for kernel QPs (yet) */
+	cap->max_inline_data = 0;
+
+	return 0;
+}
+
+static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
+				     struct ib_pd *ib_pd,
+				     struct ib_qp_init_attr *init_attr,
+				     struct ib_udata *udata, unsigned long sqpn,
+				     struct hns_roce_qp *hr_qp)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_ib_create_qp ucmd;
+	unsigned long qpn = 0;
+	int ret = 0;
+
+	mutex_init(&hr_qp->mutex);
+	spin_lock_init(&hr_qp->sq.lock);
+	spin_lock_init(&hr_qp->rq.lock);
+
+	hr_qp->state = IB_QPS_RESET;
+
+	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
+		hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
+	else
+		hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
+
+	ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, !!ib_pd->uobject,
+				   !!init_attr->srq, hr_qp);
+	if (ret) {
+		dev_err(dev, "hns_roce_set_rq_size failed\n");
+		goto err_out;
+	}
+
+	if (ib_pd->uobject) {
+		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
+			dev_err(dev, "ib_copy_from_udata error for create qp\n");
+			ret = -EFAULT;
+			goto err_out;
+		}
+
+		ret = hns_roce_set_user_sq_size(hr_dev, hr_qp, &ucmd);
+		if (ret) {
+			dev_err(dev, "hns_roce_set_user_sq_size error for create qp\n");
+			goto err_out;
+		}
+
+		hr_qp->umem = ib_umem_get(ib_pd->uobject->context,
+					  ucmd.buf_addr, hr_qp->buff_size, 0,
+					  0);
+		if (IS_ERR(hr_qp->umem)) {
+			dev_err(dev, "ib_umem_get error for create qp\n");
+			ret = PTR_ERR(hr_qp->umem);
+			goto err_out;
+		}
+
+		ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(hr_qp->umem),
+				    ilog2((unsigned int)hr_qp->umem->page_size),
+				    &hr_qp->mtt);
+		if (ret) {
+			dev_err(dev, "hns_roce_mtt_init error for create qp\n");
+			goto err_buf;
+		}
+
+		ret = hns_roce_ib_umem_write_mtt(hr_dev, &hr_qp->mtt,
+						 hr_qp->umem);
+		if (ret) {
+			dev_err(dev, "hns_roce_ib_umem_write_mtt error for create qp\n");
+			goto err_mtt;
+		}
+	} else {
+		if (init_attr->create_flags &
+		    IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
+			dev_err(dev, "init_attr->create_flags error!\n");
+			ret = -EINVAL;
+			goto err_out;
+		}
+
+		if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) {
+			dev_err(dev, "init_attr->create_flags error!\n");
+			ret = -EINVAL;
+			goto err_out;
+		}
+
+		/* Set SQ size */
+		ret = hns_roce_set_kernel_sq_size(hr_dev, &init_attr->cap,
+						  init_attr->qp_type, hr_qp);
+		if (ret) {
+			dev_err(dev, "hns_roce_set_kernel_sq_size error!\n");
+			goto err_out;
+		}
+
+		/* QP doorbell register address */
+		hr_qp->sq.db_reg_l = hr_dev->reg_base + ROCEE_DB_SQ_L_0_REG +
+				     DB_REG_OFFSET * hr_dev->priv_uar.index;
+		hr_qp->rq.db_reg_l = hr_dev->reg_base +
+				     ROCEE_DB_OTHERS_L_0_REG +
+				     DB_REG_OFFSET * hr_dev->priv_uar.index;
+
+		/* Allocate QP buf */
+		if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, PAGE_SIZE * 2,
+				       &hr_qp->hr_buf)) {
+			dev_err(dev, "hns_roce_buf_alloc error!\n");
+			ret = -ENOMEM;
+			goto err_out;
+		}
+
+		/* Write MTT */
+		ret = hns_roce_mtt_init(hr_dev, hr_qp->hr_buf.npages,
+					hr_qp->hr_buf.page_shift, &hr_qp->mtt);
+		if (ret) {
+			dev_err(dev, "hns_roce_mtt_init error for kernel create qp\n");
+			goto err_buf;
+		}
+
+		ret = hns_roce_buf_write_mtt(hr_dev, &hr_qp->mtt,
+					     &hr_qp->hr_buf);
+		if (ret) {
+			dev_err(dev, "hns_roce_buf_write_mtt error for kernel create qp\n");
+			goto err_mtt;
+		}
+
+		hr_qp->sq.wrid = kmalloc_array(hr_qp->sq.wqe_cnt, sizeof(u64),
+					       GFP_KERNEL);
+		hr_qp->rq.wrid = kmalloc_array(hr_qp->rq.wqe_cnt, sizeof(u64),
+					       GFP_KERNEL);
+		if (!hr_qp->sq.wrid || !hr_qp->rq.wrid) {
+			ret = -ENOMEM;
+			goto err_wrid;
+		}
+	}
+
+	if (sqpn) {
+		qpn = sqpn;
+	} else {
+		/* Get QPN */
+		ret = hns_roce_reserve_range_qp(hr_dev, 1, 1, &qpn);
+		if (ret) {
+			dev_err(dev, "hns_roce_reserve_range_qp alloc qpn error\n");
+			goto err_wrid;
+		}
+	}
+
+	if ((init_attr->qp_type) == IB_QPT_GSI) {
+		ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp);
+		if (ret) {
+			dev_err(dev, "hns_roce_qp_alloc failed!\n");
+			goto err_qpn;
+		}
+	} else {
+		ret = hns_roce_qp_alloc(hr_dev, qpn, hr_qp);
+		if (ret) {
+			dev_err(dev, "hns_roce_qp_alloc failed!\n");
+			goto err_qpn;
+		}
+	}
+
+	if (sqpn)
+		hr_qp->doorbell_qpn = 1;
+	else
+		hr_qp->doorbell_qpn = cpu_to_le64(hr_qp->qpn);
+
+	hr_qp->event = hns_roce_ib_qp_event;
+
+	return 0;
+
+err_qpn:
+	if (!sqpn)
+		hns_roce_release_range_qp(hr_dev, qpn, 1);
+
+err_wrid:
+	kfree(hr_qp->sq.wrid);
+	kfree(hr_qp->rq.wrid);
+
+err_mtt:
+	hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
+
+err_buf:
+	if (ib_pd->uobject)
+		ib_umem_release(hr_qp->umem);
+	else
+		hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
+
+err_out:
+	return ret;
+}
+
+struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
+				 struct ib_qp_init_attr *init_attr,
+				 struct ib_udata *udata)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_sqp *hr_sqp;
+	struct hns_roce_qp *hr_qp;
+	int ret;
+
+	switch (init_attr->qp_type) {
+	case IB_QPT_RC: {
+		hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
+		if (!hr_qp)
+			return ERR_PTR(-ENOMEM);
+
+		ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 0,
+						hr_qp);
+		if (ret) {
+			dev_err(dev, "Create RC QP failed\n");
+			kfree(hr_qp);
+			return ERR_PTR(ret);
+		}
+
+		hr_qp->ibqp.qp_num = hr_qp->qpn;
+
+		break;
+	}
+	case IB_QPT_GSI: {
+		/* Userspace is not allowed to create special QPs: */
+		if (pd->uobject) {
+			dev_err(dev, "not support usr space GSI\n");
+			return ERR_PTR(-EINVAL);
+		}
+
+		hr_sqp = kzalloc(sizeof(*hr_sqp), GFP_KERNEL);
+		if (!hr_sqp)
+			return ERR_PTR(-ENOMEM);
+
+		hr_qp = &hr_sqp->hr_qp;
+
+		ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata,
+						hr_dev->caps.sqp_start +
+						hr_dev->caps.num_ports +
+						init_attr->port_num - 1, hr_qp);
+		if (ret) {
+			dev_err(dev, "Create GSI QP failed!\n");
+			kfree(hr_sqp);
+			return ERR_PTR(ret);
+		}
+
+		hr_qp->port = (init_attr->port_num - 1);
+		hr_qp->ibqp.qp_num = hr_dev->caps.sqp_start +
+				     hr_dev->caps.num_ports +
+				     init_attr->port_num - 1;
+		break;
+	}
+	default:{
+		dev_err(dev, "not support QP type %d\n", init_attr->qp_type);
+		return ERR_PTR(-EINVAL);
+	}
+	}
+
+	return &hr_qp->ibqp;
+}
+
+int to_hr_qp_type(int qp_type)
+{
+	int transport_type;
+
+	if (qp_type == IB_QPT_RC)
+		transport_type = SERV_TYPE_RC;
+	else if (qp_type == IB_QPT_UC)
+		transport_type = SERV_TYPE_UC;
+	else if (qp_type == IB_QPT_UD)
+		transport_type = SERV_TYPE_UD;
+	else if (qp_type == IB_QPT_GSI)
+		transport_type = SERV_TYPE_UD;
+	else
+		transport_type = -1;
+
+	return transport_type;
+}
+
+int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+		       int attr_mask, struct ib_udata *udata)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	enum ib_qp_state cur_state, new_state;
+	struct device *dev = &hr_dev->pdev->dev;
+	int ret = -EINVAL;
+	int p;
+
+	mutex_lock(&hr_qp->mutex);
+
+	cur_state = attr_mask & IB_QP_CUR_STATE ?
+		    attr->cur_qp_state : (enum ib_qp_state)hr_qp->state;
+	new_state = attr_mask & IB_QP_STATE ?
+		    attr->qp_state : cur_state;
+
+	if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask,
+				IB_LINK_LAYER_ETHERNET)) {
+		dev_err(dev, "ib_modify_qp_is_ok failed\n");
+		goto out;
+	}
+
+	if ((attr_mask & IB_QP_PORT) &&
+	    (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) {
+		dev_err(dev, "attr port_num invalid.attr->port_num=%d\n",
+			attr->port_num);
+		goto out;
+	}
+
+	if (attr_mask & IB_QP_PKEY_INDEX) {
+		p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
+		if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) {
+			dev_err(dev, "attr pkey_index invalid.attr->pkey_index=%d\n",
+				attr->pkey_index);
+			goto out;
+		}
+	}
+
+	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
+	    attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) {
+		dev_err(dev, "attr max_rd_atomic invalid.attr->max_rd_atomic=%d\n",
+			attr->max_rd_atomic);
+		goto out;
+	}
+
+	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
+	    attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) {
+		dev_err(dev, "attr max_dest_rd_atomic invalid.attr->max_dest_rd_atomic=%d\n",
+			attr->max_dest_rd_atomic);
+		goto out;
+	}
+
+	if (cur_state == new_state && cur_state == IB_QPS_RESET) {
+		ret = -EPERM;
+		dev_err(dev, "cur_state=%d new_state=%d\n", cur_state,
+			new_state);
+		goto out;
+	}
+
+	ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
+				    new_state);
+
+out:
+	mutex_unlock(&hr_qp->mutex);
+
+	return ret;
+}
+
+void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)
+		       __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
+{
+	if (send_cq == recv_cq) {
+		spin_lock_irq(&send_cq->lock);
+		__acquire(&recv_cq->lock);
+	} else if (send_cq->cqn < recv_cq->cqn) {
+		spin_lock_irq(&send_cq->lock);
+		spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
+	} else {
+		spin_lock_irq(&recv_cq->lock);
+		spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
+	}
+}
+
+void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
+			 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock)
+			 __releases(&recv_cq->lock)
+{
+	if (send_cq == recv_cq) {
+		__release(&recv_cq->lock);
+		spin_unlock_irq(&send_cq->lock);
+	} else if (send_cq->cqn < recv_cq->cqn) {
+		spin_unlock(&recv_cq->lock);
+		spin_unlock_irq(&send_cq->lock);
+	} else {
+		spin_unlock(&send_cq->lock);
+		spin_unlock_irq(&recv_cq->lock);
+	}
+}
+
+__be32 send_ieth(struct ib_send_wr *wr)
+{
+	switch (wr->opcode) {
+	case IB_WR_SEND_WITH_IMM:
+	case IB_WR_RDMA_WRITE_WITH_IMM:
+		return cpu_to_le32(wr->ex.imm_data);
+	case IB_WR_SEND_WITH_INV:
+		return cpu_to_le32(wr->ex.invalidate_rkey);
+	default:
+		return 0;
+	}
+}
+
+static void *get_wqe(struct hns_roce_qp *hr_qp, int offset)
+{
+
+	return hns_roce_buf_offset(&hr_qp->hr_buf, offset);
+}
+
+void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n)
+{
+	struct ib_qp *ibqp = &hr_qp->ibqp;
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+
+	if ((n < 0) || (n > hr_qp->rq.wqe_cnt)) {
+		dev_err(&hr_dev->pdev->dev, "rq wqe index:%d,rq wqe cnt:%d\r\n",
+			n, hr_qp->rq.wqe_cnt);
+		return NULL;
+	}
+
+	return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
+}
+
+void *get_send_wqe(struct hns_roce_qp *hr_qp, int n)
+{
+	struct ib_qp *ibqp = &hr_qp->ibqp;
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+
+	if ((n < 0) || (n > hr_qp->sq.wqe_cnt)) {
+		dev_err(&hr_dev->pdev->dev, "sq wqe index:%d,sq wqe cnt:%d\r\n",
+			n, hr_qp->sq.wqe_cnt);
+		return NULL;
+	}
+
+	return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
+}
+
+bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
+			  struct ib_cq *ib_cq)
+{
+	struct hns_roce_cq *hr_cq;
+	u32 cur;
+
+	cur = hr_wq->head - hr_wq->tail;
+	if (likely(cur + nreq < hr_wq->max_post))
+		return 0;
+
+	hr_cq = to_hr_cq(ib_cq);
+	spin_lock(&hr_cq->lock);
+	cur = hr_wq->head - hr_wq->tail;
+	spin_unlock(&hr_cq->lock);
+
+	return cur + nreq >= hr_wq->max_post;
+}
+
+int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+	int reserved_from_top = 0;
+	int ret;
+
+	spin_lock_init(&qp_table->lock);
+	INIT_RADIX_TREE(&hr_dev->qp_table_tree, GFP_ATOMIC);
+
+	/* A port include two SQP, six port total 12 */
+	ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps,
+				   hr_dev->caps.num_qps - 1,
+				   hr_dev->caps.sqp_start + SQP_NUM,
+				   reserved_from_top);
+	if (ret) {
+		dev_err(&hr_dev->pdev->dev, "qp bitmap init failed!error=%d\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)
+{
+	hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap);
+}
diff --git a/drivers/infiniband/hw/cxgb3/iwch_user.h b/drivers/infiniband/hw/hns/hns_roce_user.h
index a277c31fcaf7..a28f761a9f65 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_user.h
+++ b/drivers/infiniband/hw/hns/hns_roce_user.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2016 Hisilicon Limited.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -29,46 +29,25 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-#ifndef __IWCH_USER_H__
-#define __IWCH_USER_H__
 
-#define IWCH_UVERBS_ABI_VERSION	1
+#ifndef _HNS_ROCE_USER_H
+#define _HNS_ROCE_USER_H
 
-/*
- * Make sure that all structs defined in this file remain laid out so
- * that they pack the same way on 32-bit and 64-bit architectures (to
- * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __u64
- * instead.
- */
-struct iwch_create_cq_req {
-	__u64 user_rptr_addr;
+struct hns_roce_ib_create_cq {
+	__u64   buf_addr;
 };
 
-struct iwch_create_cq_resp_v0 {
-	__u64 key;
-	__u32 cqid;
-	__u32 size_log2;
+struct hns_roce_ib_create_qp {
+	__u64	buf_addr;
+	__u64   db_addr;
+	__u8    log_sq_bb_count;
+	__u8    log_sq_stride;
+	__u8    sq_no_prefetch;
+	__u8    reserved[5];
 };
 
-struct iwch_create_cq_resp {
-	__u64 key;
-	__u32 cqid;
-	__u32 size_log2;
-	__u32 memsize;
-	__u32 reserved;
+struct hns_roce_ib_alloc_ucontext_resp {
+	__u32	qp_tab_size;
 };
 
-struct iwch_create_qp_resp {
-	__u64 key;
-	__u64 db_key;
-	__u32 qpid;
-	__u32 size_log2;
-	__u32 sq_size_log2;
-	__u32 rq_size_log2;
-};
-
-struct iwch_reg_user_mr_resp {
-	__u32 pbl_addr;
-};
-#endif
+#endif /*_HNS_ROCE_USER_H */
diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h
index b738acdb9b02..8ec09e470f84 100644
--- a/drivers/infiniband/hw/i40iw/i40iw.h
+++ b/drivers/infiniband/hw/i40iw/i40iw.h
@@ -232,7 +232,7 @@ struct i40iw_device {
 	struct i40e_client *client;
 	struct i40iw_hw hw;
 	struct i40iw_cm_core cm_core;
-	unsigned long *mem_resources;
+	u8 *mem_resources;
 	unsigned long *allocated_qps;
 	unsigned long *allocated_cqs;
 	unsigned long *allocated_mrs;
@@ -435,8 +435,8 @@ static inline int i40iw_alloc_resource(struct i40iw_device *iwdev,
 	*next = resource_num + 1;
 	if (*next == max_resources)
 		*next = 0;
-	spin_unlock_irqrestore(&iwdev->resource_lock, flags);
 	*req_resource_num = resource_num;
+	spin_unlock_irqrestore(&iwdev->resource_lock, flags);
 
 	return 0;
 }
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c
index 5026dc79978a..85637696f6e9 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c
@@ -535,8 +535,8 @@ static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
 		buf += hdr_len;
 	}
 
-	if (pd_len)
-		memcpy(buf, pdata->addr, pd_len);
+	if (pdata && pdata->addr)
+		memcpy(buf, pdata->addr, pdata->size);
 
 	atomic_set(&sqbuf->refcount, 1);
 
@@ -3166,8 +3166,11 @@ void i40iw_setup_cm_core(struct i40iw_device *iwdev)
 	spin_lock_init(&cm_core->ht_lock);
 	spin_lock_init(&cm_core->listen_list_lock);
 
-	cm_core->event_wq = create_singlethread_workqueue("iwewq");
-	cm_core->disconn_wq = create_singlethread_workqueue("iwdwq");
+	cm_core->event_wq = alloc_ordered_workqueue("iwewq",
+						    WQ_MEM_RECLAIM);
+
+	cm_core->disconn_wq = alloc_ordered_workqueue("iwdwq",
+						      WQ_MEM_RECLAIM);
 }
 
 /**
@@ -3347,26 +3350,6 @@ int i40iw_cm_disconn(struct i40iw_qp *iwqp)
 }
 
 /**
- * i40iw_loopback_nop - Send a nop
- * @qp: associated hw qp
- */
-static void i40iw_loopback_nop(struct i40iw_sc_qp *qp)
-{
-	u64 *wqe;
-	u64 header;
-
-	wqe = qp->qp_uk.sq_base->elem;
-	set_64bit_val(wqe, 0, 0);
-	set_64bit_val(wqe, 8, 0);
-	set_64bit_val(wqe, 16, 0);
-
-	header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
-	    LS_64(0, I40IWQPSQ_SIGCOMPL) |
-	    LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
-	set_64bit_val(wqe, 24, header);
-}
-
-/**
  * i40iw_qp_disconnect - free qp and close cm
  * @iwqp: associate qp for the connection
  */
@@ -3638,7 +3621,7 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	} else {
 		if (iwqp->page)
 			iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page);
-		i40iw_loopback_nop(&iwqp->sc_qp);
+		dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp, NULL, 0, 0);
 	}
 
 	if (iwqp->page)
diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c
index 3ee0cad96bc6..0c92a40b3e86 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_hw.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_hw.c
@@ -265,6 +265,7 @@ void i40iw_next_iw_state(struct i40iw_qp *iwqp,
 		info.dont_send_fin = false;
 	if (iwqp->sc_qp.term_flags && (state == I40IW_QP_STATE_ERROR))
 		info.reset_tcp_conn = true;
+	iwqp->hw_iwarp_state = state;
 	i40iw_hw_modify_qp(iwqp->iwdev, iwqp, &info, 0);
 }
 
diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c
index 6e9081380a27..ac2f3cd9478c 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_main.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_main.c
@@ -100,7 +100,7 @@ static struct notifier_block i40iw_net_notifier = {
 	.notifier_call = i40iw_net_event
 };
 
-static int i40iw_notifiers_registered;
+static atomic_t i40iw_notifiers_registered;
 
 /**
  * i40iw_find_i40e_handler - find a handler given a client info
@@ -1342,12 +1342,11 @@ exit:
  */
 static void i40iw_register_notifiers(void)
 {
-	if (!i40iw_notifiers_registered) {
+	if (atomic_inc_return(&i40iw_notifiers_registered) == 1) {
 		register_inetaddr_notifier(&i40iw_inetaddr_notifier);
 		register_inet6addr_notifier(&i40iw_inetaddr6_notifier);
 		register_netevent_notifier(&i40iw_net_notifier);
 	}
-	i40iw_notifiers_registered++;
 }
 
 /**
@@ -1429,8 +1428,7 @@ static void i40iw_deinit_device(struct i40iw_device *iwdev, bool reset, bool del
 			i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
 		/* fallthrough */
 	case INET_NOTIFIER:
-		if (i40iw_notifiers_registered > 0) {
-			i40iw_notifiers_registered--;
+		if (!atomic_dec_return(&i40iw_notifiers_registered)) {
 			unregister_netevent_notifier(&i40iw_net_notifier);
 			unregister_inetaddr_notifier(&i40iw_inetaddr_notifier);
 			unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier);
@@ -1558,6 +1556,10 @@ static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
 	enum i40iw_status_code status;
 	struct i40iw_handler *hdl;
 
+	hdl = i40iw_find_netdev(ldev->netdev);
+	if (hdl)
+		return 0;
+
 	hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);
 	if (!hdl)
 		return -ENOMEM;
@@ -1613,7 +1615,7 @@ static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
 		status = i40iw_hmc_init_pble(&iwdev->sc_dev, iwdev->pble_rsrc);
 		if (status)
 			break;
-		iwdev->virtchnl_wq = create_singlethread_workqueue("iwvch");
+		iwdev->virtchnl_wq = alloc_ordered_workqueue("iwvch", WQ_MEM_RECLAIM);
 		i40iw_register_notifiers();
 		iwdev->init_state = INET_NOTIFIER;
 		status = i40iw_add_mac_ip(iwdev);
diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c
index 0e8db0a35141..6fd043b1d714 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_utils.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_utils.c
@@ -673,8 +673,11 @@ enum i40iw_status_code i40iw_free_virt_mem(struct i40iw_hw *hw,
 {
 	if (!mem)
 		return I40IW_ERR_PARAM;
+	/*
+	 * mem->va points to the parent of mem, so both mem and mem->va
+	 * can not be touched once mem->va is freed
+	 */
 	kfree(mem->va);
-	mem->va = NULL;
 	return 0;
 }
 
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 2360338877bf..6329c971c22f 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -794,7 +794,6 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd,
 	return &iwqp->ibqp;
 error:
 	i40iw_free_qp_resources(iwdev, iwqp, qp_num);
-	kfree(mem);
 	return ERR_PTR(err_code);
 }
 
@@ -1926,8 +1925,7 @@ static int i40iw_dereg_mr(struct ib_mr *ib_mr)
 		}
 		if (iwpbl->pbl_allocated)
 			i40iw_free_pble(iwdev->pble_rsrc, palloc);
-		kfree(iwpbl->iwmr);
-		iwpbl->iwmr = NULL;
+		kfree(iwmr);
 		return 0;
 	}
 
diff --git a/drivers/infiniband/hw/mlx4/alias_GUID.c b/drivers/infiniband/hw/mlx4/alias_GUID.c
index c74ef2620b85..5e9939045852 100644
--- a/drivers/infiniband/hw/mlx4/alias_GUID.c
+++ b/drivers/infiniband/hw/mlx4/alias_GUID.c
@@ -881,7 +881,7 @@ int mlx4_ib_init_alias_guid_service(struct mlx4_ib_dev *dev)
 
 		snprintf(alias_wq_name, sizeof alias_wq_name, "alias_guid%d", i);
 		dev->sriov.alias_guid.ports_guid[i].wq =
-			create_singlethread_workqueue(alias_wq_name);
+			alloc_ordered_workqueue(alias_wq_name, WQ_MEM_RECLAIM);
 		if (!dev->sriov.alias_guid.ports_guid[i].wq) {
 			ret = -ENOMEM;
 			goto err_thread;
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index d6fc8a6e8c33..1ea686b9e0f9 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -37,7 +37,7 @@
 #include <linux/slab.h>
 
 #include "mlx4_ib.h"
-#include "user.h"
+#include <rdma/mlx4-abi.h>
 
 static void mlx4_ib_cq_comp(struct mlx4_cq *cq)
 {
@@ -576,8 +576,8 @@ static int mlx4_ib_ipoib_csum_ok(__be16 status, __be16 checksum)
 		checksum == cpu_to_be16(0xffff);
 }
 
-static int use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc,
-			   unsigned tail, struct mlx4_cqe *cqe, int is_eth)
+static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc,
+			    unsigned tail, struct mlx4_cqe *cqe, int is_eth)
 {
 	struct mlx4_ib_proxy_sqp_hdr *hdr;
 
@@ -600,8 +600,6 @@ static int use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct
 		wc->slid        = be16_to_cpu(hdr->tun.slid_mac_47_32);
 		wc->sl          = (u8) (be16_to_cpu(hdr->tun.sl_vid) >> 12);
 	}
-
-	return 0;
 }
 
 static void mlx4_ib_qp_sw_comp(struct mlx4_ib_qp *qp, int num_entries,
@@ -689,12 +687,6 @@ repoll:
 	is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
 		MLX4_CQE_OPCODE_ERROR;
 
-	if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_OPCODE_NOP &&
-		     is_send)) {
-		pr_warn("Completion for NOP opcode detected!\n");
-		return -EINVAL;
-	}
-
 	/* Resize CQ in progress */
 	if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_RESIZE)) {
 		if (cq->resize_buf) {
@@ -720,12 +712,6 @@ repoll:
 		 */
 		mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev,
 				       be32_to_cpu(cqe->vlan_my_qpn));
-		if (unlikely(!mqp)) {
-			pr_warn("CQ %06x with entry for unknown QPN %06x\n",
-			       cq->mcq.cqn, be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK);
-			return -EINVAL;
-		}
-
 		*cur_qp = to_mibqp(mqp);
 	}
 
@@ -738,11 +724,6 @@ repoll:
 		/* SRQ is also in the radix tree */
 		msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev,
 				       srq_num);
-		if (unlikely(!msrq)) {
-			pr_warn("CQ %06x with entry for unknown SRQN %06x\n",
-				cq->mcq.cqn, srq_num);
-			return -EINVAL;
-		}
 	}
 
 	if (is_send) {
@@ -852,9 +833,11 @@ repoll:
 		if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) {
 			if ((*cur_qp)->mlx4_ib_qp_type &
 			    (MLX4_IB_QPT_PROXY_SMI_OWNER |
-			     MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI))
-				return use_tunnel_data(*cur_qp, cq, wc, tail,
-						       cqe, is_eth);
+			     MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
+				use_tunnel_data(*cur_qp, cq, wc, tail, cqe,
+						is_eth);
+				return 0;
+			}
 		}
 
 		wc->slid	   = be16_to_cpu(cqe->rlid);
@@ -891,7 +874,6 @@ int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 	struct mlx4_ib_qp *cur_qp = NULL;
 	unsigned long flags;
 	int npolled;
-	int err = 0;
 	struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device);
 
 	spin_lock_irqsave(&cq->lock, flags);
@@ -901,8 +883,7 @@ int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 	}
 
 	for (npolled = 0; npolled < num_entries; ++npolled) {
-		err = mlx4_ib_poll_one(cq, &cur_qp, wc + npolled);
-		if (err)
+		if (mlx4_ib_poll_one(cq, &cur_qp, wc + npolled))
 			break;
 	}
 
@@ -911,10 +892,7 @@ int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 out:
 	spin_unlock_irqrestore(&cq->lock, flags);
 
-	if (err == 0 || err == -EAGAIN)
-		return npolled;
-	else
-		return err;
+	return npolled;
 }
 
 int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c
index 9c2e53d28f98..1672907ff219 100644
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -230,6 +230,8 @@ static void smp_snoop(struct ib_device *ibdev, u8 port_num, const struct ib_mad
 	    mad->mad_hdr.method == IB_MGMT_METHOD_SET)
 		switch (mad->mad_hdr.attr_id) {
 		case IB_SMP_ATTR_PORT_INFO:
+			if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV)
+				return;
 			pinfo = (struct ib_port_info *) ((struct ib_smp *) mad)->data;
 			lid = be16_to_cpu(pinfo->lid);
 
@@ -245,6 +247,8 @@ static void smp_snoop(struct ib_device *ibdev, u8 port_num, const struct ib_mad
 			break;
 
 		case IB_SMP_ATTR_PKEY_TABLE:
+			if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV)
+				return;
 			if (!mlx4_is_mfunc(dev->dev)) {
 				mlx4_ib_dispatch_event(dev, port_num,
 						       IB_EVENT_PKEY_CHANGE);
@@ -281,6 +285,8 @@ static void smp_snoop(struct ib_device *ibdev, u8 port_num, const struct ib_mad
 			break;
 
 		case IB_SMP_ATTR_GUID_INFO:
+			if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV)
+				return;
 			/* paravirtualized master's guid is guid 0 -- does not change */
 			if (!mlx4_is_master(dev->dev))
 				mlx4_ib_dispatch_event(dev, port_num,
@@ -296,6 +302,26 @@ static void smp_snoop(struct ib_device *ibdev, u8 port_num, const struct ib_mad
 			}
 			break;
 
+		case IB_SMP_ATTR_SL_TO_VL_TABLE:
+			/* cache sl to vl mapping changes for use in
+			 * filling QP1 LRH VL field when sending packets
+			 */
+			if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV &&
+			    dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SL_TO_VL_CHANGE_EVENT)
+				return;
+			if (!mlx4_is_slave(dev->dev)) {
+				union sl2vl_tbl_to_u64 sl2vl64;
+				int jj;
+
+				for (jj = 0; jj < 8; jj++) {
+					sl2vl64.sl8[jj] = ((struct ib_smp *)mad)->data[jj];
+					pr_debug("port %u, sl2vl[%d] = %02x\n",
+						 port_num, jj, sl2vl64.sl8[jj]);
+				}
+				atomic64_set(&dev->sl2vl[port_num - 1], sl2vl64.sl64);
+			}
+			break;
+
 		default:
 			break;
 		}
@@ -345,7 +371,8 @@ static void node_desc_override(struct ib_device *dev,
 	    mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP &&
 	    mad->mad_hdr.attr_id == IB_SMP_ATTR_NODE_DESC) {
 		spin_lock_irqsave(&to_mdev(dev)->sm_lock, flags);
-		memcpy(((struct ib_smp *) mad)->data, dev->node_desc, 64);
+		memcpy(((struct ib_smp *) mad)->data, dev->node_desc,
+		       IB_DEVICE_NODE_DESC_MAX);
 		spin_unlock_irqrestore(&to_mdev(dev)->sm_lock, flags);
 	}
 }
@@ -805,8 +832,7 @@ static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 		return IB_MAD_RESULT_FAILURE;
 
 	if (!out_mad->mad_hdr.status) {
-		if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV))
-			smp_snoop(ibdev, port_num, in_mad, prev_lid);
+		smp_snoop(ibdev, port_num, in_mad, prev_lid);
 		/* slaves get node desc from FW */
 		if (!mlx4_is_slave(to_mdev(ibdev)->dev))
 			node_desc_override(ibdev, out_mad);
@@ -1037,6 +1063,23 @@ static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num)
 						    MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK);
 		}
 	}
+
+	/* Update the sl to vl table from inside client rereg
+	 * only if in secure-host mode (snooping is not possible)
+	 * and the sl-to-vl change event is not generated by FW.
+	 */
+	if (!mlx4_is_slave(dev->dev) &&
+	    dev->dev->flags & MLX4_FLAG_SECURE_HOST &&
+	    !(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SL_TO_VL_CHANGE_EVENT)) {
+		if (mlx4_is_master(dev->dev))
+			/* already in work queue from mlx4_ib_event queueing
+			 * mlx4_handle_port_mgmt_change_event, which calls
+			 * this procedure. Therefore, call sl2vl_update directly.
+			 */
+			mlx4_ib_sl2vl_update(dev, port_num);
+		else
+			mlx4_sched_ib_sl2vl_update_work(dev, port_num);
+	}
 	mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_CLIENT_REREGISTER);
 }
 
@@ -1128,6 +1171,27 @@ void handle_port_mgmt_change_event(struct work_struct *work)
 
 		/* Generate GUID changed event */
 		if (changed_attr & MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK) {
+			if (mlx4_is_master(dev->dev)) {
+				union ib_gid gid;
+				int err = 0;
+
+				if (!eqe->event.port_mgmt_change.params.port_info.gid_prefix)
+					err = __mlx4_ib_query_gid(&dev->ib_dev, port, 0, &gid, 1);
+				else
+					gid.global.subnet_prefix =
+						eqe->event.port_mgmt_change.params.port_info.gid_prefix;
+				if (err) {
+					pr_warn("Could not change QP1 subnet prefix for port %d: query_gid error (%d)\n",
+						port, err);
+				} else {
+					pr_debug("Changing QP1 subnet prefix for port %d. old=0x%llx. new=0x%llx\n",
+						 port,
+						 (u64)atomic64_read(&dev->sriov.demux[port - 1].subnet_prefix),
+						 be64_to_cpu(gid.global.subnet_prefix));
+					atomic64_set(&dev->sriov.demux[port - 1].subnet_prefix,
+						     be64_to_cpu(gid.global.subnet_prefix));
+				}
+			}
 			mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE);
 			/*if master, notify all slaves*/
 			if (mlx4_is_master(dev->dev))
@@ -1155,6 +1219,24 @@ void handle_port_mgmt_change_event(struct work_struct *work)
 			handle_slaves_guid_change(dev, port, tbl_block, change_bitmap);
 		}
 		break;
+
+	case MLX4_DEV_PMC_SUBTYPE_SL_TO_VL_MAP:
+		/* cache sl to vl mapping changes for use in
+		 * filling QP1 LRH VL field when sending packets
+		 */
+		if (!mlx4_is_slave(dev->dev)) {
+			union sl2vl_tbl_to_u64 sl2vl64;
+			int jj;
+
+			for (jj = 0; jj < 8; jj++) {
+				sl2vl64.sl8[jj] =
+					eqe->event.port_mgmt_change.params.sl2vl_tbl_change_info.sl2vl_table[jj];
+				pr_debug("port %u, sl2vl[%d] = %02x\n",
+					 port, jj, sl2vl64.sl8[jj]);
+			}
+			atomic64_set(&dev->sl2vl[port - 1], sl2vl64.sl64);
+		}
+		break;
 	default:
 		pr_warn("Unsupported subtype 0x%x for "
 			"Port Management Change event\n", eqe->subtype);
@@ -1897,7 +1979,7 @@ static int create_pv_resources(struct ib_device *ibdev, int slave, int port,
 		goto err_buf;
 	}
 
-	ctx->pd = ib_alloc_pd(ctx->ib_dev);
+	ctx->pd = ib_alloc_pd(ctx->ib_dev, 0);
 	if (IS_ERR(ctx->pd)) {
 		ret = PTR_ERR(ctx->pd);
 		pr_err("Couldn't create tunnel PD (%d)\n", ret);
@@ -2070,7 +2152,7 @@ static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev,
 	}
 
 	snprintf(name, sizeof name, "mlx4_ibt%d", port);
-	ctx->wq = create_singlethread_workqueue(name);
+	ctx->wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM);
 	if (!ctx->wq) {
 		pr_err("Failed to create tunnelling WQ for port %d\n", port);
 		ret = -ENOMEM;
@@ -2078,7 +2160,7 @@ static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev,
 	}
 
 	snprintf(name, sizeof name, "mlx4_ibud%d", port);
-	ctx->ud_wq = create_singlethread_workqueue(name);
+	ctx->ud_wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM);
 	if (!ctx->ud_wq) {
 		pr_err("Failed to create up/down WQ for port %d\n", port);
 		ret = -ENOMEM;
@@ -2202,6 +2284,8 @@ int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev)
 		if (err)
 			goto demux_err;
 		dev->sriov.demux[i].guid_cache[0] = gid.global.interface_id;
+		atomic64_set(&dev->sriov.demux[i].subnet_prefix,
+			     be64_to_cpu(gid.global.subnet_prefix));
 		err = alloc_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1,
 				      &dev->sriov.sqps[i]);
 		if (err)
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 2af44c2de262..b597e8227591 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -55,7 +55,7 @@
 #include <linux/mlx4/qp.h>
 
 #include "mlx4_ib.h"
-#include "user.h"
+#include <rdma/mlx4-abi.h>
 
 #define DRV_NAME	MLX4_IB_DRV_NAME
 #define DRV_VERSION	"2.2-1"
@@ -832,6 +832,66 @@ static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 	return ret;
 }
 
+static int mlx4_ib_query_sl2vl(struct ib_device *ibdev, u8 port, u64 *sl2vl_tbl)
+{
+	union sl2vl_tbl_to_u64 sl2vl64;
+	struct ib_smp *in_mad  = NULL;
+	struct ib_smp *out_mad = NULL;
+	int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
+	int err = -ENOMEM;
+	int jj;
+
+	if (mlx4_is_slave(to_mdev(ibdev)->dev)) {
+		*sl2vl_tbl = 0;
+		return 0;
+	}
+
+	in_mad  = kzalloc(sizeof(*in_mad), GFP_KERNEL);
+	out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
+	if (!in_mad || !out_mad)
+		goto out;
+
+	init_query_mad(in_mad);
+	in_mad->attr_id  = IB_SMP_ATTR_SL_TO_VL_TABLE;
+	in_mad->attr_mod = 0;
+
+	if (mlx4_is_mfunc(to_mdev(ibdev)->dev))
+		mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
+
+	err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL,
+			   in_mad, out_mad);
+	if (err)
+		goto out;
+
+	for (jj = 0; jj < 8; jj++)
+		sl2vl64.sl8[jj] = ((struct ib_smp *)out_mad)->data[jj];
+	*sl2vl_tbl = sl2vl64.sl64;
+
+out:
+	kfree(in_mad);
+	kfree(out_mad);
+	return err;
+}
+
+static void mlx4_init_sl2vl_tbl(struct mlx4_ib_dev *mdev)
+{
+	u64 sl2vl;
+	int i;
+	int err;
+
+	for (i = 1; i <= mdev->dev->caps.num_ports; i++) {
+		if (mdev->dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH)
+			continue;
+		err = mlx4_ib_query_sl2vl(&mdev->ib_dev, i, &sl2vl);
+		if (err) {
+			pr_err("Unable to get default sl to vl mapping for port %d.  Using all zeroes (%d)\n",
+			       i, err);
+			sl2vl = 0;
+		}
+		atomic64_set(&mdev->sl2vl[i - 1], sl2vl);
+	}
+}
+
 int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 			 u16 *pkey, int netw_view)
 {
@@ -886,7 +946,7 @@ static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask,
 		return -EOPNOTSUPP;
 
 	spin_lock_irqsave(&to_mdev(ibdev)->sm_lock, flags);
-	memcpy(ibdev->node_desc, props->node_desc, 64);
+	memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
 	spin_unlock_irqrestore(&to_mdev(ibdev)->sm_lock, flags);
 
 	/*
@@ -897,7 +957,7 @@ static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask,
 	if (IS_ERR(mailbox))
 		return 0;
 
-	memcpy(mailbox->buf, props->node_desc, 64);
+	memcpy(mailbox->buf, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
 	mlx4_cmd(to_mdev(ibdev)->dev, mailbox->dma, 1, 0,
 		 MLX4_CMD_SET_NODE, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
 
@@ -1259,7 +1319,7 @@ static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,
 	if (err)
 		goto err1;
 
-	xrcd->pd = ib_alloc_pd(ibdev);
+	xrcd->pd = ib_alloc_pd(ibdev, 0);
 	if (IS_ERR(xrcd->pd)) {
 		err = PTR_ERR(xrcd->pd);
 		goto err2;
@@ -1361,6 +1421,19 @@ struct mlx4_ib_steering {
 	union ib_gid gid;
 };
 
+#define LAST_ETH_FIELD vlan_tag
+#define LAST_IB_FIELD sl
+#define LAST_IPV4_FIELD dst_ip
+#define LAST_TCP_UDP_FIELD src_port
+
+/* Field is the last supported field */
+#define FIELDS_NOT_SUPPORTED(filter, field)\
+	memchr_inv((void *)&filter.field  +\
+		   sizeof(filter.field), 0,\
+		   sizeof(filter) -\
+		   offsetof(typeof(filter), field) -\
+		   sizeof(filter.field))
+
 static int parse_flow_attr(struct mlx4_dev *dev,
 			   u32 qp_num,
 			   union ib_flow_spec *ib_spec,
@@ -1370,6 +1443,9 @@ static int parse_flow_attr(struct mlx4_dev *dev,
 
 	switch (ib_spec->type) {
 	case IB_FLOW_SPEC_ETH:
+		if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
+			return -ENOTSUPP;
+
 		type = MLX4_NET_TRANS_RULE_ID_ETH;
 		memcpy(mlx4_spec->eth.dst_mac, ib_spec->eth.val.dst_mac,
 		       ETH_ALEN);
@@ -1379,6 +1455,9 @@ static int parse_flow_attr(struct mlx4_dev *dev,
 		mlx4_spec->eth.vlan_tag_msk = ib_spec->eth.mask.vlan_tag;
 		break;
 	case IB_FLOW_SPEC_IB:
+		if (FIELDS_NOT_SUPPORTED(ib_spec->ib.mask, LAST_IB_FIELD))
+			return -ENOTSUPP;
+
 		type = MLX4_NET_TRANS_RULE_ID_IB;
 		mlx4_spec->ib.l3_qpn =
 			cpu_to_be32(qp_num);
@@ -1388,6 +1467,9 @@ static int parse_flow_attr(struct mlx4_dev *dev,
 
 
 	case IB_FLOW_SPEC_IPV4:
+		if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
+			return -ENOTSUPP;
+
 		type = MLX4_NET_TRANS_RULE_ID_IPV4;
 		mlx4_spec->ipv4.src_ip = ib_spec->ipv4.val.src_ip;
 		mlx4_spec->ipv4.src_ip_msk = ib_spec->ipv4.mask.src_ip;
@@ -1397,6 +1479,9 @@ static int parse_flow_attr(struct mlx4_dev *dev,
 
 	case IB_FLOW_SPEC_TCP:
 	case IB_FLOW_SPEC_UDP:
+		if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, LAST_TCP_UDP_FIELD))
+			return -ENOTSUPP;
+
 		type = ib_spec->type == IB_FLOW_SPEC_TCP ?
 					MLX4_NET_TRANS_RULE_ID_TCP :
 					MLX4_NET_TRANS_RULE_ID_UDP;
@@ -2000,7 +2085,7 @@ static int init_node_data(struct mlx4_ib_dev *dev)
 	if (err)
 		goto out;
 
-	memcpy(dev->ib_dev.node_desc, out_mad->data, 64);
+	memcpy(dev->ib_dev.node_desc, out_mad->data, IB_DEVICE_NODE_DESC_MAX);
 
 	in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
 
@@ -2202,6 +2287,9 @@ static int mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev *ibdev)
 	bool per_port = !!(ibdev->dev->caps.flags2 &
 		MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT);
 
+	if (mlx4_is_slave(ibdev->dev))
+		return 0;
+
 	for (i = 0; i < MLX4_DIAG_COUNTERS_TYPES; i++) {
 		/* i == 1 means we are building port counters */
 		if (i && !per_port)
@@ -2650,6 +2738,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 
 	if (init_node_data(ibdev))
 		goto err_map;
+	mlx4_init_sl2vl_tbl(ibdev);
 
 	for (i = 0; i < ibdev->num_ports; ++i) {
 		mutex_init(&ibdev->counters_table[i].mutex);
@@ -3098,6 +3187,47 @@ static void handle_bonded_port_state_event(struct work_struct *work)
 	ib_dispatch_event(&ibev);
 }
 
+void mlx4_ib_sl2vl_update(struct mlx4_ib_dev *mdev, int port)
+{
+	u64 sl2vl;
+	int err;
+
+	err = mlx4_ib_query_sl2vl(&mdev->ib_dev, port, &sl2vl);
+	if (err) {
+		pr_err("Unable to get current sl to vl mapping for port %d.  Using all zeroes (%d)\n",
+		       port, err);
+		sl2vl = 0;
+	}
+	atomic64_set(&mdev->sl2vl[port - 1], sl2vl);
+}
+
+static void ib_sl2vl_update_work(struct work_struct *work)
+{
+	struct ib_event_work *ew = container_of(work, struct ib_event_work, work);
+	struct mlx4_ib_dev *mdev = ew->ib_dev;
+	int port = ew->port;
+
+	mlx4_ib_sl2vl_update(mdev, port);
+
+	kfree(ew);
+}
+
+void mlx4_sched_ib_sl2vl_update_work(struct mlx4_ib_dev *ibdev,
+				     int port)
+{
+	struct ib_event_work *ew;
+
+	ew = kmalloc(sizeof(*ew), GFP_ATOMIC);
+	if (ew) {
+		INIT_WORK(&ew->work, ib_sl2vl_update_work);
+		ew->port = port;
+		ew->ib_dev = ibdev;
+		queue_work(wq, &ew->work);
+	} else {
+		pr_err("failed to allocate memory for sl2vl update work\n");
+	}
+}
+
 static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
 			  enum mlx4_dev_event event, unsigned long param)
 {
@@ -3128,10 +3258,14 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
 	case MLX4_DEV_EVENT_PORT_UP:
 		if (p > ibdev->num_ports)
 			return;
-		if (mlx4_is_master(dev) &&
+		if (!mlx4_is_slave(dev) &&
 		    rdma_port_get_link_layer(&ibdev->ib_dev, p) ==
 			IB_LINK_LAYER_INFINIBAND) {
-			mlx4_ib_invalidate_all_guid_record(ibdev, p);
+			if (mlx4_is_master(dev))
+				mlx4_ib_invalidate_all_guid_record(ibdev, p);
+			if (ibdev->dev->flags & MLX4_FLAG_SECURE_HOST &&
+			    !(ibdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SL_TO_VL_CHANGE_EVENT))
+				mlx4_sched_ib_sl2vl_update_work(ibdev, p);
 		}
 		ibev.event = IB_EVENT_PORT_ACTIVE;
 		break;
@@ -3219,7 +3353,7 @@ static int __init mlx4_ib_init(void)
 {
 	int err;
 
-	wq = create_singlethread_workqueue("mlx4_ib");
+	wq = alloc_ordered_workqueue("mlx4_ib", WQ_MEM_RECLAIM);
 	if (!wq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c
index 8f7ad07915b0..a21d37f02f35 100644
--- a/drivers/infiniband/hw/mlx4/mcg.c
+++ b/drivers/infiniband/hw/mlx4/mcg.c
@@ -489,7 +489,7 @@ static u8 get_leave_state(struct mcast_group *group)
 		if (!group->members[i])
 			leave_state |= (1 << i);
 
-	return leave_state & (group->rec.scope_join_state & 7);
+	return leave_state & (group->rec.scope_join_state & 0xf);
 }
 
 static int join_group(struct mcast_group *group, int slave, u8 join_mask)
@@ -564,8 +564,8 @@ static void mlx4_ib_mcg_timeout_handler(struct work_struct *work)
 		} else
 			mcg_warn_group(group, "DRIVER BUG\n");
 	} else if (group->state == MCAST_LEAVE_SENT) {
-		if (group->rec.scope_join_state & 7)
-			group->rec.scope_join_state &= 0xf8;
+		if (group->rec.scope_join_state & 0xf)
+			group->rec.scope_join_state &= 0xf0;
 		group->state = MCAST_IDLE;
 		mutex_unlock(&group->lock);
 		if (release_group(group, 1))
@@ -605,7 +605,7 @@ static int handle_leave_req(struct mcast_group *group, u8 leave_mask,
 static int handle_join_req(struct mcast_group *group, u8 join_mask,
 			   struct mcast_req *req)
 {
-	u8 group_join_state = group->rec.scope_join_state & 7;
+	u8 group_join_state = group->rec.scope_join_state & 0xf;
 	int ref = 0;
 	u16 status;
 	struct ib_sa_mcmember_data *sa_data = (struct ib_sa_mcmember_data *)req->sa_mad.data;
@@ -690,8 +690,8 @@ static void mlx4_ib_mcg_work_handler(struct work_struct *work)
 			u8 cur_join_state;
 
 			resp_join_state = ((struct ib_sa_mcmember_data *)
-						group->response_sa_mad.data)->scope_join_state & 7;
-			cur_join_state = group->rec.scope_join_state & 7;
+						group->response_sa_mad.data)->scope_join_state & 0xf;
+			cur_join_state = group->rec.scope_join_state & 0xf;
 
 			if (method == IB_MGMT_METHOD_GET_RESP) {
 				/* successfull join */
@@ -710,7 +710,7 @@ process_requests:
 		req = list_first_entry(&group->pending_list, struct mcast_req,
 				       group_list);
 		sa_data = (struct ib_sa_mcmember_data *)req->sa_mad.data;
-		req_join_state = sa_data->scope_join_state & 0x7;
+		req_join_state = sa_data->scope_join_state & 0xf;
 
 		/* For a leave request, we will immediately answer the VF, and
 		 * update our internal counters. The actual leave will be sent
@@ -1045,7 +1045,7 @@ int mlx4_ib_mcg_port_init(struct mlx4_ib_demux_ctx *ctx)
 
 	atomic_set(&ctx->tid, 0);
 	sprintf(name, "mlx4_ib_mcg%d", ctx->port);
-	ctx->mcg_wq = create_singlethread_workqueue(name);
+	ctx->mcg_wq = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM);
 	if (!ctx->mcg_wq)
 		return -ENOMEM;
 
@@ -1246,7 +1246,7 @@ void clean_vf_mcast(struct mlx4_ib_demux_ctx *ctx, int slave)
 
 int mlx4_ib_mcg_init(void)
 {
-	clean_wq = create_singlethread_workqueue("mlx4_ib_mcg");
+	clean_wq = alloc_ordered_workqueue("mlx4_ib_mcg", WQ_MEM_RECLAIM);
 	if (!clean_wq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 7c5832ede4bd..35141f451e5c 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -448,7 +448,7 @@ struct mlx4_ib_demux_ctx {
 	struct workqueue_struct *wq;
 	struct workqueue_struct *ud_wq;
 	spinlock_t ud_lock;
-	__be64 subnet_prefix;
+	atomic64_t subnet_prefix;
 	__be64 guid_cache[128];
 	struct mlx4_ib_dev *dev;
 	/* the following lock protects both mcg_table and mcg_mgid0_list */
@@ -570,6 +570,7 @@ struct mlx4_ib_dev {
 	struct ib_mad_agent    *send_agent[MLX4_MAX_PORTS][2];
 	struct ib_ah	       *sm_ah[MLX4_MAX_PORTS];
 	spinlock_t		sm_lock;
+	atomic64_t		sl2vl[MLX4_MAX_PORTS];
 	struct mlx4_ib_sriov	sriov;
 
 	struct mutex		cap_mask_mutex;
@@ -600,6 +601,7 @@ struct ib_event_work {
 	struct work_struct	work;
 	struct mlx4_ib_dev	*ib_dev;
 	struct mlx4_eqe		ib_eqe;
+	int			port;
 };
 
 struct mlx4_ib_qp_tunnel_init_attr {
@@ -883,4 +885,9 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags,
 int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,
 				    u8 port_num, int index);
 
+void mlx4_sched_ib_sl2vl_update_work(struct mlx4_ib_dev *ibdev,
+				     int port);
+
+void mlx4_ib_sl2vl_update(struct mlx4_ib_dev *mdev, int port);
+
 #endif /* MLX4_IB_H */
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 768085f59566..570bc866b1d6 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -47,7 +47,7 @@
 #include <linux/mlx4/qp.h>
 
 #include "mlx4_ib.h"
-#include "user.h"
+#include <rdma/mlx4-abi.h>
 
 static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq,
 			     struct mlx4_ib_cq *recv_cq);
@@ -2405,6 +2405,22 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
 	return 0;
 }
 
+static u8 sl_to_vl(struct mlx4_ib_dev *dev, u8 sl, int port_num)
+{
+	union sl2vl_tbl_to_u64 tmp_vltab;
+	u8 vl;
+
+	if (sl > 15)
+		return 0xf;
+	tmp_vltab.sl64 = atomic64_read(&dev->sl2vl[port_num - 1]);
+	vl = tmp_vltab.sl8[sl >> 1];
+	if (sl & 1)
+		vl &= 0x0f;
+	else
+		vl >>= 4;
+	return vl;
+}
+
 #define MLX4_ROCEV2_QP1_SPORT 0xC000
 static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 			    void *wqe, unsigned *mlx_seg_len)
@@ -2493,24 +2509,27 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 		sqp->ud_header.grh.flow_label    =
 			ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
 		sqp->ud_header.grh.hop_limit     = ah->av.ib.hop_limit;
-		if (is_eth)
+		if (is_eth) {
 			memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16);
-		else {
-		if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
-			/* When multi-function is enabled, the ib_core gid
-			 * indexes don't necessarily match the hw ones, so
-			 * we must use our own cache */
-			sqp->ud_header.grh.source_gid.global.subnet_prefix =
-				to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
-						       subnet_prefix;
-			sqp->ud_header.grh.source_gid.global.interface_id =
-				to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
-					       guid_cache[ah->av.ib.gid_index];
-		} else
-			ib_get_cached_gid(ib_dev,
-					  be32_to_cpu(ah->av.ib.port_pd) >> 24,
-					  ah->av.ib.gid_index,
-					  &sqp->ud_header.grh.source_gid, NULL);
+		} else {
+			if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
+				/* When multi-function is enabled, the ib_core gid
+				 * indexes don't necessarily match the hw ones, so
+				 * we must use our own cache
+				 */
+				sqp->ud_header.grh.source_gid.global.subnet_prefix =
+					cpu_to_be64(atomic64_read(&(to_mdev(ib_dev)->sriov.
+								    demux[sqp->qp.port - 1].
+								    subnet_prefix)));
+				sqp->ud_header.grh.source_gid.global.interface_id =
+					to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
+						       guid_cache[ah->av.ib.gid_index];
+			} else {
+				ib_get_cached_gid(ib_dev,
+						  be32_to_cpu(ah->av.ib.port_pd) >> 24,
+						  ah->av.ib.gid_index,
+						  &sqp->ud_header.grh.source_gid, NULL);
+			}
 		}
 		memcpy(sqp->ud_header.grh.destination_gid.raw,
 		       ah->av.ib.dgid, 16);
@@ -2587,7 +2606,12 @@ static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
 			sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
 		}
 	} else {
-		sqp->ud_header.lrh.virtual_lane    = !sqp->qp.ibqp.qp_num ? 15 : 0;
+		sqp->ud_header.lrh.virtual_lane    = !sqp->qp.ibqp.qp_num ? 15 :
+							sl_to_vl(to_mdev(ib_dev),
+								 sqp->ud_header.lrh.service_level,
+								 sqp->qp.port);
+		if (sqp->qp.ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15)
+			return -EINVAL;
 		if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
 			sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
 	}
diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c
index 0597f3eef5d0..7dd3f267f06b 100644
--- a/drivers/infiniband/hw/mlx4/srq.c
+++ b/drivers/infiniband/hw/mlx4/srq.c
@@ -37,7 +37,7 @@
 #include <linux/vmalloc.h>
 
 #include "mlx4_ib.h"
-#include "user.h"
+#include <rdma/mlx4-abi.h>
 
 static void *get_wqe(struct mlx4_ib_srq *srq, int n)
 {
diff --git a/drivers/infiniband/hw/mlx4/user.h b/drivers/infiniband/hw/mlx4/user.h
deleted file mode 100644
index 07e6769ef43b..000000000000
--- a/drivers/infiniband/hw/mlx4/user.h
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
- * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef MLX4_IB_USER_H
-#define MLX4_IB_USER_H
-
-#include <linux/types.h>
-
-/*
- * Increment this value if any changes that break userspace ABI
- * compatibility are made.
- */
-
-#define MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION	3
-#define MLX4_IB_UVERBS_ABI_VERSION		4
-
-/*
- * Make sure that all structs defined in this file remain laid out so
- * that they pack the same way on 32-bit and 64-bit architectures (to
- * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __u64
- * instead.
- */
-
-struct mlx4_ib_alloc_ucontext_resp_v3 {
-	__u32	qp_tab_size;
-	__u16	bf_reg_size;
-	__u16	bf_regs_per_page;
-};
-
-struct mlx4_ib_alloc_ucontext_resp {
-	__u32	dev_caps;
-	__u32	qp_tab_size;
-	__u16	bf_reg_size;
-	__u16	bf_regs_per_page;
-	__u32	cqe_size;
-};
-
-struct mlx4_ib_alloc_pd_resp {
-	__u32	pdn;
-	__u32	reserved;
-};
-
-struct mlx4_ib_create_cq {
-	__u64	buf_addr;
-	__u64	db_addr;
-};
-
-struct mlx4_ib_create_cq_resp {
-	__u32	cqn;
-	__u32	reserved;
-};
-
-struct mlx4_ib_resize_cq {
-	__u64	buf_addr;
-};
-
-struct mlx4_ib_create_srq {
-	__u64	buf_addr;
-	__u64	db_addr;
-};
-
-struct mlx4_ib_create_srq_resp {
-	__u32	srqn;
-	__u32	reserved;
-};
-
-struct mlx4_ib_create_qp {
-	__u64	buf_addr;
-	__u64	db_addr;
-	__u8	log_sq_bb_count;
-	__u8	log_sq_stride;
-	__u8	sq_no_prefetch;
-	__u8	reserved[5];
-};
-
-#endif /* MLX4_IB_USER_H */
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 308a358e5b46..79d017baf6f4 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -35,7 +35,6 @@
 #include <rdma/ib_user_verbs.h>
 #include <rdma/ib_cache.h>
 #include "mlx5_ib.h"
-#include "user.h"
 
 static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq)
 {
@@ -553,12 +552,6 @@ repoll:
 		 * from the table.
 		 */
 		mqp = __mlx5_qp_lookup(dev->mdev, qpn);
-		if (unlikely(!mqp)) {
-			mlx5_ib_warn(dev, "CQE@CQ %06x for unknown QPN %6x\n",
-				     cq->mcq.cqn, qpn);
-			return -EINVAL;
-		}
-
 		*cur_qp = to_mibqp(mqp);
 	}
 
@@ -619,13 +612,6 @@ repoll:
 		read_lock(&dev->mdev->priv.mkey_table.lock);
 		mmkey = __mlx5_mr_lookup(dev->mdev,
 					 mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));
-		if (unlikely(!mmkey)) {
-			read_unlock(&dev->mdev->priv.mkey_table.lock);
-			mlx5_ib_warn(dev, "CQE@CQ %06x for unknown MR %6x\n",
-				     cq->mcq.cqn, be32_to_cpu(sig_err_cqe->mkey));
-			return -EINVAL;
-		}
-
 		mr = to_mibmr(mmkey);
 		get_sig_err_item(sig_err_cqe, &mr->sig->err_item);
 		mr->sig->sig_err_exists = true;
@@ -676,7 +662,6 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 	unsigned long flags;
 	int soft_polled = 0;
 	int npolled;
-	int err = 0;
 
 	spin_lock_irqsave(&cq->lock, flags);
 	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
@@ -688,8 +673,7 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 		soft_polled = poll_soft_wc(cq, num_entries, wc);
 
 	for (npolled = 0; npolled < num_entries - soft_polled; npolled++) {
-		err = mlx5_poll_one(cq, &cur_qp, wc + soft_polled + npolled);
-		if (err)
+		if (mlx5_poll_one(cq, &cur_qp, wc + soft_polled + npolled))
 			break;
 	}
 
@@ -698,10 +682,7 @@ int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 out:
 	spin_unlock_irqrestore(&cq->lock, flags);
 
-	if (err == 0 || err == -EAGAIN)
-		return soft_polled + npolled;
-	else
-		return err;
+	return soft_polled + npolled;
 }
 
 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
@@ -747,14 +728,16 @@ static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf,
 
 static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 			  struct ib_ucontext *context, struct mlx5_ib_cq *cq,
-			  int entries, struct mlx5_create_cq_mbox_in **cqb,
+			  int entries, u32 **cqb,
 			  int *cqe_size, int *index, int *inlen)
 {
 	struct mlx5_ib_create_cq ucmd;
 	size_t ucmdlen;
 	int page_shift;
+	__be64 *pas;
 	int npages;
 	int ncont;
+	void *cqc;
 	int err;
 
 	ucmdlen =
@@ -792,14 +775,20 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
 		    ucmd.buf_addr, entries * ucmd.cqe_size, npages, page_shift, ncont);
 
-	*inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * ncont;
+	*inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
+		 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * ncont;
 	*cqb = mlx5_vzalloc(*inlen);
 	if (!*cqb) {
 		err = -ENOMEM;
 		goto err_db;
 	}
-	mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, (*cqb)->pas, 0);
-	(*cqb)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
+
+	pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);
+	mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, pas, 0);
+
+	cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);
+	MLX5_SET(cqc, cqc, log_page_size,
+		 page_shift - MLX5_ADAPTER_PAGE_SHIFT);
 
 	*index = to_mucontext(context)->uuari.uars[0].index;
 
@@ -834,9 +823,10 @@ static void init_cq_buf(struct mlx5_ib_cq *cq, struct mlx5_ib_cq_buf *buf)
 
 static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
 			    int entries, int cqe_size,
-			    struct mlx5_create_cq_mbox_in **cqb,
-			    int *index, int *inlen)
+			    u32 **cqb, int *index, int *inlen)
 {
+	__be64 *pas;
+	void *cqc;
 	int err;
 
 	err = mlx5_db_alloc(dev->mdev, &cq->db);
@@ -853,15 +843,21 @@ static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
 
 	init_cq_buf(cq, &cq->buf);
 
-	*inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * cq->buf.buf.npages;
+	*inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
+		 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * cq->buf.buf.npages;
 	*cqb = mlx5_vzalloc(*inlen);
 	if (!*cqb) {
 		err = -ENOMEM;
 		goto err_buf;
 	}
-	mlx5_fill_page_array(&cq->buf.buf, (*cqb)->pas);
 
-	(*cqb)->ctx.log_pg_sz = cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
+	pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);
+	mlx5_fill_page_array(&cq->buf.buf, pas);
+
+	cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);
+	MLX5_SET(cqc, cqc, log_page_size,
+		 cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
+
 	*index = dev->mdev->priv.uuari.uars[0].index;
 
 	return 0;
@@ -895,11 +891,12 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 {
 	int entries = attr->cqe;
 	int vector = attr->comp_vector;
-	struct mlx5_create_cq_mbox_in *cqb = NULL;
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
 	struct mlx5_ib_cq *cq;
 	int uninitialized_var(index);
 	int uninitialized_var(inlen);
+	u32 *cqb = NULL;
+	void *cqc;
 	int cqe_size;
 	unsigned int irqn;
 	int eqn;
@@ -945,19 +942,20 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 		INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
 	}
 
-	cq->cqe_size = cqe_size;
-	cqb->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5;
-
-	if (cq->create_flags & IB_CQ_FLAGS_IGNORE_OVERRUN)
-		cqb->ctx.cqe_sz_flags |= (1 << 1);
-
-	cqb->ctx.log_sz_usr_page = cpu_to_be32((ilog2(entries) << 24) | index);
 	err = mlx5_vector2eqn(dev->mdev, vector, &eqn, &irqn);
 	if (err)
 		goto err_cqb;
 
-	cqb->ctx.c_eqn = cpu_to_be16(eqn);
-	cqb->ctx.db_record_addr = cpu_to_be64(cq->db.dma);
+	cq->cqe_size = cqe_size;
+
+	cqc = MLX5_ADDR_OF(create_cq_in, cqb, cq_context);
+	MLX5_SET(cqc, cqc, cqe_sz, cqe_sz_to_mlx_sz(cqe_size));
+	MLX5_SET(cqc, cqc, log_cq_size, ilog2(entries));
+	MLX5_SET(cqc, cqc, uar_page, index);
+	MLX5_SET(cqc, cqc, c_eqn, eqn);
+	MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma);
+	if (cq->create_flags & IB_CQ_FLAGS_IGNORE_OVERRUN)
+		MLX5_SET(cqc, cqc, oi, 1);
 
 	err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen);
 	if (err)
@@ -1088,27 +1086,15 @@ void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq)
 
 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 {
-	struct mlx5_modify_cq_mbox_in *in;
 	struct mlx5_ib_dev *dev = to_mdev(cq->device);
 	struct mlx5_ib_cq *mcq = to_mcq(cq);
 	int err;
-	u32 fsel;
 
 	if (!MLX5_CAP_GEN(dev->mdev, cq_moderation))
 		return -ENOSYS;
 
-	in = kzalloc(sizeof(*in), GFP_KERNEL);
-	if (!in)
-		return -ENOMEM;
-
-	in->cqn = cpu_to_be32(mcq->mcq.cqn);
-	fsel = (MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT);
-	in->ctx.cq_period = cpu_to_be16(cq_period);
-	in->ctx.cq_max_count = cpu_to_be16(cq_count);
-	in->field_select = cpu_to_be32(fsel);
-	err = mlx5_core_modify_cq(dev->mdev, &mcq->mcq, in, sizeof(*in));
-	kfree(in);
-
+	err = mlx5_core_modify_cq_moderation(dev->mdev, &mcq->mcq,
+					     cq_period, cq_count);
 	if (err)
 		mlx5_ib_warn(dev, "modify cq 0x%x failed\n", mcq->mcq.cqn);
 
@@ -1241,9 +1227,11 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(ibcq->device);
 	struct mlx5_ib_cq *cq = to_mcq(ibcq);
-	struct mlx5_modify_cq_mbox_in *in;
+	void *cqc;
+	u32 *in;
 	int err;
 	int npas;
+	__be64 *pas;
 	int page_shift;
 	int inlen;
 	int uninitialized_var(cqe_size);
@@ -1285,28 +1273,37 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
 	if (err)
 		goto ex;
 
-	inlen = sizeof(*in) + npas * sizeof(in->pas[0]);
+	inlen = MLX5_ST_SZ_BYTES(modify_cq_in) +
+		MLX5_FLD_SZ_BYTES(modify_cq_in, pas[0]) * npas;
+
 	in = mlx5_vzalloc(inlen);
 	if (!in) {
 		err = -ENOMEM;
 		goto ex_resize;
 	}
 
+	pas = (__be64 *)MLX5_ADDR_OF(modify_cq_in, in, pas);
 	if (udata)
 		mlx5_ib_populate_pas(dev, cq->resize_umem, page_shift,
-				     in->pas, 0);
+				     pas, 0);
 	else
-		mlx5_fill_page_array(&cq->resize_buf->buf, in->pas);
-
-	in->field_select = cpu_to_be32(MLX5_MODIFY_CQ_MASK_LOG_SIZE  |
-				       MLX5_MODIFY_CQ_MASK_PG_OFFSET |
-				       MLX5_MODIFY_CQ_MASK_PG_SIZE);
-	in->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
-	in->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5;
-	in->ctx.page_offset = 0;
-	in->ctx.log_sz_usr_page = cpu_to_be32(ilog2(entries) << 24);
-	in->hdr.opmod = cpu_to_be16(MLX5_CQ_OPMOD_RESIZE);
-	in->cqn = cpu_to_be32(cq->mcq.cqn);
+		mlx5_fill_page_array(&cq->resize_buf->buf, pas);
+
+	MLX5_SET(modify_cq_in, in,
+		 modify_field_select_resize_field_select.resize_field_select.resize_field_select,
+		 MLX5_MODIFY_CQ_MASK_LOG_SIZE  |
+		 MLX5_MODIFY_CQ_MASK_PG_OFFSET |
+		 MLX5_MODIFY_CQ_MASK_PG_SIZE);
+
+	cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context);
+
+	MLX5_SET(cqc, cqc, log_page_size,
+		 page_shift - MLX5_ADAPTER_PAGE_SHIFT);
+	MLX5_SET(cqc, cqc, cqe_sz, cqe_sz_to_mlx_sz(cqe_size));
+	MLX5_SET(cqc, cqc, log_cq_size, ilog2(entries));
+
+	MLX5_SET(modify_cq_in, in, op_mod, MLX5_CQ_OPMOD_RESIZE);
+	MLX5_SET(modify_cq_in, in, cqn, cq->mcq.cqn);
 
 	err = mlx5_core_modify_cq(dev->mdev, &cq->mcq, in, inlen);
 	if (err)
diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c
index 364aab9f3c9e..39e58489dcc2 100644
--- a/drivers/infiniband/hw/mlx5/mad.c
+++ b/drivers/infiniband/hw/mlx5/mad.c
@@ -394,7 +394,7 @@ int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
 	if (err)
 		goto out;
 
-	memcpy(node_desc, out_mad->data, 64);
+	memcpy(node_desc, out_mad->data, IB_DEVICE_NODE_DESC_MAX);
 out:
 	kfree(in_mad);
 	kfree(out_mad);
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index a84bb766fc62..22174774dbb8 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -37,7 +37,6 @@
 #include <linux/pci.h>
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
-#include <linux/io-mapping.h>
 #if defined(CONFIG_X86)
 #include <asm/pat.h>
 #endif
@@ -54,7 +53,6 @@
 #include <linux/in.h>
 #include <linux/etherdevice.h>
 #include <linux/mlx5/fs.h>
-#include "user.h"
 #include "mlx5_ib.h"
 
 #define DRIVER_NAME "mlx5_ib"
@@ -107,13 +105,42 @@ static int mlx5_netdev_event(struct notifier_block *this,
 	struct mlx5_ib_dev *ibdev = container_of(this, struct mlx5_ib_dev,
 						 roce.nb);
 
-	if ((event != NETDEV_UNREGISTER) && (event != NETDEV_REGISTER))
-		return NOTIFY_DONE;
+	switch (event) {
+	case NETDEV_REGISTER:
+	case NETDEV_UNREGISTER:
+		write_lock(&ibdev->roce.netdev_lock);
+		if (ndev->dev.parent == &ibdev->mdev->pdev->dev)
+			ibdev->roce.netdev = (event == NETDEV_UNREGISTER) ?
+					     NULL : ndev;
+		write_unlock(&ibdev->roce.netdev_lock);
+		break;
+
+	case NETDEV_UP:
+	case NETDEV_DOWN: {
+		struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(ibdev->mdev);
+		struct net_device *upper = NULL;
+
+		if (lag_ndev) {
+			upper = netdev_master_upper_dev_get(lag_ndev);
+			dev_put(lag_ndev);
+		}
 
-	write_lock(&ibdev->roce.netdev_lock);
-	if (ndev->dev.parent == &ibdev->mdev->pdev->dev)
-		ibdev->roce.netdev = (event == NETDEV_UNREGISTER) ? NULL : ndev;
-	write_unlock(&ibdev->roce.netdev_lock);
+		if ((upper == ndev || (!upper && ndev == ibdev->roce.netdev))
+		    && ibdev->ib_active) {
+			struct ib_event ibev = {0};
+
+			ibev.device = &ibdev->ib_dev;
+			ibev.event = (event == NETDEV_UP) ?
+				     IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
+			ibev.element.port_num = 1;
+			ib_dispatch_event(&ibev);
+		}
+		break;
+	}
+
+	default:
+		break;
+	}
 
 	return NOTIFY_DONE;
 }
@@ -124,6 +151,10 @@ static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
 	struct mlx5_ib_dev *ibdev = to_mdev(device);
 	struct net_device *ndev;
 
+	ndev = mlx5_lag_get_roce_netdev(ibdev->mdev);
+	if (ndev)
+		return ndev;
+
 	/* Ensure ndev does not disappear before we invoke dev_hold()
 	 */
 	read_lock(&ibdev->roce.netdev_lock);
@@ -139,7 +170,7 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 				struct ib_port_attr *props)
 {
 	struct mlx5_ib_dev *dev = to_mdev(device);
-	struct net_device *ndev;
+	struct net_device *ndev, *upper;
 	enum ib_mtu ndev_ib_mtu;
 	u16 qkey_viol_cntr;
 
@@ -163,6 +194,17 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 	if (!ndev)
 		return 0;
 
+	if (mlx5_lag_is_active(dev->mdev)) {
+		rcu_read_lock();
+		upper = netdev_master_upper_dev_get_rcu(ndev);
+		if (upper) {
+			dev_put(ndev);
+			ndev = upper;
+			dev_hold(ndev);
+		}
+		rcu_read_unlock();
+	}
+
 	if (netif_running(ndev) && netif_carrier_ok(ndev)) {
 		props->state      = IB_PORT_ACTIVE;
 		props->phys_state = 5;
@@ -233,23 +275,19 @@ static int set_roce_addr(struct ib_device *device, u8 port_num,
 			 const union ib_gid *gid,
 			 const struct ib_gid_attr *attr)
 {
-	struct mlx5_ib_dev *dev	= to_mdev(device);
-	u32  in[MLX5_ST_SZ_DW(set_roce_address_in)];
-	u32 out[MLX5_ST_SZ_DW(set_roce_address_out)];
+	struct mlx5_ib_dev *dev = to_mdev(device);
+	u32  in[MLX5_ST_SZ_DW(set_roce_address_in)]  = {0};
+	u32 out[MLX5_ST_SZ_DW(set_roce_address_out)] = {0};
 	void *in_addr = MLX5_ADDR_OF(set_roce_address_in, in, roce_address);
 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(device, port_num);
 
 	if (ll != IB_LINK_LAYER_ETHERNET)
 		return -EINVAL;
 
-	memset(in, 0, sizeof(in));
-
 	ib_gid_to_mlx5_roce_addr(gid, attr, in_addr);
 
 	MLX5_SET(set_roce_address_in, in, roce_address_index, index);
 	MLX5_SET(set_roce_address_in, in, opcode, MLX5_CMD_OP_SET_ROCE_ADDRESS);
-
-	memset(out, 0, sizeof(out));
 	return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 }
 
@@ -289,7 +327,9 @@ __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
 
 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
 {
-	return !MLX5_CAP_GEN(dev->mdev, ib_virt);
+	if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
+		return !MLX5_CAP_GEN(dev->mdev, ib_virt);
+	return 0;
 }
 
 enum {
@@ -432,7 +472,7 @@ static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
 }
 
 struct mlx5_reg_node_desc {
-	u8	desc[64];
+	u8	desc[IB_DEVICE_NODE_DESC_MAX];
 };
 
 static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
@@ -535,6 +575,26 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 				resp.response_length += sizeof(resp.tso_caps);
 			}
 		}
+
+		if (field_avail(typeof(resp), rss_caps, uhw->outlen)) {
+			resp.rss_caps.rx_hash_function =
+						MLX5_RX_HASH_FUNC_TOEPLITZ;
+			resp.rss_caps.rx_hash_fields_mask =
+						MLX5_RX_HASH_SRC_IPV4 |
+						MLX5_RX_HASH_DST_IPV4 |
+						MLX5_RX_HASH_SRC_IPV6 |
+						MLX5_RX_HASH_DST_IPV6 |
+						MLX5_RX_HASH_SRC_PORT_TCP |
+						MLX5_RX_HASH_DST_PORT_TCP |
+						MLX5_RX_HASH_SRC_PORT_UDP |
+						MLX5_RX_HASH_DST_PORT_UDP;
+			resp.response_length += sizeof(resp.rss_caps);
+		}
+	} else {
+		if (field_avail(typeof(resp), tso_caps, uhw->outlen))
+			resp.response_length += sizeof(resp.tso_caps);
+		if (field_avail(typeof(resp), rss_caps, uhw->outlen))
+			resp.response_length += sizeof(resp.rss_caps);
 	}
 
 	if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
@@ -598,6 +658,17 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	if (!mlx5_core_is_pf(mdev))
 		props->device_cap_flags |= IB_DEVICE_VIRTUAL_FUNCTION;
 
+	if (mlx5_ib_port_link_layer(ibdev, 1) ==
+	    IB_LINK_LAYER_ETHERNET) {
+		props->rss_caps.max_rwq_indirection_tables =
+			1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
+		props->rss_caps.max_rwq_indirection_table_size =
+			1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
+		props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
+		props->max_wq_type_rq =
+			1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
+	}
+
 	if (uhw->outlen) {
 		err = ib_copy_to_udata(uhw, &resp, resp.response_length);
 
@@ -752,8 +823,7 @@ static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,
 				     &props->active_width);
 	if (err)
 		goto out;
-	err = mlx5_query_port_proto_oper(mdev, &props->active_speed, MLX5_PTYS_IB,
-					 port);
+	err = mlx5_query_port_ib_proto_oper(mdev, &props->active_speed, port);
 	if (err)
 		goto out;
 
@@ -850,13 +920,13 @@ static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
 	 * If possible, pass node desc to FW, so it can generate
 	 * a 144 trap.  If cmd fails, just ignore.
 	 */
-	memcpy(&in, props->node_desc, 64);
+	memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
 	err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
 				   sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
 	if (err)
 		return err;
 
-	memcpy(ibdev->node_desc, props->node_desc, 64);
+	memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
 
 	return err;
 }
@@ -1399,28 +1469,77 @@ static int mlx5_ib_dealloc_pd(struct ib_pd *pd)
 	return 0;
 }
 
-static bool outer_header_zero(u32 *match_criteria)
+enum {
+	MATCH_CRITERIA_ENABLE_OUTER_BIT,
+	MATCH_CRITERIA_ENABLE_MISC_BIT,
+	MATCH_CRITERIA_ENABLE_INNER_BIT
+};
+
+#define HEADER_IS_ZERO(match_criteria, headers)			           \
+	!(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
+		    0, MLX5_FLD_SZ_BYTES(fte_match_param, headers)))       \
+
+static u8 get_match_criteria_enable(u32 *match_criteria)
 {
-	int size = MLX5_ST_SZ_BYTES(fte_match_param);
-	char *outer_headers_c = MLX5_ADDR_OF(fte_match_param, match_criteria,
-					     outer_headers);
+	u8 match_criteria_enable;
+
+	match_criteria_enable =
+		(!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
+		MATCH_CRITERIA_ENABLE_OUTER_BIT;
+	match_criteria_enable |=
+		(!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
+		MATCH_CRITERIA_ENABLE_MISC_BIT;
+	match_criteria_enable |=
+		(!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
+		MATCH_CRITERIA_ENABLE_INNER_BIT;
+
+	return match_criteria_enable;
+}
+
+static void set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
+{
+	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
+	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
+}
 
-	return outer_headers_c[0] == 0 && !memcmp(outer_headers_c,
-						  outer_headers_c + 1,
-						  size - 1);
+static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
+{
+	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
+	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
+	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
+	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
 }
 
+#define LAST_ETH_FIELD vlan_tag
+#define LAST_IB_FIELD sl
+#define LAST_IPV4_FIELD tos
+#define LAST_IPV6_FIELD traffic_class
+#define LAST_TCP_UDP_FIELD src_port
+
+/* Field is the last supported field */
+#define FIELDS_NOT_SUPPORTED(filter, field)\
+	memchr_inv((void *)&filter.field  +\
+		   sizeof(filter.field), 0,\
+		   sizeof(filter) -\
+		   offsetof(typeof(filter), field) -\
+		   sizeof(filter.field))
+
 static int parse_flow_attr(u32 *match_c, u32 *match_v,
-			   union ib_flow_spec *ib_spec)
+			   const union ib_flow_spec *ib_spec)
 {
 	void *outer_headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
 					     outer_headers);
 	void *outer_headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
 					     outer_headers);
+	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
+					   misc_parameters);
+	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
+					   misc_parameters);
+
 	switch (ib_spec->type) {
 	case IB_FLOW_SPEC_ETH:
-		if (ib_spec->size != sizeof(ib_spec->eth))
-			return -EINVAL;
+		if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
+			return -ENOTSUPP;
 
 		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
 					     dmac_47_16),
@@ -1429,6 +1548,13 @@ static int parse_flow_attr(u32 *match_c, u32 *match_v,
 					     dmac_47_16),
 				ib_spec->eth.val.dst_mac);
 
+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
+					     smac_47_16),
+				ib_spec->eth.mask.src_mac);
+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
+					     smac_47_16),
+				ib_spec->eth.val.src_mac);
+
 		if (ib_spec->eth.mask.vlan_tag) {
 			MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
 				 vlan_tag, 1);
@@ -1460,8 +1586,8 @@ static int parse_flow_attr(u32 *match_c, u32 *match_v,
 			 ethertype, ntohs(ib_spec->eth.val.ether_type));
 		break;
 	case IB_FLOW_SPEC_IPV4:
-		if (ib_spec->size != sizeof(ib_spec->ipv4))
-			return -EINVAL;
+		if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
+			return -ENOTSUPP;
 
 		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
 			 ethertype, 0xffff);
@@ -1484,10 +1610,16 @@ static int parse_flow_attr(u32 *match_c, u32 *match_v,
 				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 		       &ib_spec->ipv4.val.dst_ip,
 		       sizeof(ib_spec->ipv4.val.dst_ip));
+
+		set_tos(outer_headers_c, outer_headers_v,
+			ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
+
+		set_proto(outer_headers_c, outer_headers_v,
+			  ib_spec->ipv4.mask.proto, ib_spec->ipv4.val.proto);
 		break;
 	case IB_FLOW_SPEC_IPV6:
-		if (ib_spec->size != sizeof(ib_spec->ipv6))
-			return -EINVAL;
+		if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
+			return -ENOTSUPP;
 
 		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
 			 ethertype, 0xffff);
@@ -1510,10 +1642,26 @@ static int parse_flow_attr(u32 *match_c, u32 *match_v,
 				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 		       &ib_spec->ipv6.val.dst_ip,
 		       sizeof(ib_spec->ipv6.val.dst_ip));
+
+		set_tos(outer_headers_c, outer_headers_v,
+			ib_spec->ipv6.mask.traffic_class,
+			ib_spec->ipv6.val.traffic_class);
+
+		set_proto(outer_headers_c, outer_headers_v,
+			  ib_spec->ipv6.mask.next_hdr,
+			  ib_spec->ipv6.val.next_hdr);
+
+		MLX5_SET(fte_match_set_misc, misc_params_c,
+			 outer_ipv6_flow_label,
+			 ntohl(ib_spec->ipv6.mask.flow_label));
+		MLX5_SET(fte_match_set_misc, misc_params_v,
+			 outer_ipv6_flow_label,
+			 ntohl(ib_spec->ipv6.val.flow_label));
 		break;
 	case IB_FLOW_SPEC_TCP:
-		if (ib_spec->size != sizeof(ib_spec->tcp_udp))
-			return -EINVAL;
+		if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
+					 LAST_TCP_UDP_FIELD))
+			return -ENOTSUPP;
 
 		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol,
 			 0xff);
@@ -1531,8 +1679,9 @@ static int parse_flow_attr(u32 *match_c, u32 *match_v,
 			 ntohs(ib_spec->tcp_udp.val.dst_port));
 		break;
 	case IB_FLOW_SPEC_UDP:
-		if (ib_spec->size != sizeof(ib_spec->tcp_udp))
-			return -EINVAL;
+		if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
+					 LAST_TCP_UDP_FIELD))
+			return -ENOTSUPP;
 
 		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol,
 			 0xff);
@@ -1579,7 +1728,7 @@ static bool flow_is_multicast_only(struct ib_flow_attr *ib_attr)
 	       is_multicast_ether_addr(eth_spec->val.dst_mac);
 }
 
-static bool is_valid_attr(struct ib_flow_attr *flow_attr)
+static bool is_valid_attr(const struct ib_flow_attr *flow_attr)
 {
 	union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
 	bool has_ipv4_spec = false;
@@ -1623,12 +1772,13 @@ static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
 
 	list_for_each_entry_safe(iter, tmp, &handler->list, list) {
 		mlx5_del_flow_rule(iter->rule);
+		put_flow_table(dev, iter->prio, true);
 		list_del(&iter->list);
 		kfree(iter);
 	}
 
 	mlx5_del_flow_rule(handler->rule);
-	put_flow_table(dev, &dev->flow_db.prios[handler->prio], true);
+	put_flow_table(dev, handler->prio, true);
 	mutex_unlock(&dev->flow_db.lock);
 
 	kfree(handler);
@@ -1644,10 +1794,16 @@ static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
 	return priority;
 }
 
+enum flow_table_type {
+	MLX5_IB_FT_RX,
+	MLX5_IB_FT_TX
+};
+
 #define MLX5_FS_MAX_TYPES	 10
 #define MLX5_FS_MAX_ENTRIES	 32000UL
 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
-						struct ib_flow_attr *flow_attr)
+						struct ib_flow_attr *flow_attr,
+						enum flow_table_type ft_type)
 {
 	bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
 	struct mlx5_flow_namespace *ns = NULL;
@@ -1678,6 +1834,19 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
 					 &num_entries,
 					 &num_groups);
 		prio = &dev->flow_db.prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
+	} else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
+		if (!MLX5_CAP_FLOWTABLE(dev->mdev,
+					allow_sniffer_and_nic_rx_shared_tir))
+			return ERR_PTR(-ENOTSUPP);
+
+		ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ?
+					     MLX5_FLOW_NAMESPACE_SNIFFER_RX :
+					     MLX5_FLOW_NAMESPACE_SNIFFER_TX);
+
+		prio = &dev->flow_db.sniffer[ft_type];
+		priority = 0;
+		num_entries = 1;
+		num_groups = 1;
 	}
 
 	if (!ns)
@@ -1703,13 +1872,13 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
 
 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
 						     struct mlx5_ib_flow_prio *ft_prio,
-						     struct ib_flow_attr *flow_attr,
+						     const struct ib_flow_attr *flow_attr,
 						     struct mlx5_flow_destination *dst)
 {
 	struct mlx5_flow_table	*ft = ft_prio->flow_table;
 	struct mlx5_ib_flow_handler *handler;
 	struct mlx5_flow_spec *spec;
-	void *ib_flow = flow_attr + 1;
+	const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
 	unsigned int spec_index;
 	u32 action;
 	int err = 0;
@@ -1735,9 +1904,7 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
 		ib_flow += ((union ib_flow_spec *)ib_flow)->size;
 	}
 
-	/* Outer header support only */
-	spec->match_criteria_enable = (!outer_header_zero(spec->match_criteria))
-		<< 0;
+	spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
 	action = dst ? MLX5_FLOW_CONTEXT_ACTION_FWD_DEST :
 		MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
 	handler->rule = mlx5_add_flow_rule(ft, spec,
@@ -1750,7 +1917,8 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
 		goto free;
 	}
 
-	handler->prio = ft_prio - dev->flow_db.prios;
+	ft_prio->refcount++;
+	handler->prio = ft_prio;
 
 	ft_prio->flow_table = ft;
 free:
@@ -1774,6 +1942,7 @@ static struct mlx5_ib_flow_handler *create_dont_trap_rule(struct mlx5_ib_dev *de
 					       flow_attr, dst);
 		if (IS_ERR(handler_dst)) {
 			mlx5_del_flow_rule(handler->rule);
+			ft_prio->refcount--;
 			kfree(handler);
 			handler = handler_dst;
 		} else {
@@ -1835,6 +2004,8 @@ static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *de
 						 &leftovers_specs[LEFTOVERS_UC].flow_attr,
 						 dst);
 		if (IS_ERR(handler_ucast)) {
+			mlx5_del_flow_rule(handler->rule);
+			ft_prio->refcount--;
 			kfree(handler);
 			handler = handler_ucast;
 		} else {
@@ -1845,13 +2016,52 @@ static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *de
 	return handler;
 }
 
+static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
+							struct mlx5_ib_flow_prio *ft_rx,
+							struct mlx5_ib_flow_prio *ft_tx,
+							struct mlx5_flow_destination *dst)
+{
+	struct mlx5_ib_flow_handler *handler_rx;
+	struct mlx5_ib_flow_handler *handler_tx;
+	int err;
+	static const struct ib_flow_attr flow_attr  = {
+		.num_of_specs = 0,
+		.size = sizeof(flow_attr)
+	};
+
+	handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
+	if (IS_ERR(handler_rx)) {
+		err = PTR_ERR(handler_rx);
+		goto err;
+	}
+
+	handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
+	if (IS_ERR(handler_tx)) {
+		err = PTR_ERR(handler_tx);
+		goto err_tx;
+	}
+
+	list_add(&handler_tx->list, &handler_rx->list);
+
+	return handler_rx;
+
+err_tx:
+	mlx5_del_flow_rule(handler_rx->rule);
+	ft_rx->refcount--;
+	kfree(handler_rx);
+err:
+	return ERR_PTR(err);
+}
+
 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
 					   struct ib_flow_attr *flow_attr,
 					   int domain)
 {
 	struct mlx5_ib_dev *dev = to_mdev(qp->device);
+	struct mlx5_ib_qp *mqp = to_mqp(qp);
 	struct mlx5_ib_flow_handler *handler = NULL;
 	struct mlx5_flow_destination *dst = NULL;
+	struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
 	struct mlx5_ib_flow_prio *ft_prio;
 	int err;
 
@@ -1869,14 +2079,25 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
 
 	mutex_lock(&dev->flow_db.lock);
 
-	ft_prio = get_flow_table(dev, flow_attr);
+	ft_prio = get_flow_table(dev, flow_attr, MLX5_IB_FT_RX);
 	if (IS_ERR(ft_prio)) {
 		err = PTR_ERR(ft_prio);
 		goto unlock;
 	}
+	if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
+		ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
+		if (IS_ERR(ft_prio_tx)) {
+			err = PTR_ERR(ft_prio_tx);
+			ft_prio_tx = NULL;
+			goto destroy_ft;
+		}
+	}
 
 	dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
-	dst->tir_num = to_mqp(qp)->raw_packet_qp.rq.tirn;
+	if (mqp->flags & MLX5_IB_QP_RSS)
+		dst->tir_num = mqp->rss_qp.tirn;
+	else
+		dst->tir_num = mqp->raw_packet_qp.rq.tirn;
 
 	if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
 		if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)  {
@@ -1890,6 +2111,8 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
 		   flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
 		handler = create_leftovers_rule(dev, ft_prio, flow_attr,
 						dst);
+	} else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
+		handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
 	} else {
 		err = -EINVAL;
 		goto destroy_ft;
@@ -1901,7 +2124,6 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
 		goto destroy_ft;
 	}
 
-	ft_prio->refcount++;
 	mutex_unlock(&dev->flow_db.lock);
 	kfree(dst);
 
@@ -1909,6 +2131,8 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
 
 destroy_ft:
 	put_flow_table(dev, ft_prio, false);
+	if (ft_prio_tx)
+		put_flow_table(dev, ft_prio_tx, false);
 unlock:
 	mutex_unlock(&dev->flow_db.lock);
 	kfree(dst);
@@ -2098,14 +2322,19 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
 		break;
 
 	case MLX5_DEV_EVENT_PORT_UP:
-		ibev.event = IB_EVENT_PORT_ACTIVE;
-		port = (u8)param;
-		break;
-
 	case MLX5_DEV_EVENT_PORT_DOWN:
 	case MLX5_DEV_EVENT_PORT_INITIALIZED:
-		ibev.event = IB_EVENT_PORT_ERR;
 		port = (u8)param;
+
+		/* In RoCE, port up/down events are handled in
+		 * mlx5_netdev_event().
+		 */
+		if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
+			IB_LINK_LAYER_ETHERNET)
+			return;
+
+		ibev.event = (event == MLX5_DEV_EVENT_PORT_UP) ?
+			     IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
 		break;
 
 	case MLX5_DEV_EVENT_LID_CHANGE:
@@ -2228,7 +2457,7 @@ static int create_umr_res(struct mlx5_ib_dev *dev)
 		goto error_0;
 	}
 
-	pd = ib_alloc_pd(&dev->ib_dev);
+	pd = ib_alloc_pd(&dev->ib_dev, 0);
 	if (IS_ERR(pd)) {
 		mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
 		ret = PTR_ERR(pd);
@@ -2510,30 +2739,88 @@ static void get_dev_fw_str(struct ib_device *ibdev, char *str,
 		       fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev));
 }
 
+static int mlx5_roce_lag_init(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_core_dev *mdev = dev->mdev;
+	struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
+								 MLX5_FLOW_NAMESPACE_LAG);
+	struct mlx5_flow_table *ft;
+	int err;
+
+	if (!ns || !mlx5_lag_is_active(mdev))
+		return 0;
+
+	err = mlx5_cmd_create_vport_lag(mdev);
+	if (err)
+		return err;
+
+	ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
+		goto err_destroy_vport_lag;
+	}
+
+	dev->flow_db.lag_demux_ft = ft;
+	return 0;
+
+err_destroy_vport_lag:
+	mlx5_cmd_destroy_vport_lag(mdev);
+	return err;
+}
+
+static void mlx5_roce_lag_cleanup(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_core_dev *mdev = dev->mdev;
+
+	if (dev->flow_db.lag_demux_ft) {
+		mlx5_destroy_flow_table(dev->flow_db.lag_demux_ft);
+		dev->flow_db.lag_demux_ft = NULL;
+
+		mlx5_cmd_destroy_vport_lag(mdev);
+	}
+}
+
+static void mlx5_remove_roce_notifier(struct mlx5_ib_dev *dev)
+{
+	if (dev->roce.nb.notifier_call) {
+		unregister_netdevice_notifier(&dev->roce.nb);
+		dev->roce.nb.notifier_call = NULL;
+	}
+}
+
 static int mlx5_enable_roce(struct mlx5_ib_dev *dev)
 {
 	int err;
 
 	dev->roce.nb.notifier_call = mlx5_netdev_event;
 	err = register_netdevice_notifier(&dev->roce.nb);
-	if (err)
+	if (err) {
+		dev->roce.nb.notifier_call = NULL;
 		return err;
+	}
 
 	err = mlx5_nic_vport_enable_roce(dev->mdev);
 	if (err)
 		goto err_unregister_netdevice_notifier;
 
+	err = mlx5_roce_lag_init(dev);
+	if (err)
+		goto err_disable_roce;
+
 	return 0;
 
+err_disable_roce:
+	mlx5_nic_vport_disable_roce(dev->mdev);
+
 err_unregister_netdevice_notifier:
-	unregister_netdevice_notifier(&dev->roce.nb);
+	mlx5_remove_roce_notifier(dev);
 	return err;
 }
 
 static void mlx5_disable_roce(struct mlx5_ib_dev *dev)
 {
+	mlx5_roce_lag_cleanup(dev);
 	mlx5_nic_vport_disable_roce(dev->mdev);
-	unregister_netdevice_notifier(&dev->roce.nb);
 }
 
 static void mlx5_ib_dealloc_q_counters(struct mlx5_ib_dev *dev)
@@ -2648,6 +2935,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	struct mlx5_ib_dev *dev;
 	enum rdma_link_layer ll;
 	int port_type_cap;
+	const char *name;
 	int err;
 	int i;
 
@@ -2680,7 +2968,12 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 
 	MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);
 
-	strlcpy(dev->ib_dev.name, "mlx5_%d", IB_DEVICE_NAME_MAX);
+	if (!mlx5_lag_is_active(mdev))
+		name = "mlx5_%d";
+	else
+		name = "mlx5_bond_%d";
+
+	strlcpy(dev->ib_dev.name, name, IB_DEVICE_NAME_MAX);
 	dev->ib_dev.owner		= THIS_MODULE;
 	dev->ib_dev.node_type		= RDMA_NODE_IB_CA;
 	dev->ib_dev.local_dma_lkey	= 0 /* not supported for now */;
@@ -2882,8 +3175,10 @@ err_rsrc:
 	destroy_dev_resources(&dev->devr);
 
 err_disable_roce:
-	if (ll == IB_LINK_LAYER_ETHERNET)
+	if (ll == IB_LINK_LAYER_ETHERNET) {
 		mlx5_disable_roce(dev);
+		mlx5_remove_roce_notifier(dev);
+	}
 
 err_free_port:
 	kfree(dev->port);
@@ -2899,6 +3194,7 @@ static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
 	struct mlx5_ib_dev *dev = context;
 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
 
+	mlx5_remove_roce_notifier(dev);
 	ib_unregister_device(&dev->ib_dev);
 	mlx5_ib_dealloc_q_counters(dev);
 	destroy_umrc_res(dev);
diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c
index 40df2cca0609..996b54e366b0 100644
--- a/drivers/infiniband/hw/mlx5/mem.c
+++ b/drivers/infiniband/hw/mlx5/mem.c
@@ -71,7 +71,7 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
 
 	addr = addr >> page_shift;
 	tmp = (unsigned long)addr;
-	m = find_first_bit(&tmp, sizeof(tmp));
+	m = find_first_bit(&tmp, BITS_PER_LONG);
 	skip = 1 << m;
 	mask = skip - 1;
 	i = 0;
@@ -81,7 +81,7 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
 		for (k = 0; k < len; k++) {
 			if (!(i & mask)) {
 				tmp = (unsigned long)pfn;
-				m = min_t(unsigned long, m, find_first_bit(&tmp, sizeof(tmp)));
+				m = min_t(unsigned long, m, find_first_bit(&tmp, BITS_PER_LONG));
 				skip = 1 << m;
 				mask = skip - 1;
 				base = pfn;
@@ -89,7 +89,7 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
 			} else {
 				if (base + p != pfn) {
 					tmp = (unsigned long)p;
-					m = find_first_bit(&tmp, sizeof(tmp));
+					m = find_first_bit(&tmp, BITS_PER_LONG);
 					skip = 1 << m;
 					mask = skip - 1;
 					base = pfn;
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 372385d0f993..dcdcd195fe53 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -44,6 +44,7 @@
 #include <linux/types.h>
 #include <linux/mlx5/transobj.h>
 #include <rdma/ib_user_verbs.h>
+#include <rdma/mlx5-abi.h>
 
 #define mlx5_ib_dbg(dev, format, arg...)				\
 pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__,	\
@@ -142,6 +143,7 @@ struct mlx5_ib_pd {
 #define MLX5_IB_FLOW_LEFTOVERS_PRIO	(MLX5_IB_FLOW_MCAST_PRIO + 1)
 
 #define MLX5_IB_NUM_FLOW_FT		(MLX5_IB_FLOW_LEFTOVERS_PRIO + 1)
+#define MLX5_IB_NUM_SNIFFER_FTS		2
 struct mlx5_ib_flow_prio {
 	struct mlx5_flow_table		*flow_table;
 	unsigned int			refcount;
@@ -150,12 +152,14 @@ struct mlx5_ib_flow_prio {
 struct mlx5_ib_flow_handler {
 	struct list_head		list;
 	struct ib_flow			ibflow;
-	unsigned int			prio;
+	struct mlx5_ib_flow_prio	*prio;
 	struct mlx5_flow_rule	*rule;
 };
 
 struct mlx5_ib_flow_db {
 	struct mlx5_ib_flow_prio	prios[MLX5_IB_NUM_FLOW_FT];
+	struct mlx5_ib_flow_prio	sniffer[MLX5_IB_NUM_SNIFFER_FTS];
+	struct mlx5_flow_table		*lag_demux_ft;
 	/* Protect flow steering bypass flow tables
 	 * when add/del flow rules.
 	 * only single add/removal of flow steering rule could be done
@@ -225,7 +229,7 @@ struct mlx5_ib_wq {
 
 struct mlx5_ib_rwq {
 	struct ib_wq		ibwq;
-	u32			rqn;
+	struct mlx5_core_qp	core_qp;
 	u32			rq_num_pas;
 	u32			log_rq_stride;
 	u32			log_rq_size;
@@ -402,6 +406,7 @@ enum mlx5_ib_qp_flags {
 	/* QP uses 1 as its source QP number */
 	MLX5_IB_QP_SQPN_QP1			= 1 << 6,
 	MLX5_IB_QP_CAP_SCATTER_FCS		= 1 << 7,
+	MLX5_IB_QP_RSS				= 1 << 8,
 };
 
 struct mlx5_umr_wr {
@@ -504,7 +509,7 @@ struct mlx5_ib_mr {
 	int			umred;
 	int			npages;
 	struct mlx5_ib_dev     *dev;
-	struct mlx5_create_mkey_mbox_out out;
+	u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
 	struct mlx5_core_sig_ctx    *sig;
 	int			live;
 	void			*descs_alloc;
@@ -602,6 +607,7 @@ struct mlx5_roce {
 	rwlock_t		netdev_lock;
 	struct net_device	*netdev;
 	struct notifier_block	nb;
+	atomic_t		next_port;
 };
 
 struct mlx5_ib_dev {
@@ -662,6 +668,11 @@ static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
 	return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;
 }
 
+static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp)
+{
+	return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
+}
+
 static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
 {
 	return container_of(mmkey, struct mlx5_ib_mr, mmkey);
@@ -946,4 +957,40 @@ static inline int verify_assign_uidx(u8 cqe_version, u32 cmd_uidx,
 
 	return 0;
 }
+
+static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
+				    struct mlx5_ib_create_qp *ucmd,
+				    int inlen,
+				    u32 *user_index)
+{
+	u8 cqe_version = ucontext->cqe_version;
+
+	if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
+	    !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
+		return 0;
+
+	if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
+	       !!cqe_version))
+		return -EINVAL;
+
+	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
+}
+
+static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
+				     struct mlx5_ib_create_srq *ucmd,
+				     int inlen,
+				     u32 *user_index)
+{
+	u8 cqe_version = ucontext->cqe_version;
+
+	if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
+	    !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
+		return 0;
+
+	if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
+	       !!cqe_version))
+		return -EINVAL;
+
+	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
+}
 #endif /* MLX5_IB_H */
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 4b021305c321..d4ad672b905b 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -40,7 +40,6 @@
 #include <rdma/ib_umem_odp.h>
 #include <rdma/ib_verbs.h>
 #include "mlx5_ib.h"
-#include "user.h"
 
 enum {
 	MAX_PENDING_REG_MR = 8,
@@ -135,20 +134,10 @@ static void reg_mr_callback(int status, void *context)
 		return;
 	}
 
-	if (mr->out.hdr.status) {
-		mlx5_ib_warn(dev, "failed - status %d, syndorme 0x%x\n",
-			     mr->out.hdr.status,
-			     be32_to_cpu(mr->out.hdr.syndrome));
-		kfree(mr);
-		dev->fill_delay = 1;
-		mod_timer(&dev->delay_timer, jiffies + HZ);
-		return;
-	}
-
 	spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags);
 	key = dev->mdev->priv.mkey_key++;
 	spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags);
-	mr->mmkey.key = mlx5_idx_to_mkey(be32_to_cpu(mr->out.mkey) & 0xffffff) | key;
+	mr->mmkey.key = mlx5_idx_to_mkey(MLX5_GET(create_mkey_out, mr->out, mkey_index)) | key;
 
 	cache->last_add = jiffies;
 
@@ -170,16 +159,19 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
 {
 	struct mlx5_mr_cache *cache = &dev->cache;
 	struct mlx5_cache_ent *ent = &cache->ent[c];
-	struct mlx5_create_mkey_mbox_in *in;
+	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 	struct mlx5_ib_mr *mr;
 	int npages = 1 << ent->order;
+	void *mkc;
+	u32 *in;
 	int err = 0;
 	int i;
 
-	in = kzalloc(sizeof(*in), GFP_KERNEL);
+	in = kzalloc(inlen, GFP_KERNEL);
 	if (!in)
 		return -ENOMEM;
 
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 	for (i = 0; i < num; i++) {
 		if (ent->pending >= MAX_PENDING_REG_MR) {
 			err = -EAGAIN;
@@ -194,18 +186,22 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
 		mr->order = ent->order;
 		mr->umred = 1;
 		mr->dev = dev;
-		in->seg.status = MLX5_MKEY_STATUS_FREE;
-		in->seg.xlt_oct_size = cpu_to_be32((npages + 1) / 2);
-		in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
-		in->seg.flags = MLX5_ACCESS_MODE_MTT | MLX5_PERM_UMR_EN;
-		in->seg.log2_page_size = 12;
+
+		MLX5_SET(mkc, mkc, free, 1);
+		MLX5_SET(mkc, mkc, umr_en, 1);
+		MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
+
+		MLX5_SET(mkc, mkc, qpn, 0xffffff);
+		MLX5_SET(mkc, mkc, translations_octword_size, (npages + 1) / 2);
+		MLX5_SET(mkc, mkc, log_page_size, 12);
 
 		spin_lock_irq(&ent->lock);
 		ent->pending++;
 		spin_unlock_irq(&ent->lock);
-		err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in,
-					    sizeof(*in), reg_mr_callback,
-					    mr, &mr->out);
+		err = mlx5_core_create_mkey_cb(dev->mdev, &mr->mmkey,
+					       in, inlen,
+					       mr->out, sizeof(mr->out),
+					       reg_mr_callback, mr);
 		if (err) {
 			spin_lock_irq(&ent->lock);
 			ent->pending--;
@@ -614,7 +610,7 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
 	int err;
 	int i;
 
-	cache->wq = create_singlethread_workqueue("mkey_cache");
+	cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
 	if (!cache->wq) {
 		mlx5_ib_warn(dev, "failed to create work queue\n");
 		return -ENOMEM;
@@ -670,30 +666,38 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
+	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 	struct mlx5_core_dev *mdev = dev->mdev;
-	struct mlx5_create_mkey_mbox_in *in;
-	struct mlx5_mkey_seg *seg;
 	struct mlx5_ib_mr *mr;
+	void *mkc;
+	u32 *in;
 	int err;
 
 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	in = kzalloc(sizeof(*in), GFP_KERNEL);
+	in = kzalloc(inlen, GFP_KERNEL);
 	if (!in) {
 		err = -ENOMEM;
 		goto err_free;
 	}
 
-	seg = &in->seg;
-	seg->flags = convert_access(acc) | MLX5_ACCESS_MODE_PA;
-	seg->flags_pd = cpu_to_be32(to_mpd(pd)->pdn | MLX5_MKEY_LEN64);
-	seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
-	seg->start_addr = 0;
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+
+	MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_PA);
+	MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
+	MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
+	MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
+	MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
+	MLX5_SET(mkc, mkc, lr, 1);
 
-	err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, sizeof(*in), NULL, NULL,
-				    NULL);
+	MLX5_SET(mkc, mkc, length64, 1);
+	MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
+	MLX5_SET(mkc, mkc, qpn, 0xffffff);
+	MLX5_SET64(mkc, mkc, start_addr, 0);
+
+	err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen);
 	if (err)
 		goto err_in;
 
@@ -1063,9 +1067,11 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
 				     int page_shift, int access_flags)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
-	struct mlx5_create_mkey_mbox_in *in;
 	struct mlx5_ib_mr *mr;
+	__be64 *pas;
+	void *mkc;
 	int inlen;
+	u32 *in;
 	int err;
 	bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg));
 
@@ -1073,31 +1079,41 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	inlen = sizeof(*in) + sizeof(*in->pas) * ((npages + 1) / 2) * 2;
+	inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
+		sizeof(*pas) * ((npages + 1) / 2) * 2;
 	in = mlx5_vzalloc(inlen);
 	if (!in) {
 		err = -ENOMEM;
 		goto err_1;
 	}
-	mlx5_ib_populate_pas(dev, umem, page_shift, in->pas,
+	pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
+	mlx5_ib_populate_pas(dev, umem, page_shift, pas,
 			     pg_cap ? MLX5_IB_MTT_PRESENT : 0);
 
-	/* The MLX5_MKEY_INBOX_PG_ACCESS bit allows setting the access flags
+	/* The pg_access bit allows setting the access flags
 	 * in the page list submitted with the command. */
-	in->flags = pg_cap ? cpu_to_be32(MLX5_MKEY_INBOX_PG_ACCESS) : 0;
-	in->seg.flags = convert_access(access_flags) |
-		MLX5_ACCESS_MODE_MTT;
-	in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
-	in->seg.start_addr = cpu_to_be64(virt_addr);
-	in->seg.len = cpu_to_be64(length);
-	in->seg.bsfs_octo_size = 0;
-	in->seg.xlt_oct_size = cpu_to_be32(get_octo_len(virt_addr, length, 1 << page_shift));
-	in->seg.log2_page_size = page_shift;
-	in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
-	in->xlat_oct_act_size = cpu_to_be32(get_octo_len(virt_addr, length,
-							 1 << page_shift));
-	err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen, NULL,
-				    NULL, NULL);
+	MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
+
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+	MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
+	MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
+	MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
+	MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
+	MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE));
+	MLX5_SET(mkc, mkc, lr, 1);
+
+	MLX5_SET64(mkc, mkc, start_addr, virt_addr);
+	MLX5_SET64(mkc, mkc, len, length);
+	MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
+	MLX5_SET(mkc, mkc, bsf_octword_size, 0);
+	MLX5_SET(mkc, mkc, translations_octword_size,
+		 get_octo_len(virt_addr, length, 1 << page_shift));
+	MLX5_SET(mkc, mkc, log_page_size, page_shift);
+	MLX5_SET(mkc, mkc, qpn, 0xffffff);
+	MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
+		 get_octo_len(virt_addr, length, 1 << page_shift));
+
+	err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
 	if (err) {
 		mlx5_ib_warn(dev, "create mkey failed\n");
 		goto err_2;
@@ -1523,30 +1539,32 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
 			       u32 max_num_sg)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
-	struct mlx5_create_mkey_mbox_in *in;
-	struct mlx5_ib_mr *mr;
+	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 	int ndescs = ALIGN(max_num_sg, 4);
+	struct mlx5_ib_mr *mr;
+	void *mkc;
+	u32 *in;
 	int err;
 
 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	in = kzalloc(sizeof(*in), GFP_KERNEL);
+	in = kzalloc(inlen, GFP_KERNEL);
 	if (!in) {
 		err = -ENOMEM;
 		goto err_free;
 	}
 
-	in->seg.status = MLX5_MKEY_STATUS_FREE;
-	in->seg.xlt_oct_size = cpu_to_be32(ndescs);
-	in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
-	in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+	MLX5_SET(mkc, mkc, free, 1);
+	MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
+	MLX5_SET(mkc, mkc, qpn, 0xffffff);
+	MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
 
 	if (mr_type == IB_MR_TYPE_MEM_REG) {
-		mr->access_mode = MLX5_ACCESS_MODE_MTT;
-		in->seg.log2_page_size = PAGE_SHIFT;
-
+		mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
+		MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
 		err = mlx5_alloc_priv_descs(pd->device, mr,
 					    ndescs, sizeof(u64));
 		if (err)
@@ -1555,7 +1573,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
 		mr->desc_size = sizeof(u64);
 		mr->max_descs = ndescs;
 	} else if (mr_type == IB_MR_TYPE_SG_GAPS) {
-		mr->access_mode = MLX5_ACCESS_MODE_KLM;
+		mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
 
 		err = mlx5_alloc_priv_descs(pd->device, mr,
 					    ndescs, sizeof(struct mlx5_klm));
@@ -1566,9 +1584,8 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
 	} else if (mr_type == IB_MR_TYPE_SIGNATURE) {
 		u32 psv_index[2];
 
-		in->seg.flags_pd = cpu_to_be32(be32_to_cpu(in->seg.flags_pd) |
-							   MLX5_MKEY_BSF_EN);
-		in->seg.bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
+		MLX5_SET(mkc, mkc, bsf_en, 1);
+		MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
 		mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
 		if (!mr->sig) {
 			err = -ENOMEM;
@@ -1581,7 +1598,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
 		if (err)
 			goto err_free_sig;
 
-		mr->access_mode = MLX5_ACCESS_MODE_KLM;
+		mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
 		mr->sig->psv_memory.psv_idx = psv_index[0];
 		mr->sig->psv_wire.psv_idx = psv_index[1];
 
@@ -1595,9 +1612,10 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
 		goto err_free_in;
 	}
 
-	in->seg.flags = MLX5_PERM_UMR_EN | mr->access_mode;
-	err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, sizeof(*in),
-				    NULL, NULL, NULL);
+	MLX5_SET(mkc, mkc, access_mode, mr->access_mode);
+	MLX5_SET(mkc, mkc, umr_en, 1);
+
+	err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
 	if (err)
 		goto err_destroy_psv;
 
@@ -1633,8 +1651,10 @@ struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
 			       struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
-	struct mlx5_create_mkey_mbox_in *in = NULL;
+	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 	struct mlx5_ib_mw *mw = NULL;
+	u32 *in = NULL;
+	void *mkc;
 	int ndescs;
 	int err;
 	struct mlx5_ib_alloc_mw req = {};
@@ -1658,23 +1678,24 @@ struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
 	ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
 
 	mw = kzalloc(sizeof(*mw), GFP_KERNEL);
-	in = kzalloc(sizeof(*in), GFP_KERNEL);
+	in = kzalloc(inlen, GFP_KERNEL);
 	if (!mw || !in) {
 		err = -ENOMEM;
 		goto free;
 	}
 
-	in->seg.status = MLX5_MKEY_STATUS_FREE;
-	in->seg.xlt_oct_size = cpu_to_be32(ndescs);
-	in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
-	in->seg.flags = MLX5_PERM_UMR_EN | MLX5_ACCESS_MODE_KLM |
-		MLX5_PERM_LOCAL_READ;
-	if (type == IB_MW_TYPE_2)
-		in->seg.flags_pd |= cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
-	in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
-
-	err = mlx5_core_create_mkey(dev->mdev, &mw->mmkey, in, sizeof(*in),
-				    NULL, NULL, NULL);
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+
+	MLX5_SET(mkc, mkc, free, 1);
+	MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
+	MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
+	MLX5_SET(mkc, mkc, umr_en, 1);
+	MLX5_SET(mkc, mkc, lr, 1);
+	MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_KLMS);
+	MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2)));
+	MLX5_SET(mkc, mkc, qpn, 0xffffff);
+
+	err = mlx5_core_create_mkey(dev->mdev, &mw->mmkey, in, inlen);
 	if (err)
 		goto free;
 
@@ -1811,7 +1832,7 @@ int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 				   mr->desc_size * mr->max_descs,
 				   DMA_TO_DEVICE);
 
-	if (mr->access_mode == MLX5_ACCESS_MODE_KLM)
+	if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
 		n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset);
 	else
 		n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 34e79e709c67..cacb631a7b0a 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -782,8 +782,8 @@ void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev)
 
 int __init mlx5_ib_odp_init(void)
 {
-	mlx5_ib_page_fault_wq =
-		create_singlethread_workqueue("mlx5_ib_page_faults");
+	mlx5_ib_page_fault_wq = alloc_ordered_workqueue("mlx5_ib_page_faults",
+							WQ_MEM_RECLAIM);
 	if (!mlx5_ib_page_fault_wq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 0dd7d93cac95..41f4c2afbcdd 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -35,7 +35,6 @@
 #include <rdma/ib_cache.h>
 #include <rdma/ib_user_verbs.h>
 #include "mlx5_ib.h"
-#include "user.h"
 
 /* not supported currently */
 static int wq_signature;
@@ -77,6 +76,17 @@ struct mlx5_wqe_eth_pad {
 	u8 rsvd0[16];
 };
 
+enum raw_qp_set_mask_map {
+	MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID		= 1UL << 0,
+};
+
+struct mlx5_modify_raw_qp_param {
+	u16 operation;
+
+	u32 set_mask; /* raw_qp_set_mask_map */
+	u8 rq_q_ctr_id;
+};
+
 static void get_cqs(enum ib_qp_type qp_type,
 		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
 		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
@@ -726,7 +736,7 @@ err_umem:
 static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 			  struct mlx5_ib_qp *qp, struct ib_udata *udata,
 			  struct ib_qp_init_attr *attr,
-			  struct mlx5_create_qp_mbox_in **in,
+			  u32 **in,
 			  struct mlx5_ib_create_qp_resp *resp, int *inlen,
 			  struct mlx5_ib_qp_base *base)
 {
@@ -739,6 +749,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	u32 offset = 0;
 	int uuarn;
 	int ncont = 0;
+	__be64 *pas;
+	void *qpc;
 	int err;
 
 	err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
@@ -795,20 +807,24 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		ubuffer->umem = NULL;
 	}
 
-	*inlen = sizeof(**in) + sizeof(*(*in)->pas) * ncont;
+	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
+		 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont;
 	*in = mlx5_vzalloc(*inlen);
 	if (!*in) {
 		err = -ENOMEM;
 		goto err_umem;
 	}
+
+	pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
 	if (ubuffer->umem)
-		mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift,
-				     (*in)->pas, 0);
-	(*in)->ctx.log_pg_sz_remote_qpn =
-		cpu_to_be32((page_shift - MLX5_ADAPTER_PAGE_SHIFT) << 24);
-	(*in)->ctx.params2 = cpu_to_be32(offset << 6);
+		mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
+
+	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
+
+	MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
+	MLX5_SET(qpc, qpc, page_offset, offset);
 
-	(*in)->ctx.qp_counter_set_usr_page = cpu_to_be32(uar_index);
+	MLX5_SET(qpc, qpc, uar_page, uar_index);
 	resp->uuar_index = uuarn;
 	qp->uuarn = uuarn;
 
@@ -857,12 +873,13 @@ static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp,
 static int create_kernel_qp(struct mlx5_ib_dev *dev,
 			    struct ib_qp_init_attr *init_attr,
 			    struct mlx5_ib_qp *qp,
-			    struct mlx5_create_qp_mbox_in **in, int *inlen,
+			    u32 **in, int *inlen,
 			    struct mlx5_ib_qp_base *base)
 {
 	enum mlx5_ib_latency_class lc = MLX5_IB_LATENCY_CLASS_LOW;
 	struct mlx5_uuar_info *uuari;
 	int uar_index;
+	void *qpc;
 	int uuarn;
 	int err;
 
@@ -902,25 +919,29 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
 	}
 
 	qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt);
-	*inlen = sizeof(**in) + sizeof(*(*in)->pas) * qp->buf.npages;
+	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
+		 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
 	*in = mlx5_vzalloc(*inlen);
 	if (!*in) {
 		err = -ENOMEM;
 		goto err_buf;
 	}
-	(*in)->ctx.qp_counter_set_usr_page = cpu_to_be32(uar_index);
-	(*in)->ctx.log_pg_sz_remote_qpn =
-		cpu_to_be32((qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT) << 24);
+
+	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
+	MLX5_SET(qpc, qpc, uar_page, uar_index);
+	MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
+
 	/* Set "fast registration enabled" for all kernel QPs */
-	(*in)->ctx.params1 |= cpu_to_be32(1 << 11);
-	(*in)->ctx.sq_crq_size |= cpu_to_be16(1 << 4);
+	MLX5_SET(qpc, qpc, fre, 1);
+	MLX5_SET(qpc, qpc, rlky, 1);
 
 	if (init_attr->create_flags & mlx5_ib_create_qp_sqpn_qp1()) {
-		(*in)->ctx.deth_sqpn = cpu_to_be32(1);
+		MLX5_SET(qpc, qpc, deth_sqpn, 1);
 		qp->flags |= MLX5_IB_QP_SQPN_QP1;
 	}
 
-	mlx5_fill_page_array(&qp->buf, (*in)->pas);
+	mlx5_fill_page_array(&qp->buf,
+			     (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas));
 
 	err = mlx5_db_alloc(dev->mdev, &qp->db);
 	if (err) {
@@ -974,15 +995,15 @@ static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 	free_uuar(&dev->mdev->priv.uuari, qp->bf->uuarn);
 }
 
-static __be32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
+static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
 {
 	if (attr->srq || (attr->qp_type == IB_QPT_XRC_TGT) ||
 	    (attr->qp_type == IB_QPT_XRC_INI))
-		return cpu_to_be32(MLX5_SRQ_RQ);
+		return MLX5_SRQ_RQ;
 	else if (!qp->has_rq)
-		return cpu_to_be32(MLX5_ZERO_LEN_RQ);
+		return MLX5_ZERO_LEN_RQ;
 	else
-		return cpu_to_be32(MLX5_NON_ZERO_RQ);
+		return MLX5_NON_ZERO_RQ;
 }
 
 static int is_connected(enum ib_qp_type qp_type)
@@ -996,13 +1017,10 @@ static int is_connected(enum ib_qp_type qp_type)
 static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
 				    struct mlx5_ib_sq *sq, u32 tdn)
 {
-	u32 in[MLX5_ST_SZ_DW(create_tis_in)];
+	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
 	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(tisc, tisc, transport_domain, tdn);
-
 	return mlx5_core_create_tis(dev->mdev, in, sizeof(in), &sq->tisn);
 }
 
@@ -1191,7 +1209,7 @@ static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
 }
 
 static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
-				struct mlx5_create_qp_mbox_in *in,
+				u32 *in,
 				struct ib_pd *pd)
 {
 	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
@@ -1449,6 +1467,7 @@ create_tir:
 	kvfree(in);
 	/* qpn is reserved for that QP */
 	qp->trans_qp.base.mqp.qpn = 0;
+	qp->flags |= MLX5_IB_QP_RSS;
 	return 0;
 
 err:
@@ -1461,18 +1480,18 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 			    struct ib_udata *udata, struct mlx5_ib_qp *qp)
 {
 	struct mlx5_ib_resources *devr = &dev->devr;
+	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
 	struct mlx5_core_dev *mdev = dev->mdev;
-	struct mlx5_ib_qp_base *base;
 	struct mlx5_ib_create_qp_resp resp;
-	struct mlx5_create_qp_mbox_in *in;
-	struct mlx5_ib_create_qp ucmd;
 	struct mlx5_ib_cq *send_cq;
 	struct mlx5_ib_cq *recv_cq;
 	unsigned long flags;
-	int inlen = sizeof(*in);
-	int err;
 	u32 uidx = MLX5_IB_DEFAULT_UIDX;
+	struct mlx5_ib_create_qp ucmd;
+	struct mlx5_ib_qp_base *base;
 	void *qpc;
+	u32 *in;
+	int err;
 
 	base = init_attr->qp_type == IB_QPT_RAW_PACKET ?
 	       &qp->raw_packet_qp.rq.base :
@@ -1600,7 +1619,7 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		if (err)
 			return err;
 	} else {
-		in = mlx5_vzalloc(sizeof(*in));
+		in = mlx5_vzalloc(inlen);
 		if (!in)
 			return -ENOMEM;
 
@@ -1610,26 +1629,29 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	if (is_sqp(init_attr->qp_type))
 		qp->port = init_attr->port_num;
 
-	in->ctx.flags = cpu_to_be32(to_mlx5_st(init_attr->qp_type) << 16 |
-				    MLX5_QP_PM_MIGRATED << 11);
+	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
+
+	MLX5_SET(qpc, qpc, st, to_mlx5_st(init_attr->qp_type));
+	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
 
 	if (init_attr->qp_type != MLX5_IB_QPT_REG_UMR)
-		in->ctx.flags_pd = cpu_to_be32(to_mpd(pd ? pd : devr->p0)->pdn);
+		MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
 	else
-		in->ctx.flags_pd = cpu_to_be32(MLX5_QP_LAT_SENSITIVE);
+		MLX5_SET(qpc, qpc, latency_sensitive, 1);
+
 
 	if (qp->wq_sig)
-		in->ctx.flags_pd |= cpu_to_be32(MLX5_QP_ENABLE_SIG);
+		MLX5_SET(qpc, qpc, wq_signature, 1);
 
 	if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
-		in->ctx.flags_pd |= cpu_to_be32(MLX5_QP_BLOCK_MCAST);
+		MLX5_SET(qpc, qpc, block_lb_mc, 1);
 
 	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
-		in->ctx.params2 |= cpu_to_be32(MLX5_QP_BIT_CC_MASTER);
+		MLX5_SET(qpc, qpc, cd_master, 1);
 	if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
-		in->ctx.params2 |= cpu_to_be32(MLX5_QP_BIT_CC_SLAVE_SEND);
+		MLX5_SET(qpc, qpc, cd_slave_send, 1);
 	if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
-		in->ctx.params2 |= cpu_to_be32(MLX5_QP_BIT_CC_SLAVE_RECV);
+		MLX5_SET(qpc, qpc, cd_slave_receive, 1);
 
 	if (qp->scat_cqe && is_connected(init_attr->qp_type)) {
 		int rcqe_sz;
@@ -1639,71 +1661,68 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		scqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->send_cq);
 
 		if (rcqe_sz == 128)
-			in->ctx.cs_res = MLX5_RES_SCAT_DATA64_CQE;
+			MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
 		else
-			in->ctx.cs_res = MLX5_RES_SCAT_DATA32_CQE;
+			MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA32_CQE);
 
 		if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) {
 			if (scqe_sz == 128)
-				in->ctx.cs_req = MLX5_REQ_SCAT_DATA64_CQE;
+				MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
 			else
-				in->ctx.cs_req = MLX5_REQ_SCAT_DATA32_CQE;
+				MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
 		}
 	}
 
 	if (qp->rq.wqe_cnt) {
-		in->ctx.rq_size_stride = (qp->rq.wqe_shift - 4);
-		in->ctx.rq_size_stride |= ilog2(qp->rq.wqe_cnt) << 3;
+		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
+		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
 	}
 
-	in->ctx.rq_type_srqn = get_rx_type(qp, init_attr);
+	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
 
 	if (qp->sq.wqe_cnt)
-		in->ctx.sq_crq_size |= cpu_to_be16(ilog2(qp->sq.wqe_cnt) << 11);
+		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
 	else
-		in->ctx.sq_crq_size |= cpu_to_be16(0x8000);
+		MLX5_SET(qpc, qpc, no_sq, 1);
 
 	/* Set default resources */
 	switch (init_attr->qp_type) {
 	case IB_QPT_XRC_TGT:
-		in->ctx.cqn_recv = cpu_to_be32(to_mcq(devr->c0)->mcq.cqn);
-		in->ctx.cqn_send = cpu_to_be32(to_mcq(devr->c0)->mcq.cqn);
-		in->ctx.rq_type_srqn |= cpu_to_be32(to_msrq(devr->s0)->msrq.srqn);
-		in->ctx.xrcd = cpu_to_be32(to_mxrcd(init_attr->xrcd)->xrcdn);
+		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
+		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
+		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
+		MLX5_SET(qpc, qpc, xrcd, to_mxrcd(init_attr->xrcd)->xrcdn);
 		break;
 	case IB_QPT_XRC_INI:
-		in->ctx.cqn_recv = cpu_to_be32(to_mcq(devr->c0)->mcq.cqn);
-		in->ctx.xrcd = cpu_to_be32(to_mxrcd(devr->x1)->xrcdn);
-		in->ctx.rq_type_srqn |= cpu_to_be32(to_msrq(devr->s0)->msrq.srqn);
+		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
+		MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
+		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
 		break;
 	default:
 		if (init_attr->srq) {
-			in->ctx.xrcd = cpu_to_be32(to_mxrcd(devr->x0)->xrcdn);
-			in->ctx.rq_type_srqn |= cpu_to_be32(to_msrq(init_attr->srq)->msrq.srqn);
+			MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
+			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
 		} else {
-			in->ctx.xrcd = cpu_to_be32(to_mxrcd(devr->x1)->xrcdn);
-			in->ctx.rq_type_srqn |=
-				cpu_to_be32(to_msrq(devr->s1)->msrq.srqn);
+			MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
+			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
 		}
 	}
 
 	if (init_attr->send_cq)
-		in->ctx.cqn_send = cpu_to_be32(to_mcq(init_attr->send_cq)->mcq.cqn);
+		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);
 
 	if (init_attr->recv_cq)
-		in->ctx.cqn_recv = cpu_to_be32(to_mcq(init_attr->recv_cq)->mcq.cqn);
+		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);
 
-	in->ctx.db_rec_addr = cpu_to_be64(qp->db.dma);
+	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
 
-	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1) {
-		qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
-		/* 0xffffff means we ask to work with cqe version 0 */
+	/* 0xffffff means we ask to work with cqe version 0 */
+	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
 		MLX5_SET(qpc, qpc, user_index, uidx);
-	}
+
 	/* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
 	if (init_attr->qp_type == IB_QPT_UD &&
 	    (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)) {
-		qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 		MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
 		qp->flags |= MLX5_IB_QP_LSO;
 	}
@@ -1854,13 +1873,13 @@ static void get_cqs(enum ib_qp_type qp_type,
 }
 
 static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
-				u16 operation);
+				const struct mlx5_modify_raw_qp_param *raw_qp_param,
+				u8 lag_tx_affinity);
 
 static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 {
 	struct mlx5_ib_cq *send_cq, *recv_cq;
 	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
-	struct mlx5_modify_qp_mbox_in *in;
 	unsigned long flags;
 	int err;
 
@@ -1873,19 +1892,18 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 	       &qp->raw_packet_qp.rq.base :
 	       &qp->trans_qp.base;
 
-	in = kzalloc(sizeof(*in), GFP_KERNEL);
-	if (!in)
-		return;
-
 	if (qp->state != IB_QPS_RESET) {
 		if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET) {
 			mlx5_ib_qp_disable_pagefaults(qp);
 			err = mlx5_core_qp_modify(dev->mdev,
-						  MLX5_CMD_OP_2RST_QP, in, 0,
-						  &base->mqp);
+						  MLX5_CMD_OP_2RST_QP, 0,
+						  NULL, &base->mqp);
 		} else {
-			err = modify_raw_packet_qp(dev, qp,
-						   MLX5_CMD_OP_2RST_QP);
+			struct mlx5_modify_raw_qp_param raw_qp_param = {
+				.operation = MLX5_CMD_OP_2RST_QP
+			};
+
+			err = modify_raw_packet_qp(dev, qp, &raw_qp_param, 0);
 		}
 		if (err)
 			mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
@@ -1924,8 +1942,6 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 				     base->mqp.qpn);
 	}
 
-	kfree(in);
-
 	if (qp->create_type == MLX5_QP_KERNEL)
 		destroy_qp_kernel(dev, qp);
 	else if (qp->create_type == MLX5_QP_USER)
@@ -2151,6 +2167,31 @@ static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
 	return err;
 }
 
+static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
+					 struct mlx5_ib_sq *sq, u8 tx_affinity)
+{
+	void *in;
+	void *tisc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
+	in = mlx5_vzalloc(inlen);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
+
+	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
+	MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
+
+	err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);
+
+	kvfree(in);
+
+	return err;
+}
+
 static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			 const struct ib_ah_attr *ah,
 			 struct mlx5_qp_path *path, u8 port, int attr_mask,
@@ -2361,8 +2402,9 @@ static int ib_mask_to_mlx5_opt(int ib_mask)
 	return result;
 }
 
-static int modify_raw_packet_qp_rq(struct mlx5_core_dev *dev,
-				   struct mlx5_ib_rq *rq, int new_state)
+static int modify_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
+				   struct mlx5_ib_rq *rq, int new_state,
+				   const struct mlx5_modify_raw_qp_param *raw_qp_param)
 {
 	void *in;
 	void *rqc;
@@ -2379,7 +2421,17 @@ static int modify_raw_packet_qp_rq(struct mlx5_core_dev *dev,
 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 	MLX5_SET(rqc, rqc, state, new_state);
 
-	err = mlx5_core_modify_rq(dev, rq->base.mqp.qpn, in, inlen);
+	if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) {
+		if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
+			MLX5_SET64(modify_rq_in, in, modify_bitmask,
+				   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_MODIFY_RQ_COUNTER_SET_ID);
+			MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
+		} else
+			pr_info_once("%s: RAW PACKET QP counters are not supported on current FW\n",
+				     dev->ib_dev.name);
+	}
+
+	err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in, inlen);
 	if (err)
 		goto out;
 
@@ -2420,7 +2472,8 @@ out:
 }
 
 static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
-				u16 operation)
+				const struct mlx5_modify_raw_qp_param *raw_qp_param,
+				u8 tx_affinity)
 {
 	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
 	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
@@ -2429,7 +2482,7 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	int sq_state;
 	int err;
 
-	switch (operation) {
+	switch (raw_qp_param->operation) {
 	case MLX5_CMD_OP_RST2INIT_QP:
 		rq_state = MLX5_RQC_STATE_RDY;
 		sq_state = MLX5_SQC_STATE_RDY;
@@ -2446,21 +2499,31 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	case MLX5_CMD_OP_INIT2RTR_QP:
 	case MLX5_CMD_OP_RTR2RTS_QP:
 	case MLX5_CMD_OP_RTS2RTS_QP:
-		/* Nothing to do here... */
-		return 0;
+		if (raw_qp_param->set_mask)
+			return -EINVAL;
+		else
+			return 0;
 	default:
 		WARN_ON(1);
 		return -EINVAL;
 	}
 
 	if (qp->rq.wqe_cnt) {
-		err =  modify_raw_packet_qp_rq(dev->mdev, rq, rq_state);
+		err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param);
 		if (err)
 			return err;
 	}
 
-	if (qp->sq.wqe_cnt)
+	if (qp->sq.wqe_cnt) {
+		if (tx_affinity) {
+			err = modify_raw_packet_tx_affinity(dev->mdev, sq,
+							    tx_affinity);
+			if (err)
+				return err;
+		}
+
 		return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state);
+	}
 
 	return 0;
 }
@@ -2511,20 +2574,20 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 	struct mlx5_ib_cq *send_cq, *recv_cq;
 	struct mlx5_qp_context *context;
-	struct mlx5_modify_qp_mbox_in *in;
 	struct mlx5_ib_pd *pd;
+	struct mlx5_ib_port *mibport = NULL;
 	enum mlx5_qp_state mlx5_cur, mlx5_new;
 	enum mlx5_qp_optpar optpar;
 	int sqd_event;
 	int mlx5_st;
 	int err;
 	u16 op;
+	u8 tx_affinity = 0;
 
-	in = kzalloc(sizeof(*in), GFP_KERNEL);
-	if (!in)
+	context = kzalloc(sizeof(*context), GFP_KERNEL);
+	if (!context)
 		return -ENOMEM;
 
-	context = &in->ctx;
 	err = to_mlx5_st(ibqp->qp_type);
 	if (err < 0) {
 		mlx5_ib_dbg(dev, "unsupported qp type %d\n", ibqp->qp_type);
@@ -2549,6 +2612,23 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 		}
 	}
 
+	if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
+		if ((ibqp->qp_type == IB_QPT_RC) ||
+		    (ibqp->qp_type == IB_QPT_UD &&
+		     !(qp->flags & MLX5_IB_QP_SQPN_QP1)) ||
+		    (ibqp->qp_type == IB_QPT_UC) ||
+		    (ibqp->qp_type == IB_QPT_RAW_PACKET) ||
+		    (ibqp->qp_type == IB_QPT_XRC_INI) ||
+		    (ibqp->qp_type == IB_QPT_XRC_TGT)) {
+			if (mlx5_lag_is_active(dev->mdev)) {
+				tx_affinity = (unsigned int)atomic_add_return(1,
+						&dev->roce.next_port) %
+						MLX5_MAX_PORTS + 1;
+				context->flags |= cpu_to_be32(tx_affinity << 24);
+			}
+		}
+	}
+
 	if (is_sqp(ibqp->qp_type)) {
 		context->mtu_msgmax = (IB_MTU_256 << 5) | 8;
 	} else if (ibqp->qp_type == IB_QPT_UD ||
@@ -2654,8 +2734,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 		u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
 			       qp->port) - 1;
-		struct mlx5_ib_port *mibport = &dev->port[port_num];
-
+		mibport = &dev->port[port_num];
 		context->qp_counter_set_usr_page |=
 			cpu_to_be32((u32)(mibport->q_cnt_id) << 24);
 	}
@@ -2689,13 +2768,21 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	op = optab[mlx5_cur][mlx5_new];
 	optpar = ib_mask_to_mlx5_opt(attr_mask);
 	optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
-	in->optparam = cpu_to_be32(optpar);
 
-	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET)
-		err = modify_raw_packet_qp(dev, qp, op);
-	else
-		err = mlx5_core_qp_modify(dev->mdev, op, in, sqd_event,
+	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
+		struct mlx5_modify_raw_qp_param raw_qp_param = {};
+
+		raw_qp_param.operation = op;
+		if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
+			raw_qp_param.rq_q_ctr_id = mibport->q_cnt_id;
+			raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
+		}
+		err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
+	} else {
+		err = mlx5_core_qp_modify(dev->mdev, op, optpar, context,
 					  &base->mqp);
+	}
+
 	if (err)
 		goto out;
 
@@ -2735,7 +2822,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	}
 
 out:
-	kfree(in);
+	kfree(context);
 	return err;
 }
 
@@ -2968,7 +3055,7 @@ static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
 
 	memset(umr, 0, sizeof(*umr));
 
-	if (mr->access_mode == MLX5_ACCESS_MODE_KLM)
+	if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
 		/* KLMs take twice the size of MTTs */
 		ndescs *= 2;
 
@@ -3111,9 +3198,9 @@ static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
 
 	memset(seg, 0, sizeof(*seg));
 
-	if (mr->access_mode == MLX5_ACCESS_MODE_MTT)
+	if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
 		seg->log2_page_size = ilog2(mr->ibmr.page_size);
-	else if (mr->access_mode == MLX5_ACCESS_MODE_KLM)
+	else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
 		/* KLMs take twice the size of MTTs */
 		ndescs *= 2;
 
@@ -3454,7 +3541,7 @@ static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
 	memset(seg, 0, sizeof(*seg));
 
 	seg->flags = get_umr_flags(wr->access_flags) |
-				   MLX5_ACCESS_MODE_KLM;
+				   MLX5_MKC_ACCESS_MODE_KLMS;
 	seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
 	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
 				    MLX5_MKEY_BSF_EN | pdn);
@@ -3658,12 +3745,8 @@ static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
 		     struct ib_send_wr *wr, unsigned *idx,
 		     int *size, int nreq)
 {
-	int err = 0;
-
-	if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq))) {
-		err = -ENOMEM;
-		return err;
-	}
+	if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
+		return -ENOMEM;
 
 	*idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
 	*seg = mlx5_get_send_wqe(qp, *idx);
@@ -3679,7 +3762,7 @@ static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
 	*seg += sizeof(**ctrl);
 	*size = sizeof(**ctrl) / 16;
 
-	return err;
+	return 0;
 }
 
 static void finish_wqe(struct mlx5_ib_qp *qp,
@@ -3758,7 +3841,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 		num_sge = wr->num_sge;
 		if (unlikely(num_sge > qp->sq.max_gs)) {
 			mlx5_ib_warn(dev, "\n");
-			err = -ENOMEM;
+			err = -EINVAL;
 			*bad_wr = wr;
 			goto out;
 		}
@@ -4320,21 +4403,24 @@ static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
 static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			 struct ib_qp_attr *qp_attr)
 {
-	struct mlx5_query_qp_mbox_out *outb;
+	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
 	struct mlx5_qp_context *context;
 	int mlx5_state;
+	u32 *outb;
 	int err = 0;
 
-	outb = kzalloc(sizeof(*outb), GFP_KERNEL);
+	outb = kzalloc(outlen, GFP_KERNEL);
 	if (!outb)
 		return -ENOMEM;
 
-	context = &outb->ctx;
 	err = mlx5_core_qp_query(dev->mdev, &qp->trans_qp.base.mqp, outb,
-				 sizeof(*outb));
+				 outlen);
 	if (err)
 		goto out;
 
+	/* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */
+	context = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, outb, qpc);
+
 	mlx5_state = be32_to_cpu(context->flags) >> 28;
 
 	qp->state		     = to_ib_qp_state(mlx5_state);
@@ -4499,6 +4585,28 @@ int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
 	return 0;
 }
 
+static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
+{
+	struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);
+	struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device);
+	struct ib_event event;
+
+	if (rwq->ibwq.event_handler) {
+		event.device     = rwq->ibwq.device;
+		event.element.wq = &rwq->ibwq;
+		switch (type) {
+		case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+			event.event = IB_EVENT_WQ_FATAL;
+			break;
+		default:
+			mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
+			return;
+		}
+
+		rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
+	}
+}
+
 static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
 		      struct ib_wq_init_attr *init_attr)
 {
@@ -4536,7 +4644,7 @@ static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
 	MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);
 	rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
 	mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0);
-	err = mlx5_core_create_rq(dev->mdev, in, inlen, &rwq->rqn);
+	err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rwq->core_qp);
 	kvfree(in);
 	return err;
 }
@@ -4652,7 +4760,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 		return ERR_PTR(-EINVAL);
 	}
 
-	rwq->ibwq.wq_num = rwq->rqn;
+	rwq->ibwq.wq_num = rwq->core_qp.qpn;
 	rwq->ibwq.state = IB_WQS_RESET;
 	if (udata->outlen) {
 		resp.response_length = offsetof(typeof(resp), response_length) +
@@ -4662,10 +4770,12 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 			goto err_copy;
 	}
 
+	rwq->core_qp.event = mlx5_ib_wq_event;
+	rwq->ibwq.event_handler = init_attr->event_handler;
 	return &rwq->ibwq;
 
 err_copy:
-	mlx5_core_destroy_rq(dev->mdev, rwq->rqn);
+	mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
 err_user_rq:
 	destroy_user_rq(pd, rwq);
 err:
@@ -4678,7 +4788,7 @@ int mlx5_ib_destroy_wq(struct ib_wq *wq)
 	struct mlx5_ib_dev *dev = to_mdev(wq->device);
 	struct mlx5_ib_rwq *rwq = to_mrwq(wq);
 
-	mlx5_core_destroy_rq(dev->mdev, rwq->rqn);
+	mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
 	destroy_user_rq(wq->pd, rwq);
 	kfree(rwq);
 
@@ -4810,7 +4920,7 @@ int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
 	MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
 	MLX5_SET(rqc, rqc, state, wq_state);
 
-	err = mlx5_core_modify_rq(dev->mdev, rwq->rqn, in, inlen);
+	err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in, inlen);
 	kvfree(in);
 	if (!err)
 		rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index ed6ac52355f1..3857dbd9c956 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -38,7 +38,6 @@
 #include <rdma/ib_user_verbs.h>
 
 #include "mlx5_ib.h"
-#include "user.h"
 
 /* not supported currently */
 static int srq_signature;
diff --git a/drivers/infiniband/hw/mlx5/user.h b/drivers/infiniband/hw/mlx5/user.h
deleted file mode 100644
index 188dac4301b5..000000000000
--- a/drivers/infiniband/hw/mlx5/user.h
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef MLX5_IB_USER_H
-#define MLX5_IB_USER_H
-
-#include <linux/types.h>
-
-#include "mlx5_ib.h"
-
-enum {
-	MLX5_QP_FLAG_SIGNATURE		= 1 << 0,
-	MLX5_QP_FLAG_SCATTER_CQE	= 1 << 1,
-};
-
-enum {
-	MLX5_SRQ_FLAG_SIGNATURE		= 1 << 0,
-};
-
-enum {
-	MLX5_WQ_FLAG_SIGNATURE		= 1 << 0,
-};
-
-
-/* Increment this value if any changes that break userspace ABI
- * compatibility are made.
- */
-#define MLX5_IB_UVERBS_ABI_VERSION	1
-
-/* Make sure that all structs defined in this file remain laid out so
- * that they pack the same way on 32-bit and 64-bit architectures (to
- * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __u64
- * instead.
- */
-
-struct mlx5_ib_alloc_ucontext_req {
-	__u32	total_num_uuars;
-	__u32	num_low_latency_uuars;
-};
-
-struct mlx5_ib_alloc_ucontext_req_v2 {
-	__u32	total_num_uuars;
-	__u32	num_low_latency_uuars;
-	__u32	flags;
-	__u32	comp_mask;
-	__u8	max_cqe_version;
-	__u8	reserved0;
-	__u16	reserved1;
-	__u32	reserved2;
-};
-
-enum mlx5_ib_alloc_ucontext_resp_mask {
-	MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET = 1UL << 0,
-};
-
-enum mlx5_user_cmds_supp_uhw {
-	MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE = 1 << 0,
-};
-
-struct mlx5_ib_alloc_ucontext_resp {
-	__u32	qp_tab_size;
-	__u32	bf_reg_size;
-	__u32	tot_uuars;
-	__u32	cache_line_size;
-	__u16	max_sq_desc_sz;
-	__u16	max_rq_desc_sz;
-	__u32	max_send_wqebb;
-	__u32	max_recv_wr;
-	__u32	max_srq_recv_wr;
-	__u16	num_ports;
-	__u16	reserved1;
-	__u32	comp_mask;
-	__u32	response_length;
-	__u8	cqe_version;
-	__u8	cmds_supp_uhw;
-	__u16	reserved2;
-	__u64	hca_core_clock_offset;
-};
-
-struct mlx5_ib_alloc_pd_resp {
-	__u32	pdn;
-};
-
-struct mlx5_ib_tso_caps {
-	__u32 max_tso; /* Maximum tso payload size in bytes */
-
-	/* Corresponding bit will be set if qp type from
-	 * 'enum ib_qp_type' is supported, e.g.
-	 * supported_qpts |= 1 << IB_QPT_UD
-	 */
-	__u32 supported_qpts;
-};
-
-struct mlx5_ib_query_device_resp {
-	__u32	comp_mask;
-	__u32	response_length;
-	struct	mlx5_ib_tso_caps tso_caps;
-};
-
-struct mlx5_ib_create_cq {
-	__u64	buf_addr;
-	__u64	db_addr;
-	__u32	cqe_size;
-	__u32	reserved; /* explicit padding (optional on i386) */
-};
-
-struct mlx5_ib_create_cq_resp {
-	__u32	cqn;
-	__u32	reserved;
-};
-
-struct mlx5_ib_resize_cq {
-	__u64	buf_addr;
-	__u16	cqe_size;
-	__u16	reserved0;
-	__u32	reserved1;
-};
-
-struct mlx5_ib_create_srq {
-	__u64	buf_addr;
-	__u64	db_addr;
-	__u32	flags;
-	__u32	reserved0; /* explicit padding (optional on i386) */
-	__u32	uidx;
-	__u32	reserved1;
-};
-
-struct mlx5_ib_create_srq_resp {
-	__u32	srqn;
-	__u32	reserved;
-};
-
-struct mlx5_ib_create_qp {
-	__u64	buf_addr;
-	__u64	db_addr;
-	__u32	sq_wqe_count;
-	__u32	rq_wqe_count;
-	__u32	rq_wqe_shift;
-	__u32	flags;
-	__u32	uidx;
-	__u32	reserved0;
-	__u64	sq_buf_addr;
-};
-
-/* RX Hash function flags */
-enum mlx5_rx_hash_function_flags {
-	MLX5_RX_HASH_FUNC_TOEPLITZ	= 1 << 0,
-};
-
-/*
- * RX Hash flags, these flags allows to set which incoming packet's field should
- * participates in RX Hash. Each flag represent certain packet's field,
- * when the flag is set the field that is represented by the flag will
- * participate in RX Hash calculation.
- * Note: *IPV4 and *IPV6 flags can't be enabled together on the same QP
- * and *TCP and *UDP flags can't be enabled together on the same QP.
-*/
-enum mlx5_rx_hash_fields {
-	MLX5_RX_HASH_SRC_IPV4	= 1 << 0,
-	MLX5_RX_HASH_DST_IPV4	= 1 << 1,
-	MLX5_RX_HASH_SRC_IPV6	= 1 << 2,
-	MLX5_RX_HASH_DST_IPV6	= 1 << 3,
-	MLX5_RX_HASH_SRC_PORT_TCP	= 1 << 4,
-	MLX5_RX_HASH_DST_PORT_TCP	= 1 << 5,
-	MLX5_RX_HASH_SRC_PORT_UDP	= 1 << 6,
-	MLX5_RX_HASH_DST_PORT_UDP	= 1 << 7
-};
-
-struct mlx5_ib_create_qp_rss {
-	__u64 rx_hash_fields_mask; /* enum mlx5_rx_hash_fields */
-	__u8 rx_hash_function; /* enum mlx5_rx_hash_function_flags */
-	__u8 rx_key_len; /* valid only for Toeplitz */
-	__u8 reserved[6];
-	__u8 rx_hash_key[128]; /* valid only for Toeplitz */
-	__u32   comp_mask;
-	__u32   reserved1;
-};
-
-struct mlx5_ib_create_qp_resp {
-	__u32	uuar_index;
-};
-
-struct mlx5_ib_alloc_mw {
-	__u32	comp_mask;
-	__u8	num_klms;
-	__u8	reserved1;
-	__u16	reserved2;
-};
-
-struct mlx5_ib_create_wq {
-	__u64   buf_addr;
-	__u64   db_addr;
-	__u32   rq_wqe_count;
-	__u32   rq_wqe_shift;
-	__u32   user_index;
-	__u32   flags;
-	__u32   comp_mask;
-	__u32   reserved;
-};
-
-struct mlx5_ib_create_wq_resp {
-	__u32	response_length;
-	__u32	reserved;
-};
-
-struct mlx5_ib_create_rwq_ind_tbl_resp {
-	__u32	response_length;
-	__u32	reserved;
-};
-
-struct mlx5_ib_modify_wq {
-	__u32	comp_mask;
-	__u32	reserved;
-};
-
-static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
-				    struct mlx5_ib_create_qp *ucmd,
-				    int inlen,
-				    u32 *user_index)
-{
-	u8 cqe_version = ucontext->cqe_version;
-
-	if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
-	    !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
-		return 0;
-
-	if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
-	       !!cqe_version))
-		return -EINVAL;
-
-	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
-}
-
-static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
-				     struct mlx5_ib_create_srq *ucmd,
-				     int inlen,
-				     u32 *user_index)
-{
-	u8 cqe_version = ucontext->cqe_version;
-
-	if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
-	    !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
-		return 0;
-
-	if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
-	       !!cqe_version))
-		return -EINVAL;
-
-	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
-}
-#endif /* MLX5_IB_USER_H */
diff --git a/drivers/infiniband/hw/mthca/mthca_catas.c b/drivers/infiniband/hw/mthca/mthca_catas.c
index 712d2a30fbe5..f6474c24f193 100644
--- a/drivers/infiniband/hw/mthca/mthca_catas.c
+++ b/drivers/infiniband/hw/mthca/mthca_catas.c
@@ -187,7 +187,7 @@ int __init mthca_catas_init(void)
 {
 	INIT_WORK(&catas_work, catas_reset);
 
-	catas_wq = create_singlethread_workqueue("mthca_catas");
+	catas_wq = alloc_ordered_workqueue("mthca_catas", WQ_MEM_RECLAIM);
 	if (!catas_wq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/hw/mthca/mthca_mad.c b/drivers/infiniband/hw/mthca/mthca_mad.c
index 7c3f2fb44ba5..9139405c4810 100644
--- a/drivers/infiniband/hw/mthca/mthca_mad.c
+++ b/drivers/infiniband/hw/mthca/mthca_mad.c
@@ -153,7 +153,8 @@ static void node_desc_override(struct ib_device *dev,
 	    mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP &&
 	    mad->mad_hdr.attr_id == IB_SMP_ATTR_NODE_DESC) {
 		mutex_lock(&to_mdev(dev)->cap_mask_mutex);
-		memcpy(((struct ib_smp *) mad)->data, dev->node_desc, 64);
+		memcpy(((struct ib_smp *) mad)->data, dev->node_desc,
+		       IB_DEVICE_NODE_DESC_MAX);
 		mutex_unlock(&to_mdev(dev)->cap_mask_mutex);
 	}
 }
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index da2335f7f7c3..358930a41e36 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -46,7 +46,7 @@
 
 #include "mthca_dev.h"
 #include "mthca_cmd.h"
-#include "mthca_user.h"
+#include <rdma/mthca-abi.h>
 #include "mthca_memfree.h"
 
 static void init_query_mad(struct ib_smp *mad)
@@ -193,7 +193,8 @@ static int mthca_modify_device(struct ib_device *ibdev,
 	if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
 		if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
 			return -ERESTARTSYS;
-		memcpy(ibdev->node_desc, props->node_desc, 64);
+		memcpy(ibdev->node_desc, props->node_desc,
+		       IB_DEVICE_NODE_DESC_MAX);
 		mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);
 	}
 
@@ -1138,7 +1139,7 @@ static int mthca_init_node_data(struct mthca_dev *dev)
 	if (err)
 		goto out;
 
-	memcpy(dev->ib_dev.node_desc, out_mad->data, 64);
+	memcpy(dev->ib_dev.node_desc, out_mad->data, IB_DEVICE_NODE_DESC_MAX);
 
 	in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
 
diff --git a/drivers/infiniband/hw/nes/nes.h b/drivers/infiniband/hw/nes/nes.h
index bd9d132f11c7..e7430c9254d3 100644
--- a/drivers/infiniband/hw/nes/nes.h
+++ b/drivers/infiniband/hw/nes/nes.h
@@ -165,7 +165,7 @@ do { \
 #include "nes_hw.h"
 #include "nes_verbs.h"
 #include "nes_context.h"
-#include "nes_user.h"
+#include <rdma/nes-abi.h>
 #include "nes_cm.h"
 #include "nes_mgt.h"
 
diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index 7f0aa23aef9d..57db9b332f44 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -2692,12 +2692,12 @@ static struct nes_cm_core *nes_cm_alloc_core(void)
 	nes_debug(NES_DBG_CM, "Init CM Core completed -- cm_core=%p\n", cm_core);
 
 	nes_debug(NES_DBG_CM, "Enable QUEUE EVENTS\n");
-	cm_core->event_wq = create_singlethread_workqueue("nesewq");
+	cm_core->event_wq = alloc_ordered_workqueue("nesewq", 0);
 	if (!cm_core->event_wq)
 		goto out_free_cmcore;
 	cm_core->post_event = nes_cm_post_event;
 	nes_debug(NES_DBG_CM, "Enable QUEUE DISCONNECTS\n");
-	cm_core->disconn_wq = create_singlethread_workqueue("nesdwq");
+	cm_core->disconn_wq = alloc_ordered_workqueue("nesdwq", 0);
 	if (!cm_core->disconn_wq)
 		goto out_free_wq;
 
diff --git a/drivers/infiniband/hw/nes/nes_user.h b/drivers/infiniband/hw/nes/nes_user.h
deleted file mode 100644
index 529c421bb15c..000000000000
--- a/drivers/infiniband/hw/nes/nes_user.h
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation.  All rights reserved.
- * Copyright (c) 2005 Topspin Communications.  All rights reserved.
- * Copyright (c) 2005 Cisco Systems.  All rights reserved.
- * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#ifndef NES_USER_H
-#define NES_USER_H
-
-#include <linux/types.h>
-
-#define NES_ABI_USERSPACE_VER 2
-#define NES_ABI_KERNEL_VER    2
-
-/*
- * Make sure that all structs defined in this file remain laid out so
- * that they pack the same way on 32-bit and 64-bit architectures (to
- * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __u64
- * instead.
- */
-
-struct nes_alloc_ucontext_req {
-	__u32 reserved32;
-	__u8  userspace_ver;
-	__u8  reserved8[3];
-};
-
-struct nes_alloc_ucontext_resp {
-	__u32 max_pds; /* maximum pds allowed for this user process */
-	__u32 max_qps; /* maximum qps allowed for this user process */
-	__u32 wq_size; /* size of the WQs (sq+rq) allocated to the mmaped area */
-	__u8  virtwq;  /* flag to indicate if virtual WQ are to be used or not */
-	__u8  kernel_ver;
-	__u8  reserved[2];
-};
-
-struct nes_alloc_pd_resp {
-	__u32 pd_id;
-	__u32 mmap_db_index;
-};
-
-struct nes_create_cq_req {
-	__u64 user_cq_buffer;
-	__u32 mcrqf;
-	__u8 reserved[4];
-};
-
-struct nes_create_qp_req {
-	__u64 user_wqe_buffers;
-	__u64 user_qp_buffer;
-};
-
-enum iwnes_memreg_type {
-	IWNES_MEMREG_TYPE_MEM = 0x0000,
-	IWNES_MEMREG_TYPE_QP = 0x0001,
-	IWNES_MEMREG_TYPE_CQ = 0x0002,
-	IWNES_MEMREG_TYPE_MW = 0x0003,
-	IWNES_MEMREG_TYPE_FMR = 0x0004,
-	IWNES_MEMREG_TYPE_FMEM = 0x0005,
-};
-
-struct nes_mem_reg_req {
-	__u32 reg_type;	/* indicates if id is memory, QP or CQ */
-	__u32 reserved;
-};
-
-struct nes_create_cq_resp {
-	__u32 cq_id;
-	__u32 cq_size;
-	__u32 mmap_db_index;
-	__u32 reserved;
-};
-
-struct nes_create_qp_resp {
-	__u32 qp_id;
-	__u32 actual_sq_size;
-	__u32 actual_rq_size;
-	__u32 mmap_sq_db_index;
-	__u32 mmap_rq_db_index;
-	__u32 nes_drv_opt;
-};
-
-#endif				/* NES_USER_H */
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_abi.h b/drivers/infiniband/hw/ocrdma/ocrdma_abi.h
deleted file mode 100644
index 430b1350fe96..000000000000
--- a/drivers/infiniband/hw/ocrdma/ocrdma_abi.h
+++ /dev/null
@@ -1,149 +0,0 @@
-/* This file is part of the Emulex RoCE Device Driver for
- * RoCE (RDMA over Converged Ethernet) adapters.
- * Copyright (C) 2012-2015 Emulex. All rights reserved.
- * EMULEX and SLI are trademarks of Emulex.
- * www.emulex.com
- *
- * This software is available to you under a choice of one of two licenses.
- * You may choose to be licensed under the terms of the GNU General Public
- * License (GPL) Version 2, available from the file COPYING in the main
- * directory of this source tree, or the BSD license below:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * - Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- *
- * - Redistributions in binary form must reproduce the above copyright
- *   notice, this list of conditions and the following disclaimer in
- *   the documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Contact Information:
- * linux-drivers@emulex.com
- *
- * Emulex
- * 3333 Susan Street
- * Costa Mesa, CA 92626
- */
-
-#ifndef __OCRDMA_ABI_H__
-#define __OCRDMA_ABI_H__
-
-#define OCRDMA_ABI_VERSION 2
-#define OCRDMA_BE_ROCE_ABI_VERSION 1
-/* user kernel communication data structures. */
-
-struct ocrdma_alloc_ucontext_resp {
-	u32 dev_id;
-	u32 wqe_size;
-	u32 max_inline_data;
-	u32 dpp_wqe_size;
-	u64 ah_tbl_page;
-	u32 ah_tbl_len;
-	u32 rqe_size;
-	u8 fw_ver[32];
-	/* for future use/new features in progress */
-	u64 rsvd1;
-	u64 rsvd2;
-};
-
-struct ocrdma_alloc_pd_ureq {
-	u64 rsvd1;
-};
-
-struct ocrdma_alloc_pd_uresp {
-	u32 id;
-	u32 dpp_enabled;
-	u32 dpp_page_addr_hi;
-	u32 dpp_page_addr_lo;
-	u64 rsvd1;
-};
-
-struct ocrdma_create_cq_ureq {
-	u32 dpp_cq;
-	u32 rsvd; /* pad */
-};
-
-#define MAX_CQ_PAGES 8
-struct ocrdma_create_cq_uresp {
-	u32 cq_id;
-	u32 page_size;
-	u32 num_pages;
-	u32 max_hw_cqe;
-	u64 page_addr[MAX_CQ_PAGES];
-	u64 db_page_addr;
-	u32 db_page_size;
-	u32 phase_change;
-	/* for future use/new features in progress */
-	u64 rsvd1;
-	u64 rsvd2;
-};
-
-#define MAX_QP_PAGES 8
-#define MAX_UD_AV_PAGES 8
-
-struct ocrdma_create_qp_ureq {
-	u8 enable_dpp_cq;
-	u8 rsvd;
-	u16 dpp_cq_id;
-	u32 rsvd1;	/* pad */
-};
-
-struct ocrdma_create_qp_uresp {
-	u16 qp_id;
-	u16 sq_dbid;
-	u16 rq_dbid;
-	u16 resv0;	/* pad */
-	u32 sq_page_size;
-	u32 rq_page_size;
-	u32 num_sq_pages;
-	u32 num_rq_pages;
-	u64 sq_page_addr[MAX_QP_PAGES];
-	u64 rq_page_addr[MAX_QP_PAGES];
-	u64 db_page_addr;
-	u32 db_page_size;
-	u32 dpp_credit;
-	u32 dpp_offset;
-	u32 num_wqe_allocated;
-	u32 num_rqe_allocated;
-	u32 db_sq_offset;
-	u32 db_rq_offset;
-	u32 db_shift;
-	u64 rsvd[11];
-} __packed;
-
-struct ocrdma_create_srq_uresp {
-	u16 rq_dbid;
-	u16 resv0;	/* pad */
-	u32 resv1;
-
-	u32 rq_page_size;
-	u32 num_rq_pages;
-
-	u64 rq_page_addr[MAX_QP_PAGES];
-	u64 db_page_addr;
-
-	u32 db_page_size;
-	u32 num_rqe_allocated;
-	u32 db_rq_offset;
-	u32 db_shift;
-
-	u64 rsvd2;
-	u64 rsvd3;
-};
-
-#endif				/* __OCRDMA_ABI_H__ */
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
index 16740dcb876b..67fc0b6857e1 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
@@ -1156,18 +1156,18 @@ static void ocrdma_get_attr(struct ocrdma_dev *dev,
 	attr->max_srq =
 		(rsp->max_srq_rpir_qps & OCRDMA_MBX_QUERY_CFG_MAX_SRQ_MASK) >>
 		OCRDMA_MBX_QUERY_CFG_MAX_SRQ_OFFSET;
-	attr->max_send_sge = ((rsp->max_write_send_sge &
+	attr->max_send_sge = ((rsp->max_recv_send_sge &
 			       OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_MASK) >>
 			      OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_SHIFT);
-	attr->max_recv_sge = (rsp->max_write_send_sge &
-			      OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_MASK) >>
-	    OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_SHIFT;
+	attr->max_recv_sge = (rsp->max_recv_send_sge &
+			      OCRDMA_MBX_QUERY_CFG_MAX_RECV_SGE_MASK) >>
+	    OCRDMA_MBX_QUERY_CFG_MAX_RECV_SGE_SHIFT;
 	attr->max_srq_sge = (rsp->max_srq_rqe_sge &
 			      OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_MASK) >>
 	    OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_OFFSET;
-	attr->max_rdma_sge = (rsp->max_write_send_sge &
-			      OCRDMA_MBX_QUERY_CFG_MAX_WRITE_SGE_MASK) >>
-	    OCRDMA_MBX_QUERY_CFG_MAX_WRITE_SGE_SHIFT;
+	attr->max_rdma_sge = (rsp->max_wr_rd_sge &
+			      OCRDMA_MBX_QUERY_CFG_MAX_RD_SGE_MASK) >>
+	    OCRDMA_MBX_QUERY_CFG_MAX_RD_SGE_SHIFT;
 	attr->max_ord_per_qp = (rsp->max_ird_ord_per_qp &
 				OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_MASK) >>
 	    OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_SHIFT;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
index 07d0c6c5b046..896071502739 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
@@ -56,7 +56,7 @@
 #include "be_roce.h"
 #include "ocrdma_hw.h"
 #include "ocrdma_stats.h"
-#include "ocrdma_abi.h"
+#include <rdma/ocrdma-abi.h>
 
 MODULE_VERSION(OCRDMA_ROCE_DRV_VERSION);
 MODULE_DESCRIPTION(OCRDMA_ROCE_DRV_DESC " " OCRDMA_ROCE_DRV_VERSION);
@@ -119,6 +119,7 @@ static int ocrdma_register_device(struct ocrdma_dev *dev)
 {
 	strlcpy(dev->ibdev.name, "ocrdma%d", IB_DEVICE_NAME_MAX);
 	ocrdma_get_guid(dev, (u8 *)&dev->ibdev.node_guid);
+	BUILD_BUG_ON(sizeof(OCRDMA_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
 	memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC,
 	       sizeof(OCRDMA_NODE_DESC));
 	dev->ibdev.owner = THIS_MODULE;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_sli.h b/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
index 0efc9662c6d8..37df4481bb8f 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
@@ -554,9 +554,9 @@ enum {
 	OCRDMA_MBX_QUERY_CFG_L3_TYPE_MASK		= 0x18,
 	OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_SHIFT		= 0,
 	OCRDMA_MBX_QUERY_CFG_MAX_SEND_SGE_MASK		= 0xFFFF,
-	OCRDMA_MBX_QUERY_CFG_MAX_WRITE_SGE_SHIFT	= 16,
-	OCRDMA_MBX_QUERY_CFG_MAX_WRITE_SGE_MASK		= 0xFFFF <<
-				OCRDMA_MBX_QUERY_CFG_MAX_WRITE_SGE_SHIFT,
+	OCRDMA_MBX_QUERY_CFG_MAX_RECV_SGE_SHIFT	= 16,
+	OCRDMA_MBX_QUERY_CFG_MAX_RECV_SGE_MASK		= 0xFFFF <<
+				OCRDMA_MBX_QUERY_CFG_MAX_RECV_SGE_SHIFT,
 
 	OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_SHIFT	= 0,
 	OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_MASK	= 0xFFFF,
@@ -612,6 +612,8 @@ enum {
 	OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_OFFSET		= 0,
 	OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_MASK		= 0xFFFF <<
 				OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_OFFSET,
+	OCRDMA_MBX_QUERY_CFG_MAX_RD_SGE_SHIFT		= 0,
+	OCRDMA_MBX_QUERY_CFG_MAX_RD_SGE_MASK		= 0xFFFF,
 };
 
 struct ocrdma_mbx_query_config {
@@ -619,7 +621,7 @@ struct ocrdma_mbx_query_config {
 	struct ocrdma_mbx_rsp rsp;
 	u32 qp_srq_cq_ird_ord;
 	u32 max_pd_ca_ack_delay;
-	u32 max_write_send_sge;
+	u32 max_recv_send_sge;
 	u32 max_ird_ord_per_qp;
 	u32 max_shared_ird_ord;
 	u32 max_mr;
@@ -639,6 +641,8 @@ struct ocrdma_mbx_query_config {
 	u32 max_wqes_rqes_per_q;
 	u32 max_cq_cqes_per_cq;
 	u32 max_srq_rqe_sge;
+	u32 max_wr_rd_sge;
+	u32 ird_pgsz_num_pages;
 };
 
 struct ocrdma_fw_ver_rsp {
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index b1a3d91fe8b9..6af44f8db3d5 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -51,7 +51,7 @@
 #include "ocrdma.h"
 #include "ocrdma_hw.h"
 #include "ocrdma_verbs.h"
-#include "ocrdma_abi.h"
+#include <rdma/ocrdma-abi.h>
 
 int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
 {
@@ -125,8 +125,8 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
 					IB_DEVICE_SYS_IMAGE_GUID |
 					IB_DEVICE_LOCAL_DMA_LKEY |
 					IB_DEVICE_MEM_MGT_EXTENSIONS;
-	attr->max_sge = dev->attr.max_send_sge;
-	attr->max_sge_rd = attr->max_sge;
+	attr->max_sge = min(dev->attr.max_send_sge, dev->attr.max_recv_sge);
+	attr->max_sge_rd = dev->attr.max_rdma_sge;
 	attr->max_cq = dev->attr.max_cq;
 	attr->max_cqe = dev->attr.max_cqe;
 	attr->max_mr = dev->attr.max_mr;
diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index bbf0a163aeab..a3e21a25cea5 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -52,6 +52,7 @@
 #include <linux/kref.h>
 #include <linux/sched.h>
 #include <linux/kthread.h>
+#include <rdma/ib_hdrs.h>
 #include <rdma/rdma_vt.h>
 
 #include "qib_common.h"
@@ -1131,7 +1132,6 @@ extern spinlock_t qib_devs_lock;
 extern struct qib_devdata *qib_lookup(int unit);
 extern u32 qib_cpulist_count;
 extern unsigned long *qib_cpulist;
-extern u16 qpt_mask;
 extern unsigned qib_cc_table_size;
 
 int qib_init(struct qib_devdata *, int);
diff --git a/drivers/infiniband/hw/qib/qib_debugfs.c b/drivers/infiniband/hw/qib/qib_debugfs.c
index 5e75b43c596b..5bad8e3b40bb 100644
--- a/drivers/infiniband/hw/qib/qib_debugfs.c
+++ b/drivers/infiniband/hw/qib/qib_debugfs.c
@@ -189,27 +189,32 @@ static int _ctx_stats_seq_show(struct seq_file *s, void *v)
 DEBUGFS_FILE(ctx_stats)
 
 static void *_qp_stats_seq_start(struct seq_file *s, loff_t *pos)
+	__acquires(RCU)
 {
 	struct qib_qp_iter *iter;
 	loff_t n = *pos;
 
-	rcu_read_lock();
 	iter = qib_qp_iter_init(s->private);
+
+	/* stop calls rcu_read_unlock */
+	rcu_read_lock();
+
 	if (!iter)
 		return NULL;
 
-	while (n--) {
+	do {
 		if (qib_qp_iter_next(iter)) {
 			kfree(iter);
 			return NULL;
 		}
-	}
+	} while (n--);
 
 	return iter;
 }
 
 static void *_qp_stats_seq_next(struct seq_file *s, void *iter_ptr,
 				   loff_t *pos)
+	__must_hold(RCU)
 {
 	struct qib_qp_iter *iter = iter_ptr;
 
@@ -224,6 +229,7 @@ static void *_qp_stats_seq_next(struct seq_file *s, void *iter_ptr,
 }
 
 static void _qp_stats_seq_stop(struct seq_file *s, void *iter_ptr)
+	__releases(RCU)
 {
 	rcu_read_unlock();
 }
diff --git a/drivers/infiniband/hw/qib/qib_driver.c b/drivers/infiniband/hw/qib/qib_driver.c
index 67ee6438cf59..728e0a030d2e 100644
--- a/drivers/infiniband/hw/qib/qib_driver.c
+++ b/drivers/infiniband/hw/qib/qib_driver.c
@@ -319,8 +319,8 @@ static u32 qib_rcv_hdrerr(struct qib_ctxtdata *rcd, struct qib_pportdata *ppd,
 		ret = 1;
 	else if (eflags == QLOGIC_IB_RHF_H_TIDERR) {
 		/* For TIDERR and RC QPs premptively schedule a NAK */
-		struct qib_ib_header *hdr = (struct qib_ib_header *) rhdr;
-		struct qib_other_headers *ohdr = NULL;
+		struct ib_header *hdr = (struct ib_header *)rhdr;
+		struct ib_other_headers *ohdr = NULL;
 		struct qib_ibport *ibp = &ppd->ibport_data;
 		struct qib_devdata *dd = ppd->dd;
 		struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
@@ -588,8 +588,7 @@ move_along:
 				qib_schedule_send(qp);
 			spin_unlock_irqrestore(&qp->s_lock, flags);
 		}
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
+		rvt_put_qp(qp);
 	}
 
 bail:
diff --git a/drivers/infiniband/hw/qib/qib_fs.c b/drivers/infiniband/hw/qib/qib_fs.c
index fcdf37913a26..f1e66efea98a 100644
--- a/drivers/infiniband/hw/qib/qib_fs.c
+++ b/drivers/infiniband/hw/qib/qib_fs.c
@@ -64,7 +64,7 @@ static int qibfs_mknod(struct inode *dir, struct dentry *dentry,
 	inode->i_uid = GLOBAL_ROOT_UID;
 	inode->i_gid = GLOBAL_ROOT_GID;
 	inode->i_blocks = 0;
-	inode->i_atime = CURRENT_TIME;
+	inode->i_atime = current_time(inode);
 	inode->i_mtime = inode->i_atime;
 	inode->i_ctime = inode->i_atime;
 	inode->i_private = data;
@@ -328,26 +328,12 @@ static ssize_t flash_write(struct file *file, const char __user *buf,
 
 	pos = *ppos;
 
-	if (pos != 0) {
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	if (count != sizeof(struct qib_flash)) {
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	tmp = kmalloc(count, GFP_KERNEL);
-	if (!tmp) {
-		ret = -ENOMEM;
-		goto bail;
-	}
+	if (pos != 0 || count != sizeof(struct qib_flash))
+		return -EINVAL;
 
-	if (copy_from_user(tmp, buf, count)) {
-		ret = -EFAULT;
-		goto bail_tmp;
-	}
+	tmp = memdup_user(buf, count);
+	if (IS_ERR(tmp))
+		return PTR_ERR(tmp);
 
 	dd = private2dd(file);
 	if (qib_eeprom_write(dd, pos, tmp, count)) {
@@ -361,8 +347,6 @@ static ssize_t flash_write(struct file *file, const char __user *buf,
 
 bail_tmp:
 	kfree(tmp);
-
-bail:
 	return ret;
 }
 
diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c
index ce4034071f9c..ded27172320e 100644
--- a/drivers/infiniband/hw/qib/qib_iba7322.c
+++ b/drivers/infiniband/hw/qib/qib_iba7322.c
@@ -1415,7 +1415,7 @@ static void flush_fifo(struct qib_pportdata *ppd)
 	u32 *hdr;
 	u64 pbc;
 	const unsigned hdrwords = 7;
-	static struct qib_ib_header ibhdr = {
+	static struct ib_header ibhdr = {
 		.lrh[0] = cpu_to_be16(0xF000 | QIB_LRH_BTH),
 		.lrh[1] = IB_LID_PERMISSIVE,
 		.lrh[2] = cpu_to_be16(hdrwords + SIZE_OF_CRC),
diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c
index f253111e682e..1730aa839a47 100644
--- a/drivers/infiniband/hw/qib/qib_init.c
+++ b/drivers/infiniband/hw/qib/qib_init.c
@@ -614,8 +614,8 @@ static int qib_create_workqueues(struct qib_devdata *dd)
 
 			snprintf(wq_name, sizeof(wq_name), "qib%d_%d",
 				dd->unit, pidx);
-			ppd->qib_wq =
-				create_singlethread_workqueue(wq_name);
+			ppd->qib_wq = alloc_ordered_workqueue(wq_name,
+							      WQ_MEM_RECLAIM);
 			if (!ppd->qib_wq)
 				goto wq_error;
 		}
diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c
index 9cc0aae1d781..99d31efe4c2f 100644
--- a/drivers/infiniband/hw/qib/qib_qp.c
+++ b/drivers/infiniband/hw/qib/qib_qp.c
@@ -41,14 +41,6 @@
 
 #include "qib.h"
 
-/*
- * mask field which was present in now deleted qib_qpn_table
- * is not present in rvt_qpn_table. Defining the same field
- * as qpt_mask here instead of adding the mask field to
- * rvt_qpn_table.
- */
-u16 qpt_mask;
-
 static inline unsigned mk_qpn(struct rvt_qpn_table *qpt,
 			      struct rvt_qpn_map *map, unsigned off)
 {
@@ -57,7 +49,7 @@ static inline unsigned mk_qpn(struct rvt_qpn_table *qpt,
 
 static inline unsigned find_next_offset(struct rvt_qpn_table *qpt,
 					struct rvt_qpn_map *map, unsigned off,
-					unsigned n)
+					unsigned n, u16 qpt_mask)
 {
 	if (qpt_mask) {
 		off++;
@@ -179,6 +171,7 @@ int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
 	struct qib_ibdev *verbs_dev = container_of(rdi, struct qib_ibdev, rdi);
 	struct qib_devdata *dd = container_of(verbs_dev, struct qib_devdata,
 					      verbs_dev);
+	u16 qpt_mask = dd->qpn_mask;
 
 	if (type == IB_QPT_SMI || type == IB_QPT_GSI) {
 		unsigned n;
@@ -215,7 +208,7 @@ int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
 				goto bail;
 			}
 			offset = find_next_offset(qpt, map, offset,
-				dd->n_krcv_queues);
+				dd->n_krcv_queues, qpt_mask);
 			qpn = mk_qpn(qpt, map, offset);
 			/*
 			 * This test differs from alloc_pidmap().
@@ -573,10 +566,6 @@ struct qib_qp_iter *qib_qp_iter_init(struct qib_ibdev *dev)
 		return NULL;
 
 	iter->dev = dev;
-	if (qib_qp_iter_next(iter)) {
-		kfree(iter);
-		return NULL;
-	}
 
 	return iter;
 }
diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c
index 444028a3582a..2097512e75aa 100644
--- a/drivers/infiniband/hw/qib/qib_rc.c
+++ b/drivers/infiniband/hw/qib/qib_rc.c
@@ -75,7 +75,7 @@ static void start_timer(struct rvt_qp *qp)
  * Note the QP s_lock must be held.
  */
 static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp,
-			   struct qib_other_headers *ohdr, u32 pmtu)
+			   struct ib_other_headers *ohdr, u32 pmtu)
 {
 	struct rvt_ack_entry *e;
 	u32 hwords;
@@ -154,10 +154,7 @@ static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp,
 			len = 0;
 			qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
 			ohdr->u.at.aeth = qib_compute_aeth(qp);
-			ohdr->u.at.atomic_ack_eth[0] =
-				cpu_to_be32(e->atomic_data >> 32);
-			ohdr->u.at.atomic_ack_eth[1] =
-				cpu_to_be32(e->atomic_data);
+			ib_u64_put(e->atomic_data, &ohdr->u.at.atomic_ack_eth);
 			hwords += sizeof(ohdr->u.at) / sizeof(u32);
 			bth2 = e->psn & QIB_PSN_MASK;
 			e->sent = 1;
@@ -234,7 +231,7 @@ int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
 {
 	struct qib_qp_priv *priv = qp->priv;
 	struct qib_ibdev *dev = to_idev(qp->ibqp.device);
-	struct qib_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct rvt_sge_state *ss;
 	struct rvt_swqe *wqe;
 	u32 hwords;
@@ -444,20 +441,18 @@ int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
 			}
 			if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
 				qp->s_state = OP(COMPARE_SWAP);
-				ohdr->u.atomic_eth.swap_data = cpu_to_be64(
-					wqe->atomic_wr.swap);
-				ohdr->u.atomic_eth.compare_data = cpu_to_be64(
-					wqe->atomic_wr.compare_add);
+				put_ib_ateth_swap(wqe->atomic_wr.swap,
+						  &ohdr->u.atomic_eth);
+				put_ib_ateth_swap(wqe->atomic_wr.compare_add,
+						  &ohdr->u.atomic_eth);
 			} else {
 				qp->s_state = OP(FETCH_ADD);
-				ohdr->u.atomic_eth.swap_data = cpu_to_be64(
-					wqe->atomic_wr.compare_add);
-				ohdr->u.atomic_eth.compare_data = 0;
+				put_ib_ateth_swap(wqe->atomic_wr.compare_add,
+						  &ohdr->u.atomic_eth);
+				put_ib_ateth_swap(0, &ohdr->u.atomic_eth);
 			}
-			ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32(
-				wqe->atomic_wr.remote_addr >> 32);
-			ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32(
-				wqe->atomic_wr.remote_addr);
+			put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr,
+					   &ohdr->u.atomic_eth);
 			ohdr->u.atomic_eth.rkey = cpu_to_be32(
 				wqe->atomic_wr.rkey);
 			hwords += sizeof(struct ib_atomic_eth) / sizeof(u32);
@@ -632,8 +627,8 @@ void qib_send_rc_ack(struct rvt_qp *qp)
 	u32 hwords;
 	u32 pbufn;
 	u32 __iomem *piobuf;
-	struct qib_ib_header hdr;
-	struct qib_other_headers *ohdr;
+	struct ib_header hdr;
+	struct ib_other_headers *ohdr;
 	u32 control;
 	unsigned long flags;
 
@@ -942,9 +937,9 @@ static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
 /*
  * This should be called with the QP s_lock held and interrupts disabled.
  */
-void qib_rc_send_complete(struct rvt_qp *qp, struct qib_ib_header *hdr)
+void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 {
-	struct qib_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct rvt_swqe *wqe;
 	struct ib_wc wc;
 	unsigned i;
@@ -1177,7 +1172,7 @@ static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
 				qib_restart_rc(qp, qp->s_last_psn + 1, 0);
 				if (list_empty(&qp->rspwait)) {
 					qp->r_flags |= RVT_R_RSP_SEND;
-					atomic_inc(&qp->refcount);
+					rvt_get_qp(qp);
 					list_add_tail(&qp->rspwait,
 						      &rcd->qp_wait_list);
 				}
@@ -1361,7 +1356,7 @@ static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn,
 	qib_restart_rc(qp, qp->s_last_psn + 1, 0);
 	if (list_empty(&qp->rspwait)) {
 		qp->r_flags |= RVT_R_RSP_SEND;
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
 	}
 }
@@ -1383,7 +1378,7 @@ static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn,
  * Called at interrupt level.
  */
 static void qib_rc_rcv_resp(struct qib_ibport *ibp,
-			    struct qib_other_headers *ohdr,
+			    struct ib_other_headers *ohdr,
 			    void *data, u32 tlen,
 			    struct rvt_qp *qp,
 			    u32 opcode,
@@ -1463,12 +1458,9 @@ static void qib_rc_rcv_resp(struct qib_ibport *ibp,
 	case OP(ATOMIC_ACKNOWLEDGE):
 	case OP(RDMA_READ_RESPONSE_FIRST):
 		aeth = be32_to_cpu(ohdr->u.aeth);
-		if (opcode == OP(ATOMIC_ACKNOWLEDGE)) {
-			__be32 *p = ohdr->u.at.atomic_ack_eth;
-
-			val = ((u64) be32_to_cpu(p[0]) << 32) |
-				be32_to_cpu(p[1]);
-		} else
+		if (opcode == OP(ATOMIC_ACKNOWLEDGE))
+			val = ib_u64_get(&ohdr->u.at.atomic_ack_eth);
+		else
 			val = 0;
 		if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
 		    opcode != OP(RDMA_READ_RESPONSE_FIRST))
@@ -1608,7 +1600,7 @@ bail:
  * Return 1 if no more processing is needed; otherwise return 0 to
  * schedule a response to be sent.
  */
-static int qib_rc_rcv_error(struct qib_other_headers *ohdr,
+static int qib_rc_rcv_error(struct ib_other_headers *ohdr,
 			    void *data,
 			    struct rvt_qp *qp,
 			    u32 opcode,
@@ -1640,7 +1632,7 @@ static int qib_rc_rcv_error(struct qib_other_headers *ohdr,
 			 */
 			if (list_empty(&qp->rspwait)) {
 				qp->r_flags |= RVT_R_RSP_NAK;
-				atomic_inc(&qp->refcount);
+				rvt_get_qp(qp);
 				list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
 			}
 		}
@@ -1848,11 +1840,11 @@ static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n)
  * for the given QP.
  * Called at interrupt level.
  */
-void qib_rc_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr,
+void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
 		int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
 {
 	struct qib_ibport *ibp = &rcd->ppd->ibport_data;
-	struct qib_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	u32 opcode;
 	u32 hdrsize;
 	u32 psn;
@@ -2177,8 +2169,7 @@ send_last:
 			e->rdma_sge.mr = NULL;
 		}
 		ateth = &ohdr->u.atomic_eth;
-		vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) |
-			be32_to_cpu(ateth->vaddr[1]);
+		vaddr = get_ib_ateth_vaddr(ateth);
 		if (unlikely(vaddr & (sizeof(u64) - 1)))
 			goto nack_inv_unlck;
 		rkey = be32_to_cpu(ateth->rkey);
@@ -2189,11 +2180,11 @@ send_last:
 			goto nack_acc_unlck;
 		/* Perform atomic OP and save result. */
 		maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
-		sdata = be64_to_cpu(ateth->swap_data);
+		sdata = get_ib_ateth_swap(ateth);
 		e->atomic_data = (opcode == OP(FETCH_ADD)) ?
 			(u64) atomic64_add_return(sdata, maddr) - sdata :
 			(u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
-				      be64_to_cpu(ateth->compare_data),
+				      get_ib_ateth_compare(ateth),
 				      sdata);
 		rvt_put_mr(qp->r_sge.sge.mr);
 		qp->r_sge.num_sge = 0;
@@ -2233,7 +2224,7 @@ rnr_nak:
 	/* Queue RNR NAK for later */
 	if (list_empty(&qp->rspwait)) {
 		qp->r_flags |= RVT_R_RSP_NAK;
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
 	}
 	return;
@@ -2245,7 +2236,7 @@ nack_op_err:
 	/* Queue NAK for later */
 	if (list_empty(&qp->rspwait)) {
 		qp->r_flags |= RVT_R_RSP_NAK;
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
 	}
 	return;
@@ -2259,7 +2250,7 @@ nack_inv:
 	/* Queue NAK for later */
 	if (list_empty(&qp->rspwait)) {
 		qp->r_flags |= RVT_R_RSP_NAK;
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
 	}
 	return;
diff --git a/drivers/infiniband/hw/qib/qib_ruc.c b/drivers/infiniband/hw/qib/qib_ruc.c
index b67779256297..de1bde5950f5 100644
--- a/drivers/infiniband/hw/qib/qib_ruc.c
+++ b/drivers/infiniband/hw/qib/qib_ruc.c
@@ -265,7 +265,7 @@ static int gid_ok(union ib_gid *gid, __be64 gid_prefix, __be64 id)
  *
  * The s_lock will be acquired around the qib_migrate_qp() call.
  */
-int qib_ruc_check_hdr(struct qib_ibport *ibp, struct qib_ib_header *hdr,
+int qib_ruc_check_hdr(struct qib_ibport *ibp, struct ib_header *hdr,
 		      int has_grh, struct rvt_qp *qp, u32 bth0)
 {
 	__be64 guid;
@@ -680,7 +680,7 @@ u32 qib_make_grh(struct qib_ibport *ibp, struct ib_grh *hdr,
 	return sizeof(struct ib_grh) / sizeof(u32);
 }
 
-void qib_make_ruc_header(struct rvt_qp *qp, struct qib_other_headers *ohdr,
+void qib_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
 			 u32 bth0, u32 bth2)
 {
 	struct qib_qp_priv *priv = qp->priv;
diff --git a/drivers/infiniband/hw/qib/qib_uc.c b/drivers/infiniband/hw/qib/qib_uc.c
index 1d61bd04f449..5b2d483451ad 100644
--- a/drivers/infiniband/hw/qib/qib_uc.c
+++ b/drivers/infiniband/hw/qib/qib_uc.c
@@ -48,7 +48,7 @@
 int qib_make_uc_req(struct rvt_qp *qp, unsigned long *flags)
 {
 	struct qib_qp_priv *priv = qp->priv;
-	struct qib_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct rvt_swqe *wqe;
 	u32 hwords;
 	u32 bth0;
@@ -236,10 +236,10 @@ bail:
  * for the given QP.
  * Called at interrupt level.
  */
-void qib_uc_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr,
+void qib_uc_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 		int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
 {
-	struct qib_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	u32 opcode;
 	u32 hdrsize;
 	u32 psn;
diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c
index 10d062561bd9..f45cad1198b0 100644
--- a/drivers/infiniband/hw/qib/qib_ud.c
+++ b/drivers/infiniband/hw/qib/qib_ud.c
@@ -245,7 +245,7 @@ drop:
 int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags)
 {
 	struct qib_qp_priv *priv = qp->priv;
-	struct qib_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct ib_ah_attr *ah_attr;
 	struct qib_pportdata *ppd;
 	struct qib_ibport *ibp;
@@ -435,10 +435,10 @@ static unsigned qib_lookup_pkey(struct qib_ibport *ibp, u16 pkey)
  * for the given QP.
  * Called at interrupt level.
  */
-void qib_ud_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr,
+void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 		int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
 {
-	struct qib_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	int opcode;
 	u32 hdrsize;
 	u32 pad;
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index fd1dfbce5539..954f15064514 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -313,7 +313,7 @@ static void qib_copy_from_sge(void *data, struct rvt_sge_state *ss, u32 length)
  * for the given QP.
  * Called at interrupt level.
  */
-static void qib_qp_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr,
+static void qib_qp_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
 		       int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
 {
 	struct qib_ibport *ibp = &rcd->ppd->ibport_data;
@@ -366,10 +366,10 @@ void qib_ib_rcv(struct qib_ctxtdata *rcd, void *rhdr, void *data, u32 tlen)
 {
 	struct qib_pportdata *ppd = rcd->ppd;
 	struct qib_ibport *ibp = &ppd->ibport_data;
-	struct qib_ib_header *hdr = rhdr;
+	struct ib_header *hdr = rhdr;
 	struct qib_devdata *dd = ppd->dd;
 	struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
-	struct qib_other_headers *ohdr;
+	struct ib_other_headers *ohdr;
 	struct rvt_qp *qp;
 	u32 qp_num;
 	int lnh;
@@ -841,7 +841,7 @@ static void sdma_complete(struct qib_sdma_txreq *cookie, int status)
 	if (tx->wqe)
 		qib_send_complete(qp, tx->wqe, IB_WC_SUCCESS);
 	else if (qp->ibqp.qp_type == IB_QPT_RC) {
-		struct qib_ib_header *hdr;
+		struct ib_header *hdr;
 
 		if (tx->txreq.flags & QIB_SDMA_TXREQ_F_FREEBUF)
 			hdr = &tx->align_buf->hdr;
@@ -889,7 +889,7 @@ static int wait_kmem(struct qib_ibdev *dev, struct rvt_qp *qp)
 	return ret;
 }
 
-static int qib_verbs_send_dma(struct rvt_qp *qp, struct qib_ib_header *hdr,
+static int qib_verbs_send_dma(struct rvt_qp *qp, struct ib_header *hdr,
 			      u32 hdrwords, struct rvt_sge_state *ss, u32 len,
 			      u32 plen, u32 dwords)
 {
@@ -1025,7 +1025,7 @@ static int no_bufs_available(struct rvt_qp *qp)
 	return ret;
 }
 
-static int qib_verbs_send_pio(struct rvt_qp *qp, struct qib_ib_header *ibhdr,
+static int qib_verbs_send_pio(struct rvt_qp *qp, struct ib_header *ibhdr,
 			      u32 hdrwords, struct rvt_sge_state *ss, u32 len,
 			      u32 plen, u32 dwords)
 {
@@ -1133,7 +1133,7 @@ done:
  * Return zero if packet is sent or queued OK.
  * Return non-zero and clear qp->s_flags RVT_S_BUSY otherwise.
  */
-int qib_verbs_send(struct rvt_qp *qp, struct qib_ib_header *hdr,
+int qib_verbs_send(struct rvt_qp *qp, struct ib_header *hdr,
 		   u32 hdrwords, struct rvt_sge_state *ss, u32 len)
 {
 	struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device);
@@ -1370,7 +1370,8 @@ static int qib_modify_device(struct ib_device *device,
 	}
 
 	if (device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC) {
-		memcpy(device->node_desc, device_modify->node_desc, 64);
+		memcpy(device->node_desc, device_modify->node_desc,
+		       IB_DEVICE_NODE_DESC_MAX);
 		for (i = 0; i < dd->num_pports; i++) {
 			struct qib_ibport *ibp = &dd->pport[i].ibport_data;
 
@@ -1606,8 +1607,6 @@ int qib_register_ib_device(struct qib_devdata *dd)
 	/* Only need to initialize non-zero fields. */
 	setup_timer(&dev->mem_timer, mem_timer, (unsigned long)dev);
 
-	qpt_mask = dd->qpn_mask;
-
 	INIT_LIST_HEAD(&dev->piowait);
 	INIT_LIST_HEAD(&dev->dmawait);
 	INIT_LIST_HEAD(&dev->txwait);
diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h
index 736ced684842..94fd30fdedac 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -45,6 +45,7 @@
 #include <linux/completion.h>
 #include <rdma/ib_pack.h>
 #include <rdma/ib_user_verbs.h>
+#include <rdma/ib_hdrs.h>
 #include <rdma/rdma_vt.h>
 #include <rdma/rdmavt_cq.h>
 
@@ -63,16 +64,6 @@ struct qib_verbs_txreq;
  */
 #define QIB_UVERBS_ABI_VERSION       2
 
-#define IB_SEQ_NAK	(3 << 29)
-
-/* AETH NAK opcode values */
-#define IB_RNR_NAK                      0x20
-#define IB_NAK_PSN_ERROR                0x60
-#define IB_NAK_INVALID_REQUEST          0x61
-#define IB_NAK_REMOTE_ACCESS_ERROR      0x62
-#define IB_NAK_REMOTE_OPERATIONAL_ERROR 0x63
-#define IB_NAK_INVALID_RD_REQUEST       0x64
-
 /* IB Performance Manager status values */
 #define IB_PMA_SAMPLE_STATUS_DONE       0x00
 #define IB_PMA_SAMPLE_STATUS_STARTED    0x01
@@ -87,22 +78,9 @@ struct qib_verbs_txreq;
 
 #define QIB_VENDOR_IPG		cpu_to_be16(0xFFA0)
 
-#define IB_BTH_REQ_ACK		(1 << 31)
-#define IB_BTH_SOLICITED	(1 << 23)
-#define IB_BTH_MIG_REQ		(1 << 22)
-
 /* XXX Should be defined in ib_verbs.h enum ib_port_cap_flags */
 #define IB_PORT_OTHER_LOCAL_CHANGES_SUP (1 << 26)
 
-#define IB_GRH_VERSION		6
-#define IB_GRH_VERSION_MASK	0xF
-#define IB_GRH_VERSION_SHIFT	28
-#define IB_GRH_TCLASS_MASK	0xFF
-#define IB_GRH_TCLASS_SHIFT	20
-#define IB_GRH_FLOW_MASK	0xFFFFF
-#define IB_GRH_FLOW_SHIFT	0
-#define IB_GRH_NEXT_HDR		0x1B
-
 #define IB_DEFAULT_GID_PREFIX	cpu_to_be64(0xfe80000000000000ULL)
 
 /* Values for set/get portinfo VLCap OperationalVLs */
@@ -129,61 +107,9 @@ static inline int qib_num_vls(int vls)
 	}
 }
 
-struct ib_reth {
-	__be64 vaddr;
-	__be32 rkey;
-	__be32 length;
-} __packed;
-
-struct ib_atomic_eth {
-	__be32 vaddr[2];        /* unaligned so access as 2 32-bit words */
-	__be32 rkey;
-	__be64 swap_data;
-	__be64 compare_data;
-} __packed;
-
-struct qib_other_headers {
-	__be32 bth[3];
-	union {
-		struct {
-			__be32 deth[2];
-			__be32 imm_data;
-		} ud;
-		struct {
-			struct ib_reth reth;
-			__be32 imm_data;
-		} rc;
-		struct {
-			__be32 aeth;
-			__be32 atomic_ack_eth[2];
-		} at;
-		__be32 imm_data;
-		__be32 aeth;
-		__be32 ieth;
-		struct ib_atomic_eth atomic_eth;
-	} u;
-} __packed;
-
-/*
- * Note that UD packets with a GRH header are 8+40+12+8 = 68 bytes
- * long (72 w/ imm_data).  Only the first 56 bytes of the IB header
- * will be in the eager header buffer.  The remaining 12 or 16 bytes
- * are in the data buffer.
- */
-struct qib_ib_header {
-	__be16 lrh[4];
-	union {
-		struct {
-			struct ib_grh grh;
-			struct qib_other_headers oth;
-		} l;
-		struct qib_other_headers oth;
-	} u;
-} __packed;
-
 struct qib_pio_header {
 	__le32 pbc[2];
-	struct qib_ib_header hdr;
+	struct ib_header hdr;
 } __packed;
 
 /*
@@ -191,7 +117,7 @@ struct qib_pio_header {
  * is made common.
  */
 struct qib_qp_priv {
-	struct qib_ib_header *s_hdr;    /* next packet header to send */
+	struct ib_header *s_hdr;        /* next packet header to send */
 	struct list_head iowait;        /* link for wait PIO buf */
 	atomic_t s_dma_busy;
 	struct qib_verbs_txreq *s_tx;
@@ -376,7 +302,7 @@ void qib_verbs_sdma_desc_avail(struct qib_pportdata *ppd, unsigned avail);
 
 void qib_put_txreq(struct qib_verbs_txreq *tx);
 
-int qib_verbs_send(struct rvt_qp *qp, struct qib_ib_header *hdr,
+int qib_verbs_send(struct rvt_qp *qp, struct ib_header *hdr,
 		   u32 hdrwords, struct rvt_sge_state *ss, u32 len);
 
 void qib_copy_sge(struct rvt_sge_state *ss, void *data, u32 length,
@@ -384,10 +310,10 @@ void qib_copy_sge(struct rvt_sge_state *ss, void *data, u32 length,
 
 void qib_skip_sge(struct rvt_sge_state *ss, u32 length, int release);
 
-void qib_uc_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr,
+void qib_uc_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 		int has_grh, void *data, u32 tlen, struct rvt_qp *qp);
 
-void qib_rc_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr,
+void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
 		int has_grh, void *data, u32 tlen, struct rvt_qp *qp);
 
 int qib_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr);
@@ -398,13 +324,13 @@ struct ib_ah *qib_create_qp0_ah(struct qib_ibport *ibp, u16 dlid);
 
 void qib_rc_rnr_retry(unsigned long arg);
 
-void qib_rc_send_complete(struct rvt_qp *qp, struct qib_ib_header *hdr);
+void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr);
 
 void qib_rc_error(struct rvt_qp *qp, enum ib_wc_status err);
 
 int qib_post_ud_send(struct rvt_qp *qp, struct ib_send_wr *wr);
 
-void qib_ud_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr,
+void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 		int has_grh, void *data, u32 tlen, struct rvt_qp *qp);
 
 void mr_rcu_callback(struct rcu_head *list);
@@ -413,13 +339,13 @@ int qib_get_rwqe(struct rvt_qp *qp, int wr_id_only);
 
 void qib_migrate_qp(struct rvt_qp *qp);
 
-int qib_ruc_check_hdr(struct qib_ibport *ibp, struct qib_ib_header *hdr,
+int qib_ruc_check_hdr(struct qib_ibport *ibp, struct ib_header *hdr,
 		      int has_grh, struct rvt_qp *qp, u32 bth0);
 
 u32 qib_make_grh(struct qib_ibport *ibp, struct ib_grh *hdr,
 		 struct ib_global_route *grh, u32 hwords, u32 nwords);
 
-void qib_make_ruc_header(struct rvt_qp *qp, struct qib_other_headers *ohdr,
+void qib_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
 			 u32 bth0, u32 bth2);
 
 void _qib_do_send(struct work_struct *work);
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c
index c229b9f4a52d..0a89a955550b 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_main.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c
@@ -664,7 +664,8 @@ static int __init usnic_ib_init(void)
 		return err;
 	}
 
-	if (pci_register_driver(&usnic_ib_pci_driver)) {
+	err = pci_register_driver(&usnic_ib_pci_driver);
+	if (err) {
 		usnic_err("Unable to register with PCI\n");
 		goto out_umem_fini;
 	}
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index f2f229efbe64..6d9904a4a0ab 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -129,7 +129,7 @@ void rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited)
 		if (likely(worker)) {
 			cq->notify = RVT_CQ_NONE;
 			cq->triggered++;
-			queue_kthread_work(worker, &cq->comptask);
+			kthread_queue_work(worker, &cq->comptask);
 		}
 	}
 
@@ -265,7 +265,7 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 	cq->ibcq.cqe = entries;
 	cq->notify = RVT_CQ_NONE;
 	spin_lock_init(&cq->lock);
-	init_kthread_work(&cq->comptask, send_complete);
+	kthread_init_work(&cq->comptask, send_complete);
 	cq->queue = wc;
 
 	ret = &cq->ibcq;
@@ -295,7 +295,7 @@ int rvt_destroy_cq(struct ib_cq *ibcq)
 	struct rvt_cq *cq = ibcq_to_rvtcq(ibcq);
 	struct rvt_dev_info *rdi = cq->rdi;
 
-	flush_kthread_work(&cq->comptask);
+	kthread_flush_work(&cq->comptask);
 	spin_lock(&rdi->n_cqs_lock);
 	rdi->n_cqs_allocated--;
 	spin_unlock(&rdi->n_cqs_lock);
@@ -514,7 +514,7 @@ int rvt_driver_cq_init(struct rvt_dev_info *rdi)
 	rdi->worker = kzalloc(sizeof(*rdi->worker), GFP_KERNEL);
 	if (!rdi->worker)
 		return -ENOMEM;
-	init_kthread_worker(rdi->worker);
+	kthread_init_worker(rdi->worker);
 	task = kthread_create_on_node(
 		kthread_worker_fn,
 		rdi->worker,
@@ -547,7 +547,7 @@ void rvt_cq_exit(struct rvt_dev_info *rdi)
 	/* blocks future queuing from send_complete() */
 	rdi->worker = NULL;
 	smp_wmb(); /* See rdi_cq_enter */
-	flush_kthread_worker(worker);
+	kthread_flush_worker(worker);
 	kthread_stop(worker->task);
 	kfree(worker);
 }
diff --git a/drivers/infiniband/sw/rdmavt/dma.c b/drivers/infiniband/sw/rdmavt/dma.c
index 33076a5eee2f..01f71caa3ac4 100644
--- a/drivers/infiniband/sw/rdmavt/dma.c
+++ b/drivers/infiniband/sw/rdmavt/dma.c
@@ -138,6 +138,21 @@ static void rvt_unmap_sg(struct ib_device *dev,
 	/* This is a stub, nothing to be done here */
 }
 
+static int rvt_map_sg_attrs(struct ib_device *dev, struct scatterlist *sgl,
+			    int nents, enum dma_data_direction direction,
+			    unsigned long attrs)
+{
+	return rvt_map_sg(dev, sgl, nents, direction);
+}
+
+static void rvt_unmap_sg_attrs(struct ib_device *dev,
+			       struct scatterlist *sg, int nents,
+			       enum dma_data_direction direction,
+			       unsigned long attrs)
+{
+	return rvt_unmap_sg(dev, sg, nents, direction);
+}
+
 static void rvt_sync_single_for_cpu(struct ib_device *dev, u64 addr,
 				    size_t size, enum dma_data_direction dir)
 {
@@ -177,6 +192,8 @@ struct ib_dma_mapping_ops rvt_default_dma_mapping_ops = {
 	.unmap_page = rvt_dma_unmap_page,
 	.map_sg = rvt_map_sg,
 	.unmap_sg = rvt_unmap_sg,
+	.map_sg_attrs = rvt_map_sg_attrs,
+	.unmap_sg_attrs = rvt_unmap_sg_attrs,
 	.sync_single_for_cpu = rvt_sync_single_for_cpu,
 	.sync_single_for_device = rvt_sync_single_for_device,
 	.alloc_coherent = rvt_dma_alloc_coherent,
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index 80c4b6b401b8..46b64970058e 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -294,7 +294,7 @@ static void __rvt_free_mr(struct rvt_mr *mr)
 {
 	rvt_deinit_mregion(&mr->mr);
 	rvt_free_lkey(&mr->mr);
-	vfree(mr);
+	kfree(mr);
 }
 
 /**
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index bdb540f25a88..6500c3b5a89c 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -488,60 +488,23 @@ static void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp)
 	spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags);
 	if (removed) {
 		synchronize_rcu();
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
+		rvt_put_qp(qp);
 	}
 }
 
 /**
- * reset_qp - initialize the QP state to the reset state
- * @qp: the QP to reset
+ * rvt_init_qp - initialize the QP state to the reset state
+ * @qp: the QP to init or reinit
  * @type: the QP type
- * r and s lock are required to be held by the caller
+ *
+ * This function is called from both rvt_create_qp() and
+ * rvt_reset_qp().   The difference is that the reset
+ * patch the necessary locks to protect against concurent
+ * access.
  */
-static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
-		  enum ib_qp_type type)
-	__releases(&qp->s_lock)
-	__releases(&qp->s_hlock)
-	__releases(&qp->r_lock)
-	__acquires(&qp->r_lock)
-	__acquires(&qp->s_hlock)
-	__acquires(&qp->s_lock)
+static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
+			enum ib_qp_type type)
 {
-	if (qp->state != IB_QPS_RESET) {
-		qp->state = IB_QPS_RESET;
-
-		/* Let drivers flush their waitlist */
-		rdi->driver_f.flush_qp_waiters(qp);
-		qp->s_flags &= ~(RVT_S_TIMER | RVT_S_ANY_WAIT);
-		spin_unlock(&qp->s_lock);
-		spin_unlock(&qp->s_hlock);
-		spin_unlock_irq(&qp->r_lock);
-
-		/* Stop the send queue and the retry timer */
-		rdi->driver_f.stop_send_queue(qp);
-
-		/* Wait for things to stop */
-		rdi->driver_f.quiesce_qp(qp);
-
-		/* take qp out the hash and wait for it to be unused */
-		rvt_remove_qp(rdi, qp);
-		wait_event(qp->wait, !atomic_read(&qp->refcount));
-
-		/* grab the lock b/c it was locked at call time */
-		spin_lock_irq(&qp->r_lock);
-		spin_lock(&qp->s_hlock);
-		spin_lock(&qp->s_lock);
-
-		rvt_clear_mr_refs(qp, 1);
-	}
-
-	/*
-	 * Let the driver do any tear down it needs to for a qp
-	 * that has been reset
-	 */
-	rdi->driver_f.notify_qp_reset(qp);
-
 	qp->remote_qpn = 0;
 	qp->qkey = 0;
 	qp->qp_access_flags = 0;
@@ -587,6 +550,60 @@ static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
 }
 
 /**
+ * rvt_reset_qp - initialize the QP state to the reset state
+ * @qp: the QP to reset
+ * @type: the QP type
+ *
+ * r_lock, s_hlock, and s_lock are required to be held by the caller
+ */
+static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
+			 enum ib_qp_type type)
+	__must_hold(&qp->s_lock)
+	__must_hold(&qp->s_hlock)
+	__must_hold(&qp->r_lock)
+{
+	lockdep_assert_held(&qp->r_lock);
+	lockdep_assert_held(&qp->s_hlock);
+	lockdep_assert_held(&qp->s_lock);
+	if (qp->state != IB_QPS_RESET) {
+		qp->state = IB_QPS_RESET;
+
+		/* Let drivers flush their waitlist */
+		rdi->driver_f.flush_qp_waiters(qp);
+		qp->s_flags &= ~(RVT_S_TIMER | RVT_S_ANY_WAIT);
+		spin_unlock(&qp->s_lock);
+		spin_unlock(&qp->s_hlock);
+		spin_unlock_irq(&qp->r_lock);
+
+		/* Stop the send queue and the retry timer */
+		rdi->driver_f.stop_send_queue(qp);
+
+		/* Wait for things to stop */
+		rdi->driver_f.quiesce_qp(qp);
+
+		/* take qp out the hash and wait for it to be unused */
+		rvt_remove_qp(rdi, qp);
+		wait_event(qp->wait, !atomic_read(&qp->refcount));
+
+		/* grab the lock b/c it was locked at call time */
+		spin_lock_irq(&qp->r_lock);
+		spin_lock(&qp->s_hlock);
+		spin_lock(&qp->s_lock);
+
+		rvt_clear_mr_refs(qp, 1);
+		/*
+		 * Let the driver do any tear down or re-init it needs to for
+		 * a qp that has been reset
+		 */
+		rdi->driver_f.notify_qp_reset(qp);
+	}
+	rvt_init_qp(rdi, qp, type);
+	lockdep_assert_held(&qp->r_lock);
+	lockdep_assert_held(&qp->s_hlock);
+	lockdep_assert_held(&qp->s_lock);
+}
+
+/**
  * rvt_create_qp - create a queue pair for a device
  * @ibpd: the protection domain who's device we create the queue pair for
  * @init_attr: the attributes of the queue pair
@@ -766,7 +783,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 		}
 		qp->ibqp.qp_num = err;
 		qp->port_num = init_attr->port_num;
-		rvt_reset_qp(rdi, qp, init_attr->qp_type);
+		rvt_init_qp(rdi, qp, init_attr->qp_type);
 		break;
 
 	default:
@@ -873,7 +890,8 @@ bail_qpn:
 	free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num);
 
 bail_rq_wq:
-	vfree(qp->r_rq.wq);
+	if (!qp->ip)
+		vfree(qp->r_rq.wq);
 
 bail_driver_priv:
 	rdi->driver_f.qp_priv_free(rdi, qp);
@@ -905,6 +923,8 @@ int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err)
 	int ret = 0;
 	struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
 
+	lockdep_assert_held(&qp->r_lock);
+	lockdep_assert_held(&qp->s_lock);
 	if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET)
 		goto bail;
 
@@ -979,7 +999,7 @@ static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp)
 	struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1];
 	unsigned long flags;
 
-	atomic_inc(&qp->refcount);
+	rvt_get_qp(qp);
 	spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags);
 
 	if (qp->ibqp.qp_num <= 1) {
@@ -996,7 +1016,7 @@ static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp)
 }
 
 /**
- * qib_modify_qp - modify the attributes of a queue pair
+ * rvt_modify_qp - modify the attributes of a queue pair
  * @ibqp: the queue pair who's attributes we're modifying
  * @attr: the new attributes
  * @attr_mask: the mask of attributes to modify
diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 55f0e8f0ca79..ab6c3c25d7ff 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -358,18 +358,15 @@ static int __init rxe_module_init(void)
 	/* initialize slab caches for managed objects */
 	err = rxe_cache_init();
 	if (err) {
-		pr_err("rxe: unable to init object pools\n");
+		pr_err("unable to init object pools\n");
 		return err;
 	}
 
 	err = rxe_net_init();
-	if (err) {
-		pr_err("rxe: unable to init\n");
-		rxe_cache_exit();
+	if (err)
 		return err;
-	}
-	pr_info("rxe: loaded\n");
 
+	pr_info("loaded\n");
 	return 0;
 }
 
@@ -379,8 +376,8 @@ static void __exit rxe_module_exit(void)
 	rxe_net_exit();
 	rxe_cache_exit();
 
-	pr_info("rxe: unloaded\n");
+	pr_info("unloaded\n");
 }
 
-module_init(rxe_module_init);
+late_initcall(rxe_module_init);
 module_exit(rxe_module_exit);
diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h
index 12c71c549f97..a696af81e4a5 100644
--- a/drivers/infiniband/sw/rxe/rxe.h
+++ b/drivers/infiniband/sw/rxe/rxe.h
@@ -34,6 +34,11 @@
 #ifndef RXE_H
 #define RXE_H
 
+#ifdef pr_fmt
+#undef pr_fmt
+#endif
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/module.h>
 #include <linux/skbuff.h>
 #include <linux/crc32.h>
diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c
index 5c9474212d4e..604f6fee96bd 100644
--- a/drivers/infiniband/sw/rxe/rxe_av.c
+++ b/drivers/infiniband/sw/rxe/rxe_av.c
@@ -39,7 +39,7 @@ int rxe_av_chk_attr(struct rxe_dev *rxe, struct ib_ah_attr *attr)
 	struct rxe_port *port;
 
 	if (attr->port_num != 1) {
-		pr_info("rxe: invalid port_num = %d\n", attr->port_num);
+		pr_info("invalid port_num = %d\n", attr->port_num);
 		return -EINVAL;
 	}
 
@@ -47,7 +47,7 @@ int rxe_av_chk_attr(struct rxe_dev *rxe, struct ib_ah_attr *attr)
 
 	if (attr->ah_flags & IB_AH_GRH) {
 		if (attr->grh.sgid_index > port->attr.gid_tbl_len) {
-			pr_info("rxe: invalid sgid index = %d\n",
+			pr_info("invalid sgid index = %d\n",
 				attr->grh.sgid_index);
 			return -EINVAL;
 		}
diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c
index 36f67de44095..6c5e29db88e3 100644
--- a/drivers/infiniband/sw/rxe/rxe_comp.c
+++ b/drivers/infiniband/sw/rxe/rxe_comp.c
@@ -567,7 +567,8 @@ int rxe_completer(void *arg)
 	state = COMPST_GET_ACK;
 
 	while (1) {
-		pr_debug("state = %s\n", comp_state_name[state]);
+		pr_debug("qp#%d state = %s\n", qp_num(qp),
+			 comp_state_name[state]);
 		switch (state) {
 		case COMPST_GET_ACK:
 			skb = skb_dequeue(&qp->resp_pkts);
@@ -689,7 +690,14 @@ int rxe_completer(void *arg)
 					qp->req.need_retry = 1;
 					rxe_run_task(&qp->req.task, 1);
 				}
+
+				if (pkt) {
+					rxe_drop_ref(pkt->qp);
+					kfree_skb(skb);
+				}
+
 				goto exit;
+
 			} else {
 				wqe->status = IB_WC_RETRY_EXC_ERR;
 				state = COMPST_ERROR;
@@ -702,7 +710,8 @@ int rxe_completer(void *arg)
 					qp->comp.rnr_retry--;
 
 				qp->req.need_retry = 1;
-				pr_debug("set rnr nak timer\n");
+				pr_debug("qp#%d set rnr nak timer\n",
+					 qp_num(qp));
 				mod_timer(&qp->rnr_nak_timer,
 					  jiffies + rnrnak_jiffies(aeth_syn(pkt)
 						& ~AETH_TYPE_MASK));
@@ -716,6 +725,12 @@ int rxe_completer(void *arg)
 		case COMPST_ERROR:
 			do_complete(qp, wqe);
 			rxe_qp_error(qp);
+
+			if (pkt) {
+				rxe_drop_ref(pkt->qp);
+				kfree_skb(skb);
+			}
+
 			goto exit;
 		}
 	}
diff --git a/drivers/infiniband/sw/rxe/rxe_dma.c b/drivers/infiniband/sw/rxe/rxe_dma.c
index 7634c1a81b2b..a0f8af5851ae 100644
--- a/drivers/infiniband/sw/rxe/rxe_dma.c
+++ b/drivers/infiniband/sw/rxe/rxe_dma.c
@@ -117,6 +117,21 @@ static void rxe_unmap_sg(struct ib_device *dev,
 	WARN_ON(!valid_dma_direction(direction));
 }
 
+static int rxe_map_sg_attrs(struct ib_device *dev, struct scatterlist *sgl,
+			    int nents, enum dma_data_direction direction,
+			    unsigned long attrs)
+{
+	return rxe_map_sg(dev, sgl, nents, direction);
+}
+
+static void rxe_unmap_sg_attrs(struct ib_device *dev,
+			       struct scatterlist *sg, int nents,
+			       enum dma_data_direction direction,
+			       unsigned long attrs)
+{
+	rxe_unmap_sg(dev, sg, nents, direction);
+}
+
 static void rxe_sync_single_for_cpu(struct ib_device *dev,
 				    u64 addr,
 				    size_t size, enum dma_data_direction dir)
@@ -159,6 +174,8 @@ struct ib_dma_mapping_ops rxe_dma_mapping_ops = {
 	.unmap_page		= rxe_dma_unmap_page,
 	.map_sg			= rxe_map_sg,
 	.unmap_sg		= rxe_unmap_sg,
+	.map_sg_attrs		= rxe_map_sg_attrs,
+	.unmap_sg_attrs		= rxe_unmap_sg_attrs,
 	.sync_single_for_cpu	= rxe_sync_single_for_cpu,
 	.sync_single_for_device	= rxe_sync_single_for_device,
 	.alloc_coherent		= rxe_dma_alloc_coherent,
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 4a5484ef604f..73849a5a91b3 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -198,7 +198,7 @@ void free_rd_atomic_resource(struct rxe_qp *qp, struct resp_res *res);
 static inline void rxe_advance_resp_resource(struct rxe_qp *qp)
 {
 	qp->resp.res_head++;
-	if (unlikely(qp->resp.res_head == qp->attr.max_rd_atomic))
+	if (unlikely(qp->resp.res_head == qp->attr.max_dest_rd_atomic))
 		qp->resp.res_head = 0;
 }
 
diff --git a/drivers/infiniband/sw/rxe/rxe_mmap.c b/drivers/infiniband/sw/rxe/rxe_mmap.c
index 54b3c7c99eff..c572a4c09359 100644
--- a/drivers/infiniband/sw/rxe/rxe_mmap.c
+++ b/drivers/infiniband/sw/rxe/rxe_mmap.c
@@ -126,7 +126,7 @@ found_it:
 
 	ret = remap_vmalloc_range(vma, ip->obj, 0);
 	if (ret) {
-		pr_err("rxe: err %d from remap_vmalloc_range\n", ret);
+		pr_err("err %d from remap_vmalloc_range\n", ret);
 		goto done;
 	}
 
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index f3dab6574504..1869152f1d23 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -39,7 +39,7 @@
  */
 static u8 rxe_get_key(void)
 {
-	static unsigned key = 1;
+	static u32 key = 1;
 
 	key = key << 1;
 
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 0b8d2ea8b41d..b8258e4f0aea 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -65,7 +65,7 @@ struct rxe_dev *net_to_rxe(struct net_device *ndev)
 	return found;
 }
 
-struct rxe_dev *get_rxe_by_name(const char* name)
+struct rxe_dev *get_rxe_by_name(const char *name)
 {
 	struct rxe_dev *rxe;
 	struct rxe_dev *found = NULL;
@@ -275,9 +275,10 @@ static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port,
 	return sock;
 }
 
-static void rxe_release_udp_tunnel(struct socket *sk)
+void rxe_release_udp_tunnel(struct socket *sk)
 {
-	udp_tunnel_sock_release(sk);
+	if (sk)
+		udp_tunnel_sock_release(sk);
 }
 
 static void prepare_udp_hdr(struct sk_buff *skb, __be16 src_port,
@@ -349,14 +350,14 @@ static void prepare_ipv6_hdr(struct dst_entry *dst, struct sk_buff *skb,
 	ip6h->payload_len = htons(skb->len - sizeof(*ip6h));
 }
 
-static int prepare4(struct rxe_dev *rxe, struct sk_buff *skb, struct rxe_av *av)
+static int prepare4(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
+		    struct sk_buff *skb, struct rxe_av *av)
 {
 	struct dst_entry *dst;
 	bool xnet = false;
 	__be16 df = htons(IP_DF);
 	struct in_addr *saddr = &av->sgid_addr._sockaddr_in.sin_addr;
 	struct in_addr *daddr = &av->dgid_addr._sockaddr_in.sin_addr;
-	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
 
 	dst = rxe_find_route4(rxe->ndev, saddr, daddr);
 	if (!dst) {
@@ -375,12 +376,12 @@ static int prepare4(struct rxe_dev *rxe, struct sk_buff *skb, struct rxe_av *av)
 	return 0;
 }
 
-static int prepare6(struct rxe_dev *rxe, struct sk_buff *skb, struct rxe_av *av)
+static int prepare6(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
+		    struct sk_buff *skb, struct rxe_av *av)
 {
 	struct dst_entry *dst;
 	struct in6_addr *saddr = &av->sgid_addr._sockaddr_in6.sin6_addr;
 	struct in6_addr *daddr = &av->dgid_addr._sockaddr_in6.sin6_addr;
-	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
 
 	dst = rxe_find_route6(rxe->ndev, saddr, daddr);
 	if (!dst) {
@@ -407,9 +408,9 @@ static int prepare(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
 	struct rxe_av *av = rxe_get_av(pkt);
 
 	if (av->network_type == RDMA_NETWORK_IPV4)
-		err = prepare4(rxe, skb, av);
+		err = prepare4(rxe, pkt, skb, av);
 	else if (av->network_type == RDMA_NETWORK_IPV6)
-		err = prepare6(rxe, skb, av);
+		err = prepare6(rxe, pkt, skb, av);
 
 	*crc = rxe_icrc_hdr(pkt, skb);
 
@@ -600,8 +601,7 @@ void rxe_port_up(struct rxe_dev *rxe)
 	port->attr.phys_state = IB_PHYS_STATE_LINK_UP;
 
 	rxe_port_event(rxe, IB_EVENT_PORT_ACTIVE);
-	pr_info("rxe: set %s active\n", rxe->ib_dev.name);
-	return;
+	pr_info("set %s active\n", rxe->ib_dev.name);
 }
 
 /* Caller must hold net_info_lock */
@@ -614,8 +614,7 @@ void rxe_port_down(struct rxe_dev *rxe)
 	port->attr.phys_state = IB_PHYS_STATE_LINK_DOWN;
 
 	rxe_port_event(rxe, IB_EVENT_PORT_ERR);
-	pr_info("rxe: set %s down\n", rxe->ib_dev.name);
-	return;
+	pr_info("set %s down\n", rxe->ib_dev.name);
 }
 
 static int rxe_notify(struct notifier_block *not_blk,
@@ -640,7 +639,7 @@ static int rxe_notify(struct notifier_block *not_blk,
 		rxe_port_down(rxe);
 		break;
 	case NETDEV_CHANGEMTU:
-		pr_info("rxe: %s changed mtu to %d\n", ndev->name, ndev->mtu);
+		pr_info("%s changed mtu to %d\n", ndev->name, ndev->mtu);
 		rxe_set_mtu(rxe, ndev->mtu);
 		break;
 	case NETDEV_REBOOT:
@@ -650,7 +649,7 @@ static int rxe_notify(struct notifier_block *not_blk,
 	case NETDEV_CHANGENAME:
 	case NETDEV_FEAT_CHANGE:
 	default:
-		pr_info("rxe: ignoring netdev event = %ld for %s\n",
+		pr_info("ignoring netdev event = %ld for %s\n",
 			event, ndev->name);
 		break;
 	}
@@ -658,51 +657,68 @@ out:
 	return NOTIFY_OK;
 }
 
-static struct notifier_block rxe_net_notifier = {
+struct notifier_block rxe_net_notifier = {
 	.notifier_call = rxe_notify,
 };
 
-int rxe_net_init(void)
+int rxe_net_ipv4_init(void)
 {
-	int err;
-
 	spin_lock_init(&dev_list_lock);
 
-	recv_sockets.sk6 = rxe_setup_udp_tunnel(&init_net,
-			htons(ROCE_V2_UDP_DPORT), true);
-	if (IS_ERR(recv_sockets.sk6)) {
-		recv_sockets.sk6 = NULL;
-		pr_err("rxe: Failed to create IPv6 UDP tunnel\n");
-		return -1;
-	}
-
 	recv_sockets.sk4 = rxe_setup_udp_tunnel(&init_net,
-			htons(ROCE_V2_UDP_DPORT), false);
+				htons(ROCE_V2_UDP_DPORT), false);
 	if (IS_ERR(recv_sockets.sk4)) {
-		rxe_release_udp_tunnel(recv_sockets.sk6);
 		recv_sockets.sk4 = NULL;
-		recv_sockets.sk6 = NULL;
-		pr_err("rxe: Failed to create IPv4 UDP tunnel\n");
+		pr_err("Failed to create IPv4 UDP tunnel\n");
 		return -1;
 	}
 
-	err = register_netdevice_notifier(&rxe_net_notifier);
-	if (err) {
-		rxe_release_udp_tunnel(recv_sockets.sk6);
-		rxe_release_udp_tunnel(recv_sockets.sk4);
-		pr_err("rxe: Failed to rigister netdev notifier\n");
-	}
+	return 0;
+}
 
-	return err;
+int rxe_net_ipv6_init(void)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+
+	spin_lock_init(&dev_list_lock);
+
+	recv_sockets.sk6 = rxe_setup_udp_tunnel(&init_net,
+						htons(ROCE_V2_UDP_DPORT), true);
+	if (IS_ERR(recv_sockets.sk6)) {
+		recv_sockets.sk6 = NULL;
+		pr_err("Failed to create IPv6 UDP tunnel\n");
+		return -1;
+	}
+#endif
+	return 0;
 }
 
 void rxe_net_exit(void)
 {
-	if (recv_sockets.sk6)
-		rxe_release_udp_tunnel(recv_sockets.sk6);
+	rxe_release_udp_tunnel(recv_sockets.sk6);
+	rxe_release_udp_tunnel(recv_sockets.sk4);
+	unregister_netdevice_notifier(&rxe_net_notifier);
+}
 
-	if (recv_sockets.sk4)
-		rxe_release_udp_tunnel(recv_sockets.sk4);
+int rxe_net_init(void)
+{
+	int err;
 
-	unregister_netdevice_notifier(&rxe_net_notifier);
+	recv_sockets.sk6 = NULL;
+
+	err = rxe_net_ipv4_init();
+	if (err)
+		return err;
+	err = rxe_net_ipv6_init();
+	if (err)
+		goto err_out;
+	err = register_netdevice_notifier(&rxe_net_notifier);
+	if (err) {
+		pr_err("Failed to register netdev notifier\n");
+		goto err_out;
+	}
+	return 0;
+err_out:
+	rxe_net_exit();
+	return err;
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_net.h b/drivers/infiniband/sw/rxe/rxe_net.h
index 7b06f76d16cc..1c06b3bfe1b6 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.h
+++ b/drivers/infiniband/sw/rxe/rxe_net.h
@@ -44,6 +44,8 @@ struct rxe_recv_sockets {
 };
 
 extern struct rxe_recv_sockets recv_sockets;
+extern struct notifier_block rxe_net_notifier;
+void rxe_release_udp_tunnel(struct socket *sk);
 
 struct rxe_dev *rxe_net_add(struct net_device *ndev);
 
diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index 22ba24f2a2c1..b8036cfbce04 100644
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -146,7 +146,7 @@ static void free_rd_atomic_resources(struct rxe_qp *qp)
 	if (qp->resp.resources) {
 		int i;
 
-		for (i = 0; i < qp->attr.max_rd_atomic; i++) {
+		for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {
 			struct resp_res *res = &qp->resp.resources[i];
 
 			free_rd_atomic_resource(qp, res);
@@ -174,7 +174,7 @@ static void cleanup_rd_atomic_resources(struct rxe_qp *qp)
 	struct resp_res *res;
 
 	if (qp->resp.resources) {
-		for (i = 0; i < qp->attr.max_rd_atomic; i++) {
+		for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {
 			res = &qp->resp.resources[i];
 			free_rd_atomic_resource(qp, res);
 		}
@@ -298,8 +298,8 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,
 
 		wqe_size = rcv_wqe_size(qp->rq.max_sge);
 
-		pr_debug("max_wr = %d, max_sge = %d, wqe_size = %d\n",
-			 qp->rq.max_wr, qp->rq.max_sge, wqe_size);
+		pr_debug("qp#%d max_wr = %d, max_sge = %d, wqe_size = %d\n",
+			 qp_num(qp), qp->rq.max_wr, qp->rq.max_sge, wqe_size);
 
 		qp->rq.queue = rxe_queue_init(rxe,
 					      &qp->rq.max_wr,
@@ -596,14 +596,21 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
 	if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
 		int max_rd_atomic = __roundup_pow_of_two(attr->max_rd_atomic);
 
+		qp->attr.max_rd_atomic = max_rd_atomic;
+		atomic_set(&qp->req.rd_atomic, max_rd_atomic);
+	}
+
+	if (mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+		int max_dest_rd_atomic =
+			__roundup_pow_of_two(attr->max_dest_rd_atomic);
+
+		qp->attr.max_dest_rd_atomic = max_dest_rd_atomic;
+
 		free_rd_atomic_resources(qp);
 
-		err = alloc_rd_atomic_resources(qp, max_rd_atomic);
+		err = alloc_rd_atomic_resources(qp, max_dest_rd_atomic);
 		if (err)
 			return err;
-
-		qp->attr.max_rd_atomic = max_rd_atomic;
-		atomic_set(&qp->req.rd_atomic, max_rd_atomic);
 	}
 
 	if (mask & IB_QP_CUR_STATE)
@@ -673,24 +680,27 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
 	if (mask & IB_QP_RETRY_CNT) {
 		qp->attr.retry_cnt = attr->retry_cnt;
 		qp->comp.retry_cnt = attr->retry_cnt;
-		pr_debug("set retry count = %d\n", attr->retry_cnt);
+		pr_debug("qp#%d set retry count = %d\n", qp_num(qp),
+			 attr->retry_cnt);
 	}
 
 	if (mask & IB_QP_RNR_RETRY) {
 		qp->attr.rnr_retry = attr->rnr_retry;
 		qp->comp.rnr_retry = attr->rnr_retry;
-		pr_debug("set rnr retry count = %d\n", attr->rnr_retry);
+		pr_debug("qp#%d set rnr retry count = %d\n", qp_num(qp),
+			 attr->rnr_retry);
 	}
 
 	if (mask & IB_QP_RQ_PSN) {
 		qp->attr.rq_psn = (attr->rq_psn & BTH_PSN_MASK);
 		qp->resp.psn = qp->attr.rq_psn;
-		pr_debug("set resp psn = 0x%x\n", qp->resp.psn);
+		pr_debug("qp#%d set resp psn = 0x%x\n", qp_num(qp),
+			 qp->resp.psn);
 	}
 
 	if (mask & IB_QP_MIN_RNR_TIMER) {
 		qp->attr.min_rnr_timer = attr->min_rnr_timer;
-		pr_debug("set min rnr timer = 0x%x\n",
+		pr_debug("qp#%d set min rnr timer = 0x%x\n", qp_num(qp),
 			 attr->min_rnr_timer);
 	}
 
@@ -698,12 +708,7 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
 		qp->attr.sq_psn = (attr->sq_psn & BTH_PSN_MASK);
 		qp->req.psn = qp->attr.sq_psn;
 		qp->comp.psn = qp->attr.sq_psn;
-		pr_debug("set req psn = 0x%x\n", qp->req.psn);
-	}
-
-	if (mask & IB_QP_MAX_DEST_RD_ATOMIC) {
-		qp->attr.max_dest_rd_atomic =
-			__roundup_pow_of_two(attr->max_dest_rd_atomic);
+		pr_debug("qp#%d set req psn = 0x%x\n", qp_num(qp), qp->req.psn);
 	}
 
 	if (mask & IB_QP_PATH_MIG_STATE)
@@ -717,38 +722,38 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
 
 		switch (attr->qp_state) {
 		case IB_QPS_RESET:
-			pr_debug("qp state -> RESET\n");
+			pr_debug("qp#%d state -> RESET\n", qp_num(qp));
 			rxe_qp_reset(qp);
 			break;
 
 		case IB_QPS_INIT:
-			pr_debug("qp state -> INIT\n");
+			pr_debug("qp#%d state -> INIT\n", qp_num(qp));
 			qp->req.state = QP_STATE_INIT;
 			qp->resp.state = QP_STATE_INIT;
 			break;
 
 		case IB_QPS_RTR:
-			pr_debug("qp state -> RTR\n");
+			pr_debug("qp#%d state -> RTR\n", qp_num(qp));
 			qp->resp.state = QP_STATE_READY;
 			break;
 
 		case IB_QPS_RTS:
-			pr_debug("qp state -> RTS\n");
+			pr_debug("qp#%d state -> RTS\n", qp_num(qp));
 			qp->req.state = QP_STATE_READY;
 			break;
 
 		case IB_QPS_SQD:
-			pr_debug("qp state -> SQD\n");
+			pr_debug("qp#%d state -> SQD\n", qp_num(qp));
 			rxe_qp_drain(qp);
 			break;
 
 		case IB_QPS_SQE:
-			pr_warn("qp state -> SQE !!?\n");
+			pr_warn("qp#%d state -> SQE !!?\n", qp_num(qp));
 			/* Not possible from modify_qp. */
 			break;
 
 		case IB_QPS_ERR:
-			pr_debug("qp state -> ERR\n");
+			pr_debug("qp#%d state -> ERR\n", qp_num(qp));
 			rxe_qp_error(qp);
 			break;
 		}
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index 3d464c23e08b..46f062842a9a 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -312,7 +312,7 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)
 		 * make a copy of the skb to post to the next qp
 		 */
 		skb_copy = (mce->qp_list.next != &mcg->qp_list) ?
-				skb_clone(skb, GFP_KERNEL) : NULL;
+				skb_clone(skb, GFP_ATOMIC) : NULL;
 
 		pkt->qp = qp;
 		rxe_add_ref(qp);
@@ -387,7 +387,8 @@ int rxe_rcv(struct sk_buff *skb)
 	pack_icrc = be32_to_cpu(*icrcp);
 
 	calc_icrc = rxe_icrc_hdr(pkt, skb);
-	calc_icrc = crc32_le(calc_icrc, (u8 *)payload_addr(pkt), payload_size(pkt));
+	calc_icrc = crc32_le(calc_icrc, (u8 *)payload_addr(pkt),
+			     payload_size(pkt));
 	calc_icrc = cpu_to_be32(~calc_icrc);
 	if (unlikely(calc_icrc != pack_icrc)) {
 		char saddr[sizeof(struct in6_addr)];
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 33b2d9d77021..832846b73ea0 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -38,7 +38,7 @@
 #include "rxe_queue.h"
 
 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
-		       unsigned opcode);
+		       u32 opcode);
 
 static inline void retry_first_write_send(struct rxe_qp *qp,
 					  struct rxe_send_wqe *wqe,
@@ -121,7 +121,7 @@ void rnr_nak_timer(unsigned long data)
 {
 	struct rxe_qp *qp = (struct rxe_qp *)data;
 
-	pr_debug("rnr nak timer fired\n");
+	pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
 	rxe_run_task(&qp->req.task, 1);
 }
 
@@ -187,7 +187,7 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
 	return wqe;
 }
 
-static int next_opcode_rc(struct rxe_qp *qp, unsigned opcode, int fits)
+static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits)
 {
 	switch (opcode) {
 	case IB_WR_RDMA_WRITE:
@@ -259,7 +259,7 @@ static int next_opcode_rc(struct rxe_qp *qp, unsigned opcode, int fits)
 	return -EINVAL;
 }
 
-static int next_opcode_uc(struct rxe_qp *qp, unsigned opcode, int fits)
+static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits)
 {
 	switch (opcode) {
 	case IB_WR_RDMA_WRITE:
@@ -311,7 +311,7 @@ static int next_opcode_uc(struct rxe_qp *qp, unsigned opcode, int fits)
 }
 
 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
-		       unsigned opcode)
+		       u32 opcode)
 {
 	int fits = (wqe->dma.resid <= qp->mtu);
 
@@ -511,24 +511,21 @@ static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 }
 
 static void update_wqe_state(struct rxe_qp *qp,
-			     struct rxe_send_wqe *wqe,
-			     struct rxe_pkt_info *pkt,
-			     enum wqe_state *prev_state)
+		struct rxe_send_wqe *wqe,
+		struct rxe_pkt_info *pkt)
 {
-	enum wqe_state prev_state_ = wqe->state;
-
 	if (pkt->mask & RXE_END_MASK) {
 		if (qp_type(qp) == IB_QPT_RC)
 			wqe->state = wqe_state_pending;
 	} else {
 		wqe->state = wqe_state_processing;
 	}
-
-	*prev_state = prev_state_;
 }
 
-static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
-			 struct rxe_pkt_info *pkt, int payload)
+static void update_wqe_psn(struct rxe_qp *qp,
+			   struct rxe_send_wqe *wqe,
+			   struct rxe_pkt_info *pkt,
+			   int payload)
 {
 	/* number of packets left to send including current one */
 	int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu;
@@ -546,9 +543,34 @@ static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 		qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK;
 	else
 		qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
+}
 
-	qp->req.opcode = pkt->opcode;
+static void save_state(struct rxe_send_wqe *wqe,
+		       struct rxe_qp *qp,
+		       struct rxe_send_wqe *rollback_wqe,
+		       struct rxe_qp *rollback_qp)
+{
+	rollback_wqe->state     = wqe->state;
+	rollback_wqe->first_psn = wqe->first_psn;
+	rollback_wqe->last_psn  = wqe->last_psn;
+	rollback_qp->req.psn    = qp->req.psn;
+}
 
+static void rollback_state(struct rxe_send_wqe *wqe,
+			   struct rxe_qp *qp,
+			   struct rxe_send_wqe *rollback_wqe,
+			   struct rxe_qp *rollback_qp)
+{
+	wqe->state     = rollback_wqe->state;
+	wqe->first_psn = rollback_wqe->first_psn;
+	wqe->last_psn  = rollback_wqe->last_psn;
+	qp->req.psn    = rollback_qp->req.psn;
+}
+
+static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
+			 struct rxe_pkt_info *pkt, int payload)
+{
+	qp->req.opcode = pkt->opcode;
 
 	if (pkt->mask & RXE_END_MASK)
 		qp->req.wqe_index = next_index(qp->sq.queue, qp->req.wqe_index);
@@ -566,12 +588,13 @@ int rxe_requester(void *arg)
 	struct rxe_pkt_info pkt;
 	struct sk_buff *skb;
 	struct rxe_send_wqe *wqe;
-	unsigned mask;
+	enum rxe_hdr_mask mask;
 	int payload;
 	int mtu;
 	int opcode;
 	int ret;
-	enum wqe_state prev_state;
+	struct rxe_qp rollback_qp;
+	struct rxe_send_wqe rollback_wqe;
 
 next_wqe:
 	if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR))
@@ -603,7 +626,8 @@ next_wqe:
 			rmr = rxe_pool_get_index(&rxe->mr_pool,
 						 wqe->wr.ex.invalidate_rkey >> 8);
 			if (!rmr) {
-				pr_err("No mr for key %#x\n", wqe->wr.ex.invalidate_rkey);
+				pr_err("No mr for key %#x\n",
+				       wqe->wr.ex.invalidate_rkey);
 				wqe->state = wqe_state_error;
 				wqe->status = IB_WC_MW_BIND_ERR;
 				goto exit;
@@ -679,22 +703,30 @@ next_wqe:
 
 	skb = init_req_packet(qp, wqe, opcode, payload, &pkt);
 	if (unlikely(!skb)) {
-		pr_err("Failed allocating skb\n");
+		pr_err("qp#%d Failed allocating skb\n", qp_num(qp));
 		goto err;
 	}
 
 	if (fill_packet(qp, wqe, &pkt, skb, payload)) {
-		pr_debug("Error during fill packet\n");
+		pr_debug("qp#%d Error during fill packet\n", qp_num(qp));
 		goto err;
 	}
 
-	update_wqe_state(qp, wqe, &pkt, &prev_state);
+	/*
+	 * To prevent a race on wqe access between requester and completer,
+	 * wqe members state and psn need to be set before calling
+	 * rxe_xmit_packet().
+	 * Otherwise, completer might initiate an unjustified retry flow.
+	 */
+	save_state(wqe, qp, &rollback_wqe, &rollback_qp);
+	update_wqe_state(qp, wqe, &pkt);
+	update_wqe_psn(qp, wqe, &pkt, payload);
 	ret = rxe_xmit_packet(to_rdev(qp->ibqp.device), qp, &pkt, skb);
 	if (ret) {
 		qp->need_req_skb = 1;
 		kfree_skb(skb);
 
-		wqe->state = prev_state;
+		rollback_state(wqe, qp, &rollback_wqe, &rollback_qp);
 
 		if (ret == -EAGAIN) {
 			rxe_run_task(&qp->req.task, 1);
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index ebb03b46e2ad..dd3d88adc003 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -383,7 +383,7 @@ static enum resp_states check_resource(struct rxe_qp *qp,
 		 * too many read/atomic ops, we just
 		 * recycle the responder resource queue
 		 */
-		if (likely(qp->attr.max_rd_atomic > 0))
+		if (likely(qp->attr.max_dest_rd_atomic > 0))
 			return RESPST_CHK_LENGTH;
 		else
 			return RESPST_ERR_TOO_MANY_RDMA_ATM_REQ;
@@ -749,6 +749,18 @@ static enum resp_states read_reply(struct rxe_qp *qp,
 	return state;
 }
 
+static void build_rdma_network_hdr(union rdma_network_hdr *hdr,
+				   struct rxe_pkt_info *pkt)
+{
+	struct sk_buff *skb = PKT_TO_SKB(pkt);
+
+	memset(hdr, 0, sizeof(*hdr));
+	if (skb->protocol == htons(ETH_P_IP))
+		memcpy(&hdr->roce4grh, ip_hdr(skb), sizeof(hdr->roce4grh));
+	else if (skb->protocol == htons(ETH_P_IPV6))
+		memcpy(&hdr->ibgrh, ipv6_hdr(skb), sizeof(hdr->ibgrh));
+}
+
 /* Executes a new request. A retried request never reach that function (send
  * and writes are discarded, and reads and atomics are retried elsewhere.
  */
@@ -761,13 +773,8 @@ static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
 		    qp_type(qp) == IB_QPT_SMI ||
 		    qp_type(qp) == IB_QPT_GSI) {
 			union rdma_network_hdr hdr;
-			struct sk_buff *skb = PKT_TO_SKB(pkt);
 
-			memset(&hdr, 0, sizeof(hdr));
-			if (skb->protocol == htons(ETH_P_IP))
-				memcpy(&hdr.roce4grh, ip_hdr(skb), sizeof(hdr.roce4grh));
-			else if (skb->protocol == htons(ETH_P_IPV6))
-				memcpy(&hdr.ibgrh, ipv6_hdr(skb), sizeof(hdr.ibgrh));
+			build_rdma_network_hdr(&hdr, pkt);
 
 			err = send_data_in(qp, &hdr, sizeof(hdr));
 			if (err)
@@ -881,7 +888,8 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 				rmr = rxe_pool_get_index(&rxe->mr_pool,
 							 wc->ex.invalidate_rkey >> 8);
 				if (unlikely(!rmr)) {
-					pr_err("Bad rkey %#x invalidation\n", wc->ex.invalidate_rkey);
+					pr_err("Bad rkey %#x invalidation\n",
+					       wc->ex.invalidate_rkey);
 					return RESPST_ERROR;
 				}
 				rmr->state = RXE_MEM_STATE_FREE;
@@ -972,11 +980,13 @@ static int send_atomic_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
 	free_rd_atomic_resource(qp, res);
 	rxe_advance_resp_resource(qp);
 
+	memcpy(SKB_TO_PKT(skb), &ack_pkt, sizeof(skb->cb));
+
 	res->type = RXE_ATOMIC_MASK;
 	res->atomic.skb = skb;
-	res->first_psn = qp->resp.psn;
-	res->last_psn = qp->resp.psn;
-	res->cur_psn = qp->resp.psn;
+	res->first_psn = ack_pkt.psn;
+	res->last_psn  = ack_pkt.psn;
+	res->cur_psn   = ack_pkt.psn;
 
 	rc = rxe_xmit_packet(rxe, qp, &ack_pkt, skb_copy);
 	if (rc) {
@@ -1116,8 +1126,7 @@ static enum resp_states duplicate_request(struct rxe_qp *qp,
 				rc = RESPST_CLEANUP;
 				goto out;
 			}
-			bth_set_psn(SKB_TO_PKT(skb_copy),
-				    qp->resp.psn - 1);
+
 			/* Resend the result. */
 			rc = rxe_xmit_packet(to_rdev(qp->ibqp.device), qp,
 					     pkt, skb_copy);
@@ -1207,7 +1216,8 @@ int rxe_responder(void *arg)
 	}
 
 	while (1) {
-		pr_debug("state = %s\n", resp_state_name[state]);
+		pr_debug("qp#%d state = %s\n", qp_num(qp),
+			 resp_state_name[state]);
 		switch (state) {
 		case RESPST_GET_REQ:
 			state = get_req(qp, &pkt);
diff --git a/drivers/infiniband/sw/rxe/rxe_sysfs.c b/drivers/infiniband/sw/rxe/rxe_sysfs.c
index cf8e77800046..d5ed7571128f 100644
--- a/drivers/infiniband/sw/rxe/rxe_sysfs.c
+++ b/drivers/infiniband/sw/rxe/rxe_sysfs.c
@@ -79,7 +79,7 @@ static int rxe_param_set_add(const char *val, const struct kernel_param *kp)
 
 	len = sanitize_arg(val, intf, sizeof(intf));
 	if (!len) {
-		pr_err("rxe: add: invalid interface name\n");
+		pr_err("add: invalid interface name\n");
 		err = -EINVAL;
 		goto err;
 	}
@@ -92,20 +92,20 @@ static int rxe_param_set_add(const char *val, const struct kernel_param *kp)
 	}
 
 	if (net_to_rxe(ndev)) {
-		pr_err("rxe: already configured on %s\n", intf);
+		pr_err("already configured on %s\n", intf);
 		err = -EINVAL;
 		goto err;
 	}
 
 	rxe = rxe_net_add(ndev);
 	if (!rxe) {
-		pr_err("rxe: failed to add %s\n", intf);
+		pr_err("failed to add %s\n", intf);
 		err = -EINVAL;
 		goto err;
 	}
 
 	rxe_set_port_state(ndev);
-	pr_info("rxe: added %s to %s\n", rxe->ib_dev.name, intf);
+	pr_info("added %s to %s\n", rxe->ib_dev.name, intf);
 err:
 	if (ndev)
 		dev_put(ndev);
@@ -120,7 +120,7 @@ static int rxe_param_set_remove(const char *val, const struct kernel_param *kp)
 
 	len = sanitize_arg(val, intf, sizeof(intf));
 	if (!len) {
-		pr_err("rxe: add: invalid interface name\n");
+		pr_err("add: invalid interface name\n");
 		return -EINVAL;
 	}
 
@@ -133,7 +133,7 @@ static int rxe_param_set_remove(const char *val, const struct kernel_param *kp)
 	rxe = get_rxe_by_name(intf);
 
 	if (!rxe) {
-		pr_err("rxe: not configured on %s\n", intf);
+		pr_err("not configured on %s\n", intf);
 		return -EINVAL;
 	}
 
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 4552be960c6a..19841c863daf 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -100,10 +100,12 @@ static int rxe_query_port(struct ib_device *dev,
 		rxe->ndev->ethtool_ops->get_settings(rxe->ndev, &cmd);
 		speed = cmd.speed;
 	} else {
-		pr_warn("%s speed is unknown, defaulting to 1000\n", rxe->ndev->name);
+		pr_warn("%s speed is unknown, defaulting to 1000\n",
+			rxe->ndev->name);
 		speed = 1000;
 	}
-	rxe_eth_speed_to_ib_speed(speed, &attr->active_speed, &attr->active_width);
+	rxe_eth_speed_to_ib_speed(speed, &attr->active_speed,
+				  &attr->active_width);
 	mutex_unlock(&rxe->usdev_lock);
 
 	return 0;
@@ -761,7 +763,7 @@ static int init_send_wqe(struct rxe_qp *qp, struct ib_send_wr *ibwr,
 }
 
 static int post_one_send(struct rxe_qp *qp, struct ib_send_wr *ibwr,
-			 unsigned mask, u32 length)
+			 unsigned int mask, u32 length)
 {
 	int err;
 	struct rxe_sq *sq = &qp->sq;
@@ -801,26 +803,15 @@ err1:
 	return err;
 }
 
-static int rxe_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
-			 struct ib_send_wr **bad_wr)
+static int rxe_post_send_kernel(struct rxe_qp *qp, struct ib_send_wr *wr,
+				struct ib_send_wr **bad_wr)
 {
 	int err = 0;
-	struct rxe_qp *qp = to_rqp(ibqp);
 	unsigned int mask;
 	unsigned int length = 0;
 	int i;
 	int must_sched;
 
-	if (unlikely(!qp->valid)) {
-		*bad_wr = wr;
-		return -EINVAL;
-	}
-
-	if (unlikely(qp->req.state < QP_STATE_READY)) {
-		*bad_wr = wr;
-		return -EINVAL;
-	}
-
 	while (wr) {
 		mask = wr_opcode_mask(wr->opcode, qp);
 		if (unlikely(!mask)) {
@@ -861,6 +852,29 @@ static int rxe_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 	return err;
 }
 
+static int rxe_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
+			 struct ib_send_wr **bad_wr)
+{
+	struct rxe_qp *qp = to_rqp(ibqp);
+
+	if (unlikely(!qp->valid)) {
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	if (unlikely(qp->req.state < QP_STATE_READY)) {
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	if (qp->is_user) {
+		/* Utilize process context to do protocol processing */
+		rxe_run_task(&qp->req.task, 0);
+		return 0;
+	} else
+		return rxe_post_send_kernel(qp, wr, bad_wr);
+}
+
 static int rxe_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 			 struct ib_recv_wr **bad_wr)
 {
@@ -1133,8 +1147,8 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
 	return 0;
 }
 
-static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
-			 unsigned int *sg_offset)
+static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+			 int sg_nents, unsigned int *sg_offset)
 {
 	struct rxe_mem *mr = to_rmr(ibmr);
 	int n;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index 4f7d9b48df64..7b8d2d9e2263 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -478,6 +478,7 @@ void ipoib_send(struct net_device *dev, struct sk_buff *skb,
 		struct ipoib_ah *address, u32 qpn);
 void ipoib_reap_ah(struct work_struct *work);
 
+struct ipoib_path *__path_find(struct net_device *dev, void *gid);
 void ipoib_mark_paths_invalid(struct net_device *dev);
 void ipoib_flush_paths(struct net_device *dev);
 int ipoib_check_sm_sendonly_fullmember_support(struct ipoib_dev_priv *priv);
@@ -771,7 +772,13 @@ static inline void ipoib_unregister_debugfs(void) { }
 #define ipoib_printk(level, priv, format, arg...)	\
 	printk(level "%s: " format, ((struct ipoib_dev_priv *) priv)->dev->name , ## arg)
 #define ipoib_warn(priv, format, arg...)		\
-	ipoib_printk(KERN_WARNING, priv, format , ## arg)
+do {							\
+	static DEFINE_RATELIMIT_STATE(_rs,		\
+		10 * HZ /*10 seconds */,		\
+		100);		\
+	if (__ratelimit(&_rs))				\
+		ipoib_printk(KERN_WARNING, priv, format , ## arg);\
+} while (0)
 
 extern int ipoib_sendq_size;
 extern int ipoib_recvq_size;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index 951d9abcca8b..4ad297d3de89 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -1318,6 +1318,8 @@ void ipoib_cm_destroy_tx(struct ipoib_cm_tx *tx)
 	}
 }
 
+#define QPN_AND_OPTIONS_OFFSET	4
+
 static void ipoib_cm_tx_start(struct work_struct *work)
 {
 	struct ipoib_dev_priv *priv = container_of(work, struct ipoib_dev_priv,
@@ -1326,6 +1328,7 @@ static void ipoib_cm_tx_start(struct work_struct *work)
 	struct ipoib_neigh *neigh;
 	struct ipoib_cm_tx *p;
 	unsigned long flags;
+	struct ipoib_path *path;
 	int ret;
 
 	struct ib_sa_path_rec pathrec;
@@ -1338,7 +1341,19 @@ static void ipoib_cm_tx_start(struct work_struct *work)
 		p = list_entry(priv->cm.start_list.next, typeof(*p), list);
 		list_del_init(&p->list);
 		neigh = p->neigh;
+
 		qpn = IPOIB_QPN(neigh->daddr);
+		/*
+		 * As long as the search is with these 2 locks,
+		 * path existence indicates its validity.
+		 */
+		path = __path_find(dev, neigh->daddr + QPN_AND_OPTIONS_OFFSET);
+		if (!path) {
+			pr_info("%s ignore not valid path %pI6\n",
+				__func__,
+				neigh->daddr + QPN_AND_OPTIONS_OFFSET);
+			goto free_neigh;
+		}
 		memcpy(&pathrec, &p->path->pathrec, sizeof pathrec);
 
 		spin_unlock_irqrestore(&priv->lock, flags);
@@ -1350,6 +1365,7 @@ static void ipoib_cm_tx_start(struct work_struct *work)
 		spin_lock_irqsave(&priv->lock, flags);
 
 		if (ret) {
+free_neigh:
 			neigh = p->neigh;
 			if (neigh) {
 				neigh->cm = NULL;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index dc6d241b9406..be11d5d5b8c1 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -1161,8 +1161,17 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv,
 	}
 
 	if (level == IPOIB_FLUSH_LIGHT) {
+		int oper_up;
 		ipoib_mark_paths_invalid(dev);
+		/* Set IPoIB operation as down to prevent races between:
+		 * the flush flow which leaves MCG and on the fly joins
+		 * which can happen during that time. mcast restart task
+		 * should deal with join requests we missed.
+		 */
+		oper_up = test_and_clear_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
 		ipoib_mcast_dev_flush(dev);
+		if (oper_up)
+			set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
 		ipoib_flush_ah(dev);
 	}
 
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 74bcaa064226..5636fc3da6b8 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -485,7 +485,7 @@ int ipoib_set_mode(struct net_device *dev, const char *buf)
 	return -EINVAL;
 }
 
-static struct ipoib_path *__path_find(struct net_device *dev, void *gid)
+struct ipoib_path *__path_find(struct net_device *dev, void *gid)
 {
 	struct ipoib_dev_priv *priv = netdev_priv(dev);
 	struct rb_node *n = priv->path_tree.rb_node;
@@ -2196,7 +2196,8 @@ static int __init ipoib_init_module(void)
 	 * its private workqueue, and we only queue up flush events
 	 * on our global flush workqueue.  This avoids the deadlocks.
 	 */
-	ipoib_workqueue = create_singlethread_workqueue("ipoib_flush");
+	ipoib_workqueue = alloc_ordered_workqueue("ipoib_flush",
+						  WQ_MEM_RECLAIM);
 	if (!ipoib_workqueue) {
 		ret = -ENOMEM;
 		goto err_fs;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index c55ecb2c3736..189dcd1709d2 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -147,7 +147,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
 	int ret, size;
 	int i;
 
-	priv->pd = ib_alloc_pd(priv->ca);
+	priv->pd = ib_alloc_pd(priv->ca, 0);
 	if (IS_ERR(priv->pd)) {
 		printk(KERN_WARNING "%s: failed to allocate PD\n", ca->name);
 		return -ENODEV;
@@ -157,7 +157,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
 	 * the various IPoIB tasks assume they will never race against
 	 * themselves, so always use a single thread workqueue
 	 */
-	priv->wq = create_singlethread_workqueue("ipoib_wq");
+	priv->wq = alloc_ordered_workqueue("ipoib_wq", WQ_MEM_RECLAIM);
 	if (!priv->wq) {
 		printk(KERN_WARNING "ipoib: failed to allocate device WQ\n");
 		goto out_free_pd;
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 0351059783b1..0be6a7c5ddb5 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -374,7 +374,6 @@ struct iser_reg_ops {
 struct iser_device {
 	struct ib_device             *ib_device;
 	struct ib_pd	             *pd;
-	struct ib_mr	             *mr;
 	struct ib_event_handler      event_handler;
 	struct list_head             ig_list;
 	int                          refcount;
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index 90be56893414..9c3e9ab53a41 100644
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -199,7 +199,11 @@ iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,
 	 * FIXME: rework the registration code path to differentiate
 	 * rkey/lkey use cases
 	 */
-	reg->rkey = device->mr ? device->mr->rkey : 0;
+
+	if (device->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)
+		reg->rkey = device->pd->unsafe_global_rkey;
+	else
+		reg->rkey = 0;
 	reg->sge.addr = ib_sg_dma_address(device->ib_device, &sg[0]);
 	reg->sge.length = ib_sg_dma_len(device->ib_device, &sg[0]);
 
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 1b4945367e4f..a4b791dfaa1d 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -88,7 +88,8 @@ static int iser_create_device_ib_res(struct iser_device *device)
 		  device->comps_used, ib_dev->name,
 		  ib_dev->num_comp_vectors, max_cqe);
 
-	device->pd = ib_alloc_pd(ib_dev);
+	device->pd = ib_alloc_pd(ib_dev,
+		iser_always_reg ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
 	if (IS_ERR(device->pd))
 		goto pd_err;
 
@@ -103,26 +104,13 @@ static int iser_create_device_ib_res(struct iser_device *device)
 		}
 	}
 
-	if (!iser_always_reg) {
-		int access = IB_ACCESS_LOCAL_WRITE |
-			     IB_ACCESS_REMOTE_WRITE |
-			     IB_ACCESS_REMOTE_READ;
-
-		device->mr = ib_get_dma_mr(device->pd, access);
-		if (IS_ERR(device->mr))
-			goto cq_err;
-	}
-
 	INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev,
 			      iser_event_handler);
 	if (ib_register_event_handler(&device->event_handler))
-		goto handler_err;
+		goto cq_err;
 
 	return 0;
 
-handler_err:
-	if (device->mr)
-		ib_dereg_mr(device->mr);
 cq_err:
 	for (i = 0; i < device->comps_used; i++) {
 		struct iser_comp *comp = &device->comps[i];
@@ -154,14 +142,10 @@ static void iser_free_device_ib_res(struct iser_device *device)
 	}
 
 	(void)ib_unregister_event_handler(&device->event_handler);
-	if (device->mr)
-		(void)ib_dereg_mr(device->mr);
 	ib_dealloc_pd(device->pd);
 
 	kfree(device->comps);
 	device->comps = NULL;
-
-	device->mr = NULL;
 	device->pd = NULL;
 }
 
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index ba6be060a476..6dd43f63238e 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -309,7 +309,7 @@ isert_create_device_ib_res(struct isert_device *device)
 	if (ret)
 		goto out;
 
-	device->pd = ib_alloc_pd(ib_dev);
+	device->pd = ib_alloc_pd(ib_dev, 0);
 	if (IS_ERR(device->pd)) {
 		ret = PTR_ERR(device->pd);
 		isert_err("failed to allocate pd, device %p, ret=%d\n",
@@ -403,6 +403,7 @@ isert_init_conn(struct isert_conn *isert_conn)
 	INIT_LIST_HEAD(&isert_conn->node);
 	init_completion(&isert_conn->login_comp);
 	init_completion(&isert_conn->login_req_comp);
+	init_waitqueue_head(&isert_conn->rem_wait);
 	kref_init(&isert_conn->kref);
 	mutex_init(&isert_conn->mutex);
 	INIT_WORK(&isert_conn->release_work, isert_release_work);
@@ -448,7 +449,7 @@ isert_alloc_login_buf(struct isert_conn *isert_conn,
 
 	isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL);
 	if (!isert_conn->login_rsp_buf) {
-		isert_err("Unable to allocate isert_conn->login_rspbuf\n");
+		ret = -ENOMEM;
 		goto out_unmap_login_req_buf;
 	}
 
@@ -578,7 +579,8 @@ isert_connect_release(struct isert_conn *isert_conn)
 	BUG_ON(!device);
 
 	isert_free_rx_descriptors(isert_conn);
-	if (isert_conn->cm_id)
+	if (isert_conn->cm_id &&
+	    !isert_conn->dev_removed)
 		rdma_destroy_id(isert_conn->cm_id);
 
 	if (isert_conn->qp) {
@@ -593,7 +595,10 @@ isert_connect_release(struct isert_conn *isert_conn)
 
 	isert_device_put(device);
 
-	kfree(isert_conn);
+	if (isert_conn->dev_removed)
+		wake_up_interruptible(&isert_conn->rem_wait);
+	else
+		kfree(isert_conn);
 }
 
 static void
@@ -753,6 +758,7 @@ static int
 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 {
 	struct isert_np *isert_np = cma_id->context;
+	struct isert_conn *isert_conn;
 	int ret = 0;
 
 	isert_info("%s (%d): status %d id %p np %p\n",
@@ -773,10 +779,21 @@ isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 		break;
 	case RDMA_CM_EVENT_ADDR_CHANGE:    /* FALLTHRU */
 	case RDMA_CM_EVENT_DISCONNECTED:   /* FALLTHRU */
-	case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */
 	case RDMA_CM_EVENT_TIMEWAIT_EXIT:  /* FALLTHRU */
 		ret = isert_disconnected_handler(cma_id, event->event);
 		break;
+	case RDMA_CM_EVENT_DEVICE_REMOVAL:
+		isert_conn = cma_id->qp->qp_context;
+		isert_conn->dev_removed = true;
+		isert_disconnected_handler(cma_id, event->event);
+		wait_event_interruptible(isert_conn->rem_wait,
+					 isert_conn->state == ISER_CONN_DOWN);
+		kfree(isert_conn);
+		/*
+		 * return non-zero from the callback to destroy
+		 * the rdma cm id
+		 */
+		return 1;
 	case RDMA_CM_EVENT_REJECTED:       /* FALLTHRU */
 	case RDMA_CM_EVENT_UNREACHABLE:    /* FALLTHRU */
 	case RDMA_CM_EVENT_CONNECT_ERROR:
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
index fc791efe3a10..c02ada57d7f5 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.h
+++ b/drivers/infiniband/ulp/isert/ib_isert.h
@@ -158,6 +158,8 @@ struct isert_conn {
 	struct work_struct	release_work;
 	bool                    logout_posted;
 	bool                    snd_w_inv;
+	wait_queue_head_t	rem_wait;
+	bool			dev_removed;
 };
 
 #define ISERT_MAX_CQ 64
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 3322ed750172..d980fb458ad4 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -1262,6 +1262,7 @@ static int srp_map_finish_fmr(struct srp_map_state *state,
 {
 	struct srp_target_port *target = ch->target;
 	struct srp_device *dev = target->srp_host->srp_dev;
+	struct ib_pd *pd = target->pd;
 	struct ib_pool_fmr *fmr;
 	u64 io_addr = 0;
 
@@ -1273,9 +1274,9 @@ static int srp_map_finish_fmr(struct srp_map_state *state,
 	if (state->npages == 0)
 		return 0;
 
-	if (state->npages == 1 && target->global_mr) {
+	if (state->npages == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
 		srp_map_desc(state, state->base_dma_addr, state->dma_len,
-			     target->global_mr->rkey);
+			     pd->unsafe_global_rkey);
 		goto reset_state;
 	}
 
@@ -1315,6 +1316,7 @@ static int srp_map_finish_fr(struct srp_map_state *state,
 {
 	struct srp_target_port *target = ch->target;
 	struct srp_device *dev = target->srp_host->srp_dev;
+	struct ib_pd *pd = target->pd;
 	struct ib_send_wr *bad_wr;
 	struct ib_reg_wr wr;
 	struct srp_fr_desc *desc;
@@ -1326,12 +1328,12 @@ static int srp_map_finish_fr(struct srp_map_state *state,
 
 	WARN_ON_ONCE(!dev->use_fast_reg);
 
-	if (sg_nents == 1 && target->global_mr) {
+	if (sg_nents == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
 		unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
 
 		srp_map_desc(state, sg_dma_address(state->sg) + sg_offset,
 			     sg_dma_len(state->sg) - sg_offset,
-			     target->global_mr->rkey);
+			     pd->unsafe_global_rkey);
 		if (sg_offset_p)
 			*sg_offset_p = 0;
 		return 1;
@@ -1386,7 +1388,7 @@ static int srp_map_finish_fr(struct srp_map_state *state,
 
 static int srp_map_sg_entry(struct srp_map_state *state,
 			    struct srp_rdma_ch *ch,
-			    struct scatterlist *sg, int sg_index)
+			    struct scatterlist *sg)
 {
 	struct srp_target_port *target = ch->target;
 	struct srp_device *dev = target->srp_host->srp_dev;
@@ -1400,7 +1402,9 @@ static int srp_map_sg_entry(struct srp_map_state *state,
 
 	while (dma_len) {
 		unsigned offset = dma_addr & ~dev->mr_page_mask;
-		if (state->npages == dev->max_pages_per_mr || offset != 0) {
+
+		if (state->npages == dev->max_pages_per_mr ||
+		    (state->npages > 0 && offset != 0)) {
 			ret = srp_map_finish_fmr(state, ch);
 			if (ret)
 				return ret;
@@ -1417,12 +1421,12 @@ static int srp_map_sg_entry(struct srp_map_state *state,
 	}
 
 	/*
-	 * If the last entry of the MR wasn't a full page, then we need to
+	 * If the end of the MR is not on a page boundary then we need to
 	 * close it out and start a new one -- we can only merge at page
 	 * boundaries.
 	 */
 	ret = 0;
-	if (len != dev->mr_page_size)
+	if ((dma_addr & ~dev->mr_page_mask) != 0)
 		ret = srp_map_finish_fmr(state, ch);
 	return ret;
 }
@@ -1439,7 +1443,7 @@ static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
 	state->fmr.end = req->fmr_list + ch->target->mr_per_cmd;
 
 	for_each_sg(scat, sg, count, i) {
-		ret = srp_map_sg_entry(state, ch, sg, i);
+		ret = srp_map_sg_entry(state, ch, sg);
 		if (ret)
 			return ret;
 	}
@@ -1491,7 +1495,7 @@ static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
 	for_each_sg(scat, sg, count, i) {
 		srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
 			     ib_sg_dma_len(dev->dev, sg),
-			     target->global_mr->rkey);
+			     target->pd->unsafe_global_rkey);
 	}
 
 	return 0;
@@ -1591,6 +1595,7 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
 			struct srp_request *req)
 {
 	struct srp_target_port *target = ch->target;
+	struct ib_pd *pd = target->pd;
 	struct scatterlist *scat;
 	struct srp_cmd *cmd = req->cmd->buf;
 	int len, nents, count, ret;
@@ -1626,7 +1631,7 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
 	fmt = SRP_DATA_DESC_DIRECT;
 	len = sizeof (struct srp_cmd) +	sizeof (struct srp_direct_buf);
 
-	if (count == 1 && target->global_mr) {
+	if (count == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
 		/*
 		 * The midlayer only generated a single gather/scatter
 		 * entry, or DMA mapping coalesced everything to a
@@ -1636,7 +1641,7 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
 		struct srp_direct_buf *buf = (void *) cmd->add_data;
 
 		buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
-		buf->key = cpu_to_be32(target->global_mr->rkey);
+		buf->key = cpu_to_be32(pd->unsafe_global_rkey);
 		buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
 
 		req->nmdesc = 0;
@@ -1709,14 +1714,14 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
 	memcpy(indirect_hdr->desc_list, req->indirect_desc,
 	       count * sizeof (struct srp_direct_buf));
 
-	if (!target->global_mr) {
+	if (!(pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
 		ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
 				  idb_len, &idb_rkey);
 		if (ret < 0)
 			goto unmap;
 		req->nmdesc++;
 	} else {
-		idb_rkey = cpu_to_be32(target->global_mr->rkey);
+		idb_rkey = cpu_to_be32(pd->unsafe_global_rkey);
 	}
 
 	indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
@@ -3268,8 +3273,8 @@ static ssize_t srp_create_target(struct device *dev,
 	target->io_class	= SRP_REV16A_IB_IO_CLASS;
 	target->scsi_host	= target_host;
 	target->srp_host	= host;
+	target->pd		= host->srp_dev->pd;
 	target->lkey		= host->srp_dev->pd->local_dma_lkey;
-	target->global_mr	= host->srp_dev->global_mr;
 	target->cmd_sg_cnt	= cmd_sg_entries;
 	target->sg_tablesize	= indirect_sg_entries ? : cmd_sg_entries;
 	target->allow_ext_sg	= allow_ext_sg;
@@ -3524,6 +3529,7 @@ static void srp_add_one(struct ib_device *device)
 	struct srp_host *host;
 	int mr_page_shift, p;
 	u64 max_pages_per_mr;
+	unsigned int flags = 0;
 
 	srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL);
 	if (!srp_dev)
@@ -3558,6 +3564,10 @@ static void srp_add_one(struct ib_device *device)
 		srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
 	}
 
+	if (never_register || !register_always ||
+	    (!srp_dev->has_fmr && !srp_dev->has_fr))
+		flags |= IB_PD_UNSAFE_GLOBAL_RKEY;
+
 	if (srp_dev->use_fast_reg) {
 		srp_dev->max_pages_per_mr =
 			min_t(u32, srp_dev->max_pages_per_mr,
@@ -3573,19 +3583,10 @@ static void srp_add_one(struct ib_device *device)
 	INIT_LIST_HEAD(&srp_dev->dev_list);
 
 	srp_dev->dev = device;
-	srp_dev->pd  = ib_alloc_pd(device);
+	srp_dev->pd  = ib_alloc_pd(device, flags);
 	if (IS_ERR(srp_dev->pd))
 		goto free_dev;
 
-	if (never_register || !register_always ||
-	    (!srp_dev->has_fmr && !srp_dev->has_fr)) {
-		srp_dev->global_mr = ib_get_dma_mr(srp_dev->pd,
-						   IB_ACCESS_LOCAL_WRITE |
-						   IB_ACCESS_REMOTE_READ |
-						   IB_ACCESS_REMOTE_WRITE);
-		if (IS_ERR(srp_dev->global_mr))
-			goto err_pd;
-	}
 
 	for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
 		host = srp_add_port(srp_dev, p);
@@ -3596,9 +3597,6 @@ static void srp_add_one(struct ib_device *device)
 	ib_set_client_data(device, &srp_client, srp_dev);
 	return;
 
-err_pd:
-	ib_dealloc_pd(srp_dev->pd);
-
 free_dev:
 	kfree(srp_dev);
 }
@@ -3638,8 +3636,6 @@ static void srp_remove_one(struct ib_device *device, void *client_data)
 		kfree(host);
 	}
 
-	if (srp_dev->global_mr)
-		ib_dereg_mr(srp_dev->global_mr);
 	ib_dealloc_pd(srp_dev->pd);
 
 	kfree(srp_dev);
diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index 26bb9b0a7a63..21c69695f9d4 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -90,7 +90,6 @@ struct srp_device {
 	struct list_head	dev_list;
 	struct ib_device       *dev;
 	struct ib_pd	       *pd;
-	struct ib_mr	       *global_mr;
 	u64			mr_page_mask;
 	int			mr_page_size;
 	int			mr_max_size;
@@ -179,7 +178,7 @@ struct srp_target_port {
 	spinlock_t		lock;
 
 	/* read only in the hot path */
-	struct ib_mr		*global_mr;
+	struct ib_pd		*pd;
 	struct srp_rdma_ch	*ch;
 	u32			ch_count;
 	u32			lkey;
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index dfa23b075a88..0b1f69ed2e92 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -522,6 +522,11 @@ static int srpt_refresh_port(struct srpt_port *sport)
 	if (ret)
 		goto err_query_port;
 
+	snprintf(sport->port_guid, sizeof(sport->port_guid),
+		"0x%016llx%016llx",
+		be64_to_cpu(sport->gid.global.subnet_prefix),
+		be64_to_cpu(sport->gid.global.interface_id));
+
 	if (!sport->mad_agent) {
 		memset(&reg_req, 0, sizeof(reg_req));
 		reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
@@ -2475,7 +2480,7 @@ static void srpt_add_one(struct ib_device *device)
 	init_waitqueue_head(&sdev->ch_releaseQ);
 	mutex_init(&sdev->mutex);
 
-	sdev->pd = ib_alloc_pd(device);
+	sdev->pd = ib_alloc_pd(device, 0);
 	if (IS_ERR(sdev->pd))
 		goto free_dev;
 
@@ -2548,10 +2553,6 @@ static void srpt_add_one(struct ib_device *device)
 			       sdev->device->name, i);
 			goto err_ring;
 		}
-		snprintf(sport->port_guid, sizeof(sport->port_guid),
-			"0x%016llx%016llx",
-			be64_to_cpu(sport->gid.global.subnet_prefix),
-			be64_to_cpu(sport->gid.global.interface_id));
 	}
 
 	spin_lock(&srpt_dev_lock);
diff --git a/drivers/input/joydev.c b/drivers/input/joydev.c
index 5d11fea3c8ec..f3135ae22df4 100644
--- a/drivers/input/joydev.c
+++ b/drivers/input/joydev.c
@@ -950,6 +950,12 @@ static const struct input_device_id joydev_ids[] = {
 		.flags = INPUT_DEVICE_ID_MATCH_EVBIT |
 				INPUT_DEVICE_ID_MATCH_ABSBIT,
 		.evbit = { BIT_MASK(EV_ABS) },
+		.absbit = { BIT_MASK(ABS_Z) },
+	},
+	{
+		.flags = INPUT_DEVICE_ID_MATCH_EVBIT |
+				INPUT_DEVICE_ID_MATCH_ABSBIT,
+		.evbit = { BIT_MASK(EV_ABS) },
 		.absbit = { BIT_MASK(ABS_WHEEL) },
 	},
 	{
diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig
index 509608c95994..cbd75cf44739 100644
--- a/drivers/input/keyboard/Kconfig
+++ b/drivers/input/keyboard/Kconfig
@@ -12,6 +12,21 @@ menuconfig INPUT_KEYBOARD
 
 if INPUT_KEYBOARD
 
+config KEYBOARD_ADC
+	tristate "ADC Ladder Buttons"
+	depends on IIO
+	select INPUT_POLLDEV
+	help
+	  This driver implements support for buttons connected
+	  to an ADC using a resistor ladder.
+
+	  Say Y here if your device has such buttons connected to an ADC.  Your
+	  board-specific setup logic must also provide a configuration data
+	  for mapping voltages to buttons.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called adc_keys.
+
 config KEYBOARD_ADP5520
 	tristate "Keypad Support for ADP5520 PMIC"
 	depends on PMIC_ADP5520
diff --git a/drivers/input/keyboard/Makefile b/drivers/input/keyboard/Makefile
index 1d416ddf84e4..d9f4cfcf3410 100644
--- a/drivers/input/keyboard/Makefile
+++ b/drivers/input/keyboard/Makefile
@@ -4,6 +4,7 @@
 
 # Each configuration option enables a list of files.
 
+obj-$(CONFIG_KEYBOARD_ADC)		+= adc-keys.o
 obj-$(CONFIG_KEYBOARD_ADP5520)		+= adp5520-keys.o
 obj-$(CONFIG_KEYBOARD_ADP5588)		+= adp5588-keys.o
 obj-$(CONFIG_KEYBOARD_ADP5589)		+= adp5589-keys.o
diff --git a/drivers/input/keyboard/adc-keys.c b/drivers/input/keyboard/adc-keys.c
new file mode 100644
index 000000000000..f8cf2ccacefd
--- /dev/null
+++ b/drivers/input/keyboard/adc-keys.c
@@ -0,0 +1,210 @@
+/*
+ * Input driver for resistor ladder connected on ADC
+ *
+ * Copyright (c) 2016 Alexandre Belloni
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/err.h>
+#include <linux/iio/consumer.h>
+#include <linux/iio/types.h>
+#include <linux/input.h>
+#include <linux/input-polldev.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/slab.h>
+
+struct adc_keys_button {
+	u32 voltage;
+	u32 keycode;
+};
+
+struct adc_keys_state {
+	struct iio_channel *channel;
+	u32 num_keys;
+	u32 last_key;
+	u32 keyup_voltage;
+	const struct adc_keys_button *map;
+};
+
+static void adc_keys_poll(struct input_polled_dev *dev)
+{
+	struct adc_keys_state *st = dev->private;
+	int i, value, ret;
+	u32 diff, closest = 0xffffffff;
+	int keycode = 0;
+
+	ret = iio_read_channel_processed(st->channel, &value);
+	if (unlikely(ret < 0)) {
+		/* Forcibly release key if any was pressed */
+		value = st->keyup_voltage;
+	} else {
+		for (i = 0; i < st->num_keys; i++) {
+			diff = abs(st->map[i].voltage - value);
+			if (diff < closest) {
+				closest = diff;
+				keycode = st->map[i].keycode;
+			}
+		}
+	}
+
+	if (abs(st->keyup_voltage - value) < closest)
+		keycode = 0;
+
+	if (st->last_key && st->last_key != keycode)
+		input_report_key(dev->input, st->last_key, 0);
+
+	if (keycode)
+		input_report_key(dev->input, keycode, 1);
+
+	input_sync(dev->input);
+	st->last_key = keycode;
+}
+
+static int adc_keys_load_keymap(struct device *dev, struct adc_keys_state *st)
+{
+	struct adc_keys_button *map;
+	struct fwnode_handle *child;
+	int i;
+
+	st->num_keys = device_get_child_node_count(dev);
+	if (st->num_keys == 0) {
+		dev_err(dev, "keymap is missing\n");
+		return -EINVAL;
+	}
+
+	map = devm_kmalloc_array(dev, st->num_keys, sizeof(*map), GFP_KERNEL);
+	if (!map)
+		return -ENOMEM;
+
+	i = 0;
+	device_for_each_child_node(dev, child) {
+		if (fwnode_property_read_u32(child, "press-threshold-microvolt",
+					     &map[i].voltage)) {
+			dev_err(dev, "Key with invalid or missing voltage\n");
+			fwnode_handle_put(child);
+			return -EINVAL;
+		}
+		map[i].voltage /= 1000;
+
+		if (fwnode_property_read_u32(child, "linux,code",
+					     &map[i].keycode)) {
+			dev_err(dev, "Key with invalid or missing linux,code\n");
+			fwnode_handle_put(child);
+			return -EINVAL;
+		}
+
+		i++;
+	}
+
+	st->map = map;
+	return 0;
+}
+
+static int adc_keys_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct adc_keys_state *st;
+	struct input_polled_dev *poll_dev;
+	struct input_dev *input;
+	enum iio_chan_type type;
+	int i, value;
+	int error;
+
+	st = devm_kzalloc(dev, sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
+
+	st->channel = devm_iio_channel_get(dev, "buttons");
+	if (IS_ERR(st->channel))
+		return PTR_ERR(st->channel);
+
+	if (!st->channel->indio_dev)
+		return -ENXIO;
+
+	error = iio_get_channel_type(st->channel, &type);
+	if (error < 0)
+		return error;
+
+	if (type != IIO_VOLTAGE) {
+		dev_err(dev, "Incompatible channel type %d\n", type);
+		return -EINVAL;
+	}
+
+	if (device_property_read_u32(dev, "keyup-threshold-microvolt",
+				     &st->keyup_voltage)) {
+		dev_err(dev, "Invalid or missing keyup voltage\n");
+		return -EINVAL;
+	}
+	st->keyup_voltage /= 1000;
+
+	error = adc_keys_load_keymap(dev, st);
+	if (error)
+		return error;
+
+	platform_set_drvdata(pdev, st);
+
+	poll_dev = devm_input_allocate_polled_device(dev);
+	if (!poll_dev) {
+		dev_err(dev, "failed to allocate input device\n");
+		return -ENOMEM;
+	}
+
+	if (!device_property_read_u32(dev, "poll-interval", &value))
+		poll_dev->poll_interval = value;
+
+	poll_dev->poll = adc_keys_poll;
+	poll_dev->private = st;
+
+	input = poll_dev->input;
+
+	input->name = pdev->name;
+	input->phys = "adc-keys/input0";
+
+	input->id.bustype = BUS_HOST;
+	input->id.vendor = 0x0001;
+	input->id.product = 0x0001;
+	input->id.version = 0x0100;
+
+	__set_bit(EV_KEY, input->evbit);
+	for (i = 0; i < st->num_keys; i++)
+		__set_bit(st->map[i].keycode, input->keybit);
+
+	if (device_property_read_bool(dev, "autorepeat"))
+		__set_bit(EV_REP, input->evbit);
+
+	error = input_register_polled_device(poll_dev);
+	if (error) {
+		dev_err(dev, "Unable to register input device: %d\n", error);
+		return error;
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id adc_keys_of_match[] = {
+	{ .compatible = "adc-keys", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, adc_keys_of_match);
+#endif
+
+static struct platform_driver __refdata adc_keys_driver = {
+	.driver = {
+		.name = "adc_keys",
+		.of_match_table = of_match_ptr(adc_keys_of_match),
+	},
+	.probe = adc_keys_probe,
+};
+module_platform_driver(adc_keys_driver);
+
+MODULE_AUTHOR("Alexandre Belloni <alexandre.belloni@free-electrons.com>");
+MODULE_DESCRIPTION("Input driver for resistor ladder connected on ADC");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/input/keyboard/cros_ec_keyb.c b/drivers/input/keyboard/cros_ec_keyb.c
index 4b0878f35471..25943e9bc8bf 100644
--- a/drivers/input/keyboard/cros_ec_keyb.c
+++ b/drivers/input/keyboard/cros_ec_keyb.c
@@ -27,6 +27,7 @@
 #include <linux/input.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
+#include <linux/notifier.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/input/matrix_keypad.h>
@@ -44,6 +45,7 @@
  * @dev: Device pointer
  * @idev: Input device
  * @ec: Top level ChromeOS device to use to talk to EC
+ * @notifier: interrupt event notifier for transport devices
  */
 struct cros_ec_keyb {
 	unsigned int rows;
@@ -57,6 +59,7 @@ struct cros_ec_keyb {
 	struct device *dev;
 	struct input_dev *idev;
 	struct cros_ec_device *ec;
+	struct notifier_block notifier;
 };
 
 
@@ -146,67 +149,44 @@ static void cros_ec_keyb_process(struct cros_ec_keyb *ckdev,
 	input_sync(ckdev->idev);
 }
 
-static int cros_ec_keyb_get_state(struct cros_ec_keyb *ckdev, uint8_t *kb_state)
-{
-	int ret = 0;
-	struct cros_ec_command *msg;
-
-	msg = kmalloc(sizeof(*msg) + ckdev->cols, GFP_KERNEL);
-	if (!msg)
-		return -ENOMEM;
-
-	msg->version = 0;
-	msg->command = EC_CMD_MKBP_STATE;
-	msg->insize = ckdev->cols;
-	msg->outsize = 0;
-
-	ret = cros_ec_cmd_xfer(ckdev->ec, msg);
-	if (ret < 0) {
-		dev_err(ckdev->dev, "Error transferring EC message %d\n", ret);
-		goto exit;
-	}
-
-	memcpy(kb_state, msg->data, ckdev->cols);
-exit:
-	kfree(msg);
-	return ret;
-}
-
-static irqreturn_t cros_ec_keyb_irq(int irq, void *data)
+static int cros_ec_keyb_open(struct input_dev *dev)
 {
-	struct cros_ec_keyb *ckdev = data;
-	struct cros_ec_device *ec = ckdev->ec;
-	int ret;
-	uint8_t kb_state[ckdev->cols];
-
-	if (device_may_wakeup(ec->dev))
-		pm_wakeup_event(ec->dev, 0);
-
-	ret = cros_ec_keyb_get_state(ckdev, kb_state);
-	if (ret >= 0)
-		cros_ec_keyb_process(ckdev, kb_state, ret);
-	else
-		dev_err(ckdev->dev, "failed to get keyboard state: %d\n", ret);
+	struct cros_ec_keyb *ckdev = input_get_drvdata(dev);
 
-	return IRQ_HANDLED;
+	return blocking_notifier_chain_register(&ckdev->ec->event_notifier,
+						&ckdev->notifier);
 }
 
-static int cros_ec_keyb_open(struct input_dev *dev)
+static void cros_ec_keyb_close(struct input_dev *dev)
 {
 	struct cros_ec_keyb *ckdev = input_get_drvdata(dev);
-	struct cros_ec_device *ec = ckdev->ec;
 
-	return request_threaded_irq(ec->irq, NULL, cros_ec_keyb_irq,
-					IRQF_TRIGGER_LOW | IRQF_ONESHOT,
-					"cros_ec_keyb", ckdev);
+	blocking_notifier_chain_unregister(&ckdev->ec->event_notifier,
+					   &ckdev->notifier);
 }
 
-static void cros_ec_keyb_close(struct input_dev *dev)
+static int cros_ec_keyb_work(struct notifier_block *nb,
+			     unsigned long queued_during_suspend, void *_notify)
 {
-	struct cros_ec_keyb *ckdev = input_get_drvdata(dev);
-	struct cros_ec_device *ec = ckdev->ec;
+	struct cros_ec_keyb *ckdev = container_of(nb, struct cros_ec_keyb,
+						  notifier);
 
-	free_irq(ec->irq, ckdev);
+	if (ckdev->ec->event_data.event_type != EC_MKBP_EVENT_KEY_MATRIX)
+		return NOTIFY_DONE;
+	/*
+	 * If EC is not the wake source, discard key state changes during
+	 * suspend.
+	 */
+	if (queued_during_suspend)
+		return NOTIFY_OK;
+	if (ckdev->ec->event_size != ckdev->cols) {
+		dev_err(ckdev->dev,
+			"Discarded incomplete key matrix event.\n");
+		return NOTIFY_OK;
+	}
+	cros_ec_keyb_process(ckdev, ckdev->ec->event_data.data.key_matrix,
+			     ckdev->ec->event_size);
+	return NOTIFY_OK;
 }
 
 /*
@@ -265,12 +245,8 @@ static int cros_ec_keyb_probe(struct platform_device *pdev)
 	if (!idev)
 		return -ENOMEM;
 
-	if (!ec->irq) {
-		dev_err(dev, "no EC IRQ specified\n");
-		return -EINVAL;
-	}
-
 	ckdev->ec = ec;
+	ckdev->notifier.notifier_call = cros_ec_keyb_work;
 	ckdev->dev = dev;
 	dev_set_drvdata(dev, ckdev);
 
@@ -311,54 +287,6 @@ static int cros_ec_keyb_probe(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-/* Clear any keys in the buffer */
-static void cros_ec_keyb_clear_keyboard(struct cros_ec_keyb *ckdev)
-{
-	uint8_t old_state[ckdev->cols];
-	uint8_t new_state[ckdev->cols];
-	unsigned long duration;
-	int i, ret;
-
-	/*
-	 * Keep reading until we see that the scan state does not change.
-	 * That indicates that we are done.
-	 *
-	 * Assume that the EC keyscan buffer is at most 32 deep.
-	 */
-	duration = jiffies;
-	ret = cros_ec_keyb_get_state(ckdev, new_state);
-	for (i = 1; !ret && i < 32; i++) {
-		memcpy(old_state, new_state, sizeof(old_state));
-		ret = cros_ec_keyb_get_state(ckdev, new_state);
-		if (0 == memcmp(old_state, new_state, sizeof(old_state)))
-			break;
-	}
-	duration = jiffies - duration;
-	dev_info(ckdev->dev, "Discarded %d keyscan(s) in %dus\n", i,
-		jiffies_to_usecs(duration));
-}
-
-static int cros_ec_keyb_resume(struct device *dev)
-{
-	struct cros_ec_keyb *ckdev = dev_get_drvdata(dev);
-
-	/*
-	 * When the EC is not a wake source, then it could not have caused the
-	 * resume, so we clear the EC's key scan buffer. If the EC was a
-	 * wake source (e.g. the lid is open and the user might press a key to
-	 * wake) then the key scan buffer should be preserved.
-	 */
-	if (!ckdev->ec->was_wake_device)
-		cros_ec_keyb_clear_keyboard(ckdev);
-
-	return 0;
-}
-
-#endif
-
-static SIMPLE_DEV_PM_OPS(cros_ec_keyb_pm_ops, NULL, cros_ec_keyb_resume);
-
 #ifdef CONFIG_OF
 static const struct of_device_id cros_ec_keyb_of_match[] = {
 	{ .compatible = "google,cros-ec-keyb" },
@@ -372,7 +300,6 @@ static struct platform_driver cros_ec_keyb_driver = {
 	.driver = {
 		.name = "cros-ec-keyb",
 		.of_match_table = of_match_ptr(cros_ec_keyb_of_match),
-		.pm	= &cros_ec_keyb_pm_ops,
 	},
 };
 
diff --git a/drivers/input/keyboard/jornada720_kbd.c b/drivers/input/keyboard/jornada720_kbd.c
index 421d9c55b0e8..1277c39f9482 100644
--- a/drivers/input/keyboard/jornada720_kbd.c
+++ b/drivers/input/keyboard/jornada720_kbd.c
@@ -25,8 +25,6 @@
 #include <linux/slab.h>
 
 #include <mach/jornada720.h>
-#include <mach/hardware.h>
-#include <mach/irqs.h>
 
 MODULE_AUTHOR("Kristoffer Ericson <Kristoffer.Ericson@gmail.com>");
 MODULE_DESCRIPTION("HP Jornada 710/720/728 keyboard driver");
@@ -66,10 +64,8 @@ static irqreturn_t jornada720_kbd_interrupt(int irq, void *dev_id)
 	jornada_ssp_start();
 
 	if (jornada_ssp_inout(GETSCANKEYCODE) != TXDUMMY) {
-		printk(KERN_DEBUG
-			"jornada720_kbd: "
-			"GetKeycode command failed with ETIMEDOUT, "
-			"flushed bus\n");
+		dev_dbg(&pdev->dev,
+			"GetKeycode command failed with ETIMEDOUT, flushed bus\n");
 	} else {
 		/* How many keycodes are waiting for us? */
 		count = jornada_ssp_byte(TXDUMMY);
@@ -97,14 +93,16 @@ static int jornada720_kbd_probe(struct platform_device *pdev)
 {
 	struct jornadakbd *jornadakbd;
 	struct input_dev *input_dev;
-	int i, err;
+	int i, err, irq;
 
-	jornadakbd = kzalloc(sizeof(struct jornadakbd), GFP_KERNEL);
-	input_dev = input_allocate_device();
-	if (!jornadakbd || !input_dev) {
-		err = -ENOMEM;
-		goto fail1;
-	}
+	irq = platform_get_irq(pdev, 0);
+	if (irq <= 0)
+		return irq < 0 ? irq : -EINVAL;
+
+	jornadakbd = devm_kzalloc(&pdev->dev, sizeof(*jornadakbd), GFP_KERNEL);
+	input_dev = devm_input_allocate_device(&pdev->dev);
+	if (!jornadakbd || !input_dev)
+		return -ENOMEM;
 
 	platform_set_drvdata(pdev, jornadakbd);
 
@@ -127,40 +125,16 @@ static int jornada720_kbd_probe(struct platform_device *pdev)
 
 	input_set_capability(input_dev, EV_MSC, MSC_SCAN);
 
-	err = request_irq(IRQ_GPIO0,
-			  jornada720_kbd_interrupt,
-			  IRQF_TRIGGER_FALLING,
-			  "jornadakbd", pdev);
+	err = devm_request_irq(&pdev->dev, irq, jornada720_kbd_interrupt,
+			       IRQF_TRIGGER_FALLING, "jornadakbd", pdev);
 	if (err) {
-		printk(KERN_INFO "jornadakbd720_kbd: Unable to grab IRQ\n");
-		goto fail1;
+		dev_err(&pdev->dev, "unable to grab IRQ%d: %d\n", irq, err);
+		return err;
 	}
 
-	err = input_register_device(jornadakbd->input);
-	if (err)
-		goto fail2;
-
-	return 0;
-
- fail2:	/* IRQ, DEVICE, MEMORY */
-	free_irq(IRQ_GPIO0, pdev);
- fail1:	/* DEVICE, MEMORY */
-	input_free_device(input_dev);
-	kfree(jornadakbd);
-	return err;
+	return input_register_device(jornadakbd->input);
 };
 
-static int jornada720_kbd_remove(struct platform_device *pdev)
-{
-	struct jornadakbd *jornadakbd = platform_get_drvdata(pdev);
-
-	free_irq(IRQ_GPIO0, pdev);
-	input_unregister_device(jornadakbd->input);
-	kfree(jornadakbd);
-
-	return 0;
-}
-
 /* work with hotplug and coldplug */
 MODULE_ALIAS("platform:jornada720_kbd");
 
@@ -169,6 +143,5 @@ static struct platform_driver jornada720_kbd_driver = {
 		.name    = "jornada720_kbd",
 	 },
 	.probe   = jornada720_kbd_probe,
-	.remove  = jornada720_kbd_remove,
 };
 module_platform_driver(jornada720_kbd_driver);
diff --git a/drivers/input/keyboard/snvs_pwrkey.c b/drivers/input/keyboard/snvs_pwrkey.c
index 24a9f599082f..7544888c4749 100644
--- a/drivers/input/keyboard/snvs_pwrkey.c
+++ b/drivers/input/keyboard/snvs_pwrkey.c
@@ -168,7 +168,6 @@ static int imx_snvs_pwrkey_probe(struct platform_device *pdev)
 	error = input_register_device(input);
 	if (error < 0) {
 		dev_err(&pdev->dev, "failed to register input device\n");
-		input_free_device(input);
 		return error;
 	}
 
diff --git a/drivers/input/keyboard/tegra-kbc.c b/drivers/input/keyboard/tegra-kbc.c
index 7d61439be5f2..0c07e1023a46 100644
--- a/drivers/input/keyboard/tegra-kbc.c
+++ b/drivers/input/keyboard/tegra-kbc.c
@@ -376,7 +376,7 @@ static int tegra_kbc_start(struct tegra_kbc *kbc)
 	/* Reset the KBC controller to clear all previous status.*/
 	reset_control_assert(kbc->rst);
 	udelay(100);
-	reset_control_assert(kbc->rst);
+	reset_control_deassert(kbc->rst);
 	udelay(100);
 
 	tegra_kbc_config_pins(kbc);
diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig
index efb0ca871327..7ffb614ce566 100644
--- a/drivers/input/misc/Kconfig
+++ b/drivers/input/misc/Kconfig
@@ -292,6 +292,18 @@ config INPUT_GPIO_TILT_POLLED
 	  To compile this driver as a module, choose M here: the
 	  module will be called gpio_tilt_polled.
 
+config INPUT_GPIO_DECODER
+	tristate "Polled GPIO Decoder Input driver"
+	depends on GPIOLIB || COMPILE_TEST
+	select INPUT_POLLDEV
+	help
+	 Say Y here if you want driver to read status of multiple GPIO
+	 lines and report the encoded value as an absolute integer to
+	 input subsystem.
+
+	 To compile this driver as a module, choose M here: the module
+	 will be called gpio_decoder.
+
 config INPUT_IXP4XX_BEEPER
 	tristate "IXP4XX Beeper support"
 	depends on ARCH_IXP4XX
@@ -454,10 +466,10 @@ config INPUT_RETU_PWRBUTTON
 
 config INPUT_TPS65218_PWRBUTTON
 	tristate "TPS65218 Power button driver"
-	depends on MFD_TPS65218
+	depends on (MFD_TPS65217 || MFD_TPS65218)
 	help
 	  Say Y here if you want to enable power buttong reporting for
-	  the TPS65218 Power Management IC device.
+	  TPS65217 and TPS65218 Power Management IC devices.
 
 	  To compile this driver as a module, choose M here. The module will
 	  be called tps65218-pwrbutton.
diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile
index 6a1e5e20fc1c..0b6d025f0487 100644
--- a/drivers/input/misc/Makefile
+++ b/drivers/input/misc/Makefile
@@ -35,6 +35,7 @@ obj-$(CONFIG_INPUT_DRV2667_HAPTICS)	+= drv2667.o
 obj-$(CONFIG_INPUT_GP2A)		+= gp2ap002a00f.o
 obj-$(CONFIG_INPUT_GPIO_BEEPER)		+= gpio-beeper.o
 obj-$(CONFIG_INPUT_GPIO_TILT_POLLED)	+= gpio_tilt_polled.o
+obj-$(CONFIG_INPUT_GPIO_DECODER)	+= gpio_decoder.o
 obj-$(CONFIG_INPUT_HISI_POWERKEY)	+= hisi_powerkey.o
 obj-$(CONFIG_HP_SDC_RTC)		+= hp_sdc_rtc.o
 obj-$(CONFIG_INPUT_IMS_PCU)		+= ims-pcu.o
diff --git a/drivers/input/misc/gpio_decoder.c b/drivers/input/misc/gpio_decoder.c
new file mode 100644
index 000000000000..ca7e0bacb2d8
--- /dev/null
+++ b/drivers/input/misc/gpio_decoder.c
@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * A generic driver to read multiple gpio lines and translate the
+ * encoded numeric value into an input event.
+ */
+
+#include <linux/device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/input.h>
+#include <linux/input-polldev.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+struct gpio_decoder {
+	struct input_polled_dev *poll_dev;
+	struct gpio_descs *input_gpios;
+	struct device *dev;
+	u32 axis;
+	u32 last_stable;
+};
+
+static int gpio_decoder_get_gpios_state(struct gpio_decoder *decoder)
+{
+	struct gpio_descs *gpios = decoder->input_gpios;
+	unsigned int ret = 0;
+	int i, val;
+
+	for (i = 0; i < gpios->ndescs; i++) {
+		val = gpiod_get_value_cansleep(gpios->desc[i]);
+		if (val < 0) {
+			dev_err(decoder->dev,
+				"Error reading gpio %d: %d\n",
+				desc_to_gpio(gpios->desc[i]), val);
+			return val;
+		}
+
+		val = !!val;
+		ret = (ret << 1) | val;
+	}
+
+	return ret;
+}
+
+static void gpio_decoder_poll_gpios(struct input_polled_dev *poll_dev)
+{
+	struct gpio_decoder *decoder = poll_dev->private;
+	int state;
+
+	state = gpio_decoder_get_gpios_state(decoder);
+	if (state >= 0 && state != decoder->last_stable) {
+		input_report_abs(poll_dev->input, decoder->axis, state);
+		input_sync(poll_dev->input);
+		decoder->last_stable = state;
+	}
+}
+
+static int gpio_decoder_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct gpio_decoder *decoder;
+	struct input_polled_dev *poll_dev;
+	u32  max;
+	int err;
+
+	decoder = devm_kzalloc(dev, sizeof(struct gpio_decoder), GFP_KERNEL);
+	if (!decoder)
+		return -ENOMEM;
+
+	device_property_read_u32(dev, "linux,axis", &decoder->axis);
+	decoder->input_gpios = devm_gpiod_get_array(dev, NULL, GPIOD_IN);
+	if (IS_ERR(decoder->input_gpios)) {
+		dev_err(dev, "unable to acquire input gpios\n");
+		return PTR_ERR(decoder->input_gpios);
+	}
+	if (decoder->input_gpios->ndescs < 2) {
+		dev_err(dev, "not enough gpios found\n");
+		return -EINVAL;
+	}
+
+	if (device_property_read_u32(dev, "decoder-max-value", &max))
+		max = (1U << decoder->input_gpios->ndescs) - 1;
+
+	decoder->dev = dev;
+	poll_dev = devm_input_allocate_polled_device(decoder->dev);
+	if (!poll_dev)
+		return -ENOMEM;
+
+	poll_dev->private = decoder;
+	poll_dev->poll = gpio_decoder_poll_gpios;
+	decoder->poll_dev = poll_dev;
+
+	poll_dev->input->name = pdev->name;
+	poll_dev->input->id.bustype = BUS_HOST;
+	input_set_abs_params(poll_dev->input, decoder->axis, 0, max, 0, 0);
+
+	err = input_register_polled_device(poll_dev);
+	if (err) {
+		dev_err(dev, "failed to register polled  device\n");
+		return err;
+	}
+	platform_set_drvdata(pdev, decoder);
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id gpio_decoder_of_match[] = {
+	{ .compatible = "gpio-decoder", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, gpio_decoder_of_match);
+#endif
+
+static struct platform_driver gpio_decoder_driver = {
+	.probe		= gpio_decoder_probe,
+	.driver		= {
+		.name	= "gpio-decoder",
+		.of_match_table = of_match_ptr(gpio_decoder_of_match),
+	}
+};
+module_platform_driver(gpio_decoder_driver);
+
+MODULE_DESCRIPTION("GPIO decoder input driver");
+MODULE_AUTHOR("Vignesh R <vigneshr@ti.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/input/misc/max77693-haptic.c b/drivers/input/misc/max77693-haptic.c
index 29ddeb7be84b..46b0f48fbf49 100644
--- a/drivers/input/misc/max77693-haptic.c
+++ b/drivers/input/misc/max77693-haptic.c
@@ -3,7 +3,7 @@
  *
  * Copyright (C) 2014,2015 Samsung Electronics
  * Jaewon Kim <jaewon02.kim@samsung.com>
- * Krzysztof Kozlowski <k.kozlowski@samsung.com>
+ * Krzysztof Kozlowski <krzk@kernel.org>
  *
  * This program is not provided / owned by Maxim Integrated Products.
  *
@@ -426,7 +426,7 @@ static struct platform_driver max77693_haptic_driver = {
 module_platform_driver(max77693_haptic_driver);
 
 MODULE_AUTHOR("Jaewon Kim <jaewon02.kim@samsung.com>");
-MODULE_AUTHOR("Krzysztof Kozlowski <k.kozlowski@samsung.com>");
+MODULE_AUTHOR("Krzysztof Kozlowski <krzk@kernel.org>");
 MODULE_DESCRIPTION("MAXIM 77693/77843 Haptic driver");
 MODULE_ALIAS("platform:max77693-haptic");
 MODULE_LICENSE("GPL");
diff --git a/drivers/input/misc/tps65218-pwrbutton.c b/drivers/input/misc/tps65218-pwrbutton.c
index a39b62651a4b..3273217ce80c 100644
--- a/drivers/input/misc/tps65218-pwrbutton.c
+++ b/drivers/input/misc/tps65218-pwrbutton.c
@@ -1,8 +1,9 @@
 /*
- * Texas Instruments' TPS65218 Power Button Input Driver
+ * Texas Instruments' TPS65217 and TPS65218 Power Button Input Driver
  *
  * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Felipe Balbi <balbi@ti.com>
+ * Author: Marcin Niestroj <m.niestroj@grinn-global.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -18,31 +19,61 @@
 #include <linux/input.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
+#include <linux/mfd/tps65217.h>
 #include <linux/mfd/tps65218.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
 #include <linux/slab.h>
 
-struct tps65218_pwrbutton {
+struct tps6521x_data {
+	unsigned int reg_status;
+	unsigned int pb_mask;
+	const char *name;
+};
+
+static const struct tps6521x_data tps65217_data = {
+	.reg_status = TPS65217_REG_STATUS,
+	.pb_mask = TPS65217_STATUS_PB,
+	.name = "tps65217_pwrbutton",
+};
+
+static const struct tps6521x_data tps65218_data = {
+	.reg_status = TPS65218_REG_STATUS,
+	.pb_mask = TPS65218_STATUS_PB_STATE,
+	.name = "tps65218_pwrbutton",
+};
+
+struct tps6521x_pwrbutton {
 	struct device *dev;
-	struct tps65218 *tps;
+	struct regmap *regmap;
 	struct input_dev *idev;
+	const struct tps6521x_data *data;
+	char phys[32];
+};
+
+static const struct of_device_id of_tps6521x_pb_match[] = {
+	{ .compatible = "ti,tps65217-pwrbutton", .data = &tps65217_data },
+	{ .compatible = "ti,tps65218-pwrbutton", .data = &tps65218_data },
+	{ },
 };
+MODULE_DEVICE_TABLE(of, of_tps6521x_pb_match);
 
-static irqreturn_t tps65218_pwr_irq(int irq, void *_pwr)
+static irqreturn_t tps6521x_pb_irq(int irq, void *_pwr)
 {
-	struct tps65218_pwrbutton *pwr = _pwr;
+	struct tps6521x_pwrbutton *pwr = _pwr;
+	const struct tps6521x_data *tps_data = pwr->data;
 	unsigned int reg;
 	int error;
 
-	error = tps65218_reg_read(pwr->tps, TPS65218_REG_STATUS, &reg);
+	error = regmap_read(pwr->regmap, tps_data->reg_status, &reg);
 	if (error) {
 		dev_err(pwr->dev, "can't read register: %d\n", error);
 		goto out;
 	}
 
-	if (reg & TPS65218_STATUS_PB_STATE) {
+	if (reg & tps_data->pb_mask) {
 		input_report_key(pwr->idev, KEY_POWER, 1);
 		pm_wakeup_event(pwr->dev, 0);
 	} else {
@@ -55,42 +86,55 @@ out:
 	return IRQ_HANDLED;
 }
 
-static int tps65218_pwron_probe(struct platform_device *pdev)
+static int tps6521x_pb_probe(struct platform_device *pdev)
 {
-	struct tps65218 *tps = dev_get_drvdata(pdev->dev.parent);
 	struct device *dev = &pdev->dev;
-	struct tps65218_pwrbutton *pwr;
+	struct tps6521x_pwrbutton *pwr;
 	struct input_dev *idev;
+	const struct of_device_id *match;
 	int error;
 	int irq;
 
+	match = of_match_node(of_tps6521x_pb_match, pdev->dev.of_node);
+	if (!match)
+		return -ENXIO;
+
 	pwr = devm_kzalloc(dev, sizeof(*pwr), GFP_KERNEL);
 	if (!pwr)
 		return -ENOMEM;
 
+	pwr->data = match->data;
+
 	idev = devm_input_allocate_device(dev);
 	if (!idev)
 		return -ENOMEM;
 
-	idev->name = "tps65218_pwrbutton";
-	idev->phys = "tps65218_pwrbutton/input0";
+	idev->name = pwr->data->name;
+	snprintf(pwr->phys, sizeof(pwr->phys), "%s/input0",
+		pwr->data->name);
+	idev->phys = pwr->phys;
 	idev->dev.parent = dev;
 	idev->id.bustype = BUS_I2C;
 
 	input_set_capability(idev, EV_KEY, KEY_POWER);
 
-	pwr->tps = tps;
+	pwr->regmap = dev_get_regmap(pdev->dev.parent, NULL);
 	pwr->dev = dev;
 	pwr->idev = idev;
 	platform_set_drvdata(pdev, pwr);
 	device_init_wakeup(dev, true);
 
 	irq = platform_get_irq(pdev, 0);
-	error = devm_request_threaded_irq(dev, irq, NULL, tps65218_pwr_irq,
+	if (irq < 0) {
+		dev_err(dev, "No IRQ resource!\n");
+		return -EINVAL;
+	}
+
+	error = devm_request_threaded_irq(dev, irq, NULL, tps6521x_pb_irq,
 					  IRQF_TRIGGER_RISING |
 						IRQF_TRIGGER_FALLING |
 						IRQF_ONESHOT,
-					  "tps65218-pwrbutton", pwr);
+					  pwr->data->name, pwr);
 	if (error) {
 		dev_err(dev, "failed to request IRQ #%d: %d\n",
 			irq, error);
@@ -106,21 +150,15 @@ static int tps65218_pwron_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct of_device_id of_tps65218_pwr_match[] = {
-	{ .compatible = "ti,tps65218-pwrbutton" },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, of_tps65218_pwr_match);
-
-static struct platform_driver tps65218_pwron_driver = {
-	.probe	= tps65218_pwron_probe,
+static struct platform_driver tps6521x_pb_driver = {
+	.probe	= tps6521x_pb_probe,
 	.driver	= {
-		.name	= "tps65218_pwrbutton",
-		.of_match_table = of_tps65218_pwr_match,
+		.name	= "tps6521x_pwrbutton",
+		.of_match_table = of_tps6521x_pb_match,
 	},
 };
-module_platform_driver(tps65218_pwron_driver);
+module_platform_driver(tps6521x_pb_driver);
 
-MODULE_DESCRIPTION("TPS65218 Power Button");
+MODULE_DESCRIPTION("TPS6521X Power Button");
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Felipe Balbi <balbi@ti.com>");
diff --git a/drivers/input/misc/uinput.c b/drivers/input/misc/uinput.c
index 65ebbd111702..92595b98e7ed 100644
--- a/drivers/input/misc/uinput.c
+++ b/drivers/input/misc/uinput.c
@@ -1013,23 +1013,12 @@ static struct miscdevice uinput_misc = {
 	.minor		= UINPUT_MINOR,
 	.name		= UINPUT_NAME,
 };
+module_misc_device(uinput_misc);
+
 MODULE_ALIAS_MISCDEV(UINPUT_MINOR);
 MODULE_ALIAS("devname:" UINPUT_NAME);
 
-static int __init uinput_init(void)
-{
-	return misc_register(&uinput_misc);
-}
-
-static void __exit uinput_exit(void)
-{
-	misc_deregister(&uinput_misc);
-}
-
 MODULE_AUTHOR("Aristeu Sergio Rozanski Filho");
 MODULE_DESCRIPTION("User level driver support for input subsystem");
 MODULE_LICENSE("GPL");
 MODULE_VERSION("0.3");
-
-module_init(uinput_init);
-module_exit(uinput_exit);
diff --git a/drivers/input/mouse/elan_i2c_smbus.c b/drivers/input/mouse/elan_i2c_smbus.c
index cb6aecbc1dc2..e23b2495d52e 100644
--- a/drivers/input/mouse/elan_i2c_smbus.c
+++ b/drivers/input/mouse/elan_i2c_smbus.c
@@ -222,11 +222,13 @@ static int elan_smbus_get_checksum(struct i2c_client *client,
 static int elan_smbus_get_max(struct i2c_client *client,
 			      unsigned int *max_x, unsigned int *max_y)
 {
+	int ret;
 	int error;
 	u8 val[3];
 
-	error = i2c_smbus_read_block_data(client, ETP_SMBUS_RANGE_CMD, val);
-	if (error) {
+	ret = i2c_smbus_read_block_data(client, ETP_SMBUS_RANGE_CMD, val);
+	if (ret != 3) {
+		error = ret < 0 ? ret : -EIO;
 		dev_err(&client->dev, "failed to get dimensions: %d\n", error);
 		return error;
 	}
@@ -240,12 +242,13 @@ static int elan_smbus_get_max(struct i2c_client *client,
 static int elan_smbus_get_resolution(struct i2c_client *client,
 				     u8 *hw_res_x, u8 *hw_res_y)
 {
+	int ret;
 	int error;
 	u8 val[3];
 
-	error = i2c_smbus_read_block_data(client,
-					  ETP_SMBUS_RESOLUTION_CMD, val);
-	if (error) {
+	ret = i2c_smbus_read_block_data(client, ETP_SMBUS_RESOLUTION_CMD, val);
+	if (ret != 3) {
+		error = ret < 0 ? ret : -EIO;
 		dev_err(&client->dev, "failed to get resolution: %d\n", error);
 		return error;
 	}
@@ -260,12 +263,13 @@ static int elan_smbus_get_num_traces(struct i2c_client *client,
 				     unsigned int *x_traces,
 				     unsigned int *y_traces)
 {
+	int ret;
 	int error;
 	u8 val[3];
 
-	error = i2c_smbus_read_block_data(client,
-					  ETP_SMBUS_XY_TRACENUM_CMD, val);
-	if (error) {
+	ret = i2c_smbus_read_block_data(client, ETP_SMBUS_XY_TRACENUM_CMD, val);
+	if (ret != 3) {
+		error = ret < 0 ? ret : -EIO;
 		dev_err(&client->dev, "failed to get trace info: %d\n", error);
 		return error;
 	}
diff --git a/drivers/input/mouse/focaltech.c b/drivers/input/mouse/focaltech.c
index c8c6a8cc329d..54eceb30ede5 100644
--- a/drivers/input/mouse/focaltech.c
+++ b/drivers/input/mouse/focaltech.c
@@ -390,7 +390,8 @@ static int focaltech_read_size(struct psmouse *psmouse)
 	return 0;
 }
 
-void focaltech_set_resolution(struct psmouse *psmouse, unsigned int resolution)
+static void focaltech_set_resolution(struct psmouse *psmouse,
+				     unsigned int resolution)
 {
 	/* not supported yet */
 }
diff --git a/drivers/input/mouse/psmouse-base.c b/drivers/input/mouse/psmouse-base.c
index 5784e20542a4..fb4b185dea96 100644
--- a/drivers/input/mouse/psmouse-base.c
+++ b/drivers/input/mouse/psmouse-base.c
@@ -1916,7 +1916,7 @@ static int __init psmouse_init(void)
 	synaptics_module_init();
 	hgpk_module_init();
 
-	kpsmoused_wq = create_singlethread_workqueue("kpsmoused");
+	kpsmoused_wq = alloc_ordered_workqueue("kpsmoused", 0);
 	if (!kpsmoused_wq) {
 		pr_err("failed to create kpsmoused workqueue\n");
 		return -ENOMEM;
diff --git a/drivers/input/rmi4/Kconfig b/drivers/input/rmi4/Kconfig
index f73df2495fed..4c8a55857e00 100644
--- a/drivers/input/rmi4/Kconfig
+++ b/drivers/input/rmi4/Kconfig
@@ -61,3 +61,14 @@ config RMI4_F30
 
 	  Function 30 provides GPIO and LED support for RMI4 devices. This
 	  includes support for buttons on TouchPads and ClickPads.
+
+config RMI4_F54
+	bool "RMI4 Function 54 (Analog diagnostics)"
+	depends on RMI4_CORE
+	depends on VIDEO_V4L2=y || (RMI4_CORE=m && VIDEO_V4L2=m)
+	select VIDEOBUF2_VMALLOC
+	help
+	  Say Y here if you want to add support for RMI4 function 54
+
+	  Function 54 provides access to various diagnostic features in certain
+	  RMI4 touch sensors.
diff --git a/drivers/input/rmi4/Makefile b/drivers/input/rmi4/Makefile
index 95c00a783992..0bafc8502c4b 100644
--- a/drivers/input/rmi4/Makefile
+++ b/drivers/input/rmi4/Makefile
@@ -7,6 +7,7 @@ rmi_core-$(CONFIG_RMI4_2D_SENSOR) += rmi_2d_sensor.o
 rmi_core-$(CONFIG_RMI4_F11) += rmi_f11.o
 rmi_core-$(CONFIG_RMI4_F12) += rmi_f12.o
 rmi_core-$(CONFIG_RMI4_F30) += rmi_f30.o
+rmi_core-$(CONFIG_RMI4_F54) += rmi_f54.o
 
 # Transports
 obj-$(CONFIG_RMI4_I2C) += rmi_i2c.o
diff --git a/drivers/input/rmi4/rmi_bus.c b/drivers/input/rmi4/rmi_bus.c
index a73580654c6b..ef8c747c35e7 100644
--- a/drivers/input/rmi4/rmi_bus.c
+++ b/drivers/input/rmi4/rmi_bus.c
@@ -9,7 +9,6 @@
 
 #include <linux/kernel.h>
 #include <linux/device.h>
-#include <linux/kconfig.h>
 #include <linux/list.h>
 #include <linux/pm.h>
 #include <linux/rmi.h>
@@ -312,6 +311,9 @@ static struct rmi_function_handler *fn_handlers[] = {
 #ifdef CONFIG_RMI4_F30
 	&rmi_f30_handler,
 #endif
+#ifdef CONFIG_RMI4_F54
+	&rmi_f54_handler,
+#endif
 };
 
 static void __rmi_unregister_function_handlers(int start_idx)
diff --git a/drivers/input/rmi4/rmi_driver.c b/drivers/input/rmi4/rmi_driver.c
index faa295ec4f31..4a88312fbd25 100644
--- a/drivers/input/rmi4/rmi_driver.c
+++ b/drivers/input/rmi4/rmi_driver.c
@@ -17,7 +17,6 @@
 #include <linux/bitmap.h>
 #include <linux/delay.h>
 #include <linux/fs.h>
-#include <linux/kconfig.h>
 #include <linux/pm.h>
 #include <linux/slab.h>
 #include <linux/of.h>
@@ -553,7 +552,6 @@ int rmi_read_register_desc(struct rmi_device *d, u16 addr,
 		goto free_struct_buff;
 
 	reg = find_first_bit(rdesc->presense_map, RMI_REG_DESC_PRESENSE_BITS);
-	map_offset = 0;
 	for (i = 0; i < rdesc->num_registers; i++) {
 		struct rmi_register_desc_item *item = &rdesc->registers[i];
 		int reg_size = struct_buf[offset];
@@ -576,6 +574,8 @@ int rmi_read_register_desc(struct rmi_device *d, u16 addr,
 		item->reg = reg;
 		item->reg_size = reg_size;
 
+		map_offset = 0;
+
 		do {
 			for (b = 0; b < 7; b++) {
 				if (struct_buf[offset] & (0x1 << b))
diff --git a/drivers/input/rmi4/rmi_driver.h b/drivers/input/rmi4/rmi_driver.h
index 6e140fa3cce1..8dfbebe9bf86 100644
--- a/drivers/input/rmi4/rmi_driver.h
+++ b/drivers/input/rmi4/rmi_driver.h
@@ -102,4 +102,5 @@ extern struct rmi_function_handler rmi_f01_handler;
 extern struct rmi_function_handler rmi_f11_handler;
 extern struct rmi_function_handler rmi_f12_handler;
 extern struct rmi_function_handler rmi_f30_handler;
+extern struct rmi_function_handler rmi_f54_handler;
 #endif
diff --git a/drivers/input/rmi4/rmi_f01.c b/drivers/input/rmi4/rmi_f01.c
index fac81fc9bcf6..b5d2dfc23bad 100644
--- a/drivers/input/rmi4/rmi_f01.c
+++ b/drivers/input/rmi4/rmi_f01.c
@@ -8,7 +8,6 @@
  */
 
 #include <linux/kernel.h>
-#include <linux/kconfig.h>
 #include <linux/rmi.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
diff --git a/drivers/input/rmi4/rmi_f11.c b/drivers/input/rmi4/rmi_f11.c
index 20c7134b3d3b..f798f427a46f 100644
--- a/drivers/input/rmi4/rmi_f11.c
+++ b/drivers/input/rmi4/rmi_f11.c
@@ -12,7 +12,6 @@
 #include <linux/device.h>
 #include <linux/input.h>
 #include <linux/input/mt.h>
-#include <linux/kconfig.h>
 #include <linux/rmi.h>
 #include <linux/slab.h>
 #include <linux/of.h>
diff --git a/drivers/input/rmi4/rmi_f54.c b/drivers/input/rmi4/rmi_f54.c
new file mode 100644
index 000000000000..cf805b960866
--- /dev/null
+++ b/drivers/input/rmi4/rmi_f54.c
@@ -0,0 +1,757 @@
+/*
+ * Copyright (c) 2012-2015 Synaptics Incorporated
+ * Copyright (C) 2016 Zodiac Inflight Innovations
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/kernel.h>
+#include <linux/rmi.h>
+#include <linux/input.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-vmalloc.h>
+#include "rmi_driver.h"
+
+#define F54_NAME		"rmi4_f54"
+
+/* F54 data offsets */
+#define F54_REPORT_DATA_OFFSET  3
+#define F54_FIFO_OFFSET         1
+#define F54_NUM_TX_OFFSET       1
+#define F54_NUM_RX_OFFSET       0
+
+/* F54 commands */
+#define F54_GET_REPORT          1
+#define F54_FORCE_CAL           2
+
+/* Fixed sizes of reports */
+#define F54_QUERY_LEN			27
+
+/* F54 capabilities */
+#define F54_CAP_BASELINE	(1 << 2)
+#define F54_CAP_IMAGE8		(1 << 3)
+#define F54_CAP_IMAGE16		(1 << 6)
+
+/**
+ * enum rmi_f54_report_type - RMI4 F54 report types
+ *
+ * @F54_8BIT_IMAGE:	Normalized 8-Bit Image Report. The capacitance variance
+ *			from baseline for each pixel.
+ *
+ * @F54_16BIT_IMAGE:	Normalized 16-Bit Image Report. The capacitance variance
+ *			from baseline for each pixel.
+ *
+ * @F54_RAW_16BIT_IMAGE:
+ *			Raw 16-Bit Image Report. The raw capacitance for each
+ *			pixel.
+ *
+ * @F54_TRUE_BASELINE:	True Baseline Report. The baseline capacitance for each
+ *			pixel.
+ *
+ * @F54_FULL_RAW_CAP:   Full Raw Capacitance Report. The raw capacitance with
+ *			low reference set to its minimum value and high
+ *			reference set to its maximum value.
+ *
+ * @F54_FULL_RAW_CAP_RX_OFFSET_REMOVED:
+ *			Full Raw Capacitance with Receiver Offset Removed
+ *			Report. Set Low reference to its minimum value and high
+ *			references to its maximum value, then report the raw
+ *			capacitance for each pixel.
+ */
+enum rmi_f54_report_type {
+	F54_REPORT_NONE = 0,
+	F54_8BIT_IMAGE = 1,
+	F54_16BIT_IMAGE = 2,
+	F54_RAW_16BIT_IMAGE = 3,
+	F54_TRUE_BASELINE = 9,
+	F54_FULL_RAW_CAP = 19,
+	F54_FULL_RAW_CAP_RX_OFFSET_REMOVED = 20,
+	F54_MAX_REPORT_TYPE,
+};
+
+const char *rmi_f54_report_type_names[] = {
+	[F54_REPORT_NONE]		= "Unknown",
+	[F54_8BIT_IMAGE]		= "Normalized 8-Bit Image",
+	[F54_16BIT_IMAGE]		= "Normalized 16-Bit Image",
+	[F54_RAW_16BIT_IMAGE]		= "Raw 16-Bit Image",
+	[F54_TRUE_BASELINE]		= "True Baseline",
+	[F54_FULL_RAW_CAP]		= "Full Raw Capacitance",
+	[F54_FULL_RAW_CAP_RX_OFFSET_REMOVED]
+					= "Full Raw Capacitance RX Offset Removed",
+};
+
+struct rmi_f54_reports {
+	int start;
+	int size;
+};
+
+struct f54_data {
+	struct rmi_function *fn;
+
+	u8 qry[F54_QUERY_LEN];
+	u8 num_rx_electrodes;
+	u8 num_tx_electrodes;
+	u8 capabilities;
+	u16 clock_rate;
+	u8 family;
+
+	enum rmi_f54_report_type report_type;
+	u8 *report_data;
+	int report_size;
+	struct rmi_f54_reports standard_report[2];
+
+	bool is_busy;
+	struct mutex status_mutex;
+	struct mutex data_mutex;
+
+	struct workqueue_struct *workqueue;
+	struct delayed_work work;
+	unsigned long timeout;
+
+	struct completion cmd_done;
+
+	/* V4L2 support */
+	struct v4l2_device v4l2;
+	struct v4l2_pix_format format;
+	struct video_device vdev;
+	struct vb2_queue queue;
+	struct mutex lock;
+	int input;
+	enum rmi_f54_report_type inputs[F54_MAX_REPORT_TYPE];
+};
+
+/*
+ * Basic checks on report_type to ensure we write a valid type
+ * to the sensor.
+ */
+static bool is_f54_report_type_valid(struct f54_data *f54,
+				     enum rmi_f54_report_type reptype)
+{
+	switch (reptype) {
+	case F54_8BIT_IMAGE:
+		return f54->capabilities & F54_CAP_IMAGE8;
+	case F54_16BIT_IMAGE:
+	case F54_RAW_16BIT_IMAGE:
+		return f54->capabilities & F54_CAP_IMAGE16;
+	case F54_TRUE_BASELINE:
+		return f54->capabilities & F54_CAP_IMAGE16;
+	case F54_FULL_RAW_CAP:
+	case F54_FULL_RAW_CAP_RX_OFFSET_REMOVED:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static enum rmi_f54_report_type rmi_f54_get_reptype(struct f54_data *f54,
+						unsigned int i)
+{
+	if (i >= F54_MAX_REPORT_TYPE)
+		return F54_REPORT_NONE;
+
+	return f54->inputs[i];
+}
+
+static void rmi_f54_create_input_map(struct f54_data *f54)
+{
+	int i = 0;
+	enum rmi_f54_report_type reptype;
+
+	for (reptype = 1; reptype < F54_MAX_REPORT_TYPE; reptype++) {
+		if (!is_f54_report_type_valid(f54, reptype))
+			continue;
+
+		f54->inputs[i++] = reptype;
+	}
+
+	/* Remaining values are zero via kzalloc */
+}
+
+static int rmi_f54_request_report(struct rmi_function *fn, u8 report_type)
+{
+	struct f54_data *f54 = dev_get_drvdata(&fn->dev);
+	struct rmi_device *rmi_dev = fn->rmi_dev;
+	int error;
+
+	/* Write Report Type into F54_AD_Data0 */
+	if (f54->report_type != report_type) {
+		error = rmi_write(rmi_dev, f54->fn->fd.data_base_addr,
+				  report_type);
+		if (error)
+			return error;
+		f54->report_type = report_type;
+	}
+
+	/*
+	 * Small delay after disabling interrupts to avoid race condition
+	 * in firmare. This value is a bit higher than absolutely necessary.
+	 * Should be removed once issue is resolved in firmware.
+	 */
+	usleep_range(2000, 3000);
+
+	mutex_lock(&f54->data_mutex);
+
+	error = rmi_write(rmi_dev, fn->fd.command_base_addr, F54_GET_REPORT);
+	if (error < 0)
+		return error;
+
+	init_completion(&f54->cmd_done);
+
+	f54->is_busy = 1;
+	f54->timeout = jiffies + msecs_to_jiffies(100);
+
+	queue_delayed_work(f54->workqueue, &f54->work, 0);
+
+	mutex_unlock(&f54->data_mutex);
+
+	return 0;
+}
+
+static size_t rmi_f54_get_report_size(struct f54_data *f54)
+{
+	u8 rx = f54->num_rx_electrodes ? : f54->num_rx_electrodes;
+	u8 tx = f54->num_tx_electrodes ? : f54->num_tx_electrodes;
+	size_t size;
+
+	switch (rmi_f54_get_reptype(f54, f54->input)) {
+	case F54_8BIT_IMAGE:
+		size = rx * tx;
+		break;
+	case F54_16BIT_IMAGE:
+	case F54_RAW_16BIT_IMAGE:
+	case F54_TRUE_BASELINE:
+	case F54_FULL_RAW_CAP:
+	case F54_FULL_RAW_CAP_RX_OFFSET_REMOVED:
+		size = sizeof(u16) * rx * tx;
+		break;
+	default:
+		size = 0;
+	}
+
+	return size;
+}
+
+static int rmi_f54_get_pixel_fmt(enum rmi_f54_report_type reptype, u32 *pixfmt)
+{
+	int ret = 0;
+
+	switch (reptype) {
+	case F54_8BIT_IMAGE:
+		*pixfmt = V4L2_TCH_FMT_DELTA_TD08;
+		break;
+
+	case F54_16BIT_IMAGE:
+		*pixfmt = V4L2_TCH_FMT_DELTA_TD16;
+		break;
+
+	case F54_RAW_16BIT_IMAGE:
+	case F54_TRUE_BASELINE:
+	case F54_FULL_RAW_CAP:
+	case F54_FULL_RAW_CAP_RX_OFFSET_REMOVED:
+		*pixfmt = V4L2_TCH_FMT_TU16;
+		break;
+
+	case F54_REPORT_NONE:
+	case F54_MAX_REPORT_TYPE:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static const struct v4l2_file_operations rmi_f54_video_fops = {
+	.owner = THIS_MODULE,
+	.open = v4l2_fh_open,
+	.release = vb2_fop_release,
+	.unlocked_ioctl = video_ioctl2,
+	.read = vb2_fop_read,
+	.mmap = vb2_fop_mmap,
+	.poll = vb2_fop_poll,
+};
+
+static int rmi_f54_queue_setup(struct vb2_queue *q, unsigned int *nbuffers,
+			       unsigned int *nplanes, unsigned int sizes[],
+			       struct device *alloc_devs[])
+{
+	struct f54_data *f54 = q->drv_priv;
+
+	if (*nplanes)
+		return sizes[0] < rmi_f54_get_report_size(f54) ? -EINVAL : 0;
+
+	*nplanes = 1;
+	sizes[0] = rmi_f54_get_report_size(f54);
+
+	return 0;
+}
+
+static void rmi_f54_buffer_queue(struct vb2_buffer *vb)
+{
+	struct f54_data *f54 = vb2_get_drv_priv(vb->vb2_queue);
+	u16 *ptr;
+	enum vb2_buffer_state state;
+	enum rmi_f54_report_type reptype;
+	int ret;
+
+	mutex_lock(&f54->status_mutex);
+
+	reptype = rmi_f54_get_reptype(f54, f54->input);
+	if (reptype == F54_REPORT_NONE) {
+		state = VB2_BUF_STATE_ERROR;
+		goto done;
+	}
+
+	if (f54->is_busy) {
+		state = VB2_BUF_STATE_ERROR;
+		goto done;
+	}
+
+	ret = rmi_f54_request_report(f54->fn, reptype);
+	if (ret) {
+		dev_err(&f54->fn->dev, "Error requesting F54 report\n");
+		state = VB2_BUF_STATE_ERROR;
+		goto done;
+	}
+
+	/* get frame data */
+	mutex_lock(&f54->data_mutex);
+
+	while (f54->is_busy) {
+		mutex_unlock(&f54->data_mutex);
+		if (!wait_for_completion_timeout(&f54->cmd_done,
+						 msecs_to_jiffies(1000))) {
+			dev_err(&f54->fn->dev, "Timed out\n");
+			state = VB2_BUF_STATE_ERROR;
+			goto done;
+		}
+		mutex_lock(&f54->data_mutex);
+	}
+
+	ptr = vb2_plane_vaddr(vb, 0);
+	if (!ptr) {
+		dev_err(&f54->fn->dev, "Error acquiring frame ptr\n");
+		state = VB2_BUF_STATE_ERROR;
+		goto data_done;
+	}
+
+	memcpy(ptr, f54->report_data, f54->report_size);
+	vb2_set_plane_payload(vb, 0, rmi_f54_get_report_size(f54));
+	state = VB2_BUF_STATE_DONE;
+
+data_done:
+	mutex_unlock(&f54->data_mutex);
+done:
+	vb2_buffer_done(vb, state);
+	mutex_unlock(&f54->status_mutex);
+}
+
+/* V4L2 structures */
+static const struct vb2_ops rmi_f54_queue_ops = {
+	.queue_setup            = rmi_f54_queue_setup,
+	.buf_queue              = rmi_f54_buffer_queue,
+	.wait_prepare           = vb2_ops_wait_prepare,
+	.wait_finish            = vb2_ops_wait_finish,
+};
+
+static const struct vb2_queue rmi_f54_queue = {
+	.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
+	.io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF | VB2_READ,
+	.buf_struct_size = sizeof(struct vb2_buffer),
+	.ops = &rmi_f54_queue_ops,
+	.mem_ops = &vb2_vmalloc_memops,
+	.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC,
+	.min_buffers_needed = 1,
+};
+
+static int rmi_f54_vidioc_querycap(struct file *file, void *priv,
+				   struct v4l2_capability *cap)
+{
+	struct f54_data *f54 = video_drvdata(file);
+
+	strlcpy(cap->driver, F54_NAME, sizeof(cap->driver));
+	strlcpy(cap->card, SYNAPTICS_INPUT_DEVICE_NAME, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info),
+		"rmi4:%s", dev_name(&f54->fn->dev));
+
+	return 0;
+}
+
+static int rmi_f54_vidioc_enum_input(struct file *file, void *priv,
+				     struct v4l2_input *i)
+{
+	struct f54_data *f54 = video_drvdata(file);
+	enum rmi_f54_report_type reptype;
+
+	reptype = rmi_f54_get_reptype(f54, i->index);
+	if (reptype == F54_REPORT_NONE)
+		return -EINVAL;
+
+	i->type = V4L2_INPUT_TYPE_TOUCH;
+
+	strlcpy(i->name, rmi_f54_report_type_names[reptype], sizeof(i->name));
+	return 0;
+}
+
+static int rmi_f54_set_input(struct f54_data *f54, unsigned int i)
+{
+	struct v4l2_pix_format *f = &f54->format;
+	enum rmi_f54_report_type reptype;
+	int ret;
+
+	reptype = rmi_f54_get_reptype(f54, i);
+	if (reptype == F54_REPORT_NONE)
+		return -EINVAL;
+
+	ret = rmi_f54_get_pixel_fmt(reptype, &f->pixelformat);
+	if (ret)
+		return ret;
+
+	f54->input = i;
+
+	f->width = f54->num_rx_electrodes;
+	f->height = f54->num_tx_electrodes;
+	f->field = V4L2_FIELD_NONE;
+	f->colorspace = V4L2_COLORSPACE_RAW;
+	f->bytesperline = f->width * sizeof(u16);
+	f->sizeimage = f->width * f->height * sizeof(u16);
+
+	return 0;
+}
+
+static int rmi_f54_vidioc_s_input(struct file *file, void *priv, unsigned int i)
+{
+	return rmi_f54_set_input(video_drvdata(file), i);
+}
+
+static int rmi_f54_vidioc_g_input(struct file *file, void *priv,
+				  unsigned int *i)
+{
+	struct f54_data *f54 = video_drvdata(file);
+
+	*i = f54->input;
+
+	return 0;
+}
+
+static int rmi_f54_vidioc_fmt(struct file *file, void *priv,
+			      struct v4l2_format *f)
+{
+	struct f54_data *f54 = video_drvdata(file);
+
+	f->fmt.pix = f54->format;
+
+	return 0;
+}
+
+static int rmi_f54_vidioc_enum_fmt(struct file *file, void *priv,
+				   struct v4l2_fmtdesc *fmt)
+{
+	if (fmt->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	switch (fmt->index) {
+	case 0:
+		fmt->pixelformat = V4L2_TCH_FMT_DELTA_TD16;
+		break;
+
+	case 1:
+		fmt->pixelformat = V4L2_TCH_FMT_DELTA_TD08;
+		break;
+
+	case 2:
+		fmt->pixelformat = V4L2_TCH_FMT_TU16;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int rmi_f54_vidioc_g_parm(struct file *file, void *fh,
+				 struct v4l2_streamparm *a)
+{
+	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	a->parm.capture.readbuffers = 1;
+	a->parm.capture.timeperframe.numerator = 1;
+	a->parm.capture.timeperframe.denominator = 10;
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops rmi_f54_video_ioctl_ops = {
+	.vidioc_querycap	= rmi_f54_vidioc_querycap,
+
+	.vidioc_enum_fmt_vid_cap = rmi_f54_vidioc_enum_fmt,
+	.vidioc_s_fmt_vid_cap	= rmi_f54_vidioc_fmt,
+	.vidioc_g_fmt_vid_cap	= rmi_f54_vidioc_fmt,
+	.vidioc_try_fmt_vid_cap	= rmi_f54_vidioc_fmt,
+	.vidioc_g_parm		= rmi_f54_vidioc_g_parm,
+
+	.vidioc_enum_input	= rmi_f54_vidioc_enum_input,
+	.vidioc_g_input		= rmi_f54_vidioc_g_input,
+	.vidioc_s_input		= rmi_f54_vidioc_s_input,
+
+	.vidioc_reqbufs		= vb2_ioctl_reqbufs,
+	.vidioc_create_bufs	= vb2_ioctl_create_bufs,
+	.vidioc_querybuf	= vb2_ioctl_querybuf,
+	.vidioc_qbuf		= vb2_ioctl_qbuf,
+	.vidioc_dqbuf		= vb2_ioctl_dqbuf,
+	.vidioc_expbuf		= vb2_ioctl_expbuf,
+
+	.vidioc_streamon	= vb2_ioctl_streamon,
+	.vidioc_streamoff	= vb2_ioctl_streamoff,
+};
+
+static const struct video_device rmi_f54_video_device = {
+	.name = "Synaptics RMI4",
+	.fops = &rmi_f54_video_fops,
+	.ioctl_ops = &rmi_f54_video_ioctl_ops,
+	.release = video_device_release_empty,
+	.device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_TOUCH |
+		       V4L2_CAP_READWRITE | V4L2_CAP_STREAMING,
+};
+
+static void rmi_f54_work(struct work_struct *work)
+{
+	struct f54_data *f54 = container_of(work, struct f54_data, work.work);
+	struct rmi_function *fn = f54->fn;
+	u8 fifo[2];
+	struct rmi_f54_reports *report;
+	int report_size;
+	u8 command;
+	u8 *data;
+	int error;
+
+	data = f54->report_data;
+	report_size = rmi_f54_get_report_size(f54);
+	if (report_size == 0) {
+		dev_err(&fn->dev, "Bad report size, report type=%d\n",
+				f54->report_type);
+		error = -EINVAL;
+		goto error;     /* retry won't help */
+	}
+	f54->standard_report[0].size = report_size;
+	report = f54->standard_report;
+
+	mutex_lock(&f54->data_mutex);
+
+	/*
+	 * Need to check if command has completed.
+	 * If not try again later.
+	 */
+	error = rmi_read(fn->rmi_dev, f54->fn->fd.command_base_addr,
+			 &command);
+	if (error) {
+		dev_err(&fn->dev, "Failed to read back command\n");
+		goto error;
+	}
+	if (command & F54_GET_REPORT) {
+		if (time_after(jiffies, f54->timeout)) {
+			dev_err(&fn->dev, "Get report command timed out\n");
+			error = -ETIMEDOUT;
+		}
+		report_size = 0;
+		goto error;
+	}
+
+	rmi_dbg(RMI_DEBUG_FN, &fn->dev, "Get report command completed, reading data\n");
+
+	report_size = 0;
+	for (; report->size; report++) {
+		fifo[0] = report->start & 0xff;
+		fifo[1] = (report->start >> 8) & 0xff;
+		error = rmi_write_block(fn->rmi_dev,
+					fn->fd.data_base_addr + F54_FIFO_OFFSET,
+					fifo, sizeof(fifo));
+		if (error) {
+			dev_err(&fn->dev, "Failed to set fifo start offset\n");
+			goto abort;
+		}
+
+		error = rmi_read_block(fn->rmi_dev, fn->fd.data_base_addr +
+				       F54_REPORT_DATA_OFFSET, data,
+				       report->size);
+		if (error) {
+			dev_err(&fn->dev, "%s: read [%d bytes] returned %d\n",
+				__func__, report->size, error);
+			goto abort;
+		}
+		data += report->size;
+		report_size += report->size;
+	}
+
+abort:
+	f54->report_size = error ? 0 : report_size;
+error:
+	if (error)
+		report_size = 0;
+
+	if (report_size == 0 && !error) {
+		queue_delayed_work(f54->workqueue, &f54->work,
+				   msecs_to_jiffies(1));
+	} else {
+		f54->is_busy = false;
+		complete(&f54->cmd_done);
+	}
+
+	mutex_unlock(&f54->data_mutex);
+}
+
+static int rmi_f54_attention(struct rmi_function *fn, unsigned long *irqbits)
+{
+	return 0;
+}
+
+static int rmi_f54_config(struct rmi_function *fn)
+{
+	struct rmi_driver *drv = fn->rmi_dev->driver;
+
+	drv->set_irq_bits(fn->rmi_dev, fn->irq_mask);
+
+	return 0;
+}
+
+static int rmi_f54_detect(struct rmi_function *fn)
+{
+	int error;
+	struct f54_data *f54;
+
+	f54 = dev_get_drvdata(&fn->dev);
+
+	error = rmi_read_block(fn->rmi_dev, fn->fd.query_base_addr,
+			       &f54->qry, sizeof(f54->qry));
+	if (error) {
+		dev_err(&fn->dev, "%s: Failed to query F54 properties\n",
+			__func__);
+		return error;
+	}
+
+	f54->num_rx_electrodes = f54->qry[0];
+	f54->num_tx_electrodes = f54->qry[1];
+	f54->capabilities = f54->qry[2];
+	f54->clock_rate = f54->qry[3] | (f54->qry[4] << 8);
+	f54->family = f54->qry[5];
+
+	rmi_dbg(RMI_DEBUG_FN, &fn->dev, "F54 num_rx_electrodes: %d\n",
+		f54->num_rx_electrodes);
+	rmi_dbg(RMI_DEBUG_FN, &fn->dev, "F54 num_tx_electrodes: %d\n",
+		f54->num_tx_electrodes);
+	rmi_dbg(RMI_DEBUG_FN, &fn->dev, "F54 capabilities: 0x%x\n",
+		f54->capabilities);
+	rmi_dbg(RMI_DEBUG_FN, &fn->dev, "F54 clock rate: 0x%x\n",
+		f54->clock_rate);
+	rmi_dbg(RMI_DEBUG_FN, &fn->dev, "F54 family: 0x%x\n",
+		f54->family);
+
+	f54->is_busy = false;
+
+	return 0;
+}
+
+static int rmi_f54_probe(struct rmi_function *fn)
+{
+	struct f54_data *f54;
+	int ret;
+	u8 rx, tx;
+
+	f54 = devm_kzalloc(&fn->dev, sizeof(struct f54_data), GFP_KERNEL);
+	if (!f54)
+		return -ENOMEM;
+
+	f54->fn = fn;
+	dev_set_drvdata(&fn->dev, f54);
+
+	ret = rmi_f54_detect(fn);
+	if (ret)
+		return ret;
+
+	mutex_init(&f54->data_mutex);
+	mutex_init(&f54->status_mutex);
+
+	rx = f54->num_rx_electrodes;
+	tx = f54->num_tx_electrodes;
+	f54->report_data = devm_kzalloc(&fn->dev,
+					sizeof(u16) * tx * rx,
+					GFP_KERNEL);
+	if (f54->report_data == NULL)
+		return -ENOMEM;
+
+	INIT_DELAYED_WORK(&f54->work, rmi_f54_work);
+
+	f54->workqueue = create_singlethread_workqueue("rmi4-poller");
+	if (!f54->workqueue)
+		return -ENOMEM;
+
+	rmi_f54_create_input_map(f54);
+
+	/* register video device */
+	strlcpy(f54->v4l2.name, F54_NAME, sizeof(f54->v4l2.name));
+	ret = v4l2_device_register(&fn->dev, &f54->v4l2);
+	if (ret) {
+		dev_err(&fn->dev, "Unable to register video dev.\n");
+		goto remove_wq;
+	}
+
+	/* initialize the queue */
+	mutex_init(&f54->lock);
+	f54->queue = rmi_f54_queue;
+	f54->queue.drv_priv = f54;
+	f54->queue.lock = &f54->lock;
+	f54->queue.dev = &fn->dev;
+
+	ret = vb2_queue_init(&f54->queue);
+	if (ret)
+		goto remove_v4l2;
+
+	f54->vdev = rmi_f54_video_device;
+	f54->vdev.v4l2_dev = &f54->v4l2;
+	f54->vdev.lock = &f54->lock;
+	f54->vdev.vfl_dir = VFL_DIR_RX;
+	f54->vdev.queue = &f54->queue;
+	video_set_drvdata(&f54->vdev, f54);
+
+	ret = video_register_device(&f54->vdev, VFL_TYPE_TOUCH, -1);
+	if (ret) {
+		dev_err(&fn->dev, "Unable to register video subdevice.");
+		goto remove_v4l2;
+	}
+
+	return 0;
+
+remove_v4l2:
+	v4l2_device_unregister(&f54->v4l2);
+remove_wq:
+	cancel_delayed_work_sync(&f54->work);
+	flush_workqueue(f54->workqueue);
+	destroy_workqueue(f54->workqueue);
+	return ret;
+}
+
+static void rmi_f54_remove(struct rmi_function *fn)
+{
+	struct f54_data *f54 = dev_get_drvdata(&fn->dev);
+
+	video_unregister_device(&f54->vdev);
+	v4l2_device_unregister(&f54->v4l2);
+}
+
+struct rmi_function_handler rmi_f54_handler = {
+	.driver = {
+		.name = F54_NAME,
+	},
+	.func = 0x54,
+	.probe = rmi_f54_probe,
+	.config = rmi_f54_config,
+	.attention = rmi_f54_attention,
+	.remove = rmi_f54_remove,
+};
diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c
index b4d34086e73f..405252a884dd 100644
--- a/drivers/input/serio/i8042.c
+++ b/drivers/input/serio/i8042.c
@@ -1305,6 +1305,7 @@ static int __init i8042_create_aux_port(int idx)
 	serio->write		= i8042_aux_write;
 	serio->start		= i8042_start;
 	serio->stop		= i8042_stop;
+	serio->ps2_cmd_mutex	= &i8042_mutex;
 	serio->port_data	= port;
 	serio->dev.parent	= &i8042_platform_device->dev;
 	if (idx < 0) {
diff --git a/drivers/input/serio/serport.c b/drivers/input/serio/serport.c
index 9c927d35c1f5..d189843f3727 100644
--- a/drivers/input/serio/serport.c
+++ b/drivers/input/serio/serport.c
@@ -71,10 +71,7 @@ static void serport_serio_close(struct serio *serio)
 
 	spin_lock_irqsave(&serport->lock, flags);
 	clear_bit(SERPORT_ACTIVE, &serport->flags);
-	set_bit(SERPORT_DEAD, &serport->flags);
 	spin_unlock_irqrestore(&serport->lock, flags);
-
-	wake_up_interruptible(&serport->wait);
 }
 
 /*
@@ -248,6 +245,19 @@ static long serport_ldisc_compat_ioctl(struct tty_struct *tty,
 }
 #endif
 
+static int serport_ldisc_hangup(struct tty_struct *tty)
+{
+	struct serport *serport = (struct serport *) tty->disc_data;
+	unsigned long flags;
+
+	spin_lock_irqsave(&serport->lock, flags);
+	set_bit(SERPORT_DEAD, &serport->flags);
+	spin_unlock_irqrestore(&serport->lock, flags);
+
+	wake_up_interruptible(&serport->wait);
+	return 0;
+}
+
 static void serport_ldisc_write_wakeup(struct tty_struct * tty)
 {
 	struct serport *serport = (struct serport *) tty->disc_data;
@@ -274,6 +284,7 @@ static struct tty_ldisc_ops serport_ldisc = {
 	.compat_ioctl =	serport_ldisc_compat_ioctl,
 #endif
 	.receive_buf =	serport_ldisc_receive,
+	.hangup =	serport_ldisc_hangup,
 	.write_wakeup =	serport_ldisc_write_wakeup
 };
 
diff --git a/drivers/input/tablet/pegasus_notetaker.c b/drivers/input/tablet/pegasus_notetaker.c
index 949dacc78664..47de5a81172f 100644
--- a/drivers/input/tablet/pegasus_notetaker.c
+++ b/drivers/input/tablet/pegasus_notetaker.c
@@ -40,6 +40,7 @@
 #include <linux/input.h>
 #include <linux/usb/input.h>
 #include <linux/slab.h>
+#include <linux/workqueue.h>
 
 /* USB HID defines */
 #define USB_REQ_GET_REPORT		0x01
diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig
index 2fb1f430a431..efca0133e266 100644
--- a/drivers/input/touchscreen/Kconfig
+++ b/drivers/input/touchscreen/Kconfig
@@ -115,6 +115,15 @@ config TOUCHSCREEN_ATMEL_MXT
 	  To compile this driver as a module, choose M here: the
 	  module will be called atmel_mxt_ts.
 
+config TOUCHSCREEN_ATMEL_MXT_T37
+	bool "Support T37 Diagnostic Data"
+	depends on TOUCHSCREEN_ATMEL_MXT
+	depends on VIDEO_V4L2=y || (TOUCHSCREEN_ATMEL_MXT=m && VIDEO_V4L2=m)
+	select VIDEOBUF2_VMALLOC
+	help
+	  Say Y here if you want support to output data from the T37
+	  Diagnostic Data object using a V4L device.
+
 config TOUCHSCREEN_AUO_PIXCIR
 	tristate "AUO in-cell touchscreen using Pixcir ICs"
 	depends on I2C
@@ -305,19 +314,6 @@ config TOUCHSCREEN_EGALAX_SERIAL
 	  To compile this driver as a module, choose M here: the
 	  module will be called egalax_ts_serial.
 
-config TOUCHSCREEN_FT6236
-	tristate "FT6236 I2C touchscreen"
-	depends on I2C
-	depends on GPIOLIB || COMPILE_TEST
-	help
-	  Say Y here to enable support for the I2C connected FT6x06 and
-	  FT6x36 family of capacitive touchscreen drivers.
-
-	  If unsure, say N.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ft6236.
-
 config TOUCHSCREEN_FUJITSU
 	tristate "Fujitsu serial touchscreen"
 	select SERIO
@@ -397,6 +393,18 @@ config TOUCHSCREEN_GUNZE
 	  To compile this driver as a module, choose M here: the
 	  module will be called gunze.
 
+config TOUCHSCREEN_EKTF2127
+	tristate "Elan eKTF2127 I2C touchscreen"
+	depends on I2C
+	help
+	  Say Y here if you have an Elan eKTF2127 touchscreen
+	  connected to your system.
+
+	  If unsure, say N.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ektf2127.
+
 config TOUCHSCREEN_ELAN
 	tristate "Elan eKTH I2C touchscreen"
 	depends on I2C
diff --git a/drivers/input/touchscreen/Makefile b/drivers/input/touchscreen/Makefile
index b4373d6be402..81b86451782d 100644
--- a/drivers/input/touchscreen/Makefile
+++ b/drivers/input/touchscreen/Makefile
@@ -32,11 +32,11 @@ obj-$(CONFIG_TOUCHSCREEN_EDT_FT5X06)	+= edt-ft5x06.o
 obj-$(CONFIG_TOUCHSCREEN_HAMPSHIRE)	+= hampshire.o
 obj-$(CONFIG_TOUCHSCREEN_GUNZE)		+= gunze.o
 obj-$(CONFIG_TOUCHSCREEN_EETI)		+= eeti_ts.o
+obj-$(CONFIG_TOUCHSCREEN_EKTF2127)	+= ektf2127.o
 obj-$(CONFIG_TOUCHSCREEN_ELAN)		+= elants_i2c.o
 obj-$(CONFIG_TOUCHSCREEN_ELO)		+= elo.o
 obj-$(CONFIG_TOUCHSCREEN_EGALAX)	+= egalax_ts.o
 obj-$(CONFIG_TOUCHSCREEN_EGALAX_SERIAL)	+= egalax_ts_serial.o
-obj-$(CONFIG_TOUCHSCREEN_FT6236)	+= ft6236.o
 obj-$(CONFIG_TOUCHSCREEN_FUJITSU)	+= fujitsu_ts.o
 obj-$(CONFIG_TOUCHSCREEN_GOODIX)	+= goodix.o
 obj-$(CONFIG_TOUCHSCREEN_ILI210X)	+= ili210x.o
diff --git a/drivers/input/touchscreen/ads7846.c b/drivers/input/touchscreen/ads7846.c
index a61b2153ab8c..1ce3ecbe37f8 100644
--- a/drivers/input/touchscreen/ads7846.c
+++ b/drivers/input/touchscreen/ads7846.c
@@ -1473,7 +1473,6 @@ static int ads7846_remove(struct spi_device *spi)
 
 	ads784x_hwmon_unregister(spi, ts);
 
-	regulator_disable(ts->reg);
 	regulator_put(ts->reg);
 
 	if (!ts->get_pendown_state) {
diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c
index 5af7907d0af4..e5d185fe69b9 100644
--- a/drivers/input/touchscreen/atmel_mxt_ts.c
+++ b/drivers/input/touchscreen/atmel_mxt_ts.c
@@ -4,6 +4,7 @@
  * Copyright (C) 2010 Samsung Electronics Co.Ltd
  * Copyright (C) 2011-2014 Atmel Corporation
  * Copyright (C) 2012 Google, Inc.
+ * Copyright (C) 2016 Zodiac Inflight Innovations
  *
  * Author: Joonyoung Shim <jy0922.shim@samsung.com>
  *
@@ -28,6 +29,10 @@
 #include <linux/of.h>
 #include <linux/slab.h>
 #include <asm/unaligned.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-vmalloc.h>
 
 /* Firmware files */
 #define MXT_FW_NAME		"maxtouch.fw"
@@ -99,6 +104,8 @@ struct t7_config {
 
 /* MXT_TOUCH_MULTI_T9 field */
 #define MXT_T9_CTRL		0
+#define MXT_T9_XSIZE		3
+#define MXT_T9_YSIZE		4
 #define MXT_T9_ORIENT		9
 #define MXT_T9_RANGE		18
 
@@ -119,11 +126,31 @@ struct t9_range {
 
 /* MXT_TOUCH_MULTI_T9 orient */
 #define MXT_T9_ORIENT_SWITCH	(1 << 0)
+#define MXT_T9_ORIENT_INVERTX	(1 << 1)
+#define MXT_T9_ORIENT_INVERTY	(1 << 2)
 
 /* MXT_SPT_COMMSCONFIG_T18 */
 #define MXT_COMMS_CTRL		0
 #define MXT_COMMS_CMD		1
 
+/* MXT_DEBUG_DIAGNOSTIC_T37 */
+#define MXT_DIAGNOSTIC_PAGEUP	0x01
+#define MXT_DIAGNOSTIC_DELTAS	0x10
+#define MXT_DIAGNOSTIC_REFS	0x11
+#define MXT_DIAGNOSTIC_SIZE	128
+
+#define MXT_FAMILY_1386			160
+#define MXT1386_COLUMNS			3
+#define MXT1386_PAGES_PER_COLUMN	8
+
+struct t37_debug {
+#ifdef CONFIG_TOUCHSCREEN_ATMEL_MXT_T37
+	u8 mode;
+	u8 page;
+	u8 data[MXT_DIAGNOSTIC_SIZE];
+#endif
+};
+
 /* Define for MXT_GEN_COMMAND_T6 */
 #define MXT_BOOT_VALUE		0xa5
 #define MXT_RESET_VALUE		0x01
@@ -133,10 +160,14 @@ struct t9_range {
 #define MXT_T100_CTRL		0
 #define MXT_T100_CFG1		1
 #define MXT_T100_TCHAUX		3
+#define MXT_T100_XSIZE		9
 #define MXT_T100_XRANGE		13
+#define MXT_T100_YSIZE		20
 #define MXT_T100_YRANGE		24
 
 #define MXT_T100_CFG_SWITCHXY	BIT(5)
+#define MXT_T100_CFG_INVERTY	BIT(6)
+#define MXT_T100_CFG_INVERTX	BIT(7)
 
 #define MXT_T100_TCHAUX_VECT	BIT(0)
 #define MXT_T100_TCHAUX_AMPL	BIT(1)
@@ -205,6 +236,37 @@ struct mxt_object {
 	u8 num_report_ids;
 } __packed;
 
+struct mxt_dbg {
+	u16 t37_address;
+	u16 diag_cmd_address;
+	struct t37_debug *t37_buf;
+	unsigned int t37_pages;
+	unsigned int t37_nodes;
+
+	struct v4l2_device v4l2;
+	struct v4l2_pix_format format;
+	struct video_device vdev;
+	struct vb2_queue queue;
+	struct mutex lock;
+	int input;
+};
+
+enum v4l_dbg_inputs {
+	MXT_V4L_INPUT_DELTAS,
+	MXT_V4L_INPUT_REFS,
+	MXT_V4L_INPUT_MAX,
+};
+
+static const struct v4l2_file_operations mxt_video_fops = {
+	.owner = THIS_MODULE,
+	.open = v4l2_fh_open,
+	.release = vb2_fop_release,
+	.unlocked_ioctl = video_ioctl2,
+	.read = vb2_fop_read,
+	.mmap = vb2_fop_mmap,
+	.poll = vb2_fop_poll,
+};
+
 /* Each client has this additional data */
 struct mxt_data {
 	struct i2c_client *client;
@@ -216,7 +278,11 @@ struct mxt_data {
 	unsigned int irq;
 	unsigned int max_x;
 	unsigned int max_y;
+	bool invertx;
+	bool inverty;
 	bool xy_switch;
+	u8 xsize;
+	u8 ysize;
 	bool in_bootloader;
 	u16 mem_size;
 	u8 t100_aux_ampl;
@@ -233,6 +299,7 @@ struct mxt_data {
 	u8 num_touchids;
 	u8 multitouch;
 	struct t7_config t7_cfg;
+	struct mxt_dbg dbg;
 
 	/* Cached parameters from object table */
 	u16 T5_address;
@@ -257,6 +324,11 @@ struct mxt_data {
 	struct completion crc_completion;
 };
 
+struct mxt_vb2_buffer {
+	struct vb2_buffer	vb;
+	struct list_head	list;
+};
+
 static size_t mxt_obj_size(const struct mxt_object *obj)
 {
 	return obj->size_minus_one + 1;
@@ -1503,6 +1575,11 @@ static void mxt_free_input_device(struct mxt_data *data)
 
 static void mxt_free_object_table(struct mxt_data *data)
 {
+#ifdef CONFIG_TOUCHSCREEN_ATMEL_MXT_T37
+	video_unregister_device(&data->dbg.vdev);
+	v4l2_device_unregister(&data->dbg.v4l2);
+#endif
+
 	kfree(data->object_table);
 	data->object_table = NULL;
 	kfree(data->msg_buf);
@@ -1661,6 +1738,18 @@ static int mxt_read_t9_resolution(struct mxt_data *data)
 		return -EINVAL;
 
 	error = __mxt_read_reg(client,
+			       object->start_address + MXT_T9_XSIZE,
+			       sizeof(data->xsize), &data->xsize);
+	if (error)
+		return error;
+
+	error = __mxt_read_reg(client,
+			       object->start_address + MXT_T9_YSIZE,
+			       sizeof(data->ysize), &data->ysize);
+	if (error)
+		return error;
+
+	error = __mxt_read_reg(client,
 			       object->start_address + MXT_T9_RANGE,
 			       sizeof(range), &range);
 	if (error)
@@ -1676,6 +1765,8 @@ static int mxt_read_t9_resolution(struct mxt_data *data)
 		return error;
 
 	data->xy_switch = orient & MXT_T9_ORIENT_SWITCH;
+	data->invertx = orient & MXT_T9_ORIENT_INVERTX;
+	data->inverty = orient & MXT_T9_ORIENT_INVERTY;
 
 	return 0;
 }
@@ -1710,6 +1801,18 @@ static int mxt_read_t100_config(struct mxt_data *data)
 
 	data->max_y = get_unaligned_le16(&range_y);
 
+	error = __mxt_read_reg(client,
+			       object->start_address + MXT_T100_XSIZE,
+			       sizeof(data->xsize), &data->xsize);
+	if (error)
+		return error;
+
+	error = __mxt_read_reg(client,
+			       object->start_address + MXT_T100_YSIZE,
+			       sizeof(data->ysize), &data->ysize);
+	if (error)
+		return error;
+
 	/* read orientation config */
 	error =  __mxt_read_reg(client,
 				object->start_address + MXT_T100_CFG1,
@@ -1718,6 +1821,8 @@ static int mxt_read_t100_config(struct mxt_data *data)
 		return error;
 
 	data->xy_switch = cfg & MXT_T100_CFG_SWITCHXY;
+	data->invertx = cfg & MXT_T100_CFG_INVERTX;
+	data->inverty = cfg & MXT_T100_CFG_INVERTY;
 
 	/* allocate aux bytes */
 	error =  __mxt_read_reg(client,
@@ -2043,6 +2148,420 @@ recheck:
 	return 0;
 }
 
+#ifdef CONFIG_TOUCHSCREEN_ATMEL_MXT_T37
+static u16 mxt_get_debug_value(struct mxt_data *data, unsigned int x,
+			       unsigned int y)
+{
+	struct mxt_info *info = &data->info;
+	struct mxt_dbg *dbg = &data->dbg;
+	unsigned int ofs, page;
+	unsigned int col = 0;
+	unsigned int col_width;
+
+	if (info->family_id == MXT_FAMILY_1386) {
+		col_width = info->matrix_ysize / MXT1386_COLUMNS;
+		col = y / col_width;
+		y = y % col_width;
+	} else {
+		col_width = info->matrix_ysize;
+	}
+
+	ofs = (y + (x * col_width)) * sizeof(u16);
+	page = ofs / MXT_DIAGNOSTIC_SIZE;
+	ofs %= MXT_DIAGNOSTIC_SIZE;
+
+	if (info->family_id == MXT_FAMILY_1386)
+		page += col * MXT1386_PAGES_PER_COLUMN;
+
+	return get_unaligned_le16(&dbg->t37_buf[page].data[ofs]);
+}
+
+static int mxt_convert_debug_pages(struct mxt_data *data, u16 *outbuf)
+{
+	struct mxt_dbg *dbg = &data->dbg;
+	unsigned int x = 0;
+	unsigned int y = 0;
+	unsigned int i, rx, ry;
+
+	for (i = 0; i < dbg->t37_nodes; i++) {
+		/* Handle orientation */
+		rx = data->xy_switch ? y : x;
+		ry = data->xy_switch ? x : y;
+		rx = data->invertx ? (data->xsize - 1 - rx) : rx;
+		ry = data->inverty ? (data->ysize - 1 - ry) : ry;
+
+		outbuf[i] = mxt_get_debug_value(data, rx, ry);
+
+		/* Next value */
+		if (++x >= (data->xy_switch ? data->ysize : data->xsize)) {
+			x = 0;
+			y++;
+		}
+	}
+
+	return 0;
+}
+
+static int mxt_read_diagnostic_debug(struct mxt_data *data, u8 mode,
+				     u16 *outbuf)
+{
+	struct mxt_dbg *dbg = &data->dbg;
+	int retries = 0;
+	int page;
+	int ret;
+	u8 cmd = mode;
+	struct t37_debug *p;
+	u8 cmd_poll;
+
+	for (page = 0; page < dbg->t37_pages; page++) {
+		p = dbg->t37_buf + page;
+
+		ret = mxt_write_reg(data->client, dbg->diag_cmd_address,
+				    cmd);
+		if (ret)
+			return ret;
+
+		retries = 0;
+		msleep(20);
+wait_cmd:
+		/* Read back command byte */
+		ret = __mxt_read_reg(data->client, dbg->diag_cmd_address,
+				     sizeof(cmd_poll), &cmd_poll);
+		if (ret)
+			return ret;
+
+		/* Field is cleared once the command has been processed */
+		if (cmd_poll) {
+			if (retries++ > 100)
+				return -EINVAL;
+
+			msleep(20);
+			goto wait_cmd;
+		}
+
+		/* Read T37 page */
+		ret = __mxt_read_reg(data->client, dbg->t37_address,
+				     sizeof(struct t37_debug), p);
+		if (ret)
+			return ret;
+
+		if (p->mode != mode || p->page != page) {
+			dev_err(&data->client->dev, "T37 page mismatch\n");
+			return -EINVAL;
+		}
+
+		dev_dbg(&data->client->dev, "%s page:%d retries:%d\n",
+			__func__, page, retries);
+
+		/* For remaining pages, write PAGEUP rather than mode */
+		cmd = MXT_DIAGNOSTIC_PAGEUP;
+	}
+
+	return mxt_convert_debug_pages(data, outbuf);
+}
+
+static int mxt_queue_setup(struct vb2_queue *q,
+		       unsigned int *nbuffers, unsigned int *nplanes,
+		       unsigned int sizes[], struct device *alloc_devs[])
+{
+	struct mxt_data *data = q->drv_priv;
+	size_t size = data->dbg.t37_nodes * sizeof(u16);
+
+	if (*nplanes)
+		return sizes[0] < size ? -EINVAL : 0;
+
+	*nplanes = 1;
+	sizes[0] = size;
+
+	return 0;
+}
+
+static void mxt_buffer_queue(struct vb2_buffer *vb)
+{
+	struct mxt_data *data = vb2_get_drv_priv(vb->vb2_queue);
+	u16 *ptr;
+	int ret;
+	u8 mode;
+
+	ptr = vb2_plane_vaddr(vb, 0);
+	if (!ptr) {
+		dev_err(&data->client->dev, "Error acquiring frame ptr\n");
+		goto fault;
+	}
+
+	switch (data->dbg.input) {
+	case MXT_V4L_INPUT_DELTAS:
+	default:
+		mode = MXT_DIAGNOSTIC_DELTAS;
+		break;
+
+	case MXT_V4L_INPUT_REFS:
+		mode = MXT_DIAGNOSTIC_REFS;
+		break;
+	}
+
+	ret = mxt_read_diagnostic_debug(data, mode, ptr);
+	if (ret)
+		goto fault;
+
+	vb2_set_plane_payload(vb, 0, data->dbg.t37_nodes * sizeof(u16));
+	vb2_buffer_done(vb, VB2_BUF_STATE_DONE);
+	return;
+
+fault:
+	vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
+}
+
+/* V4L2 structures */
+static const struct vb2_ops mxt_queue_ops = {
+	.queue_setup		= mxt_queue_setup,
+	.buf_queue		= mxt_buffer_queue,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+};
+
+static const struct vb2_queue mxt_queue = {
+	.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
+	.io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF | VB2_READ,
+	.buf_struct_size = sizeof(struct mxt_vb2_buffer),
+	.ops = &mxt_queue_ops,
+	.mem_ops = &vb2_vmalloc_memops,
+	.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC,
+	.min_buffers_needed = 1,
+};
+
+static int mxt_vidioc_querycap(struct file *file, void *priv,
+				 struct v4l2_capability *cap)
+{
+	struct mxt_data *data = video_drvdata(file);
+
+	strlcpy(cap->driver, "atmel_mxt_ts", sizeof(cap->driver));
+	strlcpy(cap->card, "atmel_mxt_ts touch", sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info),
+		 "I2C:%s", dev_name(&data->client->dev));
+	return 0;
+}
+
+static int mxt_vidioc_enum_input(struct file *file, void *priv,
+				   struct v4l2_input *i)
+{
+	if (i->index >= MXT_V4L_INPUT_MAX)
+		return -EINVAL;
+
+	i->type = V4L2_INPUT_TYPE_TOUCH;
+
+	switch (i->index) {
+	case MXT_V4L_INPUT_REFS:
+		strlcpy(i->name, "Mutual Capacitance References",
+			sizeof(i->name));
+		break;
+	case MXT_V4L_INPUT_DELTAS:
+		strlcpy(i->name, "Mutual Capacitance Deltas", sizeof(i->name));
+		break;
+	}
+
+	return 0;
+}
+
+static int mxt_set_input(struct mxt_data *data, unsigned int i)
+{
+	struct v4l2_pix_format *f = &data->dbg.format;
+
+	if (i >= MXT_V4L_INPUT_MAX)
+		return -EINVAL;
+
+	if (i == MXT_V4L_INPUT_DELTAS)
+		f->pixelformat = V4L2_TCH_FMT_DELTA_TD16;
+	else
+		f->pixelformat = V4L2_TCH_FMT_TU16;
+
+	f->width = data->xy_switch ? data->ysize : data->xsize;
+	f->height = data->xy_switch ? data->xsize : data->ysize;
+	f->field = V4L2_FIELD_NONE;
+	f->colorspace = V4L2_COLORSPACE_RAW;
+	f->bytesperline = f->width * sizeof(u16);
+	f->sizeimage = f->width * f->height * sizeof(u16);
+
+	data->dbg.input = i;
+
+	return 0;
+}
+
+static int mxt_vidioc_s_input(struct file *file, void *priv, unsigned int i)
+{
+	return mxt_set_input(video_drvdata(file), i);
+}
+
+static int mxt_vidioc_g_input(struct file *file, void *priv, unsigned int *i)
+{
+	struct mxt_data *data = video_drvdata(file);
+
+	*i = data->dbg.input;
+
+	return 0;
+}
+
+static int mxt_vidioc_fmt(struct file *file, void *priv, struct v4l2_format *f)
+{
+	struct mxt_data *data = video_drvdata(file);
+
+	f->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	f->fmt.pix = data->dbg.format;
+
+	return 0;
+}
+
+static int mxt_vidioc_enum_fmt(struct file *file, void *priv,
+				 struct v4l2_fmtdesc *fmt)
+{
+	if (fmt->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	switch (fmt->index) {
+	case 0:
+		fmt->pixelformat = V4L2_TCH_FMT_TU16;
+		break;
+
+	case 1:
+		fmt->pixelformat = V4L2_TCH_FMT_DELTA_TD16;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int mxt_vidioc_g_parm(struct file *file, void *fh,
+			     struct v4l2_streamparm *a)
+{
+	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	a->parm.capture.readbuffers = 1;
+	a->parm.capture.timeperframe.numerator = 1;
+	a->parm.capture.timeperframe.denominator = 10;
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops mxt_video_ioctl_ops = {
+	.vidioc_querycap        = mxt_vidioc_querycap,
+
+	.vidioc_enum_fmt_vid_cap = mxt_vidioc_enum_fmt,
+	.vidioc_s_fmt_vid_cap   = mxt_vidioc_fmt,
+	.vidioc_g_fmt_vid_cap   = mxt_vidioc_fmt,
+	.vidioc_try_fmt_vid_cap	= mxt_vidioc_fmt,
+	.vidioc_g_parm		= mxt_vidioc_g_parm,
+
+	.vidioc_enum_input      = mxt_vidioc_enum_input,
+	.vidioc_g_input         = mxt_vidioc_g_input,
+	.vidioc_s_input         = mxt_vidioc_s_input,
+
+	.vidioc_reqbufs         = vb2_ioctl_reqbufs,
+	.vidioc_create_bufs     = vb2_ioctl_create_bufs,
+	.vidioc_querybuf        = vb2_ioctl_querybuf,
+	.vidioc_qbuf            = vb2_ioctl_qbuf,
+	.vidioc_dqbuf           = vb2_ioctl_dqbuf,
+	.vidioc_expbuf          = vb2_ioctl_expbuf,
+
+	.vidioc_streamon        = vb2_ioctl_streamon,
+	.vidioc_streamoff       = vb2_ioctl_streamoff,
+};
+
+static const struct video_device mxt_video_device = {
+	.name = "Atmel maxTouch",
+	.fops = &mxt_video_fops,
+	.ioctl_ops = &mxt_video_ioctl_ops,
+	.release = video_device_release_empty,
+	.device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_TOUCH |
+		       V4L2_CAP_READWRITE | V4L2_CAP_STREAMING,
+};
+
+static void mxt_debug_init(struct mxt_data *data)
+{
+	struct mxt_info *info = &data->info;
+	struct mxt_dbg *dbg = &data->dbg;
+	struct mxt_object *object;
+	int error;
+
+	object = mxt_get_object(data, MXT_GEN_COMMAND_T6);
+	if (!object)
+		goto error;
+
+	dbg->diag_cmd_address = object->start_address + MXT_COMMAND_DIAGNOSTIC;
+
+	object = mxt_get_object(data, MXT_DEBUG_DIAGNOSTIC_T37);
+	if (!object)
+		goto error;
+
+	if (mxt_obj_size(object) != sizeof(struct t37_debug)) {
+		dev_warn(&data->client->dev, "Bad T37 size");
+		goto error;
+	}
+
+	dbg->t37_address = object->start_address;
+
+	/* Calculate size of data and allocate buffer */
+	dbg->t37_nodes = data->xsize * data->ysize;
+
+	if (info->family_id == MXT_FAMILY_1386)
+		dbg->t37_pages = MXT1386_COLUMNS * MXT1386_PAGES_PER_COLUMN;
+	else
+		dbg->t37_pages = DIV_ROUND_UP(data->xsize *
+					      data->info.matrix_ysize *
+					      sizeof(u16),
+					      sizeof(dbg->t37_buf->data));
+
+	dbg->t37_buf = devm_kmalloc_array(&data->client->dev, dbg->t37_pages,
+					  sizeof(struct t37_debug), GFP_KERNEL);
+	if (!dbg->t37_buf)
+		goto error;
+
+	/* init channel to zero */
+	mxt_set_input(data, 0);
+
+	/* register video device */
+	snprintf(dbg->v4l2.name, sizeof(dbg->v4l2.name), "%s", "atmel_mxt_ts");
+	error = v4l2_device_register(&data->client->dev, &dbg->v4l2);
+	if (error)
+		goto error;
+
+	/* initialize the queue */
+	mutex_init(&dbg->lock);
+	dbg->queue = mxt_queue;
+	dbg->queue.drv_priv = data;
+	dbg->queue.lock = &dbg->lock;
+	dbg->queue.dev = &data->client->dev;
+
+	error = vb2_queue_init(&dbg->queue);
+	if (error)
+		goto error_unreg_v4l2;
+
+	dbg->vdev = mxt_video_device;
+	dbg->vdev.v4l2_dev = &dbg->v4l2;
+	dbg->vdev.lock = &dbg->lock;
+	dbg->vdev.vfl_dir = VFL_DIR_RX;
+	dbg->vdev.queue = &dbg->queue;
+	video_set_drvdata(&dbg->vdev, data);
+
+	error = video_register_device(&dbg->vdev, VFL_TYPE_TOUCH, -1);
+	if (error)
+		goto error_unreg_v4l2;
+
+	return;
+
+error_unreg_v4l2:
+	v4l2_device_unregister(&dbg->v4l2);
+error:
+	dev_warn(&data->client->dev, "Error initializing T37\n");
+}
+#else
+static void mxt_debug_init(struct mxt_data *data)
+{
+}
+#endif
+
 static int mxt_configure_objects(struct mxt_data *data,
 				 const struct firmware *cfg)
 {
@@ -2070,6 +2589,8 @@ static int mxt_configure_objects(struct mxt_data *data,
 		dev_warn(dev, "No touch object detected\n");
 	}
 
+	mxt_debug_init(data);
+
 	dev_info(dev,
 		 "Family: %u Variant: %u Firmware V%u.%u.%02X Objects: %u\n",
 		 info->family_id, info->variant_id, info->version >> 4,
diff --git a/drivers/input/touchscreen/edt-ft5x06.c b/drivers/input/touchscreen/edt-ft5x06.c
index 703e295a37ed..28466e358fee 100644
--- a/drivers/input/touchscreen/edt-ft5x06.c
+++ b/drivers/input/touchscreen/edt-ft5x06.c
@@ -1063,9 +1063,15 @@ static const struct edt_i2c_chip_data edt_ft5506_data = {
 	.max_support_points = 10,
 };
 
+static const struct edt_i2c_chip_data edt_ft6236_data = {
+	.max_support_points = 2,
+};
+
 static const struct i2c_device_id edt_ft5x06_ts_id[] = {
 	{ .name = "edt-ft5x06", .driver_data = (long)&edt_ft5x06_data },
 	{ .name = "edt-ft5506", .driver_data = (long)&edt_ft5506_data },
+	/* Note no edt- prefix for compatibility with the ft6236.c driver */
+	{ .name = "ft6236", .driver_data = (long)&edt_ft6236_data },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, edt_ft5x06_ts_id);
@@ -1076,6 +1082,8 @@ static const struct of_device_id edt_ft5x06_of_match[] = {
 	{ .compatible = "edt,edt-ft5306", .data = &edt_ft5x06_data },
 	{ .compatible = "edt,edt-ft5406", .data = &edt_ft5x06_data },
 	{ .compatible = "edt,edt-ft5506", .data = &edt_ft5506_data },
+	/* Note focaltech vendor prefix for compatibility with ft6236.c */
+	{ .compatible = "focaltech,ft6236", .data = &edt_ft6236_data },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, edt_ft5x06_of_match);
diff --git a/drivers/input/touchscreen/ektf2127.c b/drivers/input/touchscreen/ektf2127.c
new file mode 100644
index 000000000000..0ed34ff787ce
--- /dev/null
+++ b/drivers/input/touchscreen/ektf2127.c
@@ -0,0 +1,336 @@
+/*
+ * Driver for ELAN eKTF2127 i2c touchscreen controller
+ *
+ * For this driver the layout of the Chipone icn8318 i2c
+ * touchscreencontroller is used.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * Author:
+ * Michel Verlaan <michel.verl@gmail.com>
+ * Siebren Vroegindeweij <siebren.vroegindeweij@hotmail.com>
+ *
+ * Original chipone_icn8318 driver:
+ * Hans de Goede <hdegoede@redhat.com>
+ */
+
+#include <linux/gpio/consumer.h>
+#include <linux/interrupt.h>
+#include <linux/i2c.h>
+#include <linux/input.h>
+#include <linux/input/mt.h>
+#include <linux/input/touchscreen.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/delay.h>
+
+/* Packet header defines (first byte of data send / received) */
+#define EKTF2127_NOISE			0x40
+#define EKTF2127_RESPONSE		0x52
+#define EKTF2127_REQUEST		0x53
+#define EKTF2127_HELLO			0x55
+#define EKTF2127_REPORT			0x5d
+#define EKTF2127_CALIB_DONE		0x66
+
+/* Register defines (second byte of data send / received) */
+#define EKTF2127_ENV_NOISY		0x41
+#define EKTF2127_HEIGHT			0x60
+#define EKTF2127_WIDTH			0x63
+
+/* 2 bytes header + 5 * 3 bytes coordinates + 3 bytes pressure info + footer */
+#define EKTF2127_TOUCH_REPORT_SIZE	21
+#define EKTF2127_MAX_TOUCHES		5
+
+struct ektf2127_ts {
+	struct i2c_client *client;
+	struct input_dev *input;
+	struct gpio_desc *power_gpios;
+	struct touchscreen_properties prop;
+};
+
+static void ektf2127_parse_coordinates(const u8* buf, unsigned int touch_count,
+				       struct input_mt_pos *touches)
+{
+	int index = 0;
+	int i;
+
+	for (i = 0; i < touch_count; i++) {
+		index = 2 + i * 3;
+
+		touches[i].x = (buf[index] & 0x0f);
+		touches[i].x <<= 8;
+		touches[i].x |= buf[index + 2];
+
+		touches[i].y = (buf[index] & 0xf0);
+		touches[i].y <<= 4;
+		touches[i].y |= buf[index + 1];
+	}
+}
+
+static void ektf2127_report_event(struct ektf2127_ts *ts, const u8 *buf)
+{
+	struct input_mt_pos touches[EKTF2127_MAX_TOUCHES];
+	int slots[EKTF2127_MAX_TOUCHES];
+	unsigned int touch_count, i;
+
+	touch_count = buf[1] & 0x07;
+	if (touch_count > EKTF2127_MAX_TOUCHES) {
+		dev_err(&ts->client->dev,
+			"Too many touches %d > %d\n",
+			touch_count, EKTF2127_MAX_TOUCHES);
+		touch_count = EKTF2127_MAX_TOUCHES;
+	}
+
+	ektf2127_parse_coordinates(buf, touch_count, touches);
+	input_mt_assign_slots(ts->input, slots, touches,
+			      touch_count, 0);
+
+	for (i = 0; i < touch_count; i++) {
+		input_mt_slot(ts->input, slots[i]);
+		input_mt_report_slot_state(ts->input, MT_TOOL_FINGER, true);
+		touchscreen_report_pos(ts->input, &ts->prop,
+				       touches[i].x, touches[i].y, true);
+	}
+
+	input_mt_sync_frame(ts->input);
+	input_sync(ts->input);
+}
+
+static irqreturn_t ektf2127_irq(int irq, void *dev_id)
+{
+	struct ektf2127_ts *ts = dev_id;
+	struct device *dev = &ts->client->dev;
+	char buf[EKTF2127_TOUCH_REPORT_SIZE];
+	int ret;
+
+	ret = i2c_master_recv(ts->client, buf, EKTF2127_TOUCH_REPORT_SIZE);
+	if (ret != EKTF2127_TOUCH_REPORT_SIZE) {
+		dev_err(dev, "Error reading touch data: %d\n", ret);
+		goto out;
+	}
+
+	switch (buf[0]) {
+	case EKTF2127_REPORT:
+		ektf2127_report_event(ts, buf);
+		break;
+
+	case EKTF2127_NOISE:
+		if (buf[1] == EKTF2127_ENV_NOISY)
+			dev_dbg(dev, "Environment is electrically noisy\n");
+		break;
+
+	case EKTF2127_HELLO:
+	case EKTF2127_CALIB_DONE:
+		break;
+
+	default:
+		dev_err(dev, "Unexpected packet header byte %#02x\n", buf[0]);
+		break;
+	}
+
+out:
+	return IRQ_HANDLED;
+}
+
+static int ektf2127_start(struct input_dev *dev)
+{
+	struct ektf2127_ts *ts = input_get_drvdata(dev);
+
+	enable_irq(ts->client->irq);
+	gpiod_set_value_cansleep(ts->power_gpios, 1);
+
+	return 0;
+}
+
+static void ektf2127_stop(struct input_dev *dev)
+{
+	struct ektf2127_ts *ts = input_get_drvdata(dev);
+
+	disable_irq(ts->client->irq);
+	gpiod_set_value_cansleep(ts->power_gpios, 0);
+}
+
+static int __maybe_unused ektf2127_suspend(struct device *dev)
+{
+	struct ektf2127_ts *ts = i2c_get_clientdata(to_i2c_client(dev));
+
+	mutex_lock(&ts->input->mutex);
+	if (ts->input->users)
+		ektf2127_stop(ts->input);
+	mutex_unlock(&ts->input->mutex);
+
+	return 0;
+}
+
+static int __maybe_unused ektf2127_resume(struct device *dev)
+{
+	struct ektf2127_ts *ts = i2c_get_clientdata(to_i2c_client(dev));
+
+	mutex_lock(&ts->input->mutex);
+	if (ts->input->users)
+		ektf2127_start(ts->input);
+	mutex_unlock(&ts->input->mutex);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(ektf2127_pm_ops, ektf2127_suspend,
+			 ektf2127_resume);
+
+static int ektf2127_query_dimension(struct i2c_client *client, bool width)
+{
+	struct device *dev = &client->dev;
+	const char *what = width ? "width" : "height";
+	u8 what_code = width ? EKTF2127_WIDTH : EKTF2127_HEIGHT;
+	u8 buf[4];
+	int ret;
+	int error;
+
+	/* Request dimension */
+	buf[0] = EKTF2127_REQUEST;
+	buf[1] = width ? EKTF2127_WIDTH : EKTF2127_HEIGHT;
+	buf[2] = 0x00;
+	buf[3] = 0x00;
+	ret = i2c_master_send(client, buf, sizeof(buf));
+	if (ret != sizeof(buf)) {
+		error = ret < 0 ? ret : -EIO;
+		dev_err(dev, "Failed to request %s: %d\n", what, error);
+		return error;
+	}
+
+	msleep(20);
+
+	/* Read response */
+	ret = i2c_master_recv(client, buf, sizeof(buf));
+	if (ret != sizeof(buf)) {
+		error = ret < 0 ? ret : -EIO;
+		dev_err(dev, "Failed to receive %s data: %d\n", what, error);
+		return error;
+	}
+
+	if (buf[0] != EKTF2127_RESPONSE || buf[1] != what_code) {
+		dev_err(dev, "Unexpected %s data: %#02x %#02x\n",
+			what, buf[0], buf[1]);
+		return -EIO;
+	}
+
+	return (((buf[3] & 0xf0) << 4) | buf[2]) - 1;
+}
+
+static int ektf2127_probe(struct i2c_client *client,
+			  const struct i2c_device_id *id)
+{
+	struct device *dev = &client->dev;
+	struct ektf2127_ts *ts;
+	struct input_dev *input;
+	u8 buf[4];
+	int max_x, max_y;
+	int error;
+
+	if (!client->irq) {
+		dev_err(dev, "Error no irq specified\n");
+		return -EINVAL;
+	}
+
+	ts = devm_kzalloc(dev, sizeof(*ts), GFP_KERNEL);
+	if (!ts)
+		return -ENOMEM;
+
+	/* This requests the gpio *and* turns on the touchscreen controller */
+	ts->power_gpios = devm_gpiod_get(dev, "power", GPIOD_OUT_HIGH);
+	if (IS_ERR(ts->power_gpios)) {
+		error = PTR_ERR(ts->power_gpios);
+		if (error != -EPROBE_DEFER)
+			dev_err(dev, "Error getting power gpio: %d\n", error);
+		return error;
+	}
+
+	input = devm_input_allocate_device(dev);
+	if (!input)
+		return -ENOMEM;
+
+	input->name = client->name;
+	input->id.bustype = BUS_I2C;
+	input->open = ektf2127_start;
+	input->close = ektf2127_stop;
+
+	ts->client = client;
+
+	/* Read hello (ignore result, depends on initial power state) */
+	msleep(20);
+	i2c_master_recv(ts->client, buf, sizeof(buf));
+
+	/* Read resolution from chip */
+	max_x = ektf2127_query_dimension(client, true);
+	if (max_x < 0)
+		return max_x;
+
+	max_y = ektf2127_query_dimension(client, false);
+	if (max_y < 0)
+		return max_y;
+
+	input_set_abs_params(input, ABS_MT_POSITION_X, 0, max_x, 0, 0);
+	input_set_abs_params(input, ABS_MT_POSITION_Y, 0, max_y, 0, 0);
+	touchscreen_parse_properties(input, true, &ts->prop);
+
+	error = input_mt_init_slots(input, EKTF2127_MAX_TOUCHES,
+				    INPUT_MT_DIRECT |
+					INPUT_MT_DROP_UNUSED |
+					INPUT_MT_TRACK);
+	if (error)
+		return error;
+
+	ts->input = input;
+	input_set_drvdata(input, ts);
+
+	error = devm_request_threaded_irq(dev, client->irq,
+					  NULL, ektf2127_irq,
+					  IRQF_ONESHOT, client->name, ts);
+	if (error) {
+		dev_err(dev, "Error requesting irq: %d\n", error);
+		return error;
+	}
+
+	/* Stop device till opened */
+	ektf2127_stop(ts->input);
+
+	error = input_register_device(input);
+	if (error)
+		return error;
+
+	i2c_set_clientdata(client, ts);
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id ektf2127_of_match[] = {
+	{ .compatible = "elan,ektf2127" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ektf2127_of_match);
+#endif
+
+static const struct i2c_device_id ektf2127_i2c_id[] = {
+	{ "ektf2127", 0 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, ektf2127_i2c_id);
+
+static struct i2c_driver ektf2127_driver = {
+	.driver = {
+		.name	= "elan_ektf2127",
+		.pm	= &ektf2127_pm_ops,
+		.of_match_table = of_match_ptr(ektf2127_of_match),
+	},
+	.probe = ektf2127_probe,
+	.id_table = ektf2127_i2c_id,
+};
+module_i2c_driver(ektf2127_driver);
+
+MODULE_DESCRIPTION("ELAN eKTF2127 I2C Touchscreen Driver");
+MODULE_AUTHOR("Michel Verlaan, Siebren Vroegindeweij");
+MODULE_LICENSE("GPL");
diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c
index ac09855fa435..02aec284deca 100644
--- a/drivers/input/touchscreen/elants_i2c.c
+++ b/drivers/input/touchscreen/elants_i2c.c
@@ -298,7 +298,7 @@ static u16 elants_i2c_parse_version(u8 *buf)
 	return get_unaligned_be32(buf) >> 4;
 }
 
-static int elants_i2c_query_fw_id(struct elants_data *ts)
+static int elants_i2c_query_hw_version(struct elants_data *ts)
 {
 	struct i2c_client *client = ts->client;
 	int error, retry_cnt;
@@ -318,8 +318,13 @@ static int elants_i2c_query_fw_id(struct elants_data *ts)
 			error, (int)sizeof(resp), resp);
 	}
 
-	dev_err(&client->dev,
-		"Failed to read fw id or fw id is invalid\n");
+	if (error) {
+		dev_err(&client->dev,
+			"Failed to read fw id: %d\n", error);
+		return error;
+	}
+
+	dev_err(&client->dev, "Invalid fw id: %#04x\n", ts->hw_version);
 
 	return -EINVAL;
 }
@@ -508,7 +513,7 @@ static int elants_i2c_fastboot(struct i2c_client *client)
 static int elants_i2c_initialize(struct elants_data *ts)
 {
 	struct i2c_client *client = ts->client;
-	int error, retry_cnt;
+	int error, error2, retry_cnt;
 	const u8 hello_packet[] = { 0x55, 0x55, 0x55, 0x55 };
 	const u8 recov_packet[] = { 0x55, 0x55, 0x80, 0x80 };
 	u8 buf[HEADER_SIZE];
@@ -553,18 +558,22 @@ static int elants_i2c_initialize(struct elants_data *ts)
 		}
 	}
 
+	/* hw version is available even if device in recovery state */
+	error2 = elants_i2c_query_hw_version(ts);
 	if (!error)
-		error = elants_i2c_query_fw_id(ts);
+		error = error2;
+
 	if (!error)
 		error = elants_i2c_query_fw_version(ts);
+	if (!error)
+		error = elants_i2c_query_test_version(ts);
+	if (!error)
+		error = elants_i2c_query_bc_version(ts);
+	if (!error)
+		error = elants_i2c_query_ts_info(ts);
 
-	if (error) {
+	if (error)
 		ts->iap_mode = ELAN_IAP_RECOVERY;
-	} else {
-		elants_i2c_query_test_version(ts);
-		elants_i2c_query_bc_version(ts);
-		elants_i2c_query_ts_info(ts);
-	}
 
 	return 0;
 }
diff --git a/drivers/input/touchscreen/ft6236.c b/drivers/input/touchscreen/ft6236.c
deleted file mode 100644
index d240d2e212bd..000000000000
--- a/drivers/input/touchscreen/ft6236.c
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * FocalTech FT6236 TouchScreen driver.
- *
- * Copyright (c) 2010  Focal tech Ltd.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/delay.h>
-#include <linux/gpio/consumer.h>
-#include <linux/i2c.h>
-#include <linux/input.h>
-#include <linux/input/mt.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/property.h>
-
-#define FT6236_MAX_TOUCH_POINTS		2
-
-#define FT6236_REG_TH_GROUP		0x80
-#define FT6236_REG_PERIODACTIVE		0x88
-#define FT6236_REG_LIB_VER_H		0xa1
-#define FT6236_REG_LIB_VER_L		0xa2
-#define FT6236_REG_CIPHER		0xa3
-#define FT6236_REG_FIRMID		0xa6
-#define FT6236_REG_FOCALTECH_ID		0xa8
-#define FT6236_REG_RELEASE_CODE_ID	0xaf
-
-#define FT6236_EVENT_PRESS_DOWN		0
-#define FT6236_EVENT_LIFT_UP		1
-#define FT6236_EVENT_CONTACT		2
-#define FT6236_EVENT_NO_EVENT		3
-
-struct ft6236_data {
-	struct i2c_client *client;
-	struct input_dev *input;
-	struct gpio_desc *reset_gpio;
-	u32 max_x;
-	u32 max_y;
-	bool invert_x;
-	bool invert_y;
-	bool swap_xy;
-};
-
-/*
- * This struct is a touchpoint as stored in hardware.  Note that the id,
- * as well as the event, are stored in the upper nybble of the hi byte.
- */
-struct ft6236_touchpoint {
-	union {
-		u8 xhi;
-		u8 event;
-	};
-	u8 xlo;
-	union {
-		u8 yhi;
-		u8 id;
-	};
-	u8 ylo;
-	u8 weight;
-	u8 misc;
-} __packed;
-
-/* This packet represents the register map as read from offset 0 */
-struct ft6236_packet {
-	u8 dev_mode;
-	u8 gest_id;
-	u8 touches;
-	struct ft6236_touchpoint points[FT6236_MAX_TOUCH_POINTS];
-} __packed;
-
-static int ft6236_read(struct i2c_client *client, u8 reg, u8 len, void *data)
-{
-	int error;
-
-	error = i2c_smbus_read_i2c_block_data(client, reg, len, data);
-	if (error < 0)
-		return error;
-
-	if (error != len)
-		return -EIO;
-
-	return 0;
-}
-
-static irqreturn_t ft6236_interrupt(int irq, void *dev_id)
-{
-	struct ft6236_data *ft6236 = dev_id;
-	struct device *dev = &ft6236->client->dev;
-	struct input_dev *input = ft6236->input;
-	struct ft6236_packet buf;
-	u8 touches;
-	int i, error;
-
-	error = ft6236_read(ft6236->client, 0, sizeof(buf), &buf);
-	if (error) {
-		dev_err(dev, "read touchdata failed %d\n", error);
-		return IRQ_HANDLED;
-	}
-
-	touches = buf.touches & 0xf;
-	if (touches > FT6236_MAX_TOUCH_POINTS) {
-		dev_dbg(dev,
-			"%d touch points reported, only %d are supported\n",
-			touches, FT6236_MAX_TOUCH_POINTS);
-		touches = FT6236_MAX_TOUCH_POINTS;
-	}
-
-	for (i = 0; i < touches; i++) {
-		struct ft6236_touchpoint *point = &buf.points[i];
-		u16 x = ((point->xhi & 0xf) << 8) | buf.points[i].xlo;
-		u16 y = ((point->yhi & 0xf) << 8) | buf.points[i].ylo;
-		u8 event = point->event >> 6;
-		u8 id = point->id >> 4;
-		bool act = (event == FT6236_EVENT_PRESS_DOWN ||
-			    event == FT6236_EVENT_CONTACT);
-
-		input_mt_slot(input, id);
-		input_mt_report_slot_state(input, MT_TOOL_FINGER, act);
-		if (!act)
-			continue;
-
-		if (ft6236->invert_x)
-			x = ft6236->max_x - x;
-
-		if (ft6236->invert_y)
-			y = ft6236->max_y - y;
-
-		if (ft6236->swap_xy) {
-			input_report_abs(input, ABS_MT_POSITION_X, y);
-			input_report_abs(input, ABS_MT_POSITION_Y, x);
-		} else {
-			input_report_abs(input, ABS_MT_POSITION_X, x);
-			input_report_abs(input, ABS_MT_POSITION_Y, y);
-		}
-	}
-
-	input_mt_sync_frame(input);
-	input_sync(input);
-
-	return IRQ_HANDLED;
-}
-
-static u8 ft6236_debug_read_byte(struct ft6236_data *ft6236, u8 reg)
-{
-	struct i2c_client *client = ft6236->client;
-	u8 val = 0;
-	int error;
-
-	error = ft6236_read(client, reg, 1, &val);
-	if (error)
-		dev_dbg(&client->dev,
-			"error reading register 0x%02x: %d\n", reg, error);
-
-	return val;
-}
-
-static void ft6236_debug_info(struct ft6236_data *ft6236)
-{
-	struct device *dev = &ft6236->client->dev;
-
-	dev_dbg(dev, "Touch threshold is %d\n",
-		ft6236_debug_read_byte(ft6236, FT6236_REG_TH_GROUP) * 4);
-	dev_dbg(dev, "Report rate is %dHz\n",
-		ft6236_debug_read_byte(ft6236, FT6236_REG_PERIODACTIVE) * 10);
-	dev_dbg(dev, "Firmware library version 0x%02x%02x\n",
-		ft6236_debug_read_byte(ft6236, FT6236_REG_LIB_VER_H),
-		ft6236_debug_read_byte(ft6236, FT6236_REG_LIB_VER_L));
-	dev_dbg(dev, "Firmware version 0x%02x\n",
-		ft6236_debug_read_byte(ft6236, FT6236_REG_FIRMID));
-	dev_dbg(dev, "Chip vendor ID 0x%02x\n",
-		ft6236_debug_read_byte(ft6236, FT6236_REG_CIPHER));
-	dev_dbg(dev, "CTPM vendor ID 0x%02x\n",
-		ft6236_debug_read_byte(ft6236, FT6236_REG_FOCALTECH_ID));
-	dev_dbg(dev, "Release code version 0x%02x\n",
-		ft6236_debug_read_byte(ft6236, FT6236_REG_RELEASE_CODE_ID));
-}
-
-static void ft6236_reset(struct ft6236_data *ft6236)
-{
-	if (!ft6236->reset_gpio)
-		return;
-
-	gpiod_set_value_cansleep(ft6236->reset_gpio, 1);
-	usleep_range(5000, 20000);
-	gpiod_set_value_cansleep(ft6236->reset_gpio, 0);
-	msleep(300);
-}
-
-static int ft6236_probe(struct i2c_client *client,
-			const struct i2c_device_id *id)
-{
-	struct device *dev = &client->dev;
-	struct ft6236_data *ft6236;
-	struct input_dev *input;
-	u32 fuzz_x = 0, fuzz_y = 0;
-	u8 val;
-	int error;
-
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
-		return -ENXIO;
-
-	if (!client->irq) {
-		dev_err(dev, "irq is missing\n");
-		return -EINVAL;
-	}
-
-	ft6236 = devm_kzalloc(dev, sizeof(*ft6236), GFP_KERNEL);
-	if (!ft6236)
-		return -ENOMEM;
-
-	ft6236->client = client;
-	ft6236->reset_gpio = devm_gpiod_get_optional(dev, "reset",
-						     GPIOD_OUT_LOW);
-	if (IS_ERR(ft6236->reset_gpio)) {
-		error = PTR_ERR(ft6236->reset_gpio);
-		if (error != -EPROBE_DEFER)
-			dev_err(dev, "error getting reset gpio: %d\n", error);
-		return error;
-	}
-
-	ft6236_reset(ft6236);
-
-	/* verify that the controller is present */
-	error = ft6236_read(client, 0x00, 1, &val);
-	if (error) {
-		dev_err(dev, "failed to read from controller: %d\n", error);
-		return error;
-	}
-
-	ft6236_debug_info(ft6236);
-
-	input = devm_input_allocate_device(dev);
-	if (!input)
-		return -ENOMEM;
-
-	ft6236->input = input;
-	input->name = client->name;
-	input->id.bustype = BUS_I2C;
-
-	if (device_property_read_u32(dev, "touchscreen-size-x",
-				     &ft6236->max_x) ||
-	    device_property_read_u32(dev, "touchscreen-size-y",
-				     &ft6236->max_y)) {
-		dev_err(dev, "touchscreen-size-x and/or -y missing\n");
-		return -EINVAL;
-	}
-
-	device_property_read_u32(dev, "touchscreen-fuzz-x", &fuzz_x);
-	device_property_read_u32(dev, "touchscreen-fuzz-y", &fuzz_y);
-	ft6236->invert_x = device_property_read_bool(dev,
-						     "touchscreen-inverted-x");
-	ft6236->invert_y = device_property_read_bool(dev,
-						     "touchscreen-inverted-y");
-	ft6236->swap_xy = device_property_read_bool(dev,
-						    "touchscreen-swapped-x-y");
-
-	if (ft6236->swap_xy) {
-		input_set_abs_params(input, ABS_MT_POSITION_X, 0,
-				     ft6236->max_y, fuzz_y, 0);
-		input_set_abs_params(input, ABS_MT_POSITION_Y, 0,
-				     ft6236->max_x, fuzz_x, 0);
-	} else {
-		input_set_abs_params(input, ABS_MT_POSITION_X, 0,
-				     ft6236->max_x, fuzz_x, 0);
-		input_set_abs_params(input, ABS_MT_POSITION_Y, 0,
-				     ft6236->max_y, fuzz_y, 0);
-	}
-
-	error = input_mt_init_slots(input, FT6236_MAX_TOUCH_POINTS,
-				    INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED);
-	if (error)
-		return error;
-
-	error = devm_request_threaded_irq(dev, client->irq, NULL,
-					  ft6236_interrupt, IRQF_ONESHOT,
-					  client->name, ft6236);
-	if (error) {
-		dev_err(dev, "request irq %d failed: %d\n", client->irq, error);
-		return error;
-	}
-
-	error = input_register_device(input);
-	if (error) {
-		dev_err(dev, "failed to register input device: %d\n", error);
-		return error;
-	}
-
-	return 0;
-}
-
-#ifdef CONFIG_OF
-static const struct of_device_id ft6236_of_match[] = {
-	{ .compatible = "focaltech,ft6236", },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, ft6236_of_match);
-#endif
-
-static const struct i2c_device_id ft6236_id[] = {
-	{ "ft6236", },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, ft6236_id);
-
-static struct i2c_driver ft6236_driver = {
-	.driver = {
-		.name = "ft6236",
-		.of_match_table = of_match_ptr(ft6236_of_match),
-	},
-	.probe = ft6236_probe,
-	.id_table = ft6236_id,
-};
-module_i2c_driver(ft6236_driver);
-
-MODULE_AUTHOR("Sean Cross <xobs@kosagi.com>");
-MODULE_AUTHOR("Noralf Trønnes <noralf@tronnes.org>");
-MODULE_DESCRIPTION("FocalTech FT6236 TouchScreen driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/input/touchscreen/jornada720_ts.c b/drivers/input/touchscreen/jornada720_ts.c
index ea3b6a5b83e6..729b3c89324c 100644
--- a/drivers/input/touchscreen/jornada720_ts.c
+++ b/drivers/input/touchscreen/jornada720_ts.c
@@ -13,6 +13,7 @@
  * HP Jornada 710/720/729 Touchscreen Driver
  */
 
+#include <linux/gpio/consumer.h>
 #include <linux/platform_device.h>
 #include <linux/input.h>
 #include <linux/interrupt.h>
@@ -20,9 +21,7 @@
 #include <linux/slab.h>
 #include <linux/io.h>
 
-#include <mach/hardware.h>
 #include <mach/jornada720.h>
-#include <mach/irqs.h>
 
 MODULE_AUTHOR("Kristoffer Ericson <kristoffer.ericson@gmail.com>");
 MODULE_DESCRIPTION("HP Jornada 710/720/728 touchscreen driver");
@@ -30,6 +29,7 @@ MODULE_LICENSE("GPL v2");
 
 struct jornada_ts {
 	struct input_dev *dev;
+	struct gpio_desc *gpio;
 	int x_data[4];		/* X sample values */
 	int y_data[4];		/* Y sample values */
 };
@@ -71,8 +71,8 @@ static irqreturn_t jornada720_ts_interrupt(int irq, void *dev_id)
 	struct input_dev *input = jornada_ts->dev;
 	int x, y;
 
-	/* If GPIO_GPIO9 is set to high then report pen up */
-	if (GPLR & GPIO_GPIO(9)) {
+	/* If gpio is high then report pen up */
+	if (gpiod_get_value(jornada_ts->gpio)) {
 		input_report_key(input, BTN_TOUCH, 0);
 		input_sync(input);
 	} else {
@@ -101,7 +101,7 @@ static int jornada720_ts_probe(struct platform_device *pdev)
 {
 	struct jornada_ts *jornada_ts;
 	struct input_dev *input_dev;
-	int error;
+	int error, irq;
 
 	jornada_ts = devm_kzalloc(&pdev->dev, sizeof(*jornada_ts), GFP_KERNEL);
 	if (!jornada_ts)
@@ -113,6 +113,14 @@ static int jornada720_ts_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, jornada_ts);
 
+	jornada_ts->gpio = devm_gpiod_get(&pdev->dev, "penup", GPIOD_IN);
+	if (IS_ERR(jornada_ts->gpio))
+		return PTR_ERR(jornada_ts->gpio);
+
+	irq = gpiod_to_irq(jornada_ts->gpio);
+	if (irq <= 0)
+		return irq < 0 ? irq : -EINVAL;
+
 	jornada_ts->dev = input_dev;
 
 	input_dev->name = "HP Jornada 7xx Touchscreen";
@@ -125,8 +133,7 @@ static int jornada720_ts_probe(struct platform_device *pdev)
 	input_set_abs_params(input_dev, ABS_X, 270, 3900, 0, 0);
 	input_set_abs_params(input_dev, ABS_Y, 180, 3700, 0, 0);
 
-	error = devm_request_irq(&pdev->dev, IRQ_GPIO9,
-				 jornada720_ts_interrupt,
+	error = devm_request_irq(&pdev->dev, irq, jornada720_ts_interrupt,
 				 IRQF_TRIGGER_RISING,
 				 "HP7XX Touchscreen driver", pdev);
 	if (error) {
diff --git a/drivers/input/touchscreen/mc13783_ts.c b/drivers/input/touchscreen/mc13783_ts.c
index 913e25a994b4..ef64f36c5ffc 100644
--- a/drivers/input/touchscreen/mc13783_ts.c
+++ b/drivers/input/touchscreen/mc13783_ts.c
@@ -37,7 +37,6 @@ struct mc13783_ts_priv {
 	struct input_dev *idev;
 	struct mc13xxx *mc13xxx;
 	struct delayed_work work;
-	struct workqueue_struct *workq;
 	unsigned int sample[4];
 	struct mc13xxx_ts_platform_data *touch;
 };
@@ -54,7 +53,7 @@ static irqreturn_t mc13783_ts_handler(int irq, void *data)
 	 * be rescheduled for immediate execution here. However the rearm
 	 * delay is HZ / 50 which is acceptable.
 	 */
-	queue_delayed_work(priv->workq, &priv->work, 0);
+	schedule_delayed_work(&priv->work, 0);
 
 	return IRQ_HANDLED;
 }
@@ -106,16 +105,18 @@ static void mc13783_ts_report_sample(struct mc13783_ts_priv *priv)
 
 			dev_dbg(&idev->dev, "report (%d, %d, %d)\n",
 					x1, y1, 0x1000 - cr0);
-			queue_delayed_work(priv->workq, &priv->work, HZ / 50);
-		} else
+			schedule_delayed_work(&priv->work, HZ / 50);
+		} else {
 			dev_dbg(&idev->dev, "report release\n");
+		}
 
 		input_report_abs(idev, ABS_PRESSURE,
 				cr0 ? 0x1000 - cr0 : cr0);
 		input_report_key(idev, BTN_TOUCH, cr0);
 		input_sync(idev);
-	} else
+	} else {
 		dev_dbg(&idev->dev, "discard event\n");
+	}
 }
 
 static void mc13783_ts_work(struct work_struct *work)
@@ -189,14 +190,6 @@ static int __init mc13783_ts_probe(struct platform_device *pdev)
 		goto err_free_mem;
 	}
 
-	/*
-	 * We need separate workqueue because mc13783_adc_do_conversion
-	 * uses keventd and thus would deadlock.
-	 */
-	priv->workq = create_singlethread_workqueue("mc13783_ts");
-	if (!priv->workq)
-		goto err_free_mem;
-
 	idev->name = MC13783_TS_NAME;
 	idev->dev.parent = &pdev->dev;
 
@@ -215,14 +208,12 @@ static int __init mc13783_ts_probe(struct platform_device *pdev)
 	if (ret) {
 		dev_err(&pdev->dev,
 			"register input device failed with %d\n", ret);
-		goto err_destroy_wq;
+		goto err_free_mem;
 	}
 
 	platform_set_drvdata(pdev, priv);
 	return 0;
 
-err_destroy_wq:
-	destroy_workqueue(priv->workq);
 err_free_mem:
 	input_free_device(idev);
 	kfree(priv);
@@ -233,7 +224,6 @@ static int mc13783_ts_remove(struct platform_device *pdev)
 {
 	struct mc13783_ts_priv *priv = platform_get_drvdata(pdev);
 
-	destroy_workqueue(priv->workq);
 	input_unregister_device(priv->idev);
 	kfree(priv);
 
diff --git a/drivers/input/touchscreen/pixcir_i2c_ts.c b/drivers/input/touchscreen/pixcir_i2c_ts.c
index d159e14f4d20..3bb0637d832e 100644
--- a/drivers/input/touchscreen/pixcir_i2c_ts.c
+++ b/drivers/input/touchscreen/pixcir_i2c_ts.c
@@ -11,10 +11,6 @@
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  */
 
 #include <linux/delay.h>
@@ -404,7 +400,6 @@ static int __maybe_unused pixcir_i2c_ts_resume(struct device *dev)
 	mutex_lock(&input->mutex);
 
 	if (device_may_wakeup(&client->dev)) {
-
 		if (!input->users) {
 			ret = pixcir_stop(ts);
 			if (ret) {
@@ -431,13 +426,7 @@ static const struct of_device_id pixcir_of_match[];
 static int pixcir_parse_dt(struct device *dev,
 			   struct pixcir_i2c_ts_data *tsdata)
 {
-	const struct of_device_id *match;
-
-	match = of_match_device(of_match_ptr(pixcir_of_match), dev);
-	if (!match)
-		return -EINVAL;
-
-	tsdata->chip = (const struct pixcir_i2c_chip_data *)match->data;
+	tsdata->chip = of_device_get_match_data(dev);
 	if (!tsdata->chip)
 		return -EINVAL;
 
diff --git a/drivers/input/touchscreen/silead.c b/drivers/input/touchscreen/silead.c
index 7379fe153cf9..f502c8488be8 100644
--- a/drivers/input/touchscreen/silead.c
+++ b/drivers/input/touchscreen/silead.c
@@ -390,9 +390,10 @@ static void silead_ts_read_props(struct i2c_client *client)
 		data->max_fingers = 5; /* Most devices handle up-to 5 fingers */
 	}
 
-	error = device_property_read_string(dev, "touchscreen-fw-name", &str);
+	error = device_property_read_string(dev, "firmware-name", &str);
 	if (!error)
-		snprintf(data->fw_name, sizeof(data->fw_name), "%s", str);
+		snprintf(data->fw_name, sizeof(data->fw_name),
+			 "silead/%s", str);
 	else
 		dev_dbg(dev, "Firmware file name read error. Using default.");
 }
@@ -410,14 +411,14 @@ static int silead_ts_set_default_fw_name(struct silead_ts_data *data,
 		if (!acpi_id)
 			return -ENODEV;
 
-		snprintf(data->fw_name, sizeof(data->fw_name), "%s.fw",
-			acpi_id->id);
+		snprintf(data->fw_name, sizeof(data->fw_name),
+			 "silead/%s.fw", acpi_id->id);
 
 		for (i = 0; i < strlen(data->fw_name); i++)
 			data->fw_name[i] = tolower(data->fw_name[i]);
 	} else {
-		snprintf(data->fw_name, sizeof(data->fw_name), "%s.fw",
-			id->name);
+		snprintf(data->fw_name, sizeof(data->fw_name),
+			 "silead/%s.fw", id->name);
 	}
 
 	return 0;
@@ -426,7 +427,8 @@ static int silead_ts_set_default_fw_name(struct silead_ts_data *data,
 static int silead_ts_set_default_fw_name(struct silead_ts_data *data,
 					 const struct i2c_device_id *id)
 {
-	snprintf(data->fw_name, sizeof(data->fw_name), "%s.fw", id->name);
+	snprintf(data->fw_name, sizeof(data->fw_name),
+		 "silead/%s.fw", id->name);
 	return 0;
 }
 #endif
@@ -464,7 +466,7 @@ static int silead_ts_probe(struct i2c_client *client,
 		return -ENODEV;
 
 	/* Power GPIO pin */
-	data->gpio_power = gpiod_get_optional(dev, "power", GPIOD_OUT_LOW);
+	data->gpio_power = devm_gpiod_get_optional(dev, "power", GPIOD_OUT_LOW);
 	if (IS_ERR(data->gpio_power)) {
 		if (PTR_ERR(data->gpio_power) != -EPROBE_DEFER)
 			dev_err(dev, "Shutdown GPIO request failed\n");
diff --git a/drivers/input/touchscreen/sur40.c b/drivers/input/touchscreen/sur40.c
index 4ea475775d58..aefb6e11f88a 100644
--- a/drivers/input/touchscreen/sur40.c
+++ b/drivers/input/touchscreen/sur40.c
@@ -139,6 +139,27 @@ struct sur40_image_header {
 #define SUR40_GET_STATE   0xc5 /*  4 bytes state (?) */
 #define SUR40_GET_SENSORS 0xb1 /*  8 bytes sensors   */
 
+static const struct v4l2_pix_format sur40_pix_format[] = {
+	{
+		.pixelformat = V4L2_TCH_FMT_TU08,
+		.width  = SENSOR_RES_X / 2,
+		.height = SENSOR_RES_Y / 2,
+		.field = V4L2_FIELD_NONE,
+		.colorspace = V4L2_COLORSPACE_SRGB,
+		.bytesperline = SENSOR_RES_X / 2,
+		.sizeimage = (SENSOR_RES_X/2) * (SENSOR_RES_Y/2),
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_GREY,
+		.width  = SENSOR_RES_X / 2,
+		.height = SENSOR_RES_Y / 2,
+		.field = V4L2_FIELD_NONE,
+		.colorspace = V4L2_COLORSPACE_SRGB,
+		.bytesperline = SENSOR_RES_X / 2,
+		.sizeimage = (SENSOR_RES_X/2) * (SENSOR_RES_Y/2),
+	}
+};
+
 /* master device state */
 struct sur40_state {
 
@@ -149,6 +170,7 @@ struct sur40_state {
 	struct v4l2_device v4l2;
 	struct video_device vdev;
 	struct mutex lock;
+	struct v4l2_pix_format pix_fmt;
 
 	struct vb2_queue queue;
 	struct list_head buf_list;
@@ -169,7 +191,6 @@ struct sur40_buffer {
 
 /* forward declarations */
 static const struct video_device sur40_video_device;
-static const struct v4l2_pix_format sur40_video_format;
 static const struct vb2_queue sur40_queue;
 static void sur40_process_video(struct sur40_state *sur40);
 
@@ -420,7 +441,7 @@ static void sur40_process_video(struct sur40_state *sur40)
 		goto err_poll;
 	}
 
-	if (le32_to_cpu(img->size) != sur40_video_format.sizeimage) {
+	if (le32_to_cpu(img->size) != sur40->pix_fmt.sizeimage) {
 		dev_err(sur40->dev, "image size mismatch\n");
 		goto err_poll;
 	}
@@ -431,7 +452,7 @@ static void sur40_process_video(struct sur40_state *sur40)
 
 	result = usb_sg_init(&sgr, sur40->usbdev,
 		usb_rcvbulkpipe(sur40->usbdev, VIDEO_ENDPOINT), 0,
-		sgt->sgl, sgt->nents, sur40_video_format.sizeimage, 0);
+		sgt->sgl, sgt->nents, sur40->pix_fmt.sizeimage, 0);
 	if (result < 0) {
 		dev_err(sur40->dev, "error %d in usb_sg_init\n", result);
 		goto err_poll;
@@ -586,13 +607,14 @@ static int sur40_probe(struct usb_interface *interface,
 	if (error)
 		goto err_unreg_v4l2;
 
+	sur40->pix_fmt = sur40_pix_format[0];
 	sur40->vdev = sur40_video_device;
 	sur40->vdev.v4l2_dev = &sur40->v4l2;
 	sur40->vdev.lock = &sur40->lock;
 	sur40->vdev.queue = &sur40->queue;
 	video_set_drvdata(&sur40->vdev, sur40);
 
-	error = video_register_device(&sur40->vdev, VFL_TYPE_GRABBER, -1);
+	error = video_register_device(&sur40->vdev, VFL_TYPE_TOUCH, -1);
 	if (error) {
 		dev_err(&interface->dev,
 			"Unable to register video subdevice.");
@@ -647,14 +669,16 @@ static int sur40_queue_setup(struct vb2_queue *q,
 		       unsigned int *nbuffers, unsigned int *nplanes,
 		       unsigned int sizes[], struct device *alloc_devs[])
 {
+	struct sur40_state *sur40 = vb2_get_drv_priv(q);
+
 	if (q->num_buffers + *nbuffers < 3)
 		*nbuffers = 3 - q->num_buffers;
 
 	if (*nplanes)
-		return sizes[0] < sur40_video_format.sizeimage ? -EINVAL : 0;
+		return sizes[0] < sur40->pix_fmt.sizeimage ? -EINVAL : 0;
 
 	*nplanes = 1;
-	sizes[0] = sur40_video_format.sizeimage;
+	sizes[0] = sur40->pix_fmt.sizeimage;
 
 	return 0;
 }
@@ -666,7 +690,7 @@ static int sur40_queue_setup(struct vb2_queue *q,
 static int sur40_buffer_prepare(struct vb2_buffer *vb)
 {
 	struct sur40_state *sur40 = vb2_get_drv_priv(vb->vb2_queue);
-	unsigned long size = sur40_video_format.sizeimage;
+	unsigned long size = sur40->pix_fmt.sizeimage;
 
 	if (vb2_plane_size(vb, 0) < size) {
 		dev_err(&sur40->usbdev->dev, "buffer too small (%lu < %lu)\n",
@@ -741,7 +765,7 @@ static int sur40_vidioc_querycap(struct file *file, void *priv,
 	strlcpy(cap->driver, DRIVER_SHORT, sizeof(cap->driver));
 	strlcpy(cap->card, DRIVER_LONG, sizeof(cap->card));
 	usb_make_path(sur40->usbdev, cap->bus_info, sizeof(cap->bus_info));
-	cap->device_caps = V4L2_CAP_VIDEO_CAPTURE |
+	cap->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_TOUCH |
 		V4L2_CAP_READWRITE |
 		V4L2_CAP_STREAMING;
 	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
@@ -753,7 +777,7 @@ static int sur40_vidioc_enum_input(struct file *file, void *priv,
 {
 	if (i->index != 0)
 		return -EINVAL;
-	i->type = V4L2_INPUT_TYPE_CAMERA;
+	i->type = V4L2_INPUT_TYPE_TOUCH;
 	i->std = V4L2_STD_UNKNOWN;
 	strlcpy(i->name, "In-Cell Sensor", sizeof(i->name));
 	i->capabilities = 0;
@@ -771,19 +795,70 @@ static int sur40_vidioc_g_input(struct file *file, void *priv, unsigned int *i)
 	return 0;
 }
 
-static int sur40_vidioc_fmt(struct file *file, void *priv,
+static int sur40_vidioc_try_fmt(struct file *file, void *priv,
+			    struct v4l2_format *f)
+{
+	switch (f->fmt.pix.pixelformat) {
+	case V4L2_PIX_FMT_GREY:
+		f->fmt.pix = sur40_pix_format[1];
+		break;
+
+	default:
+		f->fmt.pix = sur40_pix_format[0];
+		break;
+	}
+
+	return 0;
+}
+
+static int sur40_vidioc_s_fmt(struct file *file, void *priv,
+			    struct v4l2_format *f)
+{
+	struct sur40_state *sur40 = video_drvdata(file);
+
+	switch (f->fmt.pix.pixelformat) {
+	case V4L2_PIX_FMT_GREY:
+		sur40->pix_fmt = sur40_pix_format[1];
+		break;
+
+	default:
+		sur40->pix_fmt = sur40_pix_format[0];
+		break;
+	}
+
+	f->fmt.pix = sur40->pix_fmt;
+	return 0;
+}
+
+static int sur40_vidioc_g_fmt(struct file *file, void *priv,
 			    struct v4l2_format *f)
 {
-	f->fmt.pix = sur40_video_format;
+	struct sur40_state *sur40 = video_drvdata(file);
+
+	f->fmt.pix = sur40->pix_fmt;
+	return 0;
+}
+
+static int sur40_ioctl_parm(struct file *file, void *priv,
+			    struct v4l2_streamparm *p)
+{
+	if (p->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	p->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
+	p->parm.capture.timeperframe.numerator = 1;
+	p->parm.capture.timeperframe.denominator = 60;
+	p->parm.capture.readbuffers = 3;
 	return 0;
 }
 
 static int sur40_vidioc_enum_fmt(struct file *file, void *priv,
 				 struct v4l2_fmtdesc *f)
 {
-	if (f->index != 0)
+	if (f->index >= ARRAY_SIZE(sur40_pix_format))
 		return -EINVAL;
-	f->pixelformat = V4L2_PIX_FMT_GREY;
+
+	f->pixelformat = sur40_pix_format[f->index].pixelformat;
 	f->flags = 0;
 	return 0;
 }
@@ -791,25 +866,31 @@ static int sur40_vidioc_enum_fmt(struct file *file, void *priv,
 static int sur40_vidioc_enum_framesizes(struct file *file, void *priv,
 					struct v4l2_frmsizeenum *f)
 {
-	if ((f->index != 0) || (f->pixel_format != V4L2_PIX_FMT_GREY))
+	struct sur40_state *sur40 = video_drvdata(file);
+
+	if ((f->index != 0) || ((f->pixel_format != V4L2_TCH_FMT_TU08)
+		&& (f->pixel_format != V4L2_PIX_FMT_GREY)))
 		return -EINVAL;
 
 	f->type = V4L2_FRMSIZE_TYPE_DISCRETE;
-	f->discrete.width  = sur40_video_format.width;
-	f->discrete.height = sur40_video_format.height;
+	f->discrete.width  = sur40->pix_fmt.width;
+	f->discrete.height = sur40->pix_fmt.height;
 	return 0;
 }
 
 static int sur40_vidioc_enum_frameintervals(struct file *file, void *priv,
 					    struct v4l2_frmivalenum *f)
 {
-	if ((f->index > 1) || (f->pixel_format != V4L2_PIX_FMT_GREY)
-		|| (f->width  != sur40_video_format.width)
-		|| (f->height != sur40_video_format.height))
-			return -EINVAL;
+	struct sur40_state *sur40 = video_drvdata(file);
+
+	if ((f->index > 0) || ((f->pixel_format != V4L2_TCH_FMT_TU08)
+		&& (f->pixel_format != V4L2_PIX_FMT_GREY))
+		|| (f->width  != sur40->pix_fmt.width)
+		|| (f->height != sur40->pix_fmt.height))
+		return -EINVAL;
 
 	f->type = V4L2_FRMIVAL_TYPE_DISCRETE;
-	f->discrete.denominator  = 60/(f->index+1);
+	f->discrete.denominator  = 60;
 	f->discrete.numerator = 1;
 	return 0;
 }
@@ -862,13 +943,16 @@ static const struct v4l2_ioctl_ops sur40_video_ioctl_ops = {
 	.vidioc_querycap	= sur40_vidioc_querycap,
 
 	.vidioc_enum_fmt_vid_cap = sur40_vidioc_enum_fmt,
-	.vidioc_try_fmt_vid_cap	= sur40_vidioc_fmt,
-	.vidioc_s_fmt_vid_cap	= sur40_vidioc_fmt,
-	.vidioc_g_fmt_vid_cap	= sur40_vidioc_fmt,
+	.vidioc_try_fmt_vid_cap	= sur40_vidioc_try_fmt,
+	.vidioc_s_fmt_vid_cap	= sur40_vidioc_s_fmt,
+	.vidioc_g_fmt_vid_cap	= sur40_vidioc_g_fmt,
 
 	.vidioc_enum_framesizes = sur40_vidioc_enum_framesizes,
 	.vidioc_enum_frameintervals = sur40_vidioc_enum_frameintervals,
 
+	.vidioc_g_parm = sur40_ioctl_parm,
+	.vidioc_s_parm = sur40_ioctl_parm,
+
 	.vidioc_enum_input	= sur40_vidioc_enum_input,
 	.vidioc_g_input		= sur40_vidioc_g_input,
 	.vidioc_s_input		= sur40_vidioc_s_input,
@@ -891,16 +975,6 @@ static const struct video_device sur40_video_device = {
 	.release = video_device_release_empty,
 };
 
-static const struct v4l2_pix_format sur40_video_format = {
-	.pixelformat = V4L2_PIX_FMT_GREY,
-	.width  = SENSOR_RES_X / 2,
-	.height = SENSOR_RES_Y / 2,
-	.field = V4L2_FIELD_NONE,
-	.colorspace = V4L2_COLORSPACE_SRGB,
-	.bytesperline = SENSOR_RES_X / 2,
-	.sizeimage = (SENSOR_RES_X/2) * (SENSOR_RES_Y/2),
-};
-
 /* USB-specific object needed to register this driver with the USB subsystem. */
 static struct usb_driver sur40_driver = {
 	.name = DRIVER_SHORT,
diff --git a/drivers/input/touchscreen/wdt87xx_i2c.c b/drivers/input/touchscreen/wdt87xx_i2c.c
index 73861ad22df4..a9132603ab34 100644
--- a/drivers/input/touchscreen/wdt87xx_i2c.c
+++ b/drivers/input/touchscreen/wdt87xx_i2c.c
@@ -23,7 +23,7 @@
 #include <asm/unaligned.h>
 
 #define WDT87XX_NAME		"wdt87xx_i2c"
-#define WDT87XX_DRV_VER		"0.9.7"
+#define WDT87XX_DRV_VER		"0.9.8"
 #define WDT87XX_FW_NAME		"wdt87xx_fw.bin"
 #define WDT87XX_CFG_NAME	"wdt87xx_cfg.bin"
 
@@ -157,6 +157,7 @@
 /* Controller requires minimum 300us between commands */
 #define WDT_COMMAND_DELAY_MS		2
 #define WDT_FLASH_WRITE_DELAY_MS	4
+#define WDT_FLASH_ERASE_DELAY_MS	200
 #define WDT_FW_RESET_TIME		2500
 
 struct wdt87xx_sys_param {
@@ -726,7 +727,7 @@ static int wdt87xx_write_firmware(struct i2c_client *client, const void *chunk)
 				break;
 			}
 
-			msleep(50);
+			msleep(WDT_FLASH_ERASE_DELAY_MS);
 
 			error = wdt87xx_write_data(client, data, start_addr,
 						   page_size);
diff --git a/drivers/input/touchscreen/wm97xx-core.c b/drivers/input/touchscreen/wm97xx-core.c
index 1534e9b0788c..90d6be3c26cc 100644
--- a/drivers/input/touchscreen/wm97xx-core.c
+++ b/drivers/input/touchscreen/wm97xx-core.c
@@ -500,7 +500,7 @@ static int wm97xx_ts_input_open(struct input_dev *idev)
 {
 	struct wm97xx *wm = input_get_drvdata(idev);
 
-	wm->ts_workq = create_singlethread_workqueue("kwm97xx");
+	wm->ts_workq = alloc_ordered_workqueue("kwm97xx", 0);
 	if (wm->ts_workq == NULL) {
 		dev_err(wm->dev,
 			"Failed to create workqueue\n");
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index d432ca828472..8ee54d71c7eb 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -309,7 +309,7 @@ config ARM_SMMU
 
 config ARM_SMMU_V3
 	bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
-	depends on ARM64 && PCI
+	depends on ARM64
 	select IOMMU_API
 	select IOMMU_IO_PGTABLE_LPAE
 	select GENERIC_MSI_IRQ_DOMAIN
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 96de97a46079..754595ee11b6 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -103,7 +103,7 @@ struct flush_queue {
 	struct flush_queue_entry *entries;
 };
 
-DEFINE_PER_CPU(struct flush_queue, flush_queue);
+static DEFINE_PER_CPU(struct flush_queue, flush_queue);
 
 static atomic_t queue_timer_on;
 static struct timer_list queue_timer;
@@ -137,6 +137,7 @@ struct iommu_dev_data {
 	bool pri_tlp;			  /* PASID TLB required for
 					     PPR completions */
 	u32 errata;			  /* Bitmap for errata to apply */
+	bool use_vapic;			  /* Enable device to use vapic mode */
 };
 
 /*
@@ -707,14 +708,74 @@ static void iommu_poll_ppr_log(struct amd_iommu *iommu)
 	}
 }
 
+#ifdef CONFIG_IRQ_REMAP
+static int (*iommu_ga_log_notifier)(u32);
+
+int amd_iommu_register_ga_log_notifier(int (*notifier)(u32))
+{
+	iommu_ga_log_notifier = notifier;
+
+	return 0;
+}
+EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier);
+
+static void iommu_poll_ga_log(struct amd_iommu *iommu)
+{
+	u32 head, tail, cnt = 0;
+
+	if (iommu->ga_log == NULL)
+		return;
+
+	head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
+	tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
+
+	while (head != tail) {
+		volatile u64 *raw;
+		u64 log_entry;
+
+		raw = (u64 *)(iommu->ga_log + head);
+		cnt++;
+
+		/* Avoid memcpy function-call overhead */
+		log_entry = *raw;
+
+		/* Update head pointer of hardware ring-buffer */
+		head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE;
+		writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
+
+		/* Handle GA entry */
+		switch (GA_REQ_TYPE(log_entry)) {
+		case GA_GUEST_NR:
+			if (!iommu_ga_log_notifier)
+				break;
+
+			pr_debug("AMD-Vi: %s: devid=%#x, ga_tag=%#x\n",
+				 __func__, GA_DEVID(log_entry),
+				 GA_TAG(log_entry));
+
+			if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0)
+				pr_err("AMD-Vi: GA log notifier failed.\n");
+			break;
+		default:
+			break;
+		}
+	}
+}
+#endif /* CONFIG_IRQ_REMAP */
+
+#define AMD_IOMMU_INT_MASK	\
+	(MMIO_STATUS_EVT_INT_MASK | \
+	 MMIO_STATUS_PPR_INT_MASK | \
+	 MMIO_STATUS_GALOG_INT_MASK)
+
 irqreturn_t amd_iommu_int_thread(int irq, void *data)
 {
 	struct amd_iommu *iommu = (struct amd_iommu *) data;
 	u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
 
-	while (status & (MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK)) {
-		/* Enable EVT and PPR interrupts again */
-		writel((MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK),
+	while (status & AMD_IOMMU_INT_MASK) {
+		/* Enable EVT and PPR and GA interrupts again */
+		writel(AMD_IOMMU_INT_MASK,
 			iommu->mmio_base + MMIO_STATUS_OFFSET);
 
 		if (status & MMIO_STATUS_EVT_INT_MASK) {
@@ -727,6 +788,13 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data)
 			iommu_poll_ppr_log(iommu);
 		}
 
+#ifdef CONFIG_IRQ_REMAP
+		if (status & MMIO_STATUS_GALOG_INT_MASK) {
+			pr_devel("AMD-Vi: Processing IOMMU GA Log\n");
+			iommu_poll_ga_log(iommu);
+		}
+#endif
+
 		/*
 		 * Hardware bug: ERBT1312
 		 * When re-enabling interrupt (by writing 1
@@ -940,15 +1008,13 @@ static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
  * Writes the command to the IOMMUs command buffer and informs the
  * hardware about the new command.
  */
-static int iommu_queue_command_sync(struct amd_iommu *iommu,
-				    struct iommu_cmd *cmd,
-				    bool sync)
+static int __iommu_queue_command_sync(struct amd_iommu *iommu,
+				      struct iommu_cmd *cmd,
+				      bool sync)
 {
 	u32 left, tail, head, next_tail;
-	unsigned long flags;
 
 again:
-	spin_lock_irqsave(&iommu->lock, flags);
 
 	head      = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
 	tail      = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
@@ -957,15 +1023,14 @@ again:
 
 	if (left <= 2) {
 		struct iommu_cmd sync_cmd;
-		volatile u64 sem = 0;
 		int ret;
 
-		build_completion_wait(&sync_cmd, (u64)&sem);
-		copy_cmd_to_buffer(iommu, &sync_cmd, tail);
+		iommu->cmd_sem = 0;
 
-		spin_unlock_irqrestore(&iommu->lock, flags);
+		build_completion_wait(&sync_cmd, (u64)&iommu->cmd_sem);
+		copy_cmd_to_buffer(iommu, &sync_cmd, tail);
 
-		if ((ret = wait_on_sem(&sem)) != 0)
+		if ((ret = wait_on_sem(&iommu->cmd_sem)) != 0)
 			return ret;
 
 		goto again;
@@ -976,9 +1041,21 @@ again:
 	/* We need to sync now to make sure all commands are processed */
 	iommu->need_sync = sync;
 
+	return 0;
+}
+
+static int iommu_queue_command_sync(struct amd_iommu *iommu,
+				    struct iommu_cmd *cmd,
+				    bool sync)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+	ret = __iommu_queue_command_sync(iommu, cmd, sync);
 	spin_unlock_irqrestore(&iommu->lock, flags);
 
-	return 0;
+	return ret;
 }
 
 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
@@ -993,19 +1070,29 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
 static int iommu_completion_wait(struct amd_iommu *iommu)
 {
 	struct iommu_cmd cmd;
-	volatile u64 sem = 0;
+	unsigned long flags;
 	int ret;
 
 	if (!iommu->need_sync)
 		return 0;
 
-	build_completion_wait(&cmd, (u64)&sem);
 
-	ret = iommu_queue_command_sync(iommu, &cmd, false);
+	build_completion_wait(&cmd, (u64)&iommu->cmd_sem);
+
+	spin_lock_irqsave(&iommu->lock, flags);
+
+	iommu->cmd_sem = 0;
+
+	ret = __iommu_queue_command_sync(iommu, &cmd, false);
 	if (ret)
-		return ret;
+		goto out_unlock;
 
-	return wait_on_sem(&sem);
+	ret = wait_on_sem(&iommu->cmd_sem);
+
+out_unlock:
+	spin_unlock_irqrestore(&iommu->lock, flags);
+
+	return ret;
 }
 
 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
@@ -1274,7 +1361,8 @@ static u64 *alloc_pte(struct protection_domain *domain,
 
 			__npte = PM_LEVEL_PDE(level, virt_to_phys(page));
 
-			if (cmpxchg64(pte, __pte, __npte)) {
+			/* pte could have been changed somewhere. */
+			if (cmpxchg64(pte, __pte, __npte) != __pte) {
 				free_page((unsigned long)page);
 				continue;
 			}
@@ -1654,6 +1742,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 
 	free_pagetable(&dom->domain);
 
+	if (dom->domain.id)
+		domain_id_free(dom->domain.id);
+
 	kfree(dom);
 }
 
@@ -2948,6 +3039,12 @@ static void amd_iommu_detach_device(struct iommu_domain *dom,
 	if (!iommu)
 		return;
 
+#ifdef CONFIG_IRQ_REMAP
+	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
+	    (dom->type == IOMMU_DOMAIN_UNMANAGED))
+		dev_data->use_vapic = 0;
+#endif
+
 	iommu_completion_wait(iommu);
 }
 
@@ -2973,6 +3070,15 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
 
 	ret = attach_device(dev, domain);
 
+#ifdef CONFIG_IRQ_REMAP
+	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
+		if (dom->type == IOMMU_DOMAIN_UNMANAGED)
+			dev_data->use_vapic = 1;
+		else
+			dev_data->use_vapic = 0;
+	}
+#endif
+
 	iommu_completion_wait(iommu);
 
 	return ret;
@@ -3511,34 +3617,6 @@ EXPORT_SYMBOL(amd_iommu_device_info);
  *
  *****************************************************************************/
 
-union irte {
-	u32 val;
-	struct {
-		u32 valid	: 1,
-		    no_fault	: 1,
-		    int_type	: 3,
-		    rq_eoi	: 1,
-		    dm		: 1,
-		    rsvd_1	: 1,
-		    destination	: 8,
-		    vector	: 8,
-		    rsvd_2	: 8;
-	} fields;
-};
-
-struct irq_2_irte {
-	u16 devid; /* Device ID for IRTE table */
-	u16 index; /* Index into IRTE table*/
-};
-
-struct amd_ir_data {
-	struct irq_2_irte			irq_2_irte;
-	union irte				irte_entry;
-	union {
-		struct msi_msg			msi_entry;
-	};
-};
-
 static struct irq_chip amd_ir_chip;
 
 #define DTE_IRQ_PHYS_ADDR_MASK	(((1ULL << 45)-1) << 6)
@@ -3560,8 +3638,6 @@ static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
 	amd_iommu_dev_table[devid].data[2] = dte;
 }
 
-#define IRTE_ALLOCATED (~1U)
-
 static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
 {
 	struct irq_remap_table *table = NULL;
@@ -3577,7 +3653,7 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
 
 	table = irq_lookup_table[devid];
 	if (table)
-		goto out;
+		goto out_unlock;
 
 	alias = amd_iommu_alias_table[devid];
 	table = irq_lookup_table[alias];
@@ -3591,7 +3667,7 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
 	/* Nothing there yet, allocate new irq remapping table */
 	table = kzalloc(sizeof(*table), GFP_ATOMIC);
 	if (!table)
-		goto out;
+		goto out_unlock;
 
 	/* Initialize table spin-lock */
 	spin_lock_init(&table->lock);
@@ -3604,16 +3680,21 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
 	if (!table->table) {
 		kfree(table);
 		table = NULL;
-		goto out;
+		goto out_unlock;
 	}
 
-	memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32));
+	if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
+		memset(table->table, 0,
+		       MAX_IRQS_PER_TABLE * sizeof(u32));
+	else
+		memset(table->table, 0,
+		       (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2)));
 
 	if (ioapic) {
 		int i;
 
 		for (i = 0; i < 32; ++i)
-			table->table[i] = IRTE_ALLOCATED;
+			iommu->irte_ops->set_allocated(table, i);
 	}
 
 	irq_lookup_table[devid] = table;
@@ -3639,6 +3720,10 @@ static int alloc_irq_index(u16 devid, int count)
 	struct irq_remap_table *table;
 	unsigned long flags;
 	int index, c;
+	struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
+
+	if (!iommu)
+		return -ENODEV;
 
 	table = get_irq_table(devid, false);
 	if (!table)
@@ -3650,14 +3735,14 @@ static int alloc_irq_index(u16 devid, int count)
 	for (c = 0, index = table->min_index;
 	     index < MAX_IRQS_PER_TABLE;
 	     ++index) {
-		if (table->table[index] == 0)
+		if (!iommu->irte_ops->is_allocated(table, index))
 			c += 1;
 		else
 			c = 0;
 
 		if (c == count)	{
 			for (; c != 0; --c)
-				table->table[index - c + 1] = IRTE_ALLOCATED;
+				iommu->irte_ops->set_allocated(table, index - c + 1);
 
 			index -= count - 1;
 			goto out;
@@ -3672,7 +3757,42 @@ out:
 	return index;
 }
 
-static int modify_irte(u16 devid, int index, union irte irte)
+static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte,
+			  struct amd_ir_data *data)
+{
+	struct irq_remap_table *table;
+	struct amd_iommu *iommu;
+	unsigned long flags;
+	struct irte_ga *entry;
+
+	iommu = amd_iommu_rlookup_table[devid];
+	if (iommu == NULL)
+		return -EINVAL;
+
+	table = get_irq_table(devid, false);
+	if (!table)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&table->lock, flags);
+
+	entry = (struct irte_ga *)table->table;
+	entry = &entry[index];
+	entry->lo.fields_remap.valid = 0;
+	entry->hi.val = irte->hi.val;
+	entry->lo.val = irte->lo.val;
+	entry->lo.fields_remap.valid = 1;
+	if (data)
+		data->ref = entry;
+
+	spin_unlock_irqrestore(&table->lock, flags);
+
+	iommu_flush_irt(iommu, devid);
+	iommu_completion_wait(iommu);
+
+	return 0;
+}
+
+static int modify_irte(u16 devid, int index, union irte *irte)
 {
 	struct irq_remap_table *table;
 	struct amd_iommu *iommu;
@@ -3687,7 +3807,7 @@ static int modify_irte(u16 devid, int index, union irte irte)
 		return -ENOMEM;
 
 	spin_lock_irqsave(&table->lock, flags);
-	table->table[index] = irte.val;
+	table->table[index] = irte->val;
 	spin_unlock_irqrestore(&table->lock, flags);
 
 	iommu_flush_irt(iommu, devid);
@@ -3711,13 +3831,146 @@ static void free_irte(u16 devid, int index)
 		return;
 
 	spin_lock_irqsave(&table->lock, flags);
-	table->table[index] = 0;
+	iommu->irte_ops->clear_allocated(table, index);
 	spin_unlock_irqrestore(&table->lock, flags);
 
 	iommu_flush_irt(iommu, devid);
 	iommu_completion_wait(iommu);
 }
 
+static void irte_prepare(void *entry,
+			 u32 delivery_mode, u32 dest_mode,
+			 u8 vector, u32 dest_apicid, int devid)
+{
+	union irte *irte = (union irte *) entry;
+
+	irte->val                = 0;
+	irte->fields.vector      = vector;
+	irte->fields.int_type    = delivery_mode;
+	irte->fields.destination = dest_apicid;
+	irte->fields.dm          = dest_mode;
+	irte->fields.valid       = 1;
+}
+
+static void irte_ga_prepare(void *entry,
+			    u32 delivery_mode, u32 dest_mode,
+			    u8 vector, u32 dest_apicid, int devid)
+{
+	struct irte_ga *irte = (struct irte_ga *) entry;
+	struct iommu_dev_data *dev_data = search_dev_data(devid);
+
+	irte->lo.val                      = 0;
+	irte->hi.val                      = 0;
+	irte->lo.fields_remap.guest_mode  = dev_data ? dev_data->use_vapic : 0;
+	irte->lo.fields_remap.int_type    = delivery_mode;
+	irte->lo.fields_remap.dm          = dest_mode;
+	irte->hi.fields.vector            = vector;
+	irte->lo.fields_remap.destination = dest_apicid;
+	irte->lo.fields_remap.valid       = 1;
+}
+
+static void irte_activate(void *entry, u16 devid, u16 index)
+{
+	union irte *irte = (union irte *) entry;
+
+	irte->fields.valid = 1;
+	modify_irte(devid, index, irte);
+}
+
+static void irte_ga_activate(void *entry, u16 devid, u16 index)
+{
+	struct irte_ga *irte = (struct irte_ga *) entry;
+
+	irte->lo.fields_remap.valid = 1;
+	modify_irte_ga(devid, index, irte, NULL);
+}
+
+static void irte_deactivate(void *entry, u16 devid, u16 index)
+{
+	union irte *irte = (union irte *) entry;
+
+	irte->fields.valid = 0;
+	modify_irte(devid, index, irte);
+}
+
+static void irte_ga_deactivate(void *entry, u16 devid, u16 index)
+{
+	struct irte_ga *irte = (struct irte_ga *) entry;
+
+	irte->lo.fields_remap.valid = 0;
+	modify_irte_ga(devid, index, irte, NULL);
+}
+
+static void irte_set_affinity(void *entry, u16 devid, u16 index,
+			      u8 vector, u32 dest_apicid)
+{
+	union irte *irte = (union irte *) entry;
+
+	irte->fields.vector = vector;
+	irte->fields.destination = dest_apicid;
+	modify_irte(devid, index, irte);
+}
+
+static void irte_ga_set_affinity(void *entry, u16 devid, u16 index,
+				 u8 vector, u32 dest_apicid)
+{
+	struct irte_ga *irte = (struct irte_ga *) entry;
+	struct iommu_dev_data *dev_data = search_dev_data(devid);
+
+	if (!dev_data || !dev_data->use_vapic) {
+		irte->hi.fields.vector = vector;
+		irte->lo.fields_remap.destination = dest_apicid;
+		irte->lo.fields_remap.guest_mode = 0;
+		modify_irte_ga(devid, index, irte, NULL);
+	}
+}
+
+#define IRTE_ALLOCATED (~1U)
+static void irte_set_allocated(struct irq_remap_table *table, int index)
+{
+	table->table[index] = IRTE_ALLOCATED;
+}
+
+static void irte_ga_set_allocated(struct irq_remap_table *table, int index)
+{
+	struct irte_ga *ptr = (struct irte_ga *)table->table;
+	struct irte_ga *irte = &ptr[index];
+
+	memset(&irte->lo.val, 0, sizeof(u64));
+	memset(&irte->hi.val, 0, sizeof(u64));
+	irte->hi.fields.vector = 0xff;
+}
+
+static bool irte_is_allocated(struct irq_remap_table *table, int index)
+{
+	union irte *ptr = (union irte *)table->table;
+	union irte *irte = &ptr[index];
+
+	return irte->val != 0;
+}
+
+static bool irte_ga_is_allocated(struct irq_remap_table *table, int index)
+{
+	struct irte_ga *ptr = (struct irte_ga *)table->table;
+	struct irte_ga *irte = &ptr[index];
+
+	return irte->hi.fields.vector != 0;
+}
+
+static void irte_clear_allocated(struct irq_remap_table *table, int index)
+{
+	table->table[index] = 0;
+}
+
+static void irte_ga_clear_allocated(struct irq_remap_table *table, int index)
+{
+	struct irte_ga *ptr = (struct irte_ga *)table->table;
+	struct irte_ga *irte = &ptr[index];
+
+	memset(&irte->lo.val, 0, sizeof(u64));
+	memset(&irte->hi.val, 0, sizeof(u64));
+}
+
 static int get_devid(struct irq_alloc_info *info)
 {
 	int devid = -1;
@@ -3802,19 +4055,17 @@ static void irq_remapping_prepare_irte(struct amd_ir_data *data,
 {
 	struct irq_2_irte *irte_info = &data->irq_2_irte;
 	struct msi_msg *msg = &data->msi_entry;
-	union irte *irte = &data->irte_entry;
 	struct IO_APIC_route_entry *entry;
+	struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
+
+	if (!iommu)
+		return;
 
 	data->irq_2_irte.devid = devid;
 	data->irq_2_irte.index = index + sub_handle;
-
-	/* Setup IRTE for IOMMU */
-	irte->val = 0;
-	irte->fields.vector      = irq_cfg->vector;
-	irte->fields.int_type    = apic->irq_delivery_mode;
-	irte->fields.destination = irq_cfg->dest_apicid;
-	irte->fields.dm          = apic->irq_dest_mode;
-	irte->fields.valid       = 1;
+	iommu->irte_ops->prepare(data->entry, apic->irq_delivery_mode,
+				 apic->irq_dest_mode, irq_cfg->vector,
+				 irq_cfg->dest_apicid, devid);
 
 	switch (info->type) {
 	case X86_IRQ_ALLOC_TYPE_IOAPIC:
@@ -3845,12 +4096,32 @@ static void irq_remapping_prepare_irte(struct amd_ir_data *data,
 	}
 }
 
+struct amd_irte_ops irte_32_ops = {
+	.prepare = irte_prepare,
+	.activate = irte_activate,
+	.deactivate = irte_deactivate,
+	.set_affinity = irte_set_affinity,
+	.set_allocated = irte_set_allocated,
+	.is_allocated = irte_is_allocated,
+	.clear_allocated = irte_clear_allocated,
+};
+
+struct amd_irte_ops irte_128_ops = {
+	.prepare = irte_ga_prepare,
+	.activate = irte_ga_activate,
+	.deactivate = irte_ga_deactivate,
+	.set_affinity = irte_ga_set_affinity,
+	.set_allocated = irte_ga_set_allocated,
+	.is_allocated = irte_ga_is_allocated,
+	.clear_allocated = irte_ga_clear_allocated,
+};
+
 static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
 			       unsigned int nr_irqs, void *arg)
 {
 	struct irq_alloc_info *info = arg;
 	struct irq_data *irq_data;
-	struct amd_ir_data *data;
+	struct amd_ir_data *data = NULL;
 	struct irq_cfg *cfg;
 	int i, ret, devid;
 	int index = -1;
@@ -3886,6 +4157,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
 	}
 	if (index < 0) {
 		pr_warn("Failed to allocate IRTE\n");
+		ret = index;
 		goto out_free_parent;
 	}
 
@@ -3902,6 +4174,16 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
 		if (!data)
 			goto out_free_data;
 
+		if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
+			data->entry = kzalloc(sizeof(union irte), GFP_KERNEL);
+		else
+			data->entry = kzalloc(sizeof(struct irte_ga),
+						     GFP_KERNEL);
+		if (!data->entry) {
+			kfree(data);
+			goto out_free_data;
+		}
+
 		irq_data->hwirq = (devid << 16) + i;
 		irq_data->chip_data = data;
 		irq_data->chip = &amd_ir_chip;
@@ -3938,6 +4220,7 @@ static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,
 			data = irq_data->chip_data;
 			irte_info = &data->irq_2_irte;
 			free_irte(irte_info->devid, irte_info->index);
+			kfree(data->entry);
 			kfree(data);
 		}
 	}
@@ -3949,8 +4232,11 @@ static void irq_remapping_activate(struct irq_domain *domain,
 {
 	struct amd_ir_data *data = irq_data->chip_data;
 	struct irq_2_irte *irte_info = &data->irq_2_irte;
+	struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid];
 
-	modify_irte(irte_info->devid, irte_info->index, data->irte_entry);
+	if (iommu)
+		iommu->irte_ops->activate(data->entry, irte_info->devid,
+					  irte_info->index);
 }
 
 static void irq_remapping_deactivate(struct irq_domain *domain,
@@ -3958,10 +4244,11 @@ static void irq_remapping_deactivate(struct irq_domain *domain,
 {
 	struct amd_ir_data *data = irq_data->chip_data;
 	struct irq_2_irte *irte_info = &data->irq_2_irte;
-	union irte entry;
+	struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid];
 
-	entry.val = 0;
-	modify_irte(irte_info->devid, irte_info->index, data->irte_entry);
+	if (iommu)
+		iommu->irte_ops->deactivate(data->entry, irte_info->devid,
+					    irte_info->index);
 }
 
 static struct irq_domain_ops amd_ir_domain_ops = {
@@ -3971,6 +4258,70 @@ static struct irq_domain_ops amd_ir_domain_ops = {
 	.deactivate = irq_remapping_deactivate,
 };
 
+static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
+{
+	struct amd_iommu *iommu;
+	struct amd_iommu_pi_data *pi_data = vcpu_info;
+	struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data;
+	struct amd_ir_data *ir_data = data->chip_data;
+	struct irte_ga *irte = (struct irte_ga *) ir_data->entry;
+	struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
+	struct iommu_dev_data *dev_data = search_dev_data(irte_info->devid);
+
+	/* Note:
+	 * This device has never been set up for guest mode.
+	 * we should not modify the IRTE
+	 */
+	if (!dev_data || !dev_data->use_vapic)
+		return 0;
+
+	pi_data->ir_data = ir_data;
+
+	/* Note:
+	 * SVM tries to set up for VAPIC mode, but we are in
+	 * legacy mode. So, we force legacy mode instead.
+	 */
+	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
+		pr_debug("AMD-Vi: %s: Fall back to using intr legacy remap\n",
+			 __func__);
+		pi_data->is_guest_mode = false;
+	}
+
+	iommu = amd_iommu_rlookup_table[irte_info->devid];
+	if (iommu == NULL)
+		return -EINVAL;
+
+	pi_data->prev_ga_tag = ir_data->cached_ga_tag;
+	if (pi_data->is_guest_mode) {
+		/* Setting */
+		irte->hi.fields.ga_root_ptr = (pi_data->base >> 12);
+		irte->hi.fields.vector = vcpu_pi_info->vector;
+		irte->lo.fields_vapic.guest_mode = 1;
+		irte->lo.fields_vapic.ga_tag = pi_data->ga_tag;
+
+		ir_data->cached_ga_tag = pi_data->ga_tag;
+	} else {
+		/* Un-Setting */
+		struct irq_cfg *cfg = irqd_cfg(data);
+
+		irte->hi.val = 0;
+		irte->lo.val = 0;
+		irte->hi.fields.vector = cfg->vector;
+		irte->lo.fields_remap.guest_mode = 0;
+		irte->lo.fields_remap.destination = cfg->dest_apicid;
+		irte->lo.fields_remap.int_type = apic->irq_delivery_mode;
+		irte->lo.fields_remap.dm = apic->irq_dest_mode;
+
+		/*
+		 * This communicates the ga_tag back to the caller
+		 * so that it can do all the necessary clean up.
+		 */
+		ir_data->cached_ga_tag = 0;
+	}
+
+	return modify_irte_ga(irte_info->devid, irte_info->index, irte, ir_data);
+}
+
 static int amd_ir_set_affinity(struct irq_data *data,
 			       const struct cpumask *mask, bool force)
 {
@@ -3978,8 +4329,12 @@ static int amd_ir_set_affinity(struct irq_data *data,
 	struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
 	struct irq_cfg *cfg = irqd_cfg(data);
 	struct irq_data *parent = data->parent_data;
+	struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid];
 	int ret;
 
+	if (!iommu)
+		return -ENODEV;
+
 	ret = parent->chip->irq_set_affinity(parent, mask, force);
 	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
 		return ret;
@@ -3988,9 +4343,8 @@ static int amd_ir_set_affinity(struct irq_data *data,
 	 * Atomically updates the IRTE with the new destination, vector
 	 * and flushes the interrupt entry cache.
 	 */
-	ir_data->irte_entry.fields.vector = cfg->vector;
-	ir_data->irte_entry.fields.destination = cfg->dest_apicid;
-	modify_irte(irte_info->devid, irte_info->index, ir_data->irte_entry);
+	iommu->irte_ops->set_affinity(ir_data->entry, irte_info->devid,
+			    irte_info->index, cfg->vector, cfg->dest_apicid);
 
 	/*
 	 * After this point, all the interrupts will start arriving
@@ -4012,6 +4366,7 @@ static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg)
 static struct irq_chip amd_ir_chip = {
 	.irq_ack = ir_ack_apic_edge,
 	.irq_set_affinity = amd_ir_set_affinity,
+	.irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity,
 	.irq_compose_msi_msg = ir_compose_msi_msg,
 };
 
@@ -4026,4 +4381,43 @@ int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
 
 	return 0;
 }
+
+int amd_iommu_update_ga(int cpu, bool is_run, void *data)
+{
+	unsigned long flags;
+	struct amd_iommu *iommu;
+	struct irq_remap_table *irt;
+	struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
+	int devid = ir_data->irq_2_irte.devid;
+	struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
+	struct irte_ga *ref = (struct irte_ga *) ir_data->ref;
+
+	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||
+	    !ref || !entry || !entry->lo.fields_vapic.guest_mode)
+		return 0;
+
+	iommu = amd_iommu_rlookup_table[devid];
+	if (!iommu)
+		return -ENODEV;
+
+	irt = get_irq_table(devid, false);
+	if (!irt)
+		return -ENODEV;
+
+	spin_lock_irqsave(&irt->lock, flags);
+
+	if (ref->lo.fields_vapic.guest_mode) {
+		if (cpu >= 0)
+			ref->lo.fields_vapic.destination = cpu;
+		ref->lo.fields_vapic.is_run = is_run;
+		barrier();
+	}
+
+	spin_unlock_irqrestore(&irt->lock, flags);
+
+	iommu_flush_irt(iommu, devid);
+	iommu_completion_wait(iommu);
+	return 0;
+}
+EXPORT_SYMBOL(amd_iommu_update_ga);
 #endif
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 59741ead7e15..157e93421fb8 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -20,6 +20,7 @@
 #include <linux/pci.h>
 #include <linux/acpi.h>
 #include <linux/list.h>
+#include <linux/bitmap.h>
 #include <linux/slab.h>
 #include <linux/syscore_ops.h>
 #include <linux/interrupt.h>
@@ -84,6 +85,7 @@
 #define ACPI_DEVFLAG_LINT1              0x80
 #define ACPI_DEVFLAG_ATSDIS             0x10000000
 
+#define LOOP_TIMEOUT	100000
 /*
  * ACPI table definitions
  *
@@ -145,6 +147,8 @@ struct ivmd_header {
 bool amd_iommu_dump;
 bool amd_iommu_irq_remap __read_mostly;
 
+int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
+
 static bool amd_iommu_detected;
 static bool __initdata amd_iommu_disabled;
 static int amd_iommu_target_ivhd_type;
@@ -386,6 +390,10 @@ static void iommu_disable(struct amd_iommu *iommu)
 	iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
 	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
 
+	/* Disable IOMMU GA_LOG */
+	iommu_feature_disable(iommu, CONTROL_GALOG_EN);
+	iommu_feature_disable(iommu, CONTROL_GAINT_EN);
+
 	/* Disable IOMMU hardware itself */
 	iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
 }
@@ -671,6 +679,99 @@ static void __init free_ppr_log(struct amd_iommu *iommu)
 	free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
 }
 
+static void free_ga_log(struct amd_iommu *iommu)
+{
+#ifdef CONFIG_IRQ_REMAP
+	if (iommu->ga_log)
+		free_pages((unsigned long)iommu->ga_log,
+			    get_order(GA_LOG_SIZE));
+	if (iommu->ga_log_tail)
+		free_pages((unsigned long)iommu->ga_log_tail,
+			    get_order(8));
+#endif
+}
+
+static int iommu_ga_log_enable(struct amd_iommu *iommu)
+{
+#ifdef CONFIG_IRQ_REMAP
+	u32 status, i;
+
+	if (!iommu->ga_log)
+		return -EINVAL;
+
+	status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
+
+	/* Check if already running */
+	if (status & (MMIO_STATUS_GALOG_RUN_MASK))
+		return 0;
+
+	iommu_feature_enable(iommu, CONTROL_GAINT_EN);
+	iommu_feature_enable(iommu, CONTROL_GALOG_EN);
+
+	for (i = 0; i < LOOP_TIMEOUT; ++i) {
+		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
+		if (status & (MMIO_STATUS_GALOG_RUN_MASK))
+			break;
+	}
+
+	if (i >= LOOP_TIMEOUT)
+		return -EINVAL;
+#endif /* CONFIG_IRQ_REMAP */
+	return 0;
+}
+
+#ifdef CONFIG_IRQ_REMAP
+static int iommu_init_ga_log(struct amd_iommu *iommu)
+{
+	u64 entry;
+
+	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
+		return 0;
+
+	iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+					get_order(GA_LOG_SIZE));
+	if (!iommu->ga_log)
+		goto err_out;
+
+	iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+					get_order(8));
+	if (!iommu->ga_log_tail)
+		goto err_out;
+
+	entry = (u64)virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
+	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
+		    &entry, sizeof(entry));
+	entry = ((u64)virt_to_phys(iommu->ga_log) & 0xFFFFFFFFFFFFFULL) & ~7ULL;
+	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
+		    &entry, sizeof(entry));
+	writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
+	writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
+
+	return 0;
+err_out:
+	free_ga_log(iommu);
+	return -EINVAL;
+}
+#endif /* CONFIG_IRQ_REMAP */
+
+static int iommu_init_ga(struct amd_iommu *iommu)
+{
+	int ret = 0;
+
+#ifdef CONFIG_IRQ_REMAP
+	/* Note: We have already checked GASup from IVRS table.
+	 *       Now, we need to make sure that GAMSup is set.
+	 */
+	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
+	    !iommu_feature(iommu, FEATURE_GAM_VAPIC))
+		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
+
+	ret = iommu_init_ga_log(iommu);
+#endif /* CONFIG_IRQ_REMAP */
+
+	return ret;
+}
+
 static void iommu_enable_gt(struct amd_iommu *iommu)
 {
 	if (!iommu_feature(iommu, FEATURE_GT))
@@ -1144,6 +1245,7 @@ static void __init free_iommu_one(struct amd_iommu *iommu)
 	free_command_buffer(iommu);
 	free_event_buffer(iommu);
 	free_ppr_log(iommu);
+	free_ga_log(iommu);
 	iommu_unmap_mmio_space(iommu);
 }
 
@@ -1258,6 +1360,8 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
 		else
 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
+		if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0))
+			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
 		break;
 	case 0x11:
 	case 0x40:
@@ -1265,6 +1369,8 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
 		else
 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
+		if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0))
+			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
 		break;
 	default:
 		return -EINVAL;
@@ -1432,6 +1538,7 @@ static int iommu_init_pci(struct amd_iommu *iommu)
 {
 	int cap_ptr = iommu->cap_ptr;
 	u32 range, misc, low, high;
+	int ret;
 
 	iommu->dev = pci_get_bus_and_slot(PCI_BUS_NUM(iommu->devid),
 					  iommu->devid & 0xff);
@@ -1488,6 +1595,10 @@ static int iommu_init_pci(struct amd_iommu *iommu)
 	if (iommu_feature(iommu, FEATURE_PPR) && alloc_ppr_log(iommu))
 		return -ENOMEM;
 
+	ret = iommu_init_ga(iommu);
+	if (ret)
+		return ret;
+
 	if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))
 		amd_iommu_np_cache = true;
 
@@ -1545,16 +1656,24 @@ static void print_iommu_info(void)
 			dev_name(&iommu->dev->dev), iommu->cap_ptr);
 
 		if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
-			pr_info("AMD-Vi:  Extended features: ");
+			pr_info("AMD-Vi: Extended features (%#llx):\n",
+				iommu->features);
 			for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
 				if (iommu_feature(iommu, (1ULL << i)))
 					pr_cont(" %s", feat_str[i]);
 			}
+
+			if (iommu->features & FEATURE_GAM_VAPIC)
+				pr_cont(" GA_vAPIC");
+
 			pr_cont("\n");
 		}
 	}
-	if (irq_remapping_enabled)
+	if (irq_remapping_enabled) {
 		pr_info("AMD-Vi: Interrupt remapping enabled\n");
+		if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
+			pr_info("AMD-Vi: virtual APIC enabled\n");
+	}
 }
 
 static int __init amd_iommu_init_pci(void)
@@ -1645,6 +1764,8 @@ enable_faults:
 	if (iommu->ppr_log != NULL)
 		iommu_feature_enable(iommu, CONTROL_PPFINT_EN);
 
+	iommu_ga_log_enable(iommu);
+
 	return 0;
 }
 
@@ -1862,6 +1983,24 @@ static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
 			       iommu->stored_addr_lo | 1);
 }
 
+static void iommu_enable_ga(struct amd_iommu *iommu)
+{
+#ifdef CONFIG_IRQ_REMAP
+	switch (amd_iommu_guest_ir) {
+	case AMD_IOMMU_GUEST_IR_VAPIC:
+		iommu_feature_enable(iommu, CONTROL_GAM_EN);
+		/* Fall through */
+	case AMD_IOMMU_GUEST_IR_LEGACY_GA:
+		iommu_feature_enable(iommu, CONTROL_GA_EN);
+		iommu->irte_ops = &irte_128_ops;
+		break;
+	default:
+		iommu->irte_ops = &irte_32_ops;
+		break;
+	}
+#endif
+}
+
 /*
  * This function finally enables all IOMMUs found in the system after
  * they have been initialized
@@ -1877,9 +2016,15 @@ static void early_enable_iommus(void)
 		iommu_enable_command_buffer(iommu);
 		iommu_enable_event_buffer(iommu);
 		iommu_set_exclusion_range(iommu);
+		iommu_enable_ga(iommu);
 		iommu_enable(iommu);
 		iommu_flush_all_caches(iommu);
 	}
+
+#ifdef CONFIG_IRQ_REMAP
+	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
+		amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP);
+#endif
 }
 
 static void enable_iommus_v2(void)
@@ -1905,6 +2050,11 @@ static void disable_iommus(void)
 
 	for_each_iommu(iommu)
 		iommu_disable(iommu);
+
+#ifdef CONFIG_IRQ_REMAP
+	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
+		amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP);
+#endif
 }
 
 /*
@@ -2059,7 +2209,7 @@ static int __init early_amd_iommu_init(void)
 	struct acpi_table_header *ivrs_base;
 	acpi_size ivrs_size;
 	acpi_status status;
-	int i, ret = 0;
+	int i, remap_cache_sz, ret = 0;
 
 	if (!amd_iommu_detected)
 		return -ENODEV;
@@ -2136,7 +2286,7 @@ static int __init early_amd_iommu_init(void)
 	 * never allocate domain 0 because its used as the non-allocated and
 	 * error value placeholder
 	 */
-	amd_iommu_pd_alloc_bitmap[0] = 1;
+	__set_bit(0, amd_iommu_pd_alloc_bitmap);
 
 	spin_lock_init(&amd_iommu_pd_lock);
 
@@ -2157,10 +2307,14 @@ static int __init early_amd_iommu_init(void)
 		 * remapping tables.
 		 */
 		ret = -ENOMEM;
+		if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
+			remap_cache_sz = MAX_IRQS_PER_TABLE * sizeof(u32);
+		else
+			remap_cache_sz = MAX_IRQS_PER_TABLE * (sizeof(u64) * 2);
 		amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
-				MAX_IRQS_PER_TABLE * sizeof(u32),
-				IRQ_TABLE_ALIGNMENT,
-				0, NULL);
+							remap_cache_sz,
+							IRQ_TABLE_ALIGNMENT,
+							0, NULL);
 		if (!amd_iommu_irq_cache)
 			goto out;
 
@@ -2413,6 +2567,21 @@ static int __init parse_amd_iommu_dump(char *str)
 	return 1;
 }
 
+static int __init parse_amd_iommu_intr(char *str)
+{
+	for (; *str; ++str) {
+		if (strncmp(str, "legacy", 6) == 0) {
+			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
+			break;
+		}
+		if (strncmp(str, "vapic", 5) == 0) {
+			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
+			break;
+		}
+	}
+	return 1;
+}
+
 static int __init parse_amd_iommu_options(char *str)
 {
 	for (; *str; ++str) {
@@ -2521,6 +2690,7 @@ static int __init parse_ivrs_acpihid(char *str)
 
 __setup("amd_iommu_dump",	parse_amd_iommu_dump);
 __setup("amd_iommu=",		parse_amd_iommu_options);
+__setup("amd_iommu_intr=",	parse_amd_iommu_intr);
 __setup("ivrs_ioapic",		parse_ivrs_ioapic);
 __setup("ivrs_hpet",		parse_ivrs_hpet);
 __setup("ivrs_acpihid",		parse_ivrs_acpihid);
diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h
index 0bd9eb374462..7eb60c15c582 100644
--- a/drivers/iommu/amd_iommu_proto.h
+++ b/drivers/iommu/amd_iommu_proto.h
@@ -38,6 +38,7 @@ extern int amd_iommu_enable(void);
 extern void amd_iommu_disable(void);
 extern int amd_iommu_reenable(int);
 extern int amd_iommu_enable_faulting(void);
+extern int amd_iommu_guest_ir;
 
 /* IOMMUv2 specific functions */
 struct iommu_domain;
@@ -78,12 +79,6 @@ static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
 extern int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
 				  int status, int tag);
 
-#ifndef CONFIG_AMD_IOMMU_STATS
-
-static inline void amd_iommu_stats_init(void) { }
-
-#endif /* !CONFIG_AMD_IOMMU_STATS */
-
 static inline bool is_rd890_iommu(struct pci_dev *pdev)
 {
 	return (pdev->vendor == PCI_VENDOR_ID_ATI) &&
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index caf5e3822715..0d91785ebdc3 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -22,6 +22,7 @@
 
 #include <linux/types.h>
 #include <linux/mutex.h>
+#include <linux/msi.h>
 #include <linux/list.h>
 #include <linux/spinlock.h>
 #include <linux/pci.h>
@@ -69,6 +70,8 @@
 #define MMIO_EXCL_LIMIT_OFFSET  0x0028
 #define MMIO_EXT_FEATURES	0x0030
 #define MMIO_PPR_LOG_OFFSET	0x0038
+#define MMIO_GA_LOG_BASE_OFFSET	0x00e0
+#define MMIO_GA_LOG_TAIL_OFFSET	0x00e8
 #define MMIO_CMD_HEAD_OFFSET	0x2000
 #define MMIO_CMD_TAIL_OFFSET	0x2008
 #define MMIO_EVT_HEAD_OFFSET	0x2010
@@ -76,6 +79,8 @@
 #define MMIO_STATUS_OFFSET	0x2020
 #define MMIO_PPR_HEAD_OFFSET	0x2030
 #define MMIO_PPR_TAIL_OFFSET	0x2038
+#define MMIO_GA_HEAD_OFFSET	0x2040
+#define MMIO_GA_TAIL_OFFSET	0x2048
 #define MMIO_CNTR_CONF_OFFSET	0x4000
 #define MMIO_CNTR_REG_OFFSET	0x40000
 #define MMIO_REG_END_OFFSET	0x80000
@@ -92,6 +97,7 @@
 #define FEATURE_GA		(1ULL<<7)
 #define FEATURE_HE		(1ULL<<8)
 #define FEATURE_PC		(1ULL<<9)
+#define FEATURE_GAM_VAPIC	(1ULL<<21)
 
 #define FEATURE_PASID_SHIFT	32
 #define FEATURE_PASID_MASK	(0x1fULL << FEATURE_PASID_SHIFT)
@@ -110,6 +116,9 @@
 #define MMIO_STATUS_EVT_INT_MASK	(1 << 1)
 #define MMIO_STATUS_COM_WAIT_INT_MASK	(1 << 2)
 #define MMIO_STATUS_PPR_INT_MASK	(1 << 6)
+#define MMIO_STATUS_GALOG_RUN_MASK	(1 << 8)
+#define MMIO_STATUS_GALOG_OVERFLOW_MASK	(1 << 9)
+#define MMIO_STATUS_GALOG_INT_MASK	(1 << 10)
 
 /* event logging constants */
 #define EVENT_ENTRY_SIZE	0x10
@@ -146,6 +155,10 @@
 #define CONTROL_PPFINT_EN       0x0eULL
 #define CONTROL_PPR_EN          0x0fULL
 #define CONTROL_GT_EN           0x10ULL
+#define CONTROL_GA_EN           0x11ULL
+#define CONTROL_GAM_EN          0x19ULL
+#define CONTROL_GALOG_EN        0x1CULL
+#define CONTROL_GAINT_EN        0x1DULL
 
 #define CTRL_INV_TO_MASK	(7 << CONTROL_INV_TIMEOUT)
 #define CTRL_INV_TO_NONE	0
@@ -224,6 +237,19 @@
 
 #define PPR_REQ_FAULT		0x01
 
+/* Constants for GA Log handling */
+#define GA_LOG_ENTRIES		512
+#define GA_LOG_SIZE_SHIFT	56
+#define GA_LOG_SIZE_512		(0x8ULL << GA_LOG_SIZE_SHIFT)
+#define GA_ENTRY_SIZE		8
+#define GA_LOG_SIZE		(GA_ENTRY_SIZE * GA_LOG_ENTRIES)
+
+#define GA_TAG(x)		(u32)(x & 0xffffffffULL)
+#define GA_DEVID(x)		(u16)(((x) >> 32) & 0xffffULL)
+#define GA_REQ_TYPE(x)		(((x) >> 60) & 0xfULL)
+
+#define GA_GUEST_NR		0x1
+
 #define PAGE_MODE_NONE    0x00
 #define PAGE_MODE_1_LEVEL 0x01
 #define PAGE_MODE_2_LEVEL 0x02
@@ -329,6 +355,12 @@
 #define IOMMU_CAP_NPCACHE 26
 #define IOMMU_CAP_EFR     27
 
+/* IOMMU Feature Reporting Field (for IVHD type 10h */
+#define IOMMU_FEAT_GASUP_SHIFT	6
+
+/* IOMMU Extended Feature Register (EFR) */
+#define IOMMU_EFR_GASUP_SHIFT	7
+
 #define MAX_DOMAIN_ID 65536
 
 /* Protection domain flags */
@@ -400,6 +432,7 @@ struct amd_iommu_fault {
 
 struct iommu_domain;
 struct irq_domain;
+struct amd_irte_ops;
 
 /*
  * This structure contains generic data for  IOMMU protection domains
@@ -490,6 +523,12 @@ struct amd_iommu {
 	/* Base of the PPR log, if present */
 	u8 *ppr_log;
 
+	/* Base of the GA log, if present */
+	u8 *ga_log;
+
+	/* Tail of the GA log, if present */
+	u8 *ga_log_tail;
+
 	/* true if interrupts for this IOMMU are already enabled */
 	bool int_enabled;
 
@@ -523,7 +562,11 @@ struct amd_iommu {
 #ifdef CONFIG_IRQ_REMAP
 	struct irq_domain *ir_domain;
 	struct irq_domain *msi_domain;
+
+	struct amd_irte_ops *irte_ops;
 #endif
+
+	volatile u64 __aligned(8) cmd_sem;
 };
 
 #define ACPIHID_UID_LEN 256
@@ -681,4 +724,112 @@ static inline int get_hpet_devid(int id)
 	return -EINVAL;
 }
 
+enum amd_iommu_intr_mode_type {
+	AMD_IOMMU_GUEST_IR_LEGACY,
+
+	/* This mode is not visible to users. It is used when
+	 * we cannot fully enable vAPIC and fallback to only support
+	 * legacy interrupt remapping via 128-bit IRTE.
+	 */
+	AMD_IOMMU_GUEST_IR_LEGACY_GA,
+	AMD_IOMMU_GUEST_IR_VAPIC,
+};
+
+#define AMD_IOMMU_GUEST_IR_GA(x)	(x == AMD_IOMMU_GUEST_IR_VAPIC || \
+					 x == AMD_IOMMU_GUEST_IR_LEGACY_GA)
+
+#define AMD_IOMMU_GUEST_IR_VAPIC(x)	(x == AMD_IOMMU_GUEST_IR_VAPIC)
+
+union irte {
+	u32 val;
+	struct {
+		u32 valid	: 1,
+		    no_fault	: 1,
+		    int_type	: 3,
+		    rq_eoi	: 1,
+		    dm		: 1,
+		    rsvd_1	: 1,
+		    destination	: 8,
+		    vector	: 8,
+		    rsvd_2	: 8;
+	} fields;
+};
+
+union irte_ga_lo {
+	u64 val;
+
+	/* For int remapping */
+	struct {
+		u64 valid	: 1,
+		    no_fault	: 1,
+		    /* ------ */
+		    int_type	: 3,
+		    rq_eoi	: 1,
+		    dm		: 1,
+		    /* ------ */
+		    guest_mode	: 1,
+		    destination	: 8,
+		    rsvd	: 48;
+	} fields_remap;
+
+	/* For guest vAPIC */
+	struct {
+		u64 valid	: 1,
+		    no_fault	: 1,
+		    /* ------ */
+		    ga_log_intr	: 1,
+		    rsvd1	: 3,
+		    is_run	: 1,
+		    /* ------ */
+		    guest_mode	: 1,
+		    destination	: 8,
+		    rsvd2	: 16,
+		    ga_tag	: 32;
+	} fields_vapic;
+};
+
+union irte_ga_hi {
+	u64 val;
+	struct {
+		u64 vector	: 8,
+		    rsvd_1	: 4,
+		    ga_root_ptr	: 40,
+		    rsvd_2	: 12;
+	} fields;
+};
+
+struct irte_ga {
+	union irte_ga_lo lo;
+	union irte_ga_hi hi;
+};
+
+struct irq_2_irte {
+	u16 devid; /* Device ID for IRTE table */
+	u16 index; /* Index into IRTE table*/
+};
+
+struct amd_ir_data {
+	u32 cached_ga_tag;
+	struct irq_2_irte irq_2_irte;
+	struct msi_msg msi_entry;
+	void *entry;    /* Pointer to union irte or struct irte_ga */
+	void *ref;      /* Pointer to the actual irte */
+};
+
+struct amd_irte_ops {
+	void (*prepare)(void *, u32, u32, u8, u32, int);
+	void (*activate)(void *, u16, u16);
+	void (*deactivate)(void *, u16, u16);
+	void (*set_affinity)(void *, u16, u16, u8, u32);
+	void *(*get)(struct irq_remap_table *, int);
+	void (*set_allocated)(struct irq_remap_table *, int);
+	bool (*is_allocated)(struct irq_remap_table *, int);
+	void (*clear_allocated)(struct irq_remap_table *, int);
+};
+
+#ifdef CONFIG_IRQ_REMAP
+extern struct amd_irte_ops irte_32_ops;
+extern struct amd_irte_ops irte_128_ops;
+#endif
+
 #endif /* _ASM_X86_AMD_IOMMU_TYPES_H */
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index ce801170d5f2..15c01c3cd540 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -30,10 +30,13 @@
 #include <linux/msi.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/of_iommu.h>
 #include <linux/of_platform.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 
+#include <linux/amba/bus.h>
+
 #include "io-pgtable.h"
 
 /* MMIO registers */
@@ -123,6 +126,10 @@
 #define CR2_RECINVSID			(1 << 1)
 #define CR2_E2H				(1 << 0)
 
+#define ARM_SMMU_GBPA			0x44
+#define GBPA_ABORT			(1 << 20)
+#define GBPA_UPDATE			(1 << 31)
+
 #define ARM_SMMU_IRQ_CTRL		0x50
 #define IRQ_CTRL_EVTQ_IRQEN		(1 << 2)
 #define IRQ_CTRL_PRIQ_IRQEN		(1 << 1)
@@ -260,6 +267,9 @@
 #define STRTAB_STE_1_SHCFG_INCOMING	1UL
 #define STRTAB_STE_1_SHCFG_SHIFT	44
 
+#define STRTAB_STE_1_PRIVCFG_UNPRIV	2UL
+#define STRTAB_STE_1_PRIVCFG_SHIFT	48
+
 #define STRTAB_STE_2_S2VMID_SHIFT	0
 #define STRTAB_STE_2_S2VMID_MASK	0xffffUL
 #define STRTAB_STE_2_VTCR_SHIFT		32
@@ -606,12 +616,9 @@ struct arm_smmu_device {
 	struct arm_smmu_strtab_cfg	strtab_cfg;
 };
 
-/* SMMU private data for an IOMMU group */
-struct arm_smmu_group {
+/* SMMU private data for each master */
+struct arm_smmu_master_data {
 	struct arm_smmu_device		*smmu;
-	struct arm_smmu_domain		*domain;
-	int				num_sids;
-	u32				*sids;
 	struct arm_smmu_strtab_ent	ste;
 };
 
@@ -713,19 +720,15 @@ static void queue_inc_prod(struct arm_smmu_queue *q)
 	writel(q->prod, q->prod_reg);
 }
 
-static bool __queue_cons_before(struct arm_smmu_queue *q, u32 until)
-{
-	if (Q_WRP(q, q->cons) == Q_WRP(q, until))
-		return Q_IDX(q, q->cons) < Q_IDX(q, until);
-
-	return Q_IDX(q, q->cons) >= Q_IDX(q, until);
-}
-
-static int queue_poll_cons(struct arm_smmu_queue *q, u32 until, bool wfe)
+/*
+ * Wait for the SMMU to consume items. If drain is true, wait until the queue
+ * is empty. Otherwise, wait until there is at least one free slot.
+ */
+static int queue_poll_cons(struct arm_smmu_queue *q, bool drain, bool wfe)
 {
 	ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
 
-	while (queue_sync_cons(q), __queue_cons_before(q, until)) {
+	while (queue_sync_cons(q), (drain ? !queue_empty(q) : queue_full(q))) {
 		if (ktime_compare(ktime_get(), timeout) > 0)
 			return -ETIMEDOUT;
 
@@ -879,7 +882,7 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
 	 * We may have concurrent producers, so we need to be careful
 	 * not to touch any of the shadow cmdq state.
 	 */
-	queue_read(cmd, Q_ENT(q, idx), q->ent_dwords);
+	queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
 	dev_err(smmu->dev, "skipping command in error state:\n");
 	for (i = 0; i < ARRAY_SIZE(cmd); ++i)
 		dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
@@ -890,14 +893,14 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
 		return;
 	}
 
-	queue_write(cmd, Q_ENT(q, idx), q->ent_dwords);
+	queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
 }
 
 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
 				    struct arm_smmu_cmdq_ent *ent)
 {
-	u32 until;
 	u64 cmd[CMDQ_ENT_DWORDS];
+	unsigned long flags;
 	bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
 	struct arm_smmu_queue *q = &smmu->cmdq.q;
 
@@ -907,20 +910,15 @@ static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
 		return;
 	}
 
-	spin_lock(&smmu->cmdq.lock);
-	while (until = q->prod + 1, queue_insert_raw(q, cmd) == -ENOSPC) {
-		/*
-		 * Keep the queue locked, otherwise the producer could wrap
-		 * twice and we could see a future consumer pointer that looks
-		 * like it's behind us.
-		 */
-		if (queue_poll_cons(q, until, wfe))
+	spin_lock_irqsave(&smmu->cmdq.lock, flags);
+	while (queue_insert_raw(q, cmd) == -ENOSPC) {
+		if (queue_poll_cons(q, false, wfe))
 			dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
 	}
 
-	if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, until, wfe))
+	if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, true, wfe))
 		dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
-	spin_unlock(&smmu->cmdq.lock);
+	spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
 }
 
 /* Context descriptor manipulation functions */
@@ -1034,6 +1032,9 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
 		case STRTAB_STE_0_CFG_S2_TRANS:
 			ste_live = true;
 			break;
+		case STRTAB_STE_0_CFG_ABORT:
+			if (disable_bypass)
+				break;
 		default:
 			BUG(); /* STE corruption */
 		}
@@ -1070,7 +1071,9 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
 #ifdef CONFIG_PCI_ATS
 			 STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
 #endif
-			 STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT);
+			 STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT |
+			 STRTAB_STE_1_PRIVCFG_UNPRIV <<
+			 STRTAB_STE_1_PRIVCFG_SHIFT);
 
 		if (smmu->features & ARM_SMMU_FEAT_STALLS)
 			dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
@@ -1158,36 +1161,66 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
 	struct arm_smmu_queue *q = &smmu->evtq.q;
 	u64 evt[EVTQ_ENT_DWORDS];
 
-	while (!queue_remove_raw(q, evt)) {
-		u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
+	do {
+		while (!queue_remove_raw(q, evt)) {
+			u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
 
-		dev_info(smmu->dev, "event 0x%02x received:\n", id);
-		for (i = 0; i < ARRAY_SIZE(evt); ++i)
-			dev_info(smmu->dev, "\t0x%016llx\n",
-				 (unsigned long long)evt[i]);
-	}
+			dev_info(smmu->dev, "event 0x%02x received:\n", id);
+			for (i = 0; i < ARRAY_SIZE(evt); ++i)
+				dev_info(smmu->dev, "\t0x%016llx\n",
+					 (unsigned long long)evt[i]);
+
+		}
+
+		/*
+		 * Not much we can do on overflow, so scream and pretend we're
+		 * trying harder.
+		 */
+		if (queue_sync_prod(q) == -EOVERFLOW)
+			dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
+	} while (!queue_empty(q));
 
 	/* Sync our overflow flag, as we believe we're up to speed */
 	q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
 	return IRQ_HANDLED;
 }
 
-static irqreturn_t arm_smmu_evtq_handler(int irq, void *dev)
-{
-	irqreturn_t ret = IRQ_WAKE_THREAD;
-	struct arm_smmu_device *smmu = dev;
-	struct arm_smmu_queue *q = &smmu->evtq.q;
+static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
+{
+	u32 sid, ssid;
+	u16 grpid;
+	bool ssv, last;
+
+	sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK;
+	ssv = evt[0] & PRIQ_0_SSID_V;
+	ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0;
+	last = evt[0] & PRIQ_0_PRG_LAST;
+	grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK;
+
+	dev_info(smmu->dev, "unexpected PRI request received:\n");
+	dev_info(smmu->dev,
+		 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
+		 sid, ssid, grpid, last ? "L" : "",
+		 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
+		 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
+		 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
+		 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
+		 evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT);
+
+	if (last) {
+		struct arm_smmu_cmdq_ent cmd = {
+			.opcode			= CMDQ_OP_PRI_RESP,
+			.substream_valid	= ssv,
+			.pri			= {
+				.sid	= sid,
+				.ssid	= ssid,
+				.grpid	= grpid,
+				.resp	= PRI_RESP_DENY,
+			},
+		};
 
-	/*
-	 * Not much we can do on overflow, so scream and pretend we're
-	 * trying harder.
-	 */
-	if (queue_sync_prod(q) == -EOVERFLOW)
-		dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
-	else if (queue_empty(q))
-		ret = IRQ_NONE;
-
-	return ret;
+		arm_smmu_cmdq_issue_cmd(smmu, &cmd);
+	}
 }
 
 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
@@ -1196,63 +1229,19 @@ static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
 	struct arm_smmu_queue *q = &smmu->priq.q;
 	u64 evt[PRIQ_ENT_DWORDS];
 
-	while (!queue_remove_raw(q, evt)) {
-		u32 sid, ssid;
-		u16 grpid;
-		bool ssv, last;
+	do {
+		while (!queue_remove_raw(q, evt))
+			arm_smmu_handle_ppr(smmu, evt);
 
-		sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK;
-		ssv = evt[0] & PRIQ_0_SSID_V;
-		ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0;
-		last = evt[0] & PRIQ_0_PRG_LAST;
-		grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK;
-
-		dev_info(smmu->dev, "unexpected PRI request received:\n");
-		dev_info(smmu->dev,
-			 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
-			 sid, ssid, grpid, last ? "L" : "",
-			 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
-			 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
-			 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
-			 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
-			 evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT);
-
-		if (last) {
-			struct arm_smmu_cmdq_ent cmd = {
-				.opcode			= CMDQ_OP_PRI_RESP,
-				.substream_valid	= ssv,
-				.pri			= {
-					.sid	= sid,
-					.ssid	= ssid,
-					.grpid	= grpid,
-					.resp	= PRI_RESP_DENY,
-				},
-			};
-
-			arm_smmu_cmdq_issue_cmd(smmu, &cmd);
-		}
-	}
+		if (queue_sync_prod(q) == -EOVERFLOW)
+			dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
+	} while (!queue_empty(q));
 
 	/* Sync our overflow flag, as we believe we're up to speed */
 	q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
 	return IRQ_HANDLED;
 }
 
-static irqreturn_t arm_smmu_priq_handler(int irq, void *dev)
-{
-	irqreturn_t ret = IRQ_WAKE_THREAD;
-	struct arm_smmu_device *smmu = dev;
-	struct arm_smmu_queue *q = &smmu->priq.q;
-
-	/* PRIQ overflow indicates a programming error */
-	if (queue_sync_prod(q) == -EOVERFLOW)
-		dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
-	else if (queue_empty(q))
-		ret = IRQ_NONE;
-
-	return ret;
-}
-
 static irqreturn_t arm_smmu_cmdq_sync_handler(int irq, void *dev)
 {
 	/* We don't actually use CMD_SYNC interrupts for anything */
@@ -1285,15 +1274,11 @@ static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
 	if (active & GERROR_MSI_GERROR_ABT_ERR)
 		dev_warn(smmu->dev, "GERROR MSI write aborted\n");
 
-	if (active & GERROR_MSI_PRIQ_ABT_ERR) {
+	if (active & GERROR_MSI_PRIQ_ABT_ERR)
 		dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
-		arm_smmu_priq_handler(irq, smmu->dev);
-	}
 
-	if (active & GERROR_MSI_EVTQ_ABT_ERR) {
+	if (active & GERROR_MSI_EVTQ_ABT_ERR)
 		dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
-		arm_smmu_evtq_handler(irq, smmu->dev);
-	}
 
 	if (active & GERROR_MSI_CMDQ_ABT_ERR) {
 		dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
@@ -1566,6 +1551,8 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain)
 		return -ENOMEM;
 
 	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
+	domain->geometry.aperture_end = (1UL << ias) - 1;
+	domain->geometry.force_aperture = true;
 	smmu_domain->pgtbl_ops = pgtbl_ops;
 
 	ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
@@ -1575,20 +1562,6 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain)
 	return ret;
 }
 
-static struct arm_smmu_group *arm_smmu_group_get(struct device *dev)
-{
-	struct iommu_group *group;
-	struct arm_smmu_group *smmu_group;
-
-	group = iommu_group_get(dev);
-	if (!group)
-		return NULL;
-
-	smmu_group = iommu_group_get_iommudata(group);
-	iommu_group_put(group);
-	return smmu_group;
-}
-
 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
 {
 	__le64 *step;
@@ -1611,27 +1584,17 @@ static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
 	return step;
 }
 
-static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group)
+static int arm_smmu_install_ste_for_dev(struct iommu_fwspec *fwspec)
 {
 	int i;
-	struct arm_smmu_domain *smmu_domain = smmu_group->domain;
-	struct arm_smmu_strtab_ent *ste = &smmu_group->ste;
-	struct arm_smmu_device *smmu = smmu_group->smmu;
+	struct arm_smmu_master_data *master = fwspec->iommu_priv;
+	struct arm_smmu_device *smmu = master->smmu;
 
-	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-		ste->s1_cfg = &smmu_domain->s1_cfg;
-		ste->s2_cfg = NULL;
-		arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
-	} else {
-		ste->s1_cfg = NULL;
-		ste->s2_cfg = &smmu_domain->s2_cfg;
-	}
-
-	for (i = 0; i < smmu_group->num_sids; ++i) {
-		u32 sid = smmu_group->sids[i];
+	for (i = 0; i < fwspec->num_ids; ++i) {
+		u32 sid = fwspec->ids[i];
 		__le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
 
-		arm_smmu_write_strtab_ent(smmu, sid, step, ste);
+		arm_smmu_write_strtab_ent(smmu, sid, step, &master->ste);
 	}
 
 	return 0;
@@ -1639,13 +1602,11 @@ static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group)
 
 static void arm_smmu_detach_dev(struct device *dev)
 {
-	struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
+	struct arm_smmu_master_data *master = dev->iommu_fwspec->iommu_priv;
 
-	smmu_group->ste.bypass = true;
-	if (arm_smmu_install_ste_for_group(smmu_group) < 0)
+	master->ste.bypass = true;
+	if (arm_smmu_install_ste_for_dev(dev->iommu_fwspec) < 0)
 		dev_warn(dev, "failed to install bypass STE\n");
-
-	smmu_group->domain = NULL;
 }
 
 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
@@ -1653,16 +1614,20 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 	int ret = 0;
 	struct arm_smmu_device *smmu;
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
-	struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
+	struct arm_smmu_master_data *master;
+	struct arm_smmu_strtab_ent *ste;
 
-	if (!smmu_group)
+	if (!dev->iommu_fwspec)
 		return -ENOENT;
 
+	master = dev->iommu_fwspec->iommu_priv;
+	smmu = master->smmu;
+	ste = &master->ste;
+
 	/* Already attached to a different domain? */
-	if (smmu_group->domain && smmu_group->domain != smmu_domain)
+	if (!ste->bypass)
 		arm_smmu_detach_dev(dev);
 
-	smmu = smmu_group->smmu;
 	mutex_lock(&smmu_domain->init_mutex);
 
 	if (!smmu_domain->smmu) {
@@ -1681,21 +1646,21 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 		goto out_unlock;
 	}
 
-	/* Group already attached to this domain? */
-	if (smmu_group->domain)
-		goto out_unlock;
-
-	smmu_group->domain	= smmu_domain;
+	ste->bypass = false;
+	ste->valid = true;
 
-	/*
-	 * FIXME: This should always be "false" once we have IOMMU-backed
-	 * DMA ops for all devices behind the SMMU.
-	 */
-	smmu_group->ste.bypass	= domain->type == IOMMU_DOMAIN_DMA;
+	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
+		ste->s1_cfg = &smmu_domain->s1_cfg;
+		ste->s2_cfg = NULL;
+		arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
+	} else {
+		ste->s1_cfg = NULL;
+		ste->s2_cfg = &smmu_domain->s2_cfg;
+	}
 
-	ret = arm_smmu_install_ste_for_group(smmu_group);
+	ret = arm_smmu_install_ste_for_dev(dev->iommu_fwspec);
 	if (ret < 0)
-		smmu_group->domain = NULL;
+		ste->valid = false;
 
 out_unlock:
 	mutex_unlock(&smmu_domain->init_mutex);
@@ -1754,40 +1719,19 @@ arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
 	return ret;
 }
 
-static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *sidp)
-{
-	*(u32 *)sidp = alias;
-	return 0; /* Continue walking */
-}
+static struct platform_driver arm_smmu_driver;
 
-static void __arm_smmu_release_pci_iommudata(void *data)
+static int arm_smmu_match_node(struct device *dev, void *data)
 {
-	kfree(data);
+	return dev->of_node == data;
 }
 
-static struct arm_smmu_device *arm_smmu_get_for_pci_dev(struct pci_dev *pdev)
+static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
 {
-	struct device_node *of_node;
-	struct platform_device *smmu_pdev;
-	struct arm_smmu_device *smmu = NULL;
-	struct pci_bus *bus = pdev->bus;
-
-	/* Walk up to the root bus */
-	while (!pci_is_root_bus(bus))
-		bus = bus->parent;
-
-	/* Follow the "iommus" phandle from the host controller */
-	of_node = of_parse_phandle(bus->bridge->parent->of_node, "iommus", 0);
-	if (!of_node)
-		return NULL;
-
-	/* See if we can find an SMMU corresponding to the phandle */
-	smmu_pdev = of_find_device_by_node(of_node);
-	if (smmu_pdev)
-		smmu = platform_get_drvdata(smmu_pdev);
-
-	of_node_put(of_node);
-	return smmu;
+	struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
+						np, arm_smmu_match_node);
+	put_device(dev);
+	return dev ? dev_get_drvdata(dev) : NULL;
 }
 
 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
@@ -1800,94 +1744,91 @@ static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
 	return sid < limit;
 }
 
+static struct iommu_ops arm_smmu_ops;
+
 static int arm_smmu_add_device(struct device *dev)
 {
 	int i, ret;
-	u32 sid, *sids;
-	struct pci_dev *pdev;
-	struct iommu_group *group;
-	struct arm_smmu_group *smmu_group;
 	struct arm_smmu_device *smmu;
+	struct arm_smmu_master_data *master;
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	struct iommu_group *group;
 
-	/* We only support PCI, for now */
-	if (!dev_is_pci(dev))
+	if (!fwspec || fwspec->ops != &arm_smmu_ops)
 		return -ENODEV;
-
-	pdev = to_pci_dev(dev);
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return PTR_ERR(group);
-
-	smmu_group = iommu_group_get_iommudata(group);
-	if (!smmu_group) {
-		smmu = arm_smmu_get_for_pci_dev(pdev);
-		if (!smmu) {
-			ret = -ENOENT;
-			goto out_remove_dev;
-		}
-
-		smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL);
-		if (!smmu_group) {
-			ret = -ENOMEM;
-			goto out_remove_dev;
-		}
-
-		smmu_group->ste.valid	= true;
-		smmu_group->smmu	= smmu;
-		iommu_group_set_iommudata(group, smmu_group,
-					  __arm_smmu_release_pci_iommudata);
+	/*
+	 * We _can_ actually withstand dodgy bus code re-calling add_device()
+	 * without an intervening remove_device()/of_xlate() sequence, but
+	 * we're not going to do so quietly...
+	 */
+	if (WARN_ON_ONCE(fwspec->iommu_priv)) {
+		master = fwspec->iommu_priv;
+		smmu = master->smmu;
 	} else {
-		smmu = smmu_group->smmu;
-	}
+		smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
+		if (!smmu)
+			return -ENODEV;
+		master = kzalloc(sizeof(*master), GFP_KERNEL);
+		if (!master)
+			return -ENOMEM;
 
-	/* Assume SID == RID until firmware tells us otherwise */
-	pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
-	for (i = 0; i < smmu_group->num_sids; ++i) {
-		/* If we already know about this SID, then we're done */
-		if (smmu_group->sids[i] == sid)
-			goto out_put_group;
+		master->smmu = smmu;
+		fwspec->iommu_priv = master;
 	}
 
-	/* Check the SID is in range of the SMMU and our stream table */
-	if (!arm_smmu_sid_in_range(smmu, sid)) {
-		ret = -ERANGE;
-		goto out_remove_dev;
-	}
+	/* Check the SIDs are in range of the SMMU and our stream table */
+	for (i = 0; i < fwspec->num_ids; i++) {
+		u32 sid = fwspec->ids[i];
 
-	/* Ensure l2 strtab is initialised */
-	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
-		ret = arm_smmu_init_l2_strtab(smmu, sid);
-		if (ret)
-			goto out_remove_dev;
-	}
+		if (!arm_smmu_sid_in_range(smmu, sid))
+			return -ERANGE;
 
-	/* Resize the SID array for the group */
-	smmu_group->num_sids++;
-	sids = krealloc(smmu_group->sids, smmu_group->num_sids * sizeof(*sids),
-			GFP_KERNEL);
-	if (!sids) {
-		smmu_group->num_sids--;
-		ret = -ENOMEM;
-		goto out_remove_dev;
+		/* Ensure l2 strtab is initialised */
+		if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
+			ret = arm_smmu_init_l2_strtab(smmu, sid);
+			if (ret)
+				return ret;
+		}
 	}
 
-	/* Add the new SID */
-	sids[smmu_group->num_sids - 1] = sid;
-	smmu_group->sids = sids;
-
-out_put_group:
-	iommu_group_put(group);
-	return 0;
+	group = iommu_group_get_for_dev(dev);
+	if (!IS_ERR(group))
+		iommu_group_put(group);
 
-out_remove_dev:
-	iommu_group_remove_device(dev);
-	iommu_group_put(group);
-	return ret;
+	return PTR_ERR_OR_ZERO(group);
 }
 
 static void arm_smmu_remove_device(struct device *dev)
 {
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	struct arm_smmu_master_data *master;
+
+	if (!fwspec || fwspec->ops != &arm_smmu_ops)
+		return;
+
+	master = fwspec->iommu_priv;
+	if (master && master->ste.valid)
+		arm_smmu_detach_dev(dev);
 	iommu_group_remove_device(dev);
+	kfree(master);
+	iommu_fwspec_free(dev);
+}
+
+static struct iommu_group *arm_smmu_device_group(struct device *dev)
+{
+	struct iommu_group *group;
+
+	/*
+	 * We don't support devices sharing stream IDs other than PCI RID
+	 * aliases, since the necessary ID-to-device lookup becomes rather
+	 * impractical given a potential sparse 32-bit stream ID space.
+	 */
+	if (dev_is_pci(dev))
+		group = pci_device_group(dev);
+	else
+		group = generic_device_group(dev);
+
+	return group;
 }
 
 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
@@ -1934,6 +1875,11 @@ out_unlock:
 	return ret;
 }
 
+static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
+{
+	return iommu_fwspec_add_ids(dev, args->args, 1);
+}
+
 static struct iommu_ops arm_smmu_ops = {
 	.capable		= arm_smmu_capable,
 	.domain_alloc		= arm_smmu_domain_alloc,
@@ -1945,9 +1891,10 @@ static struct iommu_ops arm_smmu_ops = {
 	.iova_to_phys		= arm_smmu_iova_to_phys,
 	.add_device		= arm_smmu_add_device,
 	.remove_device		= arm_smmu_remove_device,
-	.device_group		= pci_device_group,
+	.device_group		= arm_smmu_device_group,
 	.domain_get_attr	= arm_smmu_domain_get_attr,
 	.domain_set_attr	= arm_smmu_domain_set_attr,
+	.of_xlate		= arm_smmu_of_xlate,
 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
 };
 
@@ -2148,6 +2095,24 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
 					  1, ARM_SMMU_POLL_TIMEOUT_US);
 }
 
+/* GBPA is "special" */
+static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
+{
+	int ret;
+	u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
+
+	ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
+					 1, ARM_SMMU_POLL_TIMEOUT_US);
+	if (ret)
+		return ret;
+
+	reg &= ~clr;
+	reg |= set;
+	writel_relaxed(reg | GBPA_UPDATE, gbpa);
+	return readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
+					  1, ARM_SMMU_POLL_TIMEOUT_US);
+}
+
 static void arm_smmu_free_msis(void *data)
 {
 	struct device *dev = data;
@@ -2232,10 +2197,10 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
 	/* Request interrupt lines */
 	irq = smmu->evtq.q.irq;
 	if (irq) {
-		ret = devm_request_threaded_irq(smmu->dev, irq,
-						arm_smmu_evtq_handler,
+		ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
 						arm_smmu_evtq_thread,
-						0, "arm-smmu-v3-evtq", smmu);
+						IRQF_ONESHOT,
+						"arm-smmu-v3-evtq", smmu);
 		if (ret < 0)
 			dev_warn(smmu->dev, "failed to enable evtq irq\n");
 	}
@@ -2260,10 +2225,10 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
 	if (smmu->features & ARM_SMMU_FEAT_PRI) {
 		irq = smmu->priq.q.irq;
 		if (irq) {
-			ret = devm_request_threaded_irq(smmu->dev, irq,
-							arm_smmu_priq_handler,
+			ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
 							arm_smmu_priq_thread,
-							0, "arm-smmu-v3-priq",
+							IRQF_ONESHOT,
+							"arm-smmu-v3-priq",
 							smmu);
 			if (ret < 0)
 				dev_warn(smmu->dev,
@@ -2293,7 +2258,7 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
 	return ret;
 }
 
-static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
+static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
 {
 	int ret;
 	u32 reg, enables;
@@ -2394,8 +2359,17 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
 		return ret;
 	}
 
-	/* Enable the SMMU interface */
-	enables |= CR0_SMMUEN;
+
+	/* Enable the SMMU interface, or ensure bypass */
+	if (!bypass || disable_bypass) {
+		enables |= CR0_SMMUEN;
+	} else {
+		ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
+		if (ret) {
+			dev_err(smmu->dev, "GBPA not responding to update\n");
+			return ret;
+		}
+	}
 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
 				      ARM_SMMU_CR0ACK);
 	if (ret) {
@@ -2594,6 +2568,15 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	struct resource *res;
 	struct arm_smmu_device *smmu;
 	struct device *dev = &pdev->dev;
+	bool bypass = true;
+	u32 cells;
+
+	if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
+		dev_err(dev, "missing #iommu-cells property\n");
+	else if (cells != 1)
+		dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
+	else
+		bypass = false;
 
 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
 	if (!smmu) {
@@ -2646,7 +2629,24 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, smmu);
 
 	/* Reset the device */
-	return arm_smmu_device_reset(smmu);
+	ret = arm_smmu_device_reset(smmu, bypass);
+	if (ret)
+		return ret;
+
+	/* And we're up. Go go go! */
+	of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
+#ifdef CONFIG_PCI
+	pci_request_acs();
+	ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
+	if (ret)
+		return ret;
+#endif
+#ifdef CONFIG_ARM_AMBA
+	ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
+	if (ret)
+		return ret;
+#endif
+	return bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
 }
 
 static int arm_smmu_device_remove(struct platform_device *pdev)
@@ -2674,22 +2674,14 @@ static struct platform_driver arm_smmu_driver = {
 
 static int __init arm_smmu_init(void)
 {
-	struct device_node *np;
-	int ret;
-
-	np = of_find_matching_node(NULL, arm_smmu_of_match);
-	if (!np)
-		return 0;
-
-	of_node_put(np);
-
-	ret = platform_driver_register(&arm_smmu_driver);
-	if (ret)
-		return ret;
-
-	pci_request_acs();
+	static bool registered;
+	int ret = 0;
 
-	return bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
+	if (!registered) {
+		ret = platform_driver_register(&arm_smmu_driver);
+		registered = !ret;
+	}
+	return ret;
 }
 
 static void __exit arm_smmu_exit(void)
@@ -2700,6 +2692,20 @@ static void __exit arm_smmu_exit(void)
 subsys_initcall(arm_smmu_init);
 module_exit(arm_smmu_exit);
 
+static int __init arm_smmu_of_init(struct device_node *np)
+{
+	int ret = arm_smmu_init();
+
+	if (ret)
+		return ret;
+
+	if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
+		return -ENODEV;
+
+	return 0;
+}
+IOMMU_OF_DECLARE(arm_smmuv3, "arm,smmu-v3", arm_smmu_of_init);
+
 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 4f49fe29f202..c841eb7a1a74 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -28,6 +28,7 @@
 
 #define pr_fmt(fmt) "arm-smmu: " fmt
 
+#include <linux/atomic.h>
 #include <linux/delay.h>
 #include <linux/dma-iommu.h>
 #include <linux/dma-mapping.h>
@@ -40,6 +41,8 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_iommu.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
@@ -49,15 +52,9 @@
 
 #include "io-pgtable.h"
 
-/* Maximum number of stream IDs assigned to a single device */
-#define MAX_MASTER_STREAMIDS		128
-
 /* Maximum number of context banks per SMMU */
 #define ARM_SMMU_MAX_CBS		128
 
-/* Maximum number of mapping groups per SMMU */
-#define ARM_SMMU_MAX_SMRS		128
-
 /* SMMU global address space */
 #define ARM_SMMU_GR0(smmu)		((smmu)->base)
 #define ARM_SMMU_GR1(smmu)		((smmu)->base + (1 << (smmu)->pgshift))
@@ -165,21 +162,27 @@
 #define ARM_SMMU_GR0_SMR(n)		(0x800 + ((n) << 2))
 #define SMR_VALID			(1 << 31)
 #define SMR_MASK_SHIFT			16
-#define SMR_MASK_MASK			0x7fff
 #define SMR_ID_SHIFT			0
-#define SMR_ID_MASK			0x7fff
 
 #define ARM_SMMU_GR0_S2CR(n)		(0xc00 + ((n) << 2))
 #define S2CR_CBNDX_SHIFT		0
 #define S2CR_CBNDX_MASK			0xff
 #define S2CR_TYPE_SHIFT			16
 #define S2CR_TYPE_MASK			0x3
-#define S2CR_TYPE_TRANS			(0 << S2CR_TYPE_SHIFT)
-#define S2CR_TYPE_BYPASS		(1 << S2CR_TYPE_SHIFT)
-#define S2CR_TYPE_FAULT			(2 << S2CR_TYPE_SHIFT)
+enum arm_smmu_s2cr_type {
+	S2CR_TYPE_TRANS,
+	S2CR_TYPE_BYPASS,
+	S2CR_TYPE_FAULT,
+};
 
 #define S2CR_PRIVCFG_SHIFT		24
-#define S2CR_PRIVCFG_UNPRIV		(2 << S2CR_PRIVCFG_SHIFT)
+#define S2CR_PRIVCFG_MASK		0x3
+enum arm_smmu_s2cr_privcfg {
+	S2CR_PRIVCFG_DEFAULT,
+	S2CR_PRIVCFG_DIPAN,
+	S2CR_PRIVCFG_UNPRIV,
+	S2CR_PRIVCFG_PRIV,
+};
 
 /* Context bank attribute registers */
 #define ARM_SMMU_GR1_CBAR(n)		(0x0 + ((n) << 2))
@@ -217,6 +220,7 @@
 #define ARM_SMMU_CB_TTBR0		0x20
 #define ARM_SMMU_CB_TTBR1		0x28
 #define ARM_SMMU_CB_TTBCR		0x30
+#define ARM_SMMU_CB_CONTEXTIDR		0x34
 #define ARM_SMMU_CB_S1_MAIR0		0x38
 #define ARM_SMMU_CB_S1_MAIR1		0x3c
 #define ARM_SMMU_CB_PAR			0x50
@@ -239,7 +243,6 @@
 #define SCTLR_AFE			(1 << 2)
 #define SCTLR_TRE			(1 << 1)
 #define SCTLR_M				(1 << 0)
-#define SCTLR_EAE_SBOP			(SCTLR_AFE | SCTLR_TRE)
 
 #define ARM_MMU500_ACTLR_CPRE		(1 << 1)
 
@@ -296,23 +299,33 @@ enum arm_smmu_implementation {
 	CAVIUM_SMMUV2,
 };
 
+struct arm_smmu_s2cr {
+	struct iommu_group		*group;
+	int				count;
+	enum arm_smmu_s2cr_type		type;
+	enum arm_smmu_s2cr_privcfg	privcfg;
+	u8				cbndx;
+};
+
+#define s2cr_init_val (struct arm_smmu_s2cr){				\
+	.type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,	\
+}
+
 struct arm_smmu_smr {
-	u8				idx;
 	u16				mask;
 	u16				id;
+	bool				valid;
 };
 
 struct arm_smmu_master_cfg {
-	int				num_streamids;
-	u16				streamids[MAX_MASTER_STREAMIDS];
-	struct arm_smmu_smr		*smrs;
-};
-
-struct arm_smmu_master {
-	struct device_node		*of_node;
-	struct rb_node			node;
-	struct arm_smmu_master_cfg	cfg;
+	struct arm_smmu_device		*smmu;
+	s16				smendx[];
 };
+#define INVALID_SMENDX			-1
+#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
+#define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
+#define for_each_cfg_sme(fw, i, idx) \
+	for (i = 0; idx = __fwspec_cfg(fw)->smendx[i], i < fw->num_ids; ++i)
 
 struct arm_smmu_device {
 	struct device			*dev;
@@ -346,7 +359,11 @@ struct arm_smmu_device {
 	atomic_t			irptndx;
 
 	u32				num_mapping_groups;
-	DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
+	u16				streamid_mask;
+	u16				smr_mask_mask;
+	struct arm_smmu_smr		*smrs;
+	struct arm_smmu_s2cr		*s2crs;
+	struct mutex			stream_map_mutex;
 
 	unsigned long			va_size;
 	unsigned long			ipa_size;
@@ -357,9 +374,6 @@ struct arm_smmu_device {
 	u32				num_context_irqs;
 	unsigned int			*irqs;
 
-	struct list_head		list;
-	struct rb_root			masters;
-
 	u32				cavium_id_base; /* Specific to Cavium */
 };
 
@@ -397,15 +411,6 @@ struct arm_smmu_domain {
 	struct iommu_domain		domain;
 };
 
-struct arm_smmu_phandle_args {
-	struct device_node *np;
-	int args_count;
-	uint32_t args[MAX_MASTER_STREAMIDS];
-};
-
-static DEFINE_SPINLOCK(arm_smmu_devices_lock);
-static LIST_HEAD(arm_smmu_devices);
-
 struct arm_smmu_option_prop {
 	u32 opt;
 	const char *prop;
@@ -413,6 +418,8 @@ struct arm_smmu_option_prop {
 
 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 
+static bool using_legacy_binding, using_generic_binding;
+
 static struct arm_smmu_option_prop arm_smmu_options[] = {
 	{ ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 	{ 0, NULL},
@@ -444,131 +451,86 @@ static struct device_node *dev_get_dev_node(struct device *dev)
 
 		while (!pci_is_root_bus(bus))
 			bus = bus->parent;
-		return bus->bridge->parent->of_node;
+		return of_node_get(bus->bridge->parent->of_node);
 	}
 
-	return dev->of_node;
+	return of_node_get(dev->of_node);
 }
 
-static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
-						struct device_node *dev_node)
+static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 {
-	struct rb_node *node = smmu->masters.rb_node;
-
-	while (node) {
-		struct arm_smmu_master *master;
-
-		master = container_of(node, struct arm_smmu_master, node);
-
-		if (dev_node < master->of_node)
-			node = node->rb_left;
-		else if (dev_node > master->of_node)
-			node = node->rb_right;
-		else
-			return master;
-	}
-
-	return NULL;
+	*((__be32 *)data) = cpu_to_be32(alias);
+	return 0; /* Continue walking */
 }
 
-static struct arm_smmu_master_cfg *
-find_smmu_master_cfg(struct device *dev)
+static int __find_legacy_master_phandle(struct device *dev, void *data)
 {
-	struct arm_smmu_master_cfg *cfg = NULL;
-	struct iommu_group *group = iommu_group_get(dev);
-
-	if (group) {
-		cfg = iommu_group_get_iommudata(group);
-		iommu_group_put(group);
-	}
-
-	return cfg;
+	struct of_phandle_iterator *it = *(void **)data;
+	struct device_node *np = it->node;
+	int err;
+
+	of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
+			    "#stream-id-cells", 0)
+		if (it->node == np) {
+			*(void **)data = dev;
+			return 1;
+		}
+	it->node = np;
+	return err == -ENOENT ? 0 : err;
 }
 
-static int insert_smmu_master(struct arm_smmu_device *smmu,
-			      struct arm_smmu_master *master)
+static struct platform_driver arm_smmu_driver;
+static struct iommu_ops arm_smmu_ops;
+
+static int arm_smmu_register_legacy_master(struct device *dev,
+					   struct arm_smmu_device **smmu)
 {
-	struct rb_node **new, *parent;
-
-	new = &smmu->masters.rb_node;
-	parent = NULL;
-	while (*new) {
-		struct arm_smmu_master *this
-			= container_of(*new, struct arm_smmu_master, node);
-
-		parent = *new;
-		if (master->of_node < this->of_node)
-			new = &((*new)->rb_left);
-		else if (master->of_node > this->of_node)
-			new = &((*new)->rb_right);
-		else
-			return -EEXIST;
+	struct device *smmu_dev;
+	struct device_node *np;
+	struct of_phandle_iterator it;
+	void *data = &it;
+	u32 *sids;
+	__be32 pci_sid;
+	int err;
+
+	np = dev_get_dev_node(dev);
+	if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
+		of_node_put(np);
+		return -ENODEV;
 	}
 
-	rb_link_node(&master->node, parent, new);
-	rb_insert_color(&master->node, &smmu->masters);
-	return 0;
-}
-
-static int register_smmu_master(struct arm_smmu_device *smmu,
-				struct device *dev,
-				struct arm_smmu_phandle_args *masterspec)
-{
-	int i;
-	struct arm_smmu_master *master;
+	it.node = np;
+	err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
+				     __find_legacy_master_phandle);
+	smmu_dev = data;
+	of_node_put(np);
+	if (err == 0)
+		return -ENODEV;
+	if (err < 0)
+		return err;
 
-	master = find_smmu_master(smmu, masterspec->np);
-	if (master) {
-		dev_err(dev,
-			"rejecting multiple registrations for master device %s\n",
-			masterspec->np->name);
-		return -EBUSY;
+	if (dev_is_pci(dev)) {
+		/* "mmu-masters" assumes Stream ID == Requester ID */
+		pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
+				       &pci_sid);
+		it.cur = &pci_sid;
+		it.cur_count = 1;
 	}
 
-	if (masterspec->args_count > MAX_MASTER_STREAMIDS) {
-		dev_err(dev,
-			"reached maximum number (%d) of stream IDs for master device %s\n",
-			MAX_MASTER_STREAMIDS, masterspec->np->name);
-		return -ENOSPC;
-	}
+	err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
+				&arm_smmu_ops);
+	if (err)
+		return err;
 
-	master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
-	if (!master)
+	sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
+	if (!sids)
 		return -ENOMEM;
 
-	master->of_node			= masterspec->np;
-	master->cfg.num_streamids	= masterspec->args_count;
-
-	for (i = 0; i < master->cfg.num_streamids; ++i) {
-		u16 streamid = masterspec->args[i];
-
-		if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
-		     (streamid >= smmu->num_mapping_groups)) {
-			dev_err(dev,
-				"stream ID for master device %s greater than maximum allowed (%d)\n",
-				masterspec->np->name, smmu->num_mapping_groups);
-			return -ERANGE;
-		}
-		master->cfg.streamids[i] = streamid;
-	}
-	return insert_smmu_master(smmu, master);
-}
-
-static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
-{
-	struct arm_smmu_device *smmu;
-	struct arm_smmu_master *master = NULL;
-	struct device_node *dev_node = dev_get_dev_node(dev);
-
-	spin_lock(&arm_smmu_devices_lock);
-	list_for_each_entry(smmu, &arm_smmu_devices, list) {
-		master = find_smmu_master(smmu, dev_node);
-		if (master)
-			break;
-	}
-	spin_unlock(&arm_smmu_devices_lock);
-
-	return master ? smmu : NULL;
+	*smmu = dev_get_drvdata(smmu_dev);
+	of_phandle_iterator_args(&it, sids, it.cur_count);
+	err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
+	kfree(sids);
+	return err;
 }
 
 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
@@ -686,8 +648,7 @@ static struct iommu_gather_ops arm_smmu_gather_ops = {
 
 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 {
-	int flags, ret;
-	u32 fsr, fsynr, resume;
+	u32 fsr, fsynr;
 	unsigned long iova;
 	struct iommu_domain *domain = dev;
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
@@ -701,34 +662,15 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 	if (!(fsr & FSR_FAULT))
 		return IRQ_NONE;
 
-	if (fsr & FSR_IGN)
-		dev_err_ratelimited(smmu->dev,
-				    "Unexpected context fault (fsr 0x%x)\n",
-				    fsr);
-
 	fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
-	flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
-
 	iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
-	if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {
-		ret = IRQ_HANDLED;
-		resume = RESUME_RETRY;
-	} else {
-		dev_err_ratelimited(smmu->dev,
-		    "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n",
-		    iova, fsynr, cfg->cbndx);
-		ret = IRQ_NONE;
-		resume = RESUME_TERMINATE;
-	}
-
-	/* Clear the faulting FSR */
-	writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 
-	/* Retry or terminate any stalled transactions */
-	if (fsr & FSR_SS)
-		writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME);
+	dev_err_ratelimited(smmu->dev,
+	"Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
+			    fsr, iova, fsynr, cfg->cbndx);
 
-	return ret;
+	writel(fsr, cb_base + ARM_SMMU_CB_FSR);
+	return IRQ_HANDLED;
 }
 
 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
@@ -758,7 +700,7 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 				       struct io_pgtable_cfg *pgtbl_cfg)
 {
-	u32 reg;
+	u32 reg, reg2;
 	u64 reg64;
 	bool stage1;
 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
@@ -801,14 +743,22 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 
 	/* TTBRs */
 	if (stage1) {
-		reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
-
-		reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
-		writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
-
-		reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
-		reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT;
-		writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
+		u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
+
+		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+			reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
+			writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
+			reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
+			writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
+			writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
+		} else {
+			reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
+			reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
+			writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
+			reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
+			reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
+			writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
+		}
 	} else {
 		reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 		writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
@@ -816,28 +766,36 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 
 	/* TTBCR */
 	if (stage1) {
-		reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
-		writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
-		if (smmu->version > ARM_SMMU_V1) {
-			reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
-			reg |= TTBCR2_SEP_UPSTREAM;
-			writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
+		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+			reg = pgtbl_cfg->arm_v7s_cfg.tcr;
+			reg2 = 0;
+		} else {
+			reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
+			reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
+			reg2 |= TTBCR2_SEP_UPSTREAM;
 		}
+		if (smmu->version > ARM_SMMU_V1)
+			writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 	} else {
 		reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
-		writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 	}
+	writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 
 	/* MAIRs (stage-1 only) */
 	if (stage1) {
-		reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
+		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+			reg = pgtbl_cfg->arm_v7s_cfg.prrr;
+			reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
+		} else {
+			reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
+			reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
+		}
 		writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
-		reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
-		writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1);
+		writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 	}
 
 	/* SCTLR */
-	reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
+	reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 	if (stage1)
 		reg |= SCTLR_S1_ASIDPNE;
 #ifdef __BIG_ENDIAN
@@ -861,12 +819,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 	if (smmu_domain->smmu)
 		goto out_unlock;
 
-	/* We're bypassing these SIDs, so don't allocate an actual context */
-	if (domain->type == IOMMU_DOMAIN_DMA) {
-		smmu_domain->smmu = smmu;
-		goto out_unlock;
-	}
-
 	/*
 	 * Mapping the requested stage onto what we support is surprisingly
 	 * complicated, mainly because the spec allows S1+S2 SMMUs without
@@ -900,6 +852,11 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 	 */
 	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 		cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
+	if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
+	    !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
+	    (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
+	    (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
+		cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 	if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 	    (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 			       ARM_SMMU_FEAT_FMT_AARCH64_16K |
@@ -919,10 +876,14 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 		oas = smmu->ipa_size;
 		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 			fmt = ARM_64_LPAE_S1;
-		} else {
+		} else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 			fmt = ARM_32_LPAE_S1;
 			ias = min(ias, 32UL);
 			oas = min(oas, 40UL);
+		} else {
+			fmt = ARM_V7S;
+			ias = min(ias, 32UL);
+			oas = min(oas, 32UL);
 		}
 		break;
 	case ARM_SMMU_DOMAIN_NESTED:
@@ -978,6 +939,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 
 	/* Update the domain's page sizes to reflect the page table format */
 	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
+	domain->geometry.aperture_end = (1UL << ias) - 1;
+	domain->geometry.force_aperture = true;
 
 	/* Initialise the context bank with our page table cfg */
 	arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
@@ -1016,7 +979,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 	void __iomem *cb_base;
 	int irq;
 
-	if (!smmu || domain->type == IOMMU_DOMAIN_DMA)
+	if (!smmu)
 		return;
 
 	/*
@@ -1050,8 +1013,8 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 	if (!smmu_domain)
 		return NULL;
 
-	if (type == IOMMU_DOMAIN_DMA &&
-	    iommu_get_dma_cookie(&smmu_domain->domain)) {
+	if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
+	    iommu_get_dma_cookie(&smmu_domain->domain))) {
 		kfree(smmu_domain);
 		return NULL;
 	}
@@ -1075,162 +1038,197 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
 	kfree(smmu_domain);
 }
 
-static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
-					  struct arm_smmu_master_cfg *cfg)
+static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
 {
-	int i;
-	struct arm_smmu_smr *smrs;
-	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+	struct arm_smmu_smr *smr = smmu->smrs + idx;
+	u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
 
-	if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))
-		return 0;
+	if (smr->valid)
+		reg |= SMR_VALID;
+	writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
+}
 
-	if (cfg->smrs)
-		return -EEXIST;
+static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
+{
+	struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
+	u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
+		  (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
+		  (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
 
-	smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL);
-	if (!smrs) {
-		dev_err(smmu->dev, "failed to allocate %d SMRs\n",
-			cfg->num_streamids);
-		return -ENOMEM;
-	}
+	writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
+}
 
-	/* Allocate the SMRs on the SMMU */
-	for (i = 0; i < cfg->num_streamids; ++i) {
-		int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,
-						  smmu->num_mapping_groups);
-		if (idx < 0) {
-			dev_err(smmu->dev, "failed to allocate free SMR\n");
-			goto err_free_smrs;
-		}
+static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
+{
+	arm_smmu_write_s2cr(smmu, idx);
+	if (smmu->smrs)
+		arm_smmu_write_smr(smmu, idx);
+}
 
-		smrs[i] = (struct arm_smmu_smr) {
-			.idx	= idx,
-			.mask	= 0, /* We don't currently share SMRs */
-			.id	= cfg->streamids[i],
-		};
-	}
+static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
+{
+	struct arm_smmu_smr *smrs = smmu->smrs;
+	int i, free_idx = -ENOSPC;
 
-	/* It worked! Now, poke the actual hardware */
-	for (i = 0; i < cfg->num_streamids; ++i) {
-		u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT |
-			  smrs[i].mask << SMR_MASK_SHIFT;
-		writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx));
-	}
+	/* Stream indexing is blissfully easy */
+	if (!smrs)
+		return id;
 
-	cfg->smrs = smrs;
-	return 0;
+	/* Validating SMRs is... less so */
+	for (i = 0; i < smmu->num_mapping_groups; ++i) {
+		if (!smrs[i].valid) {
+			/*
+			 * Note the first free entry we come across, which
+			 * we'll claim in the end if nothing else matches.
+			 */
+			if (free_idx < 0)
+				free_idx = i;
+			continue;
+		}
+		/*
+		 * If the new entry is _entirely_ matched by an existing entry,
+		 * then reuse that, with the guarantee that there also cannot
+		 * be any subsequent conflicting entries. In normal use we'd
+		 * expect simply identical entries for this case, but there's
+		 * no harm in accommodating the generalisation.
+		 */
+		if ((mask & smrs[i].mask) == mask &&
+		    !((id ^ smrs[i].id) & ~smrs[i].mask))
+			return i;
+		/*
+		 * If the new entry has any other overlap with an existing one,
+		 * though, then there always exists at least one stream ID
+		 * which would cause a conflict, and we can't allow that risk.
+		 */
+		if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
+			return -EINVAL;
+	}
 
-err_free_smrs:
-	while (--i >= 0)
-		__arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);
-	kfree(smrs);
-	return -ENOSPC;
+	return free_idx;
 }
 
-static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
-				      struct arm_smmu_master_cfg *cfg)
+static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
 {
-	int i;
-	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
-	struct arm_smmu_smr *smrs = cfg->smrs;
-
-	if (!smrs)
-		return;
+	if (--smmu->s2crs[idx].count)
+		return false;
 
-	/* Invalidate the SMRs before freeing back to the allocator */
-	for (i = 0; i < cfg->num_streamids; ++i) {
-		u8 idx = smrs[i].idx;
+	smmu->s2crs[idx] = s2cr_init_val;
+	if (smmu->smrs)
+		smmu->smrs[idx].valid = false;
 
-		writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx));
-		__arm_smmu_free_bitmap(smmu->smr_map, idx);
-	}
-
-	cfg->smrs = NULL;
-	kfree(smrs);
+	return true;
 }
 
-static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
-				      struct arm_smmu_master_cfg *cfg)
+static int arm_smmu_master_alloc_smes(struct device *dev)
 {
-	int i, ret;
-	struct arm_smmu_device *smmu = smmu_domain->smmu;
-	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
+	struct arm_smmu_device *smmu = cfg->smmu;
+	struct arm_smmu_smr *smrs = smmu->smrs;
+	struct iommu_group *group;
+	int i, idx, ret;
 
-	/*
-	 * FIXME: This won't be needed once we have IOMMU-backed DMA ops
-	 * for all devices behind the SMMU. Note that we need to take
-	 * care configuring SMRs for devices both a platform_device and
-	 * and a PCI device (i.e. a PCI host controller)
-	 */
-	if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA)
-		return 0;
+	mutex_lock(&smmu->stream_map_mutex);
+	/* Figure out a viable stream map entry allocation */
+	for_each_cfg_sme(fwspec, i, idx) {
+		u16 sid = fwspec->ids[i];
+		u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
 
-	/* Devices in an IOMMU group may already be configured */
-	ret = arm_smmu_master_configure_smrs(smmu, cfg);
-	if (ret)
-		return ret == -EEXIST ? 0 : ret;
+		if (idx != INVALID_SMENDX) {
+			ret = -EEXIST;
+			goto out_err;
+		}
+
+		ret = arm_smmu_find_sme(smmu, sid, mask);
+		if (ret < 0)
+			goto out_err;
 
-	for (i = 0; i < cfg->num_streamids; ++i) {
-		u32 idx, s2cr;
+		idx = ret;
+		if (smrs && smmu->s2crs[idx].count == 0) {
+			smrs[idx].id = sid;
+			smrs[idx].mask = mask;
+			smrs[idx].valid = true;
+		}
+		smmu->s2crs[idx].count++;
+		cfg->smendx[i] = (s16)idx;
+	}
+
+	group = iommu_group_get_for_dev(dev);
+	if (!group)
+		group = ERR_PTR(-ENOMEM);
+	if (IS_ERR(group)) {
+		ret = PTR_ERR(group);
+		goto out_err;
+	}
+	iommu_group_put(group);
 
-		idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
-		s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV |
-		       (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT);
-		writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
+	/* It worked! Now, poke the actual hardware */
+	for_each_cfg_sme(fwspec, i, idx) {
+		arm_smmu_write_sme(smmu, idx);
+		smmu->s2crs[idx].group = group;
 	}
 
+	mutex_unlock(&smmu->stream_map_mutex);
 	return 0;
+
+out_err:
+	while (i--) {
+		arm_smmu_free_sme(smmu, cfg->smendx[i]);
+		cfg->smendx[i] = INVALID_SMENDX;
+	}
+	mutex_unlock(&smmu->stream_map_mutex);
+	return ret;
 }
 
-static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
-					  struct arm_smmu_master_cfg *cfg)
+static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
 {
-	int i;
-	struct arm_smmu_device *smmu = smmu_domain->smmu;
-	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
-
-	/* An IOMMU group is torn down by the first device to be removed */
-	if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
-		return;
+	struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
+	struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
+	int i, idx;
 
-	/*
-	 * We *must* clear the S2CR first, because freeing the SMR means
-	 * that it can be re-allocated immediately.
-	 */
-	for (i = 0; i < cfg->num_streamids; ++i) {
-		u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
-		u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
-
-		writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx));
+	mutex_lock(&smmu->stream_map_mutex);
+	for_each_cfg_sme(fwspec, i, idx) {
+		if (arm_smmu_free_sme(smmu, idx))
+			arm_smmu_write_sme(smmu, idx);
+		cfg->smendx[i] = INVALID_SMENDX;
 	}
-
-	arm_smmu_master_free_smrs(smmu, cfg);
+	mutex_unlock(&smmu->stream_map_mutex);
 }
 
-static void arm_smmu_detach_dev(struct device *dev,
-				struct arm_smmu_master_cfg *cfg)
+static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
+				      struct iommu_fwspec *fwspec)
 {
-	struct iommu_domain *domain = dev->archdata.iommu;
-	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+	struct arm_smmu_device *smmu = smmu_domain->smmu;
+	struct arm_smmu_s2cr *s2cr = smmu->s2crs;
+	enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
+	u8 cbndx = smmu_domain->cfg.cbndx;
+	int i, idx;
+
+	for_each_cfg_sme(fwspec, i, idx) {
+		if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
+			continue;
 
-	dev->archdata.iommu = NULL;
-	arm_smmu_domain_remove_master(smmu_domain, cfg);
+		s2cr[idx].type = type;
+		s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
+		s2cr[idx].cbndx = cbndx;
+		arm_smmu_write_s2cr(smmu, idx);
+	}
+	return 0;
 }
 
 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 {
 	int ret;
-	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
 	struct arm_smmu_device *smmu;
-	struct arm_smmu_master_cfg *cfg;
+	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 
-	smmu = find_smmu_for_device(dev);
-	if (!smmu) {
+	if (!fwspec || fwspec->ops != &arm_smmu_ops) {
 		dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
 		return -ENXIO;
 	}
 
+	smmu = fwspec_smmu(fwspec);
 	/* Ensure that the domain is finalised */
 	ret = arm_smmu_init_domain_context(domain, smmu);
 	if (ret < 0)
@@ -1248,18 +1246,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 	}
 
 	/* Looks ok, so add the device to the domain */
-	cfg = find_smmu_master_cfg(dev);
-	if (!cfg)
-		return -ENODEV;
-
-	/* Detach the dev from its current domain */
-	if (dev->archdata.iommu)
-		arm_smmu_detach_dev(dev, cfg);
-
-	ret = arm_smmu_domain_add_master(smmu_domain, cfg);
-	if (!ret)
-		dev->archdata.iommu = domain;
-	return ret;
+	return arm_smmu_domain_add_master(smmu_domain, fwspec);
 }
 
 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
@@ -1378,110 +1365,113 @@ static bool arm_smmu_capable(enum iommu_cap cap)
 	}
 }
 
-static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
+static int arm_smmu_match_node(struct device *dev, void *data)
 {
-	*((u16 *)data) = alias;
-	return 0; /* Continue walking */
+	return dev->of_node == data;
 }
 
-static void __arm_smmu_release_pci_iommudata(void *data)
+static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
 {
-	kfree(data);
+	struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
+						np, arm_smmu_match_node);
+	put_device(dev);
+	return dev ? dev_get_drvdata(dev) : NULL;
 }
 
-static int arm_smmu_init_pci_device(struct pci_dev *pdev,
-				    struct iommu_group *group)
+static int arm_smmu_add_device(struct device *dev)
 {
+	struct arm_smmu_device *smmu;
 	struct arm_smmu_master_cfg *cfg;
-	u16 sid;
-	int i;
-
-	cfg = iommu_group_get_iommudata(group);
-	if (!cfg) {
-		cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
-		if (!cfg)
-			return -ENOMEM;
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	int i, ret;
 
-		iommu_group_set_iommudata(group, cfg,
-					  __arm_smmu_release_pci_iommudata);
+	if (using_legacy_binding) {
+		ret = arm_smmu_register_legacy_master(dev, &smmu);
+		fwspec = dev->iommu_fwspec;
+		if (ret)
+			goto out_free;
+	} else if (fwspec) {
+		smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
+	} else {
+		return -ENODEV;
 	}
 
-	if (cfg->num_streamids >= MAX_MASTER_STREAMIDS)
-		return -ENOSPC;
-
-	/*
-	 * Assume Stream ID == Requester ID for now.
-	 * We need a way to describe the ID mappings in FDT.
-	 */
-	pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
-	for (i = 0; i < cfg->num_streamids; ++i)
-		if (cfg->streamids[i] == sid)
-			break;
-
-	/* Avoid duplicate SIDs, as this can lead to SMR conflicts */
-	if (i == cfg->num_streamids)
-		cfg->streamids[cfg->num_streamids++] = sid;
-
-	return 0;
-}
+	ret = -EINVAL;
+	for (i = 0; i < fwspec->num_ids; i++) {
+		u16 sid = fwspec->ids[i];
+		u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
 
-static int arm_smmu_init_platform_device(struct device *dev,
-					 struct iommu_group *group)
-{
-	struct arm_smmu_device *smmu = find_smmu_for_device(dev);
-	struct arm_smmu_master *master;
+		if (sid & ~smmu->streamid_mask) {
+			dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
+				sid, smmu->streamid_mask);
+			goto out_free;
+		}
+		if (mask & ~smmu->smr_mask_mask) {
+			dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
+				sid, smmu->smr_mask_mask);
+			goto out_free;
+		}
+	}
 
-	if (!smmu)
-		return -ENODEV;
+	ret = -ENOMEM;
+	cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
+		      GFP_KERNEL);
+	if (!cfg)
+		goto out_free;
 
-	master = find_smmu_master(smmu, dev->of_node);
-	if (!master)
-		return -ENODEV;
+	cfg->smmu = smmu;
+	fwspec->iommu_priv = cfg;
+	while (i--)
+		cfg->smendx[i] = INVALID_SMENDX;
 
-	iommu_group_set_iommudata(group, &master->cfg, NULL);
+	ret = arm_smmu_master_alloc_smes(dev);
+	if (ret)
+		goto out_free;
 
 	return 0;
-}
-
-static int arm_smmu_add_device(struct device *dev)
-{
-	struct iommu_group *group;
-
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return PTR_ERR(group);
 
-	iommu_group_put(group);
-	return 0;
+out_free:
+	if (fwspec)
+		kfree(fwspec->iommu_priv);
+	iommu_fwspec_free(dev);
+	return ret;
 }
 
 static void arm_smmu_remove_device(struct device *dev)
 {
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+
+	if (!fwspec || fwspec->ops != &arm_smmu_ops)
+		return;
+
+	arm_smmu_master_free_smes(fwspec);
 	iommu_group_remove_device(dev);
+	kfree(fwspec->iommu_priv);
+	iommu_fwspec_free(dev);
 }
 
 static struct iommu_group *arm_smmu_device_group(struct device *dev)
 {
-	struct iommu_group *group;
-	int ret;
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
+	struct iommu_group *group = NULL;
+	int i, idx;
 
-	if (dev_is_pci(dev))
-		group = pci_device_group(dev);
-	else
-		group = generic_device_group(dev);
+	for_each_cfg_sme(fwspec, i, idx) {
+		if (group && smmu->s2crs[idx].group &&
+		    group != smmu->s2crs[idx].group)
+			return ERR_PTR(-EINVAL);
 
-	if (IS_ERR(group))
+		group = smmu->s2crs[idx].group;
+	}
+
+	if (group)
 		return group;
 
 	if (dev_is_pci(dev))
-		ret = arm_smmu_init_pci_device(to_pci_dev(dev), group);
+		group = pci_device_group(dev);
 	else
-		ret = arm_smmu_init_platform_device(dev, group);
-
-	if (ret) {
-		iommu_group_put(group);
-		group = ERR_PTR(ret);
-	}
+		group = generic_device_group(dev);
 
 	return group;
 }
@@ -1530,6 +1520,19 @@ out_unlock:
 	return ret;
 }
 
+static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
+{
+	u32 fwid = 0;
+
+	if (args->args_count > 0)
+		fwid |= (u16)args->args[0];
+
+	if (args->args_count > 1)
+		fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
+
+	return iommu_fwspec_add_ids(dev, &fwid, 1);
+}
+
 static struct iommu_ops arm_smmu_ops = {
 	.capable		= arm_smmu_capable,
 	.domain_alloc		= arm_smmu_domain_alloc,
@@ -1544,6 +1547,7 @@ static struct iommu_ops arm_smmu_ops = {
 	.device_group		= arm_smmu_device_group,
 	.domain_get_attr	= arm_smmu_domain_get_attr,
 	.domain_set_attr	= arm_smmu_domain_set_attr,
+	.of_xlate		= arm_smmu_of_xlate,
 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
 };
 
@@ -1551,19 +1555,19 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
 {
 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 	void __iomem *cb_base;
-	int i = 0;
+	int i;
 	u32 reg, major;
 
 	/* clear global FSR */
 	reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
 	writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
 
-	/* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */
-	reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
-	for (i = 0; i < smmu->num_mapping_groups; ++i) {
-		writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
-		writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i));
-	}
+	/*
+	 * Reset stream mapping groups: Initial values mark all SMRn as
+	 * invalid and all S2CRn as bypass unless overridden.
+	 */
+	for (i = 0; i < smmu->num_mapping_groups; ++i)
+		arm_smmu_write_sme(smmu, i);
 
 	/*
 	 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
@@ -1652,6 +1656,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 	u32 id;
 	bool cttw_dt, cttw_reg;
+	int i;
 
 	dev_notice(smmu->dev, "probing hardware configuration...\n");
 	dev_notice(smmu->dev, "SMMUv%d with:\n",
@@ -1710,39 +1715,55 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
 		dev_notice(smmu->dev,
 			   "\t(IDR0.CTTW overridden by dma-coherent property)\n");
 
+	/* Max. number of entries we have for stream matching/indexing */
+	size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
+	smmu->streamid_mask = size - 1;
 	if (id & ID0_SMS) {
-		u32 smr, sid, mask;
+		u32 smr;
 
 		smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
-		smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
-					   ID0_NUMSMRG_MASK;
-		if (smmu->num_mapping_groups == 0) {
+		size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
+		if (size == 0) {
 			dev_err(smmu->dev,
 				"stream-matching supported, but no SMRs present!\n");
 			return -ENODEV;
 		}
 
-		smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
-		smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
+		/*
+		 * SMR.ID bits may not be preserved if the corresponding MASK
+		 * bits are set, so check each one separately. We can reject
+		 * masters later if they try to claim IDs outside these masks.
+		 */
+		smr = smmu->streamid_mask << SMR_ID_SHIFT;
+		writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
+		smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
+		smmu->streamid_mask = smr >> SMR_ID_SHIFT;
+
+		smr = smmu->streamid_mask << SMR_MASK_SHIFT;
 		writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
 		smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
+		smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
 
-		mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
-		sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
-		if ((mask & sid) != sid) {
-			dev_err(smmu->dev,
-				"SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
-				mask, sid);
-			return -ENODEV;
-		}
+		/* Zero-initialised to mark as invalid */
+		smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
+					  GFP_KERNEL);
+		if (!smmu->smrs)
+			return -ENOMEM;
 
 		dev_notice(smmu->dev,
-			   "\tstream matching with %u register groups, mask 0x%x",
-			   smmu->num_mapping_groups, mask);
-	} else {
-		smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
-					   ID0_NUMSIDB_MASK;
+			   "\tstream matching with %lu register groups, mask 0x%x",
+			   size, smmu->smr_mask_mask);
 	}
+	/* s2cr->type == 0 means translation, so initialise explicitly */
+	smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
+					 GFP_KERNEL);
+	if (!smmu->s2crs)
+		return -ENOMEM;
+	for (i = 0; i < size; i++)
+		smmu->s2crs[i] = s2cr_init_val;
+
+	smmu->num_mapping_groups = size;
+	mutex_init(&smmu->stream_map_mutex);
 
 	if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
 		smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
@@ -1875,15 +1896,24 @@ MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
 
 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *of_id;
 	const struct arm_smmu_match_data *data;
 	struct resource *res;
 	struct arm_smmu_device *smmu;
 	struct device *dev = &pdev->dev;
-	struct rb_node *node;
-	struct of_phandle_iterator it;
-	struct arm_smmu_phandle_args *masterspec;
 	int num_irqs, i, err;
+	bool legacy_binding;
+
+	legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
+	if (legacy_binding && !using_generic_binding) {
+		if (!using_legacy_binding)
+			pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
+		using_legacy_binding = true;
+	} else if (!legacy_binding && !using_legacy_binding) {
+		using_generic_binding = true;
+	} else {
+		dev_err(dev, "not probing due to mismatched DT properties\n");
+		return -ENODEV;
+	}
 
 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
 	if (!smmu) {
@@ -1892,8 +1922,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	}
 	smmu->dev = dev;
 
-	of_id = of_match_node(arm_smmu_of_match, dev->of_node);
-	data = of_id->data;
+	data = of_device_get_match_data(dev);
 	smmu->version = data->version;
 	smmu->model = data->model;
 
@@ -1943,37 +1972,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	i = 0;
-	smmu->masters = RB_ROOT;
-
-	err = -ENOMEM;
-	/* No need to zero the memory for masterspec */
-	masterspec = kmalloc(sizeof(*masterspec), GFP_KERNEL);
-	if (!masterspec)
-		goto out_put_masters;
-
-	of_for_each_phandle(&it, err, dev->of_node,
-			    "mmu-masters", "#stream-id-cells", 0) {
-		int count = of_phandle_iterator_args(&it, masterspec->args,
-						     MAX_MASTER_STREAMIDS);
-		masterspec->np		= of_node_get(it.node);
-		masterspec->args_count	= count;
-
-		err = register_smmu_master(smmu, dev, masterspec);
-		if (err) {
-			dev_err(dev, "failed to add master %s\n",
-				masterspec->np->name);
-			kfree(masterspec);
-			goto out_put_masters;
-		}
-
-		i++;
-	}
-
-	dev_notice(dev, "registered %d master devices\n", i);
-
-	kfree(masterspec);
-
 	parse_driver_options(smmu);
 
 	if (smmu->version == ARM_SMMU_V2 &&
@@ -1981,8 +1979,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 		dev_err(dev,
 			"found only %d context interrupt(s) but %d required\n",
 			smmu->num_context_irqs, smmu->num_context_banks);
-		err = -ENODEV;
-		goto out_put_masters;
+		return -ENODEV;
 	}
 
 	for (i = 0; i < smmu->num_global_irqs; ++i) {
@@ -1994,59 +1991,39 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 		if (err) {
 			dev_err(dev, "failed to request global IRQ %d (%u)\n",
 				i, smmu->irqs[i]);
-			goto out_put_masters;
+			return err;
 		}
 	}
 
-	INIT_LIST_HEAD(&smmu->list);
-	spin_lock(&arm_smmu_devices_lock);
-	list_add(&smmu->list, &arm_smmu_devices);
-	spin_unlock(&arm_smmu_devices_lock);
-
+	of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
+	platform_set_drvdata(pdev, smmu);
 	arm_smmu_device_reset(smmu);
-	return 0;
 
-out_put_masters:
-	for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
-		struct arm_smmu_master *master
-			= container_of(node, struct arm_smmu_master, node);
-		of_node_put(master->of_node);
+	/* Oh, for a proper bus abstraction */
+	if (!iommu_present(&platform_bus_type))
+		bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
+#ifdef CONFIG_ARM_AMBA
+	if (!iommu_present(&amba_bustype))
+		bus_set_iommu(&amba_bustype, &arm_smmu_ops);
+#endif
+#ifdef CONFIG_PCI
+	if (!iommu_present(&pci_bus_type)) {
+		pci_request_acs();
+		bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
 	}
-
-	return err;
+#endif
+	return 0;
 }
 
 static int arm_smmu_device_remove(struct platform_device *pdev)
 {
-	int i;
-	struct device *dev = &pdev->dev;
-	struct arm_smmu_device *curr, *smmu = NULL;
-	struct rb_node *node;
-
-	spin_lock(&arm_smmu_devices_lock);
-	list_for_each_entry(curr, &arm_smmu_devices, list) {
-		if (curr->dev == dev) {
-			smmu = curr;
-			list_del(&smmu->list);
-			break;
-		}
-	}
-	spin_unlock(&arm_smmu_devices_lock);
+	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
 
 	if (!smmu)
 		return -ENODEV;
 
-	for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
-		struct arm_smmu_master *master
-			= container_of(node, struct arm_smmu_master, node);
-		of_node_put(master->of_node);
-	}
-
 	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
-		dev_err(dev, "removing device with active domains!\n");
-
-	for (i = 0; i < smmu->num_global_irqs; ++i)
-		devm_free_irq(smmu->dev, smmu->irqs[i], smmu);
+		dev_err(&pdev->dev, "removing device with active domains!\n");
 
 	/* Turn the thing off */
 	writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
@@ -2064,41 +2041,14 @@ static struct platform_driver arm_smmu_driver = {
 
 static int __init arm_smmu_init(void)
 {
-	struct device_node *np;
-	int ret;
-
-	/*
-	 * Play nice with systems that don't have an ARM SMMU by checking that
-	 * an ARM SMMU exists in the system before proceeding with the driver
-	 * and IOMMU bus operation registration.
-	 */
-	np = of_find_matching_node(NULL, arm_smmu_of_match);
-	if (!np)
-		return 0;
-
-	of_node_put(np);
-
-	ret = platform_driver_register(&arm_smmu_driver);
-	if (ret)
-		return ret;
-
-	/* Oh, for a proper bus abstraction */
-	if (!iommu_present(&platform_bus_type))
-		bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
-
-#ifdef CONFIG_ARM_AMBA
-	if (!iommu_present(&amba_bustype))
-		bus_set_iommu(&amba_bustype, &arm_smmu_ops);
-#endif
+	static bool registered;
+	int ret = 0;
 
-#ifdef CONFIG_PCI
-	if (!iommu_present(&pci_bus_type)) {
-		pci_request_acs();
-		bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
+	if (!registered) {
+		ret = platform_driver_register(&arm_smmu_driver);
+		registered = !ret;
 	}
-#endif
-
-	return 0;
+	return ret;
 }
 
 static void __exit arm_smmu_exit(void)
@@ -2109,6 +2059,25 @@ static void __exit arm_smmu_exit(void)
 subsys_initcall(arm_smmu_init);
 module_exit(arm_smmu_exit);
 
+static int __init arm_smmu_of_init(struct device_node *np)
+{
+	int ret = arm_smmu_init();
+
+	if (ret)
+		return ret;
+
+	if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
+		return -ENODEV;
+
+	return 0;
+}
+IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
+IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
+IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
+IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
+IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
+IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
+
 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 08a1e2f3690f..c5ab8667e6f2 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -25,10 +25,29 @@
 #include <linux/huge_mm.h>
 #include <linux/iommu.h>
 #include <linux/iova.h>
+#include <linux/irq.h>
 #include <linux/mm.h>
+#include <linux/pci.h>
 #include <linux/scatterlist.h>
 #include <linux/vmalloc.h>
 
+struct iommu_dma_msi_page {
+	struct list_head	list;
+	dma_addr_t		iova;
+	phys_addr_t		phys;
+};
+
+struct iommu_dma_cookie {
+	struct iova_domain	iovad;
+	struct list_head	msi_page_list;
+	spinlock_t		msi_lock;
+};
+
+static inline struct iova_domain *cookie_iovad(struct iommu_domain *domain)
+{
+	return &((struct iommu_dma_cookie *)domain->iova_cookie)->iovad;
+}
+
 int iommu_dma_init(void)
 {
 	return iova_cache_get();
@@ -43,15 +62,19 @@ int iommu_dma_init(void)
  */
 int iommu_get_dma_cookie(struct iommu_domain *domain)
 {
-	struct iova_domain *iovad;
+	struct iommu_dma_cookie *cookie;
 
 	if (domain->iova_cookie)
 		return -EEXIST;
 
-	iovad = kzalloc(sizeof(*iovad), GFP_KERNEL);
-	domain->iova_cookie = iovad;
+	cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
+	if (!cookie)
+		return -ENOMEM;
 
-	return iovad ? 0 : -ENOMEM;
+	spin_lock_init(&cookie->msi_lock);
+	INIT_LIST_HEAD(&cookie->msi_page_list);
+	domain->iova_cookie = cookie;
+	return 0;
 }
 EXPORT_SYMBOL(iommu_get_dma_cookie);
 
@@ -63,31 +86,58 @@ EXPORT_SYMBOL(iommu_get_dma_cookie);
  */
 void iommu_put_dma_cookie(struct iommu_domain *domain)
 {
-	struct iova_domain *iovad = domain->iova_cookie;
+	struct iommu_dma_cookie *cookie = domain->iova_cookie;
+	struct iommu_dma_msi_page *msi, *tmp;
 
-	if (!iovad)
+	if (!cookie)
 		return;
 
-	put_iova_domain(iovad);
-	kfree(iovad);
+	if (cookie->iovad.granule)
+		put_iova_domain(&cookie->iovad);
+
+	list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
+		list_del(&msi->list);
+		kfree(msi);
+	}
+	kfree(cookie);
 	domain->iova_cookie = NULL;
 }
 EXPORT_SYMBOL(iommu_put_dma_cookie);
 
+static void iova_reserve_pci_windows(struct pci_dev *dev,
+		struct iova_domain *iovad)
+{
+	struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus);
+	struct resource_entry *window;
+	unsigned long lo, hi;
+
+	resource_list_for_each_entry(window, &bridge->windows) {
+		if (resource_type(window->res) != IORESOURCE_MEM &&
+		    resource_type(window->res) != IORESOURCE_IO)
+			continue;
+
+		lo = iova_pfn(iovad, window->res->start - window->offset);
+		hi = iova_pfn(iovad, window->res->end - window->offset);
+		reserve_iova(iovad, lo, hi);
+	}
+}
+
 /**
  * iommu_dma_init_domain - Initialise a DMA mapping domain
  * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
  * @base: IOVA at which the mappable address space starts
  * @size: Size of IOVA space
+ * @dev: Device the domain is being initialised for
  *
  * @base and @size should be exact multiples of IOMMU page granularity to
  * avoid rounding surprises. If necessary, we reserve the page at address 0
  * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but
  * any change which could make prior IOVAs invalid will fail.
  */
-int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, u64 size)
+int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
+		u64 size, struct device *dev)
 {
-	struct iova_domain *iovad = domain->iova_cookie;
+	struct iova_domain *iovad = cookie_iovad(domain);
 	unsigned long order, base_pfn, end_pfn;
 
 	if (!iovad)
@@ -123,6 +173,8 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, u64 size
 		iovad->dma_32bit_pfn = end_pfn;
 	} else {
 		init_iova_domain(iovad, 1UL << order, base_pfn, end_pfn);
+		if (dev && dev_is_pci(dev))
+			iova_reserve_pci_windows(to_pci_dev(dev), iovad);
 	}
 	return 0;
 }
@@ -151,12 +203,15 @@ int dma_direction_to_prot(enum dma_data_direction dir, bool coherent)
 	}
 }
 
-static struct iova *__alloc_iova(struct iova_domain *iovad, size_t size,
+static struct iova *__alloc_iova(struct iommu_domain *domain, size_t size,
 		dma_addr_t dma_limit)
 {
+	struct iova_domain *iovad = cookie_iovad(domain);
 	unsigned long shift = iova_shift(iovad);
 	unsigned long length = iova_align(iovad, size) >> shift;
 
+	if (domain->geometry.force_aperture)
+		dma_limit = min(dma_limit, domain->geometry.aperture_end);
 	/*
 	 * Enforce size-alignment to be safe - there could perhaps be an
 	 * attribute to control this per-device, or at least per-domain...
@@ -167,7 +222,7 @@ static struct iova *__alloc_iova(struct iova_domain *iovad, size_t size,
 /* The IOVA allocator knows what we mapped, so just unmap whatever that was */
 static void __iommu_dma_unmap(struct iommu_domain *domain, dma_addr_t dma_addr)
 {
-	struct iova_domain *iovad = domain->iova_cookie;
+	struct iova_domain *iovad = cookie_iovad(domain);
 	unsigned long shift = iova_shift(iovad);
 	unsigned long pfn = dma_addr >> shift;
 	struct iova *iova = find_iova(iovad, pfn);
@@ -290,7 +345,7 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp,
 		void (*flush_page)(struct device *, const void *, phys_addr_t))
 {
 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
-	struct iova_domain *iovad = domain->iova_cookie;
+	struct iova_domain *iovad = cookie_iovad(domain);
 	struct iova *iova;
 	struct page **pages;
 	struct sg_table sgt;
@@ -314,7 +369,7 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp,
 	if (!pages)
 		return NULL;
 
-	iova = __alloc_iova(iovad, size, dev->coherent_dma_mask);
+	iova = __alloc_iova(domain, size, dev->coherent_dma_mask);
 	if (!iova)
 		goto out_free_pages;
 
@@ -382,11 +437,11 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
 {
 	dma_addr_t dma_addr;
 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
-	struct iova_domain *iovad = domain->iova_cookie;
+	struct iova_domain *iovad = cookie_iovad(domain);
 	phys_addr_t phys = page_to_phys(page) + offset;
 	size_t iova_off = iova_offset(iovad, phys);
 	size_t len = iova_align(iovad, size + iova_off);
-	struct iova *iova = __alloc_iova(iovad, len, dma_get_mask(dev));
+	struct iova *iova = __alloc_iova(domain, len, dma_get_mask(dev));
 
 	if (!iova)
 		return DMA_ERROR_CODE;
@@ -491,7 +546,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
 		int nents, int prot)
 {
 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
-	struct iova_domain *iovad = domain->iova_cookie;
+	struct iova_domain *iovad = cookie_iovad(domain);
 	struct iova *iova;
 	struct scatterlist *s, *prev = NULL;
 	dma_addr_t dma_addr;
@@ -538,7 +593,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
 		prev = s;
 	}
 
-	iova = __alloc_iova(iovad, iova_len, dma_get_mask(dev));
+	iova = __alloc_iova(domain, iova_len, dma_get_mask(dev));
 	if (!iova)
 		goto out_restore_sg;
 
@@ -583,3 +638,81 @@ int iommu_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 {
 	return dma_addr == DMA_ERROR_CODE;
 }
+
+static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
+		phys_addr_t msi_addr, struct iommu_domain *domain)
+{
+	struct iommu_dma_cookie *cookie = domain->iova_cookie;
+	struct iommu_dma_msi_page *msi_page;
+	struct iova_domain *iovad = &cookie->iovad;
+	struct iova *iova;
+	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
+
+	msi_addr &= ~(phys_addr_t)iova_mask(iovad);
+	list_for_each_entry(msi_page, &cookie->msi_page_list, list)
+		if (msi_page->phys == msi_addr)
+			return msi_page;
+
+	msi_page = kzalloc(sizeof(*msi_page), GFP_ATOMIC);
+	if (!msi_page)
+		return NULL;
+
+	iova = __alloc_iova(domain, iovad->granule, dma_get_mask(dev));
+	if (!iova)
+		goto out_free_page;
+
+	msi_page->phys = msi_addr;
+	msi_page->iova = iova_dma_addr(iovad, iova);
+	if (iommu_map(domain, msi_page->iova, msi_addr, iovad->granule, prot))
+		goto out_free_iova;
+
+	INIT_LIST_HEAD(&msi_page->list);
+	list_add(&msi_page->list, &cookie->msi_page_list);
+	return msi_page;
+
+out_free_iova:
+	__free_iova(iovad, iova);
+out_free_page:
+	kfree(msi_page);
+	return NULL;
+}
+
+void iommu_dma_map_msi_msg(int irq, struct msi_msg *msg)
+{
+	struct device *dev = msi_desc_to_dev(irq_get_msi_desc(irq));
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iommu_dma_cookie *cookie;
+	struct iommu_dma_msi_page *msi_page;
+	phys_addr_t msi_addr = (u64)msg->address_hi << 32 | msg->address_lo;
+	unsigned long flags;
+
+	if (!domain || !domain->iova_cookie)
+		return;
+
+	cookie = domain->iova_cookie;
+
+	/*
+	 * We disable IRQs to rule out a possible inversion against
+	 * irq_desc_lock if, say, someone tries to retarget the affinity
+	 * of an MSI from within an IPI handler.
+	 */
+	spin_lock_irqsave(&cookie->msi_lock, flags);
+	msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain);
+	spin_unlock_irqrestore(&cookie->msi_lock, flags);
+
+	if (WARN_ON(!msi_page)) {
+		/*
+		 * We're called from a void callback, so the best we can do is
+		 * 'fail' by filling the message with obviously bogus values.
+		 * Since we got this far due to an IOMMU being present, it's
+		 * not like the existing address would have worked anyway...
+		 */
+		msg->address_hi = ~0U;
+		msg->address_lo = ~0U;
+		msg->data = ~0U;
+	} else {
+		msg->address_hi = upper_32_bits(msi_page->iova);
+		msg->address_lo &= iova_mask(&cookie->iovad);
+		msg->address_lo += lower_32_bits(msi_page->iova);
+	}
+}
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 33dcc29ec200..30808e91b775 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -1345,8 +1345,8 @@ static int __init exynos_iommu_of_setup(struct device_node *np)
 		exynos_iommu_init();
 
 	pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
-	if (IS_ERR(pdev))
-		return PTR_ERR(pdev);
+	if (!pdev)
+		return -ENODEV;
 
 	/*
 	 * use the first registered sysmmu device for performing
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index ebb5bf3ddbd9..a4407eabf0e6 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -2452,20 +2452,15 @@ static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
 	return 0;
 }
 
-/* domain is initialized */
-static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
+static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
 {
 	struct device_domain_info *info = NULL;
-	struct dmar_domain *domain, *tmp;
+	struct dmar_domain *domain = NULL;
 	struct intel_iommu *iommu;
 	u16 req_id, dma_alias;
 	unsigned long flags;
 	u8 bus, devfn;
 
-	domain = find_domain(dev);
-	if (domain)
-		return domain;
-
 	iommu = device_to_iommu(dev, &bus, &devfn);
 	if (!iommu)
 		return NULL;
@@ -2487,9 +2482,9 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
 		}
 		spin_unlock_irqrestore(&device_domain_lock, flags);
 
-		/* DMA alias already has a domain, uses it */
+		/* DMA alias already has a domain, use it */
 		if (info)
-			goto found_domain;
+			goto out;
 	}
 
 	/* Allocate and initialize new domain for the device */
@@ -2501,28 +2496,67 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
 		return NULL;
 	}
 
-	/* register PCI DMA alias device */
-	if (dev_is_pci(dev) && req_id != dma_alias) {
-		tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
-					       dma_alias & 0xff, NULL, domain);
+out:
 
-		if (!tmp || tmp != domain) {
-			domain_exit(domain);
-			domain = tmp;
-		}
+	return domain;
+}
 
-		if (!domain)
-			return NULL;
+static struct dmar_domain *set_domain_for_dev(struct device *dev,
+					      struct dmar_domain *domain)
+{
+	struct intel_iommu *iommu;
+	struct dmar_domain *tmp;
+	u16 req_id, dma_alias;
+	u8 bus, devfn;
+
+	iommu = device_to_iommu(dev, &bus, &devfn);
+	if (!iommu)
+		return NULL;
+
+	req_id = ((u16)bus << 8) | devfn;
+
+	if (dev_is_pci(dev)) {
+		struct pci_dev *pdev = to_pci_dev(dev);
+
+		pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
+
+		/* register PCI DMA alias device */
+		if (req_id != dma_alias) {
+			tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
+					dma_alias & 0xff, NULL, domain);
+
+			if (!tmp || tmp != domain)
+				return tmp;
+		}
 	}
 
-found_domain:
 	tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
+	if (!tmp || tmp != domain)
+		return tmp;
+
+	return domain;
+}
 
-	if (!tmp || tmp != domain) {
+static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
+{
+	struct dmar_domain *domain, *tmp;
+
+	domain = find_domain(dev);
+	if (domain)
+		goto out;
+
+	domain = find_or_alloc_domain(dev, gaw);
+	if (!domain)
+		goto out;
+
+	tmp = set_domain_for_dev(dev, domain);
+	if (!tmp || domain != tmp) {
 		domain_exit(domain);
 		domain = tmp;
 	}
 
+out:
+
 	return domain;
 }
 
@@ -3394,17 +3428,18 @@ static unsigned long intel_alloc_iova(struct device *dev,
 
 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
 {
+	struct dmar_domain *domain, *tmp;
 	struct dmar_rmrr_unit *rmrr;
-	struct dmar_domain *domain;
 	struct device *i_dev;
 	int i, ret;
 
-	domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
-	if (!domain) {
-		pr_err("Allocating domain for %s failed\n",
-		       dev_name(dev));
-		return NULL;
-	}
+	domain = find_domain(dev);
+	if (domain)
+		goto out;
+
+	domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
+	if (!domain)
+		goto out;
 
 	/* We have a new domain - setup possible RMRRs for the device */
 	rcu_read_lock();
@@ -3423,6 +3458,18 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
 	}
 	rcu_read_unlock();
 
+	tmp = set_domain_for_dev(dev, domain);
+	if (!tmp || domain != tmp) {
+		domain_exit(domain);
+		domain = tmp;
+	}
+
+out:
+
+	if (!domain)
+		pr_err("Allocating domain for %s failed\n", dev_name(dev));
+
+
 	return domain;
 }
 
diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c
index 8c6139986d7d..f50e51c1a9c8 100644
--- a/drivers/iommu/io-pgtable-arm-v7s.c
+++ b/drivers/iommu/io-pgtable-arm-v7s.c
@@ -286,12 +286,14 @@ static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl)
 	int prot = IOMMU_READ;
 	arm_v7s_iopte attr = pte >> ARM_V7S_ATTR_SHIFT(lvl);
 
-	if (attr & ARM_V7S_PTE_AP_RDONLY)
+	if (!(attr & ARM_V7S_PTE_AP_RDONLY))
 		prot |= IOMMU_WRITE;
 	if ((attr & (ARM_V7S_TEX_MASK << ARM_V7S_TEX_SHIFT)) == 0)
 		prot |= IOMMU_MMIO;
 	else if (pte & ARM_V7S_ATTR_C)
 		prot |= IOMMU_CACHE;
+	if (pte & ARM_V7S_ATTR_XN(lvl))
+		prot |= IOMMU_NOEXEC;
 
 	return prot;
 }
@@ -631,6 +633,10 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,
 {
 	struct arm_v7s_io_pgtable *data;
 
+#ifdef PHYS_OFFSET
+	if (upper_32_bits(PHYS_OFFSET))
+		return NULL;
+#endif
 	if (cfg->ias > ARM_V7S_ADDR_BITS || cfg->oas > ARM_V7S_ADDR_BITS)
 		return NULL;
 
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index b06d93594436..9a2f1960873b 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -31,6 +31,7 @@
 #include <linux/err.h>
 #include <linux/pci.h>
 #include <linux/bitops.h>
+#include <linux/property.h>
 #include <trace/events/iommu.h>
 
 static struct kset *iommu_group_kset;
@@ -1613,3 +1614,60 @@ out:
 
 	return ret;
 }
+
+int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
+		      const struct iommu_ops *ops)
+{
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+
+	if (fwspec)
+		return ops == fwspec->ops ? 0 : -EINVAL;
+
+	fwspec = kzalloc(sizeof(*fwspec), GFP_KERNEL);
+	if (!fwspec)
+		return -ENOMEM;
+
+	of_node_get(to_of_node(iommu_fwnode));
+	fwspec->iommu_fwnode = iommu_fwnode;
+	fwspec->ops = ops;
+	dev->iommu_fwspec = fwspec;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(iommu_fwspec_init);
+
+void iommu_fwspec_free(struct device *dev)
+{
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+
+	if (fwspec) {
+		fwnode_handle_put(fwspec->iommu_fwnode);
+		kfree(fwspec);
+		dev->iommu_fwspec = NULL;
+	}
+}
+EXPORT_SYMBOL_GPL(iommu_fwspec_free);
+
+int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids)
+{
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	size_t size;
+	int i;
+
+	if (!fwspec)
+		return -EINVAL;
+
+	size = offsetof(struct iommu_fwspec, ids[fwspec->num_ids + num_ids]);
+	if (size > sizeof(*fwspec)) {
+		fwspec = krealloc(dev->iommu_fwspec, size, GFP_KERNEL);
+		if (!fwspec)
+			return -ENOMEM;
+	}
+
+	for (i = 0; i < num_ids; i++)
+		fwspec->ids[fwspec->num_ids + i] = ids[i];
+
+	fwspec->num_ids += num_ids;
+	dev->iommu_fwspec = fwspec;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids);
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 2fdbac67a77f..ace331da6459 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -636,7 +636,7 @@ static int ipmmu_add_device(struct device *dev)
 	spin_unlock(&ipmmu_devices_lock);
 
 	if (ret < 0)
-		return -ENODEV;
+		goto error;
 
 	for (i = 0; i < num_utlbs; ++i) {
 		if (utlbs[i] >= mmu->num_utlbs) {
diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h
index 9ed0a8462ccf..3dab13b4a211 100644
--- a/drivers/iommu/mtk_iommu.h
+++ b/drivers/iommu/mtk_iommu.h
@@ -55,19 +55,19 @@ struct mtk_iommu_data {
 	bool                            enable_4GB;
 };
 
-static int compare_of(struct device *dev, void *data)
+static inline int compare_of(struct device *dev, void *data)
 {
 	return dev->of_node == data;
 }
 
-static int mtk_iommu_bind(struct device *dev)
+static inline int mtk_iommu_bind(struct device *dev)
 {
 	struct mtk_iommu_data *data = dev_get_drvdata(dev);
 
 	return component_bind_all(dev, &data->smi_imu);
 }
 
-static void mtk_iommu_unbind(struct device *dev)
+static inline void mtk_iommu_unbind(struct device *dev)
 {
 	struct mtk_iommu_data *data = dev_get_drvdata(dev);
 
diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 57f23eaaa2f9..5b82862f571f 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -22,6 +22,7 @@
 #include <linux/limits.h>
 #include <linux/of.h>
 #include <linux/of_iommu.h>
+#include <linux/of_pci.h>
 #include <linux/slab.h>
 
 static const struct of_device_id __iommu_of_table_sentinel
@@ -134,6 +135,47 @@ const struct iommu_ops *of_iommu_get_ops(struct device_node *np)
 	return ops;
 }
 
+static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)
+{
+	struct of_phandle_args *iommu_spec = data;
+
+	iommu_spec->args[0] = alias;
+	return iommu_spec->np == pdev->bus->dev.of_node;
+}
+
+static const struct iommu_ops
+*of_pci_iommu_configure(struct pci_dev *pdev, struct device_node *bridge_np)
+{
+	const struct iommu_ops *ops;
+	struct of_phandle_args iommu_spec;
+
+	/*
+	 * Start by tracing the RID alias down the PCI topology as
+	 * far as the host bridge whose OF node we have...
+	 * (we're not even attempting to handle multi-alias devices yet)
+	 */
+	iommu_spec.args_count = 1;
+	iommu_spec.np = bridge_np;
+	pci_for_each_dma_alias(pdev, __get_pci_rid, &iommu_spec);
+	/*
+	 * ...then find out what that becomes once it escapes the PCI
+	 * bus into the system beyond, and which IOMMU it ends up at.
+	 */
+	iommu_spec.np = NULL;
+	if (of_pci_map_rid(bridge_np, iommu_spec.args[0], "iommu-map",
+			   "iommu-map-mask", &iommu_spec.np, iommu_spec.args))
+		return NULL;
+
+	ops = of_iommu_get_ops(iommu_spec.np);
+	if (!ops || !ops->of_xlate ||
+	    iommu_fwspec_init(&pdev->dev, &iommu_spec.np->fwnode, ops) ||
+	    ops->of_xlate(&pdev->dev, &iommu_spec))
+		ops = NULL;
+
+	of_node_put(iommu_spec.np);
+	return ops;
+}
+
 const struct iommu_ops *of_iommu_configure(struct device *dev,
 					   struct device_node *master_np)
 {
@@ -142,12 +184,8 @@ const struct iommu_ops *of_iommu_configure(struct device *dev,
 	const struct iommu_ops *ops = NULL;
 	int idx = 0;
 
-	/*
-	 * We can't do much for PCI devices without knowing how
-	 * device IDs are wired up from the PCI bus to the IOMMU.
-	 */
 	if (dev_is_pci(dev))
-		return NULL;
+		return of_pci_iommu_configure(to_pci_dev(dev), master_np);
 
 	/*
 	 * We don't currently walk up the tree looking for a parent IOMMU.
@@ -160,7 +198,9 @@ const struct iommu_ops *of_iommu_configure(struct device *dev,
 		np = iommu_spec.np;
 		ops = of_iommu_get_ops(np);
 
-		if (!ops || !ops->of_xlate || ops->of_xlate(dev, &iommu_spec))
+		if (!ops || !ops->of_xlate ||
+		    iommu_fwspec_init(dev, &np->fwnode, ops) ||
+		    ops->of_xlate(dev, &iommu_spec))
 			goto err_put_node;
 
 		of_node_put(np);
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index a04d491cf431..3b44b1d82f3b 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -101,8 +101,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
 		zpci_dma_exit_device(zdev);
 
 	zdev->dma_table = s390_domain->dma_table;
-	rc = zpci_register_ioat(zdev, 0, zdev->start_dma + PAGE_OFFSET,
-				zdev->start_dma + zdev->iommu_size - 1,
+	rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
 				(u64) zdev->dma_table);
 	if (rc)
 		goto out_restore;
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index 7f8728984f44..82b0b5daf3f5 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -39,6 +39,7 @@ config ARM_GIC_V3_ITS
 	bool
 	depends on PCI
 	depends on PCI_MSI
+	select ACPI_IORT if ACPI
 
 config ARM_NVIC
 	bool
@@ -156,6 +157,13 @@ config PIC32_EVIC
 	select GENERIC_IRQ_CHIP
 	select IRQ_DOMAIN
 
+config JCORE_AIC
+	bool "J-Core integrated AIC"
+	depends on OF && (SUPERH || COMPILE_TEST)
+	select IRQ_DOMAIN
+	help
+	  Support for the J-Core integrated AIC.
+
 config RENESAS_INTC_IRQPIN
 	bool
 	select IRQ_DOMAIN
@@ -251,6 +259,9 @@ config IRQ_MXS
 config MVEBU_ODMI
 	bool
 
+config MVEBU_PIC
+	bool
+
 config LS_SCFG_MSI
 	def_bool y if SOC_LS1021A || ARCH_LAYERSCAPE
 	depends on PCI && PCI_MSI
@@ -264,3 +275,7 @@ config EZNPS_GIC
 	select IRQ_DOMAIN
 	help
 	  Support the EZchip NPS400 global interrupt controller
+
+config STM32_EXTI
+	bool
+	select IRQ_DOMAIN
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 4c203b6b8163..e4dbfc85abdb 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -25,7 +25,7 @@ obj-$(CONFIG_ARCH_SUNXI)		+= irq-sunxi-nmi.o
 obj-$(CONFIG_ARCH_SPEAR3XX)		+= spear-shirq.o
 obj-$(CONFIG_ARM_GIC)			+= irq-gic.o irq-gic-common.o
 obj-$(CONFIG_ARM_GIC_PM)		+= irq-gic-pm.o
-obj-$(CONFIG_REALVIEW_DT)		+= irq-gic-realview.o
+obj-$(CONFIG_ARCH_REALVIEW)		+= irq-gic-realview.o
 obj-$(CONFIG_ARM_GIC_V2M)		+= irq-gic-v2m.o
 obj-$(CONFIG_ARM_GIC_V3)		+= irq-gic-v3.o irq-gic-common.o
 obj-$(CONFIG_ARM_GIC_V3_ITS)		+= irq-gic-v3-its.o irq-gic-v3-its-pci-msi.o irq-gic-v3-its-platform-msi.o
@@ -40,6 +40,7 @@ obj-$(CONFIG_I8259)			+= irq-i8259.o
 obj-$(CONFIG_IMGPDC_IRQ)		+= irq-imgpdc.o
 obj-$(CONFIG_IRQ_MIPS_CPU)		+= irq-mips-cpu.o
 obj-$(CONFIG_SIRF_IRQ)			+= irq-sirfsoc.o
+obj-$(CONFIG_JCORE_AIC)			+= irq-jcore-aic.o
 obj-$(CONFIG_RENESAS_INTC_IRQPIN)	+= irq-renesas-intc-irqpin.o
 obj-$(CONFIG_RENESAS_IRQC)		+= irq-renesas-irqc.o
 obj-$(CONFIG_VERSATILE_FPGA_IRQ)	+= irq-versatile-fpga.o
@@ -68,6 +69,8 @@ obj-$(CONFIG_INGENIC_IRQ)		+= irq-ingenic.o
 obj-$(CONFIG_IMX_GPCV2)			+= irq-imx-gpcv2.o
 obj-$(CONFIG_PIC32_EVIC)		+= irq-pic32-evic.o
 obj-$(CONFIG_MVEBU_ODMI)		+= irq-mvebu-odmi.o
+obj-$(CONFIG_MVEBU_PIC)			+= irq-mvebu-pic.o
 obj-$(CONFIG_LS_SCFG_MSI)		+= irq-ls-scfg-msi.o
 obj-$(CONFIG_EZNPS_GIC)			+= irq-eznps.o
 obj-$(CONFIG_ARCH_ASPEED)		+= irq-aspeed-vic.o
+obj-$(CONFIG_STM32_EXTI) 		+= irq-stm32-exti.o
diff --git a/drivers/irqchip/irq-atmel-aic.c b/drivers/irqchip/irq-atmel-aic.c
index 112e17c2768b..37f952dd9fc9 100644
--- a/drivers/irqchip/irq-atmel-aic.c
+++ b/drivers/irqchip/irq-atmel-aic.c
@@ -176,6 +176,7 @@ static int aic_irq_domain_xlate(struct irq_domain *d,
 {
 	struct irq_domain_chip_generic *dgc = d->gc;
 	struct irq_chip_generic *gc;
+	unsigned long flags;
 	unsigned smr;
 	int idx;
 	int ret;
@@ -194,11 +195,11 @@ static int aic_irq_domain_xlate(struct irq_domain *d,
 
 	gc = dgc->gc[idx];
 
-	irq_gc_lock(gc);
+	irq_gc_lock_irqsave(gc, flags);
 	smr = irq_reg_readl(gc, AT91_AIC_SMR(*out_hwirq));
 	aic_common_set_priority(intspec[2], &smr);
 	irq_reg_writel(gc, smr, AT91_AIC_SMR(*out_hwirq));
-	irq_gc_unlock(gc);
+	irq_gc_unlock_irqrestore(gc, flags);
 
 	return ret;
 }
diff --git a/drivers/irqchip/irq-atmel-aic5.c b/drivers/irqchip/irq-atmel-aic5.c
index 4f0d068e1abe..2a624d87a035 100644
--- a/drivers/irqchip/irq-atmel-aic5.c
+++ b/drivers/irqchip/irq-atmel-aic5.c
@@ -258,6 +258,7 @@ static int aic5_irq_domain_xlate(struct irq_domain *d,
 				 unsigned int *out_type)
 {
 	struct irq_chip_generic *bgc = irq_get_domain_generic_chip(d, 0);
+	unsigned long flags;
 	unsigned smr;
 	int ret;
 
@@ -269,12 +270,12 @@ static int aic5_irq_domain_xlate(struct irq_domain *d,
 	if (ret)
 		return ret;
 
-	irq_gc_lock(bgc);
+	irq_gc_lock_irqsave(bgc, flags);
 	irq_reg_writel(bgc, *out_hwirq, AT91_AIC5_SSR);
 	smr = irq_reg_readl(bgc, AT91_AIC5_SMR);
 	aic_common_set_priority(intspec[2], &smr);
 	irq_reg_writel(bgc, smr, AT91_AIC5_SMR);
-	irq_gc_unlock(bgc);
+	irq_gc_unlock_irqrestore(bgc, flags);
 
 	return ret;
 }
diff --git a/drivers/irqchip/irq-bcm6345-l1.c b/drivers/irqchip/irq-bcm6345-l1.c
index b844c89a9506..daa4ae89e466 100644
--- a/drivers/irqchip/irq-bcm6345-l1.c
+++ b/drivers/irqchip/irq-bcm6345-l1.c
@@ -52,7 +52,6 @@
 
 #include <linux/bitops.h>
 #include <linux/cpumask.h>
-#include <linux/kconfig.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c
index 0fea985ef1dc..353c54986211 100644
--- a/drivers/irqchip/irq-bcm7038-l1.c
+++ b/drivers/irqchip/irq-bcm7038-l1.c
@@ -12,7 +12,6 @@
 #define pr_fmt(fmt)	KBUILD_MODNAME	": " fmt
 
 #include <linux/bitops.h>
-#include <linux/kconfig.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
diff --git a/drivers/irqchip/irq-bcm7120-l2.c b/drivers/irqchip/irq-bcm7120-l2.c
index 0ec92631e23c..64c2692070ef 100644
--- a/drivers/irqchip/irq-bcm7120-l2.c
+++ b/drivers/irqchip/irq-bcm7120-l2.c
@@ -13,7 +13,6 @@
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-#include <linux/kconfig.h>
 #include <linux/kernel.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c
index 1d4a5b46d9ae..bddf169c4b37 100644
--- a/drivers/irqchip/irq-brcmstb-l2.c
+++ b/drivers/irqchip/irq-brcmstb-l2.c
@@ -18,7 +18,6 @@
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-#include <linux/kconfig.h>
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
 #include <linux/of.h>
diff --git a/drivers/irqchip/irq-gic-pm.c b/drivers/irqchip/irq-gic-pm.c
index 4cbffba3ff13..ecafd295c31c 100644
--- a/drivers/irqchip/irq-gic-pm.c
+++ b/drivers/irqchip/irq-gic-pm.c
@@ -64,7 +64,6 @@ static int gic_runtime_suspend(struct device *dev)
 
 static int gic_get_clocks(struct device *dev, const struct gic_clk_data *data)
 {
-	struct clk *clk;
 	unsigned int i;
 	int ret;
 
@@ -76,28 +75,16 @@ static int gic_get_clocks(struct device *dev, const struct gic_clk_data *data)
 		return ret;
 
 	for (i = 0; i < data->num_clocks; i++) {
-		clk = of_clk_get_by_name(dev->of_node, data->clocks[i]);
-		if (IS_ERR(clk)) {
-			dev_err(dev, "failed to get clock %s\n",
-				data->clocks[i]);
-			ret = PTR_ERR(clk);
-			goto error;
-		}
-
-		ret = pm_clk_add_clk(dev, clk);
+		ret = of_pm_clk_add_clk(dev, data->clocks[i]);
 		if (ret) {
-			dev_err(dev, "failed to add clock at index %d\n", i);
-			clk_put(clk);
-			goto error;
+			dev_err(dev, "failed to add clock %s\n",
+				data->clocks[i]);
+			pm_clk_destroy(dev);
+			return ret;
 		}
 	}
 
 	return 0;
-
-error:
-	pm_clk_destroy(dev);
-
-	return ret;
 }
 
 static int gic_probe(struct platform_device *pdev)
diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c
index 35eb7ac5d21f..863e073c6f7f 100644
--- a/drivers/irqchip/irq-gic-v2m.c
+++ b/drivers/irqchip/irq-gic-v2m.c
@@ -16,6 +16,7 @@
 #define pr_fmt(fmt) "GICv2m: " fmt
 
 #include <linux/acpi.h>
+#include <linux/dma-iommu.h>
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
@@ -108,6 +109,8 @@ static void gicv2m_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 
 	if (v2m->flags & GICV2M_NEEDS_SPI_OFFSET)
 		msg->data -= v2m->spi_offset;
+
+	iommu_dma_map_msi_msg(data->irq, msg);
 }
 
 static struct irq_chip gicv2m_irq_chip = {
diff --git a/drivers/irqchip/irq-gic-v3-its-pci-msi.c b/drivers/irqchip/irq-gic-v3-its-pci-msi.c
index aee60ed025dc..aee1c60d7ab5 100644
--- a/drivers/irqchip/irq-gic-v3-its-pci-msi.c
+++ b/drivers/irqchip/irq-gic-v3-its-pci-msi.c
@@ -15,6 +15,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/acpi_iort.h>
 #include <linux/msi.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
@@ -106,34 +107,91 @@ static struct of_device_id its_device_id[] = {
 	{},
 };
 
-static int __init its_pci_msi_init(void)
+static int __init its_pci_msi_init_one(struct fwnode_handle *handle,
+				       const char *name)
 {
-	struct device_node *np;
 	struct irq_domain *parent;
 
+	parent = irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS);
+	if (!parent || !msi_get_domain_info(parent)) {
+		pr_err("%s: Unable to locate ITS domain\n", name);
+		return -ENXIO;
+	}
+
+	if (!pci_msi_create_irq_domain(handle, &its_pci_msi_domain_info,
+				       parent)) {
+		pr_err("%s: Unable to create PCI domain\n", name);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int __init its_pci_of_msi_init(void)
+{
+	struct device_node *np;
+
 	for (np = of_find_matching_node(NULL, its_device_id); np;
 	     np = of_find_matching_node(np, its_device_id)) {
 		if (!of_property_read_bool(np, "msi-controller"))
 			continue;
 
-		parent = irq_find_matching_host(np, DOMAIN_BUS_NEXUS);
-		if (!parent || !msi_get_domain_info(parent)) {
-			pr_err("%s: unable to locate ITS domain\n",
-			       np->full_name);
-			continue;
-		}
-
-		if (!pci_msi_create_irq_domain(of_node_to_fwnode(np),
-					       &its_pci_msi_domain_info,
-					       parent)) {
-			pr_err("%s: unable to create PCI domain\n",
-			       np->full_name);
+		if (its_pci_msi_init_one(of_node_to_fwnode(np), np->full_name))
 			continue;
-		}
 
 		pr_info("PCI/MSI: %s domain created\n", np->full_name);
 	}
 
 	return 0;
 }
+
+#ifdef CONFIG_ACPI
+
+static int __init
+its_pci_msi_parse_madt(struct acpi_subtable_header *header,
+		       const unsigned long end)
+{
+	struct acpi_madt_generic_translator *its_entry;
+	struct fwnode_handle *dom_handle;
+	const char *node_name;
+	int err = -ENXIO;
+
+	its_entry = (struct acpi_madt_generic_translator *)header;
+	node_name = kasprintf(GFP_KERNEL, "ITS@0x%lx",
+			      (long)its_entry->base_address);
+	dom_handle = iort_find_domain_token(its_entry->translation_id);
+	if (!dom_handle) {
+		pr_err("%s: Unable to locate ITS domain handle\n", node_name);
+		goto out;
+	}
+
+	err = its_pci_msi_init_one(dom_handle, node_name);
+	if (!err)
+		pr_info("PCI/MSI: %s domain created\n", node_name);
+
+out:
+	kfree(node_name);
+	return err;
+}
+
+static int __init its_pci_acpi_msi_init(void)
+{
+	acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
+			      its_pci_msi_parse_madt, 0);
+	return 0;
+}
+#else
+static int __init its_pci_acpi_msi_init(void)
+{
+	return 0;
+}
+#endif
+
+static int __init its_pci_msi_init(void)
+{
+	its_pci_of_msi_init();
+	its_pci_acpi_msi_init();
+
+	return 0;
+}
 early_initcall(its_pci_msi_init);
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 7ceaba81efb4..003495d91f9c 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -15,10 +15,14 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/acpi.h>
 #include <linux/bitmap.h>
 #include <linux/cpu.h>
 #include <linux/delay.h>
+#include <linux/dma-iommu.h>
 #include <linux/interrupt.h>
+#include <linux/irqdomain.h>
+#include <linux/acpi_iort.h>
 #include <linux/log2.h>
 #include <linux/mm.h>
 #include <linux/msi.h>
@@ -75,7 +79,7 @@ struct its_node {
 	raw_spinlock_t		lock;
 	struct list_head	entry;
 	void __iomem		*base;
-	unsigned long		phys_base;
+	phys_addr_t		phys_base;
 	struct its_cmd_block	*cmd_base;
 	struct its_cmd_block	*cmd_write;
 	struct its_baser	tables[GITS_BASER_NR_REGS];
@@ -115,6 +119,7 @@ struct its_device {
 static LIST_HEAD(its_nodes);
 static DEFINE_SPINLOCK(its_lock);
 static struct rdists *gic_rdists;
+static struct irq_domain *its_parent;
 
 #define gic_data_rdist()		(raw_cpu_ptr(gic_rdists->rdist))
 #define gic_data_rdist_rd_base()	(gic_data_rdist()->rd_base)
@@ -655,6 +660,8 @@ static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
 	msg->address_lo		= addr & ((1UL << 32) - 1);
 	msg->address_hi		= addr >> 32;
 	msg->data		= its_get_event_id(d);
+
+	iommu_dma_map_msi_msg(d->irq, msg);
 }
 
 static struct irq_chip its_irq_chip = {
@@ -1437,6 +1444,11 @@ static int its_irq_gic_domain_alloc(struct irq_domain *domain,
 		fwspec.param[0] = GIC_IRQ_TYPE_LPI;
 		fwspec.param[1] = hwirq;
 		fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
+	} else if (is_fwnode_irqchip(domain->parent->fwnode)) {
+		fwspec.fwnode = domain->parent->fwnode;
+		fwspec.param_count = 2;
+		fwspec.param[0] = hwirq;
+		fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
 	} else {
 		return -EINVAL;
 	}
@@ -1545,7 +1557,12 @@ static int its_force_quiescent(void __iomem *base)
 	u32 val;
 
 	val = readl_relaxed(base + GITS_CTLR);
-	if (val & GITS_CTLR_QUIESCENT)
+	/*
+	 * GIC architecture specification requires the ITS to be both
+	 * disabled and quiescent for writes to GITS_BASER<n> or
+	 * GITS_CBASER to not have UNPREDICTABLE results.
+	 */
+	if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
 		return 0;
 
 	/* Disable the generation of all interrupts to this ITS */
@@ -1609,44 +1626,59 @@ static void its_enable_quirks(struct its_node *its)
 	gic_enable_quirks(iidr, its_quirks, its);
 }
 
-static int __init its_probe(struct device_node *node,
-			    struct irq_domain *parent)
+static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
+{
+	struct irq_domain *inner_domain;
+	struct msi_domain_info *info;
+
+	info = kzalloc(sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its);
+	if (!inner_domain) {
+		kfree(info);
+		return -ENOMEM;
+	}
+
+	inner_domain->parent = its_parent;
+	inner_domain->bus_token = DOMAIN_BUS_NEXUS;
+	info->ops = &its_msi_domain_ops;
+	info->data = its;
+	inner_domain->host_data = info;
+
+	return 0;
+}
+
+static int __init its_probe_one(struct resource *res,
+				struct fwnode_handle *handle, int numa_node)
 {
-	struct resource res;
 	struct its_node *its;
 	void __iomem *its_base;
-	struct irq_domain *inner_domain;
 	u32 val;
 	u64 baser, tmp;
 	int err;
 
-	err = of_address_to_resource(node, 0, &res);
-	if (err) {
-		pr_warn("%s: no regs?\n", node->full_name);
-		return -ENXIO;
-	}
-
-	its_base = ioremap(res.start, resource_size(&res));
+	its_base = ioremap(res->start, resource_size(res));
 	if (!its_base) {
-		pr_warn("%s: unable to map registers\n", node->full_name);
+		pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
 		return -ENOMEM;
 	}
 
 	val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
 	if (val != 0x30 && val != 0x40) {
-		pr_warn("%s: no ITS detected, giving up\n", node->full_name);
+		pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
 		err = -ENODEV;
 		goto out_unmap;
 	}
 
 	err = its_force_quiescent(its_base);
 	if (err) {
-		pr_warn("%s: failed to quiesce, giving up\n",
-			node->full_name);
+		pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
 		goto out_unmap;
 	}
 
-	pr_info("ITS: %s\n", node->full_name);
+	pr_info("ITS %pR\n", res);
 
 	its = kzalloc(sizeof(*its), GFP_KERNEL);
 	if (!its) {
@@ -1658,9 +1690,9 @@ static int __init its_probe(struct device_node *node,
 	INIT_LIST_HEAD(&its->entry);
 	INIT_LIST_HEAD(&its->its_device_list);
 	its->base = its_base;
-	its->phys_base = res.start;
+	its->phys_base = res->start;
 	its->ite_size = ((readl_relaxed(its_base + GITS_TYPER) >> 4) & 0xf) + 1;
-	its->numa_node = of_node_to_nid(node);
+	its->numa_node = numa_node;
 
 	its->cmd_base = kzalloc(ITS_CMD_QUEUE_SZ, GFP_KERNEL);
 	if (!its->cmd_base) {
@@ -1707,28 +1739,9 @@ static int __init its_probe(struct device_node *node,
 	writeq_relaxed(0, its->base + GITS_CWRITER);
 	writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR);
 
-	if (of_property_read_bool(node, "msi-controller")) {
-		struct msi_domain_info *info;
-
-		info = kzalloc(sizeof(*info), GFP_KERNEL);
-		if (!info) {
-			err = -ENOMEM;
-			goto out_free_tables;
-		}
-
-		inner_domain = irq_domain_add_tree(node, &its_domain_ops, its);
-		if (!inner_domain) {
-			err = -ENOMEM;
-			kfree(info);
-			goto out_free_tables;
-		}
-
-		inner_domain->parent = parent;
-		inner_domain->bus_token = DOMAIN_BUS_NEXUS;
-		info->ops = &its_msi_domain_ops;
-		info->data = its;
-		inner_domain->host_data = info;
-	}
+	err = its_init_domain(handle, its);
+	if (err)
+		goto out_free_tables;
 
 	spin_lock(&its_lock);
 	list_add(&its->entry, &its_nodes);
@@ -1744,7 +1757,7 @@ out_free_its:
 	kfree(its);
 out_unmap:
 	iounmap(its_base);
-	pr_err("ITS: failed probing %s (%d)\n", node->full_name, err);
+	pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err);
 	return err;
 }
 
@@ -1772,16 +1785,92 @@ static struct of_device_id its_device_id[] = {
 	{},
 };
 
-int __init its_init(struct device_node *node, struct rdists *rdists,
-	     struct irq_domain *parent_domain)
+static int __init its_of_probe(struct device_node *node)
 {
 	struct device_node *np;
+	struct resource res;
 
 	for (np = of_find_matching_node(node, its_device_id); np;
 	     np = of_find_matching_node(np, its_device_id)) {
-		its_probe(np, parent_domain);
+		if (!of_property_read_bool(np, "msi-controller")) {
+			pr_warn("%s: no msi-controller property, ITS ignored\n",
+				np->full_name);
+			continue;
+		}
+
+		if (of_address_to_resource(np, 0, &res)) {
+			pr_warn("%s: no regs?\n", np->full_name);
+			continue;
+		}
+
+		its_probe_one(&res, &np->fwnode, of_node_to_nid(np));
+	}
+	return 0;
+}
+
+#ifdef CONFIG_ACPI
+
+#define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
+
+static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header,
+					  const unsigned long end)
+{
+	struct acpi_madt_generic_translator *its_entry;
+	struct fwnode_handle *dom_handle;
+	struct resource res;
+	int err;
+
+	its_entry = (struct acpi_madt_generic_translator *)header;
+	memset(&res, 0, sizeof(res));
+	res.start = its_entry->base_address;
+	res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
+	res.flags = IORESOURCE_MEM;
+
+	dom_handle = irq_domain_alloc_fwnode((void *)its_entry->base_address);
+	if (!dom_handle) {
+		pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
+		       &res.start);
+		return -ENOMEM;
 	}
 
+	err = iort_register_domain_token(its_entry->translation_id, dom_handle);
+	if (err) {
+		pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
+		       &res.start, its_entry->translation_id);
+		goto dom_err;
+	}
+
+	err = its_probe_one(&res, dom_handle, NUMA_NO_NODE);
+	if (!err)
+		return 0;
+
+	iort_deregister_domain_token(its_entry->translation_id);
+dom_err:
+	irq_domain_free_fwnode(dom_handle);
+	return err;
+}
+
+static void __init its_acpi_probe(void)
+{
+	acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
+			      gic_acpi_parse_madt_its, 0);
+}
+#else
+static void __init its_acpi_probe(void) { }
+#endif
+
+int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
+		    struct irq_domain *parent_domain)
+{
+	struct device_node *of_node;
+
+	its_parent = parent_domain;
+	of_node = to_of_node(handle);
+	if (of_node)
+		its_of_probe(of_node);
+	else
+		its_acpi_probe();
+
 	if (list_empty(&its_nodes)) {
 		pr_warn("ITS: No ITS available, not enabling LPIs\n");
 		return -ENXIO;
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 6fc56c3466b0..9b81bd8b929c 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -495,6 +495,14 @@ static void gic_cpu_sys_reg_init(void)
 	/* Set priority mask register */
 	gic_write_pmr(DEFAULT_PMR_VALUE);
 
+	/*
+	 * Some firmwares hand over to the kernel with the BPR changed from
+	 * its reset value (and with a value large enough to prevent
+	 * any pre-emptive interrupts from working at all). Writing a zero
+	 * to BPR restores is reset value.
+	 */
+	gic_write_bpr1(0);
+
 	if (static_key_true(&supports_deactivate)) {
 		/* EOI drops priority only (mode 1) */
 		gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
@@ -548,7 +556,7 @@ static int gic_starting_cpu(unsigned int cpu)
 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
 				   unsigned long cluster_id)
 {
-	int cpu = *base_cpu;
+	int next_cpu, cpu = *base_cpu;
 	unsigned long mpidr = cpu_logical_map(cpu);
 	u16 tlist = 0;
 
@@ -562,9 +570,10 @@ static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
 
 		tlist |= 1 << (mpidr & 0xf);
 
-		cpu = cpumask_next(cpu, mask);
-		if (cpu >= nr_cpu_ids)
+		next_cpu = cpumask_next(cpu, mask);
+		if (next_cpu >= nr_cpu_ids)
 			goto out;
+		cpu = next_cpu;
 
 		mpidr = cpu_logical_map(cpu);
 
@@ -667,13 +676,20 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 #endif
 
 #ifdef CONFIG_CPU_PM
+/* Check whether it's single security state view */
+static bool gic_dist_security_disabled(void)
+{
+	return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
+}
+
 static int gic_cpu_pm_notifier(struct notifier_block *self,
 			       unsigned long cmd, void *v)
 {
 	if (cmd == CPU_PM_EXIT) {
-		gic_enable_redist(true);
+		if (gic_dist_security_disabled())
+			gic_enable_redist(true);
 		gic_cpu_sys_reg_init();
-	} else if (cmd == CPU_PM_ENTER) {
+	} else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
 		gic_write_grpen1(0);
 		gic_enable_redist(false);
 	}
@@ -903,7 +919,6 @@ static int __init gic_init_bases(void __iomem *dist_base,
 				 u64 redist_stride,
 				 struct fwnode_handle *handle)
 {
-	struct device_node *node;
 	u32 typer;
 	int gic_irqs;
 	int err;
@@ -944,10 +959,8 @@ static int __init gic_init_bases(void __iomem *dist_base,
 
 	set_handle_irq(gic_handle_irq);
 
-	node = to_of_node(handle);
-	if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis() &&
-	    node) /* Temp hack to prevent ITS init for ACPI */
-		its_init(node, &gic_data.rdists, gic_data.domain);
+	if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
+		its_init(handle, &gic_data.rdists, gic_data.domain);
 
 	gic_smp_init();
 	gic_dist_init();
diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c
index c2cab572c511..58e5b4e87056 100644
--- a/drivers/irqchip/irq-gic.c
+++ b/drivers/irqchip/irq-gic.c
@@ -91,7 +91,27 @@ struct gic_chip_data {
 #endif
 };
 
-static DEFINE_RAW_SPINLOCK(irq_controller_lock);
+#ifdef CONFIG_BL_SWITCHER
+
+static DEFINE_RAW_SPINLOCK(cpu_map_lock);
+
+#define gic_lock_irqsave(f)		\
+	raw_spin_lock_irqsave(&cpu_map_lock, (f))
+#define gic_unlock_irqrestore(f)	\
+	raw_spin_unlock_irqrestore(&cpu_map_lock, (f))
+
+#define gic_lock()			raw_spin_lock(&cpu_map_lock)
+#define gic_unlock()			raw_spin_unlock(&cpu_map_lock)
+
+#else
+
+#define gic_lock_irqsave(f)		do { (void)(f); } while(0)
+#define gic_unlock_irqrestore(f)	do { (void)(f); } while(0)
+
+#define gic_lock()			do { } while(0)
+#define gic_unlock()			do { } while(0)
+
+#endif
 
 /*
  * The GIC mapping of CPU interfaces does not necessarily match
@@ -317,12 +337,12 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 	if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids)
 		return -EINVAL;
 
-	raw_spin_lock_irqsave(&irq_controller_lock, flags);
+	gic_lock_irqsave(flags);
 	mask = 0xff << shift;
 	bit = gic_cpu_map[cpu] << shift;
 	val = readl_relaxed(reg) & ~mask;
 	writel_relaxed(val | bit, reg);
-	raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
+	gic_unlock_irqrestore(flags);
 
 	return IRQ_SET_MASK_OK_DONE;
 }
@@ -374,9 +394,7 @@ static void gic_handle_cascade_irq(struct irq_desc *desc)
 
 	chained_irq_enter(chip, desc);
 
-	raw_spin_lock(&irq_controller_lock);
 	status = readl_relaxed(gic_data_cpu_base(chip_data) + GIC_CPU_INTACK);
-	raw_spin_unlock(&irq_controller_lock);
 
 	gic_irq = (status & GICC_IAR_INT_ID_MASK);
 	if (gic_irq == GICC_INT_SPURIOUS)
@@ -769,7 +787,14 @@ static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
 	int cpu;
 	unsigned long flags, map = 0;
 
-	raw_spin_lock_irqsave(&irq_controller_lock, flags);
+	if (unlikely(nr_cpu_ids == 1)) {
+		/* Only one CPU? let's do a self-IPI... */
+		writel_relaxed(2 << 24 | irq,
+			       gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
+		return;
+	}
+
+	gic_lock_irqsave(flags);
 
 	/* Convert our logical CPU mask into a physical one. */
 	for_each_cpu(cpu, mask)
@@ -784,7 +809,7 @@ static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
 	/* this always happens on GIC0 */
 	writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
 
-	raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
+	gic_unlock_irqrestore(flags);
 }
 #endif
 
@@ -852,7 +877,7 @@ void gic_migrate_target(unsigned int new_cpu_id)
 	cur_target_mask = 0x01010101 << cur_cpu_id;
 	ror_val = (cur_cpu_id - new_cpu_id) & 31;
 
-	raw_spin_lock(&irq_controller_lock);
+	gic_lock();
 
 	/* Update the target interface for this logical CPU */
 	gic_cpu_map[cpu] = 1 << new_cpu_id;
@@ -872,7 +897,7 @@ void gic_migrate_target(unsigned int new_cpu_id)
 		}
 	}
 
-	raw_spin_unlock(&irq_controller_lock);
+	gic_unlock();
 
 	/*
 	 * Now let's migrate and clear any potential SGIs that might be
@@ -914,7 +939,7 @@ unsigned long gic_get_sgir_physaddr(void)
 	return gic_dist_physaddr + GIC_DIST_SOFTINT;
 }
 
-void __init gic_init_physaddr(struct device_node *node)
+static void __init gic_init_physaddr(struct device_node *node)
 {
 	struct resource res;
 	if (of_address_to_resource(node, 0, &res) == 0) {
diff --git a/drivers/irqchip/irq-jcore-aic.c b/drivers/irqchip/irq-jcore-aic.c
new file mode 100644
index 000000000000..84b01dec277d
--- /dev/null
+++ b/drivers/irqchip/irq-jcore-aic.c
@@ -0,0 +1,95 @@
+/*
+ * J-Core SoC AIC driver
+ *
+ * Copyright (C) 2015-2016 Smart Energy Instruments, Inc.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+
+#include <linux/irq.h>
+#include <linux/io.h>
+#include <linux/irqchip.h>
+#include <linux/irqdomain.h>
+#include <linux/cpu.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+
+#define JCORE_AIC_MAX_HWIRQ	127
+#define JCORE_AIC1_MIN_HWIRQ	16
+#define JCORE_AIC2_MIN_HWIRQ	64
+
+#define JCORE_AIC1_INTPRI_REG	8
+
+static struct irq_chip jcore_aic;
+
+static int jcore_aic_irqdomain_map(struct irq_domain *d, unsigned int irq,
+				   irq_hw_number_t hwirq)
+{
+	struct irq_chip *aic = d->host_data;
+
+	irq_set_chip_and_handler(irq, aic, handle_simple_irq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops jcore_aic_irqdomain_ops = {
+	.map = jcore_aic_irqdomain_map,
+	.xlate = irq_domain_xlate_onecell,
+};
+
+static void noop(struct irq_data *data)
+{
+}
+
+static int __init aic_irq_of_init(struct device_node *node,
+				  struct device_node *parent)
+{
+	unsigned min_irq = JCORE_AIC2_MIN_HWIRQ;
+	unsigned dom_sz = JCORE_AIC_MAX_HWIRQ+1;
+	struct irq_domain *domain;
+
+	pr_info("Initializing J-Core AIC\n");
+
+	/* AIC1 needs priority initialization to receive interrupts. */
+	if (of_device_is_compatible(node, "jcore,aic1")) {
+		unsigned cpu;
+
+		for_each_present_cpu(cpu) {
+			void __iomem *base = of_iomap(node, cpu);
+
+			if (!base) {
+				pr_err("Unable to map AIC for cpu %u\n", cpu);
+				return -ENOMEM;
+			}
+			__raw_writel(0xffffffff, base + JCORE_AIC1_INTPRI_REG);
+			iounmap(base);
+		}
+		min_irq = JCORE_AIC1_MIN_HWIRQ;
+	}
+
+	/*
+	 * The irq chip framework requires either mask/unmask or enable/disable
+	 * function pointers to be provided, but the hardware does not have any
+	 * such mechanism; the only interrupt masking is at the cpu level and
+	 * it affects all interrupts. We provide dummy mask/unmask. The hardware
+	 * handles all interrupt control and clears pending status when the cpu
+	 * accepts the interrupt.
+	 */
+	jcore_aic.irq_mask = noop;
+	jcore_aic.irq_unmask = noop;
+	jcore_aic.name = "AIC";
+
+	domain = irq_domain_add_linear(node, dom_sz, &jcore_aic_irqdomain_ops,
+				       &jcore_aic);
+	if (!domain)
+		return -ENOMEM;
+	irq_create_strict_mappings(domain, min_irq, min_irq, dom_sz - min_irq);
+
+	return 0;
+}
+
+IRQCHIP_DECLARE(jcore_aic2, "jcore,aic2", aic_irq_of_init);
+IRQCHIP_DECLARE(jcore_aic1, "jcore,aic1", aic_irq_of_init);
diff --git a/drivers/irqchip/irq-keystone.c b/drivers/irqchip/irq-keystone.c
index deb89d63a728..54a5e870a8f5 100644
--- a/drivers/irqchip/irq-keystone.c
+++ b/drivers/irqchip/irq-keystone.c
@@ -109,7 +109,7 @@ static void keystone_irq_handler(struct irq_desc *desc)
 			dev_dbg(kirq->dev, "dispatch bit %d, virq %d\n",
 				src, virq);
 			if (!virq)
-				dev_warn(kirq->dev, "sporious irq detected hwirq %d, virq %d\n",
+				dev_warn(kirq->dev, "spurious irq detected hwirq %d, virq %d\n",
 					 src, virq);
 			generic_handle_irq(virq);
 		}
diff --git a/drivers/irqchip/irq-metag-ext.c b/drivers/irqchip/irq-metag-ext.c
index 8c38b3d92e1c..0cdd923d1535 100644
--- a/drivers/irqchip/irq-metag-ext.c
+++ b/drivers/irqchip/irq-metag-ext.c
@@ -436,7 +436,6 @@ static int meta_intc_irq_set_type(struct irq_data *data, unsigned int flow_type)
 
 /**
  * meta_intc_irq_demux() - external irq de-multiplexer
- * @irq:	the virtual interrupt number
  * @desc:	the interrupt description structure for this irq
  *
  * The cpu receives an interrupt on TR2 when a SoC interrupt has occurred. It is
diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index c5f33c3bd228..c0178a122940 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -371,18 +371,13 @@ static void gic_handle_shared_int(bool chained)
 	bitmap_and(pending, pending, intrmask, gic_shared_intrs);
 	bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs);
 
-	intr = find_first_bit(pending, gic_shared_intrs);
-	while (intr != gic_shared_intrs) {
+	for_each_set_bit(intr, pending, gic_shared_intrs) {
 		virq = irq_linear_revmap(gic_irq_domain,
 					 GIC_SHARED_TO_HWIRQ(intr));
 		if (chained)
 			generic_handle_irq(virq);
 		else
 			do_IRQ(virq);
-
-		/* go to next pending bit */
-		bitmap_clear(pending, intr, 1);
-		intr = find_first_bit(pending, gic_shared_intrs);
 	}
 }
 
@@ -518,18 +513,13 @@ static void gic_handle_local_int(bool chained)
 
 	bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS);
 
-	intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
-	while (intr != GIC_NUM_LOCAL_INTRS) {
+	for_each_set_bit(intr, &pending, GIC_NUM_LOCAL_INTRS) {
 		virq = irq_linear_revmap(gic_irq_domain,
 					 GIC_LOCAL_TO_HWIRQ(intr));
 		if (chained)
 			generic_handle_irq(virq);
 		else
 			do_IRQ(virq);
-
-		/* go to next pending bit */
-		bitmap_clear(&pending, intr, 1);
-		intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
 	}
 }
 
@@ -638,27 +628,6 @@ static int gic_local_irq_domain_map(struct irq_domain *d, unsigned int virq,
 	if (!gic_local_irq_is_routable(intr))
 		return -EPERM;
 
-	/*
-	 * HACK: These are all really percpu interrupts, but the rest
-	 * of the MIPS kernel code does not use the percpu IRQ API for
-	 * the CP0 timer and performance counter interrupts.
-	 */
-	switch (intr) {
-	case GIC_LOCAL_INT_TIMER:
-	case GIC_LOCAL_INT_PERFCTR:
-	case GIC_LOCAL_INT_FDC:
-		irq_set_chip_and_handler(virq,
-					 &gic_all_vpes_local_irq_controller,
-					 handle_percpu_irq);
-		break;
-	default:
-		irq_set_chip_and_handler(virq,
-					 &gic_local_irq_controller,
-					 handle_percpu_devid_irq);
-		irq_set_percpu_devid(virq);
-		break;
-	}
-
 	spin_lock_irqsave(&gic_lock, flags);
 	for (i = 0; i < gic_vpes; i++) {
 		u32 val = GIC_MAP_TO_PIN_MSK | gic_cpu_pin;
@@ -713,9 +682,6 @@ static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
 	unsigned long flags;
 	int i;
 
-	irq_set_chip_and_handler(virq, &gic_level_irq_controller,
-				 handle_level_irq);
-
 	spin_lock_irqsave(&gic_lock, flags);
 	gic_map_to_pin(intr, gic_cpu_pin);
 	gic_map_to_vpe(intr, mips_cm_vp_id(vpe));
@@ -727,12 +693,42 @@ static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
 	return 0;
 }
 
-static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
-			      irq_hw_number_t hw)
+static int gic_setup_dev_chip(struct irq_domain *d, unsigned int virq,
+			      unsigned int hwirq)
 {
-	if (GIC_HWIRQ_TO_LOCAL(hw) < GIC_NUM_LOCAL_INTRS)
-		return gic_local_irq_domain_map(d, virq, hw);
-	return gic_shared_irq_domain_map(d, virq, hw, 0);
+	struct irq_chip *chip;
+	int err;
+
+	if (hwirq >= GIC_SHARED_HWIRQ_BASE) {
+		err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
+						    &gic_level_irq_controller,
+						    NULL);
+	} else {
+		switch (GIC_HWIRQ_TO_LOCAL(hwirq)) {
+		case GIC_LOCAL_INT_TIMER:
+		case GIC_LOCAL_INT_PERFCTR:
+		case GIC_LOCAL_INT_FDC:
+			/*
+			 * HACK: These are all really percpu interrupts, but
+			 * the rest of the MIPS kernel code does not use the
+			 * percpu IRQ API for them.
+			 */
+			chip = &gic_all_vpes_local_irq_controller;
+			irq_set_handler(virq, handle_percpu_irq);
+			break;
+
+		default:
+			chip = &gic_local_irq_controller;
+			irq_set_handler(virq, handle_percpu_devid_irq);
+			irq_set_percpu_devid(virq);
+			break;
+		}
+
+		err = irq_domain_set_hwirq_and_chip(d, virq, hwirq,
+						    chip, NULL);
+	}
+
+	return err;
 }
 
 static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
@@ -743,15 +739,12 @@ static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
 	int cpu, ret, i;
 
 	if (spec->type == GIC_DEVICE) {
-		/* verify that it doesn't conflict with an IPI irq */
-		if (test_bit(spec->hwirq, ipi_resrv))
+		/* verify that shared irqs don't conflict with an IPI irq */
+		if ((spec->hwirq >= GIC_SHARED_HWIRQ_BASE) &&
+		    test_bit(GIC_HWIRQ_TO_SHARED(spec->hwirq), ipi_resrv))
 			return -EBUSY;
 
-		hwirq = GIC_SHARED_TO_HWIRQ(spec->hwirq);
-
-		return irq_domain_set_hwirq_and_chip(d, virq, hwirq,
-						     &gic_level_irq_controller,
-						     NULL);
+		return gic_setup_dev_chip(d, virq, spec->hwirq);
 	} else {
 		base_hwirq = find_first_bit(ipi_resrv, gic_shared_intrs);
 		if (base_hwirq == gic_shared_intrs) {
@@ -771,11 +764,13 @@ static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
 			hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i);
 
 			ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq,
-							    &gic_edge_irq_controller,
+							    &gic_level_irq_controller,
 							    NULL);
 			if (ret)
 				goto error;
 
+			irq_set_handler(virq + i, handle_level_irq);
+
 			ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu);
 			if (ret)
 				goto error;
@@ -818,7 +813,6 @@ int gic_irq_domain_match(struct irq_domain *d, struct device_node *node,
 }
 
 static const struct irq_domain_ops gic_irq_domain_ops = {
-	.map = gic_irq_domain_map,
 	.alloc = gic_irq_domain_alloc,
 	.free = gic_irq_domain_free,
 	.match = gic_irq_domain_match,
@@ -849,29 +843,20 @@ static int gic_dev_domain_alloc(struct irq_domain *d, unsigned int virq,
 	struct irq_fwspec *fwspec = arg;
 	struct gic_irq_spec spec = {
 		.type = GIC_DEVICE,
-		.hwirq = fwspec->param[1],
 	};
 	int i, ret;
-	bool is_shared = fwspec->param[0] == GIC_SHARED;
 
-	if (is_shared) {
-		ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
-		if (ret)
-			return ret;
-	}
-
-	for (i = 0; i < nr_irqs; i++) {
-		irq_hw_number_t hwirq;
+	if (fwspec->param[0] == GIC_SHARED)
+		spec.hwirq = GIC_SHARED_TO_HWIRQ(fwspec->param[1]);
+	else
+		spec.hwirq = GIC_LOCAL_TO_HWIRQ(fwspec->param[1]);
 
-		if (is_shared)
-			hwirq = GIC_SHARED_TO_HWIRQ(spec.hwirq + i);
-		else
-			hwirq = GIC_LOCAL_TO_HWIRQ(spec.hwirq + i);
+	ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
+	if (ret)
+		return ret;
 
-		ret = irq_domain_set_hwirq_and_chip(d, virq + i,
-						    hwirq,
-						    &gic_level_irq_controller,
-						    NULL);
+	for (i = 0; i < nr_irqs; i++) {
+		ret = gic_setup_dev_chip(d, virq + i, spec.hwirq + i);
 		if (ret)
 			goto error;
 	}
@@ -890,10 +875,20 @@ void gic_dev_domain_free(struct irq_domain *d, unsigned int virq,
 	return;
 }
 
+static void gic_dev_domain_activate(struct irq_domain *domain,
+				    struct irq_data *d)
+{
+	if (GIC_HWIRQ_TO_LOCAL(d->hwirq) < GIC_NUM_LOCAL_INTRS)
+		gic_local_irq_domain_map(domain, d->irq, d->hwirq);
+	else
+		gic_shared_irq_domain_map(domain, d->irq, d->hwirq, 0);
+}
+
 static struct irq_domain_ops gic_dev_domain_ops = {
 	.xlate = gic_dev_domain_xlate,
 	.alloc = gic_dev_domain_alloc,
 	.free = gic_dev_domain_free,
+	.activate = gic_dev_domain_activate,
 };
 
 static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
diff --git a/drivers/irqchip/irq-mvebu-pic.c b/drivers/irqchip/irq-mvebu-pic.c
new file mode 100644
index 000000000000..eec63951129a
--- /dev/null
+++ b/drivers/irqchip/irq-mvebu-pic.c
@@ -0,0 +1,197 @@
+/*
+ * Copyright (C) 2016 Marvell
+ *
+ * Yehuda Yitschak <yehuday@marvell.com>
+ * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2.  This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/module.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+
+#define PIC_CAUSE	       0x0
+#define PIC_MASK	       0x4
+
+#define PIC_MAX_IRQS		32
+#define PIC_MAX_IRQ_MASK	((1UL << PIC_MAX_IRQS) - 1)
+
+struct mvebu_pic {
+	void __iomem *base;
+	u32 parent_irq;
+	struct irq_domain *domain;
+	struct irq_chip irq_chip;
+};
+
+static void mvebu_pic_reset(struct mvebu_pic *pic)
+{
+	/* ACK and mask all interrupts */
+	writel(0, pic->base + PIC_MASK);
+	writel(PIC_MAX_IRQ_MASK, pic->base + PIC_CAUSE);
+}
+
+static void mvebu_pic_eoi_irq(struct irq_data *d)
+{
+	struct mvebu_pic *pic = irq_data_get_irq_chip_data(d);
+
+	writel(1 << d->hwirq, pic->base + PIC_CAUSE);
+}
+
+static void mvebu_pic_mask_irq(struct irq_data *d)
+{
+	struct mvebu_pic *pic = irq_data_get_irq_chip_data(d);
+	u32 reg;
+
+	reg =  readl(pic->base + PIC_MASK);
+	reg |= (1 << d->hwirq);
+	writel(reg, pic->base + PIC_MASK);
+}
+
+static void mvebu_pic_unmask_irq(struct irq_data *d)
+{
+	struct mvebu_pic *pic = irq_data_get_irq_chip_data(d);
+	u32 reg;
+
+	reg = readl(pic->base + PIC_MASK);
+	reg &= ~(1 << d->hwirq);
+	writel(reg, pic->base + PIC_MASK);
+}
+
+static int mvebu_pic_irq_map(struct irq_domain *domain, unsigned int virq,
+			     irq_hw_number_t hwirq)
+{
+	struct mvebu_pic *pic = domain->host_data;
+
+	irq_set_percpu_devid(virq);
+	irq_set_chip_data(virq, pic);
+	irq_set_chip_and_handler(virq, &pic->irq_chip,
+				 handle_percpu_devid_irq);
+	irq_set_status_flags(virq, IRQ_LEVEL);
+	irq_set_probe(virq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops mvebu_pic_domain_ops = {
+	.map = mvebu_pic_irq_map,
+	.xlate = irq_domain_xlate_onecell,
+};
+
+static void mvebu_pic_handle_cascade_irq(struct irq_desc *desc)
+{
+	struct mvebu_pic *pic = irq_desc_get_handler_data(desc);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	unsigned long irqmap, irqn;
+	unsigned int cascade_irq;
+
+	irqmap = readl_relaxed(pic->base + PIC_CAUSE);
+	chained_irq_enter(chip, desc);
+
+	for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) {
+		cascade_irq = irq_find_mapping(pic->domain, irqn);
+		generic_handle_irq(cascade_irq);
+	}
+
+	chained_irq_exit(chip, desc);
+}
+
+static void mvebu_pic_enable_percpu_irq(void *data)
+{
+	struct mvebu_pic *pic = data;
+
+	mvebu_pic_reset(pic);
+	enable_percpu_irq(pic->parent_irq, IRQ_TYPE_NONE);
+}
+
+static void mvebu_pic_disable_percpu_irq(void *data)
+{
+	struct mvebu_pic *pic = data;
+
+	disable_percpu_irq(pic->parent_irq);
+}
+
+static int mvebu_pic_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct mvebu_pic *pic;
+	struct irq_chip *irq_chip;
+	struct resource *res;
+
+	pic = devm_kzalloc(&pdev->dev, sizeof(struct mvebu_pic), GFP_KERNEL);
+	if (!pic)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	pic->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pic->base))
+		return PTR_ERR(pic->base);
+
+	irq_chip = &pic->irq_chip;
+	irq_chip->name = dev_name(&pdev->dev);
+	irq_chip->irq_mask = mvebu_pic_mask_irq;
+	irq_chip->irq_unmask = mvebu_pic_unmask_irq;
+	irq_chip->irq_eoi = mvebu_pic_eoi_irq;
+
+	pic->parent_irq = irq_of_parse_and_map(node, 0);
+	if (pic->parent_irq <= 0) {
+		dev_err(&pdev->dev, "Failed to parse parent interrupt\n");
+		return -EINVAL;
+	}
+
+	pic->domain = irq_domain_add_linear(node, PIC_MAX_IRQS,
+					    &mvebu_pic_domain_ops, pic);
+	if (!pic->domain) {
+		dev_err(&pdev->dev, "Failed to allocate irq domain\n");
+		return -ENOMEM;
+	}
+
+	irq_set_chained_handler(pic->parent_irq, mvebu_pic_handle_cascade_irq);
+	irq_set_handler_data(pic->parent_irq, pic);
+
+	on_each_cpu(mvebu_pic_enable_percpu_irq, pic, 1);
+
+	platform_set_drvdata(pdev, pic);
+
+	return 0;
+}
+
+static int mvebu_pic_remove(struct platform_device *pdev)
+{
+	struct mvebu_pic *pic = platform_get_drvdata(pdev);
+
+	on_each_cpu(mvebu_pic_disable_percpu_irq, pic, 1);
+	irq_domain_remove(pic->domain);
+
+	return 0;
+}
+
+static const struct of_device_id mvebu_pic_of_match[] = {
+	{ .compatible = "marvell,armada-8k-pic", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, mvebu_pic_of_match);
+
+static struct platform_driver mvebu_pic_driver = {
+	.probe  = mvebu_pic_probe,
+	.remove = mvebu_pic_remove,
+	.driver = {
+		.name = "mvebu-pic",
+		.of_match_table = mvebu_pic_of_match,
+	},
+};
+module_platform_driver(mvebu_pic_driver);
+
+MODULE_AUTHOR("Yehuda Yitschak <yehuday@marvell.com>");
+MODULE_AUTHOR("Thomas Petazzoni <thomas.petazzoni@free-electrons.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:mvebu_pic");
+
diff --git a/drivers/irqchip/irq-stm32-exti.c b/drivers/irqchip/irq-stm32-exti.c
new file mode 100644
index 000000000000..491568c95aa5
--- /dev/null
+++ b/drivers/irqchip/irq-stm32-exti.c
@@ -0,0 +1,201 @@
+/*
+ * Copyright (C) Maxime Coquelin 2015
+ * Author:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+
+#define EXTI_IMR	0x0
+#define EXTI_EMR	0x4
+#define EXTI_RTSR	0x8
+#define EXTI_FTSR	0xc
+#define EXTI_SWIER	0x10
+#define EXTI_PR		0x14
+
+static void stm32_irq_handler(struct irq_desc *desc)
+{
+	struct irq_domain *domain = irq_desc_get_handler_data(desc);
+	struct irq_chip_generic *gc = domain->gc->gc[0];
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	unsigned long pending;
+	int n;
+
+	chained_irq_enter(chip, desc);
+
+	while ((pending = irq_reg_readl(gc, EXTI_PR))) {
+		for_each_set_bit(n, &pending, BITS_PER_LONG) {
+			generic_handle_irq(irq_find_mapping(domain, n));
+			irq_reg_writel(gc, BIT(n), EXTI_PR);
+		}
+	}
+
+	chained_irq_exit(chip, desc);
+}
+
+static int stm32_irq_set_type(struct irq_data *data, unsigned int type)
+{
+	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(data);
+	int pin = data->hwirq;
+	u32 rtsr, ftsr;
+
+	irq_gc_lock(gc);
+
+	rtsr = irq_reg_readl(gc, EXTI_RTSR);
+	ftsr = irq_reg_readl(gc, EXTI_FTSR);
+
+	switch (type) {
+	case IRQ_TYPE_EDGE_RISING:
+		rtsr |= BIT(pin);
+		ftsr &= ~BIT(pin);
+		break;
+	case IRQ_TYPE_EDGE_FALLING:
+		rtsr &= ~BIT(pin);
+		ftsr |= BIT(pin);
+		break;
+	case IRQ_TYPE_EDGE_BOTH:
+		rtsr |= BIT(pin);
+		ftsr |= BIT(pin);
+		break;
+	default:
+		irq_gc_unlock(gc);
+		return -EINVAL;
+	}
+
+	irq_reg_writel(gc, rtsr, EXTI_RTSR);
+	irq_reg_writel(gc, ftsr, EXTI_FTSR);
+
+	irq_gc_unlock(gc);
+
+	return 0;
+}
+
+static int stm32_irq_set_wake(struct irq_data *data, unsigned int on)
+{
+	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(data);
+	int pin = data->hwirq;
+	u32 emr;
+
+	irq_gc_lock(gc);
+
+	emr = irq_reg_readl(gc, EXTI_EMR);
+	if (on)
+		emr |= BIT(pin);
+	else
+		emr &= ~BIT(pin);
+	irq_reg_writel(gc, emr, EXTI_EMR);
+
+	irq_gc_unlock(gc);
+
+	return 0;
+}
+
+static int stm32_exti_alloc(struct irq_domain *d, unsigned int virq,
+			    unsigned int nr_irqs, void *data)
+{
+	struct irq_chip_generic *gc = d->gc->gc[0];
+	struct irq_fwspec *fwspec = data;
+	irq_hw_number_t hwirq;
+
+	hwirq = fwspec->param[0];
+
+	irq_map_generic_chip(d, virq, hwirq);
+	irq_domain_set_info(d, virq, hwirq, &gc->chip_types->chip, gc,
+			    handle_simple_irq, NULL, NULL);
+
+	return 0;
+}
+
+static void stm32_exti_free(struct irq_domain *d, unsigned int virq,
+			    unsigned int nr_irqs)
+{
+	struct irq_data *data = irq_domain_get_irq_data(d, virq);
+
+	irq_domain_reset_irq_data(data);
+}
+
+struct irq_domain_ops irq_exti_domain_ops = {
+	.map	= irq_map_generic_chip,
+	.xlate	= irq_domain_xlate_onetwocell,
+	.alloc  = stm32_exti_alloc,
+	.free	= stm32_exti_free,
+};
+
+static int __init stm32_exti_init(struct device_node *node,
+				  struct device_node *parent)
+{
+	unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN;
+	int nr_irqs, nr_exti, ret, i;
+	struct irq_chip_generic *gc;
+	struct irq_domain *domain;
+	void *base;
+
+	base = of_iomap(node, 0);
+	if (!base) {
+		pr_err("%s: Unable to map registers\n", node->full_name);
+		return -ENOMEM;
+	}
+
+	/* Determine number of irqs supported */
+	writel_relaxed(~0UL, base + EXTI_RTSR);
+	nr_exti = fls(readl_relaxed(base + EXTI_RTSR));
+	writel_relaxed(0, base + EXTI_RTSR);
+
+	pr_info("%s: %d External IRQs detected\n", node->full_name, nr_exti);
+
+	domain = irq_domain_add_linear(node, nr_exti,
+				       &irq_exti_domain_ops, NULL);
+	if (!domain) {
+		pr_err("%s: Could not register interrupt domain.\n",
+				node->name);
+		ret = -ENOMEM;
+		goto out_unmap;
+	}
+
+	ret = irq_alloc_domain_generic_chips(domain, nr_exti, 1, "exti",
+					     handle_edge_irq, clr, 0, 0);
+	if (ret) {
+		pr_err("%s: Could not allocate generic interrupt chip.\n",
+			node->full_name);
+		goto out_free_domain;
+	}
+
+	gc = domain->gc->gc[0];
+	gc->reg_base                         = base;
+	gc->chip_types->type               = IRQ_TYPE_EDGE_BOTH;
+	gc->chip_types->chip.name          = gc->chip_types[0].chip.name;
+	gc->chip_types->chip.irq_ack       = irq_gc_ack_set_bit;
+	gc->chip_types->chip.irq_mask      = irq_gc_mask_clr_bit;
+	gc->chip_types->chip.irq_unmask    = irq_gc_mask_set_bit;
+	gc->chip_types->chip.irq_set_type  = stm32_irq_set_type;
+	gc->chip_types->chip.irq_set_wake  = stm32_irq_set_wake;
+	gc->chip_types->regs.ack           = EXTI_PR;
+	gc->chip_types->regs.mask          = EXTI_IMR;
+	gc->chip_types->handler            = handle_edge_irq;
+
+	nr_irqs = of_irq_count(node);
+	for (i = 0; i < nr_irqs; i++) {
+		unsigned int irq = irq_of_parse_and_map(node, i);
+
+		irq_set_handler_data(irq, domain);
+		irq_set_chained_handler(irq, stm32_irq_handler);
+	}
+
+	return 0;
+
+out_free_domain:
+	irq_domain_remove(domain);
+out_unmap:
+	iounmap(base);
+	return ret;
+}
+
+IRQCHIP_DECLARE(stm32_exti, "st,stm32-exti", stm32_exti_init);
diff --git a/drivers/irqchip/irq-vic.c b/drivers/irqchip/irq-vic.c
index f811a7de5857..74192f62dd67 100644
--- a/drivers/irqchip/irq-vic.c
+++ b/drivers/irqchip/irq-vic.c
@@ -498,7 +498,6 @@ void __init vic_init(void __iomem *base, unsigned int irq_start,
  * vic_init_cascaded() - initialise a cascaded vectored interrupt controller
  * @base: iomem base address
  * @parent_irq: the parent IRQ we're cascaded off
- * @irq_start: starting interrupt number, must be muliple of 32
  * @vic_sources: bitmask of interrupt sources to allow
  * @resume_sources: bitmask of interrupt sources to allow for resume
  *
diff --git a/drivers/isdn/hardware/mISDN/avmfritz.c b/drivers/isdn/hardware/mISDN/avmfritz.c
index 292991c90c02..e3fa1cd64470 100644
--- a/drivers/isdn/hardware/mISDN/avmfritz.c
+++ b/drivers/isdn/hardware/mISDN/avmfritz.c
@@ -284,7 +284,7 @@ __write_ctrl_pciv2(struct fritzcard *fc, struct hdlc_hw *hdlc, u32 channel) {
 					  AVM_HDLC_STATUS_1));
 }
 
-void
+static void
 write_ctrl(struct bchannel *bch, int which) {
 	struct fritzcard *fc = bch->hw;
 	struct hdlc_hw *hdlc;
@@ -741,7 +741,7 @@ inithdlc(struct fritzcard *fc)
 	modehdlc(&fc->bch[1], -1);
 }
 
-void
+static void
 clear_pending_hdlc_ints(struct fritzcard *fc)
 {
 	u32 val;
@@ -962,7 +962,7 @@ avm_dctrl(struct mISDNchannel *ch, u32 cmd, void *arg)
 	return err;
 }
 
-int
+static int
 setup_fritz(struct fritzcard *fc)
 {
 	u32 val, ver;
diff --git a/drivers/isdn/hardware/mISDN/hfcmulti.c b/drivers/isdn/hardware/mISDN/hfcmulti.c
index 28543d795188..480c2d7794eb 100644
--- a/drivers/isdn/hardware/mISDN/hfcmulti.c
+++ b/drivers/isdn/hardware/mISDN/hfcmulti.c
@@ -564,19 +564,19 @@ disable_hwirq(struct hfc_multi *hc)
 #define	MAX_TDM_CHAN 32
 
 
-inline void
+static inline void
 enablepcibridge(struct hfc_multi *c)
 {
 	HFC_outb(c, R_BRG_PCM_CFG, (0x0 << 6) | 0x3); /* was _io before */
 }
 
-inline void
+static inline void
 disablepcibridge(struct hfc_multi *c)
 {
 	HFC_outb(c, R_BRG_PCM_CFG, (0x0 << 6) | 0x2); /* was _io before */
 }
 
-inline unsigned char
+static inline unsigned char
 readpcibridge(struct hfc_multi *hc, unsigned char address)
 {
 	unsigned short cipv;
@@ -604,7 +604,7 @@ readpcibridge(struct hfc_multi *hc, unsigned char address)
 	return data;
 }
 
-inline void
+static inline void
 writepcibridge(struct hfc_multi *hc, unsigned char address, unsigned char data)
 {
 	unsigned short cipv;
@@ -634,14 +634,14 @@ writepcibridge(struct hfc_multi *hc, unsigned char address, unsigned char data)
 	outl(datav, hc->pci_iobase);
 }
 
-inline void
+static inline void
 cpld_set_reg(struct hfc_multi *hc, unsigned char reg)
 {
 	/* Do data pin read low byte */
 	HFC_outb(hc, R_GPIO_OUT1, reg);
 }
 
-inline void
+static inline void
 cpld_write_reg(struct hfc_multi *hc, unsigned char reg, unsigned char val)
 {
 	cpld_set_reg(hc, reg);
@@ -653,7 +653,7 @@ cpld_write_reg(struct hfc_multi *hc, unsigned char reg, unsigned char val)
 	return;
 }
 
-inline unsigned char
+static inline unsigned char
 cpld_read_reg(struct hfc_multi *hc, unsigned char reg)
 {
 	unsigned char bytein;
@@ -670,14 +670,14 @@ cpld_read_reg(struct hfc_multi *hc, unsigned char reg)
 	return bytein;
 }
 
-inline void
+static inline void
 vpm_write_address(struct hfc_multi *hc, unsigned short addr)
 {
 	cpld_write_reg(hc, 0, 0xff & addr);
 	cpld_write_reg(hc, 1, 0x01 & (addr >> 8));
 }
 
-inline unsigned short
+static inline unsigned short
 vpm_read_address(struct hfc_multi *c)
 {
 	unsigned short addr;
@@ -691,7 +691,7 @@ vpm_read_address(struct hfc_multi *c)
 	return addr & 0x1ff;
 }
 
-inline unsigned char
+static inline unsigned char
 vpm_in(struct hfc_multi *c, int which, unsigned short addr)
 {
 	unsigned char res;
@@ -712,7 +712,7 @@ vpm_in(struct hfc_multi *c, int which, unsigned short addr)
 	return res;
 }
 
-inline void
+static inline void
 vpm_out(struct hfc_multi *c, int which, unsigned short addr,
 	unsigned char data)
 {
@@ -1024,7 +1024,7 @@ hfcmulti_resync(struct hfc_multi *locked, struct hfc_multi *newmaster, int rm)
 }
 
 /* This must be called AND hc must be locked irqsave!!! */
-inline void
+static inline void
 plxsd_checksync(struct hfc_multi *hc, int rm)
 {
 	if (hc->syncronized) {
diff --git a/drivers/isdn/hardware/mISDN/mISDNipac.c b/drivers/isdn/hardware/mISDN/mISDNipac.c
index aa9b6c3cadc1..8d338ba366d0 100644
--- a/drivers/isdn/hardware/mISDN/mISDNipac.c
+++ b/drivers/isdn/hardware/mISDN/mISDNipac.c
@@ -113,7 +113,7 @@ isac_ph_state_bh(struct dchannel *dch)
 	pr_debug("%s: TE newstate %x\n", isac->name, dch->state);
 }
 
-void
+static void
 isac_empty_fifo(struct isac_hw *isac, int count)
 {
 	u8 *ptr;
diff --git a/drivers/isdn/hardware/mISDN/w6692.c b/drivers/isdn/hardware/mISDN/w6692.c
index 741675525b53..3b067ea656bd 100644
--- a/drivers/isdn/hardware/mISDN/w6692.c
+++ b/drivers/isdn/hardware/mISDN/w6692.c
@@ -848,7 +848,7 @@ dbusy_timer_handler(struct dchannel *dch)
 	}
 }
 
-void initW6692(struct w6692_hw *card)
+static void initW6692(struct w6692_hw *card)
 {
 	u8	val;
 
diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig
index 9dcc9b13d495..7a628c6516f6 100644
--- a/drivers/leds/Kconfig
+++ b/drivers/leds/Kconfig
@@ -584,6 +584,18 @@ config LEDS_SEAD3
 	  This driver can also be built as a module. If so the module
 	  will be called leds-sead3.
 
+config LEDS_IS31FL319X
+	tristate "LED Support for ISSI IS31FL319x I2C LED controller family"
+	depends on LEDS_CLASS && I2C && OF
+	select REGMAP_I2C
+	help
+	  This option enables support for LEDs connected to ISSI IS31FL319x
+	  fancy LED driver chips accessed via the I2C bus.
+	  Driver supports individual PWM brightness control for each channel.
+
+	  This driver can also be built as a module. If so the module will be
+	  called leds-is31fl319x.
+
 config LEDS_IS31FL32XX
 	tristate "LED support for ISSI IS31FL32XX I2C LED controller family"
 	depends on LEDS_CLASS && I2C && OF
@@ -631,6 +643,22 @@ config LEDS_VERSATILE
 	  This option enabled support for the LEDs on the ARM Versatile
 	  and RealView boards. Say Y to enabled these.
 
+config LEDS_PM8058
+	tristate "LED Support for the Qualcomm PM8058 PMIC"
+	depends on MFD_PM8921_CORE
+	depends on LEDS_CLASS
+	help
+	  Choose this option if you want to use the LED drivers in
+	  the Qualcomm PM8058 PMIC.
+
+config LEDS_MLXCPLD
+	tristate "LED support for the Mellanox boards"
+	depends on X86_64 && DMI
+	depends on LEDS_CLASS
+	help
+	  This option enabled support for the LEDs on the Mellanox
+	  boards. Say Y to enabled these.
+
 comment "LED Triggers"
 source "drivers/leds/trigger/Kconfig"
 
diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile
index 0684c865a1c0..3965070190f5 100644
--- a/drivers/leds/Makefile
+++ b/drivers/leds/Makefile
@@ -67,7 +67,10 @@ obj-$(CONFIG_LEDS_MENF21BMC)		+= leds-menf21bmc.o
 obj-$(CONFIG_LEDS_KTD2692)		+= leds-ktd2692.o
 obj-$(CONFIG_LEDS_POWERNV)		+= leds-powernv.o
 obj-$(CONFIG_LEDS_SEAD3)		+= leds-sead3.o
+obj-$(CONFIG_LEDS_IS31FL319X)		+= leds-is31fl319x.o
 obj-$(CONFIG_LEDS_IS31FL32XX)		+= leds-is31fl32xx.o
+obj-$(CONFIG_LEDS_PM8058)		+= leds-pm8058.o
+obj-$(CONFIG_LEDS_MLXCPLD)		+= leds-mlxcpld.o
 
 # LED SPI Drivers
 obj-$(CONFIG_LEDS_DAC124S085)		+= leds-dac124s085.o
diff --git a/drivers/leds/led-triggers.c b/drivers/leds/led-triggers.c
index c92702a684ce..431123b048a2 100644
--- a/drivers/leds/led-triggers.c
+++ b/drivers/leds/led-triggers.c
@@ -11,7 +11,7 @@
  *
  */
 
-#include <linux/module.h>
+#include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/spinlock.h>
@@ -81,21 +81,23 @@ ssize_t led_trigger_show(struct device *dev, struct device_attribute *attr,
 	down_read(&led_cdev->trigger_lock);
 
 	if (!led_cdev->trigger)
-		len += sprintf(buf+len, "[none] ");
+		len += scnprintf(buf+len, PAGE_SIZE - len, "[none] ");
 	else
-		len += sprintf(buf+len, "none ");
+		len += scnprintf(buf+len, PAGE_SIZE - len, "none ");
 
 	list_for_each_entry(trig, &trigger_list, next_trig) {
 		if (led_cdev->trigger && !strcmp(led_cdev->trigger->name,
 							trig->name))
-			len += sprintf(buf+len, "[%s] ", trig->name);
+			len += scnprintf(buf+len, PAGE_SIZE - len, "[%s] ",
+					 trig->name);
 		else
-			len += sprintf(buf+len, "%s ", trig->name);
+			len += scnprintf(buf+len, PAGE_SIZE - len, "%s ",
+					 trig->name);
 	}
 	up_read(&led_cdev->trigger_lock);
 	up_read(&triggers_list_lock);
 
-	len += sprintf(len+buf, "\n");
+	len += scnprintf(len+buf, PAGE_SIZE - len, "\n");
 	return len;
 }
 EXPORT_SYMBOL_GPL(led_trigger_show);
@@ -108,6 +110,9 @@ void led_trigger_set(struct led_classdev *led_cdev, struct led_trigger *trig)
 	char *envp[2];
 	const char *name;
 
+	if (!led_cdev->trigger && !trig)
+		return;
+
 	name = trig ? trig->name : "none";
 	event = kasprintf(GFP_KERNEL, "TRIGGER=%s", name);
 
@@ -136,7 +141,9 @@ void led_trigger_set(struct led_classdev *led_cdev, struct led_trigger *trig)
 	if (event) {
 		envp[0] = event;
 		envp[1] = NULL;
-		kobject_uevent_env(&led_cdev->dev->kobj, KOBJ_CHANGE, envp);
+		if (kobject_uevent_env(&led_cdev->dev->kobj, KOBJ_CHANGE, envp))
+			dev_err(led_cdev->dev,
+				"%s: Error sending uevent\n", __func__);
 		kfree(event);
 	}
 }
@@ -357,7 +364,3 @@ void led_trigger_unregister_simple(struct led_trigger *trig)
 	kfree(trig);
 }
 EXPORT_SYMBOL_GPL(led_trigger_unregister_simple);
-
-MODULE_AUTHOR("Richard Purdie");
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("LED Triggers Core");
diff --git a/drivers/leds/leds-gpio.c b/drivers/leds/leds-gpio.c
index 9b991d46ed84..d400dcaf4d29 100644
--- a/drivers/leds/leds-gpio.c
+++ b/drivers/leds/leds-gpio.c
@@ -26,15 +26,19 @@ struct gpio_led_data {
 	struct gpio_desc *gpiod;
 	u8 can_sleep;
 	u8 blinking;
-	int (*platform_gpio_blink_set)(struct gpio_desc *desc, int state,
-			unsigned long *delay_on, unsigned long *delay_off);
+	gpio_blink_set_t platform_gpio_blink_set;
 };
 
+static inline struct gpio_led_data *
+			cdev_to_gpio_led_data(struct led_classdev *led_cdev)
+{
+	return container_of(led_cdev, struct gpio_led_data, cdev);
+}
+
 static void gpio_led_set(struct led_classdev *led_cdev,
 	enum led_brightness value)
 {
-	struct gpio_led_data *led_dat =
-		container_of(led_cdev, struct gpio_led_data, cdev);
+	struct gpio_led_data *led_dat = cdev_to_gpio_led_data(led_cdev);
 	int level;
 
 	if (value == LED_OFF)
@@ -64,8 +68,7 @@ static int gpio_led_set_blocking(struct led_classdev *led_cdev,
 static int gpio_blink_set(struct led_classdev *led_cdev,
 	unsigned long *delay_on, unsigned long *delay_off)
 {
-	struct gpio_led_data *led_dat =
-		container_of(led_cdev, struct gpio_led_data, cdev);
+	struct gpio_led_data *led_dat = cdev_to_gpio_led_data(led_cdev);
 
 	led_dat->blinking = 1;
 	return led_dat->platform_gpio_blink_set(led_dat->gpiod, GPIO_LED_BLINK,
@@ -74,8 +77,7 @@ static int gpio_blink_set(struct led_classdev *led_cdev,
 
 static int create_gpio_led(const struct gpio_led *template,
 	struct gpio_led_data *led_dat, struct device *parent,
-	int (*blink_set)(struct gpio_desc *, int, unsigned long *,
-			 unsigned long *))
+	gpio_blink_set_t blink_set)
 {
 	int ret, state;
 
@@ -120,10 +122,13 @@ static int create_gpio_led(const struct gpio_led *template,
 		led_dat->platform_gpio_blink_set = blink_set;
 		led_dat->cdev.blink_set = gpio_blink_set;
 	}
-	if (template->default_state == LEDS_GPIO_DEFSTATE_KEEP)
-		state = !!gpiod_get_value_cansleep(led_dat->gpiod);
-	else
+	if (template->default_state == LEDS_GPIO_DEFSTATE_KEEP) {
+		state = gpiod_get_value_cansleep(led_dat->gpiod);
+		if (state < 0)
+			return state;
+	} else {
 		state = (template->default_state == LEDS_GPIO_DEFSTATE_ON);
+	}
 	led_dat->cdev.brightness = state ? LED_FULL : LED_OFF;
 	if (!template->retain_state_suspended)
 		led_dat->cdev.flags |= LED_CORE_SUSPENDRESUME;
@@ -134,7 +139,7 @@ static int create_gpio_led(const struct gpio_led *template,
 	if (ret < 0)
 		return ret;
 
-	return led_classdev_register(parent, &led_dat->cdev);
+	return devm_led_classdev_register(parent, &led_dat->cdev);
 }
 
 struct gpio_leds_priv {
@@ -154,7 +159,6 @@ static struct gpio_leds_priv *gpio_leds_create(struct platform_device *pdev)
 	struct fwnode_handle *child;
 	struct gpio_leds_priv *priv;
 	int count, ret;
-	struct device_node *np;
 
 	count = device_get_child_node_count(dev);
 	if (!count)
@@ -168,26 +172,22 @@ static struct gpio_leds_priv *gpio_leds_create(struct platform_device *pdev)
 		struct gpio_led_data *led_dat = &priv->leds[priv->num_leds];
 		struct gpio_led led = {};
 		const char *state = NULL;
+		struct device_node *np = to_of_node(child);
 
 		led.gpiod = devm_get_gpiod_from_child(dev, NULL, child);
 		if (IS_ERR(led.gpiod)) {
 			fwnode_handle_put(child);
-			ret = PTR_ERR(led.gpiod);
-			goto err;
+			return ERR_CAST(led.gpiod);
 		}
 
-		np = to_of_node(child);
-
-		if (fwnode_property_present(child, "label")) {
-			fwnode_property_read_string(child, "label", &led.name);
-		} else {
-			if (IS_ENABLED(CONFIG_OF) && !led.name && np)
-				led.name = np->name;
-			if (!led.name) {
-				ret = -EINVAL;
-				goto err;
-			}
+		ret = fwnode_property_read_string(child, "label", &led.name);
+		if (ret && IS_ENABLED(CONFIG_OF) && np)
+			led.name = np->name;
+		if (!led.name) {
+			fwnode_handle_put(child);
+			return ERR_PTR(-EINVAL);
 		}
+
 		fwnode_property_read_string(child, "linux,default-trigger",
 					    &led.default_trigger);
 
@@ -209,18 +209,13 @@ static struct gpio_leds_priv *gpio_leds_create(struct platform_device *pdev)
 		ret = create_gpio_led(&led, led_dat, dev, NULL);
 		if (ret < 0) {
 			fwnode_handle_put(child);
-			goto err;
+			return ERR_PTR(ret);
 		}
 		led_dat->cdev.dev->of_node = np;
 		priv->num_leds++;
 	}
 
 	return priv;
-
-err:
-	for (count = priv->num_leds - 1; count >= 0; count--)
-		led_classdev_unregister(&priv->leds[count].cdev);
-	return ERR_PTR(ret);
 }
 
 static const struct of_device_id of_gpio_leds_match[] = {
@@ -248,13 +243,8 @@ static int gpio_led_probe(struct platform_device *pdev)
 			ret = create_gpio_led(&pdata->leds[i],
 					      &priv->leds[i],
 					      &pdev->dev, pdata->gpio_blink_set);
-			if (ret < 0) {
-				/* On failure: unwind the led creations */
-				for (i = i - 1; i >= 0; i--)
-					led_classdev_unregister(
-							&priv->leds[i].cdev);
+			if (ret < 0)
 				return ret;
-			}
 		}
 	} else {
 		priv = gpio_leds_create(pdev);
@@ -267,17 +257,6 @@ static int gpio_led_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int gpio_led_remove(struct platform_device *pdev)
-{
-	struct gpio_leds_priv *priv = platform_get_drvdata(pdev);
-	int i;
-
-	for (i = 0; i < priv->num_leds; i++)
-		led_classdev_unregister(&priv->leds[i].cdev);
-
-	return 0;
-}
-
 static void gpio_led_shutdown(struct platform_device *pdev)
 {
 	struct gpio_leds_priv *priv = platform_get_drvdata(pdev);
@@ -292,7 +271,6 @@ static void gpio_led_shutdown(struct platform_device *pdev)
 
 static struct platform_driver gpio_led_driver = {
 	.probe		= gpio_led_probe,
-	.remove		= gpio_led_remove,
 	.shutdown	= gpio_led_shutdown,
 	.driver		= {
 		.name	= "leds-gpio",
diff --git a/drivers/leds/leds-is31fl319x.c b/drivers/leds/leds-is31fl319x.c
new file mode 100644
index 000000000000..f123309597e4
--- /dev/null
+++ b/drivers/leds/leds-is31fl319x.c
@@ -0,0 +1,450 @@
+/*
+ * Copyright 2015-16 Golden Delicious Computers
+ *
+ * Author: Nikolaus Schaller <hns@goldelico.com>
+ *
+ * This file is subject to the terms and conditions of version 2 of
+ * the GNU General Public License.  See the file COPYING in the main
+ * directory of this archive for more details.
+ *
+ * LED driver for the IS31FL319{0,1,3,6,9} to drive 1, 3, 6 or 9 light
+ * effect LEDs.
+ *
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+/* register numbers */
+#define IS31FL319X_SHUTDOWN		0x00
+#define IS31FL319X_CTRL1		0x01
+#define IS31FL319X_CTRL2		0x02
+#define IS31FL319X_CONFIG1		0x03
+#define IS31FL319X_CONFIG2		0x04
+#define IS31FL319X_RAMP_MODE		0x05
+#define IS31FL319X_BREATH_MASK		0x06
+#define IS31FL319X_PWM(channel)		(0x07 + channel)
+#define IS31FL319X_DATA_UPDATE		0x10
+#define IS31FL319X_T0(channel)		(0x11 + channel)
+#define IS31FL319X_T123_1		0x1a
+#define IS31FL319X_T123_2		0x1b
+#define IS31FL319X_T123_3		0x1c
+#define IS31FL319X_T4(channel)		(0x1d + channel)
+#define IS31FL319X_TIME_UPDATE		0x26
+#define IS31FL319X_RESET		0xff
+
+#define IS31FL319X_REG_CNT		(IS31FL319X_RESET + 1)
+
+#define IS31FL319X_MAX_LEDS		9
+
+/* CS (Current Setting) in CONFIG2 register */
+#define IS31FL319X_CONFIG2_CS_SHIFT	4
+#define IS31FL319X_CONFIG2_CS_MASK	0x7
+#define IS31FL319X_CONFIG2_CS_STEP_REF	12
+
+#define IS31FL319X_CURRENT_MIN		((u32)5000)
+#define IS31FL319X_CURRENT_MAX		((u32)40000)
+#define IS31FL319X_CURRENT_STEP		((u32)5000)
+#define IS31FL319X_CURRENT_DEFAULT	((u32)20000)
+
+/* Audio gain in CONFIG2 register */
+#define IS31FL319X_AUDIO_GAIN_DB_MAX	((u32)21)
+#define IS31FL319X_AUDIO_GAIN_DB_STEP	((u32)3)
+
+/*
+ * regmap is used as a cache of chip's register space,
+ * to avoid reading back brightness values from chip,
+ * which is known to hang.
+ */
+struct is31fl319x_chip {
+	const struct is31fl319x_chipdef *cdef;
+	struct i2c_client               *client;
+	struct regmap                   *regmap;
+	struct mutex                    lock;
+	u32                             audio_gain_db;
+
+	struct is31fl319x_led {
+		struct is31fl319x_chip  *chip;
+		struct led_classdev     cdev;
+		u32                     max_microamp;
+		bool                    configured;
+	} leds[IS31FL319X_MAX_LEDS];
+};
+
+struct is31fl319x_chipdef {
+	int num_leds;
+};
+
+static const struct is31fl319x_chipdef is31fl3190_cdef = {
+	.num_leds = 1,
+};
+
+static const struct is31fl319x_chipdef is31fl3193_cdef = {
+	.num_leds = 3,
+};
+
+static const struct is31fl319x_chipdef is31fl3196_cdef = {
+	.num_leds = 6,
+};
+
+static const struct is31fl319x_chipdef is31fl3199_cdef = {
+	.num_leds = 9,
+};
+
+static const struct of_device_id of_is31fl319x_match[] = {
+	{ .compatible = "issi,is31fl3190", .data = &is31fl3190_cdef, },
+	{ .compatible = "issi,is31fl3191", .data = &is31fl3190_cdef, },
+	{ .compatible = "issi,is31fl3193", .data = &is31fl3193_cdef, },
+	{ .compatible = "issi,is31fl3196", .data = &is31fl3196_cdef, },
+	{ .compatible = "issi,is31fl3199", .data = &is31fl3199_cdef, },
+	{ .compatible = "si-en,sn3199",    .data = &is31fl3199_cdef, },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, of_is31fl319x_match);
+
+static int is31fl319x_brightness_set(struct led_classdev *cdev,
+				     enum led_brightness brightness)
+{
+	struct is31fl319x_led *led = container_of(cdev, struct is31fl319x_led,
+						  cdev);
+	struct is31fl319x_chip *is31 = led->chip;
+	int chan = led - is31->leds;
+	int ret;
+	int i;
+	u8 ctrl1 = 0, ctrl2 = 0;
+
+	dev_dbg(&is31->client->dev, "%s %d: %d\n", __func__, chan, brightness);
+
+	mutex_lock(&is31->lock);
+
+	/* update PWM register */
+	ret = regmap_write(is31->regmap, IS31FL319X_PWM(chan), brightness);
+	if (ret < 0)
+		goto out;
+
+	/* read current brightness of all PWM channels */
+	for (i = 0; i < is31->cdef->num_leds; i++) {
+		unsigned int pwm_value;
+		bool on;
+
+		/*
+		 * since neither cdev nor the chip can provide
+		 * the current setting, we read from the regmap cache
+		 */
+
+		ret = regmap_read(is31->regmap, IS31FL319X_PWM(i), &pwm_value);
+		dev_dbg(&is31->client->dev, "%s read %d: ret=%d: %d\n",
+			__func__, i, ret, pwm_value);
+		on = ret >= 0 && pwm_value > LED_OFF;
+
+		if (i < 3)
+			ctrl1 |= on << i;       /* 0..2 => bit 0..2 */
+		else if (i < 6)
+			ctrl1 |= on << (i + 1); /* 3..5 => bit 4..6 */
+		else
+			ctrl2 |= on << (i - 6); /* 6..8 => bit 0..2 */
+	}
+
+	if (ctrl1 > 0 || ctrl2 > 0) {
+		dev_dbg(&is31->client->dev, "power up %02x %02x\n",
+			ctrl1, ctrl2);
+		regmap_write(is31->regmap, IS31FL319X_CTRL1, ctrl1);
+		regmap_write(is31->regmap, IS31FL319X_CTRL2, ctrl2);
+		/* update PWMs */
+		regmap_write(is31->regmap, IS31FL319X_DATA_UPDATE, 0x00);
+		/* enable chip from shut down */
+		ret = regmap_write(is31->regmap, IS31FL319X_SHUTDOWN, 0x01);
+	} else {
+		dev_dbg(&is31->client->dev, "power down\n");
+		/* shut down (no need to clear CTRL1/2) */
+		ret = regmap_write(is31->regmap, IS31FL319X_SHUTDOWN, 0x00);
+	}
+
+out:
+	mutex_unlock(&is31->lock);
+
+	return ret;
+}
+
+static int is31fl319x_parse_child_dt(const struct device *dev,
+				     const struct device_node *child,
+				     struct is31fl319x_led *led)
+{
+	struct led_classdev *cdev = &led->cdev;
+	int ret;
+
+	if (of_property_read_string(child, "label", &cdev->name))
+		cdev->name = child->name;
+
+	ret = of_property_read_string(child, "linux,default-trigger",
+				      &cdev->default_trigger);
+	if (ret < 0 && ret != -EINVAL) /* is optional */
+		return ret;
+
+	led->max_microamp = IS31FL319X_CURRENT_DEFAULT;
+	ret = of_property_read_u32(child, "led-max-microamp",
+				   &led->max_microamp);
+	if (!ret) {
+		if (led->max_microamp < IS31FL319X_CURRENT_MIN)
+			return -EINVAL;	/* not supported */
+		led->max_microamp = min(led->max_microamp,
+					  IS31FL319X_CURRENT_MAX);
+	}
+
+	return 0;
+}
+
+static int is31fl319x_parse_dt(struct device *dev,
+			       struct is31fl319x_chip *is31)
+{
+	struct device_node *np = dev->of_node, *child;
+	const struct of_device_id *of_dev_id;
+	int count;
+	int ret;
+
+	if (!np)
+		return -ENODEV;
+
+	of_dev_id = of_match_device(of_is31fl319x_match, dev);
+	if (!of_dev_id) {
+		dev_err(dev, "Failed to match device with supported chips\n");
+		return -EINVAL;
+	}
+
+	is31->cdef = of_dev_id->data;
+
+	count = of_get_child_count(np);
+
+	dev_dbg(dev, "probe %s with %d leds defined in DT\n",
+		of_dev_id->compatible, count);
+
+	if (!count || count > is31->cdef->num_leds) {
+		dev_err(dev, "Number of leds defined must be between 1 and %u\n",
+			is31->cdef->num_leds);
+		return -ENODEV;
+	}
+
+	for_each_child_of_node(np, child) {
+		struct is31fl319x_led *led;
+		u32 reg;
+
+		ret = of_property_read_u32(child, "reg", &reg);
+		if (ret) {
+			dev_err(dev, "Failed to read led 'reg' property\n");
+			goto put_child_node;
+		}
+
+		if (reg < 1 || reg > is31->cdef->num_leds) {
+			dev_err(dev, "invalid led reg %u\n", reg);
+			ret = -EINVAL;
+			goto put_child_node;
+		}
+
+		led = &is31->leds[reg - 1];
+
+		if (led->configured) {
+			dev_err(dev, "led %u is already configured\n", reg);
+			ret = -EINVAL;
+			goto put_child_node;
+		}
+
+		ret = is31fl319x_parse_child_dt(dev, child, led);
+		if (ret) {
+			dev_err(dev, "led %u DT parsing failed\n", reg);
+			goto put_child_node;
+		}
+
+		led->configured = true;
+	}
+
+	is31->audio_gain_db = 0;
+	ret = of_property_read_u32(np, "audio-gain-db", &is31->audio_gain_db);
+	if (!ret)
+		is31->audio_gain_db = min(is31->audio_gain_db,
+					  IS31FL319X_AUDIO_GAIN_DB_MAX);
+
+	return 0;
+
+put_child_node:
+	of_node_put(child);
+	return ret;
+}
+
+static bool is31fl319x_readable_reg(struct device *dev, unsigned int reg)
+{ /* we have no readable registers */
+	return false;
+}
+
+static bool is31fl319x_volatile_reg(struct device *dev, unsigned int reg)
+{ /* volatile registers are not cached */
+	switch (reg) {
+	case IS31FL319X_DATA_UPDATE:
+	case IS31FL319X_TIME_UPDATE:
+	case IS31FL319X_RESET:
+		return true; /* always write-through */
+	default:
+		return false;
+	}
+}
+
+static const struct reg_default is31fl319x_reg_defaults[] = {
+	{ IS31FL319X_CONFIG1, 0x00},
+	{ IS31FL319X_CONFIG2, 0x00},
+	{ IS31FL319X_PWM(0), 0x00},
+	{ IS31FL319X_PWM(1), 0x00},
+	{ IS31FL319X_PWM(2), 0x00},
+	{ IS31FL319X_PWM(3), 0x00},
+	{ IS31FL319X_PWM(4), 0x00},
+	{ IS31FL319X_PWM(5), 0x00},
+	{ IS31FL319X_PWM(6), 0x00},
+	{ IS31FL319X_PWM(7), 0x00},
+	{ IS31FL319X_PWM(8), 0x00},
+};
+
+static struct regmap_config regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = IS31FL319X_REG_CNT,
+	.cache_type = REGCACHE_FLAT,
+	.readable_reg = is31fl319x_readable_reg,
+	.volatile_reg = is31fl319x_volatile_reg,
+	.reg_defaults = is31fl319x_reg_defaults,
+	.num_reg_defaults = ARRAY_SIZE(is31fl319x_reg_defaults),
+};
+
+static inline int is31fl319x_microamp_to_cs(struct device *dev, u32 microamp)
+{ /* round down to nearest supported value (range check done by caller) */
+	u32 step = microamp / IS31FL319X_CURRENT_STEP;
+
+	return ((IS31FL319X_CONFIG2_CS_STEP_REF - step) &
+		IS31FL319X_CONFIG2_CS_MASK) <<
+		IS31FL319X_CONFIG2_CS_SHIFT; /* CS encoding */
+}
+
+static inline int is31fl319x_db_to_gain(u32 dezibel)
+{ /* round down to nearest supported value (range check done by caller) */
+	return dezibel / IS31FL319X_AUDIO_GAIN_DB_STEP;
+}
+
+static int is31fl319x_probe(struct i2c_client *client,
+			    const struct i2c_device_id *id)
+{
+	struct is31fl319x_chip *is31;
+	struct device *dev = &client->dev;
+	struct i2c_adapter *adapter = to_i2c_adapter(dev->parent);
+	int err;
+	int i = 0;
+	u32 aggregated_led_microamp = IS31FL319X_CURRENT_MAX;
+
+	if (!i2c_check_functionality(adapter, I2C_FUNC_I2C))
+		return -EIO;
+
+	is31 = devm_kzalloc(&client->dev, sizeof(*is31), GFP_KERNEL);
+	if (!is31)
+		return -ENOMEM;
+
+	mutex_init(&is31->lock);
+
+	err = is31fl319x_parse_dt(&client->dev, is31);
+	if (err)
+		goto free_mutex;
+
+	is31->client = client;
+	is31->regmap = devm_regmap_init_i2c(client, &regmap_config);
+	if (IS_ERR(is31->regmap)) {
+		dev_err(&client->dev, "failed to allocate register map\n");
+		err = PTR_ERR(is31->regmap);
+		goto free_mutex;
+	}
+
+	i2c_set_clientdata(client, is31);
+
+	/* check for write-reply from chip (we can't read any registers) */
+	err = regmap_write(is31->regmap, IS31FL319X_RESET, 0x00);
+	if (err < 0) {
+		dev_err(&client->dev, "no response from chip write: err = %d\n",
+			err);
+		err = -EIO; /* does not answer */
+		goto free_mutex;
+	}
+
+	/*
+	 * Kernel conventions require per-LED led-max-microamp property.
+	 * But the chip does not allow to limit individual LEDs.
+	 * So we take minimum from all subnodes for safety of hardware.
+	 */
+	for (i = 0; i < is31->cdef->num_leds; i++)
+		if (is31->leds[i].configured &&
+		    is31->leds[i].max_microamp < aggregated_led_microamp)
+			aggregated_led_microamp = is31->leds[i].max_microamp;
+
+	regmap_write(is31->regmap, IS31FL319X_CONFIG2,
+		     is31fl319x_microamp_to_cs(dev, aggregated_led_microamp) |
+		     is31fl319x_db_to_gain(is31->audio_gain_db));
+
+	for (i = 0; i < is31->cdef->num_leds; i++) {
+		struct is31fl319x_led *led = &is31->leds[i];
+
+		if (!led->configured)
+			continue;
+
+		led->chip = is31;
+		led->cdev.brightness_set_blocking = is31fl319x_brightness_set;
+
+		err = devm_led_classdev_register(&client->dev, &led->cdev);
+		if (err < 0)
+			goto free_mutex;
+	}
+
+	return 0;
+
+free_mutex:
+	mutex_destroy(&is31->lock);
+	return err;
+}
+
+static int is31fl319x_remove(struct i2c_client *client)
+{
+	struct is31fl319x_chip *is31 = i2c_get_clientdata(client);
+
+	mutex_destroy(&is31->lock);
+	return 0;
+}
+
+/*
+ * i2c-core (and modalias) requires that id_table be properly filled,
+ * even though it is not used for DeviceTree based instantiation.
+ */
+static const struct i2c_device_id is31fl319x_id[] = {
+	{ "is31fl3190" },
+	{ "is31fl3191" },
+	{ "is31fl3193" },
+	{ "is31fl3196" },
+	{ "is31fl3199" },
+	{ "sn3199" },
+	{},
+};
+MODULE_DEVICE_TABLE(i2c, is31fl319x_id);
+
+static struct i2c_driver is31fl319x_driver = {
+	.driver   = {
+		.name           = "leds-is31fl319x",
+		.of_match_table = of_match_ptr(of_is31fl319x_match),
+	},
+	.probe    = is31fl319x_probe,
+	.remove   = is31fl319x_remove,
+	.id_table = is31fl319x_id,
+};
+
+module_i2c_driver(is31fl319x_driver);
+
+MODULE_AUTHOR("H. Nikolaus Schaller <hns@goldelico.com>");
+MODULE_AUTHOR("Andrey Utkin <andrey_utkin@fastmail.com>");
+MODULE_DESCRIPTION("IS31FL319X LED driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/leds-mlxcpld.c b/drivers/leds/leds-mlxcpld.c
new file mode 100644
index 000000000000..197ab9b29a9c
--- /dev/null
+++ b/drivers/leds/leds-mlxcpld.c
@@ -0,0 +1,430 @@
+/*
+ * drivers/leds/leds-mlxcpld.c
+ * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2016 Vadim Pasternak <vadimp@mellanox.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/acpi.h>
+#include <linux/device.h>
+#include <linux/dmi.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/io.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define MLXPLAT_CPLD_LPC_REG_BASE_ADRR     0x2500 /* LPC bus access */
+
+/* Color codes for LEDs */
+#define MLXCPLD_LED_OFFSET_HALF		0x01 /* Offset from solid: 3Hz blink */
+#define MLXCPLD_LED_OFFSET_FULL		0x02 /* Offset from solid: 6Hz blink */
+#define MLXCPLD_LED_IS_OFF		0x00 /* Off */
+#define MLXCPLD_LED_RED_STATIC_ON	0x05 /* Solid red */
+#define MLXCPLD_LED_RED_BLINK_HALF	(MLXCPLD_LED_RED_STATIC_ON + \
+					 MLXCPLD_LED_OFFSET_HALF)
+#define MLXCPLD_LED_RED_BLINK_FULL	(MLXCPLD_LED_RED_STATIC_ON + \
+					 MLXCPLD_LED_OFFSET_FULL)
+#define MLXCPLD_LED_GREEN_STATIC_ON	0x0D /* Solid green */
+#define MLXCPLD_LED_GREEN_BLINK_HALF	(MLXCPLD_LED_GREEN_STATIC_ON + \
+					 MLXCPLD_LED_OFFSET_HALF)
+#define MLXCPLD_LED_GREEN_BLINK_FULL	(MLXCPLD_LED_GREEN_STATIC_ON + \
+					 MLXCPLD_LED_OFFSET_FULL)
+#define MLXCPLD_LED_BLINK_3HZ		167 /* ~167 msec off/on */
+#define MLXCPLD_LED_BLINK_6HZ		83 /* ~83 msec off/on */
+
+/**
+ * mlxcpld_param - LED access parameters:
+ * @offset - offset for LED access in CPLD device
+ * @mask - mask for LED access in CPLD device
+ * @base_color - base color code for LED
+**/
+struct mlxcpld_param {
+	u8 offset;
+	u8 mask;
+	u8 base_color;
+};
+
+/**
+ * mlxcpld_led_priv - LED private data:
+ * @cled - LED class device instance
+ * @param - LED CPLD access parameters
+**/
+struct mlxcpld_led_priv {
+	struct led_classdev cdev;
+	struct mlxcpld_param param;
+};
+
+#define cdev_to_priv(c)		container_of(c, struct mlxcpld_led_priv, cdev)
+
+/**
+ * mlxcpld_led_profile - system LED profile (defined per system class):
+ * @offset - offset for LED access in CPLD device
+ * @mask - mask for LED access in CPLD device
+ * @base_color - base color code
+ * @brightness - default brightness setting (on/off)
+ * @name - LED name
+**/
+struct mlxcpld_led_profile {
+	u8 offset;
+	u8 mask;
+	u8 base_color;
+	enum led_brightness brightness;
+	const char *name;
+};
+
+/**
+ * mlxcpld_led_pdata - system LED private data
+ * @pdev - platform device pointer
+ * @pled - LED class device instance
+ * @profile - system configuration profile
+ * @num_led_instances - number of LED instances
+ * @lock - device access lock
+**/
+struct mlxcpld_led_pdata {
+	struct platform_device *pdev;
+	struct mlxcpld_led_priv *pled;
+	struct mlxcpld_led_profile *profile;
+	int num_led_instances;
+	spinlock_t lock;
+};
+
+static struct mlxcpld_led_pdata *mlxcpld_led;
+
+/* Default profile fit the next Mellanox systems:
+ * "msx6710", "msx6720", "msb7700", "msn2700", "msx1410",
+ * "msn2410", "msb7800", "msn2740"
+ */
+static struct mlxcpld_led_profile mlxcpld_led_default_profile[] = {
+	{
+		0x21, 0xf0, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:fan1:green",
+	},
+	{
+		0x21, 0xf0, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:fan1:red",
+	},
+	{
+		0x21, 0x0f, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:fan2:green",
+	},
+	{
+		0x21, 0x0f, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:fan2:red",
+	},
+	{
+		0x22, 0xf0, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:fan3:green",
+	},
+	{
+		0x22, 0xf0, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:fan3:red",
+	},
+	{
+		0x22, 0x0f, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:fan4:green",
+	},
+	{
+		0x22, 0x0f, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:fan4:red",
+	},
+	{
+		0x20, 0x0f, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:psu:green",
+	},
+	{
+		0x20, 0x0f, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:psu:red",
+	},
+	{
+		0x20, 0xf0, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:status:green",
+	},
+	{
+		0x20, 0xf0, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:status:red",
+	},
+};
+
+/* Profile fit the Mellanox systems based on "msn2100" */
+static struct mlxcpld_led_profile mlxcpld_led_msn2100_profile[] = {
+	{
+		0x21, 0xf0, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:fan:green",
+	},
+	{
+		0x21, 0xf0, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:fan:red",
+	},
+	{
+		0x23, 0xf0, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:psu1:green",
+	},
+	{
+		0x23, 0xf0, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:psu1:red",
+	},
+	{
+		0x23, 0x0f, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:psu2:green",
+	},
+	{
+		0x23, 0x0f, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:psu2:red",
+	},
+	{
+		0x20, 0xf0, MLXCPLD_LED_GREEN_STATIC_ON, 1,
+		"mlxcpld:status:green",
+	},
+	{
+		0x20, 0xf0, MLXCPLD_LED_RED_STATIC_ON, LED_OFF,
+		"mlxcpld:status:red",
+	},
+	{
+		0x24, 0xf0, MLXCPLD_LED_GREEN_STATIC_ON, LED_OFF,
+		"mlxcpld:uid:blue",
+	},
+};
+
+enum mlxcpld_led_platform_types {
+	MLXCPLD_LED_PLATFORM_DEFAULT,
+	MLXCPLD_LED_PLATFORM_MSN2100,
+};
+
+static const char *mlx_product_names[] = {
+	"DEFAULT",
+	"MSN2100",
+};
+
+static enum
+mlxcpld_led_platform_types mlxcpld_led_platform_check_sys_type(void)
+{
+	const char *mlx_product_name;
+	int i;
+
+	mlx_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
+	if (!mlx_product_name)
+		return MLXCPLD_LED_PLATFORM_DEFAULT;
+
+	for (i = 1;  i < ARRAY_SIZE(mlx_product_names); i++) {
+		if (strstr(mlx_product_name, mlx_product_names[i]))
+			return i;
+	}
+
+	return MLXCPLD_LED_PLATFORM_DEFAULT;
+}
+
+static void mlxcpld_led_bus_access_func(u16 base, u8 offset, u8 rw_flag,
+					u8 *data)
+{
+	u32 addr = base + offset;
+
+	if (rw_flag == 0)
+		outb(*data, addr);
+	else
+		*data = inb(addr);
+}
+
+static void mlxcpld_led_store_hw(u8 mask, u8 off, u8 vset)
+{
+	u8 nib, val;
+
+	/*
+	 * Each LED is controlled through low or high nibble of the relevant
+	 * CPLD register. Register offset is specified by off parameter.
+	 * Parameter vset provides color code: 0x0 for off, 0x5 for solid red,
+	 * 0x6 for 3Hz blink red, 0xd for solid green, 0xe for 3Hz blink
+	 * green.
+	 * Parameter mask specifies which nibble is used for specific LED: mask
+	 * 0xf0 - lower nibble is to be used (bits from 0 to 3), mask 0x0f -
+	 * higher nibble (bits from 4 to 7).
+	 */
+	spin_lock(&mlxcpld_led->lock);
+	mlxcpld_led_bus_access_func(MLXPLAT_CPLD_LPC_REG_BASE_ADRR, off, 1,
+				    &val);
+	nib = (mask == 0xf0) ? vset : (vset << 4);
+	val = (val & mask) | nib;
+	mlxcpld_led_bus_access_func(MLXPLAT_CPLD_LPC_REG_BASE_ADRR, off, 0,
+				    &val);
+	spin_unlock(&mlxcpld_led->lock);
+}
+
+static void mlxcpld_led_brightness_set(struct led_classdev *led,
+				       enum led_brightness value)
+{
+	struct mlxcpld_led_priv *pled = cdev_to_priv(led);
+
+	if (value) {
+		mlxcpld_led_store_hw(pled->param.mask, pled->param.offset,
+				     pled->param.base_color);
+		return;
+	}
+
+	mlxcpld_led_store_hw(pled->param.mask, pled->param.offset,
+			     MLXCPLD_LED_IS_OFF);
+}
+
+static int mlxcpld_led_blink_set(struct led_classdev *led,
+				 unsigned long *delay_on,
+				 unsigned long *delay_off)
+{
+	struct mlxcpld_led_priv *pled = cdev_to_priv(led);
+
+	/*
+	 * HW supports two types of blinking: full (6Hz) and half (3Hz).
+	 * For delay on/off zero default setting 3Hz is used.
+	 */
+	if (!(*delay_on == 0 && *delay_off == 0) &&
+	    !(*delay_on == MLXCPLD_LED_BLINK_3HZ &&
+	      *delay_off == MLXCPLD_LED_BLINK_3HZ) &&
+	    !(*delay_on == MLXCPLD_LED_BLINK_6HZ &&
+	      *delay_off == MLXCPLD_LED_BLINK_6HZ))
+		return -EINVAL;
+
+	if (*delay_on == MLXCPLD_LED_BLINK_6HZ)
+		mlxcpld_led_store_hw(pled->param.mask, pled->param.offset,
+				     pled->param.base_color +
+				     MLXCPLD_LED_OFFSET_FULL);
+	else
+		mlxcpld_led_store_hw(pled->param.mask, pled->param.offset,
+				     pled->param.base_color +
+				     MLXCPLD_LED_OFFSET_HALF);
+
+	return 0;
+}
+
+static int mlxcpld_led_config(struct device *dev,
+			      struct mlxcpld_led_pdata *cpld)
+{
+	int i;
+	int err;
+
+	cpld->pled = devm_kzalloc(dev, sizeof(struct mlxcpld_led_priv) *
+				  cpld->num_led_instances, GFP_KERNEL);
+	if (!cpld->pled)
+		return -ENOMEM;
+
+	for (i = 0; i < cpld->num_led_instances; i++) {
+		cpld->pled[i].cdev.name = cpld->profile[i].name;
+		cpld->pled[i].cdev.brightness = cpld->profile[i].brightness;
+		cpld->pled[i].cdev.max_brightness = 1;
+		cpld->pled[i].cdev.brightness_set = mlxcpld_led_brightness_set;
+		cpld->pled[i].cdev.blink_set = mlxcpld_led_blink_set;
+		cpld->pled[i].cdev.flags = LED_CORE_SUSPENDRESUME;
+		err = devm_led_classdev_register(dev, &cpld->pled[i].cdev);
+		if (err)
+			return err;
+
+		cpld->pled[i].param.offset = mlxcpld_led->profile[i].offset;
+		cpld->pled[i].param.mask = mlxcpld_led->profile[i].mask;
+		cpld->pled[i].param.base_color =
+					mlxcpld_led->profile[i].base_color;
+
+		if (mlxcpld_led->profile[i].brightness)
+			mlxcpld_led_brightness_set(&cpld->pled[i].cdev,
+					mlxcpld_led->profile[i].brightness);
+	}
+
+	return 0;
+}
+
+static int __init mlxcpld_led_probe(struct platform_device *pdev)
+{
+	enum mlxcpld_led_platform_types mlxcpld_led_plat =
+					mlxcpld_led_platform_check_sys_type();
+
+	mlxcpld_led = devm_kzalloc(&pdev->dev, sizeof(*mlxcpld_led),
+				   GFP_KERNEL);
+	if (!mlxcpld_led)
+		return -ENOMEM;
+
+	mlxcpld_led->pdev = pdev;
+
+	switch (mlxcpld_led_plat) {
+	case MLXCPLD_LED_PLATFORM_MSN2100:
+		mlxcpld_led->profile = mlxcpld_led_msn2100_profile;
+		mlxcpld_led->num_led_instances =
+				ARRAY_SIZE(mlxcpld_led_msn2100_profile);
+		break;
+
+	default:
+		mlxcpld_led->profile = mlxcpld_led_default_profile;
+		mlxcpld_led->num_led_instances =
+				ARRAY_SIZE(mlxcpld_led_default_profile);
+		break;
+	}
+
+	spin_lock_init(&mlxcpld_led->lock);
+
+	return mlxcpld_led_config(&pdev->dev, mlxcpld_led);
+}
+
+static struct platform_driver mlxcpld_led_driver = {
+	.driver = {
+		.name	= KBUILD_MODNAME,
+	},
+};
+
+static int __init mlxcpld_led_init(void)
+{
+	struct platform_device *pdev;
+	int err;
+
+	pdev = platform_device_register_simple(KBUILD_MODNAME, -1, NULL, 0);
+	if (IS_ERR(pdev)) {
+		pr_err("Device allocation failed\n");
+		return PTR_ERR(pdev);
+	}
+
+	err = platform_driver_probe(&mlxcpld_led_driver, mlxcpld_led_probe);
+	if (err) {
+		pr_err("Probe platform driver failed\n");
+		platform_device_unregister(pdev);
+	}
+
+	return err;
+}
+
+static void __exit mlxcpld_led_exit(void)
+{
+	platform_device_unregister(mlxcpld_led->pdev);
+	platform_driver_unregister(&mlxcpld_led_driver);
+}
+
+module_init(mlxcpld_led_init);
+module_exit(mlxcpld_led_exit);
+
+MODULE_AUTHOR("Vadim Pasternak <vadimp@mellanox.com>");
+MODULE_DESCRIPTION("Mellanox board LED driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:leds_mlxcpld");
diff --git a/drivers/leds/leds-pm8058.c b/drivers/leds/leds-pm8058.c
new file mode 100644
index 000000000000..a52674327857
--- /dev/null
+++ b/drivers/leds/leds-pm8058.c
@@ -0,0 +1,191 @@
+/* Copyright (c) 2010, 2011, 2016 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/regmap.h>
+
+#define PM8058_LED_TYPE_COMMON	0x00
+#define PM8058_LED_TYPE_KEYPAD	0x01
+#define PM8058_LED_TYPE_FLASH	0x02
+
+#define PM8058_LED_TYPE_COMMON_MASK	0xf8
+#define PM8058_LED_TYPE_KEYPAD_MASK	0xf0
+#define PM8058_LED_TYPE_COMMON_SHIFT	3
+#define PM8058_LED_TYPE_KEYPAD_SHIFT	4
+
+struct pm8058_led {
+	struct regmap *map;
+	u32 reg;
+	u32 ledtype;
+	struct led_classdev cdev;
+};
+
+static void pm8058_led_set(struct led_classdev *cled,
+	enum led_brightness value)
+{
+	struct pm8058_led *led;
+	int ret = 0;
+	unsigned int mask = 0;
+	unsigned int val = 0;
+
+	led = container_of(cled, struct pm8058_led, cdev);
+	switch (led->ledtype) {
+	case PM8058_LED_TYPE_COMMON:
+		mask = PM8058_LED_TYPE_COMMON_MASK;
+		val = value << PM8058_LED_TYPE_COMMON_SHIFT;
+		break;
+	case PM8058_LED_TYPE_KEYPAD:
+	case PM8058_LED_TYPE_FLASH:
+		mask = PM8058_LED_TYPE_KEYPAD_MASK;
+		val = value << PM8058_LED_TYPE_KEYPAD_SHIFT;
+		break;
+	default:
+		break;
+	}
+
+	ret = regmap_update_bits(led->map, led->reg, mask, val);
+	if (ret)
+		pr_err("Failed to set LED brightness\n");
+}
+
+static enum led_brightness pm8058_led_get(struct led_classdev *cled)
+{
+	struct pm8058_led *led;
+	int ret;
+	unsigned int val;
+
+	led = container_of(cled, struct pm8058_led, cdev);
+
+	ret = regmap_read(led->map, led->reg, &val);
+	if (ret) {
+		pr_err("Failed to get LED brightness\n");
+		return LED_OFF;
+	}
+
+	switch (led->ledtype) {
+	case PM8058_LED_TYPE_COMMON:
+		val &= PM8058_LED_TYPE_COMMON_MASK;
+		val >>= PM8058_LED_TYPE_COMMON_SHIFT;
+		break;
+	case PM8058_LED_TYPE_KEYPAD:
+	case PM8058_LED_TYPE_FLASH:
+		val &= PM8058_LED_TYPE_KEYPAD_MASK;
+		val >>= PM8058_LED_TYPE_KEYPAD_SHIFT;
+		break;
+	default:
+		val = LED_OFF;
+		break;
+	}
+
+	return val;
+}
+
+static int pm8058_led_probe(struct platform_device *pdev)
+{
+	struct pm8058_led *led;
+	struct device_node *np = pdev->dev.of_node;
+	int ret;
+	struct regmap *map;
+	const char *state;
+	enum led_brightness maxbright;
+
+	led = devm_kzalloc(&pdev->dev, sizeof(*led), GFP_KERNEL);
+	if (!led)
+		return -ENOMEM;
+
+	led->ledtype = (u32)of_device_get_match_data(&pdev->dev);
+
+	map = dev_get_regmap(pdev->dev.parent, NULL);
+	if (!map) {
+		dev_err(&pdev->dev, "Parent regmap unavailable.\n");
+		return -ENXIO;
+	}
+	led->map = map;
+
+	ret = of_property_read_u32(np, "reg", &led->reg);
+	if (ret) {
+		dev_err(&pdev->dev, "no register offset specified\n");
+		return -EINVAL;
+	}
+
+	/* Use label else node name */
+	led->cdev.name = of_get_property(np, "label", NULL) ? : np->name;
+	led->cdev.default_trigger =
+		of_get_property(np, "linux,default-trigger", NULL);
+	led->cdev.brightness_set = pm8058_led_set;
+	led->cdev.brightness_get = pm8058_led_get;
+	if (led->ledtype == PM8058_LED_TYPE_COMMON)
+		maxbright = 31; /* 5 bits */
+	else
+		maxbright = 15; /* 4 bits */
+	led->cdev.max_brightness = maxbright;
+
+	state = of_get_property(np, "default-state", NULL);
+	if (state) {
+		if (!strcmp(state, "keep")) {
+			led->cdev.brightness = pm8058_led_get(&led->cdev);
+		} else if (!strcmp(state, "on")) {
+			led->cdev.brightness = maxbright;
+			pm8058_led_set(&led->cdev, maxbright);
+		} else {
+			led->cdev.brightness = LED_OFF;
+			pm8058_led_set(&led->cdev, LED_OFF);
+		}
+	}
+
+	if (led->ledtype == PM8058_LED_TYPE_KEYPAD ||
+	    led->ledtype == PM8058_LED_TYPE_FLASH)
+		led->cdev.flags	= LED_CORE_SUSPENDRESUME;
+
+	ret = devm_led_classdev_register(&pdev->dev, &led->cdev);
+	if (ret) {
+		dev_err(&pdev->dev, "unable to register led \"%s\"\n",
+			led->cdev.name);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id pm8058_leds_id_table[] = {
+	{
+		.compatible = "qcom,pm8058-led",
+		.data = (void *)PM8058_LED_TYPE_COMMON
+	},
+	{
+		.compatible = "qcom,pm8058-keypad-led",
+		.data = (void *)PM8058_LED_TYPE_KEYPAD
+	},
+	{
+		.compatible = "qcom,pm8058-flash-led",
+		.data = (void *)PM8058_LED_TYPE_FLASH
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, pm8058_leds_id_table);
+
+static struct platform_driver pm8058_led_driver = {
+	.probe		= pm8058_led_probe,
+	.driver		= {
+		.name	= "pm8058-leds",
+		.of_match_table = pm8058_leds_id_table,
+	},
+};
+module_platform_driver(pm8058_led_driver);
+
+MODULE_DESCRIPTION("PM8058 LEDs driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:pm8058-leds");
diff --git a/drivers/lightnvm/Kconfig b/drivers/lightnvm/Kconfig
index 61c68a1f054a..2f5d5f4a4c75 100644
--- a/drivers/lightnvm/Kconfig
+++ b/drivers/lightnvm/Kconfig
@@ -4,7 +4,7 @@
 
 menuconfig NVM
 	bool "Open-Channel SSD target support"
-	depends on BLOCK
+	depends on BLOCK && HAS_DMA
 	help
 	  Say Y here to get to enable Open-channel SSDs.
 
diff --git a/drivers/lightnvm/Makefile b/drivers/lightnvm/Makefile
index a7a0a22cf1a5..1f6b6521016a 100644
--- a/drivers/lightnvm/Makefile
+++ b/drivers/lightnvm/Makefile
@@ -2,6 +2,6 @@
 # Makefile for Open-Channel SSDs.
 #
 
-obj-$(CONFIG_NVM)		:= core.o sysblk.o
+obj-$(CONFIG_NVM)		:= core.o sysblk.o sysfs.o
 obj-$(CONFIG_NVM_GENNVM) 	+= gennvm.o
 obj-$(CONFIG_NVM_RRPC)		+= rrpc.o
diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c
index 9ebd2cfbd849..1cac0f8bc0dc 100644
--- a/drivers/lightnvm/core.c
+++ b/drivers/lightnvm/core.c
@@ -27,6 +27,8 @@
 #include <linux/lightnvm.h>
 #include <linux/sched/sysctl.h>
 
+#include "lightnvm.h"
+
 static LIST_HEAD(nvm_tgt_types);
 static DECLARE_RWSEM(nvm_tgtt_lock);
 static LIST_HEAD(nvm_mgrs);
@@ -581,6 +583,8 @@ static int nvm_core_init(struct nvm_dev *dev)
 	mutex_init(&dev->mlock);
 	spin_lock_init(&dev->lock);
 
+	blk_queue_logical_block_size(dev->q, dev->sec_size);
+
 	return 0;
 err_fmtype:
 	kfree(dev->lun_map);
@@ -596,15 +600,19 @@ static void nvm_free_mgr(struct nvm_dev *dev)
 	dev->mt = NULL;
 }
 
-static void nvm_free(struct nvm_dev *dev)
+void nvm_free(struct nvm_dev *dev)
 {
 	if (!dev)
 		return;
 
 	nvm_free_mgr(dev);
 
+	if (dev->dma_pool)
+		dev->ops->destroy_dma_pool(dev->dma_pool);
+
 	kfree(dev->lptbl);
 	kfree(dev->lun_map);
+	kfree(dev);
 }
 
 static int nvm_init(struct nvm_dev *dev)
@@ -651,30 +659,19 @@ err:
 
 static void nvm_exit(struct nvm_dev *dev)
 {
-	if (dev->dma_pool)
-		dev->ops->destroy_dma_pool(dev->dma_pool);
-	nvm_free(dev);
+	nvm_sysfs_unregister_dev(dev);
+}
 
-	pr_info("nvm: successfully unloaded\n");
+struct nvm_dev *nvm_alloc_dev(int node)
+{
+	return kzalloc_node(sizeof(struct nvm_dev), GFP_KERNEL, node);
 }
+EXPORT_SYMBOL(nvm_alloc_dev);
 
-int nvm_register(struct request_queue *q, char *disk_name,
-							struct nvm_dev_ops *ops)
+int nvm_register(struct nvm_dev *dev)
 {
-	struct nvm_dev *dev;
 	int ret;
 
-	if (!ops->identity)
-		return -EINVAL;
-
-	dev = kzalloc(sizeof(struct nvm_dev), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	dev->q = q;
-	dev->ops = ops;
-	strncpy(dev->name, disk_name, DISK_NAME_LEN);
-
 	ret = nvm_init(dev);
 	if (ret)
 		goto err_init;
@@ -694,6 +691,10 @@ int nvm_register(struct request_queue *q, char *disk_name,
 		}
 	}
 
+	ret = nvm_sysfs_register_dev(dev);
+	if (ret)
+		goto err_ppalist;
+
 	if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT) {
 		ret = nvm_get_sysblock(dev, &dev->sb);
 		if (!ret)
@@ -710,31 +711,21 @@ int nvm_register(struct request_queue *q, char *disk_name,
 	up_write(&nvm_lock);
 
 	return 0;
+err_ppalist:
+	dev->ops->destroy_dma_pool(dev->dma_pool);
 err_init:
 	kfree(dev->lun_map);
-	kfree(dev);
 	return ret;
 }
 EXPORT_SYMBOL(nvm_register);
 
-void nvm_unregister(char *disk_name)
+void nvm_unregister(struct nvm_dev *dev)
 {
-	struct nvm_dev *dev;
-
 	down_write(&nvm_lock);
-	dev = nvm_find_nvm_dev(disk_name);
-	if (!dev) {
-		pr_err("nvm: could not find device %s to unregister\n",
-								disk_name);
-		up_write(&nvm_lock);
-		return;
-	}
-
 	list_del(&dev->devices);
 	up_write(&nvm_lock);
 
 	nvm_exit(dev);
-	kfree(dev);
 }
 EXPORT_SYMBOL(nvm_unregister);
 
@@ -1171,27 +1162,10 @@ static struct miscdevice _nvm_misc = {
 	.nodename	= "lightnvm/control",
 	.fops		= &_ctl_fops,
 };
+module_misc_device(_nvm_misc);
 
 MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR);
 
-static int __init nvm_mod_init(void)
-{
-	int ret;
-
-	ret = misc_register(&_nvm_misc);
-	if (ret)
-		pr_err("nvm: misc_register failed for control device");
-
-	return ret;
-}
-
-static void __exit nvm_mod_exit(void)
-{
-	misc_deregister(&_nvm_misc);
-}
-
 MODULE_AUTHOR("Matias Bjorling <m@bjorling.me>");
 MODULE_LICENSE("GPL v2");
 MODULE_VERSION("0.1");
-module_init(nvm_mod_init);
-module_exit(nvm_mod_exit);
diff --git a/drivers/lightnvm/lightnvm.h b/drivers/lightnvm/lightnvm.h
new file mode 100644
index 000000000000..305c181509a6
--- /dev/null
+++ b/drivers/lightnvm/lightnvm.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2016 CNEX Labs. All rights reserved.
+ * Initial release: Matias Bjorling <matias@cnexlabs.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
+ * USA.
+ *
+ */
+
+#ifndef LIGHTNVM_H
+#define LIGHTNVM_H
+
+#include <linux/lightnvm.h>
+
+/* core -> sysfs.c */
+int __must_check nvm_sysfs_register_dev(struct nvm_dev *);
+void nvm_sysfs_unregister_dev(struct nvm_dev *);
+int nvm_sysfs_register(void);
+void nvm_sysfs_unregister(void);
+
+/* sysfs > core */
+void nvm_free(struct nvm_dev *);
+
+#endif
diff --git a/drivers/lightnvm/sysfs.c b/drivers/lightnvm/sysfs.c
new file mode 100644
index 000000000000..0338c27ab95a
--- /dev/null
+++ b/drivers/lightnvm/sysfs.c
@@ -0,0 +1,198 @@
+#include <linux/kernel.h>
+#include <linux/lightnvm.h>
+#include <linux/miscdevice.h>
+#include <linux/kobject.h>
+#include <linux/blk-mq.h>
+
+#include "lightnvm.h"
+
+static ssize_t nvm_dev_attr_show(struct device *dev,
+				 struct device_attribute *dattr, char *page)
+{
+	struct nvm_dev *ndev = container_of(dev, struct nvm_dev, dev);
+	struct nvm_id *id = &ndev->identity;
+	struct nvm_id_group *grp = &id->groups[0];
+	struct attribute *attr = &dattr->attr;
+
+	if (strcmp(attr->name, "version") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", id->ver_id);
+	} else if (strcmp(attr->name, "vendor_opcode") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", id->vmnt);
+	} else if (strcmp(attr->name, "capabilities") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", id->cap);
+	} else if (strcmp(attr->name, "device_mode") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", id->dom);
+	} else if (strcmp(attr->name, "media_manager") == 0) {
+		if (!ndev->mt)
+			return scnprintf(page, PAGE_SIZE, "%s\n", "none");
+		return scnprintf(page, PAGE_SIZE, "%s\n", ndev->mt->name);
+	} else if (strcmp(attr->name, "ppa_format") == 0) {
+		return scnprintf(page, PAGE_SIZE,
+			"0x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
+			id->ppaf.ch_offset, id->ppaf.ch_len,
+			id->ppaf.lun_offset, id->ppaf.lun_len,
+			id->ppaf.pln_offset, id->ppaf.pln_len,
+			id->ppaf.blk_offset, id->ppaf.blk_len,
+			id->ppaf.pg_offset, id->ppaf.pg_len,
+			id->ppaf.sect_offset, id->ppaf.sect_len);
+	} else if (strcmp(attr->name, "media_type") == 0) {	/* u8 */
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->mtype);
+	} else if (strcmp(attr->name, "flash_media_type") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->fmtype);
+	} else if (strcmp(attr->name, "num_channels") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_ch);
+	} else if (strcmp(attr->name, "num_luns") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_lun);
+	} else if (strcmp(attr->name, "num_planes") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pln);
+	} else if (strcmp(attr->name, "num_blocks") == 0) {	/* u16 */
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_blk);
+	} else if (strcmp(attr->name, "num_pages") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pg);
+	} else if (strcmp(attr->name, "page_size") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->fpg_sz);
+	} else if (strcmp(attr->name, "hw_sector_size") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->csecs);
+	} else if (strcmp(attr->name, "oob_sector_size") == 0) {/* u32 */
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->sos);
+	} else if (strcmp(attr->name, "read_typ") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->trdt);
+	} else if (strcmp(attr->name, "read_max") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->trdm);
+	} else if (strcmp(attr->name, "prog_typ") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tprt);
+	} else if (strcmp(attr->name, "prog_max") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tprm);
+	} else if (strcmp(attr->name, "erase_typ") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tbet);
+	} else if (strcmp(attr->name, "erase_max") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tbem);
+	} else if (strcmp(attr->name, "multiplane_modes") == 0) {
+		return scnprintf(page, PAGE_SIZE, "0x%08x\n", grp->mpos);
+	} else if (strcmp(attr->name, "media_capabilities") == 0) {
+		return scnprintf(page, PAGE_SIZE, "0x%08x\n", grp->mccap);
+	} else if (strcmp(attr->name, "max_phys_secs") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n",
+				ndev->ops->max_phys_sect);
+	} else {
+		return scnprintf(page,
+				 PAGE_SIZE,
+				 "Unhandled attr(%s) in `nvm_dev_attr_show`\n",
+				 attr->name);
+	}
+}
+
+#define NVM_DEV_ATTR_RO(_name)						\
+	DEVICE_ATTR(_name, S_IRUGO, nvm_dev_attr_show, NULL)
+
+static NVM_DEV_ATTR_RO(version);
+static NVM_DEV_ATTR_RO(vendor_opcode);
+static NVM_DEV_ATTR_RO(capabilities);
+static NVM_DEV_ATTR_RO(device_mode);
+static NVM_DEV_ATTR_RO(ppa_format);
+static NVM_DEV_ATTR_RO(media_manager);
+
+static NVM_DEV_ATTR_RO(media_type);
+static NVM_DEV_ATTR_RO(flash_media_type);
+static NVM_DEV_ATTR_RO(num_channels);
+static NVM_DEV_ATTR_RO(num_luns);
+static NVM_DEV_ATTR_RO(num_planes);
+static NVM_DEV_ATTR_RO(num_blocks);
+static NVM_DEV_ATTR_RO(num_pages);
+static NVM_DEV_ATTR_RO(page_size);
+static NVM_DEV_ATTR_RO(hw_sector_size);
+static NVM_DEV_ATTR_RO(oob_sector_size);
+static NVM_DEV_ATTR_RO(read_typ);
+static NVM_DEV_ATTR_RO(read_max);
+static NVM_DEV_ATTR_RO(prog_typ);
+static NVM_DEV_ATTR_RO(prog_max);
+static NVM_DEV_ATTR_RO(erase_typ);
+static NVM_DEV_ATTR_RO(erase_max);
+static NVM_DEV_ATTR_RO(multiplane_modes);
+static NVM_DEV_ATTR_RO(media_capabilities);
+static NVM_DEV_ATTR_RO(max_phys_secs);
+
+#define NVM_DEV_ATTR(_name) (dev_attr_##_name##)
+
+static struct attribute *nvm_dev_attrs[] = {
+	&dev_attr_version.attr,
+	&dev_attr_vendor_opcode.attr,
+	&dev_attr_capabilities.attr,
+	&dev_attr_device_mode.attr,
+	&dev_attr_media_manager.attr,
+
+	&dev_attr_ppa_format.attr,
+	&dev_attr_media_type.attr,
+	&dev_attr_flash_media_type.attr,
+	&dev_attr_num_channels.attr,
+	&dev_attr_num_luns.attr,
+	&dev_attr_num_planes.attr,
+	&dev_attr_num_blocks.attr,
+	&dev_attr_num_pages.attr,
+	&dev_attr_page_size.attr,
+	&dev_attr_hw_sector_size.attr,
+	&dev_attr_oob_sector_size.attr,
+	&dev_attr_read_typ.attr,
+	&dev_attr_read_max.attr,
+	&dev_attr_prog_typ.attr,
+	&dev_attr_prog_max.attr,
+	&dev_attr_erase_typ.attr,
+	&dev_attr_erase_max.attr,
+	&dev_attr_multiplane_modes.attr,
+	&dev_attr_media_capabilities.attr,
+	&dev_attr_max_phys_secs.attr,
+	NULL,
+};
+
+static struct attribute_group nvm_dev_attr_group = {
+	.name = "lightnvm",
+	.attrs = nvm_dev_attrs,
+};
+
+static const struct attribute_group *nvm_dev_attr_groups[] = {
+	&nvm_dev_attr_group,
+	NULL,
+};
+
+static void nvm_dev_release(struct device *device)
+{
+	struct nvm_dev *dev = container_of(device, struct nvm_dev, dev);
+	struct request_queue *q = dev->q;
+
+	pr_debug("nvm/sysfs: `nvm_dev_release`\n");
+
+	blk_mq_unregister_dev(device, q);
+
+	nvm_free(dev);
+}
+
+static struct device_type nvm_type = {
+	.name		= "lightnvm",
+	.groups		= nvm_dev_attr_groups,
+	.release	= nvm_dev_release,
+};
+
+int nvm_sysfs_register_dev(struct nvm_dev *dev)
+{
+	int ret;
+
+	if (!dev->parent_dev)
+		return 0;
+
+	dev->dev.parent = dev->parent_dev;
+	dev_set_name(&dev->dev, "%s", dev->name);
+	dev->dev.type = &nvm_type;
+	device_initialize(&dev->dev);
+	ret = device_add(&dev->dev);
+
+	if (!ret)
+		blk_mq_register_dev(&dev->dev, dev->q);
+
+	return ret;
+}
+
+void nvm_sysfs_unregister_dev(struct nvm_dev *dev)
+{
+	if (dev && dev->parent_dev)
+		kobject_put(&dev->dev.kobj);
+}
diff --git a/drivers/macintosh/ams/ams-i2c.c b/drivers/macintosh/ams/ams-i2c.c
index 978eda8d6678..8a3ba565106f 100644
--- a/drivers/macintosh/ams/ams-i2c.c
+++ b/drivers/macintosh/ams/ams-i2c.c
@@ -73,7 +73,6 @@ MODULE_DEVICE_TABLE(i2c, ams_id);
 static struct i2c_driver ams_i2c_driver = {
 	.driver = {
 		.name   = "ams",
-		.owner  = THIS_MODULE,
 	},
 	.probe          = ams_i2c_probe,
 	.remove         = ams_i2c_remove,
diff --git a/drivers/macintosh/macio_asic.c b/drivers/macintosh/macio_asic.c
index b6819f0fc608..3f041b187033 100644
--- a/drivers/macintosh/macio_asic.c
+++ b/drivers/macintosh/macio_asic.c
@@ -236,7 +236,7 @@ static void macio_create_fixup_irq(struct macio_dev *dev, int index,
 	unsigned int irq;
 
 	irq = irq_create_mapping(NULL, line);
-	if (irq != NO_IRQ) {
+	if (!irq) {
 		dev->interrupt[index].start = irq;
 		dev->interrupt[index].flags = IORESOURCE_IRQ;
 		dev->interrupt[index].name = dev_name(&dev->ofdev.dev);
@@ -299,7 +299,7 @@ static void macio_setup_interrupts(struct macio_dev *dev)
 			break;
 		res = &dev->interrupt[j];
 		irq = irq_of_parse_and_map(np, i++);
-		if (irq == NO_IRQ)
+		if (!irq)
 			break;
 		res->start = irq;
 		res->flags = IORESOURCE_IRQ;
diff --git a/drivers/macintosh/rack-meter.c b/drivers/macintosh/rack-meter.c
index 465c52219639..775527135b93 100644
--- a/drivers/macintosh/rack-meter.c
+++ b/drivers/macintosh/rack-meter.c
@@ -427,7 +427,7 @@ static int rackmeter_probe(struct macio_dev* mdev,
 	rm->irq = macio_irq(mdev, 1);
 #else
 	rm->irq = irq_of_parse_and_map(i2s, 1);
-	if (rm->irq == NO_IRQ ||
+	if (!rm->irq ||
 	    of_address_to_resource(i2s, 0, &ri2s) ||
 	    of_address_to_resource(i2s, 1, &rdma)) {
 		printk(KERN_ERR
diff --git a/drivers/macintosh/smu.c b/drivers/macintosh/smu.c
index d6f72c826c1c..08edb2c25b60 100644
--- a/drivers/macintosh/smu.c
+++ b/drivers/macintosh/smu.c
@@ -279,7 +279,7 @@ int smu_queue_cmd(struct smu_cmd *cmd)
 	spin_unlock_irqrestore(&smu->lock, flags);
 
 	/* Workaround for early calls when irq isn't available */
-	if (!smu_irq_inited || smu->db_irq == NO_IRQ)
+	if (!smu_irq_inited || !smu->db_irq)
 		smu_spinwait_cmd(cmd);
 
 	return 0;
@@ -498,8 +498,8 @@ int __init smu_init (void)
 	INIT_LIST_HEAD(&smu->cmd_list);
 	INIT_LIST_HEAD(&smu->cmd_i2c_list);
 	smu->of_node = np;
-	smu->db_irq = NO_IRQ;
-	smu->msg_irq = NO_IRQ;
+	smu->db_irq = 0;
+	smu->msg_irq = 0;
 
 	/* smu_cmdbuf_abs is in the low 2G of RAM, can be converted to a
 	 * 32 bits value safely
@@ -587,13 +587,13 @@ static int smu_late_init(void)
 
 	if (smu->db_node) {
 		smu->db_irq = irq_of_parse_and_map(smu->db_node, 0);
-		if (smu->db_irq == NO_IRQ)
+		if (!smu->db_irq)
 			printk(KERN_ERR "smu: failed to map irq for node %s\n",
 			       smu->db_node->full_name);
 	}
 	if (smu->msg_node) {
 		smu->msg_irq = irq_of_parse_and_map(smu->msg_node, 0);
-		if (smu->msg_irq == NO_IRQ)
+		if (!smu->msg_irq)
 			printk(KERN_ERR "smu: failed to map irq for node %s\n",
 			       smu->msg_node->full_name);
 	}
@@ -602,23 +602,23 @@ static int smu_late_init(void)
 	 * Try to request the interrupts
 	 */
 
-	if (smu->db_irq != NO_IRQ) {
+	if (smu->db_irq) {
 		if (request_irq(smu->db_irq, smu_db_intr,
 				IRQF_SHARED, "SMU doorbell", smu) < 0) {
 			printk(KERN_WARNING "SMU: can't "
 			       "request interrupt %d\n",
 			       smu->db_irq);
-			smu->db_irq = NO_IRQ;
+			smu->db_irq = 0;
 		}
 	}
 
-	if (smu->msg_irq != NO_IRQ) {
+	if (smu->msg_irq) {
 		if (request_irq(smu->msg_irq, smu_msg_intr,
 				IRQF_SHARED, "SMU message", smu) < 0) {
 			printk(KERN_WARNING "SMU: can't "
 			       "request interrupt %d\n",
 			       smu->msg_irq);
-			smu->msg_irq = NO_IRQ;
+			smu->msg_irq = 0;
 		}
 	}
 
diff --git a/drivers/macintosh/via-cuda.c b/drivers/macintosh/via-cuda.c
index bad18130f125..2088e23a8002 100644
--- a/drivers/macintosh/via-cuda.c
+++ b/drivers/macintosh/via-cuda.c
@@ -209,7 +209,7 @@ static int __init via_cuda_start(void)
     cuda_irq = IRQ_MAC_ADB;
 #else
     cuda_irq = irq_of_parse_and_map(vias, 0);
-    if (cuda_irq == NO_IRQ) {
+    if (!cuda_irq) {
 	printk(KERN_ERR "via-cuda: can't map interrupts for %s\n",
 	       vias->full_name);
 	return -ENODEV;
diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
index f8b6d1403c16..91081dcdc272 100644
--- a/drivers/macintosh/via-pmu.c
+++ b/drivers/macintosh/via-pmu.c
@@ -145,7 +145,7 @@ static int pmu_fully_inited;
 static int pmu_has_adb;
 static struct device_node *gpio_node;
 static unsigned char __iomem *gpio_reg;
-static int gpio_irq = NO_IRQ;
+static int gpio_irq = 0;
 static int gpio_irq_enabled = -1;
 static volatile int pmu_suspended;
 static spinlock_t pmu_lock;
@@ -402,7 +402,7 @@ static int __init via_pmu_start(void)
 	batt_req.complete = 1;
 
 	irq = irq_of_parse_and_map(vias, 0);
-	if (irq == NO_IRQ) {
+	if (!irq) {
 		printk(KERN_ERR "via-pmu: can't map interrupt\n");
 		return -ENODEV;
 	}
@@ -424,7 +424,7 @@ static int __init via_pmu_start(void)
 		if (gpio_node)
 			gpio_irq = irq_of_parse_and_map(gpio_node, 0);
 
-		if (gpio_irq != NO_IRQ) {
+		if (gpio_irq) {
 			if (request_irq(gpio_irq, gpio1_interrupt,
 					IRQF_NO_SUSPEND, "GPIO1 ADB",
 					(void *)0))
diff --git a/drivers/macintosh/windfarm_pm112.c b/drivers/macintosh/windfarm_pm112.c
index 3024685e4cca..96d16fca68b2 100644
--- a/drivers/macintosh/windfarm_pm112.c
+++ b/drivers/macintosh/windfarm_pm112.c
@@ -668,7 +668,6 @@ static struct platform_driver wf_pm112_driver = {
 	.remove = wf_pm112_remove,
 	.driver = {
 		.name = "windfarm",
-		.owner	= THIS_MODULE,
 	},
 };
 
diff --git a/drivers/macintosh/windfarm_pm72.c b/drivers/macintosh/windfarm_pm72.c
index 2f506b9d5a52..e88cfb36a74d 100644
--- a/drivers/macintosh/windfarm_pm72.c
+++ b/drivers/macintosh/windfarm_pm72.c
@@ -789,7 +789,6 @@ static struct platform_driver wf_pm72_driver = {
 	.remove	= wf_pm72_remove,
 	.driver	= {
 		.name = "windfarm",
-		.owner	= THIS_MODULE,
 	},
 };
 
diff --git a/drivers/macintosh/windfarm_rm31.c b/drivers/macintosh/windfarm_rm31.c
index 82fc86a90c1a..bdfcb8a8bfbb 100644
--- a/drivers/macintosh/windfarm_rm31.c
+++ b/drivers/macintosh/windfarm_rm31.c
@@ -682,7 +682,6 @@ static struct platform_driver wf_rm31_driver = {
 	.remove	= wf_rm31_remove,
 	.driver	= {
 		.name = "windfarm",
-		.owner	= THIS_MODULE,
 	},
 };
 
diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
index 97c372908e78..11eebfe8a4cb 100644
--- a/drivers/mailbox/Kconfig
+++ b/drivers/mailbox/Kconfig
@@ -15,6 +15,16 @@ config ARM_MHU
 	  The controller has 3 mailbox channels, the last of which can be
 	  used in Secure mode only.
 
+config PLATFORM_MHU
+	tristate "Platform MHU Mailbox"
+	depends on OF
+	depends on HAS_IOMEM
+	help
+	  Say Y here if you want to build a platform specific variant MHU
+	  controller driver.
+	  The controller has a maximum of 3 mailbox channels, the last of
+	  which can be used in Secure mode only.
+
 config PL320_MBOX
 	bool "ARM PL320 Mailbox"
 	depends on ARM_AMBA
@@ -127,6 +137,7 @@ config XGENE_SLIMPRO_MBOX
 config BCM_PDC_MBOX
 	tristate "Broadcom PDC Mailbox"
 	depends on ARM64 || COMPILE_TEST
+	depends on HAS_DMA
 	default ARCH_BCM_IPROC
 	help
 	  Mailbox implementation for the Broadcom PDC ring manager,
diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile
index 66c38e300dfc..ace6fed8fea9 100644
--- a/drivers/mailbox/Makefile
+++ b/drivers/mailbox/Makefile
@@ -6,6 +6,8 @@ obj-$(CONFIG_MAILBOX_TEST)	+= mailbox-test.o
 
 obj-$(CONFIG_ARM_MHU)	+= arm_mhu.o
 
+obj-$(CONFIG_PLATFORM_MHU)	+= platform_mhu.o
+
 obj-$(CONFIG_PL320_MBOX)	+= pl320-ipc.o
 
 obj-$(CONFIG_OMAP2PLUS_MBOX)	+= omap-mailbox.o
diff --git a/drivers/mailbox/bcm-pdc-mailbox.c b/drivers/mailbox/bcm-pdc-mailbox.c
index cbe0c1ee4ba9..c19dd820ea9b 100644
--- a/drivers/mailbox/bcm-pdc-mailbox.c
+++ b/drivers/mailbox/bcm-pdc-mailbox.c
@@ -469,7 +469,7 @@ static const struct file_operations pdc_debugfs_stats = {
  * this directory for a SPU.
  * @pdcs: PDC state structure
  */
-void pdc_setup_debugfs(struct pdc_state *pdcs)
+static void pdc_setup_debugfs(struct pdc_state *pdcs)
 {
 	char spu_stats_name[16];
 
@@ -485,7 +485,7 @@ void pdc_setup_debugfs(struct pdc_state *pdcs)
 						  &pdc_debugfs_stats);
 }
 
-void pdc_free_debugfs(void)
+static void pdc_free_debugfs(void)
 {
 	if (debugfs_dir && simple_empty(debugfs_dir)) {
 		debugfs_remove_recursive(debugfs_dir);
@@ -1191,10 +1191,11 @@ static void pdc_shutdown(struct mbox_chan *chan)
 {
 	struct pdc_state *pdcs = chan->con_priv;
 
-	if (pdcs)
-		dev_dbg(&pdcs->pdev->dev,
-			"Shutdown mailbox channel for PDC %u", pdcs->pdc_idx);
+	if (!pdcs)
+		return;
 
+	dev_dbg(&pdcs->pdev->dev,
+		"Shutdown mailbox channel for PDC %u", pdcs->pdc_idx);
 	pdc_ring_free(pdcs);
 }
 
diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c
index 043828d541f7..08c87fadca8c 100644
--- a/drivers/mailbox/pcc.c
+++ b/drivers/mailbox/pcc.c
@@ -59,6 +59,7 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/list.h>
 #include <linux/platform_device.h>
 #include <linux/mailbox_controller.h>
@@ -68,11 +69,16 @@
 #include "mailbox.h"
 
 #define MAX_PCC_SUBSPACES	256
+#define MBOX_IRQ_NAME		"pcc-mbox"
 
 static struct mbox_chan *pcc_mbox_channels;
 
 /* Array of cached virtual address for doorbell registers */
 static void __iomem **pcc_doorbell_vaddr;
+/* Array of cached virtual address for doorbell ack registers */
+static void __iomem **pcc_doorbell_ack_vaddr;
+/* Array of doorbell interrupts */
+static int *pcc_doorbell_irq;
 
 static struct mbox_controller pcc_mbox_ctrl = {};
 /**
@@ -91,6 +97,132 @@ static struct mbox_chan *get_pcc_channel(int id)
 	return &pcc_mbox_channels[id];
 }
 
+/*
+ * PCC can be used with perf critical drivers such as CPPC
+ * So it makes sense to locally cache the virtual address and
+ * use it to read/write to PCC registers such as doorbell register
+ *
+ * The below read_register and write_registers are used to read and
+ * write from perf critical registers such as PCC doorbell register
+ */
+static int read_register(void __iomem *vaddr, u64 *val, unsigned int bit_width)
+{
+	int ret_val = 0;
+
+	switch (bit_width) {
+	case 8:
+		*val = readb(vaddr);
+		break;
+	case 16:
+		*val = readw(vaddr);
+		break;
+	case 32:
+		*val = readl(vaddr);
+		break;
+	case 64:
+		*val = readq(vaddr);
+		break;
+	default:
+		pr_debug("Error: Cannot read register of %u bit width",
+			bit_width);
+		ret_val = -EFAULT;
+		break;
+	}
+	return ret_val;
+}
+
+static int write_register(void __iomem *vaddr, u64 val, unsigned int bit_width)
+{
+	int ret_val = 0;
+
+	switch (bit_width) {
+	case 8:
+		writeb(val, vaddr);
+		break;
+	case 16:
+		writew(val, vaddr);
+		break;
+	case 32:
+		writel(val, vaddr);
+		break;
+	case 64:
+		writeq(val, vaddr);
+		break;
+	default:
+		pr_debug("Error: Cannot write register of %u bit width",
+			bit_width);
+		ret_val = -EFAULT;
+		break;
+	}
+	return ret_val;
+}
+
+/**
+ * pcc_map_interrupt - Map a PCC subspace GSI to a linux IRQ number
+ * @interrupt: GSI number.
+ * @flags: interrupt flags
+ *
+ * Returns: a valid linux IRQ number on success
+ *		0 or -EINVAL on failure
+ */
+static int pcc_map_interrupt(u32 interrupt, u32 flags)
+{
+	int trigger, polarity;
+
+	if (!interrupt)
+		return 0;
+
+	trigger = (flags & ACPI_PCCT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
+			: ACPI_LEVEL_SENSITIVE;
+
+	polarity = (flags & ACPI_PCCT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
+			: ACPI_ACTIVE_HIGH;
+
+	return acpi_register_gsi(NULL, interrupt, trigger, polarity);
+}
+
+/**
+ * pcc_mbox_irq - PCC mailbox interrupt handler
+ */
+static irqreturn_t pcc_mbox_irq(int irq, void *p)
+{
+	struct acpi_generic_address *doorbell_ack;
+	struct acpi_pcct_hw_reduced *pcct_ss;
+	struct mbox_chan *chan = p;
+	u64 doorbell_ack_preserve;
+	u64 doorbell_ack_write;
+	u64 doorbell_ack_val;
+	int ret;
+
+	pcct_ss = chan->con_priv;
+
+	mbox_chan_received_data(chan, NULL);
+
+	if (pcct_ss->header.type == ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2) {
+		struct acpi_pcct_hw_reduced_type2 *pcct2_ss = chan->con_priv;
+		u32 id = chan - pcc_mbox_channels;
+
+		doorbell_ack = &pcct2_ss->doorbell_ack_register;
+		doorbell_ack_preserve = pcct2_ss->ack_preserve_mask;
+		doorbell_ack_write = pcct2_ss->ack_write_mask;
+
+		ret = read_register(pcc_doorbell_ack_vaddr[id],
+				    &doorbell_ack_val,
+				    doorbell_ack->bit_width);
+		if (ret)
+			return IRQ_NONE;
+
+		ret = write_register(pcc_doorbell_ack_vaddr[id],
+				     (doorbell_ack_val & doorbell_ack_preserve)
+					| doorbell_ack_write,
+				     doorbell_ack->bit_width);
+		if (ret)
+			return IRQ_NONE;
+	}
+
+	return IRQ_HANDLED;
+}
+
 /**
  * pcc_mbox_request_channel - PCC clients call this function to
  *		request a pointer to their PCC subspace, from which they
@@ -135,6 +267,18 @@ struct mbox_chan *pcc_mbox_request_channel(struct mbox_client *cl,
 	if (chan->txdone_method == TXDONE_BY_POLL && cl->knows_txdone)
 		chan->txdone_method |= TXDONE_BY_ACK;
 
+	if (pcc_doorbell_irq[subspace_id] > 0) {
+		int rc;
+
+		rc = devm_request_irq(dev, pcc_doorbell_irq[subspace_id],
+				      pcc_mbox_irq, 0, MBOX_IRQ_NAME, chan);
+		if (unlikely(rc)) {
+			dev_err(dev, "failed to register PCC interrupt %d\n",
+				pcc_doorbell_irq[subspace_id]);
+			chan = ERR_PTR(rc);
+		}
+	}
+
 	spin_unlock_irqrestore(&chan->lock, flags);
 
 	return chan;
@@ -149,80 +293,30 @@ EXPORT_SYMBOL_GPL(pcc_mbox_request_channel);
  */
 void pcc_mbox_free_channel(struct mbox_chan *chan)
 {
+	u32 id = chan - pcc_mbox_channels;
 	unsigned long flags;
 
 	if (!chan || !chan->cl)
 		return;
 
+	if (id >= pcc_mbox_ctrl.num_chans) {
+		pr_debug("pcc_mbox_free_channel: Invalid mbox_chan passed\n");
+		return;
+	}
+
 	spin_lock_irqsave(&chan->lock, flags);
 	chan->cl = NULL;
 	chan->active_req = NULL;
 	if (chan->txdone_method == (TXDONE_BY_POLL | TXDONE_BY_ACK))
 		chan->txdone_method = TXDONE_BY_POLL;
 
+	if (pcc_doorbell_irq[id] > 0)
+		devm_free_irq(chan->mbox->dev, pcc_doorbell_irq[id], chan);
+
 	spin_unlock_irqrestore(&chan->lock, flags);
 }
 EXPORT_SYMBOL_GPL(pcc_mbox_free_channel);
 
-/*
- * PCC can be used with perf critical drivers such as CPPC
- * So it makes sense to locally cache the virtual address and
- * use it to read/write to PCC registers such as doorbell register
- *
- * The below read_register and write_registers are used to read and
- * write from perf critical registers such as PCC doorbell register
- */
-static int read_register(void __iomem *vaddr, u64 *val, unsigned int bit_width)
-{
-	int ret_val = 0;
-
-	switch (bit_width) {
-	case 8:
-		*val = readb(vaddr);
-		break;
-	case 16:
-		*val = readw(vaddr);
-		break;
-	case 32:
-		*val = readl(vaddr);
-		break;
-	case 64:
-		*val = readq(vaddr);
-		break;
-	default:
-		pr_debug("Error: Cannot read register of %u bit width",
-			bit_width);
-		ret_val = -EFAULT;
-		break;
-	}
-	return ret_val;
-}
-
-static int write_register(void __iomem *vaddr, u64 val, unsigned int bit_width)
-{
-	int ret_val = 0;
-
-	switch (bit_width) {
-	case 8:
-		writeb(val, vaddr);
-		break;
-	case 16:
-		writew(val, vaddr);
-		break;
-	case 32:
-		writel(val, vaddr);
-		break;
-	case 64:
-		writeq(val, vaddr);
-		break;
-	default:
-		pr_debug("Error: Cannot write register of %u bit width",
-			bit_width);
-		ret_val = -EFAULT;
-		break;
-	}
-	return ret_val;
-}
 
 /**
  * pcc_send_data - Called from Mailbox Controller code. Used
@@ -296,8 +390,10 @@ static int parse_pcc_subspace(struct acpi_subtable_header *header,
 	if (pcc_mbox_ctrl.num_chans <= MAX_PCC_SUBSPACES) {
 		pcct_ss = (struct acpi_pcct_hw_reduced *) header;
 
-		if (pcct_ss->header.type !=
-				ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE) {
+		if ((pcct_ss->header.type !=
+				ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE)
+		    && (pcct_ss->header.type !=
+				ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2)) {
 			pr_err("Incorrect PCC Subspace type detected\n");
 			return -EINVAL;
 		}
@@ -307,6 +403,43 @@ static int parse_pcc_subspace(struct acpi_subtable_header *header,
 }
 
 /**
+ * pcc_parse_subspace_irq - Parse the PCC IRQ and PCC ACK register
+ *		There should be one entry per PCC client.
+ * @id: PCC subspace index.
+ * @pcct_ss: Pointer to the ACPI subtable header under the PCCT.
+ *
+ * Return: 0 for Success, else errno.
+ *
+ * This gets called for each entry in the PCC table.
+ */
+static int pcc_parse_subspace_irq(int id,
+				  struct acpi_pcct_hw_reduced *pcct_ss)
+{
+	pcc_doorbell_irq[id] = pcc_map_interrupt(pcct_ss->doorbell_interrupt,
+						 (u32)pcct_ss->flags);
+	if (pcc_doorbell_irq[id] <= 0) {
+		pr_err("PCC GSI %d not registered\n",
+		       pcct_ss->doorbell_interrupt);
+		return -EINVAL;
+	}
+
+	if (pcct_ss->header.type
+		== ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2) {
+		struct acpi_pcct_hw_reduced_type2 *pcct2_ss = (void *)pcct_ss;
+
+		pcc_doorbell_ack_vaddr[id] = acpi_os_ioremap(
+				pcct2_ss->doorbell_ack_register.address,
+				pcct2_ss->doorbell_ack_register.bit_width / 8);
+		if (!pcc_doorbell_ack_vaddr[id]) {
+			pr_err("Failed to ioremap PCC ACK register\n");
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+/**
  * acpi_pcc_probe - Parse the ACPI tree for the PCCT.
  *
  * Return: 0 for Success, else errno.
@@ -316,7 +449,9 @@ static int __init acpi_pcc_probe(void)
 	acpi_size pcct_tbl_header_size;
 	struct acpi_table_header *pcct_tbl;
 	struct acpi_subtable_header *pcct_entry;
-	int count, i;
+	struct acpi_table_pcct *acpi_pcct_tbl;
+	int count, i, rc;
+	int sum = 0;
 	acpi_status status = AE_OK;
 
 	/* Search for PCCT */
@@ -333,37 +468,66 @@ static int __init acpi_pcc_probe(void)
 			sizeof(struct acpi_table_pcct),
 			ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE,
 			parse_pcc_subspace, MAX_PCC_SUBSPACES);
+	sum += (count > 0) ? count : 0;
+
+	count = acpi_table_parse_entries(ACPI_SIG_PCCT,
+			sizeof(struct acpi_table_pcct),
+			ACPI_PCCT_TYPE_HW_REDUCED_SUBSPACE_TYPE2,
+			parse_pcc_subspace, MAX_PCC_SUBSPACES);
+	sum += (count > 0) ? count : 0;
 
-	if (count <= 0) {
+	if (sum == 0 || sum >= MAX_PCC_SUBSPACES) {
 		pr_err("Error parsing PCC subspaces from PCCT\n");
 		return -EINVAL;
 	}
 
 	pcc_mbox_channels = kzalloc(sizeof(struct mbox_chan) *
-			count, GFP_KERNEL);
-
+			sum, GFP_KERNEL);
 	if (!pcc_mbox_channels) {
 		pr_err("Could not allocate space for PCC mbox channels\n");
 		return -ENOMEM;
 	}
 
-	pcc_doorbell_vaddr = kcalloc(count, sizeof(void *), GFP_KERNEL);
+	pcc_doorbell_vaddr = kcalloc(sum, sizeof(void *), GFP_KERNEL);
 	if (!pcc_doorbell_vaddr) {
-		kfree(pcc_mbox_channels);
-		return -ENOMEM;
+		rc = -ENOMEM;
+		goto err_free_mbox;
+	}
+
+	pcc_doorbell_ack_vaddr = kcalloc(sum, sizeof(void *), GFP_KERNEL);
+	if (!pcc_doorbell_ack_vaddr) {
+		rc = -ENOMEM;
+		goto err_free_db_vaddr;
+	}
+
+	pcc_doorbell_irq = kcalloc(sum, sizeof(int), GFP_KERNEL);
+	if (!pcc_doorbell_irq) {
+		rc = -ENOMEM;
+		goto err_free_db_ack_vaddr;
 	}
 
 	/* Point to the first PCC subspace entry */
 	pcct_entry = (struct acpi_subtable_header *) (
 		(unsigned long) pcct_tbl + sizeof(struct acpi_table_pcct));
 
-	for (i = 0; i < count; i++) {
+	acpi_pcct_tbl = (struct acpi_table_pcct *) pcct_tbl;
+	if (acpi_pcct_tbl->flags & ACPI_PCCT_DOORBELL)
+		pcc_mbox_ctrl.txdone_irq = true;
+
+	for (i = 0; i < sum; i++) {
 		struct acpi_generic_address *db_reg;
 		struct acpi_pcct_hw_reduced *pcct_ss;
 		pcc_mbox_channels[i].con_priv = pcct_entry;
 
+		pcct_ss = (struct acpi_pcct_hw_reduced *) pcct_entry;
+
+		if (pcc_mbox_ctrl.txdone_irq) {
+			rc = pcc_parse_subspace_irq(i, pcct_ss);
+			if (rc < 0)
+				goto err;
+		}
+
 		/* If doorbell is in system memory cache the virt address */
-		pcct_ss = (struct acpi_pcct_hw_reduced *)pcct_entry;
 		db_reg = &pcct_ss->doorbell_register;
 		if (db_reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY)
 			pcc_doorbell_vaddr[i] = acpi_os_ioremap(db_reg->address,
@@ -372,11 +536,21 @@ static int __init acpi_pcc_probe(void)
 			((unsigned long) pcct_entry + pcct_entry->length);
 	}
 
-	pcc_mbox_ctrl.num_chans = count;
+	pcc_mbox_ctrl.num_chans = sum;
 
 	pr_info("Detected %d PCC Subspaces\n", pcc_mbox_ctrl.num_chans);
 
 	return 0;
+
+err:
+	kfree(pcc_doorbell_irq);
+err_free_db_ack_vaddr:
+	kfree(pcc_doorbell_ack_vaddr);
+err_free_db_vaddr:
+	kfree(pcc_doorbell_vaddr);
+err_free_mbox:
+	kfree(pcc_mbox_channels);
+	return rc;
 }
 
 /**
diff --git a/drivers/mailbox/platform_mhu.c b/drivers/mailbox/platform_mhu.c
new file mode 100644
index 000000000000..e13201a5cec6
--- /dev/null
+++ b/drivers/mailbox/platform_mhu.c
@@ -0,0 +1,205 @@
+/*
+ * Copyright (C) 2016 BayLibre SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Synchronised with arm_mhu.c from :
+ * Copyright (C) 2013-2015 Fujitsu Semiconductor Ltd.
+ * Copyright (C) 2015 Linaro Ltd.
+ * Author: Jassi Brar <jaswinder.singh@linaro.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/mailbox_controller.h>
+
+#define INTR_SET_OFS	0x0
+#define INTR_STAT_OFS	0x4
+#define INTR_CLR_OFS	0x8
+
+#define MHU_SEC_OFFSET	0x0
+#define MHU_LP_OFFSET	0xc
+#define MHU_HP_OFFSET	0x18
+#define TX_REG_OFFSET	0x24
+
+#define MHU_CHANS	3
+
+struct platform_mhu_link {
+	int irq;
+	void __iomem *tx_reg;
+	void __iomem *rx_reg;
+};
+
+struct platform_mhu {
+	void __iomem *base;
+	struct platform_mhu_link mlink[MHU_CHANS];
+	struct mbox_chan chan[MHU_CHANS];
+	struct mbox_controller mbox;
+};
+
+static irqreturn_t platform_mhu_rx_interrupt(int irq, void *p)
+{
+	struct mbox_chan *chan = p;
+	struct platform_mhu_link *mlink = chan->con_priv;
+	u32 val;
+
+	val = readl_relaxed(mlink->rx_reg + INTR_STAT_OFS);
+	if (!val)
+		return IRQ_NONE;
+
+	mbox_chan_received_data(chan, (void *)&val);
+
+	writel_relaxed(val, mlink->rx_reg + INTR_CLR_OFS);
+
+	return IRQ_HANDLED;
+}
+
+static bool platform_mhu_last_tx_done(struct mbox_chan *chan)
+{
+	struct platform_mhu_link *mlink = chan->con_priv;
+	u32 val = readl_relaxed(mlink->tx_reg + INTR_STAT_OFS);
+
+	return (val == 0);
+}
+
+static int platform_mhu_send_data(struct mbox_chan *chan, void *data)
+{
+	struct platform_mhu_link *mlink = chan->con_priv;
+	u32 *arg = data;
+
+	writel_relaxed(*arg, mlink->tx_reg + INTR_SET_OFS);
+
+	return 0;
+}
+
+static int platform_mhu_startup(struct mbox_chan *chan)
+{
+	struct platform_mhu_link *mlink = chan->con_priv;
+	u32 val;
+	int ret;
+
+	val = readl_relaxed(mlink->tx_reg + INTR_STAT_OFS);
+	writel_relaxed(val, mlink->tx_reg + INTR_CLR_OFS);
+
+	ret = request_irq(mlink->irq, platform_mhu_rx_interrupt,
+			  IRQF_SHARED, "platform_mhu_link", chan);
+	if (ret) {
+		dev_err(chan->mbox->dev,
+			"Unable to acquire IRQ %d\n", mlink->irq);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void platform_mhu_shutdown(struct mbox_chan *chan)
+{
+	struct platform_mhu_link *mlink = chan->con_priv;
+
+	free_irq(mlink->irq, chan);
+}
+
+static const struct mbox_chan_ops platform_mhu_ops = {
+	.send_data = platform_mhu_send_data,
+	.startup = platform_mhu_startup,
+	.shutdown = platform_mhu_shutdown,
+	.last_tx_done = platform_mhu_last_tx_done,
+};
+
+static int platform_mhu_probe(struct platform_device *pdev)
+{
+	int i, err;
+	struct platform_mhu *mhu;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	int platform_mhu_reg[MHU_CHANS] = {
+		MHU_SEC_OFFSET, MHU_LP_OFFSET, MHU_HP_OFFSET
+	};
+
+	/* Allocate memory for device */
+	mhu = devm_kzalloc(dev, sizeof(*mhu), GFP_KERNEL);
+	if (!mhu)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	mhu->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(mhu->base)) {
+		dev_err(dev, "ioremap failed\n");
+		return PTR_ERR(mhu->base);
+	}
+
+	for (i = 0; i < MHU_CHANS; i++) {
+		mhu->chan[i].con_priv = &mhu->mlink[i];
+		mhu->mlink[i].irq = platform_get_irq(pdev, i);
+		if (mhu->mlink[i].irq < 0) {
+			dev_err(dev, "failed to get irq%d\n", i);
+			return mhu->mlink[i].irq;
+		}
+		mhu->mlink[i].rx_reg = mhu->base + platform_mhu_reg[i];
+		mhu->mlink[i].tx_reg = mhu->mlink[i].rx_reg + TX_REG_OFFSET;
+	}
+
+	mhu->mbox.dev = dev;
+	mhu->mbox.chans = &mhu->chan[0];
+	mhu->mbox.num_chans = MHU_CHANS;
+	mhu->mbox.ops = &platform_mhu_ops;
+	mhu->mbox.txdone_irq = false;
+	mhu->mbox.txdone_poll = true;
+	mhu->mbox.txpoll_period = 1;
+
+	platform_set_drvdata(pdev, mhu);
+
+	err = mbox_controller_register(&mhu->mbox);
+	if (err) {
+		dev_err(dev, "Failed to register mailboxes %d\n", err);
+		return err;
+	}
+
+	dev_info(dev, "Platform MHU Mailbox registered\n");
+	return 0;
+}
+
+static int platform_mhu_remove(struct platform_device *pdev)
+{
+	struct platform_mhu *mhu = platform_get_drvdata(pdev);
+
+	mbox_controller_unregister(&mhu->mbox);
+
+	return 0;
+}
+
+static const struct of_device_id platform_mhu_dt_ids[] = {
+	{ .compatible = "amlogic,meson-gxbb-mhu", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, platform_mhu_dt_ids);
+
+static struct platform_driver platform_mhu_driver = {
+	.probe	= platform_mhu_probe,
+	.remove	= platform_mhu_remove,
+	.driver = {
+		.name = "platform-mhu",
+		.of_match_table	= platform_mhu_dt_ids,
+	},
+};
+
+module_platform_driver(platform_mhu_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:platform-mhu");
+MODULE_DESCRIPTION("Platform MHU Driver");
+MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
diff --git a/drivers/mcb/Kconfig b/drivers/mcb/Kconfig
index e9a6976e1010..76d9c51de6c9 100644
--- a/drivers/mcb/Kconfig
+++ b/drivers/mcb/Kconfig
@@ -28,4 +28,13 @@ config MCB_PCI
 
 	   If build as a module, the module is called mcb-pci.ko
 
+config MCB_LPC
+	   tristate "LPC (non PCI) based MCB carrier"
+	   default n
+	   help
+
+	   This is a MCB carrier on a LPC or non PCI device.
+
+	   If build as a module, the module is called mcb-lpc.ko
+
 endif # MCB
diff --git a/drivers/mcb/Makefile b/drivers/mcb/Makefile
index 1ae141311def..bcc7745774ab 100644
--- a/drivers/mcb/Makefile
+++ b/drivers/mcb/Makefile
@@ -5,3 +5,4 @@ mcb-y += mcb-core.o
 mcb-y += mcb-parse.o
 
 obj-$(CONFIG_MCB_PCI) += mcb-pci.o
+obj-$(CONFIG_MCB_LPC) += mcb-lpc.o
diff --git a/drivers/mcb/mcb-core.c b/drivers/mcb/mcb-core.c
index 6f2c8522e14a..921a5d2a802b 100644
--- a/drivers/mcb/mcb-core.c
+++ b/drivers/mcb/mcb-core.c
@@ -233,6 +233,7 @@ int mcb_device_register(struct mcb_bus *bus, struct mcb_device *dev)
 	dev->dev.bus = &mcb_bus_type;
 	dev->dev.parent = bus->dev.parent;
 	dev->dev.release = mcb_release_dev;
+	dev->dma_dev = bus->carrier;
 
 	device_id = dev->id;
 	dev_set_name(&dev->dev, "mcb%d-16z%03d-%d:%d:%d",
@@ -369,7 +370,6 @@ struct mcb_device *mcb_alloc_dev(struct mcb_bus *bus)
 	if (!dev)
 		return NULL;
 
-	INIT_LIST_HEAD(&dev->bus_list);
 	dev->bus = bus;
 
 	return dev;
@@ -405,20 +405,6 @@ static int __mcb_bus_add_devices(struct device *dev, void *data)
 	return 0;
 }
 
-static int __mcb_bus_add_child(struct device *dev, void *data)
-{
-	struct mcb_device *mdev = to_mcb_device(dev);
-	struct mcb_bus *child;
-
-	BUG_ON(!mdev->is_added);
-	child = mdev->subordinate;
-
-	if (child)
-		mcb_bus_add_devices(child);
-
-	return 0;
-}
-
 /**
  * mcb_bus_add_devices() - Add devices in the bus' internal device list
  * @bus: The @mcb_bus we add the devices
@@ -428,8 +414,6 @@ static int __mcb_bus_add_child(struct device *dev, void *data)
 void mcb_bus_add_devices(const struct mcb_bus *bus)
 {
 	bus_for_each_dev(&mcb_bus_type, NULL, NULL, __mcb_bus_add_devices);
-	bus_for_each_dev(&mcb_bus_type, NULL, NULL, __mcb_bus_add_child);
-
 }
 EXPORT_SYMBOL_GPL(mcb_bus_add_devices);
 
diff --git a/drivers/mcb/mcb-internal.h b/drivers/mcb/mcb-internal.h
index 5254e0285725..d6e6933b19f1 100644
--- a/drivers/mcb/mcb-internal.h
+++ b/drivers/mcb/mcb-internal.h
@@ -112,6 +112,15 @@ struct chameleon_bdd {
 	u32 size;
 } __packed;
 
+struct chameleon_bar {
+	u32 addr;
+	u32 size;
+};
+
+#define BAR_CNT(x) ((x) & 0x07)
+#define CHAMELEON_BAR_MAX	6
+#define BAR_DESC_SIZE(x)	((x) * sizeof(struct chameleon_bar) + sizeof(__le32))
+
 int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 			  void __iomem *base);
 
diff --git a/drivers/mcb/mcb-lpc.c b/drivers/mcb/mcb-lpc.c
new file mode 100644
index 000000000000..d072c088ce73
--- /dev/null
+++ b/drivers/mcb/mcb-lpc.c
@@ -0,0 +1,158 @@
+/*
+ * MEN Chameleon Bus.
+ *
+ * Copyright (C) 2014 MEN Mikroelektronik GmbH (www.men.de)
+ * Author: Andreas Werner <andreas.werner@men.de>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/dmi.h>
+#include <linux/mcb.h>
+#include <linux/io.h>
+#include "mcb-internal.h"
+
+struct priv {
+	struct mcb_bus *bus;
+	struct resource *mem;
+	void __iomem *base;
+};
+
+static int mcb_lpc_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct priv *priv;
+	int ret = 0;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!priv->mem) {
+		dev_err(&pdev->dev, "No Memory resource\n");
+		return -ENODEV;
+	}
+
+	res = devm_request_mem_region(&pdev->dev, priv->mem->start,
+				      resource_size(priv->mem),
+				      KBUILD_MODNAME);
+	if (!res) {
+		dev_err(&pdev->dev, "Failed to request IO memory\n");
+		return -EBUSY;
+	}
+
+	priv->base = devm_ioremap(&pdev->dev, priv->mem->start,
+				  resource_size(priv->mem));
+	if (!priv->base) {
+		dev_err(&pdev->dev, "Cannot ioremap\n");
+		return -ENOMEM;
+	}
+
+	platform_set_drvdata(pdev, priv);
+
+	priv->bus = mcb_alloc_bus(&pdev->dev);
+	if (IS_ERR(priv->bus))
+		return PTR_ERR(priv->bus);
+
+	ret = chameleon_parse_cells(priv->bus, priv->mem->start, priv->base);
+	if (ret < 0) {
+		mcb_release_bus(priv->bus);
+		return ret;
+	}
+
+	dev_dbg(&pdev->dev, "Found %d cells\n", ret);
+
+	mcb_bus_add_devices(priv->bus);
+
+	return 0;
+
+}
+
+static int mcb_lpc_remove(struct platform_device *pdev)
+{
+	struct priv *priv = platform_get_drvdata(pdev);
+
+	mcb_release_bus(priv->bus);
+
+	return 0;
+}
+
+static struct platform_device *mcb_lpc_pdev;
+
+static int mcb_lpc_create_platform_device(const struct dmi_system_id *id)
+{
+	struct resource *res = id->driver_data;
+	int ret;
+
+	mcb_lpc_pdev = platform_device_alloc("mcb-lpc", -1);
+	if (!mcb_lpc_pdev)
+		return -ENOMEM;
+
+	ret = platform_device_add_resources(mcb_lpc_pdev, res, 1);
+	if (ret)
+		goto out_put;
+
+	ret = platform_device_add(mcb_lpc_pdev);
+	if (ret)
+		goto out_put;
+
+	return 0;
+
+out_put:
+	platform_device_put(mcb_lpc_pdev);
+	return ret;
+}
+
+static struct resource sc24_fpga_resource = {
+	.start = 0xe000e000,
+	.end = 0xe000e000 + CHAM_HEADER_SIZE,
+	.flags = IORESOURCE_MEM,
+};
+
+static struct platform_driver mcb_lpc_driver = {
+	.driver		= {
+		.name = "mcb-lpc",
+	},
+	.probe		= mcb_lpc_probe,
+	.remove		= mcb_lpc_remove,
+};
+
+static const struct dmi_system_id mcb_lpc_dmi_table[] = {
+	{
+		.ident = "SC24",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "MEN"),
+			DMI_MATCH(DMI_PRODUCT_VERSION, "14SC24"),
+		},
+		.driver_data = (void *)&sc24_fpga_resource,
+		.callback = mcb_lpc_create_platform_device,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(dmi, mcb_lpc_dmi_table);
+
+static int __init mcb_lpc_init(void)
+{
+	if (!dmi_check_system(mcb_lpc_dmi_table))
+		return -ENODEV;
+
+	return platform_driver_register(&mcb_lpc_driver);
+}
+
+static void __exit mcb_lpc_exit(void)
+{
+	platform_device_unregister(mcb_lpc_pdev);
+	platform_driver_unregister(&mcb_lpc_driver);
+}
+
+module_init(mcb_lpc_init);
+module_exit(mcb_lpc_exit);
+
+MODULE_AUTHOR("Andreas Werner <andreas.werner@men.de>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("MCB over LPC support");
diff --git a/drivers/mcb/mcb-parse.c b/drivers/mcb/mcb-parse.c
index dbecbed0d258..4ca2739b4fad 100644
--- a/drivers/mcb/mcb-parse.c
+++ b/drivers/mcb/mcb-parse.c
@@ -26,19 +26,20 @@ static inline uint32_t get_next_dtype(void __iomem *p)
 }
 
 static int chameleon_parse_bdd(struct mcb_bus *bus,
-			phys_addr_t mapbase,
+			struct chameleon_bar *cb,
 			void __iomem *base)
 {
 	return 0;
 }
 
 static int chameleon_parse_gdd(struct mcb_bus *bus,
-			phys_addr_t mapbase,
-			void __iomem *base)
+			struct chameleon_bar *cb,
+			void __iomem *base, int bar_count)
 {
 	struct chameleon_gdd __iomem *gdd =
 		(struct chameleon_gdd __iomem *) base;
 	struct mcb_device *mdev;
+	u32 dev_mapbase;
 	u32 offset;
 	u32 size;
 	int ret;
@@ -61,13 +62,39 @@ static int chameleon_parse_gdd(struct mcb_bus *bus,
 	mdev->group = GDD_GRP(reg2);
 	mdev->inst = GDD_INS(reg2);
 
+	/*
+	 * If the BAR is missing, dev_mapbase is zero, or if the
+	 * device is IO mapped we just print a warning and go on with the
+	 * next device, instead of completely stop the gdd parser
+	 */
+	if (mdev->bar > bar_count - 1) {
+		pr_info("No BAR for 16z%03d\n", mdev->id);
+		ret = 0;
+		goto err;
+	}
+
+	dev_mapbase = cb[mdev->bar].addr;
+	if (!dev_mapbase) {
+		pr_info("BAR not assigned for 16z%03d\n", mdev->id);
+		ret = 0;
+		goto err;
+	}
+
+	if (dev_mapbase & 0x01) {
+		pr_info("IO mapped Device (16z%03d) not yet supported\n",
+			mdev->id);
+		ret = 0;
+		goto err;
+	}
+
 	pr_debug("Found a 16z%03d\n", mdev->id);
 
 	mdev->irq.start = GDD_IRQ(reg1);
 	mdev->irq.end = GDD_IRQ(reg1);
 	mdev->irq.flags = IORESOURCE_IRQ;
 
-	mdev->mem.start = mapbase + offset;
+	mdev->mem.start = dev_mapbase + offset;
+
 	mdev->mem.end = mdev->mem.start + size - 1;
 	mdev->mem.flags = IORESOURCE_MEM;
 
@@ -85,13 +112,76 @@ err:
 	return ret;
 }
 
+static void chameleon_parse_bar(void __iomem *base,
+				struct chameleon_bar *cb, int bar_count)
+{
+	char __iomem *p = base;
+	int i;
+
+	/* skip reg1 */
+	p += sizeof(__le32);
+
+	for (i = 0; i < bar_count; i++) {
+		cb[i].addr = readl(p);
+		cb[i].size = readl(p + 4);
+
+		p += sizeof(struct chameleon_bar);
+	}
+}
+
+static int chameleon_get_bar(char __iomem **base, phys_addr_t mapbase,
+			     struct chameleon_bar **cb)
+{
+	struct chameleon_bar *c;
+	int bar_count;
+	__le32 reg;
+	u32 dtype;
+
+	/*
+	 * For those devices which are not connected
+	 * to the PCI Bus (e.g. LPC) there is a bar
+	 * descriptor located directly after the
+	 * chameleon header. This header is comparable
+	 * to a PCI header.
+	 */
+	dtype = get_next_dtype(*base);
+	if (dtype == CHAMELEON_DTYPE_BAR) {
+		reg = readl(*base);
+
+		bar_count = BAR_CNT(reg);
+		if (bar_count <= 0 && bar_count > CHAMELEON_BAR_MAX)
+			return -ENODEV;
+
+		c = kcalloc(bar_count, sizeof(struct chameleon_bar),
+			    GFP_KERNEL);
+		if (!c)
+			return -ENOMEM;
+
+		chameleon_parse_bar(*base, c, bar_count);
+		*base += BAR_DESC_SIZE(bar_count);
+	} else {
+		c = kzalloc(sizeof(struct chameleon_bar), GFP_KERNEL);
+		if (!c)
+			return -ENOMEM;
+
+		bar_count = 1;
+		c->addr = mapbase;
+	}
+
+	*cb = c;
+
+	return bar_count;
+}
+
 int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 			void __iomem *base)
 {
-	char __iomem *p = base;
 	struct chameleon_fpga_header *header;
-	uint32_t dtype;
+	struct chameleon_bar *cb;
+	char __iomem *p = base;
 	int num_cells = 0;
+	uint32_t dtype;
+	int bar_count;
 	int ret = 0;
 	u32 hsize;
 
@@ -108,8 +198,8 @@ int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 	if (header->magic != CHAMELEONV2_MAGIC) {
 		pr_err("Unsupported chameleon version 0x%x\n",
 				header->magic);
-		kfree(header);
-		return -ENODEV;
+		ret = -ENODEV;
+		goto free_header;
 	}
 	p += hsize;
 
@@ -119,16 +209,20 @@ int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 	snprintf(bus->name, CHAMELEON_FILENAME_LEN + 1, "%s",
 		 header->filename);
 
+	bar_count = chameleon_get_bar(&p, mapbase, &cb);
+	if (bar_count < 0)
+		goto free_header;
+
 	for_each_chameleon_cell(dtype, p) {
 		switch (dtype) {
 		case CHAMELEON_DTYPE_GENERAL:
-			ret = chameleon_parse_gdd(bus, mapbase, p);
+			ret = chameleon_parse_gdd(bus, cb, p, bar_count);
 			if (ret < 0)
-				goto out;
+				goto free_bar;
 			p += sizeof(struct chameleon_gdd);
 			break;
 		case CHAMELEON_DTYPE_BRIDGE:
-			chameleon_parse_bdd(bus, mapbase, p);
+			chameleon_parse_bdd(bus, cb, p);
 			p += sizeof(struct chameleon_bdd);
 			break;
 		case CHAMELEON_DTYPE_END:
@@ -136,8 +230,8 @@ int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 		default:
 			pr_err("Invalid chameleon descriptor type 0x%x\n",
 				dtype);
-			kfree(header);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto free_bar;
 		}
 		num_cells++;
 	}
@@ -145,11 +239,15 @@ int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 	if (num_cells == 0)
 		num_cells = -EINVAL;
 
+	kfree(cb);
 	kfree(header);
 	return num_cells;
 
-out:
+free_bar:
+	kfree(cb);
+free_header:
 	kfree(header);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(chameleon_parse_cells);
diff --git a/drivers/mcb/mcb-pci.c b/drivers/mcb/mcb-pci.c
index b15a0349cd97..af4d2f26f1c6 100644
--- a/drivers/mcb/mcb-pci.c
+++ b/drivers/mcb/mcb-pci.c
@@ -46,6 +46,7 @@ static int mcb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		dev_err(&pdev->dev, "Failed to enable PCI device\n");
 		return -ENODEV;
 	}
+	pci_set_master(pdev);
 
 	priv->mapbase = pci_resource_start(pdev, 0);
 	if (!priv->mapbase) {
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index 76f7534d1dd1..81d3db40cd7b 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -361,12 +361,8 @@ static void __btree_node_write_done(struct closure *cl)
 static void btree_node_write_done(struct closure *cl)
 {
 	struct btree *b = container_of(cl, struct btree, io);
-	struct bio_vec *bv;
-	int n;
-
-	bio_for_each_segment_all(bv, b->bio, n)
-		__free_page(bv->bv_page);
 
+	bio_free_pages(b->bio);
 	__btree_node_write_done(cl);
 }
 
diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c
index c28df164701e..333a1e5f6ae6 100644
--- a/drivers/md/bcache/debug.c
+++ b/drivers/md/bcache/debug.c
@@ -107,9 +107,8 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio)
 {
 	char name[BDEVNAME_SIZE];
 	struct bio *check;
-	struct bio_vec bv, *bv2;
+	struct bio_vec bv;
 	struct bvec_iter iter;
-	int i;
 
 	check = bio_clone(bio, GFP_NOIO);
 	if (!check)
@@ -136,8 +135,7 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio)
 		kunmap_atomic(p1);
 	}
 
-	bio_for_each_segment_all(bv2, check, i)
-		__free_page(bv2->bv_page);
+	bio_free_pages(check);
 out_put:
 	bio_put(check);
 }
diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c
index 1881319f2298..5c4bddecfaf0 100644
--- a/drivers/md/bcache/movinggc.c
+++ b/drivers/md/bcache/movinggc.c
@@ -44,11 +44,8 @@ static void write_moving_finish(struct closure *cl)
 {
 	struct moving_io *io = container_of(cl, struct moving_io, cl);
 	struct bio *bio = &io->bio.bio;
-	struct bio_vec *bv;
-	int i;
 
-	bio_for_each_segment_all(bv, bio, i)
-		__free_page(bv->bv_page);
+	bio_free_pages(bio);
 
 	if (io->op.replace_collision)
 		trace_bcache_gc_copy_collision(&io->w->key);
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 4b177fe11ebb..40ffe5e424b3 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -694,13 +694,8 @@ static void cached_dev_cache_miss_done(struct closure *cl)
 	if (s->iop.replace_collision)
 		bch_mark_cache_miss_collision(s->iop.c, s->d);
 
-	if (s->iop.bio) {
-		int i;
-		struct bio_vec *bv;
-
-		bio_for_each_segment_all(bv, s->iop.bio, i)
-			__free_page(bv->bv_page);
-	}
+	if (s->iop.bio)
+		bio_free_pages(s->iop.bio);
 
 	cached_dev_bio_complete(cl);
 }
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 95a4ca6ce6ff..849ad441cd76 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -760,7 +760,8 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size,
 	if (!d->nr_stripes ||
 	    d->nr_stripes > INT_MAX ||
 	    d->nr_stripes > SIZE_MAX / sizeof(atomic_t)) {
-		pr_err("nr_stripes too large");
+		pr_err("nr_stripes too large or invalid: %u (start sector beyond end of disk?)",
+			(unsigned)d->nr_stripes);
 		return -ENOMEM;
 	}
 
@@ -1820,7 +1821,7 @@ static int cache_alloc(struct cache *ca)
 	free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;
 
 	if (!init_fifo(&ca->free[RESERVE_BTREE], 8, GFP_KERNEL) ||
-	    !init_fifo(&ca->free[RESERVE_PRIO], prio_buckets(ca), GFP_KERNEL) ||
+	    !init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca), GFP_KERNEL) ||
 	    !init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL) ||
 	    !init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL) ||
 	    !init_fifo(&ca->free_inc,	free << 2, GFP_KERNEL) ||
@@ -1844,7 +1845,7 @@ static int register_cache(struct cache_sb *sb, struct page *sb_page,
 				struct block_device *bdev, struct cache *ca)
 {
 	char name[BDEVNAME_SIZE];
-	const char *err = NULL;
+	const char *err = NULL; /* must be set for any error case */
 	int ret = 0;
 
 	memcpy(&ca->sb, sb, sizeof(struct cache_sb));
@@ -1861,8 +1862,13 @@ static int register_cache(struct cache_sb *sb, struct page *sb_page,
 		ca->discard = CACHE_DISCARD(&ca->sb);
 
 	ret = cache_alloc(ca);
-	if (ret != 0)
+	if (ret != 0) {
+		if (ret == -ENOMEM)
+			err = "cache_alloc(): -ENOMEM";
+		else
+			err = "cache_alloc(): unknown error";
 		goto err;
+	}
 
 	if (kobject_add(&ca->kobj, &part_to_dev(bdev->bd_part)->kobj, "bcache")) {
 		err = "error calling kobject_add";
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index d9fd2a62e5f6..e51644e503a5 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -128,11 +128,8 @@ static void write_dirty_finish(struct closure *cl)
 	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
 	struct keybuf_key *w = io->bio.bi_private;
 	struct cached_dev *dc = io->dc;
-	struct bio_vec *bv;
-	int i;
 
-	bio_for_each_segment_all(bv, &io->bio, i)
-		__free_page(bv->bv_page);
+	bio_free_pages(&io->bio);
 
 	/* This is kind of a dumb way of signalling errors. */
 	if (KEY_DIRTY(&w->key)) {
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 6fff794e0c72..2d826927a3bf 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -1903,10 +1903,8 @@ int bitmap_copy_from_slot(struct mddev *mddev, int slot,
 	struct bitmap_counts *counts;
 	struct bitmap *bitmap = bitmap_create(mddev, slot);
 
-	if (IS_ERR(bitmap)) {
-		bitmap_free(bitmap);
+	if (IS_ERR(bitmap))
 		return PTR_ERR(bitmap);
-	}
 
 	rv = bitmap_init_from_disk(bitmap, 0);
 	if (rv)
@@ -2183,19 +2181,29 @@ location_show(struct mddev *mddev, char *page)
 static ssize_t
 location_store(struct mddev *mddev, const char *buf, size_t len)
 {
+	int rv;
 
+	rv = mddev_lock(mddev);
+	if (rv)
+		return rv;
 	if (mddev->pers) {
-		if (!mddev->pers->quiesce)
-			return -EBUSY;
-		if (mddev->recovery || mddev->sync_thread)
-			return -EBUSY;
+		if (!mddev->pers->quiesce) {
+			rv = -EBUSY;
+			goto out;
+		}
+		if (mddev->recovery || mddev->sync_thread) {
+			rv = -EBUSY;
+			goto out;
+		}
 	}
 
 	if (mddev->bitmap || mddev->bitmap_info.file ||
 	    mddev->bitmap_info.offset) {
 		/* bitmap already configured.  Only option is to clear it */
-		if (strncmp(buf, "none", 4) != 0)
-			return -EBUSY;
+		if (strncmp(buf, "none", 4) != 0) {
+			rv = -EBUSY;
+			goto out;
+		}
 		if (mddev->pers) {
 			mddev->pers->quiesce(mddev, 1);
 			bitmap_destroy(mddev);
@@ -2214,21 +2222,25 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
 			/* nothing to be done */;
 		else if (strncmp(buf, "file:", 5) == 0) {
 			/* Not supported yet */
-			return -EINVAL;
+			rv = -EINVAL;
+			goto out;
 		} else {
-			int rv;
 			if (buf[0] == '+')
 				rv = kstrtoll(buf+1, 10, &offset);
 			else
 				rv = kstrtoll(buf, 10, &offset);
 			if (rv)
-				return rv;
-			if (offset == 0)
-				return -EINVAL;
+				goto out;
+			if (offset == 0) {
+				rv = -EINVAL;
+				goto out;
+			}
 			if (mddev->bitmap_info.external == 0 &&
 			    mddev->major_version == 0 &&
-			    offset != mddev->bitmap_info.default_offset)
-				return -EINVAL;
+			    offset != mddev->bitmap_info.default_offset) {
+				rv = -EINVAL;
+				goto out;
+			}
 			mddev->bitmap_info.offset = offset;
 			if (mddev->pers) {
 				struct bitmap *bitmap;
@@ -2245,7 +2257,7 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
 				mddev->pers->quiesce(mddev, 0);
 				if (rv) {
 					bitmap_destroy(mddev);
-					return rv;
+					goto out;
 				}
 			}
 		}
@@ -2257,6 +2269,11 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
 		set_bit(MD_CHANGE_DEVS, &mddev->flags);
 		md_wakeup_thread(mddev->thread);
 	}
+	rv = 0;
+out:
+	mddev_unlock(mddev);
+	if (rv)
+		return rv;
 	return len;
 }
 
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index 6571c81465e1..125aedc3875f 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -191,19 +191,6 @@ static void dm_bufio_unlock(struct dm_bufio_client *c)
 	mutex_unlock(&c->lock);
 }
 
-/*
- * FIXME Move to sched.h?
- */
-#ifdef CONFIG_PREEMPT_VOLUNTARY
-#  define dm_bufio_cond_resched()		\
-do {						\
-	if (unlikely(need_resched()))		\
-		_cond_resched();		\
-} while (0)
-#else
-#  define dm_bufio_cond_resched()                do { } while (0)
-#endif
-
 /*----------------------------------------------------------------*/
 
 /*
@@ -741,7 +728,7 @@ static void __flush_write_list(struct list_head *write_list)
 			list_entry(write_list->next, struct dm_buffer, write_list);
 		list_del(&b->write_list);
 		submit_io(b, WRITE, b->block, write_endio);
-		dm_bufio_cond_resched();
+		cond_resched();
 	}
 	blk_finish_plug(&plug);
 }
@@ -780,7 +767,7 @@ static struct dm_buffer *__get_unclaimed_buffer(struct dm_bufio_client *c)
 			__unlink_buffer(b);
 			return b;
 		}
-		dm_bufio_cond_resched();
+		cond_resched();
 	}
 
 	list_for_each_entry_reverse(b, &c->lru[LIST_DIRTY], lru_list) {
@@ -791,7 +778,7 @@ static struct dm_buffer *__get_unclaimed_buffer(struct dm_bufio_client *c)
 			__unlink_buffer(b);
 			return b;
 		}
-		dm_bufio_cond_resched();
+		cond_resched();
 	}
 
 	return NULL;
@@ -923,7 +910,7 @@ static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait,
 			return;
 
 		__write_dirty_buffer(b, write_list);
-		dm_bufio_cond_resched();
+		cond_resched();
 	}
 }
 
@@ -973,7 +960,7 @@ static void __check_watermark(struct dm_bufio_client *c,
 			return;
 
 		__free_buffer_wake(b);
-		dm_bufio_cond_resched();
+		cond_resched();
 	}
 
 	if (c->n_buffers[LIST_DIRTY] > threshold_buffers)
@@ -1170,7 +1157,7 @@ void dm_bufio_prefetch(struct dm_bufio_client *c,
 				submit_io(b, READ, b->block, read_endio);
 			dm_bufio_release(b);
 
-			dm_bufio_cond_resched();
+			cond_resched();
 
 			if (!n_blocks)
 				goto flush_plug;
@@ -1291,7 +1278,7 @@ again:
 		    !test_bit(B_WRITING, &b->state))
 			__relink_lru(b, LIST_CLEAN);
 
-		dm_bufio_cond_resched();
+		cond_resched();
 
 		/*
 		 * If we dropped the lock, the list is no longer consistent,
@@ -1574,7 +1561,7 @@ static unsigned long __scan(struct dm_bufio_client *c, unsigned long nr_to_scan,
 				freed++;
 			if (!--nr_to_scan || ((count - freed) <= retain_target))
 				return freed;
-			dm_bufio_cond_resched();
+			cond_resched();
 		}
 	}
 	return freed;
@@ -1808,7 +1795,7 @@ static void __evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
 		if (__try_evict_buffer(b, 0))
 			count--;
 
-		dm_bufio_cond_resched();
+		cond_resched();
 	}
 
 	dm_bufio_unlock(c);
@@ -1879,7 +1866,7 @@ static int __init dm_bufio_init(void)
 	__cache_size_refresh();
 	mutex_unlock(&dm_bufio_clients_lock);
 
-	dm_bufio_wq = create_singlethread_workqueue("dm_bufio_cache");
+	dm_bufio_wq = alloc_workqueue("dm_bufio_cache", WQ_MEM_RECLAIM, 0);
 	if (!dm_bufio_wq)
 		return -ENOMEM;
 
diff --git a/drivers/md/dm-cache-metadata.c b/drivers/md/dm-cache-metadata.c
index 3970cda10080..695577812cf6 100644
--- a/drivers/md/dm-cache-metadata.c
+++ b/drivers/md/dm-cache-metadata.c
@@ -140,6 +140,13 @@ struct dm_cache_metadata {
 	 * the device.
 	 */
 	bool fail_io:1;
+
+	/*
+	 * These structures are used when loading metadata.  They're too
+	 * big to put on the stack.
+	 */
+	struct dm_array_cursor mapping_cursor;
+	struct dm_array_cursor hint_cursor;
 };
 
 /*-------------------------------------------------------------------
@@ -1171,31 +1178,37 @@ static bool hints_array_available(struct dm_cache_metadata *cmd,
 		hints_array_initialized(cmd);
 }
 
-static int __load_mapping(void *context, uint64_t cblock, void *leaf)
+static int __load_mapping(struct dm_cache_metadata *cmd,
+			  uint64_t cb, bool hints_valid,
+			  struct dm_array_cursor *mapping_cursor,
+			  struct dm_array_cursor *hint_cursor,
+			  load_mapping_fn fn, void *context)
 {
 	int r = 0;
-	bool dirty;
-	__le64 value;
-	__le32 hint_value = 0;
+
+	__le64 mapping;
+	__le32 hint = 0;
+
+	__le64 *mapping_value_le;
+	__le32 *hint_value_le;
+
 	dm_oblock_t oblock;
 	unsigned flags;
-	struct thunk *thunk = context;
-	struct dm_cache_metadata *cmd = thunk->cmd;
 
-	memcpy(&value, leaf, sizeof(value));
-	unpack_value(value, &oblock, &flags);
+	dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);
+	memcpy(&mapping, mapping_value_le, sizeof(mapping));
+	unpack_value(mapping, &oblock, &flags);
 
 	if (flags & M_VALID) {
-		if (thunk->hints_valid) {
-			r = dm_array_get_value(&cmd->hint_info, cmd->hint_root,
-					       cblock, &hint_value);
-			if (r && r != -ENODATA)
-				return r;
+		if (hints_valid) {
+			dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);
+			memcpy(&hint, hint_value_le, sizeof(hint));
 		}
 
-		dirty = thunk->respect_dirty_flags ? (flags & M_DIRTY) : true;
-		r = thunk->fn(thunk->context, oblock, to_cblock(cblock),
-			      dirty, le32_to_cpu(hint_value), thunk->hints_valid);
+		r = fn(context, oblock, to_cblock(cb), flags & M_DIRTY,
+		       le32_to_cpu(hint), hints_valid);
+		if (r)
+			DMERR("policy couldn't load cblock");
 	}
 
 	return r;
@@ -1205,16 +1218,60 @@ static int __load_mappings(struct dm_cache_metadata *cmd,
 			   struct dm_cache_policy *policy,
 			   load_mapping_fn fn, void *context)
 {
-	struct thunk thunk;
+	int r;
+	uint64_t cb;
+
+	bool hints_valid = hints_array_available(cmd, policy);
+
+	if (from_cblock(cmd->cache_blocks) == 0)
+		/* Nothing to do */
+		return 0;
+
+	r = dm_array_cursor_begin(&cmd->info, cmd->root, &cmd->mapping_cursor);
+	if (r)
+		return r;
 
-	thunk.fn = fn;
-	thunk.context = context;
+	if (hints_valid) {
+		r = dm_array_cursor_begin(&cmd->hint_info, cmd->hint_root, &cmd->hint_cursor);
+		if (r) {
+			dm_array_cursor_end(&cmd->mapping_cursor);
+			return r;
+		}
+	}
+
+	for (cb = 0; ; cb++) {
+		r = __load_mapping(cmd, cb, hints_valid,
+				   &cmd->mapping_cursor, &cmd->hint_cursor,
+				   fn, context);
+		if (r)
+			goto out;
+
+		/*
+		 * We need to break out before we move the cursors.
+		 */
+		if (cb >= (from_cblock(cmd->cache_blocks) - 1))
+			break;
 
-	thunk.cmd = cmd;
-	thunk.respect_dirty_flags = cmd->clean_when_opened;
-	thunk.hints_valid = hints_array_available(cmd, policy);
+		r = dm_array_cursor_next(&cmd->mapping_cursor);
+		if (r) {
+			DMERR("dm_array_cursor_next for mapping failed");
+			goto out;
+		}
 
-	return dm_array_walk(&cmd->info, cmd->root, __load_mapping, &thunk);
+		if (hints_valid) {
+			r = dm_array_cursor_next(&cmd->hint_cursor);
+			if (r) {
+				DMERR("dm_array_cursor_next for hint failed");
+				goto out;
+			}
+		}
+	}
+out:
+	dm_array_cursor_end(&cmd->mapping_cursor);
+	if (hints_valid)
+		dm_array_cursor_end(&cmd->hint_cursor);
+
+	return r;
 }
 
 int dm_cache_load_mappings(struct dm_cache_metadata *cmd,
@@ -1368,10 +1425,24 @@ int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,
 
 /*----------------------------------------------------------------*/
 
-static int begin_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
+static int get_hint(uint32_t index, void *value_le, void *context)
+{
+	uint32_t value;
+	struct dm_cache_policy *policy = context;
+
+	value = policy_get_hint(policy, to_cblock(index));
+	*((__le32 *) value_le) = cpu_to_le32(value);
+
+	return 0;
+}
+
+/*
+ * It's quicker to always delete the hint array, and recreate with
+ * dm_array_new().
+ */
+static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
 {
 	int r;
-	__le32 value;
 	size_t hint_size;
 	const char *policy_name = dm_cache_policy_get_name(policy);
 	const unsigned *policy_version = dm_cache_policy_get_version(policy);
@@ -1380,63 +1451,23 @@ static int begin_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *po
 	    (strlen(policy_name) > sizeof(cmd->policy_name) - 1))
 		return -EINVAL;
 
-	if (!policy_unchanged(cmd, policy)) {
-		strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));
-		memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));
-
-		hint_size = dm_cache_policy_get_hint_size(policy);
-		if (!hint_size)
-			return 0; /* short-circuit hints initialization */
-		cmd->policy_hint_size = hint_size;
+	strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));
+	memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));
 
-		if (cmd->hint_root) {
-			r = dm_array_del(&cmd->hint_info, cmd->hint_root);
-			if (r)
-				return r;
-		}
+	hint_size = dm_cache_policy_get_hint_size(policy);
+	if (!hint_size)
+		return 0; /* short-circuit hints initialization */
+	cmd->policy_hint_size = hint_size;
 
-		r = dm_array_empty(&cmd->hint_info, &cmd->hint_root);
+	if (cmd->hint_root) {
+		r = dm_array_del(&cmd->hint_info, cmd->hint_root);
 		if (r)
 			return r;
-
-		value = cpu_to_le32(0);
-		__dm_bless_for_disk(&value);
-		r = dm_array_resize(&cmd->hint_info, cmd->hint_root, 0,
-				    from_cblock(cmd->cache_blocks),
-				    &value, &cmd->hint_root);
-		if (r)
-			return r;
-	}
-
-	return 0;
-}
-
-static int save_hint(void *context, dm_cblock_t cblock, dm_oblock_t oblock, uint32_t hint)
-{
-	struct dm_cache_metadata *cmd = context;
-	__le32 value = cpu_to_le32(hint);
-	int r;
-
-	__dm_bless_for_disk(&value);
-
-	r = dm_array_set_value(&cmd->hint_info, cmd->hint_root,
-			       from_cblock(cblock), &value, &cmd->hint_root);
-	cmd->changed = true;
-
-	return r;
-}
-
-static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
-{
-	int r;
-
-	r = begin_hints(cmd, policy);
-	if (r) {
-		DMERR("begin_hints failed");
-		return r;
 	}
 
-	return policy_walk_mappings(policy, save_hint, cmd);
+	return dm_array_new(&cmd->hint_info, &cmd->hint_root,
+			    from_cblock(cmd->cache_blocks),
+			    get_hint, policy);
 }
 
 int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
diff --git a/drivers/md/dm-cache-policy-cleaner.c b/drivers/md/dm-cache-policy-cleaner.c
index 14aaaf059f06..2e8a8f1d8358 100644
--- a/drivers/md/dm-cache-policy-cleaner.c
+++ b/drivers/md/dm-cache-policy-cleaner.c
@@ -395,7 +395,7 @@ static void init_policy_functions(struct policy *p)
 	p->policy.set_dirty = wb_set_dirty;
 	p->policy.clear_dirty = wb_clear_dirty;
 	p->policy.load_mapping = wb_load_mapping;
-	p->policy.walk_mappings = NULL;
+	p->policy.get_hint = NULL;
 	p->policy.remove_mapping = wb_remove_mapping;
 	p->policy.writeback_work = wb_writeback_work;
 	p->policy.force_mapping = wb_force_mapping;
diff --git a/drivers/md/dm-cache-policy-internal.h b/drivers/md/dm-cache-policy-internal.h
index 2816018faa7f..808ee0e2b2c4 100644
--- a/drivers/md/dm-cache-policy-internal.h
+++ b/drivers/md/dm-cache-policy-internal.h
@@ -48,10 +48,10 @@ static inline int policy_load_mapping(struct dm_cache_policy *p,
 	return p->load_mapping(p, oblock, cblock, hint, hint_valid);
 }
 
-static inline int policy_walk_mappings(struct dm_cache_policy *p,
-				      policy_walk_fn fn, void *context)
+static inline uint32_t policy_get_hint(struct dm_cache_policy *p,
+				       dm_cblock_t cblock)
 {
-	return p->walk_mappings ? p->walk_mappings(p, fn, context) : 0;
+	return p->get_hint ? p->get_hint(p, cblock) : 0;
 }
 
 static inline int policy_writeback_work(struct dm_cache_policy *p,
diff --git a/drivers/md/dm-cache-policy-smq.c b/drivers/md/dm-cache-policy-smq.c
index cf48a617a3a4..c33f4a6e1d7d 100644
--- a/drivers/md/dm-cache-policy-smq.c
+++ b/drivers/md/dm-cache-policy-smq.c
@@ -1359,6 +1359,11 @@ static void smq_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
 	spin_unlock_irqrestore(&mq->lock, flags);
 }
 
+static unsigned random_level(dm_cblock_t cblock)
+{
+	return hash_32_generic(from_cblock(cblock), 9) & (NR_CACHE_LEVELS - 1);
+}
+
 static int smq_load_mapping(struct dm_cache_policy *p,
 			    dm_oblock_t oblock, dm_cblock_t cblock,
 			    uint32_t hint, bool hint_valid)
@@ -1369,47 +1374,21 @@ static int smq_load_mapping(struct dm_cache_policy *p,
 	e = alloc_particular_entry(&mq->cache_alloc, from_cblock(cblock));
 	e->oblock = oblock;
 	e->dirty = false;	/* this gets corrected in a minute */
-	e->level = hint_valid ? min(hint, NR_CACHE_LEVELS - 1) : 1;
+	e->level = hint_valid ? min(hint, NR_CACHE_LEVELS - 1) : random_level(cblock);
 	push(mq, e);
 
 	return 0;
 }
 
-static int smq_save_hints(struct smq_policy *mq, struct queue *q,
-			  policy_walk_fn fn, void *context)
-{
-	int r;
-	unsigned level;
-	struct entry *e;
-
-	for (level = 0; level < q->nr_levels; level++)
-		for (e = l_head(q->es, q->qs + level); e; e = l_next(q->es, e)) {
-			if (!e->sentinel) {
-				r = fn(context, infer_cblock(mq, e),
-				       e->oblock, e->level);
-				if (r)
-					return r;
-			}
-		}
-
-	return 0;
-}
-
-static int smq_walk_mappings(struct dm_cache_policy *p, policy_walk_fn fn,
-			     void *context)
+static uint32_t smq_get_hint(struct dm_cache_policy *p, dm_cblock_t cblock)
 {
 	struct smq_policy *mq = to_smq_policy(p);
-	int r = 0;
+	struct entry *e = get_entry(&mq->cache_alloc, from_cblock(cblock));
 
-	/*
-	 * We don't need to lock here since this method is only called once
-	 * the IO has stopped.
-	 */
-	r = smq_save_hints(mq, &mq->clean, fn, context);
-	if (!r)
-		r = smq_save_hints(mq, &mq->dirty, fn, context);
+	if (!e->allocated)
+		return 0;
 
-	return r;
+	return e->level;
 }
 
 static void __remove_mapping(struct smq_policy *mq, dm_oblock_t oblock)
@@ -1616,7 +1595,7 @@ static void init_policy_functions(struct smq_policy *mq, bool mimic_mq)
 	mq->policy.set_dirty = smq_set_dirty;
 	mq->policy.clear_dirty = smq_clear_dirty;
 	mq->policy.load_mapping = smq_load_mapping;
-	mq->policy.walk_mappings = smq_walk_mappings;
+	mq->policy.get_hint = smq_get_hint;
 	mq->policy.remove_mapping = smq_remove_mapping;
 	mq->policy.remove_cblock = smq_remove_cblock;
 	mq->policy.writeback_work = smq_writeback_work;
diff --git a/drivers/md/dm-cache-policy.h b/drivers/md/dm-cache-policy.h
index 05db56eedb6a..aa10b1493f34 100644
--- a/drivers/md/dm-cache-policy.h
+++ b/drivers/md/dm-cache-policy.h
@@ -90,9 +90,6 @@ struct policy_result {
 	dm_cblock_t cblock;	/* POLICY_HIT, POLICY_NEW, POLICY_REPLACE */
 };
 
-typedef int (*policy_walk_fn)(void *context, dm_cblock_t cblock,
-			      dm_oblock_t oblock, uint32_t hint);
-
 /*
  * The cache policy object.  Just a bunch of methods.  It is envisaged that
  * this structure will be embedded in a bigger, policy specific structure
@@ -158,8 +155,11 @@ struct dm_cache_policy {
 	int (*load_mapping)(struct dm_cache_policy *p, dm_oblock_t oblock,
 			    dm_cblock_t cblock, uint32_t hint, bool hint_valid);
 
-	int (*walk_mappings)(struct dm_cache_policy *p, policy_walk_fn fn,
-			     void *context);
+	/*
+	 * Gets the hint for a given cblock.  Called in a single threaded
+	 * context.  So no locking required.
+	 */
+	uint32_t (*get_hint)(struct dm_cache_policy *p, dm_cblock_t cblock);
 
 	/*
 	 * Override functions used on the error paths of the core target.
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 4e9784b4e0ac..a2768835d394 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -113,8 +113,7 @@ struct iv_tcw_private {
  * and encrypts / decrypts at the same time.
  */
 enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
-	     DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD,
-	     DM_CRYPT_EXIT_THREAD};
+	     DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD };
 
 /*
  * The fields in here must be read only after initialization.
@@ -181,7 +180,7 @@ struct crypt_config {
 	u8 key[0];
 };
 
-#define MIN_IOS        16
+#define MIN_IOS        64
 
 static void clone_init(struct dm_crypt_io *, struct bio *);
 static void kcryptd_queue_crypt(struct dm_crypt_io *io);
@@ -1136,7 +1135,7 @@ static void clone_init(struct dm_crypt_io *io, struct bio *clone)
 	clone->bi_private = io;
 	clone->bi_end_io  = crypt_endio;
 	clone->bi_bdev    = cc->dev->bdev;
-	bio_set_op_attrs(clone, bio_op(io->base_bio), io->base_bio->bi_opf);
+	bio_set_op_attrs(clone, bio_op(io->base_bio), bio_flags(io->base_bio));
 }
 
 static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
@@ -1207,18 +1206,20 @@ continue_locked:
 		if (!RB_EMPTY_ROOT(&cc->write_tree))
 			goto pop_from_list;
 
-		if (unlikely(test_bit(DM_CRYPT_EXIT_THREAD, &cc->flags))) {
-			spin_unlock_irq(&cc->write_thread_wait.lock);
-			break;
-		}
-
-		__set_current_state(TASK_INTERRUPTIBLE);
+		set_current_state(TASK_INTERRUPTIBLE);
 		__add_wait_queue(&cc->write_thread_wait, &wait);
 
 		spin_unlock_irq(&cc->write_thread_wait.lock);
 
+		if (unlikely(kthread_should_stop())) {
+			set_task_state(current, TASK_RUNNING);
+			remove_wait_queue(&cc->write_thread_wait, &wait);
+			break;
+		}
+
 		schedule();
 
+		set_task_state(current, TASK_RUNNING);
 		spin_lock_irq(&cc->write_thread_wait.lock);
 		__remove_wait_queue(&cc->write_thread_wait, &wait);
 		goto continue_locked;
@@ -1453,7 +1454,7 @@ static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
 	unsigned i;
 	int err;
 
-	cc->tfms = kmalloc(cc->tfms_count * sizeof(struct crypto_skcipher *),
+	cc->tfms = kzalloc(cc->tfms_count * sizeof(struct crypto_skcipher *),
 			   GFP_KERNEL);
 	if (!cc->tfms)
 		return -ENOMEM;
@@ -1533,13 +1534,8 @@ static void crypt_dtr(struct dm_target *ti)
 	if (!cc)
 		return;
 
-	if (cc->write_thread) {
-		spin_lock_irq(&cc->write_thread_wait.lock);
-		set_bit(DM_CRYPT_EXIT_THREAD, &cc->flags);
-		wake_up_locked(&cc->write_thread_wait);
-		spin_unlock_irq(&cc->write_thread_wait.lock);
+	if (cc->write_thread)
 		kthread_stop(cc->write_thread);
-	}
 
 	if (cc->io_queue)
 		destroy_workqueue(cc->io_queue);
@@ -1924,6 +1920,13 @@ static int crypt_map(struct dm_target *ti, struct bio *bio)
 		return DM_MAPIO_REMAPPED;
 	}
 
+	/*
+	 * Check if bio is too large, split as needed.
+	 */
+	if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_PAGES << PAGE_SHIFT)) &&
+	    bio_data_dir(bio) == WRITE)
+		dm_accept_partial_bio(bio, ((BIO_MAX_PAGES << PAGE_SHIFT) >> SECTOR_SHIFT));
+
 	io = dm_per_bio_data(bio, cc->per_bio_data_size);
 	crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
 	io->ctx.req = (struct skcipher_request *)(io + 1);
diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c
index 97e446d54a15..6a2e8dd44a1b 100644
--- a/drivers/md/dm-flakey.c
+++ b/drivers/md/dm-flakey.c
@@ -289,15 +289,13 @@ static int flakey_map(struct dm_target *ti, struct bio *bio)
 		pb->bio_submitted = true;
 
 		/*
-		 * Map reads as normal only if corrupt_bio_byte set.
+		 * Error reads if neither corrupt_bio_byte or drop_writes are set.
+		 * Otherwise, flakey_end_io() will decide if the reads should be modified.
 		 */
 		if (bio_data_dir(bio) == READ) {
-			/* If flags were specified, only corrupt those that match. */
-			if (fc->corrupt_bio_byte && (fc->corrupt_bio_rw == READ) &&
-			    all_corrupt_bio_flags_match(bio, fc))
-				goto map_bio;
-			else
+			if (!fc->corrupt_bio_byte && !test_bit(DROP_WRITES, &fc->flags))
 				return -EIO;
+			goto map_bio;
 		}
 
 		/*
@@ -334,14 +332,21 @@ static int flakey_end_io(struct dm_target *ti, struct bio *bio, int error)
 	struct flakey_c *fc = ti->private;
 	struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
 
-	/*
-	 * Corrupt successful READs while in down state.
-	 */
 	if (!error && pb->bio_submitted && (bio_data_dir(bio) == READ)) {
-		if (fc->corrupt_bio_byte)
+		if (fc->corrupt_bio_byte && (fc->corrupt_bio_rw == READ) &&
+		    all_corrupt_bio_flags_match(bio, fc)) {
+			/*
+			 * Corrupt successful matching READs while in down state.
+			 */
 			corrupt_bio_data(bio, fc);
-		else
+
+		} else if (!test_bit(DROP_WRITES, &fc->flags)) {
+			/*
+			 * Error read during the down_interval if drop_writes
+			 * wasn't configured.
+			 */
 			return -EIO;
+		}
 	}
 
 	return error;
diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c
index 4ab68033f9d1..4dfe38655a49 100644
--- a/drivers/md/dm-log-writes.c
+++ b/drivers/md/dm-log-writes.c
@@ -149,8 +149,6 @@ static void put_io_block(struct log_writes_c *lc)
 static void log_end_io(struct bio *bio)
 {
 	struct log_writes_c *lc = bio->bi_private;
-	struct bio_vec *bvec;
-	int i;
 
 	if (bio->bi_error) {
 		unsigned long flags;
@@ -161,9 +159,7 @@ static void log_end_io(struct bio *bio)
 		spin_unlock_irqrestore(&lc->blocks_lock, flags);
 	}
 
-	bio_for_each_segment_all(bvec, bio, i)
-		__free_page(bvec->bv_page);
-
+	bio_free_pages(bio);
 	put_io_block(lc);
 	bio_put(bio);
 }
@@ -259,12 +255,12 @@ static int log_one_block(struct log_writes_c *lc,
 		goto out;
 	sector++;
 
-	bio = bio_alloc(GFP_KERNEL, block->vec_cnt);
+	atomic_inc(&lc->io_blocks);
+	bio = bio_alloc(GFP_KERNEL, min(block->vec_cnt, BIO_MAX_PAGES));
 	if (!bio) {
 		DMERR("Couldn't alloc log bio");
 		goto error;
 	}
-	atomic_inc(&lc->io_blocks);
 	bio->bi_iter.bi_size = 0;
 	bio->bi_iter.bi_sector = sector;
 	bio->bi_bdev = lc->logdev->bdev;
@@ -282,7 +278,7 @@ static int log_one_block(struct log_writes_c *lc,
 		if (ret != block->vecs[i].bv_len) {
 			atomic_inc(&lc->io_blocks);
 			submit_bio(bio);
-			bio = bio_alloc(GFP_KERNEL, block->vec_cnt - i);
+			bio = bio_alloc(GFP_KERNEL, min(block->vec_cnt - i, BIO_MAX_PAGES));
 			if (!bio) {
 				DMERR("Couldn't alloc log bio");
 				goto error;
@@ -459,9 +455,9 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		goto bad;
 	}
 
-	ret = -EINVAL;
 	lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");
-	if (!lc->log_kthread) {
+	if (IS_ERR(lc->log_kthread)) {
+		ret = PTR_ERR(lc->log_kthread);
 		ti->error = "Couldn't alloc kthread";
 		dm_put_device(ti, lc->dev);
 		dm_put_device(ti, lc->logdev);
diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c
index 4ca2d1df5b44..07fc1ad42ec5 100644
--- a/drivers/md/dm-log.c
+++ b/drivers/md/dm-log.c
@@ -291,9 +291,10 @@ static void header_from_disk(struct log_header_core *core, struct log_header_dis
 	core->nr_regions = le64_to_cpu(disk->nr_regions);
 }
 
-static int rw_header(struct log_c *lc, int rw)
+static int rw_header(struct log_c *lc, int op)
 {
-	lc->io_req.bi_op = rw;
+	lc->io_req.bi_op = op;
+	lc->io_req.bi_op_flags = 0;
 
 	return dm_io(&lc->io_req, 1, &lc->header_location, NULL);
 }
@@ -316,7 +317,7 @@ static int read_header(struct log_c *log)
 {
 	int r;
 
-	r = rw_header(log, READ);
+	r = rw_header(log, REQ_OP_READ);
 	if (r)
 		return r;
 
@@ -630,7 +631,7 @@ static int disk_resume(struct dm_dirty_log *log)
 	header_to_disk(&lc->header, lc->disk_header);
 
 	/* write the new header */
-	r = rw_header(lc, WRITE);
+	r = rw_header(lc, REQ_OP_WRITE);
 	if (!r) {
 		r = flush_header(lc);
 		if (r)
@@ -698,7 +699,7 @@ static int disk_flush(struct dm_dirty_log *log)
 			log_clear_bit(lc, lc->clean_bits, i);
 	}
 
-	r = rw_header(lc, WRITE);
+	r = rw_header(lc, REQ_OP_WRITE);
 	if (r)
 		fail_log_device(lc);
 	else {
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index ac734e5bbe48..e477af8596e2 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -550,9 +550,9 @@ static int __multipath_map(struct dm_target *ti, struct request *clone,
 		pgpath = choose_pgpath(m, nr_bytes);
 
 	if (!pgpath) {
-		if (!must_push_back_rq(m))
-			r = -EIO;	/* Failed */
-		return r;
+		if (must_push_back_rq(m))
+			return DM_MAPIO_DELAY_REQUEUE;
+		return -EIO;	/* Failed */
 	} else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
 		   test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
 		pg_init_all_paths(m);
@@ -680,9 +680,11 @@ static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
 	return __multipath_map_bio(m, bio, mpio);
 }
 
-static void process_queued_bios_list(struct multipath *m)
+static void process_queued_io_list(struct multipath *m)
 {
-	if (m->queue_mode == DM_TYPE_BIO_BASED)
+	if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
+		dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
+	else if (m->queue_mode == DM_TYPE_BIO_BASED)
 		queue_work(kmultipathd, &m->process_queued_bios);
 }
 
@@ -752,7 +754,7 @@ static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
 
 	if (!queue_if_no_path) {
 		dm_table_run_md_queue_async(m->ti->table);
-		process_queued_bios_list(m);
+		process_queued_io_list(m);
 	}
 
 	return 0;
@@ -1193,21 +1195,17 @@ static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
 
 static void multipath_wait_for_pg_init_completion(struct multipath *m)
 {
-	DECLARE_WAITQUEUE(wait, current);
-
-	add_wait_queue(&m->pg_init_wait, &wait);
+	DEFINE_WAIT(wait);
 
 	while (1) {
-		set_current_state(TASK_UNINTERRUPTIBLE);
+		prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
 
 		if (!atomic_read(&m->pg_init_in_progress))
 			break;
 
 		io_schedule();
 	}
-	set_current_state(TASK_RUNNING);
-
-	remove_wait_queue(&m->pg_init_wait, &wait);
+	finish_wait(&m->pg_init_wait, &wait);
 }
 
 static void flush_multipath_work(struct multipath *m)
@@ -1308,7 +1306,7 @@ out:
 	spin_unlock_irqrestore(&m->lock, flags);
 	if (run_queue) {
 		dm_table_run_md_queue_async(m->ti->table);
-		process_queued_bios_list(m);
+		process_queued_io_list(m);
 	}
 
 	return r;
@@ -1506,7 +1504,7 @@ static void pg_init_done(void *data, int errors)
 	}
 	clear_bit(MPATHF_QUEUE_IO, &m->flags);
 
-	process_queued_bios_list(m);
+	process_queued_io_list(m);
 
 	/*
 	 * Wake up any thread waiting to suspend.
@@ -1521,10 +1519,10 @@ static void activate_path(struct work_struct *work)
 {
 	struct pgpath *pgpath =
 		container_of(work, struct pgpath, activate_path.work);
+	struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
 
-	if (pgpath->is_active)
-		scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
-				 pg_init_done, pgpath);
+	if (pgpath->is_active && !blk_queue_dying(q))
+		scsi_dh_activate(q, pg_init_done, pgpath);
 	else
 		pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
 }
@@ -1532,6 +1530,14 @@ static void activate_path(struct work_struct *work)
 static int noretry_error(int error)
 {
 	switch (error) {
+	case -EBADE:
+		/*
+		 * EBADE signals an reservation conflict.
+		 * We shouldn't fail the path here as we can communicate with
+		 * the target.  We should failover to the next path, but in
+		 * doing so we might be causing a ping-pong between paths.
+		 * So just return the reservation conflict error.
+		 */
 	case -EOPNOTSUPP:
 	case -EREMOTEIO:
 	case -EILSEQ:
@@ -1576,9 +1582,6 @@ static int do_end_io(struct multipath *m, struct request *clone,
 		if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
 			if (!must_push_back_rq(m))
 				r = -EIO;
-		} else {
-			if (error == -EBADE)
-				r = error;
 		}
 	}
 
@@ -1627,9 +1630,6 @@ static int do_end_io_bio(struct multipath *m, struct bio *clone,
 			if (!must_push_back_bio(m))
 				return -EIO;
 			return DM_ENDIO_REQUEUE;
-		} else {
-			if (error == -EBADE)
-				return error;
 		}
 	}
 
@@ -1941,7 +1941,7 @@ static int multipath_prepare_ioctl(struct dm_target *ti,
 		if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
 			pg_init_all_paths(m);
 		dm_table_run_md_queue_async(m->ti->table);
-		process_queued_bios_list(m);
+		process_queued_io_list(m);
 	}
 
 	/*
@@ -1994,11 +1994,14 @@ static int multipath_busy(struct dm_target *ti)
 	struct priority_group *pg, *next_pg;
 	struct pgpath *pgpath;
 
-	/* pg_init in progress or no paths available */
-	if (atomic_read(&m->pg_init_in_progress) ||
-	    (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)))
+	/* pg_init in progress */
+	if (atomic_read(&m->pg_init_in_progress))
 		return true;
 
+	/* no paths available, for blk-mq: rely on IO mapping to delay requeue */
+	if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
+		return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED);
+
 	/* Guess which priority_group will be used at next mapping time */
 	pg = lockless_dereference(m->current_pg);
 	next_pg = lockless_dereference(m->next_pg);
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c
index 1b9795d75ef8..8abde6b8cedc 100644
--- a/drivers/md/dm-raid.c
+++ b/drivers/md/dm-raid.c
@@ -191,7 +191,6 @@ struct raid_dev {
 #define RT_FLAG_RS_BITMAP_LOADED	2
 #define RT_FLAG_UPDATE_SBS		3
 #define RT_FLAG_RESHAPE_RS		4
-#define RT_FLAG_KEEP_RS_FROZEN		5
 
 /* Array elements of 64 bit needed for rebuild/failed disk bits */
 #define DISKS_ARRAY_ELEMS ((MAX_RAID_DEVICES + (sizeof(uint64_t) * 8 - 1)) / sizeof(uint64_t) / 8)
@@ -861,6 +860,9 @@ static int validate_region_size(struct raid_set *rs, unsigned long region_size)
 {
 	unsigned long min_region_size = rs->ti->len / (1 << 21);
 
+	if (rs_is_raid0(rs))
+		return 0;
+
 	if (!region_size) {
 		/*
 		 * Choose a reasonable default.	 All figures in sectors.
@@ -930,6 +932,8 @@ static int validate_raid_redundancy(struct raid_set *rs)
 			rebuild_cnt++;
 
 	switch (rs->raid_type->level) {
+	case 0:
+		break;
 	case 1:
 		if (rebuild_cnt >= rs->md.raid_disks)
 			goto too_many;
@@ -2335,6 +2339,13 @@ static int analyse_superblocks(struct dm_target *ti, struct raid_set *rs)
 		case 0:
 			break;
 		default:
+			/*
+			 * We have to keep any raid0 data/metadata device pairs or
+			 * the MD raid0 personality will fail to start the array.
+			 */
+			if (rs_is_raid0(rs))
+				continue;
+
 			dev = container_of(rdev, struct raid_dev, rdev);
 			if (dev->meta_dev)
 				dm_put_device(ti, dev->meta_dev);
@@ -2579,7 +2590,6 @@ static int rs_prepare_reshape(struct raid_set *rs)
 		} else {
 			/* Process raid1 without delta_disks */
 			mddev->raid_disks = rs->raid_disks;
-			set_bit(RT_FLAG_KEEP_RS_FROZEN, &rs->runtime_flags);
 			reshape = false;
 		}
 	} else {
@@ -2590,7 +2600,6 @@ static int rs_prepare_reshape(struct raid_set *rs)
 	if (reshape) {
 		set_bit(RT_FLAG_RESHAPE_RS, &rs->runtime_flags);
 		set_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags);
-		set_bit(RT_FLAG_KEEP_RS_FROZEN, &rs->runtime_flags);
 	} else if (mddev->raid_disks < rs->raid_disks)
 		/* Create new superblocks and bitmaps, if any new disks */
 		set_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags);
@@ -2902,7 +2911,6 @@ static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 			goto bad;
 
 		set_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags);
-		set_bit(RT_FLAG_KEEP_RS_FROZEN, &rs->runtime_flags);
 		/* Takeover ain't recovery, so disable recovery */
 		rs_setup_recovery(rs, MaxSector);
 		rs_set_new(rs);
@@ -3386,21 +3394,28 @@ static void raid_postsuspend(struct dm_target *ti)
 {
 	struct raid_set *rs = ti->private;
 
-	if (test_and_clear_bit(RT_FLAG_RS_RESUMED, &rs->runtime_flags)) {
-		if (!rs->md.suspended)
-			mddev_suspend(&rs->md);
-		rs->md.ro = 1;
-	}
+	if (!rs->md.suspended)
+		mddev_suspend(&rs->md);
+
+	rs->md.ro = 1;
 }
 
 static void attempt_restore_of_faulty_devices(struct raid_set *rs)
 {
 	int i;
-	uint64_t failed_devices, cleared_failed_devices = 0;
+	uint64_t cleared_failed_devices[DISKS_ARRAY_ELEMS];
 	unsigned long flags;
+	bool cleared = false;
 	struct dm_raid_superblock *sb;
+	struct mddev *mddev = &rs->md;
 	struct md_rdev *r;
 
+	/* RAID personalities have to provide hot add/remove methods or we need to bail out. */
+	if (!mddev->pers || !mddev->pers->hot_add_disk || !mddev->pers->hot_remove_disk)
+		return;
+
+	memset(cleared_failed_devices, 0, sizeof(cleared_failed_devices));
+
 	for (i = 0; i < rs->md.raid_disks; i++) {
 		r = &rs->dev[i].rdev;
 		if (test_bit(Faulty, &r->flags) && r->sb_page &&
@@ -3420,7 +3435,7 @@ static void attempt_restore_of_faulty_devices(struct raid_set *rs)
 			 * ourselves.
 			 */
 			if ((r->raid_disk >= 0) &&
-			    (r->mddev->pers->hot_remove_disk(r->mddev, r) != 0))
+			    (mddev->pers->hot_remove_disk(mddev, r) != 0))
 				/* Failed to revive this device, try next */
 				continue;
 
@@ -3430,22 +3445,30 @@ static void attempt_restore_of_faulty_devices(struct raid_set *rs)
 			clear_bit(Faulty, &r->flags);
 			clear_bit(WriteErrorSeen, &r->flags);
 			clear_bit(In_sync, &r->flags);
-			if (r->mddev->pers->hot_add_disk(r->mddev, r)) {
+			if (mddev->pers->hot_add_disk(mddev, r)) {
 				r->raid_disk = -1;
 				r->saved_raid_disk = -1;
 				r->flags = flags;
 			} else {
 				r->recovery_offset = 0;
-				cleared_failed_devices |= 1 << i;
+				set_bit(i, (void *) cleared_failed_devices);
+				cleared = true;
 			}
 		}
 	}
-	if (cleared_failed_devices) {
+
+	/* If any failed devices could be cleared, update all sbs failed_devices bits */
+	if (cleared) {
+		uint64_t failed_devices[DISKS_ARRAY_ELEMS];
+
 		rdev_for_each(r, &rs->md) {
 			sb = page_address(r->sb_page);
-			failed_devices = le64_to_cpu(sb->failed_devices);
-			failed_devices &= ~cleared_failed_devices;
-			sb->failed_devices = cpu_to_le64(failed_devices);
+			sb_retrieve_failed_devices(sb, failed_devices);
+
+			for (i = 0; i < DISKS_ARRAY_ELEMS; i++)
+				failed_devices[i] &= ~cleared_failed_devices[i];
+
+			sb_update_failed_devices(sb, failed_devices);
 		}
 	}
 }
@@ -3610,26 +3633,15 @@ static void raid_resume(struct dm_target *ti)
 		 * devices are reachable again.
 		 */
 		attempt_restore_of_faulty_devices(rs);
-	} else {
-		mddev->ro = 0;
-		mddev->in_sync = 0;
+	}
 
-		/*
-		 * When passing in flags to the ctr, we expect userspace
-		 * to reset them because they made it to the superblocks
-		 * and reload the mapping anyway.
-		 *
-		 * -> only unfreeze recovery in case of a table reload or
-		 *    we'll have a bogus recovery/reshape position
-		 *    retrieved from the superblock by the ctr because
-		 *    the ongoing recovery/reshape will change it after read.
-		 */
-		if (!test_bit(RT_FLAG_KEEP_RS_FROZEN, &rs->runtime_flags))
-			clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
+	mddev->ro = 0;
+	mddev->in_sync = 0;
 
-		if (mddev->suspended)
-			mddev_resume(mddev);
-	}
+	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
+
+	if (mddev->suspended)
+		mddev_resume(mddev);
 }
 
 static struct target_type raid_target = {
diff --git a/drivers/md/dm-round-robin.c b/drivers/md/dm-round-robin.c
index 4ace1da17db8..6c25213ab38c 100644
--- a/drivers/md/dm-round-robin.c
+++ b/drivers/md/dm-round-robin.c
@@ -210,14 +210,17 @@ static struct dm_path *rr_select_path(struct path_selector *ps, size_t nr_bytes)
 	struct path_info *pi = NULL;
 	struct dm_path *current_path = NULL;
 
+	local_irq_save(flags);
 	current_path = *this_cpu_ptr(s->current_path);
 	if (current_path) {
 		percpu_counter_dec(&s->repeat_count);
-		if (percpu_counter_read_positive(&s->repeat_count) > 0)
+		if (percpu_counter_read_positive(&s->repeat_count) > 0) {
+			local_irq_restore(flags);
 			return current_path;
+		}
 	}
 
-	spin_lock_irqsave(&s->lock, flags);
+	spin_lock(&s->lock);
 	if (!list_empty(&s->valid_paths)) {
 		pi = list_entry(s->valid_paths.next, struct path_info, list);
 		list_move_tail(&pi->list, &s->valid_paths);
diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c
index 1ca7463e8bb2..dc75bea0d541 100644
--- a/drivers/md/dm-rq.c
+++ b/drivers/md/dm-rq.c
@@ -73,15 +73,24 @@ static void dm_old_start_queue(struct request_queue *q)
 	spin_unlock_irqrestore(q->queue_lock, flags);
 }
 
+static void dm_mq_start_queue(struct request_queue *q)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(q->queue_lock, flags);
+	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
+	spin_unlock_irqrestore(q->queue_lock, flags);
+
+	blk_mq_start_stopped_hw_queues(q, true);
+	blk_mq_kick_requeue_list(q);
+}
+
 void dm_start_queue(struct request_queue *q)
 {
 	if (!q->mq_ops)
 		dm_old_start_queue(q);
-	else {
-		queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, q);
-		blk_mq_start_stopped_hw_queues(q, true);
-		blk_mq_kick_requeue_list(q);
-	}
+	else
+		dm_mq_start_queue(q);
 }
 
 static void dm_old_stop_queue(struct request_queue *q)
@@ -89,27 +98,35 @@ static void dm_old_stop_queue(struct request_queue *q)
 	unsigned long flags;
 
 	spin_lock_irqsave(q->queue_lock, flags);
+	if (!blk_queue_stopped(q))
+		blk_stop_queue(q);
+	spin_unlock_irqrestore(q->queue_lock, flags);
+}
+
+static void dm_mq_stop_queue(struct request_queue *q)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(q->queue_lock, flags);
 	if (blk_queue_stopped(q)) {
 		spin_unlock_irqrestore(q->queue_lock, flags);
 		return;
 	}
 
-	blk_stop_queue(q);
+	queue_flag_set(QUEUE_FLAG_STOPPED, q);
 	spin_unlock_irqrestore(q->queue_lock, flags);
+
+	/* Avoid that requeuing could restart the queue. */
+	blk_mq_cancel_requeue_work(q);
+	blk_mq_stop_hw_queues(q);
 }
 
 void dm_stop_queue(struct request_queue *q)
 {
 	if (!q->mq_ops)
 		dm_old_stop_queue(q);
-	else {
-		spin_lock_irq(q->queue_lock);
-		queue_flag_set(QUEUE_FLAG_STOPPED, q);
-		spin_unlock_irq(q->queue_lock);
-
-		blk_mq_cancel_requeue_work(q);
-		blk_mq_stop_hw_queues(q);
-	}
+	else
+		dm_mq_stop_queue(q);
 }
 
 static struct dm_rq_target_io *alloc_old_rq_tio(struct mapped_device *md,
@@ -319,21 +336,32 @@ static void dm_old_requeue_request(struct request *rq)
 	spin_unlock_irqrestore(q->queue_lock, flags);
 }
 
-static void dm_mq_requeue_request(struct request *rq)
+static void __dm_mq_kick_requeue_list(struct request_queue *q, unsigned long msecs)
 {
-	struct request_queue *q = rq->q;
 	unsigned long flags;
 
-	blk_mq_requeue_request(rq);
 	spin_lock_irqsave(q->queue_lock, flags);
 	if (!blk_queue_stopped(q))
-		blk_mq_kick_requeue_list(q);
+		blk_mq_delay_kick_requeue_list(q, msecs);
 	spin_unlock_irqrestore(q->queue_lock, flags);
 }
 
-static void dm_requeue_original_request(struct mapped_device *md,
-					struct request *rq)
+void dm_mq_kick_requeue_list(struct mapped_device *md)
+{
+	__dm_mq_kick_requeue_list(dm_get_md_queue(md), 0);
+}
+EXPORT_SYMBOL(dm_mq_kick_requeue_list);
+
+static void dm_mq_delay_requeue_request(struct request *rq, unsigned long msecs)
+{
+	blk_mq_requeue_request(rq);
+	__dm_mq_kick_requeue_list(rq->q, msecs);
+}
+
+static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_requeue)
 {
+	struct mapped_device *md = tio->md;
+	struct request *rq = tio->orig;
 	int rw = rq_data_dir(rq);
 
 	rq_end_stats(md, rq);
@@ -342,7 +370,7 @@ static void dm_requeue_original_request(struct mapped_device *md,
 	if (!rq->q->mq_ops)
 		dm_old_requeue_request(rq);
 	else
-		dm_mq_requeue_request(rq);
+		dm_mq_delay_requeue_request(rq, delay_requeue ? 5000 : 0);
 
 	rq_completed(md, rw, false);
 }
@@ -372,7 +400,7 @@ static void dm_done(struct request *clone, int error, bool mapped)
 		return;
 	else if (r == DM_ENDIO_REQUEUE)
 		/* The target wants to requeue the I/O */
-		dm_requeue_original_request(tio->md, tio->orig);
+		dm_requeue_original_request(tio, false);
 	else {
 		DMWARN("unimplemented target endio return value: %d", r);
 		BUG();
@@ -553,7 +581,7 @@ static void init_tio(struct dm_rq_target_io *tio, struct request *rq,
 	if (!md->init_tio_pdu)
 		memset(&tio->info, 0, sizeof(tio->info));
 	if (md->kworker_task)
-		init_kthread_work(&tio->work, map_tio_request);
+		kthread_init_work(&tio->work, map_tio_request);
 }
 
 static struct dm_rq_target_io *dm_old_prep_tio(struct request *rq,
@@ -612,20 +640,23 @@ static int dm_old_prep_fn(struct request_queue *q, struct request *rq)
 
 /*
  * Returns:
- * 0                : the request has been processed
- * DM_MAPIO_REQUEUE : the original request needs to be requeued
+ * DM_MAPIO_*       : the request has been processed as indicated
+ * DM_MAPIO_REQUEUE : the original request needs to be immediately requeued
  * < 0              : the request was completed due to failure
  */
-static int map_request(struct dm_rq_target_io *tio, struct request *rq,
-		       struct mapped_device *md)
+static int map_request(struct dm_rq_target_io *tio)
 {
 	int r;
 	struct dm_target *ti = tio->ti;
+	struct mapped_device *md = tio->md;
+	struct request *rq = tio->orig;
 	struct request *clone = NULL;
 
 	if (tio->clone) {
 		clone = tio->clone;
 		r = ti->type->map_rq(ti, clone, &tio->info);
+		if (r == DM_MAPIO_DELAY_REQUEUE)
+			return DM_MAPIO_REQUEUE; /* .request_fn requeue is always immediate */
 	} else {
 		r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);
 		if (r < 0) {
@@ -633,9 +664,8 @@ static int map_request(struct dm_rq_target_io *tio, struct request *rq,
 			dm_kill_unmapped_request(rq, r);
 			return r;
 		}
-		if (r != DM_MAPIO_REMAPPED)
-			return r;
-		if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {
+		if (r == DM_MAPIO_REMAPPED &&
+		    setup_clone(clone, rq, tio, GFP_ATOMIC)) {
 			/* -ENOMEM */
 			ti->type->release_clone_rq(clone);
 			return DM_MAPIO_REQUEUE;
@@ -654,7 +684,10 @@ static int map_request(struct dm_rq_target_io *tio, struct request *rq,
 		break;
 	case DM_MAPIO_REQUEUE:
 		/* The target wants to requeue the I/O */
-		dm_requeue_original_request(md, tio->orig);
+		break;
+	case DM_MAPIO_DELAY_REQUEUE:
+		/* The target wants to requeue the I/O after a delay */
+		dm_requeue_original_request(tio, true);
 		break;
 	default:
 		if (r > 0) {
@@ -664,10 +697,9 @@ static int map_request(struct dm_rq_target_io *tio, struct request *rq,
 
 		/* The target wants to complete the I/O */
 		dm_kill_unmapped_request(rq, r);
-		return r;
 	}
 
-	return 0;
+	return r;
 }
 
 static void dm_start_request(struct mapped_device *md, struct request *orig)
@@ -706,11 +738,9 @@ static void dm_start_request(struct mapped_device *md, struct request *orig)
 static void map_tio_request(struct kthread_work *work)
 {
 	struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work);
-	struct request *rq = tio->orig;
-	struct mapped_device *md = tio->md;
 
-	if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE)
-		dm_requeue_original_request(md, rq);
+	if (map_request(tio) == DM_MAPIO_REQUEUE)
+		dm_requeue_original_request(tio, false);
 }
 
 ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf)
@@ -801,7 +831,7 @@ static void dm_old_request_fn(struct request_queue *q)
 		tio = tio_from_request(rq);
 		/* Establish tio->ti before queuing work (map_tio_request) */
 		tio->ti = ti;
-		queue_kthread_work(&md->kworker, &tio->work);
+		kthread_queue_work(&md->kworker, &tio->work);
 		BUG_ON(!irqs_disabled());
 	}
 }
@@ -823,7 +853,7 @@ int dm_old_init_request_queue(struct mapped_device *md)
 	blk_queue_prep_rq(md->queue, dm_old_prep_fn);
 
 	/* Initialize the request-based DM worker thread */
-	init_kthread_worker(&md->kworker);
+	kthread_init_worker(&md->kworker);
 	md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker,
 				       "kdmwork-%s", dm_device_name(md));
 	if (IS_ERR(md->kworker_task))
@@ -896,7 +926,7 @@ static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
 	tio->ti = ti;
 
 	/* Direct call is fine since .queue_rq allows allocations */
-	if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) {
+	if (map_request(tio) == DM_MAPIO_REQUEUE) {
 		/* Undo dm_start_request() before requeuing */
 		rq_end_stats(md, rq);
 		rq_completed(md, rq_data_dir(rq), false);
@@ -908,7 +938,6 @@ static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
 
 static struct blk_mq_ops dm_mq_ops = {
 	.queue_rq = dm_mq_queue_rq,
-	.map_queue = blk_mq_map_queue,
 	.complete = dm_softirq_done,
 	.init_request = dm_mq_init_request,
 };
@@ -955,7 +984,7 @@ int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t)
 	dm_init_md_queue(md);
 
 	/* backfill 'mq' sysfs registration normally done in blk_register_queue */
-	blk_mq_register_disk(md->disk);
+	blk_mq_register_dev(disk_to_dev(md->disk), q);
 
 	return 0;
 
diff --git a/drivers/md/dm-rq.h b/drivers/md/dm-rq.h
index 9e6f0a3773d4..4da06cae7bad 100644
--- a/drivers/md/dm-rq.h
+++ b/drivers/md/dm-rq.h
@@ -55,6 +55,8 @@ void dm_mq_cleanup_mapped_device(struct mapped_device *md);
 void dm_start_queue(struct request_queue *q);
 void dm_stop_queue(struct request_queue *q);
 
+void dm_mq_kick_requeue_list(struct mapped_device *md);
+
 unsigned dm_get_reserved_rq_based_ios(void);
 
 ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf);
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index fa9b1cb4438a..147af9536d0c 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1648,6 +1648,8 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
 	struct request_queue *q = md->queue;
 	sector_t size;
 
+	lockdep_assert_held(&md->suspend_lock);
+
 	size = dm_table_get_size(t);
 
 	/*
@@ -1873,6 +1875,7 @@ EXPORT_SYMBOL_GPL(dm_device_name);
 
 static void __dm_destroy(struct mapped_device *md, bool wait)
 {
+	struct request_queue *q = dm_get_md_queue(md);
 	struct dm_table *map;
 	int srcu_idx;
 
@@ -1883,8 +1886,12 @@ static void __dm_destroy(struct mapped_device *md, bool wait)
 	set_bit(DMF_FREEING, &md->flags);
 	spin_unlock(&_minor_lock);
 
+	spin_lock_irq(q->queue_lock);
+	queue_flag_set(QUEUE_FLAG_DYING, q);
+	spin_unlock_irq(q->queue_lock);
+
 	if (dm_request_based(md) && md->kworker_task)
-		flush_kthread_worker(&md->kworker);
+		kthread_flush_worker(&md->kworker);
 
 	/*
 	 * Take suspend_lock so that presuspend and postsuspend methods
@@ -1934,30 +1941,25 @@ void dm_put(struct mapped_device *md)
 }
 EXPORT_SYMBOL_GPL(dm_put);
 
-static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
+static int dm_wait_for_completion(struct mapped_device *md, long task_state)
 {
 	int r = 0;
-	DECLARE_WAITQUEUE(wait, current);
-
-	add_wait_queue(&md->wait, &wait);
+	DEFINE_WAIT(wait);
 
 	while (1) {
-		set_current_state(interruptible);
+		prepare_to_wait(&md->wait, &wait, task_state);
 
 		if (!md_in_flight(md))
 			break;
 
-		if (interruptible == TASK_INTERRUPTIBLE &&
-		    signal_pending(current)) {
+		if (signal_pending_state(task_state, current)) {
 			r = -EINTR;
 			break;
 		}
 
 		io_schedule();
 	}
-	set_current_state(TASK_RUNNING);
-
-	remove_wait_queue(&md->wait, &wait);
+	finish_wait(&md->wait, &wait);
 
 	return r;
 }
@@ -2075,6 +2077,10 @@ static void unlock_fs(struct mapped_device *md)
 }
 
 /*
+ * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
+ * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
+ * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
+ *
  * If __dm_suspend returns 0, the device is completely quiescent
  * now. There is no request-processing activity. All new requests
  * are being added to md->deferred list.
@@ -2082,13 +2088,15 @@ static void unlock_fs(struct mapped_device *md)
  * Caller must hold md->suspend_lock
  */
 static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
-			unsigned suspend_flags, int interruptible,
+			unsigned suspend_flags, long task_state,
 			int dmf_suspended_flag)
 {
 	bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
 	bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
 	int r;
 
+	lockdep_assert_held(&md->suspend_lock);
+
 	/*
 	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
 	 * This flag is cleared before dm_suspend returns.
@@ -2139,7 +2147,7 @@ static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
 	if (dm_request_based(md)) {
 		dm_stop_queue(md->queue);
 		if (md->kworker_task)
-			flush_kthread_worker(&md->kworker);
+			kthread_flush_worker(&md->kworker);
 	}
 
 	flush_workqueue(md->wq);
@@ -2149,7 +2157,7 @@ static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
 	 * We call dm_wait_for_completion to wait for all existing requests
 	 * to finish.
 	 */
-	r = dm_wait_for_completion(md, interruptible);
+	r = dm_wait_for_completion(md, task_state);
 	if (!r)
 		set_bit(dmf_suspended_flag, &md->flags);
 
@@ -2249,10 +2257,11 @@ static int __dm_resume(struct mapped_device *md, struct dm_table *map)
 
 int dm_resume(struct mapped_device *md)
 {
-	int r = -EINVAL;
+	int r;
 	struct dm_table *map = NULL;
 
 retry:
+	r = -EINVAL;
 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
 
 	if (!dm_suspended_md(md))
@@ -2276,8 +2285,6 @@ retry:
 		goto out;
 
 	clear_bit(DMF_SUSPENDED, &md->flags);
-
-	r = 0;
 out:
 	mutex_unlock(&md->suspend_lock);
 
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 41573f1f626f..2b13117fb918 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -10,6 +10,7 @@
 
 
 #include <linux/module.h>
+#include <linux/kthread.h>
 #include <linux/dlm.h>
 #include <linux/sched.h>
 #include <linux/raid/md_p.h>
@@ -25,7 +26,8 @@ struct dlm_lock_resource {
 	struct dlm_lksb lksb;
 	char *name; /* lock name. */
 	uint32_t flags; /* flags to pass to dlm_lock() */
-	struct completion completion; /* completion for synchronized locking */
+	wait_queue_head_t sync_locking; /* wait queue for synchronized locking */
+	bool sync_locking_done;
 	void (*bast)(void *arg, int mode); /* blocking AST function pointer*/
 	struct mddev *mddev; /* pointing back to mddev. */
 	int mode;
@@ -118,7 +120,8 @@ static void sync_ast(void *arg)
 	struct dlm_lock_resource *res;
 
 	res = arg;
-	complete(&res->completion);
+	res->sync_locking_done = true;
+	wake_up(&res->sync_locking);
 }
 
 static int dlm_lock_sync(struct dlm_lock_resource *res, int mode)
@@ -130,7 +133,8 @@ static int dlm_lock_sync(struct dlm_lock_resource *res, int mode)
 			0, sync_ast, res, res->bast);
 	if (ret)
 		return ret;
-	wait_for_completion(&res->completion);
+	wait_event(res->sync_locking, res->sync_locking_done);
+	res->sync_locking_done = false;
 	if (res->lksb.sb_status == 0)
 		res->mode = mode;
 	return res->lksb.sb_status;
@@ -141,6 +145,44 @@ static int dlm_unlock_sync(struct dlm_lock_resource *res)
 	return dlm_lock_sync(res, DLM_LOCK_NL);
 }
 
+/*
+ * An variation of dlm_lock_sync, which make lock request could
+ * be interrupted
+ */
+static int dlm_lock_sync_interruptible(struct dlm_lock_resource *res, int mode,
+				       struct mddev *mddev)
+{
+	int ret = 0;
+
+	ret = dlm_lock(res->ls, mode, &res->lksb,
+			res->flags, res->name, strlen(res->name),
+			0, sync_ast, res, res->bast);
+	if (ret)
+		return ret;
+
+	wait_event(res->sync_locking, res->sync_locking_done
+				      || kthread_should_stop()
+				      || test_bit(MD_CLOSING, &mddev->flags));
+	if (!res->sync_locking_done) {
+		/*
+		 * the convert queue contains the lock request when request is
+		 * interrupted, and sync_ast could still be run, so need to
+		 * cancel the request and reset completion
+		 */
+		ret = dlm_unlock(res->ls, res->lksb.sb_lkid, DLM_LKF_CANCEL,
+			&res->lksb, res);
+		res->sync_locking_done = false;
+		if (unlikely(ret != 0))
+			pr_info("failed to cancel previous lock request "
+				 "%s return %d\n", res->name, ret);
+		return -EPERM;
+	} else
+		res->sync_locking_done = false;
+	if (res->lksb.sb_status == 0)
+		res->mode = mode;
+	return res->lksb.sb_status;
+}
+
 static struct dlm_lock_resource *lockres_init(struct mddev *mddev,
 		char *name, void (*bastfn)(void *arg, int mode), int with_lvb)
 {
@@ -151,7 +193,8 @@ static struct dlm_lock_resource *lockres_init(struct mddev *mddev,
 	res = kzalloc(sizeof(struct dlm_lock_resource), GFP_KERNEL);
 	if (!res)
 		return NULL;
-	init_completion(&res->completion);
+	init_waitqueue_head(&res->sync_locking);
+	res->sync_locking_done = false;
 	res->ls = cinfo->lockspace;
 	res->mddev = mddev;
 	res->mode = DLM_LOCK_IV;
@@ -194,25 +237,21 @@ out_err:
 
 static void lockres_free(struct dlm_lock_resource *res)
 {
-	int ret;
+	int ret = 0;
 
 	if (!res)
 		return;
 
-	/* cancel a lock request or a conversion request that is blocked */
-	res->flags |= DLM_LKF_CANCEL;
-retry:
-	ret = dlm_unlock(res->ls, res->lksb.sb_lkid, 0, &res->lksb, res);
-	if (unlikely(ret != 0)) {
-		pr_info("%s: failed to unlock %s return %d\n", __func__, res->name, ret);
-
-		/* if a lock conversion is cancelled, then the lock is put
-		 * back to grant queue, need to ensure it is unlocked */
-		if (ret == -DLM_ECANCEL)
-			goto retry;
-	}
-	res->flags &= ~DLM_LKF_CANCEL;
-	wait_for_completion(&res->completion);
+	/*
+	 * use FORCEUNLOCK flag, so we can unlock even the lock is on the
+	 * waiting or convert queue
+	 */
+	ret = dlm_unlock(res->ls, res->lksb.sb_lkid, DLM_LKF_FORCEUNLOCK,
+		&res->lksb, res);
+	if (unlikely(ret != 0))
+		pr_err("failed to unlock %s return %d\n", res->name, ret);
+	else
+		wait_event(res->sync_locking, res->sync_locking_done);
 
 	kfree(res->name);
 	kfree(res->lksb.sb_lvbptr);
@@ -279,7 +318,7 @@ static void recover_bitmaps(struct md_thread *thread)
 			goto clear_bit;
 		}
 
-		ret = dlm_lock_sync(bm_lockres, DLM_LOCK_PW);
+		ret = dlm_lock_sync_interruptible(bm_lockres, DLM_LOCK_PW, mddev);
 		if (ret) {
 			pr_err("md-cluster: Could not DLM lock %s: %d\n",
 					str, ret);
@@ -288,7 +327,7 @@ static void recover_bitmaps(struct md_thread *thread)
 		ret = bitmap_copy_from_slot(mddev, slot, &lo, &hi, true);
 		if (ret) {
 			pr_err("md-cluster: Could not copy data from bitmap %d\n", slot);
-			goto dlm_unlock;
+			goto clear_bit;
 		}
 		if (hi > 0) {
 			if (lo < mddev->recovery_cp)
@@ -300,8 +339,6 @@ static void recover_bitmaps(struct md_thread *thread)
 			    md_wakeup_thread(mddev->thread);
 			}
 		}
-dlm_unlock:
-		dlm_unlock_sync(bm_lockres);
 clear_bit:
 		lockres_free(bm_lockres);
 		clear_bit(slot, &cinfo->recovery_map);
@@ -495,9 +532,10 @@ static void process_metadata_update(struct mddev *mddev, struct cluster_msg *msg
 
 static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg)
 {
-	struct md_rdev *rdev = md_find_rdev_nr_rcu(mddev,
-						   le32_to_cpu(msg->raid_slot));
+	struct md_rdev *rdev;
 
+	rcu_read_lock();
+	rdev = md_find_rdev_nr_rcu(mddev, le32_to_cpu(msg->raid_slot));
 	if (rdev) {
 		set_bit(ClusterRemove, &rdev->flags);
 		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
@@ -506,18 +544,21 @@ static void process_remove_disk(struct mddev *mddev, struct cluster_msg *msg)
 	else
 		pr_warn("%s: %d Could not find disk(%d) to REMOVE\n",
 			__func__, __LINE__, le32_to_cpu(msg->raid_slot));
+	rcu_read_unlock();
 }
 
 static void process_readd_disk(struct mddev *mddev, struct cluster_msg *msg)
 {
-	struct md_rdev *rdev = md_find_rdev_nr_rcu(mddev,
-						   le32_to_cpu(msg->raid_slot));
+	struct md_rdev *rdev;
 
+	rcu_read_lock();
+	rdev = md_find_rdev_nr_rcu(mddev, le32_to_cpu(msg->raid_slot));
 	if (rdev && test_bit(Faulty, &rdev->flags))
 		clear_bit(Faulty, &rdev->flags);
 	else
 		pr_warn("%s: %d Could not find disk(%d) which is faulty",
 			__func__, __LINE__, le32_to_cpu(msg->raid_slot));
+	rcu_read_unlock();
 }
 
 static int process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
@@ -770,7 +811,6 @@ static int gather_all_resync_info(struct mddev *mddev, int total_slots)
 			md_check_recovery(mddev);
 		}
 
-		dlm_unlock_sync(bm_lockres);
 		lockres_free(bm_lockres);
 	}
 out:
@@ -834,8 +874,10 @@ static int join(struct mddev *mddev, int nodes)
 		goto err;
 	}
 	cinfo->ack_lockres = lockres_init(mddev, "ack", ack_bast, 0);
-	if (!cinfo->ack_lockres)
+	if (!cinfo->ack_lockres) {
+		ret = -ENOMEM;
 		goto err;
+	}
 	/* get sync CR lock on ACK. */
 	if (dlm_lock_sync(cinfo->ack_lockres, DLM_LOCK_CR))
 		pr_err("md-cluster: failed to get a sync CR lock on ACK!(%d)\n",
@@ -849,8 +891,10 @@ static int join(struct mddev *mddev, int nodes)
 	pr_info("md-cluster: Joined cluster %s slot %d\n", str, cinfo->slot_number);
 	snprintf(str, 64, "bitmap%04d", cinfo->slot_number - 1);
 	cinfo->bitmap_lockres = lockres_init(mddev, str, NULL, 1);
-	if (!cinfo->bitmap_lockres)
+	if (!cinfo->bitmap_lockres) {
+		ret = -ENOMEM;
 		goto err;
+	}
 	if (dlm_lock_sync(cinfo->bitmap_lockres, DLM_LOCK_PW)) {
 		pr_err("Failed to get bitmap lock\n");
 		ret = -EINVAL;
@@ -858,8 +902,10 @@ static int join(struct mddev *mddev, int nodes)
 	}
 
 	cinfo->resync_lockres = lockres_init(mddev, "resync", NULL, 0);
-	if (!cinfo->resync_lockres)
+	if (!cinfo->resync_lockres) {
+		ret = -ENOMEM;
 		goto err;
+	}
 
 	return 0;
 err:
@@ -1000,7 +1046,7 @@ static void metadata_update_cancel(struct mddev *mddev)
 static int resync_start(struct mddev *mddev)
 {
 	struct md_cluster_info *cinfo = mddev->cluster_info;
-	return dlm_lock_sync(cinfo->resync_lockres, DLM_LOCK_EX);
+	return dlm_lock_sync_interruptible(cinfo->resync_lockres, DLM_LOCK_EX, mddev);
 }
 
 static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi)
@@ -1180,7 +1226,6 @@ static void unlock_all_bitmaps(struct mddev *mddev)
 	if (cinfo->other_bitmap_lockres) {
 		for (i = 0; i < mddev->bitmap_info.nodes - 1; i++) {
 			if (cinfo->other_bitmap_lockres[i]) {
-				dlm_unlock_sync(cinfo->other_bitmap_lockres[i]);
 				lockres_free(cinfo->other_bitmap_lockres[i]);
 			}
 		}
diff --git a/drivers/md/md.c b/drivers/md/md.c
index d646f6e444f0..eac84d8ff724 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -1604,11 +1604,8 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *rdev)
 			mddev->new_chunk_sectors = mddev->chunk_sectors;
 		}
 
-		if (le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL) {
+		if (le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)
 			set_bit(MD_HAS_JOURNAL, &mddev->flags);
-			if (mddev->recovery_cp == MaxSector)
-				set_bit(MD_JOURNAL_CLEAN, &mddev->flags);
-		}
 	} else if (mddev->pers == NULL) {
 		/* Insist of good event counter while assembling, except for
 		 * spares (which don't need an event count) */
@@ -5300,6 +5297,21 @@ int md_run(struct mddev *mddev)
 		return err;
 	}
 	if (mddev->queue) {
+		bool nonrot = true;
+
+		rdev_for_each(rdev, mddev) {
+			if (rdev->raid_disk >= 0 &&
+			    !blk_queue_nonrot(bdev_get_queue(rdev->bdev))) {
+				nonrot = false;
+				break;
+			}
+		}
+		if (mddev->degraded)
+			nonrot = false;
+		if (nonrot)
+			queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mddev->queue);
+		else
+			queue_flag_clear_unlocked(QUEUE_FLAG_NONROT, mddev->queue);
 		mddev->queue->backing_dev_info.congested_data = mddev;
 		mddev->queue->backing_dev_info.congested_fn = md_congested;
 	}
@@ -5457,12 +5469,14 @@ static void md_clean(struct mddev *mddev)
 	mddev->degraded = 0;
 	mddev->safemode = 0;
 	mddev->private = NULL;
+	mddev->cluster_info = NULL;
 	mddev->bitmap_info.offset = 0;
 	mddev->bitmap_info.default_offset = 0;
 	mddev->bitmap_info.default_space = 0;
 	mddev->bitmap_info.chunksize = 0;
 	mddev->bitmap_info.daemon_sleep = 0;
 	mddev->bitmap_info.max_write_behind = 0;
+	mddev->bitmap_info.nodes = 0;
 }
 
 static void __md_stop_writes(struct mddev *mddev)
@@ -5576,8 +5590,7 @@ static int md_set_readonly(struct mddev *mddev, struct block_device *bdev)
 	mutex_lock(&mddev->open_mutex);
 	if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) ||
 	    mddev->sync_thread ||
-	    test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) ||
-	    (bdev && !test_bit(MD_STILL_CLOSED, &mddev->flags))) {
+	    test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
 		printk("md: %s still in use.\n",mdname(mddev));
 		if (did_freeze) {
 			clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
@@ -5639,8 +5652,7 @@ static int do_md_stop(struct mddev *mddev, int mode,
 	if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) ||
 	    mddev->sysfs_active ||
 	    mddev->sync_thread ||
-	    test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) ||
-	    (bdev && !test_bit(MD_STILL_CLOSED, &mddev->flags))) {
+	    test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
 		printk("md: %s still in use.\n",mdname(mddev));
 		mutex_unlock(&mddev->open_mutex);
 		if (did_freeze) {
@@ -5851,6 +5863,9 @@ static int get_array_info(struct mddev *mddev, void __user *arg)
 			working++;
 			if (test_bit(In_sync, &rdev->flags))
 				insync++;
+			else if (test_bit(Journal, &rdev->flags))
+				/* TODO: add journal count to md_u.h */
+				;
 			else
 				spare++;
 		}
@@ -6101,9 +6116,14 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 			export_rdev(rdev);
 
 		if (mddev_is_clustered(mddev)) {
-			if (info->state & (1 << MD_DISK_CANDIDATE))
-				md_cluster_ops->new_disk_ack(mddev, (err == 0));
-			else {
+			if (info->state & (1 << MD_DISK_CANDIDATE)) {
+				if (!err) {
+					err = md_cluster_ops->new_disk_ack(mddev,
+						err == 0);
+					if (err)
+						md_kick_rdev_from_array(rdev);
+				}
+			} else {
 				if (err)
 					md_cluster_ops->add_new_disk_cancel(mddev);
 				else
@@ -6821,7 +6841,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode,
 			err = -EBUSY;
 			goto out;
 		}
-		set_bit(MD_STILL_CLOSED, &mddev->flags);
+		set_bit(MD_CLOSING, &mddev->flags);
 		mutex_unlock(&mddev->open_mutex);
 		sync_blockdev(bdev);
 	}
@@ -7070,9 +7090,13 @@ static int md_open(struct block_device *bdev, fmode_t mode)
 	if ((err = mutex_lock_interruptible(&mddev->open_mutex)))
 		goto out;
 
+	if (test_bit(MD_CLOSING, &mddev->flags)) {
+		mutex_unlock(&mddev->open_mutex);
+		return -ENODEV;
+	}
+
 	err = 0;
 	atomic_inc(&mddev->openers);
-	clear_bit(MD_STILL_CLOSED, &mddev->flags);
 	mutex_unlock(&mddev->open_mutex);
 
 	check_disk_change(bdev);
@@ -7610,16 +7634,12 @@ EXPORT_SYMBOL(unregister_md_cluster_operations);
 
 int md_setup_cluster(struct mddev *mddev, int nodes)
 {
-	int err;
-
-	err = request_module("md-cluster");
-	if (err) {
-		pr_err("md-cluster module not found.\n");
-		return -ENOENT;
-	}
-
+	if (!md_cluster_ops)
+		request_module("md-cluster");
 	spin_lock(&pers_lock);
+	/* ensure module won't be unloaded */
 	if (!md_cluster_ops || !try_module_get(md_cluster_mod)) {
+		pr_err("can't find md-cluster module or get it's reference.\n");
 		spin_unlock(&pers_lock);
 		return -ENOENT;
 	}
@@ -7862,6 +7882,7 @@ void md_do_sync(struct md_thread *thread)
 	 */
 
 	do {
+		int mddev2_minor = -1;
 		mddev->curr_resync = 2;
 
 	try_again:
@@ -7891,10 +7912,14 @@ void md_do_sync(struct md_thread *thread)
 				prepare_to_wait(&resync_wait, &wq, TASK_INTERRUPTIBLE);
 				if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery) &&
 				    mddev2->curr_resync >= mddev->curr_resync) {
-					printk(KERN_INFO "md: delaying %s of %s"
-					       " until %s has finished (they"
-					       " share one or more physical units)\n",
-					       desc, mdname(mddev), mdname(mddev2));
+					if (mddev2_minor != mddev2->md_minor) {
+						mddev2_minor = mddev2->md_minor;
+						printk(KERN_INFO "md: delaying %s of %s"
+						       " until %s has finished (they"
+						       " share one or more physical units)\n",
+						       desc, mdname(mddev),
+						       mdname(mddev2));
+					}
 					mddev_put(mddev2);
 					if (signal_pending(current))
 						flush_signals(current);
@@ -8275,16 +8300,13 @@ no_add:
 static void md_start_sync(struct work_struct *ws)
 {
 	struct mddev *mddev = container_of(ws, struct mddev, del_work);
-	int ret = 0;
 
 	mddev->sync_thread = md_register_thread(md_do_sync,
 						mddev,
 						"resync");
 	if (!mddev->sync_thread) {
-		if (!(mddev_is_clustered(mddev) && ret == -EAGAIN))
-			printk(KERN_ERR "%s: could not start resync"
-			       " thread...\n",
-			       mdname(mddev));
+		printk(KERN_ERR "%s: could not start resync thread...\n",
+		       mdname(mddev));
 		/* leave the spares where they are, it shouldn't hurt */
 		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
 		clear_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
@@ -8875,7 +8897,9 @@ static void autostart_arrays(int part)
 		list_del(&node_detected_dev->list);
 		dev = node_detected_dev->dev;
 		kfree(node_detected_dev);
+		mutex_unlock(&detected_devices_mutex);
 		rdev = md_import_device(dev,0, 90);
+		mutex_lock(&detected_devices_mutex);
 		if (IS_ERR(rdev))
 			continue;
 
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 20c667579ede..2b2041773e79 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -201,9 +201,8 @@ struct mddev {
 #define MD_CHANGE_PENDING 2	/* switch from 'clean' to 'active' in progress */
 #define MD_UPDATE_SB_FLAGS (1 | 2 | 4)	/* If these are set, md_update_sb needed */
 #define MD_ARRAY_FIRST_USE 3    /* First use of array, needs initialization */
-#define MD_STILL_CLOSED	4	/* If set, then array has not been opened since
-				 * md_ioctl checked on it.
-				 */
+#define MD_CLOSING	4	/* If set, we are closing the array, do not open
+				 * it then */
 #define MD_JOURNAL_CLEAN 5	/* A raid with journal is already clean */
 #define MD_HAS_JOURNAL	6	/* The raid array has journal feature set */
 #define MD_RELOAD_SB	7	/* Reload the superblock because another node
diff --git a/drivers/md/persistent-data/dm-array.c b/drivers/md/persistent-data/dm-array.c
index 431a03067d64..e83047cbb2da 100644
--- a/drivers/md/persistent-data/dm-array.c
+++ b/drivers/md/persistent-data/dm-array.c
@@ -277,6 +277,48 @@ static int insert_ablock(struct dm_array_info *info, uint64_t index,
 	return dm_btree_insert(&info->btree_info, *root, &index, &block_le, root);
 }
 
+/*----------------------------------------------------------------*/
+
+static int __shadow_ablock(struct dm_array_info *info, dm_block_t b,
+			   struct dm_block **block, struct array_block **ab)
+{
+	int inc;
+	int r = dm_tm_shadow_block(info->btree_info.tm, b,
+				   &array_validator, block, &inc);
+	if (r)
+		return r;
+
+	*ab = dm_block_data(*block);
+	if (inc)
+		inc_ablock_entries(info, *ab);
+
+	return 0;
+}
+
+/*
+ * The shadow op will often be a noop.  Only insert if it really
+ * copied data.
+ */
+static int __reinsert_ablock(struct dm_array_info *info, unsigned index,
+			     struct dm_block *block, dm_block_t b,
+			     dm_block_t *root)
+{
+	int r = 0;
+
+	if (dm_block_location(block) != b) {
+		/*
+		 * dm_tm_shadow_block will have already decremented the old
+		 * block, but it is still referenced by the btree.  We
+		 * increment to stop the insert decrementing it below zero
+		 * when overwriting the old value.
+		 */
+		dm_tm_inc(info->btree_info.tm, b);
+		r = insert_ablock(info, index, block, root);
+	}
+
+	return r;
+}
+
 /*
  * Looks up an array block in the btree.  Then shadows it, and updates the
  * btree to point to this new shadow.  'root' is an input/output parameter
@@ -286,49 +328,21 @@ static int shadow_ablock(struct dm_array_info *info, dm_block_t *root,
 			 unsigned index, struct dm_block **block,
 			 struct array_block **ab)
 {
-	int r, inc;
+	int r;
 	uint64_t key = index;
 	dm_block_t b;
 	__le64 block_le;
 
-	/*
-	 * lookup
-	 */
 	r = dm_btree_lookup(&info->btree_info, *root, &key, &block_le);
 	if (r)
 		return r;
 	b = le64_to_cpu(block_le);
 
-	/*
-	 * shadow
-	 */
-	r = dm_tm_shadow_block(info->btree_info.tm, b,
-			       &array_validator, block, &inc);
+	r = __shadow_ablock(info, b, block, ab);
 	if (r)
 		return r;
 
-	*ab = dm_block_data(*block);
-	if (inc)
-		inc_ablock_entries(info, *ab);
-
-	/*
-	 * Reinsert.
-	 *
-	 * The shadow op will often be a noop.  Only insert if it really
-	 * copied data.
-	 */
-	if (dm_block_location(*block) != b) {
-		/*
-		 * dm_tm_shadow_block will have already decremented the old
-		 * block, but it is still referenced by the btree.  We
-		 * increment to stop the insert decrementing it below zero
-		 * when overwriting the old value.
-		 */
-		dm_tm_inc(info->btree_info.tm, b);
-		r = insert_ablock(info, index, *block, root);
-	}
-
-	return r;
+	return __reinsert_ablock(info, index, *block, b, root);
 }
 
 /*
@@ -681,6 +695,72 @@ int dm_array_resize(struct dm_array_info *info, dm_block_t root,
 }
 EXPORT_SYMBOL_GPL(dm_array_resize);
 
+static int populate_ablock_with_values(struct dm_array_info *info, struct array_block *ab,
+				       value_fn fn, void *context, unsigned base, unsigned new_nr)
+{
+	int r;
+	unsigned i;
+	uint32_t nr_entries;
+	struct dm_btree_value_type *vt = &info->value_type;
+
+	BUG_ON(le32_to_cpu(ab->nr_entries));
+	BUG_ON(new_nr > le32_to_cpu(ab->max_entries));
+
+	nr_entries = le32_to_cpu(ab->nr_entries);
+	for (i = 0; i < new_nr; i++) {
+		r = fn(base + i, element_at(info, ab, i), context);
+		if (r)
+			return r;
+
+		if (vt->inc)
+			vt->inc(vt->context, element_at(info, ab, i));
+	}
+
+	ab->nr_entries = cpu_to_le32(new_nr);
+	return 0;
+}
+
+int dm_array_new(struct dm_array_info *info, dm_block_t *root,
+		 uint32_t size, value_fn fn, void *context)
+{
+	int r;
+	struct dm_block *block;
+	struct array_block *ab;
+	unsigned block_index, end_block, size_of_block, max_entries;
+
+	r = dm_array_empty(info, root);
+	if (r)
+		return r;
+
+	size_of_block = dm_bm_block_size(dm_tm_get_bm(info->btree_info.tm));
+	max_entries = calc_max_entries(info->value_type.size, size_of_block);
+	end_block = dm_div_up(size, max_entries);
+
+	for (block_index = 0; block_index != end_block; block_index++) {
+		r = alloc_ablock(info, size_of_block, max_entries, &block, &ab);
+		if (r)
+			break;
+
+		r = populate_ablock_with_values(info, ab, fn, context,
+						block_index * max_entries,
+						min(max_entries, size));
+		if (r) {
+			unlock_ablock(info, block);
+			break;
+		}
+
+		r = insert_ablock(info, block_index, block, root);
+		unlock_ablock(info, block);
+		if (r)
+			break;
+
+		size -= max_entries;
+	}
+
+	return r;
+}
+EXPORT_SYMBOL_GPL(dm_array_new);
+
 int dm_array_del(struct dm_array_info *info, dm_block_t root)
 {
 	return dm_btree_del(&info->btree_info, root);
@@ -819,3 +899,89 @@ int dm_array_walk(struct dm_array_info *info, dm_block_t root,
 EXPORT_SYMBOL_GPL(dm_array_walk);
 
 /*----------------------------------------------------------------*/
+
+static int load_ablock(struct dm_array_cursor *c)
+{
+	int r;
+	__le64 value_le;
+	uint64_t key;
+
+	if (c->block)
+		unlock_ablock(c->info, c->block);
+
+	c->block = NULL;
+	c->ab = NULL;
+	c->index = 0;
+
+	r = dm_btree_cursor_get_value(&c->cursor, &key, &value_le);
+	if (r) {
+		DMERR("dm_btree_cursor_get_value failed");
+		dm_btree_cursor_end(&c->cursor);
+
+	} else {
+		r = get_ablock(c->info, le64_to_cpu(value_le), &c->block, &c->ab);
+		if (r) {
+			DMERR("get_ablock failed");
+			dm_btree_cursor_end(&c->cursor);
+		}
+	}
+
+	return r;
+}
+
+int dm_array_cursor_begin(struct dm_array_info *info, dm_block_t root,
+			  struct dm_array_cursor *c)
+{
+	int r;
+
+	memset(c, 0, sizeof(*c));
+	c->info = info;
+	r = dm_btree_cursor_begin(&info->btree_info, root, true, &c->cursor);
+	if (r) {
+		DMERR("couldn't create btree cursor");
+		return r;
+	}
+
+	return load_ablock(c);
+}
+EXPORT_SYMBOL_GPL(dm_array_cursor_begin);
+
+void dm_array_cursor_end(struct dm_array_cursor *c)
+{
+	if (c->block) {
+		unlock_ablock(c->info, c->block);
+		dm_btree_cursor_end(&c->cursor);
+	}
+}
+EXPORT_SYMBOL_GPL(dm_array_cursor_end);
+
+int dm_array_cursor_next(struct dm_array_cursor *c)
+{
+	int r;
+
+	if (!c->block)
+		return -ENODATA;
+
+	c->index++;
+
+	if (c->index >= le32_to_cpu(c->ab->nr_entries)) {
+		r = dm_btree_cursor_next(&c->cursor);
+		if (r)
+			return r;
+
+		r = load_ablock(c);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(dm_array_cursor_next);
+
+void dm_array_cursor_get_value(struct dm_array_cursor *c, void **value_le)
+{
+	*value_le = element_at(c->info, c->ab, c->index);
+}
+EXPORT_SYMBOL_GPL(dm_array_cursor_get_value);
+
+/*----------------------------------------------------------------*/
diff --git a/drivers/md/persistent-data/dm-array.h b/drivers/md/persistent-data/dm-array.h
index ea177d6fa58f..27ee49a55473 100644
--- a/drivers/md/persistent-data/dm-array.h
+++ b/drivers/md/persistent-data/dm-array.h
@@ -112,6 +112,25 @@ int dm_array_resize(struct dm_array_info *info, dm_block_t root,
 	__dm_written_to_disk(value);
 
 /*
+ * Creates a new array populated with values provided by a callback
+ * function.  This is more efficient than creating an empty array,
+ * resizing, and then setting values since that process incurs a lot of
+ * copying.
+ *
+ * Assumes 32bit values for now since it's only used by the cache hint
+ * array.
+ *
+ * info - describes the array
+ * root - the root block of the array on disk
+ * size - the number of entries in the array
+ * fn - the callback
+ * context - passed to the callback
+ */
+typedef int (*value_fn)(uint32_t index, void *value_le, void *context);
+int dm_array_new(struct dm_array_info *info, dm_block_t *root,
+		 uint32_t size, value_fn fn, void *context);
+
+/*
  * Frees a whole array.  The value_type's decrement operation will be called
  * for all values in the array
  */
@@ -163,4 +182,37 @@ int dm_array_walk(struct dm_array_info *info, dm_block_t root,
 
 /*----------------------------------------------------------------*/
 
+/*
+ * Cursor api.
+ *
+ * This lets you iterate through all the entries in an array efficiently
+ * (it will preload metadata).
+ *
+ * I'm using a cursor, rather than a walk function with a callback because
+ * the cache target needs to iterate both the mapping and hint arrays in
+ * unison.
+ */
+struct dm_array_cursor {
+	struct dm_array_info *info;
+	struct dm_btree_cursor cursor;
+
+	struct dm_block *block;
+	struct array_block *ab;
+	unsigned index;
+};
+
+int dm_array_cursor_begin(struct dm_array_info *info,
+			  dm_block_t root, struct dm_array_cursor *c);
+void dm_array_cursor_end(struct dm_array_cursor *c);
+
+uint32_t dm_array_cursor_index(struct dm_array_cursor *c);
+int dm_array_cursor_next(struct dm_array_cursor *c);
+
+/*
+ * value_le is only valid while the cursor points at the current value.
+ */
+void dm_array_cursor_get_value(struct dm_array_cursor *c, void **value_le);
+
+/*----------------------------------------------------------------*/
+
 #endif	/* _LINUX_DM_ARRAY_H */
diff --git a/drivers/md/persistent-data/dm-btree.c b/drivers/md/persistent-data/dm-btree.c
index 2cc1877804c2..20a40329d84a 100644
--- a/drivers/md/persistent-data/dm-btree.c
+++ b/drivers/md/persistent-data/dm-btree.c
@@ -994,3 +994,165 @@ int dm_btree_walk(struct dm_btree_info *info, dm_block_t root,
 	return walk_node(info, root, fn, context);
 }
 EXPORT_SYMBOL_GPL(dm_btree_walk);
+
+/*----------------------------------------------------------------*/
+
+static void prefetch_values(struct dm_btree_cursor *c)
+{
+	unsigned i, nr;
+	__le64 value_le;
+	struct cursor_node *n = c->nodes + c->depth - 1;
+	struct btree_node *bn = dm_block_data(n->b);
+	struct dm_block_manager *bm = dm_tm_get_bm(c->info->tm);
+
+	BUG_ON(c->info->value_type.size != sizeof(value_le));
+
+	nr = le32_to_cpu(bn->header.nr_entries);
+	for (i = 0; i < nr; i++) {
+		memcpy(&value_le, value_ptr(bn, i), sizeof(value_le));
+		dm_bm_prefetch(bm, le64_to_cpu(value_le));
+	}
+}
+
+static bool leaf_node(struct dm_btree_cursor *c)
+{
+	struct cursor_node *n = c->nodes + c->depth - 1;
+	struct btree_node *bn = dm_block_data(n->b);
+
+	return le32_to_cpu(bn->header.flags) & LEAF_NODE;
+}
+
+static int push_node(struct dm_btree_cursor *c, dm_block_t b)
+{
+	int r;
+	struct cursor_node *n = c->nodes + c->depth;
+
+	if (c->depth >= DM_BTREE_CURSOR_MAX_DEPTH - 1) {
+		DMERR("couldn't push cursor node, stack depth too high");
+		return -EINVAL;
+	}
+
+	r = bn_read_lock(c->info, b, &n->b);
+	if (r)
+		return r;
+
+	n->index = 0;
+	c->depth++;
+
+	if (c->prefetch_leaves || !leaf_node(c))
+		prefetch_values(c);
+
+	return 0;
+}
+
+static void pop_node(struct dm_btree_cursor *c)
+{
+	c->depth--;
+	unlock_block(c->info, c->nodes[c->depth].b);
+}
+
+static int inc_or_backtrack(struct dm_btree_cursor *c)
+{
+	struct cursor_node *n;
+	struct btree_node *bn;
+
+	for (;;) {
+		if (!c->depth)
+			return -ENODATA;
+
+		n = c->nodes + c->depth - 1;
+		bn = dm_block_data(n->b);
+
+		n->index++;
+		if (n->index < le32_to_cpu(bn->header.nr_entries))
+			break;
+
+		pop_node(c);
+	}
+
+	return 0;
+}
+
+static int find_leaf(struct dm_btree_cursor *c)
+{
+	int r = 0;
+	struct cursor_node *n;
+	struct btree_node *bn;
+	__le64 value_le;
+
+	for (;;) {
+		n = c->nodes + c->depth - 1;
+		bn = dm_block_data(n->b);
+
+		if (le32_to_cpu(bn->header.flags) & LEAF_NODE)
+			break;
+
+		memcpy(&value_le, value_ptr(bn, n->index), sizeof(value_le));
+		r = push_node(c, le64_to_cpu(value_le));
+		if (r) {
+			DMERR("push_node failed");
+			break;
+		}
+	}
+
+	if (!r && (le32_to_cpu(bn->header.nr_entries) == 0))
+		return -ENODATA;
+
+	return r;
+}
+
+int dm_btree_cursor_begin(struct dm_btree_info *info, dm_block_t root,
+			  bool prefetch_leaves, struct dm_btree_cursor *c)
+{
+	int r;
+
+	c->info = info;
+	c->root = root;
+	c->depth = 0;
+	c->prefetch_leaves = prefetch_leaves;
+
+	r = push_node(c, root);
+	if (r)
+		return r;
+
+	return find_leaf(c);
+}
+EXPORT_SYMBOL_GPL(dm_btree_cursor_begin);
+
+void dm_btree_cursor_end(struct dm_btree_cursor *c)
+{
+	while (c->depth)
+		pop_node(c);
+}
+EXPORT_SYMBOL_GPL(dm_btree_cursor_end);
+
+int dm_btree_cursor_next(struct dm_btree_cursor *c)
+{
+	int r = inc_or_backtrack(c);
+	if (!r) {
+		r = find_leaf(c);
+		if (r)
+			DMERR("find_leaf failed");
+	}
+
+	return r;
+}
+EXPORT_SYMBOL_GPL(dm_btree_cursor_next);
+
+int dm_btree_cursor_get_value(struct dm_btree_cursor *c, uint64_t *key, void *value_le)
+{
+	if (c->depth) {
+		struct cursor_node *n = c->nodes + c->depth - 1;
+		struct btree_node *bn = dm_block_data(n->b);
+
+		if (le32_to_cpu(bn->header.flags) & INTERNAL_NODE)
+			return -EINVAL;
+
+		*key = le64_to_cpu(*key_ptr(bn, n->index));
+		memcpy(value_le, value_ptr(bn, n->index), c->info->value_type.size);
+		return 0;
+
+	} else
+		return -ENODATA;
+}
+EXPORT_SYMBOL_GPL(dm_btree_cursor_get_value);
diff --git a/drivers/md/persistent-data/dm-btree.h b/drivers/md/persistent-data/dm-btree.h
index c74301fa5a37..db9bd26adf31 100644
--- a/drivers/md/persistent-data/dm-btree.h
+++ b/drivers/md/persistent-data/dm-btree.h
@@ -176,4 +176,39 @@ int dm_btree_walk(struct dm_btree_info *info, dm_block_t root,
 		  int (*fn)(void *context, uint64_t *keys, void *leaf),
 		  void *context);
 
+
+/*----------------------------------------------------------------*/
+
+/*
+ * Cursor API.  This does not follow the rolling lock convention.  Since we
+ * know the order that values are required we can issue prefetches to speed
+ * up iteration.  Use on a single level btree only.
+ */
+#define DM_BTREE_CURSOR_MAX_DEPTH 16
+
+struct cursor_node {
+	struct dm_block *b;
+	unsigned index;
+};
+
+struct dm_btree_cursor {
+	struct dm_btree_info *info;
+	dm_block_t root;
+
+	bool prefetch_leaves;
+	unsigned depth;
+	struct cursor_node nodes[DM_BTREE_CURSOR_MAX_DEPTH];
+};
+
+/*
+ * Creates a fresh cursor.  If prefetch_leaves is set then it is assumed
+ * the btree contains block indexes that will be prefetched.  The cursor is
+ * quite large, so you probably don't want to put it on the stack.
+ */
+int dm_btree_cursor_begin(struct dm_btree_info *info, dm_block_t root,
+			  bool prefetch_leaves, struct dm_btree_cursor *c);
+void dm_btree_cursor_end(struct dm_btree_cursor *c);
+int dm_btree_cursor_next(struct dm_btree_cursor *c);
+int dm_btree_cursor_get_value(struct dm_btree_cursor *c, uint64_t *key, void *value_le);
+
 #endif	/* _LINUX_DM_BTREE_H */
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 21dc00eb1989..1961d827dbd1 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -145,12 +145,8 @@ static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data)
 	return r1_bio;
 
 out_free_pages:
-	while (--j >= 0) {
-		struct bio_vec *bv;
-
-		bio_for_each_segment_all(bv, r1_bio->bios[j], i)
-			__free_page(bv->bv_page);
-	}
+	while (--j >= 0)
+		bio_free_pages(r1_bio->bios[j]);
 
 out_free_bio:
 	while (++j < pi->raid_disks)
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 0e4efcd10795..be1a9fca3b2d 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -1064,6 +1064,8 @@ static void __make_request(struct mddev *mddev, struct bio *bio)
 	int max_sectors;
 	int sectors;
 
+	md_write_start(mddev, bio);
+
 	/*
 	 * Register the new request and wait if the reconstruction
 	 * thread has put up a bar for new requests.
@@ -1445,8 +1447,6 @@ static void raid10_make_request(struct mddev *mddev, struct bio *bio)
 		return;
 	}
 
-	md_write_start(mddev, bio);
-
 	do {
 
 		/*
@@ -2465,20 +2465,21 @@ static int narrow_write_error(struct r10bio *r10_bio, int i)
 
 	while (sect_to_write) {
 		struct bio *wbio;
+		sector_t wsector;
 		if (sectors > sect_to_write)
 			sectors = sect_to_write;
 		/* Write at 'sector' for 'sectors' */
 		wbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
 		bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors);
-		wbio->bi_iter.bi_sector = (r10_bio->devs[i].addr+
-				   choose_data_offset(r10_bio, rdev) +
-				   (sector - r10_bio->sector));
+		wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
+		wbio->bi_iter.bi_sector = wsector +
+				   choose_data_offset(r10_bio, rdev);
 		wbio->bi_bdev = rdev->bdev;
 		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
 
 		if (submit_bio_wait(wbio) < 0)
 			/* Failure! */
-			ok = rdev_set_badblocks(rdev, sector,
+			ok = rdev_set_badblocks(rdev, wsector,
 						sectors, 0)
 				&& ok;
 
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index 51f76ddbe265..1b1ab4a1d132 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -96,7 +96,6 @@ struct r5l_log {
 	spinlock_t no_space_stripes_lock;
 
 	bool need_cache_flush;
-	bool in_teardown;
 };
 
 /*
@@ -704,31 +703,22 @@ static void r5l_write_super_and_discard_space(struct r5l_log *log,
 
 	mddev = log->rdev->mddev;
 	/*
-	 * This is to avoid a deadlock. r5l_quiesce holds reconfig_mutex and
-	 * wait for this thread to finish. This thread waits for
-	 * MD_CHANGE_PENDING clear, which is supposed to be done in
-	 * md_check_recovery(). md_check_recovery() tries to get
-	 * reconfig_mutex. Since r5l_quiesce already holds the mutex,
-	 * md_check_recovery() fails, so the PENDING never get cleared. The
-	 * in_teardown check workaround this issue.
+	 * Discard could zero data, so before discard we must make sure
+	 * superblock is updated to new log tail. Updating superblock (either
+	 * directly call md_update_sb() or depend on md thread) must hold
+	 * reconfig mutex. On the other hand, raid5_quiesce is called with
+	 * reconfig_mutex hold. The first step of raid5_quiesce() is waitting
+	 * for all IO finish, hence waitting for reclaim thread, while reclaim
+	 * thread is calling this function and waitting for reconfig mutex. So
+	 * there is a deadlock. We workaround this issue with a trylock.
+	 * FIXME: we could miss discard if we can't take reconfig mutex
 	 */
-	if (!log->in_teardown) {
-		set_mask_bits(&mddev->flags, 0,
-			      BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_PENDING));
-		md_wakeup_thread(mddev->thread);
-		wait_event(mddev->sb_wait,
-			!test_bit(MD_CHANGE_PENDING, &mddev->flags) ||
-			log->in_teardown);
-		/*
-		 * r5l_quiesce could run after in_teardown check and hold
-		 * mutex first. Superblock might get updated twice.
-		 */
-		if (log->in_teardown)
-			md_update_sb(mddev, 1);
-	} else {
-		WARN_ON(!mddev_is_locked(mddev));
-		md_update_sb(mddev, 1);
-	}
+	set_mask_bits(&mddev->flags, 0,
+		BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_PENDING));
+	if (!mddev_trylock(mddev))
+		return;
+	md_update_sb(mddev, 1);
+	mddev_unlock(mddev);
 
 	/* discard IO error really doesn't matter, ignore it */
 	if (log->last_checkpoint < end) {
@@ -827,7 +817,6 @@ void r5l_quiesce(struct r5l_log *log, int state)
 	if (!log || state == 2)
 		return;
 	if (state == 0) {
-		log->in_teardown = 0;
 		/*
 		 * This is a special case for hotadd. In suspend, the array has
 		 * no journal. In resume, journal is initialized as well as the
@@ -838,11 +827,6 @@ void r5l_quiesce(struct r5l_log *log, int state)
 		log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
 					log->rdev->mddev, "reclaim");
 	} else if (state == 1) {
-		/*
-		 * at this point all stripes are finished, so io_unit is at
-		 * least in STRIPE_END state
-		 */
-		log->in_teardown = 1;
 		/* make sure r5l_write_super_and_discard_space exits */
 		mddev = log->rdev->mddev;
 		wake_up(&mddev->sb_wait);
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 8912407a4dd0..92ac251e91e6 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -659,6 +659,7 @@ raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
 {
 	struct stripe_head *sh;
 	int hash = stripe_hash_locks_hash(sector);
+	int inc_empty_inactive_list_flag;
 
 	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
 
@@ -703,7 +704,12 @@ raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
 					atomic_inc(&conf->active_stripes);
 				BUG_ON(list_empty(&sh->lru) &&
 				       !test_bit(STRIPE_EXPANDING, &sh->state));
+				inc_empty_inactive_list_flag = 0;
+				if (!list_empty(conf->inactive_list + hash))
+					inc_empty_inactive_list_flag = 1;
 				list_del_init(&sh->lru);
+				if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag)
+					atomic_inc(&conf->empty_inactive_list_nr);
 				if (sh->group) {
 					sh->group->stripes_cnt--;
 					sh->group = NULL;
@@ -762,6 +768,7 @@ static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh
 	sector_t head_sector, tmp_sec;
 	int hash;
 	int dd_idx;
+	int inc_empty_inactive_list_flag;
 
 	/* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
 	tmp_sec = sh->sector;
@@ -779,7 +786,12 @@ static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh
 				atomic_inc(&conf->active_stripes);
 			BUG_ON(list_empty(&head->lru) &&
 			       !test_bit(STRIPE_EXPANDING, &head->state));
+			inc_empty_inactive_list_flag = 0;
+			if (!list_empty(conf->inactive_list + hash))
+				inc_empty_inactive_list_flag = 1;
 			list_del_init(&head->lru);
+			if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag)
+				atomic_inc(&conf->empty_inactive_list_nr);
 			if (head->group) {
 				head->group->stripes_cnt--;
 				head->group = NULL;
@@ -993,7 +1005,6 @@ again:
 
 			set_bit(STRIPE_IO_STARTED, &sh->state);
 
-			bio_reset(bi);
 			bi->bi_bdev = rdev->bdev;
 			bio_set_op_attrs(bi, op, op_flags);
 			bi->bi_end_io = op_is_write(op)
@@ -1045,7 +1056,6 @@ again:
 
 			set_bit(STRIPE_IO_STARTED, &sh->state);
 
-			bio_reset(rbi);
 			rbi->bi_bdev = rrdev->bdev;
 			bio_set_op_attrs(rbi, op, op_flags);
 			BUG_ON(!op_is_write(op));
@@ -1978,9 +1988,11 @@ static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
 	put_cpu();
 }
 
-static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp)
+static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp,
+	int disks)
 {
 	struct stripe_head *sh;
+	int i;
 
 	sh = kmem_cache_zalloc(sc, gfp);
 	if (sh) {
@@ -1989,6 +2001,17 @@ static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp)
 		INIT_LIST_HEAD(&sh->batch_list);
 		INIT_LIST_HEAD(&sh->lru);
 		atomic_set(&sh->count, 1);
+		for (i = 0; i < disks; i++) {
+			struct r5dev *dev = &sh->dev[i];
+
+			bio_init(&dev->req);
+			dev->req.bi_io_vec = &dev->vec;
+			dev->req.bi_max_vecs = 1;
+
+			bio_init(&dev->rreq);
+			dev->rreq.bi_io_vec = &dev->rvec;
+			dev->rreq.bi_max_vecs = 1;
+		}
 	}
 	return sh;
 }
@@ -1996,7 +2019,7 @@ static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
 {
 	struct stripe_head *sh;
 
-	sh = alloc_stripe(conf->slab_cache, gfp);
+	sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size);
 	if (!sh)
 		return 0;
 
@@ -2167,7 +2190,7 @@ static int resize_stripes(struct r5conf *conf, int newsize)
 	mutex_lock(&conf->cache_size_mutex);
 
 	for (i = conf->max_nr_stripes; i; i--) {
-		nsh = alloc_stripe(sc, GFP_KERNEL);
+		nsh = alloc_stripe(sc, GFP_KERNEL, newsize);
 		if (!nsh)
 			break;
 
@@ -2299,6 +2322,7 @@ static void raid5_end_read_request(struct bio * bi)
 		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
 		bi->bi_error);
 	if (i == disks) {
+		bio_reset(bi);
 		BUG();
 		return;
 	}
@@ -2399,6 +2423,7 @@ static void raid5_end_read_request(struct bio * bi)
 		}
 	}
 	rdev_dec_pending(rdev, conf->mddev);
+	bio_reset(bi);
 	clear_bit(R5_LOCKED, &sh->dev[i].flags);
 	set_bit(STRIPE_HANDLE, &sh->state);
 	raid5_release_stripe(sh);
@@ -2436,6 +2461,7 @@ static void raid5_end_write_request(struct bio *bi)
 		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
 		bi->bi_error);
 	if (i == disks) {
+		bio_reset(bi);
 		BUG();
 		return;
 	}
@@ -2472,6 +2498,7 @@ static void raid5_end_write_request(struct bio *bi)
 	if (sh->batch_head && bi->bi_error && !replacement)
 		set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
 
+	bio_reset(bi);
 	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
 		clear_bit(R5_LOCKED, &sh->dev[i].flags);
 	set_bit(STRIPE_HANDLE, &sh->state);
@@ -2485,16 +2512,6 @@ static void raid5_build_block(struct stripe_head *sh, int i, int previous)
 {
 	struct r5dev *dev = &sh->dev[i];
 
-	bio_init(&dev->req);
-	dev->req.bi_io_vec = &dev->vec;
-	dev->req.bi_max_vecs = 1;
-	dev->req.bi_private = sh;
-
-	bio_init(&dev->rreq);
-	dev->rreq.bi_io_vec = &dev->rvec;
-	dev->rreq.bi_max_vecs = 1;
-	dev->rreq.bi_private = sh;
-
 	dev->flags = 0;
 	dev->sector = raid5_compute_blocknr(sh, i, previous);
 }
@@ -4628,7 +4645,9 @@ finish:
 	}
 
 	if (!bio_list_empty(&s.return_bi)) {
-		if (test_bit(MD_CHANGE_PENDING, &conf->mddev->flags)) {
+		if (test_bit(MD_CHANGE_PENDING, &conf->mddev->flags) &&
+				(s.failed <= conf->max_degraded ||
+					conf->mddev->external == 0)) {
 			spin_lock_irq(&conf->device_lock);
 			bio_list_merge(&conf->return_bi, &s.return_bi);
 			spin_unlock_irq(&conf->device_lock);
@@ -6330,22 +6349,20 @@ static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu
 	return 0;
 }
 
-static void raid5_free_percpu(struct r5conf *conf)
+static int raid456_cpu_dead(unsigned int cpu, struct hlist_node *node)
 {
-	unsigned long cpu;
+	struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);
+
+	free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
+	return 0;
+}
 
+static void raid5_free_percpu(struct r5conf *conf)
+{
 	if (!conf->percpu)
 		return;
 
-#ifdef CONFIG_HOTPLUG_CPU
-	unregister_cpu_notifier(&conf->cpu_notify);
-#endif
-
-	get_online_cpus();
-	for_each_possible_cpu(cpu)
-		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
-	put_online_cpus();
-
+	cpuhp_state_remove_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
 	free_percpu(conf->percpu);
 }
 
@@ -6353,7 +6370,7 @@ static void free_conf(struct r5conf *conf)
 {
 	if (conf->log)
 		r5l_exit_log(conf->log);
-	if (conf->shrinker.seeks)
+	if (conf->shrinker.nr_deferred)
 		unregister_shrinker(&conf->shrinker);
 
 	free_thread_groups(conf);
@@ -6364,64 +6381,28 @@ static void free_conf(struct r5conf *conf)
 	kfree(conf);
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
-static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
-			      void *hcpu)
+static int raid456_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)
 {
-	struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
-	long cpu = (long)hcpu;
+	struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);
 	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
 
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		if (alloc_scratch_buffer(conf, percpu)) {
-			pr_err("%s: failed memory allocation for cpu%ld\n",
-			       __func__, cpu);
-			return notifier_from_errno(-ENOMEM);
-		}
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
-		break;
-	default:
-		break;
+	if (alloc_scratch_buffer(conf, percpu)) {
+		pr_err("%s: failed memory allocation for cpu%u\n",
+		       __func__, cpu);
+		return -ENOMEM;
 	}
-	return NOTIFY_OK;
+	return 0;
 }
-#endif
 
 static int raid5_alloc_percpu(struct r5conf *conf)
 {
-	unsigned long cpu;
 	int err = 0;
 
 	conf->percpu = alloc_percpu(struct raid5_percpu);
 	if (!conf->percpu)
 		return -ENOMEM;
 
-#ifdef CONFIG_HOTPLUG_CPU
-	conf->cpu_notify.notifier_call = raid456_cpu_notify;
-	conf->cpu_notify.priority = 0;
-	err = register_cpu_notifier(&conf->cpu_notify);
-	if (err)
-		return err;
-#endif
-
-	get_online_cpus();
-	for_each_present_cpu(cpu) {
-		err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
-		if (err) {
-			pr_err("%s: failed memory allocation for cpu%ld\n",
-			       __func__, cpu);
-			break;
-		}
-	}
-	put_online_cpus();
-
+	err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
 	if (!err) {
 		conf->scribble_disks = max(conf->raid_disks,
 			conf->previous_raid_disks);
@@ -6620,6 +6601,16 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 	}
 
 	conf->min_nr_stripes = NR_STRIPES;
+	if (mddev->reshape_position != MaxSector) {
+		int stripes = max_t(int,
+			((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4,
+			((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4);
+		conf->min_nr_stripes = max(NR_STRIPES, stripes);
+		if (conf->min_nr_stripes != NR_STRIPES)
+			printk(KERN_INFO
+				"md/raid:%s: force stripe size %d for reshape\n",
+				mdname(mddev), conf->min_nr_stripes);
+	}
 	memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +
 		 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
 	atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
@@ -6641,7 +6632,12 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 	conf->shrinker.count_objects = raid5_cache_count;
 	conf->shrinker.batch = 128;
 	conf->shrinker.flags = 0;
-	register_shrinker(&conf->shrinker);
+	if (register_shrinker(&conf->shrinker)) {
+		printk(KERN_ERR
+		       "md/raid:%s: couldn't register shrinker.\n",
+		       mdname(mddev));
+		goto abort;
+	}
 
 	sprintf(pers_name, "raid%d", mddev->new_level);
 	conf->thread = md_register_thread(raid5d, mddev, pers_name);
@@ -6826,11 +6822,14 @@ static int raid5_run(struct mddev *mddev)
 	if (IS_ERR(conf))
 		return PTR_ERR(conf);
 
-	if (test_bit(MD_HAS_JOURNAL, &mddev->flags) && !journal_dev) {
-		printk(KERN_ERR "md/raid:%s: journal disk is missing, force array readonly\n",
-		       mdname(mddev));
-		mddev->ro = 1;
-		set_disk_ro(mddev->gendisk, 1);
+	if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
+		if (!journal_dev) {
+			pr_err("md/raid:%s: journal disk is missing, force array readonly\n",
+			       mdname(mddev));
+			mddev->ro = 1;
+			set_disk_ro(mddev->gendisk, 1);
+		} else if (mddev->recovery_cp == MaxSector)
+			set_bit(MD_JOURNAL_CLEAN, &mddev->flags);
 	}
 
 	conf->min_offset_diff = min_offset_diff;
@@ -7034,6 +7033,8 @@ static int raid5_run(struct mddev *mddev)
 		else
 			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
 						mddev->queue);
+
+		blk_queue_max_hw_sectors(mddev->queue, UINT_MAX);
 	}
 
 	if (journal_dev) {
@@ -7953,10 +7954,21 @@ static struct md_personality raid4_personality =
 
 static int __init raid5_init(void)
 {
+	int ret;
+
 	raid5_wq = alloc_workqueue("raid5wq",
 		WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
 	if (!raid5_wq)
 		return -ENOMEM;
+
+	ret = cpuhp_setup_state_multi(CPUHP_MD_RAID5_PREPARE,
+				      "md/raid5:prepare",
+				      raid456_cpu_up_prepare,
+				      raid456_cpu_dead);
+	if (ret) {
+		destroy_workqueue(raid5_wq);
+		return ret;
+	}
 	register_md_personality(&raid6_personality);
 	register_md_personality(&raid5_personality);
 	register_md_personality(&raid4_personality);
@@ -7968,6 +7980,7 @@ static void raid5_exit(void)
 	unregister_md_personality(&raid6_personality);
 	unregister_md_personality(&raid5_personality);
 	unregister_md_personality(&raid4_personality);
+	cpuhp_remove_multi_state(CPUHP_MD_RAID5_PREPARE);
 	destroy_workqueue(raid5_wq);
 }
 
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h
index 517d4b68a1be..57ec49f0839e 100644
--- a/drivers/md/raid5.h
+++ b/drivers/md/raid5.h
@@ -512,9 +512,7 @@ struct r5conf {
 	} __percpu *percpu;
 	int scribble_disks;
 	int scribble_sectors;
-#ifdef CONFIG_HOTPLUG_CPU
-	struct notifier_block	cpu_notify;
-#endif
+	struct hlist_node node;
 
 	/*
 	 * Free stripes pool
diff --git a/drivers/media/Kconfig b/drivers/media/Kconfig
index 962f2a9a6614..7b8540291217 100644
--- a/drivers/media/Kconfig
+++ b/drivers/media/Kconfig
@@ -180,14 +180,14 @@ source "drivers/media/firewire/Kconfig"
 # Common driver options
 source "drivers/media/common/Kconfig"
 
-comment "Media ancillary drivers (tuners, sensors, i2c, frontends)"
+comment "Media ancillary drivers (tuners, sensors, i2c, spi, frontends)"
 
 #
-# Ancillary drivers (tuners, i2c, frontends)
+# Ancillary drivers (tuners, i2c, spi, frontends)
 #
 
 config MEDIA_SUBDRV_AUTOSELECT
-	bool "Autoselect ancillary drivers (tuners, sensors, i2c, frontends)"
+	bool "Autoselect ancillary drivers (tuners, sensors, i2c, spi, frontends)"
 	depends on MEDIA_ANALOG_TV_SUPPORT || MEDIA_DIGITAL_TV_SUPPORT || MEDIA_CAMERA_SUPPORT || MEDIA_SDR_SUPPORT
 	depends on HAS_IOMEM
 	select I2C
@@ -216,6 +216,7 @@ config MEDIA_ATTACH
 	default MODULES
 
 source "drivers/media/i2c/Kconfig"
+source "drivers/media/spi/Kconfig"
 source "drivers/media/tuners/Kconfig"
 source "drivers/media/dvb-frontends/Kconfig"
 
diff --git a/drivers/media/Makefile b/drivers/media/Makefile
index 081a7866fd44..0deaa93efdee 100644
--- a/drivers/media/Makefile
+++ b/drivers/media/Makefile
@@ -32,6 +32,6 @@ obj-y += rc/
 # Finally, merge the drivers that require the core
 #
 
-obj-y += common/ platform/ pci/ usb/ mmc/ firewire/
+obj-y += common/ platform/ pci/ usb/ mmc/ firewire/ spi/
 obj-$(CONFIG_VIDEO_DEV) += radio/
 
diff --git a/drivers/media/cec-edid.c b/drivers/media/cec-edid.c
index 70018247bdda..5719b991e340 100644
--- a/drivers/media/cec-edid.c
+++ b/drivers/media/cec-edid.c
@@ -70,7 +70,10 @@ static unsigned int cec_get_edid_spa_location(const u8 *edid, unsigned int size)
 				u8 tag = edid[i] >> 5;
 				u8 len = edid[i] & 0x1f;
 
-				if (tag == 3 && len >= 5 && i + len <= end)
+				if (tag == 3 && len >= 5 && i + len <= end &&
+				    edid[i + 1] == 0x03 &&
+				    edid[i + 2] == 0x0c &&
+				    edid[i + 3] == 0x00)
 					return i + 4;
 				i += len + 1;
 			} while (i < end);
diff --git a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
index 3ec3cebe62b9..1684810cab83 100644
--- a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
+++ b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
@@ -504,14 +504,14 @@ static void color_to_ycbcr(struct tpg_data *tpg, int r, int g, int b,
 #define COEFF(v, r) ((int)(0.5 + (v) * (r) * 256.0))
 
 	static const int bt601[3][3] = {
-		{ COEFF(0.299, 219),  COEFF(0.587, 219),  COEFF(0.114, 219)  },
-		{ COEFF(-0.169, 224), COEFF(-0.331, 224), COEFF(0.5, 224)    },
-		{ COEFF(0.5, 224),    COEFF(-0.419, 224), COEFF(-0.081, 224) },
+		{ COEFF(0.299, 219),   COEFF(0.587, 219),   COEFF(0.114, 219)   },
+		{ COEFF(-0.1687, 224), COEFF(-0.3313, 224), COEFF(0.5, 224)     },
+		{ COEFF(0.5, 224),     COEFF(-0.4187, 224), COEFF(-0.0813, 224) },
 	};
 	static const int bt601_full[3][3] = {
-		{ COEFF(0.299, 255),  COEFF(0.587, 255),  COEFF(0.114, 255)  },
-		{ COEFF(-0.169, 255), COEFF(-0.331, 255), COEFF(0.5, 255)    },
-		{ COEFF(0.5, 255),    COEFF(-0.419, 255), COEFF(-0.081, 255) },
+		{ COEFF(0.299, 255),   COEFF(0.587, 255),   COEFF(0.114, 255)   },
+		{ COEFF(-0.1687, 255), COEFF(-0.3313, 255), COEFF(0.5, 255)     },
+		{ COEFF(0.5, 255),     COEFF(-0.4187, 255), COEFF(-0.0813, 255) },
 	};
 	static const int rec709[3][3] = {
 		{ COEFF(0.2126, 219),  COEFF(0.7152, 219),  COEFF(0.0722, 219)  },
@@ -558,7 +558,6 @@ static void color_to_ycbcr(struct tpg_data *tpg, int r, int g, int b,
 
 	switch (tpg->real_ycbcr_enc) {
 	case V4L2_YCBCR_ENC_601:
-	case V4L2_YCBCR_ENC_SYCC:
 		rgb2ycbcr(full ? bt601_full : bt601, r, g, b, y_offset, y, cb, cr);
 		break;
 	case V4L2_YCBCR_ENC_XV601:
@@ -674,7 +673,6 @@ static void ycbcr_to_color(struct tpg_data *tpg, int y, int cb, int cr,
 
 	switch (tpg->real_ycbcr_enc) {
 	case V4L2_YCBCR_ENC_601:
-	case V4L2_YCBCR_ENC_SYCC:
 		ycbcr2rgb(full ? bt601_full : bt601, y, cb, cr, y_offset, r, g, b);
 		break;
 	case V4L2_YCBCR_ENC_XV601:
diff --git a/drivers/media/dvb-core/demux.h b/drivers/media/dvb-core/demux.h
index 4b4c1da20f4b..aeda2b64931c 100644
--- a/drivers/media/dvb-core/demux.h
+++ b/drivers/media/dvb-core/demux.h
@@ -202,7 +202,7 @@ struct dmx_section_feed {
  *
  * This function callback prototype, provided by the client of the demux API,
  * is called from the demux code. The function is only called when filtering
- * on ae TS feed has been enabled using the start_filtering() function at
+ * on a TS feed has been enabled using the start_filtering\(\) function at
  * the &dmx_demux.
  * Any TS packets that match the filter settings are copied to a circular
  * buffer. The filtered TS packets are delivered to the client using this
@@ -243,8 +243,10 @@ struct dmx_section_feed {
  * will also be sent to the hardware MPEG decoder.
  *
  * Return:
- * 	0, on success;
- * 	-EOVERFLOW, on buffer overflow.
+ *
+ * - 0, on success;
+ *
+ * - -EOVERFLOW, on buffer overflow.
  */
 typedef int (*dmx_ts_cb)(const u8 *buffer1,
 			 size_t buffer1_length,
@@ -293,9 +295,9 @@ typedef int (*dmx_section_cb)(const u8 *buffer1,
 			      size_t buffer2_len,
 			      struct dmx_section_filter *source);
 
-/*--------------------------------------------------------------------------*/
-/* DVB Front-End */
-/*--------------------------------------------------------------------------*/
+/*
+ * DVB Front-End
+ */
 
 /**
  * enum dmx_frontend_source - Used to identify the type of frontend
@@ -349,15 +351,15 @@ enum dmx_demux_caps {
 
 /*
  * Demux resource type identifier.
-*/
-
-/*
- * DMX_FE_ENTRY(): Casts elements in the list of registered
- * front-ends from the generic type struct list_head
- * to the type * struct dmx_frontend
- *.
-*/
+ */
 
+/**
+ * DMX_FE_ENTRY - Casts elements in the list of registered
+ *		  front-ends from the generic type struct list_head
+ *		  to the type * struct dmx_frontend
+ *
+ * @list: list of struct dmx_frontend
+ */
 #define DMX_FE_ENTRY(list) \
 	list_entry(list, struct dmx_frontend, connectivity_list)
 
@@ -551,7 +553,6 @@ enum dmx_demux_caps {
  *	0 on success;
  *	-EINVAL on bad parameter.
  */
-
 struct dmx_demux {
 	enum dmx_demux_caps capabilities;
 	struct dmx_frontend *frontend;
@@ -581,15 +582,12 @@ struct dmx_demux {
 
 	int (*get_pes_pids)(struct dmx_demux *demux, u16 *pids);
 
-	/* private: Not used upstream and never documented */
-#if 0
-	int (*get_caps)(struct dmx_demux *demux, struct dmx_caps *caps);
-	int (*set_source)(struct dmx_demux *demux, const dmx_source_t *src);
-#endif
+	/* private: */
+
 	/*
-	 * private: Only used at av7110, to read some data from firmware.
-	 *	As this was never documented, we have no clue about what's
-	 *	there, and its usage on other drivers aren't encouraged.
+	 * Only used at av7110, to read some data from firmware.
+	 * As this was never documented, we have no clue about what's
+	 * there, and its usage on other drivers aren't encouraged.
 	 */
 	int (*get_stc)(struct dmx_demux *demux, unsigned int num,
 		       u64 *stc, unsigned int *base);
diff --git a/drivers/media/dvb-core/dvb_frontend.c b/drivers/media/dvb-core/dvb_frontend.c
index be99c8dbc5f8..01511e5a5566 100644
--- a/drivers/media/dvb-core/dvb_frontend.c
+++ b/drivers/media/dvb-core/dvb_frontend.c
@@ -1969,17 +1969,9 @@ static int dvb_frontend_ioctl_properties(struct file *file,
 		if ((tvps->num == 0) || (tvps->num > DTV_IOCTL_MAX_MSGS))
 			return -EINVAL;
 
-		tvp = kmalloc(tvps->num * sizeof(struct dtv_property), GFP_KERNEL);
-		if (!tvp) {
-			err = -ENOMEM;
-			goto out;
-		}
-
-		if (copy_from_user(tvp, (void __user *)tvps->props,
-				   tvps->num * sizeof(struct dtv_property))) {
-			err = -EFAULT;
-			goto out;
-		}
+		tvp = memdup_user(tvps->props, tvps->num * sizeof(*tvp));
+		if (IS_ERR(tvp))
+			return PTR_ERR(tvp);
 
 		for (i = 0; i < tvps->num; i++) {
 			err = dtv_property_process_set(fe, tvp + i, file);
@@ -2002,17 +1994,9 @@ static int dvb_frontend_ioctl_properties(struct file *file,
 		if ((tvps->num == 0) || (tvps->num > DTV_IOCTL_MAX_MSGS))
 			return -EINVAL;
 
-		tvp = kmalloc(tvps->num * sizeof(struct dtv_property), GFP_KERNEL);
-		if (!tvp) {
-			err = -ENOMEM;
-			goto out;
-		}
-
-		if (copy_from_user(tvp, (void __user *)tvps->props,
-				   tvps->num * sizeof(struct dtv_property))) {
-			err = -EFAULT;
-			goto out;
-		}
+		tvp = memdup_user(tvps->props, tvps->num * sizeof(*tvp));
+		if (IS_ERR(tvp))
+			return PTR_ERR(tvp);
 
 		/*
 		 * Let's use our own copy of property cache, in order to
diff --git a/drivers/media/dvb-core/dvb_math.h b/drivers/media/dvb-core/dvb_math.h
index 2f0326674ca6..4d11d3529c14 100644
--- a/drivers/media/dvb-core/dvb_math.h
+++ b/drivers/media/dvb-core/dvb_math.h
@@ -25,7 +25,7 @@
 #include <linux/types.h>
 
 /**
- * cintlog2 - computes log2 of a value; the result is shifted left by 24 bits
+ * intlog2 - computes log2 of a value; the result is shifted left by 24 bits
  *
  * @value: The value (must be != 0)
  *
diff --git a/drivers/media/dvb-core/dvb_ringbuffer.h b/drivers/media/dvb-core/dvb_ringbuffer.h
index 8af642399f1e..bbe94873d44d 100644
--- a/drivers/media/dvb-core/dvb_ringbuffer.h
+++ b/drivers/media/dvb-core/dvb_ringbuffer.h
@@ -18,10 +18,6 @@
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  */
 
 #ifndef _DVB_RINGBUFFER_H_
@@ -30,6 +26,18 @@
 #include <linux/spinlock.h>
 #include <linux/wait.h>
 
+/**
+ * struct dvb_ringbuffer - Describes a ring buffer used at DVB framework
+ *
+ * @data: Area were the ringbuffer data is written
+ * @size: size of the ringbuffer
+ * @pread: next position to read
+ * @pwrite: next position to write
+ * @error: used by ringbuffer clients to indicate that an error happened.
+ * @queue: Wait queue used by ringbuffer clients to indicate when buffer
+ *         was filled
+ * @lock: Spinlock used to protect the ringbuffer
+ */
 struct dvb_ringbuffer {
 	u8               *data;
 	ssize_t           size;
@@ -43,99 +51,161 @@ struct dvb_ringbuffer {
 
 #define DVB_RINGBUFFER_PKTHDRSIZE 3
 
+/**
+ * dvb_ringbuffer_init - initialize ring buffer, lock and queue
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ * @data: pointer to the buffer where the data will be stored
+ * @len: bytes from ring buffer into @buf
+ */
+extern void dvb_ringbuffer_init(struct dvb_ringbuffer *rbuf, void *data,
+				size_t len);
 
-/*
- * Notes:
- * ------
- * (1) For performance reasons read and write routines don't check buffer sizes
- *     and/or number of bytes free/available. This has to be done before these
- *     routines are called. For example:
- *
- *     *** write @buflen: bytes ***
- *     free = dvb_ringbuffer_free(rbuf);
- *     if (free >= buflen)
- *         count = dvb_ringbuffer_write(rbuf, buffer, buflen);
- *     else
- *         ...
- *
- *     *** read min. 1000, max. @bufsize: bytes ***
- *     avail = dvb_ringbuffer_avail(rbuf);
- *     if (avail >= 1000)
- *         count = dvb_ringbuffer_read(rbuf, buffer, min(avail, bufsize));
- *     else
- *         ...
- *
- * (2) If there is exactly one reader and one writer, there is no need
- *     to lock read or write operations.
- *     Two or more readers must be locked against each other.
- *     Flushing the buffer counts as a read operation.
- *     Resetting the buffer counts as a read and write operation.
- *     Two or more writers must be locked against each other.
- */
-
-/* initialize ring buffer, lock and queue */
-extern void dvb_ringbuffer_init(struct dvb_ringbuffer *rbuf, void *data, size_t len);
-
-/* test whether buffer is empty */
+/**
+ * dvb_ringbuffer_empty - test whether buffer is empty
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ */
 extern int dvb_ringbuffer_empty(struct dvb_ringbuffer *rbuf);
 
-/* return the number of free bytes in the buffer */
+/**
+ * dvb_ringbuffer_free - returns the number of free bytes in the buffer
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ *
+ * Return: number of free bytes in the buffer
+ */
 extern ssize_t dvb_ringbuffer_free(struct dvb_ringbuffer *rbuf);
 
-/* return the number of bytes waiting in the buffer */
+/**
+ * dvb_ringbuffer_avail - returns the number of bytes waiting in the buffer
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ *
+ * Return: number of bytes waiting in the buffer
+ */
 extern ssize_t dvb_ringbuffer_avail(struct dvb_ringbuffer *rbuf);
 
-
-/*
- * Reset the read and write pointers to zero and flush the buffer
+/**
+ * dvb_ringbuffer_reset - resets the ringbuffer to initial state
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ *
+ * Resets the read and write pointers to zero and flush the buffer.
+ *
  * This counts as a read and write operation
  */
 extern void dvb_ringbuffer_reset(struct dvb_ringbuffer *rbuf);
 
+/*
+ * read routines & macros
+ */
 
-/* read routines & macros */
-/* ---------------------- */
-/* flush buffer */
+/**
+ * dvb_ringbuffer_flush - flush buffer
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ */
 extern void dvb_ringbuffer_flush(struct dvb_ringbuffer *rbuf);
 
-/* flush buffer protected by spinlock and wake-up waiting task(s) */
+/**
+ * dvb_ringbuffer_flush_spinlock_wakeup- flush buffer protected by spinlock
+ *      and wake-up waiting task(s)
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ */
 extern void dvb_ringbuffer_flush_spinlock_wakeup(struct dvb_ringbuffer *rbuf);
 
-/* peek at byte @offs: in the buffer */
-#define DVB_RINGBUFFER_PEEK(rbuf,offs)	\
-			(rbuf)->data[((rbuf)->pread+(offs))%(rbuf)->size]
+/**
+ * DVB_RINGBUFFER_PEEK - peek at byte @offs in the buffer
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ * @offs: offset inside the ringbuffer
+ */
+#define DVB_RINGBUFFER_PEEK(rbuf, offs)	\
+			((rbuf)->data[((rbuf)->pread + (offs)) % (rbuf)->size])
 
-/* advance read ptr by @num: bytes */
-#define DVB_RINGBUFFER_SKIP(rbuf,num)	\
-			(rbuf)->pread=((rbuf)->pread+(num))%(rbuf)->size
+/**
+ * DVB_RINGBUFFER_SKIP - advance read ptr by @num bytes
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ * @num: number of bytes to advance
+ */
+#define DVB_RINGBUFFER_SKIP(rbuf, num)	{\
+			(rbuf)->pread = ((rbuf)->pread + (num)) % (rbuf)->size;\
+}
 
-/*
- * read @len: bytes from ring buffer into @buf:
- * @usermem: specifies whether @buf: resides in user space
- * returns number of bytes transferred or -EFAULT
+/**
+ * dvb_ringbuffer_read_user - Reads a buffer into an user pointer
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ * @buf: pointer to the buffer where the data will be stored
+ * @len: bytes from ring buffer into @buf
+ *
+ * This variant assumes that the buffer is a memory at the userspace. So,
+ * it will internally call copy_to_user().
+ *
+ * Return: number of bytes transferred or -EFAULT
  */
 extern ssize_t dvb_ringbuffer_read_user(struct dvb_ringbuffer *rbuf,
 				   u8 __user *buf, size_t len);
+
+/**
+ * dvb_ringbuffer_read - Reads a buffer into a pointer
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ * @buf: pointer to the buffer where the data will be stored
+ * @len: bytes from ring buffer into @buf
+ *
+ * This variant assumes that the buffer is a memory at the Kernel space
+ *
+ * Return: number of bytes transferred or -EFAULT
+ */
 extern void dvb_ringbuffer_read(struct dvb_ringbuffer *rbuf,
 				   u8 *buf, size_t len);
 
-
-/* write routines & macros */
-/* ----------------------- */
-/* write single byte to ring buffer */
-#define DVB_RINGBUFFER_WRITE_BYTE(rbuf,byte)	\
-			{ (rbuf)->data[(rbuf)->pwrite]=(byte); \
-			(rbuf)->pwrite=((rbuf)->pwrite+1)%(rbuf)->size; }
 /*
- * write @len: bytes to ring buffer
- * @usermem: specifies whether @buf: resides in user space
- * returns number of bytes transferred or -EFAULT
-*/
+ * write routines & macros
+ */
+
+/**
+ * DVB_RINGBUFFER_WRITE_BYTE - write single byte to ring buffer
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ * @byte: byte to write
+ */
+#define DVB_RINGBUFFER_WRITE_BYTE(rbuf, byte)	\
+			{ (rbuf)->data[(rbuf)->pwrite] = (byte); \
+			(rbuf)->pwrite = ((rbuf)->pwrite + 1) % (rbuf)->size; }
+
+/**
+ * dvb_ringbuffer_write - Writes a buffer into the ringbuffer
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ * @buf: pointer to the buffer where the data will be read
+ * @len: bytes from ring buffer into @buf
+ *
+ * This variant assumes that the buffer is a memory at the Kernel space
+ *
+ * return: number of bytes transferred or -EFAULT
+ */
 extern ssize_t dvb_ringbuffer_write(struct dvb_ringbuffer *rbuf, const u8 *buf,
 				    size_t len);
-extern ssize_t dvb_ringbuffer_write_user(struct dvb_ringbuffer *rbuf,
-				         const u8 __user *buf, size_t len);
 
+/**
+ * dvb_ringbuffer_write_user - Writes a buffer received via an user pointer
+ *
+ * @rbuf: pointer to struct dvb_ringbuffer
+ * @buf: pointer to the buffer where the data will be read
+ * @len: bytes from ring buffer into @buf
+ *
+ * This variant assumes that the buffer is a memory at the userspace. So,
+ * it will internally call copy_from_user().
+ *
+ * Return: number of bytes transferred or -EFAULT
+ */
+extern ssize_t dvb_ringbuffer_write_user(struct dvb_ringbuffer *rbuf,
+					 const u8 __user *buf, size_t len);
 
 /**
  * dvb_ringbuffer_pkt_write - Write a packet into the ringbuffer.
@@ -143,9 +213,10 @@ extern ssize_t dvb_ringbuffer_write_user(struct dvb_ringbuffer *rbuf,
  * @rbuf: Ringbuffer to write to.
  * @buf: Buffer to write.
  * @len: Length of buffer (currently limited to 65535 bytes max).
- * returns Number of bytes written, or -EFAULT, -ENOMEM, -EVINAL.
+ *
+ * Return: Number of bytes written, or -EFAULT, -ENOMEM, -EVINAL.
  */
-extern ssize_t dvb_ringbuffer_pkt_write(struct dvb_ringbuffer *rbuf, u8* buf,
+extern ssize_t dvb_ringbuffer_pkt_write(struct dvb_ringbuffer *rbuf, u8 *buf,
 					size_t len);
 
 /**
@@ -157,7 +228,7 @@ extern ssize_t dvb_ringbuffer_pkt_write(struct dvb_ringbuffer *rbuf, u8* buf,
  * @buf: Destination buffer for data.
  * @len: Size of destination buffer.
  *
- * returns Number of bytes read, or -EFAULT.
+ * Return: Number of bytes read, or -EFAULT.
  *
  * .. note::
  *
@@ -167,7 +238,7 @@ extern ssize_t dvb_ringbuffer_pkt_write(struct dvb_ringbuffer *rbuf, u8* buf,
  */
 extern ssize_t dvb_ringbuffer_pkt_read_user(struct dvb_ringbuffer *rbuf,
 					    size_t idx,
-				            int offset, u8 __user *buf,
+					    int offset, u8 __user *buf,
 					    size_t len);
 
 /**
@@ -181,7 +252,7 @@ extern ssize_t dvb_ringbuffer_pkt_read_user(struct dvb_ringbuffer *rbuf,
  * @buf: Destination buffer for data.
  * @len: Size of destination buffer.
  *
- * returns Number of bytes read, or -EFAULT.
+ * Return: Number of bytes read, or -EFAULT.
  */
 extern ssize_t dvb_ringbuffer_pkt_read(struct dvb_ringbuffer *rbuf, size_t idx,
 				       int offset, u8 *buf, size_t len);
@@ -199,10 +270,11 @@ extern void dvb_ringbuffer_pkt_dispose(struct dvb_ringbuffer *rbuf, size_t idx);
  *
  * @rbuf: Ringbuffer concerned.
  * @idx: Previous packet index, or -1 to return the first packet index.
- * @pktlen: On success, will be updated to contain the length of the packet in bytes.
+ * @pktlen: On success, will be updated to contain the length of the packet
+ *          in bytes.
  * returns Packet index (if >=0), or -1 if no packets available.
  */
-extern ssize_t dvb_ringbuffer_pkt_next(struct dvb_ringbuffer *rbuf, size_t idx, size_t* pktlen);
-
+extern ssize_t dvb_ringbuffer_pkt_next(struct dvb_ringbuffer *rbuf,
+				       size_t idx, size_t *pktlen);
 
 #endif /* _DVB_RINGBUFFER_H_ */
diff --git a/drivers/media/dvb-frontends/Kconfig b/drivers/media/dvb-frontends/Kconfig
index c645aa81f423..012225587c25 100644
--- a/drivers/media/dvb-frontends/Kconfig
+++ b/drivers/media/dvb-frontends/Kconfig
@@ -67,6 +67,7 @@ config DVB_TDA18271C2DD
 config DVB_SI2165
 	tristate "Silicon Labs si2165 based"
 	depends on DVB_CORE && I2C
+	select REGMAP_I2C
 	default m if !MEDIA_SUBDRV_AUTOSELECT
 	help
 	  A DVB-C/T demodulator.
@@ -463,6 +464,7 @@ config DVB_STV0367
 config DVB_CXD2820R
 	tristate "Sony CXD2820R"
 	depends on DVB_CORE && I2C
+	select REGMAP_I2C
 	default m if !MEDIA_SUBDRV_AUTOSELECT
 	help
 	  Say Y when you want to support this frontend.
diff --git a/drivers/media/dvb-frontends/af9013.h b/drivers/media/dvb-frontends/af9013.h
index 1dcc936e1661..dcdd163ace85 100644
--- a/drivers/media/dvb-frontends/af9013.h
+++ b/drivers/media/dvb-frontends/af9013.h
@@ -25,7 +25,6 @@
 #ifndef AF9013_H
 #define AF9013_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 /* AF9013/5 GPIOs (mostly guessed)
diff --git a/drivers/media/dvb-frontends/af9033.h b/drivers/media/dvb-frontends/af9033.h
index 6ad22b69a636..5b83e4f96297 100644
--- a/drivers/media/dvb-frontends/af9033.h
+++ b/drivers/media/dvb-frontends/af9033.h
@@ -22,8 +22,6 @@
 #ifndef AF9033_H
 #define AF9033_H
 
-#include <linux/kconfig.h>
-
 /*
  * I2C address (TODO: are these in 8-bit format?)
  * 0x38, 0x3a, 0x3c, 0x3e
diff --git a/drivers/media/dvb-frontends/ascot2e.c b/drivers/media/dvb-frontends/ascot2e.c
index 8cc8c4597b6a..ad304eed656d 100644
--- a/drivers/media/dvb-frontends/ascot2e.c
+++ b/drivers/media/dvb-frontends/ascot2e.c
@@ -464,7 +464,7 @@ static int ascot2e_get_frequency(struct dvb_frontend *fe, u32 *frequency)
 	return 0;
 }
 
-static struct dvb_tuner_ops ascot2e_tuner_ops = {
+static const struct dvb_tuner_ops ascot2e_tuner_ops = {
 	.info = {
 		.name = "Sony ASCOT2E",
 		.frequency_min = 1000000,
diff --git a/drivers/media/dvb-frontends/ascot2e.h b/drivers/media/dvb-frontends/ascot2e.h
index 6da4ae6d6cc3..dc61bf7d1b09 100644
--- a/drivers/media/dvb-frontends/ascot2e.h
+++ b/drivers/media/dvb-frontends/ascot2e.h
@@ -22,7 +22,6 @@
 #ifndef __DVB_ASCOT2E_H__
 #define __DVB_ASCOT2E_H__
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include <linux/i2c.h>
 
diff --git a/drivers/media/dvb-frontends/atbm8830.h b/drivers/media/dvb-frontends/atbm8830.h
index 5446d13fdfe8..bb862387080f 100644
--- a/drivers/media/dvb-frontends/atbm8830.h
+++ b/drivers/media/dvb-frontends/atbm8830.h
@@ -22,7 +22,6 @@
 #ifndef __ATBM8830_H__
 #define __ATBM8830_H__
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include <linux/i2c.h>
 
diff --git a/drivers/media/dvb-frontends/au8522.h b/drivers/media/dvb-frontends/au8522.h
index 78bf3f73e58d..21c51a4c519a 100644
--- a/drivers/media/dvb-frontends/au8522.h
+++ b/drivers/media/dvb-frontends/au8522.h
@@ -22,7 +22,6 @@
 #ifndef __AU8522_H__
 #define __AU8522_H__
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 enum au8522_if_freq {
diff --git a/drivers/media/dvb-frontends/cx22702.h b/drivers/media/dvb-frontends/cx22702.h
index 68b69a7660d2..a1956a9ba406 100644
--- a/drivers/media/dvb-frontends/cx22702.h
+++ b/drivers/media/dvb-frontends/cx22702.h
@@ -28,7 +28,6 @@
 #ifndef CX22702_H
 #define CX22702_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct cx22702_config {
diff --git a/drivers/media/dvb-frontends/cx24113.h b/drivers/media/dvb-frontends/cx24113.h
index 962919b9b6e6..194c703611b4 100644
--- a/drivers/media/dvb-frontends/cx24113.h
+++ b/drivers/media/dvb-frontends/cx24113.h
@@ -22,8 +22,6 @@
 #ifndef CX24113_H
 #define CX24113_H
 
-#include <linux/kconfig.h>
-
 struct dvb_frontend;
 
 struct cx24113_config {
diff --git a/drivers/media/dvb-frontends/cx24116.h b/drivers/media/dvb-frontends/cx24116.h
index f6dbabc1d62b..9ff8df8d44b8 100644
--- a/drivers/media/dvb-frontends/cx24116.h
+++ b/drivers/media/dvb-frontends/cx24116.h
@@ -21,7 +21,6 @@
 #ifndef CX24116_H
 #define CX24116_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct cx24116_config {
diff --git a/drivers/media/dvb-frontends/cx24117.h b/drivers/media/dvb-frontends/cx24117.h
index 1648ab432168..445f13faf63a 100644
--- a/drivers/media/dvb-frontends/cx24117.h
+++ b/drivers/media/dvb-frontends/cx24117.h
@@ -22,7 +22,6 @@
 #ifndef CX24117_H
 #define CX24117_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct cx24117_config {
diff --git a/drivers/media/dvb-frontends/cx24120.h b/drivers/media/dvb-frontends/cx24120.h
index f0970423e16f..de4ca9aa0923 100644
--- a/drivers/media/dvb-frontends/cx24120.h
+++ b/drivers/media/dvb-frontends/cx24120.h
@@ -20,7 +20,6 @@
 #ifndef CX24120_H
 #define CX24120_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include <linux/firmware.h>
 
diff --git a/drivers/media/dvb-frontends/cx24123.h b/drivers/media/dvb-frontends/cx24123.h
index 975f3c926fe8..aac23444aa9a 100644
--- a/drivers/media/dvb-frontends/cx24123.h
+++ b/drivers/media/dvb-frontends/cx24123.h
@@ -21,7 +21,6 @@
 #ifndef CX24123_H
 #define CX24123_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct cx24123_config {
diff --git a/drivers/media/dvb-frontends/cxd2820r.h b/drivers/media/dvb-frontends/cxd2820r.h
index 56d42760263d..f3ff8f6eb3bb 100644
--- a/drivers/media/dvb-frontends/cxd2820r.h
+++ b/drivers/media/dvb-frontends/cxd2820r.h
@@ -22,7 +22,6 @@
 #ifndef CXD2820R_H
 #define CXD2820R_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 #define CXD2820R_GPIO_D (0 << 0) /* disable */
@@ -37,6 +36,32 @@
 #define CXD2820R_TS_PARALLEL      0x30
 #define CXD2820R_TS_PARALLEL_MSB  0x70
 
+/*
+ * I2C address: 0x6c, 0x6d
+ */
+
+/**
+ * struct cxd2820r_platform_data - Platform data for the cxd2820r driver
+ * @ts_mode: TS mode.
+ * @ts_clk_inv: TS clock inverted.
+ * @if_agc_polarity: IF AGC polarity.
+ * @spec_inv: Input spectrum inverted.
+ * @gpio_chip_base: GPIO.
+ * @get_dvb_frontend: Get DVB frontend.
+ */
+
+struct cxd2820r_platform_data {
+	u8 ts_mode;
+	bool ts_clk_inv;
+	bool if_agc_polarity;
+	bool spec_inv;
+	int **gpio_chip_base;
+
+	struct dvb_frontend* (*get_dvb_frontend)(struct i2c_client *);
+/* private: For legacy media attach wrapper. Do not set value. */
+	bool attach_in_use;
+};
+
 struct cxd2820r_config {
 	/* Demodulator I2C address.
 	 * Driver determines DVB-C slave I2C address automatically from master
diff --git a/drivers/media/dvb-frontends/cxd2820r_c.c b/drivers/media/dvb-frontends/cxd2820r_c.c
index a674a6312c38..d75b0776d5b5 100644
--- a/drivers/media/dvb-frontends/cxd2820r_c.c
+++ b/drivers/media/dvb-frontends/cxd2820r_c.c
@@ -24,12 +24,12 @@
 int cxd2820r_set_frontend_c(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
-	int ret, i;
+	int ret;
+	unsigned int utmp;
 	u8 buf[2];
-	u32 if_freq;
-	u16 if_ctl;
-	u64 num;
+	u32 if_frequency;
 	struct reg_val_mask tab[] = {
 		{ 0x00080, 0x01, 0xff },
 		{ 0x00081, 0x05, 0xff },
@@ -43,25 +43,24 @@ int cxd2820r_set_frontend_c(struct dvb_frontend *fe)
 		{ 0x10059, 0x50, 0xff },
 		{ 0x10087, 0x0c, 0x3c },
 		{ 0x1008b, 0x07, 0xff },
-		{ 0x1001f, priv->cfg.if_agc_polarity << 7, 0x80 },
-		{ 0x10070, priv->cfg.ts_mode, 0xff },
-		{ 0x10071, !priv->cfg.ts_clock_inv << 4, 0x10 },
+		{ 0x1001f, priv->if_agc_polarity << 7, 0x80 },
+		{ 0x10070, priv->ts_mode, 0xff },
+		{ 0x10071, !priv->ts_clk_inv << 4, 0x10 },
 	};
 
-	dev_dbg(&priv->i2c->dev, "%s: frequency=%d symbol_rate=%d\n", __func__,
-			c->frequency, c->symbol_rate);
+	dev_dbg(&client->dev,
+		"delivery_system=%d modulation=%d frequency=%u symbol_rate=%u inversion=%d\n",
+		c->delivery_system, c->modulation, c->frequency,
+		c->symbol_rate, c->inversion);
 
 	/* program tuner */
 	if (fe->ops.tuner_ops.set_params)
 		fe->ops.tuner_ops.set_params(fe);
 
 	if (priv->delivery_system !=  SYS_DVBC_ANNEX_A) {
-		for (i = 0; i < ARRAY_SIZE(tab); i++) {
-			ret = cxd2820r_wr_reg_mask(priv, tab[i].reg,
-				tab[i].val, tab[i].mask);
-			if (ret)
-				goto error;
-		}
+		ret = cxd2820r_wr_reg_val_mask_tab(priv, tab, ARRAY_SIZE(tab));
+		if (ret)
+			goto error;
 	}
 
 	priv->delivery_system = SYS_DVBC_ANNEX_A;
@@ -69,35 +68,33 @@ int cxd2820r_set_frontend_c(struct dvb_frontend *fe)
 
 	/* program IF frequency */
 	if (fe->ops.tuner_ops.get_if_frequency) {
-		ret = fe->ops.tuner_ops.get_if_frequency(fe, &if_freq);
+		ret = fe->ops.tuner_ops.get_if_frequency(fe, &if_frequency);
 		if (ret)
 			goto error;
-	} else
-		if_freq = 0;
-
-	dev_dbg(&priv->i2c->dev, "%s: if_freq=%d\n", __func__, if_freq);
-
-	num = if_freq / 1000; /* Hz => kHz */
-	num *= 0x4000;
-	if_ctl = 0x4000 - DIV_ROUND_CLOSEST_ULL(num, 41000);
-	buf[0] = (if_ctl >> 8) & 0x3f;
-	buf[1] = (if_ctl >> 0) & 0xff;
+		dev_dbg(&client->dev, "if_frequency=%u\n", if_frequency);
+	} else {
+		ret = -EINVAL;
+		goto error;
+	}
 
-	ret = cxd2820r_wr_regs(priv, 0x10042, buf, 2);
+	utmp = 0x4000 - DIV_ROUND_CLOSEST_ULL((u64)if_frequency * 0x4000, CXD2820R_CLK);
+	buf[0] = (utmp >> 8) & 0xff;
+	buf[1] = (utmp >> 0) & 0xff;
+	ret = regmap_bulk_write(priv->regmap[1], 0x0042, buf, 2);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_reg(priv, 0x000ff, 0x08);
+	ret = regmap_write(priv->regmap[0], 0x00ff, 0x08);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_reg(priv, 0x000fe, 0x01);
+	ret = regmap_write(priv->regmap[0], 0x00fe, 0x01);
 	if (ret)
 		goto error;
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
@@ -105,20 +102,24 @@ int cxd2820r_get_frontend_c(struct dvb_frontend *fe,
 			    struct dtv_frontend_properties *c)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 	int ret;
+	unsigned int utmp;
 	u8 buf[2];
 
-	ret = cxd2820r_rd_regs(priv, 0x1001a, buf, 2);
+	dev_dbg(&client->dev, "\n");
+
+	ret = regmap_bulk_read(priv->regmap[1], 0x001a, buf, 2);
 	if (ret)
 		goto error;
 
 	c->symbol_rate = 2500 * ((buf[0] & 0x0f) << 8 | buf[1]);
 
-	ret = cxd2820r_rd_reg(priv, 0x10019, &buf[0]);
+	ret = regmap_read(priv->regmap[1], 0x0019, &utmp);
 	if (ret)
 		goto error;
 
-	switch ((buf[0] >> 0) & 0x07) {
+	switch ((utmp >> 0) & 0x07) {
 	case 0:
 		c->modulation = QAM_16;
 		break;
@@ -136,7 +137,7 @@ int cxd2820r_get_frontend_c(struct dvb_frontend *fe,
 		break;
 	}
 
-	switch ((buf[0] >> 7) & 0x01) {
+	switch ((utmp >> 7) & 0x01) {
 	case 0:
 		c->inversion = INVERSION_OFF;
 		break;
@@ -147,167 +148,169 @@ int cxd2820r_get_frontend_c(struct dvb_frontend *fe,
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
-int cxd2820r_read_ber_c(struct dvb_frontend *fe, u32 *ber)
+int cxd2820r_read_status_c(struct dvb_frontend *fe, enum fe_status *status)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	int ret;
-	u8 buf[3], start_ber = 0;
-	*ber = 0;
+	unsigned int utmp, utmp1, utmp2;
+	u8 buf[3];
 
-	if (priv->ber_running) {
-		ret = cxd2820r_rd_regs(priv, 0x10076, buf, sizeof(buf));
-		if (ret)
-			goto error;
+	/* Lock detection */
+	ret = regmap_bulk_read(priv->regmap[1], 0x0088, &buf[0], 1);
+	if (ret)
+		goto error;
+	ret = regmap_bulk_read(priv->regmap[1], 0x0073, &buf[1], 1);
+	if (ret)
+		goto error;
 
-		if ((buf[2] >> 7) & 0x01 || (buf[2] >> 4) & 0x01) {
-			*ber = (buf[2] & 0x0f) << 16 | buf[1] << 8 | buf[0];
-			start_ber = 1;
-		}
+	utmp1 = (buf[0] >> 0) & 0x01;
+	utmp2 = (buf[1] >> 3) & 0x01;
+
+	if (utmp1 == 1 && utmp2 == 1) {
+		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER |
+			  FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK;
+	} else if (utmp1 == 1 || utmp2 == 1) {
+		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER |
+			  FE_HAS_VITERBI | FE_HAS_SYNC;
 	} else {
-		priv->ber_running = true;
-		start_ber = 1;
+		*status = 0;
 	}
 
-	if (start_ber) {
-		/* (re)start BER */
-		ret = cxd2820r_wr_reg(priv, 0x10079, 0x01);
-		if (ret)
-			goto error;
-	}
+	dev_dbg(&client->dev, "status=%02x raw=%*ph sync=%u ts=%u\n",
+		*status, 2, buf, utmp1, utmp2);
 
-	return ret;
-error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
-	return ret;
-}
+	/* Signal strength */
+	if (*status & FE_HAS_SIGNAL) {
+		unsigned int strength;
 
-int cxd2820r_read_signal_strength_c(struct dvb_frontend *fe,
-	u16 *strength)
-{
-	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
-	u8 buf[2];
-	u16 tmp;
+		ret = regmap_bulk_read(priv->regmap[1], 0x0049, buf, 2);
+		if (ret)
+			goto error;
 
-	ret = cxd2820r_rd_regs(priv, 0x10049, buf, sizeof(buf));
-	if (ret)
-		goto error;
+		utmp = buf[0] << 8 | buf[1] << 0;
+		utmp = 511 - sign_extend32(utmp, 9);
+		/* Scale value to 0x0000-0xffff */
+		strength = utmp << 6 | utmp >> 4;
 
-	tmp = (buf[0] & 0x03) << 8 | buf[1];
-	tmp = (~tmp & 0x03ff);
+		c->strength.len = 1;
+		c->strength.stat[0].scale = FE_SCALE_RELATIVE;
+		c->strength.stat[0].uvalue = strength;
+	} else {
+		c->strength.len = 1;
+		c->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
 
-	if (tmp == 512)
-		/* ~no signal */
-		tmp = 0;
-	else if (tmp > 350)
-		tmp = 350;
+	/* CNR */
+	if (*status & FE_HAS_VITERBI) {
+		unsigned int cnr, const_a, const_b;
 
-	/* scale value to 0x0000-0xffff */
-	*strength = tmp * 0xffff / (350-0);
+		ret = regmap_read(priv->regmap[1], 0x0019, &utmp);
+		if (ret)
+			goto error;
 
-	return ret;
-error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
-	return ret;
-}
+		if (((utmp >> 0) & 0x03) % 2) {
+			const_a = 8750;
+			const_b = 650;
+		} else {
+			const_a = 9500;
+			const_b = 760;
+		}
 
-int cxd2820r_read_snr_c(struct dvb_frontend *fe, u16 *snr)
-{
-	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
-	u8 tmp;
-	unsigned int A, B;
-	/* report SNR in dB * 10 */
+		ret = regmap_read(priv->regmap[1], 0x004d, &utmp);
+		if (ret)
+			goto error;
 
-	ret = cxd2820r_rd_reg(priv, 0x10019, &tmp);
-	if (ret)
-		goto error;
+		#define CXD2820R_LOG2_E_24 24204406 /* log2(e) << 24 */
+		if (utmp)
+			cnr = div_u64((u64)(intlog2(const_b) - intlog2(utmp))
+				      * const_a, CXD2820R_LOG2_E_24);
+		else
+			cnr = 0;
 
-	if (((tmp >> 0) & 0x03) % 2) {
-		A = 875;
-		B = 650;
+		c->cnr.len = 1;
+		c->cnr.stat[0].scale = FE_SCALE_DECIBEL;
+		c->cnr.stat[0].svalue = cnr;
 	} else {
-		A = 950;
-		B = 760;
+		c->cnr.len = 1;
+		c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 	}
 
-	ret = cxd2820r_rd_reg(priv, 0x1004d, &tmp);
-	if (ret)
-		goto error;
-
-	#define CXD2820R_LOG2_E_24 24204406 /* log2(e) << 24 */
-	if (tmp)
-		*snr = A * (intlog2(B / tmp) >> 5) / (CXD2820R_LOG2_E_24 >> 5)
-			/ 10;
-	else
-		*snr = 0;
-
-	return ret;
-error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
-	return ret;
-}
+	/* BER */
+	if (*status & FE_HAS_SYNC) {
+		unsigned int post_bit_error;
+		bool start_ber;
 
-int cxd2820r_read_ucblocks_c(struct dvb_frontend *fe, u32 *ucblocks)
-{
-	*ucblocks = 0;
-	/* no way to read ? */
-	return 0;
-}
+		if (priv->ber_running) {
+			ret = regmap_bulk_read(priv->regmap[1], 0x0076, buf, 3);
+			if (ret)
+				goto error;
 
-int cxd2820r_read_status_c(struct dvb_frontend *fe, enum fe_status *status)
-{
-	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
-	u8 buf[2];
-	*status = 0;
+			if ((buf[2] >> 7) & 0x01) {
+				post_bit_error = buf[2] << 16 | buf[1] << 8 |
+						 buf[0] << 0;
+				post_bit_error &= 0x0fffff;
+				start_ber = true;
+			} else {
+				post_bit_error = 0;
+				start_ber = false;
+			}
+		} else {
+			post_bit_error = 0;
+			start_ber = true;
+		}
 
-	ret = cxd2820r_rd_regs(priv, 0x10088, buf, sizeof(buf));
-	if (ret)
-		goto error;
+		if (start_ber) {
+			ret = regmap_write(priv->regmap[1], 0x0079, 0x01);
+			if (ret)
+				goto error;
+			priv->ber_running = true;
+		}
 
-	if (((buf[0] >> 0) & 0x01) == 1) {
-		*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER |
-			FE_HAS_VITERBI | FE_HAS_SYNC;
+		priv->post_bit_error += post_bit_error;
 
-		if (((buf[1] >> 3) & 0x01) == 1) {
-			*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER |
-				FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK;
-		}
+		c->post_bit_error.len = 1;
+		c->post_bit_error.stat[0].scale = FE_SCALE_COUNTER;
+		c->post_bit_error.stat[0].uvalue = priv->post_bit_error;
+	} else {
+		c->post_bit_error.len = 1;
+		c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 	}
 
-	dev_dbg(&priv->i2c->dev, "%s: lock=%02x %02x\n", __func__, buf[0],
-			buf[1]);
-
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
 int cxd2820r_init_c(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 	int ret;
 
-	ret = cxd2820r_wr_reg(priv, 0x00085, 0x07);
+	dev_dbg(&client->dev, "\n");
+
+	ret = regmap_write(priv->regmap[0], 0x0085, 0x07);
 	if (ret)
 		goto error;
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
 int cxd2820r_sleep_c(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret, i;
+	struct i2c_client *client = priv->client[0];
+	int ret;
 	struct reg_val_mask tab[] = {
 		{ 0x000ff, 0x1f, 0xff },
 		{ 0x00085, 0x00, 0xff },
@@ -316,20 +319,17 @@ int cxd2820r_sleep_c(struct dvb_frontend *fe)
 		{ 0x00080, 0x00, 0xff },
 	};
 
-	dev_dbg(&priv->i2c->dev, "%s\n", __func__);
+	dev_dbg(&client->dev, "\n");
 
 	priv->delivery_system = SYS_UNDEFINED;
 
-	for (i = 0; i < ARRAY_SIZE(tab); i++) {
-		ret = cxd2820r_wr_reg_mask(priv, tab[i].reg, tab[i].val,
-			tab[i].mask);
-		if (ret)
-			goto error;
-	}
+	ret = cxd2820r_wr_reg_val_mask_tab(priv, tab, ARRAY_SIZE(tab));
+	if (ret)
+		goto error;
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
diff --git a/drivers/media/dvb-frontends/cxd2820r_core.c b/drivers/media/dvb-frontends/cxd2820r_core.c
index 314d3b8c1080..95267c6edb3a 100644
--- a/drivers/media/dvb-frontends/cxd2820r_core.c
+++ b/drivers/media/dvb-frontends/cxd2820r_core.c
@@ -21,178 +21,50 @@
 
 #include "cxd2820r_priv.h"
 
-/* Max transfer size done by I2C transfer functions */
-#define MAX_XFER_SIZE  64
-
-/* write multiple registers */
-static int cxd2820r_wr_regs_i2c(struct cxd2820r_priv *priv, u8 i2c, u8 reg,
-	u8 *val, int len)
-{
-	int ret;
-	u8 buf[MAX_XFER_SIZE];
-	struct i2c_msg msg[1] = {
-		{
-			.addr = i2c,
-			.flags = 0,
-			.len = len + 1,
-			.buf = buf,
-		}
-	};
-
-	if (1 + len > sizeof(buf)) {
-		dev_warn(&priv->i2c->dev,
-			 "%s: i2c wr reg=%04x: len=%d is too big!\n",
-			 KBUILD_MODNAME, reg, len);
-		return -EINVAL;
-	}
-
-	buf[0] = reg;
-	memcpy(&buf[1], val, len);
-
-	ret = i2c_transfer(priv->i2c, msg, 1);
-	if (ret == 1) {
-		ret = 0;
-	} else {
-		dev_warn(&priv->i2c->dev, "%s: i2c wr failed=%d reg=%02x " \
-				"len=%d\n", KBUILD_MODNAME, ret, reg, len);
-		ret = -EREMOTEIO;
-	}
-	return ret;
-}
-
-/* read multiple registers */
-static int cxd2820r_rd_regs_i2c(struct cxd2820r_priv *priv, u8 i2c, u8 reg,
-	u8 *val, int len)
-{
-	int ret;
-	u8 buf[MAX_XFER_SIZE];
-	struct i2c_msg msg[2] = {
-		{
-			.addr = i2c,
-			.flags = 0,
-			.len = 1,
-			.buf = &reg,
-		}, {
-			.addr = i2c,
-			.flags = I2C_M_RD,
-			.len = len,
-			.buf = buf,
-		}
-	};
-
-	if (len > sizeof(buf)) {
-		dev_warn(&priv->i2c->dev,
-			 "%s: i2c wr reg=%04x: len=%d is too big!\n",
-			 KBUILD_MODNAME, reg, len);
-		return -EINVAL;
-	}
-
-	ret = i2c_transfer(priv->i2c, msg, 2);
-	if (ret == 2) {
-		memcpy(val, buf, len);
-		ret = 0;
-	} else {
-		dev_warn(&priv->i2c->dev, "%s: i2c rd failed=%d reg=%02x " \
-				"len=%d\n", KBUILD_MODNAME, ret, reg, len);
-		ret = -EREMOTEIO;
-	}
-
-	return ret;
-}
-
-/* write multiple registers */
-int cxd2820r_wr_regs(struct cxd2820r_priv *priv, u32 reginfo, u8 *val,
-	int len)
+/* Write register table */
+int cxd2820r_wr_reg_val_mask_tab(struct cxd2820r_priv *priv,
+				 const struct reg_val_mask *tab, int tab_len)
 {
+	struct i2c_client *client = priv->client[0];
 	int ret;
-	u8 i2c_addr;
-	u8 reg = (reginfo >> 0) & 0xff;
-	u8 bank = (reginfo >> 8) & 0xff;
-	u8 i2c = (reginfo >> 16) & 0x01;
-
-	/* select I2C */
-	if (i2c)
-		i2c_addr = priv->cfg.i2c_address | (1 << 1); /* DVB-C */
-	else
-		i2c_addr = priv->cfg.i2c_address; /* DVB-T/T2 */
+	unsigned int i, reg, mask, val;
+	struct regmap *regmap;
 
-	/* switch bank if needed */
-	if (bank != priv->bank[i2c]) {
-		ret = cxd2820r_wr_regs_i2c(priv, i2c_addr, 0x00, &bank, 1);
-		if (ret)
-			return ret;
-		priv->bank[i2c] = bank;
-	}
-	return cxd2820r_wr_regs_i2c(priv, i2c_addr, reg, val, len);
-}
-
-/* read multiple registers */
-int cxd2820r_rd_regs(struct cxd2820r_priv *priv, u32 reginfo, u8 *val,
-	int len)
-{
-	int ret;
-	u8 i2c_addr;
-	u8 reg = (reginfo >> 0) & 0xff;
-	u8 bank = (reginfo >> 8) & 0xff;
-	u8 i2c = (reginfo >> 16) & 0x01;
-
-	/* select I2C */
-	if (i2c)
-		i2c_addr = priv->cfg.i2c_address | (1 << 1); /* DVB-C */
-	else
-		i2c_addr = priv->cfg.i2c_address; /* DVB-T/T2 */
+	dev_dbg(&client->dev, "tab_len=%d\n", tab_len);
 
-	/* switch bank if needed */
-	if (bank != priv->bank[i2c]) {
-		ret = cxd2820r_wr_regs_i2c(priv, i2c_addr, 0x00, &bank, 1);
-		if (ret)
-			return ret;
-		priv->bank[i2c] = bank;
-	}
-	return cxd2820r_rd_regs_i2c(priv, i2c_addr, reg, val, len);
-}
-
-/* write single register */
-int cxd2820r_wr_reg(struct cxd2820r_priv *priv, u32 reg, u8 val)
-{
-	return cxd2820r_wr_regs(priv, reg, &val, 1);
-}
-
-/* read single register */
-int cxd2820r_rd_reg(struct cxd2820r_priv *priv, u32 reg, u8 *val)
-{
-	return cxd2820r_rd_regs(priv, reg, val, 1);
-}
+	for (i = 0; i < tab_len; i++) {
+		if ((tab[i].reg >> 16) & 0x1)
+			regmap = priv->regmap[1];
+		else
+			regmap = priv->regmap[0];
 
-/* write single register with mask */
-int cxd2820r_wr_reg_mask(struct cxd2820r_priv *priv, u32 reg, u8 val,
-	u8 mask)
-{
-	int ret;
-	u8 tmp;
+		reg = (tab[i].reg >> 0) & 0xffff;
+		val = tab[i].val;
+		mask = tab[i].mask;
 
-	/* no need for read if whole reg is written */
-	if (mask != 0xff) {
-		ret = cxd2820r_rd_reg(priv, reg, &tmp);
+		if (mask == 0xff)
+			ret = regmap_write(regmap, reg, val);
+		else
+			ret = regmap_write_bits(regmap, reg, mask, val);
 		if (ret)
-			return ret;
-
-		val &= mask;
-		tmp &= ~mask;
-		val |= tmp;
+			goto error;
 	}
 
-	return cxd2820r_wr_reg(priv, reg, val);
+	return 0;
+error:
+	dev_dbg(&client->dev, "failed=%d\n", ret);
+	return ret;
 }
 
 int cxd2820r_gpio(struct dvb_frontend *fe, u8 *gpio)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	int ret, i;
 	u8 tmp0, tmp1;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
 	/* update GPIOs only when needed */
 	if (!memcmp(gpio, priv->gpio, sizeof(priv->gpio)))
@@ -219,20 +91,18 @@ int cxd2820r_gpio(struct dvb_frontend *fe, u8 *gpio)
 		else
 			tmp1 |= (0 << (0 + i));
 
-		dev_dbg(&priv->i2c->dev, "%s: gpio i=%d %02x %02x\n", __func__,
-				i, tmp0, tmp1);
+		dev_dbg(&client->dev, "gpio i=%d %02x %02x\n", i, tmp0, tmp1);
 	}
 
-	dev_dbg(&priv->i2c->dev, "%s: wr gpio=%02x %02x\n", __func__, tmp0,
-			tmp1);
+	dev_dbg(&client->dev, "wr gpio=%02x %02x\n", tmp0, tmp1);
 
 	/* write bits [7:2] */
-	ret = cxd2820r_wr_reg_mask(priv, 0x00089, tmp0, 0xfc);
+	ret = regmap_update_bits(priv->regmap[0], 0x0089, 0xfc, tmp0);
 	if (ret)
 		goto error;
 
 	/* write bits [5:0] */
-	ret = cxd2820r_wr_reg_mask(priv, 0x0008e, tmp1, 0x3f);
+	ret = regmap_update_bits(priv->regmap[0], 0x008e, 0x3f, tmp1);
 	if (ret)
 		goto error;
 
@@ -240,18 +110,18 @@ int cxd2820r_gpio(struct dvb_frontend *fe, u8 *gpio)
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
 static int cxd2820r_set_frontend(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	int ret;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
 	switch (c->delivery_system) {
 	case SYS_DVBT:
@@ -279,8 +149,7 @@ static int cxd2820r_set_frontend(struct dvb_frontend *fe)
 			goto err;
 		break;
 	default:
-		dev_dbg(&priv->i2c->dev, "%s: error state=%d\n", __func__,
-				fe->dtv_property_cache.delivery_system);
+		dev_dbg(&client->dev, "invalid delivery_system\n");
 		ret = -EINVAL;
 		break;
 	}
@@ -291,12 +160,13 @@ err:
 static int cxd2820r_read_status(struct dvb_frontend *fe, enum fe_status *status)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	int ret;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
-	switch (fe->dtv_property_cache.delivery_system) {
+	switch (c->delivery_system) {
 	case SYS_DVBT:
 		ret = cxd2820r_read_status_t(fe, status);
 		break;
@@ -317,15 +187,16 @@ static int cxd2820r_get_frontend(struct dvb_frontend *fe,
 				 struct dtv_frontend_properties *p)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	int ret;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
 	if (priv->delivery_system == SYS_UNDEFINED)
 		return 0;
 
-	switch (fe->dtv_property_cache.delivery_system) {
+	switch (c->delivery_system) {
 	case SYS_DVBT:
 		ret = cxd2820r_get_frontend_t(fe, p);
 		break;
@@ -345,101 +216,60 @@ static int cxd2820r_get_frontend(struct dvb_frontend *fe,
 static int cxd2820r_read_ber(struct dvb_frontend *fe, u32 *ber)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
-	switch (fe->dtv_property_cache.delivery_system) {
-	case SYS_DVBT:
-		ret = cxd2820r_read_ber_t(fe, ber);
-		break;
-	case SYS_DVBT2:
-		ret = cxd2820r_read_ber_t2(fe, ber);
-		break;
-	case SYS_DVBC_ANNEX_A:
-		ret = cxd2820r_read_ber_c(fe, ber);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-	return ret;
+	*ber = (priv->post_bit_error - priv->post_bit_error_prev_dvbv3);
+	priv->post_bit_error_prev_dvbv3 = priv->post_bit_error;
+
+	return 0;
 }
 
 static int cxd2820r_read_signal_strength(struct dvb_frontend *fe, u16 *strength)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
-	switch (fe->dtv_property_cache.delivery_system) {
-	case SYS_DVBT:
-		ret = cxd2820r_read_signal_strength_t(fe, strength);
-		break;
-	case SYS_DVBT2:
-		ret = cxd2820r_read_signal_strength_t2(fe, strength);
-		break;
-	case SYS_DVBC_ANNEX_A:
-		ret = cxd2820r_read_signal_strength_c(fe, strength);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-	return ret;
+	if (c->strength.stat[0].scale == FE_SCALE_RELATIVE)
+		*strength = c->strength.stat[0].uvalue;
+	else
+		*strength = 0;
+
+	return 0;
 }
 
 static int cxd2820r_read_snr(struct dvb_frontend *fe, u16 *snr)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
-	switch (fe->dtv_property_cache.delivery_system) {
-	case SYS_DVBT:
-		ret = cxd2820r_read_snr_t(fe, snr);
-		break;
-	case SYS_DVBT2:
-		ret = cxd2820r_read_snr_t2(fe, snr);
-		break;
-	case SYS_DVBC_ANNEX_A:
-		ret = cxd2820r_read_snr_c(fe, snr);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-	return ret;
+	if (c->cnr.stat[0].scale == FE_SCALE_DECIBEL)
+		*snr = div_s64(c->cnr.stat[0].svalue, 100);
+	else
+		*snr = 0;
+
+	return 0;
 }
 
 static int cxd2820r_read_ucblocks(struct dvb_frontend *fe, u32 *ucblocks)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
-	switch (fe->dtv_property_cache.delivery_system) {
-	case SYS_DVBT:
-		ret = cxd2820r_read_ucblocks_t(fe, ucblocks);
-		break;
-	case SYS_DVBT2:
-		ret = cxd2820r_read_ucblocks_t2(fe, ucblocks);
-		break;
-	case SYS_DVBC_ANNEX_A:
-		ret = cxd2820r_read_ucblocks_c(fe, ucblocks);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-	return ret;
+	*ucblocks = 0;
+
+	return 0;
 }
 
 static int cxd2820r_init(struct dvb_frontend *fe)
@@ -450,12 +280,13 @@ static int cxd2820r_init(struct dvb_frontend *fe)
 static int cxd2820r_sleep(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	int ret;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
-	switch (fe->dtv_property_cache.delivery_system) {
+	switch (c->delivery_system) {
 	case SYS_DVBT:
 		ret = cxd2820r_sleep_t(fe);
 		break;
@@ -476,12 +307,13 @@ static int cxd2820r_get_tune_settings(struct dvb_frontend *fe,
 				      struct dvb_frontend_tune_settings *s)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	int ret;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
-	switch (fe->dtv_property_cache.delivery_system) {
+	switch (c->delivery_system) {
 	case SYS_DVBT:
 		ret = cxd2820r_get_tune_settings_t(fe, s);
 		break;
@@ -501,12 +333,12 @@ static int cxd2820r_get_tune_settings(struct dvb_frontend *fe,
 static enum dvbfe_search cxd2820r_search(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	int ret, i;
 	enum fe_status status = 0;
 
-	dev_dbg(&priv->i2c->dev, "%s: delsys=%d\n", __func__,
-			fe->dtv_property_cache.delivery_system);
+	dev_dbg(&client->dev, "delivery_system=%d\n", c->delivery_system);
 
 	/* switch between DVB-T and DVB-T2 when tune fails */
 	if (priv->last_tune_failed) {
@@ -530,7 +362,6 @@ static enum dvbfe_search cxd2820r_search(struct dvb_frontend *fe)
 	if (ret)
 		goto error;
 
-
 	/* frontend lock wait loop count */
 	switch (priv->delivery_system) {
 	case SYS_DVBT:
@@ -548,7 +379,7 @@ static enum dvbfe_search cxd2820r_search(struct dvb_frontend *fe)
 
 	/* wait frontend lock */
 	for (; i > 0; i--) {
-		dev_dbg(&priv->i2c->dev, "%s: loop=%d\n", __func__, i);
+		dev_dbg(&client->dev, "loop=%d\n", i);
 		msleep(50);
 		ret = cxd2820r_read_status(fe, &status);
 		if (ret)
@@ -568,7 +399,7 @@ static enum dvbfe_search cxd2820r_search(struct dvb_frontend *fe)
 	}
 
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return DVBFE_ALGO_SEARCH_ERROR;
 }
 
@@ -580,27 +411,23 @@ static int cxd2820r_get_frontend_algo(struct dvb_frontend *fe)
 static void cxd2820r_release(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 
-	dev_dbg(&priv->i2c->dev, "%s\n", __func__);
+	dev_dbg(&client->dev, "\n");
 
-#ifdef CONFIG_GPIOLIB
-	/* remove GPIOs */
-	if (priv->gpio_chip.label)
-		gpiochip_remove(&priv->gpio_chip);
+	i2c_unregister_device(client);
 
-#endif
-	kfree(priv);
 	return;
 }
 
 static int cxd2820r_i2c_gate_ctrl(struct dvb_frontend *fe, int enable)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 
-	dev_dbg(&priv->i2c->dev, "%s: %d\n", __func__, enable);
+	dev_dbg_ratelimited(&client->dev, "enable=%d\n", enable);
 
-	/* Bit 0 of reg 0xdb in bank 0x00 controls I2C repeater */
-	return cxd2820r_wr_reg_mask(priv, 0xdb, enable ? 1 : 0, 0x1);
+	return regmap_update_bits(priv->regmap[0], 0x00db, 0x01, enable ? 1 : 0);
 }
 
 #ifdef CONFIG_GPIOLIB
@@ -608,9 +435,10 @@ static int cxd2820r_gpio_direction_output(struct gpio_chip *chip, unsigned nr,
 		int val)
 {
 	struct cxd2820r_priv *priv = gpiochip_get_data(chip);
+	struct i2c_client *client = priv->client[0];
 	u8 gpio[GPIO_COUNT];
 
-	dev_dbg(&priv->i2c->dev, "%s: nr=%d val=%d\n", __func__, nr, val);
+	dev_dbg(&client->dev, "nr=%u val=%d\n", nr, val);
 
 	memcpy(gpio, priv->gpio, sizeof(gpio));
 	gpio[nr] = CXD2820R_GPIO_E | CXD2820R_GPIO_O | (val << 2);
@@ -621,9 +449,10 @@ static int cxd2820r_gpio_direction_output(struct gpio_chip *chip, unsigned nr,
 static void cxd2820r_gpio_set(struct gpio_chip *chip, unsigned nr, int val)
 {
 	struct cxd2820r_priv *priv = gpiochip_get_data(chip);
+	struct i2c_client *client = priv->client[0];
 	u8 gpio[GPIO_COUNT];
 
-	dev_dbg(&priv->i2c->dev, "%s: nr=%d val=%d\n", __func__, nr, val);
+	dev_dbg(&client->dev, "nr=%u val=%d\n", nr, val);
 
 	memcpy(gpio, priv->gpio, sizeof(gpio));
 	gpio[nr] = CXD2820R_GPIO_E | CXD2820R_GPIO_O | (val << 2);
@@ -636,8 +465,9 @@ static void cxd2820r_gpio_set(struct gpio_chip *chip, unsigned nr, int val)
 static int cxd2820r_gpio_get(struct gpio_chip *chip, unsigned nr)
 {
 	struct cxd2820r_priv *priv = gpiochip_get_data(chip);
+	struct i2c_client *client = priv->client[0];
 
-	dev_dbg(&priv->i2c->dev, "%s: nr=%d\n", __func__, nr);
+	dev_dbg(&client->dev, "nr=%u\n", nr);
 
 	return (priv->gpio[nr] >> 2) & 0x01;
 }
@@ -689,52 +519,163 @@ static const struct dvb_frontend_ops cxd2820r_ops = {
 	.read_signal_strength	= cxd2820r_read_signal_strength,
 };
 
-struct dvb_frontend *cxd2820r_attach(const struct cxd2820r_config *cfg,
-		struct i2c_adapter *i2c, int *gpio_chip_base
-)
+/*
+ * XXX: That is wrapper to cxd2820r_probe() via driver core in order to provide
+ * proper I2C client for legacy media attach binding.
+ * New users must use I2C client binding directly!
+ */
+struct dvb_frontend *cxd2820r_attach(const struct cxd2820r_config *config,
+				     struct i2c_adapter *adapter,
+				     int *gpio_chip_base)
+{
+	struct i2c_client *client;
+	struct i2c_board_info board_info;
+	struct cxd2820r_platform_data pdata;
+
+	pdata.ts_mode = config->ts_mode;
+	pdata.ts_clk_inv = config->ts_clock_inv;
+	pdata.if_agc_polarity = config->if_agc_polarity;
+	pdata.spec_inv = config->spec_inv;
+	pdata.gpio_chip_base = &gpio_chip_base;
+	pdata.attach_in_use = true;
+
+	memset(&board_info, 0, sizeof(board_info));
+	strlcpy(board_info.type, "cxd2820r", I2C_NAME_SIZE);
+	board_info.addr = config->i2c_address;
+	board_info.platform_data = &pdata;
+	client = i2c_new_device(adapter, &board_info);
+	if (!client || !client->dev.driver)
+		return NULL;
+
+	return pdata.get_dvb_frontend(client);
+}
+EXPORT_SYMBOL(cxd2820r_attach);
+
+static struct dvb_frontend *cxd2820r_get_dvb_frontend(struct i2c_client *client)
+{
+	struct cxd2820r_priv *priv = i2c_get_clientdata(client);
+
+	dev_dbg(&client->dev, "\n");
+
+	return &priv->fe;
+}
+
+static int cxd2820r_probe(struct i2c_client *client,
+			  const struct i2c_device_id *id)
 {
+	struct cxd2820r_platform_data *pdata = client->dev.platform_data;
 	struct cxd2820r_priv *priv;
-	int ret;
-	u8 tmp;
+	int ret, *gpio_chip_base;
+	unsigned int utmp;
+	static const struct regmap_range_cfg regmap_range_cfg0[] = {
+		{
+			.range_min        = 0x0000,
+			.range_max        = 0x3fff,
+			.selector_reg     = 0x00,
+			.selector_mask    = 0xff,
+			.selector_shift   = 0,
+			.window_start     = 0x00,
+			.window_len       = 0x100,
+		},
+	};
+	static const struct regmap_range_cfg regmap_range_cfg1[] = {
+		{
+			.range_min        = 0x0000,
+			.range_max        = 0x01ff,
+			.selector_reg     = 0x00,
+			.selector_mask    = 0xff,
+			.selector_shift   = 0,
+			.window_start     = 0x00,
+			.window_len       = 0x100,
+		},
+	};
+	static const struct regmap_config regmap_config0 = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = 0x3fff,
+		.ranges = regmap_range_cfg0,
+		.num_ranges = ARRAY_SIZE(regmap_range_cfg0),
+		.cache_type = REGCACHE_NONE,
+	};
+	static const struct regmap_config regmap_config1 = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = 0x01ff,
+		.ranges = regmap_range_cfg1,
+		.num_ranges = ARRAY_SIZE(regmap_range_cfg1),
+		.cache_type = REGCACHE_NONE,
+	};
+
+	dev_dbg(&client->dev, "\n");
 
-	priv = kzalloc(sizeof(struct cxd2820r_priv), GFP_KERNEL);
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv) {
 		ret = -ENOMEM;
-		dev_err(&i2c->dev, "%s: kzalloc() failed\n",
-				KBUILD_MODNAME);
-		goto error;
+		goto err;
 	}
 
-	priv->i2c = i2c;
-	memcpy(&priv->cfg, cfg, sizeof(struct cxd2820r_config));
-	memcpy(&priv->fe.ops, &cxd2820r_ops, sizeof(struct dvb_frontend_ops));
-	priv->fe.demodulator_priv = priv;
+	priv->client[0] = client;
+	priv->i2c = client->adapter;
+	priv->ts_mode = pdata->ts_mode;
+	priv->ts_clk_inv = pdata->ts_clk_inv;
+	priv->if_agc_polarity = pdata->if_agc_polarity;
+	priv->spec_inv = pdata->spec_inv;
+	gpio_chip_base = *pdata->gpio_chip_base;
+	priv->regmap[0] = regmap_init_i2c(priv->client[0], &regmap_config0);
+	if (IS_ERR(priv->regmap[0])) {
+		ret = PTR_ERR(priv->regmap[0]);
+		goto err_kfree;
+	}
 
-	priv->bank[0] = priv->bank[1] = 0xff;
-	ret = cxd2820r_rd_reg(priv, 0x000fd, &tmp);
-	dev_dbg(&priv->i2c->dev, "%s: chip id=%02x\n", __func__, tmp);
-	if (ret || tmp != 0xe1)
-		goto error;
+	/* Check demod answers with correct chip id */
+	ret = regmap_read(priv->regmap[0], 0x00fd, &utmp);
+	if (ret)
+		goto err_regmap_0_regmap_exit;
+
+	dev_dbg(&client->dev, "chip_id=%02x\n", utmp);
+
+	if (utmp != 0xe1) {
+		ret = -ENODEV;
+		goto err_regmap_0_regmap_exit;
+	}
+
+	/*
+	 * Chip has two I2C addresses for different register banks. We register
+	 * one dummy I2C client in in order to get own I2C client for each
+	 * register bank.
+	 */
+	priv->client[1] = i2c_new_dummy(client->adapter, client->addr | (1 << 1));
+	if (!priv->client[1]) {
+		ret = -ENODEV;
+		dev_err(&client->dev, "I2C registration failed\n");
+		if (ret)
+			goto err_regmap_0_regmap_exit;
+	}
+
+	priv->regmap[1] = regmap_init_i2c(priv->client[1], &regmap_config1);
+	if (IS_ERR(priv->regmap[1])) {
+		ret = PTR_ERR(priv->regmap[1]);
+		goto err_client_1_i2c_unregister_device;
+	}
 
 	if (gpio_chip_base) {
 #ifdef CONFIG_GPIOLIB
-		/* add GPIOs */
+		/* Add GPIOs */
 		priv->gpio_chip.label = KBUILD_MODNAME;
-		priv->gpio_chip.parent = &priv->i2c->dev;
+		priv->gpio_chip.parent = &client->dev;
 		priv->gpio_chip.owner = THIS_MODULE;
-		priv->gpio_chip.direction_output =
-				cxd2820r_gpio_direction_output;
+		priv->gpio_chip.direction_output = cxd2820r_gpio_direction_output;
 		priv->gpio_chip.set = cxd2820r_gpio_set;
 		priv->gpio_chip.get = cxd2820r_gpio_get;
-		priv->gpio_chip.base = -1; /* dynamic allocation */
+		priv->gpio_chip.base = -1; /* Dynamic allocation */
 		priv->gpio_chip.ngpio = GPIO_COUNT;
 		priv->gpio_chip.can_sleep = 1;
 		ret = gpiochip_add_data(&priv->gpio_chip, priv);
 		if (ret)
-			goto error;
+			goto err_regmap_1_regmap_exit;
 
-		dev_dbg(&priv->i2c->dev, "%s: gpio_chip.base=%d\n", __func__,
-				priv->gpio_chip.base);
+		dev_dbg(&client->dev, "gpio_chip.base=%d\n",
+			priv->gpio_chip.base);
 
 		*gpio_chip_base = priv->gpio_chip.base;
 #else
@@ -748,17 +689,73 @@ struct dvb_frontend *cxd2820r_attach(const struct cxd2820r_config *cfg,
 		gpio[2] = 0;
 		ret = cxd2820r_gpio(&priv->fe, gpio);
 		if (ret)
-			goto error;
+			goto err_regmap_1_regmap_exit;
 #endif
 	}
 
-	return &priv->fe;
-error:
-	dev_dbg(&i2c->dev, "%s: failed=%d\n", __func__, ret);
+	/* Create dvb frontend */
+	memcpy(&priv->fe.ops, &cxd2820r_ops, sizeof(priv->fe.ops));
+	if (!pdata->attach_in_use)
+		priv->fe.ops.release = NULL;
+	priv->fe.demodulator_priv = priv;
+	i2c_set_clientdata(client, priv);
+
+	/* Setup callbacks */
+	pdata->get_dvb_frontend = cxd2820r_get_dvb_frontend;
+
+	dev_info(&client->dev, "Sony CXD2820R successfully identified\n");
+
+	return 0;
+err_regmap_1_regmap_exit:
+	regmap_exit(priv->regmap[1]);
+err_client_1_i2c_unregister_device:
+	i2c_unregister_device(priv->client[1]);
+err_regmap_0_regmap_exit:
+	regmap_exit(priv->regmap[0]);
+err_kfree:
 	kfree(priv);
-	return NULL;
+err:
+	dev_dbg(&client->dev, "failed=%d\n", ret);
+	return ret;
 }
-EXPORT_SYMBOL(cxd2820r_attach);
+
+static int cxd2820r_remove(struct i2c_client *client)
+{
+	struct cxd2820r_priv *priv = i2c_get_clientdata(client);
+
+	dev_dbg(&client->dev, "\n");
+
+#ifdef CONFIG_GPIOLIB
+	if (priv->gpio_chip.label)
+		gpiochip_remove(&priv->gpio_chip);
+#endif
+	regmap_exit(priv->regmap[1]);
+	i2c_unregister_device(priv->client[1]);
+
+	regmap_exit(priv->regmap[0]);
+
+	kfree(priv);
+
+	return 0;
+}
+
+static const struct i2c_device_id cxd2820r_id_table[] = {
+	{"cxd2820r", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, cxd2820r_id_table);
+
+static struct i2c_driver cxd2820r_driver = {
+	.driver = {
+		.name                = "cxd2820r",
+		.suppress_bind_attrs = true,
+	},
+	.probe    = cxd2820r_probe,
+	.remove   = cxd2820r_remove,
+	.id_table = cxd2820r_id_table,
+};
+
+module_i2c_driver(cxd2820r_driver);
 
 MODULE_AUTHOR("Antti Palosaari <crope@iki.fi>");
 MODULE_DESCRIPTION("Sony CXD2820R demodulator driver");
diff --git a/drivers/media/dvb-frontends/cxd2820r_priv.h b/drivers/media/dvb-frontends/cxd2820r_priv.h
index e31c48e53097..0d096206ac66 100644
--- a/drivers/media/dvb-frontends/cxd2820r_priv.h
+++ b/drivers/media/dvb-frontends/cxd2820r_priv.h
@@ -27,6 +27,8 @@
 #include "dvb_math.h"
 #include "cxd2820r.h"
 #include <linux/gpio.h>
+#include <linux/math64.h>
+#include <linux/regmap.h>
 
 struct reg_val_mask {
 	u32 reg;
@@ -34,14 +36,23 @@ struct reg_val_mask {
 	u8  mask;
 };
 
+#define CXD2820R_CLK 41000000
+
 struct cxd2820r_priv {
+	struct i2c_client *client[2];
+	struct regmap *regmap[2];
 	struct i2c_adapter *i2c;
 	struct dvb_frontend fe;
-	struct cxd2820r_config cfg;
+	u8 ts_mode;
+	bool ts_clk_inv;
+	bool if_agc_polarity;
+	bool spec_inv;
+
+	u64 post_bit_error_prev_dvbv3;
+	u64 post_bit_error;
 
 	bool ber_running;
 
-	u8 bank[2];
 #define GPIO_COUNT 3
 	u8 gpio[GPIO_COUNT];
 #ifdef CONFIG_GPIOLIB
@@ -58,6 +69,9 @@ extern int cxd2820r_debug;
 
 int cxd2820r_gpio(struct dvb_frontend *fe, u8 *gpio);
 
+int cxd2820r_wr_reg_val_mask_tab(struct cxd2820r_priv *priv,
+				 const struct reg_val_mask *tab, int tab_len);
+
 int cxd2820r_wr_reg_mask(struct cxd2820r_priv *priv, u32 reg, u8 val,
 	u8 mask);
 
@@ -83,14 +97,6 @@ int cxd2820r_set_frontend_c(struct dvb_frontend *fe);
 
 int cxd2820r_read_status_c(struct dvb_frontend *fe, enum fe_status *status);
 
-int cxd2820r_read_ber_c(struct dvb_frontend *fe, u32 *ber);
-
-int cxd2820r_read_signal_strength_c(struct dvb_frontend *fe, u16 *strength);
-
-int cxd2820r_read_snr_c(struct dvb_frontend *fe, u16 *snr);
-
-int cxd2820r_read_ucblocks_c(struct dvb_frontend *fe, u32 *ucblocks);
-
 int cxd2820r_init_c(struct dvb_frontend *fe);
 
 int cxd2820r_sleep_c(struct dvb_frontend *fe);
@@ -107,14 +113,6 @@ int cxd2820r_set_frontend_t(struct dvb_frontend *fe);
 
 int cxd2820r_read_status_t(struct dvb_frontend *fe, enum fe_status *status);
 
-int cxd2820r_read_ber_t(struct dvb_frontend *fe, u32 *ber);
-
-int cxd2820r_read_signal_strength_t(struct dvb_frontend *fe, u16 *strength);
-
-int cxd2820r_read_snr_t(struct dvb_frontend *fe, u16 *snr);
-
-int cxd2820r_read_ucblocks_t(struct dvb_frontend *fe, u32 *ucblocks);
-
 int cxd2820r_init_t(struct dvb_frontend *fe);
 
 int cxd2820r_sleep_t(struct dvb_frontend *fe);
@@ -131,14 +129,6 @@ int cxd2820r_set_frontend_t2(struct dvb_frontend *fe);
 
 int cxd2820r_read_status_t2(struct dvb_frontend *fe, enum fe_status *status);
 
-int cxd2820r_read_ber_t2(struct dvb_frontend *fe, u32 *ber);
-
-int cxd2820r_read_signal_strength_t2(struct dvb_frontend *fe, u16 *strength);
-
-int cxd2820r_read_snr_t2(struct dvb_frontend *fe, u16 *snr);
-
-int cxd2820r_read_ucblocks_t2(struct dvb_frontend *fe, u32 *ucblocks);
-
 int cxd2820r_init_t2(struct dvb_frontend *fe);
 
 int cxd2820r_sleep_t2(struct dvb_frontend *fe);
diff --git a/drivers/media/dvb-frontends/cxd2820r_t.c b/drivers/media/dvb-frontends/cxd2820r_t.c
index 75ce7d8ded00..c2e7caf9b010 100644
--- a/drivers/media/dvb-frontends/cxd2820r_t.c
+++ b/drivers/media/dvb-frontends/cxd2820r_t.c
@@ -24,10 +24,11 @@
 int cxd2820r_set_frontend_t(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
-	int ret, i, bw_i;
-	u32 if_freq, if_ctl;
-	u64 num;
+	int ret, bw_i;
+	unsigned int utmp;
+	u32 if_frequency;
 	u8 buf[3], bw_param;
 	u8 bw_params1[][5] = {
 		{ 0x17, 0xea, 0xaa, 0xaa, 0xaa }, /* 6 MHz */
@@ -45,9 +46,9 @@ int cxd2820r_set_frontend_t(struct dvb_frontend *fe)
 		{ 0x00085, 0x07, 0xff },
 		{ 0x00088, 0x01, 0xff },
 
-		{ 0x00070, priv->cfg.ts_mode, 0xff },
-		{ 0x00071, !priv->cfg.ts_clock_inv << 4, 0x10 },
-		{ 0x000cb, priv->cfg.if_agc_polarity << 6, 0x40 },
+		{ 0x00070, priv->ts_mode, 0xff },
+		{ 0x00071, !priv->ts_clk_inv << 4, 0x10 },
+		{ 0x000cb, priv->if_agc_polarity << 6, 0x40 },
 		{ 0x000a5, 0x00, 0x01 },
 		{ 0x00082, 0x20, 0x60 },
 		{ 0x000c2, 0xc3, 0xff },
@@ -55,8 +56,10 @@ int cxd2820r_set_frontend_t(struct dvb_frontend *fe)
 		{ 0x00427, 0x41, 0xff },
 	};
 
-	dev_dbg(&priv->i2c->dev, "%s: frequency=%d bandwidth_hz=%d\n", __func__,
-			c->frequency, c->bandwidth_hz);
+	dev_dbg(&client->dev,
+		"delivery_system=%d modulation=%d frequency=%u bandwidth_hz=%u inversion=%d\n",
+		c->delivery_system, c->modulation, c->frequency,
+		c->bandwidth_hz, c->inversion);
 
 	switch (c->bandwidth_hz) {
 	case 6000000:
@@ -80,12 +83,9 @@ int cxd2820r_set_frontend_t(struct dvb_frontend *fe)
 		fe->ops.tuner_ops.set_params(fe);
 
 	if (priv->delivery_system != SYS_DVBT) {
-		for (i = 0; i < ARRAY_SIZE(tab); i++) {
-			ret = cxd2820r_wr_reg_mask(priv, tab[i].reg,
-				tab[i].val, tab[i].mask);
-			if (ret)
-				goto error;
-		}
+		ret = cxd2820r_wr_reg_val_mask_tab(priv, tab, ARRAY_SIZE(tab));
+		if (ret)
+			goto error;
 	}
 
 	priv->delivery_system = SYS_DVBT;
@@ -93,48 +93,46 @@ int cxd2820r_set_frontend_t(struct dvb_frontend *fe)
 
 	/* program IF frequency */
 	if (fe->ops.tuner_ops.get_if_frequency) {
-		ret = fe->ops.tuner_ops.get_if_frequency(fe, &if_freq);
+		ret = fe->ops.tuner_ops.get_if_frequency(fe, &if_frequency);
 		if (ret)
 			goto error;
-	} else
-		if_freq = 0;
-
-	dev_dbg(&priv->i2c->dev, "%s: if_freq=%d\n", __func__, if_freq);
-
-	num = if_freq / 1000; /* Hz => kHz */
-	num *= 0x1000000;
-	if_ctl = DIV_ROUND_CLOSEST_ULL(num, 41000);
-	buf[0] = ((if_ctl >> 16) & 0xff);
-	buf[1] = ((if_ctl >>  8) & 0xff);
-	buf[2] = ((if_ctl >>  0) & 0xff);
+		dev_dbg(&client->dev, "if_frequency=%u\n", if_frequency);
+	} else {
+		ret = -EINVAL;
+		goto error;
+	}
 
-	ret = cxd2820r_wr_regs(priv, 0x000b6, buf, 3);
+	utmp = DIV_ROUND_CLOSEST_ULL((u64)if_frequency * 0x1000000, CXD2820R_CLK);
+	buf[0] = (utmp >> 16) & 0xff;
+	buf[1] = (utmp >>  8) & 0xff;
+	buf[2] = (utmp >>  0) & 0xff;
+	ret = regmap_bulk_write(priv->regmap[0], 0x00b6, buf, 3);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_regs(priv, 0x0009f, bw_params1[bw_i], 5);
+	ret = regmap_bulk_write(priv->regmap[0], 0x009f, bw_params1[bw_i], 5);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_reg_mask(priv, 0x000d7, bw_param << 6, 0xc0);
+	ret = regmap_update_bits(priv->regmap[0], 0x00d7, 0xc0, bw_param << 6);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_regs(priv, 0x000d9, bw_params2[bw_i], 2);
+	ret = regmap_bulk_write(priv->regmap[0], 0x00d9, bw_params2[bw_i], 2);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_reg(priv, 0x000ff, 0x08);
+	ret = regmap_write(priv->regmap[0], 0x00ff, 0x08);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_reg(priv, 0x000fe, 0x01);
+	ret = regmap_write(priv->regmap[0], 0x00fe, 0x01);
 	if (ret)
 		goto error;
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
@@ -142,10 +140,14 @@ int cxd2820r_get_frontend_t(struct dvb_frontend *fe,
 			    struct dtv_frontend_properties *c)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 	int ret;
+	unsigned int utmp;
 	u8 buf[2];
 
-	ret = cxd2820r_rd_regs(priv, 0x0002f, buf, sizeof(buf));
+	dev_dbg(&client->dev, "\n");
+
+	ret = regmap_bulk_read(priv->regmap[0], 0x002f, buf, sizeof(buf));
 	if (ret)
 		goto error;
 
@@ -236,11 +238,11 @@ int cxd2820r_get_frontend_t(struct dvb_frontend *fe,
 		break;
 	}
 
-	ret = cxd2820r_rd_reg(priv, 0x007c6, &buf[0]);
+	ret = regmap_read(priv->regmap[0], 0x07c6, &utmp);
 	if (ret)
 		goto error;
 
-	switch ((buf[0] >> 0) & 0x01) {
+	switch ((utmp >> 0) & 0x01) {
 	case 0:
 		c->inversion = INVERSION_OFF;
 		break;
@@ -251,169 +253,158 @@ int cxd2820r_get_frontend_t(struct dvb_frontend *fe,
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
-	return ret;
-}
-
-int cxd2820r_read_ber_t(struct dvb_frontend *fe, u32 *ber)
-{
-	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
-	u8 buf[3], start_ber = 0;
-	*ber = 0;
-
-	if (priv->ber_running) {
-		ret = cxd2820r_rd_regs(priv, 0x00076, buf, sizeof(buf));
-		if (ret)
-			goto error;
-
-		if ((buf[2] >> 7) & 0x01 || (buf[2] >> 4) & 0x01) {
-			*ber = (buf[2] & 0x0f) << 16 | buf[1] << 8 | buf[0];
-			start_ber = 1;
-		}
-	} else {
-		priv->ber_running = true;
-		start_ber = 1;
-	}
-
-	if (start_ber) {
-		/* (re)start BER */
-		ret = cxd2820r_wr_reg(priv, 0x00079, 0x01);
-		if (ret)
-			goto error;
-	}
-
-	return ret;
-error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
-int cxd2820r_read_signal_strength_t(struct dvb_frontend *fe,
-	u16 *strength)
+int cxd2820r_read_status_t(struct dvb_frontend *fe, enum fe_status *status)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	int ret;
-	u8 buf[2];
-	u16 tmp;
+	unsigned int utmp, utmp1, utmp2;
+	u8 buf[3];
 
-	ret = cxd2820r_rd_regs(priv, 0x00026, buf, sizeof(buf));
+	/* Lock detection */
+	ret = regmap_bulk_read(priv->regmap[0], 0x0010, &buf[0], 1);
+	if (ret)
+		goto error;
+	ret = regmap_bulk_read(priv->regmap[0], 0x0073, &buf[1], 1);
 	if (ret)
 		goto error;
 
-	tmp = (buf[0] & 0x0f) << 8 | buf[1];
-	tmp = ~tmp & 0x0fff;
+	utmp1 = (buf[0] >> 0) & 0x07;
+	utmp2 = (buf[1] >> 3) & 0x01;
 
-	/* scale value to 0x0000-0xffff from 0x0000-0x0fff */
-	*strength = tmp * 0xffff / 0x0fff;
+	if (utmp1 == 6 && utmp2 == 1) {
+		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER |
+			  FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK;
+	} else if (utmp1 == 6 || utmp2 == 1) {
+		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER |
+			  FE_HAS_VITERBI | FE_HAS_SYNC;
+	} else {
+		*status = 0;
+	}
 
-	return ret;
-error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
-	return ret;
-}
+	dev_dbg(&client->dev, "status=%02x raw=%*ph sync=%u ts=%u\n",
+		*status, 2, buf, utmp1, utmp2);
 
-int cxd2820r_read_snr_t(struct dvb_frontend *fe, u16 *snr)
-{
-	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
-	u8 buf[2];
-	u16 tmp;
-	/* report SNR in dB * 10 */
+	/* Signal strength */
+	if (*status & FE_HAS_SIGNAL) {
+		unsigned int strength;
 
-	ret = cxd2820r_rd_regs(priv, 0x00028, buf, sizeof(buf));
-	if (ret)
-		goto error;
+		ret = regmap_bulk_read(priv->regmap[0], 0x0026, buf, 2);
+		if (ret)
+			goto error;
 
-	tmp = (buf[0] & 0x1f) << 8 | buf[1];
-	#define CXD2820R_LOG10_8_24 15151336 /* log10(8) << 24 */
-	if (tmp)
-		*snr = (intlog10(tmp) - CXD2820R_LOG10_8_24) / ((1 << 24)
-			/ 100);
-	else
-		*snr = 0;
+		utmp = buf[0] << 8 | buf[1] << 0;
+		utmp = ~utmp & 0x0fff;
+		/* Scale value to 0x0000-0xffff */
+		strength = utmp << 4 | utmp >> 8;
 
-	dev_dbg(&priv->i2c->dev, "%s: dBx10=%d val=%04x\n", __func__, *snr,
-			tmp);
+		c->strength.len = 1;
+		c->strength.stat[0].scale = FE_SCALE_RELATIVE;
+		c->strength.stat[0].uvalue = strength;
+	} else {
+		c->strength.len = 1;
+		c->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
 
-	return ret;
-error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
-	return ret;
-}
+	/* CNR */
+	if (*status & FE_HAS_VITERBI) {
+		unsigned int cnr;
 
-int cxd2820r_read_ucblocks_t(struct dvb_frontend *fe, u32 *ucblocks)
-{
-	*ucblocks = 0;
-	/* no way to read ? */
-	return 0;
-}
+		ret = regmap_bulk_read(priv->regmap[0], 0x002c, buf, 2);
+		if (ret)
+			goto error;
 
-int cxd2820r_read_status_t(struct dvb_frontend *fe, enum fe_status *status)
-{
-	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
-	u8 buf[4];
-	*status = 0;
+		utmp = buf[0] << 8 | buf[1] << 0;
+		if (utmp)
+			cnr = div_u64((u64)(intlog10(utmp)
+				      - intlog10(32000 - utmp) + 55532585)
+				      * 10000, (1 << 24));
+		else
+			cnr = 0;
+
+		c->cnr.len = 1;
+		c->cnr.stat[0].scale = FE_SCALE_DECIBEL;
+		c->cnr.stat[0].svalue = cnr;
+	} else {
+		c->cnr.len = 1;
+		c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
 
-	ret = cxd2820r_rd_reg(priv, 0x00010, &buf[0]);
-	if (ret)
-		goto error;
+	/* BER */
+	if (*status & FE_HAS_SYNC) {
+		unsigned int post_bit_error;
+		bool start_ber;
 
-	if ((buf[0] & 0x07) == 6) {
-		ret = cxd2820r_rd_reg(priv, 0x00073, &buf[1]);
-		if (ret)
-			goto error;
+		if (priv->ber_running) {
+			ret = regmap_bulk_read(priv->regmap[0], 0x0076, buf, 3);
+			if (ret)
+				goto error;
 
-		if (((buf[1] >> 3) & 0x01) == 1) {
-			*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER |
-				FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK;
+			if ((buf[2] >> 7) & 0x01) {
+				post_bit_error = buf[2] << 16 | buf[1] << 8 |
+						 buf[0] << 0;
+				post_bit_error &= 0x0fffff;
+				start_ber = true;
+			} else {
+				post_bit_error = 0;
+				start_ber = false;
+			}
 		} else {
-			*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER |
-				FE_HAS_VITERBI | FE_HAS_SYNC;
+			post_bit_error = 0;
+			start_ber = true;
 		}
-	} else {
-		ret = cxd2820r_rd_reg(priv, 0x00014, &buf[2]);
-		if (ret)
-			goto error;
 
-		if ((buf[2] & 0x0f) >= 4) {
-			ret = cxd2820r_rd_reg(priv, 0x00a14, &buf[3]);
+		if (start_ber) {
+			ret = regmap_write(priv->regmap[0], 0x0079, 0x01);
 			if (ret)
 				goto error;
-
-			if (((buf[3] >> 4) & 0x01) == 1)
-				*status |= FE_HAS_SIGNAL;
+			priv->ber_running = true;
 		}
-	}
 
-	dev_dbg(&priv->i2c->dev, "%s: lock=%*ph\n", __func__, 4, buf);
+		priv->post_bit_error += post_bit_error;
+
+		c->post_bit_error.len = 1;
+		c->post_bit_error.stat[0].scale = FE_SCALE_COUNTER;
+		c->post_bit_error.stat[0].uvalue = priv->post_bit_error;
+	} else {
+		c->post_bit_error.len = 1;
+		c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
 int cxd2820r_init_t(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 	int ret;
 
-	ret = cxd2820r_wr_reg(priv, 0x00085, 0x07);
+	dev_dbg(&client->dev, "\n");
+
+	ret = regmap_write(priv->regmap[0], 0x0085, 0x07);
 	if (ret)
 		goto error;
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
 int cxd2820r_sleep_t(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret, i;
+	struct i2c_client *client = priv->client[0];
+	int ret;
 	struct reg_val_mask tab[] = {
 		{ 0x000ff, 0x1f, 0xff },
 		{ 0x00085, 0x00, 0xff },
@@ -422,20 +413,17 @@ int cxd2820r_sleep_t(struct dvb_frontend *fe)
 		{ 0x00080, 0x00, 0xff },
 	};
 
-	dev_dbg(&priv->i2c->dev, "%s\n", __func__);
+	dev_dbg(&client->dev, "\n");
 
 	priv->delivery_system = SYS_UNDEFINED;
 
-	for (i = 0; i < ARRAY_SIZE(tab); i++) {
-		ret = cxd2820r_wr_reg_mask(priv, tab[i].reg, tab[i].val,
-			tab[i].mask);
-		if (ret)
-			goto error;
-	}
+	ret = cxd2820r_wr_reg_val_mask_tab(priv, tab, ARRAY_SIZE(tab));
+	if (ret)
+		goto error;
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
diff --git a/drivers/media/dvb-frontends/cxd2820r_t2.c b/drivers/media/dvb-frontends/cxd2820r_t2.c
index 704475676234..e641fde75379 100644
--- a/drivers/media/dvb-frontends/cxd2820r_t2.c
+++ b/drivers/media/dvb-frontends/cxd2820r_t2.c
@@ -23,11 +23,12 @@
 
 int cxd2820r_set_frontend_t2(struct dvb_frontend *fe)
 {
-	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret, i, bw_i;
-	u32 if_freq, if_ctl;
-	u64 num;
+	struct i2c_client *client = priv->client[0];
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+	int ret, bw_i;
+	unsigned int utmp;
+	u32 if_frequency;
 	u8 buf[3], bw_param;
 	u8 bw_params1[][5] = {
 		{ 0x1c, 0xb3, 0x33, 0x33, 0x33 }, /* 5 MHz */
@@ -45,10 +46,10 @@ int cxd2820r_set_frontend_t2(struct dvb_frontend *fe)
 		{ 0x0207f, 0x2a, 0xff },
 		{ 0x02082, 0x0a, 0xff },
 		{ 0x02083, 0x0a, 0xff },
-		{ 0x020cb, priv->cfg.if_agc_polarity << 6, 0x40 },
-		{ 0x02070, priv->cfg.ts_mode, 0xff },
-		{ 0x02071, !priv->cfg.ts_clock_inv << 6, 0x40 },
-		{ 0x020b5, priv->cfg.spec_inv << 4, 0x10 },
+		{ 0x020cb, priv->if_agc_polarity << 6, 0x40 },
+		{ 0x02070, priv->ts_mode, 0xff },
+		{ 0x02071, !priv->ts_clk_inv << 6, 0x40 },
+		{ 0x020b5, priv->spec_inv << 4, 0x10 },
 		{ 0x02567, 0x07, 0x0f },
 		{ 0x02569, 0x03, 0x03 },
 		{ 0x02595, 0x1a, 0xff },
@@ -69,8 +70,10 @@ int cxd2820r_set_frontend_t2(struct dvb_frontend *fe)
 		{ 0x027ef, 0x10, 0x18 },
 	};
 
-	dev_dbg(&priv->i2c->dev, "%s: frequency=%d bandwidth_hz=%d\n", __func__,
-			c->frequency, c->bandwidth_hz);
+	dev_dbg(&client->dev,
+		"delivery_system=%d modulation=%d frequency=%u bandwidth_hz=%u inversion=%d stream_id=%u\n",
+		c->delivery_system, c->modulation, c->frequency,
+		c->bandwidth_hz, c->inversion, c->stream_id);
 
 	switch (c->bandwidth_hz) {
 	case 5000000:
@@ -98,73 +101,67 @@ int cxd2820r_set_frontend_t2(struct dvb_frontend *fe)
 		fe->ops.tuner_ops.set_params(fe);
 
 	if (priv->delivery_system != SYS_DVBT2) {
-		for (i = 0; i < ARRAY_SIZE(tab); i++) {
-			ret = cxd2820r_wr_reg_mask(priv, tab[i].reg,
-				tab[i].val, tab[i].mask);
-			if (ret)
-				goto error;
-		}
+		ret = cxd2820r_wr_reg_val_mask_tab(priv, tab, ARRAY_SIZE(tab));
+		if (ret)
+			goto error;
 	}
 
 	priv->delivery_system = SYS_DVBT2;
 
 	/* program IF frequency */
 	if (fe->ops.tuner_ops.get_if_frequency) {
-		ret = fe->ops.tuner_ops.get_if_frequency(fe, &if_freq);
+		ret = fe->ops.tuner_ops.get_if_frequency(fe, &if_frequency);
 		if (ret)
 			goto error;
-	} else
-		if_freq = 0;
-
-	dev_dbg(&priv->i2c->dev, "%s: if_freq=%d\n", __func__, if_freq);
+		dev_dbg(&client->dev, "if_frequency=%u\n", if_frequency);
+	} else {
+		ret = -EINVAL;
+		goto error;
+	}
 
-	num = if_freq / 1000; /* Hz => kHz */
-	num *= 0x1000000;
-	if_ctl = DIV_ROUND_CLOSEST_ULL(num, 41000);
-	buf[0] = ((if_ctl >> 16) & 0xff);
-	buf[1] = ((if_ctl >>  8) & 0xff);
-	buf[2] = ((if_ctl >>  0) & 0xff);
+	utmp = DIV_ROUND_CLOSEST_ULL((u64)if_frequency * 0x1000000, CXD2820R_CLK);
+	buf[0] = (utmp >> 16) & 0xff;
+	buf[1] = (utmp >>  8) & 0xff;
+	buf[2] = (utmp >>  0) & 0xff;
+	ret = regmap_bulk_write(priv->regmap[0], 0x20b6, buf, 3);
+	if (ret)
+		goto error;
 
 	/* PLP filtering */
 	if (c->stream_id > 255) {
-		dev_dbg(&priv->i2c->dev, "%s: Disable PLP filtering\n", __func__);
-		ret = cxd2820r_wr_reg(priv, 0x023ad , 0);
+		dev_dbg(&client->dev, "disable PLP filtering\n");
+		ret = regmap_write(priv->regmap[0], 0x23ad, 0x00);
 		if (ret)
 			goto error;
 	} else {
-		dev_dbg(&priv->i2c->dev, "%s: Enable PLP filtering = %d\n", __func__,
-				c->stream_id);
-		ret = cxd2820r_wr_reg(priv, 0x023af , c->stream_id & 0xFF);
+		dev_dbg(&client->dev, "enable PLP filtering\n");
+		ret = regmap_write(priv->regmap[0], 0x23af, c->stream_id & 0xff);
 		if (ret)
 			goto error;
-		ret = cxd2820r_wr_reg(priv, 0x023ad , 1);
+		ret = regmap_write(priv->regmap[0], 0x23ad, 0x01);
 		if (ret)
 			goto error;
 	}
 
-	ret = cxd2820r_wr_regs(priv, 0x020b6, buf, 3);
+	ret = regmap_bulk_write(priv->regmap[0], 0x209f, bw_params1[bw_i], 5);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_regs(priv, 0x0209f, bw_params1[bw_i], 5);
+	ret = regmap_update_bits(priv->regmap[0], 0x20d7, 0xc0, bw_param << 6);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_reg_mask(priv, 0x020d7, bw_param << 6, 0xc0);
+	ret = regmap_write(priv->regmap[0], 0x00ff, 0x08);
 	if (ret)
 		goto error;
 
-	ret = cxd2820r_wr_reg(priv, 0x000ff, 0x08);
-	if (ret)
-		goto error;
-
-	ret = cxd2820r_wr_reg(priv, 0x000fe, 0x01);
+	ret = regmap_write(priv->regmap[0], 0x00fe, 0x01);
 	if (ret)
 		goto error;
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 
 }
@@ -173,10 +170,14 @@ int cxd2820r_get_frontend_t2(struct dvb_frontend *fe,
 			     struct dtv_frontend_properties *c)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct i2c_client *client = priv->client[0];
 	int ret;
+	unsigned int utmp;
 	u8 buf[2];
 
-	ret = cxd2820r_rd_regs(priv, 0x0205c, buf, 2);
+	dev_dbg(&client->dev, "\n");
+
+	ret = regmap_bulk_read(priv->regmap[0], 0x205c, buf, 2);
 	if (ret)
 		goto error;
 
@@ -225,7 +226,7 @@ int cxd2820r_get_frontend_t2(struct dvb_frontend *fe,
 		break;
 	}
 
-	ret = cxd2820r_rd_regs(priv, 0x0225b, buf, 2);
+	ret = regmap_bulk_read(priv->regmap[0], 0x225b, buf, 2);
 	if (ret)
 		goto error;
 
@@ -265,11 +266,11 @@ int cxd2820r_get_frontend_t2(struct dvb_frontend *fe,
 		break;
 	}
 
-	ret = cxd2820r_rd_reg(priv, 0x020b5, &buf[0]);
+	ret = regmap_read(priv->regmap[0], 0x20b5, &utmp);
 	if (ret)
 		goto error;
 
-	switch ((buf[0] >> 4) & 0x01) {
+	switch ((utmp >> 4) & 0x01) {
 	case 0:
 		c->inversion = INVERSION_OFF;
 		break;
@@ -280,130 +281,124 @@ int cxd2820r_get_frontend_t2(struct dvb_frontend *fe,
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
 int cxd2820r_read_status_t2(struct dvb_frontend *fe, enum fe_status *status)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+	struct i2c_client *client = priv->client[0];
 	int ret;
-	u8 buf[1];
-	*status = 0;
+	unsigned int utmp, utmp1, utmp2;
+	u8 buf[4];
 
-	ret = cxd2820r_rd_reg(priv, 0x02010 , &buf[0]);
+	/* Lock detection */
+	ret = regmap_bulk_read(priv->regmap[0], 0x2010, &buf[0], 1);
 	if (ret)
 		goto error;
 
-	if ((buf[0] & 0x07) == 6) {
-		if (((buf[0] >> 5) & 0x01) == 1) {
-			*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER |
-				FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK;
-		} else {
-			*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER |
-				FE_HAS_VITERBI | FE_HAS_SYNC;
-		}
-	}
+	utmp1 = (buf[0] >> 0) & 0x07;
+	utmp2 = (buf[0] >> 5) & 0x01;
 
-	dev_dbg(&priv->i2c->dev, "%s: lock=%02x\n", __func__, buf[0]);
+	if (utmp1 == 6 && utmp2 == 1) {
+		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER |
+			  FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK;
+	} else if (utmp1 == 6 || utmp2 == 1) {
+		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER |
+			  FE_HAS_VITERBI | FE_HAS_SYNC;
+	} else {
+		*status = 0;
+	}
 
-	return ret;
-error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
-	return ret;
-}
+	dev_dbg(&client->dev, "status=%02x raw=%*ph sync=%u ts=%u\n",
+		*status, 1, buf, utmp1, utmp2);
 
-int cxd2820r_read_ber_t2(struct dvb_frontend *fe, u32 *ber)
-{
-	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
-	u8 buf[4];
-	unsigned int errbits;
-	*ber = 0;
-	/* FIXME: correct calculation */
+	/* Signal strength */
+	if (*status & FE_HAS_SIGNAL) {
+		unsigned int strength;
 
-	ret = cxd2820r_rd_regs(priv, 0x02039, buf, sizeof(buf));
-	if (ret)
-		goto error;
+		ret = regmap_bulk_read(priv->regmap[0], 0x2026, buf, 2);
+		if (ret)
+			goto error;
 
-	if ((buf[0] >> 4) & 0x01) {
-		errbits = (buf[0] & 0x0f) << 24 | buf[1] << 16 |
-			buf[2] << 8 | buf[3];
+		utmp = buf[0] << 8 | buf[1] << 0;
+		utmp = ~utmp & 0x0fff;
+		/* Scale value to 0x0000-0xffff */
+		strength = utmp << 4 | utmp >> 8;
 
-		if (errbits)
-			*ber = errbits * 64 / 16588800;
+		c->strength.len = 1;
+		c->strength.stat[0].scale = FE_SCALE_RELATIVE;
+		c->strength.stat[0].uvalue = strength;
+	} else {
+		c->strength.len = 1;
+		c->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 	}
 
-	return ret;
-error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
-	return ret;
-}
+	/* CNR */
+	if (*status & FE_HAS_VITERBI) {
+		unsigned int cnr;
 
-int cxd2820r_read_signal_strength_t2(struct dvb_frontend *fe,
-	u16 *strength)
-{
-	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
-	u8 buf[2];
-	u16 tmp;
-
-	ret = cxd2820r_rd_regs(priv, 0x02026, buf, sizeof(buf));
-	if (ret)
-		goto error;
-
-	tmp = (buf[0] & 0x0f) << 8 | buf[1];
-	tmp = ~tmp & 0x0fff;
+		ret = regmap_bulk_read(priv->regmap[0], 0x2028, buf, 2);
+		if (ret)
+			goto error;
 
-	/* scale value to 0x0000-0xffff from 0x0000-0x0fff */
-	*strength = tmp * 0xffff / 0x0fff;
+		utmp = buf[0] << 8 | buf[1] << 0;
+		utmp = utmp & 0x0fff;
+		#define CXD2820R_LOG10_8_24 15151336 /* log10(8) << 24 */
+		if (utmp)
+			cnr = div_u64((u64)(intlog10(utmp)
+				      - CXD2820R_LOG10_8_24) * 10000,
+				      (1 << 24));
+		else
+			cnr = 0;
+
+		c->cnr.len = 1;
+		c->cnr.stat[0].scale = FE_SCALE_DECIBEL;
+		c->cnr.stat[0].svalue = cnr;
+	} else {
+		c->cnr.len = 1;
+		c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
 
-	return ret;
-error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
-	return ret;
-}
+	/* BER */
+	if (*status & FE_HAS_SYNC) {
+		unsigned int post_bit_error;
 
-int cxd2820r_read_snr_t2(struct dvb_frontend *fe, u16 *snr)
-{
-	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret;
-	u8 buf[2];
-	u16 tmp;
-	/* report SNR in dB * 10 */
+		ret = regmap_bulk_read(priv->regmap[0], 0x2039, buf, 4);
+		if (ret)
+			goto error;
 
-	ret = cxd2820r_rd_regs(priv, 0x02028, buf, sizeof(buf));
-	if (ret)
-		goto error;
+		if ((buf[0] >> 4) & 0x01) {
+			post_bit_error = buf[0] << 24 | buf[1] << 16 |
+					 buf[2] << 8 | buf[3] << 0;
+			post_bit_error &= 0x0fffffff;
+		} else {
+			post_bit_error = 0;
+		}
 
-	tmp = (buf[0] & 0x0f) << 8 | buf[1];
-	#define CXD2820R_LOG10_8_24 15151336 /* log10(8) << 24 */
-	if (tmp)
-		*snr = (intlog10(tmp) - CXD2820R_LOG10_8_24) / ((1 << 24)
-			/ 100);
-	else
-		*snr = 0;
+		priv->post_bit_error += post_bit_error;
 
-	dev_dbg(&priv->i2c->dev, "%s: dBx10=%d val=%04x\n", __func__, *snr,
-			tmp);
+		c->post_bit_error.len = 1;
+		c->post_bit_error.stat[0].scale = FE_SCALE_COUNTER;
+		c->post_bit_error.stat[0].uvalue = priv->post_bit_error;
+	} else {
+		c->post_bit_error.len = 1;
+		c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
-int cxd2820r_read_ucblocks_t2(struct dvb_frontend *fe, u32 *ucblocks)
-{
-	*ucblocks = 0;
-	/* no way to read ? */
-	return 0;
-}
-
 int cxd2820r_sleep_t2(struct dvb_frontend *fe)
 {
 	struct cxd2820r_priv *priv = fe->demodulator_priv;
-	int ret, i;
+	struct i2c_client *client = priv->client[0];
+	int ret;
 	struct reg_val_mask tab[] = {
 		{ 0x000ff, 0x1f, 0xff },
 		{ 0x00085, 0x00, 0xff },
@@ -413,20 +408,17 @@ int cxd2820r_sleep_t2(struct dvb_frontend *fe)
 		{ 0x00080, 0x00, 0xff },
 	};
 
-	dev_dbg(&priv->i2c->dev, "%s\n", __func__);
+	dev_dbg(&client->dev, "\n");
 
-	for (i = 0; i < ARRAY_SIZE(tab); i++) {
-		ret = cxd2820r_wr_reg_mask(priv, tab[i].reg, tab[i].val,
-			tab[i].mask);
-		if (ret)
-			goto error;
-	}
+	ret = cxd2820r_wr_reg_val_mask_tab(priv, tab, ARRAY_SIZE(tab));
+	if (ret)
+		goto error;
 
 	priv->delivery_system = SYS_UNDEFINED;
 
 	return ret;
 error:
-	dev_dbg(&priv->i2c->dev, "%s: failed=%d\n", __func__, ret);
+	dev_dbg(&client->dev, "failed=%d\n", ret);
 	return ret;
 }
 
diff --git a/drivers/media/dvb-frontends/cxd2841er.c b/drivers/media/dvb-frontends/cxd2841er.c
index ffe88bc6b813..5afb9c508f65 100644
--- a/drivers/media/dvb-frontends/cxd2841er.c
+++ b/drivers/media/dvb-frontends/cxd2841er.c
@@ -206,6 +206,9 @@ static const struct cxd2841er_cnr_data s2_cn_data[] = {
 		(u32)(((iffreq)/48.0)*16777216.0 + 0.5) : \
 		(u32)(((iffreq)/41.0)*16777216.0 + 0.5))
 
+static int cxd2841er_freeze_regs(struct cxd2841er_priv *priv);
+static int cxd2841er_unfreeze_regs(struct cxd2841er_priv *priv);
+
 static void cxd2841er_i2c_debug(struct cxd2841er_priv *priv,
 				u8 addr, u8 reg, u8 write,
 				const u8 *data, u32 len)
@@ -1401,6 +1404,41 @@ static int cxd2841er_read_ber_c(struct cxd2841er_priv *priv,
 	return 0;
 }
 
+static int cxd2841er_read_ber_i(struct cxd2841er_priv *priv,
+		u32 *bit_error, u32 *bit_count)
+{
+	u8 data[3];
+	u8 pktnum[2];
+
+	dev_dbg(&priv->i2c->dev, "%s()\n", __func__);
+	if (priv->state != STATE_ACTIVE_TC) {
+		dev_dbg(&priv->i2c->dev, "%s(): invalid state %d\n",
+				__func__, priv->state);
+		return -EINVAL;
+	}
+
+	cxd2841er_freeze_regs(priv);
+	cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x60);
+	cxd2841er_read_regs(priv, I2C_SLVT, 0x5B, pktnum, sizeof(pktnum));
+	cxd2841er_read_regs(priv, I2C_SLVT, 0x16, data, sizeof(data));
+
+	if (!pktnum[0] && !pktnum[1]) {
+		dev_dbg(&priv->i2c->dev,
+				"%s(): no valid BER data\n", __func__);
+		cxd2841er_unfreeze_regs(priv);
+		return -EINVAL;
+	}
+
+	*bit_error = ((u32)(data[0] & 0x7F) << 16) |
+		((u32)data[1] << 8) | data[2];
+	*bit_count = ((((u32)pktnum[0] << 8) | pktnum[1]) * 204 * 8);
+	dev_dbg(&priv->i2c->dev, "%s(): bit_error=%u bit_count=%u\n",
+			__func__, *bit_error, *bit_count);
+
+	cxd2841er_unfreeze_regs(priv);
+	return 0;
+}
+
 static int cxd2841er_mon_read_ber_s(struct cxd2841er_priv *priv,
 				    u32 *bit_error, u32 *bit_count)
 {
@@ -1570,6 +1608,25 @@ static int cxd2841er_read_ber_t(struct cxd2841er_priv *priv,
 	return 0;
 }
 
+static int cxd2841er_freeze_regs(struct cxd2841er_priv *priv)
+{
+	/*
+	 * Freeze registers: ensure multiple separate register reads
+	 * are from the same snapshot
+	 */
+	cxd2841er_write_reg(priv, I2C_SLVT, 0x01, 0x01);
+	return 0;
+}
+
+static int cxd2841er_unfreeze_regs(struct cxd2841er_priv *priv)
+{
+	/*
+	 * un-freeze registers
+	 */
+	cxd2841er_write_reg(priv, I2C_SLVT, 0x01, 0x00);
+	return 0;
+}
+
 static u32 cxd2841er_dvbs_read_snr(struct cxd2841er_priv *priv,
 		u8 delsys, u32 *snr)
 {
@@ -1578,6 +1635,7 @@ static u32 cxd2841er_dvbs_read_snr(struct cxd2841er_priv *priv,
 	int min_index, max_index, index;
 	static const struct cxd2841er_cnr_data *cn_data;
 
+	cxd2841er_freeze_regs(priv);
 	/* Set SLV-T Bank : 0xA1 */
 	cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0xa1);
 	/*
@@ -1629,9 +1687,11 @@ static u32 cxd2841er_dvbs_read_snr(struct cxd2841er_priv *priv,
 	} else {
 		dev_dbg(&priv->i2c->dev,
 			"%s(): no data available\n", __func__);
+		cxd2841er_unfreeze_regs(priv);
 		return -EINVAL;
 	}
 done:
+	cxd2841er_unfreeze_regs(priv);
 	*snr = res;
 	return 0;
 }
@@ -1655,12 +1715,7 @@ static int cxd2841er_read_snr_c(struct cxd2841er_priv *priv, u32 *snr)
 		return -EINVAL;
 	}
 
-	/*
-	 * Freeze registers: ensure multiple separate register reads
-	 * are from the same snapshot
-	 */
-	cxd2841er_write_reg(priv, I2C_SLVT, 0x01, 0x01);
-
+	cxd2841er_freeze_regs(priv);
 	cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x40);
 	cxd2841er_read_regs(priv, I2C_SLVT, 0x19, data, 1);
 	qam = (enum sony_dvbc_constellation_t) (data[0] & 0x07);
@@ -1670,6 +1725,7 @@ static int cxd2841er_read_snr_c(struct cxd2841er_priv *priv, u32 *snr)
 	if (reg == 0) {
 		dev_dbg(&priv->i2c->dev,
 				"%s(): reg value out of range\n", __func__);
+		cxd2841er_unfreeze_regs(priv);
 		return 0;
 	}
 
@@ -1690,9 +1746,11 @@ static int cxd2841er_read_snr_c(struct cxd2841er_priv *priv, u32 *snr)
 		*snr = -88 * (int32_t)sony_log(reg) + 86999;
 		break;
 	default:
+		cxd2841er_unfreeze_regs(priv);
 		return -EINVAL;
 	}
 
+	cxd2841er_unfreeze_regs(priv);
 	return 0;
 }
 
@@ -1707,17 +1765,21 @@ static int cxd2841er_read_snr_t(struct cxd2841er_priv *priv, u32 *snr)
 			"%s(): invalid state %d\n", __func__, priv->state);
 		return -EINVAL;
 	}
+
+	cxd2841er_freeze_regs(priv);
 	cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10);
 	cxd2841er_read_regs(priv, I2C_SLVT, 0x28, data, sizeof(data));
 	reg = ((u32)data[0] << 8) | (u32)data[1];
 	if (reg == 0) {
 		dev_dbg(&priv->i2c->dev,
 			"%s(): reg value out of range\n", __func__);
+		cxd2841er_unfreeze_regs(priv);
 		return 0;
 	}
 	if (reg > 4996)
 		reg = 4996;
 	*snr = 10000 * ((intlog10(reg) - intlog10(5350 - reg)) >> 24) + 28500;
+	cxd2841er_unfreeze_regs(priv);
 	return 0;
 }
 
@@ -1732,18 +1794,22 @@ static int cxd2841er_read_snr_t2(struct cxd2841er_priv *priv, u32 *snr)
 			"%s(): invalid state %d\n", __func__, priv->state);
 		return -EINVAL;
 	}
+
+	cxd2841er_freeze_regs(priv);
 	cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x20);
 	cxd2841er_read_regs(priv, I2C_SLVT, 0x28, data, sizeof(data));
 	reg = ((u32)data[0] << 8) | (u32)data[1];
 	if (reg == 0) {
 		dev_dbg(&priv->i2c->dev,
 			"%s(): reg value out of range\n", __func__);
+		cxd2841er_unfreeze_regs(priv);
 		return 0;
 	}
 	if (reg > 10876)
 		reg = 10876;
 	*snr = 10000 * ((intlog10(reg) -
 		intlog10(12600 - reg)) >> 24) + 32000;
+	cxd2841er_unfreeze_regs(priv);
 	return 0;
 }
 
@@ -1760,21 +1826,18 @@ static int cxd2841er_read_snr_i(struct cxd2841er_priv *priv, u32 *snr)
 		return -EINVAL;
 	}
 
-	/* Freeze all registers */
-	cxd2841er_write_reg(priv, I2C_SLVT, 0x01, 0x01);
-
-
+	cxd2841er_freeze_regs(priv);
 	cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x60);
 	cxd2841er_read_regs(priv, I2C_SLVT, 0x28, data, sizeof(data));
 	reg = ((u32)data[0] << 8) | (u32)data[1];
 	if (reg == 0) {
 		dev_dbg(&priv->i2c->dev,
 				"%s(): reg value out of range\n", __func__);
+		cxd2841er_unfreeze_regs(priv);
 		return 0;
 	}
-	if (reg > 4996)
-		reg = 4996;
-	*snr = 100 * intlog10(reg) - 9031;
+	*snr = 10000 * (intlog10(reg) >> 24) - 9031;
+	cxd2841er_unfreeze_regs(priv);
 	return 0;
 }
 
@@ -1852,6 +1915,9 @@ static void cxd2841er_read_ber(struct dvb_frontend *fe)
 	case SYS_DVBC_ANNEX_C:
 		ret = cxd2841er_read_ber_c(priv, &bit_error, &bit_count);
 		break;
+	case SYS_ISDBT:
+		ret = cxd2841er_read_ber_i(priv, &bit_error, &bit_count);
+		break;
 	case SYS_DVBS:
 		ret = cxd2841er_mon_read_ber_s(priv, &bit_error, &bit_count);
 		break;
@@ -1965,6 +2031,9 @@ static void cxd2841er_read_snr(struct dvb_frontend *fe)
 		return;
 	}
 
+	dev_dbg(&priv->i2c->dev, "%s(): snr=%d\n",
+			__func__, (int32_t)tmp);
+
 	if (!ret) {
 		p->cnr.stat[0].scale = FE_SCALE_DECIBEL;
 		p->cnr.stat[0].svalue = tmp;
@@ -1977,7 +2046,7 @@ static void cxd2841er_read_ucblocks(struct dvb_frontend *fe)
 {
 	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
 	struct cxd2841er_priv *priv = fe->demodulator_priv;
-	u32 ucblocks;
+	u32 ucblocks = 0;
 
 	dev_dbg(&priv->i2c->dev, "%s()\n", __func__);
 	switch (p->delivery_system) {
@@ -1999,7 +2068,7 @@ static void cxd2841er_read_ucblocks(struct dvb_frontend *fe)
 		p->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
 		return;
 	}
-	dev_dbg(&priv->i2c->dev, "%s()\n", __func__);
+	dev_dbg(&priv->i2c->dev, "%s() ucblocks=%u\n", __func__, ucblocks);
 
 	p->block_error.stat[0].scale = FE_SCALE_COUNTER;
 	p->block_error.stat[0].uvalue = ucblocks;
@@ -2694,6 +2763,14 @@ static int cxd2841er_sleep_tc_to_active_c_band(struct cxd2841er_priv *priv,
 	u8 b10_b6[3];
 	u32 iffreq;
 
+	if (bandwidth != 6000000 &&
+			bandwidth != 7000000 &&
+			bandwidth != 8000000) {
+		dev_info(&priv->i2c->dev, "%s(): unsupported bandwidth %d. Forcing 8Mhz!\n",
+				__func__, bandwidth);
+		bandwidth = 8000000;
+	}
+
 	dev_dbg(&priv->i2c->dev, "%s() bw=%d\n", __func__, bandwidth);
 	cxd2841er_write_reg(priv, I2C_SLVT, 0x00, 0x10);
 	switch (bandwidth) {
@@ -3076,6 +3153,7 @@ static int cxd2841er_sleep_tc_to_active_c(struct cxd2841er_priv *priv,
 	/* Enable demod clock */
 	cxd2841er_write_reg(priv, I2C_SLVT, 0x2c, 0x01);
 	/* Disable RF level monitor */
+	cxd2841er_write_reg(priv, I2C_SLVT, 0x59, 0x00);
 	cxd2841er_write_reg(priv, I2C_SLVT, 0x2f, 0x00);
 	/* Enable ADC clock */
 	cxd2841er_write_reg(priv, I2C_SLVT, 0x30, 0x00);
diff --git a/drivers/media/dvb-frontends/cxd2841er.h b/drivers/media/dvb-frontends/cxd2841er.h
index 62ad5f07390b..7f1acfb8f4f5 100644
--- a/drivers/media/dvb-frontends/cxd2841er.h
+++ b/drivers/media/dvb-frontends/cxd2841er.h
@@ -22,7 +22,6 @@
 #ifndef CXD2841ER_H
 #define CXD2841ER_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 enum cxd2841er_xtal {
diff --git a/drivers/media/dvb-frontends/dib3000mc.h b/drivers/media/dvb-frontends/dib3000mc.h
index b37e69e6a58c..67a6d50865fb 100644
--- a/drivers/media/dvb-frontends/dib3000mc.h
+++ b/drivers/media/dvb-frontends/dib3000mc.h
@@ -13,8 +13,6 @@
 #ifndef DIB3000MC_H
 #define DIB3000MC_H
 
-#include <linux/kconfig.h>
-
 #include "dibx000_common.h"
 
 struct dib3000mc_config {
diff --git a/drivers/media/dvb-frontends/dib7000m.h b/drivers/media/dvb-frontends/dib7000m.h
index 6468c278cc4d..8f84dfa9bb58 100644
--- a/drivers/media/dvb-frontends/dib7000m.h
+++ b/drivers/media/dvb-frontends/dib7000m.h
@@ -1,8 +1,6 @@
 #ifndef DIB7000M_H
 #define DIB7000M_H
 
-#include <linux/kconfig.h>
-
 #include "dibx000_common.h"
 
 struct dib7000m_config {
diff --git a/drivers/media/dvb-frontends/dib7000p.h b/drivers/media/dvb-frontends/dib7000p.h
index baa278928cf3..205fbbff632b 100644
--- a/drivers/media/dvb-frontends/dib7000p.h
+++ b/drivers/media/dvb-frontends/dib7000p.h
@@ -1,8 +1,6 @@
 #ifndef DIB7000P_H
 #define DIB7000P_H
 
-#include <linux/kconfig.h>
-
 #include "dibx000_common.h"
 
 struct dib7000p_config {
diff --git a/drivers/media/dvb-frontends/drxd.h b/drivers/media/dvb-frontends/drxd.h
index a47c22d6667e..f0507cdbb503 100644
--- a/drivers/media/dvb-frontends/drxd.h
+++ b/drivers/media/dvb-frontends/drxd.h
@@ -24,7 +24,6 @@
 #ifndef _DRXD_H_
 #define _DRXD_H_
 
-#include <linux/kconfig.h>
 #include <linux/types.h>
 #include <linux/i2c.h>
 
diff --git a/drivers/media/dvb-frontends/drxk.h b/drivers/media/dvb-frontends/drxk.h
index 8f0b9eec528f..a629897eb905 100644
--- a/drivers/media/dvb-frontends/drxk.h
+++ b/drivers/media/dvb-frontends/drxk.h
@@ -1,7 +1,6 @@
 #ifndef _DRXK_H_
 #define _DRXK_H_
 
-#include <linux/kconfig.h>
 #include <linux/types.h>
 #include <linux/i2c.h>
 
diff --git a/drivers/media/dvb-frontends/drxk_hard.c b/drivers/media/dvb-frontends/drxk_hard.c
index b975da099929..c595adc61c6f 100644
--- a/drivers/media/dvb-frontends/drxk_hard.c
+++ b/drivers/media/dvb-frontends/drxk_hard.c
@@ -6448,7 +6448,7 @@ static int get_strength(struct drxk_state *state, u64 *strength)
 			return status;
 
 		/* SCU c.o.c. */
-		read16(state, SCU_RAM_AGC_RF_IACCU_HI_CO__A, &scu_coc);
+		status = read16(state, SCU_RAM_AGC_RF_IACCU_HI_CO__A, &scu_coc);
 		if (status < 0)
 			return status;
 
diff --git a/drivers/media/dvb-frontends/ds3000.h b/drivers/media/dvb-frontends/ds3000.h
index 153169da9017..82e8c2531f26 100644
--- a/drivers/media/dvb-frontends/ds3000.h
+++ b/drivers/media/dvb-frontends/ds3000.h
@@ -22,7 +22,6 @@
 #ifndef DS3000_H
 #define DS3000_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct ds3000_config {
diff --git a/drivers/media/dvb-frontends/dvb-pll.c b/drivers/media/dvb-frontends/dvb-pll.c
index 53089e142715..735a96662022 100644
--- a/drivers/media/dvb-frontends/dvb-pll.c
+++ b/drivers/media/dvb-frontends/dvb-pll.c
@@ -739,7 +739,7 @@ static int dvb_pll_init(struct dvb_frontend *fe)
 	return -EINVAL;
 }
 
-static struct dvb_tuner_ops dvb_pll_tuner_ops = {
+static const struct dvb_tuner_ops dvb_pll_tuner_ops = {
 	.release = dvb_pll_release,
 	.sleep = dvb_pll_sleep,
 	.init = dvb_pll_init,
diff --git a/drivers/media/dvb-frontends/dvb_dummy_fe.h b/drivers/media/dvb-frontends/dvb_dummy_fe.h
index 15e4ceab869a..50f1af512b62 100644
--- a/drivers/media/dvb-frontends/dvb_dummy_fe.h
+++ b/drivers/media/dvb-frontends/dvb_dummy_fe.h
@@ -22,7 +22,6 @@
 #ifndef DVB_DUMMY_FE_H
 #define DVB_DUMMY_FE_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/ec100.h b/drivers/media/dvb-frontends/ec100.h
index 9544bab5cd1d..e894bdcf35a3 100644
--- a/drivers/media/dvb-frontends/ec100.h
+++ b/drivers/media/dvb-frontends/ec100.h
@@ -22,7 +22,6 @@
 #ifndef EC100_H
 #define EC100_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct ec100_config {
diff --git a/drivers/media/dvb-frontends/hd29l2.h b/drivers/media/dvb-frontends/hd29l2.h
index 48e9ab74c883..a14d6f36dbf6 100644
--- a/drivers/media/dvb-frontends/hd29l2.h
+++ b/drivers/media/dvb-frontends/hd29l2.h
@@ -23,7 +23,6 @@
 #ifndef HD29L2_H
 #define HD29L2_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct hd29l2_config {
diff --git a/drivers/media/dvb-frontends/helene.c b/drivers/media/dvb-frontends/helene.c
index 97a8982740a6..dc43c5f6d0ea 100644
--- a/drivers/media/dvb-frontends/helene.c
+++ b/drivers/media/dvb-frontends/helene.c
@@ -842,7 +842,7 @@ static int helene_get_frequency(struct dvb_frontend *fe, u32 *frequency)
 	return 0;
 }
 
-static struct dvb_tuner_ops helene_tuner_ops = {
+static const struct dvb_tuner_ops helene_tuner_ops = {
 	.info = {
 		.name = "Sony HELENE Ter tuner",
 		.frequency_min = 1000000,
@@ -856,7 +856,7 @@ static struct dvb_tuner_ops helene_tuner_ops = {
 	.get_frequency = helene_get_frequency,
 };
 
-static struct dvb_tuner_ops helene_tuner_ops_s = {
+static const struct dvb_tuner_ops helene_tuner_ops_s = {
 	.info = {
 		.name = "Sony HELENE Sat tuner",
 		.frequency_min = 500000,
@@ -987,8 +987,10 @@ struct dvb_frontend *helene_attach_s(struct dvb_frontend *fe,
 	if (fe->ops.i2c_gate_ctrl)
 		fe->ops.i2c_gate_ctrl(fe, 1);
 
-	if (helene_x_pon(priv) != 0)
+	if (helene_x_pon(priv) != 0) {
+		kfree(priv);
 		return NULL;
+	}
 
 	if (fe->ops.i2c_gate_ctrl)
 		fe->ops.i2c_gate_ctrl(fe, 0);
@@ -1021,8 +1023,10 @@ struct dvb_frontend *helene_attach(struct dvb_frontend *fe,
 	if (fe->ops.i2c_gate_ctrl)
 		fe->ops.i2c_gate_ctrl(fe, 1);
 
-	if (helene_x_pon(priv) != 0)
+	if (helene_x_pon(priv) != 0) {
+		kfree(priv);
 		return NULL;
+	}
 
 	if (fe->ops.i2c_gate_ctrl)
 		fe->ops.i2c_gate_ctrl(fe, 0);
diff --git a/drivers/media/dvb-frontends/helene.h b/drivers/media/dvb-frontends/helene.h
index e1b9224cfc55..333615491d9e 100644
--- a/drivers/media/dvb-frontends/helene.h
+++ b/drivers/media/dvb-frontends/helene.h
@@ -21,7 +21,6 @@
 #ifndef __DVB_HELENE_H__
 #define __DVB_HELENE_H__
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include <linux/i2c.h>
 
diff --git a/drivers/media/dvb-frontends/horus3a.c b/drivers/media/dvb-frontends/horus3a.c
index a98bca5270d9..0c089b5986a1 100644
--- a/drivers/media/dvb-frontends/horus3a.c
+++ b/drivers/media/dvb-frontends/horus3a.c
@@ -326,7 +326,7 @@ static int horus3a_get_frequency(struct dvb_frontend *fe, u32 *frequency)
 	return 0;
 }
 
-static struct dvb_tuner_ops horus3a_tuner_ops = {
+static const struct dvb_tuner_ops horus3a_tuner_ops = {
 	.info = {
 		.name = "Sony Horus3a",
 		.frequency_min = 950000,
diff --git a/drivers/media/dvb-frontends/horus3a.h b/drivers/media/dvb-frontends/horus3a.h
index c1e2d1834b78..672a556df71a 100644
--- a/drivers/media/dvb-frontends/horus3a.h
+++ b/drivers/media/dvb-frontends/horus3a.h
@@ -22,7 +22,6 @@
 #ifndef __DVB_HORUS3A_H__
 #define __DVB_HORUS3A_H__
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include <linux/i2c.h>
 
diff --git a/drivers/media/dvb-frontends/ix2505v.c b/drivers/media/dvb-frontends/ix2505v.c
index 0e3387e00952..2826bbb36b73 100644
--- a/drivers/media/dvb-frontends/ix2505v.c
+++ b/drivers/media/dvb-frontends/ix2505v.c
@@ -258,7 +258,7 @@ static int ix2505v_get_frequency(struct dvb_frontend *fe, u32 *frequency)
 	return 0;
 }
 
-static struct dvb_tuner_ops ix2505v_tuner_ops = {
+static const struct dvb_tuner_ops ix2505v_tuner_ops = {
 	.info = {
 		.name = "Sharp IX2505V (B0017)",
 		.frequency_min = 950000,
diff --git a/drivers/media/dvb-frontends/ix2505v.h b/drivers/media/dvb-frontends/ix2505v.h
index af107a2dd357..5eab39744b23 100644
--- a/drivers/media/dvb-frontends/ix2505v.h
+++ b/drivers/media/dvb-frontends/ix2505v.h
@@ -20,7 +20,6 @@
 #ifndef DVB_IX2505V_H
 #define DVB_IX2505V_H
 
-#include <linux/kconfig.h>
 #include <linux/i2c.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/lg2160.h b/drivers/media/dvb-frontends/lg2160.h
index d20bd909de39..8c74ddc6b88a 100644
--- a/drivers/media/dvb-frontends/lg2160.h
+++ b/drivers/media/dvb-frontends/lg2160.h
@@ -22,7 +22,6 @@
 #ifndef _LG2160_H_
 #define _LG2160_H_
 
-#include <linux/kconfig.h>
 #include <linux/i2c.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/lgdt3305.h b/drivers/media/dvb-frontends/lgdt3305.h
index f91a1b49ce2f..e7dceb60e572 100644
--- a/drivers/media/dvb-frontends/lgdt3305.h
+++ b/drivers/media/dvb-frontends/lgdt3305.h
@@ -22,7 +22,6 @@
 #ifndef _LGDT3305_H_
 #define _LGDT3305_H_
 
-#include <linux/kconfig.h>
 #include <linux/i2c.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/lgdt3306a.c b/drivers/media/dvb-frontends/lgdt3306a.c
index 179c26e5eb4e..0ca4e810e9d8 100644
--- a/drivers/media/dvb-frontends/lgdt3306a.c
+++ b/drivers/media/dvb-frontends/lgdt3306a.c
@@ -731,7 +731,7 @@ static int lgdt3306a_set_if(struct lgdt3306a_state *state,
 
 	switch (if_freq_khz) {
 	default:
-		pr_warn("IF=%d KHz is not supportted, 3250 assumed\n",
+		pr_warn("IF=%d KHz is not supported, 3250 assumed\n",
 			if_freq_khz);
 		/* fallthrough */
 	case 3250: /* 3.25Mhz */
@@ -1737,24 +1737,16 @@ static int lgdt3306a_get_tune_settings(struct dvb_frontend *fe,
 static int lgdt3306a_search(struct dvb_frontend *fe)
 {
 	enum fe_status status = 0;
-	int i, ret;
+	int ret;
 
 	/* set frontend */
 	ret = lgdt3306a_set_parameters(fe);
 	if (ret)
 		goto error;
 
-	/* wait frontend lock */
-	for (i = 20; i > 0; i--) {
-		dbg_info(": loop=%d\n", i);
-		msleep(50);
-		ret = lgdt3306a_read_status(fe, &status);
-		if (ret)
-			goto error;
-
-		if (status & FE_HAS_LOCK)
-			break;
-	}
+	ret = lgdt3306a_read_status(fe, &status);
+	if (ret)
+		goto error;
 
 	/* check if we have a valid signal */
 	if (status & FE_HAS_LOCK)
diff --git a/drivers/media/dvb-frontends/lgs8gl5.h b/drivers/media/dvb-frontends/lgs8gl5.h
index a5b3faf121f0..f36a7fd0b102 100644
--- a/drivers/media/dvb-frontends/lgs8gl5.h
+++ b/drivers/media/dvb-frontends/lgs8gl5.h
@@ -23,7 +23,6 @@
 #ifndef LGS8GL5_H
 #define LGS8GL5_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct lgs8gl5_config {
diff --git a/drivers/media/dvb-frontends/lgs8gxx.h b/drivers/media/dvb-frontends/lgs8gxx.h
index 368c9928ef7f..7519c0210399 100644
--- a/drivers/media/dvb-frontends/lgs8gxx.h
+++ b/drivers/media/dvb-frontends/lgs8gxx.h
@@ -26,7 +26,6 @@
 #ifndef __LGS8GXX_H__
 #define __LGS8GXX_H__
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include <linux/i2c.h>
 
diff --git a/drivers/media/dvb-frontends/lnbh24.h b/drivers/media/dvb-frontends/lnbh24.h
index a088b8ec1e53..24431dfdce1f 100644
--- a/drivers/media/dvb-frontends/lnbh24.h
+++ b/drivers/media/dvb-frontends/lnbh24.h
@@ -23,8 +23,6 @@
 #ifndef _LNBH24_H
 #define _LNBH24_H
 
-#include <linux/kconfig.h>
-
 /* system register bits */
 #define LNBH24_OLF	0x01
 #define LNBH24_OTF	0x02
diff --git a/drivers/media/dvb-frontends/lnbh25.h b/drivers/media/dvb-frontends/lnbh25.h
index 1f329ef05acc..f13fd0308b3e 100644
--- a/drivers/media/dvb-frontends/lnbh25.h
+++ b/drivers/media/dvb-frontends/lnbh25.h
@@ -22,7 +22,6 @@
 #define LNBH25_H
 
 #include <linux/i2c.h>
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 /* 22 kHz tone enabled. Tone output controlled by DSQIN pin */
diff --git a/drivers/media/dvb-frontends/lnbp21.h b/drivers/media/dvb-frontends/lnbp21.h
index cd9101f6e579..4bb6439068ec 100644
--- a/drivers/media/dvb-frontends/lnbp21.h
+++ b/drivers/media/dvb-frontends/lnbp21.h
@@ -27,8 +27,6 @@
 #ifndef _LNBP21_H
 #define _LNBP21_H
 
-#include <linux/kconfig.h>
-
 /* system register bits */
 /* [RO] 0=OK; 1=over current limit flag */
 #define LNBP21_OLF	0x01
diff --git a/drivers/media/dvb-frontends/lnbp22.h b/drivers/media/dvb-frontends/lnbp22.h
index 5d01d92814c2..0cb72126c498 100644
--- a/drivers/media/dvb-frontends/lnbp22.h
+++ b/drivers/media/dvb-frontends/lnbp22.h
@@ -28,8 +28,6 @@
 #ifndef _LNBP22_H
 #define _LNBP22_H
 
-#include <linux/kconfig.h>
-
 /* Enable */
 #define LNBP22_EN	  0x10
 /* Voltage selection */
diff --git a/drivers/media/dvb-frontends/m88rs2000.h b/drivers/media/dvb-frontends/m88rs2000.h
index de7430178e9e..1a313b0f5875 100644
--- a/drivers/media/dvb-frontends/m88rs2000.h
+++ b/drivers/media/dvb-frontends/m88rs2000.h
@@ -20,7 +20,6 @@
 #ifndef M88RS2000_H
 #define M88RS2000_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/mb86a20s.c b/drivers/media/dvb-frontends/mb86a20s.c
index 41325328a22e..fe79358b035e 100644
--- a/drivers/media/dvb-frontends/mb86a20s.c
+++ b/drivers/media/dvb-frontends/mb86a20s.c
@@ -71,25 +71,27 @@ static struct regdata mb86a20s_init1[] = {
 };
 
 static struct regdata mb86a20s_init2[] = {
-	{ 0x28, 0x22 }, { 0x29, 0x00 }, { 0x2a, 0x1f }, { 0x2b, 0xf0 },
+	{ 0x50, 0xd1 }, { 0x51, 0x22 },
+	{ 0x39, 0x01 },
+	{ 0x71, 0x00 },
 	{ 0x3b, 0x21 },
-	{ 0x3c, 0x38 },
+	{ 0x3c, 0x3a },
 	{ 0x01, 0x0d },
-	{ 0x04, 0x08 }, { 0x05, 0x03 },
+	{ 0x04, 0x08 }, { 0x05, 0x05 },
 	{ 0x04, 0x0e }, { 0x05, 0x00 },
-	{ 0x04, 0x0f }, { 0x05, 0x37 },
-	{ 0x04, 0x0b }, { 0x05, 0x78 },
+	{ 0x04, 0x0f }, { 0x05, 0x14 },
+	{ 0x04, 0x0b }, { 0x05, 0x8c },
 	{ 0x04, 0x00 }, { 0x05, 0x00 },
-	{ 0x04, 0x01 }, { 0x05, 0x1e },
-	{ 0x04, 0x02 }, { 0x05, 0x07 },
-	{ 0x04, 0x03 }, { 0x05, 0xd0 },
+	{ 0x04, 0x01 }, { 0x05, 0x07 },
+	{ 0x04, 0x02 }, { 0x05, 0x0f },
+	{ 0x04, 0x03 }, { 0x05, 0xa0 },
 	{ 0x04, 0x09 }, { 0x05, 0x00 },
 	{ 0x04, 0x0a }, { 0x05, 0xff },
-	{ 0x04, 0x27 }, { 0x05, 0x00 },
+	{ 0x04, 0x27 }, { 0x05, 0x64 },
 	{ 0x04, 0x28 }, { 0x05, 0x00 },
-	{ 0x04, 0x1e }, { 0x05, 0x00 },
-	{ 0x04, 0x29 }, { 0x05, 0x64 },
-	{ 0x04, 0x32 }, { 0x05, 0x02 },
+	{ 0x04, 0x1e }, { 0x05, 0xff },
+	{ 0x04, 0x29 }, { 0x05, 0x0a },
+	{ 0x04, 0x32 }, { 0x05, 0x0a },
 	{ 0x04, 0x14 }, { 0x05, 0x02 },
 	{ 0x04, 0x04 }, { 0x05, 0x00 },
 	{ 0x04, 0x05 }, { 0x05, 0x22 },
@@ -97,8 +99,6 @@ static struct regdata mb86a20s_init2[] = {
 	{ 0x04, 0x07 }, { 0x05, 0xd8 },
 	{ 0x04, 0x12 }, { 0x05, 0x00 },
 	{ 0x04, 0x13 }, { 0x05, 0xff },
-	{ 0x04, 0x15 }, { 0x05, 0x4e },
-	{ 0x04, 0x16 }, { 0x05, 0x20 },
 
 	/*
 	 * On this demod, when the bit count reaches the count below,
@@ -152,42 +152,36 @@ static struct regdata mb86a20s_init2[] = {
 	{ 0x50, 0x51 }, { 0x51, 0x04 },		/* MER symbol 4 */
 	{ 0x45, 0x04 },				/* CN symbol 4 */
 	{ 0x48, 0x04 },				/* CN manual mode */
-
+	{ 0x50, 0xd5 }, { 0x51, 0x01 },
 	{ 0x50, 0xd6 }, { 0x51, 0x1f },
 	{ 0x50, 0xd2 }, { 0x51, 0x03 },
-	{ 0x50, 0xd7 }, { 0x51, 0xbf },
-	{ 0x28, 0x74 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0xff },
-	{ 0x28, 0x46 }, { 0x29, 0x00 }, { 0x2a, 0x1a }, { 0x2b, 0x0c },
-
-	{ 0x04, 0x40 }, { 0x05, 0x00 },
-	{ 0x28, 0x00 }, { 0x2b, 0x08 },
-	{ 0x28, 0x05 }, { 0x2b, 0x00 },
+	{ 0x50, 0xd7 }, { 0x51, 0x3f },
 	{ 0x1c, 0x01 },
-	{ 0x28, 0x06 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x1f },
-	{ 0x28, 0x07 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x18 },
-	{ 0x28, 0x08 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x12 },
-	{ 0x28, 0x09 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x30 },
-	{ 0x28, 0x0a }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x37 },
-	{ 0x28, 0x0b }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x02 },
-	{ 0x28, 0x0c }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x09 },
-	{ 0x28, 0x0d }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x06 },
-	{ 0x28, 0x0e }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x7b },
-	{ 0x28, 0x0f }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x76 },
-	{ 0x28, 0x10 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x7d },
-	{ 0x28, 0x11 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x08 },
-	{ 0x28, 0x12 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x0b },
-	{ 0x28, 0x13 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x00 },
-	{ 0x28, 0x14 }, { 0x29, 0x00 }, { 0x2a, 0x01 }, { 0x2b, 0xf2 },
-	{ 0x28, 0x15 }, { 0x29, 0x00 }, { 0x2a, 0x01 }, { 0x2b, 0xf3 },
-	{ 0x28, 0x16 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x05 },
-	{ 0x28, 0x17 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x16 },
-	{ 0x28, 0x18 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x0f },
-	{ 0x28, 0x19 }, { 0x29, 0x00 }, { 0x2a, 0x07 }, { 0x2b, 0xef },
-	{ 0x28, 0x1a }, { 0x29, 0x00 }, { 0x2a, 0x07 }, { 0x2b, 0xd8 },
-	{ 0x28, 0x1b }, { 0x29, 0x00 }, { 0x2a, 0x07 }, { 0x2b, 0xf1 },
-	{ 0x28, 0x1c }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x3d },
-	{ 0x28, 0x1d }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x94 },
-	{ 0x28, 0x1e }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0xba },
+	{ 0x28, 0x06 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x03 },
+	{ 0x28, 0x07 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x0d },
+	{ 0x28, 0x08 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x02 },
+	{ 0x28, 0x09 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x01 },
+	{ 0x28, 0x0a }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x21 },
+	{ 0x28, 0x0b }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x29 },
+	{ 0x28, 0x0c }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x16 },
+	{ 0x28, 0x0d }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x31 },
+	{ 0x28, 0x0e }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x0e },
+	{ 0x28, 0x0f }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x4e },
+	{ 0x28, 0x10 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x46 },
+	{ 0x28, 0x11 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x0f },
+	{ 0x28, 0x12 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x56 },
+	{ 0x28, 0x13 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x35 },
+	{ 0x28, 0x14 }, { 0x29, 0x00 }, { 0x2a, 0x01 }, { 0x2b, 0xbe },
+	{ 0x28, 0x15 }, { 0x29, 0x00 }, { 0x2a, 0x01 }, { 0x2b, 0x84 },
+	{ 0x28, 0x16 }, { 0x29, 0x00 }, { 0x2a, 0x03 }, { 0x2b, 0xee },
+	{ 0x28, 0x17 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x98 },
+	{ 0x28, 0x18 }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x9f },
+	{ 0x28, 0x19 }, { 0x29, 0x00 }, { 0x2a, 0x07 }, { 0x2b, 0xb2 },
+	{ 0x28, 0x1a }, { 0x29, 0x00 }, { 0x2a, 0x06 }, { 0x2b, 0xc2 },
+	{ 0x28, 0x1b }, { 0x29, 0x00 }, { 0x2a, 0x07 }, { 0x2b, 0x4a },
+	{ 0x28, 0x1c }, { 0x29, 0x00 }, { 0x2a, 0x01 }, { 0x2b, 0xbc },
+	{ 0x28, 0x1d }, { 0x29, 0x00 }, { 0x2a, 0x04 }, { 0x2b, 0xba },
+	{ 0x28, 0x1e }, { 0x29, 0x00 }, { 0x2a, 0x06 }, { 0x2b, 0x14 },
 	{ 0x50, 0x1e }, { 0x51, 0x5d },
 	{ 0x50, 0x22 }, { 0x51, 0x00 },
 	{ 0x50, 0x23 }, { 0x51, 0xc8 },
@@ -196,9 +190,7 @@ static struct regdata mb86a20s_init2[] = {
 	{ 0x50, 0x26 }, { 0x51, 0x00 },
 	{ 0x50, 0x27 }, { 0x51, 0xc3 },
 	{ 0x50, 0x39 }, { 0x51, 0x02 },
-	{ 0xec, 0x0f },
-	{ 0xeb, 0x1f },
-	{ 0x28, 0x6a }, { 0x29, 0x00 }, { 0x2a, 0x00 }, { 0x2b, 0x00 },
+	{ 0x50, 0xd5 }, { 0x51, 0x01 },
 	{ 0xd0, 0x00 },
 };
 
@@ -318,7 +310,11 @@ static int mb86a20s_read_status(struct dvb_frontend *fe, enum fe_status *status)
 	if (val >= 7)
 		*status |= FE_HAS_SYNC;
 
-	if (val >= 8)				/* Maybe 9? */
+	/*
+	 * Actually, on state S8, it starts receiving TS, but the TS
+	 * output is only on normal state after the transition to S9.
+	 */
+	if (val >= 9)
 		*status |= FE_HAS_LOCK;
 
 	dev_dbg(&state->i2c->dev, "%s: Status = 0x%02x (state = %d)\n",
@@ -2058,6 +2054,11 @@ static void mb86a20s_release(struct dvb_frontend *fe)
 	kfree(state);
 }
 
+static int mb86a20s_get_frontend_algo(struct dvb_frontend *fe)
+{
+        return DVBFE_ALGO_HW;
+}
+
 static struct dvb_frontend_ops mb86a20s_ops;
 
 struct dvb_frontend *mb86a20s_attach(const struct mb86a20s_config *config,
@@ -2130,6 +2131,7 @@ static struct dvb_frontend_ops mb86a20s_ops = {
 	.read_status = mb86a20s_read_status_and_stats,
 	.read_signal_strength = mb86a20s_read_signal_strength_from_cache,
 	.tune = mb86a20s_tune,
+	.get_frontend_algo = mb86a20s_get_frontend_algo,
 };
 
 MODULE_DESCRIPTION("DVB Frontend module for Fujitsu mb86A20s hardware");
diff --git a/drivers/media/dvb-frontends/mb86a20s.h b/drivers/media/dvb-frontends/mb86a20s.h
index a113282d6956..dfb02db2126c 100644
--- a/drivers/media/dvb-frontends/mb86a20s.h
+++ b/drivers/media/dvb-frontends/mb86a20s.h
@@ -16,7 +16,6 @@
 #ifndef MB86A20S_H
 #define MB86A20S_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 /**
diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c
index 6e22af36b637..e038e886731b 100644
--- a/drivers/media/dvb-frontends/rtl2832_sdr.c
+++ b/drivers/media/dvb-frontends/rtl2832_sdr.c
@@ -392,7 +392,6 @@ static int rtl2832_sdr_alloc_urbs(struct rtl2832_sdr_dev *dev)
 		dev_dbg(&pdev->dev, "alloc urb=%d\n", i);
 		dev->urb_list[i] = usb_alloc_urb(0, GFP_ATOMIC);
 		if (!dev->urb_list[i]) {
-			dev_dbg(&pdev->dev, "failed\n");
 			for (j = 0; j < i; j++)
 				usb_free_urb(dev->urb_list[j]);
 			return -ENOMEM;
diff --git a/drivers/media/dvb-frontends/s5h1409.h b/drivers/media/dvb-frontends/s5h1409.h
index f58b9ca5557a..b38557c451b9 100644
--- a/drivers/media/dvb-frontends/s5h1409.h
+++ b/drivers/media/dvb-frontends/s5h1409.h
@@ -22,7 +22,6 @@
 #ifndef __S5H1409_H__
 #define __S5H1409_H__
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct s5h1409_config {
diff --git a/drivers/media/dvb-frontends/s5h1411.h b/drivers/media/dvb-frontends/s5h1411.h
index f3a87f7ec360..791bab0e16e9 100644
--- a/drivers/media/dvb-frontends/s5h1411.h
+++ b/drivers/media/dvb-frontends/s5h1411.h
@@ -22,7 +22,6 @@
 #ifndef __S5H1411_H__
 #define __S5H1411_H__
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 #define S5H1411_I2C_TOP_ADDR (0x32 >> 1)
diff --git a/drivers/media/dvb-frontends/s5h1432.h b/drivers/media/dvb-frontends/s5h1432.h
index f490c5ee5801..b81c9bd4e422 100644
--- a/drivers/media/dvb-frontends/s5h1432.h
+++ b/drivers/media/dvb-frontends/s5h1432.h
@@ -22,7 +22,6 @@
 #ifndef __S5H1432_H__
 #define __S5H1432_H__
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 #define S5H1432_I2C_TOP_ADDR (0x02 >> 1)
diff --git a/drivers/media/dvb-frontends/s921.h b/drivers/media/dvb-frontends/s921.h
index f5b722d8081b..a47ed894d4ae 100644
--- a/drivers/media/dvb-frontends/s921.h
+++ b/drivers/media/dvb-frontends/s921.h
@@ -17,7 +17,6 @@
 #ifndef S921_H
 #define S921_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct s921_config {
diff --git a/drivers/media/dvb-frontends/si2165.c b/drivers/media/dvb-frontends/si2165.c
index 8bf716a8ea58..78669ea68c61 100644
--- a/drivers/media/dvb-frontends/si2165.c
+++ b/drivers/media/dvb-frontends/si2165.c
@@ -25,6 +25,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/firmware.h>
+#include <linux/regmap.h>
 
 #include "dvb_frontend.h"
 #include "dvb_math.h"
@@ -40,7 +41,9 @@
  */
 
 struct si2165_state {
-	struct i2c_adapter *i2c;
+	struct i2c_client *client;
+
+	struct regmap *regmap;
 
 	struct dvb_frontend fe;
 
@@ -108,61 +111,27 @@ static int si2165_write(struct si2165_state *state, const u16 reg,
 		       const u8 *src, const int count)
 {
 	int ret;
-	struct i2c_msg msg;
-	u8 buf[2 + 4]; /* write a maximum of 4 bytes of data */
-
-	if (count + 2 > sizeof(buf)) {
-		dev_warn(&state->i2c->dev,
-			  "%s: i2c wr reg=%04x: count=%d is too big!\n",
-			  KBUILD_MODNAME, reg, count);
-		return -EINVAL;
-	}
-	buf[0] = reg >> 8;
-	buf[1] = reg & 0xff;
-	memcpy(buf + 2, src, count);
-
-	msg.addr = state->config.i2c_addr;
-	msg.flags = 0;
-	msg.buf = buf;
-	msg.len = count + 2;
 
 	if (debug & DEBUG_I2C_WRITE)
 		deb_i2c_write("reg: 0x%04x, data: %*ph\n", reg, count, src);
 
-	ret = i2c_transfer(state->i2c, &msg, 1);
+	ret = regmap_bulk_write(state->regmap, reg, src, count);
 
-	if (ret != 1) {
-		dev_err(&state->i2c->dev, "%s: ret == %d\n", __func__, ret);
-		if (ret < 0)
-			return ret;
-		else
-			return -EREMOTEIO;
-	}
+	if (ret)
+		dev_err(&state->client->dev, "%s: ret == %d\n", __func__, ret);
 
-	return 0;
+	return ret;
 }
 
 static int si2165_read(struct si2165_state *state,
 		       const u16 reg, u8 *val, const int count)
 {
-	int ret;
-	u8 reg_buf[] = { reg >> 8, reg & 0xff };
-	struct i2c_msg msg[] = {
-		{ .addr = state->config.i2c_addr,
-		  .flags = 0, .buf = reg_buf, .len = 2 },
-		{ .addr = state->config.i2c_addr,
-		  .flags = I2C_M_RD, .buf = val, .len = count },
-	};
-
-	ret = i2c_transfer(state->i2c, msg, 2);
+	int ret = regmap_bulk_read(state->regmap, reg, val, count);
 
-	if (ret != 2) {
-		dev_err(&state->i2c->dev, "%s: error (addr %02x reg %04x error (ret == %i)\n",
+	if (ret) {
+		dev_err(&state->client->dev, "%s: error (addr %02x reg %04x error (ret == %i)\n",
 			__func__, state->config.i2c_addr, reg, ret);
-		if (ret < 0)
-			return ret;
-		else
-			return -EREMOTEIO;
+		return ret;
 	}
 
 	if (debug & DEBUG_I2C_READ)
@@ -174,9 +143,9 @@ static int si2165_read(struct si2165_state *state,
 static int si2165_readreg8(struct si2165_state *state,
 		       const u16 reg, u8 *val)
 {
-	int ret;
-
-	ret = si2165_read(state, reg, val, 1);
+	unsigned int val_tmp;
+	int ret = regmap_read(state->regmap, reg, &val_tmp);
+	*val = (u8)val_tmp;
 	deb_readreg("R(0x%04x)=0x%02x\n", reg, *val);
 	return ret;
 }
@@ -194,7 +163,7 @@ static int si2165_readreg16(struct si2165_state *state,
 
 static int si2165_writereg8(struct si2165_state *state, const u16 reg, u8 val)
 {
-	return si2165_write(state, reg, &val, 1);
+	return regmap_write(state->regmap, reg, val);
 }
 
 static int si2165_writereg16(struct si2165_state *state, const u16 reg, u16 val)
@@ -345,7 +314,7 @@ static int si2165_wait_init_done(struct si2165_state *state)
 			return 0;
 		usleep_range(1000, 50000);
 	}
-	dev_err(&state->i2c->dev, "%s: init_done was not set\n",
+	dev_err(&state->client->dev, "%s: init_done was not set\n",
 		KBUILD_MODNAME);
 	return ret;
 }
@@ -374,14 +343,14 @@ static int si2165_upload_firmware_block(struct si2165_state *state,
 		wordcount = data[offset];
 		if (wordcount < 1 || data[offset+1] ||
 		    data[offset+2] || data[offset+3]) {
-			dev_warn(&state->i2c->dev,
+			dev_warn(&state->client->dev,
 				 "%s: bad fw data[0..3] = %*ph\n",
 				KBUILD_MODNAME, 4, data);
 			return -EINVAL;
 		}
 
 		if (offset + 8 + wordcount * 4 > len) {
-			dev_warn(&state->i2c->dev,
+			dev_warn(&state->client->dev,
 				 "%s: len is too small for block len=%d, wordcount=%d\n",
 				KBUILD_MODNAME, len, wordcount);
 			return -EINVAL;
@@ -444,15 +413,15 @@ static int si2165_upload_firmware(struct si2165_state *state)
 		fw_file = SI2165_FIRMWARE_REV_D;
 		break;
 	default:
-		dev_info(&state->i2c->dev, "%s: no firmware file for revision=%d\n",
+		dev_info(&state->client->dev, "%s: no firmware file for revision=%d\n",
 			KBUILD_MODNAME, state->chip_revcode);
 		return 0;
 	}
 
 	/* request the firmware, this will block and timeout */
-	ret = request_firmware(&fw, fw_file, state->i2c->dev.parent);
+	ret = request_firmware(&fw, fw_file, &state->client->dev);
 	if (ret) {
-		dev_warn(&state->i2c->dev, "%s: firmware file '%s' not found\n",
+		dev_warn(&state->client->dev, "%s: firmware file '%s' not found\n",
 				KBUILD_MODNAME, fw_file);
 		goto error;
 	}
@@ -460,11 +429,11 @@ static int si2165_upload_firmware(struct si2165_state *state)
 	data = fw->data;
 	len = fw->size;
 
-	dev_info(&state->i2c->dev, "%s: downloading firmware from file '%s' size=%d\n",
+	dev_info(&state->client->dev, "%s: downloading firmware from file '%s' size=%d\n",
 			KBUILD_MODNAME, fw_file, len);
 
 	if (len % 4 != 0) {
-		dev_warn(&state->i2c->dev, "%s: firmware size is not multiple of 4\n",
+		dev_warn(&state->client->dev, "%s: firmware size is not multiple of 4\n",
 				KBUILD_MODNAME);
 		ret = -EINVAL;
 		goto error;
@@ -472,14 +441,14 @@ static int si2165_upload_firmware(struct si2165_state *state)
 
 	/* check header (8 bytes) */
 	if (len < 8) {
-		dev_warn(&state->i2c->dev, "%s: firmware header is missing\n",
+		dev_warn(&state->client->dev, "%s: firmware header is missing\n",
 				KBUILD_MODNAME);
 		ret = -EINVAL;
 		goto error;
 	}
 
 	if (data[0] != 1 || data[1] != 0) {
-		dev_warn(&state->i2c->dev, "%s: firmware file version is wrong\n",
+		dev_warn(&state->client->dev, "%s: firmware file version is wrong\n",
 				KBUILD_MODNAME);
 		ret = -EINVAL;
 		goto error;
@@ -517,7 +486,7 @@ static int si2165_upload_firmware(struct si2165_state *state)
 	/* start right after the header */
 	offset = 8;
 
-	dev_info(&state->i2c->dev, "%s: si2165_upload_firmware extracted patch_version=0x%02x, block_count=0x%02x, crc_expected=0x%04x\n",
+	dev_info(&state->client->dev, "%s: si2165_upload_firmware extracted patch_version=0x%02x, block_count=0x%02x, crc_expected=0x%04x\n",
 		KBUILD_MODNAME, patch_version, block_count, crc_expected);
 
 	ret = si2165_upload_firmware_block(state, data, len, &offset, 1);
@@ -536,7 +505,7 @@ static int si2165_upload_firmware(struct si2165_state *state)
 	ret = si2165_upload_firmware_block(state, data, len,
 					   &offset, block_count);
 	if (ret < 0) {
-		dev_err(&state->i2c->dev,
+		dev_err(&state->client->dev,
 			"%s: firmware could not be uploaded\n",
 			KBUILD_MODNAME);
 		goto error;
@@ -548,7 +517,7 @@ static int si2165_upload_firmware(struct si2165_state *state)
 		goto error;
 
 	if (val16 != crc_expected) {
-		dev_err(&state->i2c->dev,
+		dev_err(&state->client->dev,
 			"%s: firmware crc mismatch %04x != %04x\n",
 			KBUILD_MODNAME, val16, crc_expected);
 		ret = -EINVAL;
@@ -560,7 +529,7 @@ static int si2165_upload_firmware(struct si2165_state *state)
 		goto error;
 
 	if (len != offset) {
-		dev_err(&state->i2c->dev,
+		dev_err(&state->client->dev,
 			"%s: firmware len mismatch %04x != %04x\n",
 			KBUILD_MODNAME, len, offset);
 		ret = -EINVAL;
@@ -577,7 +546,7 @@ static int si2165_upload_firmware(struct si2165_state *state)
 	if (ret < 0)
 		goto error;
 
-	dev_info(&state->i2c->dev, "%s: fw load finished\n", KBUILD_MODNAME);
+	dev_info(&state->client->dev, "%s: fw load finished\n", KBUILD_MODNAME);
 
 	ret = 0;
 	state->firmware_loaded = true;
@@ -611,7 +580,7 @@ static int si2165_init(struct dvb_frontend *fe)
 	if (ret < 0)
 		goto error;
 	if (val != state->config.chip_mode) {
-		dev_err(&state->i2c->dev, "%s: could not set chip_mode\n",
+		dev_err(&state->client->dev, "%s: could not set chip_mode\n",
 			KBUILD_MODNAME);
 		return -EINVAL;
 	}
@@ -751,6 +720,9 @@ static int si2165_set_oversamp(struct si2165_state *state, u32 dvb_rate)
 	u64 oversamp;
 	u32 reg_value;
 
+	if (!dvb_rate)
+		return -EINVAL;
+
 	oversamp = si2165_get_fe_clk(state);
 	oversamp <<= 23;
 	do_div(oversamp, dvb_rate);
@@ -769,12 +741,15 @@ static int si2165_set_if_freq_shift(struct si2165_state *state)
 	u32 IF = 0;
 
 	if (!fe->ops.tuner_ops.get_if_frequency) {
-		dev_err(&state->i2c->dev,
+		dev_err(&state->client->dev,
 			"%s: Error: get_if_frequency() not defined at tuner. Can't work without it!\n",
 			KBUILD_MODNAME);
 		return -EINVAL;
 	}
 
+	if (!fe_clk)
+		return -EINVAL;
+
 	fe->ops.tuner_ops.get_if_frequency(fe, &IF);
 	if_freq_shift = IF;
 	if_freq_shift <<= 29;
@@ -1003,14 +978,6 @@ static int si2165_set_frontend(struct dvb_frontend *fe)
 	return 0;
 }
 
-static void si2165_release(struct dvb_frontend *fe)
-{
-	struct si2165_state *state = fe->demodulator_priv;
-
-	dprintk("%s: called\n", __func__);
-	kfree(state);
-}
-
 static struct dvb_frontend_ops si2165_ops = {
 	.info = {
 		.name = "Silicon Labs ",
@@ -1046,67 +1013,83 @@ static struct dvb_frontend_ops si2165_ops = {
 
 	.set_frontend      = si2165_set_frontend,
 	.read_status       = si2165_read_status,
-
-	.release = si2165_release,
 };
 
-struct dvb_frontend *si2165_attach(const struct si2165_config *config,
-				   struct i2c_adapter *i2c)
+static int si2165_probe(struct i2c_client *client,
+		const struct i2c_device_id *id)
 {
 	struct si2165_state *state = NULL;
+	struct si2165_platform_data *pdata = client->dev.platform_data;
 	int n;
-	int io_ret;
+	int ret = 0;
 	u8 val;
 	char rev_char;
 	const char *chip_name;
-
-	if (config == NULL || i2c == NULL)
-		goto error;
+	static const struct regmap_config regmap_config = {
+		.reg_bits = 16,
+		.val_bits = 8,
+		.max_register = 0x08ff,
+	};
 
 	/* allocate memory for the internal state */
 	state = kzalloc(sizeof(struct si2165_state), GFP_KERNEL);
-	if (state == NULL)
+	if (state == NULL) {
+		ret = -ENOMEM;
 		goto error;
+	}
+
+	/* create regmap */
+	state->regmap = devm_regmap_init_i2c(client, &regmap_config);
+	if (IS_ERR(state->regmap)) {
+		ret = PTR_ERR(state->regmap);
+		goto error;
+	}
 
 	/* setup the state */
-	state->i2c = i2c;
-	state->config = *config;
+	state->client = client;
+	state->config.i2c_addr = client->addr;
+	state->config.chip_mode = pdata->chip_mode;
+	state->config.ref_freq_Hz = pdata->ref_freq_Hz;
+	state->config.inversion = pdata->inversion;
 
 	if (state->config.ref_freq_Hz < 4000000
 	    || state->config.ref_freq_Hz > 27000000) {
-		dev_err(&state->i2c->dev, "%s: ref_freq of %d Hz not supported by this driver\n",
+		dev_err(&state->client->dev, "%s: ref_freq of %d Hz not supported by this driver\n",
 			 KBUILD_MODNAME, state->config.ref_freq_Hz);
+		ret = -EINVAL;
 		goto error;
 	}
 
 	/* create dvb_frontend */
 	memcpy(&state->fe.ops, &si2165_ops,
 		sizeof(struct dvb_frontend_ops));
+	state->fe.ops.release = NULL;
 	state->fe.demodulator_priv = state;
+	i2c_set_clientdata(client, state);
 
 	/* powerup */
-	io_ret = si2165_writereg8(state, 0x0000, state->config.chip_mode);
-	if (io_ret < 0)
-		goto error;
+	ret = si2165_writereg8(state, 0x0000, state->config.chip_mode);
+	if (ret < 0)
+		goto nodev_error;
 
-	io_ret = si2165_readreg8(state, 0x0000, &val);
-	if (io_ret < 0)
-		goto error;
+	ret = si2165_readreg8(state, 0x0000, &val);
+	if (ret < 0)
+		goto nodev_error;
 	if (val != state->config.chip_mode)
-		goto error;
+		goto nodev_error;
 
-	io_ret = si2165_readreg8(state, 0x0023, &state->chip_revcode);
-	if (io_ret < 0)
-		goto error;
+	ret = si2165_readreg8(state, 0x0023, &state->chip_revcode);
+	if (ret < 0)
+		goto nodev_error;
 
-	io_ret = si2165_readreg8(state, 0x0118, &state->chip_type);
-	if (io_ret < 0)
-		goto error;
+	ret = si2165_readreg8(state, 0x0118, &state->chip_type);
+	if (ret < 0)
+		goto nodev_error;
 
 	/* powerdown */
-	io_ret = si2165_writereg8(state, 0x0000, SI2165_MODE_OFF);
-	if (io_ret < 0)
-		goto error;
+	ret = si2165_writereg8(state, 0x0000, SI2165_MODE_OFF);
+	if (ret < 0)
+		goto nodev_error;
 
 	if (state->chip_revcode < 26)
 		rev_char = 'A' + state->chip_revcode;
@@ -1124,12 +1107,12 @@ struct dvb_frontend *si2165_attach(const struct si2165_config *config,
 		state->has_dvbc = true;
 		break;
 	default:
-		dev_err(&state->i2c->dev, "%s: Unsupported Silicon Labs chip (type %d, rev %d)\n",
+		dev_err(&state->client->dev, "%s: Unsupported Silicon Labs chip (type %d, rev %d)\n",
 			KBUILD_MODNAME, state->chip_type, state->chip_revcode);
-		goto error;
+		goto nodev_error;
 	}
 
-	dev_info(&state->i2c->dev,
+	dev_info(&state->client->dev,
 		"%s: Detected Silicon Labs %s-%c (type %d, rev %d)\n",
 		KBUILD_MODNAME, chip_name, rev_char, state->chip_type,
 		state->chip_revcode);
@@ -1149,13 +1132,46 @@ struct dvb_frontend *si2165_attach(const struct si2165_config *config,
 			sizeof(state->fe.ops.info.name));
 	}
 
-	return &state->fe;
+	/* return fe pointer */
+	*pdata->fe = &state->fe;
+
+	return 0;
 
+nodev_error:
+	ret = -ENODEV;
 error:
 	kfree(state);
-	return NULL;
+	dev_dbg(&client->dev, "failed=%d\n", ret);
+	return ret;
 }
-EXPORT_SYMBOL(si2165_attach);
+
+static int si2165_remove(struct i2c_client *client)
+{
+	struct si2165_state *state = i2c_get_clientdata(client);
+
+	dev_dbg(&client->dev, "\n");
+
+	kfree(state);
+	return 0;
+}
+
+static const struct i2c_device_id si2165_id_table[] = {
+	{"si2165", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, si2165_id_table);
+
+static struct i2c_driver si2165_driver = {
+	.driver = {
+		.owner	= THIS_MODULE,
+		.name	= "si2165",
+	},
+	.probe		= si2165_probe,
+	.remove		= si2165_remove,
+	.id_table	= si2165_id_table,
+};
+
+module_i2c_driver(si2165_driver);
 
 module_param(debug, int, 0644);
 MODULE_PARM_DESC(debug, "Turn on/off frontend debugging (default:off).");
diff --git a/drivers/media/dvb-frontends/si2165.h b/drivers/media/dvb-frontends/si2165.h
index 8a15d6a9c552..76c2ca7d7edb 100644
--- a/drivers/media/dvb-frontends/si2165.h
+++ b/drivers/media/dvb-frontends/si2165.h
@@ -28,10 +28,15 @@ enum {
 	SI2165_MODE_PLL_XTAL = 0x21
 };
 
-struct si2165_config {
-	/* i2c addr
-	 * possible values: 0x64,0x65,0x66,0x67 */
-	u8 i2c_addr;
+/* I2C addresses
+ * possible values: 0x64,0x65,0x66,0x67
+ */
+struct si2165_platform_data {
+	/*
+	 * frontend
+	 * returned by driver
+	 */
+	struct dvb_frontend **fe;
 
 	/* external clock or XTAL */
 	u8 chip_mode;
@@ -45,18 +50,4 @@ struct si2165_config {
 	bool inversion;
 };
 
-#if IS_REACHABLE(CONFIG_DVB_SI2165)
-struct dvb_frontend *si2165_attach(
-	const struct si2165_config *config,
-	struct i2c_adapter *i2c);
-#else
-static inline struct dvb_frontend *si2165_attach(
-	const struct si2165_config *config,
-	struct i2c_adapter *i2c)
-{
-	pr_warn("%s: driver disabled by Kconfig\n", __func__);
-	return NULL;
-}
-#endif /* CONFIG_DVB_SI2165 */
-
 #endif /* _DVB_SI2165_H */
diff --git a/drivers/media/dvb-frontends/si2165_priv.h b/drivers/media/dvb-frontends/si2165_priv.h
index 2b70cf12cd79..e5932118834b 100644
--- a/drivers/media/dvb-frontends/si2165_priv.h
+++ b/drivers/media/dvb-frontends/si2165_priv.h
@@ -20,4 +20,21 @@
 
 #define SI2165_FIRMWARE_REV_D "dvb-demod-si2165.fw"
 
+struct si2165_config {
+	/* i2c addr
+	 * possible values: 0x64,0x65,0x66,0x67 */
+	u8 i2c_addr;
+
+	/* external clock or XTAL */
+	u8 chip_mode;
+
+	/* frequency of external clock or xtal in Hz
+	 * possible values: 4000000, 16000000, 20000000, 240000000, 27000000
+	 */
+	u32 ref_freq_Hz;
+
+	/* invert the spectrum */
+	bool inversion;
+};
+
 #endif /* _DVB_SI2165_PRIV */
diff --git a/drivers/media/dvb-frontends/si21xx.h b/drivers/media/dvb-frontends/si21xx.h
index ef5f351ca68e..b1be62f1983a 100644
--- a/drivers/media/dvb-frontends/si21xx.h
+++ b/drivers/media/dvb-frontends/si21xx.h
@@ -1,7 +1,6 @@
 #ifndef SI21XX_H
 #define SI21XX_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/sp2.h b/drivers/media/dvb-frontends/sp2.h
index 6cceea022d49..3901cd74b3f7 100644
--- a/drivers/media/dvb-frontends/sp2.h
+++ b/drivers/media/dvb-frontends/sp2.h
@@ -17,7 +17,6 @@
 #ifndef SP2_H
 #define SP2_H
 
-#include <linux/kconfig.h>
 #include "dvb_ca_en50221.h"
 
 /*
diff --git a/drivers/media/dvb-frontends/stb6000.c b/drivers/media/dvb-frontends/stb6000.c
index a0c3c526b132..73347d51f340 100644
--- a/drivers/media/dvb-frontends/stb6000.c
+++ b/drivers/media/dvb-frontends/stb6000.c
@@ -186,7 +186,7 @@ static int stb6000_get_frequency(struct dvb_frontend *fe, u32 *frequency)
 	return 0;
 }
 
-static struct dvb_tuner_ops stb6000_tuner_ops = {
+static const struct dvb_tuner_ops stb6000_tuner_ops = {
 	.info = {
 		.name = "ST STB6000",
 		.frequency_min = 950000,
diff --git a/drivers/media/dvb-frontends/stb6000.h b/drivers/media/dvb-frontends/stb6000.h
index da581b652cb9..78e75dfc317f 100644
--- a/drivers/media/dvb-frontends/stb6000.h
+++ b/drivers/media/dvb-frontends/stb6000.h
@@ -23,7 +23,6 @@
 #ifndef __DVB_STB6000_H__
 #define __DVB_STB6000_H__
 
-#include <linux/kconfig.h>
 #include <linux/i2c.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/stb6100.c b/drivers/media/dvb-frontends/stb6100.c
index b9c2511bf019..5add1182c3ca 100644
--- a/drivers/media/dvb-frontends/stb6100.c
+++ b/drivers/media/dvb-frontends/stb6100.c
@@ -522,7 +522,7 @@ static int stb6100_set_params(struct dvb_frontend *fe)
 	return 0;
 }
 
-static struct dvb_tuner_ops stb6100_ops = {
+static const struct dvb_tuner_ops stb6100_ops = {
 	.info = {
 		.name			= "STB6100 Silicon Tuner",
 		.frequency_min		= 950000,
diff --git a/drivers/media/dvb-frontends/stv0288.h b/drivers/media/dvb-frontends/stv0288.h
index b58603c00c80..803acb917282 100644
--- a/drivers/media/dvb-frontends/stv0288.h
+++ b/drivers/media/dvb-frontends/stv0288.h
@@ -27,7 +27,6 @@
 #ifndef STV0288_H
 #define STV0288_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/stv0367.h b/drivers/media/dvb-frontends/stv0367.h
index 92b3e85fb818..b88166a9716f 100644
--- a/drivers/media/dvb-frontends/stv0367.h
+++ b/drivers/media/dvb-frontends/stv0367.h
@@ -26,7 +26,6 @@
 #ifndef STV0367_H
 #define STV0367_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/stv0900.h b/drivers/media/dvb-frontends/stv0900.h
index c90bf00ea9ce..9ca2da90c7d7 100644
--- a/drivers/media/dvb-frontends/stv0900.h
+++ b/drivers/media/dvb-frontends/stv0900.h
@@ -26,7 +26,6 @@
 #ifndef STV0900_H
 #define STV0900_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/stv6110.c b/drivers/media/dvb-frontends/stv6110.c
index 91c6dcf65d2a..66a5a7f2295c 100644
--- a/drivers/media/dvb-frontends/stv6110.c
+++ b/drivers/media/dvb-frontends/stv6110.c
@@ -382,7 +382,7 @@ static int stv6110_get_bandwidth(struct dvb_frontend *fe, u32 *bandwidth)
 	return 0;
 }
 
-static struct dvb_tuner_ops stv6110_tuner_ops = {
+static const struct dvb_tuner_ops stv6110_tuner_ops = {
 	.info = {
 		.name = "ST STV6110",
 		.frequency_min = 950000,
diff --git a/drivers/media/dvb-frontends/stv6110.h b/drivers/media/dvb-frontends/stv6110.h
index f3c8a5c6b77d..4604f793d954 100644
--- a/drivers/media/dvb-frontends/stv6110.h
+++ b/drivers/media/dvb-frontends/stv6110.h
@@ -25,7 +25,6 @@
 #ifndef __DVB_STV6110_H__
 #define __DVB_STV6110_H__
 
-#include <linux/kconfig.h>
 #include <linux/i2c.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/stv6110x.c b/drivers/media/dvb-frontends/stv6110x.c
index a62c01e454f5..c611ad210b5c 100644
--- a/drivers/media/dvb-frontends/stv6110x.c
+++ b/drivers/media/dvb-frontends/stv6110x.c
@@ -345,7 +345,7 @@ static int stv6110x_release(struct dvb_frontend *fe)
 	return 0;
 }
 
-static struct dvb_tuner_ops stv6110x_ops = {
+static const struct dvb_tuner_ops stv6110x_ops = {
 	.info = {
 		.name		= "STV6110(A) Silicon Tuner",
 		.frequency_min	=  950000,
diff --git a/drivers/media/dvb-frontends/tda10048.h b/drivers/media/dvb-frontends/tda10048.h
index bc77a7311de1..a2cebb0cceba 100644
--- a/drivers/media/dvb-frontends/tda10048.h
+++ b/drivers/media/dvb-frontends/tda10048.h
@@ -22,7 +22,6 @@
 #ifndef TDA10048_H
 #define TDA10048_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 #include <linux/firmware.h>
 
diff --git a/drivers/media/dvb-frontends/tda18271c2dd.c b/drivers/media/dvb-frontends/tda18271c2dd.c
index de0a1c110972..bc247f9b553a 100644
--- a/drivers/media/dvb-frontends/tda18271c2dd.c
+++ b/drivers/media/dvb-frontends/tda18271c2dd.c
@@ -1217,7 +1217,7 @@ static int get_bandwidth(struct dvb_frontend *fe, u32 *bandwidth)
 }
 
 
-static struct dvb_tuner_ops tuner_ops = {
+static const struct dvb_tuner_ops tuner_ops = {
 	.info = {
 		.name = "NXP TDA18271C2D",
 		.frequency_min  =  47125000,
diff --git a/drivers/media/dvb-frontends/tda18271c2dd.h b/drivers/media/dvb-frontends/tda18271c2dd.h
index 7ebd8eaff4eb..e6ccf240f54c 100644
--- a/drivers/media/dvb-frontends/tda18271c2dd.h
+++ b/drivers/media/dvb-frontends/tda18271c2dd.h
@@ -1,8 +1,6 @@
 #ifndef _TDA18271C2DD_H_
 #define _TDA18271C2DD_H_
 
-#include <linux/kconfig.h>
-
 #if IS_REACHABLE(CONFIG_DVB_TDA18271C2DD)
 struct dvb_frontend *tda18271c2dd_attach(struct dvb_frontend *fe,
 					 struct i2c_adapter *i2c, u8 adr);
diff --git a/drivers/media/dvb-frontends/tda665x.c b/drivers/media/dvb-frontends/tda665x.c
index 82f8cc534f33..7ca965987f40 100644
--- a/drivers/media/dvb-frontends/tda665x.c
+++ b/drivers/media/dvb-frontends/tda665x.c
@@ -206,7 +206,7 @@ static int tda665x_release(struct dvb_frontend *fe)
 	return 0;
 }
 
-static struct dvb_tuner_ops tda665x_ops = {
+static const struct dvb_tuner_ops tda665x_ops = {
 	.get_status	= tda665x_get_status,
 	.set_params	= tda665x_set_params,
 	.get_frequency	= tda665x_get_frequency,
diff --git a/drivers/media/dvb-frontends/tda8261.c b/drivers/media/dvb-frontends/tda8261.c
index 3285b1bc4642..e0df93191b9e 100644
--- a/drivers/media/dvb-frontends/tda8261.c
+++ b/drivers/media/dvb-frontends/tda8261.c
@@ -161,7 +161,7 @@ static int tda8261_release(struct dvb_frontend *fe)
 	return 0;
 }
 
-static struct dvb_tuner_ops tda8261_ops = {
+static const struct dvb_tuner_ops tda8261_ops = {
 
 	.info = {
 		.name		= "TDA8261",
diff --git a/drivers/media/dvb-frontends/tda826x.c b/drivers/media/dvb-frontends/tda826x.c
index 04bbcc24de0a..2ec671df1441 100644
--- a/drivers/media/dvb-frontends/tda826x.c
+++ b/drivers/media/dvb-frontends/tda826x.c
@@ -129,7 +129,7 @@ static int tda826x_get_frequency(struct dvb_frontend *fe, u32 *frequency)
 	return 0;
 }
 
-static struct dvb_tuner_ops tda826x_tuner_ops = {
+static const struct dvb_tuner_ops tda826x_tuner_ops = {
 	.info = {
 		.name = "Philips TDA826X",
 		.frequency_min = 950000,
diff --git a/drivers/media/dvb-frontends/ts2020.c b/drivers/media/dvb-frontends/ts2020.c
index 14b410ffe612..a9f6bbea6df3 100644
--- a/drivers/media/dvb-frontends/ts2020.c
+++ b/drivers/media/dvb-frontends/ts2020.c
@@ -496,7 +496,7 @@ static int ts2020_read_signal_strength(struct dvb_frontend *fe,
 	return 0;
 }
 
-static struct dvb_tuner_ops ts2020_tuner_ops = {
+static const struct dvb_tuner_ops ts2020_tuner_ops = {
 	.info = {
 		.name = "TS2020",
 		.frequency_min = 950000,
diff --git a/drivers/media/dvb-frontends/ts2020.h b/drivers/media/dvb-frontends/ts2020.h
index 9220e5cf0d21..facc54f0a6af 100644
--- a/drivers/media/dvb-frontends/ts2020.h
+++ b/drivers/media/dvb-frontends/ts2020.h
@@ -22,7 +22,6 @@
 #ifndef TS2020_H
 #define TS2020_H
 
-#include <linux/kconfig.h>
 #include <linux/dvb/frontend.h>
 
 struct ts2020_config {
diff --git a/drivers/media/dvb-frontends/tua6100.c b/drivers/media/dvb-frontends/tua6100.c
index 029384d1fddd..6da12b9e55eb 100644
--- a/drivers/media/dvb-frontends/tua6100.c
+++ b/drivers/media/dvb-frontends/tua6100.c
@@ -157,7 +157,7 @@ static int tua6100_get_frequency(struct dvb_frontend *fe, u32 *frequency)
 	return 0;
 }
 
-static struct dvb_tuner_ops tua6100_tuner_ops = {
+static const struct dvb_tuner_ops tua6100_tuner_ops = {
 	.info = {
 		.name = "Infineon TUA6100",
 		.frequency_min = 950000,
diff --git a/drivers/media/dvb-frontends/zl10036.c b/drivers/media/dvb-frontends/zl10036.c
index 0903d461b8fa..7ed81315965f 100644
--- a/drivers/media/dvb-frontends/zl10036.c
+++ b/drivers/media/dvb-frontends/zl10036.c
@@ -446,7 +446,7 @@ static int zl10036_init(struct dvb_frontend *fe)
 	return ret;
 }
 
-static struct dvb_tuner_ops zl10036_tuner_ops = {
+static const struct dvb_tuner_ops zl10036_tuner_ops = {
 	.info = {
 		.name = "Zarlink ZL10036",
 		.frequency_min = 950000,
diff --git a/drivers/media/dvb-frontends/zl10036.h b/drivers/media/dvb-frontends/zl10036.h
index 670e76a654ee..c568d8d59de3 100644
--- a/drivers/media/dvb-frontends/zl10036.h
+++ b/drivers/media/dvb-frontends/zl10036.h
@@ -21,7 +21,6 @@
 #ifndef DVB_ZL10036_H
 #define DVB_ZL10036_H
 
-#include <linux/kconfig.h>
 #include <linux/i2c.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/dvb-frontends/zl10039.c b/drivers/media/dvb-frontends/zl10039.c
index ee09ec26c553..f8c271be196c 100644
--- a/drivers/media/dvb-frontends/zl10039.c
+++ b/drivers/media/dvb-frontends/zl10039.c
@@ -255,7 +255,7 @@ static int zl10039_release(struct dvb_frontend *fe)
 	return 0;
 }
 
-static struct dvb_tuner_ops zl10039_ops = {
+static const struct dvb_tuner_ops zl10039_ops = {
 	.release = zl10039_release,
 	.init = zl10039_init,
 	.sleep = zl10039_sleep,
diff --git a/drivers/media/dvb-frontends/zl10039.h b/drivers/media/dvb-frontends/zl10039.h
index 070929444e71..66e708569375 100644
--- a/drivers/media/dvb-frontends/zl10039.h
+++ b/drivers/media/dvb-frontends/zl10039.h
@@ -22,8 +22,6 @@
 #ifndef ZL10039_H
 #define ZL10039_H
 
-#include <linux/kconfig.h>
-
 #if IS_REACHABLE(CONFIG_DVB_ZL10039)
 struct dvb_frontend *zl10039_attach(struct dvb_frontend *fe,
 					u8 i2c_addr,
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index ce9006e10a30..2669b4bad910 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -21,7 +21,7 @@ config VIDEO_IR_I2C
 # Encoder / Decoder module configuration
 #
 
-menu "Encoders, decoders, sensors and other helper chips"
+menu "I2C Encoders, decoders, sensors and other helper chips"
 	visible if !MEDIA_SUBDRV_AUTOSELECT || COMPILE_TEST
 
 comment "Audio decoders, processors and mixers"
@@ -187,7 +187,7 @@ comment "Video decoders"
 
 config VIDEO_ADV7180
 	tristate "Analog Devices ADV7180 decoder"
-	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
+	depends on GPIOLIB && VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
 	---help---
 	  Support for the Analog Devices ADV7180 video decoder.
 
@@ -295,6 +295,13 @@ config VIDEO_ML86V7667
 	  To compile this driver as a module, choose M here: the
 	  module will be called ml86v7667.
 
+config VIDEO_AD5820
+	tristate "AD5820 lens voice coil support"
+	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
+	---help---
+	  This is a driver for the AD5820 camera lens voice coil.
+	  It is used for example in Nokia N900 (RX-51).
+
 config VIDEO_SAA7110
 	tristate "Philips SAA7110 video decoder"
 	depends on VIDEO_V4L2 && I2C
@@ -571,6 +578,13 @@ config VIDEO_MT9M032
 	  This driver supports MT9M032 camera sensors from Aptina, monochrome
 	  models only.
 
+config VIDEO_MT9M111
+	tristate "mt9m111, mt9m112 and mt9m131 support"
+	depends on I2C && VIDEO_V4L2
+	help
+	  This driver supports MT9M111, MT9M112 and MT9M131 cameras from
+	  Micron/Aptina
+
 config VIDEO_MT9P031
 	tristate "Aptina MT9P031 support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
index 94f2c99e890d..92773b2e6225 100644
--- a/drivers/media/i2c/Makefile
+++ b/drivers/media/i2c/Makefile
@@ -19,6 +19,7 @@ obj-$(CONFIG_VIDEO_SAA717X) += saa717x.o
 obj-$(CONFIG_VIDEO_SAA7127) += saa7127.o
 obj-$(CONFIG_VIDEO_SAA7185) += saa7185.o
 obj-$(CONFIG_VIDEO_SAA6752HS) += saa6752hs.o
+obj-$(CONFIG_VIDEO_AD5820)  += ad5820.o
 obj-$(CONFIG_VIDEO_ADV7170) += adv7170.o
 obj-$(CONFIG_VIDEO_ADV7175) += adv7175.o
 obj-$(CONFIG_VIDEO_ADV7180) += adv7180.o
@@ -59,6 +60,7 @@ obj-$(CONFIG_VIDEO_OV7640) += ov7640.o
 obj-$(CONFIG_VIDEO_OV7670) += ov7670.o
 obj-$(CONFIG_VIDEO_OV9650) += ov9650.o
 obj-$(CONFIG_VIDEO_MT9M032) += mt9m032.o
+obj-$(CONFIG_VIDEO_MT9M111) += mt9m111.o
 obj-$(CONFIG_VIDEO_MT9P031) += mt9p031.o
 obj-$(CONFIG_VIDEO_MT9T001) += mt9t001.o
 obj-$(CONFIG_VIDEO_MT9V011) += mt9v011.o
diff --git a/drivers/media/i2c/ad5820.c b/drivers/media/i2c/ad5820.c
new file mode 100644
index 000000000000..beab2f381b81
--- /dev/null
+++ b/drivers/media/i2c/ad5820.c
@@ -0,0 +1,372 @@
+/*
+ * drivers/media/i2c/ad5820.c
+ *
+ * AD5820 DAC driver for camera voice coil focus.
+ *
+ * Copyright (C) 2008 Nokia Corporation
+ * Copyright (C) 2007 Texas Instruments
+ * Copyright (C) 2016 Pavel Machek <pavel@ucw.cz>
+ *
+ * Contact: Tuukka Toivonen <tuukkat76@gmail.com>
+ *	    Sakari Ailus <sakari.ailus@iki.fi>
+ *
+ * Based on af_d88.c by Texas Instruments.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+
+#define AD5820_NAME		"ad5820"
+
+/* Register definitions */
+#define AD5820_POWER_DOWN		(1 << 15)
+#define AD5820_DAC_SHIFT		4
+#define AD5820_RAMP_MODE_LINEAR		(0 << 3)
+#define AD5820_RAMP_MODE_64_16		(1 << 3)
+
+#define CODE_TO_RAMP_US(s)	((s) == 0 ? 0 : (1 << ((s) - 1)) * 50)
+#define RAMP_US_TO_CODE(c)	fls(((c) + ((c)>>1)) / 50)
+
+#define to_ad5820_device(sd)	container_of(sd, struct ad5820_device, subdev)
+
+struct ad5820_device {
+	struct v4l2_subdev subdev;
+	struct ad5820_platform_data *platform_data;
+	struct regulator *vana;
+
+	struct v4l2_ctrl_handler ctrls;
+	u32 focus_absolute;
+	u32 focus_ramp_time;
+	u32 focus_ramp_mode;
+
+	struct mutex power_lock;
+	int power_count;
+
+	bool standby;
+};
+
+static int ad5820_write(struct ad5820_device *coil, u16 data)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&coil->subdev);
+	struct i2c_msg msg;
+	int r;
+
+	if (!client->adapter)
+		return -ENODEV;
+
+	data = cpu_to_be16(data);
+	msg.addr  = client->addr;
+	msg.flags = 0;
+	msg.len   = 2;
+	msg.buf   = (u8 *)&data;
+
+	r = i2c_transfer(client->adapter, &msg, 1);
+	if (r < 0) {
+		dev_err(&client->dev, "write failed, error %d\n", r);
+		return r;
+	}
+
+	return 0;
+}
+
+/*
+ * Calculate status word and write it to the device based on current
+ * values of V4L2 controls. It is assumed that the stored V4L2 control
+ * values are properly limited and rounded.
+ */
+static int ad5820_update_hw(struct ad5820_device *coil)
+{
+	u16 status;
+
+	status = RAMP_US_TO_CODE(coil->focus_ramp_time);
+	status |= coil->focus_ramp_mode
+		? AD5820_RAMP_MODE_64_16 : AD5820_RAMP_MODE_LINEAR;
+	status |= coil->focus_absolute << AD5820_DAC_SHIFT;
+
+	if (coil->standby)
+		status |= AD5820_POWER_DOWN;
+
+	return ad5820_write(coil, status);
+}
+
+/*
+ * Power handling
+ */
+static int ad5820_power_off(struct ad5820_device *coil, bool standby)
+{
+	int ret = 0, ret2;
+
+	/*
+	 * Go to standby first as real power off my be denied by the hardware
+	 * (single power line control for both coil and sensor).
+	 */
+	if (standby) {
+		coil->standby = true;
+		ret = ad5820_update_hw(coil);
+	}
+
+	ret2 = regulator_disable(coil->vana);
+	if (ret)
+		return ret;
+	return ret2;
+}
+
+static int ad5820_power_on(struct ad5820_device *coil, bool restore)
+{
+	int ret;
+
+	ret = regulator_enable(coil->vana);
+	if (ret < 0)
+		return ret;
+
+	if (restore) {
+		/* Restore the hardware settings. */
+		coil->standby = false;
+		ret = ad5820_update_hw(coil);
+		if (ret)
+			goto fail;
+	}
+	return 0;
+
+fail:
+	coil->standby = true;
+	regulator_disable(coil->vana);
+
+	return ret;
+}
+
+/*
+ * V4L2 controls
+ */
+static int ad5820_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct ad5820_device *coil =
+		container_of(ctrl->handler, struct ad5820_device, ctrls);
+
+	switch (ctrl->id) {
+	case V4L2_CID_FOCUS_ABSOLUTE:
+		coil->focus_absolute = ctrl->val;
+		return ad5820_update_hw(coil);
+	}
+
+	return 0;
+}
+
+static const struct v4l2_ctrl_ops ad5820_ctrl_ops = {
+	.s_ctrl = ad5820_set_ctrl,
+};
+
+
+static int ad5820_init_controls(struct ad5820_device *coil)
+{
+	v4l2_ctrl_handler_init(&coil->ctrls, 1);
+
+	/*
+	 * V4L2_CID_FOCUS_ABSOLUTE
+	 *
+	 * Minimum current is 0 mA, maximum is 100 mA. Thus, 1 code is
+	 * equivalent to 100/1023 = 0.0978 mA. Nevertheless, we do not use [mA]
+	 * for focus position, because it is meaningless for user. Meaningful
+	 * would be to use focus distance or even its inverse, but since the
+	 * driver doesn't have sufficiently knowledge to do the conversion, we
+	 * will just use abstract codes here. In any case, smaller value = focus
+	 * position farther from camera. The default zero value means focus at
+	 * infinity, and also least current consumption.
+	 */
+	v4l2_ctrl_new_std(&coil->ctrls, &ad5820_ctrl_ops,
+			  V4L2_CID_FOCUS_ABSOLUTE, 0, 1023, 1, 0);
+
+	if (coil->ctrls.error)
+		return coil->ctrls.error;
+
+	coil->focus_absolute = 0;
+	coil->focus_ramp_time = 0;
+	coil->focus_ramp_mode = 0;
+
+	coil->subdev.ctrl_handler = &coil->ctrls;
+
+	return 0;
+}
+
+/*
+ * V4L2 subdev operations
+ */
+static int ad5820_registered(struct v4l2_subdev *subdev)
+{
+	struct ad5820_device *coil = to_ad5820_device(subdev);
+
+	return ad5820_init_controls(coil);
+}
+
+static int
+ad5820_set_power(struct v4l2_subdev *subdev, int on)
+{
+	struct ad5820_device *coil = to_ad5820_device(subdev);
+	int ret = 0;
+
+	mutex_lock(&coil->power_lock);
+
+	/*
+	 * If the power count is modified from 0 to != 0 or from != 0 to 0,
+	 * update the power state.
+	 */
+	if (coil->power_count == !on) {
+		ret = on ? ad5820_power_on(coil, true) :
+			ad5820_power_off(coil, true);
+		if (ret < 0)
+			goto done;
+	}
+
+	/* Update the power count. */
+	coil->power_count += on ? 1 : -1;
+	WARN_ON(coil->power_count < 0);
+
+done:
+	mutex_unlock(&coil->power_lock);
+	return ret;
+}
+
+static int ad5820_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	return ad5820_set_power(sd, 1);
+}
+
+static int ad5820_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	return ad5820_set_power(sd, 0);
+}
+
+static const struct v4l2_subdev_core_ops ad5820_core_ops = {
+	.s_power = ad5820_set_power,
+};
+
+static const struct v4l2_subdev_ops ad5820_ops = {
+	.core = &ad5820_core_ops,
+};
+
+static const struct v4l2_subdev_internal_ops ad5820_internal_ops = {
+	.registered = ad5820_registered,
+	.open = ad5820_open,
+	.close = ad5820_close,
+};
+
+/*
+ * I2C driver
+ */
+static int __maybe_unused ad5820_suspend(struct device *dev)
+{
+	struct i2c_client *client = container_of(dev, struct i2c_client, dev);
+	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
+	struct ad5820_device *coil = to_ad5820_device(subdev);
+
+	if (!coil->power_count)
+		return 0;
+
+	return ad5820_power_off(coil, false);
+}
+
+static int __maybe_unused ad5820_resume(struct device *dev)
+{
+	struct i2c_client *client = container_of(dev, struct i2c_client, dev);
+	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
+	struct ad5820_device *coil = to_ad5820_device(subdev);
+
+	if (!coil->power_count)
+		return 0;
+
+	return ad5820_power_on(coil, true);
+}
+
+static int ad5820_probe(struct i2c_client *client,
+			const struct i2c_device_id *devid)
+{
+	struct ad5820_device *coil;
+	int ret;
+
+	coil = devm_kzalloc(&client->dev, sizeof(*coil), GFP_KERNEL);
+	if (!coil)
+		return -ENOMEM;
+
+	coil->vana = devm_regulator_get(&client->dev, "VANA");
+	if (IS_ERR(coil->vana)) {
+		ret = PTR_ERR(coil->vana);
+		if (ret != -EPROBE_DEFER)
+			dev_err(&client->dev, "could not get regulator for vana\n");
+		return ret;
+	}
+
+	mutex_init(&coil->power_lock);
+
+	v4l2_i2c_subdev_init(&coil->subdev, client, &ad5820_ops);
+	coil->subdev.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	coil->subdev.internal_ops = &ad5820_internal_ops;
+	strcpy(coil->subdev.name, "ad5820 focus");
+
+	ret = media_entity_pads_init(&coil->subdev.entity, 0, NULL);
+	if (ret < 0)
+		goto cleanup2;
+
+	ret = v4l2_async_register_subdev(&coil->subdev);
+	if (ret < 0)
+		goto cleanup;
+
+	return ret;
+
+cleanup2:
+	mutex_destroy(&coil->power_lock);
+cleanup:
+	media_entity_cleanup(&coil->subdev.entity);
+	return ret;
+}
+
+static int __exit ad5820_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
+	struct ad5820_device *coil = to_ad5820_device(subdev);
+
+	v4l2_device_unregister_subdev(&coil->subdev);
+	v4l2_ctrl_handler_free(&coil->ctrls);
+	media_entity_cleanup(&coil->subdev.entity);
+	mutex_destroy(&coil->power_lock);
+	return 0;
+}
+
+static const struct i2c_device_id ad5820_id_table[] = {
+	{ AD5820_NAME, 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, ad5820_id_table);
+
+static SIMPLE_DEV_PM_OPS(ad5820_pm, ad5820_suspend, ad5820_resume);
+
+static struct i2c_driver ad5820_i2c_driver = {
+	.driver		= {
+		.name	= AD5820_NAME,
+		.pm	= &ad5820_pm,
+	},
+	.probe		= ad5820_probe,
+	.remove		= __exit_p(ad5820_remove),
+	.id_table	= ad5820_id_table,
+};
+
+module_i2c_driver(ad5820_i2c_driver);
+
+MODULE_AUTHOR("Tuukka Toivonen");
+MODULE_DESCRIPTION("AD5820 camera lens driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/ad9389b.c b/drivers/media/i2c/ad9389b.c
index 0462f461e679..50f354144ee7 100644
--- a/drivers/media/i2c/ad9389b.c
+++ b/drivers/media/i2c/ad9389b.c
@@ -98,7 +98,6 @@ struct ad9389b_state {
 	struct ad9389b_state_edid edid;
 	/* Running counter of the number of detected EDIDs (for debugging) */
 	unsigned edid_detect_counter;
-	struct workqueue_struct *work_queue;
 	struct delayed_work edid_handler; /* work entry */
 };
 
@@ -843,8 +842,7 @@ static void ad9389b_edid_handler(struct work_struct *work)
 			v4l2_dbg(1, debug, sd, "%s: edid read failed\n", __func__);
 			ad9389b_s_power(sd, false);
 			ad9389b_s_power(sd, true);
-			queue_delayed_work(state->work_queue,
-					   &state->edid_handler, EDID_DELAY);
+			schedule_delayed_work(&state->edid_handler, EDID_DELAY);
 			return;
 		}
 	}
@@ -933,8 +931,7 @@ static void ad9389b_update_monitor_present_status(struct v4l2_subdev *sd)
 		ad9389b_setup(sd);
 		ad9389b_notify_monitor_detect(sd);
 		state->edid.read_retries = EDID_MAX_RETRIES;
-		queue_delayed_work(state->work_queue,
-				   &state->edid_handler, EDID_DELAY);
+		schedule_delayed_work(&state->edid_handler, EDID_DELAY);
 	} else if (!(status & MASK_AD9389B_HPD_DETECT)) {
 		v4l2_dbg(1, debug, sd, "%s: hotplug not detected\n", __func__);
 		state->have_monitor = false;
@@ -1065,8 +1062,7 @@ static bool ad9389b_check_edid_status(struct v4l2_subdev *sd)
 		ad9389b_wr(sd, 0xc9, 0xf);
 		ad9389b_wr(sd, 0xc4, state->edid.segments);
 		state->edid.read_retries = EDID_MAX_RETRIES;
-		queue_delayed_work(state->work_queue,
-				   &state->edid_handler, EDID_DELAY);
+		schedule_delayed_work(&state->edid_handler, EDID_DELAY);
 		return false;
 	}
 
@@ -1170,13 +1166,6 @@ static int ad9389b_probe(struct i2c_client *client, const struct i2c_device_id *
 		goto err_entity;
 	}
 
-	state->work_queue = create_singlethread_workqueue(sd->name);
-	if (state->work_queue == NULL) {
-		v4l2_err(sd, "could not create workqueue\n");
-		err = -ENOMEM;
-		goto err_unreg;
-	}
-
 	INIT_DELAYED_WORK(&state->edid_handler, ad9389b_edid_handler);
 	state->dv_timings = dv1080p60;
 
@@ -1187,8 +1176,6 @@ static int ad9389b_probe(struct i2c_client *client, const struct i2c_device_id *
 		  client->addr << 1, client->adapter->name);
 	return 0;
 
-err_unreg:
-	i2c_unregister_device(state->edid_i2c_client);
 err_entity:
 	media_entity_cleanup(&sd->entity);
 err_hdl:
@@ -1211,9 +1198,8 @@ static int ad9389b_remove(struct i2c_client *client)
 	ad9389b_s_stream(sd, false);
 	ad9389b_s_audio_stream(sd, false);
 	ad9389b_init_setup(sd);
-	cancel_delayed_work(&state->edid_handler);
+	cancel_delayed_work_sync(&state->edid_handler);
 	i2c_unregister_device(state->edid_i2c_client);
-	destroy_workqueue(state->work_queue);
 	v4l2_device_unregister_subdev(sd);
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
@@ -1231,7 +1217,6 @@ MODULE_DEVICE_TABLE(i2c, ad9389b_id);
 
 static struct i2c_driver ad9389b_driver = {
 	.driver = {
-		.owner = THIS_MODULE,
 		.name = "ad9389b",
 	},
 	.probe = ad9389b_probe,
diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c
index 95cbc857f36e..cbed2bc29325 100644
--- a/drivers/media/i2c/adv7180.c
+++ b/drivers/media/i2c/adv7180.c
@@ -26,6 +26,7 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 #include <linux/of.h>
+#include <linux/gpio/consumer.h>
 #include <linux/videodev2.h>
 #include <media/v4l2-ioctl.h>
 #include <media/v4l2-event.h>
@@ -56,10 +57,11 @@
 
 #define ADV7182_REG_INPUT_VIDSEL			0x0002
 
+#define ADV7180_REG_OUTPUT_CONTROL			0x0003
 #define ADV7180_REG_EXTENDED_OUTPUT_CONTROL		0x0004
 #define ADV7180_EXTENDED_OUTPUT_CONTROL_NTSCDIS		0xC5
 
-#define ADV7180_REG_AUTODETECT_ENABLE			0x07
+#define ADV7180_REG_AUTODETECT_ENABLE			0x0007
 #define ADV7180_AUTODETECT_DEFAULT			0x7f
 /* Contrast */
 #define ADV7180_REG_CON		0x0008	/*Unsigned */
@@ -100,6 +102,20 @@
 #define ADV7180_REG_IDENT 0x0011
 #define ADV7180_ID_7180 0x18
 
+#define ADV7180_REG_STATUS3		0x0013
+#define ADV7180_REG_ANALOG_CLAMP_CTL	0x0014
+#define ADV7180_REG_SHAP_FILTER_CTL_1	0x0017
+#define ADV7180_REG_CTRL_2		0x001d
+#define ADV7180_REG_VSYNC_FIELD_CTL_1	0x0031
+#define ADV7180_REG_MANUAL_WIN_CTL_1	0x003d
+#define ADV7180_REG_MANUAL_WIN_CTL_2	0x003e
+#define ADV7180_REG_MANUAL_WIN_CTL_3	0x003f
+#define ADV7180_REG_LOCK_CNT		0x0051
+#define ADV7180_REG_CVBS_TRIM		0x0052
+#define ADV7180_REG_CLAMP_ADJ		0x005a
+#define ADV7180_REG_RES_CIR		0x005f
+#define ADV7180_REG_DIFF_MODE		0x0060
+
 #define ADV7180_REG_ICONF1		0x2040
 #define ADV7180_ICONF1_ACTIVE_LOW	0x01
 #define ADV7180_ICONF1_PSYNC_ONLY	0x10
@@ -129,9 +145,15 @@
 #define ADV7180_REG_VPP_SLAVE_ADDR	0xFD
 #define ADV7180_REG_CSI_SLAVE_ADDR	0xFE
 
-#define ADV7180_REG_FLCONTROL 0x40e0
+#define ADV7180_REG_ACE_CTRL1		0x4080
+#define ADV7180_REG_ACE_CTRL5		0x4084
+#define ADV7180_REG_FLCONTROL		0x40e0
 #define ADV7180_FLCONTROL_FL_ENABLE 0x1
 
+#define ADV7180_REG_RST_CLAMP	0x809c
+#define ADV7180_REG_AGC_ADJ1	0x80b6
+#define ADV7180_REG_AGC_ADJ2	0x80c0
+
 #define ADV7180_CSI_REG_PWRDN	0x00
 #define ADV7180_CSI_PWRDN	0x80
 
@@ -192,6 +214,7 @@ struct adv7180_state {
 	struct media_pad	pad;
 	struct mutex		mutex; /* mutual excl. when accessing chip */
 	int			irq;
+	struct gpio_desc	*pwdn_gpio;
 	v4l2_std_id		curr_norm;
 	bool			powered;
 	bool			streaming;
@@ -442,6 +465,19 @@ static int adv7180_g_std(struct v4l2_subdev *sd, v4l2_std_id *norm)
 	return 0;
 }
 
+static void adv7180_set_power_pin(struct adv7180_state *state, bool on)
+{
+	if (!state->pwdn_gpio)
+		return;
+
+	if (on) {
+		gpiod_set_value_cansleep(state->pwdn_gpio, 0);
+		usleep_range(5000, 10000);
+	} else {
+		gpiod_set_value_cansleep(state->pwdn_gpio, 1);
+	}
+}
+
 static int adv7180_set_power(struct adv7180_state *state, bool on)
 {
 	u8 val;
@@ -597,7 +633,7 @@ static int adv7180_enum_mbus_code(struct v4l2_subdev *sd,
 	if (code->index != 0)
 		return -EINVAL;
 
-	code->code = MEDIA_BUS_FMT_YUYV8_2X8;
+	code->code = MEDIA_BUS_FMT_UYVY8_2X8;
 
 	return 0;
 }
@@ -607,7 +643,7 @@ static int adv7180_mbus_fmt(struct v4l2_subdev *sd,
 {
 	struct adv7180_state *state = to_state(sd);
 
-	fmt->code = MEDIA_BUS_FMT_YUYV8_2X8;
+	fmt->code = MEDIA_BUS_FMT_UYVY8_2X8;
 	fmt->colorspace = V4L2_COLORSPACE_SMPTE170M;
 	fmt->width = 720;
 	fmt->height = state->curr_norm & V4L2_STD_525_60 ? 480 : 576;
@@ -675,6 +711,7 @@ static int adv7180_set_pad_format(struct v4l2_subdev *sd,
 {
 	struct adv7180_state *state = to_state(sd);
 	struct v4l2_mbus_framefmt *framefmt;
+	int ret;
 
 	switch (format->format.field) {
 	case V4L2_FIELD_NONE:
@@ -686,8 +723,9 @@ static int adv7180_set_pad_format(struct v4l2_subdev *sd,
 		break;
 	}
 
+	ret = adv7180_mbus_fmt(sd,  &format->format);
+
 	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
-		framefmt = &format->format;
 		if (state->field != format->format.field) {
 			state->field = format->format.field;
 			adv7180_set_power(state, false);
@@ -699,7 +737,7 @@ static int adv7180_set_pad_format(struct v4l2_subdev *sd,
 		*framefmt = format->format;
 	}
 
-	return adv7180_mbus_fmt(sd, framefmt);
+	return ret;
 }
 
 static int adv7180_g_mbus_config(struct v4l2_subdev *sd,
@@ -725,16 +763,16 @@ static int adv7180_g_mbus_config(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int adv7180_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *cropcap)
+static int adv7180_g_pixelaspect(struct v4l2_subdev *sd, struct v4l2_fract *aspect)
 {
 	struct adv7180_state *state = to_state(sd);
 
 	if (state->curr_norm & V4L2_STD_525_60) {
-		cropcap->pixelaspect.numerator = 11;
-		cropcap->pixelaspect.denominator = 10;
+		aspect->numerator = 11;
+		aspect->denominator = 10;
 	} else {
-		cropcap->pixelaspect.numerator = 54;
-		cropcap->pixelaspect.denominator = 59;
+		aspect->numerator = 54;
+		aspect->denominator = 59;
 	}
 
 	return 0;
@@ -787,7 +825,7 @@ static const struct v4l2_subdev_video_ops adv7180_video_ops = {
 	.g_input_status = adv7180_g_input_status,
 	.s_routing = adv7180_s_routing,
 	.g_mbus_config = adv7180_g_mbus_config,
-	.cropcap = adv7180_cropcap,
+	.g_pixelaspect = adv7180_g_pixelaspect,
 	.g_tvnorms = adv7180_g_tvnorms,
 	.s_stream = adv7180_s_stream,
 };
@@ -886,16 +924,20 @@ static int adv7182_init(struct adv7180_state *state)
 
 	/* ADI required writes */
 	if (state->chip_info->flags & ADV7180_FLAG_MIPI_CSI2) {
-		adv7180_write(state, 0x0003, 0x4e);
-		adv7180_write(state, 0x0004, 0x57);
-		adv7180_write(state, 0x001d, 0xc0);
+		adv7180_write(state, ADV7180_REG_OUTPUT_CONTROL, 0x4e);
+		adv7180_write(state, ADV7180_REG_EXTENDED_OUTPUT_CONTROL, 0x57);
+		adv7180_write(state, ADV7180_REG_CTRL_2, 0xc0);
 	} else {
 		if (state->chip_info->flags & ADV7180_FLAG_V2)
-			adv7180_write(state, 0x0004, 0x17);
+			adv7180_write(state,
+				      ADV7180_REG_EXTENDED_OUTPUT_CONTROL,
+				      0x17);
 		else
-			adv7180_write(state, 0x0004, 0x07);
-		adv7180_write(state, 0x0003, 0x0c);
-		adv7180_write(state, 0x001d, 0x40);
+			adv7180_write(state,
+				      ADV7180_REG_EXTENDED_OUTPUT_CONTROL,
+				      0x07);
+		adv7180_write(state, ADV7180_REG_OUTPUT_CONTROL, 0x0c);
+		adv7180_write(state, ADV7180_REG_CTRL_2, 0x40);
 	}
 
 	adv7180_write(state, 0x0013, 0x00);
@@ -972,8 +1014,8 @@ static int adv7182_select_input(struct adv7180_state *state, unsigned int input)
 		return ret;
 
 	/* Reset clamp circuitry - ADI recommended writes */
-	adv7180_write(state, 0x809c, 0x00);
-	adv7180_write(state, 0x809c, 0xff);
+	adv7180_write(state, ADV7180_REG_RST_CLAMP, 0x00);
+	adv7180_write(state, ADV7180_REG_RST_CLAMP, 0xff);
 
 	input_type = adv7182_get_input_type(input);
 
@@ -981,10 +1023,10 @@ static int adv7182_select_input(struct adv7180_state *state, unsigned int input)
 	case ADV7182_INPUT_TYPE_CVBS:
 	case ADV7182_INPUT_TYPE_DIFF_CVBS:
 		/* ADI recommends to use the SH1 filter */
-		adv7180_write(state, 0x0017, 0x41);
+		adv7180_write(state, ADV7180_REG_SHAP_FILTER_CTL_1, 0x41);
 		break;
 	default:
-		adv7180_write(state, 0x0017, 0x01);
+		adv7180_write(state, ADV7180_REG_SHAP_FILTER_CTL_1, 0x01);
 		break;
 	}
 
@@ -994,21 +1036,21 @@ static int adv7182_select_input(struct adv7180_state *state, unsigned int input)
 		lbias = adv7182_lbias_settings[input_type];
 
 	for (i = 0; i < ARRAY_SIZE(adv7182_lbias_settings[0]); i++)
-		adv7180_write(state, 0x0052 + i, lbias[i]);
+		adv7180_write(state, ADV7180_REG_CVBS_TRIM + i, lbias[i]);
 
 	if (input_type == ADV7182_INPUT_TYPE_DIFF_CVBS) {
 		/* ADI required writes to make differential CVBS work */
-		adv7180_write(state, 0x005f, 0xa8);
-		adv7180_write(state, 0x005a, 0x90);
-		adv7180_write(state, 0x0060, 0xb0);
-		adv7180_write(state, 0x80b6, 0x08);
-		adv7180_write(state, 0x80c0, 0xa0);
+		adv7180_write(state, ADV7180_REG_RES_CIR, 0xa8);
+		adv7180_write(state, ADV7180_REG_CLAMP_ADJ, 0x90);
+		adv7180_write(state, ADV7180_REG_DIFF_MODE, 0xb0);
+		adv7180_write(state, ADV7180_REG_AGC_ADJ1, 0x08);
+		adv7180_write(state, ADV7180_REG_AGC_ADJ2, 0xa0);
 	} else {
-		adv7180_write(state, 0x005f, 0xf0);
-		adv7180_write(state, 0x005a, 0xd0);
-		adv7180_write(state, 0x0060, 0x10);
-		adv7180_write(state, 0x80b6, 0x9c);
-		adv7180_write(state, 0x80c0, 0x00);
+		adv7180_write(state, ADV7180_REG_RES_CIR, 0xf0);
+		adv7180_write(state, ADV7180_REG_CLAMP_ADJ, 0xd0);
+		adv7180_write(state, ADV7180_REG_DIFF_MODE, 0x10);
+		adv7180_write(state, ADV7180_REG_AGC_ADJ1, 0x9c);
+		adv7180_write(state, ADV7180_REG_AGC_ADJ2, 0x00);
 	}
 
 	return 0;
@@ -1185,6 +1227,8 @@ static int init_device(struct adv7180_state *state)
 
 	mutex_lock(&state->mutex);
 
+	adv7180_set_power_pin(state, true);
+
 	adv7180_write(state, ADV7180_REG_PWR_MAN, ADV7180_PWR_MAN_RES);
 	usleep_range(5000, 10000);
 
@@ -1254,6 +1298,14 @@ static int adv7180_probe(struct i2c_client *client,
 	state->field = V4L2_FIELD_INTERLACED;
 	state->chip_info = (struct adv7180_chip_info *)id->driver_data;
 
+	state->pwdn_gpio = devm_gpiod_get_optional(&client->dev, "powerdown",
+						   GPIOD_OUT_HIGH);
+	if (IS_ERR(state->pwdn_gpio)) {
+		ret = PTR_ERR(state->pwdn_gpio);
+		v4l_err(client, "request for power pin failed: %d\n", ret);
+		return ret;
+	}
+
 	if (state->chip_info->flags & ADV7180_FLAG_MIPI_CSI2) {
 		state->csi_client = i2c_new_dummy(client->adapter,
 				ADV7180_DEFAULT_CSI_I2C_ADDR);
@@ -1345,6 +1397,8 @@ static int adv7180_remove(struct i2c_client *client)
 	if (state->chip_info->flags & ADV7180_FLAG_MIPI_CSI2)
 		i2c_unregister_device(state->csi_client);
 
+	adv7180_set_power_pin(state, false);
+
 	mutex_destroy(&state->mutex);
 
 	return 0;
diff --git a/drivers/media/i2c/adv7183.c b/drivers/media/i2c/adv7183.c
index 2bec737881e9..04eecda74d66 100644
--- a/drivers/media/i2c/adv7183.c
+++ b/drivers/media/i2c/adv7183.c
@@ -644,7 +644,6 @@ MODULE_DEVICE_TABLE(i2c, adv7183_id);
 
 static struct i2c_driver adv7183_driver = {
 	.driver = {
-		.owner  = THIS_MODULE,
 		.name   = "adv7183",
 	},
 	.probe          = adv7183_probe,
diff --git a/drivers/media/i2c/adv7393.c b/drivers/media/i2c/adv7393.c
index 76d987476e35..f19ad4ecd11e 100644
--- a/drivers/media/i2c/adv7393.c
+++ b/drivers/media/i2c/adv7393.c
@@ -456,7 +456,6 @@ MODULE_DEVICE_TABLE(i2c, adv7393_id);
 
 static struct i2c_driver adv7393_driver = {
 	.driver = {
-		.owner	= THIS_MODULE,
 		.name	= "adv7393",
 	},
 	.probe		= adv7393_probe,
diff --git a/drivers/media/i2c/adv7511.c b/drivers/media/i2c/adv7511.c
index 53030d631653..5ba0f21bcfe4 100644
--- a/drivers/media/i2c/adv7511.c
+++ b/drivers/media/i2c/adv7511.c
@@ -1898,6 +1898,7 @@ static int adv7511_probe(struct i2c_client *client, const struct i2c_device_id *
 					       state->i2c_cec_addr >> 1);
 		if (state->i2c_cec == NULL) {
 			v4l2_err(sd, "failed to register cec i2c client\n");
+			err = -ENOMEM;
 			goto err_unreg_edid;
 		}
 		adv7511_wr(sd, 0xe2, 0x00); /* power up cec section */
diff --git a/drivers/media/i2c/ak881x.c b/drivers/media/i2c/ak881x.c
index d9f2b6b76d59..3a795dcb7d8e 100644
--- a/drivers/media/i2c/ak881x.c
+++ b/drivers/media/i2c/ak881x.c
@@ -124,21 +124,27 @@ static int ak881x_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int ak881x_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
+static int ak881x_get_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct ak881x *ak881x = to_ak881x(client);
 
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
-	a->bounds.width			= 720;
-	a->bounds.height		= ak881x->lines;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_OUTPUT;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = 720;
+		sel->r.height = ak881x->lines;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int ak881x_s_std_output(struct v4l2_subdev *sd, v4l2_std_id std)
@@ -207,13 +213,13 @@ static struct v4l2_subdev_core_ops ak881x_subdev_core_ops = {
 };
 
 static struct v4l2_subdev_video_ops ak881x_subdev_video_ops = {
-	.cropcap	= ak881x_cropcap,
 	.s_std_output	= ak881x_s_std_output,
 	.s_stream	= ak881x_s_stream,
 };
 
 static const struct v4l2_subdev_pad_ops ak881x_subdev_pad_ops = {
 	.enum_mbus_code = ak881x_enum_mbus_code,
+	.get_selection	= ak881x_get_selection,
 	.set_fmt	= ak881x_fill_fmt,
 	.get_fmt	= ak881x_fill_fmt,
 };
diff --git a/drivers/media/i2c/cs3308.c b/drivers/media/i2c/cs3308.c
index d28b4f37fe5f..7da5f69cace6 100644
--- a/drivers/media/i2c/cs3308.c
+++ b/drivers/media/i2c/cs3308.c
@@ -127,7 +127,6 @@ MODULE_DEVICE_TABLE(i2c, cs3308_id);
 
 static struct i2c_driver cs3308_driver = {
 	.driver = {
-		.owner  = THIS_MODULE,
 		.name   = "cs3308",
 	},
 	.probe          = cs3308_probe,
diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c
index bf82726fd3f4..f95a6bc839d5 100644
--- a/drivers/media/i2c/ir-kbd-i2c.c
+++ b/drivers/media/i2c/ir-kbd-i2c.c
@@ -35,6 +35,7 @@
  *
  */
 
+#include <asm/unaligned.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -63,51 +64,80 @@ module_param(debug, int, 0644);    /* debug level (0,1,2) */
 /* ----------------------------------------------------------------------- */
 
 static int get_key_haup_common(struct IR_i2c *ir, enum rc_type *protocol,
-			       u32 *scancode, u8 *ptoggle, int size, int offset)
+					u32 *scancode, u8 *ptoggle, int size)
 {
 	unsigned char buf[6];
-	int start, range, toggle, dev, code, ircode;
+	int start, range, toggle, dev, code, ircode, vendor;
 
 	/* poll IR chip */
 	if (size != i2c_master_recv(ir->c, buf, size))
 		return -EIO;
 
-	/* split rc5 data block ... */
-	start  = (buf[offset] >> 7) &    1;
-	range  = (buf[offset] >> 6) &    1;
-	toggle = (buf[offset] >> 5) &    1;
-	dev    =  buf[offset]       & 0x1f;
-	code   = (buf[offset+1] >> 2) & 0x3f;
+	if (buf[0] & 0x80) {
+		int offset = (size == 6) ? 3 : 0;
 
-	/* rc5 has two start bits
-	 * the first bit must be one
-	 * the second bit defines the command range (1 = 0-63, 0 = 64 - 127)
-	 */
-	if (!start)
-		/* no key pressed */
-		return 0;
+		/* split rc5 data block ... */
+		start  = (buf[offset] >> 7) &    1;
+		range  = (buf[offset] >> 6) &    1;
+		toggle = (buf[offset] >> 5) &    1;
+		dev    =  buf[offset]       & 0x1f;
+		code   = (buf[offset+1] >> 2) & 0x3f;
 
-	/* filter out invalid key presses */
-	ircode = (start << 12) | (toggle << 11) | (dev << 6) | code;
-	if ((ircode & 0x1fff) == 0x1fff)
-		return 0;
+		/* rc5 has two start bits
+		 * the first bit must be one
+		 * the second bit defines the command range:
+		 * 1 = 0-63, 0 = 64 - 127
+		 */
+		if (!start)
+			/* no key pressed */
+			return 0;
 
-	if (!range)
-		code += 64;
+		/* filter out invalid key presses */
+		ircode = (start << 12) | (toggle << 11) | (dev << 6) | code;
+		if ((ircode & 0x1fff) == 0x1fff)
+			return 0;
 
-	dprintk(1,"ir hauppauge (rc5): s%d r%d t%d dev=%d code=%d\n",
-		start, range, toggle, dev, code);
+		if (!range)
+			code += 64;
 
-	*protocol = RC_TYPE_RC5;
-	*scancode = RC_SCANCODE_RC5(dev, code);
-	*ptoggle = toggle;
-	return 1;
+		dprintk(1, "ir hauppauge (rc5): s%d r%d t%d dev=%d code=%d\n",
+			start, range, toggle, dev, code);
+
+		*protocol = RC_TYPE_RC5;
+		*scancode = RC_SCANCODE_RC5(dev, code);
+		*ptoggle = toggle;
+
+		return 1;
+	} else if (size == 6 && (buf[0] & 0x40)) {
+		code = buf[4];
+		dev = buf[3];
+		vendor = get_unaligned_be16(buf + 1);
+
+		if (vendor == 0x800f) {
+			*ptoggle = (dev & 0x80) != 0;
+			*protocol = RC_TYPE_RC6_MCE;
+			dev &= 0x7f;
+			dprintk(1, "ir hauppauge (rc6-mce): t%d vendor=%d dev=%d code=%d\n",
+						toggle, vendor, dev, code);
+		} else {
+			*ptoggle = 0;
+			*protocol = RC_TYPE_RC6_6A_32;
+			dprintk(1, "ir hauppauge (rc6-6a-32): vendor=%d dev=%d code=%d\n",
+							vendor, dev, code);
+		}
+
+		*scancode = RC_SCANCODE_RC6_6A(vendor, dev, code);
+
+		return 1;
+	}
+
+	return 0;
 }
 
 static int get_key_haup(struct IR_i2c *ir, enum rc_type *protocol,
 			u32 *scancode, u8 *toggle)
 {
-	return get_key_haup_common (ir, protocol, scancode, toggle, 3, 0);
+	return get_key_haup_common(ir, protocol, scancode, toggle, 3);
 }
 
 static int get_key_haup_xvr(struct IR_i2c *ir, enum rc_type *protocol,
@@ -126,7 +156,7 @@ static int get_key_haup_xvr(struct IR_i2c *ir, enum rc_type *protocol,
 	if (ret != 1)
 		return (ret < 0) ? ret : -EINVAL;
 
-	return get_key_haup_common(ir, protocol, scancode, toggle, 6, 3);
+	return get_key_haup_common(ir, protocol, scancode, toggle, 6);
 }
 
 static int get_key_pixelview(struct IR_i2c *ir, enum rc_type *protocol,
@@ -347,7 +377,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	case 0x71:
 		name        = "Hauppauge/Zilog Z8";
 		ir->get_key = get_key_haup_xvr;
-		rc_type     = RC_BIT_RC5;
+		rc_type     = RC_BIT_RC5 | RC_BIT_RC6_MCE | RC_BIT_RC6_6A_32;
 		ir_codes    = RC_MAP_HAUPPAUGE;
 		break;
 	}
diff --git a/drivers/media/i2c/soc_camera/mt9m111.c b/drivers/media/i2c/mt9m111.c
index 6dfaead6aaa8..72e71b762827 100644
--- a/drivers/media/i2c/soc_camera/mt9m111.c
+++ b/drivers/media/i2c/mt9m111.c
@@ -16,10 +16,11 @@
 #include <linux/v4l2-mediabus.h>
 #include <linux/module.h>
 
-#include <media/soc_camera.h>
+#include <media/v4l2-async.h>
 #include <media/v4l2-clk.h>
 #include <media/v4l2-common.h>
 #include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
 
 /*
  * MT9M111, MT9M112 and MT9M131:
@@ -187,10 +188,10 @@ struct mt9m111_datafmt {
 };
 
 static const struct mt9m111_datafmt mt9m111_colour_fmts[] = {
-	{MEDIA_BUS_FMT_YUYV8_2X8, V4L2_COLORSPACE_JPEG},
-	{MEDIA_BUS_FMT_YVYU8_2X8, V4L2_COLORSPACE_JPEG},
-	{MEDIA_BUS_FMT_UYVY8_2X8, V4L2_COLORSPACE_JPEG},
-	{MEDIA_BUS_FMT_VYUY8_2X8, V4L2_COLORSPACE_JPEG},
+	{MEDIA_BUS_FMT_YUYV8_2X8, V4L2_COLORSPACE_SRGB},
+	{MEDIA_BUS_FMT_YVYU8_2X8, V4L2_COLORSPACE_SRGB},
+	{MEDIA_BUS_FMT_UYVY8_2X8, V4L2_COLORSPACE_SRGB},
+	{MEDIA_BUS_FMT_VYUY8_2X8, V4L2_COLORSPACE_SRGB},
 	{MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE, V4L2_COLORSPACE_SRGB},
 	{MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE, V4L2_COLORSPACE_SRGB},
 	{MEDIA_BUS_FMT_RGB565_2X8_LE, V4L2_COLORSPACE_SRGB},
@@ -383,30 +384,36 @@ static int mt9m111_reset(struct mt9m111 *mt9m111)
 	return ret;
 }
 
-static int mt9m111_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
+static int mt9m111_set_selection(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_selection *sel)
 {
-	struct v4l2_rect rect = a->c;
-	struct mt9m111 *mt9m111 = container_of(sd, struct mt9m111, subdev);
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct mt9m111 *mt9m111 = to_mt9m111(client);
+	struct v4l2_rect rect = sel->r;
 	int width, height;
-	int ret;
+	int ret, align = 0;
 
-	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 
 	if (mt9m111->fmt->code == MEDIA_BUS_FMT_SBGGR8_1X8 ||
 	    mt9m111->fmt->code == MEDIA_BUS_FMT_SBGGR10_2X8_PADHI_LE) {
 		/* Bayer format - even size lengths */
-		rect.width	= ALIGN(rect.width, 2);
-		rect.height	= ALIGN(rect.height, 2);
+		align = 1;
 		/* Let the user play with the starting pixel */
 	}
 
 	/* FIXME: the datasheet doesn't specify minimum sizes */
-	soc_camera_limit_side(&rect.left, &rect.width,
-		     MT9M111_MIN_DARK_COLS, 2, MT9M111_MAX_WIDTH);
-
-	soc_camera_limit_side(&rect.top, &rect.height,
-		     MT9M111_MIN_DARK_ROWS, 2, MT9M111_MAX_HEIGHT);
+	v4l_bound_align_image(&rect.width, 2, MT9M111_MAX_WIDTH, align,
+			      &rect.height, 2, MT9M111_MAX_HEIGHT, align, 0);
+	rect.left = clamp(rect.left, MT9M111_MIN_DARK_COLS,
+			  MT9M111_MIN_DARK_COLS + MT9M111_MAX_WIDTH -
+			  (__s32)rect.width);
+	rect.top = clamp(rect.top, MT9M111_MIN_DARK_ROWS,
+			 MT9M111_MIN_DARK_ROWS + MT9M111_MAX_HEIGHT -
+			 (__s32)rect.height);
 
 	width = min(mt9m111->width, rect.width);
 	height = min(mt9m111->height, rect.height);
@@ -421,30 +428,30 @@ static int mt9m111_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
 	return ret;
 }
 
-static int mt9m111_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int mt9m111_get_selection(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_selection *sel)
 {
-	struct mt9m111 *mt9m111 = container_of(sd, struct mt9m111, subdev);
-
-	a->c	= mt9m111->rect;
-	a->type	= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct mt9m111 *mt9m111 = to_mt9m111(client);
 
-static int mt9m111_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
 		return -EINVAL;
 
-	a->bounds.left			= MT9M111_MIN_DARK_COLS;
-	a->bounds.top			= MT9M111_MIN_DARK_ROWS;
-	a->bounds.width			= MT9M111_MAX_WIDTH;
-	a->bounds.height		= MT9M111_MAX_HEIGHT;
-	a->defrect			= a->bounds;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
-
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = MT9M111_MIN_DARK_COLS;
+		sel->r.top = MT9M111_MIN_DARK_ROWS;
+		sel->r.width = MT9M111_MAX_WIDTH;
+		sel->r.height = MT9M111_MAX_HEIGHT;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r = mt9m111->rect;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int mt9m111_get_fmt(struct v4l2_subdev *sd,
@@ -775,17 +782,16 @@ static int mt9m111_init(struct mt9m111 *mt9m111)
 static int mt9m111_power_on(struct mt9m111 *mt9m111)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&mt9m111->subdev);
-	struct soc_camera_subdev_desc *ssdd = soc_camera_i2c_to_desc(client);
 	int ret;
 
-	ret = soc_camera_power_on(&client->dev, ssdd, mt9m111->clk);
+	ret = v4l2_clk_enable(mt9m111->clk);
 	if (ret < 0)
 		return ret;
 
 	ret = mt9m111_resume(mt9m111);
 	if (ret < 0) {
 		dev_err(&client->dev, "Failed to resume the sensor: %d\n", ret);
-		soc_camera_power_off(&client->dev, ssdd, mt9m111->clk);
+		v4l2_clk_disable(mt9m111->clk);
 	}
 
 	return ret;
@@ -793,11 +799,8 @@ static int mt9m111_power_on(struct mt9m111 *mt9m111)
 
 static void mt9m111_power_off(struct mt9m111 *mt9m111)
 {
-	struct i2c_client *client = v4l2_get_subdevdata(&mt9m111->subdev);
-	struct soc_camera_subdev_desc *ssdd = soc_camera_i2c_to_desc(client);
-
 	mt9m111_suspend(mt9m111);
-	soc_camera_power_off(&client->dev, ssdd, mt9m111->clk);
+	v4l2_clk_disable(mt9m111->clk);
 }
 
 static int mt9m111_s_power(struct v4l2_subdev *sd, int on)
@@ -854,27 +857,22 @@ static int mt9m111_enum_mbus_code(struct v4l2_subdev *sd,
 static int mt9m111_g_mbus_config(struct v4l2_subdev *sd,
 				struct v4l2_mbus_config *cfg)
 {
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct soc_camera_subdev_desc *ssdd = soc_camera_i2c_to_desc(client);
-
 	cfg->flags = V4L2_MBUS_MASTER | V4L2_MBUS_PCLK_SAMPLE_RISING |
 		V4L2_MBUS_HSYNC_ACTIVE_HIGH | V4L2_MBUS_VSYNC_ACTIVE_HIGH |
 		V4L2_MBUS_DATA_ACTIVE_HIGH;
 	cfg->type = V4L2_MBUS_PARALLEL;
-	cfg->flags = soc_camera_apply_board_flags(ssdd, cfg);
 
 	return 0;
 }
 
 static struct v4l2_subdev_video_ops mt9m111_subdev_video_ops = {
-	.s_crop		= mt9m111_s_crop,
-	.g_crop		= mt9m111_g_crop,
-	.cropcap	= mt9m111_cropcap,
 	.g_mbus_config	= mt9m111_g_mbus_config,
 };
 
 static const struct v4l2_subdev_pad_ops mt9m111_subdev_pad_ops = {
 	.enum_mbus_code = mt9m111_enum_mbus_code,
+	.get_selection	= mt9m111_get_selection,
+	.set_selection	= mt9m111_set_selection,
 	.get_fmt	= mt9m111_get_fmt,
 	.set_fmt	= mt9m111_set_fmt,
 };
@@ -933,20 +931,8 @@ static int mt9m111_probe(struct i2c_client *client,
 {
 	struct mt9m111 *mt9m111;
 	struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent);
-	struct soc_camera_subdev_desc *ssdd = soc_camera_i2c_to_desc(client);
 	int ret;
 
-	if (client->dev.of_node) {
-		ssdd = devm_kzalloc(&client->dev, sizeof(*ssdd), GFP_KERNEL);
-		if (!ssdd)
-			return -ENOMEM;
-		client->dev.platform_data = ssdd;
-	}
-	if (!ssdd) {
-		dev_err(&client->dev, "mt9m111: driver needs platform data\n");
-		return -EINVAL;
-	}
-
 	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_WORD_DATA)) {
 		dev_warn(&adapter->dev,
 			 "I2C-Adapter doesn't support I2C_FUNC_SMBUS_WORD\n");
@@ -992,10 +978,6 @@ static int mt9m111_probe(struct i2c_client *client,
 	mt9m111->lastpage	= -1;
 	mutex_init(&mt9m111->power_lock);
 
-	ret = soc_camera_power_init(&client->dev, ssdd);
-	if (ret < 0)
-		goto out_hdlfree;
-
 	ret = mt9m111_video_probe(client);
 	if (ret < 0)
 		goto out_hdlfree;
diff --git a/drivers/media/i2c/ov9650.c b/drivers/media/i2c/ov9650.c
index be5a7fd4f076..502c72238a4a 100644
--- a/drivers/media/i2c/ov9650.c
+++ b/drivers/media/i2c/ov9650.c
@@ -23,6 +23,7 @@
 #include <linux/videodev2.h>
 
 #include <media/media-entity.h>
+#include <media/v4l2-async.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
@@ -1520,6 +1521,10 @@ static int ov965x_probe(struct i2c_client *client,
 	/* Update exposure time min/max to match frame format */
 	ov965x_update_exposure_ctrl(ov965x);
 
+	ret = v4l2_async_register_subdev(sd);
+	if (ret < 0)
+		goto err_ctrls;
+
 	return 0;
 err_ctrls:
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
@@ -1532,7 +1537,7 @@ static int ov965x_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
-	v4l2_device_unregister_subdev(sd);
+	v4l2_async_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	media_entity_cleanup(&sd->entity);
 
diff --git a/drivers/media/i2c/s5c73m3/s5c73m3-core.c b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
index 08af58fb8e7d..3844853ab0a0 100644
--- a/drivers/media/i2c/s5c73m3/s5c73m3-core.c
+++ b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
@@ -1706,7 +1706,7 @@ static int s5c73m3_probe(struct i2c_client *client,
 	state->oif_pads[OIF_ISP_PAD].flags = MEDIA_PAD_FL_SINK;
 	state->oif_pads[OIF_JPEG_PAD].flags = MEDIA_PAD_FL_SINK;
 	state->oif_pads[OIF_SOURCE_PAD].flags = MEDIA_PAD_FL_SOURCE;
-	oif_sd->entity.function = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
+	oif_sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_SCALER;
 
 	ret = media_entity_pads_init(&oif_sd->entity, OIF_NUM_PADS,
 							state->oif_pads);
diff --git a/drivers/media/i2c/s5k4ecgx.c b/drivers/media/i2c/s5k4ecgx.c
index 8a0f22da590f..6ebcf254989a 100644
--- a/drivers/media/i2c/s5k4ecgx.c
+++ b/drivers/media/i2c/s5k4ecgx.c
@@ -1019,7 +1019,6 @@ MODULE_DEVICE_TABLE(i2c, s5k4ecgx_id);
 
 static struct i2c_driver v4l2_i2c_driver = {
 	.driver = {
-		.owner	= THIS_MODULE,
 		.name = S5K4ECGX_DRIVER_NAME,
 	},
 	.probe = s5k4ecgx_probe,
diff --git a/drivers/media/i2c/s5k6a3.c b/drivers/media/i2c/s5k6a3.c
index cbe4711e9b31..769964057881 100644
--- a/drivers/media/i2c/s5k6a3.c
+++ b/drivers/media/i2c/s5k6a3.c
@@ -331,6 +331,7 @@ static int s5k6a3_probe(struct i2c_client *client,
 	sensor->format.width = S5K6A3_DEFAULT_WIDTH;
 	sensor->format.height = S5K6A3_DEFAULT_HEIGHT;
 
+	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
 	sensor->pad.flags = MEDIA_PAD_FL_SOURCE;
 	ret = media_entity_pads_init(&sd->entity, 1, &sensor->pad);
 	if (ret < 0)
@@ -376,7 +377,6 @@ static struct i2c_driver s5k6a3_driver = {
 	.driver = {
 		.of_match_table	= of_match_ptr(s5k6a3_of_match),
 		.name		= S5K6A3_DRV_NAME,
-		.owner		= THIS_MODULE,
 	},
 	.probe		= s5k6a3_probe,
 	.remove		= s5k6a3_remove,
diff --git a/drivers/media/i2c/smiapp/smiapp-core.c b/drivers/media/i2c/smiapp/smiapp-core.c
index d08ab6c8357c..44f8c7e10a35 100644
--- a/drivers/media/i2c/smiapp/smiapp-core.c
+++ b/drivers/media/i2c/smiapp/smiapp-core.c
@@ -24,8 +24,8 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/module.h>
-#include <linux/of_gpio.h>
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
 #include <linux/smiapp.h>
@@ -328,6 +328,14 @@ static void __smiapp_update_exposure_limits(struct smiapp_sensor *sensor)
  *    orders must be defined.
  */
 static const struct smiapp_csi_data_format smiapp_csi_data_formats[] = {
+	{ MEDIA_BUS_FMT_SGRBG16_1X16, 16, 16, SMIAPP_PIXEL_ORDER_GRBG, },
+	{ MEDIA_BUS_FMT_SRGGB16_1X16, 16, 16, SMIAPP_PIXEL_ORDER_RGGB, },
+	{ MEDIA_BUS_FMT_SBGGR16_1X16, 16, 16, SMIAPP_PIXEL_ORDER_BGGR, },
+	{ MEDIA_BUS_FMT_SGBRG16_1X16, 16, 16, SMIAPP_PIXEL_ORDER_GBRG, },
+	{ MEDIA_BUS_FMT_SGRBG14_1X14, 14, 14, SMIAPP_PIXEL_ORDER_GRBG, },
+	{ MEDIA_BUS_FMT_SRGGB14_1X14, 14, 14, SMIAPP_PIXEL_ORDER_RGGB, },
+	{ MEDIA_BUS_FMT_SBGGR14_1X14, 14, 14, SMIAPP_PIXEL_ORDER_BGGR, },
+	{ MEDIA_BUS_FMT_SGBRG14_1X14, 14, 14, SMIAPP_PIXEL_ORDER_GBRG, },
 	{ MEDIA_BUS_FMT_SGRBG12_1X12, 12, 12, SMIAPP_PIXEL_ORDER_GRBG, },
 	{ MEDIA_BUS_FMT_SRGGB12_1X12, 12, 12, SMIAPP_PIXEL_ORDER_RGGB, },
 	{ MEDIA_BUS_FMT_SBGGR12_1X12, 12, 12, SMIAPP_PIXEL_ORDER_BGGR, },
@@ -625,12 +633,12 @@ static int smiapp_init_late_controls(struct smiapp_sensor *sensor)
 				0, max_value, 1, max_value);
 	}
 
-	for (max = 0; sensor->platform_data->op_sys_clock[max + 1]; max++);
+	for (max = 0; sensor->hwcfg->op_sys_clock[max + 1]; max++);
 
 	sensor->link_freq = v4l2_ctrl_new_int_menu(
 		&sensor->src->ctrl_handler, &smiapp_ctrl_ops,
 		V4L2_CID_LINK_FREQ, __fls(*valid_link_freqs),
-		__ffs(*valid_link_freqs), sensor->platform_data->op_sys_clock);
+		__ffs(*valid_link_freqs), sensor->hwcfg->op_sys_clock);
 
 	return sensor->src->ctrl_handler.error;
 }
@@ -833,8 +841,8 @@ static int smiapp_get_mbus_formats(struct smiapp_sensor *sensor)
 
 		pll->bits_per_pixel = f->compressed;
 
-		for (j = 0; sensor->platform_data->op_sys_clock[j]; j++) {
-			pll->link_freq = sensor->platform_data->op_sys_clock[j];
+		for (j = 0; sensor->hwcfg->op_sys_clock[j]; j++) {
+			pll->link_freq = sensor->hwcfg->op_sys_clock[j];
 
 			rval = smiapp_pll_try(sensor, pll);
 			dev_dbg(&client->dev, "link freq %u Hz, bpp %u %s\n",
@@ -1032,22 +1040,22 @@ static int smiapp_change_cci_addr(struct smiapp_sensor *sensor)
 	int rval;
 	u32 val;
 
-	client->addr = sensor->platform_data->i2c_addr_dfl;
+	client->addr = sensor->hwcfg->i2c_addr_dfl;
 
 	rval = smiapp_write(sensor,
 			    SMIAPP_REG_U8_CCI_ADDRESS_CONTROL,
-			    sensor->platform_data->i2c_addr_alt << 1);
+			    sensor->hwcfg->i2c_addr_alt << 1);
 	if (rval)
 		return rval;
 
-	client->addr = sensor->platform_data->i2c_addr_alt;
+	client->addr = sensor->hwcfg->i2c_addr_alt;
 
 	/* verify addr change went ok */
 	rval = smiapp_read(sensor, SMIAPP_REG_U8_CCI_ADDRESS_CONTROL, &val);
 	if (rval)
 		return rval;
 
-	if (val != sensor->platform_data->i2c_addr_alt << 1)
+	if (val != sensor->hwcfg->i2c_addr_alt << 1)
 		return -ENODEV;
 
 	return 0;
@@ -1061,13 +1069,13 @@ static int smiapp_change_cci_addr(struct smiapp_sensor *sensor)
 static int smiapp_setup_flash_strobe(struct smiapp_sensor *sensor)
 {
 	struct smiapp_flash_strobe_parms *strobe_setup;
-	unsigned int ext_freq = sensor->platform_data->ext_clk;
+	unsigned int ext_freq = sensor->hwcfg->ext_clk;
 	u32 tmp;
 	u32 strobe_adjustment;
 	u32 strobe_width_high_rs;
 	int rval;
 
-	strobe_setup = sensor->platform_data->strobe_setup;
+	strobe_setup = sensor->hwcfg->strobe_setup;
 
 	/*
 	 * How to calculate registers related to strobe length. Please
@@ -1179,7 +1187,7 @@ static int smiapp_setup_flash_strobe(struct smiapp_sensor *sensor)
 			    strobe_setup->trigger);
 
 out:
-	sensor->platform_data->strobe_setup->trigger = 0;
+	sensor->hwcfg->strobe_setup->trigger = 0;
 
 	return rval;
 }
@@ -1201,21 +1209,16 @@ static int smiapp_power_on(struct smiapp_sensor *sensor)
 	}
 	usleep_range(1000, 1000);
 
-	if (sensor->platform_data->set_xclk)
-		rval = sensor->platform_data->set_xclk(
-			&sensor->src->sd, sensor->platform_data->ext_clk);
-	else
-		rval = clk_prepare_enable(sensor->ext_clk);
+	rval = clk_prepare_enable(sensor->ext_clk);
 	if (rval < 0) {
 		dev_dbg(&client->dev, "failed to enable xclk\n");
 		goto out_xclk_fail;
 	}
 	usleep_range(1000, 1000);
 
-	if (gpio_is_valid(sensor->platform_data->xshutdown))
-		gpio_set_value(sensor->platform_data->xshutdown, 1);
+	gpiod_set_value(sensor->xshutdown, 1);
 
-	sleep = SMIAPP_RESET_DELAY(sensor->platform_data->ext_clk);
+	sleep = SMIAPP_RESET_DELAY(sensor->hwcfg->ext_clk);
 	usleep_range(sleep, sleep);
 
 	/*
@@ -1229,7 +1232,7 @@ static int smiapp_power_on(struct smiapp_sensor *sensor)
 	 * is found.
 	 */
 
-	if (sensor->platform_data->i2c_addr_alt) {
+	if (sensor->hwcfg->i2c_addr_alt) {
 		rval = smiapp_change_cci_addr(sensor);
 		if (rval) {
 			dev_err(&client->dev, "cci address change error\n");
@@ -1244,7 +1247,7 @@ static int smiapp_power_on(struct smiapp_sensor *sensor)
 		goto out_cci_addr_fail;
 	}
 
-	if (sensor->platform_data->i2c_addr_alt) {
+	if (sensor->hwcfg->i2c_addr_alt) {
 		rval = smiapp_change_cci_addr(sensor);
 		if (rval) {
 			dev_err(&client->dev, "cci address change error\n");
@@ -1261,14 +1264,14 @@ static int smiapp_power_on(struct smiapp_sensor *sensor)
 
 	rval = smiapp_write(
 		sensor, SMIAPP_REG_U16_EXTCLK_FREQUENCY_MHZ,
-		sensor->platform_data->ext_clk / (1000000 / (1 << 8)));
+		sensor->hwcfg->ext_clk / (1000000 / (1 << 8)));
 	if (rval) {
 		dev_err(&client->dev, "extclk frequency set failed\n");
 		goto out_cci_addr_fail;
 	}
 
 	rval = smiapp_write(sensor, SMIAPP_REG_U8_CSI_LANE_MODE,
-			    sensor->platform_data->lanes - 1);
+			    sensor->hwcfg->lanes - 1);
 	if (rval) {
 		dev_err(&client->dev, "csi lane mode set failed\n");
 		goto out_cci_addr_fail;
@@ -1282,7 +1285,7 @@ static int smiapp_power_on(struct smiapp_sensor *sensor)
 	}
 
 	rval = smiapp_write(sensor, SMIAPP_REG_U8_CSI_SIGNALLING_MODE,
-			    sensor->platform_data->csi_signalling_mode);
+			    sensor->hwcfg->csi_signalling_mode);
 	if (rval) {
 		dev_err(&client->dev, "csi signalling mode set failed\n");
 		goto out_cci_addr_fail;
@@ -1322,12 +1325,8 @@ static int smiapp_power_on(struct smiapp_sensor *sensor)
 	return 0;
 
 out_cci_addr_fail:
-	if (gpio_is_valid(sensor->platform_data->xshutdown))
-		gpio_set_value(sensor->platform_data->xshutdown, 0);
-	if (sensor->platform_data->set_xclk)
-		sensor->platform_data->set_xclk(&sensor->src->sd, 0);
-	else
-		clk_disable_unprepare(sensor->ext_clk);
+	gpiod_set_value(sensor->xshutdown, 0);
+	clk_disable_unprepare(sensor->ext_clk);
 
 out_xclk_fail:
 	regulator_disable(sensor->vana);
@@ -1343,17 +1342,13 @@ static void smiapp_power_off(struct smiapp_sensor *sensor)
 	 * really see a power off and next time the cci address change
 	 * will fail. So do a soft reset explicitly here.
 	 */
-	if (sensor->platform_data->i2c_addr_alt)
+	if (sensor->hwcfg->i2c_addr_alt)
 		smiapp_write(sensor,
 			     SMIAPP_REG_U8_SOFTWARE_RESET,
 			     SMIAPP_SOFTWARE_RESET);
 
-	if (gpio_is_valid(sensor->platform_data->xshutdown))
-		gpio_set_value(sensor->platform_data->xshutdown, 0);
-	if (sensor->platform_data->set_xclk)
-		sensor->platform_data->set_xclk(&sensor->src->sd, 0);
-	else
-		clk_disable_unprepare(sensor->ext_clk);
+	gpiod_set_value(sensor->xshutdown, 0);
+	clk_disable_unprepare(sensor->ext_clk);
 	usleep_range(5000, 5000);
 	regulator_disable(sensor->vana);
 	sensor->streaming = false;
@@ -1491,8 +1486,8 @@ static int smiapp_start_streaming(struct smiapp_sensor *sensor)
 	if ((sensor->limits[SMIAPP_LIMIT_FLASH_MODE_CAPABILITY] &
 	     (SMIAPP_FLASH_MODE_CAPABILITY_SINGLE_STROBE |
 	      SMIAPP_FLASH_MODE_CAPABILITY_MULTIPLE_STROBE)) &&
-	    sensor->platform_data->strobe_setup != NULL &&
-	    sensor->platform_data->strobe_setup->trigger != 0) {
+	    sensor->hwcfg->strobe_setup != NULL &&
+	    sensor->hwcfg->strobe_setup->trigger != 0) {
 		rval = smiapp_setup_flash_strobe(sensor);
 		if (rval)
 			goto out;
@@ -2309,7 +2304,7 @@ smiapp_sysfs_nvm_read(struct device *dev, struct device_attribute *attr,
 
 	if (!sensor->nvm_size) {
 		/* NVM not read yet - read it now */
-		sensor->nvm_size = sensor->platform_data->nvm_size;
+		sensor->nvm_size = sensor->hwcfg->nvm_size;
 		if (smiapp_set_power(subdev, 1) < 0)
 			return -ENODEV;
 		if (smiapp_read_nvm(sensor, sensor->nvm)) {
@@ -2554,35 +2549,27 @@ static int smiapp_init(struct smiapp_sensor *sensor)
 		return PTR_ERR(sensor->vana);
 	}
 
-	if (!sensor->platform_data->set_xclk) {
-		sensor->ext_clk = devm_clk_get(&client->dev, NULL);
-		if (IS_ERR(sensor->ext_clk)) {
-			dev_err(&client->dev, "could not get clock\n");
-			return PTR_ERR(sensor->ext_clk);
-		}
-
-		rval = clk_set_rate(sensor->ext_clk,
-				    sensor->platform_data->ext_clk);
-		if (rval < 0) {
-			dev_err(&client->dev,
-				"unable to set clock freq to %u\n",
-				sensor->platform_data->ext_clk);
-			return rval;
-		}
+	sensor->ext_clk = devm_clk_get(&client->dev, NULL);
+	if (IS_ERR(sensor->ext_clk)) {
+		dev_err(&client->dev, "could not get clock (%ld)\n",
+			PTR_ERR(sensor->ext_clk));
+		return -EPROBE_DEFER;
 	}
 
-	if (gpio_is_valid(sensor->platform_data->xshutdown)) {
-		rval = devm_gpio_request_one(
-			&client->dev, sensor->platform_data->xshutdown, 0,
-			"SMIA++ xshutdown");
-		if (rval < 0) {
-			dev_err(&client->dev,
-				"unable to acquire reset gpio %d\n",
-				sensor->platform_data->xshutdown);
-			return rval;
-		}
+	rval = clk_set_rate(sensor->ext_clk,
+			    sensor->hwcfg->ext_clk);
+	if (rval < 0) {
+		dev_err(&client->dev,
+			"unable to set clock freq to %u\n",
+			sensor->hwcfg->ext_clk);
+		return rval;
 	}
 
+	sensor->xshutdown = devm_gpiod_get_optional(&client->dev, "xshutdown",
+						    GPIOD_OUT_LOW);
+	if (IS_ERR(sensor->xshutdown))
+		return PTR_ERR(sensor->xshutdown);
+
 	rval = smiapp_power_on(sensor);
 	if (rval)
 		return -ENODEV;
@@ -2612,7 +2599,7 @@ static int smiapp_init(struct smiapp_sensor *sensor)
 	 *
 	 * Rotation also changes the bayer pattern.
 	 */
-	if (sensor->platform_data->module_board_orient ==
+	if (sensor->hwcfg->module_board_orient ==
 	    SMIAPP_MODULE_BOARD_ORIENT_180)
 		sensor->hvflip_inv_mask = SMIAPP_IMAGE_ORIENTATION_HFLIP |
 					  SMIAPP_IMAGE_ORIENTATION_VFLIP;
@@ -2661,9 +2648,9 @@ static int smiapp_init(struct smiapp_sensor *sensor)
 	/* SMIA++ NVM initialization - it will be read from the sensor
 	 * when it is first requested by userspace.
 	 */
-	if (sensor->minfo.smiapp_version && sensor->platform_data->nvm_size) {
+	if (sensor->minfo.smiapp_version && sensor->hwcfg->nvm_size) {
 		sensor->nvm = devm_kzalloc(&client->dev,
-				sensor->platform_data->nvm_size, GFP_KERNEL);
+				sensor->hwcfg->nvm_size, GFP_KERNEL);
 		if (sensor->nvm == NULL) {
 			dev_err(&client->dev, "nvm buf allocation failed\n");
 			rval = -ENOMEM;
@@ -2706,8 +2693,8 @@ static int smiapp_init(struct smiapp_sensor *sensor)
 
 	/* prepare PLL configuration input values */
 	pll->bus_type = SMIAPP_PLL_BUS_TYPE_CSI2;
-	pll->csi2.lanes = sensor->platform_data->lanes;
-	pll->ext_clk_freq_hz = sensor->platform_data->ext_clk;
+	pll->csi2.lanes = sensor->hwcfg->lanes;
+	pll->ext_clk_freq_hz = sensor->hwcfg->ext_clk;
 	pll->scale_n = sensor->limits[SMIAPP_LIMIT_SCALER_N_MIN];
 	/* Profile 0 sensors have no separate OP clock branch. */
 	if (sensor->minfo.smiapp_profile == SMIAPP_PROFILE_0)
@@ -2984,9 +2971,9 @@ static int smiapp_resume(struct device *dev)
 
 #endif /* CONFIG_PM */
 
-static struct smiapp_platform_data *smiapp_get_pdata(struct device *dev)
+static struct smiapp_hwconfig *smiapp_get_hwconfig(struct device *dev)
 {
-	struct smiapp_platform_data *pdata;
+	struct smiapp_hwconfig *hwcfg;
 	struct v4l2_of_endpoint *bus_cfg;
 	struct device_node *ep;
 	int i;
@@ -3003,58 +2990,55 @@ static struct smiapp_platform_data *smiapp_get_pdata(struct device *dev)
 	if (IS_ERR(bus_cfg))
 		goto out_err;
 
-	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata)
+	hwcfg = devm_kzalloc(dev, sizeof(*hwcfg), GFP_KERNEL);
+	if (!hwcfg)
 		goto out_err;
 
 	switch (bus_cfg->bus_type) {
 	case V4L2_MBUS_CSI2:
-		pdata->csi_signalling_mode = SMIAPP_CSI_SIGNALLING_MODE_CSI2;
+		hwcfg->csi_signalling_mode = SMIAPP_CSI_SIGNALLING_MODE_CSI2;
 		break;
 		/* FIXME: add CCP2 support. */
 	default:
 		goto out_err;
 	}
 
-	pdata->lanes = bus_cfg->bus.mipi_csi2.num_data_lanes;
-	dev_dbg(dev, "lanes %u\n", pdata->lanes);
-
-	/* xshutdown GPIO is optional */
-	pdata->xshutdown = of_get_named_gpio(dev->of_node, "reset-gpios", 0);
+	hwcfg->lanes = bus_cfg->bus.mipi_csi2.num_data_lanes;
+	dev_dbg(dev, "lanes %u\n", hwcfg->lanes);
 
 	/* NVM size is not mandatory */
 	of_property_read_u32(dev->of_node, "nokia,nvm-size",
-				    &pdata->nvm_size);
+				    &hwcfg->nvm_size);
 
 	rval = of_property_read_u32(dev->of_node, "clock-frequency",
-				    &pdata->ext_clk);
+				    &hwcfg->ext_clk);
 	if (rval) {
 		dev_warn(dev, "can't get clock-frequency\n");
 		goto out_err;
 	}
 
-	dev_dbg(dev, "reset %d, nvm %d, clk %d, csi %d\n", pdata->xshutdown,
-		pdata->nvm_size, pdata->ext_clk, pdata->csi_signalling_mode);
+	dev_dbg(dev, "nvm %d, clk %d, csi %d\n", hwcfg->nvm_size,
+		hwcfg->ext_clk, hwcfg->csi_signalling_mode);
 
 	if (!bus_cfg->nr_of_link_frequencies) {
 		dev_warn(dev, "no link frequencies defined\n");
 		goto out_err;
 	}
 
-	pdata->op_sys_clock = devm_kcalloc(
+	hwcfg->op_sys_clock = devm_kcalloc(
 		dev, bus_cfg->nr_of_link_frequencies + 1 /* guardian */,
-		sizeof(*pdata->op_sys_clock), GFP_KERNEL);
-	if (!pdata->op_sys_clock)
+		sizeof(*hwcfg->op_sys_clock), GFP_KERNEL);
+	if (!hwcfg->op_sys_clock)
 		goto out_err;
 
 	for (i = 0; i < bus_cfg->nr_of_link_frequencies; i++) {
-		pdata->op_sys_clock[i] = bus_cfg->link_frequencies[i];
-		dev_dbg(dev, "freq %d: %lld\n", i, pdata->op_sys_clock[i]);
+		hwcfg->op_sys_clock[i] = bus_cfg->link_frequencies[i];
+		dev_dbg(dev, "freq %d: %lld\n", i, hwcfg->op_sys_clock[i]);
 	}
 
 	v4l2_of_free_endpoint(bus_cfg);
 	of_node_put(ep);
-	return pdata;
+	return hwcfg;
 
 out_err:
 	v4l2_of_free_endpoint(bus_cfg);
@@ -3066,17 +3050,17 @@ static int smiapp_probe(struct i2c_client *client,
 			const struct i2c_device_id *devid)
 {
 	struct smiapp_sensor *sensor;
-	struct smiapp_platform_data *pdata = smiapp_get_pdata(&client->dev);
+	struct smiapp_hwconfig *hwcfg = smiapp_get_hwconfig(&client->dev);
 	int rval;
 
-	if (pdata == NULL)
+	if (hwcfg == NULL)
 		return -ENODEV;
 
 	sensor = devm_kzalloc(&client->dev, sizeof(*sensor), GFP_KERNEL);
 	if (sensor == NULL)
 		return -ENOMEM;
 
-	sensor->platform_data = pdata;
+	sensor->hwcfg = hwcfg;
 	mutex_init(&sensor->mutex);
 	mutex_init(&sensor->power_mutex);
 	sensor->src = &sensor->ssds[sensor->ssds_used];
@@ -3119,12 +3103,8 @@ static int smiapp_remove(struct i2c_client *client)
 	v4l2_async_unregister_subdev(subdev);
 
 	if (sensor->power_count) {
-		if (gpio_is_valid(sensor->platform_data->xshutdown))
-			gpio_set_value(sensor->platform_data->xshutdown, 0);
-		if (sensor->platform_data->set_xclk)
-			sensor->platform_data->set_xclk(&sensor->src->sd, 0);
-		else
-			clk_disable_unprepare(sensor->ext_clk);
+		gpiod_set_value(sensor->xshutdown, 0);
+		clk_disable_unprepare(sensor->ext_clk);
 		sensor->power_count = 0;
 	}
 
diff --git a/drivers/media/i2c/smiapp/smiapp-quirk.c b/drivers/media/i2c/smiapp/smiapp-quirk.c
index abf9ea7a0fb7..cb128eae9c54 100644
--- a/drivers/media/i2c/smiapp/smiapp-quirk.c
+++ b/drivers/media/i2c/smiapp/smiapp-quirk.c
@@ -26,7 +26,7 @@ static int smiapp_write_8(struct smiapp_sensor *sensor, u16 reg, u8 val)
 }
 
 static int smiapp_write_8s(struct smiapp_sensor *sensor,
-			   struct smiapp_reg_8 *regs, int len)
+			   const struct smiapp_reg_8 *regs, int len)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&sensor->src->sd);
 	int rval;
@@ -71,7 +71,7 @@ static int jt8ew9_limits(struct smiapp_sensor *sensor)
 
 static int jt8ew9_post_poweron(struct smiapp_sensor *sensor)
 {
-	struct smiapp_reg_8 regs[] = {
+	const struct smiapp_reg_8 regs[] = {
 		{ 0x30a3, 0xd8 }, /* Output port control : LVDS ports only */
 		{ 0x30ae, 0x00 }, /* 0x0307 pll_multiplier maximum value on PLL input 9.6MHz ( 19.2MHz is divided on pre_pll_div) */
 		{ 0x30af, 0xd0 }, /* 0x0307 pll_multiplier maximum value on PLL input 9.6MHz ( 19.2MHz is divided on pre_pll_div) */
@@ -115,7 +115,7 @@ const struct smiapp_quirk smiapp_jt8ew9_quirk = {
 static int imx125es_post_poweron(struct smiapp_sensor *sensor)
 {
 	/* Taken from v02. No idea what the other two are. */
-	struct smiapp_reg_8 regs[] = {
+	const struct smiapp_reg_8 regs[] = {
 		/*
 		 * 0x3302: clk during frame blanking:
 		 * 0x00 - HS mode, 0x01 - LP11
@@ -145,8 +145,7 @@ static int jt8ev1_post_poweron(struct smiapp_sensor *sensor)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&sensor->src->sd);
 	int rval;
-
-	struct smiapp_reg_8 regs[] = {
+	const struct smiapp_reg_8 regs[] = {
 		{ 0x3031, 0xcd }, /* For digital binning (EQ_MONI) */
 		{ 0x30a3, 0xd0 }, /* FLASH STROBE enable */
 		{ 0x3237, 0x00 }, /* For control of pulse timing for ADC */
@@ -167,8 +166,7 @@ static int jt8ev1_post_poweron(struct smiapp_sensor *sensor)
 		{ 0x33cf, 0xec }, /* For Black sun */
 		{ 0x3328, 0x80 }, /* Ugh. No idea what's this. */
 	};
-
-	struct smiapp_reg_8 regs_96[] = {
+	const struct smiapp_reg_8 regs_96[] = {
 		{ 0x30ae, 0x00 }, /* For control of ADC clock */
 		{ 0x30af, 0xd0 },
 		{ 0x30b0, 0x01 },
@@ -178,13 +176,13 @@ static int jt8ev1_post_poweron(struct smiapp_sensor *sensor)
 	if (rval < 0)
 		return rval;
 
-	switch (sensor->platform_data->ext_clk) {
+	switch (sensor->hwcfg->ext_clk) {
 	case 9600000:
 		return smiapp_write_8s(sensor, regs_96,
 				       ARRAY_SIZE(regs_96));
 	default:
 		dev_warn(&client->dev, "no MSRs for %d Hz ext_clk\n",
-			 sensor->platform_data->ext_clk);
+			 sensor->hwcfg->ext_clk);
 		return 0;
 	}
 }
diff --git a/drivers/media/i2c/smiapp/smiapp-regs.c b/drivers/media/i2c/smiapp/smiapp-regs.c
index 6b6c20b61397..1e501c06d18c 100644
--- a/drivers/media/i2c/smiapp/smiapp-regs.c
+++ b/drivers/media/i2c/smiapp/smiapp-regs.c
@@ -188,7 +188,8 @@ int smiapp_read_no_quirk(struct smiapp_sensor *sensor, u32 reg, u32 *val)
 				   SMIAPP_QUIRK_FLAG_8BIT_READ_ONLY));
 }
 
-int smiapp_read(struct smiapp_sensor *sensor, u32 reg, u32 *val)
+static int smiapp_read_quirk(struct smiapp_sensor *sensor, u32 reg, u32 *val,
+			     bool force8)
 {
 	int rval;
 
@@ -199,21 +200,20 @@ int smiapp_read(struct smiapp_sensor *sensor, u32 reg, u32 *val)
 	if (rval < 0)
 		return rval;
 
+	if (force8)
+		return __smiapp_read(sensor, reg, val, true);
+
 	return smiapp_read_no_quirk(sensor, reg, val);
 }
 
-int smiapp_read_8only(struct smiapp_sensor *sensor, u32 reg, u32 *val)
+int smiapp_read(struct smiapp_sensor *sensor, u32 reg, u32 *val)
 {
-	int rval;
-
-	*val = 0;
-	rval = smiapp_call_quirk(sensor, reg_access, false, &reg, val);
-	if (rval == -ENOIOCTLCMD)
-		return 0;
-	if (rval < 0)
-		return rval;
+	return smiapp_read_quirk(sensor, reg, val, false);
+}
 
-	return __smiapp_read(sensor, reg, val, true);
+int smiapp_read_8only(struct smiapp_sensor *sensor, u32 reg, u32 *val)
+{
+	return smiapp_read_quirk(sensor, reg, val, true);
 }
 
 int smiapp_write_no_quirk(struct smiapp_sensor *sensor, u32 reg, u32 val)
diff --git a/drivers/media/i2c/smiapp/smiapp.h b/drivers/media/i2c/smiapp/smiapp.h
index 2174f89a00db..aae72bc87bf7 100644
--- a/drivers/media/i2c/smiapp/smiapp.h
+++ b/drivers/media/i2c/smiapp/smiapp.h
@@ -151,7 +151,7 @@ struct smiapp_csi_data_format {
 #define SMIAPP_PADS			2
 
 #define SMIAPP_COMPRESSED_BASE		8
-#define SMIAPP_COMPRESSED_MAX		12
+#define SMIAPP_COMPRESSED_MAX		16
 #define SMIAPP_NR_OF_COMPRESSED		(SMIAPP_COMPRESSED_MAX - \
 					 SMIAPP_COMPRESSED_BASE + 1)
 
@@ -197,9 +197,10 @@ struct smiapp_sensor {
 	struct smiapp_subdev *binner;
 	struct smiapp_subdev *scaler;
 	struct smiapp_subdev *pixel_array;
-	struct smiapp_platform_data *platform_data;
+	struct smiapp_hwconfig *hwcfg;
 	struct regulator *vana;
 	struct clk *ext_clk;
+	struct gpio_desc *xshutdown;
 	u32 limits[SMIAPP_LIMIT_LAST];
 	u8 nbinning_subtypes;
 	struct smiapp_binning_subtype binning_subtypes[SMIAPP_BINNING_SUBTYPES];
diff --git a/drivers/media/i2c/soc_camera/Kconfig b/drivers/media/i2c/soc_camera/Kconfig
index 23d352f0adf0..7704bcf5cc25 100644
--- a/drivers/media/i2c/soc_camera/Kconfig
+++ b/drivers/media/i2c/soc_camera/Kconfig
@@ -14,11 +14,14 @@ config SOC_CAMERA_MT9M001
 	  and colour models.
 
 config SOC_CAMERA_MT9M111
-	tristate "mt9m111, mt9m112 and mt9m131 support"
+	tristate "legacy soc_camera mt9m111, mt9m112 and mt9m131 support"
 	depends on SOC_CAMERA && I2C
+	select VIDEO_MT9M111
 	help
 	  This driver supports MT9M111, MT9M112 and MT9M131 cameras from
-	  Micron/Aptina
+	  Micron/Aptina.
+	  This is the legacy configuration which shouldn't be used anymore,
+	  while VIDEO_MT9M111 should be used instead.
 
 config SOC_CAMERA_MT9T031
 	tristate "mt9t031 support"
diff --git a/drivers/media/i2c/soc_camera/Makefile b/drivers/media/i2c/soc_camera/Makefile
index d0421feaa796..6f994f9353a0 100644
--- a/drivers/media/i2c/soc_camera/Makefile
+++ b/drivers/media/i2c/soc_camera/Makefile
@@ -1,6 +1,5 @@
 obj-$(CONFIG_SOC_CAMERA_IMX074)		+= imx074.o
 obj-$(CONFIG_SOC_CAMERA_MT9M001)	+= mt9m001.o
-obj-$(CONFIG_SOC_CAMERA_MT9M111)	+= mt9m111.o
 obj-$(CONFIG_SOC_CAMERA_MT9T031)	+= mt9t031.o
 obj-$(CONFIG_SOC_CAMERA_MT9T112)	+= mt9t112.o
 obj-$(CONFIG_SOC_CAMERA_MT9V022)	+= mt9v022.o
diff --git a/drivers/media/i2c/soc_camera/imx074.c b/drivers/media/i2c/soc_camera/imx074.c
index f68c2352c63c..05b55cfe8147 100644
--- a/drivers/media/i2c/soc_camera/imx074.c
+++ b/drivers/media/i2c/soc_camera/imx074.c
@@ -209,31 +209,26 @@ static int imx074_get_fmt(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int imx074_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int imx074_get_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_selection *sel)
 {
-	struct v4l2_rect *rect = &a->c;
-
-	a->type		= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	rect->top	= 0;
-	rect->left	= 0;
-	rect->width	= IMX074_WIDTH;
-	rect->height	= IMX074_HEIGHT;
-
-	return 0;
-}
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-static int imx074_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
-	a->bounds.width			= IMX074_WIDTH;
-	a->bounds.height		= IMX074_HEIGHT;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	sel->r.left = 0;
+	sel->r.top = 0;
+	sel->r.width = IMX074_WIDTH;
+	sel->r.height = IMX074_HEIGHT;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP:
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int imx074_enum_mbus_code(struct v4l2_subdev *sd,
@@ -278,8 +273,6 @@ static int imx074_g_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops imx074_subdev_video_ops = {
 	.s_stream	= imx074_s_stream,
-	.g_crop		= imx074_g_crop,
-	.cropcap	= imx074_cropcap,
 	.g_mbus_config	= imx074_g_mbus_config,
 };
 
@@ -289,6 +282,7 @@ static struct v4l2_subdev_core_ops imx074_subdev_core_ops = {
 
 static const struct v4l2_subdev_pad_ops imx074_subdev_pad_ops = {
 	.enum_mbus_code = imx074_enum_mbus_code,
+	.get_selection	= imx074_get_selection,
 	.get_fmt	= imx074_get_fmt,
 	.set_fmt	= imx074_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/mt9m001.c b/drivers/media/i2c/soc_camera/mt9m001.c
index 69becc358659..3d6378d4491c 100644
--- a/drivers/media/i2c/soc_camera/mt9m001.c
+++ b/drivers/media/i2c/soc_camera/mt9m001.c
@@ -171,13 +171,19 @@ static int mt9m001_s_stream(struct v4l2_subdev *sd, int enable)
 	return 0;
 }
 
-static int mt9m001_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
+static int mt9m001_set_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9m001 *mt9m001 = to_mt9m001(client);
-	struct v4l2_rect rect = a->c;
-	int ret;
+	struct v4l2_rect rect = sel->r;
 	const u16 hblank = 9, vblank = 25;
+	int ret;
+
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
 
 	if (mt9m001->fmts == mt9m001_colour_fmts)
 		/*
@@ -225,29 +231,30 @@ static int mt9m001_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
 	return ret;
 }
 
-static int mt9m001_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int mt9m001_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9m001 *mt9m001 = to_mt9m001(client);
 
-	a->c	= mt9m001->rect;
-	a->type	= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int mt9m001_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	a->bounds.left			= MT9M001_COLUMN_SKIP;
-	a->bounds.top			= MT9M001_ROW_SKIP;
-	a->bounds.width			= MT9M001_MAX_WIDTH;
-	a->bounds.height		= MT9M001_MAX_HEIGHT;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = MT9M001_COLUMN_SKIP;
+		sel->r.top = MT9M001_ROW_SKIP;
+		sel->r.width = MT9M001_MAX_WIDTH;
+		sel->r.height = MT9M001_MAX_HEIGHT;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r = mt9m001->rect;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int mt9m001_get_fmt(struct v4l2_subdev *sd,
@@ -275,18 +282,18 @@ static int mt9m001_s_fmt(struct v4l2_subdev *sd,
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9m001 *mt9m001 = to_mt9m001(client);
-	struct v4l2_crop a = {
-		.c = {
-			.left	= mt9m001->rect.left,
-			.top	= mt9m001->rect.top,
-			.width	= mf->width,
-			.height	= mf->height,
-		},
+	struct v4l2_subdev_selection sel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = V4L2_SEL_TGT_CROP,
+		.r.left = mt9m001->rect.left,
+		.r.top = mt9m001->rect.top,
+		.r.width = mf->width,
+		.r.height = mf->height,
 	};
 	int ret;
 
 	/* No support for scaling so far, just crop. TODO: use skipping */
-	ret = mt9m001_s_crop(sd, &a);
+	ret = mt9m001_set_selection(sd, NULL, &sel);
 	if (!ret) {
 		mf->width	= mt9m001->rect.width;
 		mf->height	= mt9m001->rect.height;
@@ -625,9 +632,6 @@ static int mt9m001_s_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops mt9m001_subdev_video_ops = {
 	.s_stream	= mt9m001_s_stream,
-	.s_crop		= mt9m001_s_crop,
-	.g_crop		= mt9m001_g_crop,
-	.cropcap	= mt9m001_cropcap,
 	.g_mbus_config	= mt9m001_g_mbus_config,
 	.s_mbus_config	= mt9m001_s_mbus_config,
 };
@@ -638,6 +642,8 @@ static const struct v4l2_subdev_sensor_ops mt9m001_subdev_sensor_ops = {
 
 static const struct v4l2_subdev_pad_ops mt9m001_subdev_pad_ops = {
 	.enum_mbus_code = mt9m001_enum_mbus_code,
+	.get_selection	= mt9m001_get_selection,
+	.set_selection	= mt9m001_set_selection,
 	.get_fmt	= mt9m001_get_fmt,
 	.set_fmt	= mt9m001_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/mt9t031.c b/drivers/media/i2c/soc_camera/mt9t031.c
index 5c8e3ffe3b27..3aa5569065ad 100644
--- a/drivers/media/i2c/soc_camera/mt9t031.c
+++ b/drivers/media/i2c/soc_camera/mt9t031.c
@@ -264,7 +264,7 @@ static int mt9t031_set_params(struct i2c_client *client,
 
 	/*
 	 * The caller provides a supported format, as guaranteed by
-	 * .set_fmt(FORMAT_TRY), soc_camera_s_crop() and soc_camera_cropcap()
+	 * .set_fmt(FORMAT_TRY), soc_camera_s_selection() and soc_camera_cropcap()
 	 */
 	if (ret >= 0)
 		ret = reg_write(client, MT9T031_COLUMN_START, rect->left);
@@ -294,11 +294,17 @@ static int mt9t031_set_params(struct i2c_client *client,
 	return ret < 0 ? ret : 0;
 }
 
-static int mt9t031_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
+static int mt9t031_set_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
-	struct v4l2_rect rect = a->c;
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9t031 *mt9t031 = to_mt9t031(client);
+	struct v4l2_rect rect = sel->r;
+
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
 
 	rect.width = ALIGN(rect.width, 2);
 	rect.height = ALIGN(rect.height, 2);
@@ -312,29 +318,30 @@ static int mt9t031_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
 	return mt9t031_set_params(client, &rect, mt9t031->xskip, mt9t031->yskip);
 }
 
-static int mt9t031_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int mt9t031_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9t031 *mt9t031 = to_mt9t031(client);
 
-	a->c	= mt9t031->rect;
-	a->type	= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int mt9t031_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	a->bounds.left			= MT9T031_COLUMN_SKIP;
-	a->bounds.top			= MT9T031_ROW_SKIP;
-	a->bounds.width			= MT9T031_MAX_WIDTH;
-	a->bounds.height		= MT9T031_MAX_HEIGHT;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = MT9T031_COLUMN_SKIP;
+		sel->r.top = MT9T031_ROW_SKIP;
+		sel->r.width = MT9T031_MAX_WIDTH;
+		sel->r.height = MT9T031_MAX_HEIGHT;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r = mt9t031->rect;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int mt9t031_get_fmt(struct v4l2_subdev *sd,
@@ -721,9 +728,6 @@ static int mt9t031_s_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops mt9t031_subdev_video_ops = {
 	.s_stream	= mt9t031_s_stream,
-	.s_crop		= mt9t031_s_crop,
-	.g_crop		= mt9t031_g_crop,
-	.cropcap	= mt9t031_cropcap,
 	.g_mbus_config	= mt9t031_g_mbus_config,
 	.s_mbus_config	= mt9t031_s_mbus_config,
 };
@@ -734,6 +738,8 @@ static const struct v4l2_subdev_sensor_ops mt9t031_subdev_sensor_ops = {
 
 static const struct v4l2_subdev_pad_ops mt9t031_subdev_pad_ops = {
 	.enum_mbus_code = mt9t031_enum_mbus_code,
+	.get_selection	= mt9t031_get_selection,
+	.set_selection	= mt9t031_set_selection,
 	.get_fmt	= mt9t031_get_fmt,
 	.set_fmt	= mt9t031_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/mt9t112.c b/drivers/media/i2c/soc_camera/mt9t112.c
index 6a1b2a9f9a09..2ef22241ec14 100644
--- a/drivers/media/i2c/soc_camera/mt9t112.c
+++ b/drivers/media/i2c/soc_camera/mt9t112.c
@@ -867,39 +867,48 @@ static int mt9t112_set_params(struct mt9t112_priv *priv,
 	return 0;
 }
 
-static int mt9t112_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
-	a->bounds.width			= MAX_WIDTH;
-	a->bounds.height		= MAX_HEIGHT;
-	a->defrect.left			= 0;
-	a->defrect.top			= 0;
-	a->defrect.width		= VGA_WIDTH;
-	a->defrect.height		= VGA_HEIGHT;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
-
-	return 0;
-}
-
-static int mt9t112_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int mt9t112_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9t112_priv *priv = to_mt9t112(client);
 
-	a->c	= priv->frame;
-	a->type	= V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = MAX_WIDTH;
+		sel->r.height = MAX_HEIGHT;
+		return 0;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = VGA_WIDTH;
+		sel->r.height = VGA_HEIGHT;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r = priv->frame;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
-static int mt9t112_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
+static int mt9t112_set_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9t112_priv *priv = to_mt9t112(client);
-	const struct v4l2_rect *rect = &a->c;
+	const struct v4l2_rect *rect = &sel->r;
+
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
 
 	return mt9t112_set_params(priv, rect, priv->format->code);
 }
@@ -1024,15 +1033,14 @@ static int mt9t112_s_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops mt9t112_subdev_video_ops = {
 	.s_stream	= mt9t112_s_stream,
-	.cropcap	= mt9t112_cropcap,
-	.g_crop		= mt9t112_g_crop,
-	.s_crop		= mt9t112_s_crop,
 	.g_mbus_config	= mt9t112_g_mbus_config,
 	.s_mbus_config	= mt9t112_s_mbus_config,
 };
 
 static const struct v4l2_subdev_pad_ops mt9t112_subdev_pad_ops = {
 	.enum_mbus_code = mt9t112_enum_mbus_code,
+	.get_selection	= mt9t112_get_selection,
+	.set_selection	= mt9t112_set_selection,
 	.get_fmt	= mt9t112_get_fmt,
 	.set_fmt	= mt9t112_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/mt9v022.c b/drivers/media/i2c/soc_camera/mt9v022.c
index 2721e583bfa0..6a14ab5e4f2d 100644
--- a/drivers/media/i2c/soc_camera/mt9v022.c
+++ b/drivers/media/i2c/soc_camera/mt9v022.c
@@ -276,14 +276,20 @@ static int mt9v022_s_stream(struct v4l2_subdev *sd, int enable)
 	return 0;
 }
 
-static int mt9v022_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
+static int mt9v022_set_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9v022 *mt9v022 = to_mt9v022(client);
-	struct v4l2_rect rect = a->c;
+	struct v4l2_rect rect = sel->r;
 	int min_row, min_blank;
 	int ret;
 
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
+
 	/* Bayer format - even size lengths */
 	if (mt9v022->fmts == mt9v022_colour_fmts) {
 		rect.width	= ALIGN(rect.width, 2);
@@ -350,29 +356,30 @@ static int mt9v022_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
 	return 0;
 }
 
-static int mt9v022_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int mt9v022_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9v022 *mt9v022 = to_mt9v022(client);
 
-	a->c	= mt9v022->rect;
-	a->type	= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int mt9v022_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	a->bounds.left			= MT9V022_COLUMN_SKIP;
-	a->bounds.top			= MT9V022_ROW_SKIP;
-	a->bounds.width			= MT9V022_MAX_WIDTH;
-	a->bounds.height		= MT9V022_MAX_HEIGHT;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = MT9V022_COLUMN_SKIP;
+		sel->r.top = MT9V022_ROW_SKIP;
+		sel->r.width = MT9V022_MAX_WIDTH;
+		sel->r.height = MT9V022_MAX_HEIGHT;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r = mt9v022->rect;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int mt9v022_get_fmt(struct v4l2_subdev *sd,
@@ -400,13 +407,13 @@ static int mt9v022_s_fmt(struct v4l2_subdev *sd,
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct mt9v022 *mt9v022 = to_mt9v022(client);
-	struct v4l2_crop a = {
-		.c = {
-			.left	= mt9v022->rect.left,
-			.top	= mt9v022->rect.top,
-			.width	= mf->width,
-			.height	= mf->height,
-		},
+	struct v4l2_subdev_selection sel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = V4L2_SEL_TGT_CROP,
+		.r.left = mt9v022->rect.left,
+		.r.top = mt9v022->rect.top,
+		.r.width = mf->width,
+		.r.height = mf->height,
 	};
 	int ret;
 
@@ -430,7 +437,7 @@ static int mt9v022_s_fmt(struct v4l2_subdev *sd,
 	}
 
 	/* No support for scaling on this camera, just crop. */
-	ret = mt9v022_s_crop(sd, &a);
+	ret = mt9v022_set_selection(sd, NULL, &sel);
 	if (!ret) {
 		mf->width	= mt9v022->rect.width;
 		mf->height	= mt9v022->rect.height;
@@ -853,9 +860,6 @@ static int mt9v022_s_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops mt9v022_subdev_video_ops = {
 	.s_stream	= mt9v022_s_stream,
-	.s_crop		= mt9v022_s_crop,
-	.g_crop		= mt9v022_g_crop,
-	.cropcap	= mt9v022_cropcap,
 	.g_mbus_config	= mt9v022_g_mbus_config,
 	.s_mbus_config	= mt9v022_s_mbus_config,
 };
@@ -866,6 +870,8 @@ static const struct v4l2_subdev_sensor_ops mt9v022_subdev_sensor_ops = {
 
 static const struct v4l2_subdev_pad_ops mt9v022_subdev_pad_ops = {
 	.enum_mbus_code = mt9v022_enum_mbus_code,
+	.get_selection	= mt9v022_get_selection,
+	.set_selection	= mt9v022_set_selection,
 	.get_fmt	= mt9v022_get_fmt,
 	.set_fmt	= mt9v022_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/ov2640.c b/drivers/media/i2c/soc_camera/ov2640.c
index 9b4f5deec748..56de18263359 100644
--- a/drivers/media/i2c/soc_camera/ov2640.c
+++ b/drivers/media/i2c/soc_camera/ov2640.c
@@ -928,29 +928,25 @@ static int ov2640_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int ov2640_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
-{
-	a->c.left	= 0;
-	a->c.top	= 0;
-	a->c.width	= UXGA_WIDTH;
-	a->c.height	= UXGA_HEIGHT;
-	a->type		= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int ov2640_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
+static int ov2640_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
-	a->bounds.width			= UXGA_WIDTH;
-	a->bounds.height		= UXGA_HEIGHT;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP:
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = UXGA_WIDTH;
+		sel->r.height = UXGA_HEIGHT;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int ov2640_video_probe(struct i2c_client *client)
@@ -1024,13 +1020,12 @@ static int ov2640_g_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops ov2640_subdev_video_ops = {
 	.s_stream	= ov2640_s_stream,
-	.cropcap	= ov2640_cropcap,
-	.g_crop		= ov2640_g_crop,
 	.g_mbus_config	= ov2640_g_mbus_config,
 };
 
 static const struct v4l2_subdev_pad_ops ov2640_subdev_pad_ops = {
 	.enum_mbus_code = ov2640_enum_mbus_code,
+	.get_selection	= ov2640_get_selection,
 	.get_fmt	= ov2640_get_fmt,
 	.set_fmt	= ov2640_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/ov5642.c b/drivers/media/i2c/soc_camera/ov5642.c
index bab9ac0c1764..3d185bd622a3 100644
--- a/drivers/media/i2c/soc_camera/ov5642.c
+++ b/drivers/media/i2c/soc_camera/ov5642.c
@@ -850,13 +850,19 @@ static int ov5642_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int ov5642_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
+static int ov5642_set_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct ov5642 *priv = to_ov5642(client);
-	struct v4l2_rect rect = a->c;
+	struct v4l2_rect rect = sel->r;
 	int ret;
 
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
+
 	v4l_bound_align_image(&rect.width, 48, OV5642_MAX_WIDTH, 1,
 			      &rect.height, 32, OV5642_MAX_HEIGHT, 1, 0);
 
@@ -878,32 +884,30 @@ static int ov5642_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
 	return ret;
 }
 
-static int ov5642_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int ov5642_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct ov5642 *priv = to_ov5642(client);
-	struct v4l2_rect *rect = &a->c;
 
-	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
 		return -EINVAL;
 
-	*rect = priv->crop_rect;
-
-	return 0;
-}
-
-static int ov5642_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
-	a->bounds.width			= OV5642_MAX_WIDTH;
-	a->bounds.height		= OV5642_MAX_HEIGHT;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
-
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = OV5642_MAX_WIDTH;
+		sel->r.height = OV5642_MAX_HEIGHT;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r = priv->crop_rect;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int ov5642_g_mbus_config(struct v4l2_subdev *sd,
@@ -940,14 +944,13 @@ static int ov5642_s_power(struct v4l2_subdev *sd, int on)
 }
 
 static struct v4l2_subdev_video_ops ov5642_subdev_video_ops = {
-	.s_crop		= ov5642_s_crop,
-	.g_crop		= ov5642_g_crop,
-	.cropcap	= ov5642_cropcap,
 	.g_mbus_config	= ov5642_g_mbus_config,
 };
 
 static const struct v4l2_subdev_pad_ops ov5642_subdev_pad_ops = {
 	.enum_mbus_code = ov5642_enum_mbus_code,
+	.get_selection	= ov5642_get_selection,
+	.set_selection	= ov5642_set_selection,
 	.get_fmt	= ov5642_get_fmt,
 	.set_fmt	= ov5642_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/ov6650.c b/drivers/media/i2c/soc_camera/ov6650.c
index 1f8af1ee8352..4bf2995e1cb8 100644
--- a/drivers/media/i2c/soc_camera/ov6650.c
+++ b/drivers/media/i2c/soc_camera/ov6650.c
@@ -432,25 +432,43 @@ static int ov6650_s_power(struct v4l2_subdev *sd, int on)
 	return soc_camera_set_power(&client->dev, ssdd, priv->clk, on);
 }
 
-static int ov6650_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int ov6650_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct ov6650 *priv = to_ov6650(client);
 
-	a->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->c = priv->rect;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = DEF_HSTRT << 1;
+		sel->r.top = DEF_VSTRT << 1;
+		sel->r.width = W_CIF;
+		sel->r.height = H_CIF;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r = priv->rect;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
-static int ov6650_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
+static int ov6650_set_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct ov6650 *priv = to_ov6650(client);
-	struct v4l2_rect rect = a->c;
+	struct v4l2_rect rect = sel->r;
 	int ret;
 
-	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 
 	rect.left   = ALIGN(rect.left,   2);
@@ -483,22 +501,6 @@ static int ov6650_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
 	return ret;
 }
 
-static int ov6650_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-
-	a->bounds.left			= DEF_HSTRT << 1;
-	a->bounds.top			= DEF_VSTRT << 1;
-	a->bounds.width			= W_CIF;
-	a->bounds.height		= H_CIF;
-	a->defrect			= a->bounds;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
-
-	return 0;
-}
-
 static int ov6650_get_fmt(struct v4l2_subdev *sd,
 		struct v4l2_subdev_pad_config *cfg,
 		struct v4l2_subdev_format *format)
@@ -549,16 +551,15 @@ static int ov6650_s_fmt(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *mf)
 	struct soc_camera_sense *sense = icd->sense;
 	struct ov6650 *priv = to_ov6650(client);
 	bool half_scale = !is_unscaled_ok(mf->width, mf->height, &priv->rect);
-	struct v4l2_crop a = {
-		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
-		.c = {
-			.left	= priv->rect.left + (priv->rect.width >> 1) -
-					(mf->width >> (1 - half_scale)),
-			.top	= priv->rect.top + (priv->rect.height >> 1) -
-					(mf->height >> (1 - half_scale)),
-			.width	= mf->width << half_scale,
-			.height	= mf->height << half_scale,
-		},
+	struct v4l2_subdev_selection sel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = V4L2_SEL_TGT_CROP,
+		.r.left = priv->rect.left + (priv->rect.width >> 1) -
+			(mf->width >> (1 - half_scale)),
+		.r.top = priv->rect.top + (priv->rect.height >> 1) -
+			(mf->height >> (1 - half_scale)),
+		.r.width = mf->width << half_scale,
+		.r.height = mf->height << half_scale,
 	};
 	u32 code = mf->code;
 	unsigned long mclk, pclk;
@@ -672,7 +673,7 @@ static int ov6650_s_fmt(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *mf)
 	dev_dbg(&client->dev, "pixel clock divider: %ld.%ld\n",
 			mclk / pclk, 10 * mclk % pclk / pclk);
 
-	ret = ov6650_s_crop(sd, &a);
+	ret = ov6650_set_selection(sd, NULL, &sel);
 	if (!ret)
 		ret = ov6650_reg_rmw(client, REG_COMA, coma_set, coma_mask);
 	if (!ret)
@@ -943,9 +944,6 @@ static int ov6650_s_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops ov6650_video_ops = {
 	.s_stream	= ov6650_s_stream,
-	.cropcap	= ov6650_cropcap,
-	.g_crop		= ov6650_g_crop,
-	.s_crop		= ov6650_s_crop,
 	.g_parm		= ov6650_g_parm,
 	.s_parm		= ov6650_s_parm,
 	.g_mbus_config	= ov6650_g_mbus_config,
@@ -954,6 +952,8 @@ static struct v4l2_subdev_video_ops ov6650_video_ops = {
 
 static const struct v4l2_subdev_pad_ops ov6650_pad_ops = {
 	.enum_mbus_code = ov6650_enum_mbus_code,
+	.get_selection	= ov6650_get_selection,
+	.set_selection	= ov6650_set_selection,
 	.get_fmt	= ov6650_get_fmt,
 	.set_fmt	= ov6650_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/ov772x.c b/drivers/media/i2c/soc_camera/ov772x.c
index a43410c1e254..7e68762b3a4b 100644
--- a/drivers/media/i2c/soc_camera/ov772x.c
+++ b/drivers/media/i2c/soc_camera/ov772x.c
@@ -851,29 +851,28 @@ ov772x_set_fmt_error:
 	return ret;
 }
 
-static int ov772x_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
-{
-	a->c.left	= 0;
-	a->c.top	= 0;
-	a->c.width	= VGA_WIDTH;
-	a->c.height	= VGA_HEIGHT;
-	a->type		= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int ov772x_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
+static int ov772x_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
-	a->bounds.width			= OV772X_MAX_WIDTH;
-	a->bounds.height		= OV772X_MAX_HEIGHT;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	sel->r.left = 0;
+	sel->r.top = 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.width = OV772X_MAX_WIDTH;
+		sel->r.height = OV772X_MAX_HEIGHT;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r.width = VGA_WIDTH;
+		sel->r.height = VGA_HEIGHT;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int ov772x_get_fmt(struct v4l2_subdev *sd,
@@ -1030,13 +1029,12 @@ static int ov772x_g_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops ov772x_subdev_video_ops = {
 	.s_stream	= ov772x_s_stream,
-	.cropcap	= ov772x_cropcap,
-	.g_crop		= ov772x_g_crop,
 	.g_mbus_config	= ov772x_g_mbus_config,
 };
 
 static const struct v4l2_subdev_pad_ops ov772x_subdev_pad_ops = {
 	.enum_mbus_code = ov772x_enum_mbus_code,
+	.get_selection	= ov772x_get_selection,
 	.get_fmt	= ov772x_get_fmt,
 	.set_fmt	= ov772x_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/ov9640.c b/drivers/media/i2c/soc_camera/ov9640.c
index 8caae1c07541..8c93c57af71c 100644
--- a/drivers/media/i2c/soc_camera/ov9640.c
+++ b/drivers/media/i2c/soc_camera/ov9640.c
@@ -561,29 +561,25 @@ static int ov9640_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int ov9640_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
-{
-	a->c.left	= 0;
-	a->c.top	= 0;
-	a->c.width	= W_SXGA;
-	a->c.height	= H_SXGA;
-	a->type		= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int ov9640_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
+static int ov9640_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
-	a->bounds.width			= W_SXGA;
-	a->bounds.height		= H_SXGA;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	sel->r.left = 0;
+	sel->r.top = 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP:
+		sel->r.width = W_SXGA;
+		sel->r.height = H_SXGA;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int ov9640_video_probe(struct i2c_client *client)
@@ -667,13 +663,12 @@ static int ov9640_g_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops ov9640_video_ops = {
 	.s_stream	= ov9640_s_stream,
-	.cropcap	= ov9640_cropcap,
-	.g_crop		= ov9640_g_crop,
 	.g_mbus_config	= ov9640_g_mbus_config,
 };
 
 static const struct v4l2_subdev_pad_ops ov9640_pad_ops = {
 	.enum_mbus_code = ov9640_enum_mbus_code,
+	.get_selection	= ov9640_get_selection,
 	.set_fmt	= ov9640_set_fmt,
 };
 
diff --git a/drivers/media/i2c/soc_camera/ov9740.c b/drivers/media/i2c/soc_camera/ov9740.c
index 03a7fc7316ae..0da632d7d33a 100644
--- a/drivers/media/i2c/soc_camera/ov9740.c
+++ b/drivers/media/i2c/soc_camera/ov9740.c
@@ -737,29 +737,25 @@ static int ov9740_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int ov9740_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	a->bounds.left		= 0;
-	a->bounds.top		= 0;
-	a->bounds.width		= OV9740_MAX_WIDTH;
-	a->bounds.height	= OV9740_MAX_HEIGHT;
-	a->defrect		= a->bounds;
-	a->type			= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
-
-	return 0;
-}
-
-static int ov9740_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int ov9740_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
-	a->c.left		= 0;
-	a->c.top		= 0;
-	a->c.width		= OV9740_MAX_WIDTH;
-	a->c.height		= OV9740_MAX_HEIGHT;
-	a->type			= V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP:
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = OV9740_MAX_WIDTH;
+		sel->r.height = OV9740_MAX_HEIGHT;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 /* Set status of additional camera capabilities */
@@ -914,8 +910,6 @@ static int ov9740_g_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops ov9740_video_ops = {
 	.s_stream	= ov9740_s_stream,
-	.cropcap	= ov9740_cropcap,
-	.g_crop		= ov9740_g_crop,
 	.g_mbus_config	= ov9740_g_mbus_config,
 };
 
@@ -929,6 +923,7 @@ static struct v4l2_subdev_core_ops ov9740_core_ops = {
 
 static const struct v4l2_subdev_pad_ops ov9740_pad_ops = {
 	.enum_mbus_code = ov9740_enum_mbus_code,
+	.get_selection	= ov9740_get_selection,
 	.set_fmt	= ov9740_set_fmt,
 };
 
diff --git a/drivers/media/i2c/soc_camera/rj54n1cb0c.c b/drivers/media/i2c/soc_camera/rj54n1cb0c.c
index aa7bfbb4ad71..bc8ec59a3fbd 100644
--- a/drivers/media/i2c/soc_camera/rj54n1cb0c.c
+++ b/drivers/media/i2c/soc_camera/rj54n1cb0c.c
@@ -538,15 +538,21 @@ static int rj54n1_commit(struct i2c_client *client)
 static int rj54n1_sensor_scale(struct v4l2_subdev *sd, s32 *in_w, s32 *in_h,
 			       s32 *out_w, s32 *out_h);
 
-static int rj54n1_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
+static int rj54n1_set_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct rj54n1 *rj54n1 = to_rj54n1(client);
-	const struct v4l2_rect *rect = &a->c;
+	const struct v4l2_rect *rect = &sel->r;
 	int dummy = 0, output_w, output_h,
 		input_w = rect->width, input_h = rect->height;
 	int ret;
 
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
+
 	/* arbitrary minimum width and height, edges unimportant */
 	soc_camera_limit_side(&dummy, &input_w,
 		     RJ54N1_COLUMN_SKIP, 8, RJ54N1_MAX_WIDTH);
@@ -573,29 +579,30 @@ static int rj54n1_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
 	return 0;
 }
 
-static int rj54n1_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int rj54n1_get_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct rj54n1 *rj54n1 = to_rj54n1(client);
 
-	a->c	= rj54n1->rect;
-	a->type	= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int rj54n1_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	a->bounds.left			= RJ54N1_COLUMN_SKIP;
-	a->bounds.top			= RJ54N1_ROW_SKIP;
-	a->bounds.width			= RJ54N1_MAX_WIDTH;
-	a->bounds.height		= RJ54N1_MAX_HEIGHT;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = RJ54N1_COLUMN_SKIP;
+		sel->r.top = RJ54N1_ROW_SKIP;
+		sel->r.width = RJ54N1_MAX_WIDTH;
+		sel->r.height = RJ54N1_MAX_HEIGHT;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r = rj54n1->rect;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int rj54n1_get_fmt(struct v4l2_subdev *sd,
@@ -1246,15 +1253,14 @@ static int rj54n1_s_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops rj54n1_subdev_video_ops = {
 	.s_stream	= rj54n1_s_stream,
-	.g_crop		= rj54n1_g_crop,
-	.s_crop		= rj54n1_s_crop,
-	.cropcap	= rj54n1_cropcap,
 	.g_mbus_config	= rj54n1_g_mbus_config,
 	.s_mbus_config	= rj54n1_s_mbus_config,
 };
 
 static const struct v4l2_subdev_pad_ops rj54n1_subdev_pad_ops = {
 	.enum_mbus_code = rj54n1_enum_mbus_code,
+	.get_selection	= rj54n1_get_selection,
+	.set_selection	= rj54n1_set_selection,
 	.get_fmt	= rj54n1_get_fmt,
 	.set_fmt	= rj54n1_set_fmt,
 };
diff --git a/drivers/media/i2c/soc_camera/tw9910.c b/drivers/media/i2c/soc_camera/tw9910.c
index 06aff81787a7..4002c07f3857 100644
--- a/drivers/media/i2c/soc_camera/tw9910.c
+++ b/drivers/media/i2c/soc_camera/tw9910.c
@@ -676,44 +676,28 @@ tw9910_set_fmt_error:
 	return ret;
 }
 
-static int tw9910_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
+static int tw9910_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct tw9910_priv *priv = to_tw9910(client);
 
-	a->c.left	= 0;
-	a->c.top	= 0;
-	if (priv->norm & V4L2_STD_NTSC) {
-		a->c.width	= 640;
-		a->c.height	= 480;
-	} else {
-		a->c.width	= 768;
-		a->c.height	= 576;
-	}
-	a->type		= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int tw9910_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct tw9910_priv *priv = to_tw9910(client);
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
+	/* Only CROP, CROP_DEFAULT and CROP_BOUNDS are supported */
+	if (sel->target > V4L2_SEL_TGT_CROP_BOUNDS)
+		return -EINVAL;
 
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
+	sel->r.left	= 0;
+	sel->r.top	= 0;
 	if (priv->norm & V4L2_STD_NTSC) {
-		a->bounds.width		= 640;
-		a->bounds.height	= 480;
+		sel->r.width	= 640;
+		sel->r.height	= 480;
 	} else {
-		a->bounds.width		= 768;
-		a->bounds.height	= 576;
+		sel->r.width	= 768;
+		sel->r.height	= 576;
 	}
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
-
 	return 0;
 }
 
@@ -921,8 +905,6 @@ static struct v4l2_subdev_video_ops tw9910_subdev_video_ops = {
 	.s_std		= tw9910_s_std,
 	.g_std		= tw9910_g_std,
 	.s_stream	= tw9910_s_stream,
-	.cropcap	= tw9910_cropcap,
-	.g_crop		= tw9910_g_crop,
 	.g_mbus_config	= tw9910_g_mbus_config,
 	.s_mbus_config	= tw9910_s_mbus_config,
 	.g_tvnorms	= tw9910_g_tvnorms,
@@ -930,6 +912,7 @@ static struct v4l2_subdev_video_ops tw9910_subdev_video_ops = {
 
 static const struct v4l2_subdev_pad_ops tw9910_subdev_pad_ops = {
 	.enum_mbus_code = tw9910_enum_mbus_code,
+	.get_selection	= tw9910_get_selection,
 	.get_fmt	= tw9910_get_fmt,
 	.set_fmt	= tw9910_set_fmt,
 };
diff --git a/drivers/media/i2c/ths8200.c b/drivers/media/i2c/ths8200.c
index 73fc42bc2de6..42340e364cea 100644
--- a/drivers/media/i2c/ths8200.c
+++ b/drivers/media/i2c/ths8200.c
@@ -499,7 +499,6 @@ MODULE_DEVICE_TABLE(of, ths8200_of_match);
 
 static struct i2c_driver ths8200_driver = {
 	.driver = {
-		.owner = THIS_MODULE,
 		.name = "ths8200",
 		.of_match_table = of_match_ptr(ths8200_of_match),
 	},
diff --git a/drivers/media/i2c/tlv320aic23b.c b/drivers/media/i2c/tlv320aic23b.c
index 0370dd89f1fc..2e06c06cac9b 100644
--- a/drivers/media/i2c/tlv320aic23b.c
+++ b/drivers/media/i2c/tlv320aic23b.c
@@ -210,7 +210,6 @@ MODULE_DEVICE_TABLE(i2c, tlv320aic23b_id);
 
 static struct i2c_driver tlv320aic23b_driver = {
 	.driver = {
-		.owner	= THIS_MODULE,
 		.name	= "tlv320aic23b",
 	},
 	.probe		= tlv320aic23b_probe,
diff --git a/drivers/media/i2c/tvp514x.c b/drivers/media/i2c/tvp514x.c
index 7cdd94842938..d5c9347f4c6d 100644
--- a/drivers/media/i2c/tvp514x.c
+++ b/drivers/media/i2c/tvp514x.c
@@ -977,7 +977,7 @@ static const struct v4l2_subdev_ops tvp514x_ops = {
 	.pad = &tvp514x_pad_ops,
 };
 
-static struct tvp514x_decoder tvp514x_dev = {
+static const struct tvp514x_decoder tvp514x_dev = {
 	.streaming = 0,
 	.fmt_list = tvp514x_fmt_list,
 	.num_fmts = ARRAY_SIZE(tvp514x_fmt_list),
@@ -1233,7 +1233,6 @@ MODULE_DEVICE_TABLE(of, tvp514x_of_match);
 static struct i2c_driver tvp514x_driver = {
 	.driver = {
 		.of_match_table = of_match_ptr(tvp514x_of_match),
-		.owner = THIS_MODULE,
 		.name = TVP514X_MODULE_NAME,
 	},
 	.probe = tvp514x_probe,
diff --git a/drivers/media/i2c/tvp5150.c b/drivers/media/i2c/tvp5150.c
index 0b6d46c453bf..4740da39d698 100644
--- a/drivers/media/i2c/tvp5150.c
+++ b/drivers/media/i2c/tvp5150.c
@@ -871,19 +871,22 @@ static int tvp5150_fill_fmt(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int tvp5150_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
+static int tvp5150_set_selection(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_selection *sel)
 {
-	struct v4l2_rect rect = a->c;
 	struct tvp5150 *decoder = to_tvp5150(sd);
+	struct v4l2_rect rect = sel->r;
 	v4l2_std_id std;
-	unsigned int hmax;
+	int hmax;
+
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
 
 	v4l2_dbg(1, debug, sd, "%s left=%d, top=%d, width=%d, height=%d\n",
 		__func__, rect.left, rect.top, rect.width, rect.height);
 
-	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-
 	/* tvp5150 has some special limits */
 	rect.left = clamp(rect.left, 0, TVP5150_MAX_CROP_LEFT);
 	rect.width = clamp_t(unsigned int, rect.width,
@@ -924,44 +927,39 @@ static int tvp5150_s_crop(struct v4l2_subdev *sd, const struct v4l2_crop *a)
 	return 0;
 }
 
-static int tvp5150_g_crop(struct v4l2_subdev *sd, struct v4l2_crop *a)
-{
-	struct tvp5150 *decoder = to_tvp5150(sd);
-
-	a->c	= decoder->rect;
-	a->type	= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int tvp5150_cropcap(struct v4l2_subdev *sd, struct v4l2_cropcap *a)
+static int tvp5150_get_selection(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_selection *sel)
 {
-	struct tvp5150 *decoder = to_tvp5150(sd);
+	struct tvp5150 *decoder = container_of(sd, struct tvp5150, sd);
 	v4l2_std_id std;
 
-	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
 		return -EINVAL;
 
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
-	a->bounds.width			= TVP5150_H_MAX;
-
-	/* Calculate height based on current standard */
-	if (decoder->norm == V4L2_STD_ALL)
-		std = tvp5150_read_std(sd);
-	else
-		std = decoder->norm;
-
-	if (std & V4L2_STD_525_60)
-		a->bounds.height = TVP5150_V_MAX_525_60;
-	else
-		a->bounds.height = TVP5150_V_MAX_OTHERS;
-
-	a->defrect			= a->bounds;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
-
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = TVP5150_H_MAX;
+
+		/* Calculate height based on current standard */
+		if (decoder->norm == V4L2_STD_ALL)
+			std = tvp5150_read_std(sd);
+		else
+			std = decoder->norm;
+		if (std & V4L2_STD_525_60)
+			sel->r.height = TVP5150_V_MAX_525_60;
+		else
+			sel->r.height = TVP5150_V_MAX_OTHERS;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		sel->r = decoder->rect;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int tvp5150_g_mbus_config(struct v4l2_subdev *sd,
@@ -1173,7 +1171,7 @@ static int tvp5150_g_tuner(struct v4l2_subdev *sd, struct v4l2_tuner *vt)
 	return 0;
 }
 
-static int tvp5150_registered_async(struct v4l2_subdev *sd)
+static int tvp5150_registered(struct v4l2_subdev *sd)
 {
 #ifdef CONFIG_MEDIA_CONTROLLER
 	struct tvp5150 *decoder = to_tvp5150(sd);
@@ -1222,7 +1220,6 @@ static const struct v4l2_subdev_core_ops tvp5150_core_ops = {
 	.g_register = tvp5150_g_register,
 	.s_register = tvp5150_s_register,
 #endif
-	.registered_async = tvp5150_registered_async,
 };
 
 static const struct v4l2_subdev_tuner_ops tvp5150_tuner_ops = {
@@ -1233,9 +1230,6 @@ static const struct v4l2_subdev_video_ops tvp5150_video_ops = {
 	.s_std = tvp5150_s_std,
 	.s_stream = tvp5150_s_stream,
 	.s_routing = tvp5150_s_routing,
-	.s_crop = tvp5150_s_crop,
-	.g_crop = tvp5150_g_crop,
-	.cropcap = tvp5150_cropcap,
 	.g_mbus_config = tvp5150_g_mbus_config,
 };
 
@@ -1251,6 +1245,8 @@ static const struct v4l2_subdev_pad_ops tvp5150_pad_ops = {
 	.enum_frame_size = tvp5150_enum_frame_size,
 	.set_fmt = tvp5150_fill_fmt,
 	.get_fmt = tvp5150_fill_fmt,
+	.get_selection = tvp5150_get_selection,
+	.set_selection = tvp5150_set_selection,
 };
 
 static const struct v4l2_subdev_ops tvp5150_ops = {
@@ -1261,6 +1257,10 @@ static const struct v4l2_subdev_ops tvp5150_ops = {
 	.pad = &tvp5150_pad_ops,
 };
 
+static const struct v4l2_subdev_internal_ops tvp5150_internal_ops = {
+	.registered = tvp5150_registered,
+};
+
 
 /****************************************************************************
 			I2C Client & Driver
@@ -1474,6 +1474,7 @@ static int tvp5150_probe(struct i2c_client *c,
 	}
 
 	v4l2_i2c_subdev_init(sd, c, &tvp5150_ops);
+	sd->internal_ops = &tvp5150_internal_ops;
 	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 
 #if defined(CONFIG_MEDIA_CONTROLLER)
diff --git a/drivers/media/i2c/tvp7002.c b/drivers/media/i2c/tvp7002.c
index 4df640c3aa40..3dc3341c4896 100644
--- a/drivers/media/i2c/tvp7002.c
+++ b/drivers/media/i2c/tvp7002.c
@@ -1086,7 +1086,6 @@ MODULE_DEVICE_TABLE(of, tvp7002_of_match);
 static struct i2c_driver tvp7002_driver = {
 	.driver = {
 		.of_match_table = of_match_ptr(tvp7002_of_match),
-		.owner = THIS_MODULE,
 		.name = TVP7002_MODULE_NAME,
 	},
 	.probe = tvp7002_probe,
diff --git a/drivers/media/i2c/vs6624.c b/drivers/media/i2c/vs6624.c
index 4c72a18c0b8c..be4cb7a8bdeb 100644
--- a/drivers/media/i2c/vs6624.c
+++ b/drivers/media/i2c/vs6624.c
@@ -863,7 +863,6 @@ MODULE_DEVICE_TABLE(i2c, vs6624_id);
 
 static struct i2c_driver vs6624_driver = {
 	.driver = {
-		.owner  = THIS_MODULE,
 		.name   = "vs6624",
 	},
 	.probe          = vs6624_probe,
diff --git a/drivers/media/media-device.c b/drivers/media/media-device.c
index 1795abeda658..2783531f9fc0 100644
--- a/drivers/media/media-device.c
+++ b/drivers/media/media-device.c
@@ -59,27 +59,24 @@ static int media_device_close(struct file *filp)
 }
 
 static int media_device_get_info(struct media_device *dev,
-				 struct media_device_info __user *__info)
+				 struct media_device_info *info)
 {
-	struct media_device_info info;
-
-	memset(&info, 0, sizeof(info));
+	memset(info, 0, sizeof(*info));
 
 	if (dev->driver_name[0])
-		strlcpy(info.driver, dev->driver_name, sizeof(info.driver));
+		strlcpy(info->driver, dev->driver_name, sizeof(info->driver));
 	else
-		strlcpy(info.driver, dev->dev->driver->name, sizeof(info.driver));
+		strlcpy(info->driver, dev->dev->driver->name,
+			sizeof(info->driver));
 
-	strlcpy(info.model, dev->model, sizeof(info.model));
-	strlcpy(info.serial, dev->serial, sizeof(info.serial));
-	strlcpy(info.bus_info, dev->bus_info, sizeof(info.bus_info));
+	strlcpy(info->model, dev->model, sizeof(info->model));
+	strlcpy(info->serial, dev->serial, sizeof(info->serial));
+	strlcpy(info->bus_info, dev->bus_info, sizeof(info->bus_info));
 
-	info.media_version = MEDIA_API_VERSION;
-	info.hw_revision = dev->hw_revision;
-	info.driver_version = dev->driver_version;
+	info->media_version = MEDIA_API_VERSION;
+	info->hw_revision = dev->hw_revision;
+	info->driver_version = dev->driver_version;
 
-	if (copy_to_user(__info, &info, sizeof(*__info)))
-		return -EFAULT;
 	return 0;
 }
 
@@ -101,29 +98,25 @@ static struct media_entity *find_entity(struct media_device *mdev, u32 id)
 }
 
 static long media_device_enum_entities(struct media_device *mdev,
-				       struct media_entity_desc __user *uent)
+				       struct media_entity_desc *entd)
 {
 	struct media_entity *ent;
-	struct media_entity_desc u_ent;
-
-	memset(&u_ent, 0, sizeof(u_ent));
-	if (copy_from_user(&u_ent.id, &uent->id, sizeof(u_ent.id)))
-		return -EFAULT;
-
-	ent = find_entity(mdev, u_ent.id);
 
+	ent = find_entity(mdev, entd->id);
 	if (ent == NULL)
 		return -EINVAL;
 
-	u_ent.id = media_entity_id(ent);
+	memset(entd, 0, sizeof(*entd));
+
+	entd->id = media_entity_id(ent);
 	if (ent->name)
-		strlcpy(u_ent.name, ent->name, sizeof(u_ent.name));
-	u_ent.type = ent->function;
-	u_ent.revision = 0;		/* Unused */
-	u_ent.flags = ent->flags;
-	u_ent.group_id = 0;		/* Unused */
-	u_ent.pads = ent->num_pads;
-	u_ent.links = ent->num_links - ent->num_backlinks;
+		strlcpy(entd->name, ent->name, sizeof(entd->name));
+	entd->type = ent->function;
+	entd->revision = 0;		/* Unused */
+	entd->flags = ent->flags;
+	entd->group_id = 0;		/* Unused */
+	entd->pads = ent->num_pads;
+	entd->links = ent->num_links - ent->num_backlinks;
 
 	/*
 	 * Workaround for a bug at media-ctl <= v1.10 that makes it to
@@ -139,14 +132,13 @@ static long media_device_enum_entities(struct media_device *mdev,
 	if (ent->function < MEDIA_ENT_F_OLD_BASE ||
 	    ent->function > MEDIA_ENT_T_DEVNODE_UNKNOWN) {
 		if (is_media_entity_v4l2_subdev(ent))
-			u_ent.type = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
+			entd->type = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
 		else if (ent->function != MEDIA_ENT_F_IO_V4L)
-			u_ent.type = MEDIA_ENT_T_DEVNODE_UNKNOWN;
+			entd->type = MEDIA_ENT_T_DEVNODE_UNKNOWN;
 	}
 
-	memcpy(&u_ent.raw, &ent->info, sizeof(ent->info));
-	if (copy_to_user(uent, &u_ent, sizeof(u_ent)))
-		return -EFAULT;
+	memcpy(&entd->raw, &ent->info, sizeof(ent->info));
+
 	return 0;
 }
 
@@ -158,8 +150,8 @@ static void media_device_kpad_to_upad(const struct media_pad *kpad,
 	upad->flags = kpad->flags;
 }
 
-static long __media_device_enum_links(struct media_device *mdev,
-				      struct media_links_enum *links)
+static long media_device_enum_links(struct media_device *mdev,
+				    struct media_links_enum *links)
 {
 	struct media_entity *entity;
 
@@ -206,64 +198,35 @@ static long __media_device_enum_links(struct media_device *mdev,
 	return 0;
 }
 
-static long media_device_enum_links(struct media_device *mdev,
-				    struct media_links_enum __user *ulinks)
-{
-	struct media_links_enum links;
-	int rval;
-
-	if (copy_from_user(&links, ulinks, sizeof(links)))
-		return -EFAULT;
-
-	rval = __media_device_enum_links(mdev, &links);
-	if (rval < 0)
-		return rval;
-
-	if (copy_to_user(ulinks, &links, sizeof(*ulinks)))
-		return -EFAULT;
-
-	return 0;
-}
-
 static long media_device_setup_link(struct media_device *mdev,
-				    struct media_link_desc __user *_ulink)
+				    struct media_link_desc *linkd)
 {
 	struct media_link *link = NULL;
-	struct media_link_desc ulink;
 	struct media_entity *source;
 	struct media_entity *sink;
-	int ret;
-
-	if (copy_from_user(&ulink, _ulink, sizeof(ulink)))
-		return -EFAULT;
 
 	/* Find the source and sink entities and link.
 	 */
-	source = find_entity(mdev, ulink.source.entity);
-	sink = find_entity(mdev, ulink.sink.entity);
+	source = find_entity(mdev, linkd->source.entity);
+	sink = find_entity(mdev, linkd->sink.entity);
 
 	if (source == NULL || sink == NULL)
 		return -EINVAL;
 
-	if (ulink.source.index >= source->num_pads ||
-	    ulink.sink.index >= sink->num_pads)
+	if (linkd->source.index >= source->num_pads ||
+	    linkd->sink.index >= sink->num_pads)
 		return -EINVAL;
 
-	link = media_entity_find_link(&source->pads[ulink.source.index],
-				      &sink->pads[ulink.sink.index]);
+	link = media_entity_find_link(&source->pads[linkd->source.index],
+				      &sink->pads[linkd->sink.index]);
 	if (link == NULL)
 		return -EINVAL;
 
 	/* Setup the link on both entities. */
-	ret = __media_entity_setup_link(link, ulink.flags);
-
-	if (copy_to_user(_ulink, &ulink, sizeof(ulink)))
-		return -EFAULT;
-
-	return ret;
+	return __media_entity_setup_link(link, linkd->flags);
 }
 
-static long __media_device_get_topology(struct media_device *mdev,
+static long media_device_get_topology(struct media_device *mdev,
 				      struct media_v2_topology *topo)
 {
 	struct media_entity *entity;
@@ -400,63 +363,98 @@ static long __media_device_get_topology(struct media_device *mdev,
 	return ret;
 }
 
-static long media_device_get_topology(struct media_device *mdev,
-				      struct media_v2_topology __user *utopo)
+static long copy_arg_from_user(void *karg, void __user *uarg, unsigned int cmd)
 {
-	struct media_v2_topology ktopo;
-	int ret;
-
-	if (copy_from_user(&ktopo, utopo, sizeof(ktopo)))
+	/* All media IOCTLs are _IOWR() */
+	if (copy_from_user(karg, uarg, _IOC_SIZE(cmd)))
 		return -EFAULT;
 
-	ret = __media_device_get_topology(mdev, &ktopo);
-	if (ret < 0)
-		return ret;
+	return 0;
+}
 
-	if (copy_to_user(utopo, &ktopo, sizeof(*utopo)))
+static long copy_arg_to_user(void __user *uarg, void *karg, unsigned int cmd)
+{
+	/* All media IOCTLs are _IOWR() */
+	if (copy_to_user(uarg, karg, _IOC_SIZE(cmd)))
 		return -EFAULT;
 
 	return 0;
 }
 
+/* Do acquire the graph mutex */
+#define MEDIA_IOC_FL_GRAPH_MUTEX	BIT(0)
+
+#define MEDIA_IOC_ARG(__cmd, func, fl, from_user, to_user)		\
+	[_IOC_NR(MEDIA_IOC_##__cmd)] = {				\
+		.cmd = MEDIA_IOC_##__cmd,				\
+		.fn = (long (*)(struct media_device *, void *))func,	\
+		.flags = fl,						\
+		.arg_from_user = from_user,				\
+		.arg_to_user = to_user,					\
+	}
+
+#define MEDIA_IOC(__cmd, func, fl)					\
+	MEDIA_IOC_ARG(__cmd, func, fl, copy_arg_from_user, copy_arg_to_user)
+
+/* the table is indexed by _IOC_NR(cmd) */
+struct media_ioctl_info {
+	unsigned int cmd;
+	unsigned short flags;
+	long (*fn)(struct media_device *dev, void *arg);
+	long (*arg_from_user)(void *karg, void __user *uarg, unsigned int cmd);
+	long (*arg_to_user)(void __user *uarg, void *karg, unsigned int cmd);
+};
+
+static const struct media_ioctl_info ioctl_info[] = {
+	MEDIA_IOC(DEVICE_INFO, media_device_get_info, MEDIA_IOC_FL_GRAPH_MUTEX),
+	MEDIA_IOC(ENUM_ENTITIES, media_device_enum_entities, MEDIA_IOC_FL_GRAPH_MUTEX),
+	MEDIA_IOC(ENUM_LINKS, media_device_enum_links, MEDIA_IOC_FL_GRAPH_MUTEX),
+	MEDIA_IOC(SETUP_LINK, media_device_setup_link, MEDIA_IOC_FL_GRAPH_MUTEX),
+	MEDIA_IOC(G_TOPOLOGY, media_device_get_topology, MEDIA_IOC_FL_GRAPH_MUTEX),
+};
+
 static long media_device_ioctl(struct file *filp, unsigned int cmd,
-			       unsigned long arg)
+			       unsigned long __arg)
 {
 	struct media_devnode *devnode = media_devnode_data(filp);
 	struct media_device *dev = devnode->media_dev;
+	const struct media_ioctl_info *info;
+	void __user *arg = (void __user *)__arg;
+	char __karg[256], *karg = __karg;
 	long ret;
 
-	mutex_lock(&dev->graph_mutex);
-	switch (cmd) {
-	case MEDIA_IOC_DEVICE_INFO:
-		ret = media_device_get_info(dev,
-				(struct media_device_info __user *)arg);
-		break;
+	if (_IOC_NR(cmd) >= ARRAY_SIZE(ioctl_info)
+	    || ioctl_info[_IOC_NR(cmd)].cmd != cmd)
+		return -ENOIOCTLCMD;
 
-	case MEDIA_IOC_ENUM_ENTITIES:
-		ret = media_device_enum_entities(dev,
-				(struct media_entity_desc __user *)arg);
-		break;
+	info = &ioctl_info[_IOC_NR(cmd)];
 
-	case MEDIA_IOC_ENUM_LINKS:
-		ret = media_device_enum_links(dev,
-				(struct media_links_enum __user *)arg);
-		break;
+	if (_IOC_SIZE(info->cmd) > sizeof(__karg)) {
+		karg = kmalloc(_IOC_SIZE(info->cmd), GFP_KERNEL);
+		if (!karg)
+			return -ENOMEM;
+	}
 
-	case MEDIA_IOC_SETUP_LINK:
-		ret = media_device_setup_link(dev,
-				(struct media_link_desc __user *)arg);
-		break;
+	if (info->arg_from_user) {
+		ret = info->arg_from_user(karg, arg, cmd);
+		if (ret)
+			goto out_free;
+	}
 
-	case MEDIA_IOC_G_TOPOLOGY:
-		ret = media_device_get_topology(dev,
-				(struct media_v2_topology __user *)arg);
-		break;
+	if (info->flags & MEDIA_IOC_FL_GRAPH_MUTEX)
+		mutex_lock(&dev->graph_mutex);
 
-	default:
-		ret = -ENOIOCTLCMD;
-	}
-	mutex_unlock(&dev->graph_mutex);
+	ret = info->fn(dev, karg);
+
+	if (info->flags & MEDIA_IOC_FL_GRAPH_MUTEX)
+		mutex_unlock(&dev->graph_mutex);
+
+	if (!ret && info->arg_to_user)
+		ret = info->arg_to_user(arg, karg, cmd);
+
+out_free:
+	if (karg != __karg)
+		kfree(karg);
 
 	return ret;
 }
@@ -486,7 +484,7 @@ static long media_device_enum_links32(struct media_device *mdev,
 	links.pads = compat_ptr(pads_ptr);
 	links.links = compat_ptr(links_ptr);
 
-	return __media_device_enum_links(mdev, &links);
+	return media_device_enum_links(mdev, &links);
 }
 
 #define MEDIA_IOC_ENUM_LINKS32		_IOWR('|', 0x02, struct media_links_enum32)
diff --git a/drivers/media/media-entity.c b/drivers/media/media-entity.c
index d8a2299f0c2a..c68239e60487 100644
--- a/drivers/media/media-entity.c
+++ b/drivers/media/media-entity.c
@@ -65,6 +65,8 @@ static inline const char *intf_type(struct media_interface *intf)
 		return "v4l-subdev";
 	case MEDIA_INTF_T_V4L_SWRADIO:
 		return "v4l-swradio";
+	case MEDIA_INTF_T_V4L_TOUCH:
+		return "v4l-touch";
 	case MEDIA_INTF_T_ALSA_PCM_CAPTURE:
 		return "alsa-pcm-capture";
 	case MEDIA_INTF_T_ALSA_PCM_PLAYBACK:
@@ -806,17 +808,18 @@ int __media_entity_setup_link(struct media_link *link, u32 flags)
 
 	mdev = source->graph_obj.mdev;
 
-	if (mdev->link_notify) {
-		ret = mdev->link_notify(link, flags,
-					MEDIA_DEV_NOTIFY_PRE_LINK_CH);
+	if (mdev->ops && mdev->ops->link_notify) {
+		ret = mdev->ops->link_notify(link, flags,
+					     MEDIA_DEV_NOTIFY_PRE_LINK_CH);
 		if (ret < 0)
 			return ret;
 	}
 
 	ret = __media_entity_setup_link_notify(link, flags);
 
-	if (mdev->link_notify)
-		mdev->link_notify(link, flags, MEDIA_DEV_NOTIFY_POST_LINK_CH);
+	if (mdev->ops && mdev->ops->link_notify)
+		mdev->ops->link_notify(link, flags,
+				       MEDIA_DEV_NOTIFY_POST_LINK_CH);
 
 	return ret;
 }
diff --git a/drivers/media/pci/Kconfig b/drivers/media/pci/Kconfig
index 4f6467fbaeb4..da28e68c87d8 100644
--- a/drivers/media/pci/Kconfig
+++ b/drivers/media/pci/Kconfig
@@ -13,6 +13,7 @@ if MEDIA_CAMERA_SUPPORT
 source "drivers/media/pci/meye/Kconfig"
 source "drivers/media/pci/solo6x10/Kconfig"
 source "drivers/media/pci/sta2x11/Kconfig"
+source "drivers/media/pci/tw5864/Kconfig"
 source "drivers/media/pci/tw68/Kconfig"
 source "drivers/media/pci/tw686x/Kconfig"
 source "drivers/media/pci/zoran/Kconfig"
diff --git a/drivers/media/pci/Makefile b/drivers/media/pci/Makefile
index 2e54c36441f7..a7e8af0f64a7 100644
--- a/drivers/media/pci/Makefile
+++ b/drivers/media/pci/Makefile
@@ -31,3 +31,4 @@ obj-$(CONFIG_VIDEO_MEYE) += meye/
 obj-$(CONFIG_STA2X11_VIP) += sta2x11/
 obj-$(CONFIG_VIDEO_SOLO6X10) += solo6x10/
 obj-$(CONFIG_VIDEO_COBALT) += cobalt/
+obj-$(CONFIG_VIDEO_TW5864) += tw5864/
diff --git a/drivers/media/pci/bt8xx/bttv-driver.c b/drivers/media/pci/bt8xx/bttv-driver.c
index df54e17ef864..97b91a9f9fa9 100644
--- a/drivers/media/pci/bt8xx/bttv-driver.c
+++ b/drivers/media/pci/bt8xx/bttv-driver.c
@@ -2804,30 +2804,44 @@ static int bttv_cropcap(struct file *file, void *priv,
 	    cap->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
 		return -EINVAL;
 
-	*cap = bttv_tvnorms[btv->tvnorm].cropcap;
+	/* defrect and bounds are set via g_selection */
+	cap->pixelaspect = bttv_tvnorms[btv->tvnorm].cropcap.pixelaspect;
 
 	return 0;
 }
 
-static int bttv_g_crop(struct file *file, void *f, struct v4l2_crop *crop)
+static int bttv_g_selection(struct file *file, void *f, struct v4l2_selection *sel)
 {
 	struct bttv_fh *fh = f;
 	struct bttv *btv = fh->btv;
 
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
-	    crop->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
+	    sel->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
 		return -EINVAL;
 
-	/* No fh->do_crop = 1; because btv->crop[1] may be
-	   inconsistent with fh->width or fh->height and apps
-	   do not expect a change here. */
-
-	crop->c = btv->crop[!!fh->do_crop].rect;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		/*
+		 * No fh->do_crop = 1; because btv->crop[1] may be
+		 * inconsistent with fh->width or fh->height and apps
+		 * do not expect a change here.
+		 */
+		sel->r = btv->crop[!!fh->do_crop].rect;
+		break;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r = bttv_tvnorms[btv->tvnorm].cropcap.defrect;
+		break;
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r = bttv_tvnorms[btv->tvnorm].cropcap.bounds;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	return 0;
 }
 
-static int bttv_s_crop(struct file *file, void *f, const struct v4l2_crop *crop)
+static int bttv_s_selection(struct file *file, void *f, struct v4l2_selection *sel)
 {
 	struct bttv_fh *fh = f;
 	struct bttv *btv = fh->btv;
@@ -2839,8 +2853,11 @@ static int bttv_s_crop(struct file *file, void *f, const struct v4l2_crop *crop)
 	__s32 b_right;
 	__s32 b_bottom;
 
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
-	    crop->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
+	    sel->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+		return -EINVAL;
+
+	if (sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 
 	/* Make sure tvnorm, vbi_end and the current cropping
@@ -2864,22 +2881,24 @@ static int bttv_s_crop(struct file *file, void *f, const struct v4l2_crop *crop)
 	}
 
 	/* Min. scaled size 48 x 32. */
-	c.rect.left = clamp_t(s32, crop->c.left, b_left, b_right - 48);
+	c.rect.left = clamp_t(s32, sel->r.left, b_left, b_right - 48);
 	c.rect.left = min(c.rect.left, (__s32) MAX_HDELAY);
 
-	c.rect.width = clamp_t(s32, crop->c.width,
+	c.rect.width = clamp_t(s32, sel->r.width,
 			     48, b_right - c.rect.left);
 
-	c.rect.top = clamp_t(s32, crop->c.top, b_top, b_bottom - 32);
+	c.rect.top = clamp_t(s32, sel->r.top, b_top, b_bottom - 32);
 	/* Top and height must be a multiple of two. */
 	c.rect.top = (c.rect.top + 1) & ~1;
 
-	c.rect.height = clamp_t(s32, crop->c.height,
+	c.rect.height = clamp_t(s32, sel->r.height,
 			      32, b_bottom - c.rect.top);
 	c.rect.height = (c.rect.height + 1) & ~1;
 
 	bttv_crop_calc_limits(&c);
 
+	sel->r = c.rect;
+
 	btv->crop[1] = c;
 
 	fh->do_crop = 1;
@@ -3047,10 +3066,10 @@ static int bttv_open(struct file *file)
 	   which only change on request. These are stored in btv->crop[1].
 	   However for compatibility with V4L apps and cropping unaware
 	   V4L2 apps we now reset the cropping parameters as seen through
-	   this fh, which is to say VIDIOC_G_CROP and scaling limit checks
+	   this fh, which is to say VIDIOC_G_SELECTION and scaling limit checks
 	   will use btv->crop[0], the default cropping parameters for the
 	   current video standard, and VIDIOC_S_FMT will not implicitely
-	   change the cropping parameters until VIDIOC_S_CROP has been
+	   change the cropping parameters until VIDIOC_S_SELECTION has been
 	   called. */
 	fh->do_crop = !reset_crop; /* module parameter */
 
@@ -3159,8 +3178,8 @@ static const struct v4l2_ioctl_ops bttv_ioctl_ops = {
 	.vidioc_streamoff               = bttv_streamoff,
 	.vidioc_g_tuner                 = bttv_g_tuner,
 	.vidioc_s_tuner                 = bttv_s_tuner,
-	.vidioc_g_crop                  = bttv_g_crop,
-	.vidioc_s_crop                  = bttv_s_crop,
+	.vidioc_g_selection             = bttv_g_selection,
+	.vidioc_s_selection             = bttv_s_selection,
 	.vidioc_g_fbuf                  = bttv_g_fbuf,
 	.vidioc_s_fbuf                  = bttv_s_fbuf,
 	.vidioc_overlay                 = bttv_overlay,
diff --git a/drivers/media/pci/bt8xx/bttvp.h b/drivers/media/pci/bt8xx/bttvp.h
index b1e0023f923c..9efc4559fa8e 100644
--- a/drivers/media/pci/bt8xx/bttvp.h
+++ b/drivers/media/pci/bt8xx/bttvp.h
@@ -232,7 +232,7 @@ struct bttv_fh {
 	const struct bttv_format *ovfmt;
 	struct bttv_overlay      ov;
 
-	/* Application called VIDIOC_S_CROP. */
+	/* Application called VIDIOC_S_SELECTION. */
 	int                      do_crop;
 
 	/* vbi capture */
diff --git a/drivers/media/pci/cobalt/cobalt-alsa-pcm.c b/drivers/media/pci/cobalt/cobalt-alsa-pcm.c
index f0bdf10cfd57..49013c6b8646 100644
--- a/drivers/media/pci/cobalt/cobalt-alsa-pcm.c
+++ b/drivers/media/pci/cobalt/cobalt-alsa-pcm.c
@@ -510,7 +510,7 @@ static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs,
 	return vmalloc_to_page(pageptr);
 }
 
-static struct snd_pcm_ops snd_cobalt_pcm_capture_ops = {
+static const struct snd_pcm_ops snd_cobalt_pcm_capture_ops = {
 	.open		= snd_cobalt_pcm_capture_open,
 	.close		= snd_cobalt_pcm_capture_close,
 	.ioctl		= snd_cobalt_pcm_ioctl,
@@ -522,7 +522,7 @@ static struct snd_pcm_ops snd_cobalt_pcm_capture_ops = {
 	.page		= snd_pcm_get_vmalloc_page,
 };
 
-static struct snd_pcm_ops snd_cobalt_pcm_playback_ops = {
+static const struct snd_pcm_ops snd_cobalt_pcm_playback_ops = {
 	.open		= snd_cobalt_pcm_playback_open,
 	.close		= snd_cobalt_pcm_playback_close,
 	.ioctl		= snd_cobalt_pcm_ioctl,
diff --git a/drivers/media/pci/cobalt/cobalt-driver.c b/drivers/media/pci/cobalt/cobalt-driver.c
index 476f7f0dcf81..979634000597 100644
--- a/drivers/media/pci/cobalt/cobalt-driver.c
+++ b/drivers/media/pci/cobalt/cobalt-driver.c
@@ -60,30 +60,31 @@ MODULE_DESCRIPTION("cobalt driver");
 MODULE_LICENSE("GPL");
 
 static u8 edid[256] = {
-	0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00,
-	0x50, 0x21, 0x9C, 0x27, 0x00, 0x00, 0x00, 0x00,
-	0x19, 0x12, 0x01, 0x03, 0x80, 0x00, 0x00, 0x78,
-	0x0E, 0x00, 0xB2, 0xA0, 0x57, 0x49, 0x9B, 0x26,
-	0x10, 0x48, 0x4F, 0x2F, 0xCF, 0x00, 0x31, 0x59,
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00,
+	0x50, 0x21, 0x32, 0x27, 0x00, 0x00, 0x00, 0x00,
+	0x22, 0x1a, 0x01, 0x03, 0x80, 0x30, 0x1b, 0x78,
+	0x0f, 0xee, 0x91, 0xa3, 0x54, 0x4c, 0x99, 0x26,
+	0x0f, 0x50, 0x54, 0x2f, 0xcf, 0x00, 0x31, 0x59,
 	0x45, 0x59, 0x61, 0x59, 0x81, 0x99, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3A,
-	0x80, 0x18, 0x71, 0x38, 0x2D, 0x40, 0x58, 0x2C,
-	0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1E,
-	0x00, 0x00, 0x00, 0xFD, 0x00, 0x31, 0x55, 0x18,
-	0x5E, 0x11, 0x00, 0x0A, 0x20, 0x20, 0x20, 0x20,
-	0x20, 0x20, 0x00, 0x00, 0x00, 0xFC, 0x00, 0x43,
-	0x20, 0x39, 0x30, 0x0A, 0x0A, 0x0A, 0x0A, 0x0A,
-	0x0A, 0x0A, 0x0A, 0x0A, 0x00, 0x00, 0x00, 0x10,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a,
+	0x80, 0x18, 0x71, 0x38, 0x2d, 0x40, 0x58, 0x2c,
+	0x46, 0x00, 0xe0, 0x0e, 0x11, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfd, 0x00, 0x18, 0x55, 0x18,
+	0x5e, 0x11, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20,
+	0x20, 0x20, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x63,
+	0x6f, 0x62, 0x61, 0x6c, 0x74, 0x0a, 0x20, 0x20,
+	0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x10,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x9c,
+
+	0x02, 0x03, 0x1f, 0xf0, 0x4a, 0x90, 0x1f, 0x04,
+	0x13, 0x22, 0x21, 0x20, 0x02, 0x11, 0x01, 0x23,
+	0x09, 0x07, 0x07, 0x68, 0x03, 0x0c, 0x00, 0x10,
+	0x00, 0x00, 0x22, 0x0f, 0xe2, 0x00, 0xea, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x68,
-	0x02, 0x03, 0x1a, 0xc0, 0x48, 0xa2, 0x10, 0x04,
-	0x02, 0x01, 0x21, 0x14, 0x13, 0x23, 0x09, 0x07,
-	0x07, 0x65, 0x03, 0x0c, 0x00, 0x10, 0x00, 0xe2,
-	0x00, 0x2a, 0x01, 0x1d, 0x00, 0x80, 0x51, 0xd0,
-	0x1c, 0x20, 0x40, 0x80, 0x35, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x1e, 0x8c, 0x0a, 0xd0, 0x8a,
-	0x20, 0xe0, 0x2d, 0x10, 0x10, 0x3e, 0x96, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -91,7 +92,7 @@ static u8 edid[256] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd7
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa7,
 };
 
 static void cobalt_set_interrupt(struct cobalt *cobalt, bool enable)
diff --git a/drivers/media/pci/cobalt/cobalt-v4l2.c b/drivers/media/pci/cobalt/cobalt-v4l2.c
index d05672fe9ff9..5c76637900d0 100644
--- a/drivers/media/pci/cobalt/cobalt-v4l2.c
+++ b/drivers/media/pci/cobalt/cobalt-v4l2.c
@@ -161,8 +161,11 @@ static void cobalt_enable_output(struct cobalt_stream *s)
 	struct v4l2_subdev_format sd_fmt = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
+	u64 clk = bt->pixelclock;
 
-	if (!cobalt_cpld_set_freq(cobalt, bt->pixelclock)) {
+	if (bt->flags & V4L2_DV_FL_REDUCED_FPS)
+		clk = div_u64(clk * 1000ULL, 1001);
+	if (!cobalt_cpld_set_freq(cobalt, clk)) {
 		cobalt_err("pixelclock out of range\n");
 		return;
 	}
@@ -644,7 +647,7 @@ static int cobalt_s_dv_timings(struct file *file, void *priv_fh,
 		return 0;
 	}
 
-	if (v4l2_match_dv_timings(timings, &s->timings, 0, false))
+	if (v4l2_match_dv_timings(timings, &s->timings, 0, true))
 		return 0;
 
 	if (vb2_is_busy(&s->q))
diff --git a/drivers/media/pci/cx18/cx18-alsa-pcm.c b/drivers/media/pci/cx18/cx18-alsa-pcm.c
index ffb6acdc575f..5344510fbea3 100644
--- a/drivers/media/pci/cx18/cx18-alsa-pcm.c
+++ b/drivers/media/pci/cx18/cx18-alsa-pcm.c
@@ -311,7 +311,7 @@ static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs,
 	return vmalloc_to_page(pageptr);
 }
 
-static struct snd_pcm_ops snd_cx18_pcm_capture_ops = {
+static const struct snd_pcm_ops snd_cx18_pcm_capture_ops = {
 	.open		= snd_cx18_pcm_capture_open,
 	.close		= snd_cx18_pcm_capture_close,
 	.ioctl		= snd_cx18_pcm_ioctl,
diff --git a/drivers/media/pci/cx18/cx18-i2c.c b/drivers/media/pci/cx18/cx18-i2c.c
index 4af8cd6df95d..c9329371a3f8 100644
--- a/drivers/media/pci/cx18/cx18-i2c.c
+++ b/drivers/media/pci/cx18/cx18-i2c.c
@@ -98,7 +98,8 @@ static int cx18_i2c_new_ir(struct cx18 *cx, struct i2c_adapter *adap, u32 hw,
 	case CX18_HW_Z8F0811_IR_RX_HAUP:
 		init_data->ir_codes = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
-		init_data->type = RC_BIT_RC5;
+		init_data->type = RC_BIT_RC5 | RC_BIT_RC6_MCE |
+							RC_BIT_RC6_6A_32;
 		init_data->name = cx->card_name;
 		info.platform_data = init_data;
 		break;
diff --git a/drivers/media/pci/cx23885/altera-ci.h b/drivers/media/pci/cx23885/altera-ci.h
index 6c511723fd1b..57a40c84b46e 100644
--- a/drivers/media/pci/cx23885/altera-ci.h
+++ b/drivers/media/pci/cx23885/altera-ci.h
@@ -20,8 +20,6 @@
 #ifndef __ALTERA_CI_H
 #define __ALTERA_CI_H
 
-#include <linux/kconfig.h>
-
 #define ALT_DATA	0x000000ff
 #define ALT_TDI		0x00008000
 #define ALT_TDO		0x00004000
diff --git a/drivers/media/pci/cx23885/cx23885-417.c b/drivers/media/pci/cx23885/cx23885-417.c
index efec2d1a7afd..da892f3e3c29 100644
--- a/drivers/media/pci/cx23885/cx23885-417.c
+++ b/drivers/media/pci/cx23885/cx23885-417.c
@@ -1223,7 +1223,7 @@ static void cx23885_stop_streaming(struct vb2_queue *q)
 	cx23885_cancel_buffers(&dev->ts1);
 }
 
-static struct vb2_ops cx23885_qops = {
+static const struct vb2_ops cx23885_qops = {
 	.queue_setup    = queue_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_finish = buffer_finish,
@@ -1552,6 +1552,7 @@ int cx23885_417_register(struct cx23885_dev *dev)
 	q->mem_ops = &vb2_dma_sg_memops;
 	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
 	q->lock = &dev->lock;
+	q->dev = &dev->pci->dev;
 
 	err = vb2_queue_init(q);
 	if (err < 0)
diff --git a/drivers/media/pci/cx23885/cx23885-alsa.c b/drivers/media/pci/cx23885/cx23885-alsa.c
index ae7c2e89ad1c..6115d4e148ba 100644
--- a/drivers/media/pci/cx23885/cx23885-alsa.c
+++ b/drivers/media/pci/cx23885/cx23885-alsa.c
@@ -506,7 +506,7 @@ static struct page *snd_cx23885_page(struct snd_pcm_substream *substream,
 /*
  * operators
  */
-static struct snd_pcm_ops snd_cx23885_pcm_ops = {
+static const struct snd_pcm_ops snd_cx23885_pcm_ops = {
 	.open = snd_cx23885_pcm_open,
 	.close = snd_cx23885_close,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/media/pci/cx23885/cx23885-cards.c b/drivers/media/pci/cx23885/cx23885-cards.c
index 4abf50f2694f..99ba8d6328f0 100644
--- a/drivers/media/pci/cx23885/cx23885-cards.c
+++ b/drivers/media/pci/cx23885/cx23885-cards.c
@@ -770,6 +770,11 @@ struct cx23885_board cx23885_boards[] = {
 		.portb        = CX23885_MPEG_DVB,
 		.portc        = CX23885_MPEG_DVB,
 	},
+	[CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC] = {
+		.name        = "Hauppauge WinTV-QuadHD-ATSC",
+		.portb        = CX23885_MPEG_DVB,
+		.portc        = CX23885_MPEG_DVB,
+	},
 };
 const unsigned int cx23885_bcount = ARRAY_SIZE(cx23885_boards);
 
@@ -1073,6 +1078,14 @@ struct cx23885_subid cx23885_subids[] = {
 		.subvendor = 0x0070,
 		.subdevice = 0x6b28,
 		.card      = CX23885_BOARD_HAUPPAUGE_QUADHD_DVB, /* Tuner Pair 2 */
+	}, {
+		.subvendor = 0x0070,
+		.subdevice = 0x6a18,
+		.card      = CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC, /* Tuner Pair 1 */
+	}, {
+		.subvendor = 0x0070,
+		.subdevice = 0x6b18,
+		.card      = CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC, /* Tuner Pair 2 */
 	},
 };
 const unsigned int cx23885_idcount = ARRAY_SIZE(cx23885_subids);
@@ -1278,6 +1291,18 @@ static void hauppauge_eeprom(struct cx23885_dev *dev, u8 *eeprom_data)
 		/* WinTV-QuadHD (DVB) Tuner Pair 2 (PCIe, IR, half height,
 		   DVB-T/T2/C, DVB-T/T2/C */
 		break;
+	case 165100:
+		/*
+		 * WinTV-QuadHD (ATSC) Tuner Pair 1 (PCIe, IR, half height,
+		 * ATSC, ATSC
+		 */
+		break;
+	case 165101:
+		/*
+		 * WinTV-QuadHD (DVB) Tuner Pair 2 (PCIe, IR, half height,
+		 * ATSC, ATSC
+		 */
+		break;
 	default:
 		printk(KERN_WARNING "%s: warning: "
 			"unknown hauppauge model #%d\n",
@@ -1751,6 +1776,7 @@ void cx23885_gpio_setup(struct cx23885_dev *dev)
 		break;
 	case CX23885_BOARD_HAUPPAUGE_HVR5525:
 	case CX23885_BOARD_HAUPPAUGE_QUADHD_DVB:
+	case CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC:
 		/*
 		 * HVR5525 GPIO Details:
 		 *  GPIO-00 IR_WIDE
@@ -1826,6 +1852,7 @@ int cx23885_ir_init(struct cx23885_dev *dev)
 	case CX23885_BOARD_HAUPPAUGE_HVR1255_22111:
 	case CX23885_BOARD_HAUPPAUGE_HVR1210:
 	case CX23885_BOARD_HAUPPAUGE_QUADHD_DVB:
+	case CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC:
 		/* FIXME: Implement me */
 		break;
 	case CX23885_BOARD_HAUPPAUGE_HVR1270:
@@ -2025,6 +2052,7 @@ void cx23885_card_setup(struct cx23885_dev *dev)
 	case CX23885_BOARD_HAUPPAUGE_IMPACTVCBE:
 	case CX23885_BOARD_HAUPPAUGE_HVR5525:
 	case CX23885_BOARD_HAUPPAUGE_QUADHD_DVB:
+	case CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC:
 		if (dev->i2c_bus[0].i2c_rc == 0)
 			hauppauge_eeprom(dev, eeprom+0xc0);
 		break;
@@ -2171,6 +2199,7 @@ void cx23885_card_setup(struct cx23885_dev *dev)
 		ts2->src_sel_val   = CX23885_SRC_SEL_PARALLEL_MPEG_VIDEO;
 		break;
 	case CX23885_BOARD_HAUPPAUGE_QUADHD_DVB:
+	case CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC:
 		ts1->gen_ctrl_val  = 0xc; /* Serial bus + punctured clock */
 		ts1->ts_clk_en_val = 0x1; /* Enable TS_CLK */
 		ts1->src_sel_val   = CX23885_SRC_SEL_PARALLEL_MPEG_VIDEO;
diff --git a/drivers/media/pci/cx23885/cx23885-dvb.c b/drivers/media/pci/cx23885/cx23885-dvb.c
index e5748a93c479..818f3c2fc98d 100644
--- a/drivers/media/pci/cx23885/cx23885-dvb.c
+++ b/drivers/media/pci/cx23885/cx23885-dvb.c
@@ -74,6 +74,7 @@
 #include "sp2.h"
 #include "m88ds3103.h"
 #include "m88rs6000t.h"
+#include "lgdt3306a.h"
 
 static unsigned int debug;
 
@@ -172,7 +173,7 @@ static void cx23885_stop_streaming(struct vb2_queue *q)
 	cx23885_cancel_buffers(port);
 }
 
-static struct vb2_ops dvb_qops = {
+static const struct vb2_ops dvb_qops = {
 	.queue_setup    = queue_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_finish = buffer_finish,
@@ -574,6 +575,30 @@ static struct stb6100_config prof_8000_stb6100_config = {
 	.refclock = 27000000,
 };
 
+static struct lgdt3306a_config hauppauge_quadHD_ATSC_a_config = {
+	.i2c_addr               = 0x59,
+	.qam_if_khz             = 4000,
+	.vsb_if_khz             = 3250,
+	.deny_i2c_rptr          = 1, /* Disabled */
+	.spectral_inversion     = 0, /* Disabled */
+	.mpeg_mode              = LGDT3306A_MPEG_SERIAL,
+	.tpclk_edge             = LGDT3306A_TPCLK_RISING_EDGE,
+	.tpvalid_polarity       = LGDT3306A_TP_VALID_HIGH,
+	.xtalMHz                = 25, /* 24 or 25 */
+};
+
+static struct lgdt3306a_config hauppauge_quadHD_ATSC_b_config = {
+	.i2c_addr               = 0x0e,
+	.qam_if_khz             = 4000,
+	.vsb_if_khz             = 3250,
+	.deny_i2c_rptr          = 1, /* Disabled */
+	.spectral_inversion     = 0, /* Disabled */
+	.mpeg_mode              = LGDT3306A_MPEG_SERIAL,
+	.tpclk_edge             = LGDT3306A_TPCLK_RISING_EDGE,
+	.tpvalid_polarity       = LGDT3306A_TP_VALID_HIGH,
+	.xtalMHz                = 25, /* 24 or 25 */
+};
+
 static int p8000_set_voltage(struct dvb_frontend *fe,
 			     enum fe_sec_voltage voltage)
 {
@@ -867,12 +892,6 @@ static const struct tda10071_platform_data hauppauge_tda10071_pdata = {
 	.tuner_i2c_addr = 0x54,
 };
 
-static const struct si2165_config hauppauge_hvr4400_si2165_config = {
-	.i2c_addr	= 0x64,
-	.chip_mode	= SI2165_MODE_PLL_XTAL,
-	.ref_freq_Hz	= 16000000,
-};
-
 static const struct m88ds3103_config dvbsky_t9580_m88ds3103_config = {
 	.i2c_addr = 0x68,
 	.clock = 27000000,
@@ -1182,6 +1201,7 @@ static int dvb_register(struct cx23885_tsport *port)
 	struct cx23885_i2c *i2c_bus = NULL, *i2c_bus2 = NULL;
 	struct vb2_dvb_frontend *fe0, *fe1 = NULL;
 	struct si2168_config si2168_config;
+	struct si2165_platform_data si2165_pdata;
 	struct si2157_config si2157_config;
 	struct ts2020_config ts2020_config;
 	struct i2c_board_info info;
@@ -1700,6 +1720,9 @@ static int dvb_register(struct cx23885_tsport *port)
 		}
 		break;
 	case CX23885_BOARD_NETUP_DUAL_DVB_T_C_CI_RF:
+		if (port->nr > 2)
+			return 0;
+
 		i2c_bus = &dev->i2c_bus[0];
 		mfe_shared = 1;/* MFE */
 		port->frontends.gate = 0;/* not clear for me yet */
@@ -1839,9 +1862,26 @@ static int dvb_register(struct cx23885_tsport *port)
 			break;
 		/* port c */
 		case 2:
-			fe0->dvb.frontend = dvb_attach(si2165_attach,
-					&hauppauge_hvr4400_si2165_config,
-					&i2c_bus->i2c_adap);
+			/* attach frontend */
+			memset(&si2165_pdata, 0, sizeof(si2165_pdata));
+			si2165_pdata.fe = &fe0->dvb.frontend;
+			si2165_pdata.chip_mode = SI2165_MODE_PLL_XTAL,
+			si2165_pdata.ref_freq_Hz = 16000000,
+			memset(&info, 0, sizeof(struct i2c_board_info));
+			strlcpy(info.type, "si2165", I2C_NAME_SIZE);
+			info.addr = 0x64;
+			info.platform_data = &si2165_pdata;
+			request_module(info.type);
+			client_demod = i2c_new_device(&i2c_bus->i2c_adap, &info);
+			if (client_demod == NULL ||
+					client_demod->dev.driver == NULL)
+				goto frontend_detach;
+			if (!try_module_get(client_demod->dev.driver->owner)) {
+				i2c_unregister_device(client_demod);
+				goto frontend_detach;
+			}
+			port->i2c_client_demod = client_demod;
+
 			if (fe0->dvb.frontend == NULL)
 				break;
 			fe0->dvb.frontend->ops.i2c_gate_ctrl = NULL;
@@ -2365,6 +2405,81 @@ static int dvb_register(struct cx23885_tsport *port)
 			break;
 		}
 		break;
+	case CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC:
+		switch (port->nr) {
+		/* port b - Terrestrial/cable */
+		case 1:
+			/* attach frontend */
+			i2c_bus = &dev->i2c_bus[0];
+			fe0->dvb.frontend = dvb_attach(lgdt3306a_attach,
+				&hauppauge_quadHD_ATSC_a_config, &i2c_bus->i2c_adap);
+			if (fe0->dvb.frontend == NULL)
+				break;
+
+			/* attach tuner */
+			memset(&si2157_config, 0, sizeof(si2157_config));
+			si2157_config.fe = fe0->dvb.frontend;
+			si2157_config.if_port = 1;
+			si2157_config.inversion = 1;
+			memset(&info, 0, sizeof(struct i2c_board_info));
+			strlcpy(info.type, "si2157", I2C_NAME_SIZE);
+			info.addr = 0x60;
+			info.platform_data = &si2157_config;
+			request_module("%s", info.type);
+			client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, &info);
+			if (!client_tuner || !client_tuner->dev.driver) {
+				module_put(client_demod->dev.driver->owner);
+				i2c_unregister_device(client_demod);
+				port->i2c_client_demod = NULL;
+				goto frontend_detach;
+			}
+			if (!try_module_get(client_tuner->dev.driver->owner)) {
+				i2c_unregister_device(client_tuner);
+				module_put(client_demod->dev.driver->owner);
+				i2c_unregister_device(client_demod);
+				port->i2c_client_demod = NULL;
+				goto frontend_detach;
+			}
+			port->i2c_client_tuner = client_tuner;
+			break;
+
+		/* port c - terrestrial/cable */
+		case 2:
+			/* attach frontend */
+			i2c_bus = &dev->i2c_bus[0];
+			fe0->dvb.frontend = dvb_attach(lgdt3306a_attach,
+				&hauppauge_quadHD_ATSC_b_config, &i2c_bus->i2c_adap);
+			if (fe0->dvb.frontend == NULL)
+				break;
+
+			/* attach tuner */
+			memset(&si2157_config, 0, sizeof(si2157_config));
+			si2157_config.fe = fe0->dvb.frontend;
+			si2157_config.if_port = 1;
+			si2157_config.inversion = 1;
+			memset(&info, 0, sizeof(struct i2c_board_info));
+			strlcpy(info.type, "si2157", I2C_NAME_SIZE);
+			info.addr = 0x62;
+			info.platform_data = &si2157_config;
+			request_module("%s", info.type);
+			client_tuner = i2c_new_device(&dev->i2c_bus[1].i2c_adap, &info);
+			if (!client_tuner || !client_tuner->dev.driver) {
+				module_put(client_demod->dev.driver->owner);
+				i2c_unregister_device(client_demod);
+				port->i2c_client_demod = NULL;
+				goto frontend_detach;
+			}
+			if (!try_module_get(client_tuner->dev.driver->owner)) {
+				i2c_unregister_device(client_tuner);
+				module_put(client_demod->dev.driver->owner);
+				i2c_unregister_device(client_demod);
+				port->i2c_client_demod = NULL;
+				goto frontend_detach;
+			}
+			port->i2c_client_tuner = client_tuner;
+			break;
+		}
+		break;
 
 	default:
 		printk(KERN_INFO "%s: The frontend of your DVB/ATSC card "
diff --git a/drivers/media/pci/cx23885/cx23885-i2c.c b/drivers/media/pci/cx23885/cx23885-i2c.c
index ae061b358591..61591225be9a 100644
--- a/drivers/media/pci/cx23885/cx23885-i2c.c
+++ b/drivers/media/pci/cx23885/cx23885-i2c.c
@@ -258,7 +258,7 @@ static u32 cx23885_functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_SMBUS_EMUL | I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm cx23885_i2c_algo_template = {
+static const struct i2c_algorithm cx23885_i2c_algo_template = {
 	.master_xfer	= i2c_xfer,
 	.functionality	= cx23885_functionality,
 };
diff --git a/drivers/media/pci/cx23885/cx23885-input.c b/drivers/media/pci/cx23885/cx23885-input.c
index 64328d08ac2f..410c3141c163 100644
--- a/drivers/media/pci/cx23885/cx23885-input.c
+++ b/drivers/media/pci/cx23885/cx23885-input.c
@@ -293,7 +293,7 @@ int cx23885_input_init(struct cx23885_dev *dev)
 	case CX23885_BOARD_TERRATEC_CINERGY_T_PCIE_DUAL:
 		/* Integrated CX23885 IR controller */
 		driver_type = RC_DRIVER_IR_RAW;
-		allowed_protos = RC_BIT_NEC;
+		allowed_protos = RC_BIT_ALL;
 		/* The grey Terratec remote with orange buttons */
 		rc_map = RC_MAP_NEC_TERRATEC_CINERGY_XS;
 		break;
diff --git a/drivers/media/pci/cx23885/cx23885-video.c b/drivers/media/pci/cx23885/cx23885-video.c
index 6d735222a958..33d168ef278d 100644
--- a/drivers/media/pci/cx23885/cx23885-video.c
+++ b/drivers/media/pci/cx23885/cx23885-video.c
@@ -517,7 +517,7 @@ static void cx23885_stop_streaming(struct vb2_queue *q)
 	spin_unlock_irqrestore(&dev->slock, flags);
 }
 
-static struct vb2_ops cx23885_video_qops = {
+static const struct vb2_ops cx23885_video_qops = {
 	.queue_setup    = queue_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_finish = buffer_finish,
diff --git a/drivers/media/pci/cx23885/cx23885.h b/drivers/media/pci/cx23885/cx23885.h
index 24a0a6c5b501..a6735afe2269 100644
--- a/drivers/media/pci/cx23885/cx23885.h
+++ b/drivers/media/pci/cx23885/cx23885.h
@@ -103,7 +103,8 @@
 #define CX23885_BOARD_HAUPPAUGE_STARBURST      53
 #define CX23885_BOARD_VIEWCAST_260E            54
 #define CX23885_BOARD_VIEWCAST_460E            55
-#define CX23885_BOARD_HAUPPAUGE_QUADHD_DVB    56
+#define CX23885_BOARD_HAUPPAUGE_QUADHD_DVB     56
+#define CX23885_BOARD_HAUPPAUGE_QUADHD_ATSC    57
 
 #define GPIO_0 0x00000001
 #define GPIO_1 0x00000002
@@ -256,7 +257,7 @@ struct cx23885_dmaqueue {
 struct cx23885_tsport {
 	struct cx23885_dev *dev;
 
-	int                        nr;
+	unsigned                   nr;
 	int                        sram_chno;
 
 	struct vb2_dvb_frontends   frontends;
diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c
index df189b16af12..4711583de8fe 100644
--- a/drivers/media/pci/cx25821/cx25821-alsa.c
+++ b/drivers/media/pci/cx25821/cx25821-alsa.c
@@ -649,7 +649,7 @@ static struct page *snd_cx25821_page(struct snd_pcm_substream *substream,
 /*
  * operators
  */
-static struct snd_pcm_ops snd_cx25821_pcm_ops = {
+static const struct snd_pcm_ops snd_cx25821_pcm_ops = {
 	.open = snd_cx25821_pcm_open,
 	.close = snd_cx25821_close,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/media/pci/cx25821/cx25821-audio-upstream.c b/drivers/media/pci/cx25821/cx25821-audio-upstream.c
index 68dbc2dbc982..7c8edb6181ec 100644
--- a/drivers/media/pci/cx25821/cx25821-audio-upstream.c
+++ b/drivers/media/pci/cx25821/cx25821-audio-upstream.c
@@ -242,8 +242,7 @@ void cx25821_stop_upstream_audio(struct cx25821_dev *dev)
 	dev->_audioframe_count = 0;
 	dev->_audiofile_status = END_OF_FILE;
 
-	kfree(dev->_irq_audio_queues);
-	dev->_irq_audio_queues = NULL;
+	flush_work(&dev->_audio_work_entry);
 
 	kfree(dev->_audiofilename);
 }
@@ -446,8 +445,7 @@ static int cx25821_audio_upstream_irq(struct cx25821_dev *dev, int chan_num,
 
 			dev->_audioframe_index = dev->_last_index_irq;
 
-			queue_work(dev->_irq_audio_queues,
-				   &dev->_audio_work_entry);
+			schedule_work(&dev->_audio_work_entry);
 		}
 
 		if (dev->_is_first_audio_frame) {
@@ -639,14 +637,6 @@ int cx25821_audio_upstream_init(struct cx25821_dev *dev, int channel_select)
 
 	/* Work queue */
 	INIT_WORK(&dev->_audio_work_entry, cx25821_audioups_handler);
-	dev->_irq_audio_queues =
-	    create_singlethread_workqueue("cx25821_audioworkqueue");
-
-	if (!dev->_irq_audio_queues) {
-		printk(KERN_DEBUG
-			pr_fmt("ERROR: create_singlethread_workqueue() for Audio FAILED!\n"));
-		return -ENOMEM;
-	}
 
 	dev->_last_index_irq = 0;
 	dev->_audio_is_running = 0;
diff --git a/drivers/media/pci/cx25821/cx25821-i2c.c b/drivers/media/pci/cx25821/cx25821-i2c.c
index dca37c7dba73..63ba25b82692 100644
--- a/drivers/media/pci/cx25821/cx25821-i2c.c
+++ b/drivers/media/pci/cx25821/cx25821-i2c.c
@@ -281,7 +281,7 @@ static u32 cx25821_functionality(struct i2c_adapter *adap)
 		I2C_FUNC_SMBUS_READ_WORD_DATA | I2C_FUNC_SMBUS_WRITE_WORD_DATA;
 }
 
-static struct i2c_algorithm cx25821_i2c_algo_template = {
+static const struct i2c_algorithm cx25821_i2c_algo_template = {
 	.master_xfer = i2c_xfer,
 	.functionality = cx25821_functionality,
 #ifdef NEED_ALGO_CONTROL
diff --git a/drivers/media/pci/cx25821/cx25821-video.c b/drivers/media/pci/cx25821/cx25821-video.c
index adcd09be347d..7ce352a0f2d3 100644
--- a/drivers/media/pci/cx25821/cx25821-video.c
+++ b/drivers/media/pci/cx25821/cx25821-video.c
@@ -307,7 +307,7 @@ static void cx25821_stop_streaming(struct vb2_queue *q)
 	spin_unlock_irqrestore(&dev->slock, flags);
 }
 
-static struct vb2_ops cx25821_video_qops = {
+static const struct vb2_ops cx25821_video_qops = {
 	.queue_setup    = cx25821_queue_setup,
 	.buf_prepare  = cx25821_buffer_prepare,
 	.buf_finish = cx25821_buffer_finish,
diff --git a/drivers/media/pci/cx25821/cx25821.h b/drivers/media/pci/cx25821/cx25821.h
index 35c7375e4617..ef61dea982e8 100644
--- a/drivers/media/pci/cx25821/cx25821.h
+++ b/drivers/media/pci/cx25821/cx25821.h
@@ -293,7 +293,6 @@ struct cx25821_dev {
 	u32 audio_upstream_riscbuf_size;
 	u32 audio_upstream_databuf_size;
 	int _audioframe_index;
-	struct workqueue_struct *_irq_audio_queues;
 	struct work_struct _audio_work_entry;
 	char *input_audiofilename;
 
diff --git a/drivers/media/pci/cx88/cx88-alsa.c b/drivers/media/pci/cx88/cx88-alsa.c
index f3f13eb0c16e..723f06462104 100644
--- a/drivers/media/pci/cx88/cx88-alsa.c
+++ b/drivers/media/pci/cx88/cx88-alsa.c
@@ -599,7 +599,7 @@ static struct page *snd_cx88_page(struct snd_pcm_substream *substream,
 /*
  * operators
  */
-static struct snd_pcm_ops snd_cx88_pcm_ops = {
+static const struct snd_pcm_ops snd_cx88_pcm_ops = {
 	.open = snd_cx88_pcm_open,
 	.close = snd_cx88_close,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/media/pci/cx88/cx88-blackbird.c b/drivers/media/pci/cx88/cx88-blackbird.c
index 04fe9af2a802..b532e49e8f33 100644
--- a/drivers/media/pci/cx88/cx88-blackbird.c
+++ b/drivers/media/pci/cx88/cx88-blackbird.c
@@ -756,7 +756,7 @@ static void stop_streaming(struct vb2_queue *q)
 	spin_unlock_irqrestore(&dev->slock, flags);
 }
 
-static struct vb2_ops blackbird_qops = {
+static const struct vb2_ops blackbird_qops = {
 	.queue_setup    = queue_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_finish = buffer_finish,
diff --git a/drivers/media/pci/cx88/cx88-dvb.c b/drivers/media/pci/cx88/cx88-dvb.c
index 5bb63e7a5691..ac2392d8887a 100644
--- a/drivers/media/pci/cx88/cx88-dvb.c
+++ b/drivers/media/pci/cx88/cx88-dvb.c
@@ -156,7 +156,7 @@ static void stop_streaming(struct vb2_queue *q)
 	spin_unlock_irqrestore(&dev->slock, flags);
 }
 
-static struct vb2_ops dvb_qops = {
+static const struct vb2_ops dvb_qops = {
 	.queue_setup    = queue_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_finish = buffer_finish,
diff --git a/drivers/media/pci/cx88/cx88-input.c b/drivers/media/pci/cx88/cx88-input.c
index 3f1342c98b46..cd7687183381 100644
--- a/drivers/media/pci/cx88/cx88-input.c
+++ b/drivers/media/pci/cx88/cx88-input.c
@@ -144,7 +144,8 @@ static void cx88_ir_handle_key(struct cx88_IR *ir)
 		scancode = RC_SCANCODE_NECX(addr, cmd);
 
 		if (0 == (gpio & ir->mask_keyup))
-			rc_keydown_notimeout(ir->dev, RC_TYPE_NEC, scancode, 0);
+			rc_keydown_notimeout(ir->dev, RC_TYPE_NECX, scancode,
+									0);
 		else
 			rc_keyup(ir->dev);
 
@@ -345,7 +346,7 @@ int cx88_ir_init(struct cx88_core *core, struct pci_dev *pci)
 		 * 002-T mini RC, provided with newer PV hardware
 		 */
 		ir_codes = RC_MAP_PIXELVIEW_MK12;
-		rc_type = RC_BIT_NEC;
+		rc_type = RC_BIT_NECX;
 		ir->gpio_addr = MO_GP1_IO;
 		ir->mask_keyup = 0x80;
 		ir->polling = 10; /* ms */
@@ -631,7 +632,8 @@ void cx88_i2c_init_ir(struct cx88_core *core)
 			/* Hauppauge XVR */
 			core->init_data.name = "cx88 Hauppauge XVR remote";
 			core->init_data.ir_codes = RC_MAP_HAUPPAUGE;
-			core->init_data.type = RC_BIT_RC5;
+			core->init_data.type = RC_BIT_RC5 | RC_BIT_RC6_MCE |
+							RC_BIT_RC6_6A_32;
 			core->init_data.internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
 
 			info.platform_data = &core->init_data;
diff --git a/drivers/media/pci/cx88/cx88-video.c b/drivers/media/pci/cx88/cx88-video.c
index 5dc1e3f08d50..d83eb3b10f54 100644
--- a/drivers/media/pci/cx88/cx88-video.c
+++ b/drivers/media/pci/cx88/cx88-video.c
@@ -567,7 +567,7 @@ static void stop_streaming(struct vb2_queue *q)
 	spin_unlock_irqrestore(&dev->slock, flags);
 }
 
-static struct vb2_ops cx8800_video_qops = {
+static const struct vb2_ops cx8800_video_qops = {
 	.queue_setup    = queue_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_finish = buffer_finish,
diff --git a/drivers/media/pci/ddbridge/ddbridge-core.c b/drivers/media/pci/ddbridge/ddbridge-core.c
index 47def73b3502..18e3a4deee64 100644
--- a/drivers/media/pci/ddbridge/ddbridge-core.c
+++ b/drivers/media/pci/ddbridge/ddbridge-core.c
@@ -1643,53 +1643,53 @@ fail:
 /******************************************************************************/
 /******************************************************************************/
 
-static struct ddb_info ddb_none = {
+static const struct ddb_info ddb_none = {
 	.type     = DDB_NONE,
 	.name     = "Digital Devices PCIe bridge",
 };
 
-static struct ddb_info ddb_octopus = {
+static const struct ddb_info ddb_octopus = {
 	.type     = DDB_OCTOPUS,
 	.name     = "Digital Devices Octopus DVB adapter",
 	.port_num = 4,
 };
 
-static struct ddb_info ddb_octopus_le = {
+static const struct ddb_info ddb_octopus_le = {
 	.type     = DDB_OCTOPUS,
 	.name     = "Digital Devices Octopus LE DVB adapter",
 	.port_num = 2,
 };
 
-static struct ddb_info ddb_octopus_mini = {
+static const struct ddb_info ddb_octopus_mini = {
 	.type     = DDB_OCTOPUS,
 	.name     = "Digital Devices Octopus Mini",
 	.port_num = 4,
 };
 
-static struct ddb_info ddb_v6 = {
+static const struct ddb_info ddb_v6 = {
 	.type     = DDB_OCTOPUS,
 	.name     = "Digital Devices Cine S2 V6 DVB adapter",
 	.port_num = 3,
 };
-static struct ddb_info ddb_v6_5 = {
+static const struct ddb_info ddb_v6_5 = {
 	.type     = DDB_OCTOPUS,
 	.name     = "Digital Devices Cine S2 V6.5 DVB adapter",
 	.port_num = 4,
 };
 
-static struct ddb_info ddb_dvbct = {
+static const struct ddb_info ddb_dvbct = {
 	.type     = DDB_OCTOPUS,
 	.name     = "Digital Devices DVBCT V6.1 DVB adapter",
 	.port_num = 3,
 };
 
-static struct ddb_info ddb_satixS2v3 = {
+static const struct ddb_info ddb_satixS2v3 = {
 	.type     = DDB_OCTOPUS,
 	.name     = "Mystique SaTiX-S2 V3 DVB adapter",
 	.port_num = 3,
 };
 
-static struct ddb_info ddb_octopusv3 = {
+static const struct ddb_info ddb_octopusv3 = {
 	.type     = DDB_OCTOPUS,
 	.name     = "Digital Devices Octopus V3 DVB adapter",
 	.port_num = 4,
diff --git a/drivers/media/pci/ivtv/ivtv-alsa-pcm.c b/drivers/media/pci/ivtv/ivtv-alsa-pcm.c
index f198b9826ed8..a26f9800eca3 100644
--- a/drivers/media/pci/ivtv/ivtv-alsa-pcm.c
+++ b/drivers/media/pci/ivtv/ivtv-alsa-pcm.c
@@ -318,7 +318,7 @@ static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs,
 	return vmalloc_to_page(pageptr);
 }
 
-static struct snd_pcm_ops snd_ivtv_pcm_capture_ops = {
+static const struct snd_pcm_ops snd_ivtv_pcm_capture_ops = {
 	.open		= snd_ivtv_pcm_capture_open,
 	.close		= snd_ivtv_pcm_capture_close,
 	.ioctl		= snd_ivtv_pcm_ioctl,
diff --git a/drivers/media/pci/ivtv/ivtv-driver.c b/drivers/media/pci/ivtv/ivtv-driver.c
index 374033a5bdaf..ee48c3e09de4 100644
--- a/drivers/media/pci/ivtv/ivtv-driver.c
+++ b/drivers/media/pci/ivtv/ivtv-driver.c
@@ -750,7 +750,7 @@ static int ivtv_init_struct1(struct ivtv *itv)
 	spin_lock_init(&itv->lock);
 	spin_lock_init(&itv->dma_reg_lock);
 
-	init_kthread_worker(&itv->irq_worker);
+	kthread_init_worker(&itv->irq_worker);
 	itv->irq_worker_task = kthread_run(kthread_worker_fn, &itv->irq_worker,
 					   "%s", itv->v4l2_dev.name);
 	if (IS_ERR(itv->irq_worker_task)) {
@@ -760,7 +760,7 @@ static int ivtv_init_struct1(struct ivtv *itv)
 	/* must use the FIFO scheduler as it is realtime sensitive */
 	sched_setscheduler(itv->irq_worker_task, SCHED_FIFO, &param);
 
-	init_kthread_work(&itv->irq_work, ivtv_irq_work_handler);
+	kthread_init_work(&itv->irq_work, ivtv_irq_work_handler);
 
 	/* Initial settings */
 	itv->cxhdl.port = CX2341X_PORT_MEMORY;
@@ -1441,7 +1441,7 @@ static void ivtv_remove(struct pci_dev *pdev)
 	del_timer_sync(&itv->dma_timer);
 
 	/* Kill irq worker */
-	flush_kthread_worker(&itv->irq_worker);
+	kthread_flush_worker(&itv->irq_worker);
 	kthread_stop(itv->irq_worker_task);
 
 	ivtv_streams_cleanup(itv);
diff --git a/drivers/media/pci/ivtv/ivtv-i2c.c b/drivers/media/pci/ivtv/ivtv-i2c.c
index bccbf2d18e30..dea80efd5836 100644
--- a/drivers/media/pci/ivtv/ivtv-i2c.c
+++ b/drivers/media/pci/ivtv/ivtv-i2c.c
@@ -215,7 +215,8 @@ static int ivtv_i2c_new_ir(struct ivtv *itv, u32 hw, const char *type, u8 addr)
 		/* Default to grey remote */
 		init_data->ir_codes = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
-		init_data->type = RC_BIT_RC5;
+		init_data->type = RC_BIT_RC5 | RC_BIT_RC6_MCE |
+							RC_BIT_RC6_6A_32;
 		init_data->name = itv->card_name;
 		break;
 	case IVTV_HW_I2C_IR_RX_ADAPTEC:
@@ -625,7 +626,7 @@ static u32 ivtv_functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm ivtv_algo = {
+static const struct i2c_algorithm ivtv_algo = {
 	.master_xfer   = ivtv_xfer,
 	.functionality = ivtv_functionality,
 };
diff --git a/drivers/media/pci/ivtv/ivtv-irq.c b/drivers/media/pci/ivtv/ivtv-irq.c
index 36ca2d67c812..6efe1f71262c 100644
--- a/drivers/media/pci/ivtv/ivtv-irq.c
+++ b/drivers/media/pci/ivtv/ivtv-irq.c
@@ -1062,7 +1062,7 @@ irqreturn_t ivtv_irq_handler(int irq, void *dev_id)
 	}
 
 	if (test_and_clear_bit(IVTV_F_I_HAVE_WORK, &itv->i_flags)) {
-		queue_kthread_work(&itv->irq_worker, &itv->irq_work);
+		kthread_queue_work(&itv->irq_worker, &itv->irq_work);
 	}
 
 	spin_unlock(&itv->dma_reg_lock);
diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
index ac547cb84de8..b078ac2a682c 100644
--- a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
+++ b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
@@ -353,7 +353,7 @@ static void netup_unidvb_stop_streaming(struct vb2_queue *q)
 	netup_unidvb_queue_cleanup(dma);
 }
 
-static struct vb2_ops dvb_qops = {
+static const struct vb2_ops dvb_qops = {
 	.queue_setup		= netup_unidvb_queue_setup,
 	.buf_prepare		= netup_unidvb_buf_prepare,
 	.buf_queue		= netup_unidvb_buf_queue,
diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_i2c.c b/drivers/media/pci/netup_unidvb/netup_unidvb_i2c.c
index c09c52bc6eab..b49e4f9788e8 100644
--- a/drivers/media/pci/netup_unidvb/netup_unidvb_i2c.c
+++ b/drivers/media/pci/netup_unidvb/netup_unidvb_i2c.c
@@ -327,11 +327,8 @@ static int netup_i2c_init(struct netup_unidvb_dev *ndev, int bus_num)
 	i2c->adap.dev.parent = &ndev->pci_dev->dev;
 	i2c_set_adapdata(&i2c->adap, i2c);
 	ret = i2c_add_adapter(&i2c->adap);
-	if (ret) {
-		dev_err(&ndev->pci_dev->dev,
-			"%s(): failed to add I2C adapter\n", __func__);
+	if (ret)
 		return ret;
-	}
 	dev_info(&ndev->pci_dev->dev,
 		"%s(): registered I2C bus %d at 0x%x\n",
 		__func__,
diff --git a/drivers/media/pci/ngene/ngene-cards.c b/drivers/media/pci/ngene/ngene-cards.c
index 4e783a68bf4a..423e8c889310 100644
--- a/drivers/media/pci/ngene/ngene-cards.c
+++ b/drivers/media/pci/ngene/ngene-cards.c
@@ -613,7 +613,7 @@ static struct stv6110x_config tuner_cineS2_1 = {
 	.clk_div = 1,
 };
 
-static struct ngene_info ngene_info_cineS2 = {
+static const struct ngene_info ngene_info_cineS2 = {
 	.type		= NGENE_SIDEWINDER,
 	.name		= "Linux4Media cineS2 DVB-S2 Twin Tuner",
 	.io_type	= {NGENE_IO_TSIN, NGENE_IO_TSIN},
@@ -627,7 +627,7 @@ static struct ngene_info ngene_info_cineS2 = {
 	.msi_supported	= true,
 };
 
-static struct ngene_info ngene_info_satixS2 = {
+static const struct ngene_info ngene_info_satixS2 = {
 	.type		= NGENE_SIDEWINDER,
 	.name		= "Mystique SaTiX-S2 Dual",
 	.io_type	= {NGENE_IO_TSIN, NGENE_IO_TSIN},
@@ -641,7 +641,7 @@ static struct ngene_info ngene_info_satixS2 = {
 	.msi_supported	= true,
 };
 
-static struct ngene_info ngene_info_satixS2v2 = {
+static const struct ngene_info ngene_info_satixS2v2 = {
 	.type		= NGENE_SIDEWINDER,
 	.name		= "Mystique SaTiX-S2 Dual (v2)",
 	.io_type	= {NGENE_IO_TSIN, NGENE_IO_TSIN, NGENE_IO_TSIN, NGENE_IO_TSIN,
@@ -656,7 +656,7 @@ static struct ngene_info ngene_info_satixS2v2 = {
 	.msi_supported	= true,
 };
 
-static struct ngene_info ngene_info_cineS2v5 = {
+static const struct ngene_info ngene_info_cineS2v5 = {
 	.type		= NGENE_SIDEWINDER,
 	.name		= "Linux4Media cineS2 DVB-S2 Twin Tuner (v5)",
 	.io_type	= {NGENE_IO_TSIN, NGENE_IO_TSIN, NGENE_IO_TSIN, NGENE_IO_TSIN,
@@ -672,7 +672,7 @@ static struct ngene_info ngene_info_cineS2v5 = {
 };
 
 
-static struct ngene_info ngene_info_duoFlex = {
+static const struct ngene_info ngene_info_duoFlex = {
 	.type           = NGENE_SIDEWINDER,
 	.name           = "Digital Devices DuoFlex PCIe or miniPCIe",
 	.io_type        = {NGENE_IO_TSIN, NGENE_IO_TSIN, NGENE_IO_TSIN, NGENE_IO_TSIN,
@@ -687,7 +687,7 @@ static struct ngene_info ngene_info_duoFlex = {
 	.msi_supported	= true,
 };
 
-static struct ngene_info ngene_info_m780 = {
+static const struct ngene_info ngene_info_m780 = {
 	.type           = NGENE_APP,
 	.name           = "Aver M780 ATSC/QAM-B",
 
@@ -727,7 +727,7 @@ static struct drxd_config fe_terratec_dvbt_1 = {
 	.osc_deviation  = osc_deviation,
 };
 
-static struct ngene_info ngene_info_terratec = {
+static const struct ngene_info ngene_info_terratec = {
 	.type           = NGENE_TERRATEC,
 	.name           = "Terratec Integra/Cinergy2400i Dual DVB-T",
 	.io_type        = {NGENE_IO_TSIN, NGENE_IO_TSIN},
diff --git a/drivers/media/pci/pt3/pt3.c b/drivers/media/pci/pt3/pt3.c
index eff5e9f51ace..7fb649e523f4 100644
--- a/drivers/media/pci/pt3/pt3.c
+++ b/drivers/media/pci/pt3/pt3.c
@@ -798,10 +798,8 @@ static int pt3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	strlcpy(i2c->name, DRV_NAME, sizeof(i2c->name));
 	i2c_set_adapdata(i2c, pt3);
 	ret = i2c_add_adapter(i2c);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "Failed to add i2c adapter\n");
+	if (ret < 0)
 		goto err_i2cbuf;
-	}
 
 	for (i = 0; i < PT3_NUM_FE; i++) {
 		ret = pt3_alloc_adapter(pt3, i);
diff --git a/drivers/media/pci/saa7134/saa7134-alsa.c b/drivers/media/pci/saa7134/saa7134-alsa.c
index 94f816244407..dc0e2fc5f68b 100644
--- a/drivers/media/pci/saa7134/saa7134-alsa.c
+++ b/drivers/media/pci/saa7134/saa7134-alsa.c
@@ -877,7 +877,7 @@ static struct page *snd_card_saa7134_page(struct snd_pcm_substream *substream,
  * ALSA capture callbacks definition
  */
 
-static struct snd_pcm_ops snd_card_saa7134_capture_ops = {
+static const struct snd_pcm_ops snd_card_saa7134_capture_ops = {
 	.open =			snd_card_saa7134_capture_open,
 	.close =		snd_card_saa7134_capture_close,
 	.ioctl =		snd_pcm_lib_ioctl,
diff --git a/drivers/media/pci/saa7134/saa7134-dvb.c b/drivers/media/pci/saa7134/saa7134-dvb.c
index db987e5b93eb..59a4b5f7724e 100644
--- a/drivers/media/pci/saa7134/saa7134-dvb.c
+++ b/drivers/media/pci/saa7134/saa7134-dvb.c
@@ -1238,6 +1238,7 @@ static int dvb_init(struct saa7134_dev *dev)
 	q->buf_struct_size = sizeof(struct saa7134_buf);
 	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
 	q->lock = &dev->lock;
+	q->dev = &dev->pci->dev;
 	ret = vb2_queue_init(q);
 	if (ret) {
 		vb2_dvb_dealloc_frontends(&dev->frontends);
diff --git a/drivers/media/pci/saa7134/saa7134-empress.c b/drivers/media/pci/saa7134/saa7134-empress.c
index ca417a454d67..f0fe2524259f 100644
--- a/drivers/media/pci/saa7134/saa7134-empress.c
+++ b/drivers/media/pci/saa7134/saa7134-empress.c
@@ -85,7 +85,7 @@ static void stop_streaming(struct vb2_queue *vq)
 	dev->empress_started = 0;
 }
 
-static struct vb2_ops saa7134_empress_qops = {
+static const struct vb2_ops saa7134_empress_qops = {
 	.queue_setup	= saa7134_ts_queue_setup,
 	.buf_init	= saa7134_ts_buffer_init,
 	.buf_prepare	= saa7134_ts_buffer_prepare,
@@ -295,6 +295,7 @@ static int empress_init(struct saa7134_dev *dev)
 	q->buf_struct_size = sizeof(struct saa7134_buf);
 	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
 	q->lock = &dev->lock;
+	q->dev = &dev->pci->dev;
 	err = vb2_queue_init(q);
 	if (err)
 		return err;
diff --git a/drivers/media/pci/saa7134/saa7134-i2c.c b/drivers/media/pci/saa7134/saa7134-i2c.c
index 8ef6399d794f..2dac48fa1386 100644
--- a/drivers/media/pci/saa7134/saa7134-i2c.c
+++ b/drivers/media/pci/saa7134/saa7134-i2c.c
@@ -338,7 +338,7 @@ static u32 functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm saa7134_algo = {
+static const struct i2c_algorithm saa7134_algo = {
 	.master_xfer   = saa7134_i2c_xfer,
 	.functionality = functionality,
 };
diff --git a/drivers/media/pci/saa7134/saa7134-input.c b/drivers/media/pci/saa7134/saa7134-input.c
index c8042c3888cd..eff52bbbfd66 100644
--- a/drivers/media/pci/saa7134/saa7134-input.c
+++ b/drivers/media/pci/saa7134/saa7134-input.c
@@ -345,7 +345,7 @@ static int get_key_beholdm6xx(struct IR_i2c *ir, enum rc_type *protocol,
 	if (data[9] != (unsigned char)(~data[8]))
 		return 0;
 
-	*protocol = RC_TYPE_NEC;
+	*protocol = RC_TYPE_NECX;
 	*scancode = RC_SCANCODE_NECX(data[11] << 8 | data[10], data[9]);
 	*toggle = 0;
 	return 1;
@@ -1035,7 +1035,7 @@ void saa7134_probe_i2c_ir(struct saa7134_dev *dev)
 		dev->init_data.name = "BeholdTV";
 		dev->init_data.get_key = get_key_beholdm6xx;
 		dev->init_data.ir_codes = RC_MAP_BEHOLD;
-		dev->init_data.type = RC_BIT_NEC;
+		dev->init_data.type = RC_BIT_NECX;
 		info.addr = 0x2d;
 		break;
 	case SAA7134_BOARD_AVERMEDIA_CARDBUS_501:
diff --git a/drivers/media/pci/saa7134/saa7134-video.c b/drivers/media/pci/saa7134/saa7134-video.c
index 8a6ebd087889..cbb173d99085 100644
--- a/drivers/media/pci/saa7134/saa7134-video.c
+++ b/drivers/media/pci/saa7134/saa7134-video.c
@@ -1054,7 +1054,7 @@ void saa7134_vb2_stop_streaming(struct vb2_queue *vq)
 		pm_qos_remove_request(&dev->qos_request);
 }
 
-static struct vb2_ops vb2_qops = {
+static const struct vb2_ops vb2_qops = {
 	.queue_setup	= queue_setup,
 	.buf_init	= buffer_init,
 	.buf_prepare	= buffer_prepare,
@@ -1659,8 +1659,6 @@ static int saa7134_cropcap(struct file *file, void *priv,
 	if (cap->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
 	    cap->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
 		return -EINVAL;
-	cap->bounds  = dev->crop_bounds;
-	cap->defrect = dev->crop_defrect;
 	cap->pixelaspect.numerator   = 1;
 	cap->pixelaspect.denominator = 1;
 	if (dev->tvnorm->id & V4L2_STD_525_60) {
@@ -1674,25 +1672,41 @@ static int saa7134_cropcap(struct file *file, void *priv,
 	return 0;
 }
 
-static int saa7134_g_crop(struct file *file, void *f, struct v4l2_crop *crop)
+static int saa7134_g_selection(struct file *file, void *f, struct v4l2_selection *sel)
 {
 	struct saa7134_dev *dev = video_drvdata(file);
 
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
-	    crop->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
+	    sel->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
 		return -EINVAL;
-	crop->c = dev->crop_current;
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		sel->r = dev->crop_current;
+		break;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r = dev->crop_defrect;
+		break;
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r  = dev->crop_bounds;
+		break;
+	default:
+		return -EINVAL;
+	}
 	return 0;
 }
 
-static int saa7134_s_crop(struct file *file, void *f, const struct v4l2_crop *crop)
+static int saa7134_s_selection(struct file *file, void *f, struct v4l2_selection *sel)
 {
 	struct saa7134_dev *dev = video_drvdata(file);
 	struct v4l2_rect *b = &dev->crop_bounds;
 	struct v4l2_rect *c = &dev->crop_current;
 
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
-	    crop->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
+	    sel->type != V4L2_BUF_TYPE_VIDEO_OVERLAY)
+		return -EINVAL;
+
+	if (sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 
 	if (dev->overlay_owner)
@@ -1700,7 +1714,7 @@ static int saa7134_s_crop(struct file *file, void *f, const struct v4l2_crop *cr
 	if (vb2_is_streaming(&dev->video_vbq))
 		return -EBUSY;
 
-	*c = crop->c;
+	*c = sel->r;
 	if (c->top < b->top)
 		c->top = b->top;
 	if (c->top > b->top + b->height)
@@ -1714,6 +1728,7 @@ static int saa7134_s_crop(struct file *file, void *f, const struct v4l2_crop *cr
 		c->left = b->left + b->width;
 	if (c->width > b->left - c->left + b->width)
 		c->width = b->left - c->left + b->width;
+	sel->r = *c;
 	return 0;
 }
 
@@ -1989,8 +2004,8 @@ static const struct v4l2_ioctl_ops video_ioctl_ops = {
 	.vidioc_streamoff		= vb2_ioctl_streamoff,
 	.vidioc_g_tuner			= saa7134_g_tuner,
 	.vidioc_s_tuner			= saa7134_s_tuner,
-	.vidioc_g_crop			= saa7134_g_crop,
-	.vidioc_s_crop			= saa7134_s_crop,
+	.vidioc_g_selection		= saa7134_g_selection,
+	.vidioc_s_selection		= saa7134_s_selection,
 	.vidioc_g_fbuf			= saa7134_g_fbuf,
 	.vidioc_s_fbuf			= saa7134_s_fbuf,
 	.vidioc_overlay			= saa7134_overlay,
diff --git a/drivers/media/pci/saa7164/saa7164-i2c.c b/drivers/media/pci/saa7164/saa7164-i2c.c
index 0342d84913b8..024f4e29e840 100644
--- a/drivers/media/pci/saa7164/saa7164-i2c.c
+++ b/drivers/media/pci/saa7164/saa7164-i2c.c
@@ -75,7 +75,7 @@ static u32 saa7164_functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm saa7164_i2c_algo_template = {
+static const struct i2c_algorithm saa7164_i2c_algo_template = {
 	.master_xfer	= i2c_xfer,
 	.functionality	= saa7164_functionality,
 };
diff --git a/drivers/media/pci/smipcie/smipcie-main.c b/drivers/media/pci/smipcie/smipcie-main.c
index 83981d611a79..6dbe3b4d09ce 100644
--- a/drivers/media/pci/smipcie/smipcie-main.c
+++ b/drivers/media/pci/smipcie/smipcie-main.c
@@ -1060,7 +1060,7 @@ static void smi_remove(struct pci_dev *pdev)
 }
 
 /* DVBSky cards */
-static struct smi_cfg_info dvbsky_s950_cfg = {
+static const struct smi_cfg_info dvbsky_s950_cfg = {
 	.type = SMI_DVBSKY_S950,
 	.name = "DVBSky S950 V3",
 	.ts_0 = SMI_TS_NULL,
@@ -1070,7 +1070,7 @@ static struct smi_cfg_info dvbsky_s950_cfg = {
 	.rc_map = RC_MAP_DVBSKY,
 };
 
-static struct smi_cfg_info dvbsky_s952_cfg = {
+static const struct smi_cfg_info dvbsky_s952_cfg = {
 	.type = SMI_DVBSKY_S952,
 	.name = "DVBSky S952 V3",
 	.ts_0 = SMI_TS_DMA_BOTH,
@@ -1080,7 +1080,7 @@ static struct smi_cfg_info dvbsky_s952_cfg = {
 	.rc_map = RC_MAP_DVBSKY,
 };
 
-static struct smi_cfg_info dvbsky_t9580_cfg = {
+static const struct smi_cfg_info dvbsky_t9580_cfg = {
 	.type = SMI_DVBSKY_T9580,
 	.name = "DVBSky T9580 V3",
 	.ts_0 = SMI_TS_DMA_BOTH,
@@ -1090,7 +1090,7 @@ static struct smi_cfg_info dvbsky_t9580_cfg = {
 	.rc_map = RC_MAP_DVBSKY,
 };
 
-static struct smi_cfg_info technotrend_s2_4200_cfg = {
+static const struct smi_cfg_info technotrend_s2_4200_cfg = {
 	.type = SMI_TECHNOTREND_S2_4200,
 	.name = "TechnoTrend TT-budget S2-4200 Twin",
 	.ts_0 = SMI_TS_DMA_BOTH,
diff --git a/drivers/media/pci/solo6x10/solo6x10-g723.c b/drivers/media/pci/solo6x10/solo6x10-g723.c
index 4a37a1c51c48..6a35107aca25 100644
--- a/drivers/media/pci/solo6x10/solo6x10-g723.c
+++ b/drivers/media/pci/solo6x10/solo6x10-g723.c
@@ -252,7 +252,7 @@ static int snd_solo_pcm_copy(struct snd_pcm_substream *ss, int channel,
 	return 0;
 }
 
-static struct snd_pcm_ops snd_solo_pcm_ops = {
+static const struct snd_pcm_ops snd_solo_pcm_ops = {
 	.open = snd_solo_pcm_open,
 	.close = snd_solo_pcm_close,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c b/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c
index 399164314c28..25a2137ab799 100644
--- a/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c
+++ b/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c
@@ -759,7 +759,7 @@ static void solo_enc_buf_finish(struct vb2_buffer *vb)
 	}
 }
 
-static struct vb2_ops solo_enc_video_qops = {
+static const struct vb2_ops solo_enc_video_qops = {
 	.queue_setup	= solo_enc_queue_setup,
 	.buf_queue	= solo_enc_buf_queue,
 	.buf_finish	= solo_enc_buf_finish,
diff --git a/drivers/media/pci/tw5864/Kconfig b/drivers/media/pci/tw5864/Kconfig
new file mode 100644
index 000000000000..87c8f327e2d4
--- /dev/null
+++ b/drivers/media/pci/tw5864/Kconfig
@@ -0,0 +1,12 @@
+config VIDEO_TW5864
+	tristate "Techwell TW5864 video/audio grabber and encoder"
+	depends on VIDEO_DEV && PCI && VIDEO_V4L2
+	depends on HAS_DMA
+	select VIDEOBUF2_DMA_CONTIG
+	---help---
+	  Support for boards based on Techwell TW5864 chip which provides
+	  multichannel video & audio grabbing and encoding (H.264, MJPEG,
+	  ADPCM G.726).
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called tw5864.
diff --git a/drivers/media/pci/tw5864/Makefile b/drivers/media/pci/tw5864/Makefile
new file mode 100644
index 000000000000..4fc8b3b1a45a
--- /dev/null
+++ b/drivers/media/pci/tw5864/Makefile
@@ -0,0 +1,3 @@
+tw5864-objs := tw5864-core.o tw5864-video.o tw5864-h264.o tw5864-util.o
+
+obj-$(CONFIG_VIDEO_TW5864) += tw5864.o
diff --git a/drivers/media/pci/tw5864/tw5864-core.c b/drivers/media/pci/tw5864/tw5864-core.c
new file mode 100644
index 000000000000..1d43b96958ea
--- /dev/null
+++ b/drivers/media/pci/tw5864/tw5864-core.c
@@ -0,0 +1,359 @@
+/*
+ *  TW5864 driver - core functions
+ *
+ *  Copyright (C) 2016 Bluecherry, LLC <maintainers@bluecherrydvr.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ */
+
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/kmod.h>
+#include <linux/sound.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/pm.h>
+#include <linux/pci_ids.h>
+#include <linux/jiffies.h>
+#include <asm/dma.h>
+#include <media/v4l2-dev.h>
+
+#include "tw5864.h"
+#include "tw5864-reg.h"
+
+MODULE_DESCRIPTION("V4L2 driver module for tw5864-based multimedia capture & encoding devices");
+MODULE_AUTHOR("Bluecherry Maintainers <maintainers@bluecherrydvr.com>");
+MODULE_AUTHOR("Andrey Utkin <andrey.utkin@corp.bluecherry.net>");
+MODULE_LICENSE("GPL");
+
+/*
+ * BEWARE OF KNOWN ISSUES WITH VIDEO QUALITY
+ *
+ * This driver was developed by Bluecherry LLC by deducing behaviour of
+ * original manufacturer's driver, from both source code and execution traces.
+ * It is known that there are some artifacts on output video with this driver:
+ *  - on all known hardware samples: random pixels of wrong color (mostly
+ *    white, red or blue) appearing and disappearing on sequences of P-frames;
+ *  - on some hardware samples (known with H.264 core version e006:2800):
+ *    total madness on P-frames: blocks of wrong luminance; blocks of wrong
+ *    colors "creeping" across the picture.
+ * There is a workaround for both issues: avoid P-frames by setting GOP size
+ * to 1. To do that, run this command on device files created by this driver:
+ *
+ * v4l2-ctl --device /dev/videoX --set-ctrl=video_gop_size=1
+ *
+ * These issues are not decoding errors; all produced H.264 streams are decoded
+ * properly. Streams without P-frames don't have these artifacts so it's not
+ * analog-to-digital conversion issues nor internal memory errors; we conclude
+ * it's internal H.264 encoder issues.
+ * We cannot even check the original driver's behaviour because it has never
+ * worked properly at all in our development environment. So these issues may
+ * be actually related to firmware or hardware. However it may be that there's
+ * just some more register settings missing in the driver which would please
+ * the hardware.
+ * Manufacturer didn't help much on our inquiries, but feel free to disturb
+ * again the support of Intersil (owner of former Techwell).
+ */
+
+/* take first free /dev/videoX indexes by default */
+static unsigned int video_nr[] = {[0 ... (TW5864_INPUTS - 1)] = -1 };
+
+module_param_array(video_nr, int, NULL, 0444);
+MODULE_PARM_DESC(video_nr, "video devices numbers array");
+
+/*
+ * Please add any new PCI IDs to: http://pci-ids.ucw.cz.  This keeps
+ * the PCI ID database up to date.  Note that the entries must be
+ * added under vendor 0x1797 (Techwell Inc.) as subsystem IDs.
+ */
+static const struct pci_device_id tw5864_pci_tbl[] = {
+	{PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, PCI_DEVICE_ID_TECHWELL_5864)},
+	{0,}
+};
+
+void tw5864_irqmask_apply(struct tw5864_dev *dev)
+{
+	tw_writel(TW5864_INTR_ENABLE_L, dev->irqmask & 0xffff);
+	tw_writel(TW5864_INTR_ENABLE_H, (dev->irqmask >> 16));
+}
+
+static void tw5864_interrupts_disable(struct tw5864_dev *dev)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->slock, flags);
+	dev->irqmask = 0;
+	tw5864_irqmask_apply(dev);
+	spin_unlock_irqrestore(&dev->slock, flags);
+}
+
+static void tw5864_timer_isr(struct tw5864_dev *dev);
+static void tw5864_h264_isr(struct tw5864_dev *dev);
+
+static irqreturn_t tw5864_isr(int irq, void *dev_id)
+{
+	struct tw5864_dev *dev = dev_id;
+	u32 status;
+
+	status = tw_readl(TW5864_INTR_STATUS_L) |
+		tw_readl(TW5864_INTR_STATUS_H) << 16;
+	if (!status)
+		return IRQ_NONE;
+
+	tw_writel(TW5864_INTR_CLR_L, 0xffff);
+	tw_writel(TW5864_INTR_CLR_H, 0xffff);
+
+	if (status & TW5864_INTR_VLC_DONE)
+		tw5864_h264_isr(dev);
+
+	if (status & TW5864_INTR_TIMER)
+		tw5864_timer_isr(dev);
+
+	if (!(status & (TW5864_INTR_TIMER | TW5864_INTR_VLC_DONE))) {
+		dev_dbg(&dev->pci->dev, "Unknown interrupt, status 0x%08X\n",
+			status);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void tw5864_h264_isr(struct tw5864_dev *dev)
+{
+	int channel = tw_readl(TW5864_DSP) & TW5864_DSP_ENC_CHN;
+	struct tw5864_input *input = &dev->inputs[channel];
+	int cur_frame_index, next_frame_index;
+	struct tw5864_h264_frame *cur_frame, *next_frame;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->slock, flags);
+
+	cur_frame_index = dev->h264_buf_w_index;
+	next_frame_index = (cur_frame_index + 1) % H264_BUF_CNT;
+	cur_frame = &dev->h264_buf[cur_frame_index];
+	next_frame = &dev->h264_buf[next_frame_index];
+
+	if (next_frame_index != dev->h264_buf_r_index) {
+		cur_frame->vlc_len = tw_readl(TW5864_VLC_LENGTH) << 2;
+		cur_frame->checksum = tw_readl(TW5864_VLC_CRC_REG);
+		cur_frame->input = input;
+		cur_frame->timestamp = ktime_get_ns();
+		cur_frame->seqno = input->frame_seqno;
+		cur_frame->gop_seqno = input->frame_gop_seqno;
+
+		dev->h264_buf_w_index = next_frame_index;
+		tasklet_schedule(&dev->tasklet);
+
+		cur_frame = next_frame;
+
+		spin_lock(&input->slock);
+		input->frame_seqno++;
+		input->frame_gop_seqno++;
+		if (input->frame_gop_seqno >= input->gop)
+			input->frame_gop_seqno = 0;
+		spin_unlock(&input->slock);
+	} else {
+		dev_err(&dev->pci->dev,
+			"Skipped frame on input %d because all buffers busy\n",
+			channel);
+	}
+
+	dev->encoder_busy = 0;
+
+	spin_unlock_irqrestore(&dev->slock, flags);
+
+	tw_writel(TW5864_VLC_STREAM_BASE_ADDR, cur_frame->vlc.dma_addr);
+	tw_writel(TW5864_MV_STREAM_BASE_ADDR, cur_frame->mv.dma_addr);
+
+	/* Additional ack for this interrupt */
+	tw_writel(TW5864_VLC_DSP_INTR, 0x00000001);
+	tw_writel(TW5864_PCI_INTR_STATUS, TW5864_VLC_DONE_INTR);
+}
+
+static void tw5864_input_deadline_update(struct tw5864_input *input)
+{
+	input->new_frame_deadline = jiffies + msecs_to_jiffies(1000);
+}
+
+static void tw5864_timer_isr(struct tw5864_dev *dev)
+{
+	unsigned long flags;
+	int i;
+	int encoder_busy;
+
+	/* Additional ack for this interrupt */
+	tw_writel(TW5864_PCI_INTR_STATUS, TW5864_TIMER_INTR);
+
+	spin_lock_irqsave(&dev->slock, flags);
+	encoder_busy = dev->encoder_busy;
+	spin_unlock_irqrestore(&dev->slock, flags);
+
+	if (encoder_busy)
+		return;
+
+	/*
+	 * Traversing inputs in round-robin fashion, starting from next to the
+	 * last processed one
+	 */
+	for (i = 0; i < TW5864_INPUTS; i++) {
+		int next_input = (i + dev->next_input) % TW5864_INPUTS;
+		struct tw5864_input *input = &dev->inputs[next_input];
+		int raw_buf_id; /* id of internal buf with last raw frame */
+
+		spin_lock_irqsave(&input->slock, flags);
+		if (!input->enabled)
+			goto next;
+
+		/* Check if new raw frame is available */
+		raw_buf_id = tw_mask_shift_readl(TW5864_SENIF_ORG_FRM_PTR1, 0x3,
+						 2 * input->nr);
+
+		if (input->buf_id != raw_buf_id) {
+			input->buf_id = raw_buf_id;
+			tw5864_input_deadline_update(input);
+			spin_unlock_irqrestore(&input->slock, flags);
+
+			spin_lock_irqsave(&dev->slock, flags);
+			dev->encoder_busy = 1;
+			dev->next_input = (next_input + 1) % TW5864_INPUTS;
+			spin_unlock_irqrestore(&dev->slock, flags);
+
+			tw5864_request_encoded_frame(input);
+			break;
+		}
+
+		/* No new raw frame; check if channel is stuck */
+		if (time_is_after_jiffies(input->new_frame_deadline)) {
+			/* If stuck, request new raw frames again */
+			tw_mask_shift_writel(TW5864_ENC_BUF_PTR_REC1, 0x3,
+					     2 * input->nr, input->buf_id + 3);
+			tw5864_input_deadline_update(input);
+		}
+next:
+		spin_unlock_irqrestore(&input->slock, flags);
+	}
+}
+
+static int tw5864_initdev(struct pci_dev *pci_dev,
+			  const struct pci_device_id *pci_id)
+{
+	struct tw5864_dev *dev;
+	int err;
+
+	dev = devm_kzalloc(&pci_dev->dev, sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	snprintf(dev->name, sizeof(dev->name), "tw5864:%s", pci_name(pci_dev));
+
+	err = v4l2_device_register(&pci_dev->dev, &dev->v4l2_dev);
+	if (err)
+		return err;
+
+	/* pci init */
+	dev->pci = pci_dev;
+	err = pci_enable_device(pci_dev);
+	if (err) {
+		dev_err(&dev->pci->dev, "pci_enable_device() failed\n");
+		goto unreg_v4l2;
+	}
+
+	pci_set_master(pci_dev);
+
+	err = pci_set_dma_mask(pci_dev, DMA_BIT_MASK(32));
+	if (err) {
+		dev_err(&dev->pci->dev, "32 bit PCI DMA is not supported\n");
+		goto disable_pci;
+	}
+
+	/* get mmio */
+	err = pci_request_regions(pci_dev, dev->name);
+	if (err) {
+		dev_err(&dev->pci->dev, "Cannot request regions for MMIO\n");
+		goto disable_pci;
+	}
+	dev->mmio = pci_ioremap_bar(pci_dev, 0);
+	if (!dev->mmio) {
+		err = -EIO;
+		dev_err(&dev->pci->dev, "can't ioremap() MMIO memory\n");
+		goto release_mmio;
+	}
+
+	spin_lock_init(&dev->slock);
+
+	dev_info(&pci_dev->dev, "TW5864 hardware version: %04x\n",
+		 tw_readl(TW5864_HW_VERSION));
+	dev_info(&pci_dev->dev, "TW5864 H.264 core version: %04x:%04x\n",
+		 tw_readl(TW5864_H264REV),
+		 tw_readl(TW5864_UNDECLARED_H264REV_PART2));
+
+	err = tw5864_video_init(dev, video_nr);
+	if (err)
+		goto unmap_mmio;
+
+	/* get irq */
+	err = devm_request_irq(&pci_dev->dev, pci_dev->irq, tw5864_isr,
+			       IRQF_SHARED, "tw5864", dev);
+	if (err < 0) {
+		dev_err(&dev->pci->dev, "can't get IRQ %d\n", pci_dev->irq);
+		goto fini_video;
+	}
+
+	dev_info(&pci_dev->dev, "Note: there are known video quality issues. For details\n");
+	dev_info(&pci_dev->dev, "see the comment in drivers/media/pci/tw5864/tw5864-core.c.\n");
+
+	return 0;
+
+fini_video:
+	tw5864_video_fini(dev);
+unmap_mmio:
+	iounmap(dev->mmio);
+release_mmio:
+	pci_release_regions(pci_dev);
+disable_pci:
+	pci_disable_device(pci_dev);
+unreg_v4l2:
+	v4l2_device_unregister(&dev->v4l2_dev);
+	return err;
+}
+
+static void tw5864_finidev(struct pci_dev *pci_dev)
+{
+	struct v4l2_device *v4l2_dev = pci_get_drvdata(pci_dev);
+	struct tw5864_dev *dev =
+		container_of(v4l2_dev, struct tw5864_dev, v4l2_dev);
+
+	/* shutdown subsystems */
+	tw5864_interrupts_disable(dev);
+
+	/* unregister */
+	tw5864_video_fini(dev);
+
+	/* release resources */
+	iounmap(dev->mmio);
+	release_mem_region(pci_resource_start(pci_dev, 0),
+			   pci_resource_len(pci_dev, 0));
+
+	v4l2_device_unregister(&dev->v4l2_dev);
+	devm_kfree(&pci_dev->dev, dev);
+}
+
+static struct pci_driver tw5864_pci_driver = {
+	.name = "tw5864",
+	.id_table = tw5864_pci_tbl,
+	.probe = tw5864_initdev,
+	.remove = tw5864_finidev,
+};
+
+module_pci_driver(tw5864_pci_driver);
diff --git a/drivers/media/pci/tw5864/tw5864-h264.c b/drivers/media/pci/tw5864/tw5864-h264.c
new file mode 100644
index 000000000000..330d200f52cd
--- /dev/null
+++ b/drivers/media/pci/tw5864/tw5864-h264.c
@@ -0,0 +1,259 @@
+/*
+ *  TW5864 driver - H.264 headers generation functions
+ *
+ *  Copyright (C) 2016 Bluecherry, LLC <maintainers@bluecherrydvr.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ */
+
+#include <linux/log2.h>
+
+#include "tw5864.h"
+
+static u8 marker[] = { 0x00, 0x00, 0x00, 0x01 };
+
+/*
+ * Exponential-Golomb coding functions
+ *
+ * These functions are used for generation of H.264 bitstream headers.
+ *
+ * This code is derived from tw5864 reference driver by manufacturers, which
+ * itself apparently was derived from x264 project.
+ */
+
+/* Bitstream writing context */
+struct bs {
+	u8 *buf; /* pointer to buffer beginning */
+	u8 *buf_end; /* pointer to buffer end */
+	u8 *ptr; /* pointer to current byte in buffer */
+	unsigned int bits_left; /* number of available bits in current byte */
+};
+
+static void bs_init(struct bs *s, void *buf, int size)
+{
+	s->buf = buf;
+	s->ptr = buf;
+	s->buf_end = s->ptr + size;
+	s->bits_left = 8;
+}
+
+static int bs_len(struct bs *s)
+{
+	return s->ptr - s->buf;
+}
+
+static void bs_write(struct bs *s, int count, u32 bits)
+{
+	if (s->ptr >= s->buf_end - 4)
+		return;
+	while (count > 0) {
+		if (count < 32)
+			bits &= (1 << count) - 1;
+		if (count < s->bits_left) {
+			*s->ptr = (*s->ptr << count) | bits;
+			s->bits_left -= count;
+			break;
+		}
+		*s->ptr = (*s->ptr << s->bits_left) |
+			(bits >> (count - s->bits_left));
+		count -= s->bits_left;
+		s->ptr++;
+		s->bits_left = 8;
+	}
+}
+
+static void bs_write1(struct bs *s, u32 bit)
+{
+	if (s->ptr < s->buf_end) {
+		*s->ptr <<= 1;
+		*s->ptr |= bit;
+		s->bits_left--;
+		if (s->bits_left == 0) {
+			s->ptr++;
+			s->bits_left = 8;
+		}
+	}
+}
+
+static void bs_write_ue(struct bs *s, u32 val)
+{
+	if (val == 0) {
+		bs_write1(s, 1);
+	} else {
+		val++;
+		bs_write(s, 2 * fls(val) - 1, val);
+	}
+}
+
+static void bs_write_se(struct bs *s, int val)
+{
+	bs_write_ue(s, val <= 0 ? -val * 2 : val * 2 - 1);
+}
+
+static void bs_rbsp_trailing(struct bs *s)
+{
+	bs_write1(s, 1);
+	if (s->bits_left != 8)
+		bs_write(s, s->bits_left, 0x00);
+}
+
+/* H.264 headers generation functions */
+
+static int tw5864_h264_gen_sps_rbsp(u8 *buf, size_t size, int width, int height)
+{
+	struct bs bs, *s;
+
+	s = &bs;
+	bs_init(s, buf, size);
+	bs_write(s, 8, 0x42); /* profile_idc, baseline */
+	bs_write(s, 1, 1); /* constraint_set0_flag */
+	bs_write(s, 1, 1); /* constraint_set1_flag */
+	bs_write(s, 1, 0); /* constraint_set2_flag */
+	bs_write(s, 5, 0); /* reserved_zero_5bits */
+	bs_write(s, 8, 0x1e); /* level_idc */
+	bs_write_ue(s, 0); /* seq_parameter_set_id */
+	bs_write_ue(s, ilog2(MAX_GOP_SIZE) - 4); /* log2_max_frame_num_minus4 */
+	bs_write_ue(s, 0); /* pic_order_cnt_type */
+	/* log2_max_pic_order_cnt_lsb_minus4 */
+	bs_write_ue(s, ilog2(MAX_GOP_SIZE) - 4);
+	bs_write_ue(s, 1); /* num_ref_frames */
+	bs_write(s, 1, 0); /* gaps_in_frame_num_value_allowed_flag */
+	bs_write_ue(s, width / 16 - 1); /* pic_width_in_mbs_minus1 */
+	bs_write_ue(s, height / 16 - 1); /* pic_height_in_map_units_minus1 */
+	bs_write(s, 1, 1); /* frame_mbs_only_flag */
+	bs_write(s, 1, 0); /* direct_8x8_inference_flag */
+	bs_write(s, 1, 0); /* frame_cropping_flag */
+	bs_write(s, 1, 0); /* vui_parameters_present_flag */
+	bs_rbsp_trailing(s);
+	return bs_len(s);
+}
+
+static int tw5864_h264_gen_pps_rbsp(u8 *buf, size_t size, int qp)
+{
+	struct bs bs, *s;
+
+	s = &bs;
+	bs_init(s, buf, size);
+	bs_write_ue(s, 0); /* pic_parameter_set_id */
+	bs_write_ue(s, 0); /* seq_parameter_set_id */
+	bs_write(s, 1, 0); /* entropy_coding_mode_flag */
+	bs_write(s, 1, 0); /* pic_order_present_flag */
+	bs_write_ue(s, 0); /* num_slice_groups_minus1 */
+	bs_write_ue(s, 0); /* i_num_ref_idx_l0_active_minus1 */
+	bs_write_ue(s, 0); /* i_num_ref_idx_l1_active_minus1 */
+	bs_write(s, 1, 0); /* weighted_pred_flag */
+	bs_write(s, 2, 0); /* weighted_bipred_idc */
+	bs_write_se(s, qp - 26); /* pic_init_qp_minus26 */
+	bs_write_se(s, qp - 26); /* pic_init_qs_minus26 */
+	bs_write_se(s, 0); /* chroma_qp_index_offset */
+	bs_write(s, 1, 0); /* deblocking_filter_control_present_flag */
+	bs_write(s, 1, 0); /* constrained_intra_pred_flag */
+	bs_write(s, 1, 0); /* redundant_pic_cnt_present_flag */
+	bs_rbsp_trailing(s);
+	return bs_len(s);
+}
+
+static int tw5864_h264_gen_slice_head(u8 *buf, size_t size,
+				      unsigned int idr_pic_id,
+				      unsigned int frame_gop_seqno,
+				      int *tail_nb_bits, u8 *tail)
+{
+	struct bs bs, *s;
+	int is_i_frame = frame_gop_seqno == 0;
+
+	s = &bs;
+	bs_init(s, buf, size);
+	bs_write_ue(s, 0); /* first_mb_in_slice */
+	bs_write_ue(s, is_i_frame ? 2 : 5); /* slice_type - I or P */
+	bs_write_ue(s, 0); /* pic_parameter_set_id */
+	bs_write(s, ilog2(MAX_GOP_SIZE), frame_gop_seqno); /* frame_num */
+	if (is_i_frame)
+		bs_write_ue(s, idr_pic_id);
+
+	/* pic_order_cnt_lsb */
+	bs_write(s, ilog2(MAX_GOP_SIZE), frame_gop_seqno);
+
+	if (is_i_frame) {
+		bs_write1(s, 0); /* no_output_of_prior_pics_flag */
+		bs_write1(s, 0); /* long_term_reference_flag */
+	} else {
+		bs_write1(s, 0); /* num_ref_idx_active_override_flag */
+		bs_write1(s, 0); /* ref_pic_list_reordering_flag_l0 */
+		bs_write1(s, 0); /* adaptive_ref_pic_marking_mode_flag */
+	}
+
+	bs_write_se(s, 0); /* slice_qp_delta */
+
+	if (s->bits_left != 8) {
+		*tail = ((s->ptr[0]) << s->bits_left);
+		*tail_nb_bits = 8 - s->bits_left;
+	} else {
+		*tail = 0;
+		*tail_nb_bits = 0;
+	}
+
+	return bs_len(s);
+}
+
+void tw5864_h264_put_stream_header(u8 **buf, size_t *space_left, int qp,
+				   int width, int height)
+{
+	int nal_len;
+
+	/* SPS */
+	memcpy(*buf, marker, sizeof(marker));
+	*buf += 4;
+	*space_left -= 4;
+
+	**buf = 0x67; /* SPS NAL header */
+	*buf += 1;
+	*space_left -= 1;
+
+	nal_len = tw5864_h264_gen_sps_rbsp(*buf, *space_left, width, height);
+	*buf += nal_len;
+	*space_left -= nal_len;
+
+	/* PPS */
+	memcpy(*buf, marker, sizeof(marker));
+	*buf += 4;
+	*space_left -= 4;
+
+	**buf = 0x68; /* PPS NAL header */
+	*buf += 1;
+	*space_left -= 1;
+
+	nal_len = tw5864_h264_gen_pps_rbsp(*buf, *space_left, qp);
+	*buf += nal_len;
+	*space_left -= nal_len;
+}
+
+void tw5864_h264_put_slice_header(u8 **buf, size_t *space_left,
+				  unsigned int idr_pic_id,
+				  unsigned int frame_gop_seqno,
+				  int *tail_nb_bits, u8 *tail)
+{
+	int nal_len;
+
+	memcpy(*buf, marker, sizeof(marker));
+	*buf += 4;
+	*space_left -= 4;
+
+	/* Frame NAL header */
+	**buf = (frame_gop_seqno == 0) ? 0x25 : 0x21;
+	*buf += 1;
+	*space_left -= 1;
+
+	nal_len = tw5864_h264_gen_slice_head(*buf, *space_left, idr_pic_id,
+					     frame_gop_seqno, tail_nb_bits,
+					     tail);
+	*buf += nal_len;
+	*space_left -= nal_len;
+}
diff --git a/drivers/media/pci/tw5864/tw5864-reg.h b/drivers/media/pci/tw5864/tw5864-reg.h
new file mode 100644
index 000000000000..92a1b077ef8a
--- /dev/null
+++ b/drivers/media/pci/tw5864/tw5864-reg.h
@@ -0,0 +1,2133 @@
+/*
+ *  TW5864 driver - registers description
+ *
+ *  Copyright (C) 2016 Bluecherry, LLC <maintainers@bluecherrydvr.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ */
+
+/* According to TW5864_datasheet_0.6d.pdf, tw5864b1-ds.pdf */
+
+/* Register Description - Direct Map Space */
+/* 0x0000 ~ 0x1ffc - H264 Register Map */
+/* [15:0] The Version register for H264 core (Read Only) */
+#define TW5864_H264REV 0x0000
+
+#define TW5864_EMU 0x0004
+/* Define controls in register TW5864_EMU */
+/* DDR controller enabled */
+#define TW5864_EMU_EN_DDR BIT(0)
+/* Enable bit for Inter module */
+#define TW5864_EMU_EN_ME BIT(1)
+/* Enable bit for Sensor Interface module */
+#define TW5864_EMU_EN_SEN BIT(2)
+/* Enable bit for Host Burst Access */
+#define TW5864_EMU_EN_BHOST BIT(3)
+/* Enable bit for Loop Filter module */
+#define TW5864_EMU_EN_LPF BIT(4)
+/* Enable bit for PLBK module */
+#define TW5864_EMU_EN_PLBK BIT(5)
+/*
+ * Video Frame mapping in DDR
+ * 00 CIF
+ * 01 D1
+ * 10 Reserved
+ * 11 Reserved
+ *
+ */
+#define TW5864_DSP_FRAME_TYPE (3 << 6)
+#define TW5864_DSP_FRAME_TYPE_D1 BIT(6)
+
+#define TW5864_UNDECLARED_H264REV_PART2 0x0008
+
+#define TW5864_SLICE 0x000c
+/* Define controls in register TW5864_SLICE */
+/* VLC Slice end flag */
+#define TW5864_VLC_SLICE_END BIT(0)
+/* Master Slice End Flag */
+#define TW5864_MAS_SLICE_END BIT(4)
+/* Host to start a new slice Address */
+#define TW5864_START_NSLICE BIT(15)
+
+/*
+ * [15:0] Two bit for each channel (channel 0 ~ 7). Each two bits are the buffer
+ * pointer for the last encoded frame of the corresponding channel.
+ */
+#define TW5864_ENC_BUF_PTR_REC1 0x0010
+
+/* [5:0] DSP_MB_QP and [15:10] DSP_LPF_OFFSET */
+#define TW5864_DSP_QP 0x0018
+/* Define controls in register TW5864_DSP_QP */
+/* [5:0] H264 QP Value for codec */
+#define TW5864_DSP_MB_QP 0x003f
+/*
+ * [15:10] H264 LPF_OFFSET Address
+ * (Default 0)
+ */
+#define TW5864_DSP_LPF_OFFSET 0xfc00
+
+#define TW5864_DSP_CODEC 0x001c
+/* Define controls in register TW5864_DSP_CODEC */
+/*
+ * 0: Encode (TW5864 Default)
+ * 1: Decode
+ */
+#define TW5864_DSP_CODEC_MODE BIT(0)
+/*
+ * 0->3 4 VLC data buffer in DDR (1M each)
+ * 0->7 8 VLC data buffer in DDR (512k each)
+ */
+#define TW5864_VLC_BUF_ID (7 << 2)
+/*
+ * 0 4CIF in 1 MB
+ * 1 1CIF in 1 MB
+ */
+#define TW5864_CIF_MAP_MD BIT(6)
+/*
+ * 0 2 falf D1 in 1 MB
+ * 1 1 half D1 in 1 MB
+ */
+#define TW5864_HD1_MAP_MD BIT(7)
+/* VLC Stream valid */
+#define TW5864_VLC_VLD BIT(8)
+/* MV Vector Valid */
+#define TW5864_MV_VECT_VLD BIT(9)
+/* MV Flag Valid */
+#define TW5864_MV_FLAG_VLD BIT(10)
+
+#define TW5864_DSP_SEN 0x0020
+/* Define controls in register TW5864_DSP_SEN */
+/* Org Buffer Base for Luma (default 0) */
+#define TW5864_DSP_SEN_PIC_LU 0x000f
+/* Org Buffer Base for Chroma (default 4) */
+#define TW5864_DSP_SEN_PIC_CHM 0x00f0
+/* Maximum Number of Buffers (default 4) */
+#define TW5864_DSP_SEN_PIC_MAX 0x0700
+/*
+ * Original Frame D1 or HD1 switch
+ * (Default 0)
+ */
+#define TW5864_DSP_SEN_HFULL 0x1000
+
+#define TW5864_DSP_REF_PIC 0x0024
+/* Define controls in register TW5864_DSP_REF_PIC */
+/* Ref Buffer Base for Luma (default 0) */
+#define TW5864_DSP_REF_PIC_LU 0x000f
+/* Ref Buffer Base for Chroma (default 4) */
+#define TW5864_DSP_REF_PIC_CHM 0x00f0
+/* Maximum Number of Buffers (default 4) */
+#define TW5864_DSP_REF_PIC_MAX 0x0700
+
+/* [15:0] SEN_EN_CH[n] SENIF original frame capture enable for each channel */
+#define TW5864_SEN_EN_CH 0x0028
+
+#define TW5864_DSP 0x002c
+/* Define controls in register TW5864_DSP */
+/* The ID for channel selected for encoding operation */
+#define TW5864_DSP_ENC_CHN 0x000f
+/* See DSP_MB_DELAY below */
+#define TW5864_DSP_MB_WAIT 0x0010
+/*
+ * DSP Chroma Switch
+ * 0 DDRB
+ * 1 DDRA
+ */
+#define TW5864_DSP_CHROM_SW 0x0020
+/* VLC Flow Control: 1 for enable */
+#define TW5864_DSP_FLW_CNTL 0x0040
+/*
+ * If DSP_MB_WAIT == 0, MB delay is DSP_MB_DELAY * 16
+ * If DSP_MB_DELAY == 1, MB delay is DSP_MB_DELAY * 128
+ */
+#define TW5864_DSP_MB_DELAY 0x0f00
+
+#define TW5864_DDR 0x0030
+/* Define controls in register TW5864_DDR */
+/* DDR Single Access Page Number */
+#define TW5864_DDR_PAGE_CNTL 0x00ff
+/* DDR-DPR Burst Read Enable */
+#define TW5864_DDR_BRST_EN BIT(13)
+/*
+ * DDR A/B Select as HOST access
+ * 0 Select DDRA
+ * 1 Select DDRB
+ */
+#define TW5864_DDR_AB_SEL BIT(14)
+/*
+ * DDR Access Mode Select
+ * 0 Single R/W Access (Host <-> DDR)
+ * 1 Burst R/W Access (Host <-> DPR)
+ */
+#define TW5864_DDR_MODE BIT(15)
+
+/* The original frame capture pointer. Two bits for each channel */
+/* SENIF_ORG_FRM_PTR [15:0] */
+#define TW5864_SENIF_ORG_FRM_PTR1 0x0038
+/* SENIF_ORG_FRM_PTR [31:16] */
+#define TW5864_SENIF_ORG_FRM_PTR2 0x003c
+
+#define TW5864_DSP_SEN_MODE 0x0040
+/* Define controls in register TW5864_DSP_SEN_MODE */
+#define TW5864_DSP_SEN_MODE_CH0 0x000f
+#define TW5864_DSP_SEN_MODE_CH1 0x00f0
+
+/*
+ * [15:0]: ENC_BUF_PTR_REC[31:16] Two bit for each channel (channel 8 ~ 15).
+ * Each two bits are the buffer pointer for the last encoded frame of a channel
+ */
+#define TW5864_ENC_BUF_PTR_REC2 0x004c
+
+/* Current MV Flag Status Pointer for Channel n. (Read only) */
+/*
+ * [1:0] CH0_MV_PTR, ..., [15:14] CH7_MV_PTR
+ */
+#define TW5864_CH_MV_PTR1 0x0060
+/*
+ * [1:0] CH8_MV_PTR, ..., [15:14] CH15_MV_PTR
+ */
+#define TW5864_CH_MV_PTR2 0x0064
+
+/*
+ * [15:0] Reset Current MV Flag Status Pointer for Channel n (one bit each)
+ */
+#define TW5864_RST_MV_PTR 0x0068
+#define TW5864_INTERLACING 0x0200
+/* Define controls in register TW5864_INTERLACING */
+/*
+ * Inter_Mode Start. 2-nd bit? A guess. Missing in datasheet. Without this bit
+ * set, the output video is interlaced (stripy).
+ */
+#define TW5864_DSP_INTER_ST BIT(1)
+/* Deinterlacer Enable */
+#define TW5864_DI_EN BIT(2)
+/*
+ * De-interlacer Mode
+ * 1 Shuffled frame
+ * 0 Normal Un-Shuffled Frame
+ */
+#define TW5864_DI_MD BIT(3)
+/*
+ * Down scale original frame in X direction
+ * 11: Un-used
+ * 10: down-sample to 1/4
+ * 01: down-sample to 1/2
+ * 00: down-sample disabled
+ */
+#define TW5864_DSP_DWN_X (3 << 4)
+/*
+ * Down scale original frame in Y direction
+ * 11: Un-used
+ * 10: down-sample to 1/4
+ * 01: down-sample to 1/2
+ * 00: down-sample disabled
+ */
+#define TW5864_DSP_DWN_Y (3 << 6)
+/*
+ * 1 Dual Stream
+ * 0 Single Stream
+ */
+#define TW5864_DUAL_STR BIT(8)
+
+#define TW5864_DSP_REF 0x0204
+/* Define controls in register TW5864_DSP_REF */
+/* Number of reference frame (Default 1 for TW5864B) */
+#define TW5864_DSP_REF_FRM 0x000f
+/* Window size */
+#define TW5864_DSP_WIN_SIZE 0x02f0
+
+#define TW5864_DSP_SKIP 0x0208
+/* Define controls in register TW5864_DSP_SKIP */
+/*
+ * Skip Offset Enable bit
+ * 0 DSP_SKIP_OFFSET value is not used (default 8)
+ * 1 DSP_SKIP_OFFSET value is used in HW
+ */
+#define TW5864_DSP_SKIP_OFEN 0x0080
+/* Skip mode cost offset (default 8) */
+#define TW5864_DSP_SKIP_OFFSET 0x007f
+
+#define TW5864_MOTION_SEARCH_ETC 0x020c
+/* Define controls in register TW5864_MOTION_SEARCH_ETC */
+/* Enable quarter pel search mode */
+#define TW5864_QPEL_EN BIT(0)
+/* Enable half pel search mode */
+#define TW5864_HPEL_EN BIT(1)
+/* Enable motion search mode */
+#define TW5864_ME_EN BIT(2)
+/* Enable Intra mode */
+#define TW5864_INTRA_EN BIT(3)
+/* Enable Skip Mode */
+#define TW5864_SKIP_EN BIT(4)
+/* Search Option (Default 2"b01) */
+#define TW5864_SRCH_OPT (3 << 5)
+
+#define TW5864_DSP_ENC_REC 0x0210
+/* Define controls in register TW5864_DSP_ENC_REC */
+/* Reference Buffer Pointer for encoding */
+#define TW5864_DSP_ENC_REF_PTR 0x0007
+/* Reconstruct Buffer pointer */
+#define TW5864_DSP_REC_BUF_PTR 0x7000
+
+/* [15:0] Lambda Value for H264 */
+#define TW5864_DSP_REF_MVP_LAMBDA 0x0214
+
+#define TW5864_DSP_PIC_MAX_MB 0x0218
+/* Define controls in register TW5864_DSP_PIC_MAX_MB */
+/* The MB number in Y direction for a frame */
+#define TW5864_DSP_PIC_MAX_MB_Y 0x007f
+/* The MB number in X direction for a frame */
+#define TW5864_DSP_PIC_MAX_MB_X 0x7f00
+
+/* The original frame pointer for encoding */
+#define TW5864_DSP_ENC_ORG_PTR_REG 0x021c
+/* Mask to use with TW5864_DSP_ENC_ORG_PTR */
+#define TW5864_DSP_ENC_ORG_PTR_MASK 0x7000
+/* Number of bits to shift with TW5864_DSP_ENC_ORG_PTR */
+#define TW5864_DSP_ENC_ORG_PTR_SHIFT 12
+
+/* DDR base address of OSD rectangle attribute data */
+#define TW5864_DSP_OSD_ATTRI_BASE 0x0220
+/* OSD enable bit for each channel */
+#define TW5864_DSP_OSD_ENABLE 0x0228
+
+/* 0x0280 ~ 0x029c – Motion Vector for 1st 4x4 Block, e.g., 80 (X), 84 (Y) */
+#define TW5864_ME_MV_VEC1 0x0280
+/* 0x02a0 ~ 0x02bc – Motion Vector for 2nd 4x4 Block, e.g., A0 (X), A4 (Y) */
+#define TW5864_ME_MV_VEC2 0x02a0
+/* 0x02c0 ~ 0x02dc – Motion Vector for 3rd 4x4 Block, e.g., C0 (X), C4 (Y) */
+#define TW5864_ME_MV_VEC3 0x02c0
+/* 0x02e0 ~ 0x02fc – Motion Vector for 4th 4x4 Block, e.g., E0 (X), E4 (Y) */
+#define TW5864_ME_MV_VEC4 0x02e0
+
+/*
+ * [5:0]
+ * if (intra16x16_cost < (intra4x4_cost+dsp_i4x4_offset))
+ * Intra_mode = intra16x16_mode
+ * Else
+ * Intra_mode = intra4x4_mode
+ */
+#define TW5864_DSP_I4x4_OFFSET 0x040c
+
+/*
+ * [6:4]
+ * 0x5 Only 4x4
+ * 0x6 Only 16x16
+ * 0x7 16x16 & 4x4
+ */
+#define TW5864_DSP_INTRA_MODE 0x0410
+#define TW5864_DSP_INTRA_MODE_SHIFT 4
+#define TW5864_DSP_INTRA_MODE_MASK (7 << 4)
+#define TW5864_DSP_INTRA_MODE_4x4 0x5
+#define TW5864_DSP_INTRA_MODE_16x16 0x6
+#define TW5864_DSP_INTRA_MODE_4x4_AND_16x16 0x7
+/*
+ * [5:0] WEIGHT Factor for I4x4 cost calculation (QP dependent)
+ */
+#define TW5864_DSP_I4x4_WEIGHT 0x0414
+
+/*
+ * [7:0] Offset used to affect Intra/ME model decision
+ * If (me_cost < intra_cost + dsp_resid_mode_offset)
+ * Pred_Mode = me_mode
+ * Else
+ * Pred_mode = intra_mode
+ */
+#define TW5864_DSP_RESID_MODE_OFFSET 0x0604
+
+/* 0x0800 ~ 0x09ff - Quantization TABLE Values */
+#define TW5864_QUAN_TAB 0x0800
+
+/* Valid channel value [0; f], frame value [0; 3] */
+#define TW5864_RT_CNTR_CH_FRM(channel, frame) \
+	(0x0c00 | (channel << 4) | (frame << 2))
+
+#define TW5864_FRAME_BUS1 0x0d00
+/*
+ * 1 Progressive in part A in bus n
+ * 0 Interlaced in part A in bus n
+ */
+#define TW5864_PROG_A BIT(0)
+/*
+ * 1 Progressive in part B in bus n
+ * 0 Interlaced in part B in bus n
+ */
+#define TW5864_PROG_B BIT(1)
+/*
+ * 1 Frame Mode in bus n
+ * 0 Field Mode in bus n
+ */
+#define TW5864_FRAME BIT(2)
+/*
+ * 0 4CIF in bus n
+ * 1 1D1 + 4 CIF in bus n
+ * 2 2D1 in bus n
+ */
+#define TW5864_BUS_D1 (3 << 3)
+/* Bus 1 goes in TW5864_FRAME_BUS1 in [4:0] */
+/* Bus 2 goes in TW5864_FRAME_BUS1 in [12:8] */
+#define TW5864_FRAME_BUS2 0x0d04
+/* Bus 3 goes in TW5864_FRAME_BUS2 in [4:0] */
+/* Bus 4 goes in TW5864_FRAME_BUS2 in [12:8] */
+
+/* [15:0] Horizontal Mirror for channel n */
+#define TW5864_SENIF_HOR_MIR 0x0d08
+/* [15:0] Vertical Mirror for channel n */
+#define TW5864_SENIF_VER_MIR 0x0d0c
+
+/*
+ * FRAME_WIDTH_BUSn_A
+ * 0x15f: 4 CIF
+ * 0x2cf: 1 D1 + 3 CIF
+ * 0x2cf: 2 D1
+ * FRAME_WIDTH_BUSn_B
+ * 0x15f: 4 CIF
+ * 0x2cf: 1 D1 + 3 CIF
+ * 0x2cf: 2 D1
+ * FRAME_HEIGHT_BUSn_A
+ * 0x11f: 4CIF (PAL)
+ * 0x23f: 1D1 + 3CIF (PAL)
+ * 0x23f: 2 D1 (PAL)
+ * 0x0ef: 4CIF (NTSC)
+ * 0x1df: 1D1 + 3CIF (NTSC)
+ * 0x1df: 2 D1 (NTSC)
+ * FRAME_HEIGHT_BUSn_B
+ * 0x11f: 4CIF (PAL)
+ * 0x23f: 1D1 + 3CIF (PAL)
+ * 0x23f: 2 D1 (PAL)
+ * 0x0ef: 4CIF (NTSC)
+ * 0x1df: 1D1 + 3CIF (NTSC)
+ * 0x1df: 2 D1 (NTSC)
+ */
+#define TW5864_FRAME_WIDTH_BUS_A(bus) (0x0d10 + 0x0010 * bus)
+#define TW5864_FRAME_WIDTH_BUS_B(bus) (0x0d14 + 0x0010 * bus)
+#define TW5864_FRAME_HEIGHT_BUS_A(bus) (0x0d18 + 0x0010 * bus)
+#define TW5864_FRAME_HEIGHT_BUS_B(bus) (0x0d1c + 0x0010 * bus)
+
+/*
+ * 1: the bus mapped Channel n Full D1
+ * 0: the bus mapped Channel n Half D1
+ */
+#define TW5864_FULL_HALF_FLAG 0x0d50
+
+/*
+ * 0 The bus mapped Channel select partA Mode
+ * 1 The bus mapped Channel select partB Mode
+ */
+#define TW5864_FULL_HALF_MODE_SEL 0x0d54
+
+#define TW5864_VLC 0x1000
+/* Define controls in register TW5864_VLC */
+/* QP Value used by H264 CAVLC */
+#define TW5864_VLC_SLICE_QP 0x003f
+/*
+ * Swap byte order of VLC stream in d-word.
+ * 1 Normal (VLC output= [31:0])
+ * 0 Swap (VLC output={[23:16],[31:24],[7:0], [15:8]})
+ */
+#define TW5864_VLC_BYTE_SWP BIT(6)
+/* Enable Adding 03 circuit for VLC stream */
+#define TW5864_VLC_ADD03_EN BIT(7)
+/* Number of bit for VLC bit Align */
+#define TW5864_VLC_BIT_ALIGN_SHIFT 8
+#define TW5864_VLC_BIT_ALIGN_MASK (0x1f << 8)
+/*
+ * Synchronous Interface select for VLC Stream
+ * 1 CDC_VLCS_MAS read VLC stream
+ * 0 CPU read VLC stream
+ */
+#define TW5864_VLC_INF_SEL BIT(13)
+/* Enable VLC overflow control */
+#define TW5864_VLC_OVFL_CNTL BIT(14)
+/*
+ * 1 PCI Master Mode
+ * 0 Non PCI Master Mode
+ */
+#define TW5864_VLC_PCI_SEL BIT(15)
+/*
+ * 0 Enable Adding 03 to VLC header and stream
+ * 1 Disable Adding 03 to VLC header of "00000001"
+ */
+#define TW5864_VLC_A03_DISAB BIT(16)
+/*
+ * Status of VLC stream in DDR (one bit for each buffer)
+ * 1 VLC is ready in buffer n (HW set)
+ * 0 VLC is not ready in buffer n (SW clear)
+ */
+#define TW5864_VLC_BUF_RDY_SHIFT 24
+#define TW5864_VLC_BUF_RDY_MASK (0xff << 24)
+
+/* Total number of bit in the slice */
+#define TW5864_SLICE_TOTAL_BIT 0x1004
+/* Total number of bit in the residue */
+#define TW5864_RES_TOTAL_BIT 0x1008
+
+#define TW5864_VLC_BUF 0x100c
+/* Define controls in register TW5864_VLC_BUF */
+/* VLC BK0 full status, write ‘1’ to clear */
+#define TW5864_VLC_BK0_FULL BIT(0)
+/* VLC BK1 full status, write ‘1’ to clear */
+#define TW5864_VLC_BK1_FULL BIT(1)
+/* VLC end slice status, write ‘1’ to clear */
+#define TW5864_VLC_END_SLICE BIT(2)
+/* VLC Buffer overflow status, write ‘1’ to clear */
+#define TW5864_DSP_RD_OF BIT(3)
+/* VLC string length in either buffer 0 or 1 at end of frame */
+#define TW5864_VLC_STREAM_LEN_SHIFT 4
+#define TW5864_VLC_STREAM_LEN_MASK (0x1ff << 4)
+
+/* [15:0] Total coefficient number in a frame */
+#define TW5864_TOTAL_COEF_NO 0x1010
+/* [0] VLC Encoder Interrupt. Write ‘1’ to clear */
+#define TW5864_VLC_DSP_INTR 0x1014
+/* [31:0] VLC stream CRC checksum */
+#define TW5864_VLC_STREAM_CRC 0x1018
+
+#define TW5864_VLC_RD 0x101c
+/* Define controls in register TW5864_VLC_RD */
+/*
+ * 1 Read VLC lookup Memory
+ * 0 Read VLC Stream Memory
+ */
+#define TW5864_VLC_RD_MEM BIT(0)
+/*
+ * 1 Read VLC Stream Memory in burst mode
+ * 0 Read VLC Stream Memory in single mode
+ */
+#define TW5864_VLC_RD_BRST BIT(1)
+
+/* 0x2000 ~ 0x2ffc -- H264 Stream Memory Map */
+/*
+ * A word is 4 bytes. I.e.,
+ * VLC_STREAM_MEM[0] address: 0x2000
+ * VLC_STREAM_MEM[1] address: 0x2004
+ * ...
+ * VLC_STREAM_MEM[3FF] address: 0x2ffc
+ */
+#define TW5864_VLC_STREAM_MEM_START 0x2000
+#define TW5864_VLC_STREAM_MEM_MAX_OFFSET 0x3ff
+#define TW5864_VLC_STREAM_MEM(offset) (TW5864_VLC_STREAM_MEM_START + 4 * offset)
+
+/* 0x4000 ~ 0x4ffc -- Audio Register Map */
+/* [31:0] config 1ms cnt = Realtime clk/1000 */
+#define TW5864_CFG_1MS_CNT 0x4000
+
+#define TW5864_ADPCM 0x4004
+/* Define controls in register TW5864_ADPCM */
+/* ADPCM decoder enable */
+#define TW5864_ADPCM_DEC BIT(0)
+/* ADPCM input data enable */
+#define TW5864_ADPCM_IN_DATA BIT(1)
+/* ADPCM encoder enable */
+#define TW5864_ADPCM_ENC BIT(2)
+
+#define TW5864_AUD 0x4008
+/* Define controls in register TW5864_AUD */
+/* Record path PCM Audio enable bit for each channel */
+#define TW5864_AUD_ORG_CH_EN 0x00ff
+/* Speaker path PCM Audio Enable */
+#define TW5864_SPK_ORG_EN BIT(16)
+/*
+ * 0 16bit
+ * 1 8bit
+ */
+#define TW5864_AD_BIT_MODE BIT(17)
+#define TW5864_AUD_TYPE_SHIFT 18
+/*
+ * 0 PCM
+ * 3 ADPCM
+ */
+#define TW5864_AUD_TYPE (0xf << 18)
+#define TW5864_AUD_SAMPLE_RATE_SHIFT 22
+/*
+ * 0 8K
+ * 1 16K
+ */
+#define TW5864_AUD_SAMPLE_RATE (3 << 22)
+/* Channel ID used to select audio channel (0 to 16) for loopback */
+#define TW5864_TESTLOOP_CHID_SHIFT 24
+#define TW5864_TESTLOOP_CHID (0x1f << 24)
+/* Enable AD Loopback Test */
+#define TW5864_TEST_ADLOOP_EN BIT(30)
+/*
+ * 0 Asynchronous Mode or PCI target mode
+ * 1 PCI Initiator Mode
+ */
+#define TW5864_AUD_MODE BIT(31)
+
+#define TW5864_AUD_ADPCM 0x400c
+/* Define controls in register TW5864_AUD_ADPCM */
+/* Record path ADPCM audio channel enable, one bit for each */
+#define TW5864_AUD_ADPCM_CH_EN 0x00ff
+/* Speaker path ADPCM audio channel enable */
+#define TW5864_SPK_ADPCM_EN BIT(16)
+
+#define TW5864_PC_BLOCK_ADPCM_RD_NO 0x4018
+#define TW5864_PC_BLOCK_ADPCM_RD_NO_MASK 0x1f
+
+/*
+ * For ADPCM_ENC_WR_PTR, ADPCM_ENC_RD_PTR (see below):
+ * Bit[2:0] ch0
+ * Bit[5:3] ch1
+ * Bit[8:6] ch2
+ * Bit[11:9] ch3
+ * Bit[14:12] ch4
+ * Bit[17:15] ch5
+ * Bit[20:18] ch6
+ * Bit[23:21] ch7
+ * Bit[26:24] ch8
+ * Bit[29:27] ch9
+ * Bit[32:30] ch10
+ * Bit[35:33] ch11
+ * Bit[38:36] ch12
+ * Bit[41:39] ch13
+ * Bit[44:42] ch14
+ * Bit[47:45] ch15
+ * Bit[50:48] ch16
+ */
+#define TW5864_ADPCM_ENC_XX_MASK 0x3fff
+#define TW5864_ADPCM_ENC_XX_PTR2_SHIFT 30
+/* ADPCM_ENC_WR_PTR[29:0] */
+#define TW5864_ADPCM_ENC_WR_PTR1 0x401c
+/* ADPCM_ENC_WR_PTR[50:30] */
+#define TW5864_ADPCM_ENC_WR_PTR2 0x4020
+
+/* ADPCM_ENC_RD_PTR[29:0] */
+#define TW5864_ADPCM_ENC_RD_PTR1 0x4024
+/* ADPCM_ENC_RD_PTR[50:30] */
+#define TW5864_ADPCM_ENC_RD_PTR2 0x4028
+
+/* [3:0] rd ch0, [7:4] rd ch1, [11:8] wr ch0, [15:12] wr ch1 */
+#define TW5864_ADPCM_DEC_RD_WR_PTR 0x402c
+
+/*
+ * For TW5864_AD_ORIG_WR_PTR, TW5864_AD_ORIG_RD_PTR:
+ * Bit[3:0] ch0
+ * Bit[7:4] ch1
+ * Bit[11:8] ch2
+ * Bit[15:12] ch3
+ * Bit[19:16] ch4
+ * Bit[23:20] ch5
+ * Bit[27:24] ch6
+ * Bit[31:28] ch7
+ * Bit[35:32] ch8
+ * Bit[39:36] ch9
+ * Bit[43:40] ch10
+ * Bit[47:44] ch11
+ * Bit[51:48] ch12
+ * Bit[55:52] ch13
+ * Bit[59:56] ch14
+ * Bit[63:60] ch15
+ * Bit[67:64] ch16
+ */
+/* AD_ORIG_WR_PTR[31:0] */
+#define TW5864_AD_ORIG_WR_PTR1 0x4030
+/* AD_ORIG_WR_PTR[63:32] */
+#define TW5864_AD_ORIG_WR_PTR2 0x4034
+/* AD_ORIG_WR_PTR[67:64] */
+#define TW5864_AD_ORIG_WR_PTR3 0x4038
+
+/* AD_ORIG_RD_PTR[31:0] */
+#define TW5864_AD_ORIG_RD_PTR1 0x403c
+/* AD_ORIG_RD_PTR[63:32] */
+#define TW5864_AD_ORIG_RD_PTR2 0x4040
+/* AD_ORIG_RD_PTR[67:64] */
+#define TW5864_AD_ORIG_RD_PTR3 0x4044
+
+#define TW5864_PC_BLOCK_ORIG_RD_NO 0x4048
+#define TW5864_PC_BLOCK_ORIG_RD_NO_MASK 0x1f
+
+#define TW5864_PCI_AUD 0x404c
+/* Define controls in register TW5864_PCI_AUD */
+/*
+ * The register is applicable to PCI initiator mode only. Used to select PCM(0)
+ * or ADPCM(1) audio data sent to PC. One bit for each channel
+ */
+#define TW5864_PCI_DATA_SEL 0xffff
+/*
+ * Audio flow control mode selection bit.
+ * 0 Flow control disabled. TW5864 continuously sends audio frame to PC
+ * (initiator mode)
+ * 1 Flow control enabled
+ */
+#define TW5864_PCI_FLOW_EN BIT(16)
+/*
+ * When PCI_FLOW_EN is set, PCI need to toggle this bit to send an audio frame
+ * to PC. One toggle to send one frame.
+ */
+#define TW5864_PCI_AUD_FRM_EN BIT(17)
+
+/* [1:0] CS valid to data valid CLK cycles when writing operation */
+#define TW5864_CS2DAT_CNT 0x8000
+/* [2:0] Data valid signal width by system clock cycles */
+#define TW5864_DATA_VLD_WIDTH 0x8004
+
+#define TW5864_SYNC 0x8008
+/* Define controls in register TW5864_SYNC */
+/*
+ * 0 vlc stream to syncrous port
+ * 1 vlc stream to ddr buffers
+ */
+#define TW5864_SYNC_CFG BIT(7)
+/*
+ * 0 SYNC Address sampled on Rising edge
+ * 1 SYNC Address sampled on Falling edge
+ */
+#define TW5864_SYNC_ADR_EDGE BIT(0)
+#define TW5864_VLC_STR_DELAY_SHIFT 1
+/*
+ * 0 No system delay
+ * 1 One system clock delay
+ * 2 Two system clock delay
+ * 3 Three system clock delay
+ */
+#define TW5864_VLC_STR_DELAY (3 << 1)
+/*
+ * 0 Rising edge output
+ * 1 Falling edge output
+ */
+#define TW5864_VLC_OUT_EDGE BIT(3)
+
+/*
+ * [1:0]
+ * 2’b00 phase set to 180 degree
+ * 2’b01 phase set to 270 degree
+ * 2’b10 phase set to 0 degree
+ * 2’b11 phase set to 90 degree
+ */
+#define TW5864_I2C_PHASE_CFG 0x800c
+
+/*
+ * The system / DDR clock (166 MHz) is generated with an on-chip system clock
+ * PLL (SYSPLL) using input crystal clock of 27 MHz. The system clock PLL
+ * frequency is controlled with the following equation.
+ * CLK_OUT = CLK_IN * (M+1) / ((N+1) * P)
+ * SYSPLL_M M parameter
+ * SYSPLL_N N parameter
+ * SYSPLL_P P parameter
+ */
+/* SYSPLL_M[7:0] */
+#define TW5864_SYSPLL1 0x8018
+/* Define controls in register TW5864_SYSPLL1 */
+#define TW5864_SYSPLL_M_LOW 0x00ff
+
+/* [2:0]: SYSPLL_M[10:8], [7:3]: SYSPLL_N[4:0] */
+#define TW5864_SYSPLL2 0x8019
+/* Define controls in register TW5864_SYSPLL2 */
+#define TW5864_SYSPLL_M_HI 0x07
+#define TW5864_SYSPLL_N_LOW_SHIFT 3
+#define TW5864_SYSPLL_N_LOW (0x1f << 3)
+
+/*
+ * [1:0]: SYSPLL_N[6:5], [3:2]: SYSPLL_P, [4]: SYSPLL_IREF, [7:5]: SYSPLL_CP_SEL
+ */
+#define TW5864_SYSPLL3 0x8020
+/* Define controls in register TW5864_SYSPLL3 */
+#define TW5864_SYSPLL_N_HI 0x03
+#define TW5864_SYSPLL_P_SHIFT 2
+#define TW5864_SYSPLL_P (0x03 << 2)
+/*
+ * SYSPLL bias current control
+ * 0 Lower current (default)
+ * 1 30% higher current
+ */
+#define TW5864_SYSPLL_IREF BIT(4)
+/*
+ * SYSPLL charge pump current selection
+ * 0 1,5 uA
+ * 1 4 uA
+ * 2 9 uA
+ * 3 19 uA
+ * 4 39 uA
+ * 5 79 uA
+ * 6 159 uA
+ * 7 319 uA
+ */
+#define TW5864_SYSPLL_CP_SEL_SHIFT 5
+#define TW5864_SYSPLL_CP_SEL (0x07 << 5)
+
+/*
+ * [1:0]: SYSPLL_VCO, [3:2]: SYSPLL_LP_X8, [5:4]: SYSPLL_ICP_SEL,
+ * [6]: SYSPLL_LPF_5PF, [7]: SYSPLL_ED_SEL
+ */
+#define TW5864_SYSPLL4 0x8021
+/* Define controls in register TW5864_SYSPLL4 */
+/*
+ * SYSPLL_VCO VCO Range selection
+ * 00 5 ~ 75 MHz
+ * 01 50 ~ 140 MHz
+ * 10 110 ~ 320 MHz
+ * 11 270 ~ 700 MHz
+ */
+#define TW5864_SYSPLL_VCO 0x03
+#define TW5864_SYSPLL_LP_X8_SHIFT 2
+/*
+ * Loop resister
+ * 0 38.5K ohms
+ * 1 6.6K ohms (default)
+ * 2 2.2K ohms
+ * 3 1.1K ohms
+ */
+#define TW5864_SYSPLL_LP_X8 (0x03 << 2)
+#define TW5864_SYSPLL_ICP_SEL_SHIFT 4
+/*
+ * PLL charge pump fine tune
+ * 00 x1 (default)
+ * 01 x1/2
+ * 10 x1/7
+ * 11 x1/8
+ */
+#define TW5864_SYSPLL_ICP_SEL (0x03 << 4)
+/*
+ * PLL low pass filter phase margin adjustment
+ * 0 no 5pF (default)
+ * 1 5pF added
+ */
+#define TW5864_SYSPLL_LPF_5PF BIT(6)
+/*
+ * PFD select edge for detection
+ * 0 Falling edge (default)
+ * 1 Rising edge
+ */
+#define TW5864_SYSPLL_ED_SEL BIT(7)
+
+/* [0]: SYSPLL_RST, [4]: SYSPLL_PD */
+#define TW5864_SYSPLL5 0x8024
+/* Define controls in register TW5864_SYSPLL5 */
+/* Reset SYSPLL */
+#define TW5864_SYSPLL_RST BIT(0)
+/* Power down SYSPLL */
+#define TW5864_SYSPLL_PD BIT(4)
+
+#define TW5864_PLL_CFG 0x801c
+/* Define controls in register TW5864_PLL_CFG */
+/*
+ * Issue Soft Reset from Async Host Interface / PCI Interface clock domain.
+ * Become valid after sync to the xtal clock domain. This bit is set only if
+ * LOAD register bit is also set to 1.
+ */
+#define TW5864_SRST BIT(0)
+/*
+ * Issue SYSPLL (166 MHz) configuration latch from Async host interface / PCI
+ * Interface clock domain. The configuration setting becomes effective only if
+ * LOAD register bit is also set to 1.
+ */
+#define TW5864_SYSPLL_CFG BIT(2)
+/*
+ * Issue SPLL (108 MHz) configuration load from Async host interface / PCI
+ * Interface clock domain. The configuration setting becomes effective only if
+ * the LOAD register bit is also set to 1.
+ */
+#define TW5864_SPLL_CFG BIT(4)
+/*
+ * Set this bit to latch the SRST, SYSPLL_CFG, SPLL_CFG setting into the xtal
+ * clock domain to restart the PLL. This bit is self cleared.
+ */
+#define TW5864_LOAD BIT(3)
+
+/* SPLL_IREF, SPLL_LPX4, SPLL_CPX4, SPLL_PD, SPLL_DBG */
+#define TW5864_SPLL 0x8028
+
+/* 0x8800 ~ 0x88fc -- Interrupt Register Map */
+/*
+ * Trigger mode of interrupt source 0 ~ 15
+ * 1 Edge trigger mode
+ * 0 Level trigger mode
+ */
+#define TW5864_TRIGGER_MODE_L 0x8800
+/* Trigger mode of interrupt source 16 ~ 31 */
+#define TW5864_TRIGGER_MODE_H 0x8804
+/* Enable of interrupt source 0 ~ 15 */
+#define TW5864_INTR_ENABLE_L 0x8808
+/* Enable of interrupt source 16 ~ 31 */
+#define TW5864_INTR_ENABLE_H 0x880c
+/* Clear interrupt command of interrupt source 0 ~ 15 */
+#define TW5864_INTR_CLR_L 0x8810
+/* Clear interrupt command of interrupt source 16 ~ 31 */
+#define TW5864_INTR_CLR_H 0x8814
+/*
+ * Assertion of interrupt source 0 ~ 15
+ * 1 High level or pos-edge is assertion
+ * 0 Low level or neg-edge is assertion
+ */
+#define TW5864_INTR_ASSERT_L 0x8818
+/* Assertion of interrupt source 16 ~ 31 */
+#define TW5864_INTR_ASSERT_H 0x881c
+/*
+ * Output level of interrupt
+ * 1 Interrupt output is high assertion
+ * 0 Interrupt output is low assertion
+ */
+#define TW5864_INTR_OUT_LEVEL 0x8820
+/*
+ * Status of interrupt source 0 ~ 15
+ * Bit[0]: VLC 4k RAM interrupt
+ * Bit[1]: BURST DDR RAM interrupt
+ * Bit[2]: MV DSP interrupt
+ * Bit[3]: video lost interrupt
+ * Bit[4]: gpio 0 interrupt
+ * Bit[5]: gpio 1 interrupt
+ * Bit[6]: gpio 2 interrupt
+ * Bit[7]: gpio 3 interrupt
+ * Bit[8]: gpio 4 interrupt
+ * Bit[9]: gpio 5 interrupt
+ * Bit[10]: gpio 6 interrupt
+ * Bit[11]: gpio 7 interrupt
+ * Bit[12]: JPEG interrupt
+ * Bit[13:15]: Reserved
+ */
+#define TW5864_INTR_STATUS_L 0x8838
+/*
+ * Status of interrupt source 16 ~ 31
+ * Bit[0]: Reserved
+ * Bit[1]: VLC done interrupt
+ * Bit[2]: Reserved
+ * Bit[3]: AD Vsync interrupt
+ * Bit[4]: Preview eof interrupt
+ * Bit[5]: Preview overflow interrupt
+ * Bit[6]: Timer interrupt
+ * Bit[7]: Reserved
+ * Bit[8]: Audio eof interrupt
+ * Bit[9]: I2C done interrupt
+ * Bit[10]: AD interrupt
+ * Bit[11:15]: Reserved
+ */
+#define TW5864_INTR_STATUS_H 0x883c
+
+/* Defines of interrupt bits, united for both low and high word registers */
+#define TW5864_INTR_VLC_RAM BIT(0)
+#define TW5864_INTR_BURST BIT(1)
+#define TW5864_INTR_MV_DSP BIT(2)
+#define TW5864_INTR_VIN_LOST BIT(3)
+/* n belongs to [0; 7] */
+#define TW5864_INTR_GPIO(n) (1 << (4 + n))
+#define TW5864_INTR_JPEG BIT(12)
+#define TW5864_INTR_VLC_DONE BIT(17)
+#define TW5864_INTR_AD_VSYNC BIT(19)
+#define TW5864_INTR_PV_EOF BIT(20)
+#define TW5864_INTR_PV_OVERFLOW BIT(21)
+#define TW5864_INTR_TIMER BIT(22)
+#define TW5864_INTR_AUD_EOF BIT(24)
+#define TW5864_INTR_I2C_DONE BIT(25)
+#define TW5864_INTR_AD BIT(26)
+
+/* 0x9000 ~ 0x920c -- Video Capture (VIF) Register Map */
+/*
+ * H264EN_CH_STATUS[n] Status of Vsync synchronized H264EN_CH_EN (Read Only)
+ * 1 Channel Enabled
+ * 0 Channel Disabled
+ */
+#define TW5864_H264EN_CH_STATUS 0x9000
+/*
+ * [15:0] H264EN_CH_EN[n] H264 Encoding Path Enable for channel
+ * 1 Channel Enabled
+ * 0 Channel Disabled
+ */
+#define TW5864_H264EN_CH_EN 0x9004
+/*
+ * H264EN_CH_DNS[n] H264 Encoding Path Downscale Video Decoder Input for
+ * channel n
+ * 1 Downscale Y to 1/2
+ * 0 Does not downscale
+ */
+#define TW5864_H264EN_CH_DNS 0x9008
+/*
+ * H264EN_CH_PROG[n] H264 Encoding Path channel n is progressive
+ * 1 Progressive (Not valid for TW5864)
+ * 0 Interlaced (TW5864 default)
+ */
+#define TW5864_H264EN_CH_PROG 0x900c
+/*
+ * [3:0] H264EN_BUS_MAX_CH[n]
+ * H264 Encoding Path maximum number of channel on BUS n
+ * 0 Max 4 channels
+ * 1 Max 2 channels
+ */
+#define TW5864_H264EN_BUS_MAX_CH 0x9010
+
+/*
+ * H264EN_RATE_MAX_LINE_n H264 Encoding path Rate Mapping Maximum Line Number
+ * on Bus n
+ */
+#define TW5864_H264EN_RATE_MAX_LINE_EVEN 0x1f
+#define TW5864_H264EN_RATE_MAX_LINE_ODD_SHIFT 5
+#define TW5864_H264EN_RATE_MAX_LINE_ODD (0x1f << 5)
+/*
+ * [4:0] H264EN_RATE_MAX_LINE_0
+ * [9:5] H264EN_RATE_MAX_LINE_1
+ */
+#define TW5864_H264EN_RATE_MAX_LINE_REG1 0x9014
+/*
+ * [4:0] H264EN_RATE_MAX_LINE_2
+ * [9:5] H264EN_RATE_MAX_LINE_3
+ */
+#define TW5864_H264EN_RATE_MAX_LINE_REG2 0x9018
+
+/*
+ * H264EN_CHn_FMT H264 Encoding Path Format configuration of Channel n
+ * 00 D1 (For D1 and hD1 frame)
+ * 01 (Reserved)
+ * 10 (Reserved)
+ * 11 D1 with 1/2 size in X (for CIF frame)
+ * Note: To be used with 0x9008 register to configure the frame size
+ */
+/*
+ * [1:0]: H264EN_CH0_FMT,
+ * ..., [15:14]: H264EN_CH7_FMT
+ */
+#define TW5864_H264EN_CH_FMT_REG1 0x9020
+/*
+ * [1:0]: H264EN_CH8_FMT (?),
+ * ..., [15:14]: H264EN_CH15_FMT (?)
+ */
+#define TW5864_H264EN_CH_FMT_REG2 0x9024
+
+/*
+ * H264EN_RATE_CNTL_BUSm_CHn H264 Encoding Path BUS m Rate Control for Channel n
+ */
+#define TW5864_H264EN_RATE_CNTL_LO_WORD(bus, channel) \
+	(0x9100 + bus * 0x20 + channel * 0x08)
+#define TW5864_H264EN_RATE_CNTL_HI_WORD(bus, channel) \
+	(0x9104 + bus * 0x20 + channel * 0x08)
+
+/*
+ * H264EN_BUSm_MAP_CHn The 16-to-1 MUX configuration register for each encoding
+ * channel (total of 16 channels). Four bits for each channel.
+ */
+#define TW5864_H264EN_BUS0_MAP 0x9200
+#define TW5864_H264EN_BUS1_MAP 0x9204
+#define TW5864_H264EN_BUS2_MAP 0x9208
+#define TW5864_H264EN_BUS3_MAP 0x920c
+
+/* This register is not defined in datasheet, but used in reference driver */
+#define TW5864_UNDECLARED_ERROR_FLAGS_0x9218 0x9218
+
+#define TW5864_GPIO1 0x9800
+#define TW5864_GPIO2 0x9804
+/* Define controls in registers TW5864_GPIO1, TW5864_GPIO2 */
+/* GPIO DATA of Group n */
+#define TW5864_GPIO_DATA 0x00ff
+#define TW5864_GPIO_OEN_SHIFT 8
+/* GPIO Output Enable of Group n */
+#define TW5864_GPIO_OEN (0xff << 8)
+
+/* 0xa000 ~ 0xa8ff – DDR Controller Register Map */
+/* DDR Controller A */
+/*
+ * [2:0] Data valid counter after read command to DDR. This is the delay value
+ * to show how many cycles the data will be back from DDR after we issue a read
+ * command.
+ */
+#define TW5864_RD_ACK_VLD_MUX 0xa000
+
+#define TW5864_DDR_PERIODS 0xa004
+/* Define controls in register TW5864_DDR_PERIODS */
+/*
+ * Tras value, the minimum cycle of active to precharge command period,
+ * default is 7
+ */
+#define TW5864_TRAS_CNT_MAX 0x000f
+/*
+ * Trfc value, the minimum cycle of refresh to active or refresh command period,
+ * default is 4"hf
+ */
+#define TW5864_RFC_CNT_MAX_SHIFT 8
+#define TW5864_RFC_CNT_MAX (0x0f << 8)
+/*
+ * Trcd value, the minimum cycle of active to internal read/write command
+ * period, default is 4"h2
+ */
+#define TW5864_TCD_CNT_MAX_SHIFT 4
+#define TW5864_TCD_CNT_MAX (0x0f << 4)
+/* Twr value, write recovery time, default is 4"h3 */
+#define TW5864_TWR_CNT_MAX_SHIFT 12
+#define TW5864_TWR_CNT_MAX (0x0f << 12)
+
+/*
+ * [2:0] CAS latency, the delay cycle between internal read command and the
+ * availability of the first bit of output data, default is 3
+ */
+#define TW5864_CAS_LATENCY 0xa008
+/*
+ * [15:0] Maximum average periodic refresh, the value is based on the current
+ * frequency to match 7.8mcs
+ */
+#define TW5864_DDR_REF_CNTR_MAX 0xa00c
+/*
+ * DDR_ON_CHIP_MAP [1:0]
+ * 0 256M DDR on board
+ * 1 512M DDR on board
+ * 2 1G DDR on board
+ * DDR_ON_CHIP_MAP [2]
+ * 0 Only one DDR chip
+ * 1 Two DDR chips
+ */
+#define TW5864_DDR_ON_CHIP_MAP 0xa01c
+#define TW5864_DDR_SELFTEST_MODE 0xa020
+/* Define controls in register TW5864_DDR_SELFTEST_MODE */
+/*
+ * 0 Common read/write mode
+ * 1 DDR self-test mode
+ */
+#define TW5864_MASTER_MODE BIT(0)
+/*
+ * 0 DDR self-test single read/write
+ * 1 DDR self-test burst read/write
+ */
+#define TW5864_SINGLE_PROC BIT(1)
+/*
+ * 0 DDR self-test write command
+ * 1 DDR self-test read command
+ */
+#define TW5864_WRITE_FLAG BIT(2)
+#define TW5864_DATA_MODE_SHIFT 4
+/*
+ * 0 write 32'haaaa5555 to DDR
+ * 1 write 32'hffffffff to DDR
+ * 2 write 32'hha5a55a5a to DDR
+ * 3 write increasing data to DDR
+ */
+#define TW5864_DATA_MODE (0x3 << 4)
+
+/* [7:0] The maximum data of one burst in DDR self-test mode */
+#define TW5864_BURST_CNTR_MAX 0xa024
+/* [15:0] The maximum burst counter (bit 15~0) in DDR self-test mode */
+#define TW5864_DDR_PROC_CNTR_MAX_L 0xa028
+/* The maximum burst counter (bit 31~16) in DDR self-test mode */
+#define TW5864_DDR_PROC_CNTR_MAX_H 0xa02c
+/* [0]: Start one DDR self-test */
+#define TW5864_DDR_SELF_TEST_CMD 0xa030
+/* The maximum error counter (bit 15 ~ 0) in DDR self-test */
+#define TW5864_ERR_CNTR_L 0xa034
+
+#define TW5864_ERR_CNTR_H_AND_FLAG 0xa038
+/* Define controls in register TW5864_ERR_CNTR_H_AND_FLAG */
+/* The maximum error counter (bit 30 ~ 16) in DDR self-test */
+#define TW5864_ERR_CNTR_H_MASK 0x3fff
+/* DDR self-test end flag */
+#define TW5864_END_FLAG 0x8000
+
+/*
+ * DDR Controller B: same as 0xa000 ~ 0xa038, but add TW5864_DDR_B_OFFSET to all
+ * addresses
+ */
+#define TW5864_DDR_B_OFFSET 0x0800
+
+/* 0xb004 ~ 0xb018 – HW version/ARB12 Register Map */
+/* [15:0] Default is C013 */
+#define TW5864_HW_VERSION 0xb004
+
+#define TW5864_REQS_ENABLE 0xb010
+/* Define controls in register TW5864_REQS_ENABLE */
+/* Audio data in to DDR enable (default 1) */
+#define TW5864_AUD_DATA_IN_ENB BIT(0)
+/* Audio encode request to DDR enable (default 1) */
+#define TW5864_AUD_ENC_REQ_ENB BIT(1)
+/* Audio decode request0 to DDR enable (default 1) */
+#define TW5864_AUD_DEC_REQ0_ENB BIT(2)
+/* Audio decode request1 to DDR enable (default 1) */
+#define TW5864_AUD_DEC_REQ1_ENB BIT(3)
+/* VLC stream request to DDR enable (default 1) */
+#define TW5864_VLC_STRM_REQ_ENB BIT(4)
+/* H264 MV request to DDR enable (default 1) */
+#define TW5864_DVM_MV_REQ_ENB BIT(5)
+/* mux_core MVD request to DDR enable (default 1) */
+#define TW5864_MVD_REQ_ENB BIT(6)
+/* mux_core MVD temp data request to DDR enable (default 1) */
+#define TW5864_MVD_TMP_REQ_ENB BIT(7)
+/* JPEG request to DDR enable (default 1) */
+#define TW5864_JPEG_REQ_ENB BIT(8)
+/* mv_flag request to DDR enable (default 1) */
+#define TW5864_MV_FLAG_REQ_ENB BIT(9)
+
+#define TW5864_ARB12 0xb018
+/* Define controls in register TW5864_ARB12 */
+/* ARB12 Enable (default 1) */
+#define TW5864_ARB12_ENB BIT(15)
+/* ARB12 maximum value of time out counter (default 15"h1FF) */
+#define TW5864_ARB12_TIME_OUT_CNT 0x7fff
+
+/* 0xb800 ~ 0xb80c -- Indirect Access Register Map */
+/*
+ * Spec says:
+ * In order to access the indirect register space, the following procedure is
+ * followed.
+ * But reference driver implementation, and current driver, too, does it
+ * differently.
+ *
+ * Write Registers:
+ * (1) Write IND_DATA at 0xb804 ~ 0xb807
+ * (2) Read BUSY flag from 0xb803. Wait until BUSY signal is 0.
+ * (3) Write IND_ADDR at 0xb800 ~ 0xb801. Set R/W to "1", ENABLE to "1"
+ * Read Registers:
+ * (1) Read BUSY flag from 0xb803. Wait until BUSY signal is 0.
+ * (2) Write IND_ADDR at 0xb800 ~ 0xb801. Set R/W to "0", ENABLE to "1"
+ * (3) Read BUSY flag from 0xb803. Wait until BUSY signal is 0.
+ * (4) Read IND_DATA from 0xb804 ~ 0xb807
+ */
+#define TW5864_IND_CTL 0xb800
+/* Define controls in register TW5864_IND_CTL */
+/* Address used to access indirect register space */
+#define TW5864_IND_ADDR 0x0000ffff
+/* Wait until this bit is "0" before using indirect access */
+#define TW5864_BUSY BIT(31)
+/* Activate the indirect access. This bit is self cleared */
+#define TW5864_ENABLE BIT(25)
+/* Read/Write command */
+#define TW5864_RW BIT(24)
+
+/* [31:0] Data used to read/write indirect register space */
+#define TW5864_IND_DATA 0xb804
+
+/* 0xc000 ~ 0xc7fc -- Preview Register Map */
+/* Mostly skipped this section. */
+/*
+ * [15:0] Status of Vsync Synchronized PCI_PV_CH_EN (Read Only)
+ * 1 Channel Enabled
+ * 0 Channel Disabled
+ */
+#define TW5864_PCI_PV_CH_STATUS 0xc000
+/*
+ * [15:0] PCI Preview Path Enable for channel n
+ * 1 Channel Enable
+ * 0 Channel Disable
+ */
+#define TW5864_PCI_PV_CH_EN 0xc004
+
+/* 0xc800 ~ 0xc804 -- JPEG Capture Register Map */
+/* Skipped. */
+/* 0xd000 ~ 0xd0fc -- JPEG Control Register Map */
+/* Skipped. */
+
+/* 0xe000 ~ 0xfc04 – Motion Vector Register Map */
+
+/* ME Motion Vector data (Four Byte Each) 0xe000 ~ 0xe7fc */
+#define TW5864_ME_MV_VEC_START 0xe000
+#define TW5864_ME_MV_VEC_MAX_OFFSET 0x1ff
+#define TW5864_ME_MV_VEC(offset) (TW5864_ME_MV_VEC_START + 4 * offset)
+
+#define TW5864_MV 0xfc00
+/* Define controls in register TW5864_MV */
+/* mv bank0 full status , write "1" to clear */
+#define TW5864_MV_BK0_FULL BIT(0)
+/* mv bank1 full status , write "1" to clear */
+#define TW5864_MV_BK1_FULL BIT(1)
+/* slice end status; write "1" to clear */
+#define TW5864_MV_EOF BIT(2)
+/* mv encode interrupt status; write "1" to clear */
+#define TW5864_MV_DSP_INTR BIT(3)
+/* mv write memory overflow, write "1" to clear */
+#define TW5864_DSP_WR_OF BIT(4)
+#define TW5864_MV_LEN_SHIFT 5
+/* mv stream length */
+#define TW5864_MV_LEN (0xff << 5)
+/* The configured status bit written into bit 15 of 0xfc04 */
+#define TW5864_MPI_DDR_SEL BIT(13)
+
+#define TW5864_MPI_DDR_SEL_REG 0xfc04
+/* Define controls in register TW5864_MPI_DDR_SEL_REG */
+/*
+ * SW configure register
+ * 0 MV is saved in internal DPR
+ * 1 MV is saved in DDR
+ */
+#define TW5864_MPI_DDR_SEL2 BIT(15)
+
+/* 0x18000 ~ 0x181fc – PCI Master/Slave Control Map */
+#define TW5864_PCI_INTR_STATUS 0x18000
+/* Define controls in register TW5864_PCI_INTR_STATUS */
+/* vlc done */
+#define TW5864_VLC_DONE_INTR BIT(1)
+/* ad vsync */
+#define TW5864_AD_VSYNC_INTR BIT(3)
+/* preview eof */
+#define TW5864_PREV_EOF_INTR BIT(4)
+/* preview overflow interrupt */
+#define TW5864_PREV_OVERFLOW_INTR BIT(5)
+/* timer interrupt */
+#define TW5864_TIMER_INTR BIT(6)
+/* audio eof */
+#define TW5864_AUDIO_EOF_INTR BIT(8)
+/* IIC done */
+#define TW5864_IIC_DONE_INTR BIT(24)
+/* ad interrupt (e.g.: video lost, video format changed) */
+#define TW5864_AD_INTR_REG BIT(25)
+
+#define TW5864_PCI_INTR_CTL 0x18004
+/* Define controls in register TW5864_PCI_INTR_CTL */
+/* master enable */
+#define TW5864_PCI_MAST_ENB BIT(0)
+/* mvd&vlc master enable */
+#define TW5864_MVD_VLC_MAST_ENB 0x06
+/* (Need to set 0 in TW5864A) */
+#define TW5864_AD_MAST_ENB BIT(3)
+/* preview master enable */
+#define TW5864_PREV_MAST_ENB BIT(4)
+/* preview overflow enable */
+#define TW5864_PREV_OVERFLOW_ENB BIT(5)
+/* timer interrupt enable */
+#define TW5864_TIMER_INTR_ENB BIT(6)
+/* JPEG master (push mode) enable */
+#define TW5864_JPEG_MAST_ENB BIT(7)
+#define TW5864_AU_MAST_ENB_CHN_SHIFT 8
+/* audio master channel enable */
+#define TW5864_AU_MAST_ENB_CHN (0xffff << 8)
+/* IIC interrupt enable */
+#define TW5864_IIC_INTR_ENB BIT(24)
+/* ad interrupt enable */
+#define TW5864_AD_INTR_ENB BIT(25)
+/* target burst enable */
+#define TW5864_PCI_TAR_BURST_ENB BIT(26)
+/* vlc stream burst enable */
+#define TW5864_PCI_VLC_BURST_ENB BIT(27)
+/* ddr burst enable (1 enable, and must set DDR_BRST_EN) */
+#define TW5864_PCI_DDR_BURST_ENB BIT(28)
+
+/*
+ * Because preview and audio have 16 channels separately, so using this
+ * registers to indicate interrupt status for every channels. This is secondary
+ * interrupt status register. OR operating of the PREV_INTR_REG is
+ * PREV_EOF_INTR, OR operating of the AU_INTR_REG bits is AUDIO_EOF_INTR
+ */
+#define TW5864_PREV_AND_AU_INTR 0x18008
+/* Define controls in register TW5864_PREV_AND_AU_INTR */
+/* preview eof interrupt flag */
+#define TW5864_PREV_INTR_REG 0x0000ffff
+#define TW5864_AU_INTR_REG_SHIFT 16
+/* audio eof interrupt flag */
+#define TW5864_AU_INTR_REG (0xffff << 16)
+
+#define TW5864_MASTER_ENB_REG 0x1800c
+/* Define controls in register TW5864_MASTER_ENB_REG */
+/* master enable */
+#define TW5864_PCI_VLC_INTR_ENB BIT(1)
+/* mvd and vlc master enable */
+#define TW5864_PCI_PREV_INTR_ENB BIT(4)
+/* ad vsync master enable */
+#define TW5864_PCI_PREV_OF_INTR_ENB BIT(5)
+/* jpeg master enable */
+#define TW5864_PCI_JPEG_INTR_ENB BIT(7)
+/* preview master enable */
+#define TW5864_PCI_AUD_INTR_ENB BIT(8)
+
+/*
+ * Every channel of preview and audio have ping-pong buffers in system memory,
+ * this register is the buffer flag to notify software which buffer is been
+ * operated.
+ */
+#define TW5864_PREV_AND_AU_BUF_FLAG 0x18010
+/* Define controls in register TW5864_PREV_AND_AU_BUF_FLAG */
+/* preview buffer A/B flag */
+#define TW5864_PREV_BUF_FLAG 0xffff
+#define TW5864_AUDIO_BUF_FLAG_SHIFT 16
+/* audio buffer A/B flag */
+#define TW5864_AUDIO_BUF_FLAG (0xffff << 16)
+
+#define TW5864_IIC 0x18014
+/* Define controls in register TW5864_IIC */
+/* register data */
+#define TW5864_IIC_DATA 0x00ff
+#define TW5864_IIC_REG_ADDR_SHIFT 8
+/* register addr */
+#define TW5864_IIC_REG_ADDR (0xff << 8)
+/* rd/wr flag rd=1,wr=0 */
+#define TW5864_IIC_RW BIT(16)
+#define TW5864_IIC_DEV_ADDR_SHIFT 17
+/* device addr */
+#define TW5864_IIC_DEV_ADDR (0x7f << 17)
+/*
+ * iic done, software kick off one time iic transaction through setting this
+ * bit to 1. Then poll this bit, value 1 indicate iic transaction have
+ * completed, if read, valid data have been stored in iic_data
+ */
+#define TW5864_IIC_DONE BIT(24)
+
+#define TW5864_RST_AND_IF_INFO 0x18018
+/* Define controls in register TW5864_RST_AND_IF_INFO */
+/* application software soft reset */
+#define TW5864_APP_SOFT_RST BIT(0)
+#define TW5864_PCI_INF_VERSION_SHIFT 16
+/* PCI interface version, read only */
+#define TW5864_PCI_INF_VERSION (0xffff << 16)
+
+/* vlc stream crc value, it is calculated in pci module */
+#define TW5864_VLC_CRC_REG 0x1801c
+/*
+ * vlc max length, it is defined by software based on software assign memory
+ * space for vlc
+ */
+#define TW5864_VLC_MAX_LENGTH 0x18020
+/* vlc length of one frame */
+#define TW5864_VLC_LENGTH 0x18024
+/* vlc original crc value */
+#define TW5864_VLC_INTRA_CRC_I_REG 0x18028
+/* vlc original crc value */
+#define TW5864_VLC_INTRA_CRC_O_REG 0x1802c
+/* mv stream crc value, it is calculated in pci module */
+#define TW5864_VLC_PAR_CRC_REG 0x18030
+/* mv length */
+#define TW5864_VLC_PAR_LENGTH_REG 0x18034
+/* mv original crc value */
+#define TW5864_VLC_PAR_I_REG 0x18038
+/* mv original crc value */
+#define TW5864_VLC_PAR_O_REG 0x1803c
+
+/*
+ * Configuration register for 9[or 10] CIFs or 1D1+15QCIF Preview mode.
+ * PREV_PCI_ENB_CHN[0] Enable 9th preview channel (9CIF prev) or 1D1 channel in
+ * (1D1+15QCIF prev)
+ * PREV_PCI_ENB_CHN[1] Enable 10th preview channel
+ */
+#define TW5864_PREV_PCI_ENB_CHN 0x18040
+/* Description skipped. */
+#define TW5864_PREV_FRAME_FORMAT_IN 0x18044
+/* IIC enable */
+#define TW5864_IIC_ENB 0x18048
+/*
+ * Timer interrupt interval
+ * 0 1ms
+ * 1 2ms
+ * 2 4ms
+ * 3 8ms
+ */
+#define TW5864_PCI_INTTM_SCALE 0x1804c
+
+/*
+ * The above register is pci base address registers. Application software will
+ * initialize them to tell chip where the corresponding stream will be dumped
+ * to. Application software will select appropriate base address interval based
+ * on the stream length.
+ */
+/* VLC stream base address */
+#define TW5864_VLC_STREAM_BASE_ADDR 0x18080
+/* MV stream base address */
+#define TW5864_MV_STREAM_BASE_ADDR 0x18084
+/* 0x180a0 – 0x180bc: audio burst base address. Skipped. */
+/* 0x180c0 ~ 0x180dc – JPEG Push Mode Buffer Base Address. Skipped. */
+/* 0x18100 – 0x1817c: preview burst base address. Skipped. */
+
+/* 0x80000 ~ 0x87fff -- DDR Burst RW Register Map */
+#define TW5864_DDR_CTL 0x80000
+/* Define controls in register TW5864_DDR_CTL */
+#define TW5864_BRST_LENGTH_SHIFT 2
+/* Length of 32-bit data burst */
+#define TW5864_BRST_LENGTH (0x3fff << 2)
+/*
+ * Burst Read/Write
+ * 0 Read Burst from DDR
+ * 1 Write Burst to DDR
+ */
+#define TW5864_BRST_RW BIT(16)
+/* Begin a new DDR Burst. This bit is self cleared */
+#define TW5864_NEW_BRST_CMD BIT(17)
+/* DDR Burst End Flag */
+#define TW5864_BRST_END BIT(24)
+/* Enable Error Interrupt for Single DDR Access */
+#define TW5864_SING_ERR_INTR BIT(25)
+/* Enable Error Interrupt for Burst DDR Access */
+#define TW5864_BRST_ERR_INTR BIT(26)
+/* Enable Interrupt for End of DDR Burst Access */
+#define TW5864_BRST_END_INTR BIT(27)
+/* DDR Single Access Error Flag */
+#define TW5864_SINGLE_ERR BIT(28)
+/* DDR Single Access Busy Flag */
+#define TW5864_SINGLE_BUSY BIT(29)
+/* DDR Burst Access Error Flag */
+#define TW5864_BRST_ERR BIT(30)
+/* DDR Burst Access Busy Flag */
+#define TW5864_BRST_BUSY BIT(31)
+
+/* [27:0] DDR Access Address. Bit [1:0] has to be 0 */
+#define TW5864_DDR_ADDR 0x80004
+/* DDR Access Internal Buffer Address. Bit [1:0] has to be 0 */
+#define TW5864_DPR_BUF_ADDR 0x80008
+/* SRAM Buffer MPI Access Space. Totally 16 KB */
+#define TW5864_DPR_BUF_START 0x84000
+/* 0x84000 - 0x87ffc */
+#define TW5864_DPR_BUF_SIZE 0x4000
+
+/* Indirect Map Space */
+/*
+ * The indirect space is accessed through 0xb800 ~ 0xb807 registers in direct
+ * access space
+ */
+/* Analog Video / Audio Decoder / Encoder */
+/* Allowed channel values: [0; 3] */
+/* Read-only register */
+#define TW5864_INDIR_VIN_0(channel) (0x000 + channel * 0x010)
+/* Define controls in register TW5864_INDIR_VIN_0 */
+/*
+ * 1 Video not present. (sync is not detected in number of consecutive line
+ * periods specified by MISSCNT register)
+ * 0 Video detected.
+ */
+#define TW5864_INDIR_VIN_0_VDLOSS BIT(7)
+/*
+ * 1 Horizontal sync PLL is locked to the incoming video source.
+ * 0 Horizontal sync PLL is not locked.
+ */
+#define TW5864_INDIR_VIN_0_HLOCK BIT(6)
+/*
+ * 1 Sub-carrier PLL is locked to the incoming video source.
+ * 0 Sub-carrier PLL is not locked.
+ */
+#define TW5864_INDIR_VIN_0_SLOCK BIT(5)
+/*
+ * 1 Even field is being decoded.
+ * 0 Odd field is being decoded.
+ */
+#define TW5864_INDIR_VIN_0_FLD BIT(4)
+/*
+ * 1 Vertical logic is locked to the incoming video source.
+ * 0 Vertical logic is not locked.
+ */
+#define TW5864_INDIR_VIN_0_VLOCK BIT(3)
+/*
+ * 1 No color burst signal detected.
+ * 0 Color burst signal detected.
+ */
+#define TW5864_INDIR_VIN_0_MONO BIT(1)
+/*
+ * 0 60Hz source detected
+ * 1 50Hz source detected
+ * The actual vertical scanning frequency depends on the current standard
+ * invoked.
+ */
+#define TW5864_INDIR_VIN_0_DET50 BIT(0)
+
+#define TW5864_INDIR_VIN_1(channel) (0x001 + channel * 0x010)
+/* VCR signal indicator. Read-only. */
+#define TW5864_INDIR_VIN_1_VCR BIT(7)
+/* Weak signal indicator 2. Read-only. */
+#define TW5864_INDIR_VIN_1_WKAIR BIT(6)
+/* Weak signal indicator controlled by WKTH. Read-only. */
+#define TW5864_INDIR_VIN_1_WKAIR1 BIT(5)
+/*
+ * 1 = Standard signal
+ * 0 = Non-standard signal
+ * Read-only
+ */
+#define TW5864_INDIR_VIN_1_VSTD BIT(4)
+/*
+ * 1 = Non-interlaced signal
+ * 0 = interlaced signal
+ * Read-only
+ */
+#define TW5864_INDIR_VIN_1_NINTL BIT(3)
+/*
+ * Vertical Sharpness Control. Writable.
+ * 0 = None (default)
+ * 7 = Highest
+ * **Note: VSHP must be set to ‘0’ if COMB = 0
+ */
+#define TW5864_INDIR_VIN_1_VSHP 0x07
+
+/* HDELAY_XY[7:0] */
+#define TW5864_INDIR_VIN_2_HDELAY_XY_LO(channel) (0x002 + channel * 0x010)
+/* HACTIVE_XY[7:0] */
+#define TW5864_INDIR_VIN_3_HACTIVE_XY_LO(channel) (0x003 + channel * 0x010)
+/* VDELAY_XY[7:0] */
+#define TW5864_INDIR_VIN_4_VDELAY_XY_LO(channel) (0x004 + channel * 0x010)
+/* VACTIVE_XY[7:0] */
+#define TW5864_INDIR_VIN_5_VACTIVE_XY_LO(channel) (0x005 + channel * 0x010)
+
+#define TW5864_INDIR_VIN_6(channel) (0x006 + channel * 0x010)
+/* Define controls in register TW5864_INDIR_VIN_6 */
+#define TW5864_INDIR_VIN_6_HDELAY_XY_HI 0x03
+#define TW5864_INDIR_VIN_6_HACTIVE_XY_HI_SHIFT 2
+#define TW5864_INDIR_VIN_6_HACTIVE_XY_HI (0x03 << 2)
+#define TW5864_INDIR_VIN_6_VDELAY_XY_HI BIT(4)
+#define TW5864_INDIR_VIN_6_VACTIVE_XY_HI BIT(5)
+
+/*
+ * HDELAY_XY This 10bit register defines the starting location of horizontal
+ * active pixel for display / record path. A unit is 1 pixel. The default value
+ * is 0x00f for NTSC and 0x00a for PAL.
+ *
+ * HACTIVE_XY This 10bit register defines the number of horizontal active pixel
+ * for display / record path. A unit is 1 pixel. The default value is decimal
+ * 720.
+ *
+ * VDELAY_XY This 9bit register defines the starting location of vertical
+ * active for display / record path. A unit is 1 line. The default value is
+ * decimal 6.
+ *
+ * VACTIVE_XY This 9bit register defines the number of vertical active lines
+ * for display / record path. A unit is 1 line. The default value is decimal
+ * 240.
+ */
+
+/* HUE These bits control the color hue as 2's complement number. They have
+ * value from +36o (7Fh) to -36o (80h) with an increment of 2.8o. The 2 LSB has
+ * no effect. The positive value gives greenish tone and negative value gives
+ * purplish tone. The default value is 0o (00h). This is effective only on NTSC
+ * system. The default is 00h.
+ */
+#define TW5864_INDIR_VIN_7_HUE(channel) (0x007 + channel * 0x010)
+
+#define TW5864_INDIR_VIN_8(channel) (0x008 + channel * 0x010)
+/* Define controls in register TW5864_INDIR_VIN_8 */
+/*
+ * This bit controls the center frequency of the peaking filter.
+ * The corresponding gain adjustment is HFLT.
+ * 0 Low
+ * 1 center
+ */
+#define TW5864_INDIR_VIN_8_SCURVE BIT(7)
+/* CTI level selection. The default is 1.
+ * 0 None
+ * 3 Highest
+ */
+#define TW5864_INDIR_VIN_8_CTI_SHIFT 4
+#define TW5864_INDIR_VIN_8_CTI (0x03 << 4)
+
+/*
+ * These bits control the amount of sharpness enhancement on the luminance
+ * signals. There are 16 levels of control with "0" having no effect on the
+ * output image. 1 through 15 provides sharpness enhancement with "F" being the
+ * strongest. The default is 1.
+ */
+#define TW5864_INDIR_VIN_8_SHARPNESS 0x0f
+
+/*
+ * These bits control the luminance contrast gain. A value of 100 (64h) has a
+ * gain of 1. The range adjustment is from 0% to 255% at 1% per step. The
+ * default is 64h.
+ */
+#define TW5864_INDIR_VIN_9_CNTRST(channel) (0x009 + channel * 0x010)
+
+/*
+ * These bits control the brightness. They have value of –128 to 127 in 2's
+ * complement form. Positive value increases brightness. A value 0 has no
+ * effect on the data. The default is 00h.
+ */
+#define TW5864_INDIR_VIN_A_BRIGHT(channel) (0x00a + channel * 0x010)
+
+/*
+ * These bits control the digital gain adjustment to the U (or Cb) component of
+ * the digital video signal. The color saturation can be adjusted by adjusting
+ * the U and V color gain components by the same amount in the normal
+ * situation. The U and V can also be adjusted independently to provide greater
+ * flexibility. The range of adjustment is 0 to 200%. A value of 128 (80h) has
+ * gain of 100%. The default is 80h.
+ */
+#define TW5864_INDIR_VIN_B_SAT_U(channel) (0x00b + channel * 0x010)
+
+/*
+ * These bits control the digital gain adjustment to the V (or Cr) component of
+ * the digital video signal. The color saturation can be adjusted by adjusting
+ * the U and V color gain components by the same amount in the normal
+ * situation. The U and V can also be adjusted independently to provide greater
+ * flexibility. The range of adjustment is 0 to 200%. A value of 128 (80h) has
+ * gain of 100%. The default is 80h.
+ */
+#define TW5864_INDIR_VIN_C_SAT_V(channel) (0x00c + channel * 0x010)
+
+/* Read-only */
+#define TW5864_INDIR_VIN_D(channel) (0x00d + channel * 0x010)
+/* Define controls in register TW5864_INDIR_VIN_D */
+/* Macrovision color stripe detection may be un-reliable */
+#define TW5864_INDIR_VIN_D_CSBAD BIT(3)
+/* Macrovision AGC pulse detected */
+#define TW5864_INDIR_VIN_D_MCVSN BIT(2)
+/* Macrovision color stripe protection burst detected */
+#define TW5864_INDIR_VIN_D_CSTRIPE BIT(1)
+/*
+ * This bit is valid only when color stripe protection is detected, i.e. if
+ * CSTRIPE=1,
+ * 1 Type 2 color stripe protection
+ * 0 Type 3 color stripe protection
+ */
+#define TW5864_INDIR_VIN_D_CTYPE2 BIT(0)
+
+/* Read-only */
+#define TW5864_INDIR_VIN_E(channel) (0x00e + channel * 0x010)
+/* Define controls in register TW5864_INDIR_VIN_E */
+/*
+ * Read-only.
+ * 0 Idle
+ * 1 Detection in progress
+ */
+#define TW5864_INDIR_VIN_E_DETSTUS BIT(7)
+/*
+ * STDNOW Current standard invoked
+ * 0 NTSC (M)
+ * 1 PAL (B, D, G, H, I)
+ * 2 SECAM
+ * 3 NTSC4.43
+ * 4 PAL (M)
+ * 5 PAL (CN)
+ * 6 PAL 60
+ * 7 Not valid
+ */
+#define TW5864_INDIR_VIN_E_STDNOW_SHIFT 4
+#define TW5864_INDIR_VIN_E_STDNOW (0x07 << 4)
+
+/*
+ * 1 Disable the shadow registers
+ * 0 Enable VACTIVE and HDELAY shadow registers value depending on STANDARD.
+ * (Default)
+ */
+#define TW5864_INDIR_VIN_E_ATREG BIT(3)
+/*
+ * STANDARD Standard selection
+ * 0 NTSC (M)
+ * 1 PAL (B, D, G, H, I)
+ * 2 SECAM
+ * 3 NTSC4.43
+ * 4 PAL (M)
+ * 5 PAL (CN)
+ * 6 PAL 60
+ * 7 Auto detection (Default)
+ */
+#define TW5864_INDIR_VIN_E_STANDARD 0x07
+
+#define TW5864_INDIR_VIN_F(channel) (0x00f + channel * 0x010)
+/* Define controls in register TW5864_INDIR_VIN_F */
+/*
+ * 1 Writing 1 to this bit will manually initiate the auto format detection
+ * process. This bit is a self-clearing bit
+ * 0 Manual initiation of auto format detection is done. (Default)
+ */
+#define TW5864_INDIR_VIN_F_ATSTART BIT(7)
+/* Enable recognition of PAL60 (Default) */
+#define TW5864_INDIR_VIN_F_PAL60EN BIT(6)
+/* Enable recognition of PAL (CN). (Default) */
+#define TW5864_INDIR_VIN_F_PALCNEN BIT(5)
+/* Enable recognition of PAL (M). (Default) */
+#define TW5864_INDIR_VIN_F_PALMEN BIT(4)
+/* Enable recognition of NTSC 4.43. (Default) */
+#define TW5864_INDIR_VIN_F_NTSC44EN BIT(3)
+/* Enable recognition of SECAM. (Default) */
+#define TW5864_INDIR_VIN_F_SECAMEN BIT(2)
+/* Enable recognition of PAL (B, D, G, H, I). (Default) */
+#define TW5864_INDIR_VIN_F_PALBEN BIT(1)
+/* Enable recognition of NTSC (M). (Default) */
+#define TW5864_INDIR_VIN_F_NTSCEN BIT(0)
+
+/* Some registers skipped. */
+
+/* Use falling edge to sample VD1-VD4 from 54 MHz to 108 MHz */
+#define TW5864_INDIR_VD_108_POL 0x041
+#define TW5864_INDIR_VD_108_POL_VD12 BIT(0)
+#define TW5864_INDIR_VD_108_POL_VD34 BIT(1)
+#define TW5864_INDIR_VD_108_POL_BOTH \
+	(TW5864_INDIR_VD_108_POL_VD12 | TW5864_INDIR_VD_108_POL_VD34)
+
+/* Some registers skipped. */
+
+/*
+ * Audio Input ADC gain control
+ * 0 0.25
+ * 1 0.31
+ * 2 0.38
+ * 3 0.44
+ * 4 0.50
+ * 5 0.63
+ * 6 0.75
+ * 7 0.88
+ * 8 1.00 (default)
+ * 9 1.25
+ * 10 1.50
+ * 11 1.75
+ * 12 2.00
+ * 13 2.25
+ * 14 2.50
+ * 15 2.75
+ */
+/* [3:0] channel 0, [7:4] channel 1 */
+#define TW5864_INDIR_AIGAIN1 0x060
+/* [3:0] channel 2, [7:4] channel 3 */
+#define TW5864_INDIR_AIGAIN2 0x061
+
+/* Some registers skipped */
+
+#define TW5864_INDIR_AIN_0x06D 0x06d
+/* Define controls in register TW5864_INDIR_AIN_0x06D */
+/*
+ * LAWMD Select u-Law/A-Law/PCM/SB data output format on ADATR and ADATM pin.
+ * 0 PCM output (default)
+ * 1 SB (Signed MSB bit in PCM data is inverted) output
+ * 2 u-Law output
+ * 3 A-Law output
+ */
+#define TW5864_INDIR_AIN_LAWMD_SHIFT 6
+#define TW5864_INDIR_AIN_LAWMD (0x03 << 6)
+/*
+ * Disable the mixing ratio value for all audio.
+ * 0 Apply individual mixing ratio value for each audio (default)
+ * 1 Apply nominal value for all audio commonly
+ */
+#define TW5864_INDIR_AIN_MIX_DERATIO BIT(5)
+/*
+ * Enable the mute function for audio channel AINn when n is 0 to 3. It effects
+ * only for mixing. When n = 4, it enable the mute function of the playback
+ * audio input. It effects only for single chip or the last stage chip
+ * 0 Normal
+ * 1 Muted (default)
+ */
+#define TW5864_INDIR_AIN_MIX_MUTE 0x1f
+
+/* Some registers skipped */
+
+#define TW5864_INDIR_AIN_0x0E3 0x0e3
+/* Define controls in register TW5864_INDIR_AIN_0x0E3 */
+/*
+ * ADATP signal is coming from external ADPCM decoder, instead of on-chip ADPCM
+ * decoder
+ */
+#define TW5864_INDIR_AIN_0x0E3_EXT_ADATP BIT(7)
+/* ACLKP output signal polarity inverse */
+#define TW5864_INDIR_AIN_0x0E3_ACLKPPOLO BIT(6)
+/*
+ * ACLKR input signal polarity inverse.
+ * 0 Not inversed (Default)
+ * 1 Inversed
+ */
+#define TW5864_INDIR_AIN_0x0E3_ACLKRPOL BIT(5)
+/*
+ * ACLKP input signal polarity inverse.
+ * 0 Not inversed (Default)
+ * 1 Inversed
+ */
+#define TW5864_INDIR_AIN_0x0E3_ACLKPPOLI BIT(4)
+/*
+ * ACKI [21:0] control automatic set up with AFMD registers
+ * This mode is only effective when ACLKRMASTER=1
+ * 0 ACKI [21:0] registers set up ACKI control
+ * 1 ACKI control is automatically set up by AFMD register values
+ */
+#define TW5864_INDIR_AIN_0x0E3_AFAUTO BIT(3)
+/*
+ * AFAUTO control mode
+ * 0 8kHz setting (Default)
+ * 1 16kHz setting
+ * 2 32kHz setting
+ * 3 44.1kHz setting
+ * 4 48kHz setting
+ */
+#define TW5864_INDIR_AIN_0x0E3_AFMD 0x07
+
+#define TW5864_INDIR_AIN_0x0E4 0x0e4
+/* Define controls in register TW5864_INDIR_AIN_0x0ED */
+/*
+ * 8bit I2S Record output mode.
+ * 0 L/R half length separated output (Default).
+ * 1 One continuous packed output equal to DSP output format.
+ */
+#define TW5864_INDIR_AIN_0x0E4_I2S8MODE BIT(7)
+/*
+ * Audio Clock Master ACLKR output wave format.
+ * 0 High periods is one 27MHz clock period (default).
+ * 1 Almost duty 50-50% clock output on ACLKR pin. If this mode is selected, two
+ * times bigger number value need to be set up on the ACKI register. If
+ * AFAUTO=1, ACKI control is automatically set up even if MASCKMD=1.
+ */
+#define TW5864_INDIR_AIN_0x0E4_MASCKMD BIT(6)
+/* Playback ACLKP/ASYNP/ADATP input data MSB-LSB swapping */
+#define TW5864_INDIR_AIN_0x0E4_PBINSWAP BIT(5)
+/*
+ * ASYNR input signal delay.
+ * 0 No delay
+ * 1 Add one 27MHz period delay in ASYNR signal input
+ */
+#define TW5864_INDIR_AIN_0x0E4_ASYNRDLY BIT(4)
+/*
+ * ASYNP input signal delay.
+ * 0 no delay
+ * 1 add one 27MHz period delay in ASYNP signal input
+ */
+#define TW5864_INDIR_AIN_0x0E4_ASYNPDLY BIT(3)
+/*
+ * ADATP input data delay by one ACLKP clock.
+ * 0 No delay (Default). This is for I2S type 1T delay input interface.
+ * 1 Add 1 ACLKP clock delay in ADATP input data. This is for left-justified
+ * type 0T delay input interface.
+ */
+#define TW5864_INDIR_AIN_0x0E4_ADATPDLY BIT(2)
+/*
+ * Select u-Law/A-Law/PCM/SB data input format on ADATP pin.
+ * 0 PCM input (Default)
+ * 1 SB (Signed MSB bit in PCM data is inverted) input
+ * 2 u-Law input
+ * 3 A-Law input
+ */
+#define TW5864_INDIR_AIN_0x0E4_INLAWMD 0x03
+
+/*
+ * Enable state register updating and interrupt request of audio AIN5 detection
+ * for each input
+ */
+#define TW5864_INDIR_AIN_A5DETENA 0x0e5
+
+/* Some registers skipped */
+
+/*
+ * [7:3]: DEV_ID The TW5864 product ID code is 01000
+ * [2:0]: REV_ID The revision number is 0h
+ */
+#define TW5864_INDIR_ID 0x0fe
+
+#define TW5864_INDIR_IN_PIC_WIDTH(channel) (0x200 + 4 * channel)
+#define TW5864_INDIR_IN_PIC_HEIGHT(channel) (0x201 + 4 * channel)
+#define TW5864_INDIR_OUT_PIC_WIDTH(channel) (0x202 + 4 * channel)
+#define TW5864_INDIR_OUT_PIC_HEIGHT(channel) (0x203 + 4 * channel)
+/*
+ * Interrupt status register from the front-end. Write "1" to each bit to clear
+ * the interrupt
+ * 15:0 Motion detection interrupt for channel 0 ~ 15
+ * 31:16 Night detection interrupt for channel 0 ~ 15
+ * 47:32 Blind detection interrupt for channel 0 ~ 15
+ * 63:48 No video interrupt for channel 0 ~ 15
+ * 79:64 Line mode underflow interrupt for channel 0 ~ 15
+ * 95:80 Line mode overflow interrupt for channel 0 ~ 15
+ */
+/* 0x2d0~0x2d7: [63:0] bits */
+#define TW5864_INDIR_INTERRUPT1 0x2d0
+/* 0x2e0~0x2e3: [95:64] bits */
+#define TW5864_INDIR_INTERRUPT2 0x2e0
+
+/*
+ * Interrupt mask register for interrupts in 0x2d0 ~ 0x2d7
+ * 15:0 Motion detection interrupt for channel 0 ~ 15
+ * 31:16 Night detection interrupt for channel 0 ~ 15
+ * 47:32 Blind detection interrupt for channel 0 ~ 15
+ * 63:48 No video interrupt for channel 0 ~ 15
+ * 79:64 Line mode underflow interrupt for channel 0 ~ 15
+ * 95:80 Line mode overflow interrupt for channel 0 ~ 15
+ */
+/* 0x2d8~0x2df: [63:0] bits */
+#define TW5864_INDIR_INTERRUPT_MASK1 0x2d8
+/* 0x2e8~0x2eb: [95:64] bits */
+#define TW5864_INDIR_INTERRUPT_MASK2 0x2e8
+
+/* [11:0]: Interrupt summary register for interrupts & interrupt mask from in
+ * 0x2d0 ~ 0x2d7 and 0x2d8 ~ 0x2df
+ * bit 0: interrupt occurs in 0x2d0 & 0x2d8
+ * bit 1: interrupt occurs in 0x2d1 & 0x2d9
+ * bit 2: interrupt occurs in 0x2d2 & 0x2da
+ * bit 3: interrupt occurs in 0x2d3 & 0x2db
+ * bit 4: interrupt occurs in 0x2d4 & 0x2dc
+ * bit 5: interrupt occurs in 0x2d5 & 0x2dd
+ * bit 6: interrupt occurs in 0x2d6 & 0x2de
+ * bit 7: interrupt occurs in 0x2d7 & 0x2df
+ * bit 8: interrupt occurs in 0x2e0 & 0x2e8
+ * bit 9: interrupt occurs in 0x2e1 & 0x2e9
+ * bit 10: interrupt occurs in 0x2e2 & 0x2ea
+ * bit 11: interrupt occurs in 0x2e3 & 0x2eb
+ */
+#define TW5864_INDIR_INTERRUPT_SUMMARY 0x2f0
+
+/* Motion / Blind / Night Detection */
+/* valid value for channel is [0:15] */
+#define TW5864_INDIR_DETECTION_CTL0(channel) (0x300 + channel * 0x08)
+/* Define controls in register TW5864_INDIR_DETECTION_CTL0 */
+/*
+ * Disable the motion and blind detection.
+ * 0 Enable motion and blind detection (default)
+ * 1 Disable motion and blind detection
+ */
+#define TW5864_INDIR_DETECTION_CTL0_MD_DIS BIT(5)
+/*
+ * Request to start motion detection on manual trigger mode
+ * 0 None Operation (default)
+ * 1 Request to start motion detection
+ */
+#define TW5864_INDIR_DETECTION_CTL0_MD_STRB BIT(3)
+/*
+ * Select the trigger mode of motion detection
+ * 0 Automatic trigger mode of motion detection (default)
+ * 1 Manual trigger mode for motion detection
+ */
+#define TW5864_INDIR_DETECTION_CTL0_MD_STRB_EN BIT(2)
+/*
+ * Define the threshold of cell for blind detection.
+ * 0 Low threshold (More sensitive) (default)
+ * : :
+ * 3 High threshold (Less sensitive)
+ */
+#define TW5864_INDIR_DETECTION_CTL0_BD_CELSENS 0x03
+
+#define TW5864_INDIR_DETECTION_CTL1(channel) (0x301 + channel * 0x08)
+/* Define controls in register TW5864_INDIR_DETECTION_CTL1 */
+/*
+ * Control the temporal sensitivity of motion detector.
+ * 0 More Sensitive (default)
+ * : :
+ * 15 Less Sensitive
+ */
+#define TW5864_INDIR_DETECTION_CTL1_MD_TMPSENS_SHIFT 4
+#define TW5864_INDIR_DETECTION_CTL1_MD_TMPSENS (0x0f << 4)
+/*
+ * Adjust the horizontal starting position for motion detection
+ * 0 0 pixel (default)
+ * : :
+ * 15 15 pixels
+ */
+#define TW5864_INDIR_DETECTION_CTL1_MD_PIXEL_OS 0x0f
+
+#define TW5864_INDIR_DETECTION_CTL2(channel) (0x302 + channel * 0x08)
+/* Define controls in register TW5864_INDIR_DETECTION_CTL2 */
+/*
+ * Control the updating time of reference field for motion detection.
+ * 0 Update reference field every field (default)
+ * 1 Update reference field according to MD_SPEED
+ */
+#define TW5864_INDIR_DETECTION_CTL2_MD_REFFLD BIT(7)
+/*
+ * Select the field for motion detection.
+ * 0 Detecting motion for only odd field (default)
+ * 1 Detecting motion for only even field
+ * 2 Detecting motion for any field
+ * 3 Detecting motion for both odd and even field
+ */
+#define TW5864_INDIR_DETECTION_CTL2_MD_FIELD_SHIFT 5
+#define TW5864_INDIR_DETECTION_CTL2_MD_FIELD (0x03 << 5)
+/*
+ * Control the level sensitivity of motion detector.
+ * 0 More sensitive (default)
+ * : :
+ * 15 Less sensitive
+ */
+#define TW5864_INDIR_DETECTION_CTL2_MD_LVSENS 0x1f
+
+#define TW5864_INDIR_DETECTION_CTL3(channel) (0x303 + channel * 0x08)
+/* Define controls in register TW5864_INDIR_DETECTION_CTL3 */
+/*
+ * Define the threshold of sub-cell number for motion detection.
+ * 0 Motion is detected if 1 sub-cell has motion (More sensitive) (default)
+ * 1 Motion is detected if 2 sub-cells have motion
+ * 2 Motion is detected if 3 sub-cells have motion
+ * 3 Motion is detected if 4 sub-cells have motion (Less sensitive)
+ */
+#define TW5864_INDIR_DETECTION_CTL3_MD_CELSENS_SHIFT 6
+#define TW5864_INDIR_DETECTION_CTL3_MD_CELSENS (0x03 << 6)
+/*
+ * Control the velocity of motion detector.
+ * Large value is suitable for slow motion detection.
+ * In MD_DUAL_EN = 1, MD_SPEED should be limited to 0 ~ 31.
+ * 0 1 field intervals (default)
+ * 1 2 field intervals
+ * : :
+ * 61 62 field intervals
+ * 62 63 field intervals
+ * 63 Not supported
+ */
+#define TW5864_INDIR_DETECTION_CTL3_MD_SPEED 0x3f
+
+#define TW5864_INDIR_DETECTION_CTL4(channel) (0x304 + channel * 0x08)
+/* Define controls in register TW5864_INDIR_DETECTION_CTL4 */
+/*
+ * Control the spatial sensitivity of motion detector.
+ * 0 More Sensitive (default)
+ * : :
+ * 15 Less Sensitive
+ */
+#define TW5864_INDIR_DETECTION_CTL4_MD_SPSENS_SHIFT 4
+#define TW5864_INDIR_DETECTION_CTL4_MD_SPSENS (0x0f << 4)
+/*
+ * Define the threshold of level for blind detection.
+ * 0 Low threshold (More sensitive) (default)
+ * : :
+ * 15 High threshold (Less sensitive)
+ */
+#define TW5864_INDIR_DETECTION_CTL4_BD_LVSENS 0x0f
+
+#define TW5864_INDIR_DETECTION_CTL5(channel) (0x305 + channel * 0x08)
+/*
+ * Define the threshold of temporal sensitivity for night detection.
+ * 0 Low threshold (More sensitive) (default)
+ * : :
+ * 15 High threshold (Less sensitive)
+ */
+#define TW5864_INDIR_DETECTION_CTL5_ND_TMPSENS_SHIFT 4
+#define TW5864_INDIR_DETECTION_CTL5_ND_TMPSENS (0x0f << 4)
+/*
+ * Define the threshold of level for night detection.
+ * 0 Low threshold (More sensitive) (default)
+ * : :
+ * 3 High threshold (Less sensitive)
+ */
+#define TW5864_INDIR_DETECTION_CTL5_ND_LVSENS 0x0f
+
+/*
+ * [11:0] The base address of the motion detection buffer. This address is in
+ * unit of 64K bytes. The generated DDR address will be {MD_BASE_ADDR,
+ * 16"h0000}. The default value should be 12"h000
+ */
+#define TW5864_INDIR_MD_BASE_ADDR 0x380
+
+/*
+ * This controls the channel of the motion detection result shown in register
+ * 0x3a0 ~ 0x3b7. Before reading back motion result, always set this first.
+ */
+#define TW5864_INDIR_RGR_MOTION_SEL 0x382
+
+/* [15:0] MD strobe has been performed at channel n (read only) */
+#define TW5864_INDIR_MD_STRB 0x386
+/* NO_VIDEO Detected from channel n (read only) */
+#define TW5864_INDIR_NOVID_DET 0x388
+/* Motion Detected from channel n (read only) */
+#define TW5864_INDIR_MD_DET 0x38a
+/* Blind Detected from channel n (read only) */
+#define TW5864_INDIR_BD_DET 0x38c
+/* Night Detected from channel n (read only) */
+#define TW5864_INDIR_ND_DET 0x38e
+
+/* 192 bit motion flag of the channel specified by RGR_MOTION_SEL in 0x382 */
+#define TW5864_INDIR_MOTION_FLAG 0x3a0
+#define TW5864_INDIR_MOTION_FLAG_BYTE_COUNT 24
+
+/*
+ * [9:0] The motion cell count of a specific channel selected by 0x382. This is
+ * for DI purpose
+ */
+#define TW5864_INDIR_MD_DI_CNT 0x3b8
+/* The motion detection cell sensitivity for DI purpose */
+#define TW5864_INDIR_MD_DI_CELLSENS 0x3ba
+/* The motion detection threshold level for DI purpose */
+#define TW5864_INDIR_MD_DI_LVSENS 0x3bb
+
+/* 192 bit motion mask of the channel specified by MASK_CH_SEL in 0x3fe */
+#define TW5864_INDIR_MOTION_MASK 0x3e0
+#define TW5864_INDIR_MOTION_MASK_BYTE_COUNT 24
+
+/* [4:0] The channel selection to access masks in 0x3e0 ~ 0x3f7 */
+#define TW5864_INDIR_MASK_CH_SEL 0x3fe
+
+/* Clock PLL / Analog IP Control */
+/* Some registers skipped */
+
+#define TW5864_INDIR_DDRA_DLL_DQS_SEL0 0xee6
+#define TW5864_INDIR_DDRA_DLL_DQS_SEL1 0xee7
+#define TW5864_INDIR_DDRA_DLL_CLK90_SEL 0xee8
+#define TW5864_INDIR_DDRA_DLL_TEST_SEL_AND_TAP_S 0xee9
+
+#define TW5864_INDIR_DDRB_DLL_DQS_SEL0 0xeeb
+#define TW5864_INDIR_DDRB_DLL_DQS_SEL1 0xeec
+#define TW5864_INDIR_DDRB_DLL_CLK90_SEL 0xeed
+#define TW5864_INDIR_DDRB_DLL_TEST_SEL_AND_TAP_S 0xeee
+
+#define TW5864_INDIR_RESET 0xef0
+#define TW5864_INDIR_RESET_VD BIT(7)
+#define TW5864_INDIR_RESET_DLL BIT(6)
+#define TW5864_INDIR_RESET_MUX_CORE BIT(5)
+
+#define TW5864_INDIR_PV_VD_CK_POL 0xefd
+#define TW5864_INDIR_PV_VD_CK_POL_PV(channel) BIT(channel)
+#define TW5864_INDIR_PV_VD_CK_POL_VD(channel) BIT(channel + 4)
+
+#define TW5864_INDIR_CLK0_SEL 0xefe
+#define TW5864_INDIR_CLK0_SEL_VD_SHIFT 0
+#define TW5864_INDIR_CLK0_SEL_VD_MASK 0x3
+#define TW5864_INDIR_CLK0_SEL_PV_SHIFT 2
+#define TW5864_INDIR_CLK0_SEL_PV_MASK (0x3 << 2)
+#define TW5864_INDIR_CLK0_SEL_PV2_SHIFT 4
+#define TW5864_INDIR_CLK0_SEL_PV2_MASK (0x3 << 4)
diff --git a/drivers/media/pci/tw5864/tw5864-util.c b/drivers/media/pci/tw5864/tw5864-util.c
new file mode 100644
index 000000000000..771eef235755
--- /dev/null
+++ b/drivers/media/pci/tw5864/tw5864-util.c
@@ -0,0 +1,37 @@
+#include "tw5864.h"
+
+void tw5864_indir_writeb(struct tw5864_dev *dev, u16 addr, u8 data)
+{
+	int retries = 30000;
+
+	while (tw_readl(TW5864_IND_CTL) & BIT(31) && --retries)
+		;
+	if (!retries)
+		dev_err(&dev->pci->dev,
+			"tw_indir_writel() retries exhausted before writing\n");
+
+	tw_writel(TW5864_IND_DATA, data);
+	tw_writel(TW5864_IND_CTL, addr << 2 | TW5864_RW | TW5864_ENABLE);
+}
+
+u8 tw5864_indir_readb(struct tw5864_dev *dev, u16 addr)
+{
+	int retries = 30000;
+
+	while (tw_readl(TW5864_IND_CTL) & BIT(31) && --retries)
+		;
+	if (!retries)
+		dev_err(&dev->pci->dev,
+			"tw_indir_readl() retries exhausted before reading\n");
+
+	tw_writel(TW5864_IND_CTL, addr << 2 | TW5864_ENABLE);
+
+	retries = 30000;
+	while (tw_readl(TW5864_IND_CTL) & BIT(31) && --retries)
+		;
+	if (!retries)
+		dev_err(&dev->pci->dev,
+			"tw_indir_readl() retries exhausted at reading\n");
+
+	return tw_readl(TW5864_IND_DATA);
+}
diff --git a/drivers/media/pci/tw5864/tw5864-video.c b/drivers/media/pci/tw5864/tw5864-video.c
new file mode 100644
index 000000000000..652a059b2e0a
--- /dev/null
+++ b/drivers/media/pci/tw5864/tw5864-video.c
@@ -0,0 +1,1510 @@
+/*
+ *  TW5864 driver - video encoding functions
+ *
+ *  Copyright (C) 2016 Bluecherry, LLC <maintainers@bluecherrydvr.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-event.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "tw5864.h"
+#include "tw5864-reg.h"
+
+#define QUANTIZATION_TABLE_LEN 96
+#define VLC_LOOKUP_TABLE_LEN 1024
+
+static const u16 forward_quantization_table[QUANTIZATION_TABLE_LEN] = {
+	0x3333, 0x1f82, 0x3333, 0x1f82, 0x1f82, 0x147b, 0x1f82, 0x147b,
+	0x3333, 0x1f82, 0x3333, 0x1f82, 0x1f82, 0x147b, 0x1f82, 0x147b,
+	0x2e8c, 0x1d42, 0x2e8c, 0x1d42, 0x1d42, 0x1234, 0x1d42, 0x1234,
+	0x2e8c, 0x1d42, 0x2e8c, 0x1d42, 0x1d42, 0x1234, 0x1d42, 0x1234,
+	0x2762, 0x199a, 0x2762, 0x199a, 0x199a, 0x1062, 0x199a, 0x1062,
+	0x2762, 0x199a, 0x2762, 0x199a, 0x199a, 0x1062, 0x199a, 0x1062,
+	0x2492, 0x16c1, 0x2492, 0x16c1, 0x16c1, 0x0e3f, 0x16c1, 0x0e3f,
+	0x2492, 0x16c1, 0x2492, 0x16c1, 0x16c1, 0x0e3f, 0x16c1, 0x0e3f,
+	0x2000, 0x147b, 0x2000, 0x147b, 0x147b, 0x0d1b, 0x147b, 0x0d1b,
+	0x2000, 0x147b, 0x2000, 0x147b, 0x147b, 0x0d1b, 0x147b, 0x0d1b,
+	0x1c72, 0x11cf, 0x1c72, 0x11cf, 0x11cf, 0x0b4d, 0x11cf, 0x0b4d,
+	0x1c72, 0x11cf, 0x1c72, 0x11cf, 0x11cf, 0x0b4d, 0x11cf, 0x0b4d
+};
+
+static const u16 inverse_quantization_table[QUANTIZATION_TABLE_LEN] = {
+	0x800a, 0x800d, 0x800a, 0x800d, 0x800d, 0x8010, 0x800d, 0x8010,
+	0x800a, 0x800d, 0x800a, 0x800d, 0x800d, 0x8010, 0x800d, 0x8010,
+	0x800b, 0x800e, 0x800b, 0x800e, 0x800e, 0x8012, 0x800e, 0x8012,
+	0x800b, 0x800e, 0x800b, 0x800e, 0x800e, 0x8012, 0x800e, 0x8012,
+	0x800d, 0x8010, 0x800d, 0x8010, 0x8010, 0x8014, 0x8010, 0x8014,
+	0x800d, 0x8010, 0x800d, 0x8010, 0x8010, 0x8014, 0x8010, 0x8014,
+	0x800e, 0x8012, 0x800e, 0x8012, 0x8012, 0x8017, 0x8012, 0x8017,
+	0x800e, 0x8012, 0x800e, 0x8012, 0x8012, 0x8017, 0x8012, 0x8017,
+	0x8010, 0x8014, 0x8010, 0x8014, 0x8014, 0x8019, 0x8014, 0x8019,
+	0x8010, 0x8014, 0x8010, 0x8014, 0x8014, 0x8019, 0x8014, 0x8019,
+	0x8012, 0x8017, 0x8012, 0x8017, 0x8017, 0x801d, 0x8017, 0x801d,
+	0x8012, 0x8017, 0x8012, 0x8017, 0x8017, 0x801d, 0x8017, 0x801d
+};
+
+static const u16 encoder_vlc_lookup_table[VLC_LOOKUP_TABLE_LEN] = {
+	0x011, 0x000, 0x000, 0x000, 0x065, 0x021, 0x000, 0x000, 0x087, 0x064,
+	0x031, 0x000, 0x097, 0x086, 0x075, 0x053, 0x0a7, 0x096, 0x085, 0x063,
+	0x0b7, 0x0a6, 0x095, 0x074, 0x0df, 0x0b6, 0x0a5, 0x084, 0x0db, 0x0de,
+	0x0b5, 0x094, 0x0d8, 0x0da, 0x0dd, 0x0a4, 0x0ef, 0x0ee, 0x0d9, 0x0b4,
+	0x0eb, 0x0ea, 0x0ed, 0x0dc, 0x0ff, 0x0fe, 0x0e9, 0x0ec, 0x0fb, 0x0fa,
+	0x0fd, 0x0e8, 0x10f, 0x0f1, 0x0f9, 0x0fc, 0x10b, 0x10e, 0x10d, 0x0f8,
+	0x107, 0x10a, 0x109, 0x10c, 0x104, 0x106, 0x105, 0x108, 0x023, 0x000,
+	0x000, 0x000, 0x06b, 0x022, 0x000, 0x000, 0x067, 0x057, 0x033, 0x000,
+	0x077, 0x06a, 0x069, 0x045, 0x087, 0x066, 0x065, 0x044, 0x084, 0x076,
+	0x075, 0x056, 0x097, 0x086, 0x085, 0x068, 0x0bf, 0x096, 0x095, 0x064,
+	0x0bb, 0x0be, 0x0bd, 0x074, 0x0cf, 0x0ba, 0x0b9, 0x094, 0x0cb, 0x0ce,
+	0x0cd, 0x0bc, 0x0c8, 0x0ca, 0x0c9, 0x0b8, 0x0df, 0x0de, 0x0dd, 0x0cc,
+	0x0db, 0x0da, 0x0d9, 0x0dc, 0x0d7, 0x0eb, 0x0d6, 0x0d8, 0x0e9, 0x0e8,
+	0x0ea, 0x0d1, 0x0e7, 0x0e6, 0x0e5, 0x0e4, 0x04f, 0x000, 0x000, 0x000,
+	0x06f, 0x04e, 0x000, 0x000, 0x06b, 0x05f, 0x04d, 0x000, 0x068, 0x05c,
+	0x05e, 0x04c, 0x07f, 0x05a, 0x05b, 0x04b, 0x07b, 0x058, 0x059, 0x04a,
+	0x079, 0x06e, 0x06d, 0x049, 0x078, 0x06a, 0x069, 0x048, 0x08f, 0x07e,
+	0x07d, 0x05d, 0x08b, 0x08e, 0x07a, 0x06c, 0x09f, 0x08a, 0x08d, 0x07c,
+	0x09b, 0x09e, 0x089, 0x08c, 0x098, 0x09a, 0x09d, 0x088, 0x0ad, 0x097,
+	0x099, 0x09c, 0x0a9, 0x0ac, 0x0ab, 0x0aa, 0x0a5, 0x0a8, 0x0a7, 0x0a6,
+	0x0a1, 0x0a4, 0x0a3, 0x0a2, 0x021, 0x000, 0x000, 0x000, 0x067, 0x011,
+	0x000, 0x000, 0x064, 0x066, 0x031, 0x000, 0x063, 0x073, 0x072, 0x065,
+	0x062, 0x083, 0x082, 0x070, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x011, 0x010,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x011, 0x021, 0x020, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x023, 0x022, 0x021, 0x020, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x023, 0x022, 0x021, 0x031,
+	0x030, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x023, 0x022, 0x033, 0x032, 0x031, 0x030, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x023, 0x030,
+	0x031, 0x033, 0x032, 0x035, 0x034, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x037, 0x036, 0x035, 0x034, 0x033, 0x032,
+	0x031, 0x041, 0x051, 0x061, 0x071, 0x081, 0x091, 0x0a1, 0x0b1, 0x000,
+	0x002, 0x000, 0x0e4, 0x011, 0x0f4, 0x002, 0x024, 0x003, 0x005, 0x012,
+	0x034, 0x013, 0x065, 0x024, 0x013, 0x063, 0x015, 0x022, 0x075, 0x034,
+	0x044, 0x023, 0x023, 0x073, 0x054, 0x033, 0x033, 0x004, 0x043, 0x014,
+	0x011, 0x043, 0x014, 0x001, 0x025, 0x015, 0x035, 0x025, 0x064, 0x055,
+	0x045, 0x035, 0x074, 0x065, 0x085, 0x0d5, 0x012, 0x095, 0x055, 0x045,
+	0x095, 0x0e5, 0x084, 0x075, 0x022, 0x0a5, 0x094, 0x085, 0x032, 0x0b5,
+	0x003, 0x0c5, 0x001, 0x044, 0x0a5, 0x032, 0x0b5, 0x094, 0x0c5, 0x0a4,
+	0x0a4, 0x054, 0x0d5, 0x0b4, 0x0b4, 0x064, 0x0f5, 0x0f5, 0x053, 0x0d4,
+	0x0e5, 0x0c4, 0x105, 0x105, 0x0c4, 0x074, 0x063, 0x0e4, 0x0d4, 0x084,
+	0x073, 0x0f4, 0x004, 0x005, 0x000, 0x053, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x011, 0x021, 0x031, 0x030, 0x011, 0x021, 0x020, 0x000,
+	0x011, 0x010, 0x000, 0x000, 0x011, 0x033, 0x032, 0x043, 0x042, 0x053,
+	0x052, 0x063, 0x062, 0x073, 0x072, 0x083, 0x082, 0x093, 0x092, 0x091,
+	0x037, 0x036, 0x035, 0x034, 0x033, 0x045, 0x044, 0x043, 0x042, 0x053,
+	0x052, 0x063, 0x062, 0x061, 0x060, 0x000, 0x045, 0x037, 0x036, 0x035,
+	0x044, 0x043, 0x034, 0x033, 0x042, 0x053, 0x052, 0x061, 0x051, 0x060,
+	0x000, 0x000, 0x053, 0x037, 0x045, 0x044, 0x036, 0x035, 0x034, 0x043,
+	0x033, 0x042, 0x052, 0x051, 0x050, 0x000, 0x000, 0x000, 0x045, 0x044,
+	0x043, 0x037, 0x036, 0x035, 0x034, 0x033, 0x042, 0x051, 0x041, 0x050,
+	0x000, 0x000, 0x000, 0x000, 0x061, 0x051, 0x037, 0x036, 0x035, 0x034,
+	0x033, 0x032, 0x041, 0x031, 0x060, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x061, 0x051, 0x035, 0x034, 0x033, 0x023, 0x032, 0x041, 0x031, 0x060,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x061, 0x041, 0x051, 0x033,
+	0x023, 0x022, 0x032, 0x031, 0x060, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x061, 0x060, 0x041, 0x023, 0x022, 0x031, 0x021, 0x051,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x051, 0x050,
+	0x031, 0x023, 0x022, 0x021, 0x041, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x040, 0x041, 0x031, 0x032, 0x011, 0x033,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x040, 0x041, 0x021, 0x011, 0x031, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x030, 0x031, 0x011, 0x021,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x020, 0x021, 0x011, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x010, 0x011,
+	0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000, 0x000,
+	0x000, 0x000, 0x000, 0x000
+};
+
+static const unsigned int lambda_lookup_table[] = {
+	0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020,
+	0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020,
+	0x0040, 0x0040, 0x0040, 0x0040, 0x0060, 0x0060, 0x0060, 0x0080,
+	0x0080, 0x0080, 0x00a0, 0x00c0, 0x00c0, 0x00e0, 0x0100, 0x0120,
+	0x0140, 0x0160, 0x01a0, 0x01c0, 0x0200, 0x0240, 0x0280, 0x02e0,
+	0x0320, 0x03a0, 0x0400, 0x0480, 0x0500, 0x05a0, 0x0660, 0x0720,
+	0x0800, 0x0900, 0x0a20, 0x0b60
+};
+
+static const unsigned int intra4x4_lambda3[] = {
+	1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1,
+	2, 2, 2, 2, 3, 3, 3, 4,
+	4, 4, 5, 6, 6, 7, 8, 9,
+	10, 11, 13, 14, 16, 18, 20, 23,
+	25, 29, 32, 36, 40, 45, 51, 57,
+	64, 72, 81, 91
+};
+
+static v4l2_std_id tw5864_get_v4l2_std(enum tw5864_vid_std std);
+static enum tw5864_vid_std tw5864_from_v4l2_std(v4l2_std_id v4l2_std);
+
+static void tw5864_handle_frame_task(unsigned long data);
+static void tw5864_handle_frame(struct tw5864_h264_frame *frame);
+static void tw5864_frame_interval_set(struct tw5864_input *input);
+
+static int tw5864_queue_setup(struct vb2_queue *q, unsigned int *num_buffers,
+			      unsigned int *num_planes, unsigned int sizes[],
+			      struct device *alloc_ctxs[])
+{
+	if (*num_planes)
+		return sizes[0] < H264_VLC_BUF_SIZE ? -EINVAL : 0;
+
+	sizes[0] = H264_VLC_BUF_SIZE;
+	*num_planes = 1;
+
+	return 0;
+}
+
+static void tw5864_buf_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct vb2_queue *vq = vb->vb2_queue;
+	struct tw5864_input *dev = vb2_get_drv_priv(vq);
+	struct tw5864_buf *buf = container_of(vbuf, struct tw5864_buf, vb);
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->slock, flags);
+	list_add_tail(&buf->list, &dev->active);
+	spin_unlock_irqrestore(&dev->slock, flags);
+}
+
+static int tw5864_input_std_get(struct tw5864_input *input,
+				enum tw5864_vid_std *std)
+{
+	struct tw5864_dev *dev = input->root;
+	u8 std_reg = tw_indir_readb(TW5864_INDIR_VIN_E(input->nr));
+
+	*std = (std_reg & 0x70) >> 4;
+
+	if (std_reg & 0x80) {
+		dev_dbg(&dev->pci->dev,
+			"Video format detection is in progress, please wait\n");
+		return -EAGAIN;
+	}
+
+	return 0;
+}
+
+static int tw5864_enable_input(struct tw5864_input *input)
+{
+	struct tw5864_dev *dev = input->root;
+	int nr = input->nr;
+	unsigned long flags;
+	int d1_width = 720;
+	int d1_height;
+	int frame_width_bus_value = 0;
+	int frame_height_bus_value = 0;
+	int reg_frame_bus = 0x1c;
+	int fmt_reg_value = 0;
+	int downscale_enabled = 0;
+
+	dev_dbg(&dev->pci->dev, "Enabling channel %d\n", nr);
+
+	input->frame_seqno = 0;
+	input->frame_gop_seqno = 0;
+	input->h264_idr_pic_id = 0;
+
+	input->reg_dsp_qp = input->qp;
+	input->reg_dsp_ref_mvp_lambda = lambda_lookup_table[input->qp];
+	input->reg_dsp_i4x4_weight = intra4x4_lambda3[input->qp];
+	input->reg_emu = TW5864_EMU_EN_LPF | TW5864_EMU_EN_BHOST
+		| TW5864_EMU_EN_SEN | TW5864_EMU_EN_ME | TW5864_EMU_EN_DDR;
+	input->reg_dsp = nr /* channel id */
+		| TW5864_DSP_CHROM_SW
+		| ((0xa << 8) & TW5864_DSP_MB_DELAY)
+		;
+
+	input->resolution = D1;
+
+	d1_height = (input->std == STD_NTSC) ? 480 : 576;
+
+	input->width = d1_width;
+	input->height = d1_height;
+
+	input->reg_interlacing = 0x4;
+
+	switch (input->resolution) {
+	case D1:
+		frame_width_bus_value = 0x2cf;
+		frame_height_bus_value = input->height - 1;
+		reg_frame_bus = 0x1c;
+		fmt_reg_value = 0;
+		downscale_enabled = 0;
+		input->reg_dsp_codec |= TW5864_CIF_MAP_MD | TW5864_HD1_MAP_MD;
+		input->reg_emu |= TW5864_DSP_FRAME_TYPE_D1;
+		input->reg_interlacing = TW5864_DI_EN | TW5864_DSP_INTER_ST;
+
+		tw_setl(TW5864_FULL_HALF_FLAG, 1 << nr);
+		break;
+	case HD1:
+		input->height /= 2;
+		input->width /= 2;
+		frame_width_bus_value = 0x2cf;
+		frame_height_bus_value = input->height * 2 - 1;
+		reg_frame_bus = 0x1c;
+		fmt_reg_value = 0;
+		downscale_enabled = 0;
+		input->reg_dsp_codec |= TW5864_HD1_MAP_MD;
+		input->reg_emu |= TW5864_DSP_FRAME_TYPE_D1;
+
+		tw_clearl(TW5864_FULL_HALF_FLAG, 1 << nr);
+
+		break;
+	case CIF:
+		input->height /= 4;
+		input->width /= 2;
+		frame_width_bus_value = 0x15f;
+		frame_height_bus_value = input->height * 2 - 1;
+		reg_frame_bus = 0x07;
+		fmt_reg_value = 1;
+		downscale_enabled = 1;
+		input->reg_dsp_codec |= TW5864_CIF_MAP_MD;
+
+		tw_clearl(TW5864_FULL_HALF_FLAG, 1 << nr);
+		break;
+	case QCIF:
+		input->height /= 4;
+		input->width /= 4;
+		frame_width_bus_value = 0x15f;
+		frame_height_bus_value = input->height * 2 - 1;
+		reg_frame_bus = 0x07;
+		fmt_reg_value = 1;
+		downscale_enabled = 1;
+		input->reg_dsp_codec |= TW5864_CIF_MAP_MD;
+
+		tw_clearl(TW5864_FULL_HALF_FLAG, 1 << nr);
+		break;
+	}
+
+	/* analog input width / 4 */
+	tw_indir_writeb(TW5864_INDIR_IN_PIC_WIDTH(nr), d1_width / 4);
+	tw_indir_writeb(TW5864_INDIR_IN_PIC_HEIGHT(nr), d1_height / 4);
+
+	/* output width / 4 */
+	tw_indir_writeb(TW5864_INDIR_OUT_PIC_WIDTH(nr), input->width / 4);
+	tw_indir_writeb(TW5864_INDIR_OUT_PIC_HEIGHT(nr), input->height / 4);
+
+	tw_writel(TW5864_DSP_PIC_MAX_MB,
+		  ((input->width / 16) << 8) | (input->height / 16));
+
+	tw_writel(TW5864_FRAME_WIDTH_BUS_A(nr),
+		  frame_width_bus_value);
+	tw_writel(TW5864_FRAME_WIDTH_BUS_B(nr),
+		  frame_width_bus_value);
+	tw_writel(TW5864_FRAME_HEIGHT_BUS_A(nr),
+		  frame_height_bus_value);
+	tw_writel(TW5864_FRAME_HEIGHT_BUS_B(nr),
+		  (frame_height_bus_value + 1) / 2 - 1);
+
+	tw5864_frame_interval_set(input);
+
+	if (downscale_enabled)
+		tw_setl(TW5864_H264EN_CH_DNS, 1 << nr);
+
+	tw_mask_shift_writel(TW5864_H264EN_CH_FMT_REG1, 0x3, 2 * nr,
+			     fmt_reg_value);
+
+	tw_mask_shift_writel((nr < 2
+			      ? TW5864_H264EN_RATE_MAX_LINE_REG1
+			      : TW5864_H264EN_RATE_MAX_LINE_REG2),
+			     0x1f, 5 * (nr % 2),
+			     input->std == STD_NTSC ? 29 : 24);
+
+	tw_mask_shift_writel((nr < 2) ? TW5864_FRAME_BUS1 :
+			     TW5864_FRAME_BUS2, 0xff, (nr % 2) * 8,
+			     reg_frame_bus);
+
+	spin_lock_irqsave(&dev->slock, flags);
+	input->enabled = 1;
+	spin_unlock_irqrestore(&dev->slock, flags);
+
+	return 0;
+}
+
+void tw5864_request_encoded_frame(struct tw5864_input *input)
+{
+	struct tw5864_dev *dev = input->root;
+	u32 enc_buf_id_new;
+
+	tw_setl(TW5864_DSP_CODEC, TW5864_CIF_MAP_MD | TW5864_HD1_MAP_MD);
+	tw_writel(TW5864_EMU, input->reg_emu);
+	tw_writel(TW5864_INTERLACING, input->reg_interlacing);
+	tw_writel(TW5864_DSP, input->reg_dsp);
+
+	tw_writel(TW5864_DSP_QP, input->reg_dsp_qp);
+	tw_writel(TW5864_DSP_REF_MVP_LAMBDA, input->reg_dsp_ref_mvp_lambda);
+	tw_writel(TW5864_DSP_I4x4_WEIGHT, input->reg_dsp_i4x4_weight);
+	tw_mask_shift_writel(TW5864_DSP_INTRA_MODE, TW5864_DSP_INTRA_MODE_MASK,
+			     TW5864_DSP_INTRA_MODE_SHIFT,
+			     TW5864_DSP_INTRA_MODE_16x16);
+
+	if (input->frame_gop_seqno == 0) {
+		/* Produce I-frame */
+		tw_writel(TW5864_MOTION_SEARCH_ETC, TW5864_INTRA_EN);
+		input->h264_idr_pic_id++;
+		input->h264_idr_pic_id &= TW5864_DSP_REF_FRM;
+	} else {
+		/* Produce P-frame */
+		tw_writel(TW5864_MOTION_SEARCH_ETC, TW5864_INTRA_EN |
+			  TW5864_ME_EN | BIT(5) /* SRCH_OPT default */);
+	}
+	tw5864_prepare_frame_headers(input);
+	tw_writel(TW5864_VLC,
+		  TW5864_VLC_PCI_SEL |
+		  ((input->tail_nb_bits + 24) << TW5864_VLC_BIT_ALIGN_SHIFT) |
+		  input->reg_dsp_qp);
+
+	enc_buf_id_new = tw_mask_shift_readl(TW5864_ENC_BUF_PTR_REC1, 0x3,
+					     2 * input->nr);
+	tw_writel(TW5864_DSP_ENC_ORG_PTR_REG,
+		  enc_buf_id_new << TW5864_DSP_ENC_ORG_PTR_SHIFT);
+	tw_writel(TW5864_DSP_ENC_REC,
+		  enc_buf_id_new << 12 | ((enc_buf_id_new + 3) & 3));
+
+	tw_writel(TW5864_SLICE, TW5864_START_NSLICE);
+	tw_writel(TW5864_SLICE, 0);
+}
+
+static int tw5864_disable_input(struct tw5864_input *input)
+{
+	struct tw5864_dev *dev = input->root;
+	unsigned long flags;
+
+	dev_dbg(&dev->pci->dev, "Disabling channel %d\n", input->nr);
+
+	spin_lock_irqsave(&dev->slock, flags);
+	input->enabled = 0;
+	spin_unlock_irqrestore(&dev->slock, flags);
+	return 0;
+}
+
+static int tw5864_start_streaming(struct vb2_queue *q, unsigned int count)
+{
+	struct tw5864_input *input = vb2_get_drv_priv(q);
+	int ret;
+
+	ret = tw5864_enable_input(input);
+	if (!ret)
+		return 0;
+
+	while (!list_empty(&input->active)) {
+		struct tw5864_buf *buf = list_entry(input->active.next,
+						    struct tw5864_buf, list);
+
+		list_del(&buf->list);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
+	}
+	return ret;
+}
+
+static void tw5864_stop_streaming(struct vb2_queue *q)
+{
+	unsigned long flags;
+	struct tw5864_input *input = vb2_get_drv_priv(q);
+
+	tw5864_disable_input(input);
+
+	spin_lock_irqsave(&input->slock, flags);
+	if (input->vb) {
+		vb2_buffer_done(&input->vb->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+		input->vb = NULL;
+	}
+	while (!list_empty(&input->active)) {
+		struct tw5864_buf *buf = list_entry(input->active.next,
+						    struct tw5864_buf, list);
+
+		list_del(&buf->list);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+	}
+	spin_unlock_irqrestore(&input->slock, flags);
+}
+
+static const struct vb2_ops tw5864_video_qops = {
+	.queue_setup = tw5864_queue_setup,
+	.buf_queue = tw5864_buf_queue,
+	.start_streaming = tw5864_start_streaming,
+	.stop_streaming = tw5864_stop_streaming,
+	.wait_prepare = vb2_ops_wait_prepare,
+	.wait_finish = vb2_ops_wait_finish,
+};
+
+static int tw5864_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct tw5864_input *input =
+		container_of(ctrl->handler, struct tw5864_input, hdl);
+	struct tw5864_dev *dev = input->root;
+	unsigned long flags;
+
+	switch (ctrl->id) {
+	case V4L2_CID_BRIGHTNESS:
+		tw_indir_writeb(TW5864_INDIR_VIN_A_BRIGHT(input->nr),
+				(u8)ctrl->val);
+		break;
+	case V4L2_CID_HUE:
+		tw_indir_writeb(TW5864_INDIR_VIN_7_HUE(input->nr),
+				(u8)ctrl->val);
+		break;
+	case V4L2_CID_CONTRAST:
+		tw_indir_writeb(TW5864_INDIR_VIN_9_CNTRST(input->nr),
+				(u8)ctrl->val);
+		break;
+	case V4L2_CID_SATURATION:
+		tw_indir_writeb(TW5864_INDIR_VIN_B_SAT_U(input->nr),
+				(u8)ctrl->val);
+		tw_indir_writeb(TW5864_INDIR_VIN_C_SAT_V(input->nr),
+				(u8)ctrl->val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_GOP_SIZE:
+		input->gop = ctrl->val;
+		return 0;
+	case V4L2_CID_MPEG_VIDEO_H264_MIN_QP:
+		spin_lock_irqsave(&input->slock, flags);
+		input->qp = ctrl->val;
+		input->reg_dsp_qp = input->qp;
+		input->reg_dsp_ref_mvp_lambda = lambda_lookup_table[input->qp];
+		input->reg_dsp_i4x4_weight = intra4x4_lambda3[input->qp];
+		spin_unlock_irqrestore(&input->slock, flags);
+		return 0;
+	case V4L2_CID_DETECT_MD_GLOBAL_THRESHOLD:
+		memset(input->md_threshold_grid_values, ctrl->val,
+		       sizeof(input->md_threshold_grid_values));
+		return 0;
+	case V4L2_CID_DETECT_MD_MODE:
+		return 0;
+	case V4L2_CID_DETECT_MD_THRESHOLD_GRID:
+		/* input->md_threshold_grid_ctrl->p_new.p_u16 contains data */
+		memcpy(input->md_threshold_grid_values,
+		       input->md_threshold_grid_ctrl->p_new.p_u16,
+		       sizeof(input->md_threshold_grid_values));
+		return 0;
+	}
+	return 0;
+}
+
+static int tw5864_fmt_vid_cap(struct file *file, void *priv,
+			      struct v4l2_format *f)
+{
+	struct tw5864_input *input = video_drvdata(file);
+
+	f->fmt.pix.width = 720;
+	switch (input->std) {
+	default:
+		WARN_ON_ONCE(1);
+	case STD_NTSC:
+		f->fmt.pix.height = 480;
+		break;
+	case STD_PAL:
+	case STD_SECAM:
+		f->fmt.pix.height = 576;
+		break;
+	}
+	f->fmt.pix.field = V4L2_FIELD_INTERLACED;
+	f->fmt.pix.pixelformat = V4L2_PIX_FMT_H264;
+	f->fmt.pix.sizeimage = H264_VLC_BUF_SIZE;
+	f->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
+	return 0;
+}
+
+static int tw5864_enum_input(struct file *file, void *priv,
+			     struct v4l2_input *i)
+{
+	struct tw5864_input *input = video_drvdata(file);
+	struct tw5864_dev *dev = input->root;
+
+	u8 indir_0x000 = tw_indir_readb(TW5864_INDIR_VIN_0(input->nr));
+	u8 indir_0x00d = tw_indir_readb(TW5864_INDIR_VIN_D(input->nr));
+	u8 v1 = indir_0x000;
+	u8 v2 = indir_0x00d;
+
+	if (i->index)
+		return -EINVAL;
+
+	i->type = V4L2_INPUT_TYPE_CAMERA;
+	snprintf(i->name, sizeof(i->name), "Encoder %d", input->nr);
+	i->std = TW5864_NORMS;
+	if (v1 & (1 << 7))
+		i->status |= V4L2_IN_ST_NO_SYNC;
+	if (!(v1 & (1 << 6)))
+		i->status |= V4L2_IN_ST_NO_H_LOCK;
+	if (v1 & (1 << 2))
+		i->status |= V4L2_IN_ST_NO_SIGNAL;
+	if (v1 & (1 << 1))
+		i->status |= V4L2_IN_ST_NO_COLOR;
+	if (v2 & (1 << 2))
+		i->status |= V4L2_IN_ST_MACROVISION;
+
+	return 0;
+}
+
+static int tw5864_g_input(struct file *file, void *priv, unsigned int *i)
+{
+	*i = 0;
+	return 0;
+}
+
+static int tw5864_s_input(struct file *file, void *priv, unsigned int i)
+{
+	if (i)
+		return -EINVAL;
+	return 0;
+}
+
+static int tw5864_querycap(struct file *file, void *priv,
+			   struct v4l2_capability *cap)
+{
+	struct tw5864_input *input = video_drvdata(file);
+
+	strcpy(cap->driver, "tw5864");
+	snprintf(cap->card, sizeof(cap->card), "TW5864 Encoder %d",
+		 input->nr);
+	sprintf(cap->bus_info, "PCI:%s", pci_name(input->root->pci));
+	return 0;
+}
+
+static int tw5864_querystd(struct file *file, void *priv, v4l2_std_id *std)
+{
+	struct tw5864_input *input = video_drvdata(file);
+	enum tw5864_vid_std tw_std;
+	int ret;
+
+	ret = tw5864_input_std_get(input, &tw_std);
+	if (ret)
+		return ret;
+	*std = tw5864_get_v4l2_std(tw_std);
+
+	return 0;
+}
+
+static int tw5864_g_std(struct file *file, void *priv, v4l2_std_id *std)
+{
+	struct tw5864_input *input = video_drvdata(file);
+
+	*std = input->v4l2_std;
+	return 0;
+}
+
+static int tw5864_s_std(struct file *file, void *priv, v4l2_std_id std)
+{
+	struct tw5864_input *input = video_drvdata(file);
+	struct tw5864_dev *dev = input->root;
+
+	input->v4l2_std = std;
+	input->std = tw5864_from_v4l2_std(std);
+	tw_indir_writeb(TW5864_INDIR_VIN_E(input->nr), input->std);
+	return 0;
+}
+
+static int tw5864_enum_fmt_vid_cap(struct file *file, void *priv,
+				   struct v4l2_fmtdesc *f)
+{
+	if (f->index)
+		return -EINVAL;
+
+	f->pixelformat = V4L2_PIX_FMT_H264;
+
+	return 0;
+}
+
+static int tw5864_subscribe_event(struct v4l2_fh *fh,
+				  const struct v4l2_event_subscription *sub)
+{
+	switch (sub->type) {
+	case V4L2_EVENT_CTRL:
+		return v4l2_ctrl_subscribe_event(fh, sub);
+	case V4L2_EVENT_MOTION_DET:
+		/*
+		 * Allow for up to 30 events (1 second for NTSC) to be stored.
+		 */
+		return v4l2_event_subscribe(fh, sub, 30, NULL);
+	}
+	return -EINVAL;
+}
+
+static void tw5864_frame_interval_set(struct tw5864_input *input)
+{
+	/*
+	 * This register value seems to follow such approach: In each second
+	 * interval, when processing Nth frame, it checks Nth bit of register
+	 * value and, if the bit is 1, it processes the frame, otherwise the
+	 * frame is discarded.
+	 * So unary representation would work, but more or less equal gaps
+	 * between the frames should be preserved.
+	 *
+	 * For 1 FPS - 0x00000001
+	 * 00000000 00000000 00000000 00000001
+	 *
+	 * For max FPS - set all 25/30 lower bits:
+	 * 00111111 11111111 11111111 11111111 (NTSC)
+	 * 00000001 11111111 11111111 11111111 (PAL)
+	 *
+	 * For half of max FPS - use such pattern:
+	 * 00010101 01010101 01010101 01010101 (NTSC)
+	 * 00000001 01010101 01010101 01010101 (PAL)
+	 *
+	 * Et cetera.
+	 *
+	 * The value supplied to hardware is capped by mask of 25/30 lower bits.
+	 */
+	struct tw5864_dev *dev = input->root;
+	u32 unary_framerate = 0;
+	int shift = 0;
+	int std_max_fps = input->std == STD_NTSC ? 30 : 25;
+
+	for (shift = 0; shift < std_max_fps; shift += input->frame_interval)
+		unary_framerate |= 0x00000001 << shift;
+
+	tw_writel(TW5864_H264EN_RATE_CNTL_LO_WORD(input->nr, 0),
+		  unary_framerate >> 16);
+	tw_writel(TW5864_H264EN_RATE_CNTL_HI_WORD(input->nr, 0),
+		  unary_framerate & 0xffff);
+}
+
+static int tw5864_frameinterval_get(struct tw5864_input *input,
+				    struct v4l2_fract *frameinterval)
+{
+	switch (input->std) {
+	case STD_NTSC:
+		frameinterval->numerator = 1001;
+		frameinterval->denominator = 30000;
+		break;
+	case STD_PAL:
+	case STD_SECAM:
+		frameinterval->numerator = 1;
+		frameinterval->denominator = 25;
+		break;
+	default:
+		WARN(1, "tw5864_frameinterval_get requested for unknown std %d\n",
+		     input->std);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int tw5864_enum_framesizes(struct file *file, void *priv,
+				  struct v4l2_frmsizeenum *fsize)
+{
+	struct tw5864_input *input = video_drvdata(file);
+
+	if (fsize->index > 0)
+		return -EINVAL;
+	if (fsize->pixel_format != V4L2_PIX_FMT_H264)
+		return -EINVAL;
+
+	fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE;
+	fsize->discrete.width = 720;
+	fsize->discrete.height = input->std == STD_NTSC ? 480 : 576;
+
+	return 0;
+}
+
+static int tw5864_enum_frameintervals(struct file *file, void *priv,
+				      struct v4l2_frmivalenum *fintv)
+{
+	struct tw5864_input *input = video_drvdata(file);
+	struct v4l2_fract frameinterval;
+	int std_max_fps = input->std == STD_NTSC ? 30 : 25;
+	struct v4l2_frmsizeenum fsize = { .index = fintv->index,
+		.pixel_format = fintv->pixel_format };
+	int ret;
+
+	ret = tw5864_enum_framesizes(file, priv, &fsize);
+	if (ret)
+		return ret;
+
+	if (fintv->width != fsize.discrete.width ||
+	    fintv->height != fsize.discrete.height)
+		return -EINVAL;
+
+	fintv->type = V4L2_FRMIVAL_TYPE_STEPWISE;
+
+	ret = tw5864_frameinterval_get(input, &frameinterval);
+	fintv->stepwise.step = frameinterval;
+	fintv->stepwise.min = frameinterval;
+	fintv->stepwise.max = frameinterval;
+	fintv->stepwise.max.numerator *= std_max_fps;
+
+	return ret;
+}
+
+static int tw5864_g_parm(struct file *file, void *priv,
+			 struct v4l2_streamparm *sp)
+{
+	struct tw5864_input *input = video_drvdata(file);
+	struct v4l2_captureparm *cp = &sp->parm.capture;
+	int ret;
+
+	cp->capability = V4L2_CAP_TIMEPERFRAME;
+
+	ret = tw5864_frameinterval_get(input, &cp->timeperframe);
+	cp->timeperframe.numerator *= input->frame_interval;
+	cp->capturemode = 0;
+	cp->readbuffers = 2;
+
+	return ret;
+}
+
+static int tw5864_s_parm(struct file *file, void *priv,
+			 struct v4l2_streamparm *sp)
+{
+	struct tw5864_input *input = video_drvdata(file);
+	struct v4l2_fract *t = &sp->parm.capture.timeperframe;
+	struct v4l2_fract time_base;
+	int ret;
+
+	ret = tw5864_frameinterval_get(input, &time_base);
+	if (ret)
+		return ret;
+
+	if (!t->numerator || !t->denominator) {
+		t->numerator = time_base.numerator * input->frame_interval;
+		t->denominator = time_base.denominator;
+	} else if (t->denominator != time_base.denominator) {
+		t->numerator = t->numerator * time_base.denominator /
+			t->denominator;
+		t->denominator = time_base.denominator;
+	}
+
+	input->frame_interval = t->numerator / time_base.numerator;
+	if (input->frame_interval < 1)
+		input->frame_interval = 1;
+	tw5864_frame_interval_set(input);
+	return tw5864_g_parm(file, priv, sp);
+}
+
+static const struct v4l2_ctrl_ops tw5864_ctrl_ops = {
+	.s_ctrl = tw5864_s_ctrl,
+};
+
+static const struct v4l2_file_operations video_fops = {
+	.owner = THIS_MODULE,
+	.open = v4l2_fh_open,
+	.release = vb2_fop_release,
+	.read = vb2_fop_read,
+	.poll = vb2_fop_poll,
+	.mmap = vb2_fop_mmap,
+	.unlocked_ioctl = video_ioctl2,
+};
+
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+
+#define INDIR_SPACE_MAP_SHIFT 0x100000
+
+static int tw5864_g_reg(struct file *file, void *fh,
+			struct v4l2_dbg_register *reg)
+{
+	struct tw5864_input *input = video_drvdata(file);
+	struct tw5864_dev *dev = input->root;
+
+	if (reg->reg < INDIR_SPACE_MAP_SHIFT) {
+		if (reg->reg > 0x87fff)
+			return -EINVAL;
+		reg->size = 4;
+		reg->val = tw_readl(reg->reg);
+	} else {
+		__u64 indir_addr = reg->reg - INDIR_SPACE_MAP_SHIFT;
+
+		if (indir_addr > 0xefe)
+			return -EINVAL;
+		reg->size = 1;
+		reg->val = tw_indir_readb(reg->reg);
+	}
+	return 0;
+}
+
+static int tw5864_s_reg(struct file *file, void *fh,
+			const struct v4l2_dbg_register *reg)
+{
+	struct tw5864_input *input = video_drvdata(file);
+	struct tw5864_dev *dev = input->root;
+
+	if (reg->reg < INDIR_SPACE_MAP_SHIFT) {
+		if (reg->reg > 0x87fff)
+			return -EINVAL;
+		tw_writel(reg->reg, reg->val);
+	} else {
+		__u64 indir_addr = reg->reg - INDIR_SPACE_MAP_SHIFT;
+
+		if (indir_addr > 0xefe)
+			return -EINVAL;
+		tw_indir_writeb(reg->reg, reg->val);
+	}
+	return 0;
+}
+#endif
+
+static const struct v4l2_ioctl_ops video_ioctl_ops = {
+	.vidioc_querycap = tw5864_querycap,
+	.vidioc_enum_fmt_vid_cap = tw5864_enum_fmt_vid_cap,
+	.vidioc_reqbufs = vb2_ioctl_reqbufs,
+	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_querybuf = vb2_ioctl_querybuf,
+	.vidioc_qbuf = vb2_ioctl_qbuf,
+	.vidioc_dqbuf = vb2_ioctl_dqbuf,
+	.vidioc_expbuf = vb2_ioctl_expbuf,
+	.vidioc_querystd = tw5864_querystd,
+	.vidioc_s_std = tw5864_s_std,
+	.vidioc_g_std = tw5864_g_std,
+	.vidioc_enum_input = tw5864_enum_input,
+	.vidioc_g_input = tw5864_g_input,
+	.vidioc_s_input = tw5864_s_input,
+	.vidioc_streamon = vb2_ioctl_streamon,
+	.vidioc_streamoff = vb2_ioctl_streamoff,
+	.vidioc_try_fmt_vid_cap = tw5864_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap = tw5864_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap = tw5864_fmt_vid_cap,
+	.vidioc_log_status = v4l2_ctrl_log_status,
+	.vidioc_subscribe_event = tw5864_subscribe_event,
+	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+	.vidioc_enum_framesizes = tw5864_enum_framesizes,
+	.vidioc_enum_frameintervals = tw5864_enum_frameintervals,
+	.vidioc_s_parm = tw5864_s_parm,
+	.vidioc_g_parm = tw5864_g_parm,
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+	.vidioc_g_register = tw5864_g_reg,
+	.vidioc_s_register = tw5864_s_reg,
+#endif
+};
+
+static const struct video_device tw5864_video_template = {
+	.name = "tw5864_video",
+	.fops = &video_fops,
+	.ioctl_ops = &video_ioctl_ops,
+	.release = video_device_release_empty,
+	.tvnorms = TW5864_NORMS,
+	.device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_READWRITE |
+		V4L2_CAP_STREAMING,
+};
+
+/* Motion Detection Threshold matrix */
+static const struct v4l2_ctrl_config tw5864_md_thresholds = {
+	.ops = &tw5864_ctrl_ops,
+	.id = V4L2_CID_DETECT_MD_THRESHOLD_GRID,
+	.dims = {MD_CELLS_HOR, MD_CELLS_VERT},
+	.def = 14,
+	/* See tw5864_md_metric_from_mvd() */
+	.max = 2 * 0x0f,
+	.step = 1,
+};
+
+static int tw5864_video_input_init(struct tw5864_input *dev, int video_nr);
+static void tw5864_video_input_fini(struct tw5864_input *dev);
+static void tw5864_encoder_tables_upload(struct tw5864_dev *dev);
+
+int tw5864_video_init(struct tw5864_dev *dev, int *video_nr)
+{
+	int i;
+	int ret;
+	unsigned long flags;
+	int last_dma_allocated = -1;
+	int last_input_nr_registered = -1;
+
+	for (i = 0; i < H264_BUF_CNT; i++) {
+		struct tw5864_h264_frame *frame = &dev->h264_buf[i];
+
+		frame->vlc.addr = dma_alloc_coherent(&dev->pci->dev,
+						     H264_VLC_BUF_SIZE,
+						     &frame->vlc.dma_addr,
+						     GFP_KERNEL | GFP_DMA32);
+		if (!frame->vlc.addr) {
+			dev_err(&dev->pci->dev, "dma alloc fail\n");
+			ret = -ENOMEM;
+			goto free_dma;
+		}
+		frame->mv.addr = dma_alloc_coherent(&dev->pci->dev,
+						    H264_MV_BUF_SIZE,
+						    &frame->mv.dma_addr,
+						    GFP_KERNEL | GFP_DMA32);
+		if (!frame->mv.addr) {
+			dev_err(&dev->pci->dev, "dma alloc fail\n");
+			ret = -ENOMEM;
+			dma_free_coherent(&dev->pci->dev, H264_VLC_BUF_SIZE,
+					  frame->vlc.addr, frame->vlc.dma_addr);
+			goto free_dma;
+		}
+		last_dma_allocated = i;
+	}
+
+	tw5864_encoder_tables_upload(dev);
+
+	/* Picture is distorted without this block */
+	/* use falling edge to sample 54M to 108M */
+	tw_indir_writeb(TW5864_INDIR_VD_108_POL, TW5864_INDIR_VD_108_POL_BOTH);
+	tw_indir_writeb(TW5864_INDIR_CLK0_SEL, 0x00);
+
+	tw_indir_writeb(TW5864_INDIR_DDRA_DLL_DQS_SEL0, 0x02);
+	tw_indir_writeb(TW5864_INDIR_DDRA_DLL_DQS_SEL1, 0x02);
+	tw_indir_writeb(TW5864_INDIR_DDRA_DLL_CLK90_SEL, 0x02);
+	tw_indir_writeb(TW5864_INDIR_DDRB_DLL_DQS_SEL0, 0x02);
+	tw_indir_writeb(TW5864_INDIR_DDRB_DLL_DQS_SEL1, 0x02);
+	tw_indir_writeb(TW5864_INDIR_DDRB_DLL_CLK90_SEL, 0x02);
+
+	/* video input reset */
+	tw_indir_writeb(TW5864_INDIR_RESET, 0);
+	tw_indir_writeb(TW5864_INDIR_RESET, TW5864_INDIR_RESET_VD |
+			TW5864_INDIR_RESET_DLL | TW5864_INDIR_RESET_MUX_CORE);
+	msleep(20);
+
+	/*
+	 * Select Part A mode for all channels.
+	 * tw_setl instead of tw_clearl for Part B mode.
+	 *
+	 * I guess "Part B" is primarily for downscaled version of same channel
+	 * which goes in Part A of same bus
+	 */
+	tw_writel(TW5864_FULL_HALF_MODE_SEL, 0);
+
+	tw_indir_writeb(TW5864_INDIR_PV_VD_CK_POL,
+			TW5864_INDIR_PV_VD_CK_POL_VD(0) |
+			TW5864_INDIR_PV_VD_CK_POL_VD(1) |
+			TW5864_INDIR_PV_VD_CK_POL_VD(2) |
+			TW5864_INDIR_PV_VD_CK_POL_VD(3));
+
+	spin_lock_irqsave(&dev->slock, flags);
+	dev->encoder_busy = 0;
+	dev->h264_buf_r_index = 0;
+	dev->h264_buf_w_index = 0;
+	tw_writel(TW5864_VLC_STREAM_BASE_ADDR,
+		  dev->h264_buf[dev->h264_buf_w_index].vlc.dma_addr);
+	tw_writel(TW5864_MV_STREAM_BASE_ADDR,
+		  dev->h264_buf[dev->h264_buf_w_index].mv.dma_addr);
+	spin_unlock_irqrestore(&dev->slock, flags);
+
+	tw_writel(TW5864_SEN_EN_CH, 0x000f);
+	tw_writel(TW5864_H264EN_CH_EN, 0x000f);
+
+	tw_writel(TW5864_H264EN_BUS0_MAP, 0x00000000);
+	tw_writel(TW5864_H264EN_BUS1_MAP, 0x00001111);
+	tw_writel(TW5864_H264EN_BUS2_MAP, 0x00002222);
+	tw_writel(TW5864_H264EN_BUS3_MAP, 0x00003333);
+
+	/*
+	 * Quote from Intersil (manufacturer):
+	 * 0x0038 is managed by HW, and by default it won't pass the pointer set
+	 * at 0x0010. So if you don't do encoding, 0x0038 should stay at '3'
+	 * (with 4 frames in buffer). If you encode one frame and then move
+	 * 0x0010 to '1' for example, HW will take one more frame and set it to
+	 * buffer #0, and then you should see 0x0038 is set to '0'.  There is
+	 * only one HW encoder engine, so 4 channels cannot get encoded
+	 * simultaneously. But each channel does have its own buffer (for
+	 * original frames and reconstructed frames). So there is no problem to
+	 * manage encoding for 4 channels at same time and no need to force
+	 * I-frames in switching channels.
+	 * End of quote.
+	 *
+	 * If we set 0x0010 (TW5864_ENC_BUF_PTR_REC1) to 0 (for any channel), we
+	 * have no "rolling" (until we change this value).
+	 * If we set 0x0010 (TW5864_ENC_BUF_PTR_REC1) to 0x3, it starts to roll
+	 * continuously together with 0x0038.
+	 */
+	tw_writel(TW5864_ENC_BUF_PTR_REC1, 0x00ff);
+	tw_writel(TW5864_PCI_INTTM_SCALE, 0);
+
+	tw_writel(TW5864_INTERLACING, TW5864_DI_EN);
+	tw_writel(TW5864_MASTER_ENB_REG, TW5864_PCI_VLC_INTR_ENB);
+	tw_writel(TW5864_PCI_INTR_CTL,
+		  TW5864_TIMER_INTR_ENB | TW5864_PCI_MAST_ENB |
+		  TW5864_MVD_VLC_MAST_ENB);
+
+	dev->irqmask |= TW5864_INTR_VLC_DONE | TW5864_INTR_TIMER;
+	tw5864_irqmask_apply(dev);
+
+	tasklet_init(&dev->tasklet, tw5864_handle_frame_task,
+		     (unsigned long)dev);
+
+	for (i = 0; i < TW5864_INPUTS; i++) {
+		dev->inputs[i].root = dev;
+		dev->inputs[i].nr = i;
+		ret = tw5864_video_input_init(&dev->inputs[i], video_nr[i]);
+		if (ret)
+			goto fini_video_inputs;
+		last_input_nr_registered = i;
+	}
+
+	return 0;
+
+fini_video_inputs:
+	for (i = last_input_nr_registered; i >= 0; i--)
+		tw5864_video_input_fini(&dev->inputs[i]);
+
+	tasklet_kill(&dev->tasklet);
+
+free_dma:
+	for (i = last_dma_allocated; i >= 0; i--) {
+		dma_free_coherent(&dev->pci->dev, H264_VLC_BUF_SIZE,
+				  dev->h264_buf[i].vlc.addr,
+				  dev->h264_buf[i].vlc.dma_addr);
+		dma_free_coherent(&dev->pci->dev, H264_MV_BUF_SIZE,
+				  dev->h264_buf[i].mv.addr,
+				  dev->h264_buf[i].mv.dma_addr);
+	}
+
+	return ret;
+}
+
+static int tw5864_video_input_init(struct tw5864_input *input, int video_nr)
+{
+	struct tw5864_dev *dev = input->root;
+	int ret;
+	struct v4l2_ctrl_handler *hdl = &input->hdl;
+
+	mutex_init(&input->lock);
+	spin_lock_init(&input->slock);
+
+	/* setup video buffers queue */
+	INIT_LIST_HEAD(&input->active);
+	input->vidq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	input->vidq.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	input->vidq.io_modes = VB2_MMAP | VB2_READ | VB2_DMABUF;
+	input->vidq.ops = &tw5864_video_qops;
+	input->vidq.mem_ops = &vb2_dma_contig_memops;
+	input->vidq.drv_priv = input;
+	input->vidq.gfp_flags = 0;
+	input->vidq.buf_struct_size = sizeof(struct tw5864_buf);
+	input->vidq.lock = &input->lock;
+	input->vidq.min_buffers_needed = 2;
+	input->vidq.dev = &input->root->pci->dev;
+	ret = vb2_queue_init(&input->vidq);
+	if (ret)
+		goto free_mutex;
+
+	input->vdev = tw5864_video_template;
+	input->vdev.v4l2_dev = &input->root->v4l2_dev;
+	input->vdev.lock = &input->lock;
+	input->vdev.queue = &input->vidq;
+	video_set_drvdata(&input->vdev, input);
+
+	/* Initialize the device control structures */
+	v4l2_ctrl_handler_init(hdl, 6);
+	v4l2_ctrl_new_std(hdl, &tw5864_ctrl_ops,
+			  V4L2_CID_BRIGHTNESS, -128, 127, 1, 0);
+	v4l2_ctrl_new_std(hdl, &tw5864_ctrl_ops,
+			  V4L2_CID_CONTRAST, 0, 255, 1, 100);
+	v4l2_ctrl_new_std(hdl, &tw5864_ctrl_ops,
+			  V4L2_CID_SATURATION, 0, 255, 1, 128);
+	v4l2_ctrl_new_std(hdl, &tw5864_ctrl_ops, V4L2_CID_HUE, -128, 127, 1, 0);
+	v4l2_ctrl_new_std(hdl, &tw5864_ctrl_ops, V4L2_CID_MPEG_VIDEO_GOP_SIZE,
+			  1, MAX_GOP_SIZE, 1, GOP_SIZE);
+	v4l2_ctrl_new_std(hdl, &tw5864_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_H264_MIN_QP, 28, 51, 1, QP_VALUE);
+	v4l2_ctrl_new_std_menu(hdl, &tw5864_ctrl_ops,
+			       V4L2_CID_DETECT_MD_MODE,
+			       V4L2_DETECT_MD_MODE_THRESHOLD_GRID, 0,
+			       V4L2_DETECT_MD_MODE_DISABLED);
+	v4l2_ctrl_new_std(hdl, &tw5864_ctrl_ops,
+			  V4L2_CID_DETECT_MD_GLOBAL_THRESHOLD,
+			  tw5864_md_thresholds.min, tw5864_md_thresholds.max,
+			  tw5864_md_thresholds.step, tw5864_md_thresholds.def);
+	input->md_threshold_grid_ctrl =
+		v4l2_ctrl_new_custom(hdl, &tw5864_md_thresholds, NULL);
+	if (hdl->error) {
+		ret = hdl->error;
+		goto free_v4l2_hdl;
+	}
+	input->vdev.ctrl_handler = hdl;
+	v4l2_ctrl_handler_setup(hdl);
+
+	input->qp = QP_VALUE;
+	input->gop = GOP_SIZE;
+	input->frame_interval = 1;
+
+	ret = video_register_device(&input->vdev, VFL_TYPE_GRABBER, video_nr);
+	if (ret)
+		goto free_v4l2_hdl;
+
+	dev_info(&input->root->pci->dev, "Registered video device %s\n",
+		 video_device_node_name(&input->vdev));
+
+	/*
+	 * Set default video standard. Doesn't matter which, the detected value
+	 * will be found out by VIDIOC_QUERYSTD handler.
+	 */
+	input->v4l2_std = V4L2_STD_NTSC_M;
+	input->std = STD_NTSC;
+
+	tw_indir_writeb(TW5864_INDIR_VIN_E(video_nr), 0x07);
+	/* to initiate auto format recognition */
+	tw_indir_writeb(TW5864_INDIR_VIN_F(video_nr), 0xff);
+
+	return 0;
+
+free_v4l2_hdl:
+	v4l2_ctrl_handler_free(hdl);
+	vb2_queue_release(&input->vidq);
+free_mutex:
+	mutex_destroy(&input->lock);
+
+	return ret;
+}
+
+static void tw5864_video_input_fini(struct tw5864_input *dev)
+{
+	video_unregister_device(&dev->vdev);
+	v4l2_ctrl_handler_free(&dev->hdl);
+	vb2_queue_release(&dev->vidq);
+}
+
+void tw5864_video_fini(struct tw5864_dev *dev)
+{
+	int i;
+
+	tasklet_kill(&dev->tasklet);
+
+	for (i = 0; i < TW5864_INPUTS; i++)
+		tw5864_video_input_fini(&dev->inputs[i]);
+
+	for (i = 0; i < H264_BUF_CNT; i++) {
+		dma_free_coherent(&dev->pci->dev, H264_VLC_BUF_SIZE,
+				  dev->h264_buf[i].vlc.addr,
+				  dev->h264_buf[i].vlc.dma_addr);
+		dma_free_coherent(&dev->pci->dev, H264_MV_BUF_SIZE,
+				  dev->h264_buf[i].mv.addr,
+				  dev->h264_buf[i].mv.dma_addr);
+	}
+}
+
+void tw5864_prepare_frame_headers(struct tw5864_input *input)
+{
+	struct tw5864_buf *vb = input->vb;
+	u8 *dst;
+	size_t dst_space;
+	unsigned long flags;
+
+	if (!vb) {
+		spin_lock_irqsave(&input->slock, flags);
+		if (list_empty(&input->active)) {
+			spin_unlock_irqrestore(&input->slock, flags);
+			input->vb = NULL;
+			return;
+		}
+		vb = list_first_entry(&input->active, struct tw5864_buf, list);
+		list_del(&vb->list);
+		spin_unlock_irqrestore(&input->slock, flags);
+	}
+
+	dst = vb2_plane_vaddr(&vb->vb.vb2_buf, 0);
+	dst_space = vb2_plane_size(&vb->vb.vb2_buf, 0);
+
+	/*
+	 * Low-level bitstream writing functions don't have a fine way to say
+	 * correctly that supplied buffer is too small. So we just check there
+	 * and warn, and don't care at lower level.
+	 * Currently all headers take below 32 bytes.
+	 * The buffer is supposed to have plenty of free space at this point,
+	 * anyway.
+	 */
+	if (WARN_ON_ONCE(dst_space < 128))
+		return;
+
+	/*
+	 * Generate H264 headers:
+	 * If this is first frame, put SPS and PPS
+	 */
+	if (input->frame_gop_seqno == 0)
+		tw5864_h264_put_stream_header(&dst, &dst_space, input->qp,
+					      input->width, input->height);
+
+	/* Put slice header */
+	tw5864_h264_put_slice_header(&dst, &dst_space, input->h264_idr_pic_id,
+				     input->frame_gop_seqno,
+				     &input->tail_nb_bits, &input->tail);
+	input->vb = vb;
+	input->buf_cur_ptr = dst;
+	input->buf_cur_space_left = dst_space;
+}
+
+/*
+ * Returns heuristic motion detection metric value from known components of
+ * hardware-provided Motion Vector Data.
+ */
+static unsigned int tw5864_md_metric_from_mvd(u32 mvd)
+{
+	/*
+	 * Format of motion vector data exposed by tw5864, according to
+	 * manufacturer:
+	 * mv_x 10 bits
+	 * mv_y 10 bits
+	 * non_zero_members 8 bits
+	 * mb_type 3 bits
+	 * reserved 1 bit
+	 *
+	 * non_zero_members: number of non-zero residuals in each macro block
+	 * after quantization
+	 *
+	 * unsigned int reserved = mvd >> 31;
+	 * unsigned int mb_type = (mvd >> 28) & 0x7;
+	 * unsigned int non_zero_members = (mvd >> 20) & 0xff;
+	 */
+	unsigned int mv_y = (mvd >> 10) & 0x3ff;
+	unsigned int mv_x = mvd & 0x3ff;
+
+	/* heuristic: */
+	mv_x &= 0x0f;
+	mv_y &= 0x0f;
+
+	return mv_y + mv_x;
+}
+
+static int tw5864_is_motion_triggered(struct tw5864_h264_frame *frame)
+{
+	struct tw5864_input *input = frame->input;
+	u32 *mv = (u32 *)frame->mv.addr;
+	int i;
+	int detected = 0;
+
+	for (i = 0; i < MD_CELLS; i++) {
+		const u16 thresh = input->md_threshold_grid_values[i];
+		const unsigned int metric = tw5864_md_metric_from_mvd(mv[i]);
+
+		if (metric > thresh)
+			detected = 1;
+
+		if (detected)
+			break;
+	}
+	return detected;
+}
+
+static void tw5864_handle_frame_task(unsigned long data)
+{
+	struct tw5864_dev *dev = (struct tw5864_dev *)data;
+	unsigned long flags;
+	int batch_size = H264_BUF_CNT;
+
+	spin_lock_irqsave(&dev->slock, flags);
+	while (dev->h264_buf_r_index != dev->h264_buf_w_index && batch_size--) {
+		struct tw5864_h264_frame *frame =
+			&dev->h264_buf[dev->h264_buf_r_index];
+
+		spin_unlock_irqrestore(&dev->slock, flags);
+		dma_sync_single_for_cpu(&dev->pci->dev, frame->vlc.dma_addr,
+					H264_VLC_BUF_SIZE, DMA_FROM_DEVICE);
+		dma_sync_single_for_cpu(&dev->pci->dev, frame->mv.dma_addr,
+					H264_MV_BUF_SIZE, DMA_FROM_DEVICE);
+		tw5864_handle_frame(frame);
+		dma_sync_single_for_device(&dev->pci->dev, frame->vlc.dma_addr,
+					   H264_VLC_BUF_SIZE, DMA_FROM_DEVICE);
+		dma_sync_single_for_device(&dev->pci->dev, frame->mv.dma_addr,
+					   H264_MV_BUF_SIZE, DMA_FROM_DEVICE);
+		spin_lock_irqsave(&dev->slock, flags);
+
+		dev->h264_buf_r_index++;
+		dev->h264_buf_r_index %= H264_BUF_CNT;
+	}
+	spin_unlock_irqrestore(&dev->slock, flags);
+}
+
+#ifdef DEBUG
+static u32 tw5864_vlc_checksum(u32 *data, int len)
+{
+	u32 val, count_len = len;
+
+	val = *data++;
+	while (((count_len >> 2) - 1) > 0) {
+		val ^= *data++;
+		count_len -= 4;
+	}
+	val ^= htonl((len >> 2));
+	return val;
+}
+#endif
+
+static void tw5864_handle_frame(struct tw5864_h264_frame *frame)
+{
+#define SKIP_VLCBUF_BYTES 3
+	struct tw5864_input *input = frame->input;
+	struct tw5864_dev *dev = input->root;
+	struct tw5864_buf *vb;
+	struct vb2_v4l2_buffer *v4l2_buf;
+	int frame_len = frame->vlc_len - SKIP_VLCBUF_BYTES;
+	u8 *dst = input->buf_cur_ptr;
+	u8 tail_mask, vlc_mask = 0;
+	int i;
+	u8 vlc_first_byte = ((u8 *)(frame->vlc.addr + SKIP_VLCBUF_BYTES))[0];
+	unsigned long flags;
+	int zero_run;
+	u8 *src;
+	u8 *src_end;
+
+#ifdef DEBUG
+	if (frame->checksum !=
+	    tw5864_vlc_checksum((u32 *)frame->vlc.addr, frame_len))
+		dev_err(&dev->pci->dev,
+			"Checksum of encoded frame doesn't match!\n");
+#endif
+
+	spin_lock_irqsave(&input->slock, flags);
+	vb = input->vb;
+	input->vb = NULL;
+	spin_unlock_irqrestore(&input->slock, flags);
+
+	v4l2_buf = to_vb2_v4l2_buffer(&vb->vb.vb2_buf);
+
+	if (!vb) { /* Gone because of disabling */
+		dev_dbg(&dev->pci->dev, "vb is empty, dropping frame\n");
+		return;
+	}
+
+	/*
+	 * Check for space.
+	 * Mind the overhead of startcode emulation prevention.
+	 */
+	if (input->buf_cur_space_left < frame_len * 5 / 4) {
+		dev_err_once(&dev->pci->dev,
+			     "Left space in vb2 buffer, %d bytes, is less than considered safely enough to put frame of length %d. Dropping this frame.\n",
+			     input->buf_cur_space_left, frame_len);
+		return;
+	}
+
+	for (i = 0; i < 8 - input->tail_nb_bits; i++)
+		vlc_mask |= 1 << i;
+	tail_mask = (~vlc_mask) & 0xff;
+
+	dst[0] = (input->tail & tail_mask) | (vlc_first_byte & vlc_mask);
+	frame_len--;
+	dst++;
+
+	/* H.264 startcode emulation prevention */
+	src = frame->vlc.addr + SKIP_VLCBUF_BYTES + 1;
+	src_end = src + frame_len;
+	zero_run = 0;
+	for (; src < src_end; src++) {
+		if (zero_run < 2) {
+			if (*src == 0)
+				++zero_run;
+			else
+				zero_run = 0;
+		} else {
+			if ((*src & ~0x03) == 0)
+				*dst++ = 0x03;
+			zero_run = *src == 0;
+		}
+		*dst++ = *src;
+	}
+
+	vb2_set_plane_payload(&vb->vb.vb2_buf, 0,
+			      dst - (u8 *)vb2_plane_vaddr(&vb->vb.vb2_buf, 0));
+
+	vb->vb.vb2_buf.timestamp = frame->timestamp;
+	v4l2_buf->field = V4L2_FIELD_INTERLACED;
+	v4l2_buf->sequence = frame->seqno;
+
+	/* Check for motion flags */
+	if (frame->gop_seqno /* P-frame */ &&
+	    tw5864_is_motion_triggered(frame)) {
+		struct v4l2_event ev = {
+			.type = V4L2_EVENT_MOTION_DET,
+			.u.motion_det = {
+				.flags = V4L2_EVENT_MD_FL_HAVE_FRAME_SEQ,
+				.frame_sequence = v4l2_buf->sequence,
+			},
+		};
+
+		v4l2_event_queue(&input->vdev, &ev);
+	}
+
+	vb2_buffer_done(&vb->vb.vb2_buf, VB2_BUF_STATE_DONE);
+}
+
+static v4l2_std_id tw5864_get_v4l2_std(enum tw5864_vid_std std)
+{
+	switch (std) {
+	case STD_NTSC:    return V4L2_STD_NTSC_M;
+	case STD_PAL:     return V4L2_STD_PAL_B;
+	case STD_SECAM:   return V4L2_STD_SECAM_B;
+	case STD_NTSC443: return V4L2_STD_NTSC_443;
+	case STD_PAL_M:   return V4L2_STD_PAL_M;
+	case STD_PAL_CN:  return V4L2_STD_PAL_Nc;
+	case STD_PAL_60:  return V4L2_STD_PAL_60;
+	case STD_INVALID: return V4L2_STD_UNKNOWN;
+	}
+	return 0;
+}
+
+static enum tw5864_vid_std tw5864_from_v4l2_std(v4l2_std_id v4l2_std)
+{
+	if (v4l2_std & V4L2_STD_NTSC_M)
+		return STD_NTSC;
+	if (v4l2_std & V4L2_STD_PAL_B)
+		return STD_PAL;
+	if (v4l2_std & V4L2_STD_SECAM_B)
+		return STD_SECAM;
+	if (v4l2_std & V4L2_STD_NTSC_443)
+		return STD_NTSC443;
+	if (v4l2_std & V4L2_STD_PAL_M)
+		return STD_PAL_M;
+	if (v4l2_std & V4L2_STD_PAL_Nc)
+		return STD_PAL_CN;
+	if (v4l2_std & V4L2_STD_PAL_60)
+		return STD_PAL_60;
+
+	return STD_INVALID;
+}
+
+static void tw5864_encoder_tables_upload(struct tw5864_dev *dev)
+{
+	int i;
+
+	tw_writel(TW5864_VLC_RD, 0x1);
+	for (i = 0; i < VLC_LOOKUP_TABLE_LEN; i++) {
+		tw_writel((TW5864_VLC_STREAM_MEM_START + i * 4),
+			  encoder_vlc_lookup_table[i]);
+	}
+	tw_writel(TW5864_VLC_RD, 0x0);
+
+	for (i = 0; i < QUANTIZATION_TABLE_LEN; i++) {
+		tw_writel((TW5864_QUAN_TAB + i * 4),
+			  forward_quantization_table[i]);
+	}
+
+	for (i = 0; i < QUANTIZATION_TABLE_LEN; i++) {
+		tw_writel((TW5864_QUAN_TAB + i * 4),
+			  inverse_quantization_table[i]);
+	}
+}
diff --git a/drivers/media/pci/tw5864/tw5864.h b/drivers/media/pci/tw5864/tw5864.h
new file mode 100644
index 000000000000..f5de9f6ef119
--- /dev/null
+++ b/drivers/media/pci/tw5864/tw5864.h
@@ -0,0 +1,205 @@
+/*
+ *  TW5864 driver  - common header file
+ *
+ *  Copyright (C) 2016 Bluecherry, LLC <maintainers@bluecherrydvr.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ */
+
+#include <linux/pci.h>
+#include <linux/videodev2.h>
+#include <linux/notifier.h>
+#include <linux/delay.h>
+#include <linux/mutex.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+
+#include <media/v4l2-common.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/videobuf2-dma-sg.h>
+
+#include "tw5864-reg.h"
+
+#define PCI_DEVICE_ID_TECHWELL_5864 0x5864
+
+#define TW5864_NORMS V4L2_STD_ALL
+
+/* ----------------------------------------------------------- */
+/* card configuration   */
+
+#define TW5864_INPUTS 4
+
+/* The TW5864 uses 192 (16x12) detection cells in full screen for motion
+ * detection. Each detection cell is composed of 44 pixels and 20 lines for
+ * NTSC and 24 lines for PAL.
+ */
+#define MD_CELLS_HOR 16
+#define MD_CELLS_VERT 12
+#define MD_CELLS (MD_CELLS_HOR * MD_CELLS_VERT)
+
+#define H264_VLC_BUF_SIZE 0x80000
+#define H264_MV_BUF_SIZE 0x2000 /* device writes 5396 bytes */
+#define QP_VALUE 28
+#define MAX_GOP_SIZE 255
+#define GOP_SIZE MAX_GOP_SIZE
+
+enum resolution {
+	D1 = 1,
+	HD1 = 2, /* half d1 - 360x(240|288) */
+	CIF = 3,
+	QCIF = 4,
+};
+
+/* ----------------------------------------------------------- */
+/* device / file handle status                                 */
+
+struct tw5864_dev; /* forward delclaration */
+
+/* buffer for one video/vbi/ts frame */
+struct tw5864_buf {
+	struct vb2_v4l2_buffer vb;
+	struct list_head list;
+
+	unsigned int size;
+};
+
+struct tw5864_dma_buf {
+	void *addr;
+	dma_addr_t dma_addr;
+};
+
+enum tw5864_vid_std {
+	STD_NTSC = 0, /* NTSC (M) */
+	STD_PAL = 1, /* PAL (B, D, G, H, I) */
+	STD_SECAM = 2, /* SECAM */
+	STD_NTSC443 = 3, /* NTSC4.43 */
+	STD_PAL_M = 4, /* PAL (M) */
+	STD_PAL_CN = 5, /* PAL (CN) */
+	STD_PAL_60 = 6, /* PAL 60 */
+	STD_INVALID = 7,
+	STD_AUTO = 7,
+};
+
+struct tw5864_input {
+	int nr; /* input number */
+	struct tw5864_dev *root;
+	struct mutex lock; /* used for vidq and vdev */
+	spinlock_t slock; /* used for sync between ISR, tasklet & V4L2 API */
+	struct video_device vdev;
+	struct v4l2_ctrl_handler hdl;
+	struct vb2_queue vidq;
+	struct list_head active;
+	enum resolution resolution;
+	unsigned int width, height;
+	unsigned int frame_seqno;
+	unsigned int frame_gop_seqno;
+	unsigned int h264_idr_pic_id;
+	int enabled;
+	enum tw5864_vid_std std;
+	v4l2_std_id v4l2_std;
+	int tail_nb_bits;
+	u8 tail;
+	u8 *buf_cur_ptr;
+	int buf_cur_space_left;
+
+	u32 reg_interlacing;
+	u32 reg_vlc;
+	u32 reg_dsp_codec;
+	u32 reg_dsp;
+	u32 reg_emu;
+	u32 reg_dsp_qp;
+	u32 reg_dsp_ref_mvp_lambda;
+	u32 reg_dsp_i4x4_weight;
+	u32 buf_id;
+
+	struct tw5864_buf *vb;
+
+	struct v4l2_ctrl *md_threshold_grid_ctrl;
+	u16 md_threshold_grid_values[12 * 16];
+	int qp;
+	int gop;
+
+	/*
+	 * In (1/MAX_FPS) units.
+	 * For max FPS (default), set to 1.
+	 * For 1 FPS, set to e.g. 32.
+	 */
+	int frame_interval;
+	unsigned long new_frame_deadline;
+};
+
+struct tw5864_h264_frame {
+	struct tw5864_dma_buf vlc;
+	struct tw5864_dma_buf mv;
+	int vlc_len;
+	u32 checksum;
+	struct tw5864_input *input;
+	u64 timestamp;
+	unsigned int seqno;
+	unsigned int gop_seqno;
+};
+
+/* global device status */
+struct tw5864_dev {
+	spinlock_t slock; /* used for sync between ISR, tasklet & V4L2 API */
+	struct v4l2_device v4l2_dev;
+	struct tw5864_input inputs[TW5864_INPUTS];
+#define H264_BUF_CNT 4
+	struct tw5864_h264_frame h264_buf[H264_BUF_CNT];
+	int h264_buf_r_index;
+	int h264_buf_w_index;
+
+	struct tasklet_struct tasklet;
+
+	int encoder_busy;
+	/* Input number to check next for ready raw picture (in RR fashion) */
+	int next_input;
+
+	/* pci i/o */
+	char name[64];
+	struct pci_dev *pci;
+	void __iomem *mmio;
+	u32 irqmask;
+};
+
+#define tw_readl(reg) readl(dev->mmio + reg)
+#define tw_mask_readl(reg, mask) \
+	(tw_readl(reg) & (mask))
+#define tw_mask_shift_readl(reg, mask, shift) \
+	(tw_mask_readl((reg), ((mask) << (shift))) >> (shift))
+
+#define tw_writel(reg, value) writel((value), dev->mmio + reg)
+#define tw_mask_writel(reg, mask, value) \
+	tw_writel(reg, (tw_readl(reg) & ~(mask)) | ((value) & (mask)))
+#define tw_mask_shift_writel(reg, mask, shift, value) \
+	tw_mask_writel((reg), ((mask) << (shift)), ((value) << (shift)))
+
+#define tw_setl(reg, bit) tw_writel((reg), tw_readl(reg) | (bit))
+#define tw_clearl(reg, bit) tw_writel((reg), tw_readl(reg) & ~(bit))
+
+u8 tw5864_indir_readb(struct tw5864_dev *dev, u16 addr);
+#define tw_indir_readb(addr) tw5864_indir_readb(dev, addr)
+void tw5864_indir_writeb(struct tw5864_dev *dev, u16 addr, u8 data);
+#define tw_indir_writeb(addr, data) tw5864_indir_writeb(dev, addr, data)
+
+void tw5864_irqmask_apply(struct tw5864_dev *dev);
+int tw5864_video_init(struct tw5864_dev *dev, int *video_nr);
+void tw5864_video_fini(struct tw5864_dev *dev);
+void tw5864_prepare_frame_headers(struct tw5864_input *input);
+void tw5864_h264_put_stream_header(u8 **buf, size_t *space_left, int qp,
+				   int width, int height);
+void tw5864_h264_put_slice_header(u8 **buf, size_t *space_left,
+				  unsigned int idr_pic_id,
+				  unsigned int frame_gop_seqno,
+				  int *tail_nb_bits, u8 *tail);
+void tw5864_request_encoded_frame(struct tw5864_input *input);
diff --git a/drivers/media/pci/tw68/tw68-video.c b/drivers/media/pci/tw68/tw68-video.c
index 5e8212845c87..a45e02367321 100644
--- a/drivers/media/pci/tw68/tw68-video.c
+++ b/drivers/media/pci/tw68/tw68-video.c
@@ -535,7 +535,7 @@ static void tw68_stop_streaming(struct vb2_queue *q)
 	}
 }
 
-static struct vb2_ops tw68_video_qops = {
+static const struct vb2_ops tw68_video_qops = {
 	.queue_setup	= tw68_queue_setup,
 	.buf_queue	= tw68_buf_queue,
 	.buf_prepare	= tw68_buf_prepare,
diff --git a/drivers/media/pci/tw686x/tw686x-audio.c b/drivers/media/pci/tw686x/tw686x-audio.c
index 96e444c49173..77190768622a 100644
--- a/drivers/media/pci/tw686x/tw686x-audio.c
+++ b/drivers/media/pci/tw686x/tw686x-audio.c
@@ -269,7 +269,7 @@ static snd_pcm_uframes_t tw686x_pcm_pointer(struct snd_pcm_substream *ss)
 	return bytes_to_frames(ss->runtime, ac->ptr);
 }
 
-static struct snd_pcm_ops tw686x_pcm_ops = {
+static const struct snd_pcm_ops tw686x_pcm_ops = {
 	.open = tw686x_pcm_open,
 	.close = tw686x_pcm_close,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/media/pci/tw686x/tw686x-video.c b/drivers/media/pci/tw686x/tw686x-video.c
index cdb16de770fe..c3fafa97b2d0 100644
--- a/drivers/media/pci/tw686x/tw686x-video.c
+++ b/drivers/media/pci/tw686x/tw686x-video.c
@@ -577,7 +577,7 @@ static int tw686x_buf_prepare(struct vb2_buffer *vb)
 	return 0;
 }
 
-static struct vb2_ops tw686x_video_qops = {
+static const struct vb2_ops tw686x_video_qops = {
 	.queue_setup		= tw686x_queue_setup,
 	.buf_queue		= tw686x_buf_queue,
 	.buf_prepare		= tw686x_buf_prepare,
@@ -672,30 +672,20 @@ static int tw686x_try_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
-static int tw686x_s_fmt_vid_cap(struct file *file, void *priv,
-				struct v4l2_format *f)
+static int tw686x_set_format(struct tw686x_video_channel *vc,
+			     unsigned int pixelformat, unsigned int width,
+			     unsigned int height, bool realloc)
 {
-	struct tw686x_video_channel *vc = video_drvdata(file);
 	struct tw686x_dev *dev = vc->dev;
-	u32 val, width, line_width, height;
-	unsigned long bitsperframe;
+	u32 val, dma_width, dma_height, dma_line_width;
 	int err, pb;
 
-	if (vb2_is_busy(&vc->vidq))
-		return -EBUSY;
-
-	bitsperframe = vc->width * vc->height * vc->format->depth;
-	err = tw686x_try_fmt_vid_cap(file, priv, f);
-	if (err)
-		return err;
-
-	vc->format = format_by_fourcc(f->fmt.pix.pixelformat);
-	vc->width = f->fmt.pix.width;
-	vc->height = f->fmt.pix.height;
+	vc->format = format_by_fourcc(pixelformat);
+	vc->width = width;
+	vc->height = height;
 
 	/* We need new DMA buffers if the framesize has changed */
-	if (dev->dma_ops->alloc &&
-	    bitsperframe != vc->width * vc->height * vc->format->depth) {
+	if (dev->dma_ops->alloc && realloc) {
 		for (pb = 0; pb < 2; pb++)
 			dev->dma_ops->free(vc, pb);
 
@@ -739,14 +729,36 @@ static int tw686x_s_fmt_vid_cap(struct file *file, void *priv,
 	reg_write(vc->dev, VDMA_CHANNEL_CONFIG[vc->ch], val);
 
 	/* Program the DMA frame size */
-	width = (vc->width * 2) & 0x7ff;
-	height = vc->height / 2;
-	line_width = (vc->width * 2) & 0x7ff;
-	val = (height << 22) | (line_width << 11)  | width;
+	dma_width = (vc->width * 2) & 0x7ff;
+	dma_height = vc->height / 2;
+	dma_line_width = (vc->width * 2) & 0x7ff;
+	val = (dma_height << 22) | (dma_line_width << 11)  | dma_width;
 	reg_write(vc->dev, VDMA_WHP[vc->ch], val);
 	return 0;
 }
 
+static int tw686x_s_fmt_vid_cap(struct file *file, void *priv,
+				struct v4l2_format *f)
+{
+	struct tw686x_video_channel *vc = video_drvdata(file);
+	unsigned long area;
+	bool realloc;
+	int err;
+
+	if (vb2_is_busy(&vc->vidq))
+		return -EBUSY;
+
+	area = vc->width * vc->height;
+	err = tw686x_try_fmt_vid_cap(file, priv, f);
+	if (err)
+		return err;
+
+	realloc = area != (f->fmt.pix.width * f->fmt.pix.height);
+	return tw686x_set_format(vc, f->fmt.pix.pixelformat,
+				 f->fmt.pix.width, f->fmt.pix.height,
+				 realloc);
+}
+
 static int tw686x_querycap(struct file *file, void *priv,
 			   struct v4l2_capability *cap)
 {
@@ -763,17 +775,9 @@ static int tw686x_querycap(struct file *file, void *priv,
 	return 0;
 }
 
-static int tw686x_s_std(struct file *file, void *priv, v4l2_std_id id)
+static int tw686x_set_standard(struct tw686x_video_channel *vc, v4l2_std_id id)
 {
-	struct tw686x_video_channel *vc = video_drvdata(file);
-	struct v4l2_format f;
-	u32 val, ret;
-
-	if (vc->video_standard == id)
-		return 0;
-
-	if (vb2_is_busy(&vc->vidq))
-		return -EBUSY;
+	u32 val;
 
 	if (id & V4L2_STD_NTSC)
 		val = 0;
@@ -802,14 +806,31 @@ static int tw686x_s_std(struct file *file, void *priv, v4l2_std_id id)
 		val |= (1 << (SYS_MODE_DMA_SHIFT + vc->ch));
 	reg_write(vc->dev, VIDEO_CONTROL1, val);
 
+	return 0;
+}
+
+static int tw686x_s_std(struct file *file, void *priv, v4l2_std_id id)
+{
+	struct tw686x_video_channel *vc = video_drvdata(file);
+	struct v4l2_format f;
+	int ret;
+
+	if (vc->video_standard == id)
+		return 0;
+
+	if (vb2_is_busy(&vc->vidq))
+		return -EBUSY;
+
+	ret = tw686x_set_standard(vc, id);
+	if (ret)
+		return ret;
 	/*
 	 * Adjust format after V4L2_STD_525_60/V4L2_STD_625_50 change,
 	 * calling g_fmt and s_fmt will sanitize the height
 	 * according to the standard.
 	 */
-	ret = tw686x_g_fmt_vid_cap(file, priv, &f);
-	if (!ret)
-		tw686x_s_fmt_vid_cap(file, priv, &f);
+	tw686x_g_fmt_vid_cap(file, priv, &f);
+	tw686x_s_fmt_vid_cap(file, priv, &f);
 
 	/*
 	 * Frame decimation depends on the chosen standard,
@@ -885,6 +906,42 @@ static int tw686x_g_std(struct file *file, void *priv, v4l2_std_id *id)
 	return 0;
 }
 
+static int tw686x_enum_framesizes(struct file *file, void *priv,
+				  struct v4l2_frmsizeenum *fsize)
+{
+	struct tw686x_video_channel *vc = video_drvdata(file);
+
+	if (fsize->index)
+		return -EINVAL;
+	fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
+	fsize->stepwise.max_width = TW686X_VIDEO_WIDTH;
+	fsize->stepwise.min_width = fsize->stepwise.max_width / 2;
+	fsize->stepwise.step_width = fsize->stepwise.min_width;
+	fsize->stepwise.max_height = TW686X_VIDEO_HEIGHT(vc->video_standard);
+	fsize->stepwise.min_height = fsize->stepwise.max_height / 2;
+	fsize->stepwise.step_height = fsize->stepwise.min_height;
+	return 0;
+}
+
+static int tw686x_enum_frameintervals(struct file *file, void *priv,
+				      struct v4l2_frmivalenum *ival)
+{
+	struct tw686x_video_channel *vc = video_drvdata(file);
+	int max_fps = TW686X_MAX_FPS(vc->video_standard);
+	int max_rates = DIV_ROUND_UP(max_fps, 2);
+
+	if (ival->index >= max_rates)
+		return -EINVAL;
+
+	ival->type = V4L2_FRMIVAL_TYPE_DISCRETE;
+	ival->discrete.numerator = 1;
+	if (ival->index < (max_rates - 1))
+		ival->discrete.denominator = (ival->index + 1) * 2;
+	else
+		ival->discrete.denominator = max_fps;
+	return 0;
+}
+
 static int tw686x_g_parm(struct file *file, void *priv,
 			 struct v4l2_streamparm *sp)
 {
@@ -928,10 +985,21 @@ static int tw686x_enum_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
+static void tw686x_set_input(struct tw686x_video_channel *vc, unsigned int i)
+{
+	u32 val;
+
+	vc->input = i;
+
+	val = reg_read(vc->dev, VDMA_CHANNEL_CONFIG[vc->ch]);
+	val &= ~(0x3 << 30);
+	val |= i << 30;
+	reg_write(vc->dev, VDMA_CHANNEL_CONFIG[vc->ch], val);
+}
+
 static int tw686x_s_input(struct file *file, void *priv, unsigned int i)
 {
 	struct tw686x_video_channel *vc = video_drvdata(file);
-	u32 val;
 
 	if (i >= TW686X_INPUTS_PER_CH)
 		return -EINVAL;
@@ -943,12 +1011,7 @@ static int tw686x_s_input(struct file *file, void *priv, unsigned int i)
 	if (vb2_is_busy(&vc->vidq))
 		return -EBUSY;
 
-	vc->input = i;
-
-	val = reg_read(vc->dev, VDMA_CHANNEL_CONFIG[vc->ch]);
-	val &= ~(0x3 << 30);
-	val |= i << 30;
-	reg_write(vc->dev, VDMA_CHANNEL_CONFIG[vc->ch], val);
+	tw686x_set_input(vc, i);
 	return 0;
 }
 
@@ -1007,6 +1070,8 @@ static const struct v4l2_ioctl_ops tw686x_video_ioctl_ops = {
 
 	.vidioc_g_parm			= tw686x_g_parm,
 	.vidioc_s_parm			= tw686x_s_parm,
+	.vidioc_enum_framesizes		= tw686x_enum_framesizes,
+	.vidioc_enum_frameintervals	= tw686x_enum_frameintervals,
 
 	.vidioc_enum_input		= tw686x_enum_input,
 	.vidioc_g_input			= tw686x_g_input,
@@ -1093,8 +1158,7 @@ void tw686x_video_free(struct tw686x_dev *dev)
 	for (ch = 0; ch < max_channels(dev); ch++) {
 		struct tw686x_video_channel *vc = &dev->video_channels[ch];
 
-		if (vc->device)
-			video_unregister_device(vc->device);
+		video_unregister_device(vc->device);
 
 		if (dev->dma_ops->free)
 			for (pb = 0; pb < 2; pb++)
@@ -1104,7 +1168,7 @@ void tw686x_video_free(struct tw686x_dev *dev)
 
 int tw686x_video_init(struct tw686x_dev *dev)
 {
-	unsigned int ch, val, pb;
+	unsigned int ch, val;
 	int err;
 
 	if (dev->dma_mode == TW686X_DMA_MODE_MEMCPY)
@@ -1138,27 +1202,23 @@ int tw686x_video_init(struct tw686x_dev *dev)
 		vc->ch = ch;
 
 		/* default settings */
-		vc->format = &formats[0];
-		vc->video_standard = V4L2_STD_NTSC;
-		vc->width = TW686X_VIDEO_WIDTH;
-		vc->height = TW686X_VIDEO_HEIGHT(vc->video_standard);
-		vc->input = 0;
+		err = tw686x_set_standard(vc, V4L2_STD_NTSC);
+		if (err)
+			goto error;
 
-		reg_write(vc->dev, SDT[ch], 0);
-		tw686x_set_framerate(vc, 30);
+		err = tw686x_set_format(vc, formats[0].fourcc,
+				TW686X_VIDEO_WIDTH,
+				TW686X_VIDEO_HEIGHT(vc->video_standard),
+				true);
+		if (err)
+			goto error;
 
+		tw686x_set_input(vc, 0);
+		tw686x_set_framerate(vc, 30);
 		reg_write(dev, VDELAY_LO[ch], 0x14);
 		reg_write(dev, HACTIVE_LO[ch], 0xd0);
 		reg_write(dev, VIDEO_SIZE[ch], 0);
 
-		if (dev->dma_ops->alloc) {
-			for (pb = 0; pb < 2; pb++) {
-				err = dev->dma_ops->alloc(vc, pb);
-				if (err)
-					goto error;
-			}
-		}
-
 		vc->vidq.io_modes = VB2_READ | VB2_MMAP | VB2_DMABUF;
 		vc->vidq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 		vc->vidq.drv_priv = vc;
diff --git a/drivers/media/pci/zoran/zoran_driver.c b/drivers/media/pci/zoran/zoran_driver.c
index 80caa70c6360..d6b631add216 100644
--- a/drivers/media/pci/zoran/zoran_driver.c
+++ b/drivers/media/pci/zoran/zoran_driver.c
@@ -2365,94 +2365,80 @@ static int zoran_s_output(struct file *file, void *__fh, unsigned int output)
 }
 
 /* cropping (sub-frame capture) */
-static int zoran_cropcap(struct file *file, void *__fh,
-					struct v4l2_cropcap *cropcap)
+static int zoran_g_selection(struct file *file, void *__fh, struct v4l2_selection *sel)
 {
 	struct zoran_fh *fh = __fh;
 	struct zoran *zr = fh->zr;
-	int type = cropcap->type, res = 0;
 
-	memset(cropcap, 0, sizeof(*cropcap));
-	cropcap->type = type;
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
+	    sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
 
-	if (cropcap->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
-	    (cropcap->type != V4L2_BUF_TYPE_VIDEO_CAPTURE ||
-	     fh->map_mode == ZORAN_MAP_MODE_RAW)) {
+	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
 		dprintk(1, KERN_ERR
-			"%s: VIDIOC_CROPCAP - subcapture only supported for compressed capture\n",
+			"%s: VIDIOC_G_SELECTION - subcapture only supported for compressed capture\n",
 			ZR_DEVNAME(zr));
-		res = -EINVAL;
-		return res;
+		return -EINVAL;
 	}
 
-	cropcap->bounds.top = cropcap->bounds.left = 0;
-	cropcap->bounds.width = BUZ_MAX_WIDTH;
-	cropcap->bounds.height = BUZ_MAX_HEIGHT;
-	cropcap->defrect.top = cropcap->defrect.left = 0;
-	cropcap->defrect.width = BUZ_MIN_WIDTH;
-	cropcap->defrect.height = BUZ_MIN_HEIGHT;
-	return res;
-}
-
-static int zoran_g_crop(struct file *file, void *__fh, struct v4l2_crop *crop)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int type = crop->type, res = 0;
-
-	memset(crop, 0, sizeof(*crop));
-	crop->type = type;
-
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
-	    (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE ||
-	     fh->map_mode == ZORAN_MAP_MODE_RAW)) {
-		dprintk(1,
-			KERN_ERR
-			"%s: VIDIOC_G_CROP - subcapture only supported for compressed capture\n",
-			ZR_DEVNAME(zr));
-		res = -EINVAL;
-		return res;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		sel->r.top = fh->jpg_settings.img_y;
+		sel->r.left = fh->jpg_settings.img_x;
+		sel->r.width = fh->jpg_settings.img_width;
+		sel->r.height = fh->jpg_settings.img_height;
+		break;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r.top = sel->r.left = 0;
+		sel->r.width = BUZ_MIN_WIDTH;
+		sel->r.height = BUZ_MIN_HEIGHT;
+		break;
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r.top = sel->r.left = 0;
+		sel->r.width = BUZ_MAX_WIDTH;
+		sel->r.height = BUZ_MAX_HEIGHT;
+		break;
+	default:
+		return -EINVAL;
 	}
-
-	crop->c.top = fh->jpg_settings.img_y;
-	crop->c.left = fh->jpg_settings.img_x;
-	crop->c.width = fh->jpg_settings.img_width;
-	crop->c.height = fh->jpg_settings.img_height;
-	return res;
+	return 0;
 }
 
-static int zoran_s_crop(struct file *file, void *__fh, const struct v4l2_crop *crop)
+static int zoran_s_selection(struct file *file, void *__fh, struct v4l2_selection *sel)
 {
 	struct zoran_fh *fh = __fh;
 	struct zoran *zr = fh->zr;
-	int res = 0;
 	struct zoran_jpg_settings settings;
+	int res;
 
-	settings = fh->jpg_settings;
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
+	    sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
 
-	if (fh->buffers.allocated) {
+	if (sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
+
+	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
 		dprintk(1, KERN_ERR
-			"%s: VIDIOC_S_CROP - cannot change settings while active\n",
+			"%s: VIDIOC_S_SELECTION - subcapture only supported for compressed capture\n",
 			ZR_DEVNAME(zr));
-		res = -EBUSY;
-		return res;
+		return -EINVAL;
 	}
 
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
-	    (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE ||
-	     fh->map_mode == ZORAN_MAP_MODE_RAW)) {
+	settings = fh->jpg_settings;
+
+	if (fh->buffers.allocated) {
 		dprintk(1, KERN_ERR
-			"%s: VIDIOC_G_CROP - subcapture only supported for compressed capture\n",
+			"%s: VIDIOC_S_SELECTION - cannot change settings while active\n",
 			ZR_DEVNAME(zr));
-		res = -EINVAL;
-		return res;
+		return -EBUSY;
 	}
 
 	/* move into a form that we understand */
-	settings.img_x = crop->c.left;
-	settings.img_y = crop->c.top;
-	settings.img_width = crop->c.width;
-	settings.img_height = crop->c.height;
+	settings.img_x = sel->r.left;
+	settings.img_y = sel->r.top;
+	settings.img_width = sel->r.width;
+	settings.img_height = sel->r.height;
 
 	/* check validity */
 	res = zoran_check_jpg_settings(zr, &settings, 0);
@@ -2808,9 +2794,8 @@ zoran_mmap (struct file           *file,
 
 static const struct v4l2_ioctl_ops zoran_ioctl_ops = {
 	.vidioc_querycap    		    = zoran_querycap,
-	.vidioc_cropcap       		    = zoran_cropcap,
-	.vidioc_s_crop       		    = zoran_s_crop,
-	.vidioc_g_crop       		    = zoran_g_crop,
+	.vidioc_s_selection		    = zoran_s_selection,
+	.vidioc_g_selection		    = zoran_g_selection,
 	.vidioc_enum_input     		    = zoran_enum_input,
 	.vidioc_g_input      		    = zoran_g_input,
 	.vidioc_s_input      		    = zoran_s_input,
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index f25344bc7912..ce4a96fccc43 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -91,6 +91,15 @@ config VIDEO_OMAP3_DEBUG
 	---help---
 	  Enable debug messages on OMAP 3 camera controller driver.
 
+config VIDEO_PXA27x
+	tristate "PXA27x Quick Capture Interface driver"
+	depends on VIDEO_DEV && HAS_DMA
+	depends on PXA27x || COMPILE_TEST
+	select VIDEOBUF2_DMA_SG
+	select SG_SPLIT
+	---help---
+	  This is a v4l2 driver for the PXA27x Quick Capture Interface
+
 config VIDEO_S3C_CAMIF
 	tristate "Samsung S3C24XX/S3C64XX SoC Camera Interface driver"
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
@@ -107,10 +116,10 @@ config VIDEO_S3C_CAMIF
 
 source "drivers/media/platform/soc_camera/Kconfig"
 source "drivers/media/platform/exynos4-is/Kconfig"
-source "drivers/media/platform/s5p-tv/Kconfig"
 source "drivers/media/platform/am437x/Kconfig"
 source "drivers/media/platform/xilinx/Kconfig"
 source "drivers/media/platform/rcar-vin/Kconfig"
+source "drivers/media/platform/atmel/Kconfig"
 
 config VIDEO_TI_CAL
 	tristate "TI CAL (Camera Adaptation Layer) driver"
@@ -155,7 +164,7 @@ config VIDEO_CODA
 
 config VIDEO_MEDIATEK_VPU
 	tristate "Mediatek Video Processor Unit"
-	depends on VIDEO_DEV && VIDEO_V4L2
+	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	---help---
 	    This driver provides downloading VPU firmware and
@@ -169,7 +178,7 @@ config VIDEO_MEDIATEK_VPU
 config VIDEO_MEDIATEK_VCODEC
 	tristate "Mediatek Video Codec driver"
 	depends on MTK_IOMMU || COMPILE_TEST
-	depends on VIDEO_DEV && VIDEO_V4L2
+	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
@@ -257,6 +266,21 @@ config VIDEO_STI_BDISP
 	help
 	  This v4l2 mem2mem driver is a 2D blitter for STMicroelectronics SoC.
 
+config VIDEO_STI_HVA
+	tristate "STMicroelectronics HVA multi-format video encoder V4L2 driver"
+	depends on VIDEO_DEV && VIDEO_V4L2
+	depends on HAS_DMA
+	depends on ARCH_STI || COMPILE_TEST
+	select VIDEOBUF2_DMA_CONTIG
+	select V4L2_MEM2MEM_DEV
+	help
+	  This V4L2 driver enables HVA (Hardware Video Accelerator) multi-format
+	  video encoder of STMicroelectronics SoC, allowing hardware encoding of
+	  raw uncompressed formats in various compressed video bitstreams format.
+
+	  To compile this driver as a module, choose M here:
+	  the module will be called st-hva.
+
 config VIDEO_SH_VEU
 	tristate "SuperH VEU mem2mem video processing driver"
 	depends on VIDEO_DEV && VIDEO_V4L2 && HAS_DMA
diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile
index 21771c1a13fb..40b18d12726e 100644
--- a/drivers/media/platform/Makefile
+++ b/drivers/media/platform/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_VIDEO_CAFE_CCIC) += marvell-ccic/
 obj-$(CONFIG_VIDEO_MMP_CAMERA) += marvell-ccic/
 
 obj-$(CONFIG_VIDEO_OMAP3)	+= omap3isp/
+obj-$(CONFIG_VIDEO_PXA27x)	+= pxa_camera.o
 
 obj-$(CONFIG_VIDEO_VIU) += fsl-viu.o
 
@@ -30,12 +31,12 @@ obj-$(CONFIG_VIDEO_S3C_CAMIF) 		+= s3c-camif/
 obj-$(CONFIG_VIDEO_SAMSUNG_EXYNOS4_IS) 	+= exynos4-is/
 obj-$(CONFIG_VIDEO_SAMSUNG_S5P_JPEG)	+= s5p-jpeg/
 obj-$(CONFIG_VIDEO_SAMSUNG_S5P_MFC)	+= s5p-mfc/
-obj-$(CONFIG_VIDEO_SAMSUNG_S5P_TV)	+= s5p-tv/
 
 obj-$(CONFIG_VIDEO_SAMSUNG_S5P_G2D)	+= s5p-g2d/
 obj-$(CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC)	+= exynos-gsc/
 
 obj-$(CONFIG_VIDEO_STI_BDISP)		+= sti/bdisp/
+obj-$(CONFIG_VIDEO_STI_HVA)		+= sti/hva/
 obj-$(CONFIG_DVB_C8SECTPFE)		+= sti/c8sectpfe/
 
 obj-$(CONFIG_BLACKFIN)                  += blackfin/
@@ -58,6 +59,8 @@ obj-$(CONFIG_VIDEO_XILINX)		+= xilinx/
 
 obj-$(CONFIG_VIDEO_RCAR_VIN)		+= rcar-vin/
 
+obj-$(CONFIG_VIDEO_ATMEL_ISC)		+= atmel/
+
 ccflags-y += -I$(srctree)/drivers/media/i2c
 
 obj-$(CONFIG_VIDEO_MEDIATEK_VPU)	+= mtk-vpu/
diff --git a/drivers/media/platform/atmel/Kconfig b/drivers/media/platform/atmel/Kconfig
new file mode 100644
index 000000000000..867dca22a473
--- /dev/null
+++ b/drivers/media/platform/atmel/Kconfig
@@ -0,0 +1,9 @@
+config VIDEO_ATMEL_ISC
+	tristate "ATMEL Image Sensor Controller (ISC) support"
+	depends on VIDEO_V4L2 && COMMON_CLK && VIDEO_V4L2_SUBDEV_API && HAS_DMA
+	depends on ARCH_AT91 || COMPILE_TEST
+	select VIDEOBUF2_DMA_CONTIG
+	select REGMAP_MMIO
+	help
+	   This module makes the ATMEL Image Sensor Controller available
+	   as a v4l2 device.
\ No newline at end of file
diff --git a/drivers/media/platform/atmel/Makefile b/drivers/media/platform/atmel/Makefile
new file mode 100644
index 000000000000..9d7c999d434d
--- /dev/null
+++ b/drivers/media/platform/atmel/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_VIDEO_ATMEL_ISC) += atmel-isc.o
diff --git a/drivers/media/platform/atmel/atmel-isc-regs.h b/drivers/media/platform/atmel/atmel-isc-regs.h
new file mode 100644
index 000000000000..00c449717cde
--- /dev/null
+++ b/drivers/media/platform/atmel/atmel-isc-regs.h
@@ -0,0 +1,165 @@
+#ifndef __ATMEL_ISC_REGS_H
+#define __ATMEL_ISC_REGS_H
+
+#include <linux/bitops.h>
+
+/* ISC Control Enable Register 0 */
+#define ISC_CTRLEN      0x00000000
+
+/* ISC Control Disable Register 0 */
+#define ISC_CTRLDIS     0x00000004
+
+/* ISC Control Status Register 0 */
+#define ISC_CTRLSR      0x00000008
+
+#define ISC_CTRL_CAPTURE	BIT(0)
+#define ISC_CTRL_UPPRO		BIT(1)
+#define ISC_CTRL_HISREQ		BIT(2)
+#define ISC_CTRL_HISCLR		BIT(3)
+
+/* ISC Parallel Front End Configuration 0 Register */
+#define ISC_PFE_CFG0    0x0000000c
+
+#define ISC_PFE_CFG0_HPOL_LOW   BIT(0)
+#define ISC_PFE_CFG0_VPOL_LOW   BIT(1)
+#define ISC_PFE_CFG0_PPOL_LOW   BIT(2)
+
+#define ISC_PFE_CFG0_MODE_PROGRESSIVE   (0x0 << 4)
+#define ISC_PFE_CFG0_MODE_MASK          GENMASK(6, 4)
+
+#define ISC_PFE_CFG0_BPS_EIGHT  (0x4 << 28)
+#define ISC_PFG_CFG0_BPS_NINE   (0x3 << 28)
+#define ISC_PFG_CFG0_BPS_TEN    (0x2 << 28)
+#define ISC_PFG_CFG0_BPS_ELEVEN (0x1 << 28)
+#define ISC_PFG_CFG0_BPS_TWELVE (0x0 << 28)
+#define ISC_PFE_CFG0_BPS_MASK   GENMASK(30, 28)
+
+/* ISC Clock Enable Register */
+#define ISC_CLKEN               0x00000018
+
+/* ISC Clock Disable Register */
+#define ISC_CLKDIS              0x0000001c
+
+/* ISC Clock Status Register */
+#define ISC_CLKSR               0x00000020
+
+#define ISC_CLK(n)		BIT(n)
+
+/* ISC Clock Configuration Register */
+#define ISC_CLKCFG              0x00000024
+#define ISC_CLKCFG_DIV_SHIFT(n) ((n)*16)
+#define ISC_CLKCFG_DIV_MASK(n)  GENMASK(((n)*16 + 7), (n)*16)
+#define ISC_CLKCFG_SEL_SHIFT(n) ((n)*16 + 8)
+#define ISC_CLKCFG_SEL_MASK(n)  GENMASK(((n)*17 + 8), ((n)*16 + 8))
+
+/* ISC Interrupt Enable Register */
+#define ISC_INTEN       0x00000028
+
+/* ISC Interrupt Disable Register */
+#define ISC_INTDIS      0x0000002c
+
+/* ISC Interrupt Mask Register */
+#define ISC_INTMASK     0x00000030
+
+/* ISC Interrupt Status Register */
+#define ISC_INTSR       0x00000034
+
+#define ISC_INT_DDONE		BIT(8)
+
+/* ISC White Balance Control Register */
+#define ISC_WB_CTRL     0x00000058
+
+/* ISC White Balance Configuration Register */
+#define ISC_WB_CFG      0x0000005c
+
+/* ISC Color Filter Array Control Register */
+#define ISC_CFA_CTRL    0x00000070
+
+/* ISC Color Filter Array Configuration Register */
+#define ISC_CFA_CFG     0x00000074
+
+#define ISC_BAY_CFG_GRGR	0x0
+#define ISC_BAY_CFG_RGRG	0x1
+#define ISC_BAY_CFG_GBGB	0x2
+#define ISC_BAY_CFG_BGBG	0x3
+#define ISC_BAY_CFG_MASK	GENMASK(1, 0)
+
+/* ISC Color Correction Control Register */
+#define ISC_CC_CTRL     0x00000078
+
+/* ISC Gamma Correction Control Register */
+#define ISC_GAM_CTRL    0x00000094
+
+/* Color Space Conversion Control Register */
+#define ISC_CSC_CTRL    0x00000398
+
+/* Contrast And Brightness Control Register */
+#define ISC_CBC_CTRL    0x000003b4
+
+/* Subsampling 4:4:4 to 4:2:2 Control Register */
+#define ISC_SUB422_CTRL 0x000003c4
+
+/* Subsampling 4:2:2 to 4:2:0 Control Register */
+#define ISC_SUB420_CTRL 0x000003cc
+
+/* Rounding, Limiting and Packing Configuration Register */
+#define ISC_RLP_CFG     0x000003d0
+
+#define ISC_RLP_CFG_MODE_DAT8           0x0
+#define ISC_RLP_CFG_MODE_DAT9           0x1
+#define ISC_RLP_CFG_MODE_DAT10          0x2
+#define ISC_RLP_CFG_MODE_DAT11          0x3
+#define ISC_RLP_CFG_MODE_DAT12          0x4
+#define ISC_RLP_CFG_MODE_DATY8          0x5
+#define ISC_RLP_CFG_MODE_DATY10         0x6
+#define ISC_RLP_CFG_MODE_ARGB444        0x7
+#define ISC_RLP_CFG_MODE_ARGB555        0x8
+#define ISC_RLP_CFG_MODE_RGB565         0x9
+#define ISC_RLP_CFG_MODE_ARGB32         0xa
+#define ISC_RLP_CFG_MODE_YYCC           0xb
+#define ISC_RLP_CFG_MODE_YYCC_LIMITED   0xc
+#define ISC_RLP_CFG_MODE_MASK           GENMASK(3, 0)
+
+/* DMA Configuration Register */
+#define ISC_DCFG        0x000003e0
+#define ISC_DCFG_IMODE_PACKED8          0x0
+#define ISC_DCFG_IMODE_PACKED16         0x1
+#define ISC_DCFG_IMODE_PACKED32         0x2
+#define ISC_DCFG_IMODE_YC422SP          0x3
+#define ISC_DCFG_IMODE_YC422P           0x4
+#define ISC_DCFG_IMODE_YC420SP          0x5
+#define ISC_DCFG_IMODE_YC420P           0x6
+#define ISC_DCFG_IMODE_MASK             GENMASK(2, 0)
+
+#define ISC_DCFG_YMBSIZE_SINGLE         (0x0 << 4)
+#define ISC_DCFG_YMBSIZE_BEATS4         (0x1 << 4)
+#define ISC_DCFG_YMBSIZE_BEATS8         (0x2 << 4)
+#define ISC_DCFG_YMBSIZE_BEATS16        (0x3 << 4)
+#define ISC_DCFG_YMBSIZE_MASK           GENMASK(5, 4)
+
+#define ISC_DCFG_CMBSIZE_SINGLE         (0x0 << 8)
+#define ISC_DCFG_CMBSIZE_BEATS4         (0x1 << 8)
+#define ISC_DCFG_CMBSIZE_BEATS8         (0x2 << 8)
+#define ISC_DCFG_CMBSIZE_BEATS16        (0x3 << 8)
+#define ISC_DCFG_CMBSIZE_MASK           GENMASK(9, 8)
+
+/* DMA Control Register */
+#define ISC_DCTRL       0x000003e4
+
+#define ISC_DCTRL_DVIEW_PACKED          (0x0 << 1)
+#define ISC_DCTRL_DVIEW_SEMIPLANAR      (0x1 << 1)
+#define ISC_DCTRL_DVIEW_PLANAR          (0x2 << 1)
+#define ISC_DCTRL_DVIEW_MASK            GENMASK(2, 1)
+
+#define ISC_DCTRL_IE_IS			(0x0 << 4)
+
+/* DMA Descriptor Address Register */
+#define ISC_DNDA        0x000003e8
+
+/* DMA Address 0 Register */
+#define ISC_DAD0        0x000003ec
+
+/* DMA Stride 0 Register */
+#define ISC_DST0        0x000003f0
+
+#endif
diff --git a/drivers/media/platform/atmel/atmel-isc.c b/drivers/media/platform/atmel/atmel-isc.c
new file mode 100644
index 000000000000..ccfe13b7d3f8
--- /dev/null
+++ b/drivers/media/platform/atmel/atmel-isc.c
@@ -0,0 +1,1520 @@
+/*
+ * Atmel Image Sensor Controller (ISC) driver
+ *
+ * Copyright (C) 2016 Atmel
+ *
+ * Author: Songjun Wu <songjun.wu@microchip.com>
+ *
+ * This program is free software; you may redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * Sensor-->PFE-->WB-->CFA-->CC-->GAM-->CSC-->CBC-->SUB-->RLP-->DMA
+ *
+ * ISC video pipeline integrates the following submodules:
+ * PFE: Parallel Front End to sample the camera sensor input stream
+ *  WB: Programmable white balance in the Bayer domain
+ * CFA: Color filter array interpolation module
+ *  CC: Programmable color correction
+ * GAM: Gamma correction
+ * CSC: Programmable color space conversion
+ * CBC: Contrast and Brightness control
+ * SUB: This module performs YCbCr444 to YCbCr420 chrominance subsampling
+ * RLP: This module performs rounding, range limiting
+ *      and packing of the incoming data
+ */
+
+#include <linux/clk.h>
+#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/videodev2.h>
+
+#include <media/v4l2-device.h>
+#include <media/v4l2-image-sizes.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-of.h>
+#include <media/v4l2-subdev.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "atmel-isc-regs.h"
+
+#define ATMEL_ISC_NAME		"atmel_isc"
+
+#define ISC_MAX_SUPPORT_WIDTH   2592
+#define ISC_MAX_SUPPORT_HEIGHT  1944
+
+#define ISC_CLK_MAX_DIV		255
+
+enum isc_clk_id {
+	ISC_ISPCK = 0,
+	ISC_MCK = 1,
+};
+
+struct isc_clk {
+	struct clk_hw   hw;
+	struct clk      *clk;
+	struct regmap   *regmap;
+	u8		id;
+	u8		parent_id;
+	u32		div;
+	struct device	*dev;
+};
+
+#define to_isc_clk(hw) container_of(hw, struct isc_clk, hw)
+
+struct isc_buffer {
+	struct vb2_v4l2_buffer  vb;
+	struct list_head	list;
+};
+
+struct isc_subdev_entity {
+	struct v4l2_subdev		*sd;
+	struct v4l2_async_subdev	*asd;
+	struct v4l2_async_notifier      notifier;
+	struct v4l2_subdev_pad_config	*config;
+
+	u32 pfe_cfg0;
+
+	struct list_head list;
+};
+
+/*
+ * struct isc_format - ISC media bus format information
+ * @fourcc:		Fourcc code for this format
+ * @mbus_code:		V4L2 media bus format code.
+ * @bpp:		Bytes per pixel (when stored in memory)
+ * @reg_bps:		reg value for bits per sample
+ *			(when transferred over a bus)
+ * @support:		Indicates format supported by subdev
+ */
+struct isc_format {
+	u32	fourcc;
+	u32	mbus_code;
+	u8	bpp;
+
+	u32	reg_bps;
+	u32	reg_rlp_mode;
+	u32	reg_dcfg_imode;
+	u32	reg_dctrl_dview;
+
+	bool	support;
+};
+
+#define ISC_PIPE_LINE_NODE_NUM	11
+
+struct isc_device {
+	struct regmap		*regmap;
+	struct clk		*hclock;
+	struct clk		*ispck;
+	struct isc_clk		isc_clks[2];
+
+	struct device		*dev;
+	struct v4l2_device	v4l2_dev;
+	struct video_device	video_dev;
+
+	struct vb2_queue	vb2_vidq;
+	spinlock_t		dma_queue_lock;
+	struct list_head	dma_queue;
+	struct isc_buffer	*cur_frm;
+	unsigned int		sequence;
+	bool			stop;
+	struct completion	comp;
+
+	struct v4l2_format	fmt;
+	struct isc_format	**user_formats;
+	unsigned int		num_user_formats;
+	const struct isc_format	*current_fmt;
+
+	struct mutex		lock;
+
+	struct regmap_field	*pipeline[ISC_PIPE_LINE_NODE_NUM];
+
+	struct isc_subdev_entity	*current_subdev;
+	struct list_head		subdev_entities;
+};
+
+static struct isc_format isc_formats[] = {
+	{ V4L2_PIX_FMT_SBGGR8, MEDIA_BUS_FMT_SBGGR8_1X8,
+	  1, ISC_PFE_CFG0_BPS_EIGHT, ISC_RLP_CFG_MODE_DAT8,
+	  ISC_DCFG_IMODE_PACKED8, ISC_DCTRL_DVIEW_PACKED, false },
+	{ V4L2_PIX_FMT_SGBRG8, MEDIA_BUS_FMT_SGBRG8_1X8,
+	  1, ISC_PFE_CFG0_BPS_EIGHT, ISC_RLP_CFG_MODE_DAT8,
+	  ISC_DCFG_IMODE_PACKED8, ISC_DCTRL_DVIEW_PACKED, false },
+	{ V4L2_PIX_FMT_SGRBG8, MEDIA_BUS_FMT_SGRBG8_1X8,
+	  1, ISC_PFE_CFG0_BPS_EIGHT, ISC_RLP_CFG_MODE_DAT8,
+	  ISC_DCFG_IMODE_PACKED8, ISC_DCTRL_DVIEW_PACKED, false },
+	{ V4L2_PIX_FMT_SRGGB8, MEDIA_BUS_FMT_SRGGB8_1X8,
+	  1, ISC_PFE_CFG0_BPS_EIGHT, ISC_RLP_CFG_MODE_DAT8,
+	  ISC_DCFG_IMODE_PACKED8, ISC_DCTRL_DVIEW_PACKED, false },
+
+	{ V4L2_PIX_FMT_SBGGR10, MEDIA_BUS_FMT_SBGGR10_1X10,
+	  2, ISC_PFG_CFG0_BPS_TEN, ISC_RLP_CFG_MODE_DAT10,
+	  ISC_DCFG_IMODE_PACKED16, ISC_DCTRL_DVIEW_PACKED, false },
+	{ V4L2_PIX_FMT_SGBRG10, MEDIA_BUS_FMT_SGBRG10_1X10,
+	  2, ISC_PFG_CFG0_BPS_TEN, ISC_RLP_CFG_MODE_DAT10,
+	  ISC_DCFG_IMODE_PACKED16, ISC_DCTRL_DVIEW_PACKED, false },
+	{ V4L2_PIX_FMT_SGRBG10, MEDIA_BUS_FMT_SGRBG10_1X10,
+	  2, ISC_PFG_CFG0_BPS_TEN, ISC_RLP_CFG_MODE_DAT10,
+	  ISC_DCFG_IMODE_PACKED16, ISC_DCTRL_DVIEW_PACKED, false },
+	{ V4L2_PIX_FMT_SRGGB10, MEDIA_BUS_FMT_SRGGB10_1X10,
+	  2, ISC_PFG_CFG0_BPS_TEN, ISC_RLP_CFG_MODE_DAT10,
+	  ISC_DCFG_IMODE_PACKED16, ISC_DCTRL_DVIEW_PACKED, false },
+
+	{ V4L2_PIX_FMT_SBGGR12, MEDIA_BUS_FMT_SBGGR12_1X12,
+	  2, ISC_PFG_CFG0_BPS_TWELVE, ISC_RLP_CFG_MODE_DAT12,
+	  ISC_DCFG_IMODE_PACKED16, ISC_DCTRL_DVIEW_PACKED, false },
+	{ V4L2_PIX_FMT_SGBRG12, MEDIA_BUS_FMT_SGBRG12_1X12,
+	  2, ISC_PFG_CFG0_BPS_TWELVE, ISC_RLP_CFG_MODE_DAT12,
+	  ISC_DCFG_IMODE_PACKED16, ISC_DCTRL_DVIEW_PACKED, false },
+	{ V4L2_PIX_FMT_SGRBG12, MEDIA_BUS_FMT_SGRBG12_1X12,
+	  2, ISC_PFG_CFG0_BPS_TWELVE, ISC_RLP_CFG_MODE_DAT12,
+	  ISC_DCFG_IMODE_PACKED16, ISC_DCTRL_DVIEW_PACKED, false },
+	{ V4L2_PIX_FMT_SRGGB12, MEDIA_BUS_FMT_SRGGB12_1X12,
+	  2, ISC_PFG_CFG0_BPS_TWELVE, ISC_RLP_CFG_MODE_DAT12,
+	  ISC_DCFG_IMODE_PACKED16, ISC_DCTRL_DVIEW_PACKED, false },
+
+	{ V4L2_PIX_FMT_YUYV, MEDIA_BUS_FMT_YUYV8_2X8,
+	  2, ISC_PFE_CFG0_BPS_EIGHT, ISC_RLP_CFG_MODE_DAT8,
+	  ISC_DCFG_IMODE_PACKED8, ISC_DCTRL_DVIEW_PACKED, false },
+};
+
+static int isc_clk_enable(struct clk_hw *hw)
+{
+	struct isc_clk *isc_clk = to_isc_clk(hw);
+	u32 id = isc_clk->id;
+	struct regmap *regmap = isc_clk->regmap;
+
+	dev_dbg(isc_clk->dev, "ISC CLK: %s, div = %d, parent id = %d\n",
+		__func__, isc_clk->div, isc_clk->parent_id);
+
+	regmap_update_bits(regmap, ISC_CLKCFG,
+			   ISC_CLKCFG_DIV_MASK(id) | ISC_CLKCFG_SEL_MASK(id),
+			   (isc_clk->div << ISC_CLKCFG_DIV_SHIFT(id)) |
+			   (isc_clk->parent_id << ISC_CLKCFG_SEL_SHIFT(id)));
+
+	regmap_write(regmap, ISC_CLKEN, ISC_CLK(id));
+
+	return 0;
+}
+
+static void isc_clk_disable(struct clk_hw *hw)
+{
+	struct isc_clk *isc_clk = to_isc_clk(hw);
+	u32 id = isc_clk->id;
+
+	regmap_write(isc_clk->regmap, ISC_CLKDIS, ISC_CLK(id));
+}
+
+static int isc_clk_is_enabled(struct clk_hw *hw)
+{
+	struct isc_clk *isc_clk = to_isc_clk(hw);
+	u32 status;
+
+	regmap_read(isc_clk->regmap, ISC_CLKSR, &status);
+
+	return status & ISC_CLK(isc_clk->id) ? 1 : 0;
+}
+
+static unsigned long
+isc_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct isc_clk *isc_clk = to_isc_clk(hw);
+
+	return DIV_ROUND_CLOSEST(parent_rate, isc_clk->div + 1);
+}
+
+static int isc_clk_determine_rate(struct clk_hw *hw,
+				   struct clk_rate_request *req)
+{
+	struct isc_clk *isc_clk = to_isc_clk(hw);
+	long best_rate = -EINVAL;
+	int best_diff = -1;
+	unsigned int i, div;
+
+	for (i = 0; i < clk_hw_get_num_parents(hw); i++) {
+		struct clk_hw *parent;
+		unsigned long parent_rate;
+
+		parent = clk_hw_get_parent_by_index(hw, i);
+		if (!parent)
+			continue;
+
+		parent_rate = clk_hw_get_rate(parent);
+		if (!parent_rate)
+			continue;
+
+		for (div = 1; div < ISC_CLK_MAX_DIV + 2; div++) {
+			unsigned long rate;
+			int diff;
+
+			rate = DIV_ROUND_CLOSEST(parent_rate, div);
+			diff = abs(req->rate - rate);
+
+			if (best_diff < 0 || best_diff > diff) {
+				best_rate = rate;
+				best_diff = diff;
+				req->best_parent_rate = parent_rate;
+				req->best_parent_hw = parent;
+			}
+
+			if (!best_diff || rate < req->rate)
+				break;
+		}
+
+		if (!best_diff)
+			break;
+	}
+
+	dev_dbg(isc_clk->dev,
+		"ISC CLK: %s, best_rate = %ld, parent clk: %s @ %ld\n",
+		__func__, best_rate,
+		__clk_get_name((req->best_parent_hw)->clk),
+		req->best_parent_rate);
+
+	if (best_rate < 0)
+		return best_rate;
+
+	req->rate = best_rate;
+
+	return 0;
+}
+
+static int isc_clk_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct isc_clk *isc_clk = to_isc_clk(hw);
+
+	if (index >= clk_hw_get_num_parents(hw))
+		return -EINVAL;
+
+	isc_clk->parent_id = index;
+
+	return 0;
+}
+
+static u8 isc_clk_get_parent(struct clk_hw *hw)
+{
+	struct isc_clk *isc_clk = to_isc_clk(hw);
+
+	return isc_clk->parent_id;
+}
+
+static int isc_clk_set_rate(struct clk_hw *hw,
+			     unsigned long rate,
+			     unsigned long parent_rate)
+{
+	struct isc_clk *isc_clk = to_isc_clk(hw);
+	u32 div;
+
+	if (!rate)
+		return -EINVAL;
+
+	div = DIV_ROUND_CLOSEST(parent_rate, rate);
+	if (div > (ISC_CLK_MAX_DIV + 1) || !div)
+		return -EINVAL;
+
+	isc_clk->div = div - 1;
+
+	return 0;
+}
+
+static const struct clk_ops isc_clk_ops = {
+	.enable		= isc_clk_enable,
+	.disable	= isc_clk_disable,
+	.is_enabled	= isc_clk_is_enabled,
+	.recalc_rate	= isc_clk_recalc_rate,
+	.determine_rate	= isc_clk_determine_rate,
+	.set_parent	= isc_clk_set_parent,
+	.get_parent	= isc_clk_get_parent,
+	.set_rate	= isc_clk_set_rate,
+};
+
+static int isc_clk_register(struct isc_device *isc, unsigned int id)
+{
+	struct regmap *regmap = isc->regmap;
+	struct device_node *np = isc->dev->of_node;
+	struct isc_clk *isc_clk;
+	struct clk_init_data init;
+	const char *clk_name = np->name;
+	const char *parent_names[3];
+	int num_parents;
+
+	num_parents = of_clk_get_parent_count(np);
+	if (num_parents < 1 || num_parents > 3)
+		return -EINVAL;
+
+	if (num_parents > 2 && id == ISC_ISPCK)
+		num_parents = 2;
+
+	of_clk_parent_fill(np, parent_names, num_parents);
+
+	if (id == ISC_MCK)
+		of_property_read_string(np, "clock-output-names", &clk_name);
+	else
+		clk_name = "isc-ispck";
+
+	init.parent_names	= parent_names;
+	init.num_parents	= num_parents;
+	init.name		= clk_name;
+	init.ops		= &isc_clk_ops;
+	init.flags		= CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE;
+
+	isc_clk = &isc->isc_clks[id];
+	isc_clk->hw.init	= &init;
+	isc_clk->regmap		= regmap;
+	isc_clk->id		= id;
+	isc_clk->dev		= isc->dev;
+
+	isc_clk->clk = clk_register(isc->dev, &isc_clk->hw);
+	if (IS_ERR(isc_clk->clk)) {
+		dev_err(isc->dev, "%s: clock register fail\n", clk_name);
+		return PTR_ERR(isc_clk->clk);
+	} else if (id == ISC_MCK)
+		of_clk_add_provider(np, of_clk_src_simple_get, isc_clk->clk);
+
+	return 0;
+}
+
+static int isc_clk_init(struct isc_device *isc)
+{
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < ARRAY_SIZE(isc->isc_clks); i++)
+		isc->isc_clks[i].clk = ERR_PTR(-EINVAL);
+
+	for (i = 0; i < ARRAY_SIZE(isc->isc_clks); i++) {
+		ret = isc_clk_register(isc, i);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void isc_clk_cleanup(struct isc_device *isc)
+{
+	unsigned int i;
+
+	of_clk_del_provider(isc->dev->of_node);
+
+	for (i = 0; i < ARRAY_SIZE(isc->isc_clks); i++) {
+		struct isc_clk *isc_clk = &isc->isc_clks[i];
+
+		if (!IS_ERR(isc_clk->clk))
+			clk_unregister(isc_clk->clk);
+	}
+}
+
+static int isc_queue_setup(struct vb2_queue *vq,
+			    unsigned int *nbuffers, unsigned int *nplanes,
+			    unsigned int sizes[], struct device *alloc_devs[])
+{
+	struct isc_device *isc = vb2_get_drv_priv(vq);
+	unsigned int size = isc->fmt.fmt.pix.sizeimage;
+
+	if (*nplanes)
+		return sizes[0] < size ? -EINVAL : 0;
+
+	*nplanes = 1;
+	sizes[0] = size;
+
+	return 0;
+}
+
+static int isc_buffer_prepare(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct isc_device *isc = vb2_get_drv_priv(vb->vb2_queue);
+	unsigned long size = isc->fmt.fmt.pix.sizeimage;
+
+	if (vb2_plane_size(vb, 0) < size) {
+		v4l2_err(&isc->v4l2_dev, "buffer too small (%lu < %lu)\n",
+			 vb2_plane_size(vb, 0), size);
+		return -EINVAL;
+	}
+
+	vb2_set_plane_payload(vb, 0, size);
+
+	vbuf->field = isc->fmt.fmt.pix.field;
+
+	return 0;
+}
+
+static inline void isc_start_dma(struct regmap *regmap,
+				  struct isc_buffer *frm, u32 dview)
+{
+	dma_addr_t addr;
+
+	addr = vb2_dma_contig_plane_dma_addr(&frm->vb.vb2_buf, 0);
+
+	regmap_write(regmap, ISC_DCTRL, dview | ISC_DCTRL_IE_IS);
+	regmap_write(regmap, ISC_DAD0, addr);
+	regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_CAPTURE);
+}
+
+static void isc_set_pipeline(struct isc_device *isc, u32 pipeline)
+{
+	u32 val;
+	unsigned int i;
+
+	for (i = 0; i < ISC_PIPE_LINE_NODE_NUM; i++) {
+		val = pipeline & BIT(i) ? 1 : 0;
+		regmap_field_write(isc->pipeline[i], val);
+	}
+}
+
+static int isc_configure(struct isc_device *isc)
+{
+	struct regmap *regmap = isc->regmap;
+	const struct isc_format *current_fmt = isc->current_fmt;
+	struct isc_subdev_entity *subdev = isc->current_subdev;
+	u32 val, mask;
+	int counter = 10;
+
+	val = current_fmt->reg_bps | subdev->pfe_cfg0 |
+	      ISC_PFE_CFG0_MODE_PROGRESSIVE;
+	mask = ISC_PFE_CFG0_BPS_MASK | ISC_PFE_CFG0_HPOL_LOW |
+	       ISC_PFE_CFG0_VPOL_LOW | ISC_PFE_CFG0_PPOL_LOW |
+	       ISC_PFE_CFG0_MODE_MASK;
+
+	regmap_update_bits(regmap, ISC_PFE_CFG0, mask, val);
+
+	regmap_update_bits(regmap, ISC_RLP_CFG, ISC_RLP_CFG_MODE_MASK,
+			   current_fmt->reg_rlp_mode);
+
+	regmap_update_bits(regmap, ISC_DCFG, ISC_DCFG_IMODE_MASK,
+			   current_fmt->reg_dcfg_imode);
+
+	/* Disable the pipeline */
+	isc_set_pipeline(isc, 0x0);
+
+	/* Update profile */
+	regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_UPPRO);
+
+	regmap_read(regmap, ISC_CTRLSR, &val);
+	while ((val & ISC_CTRL_UPPRO) && counter--) {
+		usleep_range(1000, 2000);
+		regmap_read(regmap, ISC_CTRLSR, &val);
+	}
+
+	if (counter < 0)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static int isc_start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct isc_device *isc = vb2_get_drv_priv(vq);
+	struct regmap *regmap = isc->regmap;
+	struct isc_buffer *buf;
+	unsigned long flags;
+	int ret;
+	u32 val;
+
+	/* Enable stream on the sub device */
+	ret = v4l2_subdev_call(isc->current_subdev->sd, video, s_stream, 1);
+	if (ret && ret != -ENOIOCTLCMD) {
+		v4l2_err(&isc->v4l2_dev, "stream on failed in subdev\n");
+		goto err_start_stream;
+	}
+
+	pm_runtime_get_sync(isc->dev);
+
+	/* Disable all the interrupts */
+	regmap_write(isc->regmap, ISC_INTDIS, (u32)~0UL);
+
+	/* Clean the interrupt status register */
+	regmap_read(regmap, ISC_INTSR, &val);
+
+	ret = isc_configure(isc);
+	if (unlikely(ret))
+		goto err_configure;
+
+	/* Enable DMA interrupt */
+	regmap_write(regmap, ISC_INTEN, ISC_INT_DDONE);
+
+	spin_lock_irqsave(&isc->dma_queue_lock, flags);
+
+	isc->sequence = 0;
+	isc->stop = false;
+	reinit_completion(&isc->comp);
+
+	isc->cur_frm = list_first_entry(&isc->dma_queue,
+					struct isc_buffer, list);
+	list_del(&isc->cur_frm->list);
+
+	isc_start_dma(regmap, isc->cur_frm, isc->current_fmt->reg_dctrl_dview);
+
+	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
+
+	return 0;
+
+err_configure:
+	pm_runtime_put_sync(isc->dev);
+
+	v4l2_subdev_call(isc->current_subdev->sd, video, s_stream, 0);
+
+err_start_stream:
+	spin_lock_irqsave(&isc->dma_queue_lock, flags);
+	list_for_each_entry(buf, &isc->dma_queue, list)
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
+	INIT_LIST_HEAD(&isc->dma_queue);
+	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
+
+	return ret;
+}
+
+static void isc_stop_streaming(struct vb2_queue *vq)
+{
+	struct isc_device *isc = vb2_get_drv_priv(vq);
+	unsigned long flags;
+	struct isc_buffer *buf;
+	int ret;
+
+	isc->stop = true;
+
+	/* Wait until the end of the current frame */
+	if (isc->cur_frm && !wait_for_completion_timeout(&isc->comp, 5 * HZ))
+		v4l2_err(&isc->v4l2_dev,
+			 "Timeout waiting for end of the capture\n");
+
+	/* Disable DMA interrupt */
+	regmap_write(isc->regmap, ISC_INTDIS, ISC_INT_DDONE);
+
+	pm_runtime_put_sync(isc->dev);
+
+	/* Disable stream on the sub device */
+	ret = v4l2_subdev_call(isc->current_subdev->sd, video, s_stream, 0);
+	if (ret && ret != -ENOIOCTLCMD)
+		v4l2_err(&isc->v4l2_dev, "stream off failed in subdev\n");
+
+	/* Release all active buffers */
+	spin_lock_irqsave(&isc->dma_queue_lock, flags);
+	if (unlikely(isc->cur_frm)) {
+		vb2_buffer_done(&isc->cur_frm->vb.vb2_buf,
+				VB2_BUF_STATE_ERROR);
+		isc->cur_frm = NULL;
+	}
+	list_for_each_entry(buf, &isc->dma_queue, list)
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+	INIT_LIST_HEAD(&isc->dma_queue);
+	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
+}
+
+static void isc_buffer_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct isc_buffer *buf = container_of(vbuf, struct isc_buffer, vb);
+	struct isc_device *isc = vb2_get_drv_priv(vb->vb2_queue);
+	unsigned long flags;
+
+	spin_lock_irqsave(&isc->dma_queue_lock, flags);
+	list_add_tail(&buf->list, &isc->dma_queue);
+	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
+}
+
+static struct vb2_ops isc_vb2_ops = {
+	.queue_setup		= isc_queue_setup,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+	.buf_prepare		= isc_buffer_prepare,
+	.start_streaming	= isc_start_streaming,
+	.stop_streaming		= isc_stop_streaming,
+	.buf_queue		= isc_buffer_queue,
+};
+
+static int isc_querycap(struct file *file, void *priv,
+			 struct v4l2_capability *cap)
+{
+	struct isc_device *isc = video_drvdata(file);
+
+	strcpy(cap->driver, ATMEL_ISC_NAME);
+	strcpy(cap->card, "Atmel Image Sensor Controller");
+	snprintf(cap->bus_info, sizeof(cap->bus_info),
+		 "platform:%s", isc->v4l2_dev.name);
+
+	return 0;
+}
+
+static int isc_enum_fmt_vid_cap(struct file *file, void *priv,
+				 struct v4l2_fmtdesc *f)
+{
+	struct isc_device *isc = video_drvdata(file);
+	u32 index = f->index;
+
+	if (index >= isc->num_user_formats)
+		return -EINVAL;
+
+	f->pixelformat = isc->user_formats[index]->fourcc;
+
+	return 0;
+}
+
+static int isc_g_fmt_vid_cap(struct file *file, void *priv,
+			      struct v4l2_format *fmt)
+{
+	struct isc_device *isc = video_drvdata(file);
+
+	*fmt = isc->fmt;
+
+	return 0;
+}
+
+static struct isc_format *find_format_by_fourcc(struct isc_device *isc,
+						 unsigned int fourcc)
+{
+	unsigned int num_formats = isc->num_user_formats;
+	struct isc_format *fmt;
+	unsigned int i;
+
+	for (i = 0; i < num_formats; i++) {
+		fmt = isc->user_formats[i];
+		if (fmt->fourcc == fourcc)
+			return fmt;
+	}
+
+	return NULL;
+}
+
+static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f,
+			struct isc_format **current_fmt)
+{
+	struct isc_format *isc_fmt;
+	struct v4l2_pix_format *pixfmt = &f->fmt.pix;
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_TRY,
+	};
+	int ret;
+
+	if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	isc_fmt = find_format_by_fourcc(isc, pixfmt->pixelformat);
+	if (!isc_fmt) {
+		v4l2_warn(&isc->v4l2_dev, "Format 0x%x not found\n",
+			  pixfmt->pixelformat);
+		isc_fmt = isc->user_formats[isc->num_user_formats - 1];
+		pixfmt->pixelformat = isc_fmt->fourcc;
+	}
+
+	/* Limit to Atmel ISC hardware capabilities */
+	if (pixfmt->width > ISC_MAX_SUPPORT_WIDTH)
+		pixfmt->width = ISC_MAX_SUPPORT_WIDTH;
+	if (pixfmt->height > ISC_MAX_SUPPORT_HEIGHT)
+		pixfmt->height = ISC_MAX_SUPPORT_HEIGHT;
+
+	v4l2_fill_mbus_format(&format.format, pixfmt, isc_fmt->mbus_code);
+	ret = v4l2_subdev_call(isc->current_subdev->sd, pad, set_fmt,
+			       isc->current_subdev->config, &format);
+	if (ret < 0)
+		return ret;
+
+	v4l2_fill_pix_format(pixfmt, &format.format);
+
+	pixfmt->field = V4L2_FIELD_NONE;
+	pixfmt->bytesperline = pixfmt->width * isc_fmt->bpp;
+	pixfmt->sizeimage = pixfmt->bytesperline * pixfmt->height;
+
+	if (current_fmt)
+		*current_fmt = isc_fmt;
+
+	return 0;
+}
+
+static int isc_set_fmt(struct isc_device *isc, struct v4l2_format *f)
+{
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	struct isc_format *current_fmt;
+	int ret;
+
+	ret = isc_try_fmt(isc, f, &current_fmt);
+	if (ret)
+		return ret;
+
+	v4l2_fill_mbus_format(&format.format, &f->fmt.pix,
+			      current_fmt->mbus_code);
+	ret = v4l2_subdev_call(isc->current_subdev->sd, pad,
+			       set_fmt, NULL, &format);
+	if (ret < 0)
+		return ret;
+
+	isc->fmt = *f;
+	isc->current_fmt = current_fmt;
+
+	return 0;
+}
+
+static int isc_s_fmt_vid_cap(struct file *file, void *priv,
+			      struct v4l2_format *f)
+{
+	struct isc_device *isc = video_drvdata(file);
+
+	if (vb2_is_streaming(&isc->vb2_vidq))
+		return -EBUSY;
+
+	return isc_set_fmt(isc, f);
+}
+
+static int isc_try_fmt_vid_cap(struct file *file, void *priv,
+				struct v4l2_format *f)
+{
+	struct isc_device *isc = video_drvdata(file);
+
+	return isc_try_fmt(isc, f, NULL);
+}
+
+static int isc_enum_input(struct file *file, void *priv,
+			   struct v4l2_input *inp)
+{
+	if (inp->index != 0)
+		return -EINVAL;
+
+	inp->type = V4L2_INPUT_TYPE_CAMERA;
+	inp->std = 0;
+	strcpy(inp->name, "Camera");
+
+	return 0;
+}
+
+static int isc_g_input(struct file *file, void *priv, unsigned int *i)
+{
+	*i = 0;
+
+	return 0;
+}
+
+static int isc_s_input(struct file *file, void *priv, unsigned int i)
+{
+	if (i > 0)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int isc_g_parm(struct file *file, void *fh, struct v4l2_streamparm *a)
+{
+	struct isc_device *isc = video_drvdata(file);
+
+	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	return v4l2_subdev_call(isc->current_subdev->sd, video, g_parm, a);
+}
+
+static int isc_s_parm(struct file *file, void *fh, struct v4l2_streamparm *a)
+{
+	struct isc_device *isc = video_drvdata(file);
+
+	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	return v4l2_subdev_call(isc->current_subdev->sd, video, s_parm, a);
+}
+
+static int isc_enum_framesizes(struct file *file, void *fh,
+			       struct v4l2_frmsizeenum *fsize)
+{
+	struct isc_device *isc = video_drvdata(file);
+	const struct isc_format *isc_fmt;
+	struct v4l2_subdev_frame_size_enum fse = {
+		.index = fsize->index,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	int ret;
+
+	isc_fmt = find_format_by_fourcc(isc, fsize->pixel_format);
+	if (!isc_fmt)
+		return -EINVAL;
+
+	fse.code = isc_fmt->mbus_code;
+
+	ret = v4l2_subdev_call(isc->current_subdev->sd, pad, enum_frame_size,
+			       NULL, &fse);
+	if (ret)
+		return ret;
+
+	fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE;
+	fsize->discrete.width = fse.max_width;
+	fsize->discrete.height = fse.max_height;
+
+	return 0;
+}
+
+static int isc_enum_frameintervals(struct file *file, void *fh,
+				    struct v4l2_frmivalenum *fival)
+{
+	struct isc_device *isc = video_drvdata(file);
+	const struct isc_format *isc_fmt;
+	struct v4l2_subdev_frame_interval_enum fie = {
+		.index = fival->index,
+		.width = fival->width,
+		.height = fival->height,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	int ret;
+
+	isc_fmt = find_format_by_fourcc(isc, fival->pixel_format);
+	if (!isc_fmt)
+		return -EINVAL;
+
+	fie.code = isc_fmt->mbus_code;
+
+	ret = v4l2_subdev_call(isc->current_subdev->sd, pad,
+			       enum_frame_interval, NULL, &fie);
+	if (ret)
+		return ret;
+
+	fival->type = V4L2_FRMIVAL_TYPE_DISCRETE;
+	fival->discrete = fie.interval;
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops isc_ioctl_ops = {
+	.vidioc_querycap		= isc_querycap,
+	.vidioc_enum_fmt_vid_cap	= isc_enum_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap		= isc_g_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap		= isc_s_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap		= isc_try_fmt_vid_cap,
+
+	.vidioc_enum_input		= isc_enum_input,
+	.vidioc_g_input			= isc_g_input,
+	.vidioc_s_input			= isc_s_input,
+
+	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
+	.vidioc_querybuf		= vb2_ioctl_querybuf,
+	.vidioc_qbuf			= vb2_ioctl_qbuf,
+	.vidioc_expbuf			= vb2_ioctl_expbuf,
+	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
+	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
+	.vidioc_prepare_buf		= vb2_ioctl_prepare_buf,
+	.vidioc_streamon		= vb2_ioctl_streamon,
+	.vidioc_streamoff		= vb2_ioctl_streamoff,
+
+	.vidioc_g_parm			= isc_g_parm,
+	.vidioc_s_parm			= isc_s_parm,
+	.vidioc_enum_framesizes		= isc_enum_framesizes,
+	.vidioc_enum_frameintervals	= isc_enum_frameintervals,
+};
+
+static int isc_open(struct file *file)
+{
+	struct isc_device *isc = video_drvdata(file);
+	struct v4l2_subdev *sd = isc->current_subdev->sd;
+	int ret;
+
+	if (mutex_lock_interruptible(&isc->lock))
+		return -ERESTARTSYS;
+
+	ret = v4l2_fh_open(file);
+	if (ret < 0)
+		goto unlock;
+
+	if (!v4l2_fh_is_singular_file(file))
+		goto unlock;
+
+	ret = v4l2_subdev_call(sd, core, s_power, 1);
+	if (ret < 0 && ret != -ENOIOCTLCMD) {
+		v4l2_fh_release(file);
+		goto unlock;
+	}
+
+	ret = isc_set_fmt(isc, &isc->fmt);
+	if (ret) {
+		v4l2_subdev_call(sd, core, s_power, 0);
+		v4l2_fh_release(file);
+	}
+
+unlock:
+	mutex_unlock(&isc->lock);
+	return ret;
+}
+
+static int isc_release(struct file *file)
+{
+	struct isc_device *isc = video_drvdata(file);
+	struct v4l2_subdev *sd = isc->current_subdev->sd;
+	bool fh_singular;
+	int ret;
+
+	mutex_lock(&isc->lock);
+
+	fh_singular = v4l2_fh_is_singular_file(file);
+
+	ret = _vb2_fop_release(file, NULL);
+
+	if (fh_singular)
+		v4l2_subdev_call(sd, core, s_power, 0);
+
+	mutex_unlock(&isc->lock);
+
+	return ret;
+}
+
+static const struct v4l2_file_operations isc_fops = {
+	.owner		= THIS_MODULE,
+	.open		= isc_open,
+	.release	= isc_release,
+	.unlocked_ioctl	= video_ioctl2,
+	.read		= vb2_fop_read,
+	.mmap		= vb2_fop_mmap,
+	.poll		= vb2_fop_poll,
+};
+
+static irqreturn_t isc_interrupt(int irq, void *dev_id)
+{
+	struct isc_device *isc = (struct isc_device *)dev_id;
+	struct regmap *regmap = isc->regmap;
+	u32 isc_intsr, isc_intmask, pending;
+	irqreturn_t ret = IRQ_NONE;
+
+	spin_lock(&isc->dma_queue_lock);
+
+	regmap_read(regmap, ISC_INTSR, &isc_intsr);
+	regmap_read(regmap, ISC_INTMASK, &isc_intmask);
+
+	pending = isc_intsr & isc_intmask;
+
+	if (likely(pending & ISC_INT_DDONE)) {
+		if (isc->cur_frm) {
+			struct vb2_v4l2_buffer *vbuf = &isc->cur_frm->vb;
+			struct vb2_buffer *vb = &vbuf->vb2_buf;
+
+			vb->timestamp = ktime_get_ns();
+			vbuf->sequence = isc->sequence++;
+			vb2_buffer_done(vb, VB2_BUF_STATE_DONE);
+			isc->cur_frm = NULL;
+		}
+
+		if (!list_empty(&isc->dma_queue) && !isc->stop) {
+			isc->cur_frm = list_first_entry(&isc->dma_queue,
+						     struct isc_buffer, list);
+			list_del(&isc->cur_frm->list);
+
+			isc_start_dma(regmap, isc->cur_frm,
+				      isc->current_fmt->reg_dctrl_dview);
+		}
+
+		if (isc->stop)
+			complete(&isc->comp);
+
+		ret = IRQ_HANDLED;
+	}
+
+	spin_unlock(&isc->dma_queue_lock);
+
+	return ret;
+}
+
+static int isc_async_bound(struct v4l2_async_notifier *notifier,
+			    struct v4l2_subdev *subdev,
+			    struct v4l2_async_subdev *asd)
+{
+	struct isc_device *isc = container_of(notifier->v4l2_dev,
+					      struct isc_device, v4l2_dev);
+	struct isc_subdev_entity *subdev_entity =
+		container_of(notifier, struct isc_subdev_entity, notifier);
+
+	if (video_is_registered(&isc->video_dev)) {
+		v4l2_err(&isc->v4l2_dev, "only supports one sub-device.\n");
+		return -EBUSY;
+	}
+
+	subdev_entity->sd = subdev;
+
+	return 0;
+}
+
+static void isc_async_unbind(struct v4l2_async_notifier *notifier,
+			      struct v4l2_subdev *subdev,
+			      struct v4l2_async_subdev *asd)
+{
+	struct isc_device *isc = container_of(notifier->v4l2_dev,
+					      struct isc_device, v4l2_dev);
+
+	video_unregister_device(&isc->video_dev);
+	if (isc->current_subdev->config)
+		v4l2_subdev_free_pad_config(isc->current_subdev->config);
+}
+
+static struct isc_format *find_format_by_code(unsigned int code, int *index)
+{
+	struct isc_format *fmt = &isc_formats[0];
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(isc_formats); i++) {
+		if (fmt->mbus_code == code) {
+			*index = i;
+			return fmt;
+		}
+
+		fmt++;
+	}
+
+	return NULL;
+}
+
+static int isc_formats_init(struct isc_device *isc)
+{
+	struct isc_format *fmt;
+	struct v4l2_subdev *subdev = isc->current_subdev->sd;
+	int num_fmts = 0, i, j;
+	struct v4l2_subdev_mbus_code_enum mbus_code = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+
+	fmt = &isc_formats[0];
+	for (i = 0; i < ARRAY_SIZE(isc_formats); i++) {
+		fmt->support = false;
+		fmt++;
+	}
+
+	while (!v4l2_subdev_call(subdev, pad, enum_mbus_code,
+	       NULL, &mbus_code)) {
+		mbus_code.index++;
+		fmt = find_format_by_code(mbus_code.code, &i);
+		if (!fmt)
+			continue;
+
+		fmt->support = true;
+		num_fmts++;
+	}
+
+	if (!num_fmts)
+		return -ENXIO;
+
+	isc->num_user_formats = num_fmts;
+	isc->user_formats = devm_kcalloc(isc->dev,
+					 num_fmts, sizeof(struct isc_format *),
+					 GFP_KERNEL);
+	if (!isc->user_formats) {
+		v4l2_err(&isc->v4l2_dev, "could not allocate memory\n");
+		return -ENOMEM;
+	}
+
+	fmt = &isc_formats[0];
+	for (i = 0, j = 0; i < ARRAY_SIZE(isc_formats); i++) {
+		if (fmt->support)
+			isc->user_formats[j++] = fmt;
+
+		fmt++;
+	}
+
+	return 0;
+}
+
+static int isc_set_default_fmt(struct isc_device *isc)
+{
+	struct v4l2_format f = {
+		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
+		.fmt.pix = {
+			.width		= VGA_WIDTH,
+			.height		= VGA_HEIGHT,
+			.field		= V4L2_FIELD_NONE,
+			.pixelformat	= isc->user_formats[0]->fourcc,
+		},
+	};
+	int ret;
+
+	ret = isc_try_fmt(isc, &f, NULL);
+	if (ret)
+		return ret;
+
+	isc->current_fmt = isc->user_formats[0];
+	isc->fmt = f;
+
+	return 0;
+}
+
+static int isc_async_complete(struct v4l2_async_notifier *notifier)
+{
+	struct isc_device *isc = container_of(notifier->v4l2_dev,
+					      struct isc_device, v4l2_dev);
+	struct isc_subdev_entity *sd_entity;
+	struct video_device *vdev = &isc->video_dev;
+	struct vb2_queue *q = &isc->vb2_vidq;
+	int ret;
+
+	isc->current_subdev = container_of(notifier,
+					   struct isc_subdev_entity, notifier);
+	sd_entity = isc->current_subdev;
+
+	mutex_init(&isc->lock);
+	init_completion(&isc->comp);
+
+	/* Initialize videobuf2 queue */
+	q->type			= V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	q->io_modes		= VB2_MMAP | VB2_DMABUF | VB2_READ;
+	q->drv_priv		= isc;
+	q->buf_struct_size	= sizeof(struct isc_buffer);
+	q->ops			= &isc_vb2_ops;
+	q->mem_ops		= &vb2_dma_contig_memops;
+	q->timestamp_flags	= V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->lock			= &isc->lock;
+	q->min_buffers_needed	= 1;
+	q->dev			= isc->dev;
+
+	ret = vb2_queue_init(q);
+	if (ret < 0) {
+		v4l2_err(&isc->v4l2_dev,
+			 "vb2_queue_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	/* Init video dma queues */
+	INIT_LIST_HEAD(&isc->dma_queue);
+	spin_lock_init(&isc->dma_queue_lock);
+
+	sd_entity->config = v4l2_subdev_alloc_pad_config(sd_entity->sd);
+	if (sd_entity->config == NULL)
+		return -ENOMEM;
+
+	ret = isc_formats_init(isc);
+	if (ret < 0) {
+		v4l2_err(&isc->v4l2_dev,
+			 "Init format failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = isc_set_default_fmt(isc);
+	if (ret) {
+		v4l2_err(&isc->v4l2_dev, "Could not set default format\n");
+		return ret;
+	}
+
+	/* Register video device */
+	strlcpy(vdev->name, ATMEL_ISC_NAME, sizeof(vdev->name));
+	vdev->release		= video_device_release_empty;
+	vdev->fops		= &isc_fops;
+	vdev->ioctl_ops		= &isc_ioctl_ops;
+	vdev->v4l2_dev		= &isc->v4l2_dev;
+	vdev->vfl_dir		= VFL_DIR_RX;
+	vdev->queue		= q;
+	vdev->lock		= &isc->lock;
+	vdev->ctrl_handler	= isc->current_subdev->sd->ctrl_handler;
+	vdev->device_caps	= V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE;
+	video_set_drvdata(vdev, isc);
+
+	ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1);
+	if (ret < 0) {
+		v4l2_err(&isc->v4l2_dev,
+			 "video_register_device failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void isc_subdev_cleanup(struct isc_device *isc)
+{
+	struct isc_subdev_entity *subdev_entity;
+
+	list_for_each_entry(subdev_entity, &isc->subdev_entities, list)
+		v4l2_async_notifier_unregister(&subdev_entity->notifier);
+
+	INIT_LIST_HEAD(&isc->subdev_entities);
+}
+
+static int isc_pipeline_init(struct isc_device *isc)
+{
+	struct device *dev = isc->dev;
+	struct regmap *regmap = isc->regmap;
+	struct regmap_field *regs;
+	unsigned int i;
+
+	/* WB-->CFA-->CC-->GAM-->CSC-->CBC-->SUB422-->SUB420 */
+	const struct reg_field regfields[ISC_PIPE_LINE_NODE_NUM] = {
+		REG_FIELD(ISC_WB_CTRL, 0, 0),
+		REG_FIELD(ISC_CFA_CTRL, 0, 0),
+		REG_FIELD(ISC_CC_CTRL, 0, 0),
+		REG_FIELD(ISC_GAM_CTRL, 0, 0),
+		REG_FIELD(ISC_GAM_CTRL, 1, 1),
+		REG_FIELD(ISC_GAM_CTRL, 2, 2),
+		REG_FIELD(ISC_GAM_CTRL, 3, 3),
+		REG_FIELD(ISC_CSC_CTRL, 0, 0),
+		REG_FIELD(ISC_CBC_CTRL, 0, 0),
+		REG_FIELD(ISC_SUB422_CTRL, 0, 0),
+		REG_FIELD(ISC_SUB420_CTRL, 0, 0),
+	};
+
+	for (i = 0; i < ISC_PIPE_LINE_NODE_NUM; i++) {
+		regs = devm_regmap_field_alloc(dev, regmap, regfields[i]);
+		if (IS_ERR(regs))
+			return PTR_ERR(regs);
+
+		isc->pipeline[i] =  regs;
+	}
+
+	return 0;
+}
+
+static int isc_parse_dt(struct device *dev, struct isc_device *isc)
+{
+	struct device_node *np = dev->of_node;
+	struct device_node *epn = NULL, *rem;
+	struct v4l2_of_endpoint v4l2_epn;
+	struct isc_subdev_entity *subdev_entity;
+	unsigned int flags;
+	int ret;
+
+	INIT_LIST_HEAD(&isc->subdev_entities);
+
+	for (; ;) {
+		epn = of_graph_get_next_endpoint(np, epn);
+		if (!epn)
+			break;
+
+		rem = of_graph_get_remote_port_parent(epn);
+		if (!rem) {
+			dev_notice(dev, "Remote device at %s not found\n",
+				   of_node_full_name(epn));
+			continue;
+		}
+
+		ret = v4l2_of_parse_endpoint(epn, &v4l2_epn);
+		if (ret) {
+			of_node_put(rem);
+			ret = -EINVAL;
+			dev_err(dev, "Could not parse the endpoint\n");
+			break;
+		}
+
+		subdev_entity = devm_kzalloc(dev,
+					  sizeof(*subdev_entity), GFP_KERNEL);
+		if (subdev_entity == NULL) {
+			of_node_put(rem);
+			ret = -ENOMEM;
+			break;
+		}
+
+		subdev_entity->asd = devm_kzalloc(dev,
+				     sizeof(*subdev_entity->asd), GFP_KERNEL);
+		if (subdev_entity->asd == NULL) {
+			of_node_put(rem);
+			ret = -ENOMEM;
+			break;
+		}
+
+		flags = v4l2_epn.bus.parallel.flags;
+
+		if (flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)
+			subdev_entity->pfe_cfg0 = ISC_PFE_CFG0_HPOL_LOW;
+
+		if (flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
+			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_VPOL_LOW;
+
+		if (flags & V4L2_MBUS_PCLK_SAMPLE_FALLING)
+			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_PPOL_LOW;
+
+		subdev_entity->asd->match_type = V4L2_ASYNC_MATCH_OF;
+		subdev_entity->asd->match.of.node = rem;
+		list_add_tail(&subdev_entity->list, &isc->subdev_entities);
+	}
+
+	of_node_put(epn);
+	return ret;
+}
+
+/* regmap configuration */
+#define ATMEL_ISC_REG_MAX    0xbfc
+static const struct regmap_config isc_regmap_config = {
+	.reg_bits       = 32,
+	.reg_stride     = 4,
+	.val_bits       = 32,
+	.max_register	= ATMEL_ISC_REG_MAX,
+};
+
+static int atmel_isc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct isc_device *isc;
+	struct resource *res;
+	void __iomem *io_base;
+	struct isc_subdev_entity *subdev_entity;
+	int irq;
+	int ret;
+
+	isc = devm_kzalloc(dev, sizeof(*isc), GFP_KERNEL);
+	if (!isc)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, isc);
+	isc->dev = dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	io_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(io_base))
+		return PTR_ERR(io_base);
+
+	isc->regmap = devm_regmap_init_mmio(dev, io_base, &isc_regmap_config);
+	if (IS_ERR(isc->regmap)) {
+		ret = PTR_ERR(isc->regmap);
+		dev_err(dev, "failed to init register map: %d\n", ret);
+		return ret;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		ret = irq;
+		dev_err(dev, "failed to get irq: %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_request_irq(dev, irq, isc_interrupt, 0,
+			       ATMEL_ISC_NAME, isc);
+	if (ret < 0) {
+		dev_err(dev, "can't register ISR for IRQ %u (ret=%i)\n",
+			irq, ret);
+		return ret;
+	}
+
+	ret = isc_pipeline_init(isc);
+	if (ret)
+		return ret;
+
+	isc->hclock = devm_clk_get(dev, "hclock");
+	if (IS_ERR(isc->hclock)) {
+		ret = PTR_ERR(isc->hclock);
+		dev_err(dev, "failed to get hclock: %d\n", ret);
+		return ret;
+	}
+
+	ret = isc_clk_init(isc);
+	if (ret) {
+		dev_err(dev, "failed to init isc clock: %d\n", ret);
+		goto clean_isc_clk;
+	}
+
+	isc->ispck = isc->isc_clks[ISC_ISPCK].clk;
+
+	/* ispck should be greater or equal to hclock */
+	ret = clk_set_rate(isc->ispck, clk_get_rate(isc->hclock));
+	if (ret) {
+		dev_err(dev, "failed to set ispck rate: %d\n", ret);
+		goto clean_isc_clk;
+	}
+
+	ret = v4l2_device_register(dev, &isc->v4l2_dev);
+	if (ret) {
+		dev_err(dev, "unable to register v4l2 device.\n");
+		goto clean_isc_clk;
+	}
+
+	ret = isc_parse_dt(dev, isc);
+	if (ret) {
+		dev_err(dev, "fail to parse device tree\n");
+		goto unregister_v4l2_device;
+	}
+
+	if (list_empty(&isc->subdev_entities)) {
+		dev_err(dev, "no subdev found\n");
+		goto unregister_v4l2_device;
+	}
+
+	list_for_each_entry(subdev_entity, &isc->subdev_entities, list) {
+		subdev_entity->notifier.subdevs = &subdev_entity->asd;
+		subdev_entity->notifier.num_subdevs = 1;
+		subdev_entity->notifier.bound = isc_async_bound;
+		subdev_entity->notifier.unbind = isc_async_unbind;
+		subdev_entity->notifier.complete = isc_async_complete;
+
+		ret = v4l2_async_notifier_register(&isc->v4l2_dev,
+						   &subdev_entity->notifier);
+		if (ret) {
+			dev_err(dev, "fail to register async notifier\n");
+			goto cleanup_subdev;
+		}
+
+		if (video_is_registered(&isc->video_dev))
+			break;
+	}
+
+	pm_runtime_enable(dev);
+
+	return 0;
+
+cleanup_subdev:
+	isc_subdev_cleanup(isc);
+
+unregister_v4l2_device:
+	v4l2_device_unregister(&isc->v4l2_dev);
+
+clean_isc_clk:
+	isc_clk_cleanup(isc);
+
+	return ret;
+}
+
+static int atmel_isc_remove(struct platform_device *pdev)
+{
+	struct isc_device *isc = platform_get_drvdata(pdev);
+
+	pm_runtime_disable(&pdev->dev);
+
+	isc_subdev_cleanup(isc);
+
+	v4l2_device_unregister(&isc->v4l2_dev);
+
+	isc_clk_cleanup(isc);
+
+	return 0;
+}
+
+static int __maybe_unused isc_runtime_suspend(struct device *dev)
+{
+	struct isc_device *isc = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(isc->ispck);
+	clk_disable_unprepare(isc->hclock);
+
+	return 0;
+}
+
+static int __maybe_unused isc_runtime_resume(struct device *dev)
+{
+	struct isc_device *isc = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(isc->hclock);
+	if (ret)
+		return ret;
+
+	return clk_prepare_enable(isc->ispck);
+}
+
+static const struct dev_pm_ops atmel_isc_dev_pm_ops = {
+	SET_RUNTIME_PM_OPS(isc_runtime_suspend, isc_runtime_resume, NULL)
+};
+
+static const struct of_device_id atmel_isc_of_match[] = {
+	{ .compatible = "atmel,sama5d2-isc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, atmel_isc_of_match);
+
+static struct platform_driver atmel_isc_driver = {
+	.probe	= atmel_isc_probe,
+	.remove	= atmel_isc_remove,
+	.driver	= {
+		.name		= ATMEL_ISC_NAME,
+		.pm		= &atmel_isc_dev_pm_ops,
+		.of_match_table = of_match_ptr(atmel_isc_of_match),
+	},
+};
+
+module_platform_driver(atmel_isc_driver);
+
+MODULE_AUTHOR("Songjun Wu <songjun.wu@microchip.com>");
+MODULE_DESCRIPTION("The V4L2 driver for Atmel-ISC");
+MODULE_LICENSE("GPL v2");
+MODULE_SUPPORTED_DEVICE("video");
diff --git a/drivers/media/platform/davinci/vpbe_display.c b/drivers/media/platform/davinci/vpbe_display.c
index 0b1709e96673..a9bc0175e4d3 100644
--- a/drivers/media/platform/davinci/vpbe_display.c
+++ b/drivers/media/platform/davinci/vpbe_display.c
@@ -440,7 +440,7 @@ vpbe_disp_calculate_scale_factor(struct vpbe_display *disp_dev,
 	/*
 	 * Application initially set the image format. Current display
 	 * size is obtained from the vpbe display controller. expected_xsize
-	 * and expected_ysize are set through S_CROP ioctl. Based on this,
+	 * and expected_ysize are set through S_SELECTION ioctl. Based on this,
 	 * driver will calculate the scale factors for vertical and
 	 * horizontal direction so that the image is displayed scaled
 	 * and expanded. Application uses expansion to display the image
@@ -649,24 +649,23 @@ static int vpbe_display_querycap(struct file *file, void  *priv,
 	return 0;
 }
 
-static int vpbe_display_s_crop(struct file *file, void *priv,
-			     const struct v4l2_crop *crop)
+static int vpbe_display_s_selection(struct file *file, void *priv,
+			     struct v4l2_selection *sel)
 {
 	struct vpbe_layer *layer = video_drvdata(file);
 	struct vpbe_display *disp_dev = layer->disp_dev;
 	struct vpbe_device *vpbe_dev = disp_dev->vpbe_dev;
 	struct osd_layer_config *cfg = &layer->layer_info.config;
 	struct osd_state *osd_device = disp_dev->osd_device;
-	struct v4l2_rect rect = crop->c;
+	struct v4l2_rect rect = sel->r;
 	int ret;
 
 	v4l2_dbg(1, debug, &vpbe_dev->v4l2_dev,
-		"VIDIOC_S_CROP, layer id = %d\n", layer->device_id);
+		"VIDIOC_S_SELECTION, layer id = %d\n", layer->device_id);
 
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) {
-		v4l2_err(&vpbe_dev->v4l2_dev, "Invalid buf type\n");
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT ||
+	    sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
-	}
 
 	if (rect.top < 0)
 		rect.top = 0;
@@ -714,32 +713,45 @@ static int vpbe_display_s_crop(struct file *file, void *priv,
 	else
 		osd_device->ops.set_interpolation_filter(osd_device, 0);
 
+	sel->r = rect;
 	return 0;
 }
 
-static int vpbe_display_g_crop(struct file *file, void *priv,
-			     struct v4l2_crop *crop)
+static int vpbe_display_g_selection(struct file *file, void *priv,
+				    struct v4l2_selection *sel)
 {
 	struct vpbe_layer *layer = video_drvdata(file);
 	struct osd_layer_config *cfg = &layer->layer_info.config;
 	struct vpbe_device *vpbe_dev = layer->disp_dev->vpbe_dev;
 	struct osd_state *osd_device = layer->disp_dev->osd_device;
-	struct v4l2_rect *rect = &crop->c;
+	struct v4l2_rect *rect = &sel->r;
 
 	v4l2_dbg(1, debug, &vpbe_dev->v4l2_dev,
-			"VIDIOC_G_CROP, layer id = %d\n",
+			"VIDIOC_G_SELECTION, layer id = %d\n",
 			layer->device_id);
 
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) {
-		v4l2_err(&vpbe_dev->v4l2_dev, "Invalid buf type\n");
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+		return -EINVAL;
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		osd_device->ops.get_layer_config(osd_device,
+						 layer->layer_info.id, cfg);
+		rect->top = cfg->ypos;
+		rect->left = cfg->xpos;
+		rect->width = cfg->xsize;
+		rect->height = cfg->ysize;
+		break;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		rect->left = 0;
+		rect->top = 0;
+		rect->width = vpbe_dev->current_timings.xres;
+		rect->height = vpbe_dev->current_timings.yres;
+		break;
+	default:
 		return -EINVAL;
 	}
-	osd_device->ops.get_layer_config(osd_device,
-				layer->layer_info.id, cfg);
-	rect->top = cfg->ypos;
-	rect->left = cfg->xpos;
-	rect->width = cfg->xsize;
-	rect->height = cfg->ysize;
 
 	return 0;
 }
@@ -752,13 +764,10 @@ static int vpbe_display_cropcap(struct file *file, void *priv,
 
 	v4l2_dbg(1, debug, &vpbe_dev->v4l2_dev, "VIDIOC_CROPCAP ioctl\n");
 
-	cropcap->type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
-	cropcap->bounds.left = 0;
-	cropcap->bounds.top = 0;
-	cropcap->bounds.width = vpbe_dev->current_timings.xres;
-	cropcap->bounds.height = vpbe_dev->current_timings.yres;
+	if (cropcap->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+		return -EINVAL;
+
 	cropcap->pixelaspect = vpbe_dev->current_timings.aspect;
-	cropcap->defrect = cropcap->bounds;
 	return 0;
 }
 
@@ -1251,8 +1260,8 @@ static const struct v4l2_ioctl_ops vpbe_ioctl_ops = {
 	.vidioc_expbuf		 = vb2_ioctl_expbuf,
 
 	.vidioc_cropcap		 = vpbe_display_cropcap,
-	.vidioc_g_crop		 = vpbe_display_g_crop,
-	.vidioc_s_crop		 = vpbe_display_s_crop,
+	.vidioc_g_selection	 = vpbe_display_g_selection,
+	.vidioc_s_selection	 = vpbe_display_s_selection,
 
 	.vidioc_s_std		 = vpbe_display_s_std,
 	.vidioc_g_std		 = vpbe_display_g_std,
diff --git a/drivers/media/platform/davinci/vpfe_capture.c b/drivers/media/platform/davinci/vpfe_capture.c
index 7767e072d623..6efb2f1631c4 100644
--- a/drivers/media/platform/davinci/vpfe_capture.c
+++ b/drivers/media/platform/davinci/vpfe_capture.c
@@ -1610,38 +1610,53 @@ static int vpfe_cropcap(struct file *file, void *priv,
 
 	v4l2_dbg(1, debug, &vpfe_dev->v4l2_dev, "vpfe_cropcap\n");
 
-	if (vpfe_dev->std_index >= ARRAY_SIZE(vpfe_standards))
+	if (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
+	/* If std_index is invalid, then just return (== 1:1 aspect) */
+	if (vpfe_dev->std_index >= ARRAY_SIZE(vpfe_standards))
+		return 0;
 
-	memset(crop, 0, sizeof(struct v4l2_cropcap));
-	crop->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	crop->bounds.width = crop->defrect.width =
-		vpfe_standards[vpfe_dev->std_index].width;
-	crop->bounds.height = crop->defrect.height =
-		vpfe_standards[vpfe_dev->std_index].height;
 	crop->pixelaspect = vpfe_standards[vpfe_dev->std_index].pixelaspect;
 	return 0;
 }
 
-static int vpfe_g_crop(struct file *file, void *priv,
-			     struct v4l2_crop *crop)
+static int vpfe_g_selection(struct file *file, void *priv,
+			    struct v4l2_selection *sel)
 {
 	struct vpfe_device *vpfe_dev = video_drvdata(file);
 
-	v4l2_dbg(1, debug, &vpfe_dev->v4l2_dev, "vpfe_g_crop\n");
+	v4l2_dbg(1, debug, &vpfe_dev->v4l2_dev, "vpfe_g_selection\n");
 
-	crop->c = vpfe_dev->crop;
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		sel->r = vpfe_dev->crop;
+		break;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r.width = vpfe_standards[vpfe_dev->std_index].width;
+		sel->r.height = vpfe_standards[vpfe_dev->std_index].height;
+		break;
+	default:
+		return -EINVAL;
+	}
 	return 0;
 }
 
-static int vpfe_s_crop(struct file *file, void *priv,
-			     const struct v4l2_crop *crop)
+static int vpfe_s_selection(struct file *file, void *priv,
+			    struct v4l2_selection *sel)
 {
 	struct vpfe_device *vpfe_dev = video_drvdata(file);
-	struct v4l2_rect rect = crop->c;
+	struct v4l2_rect rect = sel->r;
 	int ret = 0;
 
-	v4l2_dbg(1, debug, &vpfe_dev->v4l2_dev, "vpfe_s_crop\n");
+	v4l2_dbg(1, debug, &vpfe_dev->v4l2_dev, "vpfe_s_selection\n");
+
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
 
 	if (vpfe_dev->started) {
 		/* make sure streaming is not started */
@@ -1669,7 +1684,7 @@ static int vpfe_s_crop(struct file *file, void *priv,
 		vpfe_dev->std_info.active_pixels) ||
 	    (rect.top + rect.height >
 		vpfe_dev->std_info.active_lines)) {
-		v4l2_err(&vpfe_dev->v4l2_dev, "Error in S_CROP params\n");
+		v4l2_err(&vpfe_dev->v4l2_dev, "Error in S_SELECTION params\n");
 		ret = -EINVAL;
 		goto unlock_out;
 	}
@@ -1682,6 +1697,7 @@ static int vpfe_s_crop(struct file *file, void *priv,
 		vpfe_dev->fmt.fmt.pix.bytesperline *
 		vpfe_dev->fmt.fmt.pix.height;
 	vpfe_dev->crop = rect;
+	sel->r = rect;
 unlock_out:
 	mutex_unlock(&vpfe_dev->lock);
 	return ret;
@@ -1760,8 +1776,8 @@ static const struct v4l2_ioctl_ops vpfe_ioctl_ops = {
 	.vidioc_streamon	 = vpfe_streamon,
 	.vidioc_streamoff	 = vpfe_streamoff,
 	.vidioc_cropcap		 = vpfe_cropcap,
-	.vidioc_g_crop		 = vpfe_g_crop,
-	.vidioc_s_crop		 = vpfe_s_crop,
+	.vidioc_g_selection	 = vpfe_g_selection,
+	.vidioc_s_selection	 = vpfe_s_selection,
 	.vidioc_default		 = vpfe_param_handler,
 };
 
diff --git a/drivers/media/platform/exynos-gsc/gsc-m2m.c b/drivers/media/platform/exynos-gsc/gsc-m2m.c
index ec6494cbdd45..9f03b791b711 100644
--- a/drivers/media/platform/exynos-gsc/gsc-m2m.c
+++ b/drivers/media/platform/exynos-gsc/gsc-m2m.c
@@ -261,7 +261,7 @@ static void gsc_m2m_buf_queue(struct vb2_buffer *vb)
 		v4l2_m2m_buf_queue(ctx->m2m_ctx, vbuf);
 }
 
-static struct vb2_ops gsc_m2m_qops = {
+static const struct vb2_ops gsc_m2m_qops = {
 	.queue_setup	 = gsc_m2m_queue_setup,
 	.buf_prepare	 = gsc_m2m_buf_prepare,
 	.buf_queue	 = gsc_m2m_buf_queue,
@@ -277,9 +277,10 @@ static int gsc_m2m_querycap(struct file *file, void *fh,
 	struct gsc_ctx *ctx = fh_to_ctx(fh);
 	struct gsc_dev *gsc = ctx->gsc_dev;
 
-	strlcpy(cap->driver, gsc->pdev->name, sizeof(cap->driver));
-	strlcpy(cap->card, gsc->pdev->name, sizeof(cap->card));
-	strlcpy(cap->bus_info, "platform", sizeof(cap->bus_info));
+	strlcpy(cap->driver, GSC_MODULE_NAME, sizeof(cap->driver));
+	strlcpy(cap->card, GSC_MODULE_NAME " gscaler", sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
+		 dev_name(&gsc->pdev->dev));
 	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M_MPLANE |
 		V4L2_CAP_VIDEO_CAPTURE_MPLANE |	V4L2_CAP_VIDEO_OUTPUT_MPLANE;
 
diff --git a/drivers/media/platform/exynos4-is/fimc-capture.c b/drivers/media/platform/exynos4-is/fimc-capture.c
index fdec499fbbda..964f4a681934 100644
--- a/drivers/media/platform/exynos4-is/fimc-capture.c
+++ b/drivers/media/platform/exynos4-is/fimc-capture.c
@@ -452,7 +452,7 @@ static void buffer_queue(struct vb2_buffer *vb)
 	spin_unlock_irqrestore(&fimc->slock, flags);
 }
 
-static struct vb2_ops fimc_capture_qops = {
+static const struct vb2_ops fimc_capture_qops = {
 	.queue_setup		= queue_setup,
 	.buf_prepare		= buffer_prepare,
 	.buf_queue		= buffer_queue,
@@ -1796,6 +1796,7 @@ static int fimc_register_capture_device(struct fimc_dev *fimc,
 	vid_cap->wb_fmt.code = fmt->mbus_code;
 
 	vid_cap->vd_pad.flags = MEDIA_PAD_FL_SINK;
+	vfd->entity.function = MEDIA_ENT_F_PROC_VIDEO_SCALER;
 	ret = media_entity_pads_init(&vfd->entity, 1, &vid_cap->vd_pad);
 	if (ret)
 		goto err_free_ctx;
diff --git a/drivers/media/platform/exynos4-is/fimc-is-i2c.c b/drivers/media/platform/exynos4-is/fimc-is-i2c.c
index 7521aa59b064..6bba4ca022be 100644
--- a/drivers/media/platform/exynos4-is/fimc-is-i2c.c
+++ b/drivers/media/platform/exynos4-is/fimc-is-i2c.c
@@ -55,26 +55,33 @@ static int fimc_is_i2c_probe(struct platform_device *pdev)
 	i2c_adap->algo = &fimc_is_i2c_algorithm;
 	i2c_adap->class = I2C_CLASS_SPD;
 
-	ret = i2c_add_adapter(i2c_adap);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to add I2C bus %s\n",
-						node->full_name);
-		return ret;
-	}
-
 	platform_set_drvdata(pdev, isp_i2c);
-
 	pm_runtime_enable(&pdev->dev);
-	pm_runtime_enable(&i2c_adap->dev);
 
+	ret = i2c_add_adapter(i2c_adap);
+	if (ret < 0)
+		goto err_pm_dis;
+	/*
+	 * Client drivers of this adapter don't do any I2C transfers as that
+	 * is handled by the ISP firmware.  But we rely on the runtime PM
+	 * state propagation from the clients up to the adapter driver so
+	 * clear the ignore_children flags here.  PM rutnime calls are not
+	 * used in probe() handler of clients of this adapter so there is
+	 * no issues with clearing the flag right after registering the I2C
+	 * adapter.
+	 */
+	pm_suspend_ignore_children(&i2c_adap->dev, false);
 	return 0;
+
+err_pm_dis:
+	pm_runtime_disable(&pdev->dev);
+	return ret;
 }
 
 static int fimc_is_i2c_remove(struct platform_device *pdev)
 {
 	struct fimc_is_i2c *isp_i2c = platform_get_drvdata(pdev);
 
-	pm_runtime_disable(&isp_i2c->adapter.dev);
 	pm_runtime_disable(&pdev->dev);
 	i2c_del_adapter(&isp_i2c->adapter);
 
diff --git a/drivers/media/platform/exynos4-is/fimc-is.c b/drivers/media/platform/exynos4-is/fimc-is.c
index 32ca55f16677..518ad34f80d7 100644
--- a/drivers/media/platform/exynos4-is/fimc-is.c
+++ b/drivers/media/platform/exynos4-is/fimc-is.c
@@ -52,6 +52,9 @@ static char *fimc_is_clocks[ISS_CLKS_MAX] = {
 	[ISS_CLK_DRC]			= "drc",
 	[ISS_CLK_FD]			= "fd",
 	[ISS_CLK_MCUISP]		= "mcuisp",
+	[ISS_CLK_GICISP]		= "gicisp",
+	[ISS_CLK_PWM_ISP]		= "pwm_isp",
+	[ISS_CLK_MCUCTL_ISP]		= "mcuctl_isp",
 	[ISS_CLK_UART]			= "uart",
 	[ISS_CLK_ISP_DIV0]		= "ispdiv0",
 	[ISS_CLK_ISP_DIV1]		= "ispdiv1",
@@ -165,6 +168,7 @@ static int fimc_is_parse_sensor_config(struct fimc_is *is, unsigned int index,
 						struct device_node *node)
 {
 	struct fimc_is_sensor *sensor = &is->sensor[index];
+	struct device_node *ep, *port;
 	u32 tmp = 0;
 	int ret;
 
@@ -175,22 +179,25 @@ static int fimc_is_parse_sensor_config(struct fimc_is *is, unsigned int index,
 		return -EINVAL;
 	}
 
-	node = of_graph_get_next_endpoint(node, NULL);
-	if (!node)
+	ep = of_graph_get_next_endpoint(node, NULL);
+	if (!ep)
 		return -ENXIO;
 
-	node = of_graph_get_remote_port(node);
-	if (!node)
+	port = of_graph_get_remote_port(ep);
+	of_node_put(ep);
+	if (!port)
 		return -ENXIO;
 
 	/* Use MIPI-CSIS channel id to determine the ISP I2C bus index. */
-	ret = of_property_read_u32(node, "reg", &tmp);
+	ret = of_property_read_u32(port, "reg", &tmp);
 	if (ret < 0) {
 		dev_err(&is->pdev->dev, "reg property not found at: %s\n",
-							 node->full_name);
+							 port->full_name);
+		of_node_put(port);
 		return ret;
 	}
 
+	of_node_put(port);
 	sensor->i2c_bus = tmp - FIMC_INPUT_MIPI_CSI2_0;
 	return 0;
 }
@@ -845,13 +852,18 @@ static int fimc_is_probe(struct platform_device *pdev)
 		goto err_pm;
 
 	vb2_dma_contig_set_max_seg_size(dev, DMA_BIT_MASK(32));
+
+	ret = of_platform_populate(dev->of_node, NULL, NULL, dev);
+	if (ret < 0)
+		goto err_pm;
+
 	/*
 	 * Register FIMC-IS V4L2 subdevs to this driver. The video nodes
 	 * will be created within the subdev's registered() callback.
 	 */
 	ret = fimc_is_register_subdevs(is);
 	if (ret < 0)
-		goto err_pm;
+		goto err_of_dep;
 
 	ret = fimc_is_debugfs_create(is);
 	if (ret < 0)
@@ -870,6 +882,8 @@ err_dfs:
 	fimc_is_debugfs_remove(is);
 err_sd:
 	fimc_is_unregister_subdevs(is);
+err_of_dep:
+	of_platform_depopulate(dev);
 err_pm:
 	if (!pm_runtime_enabled(dev))
 		fimc_is_runtime_suspend(dev);
@@ -929,6 +943,7 @@ static int fimc_is_remove(struct platform_device *pdev)
 	if (!pm_runtime_status_suspended(dev))
 		fimc_is_runtime_suspend(dev);
 	free_irq(is->irq, is);
+	of_platform_depopulate(dev);
 	fimc_is_unregister_subdevs(is);
 	vb2_dma_contig_clear_max_seg_size(dev);
 	fimc_is_put_clocks(is);
diff --git a/drivers/media/platform/exynos4-is/fimc-is.h b/drivers/media/platform/exynos4-is/fimc-is.h
index 3a82c6a214c7..ee05da034aa1 100644
--- a/drivers/media/platform/exynos4-is/fimc-is.h
+++ b/drivers/media/platform/exynos4-is/fimc-is.h
@@ -77,6 +77,9 @@ enum {
 	ISS_CLK_DRC,
 	ISS_CLK_FD,
 	ISS_CLK_MCUISP,
+	ISS_CLK_GICISP,
+	ISS_CLK_PWM_ISP,
+	ISS_CLK_MCUCTL_ISP,
 	ISS_CLK_UART,
 	ISS_GATE_CLKS_MAX,
 	ISS_CLK_ISP_DIV0 = ISS_GATE_CLKS_MAX,
diff --git a/drivers/media/platform/exynos4-is/fimc-isp.c b/drivers/media/platform/exynos4-is/fimc-isp.c
index 293b807020c4..8efe9160ab34 100644
--- a/drivers/media/platform/exynos4-is/fimc-isp.c
+++ b/drivers/media/platform/exynos4-is/fimc-isp.c
@@ -705,6 +705,7 @@ int fimc_isp_subdev_create(struct fimc_isp *isp)
 	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 	snprintf(sd->name, sizeof(sd->name), "FIMC-IS-ISP");
 
+	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
 	isp->subdev_pads[FIMC_ISP_SD_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
 	isp->subdev_pads[FIMC_ISP_SD_PAD_SRC_FIFO].flags = MEDIA_PAD_FL_SOURCE;
 	isp->subdev_pads[FIMC_ISP_SD_PAD_SRC_DMA].flags = MEDIA_PAD_FL_SOURCE;
diff --git a/drivers/media/platform/exynos4-is/fimc-lite.c b/drivers/media/platform/exynos4-is/fimc-lite.c
index a0f149fb88e1..b91abf1c4d43 100644
--- a/drivers/media/platform/exynos4-is/fimc-lite.c
+++ b/drivers/media/platform/exynos4-is/fimc-lite.c
@@ -1432,6 +1432,7 @@ static int fimc_lite_create_capture_subdev(struct fimc_lite *fimc)
 
 	sd->ctrl_handler = handler;
 	sd->internal_ops = &fimc_lite_subdev_internal_ops;
+	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_SCALER;
 	sd->entity.ops = &fimc_lite_subdev_media_ops;
 	sd->owner = THIS_MODULE;
 	v4l2_set_subdevdata(sd, fimc);
@@ -1454,25 +1455,17 @@ static void fimc_lite_clk_put(struct fimc_lite *fimc)
 	if (IS_ERR(fimc->clock))
 		return;
 
-	clk_unprepare(fimc->clock);
 	clk_put(fimc->clock);
 	fimc->clock = ERR_PTR(-EINVAL);
 }
 
 static int fimc_lite_clk_get(struct fimc_lite *fimc)
 {
-	int ret;
-
 	fimc->clock = clk_get(&fimc->pdev->dev, FLITE_CLK_NAME);
 	if (IS_ERR(fimc->clock))
 		return PTR_ERR(fimc->clock);
 
-	ret = clk_prepare(fimc->clock);
-	if (ret < 0) {
-		clk_put(fimc->clock);
-		fimc->clock = ERR_PTR(-EINVAL);
-	}
-	return ret;
+	return 0;
 }
 
 static const struct of_device_id flite_of_match[];
@@ -1543,7 +1536,7 @@ static int fimc_lite_probe(struct platform_device *pdev)
 	pm_runtime_enable(dev);
 
 	if (!pm_runtime_enabled(dev)) {
-		ret = clk_enable(fimc->clock);
+		ret = clk_prepare_enable(fimc->clock);
 		if (ret < 0)
 			goto err_sd;
 	}
@@ -1568,7 +1561,7 @@ static int fimc_lite_runtime_resume(struct device *dev)
 {
 	struct fimc_lite *fimc = dev_get_drvdata(dev);
 
-	clk_enable(fimc->clock);
+	clk_prepare_enable(fimc->clock);
 	return 0;
 }
 
@@ -1576,7 +1569,7 @@ static int fimc_lite_runtime_suspend(struct device *dev)
 {
 	struct fimc_lite *fimc = dev_get_drvdata(dev);
 
-	clk_disable(fimc->clock);
+	clk_disable_unprepare(fimc->clock);
 	return 0;
 }
 #endif
diff --git a/drivers/media/platform/exynos4-is/fimc-m2m.c b/drivers/media/platform/exynos4-is/fimc-m2m.c
index b1309e114edb..6028e4fbaed3 100644
--- a/drivers/media/platform/exynos4-is/fimc-m2m.c
+++ b/drivers/media/platform/exynos4-is/fimc-m2m.c
@@ -219,7 +219,7 @@ static void fimc_buf_queue(struct vb2_buffer *vb)
 	v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
 }
 
-static struct vb2_ops fimc_qops = {
+static const struct vb2_ops fimc_qops = {
 	.queue_setup	 = fimc_queue_setup,
 	.buf_prepare	 = fimc_buf_prepare,
 	.buf_queue	 = fimc_buf_queue,
diff --git a/drivers/media/platform/exynos4-is/media-dev.c b/drivers/media/platform/exynos4-is/media-dev.c
index 891625e77ef5..1a1154a9dfa4 100644
--- a/drivers/media/platform/exynos4-is/media-dev.c
+++ b/drivers/media/platform/exynos4-is/media-dev.c
@@ -1190,6 +1190,10 @@ static int fimc_md_link_notify(struct media_link *link, unsigned int flags,
 	return ret ? -EPIPE : 0;
 }
 
+static const struct media_device_ops fimc_md_ops = {
+	.link_notify = fimc_md_link_notify,
+};
+
 static ssize_t fimc_md_sysfs_show(struct device *dev,
 				  struct device_attribute *attr, char *buf)
 {
@@ -1416,7 +1420,7 @@ static int fimc_md_probe(struct platform_device *pdev)
 
 	strlcpy(fmd->media_dev.model, "SAMSUNG S5P FIMC",
 		sizeof(fmd->media_dev.model));
-	fmd->media_dev.link_notify = fimc_md_link_notify;
+	fmd->media_dev.ops = &fimc_md_ops;
 	fmd->media_dev.dev = dev;
 
 	v4l2_dev = &fmd->v4l2_dev;
diff --git a/drivers/media/platform/exynos4-is/mipi-csis.c b/drivers/media/platform/exynos4-is/mipi-csis.c
index 86e681daa89d..befd9fc0adc4 100644
--- a/drivers/media/platform/exynos4-is/mipi-csis.c
+++ b/drivers/media/platform/exynos4-is/mipi-csis.c
@@ -853,6 +853,7 @@ static int s5pcsis_probe(struct platform_device *pdev)
 	state->format.width = S5PCSIS_DEF_PIX_WIDTH;
 	state->format.height = S5PCSIS_DEF_PIX_HEIGHT;
 
+	state->sd.entity.function = MEDIA_ENT_F_IO_V4L;
 	state->pads[CSIS_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
 	state->pads[CSIS_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
 	ret = media_entity_pads_init(&state->sd.entity,
diff --git a/drivers/media/platform/m2m-deinterlace.c b/drivers/media/platform/m2m-deinterlace.c
index 0fcb5c78031d..bedc7cc4c7d6 100644
--- a/drivers/media/platform/m2m-deinterlace.c
+++ b/drivers/media/platform/m2m-deinterlace.c
@@ -852,7 +852,7 @@ static void deinterlace_buf_queue(struct vb2_buffer *vb)
 	v4l2_m2m_buf_queue(ctx->m2m_ctx, vbuf);
 }
 
-static struct vb2_ops deinterlace_qops = {
+static const struct vb2_ops deinterlace_qops = {
 	.queue_setup	 = deinterlace_queue_setup,
 	.buf_prepare	 = deinterlace_buf_prepare,
 	.buf_queue	 = deinterlace_buf_queue,
@@ -1016,7 +1016,7 @@ static int deinterlace_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	if (!dma_has_cap(DMA_INTERLEAVE, pcdev->dma_chan->device->cap_mask)) {
-		v4l2_err(&pcdev->v4l2_dev, "DMA does not support INTERLEAVE\n");
+		dev_err(&pdev->dev, "DMA does not support INTERLEAVE\n");
 		goto rel_dma;
 	}
 
diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h
index 94f0a425be42..c8eaa41c00e6 100644
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h
@@ -23,7 +23,6 @@
 #include <media/v4l2-ioctl.h>
 #include <media/videobuf2-core.h>
 
-#include "mtk_vcodec_util.h"
 
 #define MTK_VCODEC_DRV_NAME	"mtk_vcodec_drv"
 #define MTK_VCODEC_ENC_NAME	"mtk-vcodec-enc"
@@ -240,7 +239,7 @@ struct mtk_vcodec_ctx {
 	enum mtk_encode_param param_change;
 	struct mtk_enc_params enc_params;
 
-	struct venc_common_if *enc_if;
+	const struct venc_common_if *enc_if;
 	unsigned long drv_handle;
 
 	int int_cond;
diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c
index 3ed3f2d31df5..1b1a28abbf1f 100644
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c
@@ -243,6 +243,8 @@ static int vidioc_venc_s_parm(struct file *file, void *priv,
 			a->parm.output.timeperframe.numerator;
 	ctx->param_change |= MTK_ENCODE_PARAM_FRAMERATE;
 
+	a->parm.output.capability = V4L2_CAP_TIMEPERFRAME;
+
 	return 0;
 }
 
@@ -254,6 +256,7 @@ static int vidioc_venc_g_parm(struct file *file, void *priv,
 	if (a->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
 		return -EINVAL;
 
+	a->parm.output.capability = V4L2_CAP_TIMEPERFRAME;
 	a->parm.output.timeperframe.denominator =
 			ctx->enc_params.framerate_num;
 	a->parm.output.timeperframe.numerator =
@@ -487,7 +490,6 @@ static int vidioc_venc_s_fmt_out(struct file *file, void *priv,
 	struct mtk_q_data *q_data;
 	int ret, i;
 	struct mtk_video_fmt *fmt;
-	unsigned int pitch_w_div16;
 	struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp;
 
 	vq = v4l2_m2m_get_vq(ctx->m2m_ctx, f->type);
@@ -530,15 +532,6 @@ static int vidioc_venc_s_fmt_out(struct file *file, void *priv,
 	q_data->coded_width = f->fmt.pix_mp.width;
 	q_data->coded_height = f->fmt.pix_mp.height;
 
-	pitch_w_div16 = DIV_ROUND_UP(q_data->visible_width, 16);
-	if (pitch_w_div16 % 8 != 0) {
-		/* Adjust returned width/height, so application could correctly
-		 * allocate hw required memory
-		 */
-		q_data->visible_height += 32;
-		vidioc_try_fmt(f, q_data->fmt);
-	}
-
 	q_data->field = f->fmt.pix_mp.field;
 	ctx->colorspace = f->fmt.pix_mp.colorspace;
 	ctx->ycbcr_enc = f->fmt.pix_mp.ycbcr_enc;
@@ -631,6 +624,69 @@ static int vidioc_try_fmt_vid_out_mplane(struct file *file, void *priv,
 	return vidioc_try_fmt(f, fmt);
 }
 
+static int vidioc_venc_g_selection(struct file *file, void *priv,
+				     struct v4l2_selection *s)
+{
+	struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv);
+	struct mtk_q_data *q_data;
+
+	if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+		return -EINVAL;
+
+	q_data = mtk_venc_get_q_data(ctx, s->type);
+	if (!q_data)
+		return -EINVAL;
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		s->r.top = 0;
+		s->r.left = 0;
+		s->r.width = q_data->coded_width;
+		s->r.height = q_data->coded_height;
+		break;
+	case V4L2_SEL_TGT_CROP:
+		s->r.top = 0;
+		s->r.left = 0;
+		s->r.width = q_data->visible_width;
+		s->r.height = q_data->visible_height;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int vidioc_venc_s_selection(struct file *file, void *priv,
+				     struct v4l2_selection *s)
+{
+	struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv);
+	struct mtk_q_data *q_data;
+
+	if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+		return -EINVAL;
+
+	q_data = mtk_venc_get_q_data(ctx, s->type);
+	if (!q_data)
+		return -EINVAL;
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_CROP:
+		/* Only support crop from (0,0) */
+		s->r.top = 0;
+		s->r.left = 0;
+		s->r.width = min(s->r.width, q_data->coded_width);
+		s->r.height = min(s->r.height, q_data->coded_height);
+		q_data->visible_width = s->r.width;
+		q_data->visible_height = s->r.height;
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
 static int vidioc_venc_qbuf(struct file *file, void *priv,
 			    struct v4l2_buffer *buf)
 {
@@ -689,6 +745,9 @@ const struct v4l2_ioctl_ops mtk_venc_ioctl_ops = {
 
 	.vidioc_create_bufs		= v4l2_m2m_ioctl_create_bufs,
 	.vidioc_prepare_buf		= v4l2_m2m_ioctl_prepare_buf,
+
+	.vidioc_g_selection		= vidioc_venc_g_selection,
+	.vidioc_s_selection		= vidioc_venc_s_selection,
 };
 
 static int vb2ops_venc_queue_setup(struct vb2_queue *vq,
@@ -864,7 +923,7 @@ static void vb2ops_venc_stop_streaming(struct vb2_queue *q)
 	ctx->state = MTK_STATE_FREE;
 }
 
-static struct vb2_ops mtk_venc_vb2_ops = {
+static const struct vb2_ops mtk_venc_vb2_ops = {
 	.queue_setup		= vb2ops_venc_queue_setup,
 	.buf_prepare		= vb2ops_venc_buf_prepare,
 	.buf_queue		= vb2ops_venc_buf_queue,
@@ -878,7 +937,8 @@ static int mtk_venc_encode_header(void *priv)
 {
 	struct mtk_vcodec_ctx *ctx = priv;
 	int ret;
-	struct vb2_buffer *dst_buf;
+	struct vb2_buffer *src_buf, *dst_buf;
+	struct vb2_v4l2_buffer *dst_vb2_v4l2, *src_vb2_v4l2;
 	struct mtk_vcodec_mem bs_buf;
 	struct venc_done_result enc_result;
 
@@ -911,6 +971,15 @@ static int mtk_venc_encode_header(void *priv)
 		mtk_v4l2_err("venc_if_encode failed=%d", ret);
 		return -EINVAL;
 	}
+	src_buf = v4l2_m2m_next_src_buf(ctx->m2m_ctx);
+	if (src_buf) {
+		src_vb2_v4l2 = to_vb2_v4l2_buffer(src_buf);
+		dst_vb2_v4l2 = to_vb2_v4l2_buffer(dst_buf);
+		dst_buf->timestamp = src_buf->timestamp;
+		dst_vb2_v4l2->timecode = src_vb2_v4l2->timecode;
+	} else {
+		mtk_v4l2_err("No timestamp for the header buffer.");
+	}
 
 	ctx->state = MTK_STATE_HEADER;
 	dst_buf->planes[0].bytesused = enc_result.bs_size;
@@ -1003,7 +1072,7 @@ static void mtk_venc_worker(struct work_struct *work)
 	struct mtk_vcodec_mem bs_buf;
 	struct venc_done_result enc_result;
 	int ret, i;
-	struct vb2_v4l2_buffer *vb2_v4l2;
+	struct vb2_v4l2_buffer *dst_vb2_v4l2, *src_vb2_v4l2;
 
 	/* check dst_buf, dst_buf may be removed in device_run
 	 * to stored encdoe header so we need check dst_buf and
@@ -1043,9 +1112,14 @@ static void mtk_venc_worker(struct work_struct *work)
 	ret = venc_if_encode(ctx, VENC_START_OPT_ENCODE_FRAME,
 			     &frm_buf, &bs_buf, &enc_result);
 
-	vb2_v4l2 = container_of(dst_buf, struct vb2_v4l2_buffer, vb2_buf);
+	src_vb2_v4l2 = to_vb2_v4l2_buffer(src_buf);
+	dst_vb2_v4l2 = to_vb2_v4l2_buffer(dst_buf);
+
+	dst_buf->timestamp = src_buf->timestamp;
+	dst_vb2_v4l2->timecode = src_vb2_v4l2->timecode;
+
 	if (enc_result.is_key_frm)
-		vb2_v4l2->flags |= V4L2_BUF_FLAG_KEYFRAME;
+		dst_vb2_v4l2->flags |= V4L2_BUF_FLAG_KEYFRAME;
 
 	if (ret) {
 		v4l2_m2m_buf_done(to_vb2_v4l2_buffer(src_buf),
@@ -1217,7 +1291,7 @@ int mtk_vcodec_enc_ctrls_setup(struct mtk_vcodec_ctx *ctx)
 			0, V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE);
 	v4l2_ctrl_new_std_menu(handler, ops, V4L2_CID_MPEG_VIDEO_H264_PROFILE,
 			V4L2_MPEG_VIDEO_H264_PROFILE_HIGH,
-			0, V4L2_MPEG_VIDEO_H264_PROFILE_MAIN);
+			0, V4L2_MPEG_VIDEO_H264_PROFILE_HIGH);
 	v4l2_ctrl_new_std_menu(handler, ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL,
 			V4L2_MPEG_VIDEO_H264_LEVEL_4_2,
 			0, V4L2_MPEG_VIDEO_H264_LEVEL_4_0);
@@ -1288,5 +1362,10 @@ int mtk_venc_lock(struct mtk_vcodec_ctx *ctx)
 
 void mtk_vcodec_enc_release(struct mtk_vcodec_ctx *ctx)
 {
-	venc_if_deinit(ctx);
+	int ret = venc_if_deinit(ctx);
+
+	if (ret)
+		mtk_v4l2_err("venc_if_deinit failed=%d", ret);
+
+	ctx->state = MTK_STATE_FREE;
 }
diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c
index c7806ecda2dd..5cd2151431bf 100644
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c
@@ -218,11 +218,15 @@ static int fops_vcodec_release(struct file *file)
 	mtk_v4l2_debug(1, "[%d] encoder", ctx->id);
 	mutex_lock(&dev->dev_mutex);
 
+	/*
+	 * Call v4l2_m2m_ctx_release to make sure the worker thread is not
+	 * running after venc_if_deinit.
+	 */
+	v4l2_m2m_ctx_release(ctx->m2m_ctx);
 	mtk_vcodec_enc_release(ctx);
 	v4l2_fh_del(&ctx->fh);
 	v4l2_fh_exit(&ctx->fh);
 	v4l2_ctrl_handler_free(&ctx->ctrl_hdl);
-	v4l2_m2m_ctx_release(ctx->m2m_ctx);
 
 	list_del_init(&ctx->list);
 	dev->num_instances--;
diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.h
index 33e890f5aa9c..12131855b46a 100644
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.h
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.h
@@ -16,7 +16,6 @@
 #define _MTK_VCODEC_INTR_H_
 
 #define MTK_INST_IRQ_RECEIVED		0x1
-#define MTK_INST_WORK_THREAD_ABORT_DONE	0x2
 
 struct mtk_vcodec_ctx;
 
diff --git a/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c b/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c
index 9a600525b3c1..b76c80bdf30b 100644
--- a/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c
+++ b/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c
@@ -61,6 +61,8 @@ enum venc_h264_bs_mode {
 
 /*
  * struct venc_h264_vpu_config - Structure for h264 encoder configuration
+ *                               AP-W/R : AP is writer/reader on this item
+ *                               VPU-W/R: VPU is write/reader on this item
  * @input_fourcc: input fourcc
  * @bitrate: target bitrate (in bps)
  * @pic_w: picture width. Picture size is visible stream resolution, in pixels,
@@ -94,13 +96,13 @@ struct venc_h264_vpu_config {
 
 /*
  * struct venc_h264_vpu_buf - Structure for buffer information
- * @align: buffer alignment (in bytes)
+ *                            AP-W/R : AP is writer/reader on this item
+ *                            VPU-W/R: VPU is write/reader on this item
  * @iova: IO virtual address
  * @vpua: VPU side memory addr which is used by RC_CODE
  * @size: buffer size (in bytes)
  */
 struct venc_h264_vpu_buf {
-	u32 align;
 	u32 iova;
 	u32 vpua;
 	u32 size;
@@ -108,6 +110,8 @@ struct venc_h264_vpu_buf {
 
 /*
  * struct venc_h264_vsi - Structure for VPU driver control and info share
+ *                        AP-W/R : AP is writer/reader on this item
+ *                        VPU-W/R: VPU is write/reader on this item
  * This structure is allocated in VPU side and shared to AP side.
  * @config: h264 encoder configuration
  * @work_bufs: working buffer information in VPU side
@@ -150,12 +154,6 @@ struct venc_h264_inst {
 	struct mtk_vcodec_ctx *ctx;
 };
 
-static inline void h264_write_reg(struct venc_h264_inst *inst, u32 addr,
-				  u32 val)
-{
-	writel(val, inst->hw_base + addr);
-}
-
 static inline u32 h264_read_reg(struct venc_h264_inst *inst, u32 addr)
 {
 	return readl(inst->hw_base + addr);
@@ -214,6 +212,8 @@ static unsigned int h264_get_level(struct venc_h264_inst *inst,
 		return 40;
 	case V4L2_MPEG_VIDEO_H264_LEVEL_4_1:
 		return 41;
+	case V4L2_MPEG_VIDEO_H264_LEVEL_4_2:
+		return 42;
 	default:
 		mtk_vcodec_debug(inst, "unsupported level %d", level);
 		return 31;
@@ -664,16 +664,16 @@ static int h264_enc_deinit(unsigned long handle)
 	return ret;
 }
 
-static struct venc_common_if venc_h264_if = {
+static const struct venc_common_if venc_h264_if = {
 	h264_enc_init,
 	h264_enc_encode,
 	h264_enc_set_param,
 	h264_enc_deinit,
 };
 
-struct venc_common_if *get_h264_enc_comm_if(void);
+const struct venc_common_if *get_h264_enc_comm_if(void);
 
-struct venc_common_if *get_h264_enc_comm_if(void)
+const struct venc_common_if *get_h264_enc_comm_if(void)
 {
 	return &venc_h264_if;
 }
diff --git a/drivers/media/platform/mtk-vcodec/venc/venc_vp8_if.c b/drivers/media/platform/mtk-vcodec/venc/venc_vp8_if.c
index 60bbcd2a0510..544f57186243 100644
--- a/drivers/media/platform/mtk-vcodec/venc/venc_vp8_if.c
+++ b/drivers/media/platform/mtk-vcodec/venc/venc_vp8_if.c
@@ -56,6 +56,8 @@ enum venc_vp8_vpu_work_buf {
 
 /*
  * struct venc_vp8_vpu_config - Structure for vp8 encoder configuration
+ *                              AP-W/R : AP is writer/reader on this item
+ *                              VPU-W/R: VPU is write/reader on this item
  * @input_fourcc: input fourcc
  * @bitrate: target bitrate (in bps)
  * @pic_w: picture width. Picture size is visible stream resolution, in pixels,
@@ -83,14 +85,14 @@ struct venc_vp8_vpu_config {
 };
 
 /*
- * struct venc_vp8_vpu_buf -Structure for buffer information
- * @align: buffer alignment (in bytes)
+ * struct venc_vp8_vpu_buf - Structure for buffer information
+ *                           AP-W/R : AP is writer/reader on this item
+ *                           VPU-W/R: VPU is write/reader on this item
  * @iova: IO virtual address
  * @vpua: VPU side memory addr which is used by RC_CODE
  * @size: buffer size (in bytes)
  */
 struct venc_vp8_vpu_buf {
-	u32 align;
 	u32 iova;
 	u32 vpua;
 	u32 size;
@@ -98,6 +100,8 @@ struct venc_vp8_vpu_buf {
 
 /*
  * struct venc_vp8_vsi - Structure for VPU driver control and info share
+ *                       AP-W/R : AP is writer/reader on this item
+ *                       VPU-W/R: VPU is write/reader on this item
  * This structure is allocated in VPU side and shared to AP side.
  * @config: vp8 encoder configuration
  * @work_bufs: working buffer information in VPU side
@@ -138,12 +142,6 @@ struct venc_vp8_inst {
 	struct mtk_vcodec_ctx *ctx;
 };
 
-static inline void vp8_enc_write_reg(struct venc_vp8_inst *inst, u32 addr,
-				     u32 val)
-{
-	writel(val, inst->hw_base + addr);
-}
-
 static inline u32 vp8_enc_read_reg(struct venc_vp8_inst *inst, u32 addr)
 {
 	return readl(inst->hw_base + addr);
@@ -471,16 +469,16 @@ static int vp8_enc_deinit(unsigned long handle)
 	return ret;
 }
 
-static struct venc_common_if venc_vp8_if = {
+static const struct venc_common_if venc_vp8_if = {
 	vp8_enc_init,
 	vp8_enc_encode,
 	vp8_enc_set_param,
 	vp8_enc_deinit,
 };
 
-struct venc_common_if *get_vp8_enc_comm_if(void);
+const struct venc_common_if *get_vp8_enc_comm_if(void);
 
-struct venc_common_if *get_vp8_enc_comm_if(void)
+const struct venc_common_if *get_vp8_enc_comm_if(void)
 {
 	return &venc_vp8_if;
 }
diff --git a/drivers/media/platform/mtk-vcodec/venc_drv_if.c b/drivers/media/platform/mtk-vcodec/venc_drv_if.c
index c4c83e7189c3..d02d5f1df279 100644
--- a/drivers/media/platform/mtk-vcodec/venc_drv_if.c
+++ b/drivers/media/platform/mtk-vcodec/venc_drv_if.c
@@ -26,8 +26,8 @@
 #include "mtk_vcodec_enc_pm.h"
 #include "mtk_vpu.h"
 
-struct venc_common_if *get_h264_enc_comm_if(void);
-struct venc_common_if *get_vp8_enc_comm_if(void);
+const struct venc_common_if *get_h264_enc_comm_if(void);
+const struct venc_common_if *get_vp8_enc_comm_if(void);
 
 int venc_if_init(struct mtk_vcodec_ctx *ctx, unsigned int fourcc)
 {
diff --git a/drivers/media/platform/mx2_emmaprp.c b/drivers/media/platform/mx2_emmaprp.c
index c639406fe72e..e68d271b10af 100644
--- a/drivers/media/platform/mx2_emmaprp.c
+++ b/drivers/media/platform/mx2_emmaprp.c
@@ -743,7 +743,7 @@ static void emmaprp_buf_queue(struct vb2_buffer *vb)
 	v4l2_m2m_buf_queue(ctx->m2m_ctx, vbuf);
 }
 
-static struct vb2_ops emmaprp_qops = {
+static const struct vb2_ops emmaprp_qops = {
 	.queue_setup	 = emmaprp_queue_setup,
 	.buf_prepare	 = emmaprp_buf_prepare,
 	.buf_queue	 = emmaprp_buf_queue,
diff --git a/drivers/media/platform/omap/omap_vout.c b/drivers/media/platform/omap/omap_vout.c
index 6b01e126fe73..e668dde6d857 100644
--- a/drivers/media/platform/omap/omap_vout.c
+++ b/drivers/media/platform/omap/omap_vout.c
@@ -1247,36 +1247,33 @@ static int vidioc_g_fmt_vid_overlay(struct file *file, void *fh,
 	return 0;
 }
 
-static int vidioc_cropcap(struct file *file, void *fh,
-		struct v4l2_cropcap *cropcap)
+static int vidioc_g_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 {
 	struct omap_vout_device *vout = fh;
 	struct v4l2_pix_format *pix = &vout->pix;
 
-	if (cropcap->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
 		return -EINVAL;
 
-	/* Width and height are always even */
-	cropcap->bounds.width = pix->width & ~1;
-	cropcap->bounds.height = pix->height & ~1;
-
-	omap_vout_default_crop(&vout->pix, &vout->fbuf, &cropcap->defrect);
-	cropcap->pixelaspect.numerator = 1;
-	cropcap->pixelaspect.denominator = 1;
-	return 0;
-}
-
-static int vidioc_g_crop(struct file *file, void *fh, struct v4l2_crop *crop)
-{
-	struct omap_vout_device *vout = fh;
-
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		sel->r = vout->crop;
+		break;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		omap_vout_default_crop(&vout->pix, &vout->fbuf, &sel->r);
+		break;
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		/* Width and height are always even */
+		sel->r.width = pix->width & ~1;
+		sel->r.height = pix->height & ~1;
+		break;
+	default:
 		return -EINVAL;
-	crop->c = vout->crop;
+	}
 	return 0;
 }
 
-static int vidioc_s_crop(struct file *file, void *fh, const struct v4l2_crop *crop)
+static int vidioc_s_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 {
 	int ret = -EINVAL;
 	struct omap_vout_device *vout = fh;
@@ -1285,6 +1282,12 @@ static int vidioc_s_crop(struct file *file, void *fh, const struct v4l2_crop *cr
 	struct omap_video_timings *timing;
 	struct omap_dss_device *dssdev;
 
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+		return -EINVAL;
+
+	if (sel->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
+
 	if (vout->streaming)
 		return -EBUSY;
 
@@ -1309,9 +1312,8 @@ static int vidioc_s_crop(struct file *file, void *fh, const struct v4l2_crop *cr
 		vout->fbuf.fmt.width = timing->x_res;
 	}
 
-	if (crop->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
-		ret = omap_vout_new_crop(&vout->pix, &vout->crop, &vout->win,
-				&vout->fbuf, &crop->c);
+	ret = omap_vout_new_crop(&vout->pix, &vout->crop, &vout->win,
+				 &vout->fbuf, &sel->r);
 
 s_crop_err:
 	mutex_unlock(&vout->lock);
@@ -1780,9 +1782,8 @@ static const struct v4l2_ioctl_ops vout_ioctl_ops = {
 	.vidioc_try_fmt_vid_out_overlay		= vidioc_try_fmt_vid_overlay,
 	.vidioc_s_fmt_vid_out_overlay		= vidioc_s_fmt_vid_overlay,
 	.vidioc_g_fmt_vid_out_overlay		= vidioc_g_fmt_vid_overlay,
-	.vidioc_cropcap				= vidioc_cropcap,
-	.vidioc_g_crop				= vidioc_g_crop,
-	.vidioc_s_crop				= vidioc_s_crop,
+	.vidioc_g_selection			= vidioc_g_selection,
+	.vidioc_s_selection			= vidioc_s_selection,
 	.vidioc_reqbufs				= vidioc_reqbufs,
 	.vidioc_querybuf			= vidioc_querybuf,
 	.vidioc_qbuf				= vidioc_qbuf,
diff --git a/drivers/media/platform/omap3isp/isp.c b/drivers/media/platform/omap3isp/isp.c
index 5d54e2c6c16b..0321d84addc7 100644
--- a/drivers/media/platform/omap3isp/isp.c
+++ b/drivers/media/platform/omap3isp/isp.c
@@ -657,6 +657,10 @@ static irqreturn_t isp_isr(int irq, void *_isp)
 	return IRQ_HANDLED;
 }
 
+static const struct media_device_ops isp_media_ops = {
+	.link_notify = v4l2_pipeline_link_notify,
+};
+
 /* -----------------------------------------------------------------------------
  * Pipeline stream management
  */
@@ -1680,7 +1684,7 @@ static int isp_register_entities(struct isp_device *isp)
 	strlcpy(isp->media_dev.model, "TI OMAP3 ISP",
 		sizeof(isp->media_dev.model));
 	isp->media_dev.hw_revision = isp->revision;
-	isp->media_dev.link_notify = v4l2_pipeline_link_notify;
+	isp->media_dev.ops = &isp_media_ops;
 	media_device_init(&isp->media_dev);
 
 	isp->v4l2_dev.mdev = &isp->media_dev;
diff --git a/drivers/media/platform/omap3isp/ispvideo.c b/drivers/media/platform/omap3isp/ispvideo.c
index 7d9f35976d18..7354469670b7 100644
--- a/drivers/media/platform/omap3isp/ispvideo.c
+++ b/drivers/media/platform/omap3isp/ispvideo.c
@@ -772,40 +772,45 @@ isp_video_try_format(struct file *file, void *fh, struct v4l2_format *format)
 }
 
 static int
-isp_video_cropcap(struct file *file, void *fh, struct v4l2_cropcap *cropcap)
-{
-	struct isp_video *video = video_drvdata(file);
-	struct v4l2_subdev *subdev;
-	int ret;
-
-	subdev = isp_video_remote_subdev(video, NULL);
-	if (subdev == NULL)
-		return -EINVAL;
-
-	mutex_lock(&video->mutex);
-	ret = v4l2_subdev_call(subdev, video, cropcap, cropcap);
-	mutex_unlock(&video->mutex);
-
-	return ret == -ENOIOCTLCMD ? -ENOTTY : ret;
-}
-
-static int
-isp_video_get_crop(struct file *file, void *fh, struct v4l2_crop *crop)
+isp_video_get_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 {
 	struct isp_video *video = video_drvdata(file);
 	struct v4l2_subdev_format format;
 	struct v4l2_subdev *subdev;
+	struct v4l2_subdev_selection sdsel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = sel->target,
+	};
 	u32 pad;
 	int ret;
 
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		if (video->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_COMPOSE:
+	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+	case V4L2_SEL_TGT_COMPOSE_DEFAULT:
+		if (video->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		break;
+	default:
+		return -EINVAL;
+	}
 	subdev = isp_video_remote_subdev(video, &pad);
 	if (subdev == NULL)
 		return -EINVAL;
 
-	/* Try the get crop operation first and fallback to get format if not
+	/* Try the get selection operation first and fallback to get format if not
 	 * implemented.
 	 */
-	ret = v4l2_subdev_call(subdev, video, g_crop, crop);
+	sdsel.pad = pad;
+	ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel);
+	if (!ret)
+		sel->r = sdsel.r;
 	if (ret != -ENOIOCTLCMD)
 		return ret;
 
@@ -815,28 +820,50 @@ isp_video_get_crop(struct file *file, void *fh, struct v4l2_crop *crop)
 	if (ret < 0)
 		return ret == -ENOIOCTLCMD ? -ENOTTY : ret;
 
-	crop->c.left = 0;
-	crop->c.top = 0;
-	crop->c.width = format.format.width;
-	crop->c.height = format.format.height;
+	sel->r.left = 0;
+	sel->r.top = 0;
+	sel->r.width = format.format.width;
+	sel->r.height = format.format.height;
 
 	return 0;
 }
 
 static int
-isp_video_set_crop(struct file *file, void *fh, const struct v4l2_crop *crop)
+isp_video_set_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 {
 	struct isp_video *video = video_drvdata(file);
 	struct v4l2_subdev *subdev;
+	struct v4l2_subdev_selection sdsel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = sel->target,
+		.flags = sel->flags,
+		.r = sel->r,
+	};
+	u32 pad;
 	int ret;
 
-	subdev = isp_video_remote_subdev(video, NULL);
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		if (video->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_COMPOSE:
+		if (video->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+	subdev = isp_video_remote_subdev(video, &pad);
 	if (subdev == NULL)
 		return -EINVAL;
 
+	sdsel.pad = pad;
 	mutex_lock(&video->mutex);
-	ret = v4l2_subdev_call(subdev, video, s_crop, crop);
+	ret = v4l2_subdev_call(subdev, pad, set_selection, NULL, &sdsel);
 	mutex_unlock(&video->mutex);
+	if (!ret)
+		sel->r = sdsel.r;
 
 	return ret == -ENOIOCTLCMD ? -ENOTTY : ret;
 }
@@ -1252,9 +1279,8 @@ static const struct v4l2_ioctl_ops isp_video_ioctl_ops = {
 	.vidioc_g_fmt_vid_out		= isp_video_get_format,
 	.vidioc_s_fmt_vid_out		= isp_video_set_format,
 	.vidioc_try_fmt_vid_out		= isp_video_try_format,
-	.vidioc_cropcap			= isp_video_cropcap,
-	.vidioc_g_crop			= isp_video_get_crop,
-	.vidioc_s_crop			= isp_video_set_crop,
+	.vidioc_g_selection		= isp_video_get_selection,
+	.vidioc_s_selection		= isp_video_set_selection,
 	.vidioc_g_parm			= isp_video_get_param,
 	.vidioc_s_parm			= isp_video_set_param,
 	.vidioc_reqbufs			= isp_video_reqbufs,
diff --git a/drivers/media/platform/soc_camera/pxa_camera.c b/drivers/media/platform/pxa_camera.c
index 2aaf4a8f71a0..c12209c701d3 100644
--- a/drivers/media/platform/soc_camera/pxa_camera.c
+++ b/drivers/media/platform/pxa_camera.c
@@ -3,6 +3,7 @@
  *
  * Copyright (C) 2006, Sascha Hauer, Pengutronix
  * Copyright (C) 2008, Guennadi Liakhovetski <kernel@pengutronix.de>
+ * Copyright (C) 2016, Robert Jarzmik <robert.jarzmik@free.fr>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -14,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/io.h>
 #include <linux/delay.h>
+#include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/errno.h>
@@ -22,8 +24,8 @@
 #include <linux/kernel.h>
 #include <linux/mm.h>
 #include <linux/moduleparam.h>
+#include <linux/of.h>
 #include <linux/time.h>
-#include <linux/device.h>
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/sched.h>
@@ -32,13 +34,15 @@
 #include <linux/dma-mapping.h>
 #include <linux/dma/pxa-dma.h>
 
+#include <media/v4l2-async.h>
+#include <media/v4l2-clk.h>
 #include <media/v4l2-common.h>
-#include <media/v4l2-dev.h>
-#include <media/videobuf-dma-sg.h>
-#include <media/soc_camera.h>
-#include <media/drv-intf/soc_mediabus.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
 #include <media/v4l2-of.h>
 
+#include <media/videobuf2-dma-sg.h>
+
 #include <linux/videodev2.h>
 
 #include <linux/platform_data/media/camera-pxa.h>
@@ -46,6 +50,9 @@
 #define PXA_CAM_VERSION "0.0.6"
 #define PXA_CAM_DRV_NAME "pxa27x-camera"
 
+#define DEFAULT_WIDTH	640
+#define DEFAULT_HEIGHT	480
+
 /* Camera Interface */
 #define CICR0		0x0000
 #define CICR1		0x0004
@@ -168,6 +175,462 @@
 			CICR0_PERRM | CICR0_QDM | CICR0_CDM | CICR0_SOFM | \
 			CICR0_EOFM | CICR0_FOM)
 
+#define sensor_call(cam, o, f, args...) \
+	v4l2_subdev_call(cam->sensor, o, f, ##args)
+
+/*
+ * Format handling
+ */
+
+/**
+ * enum pxa_mbus_packing - data packing types on the media-bus
+ * @PXA_MBUS_PACKING_NONE:	no packing, bit-for-bit transfer to RAM, one
+ *				sample represents one pixel
+ * @PXA_MBUS_PACKING_2X8_PADHI:	16 bits transferred in 2 8-bit samples, in the
+ *				possibly incomplete byte high bits are padding
+ * @PXA_MBUS_PACKING_EXTEND16:	sample width (e.g., 10 bits) has to be extended
+ *				to 16 bits
+ */
+enum pxa_mbus_packing {
+	PXA_MBUS_PACKING_NONE,
+	PXA_MBUS_PACKING_2X8_PADHI,
+	PXA_MBUS_PACKING_EXTEND16,
+};
+
+/**
+ * enum pxa_mbus_order - sample order on the media bus
+ * @PXA_MBUS_ORDER_LE:		least significant sample first
+ * @PXA_MBUS_ORDER_BE:		most significant sample first
+ */
+enum pxa_mbus_order {
+	PXA_MBUS_ORDER_LE,
+	PXA_MBUS_ORDER_BE,
+};
+
+/**
+ * enum pxa_mbus_layout - planes layout in memory
+ * @PXA_MBUS_LAYOUT_PACKED:		color components packed
+ * @PXA_MBUS_LAYOUT_PLANAR_2Y_U_V:	YUV components stored in 3 planes (4:2:2)
+ * @PXA_MBUS_LAYOUT_PLANAR_2Y_C:	YUV components stored in a luma and a
+ *					chroma plane (C plane is half the size
+ *					of Y plane)
+ * @PXA_MBUS_LAYOUT_PLANAR_Y_C:		YUV components stored in a luma and a
+ *					chroma plane (C plane is the same size
+ *					as Y plane)
+ */
+enum pxa_mbus_layout {
+	PXA_MBUS_LAYOUT_PACKED = 0,
+	PXA_MBUS_LAYOUT_PLANAR_2Y_U_V,
+	PXA_MBUS_LAYOUT_PLANAR_2Y_C,
+	PXA_MBUS_LAYOUT_PLANAR_Y_C,
+};
+
+/**
+ * struct pxa_mbus_pixelfmt - Data format on the media bus
+ * @name:		Name of the format
+ * @fourcc:		Fourcc code, that will be obtained if the data is
+ *			stored in memory in the following way:
+ * @packing:		Type of sample-packing, that has to be used
+ * @order:		Sample order when storing in memory
+ * @bits_per_sample:	How many bits the bridge has to sample
+ */
+struct pxa_mbus_pixelfmt {
+	const char		*name;
+	u32			fourcc;
+	enum pxa_mbus_packing	packing;
+	enum pxa_mbus_order	order;
+	enum pxa_mbus_layout	layout;
+	u8			bits_per_sample;
+};
+
+/**
+ * struct pxa_mbus_lookup - Lookup FOURCC IDs by mediabus codes for pass-through
+ * @code:	mediabus pixel-code
+ * @fmt:	pixel format description
+ */
+struct pxa_mbus_lookup {
+	u32	code;
+	struct pxa_mbus_pixelfmt	fmt;
+};
+
+static const struct pxa_mbus_lookup mbus_fmt[] = {
+{
+	.code = MEDIA_BUS_FMT_YUYV8_2X8,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_YUYV,
+		.name			= "YUYV",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_YVYU8_2X8,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_YVYU,
+		.name			= "YVYU",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_UYVY8_2X8,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_UYVY,
+		.name			= "UYVY",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_VYUY8_2X8,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_VYUY,
+		.name			= "VYUY",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_RGB555,
+		.name			= "RGB555",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_RGB555X,
+		.name			= "RGB555X",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_BE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_RGB565_2X8_LE,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_RGB565,
+		.name			= "RGB565",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_RGB565_2X8_BE,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_RGB565X,
+		.name			= "RGB565X",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_BE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SBGGR8_1X8,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SBGGR8,
+		.name			= "Bayer 8 BGGR",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_NONE,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SBGGR10_1X10,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SBGGR10,
+		.name			= "Bayer 10 BGGR",
+		.bits_per_sample	= 10,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_Y8_1X8,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_GREY,
+		.name			= "Grey",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_NONE,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_Y10_1X10,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_Y10,
+		.name			= "Grey 10bit",
+		.bits_per_sample	= 10,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SBGGR10_2X8_PADHI_LE,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SBGGR10,
+		.name			= "Bayer 10 BGGR",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SBGGR10_2X8_PADHI_BE,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SBGGR10,
+		.name			= "Bayer 10 BGGR",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_BE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_RGB444_2X8_PADHI_BE,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_RGB444,
+		.name			= "RGB444",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_BE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_UYVY8_1X16,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_UYVY,
+		.name			= "UYVY 16bit",
+		.bits_per_sample	= 16,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_VYUY8_1X16,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_VYUY,
+		.name			= "VYUY 16bit",
+		.bits_per_sample	= 16,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_YUYV8_1X16,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_YUYV,
+		.name			= "YUYV 16bit",
+		.bits_per_sample	= 16,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_YVYU8_1X16,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_YVYU,
+		.name			= "YVYU 16bit",
+		.bits_per_sample	= 16,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SGRBG8_1X8,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SGRBG8,
+		.name			= "Bayer 8 GRBG",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_NONE,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SGRBG10_DPCM8_1X8,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SGRBG10DPCM8,
+		.name			= "Bayer 10 BGGR DPCM 8",
+		.bits_per_sample	= 8,
+		.packing		= PXA_MBUS_PACKING_NONE,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SGBRG10_1X10,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SGBRG10,
+		.name			= "Bayer 10 GBRG",
+		.bits_per_sample	= 10,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SGRBG10_1X10,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SGRBG10,
+		.name			= "Bayer 10 GRBG",
+		.bits_per_sample	= 10,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SRGGB10_1X10,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SRGGB10,
+		.name			= "Bayer 10 RGGB",
+		.bits_per_sample	= 10,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SBGGR12_1X12,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SBGGR12,
+		.name			= "Bayer 12 BGGR",
+		.bits_per_sample	= 12,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SGBRG12_1X12,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SGBRG12,
+		.name			= "Bayer 12 GBRG",
+		.bits_per_sample	= 12,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SGRBG12_1X12,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SGRBG12,
+		.name			= "Bayer 12 GRBG",
+		.bits_per_sample	= 12,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+}, {
+	.code = MEDIA_BUS_FMT_SRGGB12_1X12,
+	.fmt = {
+		.fourcc			= V4L2_PIX_FMT_SRGGB12,
+		.name			= "Bayer 12 RGGB",
+		.bits_per_sample	= 12,
+		.packing		= PXA_MBUS_PACKING_EXTEND16,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PACKED,
+	},
+},
+};
+
+static s32 pxa_mbus_bytes_per_line(u32 width, const struct pxa_mbus_pixelfmt *mf)
+{
+	if (mf->layout != PXA_MBUS_LAYOUT_PACKED)
+		return width * mf->bits_per_sample / 8;
+
+	switch (mf->packing) {
+	case PXA_MBUS_PACKING_NONE:
+		return width * mf->bits_per_sample / 8;
+	case PXA_MBUS_PACKING_2X8_PADHI:
+	case PXA_MBUS_PACKING_EXTEND16:
+		return width * 2;
+	}
+	return -EINVAL;
+}
+
+static s32 pxa_mbus_image_size(const struct pxa_mbus_pixelfmt *mf,
+			u32 bytes_per_line, u32 height)
+{
+	switch (mf->packing) {
+	case PXA_MBUS_PACKING_2X8_PADHI:
+		return bytes_per_line * height * 2;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct pxa_mbus_pixelfmt *pxa_mbus_find_fmtdesc(
+	u32 code,
+	const struct pxa_mbus_lookup *lookup,
+	int n)
+{
+	int i;
+
+	for (i = 0; i < n; i++)
+		if (lookup[i].code == code)
+			return &lookup[i].fmt;
+
+	return NULL;
+}
+
+static const struct pxa_mbus_pixelfmt *pxa_mbus_get_fmtdesc(
+	u32 code)
+{
+	return pxa_mbus_find_fmtdesc(code, mbus_fmt, ARRAY_SIZE(mbus_fmt));
+}
+
+static unsigned int pxa_mbus_config_compatible(const struct v4l2_mbus_config *cfg,
+					unsigned int flags)
+{
+	unsigned long common_flags;
+	bool hsync = true, vsync = true, pclk, data, mode;
+	bool mipi_lanes, mipi_clock;
+
+	common_flags = cfg->flags & flags;
+
+	switch (cfg->type) {
+	case V4L2_MBUS_PARALLEL:
+		hsync = common_flags & (V4L2_MBUS_HSYNC_ACTIVE_HIGH |
+					V4L2_MBUS_HSYNC_ACTIVE_LOW);
+		vsync = common_flags & (V4L2_MBUS_VSYNC_ACTIVE_HIGH |
+					V4L2_MBUS_VSYNC_ACTIVE_LOW);
+		/* fall through */
+	case V4L2_MBUS_BT656:
+		pclk = common_flags & (V4L2_MBUS_PCLK_SAMPLE_RISING |
+				       V4L2_MBUS_PCLK_SAMPLE_FALLING);
+		data = common_flags & (V4L2_MBUS_DATA_ACTIVE_HIGH |
+				       V4L2_MBUS_DATA_ACTIVE_LOW);
+		mode = common_flags & (V4L2_MBUS_MASTER | V4L2_MBUS_SLAVE);
+		return (!hsync || !vsync || !pclk || !data || !mode) ?
+			0 : common_flags;
+	case V4L2_MBUS_CSI2:
+		mipi_lanes = common_flags & V4L2_MBUS_CSI2_LANES;
+		mipi_clock = common_flags & (V4L2_MBUS_CSI2_NONCONTINUOUS_CLOCK |
+					     V4L2_MBUS_CSI2_CONTINUOUS_CLOCK);
+		return (!mipi_lanes || !mipi_clock) ? 0 : common_flags;
+	}
+	return 0;
+}
+
+/**
+ * struct soc_camera_format_xlate - match between host and sensor formats
+ * @code: code of a sensor provided format
+ * @host_fmt: host format after host translation from code
+ *
+ * Host and sensor translation structure. Used in table of host and sensor
+ * formats matchings in soc_camera_device. A host can override the generic list
+ * generation by implementing get_formats(), and use it for format checks and
+ * format setup.
+ */
+struct soc_camera_format_xlate {
+	u32 code;
+	const struct pxa_mbus_pixelfmt *host_fmt;
+};
+
 /*
  * Structures
  */
@@ -180,19 +643,33 @@ enum pxa_camera_active_dma {
 /* buffer for one video frame */
 struct pxa_buffer {
 	/* common v4l buffer stuff -- must be first */
-	struct videobuf_buffer		vb;
+	struct vb2_v4l2_buffer		vbuf;
+	struct list_head		queue;
 	u32	code;
+	int				nb_planes;
 	/* our descriptor lists for Y, U and V channels */
 	struct dma_async_tx_descriptor	*descs[3];
 	dma_cookie_t			cookie[3];
 	struct scatterlist		*sg[3];
 	int				sg_len[3];
+	size_t				plane_sizes[3];
 	int				inwork;
 	enum pxa_camera_active_dma	active_dma;
 };
 
 struct pxa_camera_dev {
-	struct soc_camera_host	soc_host;
+	struct v4l2_device	v4l2_dev;
+	struct video_device	vdev;
+	struct v4l2_async_notifier notifier;
+	struct vb2_queue	vb2_vq;
+	struct v4l2_subdev	*sensor;
+	struct soc_camera_format_xlate *user_formats;
+	const struct soc_camera_format_xlate *current_fmt;
+	struct v4l2_pix_format	current_pix;
+
+	struct v4l2_async_subdev asd;
+	struct v4l2_async_subdev *asds[1];
+
 	/*
 	 * PXA27x is only supposed to handle one camera on its Quick Capture
 	 * interface. If anyone ever builds hardware to enable more than
@@ -212,11 +689,14 @@ struct pxa_camera_dev {
 	unsigned long		ciclk;
 	unsigned long		mclk;
 	u32			mclk_divisor;
+	struct v4l2_clk		*mclk_clk;
 	u16			width_flags;	/* max 10 bits */
 
 	struct list_head	capture;
 
 	spinlock_t		lock;
+	struct mutex		mlock;
+	unsigned int		buf_sequence;
 
 	struct pxa_buffer	*active;
 	struct tasklet_struct	task_eof;
@@ -230,59 +710,90 @@ struct pxa_cam {
 
 static const char *pxa_cam_driver_description = "PXA_Camera";
 
-static unsigned int vid_limit = 16;	/* Video memory limit, in Mb */
-
 /*
- *  Videobuf operations
+ * Format translation functions
  */
-static int pxa_videobuf_setup(struct videobuf_queue *vq, unsigned int *count,
-			      unsigned int *size)
+static const struct soc_camera_format_xlate
+*pxa_mbus_xlate_by_fourcc(struct soc_camera_format_xlate *user_formats,
+			  unsigned int fourcc)
 {
-	struct soc_camera_device *icd = vq->priv_data;
+	unsigned int i;
 
-	dev_dbg(icd->parent, "count=%d, size=%d\n", *count, *size);
-
-	*size = icd->sizeimage;
-
-	if (0 == *count)
-		*count = 32;
-	if (*size * *count > vid_limit * 1024 * 1024)
-		*count = (vid_limit * 1024 * 1024) / *size;
-
-	return 0;
+	for (i = 0; user_formats[i].code; i++)
+		if (user_formats[i].host_fmt->fourcc == fourcc)
+			return user_formats + i;
+	return NULL;
 }
 
-static void free_buffer(struct videobuf_queue *vq, struct pxa_buffer *buf)
+static struct soc_camera_format_xlate *pxa_mbus_build_fmts_xlate(
+	struct v4l2_device *v4l2_dev, struct v4l2_subdev *subdev,
+	int (*get_formats)(struct v4l2_device *, unsigned int,
+			   struct soc_camera_format_xlate *xlate))
 {
-	struct soc_camera_device *icd = vq->priv_data;
-	struct videobuf_dmabuf *dma = videobuf_to_dma(&buf->vb);
-	int i;
-
-	BUG_ON(in_interrupt());
+	unsigned int i, fmts = 0, raw_fmts = 0;
+	int ret;
+	struct v4l2_subdev_mbus_code_enum code = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	struct soc_camera_format_xlate *user_formats;
 
-	dev_dbg(icd->parent, "%s (vb=0x%p) 0x%08lx %d\n", __func__,
-		&buf->vb, buf->vb.baddr, buf->vb.bsize);
+	while (!v4l2_subdev_call(subdev, pad, enum_mbus_code, NULL, &code)) {
+		raw_fmts++;
+		code.index++;
+	}
 
 	/*
-	 * This waits until this buffer is out of danger, i.e., until it is no
-	 * longer in STATE_QUEUED or STATE_ACTIVE
+	 * First pass - only count formats this host-sensor
+	 * configuration can provide
 	 */
-	videobuf_waiton(vq, &buf->vb, 0, 0);
+	for (i = 0; i < raw_fmts; i++) {
+		ret = get_formats(v4l2_dev, i, NULL);
+		if (ret < 0)
+			return ERR_PTR(ret);
+		fmts += ret;
+	}
 
-	for (i = 0; i < 3 && buf->descs[i]; i++) {
-		dmaengine_desc_free(buf->descs[i]);
-		kfree(buf->sg[i]);
-		buf->descs[i] = NULL;
-		buf->sg[i] = NULL;
-		buf->sg_len[i] = 0;
+	if (!fmts)
+		return ERR_PTR(-ENXIO);
+
+	user_formats = kcalloc(fmts + 1, sizeof(*user_formats), GFP_KERNEL);
+	if (!user_formats)
+		return ERR_PTR(-ENOMEM);
+
+	/* Second pass - actually fill data formats */
+	fmts = 0;
+	for (i = 0; i < raw_fmts; i++) {
+		ret = get_formats(v4l2_dev, i, user_formats + fmts);
+		if (ret < 0)
+			goto egfmt;
+		fmts += ret;
 	}
-	videobuf_dma_unmap(vq->dev, dma);
-	videobuf_dma_free(dma);
+	user_formats[fmts].code = 0;
 
-	buf->vb.state = VIDEOBUF_NEEDS_INIT;
+	return user_formats;
+egfmt:
+	kfree(user_formats);
+	return ERR_PTR(ret);
+}
 
-	dev_dbg(icd->parent, "%s end (vb=0x%p) 0x%08lx %d\n", __func__,
-		&buf->vb, buf->vb.baddr, buf->vb.bsize);
+/*
+ *  Videobuf operations
+ */
+static struct pxa_buffer *vb2_to_pxa_buffer(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+
+	return container_of(vbuf, struct pxa_buffer, vbuf);
+}
+
+static struct device *pcdev_to_dev(struct pxa_camera_dev *pcdev)
+{
+	return pcdev->v4l2_dev.dev;
+}
+
+static struct pxa_camera_dev *v4l2_dev_to_pcdev(struct v4l2_device *v4l2_dev)
+{
+	return container_of(v4l2_dev, struct pxa_camera_dev, v4l2_dev);
 }
 
 static void pxa_camera_dma_irq(struct pxa_camera_dev *pcdev,
@@ -312,31 +823,26 @@ static void pxa_camera_dma_irq_v(void *data)
 /**
  * pxa_init_dma_channel - init dma descriptors
  * @pcdev: pxa camera device
- * @buf: pxa buffer to find pxa dma channel
+ * @vb: videobuffer2 buffer
  * @dma: dma video buffer
  * @channel: dma channel (0 => 'Y', 1 => 'U', 2 => 'V')
  * @cibr: camera Receive Buffer Register
- * @size: bytes to transfer
- * @offset: offset in videobuffer of the first byte to transfer
  *
  * Prepares the pxa dma descriptors to transfer one camera channel.
  *
  * Returns 0 if success or -ENOMEM if no memory is available
  */
 static int pxa_init_dma_channel(struct pxa_camera_dev *pcdev,
-				struct pxa_buffer *buf,
-				struct videobuf_dmabuf *dma, int channel,
-				int cibr, int size, int offset)
+				struct pxa_buffer *buf, int channel,
+				struct scatterlist *sg, int sglen)
 {
 	struct dma_chan *dma_chan = pcdev->dma_chans[channel];
-	struct scatterlist *sg = buf->sg[channel];
-	int sglen = buf->sg_len[channel];
 	struct dma_async_tx_descriptor *tx;
 
 	tx = dmaengine_prep_slave_sg(dma_chan, sg, sglen, DMA_DEV_TO_MEM,
 				     DMA_PREP_INTERRUPT | DMA_CTRL_REUSE);
 	if (!tx) {
-		dev_err(pcdev->soc_host.v4l2_dev.dev,
+		dev_err(pcdev_to_dev(pcdev),
 			"dmaengine_prep_slave_sg failed\n");
 		goto fail;
 	}
@@ -357,11 +863,9 @@ static int pxa_init_dma_channel(struct pxa_camera_dev *pcdev,
 	buf->descs[channel] = tx;
 	return 0;
 fail:
-	kfree(sg);
-
-	dev_dbg(pcdev->soc_host.v4l2_dev.dev,
-		"%s (vb=0x%p) dma_tx=%p\n",
-		__func__, &buf->vb, tx);
+	dev_dbg(pcdev_to_dev(pcdev),
+		"%s (vb=%p) dma_tx=%p\n",
+		__func__, buf, tx);
 
 	return -ENOMEM;
 }
@@ -370,133 +874,10 @@ static void pxa_videobuf_set_actdma(struct pxa_camera_dev *pcdev,
 				    struct pxa_buffer *buf)
 {
 	buf->active_dma = DMA_Y;
-	if (pcdev->channels == 3)
+	if (buf->nb_planes == 3)
 		buf->active_dma |= DMA_U | DMA_V;
 }
 
-/*
- * Please check the DMA prepared buffer structure in :
- *   Documentation/video4linux/pxa_camera.txt
- * Please check also in pxa_camera_check_link_miss() to understand why DMA chain
- * modification while DMA chain is running will work anyway.
- */
-static int pxa_videobuf_prepare(struct videobuf_queue *vq,
-		struct videobuf_buffer *vb, enum v4l2_field field)
-{
-	struct soc_camera_device *icd = vq->priv_data;
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct pxa_camera_dev *pcdev = ici->priv;
-	struct device *dev = pcdev->soc_host.v4l2_dev.dev;
-	struct pxa_buffer *buf = container_of(vb, struct pxa_buffer, vb);
-	int ret;
-	int size_y, size_u = 0, size_v = 0;
-	size_t sizes[3];
-
-	dev_dbg(dev, "%s (vb=0x%p) 0x%08lx %d\n", __func__,
-		vb, vb->baddr, vb->bsize);
-
-	/* Added list head initialization on alloc */
-	WARN_ON(!list_empty(&vb->queue));
-
-#ifdef DEBUG
-	/*
-	 * This can be useful if you want to see if we actually fill
-	 * the buffer with something
-	 */
-	memset((void *)vb->baddr, 0xaa, vb->bsize);
-#endif
-
-	BUG_ON(NULL == icd->current_fmt);
-
-	/*
-	 * I think, in buf_prepare you only have to protect global data,
-	 * the actual buffer is yours
-	 */
-	buf->inwork = 1;
-
-	if (buf->code	!= icd->current_fmt->code ||
-	    vb->width	!= icd->user_width ||
-	    vb->height	!= icd->user_height ||
-	    vb->field	!= field) {
-		buf->code	= icd->current_fmt->code;
-		vb->width	= icd->user_width;
-		vb->height	= icd->user_height;
-		vb->field	= field;
-		vb->state	= VIDEOBUF_NEEDS_INIT;
-	}
-
-	vb->size = icd->sizeimage;
-	if (0 != vb->baddr && vb->bsize < vb->size) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	if (vb->state == VIDEOBUF_NEEDS_INIT) {
-		int size = vb->size;
-		struct videobuf_dmabuf *dma = videobuf_to_dma(vb);
-
-		ret = videobuf_iolock(vq, vb, NULL);
-		if (ret)
-			goto out;
-
-		if (pcdev->channels == 3) {
-			size_y = size / 2;
-			size_u = size_v = size / 4;
-		} else {
-			size_y = size;
-		}
-
-		sizes[0] = size_y;
-		sizes[1] = size_u;
-		sizes[2] = size_v;
-		ret = sg_split(dma->sglist, dma->sglen, 0, pcdev->channels,
-			       sizes, buf->sg, buf->sg_len, GFP_KERNEL);
-		if (ret < 0) {
-			dev_err(dev, "sg_split failed: %d\n", ret);
-			goto fail;
-		}
-
-		/* init DMA for Y channel */
-		ret = pxa_init_dma_channel(pcdev, buf, dma, 0, CIBR0,
-					   size_y, 0);
-		if (ret) {
-			dev_err(dev, "DMA initialization for Y/RGB failed\n");
-			goto fail;
-		}
-
-		/* init DMA for U channel */
-		if (size_u)
-			ret = pxa_init_dma_channel(pcdev, buf, dma, 1, CIBR1,
-						   size_u, size_y);
-		if (ret) {
-			dev_err(dev, "DMA initialization for U failed\n");
-			goto fail;
-		}
-
-		/* init DMA for V channel */
-		if (size_v)
-			ret = pxa_init_dma_channel(pcdev, buf, dma, 2, CIBR2,
-						   size_v, size_y + size_u);
-		if (ret) {
-			dev_err(dev, "DMA initialization for V failed\n");
-			goto fail;
-		}
-
-		vb->state = VIDEOBUF_PREPARED;
-	}
-
-	buf->inwork = 0;
-	pxa_videobuf_set_actdma(pcdev, buf);
-
-	return 0;
-
-fail:
-	free_buffer(vq, buf);
-out:
-	buf->inwork = 0;
-	return ret;
-}
-
 /**
  * pxa_dma_start_channels - start DMA channel for active buffer
  * @pcdev: pxa camera device
@@ -507,12 +888,9 @@ out:
 static void pxa_dma_start_channels(struct pxa_camera_dev *pcdev)
 {
 	int i;
-	struct pxa_buffer *active;
-
-	active = pcdev->active;
 
 	for (i = 0; i < pcdev->channels; i++) {
-		dev_dbg(pcdev->soc_host.v4l2_dev.dev,
+		dev_dbg(pcdev_to_dev(pcdev),
 			"%s (channel=%d)\n", __func__, i);
 		dma_async_issue_pending(pcdev->dma_chans[i]);
 	}
@@ -523,7 +901,7 @@ static void pxa_dma_stop_channels(struct pxa_camera_dev *pcdev)
 	int i;
 
 	for (i = 0; i < pcdev->channels; i++) {
-		dev_dbg(pcdev->soc_host.v4l2_dev.dev,
+		dev_dbg(pcdev_to_dev(pcdev),
 			"%s (channel=%d)\n", __func__, i);
 		dmaengine_terminate_all(pcdev->dma_chans[i]);
 	}
@@ -536,7 +914,7 @@ static void pxa_dma_add_tail_buf(struct pxa_camera_dev *pcdev,
 
 	for (i = 0; i < pcdev->channels; i++) {
 		buf->cookie[i] = dmaengine_submit(buf->descs[i]);
-		dev_dbg(pcdev->soc_host.v4l2_dev.dev,
+		dev_dbg(pcdev_to_dev(pcdev),
 			"%s (channel=%d) : submit vb=%p cookie=%d\n",
 			__func__, i, buf, buf->descs[i]->cookie);
 	}
@@ -554,7 +932,7 @@ static void pxa_camera_start_capture(struct pxa_camera_dev *pcdev)
 {
 	unsigned long cicr0;
 
-	dev_dbg(pcdev->soc_host.v4l2_dev.dev, "%s\n", __func__);
+	dev_dbg(pcdev_to_dev(pcdev), "%s\n", __func__);
 	__raw_writel(__raw_readl(pcdev->base + CISR), pcdev->base + CISR);
 	/* Enable End-Of-Frame Interrupt */
 	cicr0 = __raw_readl(pcdev->base + CICR0) | CICR0_ENB;
@@ -572,72 +950,24 @@ static void pxa_camera_stop_capture(struct pxa_camera_dev *pcdev)
 	__raw_writel(cicr0, pcdev->base + CICR0);
 
 	pcdev->active = NULL;
-	dev_dbg(pcdev->soc_host.v4l2_dev.dev, "%s\n", __func__);
-}
-
-/* Called under spinlock_irqsave(&pcdev->lock, ...) */
-static void pxa_videobuf_queue(struct videobuf_queue *vq,
-			       struct videobuf_buffer *vb)
-{
-	struct soc_camera_device *icd = vq->priv_data;
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct pxa_camera_dev *pcdev = ici->priv;
-	struct pxa_buffer *buf = container_of(vb, struct pxa_buffer, vb);
-
-	dev_dbg(icd->parent, "%s (vb=0x%p) 0x%08lx %d active=%p\n",
-		__func__, vb, vb->baddr, vb->bsize, pcdev->active);
-
-	list_add_tail(&vb->queue, &pcdev->capture);
-
-	vb->state = VIDEOBUF_ACTIVE;
-	pxa_dma_add_tail_buf(pcdev, buf);
-
-	if (!pcdev->active)
-		pxa_camera_start_capture(pcdev);
-}
-
-static void pxa_videobuf_release(struct videobuf_queue *vq,
-				 struct videobuf_buffer *vb)
-{
-	struct pxa_buffer *buf = container_of(vb, struct pxa_buffer, vb);
-#ifdef DEBUG
-	struct soc_camera_device *icd = vq->priv_data;
-	struct device *dev = icd->parent;
-
-	dev_dbg(dev, "%s (vb=0x%p) 0x%08lx %d\n", __func__,
-		vb, vb->baddr, vb->bsize);
-
-	switch (vb->state) {
-	case VIDEOBUF_ACTIVE:
-		dev_dbg(dev, "%s (active)\n", __func__);
-		break;
-	case VIDEOBUF_QUEUED:
-		dev_dbg(dev, "%s (queued)\n", __func__);
-		break;
-	case VIDEOBUF_PREPARED:
-		dev_dbg(dev, "%s (prepared)\n", __func__);
-		break;
-	default:
-		dev_dbg(dev, "%s (unknown)\n", __func__);
-		break;
-	}
-#endif
-
-	free_buffer(vq, buf);
+	dev_dbg(pcdev_to_dev(pcdev), "%s\n", __func__);
 }
 
 static void pxa_camera_wakeup(struct pxa_camera_dev *pcdev,
-			      struct videobuf_buffer *vb,
-			      struct pxa_buffer *buf)
+			      struct pxa_buffer *buf,
+			      enum vb2_buffer_state state)
 {
+	struct vb2_buffer *vb = &buf->vbuf.vb2_buf;
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+
 	/* _init is used to debug races, see comment in pxa_camera_reqbufs() */
-	list_del_init(&vb->queue);
-	vb->state = VIDEOBUF_DONE;
-	v4l2_get_timestamp(&vb->ts);
-	vb->field_count++;
-	wake_up(&vb->done);
-	dev_dbg(pcdev->soc_host.v4l2_dev.dev, "%s dequeud buffer (vb=0x%p)\n",
-		__func__, vb);
+	list_del_init(&buf->queue);
+	vb->timestamp = ktime_get_ns();
+	vbuf->sequence = pcdev->buf_sequence++;
+	vbuf->field = V4L2_FIELD_NONE;
+	vb2_buffer_done(vb, VB2_BUF_STATE_DONE);
+	dev_dbg(pcdev_to_dev(pcdev), "%s dequeued buffer (buf=0x%p)\n",
+		__func__, buf);
 
 	if (list_empty(&pcdev->capture)) {
 		pxa_camera_stop_capture(pcdev);
@@ -645,7 +975,7 @@ static void pxa_camera_wakeup(struct pxa_camera_dev *pcdev,
 	}
 
 	pcdev->active = list_entry(pcdev->capture.next,
-				   struct pxa_buffer, vb.queue);
+				   struct pxa_buffer, queue);
 }
 
 /**
@@ -670,7 +1000,7 @@ static void pxa_camera_check_link_miss(struct pxa_camera_dev *pcdev,
 {
 	bool is_dma_stopped = last_submitted != last_issued;
 
-	dev_dbg(pcdev->soc_host.v4l2_dev.dev,
+	dev_dbg(pcdev_to_dev(pcdev),
 		"%s : top queued buffer=%p, is_dma_stopped=%d\n",
 		__func__, pcdev->active, is_dma_stopped);
 
@@ -681,19 +1011,17 @@ static void pxa_camera_check_link_miss(struct pxa_camera_dev *pcdev,
 static void pxa_camera_dma_irq(struct pxa_camera_dev *pcdev,
 			       enum pxa_camera_active_dma act_dma)
 {
-	struct device *dev = pcdev->soc_host.v4l2_dev.dev;
 	struct pxa_buffer *buf, *last_buf;
 	unsigned long flags;
 	u32 camera_status, overrun;
 	int chan;
-	struct videobuf_buffer *vb;
 	enum dma_status last_status;
 	dma_cookie_t last_issued;
 
 	spin_lock_irqsave(&pcdev->lock, flags);
 
 	camera_status = __raw_readl(pcdev->base + CISR);
-	dev_dbg(dev, "camera dma irq, cisr=0x%x dma=%d\n",
+	dev_dbg(pcdev_to_dev(pcdev), "camera dma irq, cisr=0x%x dma=%d\n",
 		camera_status, act_dma);
 	overrun = CISR_IFO_0;
 	if (pcdev->channels == 3)
@@ -714,9 +1042,8 @@ static void pxa_camera_dma_irq(struct pxa_camera_dev *pcdev,
 	if (!pcdev->active)
 		goto out;
 
-	vb = &pcdev->active->vb;
-	buf = container_of(vb, struct pxa_buffer, vb);
-	WARN_ON(buf->inwork || list_empty(&vb->queue));
+	buf = pcdev->active;
+	WARN_ON(buf->inwork || list_empty(&buf->queue));
 
 	/*
 	 * It's normal if the last frame creates an overrun, as there
@@ -734,23 +1061,23 @@ static void pxa_camera_dma_irq(struct pxa_camera_dev *pcdev,
 		break;
 	}
 	last_buf = list_entry(pcdev->capture.prev,
-			      struct pxa_buffer, vb.queue);
+			      struct pxa_buffer, queue);
 	last_status = dma_async_is_tx_complete(pcdev->dma_chans[chan],
 					       last_buf->cookie[chan],
 					       NULL, &last_issued);
 	if (camera_status & overrun &&
 	    last_status != DMA_COMPLETE) {
-		dev_dbg(dev, "FIFO overrun! CISR: %x\n",
+		dev_dbg(pcdev_to_dev(pcdev), "FIFO overrun! CISR: %x\n",
 			camera_status);
 		pxa_camera_stop_capture(pcdev);
-		list_for_each_entry(buf, &pcdev->capture, vb.queue)
+		list_for_each_entry(buf, &pcdev->capture, queue)
 			pxa_dma_add_tail_buf(pcdev, buf);
 		pxa_camera_start_capture(pcdev);
 		goto out;
 	}
 	buf->active_dma &= ~act_dma;
 	if (!buf->active_dma) {
-		pxa_camera_wakeup(pcdev, vb, buf);
+		pxa_camera_wakeup(pcdev, buf, VB2_BUF_STATE_DONE);
 		pxa_camera_check_link_miss(pcdev, last_buf->cookie[chan],
 					   last_issued);
 	}
@@ -759,33 +1086,10 @@ out:
 	spin_unlock_irqrestore(&pcdev->lock, flags);
 }
 
-static struct videobuf_queue_ops pxa_videobuf_ops = {
-	.buf_setup      = pxa_videobuf_setup,
-	.buf_prepare    = pxa_videobuf_prepare,
-	.buf_queue      = pxa_videobuf_queue,
-	.buf_release    = pxa_videobuf_release,
-};
-
-static void pxa_camera_init_videobuf(struct videobuf_queue *q,
-			      struct soc_camera_device *icd)
-{
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct pxa_camera_dev *pcdev = ici->priv;
-
-	/*
-	 * We must pass NULL as dev pointer, then all pci_* dma operations
-	 * transform to normal dma_* ones.
-	 */
-	videobuf_queue_sg_init(q, &pxa_videobuf_ops, NULL, &pcdev->lock,
-				V4L2_BUF_TYPE_VIDEO_CAPTURE, V4L2_FIELD_NONE,
-				sizeof(struct pxa_buffer), icd, &ici->host_lock);
-}
-
 static u32 mclk_get_divisor(struct platform_device *pdev,
 			    struct pxa_camera_dev *pcdev)
 {
 	unsigned long mclk = pcdev->mclk;
-	struct device *dev = &pdev->dev;
 	u32 div;
 	unsigned long lcdclk;
 
@@ -795,7 +1099,8 @@ static u32 mclk_get_divisor(struct platform_device *pdev,
 	/* mclk <= ciclk / 4 (27.4.2) */
 	if (mclk > lcdclk / 4) {
 		mclk = lcdclk / 4;
-		dev_warn(dev, "Limiting master clock to %lu\n", mclk);
+		dev_warn(pcdev_to_dev(pcdev),
+			 "Limiting master clock to %lu\n", mclk);
 	}
 
 	/* We verify mclk != 0, so if anyone breaks it, here comes their Oops */
@@ -805,7 +1110,7 @@ static u32 mclk_get_divisor(struct platform_device *pdev,
 	if (pcdev->platform_flags & PXA_CAMERA_MCLK_EN)
 		pcdev->mclk = lcdclk / (2 * (div + 1));
 
-	dev_dbg(dev, "LCD clock %luHz, target freq %luHz, divisor %u\n",
+	dev_dbg(pcdev_to_dev(pcdev), "LCD clock %luHz, target freq %luHz, divisor %u\n",
 		lcdclk, mclk, div);
 
 	return div;
@@ -860,9 +1165,8 @@ static void pxa_camera_eof(unsigned long arg)
 	struct pxa_camera_dev *pcdev = (struct pxa_camera_dev *)arg;
 	unsigned long cifr;
 	struct pxa_buffer *buf;
-	struct videobuf_buffer *vb;
 
-	dev_dbg(pcdev->soc_host.v4l2_dev.dev,
+	dev_dbg(pcdev_to_dev(pcdev),
 		"Camera interrupt status 0x%x\n",
 		__raw_readl(pcdev->base + CISR));
 
@@ -871,9 +1175,8 @@ static void pxa_camera_eof(unsigned long arg)
 	__raw_writel(cifr, pcdev->base + CIFR);
 
 	pcdev->active = list_first_entry(&pcdev->capture,
-					 struct pxa_buffer, vb.queue);
-	vb = &pcdev->active->vb;
-	buf = container_of(vb, struct pxa_buffer, vb);
+					 struct pxa_buffer, queue);
+	buf = pcdev->active;
 	pxa_videobuf_set_actdma(pcdev, buf);
 
 	pxa_dma_start_channels(pcdev);
@@ -885,7 +1188,7 @@ static irqreturn_t pxa_camera_irq(int irq, void *data)
 	unsigned long status, cicr0;
 
 	status = __raw_readl(pcdev->base + CISR);
-	dev_dbg(pcdev->soc_host.v4l2_dev.dev,
+	dev_dbg(pcdev_to_dev(pcdev),
 		"Camera interrupt status 0x%lx\n", status);
 
 	if (!status)
@@ -902,47 +1205,6 @@ static irqreturn_t pxa_camera_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static int pxa_camera_add_device(struct soc_camera_device *icd)
-{
-	dev_info(icd->parent, "PXA Camera driver attached to camera %d\n",
-		 icd->devnum);
-
-	return 0;
-}
-
-static void pxa_camera_remove_device(struct soc_camera_device *icd)
-{
-	dev_info(icd->parent, "PXA Camera driver detached from camera %d\n",
-		 icd->devnum);
-}
-
-/*
- * The following two functions absolutely depend on the fact, that
- * there can be only one camera on PXA quick capture interface
- * Called with .host_lock held
- */
-static int pxa_camera_clock_start(struct soc_camera_host *ici)
-{
-	struct pxa_camera_dev *pcdev = ici->priv;
-
-	pxa_camera_activate(pcdev);
-
-	return 0;
-}
-
-/* Called with .host_lock held */
-static void pxa_camera_clock_stop(struct soc_camera_host *ici)
-{
-	struct pxa_camera_dev *pcdev = ici->priv;
-
-	/* disable capture, disable interrupts */
-	__raw_writel(0x3ff, pcdev->base + CICR0);
-
-	/* Stop DMA engine */
-	pxa_dma_stop_channels(pcdev);
-	pxa_camera_deactivate(pcdev);
-}
-
 static int test_platform_param(struct pxa_camera_dev *pcdev,
 			       unsigned char buswidth, unsigned long *flags)
 {
@@ -968,15 +1230,12 @@ static int test_platform_param(struct pxa_camera_dev *pcdev,
 	return -EINVAL;
 }
 
-static void pxa_camera_setup_cicr(struct soc_camera_device *icd,
+static void pxa_camera_setup_cicr(struct pxa_camera_dev *pcdev,
 				  unsigned long flags, __u32 pixfmt)
 {
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct pxa_camera_dev *pcdev = ici->priv;
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
 	unsigned long dw, bpp;
 	u32 cicr0, cicr1, cicr2, cicr3, cicr4 = 0, y_skip_top;
-	int ret = v4l2_subdev_call(sd, sensor, g_skip_top_lines, &y_skip_top);
+	int ret = sensor_call(pcdev, sensor, g_skip_top_lines, &y_skip_top);
 
 	if (ret < 0)
 		y_skip_top = 0;
@@ -985,7 +1244,7 @@ static void pxa_camera_setup_cicr(struct soc_camera_device *icd,
 	 * Datawidth is now guaranteed to be equal to one of the three values.
 	 * We fix bit-per-pixel equal to data-width...
 	 */
-	switch (icd->current_fmt->host_fmt->bits_per_sample) {
+	switch (pcdev->current_fmt->host_fmt->bits_per_sample) {
 	case 10:
 		dw = 4;
 		bpp = 0x40;
@@ -1019,7 +1278,7 @@ static void pxa_camera_setup_cicr(struct soc_camera_device *icd,
 	if (cicr0 & CICR0_ENB)
 		__raw_writel(cicr0 & ~CICR0_ENB, pcdev->base + CICR0);
 
-	cicr1 = CICR1_PPL_VAL(icd->user_width - 1) | bpp | dw;
+	cicr1 = CICR1_PPL_VAL(pcdev->current_pix.width - 1) | bpp | dw;
 
 	switch (pixfmt) {
 	case V4L2_PIX_FMT_YUV422P:
@@ -1048,7 +1307,7 @@ static void pxa_camera_setup_cicr(struct soc_camera_device *icd,
 	}
 
 	cicr2 = 0;
-	cicr3 = CICR3_LPF_VAL(icd->user_height - 1) |
+	cicr3 = CICR3_LPF_VAL(pcdev->current_pix.height - 1) |
 		CICR3_BFW_VAL(min((u32)255, y_skip_top));
 	cicr4 |= pcdev->mclk_divisor;
 
@@ -1064,28 +1323,271 @@ static void pxa_camera_setup_cicr(struct soc_camera_device *icd,
 	__raw_writel(cicr0, pcdev->base + CICR0);
 }
 
-static int pxa_camera_set_bus_param(struct soc_camera_device *icd)
+/*
+ * Videobuf2 section
+ */
+static void pxa_buffer_cleanup(struct pxa_buffer *buf)
+{
+	int i;
+
+	for (i = 0; i < 3 && buf->descs[i]; i++) {
+		dmaengine_desc_free(buf->descs[i]);
+		kfree(buf->sg[i]);
+		buf->descs[i] = NULL;
+		buf->sg[i] = NULL;
+		buf->sg_len[i] = 0;
+		buf->plane_sizes[i] = 0;
+	}
+	buf->nb_planes = 0;
+}
+
+static int pxa_buffer_init(struct pxa_camera_dev *pcdev,
+			   struct pxa_buffer *buf)
+{
+	struct vb2_buffer *vb = &buf->vbuf.vb2_buf;
+	struct sg_table *sgt = vb2_dma_sg_plane_desc(vb, 0);
+	int nb_channels = pcdev->channels;
+	int i, ret = 0;
+	unsigned long size = vb2_plane_size(vb, 0);
+
+	switch (nb_channels) {
+	case 1:
+		buf->plane_sizes[0] = size;
+		break;
+	case 3:
+		buf->plane_sizes[0] = size / 2;
+		buf->plane_sizes[1] = size / 4;
+		buf->plane_sizes[2] = size / 4;
+		break;
+	default:
+		return -EINVAL;
+	};
+	buf->nb_planes = nb_channels;
+
+	ret = sg_split(sgt->sgl, sgt->nents, 0, nb_channels,
+		       buf->plane_sizes, buf->sg, buf->sg_len, GFP_KERNEL);
+	if (ret < 0) {
+		dev_err(pcdev_to_dev(pcdev),
+			"sg_split failed: %d\n", ret);
+		return ret;
+	}
+	for (i = 0; i < nb_channels; i++) {
+		ret = pxa_init_dma_channel(pcdev, buf, i,
+					   buf->sg[i], buf->sg_len[i]);
+		if (ret) {
+			pxa_buffer_cleanup(buf);
+			return ret;
+		}
+	}
+	INIT_LIST_HEAD(&buf->queue);
+
+	return ret;
+}
+
+static void pxac_vb2_cleanup(struct vb2_buffer *vb)
+{
+	struct pxa_buffer *buf = vb2_to_pxa_buffer(vb);
+	struct pxa_camera_dev *pcdev = vb2_get_drv_priv(vb->vb2_queue);
+
+	dev_dbg(pcdev_to_dev(pcdev),
+		 "%s(vb=%p)\n", __func__, vb);
+	pxa_buffer_cleanup(buf);
+}
+
+static void pxac_vb2_queue(struct vb2_buffer *vb)
+{
+	struct pxa_buffer *buf = vb2_to_pxa_buffer(vb);
+	struct pxa_camera_dev *pcdev = vb2_get_drv_priv(vb->vb2_queue);
+
+	dev_dbg(pcdev_to_dev(pcdev),
+		 "%s(vb=%p) nb_channels=%d size=%lu active=%p\n",
+		__func__, vb, pcdev->channels, vb2_get_plane_payload(vb, 0),
+		pcdev->active);
+
+	list_add_tail(&buf->queue, &pcdev->capture);
+
+	pxa_dma_add_tail_buf(pcdev, buf);
+}
+
+/*
+ * Please check the DMA prepared buffer structure in :
+ *   Documentation/video4linux/pxa_camera.txt
+ * Please check also in pxa_camera_check_link_miss() to understand why DMA chain
+ * modification while DMA chain is running will work anyway.
+ */
+static int pxac_vb2_prepare(struct vb2_buffer *vb)
+{
+	struct pxa_camera_dev *pcdev = vb2_get_drv_priv(vb->vb2_queue);
+	struct pxa_buffer *buf = vb2_to_pxa_buffer(vb);
+	int ret = 0;
+
+	switch (pcdev->channels) {
+	case 1:
+	case 3:
+		vb2_set_plane_payload(vb, 0, pcdev->current_pix.sizeimage);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	dev_dbg(pcdev_to_dev(pcdev),
+		 "%s (vb=%p) nb_channels=%d size=%lu\n",
+		__func__, vb, pcdev->channels, vb2_get_plane_payload(vb, 0));
+
+	WARN_ON(!pcdev->current_fmt);
+
+#ifdef DEBUG
+	/*
+	 * This can be useful if you want to see if we actually fill
+	 * the buffer with something
+	 */
+	for (i = 0; i < vb->num_planes; i++)
+		memset((void *)vb2_plane_vaddr(vb, i),
+		       0xaa, vb2_get_plane_payload(vb, i));
+#endif
+
+	/*
+	 * I think, in buf_prepare you only have to protect global data,
+	 * the actual buffer is yours
+	 */
+	buf->inwork = 0;
+	pxa_videobuf_set_actdma(pcdev, buf);
+
+	return ret;
+}
+
+static int pxac_vb2_init(struct vb2_buffer *vb)
+{
+	struct pxa_camera_dev *pcdev = vb2_get_drv_priv(vb->vb2_queue);
+	struct pxa_buffer *buf = vb2_to_pxa_buffer(vb);
+
+	dev_dbg(pcdev_to_dev(pcdev),
+		 "%s(nb_channels=%d)\n",
+		__func__, pcdev->channels);
+
+	return pxa_buffer_init(pcdev, buf);
+}
+
+static int pxac_vb2_queue_setup(struct vb2_queue *vq,
+				unsigned int *nbufs,
+				unsigned int *num_planes, unsigned int sizes[],
+				struct device *alloc_devs[])
+{
+	struct pxa_camera_dev *pcdev = vb2_get_drv_priv(vq);
+	int size = pcdev->current_pix.sizeimage;
+
+	dev_dbg(pcdev_to_dev(pcdev),
+		 "%s(vq=%p nbufs=%d num_planes=%d size=%d)\n",
+		__func__, vq, *nbufs, *num_planes, size);
+	/*
+	 * Called from VIDIOC_REQBUFS or in compatibility mode For YUV422P
+	 * format, even if there are 3 planes Y, U and V, we reply there is only
+	 * one plane, containing Y, U and V data, one after the other.
+	 */
+	if (*num_planes)
+		return sizes[0] < size ? -EINVAL : 0;
+
+	*num_planes = 1;
+	switch (pcdev->channels) {
+	case 1:
+	case 3:
+		sizes[0] = size;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (!*nbufs)
+		*nbufs = 1;
+
+	return 0;
+}
+
+static int pxac_vb2_start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct pxa_camera_dev *pcdev = vb2_get_drv_priv(vq);
+
+	dev_dbg(pcdev_to_dev(pcdev), "%s(count=%d) active=%p\n",
+		__func__, count, pcdev->active);
+
+	pcdev->buf_sequence = 0;
+	if (!pcdev->active)
+		pxa_camera_start_capture(pcdev);
+
+	return 0;
+}
+
+static void pxac_vb2_stop_streaming(struct vb2_queue *vq)
+{
+	struct pxa_camera_dev *pcdev = vb2_get_drv_priv(vq);
+	struct pxa_buffer *buf, *tmp;
+
+	dev_dbg(pcdev_to_dev(pcdev), "%s active=%p\n",
+		__func__, pcdev->active);
+	pxa_camera_stop_capture(pcdev);
+
+	list_for_each_entry_safe(buf, tmp, &pcdev->capture, queue)
+		pxa_camera_wakeup(pcdev, buf, VB2_BUF_STATE_ERROR);
+}
+
+static struct vb2_ops pxac_vb2_ops = {
+	.queue_setup		= pxac_vb2_queue_setup,
+	.buf_init		= pxac_vb2_init,
+	.buf_prepare		= pxac_vb2_prepare,
+	.buf_queue		= pxac_vb2_queue,
+	.buf_cleanup		= pxac_vb2_cleanup,
+	.start_streaming	= pxac_vb2_start_streaming,
+	.stop_streaming		= pxac_vb2_stop_streaming,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+};
+
+static int pxa_camera_init_videobuf2(struct pxa_camera_dev *pcdev)
+{
+	int ret;
+	struct vb2_queue *vq = &pcdev->vb2_vq;
+
+	memset(vq, 0, sizeof(*vq));
+	vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	vq->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
+	vq->drv_priv = pcdev;
+	vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	vq->buf_struct_size = sizeof(struct pxa_buffer);
+	vq->dev = pcdev->v4l2_dev.dev;
+
+	vq->ops = &pxac_vb2_ops;
+	vq->mem_ops = &vb2_dma_sg_memops;
+	vq->lock = &pcdev->mlock;
+
+	ret = vb2_queue_init(vq);
+	dev_dbg(pcdev_to_dev(pcdev),
+		 "vb2_queue_init(vq=%p): %d\n", vq, ret);
+
+	return ret;
+}
+
+/*
+ * Video ioctls section
+ */
+static int pxa_camera_set_bus_param(struct pxa_camera_dev *pcdev)
 {
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct pxa_camera_dev *pcdev = ici->priv;
 	struct v4l2_mbus_config cfg = {.type = V4L2_MBUS_PARALLEL,};
-	u32 pixfmt = icd->current_fmt->host_fmt->fourcc;
+	u32 pixfmt = pcdev->current_fmt->host_fmt->fourcc;
 	unsigned long bus_flags, common_flags;
 	int ret;
-	struct pxa_cam *cam = icd->host_priv;
 
-	ret = test_platform_param(pcdev, icd->current_fmt->host_fmt->bits_per_sample,
+	ret = test_platform_param(pcdev,
+				  pcdev->current_fmt->host_fmt->bits_per_sample,
 				  &bus_flags);
 	if (ret < 0)
 		return ret;
 
-	ret = v4l2_subdev_call(sd, video, g_mbus_config, &cfg);
+	ret = sensor_call(pcdev, video, g_mbus_config, &cfg);
 	if (!ret) {
-		common_flags = soc_mbus_config_compatible(&cfg,
+		common_flags = pxa_mbus_config_compatible(&cfg,
 							  bus_flags);
 		if (!common_flags) {
-			dev_warn(icd->parent,
+			dev_warn(pcdev_to_dev(pcdev),
 				 "Flags incompatible: camera 0x%x, host 0x%lx\n",
 				 cfg.flags, bus_flags);
 			return -EINVAL;
@@ -1124,26 +1626,22 @@ static int pxa_camera_set_bus_param(struct soc_camera_device *icd)
 	}
 
 	cfg.flags = common_flags;
-	ret = v4l2_subdev_call(sd, video, s_mbus_config, &cfg);
+	ret = sensor_call(pcdev, video, s_mbus_config, &cfg);
 	if (ret < 0 && ret != -ENOIOCTLCMD) {
-		dev_dbg(icd->parent, "camera s_mbus_config(0x%lx) returned %d\n",
+		dev_dbg(pcdev_to_dev(pcdev),
+			"camera s_mbus_config(0x%lx) returned %d\n",
 			common_flags, ret);
 		return ret;
 	}
 
-	cam->flags = common_flags;
-
-	pxa_camera_setup_cicr(icd, common_flags, pixfmt);
+	pxa_camera_setup_cicr(pcdev, common_flags, pixfmt);
 
 	return 0;
 }
 
-static int pxa_camera_try_bus_param(struct soc_camera_device *icd,
+static int pxa_camera_try_bus_param(struct pxa_camera_dev *pcdev,
 				    unsigned char buswidth)
 {
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct pxa_camera_dev *pcdev = ici->priv;
 	struct v4l2_mbus_config cfg = {.type = V4L2_MBUS_PARALLEL,};
 	unsigned long bus_flags, common_flags;
 	int ret = test_platform_param(pcdev, buswidth, &bus_flags);
@@ -1151,12 +1649,12 @@ static int pxa_camera_try_bus_param(struct soc_camera_device *icd,
 	if (ret < 0)
 		return ret;
 
-	ret = v4l2_subdev_call(sd, video, g_mbus_config, &cfg);
+	ret = sensor_call(pcdev, video, g_mbus_config, &cfg);
 	if (!ret) {
-		common_flags = soc_mbus_config_compatible(&cfg,
+		common_flags = pxa_mbus_config_compatible(&cfg,
 							  bus_flags);
 		if (!common_flags) {
-			dev_warn(icd->parent,
+			dev_warn(pcdev_to_dev(pcdev),
 				 "Flags incompatible: camera 0x%x, host 0x%lx\n",
 				 cfg.flags, bus_flags);
 			return -EINVAL;
@@ -1168,66 +1666,56 @@ static int pxa_camera_try_bus_param(struct soc_camera_device *icd,
 	return ret;
 }
 
-static const struct soc_mbus_pixelfmt pxa_camera_formats[] = {
+static const struct pxa_mbus_pixelfmt pxa_camera_formats[] = {
 	{
 		.fourcc			= V4L2_PIX_FMT_YUV422P,
 		.name			= "Planar YUV422 16 bit",
 		.bits_per_sample	= 8,
-		.packing		= SOC_MBUS_PACKING_2X8_PADHI,
-		.order			= SOC_MBUS_ORDER_LE,
-		.layout			= SOC_MBUS_LAYOUT_PLANAR_2Y_U_V,
+		.packing		= PXA_MBUS_PACKING_2X8_PADHI,
+		.order			= PXA_MBUS_ORDER_LE,
+		.layout			= PXA_MBUS_LAYOUT_PLANAR_2Y_U_V,
 	},
 };
 
 /* This will be corrected as we get more formats */
-static bool pxa_camera_packing_supported(const struct soc_mbus_pixelfmt *fmt)
+static bool pxa_camera_packing_supported(const struct pxa_mbus_pixelfmt *fmt)
 {
-	return	fmt->packing == SOC_MBUS_PACKING_NONE ||
+	return	fmt->packing == PXA_MBUS_PACKING_NONE ||
 		(fmt->bits_per_sample == 8 &&
-		 fmt->packing == SOC_MBUS_PACKING_2X8_PADHI) ||
+		 fmt->packing == PXA_MBUS_PACKING_2X8_PADHI) ||
 		(fmt->bits_per_sample > 8 &&
-		 fmt->packing == SOC_MBUS_PACKING_EXTEND16);
+		 fmt->packing == PXA_MBUS_PACKING_EXTEND16);
 }
 
-static int pxa_camera_get_formats(struct soc_camera_device *icd, unsigned int idx,
+static int pxa_camera_get_formats(struct v4l2_device *v4l2_dev,
+				  unsigned int idx,
 				  struct soc_camera_format_xlate *xlate)
 {
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct device *dev = icd->parent;
+	struct pxa_camera_dev *pcdev = v4l2_dev_to_pcdev(v4l2_dev);
 	int formats = 0, ret;
-	struct pxa_cam *cam;
 	struct v4l2_subdev_mbus_code_enum code = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 		.index = idx,
 	};
-	const struct soc_mbus_pixelfmt *fmt;
+	const struct pxa_mbus_pixelfmt *fmt;
 
-	ret = v4l2_subdev_call(sd, pad, enum_mbus_code, NULL, &code);
+	ret = sensor_call(pcdev, pad, enum_mbus_code, NULL, &code);
 	if (ret < 0)
 		/* No more formats */
 		return 0;
 
-	fmt = soc_mbus_get_fmtdesc(code.code);
+	fmt = pxa_mbus_get_fmtdesc(code.code);
 	if (!fmt) {
-		dev_err(dev, "Invalid format code #%u: %d\n", idx, code.code);
+		dev_err(pcdev_to_dev(pcdev),
+			"Invalid format code #%u: %d\n", idx, code.code);
 		return 0;
 	}
 
 	/* This also checks support for the requested bits-per-sample */
-	ret = pxa_camera_try_bus_param(icd, fmt->bits_per_sample);
+	ret = pxa_camera_try_bus_param(pcdev, fmt->bits_per_sample);
 	if (ret < 0)
 		return 0;
 
-	if (!icd->host_priv) {
-		cam = kzalloc(sizeof(*cam), GFP_KERNEL);
-		if (!cam)
-			return -ENOMEM;
-
-		icd->host_priv = cam;
-	} else {
-		cam = icd->host_priv;
-	}
-
 	switch (code.code) {
 	case MEDIA_BUS_FMT_UYVY8_2X8:
 		formats++;
@@ -1235,25 +1723,29 @@ static int pxa_camera_get_formats(struct soc_camera_device *icd, unsigned int id
 			xlate->host_fmt	= &pxa_camera_formats[0];
 			xlate->code	= code.code;
 			xlate++;
-			dev_dbg(dev, "Providing format %s using code %d\n",
+			dev_dbg(pcdev_to_dev(pcdev),
+				"Providing format %s using code %d\n",
 				pxa_camera_formats[0].name, code.code);
 		}
+	/* fall through */
 	case MEDIA_BUS_FMT_VYUY8_2X8:
 	case MEDIA_BUS_FMT_YUYV8_2X8:
 	case MEDIA_BUS_FMT_YVYU8_2X8:
 	case MEDIA_BUS_FMT_RGB565_2X8_LE:
 	case MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE:
 		if (xlate)
-			dev_dbg(dev, "Providing format %s packed\n",
+			dev_dbg(pcdev_to_dev(pcdev),
+				"Providing format %s packed\n",
 				fmt->name);
 		break;
 	default:
 		if (!pxa_camera_packing_supported(fmt))
 			return 0;
 		if (xlate)
-			dev_dbg(dev,
+			dev_dbg(pcdev_to_dev(pcdev),
 				"Providing format %s in pass-through mode\n",
 				fmt->name);
+		break;
 	}
 
 	/* Generic pass-through */
@@ -1267,10 +1759,22 @@ static int pxa_camera_get_formats(struct soc_camera_device *icd, unsigned int id
 	return formats;
 }
 
-static void pxa_camera_put_formats(struct soc_camera_device *icd)
+static int pxa_camera_build_formats(struct pxa_camera_dev *pcdev)
 {
-	kfree(icd->host_priv);
-	icd->host_priv = NULL;
+	struct soc_camera_format_xlate *xlate;
+
+	xlate = pxa_mbus_build_fmts_xlate(&pcdev->v4l2_dev, pcdev->sensor,
+					  pxa_camera_get_formats);
+	if (IS_ERR(xlate))
+		return PTR_ERR(xlate);
+
+	pcdev->user_formats = xlate;
+	return 0;
+}
+
+static void pxa_camera_destroy_formats(struct pxa_camera_dev *pcdev)
+{
+	kfree(pcdev->user_formats);
 }
 
 static int pxa_camera_check_frame(u32 width, u32 height)
@@ -1280,158 +1784,72 @@ static int pxa_camera_check_frame(u32 width, u32 height)
 		(width & 0x01);
 }
 
-static int pxa_camera_set_crop(struct soc_camera_device *icd,
-			       const struct v4l2_crop *a)
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+static int pxac_vidioc_g_register(struct file *file, void *priv,
+				  struct v4l2_dbg_register *reg)
 {
-	const struct v4l2_rect *rect = &a->c;
-	struct device *dev = icd->parent;
-	struct soc_camera_host *ici = to_soc_camera_host(dev);
-	struct pxa_camera_dev *pcdev = ici->priv;
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct soc_camera_sense sense = {
-		.master_clock = pcdev->mclk,
-		.pixel_clock_max = pcdev->ciclk / 4,
-	};
-	struct v4l2_subdev_format fmt = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-	struct v4l2_mbus_framefmt *mf = &fmt.format;
-	struct pxa_cam *cam = icd->host_priv;
-	u32 fourcc = icd->current_fmt->host_fmt->fourcc;
-	int ret;
-
-	/* If PCLK is used to latch data from the sensor, check sense */
-	if (pcdev->platform_flags & PXA_CAMERA_PCLK_EN)
-		icd->sense = &sense;
-
-	ret = v4l2_subdev_call(sd, video, s_crop, a);
-
-	icd->sense = NULL;
-
-	if (ret < 0) {
-		dev_warn(dev, "Failed to crop to %ux%u@%u:%u\n",
-			 rect->width, rect->height, rect->left, rect->top);
-		return ret;
-	}
+	struct pxa_camera_dev *pcdev = video_drvdata(file);
 
-	ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt);
-	if (ret < 0)
-		return ret;
-
-	if (pxa_camera_check_frame(mf->width, mf->height)) {
-		/*
-		 * Camera cropping produced a frame beyond our capabilities.
-		 * FIXME: just extract a subframe, that we can process.
-		 */
-		v4l_bound_align_image(&mf->width, 48, 2048, 1,
-			&mf->height, 32, 2048, 0,
-			fourcc == V4L2_PIX_FMT_YUV422P ? 4 : 0);
-		ret = v4l2_subdev_call(sd, pad, set_fmt, NULL, &fmt);
-		if (ret < 0)
-			return ret;
+	if (reg->reg > CIBR2)
+		return -ERANGE;
 
-		if (pxa_camera_check_frame(mf->width, mf->height)) {
-			dev_warn(icd->parent,
-				 "Inconsistent state. Use S_FMT to repair\n");
-			return -EINVAL;
-		}
-	}
-
-	if (sense.flags & SOCAM_SENSE_PCLK_CHANGED) {
-		if (sense.pixel_clock > sense.pixel_clock_max) {
-			dev_err(dev,
-				"pixel clock %lu set by the camera too high!",
-				sense.pixel_clock);
-			return -EIO;
-		}
-		recalculate_fifo_timeout(pcdev, sense.pixel_clock);
-	}
-
-	icd->user_width		= mf->width;
-	icd->user_height	= mf->height;
-
-	pxa_camera_setup_cicr(icd, cam->flags, fourcc);
-
-	return ret;
+	reg->val = __raw_readl(pcdev->base + reg->reg);
+	reg->size = sizeof(__u32);
+	return 0;
 }
 
-static int pxa_camera_set_fmt(struct soc_camera_device *icd,
-			      struct v4l2_format *f)
+static int pxac_vidioc_s_register(struct file *file, void *priv,
+				  const struct v4l2_dbg_register *reg)
 {
-	struct device *dev = icd->parent;
-	struct soc_camera_host *ici = to_soc_camera_host(dev);
-	struct pxa_camera_dev *pcdev = ici->priv;
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	const struct soc_camera_format_xlate *xlate = NULL;
-	struct soc_camera_sense sense = {
-		.master_clock = pcdev->mclk,
-		.pixel_clock_max = pcdev->ciclk / 4,
-	};
-	struct v4l2_pix_format *pix = &f->fmt.pix;
-	struct v4l2_subdev_format format = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-	struct v4l2_mbus_framefmt *mf = &format.format;
-	int ret;
+	struct pxa_camera_dev *pcdev = video_drvdata(file);
 
-	xlate = soc_camera_xlate_by_fourcc(icd, pix->pixelformat);
-	if (!xlate) {
-		dev_warn(dev, "Format %x not found\n", pix->pixelformat);
+	if (reg->reg > CIBR2)
+		return -ERANGE;
+	if (reg->size != sizeof(__u32))
 		return -EINVAL;
-	}
-
-	/* If PCLK is used to latch data from the sensor, check sense */
-	if (pcdev->platform_flags & PXA_CAMERA_PCLK_EN)
-		/* The caller holds a mutex. */
-		icd->sense = &sense;
-
-	mf->width	= pix->width;
-	mf->height	= pix->height;
-	mf->field	= pix->field;
-	mf->colorspace	= pix->colorspace;
-	mf->code	= xlate->code;
+	__raw_writel(reg->val, pcdev->base + reg->reg);
+	return 0;
+}
+#endif
 
-	ret = v4l2_subdev_call(sd, pad, set_fmt, NULL, &format);
+static int pxac_vidioc_enum_fmt_vid_cap(struct file *filp, void  *priv,
+					struct v4l2_fmtdesc *f)
+{
+	struct pxa_camera_dev *pcdev = video_drvdata(filp);
+	const struct pxa_mbus_pixelfmt *format;
+	unsigned int idx;
 
-	if (mf->code != xlate->code)
+	for (idx = 0; pcdev->user_formats[idx].code; idx++);
+	if (f->index >= idx)
 		return -EINVAL;
 
-	icd->sense = NULL;
-
-	if (ret < 0) {
-		dev_warn(dev, "Failed to configure for format %x\n",
-			 pix->pixelformat);
-	} else if (pxa_camera_check_frame(mf->width, mf->height)) {
-		dev_warn(dev,
-			 "Camera driver produced an unsupported frame %dx%d\n",
-			 mf->width, mf->height);
-		ret = -EINVAL;
-	} else if (sense.flags & SOCAM_SENSE_PCLK_CHANGED) {
-		if (sense.pixel_clock > sense.pixel_clock_max) {
-			dev_err(dev,
-				"pixel clock %lu set by the camera too high!",
-				sense.pixel_clock);
-			return -EIO;
-		}
-		recalculate_fifo_timeout(pcdev, sense.pixel_clock);
-	}
-
-	if (ret < 0)
-		return ret;
+	format = pcdev->user_formats[f->index].host_fmt;
+	f->pixelformat = format->fourcc;
+	return 0;
+}
 
-	pix->width		= mf->width;
-	pix->height		= mf->height;
-	pix->field		= mf->field;
-	pix->colorspace		= mf->colorspace;
-	icd->current_fmt	= xlate;
+static int pxac_vidioc_g_fmt_vid_cap(struct file *filp, void *priv,
+				    struct v4l2_format *f)
+{
+	struct pxa_camera_dev *pcdev = video_drvdata(filp);
+	struct v4l2_pix_format *pix = &f->fmt.pix;
 
-	return ret;
+	pix->width		= pcdev->current_pix.width;
+	pix->height		= pcdev->current_pix.height;
+	pix->bytesperline	= pcdev->current_pix.bytesperline;
+	pix->sizeimage		= pcdev->current_pix.sizeimage;
+	pix->field		= pcdev->current_pix.field;
+	pix->pixelformat	= pcdev->current_fmt->host_fmt->fourcc;
+	pix->colorspace		= pcdev->current_pix.colorspace;
+	dev_dbg(pcdev_to_dev(pcdev), "current_fmt->fourcc: 0x%08x\n",
+		pcdev->current_fmt->host_fmt->fourcc);
+	return 0;
 }
 
-static int pxa_camera_try_fmt(struct soc_camera_device *icd,
-			      struct v4l2_format *f)
+static int pxac_vidioc_try_fmt_vid_cap(struct file *filp, void *priv,
+				      struct v4l2_format *f)
 {
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
+	struct pxa_camera_dev *pcdev = video_drvdata(filp);
 	const struct soc_camera_format_xlate *xlate;
 	struct v4l2_pix_format *pix = &f->fmt.pix;
 	struct v4l2_subdev_pad_config pad_cfg;
@@ -1442,9 +1860,9 @@ static int pxa_camera_try_fmt(struct soc_camera_device *icd,
 	__u32 pixfmt = pix->pixelformat;
 	int ret;
 
-	xlate = soc_camera_xlate_by_fourcc(icd, pixfmt);
+	xlate = pxa_mbus_xlate_by_fourcc(pcdev->user_formats, pixfmt);
 	if (!xlate) {
-		dev_warn(icd->parent, "Format %x not found\n", pixfmt);
+		dev_warn(pcdev_to_dev(pcdev), "Format %x not found\n", pixfmt);
 		return -EINVAL;
 	}
 
@@ -1458,90 +1876,311 @@ static int pxa_camera_try_fmt(struct soc_camera_device *icd,
 			      &pix->height, 32, 2048, 0,
 			      pixfmt == V4L2_PIX_FMT_YUV422P ? 4 : 0);
 
-	/* limit to sensor capabilities */
-	mf->width	= pix->width;
-	mf->height	= pix->height;
-	/* Only progressive video supported so far */
-	mf->field	= V4L2_FIELD_NONE;
-	mf->colorspace	= pix->colorspace;
-	mf->code	= xlate->code;
-
-	ret = v4l2_subdev_call(sd, pad, set_fmt, &pad_cfg, &format);
+	v4l2_fill_mbus_format(mf, pix, xlate->code);
+	ret = sensor_call(pcdev, pad, set_fmt, &pad_cfg, &format);
 	if (ret < 0)
 		return ret;
 
-	pix->width	= mf->width;
-	pix->height	= mf->height;
-	pix->colorspace	= mf->colorspace;
+	v4l2_fill_pix_format(pix, mf);
 
+	/* Only progressive video supported so far */
 	switch (mf->field) {
 	case V4L2_FIELD_ANY:
 	case V4L2_FIELD_NONE:
-		pix->field	= V4L2_FIELD_NONE;
+		pix->field = V4L2_FIELD_NONE;
 		break;
 	default:
 		/* TODO: support interlaced at least in pass-through mode */
-		dev_err(icd->parent, "Field type %d unsupported.\n",
+		dev_err(pcdev_to_dev(pcdev), "Field type %d unsupported.\n",
 			mf->field);
 		return -EINVAL;
 	}
 
-	return ret;
+	ret = pxa_mbus_bytes_per_line(pix->width, xlate->host_fmt);
+	if (ret < 0)
+		return ret;
+
+	pix->bytesperline = ret;
+	ret = pxa_mbus_image_size(xlate->host_fmt, pix->bytesperline,
+				  pix->height);
+	if (ret < 0)
+		return ret;
+
+	pix->sizeimage = ret;
+	return 0;
 }
 
-static int pxa_camera_reqbufs(struct soc_camera_device *icd,
-			      struct v4l2_requestbuffers *p)
+static int pxac_vidioc_s_fmt_vid_cap(struct file *filp, void *priv,
+				    struct v4l2_format *f)
 {
-	int i;
+	struct pxa_camera_dev *pcdev = video_drvdata(filp);
+	const struct soc_camera_format_xlate *xlate;
+	struct v4l2_pix_format *pix = &f->fmt.pix;
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	unsigned long flags;
+	int ret, is_busy;
 
-	/*
-	 * This is for locking debugging only. I removed spinlocks and now I
-	 * check whether .prepare is ever called on a linked buffer, or whether
-	 * a dma IRQ can occur for an in-work or unlinked buffer. Until now
-	 * it hadn't triggered
-	 */
-	for (i = 0; i < p->count; i++) {
-		struct pxa_buffer *buf = container_of(icd->vb_vidq.bufs[i],
-						      struct pxa_buffer, vb);
-		buf->inwork = 0;
-		INIT_LIST_HEAD(&buf->vb.queue);
+	dev_dbg(pcdev_to_dev(pcdev),
+		"s_fmt_vid_cap(pix=%dx%d:%x)\n",
+		pix->width, pix->height, pix->pixelformat);
+
+	spin_lock_irqsave(&pcdev->lock, flags);
+	is_busy = pcdev->active || vb2_is_busy(&pcdev->vb2_vq);
+	spin_unlock_irqrestore(&pcdev->lock, flags);
+
+	if (is_busy)
+		return -EBUSY;
+
+	ret = pxac_vidioc_try_fmt_vid_cap(filp, priv, f);
+	if (ret)
+		return ret;
+
+	xlate = pxa_mbus_xlate_by_fourcc(pcdev->user_formats,
+					 pix->pixelformat);
+	v4l2_fill_mbus_format(&format.format, pix, xlate->code);
+	ret = sensor_call(pcdev, pad, set_fmt, NULL, &format);
+	if (ret < 0) {
+		dev_warn(pcdev_to_dev(pcdev),
+			 "Failed to configure for format %x\n",
+			 pix->pixelformat);
+	} else if (pxa_camera_check_frame(pix->width, pix->height)) {
+		dev_warn(pcdev_to_dev(pcdev),
+			 "Camera driver produced an unsupported frame %dx%d\n",
+			 pix->width, pix->height);
+		return -EINVAL;
 	}
 
+	pcdev->current_fmt = xlate;
+	pcdev->current_pix = *pix;
+
+	ret = pxa_camera_set_bus_param(pcdev);
+	return ret;
+}
+
+static int pxac_vidioc_querycap(struct file *file, void *priv,
+				struct v4l2_capability *cap)
+{
+	strlcpy(cap->bus_info, "platform:pxa-camera", sizeof(cap->bus_info));
+	strlcpy(cap->driver, PXA_CAM_DRV_NAME, sizeof(cap->driver));
+	strlcpy(cap->card, pxa_cam_driver_description, sizeof(cap->card));
+	cap->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
+	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
+
 	return 0;
 }
 
-static unsigned int pxa_camera_poll(struct file *file, poll_table *pt)
+static int pxac_vidioc_enum_input(struct file *file, void *priv,
+				  struct v4l2_input *i)
 {
-	struct soc_camera_device *icd = file->private_data;
-	struct pxa_buffer *buf;
+	if (i->index > 0)
+		return -EINVAL;
 
-	buf = list_entry(icd->vb_vidq.stream.next, struct pxa_buffer,
-			 vb.stream);
+	i->type = V4L2_INPUT_TYPE_CAMERA;
+	strlcpy(i->name, "Camera", sizeof(i->name));
 
-	poll_wait(file, &buf->vb.done, pt);
+	return 0;
+}
 
-	if (buf->vb.state == VIDEOBUF_DONE ||
-	    buf->vb.state == VIDEOBUF_ERROR)
-		return POLLIN|POLLRDNORM;
+static int pxac_vidioc_g_input(struct file *file, void *priv, unsigned int *i)
+{
+	*i = 0;
 
 	return 0;
 }
 
-static int pxa_camera_querycap(struct soc_camera_host *ici,
-			       struct v4l2_capability *cap)
+static int pxac_vidioc_s_input(struct file *file, void *priv, unsigned int i)
 {
-	/* cap->name is set by the firendly caller:-> */
-	strlcpy(cap->card, pxa_cam_driver_description, sizeof(cap->card));
-	cap->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
-	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
+	if (i > 0)
+		return -EINVAL;
 
 	return 0;
 }
 
+static int pxac_fops_camera_open(struct file *filp)
+{
+	struct pxa_camera_dev *pcdev = video_drvdata(filp);
+	int ret;
+
+	mutex_lock(&pcdev->mlock);
+	ret = v4l2_fh_open(filp);
+	if (ret < 0)
+		goto out;
+
+	ret = sensor_call(pcdev, core, s_power, 1);
+	if (ret)
+		v4l2_fh_release(filp);
+out:
+	mutex_unlock(&pcdev->mlock);
+	return ret;
+}
+
+static int pxac_fops_camera_release(struct file *filp)
+{
+	struct pxa_camera_dev *pcdev = video_drvdata(filp);
+	int ret;
+
+	ret = vb2_fop_release(filp);
+	if (ret < 0)
+		return ret;
+
+	mutex_lock(&pcdev->mlock);
+	ret = sensor_call(pcdev, core, s_power, 0);
+	mutex_unlock(&pcdev->mlock);
+
+	return ret;
+}
+
+static const struct v4l2_file_operations pxa_camera_fops = {
+	.owner		= THIS_MODULE,
+	.open		= pxac_fops_camera_open,
+	.release	= pxac_fops_camera_release,
+	.read		= vb2_fop_read,
+	.poll		= vb2_fop_poll,
+	.mmap		= vb2_fop_mmap,
+	.unlocked_ioctl = video_ioctl2,
+};
+
+static const struct v4l2_ioctl_ops pxa_camera_ioctl_ops = {
+	.vidioc_querycap		= pxac_vidioc_querycap,
+
+	.vidioc_enum_input		= pxac_vidioc_enum_input,
+	.vidioc_g_input			= pxac_vidioc_g_input,
+	.vidioc_s_input			= pxac_vidioc_s_input,
+
+	.vidioc_enum_fmt_vid_cap	= pxac_vidioc_enum_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap		= pxac_vidioc_g_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap		= pxac_vidioc_s_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap		= pxac_vidioc_try_fmt_vid_cap,
+
+	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
+	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
+	.vidioc_querybuf		= vb2_ioctl_querybuf,
+	.vidioc_qbuf			= vb2_ioctl_qbuf,
+	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
+	.vidioc_expbuf			= vb2_ioctl_expbuf,
+	.vidioc_streamon		= vb2_ioctl_streamon,
+	.vidioc_streamoff		= vb2_ioctl_streamoff,
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+	.vidioc_g_register		= pxac_vidioc_g_register,
+	.vidioc_s_register		= pxac_vidioc_s_register,
+#endif
+};
+
+static struct v4l2_clk_ops pxa_camera_mclk_ops = {
+};
+
+static const struct video_device pxa_camera_videodev_template = {
+	.name = "pxa-camera",
+	.minor = -1,
+	.fops = &pxa_camera_fops,
+	.ioctl_ops = &pxa_camera_ioctl_ops,
+	.release = video_device_release_empty,
+	.device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING,
+};
+
+static int pxa_camera_sensor_bound(struct v4l2_async_notifier *notifier,
+		     struct v4l2_subdev *subdev,
+		     struct v4l2_async_subdev *asd)
+{
+	int err;
+	struct v4l2_device *v4l2_dev = notifier->v4l2_dev;
+	struct pxa_camera_dev *pcdev = v4l2_dev_to_pcdev(v4l2_dev);
+	struct video_device *vdev = &pcdev->vdev;
+	struct v4l2_pix_format *pix = &pcdev->current_pix;
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	struct v4l2_mbus_framefmt *mf = &format.format;
+
+	dev_info(pcdev_to_dev(pcdev), "%s(): trying to bind a device\n",
+		 __func__);
+	mutex_lock(&pcdev->mlock);
+	*vdev = pxa_camera_videodev_template;
+	vdev->v4l2_dev = v4l2_dev;
+	vdev->lock = &pcdev->mlock;
+	pcdev->sensor = subdev;
+	pcdev->vdev.queue = &pcdev->vb2_vq;
+	pcdev->vdev.v4l2_dev = &pcdev->v4l2_dev;
+	pcdev->vdev.ctrl_handler = subdev->ctrl_handler;
+	video_set_drvdata(&pcdev->vdev, pcdev);
+
+	err = pxa_camera_build_formats(pcdev);
+	if (err) {
+		dev_err(pcdev_to_dev(pcdev), "building formats failed: %d\n",
+			err);
+		goto out;
+	}
+
+	pcdev->current_fmt = pcdev->user_formats;
+	pix->field = V4L2_FIELD_NONE;
+	pix->width = DEFAULT_WIDTH;
+	pix->height = DEFAULT_HEIGHT;
+	pix->bytesperline =
+		pxa_mbus_bytes_per_line(pix->width,
+					pcdev->current_fmt->host_fmt);
+	pix->sizeimage =
+		pxa_mbus_image_size(pcdev->current_fmt->host_fmt,
+				    pix->bytesperline, pix->height);
+	pix->pixelformat = pcdev->current_fmt->host_fmt->fourcc;
+	v4l2_fill_mbus_format(mf, pix, pcdev->current_fmt->code);
+	err = sensor_call(pcdev, pad, set_fmt, NULL, &format);
+	if (err)
+		goto out;
+
+	v4l2_fill_pix_format(pix, mf);
+	pr_info("%s(): colorspace=0x%x pixfmt=0x%x\n",
+		__func__, pix->colorspace, pix->pixelformat);
+
+	err = pxa_camera_init_videobuf2(pcdev);
+	if (err)
+		goto out;
+
+	err = video_register_device(&pcdev->vdev, VFL_TYPE_GRABBER, -1);
+	if (err) {
+		v4l2_err(v4l2_dev, "register video device failed: %d\n", err);
+		pcdev->sensor = NULL;
+	} else {
+		dev_info(pcdev_to_dev(pcdev),
+			 "PXA Camera driver attached to camera %s\n",
+			 subdev->name);
+	}
+out:
+	mutex_unlock(&pcdev->mlock);
+	return err;
+}
+
+static void pxa_camera_sensor_unbind(struct v4l2_async_notifier *notifier,
+		     struct v4l2_subdev *subdev,
+		     struct v4l2_async_subdev *asd)
+{
+	struct pxa_camera_dev *pcdev = v4l2_dev_to_pcdev(notifier->v4l2_dev);
+
+	mutex_lock(&pcdev->mlock);
+	dev_info(pcdev_to_dev(pcdev),
+		 "PXA Camera driver detached from camera %s\n",
+		 subdev->name);
+
+	/* disable capture, disable interrupts */
+	__raw_writel(0x3ff, pcdev->base + CICR0);
+
+	/* Stop DMA engine */
+	pxa_dma_stop_channels(pcdev);
+
+	pxa_camera_destroy_formats(pcdev);
+	video_unregister_device(&pcdev->vdev);
+	pcdev->sensor = NULL;
+
+	mutex_unlock(&pcdev->mlock);
+}
+
+/*
+ * Driver probe, remove, suspend and resume operations
+ */
 static int pxa_camera_suspend(struct device *dev)
 {
-	struct soc_camera_host *ici = to_soc_camera_host(dev);
-	struct pxa_camera_dev *pcdev = ici->priv;
+	struct pxa_camera_dev *pcdev = dev_get_drvdata(dev);
 	int i = 0, ret = 0;
 
 	pcdev->save_cicr[i++] = __raw_readl(pcdev->base + CICR0);
@@ -1550,9 +2189,8 @@ static int pxa_camera_suspend(struct device *dev)
 	pcdev->save_cicr[i++] = __raw_readl(pcdev->base + CICR3);
 	pcdev->save_cicr[i++] = __raw_readl(pcdev->base + CICR4);
 
-	if (pcdev->soc_host.icd) {
-		struct v4l2_subdev *sd = soc_camera_to_subdev(pcdev->soc_host.icd);
-		ret = v4l2_subdev_call(sd, core, s_power, 0);
+	if (pcdev->sensor) {
+		ret = sensor_call(pcdev, core, s_power, 0);
 		if (ret == -ENOIOCTLCMD)
 			ret = 0;
 	}
@@ -1562,8 +2200,7 @@ static int pxa_camera_suspend(struct device *dev)
 
 static int pxa_camera_resume(struct device *dev)
 {
-	struct soc_camera_host *ici = to_soc_camera_host(dev);
-	struct pxa_camera_dev *pcdev = ici->priv;
+	struct pxa_camera_dev *pcdev = dev_get_drvdata(dev);
 	int i = 0, ret = 0;
 
 	__raw_writel(pcdev->save_cicr[i++] & ~CICR0_ENB, pcdev->base + CICR0);
@@ -1572,9 +2209,8 @@ static int pxa_camera_resume(struct device *dev)
 	__raw_writel(pcdev->save_cicr[i++], pcdev->base + CICR3);
 	__raw_writel(pcdev->save_cicr[i++], pcdev->base + CICR4);
 
-	if (pcdev->soc_host.icd) {
-		struct v4l2_subdev *sd = soc_camera_to_subdev(pcdev->soc_host.icd);
-		ret = v4l2_subdev_call(sd, core, s_power, 1);
+	if (pcdev->sensor) {
+		ret = sensor_call(pcdev, core, s_power, 1);
 		if (ret == -ENOIOCTLCMD)
 			ret = 0;
 	}
@@ -1586,29 +2222,12 @@ static int pxa_camera_resume(struct device *dev)
 	return ret;
 }
 
-static struct soc_camera_host_ops pxa_soc_camera_host_ops = {
-	.owner		= THIS_MODULE,
-	.add		= pxa_camera_add_device,
-	.remove		= pxa_camera_remove_device,
-	.clock_start	= pxa_camera_clock_start,
-	.clock_stop	= pxa_camera_clock_stop,
-	.set_crop	= pxa_camera_set_crop,
-	.get_formats	= pxa_camera_get_formats,
-	.put_formats	= pxa_camera_put_formats,
-	.set_fmt	= pxa_camera_set_fmt,
-	.try_fmt	= pxa_camera_try_fmt,
-	.init_videobuf	= pxa_camera_init_videobuf,
-	.reqbufs	= pxa_camera_reqbufs,
-	.poll		= pxa_camera_poll,
-	.querycap	= pxa_camera_querycap,
-	.set_bus_param	= pxa_camera_set_bus_param,
-};
-
 static int pxa_camera_pdata_from_dt(struct device *dev,
-				    struct pxa_camera_dev *pcdev)
+				    struct pxa_camera_dev *pcdev,
+				    struct v4l2_async_subdev *asd)
 {
 	u32 mclk_rate;
-	struct device_node *np = dev->of_node;
+	struct device_node *remote, *np = dev->of_node;
 	struct v4l2_of_endpoint ep;
 	int err = of_property_read_u32(np, "clock-frequency",
 				       &mclk_rate);
@@ -1660,6 +2279,15 @@ static int pxa_camera_pdata_from_dt(struct device *dev,
 	if (ep.bus.parallel.flags & V4L2_MBUS_PCLK_SAMPLE_FALLING)
 		pcdev->platform_flags |= PXA_CAMERA_PCLK_EN;
 
+	asd->match_type = V4L2_ASYNC_MATCH_OF;
+	remote = of_graph_get_remote_port(np);
+	if (remote) {
+		asd->match.of.node = remote;
+		of_node_put(remote);
+	} else {
+		dev_notice(dev, "no remote for %s\n", of_node_full_name(np));
+	}
+
 out:
 	of_node_put(np);
 
@@ -1678,6 +2306,7 @@ static int pxa_camera_probe(struct platform_device *pdev)
 	};
 	dma_cap_mask_t mask;
 	struct pxad_param params;
+	char clk_name[V4L2_CLK_NAME_SIZE];
 	int irq;
 	int err = 0, i;
 
@@ -1700,10 +2329,14 @@ static int pxa_camera_probe(struct platform_device *pdev)
 
 	pcdev->pdata = pdev->dev.platform_data;
 	if (&pdev->dev.of_node && !pcdev->pdata) {
-		err = pxa_camera_pdata_from_dt(&pdev->dev, pcdev);
+		err = pxa_camera_pdata_from_dt(&pdev->dev, pcdev, &pcdev->asd);
 	} else {
 		pcdev->platform_flags = pcdev->pdata->flags;
 		pcdev->mclk = pcdev->pdata->mclk_10khz * 10000;
+		pcdev->asd.match_type = V4L2_ASYNC_MATCH_I2C;
+		pcdev->asd.match.i2c.adapter_id =
+			pcdev->pdata->sensor_i2c_adapter_id;
+		pcdev->asd.match.i2c.address = pcdev->pdata->sensor_i2c_address;
 	}
 	if (err < 0)
 		return err;
@@ -1735,6 +2368,7 @@ static int pxa_camera_probe(struct platform_device *pdev)
 
 	INIT_LIST_HEAD(&pcdev->capture);
 	spin_lock_init(&pcdev->lock);
+	mutex_init(&pcdev->mlock);
 
 	/*
 	 * Request the regions.
@@ -1767,6 +2401,7 @@ static int pxa_camera_probe(struct platform_device *pdev)
 						 &params, &pdev->dev, "CI_U");
 	if (!pcdev->dma_chans[1]) {
 		dev_err(&pdev->dev, "Can't request DMA for Y\n");
+		err = -ENODEV;
 		goto exit_free_dma_y;
 	}
 
@@ -1776,6 +2411,7 @@ static int pxa_camera_probe(struct platform_device *pdev)
 						 &params, &pdev->dev, "CI_V");
 	if (!pcdev->dma_chans[2]) {
 		dev_err(&pdev->dev, "Can't request DMA for V\n");
+		err = -ENODEV;
 		goto exit_free_dma_u;
 	}
 
@@ -1797,19 +2433,50 @@ static int pxa_camera_probe(struct platform_device *pdev)
 		goto exit_free_dma;
 	}
 
-	pcdev->soc_host.drv_name	= PXA_CAM_DRV_NAME;
-	pcdev->soc_host.ops		= &pxa_soc_camera_host_ops;
-	pcdev->soc_host.priv		= pcdev;
-	pcdev->soc_host.v4l2_dev.dev	= &pdev->dev;
-	pcdev->soc_host.nr		= pdev->id;
 	tasklet_init(&pcdev->task_eof, pxa_camera_eof, (unsigned long)pcdev);
 
-	err = soc_camera_host_register(&pcdev->soc_host);
+	pxa_camera_activate(pcdev);
+
+	dev_set_drvdata(&pdev->dev, pcdev);
+	err = v4l2_device_register(&pdev->dev, &pcdev->v4l2_dev);
 	if (err)
 		goto exit_free_dma;
 
-	return 0;
+	pcdev->asds[0] = &pcdev->asd;
+	pcdev->notifier.subdevs = pcdev->asds;
+	pcdev->notifier.num_subdevs = 1;
+	pcdev->notifier.bound = pxa_camera_sensor_bound;
+	pcdev->notifier.unbind = pxa_camera_sensor_unbind;
 
+	if (!of_have_populated_dt())
+		pcdev->asd.match_type = V4L2_ASYNC_MATCH_I2C;
+
+	err = pxa_camera_init_videobuf2(pcdev);
+	if (err)
+		goto exit_free_v4l2dev;
+
+	if (pcdev->mclk) {
+		v4l2_clk_name_i2c(clk_name, sizeof(clk_name),
+				  pcdev->asd.match.i2c.adapter_id,
+				  pcdev->asd.match.i2c.address);
+
+		pcdev->mclk_clk = v4l2_clk_register(&pxa_camera_mclk_ops,
+						    clk_name, NULL);
+		if (IS_ERR(pcdev->mclk_clk)) {
+			err = PTR_ERR(pcdev->mclk_clk);
+			goto exit_free_v4l2dev;
+		}
+	}
+
+	err = v4l2_async_notifier_register(&pcdev->v4l2_dev, &pcdev->notifier);
+	if (err)
+		goto exit_free_clk;
+
+	return 0;
+exit_free_clk:
+	v4l2_clk_unregister(pcdev->mclk_clk);
+exit_free_v4l2dev:
+	v4l2_device_unregister(&pcdev->v4l2_dev);
 exit_free_dma:
 	dma_release_channel(pcdev->dma_chans[2]);
 exit_free_dma_u:
@@ -1821,15 +2488,15 @@ exit_free_dma_y:
 
 static int pxa_camera_remove(struct platform_device *pdev)
 {
-	struct soc_camera_host *soc_host = to_soc_camera_host(&pdev->dev);
-	struct pxa_camera_dev *pcdev = container_of(soc_host,
-					struct pxa_camera_dev, soc_host);
+	struct pxa_camera_dev *pcdev = dev_get_drvdata(&pdev->dev);
 
+	pxa_camera_deactivate(pcdev);
 	dma_release_channel(pcdev->dma_chans[0]);
 	dma_release_channel(pcdev->dma_chans[1]);
 	dma_release_channel(pcdev->dma_chans[2]);
 
-	soc_camera_host_unregister(soc_host);
+	v4l2_clk_unregister(pcdev->mclk_clk);
+	v4l2_device_unregister(&pcdev->v4l2_dev);
 
 	dev_info(&pdev->dev, "PXA Camera driver unloaded\n");
 
diff --git a/drivers/media/platform/rcar-fcp.c b/drivers/media/platform/rcar-fcp.c
index 6a7bcc3028b1..f3a3f31cdfa9 100644
--- a/drivers/media/platform/rcar-fcp.c
+++ b/drivers/media/platform/rcar-fcp.c
@@ -99,10 +99,16 @@ EXPORT_SYMBOL_GPL(rcar_fcp_put);
  */
 int rcar_fcp_enable(struct rcar_fcp_device *fcp)
 {
+	int ret;
+
 	if (!fcp)
 		return 0;
 
-	return pm_runtime_get_sync(fcp->dev);
+	ret = pm_runtime_get_sync(fcp->dev);
+	if (ret < 0)
+		return ret;
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(rcar_fcp_enable);
 
@@ -159,6 +165,7 @@ static int rcar_fcp_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id rcar_fcp_of_match[] = {
+	{ .compatible = "renesas,fcpf" },
 	{ .compatible = "renesas,fcpv" },
 	{ },
 };
diff --git a/drivers/media/platform/rcar-vin/Kconfig b/drivers/media/platform/rcar-vin/Kconfig
index b2ff2d4e8bb1..111d2a151f6a 100644
--- a/drivers/media/platform/rcar-vin/Kconfig
+++ b/drivers/media/platform/rcar-vin/Kconfig
@@ -1,6 +1,6 @@
 config VIDEO_RCAR_VIN
 	tristate "R-Car Video Input (VIN) Driver"
-	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && OF && HAS_DMA
+	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && OF && HAS_DMA && MEDIA_CONTROLLER
 	depends on ARCH_RENESAS || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	---help---
diff --git a/drivers/media/platform/rcar-vin/rcar-core.c b/drivers/media/platform/rcar-vin/rcar-core.c
index 4b2007b73463..098a0b1cc10a 100644
--- a/drivers/media/platform/rcar-vin/rcar-core.c
+++ b/drivers/media/platform/rcar-vin/rcar-core.c
@@ -31,26 +31,22 @@
 
 #define notifier_to_vin(n) container_of(n, struct rvin_dev, notifier)
 
-static int rvin_mbus_supported(struct rvin_dev *vin)
+static bool rvin_mbus_supported(struct rvin_graph_entity *entity)
 {
-	struct v4l2_subdev *sd;
+	struct v4l2_subdev *sd = entity->subdev;
 	struct v4l2_subdev_mbus_code_enum code = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
 
-	sd = vin_to_source(vin);
-
 	code.index = 0;
 	while (!v4l2_subdev_call(sd, pad, enum_mbus_code, NULL, &code)) {
 		code.index++;
 		switch (code.code) {
 		case MEDIA_BUS_FMT_YUYV8_1X16:
-		case MEDIA_BUS_FMT_YUYV8_2X8:
-		case MEDIA_BUS_FMT_YUYV10_2X10:
+		case MEDIA_BUS_FMT_UYVY8_2X8:
+		case MEDIA_BUS_FMT_UYVY10_2X10:
 		case MEDIA_BUS_FMT_RGB888_1X24:
-			vin->source.code = code.code;
-			vin_dbg(vin, "Found supported media bus format: %d\n",
-				vin->source.code);
+			entity->code = code.code;
 			return true;
 		default:
 			break;
@@ -60,142 +56,168 @@ static int rvin_mbus_supported(struct rvin_dev *vin)
 	return false;
 }
 
-static int rvin_graph_notify_complete(struct v4l2_async_notifier *notifier)
+static int rvin_digital_notify_complete(struct v4l2_async_notifier *notifier)
 {
 	struct rvin_dev *vin = notifier_to_vin(notifier);
 	int ret;
 
+	/* Verify subdevices mbus format */
+	if (!rvin_mbus_supported(&vin->digital)) {
+		vin_err(vin, "Unsupported media bus format for %s\n",
+			vin->digital.subdev->name);
+		return -EINVAL;
+	}
+
+	vin_dbg(vin, "Found media bus format for %s: %d\n",
+		vin->digital.subdev->name, vin->digital.code);
+
 	ret = v4l2_device_register_subdev_nodes(&vin->v4l2_dev);
 	if (ret < 0) {
 		vin_err(vin, "Failed to register subdev nodes\n");
 		return ret;
 	}
 
-	if (!rvin_mbus_supported(vin)) {
-		vin_err(vin, "No supported mediabus format found\n");
-		return -EINVAL;
+	return rvin_v4l2_probe(vin);
+}
+
+static void rvin_digital_notify_unbind(struct v4l2_async_notifier *notifier,
+				       struct v4l2_subdev *subdev,
+				       struct v4l2_async_subdev *asd)
+{
+	struct rvin_dev *vin = notifier_to_vin(notifier);
+
+	if (vin->digital.subdev == subdev) {
+		vin_dbg(vin, "unbind digital subdev %s\n", subdev->name);
+		rvin_v4l2_remove(vin);
+		vin->digital.subdev = NULL;
+		return;
 	}
 
-	return rvin_v4l2_probe(vin);
+	vin_err(vin, "no entity for subdev %s to unbind\n", subdev->name);
 }
 
-static void rvin_graph_notify_unbind(struct v4l2_async_notifier *notifier,
-				     struct v4l2_subdev *sd,
+static int rvin_digital_notify_bound(struct v4l2_async_notifier *notifier,
+				     struct v4l2_subdev *subdev,
 				     struct v4l2_async_subdev *asd)
 {
 	struct rvin_dev *vin = notifier_to_vin(notifier);
 
-	rvin_v4l2_remove(vin);
+	v4l2_set_subdev_hostdata(subdev, vin);
+
+	if (vin->digital.asd.match.of.node == subdev->dev->of_node) {
+		vin_dbg(vin, "bound digital subdev %s\n", subdev->name);
+		vin->digital.subdev = subdev;
+		return 0;
+	}
+
+	vin_err(vin, "no entity for subdev %s to bind\n", subdev->name);
+	return -EINVAL;
 }
 
-static int rvin_graph_notify_bound(struct v4l2_async_notifier *notifier,
-				   struct v4l2_subdev *subdev,
-				   struct v4l2_async_subdev *asd)
+static int rvin_digitial_parse_v4l2(struct rvin_dev *vin,
+				    struct device_node *ep,
+				    struct v4l2_mbus_config *mbus_cfg)
 {
-	struct rvin_dev *vin = notifier_to_vin(notifier);
+	struct v4l2_of_endpoint v4l2_ep;
+	int ret;
 
-	vin_dbg(vin, "subdev %s bound\n", subdev->name);
+	ret = v4l2_of_parse_endpoint(ep, &v4l2_ep);
+	if (ret) {
+		vin_err(vin, "Could not parse v4l2 endpoint\n");
+		return -EINVAL;
+	}
 
-	vin->entity.entity = &subdev->entity;
-	vin->entity.subdev = subdev;
+	mbus_cfg->type = v4l2_ep.bus_type;
+
+	switch (mbus_cfg->type) {
+	case V4L2_MBUS_PARALLEL:
+		vin_dbg(vin, "Found PARALLEL media bus\n");
+		mbus_cfg->flags = v4l2_ep.bus.parallel.flags;
+		break;
+	case V4L2_MBUS_BT656:
+		vin_dbg(vin, "Found BT656 media bus\n");
+		mbus_cfg->flags = 0;
+		break;
+	default:
+		vin_err(vin, "Unknown media bus type\n");
+		return -EINVAL;
+	}
 
 	return 0;
 }
 
-static int rvin_graph_parse(struct rvin_dev *vin,
-			    struct device_node *node)
+static int rvin_digital_graph_parse(struct rvin_dev *vin)
 {
-	struct device_node *remote;
-	struct device_node *ep = NULL;
-	struct device_node *next;
-	int ret = 0;
-
-	while (1) {
-		next = of_graph_get_next_endpoint(node, ep);
-		if (!next)
-			break;
-
-		of_node_put(ep);
-		ep = next;
+	struct device_node *ep, *np;
+	int ret;
 
-		remote = of_graph_get_remote_port_parent(ep);
-		if (!remote) {
-			ret = -EINVAL;
-			break;
-		}
+	vin->digital.asd.match.of.node = NULL;
+	vin->digital.subdev = NULL;
 
-		/* Skip entities that we have already processed. */
-		if (remote == vin->dev->of_node) {
-			of_node_put(remote);
-			continue;
-		}
+	/*
+	 * Port 0 id 0 is local digital input, try to get it.
+	 * Not all instances can or will have this, that is OK
+	 */
+	ep = of_graph_get_endpoint_by_regs(vin->dev->of_node, 0, 0);
+	if (!ep)
+		return 0;
 
-		/* Remote node to connect */
-		if (!vin->entity.node) {
-			vin->entity.node = remote;
-			vin->entity.asd.match_type = V4L2_ASYNC_MATCH_OF;
-			vin->entity.asd.match.of.node = remote;
-			ret++;
-		}
+	np = of_graph_get_remote_port_parent(ep);
+	if (!np) {
+		vin_err(vin, "No remote parent for digital input\n");
+		of_node_put(ep);
+		return -EINVAL;
 	}
+	of_node_put(np);
 
+	ret = rvin_digitial_parse_v4l2(vin, ep, &vin->digital.mbus_cfg);
 	of_node_put(ep);
+	if (ret)
+		return ret;
 
-	return ret;
+	vin->digital.asd.match.of.node = np;
+	vin->digital.asd.match_type = V4L2_ASYNC_MATCH_OF;
+
+	return 0;
 }
 
-static int rvin_graph_init(struct rvin_dev *vin)
+static int rvin_digital_graph_init(struct rvin_dev *vin)
 {
 	struct v4l2_async_subdev **subdevs = NULL;
 	int ret;
 
-	/* Parse the graph to extract a list of subdevice DT nodes. */
-	ret = rvin_graph_parse(vin, vin->dev->of_node);
-	if (ret < 0) {
-		vin_err(vin, "Graph parsing failed\n");
-		goto done;
-	}
-
-	if (!ret) {
-		vin_err(vin, "No subdev found in graph\n");
-		goto done;
-	}
+	ret = rvin_digital_graph_parse(vin);
+	if (ret)
+		return ret;
 
-	if (ret != 1) {
-		vin_err(vin, "More then one subdev found in graph\n");
-		goto done;
+	if (!vin->digital.asd.match.of.node) {
+		vin_dbg(vin, "No digital subdevice found\n");
+		return -ENODEV;
 	}
 
 	/* Register the subdevices notifier. */
 	subdevs = devm_kzalloc(vin->dev, sizeof(*subdevs), GFP_KERNEL);
-	if (subdevs == NULL) {
-		ret = -ENOMEM;
-		goto done;
-	}
+	if (subdevs == NULL)
+		return -ENOMEM;
 
-	subdevs[0] = &vin->entity.asd;
+	subdevs[0] = &vin->digital.asd;
+
+	vin_dbg(vin, "Found digital subdevice %s\n",
+		of_node_full_name(subdevs[0]->match.of.node));
 
-	vin->notifier.subdevs = subdevs;
 	vin->notifier.num_subdevs = 1;
-	vin->notifier.bound = rvin_graph_notify_bound;
-	vin->notifier.unbind = rvin_graph_notify_unbind;
-	vin->notifier.complete = rvin_graph_notify_complete;
+	vin->notifier.subdevs = subdevs;
+	vin->notifier.bound = rvin_digital_notify_bound;
+	vin->notifier.unbind = rvin_digital_notify_unbind;
+	vin->notifier.complete = rvin_digital_notify_complete;
 
 	ret = v4l2_async_notifier_register(&vin->v4l2_dev, &vin->notifier);
 	if (ret < 0) {
 		vin_err(vin, "Notifier registration failed\n");
-		goto done;
-	}
-
-	ret = 0;
-
-done:
-	if (ret < 0) {
-		v4l2_async_notifier_unregister(&vin->notifier);
-		of_node_put(vin->entity.node);
+		return ret;
 	}
 
-	return ret;
+	return 0;
 }
 
 /* -----------------------------------------------------------------------------
@@ -209,56 +231,14 @@ static const struct of_device_id rvin_of_id_table[] = {
 	{ .compatible = "renesas,vin-r8a7790", .data = (void *)RCAR_GEN2 },
 	{ .compatible = "renesas,vin-r8a7779", .data = (void *)RCAR_H1 },
 	{ .compatible = "renesas,vin-r8a7778", .data = (void *)RCAR_M1 },
+	{ .compatible = "renesas,rcar-gen2-vin", .data = (void *)RCAR_GEN2 },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, rvin_of_id_table);
 
-static int rvin_parse_dt(struct rvin_dev *vin)
-{
-	const struct of_device_id *match;
-	struct v4l2_of_endpoint ep;
-	struct device_node *np;
-	int ret;
-
-	match = of_match_device(of_match_ptr(rvin_of_id_table), vin->dev);
-	if (!match)
-		return -ENODEV;
-
-	vin->chip = (enum chip_id)match->data;
-
-	np = of_graph_get_next_endpoint(vin->dev->of_node, NULL);
-	if (!np) {
-		vin_err(vin, "Could not find endpoint\n");
-		return -EINVAL;
-	}
-
-	ret = v4l2_of_parse_endpoint(np, &ep);
-	if (ret) {
-		vin_err(vin, "Could not parse endpoint\n");
-		return ret;
-	}
-
-	of_node_put(np);
-
-	vin->mbus_cfg.type = ep.bus_type;
-
-	switch (vin->mbus_cfg.type) {
-	case V4L2_MBUS_PARALLEL:
-		vin->mbus_cfg.flags = ep.bus.parallel.flags;
-		break;
-	case V4L2_MBUS_BT656:
-		vin->mbus_cfg.flags = 0;
-		break;
-	default:
-		vin_err(vin, "Unknown media bus type\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
 static int rcar_vin_probe(struct platform_device *pdev)
 {
+	const struct of_device_id *match;
 	struct rvin_dev *vin;
 	struct resource *mem;
 	int irq, ret;
@@ -267,11 +247,12 @@ static int rcar_vin_probe(struct platform_device *pdev)
 	if (!vin)
 		return -ENOMEM;
 
-	vin->dev = &pdev->dev;
+	match = of_match_device(of_match_ptr(rvin_of_id_table), &pdev->dev);
+	if (!match)
+		return -ENODEV;
 
-	ret = rvin_parse_dt(vin);
-	if (ret)
-		return ret;
+	vin->dev = &pdev->dev;
+	vin->chip = (enum chip_id)match->data;
 
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (mem == NULL)
@@ -282,14 +263,14 @@ static int rcar_vin_probe(struct platform_device *pdev)
 		return PTR_ERR(vin->base);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq <= 0)
-		return ret;
+	if (irq < 0)
+		return irq;
 
 	ret = rvin_dma_probe(vin, irq);
 	if (ret)
 		return ret;
 
-	ret = rvin_graph_init(vin);
+	ret = rvin_digital_graph_init(vin);
 	if (ret < 0)
 		goto error;
 
diff --git a/drivers/media/platform/rcar-vin/rcar-dma.c b/drivers/media/platform/rcar-vin/rcar-dma.c
index 496aa97b6400..9ccd5ff55e19 100644
--- a/drivers/media/platform/rcar-vin/rcar-dma.c
+++ b/drivers/media/platform/rcar-vin/rcar-dma.c
@@ -95,6 +95,7 @@
 /* Video n Module Status Register bits */
 #define VNMS_FBS_MASK		(3 << 3)
 #define VNMS_FBS_SHIFT		3
+#define VNMS_FS			(1 << 2)
 #define VNMS_AV			(1 << 1)
 #define VNMS_CA			(1 << 0)
 
@@ -131,6 +132,7 @@ static u32 rvin_read(struct rvin_dev *vin, u32 offset)
 static int rvin_setup(struct rvin_dev *vin)
 {
 	u32 vnmc, dmr, dmr2, interrupts;
+	v4l2_std_id std;
 	bool progressive = false, output_is_yuv = false, input_is_yuv = false;
 
 	switch (vin->format.field) {
@@ -141,12 +143,21 @@ static int rvin_setup(struct rvin_dev *vin)
 		vnmc = VNMC_IM_EVEN;
 		break;
 	case V4L2_FIELD_INTERLACED:
+		/* Default to TB */
+		vnmc = VNMC_IM_FULL;
+		/* Use BT if video standard can be read and is 60 Hz format */
+		if (!v4l2_subdev_call(vin_to_source(vin), video, g_std, &std)) {
+			if (std & V4L2_STD_525_60)
+				vnmc = VNMC_IM_FULL | VNMC_FOC;
+		}
+		break;
 	case V4L2_FIELD_INTERLACED_TB:
 		vnmc = VNMC_IM_FULL;
 		break;
 	case V4L2_FIELD_INTERLACED_BT:
 		vnmc = VNMC_IM_FULL | VNMC_FOC;
 		break;
+	case V4L2_FIELD_ALTERNATE:
 	case V4L2_FIELD_NONE:
 		if (vin->continuous) {
 			vnmc = VNMC_IM_ODD_EVEN;
@@ -163,24 +174,24 @@ static int rvin_setup(struct rvin_dev *vin)
 	/*
 	 * Input interface
 	 */
-	switch (vin->source.code) {
+	switch (vin->digital.code) {
 	case MEDIA_BUS_FMT_YUYV8_1X16:
 		/* BT.601/BT.1358 16bit YCbCr422 */
 		vnmc |= VNMC_INF_YUV16;
 		input_is_yuv = true;
 		break;
-	case MEDIA_BUS_FMT_YUYV8_2X8:
+	case MEDIA_BUS_FMT_UYVY8_2X8:
 		/* BT.656 8bit YCbCr422 or BT.601 8bit YCbCr422 */
-		vnmc |= vin->mbus_cfg.type == V4L2_MBUS_BT656 ?
+		vnmc |= vin->digital.mbus_cfg.type == V4L2_MBUS_BT656 ?
 			VNMC_INF_YUV8_BT656 : VNMC_INF_YUV8_BT601;
 		input_is_yuv = true;
 		break;
 	case MEDIA_BUS_FMT_RGB888_1X24:
 		vnmc |= VNMC_INF_RGB888;
 		break;
-	case MEDIA_BUS_FMT_YUYV10_2X10:
+	case MEDIA_BUS_FMT_UYVY10_2X10:
 		/* BT.656 10bit YCbCr422 or BT.601 10bit YCbCr422 */
-		vnmc |= vin->mbus_cfg.type == V4L2_MBUS_BT656 ?
+		vnmc |= vin->digital.mbus_cfg.type == V4L2_MBUS_BT656 ?
 			VNMC_INF_YUV10_BT656 : VNMC_INF_YUV10_BT601;
 		input_is_yuv = true;
 		break;
@@ -192,11 +203,11 @@ static int rvin_setup(struct rvin_dev *vin)
 	dmr2 = VNDMR2_FTEV | VNDMR2_VLV(1);
 
 	/* Hsync Signal Polarity Select */
-	if (!(vin->mbus_cfg.flags & V4L2_MBUS_HSYNC_ACTIVE_LOW))
+	if (!(vin->digital.mbus_cfg.flags & V4L2_MBUS_HSYNC_ACTIVE_LOW))
 		dmr2 |= VNDMR2_HPS;
 
 	/* Vsync Signal Polarity Select */
-	if (!(vin->mbus_cfg.flags & V4L2_MBUS_VSYNC_ACTIVE_LOW))
+	if (!(vin->digital.mbus_cfg.flags & V4L2_MBUS_VSYNC_ACTIVE_LOW))
 		dmr2 |= VNDMR2_VPS;
 
 	/*
@@ -225,11 +236,9 @@ static int rvin_setup(struct rvin_dev *vin)
 		dmr = 0;
 		break;
 	case V4L2_PIX_FMT_XBGR32:
-		if (vin->chip == RCAR_GEN2 || vin->chip == RCAR_H1) {
-			dmr = VNDMR_EXRGB;
-			break;
-		}
-		/* fall through */
+		/* Note: not supported on M1 */
+		dmr = VNDMR_EXRGB;
+		break;
 	default:
 		vin_err(vin, "Invalid pixelformat (0x%x)\n",
 			vin->format.pixelformat);
@@ -322,15 +331,26 @@ static bool rvin_capture_active(struct rvin_dev *vin)
 	return rvin_read(vin, VNMS_REG) & VNMS_CA;
 }
 
-static int rvin_get_active_slot(struct rvin_dev *vin)
+static int rvin_get_active_slot(struct rvin_dev *vin, u32 vnms)
 {
 	if (vin->continuous)
-		return (rvin_read(vin, VNMS_REG) & VNMS_FBS_MASK)
-			>> VNMS_FBS_SHIFT;
+		return (vnms & VNMS_FBS_MASK) >> VNMS_FBS_SHIFT;
 
 	return 0;
 }
 
+static enum v4l2_field rvin_get_active_field(struct rvin_dev *vin, u32 vnms)
+{
+	if (vin->format.field == V4L2_FIELD_ALTERNATE) {
+		/* If FS is set it's a Even field */
+		if (vnms & VNMS_FS)
+			return V4L2_FIELD_BOTTOM;
+		return V4L2_FIELD_TOP;
+	}
+
+	return vin->format.field;
+}
+
 static void rvin_set_slot_addr(struct rvin_dev *vin, int slot, dma_addr_t addr)
 {
 	const struct rvin_video_format *fmt;
@@ -871,7 +891,7 @@ static bool rvin_fill_hw(struct rvin_dev *vin)
 static irqreturn_t rvin_irq(int irq, void *data)
 {
 	struct rvin_dev *vin = data;
-	u32 int_status;
+	u32 int_status, vnms;
 	int slot;
 	unsigned int sequence, handled = 0;
 	unsigned long flags;
@@ -898,7 +918,8 @@ static irqreturn_t rvin_irq(int irq, void *data)
 	}
 
 	/* Prepare for capture and update state */
-	slot = rvin_get_active_slot(vin);
+	vnms = rvin_read(vin, VNMS_REG);
+	slot = rvin_get_active_slot(vin, vnms);
 	sequence = vin->sequence++;
 
 	vin_dbg(vin, "IRQ %02d: %d\tbuf0: %c buf1: %c buf2: %c\tmore: %d\n",
@@ -913,7 +934,7 @@ static irqreturn_t rvin_irq(int irq, void *data)
 		goto done;
 
 	/* Capture frame */
-	vin->queue_buf[slot]->field = vin->format.field;
+	vin->queue_buf[slot]->field = rvin_get_active_field(vin, vnms);
 	vin->queue_buf[slot]->sequence = sequence;
 	vin->queue_buf[slot]->vb2_buf.timestamp = ktime_get_ns();
 	vb2_buffer_done(&vin->queue_buf[slot]->vb2_buf, VB2_BUF_STATE_DONE);
@@ -1116,7 +1137,7 @@ static void rvin_stop_streaming(struct vb2_queue *vq)
 	rvin_disable_interrupts(vin);
 }
 
-static struct vb2_ops rvin_qops = {
+static const struct vb2_ops rvin_qops = {
 	.queue_setup		= rvin_queue_setup,
 	.buf_prepare		= rvin_buffer_prepare,
 	.buf_queue		= rvin_buffer_queue,
diff --git a/drivers/media/platform/rcar-vin/rcar-v4l2.c b/drivers/media/platform/rcar-vin/rcar-v4l2.c
index 10a5c107e8b9..2bbe6d495fa6 100644
--- a/drivers/media/platform/rcar-vin/rcar-v4l2.c
+++ b/drivers/media/platform/rcar-vin/rcar-v4l2.c
@@ -92,21 +92,84 @@ static u32 rvin_format_sizeimage(struct v4l2_pix_format *pix)
  * V4L2
  */
 
+static void rvin_reset_crop_compose(struct rvin_dev *vin)
+{
+	vin->crop.top = vin->crop.left = 0;
+	vin->crop.width = vin->source.width;
+	vin->crop.height = vin->source.height;
+
+	vin->compose.top = vin->compose.left = 0;
+	vin->compose.width = vin->format.width;
+	vin->compose.height = vin->format.height;
+}
+
+static int rvin_reset_format(struct rvin_dev *vin)
+{
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	struct v4l2_mbus_framefmt *mf = &fmt.format;
+	int ret;
+
+	fmt.pad = vin->src_pad_idx;
+
+	ret = v4l2_subdev_call(vin_to_source(vin), pad, get_fmt, NULL, &fmt);
+	if (ret)
+		return ret;
+
+	vin->format.width	= mf->width;
+	vin->format.height	= mf->height;
+	vin->format.colorspace	= mf->colorspace;
+	vin->format.field	= mf->field;
+
+	/*
+	 * If the subdevice uses ALTERNATE field mode and G_STD is
+	 * implemented use the VIN HW to combine the two fields to
+	 * one INTERLACED frame. The ALTERNATE field mode can still
+	 * be requested in S_FMT and be respected, this is just the
+	 * default which is applied at probing or when S_STD is called.
+	 */
+	if (vin->format.field == V4L2_FIELD_ALTERNATE &&
+	    v4l2_subdev_has_op(vin_to_source(vin), video, g_std))
+		vin->format.field = V4L2_FIELD_INTERLACED;
+
+	switch (vin->format.field) {
+	case V4L2_FIELD_TOP:
+	case V4L2_FIELD_BOTTOM:
+	case V4L2_FIELD_ALTERNATE:
+		vin->format.height /= 2;
+		break;
+	case V4L2_FIELD_NONE:
+	case V4L2_FIELD_INTERLACED_TB:
+	case V4L2_FIELD_INTERLACED_BT:
+	case V4L2_FIELD_INTERLACED:
+		break;
+	default:
+		vin->format.field = V4L2_FIELD_NONE;
+		break;
+	}
+
+	rvin_reset_crop_compose(vin);
+
+	return 0;
+}
+
 static int __rvin_try_format_source(struct rvin_dev *vin,
-					u32 which,
-					struct v4l2_pix_format *pix,
-					struct rvin_source_fmt *source)
+				    u32 which,
+				    struct v4l2_pix_format *pix,
+				    struct rvin_source_fmt *source)
 {
 	struct v4l2_subdev *sd;
 	struct v4l2_subdev_pad_config *pad_cfg;
 	struct v4l2_subdev_format format = {
 		.which = which,
 	};
+	enum v4l2_field field;
 	int ret;
 
 	sd = vin_to_source(vin);
 
-	v4l2_fill_mbus_format(&format.format, pix, vin->source.code);
+	v4l2_fill_mbus_format(&format.format, pix, vin->digital.code);
 
 	pad_cfg = v4l2_subdev_alloc_pad_config(sd);
 	if (pad_cfg == NULL)
@@ -114,28 +177,31 @@ static int __rvin_try_format_source(struct rvin_dev *vin,
 
 	format.pad = vin->src_pad_idx;
 
-	ret = v4l2_device_call_until_err(sd->v4l2_dev, 0, pad, set_fmt,
-					 pad_cfg, &format);
-	if (ret < 0)
-		goto cleanup;
+	field = pix->field;
+
+	ret = v4l2_subdev_call(sd, pad, set_fmt, pad_cfg, &format);
+	if (ret < 0 && ret != -ENOIOCTLCMD)
+		goto done;
 
 	v4l2_fill_pix_format(pix, &format.format);
 
+	pix->field = field;
+
 	source->width = pix->width;
 	source->height = pix->height;
 
 	vin_dbg(vin, "Source resolution: %ux%u\n", source->width,
 		source->height);
 
-cleanup:
+done:
 	v4l2_subdev_free_pad_config(pad_cfg);
-	return 0;
+	return ret;
 }
 
 static int __rvin_try_format(struct rvin_dev *vin,
-				 u32 which,
-				 struct v4l2_pix_format *pix,
-				 struct rvin_source_fmt *source)
+			     u32 which,
+			     struct v4l2_pix_format *pix,
+			     struct rvin_source_fmt *source)
 {
 	const struct rvin_video_format *info;
 	u32 rwidth, rheight, walign;
@@ -144,6 +210,10 @@ static int __rvin_try_format(struct rvin_dev *vin,
 	rwidth = pix->width;
 	rheight = pix->height;
 
+	/* Keep current field if no specific one is asked for */
+	if (pix->field == V4L2_FIELD_ANY)
+		pix->field = vin->format.field;
+
 	/*
 	 * Retrieve format information and select the current format if the
 	 * requested format isn't supported.
@@ -164,21 +234,14 @@ static int __rvin_try_format(struct rvin_dev *vin,
 	/* Limit to source capabilities */
 	__rvin_try_format_source(vin, which, pix, source);
 
-	/* If source can't match format try if VIN can scale */
-	if (source->width != rwidth || source->height != rheight)
-		rvin_scale_try(vin, pix, rwidth, rheight);
-
-	/* HW limit width to a multiple of 32 (2^5) for NV16 else 2 (2^1) */
-	walign = vin->format.pixelformat == V4L2_PIX_FMT_NV16 ? 5 : 1;
-
-	/* Limit to VIN capabilities */
-	v4l_bound_align_image(&pix->width, 2, RVIN_MAX_WIDTH, walign,
-			      &pix->height, 4, RVIN_MAX_HEIGHT, 2, 0);
-
 	switch (pix->field) {
-	case V4L2_FIELD_NONE:
 	case V4L2_FIELD_TOP:
 	case V4L2_FIELD_BOTTOM:
+	case V4L2_FIELD_ALTERNATE:
+		pix->height /= 2;
+		source->height /= 2;
+		break;
+	case V4L2_FIELD_NONE:
 	case V4L2_FIELD_INTERLACED_TB:
 	case V4L2_FIELD_INTERLACED_BT:
 	case V4L2_FIELD_INTERLACED:
@@ -188,11 +251,27 @@ static int __rvin_try_format(struct rvin_dev *vin,
 		break;
 	}
 
+	/* If source can't match format try if VIN can scale */
+	if (source->width != rwidth || source->height != rheight)
+		rvin_scale_try(vin, pix, rwidth, rheight);
+
+	/* HW limit width to a multiple of 32 (2^5) for NV16 else 2 (2^1) */
+	walign = vin->format.pixelformat == V4L2_PIX_FMT_NV16 ? 5 : 1;
+
+	/* Limit to VIN capabilities */
+	v4l_bound_align_image(&pix->width, 2, RVIN_MAX_WIDTH, walign,
+			      &pix->height, 4, RVIN_MAX_HEIGHT, 2, 0);
+
 	pix->bytesperline = max_t(u32, pix->bytesperline,
 				  rvin_format_bytesperline(pix));
 	pix->sizeimage = max_t(u32, pix->sizeimage,
 			       rvin_format_sizeimage(pix));
 
+	if (vin->chip == RCAR_M1 && pix->pixelformat == V4L2_PIX_FMT_XBGR32) {
+		vin_err(vin, "pixel format XBGR32 not supported on M1\n");
+		return -EINVAL;
+	}
+
 	vin_dbg(vin, "Requested %ux%u Got %ux%u bpl: %d size: %d\n",
 		rwidth, rheight, pix->width, pix->height,
 		pix->bytesperline, pix->sizeimage);
@@ -219,7 +298,7 @@ static int rvin_try_fmt_vid_cap(struct file *file, void *priv,
 	struct rvin_source_fmt source;
 
 	return __rvin_try_format(vin, V4L2_SUBDEV_FORMAT_TRY, &f->fmt.pix,
-				     &source);
+				 &source);
 }
 
 static int rvin_s_fmt_vid_cap(struct file *file, void *priv,
@@ -233,7 +312,7 @@ static int rvin_s_fmt_vid_cap(struct file *file, void *priv,
 		return -EBUSY;
 
 	ret = __rvin_try_format(vin, V4L2_SUBDEV_FORMAT_ACTIVE, &f->fmt.pix,
-				    &source);
+				&source);
 	if (ret)
 		return ret;
 
@@ -242,6 +321,8 @@ static int rvin_s_fmt_vid_cap(struct file *file, void *priv,
 
 	vin->format = f->fmt.pix;
 
+	rvin_reset_crop_compose(vin);
+
 	return 0;
 }
 
@@ -334,8 +415,8 @@ static int rvin_s_selection(struct file *file, void *fh,
 		vin->crop = s->r = r;
 
 		vin_dbg(vin, "Cropped %dx%d@%d:%d of %dx%d\n",
-			 r.width, r.height, r.left, r.top,
-			 vin->source.width, vin->source.height);
+			r.width, r.height, r.left, r.top,
+			vin->source.width, vin->source.height);
 		break;
 	case V4L2_SEL_TGT_COMPOSE:
 		/* Make sure compose rect fits inside output format */
@@ -359,8 +440,8 @@ static int rvin_s_selection(struct file *file, void *fh,
 		vin->compose = s->r = r;
 
 		vin_dbg(vin, "Compose %dx%d@%d:%d in %dx%d\n",
-			 r.width, r.height, r.left, r.top,
-			 vin->format.width, vin->format.height);
+			r.width, r.height, r.left, r.top,
+			vin->format.width, vin->format.height);
 		break;
 	default:
 		return -EINVAL;
@@ -381,7 +462,7 @@ static int rvin_cropcap(struct file *file, void *priv,
 	if (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
-	return v4l2_subdev_call(sd, video, cropcap, crop);
+	return v4l2_subdev_call(sd, video, g_pixelaspect, &crop->pixelaspect);
 }
 
 static int rvin_enum_input(struct file *file, void *priv,
@@ -433,35 +514,14 @@ static int rvin_querystd(struct file *file, void *priv, v4l2_std_id *a)
 static int rvin_s_std(struct file *file, void *priv, v4l2_std_id a)
 {
 	struct rvin_dev *vin = video_drvdata(file);
-	struct v4l2_subdev *sd = vin_to_source(vin);
-	struct v4l2_subdev_format fmt = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-	struct v4l2_mbus_framefmt *mf = &fmt.format;
-	int ret = v4l2_subdev_call(sd, video, s_std, a);
+	int ret;
 
+	ret = v4l2_subdev_call(vin_to_source(vin), video, s_std, a);
 	if (ret < 0)
 		return ret;
 
 	/* Changing the standard will change the width/height */
-	ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt);
-	if (ret) {
-		vin_err(vin, "Failed to get initial format\n");
-		return ret;
-	}
-
-	vin->format.width = mf->width;
-	vin->format.height = mf->height;
-
-	vin->crop.top = vin->crop.left = 0;
-	vin->crop.width = mf->width;
-	vin->crop.height = mf->height;
-
-	vin->compose.top = vin->compose.left = 0;
-	vin->compose.width = mf->width;
-	vin->compose.height = mf->height;
-
-	return 0;
+	return rvin_reset_format(vin);
 }
 
 static int rvin_g_std(struct file *file, void *priv, v4l2_std_id *a)
@@ -483,14 +543,14 @@ static int rvin_subscribe_event(struct v4l2_fh *fh,
 }
 
 static int rvin_enum_dv_timings(struct file *file, void *priv_fh,
-				    struct v4l2_enum_dv_timings *timings)
+				struct v4l2_enum_dv_timings *timings)
 {
 	struct rvin_dev *vin = video_drvdata(file);
 	struct v4l2_subdev *sd = vin_to_source(vin);
 	int pad, ret;
 
 	pad = timings->pad;
-	timings->pad = vin->src_pad_idx;
+	timings->pad = vin->sink_pad_idx;
 
 	ret = v4l2_subdev_call(sd, pad, enum_dv_timings, timings);
 
@@ -500,52 +560,51 @@ static int rvin_enum_dv_timings(struct file *file, void *priv_fh,
 }
 
 static int rvin_s_dv_timings(struct file *file, void *priv_fh,
-				    struct v4l2_dv_timings *timings)
+			     struct v4l2_dv_timings *timings)
 {
 	struct rvin_dev *vin = video_drvdata(file);
 	struct v4l2_subdev *sd = vin_to_source(vin);
-	int err;
-
-	err = v4l2_subdev_call(sd,
-			video, s_dv_timings, timings);
-	if (!err) {
-		vin->source.width = timings->bt.width;
-		vin->source.height = timings->bt.height;
-		vin->format.width = timings->bt.width;
-		vin->format.height = timings->bt.height;
-	}
-	return err;
+	int ret;
+
+	ret = v4l2_subdev_call(sd, video, s_dv_timings, timings);
+	if (ret)
+		return ret;
+
+	vin->source.width = timings->bt.width;
+	vin->source.height = timings->bt.height;
+	vin->format.width = timings->bt.width;
+	vin->format.height = timings->bt.height;
+
+	return 0;
 }
 
 static int rvin_g_dv_timings(struct file *file, void *priv_fh,
-				    struct v4l2_dv_timings *timings)
+			     struct v4l2_dv_timings *timings)
 {
 	struct rvin_dev *vin = video_drvdata(file);
 	struct v4l2_subdev *sd = vin_to_source(vin);
 
-	return v4l2_subdev_call(sd,
-			video, g_dv_timings, timings);
+	return v4l2_subdev_call(sd, video, g_dv_timings, timings);
 }
 
 static int rvin_query_dv_timings(struct file *file, void *priv_fh,
-				    struct v4l2_dv_timings *timings)
+				 struct v4l2_dv_timings *timings)
 {
 	struct rvin_dev *vin = video_drvdata(file);
 	struct v4l2_subdev *sd = vin_to_source(vin);
 
-	return v4l2_subdev_call(sd,
-			video, query_dv_timings, timings);
+	return v4l2_subdev_call(sd, video, query_dv_timings, timings);
 }
 
 static int rvin_dv_timings_cap(struct file *file, void *priv_fh,
-				    struct v4l2_dv_timings_cap *cap)
+			       struct v4l2_dv_timings_cap *cap)
 {
 	struct rvin_dev *vin = video_drvdata(file);
 	struct v4l2_subdev *sd = vin_to_source(vin);
 	int pad, ret;
 
 	pad = cap->pad;
-	cap->pad = vin->src_pad_idx;
+	cap->pad = vin->sink_pad_idx;
 
 	ret = v4l2_subdev_call(sd, pad, dv_timings_cap, cap);
 
@@ -554,6 +613,44 @@ static int rvin_dv_timings_cap(struct file *file, void *priv_fh,
 	return ret;
 }
 
+static int rvin_g_edid(struct file *file, void *fh, struct v4l2_edid *edid)
+{
+	struct rvin_dev *vin = video_drvdata(file);
+	struct v4l2_subdev *sd = vin_to_source(vin);
+	int input, ret;
+
+	if (edid->pad)
+		return -EINVAL;
+
+	input = edid->pad;
+	edid->pad = vin->sink_pad_idx;
+
+	ret = v4l2_subdev_call(sd, pad, get_edid, edid);
+
+	edid->pad = input;
+
+	return ret;
+}
+
+static int rvin_s_edid(struct file *file, void *fh, struct v4l2_edid *edid)
+{
+	struct rvin_dev *vin = video_drvdata(file);
+	struct v4l2_subdev *sd = vin_to_source(vin);
+	int input, ret;
+
+	if (edid->pad)
+		return -EINVAL;
+
+	input = edid->pad;
+	edid->pad = vin->sink_pad_idx;
+
+	ret = v4l2_subdev_call(sd, pad, set_edid, edid);
+
+	edid->pad = input;
+
+	return ret;
+}
+
 static const struct v4l2_ioctl_ops rvin_ioctl_ops = {
 	.vidioc_querycap		= rvin_querycap,
 	.vidioc_try_fmt_vid_cap		= rvin_try_fmt_vid_cap,
@@ -576,6 +673,9 @@ static const struct v4l2_ioctl_ops rvin_ioctl_ops = {
 	.vidioc_s_dv_timings		= rvin_s_dv_timings,
 	.vidioc_query_dv_timings	= rvin_query_dv_timings,
 
+	.vidioc_g_edid			= rvin_g_edid,
+	.vidioc_s_edid			= rvin_s_edid,
+
 	.vidioc_querystd		= rvin_querystd,
 	.vidioc_g_std			= rvin_g_std,
 	.vidioc_s_std			= rvin_s_std,
@@ -767,16 +867,9 @@ static void rvin_notify(struct v4l2_subdev *sd,
 
 int rvin_v4l2_probe(struct rvin_dev *vin)
 {
-	struct v4l2_subdev_format fmt = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-	struct v4l2_mbus_framefmt *mf = &fmt.format;
 	struct video_device *vdev = &vin->vdev;
 	struct v4l2_subdev *sd = vin_to_source(vin);
-#if defined(CONFIG_MEDIA_CONTROLLER)
-	int pad_idx;
-#endif
-	int ret;
+	int pad_idx, ret;
 
 	v4l2_set_subdev_hostdata(sd, vin);
 
@@ -823,41 +916,23 @@ int rvin_v4l2_probe(struct rvin_dev *vin)
 		V4L2_CAP_READWRITE;
 
 	vin->src_pad_idx = 0;
-#if defined(CONFIG_MEDIA_CONTROLLER)
 	for (pad_idx = 0; pad_idx < sd->entity.num_pads; pad_idx++)
-		if (sd->entity.pads[pad_idx].flags
-				== MEDIA_PAD_FL_SOURCE)
+		if (sd->entity.pads[pad_idx].flags == MEDIA_PAD_FL_SOURCE)
 			break;
 	if (pad_idx >= sd->entity.num_pads)
 		return -EINVAL;
 
 	vin->src_pad_idx = pad_idx;
-#endif
-	fmt.pad = vin->src_pad_idx;
 
-	/* Try to improve our guess of a reasonable window format */
-	ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt);
-	if (ret) {
-		vin_err(vin, "Failed to get initial format\n");
-		return ret;
-	}
+	vin->sink_pad_idx = 0;
+	for (pad_idx = 0; pad_idx < sd->entity.num_pads; pad_idx++)
+		if (sd->entity.pads[pad_idx].flags == MEDIA_PAD_FL_SINK) {
+			vin->sink_pad_idx = pad_idx;
+			break;
+		}
 
-	/* Set default format */
-	vin->format.width	= mf->width;
-	vin->format.height	= mf->height;
-	vin->format.colorspace	= mf->colorspace;
-	vin->format.field	= mf->field;
 	vin->format.pixelformat	= RVIN_DEFAULT_FORMAT;
-
-
-	/* Set initial crop and compose */
-	vin->crop.top = vin->crop.left = 0;
-	vin->crop.width = mf->width;
-	vin->crop.height = mf->height;
-
-	vin->compose.top = vin->compose.left = 0;
-	vin->compose.width = mf->width;
-	vin->compose.height = mf->height;
+	rvin_reset_format(vin);
 
 	ret = video_register_device(&vin->vdev, VFL_TYPE_GRABBER, -1);
 	if (ret) {
diff --git a/drivers/media/platform/rcar-vin/rcar-vin.h b/drivers/media/platform/rcar-vin/rcar-vin.h
index 31ad39a39937..727e215c0871 100644
--- a/drivers/media/platform/rcar-vin/rcar-vin.h
+++ b/drivers/media/platform/rcar-vin/rcar-vin.h
@@ -30,9 +30,9 @@
 #define HW_BUFFER_MASK 0x7f
 
 enum chip_id {
-	RCAR_GEN2,
 	RCAR_H1,
 	RCAR_M1,
+	RCAR_GEN2,
 };
 
 /**
@@ -50,12 +50,10 @@ enum rvin_dma_state {
 
 /**
  * struct rvin_source_fmt - Source information
- * @code:	Media bus format from source
  * @width:	Width from source
  * @height:	Height from source
  */
 struct rvin_source_fmt {
-	u32 code;
 	u32 width;
 	u32 height;
 };
@@ -70,12 +68,19 @@ struct rvin_video_format {
 	u8 bpp;
 };
 
+/**
+ * struct rvin_graph_entity - Video endpoint from async framework
+ * @asd:	sub-device descriptor for async framework
+ * @subdev:	subdevice matched using async framework
+ * @code:	Media bus format from source
+ * @mbus_cfg:	Media bus format from DT
+ */
 struct rvin_graph_entity {
-	struct device_node *node;
-	struct media_entity *entity;
-
 	struct v4l2_async_subdev asd;
 	struct v4l2_subdev *subdev;
+
+	u32 code;
+	struct v4l2_mbus_config mbus_cfg;
 };
 
 /**
@@ -83,14 +88,14 @@ struct rvin_graph_entity {
  * @dev:		(OF) device
  * @base:		device I/O register space remapped to virtual memory
  * @chip:		type of VIN chip
- * @mbus_cfg		media bus configuration
  *
  * @vdev:		V4L2 video device associated with VIN
  * @v4l2_dev:		V4L2 device
  * @src_pad_idx:	source pad index for media controller drivers
+ * @sink_pad_idx:	sink pad index for media controller drivers
  * @ctrl_handler:	V4L2 control handler
  * @notifier:		V4L2 asynchronous subdevs notifier
- * @entity:		entity in the DT for subdevice
+ * @digital:		entity in the DT for local digital subdevice
  *
  * @lock:		protects @queue
  * @queue:		vb2 buffers queue
@@ -113,14 +118,14 @@ struct rvin_dev {
 	struct device *dev;
 	void __iomem *base;
 	enum chip_id chip;
-	struct v4l2_mbus_config mbus_cfg;
 
 	struct video_device vdev;
 	struct v4l2_device v4l2_dev;
 	int src_pad_idx;
+	int sink_pad_idx;
 	struct v4l2_ctrl_handler ctrl_handler;
 	struct v4l2_async_notifier notifier;
-	struct rvin_graph_entity entity;
+	struct rvin_graph_entity digital;
 
 	struct mutex lock;
 	struct vb2_queue queue;
@@ -139,7 +144,7 @@ struct rvin_dev {
 	struct v4l2_rect compose;
 };
 
-#define vin_to_source(vin)		vin->entity.subdev
+#define vin_to_source(vin)		vin->digital.subdev
 
 /* Debug */
 #define vin_dbg(d, fmt, arg...)		dev_dbg(d->dev, fmt, ##arg)
diff --git a/drivers/media/platform/rcar_jpu.c b/drivers/media/platform/rcar_jpu.c
index 16782ceb29c3..d1746ecc645d 100644
--- a/drivers/media/platform/rcar_jpu.c
+++ b/drivers/media/platform/rcar_jpu.c
@@ -1183,7 +1183,7 @@ static void jpu_stop_streaming(struct vb2_queue *vq)
 	}
 }
 
-static struct vb2_ops jpu_qops = {
+static const struct vb2_ops jpu_qops = {
 	.queue_setup		= jpu_queue_setup,
 	.buf_prepare		= jpu_buf_prepare,
 	.buf_queue		= jpu_buf_queue,
diff --git a/drivers/media/platform/s5p-g2d/g2d.c b/drivers/media/platform/s5p-g2d/g2d.c
index 391dd7a7b362..62c0dec30b59 100644
--- a/drivers/media/platform/s5p-g2d/g2d.c
+++ b/drivers/media/platform/s5p-g2d/g2d.c
@@ -138,7 +138,7 @@ static void g2d_buf_queue(struct vb2_buffer *vb)
 	v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
 }
 
-static struct vb2_ops g2d_qops = {
+static const struct vb2_ops g2d_qops = {
 	.queue_setup	= g2d_queue_setup,
 	.buf_prepare	= g2d_buf_prepare,
 	.buf_queue	= g2d_buf_queue,
diff --git a/drivers/media/platform/s5p-jpeg/jpeg-core.c b/drivers/media/platform/s5p-jpeg/jpeg-core.c
index 785e6936c881..52dc7941db65 100644
--- a/drivers/media/platform/s5p-jpeg/jpeg-core.c
+++ b/drivers/media/platform/s5p-jpeg/jpeg-core.c
@@ -537,6 +537,7 @@ static const u32 fourcc_to_dwngrd_schema_id[] = {
 static int s5p_jpeg_get_dwngrd_sch_id_by_fourcc(u32 fourcc)
 {
 	int i;
+
 	for (i = 0; i < ARRAY_SIZE(fourcc_to_dwngrd_schema_id); ++i) {
 		if (fourcc_to_dwngrd_schema_id[i] == fourcc)
 			return i;
@@ -1246,17 +1247,18 @@ static int s5p_jpeg_querycap(struct file *file, void *priv,
 	struct s5p_jpeg_ctx *ctx = fh_to_ctx(priv);
 
 	if (ctx->mode == S5P_JPEG_ENCODE) {
-		strlcpy(cap->driver, S5P_JPEG_M2M_NAME " encoder",
+		strlcpy(cap->driver, S5P_JPEG_M2M_NAME,
 			sizeof(cap->driver));
 		strlcpy(cap->card, S5P_JPEG_M2M_NAME " encoder",
 			sizeof(cap->card));
 	} else {
-		strlcpy(cap->driver, S5P_JPEG_M2M_NAME " decoder",
+		strlcpy(cap->driver, S5P_JPEG_M2M_NAME,
 			sizeof(cap->driver));
 		strlcpy(cap->card, S5P_JPEG_M2M_NAME " decoder",
 			sizeof(cap->card));
 	}
-	cap->bus_info[0] = 0;
+	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
+		 dev_name(ctx->jpeg->dev));
 	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M;
 	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
 	return 0;
@@ -1273,7 +1275,8 @@ static int enum_fmt(struct s5p_jpeg_fmt *sjpeg_formats, int n,
 			if (num == f->index)
 				break;
 			/* Correct type but haven't reached our index yet,
-			 * just increment per-type index */
+			 * just increment per-type index
+			 */
 			++num;
 		}
 	}
@@ -1349,6 +1352,7 @@ static int s5p_jpeg_g_fmt(struct file *file, void *priv, struct v4l2_format *f)
 	pix->bytesperline = 0;
 	if (q_data->fmt->fourcc != V4L2_PIX_FMT_JPEG) {
 		u32 bpl = q_data->w;
+
 		if (q_data->fmt->colplanes == 1)
 			bpl = (bpl * q_data->fmt->depth) >> 3;
 		pix->bytesperline = bpl;
@@ -1374,6 +1378,7 @@ static struct s5p_jpeg_fmt *s5p_jpeg_find_format(struct s5p_jpeg_ctx *ctx,
 
 	for (k = 0; k < ARRAY_SIZE(sjpeg_formats); k++) {
 		struct s5p_jpeg_fmt *fmt = &sjpeg_formats[k];
+
 		if (fmt->fourcc == pixelformat &&
 		    fmt->flags & fmt_flag &&
 		    fmt->flags & ctx->jpeg->variant->fmt_ver_flag) {
@@ -1431,7 +1436,8 @@ static int vidioc_try_fmt(struct v4l2_format *f, struct s5p_jpeg_fmt *fmt,
 		return -EINVAL;
 
 	/* V4L2 specification suggests the driver corrects the format struct
-	 * if any of the dimensions is unsupported */
+	 * if any of the dimensions is unsupported
+	 */
 	if (q_type == FMT_TYPE_OUTPUT)
 		jpeg_bound_align_image(ctx, &pix->width, S5P_JPEG_MIN_WIDTH,
 				       S5P_JPEG_MAX_WIDTH, 0,
@@ -2489,6 +2495,7 @@ static void s5p_jpeg_buf_queue(struct vb2_buffer *vb)
 	if (ctx->mode == S5P_JPEG_DECODE &&
 	    vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) {
 		struct s5p_jpeg_q_data tmp, *q_data;
+
 		ctx->hdr_parsed = s5p_jpeg_parse_hdr(&tmp,
 		     (unsigned long)vb2_plane_vaddr(vb, 0),
 		     min((unsigned long)ctx->out_q.size,
@@ -2538,7 +2545,7 @@ static void s5p_jpeg_stop_streaming(struct vb2_queue *q)
 	pm_runtime_put(ctx->jpeg->dev);
 }
 
-static struct vb2_ops s5p_jpeg_qops = {
+static const struct vb2_ops s5p_jpeg_qops = {
 	.queue_setup		= s5p_jpeg_queue_setup,
 	.buf_prepare		= s5p_jpeg_buf_prepare,
 	.buf_queue		= s5p_jpeg_buf_queue,
@@ -2996,27 +3003,11 @@ static int s5p_jpeg_runtime_resume(struct device *dev)
 }
 #endif /* CONFIG_PM */
 
-#ifdef CONFIG_PM_SLEEP
-static int s5p_jpeg_suspend(struct device *dev)
-{
-	if (pm_runtime_suspended(dev))
-		return 0;
-
-	return s5p_jpeg_runtime_suspend(dev);
-}
-
-static int s5p_jpeg_resume(struct device *dev)
-{
-	if (pm_runtime_suspended(dev))
-		return 0;
-
-	return s5p_jpeg_runtime_resume(dev);
-}
-#endif
-
 static const struct dev_pm_ops s5p_jpeg_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(s5p_jpeg_suspend, s5p_jpeg_resume)
-	SET_RUNTIME_PM_OPS(s5p_jpeg_runtime_suspend, s5p_jpeg_runtime_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(s5p_jpeg_runtime_suspend, s5p_jpeg_runtime_resume,
+			   NULL)
 };
 
 static struct s5p_jpeg_variant s5p_jpeg_drvdata = {
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc.c b/drivers/media/platform/s5p-mfc/s5p_mfc.c
index e3f104fafd0a..0a5b8f5e011e 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc.c
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc.c
@@ -153,7 +153,7 @@ static void s5p_mfc_watchdog(unsigned long arg)
 		 * error. Now it is time to kill all instances and
 		 * reset the MFC. */
 		mfc_err("Time out during waiting for HW\n");
-		queue_work(dev->watchdog_workqueue, &dev->watchdog_work);
+		schedule_work(&dev->watchdog_work);
 	}
 	dev->watchdog_timer.expires = jiffies +
 					msecs_to_jiffies(MFC_WATCHDOG_INTERVAL);
@@ -494,7 +494,6 @@ static void s5p_mfc_handle_error(struct s5p_mfc_dev *dev,
 	s5p_mfc_hw_call(dev->mfc_ops, clear_int_flags, dev);
 	s5p_mfc_clock_off();
 	wake_up_dev(dev, reason, err);
-	return;
 }
 
 /* Header parsing interrupt handling */
@@ -759,7 +758,6 @@ static int s5p_mfc_open(struct file *file)
 	/* Allocate memory for context */
 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 	if (!ctx) {
-		mfc_err("Not enough memory\n");
 		ret = -ENOMEM;
 		goto err_alloc;
 	}
@@ -776,7 +774,7 @@ static int s5p_mfc_open(struct file *file)
 	while (dev->ctx[ctx->num]) {
 		ctx->num++;
 		if (ctx->num >= MFC_NUM_CONTEXTS) {
-			mfc_err("Too many open contexts\n");
+			mfc_debug(2, "Too many open contexts\n");
 			ret = -EBUSY;
 			goto err_no_ctx;
 		}
@@ -924,39 +922,50 @@ static int s5p_mfc_release(struct file *file)
 	struct s5p_mfc_ctx *ctx = fh_to_ctx(file->private_data);
 	struct s5p_mfc_dev *dev = ctx->dev;
 
+	/* if dev is null, do cleanup that doesn't need dev */
 	mfc_debug_enter();
-	mutex_lock(&dev->mfc_mutex);
+	if (dev)
+		mutex_lock(&dev->mfc_mutex);
 	s5p_mfc_clock_on();
 	vb2_queue_release(&ctx->vq_src);
 	vb2_queue_release(&ctx->vq_dst);
-	/* Mark context as idle */
-	clear_work_bit_irqsave(ctx);
-	/* If instance was initialised and not yet freed,
-	 * return instance and free resources */
-	if (ctx->state != MFCINST_FREE && ctx->state != MFCINST_INIT) {
-		mfc_debug(2, "Has to free instance\n");
-		s5p_mfc_close_mfc_inst(dev, ctx);
-	}
-	/* hardware locking scheme */
-	if (dev->curr_ctx == ctx->num)
-		clear_bit(0, &dev->hw_lock);
-	dev->num_inst--;
-	if (dev->num_inst == 0) {
-		mfc_debug(2, "Last instance\n");
-		s5p_mfc_deinit_hw(dev);
-		del_timer_sync(&dev->watchdog_timer);
-		if (s5p_mfc_power_off() < 0)
-			mfc_err("Power off failed\n");
+	if (dev) {
+		/* Mark context as idle */
+		clear_work_bit_irqsave(ctx);
+		/*
+		 * If instance was initialised and not yet freed,
+		 * return instance and free resources
+		*/
+		if (ctx->state != MFCINST_FREE && ctx->state != MFCINST_INIT) {
+			mfc_debug(2, "Has to free instance\n");
+			s5p_mfc_close_mfc_inst(dev, ctx);
+		}
+		/* hardware locking scheme */
+		if (dev->curr_ctx == ctx->num)
+			clear_bit(0, &dev->hw_lock);
+		dev->num_inst--;
+		if (dev->num_inst == 0) {
+			mfc_debug(2, "Last instance\n");
+			s5p_mfc_deinit_hw(dev);
+			del_timer_sync(&dev->watchdog_timer);
+			if (s5p_mfc_power_off() < 0)
+				mfc_err("Power off failed\n");
+		}
 	}
 	mfc_debug(2, "Shutting down clock\n");
 	s5p_mfc_clock_off();
-	dev->ctx[ctx->num] = NULL;
+	if (dev)
+		dev->ctx[ctx->num] = NULL;
 	s5p_mfc_dec_ctrls_delete(ctx);
 	v4l2_fh_del(&ctx->fh);
-	v4l2_fh_exit(&ctx->fh);
+	/* vdev is gone if dev is null */
+	if (dev)
+		v4l2_fh_exit(&ctx->fh);
 	kfree(ctx);
 	mfc_debug_leave();
-	mutex_unlock(&dev->mfc_mutex);
+	if (dev)
+		mutex_unlock(&dev->mfc_mutex);
+
 	return 0;
 }
 
@@ -1158,10 +1167,6 @@ static int s5p_mfc_probe(struct platform_device *pdev)
 	dev->variant = mfc_get_drv_data(pdev);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (res == NULL) {
-		dev_err(&pdev->dev, "failed to get io resource\n");
-		return -ENOENT;
-	}
 	dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(dev->regs_base))
 		return PTR_ERR(dev->regs_base);
@@ -1241,7 +1246,6 @@ static int s5p_mfc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, dev);
 
 	dev->hw_lock = 0;
-	dev->watchdog_workqueue = create_singlethread_workqueue(S5P_MFC_NAME);
 	INIT_WORK(&dev->watchdog_work, s5p_mfc_watchdog_worker);
 	atomic_set(&dev->watchdog_cnt, 0);
 	init_timer(&dev->watchdog_timer);
@@ -1298,12 +1302,28 @@ err_dma:
 static int s5p_mfc_remove(struct platform_device *pdev)
 {
 	struct s5p_mfc_dev *dev = platform_get_drvdata(pdev);
+	struct s5p_mfc_ctx *ctx;
+	int i;
 
 	v4l2_info(&dev->v4l2_dev, "Removing %s\n", pdev->name);
 
+	/*
+	 * Clear ctx dev pointer to avoid races between s5p_mfc_remove()
+	 * and s5p_mfc_release() and s5p_mfc_release() accessing ctx->dev
+	 * after s5p_mfc_remove() is run during unbind.
+	*/
+	mutex_lock(&dev->mfc_mutex);
+	for (i = 0; i < MFC_NUM_CONTEXTS; i++) {
+		ctx = dev->ctx[i];
+		if (!ctx)
+			continue;
+		/* clear ctx->dev */
+		ctx->dev = NULL;
+	}
+	mutex_unlock(&dev->mfc_mutex);
+
 	del_timer_sync(&dev->watchdog_timer);
-	flush_workqueue(dev->watchdog_workqueue);
-	destroy_workqueue(dev->watchdog_workqueue);
+	flush_work(&dev->watchdog_work);
 
 	video_unregister_device(dev->vfd_enc);
 	video_unregister_device(dev->vfd_dec);
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_common.h b/drivers/media/platform/s5p-mfc/s5p_mfc_common.h
index 373e346fce3e..46b99f28cbd7 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc_common.h
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc_common.h
@@ -292,7 +292,9 @@ struct s5p_mfc_priv_buf {
  * @warn_start:		hardware error code from which warnings start
  * @mfc_ops:		ops structure holding HW operation function pointers
  * @mfc_cmds:		cmd structure holding HW commands function pointers
+ * @mfc_regs:		structure holding MFC registers
  * @fw_ver:		loaded firmware sub-version
+ * risc_on:		flag indicates RISC is on or off
  *
  */
 struct s5p_mfc_dev {
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c b/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c
index 47c997d9e8cb..52081ddc9bf2 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c
@@ -776,11 +776,12 @@ static int vidioc_g_crop(struct file *file, void *priv,
 	u32 left, right, top, bottom;
 
 	if (ctx->state != MFCINST_HEAD_PARSED &&
-	ctx->state != MFCINST_RUNNING && ctx->state != MFCINST_FINISHING
-					&& ctx->state != MFCINST_FINISHED) {
-			mfc_err("Cannont set crop\n");
-			return -EINVAL;
-		}
+	    ctx->state != MFCINST_RUNNING &&
+	    ctx->state != MFCINST_FINISHING &&
+	    ctx->state != MFCINST_FINISHED) {
+		mfc_err("Can not get crop information\n");
+		return -EINVAL;
+	}
 	if (ctx->src_fmt->fourcc == V4L2_PIX_FMT_H264) {
 		left = s5p_mfc_hw_call(dev->mfc_ops, get_crop_info_h, ctx);
 		right = left >> S5P_FIMV_SHARED_CROP_RIGHT_SHIFT;
diff --git a/drivers/media/platform/s5p-tv/Kconfig b/drivers/media/platform/s5p-tv/Kconfig
deleted file mode 100644
index 697aaed42486..000000000000
--- a/drivers/media/platform/s5p-tv/Kconfig
+++ /dev/null
@@ -1,88 +0,0 @@
-# drivers/media/platform/s5p-tv/Kconfig
-#
-# Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
-#	http://www.samsung.com/
-# Tomasz Stanislawski <t.stanislaws@samsung.com>
-#
-# Licensed under GPL
-
-config VIDEO_SAMSUNG_S5P_TV
-	bool "Samsung TV driver for S5P platform"
-	depends on PM
-	depends on ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST
-	default n
-	---help---
-	  Say Y here to enable selecting the TV output devices for
-	  Samsung S5P platform.
-
-if VIDEO_SAMSUNG_S5P_TV
-
-config VIDEO_SAMSUNG_S5P_HDMI
-	tristate "Samsung HDMI Driver"
-	depends on VIDEO_V4L2
-	depends on I2C
-	depends on VIDEO_SAMSUNG_S5P_TV
-	select VIDEO_SAMSUNG_S5P_HDMIPHY
-	help
-	  Say Y here if you want support for the HDMI output
-	  interface in S5P Samsung SoC. The driver can be compiled
-	  as module. It is an auxiliary driver, that exposes a V4L2
-	  subdev for use by other drivers. This driver requires
-	  hdmiphy driver to work correctly.
-
-config VIDEO_SAMSUNG_S5P_HDMI_DEBUG
-	bool "Enable debug for HDMI Driver"
-	depends on VIDEO_SAMSUNG_S5P_HDMI
-	default n
-	help
-	  Enables debugging for HDMI driver.
-
-config VIDEO_SAMSUNG_S5P_HDMIPHY
-	tristate "Samsung HDMIPHY Driver"
-	depends on VIDEO_DEV && VIDEO_V4L2 && I2C
-	depends on VIDEO_SAMSUNG_S5P_TV
-	help
-	  Say Y here if you want support for the physical HDMI
-	  interface in S5P Samsung SoC. The driver can be compiled
-	  as module. It is an I2C driver, that exposes a V4L2
-	  subdev for use by other drivers.
-
-config VIDEO_SAMSUNG_S5P_SII9234
-	tristate "Samsung SII9234 Driver"
-	depends on VIDEO_DEV && VIDEO_V4L2 && I2C
-	depends on VIDEO_SAMSUNG_S5P_TV
-	help
-	  Say Y here if you want support for the MHL interface
-	  in S5P Samsung SoC. The driver can be compiled
-	  as module. It is an I2C driver, that exposes a V4L2
-	  subdev for use by other drivers.
-
-config VIDEO_SAMSUNG_S5P_SDO
-	tristate "Samsung Analog TV Driver"
-	depends on VIDEO_DEV && VIDEO_V4L2
-	depends on VIDEO_SAMSUNG_S5P_TV
-	help
-	  Say Y here if you want support for the analog TV output
-	  interface in S5P Samsung SoC. The driver can be compiled
-	  as module. It is an auxiliary driver, that exposes a V4L2
-	  subdev for use by other drivers. This driver requires
-	  hdmiphy driver to work correctly.
-
-config VIDEO_SAMSUNG_S5P_MIXER
-	tristate "Samsung Mixer and Video Processor Driver"
-	depends on VIDEO_DEV && VIDEO_V4L2
-	depends on VIDEO_SAMSUNG_S5P_TV
-	depends on HAS_DMA
-	select VIDEOBUF2_DMA_CONTIG
-	help
-	  Say Y here if you want support for the Mixer in Samsung S5P SoCs.
-	  This device produce image data to one of output interfaces.
-
-config VIDEO_SAMSUNG_S5P_MIXER_DEBUG
-	bool "Enable debug for Mixer Driver"
-	depends on VIDEO_SAMSUNG_S5P_MIXER
-	default n
-	help
-	  Enables debugging for Mixer driver.
-
-endif # VIDEO_SAMSUNG_S5P_TV
diff --git a/drivers/media/platform/s5p-tv/Makefile b/drivers/media/platform/s5p-tv/Makefile
deleted file mode 100644
index 7cd47902e269..000000000000
--- a/drivers/media/platform/s5p-tv/Makefile
+++ /dev/null
@@ -1,19 +0,0 @@
-# drivers/media/platform/samsung/tvout/Makefile
-#
-# Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
-#	http://www.samsung.com/
-# Tomasz Stanislawski <t.stanislaws@samsung.com>
-#
-# Licensed under GPL
-
-obj-$(CONFIG_VIDEO_SAMSUNG_S5P_HDMIPHY) += s5p-hdmiphy.o
-s5p-hdmiphy-y += hdmiphy_drv.o
-obj-$(CONFIG_VIDEO_SAMSUNG_S5P_SII9234) += s5p-sii9234.o
-s5p-sii9234-y += sii9234_drv.o
-obj-$(CONFIG_VIDEO_SAMSUNG_S5P_HDMI) += s5p-hdmi.o
-s5p-hdmi-y += hdmi_drv.o
-obj-$(CONFIG_VIDEO_SAMSUNG_S5P_SDO) += s5p-sdo.o
-s5p-sdo-y += sdo_drv.o
-obj-$(CONFIG_VIDEO_SAMSUNG_S5P_MIXER) += s5p-mixer.o
-s5p-mixer-y += mixer_drv.o mixer_video.o mixer_reg.o mixer_grp_layer.o mixer_vp_layer.o
-
diff --git a/drivers/media/platform/s5p-tv/hdmi_drv.c b/drivers/media/platform/s5p-tv/hdmi_drv.c
deleted file mode 100644
index e71b13e40f59..000000000000
--- a/drivers/media/platform/s5p-tv/hdmi_drv.c
+++ /dev/null
@@ -1,1059 +0,0 @@
-/*
- * Samsung HDMI interface driver
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *
- * Tomasz Stanislawski, <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published
- * by the Free Software Foundiation. either version 2 of the License,
- * or (at your option) any later version
- */
-
-#define pr_fmt(fmt) "s5p-tv (hdmi_drv): " fmt
-
-#ifdef CONFIG_VIDEO_SAMSUNG_S5P_HDMI_DEBUG
-#define DEBUG
-#endif
-
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/io.h>
-#include <linux/i2c.h>
-#include <linux/platform_device.h>
-#include <media/v4l2-subdev.h>
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/delay.h>
-#include <linux/bug.h>
-#include <linux/pm_runtime.h>
-#include <linux/clk.h>
-#include <linux/regulator/consumer.h>
-#include <linux/v4l2-dv-timings.h>
-
-#include <linux/platform_data/media/s5p_hdmi.h>
-#include <media/v4l2-common.h>
-#include <media/v4l2-dev.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-dv-timings.h>
-
-#include "regs-hdmi.h"
-
-MODULE_AUTHOR("Tomasz Stanislawski, <t.stanislaws@samsung.com>");
-MODULE_DESCRIPTION("Samsung HDMI");
-MODULE_LICENSE("GPL");
-
-struct hdmi_pulse {
-	u32 beg;
-	u32 end;
-};
-
-struct hdmi_timings {
-	struct hdmi_pulse hact;
-	u32 hsyn_pol; /* 0 - high, 1 - low */
-	struct hdmi_pulse hsyn;
-	u32 interlaced;
-	struct hdmi_pulse vact[2];
-	u32 vsyn_pol; /* 0 - high, 1 - low */
-	u32 vsyn_off;
-	struct hdmi_pulse vsyn[2];
-};
-
-struct hdmi_resources {
-	struct clk *hdmi;
-	struct clk *sclk_hdmi;
-	struct clk *sclk_pixel;
-	struct clk *sclk_hdmiphy;
-	struct clk *hdmiphy;
-	struct regulator_bulk_data *regul_bulk;
-	int regul_count;
-};
-
-struct hdmi_device {
-	/** base address of HDMI registers */
-	void __iomem *regs;
-	/** HDMI interrupt */
-	unsigned int irq;
-	/** pointer to device parent */
-	struct device *dev;
-	/** subdev generated by HDMI device */
-	struct v4l2_subdev sd;
-	/** V4L2 device structure */
-	struct v4l2_device v4l2_dev;
-	/** subdev of HDMIPHY interface */
-	struct v4l2_subdev *phy_sd;
-	/** subdev of MHL interface */
-	struct v4l2_subdev *mhl_sd;
-	/** configuration of current graphic mode */
-	const struct hdmi_timings *cur_conf;
-	/** flag indicating that timings are dirty */
-	int cur_conf_dirty;
-	/** current timings */
-	struct v4l2_dv_timings cur_timings;
-	/** other resources */
-	struct hdmi_resources res;
-};
-
-static const struct platform_device_id hdmi_driver_types[] = {
-	{
-		.name		= "s5pv210-hdmi",
-	}, {
-		.name		= "exynos4-hdmi",
-	}, {
-		/* end node */
-	}
-};
-
-static const struct v4l2_subdev_ops hdmi_sd_ops;
-
-static struct hdmi_device *sd_to_hdmi_dev(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct hdmi_device, sd);
-}
-
-static inline
-void hdmi_write(struct hdmi_device *hdev, u32 reg_id, u32 value)
-{
-	writel(value, hdev->regs + reg_id);
-}
-
-static inline
-void hdmi_write_mask(struct hdmi_device *hdev, u32 reg_id, u32 value, u32 mask)
-{
-	u32 old = readl(hdev->regs + reg_id);
-	value = (value & mask) | (old & ~mask);
-	writel(value, hdev->regs + reg_id);
-}
-
-static inline
-void hdmi_writeb(struct hdmi_device *hdev, u32 reg_id, u8 value)
-{
-	writeb(value, hdev->regs + reg_id);
-}
-
-static inline
-void hdmi_writebn(struct hdmi_device *hdev, u32 reg_id, int n, u32 value)
-{
-	switch (n) {
-	default:
-		writeb(value >> 24, hdev->regs + reg_id + 12);
-	case 3:
-		writeb(value >> 16, hdev->regs + reg_id + 8);
-	case 2:
-		writeb(value >>  8, hdev->regs + reg_id + 4);
-	case 1:
-		writeb(value >>  0, hdev->regs + reg_id + 0);
-	}
-}
-
-static inline u32 hdmi_read(struct hdmi_device *hdev, u32 reg_id)
-{
-	return readl(hdev->regs + reg_id);
-}
-
-static irqreturn_t hdmi_irq_handler(int irq, void *dev_data)
-{
-	struct hdmi_device *hdev = dev_data;
-	u32 intc_flag;
-
-	(void)irq;
-	intc_flag = hdmi_read(hdev, HDMI_INTC_FLAG);
-	/* clearing flags for HPD plug/unplug */
-	if (intc_flag & HDMI_INTC_FLAG_HPD_UNPLUG) {
-		pr_info("unplugged\n");
-		hdmi_write_mask(hdev, HDMI_INTC_FLAG, ~0,
-			HDMI_INTC_FLAG_HPD_UNPLUG);
-	}
-	if (intc_flag & HDMI_INTC_FLAG_HPD_PLUG) {
-		pr_info("plugged\n");
-		hdmi_write_mask(hdev, HDMI_INTC_FLAG, ~0,
-			HDMI_INTC_FLAG_HPD_PLUG);
-	}
-
-	return IRQ_HANDLED;
-}
-
-static void hdmi_reg_init(struct hdmi_device *hdev)
-{
-	/* enable HPD interrupts */
-	hdmi_write_mask(hdev, HDMI_INTC_CON, ~0, HDMI_INTC_EN_GLOBAL |
-		HDMI_INTC_EN_HPD_PLUG | HDMI_INTC_EN_HPD_UNPLUG);
-	/* choose DVI mode */
-	hdmi_write_mask(hdev, HDMI_MODE_SEL,
-		HDMI_MODE_DVI_EN, HDMI_MODE_MASK);
-	hdmi_write_mask(hdev, HDMI_CON_2, ~0,
-		HDMI_DVI_PERAMBLE_EN | HDMI_DVI_BAND_EN);
-	/* disable bluescreen */
-	hdmi_write_mask(hdev, HDMI_CON_0, 0, HDMI_BLUE_SCR_EN);
-	/* choose bluescreen (fecal) color */
-	hdmi_writeb(hdev, HDMI_BLUE_SCREEN_0, 0x12);
-	hdmi_writeb(hdev, HDMI_BLUE_SCREEN_1, 0x34);
-	hdmi_writeb(hdev, HDMI_BLUE_SCREEN_2, 0x56);
-}
-
-static void hdmi_timing_apply(struct hdmi_device *hdev,
-	const struct hdmi_timings *t)
-{
-	/* setting core registers */
-	hdmi_writebn(hdev, HDMI_H_BLANK_0, 2, t->hact.beg);
-	hdmi_writebn(hdev, HDMI_H_SYNC_GEN_0, 3,
-		(t->hsyn_pol << 20) | (t->hsyn.end << 10) | t->hsyn.beg);
-	hdmi_writeb(hdev, HDMI_VSYNC_POL, t->vsyn_pol);
-	hdmi_writebn(hdev, HDMI_V_BLANK_0, 3,
-		(t->vact[0].beg << 11) | t->vact[0].end);
-	hdmi_writebn(hdev, HDMI_V_SYNC_GEN_1_0, 3,
-		(t->vsyn[0].beg << 12) | t->vsyn[0].end);
-	if (t->interlaced) {
-		u32 vsyn_trans = t->hsyn.beg + t->vsyn_off;
-
-		hdmi_writeb(hdev, HDMI_INT_PRO_MODE, 1);
-		hdmi_writebn(hdev, HDMI_H_V_LINE_0, 3,
-			(t->hact.end << 12) | t->vact[1].end);
-		hdmi_writebn(hdev, HDMI_V_BLANK_F_0, 3,
-			(t->vact[1].end << 11) | t->vact[1].beg);
-		hdmi_writebn(hdev, HDMI_V_SYNC_GEN_2_0, 3,
-			(t->vsyn[1].beg << 12) | t->vsyn[1].end);
-		hdmi_writebn(hdev, HDMI_V_SYNC_GEN_3_0, 3,
-			(vsyn_trans << 12) | vsyn_trans);
-	} else {
-		hdmi_writeb(hdev, HDMI_INT_PRO_MODE, 0);
-		hdmi_writebn(hdev, HDMI_H_V_LINE_0, 3,
-			(t->hact.end << 12) | t->vact[0].end);
-	}
-
-	/* Timing generator registers */
-	hdmi_writebn(hdev, HDMI_TG_H_FSZ_L, 2, t->hact.end);
-	hdmi_writebn(hdev, HDMI_TG_HACT_ST_L, 2, t->hact.beg);
-	hdmi_writebn(hdev, HDMI_TG_HACT_SZ_L, 2, t->hact.end - t->hact.beg);
-	hdmi_writebn(hdev, HDMI_TG_VSYNC_L, 2, t->vsyn[0].beg);
-	hdmi_writebn(hdev, HDMI_TG_VACT_ST_L, 2, t->vact[0].beg);
-	hdmi_writebn(hdev, HDMI_TG_VACT_SZ_L, 2,
-		t->vact[0].end - t->vact[0].beg);
-	hdmi_writebn(hdev, HDMI_TG_VSYNC_TOP_HDMI_L, 2, t->vsyn[0].beg);
-	hdmi_writebn(hdev, HDMI_TG_FIELD_TOP_HDMI_L, 2, t->vsyn[0].beg);
-	if (t->interlaced) {
-		hdmi_write_mask(hdev, HDMI_TG_CMD, ~0, HDMI_TG_FIELD_EN);
-		hdmi_writebn(hdev, HDMI_TG_V_FSZ_L, 2, t->vact[1].end);
-		hdmi_writebn(hdev, HDMI_TG_VSYNC2_L, 2, t->vsyn[1].beg);
-		hdmi_writebn(hdev, HDMI_TG_FIELD_CHG_L, 2, t->vact[0].end);
-		hdmi_writebn(hdev, HDMI_TG_VACT_ST2_L, 2, t->vact[1].beg);
-		hdmi_writebn(hdev, HDMI_TG_VSYNC_BOT_HDMI_L, 2, t->vsyn[1].beg);
-		hdmi_writebn(hdev, HDMI_TG_FIELD_BOT_HDMI_L, 2, t->vsyn[1].beg);
-	} else {
-		hdmi_write_mask(hdev, HDMI_TG_CMD, 0, HDMI_TG_FIELD_EN);
-		hdmi_writebn(hdev, HDMI_TG_V_FSZ_L, 2, t->vact[0].end);
-	}
-}
-
-static int hdmi_conf_apply(struct hdmi_device *hdmi_dev)
-{
-	struct device *dev = hdmi_dev->dev;
-	const struct hdmi_timings *conf = hdmi_dev->cur_conf;
-	int ret;
-
-	dev_dbg(dev, "%s\n", __func__);
-
-	/* skip if conf is already synchronized with HW */
-	if (!hdmi_dev->cur_conf_dirty)
-		return 0;
-
-	/* reset hdmiphy */
-	hdmi_write_mask(hdmi_dev, HDMI_PHY_RSTOUT, ~0, HDMI_PHY_SW_RSTOUT);
-	mdelay(10);
-	hdmi_write_mask(hdmi_dev, HDMI_PHY_RSTOUT,  0, HDMI_PHY_SW_RSTOUT);
-	mdelay(10);
-
-	/* configure timings */
-	ret = v4l2_subdev_call(hdmi_dev->phy_sd, video, s_dv_timings,
-				&hdmi_dev->cur_timings);
-	if (ret) {
-		dev_err(dev, "failed to set timings\n");
-		return ret;
-	}
-
-	/* resetting HDMI core */
-	hdmi_write_mask(hdmi_dev, HDMI_CORE_RSTOUT,  0, HDMI_CORE_SW_RSTOUT);
-	mdelay(10);
-	hdmi_write_mask(hdmi_dev, HDMI_CORE_RSTOUT, ~0, HDMI_CORE_SW_RSTOUT);
-	mdelay(10);
-
-	hdmi_reg_init(hdmi_dev);
-
-	/* setting core registers */
-	hdmi_timing_apply(hdmi_dev, conf);
-
-	hdmi_dev->cur_conf_dirty = 0;
-
-	return 0;
-}
-
-static void hdmi_dumpregs(struct hdmi_device *hdev, char *prefix)
-{
-#define DUMPREG(reg_id) \
-	dev_dbg(hdev->dev, "%s:" #reg_id " = %08x\n", prefix, \
-		readl(hdev->regs + reg_id))
-
-	dev_dbg(hdev->dev, "%s: ---- CONTROL REGISTERS ----\n", prefix);
-	DUMPREG(HDMI_INTC_FLAG);
-	DUMPREG(HDMI_INTC_CON);
-	DUMPREG(HDMI_HPD_STATUS);
-	DUMPREG(HDMI_PHY_RSTOUT);
-	DUMPREG(HDMI_PHY_VPLL);
-	DUMPREG(HDMI_PHY_CMU);
-	DUMPREG(HDMI_CORE_RSTOUT);
-
-	dev_dbg(hdev->dev, "%s: ---- CORE REGISTERS ----\n", prefix);
-	DUMPREG(HDMI_CON_0);
-	DUMPREG(HDMI_CON_1);
-	DUMPREG(HDMI_CON_2);
-	DUMPREG(HDMI_SYS_STATUS);
-	DUMPREG(HDMI_PHY_STATUS);
-	DUMPREG(HDMI_STATUS_EN);
-	DUMPREG(HDMI_HPD);
-	DUMPREG(HDMI_MODE_SEL);
-	DUMPREG(HDMI_HPD_GEN);
-	DUMPREG(HDMI_DC_CONTROL);
-	DUMPREG(HDMI_VIDEO_PATTERN_GEN);
-
-	dev_dbg(hdev->dev, "%s: ---- CORE SYNC REGISTERS ----\n", prefix);
-	DUMPREG(HDMI_H_BLANK_0);
-	DUMPREG(HDMI_H_BLANK_1);
-	DUMPREG(HDMI_V_BLANK_0);
-	DUMPREG(HDMI_V_BLANK_1);
-	DUMPREG(HDMI_V_BLANK_2);
-	DUMPREG(HDMI_H_V_LINE_0);
-	DUMPREG(HDMI_H_V_LINE_1);
-	DUMPREG(HDMI_H_V_LINE_2);
-	DUMPREG(HDMI_VSYNC_POL);
-	DUMPREG(HDMI_INT_PRO_MODE);
-	DUMPREG(HDMI_V_BLANK_F_0);
-	DUMPREG(HDMI_V_BLANK_F_1);
-	DUMPREG(HDMI_V_BLANK_F_2);
-	DUMPREG(HDMI_H_SYNC_GEN_0);
-	DUMPREG(HDMI_H_SYNC_GEN_1);
-	DUMPREG(HDMI_H_SYNC_GEN_2);
-	DUMPREG(HDMI_V_SYNC_GEN_1_0);
-	DUMPREG(HDMI_V_SYNC_GEN_1_1);
-	DUMPREG(HDMI_V_SYNC_GEN_1_2);
-	DUMPREG(HDMI_V_SYNC_GEN_2_0);
-	DUMPREG(HDMI_V_SYNC_GEN_2_1);
-	DUMPREG(HDMI_V_SYNC_GEN_2_2);
-	DUMPREG(HDMI_V_SYNC_GEN_3_0);
-	DUMPREG(HDMI_V_SYNC_GEN_3_1);
-	DUMPREG(HDMI_V_SYNC_GEN_3_2);
-
-	dev_dbg(hdev->dev, "%s: ---- TG REGISTERS ----\n", prefix);
-	DUMPREG(HDMI_TG_CMD);
-	DUMPREG(HDMI_TG_H_FSZ_L);
-	DUMPREG(HDMI_TG_H_FSZ_H);
-	DUMPREG(HDMI_TG_HACT_ST_L);
-	DUMPREG(HDMI_TG_HACT_ST_H);
-	DUMPREG(HDMI_TG_HACT_SZ_L);
-	DUMPREG(HDMI_TG_HACT_SZ_H);
-	DUMPREG(HDMI_TG_V_FSZ_L);
-	DUMPREG(HDMI_TG_V_FSZ_H);
-	DUMPREG(HDMI_TG_VSYNC_L);
-	DUMPREG(HDMI_TG_VSYNC_H);
-	DUMPREG(HDMI_TG_VSYNC2_L);
-	DUMPREG(HDMI_TG_VSYNC2_H);
-	DUMPREG(HDMI_TG_VACT_ST_L);
-	DUMPREG(HDMI_TG_VACT_ST_H);
-	DUMPREG(HDMI_TG_VACT_SZ_L);
-	DUMPREG(HDMI_TG_VACT_SZ_H);
-	DUMPREG(HDMI_TG_FIELD_CHG_L);
-	DUMPREG(HDMI_TG_FIELD_CHG_H);
-	DUMPREG(HDMI_TG_VACT_ST2_L);
-	DUMPREG(HDMI_TG_VACT_ST2_H);
-	DUMPREG(HDMI_TG_VSYNC_TOP_HDMI_L);
-	DUMPREG(HDMI_TG_VSYNC_TOP_HDMI_H);
-	DUMPREG(HDMI_TG_VSYNC_BOT_HDMI_L);
-	DUMPREG(HDMI_TG_VSYNC_BOT_HDMI_H);
-	DUMPREG(HDMI_TG_FIELD_TOP_HDMI_L);
-	DUMPREG(HDMI_TG_FIELD_TOP_HDMI_H);
-	DUMPREG(HDMI_TG_FIELD_BOT_HDMI_L);
-	DUMPREG(HDMI_TG_FIELD_BOT_HDMI_H);
-#undef DUMPREG
-}
-
-static const struct hdmi_timings hdmi_timings_480p = {
-	.hact = { .beg = 138, .end = 858 },
-	.hsyn_pol = 1,
-	.hsyn = { .beg = 16, .end = 16 + 62 },
-	.interlaced = 0,
-	.vact[0] = { .beg = 42 + 3, .end = 522 + 3 },
-	.vsyn_pol = 1,
-	.vsyn[0] = { .beg = 6 + 3, .end = 12 + 3},
-};
-
-static const struct hdmi_timings hdmi_timings_576p50 = {
-	.hact = { .beg = 144, .end = 864 },
-	.hsyn_pol = 1,
-	.hsyn = { .beg = 12, .end = 12 + 64 },
-	.interlaced = 0,
-	.vact[0] = { .beg = 44 + 5, .end = 620 + 5 },
-	.vsyn_pol = 1,
-	.vsyn[0] = { .beg = 0 + 5, .end = 5 + 5},
-};
-
-static const struct hdmi_timings hdmi_timings_720p60 = {
-	.hact = { .beg = 370, .end = 1650 },
-	.hsyn_pol = 0,
-	.hsyn = { .beg = 110, .end = 110 + 40 },
-	.interlaced = 0,
-	.vact[0] = { .beg = 25 + 5, .end = 745 + 5 },
-	.vsyn_pol = 0,
-	.vsyn[0] = { .beg = 0 + 5, .end = 5 + 5},
-};
-
-static const struct hdmi_timings hdmi_timings_720p50 = {
-	.hact = { .beg = 700, .end = 1980 },
-	.hsyn_pol = 0,
-	.hsyn = { .beg = 440, .end = 440 + 40 },
-	.interlaced = 0,
-	.vact[0] = { .beg = 25 + 5, .end = 745 + 5 },
-	.vsyn_pol = 0,
-	.vsyn[0] = { .beg = 0 + 5, .end = 5 + 5},
-};
-
-static const struct hdmi_timings hdmi_timings_1080p24 = {
-	.hact = { .beg = 830, .end = 2750 },
-	.hsyn_pol = 0,
-	.hsyn = { .beg = 638, .end = 638 + 44 },
-	.interlaced = 0,
-	.vact[0] = { .beg = 41 + 4, .end = 1121 + 4 },
-	.vsyn_pol = 0,
-	.vsyn[0] = { .beg = 0 + 4, .end = 5 + 4},
-};
-
-static const struct hdmi_timings hdmi_timings_1080p60 = {
-	.hact = { .beg = 280, .end = 2200 },
-	.hsyn_pol = 0,
-	.hsyn = { .beg = 88, .end = 88 + 44 },
-	.interlaced = 0,
-	.vact[0] = { .beg = 41 + 4, .end = 1121 + 4 },
-	.vsyn_pol = 0,
-	.vsyn[0] = { .beg = 0 + 4, .end = 5 + 4},
-};
-
-static const struct hdmi_timings hdmi_timings_1080i60 = {
-	.hact = { .beg = 280, .end = 2200 },
-	.hsyn_pol = 0,
-	.hsyn = { .beg = 88, .end = 88 + 44 },
-	.interlaced = 1,
-	.vact[0] = { .beg = 20 + 2, .end = 560 + 2 },
-	.vact[1] = { .beg = 583 + 2, .end = 1123 + 2 },
-	.vsyn_pol = 0,
-	.vsyn_off = 1100,
-	.vsyn[0] = { .beg = 0 + 2, .end = 5 + 2},
-	.vsyn[1] = { .beg = 562 + 2, .end = 567 + 2},
-};
-
-static const struct hdmi_timings hdmi_timings_1080i50 = {
-	.hact = { .beg = 720, .end = 2640 },
-	.hsyn_pol = 0,
-	.hsyn = { .beg = 528, .end = 528 + 44 },
-	.interlaced = 1,
-	.vact[0] = { .beg = 20 + 2, .end = 560 + 2 },
-	.vact[1] = { .beg = 583 + 2, .end = 1123 + 2 },
-	.vsyn_pol = 0,
-	.vsyn_off = 1320,
-	.vsyn[0] = { .beg = 0 + 2, .end = 5 + 2},
-	.vsyn[1] = { .beg = 562 + 2, .end = 567 + 2},
-};
-
-static const struct hdmi_timings hdmi_timings_1080p50 = {
-	.hact = { .beg = 720, .end = 2640 },
-	.hsyn_pol = 0,
-	.hsyn = { .beg = 528, .end = 528 + 44 },
-	.interlaced = 0,
-	.vact[0] = { .beg = 41 + 4, .end = 1121 + 4 },
-	.vsyn_pol = 0,
-	.vsyn[0] = { .beg = 0 + 4, .end = 5 + 4},
-};
-
-/* default hdmi_timings index of the timings configured on probe */
-#define HDMI_DEFAULT_TIMINGS_IDX (0)
-
-static const struct {
-	bool reduced_fps;
-	const struct v4l2_dv_timings dv_timings;
-	const struct hdmi_timings *hdmi_timings;
-} hdmi_timings[] = {
-	{ false, V4L2_DV_BT_CEA_720X480P59_94, &hdmi_timings_480p    },
-	{ false, V4L2_DV_BT_CEA_720X576P50,    &hdmi_timings_576p50  },
-	{ false, V4L2_DV_BT_CEA_1280X720P50,   &hdmi_timings_720p50  },
-	{ true,  V4L2_DV_BT_CEA_1280X720P60,   &hdmi_timings_720p60  },
-	{ false, V4L2_DV_BT_CEA_1920X1080P24,  &hdmi_timings_1080p24 },
-	{ false, V4L2_DV_BT_CEA_1920X1080P30,  &hdmi_timings_1080p60 },
-	{ false, V4L2_DV_BT_CEA_1920X1080P50,  &hdmi_timings_1080p50 },
-	{ false, V4L2_DV_BT_CEA_1920X1080I50,  &hdmi_timings_1080i50 },
-	{ false, V4L2_DV_BT_CEA_1920X1080I60,  &hdmi_timings_1080i60 },
-	{ false, V4L2_DV_BT_CEA_1920X1080P60,  &hdmi_timings_1080p60 },
-};
-
-static int hdmi_streamon(struct hdmi_device *hdev)
-{
-	struct device *dev = hdev->dev;
-	struct hdmi_resources *res = &hdev->res;
-	int ret, tries;
-
-	dev_dbg(dev, "%s\n", __func__);
-
-	ret = hdmi_conf_apply(hdev);
-	if (ret)
-		return ret;
-
-	ret = v4l2_subdev_call(hdev->phy_sd, video, s_stream, 1);
-	if (ret)
-		return ret;
-
-	/* waiting for HDMIPHY's PLL to get to steady state */
-	for (tries = 100; tries; --tries) {
-		u32 val = hdmi_read(hdev, HDMI_PHY_STATUS);
-		if (val & HDMI_PHY_STATUS_READY)
-			break;
-		mdelay(1);
-	}
-	/* steady state not achieved */
-	if (tries == 0) {
-		dev_err(dev, "hdmiphy's pll could not reach steady state.\n");
-		v4l2_subdev_call(hdev->phy_sd, video, s_stream, 0);
-		hdmi_dumpregs(hdev, "hdmiphy - s_stream");
-		return -EIO;
-	}
-
-	/* starting MHL */
-	ret = v4l2_subdev_call(hdev->mhl_sd, video, s_stream, 1);
-	if (hdev->mhl_sd && ret) {
-		v4l2_subdev_call(hdev->phy_sd, video, s_stream, 0);
-		hdmi_dumpregs(hdev, "mhl - s_stream");
-		return -EIO;
-	}
-
-	/* hdmiphy clock is used for HDMI in streaming mode */
-	clk_disable(res->sclk_hdmi);
-	clk_set_parent(res->sclk_hdmi, res->sclk_hdmiphy);
-	clk_enable(res->sclk_hdmi);
-
-	/* enable HDMI and timing generator */
-	hdmi_write_mask(hdev, HDMI_CON_0, ~0, HDMI_EN);
-	hdmi_write_mask(hdev, HDMI_TG_CMD, ~0, HDMI_TG_EN);
-	hdmi_dumpregs(hdev, "streamon");
-	return 0;
-}
-
-static int hdmi_streamoff(struct hdmi_device *hdev)
-{
-	struct device *dev = hdev->dev;
-	struct hdmi_resources *res = &hdev->res;
-
-	dev_dbg(dev, "%s\n", __func__);
-
-	hdmi_write_mask(hdev, HDMI_CON_0, 0, HDMI_EN);
-	hdmi_write_mask(hdev, HDMI_TG_CMD, 0, HDMI_TG_EN);
-
-	/* pixel(vpll) clock is used for HDMI in config mode */
-	clk_disable(res->sclk_hdmi);
-	clk_set_parent(res->sclk_hdmi, res->sclk_pixel);
-	clk_enable(res->sclk_hdmi);
-
-	v4l2_subdev_call(hdev->mhl_sd, video, s_stream, 0);
-	v4l2_subdev_call(hdev->phy_sd, video, s_stream, 0);
-
-	hdmi_dumpregs(hdev, "streamoff");
-	return 0;
-}
-
-static int hdmi_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct hdmi_device *hdev = sd_to_hdmi_dev(sd);
-	struct device *dev = hdev->dev;
-
-	dev_dbg(dev, "%s(%d)\n", __func__, enable);
-	if (enable)
-		return hdmi_streamon(hdev);
-	return hdmi_streamoff(hdev);
-}
-
-static int hdmi_resource_poweron(struct hdmi_resources *res)
-{
-	int ret;
-
-	/* turn HDMI power on */
-	ret = regulator_bulk_enable(res->regul_count, res->regul_bulk);
-	if (ret < 0)
-		return ret;
-	/* power-on hdmi physical interface */
-	clk_enable(res->hdmiphy);
-	/* use VPP as parent clock; HDMIPHY is not working yet */
-	clk_set_parent(res->sclk_hdmi, res->sclk_pixel);
-	/* turn clocks on */
-	clk_enable(res->sclk_hdmi);
-
-	return 0;
-}
-
-static void hdmi_resource_poweroff(struct hdmi_resources *res)
-{
-	/* turn clocks off */
-	clk_disable(res->sclk_hdmi);
-	/* power-off hdmiphy */
-	clk_disable(res->hdmiphy);
-	/* turn HDMI power off */
-	regulator_bulk_disable(res->regul_count, res->regul_bulk);
-}
-
-static int hdmi_s_power(struct v4l2_subdev *sd, int on)
-{
-	struct hdmi_device *hdev = sd_to_hdmi_dev(sd);
-	int ret;
-
-	if (on)
-		ret = pm_runtime_get_sync(hdev->dev);
-	else
-		ret = pm_runtime_put_sync(hdev->dev);
-	/* only values < 0 indicate errors */
-	return ret < 0 ? ret : 0;
-}
-
-static int hdmi_s_dv_timings(struct v4l2_subdev *sd,
-	struct v4l2_dv_timings *timings)
-{
-	struct hdmi_device *hdev = sd_to_hdmi_dev(sd);
-	struct device *dev = hdev->dev;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(hdmi_timings); i++)
-		if (v4l2_match_dv_timings(&hdmi_timings[i].dv_timings,
-					timings, 0, false))
-			break;
-	if (i == ARRAY_SIZE(hdmi_timings)) {
-		dev_err(dev, "timings not supported\n");
-		return -EINVAL;
-	}
-	hdev->cur_conf = hdmi_timings[i].hdmi_timings;
-	hdev->cur_conf_dirty = 1;
-	hdev->cur_timings = *timings;
-	if (!hdmi_timings[i].reduced_fps)
-		hdev->cur_timings.bt.flags &= ~V4L2_DV_FL_CAN_REDUCE_FPS;
-	return 0;
-}
-
-static int hdmi_g_dv_timings(struct v4l2_subdev *sd,
-	struct v4l2_dv_timings *timings)
-{
-	*timings = sd_to_hdmi_dev(sd)->cur_timings;
-	return 0;
-}
-
-static int hdmi_get_fmt(struct v4l2_subdev *sd,
-		struct v4l2_subdev_pad_config *cfg,
-		struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct hdmi_device *hdev = sd_to_hdmi_dev(sd);
-	const struct hdmi_timings *t = hdev->cur_conf;
-
-	dev_dbg(hdev->dev, "%s\n", __func__);
-	if (!hdev->cur_conf)
-		return -EINVAL;
-	if (format->pad)
-		return -EINVAL;
-
-	memset(fmt, 0, sizeof(*fmt));
-	fmt->width = t->hact.end - t->hact.beg;
-	fmt->height = t->vact[0].end - t->vact[0].beg;
-	fmt->code = MEDIA_BUS_FMT_FIXED; /* means RGB888 */
-	fmt->colorspace = V4L2_COLORSPACE_SRGB;
-	if (t->interlaced) {
-		fmt->field = V4L2_FIELD_INTERLACED;
-		fmt->height *= 2;
-	} else {
-		fmt->field = V4L2_FIELD_NONE;
-	}
-	return 0;
-}
-
-static int hdmi_enum_dv_timings(struct v4l2_subdev *sd,
-	struct v4l2_enum_dv_timings *timings)
-{
-	if (timings->pad != 0)
-		return -EINVAL;
-	if (timings->index >= ARRAY_SIZE(hdmi_timings))
-		return -EINVAL;
-	timings->timings = hdmi_timings[timings->index].dv_timings;
-	if (!hdmi_timings[timings->index].reduced_fps)
-		timings->timings.bt.flags &= ~V4L2_DV_FL_CAN_REDUCE_FPS;
-	return 0;
-}
-
-static int hdmi_dv_timings_cap(struct v4l2_subdev *sd,
-	struct v4l2_dv_timings_cap *cap)
-{
-	struct hdmi_device *hdev = sd_to_hdmi_dev(sd);
-
-	if (cap->pad != 0)
-		return -EINVAL;
-
-	/* Let the phy fill in the pixelclock range */
-	v4l2_subdev_call(hdev->phy_sd, pad, dv_timings_cap, cap);
-	cap->type = V4L2_DV_BT_656_1120;
-	cap->bt.min_width = 720;
-	cap->bt.max_width = 1920;
-	cap->bt.min_height = 480;
-	cap->bt.max_height = 1080;
-	cap->bt.standards = V4L2_DV_BT_STD_CEA861;
-	cap->bt.capabilities = V4L2_DV_BT_CAP_INTERLACED |
-			       V4L2_DV_BT_CAP_PROGRESSIVE;
-	return 0;
-}
-
-static const struct v4l2_subdev_core_ops hdmi_sd_core_ops = {
-	.s_power = hdmi_s_power,
-};
-
-static const struct v4l2_subdev_video_ops hdmi_sd_video_ops = {
-	.s_dv_timings = hdmi_s_dv_timings,
-	.g_dv_timings = hdmi_g_dv_timings,
-	.s_stream = hdmi_s_stream,
-};
-
-static const struct v4l2_subdev_pad_ops hdmi_sd_pad_ops = {
-	.enum_dv_timings = hdmi_enum_dv_timings,
-	.dv_timings_cap = hdmi_dv_timings_cap,
-	.get_fmt = hdmi_get_fmt,
-};
-
-static const struct v4l2_subdev_ops hdmi_sd_ops = {
-	.core = &hdmi_sd_core_ops,
-	.video = &hdmi_sd_video_ops,
-	.pad = &hdmi_sd_pad_ops,
-};
-
-static int hdmi_runtime_suspend(struct device *dev)
-{
-	struct v4l2_subdev *sd = dev_get_drvdata(dev);
-	struct hdmi_device *hdev = sd_to_hdmi_dev(sd);
-
-	dev_dbg(dev, "%s\n", __func__);
-	v4l2_subdev_call(hdev->mhl_sd, core, s_power, 0);
-	hdmi_resource_poweroff(&hdev->res);
-	/* flag that device context is lost */
-	hdev->cur_conf_dirty = 1;
-	return 0;
-}
-
-static int hdmi_runtime_resume(struct device *dev)
-{
-	struct v4l2_subdev *sd = dev_get_drvdata(dev);
-	struct hdmi_device *hdev = sd_to_hdmi_dev(sd);
-	int ret;
-
-	dev_dbg(dev, "%s\n", __func__);
-
-	ret = hdmi_resource_poweron(&hdev->res);
-	if (ret < 0)
-		return ret;
-
-	/* starting MHL */
-	ret = v4l2_subdev_call(hdev->mhl_sd, core, s_power, 1);
-	if (hdev->mhl_sd && ret)
-		goto fail;
-
-	dev_dbg(dev, "poweron succeed\n");
-
-	return 0;
-
-fail:
-	hdmi_resource_poweroff(&hdev->res);
-	dev_err(dev, "poweron failed\n");
-
-	return ret;
-}
-
-static const struct dev_pm_ops hdmi_pm_ops = {
-	.runtime_suspend = hdmi_runtime_suspend,
-	.runtime_resume	 = hdmi_runtime_resume,
-};
-
-static void hdmi_resource_clear_clocks(struct hdmi_resources *res)
-{
-	res->hdmi	 = ERR_PTR(-EINVAL);
-	res->sclk_hdmi	 = ERR_PTR(-EINVAL);
-	res->sclk_pixel	 = ERR_PTR(-EINVAL);
-	res->sclk_hdmiphy = ERR_PTR(-EINVAL);
-	res->hdmiphy	 = ERR_PTR(-EINVAL);
-}
-
-static void hdmi_resources_cleanup(struct hdmi_device *hdev)
-{
-	struct hdmi_resources *res = &hdev->res;
-
-	dev_dbg(hdev->dev, "HDMI resource cleanup\n");
-	/* put clocks, power */
-	if (res->regul_count)
-		regulator_bulk_free(res->regul_count, res->regul_bulk);
-	/* kfree is NULL-safe */
-	kfree(res->regul_bulk);
-	if (!IS_ERR(res->hdmiphy))
-		clk_put(res->hdmiphy);
-	if (!IS_ERR(res->sclk_hdmiphy))
-		clk_put(res->sclk_hdmiphy);
-	if (!IS_ERR(res->sclk_pixel))
-		clk_put(res->sclk_pixel);
-	if (!IS_ERR(res->sclk_hdmi))
-		clk_put(res->sclk_hdmi);
-	if (!IS_ERR(res->hdmi))
-		clk_put(res->hdmi);
-	memset(res, 0, sizeof(*res));
-	hdmi_resource_clear_clocks(res);
-}
-
-static int hdmi_resources_init(struct hdmi_device *hdev)
-{
-	struct device *dev = hdev->dev;
-	struct hdmi_resources *res = &hdev->res;
-	static char *supply[] = {
-		"hdmi-en",
-		"vdd",
-		"vdd_osc",
-		"vdd_pll",
-	};
-	int i, ret;
-
-	dev_dbg(dev, "HDMI resource init\n");
-
-	memset(res, 0, sizeof(*res));
-	hdmi_resource_clear_clocks(res);
-
-	/* get clocks, power */
-	res->hdmi = clk_get(dev, "hdmi");
-	if (IS_ERR(res->hdmi)) {
-		dev_err(dev, "failed to get clock 'hdmi'\n");
-		goto fail;
-	}
-	res->sclk_hdmi = clk_get(dev, "sclk_hdmi");
-	if (IS_ERR(res->sclk_hdmi)) {
-		dev_err(dev, "failed to get clock 'sclk_hdmi'\n");
-		goto fail;
-	}
-	res->sclk_pixel = clk_get(dev, "sclk_pixel");
-	if (IS_ERR(res->sclk_pixel)) {
-		dev_err(dev, "failed to get clock 'sclk_pixel'\n");
-		goto fail;
-	}
-	res->sclk_hdmiphy = clk_get(dev, "sclk_hdmiphy");
-	if (IS_ERR(res->sclk_hdmiphy)) {
-		dev_err(dev, "failed to get clock 'sclk_hdmiphy'\n");
-		goto fail;
-	}
-	res->hdmiphy = clk_get(dev, "hdmiphy");
-	if (IS_ERR(res->hdmiphy)) {
-		dev_err(dev, "failed to get clock 'hdmiphy'\n");
-		goto fail;
-	}
-	res->regul_bulk = kcalloc(ARRAY_SIZE(supply),
-				  sizeof(res->regul_bulk[0]), GFP_KERNEL);
-	if (!res->regul_bulk) {
-		dev_err(dev, "failed to get memory for regulators\n");
-		goto fail;
-	}
-	for (i = 0; i < ARRAY_SIZE(supply); ++i) {
-		res->regul_bulk[i].supply = supply[i];
-		res->regul_bulk[i].consumer = NULL;
-	}
-
-	ret = regulator_bulk_get(dev, ARRAY_SIZE(supply), res->regul_bulk);
-	if (ret) {
-		dev_err(dev, "failed to get regulators\n");
-		goto fail;
-	}
-	res->regul_count = ARRAY_SIZE(supply);
-
-	return 0;
-fail:
-	dev_err(dev, "HDMI resource init - failed\n");
-	hdmi_resources_cleanup(hdev);
-	return -ENODEV;
-}
-
-static int hdmi_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct resource *res;
-	struct i2c_adapter *adapter;
-	struct v4l2_subdev *sd;
-	struct hdmi_device *hdmi_dev = NULL;
-	struct s5p_hdmi_platform_data *pdata = dev->platform_data;
-	int ret;
-
-	dev_dbg(dev, "probe start\n");
-
-	if (!pdata) {
-		dev_err(dev, "platform data is missing\n");
-		ret = -ENODEV;
-		goto fail;
-	}
-
-	hdmi_dev = devm_kzalloc(&pdev->dev, sizeof(*hdmi_dev), GFP_KERNEL);
-	if (!hdmi_dev) {
-		dev_err(dev, "out of memory\n");
-		ret = -ENOMEM;
-		goto fail;
-	}
-
-	hdmi_dev->dev = dev;
-
-	ret = hdmi_resources_init(hdmi_dev);
-	if (ret)
-		goto fail;
-
-	/* mapping HDMI registers */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (res == NULL) {
-		dev_err(dev, "get memory resource failed.\n");
-		ret = -ENXIO;
-		goto fail_init;
-	}
-
-	hdmi_dev->regs = devm_ioremap(&pdev->dev, res->start,
-				      resource_size(res));
-	if (hdmi_dev->regs == NULL) {
-		dev_err(dev, "register mapping failed.\n");
-		ret = -ENXIO;
-		goto fail_init;
-	}
-
-	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (res == NULL) {
-		dev_err(dev, "get interrupt resource failed.\n");
-		ret = -ENXIO;
-		goto fail_init;
-	}
-
-	ret = devm_request_irq(&pdev->dev, res->start, hdmi_irq_handler, 0,
-			       "hdmi", hdmi_dev);
-	if (ret) {
-		dev_err(dev, "request interrupt failed.\n");
-		goto fail_init;
-	}
-	hdmi_dev->irq = res->start;
-
-	/* setting v4l2 name to prevent WARN_ON in v4l2_device_register */
-	strlcpy(hdmi_dev->v4l2_dev.name, dev_name(dev),
-		sizeof(hdmi_dev->v4l2_dev.name));
-	/* passing NULL owner prevents driver from erasing drvdata */
-	ret = v4l2_device_register(NULL, &hdmi_dev->v4l2_dev);
-	if (ret) {
-		dev_err(dev, "could not register v4l2 device.\n");
-		goto fail_init;
-	}
-
-	/* testing if hdmiphy info is present */
-	if (!pdata->hdmiphy_info) {
-		dev_err(dev, "hdmiphy info is missing in platform data\n");
-		ret = -ENXIO;
-		goto fail_vdev;
-	}
-
-	adapter = i2c_get_adapter(pdata->hdmiphy_bus);
-	if (adapter == NULL) {
-		dev_err(dev, "hdmiphy adapter request failed\n");
-		ret = -ENXIO;
-		goto fail_vdev;
-	}
-
-	hdmi_dev->phy_sd = v4l2_i2c_new_subdev_board(&hdmi_dev->v4l2_dev,
-		adapter, pdata->hdmiphy_info, NULL);
-	/* on failure or not adapter is no longer useful */
-	i2c_put_adapter(adapter);
-	if (hdmi_dev->phy_sd == NULL) {
-		dev_err(dev, "missing subdev for hdmiphy\n");
-		ret = -ENODEV;
-		goto fail_vdev;
-	}
-
-	/* initialization of MHL interface if present */
-	if (pdata->mhl_info) {
-		adapter = i2c_get_adapter(pdata->mhl_bus);
-		if (adapter == NULL) {
-			dev_err(dev, "MHL adapter request failed\n");
-			ret = -ENXIO;
-			goto fail_vdev;
-		}
-
-		hdmi_dev->mhl_sd = v4l2_i2c_new_subdev_board(
-			&hdmi_dev->v4l2_dev, adapter,
-			pdata->mhl_info, NULL);
-		/* on failure or not adapter is no longer useful */
-		i2c_put_adapter(adapter);
-		if (hdmi_dev->mhl_sd == NULL) {
-			dev_err(dev, "missing subdev for MHL\n");
-			ret = -ENODEV;
-			goto fail_vdev;
-		}
-	}
-
-	clk_enable(hdmi_dev->res.hdmi);
-
-	pm_runtime_enable(dev);
-
-	sd = &hdmi_dev->sd;
-	v4l2_subdev_init(sd, &hdmi_sd_ops);
-	sd->owner = THIS_MODULE;
-
-	strlcpy(sd->name, "s5p-hdmi", sizeof(sd->name));
-	hdmi_dev->cur_timings =
-		hdmi_timings[HDMI_DEFAULT_TIMINGS_IDX].dv_timings;
-	/* FIXME: missing fail timings is not supported */
-	hdmi_dev->cur_conf =
-		hdmi_timings[HDMI_DEFAULT_TIMINGS_IDX].hdmi_timings;
-	hdmi_dev->cur_conf_dirty = 1;
-
-	/* storing subdev for call that have only access to struct device */
-	dev_set_drvdata(dev, sd);
-
-	dev_info(dev, "probe successful\n");
-
-	return 0;
-
-fail_vdev:
-	v4l2_device_unregister(&hdmi_dev->v4l2_dev);
-
-fail_init:
-	hdmi_resources_cleanup(hdmi_dev);
-
-fail:
-	dev_err(dev, "probe failed\n");
-	return ret;
-}
-
-static int hdmi_remove(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct v4l2_subdev *sd = dev_get_drvdata(dev);
-	struct hdmi_device *hdmi_dev = sd_to_hdmi_dev(sd);
-
-	pm_runtime_disable(dev);
-	clk_disable(hdmi_dev->res.hdmi);
-	v4l2_device_unregister(&hdmi_dev->v4l2_dev);
-	disable_irq(hdmi_dev->irq);
-	hdmi_resources_cleanup(hdmi_dev);
-	dev_info(dev, "remove successful\n");
-
-	return 0;
-}
-
-static struct platform_driver hdmi_driver __refdata = {
-	.probe = hdmi_probe,
-	.remove = hdmi_remove,
-	.id_table = hdmi_driver_types,
-	.driver = {
-		.name = "s5p-hdmi",
-		.pm = &hdmi_pm_ops,
-	}
-};
-
-module_platform_driver(hdmi_driver);
diff --git a/drivers/media/platform/s5p-tv/hdmiphy_drv.c b/drivers/media/platform/s5p-tv/hdmiphy_drv.c
deleted file mode 100644
index aae652351aa8..000000000000
--- a/drivers/media/platform/s5p-tv/hdmiphy_drv.c
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Samsung HDMI Physical interface driver
- *
- * Copyright (C) 2010-2011 Samsung Electronics Co.Ltd
- * Author: Tomasz Stanislawski <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/i2c.h>
-#include <linux/slab.h>
-#include <linux/clk.h>
-#include <linux/io.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/err.h>
-
-#include <media/v4l2-subdev.h>
-
-MODULE_AUTHOR("Tomasz Stanislawski <t.stanislaws@samsung.com>");
-MODULE_DESCRIPTION("Samsung HDMI Physical interface driver");
-MODULE_LICENSE("GPL");
-
-struct hdmiphy_conf {
-	unsigned long pixclk;
-	const u8 *data;
-};
-
-struct hdmiphy_ctx {
-	struct v4l2_subdev sd;
-	const struct hdmiphy_conf *conf_tab;
-};
-
-static const struct hdmiphy_conf hdmiphy_conf_s5pv210[] = {
-	{ .pixclk = 27000000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD8, 0x10, 0x1C, 0x30, 0x40,
-		0x6B, 0x10, 0x02, 0x52, 0xDF, 0xF2, 0x54, 0x87,
-		0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x22, 0x40, 0xE3, 0x26, 0x00, 0x00, 0x00, 0x00, }
-	},
-	{ .pixclk = 27027000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD4, 0x10, 0x9C, 0x09, 0x64,
-		0x6B, 0x10, 0x02, 0x52, 0xDF, 0xF2, 0x54, 0x87,
-		0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x22, 0x40, 0xE2, 0x26, 0x00, 0x00, 0x00, 0x00, }
-	},
-	{ .pixclk = 74176000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD8, 0x10, 0x9C, 0xEF, 0x5B,
-		0x6D, 0x10, 0x01, 0x52, 0xEF, 0xF3, 0x54, 0xB9,
-		0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x22, 0x40, 0xA5, 0x26, 0x01, 0x00, 0x00, 0x00, }
-	},
-	{ .pixclk = 74250000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD8, 0x10, 0x9C, 0xF8, 0x40,
-		0x6A, 0x10, 0x01, 0x52, 0xFF, 0xF1, 0x54, 0xBA,
-		0x84, 0x00, 0x10, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x22, 0x40, 0xA4, 0x26, 0x01, 0x00, 0x00, 0x00, }
-	},
-	{ /* end marker */ }
-};
-
-static const struct hdmiphy_conf hdmiphy_conf_exynos4210[] = {
-	{ .pixclk = 27000000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD8, 0x10, 0x1C, 0x30, 0x40,
-		0x6B, 0x10, 0x02, 0x51, 0xDF, 0xF2, 0x54, 0x87,
-		0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x22, 0x40, 0xE3, 0x26, 0x00, 0x00, 0x00, 0x00, }
-	},
-	{ .pixclk = 27027000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD4, 0x10, 0x9C, 0x09, 0x64,
-		0x6B, 0x10, 0x02, 0x51, 0xDF, 0xF2, 0x54, 0x87,
-		0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x22, 0x40, 0xE2, 0x26, 0x00, 0x00, 0x00, 0x00, }
-	},
-	{ .pixclk = 74176000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD8, 0x10, 0x9C, 0xEF, 0x5B,
-		0x6D, 0x10, 0x01, 0x51, 0xEF, 0xF3, 0x54, 0xB9,
-		0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x22, 0x40, 0xA5, 0x26, 0x01, 0x00, 0x00, 0x00, }
-	},
-	{ .pixclk = 74250000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD8, 0x10, 0x9C, 0xF8, 0x40,
-		0x6A, 0x10, 0x01, 0x51, 0xFF, 0xF1, 0x54, 0xBA,
-		0x84, 0x00, 0x10, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x22, 0x40, 0xA4, 0x26, 0x01, 0x00, 0x00, 0x00, }
-	},
-	{ .pixclk = 148352000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD8, 0x10, 0x9C, 0xEF, 0x5B,
-		0x6D, 0x18, 0x00, 0x51, 0xEF, 0xF3, 0x54, 0xB9,
-		0x84, 0x00, 0x30, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x11, 0x40, 0xA5, 0x26, 0x02, 0x00, 0x00, 0x00, }
-	},
-	{ .pixclk = 148500000, .data = (u8 [32]) {
-		0x01, 0x05, 0x00, 0xD8, 0x10, 0x9C, 0xF8, 0x40,
-		0x6A, 0x18, 0x00, 0x51, 0xFF, 0xF1, 0x54, 0xBA,
-		0x84, 0x00, 0x10, 0x38, 0x00, 0x08, 0x10, 0xE0,
-		0x11, 0x40, 0xA4, 0x26, 0x02, 0x00, 0x00, 0x00, }
-	},
-	{ /* end marker */ }
-};
-
-static const struct hdmiphy_conf hdmiphy_conf_exynos4212[] = {
-	{ .pixclk = 27000000, .data = (u8 [32]) {
-		0x01, 0x11, 0x2D, 0x75, 0x00, 0x01, 0x00, 0x08,
-		0x82, 0x00, 0x0E, 0xD9, 0x45, 0xA0, 0x34, 0xC0,
-		0x0B, 0x80, 0x12, 0x87, 0x08, 0x24, 0x24, 0x71,
-		0x54, 0xE3, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ .pixclk = 27027000, .data = (u8 [32]) {
-		0x01, 0x91, 0x2D, 0x72, 0x00, 0x64, 0x12, 0x08,
-		0x43, 0x20, 0x0E, 0xD9, 0x45, 0xA0, 0x34, 0xC0,
-		0x0B, 0x80, 0x12, 0x87, 0x08, 0x24, 0x24, 0x71,
-		0x54, 0xE2, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ .pixclk = 74176000, .data = (u8 [32]) {
-		0x01, 0x91, 0x3E, 0x35, 0x00, 0x5B, 0xDE, 0x08,
-		0x82, 0x20, 0x73, 0xD9, 0x45, 0xA0, 0x34, 0xC0,
-		0x0B, 0x80, 0x12, 0x87, 0x08, 0x24, 0x24, 0x52,
-		0x54, 0xA5, 0x24, 0x01, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ .pixclk = 74250000, .data = (u8 [32]) {
-		0x01, 0x91, 0x3E, 0x35, 0x00, 0x40, 0xF0, 0x08,
-		0x82, 0x20, 0x73, 0xD9, 0x45, 0xA0, 0x34, 0xC0,
-		0x0B, 0x80, 0x12, 0x87, 0x08, 0x24, 0x24, 0x52,
-		0x54, 0xA4, 0x24, 0x01, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ .pixclk = 148500000, .data = (u8 [32]) {
-		0x01, 0x91, 0x3E, 0x15, 0x00, 0x40, 0xF0, 0x08,
-		0x82, 0x20, 0x73, 0xD9, 0x45, 0xA0, 0x34, 0xC0,
-		0x0B, 0x80, 0x12, 0x87, 0x08, 0x24, 0x24, 0xA4,
-		0x54, 0x4A, 0x25, 0x03, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ /* end marker */ }
-};
-
-static const struct hdmiphy_conf hdmiphy_conf_exynos4412[] = {
-	{ .pixclk = 27000000, .data = (u8 [32]) {
-		0x01, 0x11, 0x2D, 0x75, 0x40, 0x01, 0x00, 0x08,
-		0x82, 0x00, 0x0E, 0xD9, 0x45, 0xA0, 0xAC, 0x80,
-		0x08, 0x80, 0x11, 0x84, 0x02, 0x22, 0x44, 0x86,
-		0x54, 0xE4, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ .pixclk = 27027000, .data = (u8 [32]) {
-		0x01, 0x91, 0x2D, 0x72, 0x40, 0x64, 0x12, 0x08,
-		0x43, 0x20, 0x0E, 0xD9, 0x45, 0xA0, 0xAC, 0x80,
-		0x08, 0x80, 0x11, 0x84, 0x02, 0x22, 0x44, 0x86,
-		0x54, 0xE3, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ .pixclk = 74176000, .data = (u8 [32]) {
-		0x01, 0x91, 0x1F, 0x10, 0x40, 0x5B, 0xEF, 0x08,
-		0x81, 0x20, 0xB9, 0xD8, 0x45, 0xA0, 0xAC, 0x80,
-		0x08, 0x80, 0x11, 0x84, 0x02, 0x22, 0x44, 0x86,
-		0x54, 0xA6, 0x24, 0x01, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ .pixclk = 74250000, .data = (u8 [32]) {
-		0x01, 0x91, 0x1F, 0x10, 0x40, 0x40, 0xF8, 0x08,
-		0x81, 0x20, 0xBA, 0xD8, 0x45, 0xA0, 0xAC, 0x80,
-		0x08, 0x80, 0x11, 0x84, 0x02, 0x22, 0x44, 0x86,
-		0x54, 0xA5, 0x24, 0x01, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ .pixclk = 148500000, .data = (u8 [32]) {
-		0x01, 0x91, 0x1F, 0x00, 0x40, 0x40, 0xF8, 0x08,
-		0x81, 0x20, 0xBA, 0xD8, 0x45, 0xA0, 0xAC, 0x80,
-		0x08, 0x80, 0x11, 0x84, 0x02, 0x22, 0x44, 0x86,
-		0x54, 0x4B, 0x25, 0x03, 0x00, 0x00, 0x01, 0x00, }
-	},
-	{ /* end marker */ }
-};
-
-static inline struct hdmiphy_ctx *sd_to_ctx(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct hdmiphy_ctx, sd);
-}
-
-static const u8 *hdmiphy_find_conf(unsigned long pixclk,
-		const struct hdmiphy_conf *conf)
-{
-	for (; conf->pixclk; ++conf)
-		if (conf->pixclk == pixclk)
-			return conf->data;
-	return NULL;
-}
-
-static int hdmiphy_s_power(struct v4l2_subdev *sd, int on)
-{
-	/* to be implemented */
-	return 0;
-}
-
-static int hdmiphy_s_dv_timings(struct v4l2_subdev *sd,
-	struct v4l2_dv_timings *timings)
-{
-	const u8 *data;
-	u8 buffer[32];
-	int ret;
-	struct hdmiphy_ctx *ctx = sd_to_ctx(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct device *dev = &client->dev;
-	unsigned long pixclk = timings->bt.pixelclock;
-
-	dev_info(dev, "s_dv_timings\n");
-	if ((timings->bt.flags & V4L2_DV_FL_REDUCED_FPS) && pixclk == 74250000)
-		pixclk = 74176000;
-	data = hdmiphy_find_conf(pixclk, ctx->conf_tab);
-	if (!data) {
-		dev_err(dev, "format not supported\n");
-		return -EINVAL;
-	}
-
-	/* storing configuration to the device */
-	memcpy(buffer, data, 32);
-	ret = i2c_master_send(client, buffer, 32);
-	if (ret != 32) {
-		dev_err(dev, "failed to configure HDMIPHY via I2C\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-static int hdmiphy_dv_timings_cap(struct v4l2_subdev *sd,
-	struct v4l2_dv_timings_cap *cap)
-{
-	if (cap->pad != 0)
-		return -EINVAL;
-
-	cap->type = V4L2_DV_BT_656_1120;
-	/* The phy only determines the pixelclock, leave the other values
-	 * at 0 to signify that we have no information for them. */
-	cap->bt.min_pixelclock = 27000000;
-	cap->bt.max_pixelclock = 148500000;
-	return 0;
-}
-
-static int hdmiphy_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct device *dev = &client->dev;
-	u8 buffer[2];
-	int ret;
-
-	dev_info(dev, "s_stream(%d)\n", enable);
-	/* going to/from configuration from/to operation mode */
-	buffer[0] = 0x1f;
-	buffer[1] = enable ? 0x80 : 0x00;
-
-	ret = i2c_master_send(client, buffer, 2);
-	if (ret != 2) {
-		dev_err(dev, "stream (%d) failed\n", enable);
-		return -EIO;
-	}
-	return 0;
-}
-
-static const struct v4l2_subdev_core_ops hdmiphy_core_ops = {
-	.s_power =  hdmiphy_s_power,
-};
-
-static const struct v4l2_subdev_video_ops hdmiphy_video_ops = {
-	.s_dv_timings = hdmiphy_s_dv_timings,
-	.s_stream =  hdmiphy_s_stream,
-};
-
-static const struct v4l2_subdev_pad_ops hdmiphy_pad_ops = {
-	.dv_timings_cap = hdmiphy_dv_timings_cap,
-};
-
-static const struct v4l2_subdev_ops hdmiphy_ops = {
-	.core = &hdmiphy_core_ops,
-	.video = &hdmiphy_video_ops,
-	.pad = &hdmiphy_pad_ops,
-};
-
-static int hdmiphy_probe(struct i2c_client *client,
-			 const struct i2c_device_id *id)
-{
-	struct hdmiphy_ctx *ctx;
-
-	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
-
-	ctx->conf_tab = (struct hdmiphy_conf *)id->driver_data;
-	v4l2_i2c_subdev_init(&ctx->sd, client, &hdmiphy_ops);
-
-	dev_info(&client->dev, "probe successful\n");
-	return 0;
-}
-
-static int hdmiphy_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct hdmiphy_ctx *ctx = sd_to_ctx(sd);
-
-	kfree(ctx);
-	dev_info(&client->dev, "remove successful\n");
-
-	return 0;
-}
-
-static const struct i2c_device_id hdmiphy_id[] = {
-	{ "hdmiphy", (unsigned long)hdmiphy_conf_exynos4210 },
-	{ "hdmiphy-s5pv210", (unsigned long)hdmiphy_conf_s5pv210 },
-	{ "hdmiphy-exynos4210", (unsigned long)hdmiphy_conf_exynos4210 },
-	{ "hdmiphy-exynos4212", (unsigned long)hdmiphy_conf_exynos4212 },
-	{ "hdmiphy-exynos4412", (unsigned long)hdmiphy_conf_exynos4412 },
-	{ },
-};
-MODULE_DEVICE_TABLE(i2c, hdmiphy_id);
-
-static struct i2c_driver hdmiphy_driver = {
-	.driver = {
-		.name	= "s5p-hdmiphy",
-	},
-	.probe		= hdmiphy_probe,
-	.remove		= hdmiphy_remove,
-	.id_table = hdmiphy_id,
-};
-
-module_i2c_driver(hdmiphy_driver);
diff --git a/drivers/media/platform/s5p-tv/mixer.h b/drivers/media/platform/s5p-tv/mixer.h
deleted file mode 100644
index 869f0ce86f6e..000000000000
--- a/drivers/media/platform/s5p-tv/mixer.h
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Samsung TV Mixer driver
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *
- * Tomasz Stanislawski, <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published
- * by the Free Software Foundiation. either version 2 of the License,
- * or (at your option) any later version
- */
-
-#ifndef SAMSUNG_MIXER_H
-#define SAMSUNG_MIXER_H
-
-#ifdef CONFIG_VIDEO_SAMSUNG_S5P_MIXER_DEBUG
-	#define DEBUG
-#endif
-
-#include <linux/fb.h>
-#include <linux/irqreturn.h>
-#include <linux/kernel.h>
-#include <linux/spinlock.h>
-#include <linux/wait.h>
-#include <media/v4l2-device.h>
-#include <media/videobuf2-v4l2.h>
-
-#include "regs-mixer.h"
-
-/** maximum number of output interfaces */
-#define MXR_MAX_OUTPUTS 2
-/** maximum number of input interfaces (layers) */
-#define MXR_MAX_LAYERS 3
-#define MXR_DRIVER_NAME "s5p-mixer"
-/** maximal number of planes for every layer */
-#define MXR_MAX_PLANES	2
-
-#define MXR_ENABLE 1
-#define MXR_DISABLE 0
-
-/** description of a macroblock for packed formats */
-struct mxr_block {
-	/** vertical number of pixels in macroblock */
-	unsigned int width;
-	/** horizontal number of pixels in macroblock */
-	unsigned int height;
-	/** size of block in bytes */
-	unsigned int size;
-};
-
-/** description of supported format */
-struct mxr_format {
-	/** format name/mnemonic */
-	const char *name;
-	/** fourcc identifier */
-	u32 fourcc;
-	/** colorspace identifier */
-	enum v4l2_colorspace colorspace;
-	/** number of planes in image data */
-	int num_planes;
-	/** description of block for each plane */
-	struct mxr_block plane[MXR_MAX_PLANES];
-	/** number of subframes in image data */
-	int num_subframes;
-	/** specifies to which subframe belong given plane */
-	int plane2subframe[MXR_MAX_PLANES];
-	/** internal code, driver dependent */
-	unsigned long cookie;
-};
-
-/** description of crop configuration for image */
-struct mxr_crop {
-	/** width of layer in pixels */
-	unsigned int full_width;
-	/** height of layer in pixels */
-	unsigned int full_height;
-	/** horizontal offset of first pixel to be displayed */
-	unsigned int x_offset;
-	/** vertical offset of first pixel to be displayed */
-	unsigned int y_offset;
-	/** width of displayed data in pixels */
-	unsigned int width;
-	/** height of displayed data in pixels */
-	unsigned int height;
-	/** indicate which fields are present in buffer */
-	unsigned int field;
-};
-
-/** stages of geometry operations */
-enum mxr_geometry_stage {
-	MXR_GEOMETRY_SINK,
-	MXR_GEOMETRY_COMPOSE,
-	MXR_GEOMETRY_CROP,
-	MXR_GEOMETRY_SOURCE,
-};
-
-/* flag indicating that offset should be 0 */
-#define MXR_NO_OFFSET	0x80000000
-
-/** description of transformation from source to destination image */
-struct mxr_geometry {
-	/** cropping for source image */
-	struct mxr_crop src;
-	/** cropping for destination image */
-	struct mxr_crop dst;
-	/** layer-dependant description of horizontal scaling */
-	unsigned int x_ratio;
-	/** layer-dependant description of vertical scaling */
-	unsigned int y_ratio;
-};
-
-/** instance of a buffer */
-struct mxr_buffer {
-	/** common v4l buffer stuff -- must be first */
-	struct vb2_v4l2_buffer vb;
-	/** node for layer's lists */
-	struct list_head	list;
-};
-
-
-/** internal states of layer */
-enum mxr_layer_state {
-	/** layers is not shown */
-	MXR_LAYER_IDLE = 0,
-	/** layer is shown */
-	MXR_LAYER_STREAMING,
-	/** state before STREAMOFF is finished */
-	MXR_LAYER_STREAMING_FINISH,
-};
-
-/** forward declarations */
-struct mxr_device;
-struct mxr_layer;
-
-/** callback for layers operation */
-struct mxr_layer_ops {
-	/* TODO: try to port it to subdev API */
-	/** handler for resource release function */
-	void (*release)(struct mxr_layer *);
-	/** setting buffer to HW */
-	void (*buffer_set)(struct mxr_layer *, struct mxr_buffer *);
-	/** setting format and geometry in HW */
-	void (*format_set)(struct mxr_layer *);
-	/** streaming stop/start */
-	void (*stream_set)(struct mxr_layer *, int);
-	/** adjusting geometry */
-	void (*fix_geometry)(struct mxr_layer *,
-		enum mxr_geometry_stage, unsigned long);
-};
-
-/** layer instance, a single window and content displayed on output */
-struct mxr_layer {
-	/** parent mixer device */
-	struct mxr_device *mdev;
-	/** layer index (unique identifier) */
-	int idx;
-	/** callbacks for layer methods */
-	struct mxr_layer_ops ops;
-	/** format array */
-	const struct mxr_format **fmt_array;
-	/** size of format array */
-	unsigned long fmt_array_size;
-
-	/** lock for protection of list and state fields */
-	spinlock_t enq_slock;
-	/** list for enqueued buffers */
-	struct list_head enq_list;
-	/** buffer currently owned by hardware in temporary registers */
-	struct mxr_buffer *update_buf;
-	/** buffer currently owned by hardware in shadow registers */
-	struct mxr_buffer *shadow_buf;
-	/** state of layer IDLE/STREAMING */
-	enum mxr_layer_state state;
-
-	/** mutex for protection of fields below */
-	struct mutex mutex;
-	/** handler for video node */
-	struct video_device vfd;
-	/** queue for output buffers */
-	struct vb2_queue vb_queue;
-	/** current image format */
-	const struct mxr_format *fmt;
-	/** current geometry of image */
-	struct mxr_geometry geo;
-};
-
-/** description of mixers output interface */
-struct mxr_output {
-	/** name of output */
-	char name[32];
-	/** output subdev */
-	struct v4l2_subdev *sd;
-	/** cookie used for configuration of registers */
-	int cookie;
-};
-
-/** specify source of output subdevs */
-struct mxr_output_conf {
-	/** name of output (connector) */
-	char *output_name;
-	/** name of module that generates output subdev */
-	char *module_name;
-	/** cookie need for mixer HW */
-	int cookie;
-};
-
-struct clk;
-struct regulator;
-
-/** auxiliary resources used my mixer */
-struct mxr_resources {
-	/** interrupt index */
-	int irq;
-	/** pointer to Mixer registers */
-	void __iomem *mxr_regs;
-	/** pointer to Video Processor registers */
-	void __iomem *vp_regs;
-	/** other resources, should used under mxr_device.mutex */
-	struct clk *mixer;
-	struct clk *vp;
-	struct clk *sclk_mixer;
-	struct clk *sclk_hdmi;
-	struct clk *sclk_dac;
-};
-
-/* event flags used  */
-enum mxr_devide_flags {
-	MXR_EVENT_VSYNC = 0,
-	MXR_EVENT_TOP = 1,
-};
-
-/** drivers instance */
-struct mxr_device {
-	/** master device */
-	struct device *dev;
-	/** state of each layer */
-	struct mxr_layer *layer[MXR_MAX_LAYERS];
-	/** state of each output */
-	struct mxr_output *output[MXR_MAX_OUTPUTS];
-	/** number of registered outputs */
-	int output_cnt;
-
-	/* video resources */
-
-	/** V4L2 device */
-	struct v4l2_device v4l2_dev;
-	/** event wait queue */
-	wait_queue_head_t event_queue;
-	/** state flags */
-	unsigned long event_flags;
-
-	/** spinlock for protection of registers */
-	spinlock_t reg_slock;
-
-	/** mutex for protection of fields below */
-	struct mutex mutex;
-	/** number of entities depndant on output configuration */
-	int n_output;
-	/** number of users that do streaming */
-	int n_streamer;
-	/** index of current output */
-	int current_output;
-	/** auxiliary resources used my mixer */
-	struct mxr_resources res;
-};
-
-/** transform device structure into mixer device */
-static inline struct mxr_device *to_mdev(struct device *dev)
-{
-	struct v4l2_device *vdev = dev_get_drvdata(dev);
-	return container_of(vdev, struct mxr_device, v4l2_dev);
-}
-
-/** get current output data, should be called under mdev's mutex */
-static inline struct mxr_output *to_output(struct mxr_device *mdev)
-{
-	return mdev->output[mdev->current_output];
-}
-
-/** get current output subdev, should be called under mdev's mutex */
-static inline struct v4l2_subdev *to_outsd(struct mxr_device *mdev)
-{
-	struct mxr_output *out = to_output(mdev);
-	return out ? out->sd : NULL;
-}
-
-/** forward declaration for mixer platform data */
-struct mxr_platform_data;
-
-/** acquiring common video resources */
-int mxr_acquire_video(struct mxr_device *mdev,
-	struct mxr_output_conf *output_cont, int output_count);
-
-/** releasing common video resources */
-void mxr_release_video(struct mxr_device *mdev);
-
-struct mxr_layer *mxr_graph_layer_create(struct mxr_device *mdev, int idx);
-struct mxr_layer *mxr_vp_layer_create(struct mxr_device *mdev, int idx);
-struct mxr_layer *mxr_base_layer_create(struct mxr_device *mdev,
-	int idx, char *name, const struct mxr_layer_ops *ops);
-
-void mxr_base_layer_release(struct mxr_layer *layer);
-void mxr_layer_release(struct mxr_layer *layer);
-
-int mxr_base_layer_register(struct mxr_layer *layer);
-void mxr_base_layer_unregister(struct mxr_layer *layer);
-
-unsigned long mxr_get_plane_size(const struct mxr_block *blk,
-	unsigned int width, unsigned int height);
-
-/** adds new consumer for mixer's power */
-int __must_check mxr_power_get(struct mxr_device *mdev);
-/** removes consumer for mixer's power */
-void mxr_power_put(struct mxr_device *mdev);
-/** add new client for output configuration */
-void mxr_output_get(struct mxr_device *mdev);
-/** removes new client for output configuration */
-void mxr_output_put(struct mxr_device *mdev);
-/** add new client for streaming */
-void mxr_streamer_get(struct mxr_device *mdev);
-/** removes new client for streaming */
-void mxr_streamer_put(struct mxr_device *mdev);
-/** returns format of data delivared to current output */
-void mxr_get_mbus_fmt(struct mxr_device *mdev,
-	struct v4l2_mbus_framefmt *mbus_fmt);
-
-/* Debug */
-
-#define mxr_err(mdev, fmt, ...)  dev_err(mdev->dev, fmt, ##__VA_ARGS__)
-#define mxr_warn(mdev, fmt, ...) dev_warn(mdev->dev, fmt, ##__VA_ARGS__)
-#define mxr_info(mdev, fmt, ...) dev_info(mdev->dev, fmt, ##__VA_ARGS__)
-
-#ifdef CONFIG_VIDEO_SAMSUNG_S5P_MIXER_DEBUG
-	#define mxr_dbg(mdev, fmt, ...)  dev_dbg(mdev->dev, fmt, ##__VA_ARGS__)
-#else
-	#define mxr_dbg(mdev, fmt, ...)  do { (void) mdev; } while (0)
-#endif
-
-/* accessing Mixer's and Video Processor's registers */
-
-void mxr_vsync_set_update(struct mxr_device *mdev, int en);
-void mxr_reg_reset(struct mxr_device *mdev);
-irqreturn_t mxr_irq_handler(int irq, void *dev_data);
-void mxr_reg_s_output(struct mxr_device *mdev, int cookie);
-void mxr_reg_streamon(struct mxr_device *mdev);
-void mxr_reg_streamoff(struct mxr_device *mdev);
-int mxr_reg_wait4vsync(struct mxr_device *mdev);
-void mxr_reg_set_mbus_fmt(struct mxr_device *mdev,
-	struct v4l2_mbus_framefmt *fmt);
-void mxr_reg_graph_layer_stream(struct mxr_device *mdev, int idx, int en);
-void mxr_reg_graph_buffer(struct mxr_device *mdev, int idx, dma_addr_t addr);
-void mxr_reg_graph_format(struct mxr_device *mdev, int idx,
-	const struct mxr_format *fmt, const struct mxr_geometry *geo);
-
-void mxr_reg_vp_layer_stream(struct mxr_device *mdev, int en);
-void mxr_reg_vp_buffer(struct mxr_device *mdev,
-	dma_addr_t luma_addr[2], dma_addr_t chroma_addr[2]);
-void mxr_reg_vp_format(struct mxr_device *mdev,
-	const struct mxr_format *fmt, const struct mxr_geometry *geo);
-void mxr_reg_dump(struct mxr_device *mdev);
-
-#endif /* SAMSUNG_MIXER_H */
-
diff --git a/drivers/media/platform/s5p-tv/mixer_drv.c b/drivers/media/platform/s5p-tv/mixer_drv.c
deleted file mode 100644
index 8a5d19469ddc..000000000000
--- a/drivers/media/platform/s5p-tv/mixer_drv.c
+++ /dev/null
@@ -1,527 +0,0 @@
-/*
- * Samsung TV Mixer driver
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *
- * Tomasz Stanislawski, <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published
- * by the Free Software Foundiation. either version 2 of the License,
- * or (at your option) any later version
- */
-
-#include "mixer.h"
-
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/io.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/fb.h>
-#include <linux/delay.h>
-#include <linux/pm_runtime.h>
-#include <linux/clk.h>
-
-MODULE_AUTHOR("Tomasz Stanislawski, <t.stanislaws@samsung.com>");
-MODULE_DESCRIPTION("Samsung MIXER");
-MODULE_LICENSE("GPL");
-
-/* --------- DRIVER PARAMETERS ---------- */
-
-static struct mxr_output_conf mxr_output_conf[] = {
-	{
-		.output_name = "S5P HDMI connector",
-		.module_name = "s5p-hdmi",
-		.cookie = 1,
-	},
-	{
-		.output_name = "S5P SDO connector",
-		.module_name = "s5p-sdo",
-		.cookie = 0,
-	},
-};
-
-void mxr_get_mbus_fmt(struct mxr_device *mdev,
-	struct v4l2_mbus_framefmt *mbus_fmt)
-{
-	struct v4l2_subdev *sd;
-	struct v4l2_subdev_format fmt = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-	int ret;
-
-	mutex_lock(&mdev->mutex);
-	sd = to_outsd(mdev);
-	ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt);
-	*mbus_fmt = fmt.format;
-	WARN(ret, "failed to get mbus_fmt for output %s\n", sd->name);
-	mutex_unlock(&mdev->mutex);
-}
-
-void mxr_streamer_get(struct mxr_device *mdev)
-{
-	mutex_lock(&mdev->mutex);
-	++mdev->n_streamer;
-	mxr_dbg(mdev, "%s(%d)\n", __func__, mdev->n_streamer);
-	if (mdev->n_streamer == 1) {
-		struct v4l2_subdev *sd = to_outsd(mdev);
-		struct v4l2_subdev_format fmt = {
-			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-		};
-		struct v4l2_mbus_framefmt *mbus_fmt = &fmt.format;
-		struct mxr_resources *res = &mdev->res;
-		int ret;
-
-		if (to_output(mdev)->cookie == 0)
-			clk_set_parent(res->sclk_mixer, res->sclk_dac);
-		else
-			clk_set_parent(res->sclk_mixer, res->sclk_hdmi);
-		mxr_reg_s_output(mdev, to_output(mdev)->cookie);
-
-		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt);
-		WARN(ret, "failed to get mbus_fmt for output %s\n", sd->name);
-		ret = v4l2_subdev_call(sd, video, s_stream, 1);
-		WARN(ret, "starting stream failed for output %s\n", sd->name);
-
-		mxr_reg_set_mbus_fmt(mdev, mbus_fmt);
-		mxr_reg_streamon(mdev);
-		ret = mxr_reg_wait4vsync(mdev);
-		WARN(ret, "failed to get vsync (%d) from output\n", ret);
-	}
-	mutex_unlock(&mdev->mutex);
-	mxr_reg_dump(mdev);
-	/* FIXME: what to do when streaming fails? */
-}
-
-void mxr_streamer_put(struct mxr_device *mdev)
-{
-	mutex_lock(&mdev->mutex);
-	--mdev->n_streamer;
-	mxr_dbg(mdev, "%s(%d)\n", __func__, mdev->n_streamer);
-	if (mdev->n_streamer == 0) {
-		int ret;
-		struct v4l2_subdev *sd = to_outsd(mdev);
-
-		mxr_reg_streamoff(mdev);
-		/* vsync applies Mixer setup */
-		ret = mxr_reg_wait4vsync(mdev);
-		WARN(ret, "failed to get vsync (%d) from output\n", ret);
-		ret = v4l2_subdev_call(sd, video, s_stream, 0);
-		WARN(ret, "stopping stream failed for output %s\n", sd->name);
-	}
-	WARN(mdev->n_streamer < 0, "negative number of streamers (%d)\n",
-		mdev->n_streamer);
-	mutex_unlock(&mdev->mutex);
-	mxr_reg_dump(mdev);
-}
-
-void mxr_output_get(struct mxr_device *mdev)
-{
-	mutex_lock(&mdev->mutex);
-	++mdev->n_output;
-	mxr_dbg(mdev, "%s(%d)\n", __func__, mdev->n_output);
-	/* turn on auxiliary driver */
-	if (mdev->n_output == 1)
-		v4l2_subdev_call(to_outsd(mdev), core, s_power, 1);
-	mutex_unlock(&mdev->mutex);
-}
-
-void mxr_output_put(struct mxr_device *mdev)
-{
-	mutex_lock(&mdev->mutex);
-	--mdev->n_output;
-	mxr_dbg(mdev, "%s(%d)\n", __func__, mdev->n_output);
-	/* turn on auxiliary driver */
-	if (mdev->n_output == 0)
-		v4l2_subdev_call(to_outsd(mdev), core, s_power, 0);
-	WARN(mdev->n_output < 0, "negative number of output users (%d)\n",
-		mdev->n_output);
-	mutex_unlock(&mdev->mutex);
-}
-
-int mxr_power_get(struct mxr_device *mdev)
-{
-	int ret = pm_runtime_get_sync(mdev->dev);
-
-	/* returning 1 means that power is already enabled,
-	 * so zero success be returned */
-	if (ret < 0)
-		return ret;
-	return 0;
-}
-
-void mxr_power_put(struct mxr_device *mdev)
-{
-	pm_runtime_put_sync(mdev->dev);
-}
-
-/* --------- RESOURCE MANAGEMENT -------------*/
-
-static int mxr_acquire_plat_resources(struct mxr_device *mdev,
-				      struct platform_device *pdev)
-{
-	struct resource *res;
-	int ret;
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mxr");
-	if (res == NULL) {
-		mxr_err(mdev, "get memory resource failed.\n");
-		ret = -ENXIO;
-		goto fail;
-	}
-
-	mdev->res.mxr_regs = ioremap(res->start, resource_size(res));
-	if (mdev->res.mxr_regs == NULL) {
-		mxr_err(mdev, "register mapping failed.\n");
-		ret = -ENXIO;
-		goto fail;
-	}
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vp");
-	if (res == NULL) {
-		mxr_err(mdev, "get memory resource failed.\n");
-		ret = -ENXIO;
-		goto fail_mxr_regs;
-	}
-
-	mdev->res.vp_regs = ioremap(res->start, resource_size(res));
-	if (mdev->res.vp_regs == NULL) {
-		mxr_err(mdev, "register mapping failed.\n");
-		ret = -ENXIO;
-		goto fail_mxr_regs;
-	}
-
-	res = platform_get_resource_byname(pdev, IORESOURCE_IRQ, "irq");
-	if (res == NULL) {
-		mxr_err(mdev, "get interrupt resource failed.\n");
-		ret = -ENXIO;
-		goto fail_vp_regs;
-	}
-
-	ret = request_irq(res->start, mxr_irq_handler, 0, "s5p-mixer", mdev);
-	if (ret) {
-		mxr_err(mdev, "request interrupt failed.\n");
-		goto fail_vp_regs;
-	}
-	mdev->res.irq = res->start;
-
-	return 0;
-
-fail_vp_regs:
-	iounmap(mdev->res.vp_regs);
-
-fail_mxr_regs:
-	iounmap(mdev->res.mxr_regs);
-
-fail:
-	return ret;
-}
-
-static void mxr_resource_clear_clocks(struct mxr_resources *res)
-{
-	res->mixer	= ERR_PTR(-EINVAL);
-	res->vp		= ERR_PTR(-EINVAL);
-	res->sclk_mixer	= ERR_PTR(-EINVAL);
-	res->sclk_hdmi	= ERR_PTR(-EINVAL);
-	res->sclk_dac	= ERR_PTR(-EINVAL);
-}
-
-static void mxr_release_plat_resources(struct mxr_device *mdev)
-{
-	free_irq(mdev->res.irq, mdev);
-	iounmap(mdev->res.vp_regs);
-	iounmap(mdev->res.mxr_regs);
-}
-
-static void mxr_release_clocks(struct mxr_device *mdev)
-{
-	struct mxr_resources *res = &mdev->res;
-
-	if (!IS_ERR(res->sclk_dac))
-		clk_put(res->sclk_dac);
-	if (!IS_ERR(res->sclk_hdmi))
-		clk_put(res->sclk_hdmi);
-	if (!IS_ERR(res->sclk_mixer))
-		clk_put(res->sclk_mixer);
-	if (!IS_ERR(res->vp))
-		clk_put(res->vp);
-	if (!IS_ERR(res->mixer))
-		clk_put(res->mixer);
-}
-
-static int mxr_acquire_clocks(struct mxr_device *mdev)
-{
-	struct mxr_resources *res = &mdev->res;
-	struct device *dev = mdev->dev;
-
-	mxr_resource_clear_clocks(res);
-
-	res->mixer = clk_get(dev, "mixer");
-	if (IS_ERR(res->mixer)) {
-		mxr_err(mdev, "failed to get clock 'mixer'\n");
-		goto fail;
-	}
-	res->vp = clk_get(dev, "vp");
-	if (IS_ERR(res->vp)) {
-		mxr_err(mdev, "failed to get clock 'vp'\n");
-		goto fail;
-	}
-	res->sclk_mixer = clk_get(dev, "sclk_mixer");
-	if (IS_ERR(res->sclk_mixer)) {
-		mxr_err(mdev, "failed to get clock 'sclk_mixer'\n");
-		goto fail;
-	}
-	res->sclk_hdmi = clk_get(dev, "sclk_hdmi");
-	if (IS_ERR(res->sclk_hdmi)) {
-		mxr_err(mdev, "failed to get clock 'sclk_hdmi'\n");
-		goto fail;
-	}
-	res->sclk_dac = clk_get(dev, "sclk_dac");
-	if (IS_ERR(res->sclk_dac)) {
-		mxr_err(mdev, "failed to get clock 'sclk_dac'\n");
-		goto fail;
-	}
-
-	return 0;
-fail:
-	mxr_release_clocks(mdev);
-	return -ENODEV;
-}
-
-static int mxr_acquire_resources(struct mxr_device *mdev,
-				 struct platform_device *pdev)
-{
-	int ret;
-	ret = mxr_acquire_plat_resources(mdev, pdev);
-
-	if (ret)
-		goto fail;
-
-	ret = mxr_acquire_clocks(mdev);
-	if (ret)
-		goto fail_plat;
-
-	mxr_info(mdev, "resources acquired\n");
-	return 0;
-
-fail_plat:
-	mxr_release_plat_resources(mdev);
-fail:
-	mxr_err(mdev, "resources acquire failed\n");
-	return ret;
-}
-
-static void mxr_release_resources(struct mxr_device *mdev)
-{
-	mxr_release_clocks(mdev);
-	mxr_release_plat_resources(mdev);
-	memset(&mdev->res, 0, sizeof(mdev->res));
-	mxr_resource_clear_clocks(&mdev->res);
-}
-
-static void mxr_release_layers(struct mxr_device *mdev)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(mdev->layer); ++i)
-		if (mdev->layer[i])
-			mxr_layer_release(mdev->layer[i]);
-}
-
-static int mxr_acquire_layers(struct mxr_device *mdev,
-			      struct mxr_platform_data *pdata)
-{
-	mdev->layer[0] = mxr_graph_layer_create(mdev, 0);
-	mdev->layer[1] = mxr_graph_layer_create(mdev, 1);
-	mdev->layer[2] = mxr_vp_layer_create(mdev, 0);
-
-	if (!mdev->layer[0] || !mdev->layer[1] || !mdev->layer[2]) {
-		mxr_err(mdev, "failed to acquire layers\n");
-		goto fail;
-	}
-
-	return 0;
-
-fail:
-	mxr_release_layers(mdev);
-	return -ENODEV;
-}
-
-/* ---------- POWER MANAGEMENT ----------- */
-
-static int mxr_runtime_resume(struct device *dev)
-{
-	struct mxr_device *mdev = to_mdev(dev);
-	struct mxr_resources *res = &mdev->res;
-	int ret;
-
-	mxr_dbg(mdev, "resume - start\n");
-	mutex_lock(&mdev->mutex);
-	/* turn clocks on */
-	ret = clk_prepare_enable(res->mixer);
-	if (ret < 0) {
-		dev_err(mdev->dev, "clk_prepare_enable(mixer) failed\n");
-		goto fail;
-	}
-	ret = clk_prepare_enable(res->vp);
-	if (ret < 0) {
-		dev_err(mdev->dev, "clk_prepare_enable(vp) failed\n");
-		goto fail_mixer;
-	}
-	ret = clk_prepare_enable(res->sclk_mixer);
-	if (ret < 0) {
-		dev_err(mdev->dev, "clk_prepare_enable(sclk_mixer) failed\n");
-		goto fail_vp;
-	}
-	/* apply default configuration */
-	mxr_reg_reset(mdev);
-	mxr_dbg(mdev, "resume - finished\n");
-
-	mutex_unlock(&mdev->mutex);
-	return 0;
-
-fail_vp:
-	clk_disable_unprepare(res->vp);
-fail_mixer:
-	clk_disable_unprepare(res->mixer);
-fail:
-	mutex_unlock(&mdev->mutex);
-	dev_err(mdev->dev, "resume failed\n");
-	return ret;
-}
-
-static int mxr_runtime_suspend(struct device *dev)
-{
-	struct mxr_device *mdev = to_mdev(dev);
-	struct mxr_resources *res = &mdev->res;
-	mxr_dbg(mdev, "suspend - start\n");
-	mutex_lock(&mdev->mutex);
-	/* turn clocks off */
-	clk_disable_unprepare(res->sclk_mixer);
-	clk_disable_unprepare(res->vp);
-	clk_disable_unprepare(res->mixer);
-	mutex_unlock(&mdev->mutex);
-	mxr_dbg(mdev, "suspend - finished\n");
-	return 0;
-}
-
-static const struct dev_pm_ops mxr_pm_ops = {
-	.runtime_suspend = mxr_runtime_suspend,
-	.runtime_resume	 = mxr_runtime_resume,
-};
-
-/* --------- DRIVER INITIALIZATION ---------- */
-
-static int mxr_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct mxr_platform_data *pdata = dev->platform_data;
-	struct mxr_device *mdev;
-	int ret;
-
-	/* mdev does not exist yet so no mxr_dbg is used */
-	dev_info(dev, "probe start\n");
-
-	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
-	if (!mdev) {
-		dev_err(dev, "not enough memory.\n");
-		ret = -ENOMEM;
-		goto fail;
-	}
-
-	/* setup pointer to master device */
-	mdev->dev = dev;
-
-	mutex_init(&mdev->mutex);
-	spin_lock_init(&mdev->reg_slock);
-	init_waitqueue_head(&mdev->event_queue);
-
-	/* acquire resources: regs, irqs, clocks, regulators */
-	ret = mxr_acquire_resources(mdev, pdev);
-	if (ret)
-		goto fail_mem;
-
-	/* configure resources for video output */
-	ret = mxr_acquire_video(mdev, mxr_output_conf,
-		ARRAY_SIZE(mxr_output_conf));
-	if (ret)
-		goto fail_resources;
-
-	/* configure layers */
-	ret = mxr_acquire_layers(mdev, pdata);
-	if (ret)
-		goto fail_video;
-
-	pm_runtime_enable(dev);
-
-	mxr_info(mdev, "probe successful\n");
-	return 0;
-
-fail_video:
-	mxr_release_video(mdev);
-
-fail_resources:
-	mxr_release_resources(mdev);
-
-fail_mem:
-	kfree(mdev);
-
-fail:
-	dev_info(dev, "probe failed\n");
-	return ret;
-}
-
-static int mxr_remove(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct mxr_device *mdev = to_mdev(dev);
-
-	pm_runtime_disable(dev);
-
-	mxr_release_layers(mdev);
-	mxr_release_video(mdev);
-	mxr_release_resources(mdev);
-
-	kfree(mdev);
-
-	dev_info(dev, "remove successful\n");
-	return 0;
-}
-
-static struct platform_driver mxr_driver __refdata = {
-	.probe = mxr_probe,
-	.remove = mxr_remove,
-	.driver = {
-		.name = MXR_DRIVER_NAME,
-		.pm = &mxr_pm_ops,
-	}
-};
-
-static int __init mxr_init(void)
-{
-	int i, ret;
-	static const char banner[] __initconst =
-		"Samsung TV Mixer driver, "
-		"(c) 2010-2011 Samsung Electronics Co., Ltd.\n";
-	pr_info("%s\n", banner);
-
-	/* Loading auxiliary modules */
-	for (i = 0; i < ARRAY_SIZE(mxr_output_conf); ++i)
-		request_module(mxr_output_conf[i].module_name);
-
-	ret = platform_driver_register(&mxr_driver);
-	if (ret != 0) {
-		pr_err("s5p-tv: registration of MIXER driver failed\n");
-		return -ENXIO;
-	}
-
-	return 0;
-}
-module_init(mxr_init);
-
-static void __exit mxr_exit(void)
-{
-	platform_driver_unregister(&mxr_driver);
-}
-module_exit(mxr_exit);
diff --git a/drivers/media/platform/s5p-tv/mixer_grp_layer.c b/drivers/media/platform/s5p-tv/mixer_grp_layer.c
deleted file mode 100644
index d4d2564f7de7..000000000000
--- a/drivers/media/platform/s5p-tv/mixer_grp_layer.c
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Samsung TV Mixer driver
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *
- * Tomasz Stanislawski, <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published
- * by the Free Software Foundiation. either version 2 of the License,
- * or (at your option) any later version
- */
-
-#include "mixer.h"
-
-#include <media/videobuf2-dma-contig.h>
-
-/* FORMAT DEFINITIONS */
-
-static const struct mxr_format mxr_fb_fmt_rgb565 = {
-	.name = "RGB565",
-	.fourcc = V4L2_PIX_FMT_RGB565,
-	.colorspace = V4L2_COLORSPACE_SRGB,
-	.num_planes = 1,
-	.plane = {
-		{ .width = 1, .height = 1, .size = 2 },
-	},
-	.num_subframes = 1,
-	.cookie = 4,
-};
-
-static const struct mxr_format mxr_fb_fmt_argb1555 = {
-	.name = "ARGB1555",
-	.num_planes = 1,
-	.fourcc = V4L2_PIX_FMT_RGB555,
-	.colorspace = V4L2_COLORSPACE_SRGB,
-	.plane = {
-		{ .width = 1, .height = 1, .size = 2 },
-	},
-	.num_subframes = 1,
-	.cookie = 5,
-};
-
-static const struct mxr_format mxr_fb_fmt_argb4444 = {
-	.name = "ARGB4444",
-	.num_planes = 1,
-	.fourcc = V4L2_PIX_FMT_RGB444,
-	.colorspace = V4L2_COLORSPACE_SRGB,
-	.plane = {
-		{ .width = 1, .height = 1, .size = 2 },
-	},
-	.num_subframes = 1,
-	.cookie = 6,
-};
-
-static const struct mxr_format mxr_fb_fmt_argb8888 = {
-	.name = "ARGB8888",
-	.fourcc = V4L2_PIX_FMT_BGR32,
-	.colorspace = V4L2_COLORSPACE_SRGB,
-	.num_planes = 1,
-	.plane = {
-		{ .width = 1, .height = 1, .size = 4 },
-	},
-	.num_subframes = 1,
-	.cookie = 7,
-};
-
-static const struct mxr_format *mxr_graph_format[] = {
-	&mxr_fb_fmt_rgb565,
-	&mxr_fb_fmt_argb1555,
-	&mxr_fb_fmt_argb4444,
-	&mxr_fb_fmt_argb8888,
-};
-
-/* AUXILIARY CALLBACKS */
-
-static void mxr_graph_layer_release(struct mxr_layer *layer)
-{
-	mxr_base_layer_unregister(layer);
-	mxr_base_layer_release(layer);
-}
-
-static void mxr_graph_buffer_set(struct mxr_layer *layer,
-	struct mxr_buffer *buf)
-{
-	dma_addr_t addr = 0;
-
-	if (buf)
-		addr = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
-	mxr_reg_graph_buffer(layer->mdev, layer->idx, addr);
-}
-
-static void mxr_graph_stream_set(struct mxr_layer *layer, int en)
-{
-	mxr_reg_graph_layer_stream(layer->mdev, layer->idx, en);
-}
-
-static void mxr_graph_format_set(struct mxr_layer *layer)
-{
-	mxr_reg_graph_format(layer->mdev, layer->idx,
-		layer->fmt, &layer->geo);
-}
-
-static inline unsigned int closest(unsigned int x, unsigned int a,
-	unsigned int b, unsigned long flags)
-{
-	unsigned int mid = (a + b) / 2;
-
-	/* choosing closest value with constraints according to table:
-	 * -------------+-----+-----+-----+-------+
-	 * flags	|  0  |  LE |  GE | LE|GE |
-	 * -------------+-----+-----+-----+-------+
-	 * x <= a	|  a  |  a  |  a  |   a   |
-	 * a < x <= mid	|  a  |  a  |  b  |   a   |
-	 * mid < x < b	|  b  |  a  |  b  |   b   |
-	 * b <= x	|  b  |  b  |  b  |   b   |
-	 * -------------+-----+-----+-----+-------+
-	 */
-
-	/* remove all non-constraint flags */
-	flags &= V4L2_SEL_FLAG_LE | V4L2_SEL_FLAG_GE;
-
-	if (x <= a)
-		return  a;
-	if (x >= b)
-		return b;
-	if (flags == V4L2_SEL_FLAG_LE)
-		return a;
-	if (flags == V4L2_SEL_FLAG_GE)
-		return b;
-	if (x <= mid)
-		return a;
-	return b;
-}
-
-static inline unsigned int do_center(unsigned int center,
-	unsigned int size, unsigned int upper, unsigned int flags)
-{
-	unsigned int lower;
-
-	if (flags & MXR_NO_OFFSET)
-		return 0;
-
-	lower = center - min(center, size / 2);
-	return min(lower, upper - size);
-}
-
-static void mxr_graph_fix_geometry(struct mxr_layer *layer,
-	enum mxr_geometry_stage stage, unsigned long flags)
-{
-	struct mxr_geometry *geo = &layer->geo;
-	struct mxr_crop *src = &geo->src;
-	struct mxr_crop *dst = &geo->dst;
-	unsigned int x_center, y_center;
-
-	switch (stage) {
-
-	case MXR_GEOMETRY_SINK: /* nothing to be fixed here */
-		flags = 0;
-		/* fall through */
-
-	case MXR_GEOMETRY_COMPOSE:
-		/* remember center of the area */
-		x_center = dst->x_offset + dst->width / 2;
-		y_center = dst->y_offset + dst->height / 2;
-		/* round up/down to 2 multiple depending on flags */
-		if (flags & V4L2_SEL_FLAG_LE) {
-			dst->width = round_down(dst->width, 2);
-			dst->height = round_down(dst->height, 2);
-		} else {
-			dst->width = round_up(dst->width, 2);
-			dst->height = round_up(dst->height, 2);
-		}
-		/* assure that compose rect is inside display area */
-		dst->width = min(dst->width, dst->full_width);
-		dst->height = min(dst->height, dst->full_height);
-
-		/* ensure that compose is reachable using 2x scaling */
-		dst->width = min(dst->width, 2 * src->full_width);
-		dst->height = min(dst->height, 2 * src->full_height);
-
-		/* setup offsets */
-		dst->x_offset = do_center(x_center, dst->width,
-			dst->full_width, flags);
-		dst->y_offset = do_center(y_center, dst->height,
-			dst->full_height, flags);
-		flags = 0;
-		/* fall through */
-
-	case MXR_GEOMETRY_CROP:
-		/* remember center of the area */
-		x_center = src->x_offset + src->width / 2;
-		y_center = src->y_offset + src->height / 2;
-		/* ensure that cropping area lies inside the buffer */
-		if (src->full_width < dst->width)
-			src->width = dst->width / 2;
-		else
-			src->width = closest(src->width, dst->width / 2,
-				dst->width, flags);
-
-		if (src->width == dst->width)
-			geo->x_ratio = 0;
-		else
-			geo->x_ratio = 1;
-
-		if (src->full_height < dst->height)
-			src->height = dst->height / 2;
-		else
-			src->height = closest(src->height, dst->height / 2,
-				dst->height, flags);
-
-		if (src->height == dst->height)
-			geo->y_ratio = 0;
-		else
-			geo->y_ratio = 1;
-
-		/* setup offsets */
-		src->x_offset = do_center(x_center, src->width,
-			src->full_width, flags);
-		src->y_offset = do_center(y_center, src->height,
-			src->full_height, flags);
-		flags = 0;
-		/* fall through */
-	case MXR_GEOMETRY_SOURCE:
-		src->full_width = clamp_val(src->full_width,
-			src->width + src->x_offset, 32767);
-		src->full_height = clamp_val(src->full_height,
-			src->height + src->y_offset, 2047);
-	}
-}
-
-/* PUBLIC API */
-
-struct mxr_layer *mxr_graph_layer_create(struct mxr_device *mdev, int idx)
-{
-	struct mxr_layer *layer;
-	int ret;
-	const struct mxr_layer_ops ops = {
-		.release = mxr_graph_layer_release,
-		.buffer_set = mxr_graph_buffer_set,
-		.stream_set = mxr_graph_stream_set,
-		.format_set = mxr_graph_format_set,
-		.fix_geometry = mxr_graph_fix_geometry,
-	};
-	char name[32];
-
-	sprintf(name, "graph%d", idx);
-
-	layer = mxr_base_layer_create(mdev, idx, name, &ops);
-	if (layer == NULL) {
-		mxr_err(mdev, "failed to initialize layer(%d) base\n", idx);
-		goto fail;
-	}
-
-	layer->fmt_array = mxr_graph_format;
-	layer->fmt_array_size = ARRAY_SIZE(mxr_graph_format);
-
-	ret = mxr_base_layer_register(layer);
-	if (ret)
-		goto fail_layer;
-
-	return layer;
-
-fail_layer:
-	mxr_base_layer_release(layer);
-
-fail:
-	return NULL;
-}
-
diff --git a/drivers/media/platform/s5p-tv/mixer_reg.c b/drivers/media/platform/s5p-tv/mixer_reg.c
deleted file mode 100644
index a0ec14a1da13..000000000000
--- a/drivers/media/platform/s5p-tv/mixer_reg.c
+++ /dev/null
@@ -1,551 +0,0 @@
-/*
- * Samsung TV Mixer driver
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *
- * Tomasz Stanislawski, <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published
- * by the Free Software Foundiation. either version 2 of the License,
- * or (at your option) any later version
- */
-
-#include "mixer.h"
-#include "regs-mixer.h"
-#include "regs-vp.h"
-
-#include <linux/delay.h>
-
-/* Register access subroutines */
-
-static inline u32 vp_read(struct mxr_device *mdev, u32 reg_id)
-{
-	return readl(mdev->res.vp_regs + reg_id);
-}
-
-static inline void vp_write(struct mxr_device *mdev, u32 reg_id, u32 val)
-{
-	writel(val, mdev->res.vp_regs + reg_id);
-}
-
-static inline void vp_write_mask(struct mxr_device *mdev, u32 reg_id,
-	u32 val, u32 mask)
-{
-	u32 old = vp_read(mdev, reg_id);
-
-	val = (val & mask) | (old & ~mask);
-	writel(val, mdev->res.vp_regs + reg_id);
-}
-
-static inline u32 mxr_read(struct mxr_device *mdev, u32 reg_id)
-{
-	return readl(mdev->res.mxr_regs + reg_id);
-}
-
-static inline void mxr_write(struct mxr_device *mdev, u32 reg_id, u32 val)
-{
-	writel(val, mdev->res.mxr_regs + reg_id);
-}
-
-static inline void mxr_write_mask(struct mxr_device *mdev, u32 reg_id,
-	u32 val, u32 mask)
-{
-	u32 old = mxr_read(mdev, reg_id);
-
-	val = (val & mask) | (old & ~mask);
-	writel(val, mdev->res.mxr_regs + reg_id);
-}
-
-void mxr_vsync_set_update(struct mxr_device *mdev, int en)
-{
-	/* block update on vsync */
-	mxr_write_mask(mdev, MXR_STATUS, en ? MXR_STATUS_SYNC_ENABLE : 0,
-		MXR_STATUS_SYNC_ENABLE);
-	vp_write(mdev, VP_SHADOW_UPDATE, en ? VP_SHADOW_UPDATE_ENABLE : 0);
-}
-
-static void __mxr_reg_vp_reset(struct mxr_device *mdev)
-{
-	int tries = 100;
-
-	vp_write(mdev, VP_SRESET, VP_SRESET_PROCESSING);
-	for (tries = 100; tries; --tries) {
-		/* waiting until VP_SRESET_PROCESSING is 0 */
-		if (~vp_read(mdev, VP_SRESET) & VP_SRESET_PROCESSING)
-			break;
-		mdelay(10);
-	}
-	WARN(tries == 0, "failed to reset Video Processor\n");
-}
-
-static void mxr_reg_vp_default_filter(struct mxr_device *mdev);
-
-void mxr_reg_reset(struct mxr_device *mdev)
-{
-	unsigned long flags;
-	u32 val; /* value stored to register */
-
-	spin_lock_irqsave(&mdev->reg_slock, flags);
-	mxr_vsync_set_update(mdev, MXR_DISABLE);
-
-	/* set output in RGB888 mode */
-	mxr_write(mdev, MXR_CFG, MXR_CFG_OUT_RGB888);
-
-	/* 16 beat burst in DMA */
-	mxr_write_mask(mdev, MXR_STATUS, MXR_STATUS_16_BURST,
-		MXR_STATUS_BURST_MASK);
-
-	/* setting default layer priority: layer1 > video > layer0
-	 * because typical usage scenario would be
-	 * layer0 - framebuffer
-	 * video - video overlay
-	 * layer1 - OSD
-	 */
-	val  = MXR_LAYER_CFG_GRP0_VAL(1);
-	val |= MXR_LAYER_CFG_VP_VAL(2);
-	val |= MXR_LAYER_CFG_GRP1_VAL(3);
-	mxr_write(mdev, MXR_LAYER_CFG, val);
-
-	/* use dark gray background color */
-	mxr_write(mdev, MXR_BG_COLOR0, 0x808080);
-	mxr_write(mdev, MXR_BG_COLOR1, 0x808080);
-	mxr_write(mdev, MXR_BG_COLOR2, 0x808080);
-
-	/* setting graphical layers */
-
-	val  = MXR_GRP_CFG_COLOR_KEY_DISABLE; /* no blank key */
-	val |= MXR_GRP_CFG_BLEND_PRE_MUL; /* premul mode */
-	val |= MXR_GRP_CFG_ALPHA_VAL(0xff); /* non-transparent alpha */
-
-	/* the same configuration for both layers */
-	mxr_write(mdev, MXR_GRAPHIC_CFG(0), val);
-	mxr_write(mdev, MXR_GRAPHIC_CFG(1), val);
-
-	/* configuration of Video Processor Registers */
-	__mxr_reg_vp_reset(mdev);
-	mxr_reg_vp_default_filter(mdev);
-
-	/* enable all interrupts */
-	mxr_write_mask(mdev, MXR_INT_EN, ~0, MXR_INT_EN_ALL);
-
-	mxr_vsync_set_update(mdev, MXR_ENABLE);
-	spin_unlock_irqrestore(&mdev->reg_slock, flags);
-}
-
-void mxr_reg_graph_format(struct mxr_device *mdev, int idx,
-	const struct mxr_format *fmt, const struct mxr_geometry *geo)
-{
-	u32 val;
-	unsigned long flags;
-
-	spin_lock_irqsave(&mdev->reg_slock, flags);
-	mxr_vsync_set_update(mdev, MXR_DISABLE);
-
-	/* setup format */
-	mxr_write_mask(mdev, MXR_GRAPHIC_CFG(idx),
-		MXR_GRP_CFG_FORMAT_VAL(fmt->cookie), MXR_GRP_CFG_FORMAT_MASK);
-
-	/* setup geometry */
-	mxr_write(mdev, MXR_GRAPHIC_SPAN(idx), geo->src.full_width);
-	val  = MXR_GRP_WH_WIDTH(geo->src.width);
-	val |= MXR_GRP_WH_HEIGHT(geo->src.height);
-	val |= MXR_GRP_WH_H_SCALE(geo->x_ratio);
-	val |= MXR_GRP_WH_V_SCALE(geo->y_ratio);
-	mxr_write(mdev, MXR_GRAPHIC_WH(idx), val);
-
-	/* setup offsets in source image */
-	val  = MXR_GRP_SXY_SX(geo->src.x_offset);
-	val |= MXR_GRP_SXY_SY(geo->src.y_offset);
-	mxr_write(mdev, MXR_GRAPHIC_SXY(idx), val);
-
-	/* setup offsets in display image */
-	val  = MXR_GRP_DXY_DX(geo->dst.x_offset);
-	val |= MXR_GRP_DXY_DY(geo->dst.y_offset);
-	mxr_write(mdev, MXR_GRAPHIC_DXY(idx), val);
-
-	mxr_vsync_set_update(mdev, MXR_ENABLE);
-	spin_unlock_irqrestore(&mdev->reg_slock, flags);
-}
-
-void mxr_reg_vp_format(struct mxr_device *mdev,
-	const struct mxr_format *fmt, const struct mxr_geometry *geo)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mdev->reg_slock, flags);
-	mxr_vsync_set_update(mdev, MXR_DISABLE);
-
-	vp_write_mask(mdev, VP_MODE, fmt->cookie, VP_MODE_FMT_MASK);
-
-	/* setting size of input image */
-	vp_write(mdev, VP_IMG_SIZE_Y, VP_IMG_HSIZE(geo->src.full_width) |
-		VP_IMG_VSIZE(geo->src.full_height));
-	/* chroma height has to reduced by 2 to avoid chroma distorions */
-	vp_write(mdev, VP_IMG_SIZE_C, VP_IMG_HSIZE(geo->src.full_width) |
-		VP_IMG_VSIZE(geo->src.full_height / 2));
-
-	vp_write(mdev, VP_SRC_WIDTH, geo->src.width);
-	vp_write(mdev, VP_SRC_HEIGHT, geo->src.height);
-	vp_write(mdev, VP_SRC_H_POSITION,
-		VP_SRC_H_POSITION_VAL(geo->src.x_offset));
-	vp_write(mdev, VP_SRC_V_POSITION, geo->src.y_offset);
-
-	vp_write(mdev, VP_DST_WIDTH, geo->dst.width);
-	vp_write(mdev, VP_DST_H_POSITION, geo->dst.x_offset);
-	if (geo->dst.field == V4L2_FIELD_INTERLACED) {
-		vp_write(mdev, VP_DST_HEIGHT, geo->dst.height / 2);
-		vp_write(mdev, VP_DST_V_POSITION, geo->dst.y_offset / 2);
-	} else {
-		vp_write(mdev, VP_DST_HEIGHT, geo->dst.height);
-		vp_write(mdev, VP_DST_V_POSITION, geo->dst.y_offset);
-	}
-
-	vp_write(mdev, VP_H_RATIO, geo->x_ratio);
-	vp_write(mdev, VP_V_RATIO, geo->y_ratio);
-
-	vp_write(mdev, VP_ENDIAN_MODE, VP_ENDIAN_MODE_LITTLE);
-
-	mxr_vsync_set_update(mdev, MXR_ENABLE);
-	spin_unlock_irqrestore(&mdev->reg_slock, flags);
-
-}
-
-void mxr_reg_graph_buffer(struct mxr_device *mdev, int idx, dma_addr_t addr)
-{
-	u32 val = addr ? ~0 : 0;
-	unsigned long flags;
-
-	spin_lock_irqsave(&mdev->reg_slock, flags);
-	mxr_vsync_set_update(mdev, MXR_DISABLE);
-
-	if (idx == 0)
-		mxr_write_mask(mdev, MXR_CFG, val, MXR_CFG_GRP0_ENABLE);
-	else
-		mxr_write_mask(mdev, MXR_CFG, val, MXR_CFG_GRP1_ENABLE);
-	mxr_write(mdev, MXR_GRAPHIC_BASE(idx), addr);
-
-	mxr_vsync_set_update(mdev, MXR_ENABLE);
-	spin_unlock_irqrestore(&mdev->reg_slock, flags);
-}
-
-void mxr_reg_vp_buffer(struct mxr_device *mdev,
-	dma_addr_t luma_addr[2], dma_addr_t chroma_addr[2])
-{
-	u32 val = luma_addr[0] ? ~0 : 0;
-	unsigned long flags;
-
-	spin_lock_irqsave(&mdev->reg_slock, flags);
-	mxr_vsync_set_update(mdev, MXR_DISABLE);
-
-	mxr_write_mask(mdev, MXR_CFG, val, MXR_CFG_VP_ENABLE);
-	vp_write_mask(mdev, VP_ENABLE, val, VP_ENABLE_ON);
-	/* TODO: fix tiled mode */
-	vp_write(mdev, VP_TOP_Y_PTR, luma_addr[0]);
-	vp_write(mdev, VP_TOP_C_PTR, chroma_addr[0]);
-	vp_write(mdev, VP_BOT_Y_PTR, luma_addr[1]);
-	vp_write(mdev, VP_BOT_C_PTR, chroma_addr[1]);
-
-	mxr_vsync_set_update(mdev, MXR_ENABLE);
-	spin_unlock_irqrestore(&mdev->reg_slock, flags);
-}
-
-static void mxr_irq_layer_handle(struct mxr_layer *layer)
-{
-	struct list_head *head = &layer->enq_list;
-	struct mxr_buffer *done;
-
-	/* skip non-existing layer */
-	if (layer == NULL)
-		return;
-
-	spin_lock(&layer->enq_slock);
-	if (layer->state == MXR_LAYER_IDLE)
-		goto done;
-
-	done = layer->shadow_buf;
-	layer->shadow_buf = layer->update_buf;
-
-	if (list_empty(head)) {
-		if (layer->state != MXR_LAYER_STREAMING)
-			layer->update_buf = NULL;
-	} else {
-		struct mxr_buffer *next;
-		next = list_first_entry(head, struct mxr_buffer, list);
-		list_del(&next->list);
-		layer->update_buf = next;
-	}
-
-	layer->ops.buffer_set(layer, layer->update_buf);
-
-	if (done && done != layer->shadow_buf)
-		vb2_buffer_done(&done->vb.vb2_buf, VB2_BUF_STATE_DONE);
-
-done:
-	spin_unlock(&layer->enq_slock);
-}
-
-irqreturn_t mxr_irq_handler(int irq, void *dev_data)
-{
-	struct mxr_device *mdev = dev_data;
-	u32 i, val;
-
-	spin_lock(&mdev->reg_slock);
-	val = mxr_read(mdev, MXR_INT_STATUS);
-
-	/* wake up process waiting for VSYNC */
-	if (val & MXR_INT_STATUS_VSYNC) {
-		set_bit(MXR_EVENT_VSYNC, &mdev->event_flags);
-		/* toggle TOP field event if working in interlaced mode */
-		if (~mxr_read(mdev, MXR_CFG) & MXR_CFG_SCAN_PROGRASSIVE)
-			change_bit(MXR_EVENT_TOP, &mdev->event_flags);
-		wake_up(&mdev->event_queue);
-		/* vsync interrupt use different bit for read and clear */
-		val &= ~MXR_INT_STATUS_VSYNC;
-		val |= MXR_INT_CLEAR_VSYNC;
-	}
-
-	/* clear interrupts */
-	mxr_write(mdev, MXR_INT_STATUS, val);
-
-	spin_unlock(&mdev->reg_slock);
-	/* leave on non-vsync event */
-	if (~val & MXR_INT_CLEAR_VSYNC)
-		return IRQ_HANDLED;
-	/* skip layer update on bottom field */
-	if (!test_bit(MXR_EVENT_TOP, &mdev->event_flags))
-		return IRQ_HANDLED;
-	for (i = 0; i < MXR_MAX_LAYERS; ++i)
-		mxr_irq_layer_handle(mdev->layer[i]);
-	return IRQ_HANDLED;
-}
-
-void mxr_reg_s_output(struct mxr_device *mdev, int cookie)
-{
-	u32 val;
-
-	val = cookie == 0 ? MXR_CFG_DST_SDO : MXR_CFG_DST_HDMI;
-	mxr_write_mask(mdev, MXR_CFG, val, MXR_CFG_DST_MASK);
-}
-
-void mxr_reg_streamon(struct mxr_device *mdev)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mdev->reg_slock, flags);
-	/* single write -> no need to block vsync update */
-
-	/* start MIXER */
-	mxr_write_mask(mdev, MXR_STATUS, ~0, MXR_STATUS_REG_RUN);
-	set_bit(MXR_EVENT_TOP, &mdev->event_flags);
-
-	spin_unlock_irqrestore(&mdev->reg_slock, flags);
-}
-
-void mxr_reg_streamoff(struct mxr_device *mdev)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mdev->reg_slock, flags);
-	/* single write -> no need to block vsync update */
-
-	/* stop MIXER */
-	mxr_write_mask(mdev, MXR_STATUS, 0, MXR_STATUS_REG_RUN);
-
-	spin_unlock_irqrestore(&mdev->reg_slock, flags);
-}
-
-int mxr_reg_wait4vsync(struct mxr_device *mdev)
-{
-	long time_left;
-
-	clear_bit(MXR_EVENT_VSYNC, &mdev->event_flags);
-	/* TODO: consider adding interruptible */
-	time_left = wait_event_timeout(mdev->event_queue,
-			test_bit(MXR_EVENT_VSYNC, &mdev->event_flags),
-				 msecs_to_jiffies(1000));
-	if (time_left > 0)
-		return 0;
-	mxr_warn(mdev, "no vsync detected - timeout\n");
-	return -ETIME;
-}
-
-void mxr_reg_set_mbus_fmt(struct mxr_device *mdev,
-	struct v4l2_mbus_framefmt *fmt)
-{
-	u32 val = 0;
-	unsigned long flags;
-
-	spin_lock_irqsave(&mdev->reg_slock, flags);
-	mxr_vsync_set_update(mdev, MXR_DISABLE);
-
-	/* selecting colorspace accepted by output */
-	if (fmt->colorspace == V4L2_COLORSPACE_JPEG)
-		val |= MXR_CFG_OUT_YUV444;
-	else
-		val |= MXR_CFG_OUT_RGB888;
-
-	/* choosing between interlace and progressive mode */
-	if (fmt->field == V4L2_FIELD_INTERLACED)
-		val |= MXR_CFG_SCAN_INTERLACE;
-	else
-		val |= MXR_CFG_SCAN_PROGRASSIVE;
-
-	/* choosing between porper HD and SD mode */
-	if (fmt->height == 480)
-		val |= MXR_CFG_SCAN_NTSC | MXR_CFG_SCAN_SD;
-	else if (fmt->height == 576)
-		val |= MXR_CFG_SCAN_PAL | MXR_CFG_SCAN_SD;
-	else if (fmt->height == 720)
-		val |= MXR_CFG_SCAN_HD_720 | MXR_CFG_SCAN_HD;
-	else if (fmt->height == 1080)
-		val |= MXR_CFG_SCAN_HD_1080 | MXR_CFG_SCAN_HD;
-	else
-		WARN(1, "unrecognized mbus height %u!\n", fmt->height);
-
-	mxr_write_mask(mdev, MXR_CFG, val, MXR_CFG_SCAN_MASK |
-		MXR_CFG_OUT_MASK);
-
-	val = (fmt->field == V4L2_FIELD_INTERLACED) ? ~0 : 0;
-	vp_write_mask(mdev, VP_MODE, val,
-		VP_MODE_LINE_SKIP | VP_MODE_FIELD_ID_AUTO_TOGGLING);
-
-	mxr_vsync_set_update(mdev, MXR_ENABLE);
-	spin_unlock_irqrestore(&mdev->reg_slock, flags);
-}
-
-void mxr_reg_graph_layer_stream(struct mxr_device *mdev, int idx, int en)
-{
-	/* no extra actions need to be done */
-}
-
-void mxr_reg_vp_layer_stream(struct mxr_device *mdev, int en)
-{
-	/* no extra actions need to be done */
-}
-
-static const u8 filter_y_horiz_tap8[] = {
-	0,	-1,	-1,	-1,	-1,	-1,	-1,	-1,
-	-1,	-1,	-1,	-1,	-1,	0,	0,	0,
-	0,	2,	4,	5,	6,	6,	6,	6,
-	6,	5,	5,	4,	3,	2,	1,	1,
-	0,	-6,	-12,	-16,	-18,	-20,	-21,	-20,
-	-20,	-18,	-16,	-13,	-10,	-8,	-5,	-2,
-	127,	126,	125,	121,	114,	107,	99,	89,
-	79,	68,	57,	46,	35,	25,	16,	8,
-};
-
-static const u8 filter_y_vert_tap4[] = {
-	0,	-3,	-6,	-8,	-8,	-8,	-8,	-7,
-	-6,	-5,	-4,	-3,	-2,	-1,	-1,	0,
-	127,	126,	124,	118,	111,	102,	92,	81,
-	70,	59,	48,	37,	27,	19,	11,	5,
-	0,	5,	11,	19,	27,	37,	48,	59,
-	70,	81,	92,	102,	111,	118,	124,	126,
-	0,	0,	-1,	-1,	-2,	-3,	-4,	-5,
-	-6,	-7,	-8,	-8,	-8,	-8,	-6,	-3,
-};
-
-static const u8 filter_cr_horiz_tap4[] = {
-	0,	-3,	-6,	-8,	-8,	-8,	-8,	-7,
-	-6,	-5,	-4,	-3,	-2,	-1,	-1,	0,
-	127,	126,	124,	118,	111,	102,	92,	81,
-	70,	59,	48,	37,	27,	19,	11,	5,
-};
-
-static inline void mxr_reg_vp_filter_set(struct mxr_device *mdev,
-	int reg_id, const u8 *data, unsigned int size)
-{
-	/* assure 4-byte align */
-	BUG_ON(size & 3);
-	for (; size; size -= 4, reg_id += 4, data += 4) {
-		u32 val = (data[0] << 24) |  (data[1] << 16) |
-			(data[2] << 8) | data[3];
-		vp_write(mdev, reg_id, val);
-	}
-}
-
-static void mxr_reg_vp_default_filter(struct mxr_device *mdev)
-{
-	mxr_reg_vp_filter_set(mdev, VP_POLY8_Y0_LL,
-		filter_y_horiz_tap8, sizeof(filter_y_horiz_tap8));
-	mxr_reg_vp_filter_set(mdev, VP_POLY4_Y0_LL,
-		filter_y_vert_tap4, sizeof(filter_y_vert_tap4));
-	mxr_reg_vp_filter_set(mdev, VP_POLY4_C0_LL,
-		filter_cr_horiz_tap4, sizeof(filter_cr_horiz_tap4));
-}
-
-static void mxr_reg_mxr_dump(struct mxr_device *mdev)
-{
-#define DUMPREG(reg_id) \
-do { \
-	mxr_dbg(mdev, #reg_id " = %08x\n", \
-		(u32)readl(mdev->res.mxr_regs + reg_id)); \
-} while (0)
-
-	DUMPREG(MXR_STATUS);
-	DUMPREG(MXR_CFG);
-	DUMPREG(MXR_INT_EN);
-	DUMPREG(MXR_INT_STATUS);
-
-	DUMPREG(MXR_LAYER_CFG);
-	DUMPREG(MXR_VIDEO_CFG);
-
-	DUMPREG(MXR_GRAPHIC0_CFG);
-	DUMPREG(MXR_GRAPHIC0_BASE);
-	DUMPREG(MXR_GRAPHIC0_SPAN);
-	DUMPREG(MXR_GRAPHIC0_WH);
-	DUMPREG(MXR_GRAPHIC0_SXY);
-	DUMPREG(MXR_GRAPHIC0_DXY);
-
-	DUMPREG(MXR_GRAPHIC1_CFG);
-	DUMPREG(MXR_GRAPHIC1_BASE);
-	DUMPREG(MXR_GRAPHIC1_SPAN);
-	DUMPREG(MXR_GRAPHIC1_WH);
-	DUMPREG(MXR_GRAPHIC1_SXY);
-	DUMPREG(MXR_GRAPHIC1_DXY);
-#undef DUMPREG
-}
-
-static void mxr_reg_vp_dump(struct mxr_device *mdev)
-{
-#define DUMPREG(reg_id) \
-do { \
-	mxr_dbg(mdev, #reg_id " = %08x\n", \
-		(u32) readl(mdev->res.vp_regs + reg_id)); \
-} while (0)
-
-
-	DUMPREG(VP_ENABLE);
-	DUMPREG(VP_SRESET);
-	DUMPREG(VP_SHADOW_UPDATE);
-	DUMPREG(VP_FIELD_ID);
-	DUMPREG(VP_MODE);
-	DUMPREG(VP_IMG_SIZE_Y);
-	DUMPREG(VP_IMG_SIZE_C);
-	DUMPREG(VP_PER_RATE_CTRL);
-	DUMPREG(VP_TOP_Y_PTR);
-	DUMPREG(VP_BOT_Y_PTR);
-	DUMPREG(VP_TOP_C_PTR);
-	DUMPREG(VP_BOT_C_PTR);
-	DUMPREG(VP_ENDIAN_MODE);
-	DUMPREG(VP_SRC_H_POSITION);
-	DUMPREG(VP_SRC_V_POSITION);
-	DUMPREG(VP_SRC_WIDTH);
-	DUMPREG(VP_SRC_HEIGHT);
-	DUMPREG(VP_DST_H_POSITION);
-	DUMPREG(VP_DST_V_POSITION);
-	DUMPREG(VP_DST_WIDTH);
-	DUMPREG(VP_DST_HEIGHT);
-	DUMPREG(VP_H_RATIO);
-	DUMPREG(VP_V_RATIO);
-
-#undef DUMPREG
-}
-
-void mxr_reg_dump(struct mxr_device *mdev)
-{
-	mxr_reg_mxr_dump(mdev);
-	mxr_reg_vp_dump(mdev);
-}
-
diff --git a/drivers/media/platform/s5p-tv/mixer_video.c b/drivers/media/platform/s5p-tv/mixer_video.c
deleted file mode 100644
index ee74e2b44d69..000000000000
--- a/drivers/media/platform/s5p-tv/mixer_video.c
+++ /dev/null
@@ -1,1130 +0,0 @@
-/*
- * Samsung TV Mixer driver
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *
- * Tomasz Stanislawski, <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published
- * by the Free Software Foundation. either version 2 of the License,
- * or (at your option) any later version
- */
-
-#define pr_fmt(fmt) "s5p-tv (mixer): " fmt
-
-#include "mixer.h"
-
-#include <media/v4l2-ioctl.h>
-#include <linux/videodev2.h>
-#include <linux/mm.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/timer.h>
-#include <media/videobuf2-dma-contig.h>
-
-static int find_reg_callback(struct device *dev, void *p)
-{
-	struct v4l2_subdev **sd = p;
-
-	*sd = dev_get_drvdata(dev);
-	/* non-zero value stops iteration */
-	return 1;
-}
-
-static struct v4l2_subdev *find_and_register_subdev(
-	struct mxr_device *mdev, char *module_name)
-{
-	struct device_driver *drv;
-	struct v4l2_subdev *sd = NULL;
-	int ret;
-
-	/* TODO: add waiting until probe is finished */
-	drv = driver_find(module_name, &platform_bus_type);
-	if (!drv) {
-		mxr_warn(mdev, "module %s is missing\n", module_name);
-		return NULL;
-	}
-	/* driver refcnt is increased, it is safe to iterate over devices */
-	ret = driver_for_each_device(drv, NULL, &sd, find_reg_callback);
-	/* ret == 0 means that find_reg_callback was never executed */
-	if (sd == NULL) {
-		mxr_warn(mdev, "module %s provides no subdev!\n", module_name);
-		goto done;
-	}
-	/* v4l2_device_register_subdev detects if sd is NULL */
-	ret = v4l2_device_register_subdev(&mdev->v4l2_dev, sd);
-	if (ret) {
-		mxr_warn(mdev, "failed to register subdev %s\n", sd->name);
-		sd = NULL;
-	}
-
-done:
-	return sd;
-}
-
-int mxr_acquire_video(struct mxr_device *mdev,
-		      struct mxr_output_conf *output_conf, int output_count)
-{
-	struct device *dev = mdev->dev;
-	struct v4l2_device *v4l2_dev = &mdev->v4l2_dev;
-	int i;
-	int ret = 0;
-	struct v4l2_subdev *sd;
-
-	strlcpy(v4l2_dev->name, dev_name(mdev->dev), sizeof(v4l2_dev->name));
-	/* prepare context for V4L2 device */
-	ret = v4l2_device_register(dev, v4l2_dev);
-	if (ret) {
-		mxr_err(mdev, "could not register v4l2 device.\n");
-		goto fail;
-	}
-
-	vb2_dma_contig_set_max_seg_size(mdev->dev, DMA_BIT_MASK(32));
-
-	/* registering outputs */
-	mdev->output_cnt = 0;
-	for (i = 0; i < output_count; ++i) {
-		struct mxr_output_conf *conf = &output_conf[i];
-		struct mxr_output *out;
-
-		sd = find_and_register_subdev(mdev, conf->module_name);
-		/* trying to register next output */
-		if (sd == NULL)
-			continue;
-		out = kzalloc(sizeof(*out), GFP_KERNEL);
-		if (out == NULL) {
-			mxr_err(mdev, "no memory for '%s'\n",
-				conf->output_name);
-			ret = -ENOMEM;
-			/* registered subdevs are removed in fail_v4l2_dev */
-			goto fail_output;
-		}
-		strlcpy(out->name, conf->output_name, sizeof(out->name));
-		out->sd = sd;
-		out->cookie = conf->cookie;
-		mdev->output[mdev->output_cnt++] = out;
-		mxr_info(mdev, "added output '%s' from module '%s'\n",
-			conf->output_name, conf->module_name);
-		/* checking if maximal number of outputs is reached */
-		if (mdev->output_cnt >= MXR_MAX_OUTPUTS)
-			break;
-	}
-
-	if (mdev->output_cnt == 0) {
-		mxr_err(mdev, "failed to register any output\n");
-		ret = -ENODEV;
-		/* skipping fail_output because there is nothing to free */
-		goto fail_v4l2_dev;
-	}
-
-	return 0;
-
-fail_output:
-	/* kfree is NULL-safe */
-	for (i = 0; i < mdev->output_cnt; ++i)
-		kfree(mdev->output[i]);
-	memset(mdev->output, 0, sizeof(mdev->output));
-
-fail_v4l2_dev:
-	/* NOTE: automatically unregister all subdevs */
-	v4l2_device_unregister(v4l2_dev);
-
-fail:
-	return ret;
-}
-
-void mxr_release_video(struct mxr_device *mdev)
-{
-	int i;
-
-	/* kfree is NULL-safe */
-	for (i = 0; i < mdev->output_cnt; ++i)
-		kfree(mdev->output[i]);
-
-	vb2_dma_contig_clear_max_seg_size(mdev->dev);
-	v4l2_device_unregister(&mdev->v4l2_dev);
-}
-
-static int mxr_querycap(struct file *file, void *priv,
-	struct v4l2_capability *cap)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-
-	strlcpy(cap->driver, MXR_DRIVER_NAME, sizeof(cap->driver));
-	strlcpy(cap->card, layer->vfd.name, sizeof(cap->card));
-	sprintf(cap->bus_info, "%d", layer->idx);
-	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_OUTPUT_MPLANE;
-	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
-
-	return 0;
-}
-
-static void mxr_geometry_dump(struct mxr_device *mdev, struct mxr_geometry *geo)
-{
-	mxr_dbg(mdev, "src.full_size = (%u, %u)\n",
-		geo->src.full_width, geo->src.full_height);
-	mxr_dbg(mdev, "src.size = (%u, %u)\n",
-		geo->src.width, geo->src.height);
-	mxr_dbg(mdev, "src.offset = (%u, %u)\n",
-		geo->src.x_offset, geo->src.y_offset);
-	mxr_dbg(mdev, "dst.full_size = (%u, %u)\n",
-		geo->dst.full_width, geo->dst.full_height);
-	mxr_dbg(mdev, "dst.size = (%u, %u)\n",
-		geo->dst.width, geo->dst.height);
-	mxr_dbg(mdev, "dst.offset = (%u, %u)\n",
-		geo->dst.x_offset, geo->dst.y_offset);
-	mxr_dbg(mdev, "ratio = (%u, %u)\n",
-		geo->x_ratio, geo->y_ratio);
-}
-
-static void mxr_layer_default_geo(struct mxr_layer *layer)
-{
-	struct mxr_device *mdev = layer->mdev;
-	struct v4l2_mbus_framefmt mbus_fmt;
-
-	memset(&layer->geo, 0, sizeof(layer->geo));
-
-	mxr_get_mbus_fmt(mdev, &mbus_fmt);
-
-	layer->geo.dst.full_width = mbus_fmt.width;
-	layer->geo.dst.full_height = mbus_fmt.height;
-	layer->geo.dst.width = layer->geo.dst.full_width;
-	layer->geo.dst.height = layer->geo.dst.full_height;
-	layer->geo.dst.field = mbus_fmt.field;
-
-	layer->geo.src.full_width = mbus_fmt.width;
-	layer->geo.src.full_height = mbus_fmt.height;
-	layer->geo.src.width = layer->geo.src.full_width;
-	layer->geo.src.height = layer->geo.src.full_height;
-
-	mxr_geometry_dump(mdev, &layer->geo);
-	layer->ops.fix_geometry(layer, MXR_GEOMETRY_SINK, 0);
-	mxr_geometry_dump(mdev, &layer->geo);
-}
-
-static void mxr_layer_update_output(struct mxr_layer *layer)
-{
-	struct mxr_device *mdev = layer->mdev;
-	struct v4l2_mbus_framefmt mbus_fmt;
-
-	mxr_get_mbus_fmt(mdev, &mbus_fmt);
-	/* checking if update is needed */
-	if (layer->geo.dst.full_width == mbus_fmt.width &&
-		layer->geo.dst.full_height == mbus_fmt.width)
-		return;
-
-	layer->geo.dst.full_width = mbus_fmt.width;
-	layer->geo.dst.full_height = mbus_fmt.height;
-	layer->geo.dst.field = mbus_fmt.field;
-	layer->ops.fix_geometry(layer, MXR_GEOMETRY_SINK, 0);
-
-	mxr_geometry_dump(mdev, &layer->geo);
-}
-
-static const struct mxr_format *find_format_by_fourcc(
-	struct mxr_layer *layer, unsigned long fourcc);
-static const struct mxr_format *find_format_by_index(
-	struct mxr_layer *layer, unsigned long index);
-
-static int mxr_enum_fmt(struct file *file, void  *priv,
-	struct v4l2_fmtdesc *f)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-	const struct mxr_format *fmt;
-
-	mxr_dbg(mdev, "%s\n", __func__);
-	fmt = find_format_by_index(layer, f->index);
-	if (fmt == NULL)
-		return -EINVAL;
-
-	strlcpy(f->description, fmt->name, sizeof(f->description));
-	f->pixelformat = fmt->fourcc;
-
-	return 0;
-}
-
-static unsigned int divup(unsigned int divident, unsigned int divisor)
-{
-	return (divident + divisor - 1) / divisor;
-}
-
-unsigned long mxr_get_plane_size(const struct mxr_block *blk,
-	unsigned int width, unsigned int height)
-{
-	unsigned int bl_width = divup(width, blk->width);
-	unsigned int bl_height = divup(height, blk->height);
-
-	return bl_width * bl_height * blk->size;
-}
-
-static void mxr_mplane_fill(struct v4l2_plane_pix_format *planes,
-	const struct mxr_format *fmt, u32 width, u32 height)
-{
-	int i;
-
-	/* checking if nothing to fill */
-	if (!planes)
-		return;
-
-	memset(planes, 0, sizeof(*planes) * fmt->num_subframes);
-	for (i = 0; i < fmt->num_planes; ++i) {
-		struct v4l2_plane_pix_format *plane = planes
-			+ fmt->plane2subframe[i];
-		const struct mxr_block *blk = &fmt->plane[i];
-		u32 bl_width = divup(width, blk->width);
-		u32 bl_height = divup(height, blk->height);
-		u32 sizeimage = bl_width * bl_height * blk->size;
-		u32 bytesperline = bl_width * blk->size / blk->height;
-
-		plane->sizeimage += sizeimage;
-		plane->bytesperline = max(plane->bytesperline, bytesperline);
-	}
-}
-
-static int mxr_g_fmt(struct file *file, void *priv,
-			     struct v4l2_format *f)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp;
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-
-	pix->width = layer->geo.src.full_width;
-	pix->height = layer->geo.src.full_height;
-	pix->field = V4L2_FIELD_NONE;
-	pix->pixelformat = layer->fmt->fourcc;
-	pix->colorspace = layer->fmt->colorspace;
-	mxr_mplane_fill(pix->plane_fmt, layer->fmt, pix->width, pix->height);
-
-	return 0;
-}
-
-static int mxr_s_fmt(struct file *file, void *priv,
-	struct v4l2_format *f)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	const struct mxr_format *fmt;
-	struct v4l2_pix_format_mplane *pix;
-	struct mxr_device *mdev = layer->mdev;
-	struct mxr_geometry *geo = &layer->geo;
-
-	mxr_dbg(mdev, "%s:%d\n", __func__, __LINE__);
-
-	pix = &f->fmt.pix_mp;
-	fmt = find_format_by_fourcc(layer, pix->pixelformat);
-	if (fmt == NULL) {
-		mxr_warn(mdev, "not recognized fourcc: %08x\n",
-			pix->pixelformat);
-		return -EINVAL;
-	}
-	layer->fmt = fmt;
-	/* set source size to highest accepted value */
-	geo->src.full_width = max(geo->dst.full_width, pix->width);
-	geo->src.full_height = max(geo->dst.full_height, pix->height);
-	layer->ops.fix_geometry(layer, MXR_GEOMETRY_SOURCE, 0);
-	mxr_geometry_dump(mdev, &layer->geo);
-	/* set cropping to total visible screen */
-	geo->src.width = pix->width;
-	geo->src.height = pix->height;
-	geo->src.x_offset = 0;
-	geo->src.y_offset = 0;
-	/* assure consistency of geometry */
-	layer->ops.fix_geometry(layer, MXR_GEOMETRY_CROP, MXR_NO_OFFSET);
-	mxr_geometry_dump(mdev, &layer->geo);
-	/* set full size to lowest possible value */
-	geo->src.full_width = 0;
-	geo->src.full_height = 0;
-	layer->ops.fix_geometry(layer, MXR_GEOMETRY_SOURCE, 0);
-	mxr_geometry_dump(mdev, &layer->geo);
-
-	/* returning results */
-	mxr_g_fmt(file, priv, f);
-
-	return 0;
-}
-
-static int mxr_g_selection(struct file *file, void *fh,
-	struct v4l2_selection *s)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_geometry *geo = &layer->geo;
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-
-	if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
-		s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
-		return -EINVAL;
-
-	switch (s->target) {
-	case V4L2_SEL_TGT_CROP:
-		s->r.left = geo->src.x_offset;
-		s->r.top = geo->src.y_offset;
-		s->r.width = geo->src.width;
-		s->r.height = geo->src.height;
-		break;
-	case V4L2_SEL_TGT_CROP_DEFAULT:
-	case V4L2_SEL_TGT_CROP_BOUNDS:
-		s->r.left = 0;
-		s->r.top = 0;
-		s->r.width = geo->src.full_width;
-		s->r.height = geo->src.full_height;
-		break;
-	case V4L2_SEL_TGT_COMPOSE:
-	case V4L2_SEL_TGT_COMPOSE_PADDED:
-		s->r.left = geo->dst.x_offset;
-		s->r.top = geo->dst.y_offset;
-		s->r.width = geo->dst.width;
-		s->r.height = geo->dst.height;
-		break;
-	case V4L2_SEL_TGT_COMPOSE_DEFAULT:
-	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
-		s->r.left = 0;
-		s->r.top = 0;
-		s->r.width = geo->dst.full_width;
-		s->r.height = geo->dst.full_height;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* returns 1 if rectangle 'a' is inside 'b' */
-static int mxr_is_rect_inside(struct v4l2_rect *a, struct v4l2_rect *b)
-{
-	if (a->left < b->left)
-		return 0;
-	if (a->top < b->top)
-		return 0;
-	if (a->left + a->width > b->left + b->width)
-		return 0;
-	if (a->top + a->height > b->top + b->height)
-		return 0;
-	return 1;
-}
-
-static int mxr_s_selection(struct file *file, void *fh,
-	struct v4l2_selection *s)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_geometry *geo = &layer->geo;
-	struct mxr_crop *target = NULL;
-	enum mxr_geometry_stage stage;
-	struct mxr_geometry tmp;
-	struct v4l2_rect res;
-
-	memset(&res, 0, sizeof(res));
-
-	mxr_dbg(layer->mdev, "%s: rect: %dx%d@%d,%d\n", __func__,
-		s->r.width, s->r.height, s->r.left, s->r.top);
-
-	if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
-		s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
-		return -EINVAL;
-
-	switch (s->target) {
-	/* ignore read-only targets */
-	case V4L2_SEL_TGT_CROP_DEFAULT:
-	case V4L2_SEL_TGT_CROP_BOUNDS:
-		res.width = geo->src.full_width;
-		res.height = geo->src.full_height;
-		break;
-
-	/* ignore read-only targets */
-	case V4L2_SEL_TGT_COMPOSE_DEFAULT:
-	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
-		res.width = geo->dst.full_width;
-		res.height = geo->dst.full_height;
-		break;
-
-	case V4L2_SEL_TGT_CROP:
-		target = &geo->src;
-		stage = MXR_GEOMETRY_CROP;
-		break;
-	case V4L2_SEL_TGT_COMPOSE:
-	case V4L2_SEL_TGT_COMPOSE_PADDED:
-		target = &geo->dst;
-		stage = MXR_GEOMETRY_COMPOSE;
-		break;
-	default:
-		return -EINVAL;
-	}
-	/* apply change and update geometry if needed */
-	if (target) {
-		/* backup current geometry if setup fails */
-		memcpy(&tmp, geo, sizeof(tmp));
-
-		/* apply requested selection */
-		target->x_offset = s->r.left;
-		target->y_offset = s->r.top;
-		target->width = s->r.width;
-		target->height = s->r.height;
-
-		layer->ops.fix_geometry(layer, stage, s->flags);
-
-		/* retrieve update selection rectangle */
-		res.left = target->x_offset;
-		res.top = target->y_offset;
-		res.width = target->width;
-		res.height = target->height;
-
-		mxr_geometry_dump(layer->mdev, &layer->geo);
-	}
-
-	/* checking if the rectangle satisfies constraints */
-	if ((s->flags & V4L2_SEL_FLAG_LE) && !mxr_is_rect_inside(&res, &s->r))
-		goto fail;
-	if ((s->flags & V4L2_SEL_FLAG_GE) && !mxr_is_rect_inside(&s->r, &res))
-		goto fail;
-
-	/* return result rectangle */
-	s->r = res;
-
-	return 0;
-fail:
-	/* restore old geometry, which is not touched if target is NULL */
-	if (target)
-		memcpy(geo, &tmp, sizeof(tmp));
-	return -ERANGE;
-}
-
-static int mxr_enum_dv_timings(struct file *file, void *fh,
-	struct v4l2_enum_dv_timings *timings)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-	int ret;
-
-	timings->pad = 0;
-
-	/* lock protects from changing sd_out */
-	mutex_lock(&mdev->mutex);
-	ret = v4l2_subdev_call(to_outsd(mdev), pad, enum_dv_timings, timings);
-	mutex_unlock(&mdev->mutex);
-
-	return ret ? -EINVAL : 0;
-}
-
-static int mxr_s_dv_timings(struct file *file, void *fh,
-	struct v4l2_dv_timings *timings)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-	int ret;
-
-	/* lock protects from changing sd_out */
-	mutex_lock(&mdev->mutex);
-
-	/* timings change cannot be done while there is an entity
-	 * dependent on output configuration
-	 */
-	if (mdev->n_output > 0) {
-		mutex_unlock(&mdev->mutex);
-		return -EBUSY;
-	}
-
-	ret = v4l2_subdev_call(to_outsd(mdev), video, s_dv_timings, timings);
-
-	mutex_unlock(&mdev->mutex);
-
-	mxr_layer_update_output(layer);
-
-	/* any failure should return EINVAL according to V4L2 doc */
-	return ret ? -EINVAL : 0;
-}
-
-static int mxr_g_dv_timings(struct file *file, void *fh,
-	struct v4l2_dv_timings *timings)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-	int ret;
-
-	/* lock protects from changing sd_out */
-	mutex_lock(&mdev->mutex);
-	ret = v4l2_subdev_call(to_outsd(mdev), video, g_dv_timings, timings);
-	mutex_unlock(&mdev->mutex);
-
-	return ret ? -EINVAL : 0;
-}
-
-static int mxr_dv_timings_cap(struct file *file, void *fh,
-	struct v4l2_dv_timings_cap *cap)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-	int ret;
-
-	cap->pad = 0;
-
-	/* lock protects from changing sd_out */
-	mutex_lock(&mdev->mutex);
-	ret = v4l2_subdev_call(to_outsd(mdev), pad, dv_timings_cap, cap);
-	mutex_unlock(&mdev->mutex);
-
-	return ret ? -EINVAL : 0;
-}
-
-static int mxr_s_std(struct file *file, void *fh, v4l2_std_id norm)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-	int ret;
-
-	/* lock protects from changing sd_out */
-	mutex_lock(&mdev->mutex);
-
-	/* standard change cannot be done while there is an entity
-	 * dependent on output configuration
-	 */
-	if (mdev->n_output > 0) {
-		mutex_unlock(&mdev->mutex);
-		return -EBUSY;
-	}
-
-	ret = v4l2_subdev_call(to_outsd(mdev), video, s_std_output, norm);
-
-	mutex_unlock(&mdev->mutex);
-
-	mxr_layer_update_output(layer);
-
-	return ret ? -EINVAL : 0;
-}
-
-static int mxr_g_std(struct file *file, void *fh, v4l2_std_id *norm)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-	int ret;
-
-	/* lock protects from changing sd_out */
-	mutex_lock(&mdev->mutex);
-	ret = v4l2_subdev_call(to_outsd(mdev), video, g_std_output, norm);
-	mutex_unlock(&mdev->mutex);
-
-	return ret ? -EINVAL : 0;
-}
-
-static int mxr_enum_output(struct file *file, void *fh, struct v4l2_output *a)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-	struct mxr_output *out;
-	struct v4l2_subdev *sd;
-
-	if (a->index >= mdev->output_cnt)
-		return -EINVAL;
-	out = mdev->output[a->index];
-	BUG_ON(out == NULL);
-	sd = out->sd;
-	strlcpy(a->name, out->name, sizeof(a->name));
-
-	/* try to obtain supported tv norms */
-	v4l2_subdev_call(sd, video, g_tvnorms_output, &a->std);
-	a->capabilities = 0;
-	if (sd->ops->video && sd->ops->video->s_dv_timings)
-		a->capabilities |= V4L2_OUT_CAP_DV_TIMINGS;
-	if (sd->ops->video && sd->ops->video->s_std_output)
-		a->capabilities |= V4L2_OUT_CAP_STD;
-	a->type = V4L2_OUTPUT_TYPE_ANALOG;
-
-	return 0;
-}
-
-static int mxr_s_output(struct file *file, void *fh, unsigned int i)
-{
-	struct video_device *vfd = video_devdata(file);
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-
-	if (i >= mdev->output_cnt || mdev->output[i] == NULL)
-		return -EINVAL;
-
-	mutex_lock(&mdev->mutex);
-	if (mdev->n_output > 0) {
-		mutex_unlock(&mdev->mutex);
-		return -EBUSY;
-	}
-	mdev->current_output = i;
-	vfd->tvnorms = 0;
-	v4l2_subdev_call(to_outsd(mdev), video, g_tvnorms_output,
-		&vfd->tvnorms);
-	mutex_unlock(&mdev->mutex);
-
-	/* update layers geometry */
-	mxr_layer_update_output(layer);
-
-	mxr_dbg(mdev, "tvnorms = %08llx\n", vfd->tvnorms);
-
-	return 0;
-}
-
-static int mxr_g_output(struct file *file, void *fh, unsigned int *p)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-
-	mutex_lock(&mdev->mutex);
-	*p = mdev->current_output;
-	mutex_unlock(&mdev->mutex);
-
-	return 0;
-}
-
-static int mxr_reqbufs(struct file *file, void *priv,
-			  struct v4l2_requestbuffers *p)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-	return vb2_reqbufs(&layer->vb_queue, p);
-}
-
-static int mxr_querybuf(struct file *file, void *priv, struct v4l2_buffer *p)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-	return vb2_querybuf(&layer->vb_queue, p);
-}
-
-static int mxr_qbuf(struct file *file, void *priv, struct v4l2_buffer *p)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-
-	mxr_dbg(layer->mdev, "%s:%d(%d)\n", __func__, __LINE__, p->index);
-	return vb2_qbuf(&layer->vb_queue, p);
-}
-
-static int mxr_dqbuf(struct file *file, void *priv, struct v4l2_buffer *p)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-	return vb2_dqbuf(&layer->vb_queue, p, file->f_flags & O_NONBLOCK);
-}
-
-static int mxr_expbuf(struct file *file, void *priv,
-	struct v4l2_exportbuffer *eb)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-	return vb2_expbuf(&layer->vb_queue, eb);
-}
-
-static int mxr_streamon(struct file *file, void *priv, enum v4l2_buf_type i)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-	return vb2_streamon(&layer->vb_queue, i);
-}
-
-static int mxr_streamoff(struct file *file, void *priv, enum v4l2_buf_type i)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-	return vb2_streamoff(&layer->vb_queue, i);
-}
-
-static const struct v4l2_ioctl_ops mxr_ioctl_ops = {
-	.vidioc_querycap = mxr_querycap,
-	/* format handling */
-	.vidioc_enum_fmt_vid_out_mplane = mxr_enum_fmt,
-	.vidioc_s_fmt_vid_out_mplane = mxr_s_fmt,
-	.vidioc_g_fmt_vid_out_mplane = mxr_g_fmt,
-	/* buffer control */
-	.vidioc_reqbufs = mxr_reqbufs,
-	.vidioc_querybuf = mxr_querybuf,
-	.vidioc_qbuf = mxr_qbuf,
-	.vidioc_dqbuf = mxr_dqbuf,
-	.vidioc_expbuf = mxr_expbuf,
-	/* Streaming control */
-	.vidioc_streamon = mxr_streamon,
-	.vidioc_streamoff = mxr_streamoff,
-	/* DV Timings functions */
-	.vidioc_enum_dv_timings = mxr_enum_dv_timings,
-	.vidioc_s_dv_timings = mxr_s_dv_timings,
-	.vidioc_g_dv_timings = mxr_g_dv_timings,
-	.vidioc_dv_timings_cap = mxr_dv_timings_cap,
-	/* analog TV standard functions */
-	.vidioc_s_std = mxr_s_std,
-	.vidioc_g_std = mxr_g_std,
-	/* Output handling */
-	.vidioc_enum_output = mxr_enum_output,
-	.vidioc_s_output = mxr_s_output,
-	.vidioc_g_output = mxr_g_output,
-	/* selection ioctls */
-	.vidioc_g_selection = mxr_g_selection,
-	.vidioc_s_selection = mxr_s_selection,
-};
-
-static int mxr_video_open(struct file *file)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	struct mxr_device *mdev = layer->mdev;
-	int ret = 0;
-
-	mxr_dbg(mdev, "%s:%d\n", __func__, __LINE__);
-	if (mutex_lock_interruptible(&layer->mutex))
-		return -ERESTARTSYS;
-	/* assure device probe is finished */
-	wait_for_device_probe();
-	/* creating context for file descriptor */
-	ret = v4l2_fh_open(file);
-	if (ret) {
-		mxr_err(mdev, "v4l2_fh_open failed\n");
-		goto unlock;
-	}
-
-	/* leaving if layer is already initialized */
-	if (!v4l2_fh_is_singular_file(file))
-		goto unlock;
-
-	/* FIXME: should power be enabled on open? */
-	ret = mxr_power_get(mdev);
-	if (ret) {
-		mxr_err(mdev, "power on failed\n");
-		goto fail_fh_open;
-	}
-
-	ret = vb2_queue_init(&layer->vb_queue);
-	if (ret != 0) {
-		mxr_err(mdev, "failed to initialize vb2 queue\n");
-		goto fail_power;
-	}
-	/* set default format, first on the list */
-	layer->fmt = layer->fmt_array[0];
-	/* setup default geometry */
-	mxr_layer_default_geo(layer);
-	mutex_unlock(&layer->mutex);
-
-	return 0;
-
-fail_power:
-	mxr_power_put(mdev);
-
-fail_fh_open:
-	v4l2_fh_release(file);
-
-unlock:
-	mutex_unlock(&layer->mutex);
-
-	return ret;
-}
-
-static unsigned int
-mxr_video_poll(struct file *file, struct poll_table_struct *wait)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	unsigned int res;
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-
-	mutex_lock(&layer->mutex);
-	res = vb2_poll(&layer->vb_queue, file, wait);
-	mutex_unlock(&layer->mutex);
-	return res;
-}
-
-static int mxr_video_mmap(struct file *file, struct vm_area_struct *vma)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-	int ret;
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-
-	if (mutex_lock_interruptible(&layer->mutex))
-		return -ERESTARTSYS;
-	ret = vb2_mmap(&layer->vb_queue, vma);
-	mutex_unlock(&layer->mutex);
-	return ret;
-}
-
-static int mxr_video_release(struct file *file)
-{
-	struct mxr_layer *layer = video_drvdata(file);
-
-	mxr_dbg(layer->mdev, "%s:%d\n", __func__, __LINE__);
-	mutex_lock(&layer->mutex);
-	if (v4l2_fh_is_singular_file(file)) {
-		vb2_queue_release(&layer->vb_queue);
-		mxr_power_put(layer->mdev);
-	}
-	v4l2_fh_release(file);
-	mutex_unlock(&layer->mutex);
-	return 0;
-}
-
-static const struct v4l2_file_operations mxr_fops = {
-	.owner = THIS_MODULE,
-	.open = mxr_video_open,
-	.poll = mxr_video_poll,
-	.mmap = mxr_video_mmap,
-	.release = mxr_video_release,
-	.unlocked_ioctl = video_ioctl2,
-};
-
-static int queue_setup(struct vb2_queue *vq,
-	unsigned int *nbuffers, unsigned int *nplanes, unsigned int sizes[],
-	struct device *alloc_devs[])
-{
-	struct mxr_layer *layer = vb2_get_drv_priv(vq);
-	const struct mxr_format *fmt = layer->fmt;
-	int i;
-	struct mxr_device *mdev = layer->mdev;
-	struct v4l2_plane_pix_format planes[3];
-
-	mxr_dbg(mdev, "%s\n", __func__);
-	/* checking if format was configured */
-	if (fmt == NULL)
-		return -EINVAL;
-	mxr_dbg(mdev, "fmt = %s\n", fmt->name);
-	mxr_mplane_fill(planes, fmt, layer->geo.src.full_width,
-		layer->geo.src.full_height);
-
-	*nplanes = fmt->num_subframes;
-	for (i = 0; i < fmt->num_subframes; ++i) {
-		sizes[i] = planes[i].sizeimage;
-		mxr_dbg(mdev, "size[%d] = %08x\n", i, sizes[i]);
-	}
-
-	if (*nbuffers == 0)
-		*nbuffers = 1;
-
-	return 0;
-}
-
-static void buf_queue(struct vb2_buffer *vb)
-{
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct mxr_buffer *buffer = container_of(vbuf, struct mxr_buffer, vb);
-	struct mxr_layer *layer = vb2_get_drv_priv(vb->vb2_queue);
-	struct mxr_device *mdev = layer->mdev;
-	unsigned long flags;
-
-	spin_lock_irqsave(&layer->enq_slock, flags);
-	list_add_tail(&buffer->list, &layer->enq_list);
-	spin_unlock_irqrestore(&layer->enq_slock, flags);
-
-	mxr_dbg(mdev, "queuing buffer\n");
-}
-
-static int start_streaming(struct vb2_queue *vq, unsigned int count)
-{
-	struct mxr_layer *layer = vb2_get_drv_priv(vq);
-	struct mxr_device *mdev = layer->mdev;
-	unsigned long flags;
-
-	mxr_dbg(mdev, "%s\n", __func__);
-
-	/* block any changes in output configuration */
-	mxr_output_get(mdev);
-
-	mxr_layer_update_output(layer);
-	layer->ops.format_set(layer);
-	/* enabling layer in hardware */
-	spin_lock_irqsave(&layer->enq_slock, flags);
-	layer->state = MXR_LAYER_STREAMING;
-	spin_unlock_irqrestore(&layer->enq_slock, flags);
-
-	layer->ops.stream_set(layer, MXR_ENABLE);
-	mxr_streamer_get(mdev);
-
-	return 0;
-}
-
-static void mxr_watchdog(unsigned long arg)
-{
-	struct mxr_layer *layer = (struct mxr_layer *) arg;
-	struct mxr_device *mdev = layer->mdev;
-	unsigned long flags;
-
-	mxr_err(mdev, "watchdog fired for layer %s\n", layer->vfd.name);
-
-	spin_lock_irqsave(&layer->enq_slock, flags);
-
-	if (layer->update_buf == layer->shadow_buf)
-		layer->update_buf = NULL;
-	if (layer->update_buf) {
-		vb2_buffer_done(&layer->update_buf->vb.vb2_buf,
-				VB2_BUF_STATE_ERROR);
-		layer->update_buf = NULL;
-	}
-	if (layer->shadow_buf) {
-		vb2_buffer_done(&layer->shadow_buf->vb.vb2_buf,
-				VB2_BUF_STATE_ERROR);
-		layer->shadow_buf = NULL;
-	}
-	spin_unlock_irqrestore(&layer->enq_slock, flags);
-}
-
-static void stop_streaming(struct vb2_queue *vq)
-{
-	struct mxr_layer *layer = vb2_get_drv_priv(vq);
-	struct mxr_device *mdev = layer->mdev;
-	unsigned long flags;
-	struct timer_list watchdog;
-	struct mxr_buffer *buf, *buf_tmp;
-
-	mxr_dbg(mdev, "%s\n", __func__);
-
-	spin_lock_irqsave(&layer->enq_slock, flags);
-
-	/* reset list */
-	layer->state = MXR_LAYER_STREAMING_FINISH;
-
-	/* set all buffer to be done */
-	list_for_each_entry_safe(buf, buf_tmp, &layer->enq_list, list) {
-		list_del(&buf->list);
-		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
-	}
-
-	spin_unlock_irqrestore(&layer->enq_slock, flags);
-
-	/* give 1 seconds to complete to complete last buffers */
-	setup_timer_on_stack(&watchdog, mxr_watchdog,
-		(unsigned long)layer);
-	mod_timer(&watchdog, jiffies + msecs_to_jiffies(1000));
-
-	/* wait until all buffers are goes to done state */
-	vb2_wait_for_all_buffers(vq);
-
-	/* stop timer if all synchronization is done */
-	del_timer_sync(&watchdog);
-	destroy_timer_on_stack(&watchdog);
-
-	/* stopping hardware */
-	spin_lock_irqsave(&layer->enq_slock, flags);
-	layer->state = MXR_LAYER_IDLE;
-	spin_unlock_irqrestore(&layer->enq_slock, flags);
-
-	/* disabling layer in hardware */
-	layer->ops.stream_set(layer, MXR_DISABLE);
-	/* remove one streamer */
-	mxr_streamer_put(mdev);
-	/* allow changes in output configuration */
-	mxr_output_put(mdev);
-}
-
-static struct vb2_ops mxr_video_qops = {
-	.queue_setup = queue_setup,
-	.buf_queue = buf_queue,
-	.wait_prepare = vb2_ops_wait_prepare,
-	.wait_finish = vb2_ops_wait_finish,
-	.start_streaming = start_streaming,
-	.stop_streaming = stop_streaming,
-};
-
-/* FIXME: try to put this functions to mxr_base_layer_create */
-int mxr_base_layer_register(struct mxr_layer *layer)
-{
-	struct mxr_device *mdev = layer->mdev;
-	int ret;
-
-	ret = video_register_device(&layer->vfd, VFL_TYPE_GRABBER, -1);
-	if (ret)
-		mxr_err(mdev, "failed to register video device\n");
-	else
-		mxr_info(mdev, "registered layer %s as /dev/video%d\n",
-			layer->vfd.name, layer->vfd.num);
-	return ret;
-}
-
-void mxr_base_layer_unregister(struct mxr_layer *layer)
-{
-	video_unregister_device(&layer->vfd);
-}
-
-void mxr_layer_release(struct mxr_layer *layer)
-{
-	if (layer->ops.release)
-		layer->ops.release(layer);
-}
-
-void mxr_base_layer_release(struct mxr_layer *layer)
-{
-	kfree(layer);
-}
-
-static void mxr_vfd_release(struct video_device *vdev)
-{
-	pr_info("video device release\n");
-}
-
-struct mxr_layer *mxr_base_layer_create(struct mxr_device *mdev,
-	int idx, char *name, const struct mxr_layer_ops *ops)
-{
-	struct mxr_layer *layer;
-
-	layer = kzalloc(sizeof(*layer), GFP_KERNEL);
-	if (layer == NULL) {
-		mxr_err(mdev, "not enough memory for layer.\n");
-		goto fail;
-	}
-
-	layer->mdev = mdev;
-	layer->idx = idx;
-	layer->ops = *ops;
-
-	spin_lock_init(&layer->enq_slock);
-	INIT_LIST_HEAD(&layer->enq_list);
-	mutex_init(&layer->mutex);
-
-	layer->vfd = (struct video_device) {
-		.minor = -1,
-		.release = mxr_vfd_release,
-		.fops = &mxr_fops,
-		.vfl_dir = VFL_DIR_TX,
-		.ioctl_ops = &mxr_ioctl_ops,
-	};
-	strlcpy(layer->vfd.name, name, sizeof(layer->vfd.name));
-
-	video_set_drvdata(&layer->vfd, layer);
-	layer->vfd.lock = &layer->mutex;
-	layer->vfd.v4l2_dev = &mdev->v4l2_dev;
-
-	layer->vb_queue = (struct vb2_queue) {
-		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
-		.io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF,
-		.drv_priv = layer,
-		.buf_struct_size = sizeof(struct mxr_buffer),
-		.ops = &mxr_video_qops,
-		.min_buffers_needed = 1,
-		.mem_ops = &vb2_dma_contig_memops,
-		.lock = &layer->mutex,
-		.dev = mdev->dev,
-	};
-
-	return layer;
-
-fail:
-	return NULL;
-}
-
-static const struct mxr_format *find_format_by_fourcc(
-	struct mxr_layer *layer, unsigned long fourcc)
-{
-	int i;
-
-	for (i = 0; i < layer->fmt_array_size; ++i)
-		if (layer->fmt_array[i]->fourcc == fourcc)
-			return layer->fmt_array[i];
-	return NULL;
-}
-
-static const struct mxr_format *find_format_by_index(
-	struct mxr_layer *layer, unsigned long index)
-{
-	if (index >= layer->fmt_array_size)
-		return NULL;
-	return layer->fmt_array[index];
-}
-
diff --git a/drivers/media/platform/s5p-tv/mixer_vp_layer.c b/drivers/media/platform/s5p-tv/mixer_vp_layer.c
deleted file mode 100644
index 6fa6f673f53b..000000000000
--- a/drivers/media/platform/s5p-tv/mixer_vp_layer.c
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Samsung TV Mixer driver
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *
- * Tomasz Stanislawski, <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published
- * by the Free Software Foundiation. either version 2 of the License,
- * or (at your option) any later version
- */
-
-#include "mixer.h"
-
-#include "regs-vp.h"
-
-#include <media/videobuf2-dma-contig.h>
-
-/* FORMAT DEFINITIONS */
-static const struct mxr_format mxr_fmt_nv12 = {
-	.name = "NV12",
-	.fourcc = V4L2_PIX_FMT_NV12,
-	.colorspace = V4L2_COLORSPACE_JPEG,
-	.num_planes = 2,
-	.plane = {
-		{ .width = 1, .height = 1, .size = 1 },
-		{ .width = 2, .height = 2, .size = 2 },
-	},
-	.num_subframes = 1,
-	.cookie = VP_MODE_NV12 | VP_MODE_MEM_LINEAR,
-};
-
-static const struct mxr_format mxr_fmt_nv21 = {
-	.name = "NV21",
-	.fourcc = V4L2_PIX_FMT_NV21,
-	.colorspace = V4L2_COLORSPACE_JPEG,
-	.num_planes = 2,
-	.plane = {
-		{ .width = 1, .height = 1, .size = 1 },
-		{ .width = 2, .height = 2, .size = 2 },
-	},
-	.num_subframes = 1,
-	.cookie = VP_MODE_NV21 | VP_MODE_MEM_LINEAR,
-};
-
-static const struct mxr_format mxr_fmt_nv12m = {
-	.name = "NV12 (mplane)",
-	.fourcc = V4L2_PIX_FMT_NV12M,
-	.colorspace = V4L2_COLORSPACE_JPEG,
-	.num_planes = 2,
-	.plane = {
-		{ .width = 1, .height = 1, .size = 1 },
-		{ .width = 2, .height = 2, .size = 2 },
-	},
-	.num_subframes = 2,
-	.plane2subframe = {0, 1},
-	.cookie = VP_MODE_NV12 | VP_MODE_MEM_LINEAR,
-};
-
-static const struct mxr_format mxr_fmt_nv12mt = {
-	.name = "NV12 tiled (mplane)",
-	.fourcc = V4L2_PIX_FMT_NV12MT,
-	.colorspace = V4L2_COLORSPACE_JPEG,
-	.num_planes = 2,
-	.plane = {
-		{ .width = 128, .height = 32, .size = 4096 },
-		{ .width = 128, .height = 32, .size = 2048 },
-	},
-	.num_subframes = 2,
-	.plane2subframe = {0, 1},
-	.cookie = VP_MODE_NV12 | VP_MODE_MEM_TILED,
-};
-
-static const struct mxr_format *mxr_video_format[] = {
-	&mxr_fmt_nv12,
-	&mxr_fmt_nv21,
-	&mxr_fmt_nv12m,
-	&mxr_fmt_nv12mt,
-};
-
-/* AUXILIARY CALLBACKS */
-
-static void mxr_vp_layer_release(struct mxr_layer *layer)
-{
-	mxr_base_layer_unregister(layer);
-	mxr_base_layer_release(layer);
-}
-
-static void mxr_vp_buffer_set(struct mxr_layer *layer,
-	struct mxr_buffer *buf)
-{
-	dma_addr_t luma_addr[2] = {0, 0};
-	dma_addr_t chroma_addr[2] = {0, 0};
-
-	if (buf == NULL) {
-		mxr_reg_vp_buffer(layer->mdev, luma_addr, chroma_addr);
-		return;
-	}
-	luma_addr[0] = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
-	if (layer->fmt->num_subframes == 2) {
-		chroma_addr[0] =
-			vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 1);
-	} else {
-		/* FIXME: mxr_get_plane_size compute integer division,
-		 * which is slow and should not be performed in interrupt */
-		chroma_addr[0] = luma_addr[0] + mxr_get_plane_size(
-			&layer->fmt->plane[0], layer->geo.src.full_width,
-			layer->geo.src.full_height);
-	}
-	if (layer->fmt->cookie & VP_MODE_MEM_TILED) {
-		luma_addr[1] = luma_addr[0] + 0x40;
-		chroma_addr[1] = chroma_addr[0] + 0x40;
-	} else {
-		luma_addr[1] = luma_addr[0] + layer->geo.src.full_width;
-		chroma_addr[1] = chroma_addr[0];
-	}
-	mxr_reg_vp_buffer(layer->mdev, luma_addr, chroma_addr);
-}
-
-static void mxr_vp_stream_set(struct mxr_layer *layer, int en)
-{
-	mxr_reg_vp_layer_stream(layer->mdev, en);
-}
-
-static void mxr_vp_format_set(struct mxr_layer *layer)
-{
-	mxr_reg_vp_format(layer->mdev, layer->fmt, &layer->geo);
-}
-
-static inline unsigned int do_center(unsigned int center,
-	unsigned int size, unsigned int upper, unsigned int flags)
-{
-	unsigned int lower;
-
-	if (flags & MXR_NO_OFFSET)
-		return 0;
-
-	lower = center - min(center, size / 2);
-	return min(lower, upper - size);
-}
-
-static void mxr_vp_fix_geometry(struct mxr_layer *layer,
-	enum mxr_geometry_stage stage, unsigned long flags)
-{
-	struct mxr_geometry *geo = &layer->geo;
-	struct mxr_crop *src = &geo->src;
-	struct mxr_crop *dst = &geo->dst;
-	unsigned long x_center, y_center;
-
-	switch (stage) {
-
-	case MXR_GEOMETRY_SINK: /* nothing to be fixed here */
-	case MXR_GEOMETRY_COMPOSE:
-		/* remember center of the area */
-		x_center = dst->x_offset + dst->width / 2;
-		y_center = dst->y_offset + dst->height / 2;
-
-		/* ensure that compose is reachable using 16x scaling */
-		dst->width = clamp(dst->width, 8U, 16 * src->full_width);
-		dst->height = clamp(dst->height, 1U, 16 * src->full_height);
-
-		/* setup offsets */
-		dst->x_offset = do_center(x_center, dst->width,
-			dst->full_width, flags);
-		dst->y_offset = do_center(y_center, dst->height,
-			dst->full_height, flags);
-		flags = 0; /* remove possible MXR_NO_OFFSET flag */
-		/* fall through */
-	case MXR_GEOMETRY_CROP:
-		/* remember center of the area */
-		x_center = src->x_offset + src->width / 2;
-		y_center = src->y_offset + src->height / 2;
-
-		/* ensure scaling is between 0.25x .. 16x */
-		src->width = clamp(src->width, round_up(dst->width / 16, 4),
-			dst->width * 4);
-		src->height = clamp(src->height, round_up(dst->height / 16, 4),
-			dst->height * 4);
-
-		/* hardware limits */
-		src->width = clamp(src->width, 32U, 2047U);
-		src->height = clamp(src->height, 4U, 2047U);
-
-		/* setup offsets */
-		src->x_offset = do_center(x_center, src->width,
-			src->full_width, flags);
-		src->y_offset = do_center(y_center, src->height,
-			src->full_height, flags);
-
-		/* setting scaling ratio */
-		geo->x_ratio = (src->width << 16) / dst->width;
-		geo->y_ratio = (src->height << 16) / dst->height;
-		/* fall through */
-
-	case MXR_GEOMETRY_SOURCE:
-		src->full_width = clamp(src->full_width,
-			ALIGN(src->width + src->x_offset, 8), 8192U);
-		src->full_height = clamp(src->full_height,
-			src->height + src->y_offset, 8192U);
-	}
-}
-
-/* PUBLIC API */
-
-struct mxr_layer *mxr_vp_layer_create(struct mxr_device *mdev, int idx)
-{
-	struct mxr_layer *layer;
-	int ret;
-	const struct mxr_layer_ops ops = {
-		.release = mxr_vp_layer_release,
-		.buffer_set = mxr_vp_buffer_set,
-		.stream_set = mxr_vp_stream_set,
-		.format_set = mxr_vp_format_set,
-		.fix_geometry = mxr_vp_fix_geometry,
-	};
-	char name[32];
-
-	sprintf(name, "video%d", idx);
-
-	layer = mxr_base_layer_create(mdev, idx, name, &ops);
-	if (layer == NULL) {
-		mxr_err(mdev, "failed to initialize layer(%d) base\n", idx);
-		goto fail;
-	}
-
-	layer->fmt_array = mxr_video_format;
-	layer->fmt_array_size = ARRAY_SIZE(mxr_video_format);
-
-	ret = mxr_base_layer_register(layer);
-	if (ret)
-		goto fail_layer;
-
-	return layer;
-
-fail_layer:
-	mxr_base_layer_release(layer);
-
-fail:
-	return NULL;
-}
-
diff --git a/drivers/media/platform/s5p-tv/regs-hdmi.h b/drivers/media/platform/s5p-tv/regs-hdmi.h
deleted file mode 100644
index a889d1f57f28..000000000000
--- a/drivers/media/platform/s5p-tv/regs-hdmi.h
+++ /dev/null
@@ -1,146 +0,0 @@
-/* linux/arch/arm/mach-exynos4/include/mach/regs-hdmi.h
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- * http://www.samsung.com/
- *
- * HDMI register header file for Samsung TVOUT driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
-
-#ifndef SAMSUNG_REGS_HDMI_H
-#define SAMSUNG_REGS_HDMI_H
-
-/*
- * Register part
-*/
-
-#define HDMI_CTRL_BASE(x)		((x) + 0x00000000)
-#define HDMI_CORE_BASE(x)		((x) + 0x00010000)
-#define HDMI_TG_BASE(x)			((x) + 0x00050000)
-
-/* Control registers */
-#define HDMI_INTC_CON			HDMI_CTRL_BASE(0x0000)
-#define HDMI_INTC_FLAG			HDMI_CTRL_BASE(0x0004)
-#define HDMI_HPD_STATUS			HDMI_CTRL_BASE(0x000C)
-#define HDMI_PHY_RSTOUT			HDMI_CTRL_BASE(0x0014)
-#define HDMI_PHY_VPLL			HDMI_CTRL_BASE(0x0018)
-#define HDMI_PHY_CMU			HDMI_CTRL_BASE(0x001C)
-#define HDMI_CORE_RSTOUT		HDMI_CTRL_BASE(0x0020)
-
-/* Core registers */
-#define HDMI_CON_0			HDMI_CORE_BASE(0x0000)
-#define HDMI_CON_1			HDMI_CORE_BASE(0x0004)
-#define HDMI_CON_2			HDMI_CORE_BASE(0x0008)
-#define HDMI_SYS_STATUS			HDMI_CORE_BASE(0x0010)
-#define HDMI_PHY_STATUS			HDMI_CORE_BASE(0x0014)
-#define HDMI_STATUS_EN			HDMI_CORE_BASE(0x0020)
-#define HDMI_HPD			HDMI_CORE_BASE(0x0030)
-#define HDMI_MODE_SEL			HDMI_CORE_BASE(0x0040)
-#define HDMI_BLUE_SCREEN_0		HDMI_CORE_BASE(0x0050)
-#define HDMI_BLUE_SCREEN_1		HDMI_CORE_BASE(0x0054)
-#define HDMI_BLUE_SCREEN_2		HDMI_CORE_BASE(0x0058)
-#define HDMI_H_BLANK_0			HDMI_CORE_BASE(0x00A0)
-#define HDMI_H_BLANK_1			HDMI_CORE_BASE(0x00A4)
-#define HDMI_V_BLANK_0			HDMI_CORE_BASE(0x00B0)
-#define HDMI_V_BLANK_1			HDMI_CORE_BASE(0x00B4)
-#define HDMI_V_BLANK_2			HDMI_CORE_BASE(0x00B8)
-#define HDMI_H_V_LINE_0			HDMI_CORE_BASE(0x00C0)
-#define HDMI_H_V_LINE_1			HDMI_CORE_BASE(0x00C4)
-#define HDMI_H_V_LINE_2			HDMI_CORE_BASE(0x00C8)
-#define HDMI_VSYNC_POL			HDMI_CORE_BASE(0x00E4)
-#define HDMI_INT_PRO_MODE		HDMI_CORE_BASE(0x00E8)
-#define HDMI_V_BLANK_F_0		HDMI_CORE_BASE(0x0110)
-#define HDMI_V_BLANK_F_1		HDMI_CORE_BASE(0x0114)
-#define HDMI_V_BLANK_F_2		HDMI_CORE_BASE(0x0118)
-#define HDMI_H_SYNC_GEN_0		HDMI_CORE_BASE(0x0120)
-#define HDMI_H_SYNC_GEN_1		HDMI_CORE_BASE(0x0124)
-#define HDMI_H_SYNC_GEN_2		HDMI_CORE_BASE(0x0128)
-#define HDMI_V_SYNC_GEN_1_0		HDMI_CORE_BASE(0x0130)
-#define HDMI_V_SYNC_GEN_1_1		HDMI_CORE_BASE(0x0134)
-#define HDMI_V_SYNC_GEN_1_2		HDMI_CORE_BASE(0x0138)
-#define HDMI_V_SYNC_GEN_2_0		HDMI_CORE_BASE(0x0140)
-#define HDMI_V_SYNC_GEN_2_1		HDMI_CORE_BASE(0x0144)
-#define HDMI_V_SYNC_GEN_2_2		HDMI_CORE_BASE(0x0148)
-#define HDMI_V_SYNC_GEN_3_0		HDMI_CORE_BASE(0x0150)
-#define HDMI_V_SYNC_GEN_3_1		HDMI_CORE_BASE(0x0154)
-#define HDMI_V_SYNC_GEN_3_2		HDMI_CORE_BASE(0x0158)
-#define HDMI_AVI_CON			HDMI_CORE_BASE(0x0300)
-#define HDMI_AVI_BYTE(n)		HDMI_CORE_BASE(0x0320 + 4 * (n))
-#define	HDMI_DC_CONTROL			HDMI_CORE_BASE(0x05C0)
-#define HDMI_VIDEO_PATTERN_GEN		HDMI_CORE_BASE(0x05C4)
-#define HDMI_HPD_GEN			HDMI_CORE_BASE(0x05C8)
-
-/* Timing generator registers */
-#define HDMI_TG_CMD			HDMI_TG_BASE(0x0000)
-#define HDMI_TG_H_FSZ_L			HDMI_TG_BASE(0x0018)
-#define HDMI_TG_H_FSZ_H			HDMI_TG_BASE(0x001C)
-#define HDMI_TG_HACT_ST_L		HDMI_TG_BASE(0x0020)
-#define HDMI_TG_HACT_ST_H		HDMI_TG_BASE(0x0024)
-#define HDMI_TG_HACT_SZ_L		HDMI_TG_BASE(0x0028)
-#define HDMI_TG_HACT_SZ_H		HDMI_TG_BASE(0x002C)
-#define HDMI_TG_V_FSZ_L			HDMI_TG_BASE(0x0030)
-#define HDMI_TG_V_FSZ_H			HDMI_TG_BASE(0x0034)
-#define HDMI_TG_VSYNC_L			HDMI_TG_BASE(0x0038)
-#define HDMI_TG_VSYNC_H			HDMI_TG_BASE(0x003C)
-#define HDMI_TG_VSYNC2_L		HDMI_TG_BASE(0x0040)
-#define HDMI_TG_VSYNC2_H		HDMI_TG_BASE(0x0044)
-#define HDMI_TG_VACT_ST_L		HDMI_TG_BASE(0x0048)
-#define HDMI_TG_VACT_ST_H		HDMI_TG_BASE(0x004C)
-#define HDMI_TG_VACT_SZ_L		HDMI_TG_BASE(0x0050)
-#define HDMI_TG_VACT_SZ_H		HDMI_TG_BASE(0x0054)
-#define HDMI_TG_FIELD_CHG_L		HDMI_TG_BASE(0x0058)
-#define HDMI_TG_FIELD_CHG_H		HDMI_TG_BASE(0x005C)
-#define HDMI_TG_VACT_ST2_L		HDMI_TG_BASE(0x0060)
-#define HDMI_TG_VACT_ST2_H		HDMI_TG_BASE(0x0064)
-#define HDMI_TG_VSYNC_TOP_HDMI_L	HDMI_TG_BASE(0x0078)
-#define HDMI_TG_VSYNC_TOP_HDMI_H	HDMI_TG_BASE(0x007C)
-#define HDMI_TG_VSYNC_BOT_HDMI_L	HDMI_TG_BASE(0x0080)
-#define HDMI_TG_VSYNC_BOT_HDMI_H	HDMI_TG_BASE(0x0084)
-#define HDMI_TG_FIELD_TOP_HDMI_L	HDMI_TG_BASE(0x0088)
-#define HDMI_TG_FIELD_TOP_HDMI_H	HDMI_TG_BASE(0x008C)
-#define HDMI_TG_FIELD_BOT_HDMI_L	HDMI_TG_BASE(0x0090)
-#define HDMI_TG_FIELD_BOT_HDMI_H	HDMI_TG_BASE(0x0094)
-
-/*
- * Bit definition part
- */
-
-/* HDMI_INTC_CON */
-#define HDMI_INTC_EN_GLOBAL		(1 << 6)
-#define HDMI_INTC_EN_HPD_PLUG		(1 << 3)
-#define HDMI_INTC_EN_HPD_UNPLUG		(1 << 2)
-
-/* HDMI_INTC_FLAG */
-#define HDMI_INTC_FLAG_HPD_PLUG		(1 << 3)
-#define HDMI_INTC_FLAG_HPD_UNPLUG	(1 << 2)
-
-/* HDMI_PHY_RSTOUT */
-#define HDMI_PHY_SW_RSTOUT		(1 << 0)
-
-/* HDMI_CORE_RSTOUT */
-#define HDMI_CORE_SW_RSTOUT		(1 << 0)
-
-/* HDMI_CON_0 */
-#define HDMI_BLUE_SCR_EN		(1 << 5)
-#define HDMI_EN				(1 << 0)
-
-/* HDMI_CON_2 */
-#define HDMI_DVI_PERAMBLE_EN		(1 << 5)
-#define HDMI_DVI_BAND_EN		(1 << 1)
-
-/* HDMI_PHY_STATUS */
-#define HDMI_PHY_STATUS_READY		(1 << 0)
-
-/* HDMI_MODE_SEL */
-#define HDMI_MODE_HDMI_EN		(1 << 1)
-#define HDMI_MODE_DVI_EN		(1 << 0)
-#define HDMI_MODE_MASK			(3 << 0)
-
-/* HDMI_TG_CMD */
-#define HDMI_TG_FIELD_EN		(1 << 1)
-#define HDMI_TG_EN			(1 << 0)
-
-#endif /* SAMSUNG_REGS_HDMI_H */
diff --git a/drivers/media/platform/s5p-tv/regs-mixer.h b/drivers/media/platform/s5p-tv/regs-mixer.h
deleted file mode 100644
index 158abb43d0a4..000000000000
--- a/drivers/media/platform/s5p-tv/regs-mixer.h
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- * http://www.samsung.com/
- *
- * Mixer register header file for Samsung Mixer driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
-#ifndef SAMSUNG_REGS_MIXER_H
-#define SAMSUNG_REGS_MIXER_H
-
-/*
- * Register part
- */
-#define MXR_STATUS			0x0000
-#define MXR_CFG				0x0004
-#define MXR_INT_EN			0x0008
-#define MXR_INT_STATUS			0x000C
-#define MXR_LAYER_CFG			0x0010
-#define MXR_VIDEO_CFG			0x0014
-#define MXR_GRAPHIC0_CFG		0x0020
-#define MXR_GRAPHIC0_BASE		0x0024
-#define MXR_GRAPHIC0_SPAN		0x0028
-#define MXR_GRAPHIC0_SXY		0x002C
-#define MXR_GRAPHIC0_WH			0x0030
-#define MXR_GRAPHIC0_DXY		0x0034
-#define MXR_GRAPHIC0_BLANK		0x0038
-#define MXR_GRAPHIC1_CFG		0x0040
-#define MXR_GRAPHIC1_BASE		0x0044
-#define MXR_GRAPHIC1_SPAN		0x0048
-#define MXR_GRAPHIC1_SXY		0x004C
-#define MXR_GRAPHIC1_WH			0x0050
-#define MXR_GRAPHIC1_DXY		0x0054
-#define MXR_GRAPHIC1_BLANK		0x0058
-#define MXR_BG_CFG			0x0060
-#define MXR_BG_COLOR0			0x0064
-#define MXR_BG_COLOR1			0x0068
-#define MXR_BG_COLOR2			0x006C
-
-/* for parametrized access to layer registers */
-#define MXR_GRAPHIC_CFG(i)		(0x0020 + (i) * 0x20)
-#define MXR_GRAPHIC_BASE(i)		(0x0024 + (i) * 0x20)
-#define MXR_GRAPHIC_SPAN(i)		(0x0028 + (i) * 0x20)
-#define MXR_GRAPHIC_SXY(i)		(0x002C + (i) * 0x20)
-#define MXR_GRAPHIC_WH(i)		(0x0030 + (i) * 0x20)
-#define MXR_GRAPHIC_DXY(i)		(0x0034 + (i) * 0x20)
-
-/*
- * Bit definition part
- */
-
-/* generates mask for range of bits */
-#define MXR_MASK(high_bit, low_bit) \
-	(((2 << ((high_bit) - (low_bit))) - 1) << (low_bit))
-
-#define MXR_MASK_VAL(val, high_bit, low_bit) \
-	(((val) << (low_bit)) & MXR_MASK(high_bit, low_bit))
-
-/* bits for MXR_STATUS */
-#define MXR_STATUS_16_BURST		(1 << 7)
-#define MXR_STATUS_BURST_MASK		(1 << 7)
-#define MXR_STATUS_SYNC_ENABLE		(1 << 2)
-#define MXR_STATUS_REG_RUN		(1 << 0)
-
-/* bits for MXR_CFG */
-#define MXR_CFG_OUT_YUV444		(0 << 8)
-#define MXR_CFG_OUT_RGB888		(1 << 8)
-#define MXR_CFG_OUT_MASK		(1 << 8)
-#define MXR_CFG_DST_SDO			(0 << 7)
-#define MXR_CFG_DST_HDMI		(1 << 7)
-#define MXR_CFG_DST_MASK		(1 << 7)
-#define MXR_CFG_SCAN_HD_720		(0 << 6)
-#define MXR_CFG_SCAN_HD_1080		(1 << 6)
-#define MXR_CFG_GRP1_ENABLE		(1 << 5)
-#define MXR_CFG_GRP0_ENABLE		(1 << 4)
-#define MXR_CFG_VP_ENABLE		(1 << 3)
-#define MXR_CFG_SCAN_INTERLACE		(0 << 2)
-#define MXR_CFG_SCAN_PROGRASSIVE	(1 << 2)
-#define MXR_CFG_SCAN_NTSC		(0 << 1)
-#define MXR_CFG_SCAN_PAL		(1 << 1)
-#define MXR_CFG_SCAN_SD			(0 << 0)
-#define MXR_CFG_SCAN_HD			(1 << 0)
-#define MXR_CFG_SCAN_MASK		0x47
-
-/* bits for MXR_GRAPHICn_CFG */
-#define MXR_GRP_CFG_COLOR_KEY_DISABLE	(1 << 21)
-#define MXR_GRP_CFG_BLEND_PRE_MUL	(1 << 20)
-#define MXR_GRP_CFG_FORMAT_VAL(x)	MXR_MASK_VAL(x, 11, 8)
-#define MXR_GRP_CFG_FORMAT_MASK		MXR_GRP_CFG_FORMAT_VAL(~0)
-#define MXR_GRP_CFG_ALPHA_VAL(x)	MXR_MASK_VAL(x, 7, 0)
-
-/* bits for MXR_GRAPHICn_WH */
-#define MXR_GRP_WH_H_SCALE(x)		MXR_MASK_VAL(x, 28, 28)
-#define MXR_GRP_WH_V_SCALE(x)		MXR_MASK_VAL(x, 12, 12)
-#define MXR_GRP_WH_WIDTH(x)		MXR_MASK_VAL(x, 26, 16)
-#define MXR_GRP_WH_HEIGHT(x)		MXR_MASK_VAL(x, 10, 0)
-
-/* bits for MXR_GRAPHICn_SXY */
-#define MXR_GRP_SXY_SX(x)		MXR_MASK_VAL(x, 26, 16)
-#define MXR_GRP_SXY_SY(x)		MXR_MASK_VAL(x, 10, 0)
-
-/* bits for MXR_GRAPHICn_DXY */
-#define MXR_GRP_DXY_DX(x)		MXR_MASK_VAL(x, 26, 16)
-#define MXR_GRP_DXY_DY(x)		MXR_MASK_VAL(x, 10, 0)
-
-/* bits for MXR_INT_EN */
-#define MXR_INT_EN_VSYNC		(1 << 11)
-#define MXR_INT_EN_ALL			(0x0f << 8)
-
-/* bit for MXR_INT_STATUS */
-#define MXR_INT_CLEAR_VSYNC		(1 << 11)
-#define MXR_INT_STATUS_VSYNC		(1 << 0)
-
-/* bit for MXR_LAYER_CFG */
-#define MXR_LAYER_CFG_GRP1_VAL(x)	MXR_MASK_VAL(x, 11, 8)
-#define MXR_LAYER_CFG_GRP0_VAL(x)	MXR_MASK_VAL(x, 7, 4)
-#define MXR_LAYER_CFG_VP_VAL(x)		MXR_MASK_VAL(x, 3, 0)
-
-#endif /* SAMSUNG_REGS_MIXER_H */
-
diff --git a/drivers/media/platform/s5p-tv/regs-sdo.h b/drivers/media/platform/s5p-tv/regs-sdo.h
deleted file mode 100644
index 6f22fbfe2f6c..000000000000
--- a/drivers/media/platform/s5p-tv/regs-sdo.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/* drivers/media/platform/s5p-tv/regs-sdo.h
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com/
- *
- * SDO register description file
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef SAMSUNG_REGS_SDO_H
-#define SAMSUNG_REGS_SDO_H
-
-/*
- * Register part
- */
-
-#define SDO_CLKCON			0x0000
-#define SDO_CONFIG			0x0008
-#define SDO_VBI				0x0014
-#define SDO_DAC				0x003C
-#define SDO_CCCON			0x0180
-#define SDO_IRQ				0x0280
-#define SDO_IRQMASK			0x0284
-#define SDO_VERSION			0x03D8
-
-/*
- * Bit definition part
- */
-
-/* SDO Clock Control Register (SDO_CLKCON) */
-#define SDO_TVOUT_SW_RESET		(1 << 4)
-#define SDO_TVOUT_CLOCK_READY		(1 << 1)
-#define SDO_TVOUT_CLOCK_ON		(1 << 0)
-
-/* SDO Video Standard Configuration Register (SDO_CONFIG) */
-#define SDO_PROGRESSIVE			(1 << 4)
-#define SDO_NTSC_M			0
-#define SDO_PAL_M			1
-#define SDO_PAL_BGHID			2
-#define SDO_PAL_N			3
-#define SDO_PAL_NC			4
-#define SDO_NTSC_443			8
-#define SDO_PAL_60			9
-#define SDO_STANDARD_MASK		0xf
-
-/* SDO VBI Configuration Register (SDO_VBI) */
-#define SDO_CVBS_WSS_INS		(1 << 14)
-#define SDO_CVBS_CLOSED_CAPTION_MASK	(3 << 12)
-
-/* SDO DAC Configuration Register (SDO_DAC) */
-#define SDO_POWER_ON_DAC		(1 << 0)
-
-/* SDO Color Compensation On/Off Control (SDO_CCCON) */
-#define SDO_COMPENSATION_BHS_ADJ_OFF	(1 << 4)
-#define SDO_COMPENSATION_CVBS_COMP_OFF	(1 << 0)
-
-/* SDO Interrupt Request Register (SDO_IRQ) */
-#define SDO_VSYNC_IRQ_PEND		(1 << 0)
-
-#endif /* SAMSUNG_REGS_SDO_H */
diff --git a/drivers/media/platform/s5p-tv/regs-vp.h b/drivers/media/platform/s5p-tv/regs-vp.h
deleted file mode 100644
index 6c63984e11e8..000000000000
--- a/drivers/media/platform/s5p-tv/regs-vp.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com/
- *
- * Video processor register header file for Samsung Mixer driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef SAMSUNG_REGS_VP_H
-#define SAMSUNG_REGS_VP_H
-
-/*
- * Register part
- */
-
-#define VP_ENABLE			0x0000
-#define VP_SRESET			0x0004
-#define VP_SHADOW_UPDATE		0x0008
-#define VP_FIELD_ID			0x000C
-#define VP_MODE				0x0010
-#define VP_IMG_SIZE_Y			0x0014
-#define VP_IMG_SIZE_C			0x0018
-#define VP_PER_RATE_CTRL		0x001C
-#define VP_TOP_Y_PTR			0x0028
-#define VP_BOT_Y_PTR			0x002C
-#define VP_TOP_C_PTR			0x0030
-#define VP_BOT_C_PTR			0x0034
-#define VP_ENDIAN_MODE			0x03CC
-#define VP_SRC_H_POSITION		0x0044
-#define VP_SRC_V_POSITION		0x0048
-#define VP_SRC_WIDTH			0x004C
-#define VP_SRC_HEIGHT			0x0050
-#define VP_DST_H_POSITION		0x0054
-#define VP_DST_V_POSITION		0x0058
-#define VP_DST_WIDTH			0x005C
-#define VP_DST_HEIGHT			0x0060
-#define VP_H_RATIO			0x0064
-#define VP_V_RATIO			0x0068
-#define VP_POLY8_Y0_LL			0x006C
-#define VP_POLY4_Y0_LL			0x00EC
-#define VP_POLY4_C0_LL			0x012C
-
-/*
- * Bit definition part
- */
-
-/* generates mask for range of bits */
-
-#define VP_MASK(high_bit, low_bit) \
-	(((2 << ((high_bit) - (low_bit))) - 1) << (low_bit))
-
-#define VP_MASK_VAL(val, high_bit, low_bit) \
-	(((val) << (low_bit)) & VP_MASK(high_bit, low_bit))
-
- /* VP_ENABLE */
-#define VP_ENABLE_ON			(1 << 0)
-
-/* VP_SRESET */
-#define VP_SRESET_PROCESSING		(1 << 0)
-
-/* VP_SHADOW_UPDATE */
-#define VP_SHADOW_UPDATE_ENABLE		(1 << 0)
-
-/* VP_MODE */
-#define VP_MODE_NV12			(0 << 6)
-#define VP_MODE_NV21			(1 << 6)
-#define VP_MODE_LINE_SKIP		(1 << 5)
-#define VP_MODE_MEM_LINEAR		(0 << 4)
-#define VP_MODE_MEM_TILED		(1 << 4)
-#define VP_MODE_FMT_MASK		(5 << 4)
-#define VP_MODE_FIELD_ID_AUTO_TOGGLING	(1 << 2)
-#define VP_MODE_2D_IPC			(1 << 1)
-
-/* VP_IMG_SIZE_Y */
-/* VP_IMG_SIZE_C */
-#define VP_IMG_HSIZE(x)			VP_MASK_VAL(x, 29, 16)
-#define VP_IMG_VSIZE(x)			VP_MASK_VAL(x, 13, 0)
-
-/* VP_SRC_H_POSITION */
-#define VP_SRC_H_POSITION_VAL(x)	VP_MASK_VAL(x, 14, 4)
-
-/* VP_ENDIAN_MODE */
-#define VP_ENDIAN_MODE_LITTLE		(1 << 0)
-
-#endif /* SAMSUNG_REGS_VP_H */
diff --git a/drivers/media/platform/s5p-tv/sdo_drv.c b/drivers/media/platform/s5p-tv/sdo_drv.c
deleted file mode 100644
index c75d4354d182..000000000000
--- a/drivers/media/platform/s5p-tv/sdo_drv.c
+++ /dev/null
@@ -1,497 +0,0 @@
-/*
- * Samsung Standard Definition Output (SDO) driver
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *
- * Tomasz Stanislawski, <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published
- * by the Free Software Foundiation. either version 2 of the License,
- * or (at your option) any later version
- */
-
-#include <linux/clk.h>
-#include <linux/delay.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/irq.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/regulator/consumer.h>
-#include <linux/slab.h>
-
-#include <media/v4l2-subdev.h>
-
-#include "regs-sdo.h"
-
-MODULE_AUTHOR("Tomasz Stanislawski, <t.stanislaws@samsung.com>");
-MODULE_DESCRIPTION("Samsung Standard Definition Output (SDO)");
-MODULE_LICENSE("GPL");
-
-#define SDO_DEFAULT_STD	V4L2_STD_PAL
-
-struct sdo_format {
-	v4l2_std_id id;
-	/* all modes are 720 pixels wide */
-	unsigned int height;
-	unsigned int cookie;
-};
-
-struct sdo_device {
-	/** pointer to device parent */
-	struct device *dev;
-	/** base address of SDO registers */
-	void __iomem *regs;
-	/** SDO interrupt */
-	unsigned int irq;
-	/** DAC source clock */
-	struct clk *sclk_dac;
-	/** DAC clock */
-	struct clk *dac;
-	/** DAC physical interface */
-	struct clk *dacphy;
-	/** clock for control of VPLL */
-	struct clk *fout_vpll;
-	/** vpll rate before sdo stream was on */
-	unsigned long vpll_rate;
-	/** regulator for SDO IP power */
-	struct regulator *vdac;
-	/** regulator for SDO plug detection */
-	struct regulator *vdet;
-	/** subdev used as device interface */
-	struct v4l2_subdev sd;
-	/** current format */
-	const struct sdo_format *fmt;
-};
-
-static inline struct sdo_device *sd_to_sdev(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct sdo_device, sd);
-}
-
-static inline
-void sdo_write_mask(struct sdo_device *sdev, u32 reg_id, u32 value, u32 mask)
-{
-	u32 old = readl(sdev->regs + reg_id);
-	value = (value & mask) | (old & ~mask);
-	writel(value, sdev->regs + reg_id);
-}
-
-static inline
-void sdo_write(struct sdo_device *sdev, u32 reg_id, u32 value)
-{
-	writel(value, sdev->regs + reg_id);
-}
-
-static inline
-u32 sdo_read(struct sdo_device *sdev, u32 reg_id)
-{
-	return readl(sdev->regs + reg_id);
-}
-
-static irqreturn_t sdo_irq_handler(int irq, void *dev_data)
-{
-	struct sdo_device *sdev = dev_data;
-
-	/* clear interrupt */
-	sdo_write_mask(sdev, SDO_IRQ, ~0, SDO_VSYNC_IRQ_PEND);
-	return IRQ_HANDLED;
-}
-
-static void sdo_reg_debug(struct sdo_device *sdev)
-{
-#define DBGREG(reg_id) \
-	dev_info(sdev->dev, #reg_id " = %08x\n", \
-		sdo_read(sdev, reg_id))
-
-	DBGREG(SDO_CLKCON);
-	DBGREG(SDO_CONFIG);
-	DBGREG(SDO_VBI);
-	DBGREG(SDO_DAC);
-	DBGREG(SDO_IRQ);
-	DBGREG(SDO_IRQMASK);
-	DBGREG(SDO_VERSION);
-}
-
-static const struct sdo_format sdo_format[] = {
-	{ V4L2_STD_PAL_N,	.height = 576, .cookie = SDO_PAL_N },
-	{ V4L2_STD_PAL_Nc,	.height = 576, .cookie = SDO_PAL_NC },
-	{ V4L2_STD_PAL_M,	.height = 480, .cookie = SDO_PAL_M },
-	{ V4L2_STD_PAL_60,	.height = 480, .cookie = SDO_PAL_60 },
-	{ V4L2_STD_NTSC_443,	.height = 480, .cookie = SDO_NTSC_443 },
-	{ V4L2_STD_PAL,		.height = 576, .cookie = SDO_PAL_BGHID },
-	{ V4L2_STD_NTSC_M,	.height = 480, .cookie = SDO_NTSC_M },
-};
-
-static const struct sdo_format *sdo_find_format(v4l2_std_id id)
-{
-	int i;
-	for (i = 0; i < ARRAY_SIZE(sdo_format); ++i)
-		if (sdo_format[i].id & id)
-			return &sdo_format[i];
-	return NULL;
-}
-
-static int sdo_g_tvnorms_output(struct v4l2_subdev *sd, v4l2_std_id *std)
-{
-	*std = V4L2_STD_NTSC_M | V4L2_STD_PAL_M | V4L2_STD_PAL |
-		V4L2_STD_PAL_N | V4L2_STD_PAL_Nc |
-		V4L2_STD_NTSC_443 | V4L2_STD_PAL_60;
-	return 0;
-}
-
-static int sdo_s_std_output(struct v4l2_subdev *sd, v4l2_std_id std)
-{
-	struct sdo_device *sdev = sd_to_sdev(sd);
-	const struct sdo_format *fmt;
-	fmt = sdo_find_format(std);
-	if (fmt == NULL)
-		return -EINVAL;
-	sdev->fmt = fmt;
-	return 0;
-}
-
-static int sdo_g_std_output(struct v4l2_subdev *sd, v4l2_std_id *std)
-{
-	*std = sd_to_sdev(sd)->fmt->id;
-	return 0;
-}
-
-static int sdo_get_fmt(struct v4l2_subdev *sd,
-		struct v4l2_subdev_pad_config *cfg,
-		struct v4l2_subdev_format *format)
-{
-	struct v4l2_mbus_framefmt *fmt = &format->format;
-	struct sdo_device *sdev = sd_to_sdev(sd);
-
-	if (!sdev->fmt)
-		return -ENXIO;
-	if (format->pad)
-		return -EINVAL;
-	/* all modes are 720 pixels wide */
-	fmt->width = 720;
-	fmt->height = sdev->fmt->height;
-	fmt->code = MEDIA_BUS_FMT_FIXED;
-	fmt->field = V4L2_FIELD_INTERLACED;
-	fmt->colorspace = V4L2_COLORSPACE_JPEG;
-	return 0;
-}
-
-static int sdo_s_power(struct v4l2_subdev *sd, int on)
-{
-	struct sdo_device *sdev = sd_to_sdev(sd);
-	struct device *dev = sdev->dev;
-	int ret;
-
-	dev_info(dev, "sdo_s_power(%d)\n", on);
-
-	if (on)
-		ret = pm_runtime_get_sync(dev);
-	else
-		ret = pm_runtime_put_sync(dev);
-
-	/* only values < 0 indicate errors */
-	return ret < 0 ? ret : 0;
-}
-
-static int sdo_streamon(struct sdo_device *sdev)
-{
-	int ret;
-
-	/* set proper clock for Timing Generator */
-	sdev->vpll_rate = clk_get_rate(sdev->fout_vpll);
-	ret = clk_set_rate(sdev->fout_vpll, 54000000);
-	if (ret < 0) {
-		dev_err(sdev->dev, "Failed to set vpll rate\n");
-		return ret;
-	}
-	dev_info(sdev->dev, "fout_vpll.rate = %lu\n",
-	clk_get_rate(sdev->fout_vpll));
-	/* enable clock in SDO */
-	sdo_write_mask(sdev, SDO_CLKCON, ~0, SDO_TVOUT_CLOCK_ON);
-	ret = clk_prepare_enable(sdev->dacphy);
-	if (ret < 0) {
-		dev_err(sdev->dev, "clk_prepare_enable(dacphy) failed\n");
-		goto fail;
-	}
-	/* enable DAC */
-	sdo_write_mask(sdev, SDO_DAC, ~0, SDO_POWER_ON_DAC);
-	sdo_reg_debug(sdev);
-	return 0;
-
-fail:
-	sdo_write_mask(sdev, SDO_CLKCON, 0, SDO_TVOUT_CLOCK_ON);
-	clk_set_rate(sdev->fout_vpll, sdev->vpll_rate);
-	return ret;
-}
-
-static int sdo_streamoff(struct sdo_device *sdev)
-{
-	int tries;
-
-	sdo_write_mask(sdev, SDO_DAC, 0, SDO_POWER_ON_DAC);
-	clk_disable_unprepare(sdev->dacphy);
-	sdo_write_mask(sdev, SDO_CLKCON, 0, SDO_TVOUT_CLOCK_ON);
-	for (tries = 100; tries; --tries) {
-		if (sdo_read(sdev, SDO_CLKCON) & SDO_TVOUT_CLOCK_READY)
-			break;
-		mdelay(1);
-	}
-	if (tries == 0)
-		dev_err(sdev->dev, "failed to stop streaming\n");
-	clk_set_rate(sdev->fout_vpll, sdev->vpll_rate);
-	return tries ? 0 : -EIO;
-}
-
-static int sdo_s_stream(struct v4l2_subdev *sd, int on)
-{
-	struct sdo_device *sdev = sd_to_sdev(sd);
-	return on ? sdo_streamon(sdev) : sdo_streamoff(sdev);
-}
-
-static const struct v4l2_subdev_core_ops sdo_sd_core_ops = {
-	.s_power = sdo_s_power,
-};
-
-static const struct v4l2_subdev_video_ops sdo_sd_video_ops = {
-	.s_std_output = sdo_s_std_output,
-	.g_std_output = sdo_g_std_output,
-	.g_tvnorms_output = sdo_g_tvnorms_output,
-	.s_stream = sdo_s_stream,
-};
-
-static const struct v4l2_subdev_pad_ops sdo_sd_pad_ops = {
-	.get_fmt = sdo_get_fmt,
-};
-
-static const struct v4l2_subdev_ops sdo_sd_ops = {
-	.core = &sdo_sd_core_ops,
-	.video = &sdo_sd_video_ops,
-	.pad = &sdo_sd_pad_ops,
-};
-
-static int sdo_runtime_suspend(struct device *dev)
-{
-	struct v4l2_subdev *sd = dev_get_drvdata(dev);
-	struct sdo_device *sdev = sd_to_sdev(sd);
-
-	dev_info(dev, "suspend\n");
-	regulator_disable(sdev->vdet);
-	regulator_disable(sdev->vdac);
-	clk_disable_unprepare(sdev->sclk_dac);
-	return 0;
-}
-
-static int sdo_runtime_resume(struct device *dev)
-{
-	struct v4l2_subdev *sd = dev_get_drvdata(dev);
-	struct sdo_device *sdev = sd_to_sdev(sd);
-	int ret;
-
-	dev_info(dev, "resume\n");
-
-	ret = clk_prepare_enable(sdev->sclk_dac);
-	if (ret < 0)
-		return ret;
-
-	ret = regulator_enable(sdev->vdac);
-	if (ret < 0)
-		goto dac_clk_dis;
-
-	ret = regulator_enable(sdev->vdet);
-	if (ret < 0)
-		goto vdac_r_dis;
-
-	/* software reset */
-	sdo_write_mask(sdev, SDO_CLKCON, ~0, SDO_TVOUT_SW_RESET);
-	mdelay(10);
-	sdo_write_mask(sdev, SDO_CLKCON, 0, SDO_TVOUT_SW_RESET);
-
-	/* setting TV mode */
-	sdo_write_mask(sdev, SDO_CONFIG, sdev->fmt->cookie, SDO_STANDARD_MASK);
-	/* XXX: forcing interlaced mode using undocumented bit */
-	sdo_write_mask(sdev, SDO_CONFIG, 0, SDO_PROGRESSIVE);
-	/* turn all VBI off */
-	sdo_write_mask(sdev, SDO_VBI, 0, SDO_CVBS_WSS_INS |
-		SDO_CVBS_CLOSED_CAPTION_MASK);
-	/* turn all post processing off */
-	sdo_write_mask(sdev, SDO_CCCON, ~0, SDO_COMPENSATION_BHS_ADJ_OFF |
-		SDO_COMPENSATION_CVBS_COMP_OFF);
-	sdo_reg_debug(sdev);
-	return 0;
-
-vdac_r_dis:
-	regulator_disable(sdev->vdac);
-dac_clk_dis:
-	clk_disable_unprepare(sdev->sclk_dac);
-	return ret;
-}
-
-static const struct dev_pm_ops sdo_pm_ops = {
-	.runtime_suspend = sdo_runtime_suspend,
-	.runtime_resume	 = sdo_runtime_resume,
-};
-
-static int sdo_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct sdo_device *sdev;
-	struct resource *res;
-	int ret = 0;
-	struct clk *sclk_vpll;
-
-	dev_info(dev, "probe start\n");
-	sdev = devm_kzalloc(&pdev->dev, sizeof(*sdev), GFP_KERNEL);
-	if (!sdev) {
-		dev_err(dev, "not enough memory.\n");
-		ret = -ENOMEM;
-		goto fail;
-	}
-	sdev->dev = dev;
-
-	/* mapping registers */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (res == NULL) {
-		dev_err(dev, "get memory resource failed.\n");
-		ret = -ENXIO;
-		goto fail;
-	}
-
-	sdev->regs = devm_ioremap(&pdev->dev, res->start, resource_size(res));
-	if (sdev->regs == NULL) {
-		dev_err(dev, "register mapping failed.\n");
-		ret = -ENXIO;
-		goto fail;
-	}
-
-	/* acquiring interrupt */
-	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (res == NULL) {
-		dev_err(dev, "get interrupt resource failed.\n");
-		ret = -ENXIO;
-		goto fail;
-	}
-	ret = devm_request_irq(&pdev->dev, res->start, sdo_irq_handler, 0,
-			       "s5p-sdo", sdev);
-	if (ret) {
-		dev_err(dev, "request interrupt failed.\n");
-		goto fail;
-	}
-	sdev->irq = res->start;
-
-	/* acquire clocks */
-	sdev->sclk_dac = clk_get(dev, "sclk_dac");
-	if (IS_ERR(sdev->sclk_dac)) {
-		dev_err(dev, "failed to get clock 'sclk_dac'\n");
-		ret = PTR_ERR(sdev->sclk_dac);
-		goto fail;
-	}
-	sdev->dac = clk_get(dev, "dac");
-	if (IS_ERR(sdev->dac)) {
-		dev_err(dev, "failed to get clock 'dac'\n");
-		ret = PTR_ERR(sdev->dac);
-		goto fail_sclk_dac;
-	}
-	sdev->dacphy = clk_get(dev, "dacphy");
-	if (IS_ERR(sdev->dacphy)) {
-		dev_err(dev, "failed to get clock 'dacphy'\n");
-		ret = PTR_ERR(sdev->dacphy);
-		goto fail_dac;
-	}
-	sclk_vpll = clk_get(dev, "sclk_vpll");
-	if (IS_ERR(sclk_vpll)) {
-		dev_err(dev, "failed to get clock 'sclk_vpll'\n");
-		ret = PTR_ERR(sclk_vpll);
-		goto fail_dacphy;
-	}
-	clk_set_parent(sdev->sclk_dac, sclk_vpll);
-	clk_put(sclk_vpll);
-	sdev->fout_vpll = clk_get(dev, "fout_vpll");
-	if (IS_ERR(sdev->fout_vpll)) {
-		dev_err(dev, "failed to get clock 'fout_vpll'\n");
-		ret = PTR_ERR(sdev->fout_vpll);
-		goto fail_dacphy;
-	}
-	dev_info(dev, "fout_vpll.rate = %lu\n", clk_get_rate(sclk_vpll));
-
-	/* acquire regulator */
-	sdev->vdac = devm_regulator_get(dev, "vdd33a_dac");
-	if (IS_ERR(sdev->vdac)) {
-		dev_err(dev, "failed to get regulator 'vdac'\n");
-		ret = PTR_ERR(sdev->vdac);
-		goto fail_fout_vpll;
-	}
-	sdev->vdet = devm_regulator_get(dev, "vdet");
-	if (IS_ERR(sdev->vdet)) {
-		dev_err(dev, "failed to get regulator 'vdet'\n");
-		ret = PTR_ERR(sdev->vdet);
-		goto fail_fout_vpll;
-	}
-
-	/* enable gate for dac clock, because mixer uses it */
-	ret = clk_prepare_enable(sdev->dac);
-	if (ret < 0) {
-		dev_err(dev, "clk_prepare_enable(dac) failed\n");
-		goto fail_fout_vpll;
-	}
-
-	/* configure power management */
-	pm_runtime_enable(dev);
-
-	/* configuration of interface subdevice */
-	v4l2_subdev_init(&sdev->sd, &sdo_sd_ops);
-	sdev->sd.owner = THIS_MODULE;
-	strlcpy(sdev->sd.name, "s5p-sdo", sizeof(sdev->sd.name));
-
-	/* set default format */
-	sdev->fmt = sdo_find_format(SDO_DEFAULT_STD);
-	BUG_ON(sdev->fmt == NULL);
-
-	/* keeping subdev in device's private for use by other drivers */
-	dev_set_drvdata(dev, &sdev->sd);
-
-	dev_info(dev, "probe succeeded\n");
-	return 0;
-
-fail_fout_vpll:
-	clk_put(sdev->fout_vpll);
-fail_dacphy:
-	clk_put(sdev->dacphy);
-fail_dac:
-	clk_put(sdev->dac);
-fail_sclk_dac:
-	clk_put(sdev->sclk_dac);
-fail:
-	dev_info(dev, "probe failed\n");
-	return ret;
-}
-
-static int sdo_remove(struct platform_device *pdev)
-{
-	struct v4l2_subdev *sd = dev_get_drvdata(&pdev->dev);
-	struct sdo_device *sdev = sd_to_sdev(sd);
-
-	pm_runtime_disable(&pdev->dev);
-	clk_disable_unprepare(sdev->dac);
-	clk_put(sdev->fout_vpll);
-	clk_put(sdev->dacphy);
-	clk_put(sdev->dac);
-	clk_put(sdev->sclk_dac);
-
-	dev_info(&pdev->dev, "remove successful\n");
-	return 0;
-}
-
-static struct platform_driver sdo_driver __refdata = {
-	.probe = sdo_probe,
-	.remove = sdo_remove,
-	.driver = {
-		.name = "s5p-sdo",
-		.pm = &sdo_pm_ops,
-	}
-};
-
-module_platform_driver(sdo_driver);
diff --git a/drivers/media/platform/s5p-tv/sii9234_drv.c b/drivers/media/platform/s5p-tv/sii9234_drv.c
deleted file mode 100644
index 0a97f9ab4f76..000000000000
--- a/drivers/media/platform/s5p-tv/sii9234_drv.c
+++ /dev/null
@@ -1,407 +0,0 @@
-/*
- * Samsung MHL interface driver
- *
- * Copyright (C) 2011 Samsung Electronics Co.Ltd
- * Author: Tomasz Stanislawski <t.stanislaws@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#include <linux/delay.h>
-#include <linux/err.h>
-#include <linux/freezer.h>
-#include <linux/gpio.h>
-#include <linux/i2c.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/kthread.h>
-#include <linux/module.h>
-#include <linux/pm_runtime.h>
-#include <linux/regulator/machine.h>
-#include <linux/slab.h>
-
-#include <linux/platform_data/media/sii9234.h>
-#include <media/v4l2-subdev.h>
-
-MODULE_AUTHOR("Tomasz Stanislawski <t.stanislaws@samsung.com>");
-MODULE_DESCRIPTION("Samsung MHL interface driver");
-MODULE_LICENSE("GPL");
-
-struct sii9234_context {
-	struct i2c_client *client;
-	struct regulator *power;
-	int gpio_n_reset;
-	struct v4l2_subdev sd;
-};
-
-static inline struct sii9234_context *sd_to_context(struct v4l2_subdev *sd)
-{
-	return container_of(sd, struct sii9234_context, sd);
-}
-
-static inline int sii9234_readb(struct i2c_client *client, int addr)
-{
-	return i2c_smbus_read_byte_data(client, addr);
-}
-
-static inline int sii9234_writeb(struct i2c_client *client, int addr, int value)
-{
-	return i2c_smbus_write_byte_data(client, addr, value);
-}
-
-static inline int sii9234_writeb_mask(struct i2c_client *client, int addr,
-	int value, int mask)
-{
-	int ret;
-
-	ret = i2c_smbus_read_byte_data(client, addr);
-	if (ret < 0)
-		return ret;
-	ret = (ret & ~mask) | (value & mask);
-	return i2c_smbus_write_byte_data(client, addr, ret);
-}
-
-static inline int sii9234_readb_idx(struct i2c_client *client, int addr)
-{
-	int ret;
-	ret = i2c_smbus_write_byte_data(client, 0xbc, addr >> 8);
-	if (ret < 0)
-		return ret;
-	ret = i2c_smbus_write_byte_data(client, 0xbd, addr & 0xff);
-	if (ret < 0)
-		return ret;
-	return i2c_smbus_read_byte_data(client, 0xbe);
-}
-
-static inline int sii9234_writeb_idx(struct i2c_client *client, int addr,
-	int value)
-{
-	int ret;
-	ret = i2c_smbus_write_byte_data(client, 0xbc, addr >> 8);
-	if (ret < 0)
-		return ret;
-	ret = i2c_smbus_write_byte_data(client, 0xbd, addr & 0xff);
-	if (ret < 0)
-		return ret;
-	ret = i2c_smbus_write_byte_data(client, 0xbe, value);
-	return ret;
-}
-
-static inline int sii9234_writeb_idx_mask(struct i2c_client *client, int addr,
-	int value, int mask)
-{
-	int ret;
-
-	ret = sii9234_readb_idx(client, addr);
-	if (ret < 0)
-		return ret;
-	ret = (ret & ~mask) | (value & mask);
-	return sii9234_writeb_idx(client, addr, ret);
-}
-
-static int sii9234_reset(struct sii9234_context *ctx)
-{
-	struct i2c_client *client = ctx->client;
-	struct device *dev = &client->dev;
-	int ret, tries;
-
-	gpio_direction_output(ctx->gpio_n_reset, 1);
-	mdelay(1);
-	gpio_direction_output(ctx->gpio_n_reset, 0);
-	mdelay(1);
-	gpio_direction_output(ctx->gpio_n_reset, 1);
-	mdelay(1);
-
-	/* going to TTPI mode */
-	ret = sii9234_writeb(client, 0xc7, 0);
-	if (ret < 0) {
-		dev_err(dev, "failed to set TTPI mode\n");
-		return ret;
-	}
-	for (tries = 0; tries < 100 ; ++tries) {
-		ret = sii9234_readb(client, 0x1b);
-		if (ret > 0)
-			break;
-		if (ret < 0) {
-			dev_err(dev, "failed to reset device\n");
-			return -EIO;
-		}
-		mdelay(1);
-	}
-	if (tries == 100) {
-		dev_err(dev, "maximal number of tries reached\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-static int sii9234_verify_version(struct i2c_client *client)
-{
-	struct device *dev = &client->dev;
-	int family, rev, tpi_rev, dev_id, sub_id, hdcp, id;
-
-	family = sii9234_readb(client, 0x1b);
-	rev = sii9234_readb(client, 0x1c) & 0x0f;
-	tpi_rev = sii9234_readb(client, 0x1d) & 0x7f;
-	dev_id = sii9234_readb_idx(client, 0x0103);
-	sub_id = sii9234_readb_idx(client, 0x0102);
-	hdcp = sii9234_readb(client, 0x30);
-
-	if (family < 0 || rev < 0 || tpi_rev < 0 || dev_id < 0 ||
-		sub_id < 0 || hdcp < 0) {
-		dev_err(dev, "failed to read chip's version\n");
-		return -EIO;
-	}
-
-	id = (dev_id << 8) | sub_id;
-
-	dev_info(dev, "chip: SiL%02x family: %02x, rev: %02x\n",
-		id, family, rev);
-	dev_info(dev, "tpi_rev:%02x, hdcp: %02x\n", tpi_rev, hdcp);
-	if (id != 0x9234) {
-		dev_err(dev, "not supported chip\n");
-		return -ENODEV;
-	}
-
-	return 0;
-}
-
-static u8 data[][3] = {
-/* setup from driver created by doonsoo45.kim */
-	{ 0x01, 0x05, 0x04 }, /* Enable Auto soft reset on SCDT = 0 */
-	{ 0x01, 0x08, 0x35 }, /* Power Up TMDS Tx Core */
-	{ 0x01, 0x0d, 0x1c }, /* HDMI Transcode mode enable */
-	{ 0x01, 0x2b, 0x01 }, /* Enable HDCP Compliance workaround */
-	{ 0x01, 0x79, 0x40 }, /* daniel test...MHL_INT */
-	{ 0x01, 0x80, 0x34 }, /* Enable Rx PLL Clock Value */
-	{ 0x01, 0x90, 0x27 }, /* Enable CBUS discovery */
-	{ 0x01, 0x91, 0xe5 }, /* Skip RGND detection */
-	{ 0x01, 0x92, 0x46 }, /* Force MHD mode */
-	{ 0x01, 0x93, 0xdc }, /* Disable CBUS pull-up during RGND measurement */
-	{ 0x01, 0x94, 0x66 }, /* 1.8V CBUS VTH & GND threshold */
-	{ 0x01, 0x95, 0x31 }, /* RGND block & single discovery attempt */
-	{ 0x01, 0x96, 0x22 }, /* use 1K and 2K setting */
-	{ 0x01, 0xa0, 0x10 }, /* SIMG: Term mode */
-	{ 0x01, 0xa1, 0xfc }, /* Disable internal Mobile HD driver */
-	{ 0x01, 0xa3, 0xfa }, /* SIMG: Output Swing  default EB, 3x Clk Mult */
-	{ 0x01, 0xa5, 0x80 }, /* SIMG: RGND Hysterisis, 3x mode for Beast */
-	{ 0x01, 0xa6, 0x0c }, /* SIMG: Swing Offset */
-	{ 0x02, 0x3d, 0x3f }, /* Power up CVCC 1.2V core */
-	{ 0x03, 0x00, 0x00 }, /* SIMG: correcting HW default */
-	{ 0x03, 0x11, 0x01 }, /* Enable TxPLL Clock */
-	{ 0x03, 0x12, 0x15 }, /* Enable Tx Clock Path & Equalizer */
-	{ 0x03, 0x13, 0x60 }, /* SIMG: Set termination value */
-	{ 0x03, 0x14, 0xf0 }, /* SIMG: Change CKDT level */
-	{ 0x03, 0x17, 0x07 }, /* SIMG: PLL Calrefsel */
-	{ 0x03, 0x1a, 0x20 }, /* VCO Cal */
-	{ 0x03, 0x22, 0xe0 }, /* SIMG: Auto EQ */
-	{ 0x03, 0x23, 0xc0 }, /* SIMG: Auto EQ */
-	{ 0x03, 0x24, 0xa0 }, /* SIMG: Auto EQ */
-	{ 0x03, 0x25, 0x80 }, /* SIMG: Auto EQ */
-	{ 0x03, 0x26, 0x60 }, /* SIMG: Auto EQ */
-	{ 0x03, 0x27, 0x40 }, /* SIMG: Auto EQ */
-	{ 0x03, 0x28, 0x20 }, /* SIMG: Auto EQ */
-	{ 0x03, 0x29, 0x00 }, /* SIMG: Auto EQ */
-	{ 0x03, 0x31, 0x0b }, /* SIMG: Rx PLL BW value from I2C BW ~ 4MHz */
-	{ 0x03, 0x45, 0x06 }, /* SIMG: DPLL Mode */
-	{ 0x03, 0x4b, 0x06 }, /* SIMG: Correcting HW default */
-	{ 0x03, 0x4c, 0xa0 }, /* Manual zone control */
-	{ 0x03, 0x4d, 0x02 }, /* SIMG: PLL Mode Value (order is important) */
-};
-
-static int sii9234_set_internal(struct sii9234_context *ctx)
-{
-	struct i2c_client *client = ctx->client;
-	int i, ret;
-
-	for (i = 0; i < ARRAY_SIZE(data); ++i) {
-		int addr = (data[i][0] << 8) | data[i][1];
-		ret = sii9234_writeb_idx(client, addr, data[i][2]);
-		if (ret < 0)
-			return ret;
-	}
-	return 0;
-}
-
-static int sii9234_runtime_suspend(struct device *dev)
-{
-	struct v4l2_subdev *sd = dev_get_drvdata(dev);
-	struct sii9234_context *ctx = sd_to_context(sd);
-	struct i2c_client *client = ctx->client;
-
-	dev_info(dev, "suspend start\n");
-
-	sii9234_writeb_mask(client, 0x1e, 3, 3);
-	regulator_disable(ctx->power);
-
-	return 0;
-}
-
-static int sii9234_runtime_resume(struct device *dev)
-{
-	struct v4l2_subdev *sd = dev_get_drvdata(dev);
-	struct sii9234_context *ctx = sd_to_context(sd);
-	struct i2c_client *client = ctx->client;
-	int ret;
-
-	dev_info(dev, "resume start\n");
-	ret = regulator_enable(ctx->power);
-	if (ret < 0)
-		return ret;
-
-	ret = sii9234_reset(ctx);
-	if (ret)
-		goto fail;
-
-	/* enable tpi */
-	ret = sii9234_writeb_mask(client, 0x1e, 1, 0);
-	if (ret < 0)
-		goto fail;
-	ret = sii9234_set_internal(ctx);
-	if (ret < 0)
-		goto fail;
-
-	return 0;
-
-fail:
-	dev_err(dev, "failed to resume\n");
-	regulator_disable(ctx->power);
-
-	return ret;
-}
-
-static const struct dev_pm_ops sii9234_pm_ops = {
-	.runtime_suspend = sii9234_runtime_suspend,
-	.runtime_resume	 = sii9234_runtime_resume,
-};
-
-static int sii9234_s_power(struct v4l2_subdev *sd, int on)
-{
-	struct sii9234_context *ctx = sd_to_context(sd);
-	int ret;
-
-	if (on)
-		ret = pm_runtime_get_sync(&ctx->client->dev);
-	else
-		ret = pm_runtime_put(&ctx->client->dev);
-	/* only values < 0 indicate errors */
-	return ret < 0 ? ret : 0;
-}
-
-static int sii9234_s_stream(struct v4l2_subdev *sd, int enable)
-{
-	struct sii9234_context *ctx = sd_to_context(sd);
-
-	/* (dis/en)able TDMS output */
-	sii9234_writeb_mask(ctx->client, 0x1a, enable ? 0 : ~0 , 1 << 4);
-	return 0;
-}
-
-static const struct v4l2_subdev_core_ops sii9234_core_ops = {
-	.s_power =  sii9234_s_power,
-};
-
-static const struct v4l2_subdev_video_ops sii9234_video_ops = {
-	.s_stream =  sii9234_s_stream,
-};
-
-static const struct v4l2_subdev_ops sii9234_ops = {
-	.core = &sii9234_core_ops,
-	.video = &sii9234_video_ops,
-};
-
-static int sii9234_probe(struct i2c_client *client,
-			 const struct i2c_device_id *id)
-{
-	struct device *dev = &client->dev;
-	struct sii9234_platform_data *pdata = dev->platform_data;
-	struct sii9234_context *ctx;
-	int ret;
-
-	ctx = devm_kzalloc(&client->dev, sizeof(*ctx), GFP_KERNEL);
-	if (!ctx) {
-		dev_err(dev, "out of memory\n");
-		ret = -ENOMEM;
-		goto fail;
-	}
-	ctx->client = client;
-
-	ctx->power = devm_regulator_get(dev, "hdmi-en");
-	if (IS_ERR(ctx->power)) {
-		dev_err(dev, "failed to acquire regulator hdmi-en\n");
-		return PTR_ERR(ctx->power);
-	}
-
-	ctx->gpio_n_reset = pdata->gpio_n_reset;
-	ret = devm_gpio_request(dev, ctx->gpio_n_reset, "MHL_RST");
-	if (ret) {
-		dev_err(dev, "failed to acquire MHL_RST gpio\n");
-		return ret;
-	}
-
-	v4l2_i2c_subdev_init(&ctx->sd, client, &sii9234_ops);
-
-	pm_runtime_enable(dev);
-
-	/* enable device */
-	ret = pm_runtime_get_sync(dev);
-	if (ret)
-		goto fail_pm;
-
-	/* verify chip version */
-	ret = sii9234_verify_version(client);
-	if (ret)
-		goto fail_pm_get;
-
-	/* stop processing */
-	pm_runtime_put(dev);
-
-	dev_info(dev, "probe successful\n");
-
-	return 0;
-
-fail_pm_get:
-	pm_runtime_put_sync(dev);
-
-fail_pm:
-	pm_runtime_disable(dev);
-
-fail:
-	dev_err(dev, "probe failed\n");
-
-	return ret;
-}
-
-static int sii9234_remove(struct i2c_client *client)
-{
-	struct device *dev = &client->dev;
-
-	pm_runtime_disable(dev);
-
-	dev_info(dev, "remove successful\n");
-
-	return 0;
-}
-
-
-static const struct i2c_device_id sii9234_id[] = {
-	{ "SII9234", 0 },
-	{ },
-};
-
-MODULE_DEVICE_TABLE(i2c, sii9234_id);
-static struct i2c_driver sii9234_driver = {
-	.driver = {
-		.name	= "sii9234",
-		.pm = &sii9234_pm_ops,
-	},
-	.probe		= sii9234_probe,
-	.remove		= sii9234_remove,
-	.id_table = sii9234_id,
-};
-
-module_i2c_driver(sii9234_driver);
diff --git a/drivers/media/platform/sh_vou.c b/drivers/media/platform/sh_vou.c
index e1f39b4cf1cd..ef2a519bcd4c 100644
--- a/drivers/media/platform/sh_vou.c
+++ b/drivers/media/platform/sh_vou.c
@@ -362,7 +362,7 @@ static void sh_vou_stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irqrestore(&vou_dev->lock, flags);
 }
 
-static struct vb2_ops sh_vou_qops = {
+static const struct vb2_ops sh_vou_qops = {
 	.queue_setup		= sh_vou_queue_setup,
 	.buf_prepare		= sh_vou_buf_prepare,
 	.buf_queue		= sh_vou_buf_queue,
@@ -937,7 +937,10 @@ static int sh_vou_s_selection(struct file *file, void *fh,
 {
 	struct v4l2_rect *rect = &sel->r;
 	struct sh_vou_device *vou_dev = video_drvdata(file);
-	struct v4l2_crop sd_crop = {.type = V4L2_BUF_TYPE_VIDEO_OUTPUT};
+	struct v4l2_subdev_selection sd_sel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = V4L2_SEL_TGT_COMPOSE,
+	};
 	struct v4l2_pix_format *pix = &vou_dev->pix;
 	struct sh_vou_geometry geo;
 	struct v4l2_subdev_format format = {
@@ -978,14 +981,14 @@ static int sh_vou_s_selection(struct file *file, void *fh,
 	geo.in_height = pix->height;
 
 	/* Configure the encoder one-to-one, position at 0, ignore errors */
-	sd_crop.c.width = geo.output.width;
-	sd_crop.c.height = geo.output.height;
+	sd_sel.r.width = geo.output.width;
+	sd_sel.r.height = geo.output.height;
 	/*
-	 * We first issue a S_CROP, so that the subsequent S_FMT delivers the
+	 * We first issue a S_SELECTION, so that the subsequent S_FMT delivers the
 	 * final encoder configuration.
 	 */
-	v4l2_device_call_until_err(&vou_dev->v4l2_dev, 0, video,
-				   s_crop, &sd_crop);
+	v4l2_device_call_until_err(&vou_dev->v4l2_dev, 0, pad,
+				   set_selection, NULL, &sd_sel);
 	format.format.width = geo.output.width;
 	format.format.height = geo.output.height;
 	ret = v4l2_device_call_until_err(&vou_dev->v4l2_dev, 0, pad,
diff --git a/drivers/media/platform/soc_camera/Kconfig b/drivers/media/platform/soc_camera/Kconfig
index 39f66414f621..86d74788544f 100644
--- a/drivers/media/platform/soc_camera/Kconfig
+++ b/drivers/media/platform/soc_camera/Kconfig
@@ -17,31 +17,6 @@ config SOC_CAMERA_PLATFORM
 	help
 	  This is a generic SoC camera platform driver, useful for testing
 
-config VIDEO_PXA27x
-	tristate "PXA27x Quick Capture Interface driver"
-	depends on VIDEO_DEV && PXA27x && SOC_CAMERA
-	select VIDEOBUF_DMA_SG
-	select SG_SPLIT
-	---help---
-	  This is a v4l2 driver for the PXA27x Quick Capture Interface
-
-config VIDEO_RCAR_VIN_OLD
-	tristate "R-Car Video Input (VIN) support (DEPRECATED)"
-	depends on VIDEO_DEV && SOC_CAMERA
-	depends on ARCH_RENESAS || COMPILE_TEST
-	depends on HAS_DMA
-	select VIDEOBUF2_DMA_CONTIG
-	select SOC_CAMERA_SCALE_CROP
-	---help---
-	  This is a v4l2 driver for the R-Car VIN Interface
-
-config VIDEO_SH_MOBILE_CSI2
-	tristate "SuperH Mobile MIPI CSI-2 Interface driver"
-	depends on VIDEO_DEV && SOC_CAMERA && HAVE_CLK
-	depends on ARCH_SHMOBILE || SUPERH || COMPILE_TEST
-	---help---
-	  This is a v4l2 driver for the SuperH MIPI CSI-2 Interface
-
 config VIDEO_SH_MOBILE_CEU
 	tristate "SuperH Mobile CEU Interface driver"
 	depends on VIDEO_DEV && SOC_CAMERA && HAS_DMA && HAVE_CLK
diff --git a/drivers/media/platform/soc_camera/Makefile b/drivers/media/platform/soc_camera/Makefile
index 7703cb7ce456..7633a0f2f66f 100644
--- a/drivers/media/platform/soc_camera/Makefile
+++ b/drivers/media/platform/soc_camera/Makefile
@@ -7,7 +7,4 @@ obj-$(CONFIG_SOC_CAMERA_PLATFORM)	+= soc_camera_platform.o
 
 # soc-camera host drivers have to be linked after camera drivers
 obj-$(CONFIG_VIDEO_ATMEL_ISI)		+= atmel-isi.o
-obj-$(CONFIG_VIDEO_PXA27x)		+= pxa_camera.o
 obj-$(CONFIG_VIDEO_SH_MOBILE_CEU)	+= sh_mobile_ceu_camera.o
-obj-$(CONFIG_VIDEO_SH_MOBILE_CSI2)	+= sh_mobile_csi2.o
-obj-$(CONFIG_VIDEO_RCAR_VIN_OLD)	+= rcar_vin.o
diff --git a/drivers/media/platform/soc_camera/atmel-isi.c b/drivers/media/platform/soc_camera/atmel-isi.c
index 30211f6b4483..46de657c3e6d 100644
--- a/drivers/media/platform/soc_camera/atmel-isi.c
+++ b/drivers/media/platform/soc_camera/atmel-isi.c
@@ -536,7 +536,7 @@ static void stop_streaming(struct vb2_queue *vq)
 	pm_runtime_put(ici->v4l2_dev.dev);
 }
 
-static struct vb2_ops isi_video_qops = {
+static const struct vb2_ops isi_video_qops = {
 	.queue_setup		= queue_setup,
 	.buf_init		= buffer_init,
 	.buf_prepare		= buffer_prepare,
diff --git a/drivers/media/platform/soc_camera/rcar_vin.c b/drivers/media/platform/soc_camera/rcar_vin.c
deleted file mode 100644
index 9c137522c660..000000000000
--- a/drivers/media/platform/soc_camera/rcar_vin.c
+++ /dev/null
@@ -1,1970 +0,0 @@
-/*
- * SoC-camera host driver for Renesas R-Car VIN unit
- *
- * Copyright (C) 2011-2013 Renesas Solutions Corp.
- * Copyright (C) 2013 Cogent Embedded, Inc., <source@cogentembedded.com>
- *
- * Based on V4L2 Driver for SuperH Mobile CEU interface "sh_mobile_ceu_camera.c"
- *
- * Copyright (C) 2008 Magnus Damm
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#include <linux/delay.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_device.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/slab.h>
-#include <linux/videodev2.h>
-
-#include <media/soc_camera.h>
-#include <media/drv-intf/soc_mediabus.h>
-#include <media/v4l2-common.h>
-#include <media/v4l2-dev.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-mediabus.h>
-#include <media/v4l2-of.h>
-#include <media/v4l2-subdev.h>
-#include <media/videobuf2-dma-contig.h>
-
-#include "soc_scale_crop.h"
-
-#define DRV_NAME "rcar_vin"
-
-/* Register offsets for R-Car VIN */
-#define VNMC_REG	0x00	/* Video n Main Control Register */
-#define VNMS_REG	0x04	/* Video n Module Status Register */
-#define VNFC_REG	0x08	/* Video n Frame Capture Register */
-#define VNSLPRC_REG	0x0C	/* Video n Start Line Pre-Clip Register */
-#define VNELPRC_REG	0x10	/* Video n End Line Pre-Clip Register */
-#define VNSPPRC_REG	0x14	/* Video n Start Pixel Pre-Clip Register */
-#define VNEPPRC_REG	0x18	/* Video n End Pixel Pre-Clip Register */
-#define VNSLPOC_REG	0x1C	/* Video n Start Line Post-Clip Register */
-#define VNELPOC_REG	0x20	/* Video n End Line Post-Clip Register */
-#define VNSPPOC_REG	0x24	/* Video n Start Pixel Post-Clip Register */
-#define VNEPPOC_REG	0x28	/* Video n End Pixel Post-Clip Register */
-#define VNIS_REG	0x2C	/* Video n Image Stride Register */
-#define VNMB_REG(m)	(0x30 + ((m) << 2)) /* Video n Memory Base m Register */
-#define VNIE_REG	0x40	/* Video n Interrupt Enable Register */
-#define VNINTS_REG	0x44	/* Video n Interrupt Status Register */
-#define VNSI_REG	0x48	/* Video n Scanline Interrupt Register */
-#define VNMTC_REG	0x4C	/* Video n Memory Transfer Control Register */
-#define VNYS_REG	0x50	/* Video n Y Scale Register */
-#define VNXS_REG	0x54	/* Video n X Scale Register */
-#define VNDMR_REG	0x58	/* Video n Data Mode Register */
-#define VNDMR2_REG	0x5C	/* Video n Data Mode Register 2 */
-#define VNUVAOF_REG	0x60	/* Video n UV Address Offset Register */
-#define VNC1A_REG	0x80	/* Video n Coefficient Set C1A Register */
-#define VNC1B_REG	0x84	/* Video n Coefficient Set C1B Register */
-#define VNC1C_REG	0x88	/* Video n Coefficient Set C1C Register */
-#define VNC2A_REG	0x90	/* Video n Coefficient Set C2A Register */
-#define VNC2B_REG	0x94	/* Video n Coefficient Set C2B Register */
-#define VNC2C_REG	0x98	/* Video n Coefficient Set C2C Register */
-#define VNC3A_REG	0xA0	/* Video n Coefficient Set C3A Register */
-#define VNC3B_REG	0xA4	/* Video n Coefficient Set C3B Register */
-#define VNC3C_REG	0xA8	/* Video n Coefficient Set C3C Register */
-#define VNC4A_REG	0xB0	/* Video n Coefficient Set C4A Register */
-#define VNC4B_REG	0xB4	/* Video n Coefficient Set C4B Register */
-#define VNC4C_REG	0xB8	/* Video n Coefficient Set C4C Register */
-#define VNC5A_REG	0xC0	/* Video n Coefficient Set C5A Register */
-#define VNC5B_REG	0xC4	/* Video n Coefficient Set C5B Register */
-#define VNC5C_REG	0xC8	/* Video n Coefficient Set C5C Register */
-#define VNC6A_REG	0xD0	/* Video n Coefficient Set C6A Register */
-#define VNC6B_REG	0xD4	/* Video n Coefficient Set C6B Register */
-#define VNC6C_REG	0xD8	/* Video n Coefficient Set C6C Register */
-#define VNC7A_REG	0xE0	/* Video n Coefficient Set C7A Register */
-#define VNC7B_REG	0xE4	/* Video n Coefficient Set C7B Register */
-#define VNC7C_REG	0xE8	/* Video n Coefficient Set C7C Register */
-#define VNC8A_REG	0xF0	/* Video n Coefficient Set C8A Register */
-#define VNC8B_REG	0xF4	/* Video n Coefficient Set C8B Register */
-#define VNC8C_REG	0xF8	/* Video n Coefficient Set C8C Register */
-
-/* Register bit fields for R-Car VIN */
-/* Video n Main Control Register bits */
-#define VNMC_FOC		(1 << 21)
-#define VNMC_YCAL		(1 << 19)
-#define VNMC_INF_YUV8_BT656	(0 << 16)
-#define VNMC_INF_YUV8_BT601	(1 << 16)
-#define VNMC_INF_YUV10_BT656	(2 << 16)
-#define VNMC_INF_YUV10_BT601	(3 << 16)
-#define VNMC_INF_YUV16		(5 << 16)
-#define VNMC_INF_RGB888		(6 << 16)
-#define VNMC_VUP		(1 << 10)
-#define VNMC_IM_ODD		(0 << 3)
-#define VNMC_IM_ODD_EVEN	(1 << 3)
-#define VNMC_IM_EVEN		(2 << 3)
-#define VNMC_IM_FULL		(3 << 3)
-#define VNMC_BPS		(1 << 1)
-#define VNMC_ME			(1 << 0)
-
-/* Video n Module Status Register bits */
-#define VNMS_FBS_MASK		(3 << 3)
-#define VNMS_FBS_SHIFT		3
-#define VNMS_AV			(1 << 1)
-#define VNMS_CA			(1 << 0)
-
-/* Video n Frame Capture Register bits */
-#define VNFC_C_FRAME		(1 << 1)
-#define VNFC_S_FRAME		(1 << 0)
-
-/* Video n Interrupt Enable Register bits */
-#define VNIE_FIE		(1 << 4)
-#define VNIE_EFE		(1 << 1)
-
-/* Video n Data Mode Register bits */
-#define VNDMR_EXRGB		(1 << 8)
-#define VNDMR_BPSM		(1 << 4)
-#define VNDMR_DTMD_YCSEP	(1 << 1)
-#define VNDMR_DTMD_ARGB		(1 << 0)
-
-/* Video n Data Mode Register 2 bits */
-#define VNDMR2_VPS		(1 << 30)
-#define VNDMR2_HPS		(1 << 29)
-#define VNDMR2_FTEV		(1 << 17)
-#define VNDMR2_VLV(n)		((n & 0xf) << 12)
-
-#define VIN_MAX_WIDTH		2048
-#define VIN_MAX_HEIGHT		2048
-
-#define TIMEOUT_MS		100
-
-#define RCAR_VIN_HSYNC_ACTIVE_LOW	(1 << 0)
-#define RCAR_VIN_VSYNC_ACTIVE_LOW	(1 << 1)
-#define RCAR_VIN_BT601			(1 << 2)
-#define RCAR_VIN_BT656			(1 << 3)
-
-enum chip_id {
-	RCAR_GEN3,
-	RCAR_GEN2,
-	RCAR_H1,
-	RCAR_M1,
-	RCAR_E1,
-};
-
-struct vin_coeff {
-	unsigned short xs_value;
-	u32 coeff_set[24];
-};
-
-static const struct vin_coeff vin_coeff_set[] = {
-	{ 0x0000, {
-		0x00000000,		0x00000000,		0x00000000,
-		0x00000000,		0x00000000,		0x00000000,
-		0x00000000,		0x00000000,		0x00000000,
-		0x00000000,		0x00000000,		0x00000000,
-		0x00000000,		0x00000000,		0x00000000,
-		0x00000000,		0x00000000,		0x00000000,
-		0x00000000,		0x00000000,		0x00000000,
-		0x00000000,		0x00000000,		0x00000000 },
-	},
-	{ 0x1000, {
-		0x000fa400,		0x000fa400,		0x09625902,
-		0x000003f8,		0x00000403,		0x3de0d9f0,
-		0x001fffed,		0x00000804,		0x3cc1f9c3,
-		0x001003de,		0x00000c01,		0x3cb34d7f,
-		0x002003d2,		0x00000c00,		0x3d24a92d,
-		0x00200bca,		0x00000bff,		0x3df600d2,
-		0x002013cc,		0x000007ff,		0x3ed70c7e,
-		0x00100fde,		0x00000000,		0x3f87c036 },
-	},
-	{ 0x1200, {
-		0x002ffff1,		0x002ffff1,		0x02a0a9c8,
-		0x002003e7,		0x001ffffa,		0x000185bc,
-		0x002007dc,		0x000003ff,		0x3e52859c,
-		0x00200bd4,		0x00000002,		0x3d53996b,
-		0x00100fd0,		0x00000403,		0x3d04ad2d,
-		0x00000bd5,		0x00000403,		0x3d35ace7,
-		0x3ff003e4,		0x00000801,		0x3dc674a1,
-		0x3fffe800,		0x00000800,		0x3e76f461 },
-	},
-	{ 0x1400, {
-		0x00100be3,		0x00100be3,		0x04d1359a,
-		0x00000fdb,		0x002003ed,		0x0211fd93,
-		0x00000fd6,		0x002003f4,		0x0002d97b,
-		0x000007d6,		0x002ffffb,		0x3e93b956,
-		0x3ff003da,		0x001003ff,		0x3db49926,
-		0x3fffefe9,		0x00100001,		0x3d655cee,
-		0x3fffd400,		0x00000003,		0x3d65f4b6,
-		0x000fb421,		0x00000402,		0x3dc6547e },
-	},
-	{ 0x1600, {
-		0x00000bdd,		0x00000bdd,		0x06519578,
-		0x3ff007da,		0x00000be3,		0x03c24973,
-		0x3ff003d9,		0x00000be9,		0x01b30d5f,
-		0x3ffff7df,		0x001003f1,		0x0003c542,
-		0x000fdfec,		0x001003f7,		0x3ec4711d,
-		0x000fc400,		0x002ffffd,		0x3df504f1,
-		0x001fa81a,		0x002ffc00,		0x3d957cc2,
-		0x002f8c3c,		0x00100000,		0x3db5c891 },
-	},
-	{ 0x1800, {
-		0x3ff003dc,		0x3ff003dc,		0x0791e558,
-		0x000ff7dd,		0x3ff007de,		0x05328554,
-		0x000fe7e3,		0x3ff00be2,		0x03232546,
-		0x000fd7ee,		0x000007e9,		0x0143bd30,
-		0x001fb800,		0x000007ee,		0x00044511,
-		0x002fa015,		0x000007f4,		0x3ef4bcee,
-		0x002f8832,		0x001003f9,		0x3e4514c7,
-		0x001f7853,		0x001003fd,		0x3de54c9f },
-	},
-	{ 0x1a00, {
-		0x000fefe0,		0x000fefe0,		0x08721d3c,
-		0x001fdbe7,		0x000ffbde,		0x0652a139,
-		0x001fcbf0,		0x000003df,		0x0463292e,
-		0x002fb3ff,		0x3ff007e3,		0x0293a91d,
-		0x002f9c12,		0x3ff00be7,		0x01241905,
-		0x001f8c29,		0x000007ed,		0x3fe470eb,
-		0x000f7c46,		0x000007f2,		0x3f04b8ca,
-		0x3fef7865,		0x000007f6,		0x3e74e4a8 },
-	},
-	{ 0x1c00, {
-		0x001fd3e9,		0x001fd3e9,		0x08f23d26,
-		0x002fbff3,		0x001fe3e4,		0x0712ad23,
-		0x002fa800,		0x000ff3e0,		0x05631d1b,
-		0x001f9810,		0x000ffbe1,		0x03b3890d,
-		0x000f8c23,		0x000003e3,		0x0233e8fa,
-		0x3fef843b,		0x000003e7,		0x00f430e4,
-		0x3fbf8456,		0x3ff00bea,		0x00046cc8,
-		0x3f8f8c72,		0x3ff00bef,		0x3f3490ac },
-	},
-	{ 0x1e00, {
-		0x001fbbf4,		0x001fbbf4,		0x09425112,
-		0x001fa800,		0x002fc7ed,		0x0792b110,
-		0x000f980e,		0x001fdbe6,		0x0613110a,
-		0x3fff8c20,		0x001fe7e3,		0x04a368fd,
-		0x3fcf8c33,		0x000ff7e2,		0x0343b8ed,
-		0x3f9f8c4a,		0x000fffe3,		0x0203f8da,
-		0x3f5f9c61,		0x000003e6,		0x00e428c5,
-		0x3f1fb07b,		0x000003eb,		0x3fe440af },
-	},
-	{ 0x2000, {
-		0x000fa400,		0x000fa400,		0x09625902,
-		0x3fff980c,		0x001fb7f5,		0x0812b0ff,
-		0x3fdf901c,		0x001fc7ed,		0x06b2fcfa,
-		0x3faf902d,		0x001fd3e8,		0x055348f1,
-		0x3f7f983f,		0x001fe3e5,		0x04038ce3,
-		0x3f3fa454,		0x001fefe3,		0x02e3c8d1,
-		0x3f0fb86a,		0x001ff7e4,		0x01c3e8c0,
-		0x3ecfd880,		0x000fffe6,		0x00c404ac },
-	},
-	{ 0x2200, {
-		0x3fdf9c0b,		0x3fdf9c0b,		0x09725cf4,
-		0x3fbf9818,		0x3fffa400,		0x0842a8f1,
-		0x3f8f9827,		0x000fb3f7,		0x0702f0ec,
-		0x3f5fa037,		0x000fc3ef,		0x05d330e4,
-		0x3f2fac49,		0x001fcfea,		0x04a364d9,
-		0x3effc05c,		0x001fdbe7,		0x038394ca,
-		0x3ecfdc6f,		0x001fe7e6,		0x0273b0bb,
-		0x3ea00083,		0x001fefe6,		0x0183c0a9 },
-	},
-	{ 0x2400, {
-		0x3f9fa014,		0x3f9fa014,		0x098260e6,
-		0x3f7f9c23,		0x3fcf9c0a,		0x08629ce5,
-		0x3f4fa431,		0x3fefa400,		0x0742d8e1,
-		0x3f1fb440,		0x3fffb3f8,		0x062310d9,
-		0x3eefc850,		0x000fbbf2,		0x050340d0,
-		0x3ecfe062,		0x000fcbec,		0x041364c2,
-		0x3ea00073,		0x001fd3ea,		0x03037cb5,
-		0x3e902086,		0x001fdfe8,		0x022388a5 },
-	},
-	{ 0x2600, {
-		0x3f5fa81e,		0x3f5fa81e,		0x096258da,
-		0x3f3fac2b,		0x3f8fa412,		0x088290d8,
-		0x3f0fbc38,		0x3fafa408,		0x0772c8d5,
-		0x3eefcc47,		0x3fcfa800,		0x0672f4ce,
-		0x3ecfe456,		0x3fefaffa,		0x05531cc6,
-		0x3eb00066,		0x3fffbbf3,		0x047334bb,
-		0x3ea01c77,		0x000fc7ee,		0x039348ae,
-		0x3ea04486,		0x000fd3eb,		0x02b350a1 },
-	},
-	{ 0x2800, {
-		0x3f2fb426,		0x3f2fb426,		0x094250ce,
-		0x3f0fc032,		0x3f4fac1b,		0x086284cd,
-		0x3eefd040,		0x3f7fa811,		0x0782acc9,
-		0x3ecfe84c,		0x3f9fa807,		0x06a2d8c4,
-		0x3eb0005b,		0x3fbfac00,		0x05b2f4bc,
-		0x3eb0186a,		0x3fdfb3fa,		0x04c308b4,
-		0x3eb04077,		0x3fefbbf4,		0x03f31ca8,
-		0x3ec06884,		0x000fbff2,		0x03031c9e },
-	},
-	{ 0x2a00, {
-		0x3f0fc42d,		0x3f0fc42d,		0x090240c4,
-		0x3eefd439,		0x3f2fb822,		0x08526cc2,
-		0x3edfe845,		0x3f4fb018,		0x078294bf,
-		0x3ec00051,		0x3f6fac0f,		0x06b2b4bb,
-		0x3ec0185f,		0x3f8fac07,		0x05e2ccb4,
-		0x3ec0386b,		0x3fafac00,		0x0502e8ac,
-		0x3ed05c77,		0x3fcfb3fb,		0x0432f0a3,
-		0x3ef08482,		0x3fdfbbf6,		0x0372f898 },
-	},
-	{ 0x2c00, {
-		0x3eefdc31,		0x3eefdc31,		0x08e238b8,
-		0x3edfec3d,		0x3f0fc828,		0x082258b9,
-		0x3ed00049,		0x3f1fc01e,		0x077278b6,
-		0x3ed01455,		0x3f3fb815,		0x06c294b2,
-		0x3ed03460,		0x3f5fb40d,		0x0602acac,
-		0x3ef0506c,		0x3f7fb006,		0x0542c0a4,
-		0x3f107476,		0x3f9fb400,		0x0472c89d,
-		0x3f309c80,		0x3fbfb7fc,		0x03b2cc94 },
-	},
-	{ 0x2e00, {
-		0x3eefec37,		0x3eefec37,		0x088220b0,
-		0x3ee00041,		0x3effdc2d,		0x07f244ae,
-		0x3ee0144c,		0x3f0fd023,		0x07625cad,
-		0x3ef02c57,		0x3f1fc81a,		0x06c274a9,
-		0x3f004861,		0x3f3fbc13,		0x060288a6,
-		0x3f20686b,		0x3f5fb80c,		0x05529c9e,
-		0x3f408c74,		0x3f6fb805,		0x04b2ac96,
-		0x3f80ac7e,		0x3f8fb800,		0x0402ac8e },
-	},
-	{ 0x3000, {
-		0x3ef0003a,		0x3ef0003a,		0x084210a6,
-		0x3ef01045,		0x3effec32,		0x07b228a7,
-		0x3f00284e,		0x3f0fdc29,		0x073244a4,
-		0x3f104058,		0x3f0fd420,		0x06a258a2,
-		0x3f305c62,		0x3f2fc818,		0x0612689d,
-		0x3f508069,		0x3f3fc011,		0x05728496,
-		0x3f80a072,		0x3f4fc00a,		0x04d28c90,
-		0x3fc0c07b,		0x3f6fbc04,		0x04429088 },
-	},
-	{ 0x3200, {
-		0x3f00103e,		0x3f00103e,		0x07f1fc9e,
-		0x3f102447,		0x3f000035,		0x0782149d,
-		0x3f203c4f,		0x3f0ff02c,		0x07122c9c,
-		0x3f405458,		0x3f0fe424,		0x06924099,
-		0x3f607061,		0x3f1fd41d,		0x06024c97,
-		0x3f909068,		0x3f2fcc16,		0x05726490,
-		0x3fc0b070,		0x3f3fc80f,		0x04f26c8a,
-		0x0000d077,		0x3f4fc409,		0x04627484 },
-	},
-	{ 0x3400, {
-		0x3f202040,		0x3f202040,		0x07a1e898,
-		0x3f303449,		0x3f100c38,		0x0741fc98,
-		0x3f504c50,		0x3f10002f,		0x06e21495,
-		0x3f706459,		0x3f1ff028,		0x06722492,
-		0x3fa08060,		0x3f1fe421,		0x05f2348f,
-		0x3fd09c67,		0x3f1fdc19,		0x05824c89,
-		0x0000bc6e,		0x3f2fd014,		0x04f25086,
-		0x0040dc74,		0x3f3fcc0d,		0x04825c7f },
-	},
-	{ 0x3600, {
-		0x3f403042,		0x3f403042,		0x0761d890,
-		0x3f504848,		0x3f301c3b,		0x0701f090,
-		0x3f805c50,		0x3f200c33,		0x06a2008f,
-		0x3fa07458,		0x3f10002b,		0x06520c8d,
-		0x3fd0905e,		0x3f1ff424,		0x05e22089,
-		0x0000ac65,		0x3f1fe81d,		0x05823483,
-		0x0030cc6a,		0x3f2fdc18,		0x04f23c81,
-		0x0080e871,		0x3f2fd412,		0x0482407c },
-	},
-	{ 0x3800, {
-		0x3f604043,		0x3f604043,		0x0721c88a,
-		0x3f80544a,		0x3f502c3c,		0x06d1d88a,
-		0x3fb06851,		0x3f301c35,		0x0681e889,
-		0x3fd08456,		0x3f30082f,		0x0611fc88,
-		0x00009c5d,		0x3f200027,		0x05d20884,
-		0x0030b863,		0x3f2ff421,		0x05621880,
-		0x0070d468,		0x3f2fe81b,		0x0502247c,
-		0x00c0ec6f,		0x3f2fe015,		0x04a22877 },
-	},
-	{ 0x3a00, {
-		0x3f904c44,		0x3f904c44,		0x06e1b884,
-		0x3fb0604a,		0x3f70383e,		0x0691c885,
-		0x3fe07451,		0x3f502c36,		0x0661d483,
-		0x00009055,		0x3f401831,		0x0601ec81,
-		0x0030a85b,		0x3f300c2a,		0x05b1f480,
-		0x0070c061,		0x3f300024,		0x0562047a,
-		0x00b0d867,		0x3f3ff41e,		0x05020c77,
-		0x00f0f46b,		0x3f2fec19,		0x04a21474 },
-	},
-	{ 0x3c00, {
-		0x3fb05c43,		0x3fb05c43,		0x06c1b07e,
-		0x3fe06c4b,		0x3f902c3f,		0x0681c081,
-		0x0000844f,		0x3f703838,		0x0631cc7d,
-		0x00309855,		0x3f602433,		0x05d1d47e,
-		0x0060b459,		0x3f50142e,		0x0581e47b,
-		0x00a0c85f,		0x3f400828,		0x0531f078,
-		0x00e0e064,		0x3f300021,		0x0501fc73,
-		0x00b0fc6a,		0x3f3ff41d,		0x04a20873 },
-	},
-	{ 0x3e00, {
-		0x3fe06444,		0x3fe06444,		0x0681a07a,
-		0x00007849,		0x3fc0503f,		0x0641b07a,
-		0x0020904d,		0x3fa0403a,		0x05f1c07a,
-		0x0060a453,		0x3f803034,		0x05c1c878,
-		0x0090b858,		0x3f70202f,		0x0571d477,
-		0x00d0d05d,		0x3f501829,		0x0531e073,
-		0x0110e462,		0x3f500825,		0x04e1e471,
-		0x01510065,		0x3f40001f,		0x04a1f06d },
-	},
-	{ 0x4000, {
-		0x00007044,		0x00007044,		0x06519476,
-		0x00208448,		0x3fe05c3f,		0x0621a476,
-		0x0050984d,		0x3fc04c3a,		0x05e1b075,
-		0x0080ac52,		0x3fa03c35,		0x05a1b875,
-		0x00c0c056,		0x3f803030,		0x0561c473,
-		0x0100d45b,		0x3f70202b,		0x0521d46f,
-		0x0140e860,		0x3f601427,		0x04d1d46e,
-		0x01810064,		0x3f500822,		0x0491dc6b },
-	},
-	{ 0x5000, {
-		0x0110a442,		0x0110a442,		0x0551545e,
-		0x0140b045,		0x00e0983f,		0x0531585f,
-		0x0160c047,		0x00c08c3c,		0x0511645e,
-		0x0190cc4a,		0x00908039,		0x04f1685f,
-		0x01c0dc4c,		0x00707436,		0x04d1705e,
-		0x0200e850,		0x00506833,		0x04b1785b,
-		0x0230f453,		0x00305c30,		0x0491805a,
-		0x02710056,		0x0010542d,		0x04718059 },
-	},
-	{ 0x6000, {
-		0x01c0bc40,		0x01c0bc40,		0x04c13052,
-		0x01e0c841,		0x01a0b43d,		0x04c13851,
-		0x0210cc44,		0x0180a83c,		0x04a13453,
-		0x0230d845,		0x0160a03a,		0x04913c52,
-		0x0260e047,		0x01409838,		0x04714052,
-		0x0280ec49,		0x01208c37,		0x04514c50,
-		0x02b0f44b,		0x01008435,		0x04414c50,
-		0x02d1004c,		0x00e07c33,		0x0431544f },
-	},
-	{ 0x7000, {
-		0x0230c83e,		0x0230c83e,		0x04711c4c,
-		0x0250d03f,		0x0210c43c,		0x0471204b,
-		0x0270d840,		0x0200b83c,		0x0451244b,
-		0x0290dc42,		0x01e0b43a,		0x0441244c,
-		0x02b0e443,		0x01c0b038,		0x0441284b,
-		0x02d0ec44,		0x01b0a438,		0x0421304a,
-		0x02f0f445,		0x0190a036,		0x04213449,
-		0x0310f847,		0x01709c34,		0x04213848 },
-	},
-	{ 0x8000, {
-		0x0280d03d,		0x0280d03d,		0x04310c48,
-		0x02a0d43e,		0x0270c83c,		0x04311047,
-		0x02b0dc3e,		0x0250c83a,		0x04311447,
-		0x02d0e040,		0x0240c03a,		0x04211446,
-		0x02e0e840,		0x0220bc39,		0x04111847,
-		0x0300e842,		0x0210b438,		0x04012445,
-		0x0310f043,		0x0200b037,		0x04012045,
-		0x0330f444,		0x01e0ac36,		0x03f12445 },
-	},
-	{ 0xefff, {
-		0x0340dc3a,		0x0340dc3a,		0x03b0ec40,
-		0x0340e03a,		0x0330e039,		0x03c0f03e,
-		0x0350e03b,		0x0330dc39,		0x03c0ec3e,
-		0x0350e43a,		0x0320dc38,		0x03c0f43e,
-		0x0360e43b,		0x0320d839,		0x03b0f03e,
-		0x0360e83b,		0x0310d838,		0x03c0fc3b,
-		0x0370e83b,		0x0310d439,		0x03a0f83d,
-		0x0370e83c,		0x0300d438,		0x03b0fc3c },
-	}
-};
-
-enum rcar_vin_state {
-	STOPPED = 0,
-	RUNNING,
-	STOPPING,
-};
-
-struct rcar_vin_priv {
-	void __iomem			*base;
-	spinlock_t			lock;
-	int				sequence;
-	/* State of the VIN module in capturing mode */
-	enum rcar_vin_state		state;
-	struct soc_camera_host		ici;
-	struct list_head		capture;
-#define MAX_BUFFER_NUM			3
-	struct vb2_v4l2_buffer		*queue_buf[MAX_BUFFER_NUM];
-	enum v4l2_field			field;
-	unsigned int			pdata_flags;
-	unsigned int			vb_count;
-	unsigned int			nr_hw_slots;
-	bool				request_to_stop;
-	struct completion		capture_stop;
-	enum chip_id			chip;
-};
-
-#define is_continuous_transfer(priv)	(priv->vb_count > MAX_BUFFER_NUM)
-
-struct rcar_vin_buffer {
-	struct vb2_v4l2_buffer vb;
-	struct list_head		list;
-};
-
-#define to_buf_list(vb2_buffer)	(&container_of(vb2_buffer, \
-						       struct rcar_vin_buffer, \
-						       vb)->list)
-
-struct rcar_vin_cam {
-	/* VIN offsets within the camera output, before the VIN scaler */
-	unsigned int			vin_left;
-	unsigned int			vin_top;
-	/* Client output, as seen by the VIN */
-	unsigned int			width;
-	unsigned int			height;
-	/* User window from S_FMT */
-	unsigned int out_width;
-	unsigned int out_height;
-	/*
-	 * User window from S_CROP / G_CROP, produced by client cropping and
-	 * scaling, VIN scaling and VIN cropping, mapped back onto the client
-	 * input window
-	 */
-	struct v4l2_rect		subrect;
-	/* Camera cropping rectangle */
-	struct v4l2_rect		rect;
-	const struct soc_mbus_pixelfmt	*extra_fmt;
-};
-
-/*
- * .queue_setup() is called to check whether the driver can accept the requested
- * number of buffers and to fill in plane sizes for the current frame format if
- * required
- */
-static int rcar_vin_videobuf_setup(struct vb2_queue *vq,
-				   unsigned int *count,
-				   unsigned int *num_planes,
-				   unsigned int sizes[], struct device *alloc_devs[])
-{
-	struct soc_camera_device *icd = soc_camera_from_vb2q(vq);
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct rcar_vin_priv *priv = ici->priv;
-
-	if (!vq->num_buffers)
-		priv->sequence = 0;
-
-	if (!*count)
-		*count = 2;
-	priv->vb_count = *count;
-
-	/* Number of hardware slots */
-	if (is_continuous_transfer(priv))
-		priv->nr_hw_slots = MAX_BUFFER_NUM;
-	else
-		priv->nr_hw_slots = 1;
-
-	if (*num_planes)
-		return sizes[0] < icd->sizeimage ? -EINVAL : 0;
-
-	sizes[0] = icd->sizeimage;
-	*num_planes = 1;
-
-	dev_dbg(icd->parent, "count=%d, size=%u\n", *count, sizes[0]);
-
-	return 0;
-}
-
-static int rcar_vin_setup(struct rcar_vin_priv *priv)
-{
-	struct soc_camera_device *icd = priv->ici.icd;
-	struct rcar_vin_cam *cam = icd->host_priv;
-	u32 vnmc, dmr, interrupts;
-	bool progressive = false, output_is_yuv = false, input_is_yuv = false;
-
-	switch (priv->field) {
-	case V4L2_FIELD_TOP:
-		vnmc = VNMC_IM_ODD;
-		break;
-	case V4L2_FIELD_BOTTOM:
-		vnmc = VNMC_IM_EVEN;
-		break;
-	case V4L2_FIELD_INTERLACED:
-	case V4L2_FIELD_INTERLACED_TB:
-		vnmc = VNMC_IM_FULL;
-		break;
-	case V4L2_FIELD_INTERLACED_BT:
-		vnmc = VNMC_IM_FULL | VNMC_FOC;
-		break;
-	case V4L2_FIELD_NONE:
-		if (is_continuous_transfer(priv)) {
-			vnmc = VNMC_IM_ODD_EVEN;
-			progressive = true;
-		} else {
-			vnmc = VNMC_IM_ODD;
-		}
-		break;
-	default:
-		vnmc = VNMC_IM_ODD;
-		break;
-	}
-
-	/* input interface */
-	switch (icd->current_fmt->code) {
-	case MEDIA_BUS_FMT_YUYV8_1X16:
-		/* BT.601/BT.1358 16bit YCbCr422 */
-		vnmc |= VNMC_INF_YUV16;
-		input_is_yuv = true;
-		break;
-	case MEDIA_BUS_FMT_YUYV8_2X8:
-		/* BT.656 8bit YCbCr422 or BT.601 8bit YCbCr422 */
-		vnmc |= priv->pdata_flags & RCAR_VIN_BT656 ?
-			VNMC_INF_YUV8_BT656 : VNMC_INF_YUV8_BT601;
-		input_is_yuv = true;
-		break;
-	case MEDIA_BUS_FMT_RGB888_1X24:
-		vnmc |= VNMC_INF_RGB888;
-		break;
-	case MEDIA_BUS_FMT_YUYV10_2X10:
-		/* BT.656 10bit YCbCr422 or BT.601 10bit YCbCr422 */
-		vnmc |= priv->pdata_flags & RCAR_VIN_BT656 ?
-			VNMC_INF_YUV10_BT656 : VNMC_INF_YUV10_BT601;
-		input_is_yuv = true;
-		break;
-	default:
-		break;
-	}
-
-	/* output format */
-	switch (icd->current_fmt->host_fmt->fourcc) {
-	case V4L2_PIX_FMT_NV16:
-		iowrite32(ALIGN(cam->width * cam->height, 0x80),
-			  priv->base + VNUVAOF_REG);
-		dmr = VNDMR_DTMD_YCSEP;
-		output_is_yuv = true;
-		break;
-	case V4L2_PIX_FMT_YUYV:
-		dmr = VNDMR_BPSM;
-		output_is_yuv = true;
-		break;
-	case V4L2_PIX_FMT_UYVY:
-		dmr = 0;
-		output_is_yuv = true;
-		break;
-	case V4L2_PIX_FMT_RGB555X:
-		dmr = VNDMR_DTMD_ARGB;
-		break;
-	case V4L2_PIX_FMT_RGB565:
-		dmr = 0;
-		break;
-	case V4L2_PIX_FMT_RGB32:
-		if (priv->chip != RCAR_GEN2 && priv->chip != RCAR_H1 &&
-		    priv->chip != RCAR_E1)
-			goto e_format;
-
-		dmr = VNDMR_EXRGB;
-		break;
-	case V4L2_PIX_FMT_ARGB32:
-		if (priv->chip != RCAR_GEN3)
-			goto e_format;
-
-		dmr = VNDMR_EXRGB | VNDMR_DTMD_ARGB;
-		break;
-	default:
-		goto e_format;
-	}
-
-	/* Always update on field change */
-	vnmc |= VNMC_VUP;
-
-	/* If input and output use the same colorspace, use bypass mode */
-	if (input_is_yuv == output_is_yuv)
-		vnmc |= VNMC_BPS;
-
-	/* progressive or interlaced mode */
-	interrupts = progressive ? VNIE_FIE : VNIE_EFE;
-
-	/* ack interrupts */
-	iowrite32(interrupts, priv->base + VNINTS_REG);
-	/* enable interrupts */
-	iowrite32(interrupts, priv->base + VNIE_REG);
-	/* start capturing */
-	iowrite32(dmr, priv->base + VNDMR_REG);
-	iowrite32(vnmc | VNMC_ME, priv->base + VNMC_REG);
-
-	return 0;
-
-e_format:
-	dev_warn(icd->parent, "Invalid fourcc format (0x%x)\n",
-		 icd->current_fmt->host_fmt->fourcc);
-	return -EINVAL;
-}
-
-static void rcar_vin_capture(struct rcar_vin_priv *priv)
-{
-	if (is_continuous_transfer(priv))
-		/* Continuous Frame Capture Mode */
-		iowrite32(VNFC_C_FRAME, priv->base + VNFC_REG);
-	else
-		/* Single Frame Capture Mode */
-		iowrite32(VNFC_S_FRAME, priv->base + VNFC_REG);
-}
-
-static void rcar_vin_request_capture_stop(struct rcar_vin_priv *priv)
-{
-	priv->state = STOPPING;
-
-	/* set continuous & single transfer off */
-	iowrite32(0, priv->base + VNFC_REG);
-	/* disable capture (release DMA buffer), reset */
-	iowrite32(ioread32(priv->base + VNMC_REG) & ~VNMC_ME,
-		  priv->base + VNMC_REG);
-
-	/* update the status if stopped already */
-	if (!(ioread32(priv->base + VNMS_REG) & VNMS_CA))
-		priv->state = STOPPED;
-}
-
-static int rcar_vin_get_free_hw_slot(struct rcar_vin_priv *priv)
-{
-	int slot;
-
-	for (slot = 0; slot < priv->nr_hw_slots; slot++)
-		if (priv->queue_buf[slot] == NULL)
-			return slot;
-
-	return -1;
-}
-
-static int rcar_vin_hw_ready(struct rcar_vin_priv *priv)
-{
-	/* Ensure all HW slots are filled */
-	return rcar_vin_get_free_hw_slot(priv) < 0 ? 1 : 0;
-}
-
-/* Moves a buffer from the queue to the HW slots */
-static int rcar_vin_fill_hw_slot(struct rcar_vin_priv *priv)
-{
-	struct vb2_v4l2_buffer *vbuf;
-	dma_addr_t phys_addr_top;
-	int slot;
-
-	if (list_empty(&priv->capture))
-		return 0;
-
-	/* Find a free HW slot */
-	slot = rcar_vin_get_free_hw_slot(priv);
-	if (slot < 0)
-		return 0;
-
-	vbuf = &list_entry(priv->capture.next,
-			struct rcar_vin_buffer, list)->vb;
-	list_del_init(to_buf_list(vbuf));
-	priv->queue_buf[slot] = vbuf;
-	phys_addr_top = vb2_dma_contig_plane_dma_addr(&vbuf->vb2_buf, 0);
-	iowrite32(phys_addr_top, priv->base + VNMB_REG(slot));
-
-	return 1;
-}
-
-static void rcar_vin_videobuf_queue(struct vb2_buffer *vb)
-{
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct soc_camera_device *icd = soc_camera_from_vb2q(vb->vb2_queue);
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct rcar_vin_priv *priv = ici->priv;
-	unsigned long size;
-
-	size = icd->sizeimage;
-
-	if (vb2_plane_size(vb, 0) < size) {
-		dev_err(icd->parent, "Buffer #%d too small (%lu < %lu)\n",
-			vb->index, vb2_plane_size(vb, 0), size);
-		goto error;
-	}
-
-	vb2_set_plane_payload(vb, 0, size);
-
-	dev_dbg(icd->parent, "%s (vb=0x%p) 0x%p %lu\n", __func__,
-		vb, vb2_plane_vaddr(vb, 0), vb2_get_plane_payload(vb, 0));
-
-	spin_lock_irq(&priv->lock);
-
-	list_add_tail(to_buf_list(vbuf), &priv->capture);
-	rcar_vin_fill_hw_slot(priv);
-
-	/* If we weren't running, and have enough buffers, start capturing! */
-	if (priv->state != RUNNING && rcar_vin_hw_ready(priv)) {
-		if (rcar_vin_setup(priv)) {
-			/* Submit error */
-			list_del_init(to_buf_list(vbuf));
-			spin_unlock_irq(&priv->lock);
-			goto error;
-		}
-		priv->request_to_stop = false;
-		init_completion(&priv->capture_stop);
-		priv->state = RUNNING;
-		rcar_vin_capture(priv);
-	}
-
-	spin_unlock_irq(&priv->lock);
-
-	return;
-
-error:
-	vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
-}
-
-/*
- * Wait for capture to stop and all in-flight buffers to be finished with by
- * the video hardware. This must be called under &priv->lock
- *
- */
-static void rcar_vin_wait_stop_streaming(struct rcar_vin_priv *priv)
-{
-	while (priv->state != STOPPED) {
-		/* issue stop if running */
-		if (priv->state == RUNNING)
-			rcar_vin_request_capture_stop(priv);
-
-		/* wait until capturing has been stopped */
-		if (priv->state == STOPPING) {
-			priv->request_to_stop = true;
-			spin_unlock_irq(&priv->lock);
-			if (!wait_for_completion_timeout(
-					&priv->capture_stop,
-					msecs_to_jiffies(TIMEOUT_MS)))
-				priv->state = STOPPED;
-			spin_lock_irq(&priv->lock);
-		}
-	}
-}
-
-static void rcar_vin_stop_streaming(struct vb2_queue *vq)
-{
-	struct soc_camera_device *icd = soc_camera_from_vb2q(vq);
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct rcar_vin_priv *priv = ici->priv;
-	struct list_head *buf_head, *tmp;
-	int i;
-
-	spin_lock_irq(&priv->lock);
-	rcar_vin_wait_stop_streaming(priv);
-
-	for (i = 0; i < MAX_BUFFER_NUM; i++) {
-		if (priv->queue_buf[i]) {
-			vb2_buffer_done(&priv->queue_buf[i]->vb2_buf,
-					VB2_BUF_STATE_ERROR);
-			priv->queue_buf[i] = NULL;
-		}
-	}
-
-	list_for_each_safe(buf_head, tmp, &priv->capture) {
-		vb2_buffer_done(&list_entry(buf_head,
-				struct rcar_vin_buffer, list)->vb.vb2_buf,
-				VB2_BUF_STATE_ERROR);
-		list_del_init(buf_head);
-	}
-	spin_unlock_irq(&priv->lock);
-}
-
-static struct vb2_ops rcar_vin_vb2_ops = {
-	.queue_setup	= rcar_vin_videobuf_setup,
-	.buf_queue	= rcar_vin_videobuf_queue,
-	.stop_streaming	= rcar_vin_stop_streaming,
-	.wait_prepare	= vb2_ops_wait_prepare,
-	.wait_finish	= vb2_ops_wait_finish,
-};
-
-static irqreturn_t rcar_vin_irq(int irq, void *data)
-{
-	struct rcar_vin_priv *priv = data;
-	u32 int_status;
-	bool can_run = false, hw_stopped;
-	int slot;
-	unsigned int handled = 0;
-
-	spin_lock(&priv->lock);
-
-	int_status = ioread32(priv->base + VNINTS_REG);
-	if (!int_status)
-		goto done;
-	/* ack interrupts */
-	iowrite32(int_status, priv->base + VNINTS_REG);
-	handled = 1;
-
-	/* nothing to do if capture status is 'STOPPED' */
-	if (priv->state == STOPPED)
-		goto done;
-
-	hw_stopped = !(ioread32(priv->base + VNMS_REG) & VNMS_CA);
-
-	if (!priv->request_to_stop) {
-		if (is_continuous_transfer(priv))
-			slot = (ioread32(priv->base + VNMS_REG) &
-				VNMS_FBS_MASK) >> VNMS_FBS_SHIFT;
-		else
-			slot = 0;
-
-		priv->queue_buf[slot]->field = priv->field;
-		priv->queue_buf[slot]->sequence = priv->sequence++;
-		priv->queue_buf[slot]->vb2_buf.timestamp = ktime_get_ns();
-		vb2_buffer_done(&priv->queue_buf[slot]->vb2_buf,
-				VB2_BUF_STATE_DONE);
-		priv->queue_buf[slot] = NULL;
-
-		if (priv->state != STOPPING)
-			can_run = rcar_vin_fill_hw_slot(priv);
-
-		if (hw_stopped || !can_run) {
-			priv->state = STOPPED;
-		} else if (is_continuous_transfer(priv) &&
-			   list_empty(&priv->capture) &&
-			   priv->state == RUNNING) {
-			/*
-			 * The continuous capturing requires an explicit stop
-			 * operation when there is no buffer to be set into
-			 * the VnMBm registers.
-			 */
-			rcar_vin_request_capture_stop(priv);
-		} else {
-			rcar_vin_capture(priv);
-		}
-
-	} else if (hw_stopped) {
-		priv->state = STOPPED;
-		priv->request_to_stop = false;
-		complete(&priv->capture_stop);
-	}
-
-done:
-	spin_unlock(&priv->lock);
-
-	return IRQ_RETVAL(handled);
-}
-
-static int rcar_vin_add_device(struct soc_camera_device *icd)
-{
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct rcar_vin_priv *priv = ici->priv;
-	int i;
-
-	for (i = 0; i < MAX_BUFFER_NUM; i++)
-		priv->queue_buf[i] = NULL;
-
-	pm_runtime_get_sync(ici->v4l2_dev.dev);
-
-	dev_dbg(icd->parent, "R-Car VIN driver attached to camera %d\n",
-		icd->devnum);
-
-	return 0;
-}
-
-static void rcar_vin_remove_device(struct soc_camera_device *icd)
-{
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct rcar_vin_priv *priv = ici->priv;
-	struct vb2_v4l2_buffer *vbuf;
-	int i;
-
-	/* disable capture, disable interrupts */
-	iowrite32(ioread32(priv->base + VNMC_REG) & ~VNMC_ME,
-		  priv->base + VNMC_REG);
-	iowrite32(0, priv->base + VNIE_REG);
-
-	priv->state = STOPPED;
-	priv->request_to_stop = false;
-
-	/* make sure active buffer is cancelled */
-	spin_lock_irq(&priv->lock);
-	for (i = 0; i < MAX_BUFFER_NUM; i++) {
-		vbuf = priv->queue_buf[i];
-		if (vbuf) {
-			list_del_init(to_buf_list(vbuf));
-			vb2_buffer_done(&vbuf->vb2_buf, VB2_BUF_STATE_ERROR);
-		}
-	}
-	spin_unlock_irq(&priv->lock);
-
-	pm_runtime_put(ici->v4l2_dev.dev);
-
-	dev_dbg(icd->parent, "R-Car VIN driver detached from camera %d\n",
-		icd->devnum);
-}
-
-static void set_coeff(struct rcar_vin_priv *priv, unsigned short xs)
-{
-	int i;
-	const struct vin_coeff *p_prev_set = NULL;
-	const struct vin_coeff *p_set = NULL;
-
-	/* Look for suitable coefficient values */
-	for (i = 0; i < ARRAY_SIZE(vin_coeff_set); i++) {
-		p_prev_set = p_set;
-		p_set = &vin_coeff_set[i];
-
-		if (xs < p_set->xs_value)
-			break;
-	}
-
-	/* Use previous value if its XS value is closer */
-	if (p_prev_set && p_set &&
-	    xs - p_prev_set->xs_value < p_set->xs_value - xs)
-		p_set = p_prev_set;
-
-	/* Set coefficient registers */
-	iowrite32(p_set->coeff_set[0], priv->base + VNC1A_REG);
-	iowrite32(p_set->coeff_set[1], priv->base + VNC1B_REG);
-	iowrite32(p_set->coeff_set[2], priv->base + VNC1C_REG);
-
-	iowrite32(p_set->coeff_set[3], priv->base + VNC2A_REG);
-	iowrite32(p_set->coeff_set[4], priv->base + VNC2B_REG);
-	iowrite32(p_set->coeff_set[5], priv->base + VNC2C_REG);
-
-	iowrite32(p_set->coeff_set[6], priv->base + VNC3A_REG);
-	iowrite32(p_set->coeff_set[7], priv->base + VNC3B_REG);
-	iowrite32(p_set->coeff_set[8], priv->base + VNC3C_REG);
-
-	iowrite32(p_set->coeff_set[9], priv->base + VNC4A_REG);
-	iowrite32(p_set->coeff_set[10], priv->base + VNC4B_REG);
-	iowrite32(p_set->coeff_set[11], priv->base + VNC4C_REG);
-
-	iowrite32(p_set->coeff_set[12], priv->base + VNC5A_REG);
-	iowrite32(p_set->coeff_set[13], priv->base + VNC5B_REG);
-	iowrite32(p_set->coeff_set[14], priv->base + VNC5C_REG);
-
-	iowrite32(p_set->coeff_set[15], priv->base + VNC6A_REG);
-	iowrite32(p_set->coeff_set[16], priv->base + VNC6B_REG);
-	iowrite32(p_set->coeff_set[17], priv->base + VNC6C_REG);
-
-	iowrite32(p_set->coeff_set[18], priv->base + VNC7A_REG);
-	iowrite32(p_set->coeff_set[19], priv->base + VNC7B_REG);
-	iowrite32(p_set->coeff_set[20], priv->base + VNC7C_REG);
-
-	iowrite32(p_set->coeff_set[21], priv->base + VNC8A_REG);
-	iowrite32(p_set->coeff_set[22], priv->base + VNC8B_REG);
-	iowrite32(p_set->coeff_set[23], priv->base + VNC8C_REG);
-}
-
-/* rect is guaranteed to not exceed the scaled camera rectangle */
-static int rcar_vin_set_rect(struct soc_camera_device *icd)
-{
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct rcar_vin_cam *cam = icd->host_priv;
-	struct rcar_vin_priv *priv = ici->priv;
-	unsigned int left_offset, top_offset;
-	unsigned char dsize = 0;
-	struct v4l2_rect *cam_subrect = &cam->subrect;
-	u32 value;
-
-	dev_dbg(icd->parent, "Crop %ux%u@%u:%u\n",
-		icd->user_width, icd->user_height, cam->vin_left, cam->vin_top);
-
-	left_offset = cam->vin_left;
-	top_offset = cam->vin_top;
-
-	if (icd->current_fmt->host_fmt->fourcc == V4L2_PIX_FMT_RGB32 &&
-	    priv->chip == RCAR_E1)
-		dsize = 1;
-
-	dev_dbg(icd->parent, "Cam %ux%u@%u:%u\n",
-		cam->width, cam->height, cam->vin_left, cam->vin_top);
-	dev_dbg(icd->parent, "Cam subrect %ux%u@%u:%u\n",
-		cam_subrect->width, cam_subrect->height,
-		cam_subrect->left, cam_subrect->top);
-
-	/* Set Start/End Pixel/Line Pre-Clip */
-	iowrite32(left_offset << dsize, priv->base + VNSPPRC_REG);
-	iowrite32((left_offset + cam_subrect->width - 1) << dsize,
-		  priv->base + VNEPPRC_REG);
-	switch (priv->field) {
-	case V4L2_FIELD_INTERLACED:
-	case V4L2_FIELD_INTERLACED_TB:
-	case V4L2_FIELD_INTERLACED_BT:
-		iowrite32(top_offset / 2, priv->base + VNSLPRC_REG);
-		iowrite32((top_offset + cam_subrect->height) / 2 - 1,
-			  priv->base + VNELPRC_REG);
-		break;
-	default:
-		iowrite32(top_offset, priv->base + VNSLPRC_REG);
-		iowrite32(top_offset + cam_subrect->height - 1,
-			  priv->base + VNELPRC_REG);
-		break;
-	}
-
-	/* Set scaling coefficient */
-	value = 0;
-	if (cam_subrect->height != cam->out_height)
-		value = (4096 * cam_subrect->height) / cam->out_height;
-	dev_dbg(icd->parent, "YS Value: %x\n", value);
-	iowrite32(value, priv->base + VNYS_REG);
-
-	value = 0;
-	if (cam_subrect->width != cam->out_width)
-		value = (4096 * cam_subrect->width) / cam->out_width;
-
-	/* Horizontal upscaling is up to double size */
-	if (0 < value && value < 2048)
-		value = 2048;
-
-	dev_dbg(icd->parent, "XS Value: %x\n", value);
-	iowrite32(value, priv->base + VNXS_REG);
-
-	/* Horizontal upscaling is carried out by scaling down from double size */
-	if (value < 4096)
-		value *= 2;
-
-	set_coeff(priv, value);
-
-	/* Set Start/End Pixel/Line Post-Clip */
-	iowrite32(0, priv->base + VNSPPOC_REG);
-	iowrite32(0, priv->base + VNSLPOC_REG);
-	iowrite32((cam->out_width - 1) << dsize, priv->base + VNEPPOC_REG);
-	switch (priv->field) {
-	case V4L2_FIELD_INTERLACED:
-	case V4L2_FIELD_INTERLACED_TB:
-	case V4L2_FIELD_INTERLACED_BT:
-		iowrite32(cam->out_height / 2 - 1,
-			  priv->base + VNELPOC_REG);
-		break;
-	default:
-		iowrite32(cam->out_height - 1, priv->base + VNELPOC_REG);
-		break;
-	}
-
-	iowrite32(ALIGN(cam->out_width, 0x10), priv->base + VNIS_REG);
-
-	return 0;
-}
-
-static void capture_stop_preserve(struct rcar_vin_priv *priv, u32 *vnmc)
-{
-	*vnmc = ioread32(priv->base + VNMC_REG);
-	/* module disable */
-	iowrite32(*vnmc & ~VNMC_ME, priv->base + VNMC_REG);
-}
-
-static void capture_restore(struct rcar_vin_priv *priv, u32 vnmc)
-{
-	unsigned long timeout = jiffies + 10 * HZ;
-
-	/*
-	 * Wait until the end of the current frame. It can take a long time,
-	 * but if it has been aborted by a MRST1 reset, it should exit sooner.
-	 */
-	while ((ioread32(priv->base + VNMS_REG) & VNMS_AV) &&
-		time_before(jiffies, timeout))
-		msleep(1);
-
-	if (time_after(jiffies, timeout)) {
-		dev_err(priv->ici.v4l2_dev.dev,
-			"Timeout waiting for frame end! Interface problem?\n");
-		return;
-	}
-
-	iowrite32(vnmc, priv->base + VNMC_REG);
-}
-
-#define VIN_MBUS_FLAGS	(V4L2_MBUS_MASTER |		\
-			 V4L2_MBUS_PCLK_SAMPLE_RISING |	\
-			 V4L2_MBUS_HSYNC_ACTIVE_HIGH |	\
-			 V4L2_MBUS_HSYNC_ACTIVE_LOW |	\
-			 V4L2_MBUS_VSYNC_ACTIVE_HIGH |	\
-			 V4L2_MBUS_VSYNC_ACTIVE_LOW |	\
-			 V4L2_MBUS_DATA_ACTIVE_HIGH)
-
-static int rcar_vin_set_bus_param(struct soc_camera_device *icd)
-{
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct rcar_vin_priv *priv = ici->priv;
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct v4l2_mbus_config cfg;
-	unsigned long common_flags;
-	u32 vnmc;
-	u32 val;
-	int ret;
-
-	capture_stop_preserve(priv, &vnmc);
-
-	ret = v4l2_subdev_call(sd, video, g_mbus_config, &cfg);
-	if (!ret) {
-		common_flags = soc_mbus_config_compatible(&cfg, VIN_MBUS_FLAGS);
-		if (!common_flags) {
-			dev_warn(icd->parent,
-				 "MBUS flags incompatible: camera 0x%x, host 0x%x\n",
-				 cfg.flags, VIN_MBUS_FLAGS);
-			return -EINVAL;
-		}
-	} else if (ret != -ENOIOCTLCMD) {
-		return ret;
-	} else {
-		common_flags = VIN_MBUS_FLAGS;
-	}
-
-	/* Make choises, based on platform preferences */
-	if ((common_flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH) &&
-	    (common_flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)) {
-		if (priv->pdata_flags & RCAR_VIN_HSYNC_ACTIVE_LOW)
-			common_flags &= ~V4L2_MBUS_HSYNC_ACTIVE_HIGH;
-		else
-			common_flags &= ~V4L2_MBUS_HSYNC_ACTIVE_LOW;
-	}
-
-	if ((common_flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH) &&
-	    (common_flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)) {
-		if (priv->pdata_flags & RCAR_VIN_VSYNC_ACTIVE_LOW)
-			common_flags &= ~V4L2_MBUS_VSYNC_ACTIVE_HIGH;
-		else
-			common_flags &= ~V4L2_MBUS_VSYNC_ACTIVE_LOW;
-	}
-
-	cfg.flags = common_flags;
-	ret = v4l2_subdev_call(sd, video, s_mbus_config, &cfg);
-	if (ret < 0 && ret != -ENOIOCTLCMD)
-		return ret;
-
-	val = VNDMR2_FTEV | VNDMR2_VLV(1);
-	if (!(common_flags & V4L2_MBUS_VSYNC_ACTIVE_LOW))
-		val |= VNDMR2_VPS;
-	if (!(common_flags & V4L2_MBUS_HSYNC_ACTIVE_LOW))
-		val |= VNDMR2_HPS;
-	iowrite32(val, priv->base + VNDMR2_REG);
-
-	ret = rcar_vin_set_rect(icd);
-	if (ret < 0)
-		return ret;
-
-	capture_restore(priv, vnmc);
-
-	return 0;
-}
-
-static int rcar_vin_try_bus_param(struct soc_camera_device *icd,
-				  unsigned char buswidth)
-{
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct v4l2_mbus_config cfg;
-	int ret;
-
-	ret = v4l2_subdev_call(sd, video, g_mbus_config, &cfg);
-	if (ret == -ENOIOCTLCMD)
-		return 0;
-	else if (ret)
-		return ret;
-
-	if (buswidth > 24)
-		return -EINVAL;
-
-	/* check is there common mbus flags */
-	ret = soc_mbus_config_compatible(&cfg, VIN_MBUS_FLAGS);
-	if (ret)
-		return 0;
-
-	dev_warn(icd->parent,
-		"MBUS flags incompatible: camera 0x%x, host 0x%x\n",
-		 cfg.flags, VIN_MBUS_FLAGS);
-
-	return -EINVAL;
-}
-
-static bool rcar_vin_packing_supported(const struct soc_mbus_pixelfmt *fmt)
-{
-	return	fmt->packing == SOC_MBUS_PACKING_NONE ||
-		(fmt->bits_per_sample > 8 &&
-		 fmt->packing == SOC_MBUS_PACKING_EXTEND16);
-}
-
-static const struct soc_mbus_pixelfmt rcar_vin_formats[] = {
-	{
-		.fourcc			= V4L2_PIX_FMT_NV16,
-		.name			= "NV16",
-		.bits_per_sample	= 8,
-		.packing		= SOC_MBUS_PACKING_2X8_PADHI,
-		.order			= SOC_MBUS_ORDER_LE,
-		.layout			= SOC_MBUS_LAYOUT_PLANAR_Y_C,
-	},
-	{
-		.fourcc			= V4L2_PIX_FMT_YUYV,
-		.name			= "YUYV",
-		.bits_per_sample	= 16,
-		.packing		= SOC_MBUS_PACKING_NONE,
-		.order			= SOC_MBUS_ORDER_LE,
-		.layout			= SOC_MBUS_LAYOUT_PACKED,
-	},
-	{
-		.fourcc			= V4L2_PIX_FMT_UYVY,
-		.name			= "UYVY",
-		.bits_per_sample	= 16,
-		.packing		= SOC_MBUS_PACKING_NONE,
-		.order			= SOC_MBUS_ORDER_LE,
-		.layout			= SOC_MBUS_LAYOUT_PACKED,
-	},
-	{
-		.fourcc			= V4L2_PIX_FMT_RGB565,
-		.name			= "RGB565",
-		.bits_per_sample	= 16,
-		.packing		= SOC_MBUS_PACKING_NONE,
-		.order			= SOC_MBUS_ORDER_LE,
-		.layout			= SOC_MBUS_LAYOUT_PACKED,
-	},
-	{
-		.fourcc			= V4L2_PIX_FMT_RGB555X,
-		.name			= "ARGB1555",
-		.bits_per_sample	= 16,
-		.packing		= SOC_MBUS_PACKING_NONE,
-		.order			= SOC_MBUS_ORDER_LE,
-		.layout			= SOC_MBUS_LAYOUT_PACKED,
-	},
-	{
-		.fourcc			= V4L2_PIX_FMT_RGB32,
-		.name			= "RGB888",
-		.bits_per_sample	= 32,
-		.packing		= SOC_MBUS_PACKING_NONE,
-		.order			= SOC_MBUS_ORDER_LE,
-		.layout			= SOC_MBUS_LAYOUT_PACKED,
-	},
-	{
-		.fourcc			= V4L2_PIX_FMT_ARGB32,
-		.name			= "ARGB8888",
-		.bits_per_sample	= 32,
-		.packing		= SOC_MBUS_PACKING_NONE,
-		.order			= SOC_MBUS_ORDER_LE,
-		.layout			= SOC_MBUS_LAYOUT_PACKED,
-	},
-};
-
-static int rcar_vin_get_formats(struct soc_camera_device *icd, unsigned int idx,
-				struct soc_camera_format_xlate *xlate)
-{
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct device *dev = icd->parent;
-	int ret, k, n;
-	int formats = 0;
-	struct rcar_vin_cam *cam;
-	struct v4l2_subdev_mbus_code_enum code = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-		.index = idx,
-	};
-	const struct soc_mbus_pixelfmt *fmt;
-
-	ret = v4l2_subdev_call(sd, pad, enum_mbus_code, NULL, &code);
-	if (ret < 0)
-		return 0;
-
-	fmt = soc_mbus_get_fmtdesc(code.code);
-	if (!fmt) {
-		dev_warn(dev, "unsupported format code #%u: %d\n", idx, code.code);
-		return 0;
-	}
-
-	ret = rcar_vin_try_bus_param(icd, fmt->bits_per_sample);
-	if (ret < 0)
-		return 0;
-
-	if (!icd->host_priv) {
-		struct v4l2_subdev_format fmt = {
-			.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-		};
-		struct v4l2_mbus_framefmt *mf = &fmt.format;
-		struct v4l2_rect rect;
-		struct device *dev = icd->parent;
-		int shift;
-
-		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt);
-		if (ret < 0)
-			return ret;
-
-		/* Cache current client geometry */
-		ret = soc_camera_client_g_rect(sd, &rect);
-		if (ret == -ENOIOCTLCMD) {
-			/* Sensor driver doesn't support cropping */
-			rect.left = 0;
-			rect.top = 0;
-			rect.width = mf->width;
-			rect.height = mf->height;
-		} else if (ret < 0) {
-			return ret;
-		}
-
-		/*
-		 * If sensor proposes too large format then try smaller ones:
-		 * 1280x960, 640x480, 320x240
-		 */
-		for (shift = 0; shift < 3; shift++) {
-			if (mf->width <= VIN_MAX_WIDTH &&
-			    mf->height <= VIN_MAX_HEIGHT)
-				break;
-
-			mf->width = 1280 >> shift;
-			mf->height = 960 >> shift;
-			ret = v4l2_device_call_until_err(sd->v4l2_dev,
-							 soc_camera_grp_id(icd),
-							 pad, set_fmt, NULL,
-							 &fmt);
-			if (ret < 0)
-				return ret;
-		}
-
-		if (shift == 3) {
-			dev_err(dev,
-				"Failed to configure the client below %ux%u\n",
-				mf->width, mf->height);
-			return -EIO;
-		}
-
-		dev_dbg(dev, "camera fmt %ux%u\n", mf->width, mf->height);
-
-		cam = kzalloc(sizeof(*cam), GFP_KERNEL);
-		if (!cam)
-			return -ENOMEM;
-		/*
-		 * We are called with current camera crop,
-		 * initialise subrect with it
-		 */
-		cam->rect = rect;
-		cam->subrect = rect;
-		cam->width = mf->width;
-		cam->height = mf->height;
-		cam->out_width	= mf->width;
-		cam->out_height	= mf->height;
-
-		icd->host_priv = cam;
-	} else {
-		cam = icd->host_priv;
-	}
-
-	/* Beginning of a pass */
-	if (!idx)
-		cam->extra_fmt = NULL;
-
-	switch (code.code) {
-	case MEDIA_BUS_FMT_YUYV8_1X16:
-	case MEDIA_BUS_FMT_YUYV8_2X8:
-	case MEDIA_BUS_FMT_YUYV10_2X10:
-	case MEDIA_BUS_FMT_RGB888_1X24:
-		if (cam->extra_fmt)
-			break;
-
-		/* Add all our formats that can be generated by VIN */
-		cam->extra_fmt = rcar_vin_formats;
-
-		n = ARRAY_SIZE(rcar_vin_formats);
-		formats += n;
-		for (k = 0; xlate && k < n; k++, xlate++) {
-			xlate->host_fmt = &rcar_vin_formats[k];
-			xlate->code = code.code;
-			dev_dbg(dev, "Providing format %s using code %d\n",
-				rcar_vin_formats[k].name, code.code);
-		}
-		break;
-	default:
-		if (!rcar_vin_packing_supported(fmt))
-			return 0;
-
-		dev_dbg(dev, "Providing format %s in pass-through mode\n",
-			fmt->name);
-		break;
-	}
-
-	/* Generic pass-through */
-	formats++;
-	if (xlate) {
-		xlate->host_fmt = fmt;
-		xlate->code = code.code;
-		xlate++;
-	}
-
-	return formats;
-}
-
-static void rcar_vin_put_formats(struct soc_camera_device *icd)
-{
-	kfree(icd->host_priv);
-	icd->host_priv = NULL;
-}
-
-static int rcar_vin_set_crop(struct soc_camera_device *icd,
-			     const struct v4l2_crop *a)
-{
-	struct v4l2_crop a_writable = *a;
-	const struct v4l2_rect *rect = &a_writable.c;
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct rcar_vin_priv *priv = ici->priv;
-	struct v4l2_crop cam_crop;
-	struct rcar_vin_cam *cam = icd->host_priv;
-	struct v4l2_rect *cam_rect = &cam_crop.c;
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct device *dev = icd->parent;
-	struct v4l2_subdev_format fmt = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-	struct v4l2_mbus_framefmt *mf = &fmt.format;
-	u32 vnmc;
-	int ret, i;
-
-	dev_dbg(dev, "S_CROP(%ux%u@%u:%u)\n", rect->width, rect->height,
-		rect->left, rect->top);
-
-	/* During camera cropping its output window can change too, stop VIN */
-	capture_stop_preserve(priv, &vnmc);
-	dev_dbg(dev, "VNMC_REG 0x%x\n", vnmc);
-
-	/* Apply iterative camera S_CROP for new input window. */
-	ret = soc_camera_client_s_crop(sd, &a_writable, &cam_crop,
-				       &cam->rect, &cam->subrect);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(dev, "camera cropped to %ux%u@%u:%u\n",
-		cam_rect->width, cam_rect->height,
-		cam_rect->left, cam_rect->top);
-
-	/* On success cam_crop contains current camera crop */
-
-	/* Retrieve camera output window */
-	ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &fmt);
-	if (ret < 0)
-		return ret;
-
-	if (mf->width > VIN_MAX_WIDTH || mf->height > VIN_MAX_HEIGHT)
-		return -EINVAL;
-
-	/* Cache camera output window */
-	cam->width = mf->width;
-	cam->height = mf->height;
-
-	icd->user_width  = cam->width;
-	icd->user_height = cam->height;
-
-	cam->vin_left = rect->left & ~1;
-	cam->vin_top = rect->top & ~1;
-
-	/* Use VIN cropping to crop to the new window. */
-	ret = rcar_vin_set_rect(icd);
-	if (ret < 0)
-		return ret;
-
-	cam->subrect = *rect;
-
-	dev_dbg(dev, "VIN cropped to %ux%u@%u:%u\n",
-		icd->user_width, icd->user_height,
-		cam->vin_left, cam->vin_top);
-
-	/* Restore capture */
-	for (i = 0; i < MAX_BUFFER_NUM; i++) {
-		if (priv->queue_buf[i] && priv->state == STOPPED) {
-			vnmc |= VNMC_ME;
-			break;
-		}
-	}
-	capture_restore(priv, vnmc);
-
-	/* Even if only camera cropping succeeded */
-	return ret;
-}
-
-static int rcar_vin_get_crop(struct soc_camera_device *icd,
-			     struct v4l2_crop *a)
-{
-	struct rcar_vin_cam *cam = icd->host_priv;
-
-	a->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->c = cam->subrect;
-
-	return 0;
-}
-
-/* Similar to set_crop multistage iterative algorithm */
-static int rcar_vin_set_fmt(struct soc_camera_device *icd,
-			    struct v4l2_format *f)
-{
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct rcar_vin_priv *priv = ici->priv;
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct rcar_vin_cam *cam = icd->host_priv;
-	struct v4l2_pix_format *pix = &f->fmt.pix;
-	struct v4l2_mbus_framefmt mf;
-	struct device *dev = icd->parent;
-	__u32 pixfmt = pix->pixelformat;
-	const struct soc_camera_format_xlate *xlate;
-	unsigned int vin_sub_width = 0, vin_sub_height = 0;
-	int ret;
-	bool can_scale;
-	enum v4l2_field field;
-	v4l2_std_id std;
-
-	dev_dbg(dev, "S_FMT(pix=0x%x, %ux%u)\n",
-		pixfmt, pix->width, pix->height);
-
-	switch (pix->field) {
-	default:
-		pix->field = V4L2_FIELD_NONE;
-		/* fall-through */
-	case V4L2_FIELD_NONE:
-	case V4L2_FIELD_TOP:
-	case V4L2_FIELD_BOTTOM:
-	case V4L2_FIELD_INTERLACED_TB:
-	case V4L2_FIELD_INTERLACED_BT:
-		field = pix->field;
-		break;
-	case V4L2_FIELD_INTERLACED:
-		/* Query for standard if not explicitly mentioned _TB/_BT */
-		ret = v4l2_subdev_call(sd, video, querystd, &std);
-		if (ret == -ENOIOCTLCMD) {
-			field = V4L2_FIELD_NONE;
-		} else if (ret < 0) {
-			return ret;
-		} else {
-			field = std & V4L2_STD_625_50 ?
-				V4L2_FIELD_INTERLACED_TB :
-				V4L2_FIELD_INTERLACED_BT;
-		}
-		break;
-	}
-
-	xlate = soc_camera_xlate_by_fourcc(icd, pixfmt);
-	if (!xlate) {
-		dev_warn(dev, "Format %x not found\n", pixfmt);
-		return -EINVAL;
-	}
-	/* Calculate client output geometry */
-	soc_camera_calc_client_output(icd, &cam->rect, &cam->subrect, pix, &mf,
-				      12);
-	mf.field = pix->field;
-	mf.colorspace = pix->colorspace;
-	mf.code	 = xlate->code;
-
-	switch (pixfmt) {
-	case V4L2_PIX_FMT_RGB32:
-		can_scale = priv->chip != RCAR_E1;
-		break;
-	case V4L2_PIX_FMT_ARGB32:
-	case V4L2_PIX_FMT_UYVY:
-	case V4L2_PIX_FMT_YUYV:
-	case V4L2_PIX_FMT_RGB565:
-	case V4L2_PIX_FMT_RGB555X:
-		can_scale = true;
-		break;
-	default:
-		can_scale = false;
-		break;
-	}
-
-	dev_dbg(dev, "request camera output %ux%u\n", mf.width, mf.height);
-
-	ret = soc_camera_client_scale(icd, &cam->rect, &cam->subrect,
-				      &mf, &vin_sub_width, &vin_sub_height,
-				      can_scale, 12);
-
-	/* Done with the camera. Now see if we can improve the result */
-	dev_dbg(dev, "Camera %d fmt %ux%u, requested %ux%u\n",
-		ret, mf.width, mf.height, pix->width, pix->height);
-
-	if (ret == -ENOIOCTLCMD)
-		dev_dbg(dev, "Sensor doesn't support scaling\n");
-	else if (ret < 0)
-		return ret;
-
-	if (mf.code != xlate->code)
-		return -EINVAL;
-
-	/* Prepare VIN crop */
-	cam->width = mf.width;
-	cam->height = mf.height;
-
-	/* Use VIN scaling to scale to the requested user window. */
-
-	/* We cannot scale up */
-	if (pix->width > vin_sub_width)
-		vin_sub_width = pix->width;
-
-	if (pix->height > vin_sub_height)
-		vin_sub_height = pix->height;
-
-	pix->colorspace = mf.colorspace;
-
-	if (!can_scale) {
-		pix->width = vin_sub_width;
-		pix->height = vin_sub_height;
-	}
-
-	/*
-	 * We have calculated CFLCR, the actual configuration will be performed
-	 * in rcar_vin_set_bus_param()
-	 */
-
-	dev_dbg(dev, "W: %u : %u, H: %u : %u\n",
-		vin_sub_width, pix->width, vin_sub_height, pix->height);
-
-	cam->out_width = pix->width;
-	cam->out_height = pix->height;
-
-	icd->current_fmt = xlate;
-
-	priv->field = field;
-
-	return 0;
-}
-
-static int rcar_vin_try_fmt(struct soc_camera_device *icd,
-			    struct v4l2_format *f)
-{
-	const struct soc_camera_format_xlate *xlate;
-	struct v4l2_pix_format *pix = &f->fmt.pix;
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	struct v4l2_subdev_pad_config pad_cfg;
-	struct v4l2_subdev_format format = {
-		.which = V4L2_SUBDEV_FORMAT_TRY,
-	};
-	struct v4l2_mbus_framefmt *mf = &format.format;
-	__u32 pixfmt = pix->pixelformat;
-	int width, height;
-	int ret;
-
-	xlate = soc_camera_xlate_by_fourcc(icd, pixfmt);
-	if (!xlate) {
-		xlate = icd->current_fmt;
-		dev_dbg(icd->parent, "Format %x not found, keeping %x\n",
-			pixfmt, xlate->host_fmt->fourcc);
-		pixfmt = xlate->host_fmt->fourcc;
-		pix->pixelformat = pixfmt;
-		pix->colorspace = icd->colorspace;
-	}
-
-	/* FIXME: calculate using depth and bus width */
-	v4l_bound_align_image(&pix->width, 2, VIN_MAX_WIDTH, 1,
-			      &pix->height, 4, VIN_MAX_HEIGHT, 2, 0);
-
-	width = pix->width;
-	height = pix->height;
-
-	/* let soc-camera calculate these values */
-	pix->bytesperline = 0;
-	pix->sizeimage = 0;
-
-	/* limit to sensor capabilities */
-	mf->width = pix->width;
-	mf->height = pix->height;
-	mf->field = pix->field;
-	mf->code = xlate->code;
-	mf->colorspace = pix->colorspace;
-
-	ret = v4l2_device_call_until_err(sd->v4l2_dev, soc_camera_grp_id(icd),
-					 pad, set_fmt, &pad_cfg, &format);
-	if (ret < 0)
-		return ret;
-
-	/* Adjust only if VIN cannot scale */
-	if (pix->width > mf->width * 2)
-		pix->width = mf->width * 2;
-	if (pix->height > mf->height * 3)
-		pix->height = mf->height * 3;
-
-	pix->field = mf->field;
-	pix->colorspace = mf->colorspace;
-
-	if (pixfmt == V4L2_PIX_FMT_NV16) {
-		/* FIXME: check against rect_max after converting soc-camera */
-		/* We can scale precisely, need a bigger image from camera */
-		if (pix->width < width || pix->height < height) {
-			/*
-			 * We presume, the sensor behaves sanely, i.e. if
-			 * requested a bigger rectangle, it will not return a
-			 * smaller one.
-			 */
-			mf->width = VIN_MAX_WIDTH;
-			mf->height = VIN_MAX_HEIGHT;
-			ret = v4l2_device_call_until_err(sd->v4l2_dev,
-							 soc_camera_grp_id(icd),
-							 pad, set_fmt, &pad_cfg,
-							 &format);
-			if (ret < 0) {
-				dev_err(icd->parent,
-					"client try_fmt() = %d\n", ret);
-				return ret;
-			}
-		}
-		/* We will scale exactly */
-		if (mf->width > width)
-			pix->width = width;
-		if (mf->height > height)
-			pix->height = height;
-	}
-
-	return ret;
-}
-
-static unsigned int rcar_vin_poll(struct file *file, poll_table *pt)
-{
-	struct soc_camera_device *icd = file->private_data;
-
-	return vb2_poll(&icd->vb2_vidq, file, pt);
-}
-
-static int rcar_vin_querycap(struct soc_camera_host *ici,
-			     struct v4l2_capability *cap)
-{
-	strlcpy(cap->card, "R_Car_VIN", sizeof(cap->card));
-	cap->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
-	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
-	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s%d", DRV_NAME, ici->nr);
-
-	return 0;
-}
-
-static int rcar_vin_init_videobuf2(struct vb2_queue *vq,
-				   struct soc_camera_device *icd)
-{
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-
-	vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	vq->io_modes = VB2_MMAP | VB2_USERPTR;
-	vq->drv_priv = icd;
-	vq->ops = &rcar_vin_vb2_ops;
-	vq->mem_ops = &vb2_dma_contig_memops;
-	vq->buf_struct_size = sizeof(struct rcar_vin_buffer);
-	vq->timestamp_flags  = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
-	vq->lock = &ici->host_lock;
-	vq->dev = ici->v4l2_dev.dev;
-
-	return vb2_queue_init(vq);
-}
-
-static struct soc_camera_host_ops rcar_vin_host_ops = {
-	.owner		= THIS_MODULE,
-	.add		= rcar_vin_add_device,
-	.remove		= rcar_vin_remove_device,
-	.get_formats	= rcar_vin_get_formats,
-	.put_formats	= rcar_vin_put_formats,
-	.get_crop	= rcar_vin_get_crop,
-	.set_crop	= rcar_vin_set_crop,
-	.try_fmt	= rcar_vin_try_fmt,
-	.set_fmt	= rcar_vin_set_fmt,
-	.poll		= rcar_vin_poll,
-	.querycap	= rcar_vin_querycap,
-	.set_bus_param	= rcar_vin_set_bus_param,
-	.init_videobuf2	= rcar_vin_init_videobuf2,
-};
-
-#ifdef CONFIG_OF
-static const struct of_device_id rcar_vin_of_table[] = {
-	{ .compatible = "renesas,vin-r8a7795", .data = (void *)RCAR_GEN3 },
-	{ .compatible = "renesas,vin-r8a7794", .data = (void *)RCAR_GEN2 },
-	{ .compatible = "renesas,vin-r8a7793", .data = (void *)RCAR_GEN2 },
-	{ .compatible = "renesas,vin-r8a7791", .data = (void *)RCAR_GEN2 },
-	{ .compatible = "renesas,vin-r8a7790", .data = (void *)RCAR_GEN2 },
-	{ .compatible = "renesas,vin-r8a7779", .data = (void *)RCAR_H1 },
-	{ .compatible = "renesas,vin-r8a7778", .data = (void *)RCAR_M1 },
-	{ .compatible = "renesas,rcar-gen3-vin", .data = (void *)RCAR_GEN3 },
-	{ .compatible = "renesas,rcar-gen2-vin", .data = (void *)RCAR_GEN2 },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, rcar_vin_of_table);
-#endif
-
-static int rcar_vin_probe(struct platform_device *pdev)
-{
-	const struct of_device_id *match = NULL;
-	struct rcar_vin_priv *priv;
-	struct v4l2_of_endpoint ep;
-	struct device_node *np;
-	struct resource *mem;
-	unsigned int pdata_flags;
-	int irq, ret;
-
-	match = of_match_device(of_match_ptr(rcar_vin_of_table), &pdev->dev);
-
-	np = of_graph_get_next_endpoint(pdev->dev.of_node, NULL);
-	if (!np) {
-		dev_err(&pdev->dev, "could not find endpoint\n");
-		return -EINVAL;
-	}
-
-	ret = v4l2_of_parse_endpoint(np, &ep);
-	if (ret) {
-		dev_err(&pdev->dev, "could not parse endpoint\n");
-		return ret;
-	}
-
-	if (ep.bus_type == V4L2_MBUS_BT656)
-		pdata_flags = RCAR_VIN_BT656;
-	else {
-		pdata_flags = 0;
-		if (ep.bus.parallel.flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)
-			pdata_flags |= RCAR_VIN_HSYNC_ACTIVE_LOW;
-		if (ep.bus.parallel.flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
-			pdata_flags |= RCAR_VIN_VSYNC_ACTIVE_LOW;
-	}
-
-	of_node_put(np);
-
-	dev_dbg(&pdev->dev, "pdata_flags = %08x\n", pdata_flags);
-
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (mem == NULL)
-		return -EINVAL;
-
-	irq = platform_get_irq(pdev, 0);
-	if (irq <= 0)
-		return -EINVAL;
-
-	priv = devm_kzalloc(&pdev->dev, sizeof(struct rcar_vin_priv),
-			    GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base = devm_ioremap_resource(&pdev->dev, mem);
-	if (IS_ERR(priv->base))
-		return PTR_ERR(priv->base);
-
-	ret = devm_request_irq(&pdev->dev, irq, rcar_vin_irq, IRQF_SHARED,
-			       dev_name(&pdev->dev), priv);
-	if (ret)
-		return ret;
-
-	priv->ici.priv = priv;
-	priv->ici.v4l2_dev.dev = &pdev->dev;
-	priv->ici.drv_name = dev_name(&pdev->dev);
-	priv->ici.ops = &rcar_vin_host_ops;
-
-	priv->pdata_flags = pdata_flags;
-	if (!match) {
-		priv->ici.nr = pdev->id;
-		priv->chip = pdev->id_entry->driver_data;
-	} else {
-		priv->ici.nr = of_alias_get_id(pdev->dev.of_node, "vin");
-		priv->chip = (enum chip_id)match->data;
-	}
-
-	spin_lock_init(&priv->lock);
-	INIT_LIST_HEAD(&priv->capture);
-
-	priv->state = STOPPED;
-
-	pm_suspend_ignore_children(&pdev->dev, true);
-	pm_runtime_enable(&pdev->dev);
-
-	ret = soc_camera_host_register(&priv->ici);
-	if (ret)
-		goto cleanup;
-
-	return 0;
-
-cleanup:
-	pm_runtime_disable(&pdev->dev);
-
-	return ret;
-}
-
-static int rcar_vin_remove(struct platform_device *pdev)
-{
-	struct soc_camera_host *soc_host = to_soc_camera_host(&pdev->dev);
-
-	soc_camera_host_unregister(soc_host);
-	pm_runtime_disable(&pdev->dev);
-
-	return 0;
-}
-
-static struct platform_driver rcar_vin_driver = {
-	.probe		= rcar_vin_probe,
-	.remove		= rcar_vin_remove,
-	.driver		= {
-		.name		= DRV_NAME,
-		.of_match_table	= of_match_ptr(rcar_vin_of_table),
-	},
-};
-
-module_platform_driver(rcar_vin_driver);
-
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:rcar_vin");
-MODULE_DESCRIPTION("Renesas R-Car VIN camera host driver");
diff --git a/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c b/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c
index 02b519dde42a..a15bfb5aea47 100644
--- a/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c
+++ b/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c
@@ -41,7 +41,6 @@
 #include <media/v4l2-dev.h>
 #include <media/soc_camera.h>
 #include <media/drv-intf/sh_mobile_ceu.h>
-#include <media/drv-intf/sh_mobile_csi2.h>
 #include <media/videobuf2-dma-contig.h>
 #include <media/v4l2-mediabus.h>
 #include <media/drv-intf/soc_mediabus.h>
@@ -99,11 +98,6 @@ struct sh_mobile_ceu_buffer {
 
 struct sh_mobile_ceu_dev {
 	struct soc_camera_host ici;
-	/* Asynchronous CSI2 linking */
-	struct v4l2_async_subdev *csi2_asd;
-	struct v4l2_subdev *csi2_sd;
-	/* Synchronous probing compatibility */
-	struct platform_device *csi2_pdev;
 
 	unsigned int irq;
 	void __iomem *base;
@@ -140,7 +134,7 @@ struct sh_mobile_ceu_cam {
 	unsigned int width;
 	unsigned int height;
 	/*
-	 * User window from S_CROP / G_CROP, produced by client cropping and
+	 * User window from S_SELECTION / G_SELECTION, produced by client cropping and
 	 * scaling, CEU scaling and CEU cropping, mapped back onto the client
 	 * input window
 	 */
@@ -470,7 +464,7 @@ static void sh_mobile_ceu_stop_streaming(struct vb2_queue *q)
 	sh_mobile_ceu_soft_reset(pcdev);
 }
 
-static struct vb2_ops sh_mobile_ceu_videobuf_ops = {
+static const struct vb2_ops sh_mobile_ceu_videobuf_ops = {
 	.queue_setup	= sh_mobile_ceu_videobuf_setup,
 	.buf_prepare	= sh_mobile_ceu_videobuf_prepare,
 	.buf_queue	= sh_mobile_ceu_videobuf_queue,
@@ -517,74 +511,20 @@ out:
 	return IRQ_HANDLED;
 }
 
-static struct v4l2_subdev *find_csi2(struct sh_mobile_ceu_dev *pcdev)
-{
-	struct v4l2_subdev *sd;
-
-	if (pcdev->csi2_sd)
-		return pcdev->csi2_sd;
-
-	if (pcdev->csi2_asd) {
-		char name[] = "sh-mobile-csi2";
-		v4l2_device_for_each_subdev(sd, &pcdev->ici.v4l2_dev)
-			if (!strncmp(name, sd->name, sizeof(name) - 1)) {
-				pcdev->csi2_sd = sd;
-				return sd;
-			}
-	}
-
-	return NULL;
-}
-
-static struct v4l2_subdev *csi2_subdev(struct sh_mobile_ceu_dev *pcdev,
-				       struct soc_camera_device *icd)
-{
-	struct v4l2_subdev *sd = pcdev->csi2_sd;
-
-	return sd && sd->grp_id == soc_camera_grp_id(icd) ? sd : NULL;
-}
-
 static int sh_mobile_ceu_add_device(struct soc_camera_device *icd)
 {
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct sh_mobile_ceu_dev *pcdev = ici->priv;
-	struct v4l2_subdev *csi2_sd = find_csi2(pcdev);
-	int ret;
-
-	if (csi2_sd) {
-		csi2_sd->grp_id = soc_camera_grp_id(icd);
-		v4l2_set_subdev_hostdata(csi2_sd, icd);
-	}
-
-	ret = v4l2_subdev_call(csi2_sd, core, s_power, 1);
-	if (ret < 0 && ret != -ENOIOCTLCMD && ret != -ENODEV)
-		return ret;
-
-	/*
-	 * -ENODEV is special: either csi2_sd == NULL or the CSI-2 driver
-	 * has not found this soc-camera device among its clients
-	 */
-	if (csi2_sd && ret == -ENODEV)
-		csi2_sd->grp_id = 0;
-
 	dev_info(icd->parent,
-		 "SuperH Mobile CEU%s driver attached to camera %d\n",
-		 csi2_sd && csi2_sd->grp_id ? "/CSI-2" : "", icd->devnum);
+		 "SuperH Mobile CEU driver attached to camera %d\n",
+		 icd->devnum);
 
 	return 0;
 }
 
 static void sh_mobile_ceu_remove_device(struct soc_camera_device *icd)
 {
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct sh_mobile_ceu_dev *pcdev = ici->priv;
-	struct v4l2_subdev *csi2_sd = find_csi2(pcdev);
-
 	dev_info(icd->parent,
 		 "SuperH Mobile CEU driver detached from camera %d\n",
 		 icd->devnum);
-
-	v4l2_subdev_call(csi2_sd, core, s_power, 0);
 }
 
 /* Called with .host_lock held */
@@ -704,12 +644,6 @@ static void sh_mobile_ceu_set_rect(struct soc_camera_device *icd)
 		cdwdr_width *= 2;
 	}
 
-	/* CSI2 special configuration */
-	if (csi2_subdev(pcdev, icd)) {
-		in_width = ((in_width - 2) * 2);
-		left_offset *= 2;
-	}
-
 	/* Set CAMOR, CAPWR, CFSZR, take care of CDWDR */
 	camor = left_offset | (top_offset << 16);
 
@@ -758,13 +692,6 @@ static void capture_restore(struct sh_mobile_ceu_dev *pcdev, u32 capsr)
 		ceu_write(pcdev, CAPSR, capsr);
 }
 
-/* Find the bus subdevice driver, e.g., CSI2 */
-static struct v4l2_subdev *find_bus_subdev(struct sh_mobile_ceu_dev *pcdev,
-					   struct soc_camera_device *icd)
-{
-	return csi2_subdev(pcdev, icd) ? : soc_camera_to_subdev(icd);
-}
-
 #define CEU_BUS_FLAGS (V4L2_MBUS_MASTER |	\
 		V4L2_MBUS_PCLK_SAMPLE_RISING |	\
 		V4L2_MBUS_HSYNC_ACTIVE_HIGH |	\
@@ -778,7 +705,7 @@ static int sh_mobile_ceu_set_bus_param(struct soc_camera_device *icd)
 {
 	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
 	struct sh_mobile_ceu_dev *pcdev = ici->priv;
-	struct v4l2_subdev *sd = find_bus_subdev(pcdev, icd);
+	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
 	struct sh_mobile_ceu_cam *cam = icd->host_priv;
 	struct v4l2_mbus_config cfg = {.type = V4L2_MBUS_PARALLEL,};
 	unsigned long value, common_flags = CEU_BUS_FLAGS;
@@ -866,9 +793,7 @@ static int sh_mobile_ceu_set_bus_param(struct soc_camera_device *icd)
 	value |= common_flags & V4L2_MBUS_VSYNC_ACTIVE_LOW ? 1 << 1 : 0;
 	value |= common_flags & V4L2_MBUS_HSYNC_ACTIVE_LOW ? 1 << 0 : 0;
 
-	if (csi2_subdev(pcdev, icd)) /* CSI2 mode */
-		value |= 3 << 12;
-	else if (pcdev->is_16bit)
+	if (pcdev->is_16bit)
 		value |= 1 << 12;
 	else if (pcdev->flags & SH_CEU_FLAG_LOWER_8BIT)
 		value |= 2 << 12;
@@ -923,9 +848,7 @@ static int sh_mobile_ceu_set_bus_param(struct soc_camera_device *icd)
 static int sh_mobile_ceu_try_bus_param(struct soc_camera_device *icd,
 				       unsigned char buswidth)
 {
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	struct sh_mobile_ceu_dev *pcdev = ici->priv;
-	struct v4l2_subdev *sd = find_bus_subdev(pcdev, icd);
+	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
 	unsigned long common_flags = CEU_BUS_FLAGS;
 	struct v4l2_mbus_config cfg = {.type = V4L2_MBUS_PARALLEL,};
 	int ret;
@@ -1046,12 +969,9 @@ static int sh_mobile_ceu_get_formats(struct soc_camera_device *icd, unsigned int
 		return 0;
 	}
 
-	if (!csi2_subdev(pcdev, icd)) {
-		/* Are there any restrictions in the CSI-2 case? */
-		ret = sh_mobile_ceu_try_bus_param(icd, fmt->bits_per_sample);
-		if (ret < 0)
-			return 0;
-	}
+	ret = sh_mobile_ceu_try_bus_param(icd, fmt->bits_per_sample);
+	if (ret < 0)
+		return 0;
 
 	if (!icd->host_priv) {
 		struct v4l2_subdev_format fmt = {
@@ -1189,17 +1109,16 @@ static void sh_mobile_ceu_put_formats(struct soc_camera_device *icd)
  * Documentation/video4linux/sh_mobile_ceu_camera.txt for a description of
  * scaling and cropping algorithms and for the meaning of referenced here steps.
  */
-static int sh_mobile_ceu_set_crop(struct soc_camera_device *icd,
-				  const struct v4l2_crop *a)
+static int sh_mobile_ceu_set_selection(struct soc_camera_device *icd,
+				       struct v4l2_selection *sel)
 {
-	struct v4l2_crop a_writable = *a;
-	const struct v4l2_rect *rect = &a_writable.c;
+	struct v4l2_rect *rect = &sel->r;
 	struct device *dev = icd->parent;
 	struct soc_camera_host *ici = to_soc_camera_host(dev);
 	struct sh_mobile_ceu_dev *pcdev = ici->priv;
-	struct v4l2_crop cam_crop;
+	struct v4l2_selection cam_sel;
 	struct sh_mobile_ceu_cam *cam = icd->host_priv;
-	struct v4l2_rect *cam_rect = &cam_crop.c;
+	struct v4l2_rect *cam_rect = &cam_sel.r;
 	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
 	struct v4l2_subdev_format fmt = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
@@ -1211,7 +1130,7 @@ static int sh_mobile_ceu_set_crop(struct soc_camera_device *icd,
 	u32 capsr, cflcr;
 	int ret;
 
-	dev_geo(dev, "S_CROP(%ux%u@%u:%u)\n", rect->width, rect->height,
+	dev_geo(dev, "S_SELECTION(%ux%u@%u:%u)\n", rect->width, rect->height,
 		rect->left, rect->top);
 
 	/* During camera cropping its output window can change too, stop CEU */
@@ -1219,10 +1138,10 @@ static int sh_mobile_ceu_set_crop(struct soc_camera_device *icd,
 	dev_dbg(dev, "CAPSR 0x%x, CFLCR 0x%x\n", capsr, pcdev->cflcr);
 
 	/*
-	 * 1. - 2. Apply iterative camera S_CROP for new input window, read back
+	 * 1. - 2. Apply iterative camera S_SELECTION for new input window, read back
 	 * actual camera rectangle.
 	 */
-	ret = soc_camera_client_s_crop(sd, &a_writable, &cam_crop,
+	ret = soc_camera_client_s_selection(sd, sel, &cam_sel,
 				       &cam->rect, &cam->subrect);
 	if (ret < 0)
 		return ret;
@@ -1331,13 +1250,12 @@ static int sh_mobile_ceu_set_crop(struct soc_camera_device *icd,
 	return ret;
 }
 
-static int sh_mobile_ceu_get_crop(struct soc_camera_device *icd,
-				  struct v4l2_crop *a)
+static int sh_mobile_ceu_get_selection(struct soc_camera_device *icd,
+				       struct v4l2_selection *sel)
 {
 	struct sh_mobile_ceu_cam *cam = icd->host_priv;
 
-	a->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->c = cam->subrect;
+	sel->r = cam->subrect;
 
 	return 0;
 }
@@ -1579,8 +1497,8 @@ static int sh_mobile_ceu_try_fmt(struct soc_camera_device *icd,
 	return ret;
 }
 
-static int sh_mobile_ceu_set_livecrop(struct soc_camera_device *icd,
-				      const struct v4l2_crop *a)
+static int sh_mobile_ceu_set_liveselection(struct soc_camera_device *icd,
+					   struct v4l2_selection *sel)
 {
 	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
 	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
@@ -1599,7 +1517,7 @@ static int sh_mobile_ceu_set_livecrop(struct soc_camera_device *icd,
 			 "Client failed to stop the stream: %d\n", ret);
 	else
 		/* Do the crop, if it fails, there's nothing more we can do */
-		sh_mobile_ceu_set_crop(icd, a);
+		sh_mobile_ceu_set_selection(icd, sel);
 
 	dev_geo(icd->parent, "Output after crop: %ux%u\n", icd->user_width, icd->user_height);
 
@@ -1680,9 +1598,9 @@ static struct soc_camera_host_ops sh_mobile_ceu_host_ops = {
 	.clock_stop	= sh_mobile_ceu_clock_stop,
 	.get_formats	= sh_mobile_ceu_get_formats,
 	.put_formats	= sh_mobile_ceu_put_formats,
-	.get_crop	= sh_mobile_ceu_get_crop,
-	.set_crop	= sh_mobile_ceu_set_crop,
-	.set_livecrop	= sh_mobile_ceu_set_livecrop,
+	.get_selection	= sh_mobile_ceu_get_selection,
+	.set_selection	= sh_mobile_ceu_set_selection,
+	.set_liveselection	= sh_mobile_ceu_set_liveselection,
 	.set_fmt	= sh_mobile_ceu_set_fmt,
 	.try_fmt	= sh_mobile_ceu_try_fmt,
 	.poll		= sh_mobile_ceu_poll,
@@ -1721,12 +1639,11 @@ static int sh_mobile_ceu_probe(struct platform_device *pdev)
 	struct resource *res;
 	void __iomem *base;
 	unsigned int irq;
-	int err, i;
+	int err;
 	struct bus_wait wait = {
 		.completion = COMPLETION_INITIALIZER_ONSTACK(wait.completion),
 		.notifier.notifier_call = bus_notify,
 	};
-	struct sh_mobile_ceu_companion *csi2;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	irq = platform_get_irq(pdev, 0);
@@ -1821,132 +1738,16 @@ static int sh_mobile_ceu_probe(struct platform_device *pdev)
 	pcdev->ici.capabilities = SOCAM_HOST_CAP_STRIDE;
 
 	if (pcdev->pdata && pcdev->pdata->asd_sizes) {
-		struct v4l2_async_subdev **asd;
-		char name[] = "sh-mobile-csi2";
-		int j;
-
-		/*
-		 * CSI2 interfacing: several groups can use CSI2, pick up the
-		 * first one
-		 */
-		asd = pcdev->pdata->asd;
-		for (j = 0; pcdev->pdata->asd_sizes[j]; j++) {
-			for (i = 0; i < pcdev->pdata->asd_sizes[j]; i++, asd++) {
-				dev_dbg(&pdev->dev, "%s(): subdev #%d, type %u\n",
-					__func__, i, (*asd)->match_type);
-				if ((*asd)->match_type == V4L2_ASYNC_MATCH_DEVNAME &&
-				    !strncmp(name, (*asd)->match.device_name.name,
-					     sizeof(name) - 1)) {
-					pcdev->csi2_asd = *asd;
-					break;
-				}
-			}
-			if (pcdev->csi2_asd)
-				break;
-		}
-
 		pcdev->ici.asd = pcdev->pdata->asd;
 		pcdev->ici.asd_sizes = pcdev->pdata->asd_sizes;
 	}
 
-	/* Legacy CSI2 interfacing */
-	csi2 = pcdev->pdata ? pcdev->pdata->csi2 : NULL;
-	if (csi2) {
-		/*
-		 * TODO: remove this once all users are converted to
-		 * asynchronous CSI2 probing. If it has to be kept, csi2
-		 * platform device resources have to be added, using
-		 * platform_device_add_resources()
-		 */
-		struct platform_device *csi2_pdev =
-			platform_device_alloc("sh-mobile-csi2", csi2->id);
-		struct sh_csi2_pdata *csi2_pdata = csi2->platform_data;
-
-		if (!csi2_pdev) {
-			err = -ENOMEM;
-			goto exit_free_clk;
-		}
-
-		pcdev->csi2_pdev		= csi2_pdev;
-
-		err = platform_device_add_data(csi2_pdev, csi2_pdata,
-					       sizeof(*csi2_pdata));
-		if (err < 0)
-			goto exit_pdev_put;
-
-		csi2_pdev->resource		= csi2->resource;
-		csi2_pdev->num_resources	= csi2->num_resources;
-
-		err = platform_device_add(csi2_pdev);
-		if (err < 0)
-			goto exit_pdev_put;
-
-		wait.dev = &csi2_pdev->dev;
-
-		err = bus_register_notifier(&platform_bus_type, &wait.notifier);
-		if (err < 0)
-			goto exit_pdev_unregister;
-
-		/*
-		 * From this point the driver module will not unload, until
-		 * we complete the completion.
-		 */
-
-		if (!csi2_pdev->dev.driver) {
-			complete(&wait.completion);
-			/* Either too late, or probing failed */
-			bus_unregister_notifier(&platform_bus_type, &wait.notifier);
-			err = -ENXIO;
-			goto exit_pdev_unregister;
-		}
-
-		/*
-		 * The module is still loaded, in the worst case it is hanging
-		 * in device release on our completion. So, _now_ dereferencing
-		 * the "owner" is safe!
-		 */
-
-		err = try_module_get(csi2_pdev->dev.driver->owner);
-
-		/* Let notifier complete, if it has been locked */
-		complete(&wait.completion);
-		bus_unregister_notifier(&platform_bus_type, &wait.notifier);
-		if (!err) {
-			err = -ENODEV;
-			goto exit_pdev_unregister;
-		}
-
-		pcdev->csi2_sd = platform_get_drvdata(csi2_pdev);
-	}
-
 	err = soc_camera_host_register(&pcdev->ici);
 	if (err)
-		goto exit_csi2_unregister;
-
-	if (csi2) {
-		err = v4l2_device_register_subdev(&pcdev->ici.v4l2_dev,
-						  pcdev->csi2_sd);
-		dev_dbg(&pdev->dev, "%s(): ret(register_subdev) = %d\n",
-			__func__, err);
-		if (err < 0)
-			goto exit_host_unregister;
-		/* v4l2_device_register_subdev() took a reference too */
-		module_put(pcdev->csi2_sd->owner);
-	}
+		goto exit_free_clk;
 
 	return 0;
 
-exit_host_unregister:
-	soc_camera_host_unregister(&pcdev->ici);
-exit_csi2_unregister:
-	if (csi2) {
-		module_put(pcdev->csi2_pdev->dev.driver->owner);
-exit_pdev_unregister:
-		platform_device_del(pcdev->csi2_pdev);
-exit_pdev_put:
-		pcdev->csi2_pdev->resource = NULL;
-		platform_device_put(pcdev->csi2_pdev);
-	}
 exit_free_clk:
 	pm_runtime_disable(&pdev->dev);
 exit_release_mem:
@@ -1958,21 +1759,11 @@ exit_release_mem:
 static int sh_mobile_ceu_remove(struct platform_device *pdev)
 {
 	struct soc_camera_host *soc_host = to_soc_camera_host(&pdev->dev);
-	struct sh_mobile_ceu_dev *pcdev = container_of(soc_host,
-					struct sh_mobile_ceu_dev, ici);
-	struct platform_device *csi2_pdev = pcdev->csi2_pdev;
 
 	soc_camera_host_unregister(soc_host);
 	pm_runtime_disable(&pdev->dev);
 	if (platform_get_resource(pdev, IORESOURCE_MEM, 1))
 		dma_release_declared_memory(&pdev->dev);
-	if (csi2_pdev && csi2_pdev->dev.driver) {
-		struct module *csi2_drv = csi2_pdev->dev.driver->owner;
-		platform_device_del(csi2_pdev);
-		csi2_pdev->resource = NULL;
-		platform_device_put(csi2_pdev);
-		module_put(csi2_drv);
-	}
 
 	return 0;
 }
@@ -2012,8 +1803,6 @@ static struct platform_driver sh_mobile_ceu_driver = {
 
 static int __init sh_mobile_ceu_init(void)
 {
-	/* Whatever return code */
-	request_module("sh_mobile_csi2");
 	return platform_driver_register(&sh_mobile_ceu_driver);
 }
 
diff --git a/drivers/media/platform/soc_camera/sh_mobile_csi2.c b/drivers/media/platform/soc_camera/sh_mobile_csi2.c
deleted file mode 100644
index 09b18365a4b1..000000000000
--- a/drivers/media/platform/soc_camera/sh_mobile_csi2.c
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Driver for the SH-Mobile MIPI CSI-2 unit
- *
- * Copyright (C) 2010, Guennadi Liakhovetski <g.liakhovetski@gmx.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/delay.h>
-#include <linux/err.h>
-#include <linux/i2c.h>
-#include <linux/io.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/slab.h>
-#include <linux/videodev2.h>
-#include <linux/module.h>
-
-#include <media/drv-intf/sh_mobile_ceu.h>
-#include <media/drv-intf/sh_mobile_csi2.h>
-#include <media/soc_camera.h>
-#include <media/drv-intf/soc_mediabus.h>
-#include <media/v4l2-common.h>
-#include <media/v4l2-dev.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-mediabus.h>
-#include <media/v4l2-subdev.h>
-
-#define SH_CSI2_TREF	0x00
-#define SH_CSI2_SRST	0x04
-#define SH_CSI2_PHYCNT	0x08
-#define SH_CSI2_CHKSUM	0x0C
-#define SH_CSI2_VCDT	0x10
-
-struct sh_csi2 {
-	struct v4l2_subdev		subdev;
-	unsigned int			irq;
-	unsigned long			mipi_flags;
-	void __iomem			*base;
-	struct platform_device		*pdev;
-	struct sh_csi2_client_config	*client;
-};
-
-static void sh_csi2_hwinit(struct sh_csi2 *priv);
-
-static int sh_csi2_set_fmt(struct v4l2_subdev *sd,
-		struct v4l2_subdev_pad_config *cfg,
-		struct v4l2_subdev_format *format)
-{
-	struct sh_csi2 *priv = container_of(sd, struct sh_csi2, subdev);
-	struct sh_csi2_pdata *pdata = priv->pdev->dev.platform_data;
-	struct v4l2_mbus_framefmt *mf = &format->format;
-	u32 tmp = (priv->client->channel & 3) << 8;
-
-	if (format->pad)
-		return -EINVAL;
-
-	if (mf->width > 8188)
-		mf->width = 8188;
-	else if (mf->width & 1)
-		mf->width &= ~1;
-
-	switch (pdata->type) {
-	case SH_CSI2C:
-		switch (mf->code) {
-		case MEDIA_BUS_FMT_UYVY8_2X8:		/* YUV422 */
-		case MEDIA_BUS_FMT_YUYV8_1_5X8:		/* YUV420 */
-		case MEDIA_BUS_FMT_Y8_1X8:		/* RAW8 */
-		case MEDIA_BUS_FMT_SBGGR8_1X8:
-		case MEDIA_BUS_FMT_SGRBG8_1X8:
-			break;
-		default:
-			/* All MIPI CSI-2 devices must support one of primary formats */
-			mf->code = MEDIA_BUS_FMT_YUYV8_2X8;
-		}
-		break;
-	case SH_CSI2I:
-		switch (mf->code) {
-		case MEDIA_BUS_FMT_Y8_1X8:		/* RAW8 */
-		case MEDIA_BUS_FMT_SBGGR8_1X8:
-		case MEDIA_BUS_FMT_SGRBG8_1X8:
-		case MEDIA_BUS_FMT_SBGGR10_1X10:	/* RAW10 */
-		case MEDIA_BUS_FMT_SBGGR12_1X12:	/* RAW12 */
-			break;
-		default:
-			/* All MIPI CSI-2 devices must support one of primary formats */
-			mf->code = MEDIA_BUS_FMT_SBGGR8_1X8;
-		}
-		break;
-	}
-
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
-		cfg->try_fmt = *mf;
-		return 0;
-	}
-
-	if (mf->width > 8188 || mf->width & 1)
-		return -EINVAL;
-
-	switch (mf->code) {
-	case MEDIA_BUS_FMT_UYVY8_2X8:
-		tmp |= 0x1e;	/* YUV422 8 bit */
-		break;
-	case MEDIA_BUS_FMT_YUYV8_1_5X8:
-		tmp |= 0x18;	/* YUV420 8 bit */
-		break;
-	case MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE:
-		tmp |= 0x21;	/* RGB555 */
-		break;
-	case MEDIA_BUS_FMT_RGB565_2X8_BE:
-		tmp |= 0x22;	/* RGB565 */
-		break;
-	case MEDIA_BUS_FMT_Y8_1X8:
-	case MEDIA_BUS_FMT_SBGGR8_1X8:
-	case MEDIA_BUS_FMT_SGRBG8_1X8:
-		tmp |= 0x2a;	/* RAW8 */
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	iowrite32(tmp, priv->base + SH_CSI2_VCDT);
-
-	return 0;
-}
-
-static int sh_csi2_g_mbus_config(struct v4l2_subdev *sd,
-				 struct v4l2_mbus_config *cfg)
-{
-	struct sh_csi2 *priv = container_of(sd, struct sh_csi2, subdev);
-
-	if (!priv->mipi_flags) {
-		struct soc_camera_device *icd = v4l2_get_subdev_hostdata(sd);
-		struct v4l2_subdev *client_sd = soc_camera_to_subdev(icd);
-		struct sh_csi2_pdata *pdata = priv->pdev->dev.platform_data;
-		unsigned long common_flags, csi2_flags;
-		struct v4l2_mbus_config client_cfg = {.type = V4L2_MBUS_CSI2,};
-		int ret;
-
-		/* Check if we can support this camera */
-		csi2_flags = V4L2_MBUS_CSI2_CONTINUOUS_CLOCK |
-			V4L2_MBUS_CSI2_1_LANE;
-
-		switch (pdata->type) {
-		case SH_CSI2C:
-			if (priv->client->lanes != 1)
-				csi2_flags |= V4L2_MBUS_CSI2_2_LANE;
-			break;
-		case SH_CSI2I:
-			switch (priv->client->lanes) {
-			default:
-				csi2_flags |= V4L2_MBUS_CSI2_4_LANE;
-			case 3:
-				csi2_flags |= V4L2_MBUS_CSI2_3_LANE;
-			case 2:
-				csi2_flags |= V4L2_MBUS_CSI2_2_LANE;
-			}
-		}
-
-		ret = v4l2_subdev_call(client_sd, video, g_mbus_config, &client_cfg);
-		if (ret == -ENOIOCTLCMD)
-			common_flags = csi2_flags;
-		else if (!ret)
-			common_flags = soc_mbus_config_compatible(&client_cfg,
-								  csi2_flags);
-		else
-			common_flags = 0;
-
-		if (!common_flags)
-			return -EINVAL;
-
-		/* All good: camera MIPI configuration supported */
-		priv->mipi_flags = common_flags;
-	}
-
-	if (cfg) {
-		cfg->flags = V4L2_MBUS_PCLK_SAMPLE_RISING |
-			V4L2_MBUS_HSYNC_ACTIVE_HIGH | V4L2_MBUS_VSYNC_ACTIVE_HIGH |
-			V4L2_MBUS_MASTER | V4L2_MBUS_DATA_ACTIVE_HIGH;
-		cfg->type = V4L2_MBUS_PARALLEL;
-	}
-
-	return 0;
-}
-
-static int sh_csi2_s_mbus_config(struct v4l2_subdev *sd,
-				 const struct v4l2_mbus_config *cfg)
-{
-	struct sh_csi2 *priv = container_of(sd, struct sh_csi2, subdev);
-	struct soc_camera_device *icd = v4l2_get_subdev_hostdata(sd);
-	struct v4l2_subdev *client_sd = soc_camera_to_subdev(icd);
-	struct v4l2_mbus_config client_cfg = {.type = V4L2_MBUS_CSI2,};
-	int ret = sh_csi2_g_mbus_config(sd, NULL);
-
-	if (ret < 0)
-		return ret;
-
-	pm_runtime_get_sync(&priv->pdev->dev);
-
-	sh_csi2_hwinit(priv);
-
-	client_cfg.flags = priv->mipi_flags;
-
-	return v4l2_subdev_call(client_sd, video, s_mbus_config, &client_cfg);
-}
-
-static struct v4l2_subdev_video_ops sh_csi2_subdev_video_ops = {
-	.g_mbus_config	= sh_csi2_g_mbus_config,
-	.s_mbus_config	= sh_csi2_s_mbus_config,
-};
-
-static struct v4l2_subdev_pad_ops sh_csi2_subdev_pad_ops = {
-	.set_fmt	= sh_csi2_set_fmt,
-};
-
-static void sh_csi2_hwinit(struct sh_csi2 *priv)
-{
-	struct sh_csi2_pdata *pdata = priv->pdev->dev.platform_data;
-	__u32 tmp = 0x10; /* Enable MIPI CSI clock lane */
-
-	/* Reflect registers immediately */
-	iowrite32(0x00000001, priv->base + SH_CSI2_TREF);
-	/* reset CSI2 harware */
-	iowrite32(0x00000001, priv->base + SH_CSI2_SRST);
-	udelay(5);
-	iowrite32(0x00000000, priv->base + SH_CSI2_SRST);
-
-	switch (pdata->type) {
-	case SH_CSI2C:
-		if (priv->client->lanes == 1)
-			tmp |= 1;
-		else
-			/* Default - both lanes */
-			tmp |= 3;
-		break;
-	case SH_CSI2I:
-		if (!priv->client->lanes || priv->client->lanes > 4)
-			/* Default - all 4 lanes */
-			tmp |= 0xf;
-		else
-			tmp |= (1 << priv->client->lanes) - 1;
-	}
-
-	if (priv->client->phy == SH_CSI2_PHY_MAIN)
-		tmp |= 0x8000;
-
-	iowrite32(tmp, priv->base + SH_CSI2_PHYCNT);
-
-	tmp = 0;
-	if (pdata->flags & SH_CSI2_ECC)
-		tmp |= 2;
-	if (pdata->flags & SH_CSI2_CRC)
-		tmp |= 1;
-	iowrite32(tmp, priv->base + SH_CSI2_CHKSUM);
-}
-
-static int sh_csi2_client_connect(struct sh_csi2 *priv)
-{
-	struct device *dev = v4l2_get_subdevdata(&priv->subdev);
-	struct sh_csi2_pdata *pdata = dev->platform_data;
-	struct soc_camera_device *icd = v4l2_get_subdev_hostdata(&priv->subdev);
-	int i;
-
-	if (priv->client)
-		return -EBUSY;
-
-	for (i = 0; i < pdata->num_clients; i++)
-		if ((pdata->clients[i].pdev &&
-		     &pdata->clients[i].pdev->dev == icd->pdev) ||
-		    (icd->control &&
-		     strcmp(pdata->clients[i].name, dev_name(icd->control))))
-			break;
-
-	dev_dbg(dev, "%s(%p): found #%d\n", __func__, dev, i);
-
-	if (i == pdata->num_clients)
-		return -ENODEV;
-
-	priv->client = pdata->clients + i;
-
-	return 0;
-}
-
-static void sh_csi2_client_disconnect(struct sh_csi2 *priv)
-{
-	if (!priv->client)
-		return;
-
-	priv->client = NULL;
-
-	pm_runtime_put(v4l2_get_subdevdata(&priv->subdev));
-}
-
-static int sh_csi2_s_power(struct v4l2_subdev *sd, int on)
-{
-	struct sh_csi2 *priv = container_of(sd, struct sh_csi2, subdev);
-
-	if (on)
-		return sh_csi2_client_connect(priv);
-
-	sh_csi2_client_disconnect(priv);
-	return 0;
-}
-
-static struct v4l2_subdev_core_ops sh_csi2_subdev_core_ops = {
-	.s_power	= sh_csi2_s_power,
-};
-
-static struct v4l2_subdev_ops sh_csi2_subdev_ops = {
-	.core	= &sh_csi2_subdev_core_ops,
-	.video	= &sh_csi2_subdev_video_ops,
-	.pad	= &sh_csi2_subdev_pad_ops,
-};
-
-static int sh_csi2_probe(struct platform_device *pdev)
-{
-	struct resource *res;
-	unsigned int irq;
-	int ret;
-	struct sh_csi2 *priv;
-	/* Platform data specify the PHY, lanes, ECC, CRC */
-	struct sh_csi2_pdata *pdata = pdev->dev.platform_data;
-
-	if (!pdata)
-		return -EINVAL;
-
-	priv = devm_kzalloc(&pdev->dev, sizeof(struct sh_csi2), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	/* Interrupt unused so far */
-	irq = platform_get_irq(pdev, 0);
-
-	if (!res || (int)irq <= 0) {
-		dev_err(&pdev->dev, "Not enough CSI2 platform resources.\n");
-		return -ENODEV;
-	}
-
-	/* TODO: Add support for CSI2I. Careful: different register layout! */
-	if (pdata->type != SH_CSI2C) {
-		dev_err(&pdev->dev, "Only CSI2C supported ATM.\n");
-		return -EINVAL;
-	}
-
-	priv->irq = irq;
-
-	priv->base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(priv->base))
-		return PTR_ERR(priv->base);
-
-	priv->pdev = pdev;
-	priv->subdev.owner = THIS_MODULE;
-	priv->subdev.dev = &pdev->dev;
-	platform_set_drvdata(pdev, &priv->subdev);
-
-	v4l2_subdev_init(&priv->subdev, &sh_csi2_subdev_ops);
-	v4l2_set_subdevdata(&priv->subdev, &pdev->dev);
-
-	snprintf(priv->subdev.name, V4L2_SUBDEV_NAME_SIZE, "%s.mipi-csi",
-		 dev_name(&pdev->dev));
-
-	ret = v4l2_async_register_subdev(&priv->subdev);
-	if (ret < 0)
-		return ret;
-
-	pm_runtime_enable(&pdev->dev);
-
-	dev_dbg(&pdev->dev, "CSI2 probed.\n");
-
-	return 0;
-}
-
-static int sh_csi2_remove(struct platform_device *pdev)
-{
-	struct v4l2_subdev *subdev = platform_get_drvdata(pdev);
-	struct sh_csi2 *priv = container_of(subdev, struct sh_csi2, subdev);
-
-	v4l2_async_unregister_subdev(&priv->subdev);
-	pm_runtime_disable(&pdev->dev);
-
-	return 0;
-}
-
-static struct platform_driver __refdata sh_csi2_pdrv = {
-	.remove	= sh_csi2_remove,
-	.probe	= sh_csi2_probe,
-	.driver	= {
-		.name	= "sh-mobile-csi2",
-	},
-};
-
-module_platform_driver(sh_csi2_pdrv);
-
-MODULE_DESCRIPTION("SH-Mobile MIPI CSI-2 driver");
-MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("platform:sh-mobile-csi2");
diff --git a/drivers/media/platform/soc_camera/soc_camera.c b/drivers/media/platform/soc_camera/soc_camera.c
index 46c7186f7867..edd1c1de4e33 100644
--- a/drivers/media/platform/soc_camera/soc_camera.c
+++ b/drivers/media/platform/soc_camera/soc_camera.c
@@ -581,7 +581,7 @@ static int soc_camera_set_fmt(struct soc_camera_device *icd,
 	dev_dbg(icd->pdev, "S_FMT(%c%c%c%c, %ux%u)\n",
 		pixfmtstr(pix->pixelformat), pix->width, pix->height);
 
-	/* We always call try_fmt() before set_fmt() or set_crop() */
+	/* We always call try_fmt() before set_fmt() or set_selection() */
 	ret = soc_camera_try_fmt(icd, f);
 	if (ret < 0)
 		return ret;
@@ -1025,72 +1025,6 @@ static int soc_camera_streamoff(struct file *file, void *priv,
 	return ret;
 }
 
-static int soc_camera_cropcap(struct file *file, void *fh,
-			      struct v4l2_cropcap *a)
-{
-	struct soc_camera_device *icd = file->private_data;
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-
-	return ici->ops->cropcap(icd, a);
-}
-
-static int soc_camera_g_crop(struct file *file, void *fh,
-			     struct v4l2_crop *a)
-{
-	struct soc_camera_device *icd = file->private_data;
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	int ret;
-
-	ret = ici->ops->get_crop(icd, a);
-
-	return ret;
-}
-
-/*
- * According to the V4L2 API, drivers shall not update the struct v4l2_crop
- * argument with the actual geometry, instead, the user shall use G_CROP to
- * retrieve it.
- */
-static int soc_camera_s_crop(struct file *file, void *fh,
-			     const struct v4l2_crop *a)
-{
-	struct soc_camera_device *icd = file->private_data;
-	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	const struct v4l2_rect *rect = &a->c;
-	struct v4l2_crop current_crop;
-	int ret;
-
-	if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-
-	dev_dbg(icd->pdev, "S_CROP(%ux%u@%u:%u)\n",
-		rect->width, rect->height, rect->left, rect->top);
-
-	current_crop.type = a->type;
-
-	/* If get_crop fails, we'll let host and / or client drivers decide */
-	ret = ici->ops->get_crop(icd, &current_crop);
-
-	/* Prohibit window size change with initialised buffers */
-	if (ret < 0) {
-		dev_err(icd->pdev,
-			"S_CROP denied: getting current crop failed\n");
-	} else if ((a->c.width == current_crop.c.width &&
-		    a->c.height == current_crop.c.height) ||
-		   !is_streaming(ici, icd)) {
-		/* same size or not streaming - use .set_crop() */
-		ret = ici->ops->set_crop(icd, a);
-	} else if (ici->ops->set_livecrop) {
-		ret = ici->ops->set_livecrop(icd, a);
-	} else {
-		dev_err(icd->pdev,
-			"S_CROP denied: queue initialised and sizes differ\n");
-		ret = -EBUSY;
-	}
-
-	return ret;
-}
-
 static int soc_camera_g_selection(struct file *file, void *fh,
 				  struct v4l2_selection *s)
 {
@@ -1101,9 +1035,6 @@ static int soc_camera_g_selection(struct file *file, void *fh,
 	if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
-	if (!ici->ops->get_selection)
-		return -ENOTTY;
-
 	return ici->ops->get_selection(icd, s);
 }
 
@@ -1135,10 +1066,11 @@ static int soc_camera_s_selection(struct file *file, void *fh,
 			return -EBUSY;
 	}
 
-	if (!ici->ops->set_selection)
-		return -ENOTTY;
-
-	ret = ici->ops->set_selection(icd, s);
+	if (s->target == V4L2_SEL_TGT_CROP && is_streaming(ici, icd) &&
+	    ici->ops->set_liveselection)
+		ret = ici->ops->set_liveselection(icd, s);
+	else
+		ret = ici->ops->set_selection(icd, s);
 	if (!ret &&
 	    s->target == V4L2_SEL_TGT_COMPOSE) {
 		icd->user_width = s->r.width;
@@ -1881,23 +1813,40 @@ static int soc_camera_remove(struct soc_camera_device *icd)
 	return 0;
 }
 
-static int default_cropcap(struct soc_camera_device *icd,
-			   struct v4l2_cropcap *a)
+static int default_g_selection(struct soc_camera_device *icd,
+			       struct v4l2_selection *sel)
 {
 	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	return v4l2_subdev_call(sd, video, cropcap, a);
-}
+	struct v4l2_subdev_selection sdsel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = sel->target,
+	};
+	int ret;
 
-static int default_g_crop(struct soc_camera_device *icd, struct v4l2_crop *a)
-{
-	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	return v4l2_subdev_call(sd, video, g_crop, a);
+	ret = v4l2_subdev_call(sd, pad, get_selection, NULL, &sdsel);
+	if (ret)
+		return ret;
+	sel->r = sdsel.r;
+	return 0;
 }
 
-static int default_s_crop(struct soc_camera_device *icd, const struct v4l2_crop *a)
+static int default_s_selection(struct soc_camera_device *icd,
+			       struct v4l2_selection *sel)
 {
 	struct v4l2_subdev *sd = soc_camera_to_subdev(icd);
-	return v4l2_subdev_call(sd, video, s_crop, a);
+	struct v4l2_subdev_selection sdsel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = sel->target,
+		.flags = sel->flags,
+		.r = sel->r,
+	};
+	int ret;
+
+	ret = v4l2_subdev_call(sd, pad, set_selection, NULL, &sdsel);
+	if (ret)
+		return ret;
+	sel->r = sdsel.r;
+	return 0;
 }
 
 static int default_g_parm(struct soc_camera_device *icd,
@@ -1968,12 +1917,10 @@ int soc_camera_host_register(struct soc_camera_host *ici)
 	    !ici->v4l2_dev.dev)
 		return -EINVAL;
 
-	if (!ici->ops->set_crop)
-		ici->ops->set_crop = default_s_crop;
-	if (!ici->ops->get_crop)
-		ici->ops->get_crop = default_g_crop;
-	if (!ici->ops->cropcap)
-		ici->ops->cropcap = default_cropcap;
+	if (!ici->ops->set_selection)
+		ici->ops->set_selection = default_s_selection;
+	if (!ici->ops->get_selection)
+		ici->ops->get_selection = default_g_selection;
 	if (!ici->ops->set_parm)
 		ici->ops->set_parm = default_s_parm;
 	if (!ici->ops->get_parm)
@@ -2126,9 +2073,6 @@ static const struct v4l2_ioctl_ops soc_camera_ioctl_ops = {
 	.vidioc_expbuf		 = soc_camera_expbuf,
 	.vidioc_streamon	 = soc_camera_streamon,
 	.vidioc_streamoff	 = soc_camera_streamoff,
-	.vidioc_cropcap		 = soc_camera_cropcap,
-	.vidioc_g_crop		 = soc_camera_g_crop,
-	.vidioc_s_crop		 = soc_camera_s_crop,
 	.vidioc_g_selection	 = soc_camera_g_selection,
 	.vidioc_s_selection	 = soc_camera_s_selection,
 	.vidioc_g_parm		 = soc_camera_g_parm,
diff --git a/drivers/media/platform/soc_camera/soc_camera_platform.c b/drivers/media/platform/soc_camera/soc_camera_platform.c
index a51d2a42998c..534d6c3c6d60 100644
--- a/drivers/media/platform/soc_camera/soc_camera_platform.c
+++ b/drivers/media/platform/soc_camera/soc_camera_platform.c
@@ -76,35 +76,27 @@ static int soc_camera_platform_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int soc_camera_platform_g_crop(struct v4l2_subdev *sd,
-				      struct v4l2_crop *a)
-{
-	struct soc_camera_platform_info *p = v4l2_get_subdevdata(sd);
-
-	a->c.left	= 0;
-	a->c.top	= 0;
-	a->c.width	= p->format.width;
-	a->c.height	= p->format.height;
-	a->type		= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	return 0;
-}
-
-static int soc_camera_platform_cropcap(struct v4l2_subdev *sd,
-				       struct v4l2_cropcap *a)
+static int soc_camera_platform_get_selection(struct v4l2_subdev *sd,
+		struct v4l2_subdev_pad_config *cfg,
+		struct v4l2_subdev_selection *sel)
 {
 	struct soc_camera_platform_info *p = v4l2_get_subdevdata(sd);
 
-	a->bounds.left			= 0;
-	a->bounds.top			= 0;
-	a->bounds.width			= p->format.width;
-	a->bounds.height		= p->format.height;
-	a->defrect			= a->bounds;
-	a->type				= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	a->pixelaspect.numerator	= 1;
-	a->pixelaspect.denominator	= 1;
+	if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE)
+		return -EINVAL;
 
-	return 0;
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP:
+		sel->r.left = 0;
+		sel->r.top = 0;
+		sel->r.width = p->format.width;
+		sel->r.height = p->format.height;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int soc_camera_platform_g_mbus_config(struct v4l2_subdev *sd,
@@ -120,13 +112,12 @@ static int soc_camera_platform_g_mbus_config(struct v4l2_subdev *sd,
 
 static struct v4l2_subdev_video_ops platform_subdev_video_ops = {
 	.s_stream	= soc_camera_platform_s_stream,
-	.cropcap	= soc_camera_platform_cropcap,
-	.g_crop		= soc_camera_platform_g_crop,
 	.g_mbus_config	= soc_camera_platform_g_mbus_config,
 };
 
 static const struct v4l2_subdev_pad_ops platform_subdev_pad_ops = {
 	.enum_mbus_code = soc_camera_platform_enum_mbus_code,
+	.get_selection	= soc_camera_platform_get_selection,
 	.get_fmt	= soc_camera_platform_fill_fmt,
 	.set_fmt	= soc_camera_platform_fill_fmt,
 };
diff --git a/drivers/media/platform/soc_camera/soc_scale_crop.c b/drivers/media/platform/soc_camera/soc_scale_crop.c
index bda29bc1b933..f77252d6ccd3 100644
--- a/drivers/media/platform/soc_camera/soc_scale_crop.c
+++ b/drivers/media/platform/soc_camera/soc_scale_crop.c
@@ -40,24 +40,22 @@ static bool is_inside(const struct v4l2_rect *r1, const struct v4l2_rect *r2)
 /* Get and store current client crop */
 int soc_camera_client_g_rect(struct v4l2_subdev *sd, struct v4l2_rect *rect)
 {
-	struct v4l2_crop crop;
-	struct v4l2_cropcap cap;
+	struct v4l2_subdev_selection sdsel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = V4L2_SEL_TGT_CROP,
+	};
 	int ret;
 
-	crop.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	ret = v4l2_subdev_call(sd, video, g_crop, &crop);
+	ret = v4l2_subdev_call(sd, pad, get_selection, NULL, &sdsel);
 	if (!ret) {
-		*rect = crop.c;
+		*rect = sdsel.r;
 		return ret;
 	}
 
-	/* Camera driver doesn't support .g_crop(), assume default rectangle */
-	cap.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	ret = v4l2_subdev_call(sd, video, cropcap, &cap);
+	sdsel.target = V4L2_SEL_TGT_CROP_DEFAULT;
+	ret = v4l2_subdev_call(sd, pad, get_selection, NULL, &sdsel);
 	if (!ret)
-		*rect = cap.defrect;
+		*rect = sdsel.r;
 
 	return ret;
 }
@@ -93,17 +91,27 @@ static void update_subrect(struct v4l2_rect *rect, struct v4l2_rect *subrect)
  * 2. if (1) failed, try to double the client image until we get one big enough
  * 3. if (2) failed, try to request the maximum image
  */
-int soc_camera_client_s_crop(struct v4l2_subdev *sd,
-			struct v4l2_crop *crop, struct v4l2_crop *cam_crop,
+int soc_camera_client_s_selection(struct v4l2_subdev *sd,
+			struct v4l2_selection *sel, struct v4l2_selection *cam_sel,
 			struct v4l2_rect *target_rect, struct v4l2_rect *subrect)
 {
-	struct v4l2_rect *rect = &crop->c, *cam_rect = &cam_crop->c;
+	struct v4l2_subdev_selection sdsel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = sel->target,
+		.flags = sel->flags,
+		.r = sel->r,
+	};
+	struct v4l2_subdev_selection bounds = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = V4L2_SEL_TGT_CROP_BOUNDS,
+	};
+	struct v4l2_rect *rect = &sel->r, *cam_rect = &cam_sel->r;
 	struct device *dev = sd->v4l2_dev->dev;
-	struct v4l2_cropcap cap;
 	int ret;
 	unsigned int width, height;
 
-	v4l2_subdev_call(sd, video, s_crop, crop);
+	v4l2_subdev_call(sd, pad, set_selection, NULL, &sdsel);
+	sel->r = sdsel.r;
 	ret = soc_camera_client_g_rect(sd, cam_rect);
 	if (ret < 0)
 		return ret;
@@ -113,29 +121,29 @@ int soc_camera_client_s_crop(struct v4l2_subdev *sd,
 	 * be within camera cropcap bounds
 	 */
 	if (!memcmp(rect, cam_rect, sizeof(*rect))) {
-		/* Even if camera S_CROP failed, but camera rectangle matches */
-		dev_dbg(dev, "Camera S_CROP successful for %dx%d@%d:%d\n",
+		/* Even if camera S_SELECTION failed, but camera rectangle matches */
+		dev_dbg(dev, "Camera S_SELECTION successful for %dx%d@%d:%d\n",
 			rect->width, rect->height, rect->left, rect->top);
 		*target_rect = *cam_rect;
 		return 0;
 	}
 
 	/* Try to fix cropping, that camera hasn't managed to set */
-	dev_geo(dev, "Fix camera S_CROP for %dx%d@%d:%d to %dx%d@%d:%d\n",
+	dev_geo(dev, "Fix camera S_SELECTION for %dx%d@%d:%d to %dx%d@%d:%d\n",
 		cam_rect->width, cam_rect->height,
 		cam_rect->left, cam_rect->top,
 		rect->width, rect->height, rect->left, rect->top);
 
 	/* We need sensor maximum rectangle */
-	ret = v4l2_subdev_call(sd, video, cropcap, &cap);
+	ret = v4l2_subdev_call(sd, pad, get_selection, NULL, &bounds);
 	if (ret < 0)
 		return ret;
 
 	/* Put user requested rectangle within sensor bounds */
-	soc_camera_limit_side(&rect->left, &rect->width, cap.bounds.left, 2,
-			      cap.bounds.width);
-	soc_camera_limit_side(&rect->top, &rect->height, cap.bounds.top, 4,
-			      cap.bounds.height);
+	soc_camera_limit_side(&rect->left, &rect->width, sdsel.r.left, 2,
+			      bounds.r.width);
+	soc_camera_limit_side(&rect->top, &rect->height, sdsel.r.top, 4,
+			      bounds.r.height);
 
 	/*
 	 * Popular special case - some cameras can only handle fixed sizes like
@@ -150,7 +158,7 @@ int soc_camera_client_s_crop(struct v4l2_subdev *sd,
 	 */
 	while (!ret && (is_smaller(cam_rect, rect) ||
 			is_inside(cam_rect, rect)) &&
-	       (cap.bounds.width > width || cap.bounds.height > height)) {
+	       (bounds.r.width > width || bounds.r.height > height)) {
 
 		width *= 2;
 		height *= 2;
@@ -168,36 +176,40 @@ int soc_camera_client_s_crop(struct v4l2_subdev *sd,
 		 * Instead we just drop to the left and top bounds.
 		 */
 		if (cam_rect->left > rect->left)
-			cam_rect->left = cap.bounds.left;
+			cam_rect->left = bounds.r.left;
 
 		if (cam_rect->left + cam_rect->width < rect->left + rect->width)
 			cam_rect->width = rect->left + rect->width -
 				cam_rect->left;
 
 		if (cam_rect->top > rect->top)
-			cam_rect->top = cap.bounds.top;
+			cam_rect->top = bounds.r.top;
 
 		if (cam_rect->top + cam_rect->height < rect->top + rect->height)
 			cam_rect->height = rect->top + rect->height -
 				cam_rect->top;
 
-		v4l2_subdev_call(sd, video, s_crop, cam_crop);
+		sdsel.r = *cam_rect;
+		v4l2_subdev_call(sd, pad, set_selection, NULL, &sdsel);
+		*cam_rect = sdsel.r;
 		ret = soc_camera_client_g_rect(sd, cam_rect);
-		dev_geo(dev, "Camera S_CROP %d for %dx%d@%d:%d\n", ret,
+		dev_geo(dev, "Camera S_SELECTION %d for %dx%d@%d:%d\n", ret,
 			cam_rect->width, cam_rect->height,
 			cam_rect->left, cam_rect->top);
 	}
 
-	/* S_CROP must not modify the rectangle */
+	/* S_SELECTION must not modify the rectangle */
 	if (is_smaller(cam_rect, rect) || is_inside(cam_rect, rect)) {
 		/*
 		 * The camera failed to configure a suitable cropping,
 		 * we cannot use the current rectangle, set to max
 		 */
-		*cam_rect = cap.bounds;
-		v4l2_subdev_call(sd, video, s_crop, cam_crop);
+		sdsel.r = bounds.r;
+		v4l2_subdev_call(sd, pad, set_selection, NULL, &sdsel);
+		*cam_rect = sdsel.r;
+
 		ret = soc_camera_client_g_rect(sd, cam_rect);
-		dev_geo(dev, "Camera S_CROP %d for max %dx%d@%d:%d\n", ret,
+		dev_geo(dev, "Camera S_SELECTION %d for max %dx%d@%d:%d\n", ret,
 			cam_rect->width, cam_rect->height,
 			cam_rect->left, cam_rect->top);
 	}
@@ -209,7 +221,7 @@ int soc_camera_client_s_crop(struct v4l2_subdev *sd,
 
 	return ret;
 }
-EXPORT_SYMBOL(soc_camera_client_s_crop);
+EXPORT_SYMBOL(soc_camera_client_s_selection);
 
 /* Iterative set_fmt, also updates cached client crop on success */
 static int client_set_fmt(struct soc_camera_device *icd,
@@ -221,7 +233,10 @@ static int client_set_fmt(struct soc_camera_device *icd,
 	struct device *dev = icd->parent;
 	struct v4l2_mbus_framefmt *mf = &format->format;
 	unsigned int width = mf->width, height = mf->height, tmp_w, tmp_h;
-	struct v4l2_cropcap cap;
+	struct v4l2_subdev_selection sdsel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = V4L2_SEL_TGT_CROP_BOUNDS,
+	};
 	bool host_1to1;
 	int ret;
 
@@ -243,16 +258,14 @@ static int client_set_fmt(struct soc_camera_device *icd,
 	if (!host_can_scale)
 		goto update_cache;
 
-	cap.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-	ret = v4l2_subdev_call(sd, video, cropcap, &cap);
+	ret = v4l2_subdev_call(sd, pad, get_selection, NULL, &sdsel);
 	if (ret < 0)
 		return ret;
 
-	if (max_width > cap.bounds.width)
-		max_width = cap.bounds.width;
-	if (max_height > cap.bounds.height)
-		max_height = cap.bounds.height;
+	if (max_width > sdsel.r.width)
+		max_width = sdsel.r.width;
+	if (max_height > sdsel.r.height)
+		max_height = sdsel.r.height;
 
 	/* Camera set a format, but geometry is not precise, try to improve */
 	tmp_w = mf->width;
diff --git a/drivers/media/platform/soc_camera/soc_scale_crop.h b/drivers/media/platform/soc_camera/soc_scale_crop.h
index 184a30dff541..9ca469312a1f 100644
--- a/drivers/media/platform/soc_camera/soc_scale_crop.h
+++ b/drivers/media/platform/soc_camera/soc_scale_crop.h
@@ -16,7 +16,7 @@
 
 struct soc_camera_device;
 
-struct v4l2_crop;
+struct v4l2_selection;
 struct v4l2_mbus_framefmt;
 struct v4l2_pix_format;
 struct v4l2_rect;
@@ -31,8 +31,8 @@ static inline unsigned int soc_camera_shift_scale(unsigned int size,
 #define soc_camera_calc_scale(in, shift, out) soc_camera_shift_scale(in, shift, out)
 
 int soc_camera_client_g_rect(struct v4l2_subdev *sd, struct v4l2_rect *rect);
-int soc_camera_client_s_crop(struct v4l2_subdev *sd,
-			struct v4l2_crop *crop, struct v4l2_crop *cam_crop,
+int soc_camera_client_s_selection(struct v4l2_subdev *sd,
+			struct v4l2_selection *sel, struct v4l2_selection *cam_sel,
 			struct v4l2_rect *target_rect, struct v4l2_rect *subrect);
 int soc_camera_client_scale(struct soc_camera_device *icd,
 			struct v4l2_rect *rect, struct v4l2_rect *subrect,
diff --git a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
index 3b1ac687d0df..45f82b5ddd77 100644
--- a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
+++ b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
@@ -527,7 +527,7 @@ static void bdisp_stop_streaming(struct vb2_queue *q)
 	pm_runtime_put(ctx->bdisp_dev->dev);
 }
 
-static struct vb2_ops bdisp_qops = {
+static const struct vb2_ops bdisp_qops = {
 	.queue_setup     = bdisp_queue_setup,
 	.buf_prepare     = bdisp_buf_prepare,
 	.buf_queue       = bdisp_buf_queue,
diff --git a/drivers/media/platform/sti/hva/Makefile b/drivers/media/platform/sti/hva/Makefile
new file mode 100644
index 000000000000..ffb69cebaef3
--- /dev/null
+++ b/drivers/media/platform/sti/hva/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_VIDEO_STI_HVA) := st-hva.o
+st-hva-y := hva-v4l2.o hva-hw.o hva-mem.o hva-h264.o
diff --git a/drivers/media/platform/sti/hva/hva-h264.c b/drivers/media/platform/sti/hva/hva-h264.c
new file mode 100644
index 000000000000..8cc8467c0cd3
--- /dev/null
+++ b/drivers/media/platform/sti/hva/hva-h264.c
@@ -0,0 +1,1050 @@
+/*
+ * Copyright (C) STMicroelectronics SA 2015
+ * Authors: Yannick Fertre <yannick.fertre@st.com>
+ *          Hugues Fruchet <hugues.fruchet@st.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include "hva.h"
+#include "hva-hw.h"
+
+#define MAX_SPS_PPS_SIZE 128
+
+#define BITSTREAM_OFFSET_MASK 0x7F
+
+/* video max size*/
+#define H264_MAX_SIZE_W 1920
+#define H264_MAX_SIZE_H 1920
+
+/* macroBlocs number (width & height) */
+#define MB_W(w) ((w + 0xF)  / 0x10)
+#define MB_H(h) ((h + 0xF)  / 0x10)
+
+/* formula to get temporal or spatial data size */
+#define DATA_SIZE(w, h) (MB_W(w) * MB_H(h) * 16)
+
+#define SEARCH_WINDOW_BUFFER_MAX_SIZE(w) ((4 * MB_W(w) + 42) * 256 * 3 / 2)
+#define CABAC_CONTEXT_BUFFER_MAX_SIZE(w) (MB_W(w) * 16)
+#define CTX_MB_BUFFER_MAX_SIZE(w) (MB_W(w) * 16 * 8)
+#define SLICE_HEADER_SIZE (4 * 16)
+#define BRC_DATA_SIZE (5 * 16)
+
+/* source buffer copy in YUV 420 MB-tiled format with size=16*256*3/2 */
+#define CURRENT_WINDOW_BUFFER_MAX_SIZE (16 * 256 * 3 / 2)
+
+/*
+ * 4 lines of pixels (in Luma, Chroma blue and Chroma red) of top MB
+ * for deblocking with size=4*16*MBx*2
+ */
+#define LOCAL_RECONSTRUCTED_BUFFER_MAX_SIZE(w) (4 * 16 * MB_W(w) * 2)
+
+/* factor for bitrate and cpb buffer size max values if profile >= high */
+#define H264_FACTOR_HIGH 1200
+
+/* factor for bitrate and cpb buffer size max values if profile < high */
+#define H264_FACTOR_BASELINE 1000
+
+/* number of bytes for NALU_TYPE_FILLER_DATA header and footer */
+#define H264_FILLER_DATA_SIZE 6
+
+struct h264_profile {
+	enum v4l2_mpeg_video_h264_level level;
+	u32 max_mb_per_seconds;
+	u32 max_frame_size;
+	u32 max_bitrate;
+	u32 max_cpb_size;
+	u32 min_comp_ratio;
+};
+
+static const struct h264_profile h264_infos_list[] = {
+	{V4L2_MPEG_VIDEO_H264_LEVEL_1_0, 1485, 99, 64, 175, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_1B, 1485, 99, 128, 350, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_1_1, 3000, 396, 192, 500, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_1_2, 6000, 396, 384, 1000, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_1_3, 11880, 396, 768, 2000, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_2_0, 11880, 396, 2000, 2000, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_2_1, 19800, 792, 4000, 4000, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_2_2, 20250, 1620, 4000, 4000, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_3_0, 40500, 1620, 10000, 10000, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_3_1, 108000, 3600, 14000, 14000, 4},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_3_2, 216000, 5120, 20000, 20000, 4},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_4_0, 245760, 8192, 20000, 25000, 4},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_4_1, 245760, 8192, 50000, 62500, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_4_2, 522240, 8704, 50000, 62500, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_5_0, 589824, 22080, 135000, 135000, 2},
+	{V4L2_MPEG_VIDEO_H264_LEVEL_5_1, 983040, 36864, 240000, 240000, 2}
+};
+
+enum hva_brc_type {
+	BRC_TYPE_NONE = 0,
+	BRC_TYPE_CBR = 1,
+	BRC_TYPE_VBR = 2,
+	BRC_TYPE_VBR_LOW_DELAY = 3
+};
+
+enum hva_entropy_coding_mode {
+	CAVLC = 0,
+	CABAC = 1
+};
+
+enum hva_picture_coding_type {
+	PICTURE_CODING_TYPE_I = 0,
+	PICTURE_CODING_TYPE_P = 1,
+	PICTURE_CODING_TYPE_B = 2
+};
+
+enum hva_h264_sampling_mode {
+	SAMPLING_MODE_NV12 = 0,
+	SAMPLING_MODE_UYVY = 1,
+	SAMPLING_MODE_RGB3 = 3,
+	SAMPLING_MODE_XRGB4 = 4,
+	SAMPLING_MODE_NV21 = 8,
+	SAMPLING_MODE_VYUY = 9,
+	SAMPLING_MODE_BGR3 = 11,
+	SAMPLING_MODE_XBGR4 = 12,
+	SAMPLING_MODE_RGBX4 = 20,
+	SAMPLING_MODE_BGRX4 = 28
+};
+
+enum hva_h264_nalu_type {
+	NALU_TYPE_UNKNOWN = 0,
+	NALU_TYPE_SLICE = 1,
+	NALU_TYPE_SLICE_DPA = 2,
+	NALU_TYPE_SLICE_DPB = 3,
+	NALU_TYPE_SLICE_DPC = 4,
+	NALU_TYPE_SLICE_IDR = 5,
+	NALU_TYPE_SEI = 6,
+	NALU_TYPE_SPS = 7,
+	NALU_TYPE_PPS = 8,
+	NALU_TYPE_AU_DELIMITER = 9,
+	NALU_TYPE_SEQ_END = 10,
+	NALU_TYPE_STREAM_END = 11,
+	NALU_TYPE_FILLER_DATA = 12,
+	NALU_TYPE_SPS_EXT = 13,
+	NALU_TYPE_PREFIX_UNIT = 14,
+	NALU_TYPE_SUBSET_SPS = 15,
+	NALU_TYPE_SLICE_AUX = 19,
+	NALU_TYPE_SLICE_EXT = 20
+};
+
+enum hva_h264_sei_payload_type {
+	SEI_BUFFERING_PERIOD = 0,
+	SEI_PICTURE_TIMING = 1,
+	SEI_STEREO_VIDEO_INFO = 21,
+	SEI_FRAME_PACKING_ARRANGEMENT = 45
+};
+
+/**
+ * stereo Video Info struct
+ */
+struct hva_h264_stereo_video_sei {
+	u8 field_views_flag;
+	u8 top_field_is_left_view_flag;
+	u8 current_frame_is_left_view_flag;
+	u8 next_frame_is_second_view_flag;
+	u8 left_view_self_contained_flag;
+	u8 right_view_self_contained_flag;
+};
+
+/**
+ * @frame_width: width in pixels of the buffer containing the input frame
+ * @frame_height: height in pixels of the buffer containing the input frame
+ * @frame_num: the parameter to be written in the slice header
+ * @picture_coding_type: type I, P or B
+ * @pic_order_cnt_type: POC mode, as defined in H264 std : can be 0,1,2
+ * @first_picture_in_sequence: flag telling to encoder that this is the
+ *			       first picture in a video sequence.
+ *			       Used for VBR
+ * @slice_size_type: 0 = no constraint to close the slice
+ *		     1= a slice is closed as soon as the slice_mb_size limit
+ *			is reached
+ *		     2= a slice is closed as soon as the slice_byte_size limit
+ *			is reached
+ *		     3= a slice is closed as soon as either the slice_byte_size
+ *			limit or the slice_mb_size limit is reached
+ * @slice_mb_size: defines the slice size in number of macroblocks
+ *		   (used when slice_size_type=1 or slice_size_type=3)
+ * @ir_param_option: defines the number of macroblocks per frame to be
+ *		     refreshed by AIR algorithm OR the refresh period
+ *		     by CIR algorithm
+ * @intra_refresh_type: enables the adaptive intra refresh algorithm.
+ *			Disable=0 / Adaptative=1 and Cycle=2 as intra refresh
+ * @use_constrained_intra_flag: constrained_intra_pred_flag from PPS
+ * @transform_mode: controls the use of 4x4/8x8 transform mode
+ * @disable_deblocking_filter_idc:
+ *		     0: specifies that all luma and chroma block edges of
+ *			the slice are filtered.
+ *		     1: specifies that deblocking is disabled for all block
+ *			edges of the slice.
+ *		     2: specifies that all luma and chroma block edges of
+ *			the slice are filtered with exception of the block edges
+ *			that coincide with slice boundaries
+ * @slice_alpha_c0_offset_div2: to be written in slice header,
+ *				controls deblocking
+ * @slice_beta_offset_div2: to be written in slice header,
+ *			    controls deblocking
+ * @encoder_complexity: encoder complexity control (IME).
+ *		     0 = I_16x16, P_16x16, Full ME Complexity
+ *		     1 = I_16x16, I_NxN, P_16x16, Full ME Complexity
+ *		     2 = I_16x16, I_NXN, P_16x16, P_WxH, Full ME Complexity
+ *		     4 = I_16x16, P_16x16, Reduced ME Complexity
+ *		     5 = I_16x16, I_NxN, P_16x16, Reduced ME Complexity
+ *		     6 = I_16x16, I_NXN, P_16x16, P_WxH, Reduced ME Complexity
+ *  @chroma_qp_index_offset: coming from picture parameter set
+ *			     (PPS see [H.264 STD] 7.4.2.2)
+ *  @entropy_coding_mode: entropy coding mode.
+ *			  0 = CAVLC
+ *			  1 = CABAC
+ * @brc_type: selects the bit-rate control algorithm
+ *		     0 = constant Qp, (no BRC)
+ *		     1 = CBR
+ *		     2 = VBR
+ * @quant: Quantization param used in case of fix QP encoding (no BRC)
+ * @non_VCL_NALU_Size: size of non-VCL NALUs (SPS, PPS, filler),
+ *		       used by BRC
+ * @cpb_buffer_size: size of Coded Picture Buffer, used by BRC
+ * @bit_rate: target bitrate, for BRC
+ * @qp_min: min QP threshold
+ * @qp_max: max QP threshold
+ * @framerate_num: target framerate numerator , used by BRC
+ * @framerate_den: target framerate denomurator , used by BRC
+ * @delay: End-to-End Initial Delay
+ * @strict_HRD_compliancy: flag for HDR compliancy (1)
+ *			   May impact quality encoding
+ * @addr_source_buffer: address of input frame buffer for current frame
+ * @addr_fwd_Ref_Buffer: address of reference frame buffer
+ * @addr_rec_buffer: address of reconstructed frame buffer
+ * @addr_output_bitstream_start: output bitstream start address
+ * @addr_output_bitstream_end: output bitstream end address
+ * @addr_external_sw : address of external search window
+ * @addr_lctx : address of context picture buffer
+ * @addr_local_rec_buffer: address of local reconstructed buffer
+ * @addr_spatial_context: address of spatial context buffer
+ * @bitstream_offset: offset in bits between aligned bitstream start
+ *		      address and first bit to be written by HVA.
+ *		      Range value is [0..63]
+ * @sampling_mode: Input picture format .
+ *		     0: YUV420 semi_planar Interleaved
+ *		     1: YUV422 raster Interleaved
+ * @addr_param_out: address of output parameters structure
+ * @addr_scaling_matrix: address to the coefficient of
+ *			 the inverse scaling matrix
+ * @addr_scaling_matrix_dir: address to the coefficient of
+ *			     the direct scaling matrix
+ * @addr_cabac_context_buffer: address of cabac context buffer
+ * @GmvX: Input information about the horizontal global displacement of
+ *	  the encoded frame versus the previous one
+ * @GmvY: Input information about the vertical global displacement of
+ *	  the encoded frame versus the previous one
+ * @window_width: width in pixels of the window to be encoded inside
+ *		  the input frame
+ * @window_height: width in pixels of the window to be encoded inside
+ *		   the input frame
+ * @window_horizontal_offset: horizontal offset in pels for input window
+ *			      within input frame
+ * @window_vertical_offset: vertical offset in pels for input window
+ *			    within input frame
+ * @addr_roi: Map of QP offset for the Region of Interest algorithm and
+ *	      also used for Error map.
+ *	      Bit 0-6 used for qp offset (value -64 to 63).
+ *	      Bit 7 used to force intra
+ * @addr_slice_header: address to slice header
+ * @slice_header_size_in_bits: size in bits of the Slice header
+ * @slice_header_offset0: Slice header offset where to insert
+ *			  first_Mb_in_slice
+ * @slice_header_offset1: Slice header offset where to insert
+ *			  slice_qp_delta
+ * @slice_header_offset2: Slice header offset where to insert
+ *			  num_MBs_in_slice
+ * @slice_synchro_enable: enable "slice ready" interrupt after each slice
+ * @max_slice_number: Maximum number of slice in a frame
+ *		      (0 is strictly forbidden)
+ * @rgb2_yuv_y_coeff: Four coefficients (C0C1C2C3) to convert from RGB to
+ *		      YUV for the Y component.
+ *		      Y = C0*R + C1*G + C2*B + C3 (C0 is on byte 0)
+ * @rgb2_yuv_u_coeff: four coefficients (C0C1C2C3) to convert from RGB to
+ *		      YUV for the Y component.
+ *		      Y = C0*R + C1*G + C2*B + C3 (C0 is on byte 0)
+ * @rgb2_yuv_v_coeff: Four coefficients (C0C1C2C3) to convert from RGB to
+ *		      YUV for the U (Cb) component.
+ *		      U = C0*R + C1*G + C2*B + C3 (C0 is on byte 0)
+ * @slice_byte_size: maximum slice size in bytes
+ *		     (used when slice_size_type=2 or slice_size_type=3)
+ * @max_air_intra_mb_nb: Maximum number of intra macroblock in a frame
+ *			 for the AIR algorithm
+ * @brc_no_skip: Disable skipping in the Bitrate Controller
+ * @addr_brc_in_out_parameter: address of static buffer for BRC parameters
+ */
+struct hva_h264_td {
+	u16 frame_width;
+	u16 frame_height;
+	u32 frame_num;
+	u16 picture_coding_type;
+	u16 reserved1;
+	u16 pic_order_cnt_type;
+	u16 first_picture_in_sequence;
+	u16 slice_size_type;
+	u16 reserved2;
+	u32 slice_mb_size;
+	u16 ir_param_option;
+	u16 intra_refresh_type;
+	u16 use_constrained_intra_flag;
+	u16 transform_mode;
+	u16 disable_deblocking_filter_idc;
+	s16 slice_alpha_c0_offset_div2;
+	s16 slice_beta_offset_div2;
+	u16 encoder_complexity;
+	s16 chroma_qp_index_offset;
+	u16 entropy_coding_mode;
+	u16 brc_type;
+	u16 quant;
+	u32 non_vcl_nalu_size;
+	u32 cpb_buffer_size;
+	u32 bit_rate;
+	u16 qp_min;
+	u16 qp_max;
+	u16 framerate_num;
+	u16 framerate_den;
+	u16 delay;
+	u16 strict_hrd_compliancy;
+	u32 addr_source_buffer;
+	u32 addr_fwd_ref_buffer;
+	u32 addr_rec_buffer;
+	u32 addr_output_bitstream_start;
+	u32 addr_output_bitstream_end;
+	u32 addr_external_sw;
+	u32 addr_lctx;
+	u32 addr_local_rec_buffer;
+	u32 addr_spatial_context;
+	u16 bitstream_offset;
+	u16 sampling_mode;
+	u32 addr_param_out;
+	u32 addr_scaling_matrix;
+	u32 addr_scaling_matrix_dir;
+	u32 addr_cabac_context_buffer;
+	u32 reserved3;
+	u32 reserved4;
+	s16 gmv_x;
+	s16 gmv_y;
+	u16 window_width;
+	u16 window_height;
+	u16 window_horizontal_offset;
+	u16 window_vertical_offset;
+	u32 addr_roi;
+	u32 addr_slice_header;
+	u16 slice_header_size_in_bits;
+	u16 slice_header_offset0;
+	u16 slice_header_offset1;
+	u16 slice_header_offset2;
+	u32 reserved5;
+	u32 reserved6;
+	u16 reserved7;
+	u16 reserved8;
+	u16 slice_synchro_enable;
+	u16 max_slice_number;
+	u32 rgb2_yuv_y_coeff;
+	u32 rgb2_yuv_u_coeff;
+	u32 rgb2_yuv_v_coeff;
+	u32 slice_byte_size;
+	u16 max_air_intra_mb_nb;
+	u16 brc_no_skip;
+	u32 addr_temporal_context;
+	u32 addr_brc_in_out_parameter;
+};
+
+/**
+ * @ slice_size: slice size
+ * @ slice_start_time: start time
+ * @ slice_stop_time: stop time
+ * @ slice_num: slice number
+ */
+struct hva_h264_slice_po {
+	u32 slice_size;
+	u32 slice_start_time;
+	u32 slice_end_time;
+	u32 slice_num;
+};
+
+/**
+ * @ bitstream_size: bitstream size
+ * @ dct_bitstream_size: dtc bitstream size
+ * @ stuffing_bits: number of stuffing bits inserted by the encoder
+ * @ removal_time: removal time of current frame (nb of ticks 1/framerate)
+ * @ hvc_start_time: hvc start time
+ * @ hvc_stop_time: hvc stop time
+ * @ slice_count: slice count
+ */
+struct hva_h264_po {
+	u32 bitstream_size;
+	u32 dct_bitstream_size;
+	u32 stuffing_bits;
+	u32 removal_time;
+	u32 hvc_start_time;
+	u32 hvc_stop_time;
+	u32 slice_count;
+	u32 reserved0;
+	struct hva_h264_slice_po slice_params[16];
+};
+
+struct hva_h264_task {
+	struct hva_h264_td td;
+	struct hva_h264_po po;
+};
+
+/**
+ * @seq_info:  sequence information buffer
+ * @ref_frame: reference frame buffer
+ * @rec_frame: reconstructed frame buffer
+ * @task:      task descriptor
+ */
+struct hva_h264_ctx {
+	struct hva_buffer *seq_info;
+	struct hva_buffer *ref_frame;
+	struct hva_buffer *rec_frame;
+	struct hva_buffer *task;
+};
+
+static int hva_h264_fill_slice_header(struct hva_ctx *pctx,
+				      u8 *slice_header_addr,
+				      struct hva_controls *ctrls,
+				      int frame_num,
+				      u16 *header_size,
+				      u16 *header_offset0,
+				      u16 *header_offset1,
+				      u16 *header_offset2)
+{
+	/*
+	 * with this HVA hardware version, part of the slice header is computed
+	 * on host and part by hardware.
+	 * The part of host is precomputed and available through this array.
+	 */
+	struct device *dev = ctx_to_dev(pctx);
+	int  cabac = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC;
+	const unsigned char slice_header[] = { 0x00, 0x00, 0x00, 0x01,
+					       0x41, 0x34, 0x07, 0x00};
+	int idr_pic_id = frame_num % 2;
+	enum hva_picture_coding_type type;
+	u32 frame_order = frame_num % ctrls->gop_size;
+
+	if (!(frame_num % ctrls->gop_size))
+		type = PICTURE_CODING_TYPE_I;
+	else
+		type = PICTURE_CODING_TYPE_P;
+
+	memcpy(slice_header_addr, slice_header, sizeof(slice_header));
+
+	*header_size = 56;
+	*header_offset0 = 40;
+	*header_offset1 = 13;
+	*header_offset2 = 0;
+
+	if (type == PICTURE_CODING_TYPE_I) {
+		slice_header_addr[4] = 0x65;
+		slice_header_addr[5] = 0x11;
+
+		/* toggle the I frame */
+		if ((frame_num / ctrls->gop_size) % 2) {
+			*header_size += 4;
+			*header_offset1 += 4;
+			slice_header_addr[6] = 0x04;
+			slice_header_addr[7] = 0x70;
+
+		} else {
+			*header_size += 2;
+			*header_offset1 += 2;
+			slice_header_addr[6] = 0x09;
+			slice_header_addr[7] = 0xC0;
+		}
+	} else {
+		if (ctrls->entropy_mode == cabac) {
+			*header_size += 1;
+			*header_offset1 += 1;
+			slice_header_addr[7] = 0x80;
+		}
+		/*
+		 * update slice header with P frame order
+		 * frame order is limited to 16 (coded on 4bits only)
+		 */
+		slice_header_addr[5] += ((frame_order & 0x0C) >> 2);
+		slice_header_addr[6] += ((frame_order & 0x03) << 6);
+	}
+
+	dev_dbg(dev,
+		"%s   %s slice header order %d idrPicId %d header size %d\n",
+		pctx->name, __func__, frame_order, idr_pic_id, *header_size);
+	return 0;
+}
+
+static int hva_h264_fill_data_nal(struct hva_ctx *pctx,
+				  unsigned int stuffing_bytes, u8 *addr,
+				  unsigned int stream_size, unsigned int *size)
+{
+	struct device *dev = ctx_to_dev(pctx);
+	const u8 start[] = { 0x00, 0x00, 0x00, 0x01 };
+
+	dev_dbg(dev, "%s   %s stuffing bytes %d\n", pctx->name, __func__,
+		stuffing_bytes);
+
+	if ((*size + stuffing_bytes + H264_FILLER_DATA_SIZE) > stream_size) {
+		dev_dbg(dev, "%s   %s too many stuffing bytes %d\n",
+			pctx->name, __func__, stuffing_bytes);
+		return 0;
+	}
+
+	/* start code */
+	memcpy(addr + *size, start, sizeof(start));
+	*size += sizeof(start);
+
+	/* nal_unit_type */
+	addr[*size] = NALU_TYPE_FILLER_DATA;
+	*size += 1;
+
+	memset(addr + *size, 0xff, stuffing_bytes);
+	*size += stuffing_bytes;
+
+	addr[*size] = 0x80;
+	*size += 1;
+
+	return 0;
+}
+
+static int hva_h264_fill_sei_nal(struct hva_ctx *pctx,
+				 enum hva_h264_sei_payload_type type,
+				 u8 *addr, u32 *size)
+{
+	struct device *dev = ctx_to_dev(pctx);
+	const u8 start[] = { 0x00, 0x00, 0x00, 0x01 };
+	struct hva_h264_stereo_video_sei info;
+	u8 offset = 7;
+	u8 msg = 0;
+
+	/* start code */
+	memcpy(addr + *size, start, sizeof(start));
+	*size += sizeof(start);
+
+	/* nal_unit_type */
+	addr[*size] = NALU_TYPE_SEI;
+	*size += 1;
+
+	/* payload type */
+	addr[*size] = type;
+	*size += 1;
+
+	switch (type) {
+	case SEI_STEREO_VIDEO_INFO:
+		memset(&info, 0, sizeof(info));
+
+		/* set to top/bottom frame packing arrangement */
+		info.field_views_flag = 1;
+		info.top_field_is_left_view_flag = 1;
+
+		/* payload size */
+		addr[*size] = 1;
+		*size += 1;
+
+		/* payload */
+		msg = info.field_views_flag << offset--;
+
+		if (info.field_views_flag) {
+			msg |= info.top_field_is_left_view_flag <<
+			       offset--;
+		} else {
+			msg |= info.current_frame_is_left_view_flag <<
+			       offset--;
+			msg |= info.next_frame_is_second_view_flag <<
+			       offset--;
+		}
+		msg |= info.left_view_self_contained_flag << offset--;
+		msg |= info.right_view_self_contained_flag << offset--;
+
+		addr[*size] = msg;
+		*size += 1;
+
+		addr[*size] = 0x80;
+		*size += 1;
+
+		return 0;
+	case SEI_BUFFERING_PERIOD:
+	case SEI_PICTURE_TIMING:
+	case SEI_FRAME_PACKING_ARRANGEMENT:
+	default:
+		dev_err(dev, "%s   sei nal type not supported %d\n",
+			pctx->name, type);
+		return -EINVAL;
+	}
+}
+
+static int hva_h264_prepare_task(struct hva_ctx *pctx,
+				 struct hva_h264_task *task,
+				 struct hva_frame *frame,
+				 struct hva_stream *stream)
+{
+	struct hva_dev *hva = ctx_to_hdev(pctx);
+	struct device *dev = ctx_to_dev(pctx);
+	struct hva_h264_ctx *ctx = (struct hva_h264_ctx *)pctx->priv;
+	struct hva_buffer *seq_info = ctx->seq_info;
+	struct hva_buffer *fwd_ref_frame = ctx->ref_frame;
+	struct hva_buffer *loc_rec_frame = ctx->rec_frame;
+	struct hva_h264_td *td = &task->td;
+	struct hva_controls *ctrls = &pctx->ctrls;
+	struct v4l2_fract *time_per_frame = &pctx->ctrls.time_per_frame;
+	int cavlc =  V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC;
+	u32 frame_num = pctx->stream_num;
+	u32 addr_esram = hva->esram_addr;
+	enum v4l2_mpeg_video_h264_level level;
+	dma_addr_t paddr = 0;
+	u8 *slice_header_vaddr;
+	u32 frame_width = frame->info.aligned_width;
+	u32 frame_height = frame->info.aligned_height;
+	u32 max_cpb_buffer_size;
+	unsigned int payload = stream->bytesused;
+	u32 max_bitrate;
+
+	/* check width and height parameters */
+	if ((frame_width > max(H264_MAX_SIZE_W, H264_MAX_SIZE_H)) ||
+	    (frame_height > max(H264_MAX_SIZE_W, H264_MAX_SIZE_H))) {
+		dev_err(dev,
+			"%s   width(%d) or height(%d) exceeds limits (%dx%d)\n",
+			pctx->name, frame_width, frame_height,
+			H264_MAX_SIZE_W, H264_MAX_SIZE_H);
+		return -EINVAL;
+	}
+
+	level = ctrls->level;
+
+	memset(td, 0, sizeof(struct hva_h264_td));
+
+	td->frame_width = frame_width;
+	td->frame_height = frame_height;
+
+	/* set frame alignement */
+	td->window_width =  frame_width;
+	td->window_height = frame_height;
+	td->window_horizontal_offset = 0;
+	td->window_vertical_offset = 0;
+
+	td->first_picture_in_sequence = (!frame_num) ? 1 : 0;
+
+	/* pic_order_cnt_type hard coded to '2' as only I & P frames */
+	td->pic_order_cnt_type = 2;
+
+	/* useConstrainedIntraFlag set to false for better coding efficiency */
+	td->use_constrained_intra_flag = false;
+	td->brc_type = (ctrls->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_CBR)
+			? BRC_TYPE_CBR : BRC_TYPE_VBR;
+
+	td->entropy_coding_mode = (ctrls->entropy_mode == cavlc) ? CAVLC :
+				  CABAC;
+
+	td->bit_rate = ctrls->bitrate;
+
+	/* set framerate, framerate = 1 n/ time per frame */
+	if (time_per_frame->numerator >= 536) {
+		/*
+		 * due to a hardware bug, framerate denominator can't exceed
+		 * 536 (BRC overflow). Compute nearest framerate
+		 */
+		td->framerate_den = 1;
+		td->framerate_num = (time_per_frame->denominator +
+				    (time_per_frame->numerator >> 1) - 1) /
+				    time_per_frame->numerator;
+
+		/*
+		 * update bitrate to introduce a correction due to
+		 * the new framerate
+		 * new bitrate = (old bitrate * new framerate) / old framerate
+		 */
+		td->bit_rate /= time_per_frame->numerator;
+		td->bit_rate *= time_per_frame->denominator;
+		td->bit_rate /= td->framerate_num;
+	} else {
+		td->framerate_den = time_per_frame->numerator;
+		td->framerate_num = time_per_frame->denominator;
+	}
+
+	/* compute maximum bitrate depending on profile */
+	if (ctrls->profile >= V4L2_MPEG_VIDEO_H264_PROFILE_HIGH)
+		max_bitrate = h264_infos_list[level].max_bitrate *
+			      H264_FACTOR_HIGH;
+	else
+		max_bitrate = h264_infos_list[level].max_bitrate *
+			      H264_FACTOR_BASELINE;
+
+	/* check if bitrate doesn't exceed max size */
+	if (td->bit_rate > max_bitrate) {
+		dev_dbg(dev,
+			"%s   bitrate (%d) larger than level and profile allow, clip to %d\n",
+			pctx->name, td->bit_rate, max_bitrate);
+		td->bit_rate = max_bitrate;
+	}
+
+	/* convert cpb_buffer_size in bits */
+	td->cpb_buffer_size = ctrls->cpb_size * 8000;
+
+	/* compute maximum cpb buffer size depending on profile */
+	if (ctrls->profile >= V4L2_MPEG_VIDEO_H264_PROFILE_HIGH)
+		max_cpb_buffer_size =
+		    h264_infos_list[level].max_cpb_size * H264_FACTOR_HIGH;
+	else
+		max_cpb_buffer_size =
+		    h264_infos_list[level].max_cpb_size * H264_FACTOR_BASELINE;
+
+	/* check if cpb buffer size doesn't exceed max size */
+	if (td->cpb_buffer_size > max_cpb_buffer_size) {
+		dev_dbg(dev,
+			"%s   cpb size larger than level %d allows, clip to %d\n",
+			pctx->name, td->cpb_buffer_size, max_cpb_buffer_size);
+		td->cpb_buffer_size = max_cpb_buffer_size;
+	}
+
+	/* enable skipping in the Bitrate Controller */
+	td->brc_no_skip = 0;
+
+	/* initial delay */
+	if ((ctrls->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_CBR) &&
+	    td->bit_rate)
+		td->delay = 1000 * (td->cpb_buffer_size / td->bit_rate);
+	else
+		td->delay = 0;
+
+	switch (frame->info.pixelformat) {
+	case V4L2_PIX_FMT_NV12:
+		td->sampling_mode = SAMPLING_MODE_NV12;
+		break;
+	case V4L2_PIX_FMT_NV21:
+		td->sampling_mode = SAMPLING_MODE_NV21;
+		break;
+	default:
+		dev_err(dev, "%s   invalid source pixel format\n",
+			pctx->name);
+		return -EINVAL;
+	}
+
+	/*
+	 * fill matrix color converter (RGB to YUV)
+	 * Y = 0,299 R + 0,587 G + 0,114 B
+	 * Cb = -0,1687 R -0,3313 G + 0,5 B + 128
+	 * Cr = 0,5 R - 0,4187 G - 0,0813 B + 128
+	 */
+	td->rgb2_yuv_y_coeff = 0x12031008;
+	td->rgb2_yuv_u_coeff = 0x800EF7FB;
+	td->rgb2_yuv_v_coeff = 0x80FEF40E;
+
+	/* enable/disable transform mode */
+	td->transform_mode = ctrls->dct8x8;
+
+	/* encoder complexity fix to 2, ENCODE_I_16x16_I_NxN_P_16x16_P_WxH */
+	td->encoder_complexity = 2;
+
+	/* quant fix to 28, default VBR value */
+	td->quant = 28;
+
+	if (td->framerate_den == 0) {
+		dev_err(dev, "%s   invalid framerate\n", pctx->name);
+		return -EINVAL;
+	}
+
+	/* if automatic framerate, deactivate bitrate controller */
+	if (td->framerate_num == 0)
+		td->brc_type = 0;
+
+	/* compliancy fix to true */
+	td->strict_hrd_compliancy = 1;
+
+	/* set minimum & maximum quantizers */
+	td->qp_min = clamp_val(ctrls->qpmin, 0, 51);
+	td->qp_max = clamp_val(ctrls->qpmax, 0, 51);
+
+	td->addr_source_buffer = frame->paddr;
+	td->addr_fwd_ref_buffer = fwd_ref_frame->paddr;
+	td->addr_rec_buffer = loc_rec_frame->paddr;
+
+	td->addr_output_bitstream_end = (u32)stream->paddr + stream->size;
+
+	td->addr_output_bitstream_start = (u32)stream->paddr;
+	td->bitstream_offset = (((u32)stream->paddr & 0xF) << 3) &
+			       BITSTREAM_OFFSET_MASK;
+
+	td->addr_param_out = (u32)ctx->task->paddr +
+			     offsetof(struct hva_h264_task, po);
+
+	/* swap spatial and temporal context */
+	if (frame_num % 2) {
+		paddr = seq_info->paddr;
+		td->addr_spatial_context =  ALIGN(paddr, 0x100);
+		paddr = seq_info->paddr + DATA_SIZE(frame_width,
+							frame_height);
+		td->addr_temporal_context = ALIGN(paddr, 0x100);
+	} else {
+		paddr = seq_info->paddr;
+		td->addr_temporal_context = ALIGN(paddr, 0x100);
+		paddr = seq_info->paddr + DATA_SIZE(frame_width,
+							frame_height);
+		td->addr_spatial_context =  ALIGN(paddr, 0x100);
+	}
+
+	paddr = seq_info->paddr + 2 * DATA_SIZE(frame_width, frame_height);
+
+	td->addr_brc_in_out_parameter =  ALIGN(paddr, 0x100);
+
+	paddr = td->addr_brc_in_out_parameter + BRC_DATA_SIZE;
+	td->addr_slice_header =  ALIGN(paddr, 0x100);
+	td->addr_external_sw =  ALIGN(addr_esram, 0x100);
+
+	addr_esram += SEARCH_WINDOW_BUFFER_MAX_SIZE(frame_width);
+	td->addr_local_rec_buffer = ALIGN(addr_esram, 0x100);
+
+	addr_esram += LOCAL_RECONSTRUCTED_BUFFER_MAX_SIZE(frame_width);
+	td->addr_lctx = ALIGN(addr_esram, 0x100);
+
+	addr_esram += CTX_MB_BUFFER_MAX_SIZE(max(frame_width, frame_height));
+	td->addr_cabac_context_buffer = ALIGN(addr_esram, 0x100);
+
+	if (!(frame_num % ctrls->gop_size)) {
+		td->picture_coding_type = PICTURE_CODING_TYPE_I;
+		stream->vbuf.flags |= V4L2_BUF_FLAG_KEYFRAME;
+	} else {
+		td->picture_coding_type = PICTURE_CODING_TYPE_P;
+		stream->vbuf.flags &= ~V4L2_BUF_FLAG_KEYFRAME;
+	}
+
+	/* fill the slice header part */
+	slice_header_vaddr = seq_info->vaddr + (td->addr_slice_header -
+			     seq_info->paddr);
+
+	hva_h264_fill_slice_header(pctx, slice_header_vaddr, ctrls, frame_num,
+				   &td->slice_header_size_in_bits,
+				   &td->slice_header_offset0,
+				   &td->slice_header_offset1,
+				   &td->slice_header_offset2);
+
+	td->chroma_qp_index_offset = 2;
+	td->slice_synchro_enable = 0;
+	td->max_slice_number = 1;
+
+	/*
+	 * check the sps/pps header size for key frame only
+	 * sps/pps header was previously fill by libv4l
+	 * during qbuf of stream buffer
+	 */
+	if ((stream->vbuf.flags == V4L2_BUF_FLAG_KEYFRAME) &&
+	    (payload > MAX_SPS_PPS_SIZE)) {
+		dev_err(dev, "%s   invalid sps/pps size %d\n", pctx->name,
+			payload);
+		return -EINVAL;
+	}
+
+	if (stream->vbuf.flags != V4L2_BUF_FLAG_KEYFRAME)
+		payload = 0;
+
+	/* add SEI nal (video stereo info) */
+	if (ctrls->sei_fp && hva_h264_fill_sei_nal(pctx, SEI_STEREO_VIDEO_INFO,
+						   (u8 *)stream->vaddr,
+						   &payload)) {
+		dev_err(dev, "%s   fail to get SEI nal\n", pctx->name);
+		return -EINVAL;
+	}
+
+	/* fill size of non-VCL NAL units (SPS, PPS, filler and SEI) */
+	td->non_vcl_nalu_size = payload * 8;
+
+	/* compute bitstream offset & new start address of bitstream */
+	td->addr_output_bitstream_start += ((payload >> 4) << 4);
+	td->bitstream_offset += (payload - ((payload >> 4) << 4)) * 8;
+
+	stream->bytesused = payload;
+
+	return 0;
+}
+
+static unsigned int hva_h264_get_stream_size(struct hva_h264_task *task)
+{
+	struct hva_h264_po *po = &task->po;
+
+	return po->bitstream_size;
+}
+
+static u32 hva_h264_get_stuffing_bytes(struct hva_h264_task *task)
+{
+	struct hva_h264_po *po = &task->po;
+
+	return po->stuffing_bits >> 3;
+}
+
+static int hva_h264_open(struct hva_ctx *pctx)
+{
+	struct device *dev = ctx_to_dev(pctx);
+	struct hva_h264_ctx *ctx;
+	struct hva_dev *hva = ctx_to_hdev(pctx);
+	u32 frame_width = pctx->frameinfo.aligned_width;
+	u32 frame_height = pctx->frameinfo.aligned_height;
+	u32 size;
+	int ret;
+
+	/* check esram size necessary to encode a frame */
+	size = SEARCH_WINDOW_BUFFER_MAX_SIZE(frame_width) +
+	       LOCAL_RECONSTRUCTED_BUFFER_MAX_SIZE(frame_width) +
+	       CTX_MB_BUFFER_MAX_SIZE(max(frame_width, frame_height)) +
+	       CABAC_CONTEXT_BUFFER_MAX_SIZE(frame_width);
+
+	if (hva->esram_size < size) {
+		dev_err(dev, "%s   not enough esram (max:%d request:%d)\n",
+			pctx->name, hva->esram_size, size);
+		ret = -EINVAL;
+		goto err;
+	}
+
+	/* allocate context for codec */
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	/* allocate sequence info buffer */
+	ret = hva_mem_alloc(pctx,
+			    2 * DATA_SIZE(frame_width, frame_height) +
+			    SLICE_HEADER_SIZE +
+			    BRC_DATA_SIZE,
+			    "hva sequence info",
+			    &ctx->seq_info);
+	if (ret) {
+		dev_err(dev,
+			"%s   failed to allocate sequence info buffer\n",
+			pctx->name);
+		goto err_ctx;
+	}
+
+	/* allocate reference frame buffer */
+	ret = hva_mem_alloc(pctx,
+			    frame_width * frame_height * 3 / 2,
+			    "hva reference frame",
+			    &ctx->ref_frame);
+	if (ret) {
+		dev_err(dev, "%s   failed to allocate reference frame buffer\n",
+			pctx->name);
+		goto err_seq_info;
+	}
+
+	/* allocate reconstructed frame buffer */
+	ret = hva_mem_alloc(pctx,
+			    frame_width * frame_height * 3 / 2,
+			    "hva reconstructed frame",
+			    &ctx->rec_frame);
+	if (ret) {
+		dev_err(dev,
+			"%s   failed to allocate reconstructed frame buffer\n",
+			pctx->name);
+		goto err_ref_frame;
+	}
+
+	/* allocate task descriptor */
+	ret = hva_mem_alloc(pctx,
+			    sizeof(struct hva_h264_task),
+			    "hva task descriptor",
+			    &ctx->task);
+	if (ret) {
+		dev_err(dev,
+			"%s   failed to allocate task descriptor\n",
+			pctx->name);
+		goto err_rec_frame;
+	}
+
+	pctx->priv = (void *)ctx;
+
+	return 0;
+
+err_rec_frame:
+	hva_mem_free(pctx, ctx->rec_frame);
+err_ref_frame:
+	hva_mem_free(pctx, ctx->ref_frame);
+err_seq_info:
+	hva_mem_free(pctx, ctx->seq_info);
+err_ctx:
+	devm_kfree(dev, ctx);
+err:
+	return ret;
+}
+
+static int hva_h264_close(struct hva_ctx *pctx)
+{
+	struct hva_h264_ctx *ctx = (struct hva_h264_ctx *)pctx->priv;
+	struct device *dev = ctx_to_dev(pctx);
+
+	if (ctx->seq_info)
+		hva_mem_free(pctx, ctx->seq_info);
+
+	if (ctx->ref_frame)
+		hva_mem_free(pctx, ctx->ref_frame);
+
+	if (ctx->rec_frame)
+		hva_mem_free(pctx, ctx->rec_frame);
+
+	if (ctx->task)
+		hva_mem_free(pctx, ctx->task);
+
+	devm_kfree(dev, ctx);
+
+	return 0;
+}
+
+static int hva_h264_encode(struct hva_ctx *pctx, struct hva_frame *frame,
+			   struct hva_stream *stream)
+{
+	struct hva_h264_ctx *ctx = (struct hva_h264_ctx *)pctx->priv;
+	struct hva_h264_task *task = (struct hva_h264_task *)ctx->task->vaddr;
+	struct hva_buffer *tmp_frame;
+	u32 stuffing_bytes = 0;
+	int ret = 0;
+
+	ret = hva_h264_prepare_task(pctx, task, frame, stream);
+	if (ret)
+		goto err;
+
+	ret = hva_hw_execute_task(pctx, H264_ENC, ctx->task);
+	if (ret)
+		goto err;
+
+	pctx->stream_num++;
+	stream->bytesused += hva_h264_get_stream_size(task);
+
+	stuffing_bytes = hva_h264_get_stuffing_bytes(task);
+
+	if (stuffing_bytes)
+		hva_h264_fill_data_nal(pctx, stuffing_bytes,
+				       (u8 *)stream->vaddr,
+				       stream->size,
+				       &stream->bytesused);
+
+	/* switch reference & reconstructed frame */
+	tmp_frame = ctx->ref_frame;
+	ctx->ref_frame = ctx->rec_frame;
+	ctx->rec_frame = tmp_frame;
+
+	return 0;
+err:
+	stream->bytesused = 0;
+	return ret;
+}
+
+const struct hva_enc nv12h264enc = {
+	.name = "H264(NV12)",
+	.pixelformat = V4L2_PIX_FMT_NV12,
+	.streamformat = V4L2_PIX_FMT_H264,
+	.max_width = H264_MAX_SIZE_W,
+	.max_height = H264_MAX_SIZE_H,
+	.open = hva_h264_open,
+	.close = hva_h264_close,
+	.encode = hva_h264_encode,
+};
+
+const struct hva_enc nv21h264enc = {
+	.name = "H264(NV21)",
+	.pixelformat = V4L2_PIX_FMT_NV21,
+	.streamformat = V4L2_PIX_FMT_H264,
+	.max_width = H264_MAX_SIZE_W,
+	.max_height = H264_MAX_SIZE_H,
+	.open = hva_h264_open,
+	.close = hva_h264_close,
+	.encode = hva_h264_encode,
+};
diff --git a/drivers/media/platform/sti/hva/hva-hw.c b/drivers/media/platform/sti/hva/hva-hw.c
new file mode 100644
index 000000000000..d341d4994528
--- /dev/null
+++ b/drivers/media/platform/sti/hva/hva-hw.c
@@ -0,0 +1,538 @@
+/*
+ * Copyright (C) STMicroelectronics SA 2015
+ * Authors: Yannick Fertre <yannick.fertre@st.com>
+ *          Hugues Fruchet <hugues.fruchet@st.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include "hva.h"
+#include "hva-hw.h"
+
+/* HVA register offsets */
+#define HVA_HIF_REG_RST                 0x0100U
+#define HVA_HIF_REG_RST_ACK             0x0104U
+#define HVA_HIF_REG_MIF_CFG             0x0108U
+#define HVA_HIF_REG_HEC_MIF_CFG         0x010CU
+#define HVA_HIF_REG_CFL                 0x0110U
+#define HVA_HIF_FIFO_CMD                0x0114U
+#define HVA_HIF_FIFO_STS                0x0118U
+#define HVA_HIF_REG_SFL                 0x011CU
+#define HVA_HIF_REG_IT_ACK              0x0120U
+#define HVA_HIF_REG_ERR_IT_ACK          0x0124U
+#define HVA_HIF_REG_LMI_ERR             0x0128U
+#define HVA_HIF_REG_EMI_ERR             0x012CU
+#define HVA_HIF_REG_HEC_MIF_ERR         0x0130U
+#define HVA_HIF_REG_HEC_STS             0x0134U
+#define HVA_HIF_REG_HVC_STS             0x0138U
+#define HVA_HIF_REG_HJE_STS             0x013CU
+#define HVA_HIF_REG_CNT                 0x0140U
+#define HVA_HIF_REG_HEC_CHKSYN_DIS      0x0144U
+#define HVA_HIF_REG_CLK_GATING          0x0148U
+#define HVA_HIF_REG_VERSION             0x014CU
+#define HVA_HIF_REG_BSM                 0x0150U
+
+/* define value for version id register (HVA_HIF_REG_VERSION) */
+#define VERSION_ID_MASK	0x0000FFFF
+
+/* define values for BSM register (HVA_HIF_REG_BSM) */
+#define BSM_CFG_VAL1	0x0003F000
+#define BSM_CFG_VAL2	0x003F0000
+
+/* define values for memory interface register (HVA_HIF_REG_MIF_CFG) */
+#define MIF_CFG_VAL1	0x04460446
+#define MIF_CFG_VAL2	0x04460806
+#define MIF_CFG_VAL3	0x00000000
+
+/* define value for HEC memory interface register (HVA_HIF_REG_MIF_CFG) */
+#define HEC_MIF_CFG_VAL	0x000000C4
+
+/*  Bits definition for clock gating register (HVA_HIF_REG_CLK_GATING) */
+#define CLK_GATING_HVC	BIT(0)
+#define CLK_GATING_HEC	BIT(1)
+#define CLK_GATING_HJE	BIT(2)
+
+/* fix hva clock rate */
+#define CLK_RATE		300000000
+
+/* fix delay for pmruntime */
+#define AUTOSUSPEND_DELAY_MS	3
+
+/*
+ * hw encode error values
+ * NO_ERROR: Success, Task OK
+ * H264_BITSTREAM_OVERSIZE: VECH264 Bitstream size > bitstream buffer
+ * H264_FRAME_SKIPPED: VECH264 Frame skipped (refers to CPB Buffer Size)
+ * H264_SLICE_LIMIT_SIZE: VECH264 MB > slice limit size
+ * H264_MAX_SLICE_NUMBER: VECH264 max slice number reached
+ * H264_SLICE_READY: VECH264 Slice ready
+ * TASK_LIST_FULL: HVA/FPC task list full
+		   (discard latest transform command)
+ * UNKNOWN_COMMAND: Transform command not known by HVA/FPC
+ * WRONG_CODEC_OR_RESOLUTION: Wrong Codec or Resolution Selection
+ * NO_INT_COMPLETION: Time-out on interrupt completion
+ * LMI_ERR: Local Memory Interface Error
+ * EMI_ERR: External Memory Interface Error
+ * HECMI_ERR: HEC Memory Interface Error
+ */
+enum hva_hw_error {
+	NO_ERROR = 0x0,
+	H264_BITSTREAM_OVERSIZE = 0x2,
+	H264_FRAME_SKIPPED = 0x4,
+	H264_SLICE_LIMIT_SIZE = 0x5,
+	H264_MAX_SLICE_NUMBER = 0x7,
+	H264_SLICE_READY = 0x8,
+	TASK_LIST_FULL = 0xF0,
+	UNKNOWN_COMMAND = 0xF1,
+	WRONG_CODEC_OR_RESOLUTION = 0xF4,
+	NO_INT_COMPLETION = 0x100,
+	LMI_ERR = 0x101,
+	EMI_ERR = 0x102,
+	HECMI_ERR = 0x103,
+};
+
+static irqreturn_t hva_hw_its_interrupt(int irq, void *data)
+{
+	struct hva_dev *hva = data;
+
+	/* read status registers */
+	hva->sts_reg = readl_relaxed(hva->regs + HVA_HIF_FIFO_STS);
+	hva->sfl_reg = readl_relaxed(hva->regs + HVA_HIF_REG_SFL);
+
+	/* acknowledge interruption */
+	writel_relaxed(0x1, hva->regs + HVA_HIF_REG_IT_ACK);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t hva_hw_its_irq_thread(int irq, void *arg)
+{
+	struct hva_dev *hva = arg;
+	struct device *dev = hva_to_dev(hva);
+	u32 status = hva->sts_reg & 0xFF;
+	u8 ctx_id = 0;
+	struct hva_ctx *ctx = NULL;
+
+	dev_dbg(dev, "%s     %s: status: 0x%02x fifo level: 0x%02x\n",
+		HVA_PREFIX, __func__, hva->sts_reg & 0xFF, hva->sfl_reg & 0xF);
+
+	/*
+	 * status: task_id[31:16] client_id[15:8] status[7:0]
+	 * the context identifier is retrieved from the client identifier
+	 */
+	ctx_id = (hva->sts_reg & 0xFF00) >> 8;
+	if (ctx_id >= HVA_MAX_INSTANCES) {
+		dev_err(dev, "%s     %s: bad context identifier: %d\n",
+			ctx->name, __func__, ctx_id);
+		ctx->hw_err = true;
+		goto out;
+	}
+
+	ctx = hva->instances[ctx_id];
+	if (!ctx)
+		goto out;
+
+	switch (status) {
+	case NO_ERROR:
+		dev_dbg(dev, "%s     %s: no error\n",
+			ctx->name, __func__);
+		ctx->hw_err = false;
+		break;
+	case H264_SLICE_READY:
+		dev_dbg(dev, "%s     %s: h264 slice ready\n",
+			ctx->name, __func__);
+		ctx->hw_err = false;
+		break;
+	case H264_FRAME_SKIPPED:
+		dev_dbg(dev, "%s     %s: h264 frame skipped\n",
+			ctx->name, __func__);
+		ctx->hw_err = false;
+		break;
+	case H264_BITSTREAM_OVERSIZE:
+		dev_err(dev, "%s     %s:h264 bitstream oversize\n",
+			ctx->name, __func__);
+		ctx->hw_err = true;
+		break;
+	case H264_SLICE_LIMIT_SIZE:
+		dev_err(dev, "%s     %s: h264 slice limit size is reached\n",
+			ctx->name, __func__);
+		ctx->hw_err = true;
+		break;
+	case H264_MAX_SLICE_NUMBER:
+		dev_err(dev, "%s     %s: h264 max slice number is reached\n",
+			ctx->name, __func__);
+		ctx->hw_err = true;
+		break;
+	case TASK_LIST_FULL:
+		dev_err(dev, "%s     %s:task list full\n",
+			ctx->name, __func__);
+		ctx->hw_err = true;
+		break;
+	case UNKNOWN_COMMAND:
+		dev_err(dev, "%s     %s: command not known\n",
+			ctx->name, __func__);
+		ctx->hw_err = true;
+		break;
+	case WRONG_CODEC_OR_RESOLUTION:
+		dev_err(dev, "%s     %s: wrong codec or resolution\n",
+			ctx->name, __func__);
+		ctx->hw_err = true;
+		break;
+	default:
+		dev_err(dev, "%s     %s: status not recognized\n",
+			ctx->name, __func__);
+		ctx->hw_err = true;
+		break;
+	}
+out:
+	complete(&hva->interrupt);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t hva_hw_err_interrupt(int irq, void *data)
+{
+	struct hva_dev *hva = data;
+
+	/* read status registers */
+	hva->sts_reg = readl_relaxed(hva->regs + HVA_HIF_FIFO_STS);
+	hva->sfl_reg = readl_relaxed(hva->regs + HVA_HIF_REG_SFL);
+
+	/* read error registers */
+	hva->lmi_err_reg = readl_relaxed(hva->regs + HVA_HIF_REG_LMI_ERR);
+	hva->emi_err_reg = readl_relaxed(hva->regs + HVA_HIF_REG_EMI_ERR);
+	hva->hec_mif_err_reg = readl_relaxed(hva->regs +
+					     HVA_HIF_REG_HEC_MIF_ERR);
+
+	/* acknowledge interruption */
+	writel_relaxed(0x1, hva->regs + HVA_HIF_REG_IT_ACK);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t hva_hw_err_irq_thread(int irq, void *arg)
+{
+	struct hva_dev *hva = arg;
+	struct device *dev = hva_to_dev(hva);
+	u8 ctx_id = 0;
+	struct hva_ctx *ctx;
+
+	dev_dbg(dev, "%s     status: 0x%02x fifo level: 0x%02x\n",
+		HVA_PREFIX, hva->sts_reg & 0xFF, hva->sfl_reg & 0xF);
+
+	/*
+	 * status: task_id[31:16] client_id[15:8] status[7:0]
+	 * the context identifier is retrieved from the client identifier
+	 */
+	ctx_id = (hva->sts_reg & 0xFF00) >> 8;
+	if (ctx_id >= HVA_MAX_INSTANCES) {
+		dev_err(dev, "%s     bad context identifier: %d\n", HVA_PREFIX,
+			ctx_id);
+		goto out;
+	}
+
+	ctx = hva->instances[ctx_id];
+	if (!ctx)
+		goto out;
+
+	if (hva->lmi_err_reg) {
+		dev_err(dev, "%s     local memory interface error: 0x%08x\n",
+			ctx->name, hva->lmi_err_reg);
+		ctx->hw_err = true;
+	}
+
+	if (hva->lmi_err_reg) {
+		dev_err(dev, "%s     external memory interface error: 0x%08x\n",
+			ctx->name, hva->emi_err_reg);
+		ctx->hw_err = true;
+	}
+
+	if (hva->hec_mif_err_reg) {
+		dev_err(dev, "%s     hec memory interface error: 0x%08x\n",
+			ctx->name, hva->hec_mif_err_reg);
+		ctx->hw_err = true;
+	}
+out:
+	complete(&hva->interrupt);
+
+	return IRQ_HANDLED;
+}
+
+static unsigned long int hva_hw_get_ip_version(struct hva_dev *hva)
+{
+	struct device *dev = hva_to_dev(hva);
+	unsigned long int version;
+
+	if (pm_runtime_get_sync(dev) < 0) {
+		dev_err(dev, "%s     failed to get pm_runtime\n", HVA_PREFIX);
+		mutex_unlock(&hva->protect_mutex);
+		return -EFAULT;
+	}
+
+	version = readl_relaxed(hva->regs + HVA_HIF_REG_VERSION) &
+				VERSION_ID_MASK;
+
+	pm_runtime_put_autosuspend(dev);
+
+	switch (version) {
+	case HVA_VERSION_V400:
+		dev_dbg(dev, "%s     IP hardware version 0x%lx\n",
+			HVA_PREFIX, version);
+		break;
+	default:
+		dev_err(dev, "%s     unknown IP hardware version 0x%lx\n",
+			HVA_PREFIX, version);
+		version = HVA_VERSION_UNKNOWN;
+		break;
+	}
+
+	return version;
+}
+
+int hva_hw_probe(struct platform_device *pdev, struct hva_dev *hva)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *regs;
+	struct resource *esram;
+	int ret;
+
+	WARN_ON(!hva);
+
+	/* get memory for registers */
+	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	hva->regs = devm_ioremap_resource(dev, regs);
+	if (IS_ERR_OR_NULL(hva->regs)) {
+		dev_err(dev, "%s     failed to get regs\n", HVA_PREFIX);
+		return PTR_ERR(hva->regs);
+	}
+
+	/* get memory for esram */
+	esram = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	if (IS_ERR_OR_NULL(esram)) {
+		dev_err(dev, "%s     failed to get esram\n", HVA_PREFIX);
+		return PTR_ERR(esram);
+	}
+	hva->esram_addr = esram->start;
+	hva->esram_size = resource_size(esram);
+
+	dev_info(dev, "%s     esram reserved for address: 0x%x size:%d\n",
+		 HVA_PREFIX, hva->esram_addr, hva->esram_size);
+
+	/* get clock resource */
+	hva->clk = devm_clk_get(dev, "clk_hva");
+	if (IS_ERR(hva->clk)) {
+		dev_err(dev, "%s     failed to get clock\n", HVA_PREFIX);
+		return PTR_ERR(hva->clk);
+	}
+
+	ret = clk_prepare(hva->clk);
+	if (ret < 0) {
+		dev_err(dev, "%s     failed to prepare clock\n", HVA_PREFIX);
+		hva->clk = ERR_PTR(-EINVAL);
+		return ret;
+	}
+
+	/* get status interruption resource */
+	ret  = platform_get_irq(pdev, 0);
+	if (ret < 0) {
+		dev_err(dev, "%s     failed to get status IRQ\n", HVA_PREFIX);
+		goto err_clk;
+	}
+	hva->irq_its = ret;
+
+	ret = devm_request_threaded_irq(dev, hva->irq_its, hva_hw_its_interrupt,
+					hva_hw_its_irq_thread,
+					IRQF_ONESHOT,
+					"hva_its_irq", hva);
+	if (ret) {
+		dev_err(dev, "%s     failed to install status IRQ 0x%x\n",
+			HVA_PREFIX, hva->irq_its);
+		goto err_clk;
+	}
+	disable_irq(hva->irq_its);
+
+	/* get error interruption resource */
+	ret = platform_get_irq(pdev, 1);
+	if (ret < 0) {
+		dev_err(dev, "%s     failed to get error IRQ\n", HVA_PREFIX);
+		goto err_clk;
+	}
+	hva->irq_err = ret;
+
+	ret = devm_request_threaded_irq(dev, hva->irq_err, hva_hw_err_interrupt,
+					hva_hw_err_irq_thread,
+					IRQF_ONESHOT,
+					"hva_err_irq", hva);
+	if (ret) {
+		dev_err(dev, "%s     failed to install error IRQ 0x%x\n",
+			HVA_PREFIX, hva->irq_err);
+		goto err_clk;
+	}
+	disable_irq(hva->irq_err);
+
+	/* initialise protection mutex */
+	mutex_init(&hva->protect_mutex);
+
+	/* initialise completion signal */
+	init_completion(&hva->interrupt);
+
+	/* initialise runtime power management */
+	pm_runtime_set_autosuspend_delay(dev, AUTOSUSPEND_DELAY_MS);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_suspended(dev);
+	pm_runtime_enable(dev);
+
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "%s     failed to set PM\n", HVA_PREFIX);
+		goto err_clk;
+	}
+
+	/* check IP hardware version */
+	hva->ip_version = hva_hw_get_ip_version(hva);
+
+	if (hva->ip_version == HVA_VERSION_UNKNOWN) {
+		ret = -EINVAL;
+		goto err_pm;
+	}
+
+	dev_info(dev, "%s     found hva device (version 0x%lx)\n", HVA_PREFIX,
+		 hva->ip_version);
+
+	return 0;
+
+err_pm:
+	pm_runtime_put(dev);
+err_clk:
+	if (hva->clk)
+		clk_unprepare(hva->clk);
+
+	return ret;
+}
+
+void hva_hw_remove(struct hva_dev *hva)
+{
+	struct device *dev = hva_to_dev(hva);
+
+	disable_irq(hva->irq_its);
+	disable_irq(hva->irq_err);
+
+	pm_runtime_put_autosuspend(dev);
+	pm_runtime_disable(dev);
+}
+
+int hva_hw_runtime_suspend(struct device *dev)
+{
+	struct hva_dev *hva = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(hva->clk);
+
+	return 0;
+}
+
+int hva_hw_runtime_resume(struct device *dev)
+{
+	struct hva_dev *hva = dev_get_drvdata(dev);
+
+	if (clk_prepare_enable(hva->clk)) {
+		dev_err(hva->dev, "%s     failed to prepare hva clk\n",
+			HVA_PREFIX);
+		return -EINVAL;
+	}
+
+	if (clk_set_rate(hva->clk, CLK_RATE)) {
+		dev_err(dev, "%s     failed to set clock frequency\n",
+			HVA_PREFIX);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int hva_hw_execute_task(struct hva_ctx *ctx, enum hva_hw_cmd_type cmd,
+			struct hva_buffer *task)
+{
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+	struct device *dev = hva_to_dev(hva);
+	u8 client_id = ctx->id;
+	int ret;
+	u32 reg = 0;
+
+	mutex_lock(&hva->protect_mutex);
+
+	/* enable irqs */
+	enable_irq(hva->irq_its);
+	enable_irq(hva->irq_err);
+
+	if (pm_runtime_get_sync(dev) < 0) {
+		dev_err(dev, "%s     failed to get pm_runtime\n", ctx->name);
+		ret = -EFAULT;
+		goto out;
+	}
+
+	reg = readl_relaxed(hva->regs + HVA_HIF_REG_CLK_GATING);
+	switch (cmd) {
+	case H264_ENC:
+		reg |= CLK_GATING_HVC;
+		break;
+	default:
+		dev_dbg(dev, "%s     unknown command 0x%x\n", ctx->name, cmd);
+		ret = -EFAULT;
+		goto out;
+	}
+	writel_relaxed(reg, hva->regs + HVA_HIF_REG_CLK_GATING);
+
+	dev_dbg(dev, "%s     %s: write configuration registers\n", ctx->name,
+		__func__);
+
+	/* byte swap config */
+	writel_relaxed(BSM_CFG_VAL1, hva->regs + HVA_HIF_REG_BSM);
+
+	/* define Max Opcode Size and Max Message Size for LMI and EMI */
+	writel_relaxed(MIF_CFG_VAL3, hva->regs + HVA_HIF_REG_MIF_CFG);
+	writel_relaxed(HEC_MIF_CFG_VAL, hva->regs + HVA_HIF_REG_HEC_MIF_CFG);
+
+	/*
+	 * command FIFO: task_id[31:16] client_id[15:8] command_type[7:0]
+	 * the context identifier is provided as client identifier to the
+	 * hardware, and is retrieved in the interrupt functions from the
+	 * status register
+	 */
+	dev_dbg(dev, "%s     %s: send task (cmd: %d, task_desc: %pad)\n",
+		ctx->name, __func__, cmd + (client_id << 8), &task->paddr);
+	writel_relaxed(cmd + (client_id << 8), hva->regs + HVA_HIF_FIFO_CMD);
+	writel_relaxed(task->paddr, hva->regs + HVA_HIF_FIFO_CMD);
+
+	if (!wait_for_completion_timeout(&hva->interrupt,
+					 msecs_to_jiffies(2000))) {
+		dev_err(dev, "%s     %s: time out on completion\n", ctx->name,
+			__func__);
+		ret = -EFAULT;
+		goto out;
+	}
+
+	/* get encoding status */
+	ret = ctx->hw_err ? -EFAULT : 0;
+
+out:
+	disable_irq(hva->irq_its);
+	disable_irq(hva->irq_err);
+
+	switch (cmd) {
+	case H264_ENC:
+		reg &= ~CLK_GATING_HVC;
+		writel_relaxed(reg, hva->regs + HVA_HIF_REG_CLK_GATING);
+		break;
+	default:
+		dev_dbg(dev, "%s     unknown command 0x%x\n", ctx->name, cmd);
+	}
+
+	pm_runtime_put_autosuspend(dev);
+	mutex_unlock(&hva->protect_mutex);
+
+	return ret;
+}
diff --git a/drivers/media/platform/sti/hva/hva-hw.h b/drivers/media/platform/sti/hva/hva-hw.h
new file mode 100644
index 000000000000..efb45b927524
--- /dev/null
+++ b/drivers/media/platform/sti/hva/hva-hw.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) STMicroelectronics SA 2015
+ * Authors: Yannick Fertre <yannick.fertre@st.com>
+ *          Hugues Fruchet <hugues.fruchet@st.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#ifndef HVA_HW_H
+#define HVA_HW_H
+
+#include "hva-mem.h"
+
+/* HVA Versions */
+#define HVA_VERSION_UNKNOWN    0x000
+#define HVA_VERSION_V400       0x400
+
+/* HVA command types */
+enum hva_hw_cmd_type {
+	/* RESERVED = 0x00 */
+	/* RESERVED = 0x01 */
+	H264_ENC = 0x02,
+	/* RESERVED = 0x03 */
+	/* RESERVED = 0x04 */
+	/* RESERVED = 0x05 */
+	/* RESERVED = 0x06 */
+	/* RESERVED = 0x07 */
+	REMOVE_CLIENT = 0x08,
+	FREEZE_CLIENT = 0x09,
+	START_CLIENT = 0x0A,
+	FREEZE_ALL = 0x0B,
+	START_ALL = 0x0C,
+	REMOVE_ALL = 0x0D
+};
+
+int hva_hw_probe(struct platform_device *pdev, struct hva_dev *hva);
+void hva_hw_remove(struct hva_dev *hva);
+int hva_hw_runtime_suspend(struct device *dev);
+int hva_hw_runtime_resume(struct device *dev);
+int hva_hw_execute_task(struct hva_ctx *ctx, enum hva_hw_cmd_type cmd,
+			struct hva_buffer *task);
+
+#endif /* HVA_HW_H */
diff --git a/drivers/media/platform/sti/hva/hva-mem.c b/drivers/media/platform/sti/hva/hva-mem.c
new file mode 100644
index 000000000000..649f66007ad6
--- /dev/null
+++ b/drivers/media/platform/sti/hva/hva-mem.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) STMicroelectronics SA 2015
+ * Authors: Yannick Fertre <yannick.fertre@st.com>
+ *          Hugues Fruchet <hugues.fruchet@st.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include "hva.h"
+#include "hva-mem.h"
+
+int hva_mem_alloc(struct hva_ctx *ctx, u32 size, const char *name,
+		  struct hva_buffer **buf)
+{
+	struct device *dev = ctx_to_dev(ctx);
+	struct hva_buffer *b;
+	dma_addr_t paddr;
+	void *base;
+
+	b = devm_kzalloc(dev, sizeof(*b), GFP_KERNEL);
+	if (!b)
+		return -ENOMEM;
+
+	base = dma_alloc_attrs(dev, size, &paddr, GFP_KERNEL | GFP_DMA,
+			       DMA_ATTR_WRITE_COMBINE);
+	if (!base) {
+		dev_err(dev, "%s %s : dma_alloc_attrs failed for %s (size=%d)\n",
+			ctx->name, __func__, name, size);
+		devm_kfree(dev, b);
+		return -ENOMEM;
+	}
+
+	b->size = size;
+	b->paddr = paddr;
+	b->vaddr = base;
+	b->name = name;
+
+	dev_dbg(dev,
+		"%s allocate %d bytes of HW memory @(virt=%p, phy=%pad): %s\n",
+		ctx->name, size, b->vaddr, &b->paddr, b->name);
+
+	/* return  hva buffer to user */
+	*buf = b;
+
+	return 0;
+}
+
+void hva_mem_free(struct hva_ctx *ctx, struct hva_buffer *buf)
+{
+	struct device *dev = ctx_to_dev(ctx);
+
+	dev_dbg(dev,
+		"%s free %d bytes of HW memory @(virt=%p, phy=%pad): %s\n",
+		ctx->name, buf->size, buf->vaddr, &buf->paddr, buf->name);
+
+	dma_free_attrs(dev, buf->size, buf->vaddr, buf->paddr,
+		       DMA_ATTR_WRITE_COMBINE);
+
+	devm_kfree(dev, buf);
+}
diff --git a/drivers/media/platform/sti/hva/hva-mem.h b/drivers/media/platform/sti/hva/hva-mem.h
new file mode 100644
index 000000000000..a95c728a45e6
--- /dev/null
+++ b/drivers/media/platform/sti/hva/hva-mem.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) STMicroelectronics SA 2015
+ * Authors: Yannick Fertre <yannick.fertre@st.com>
+ *          Hugues Fruchet <hugues.fruchet@st.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#ifndef HVA_MEM_H
+#define HVA_MEM_H
+
+/**
+ * struct hva_buffer - hva buffer
+ *
+ * @name:  name of requester
+ * @paddr: physical address (for hardware)
+ * @vaddr: virtual address (kernel can read/write)
+ * @size:  size of buffer
+ */
+struct hva_buffer {
+	const char		*name;
+	dma_addr_t		paddr;
+	void			*vaddr;
+	u32			size;
+};
+
+int hva_mem_alloc(struct hva_ctx *ctx,
+		  __u32 size,
+		  const char *name,
+		  struct hva_buffer **buf);
+
+void hva_mem_free(struct hva_ctx *ctx,
+		  struct hva_buffer *buf);
+
+#endif /* HVA_MEM_H */
diff --git a/drivers/media/platform/sti/hva/hva-v4l2.c b/drivers/media/platform/sti/hva/hva-v4l2.c
new file mode 100644
index 000000000000..6bf3c8588230
--- /dev/null
+++ b/drivers/media/platform/sti/hva/hva-v4l2.c
@@ -0,0 +1,1415 @@
+/*
+ * Copyright (C) STMicroelectronics SA 2015
+ * Authors: Yannick Fertre <yannick.fertre@st.com>
+ *          Hugues Fruchet <hugues.fruchet@st.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "hva.h"
+#include "hva-hw.h"
+
+#define HVA_NAME "st-hva"
+
+#define MIN_FRAMES	1
+#define MIN_STREAMS	1
+
+#define HVA_MIN_WIDTH	32
+#define HVA_MAX_WIDTH	1920
+#define HVA_MIN_HEIGHT	32
+#define HVA_MAX_HEIGHT	1920
+
+/* HVA requires a 16x16 pixels alignment for frames */
+#define HVA_WIDTH_ALIGNMENT	16
+#define HVA_HEIGHT_ALIGNMENT	16
+
+#define HVA_DEFAULT_WIDTH	HVA_MIN_WIDTH
+#define	HVA_DEFAULT_HEIGHT	HVA_MIN_HEIGHT
+#define HVA_DEFAULT_FRAME_NUM	1
+#define HVA_DEFAULT_FRAME_DEN	30
+
+#define to_type_str(type) (type == V4L2_BUF_TYPE_VIDEO_OUTPUT ? \
+			   "frame" : "stream")
+
+#define fh_to_ctx(f)    (container_of(f, struct hva_ctx, fh))
+
+/* registry of available encoders */
+static const struct hva_enc *hva_encoders[] = {
+	&nv12h264enc,
+	&nv21h264enc,
+};
+
+static inline int frame_size(u32 w, u32 h, u32 fmt)
+{
+	switch (fmt) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_NV21:
+		return (w * h * 3) / 2;
+	default:
+		return 0;
+	}
+}
+
+static inline int frame_stride(u32 w, u32 fmt)
+{
+	switch (fmt) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_NV21:
+		return w;
+	default:
+		return 0;
+	}
+}
+
+static inline int frame_alignment(u32 fmt)
+{
+	switch (fmt) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_NV21:
+		/* multiple of 2 */
+		return 2;
+	default:
+		return 1;
+	}
+}
+
+static inline int estimated_stream_size(u32 w, u32 h)
+{
+	/*
+	 * HVA only encodes in YUV420 format, whatever the frame format.
+	 * A compression ratio of 2 is assumed: thus, the maximum size
+	 * of a stream is estimated to ((width x height x 3 / 2) / 2)
+	 */
+	return (w * h * 3) / 4;
+}
+
+static void set_default_params(struct hva_ctx *ctx)
+{
+	struct hva_frameinfo *frameinfo = &ctx->frameinfo;
+	struct hva_streaminfo *streaminfo = &ctx->streaminfo;
+
+	frameinfo->pixelformat = V4L2_PIX_FMT_NV12;
+	frameinfo->width = HVA_DEFAULT_WIDTH;
+	frameinfo->height = HVA_DEFAULT_HEIGHT;
+	frameinfo->aligned_width = ALIGN(frameinfo->width,
+					 HVA_WIDTH_ALIGNMENT);
+	frameinfo->aligned_height = ALIGN(frameinfo->height,
+					  HVA_HEIGHT_ALIGNMENT);
+	frameinfo->size = frame_size(frameinfo->aligned_width,
+				     frameinfo->aligned_height,
+				     frameinfo->pixelformat);
+
+	streaminfo->streamformat = V4L2_PIX_FMT_H264;
+	streaminfo->width = HVA_DEFAULT_WIDTH;
+	streaminfo->height = HVA_DEFAULT_HEIGHT;
+
+	ctx->colorspace = V4L2_COLORSPACE_REC709;
+	ctx->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+	ctx->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	ctx->quantization = V4L2_QUANTIZATION_DEFAULT;
+
+	ctx->max_stream_size = estimated_stream_size(streaminfo->width,
+						     streaminfo->height);
+}
+
+static const struct hva_enc *hva_find_encoder(struct hva_ctx *ctx,
+					      u32 pixelformat,
+					      u32 streamformat)
+{
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+	const struct hva_enc *enc;
+	unsigned int i;
+
+	for (i = 0; i < hva->nb_of_encoders; i++) {
+		enc = hva->encoders[i];
+		if ((enc->pixelformat == pixelformat) &&
+		    (enc->streamformat == streamformat))
+			return enc;
+	}
+
+	return NULL;
+}
+
+static void register_format(u32 format, u32 formats[], u32 *nb_of_formats)
+{
+	u32 i;
+	bool found = false;
+
+	for (i = 0; i < *nb_of_formats; i++) {
+		if (format == formats[i]) {
+			found = true;
+			break;
+		}
+	}
+
+	if (!found)
+		formats[(*nb_of_formats)++] = format;
+}
+
+static void register_formats(struct hva_dev *hva)
+{
+	unsigned int i;
+
+	for (i = 0; i < hva->nb_of_encoders; i++) {
+		register_format(hva->encoders[i]->pixelformat,
+				hva->pixelformats,
+				&hva->nb_of_pixelformats);
+
+		register_format(hva->encoders[i]->streamformat,
+				hva->streamformats,
+				&hva->nb_of_streamformats);
+	}
+}
+
+static void register_encoders(struct hva_dev *hva)
+{
+	struct device *dev = hva_to_dev(hva);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(hva_encoders); i++) {
+		if (hva->nb_of_encoders >= HVA_MAX_ENCODERS) {
+			dev_dbg(dev,
+				"%s failed to register %s encoder (%d maximum reached)\n",
+				HVA_PREFIX, hva_encoders[i]->name,
+				HVA_MAX_ENCODERS);
+			return;
+		}
+
+		hva->encoders[hva->nb_of_encoders++] = hva_encoders[i];
+		dev_info(dev, "%s %s encoder registered\n", HVA_PREFIX,
+			 hva_encoders[i]->name);
+	}
+}
+
+static int hva_open_encoder(struct hva_ctx *ctx, u32 streamformat,
+			    u32 pixelformat, struct hva_enc **penc)
+{
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+	struct device *dev = ctx_to_dev(ctx);
+	struct hva_enc *enc;
+	int ret;
+
+	/* find an encoder which can deal with these formats */
+	enc = (struct hva_enc *)hva_find_encoder(ctx, pixelformat,
+						 streamformat);
+	if (!enc) {
+		dev_err(dev, "%s no encoder found matching %4.4s => %4.4s\n",
+			ctx->name, (char *)&pixelformat, (char *)&streamformat);
+		return -EINVAL;
+	}
+
+	dev_dbg(dev, "%s one encoder matching %4.4s => %4.4s\n",
+		ctx->name, (char *)&pixelformat, (char *)&streamformat);
+
+	/* update instance name */
+	snprintf(ctx->name, sizeof(ctx->name), "[%3d:%4.4s]",
+		 hva->instance_id, (char *)&streamformat);
+
+	/* open encoder instance */
+	ret = enc->open(ctx);
+	if (ret) {
+		dev_err(dev, "%s failed to open encoder instance (%d)\n",
+			ctx->name, ret);
+		return ret;
+	}
+
+	dev_dbg(dev, "%s %s encoder opened\n", ctx->name, enc->name);
+
+	*penc = enc;
+
+	return ret;
+}
+
+/*
+ * V4L2 ioctl operations
+ */
+
+static int hva_querycap(struct file *file, void *priv,
+			struct v4l2_capability *cap)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+
+	strlcpy(cap->driver, HVA_NAME, sizeof(cap->driver));
+	strlcpy(cap->card, hva->vdev->name, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
+		 hva->pdev->name);
+
+	return 0;
+}
+
+static int hva_enum_fmt_stream(struct file *file, void *priv,
+			       struct v4l2_fmtdesc *f)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+
+	if (unlikely(f->index >= hva->nb_of_streamformats))
+		return -EINVAL;
+
+	f->pixelformat = hva->streamformats[f->index];
+
+	return 0;
+}
+
+static int hva_enum_fmt_frame(struct file *file, void *priv,
+			      struct v4l2_fmtdesc *f)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+
+	if (unlikely(f->index >= hva->nb_of_pixelformats))
+		return -EINVAL;
+
+	f->pixelformat = hva->pixelformats[f->index];
+
+	return 0;
+}
+
+static int hva_g_fmt_stream(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct hva_streaminfo *streaminfo = &ctx->streaminfo;
+
+	f->fmt.pix.width = streaminfo->width;
+	f->fmt.pix.height = streaminfo->height;
+	f->fmt.pix.field = V4L2_FIELD_NONE;
+	f->fmt.pix.colorspace = ctx->colorspace;
+	f->fmt.pix.xfer_func = ctx->xfer_func;
+	f->fmt.pix.ycbcr_enc = ctx->ycbcr_enc;
+	f->fmt.pix.quantization = ctx->quantization;
+	f->fmt.pix.pixelformat = streaminfo->streamformat;
+	f->fmt.pix.bytesperline = 0;
+	f->fmt.pix.sizeimage = ctx->max_stream_size;
+
+	return 0;
+}
+
+static int hva_g_fmt_frame(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct hva_frameinfo *frameinfo = &ctx->frameinfo;
+
+	f->fmt.pix.width = frameinfo->width;
+	f->fmt.pix.height = frameinfo->height;
+	f->fmt.pix.field = V4L2_FIELD_NONE;
+	f->fmt.pix.colorspace = ctx->colorspace;
+	f->fmt.pix.xfer_func = ctx->xfer_func;
+	f->fmt.pix.ycbcr_enc = ctx->ycbcr_enc;
+	f->fmt.pix.quantization = ctx->quantization;
+	f->fmt.pix.pixelformat = frameinfo->pixelformat;
+	f->fmt.pix.bytesperline = frame_stride(frameinfo->aligned_width,
+					       frameinfo->pixelformat);
+	f->fmt.pix.sizeimage = frameinfo->size;
+
+	return 0;
+}
+
+static int hva_try_fmt_stream(struct file *file, void *priv,
+			      struct v4l2_format *f)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct device *dev = ctx_to_dev(ctx);
+	struct v4l2_pix_format *pix = &f->fmt.pix;
+	u32 streamformat = pix->pixelformat;
+	const struct hva_enc *enc;
+	u32 width, height;
+	u32 stream_size;
+
+	enc = hva_find_encoder(ctx, ctx->frameinfo.pixelformat, streamformat);
+	if (!enc) {
+		dev_dbg(dev,
+			"%s V4L2 TRY_FMT (CAPTURE): unsupported format %.4s\n",
+			ctx->name, (char *)&pix->pixelformat);
+		return -EINVAL;
+	}
+
+	width = pix->width;
+	height = pix->height;
+	if (ctx->flags & HVA_FLAG_FRAMEINFO) {
+		/*
+		 * if the frame resolution is already fixed, only allow the
+		 * same stream resolution
+		 */
+		pix->width = ctx->frameinfo.width;
+		pix->height = ctx->frameinfo.height;
+		if ((pix->width != width) || (pix->height != height))
+			dev_dbg(dev,
+				"%s V4L2 TRY_FMT (CAPTURE): resolution updated %dx%d -> %dx%d to fit frame resolution\n",
+				ctx->name, width, height,
+				pix->width, pix->height);
+	} else {
+		/* adjust width & height */
+		v4l_bound_align_image(&pix->width,
+				      HVA_MIN_WIDTH, enc->max_width,
+				      0,
+				      &pix->height,
+				      HVA_MIN_HEIGHT, enc->max_height,
+				      0,
+				      0);
+
+		if ((pix->width != width) || (pix->height != height))
+			dev_dbg(dev,
+				"%s V4L2 TRY_FMT (CAPTURE): resolution updated %dx%d -> %dx%d to fit min/max/alignment\n",
+				ctx->name, width, height,
+				pix->width, pix->height);
+	}
+
+	stream_size = estimated_stream_size(pix->width, pix->height);
+	if (pix->sizeimage < stream_size)
+		pix->sizeimage = stream_size;
+
+	pix->bytesperline = 0;
+	pix->colorspace = ctx->colorspace;
+	pix->xfer_func = ctx->xfer_func;
+	pix->ycbcr_enc = ctx->ycbcr_enc;
+	pix->quantization = ctx->quantization;
+	pix->field = V4L2_FIELD_NONE;
+
+	return 0;
+}
+
+static int hva_try_fmt_frame(struct file *file, void *priv,
+			     struct v4l2_format *f)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct device *dev = ctx_to_dev(ctx);
+	struct v4l2_pix_format *pix = &f->fmt.pix;
+	u32 pixelformat = pix->pixelformat;
+	const struct hva_enc *enc;
+	u32 width, height;
+
+	enc = hva_find_encoder(ctx, pixelformat, ctx->streaminfo.streamformat);
+	if (!enc) {
+		dev_dbg(dev,
+			"%s V4L2 TRY_FMT (OUTPUT): unsupported format %.4s\n",
+			ctx->name, (char *)&pixelformat);
+		return -EINVAL;
+	}
+
+	/* adjust width & height */
+	width = pix->width;
+	height = pix->height;
+	v4l_bound_align_image(&pix->width,
+			      HVA_MIN_WIDTH, HVA_MAX_WIDTH,
+			      frame_alignment(pixelformat) - 1,
+			      &pix->height,
+			      HVA_MIN_HEIGHT, HVA_MAX_HEIGHT,
+			      frame_alignment(pixelformat) - 1,
+			      0);
+
+	if ((pix->width != width) || (pix->height != height))
+		dev_dbg(dev,
+			"%s V4L2 TRY_FMT (OUTPUT): resolution updated %dx%d -> %dx%d to fit min/max/alignment\n",
+			ctx->name, width, height, pix->width, pix->height);
+
+	width = ALIGN(pix->width, HVA_WIDTH_ALIGNMENT);
+	height = ALIGN(pix->height, HVA_HEIGHT_ALIGNMENT);
+
+	if (!pix->colorspace) {
+		pix->colorspace = V4L2_COLORSPACE_REC709;
+		pix->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+		pix->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+		pix->quantization = V4L2_QUANTIZATION_DEFAULT;
+	}
+
+	pix->bytesperline = frame_stride(width, pixelformat);
+	pix->sizeimage = frame_size(width, height, pixelformat);
+	pix->field = V4L2_FIELD_NONE;
+
+	return 0;
+}
+
+static int hva_s_fmt_stream(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct device *dev = ctx_to_dev(ctx);
+	struct vb2_queue *vq;
+	int ret;
+
+	ret = hva_try_fmt_stream(file, fh, f);
+	if (ret) {
+		dev_dbg(dev, "%s V4L2 S_FMT (CAPTURE): unsupported format %.4s\n",
+			ctx->name, (char *)&f->fmt.pix.pixelformat);
+		return ret;
+	}
+
+	vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
+	if (vb2_is_streaming(vq)) {
+		dev_dbg(dev, "%s V4L2 S_FMT (CAPTURE): queue busy\n",
+			ctx->name);
+		return -EBUSY;
+	}
+
+	ctx->max_stream_size = f->fmt.pix.sizeimage;
+	ctx->streaminfo.width = f->fmt.pix.width;
+	ctx->streaminfo.height = f->fmt.pix.height;
+	ctx->streaminfo.streamformat = f->fmt.pix.pixelformat;
+	ctx->flags |= HVA_FLAG_STREAMINFO;
+
+	return 0;
+}
+
+static int hva_s_fmt_frame(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct device *dev = ctx_to_dev(ctx);
+	struct v4l2_pix_format *pix = &f->fmt.pix;
+	struct vb2_queue *vq;
+	int ret;
+
+	ret = hva_try_fmt_frame(file, fh, f);
+	if (ret) {
+		dev_dbg(dev, "%s V4L2 S_FMT (OUTPUT): unsupported format %.4s\n",
+			ctx->name, (char *)&pix->pixelformat);
+		return ret;
+	}
+
+	vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
+	if (vb2_is_streaming(vq)) {
+		dev_dbg(dev, "%s V4L2 S_FMT (OUTPUT): queue busy\n", ctx->name);
+		return -EBUSY;
+	}
+
+	ctx->colorspace = pix->colorspace;
+	ctx->xfer_func = pix->xfer_func;
+	ctx->ycbcr_enc = pix->ycbcr_enc;
+	ctx->quantization = pix->quantization;
+
+	ctx->frameinfo.aligned_width = ALIGN(pix->width, HVA_WIDTH_ALIGNMENT);
+	ctx->frameinfo.aligned_height = ALIGN(pix->height,
+					      HVA_HEIGHT_ALIGNMENT);
+	ctx->frameinfo.size = pix->sizeimage;
+	ctx->frameinfo.pixelformat = pix->pixelformat;
+	ctx->frameinfo.width = pix->width;
+	ctx->frameinfo.height = pix->height;
+	ctx->flags |= HVA_FLAG_FRAMEINFO;
+
+	return 0;
+}
+
+static int hva_g_parm(struct file *file, void *fh, struct v4l2_streamparm *sp)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct v4l2_fract *time_per_frame = &ctx->ctrls.time_per_frame;
+
+	if (sp->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+		return -EINVAL;
+
+	sp->parm.output.capability = V4L2_CAP_TIMEPERFRAME;
+	sp->parm.output.timeperframe.numerator = time_per_frame->numerator;
+	sp->parm.output.timeperframe.denominator =
+		time_per_frame->denominator;
+
+	return 0;
+}
+
+static int hva_s_parm(struct file *file, void *fh, struct v4l2_streamparm *sp)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct v4l2_fract *time_per_frame = &ctx->ctrls.time_per_frame;
+
+	if (sp->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+		return -EINVAL;
+
+	if (!sp->parm.output.timeperframe.numerator ||
+	    !sp->parm.output.timeperframe.denominator)
+		return hva_g_parm(file, fh, sp);
+
+	sp->parm.output.capability = V4L2_CAP_TIMEPERFRAME;
+	time_per_frame->numerator = sp->parm.output.timeperframe.numerator;
+	time_per_frame->denominator =
+		sp->parm.output.timeperframe.denominator;
+
+	return 0;
+}
+
+static int hva_qbuf(struct file *file, void *priv, struct v4l2_buffer *buf)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct device *dev = ctx_to_dev(ctx);
+
+	if (buf->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+		/*
+		 * depending on the targeted compressed video format, the
+		 * capture buffer might contain headers (e.g. H.264 SPS/PPS)
+		 * filled in by the driver client; the size of these data is
+		 * copied from the bytesused field of the V4L2 buffer in the
+		 * payload field of the hva stream buffer
+		 */
+		struct vb2_queue *vq;
+		struct hva_stream *stream;
+
+		vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, buf->type);
+
+		if (buf->index >= vq->num_buffers) {
+			dev_dbg(dev, "%s buffer index %d out of range (%d)\n",
+				ctx->name, buf->index, vq->num_buffers);
+			return -EINVAL;
+		}
+
+		stream = (struct hva_stream *)vq->bufs[buf->index];
+		stream->bytesused = buf->bytesused;
+	}
+
+	return v4l2_m2m_qbuf(file, ctx->fh.m2m_ctx, buf);
+}
+
+/* V4L2 ioctl ops */
+static const struct v4l2_ioctl_ops hva_ioctl_ops = {
+	.vidioc_querycap		= hva_querycap,
+	.vidioc_enum_fmt_vid_cap	= hva_enum_fmt_stream,
+	.vidioc_enum_fmt_vid_out	= hva_enum_fmt_frame,
+	.vidioc_g_fmt_vid_cap		= hva_g_fmt_stream,
+	.vidioc_g_fmt_vid_out		= hva_g_fmt_frame,
+	.vidioc_try_fmt_vid_cap		= hva_try_fmt_stream,
+	.vidioc_try_fmt_vid_out		= hva_try_fmt_frame,
+	.vidioc_s_fmt_vid_cap		= hva_s_fmt_stream,
+	.vidioc_s_fmt_vid_out		= hva_s_fmt_frame,
+	.vidioc_g_parm			= hva_g_parm,
+	.vidioc_s_parm			= hva_s_parm,
+	.vidioc_reqbufs			= v4l2_m2m_ioctl_reqbufs,
+	.vidioc_create_bufs             = v4l2_m2m_ioctl_create_bufs,
+	.vidioc_querybuf		= v4l2_m2m_ioctl_querybuf,
+	.vidioc_expbuf			= v4l2_m2m_ioctl_expbuf,
+	.vidioc_qbuf			= hva_qbuf,
+	.vidioc_dqbuf			= v4l2_m2m_ioctl_dqbuf,
+	.vidioc_streamon		= v4l2_m2m_ioctl_streamon,
+	.vidioc_streamoff		= v4l2_m2m_ioctl_streamoff,
+	.vidioc_subscribe_event		= v4l2_ctrl_subscribe_event,
+	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
+};
+
+/*
+ * V4L2 control operations
+ */
+
+static int hva_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct hva_ctx *ctx = container_of(ctrl->handler, struct hva_ctx,
+					   ctrl_handler);
+	struct device *dev = ctx_to_dev(ctx);
+
+	dev_dbg(dev, "%s S_CTRL: id = %d, val = %d\n", ctx->name,
+		ctrl->id, ctrl->val);
+
+	switch (ctrl->id) {
+	case V4L2_CID_MPEG_VIDEO_BITRATE_MODE:
+		ctx->ctrls.bitrate_mode = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_GOP_SIZE:
+		ctx->ctrls.gop_size = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_BITRATE:
+		ctx->ctrls.bitrate = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_ASPECT:
+		ctx->ctrls.aspect = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_PROFILE:
+		ctx->ctrls.profile = ctrl->val;
+		if (ctx->flags & HVA_FLAG_STREAMINFO)
+			snprintf(ctx->streaminfo.profile,
+				 sizeof(ctx->streaminfo.profile),
+				 "%s profile",
+				 v4l2_ctrl_get_menu(ctrl->id)[ctrl->val]);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
+		ctx->ctrls.level = ctrl->val;
+		if (ctx->flags & HVA_FLAG_STREAMINFO)
+			snprintf(ctx->streaminfo.level,
+				 sizeof(ctx->streaminfo.level),
+				 "level %s",
+				 v4l2_ctrl_get_menu(ctrl->id)[ctrl->val]);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE:
+		ctx->ctrls.entropy_mode = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_CPB_SIZE:
+		ctx->ctrls.cpb_size = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM:
+		ctx->ctrls.dct8x8 = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_MIN_QP:
+		ctx->ctrls.qpmin = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_MAX_QP:
+		ctx->ctrls.qpmax = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE:
+		ctx->ctrls.vui_sar = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_IDC:
+		ctx->ctrls.vui_sar_idc = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_SEI_FRAME_PACKING:
+		ctx->ctrls.sei_fp = ctrl->val;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE:
+		ctx->ctrls.sei_fp_type = ctrl->val;
+		break;
+	default:
+		dev_dbg(dev, "%s S_CTRL: invalid control (id = %d)\n",
+			ctx->name, ctrl->id);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* V4L2 control ops */
+static const struct v4l2_ctrl_ops hva_ctrl_ops = {
+	.s_ctrl = hva_s_ctrl,
+};
+
+static int hva_ctrls_setup(struct hva_ctx *ctx)
+{
+	struct device *dev = ctx_to_dev(ctx);
+	u64 mask;
+	enum v4l2_mpeg_video_h264_sei_fp_arrangement_type sei_fp_type =
+		V4L2_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE_TOP_BOTTOM;
+
+	v4l2_ctrl_handler_init(&ctx->ctrl_handler, 15);
+
+	v4l2_ctrl_new_std_menu(&ctx->ctrl_handler, &hva_ctrl_ops,
+			       V4L2_CID_MPEG_VIDEO_BITRATE_MODE,
+			       V4L2_MPEG_VIDEO_BITRATE_MODE_CBR,
+			       0,
+			       V4L2_MPEG_VIDEO_BITRATE_MODE_CBR);
+
+	v4l2_ctrl_new_std(&ctx->ctrl_handler, &hva_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_GOP_SIZE,
+			  1, 60, 1, 16);
+
+	v4l2_ctrl_new_std(&ctx->ctrl_handler, &hva_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_BITRATE,
+			  1000, 60000000, 1000, 20000000);
+
+	mask = ~(1 << V4L2_MPEG_VIDEO_ASPECT_1x1);
+	v4l2_ctrl_new_std_menu(&ctx->ctrl_handler, &hva_ctrl_ops,
+			       V4L2_CID_MPEG_VIDEO_ASPECT,
+			       V4L2_MPEG_VIDEO_ASPECT_1x1,
+			       mask,
+			       V4L2_MPEG_VIDEO_ASPECT_1x1);
+
+	mask = ~((1 << V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) |
+		 (1 << V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) |
+		 (1 << V4L2_MPEG_VIDEO_H264_PROFILE_HIGH) |
+		 (1 << V4L2_MPEG_VIDEO_H264_PROFILE_STEREO_HIGH));
+	v4l2_ctrl_new_std_menu(&ctx->ctrl_handler, &hva_ctrl_ops,
+			       V4L2_CID_MPEG_VIDEO_H264_PROFILE,
+			       V4L2_MPEG_VIDEO_H264_PROFILE_STEREO_HIGH,
+			       mask,
+			       V4L2_MPEG_VIDEO_H264_PROFILE_HIGH);
+
+	v4l2_ctrl_new_std_menu(&ctx->ctrl_handler, &hva_ctrl_ops,
+			       V4L2_CID_MPEG_VIDEO_H264_LEVEL,
+			       V4L2_MPEG_VIDEO_H264_LEVEL_4_2,
+			       0,
+			       V4L2_MPEG_VIDEO_H264_LEVEL_4_0);
+
+	v4l2_ctrl_new_std_menu(&ctx->ctrl_handler, &hva_ctrl_ops,
+			       V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE,
+			       V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC,
+			       0,
+			       V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC);
+
+	v4l2_ctrl_new_std(&ctx->ctrl_handler, &hva_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_H264_CPB_SIZE,
+			  1, 10000, 1, 3000);
+
+	v4l2_ctrl_new_std(&ctx->ctrl_handler, &hva_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM,
+			  0, 1, 1, 0);
+
+	v4l2_ctrl_new_std(&ctx->ctrl_handler, &hva_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_H264_MIN_QP,
+			  0, 51, 1, 5);
+
+	v4l2_ctrl_new_std(&ctx->ctrl_handler, &hva_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_H264_MAX_QP,
+			  0, 51, 1, 51);
+
+	v4l2_ctrl_new_std(&ctx->ctrl_handler, &hva_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE,
+			  0, 1, 1, 1);
+
+	mask = ~(1 << V4L2_MPEG_VIDEO_H264_VUI_SAR_IDC_1x1);
+	v4l2_ctrl_new_std_menu(&ctx->ctrl_handler, &hva_ctrl_ops,
+			       V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_IDC,
+			       V4L2_MPEG_VIDEO_H264_VUI_SAR_IDC_1x1,
+			       mask,
+			       V4L2_MPEG_VIDEO_H264_VUI_SAR_IDC_1x1);
+
+	v4l2_ctrl_new_std(&ctx->ctrl_handler, &hva_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_H264_SEI_FRAME_PACKING,
+			  0, 1, 1, 0);
+
+	mask = ~(1 << sei_fp_type);
+	v4l2_ctrl_new_std_menu(&ctx->ctrl_handler, &hva_ctrl_ops,
+			       V4L2_CID_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE,
+			       sei_fp_type,
+			       mask,
+			       sei_fp_type);
+
+	if (ctx->ctrl_handler.error) {
+		int err = ctx->ctrl_handler.error;
+
+		dev_dbg(dev, "%s controls setup failed (%d)\n",
+			ctx->name, err);
+		v4l2_ctrl_handler_free(&ctx->ctrl_handler);
+		return err;
+	}
+
+	v4l2_ctrl_handler_setup(&ctx->ctrl_handler);
+
+	/* set default time per frame */
+	ctx->ctrls.time_per_frame.numerator = HVA_DEFAULT_FRAME_NUM;
+	ctx->ctrls.time_per_frame.denominator = HVA_DEFAULT_FRAME_DEN;
+
+	return 0;
+}
+
+/*
+ * mem-to-mem operations
+ */
+
+static void hva_run_work(struct work_struct *work)
+{
+	struct hva_ctx *ctx = container_of(work, struct hva_ctx, run_work);
+	struct vb2_v4l2_buffer *src_buf, *dst_buf;
+	const struct hva_enc *enc = ctx->enc;
+	struct hva_frame *frame;
+	struct hva_stream *stream;
+	int ret;
+
+	/* protect instance against reentrancy */
+	mutex_lock(&ctx->lock);
+
+	src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+	dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+
+	frame = to_hva_frame(src_buf);
+	stream = to_hva_stream(dst_buf);
+	frame->vbuf.sequence = ctx->frame_num++;
+
+	ret = enc->encode(ctx, frame, stream);
+
+	vb2_set_plane_payload(&dst_buf->vb2_buf, 0, stream->bytesused);
+	if (ret) {
+		v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR);
+		v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_ERROR);
+	} else {
+		/* propagate frame timestamp */
+		dst_buf->vb2_buf.timestamp = src_buf->vb2_buf.timestamp;
+		dst_buf->field = V4L2_FIELD_NONE;
+		dst_buf->sequence = ctx->stream_num - 1;
+
+		v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE);
+		v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_DONE);
+	}
+
+	mutex_unlock(&ctx->lock);
+
+	v4l2_m2m_job_finish(ctx->hva_dev->m2m_dev, ctx->fh.m2m_ctx);
+}
+
+static void hva_device_run(void *priv)
+{
+	struct hva_ctx *ctx = priv;
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+
+	queue_work(hva->work_queue, &ctx->run_work);
+}
+
+static void hva_job_abort(void *priv)
+{
+	struct hva_ctx *ctx = priv;
+	struct device *dev = ctx_to_dev(ctx);
+
+	dev_dbg(dev, "%s aborting job\n", ctx->name);
+
+	ctx->aborting = true;
+}
+
+static int hva_job_ready(void *priv)
+{
+	struct hva_ctx *ctx = priv;
+	struct device *dev = ctx_to_dev(ctx);
+
+	if (!v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx)) {
+		dev_dbg(dev, "%s job not ready: no frame buffers\n",
+			ctx->name);
+		return 0;
+	}
+
+	if (!v4l2_m2m_num_dst_bufs_ready(ctx->fh.m2m_ctx)) {
+		dev_dbg(dev, "%s job not ready: no stream buffers\n",
+			ctx->name);
+		return 0;
+	}
+
+	if (ctx->aborting) {
+		dev_dbg(dev, "%s job not ready: aborting\n", ctx->name);
+		return 0;
+	}
+
+	return 1;
+}
+
+/* mem-to-mem ops */
+static const struct v4l2_m2m_ops hva_m2m_ops = {
+	.device_run	= hva_device_run,
+	.job_abort	= hva_job_abort,
+	.job_ready	= hva_job_ready,
+};
+
+/*
+ * VB2 queue operations
+ */
+
+static int hva_queue_setup(struct vb2_queue *vq,
+			   unsigned int *num_buffers, unsigned int *num_planes,
+			   unsigned int sizes[], struct device *alloc_devs[])
+{
+	struct hva_ctx *ctx = vb2_get_drv_priv(vq);
+	struct device *dev = ctx_to_dev(ctx);
+	unsigned int size;
+
+	dev_dbg(dev, "%s %s queue setup: num_buffers %d\n", ctx->name,
+		to_type_str(vq->type), *num_buffers);
+
+	size = vq->type == V4L2_BUF_TYPE_VIDEO_OUTPUT ?
+		ctx->frameinfo.size : ctx->max_stream_size;
+
+	if (*num_planes)
+		return sizes[0] < size ? -EINVAL : 0;
+
+	/* only one plane supported */
+	*num_planes = 1;
+	sizes[0] = size;
+
+	return 0;
+}
+
+static int hva_buf_prepare(struct vb2_buffer *vb)
+{
+	struct hva_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+	struct device *dev = ctx_to_dev(ctx);
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+
+	if (vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) {
+		struct hva_frame *frame = to_hva_frame(vbuf);
+
+		if (vbuf->field == V4L2_FIELD_ANY)
+			vbuf->field = V4L2_FIELD_NONE;
+		if (vbuf->field != V4L2_FIELD_NONE) {
+			dev_dbg(dev,
+				"%s frame[%d] prepare: %d field not supported\n",
+				ctx->name, vb->index, vbuf->field);
+			return -EINVAL;
+		}
+
+		if (!frame->prepared) {
+			/* get memory addresses */
+			frame->vaddr = vb2_plane_vaddr(&vbuf->vb2_buf, 0);
+			frame->paddr = vb2_dma_contig_plane_dma_addr(
+					&vbuf->vb2_buf, 0);
+			frame->info = ctx->frameinfo;
+			frame->prepared = true;
+
+			dev_dbg(dev,
+				"%s frame[%d] prepared; virt=%p, phy=%pad\n",
+				ctx->name, vb->index,
+				frame->vaddr, &frame->paddr);
+		}
+	} else {
+		struct hva_stream *stream = to_hva_stream(vbuf);
+
+		if (!stream->prepared) {
+			/* get memory addresses */
+			stream->vaddr = vb2_plane_vaddr(&vbuf->vb2_buf, 0);
+			stream->paddr = vb2_dma_contig_plane_dma_addr(
+					&vbuf->vb2_buf, 0);
+			stream->size = vb2_plane_size(&vbuf->vb2_buf, 0);
+			stream->prepared = true;
+
+			dev_dbg(dev,
+				"%s stream[%d] prepared; virt=%p, phy=%pad\n",
+				ctx->name, vb->index,
+				stream->vaddr, &stream->paddr);
+		}
+	}
+
+	return 0;
+}
+
+static void hva_buf_queue(struct vb2_buffer *vb)
+{
+	struct hva_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+
+	if (ctx->fh.m2m_ctx)
+		v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
+}
+
+static int hva_start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct hva_ctx *ctx = vb2_get_drv_priv(vq);
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+	struct device *dev = ctx_to_dev(ctx);
+	struct vb2_v4l2_buffer *vbuf;
+	int ret;
+	unsigned int i;
+	bool found = false;
+
+	dev_dbg(dev, "%s %s start streaming\n", ctx->name,
+		to_type_str(vq->type));
+
+	/* open encoder when both start_streaming have been called */
+	if (V4L2_TYPE_IS_OUTPUT(vq->type)) {
+		if (!vb2_start_streaming_called(&ctx->fh.m2m_ctx->cap_q_ctx.q))
+			return 0;
+	} else {
+		if (!vb2_start_streaming_called(&ctx->fh.m2m_ctx->out_q_ctx.q))
+			return 0;
+	}
+
+	/* store the instance context in the instances array */
+	for (i = 0; i < HVA_MAX_INSTANCES; i++) {
+		if (!hva->instances[i]) {
+			hva->instances[i] = ctx;
+			/* save the context identifier in the context */
+			ctx->id = i;
+			found = true;
+			break;
+		}
+	}
+
+	if (!found) {
+		dev_err(dev, "%s maximum instances reached\n", ctx->name);
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	hva->nb_of_instances++;
+
+	if (!ctx->enc) {
+		ret = hva_open_encoder(ctx,
+				       ctx->streaminfo.streamformat,
+				       ctx->frameinfo.pixelformat,
+				       &ctx->enc);
+		if (ret < 0)
+			goto err_ctx;
+	}
+
+	return 0;
+
+err_ctx:
+	hva->instances[ctx->id] = NULL;
+	hva->nb_of_instances--;
+err:
+	if (vq->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) {
+		/* return of all pending buffers to vb2 (in queued state) */
+		while ((vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx)))
+			v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_QUEUED);
+	} else {
+		/* return of all pending buffers to vb2 (in queued state) */
+		while ((vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx)))
+			v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_QUEUED);
+	}
+
+	return ret;
+}
+
+static void hva_stop_streaming(struct vb2_queue *vq)
+{
+	struct hva_ctx *ctx = vb2_get_drv_priv(vq);
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+	struct device *dev = ctx_to_dev(ctx);
+	const struct hva_enc *enc = ctx->enc;
+	struct vb2_v4l2_buffer *vbuf;
+
+	dev_dbg(dev, "%s %s stop streaming\n", ctx->name,
+		to_type_str(vq->type));
+
+	if (vq->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) {
+		/* return of all pending buffers to vb2 (in error state) */
+		ctx->frame_num = 0;
+		while ((vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx)))
+			v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR);
+	} else {
+		/* return of all pending buffers to vb2 (in error state) */
+		ctx->stream_num = 0;
+		while ((vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx)))
+			v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR);
+	}
+
+	if ((V4L2_TYPE_IS_OUTPUT(vq->type) &&
+	     vb2_is_streaming(&ctx->fh.m2m_ctx->cap_q_ctx.q)) ||
+	    (!V4L2_TYPE_IS_OUTPUT(vq->type) &&
+	     vb2_is_streaming(&ctx->fh.m2m_ctx->out_q_ctx.q))) {
+		dev_dbg(dev, "%s %s out=%d cap=%d\n",
+			ctx->name, to_type_str(vq->type),
+			vb2_is_streaming(&ctx->fh.m2m_ctx->out_q_ctx.q),
+			vb2_is_streaming(&ctx->fh.m2m_ctx->cap_q_ctx.q));
+		return;
+	}
+
+	/* close encoder when both stop_streaming have been called */
+	if (enc) {
+		dev_dbg(dev, "%s %s encoder closed\n", ctx->name, enc->name);
+		enc->close(ctx);
+		ctx->enc = NULL;
+
+		/* clear instance context in instances array */
+		hva->instances[ctx->id] = NULL;
+		hva->nb_of_instances--;
+	}
+
+	ctx->aborting = false;
+}
+
+/* VB2 queue ops */
+static const struct vb2_ops hva_qops = {
+	.queue_setup		= hva_queue_setup,
+	.buf_prepare		= hva_buf_prepare,
+	.buf_queue		= hva_buf_queue,
+	.start_streaming	= hva_start_streaming,
+	.stop_streaming		= hva_stop_streaming,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
+};
+
+/*
+ * V4L2 file operations
+ */
+
+static int queue_init(struct hva_ctx *ctx, struct vb2_queue *vq)
+{
+	vq->io_modes = VB2_MMAP | VB2_DMABUF;
+	vq->drv_priv = ctx;
+	vq->ops = &hva_qops;
+	vq->mem_ops = &vb2_dma_contig_memops;
+	vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+	vq->lock = &ctx->hva_dev->lock;
+
+	return vb2_queue_init(vq);
+}
+
+static int hva_queue_init(void *priv, struct vb2_queue *src_vq,
+			  struct vb2_queue *dst_vq)
+{
+	struct hva_ctx *ctx = priv;
+	int ret;
+
+	src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+	src_vq->buf_struct_size = sizeof(struct hva_frame);
+	src_vq->min_buffers_needed = MIN_FRAMES;
+	src_vq->dev = ctx->hva_dev->dev;
+
+	ret = queue_init(ctx, src_vq);
+	if (ret)
+		return ret;
+
+	dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	dst_vq->buf_struct_size = sizeof(struct hva_stream);
+	dst_vq->min_buffers_needed = MIN_STREAMS;
+	dst_vq->dev = ctx->hva_dev->dev;
+
+	return queue_init(ctx, dst_vq);
+}
+
+static int hva_open(struct file *file)
+{
+	struct hva_dev *hva = video_drvdata(file);
+	struct device *dev = hva_to_dev(hva);
+	struct hva_ctx *ctx;
+	int ret;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	ctx->hva_dev = hva;
+
+	INIT_WORK(&ctx->run_work, hva_run_work);
+	v4l2_fh_init(&ctx->fh, video_devdata(file));
+	file->private_data = &ctx->fh;
+	v4l2_fh_add(&ctx->fh);
+
+	ret = hva_ctrls_setup(ctx);
+	if (ret) {
+		dev_err(dev, "%s [x:x] failed to setup controls\n",
+			HVA_PREFIX);
+		goto err_fh;
+	}
+	ctx->fh.ctrl_handler = &ctx->ctrl_handler;
+
+	mutex_init(&ctx->lock);
+
+	ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(hva->m2m_dev, ctx,
+					    &hva_queue_init);
+	if (IS_ERR(ctx->fh.m2m_ctx)) {
+		ret = PTR_ERR(ctx->fh.m2m_ctx);
+		dev_err(dev, "%s failed to initialize m2m context (%d)\n",
+			HVA_PREFIX, ret);
+		goto err_ctrls;
+	}
+
+	/* set the instance name */
+	mutex_lock(&hva->lock);
+	hva->instance_id++;
+	snprintf(ctx->name, sizeof(ctx->name), "[%3d:----]",
+		 hva->instance_id);
+	mutex_unlock(&hva->lock);
+
+	/* default parameters for frame and stream */
+	set_default_params(ctx);
+
+	dev_info(dev, "%s encoder instance created\n", ctx->name);
+
+	return 0;
+
+err_ctrls:
+	v4l2_ctrl_handler_free(&ctx->ctrl_handler);
+err_fh:
+	v4l2_fh_del(&ctx->fh);
+	v4l2_fh_exit(&ctx->fh);
+	kfree(ctx);
+out:
+	return ret;
+}
+
+static int hva_release(struct file *file)
+{
+	struct hva_ctx *ctx = fh_to_ctx(file->private_data);
+	struct hva_dev *hva = ctx_to_hdev(ctx);
+	struct device *dev = ctx_to_dev(ctx);
+	const struct hva_enc *enc = ctx->enc;
+
+	if (enc) {
+		dev_dbg(dev, "%s %s encoder closed\n", ctx->name, enc->name);
+		enc->close(ctx);
+		ctx->enc = NULL;
+
+		/* clear instance context in instances array */
+		hva->instances[ctx->id] = NULL;
+		hva->nb_of_instances--;
+	}
+
+	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+
+	v4l2_ctrl_handler_free(&ctx->ctrl_handler);
+
+	v4l2_fh_del(&ctx->fh);
+	v4l2_fh_exit(&ctx->fh);
+
+	dev_info(dev, "%s encoder instance released\n", ctx->name);
+
+	kfree(ctx);
+
+	return 0;
+}
+
+/* V4L2 file ops */
+static const struct v4l2_file_operations hva_fops = {
+	.owner			= THIS_MODULE,
+	.open			= hva_open,
+	.release		= hva_release,
+	.unlocked_ioctl		= video_ioctl2,
+	.mmap			= v4l2_m2m_fop_mmap,
+	.poll			= v4l2_m2m_fop_poll,
+};
+
+/*
+ * Platform device operations
+ */
+
+static int hva_register_device(struct hva_dev *hva)
+{
+	int ret;
+	struct video_device *vdev;
+	struct device *dev;
+
+	if (!hva)
+		return -ENODEV;
+	dev = hva_to_dev(hva);
+
+	hva->m2m_dev = v4l2_m2m_init(&hva_m2m_ops);
+	if (IS_ERR(hva->m2m_dev)) {
+		dev_err(dev, "%s failed to initialize v4l2-m2m device\n",
+			HVA_PREFIX);
+		ret = PTR_ERR(hva->m2m_dev);
+		goto err;
+	}
+
+	vdev = video_device_alloc();
+	if (!vdev) {
+		dev_err(dev, "%s failed to allocate video device\n",
+			HVA_PREFIX);
+		ret = -ENOMEM;
+		goto err_m2m_release;
+	}
+
+	vdev->fops = &hva_fops;
+	vdev->ioctl_ops = &hva_ioctl_ops;
+	vdev->release = video_device_release;
+	vdev->lock = &hva->lock;
+	vdev->vfl_dir = VFL_DIR_M2M;
+	vdev->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M;
+	vdev->v4l2_dev = &hva->v4l2_dev;
+	snprintf(vdev->name, sizeof(vdev->name), "%s%lx", HVA_NAME,
+		 hva->ip_version);
+
+	ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1);
+	if (ret) {
+		dev_err(dev, "%s failed to register video device\n",
+			HVA_PREFIX);
+		goto err_vdev_release;
+	}
+
+	hva->vdev = vdev;
+	video_set_drvdata(vdev, hva);
+	return 0;
+
+err_vdev_release:
+	video_device_release(vdev);
+err_m2m_release:
+	v4l2_m2m_release(hva->m2m_dev);
+err:
+	return ret;
+}
+
+static void hva_unregister_device(struct hva_dev *hva)
+{
+	if (!hva)
+		return;
+
+	if (hva->m2m_dev)
+		v4l2_m2m_release(hva->m2m_dev);
+
+	video_unregister_device(hva->vdev);
+}
+
+static int hva_probe(struct platform_device *pdev)
+{
+	struct hva_dev *hva;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	hva = devm_kzalloc(dev, sizeof(*hva), GFP_KERNEL);
+	if (!hva) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	hva->dev = dev;
+	hva->pdev = pdev;
+	platform_set_drvdata(pdev, hva);
+
+	mutex_init(&hva->lock);
+
+	/* probe hardware */
+	ret = hva_hw_probe(pdev, hva);
+	if (ret)
+		goto err;
+
+	/* register all available encoders */
+	register_encoders(hva);
+
+	/* register all supported formats */
+	register_formats(hva);
+
+	/* register on V4L2 */
+	ret = v4l2_device_register(dev, &hva->v4l2_dev);
+	if (ret) {
+		dev_err(dev, "%s %s failed to register V4L2 device\n",
+			HVA_PREFIX, HVA_NAME);
+		goto err_hw;
+	}
+
+	hva->work_queue = create_workqueue(HVA_NAME);
+	if (!hva->work_queue) {
+		dev_err(dev, "%s %s failed to allocate work queue\n",
+			HVA_PREFIX, HVA_NAME);
+		ret = -ENOMEM;
+		goto err_v4l2;
+	}
+
+	/* register device */
+	ret = hva_register_device(hva);
+	if (ret)
+		goto err_work_queue;
+
+	dev_info(dev, "%s %s registered as /dev/video%d\n", HVA_PREFIX,
+		 HVA_NAME, hva->vdev->num);
+
+	return 0;
+
+err_work_queue:
+	destroy_workqueue(hva->work_queue);
+err_v4l2:
+	v4l2_device_unregister(&hva->v4l2_dev);
+err_hw:
+	hva_hw_remove(hva);
+err:
+	return ret;
+}
+
+static int hva_remove(struct platform_device *pdev)
+{
+	struct hva_dev *hva = platform_get_drvdata(pdev);
+	struct device *dev = hva_to_dev(hva);
+
+	hva_unregister_device(hva);
+
+	destroy_workqueue(hva->work_queue);
+
+	hva_hw_remove(hva);
+
+	v4l2_device_unregister(&hva->v4l2_dev);
+
+	dev_info(dev, "%s %s removed\n", HVA_PREFIX, pdev->name);
+
+	return 0;
+}
+
+/* PM ops */
+static const struct dev_pm_ops hva_pm_ops = {
+	.runtime_suspend	= hva_hw_runtime_suspend,
+	.runtime_resume		= hva_hw_runtime_resume,
+};
+
+static const struct of_device_id hva_match_types[] = {
+	{
+	 .compatible = "st,st-hva",
+	},
+	{ /* end node */ }
+};
+
+MODULE_DEVICE_TABLE(of, hva_match_types);
+
+static struct platform_driver hva_driver = {
+	.probe  = hva_probe,
+	.remove = hva_remove,
+	.driver = {
+		.name		= HVA_NAME,
+		.of_match_table	= hva_match_types,
+		.pm		= &hva_pm_ops,
+		},
+};
+
+module_platform_driver(hva_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Yannick Fertre <yannick.fertre@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics HVA video encoder V4L2 driver");
diff --git a/drivers/media/platform/sti/hva/hva.h b/drivers/media/platform/sti/hva/hva.h
new file mode 100644
index 000000000000..caa580825541
--- /dev/null
+++ b/drivers/media/platform/sti/hva/hva.h
@@ -0,0 +1,315 @@
+/*
+ * Copyright (C) STMicroelectronics SA 2015
+ * Authors: Yannick Fertre <yannick.fertre@st.com>
+ *          Hugues Fruchet <hugues.fruchet@st.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#ifndef HVA_H
+#define HVA_H
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-mem2mem.h>
+
+#define fh_to_ctx(f)    (container_of(f, struct hva_ctx, fh))
+
+#define hva_to_dev(h)   (h->dev)
+
+#define ctx_to_dev(c)   (c->hva_dev->dev)
+
+#define ctx_to_hdev(c)  (c->hva_dev)
+
+#define HVA_PREFIX "[---:----]"
+
+extern const struct hva_enc nv12h264enc;
+extern const struct hva_enc nv21h264enc;
+
+/**
+ * struct hva_frameinfo - information about hva frame
+ *
+ * @pixelformat:    fourcc code for uncompressed video format
+ * @width:          width of frame
+ * @height:         height of frame
+ * @aligned_width:  width of frame (with encoder alignment constraint)
+ * @aligned_height: height of frame (with encoder alignment constraint)
+ * @size:           maximum size in bytes required for data
+*/
+struct hva_frameinfo {
+	u32	pixelformat;
+	u32	width;
+	u32	height;
+	u32	aligned_width;
+	u32	aligned_height;
+	u32	size;
+};
+
+/**
+ * struct hva_streaminfo - information about hva stream
+ *
+ * @streamformat: fourcc code of compressed video format (H.264...)
+ * @width:        width of stream
+ * @height:       height of stream
+ * @profile:      profile string
+ * @level:        level string
+ */
+struct hva_streaminfo {
+	u32	streamformat;
+	u32	width;
+	u32	height;
+	u8	profile[32];
+	u8	level[32];
+};
+
+/**
+ * struct hva_controls - hva controls set
+ *
+ * @time_per_frame: time per frame in seconds
+ * @bitrate_mode:   bitrate mode (constant bitrate or variable bitrate)
+ * @gop_size:       groupe of picture size
+ * @bitrate:        bitrate (in bps)
+ * @aspect:         video aspect
+ * @profile:        H.264 profile
+ * @level:          H.264 level
+ * @entropy_mode:   H.264 entropy mode (CABAC or CVLC)
+ * @cpb_size:       coded picture buffer size (in kB)
+ * @dct8x8:         transform mode 8x8 enable
+ * @qpmin:          minimum quantizer
+ * @qpmax:          maximum quantizer
+ * @vui_sar:        pixel aspect ratio enable
+ * @vui_sar_idc:    pixel aspect ratio identifier
+ * @sei_fp:         sei frame packing arrangement enable
+ * @sei_fp_type:    sei frame packing arrangement type
+ */
+struct hva_controls {
+	struct v4l2_fract					time_per_frame;
+	enum v4l2_mpeg_video_bitrate_mode			bitrate_mode;
+	u32							gop_size;
+	u32							bitrate;
+	enum v4l2_mpeg_video_aspect				aspect;
+	enum v4l2_mpeg_video_h264_profile			profile;
+	enum v4l2_mpeg_video_h264_level				level;
+	enum v4l2_mpeg_video_h264_entropy_mode			entropy_mode;
+	u32							cpb_size;
+	bool							dct8x8;
+	u32							qpmin;
+	u32							qpmax;
+	bool							vui_sar;
+	enum v4l2_mpeg_video_h264_vui_sar_idc			vui_sar_idc;
+	bool							sei_fp;
+	enum v4l2_mpeg_video_h264_sei_fp_arrangement_type	sei_fp_type;
+};
+
+/**
+ * struct hva_frame - hva frame buffer (output)
+ *
+ * @vbuf:     video buffer information for V4L2
+ * @list:     V4L2 m2m list that the frame belongs to
+ * @info:     frame information (width, height, format, alignment...)
+ * @paddr:    physical address (for hardware)
+ * @vaddr:    virtual address (kernel can read/write)
+ * @prepared: true if vaddr/paddr are resolved
+ */
+struct hva_frame {
+	struct vb2_v4l2_buffer	vbuf;
+	struct list_head	list;
+	struct hva_frameinfo	info;
+	dma_addr_t		paddr;
+	void			*vaddr;
+	bool			prepared;
+};
+
+/*
+ * to_hva_frame() - cast struct vb2_v4l2_buffer * to struct hva_frame *
+ */
+#define to_hva_frame(vb) \
+	container_of(vb, struct hva_frame, vbuf)
+
+/**
+ * struct hva_stream - hva stream buffer (capture)
+ *
+ * @v4l2:       video buffer information for V4L2
+ * @list:       V4L2 m2m list that the frame belongs to
+ * @paddr:      physical address (for hardware)
+ * @vaddr:      virtual address (kernel can read/write)
+ * @prepared:   true if vaddr/paddr are resolved
+ * @size:       size of the buffer in bytes
+ * @bytesused:  number of bytes occupied by data in the buffer
+ */
+struct hva_stream {
+	struct vb2_v4l2_buffer	vbuf;
+	struct list_head	list;
+	dma_addr_t		paddr;
+	void			*vaddr;
+	bool			prepared;
+	unsigned int		size;
+	unsigned int		bytesused;
+};
+
+/*
+ * to_hva_stream() - cast struct vb2_v4l2_buffer * to struct hva_stream *
+ */
+#define to_hva_stream(vb) \
+	container_of(vb, struct hva_stream, vbuf)
+
+struct hva_dev;
+struct hva_enc;
+
+/**
+ * struct hva_ctx - context of hva instance
+ *
+ * @hva_dev:         the device that this instance is associated with
+ * @fh:              V4L2 file handle
+ * @ctrl_handler:    V4L2 controls handler
+ * @ctrls:           hva controls set
+ * @id:              instance identifier
+ * @aborting:        true if current job aborted
+ * @name:            instance name (debug purpose)
+ * @run_work:        encode work
+ * @lock:            mutex used to lock access of this context
+ * @flags:           validity of streaminfo and frameinfo fields
+ * @frame_num:       frame number
+ * @stream_num:      stream number
+ * @max_stream_size: maximum size in bytes required for stream data
+ * @colorspace:      colorspace identifier
+ * @xfer_func:       transfer function identifier
+ * @ycbcr_enc:       Y'CbCr encoding identifier
+ * @quantization:    quantization identifier
+ * @streaminfo:      stream properties
+ * @frameinfo:       frame properties
+ * @enc:             current encoder
+ * @priv:            private codec data for this instance, allocated
+ *                   by encoder @open time
+ * @hw_err:          true if hardware error detected
+ */
+struct hva_ctx {
+	struct hva_dev		        *hva_dev;
+	struct v4l2_fh			fh;
+	struct v4l2_ctrl_handler	ctrl_handler;
+	struct hva_controls		ctrls;
+	u8				id;
+	bool				aborting;
+	char				name[100];
+	struct work_struct		run_work;
+	/* mutex protecting this data structure */
+	struct mutex			lock;
+	u32				flags;
+	u32				frame_num;
+	u32				stream_num;
+	u32				max_stream_size;
+	enum v4l2_colorspace		colorspace;
+	enum v4l2_xfer_func		xfer_func;
+	enum v4l2_ycbcr_encoding	ycbcr_enc;
+	enum v4l2_quantization		quantization;
+	struct hva_streaminfo		streaminfo;
+	struct hva_frameinfo		frameinfo;
+	struct hva_enc			*enc;
+	void				*priv;
+	bool				hw_err;
+};
+
+#define HVA_FLAG_STREAMINFO	0x0001
+#define HVA_FLAG_FRAMEINFO	0x0002
+
+#define HVA_MAX_INSTANCES	16
+#define HVA_MAX_ENCODERS	10
+#define HVA_MAX_FORMATS		HVA_MAX_ENCODERS
+
+/**
+ * struct hva_dev - abstraction for hva entity
+ *
+ * @v4l2_dev:            V4L2 device
+ * @vdev:                video device
+ * @pdev:                platform device
+ * @dev:                 device
+ * @lock:                mutex used for critical sections & V4L2 ops
+ *                       serialization
+ * @m2m_dev:             memory-to-memory V4L2 device information
+ * @instances:           opened instances
+ * @nb_of_instances:     number of opened instances
+ * @instance_id:         rolling counter identifying an instance (debug purpose)
+ * @regs:                register io memory access
+ * @esram_addr:          esram address
+ * @esram_size:          esram size
+ * @clk:                 hva clock
+ * @irq_its:             status interruption
+ * @irq_err:             error interruption
+ * @work_queue:          work queue to handle the encode jobs
+ * @protect_mutex:       mutex used to lock access of hardware
+ * @interrupt:           completion interrupt
+ * @ip_version:          IP hardware version
+ * @encoders:            registered encoders
+ * @nb_of_encoders:      number of registered encoders
+ * @pixelformats:        supported uncompressed video formats
+ * @nb_of_pixelformats:  number of supported umcompressed video formats
+ * @streamformats:       supported compressed video formats
+ * @nb_of_streamformats: number of supported compressed video formats
+ * @sfl_reg:             status fifo level register value
+ * @sts_reg:             status register value
+ * @lmi_err_reg:         local memory interface error register value
+ * @emi_err_reg:         external memory interface error register value
+ * @hec_mif_err_reg:     HEC memory interface error register value
+ */
+struct hva_dev {
+	struct v4l2_device	v4l2_dev;
+	struct video_device	*vdev;
+	struct platform_device	*pdev;
+	struct device		*dev;
+	/* mutex protecting vb2_queue structure */
+	struct mutex		lock;
+	struct v4l2_m2m_dev	*m2m_dev;
+	struct hva_ctx		*instances[HVA_MAX_INSTANCES];
+	unsigned int		nb_of_instances;
+	unsigned int		instance_id;
+	void __iomem		*regs;
+	u32			esram_addr;
+	u32			esram_size;
+	struct clk		*clk;
+	int			irq_its;
+	int			irq_err;
+	struct workqueue_struct *work_queue;
+	/* mutex protecting hardware access */
+	struct mutex		protect_mutex;
+	struct completion	interrupt;
+	unsigned long int	ip_version;
+	const struct hva_enc	*encoders[HVA_MAX_ENCODERS];
+	u32			nb_of_encoders;
+	u32			pixelformats[HVA_MAX_FORMATS];
+	u32			nb_of_pixelformats;
+	u32			streamformats[HVA_MAX_FORMATS];
+	u32			nb_of_streamformats;
+	u32			sfl_reg;
+	u32			sts_reg;
+	u32			lmi_err_reg;
+	u32			emi_err_reg;
+	u32			hec_mif_err_reg;
+};
+
+/**
+ * struct hva_enc - hva encoder
+ *
+ * @name:         encoder name
+ * @streamformat: fourcc code for compressed video format (H.264...)
+ * @pixelformat:  fourcc code for uncompressed video format
+ * @max_width:    maximum width of frame for this encoder
+ * @max_height:   maximum height of frame for this encoder
+ * @open:         open encoder
+ * @close:        close encoder
+ * @encode:       encode a frame (struct hva_frame) in a stream
+ *                (struct hva_stream)
+ */
+
+struct hva_enc {
+	const char	*name;
+	u32		streamformat;
+	u32		pixelformat;
+	u32		max_width;
+	u32		max_height;
+	int		(*open)(struct hva_ctx *ctx);
+	int		(*close)(struct hva_ctx *ctx);
+	int		(*encode)(struct hva_ctx *ctx, struct hva_frame *frame,
+				  struct hva_stream *stream);
+};
+
+#endif /* HVA_H */
diff --git a/drivers/media/platform/ti-vpe/cal.c b/drivers/media/platform/ti-vpe/cal.c
index e967fcfdc1d8..44323cb5d287 100644
--- a/drivers/media/platform/ti-vpe/cal.c
+++ b/drivers/media/platform/ti-vpe/cal.c
@@ -1379,7 +1379,7 @@ static void cal_stop_streaming(struct vb2_queue *vq)
 	cal_runtime_put(ctx->dev);
 }
 
-static struct vb2_ops cal_video_qops = {
+static const struct vb2_ops cal_video_qops = {
 	.queue_setup		= cal_queue_setup,
 	.buf_prepare		= cal_buffer_prepare,
 	.buf_queue		= cal_buffer_queue,
diff --git a/drivers/media/platform/ti-vpe/vpe.c b/drivers/media/platform/ti-vpe/vpe.c
index 55a1458ac783..0189f7f7cb03 100644
--- a/drivers/media/platform/ti-vpe/vpe.c
+++ b/drivers/media/platform/ti-vpe/vpe.c
@@ -1878,7 +1878,7 @@ static void vpe_stop_streaming(struct vb2_queue *q)
 	vpdma_dump_regs(ctx->dev->vpdma);
 }
 
-static struct vb2_ops vpe_qops = {
+static const struct vb2_ops vpe_qops = {
 	.queue_setup	 = vpe_queue_setup,
 	.buf_prepare	 = vpe_buf_prepare,
 	.buf_queue	 = vpe_buf_queue,
diff --git a/drivers/media/platform/vim2m.c b/drivers/media/platform/vim2m.c
index cd0ff4a66fdc..a98f679bd88d 100644
--- a/drivers/media/platform/vim2m.c
+++ b/drivers/media/platform/vim2m.c
@@ -815,7 +815,7 @@ static void vim2m_stop_streaming(struct vb2_queue *q)
 	}
 }
 
-static struct vb2_ops vim2m_qops = {
+static const struct vb2_ops vim2m_qops = {
 	.queue_setup	 = vim2m_queue_setup,
 	.buf_prepare	 = vim2m_buf_prepare,
 	.buf_queue	 = vim2m_buf_queue,
diff --git a/drivers/media/platform/vivid/vivid-core.c b/drivers/media/platform/vivid/vivid-core.c
index 7f937136c3f5..5464fefbaab9 100644
--- a/drivers/media/platform/vivid/vivid-core.c
+++ b/drivers/media/platform/vivid/vivid-core.c
@@ -163,38 +163,38 @@ const struct v4l2_rect vivid_max_rect = {
 
 static const u8 vivid_hdmi_edid[256] = {
 	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00,
-	0x63, 0x3a, 0xaa, 0x55, 0x00, 0x00, 0x00, 0x00,
-	0x0a, 0x18, 0x01, 0x03, 0x80, 0x10, 0x09, 0x78,
-	0x0e, 0x00, 0xb2, 0xa0, 0x57, 0x49, 0x9b, 0x26,
-	0x10, 0x48, 0x4f, 0x2f, 0xcf, 0x00, 0x31, 0x59,
+	0x31, 0xd8, 0x34, 0x12, 0x00, 0x00, 0x00, 0x00,
+	0x22, 0x1a, 0x01, 0x03, 0x80, 0x60, 0x36, 0x78,
+	0x0f, 0xee, 0x91, 0xa3, 0x54, 0x4c, 0x99, 0x26,
+	0x0f, 0x50, 0x54, 0x2f, 0xcf, 0x00, 0x31, 0x59,
 	0x45, 0x59, 0x81, 0x80, 0x81, 0x40, 0x90, 0x40,
-	0x95, 0x00, 0xa9, 0x40, 0xb3, 0x00, 0x02, 0x3a,
-	0x80, 0x18, 0x71, 0x38, 0x2d, 0x40, 0x58, 0x2c,
-	0x46, 0x00, 0x10, 0x09, 0x00, 0x00, 0x00, 0x1e,
+	0x95, 0x00, 0xa9, 0x40, 0xb3, 0x00, 0x08, 0xe8,
+	0x00, 0x30, 0xf2, 0x70, 0x5a, 0x80, 0xb0, 0x58,
+	0x8a, 0x00, 0xc0, 0x1c, 0x32, 0x00, 0x00, 0x1e,
 	0x00, 0x00, 0x00, 0xfd, 0x00, 0x18, 0x55, 0x18,
-	0x5e, 0x11, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20,
-	0x20, 0x20, 0x00, 0x00, 0x00, 0xfc, 0x00,  'v',
-	'4',   'l',  '2',  '-',  'h',  'd',  'm',  'i',
-	0x0a, 0x0a, 0x0a, 0x0a, 0x00, 0x00, 0x00, 0x10,
+	0x87, 0x3c, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20,
+	0x20, 0x20, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x76,
+	0x69, 0x76, 0x69, 0x64, 0x0a, 0x20, 0x20, 0x20,
+	0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x10,
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xf0,
-
-	0x02, 0x03, 0x1a, 0xc0, 0x48, 0xa2, 0x10, 0x04,
-	0x02, 0x01, 0x21, 0x14, 0x13, 0x23, 0x09, 0x07,
-	0x07, 0x65, 0x03, 0x0c, 0x00, 0x10, 0x00, 0xe2,
-	0x00, 0x2a, 0x01, 0x1d, 0x00, 0x80, 0x51, 0xd0,
-	0x1c, 0x20, 0x40, 0x80, 0x35, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x1e, 0x8c, 0x0a, 0xd0, 0x8a,
-	0x20, 0xe0, 0x2d, 0x10, 0x10, 0x3e, 0x96, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd7
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x7b,
+
+	0x02, 0x03, 0x3f, 0xf0, 0x51, 0x61, 0x60, 0x5f,
+	0x5e, 0x5d, 0x10, 0x1f, 0x04, 0x13, 0x22, 0x21,
+	0x20, 0x05, 0x14, 0x02, 0x11, 0x01, 0x23, 0x09,
+	0x07, 0x07, 0x83, 0x01, 0x00, 0x00, 0x6d, 0x03,
+	0x0c, 0x00, 0x10, 0x00, 0x00, 0x78, 0x21, 0x00,
+	0x60, 0x01, 0x02, 0x03, 0x67, 0xd8, 0x5d, 0xc4,
+	0x01, 0x78, 0x00, 0x00, 0xe2, 0x00, 0xea, 0xe3,
+	0x05, 0x00, 0x00, 0xe3, 0x06, 0x01, 0x00, 0x4d,
+	0xd0, 0x00, 0xa0, 0xf0, 0x70, 0x3e, 0x80, 0x30,
+	0x20, 0x35, 0x00, 0xc0, 0x1c, 0x32, 0x00, 0x00,
+	0x1e, 0x1a, 0x36, 0x80, 0xa0, 0x70, 0x38, 0x1f,
+	0x40, 0x30, 0x20, 0x35, 0x00, 0xc0, 0x1c, 0x32,
+	0x00, 0x00, 0x1a, 0x1a, 0x1d, 0x00, 0x80, 0x51,
+	0xd0, 0x1c, 0x20, 0x40, 0x80, 0x35, 0x00, 0xc0,
+	0x1c, 0x32, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27,
 };
 
 static int vidioc_querycap(struct file *file, void  *priv,
@@ -839,6 +839,7 @@ static int vivid_create_instance(struct platform_device *pdev, int inst)
 		dev->radio_tx_caps = V4L2_CAP_RDS_OUTPUT | V4L2_CAP_MODULATOR |
 				     V4L2_CAP_READWRITE;
 
+	ret = -ENOMEM;
 	/* initialize the test pattern generator */
 	tpg_init(&dev->tpg, 640, 360);
 	if (tpg_alloc(&dev->tpg, MAX_ZOOM * MAX_WIDTH))
@@ -1033,8 +1034,10 @@ static int vivid_create_instance(struct platform_device *pdev, int inst)
 	 */
 	dev->cec_workqueue =
 		alloc_ordered_workqueue("vivid-%03d-cec", WQ_MEM_RECLAIM, inst);
-	if (!dev->cec_workqueue)
+	if (!dev->cec_workqueue) {
+		ret = -ENOMEM;
 		goto unreg_dev;
+	}
 
 	/* start creating the vb2 queues */
 	if (dev->has_vid_cap) {
diff --git a/drivers/media/platform/vivid/vivid-ctrls.c b/drivers/media/platform/vivid/vivid-ctrls.c
index b98089c95ef5..aceb38d9f7e7 100644
--- a/drivers/media/platform/vivid/vivid-ctrls.c
+++ b/drivers/media/platform/vivid/vivid-ctrls.c
@@ -761,7 +761,7 @@ static const char * const vivid_ctrl_ycbcr_enc_strings[] = {
 	"Rec. 709",
 	"xvYCC 601",
 	"xvYCC 709",
-	"sYCC",
+	"",
 	"BT.2020",
 	"BT.2020 Constant Luminance",
 	"SMPTE 240M",
@@ -773,6 +773,7 @@ static const struct v4l2_ctrl_config vivid_ctrl_ycbcr_enc = {
 	.id = VIVID_CID_YCBCR_ENC,
 	.name = "Y'CbCr Encoding",
 	.type = V4L2_CTRL_TYPE_MENU,
+	.menu_skip_mask = 1 << 5,
 	.max = ARRAY_SIZE(vivid_ctrl_ycbcr_enc_strings) - 2,
 	.qmenu = vivid_ctrl_ycbcr_enc_strings,
 };
diff --git a/drivers/media/platform/vivid/vivid-vid-cap.c b/drivers/media/platform/vivid/vivid-vid-cap.c
index d404a7ce33a4..d5c84ecf2027 100644
--- a/drivers/media/platform/vivid/vivid-vid-cap.c
+++ b/drivers/media/platform/vivid/vivid-vid-cap.c
@@ -823,7 +823,7 @@ int vivid_vid_cap_g_selection(struct file *file, void *priv,
 	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 	if (vivid_is_webcam(dev))
-		return -EINVAL;
+		return -ENODATA;
 
 	sel->r.left = sel->r.top = 0;
 	switch (sel->target) {
@@ -872,7 +872,7 @@ int vivid_vid_cap_s_selection(struct file *file, void *fh, struct v4l2_selection
 	if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 	if (vivid_is_webcam(dev))
-		return -EINVAL;
+		return -ENODATA;
 
 	switch (s->target) {
 	case V4L2_SEL_TGT_CROP:
diff --git a/drivers/media/platform/vsp1/vsp1.h b/drivers/media/platform/vsp1/vsp1.h
index 06a2ec7e5ad4..b23fa879a9aa 100644
--- a/drivers/media/platform/vsp1/vsp1.h
+++ b/drivers/media/platform/vsp1/vsp1.h
@@ -53,6 +53,7 @@ struct vsp1_uds;
 
 struct vsp1_device_info {
 	u32 version;
+	const char *model;
 	unsigned int gen;
 	unsigned int features;
 	unsigned int rpf_count;
@@ -65,6 +66,7 @@ struct vsp1_device_info {
 struct vsp1_device {
 	struct device *dev;
 	const struct vsp1_device_info *info;
+	u32 version;
 
 	void __iomem *mmio;
 	struct rcar_fcp_device *fcp;
diff --git a/drivers/media/platform/vsp1/vsp1_bru.c b/drivers/media/platform/vsp1/vsp1_bru.c
index 8268b87727a7..ee8355c28f94 100644
--- a/drivers/media/platform/vsp1/vsp1_bru.c
+++ b/drivers/media/platform/vsp1/vsp1_bru.c
@@ -142,10 +142,15 @@ static int bru_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_bru *bru = to_bru(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
+
+	mutex_lock(&bru->entity.lock);
 
 	config = vsp1_entity_get_pad_config(&bru->entity, cfg, fmt->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	bru_try_format(bru, config, fmt->pad, &fmt->format);
 
@@ -174,7 +179,9 @@ static int bru_set_format(struct v4l2_subdev *subdev,
 		}
 	}
 
-	return 0;
+done:
+	mutex_unlock(&bru->entity.lock);
+	return ret;
 }
 
 static int bru_get_selection(struct v4l2_subdev *subdev,
@@ -201,7 +208,9 @@ static int bru_get_selection(struct v4l2_subdev *subdev,
 		if (!config)
 			return -EINVAL;
 
+		mutex_lock(&bru->entity.lock);
 		sel->r = *bru_get_compose(bru, config, sel->pad);
+		mutex_unlock(&bru->entity.lock);
 		return 0;
 
 	default:
@@ -217,6 +226,7 @@ static int bru_set_selection(struct v4l2_subdev *subdev,
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *compose;
+	int ret = 0;
 
 	if (sel->pad == bru->entity.source_pad)
 		return -EINVAL;
@@ -224,11 +234,16 @@ static int bru_set_selection(struct v4l2_subdev *subdev,
 	if (sel->target != V4L2_SEL_TGT_COMPOSE)
 		return -EINVAL;
 
+	mutex_lock(&bru->entity.lock);
+
 	config = vsp1_entity_get_pad_config(&bru->entity, cfg, sel->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
-	/* The compose rectangle top left corner must be inside the output
+	/*
+	 * The compose rectangle top left corner must be inside the output
 	 * frame.
 	 */
 	format = vsp1_entity_get_pad_format(&bru->entity, config,
@@ -246,7 +261,9 @@ static int bru_set_selection(struct v4l2_subdev *subdev,
 	compose = bru_get_compose(bru, config, sel->pad);
 	*compose = sel->r;
 
-	return 0;
+done:
+	mutex_unlock(&bru->entity.lock);
+	return ret;
 }
 
 static const struct v4l2_subdev_pad_ops bru_pad_ops = {
@@ -269,14 +286,15 @@ static const struct v4l2_subdev_ops bru_ops = {
 
 static void bru_configure(struct vsp1_entity *entity,
 			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl, bool full)
+			  struct vsp1_dl_list *dl,
+			  enum vsp1_entity_params params)
 {
 	struct vsp1_bru *bru = to_bru(&entity->subdev);
 	struct v4l2_mbus_framefmt *format;
 	unsigned int flags;
 	unsigned int i;
 
-	if (!full)
+	if (params != VSP1_ENTITY_PARAMS_INIT)
 		return;
 
 	format = vsp1_entity_get_pad_format(&bru->entity, bru->entity.config,
diff --git a/drivers/media/platform/vsp1/vsp1_clu.c b/drivers/media/platform/vsp1/vsp1_clu.c
index b63d2dbe5ea3..f2fb26e5ab4e 100644
--- a/drivers/media/platform/vsp1/vsp1_clu.c
+++ b/drivers/media/platform/vsp1/vsp1_clu.c
@@ -148,10 +148,15 @@ static int clu_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_clu *clu = to_clu(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
+
+	mutex_lock(&clu->entity.lock);
 
 	config = vsp1_entity_get_pad_config(&clu->entity, cfg, fmt->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	/* Default to YUV if the requested format is not supported. */
 	if (fmt->format.code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
@@ -164,7 +169,7 @@ static int clu_set_format(struct v4l2_subdev *subdev,
 	if (fmt->pad == CLU_PAD_SOURCE) {
 		/* The CLU output format can't be modified. */
 		fmt->format = *format;
-		return 0;
+		goto done;
 	}
 
 	format->code = fmt->format.code;
@@ -182,7 +187,9 @@ static int clu_set_format(struct v4l2_subdev *subdev,
 					    CLU_PAD_SOURCE);
 	*format = fmt->format;
 
-	return 0;
+done:
+	mutex_unlock(&clu->entity.lock);
+	return ret;
 }
 
 /* -----------------------------------------------------------------------------
@@ -207,42 +214,51 @@ static const struct v4l2_subdev_ops clu_ops = {
 
 static void clu_configure(struct vsp1_entity *entity,
 			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl, bool full)
+			  struct vsp1_dl_list *dl,
+			  enum vsp1_entity_params params)
 {
 	struct vsp1_clu *clu = to_clu(&entity->subdev);
 	struct vsp1_dl_body *dlb;
 	unsigned long flags;
 	u32 ctrl = VI6_CLU_CTRL_AAI | VI6_CLU_CTRL_MVS | VI6_CLU_CTRL_EN;
 
-	/* The format can't be changed during streaming, only verify it at
-	 * stream start and store the information internally for future partial
-	 * reconfiguration calls.
-	 */
-	if (full) {
+	switch (params) {
+	case VSP1_ENTITY_PARAMS_INIT: {
+		/*
+		 * The format can't be changed during streaming, only verify it
+		 * at setup time and store the information internally for future
+		 * runtime configuration calls.
+		 */
 		struct v4l2_mbus_framefmt *format;
 
 		format = vsp1_entity_get_pad_format(&clu->entity,
 						    clu->entity.config,
 						    CLU_PAD_SINK);
 		clu->yuv_mode = format->code == MEDIA_BUS_FMT_AYUV8_1X32;
-		return;
+		break;
 	}
 
-	/* 2D mode can only be used with the YCbCr pixel encoding. */
-	if (clu->mode == V4L2_CID_VSP1_CLU_MODE_2D && clu->yuv_mode)
-		ctrl |= VI6_CLU_CTRL_AX1I_2D | VI6_CLU_CTRL_AX2I_2D
-		     |  VI6_CLU_CTRL_OS0_2D | VI6_CLU_CTRL_OS1_2D
-		     |  VI6_CLU_CTRL_OS2_2D | VI6_CLU_CTRL_M2D;
+	case VSP1_ENTITY_PARAMS_PARTITION:
+		break;
+
+	case VSP1_ENTITY_PARAMS_RUNTIME:
+		/* 2D mode can only be used with the YCbCr pixel encoding. */
+		if (clu->mode == V4L2_CID_VSP1_CLU_MODE_2D && clu->yuv_mode)
+			ctrl |= VI6_CLU_CTRL_AX1I_2D | VI6_CLU_CTRL_AX2I_2D
+			     |  VI6_CLU_CTRL_OS0_2D | VI6_CLU_CTRL_OS1_2D
+			     |  VI6_CLU_CTRL_OS2_2D | VI6_CLU_CTRL_M2D;
 
-	vsp1_clu_write(clu, dl, VI6_CLU_CTRL, ctrl);
+		vsp1_clu_write(clu, dl, VI6_CLU_CTRL, ctrl);
 
-	spin_lock_irqsave(&clu->lock, flags);
-	dlb = clu->clu;
-	clu->clu = NULL;
-	spin_unlock_irqrestore(&clu->lock, flags);
+		spin_lock_irqsave(&clu->lock, flags);
+		dlb = clu->clu;
+		clu->clu = NULL;
+		spin_unlock_irqrestore(&clu->lock, flags);
 
-	if (dlb)
-		vsp1_dl_list_add_fragment(dl, dlb);
+		if (dlb)
+			vsp1_dl_list_add_fragment(dl, dlb);
+		break;
+	}
 }
 
 static const struct vsp1_entity_operations clu_entity_ops = {
diff --git a/drivers/media/platform/vsp1/vsp1_dl.c b/drivers/media/platform/vsp1/vsp1_dl.c
index 37c3518aa2a8..ad545aff4e35 100644
--- a/drivers/media/platform/vsp1/vsp1_dl.c
+++ b/drivers/media/platform/vsp1/vsp1_dl.c
@@ -21,7 +21,6 @@
 #include "vsp1_dl.h"
 
 #define VSP1_DL_NUM_ENTRIES		256
-#define VSP1_DL_NUM_LISTS		3
 
 #define VSP1_DLH_INT_ENABLE		(1 << 1)
 #define VSP1_DLH_AUTO_START		(1 << 0)
@@ -71,6 +70,7 @@ struct vsp1_dl_body {
  * @dma: DMA address for the header
  * @body0: first display list body
  * @fragments: list of extra display list bodies
+ * @chain: entry in the display list partition chain
  */
 struct vsp1_dl_list {
 	struct list_head list;
@@ -81,6 +81,9 @@ struct vsp1_dl_list {
 
 	struct vsp1_dl_body body0;
 	struct list_head fragments;
+
+	bool has_chain;
+	struct list_head chain;
 };
 
 enum vsp1_dl_mode {
@@ -262,7 +265,6 @@ static struct vsp1_dl_list *vsp1_dl_list_alloc(struct vsp1_dl_manager *dlm)
 
 		memset(dl->header, 0, sizeof(*dl->header));
 		dl->header->lists[0].addr = dl->body0.dma;
-		dl->header->flags = VSP1_DLH_INT_ENABLE;
 	}
 
 	return dl;
@@ -293,6 +295,12 @@ struct vsp1_dl_list *vsp1_dl_list_get(struct vsp1_dl_manager *dlm)
 	if (!list_empty(&dlm->free)) {
 		dl = list_first_entry(&dlm->free, struct vsp1_dl_list, list);
 		list_del(&dl->list);
+
+		/*
+		 * The display list chain must be initialised to ensure every
+		 * display list can assert list_empty() if it is not in a chain.
+		 */
+		INIT_LIST_HEAD(&dl->chain);
 	}
 
 	spin_unlock_irqrestore(&dlm->lock, flags);
@@ -303,10 +311,24 @@ struct vsp1_dl_list *vsp1_dl_list_get(struct vsp1_dl_manager *dlm)
 /* This function must be called with the display list manager lock held.*/
 static void __vsp1_dl_list_put(struct vsp1_dl_list *dl)
 {
+	struct vsp1_dl_list *dl_child;
+
 	if (!dl)
 		return;
 
-	/* We can't free fragments here as DMA memory can only be freed in
+	/*
+	 * Release any linked display-lists which were chained for a single
+	 * hardware operation.
+	 */
+	if (dl->has_chain) {
+		list_for_each_entry(dl_child, &dl->chain, chain)
+			__vsp1_dl_list_put(dl_child);
+	}
+
+	dl->has_chain = false;
+
+	/*
+	 * We can't free fragments here as DMA memory can only be freed in
 	 * interruptible context. Move all fragments to the display list
 	 * manager's list of fragments to be freed, they will be
 	 * garbage-collected by the work queue.
@@ -383,6 +405,76 @@ int vsp1_dl_list_add_fragment(struct vsp1_dl_list *dl,
 	return 0;
 }
 
+/**
+ * vsp1_dl_list_add_chain - Add a display list to a chain
+ * @head: The head display list
+ * @dl: The new display list
+ *
+ * Add a display list to an existing display list chain. The chained lists
+ * will be automatically processed by the hardware without intervention from
+ * the CPU. A display list end interrupt will only complete after the last
+ * display list in the chain has completed processing.
+ *
+ * Adding a display list to a chain passes ownership of the display list to
+ * the head display list item. The chain is released when the head dl item is
+ * put back with __vsp1_dl_list_put().
+ *
+ * Chained display lists are only usable in header mode. Attempts to add a
+ * display list to a chain in header-less mode will return an error.
+ */
+int vsp1_dl_list_add_chain(struct vsp1_dl_list *head,
+			   struct vsp1_dl_list *dl)
+{
+	/* Chained lists are only available in header mode. */
+	if (head->dlm->mode != VSP1_DL_MODE_HEADER)
+		return -EINVAL;
+
+	head->has_chain = true;
+	list_add_tail(&dl->chain, &head->chain);
+	return 0;
+}
+
+static void vsp1_dl_list_fill_header(struct vsp1_dl_list *dl, bool is_last)
+{
+	struct vsp1_dl_header_list *hdr = dl->header->lists;
+	struct vsp1_dl_body *dlb;
+	unsigned int num_lists = 0;
+
+	/*
+	 * Fill the header with the display list bodies addresses and sizes. The
+	 * address of the first body has already been filled when the display
+	 * list was allocated.
+	 */
+
+	hdr->num_bytes = dl->body0.num_entries
+		       * sizeof(*dl->header->lists);
+
+	list_for_each_entry(dlb, &dl->fragments, list) {
+		num_lists++;
+		hdr++;
+
+		hdr->addr = dlb->dma;
+		hdr->num_bytes = dlb->num_entries
+			       * sizeof(*dl->header->lists);
+	}
+
+	dl->header->num_lists = num_lists;
+
+	/*
+	 * If this display list's chain is not empty, we are on a list, where
+	 * the next item in the list is the display list entity which should be
+	 * automatically queued by the hardware.
+	 */
+	if (!list_empty(&dl->chain) && !is_last) {
+		struct vsp1_dl_list *next = list_next_entry(dl, chain);
+
+		dl->header->next_header = next->dma;
+		dl->header->flags = VSP1_DLH_AUTO_START;
+	} else {
+		dl->header->flags = VSP1_DLH_INT_ENABLE;
+	}
+}
+
 void vsp1_dl_list_commit(struct vsp1_dl_list *dl)
 {
 	struct vsp1_dl_manager *dlm = dl->dlm;
@@ -393,30 +485,26 @@ void vsp1_dl_list_commit(struct vsp1_dl_list *dl)
 	spin_lock_irqsave(&dlm->lock, flags);
 
 	if (dl->dlm->mode == VSP1_DL_MODE_HEADER) {
-		struct vsp1_dl_header_list *hdr = dl->header->lists;
-		struct vsp1_dl_body *dlb;
-		unsigned int num_lists = 0;
+		struct vsp1_dl_list *dl_child;
 
-		/* Fill the header with the display list bodies addresses and
-		 * sizes. The address of the first body has already been filled
-		 * when the display list was allocated.
-		 *
+		/*
 		 * In header mode the caller guarantees that the hardware is
 		 * idle at this point.
 		 */
-		hdr->num_bytes = dl->body0.num_entries
-			       * sizeof(*dl->header->lists);
 
-		list_for_each_entry(dlb, &dl->fragments, list) {
-			num_lists++;
-			hdr++;
+		/* Fill the header for the head and chained display lists. */
+		vsp1_dl_list_fill_header(dl, list_empty(&dl->chain));
 
-			hdr->addr = dlb->dma;
-			hdr->num_bytes = dlb->num_entries
-				       * sizeof(*dl->header->lists);
+		list_for_each_entry(dl_child, &dl->chain, chain) {
+			bool last = list_is_last(&dl_child->chain, &dl->chain);
+
+			vsp1_dl_list_fill_header(dl_child, last);
 		}
 
-		dl->header->num_lists = num_lists;
+		/*
+		 * Commit the head display list to hardware. Chained headers
+		 * will auto-start.
+		 */
 		vsp1_write(vsp1, VI6_DL_HDR_ADDR(dlm->index), dl->dma);
 
 		dlm->active = dl;
diff --git a/drivers/media/platform/vsp1/vsp1_dl.h b/drivers/media/platform/vsp1/vsp1_dl.h
index de387cd4d745..7131aa3c5978 100644
--- a/drivers/media/platform/vsp1/vsp1_dl.h
+++ b/drivers/media/platform/vsp1/vsp1_dl.h
@@ -41,5 +41,6 @@ void vsp1_dl_fragment_free(struct vsp1_dl_body *dlb);
 void vsp1_dl_fragment_write(struct vsp1_dl_body *dlb, u32 reg, u32 data);
 int vsp1_dl_list_add_fragment(struct vsp1_dl_list *dl,
 			      struct vsp1_dl_body *dlb);
+int vsp1_dl_list_add_chain(struct vsp1_dl_list *head, struct vsp1_dl_list *dl);
 
 #endif /* __VSP1_DL_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_drm.c b/drivers/media/platform/vsp1/vsp1_drm.c
index fe9665e57b3b..cd209dccff1b 100644
--- a/drivers/media/platform/vsp1/vsp1_drm.c
+++ b/drivers/media/platform/vsp1/vsp1_drm.c
@@ -276,17 +276,18 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int rpf_index,
 	}
 
 	dev_dbg(vsp1->dev,
-		"%s: RPF%u: (%u,%u)/%ux%u -> (%u,%u)/%ux%u (%08x), pitch %u dma { %pad, %pad } zpos %u\n",
+		"%s: RPF%u: (%u,%u)/%ux%u -> (%u,%u)/%ux%u (%08x), pitch %u dma { %pad, %pad, %pad } zpos %u\n",
 		__func__, rpf_index,
 		cfg->src.left, cfg->src.top, cfg->src.width, cfg->src.height,
 		cfg->dst.left, cfg->dst.top, cfg->dst.width, cfg->dst.height,
 		cfg->pixelformat, cfg->pitch, &cfg->mem[0], &cfg->mem[1],
-		cfg->zpos);
+		&cfg->mem[2], cfg->zpos);
 
-	/* Store the format, stride, memory buffer address, crop and compose
+	/*
+	 * Store the format, stride, memory buffer address, crop and compose
 	 * rectangles and Z-order position and for the input.
 	 */
-	fmtinfo = vsp1_get_format_info(cfg->pixelformat);
+	fmtinfo = vsp1_get_format_info(vsp1, cfg->pixelformat);
 	if (!fmtinfo) {
 		dev_dbg(vsp1->dev, "Unsupport pixel format %08x for RPF\n",
 			cfg->pixelformat);
@@ -301,7 +302,7 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int rpf_index,
 
 	rpf->mem.addr[0] = cfg->mem[0];
 	rpf->mem.addr[1] = cfg->mem[1];
-	rpf->mem.addr[2] = 0;
+	rpf->mem.addr[2] = cfg->mem[2];
 
 	vsp1->drm->inputs[rpf_index].crop = cfg->src;
 	vsp1->drm->inputs[rpf_index].compose = cfg->dst;
@@ -492,16 +493,13 @@ void vsp1_du_atomic_flush(struct device *dev)
 		vsp1_entity_route_setup(entity, pipe->dl);
 
 		if (entity->ops->configure) {
-			entity->ops->configure(entity, pipe, pipe->dl, true);
-			entity->ops->configure(entity, pipe, pipe->dl, false);
+			entity->ops->configure(entity, pipe, pipe->dl,
+					       VSP1_ENTITY_PARAMS_INIT);
+			entity->ops->configure(entity, pipe, pipe->dl,
+					       VSP1_ENTITY_PARAMS_RUNTIME);
+			entity->ops->configure(entity, pipe, pipe->dl,
+					       VSP1_ENTITY_PARAMS_PARTITION);
 		}
-
-		/* The memory buffer address must be applied after configuring
-		 * the RPF to make sure the crop offset are computed.
-		 */
-		if (entity->type == VSP1_ENTITY_RPF)
-			vsp1_rwpf_set_memory(to_rwpf(&entity->subdev),
-					     pipe->dl);
 	}
 
 	vsp1_dl_list_commit(pipe->dl);
diff --git a/drivers/media/platform/vsp1/vsp1_drv.c b/drivers/media/platform/vsp1/vsp1_drv.c
index cc316d281687..57c713a4e1df 100644
--- a/drivers/media/platform/vsp1/vsp1_drv.c
+++ b/drivers/media/platform/vsp1/vsp1_drv.c
@@ -60,7 +60,7 @@ static irqreturn_t vsp1_irq_handler(int irq, void *data)
 		status = vsp1_read(vsp1, VI6_WPF_IRQ_STA(i));
 		vsp1_write(vsp1, VI6_WPF_IRQ_STA(i), ~status & mask);
 
-		if (status & VI6_WFP_IRQ_STA_FRE) {
+		if (status & VI6_WFP_IRQ_STA_DFE) {
 			vsp1_pipeline_frame_end(wpf->pipe);
 			ret = IRQ_HANDLED;
 		}
@@ -220,7 +220,8 @@ static int vsp1_create_entities(struct vsp1_device *vsp1)
 	int ret;
 
 	mdev->dev = vsp1->dev;
-	strlcpy(mdev->model, "VSP1", sizeof(mdev->model));
+	mdev->hw_revision = vsp1->version;
+	strlcpy(mdev->model, vsp1->info->model, sizeof(mdev->model));
 	snprintf(mdev->bus_info, sizeof(mdev->bus_info), "platform:%s",
 		 dev_name(mdev->dev));
 	media_device_init(mdev);
@@ -559,6 +560,7 @@ static const struct dev_pm_ops vsp1_pm_ops = {
 static const struct vsp1_device_info vsp1_device_infos[] = {
 	{
 		.version = VI6_IP_VERSION_MODEL_VSPS_H2,
+		.model = "VSP1-S",
 		.gen = 2,
 		.features = VSP1_HAS_BRU | VSP1_HAS_CLU | VSP1_HAS_LUT
 			  | VSP1_HAS_SRU | VSP1_HAS_WPF_VFLIP,
@@ -569,6 +571,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.uapi = true,
 	}, {
 		.version = VI6_IP_VERSION_MODEL_VSPR_H2,
+		.model = "VSP1-R",
 		.gen = 2,
 		.features = VSP1_HAS_BRU | VSP1_HAS_SRU | VSP1_HAS_WPF_VFLIP,
 		.rpf_count = 5,
@@ -578,6 +581,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.uapi = true,
 	}, {
 		.version = VI6_IP_VERSION_MODEL_VSPD_GEN2,
+		.model = "VSP1-D",
 		.gen = 2,
 		.features = VSP1_HAS_BRU | VSP1_HAS_LIF | VSP1_HAS_LUT,
 		.rpf_count = 4,
@@ -587,6 +591,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.uapi = true,
 	}, {
 		.version = VI6_IP_VERSION_MODEL_VSPS_M2,
+		.model = "VSP1-S",
 		.gen = 2,
 		.features = VSP1_HAS_BRU | VSP1_HAS_CLU | VSP1_HAS_LUT
 			  | VSP1_HAS_SRU | VSP1_HAS_WPF_VFLIP,
@@ -596,7 +601,30 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.num_bru_inputs = 4,
 		.uapi = true,
 	}, {
+		.version = VI6_IP_VERSION_MODEL_VSPS_V2H,
+		.model = "VSP1V-S",
+		.gen = 2,
+		.features = VSP1_HAS_BRU | VSP1_HAS_CLU | VSP1_HAS_LUT
+			  | VSP1_HAS_SRU | VSP1_HAS_WPF_VFLIP,
+		.rpf_count = 4,
+		.uds_count = 1,
+		.wpf_count = 4,
+		.num_bru_inputs = 4,
+		.uapi = true,
+	}, {
+		.version = VI6_IP_VERSION_MODEL_VSPD_V2H,
+		.model = "VSP1V-D",
+		.gen = 2,
+		.features = VSP1_HAS_BRU | VSP1_HAS_CLU | VSP1_HAS_LUT
+			  | VSP1_HAS_LIF,
+		.rpf_count = 4,
+		.uds_count = 1,
+		.wpf_count = 1,
+		.num_bru_inputs = 4,
+		.uapi = true,
+	}, {
 		.version = VI6_IP_VERSION_MODEL_VSPI_GEN3,
+		.model = "VSP2-I",
 		.gen = 3,
 		.features = VSP1_HAS_CLU | VSP1_HAS_LUT | VSP1_HAS_SRU
 			  | VSP1_HAS_WPF_HFLIP | VSP1_HAS_WPF_VFLIP,
@@ -606,6 +634,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.uapi = true,
 	}, {
 		.version = VI6_IP_VERSION_MODEL_VSPBD_GEN3,
+		.model = "VSP2-BD",
 		.gen = 3,
 		.features = VSP1_HAS_BRU | VSP1_HAS_WPF_VFLIP,
 		.rpf_count = 5,
@@ -614,6 +643,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.uapi = true,
 	}, {
 		.version = VI6_IP_VERSION_MODEL_VSPBC_GEN3,
+		.model = "VSP2-BC",
 		.gen = 3,
 		.features = VSP1_HAS_BRU | VSP1_HAS_CLU | VSP1_HAS_LUT
 			  | VSP1_HAS_WPF_VFLIP,
@@ -623,6 +653,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.uapi = true,
 	}, {
 		.version = VI6_IP_VERSION_MODEL_VSPD_GEN3,
+		.model = "VSP2-D",
 		.gen = 3,
 		.features = VSP1_HAS_BRU | VSP1_HAS_LIF | VSP1_HAS_WPF_VFLIP,
 		.rpf_count = 5,
@@ -638,7 +669,6 @@ static int vsp1_probe(struct platform_device *pdev)
 	struct resource *irq;
 	struct resource *io;
 	unsigned int i;
-	u32 version;
 	int ret;
 
 	vsp1 = devm_kzalloc(&pdev->dev, sizeof(*vsp1), GFP_KERNEL);
@@ -689,11 +719,11 @@ static int vsp1_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto done;
 
-	version = vsp1_read(vsp1, VI6_IP_VERSION);
+	vsp1->version = vsp1_read(vsp1, VI6_IP_VERSION);
 	pm_runtime_put_sync(&pdev->dev);
 
 	for (i = 0; i < ARRAY_SIZE(vsp1_device_infos); ++i) {
-		if ((version & VI6_IP_VERSION_MODEL_MASK) ==
+		if ((vsp1->version & VI6_IP_VERSION_MODEL_MASK) ==
 		    vsp1_device_infos[i].version) {
 			vsp1->info = &vsp1_device_infos[i];
 			break;
@@ -701,12 +731,13 @@ static int vsp1_probe(struct platform_device *pdev)
 	}
 
 	if (!vsp1->info) {
-		dev_err(&pdev->dev, "unsupported IP version 0x%08x\n", version);
+		dev_err(&pdev->dev, "unsupported IP version 0x%08x\n",
+			vsp1->version);
 		ret = -ENXIO;
 		goto done;
 	}
 
-	dev_dbg(&pdev->dev, "IP version 0x%08x\n", version);
+	dev_dbg(&pdev->dev, "IP version 0x%08x\n", vsp1->version);
 
 	/* Instanciate entities */
 	ret = vsp1_create_entities(vsp1);
diff --git a/drivers/media/platform/vsp1/vsp1_entity.c b/drivers/media/platform/vsp1/vsp1_entity.c
index 4cf6cc719c00..da673495c222 100644
--- a/drivers/media/platform/vsp1/vsp1_entity.c
+++ b/drivers/media/platform/vsp1/vsp1_entity.c
@@ -51,6 +51,9 @@ void vsp1_entity_route_setup(struct vsp1_entity *source,
  * @cfg: the TRY pad configuration
  * @which: configuration selector (ACTIVE or TRY)
  *
+ * When called with which set to V4L2_SUBDEV_FORMAT_ACTIVE the caller must hold
+ * the entity lock to access the returned configuration.
+ *
  * Return the pad configuration requested by the which argument. The TRY
  * configuration is passed explicitly to the function through the cfg argument
  * and simply returned when requested. The ACTIVE configuration comes from the
@@ -160,7 +163,9 @@ int vsp1_subdev_get_pad_format(struct v4l2_subdev *subdev,
 	if (!config)
 		return -EINVAL;
 
+	mutex_lock(&entity->lock);
 	fmt->format = *vsp1_entity_get_pad_format(entity, config, fmt->pad);
+	mutex_unlock(&entity->lock);
 
 	return 0;
 }
@@ -204,8 +209,10 @@ int vsp1_subdev_enum_mbus_code(struct v4l2_subdev *subdev,
 		if (!config)
 			return -EINVAL;
 
+		mutex_lock(&entity->lock);
 		format = vsp1_entity_get_pad_format(entity, config, 0);
 		code->code = format->code;
+		mutex_unlock(&entity->lock);
 	}
 
 	return 0;
@@ -235,6 +242,7 @@ int vsp1_subdev_enum_frame_size(struct v4l2_subdev *subdev,
 	struct vsp1_entity *entity = to_vsp1_entity(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
 
 	config = vsp1_entity_get_pad_config(entity, cfg, fse->which);
 	if (!config)
@@ -242,8 +250,12 @@ int vsp1_subdev_enum_frame_size(struct v4l2_subdev *subdev,
 
 	format = vsp1_entity_get_pad_format(entity, config, fse->pad);
 
-	if (fse->index || fse->code != format->code)
-		return -EINVAL;
+	mutex_lock(&entity->lock);
+
+	if (fse->index || fse->code != format->code) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	if (fse->pad == 0) {
 		fse->min_width = min_width;
@@ -260,7 +272,9 @@ int vsp1_subdev_enum_frame_size(struct v4l2_subdev *subdev,
 		fse->max_height = format->height;
 	}
 
-	return 0;
+done:
+	mutex_unlock(&entity->lock);
+	return ret;
 }
 
 /* -----------------------------------------------------------------------------
@@ -358,6 +372,8 @@ int vsp1_entity_init(struct vsp1_device *vsp1, struct vsp1_entity *entity,
 	if (i == ARRAY_SIZE(vsp1_routes))
 		return -EINVAL;
 
+	mutex_init(&entity->lock);
+
 	entity->vsp1 = vsp1;
 	entity->source_pad = num_pads - 1;
 
diff --git a/drivers/media/platform/vsp1/vsp1_entity.h b/drivers/media/platform/vsp1/vsp1_entity.h
index b43457fd2c43..901146f807b9 100644
--- a/drivers/media/platform/vsp1/vsp1_entity.h
+++ b/drivers/media/platform/vsp1/vsp1_entity.h
@@ -14,7 +14,7 @@
 #define __VSP1_ENTITY_H__
 
 #include <linux/list.h>
-#include <linux/spinlock.h>
+#include <linux/mutex.h>
 
 #include <media/v4l2-subdev.h>
 
@@ -35,6 +35,18 @@ enum vsp1_entity_type {
 	VSP1_ENTITY_WPF,
 };
 
+/**
+ * enum vsp1_entity_params - Entity configuration parameters class
+ * @VSP1_ENTITY_PARAMS_INIT - Initial parameters
+ * @VSP1_ENTITY_PARAMS_PARTITION - Per-image partition parameters
+ * @VSP1_ENTITY_PARAMS_RUNTIME - Runtime-configurable parameters
+ */
+enum vsp1_entity_params {
+	VSP1_ENTITY_PARAMS_INIT,
+	VSP1_ENTITY_PARAMS_PARTITION,
+	VSP1_ENTITY_PARAMS_RUNTIME,
+};
+
 #define VSP1_ENTITY_MAX_INPUTS		5	/* For the BRU */
 
 /*
@@ -63,17 +75,16 @@ struct vsp1_route {
 /**
  * struct vsp1_entity_operations - Entity operations
  * @destroy:	Destroy the entity.
- * @set_memory:	Setup memory buffer access. This operation applies the settings
- *		stored in the rwpf mem field to the display list. Valid for RPF
- *		and WPF only.
  * @configure:	Setup the hardware based on the entity state (pipeline, formats,
  *		selection rectangles, ...)
+ * @max_width:	Return the max supported width of data that the entity can
+ *		process in a single operation.
  */
 struct vsp1_entity_operations {
 	void (*destroy)(struct vsp1_entity *);
-	void (*set_memory)(struct vsp1_entity *, struct vsp1_dl_list *dl);
 	void (*configure)(struct vsp1_entity *, struct vsp1_pipeline *,
-			  struct vsp1_dl_list *, bool);
+			  struct vsp1_dl_list *, enum vsp1_entity_params);
+	unsigned int (*max_width)(struct vsp1_entity *, struct vsp1_pipeline *);
 };
 
 struct vsp1_entity {
@@ -96,6 +107,8 @@ struct vsp1_entity {
 
 	struct v4l2_subdev subdev;
 	struct v4l2_subdev_pad_config *config;
+
+	struct mutex lock;	/* Protects the pad config */
 };
 
 static inline struct vsp1_entity *to_vsp1_entity(struct v4l2_subdev *subdev)
diff --git a/drivers/media/platform/vsp1/vsp1_hsit.c b/drivers/media/platform/vsp1/vsp1_hsit.c
index 6e5077beb38c..94316afc54ff 100644
--- a/drivers/media/platform/vsp1/vsp1_hsit.c
+++ b/drivers/media/platform/vsp1/vsp1_hsit.c
@@ -71,10 +71,15 @@ static int hsit_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_hsit *hsit = to_hsit(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
+
+	mutex_lock(&hsit->entity.lock);
 
 	config = vsp1_entity_get_pad_config(&hsit->entity, cfg, fmt->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	format = vsp1_entity_get_pad_format(&hsit->entity, config, fmt->pad);
 
@@ -83,7 +88,7 @@ static int hsit_set_format(struct v4l2_subdev *subdev,
 		 * modified.
 		 */
 		fmt->format = *format;
-		return 0;
+		goto done;
 	}
 
 	format->code = hsit->inverse ? MEDIA_BUS_FMT_AHSV8888_1X32
@@ -104,7 +109,9 @@ static int hsit_set_format(struct v4l2_subdev *subdev,
 	format->code = hsit->inverse ? MEDIA_BUS_FMT_ARGB8888_1X32
 		     : MEDIA_BUS_FMT_AHSV8888_1X32;
 
-	return 0;
+done:
+	mutex_unlock(&hsit->entity.lock);
+	return ret;
 }
 
 static const struct v4l2_subdev_pad_ops hsit_pad_ops = {
@@ -125,11 +132,12 @@ static const struct v4l2_subdev_ops hsit_ops = {
 
 static void hsit_configure(struct vsp1_entity *entity,
 			   struct vsp1_pipeline *pipe,
-			   struct vsp1_dl_list *dl, bool full)
+			   struct vsp1_dl_list *dl,
+			   enum vsp1_entity_params params)
 {
 	struct vsp1_hsit *hsit = to_hsit(&entity->subdev);
 
-	if (!full)
+	if (params != VSP1_ENTITY_PARAMS_INIT)
 		return;
 
 	if (hsit->inverse)
diff --git a/drivers/media/platform/vsp1/vsp1_lif.c b/drivers/media/platform/vsp1/vsp1_lif.c
index a720063f38c5..e32acae1fc6e 100644
--- a/drivers/media/platform/vsp1/vsp1_lif.c
+++ b/drivers/media/platform/vsp1/vsp1_lif.c
@@ -66,10 +66,15 @@ static int lif_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_lif *lif = to_lif(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
+
+	mutex_lock(&lif->entity.lock);
 
 	config = vsp1_entity_get_pad_config(&lif->entity, cfg, fmt->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	/* Default to YUV if the requested format is not supported. */
 	if (fmt->format.code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
@@ -83,7 +88,7 @@ static int lif_set_format(struct v4l2_subdev *subdev,
 		 * format.
 		 */
 		fmt->format = *format;
-		return 0;
+		goto done;
 	}
 
 	format->code = fmt->format.code;
@@ -101,7 +106,9 @@ static int lif_set_format(struct v4l2_subdev *subdev,
 					    LIF_PAD_SOURCE);
 	*format = fmt->format;
 
-	return 0;
+done:
+	mutex_unlock(&lif->entity.lock);
+	return ret;
 }
 
 static const struct v4l2_subdev_pad_ops lif_pad_ops = {
@@ -122,7 +129,8 @@ static const struct v4l2_subdev_ops lif_ops = {
 
 static void lif_configure(struct vsp1_entity *entity,
 			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl, bool full)
+			  struct vsp1_dl_list *dl,
+			  enum vsp1_entity_params params)
 {
 	const struct v4l2_mbus_framefmt *format;
 	struct vsp1_lif *lif = to_lif(&entity->subdev);
@@ -130,7 +138,7 @@ static void lif_configure(struct vsp1_entity *entity,
 	unsigned int obth = 400;
 	unsigned int lbth = 200;
 
-	if (!full)
+	if (params != VSP1_ENTITY_PARAMS_INIT)
 		return;
 
 	format = vsp1_entity_get_pad_format(&lif->entity, lif->entity.config,
diff --git a/drivers/media/platform/vsp1/vsp1_lut.c b/drivers/media/platform/vsp1/vsp1_lut.c
index dc31de9602ba..c67cc60db0db 100644
--- a/drivers/media/platform/vsp1/vsp1_lut.c
+++ b/drivers/media/platform/vsp1/vsp1_lut.c
@@ -124,10 +124,15 @@ static int lut_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_lut *lut = to_lut(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
+
+	mutex_lock(&lut->entity.lock);
 
 	config = vsp1_entity_get_pad_config(&lut->entity, cfg, fmt->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	/* Default to YUV if the requested format is not supported. */
 	if (fmt->format.code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
@@ -140,7 +145,7 @@ static int lut_set_format(struct v4l2_subdev *subdev,
 	if (fmt->pad == LUT_PAD_SOURCE) {
 		/* The LUT output format can't be modified. */
 		fmt->format = *format;
-		return 0;
+		goto done;
 	}
 
 	format->code = fmt->format.code;
@@ -158,7 +163,9 @@ static int lut_set_format(struct v4l2_subdev *subdev,
 					    LUT_PAD_SOURCE);
 	*format = fmt->format;
 
-	return 0;
+done:
+	mutex_unlock(&lut->entity.lock);
+	return ret;
 }
 
 /* -----------------------------------------------------------------------------
@@ -183,24 +190,31 @@ static const struct v4l2_subdev_ops lut_ops = {
 
 static void lut_configure(struct vsp1_entity *entity,
 			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl, bool full)
+			  struct vsp1_dl_list *dl,
+			  enum vsp1_entity_params params)
 {
 	struct vsp1_lut *lut = to_lut(&entity->subdev);
 	struct vsp1_dl_body *dlb;
 	unsigned long flags;
 
-	if (full) {
+	switch (params) {
+	case VSP1_ENTITY_PARAMS_INIT:
 		vsp1_lut_write(lut, dl, VI6_LUT_CTRL, VI6_LUT_CTRL_EN);
-		return;
-	}
+		break;
 
-	spin_lock_irqsave(&lut->lock, flags);
-	dlb = lut->lut;
-	lut->lut = NULL;
-	spin_unlock_irqrestore(&lut->lock, flags);
+	case VSP1_ENTITY_PARAMS_PARTITION:
+		break;
+
+	case VSP1_ENTITY_PARAMS_RUNTIME:
+		spin_lock_irqsave(&lut->lock, flags);
+		dlb = lut->lut;
+		lut->lut = NULL;
+		spin_unlock_irqrestore(&lut->lock, flags);
 
-	if (dlb)
-		vsp1_dl_list_add_fragment(dl, dlb);
+		if (dlb)
+			vsp1_dl_list_add_fragment(dl, dlb);
+		break;
+	}
 }
 
 static const struct vsp1_entity_operations lut_entity_ops = {
diff --git a/drivers/media/platform/vsp1/vsp1_pipe.c b/drivers/media/platform/vsp1/vsp1_pipe.c
index 3e75fb3fcace..756ca4ea7668 100644
--- a/drivers/media/platform/vsp1/vsp1_pipe.c
+++ b/drivers/media/platform/vsp1/vsp1_pipe.c
@@ -136,17 +136,23 @@ static const struct vsp1_format_info vsp1_video_formats[] = {
 	  3, { 8, 8, 8 }, false, true, 1, 1, false },
 };
 
-/*
+/**
  * vsp1_get_format_info - Retrieve format information for a 4CC
+ * @vsp1: the VSP1 device
  * @fourcc: the format 4CC
  *
  * Return a pointer to the format information structure corresponding to the
  * given V4L2 format 4CC, or NULL if no corresponding format can be found.
  */
-const struct vsp1_format_info *vsp1_get_format_info(u32 fourcc)
+const struct vsp1_format_info *vsp1_get_format_info(struct vsp1_device *vsp1,
+						    u32 fourcc)
 {
 	unsigned int i;
 
+	/* Special case, the VYUY format is supported on Gen2 only. */
+	if (vsp1->info->gen != 2 && fourcc == V4L2_PIX_FMT_VYUY)
+		return NULL;
+
 	for (i = 0; i < ARRAY_SIZE(vsp1_video_formats); ++i) {
 		const struct vsp1_format_info *info = &vsp1_video_formats[i];
 
@@ -365,6 +371,7 @@ void vsp1_pipelines_suspend(struct vsp1_device *vsp1)
 
 void vsp1_pipelines_resume(struct vsp1_device *vsp1)
 {
+	unsigned long flags;
 	unsigned int i;
 
 	/* Resume all running pipelines. */
@@ -379,7 +386,9 @@ void vsp1_pipelines_resume(struct vsp1_device *vsp1)
 		if (pipe == NULL)
 			continue;
 
+		spin_lock_irqsave(&pipe->irqlock, flags);
 		if (vsp1_pipeline_ready(pipe))
 			vsp1_pipeline_run(pipe);
+		spin_unlock_irqrestore(&pipe->irqlock, flags);
 	}
 }
diff --git a/drivers/media/platform/vsp1/vsp1_pipe.h b/drivers/media/platform/vsp1/vsp1_pipe.h
index d20d997b1fda..ac4ad2655551 100644
--- a/drivers/media/platform/vsp1/vsp1_pipe.h
+++ b/drivers/media/platform/vsp1/vsp1_pipe.h
@@ -77,6 +77,9 @@ enum vsp1_pipeline_state {
  * @uds_input: entity at the input of the UDS, if the UDS is present
  * @entities: list of entities in the pipeline
  * @dl: display list associated with the pipeline
+ * @div_size: The maximum allowed partition size for the pipeline
+ * @partitions: The number of partitions used to process one frame
+ * @current_partition: The partition number currently being configured
  */
 struct vsp1_pipeline {
 	struct media_pipeline pipe;
@@ -104,6 +107,11 @@ struct vsp1_pipeline {
 	struct list_head entities;
 
 	struct vsp1_dl_list *dl;
+
+	unsigned int div_size;
+	unsigned int partitions;
+	struct v4l2_rect partition;
+	unsigned int current_partition;
 };
 
 void vsp1_pipeline_reset(struct vsp1_pipeline *pipe);
@@ -122,6 +130,7 @@ void vsp1_pipeline_propagate_alpha(struct vsp1_pipeline *pipe,
 void vsp1_pipelines_suspend(struct vsp1_device *vsp1);
 void vsp1_pipelines_resume(struct vsp1_device *vsp1);
 
-const struct vsp1_format_info *vsp1_get_format_info(u32 fourcc);
+const struct vsp1_format_info *vsp1_get_format_info(struct vsp1_device *vsp1,
+						    u32 fourcc);
 
 #endif /* __VSP1_PIPE_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_regs.h b/drivers/media/platform/vsp1/vsp1_regs.h
index 3b03007ba625..47b1dee044fb 100644
--- a/drivers/media/platform/vsp1/vsp1_regs.h
+++ b/drivers/media/platform/vsp1/vsp1_regs.h
@@ -660,6 +660,8 @@
 #define VI6_IP_VERSION_MODEL_VSPR_H2	(0x0a << 8)
 #define VI6_IP_VERSION_MODEL_VSPD_GEN2	(0x0b << 8)
 #define VI6_IP_VERSION_MODEL_VSPS_M2	(0x0c << 8)
+#define VI6_IP_VERSION_MODEL_VSPS_V2H	(0x12 << 8)
+#define VI6_IP_VERSION_MODEL_VSPD_V2H	(0x13 << 8)
 #define VI6_IP_VERSION_MODEL_VSPI_GEN3	(0x14 << 8)
 #define VI6_IP_VERSION_MODEL_VSPBD_GEN3	(0x15 << 8)
 #define VI6_IP_VERSION_MODEL_VSPBC_GEN3	(0x16 << 8)
diff --git a/drivers/media/platform/vsp1/vsp1_rpf.c b/drivers/media/platform/vsp1/vsp1_rpf.c
index 388838913205..b2e34a800ffa 100644
--- a/drivers/media/platform/vsp1/vsp1_rpf.c
+++ b/drivers/media/platform/vsp1/vsp1_rpf.c
@@ -46,34 +46,22 @@ static const struct v4l2_subdev_ops rpf_ops = {
  * VSP1 Entity Operations
  */
 
-static void rpf_set_memory(struct vsp1_entity *entity, struct vsp1_dl_list *dl)
-{
-	struct vsp1_rwpf *rpf = entity_to_rwpf(entity);
-
-	vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_ADDR_Y,
-		       rpf->mem.addr[0] + rpf->offsets[0]);
-	vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_ADDR_C0,
-		       rpf->mem.addr[1] + rpf->offsets[1]);
-	vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_ADDR_C1,
-		       rpf->mem.addr[2] + rpf->offsets[1]);
-}
-
 static void rpf_configure(struct vsp1_entity *entity,
 			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl, bool full)
+			  struct vsp1_dl_list *dl,
+			  enum vsp1_entity_params params)
 {
 	struct vsp1_rwpf *rpf = to_rwpf(&entity->subdev);
 	const struct vsp1_format_info *fmtinfo = rpf->fmtinfo;
 	const struct v4l2_pix_format_mplane *format = &rpf->format;
 	const struct v4l2_mbus_framefmt *source_format;
 	const struct v4l2_mbus_framefmt *sink_format;
-	const struct v4l2_rect *crop;
 	unsigned int left = 0;
 	unsigned int top = 0;
 	u32 pstride;
 	u32 infmt;
 
-	if (!full) {
+	if (params == VSP1_ENTITY_PARAMS_RUNTIME) {
 		vsp1_rpf_write(rpf, dl, VI6_RPF_VRTCOL_SET,
 			       rpf->alpha << VI6_RPF_VRTCOL_SET_LAYA_SHIFT);
 		vsp1_rpf_write(rpf, dl, VI6_RPF_MULT_ALPHA, rpf->mult_alpha |
@@ -83,34 +71,80 @@ static void rpf_configure(struct vsp1_entity *entity,
 		return;
 	}
 
-	/* Source size, stride and crop offsets.
-	 *
-	 * The crop offsets correspond to the location of the crop rectangle top
-	 * left corner in the plane buffer. Only two offsets are needed, as
-	 * planes 2 and 3 always have identical strides.
-	 */
-	crop = vsp1_rwpf_get_crop(rpf, rpf->entity.config);
+	if (params == VSP1_ENTITY_PARAMS_PARTITION) {
+		unsigned int offsets[2];
+		struct v4l2_rect crop;
+
+		/*
+		 * Source size and crop offsets.
+		 *
+		 * The crop offsets correspond to the location of the crop
+		 * rectangle top left corner in the plane buffer. Only two
+		 * offsets are needed, as planes 2 and 3 always have identical
+		 * strides.
+		 */
+		crop = *vsp1_rwpf_get_crop(rpf, rpf->entity.config);
+
+		/*
+		 * Partition Algorithm Control
+		 *
+		 * The partition algorithm can split this frame into multiple
+		 * slices. We must scale our partition window based on the pipe
+		 * configuration to match the destination partition window.
+		 * To achieve this, we adjust our crop to provide a 'sub-crop'
+		 * matching the expected partition window. Only 'left' and
+		 * 'width' need to be adjusted.
+		 */
+		if (pipe->partitions > 1) {
+			const struct v4l2_mbus_framefmt *output;
+			struct vsp1_entity *wpf = &pipe->output->entity;
+			unsigned int input_width = crop.width;
+
+			/*
+			 * Scale the partition window based on the configuration
+			 * of the pipeline.
+			 */
+			output = vsp1_entity_get_pad_format(wpf, wpf->config,
+							    RWPF_PAD_SOURCE);
 
-	vsp1_rpf_write(rpf, dl, VI6_RPF_SRC_BSIZE,
-		       (crop->width << VI6_RPF_SRC_BSIZE_BHSIZE_SHIFT) |
-		       (crop->height << VI6_RPF_SRC_BSIZE_BVSIZE_SHIFT));
-	vsp1_rpf_write(rpf, dl, VI6_RPF_SRC_ESIZE,
-		       (crop->width << VI6_RPF_SRC_ESIZE_EHSIZE_SHIFT) |
-		       (crop->height << VI6_RPF_SRC_ESIZE_EVSIZE_SHIFT));
+			crop.width = pipe->partition.width * input_width
+				   / output->width;
+			crop.left += pipe->partition.left * input_width
+				   / output->width;
+		}
+
+		vsp1_rpf_write(rpf, dl, VI6_RPF_SRC_BSIZE,
+			       (crop.width << VI6_RPF_SRC_BSIZE_BHSIZE_SHIFT) |
+			       (crop.height << VI6_RPF_SRC_BSIZE_BVSIZE_SHIFT));
+		vsp1_rpf_write(rpf, dl, VI6_RPF_SRC_ESIZE,
+			       (crop.width << VI6_RPF_SRC_ESIZE_EHSIZE_SHIFT) |
+			       (crop.height << VI6_RPF_SRC_ESIZE_EVSIZE_SHIFT));
+
+		offsets[0] = crop.top * format->plane_fmt[0].bytesperline
+			   + crop.left * fmtinfo->bpp[0] / 8;
+
+		if (format->num_planes > 1)
+			offsets[1] = crop.top * format->plane_fmt[1].bytesperline
+				   + crop.left / fmtinfo->hsub
+				   * fmtinfo->bpp[1] / 8;
+		else
+			offsets[1] = 0;
+
+		vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_ADDR_Y,
+			       rpf->mem.addr[0] + offsets[0]);
+		vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_ADDR_C0,
+			       rpf->mem.addr[1] + offsets[1]);
+		vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_ADDR_C1,
+			       rpf->mem.addr[2] + offsets[1]);
+		return;
+	}
 
-	rpf->offsets[0] = crop->top * format->plane_fmt[0].bytesperline
-			+ crop->left * fmtinfo->bpp[0] / 8;
+	/* Stride */
 	pstride = format->plane_fmt[0].bytesperline
 		<< VI6_RPF_SRCM_PSTRIDE_Y_SHIFT;
-
-	if (format->num_planes > 1) {
-		rpf->offsets[1] = crop->top * format->plane_fmt[1].bytesperline
-				+ crop->left * fmtinfo->bpp[1] / 8;
+	if (format->num_planes > 1)
 		pstride |= format->plane_fmt[1].bytesperline
 			<< VI6_RPF_SRCM_PSTRIDE_C_SHIFT;
-	} else {
-		rpf->offsets[1] = 0;
-	}
 
 	vsp1_rpf_write(rpf, dl, VI6_RPF_SRCM_PSTRIDE, pstride);
 
@@ -215,7 +249,6 @@ static void rpf_configure(struct vsp1_entity *entity,
 }
 
 static const struct vsp1_entity_operations rpf_entity_ops = {
-	.set_memory = rpf_set_memory,
 	.configure = rpf_configure,
 };
 
diff --git a/drivers/media/platform/vsp1/vsp1_rwpf.c b/drivers/media/platform/vsp1/vsp1_rwpf.c
index 8d461b375e91..66e4d7ea31d6 100644
--- a/drivers/media/platform/vsp1/vsp1_rwpf.c
+++ b/drivers/media/platform/vsp1/vsp1_rwpf.c
@@ -66,11 +66,15 @@ static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_rwpf *rwpf = to_rwpf(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
-	struct v4l2_rect *crop;
+	int ret = 0;
+
+	mutex_lock(&rwpf->entity.lock);
 
 	config = vsp1_entity_get_pad_config(&rwpf->entity, cfg, fmt->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	/* Default to YUV if the requested format is not supported. */
 	if (fmt->format.code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
@@ -85,7 +89,7 @@ static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev,
 		 */
 		format->code = fmt->format.code;
 		fmt->format = *format;
-		return 0;
+		goto done;
 	}
 
 	format->code = fmt->format.code;
@@ -98,19 +102,25 @@ static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev,
 
 	fmt->format = *format;
 
-	/* Update the sink crop rectangle. */
-	crop = vsp1_rwpf_get_crop(rwpf, config);
-	crop->left = 0;
-	crop->top = 0;
-	crop->width = fmt->format.width;
-	crop->height = fmt->format.height;
+	if (rwpf->entity.type == VSP1_ENTITY_RPF) {
+		struct v4l2_rect *crop;
+
+		/* Update the sink crop rectangle. */
+		crop = vsp1_rwpf_get_crop(rwpf, config);
+		crop->left = 0;
+		crop->top = 0;
+		crop->width = fmt->format.width;
+		crop->height = fmt->format.height;
+	}
 
 	/* Propagate the format to the source pad. */
 	format = vsp1_entity_get_pad_format(&rwpf->entity, config,
 					    RWPF_PAD_SOURCE);
 	*format = fmt->format;
 
-	return 0;
+done:
+	mutex_unlock(&rwpf->entity.lock);
+	return ret;
 }
 
 static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev,
@@ -120,14 +130,22 @@ static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev,
 	struct vsp1_rwpf *rwpf = to_rwpf(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
 
-	/* Cropping is implemented on the sink pad. */
-	if (sel->pad != RWPF_PAD_SINK)
+	/*
+	 * Cropping is only supported on the RPF and is implemented on the sink
+	 * pad.
+	 */
+	if (rwpf->entity.type == VSP1_ENTITY_WPF || sel->pad != RWPF_PAD_SINK)
 		return -EINVAL;
 
+	mutex_lock(&rwpf->entity.lock);
+
 	config = vsp1_entity_get_pad_config(&rwpf->entity, cfg, sel->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	switch (sel->target) {
 	case V4L2_SEL_TGT_CROP:
@@ -144,10 +162,13 @@ static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev,
 		break;
 
 	default:
-		return -EINVAL;
+		ret = -EINVAL;
+		break;
 	}
 
-	return 0;
+done:
+	mutex_unlock(&rwpf->entity.lock);
+	return ret;
 }
 
 static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
@@ -158,21 +179,27 @@ static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *crop;
+	int ret = 0;
 
-	/* Cropping is implemented on the sink pad. */
-	if (sel->pad != RWPF_PAD_SINK)
+	/*
+	 * Cropping is only supported on the RPF and is implemented on the sink
+	 * pad.
+	 */
+	if (rwpf->entity.type == VSP1_ENTITY_WPF || sel->pad != RWPF_PAD_SINK)
 		return -EINVAL;
 
 	if (sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 
+	mutex_lock(&rwpf->entity.lock);
+
 	config = vsp1_entity_get_pad_config(&rwpf->entity, cfg, sel->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
-	/* Make sure the crop rectangle is entirely contained in the image. The
-	 * WPF top and left offsets are limited to 255.
-	 */
+	/* Make sure the crop rectangle is entirely contained in the image. */
 	format = vsp1_entity_get_pad_format(&rwpf->entity, config,
 					    RWPF_PAD_SINK);
 
@@ -188,10 +215,6 @@ static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
 
 	sel->r.left = min_t(unsigned int, sel->r.left, format->width - 2);
 	sel->r.top = min_t(unsigned int, sel->r.top, format->height - 2);
-	if (rwpf->entity.type == VSP1_ENTITY_WPF) {
-		sel->r.left = min_t(unsigned int, sel->r.left, 255);
-		sel->r.top = min_t(unsigned int, sel->r.top, 255);
-	}
 	sel->r.width = min_t(unsigned int, sel->r.width,
 			     format->width - sel->r.left);
 	sel->r.height = min_t(unsigned int, sel->r.height,
@@ -206,7 +229,9 @@ static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
 	format->width = crop->width;
 	format->height = crop->height;
 
-	return 0;
+done:
+	mutex_unlock(&rwpf->entity.lock);
+	return ret;
 }
 
 const struct v4l2_subdev_pad_ops vsp1_rwpf_pad_ops = {
diff --git a/drivers/media/platform/vsp1/vsp1_rwpf.h b/drivers/media/platform/vsp1/vsp1_rwpf.h
index cb20484e80da..1c98aff3da5d 100644
--- a/drivers/media/platform/vsp1/vsp1_rwpf.h
+++ b/drivers/media/platform/vsp1/vsp1_rwpf.h
@@ -61,7 +61,6 @@ struct vsp1_rwpf {
 		unsigned int active;
 	} flip;
 
-	unsigned int offsets[2];
 	struct vsp1_rwpf_memory mem;
 
 	struct vsp1_dl_manager *dlm;
@@ -86,17 +85,5 @@ extern const struct v4l2_subdev_pad_ops vsp1_rwpf_pad_ops;
 
 struct v4l2_rect *vsp1_rwpf_get_crop(struct vsp1_rwpf *rwpf,
 				     struct v4l2_subdev_pad_config *config);
-/**
- * vsp1_rwpf_set_memory - Configure DMA addresses for a [RW]PF
- * @rwpf: the [RW]PF instance
- * @dl: the display list
- *
- * This function applies the cached memory buffer address to the display list.
- */
-static inline void vsp1_rwpf_set_memory(struct vsp1_rwpf *rwpf,
-					struct vsp1_dl_list *dl)
-{
-	rwpf->entity.ops->set_memory(&rwpf->entity, dl);
-}
 
 #endif /* __VSP1_RWPF_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_sru.c b/drivers/media/platform/vsp1/vsp1_sru.c
index 47f5e0cea2ce..b4e568a3b4ed 100644
--- a/drivers/media/platform/vsp1/vsp1_sru.c
+++ b/drivers/media/platform/vsp1/vsp1_sru.c
@@ -128,6 +128,7 @@ static int sru_enum_frame_size(struct v4l2_subdev *subdev,
 	struct vsp1_sru *sru = to_sru(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
 
 	config = vsp1_entity_get_pad_config(&sru->entity, cfg, fse->which);
 	if (!config)
@@ -135,8 +136,12 @@ static int sru_enum_frame_size(struct v4l2_subdev *subdev,
 
 	format = vsp1_entity_get_pad_format(&sru->entity, config, SRU_PAD_SINK);
 
-	if (fse->index || fse->code != format->code)
-		return -EINVAL;
+	mutex_lock(&sru->entity.lock);
+
+	if (fse->index || fse->code != format->code) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	if (fse->pad == SRU_PAD_SINK) {
 		fse->min_width = SRU_MIN_SIZE;
@@ -156,7 +161,9 @@ static int sru_enum_frame_size(struct v4l2_subdev *subdev,
 		}
 	}
 
-	return 0;
+done:
+	mutex_unlock(&sru->entity.lock);
+	return ret;
 }
 
 static void sru_try_format(struct vsp1_sru *sru,
@@ -217,10 +224,15 @@ static int sru_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_sru *sru = to_sru(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
+
+	mutex_lock(&sru->entity.lock);
 
 	config = vsp1_entity_get_pad_config(&sru->entity, cfg, fmt->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	sru_try_format(sru, config, fmt->pad, &fmt->format);
 
@@ -236,7 +248,9 @@ static int sru_set_format(struct v4l2_subdev *subdev,
 		sru_try_format(sru, config, SRU_PAD_SOURCE, format);
 	}
 
-	return 0;
+done:
+	mutex_unlock(&sru->entity.lock);
+	return ret;
 }
 
 static const struct v4l2_subdev_pad_ops sru_pad_ops = {
@@ -257,7 +271,8 @@ static const struct v4l2_subdev_ops sru_ops = {
 
 static void sru_configure(struct vsp1_entity *entity,
 			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl, bool full)
+			  struct vsp1_dl_list *dl,
+			  enum vsp1_entity_params params)
 {
 	const struct vsp1_sru_param *param;
 	struct vsp1_sru *sru = to_sru(&entity->subdev);
@@ -265,7 +280,7 @@ static void sru_configure(struct vsp1_entity *entity,
 	struct v4l2_mbus_framefmt *output;
 	u32 ctrl0;
 
-	if (!full)
+	if (params != VSP1_ENTITY_PARAMS_INIT)
 		return;
 
 	input = vsp1_entity_get_pad_format(&sru->entity, sru->entity.config,
@@ -291,8 +306,27 @@ static void sru_configure(struct vsp1_entity *entity,
 	vsp1_sru_write(sru, dl, VI6_SRU_CTRL2, param->ctrl2);
 }
 
+static unsigned int sru_max_width(struct vsp1_entity *entity,
+				  struct vsp1_pipeline *pipe)
+{
+	struct vsp1_sru *sru = to_sru(&entity->subdev);
+	struct v4l2_mbus_framefmt *input;
+	struct v4l2_mbus_framefmt *output;
+
+	input = vsp1_entity_get_pad_format(&sru->entity, sru->entity.config,
+					   SRU_PAD_SINK);
+	output = vsp1_entity_get_pad_format(&sru->entity, sru->entity.config,
+					    SRU_PAD_SOURCE);
+
+	if (input->width != output->width)
+		return 512;
+	else
+		return 256;
+}
+
 static const struct vsp1_entity_operations sru_entity_ops = {
 	.configure = sru_configure,
+	.max_width = sru_max_width,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/vsp1/vsp1_uds.c b/drivers/media/platform/vsp1/vsp1_uds.c
index 652dcd895022..da8f89a31ea4 100644
--- a/drivers/media/platform/vsp1/vsp1_uds.c
+++ b/drivers/media/platform/vsp1/vsp1_uds.c
@@ -18,6 +18,7 @@
 
 #include "vsp1.h"
 #include "vsp1_dl.h"
+#include "vsp1_pipe.h"
 #include "vsp1_uds.h"
 
 #define UDS_MIN_SIZE				4U
@@ -133,6 +134,7 @@ static int uds_enum_frame_size(struct v4l2_subdev *subdev,
 	struct vsp1_uds *uds = to_uds(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
 
 	config = vsp1_entity_get_pad_config(&uds->entity, cfg, fse->which);
 	if (!config)
@@ -141,8 +143,12 @@ static int uds_enum_frame_size(struct v4l2_subdev *subdev,
 	format = vsp1_entity_get_pad_format(&uds->entity, config,
 					    UDS_PAD_SINK);
 
-	if (fse->index || fse->code != format->code)
-		return -EINVAL;
+	mutex_lock(&uds->entity.lock);
+
+	if (fse->index || fse->code != format->code) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	if (fse->pad == UDS_PAD_SINK) {
 		fse->min_width = UDS_MIN_SIZE;
@@ -156,7 +162,9 @@ static int uds_enum_frame_size(struct v4l2_subdev *subdev,
 				  &fse->max_height);
 	}
 
-	return 0;
+done:
+	mutex_unlock(&uds->entity.lock);
+	return ret;
 }
 
 static void uds_try_format(struct vsp1_uds *uds,
@@ -202,10 +210,15 @@ static int uds_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_uds *uds = to_uds(subdev);
 	struct v4l2_subdev_pad_config *config;
 	struct v4l2_mbus_framefmt *format;
+	int ret = 0;
+
+	mutex_lock(&uds->entity.lock);
 
 	config = vsp1_entity_get_pad_config(&uds->entity, cfg, fmt->which);
-	if (!config)
-		return -EINVAL;
+	if (!config) {
+		ret = -EINVAL;
+		goto done;
+	}
 
 	uds_try_format(uds, config, fmt->pad, &fmt->format);
 
@@ -221,7 +234,9 @@ static int uds_set_format(struct v4l2_subdev *subdev,
 		uds_try_format(uds, config, UDS_PAD_SOURCE, format);
 	}
 
-	return 0;
+done:
+	mutex_unlock(&uds->entity.lock);
+	return ret;
 }
 
 /* -----------------------------------------------------------------------------
@@ -246,7 +261,8 @@ static const struct v4l2_subdev_ops uds_ops = {
 
 static void uds_configure(struct vsp1_entity *entity,
 			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl, bool full)
+			  struct vsp1_dl_list *dl,
+			  enum vsp1_entity_params params)
 {
 	struct vsp1_uds *uds = to_uds(&entity->subdev);
 	const struct v4l2_mbus_framefmt *output;
@@ -255,7 +271,16 @@ static void uds_configure(struct vsp1_entity *entity,
 	unsigned int vscale;
 	bool multitap;
 
-	if (!full)
+	if (params == VSP1_ENTITY_PARAMS_PARTITION) {
+		const struct v4l2_rect *clip = &pipe->partition;
+
+		vsp1_uds_write(uds, dl, VI6_UDS_CLIP_SIZE,
+			       (clip->width << VI6_UDS_CLIP_SIZE_HSIZE_SHIFT) |
+			       (clip->height << VI6_UDS_CLIP_SIZE_VSIZE_SHIFT));
+		return;
+	}
+
+	if (params != VSP1_ENTITY_PARAMS_INIT)
 		return;
 
 	input = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
@@ -287,17 +312,39 @@ static void uds_configure(struct vsp1_entity *entity,
 		       (uds_passband_width(vscale)
 				<< VI6_UDS_PASS_BWIDTH_V_SHIFT));
 
-	/* Set the scaling ratios and the output size. */
+	/* Set the scaling ratios. */
 	vsp1_uds_write(uds, dl, VI6_UDS_SCALE,
 		       (hscale << VI6_UDS_SCALE_HFRAC_SHIFT) |
 		       (vscale << VI6_UDS_SCALE_VFRAC_SHIFT));
-	vsp1_uds_write(uds, dl, VI6_UDS_CLIP_SIZE,
-		       (output->width << VI6_UDS_CLIP_SIZE_HSIZE_SHIFT) |
-		       (output->height << VI6_UDS_CLIP_SIZE_VSIZE_SHIFT));
+}
+
+static unsigned int uds_max_width(struct vsp1_entity *entity,
+				  struct vsp1_pipeline *pipe)
+{
+	struct vsp1_uds *uds = to_uds(&entity->subdev);
+	const struct v4l2_mbus_framefmt *output;
+	const struct v4l2_mbus_framefmt *input;
+	unsigned int hscale;
+
+	input = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
+					   UDS_PAD_SINK);
+	output = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
+					    UDS_PAD_SOURCE);
+	hscale = output->width / input->width;
+
+	if (hscale <= 2)
+		return 256;
+	else if (hscale <= 4)
+		return 512;
+	else if (hscale <= 8)
+		return 1024;
+	else
+		return 2048;
 }
 
 static const struct vsp1_entity_operations uds_entity_ops = {
 	.configure = uds_configure,
+	.max_width = uds_max_width,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/vsp1/vsp1_video.c b/drivers/media/platform/vsp1/vsp1_video.c
index 9fb4fc26a359..d351b9c768d2 100644
--- a/drivers/media/platform/vsp1/vsp1_video.c
+++ b/drivers/media/platform/vsp1/vsp1_video.c
@@ -117,9 +117,9 @@ static int __vsp1_video_try_format(struct vsp1_video *video,
 	/* Retrieve format information and select the default format if the
 	 * requested format isn't supported.
 	 */
-	info = vsp1_get_format_info(pix->pixelformat);
+	info = vsp1_get_format_info(video->vsp1, pix->pixelformat);
 	if (info == NULL)
-		info = vsp1_get_format_info(VSP1_VIDEO_DEF_FORMAT);
+		info = vsp1_get_format_info(video->vsp1, VSP1_VIDEO_DEF_FORMAT);
 
 	pix->pixelformat = info->fourcc;
 	pix->colorspace = V4L2_COLORSPACE_SRGB;
@@ -169,6 +169,113 @@ static int __vsp1_video_try_format(struct vsp1_video *video,
 }
 
 /* -----------------------------------------------------------------------------
+ * VSP1 Partition Algorithm support
+ */
+
+static void vsp1_video_pipeline_setup_partitions(struct vsp1_pipeline *pipe)
+{
+	struct vsp1_device *vsp1 = pipe->output->entity.vsp1;
+	const struct v4l2_mbus_framefmt *format;
+	struct vsp1_entity *entity;
+	unsigned int div_size;
+
+	format = vsp1_entity_get_pad_format(&pipe->output->entity,
+					    pipe->output->entity.config,
+					    RWPF_PAD_SOURCE);
+	div_size = format->width;
+
+	/* Gen2 hardware doesn't require image partitioning. */
+	if (vsp1->info->gen == 2) {
+		pipe->div_size = div_size;
+		pipe->partitions = 1;
+		return;
+	}
+
+	list_for_each_entry(entity, &pipe->entities, list_pipe) {
+		unsigned int entity_max = VSP1_VIDEO_MAX_WIDTH;
+
+		if (entity->ops->max_width) {
+			entity_max = entity->ops->max_width(entity, pipe);
+			if (entity_max)
+				div_size = min(div_size, entity_max);
+		}
+	}
+
+	pipe->div_size = div_size;
+	pipe->partitions = DIV_ROUND_UP(format->width, div_size);
+}
+
+/**
+ * vsp1_video_partition - Calculate the active partition output window
+ *
+ * @div_size: pre-determined maximum partition division size
+ * @index: partition index
+ *
+ * Returns a v4l2_rect describing the partition window.
+ */
+static struct v4l2_rect vsp1_video_partition(struct vsp1_pipeline *pipe,
+					     unsigned int div_size,
+					     unsigned int index)
+{
+	const struct v4l2_mbus_framefmt *format;
+	struct v4l2_rect partition;
+	unsigned int modulus;
+
+	format = vsp1_entity_get_pad_format(&pipe->output->entity,
+					    pipe->output->entity.config,
+					    RWPF_PAD_SOURCE);
+
+	/* A single partition simply processes the output size in full. */
+	if (pipe->partitions <= 1) {
+		partition.left = 0;
+		partition.top = 0;
+		partition.width = format->width;
+		partition.height = format->height;
+		return partition;
+	}
+
+	/* Initialise the partition with sane starting conditions. */
+	partition.left = index * div_size;
+	partition.top = 0;
+	partition.width = div_size;
+	partition.height = format->height;
+
+	modulus = format->width % div_size;
+
+	/*
+	 * We need to prevent the last partition from being smaller than the
+	 * *minimum* width of the hardware capabilities.
+	 *
+	 * If the modulus is less than half of the partition size,
+	 * the penultimate partition is reduced to half, which is added
+	 * to the final partition: |1234|1234|1234|12|341|
+	 * to prevents this:       |1234|1234|1234|1234|1|.
+	 */
+	if (modulus) {
+		/*
+		 * pipe->partitions is 1 based, whilst index is a 0 based index.
+		 * Normalise this locally.
+		 */
+		unsigned int partitions = pipe->partitions - 1;
+
+		if (modulus < div_size / 2) {
+			if (index == partitions - 1) {
+				/* Halve the penultimate partition. */
+				partition.width = div_size / 2;
+			} else if (index == partitions) {
+				/* Increase the final partition. */
+				partition.width = (div_size / 2) + modulus;
+				partition.left -= div_size / 2;
+			}
+		} else if (index == partitions) {
+			partition.width = modulus;
+		}
+	}
+
+	return partition;
+}
+
+/* -----------------------------------------------------------------------------
  * Pipeline Management
  */
 
@@ -234,44 +341,81 @@ static void vsp1_video_frame_end(struct vsp1_pipeline *pipe,
 {
 	struct vsp1_video *video = rwpf->video;
 	struct vsp1_vb2_buffer *buf;
-	unsigned long flags;
 
 	buf = vsp1_video_complete_buffer(video);
 	if (buf == NULL)
 		return;
 
-	spin_lock_irqsave(&pipe->irqlock, flags);
-
 	video->rwpf->mem = buf->mem;
 	pipe->buffers_ready |= 1 << video->pipe_index;
+}
 
-	spin_unlock_irqrestore(&pipe->irqlock, flags);
+static void vsp1_video_pipeline_run_partition(struct vsp1_pipeline *pipe,
+					      struct vsp1_dl_list *dl)
+{
+	struct vsp1_entity *entity;
+
+	pipe->partition = vsp1_video_partition(pipe, pipe->div_size,
+					       pipe->current_partition);
+
+	list_for_each_entry(entity, &pipe->entities, list_pipe) {
+		if (entity->ops->configure)
+			entity->ops->configure(entity, pipe, dl,
+					       VSP1_ENTITY_PARAMS_PARTITION);
+	}
 }
 
 static void vsp1_video_pipeline_run(struct vsp1_pipeline *pipe)
 {
 	struct vsp1_device *vsp1 = pipe->output->entity.vsp1;
 	struct vsp1_entity *entity;
-	unsigned int i;
 
 	if (!pipe->dl)
 		pipe->dl = vsp1_dl_list_get(pipe->output->dlm);
 
+	/*
+	 * Start with the runtime parameters as the configure operation can
+	 * compute/cache information needed when configuring partitions. This
+	 * is the case with flipping in the WPF.
+	 */
 	list_for_each_entry(entity, &pipe->entities, list_pipe) {
 		if (entity->ops->configure)
-			entity->ops->configure(entity, pipe, pipe->dl, false);
+			entity->ops->configure(entity, pipe, pipe->dl,
+					       VSP1_ENTITY_PARAMS_RUNTIME);
 	}
 
-	for (i = 0; i < vsp1->info->rpf_count; ++i) {
-		struct vsp1_rwpf *rwpf = pipe->inputs[i];
+	/* Run the first partition */
+	pipe->current_partition = 0;
+	vsp1_video_pipeline_run_partition(pipe, pipe->dl);
 
-		if (rwpf)
-			vsp1_rwpf_set_memory(rwpf, pipe->dl);
-	}
+	/* Process consecutive partitions as necessary */
+	for (pipe->current_partition = 1;
+	     pipe->current_partition < pipe->partitions;
+	     pipe->current_partition++) {
+		struct vsp1_dl_list *dl;
 
-	if (!pipe->lif)
-		vsp1_rwpf_set_memory(pipe->output, pipe->dl);
+		/*
+		 * Partition configuration operations will utilise
+		 * the pipe->current_partition variable to determine
+		 * the work they should complete.
+		 */
+		dl = vsp1_dl_list_get(pipe->output->dlm);
+
+		/*
+		 * An incomplete chain will still function, but output only
+		 * the partitions that had a dl available. The frame end
+		 * interrupt will be marked on the last dl in the chain.
+		 */
+		if (!dl) {
+			dev_err(vsp1->dev, "Failed to obtain a dl list. Frame will be incomplete\n");
+			break;
+		}
+
+		vsp1_video_pipeline_run_partition(pipe, dl);
+		vsp1_dl_list_add_chain(pipe->dl, dl);
+	}
 
+	/* Complete, and commit the head display list. */
 	vsp1_dl_list_commit(pipe->dl);
 	pipe->dl = NULL;
 
@@ -285,6 +429,8 @@ static void vsp1_video_pipeline_frame_end(struct vsp1_pipeline *pipe)
 	unsigned long flags;
 	unsigned int i;
 
+	spin_lock_irqsave(&pipe->irqlock, flags);
+
 	/* Complete buffers on all video nodes. */
 	for (i = 0; i < vsp1->info->rpf_count; ++i) {
 		if (!pipe->inputs[i])
@@ -295,8 +441,6 @@ static void vsp1_video_pipeline_frame_end(struct vsp1_pipeline *pipe)
 
 	vsp1_video_frame_end(pipe, pipe->output);
 
-	spin_lock_irqsave(&pipe->irqlock, flags);
-
 	state = pipe->state;
 	pipe->state = VSP1_PIPELINE_STOPPED;
 
@@ -607,6 +751,9 @@ static int vsp1_video_setup_pipeline(struct vsp1_pipeline *pipe)
 {
 	struct vsp1_entity *entity;
 
+	/* Determine this pipelines sizes for image partitioning support. */
+	vsp1_video_pipeline_setup_partitions(pipe);
+
 	/* Prepare the display list. */
 	pipe->dl = vsp1_dl_list_get(pipe->output->dlm);
 	if (!pipe->dl)
@@ -634,7 +781,8 @@ static int vsp1_video_setup_pipeline(struct vsp1_pipeline *pipe)
 		vsp1_entity_route_setup(entity, pipe->dl);
 
 		if (entity->ops->configure)
-			entity->ops->configure(entity, pipe, pipe->dl, true);
+			entity->ops->configure(entity, pipe, pipe->dl,
+					       VSP1_ENTITY_PARAMS_INIT);
 	}
 
 	return 0;
@@ -675,6 +823,14 @@ static void vsp1_video_stop_streaming(struct vb2_queue *vq)
 	unsigned long flags;
 	int ret;
 
+	/*
+	 * Clear the buffers ready flag to make sure the device won't be started
+	 * by a QBUF on the video node on the other side of the pipeline.
+	 */
+	spin_lock_irqsave(&video->irqlock, flags);
+	pipe->buffers_ready &= ~(1 << video->pipe_index);
+	spin_unlock_irqrestore(&video->irqlock, flags);
+
 	mutex_lock(&pipe->lock);
 	if (--pipe->stream_count == pipe->num_inputs) {
 		/* Stop the pipeline. */
diff --git a/drivers/media/platform/vsp1/vsp1_wpf.c b/drivers/media/platform/vsp1/vsp1_wpf.c
index 31983169c24a..7c48f81cd5c1 100644
--- a/drivers/media/platform/vsp1/vsp1_wpf.c
+++ b/drivers/media/platform/vsp1/vsp1_wpf.c
@@ -173,58 +173,28 @@ static void vsp1_wpf_destroy(struct vsp1_entity *entity)
 	vsp1_dlm_destroy(wpf->dlm);
 }
 
-static void wpf_set_memory(struct vsp1_entity *entity, struct vsp1_dl_list *dl)
-{
-	struct vsp1_rwpf *wpf = entity_to_rwpf(entity);
-	const struct v4l2_pix_format_mplane *format = &wpf->format;
-	struct vsp1_rwpf_memory mem = wpf->mem;
-	unsigned int flip = wpf->flip.active;
-	unsigned int offset;
-
-	/* Update the memory offsets based on flipping configuration. The
-	 * destination addresses point to the locations where the VSP starts
-	 * writing to memory, which can be different corners of the image
-	 * depending on vertical flipping. Horizontal flipping is handled
-	 * through a line buffer and doesn't modify the start address.
-	 */
-	if (flip & BIT(WPF_CTRL_VFLIP)) {
-		mem.addr[0] += (format->height - 1)
-			     * format->plane_fmt[0].bytesperline;
-
-		if (format->num_planes > 1) {
-			offset = (format->height / wpf->fmtinfo->vsub - 1)
-			       * format->plane_fmt[1].bytesperline;
-			mem.addr[1] += offset;
-			mem.addr[2] += offset;
-		}
-	}
-
-	vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_ADDR_Y, mem.addr[0]);
-	vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_ADDR_C0, mem.addr[1]);
-	vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_ADDR_C1, mem.addr[2]);
-}
-
 static void wpf_configure(struct vsp1_entity *entity,
 			  struct vsp1_pipeline *pipe,
-			  struct vsp1_dl_list *dl, bool full)
+			  struct vsp1_dl_list *dl,
+			  enum vsp1_entity_params params)
 {
 	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
 	struct vsp1_device *vsp1 = wpf->entity.vsp1;
 	const struct v4l2_mbus_framefmt *source_format;
 	const struct v4l2_mbus_framefmt *sink_format;
-	const struct v4l2_rect *crop;
 	unsigned int i;
 	u32 outfmt = 0;
 	u32 srcrpf = 0;
 
-	if (!full) {
+	if (params == VSP1_ENTITY_PARAMS_RUNTIME) {
 		const unsigned int mask = BIT(WPF_CTRL_VFLIP)
 					| BIT(WPF_CTRL_HFLIP);
+		unsigned long flags;
 
-		spin_lock(&wpf->flip.lock);
+		spin_lock_irqsave(&wpf->flip.lock, flags);
 		wpf->flip.active = (wpf->flip.active & ~mask)
 				 | (wpf->flip.pending & mask);
-		spin_unlock(&wpf->flip.lock);
+		spin_unlock_irqrestore(&wpf->flip.lock, flags);
 
 		outfmt = (wpf->alpha << VI6_WPF_OUTFMT_PDV_SHIFT) | wpf->outfmt;
 
@@ -237,17 +207,6 @@ static void wpf_configure(struct vsp1_entity *entity,
 		return;
 	}
 
-	/* Cropping */
-	crop = vsp1_rwpf_get_crop(wpf, wpf->entity.config);
-
-	vsp1_wpf_write(wpf, dl, VI6_WPF_HSZCLIP, VI6_WPF_SZCLIP_EN |
-		       (crop->left << VI6_WPF_SZCLIP_OFST_SHIFT) |
-		       (crop->width << VI6_WPF_SZCLIP_SIZE_SHIFT));
-	vsp1_wpf_write(wpf, dl, VI6_WPF_VSZCLIP, VI6_WPF_SZCLIP_EN |
-		       (crop->top << VI6_WPF_SZCLIP_OFST_SHIFT) |
-		       (crop->height << VI6_WPF_SZCLIP_SIZE_SHIFT));
-
-	/* Format */
 	sink_format = vsp1_entity_get_pad_format(&wpf->entity,
 						 wpf->entity.config,
 						 RWPF_PAD_SINK);
@@ -255,6 +214,80 @@ static void wpf_configure(struct vsp1_entity *entity,
 						   wpf->entity.config,
 						   RWPF_PAD_SOURCE);
 
+	if (params == VSP1_ENTITY_PARAMS_PARTITION) {
+		const struct v4l2_pix_format_mplane *format = &wpf->format;
+		struct vsp1_rwpf_memory mem = wpf->mem;
+		unsigned int flip = wpf->flip.active;
+		unsigned int width = source_format->width;
+		unsigned int height = source_format->height;
+		unsigned int offset;
+
+		/*
+		 * Cropping. The partition algorithm can split the image into
+		 * multiple slices.
+		 */
+		if (pipe->partitions > 1)
+			width = pipe->partition.width;
+
+		vsp1_wpf_write(wpf, dl, VI6_WPF_HSZCLIP, VI6_WPF_SZCLIP_EN |
+			       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
+			       (width << VI6_WPF_SZCLIP_SIZE_SHIFT));
+		vsp1_wpf_write(wpf, dl, VI6_WPF_VSZCLIP, VI6_WPF_SZCLIP_EN |
+			       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
+			       (height << VI6_WPF_SZCLIP_SIZE_SHIFT));
+
+		if (pipe->lif)
+			return;
+
+		/*
+		 * Update the memory offsets based on flipping configuration.
+		 * The destination addresses point to the locations where the
+		 * VSP starts writing to memory, which can be different corners
+		 * of the image depending on vertical flipping.
+		 */
+		if (pipe->partitions > 1) {
+			const struct vsp1_format_info *fmtinfo = wpf->fmtinfo;
+
+			/*
+			 * Horizontal flipping is handled through a line buffer
+			 * and doesn't modify the start address, but still needs
+			 * to be handled when image partitioning is in effect to
+			 * order the partitions correctly.
+			 */
+			if (flip & BIT(WPF_CTRL_HFLIP))
+				offset = format->width - pipe->partition.left
+					- pipe->partition.width;
+			else
+				offset = pipe->partition.left;
+
+			mem.addr[0] += offset * fmtinfo->bpp[0] / 8;
+			if (format->num_planes > 1) {
+				mem.addr[1] += offset / fmtinfo->hsub
+					     * fmtinfo->bpp[1] / 8;
+				mem.addr[2] += offset / fmtinfo->hsub
+					     * fmtinfo->bpp[2] / 8;
+			}
+		}
+
+		if (flip & BIT(WPF_CTRL_VFLIP)) {
+			mem.addr[0] += (format->height - 1)
+				     * format->plane_fmt[0].bytesperline;
+
+			if (format->num_planes > 1) {
+				offset = (format->height / wpf->fmtinfo->vsub - 1)
+				       * format->plane_fmt[1].bytesperline;
+				mem.addr[1] += offset;
+				mem.addr[2] += offset;
+			}
+		}
+
+		vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_ADDR_Y, mem.addr[0]);
+		vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_ADDR_C0, mem.addr[1]);
+		vsp1_wpf_write(wpf, dl, VI6_WPF_DSTM_ADDR_C1, mem.addr[2]);
+		return;
+	}
+
+	/* Format */
 	if (!pipe->lif) {
 		const struct v4l2_pix_format_mplane *format = &wpf->format;
 		const struct vsp1_format_info *fmtinfo = wpf->fmtinfo;
@@ -318,12 +351,11 @@ static void wpf_configure(struct vsp1_entity *entity,
 	/* Enable interrupts */
 	vsp1_dl_list_write(dl, VI6_WPF_IRQ_STA(wpf->entity.index), 0);
 	vsp1_dl_list_write(dl, VI6_WPF_IRQ_ENB(wpf->entity.index),
-			   VI6_WFP_IRQ_ENB_FREE);
+			   VI6_WFP_IRQ_ENB_DFEE);
 }
 
 static const struct vsp1_entity_operations wpf_entity_ops = {
 	.destroy = vsp1_wpf_destroy,
-	.set_memory = wpf_set_memory,
 	.configure = wpf_configure,
 };
 
@@ -360,7 +392,7 @@ struct vsp1_rwpf *vsp1_wpf_create(struct vsp1_device *vsp1, unsigned int index)
 		return ERR_PTR(ret);
 
 	/* Initialize the display list manager. */
-	wpf->dlm = vsp1_dlm_create(vsp1, index, 4);
+	wpf->dlm = vsp1_dlm_create(vsp1, index, 64);
 	if (!wpf->dlm) {
 		ret = -ENOMEM;
 		goto error;
diff --git a/drivers/media/platform/xilinx/xilinx-dma.c b/drivers/media/platform/xilinx/xilinx-dma.c
index 7ae1a134b1ff..1d5836c3fb7a 100644
--- a/drivers/media/platform/xilinx/xilinx-dma.c
+++ b/drivers/media/platform/xilinx/xilinx-dma.c
@@ -474,7 +474,7 @@ static void xvip_dma_stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irq(&dma->queued_lock);
 }
 
-static struct vb2_ops xvip_dma_queue_qops = {
+static const struct vb2_ops xvip_dma_queue_qops = {
 	.queue_setup = xvip_dma_queue_setup,
 	.buf_prepare = xvip_dma_buffer_prepare,
 	.buf_queue = xvip_dma_buffer_queue,
diff --git a/drivers/media/radio/si470x/radio-si470x-i2c.c b/drivers/media/radio/si470x/radio-si470x-i2c.c
index 471d6a8ae8a4..ee0470a3196b 100644
--- a/drivers/media/radio/si470x/radio-si470x-i2c.c
+++ b/drivers/media/radio/si470x/radio-si470x-i2c.c
@@ -509,7 +509,6 @@ static SIMPLE_DEV_PM_OPS(si470x_i2c_pm, si470x_i2c_suspend, si470x_i2c_resume);
 static struct i2c_driver si470x_i2c_driver = {
 	.driver = {
 		.name		= "si470x",
-		.owner		= THIS_MODULE,
 #ifdef CONFIG_PM_SLEEP
 		.pm		= &si470x_i2c_pm,
 #endif
diff --git a/drivers/media/radio/si470x/radio-si470x-usb.c b/drivers/media/radio/si470x/radio-si470x-usb.c
index 091d793f6583..4b132c29f290 100644
--- a/drivers/media/radio/si470x/radio-si470x-usb.c
+++ b/drivers/media/radio/si470x/radio-si470x-usb.c
@@ -627,7 +627,6 @@ static int si470x_usb_driver_probe(struct usb_interface *intf,
 
 	radio->int_in_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!radio->int_in_urb) {
-		dev_info(&intf->dev, "could not allocate int_in_urb");
 		retval = -ENOMEM;
 		goto err_intbuffer;
 	}
diff --git a/drivers/media/radio/si4713/radio-usb-si4713.c b/drivers/media/radio/si4713/radio-usb-si4713.c
index 5146be2a1a50..e5e5a1672bdb 100644
--- a/drivers/media/radio/si4713/radio-usb-si4713.c
+++ b/drivers/media/radio/si4713/radio-usb-si4713.c
@@ -402,7 +402,7 @@ static u32 si4713_functionality(struct i2c_adapter *adapter)
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm si4713_algo = {
+static const struct i2c_algorithm si4713_algo = {
 	.master_xfer   = si4713_transfer,
 	.functionality = si4713_functionality,
 };
diff --git a/drivers/media/rc/igorplugusb.c b/drivers/media/rc/igorplugusb.c
index e0c531fa01da..5cf983be07a2 100644
--- a/drivers/media/rc/igorplugusb.c
+++ b/drivers/media/rc/igorplugusb.c
@@ -203,7 +203,8 @@ static int igorplugusb_probe(struct usb_interface *intf,
 	 * This device can only store 36 pulses + spaces, which is not enough
 	 * for the NEC protocol and many others.
 	 */
-	rc->allowed_protocols = RC_BIT_ALL & ~(RC_BIT_NEC | RC_BIT_RC6_6A_20 |
+	rc->allowed_protocols = RC_BIT_ALL & ~(RC_BIT_NEC | RC_BIT_NECX |
+			RC_BIT_NEC32 | RC_BIT_RC6_6A_20 |
 			RC_BIT_RC6_6A_24 | RC_BIT_RC6_6A_32 | RC_BIT_RC6_MCE |
 			RC_BIT_SONY20 | RC_BIT_MCE_KBD | RC_BIT_SANYO);
 
diff --git a/drivers/media/rc/img-ir/img-ir-nec.c b/drivers/media/rc/img-ir/img-ir-nec.c
index 27a7ea8f1260..09314933ea08 100644
--- a/drivers/media/rc/img-ir/img-ir-nec.c
+++ b/drivers/media/rc/img-ir/img-ir-nec.c
@@ -34,19 +34,21 @@ static int img_ir_nec_scancode(int len, u64 raw, u64 enabled_protocols,
 				bitrev8(addr_inv) << 16 |
 				bitrev8(data)     <<  8 |
 				bitrev8(data_inv);
+		request->protocol = RC_TYPE_NEC32;
 	} else if ((addr_inv ^ addr) != 0xff) {
 		/* Extended NEC */
 		/* scan encoding: AAaaDD */
 		request->scancode = addr     << 16 |
 				addr_inv <<  8 |
 				data;
+		request->protocol = RC_TYPE_NECX;
 	} else {
 		/* Normal NEC */
 		/* scan encoding: AADD */
 		request->scancode = addr << 8 |
 				data;
+		request->protocol = RC_TYPE_NEC;
 	}
-	request->protocol = RC_TYPE_NEC;
 	return IMG_IR_SCANCODE;
 }
 
@@ -109,7 +111,7 @@ static int img_ir_nec_filter(const struct rc_scancode_filter *in,
  *        http://wiki.altium.com/display/ADOH/NEC+Infrared+Transmission+Protocol
  */
 struct img_ir_decoder img_ir_nec = {
-	.type = RC_BIT_NEC,
+	.type = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32,
 	.control = {
 		.decoden = 1,
 		.code_type = IMG_IR_CODETYPE_PULSEDIST,
diff --git a/drivers/media/rc/imon.c b/drivers/media/rc/imon.c
index 65f80b8b9f7a..86cc70fe2534 100644
--- a/drivers/media/rc/imon.c
+++ b/drivers/media/rc/imon.c
@@ -2211,16 +2211,11 @@ static struct imon_context *imon_init_intf0(struct usb_interface *intf,
 		goto exit;
 	}
 	rx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!rx_urb) {
-		dev_err(dev, "%s: usb_alloc_urb failed for IR urb", __func__);
+	if (!rx_urb)
 		goto rx_urb_alloc_failed;
-	}
 	tx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!tx_urb) {
-		dev_err(dev, "%s: usb_alloc_urb failed for display urb",
-			__func__);
+	if (!tx_urb)
 		goto tx_urb_alloc_failed;
-	}
 
 	mutex_init(&ictx->lock);
 	spin_lock_init(&ictx->kc_lock);
@@ -2305,10 +2300,8 @@ static struct imon_context *imon_init_intf1(struct usb_interface *intf,
 	int ret = -ENOMEM;
 
 	rx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!rx_urb) {
-		pr_err("usb_alloc_urb failed for IR urb\n");
+	if (!rx_urb)
 		goto rx_urb_alloc_failed;
-	}
 
 	mutex_lock(&ictx->lock);
 
diff --git a/drivers/media/rc/ir-nec-decoder.c b/drivers/media/rc/ir-nec-decoder.c
index bea0d1eedee0..2a9d155548ab 100644
--- a/drivers/media/rc/ir-nec-decoder.c
+++ b/drivers/media/rc/ir-nec-decoder.c
@@ -49,6 +49,7 @@ static int ir_nec_decode(struct rc_dev *dev, struct ir_raw_event ev)
 {
 	struct nec_dec *data = &dev->raw->nec;
 	u32 scancode;
+	enum rc_type rc_type;
 	u8 address, not_address, command, not_command;
 	bool send_32bits = false;
 
@@ -171,22 +172,25 @@ static int ir_nec_decode(struct rc_dev *dev, struct ir_raw_event ev)
 			 * least Apple and TiVo remotes */
 			scancode = data->bits;
 			IR_dprintk(1, "NEC (modified) scancode 0x%08x\n", scancode);
+			rc_type = RC_TYPE_NEC32;
 		} else if ((address ^ not_address) != 0xff) {
 			/* Extended NEC */
 			scancode = address     << 16 |
 				   not_address <<  8 |
 				   command;
 			IR_dprintk(1, "NEC (Ext) scancode 0x%06x\n", scancode);
+			rc_type = RC_TYPE_NECX;
 		} else {
 			/* Normal NEC */
 			scancode = address << 8 | command;
 			IR_dprintk(1, "NEC scancode 0x%04x\n", scancode);
+			rc_type = RC_TYPE_NEC;
 		}
 
 		if (data->is_nec_x)
 			data->necx_repeat = true;
 
-		rc_keydown(dev, RC_TYPE_NEC, scancode, 0);
+		rc_keydown(dev, rc_type, scancode, 0);
 		data->state = STATE_INACTIVE;
 		return 0;
 	}
@@ -198,7 +202,7 @@ static int ir_nec_decode(struct rc_dev *dev, struct ir_raw_event ev)
 }
 
 static struct ir_raw_handler nec_handler = {
-	.protocols	= RC_BIT_NEC,
+	.protocols	= RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32,
 	.decode		= ir_nec_decode,
 };
 
diff --git a/drivers/media/rc/ir-rc6-decoder.c b/drivers/media/rc/ir-rc6-decoder.c
index e0e2edefa651..5cc54c967a80 100644
--- a/drivers/media/rc/ir-rc6-decoder.c
+++ b/drivers/media/rc/ir-rc6-decoder.c
@@ -248,7 +248,7 @@ again:
 				toggle = 0;
 				break;
 			case 24:
-				protocol = RC_BIT_RC6_6A_24;
+				protocol = RC_TYPE_RC6_6A_24;
 				toggle = 0;
 				break;
 			case 32:
@@ -257,7 +257,7 @@ again:
 					toggle = !!(scancode & RC6_6A_MCE_TOGGLE_MASK);
 					scancode &= ~RC6_6A_MCE_TOGGLE_MASK;
 				} else {
-					protocol = RC_BIT_RC6_6A_32;
+					protocol = RC_TYPE_RC6_6A_32;
 					toggle = 0;
 				}
 				break;
diff --git a/drivers/media/rc/meson-ir.c b/drivers/media/rc/meson-ir.c
index fcc3b82d1454..003fff07ade2 100644
--- a/drivers/media/rc/meson-ir.c
+++ b/drivers/media/rc/meson-ir.c
@@ -24,6 +24,7 @@
 
 #define DRIVER_NAME		"meson-ir"
 
+/* valid on all Meson platforms */
 #define IR_DEC_LDR_ACTIVE	0x00
 #define IR_DEC_LDR_IDLE		0x04
 #define IR_DEC_LDR_REPEAT	0x08
@@ -32,12 +33,21 @@
 #define IR_DEC_FRAME		0x14
 #define IR_DEC_STATUS		0x18
 #define IR_DEC_REG1		0x1c
+/* only available on Meson 8b and newer */
+#define IR_DEC_REG2		0x20
 
 #define REG0_RATE_MASK		(BIT(11) - 1)
 
-#define REG1_MODE_MASK		(BIT(7) | BIT(8))
-#define REG1_MODE_NEC		(0 << 7)
-#define REG1_MODE_GENERAL	(2 << 7)
+#define DECODE_MODE_NEC		0x0
+#define DECODE_MODE_RAW		0x2
+
+/* Meson 6b uses REG1 to configure the mode */
+#define REG1_MODE_MASK		GENMASK(8, 7)
+#define REG1_MODE_SHIFT		7
+
+/* Meson 8b / GXBB use REG2 to configure the mode */
+#define REG2_MODE_MASK		GENMASK(3, 0)
+#define REG2_MODE_SHIFT		0
 
 #define REG1_TIME_IV_SHIFT	16
 #define REG1_TIME_IV_MASK	((BIT(13) - 1) << REG1_TIME_IV_SHIFT)
@@ -158,8 +168,15 @@ static int meson_ir_probe(struct platform_device *pdev)
 	/* Reset the decoder */
 	meson_ir_set_mask(ir, IR_DEC_REG1, REG1_RESET, REG1_RESET);
 	meson_ir_set_mask(ir, IR_DEC_REG1, REG1_RESET, 0);
-	/* Set general operation mode */
-	meson_ir_set_mask(ir, IR_DEC_REG1, REG1_MODE_MASK, REG1_MODE_GENERAL);
+
+	/* Set general operation mode (= raw/software decoding) */
+	if (of_device_is_compatible(node, "amlogic,meson6-ir"))
+		meson_ir_set_mask(ir, IR_DEC_REG1, REG1_MODE_MASK,
+				  DECODE_MODE_RAW << REG1_MODE_SHIFT);
+	else
+		meson_ir_set_mask(ir, IR_DEC_REG2, REG2_MODE_MASK,
+				  DECODE_MODE_RAW << REG2_MODE_SHIFT);
+
 	/* Set rate */
 	meson_ir_set_mask(ir, IR_DEC_REG0, REG0_RATE_MASK, MESON_TRATE - 1);
 	/* IRQ on rising and falling edges */
@@ -197,6 +214,8 @@ static int meson_ir_remove(struct platform_device *pdev)
 
 static const struct of_device_id meson_ir_match[] = {
 	{ .compatible = "amlogic,meson6-ir" },
+	{ .compatible = "amlogic,meson8b-ir" },
+	{ .compatible = "amlogic,meson-gxbb-ir" },
 	{ },
 };
 
diff --git a/drivers/media/rc/nuvoton-cir.c b/drivers/media/rc/nuvoton-cir.c
index 00215f343819..04fedaa75612 100644
--- a/drivers/media/rc/nuvoton-cir.c
+++ b/drivers/media/rc/nuvoton-cir.c
@@ -769,21 +769,11 @@ static void nvt_process_rx_ir_data(struct nvt_dev *nvt)
 			rawir.pulse ? "pulse" : "space", rawir.duration);
 
 		ir_raw_event_store_with_filter(nvt->rdev, &rawir);
-
-		/*
-		 * BUF_PULSE_BIT indicates end of IR data, BUF_REPEAT_BYTE
-		 * indicates end of IR signal, but new data incoming. In both
-		 * cases, it means we're ready to call ir_raw_event_handle
-		 */
-		if ((sample == BUF_PULSE_BIT) && (i + 1 < nvt->pkts)) {
-			nvt_dbg("Calling ir_raw_event_handle (signal end)\n");
-			ir_raw_event_handle(nvt->rdev);
-		}
 	}
 
 	nvt->pkts = 0;
 
-	nvt_dbg("Calling ir_raw_event_handle (buffer empty)\n");
+	nvt_dbg("Calling ir_raw_event_handle\n");
 	ir_raw_event_handle(nvt->rdev);
 
 	nvt_dbg_verbose("%s done", __func__);
@@ -801,8 +791,7 @@ static void nvt_handle_rx_fifo_overrun(struct nvt_dev *nvt)
 /* copy data from hardware rx fifo into driver buffer */
 static void nvt_get_rx_ir_data(struct nvt_dev *nvt)
 {
-	u8 fifocount, val;
-	unsigned int b_idx;
+	u8 fifocount;
 	int i;
 
 	/* Get count of how many bytes to read from RX FIFO */
@@ -810,21 +799,11 @@ static void nvt_get_rx_ir_data(struct nvt_dev *nvt)
 
 	nvt_dbg("attempting to fetch %u bytes from hw rx fifo", fifocount);
 
-	b_idx = nvt->pkts;
-
-	/* This should never happen, but lets check anyway... */
-	if (b_idx + fifocount > RX_BUF_LEN) {
-		nvt_process_rx_ir_data(nvt);
-		b_idx = 0;
-	}
-
 	/* Read fifocount bytes from CIR Sample RX FIFO register */
-	for (i = 0; i < fifocount; i++) {
-		val = nvt_cir_reg_read(nvt, CIR_SRXFIFO);
-		nvt->buf[b_idx + i] = val;
-	}
+	for (i = 0; i < fifocount; i++)
+		nvt->buf[i] = nvt_cir_reg_read(nvt, CIR_SRXFIFO);
 
-	nvt->pkts += fifocount;
+	nvt->pkts = fifocount;
 	nvt_dbg("%s: pkts now %d", __func__, nvt->pkts);
 
 	nvt_process_rx_ir_data(nvt);
@@ -886,6 +865,15 @@ static irqreturn_t nvt_cir_isr(int irq, void *data)
 	status = nvt_cir_reg_read(nvt, CIR_IRSTS);
 	iren = nvt_cir_reg_read(nvt, CIR_IREN);
 
+	/* At least NCT6779D creates a spurious interrupt when the
+	 * logical device is being disabled.
+	 */
+	if (status == 0xff && iren == 0xff) {
+		spin_unlock_irqrestore(&nvt->nvt_lock, flags);
+		nvt_dbg_verbose("Spurious interrupt detected");
+		return IRQ_HANDLED;
+	}
+
 	/* IRQ may be shared with CIR WAKE, therefore check for each
 	 * status bit whether the related interrupt source is enabled
 	 */
diff --git a/drivers/media/rc/rc-ir-raw.c b/drivers/media/rc/rc-ir-raw.c
index 144304c94606..205ecc602e34 100644
--- a/drivers/media/rc/rc-ir-raw.c
+++ b/drivers/media/rc/rc-ir-raw.c
@@ -26,6 +26,7 @@ static LIST_HEAD(ir_raw_client_list);
 /* Used to handle IR raw handler extensions */
 static DEFINE_MUTEX(ir_raw_handler_lock);
 static LIST_HEAD(ir_raw_handler_list);
+static DEFINE_MUTEX(available_protocols_lock);
 static u64 available_protocols;
 
 static int ir_raw_event_thread(void *data)
@@ -234,9 +235,9 @@ u64
 ir_raw_get_allowed_protocols(void)
 {
 	u64 protocols;
-	mutex_lock(&ir_raw_handler_lock);
+	mutex_lock(&available_protocols_lock);
 	protocols = available_protocols;
-	mutex_unlock(&ir_raw_handler_lock);
+	mutex_unlock(&available_protocols_lock);
 	return protocols;
 }
 
@@ -330,7 +331,9 @@ int ir_raw_handler_register(struct ir_raw_handler *ir_raw_handler)
 	if (ir_raw_handler->raw_register)
 		list_for_each_entry(raw, &ir_raw_client_list, list)
 			ir_raw_handler->raw_register(raw->dev);
+	mutex_lock(&available_protocols_lock);
 	available_protocols |= ir_raw_handler->protocols;
+	mutex_unlock(&available_protocols_lock);
 	mutex_unlock(&ir_raw_handler_lock);
 
 	return 0;
@@ -349,7 +352,9 @@ void ir_raw_handler_unregister(struct ir_raw_handler *ir_raw_handler)
 		if (ir_raw_handler->raw_unregister)
 			ir_raw_handler->raw_unregister(raw->dev);
 	}
+	mutex_lock(&available_protocols_lock);
 	available_protocols &= ~protocols;
+	mutex_unlock(&available_protocols_lock);
 	mutex_unlock(&ir_raw_handler_lock);
 }
 EXPORT_SYMBOL(ir_raw_handler_unregister);
diff --git a/drivers/media/rc/rc-main.c b/drivers/media/rc/rc-main.c
index 8e7f2929fa6f..d9c1f2ff7119 100644
--- a/drivers/media/rc/rc-main.c
+++ b/drivers/media/rc/rc-main.c
@@ -795,7 +795,9 @@ static const struct {
 	{ RC_BIT_UNKNOWN,	"unknown",	NULL			},
 	{ RC_BIT_RC5 |
 	  RC_BIT_RC5X,		"rc-5",		"ir-rc5-decoder"	},
-	{ RC_BIT_NEC,		"nec",		"ir-nec-decoder"	},
+	{ RC_BIT_NEC |
+	  RC_BIT_NECX |
+	  RC_BIT_NEC32,		"nec",		"ir-nec-decoder"	},
 	{ RC_BIT_RC6_0 |
 	  RC_BIT_RC6_6A_20 |
 	  RC_BIT_RC6_6A_24 |
@@ -1460,6 +1462,10 @@ int rc_register_device(struct rc_dev *dev)
 	dev->input_dev->phys = dev->input_phys;
 	dev->input_dev->name = dev->input_name;
 
+	rc = input_register_device(dev->input_dev);
+	if (rc)
+		goto out_table;
+
 	/*
 	 * Default delay of 250ms is too short for some protocols, especially
 	 * since the timeout is currently set to 250ms. Increase it to 500ms,
@@ -1475,11 +1481,6 @@ int rc_register_device(struct rc_dev *dev)
 	 */
 	dev->input_dev->rep[REP_PERIOD] = 125;
 
-	/* rc_open will be called here */
-	rc = input_register_device(dev->input_dev);
-	if (rc)
-		goto out_table;
-
 	path = kobject_get_path(&dev->dev.kobj, GFP_KERNEL);
 	dev_info(&dev->dev, "%s as %s\n",
 		dev->input_name ?: "Unspecified device", path ?: "N/A");
diff --git a/drivers/media/rc/redrat3.c b/drivers/media/rc/redrat3.c
index 399f44d89a29..05ba47bc0b61 100644
--- a/drivers/media/rc/redrat3.c
+++ b/drivers/media/rc/redrat3.c
@@ -124,6 +124,41 @@
 #define USB_RR3USB_PRODUCT_ID	0x0001
 #define USB_RR3IIUSB_PRODUCT_ID	0x0005
 
+
+/*
+ * The redrat3 encodes an IR signal as set of different lengths and a set
+ * of indices into those lengths. This sets how much two lengths must
+ * differ before they are considered distinct, the value is specified
+ * in microseconds.
+ * Default 5, value 0 to 127.
+ */
+static int length_fuzz = 5;
+module_param(length_fuzz, uint, 0644);
+MODULE_PARM_DESC(length_fuzz, "Length Fuzz (0-127)");
+
+/*
+ * When receiving a continuous ir stream (for example when a user is
+ * holding a button down on a remote), this specifies the minimum size
+ * of a space when the redrat3 sends a irdata packet to the host. Specified
+ * in miliseconds. Default value 18ms.
+ * The value can be between 2 and 30 inclusive.
+ */
+static int minimum_pause = 18;
+module_param(minimum_pause, uint, 0644);
+MODULE_PARM_DESC(minimum_pause, "Minimum Pause in ms (2-30)");
+
+/*
+ * The carrier frequency is measured during the first pulse of the IR
+ * signal. The larger the number of periods used To measure, the more
+ * accurate the result is likely to be, however some signals have short
+ * initial pulses, so in some case it may be necessary to reduce this value.
+ * Default 8, value 1 to 255.
+ */
+static int periods_measure_carrier = 8;
+module_param(periods_measure_carrier, uint, 0644);
+MODULE_PARM_DESC(periods_measure_carrier, "Number of Periods to Measure Carrier (1-255)");
+
+
 struct redrat3_header {
 	__be16 length;
 	__be16 transfer_type;
@@ -188,9 +223,6 @@ struct redrat3_dev {
 	/* usb dma */
 	dma_addr_t dma_in;
 
-	/* rx signal timeout */
-	u32 hw_timeout;
-
 	/* Is the device currently transmitting?*/
 	bool transmitting;
 
@@ -372,7 +404,7 @@ static void redrat3_process_ir_data(struct redrat3_dev *rr3)
 	/* add a trailing space */
 	rawir.pulse = false;
 	rawir.timeout = true;
-	rawir.duration = US_TO_NS(rr3->hw_timeout);
+	rawir.duration = rr3->rc->timeout;
 	dev_dbg(dev, "storing trailing timeout with duration %d\n",
 							rawir.duration);
 	ir_raw_event_store_with_filter(rr3->rc, &rawir);
@@ -480,7 +512,7 @@ static int redrat3_set_timeout(struct rc_dev *rc_dev, unsigned int timeoutns)
 	struct redrat3_dev *rr3 = rc_dev->priv;
 	struct usb_device *udev = rr3->udev;
 	struct device *dev = rr3->dev;
-	u32 *timeout;
+	__be32 *timeout;
 	int ret;
 
 	timeout = kmalloc(sizeof(*timeout), GFP_KERNEL);
@@ -495,10 +527,9 @@ static int redrat3_set_timeout(struct rc_dev *rc_dev, unsigned int timeoutns)
 	dev_dbg(dev, "set ir parm timeout %d ret 0x%02x\n",
 						be32_to_cpu(*timeout), ret);
 
-	if (ret == sizeof(*timeout)) {
-		rr3->hw_timeout = timeoutns / 1000;
+	if (ret == sizeof(*timeout))
 		ret = 0;
-	} else if (ret >= 0)
+	else if (ret >= 0)
 		ret = -EIO;
 
 	kfree(timeout);
@@ -529,12 +560,25 @@ static void redrat3_reset(struct redrat3_dev *rr3)
 			     RR3_CPUCS_REG_ADDR, 0, val, len, HZ * 25);
 	dev_dbg(dev, "reset returned 0x%02x\n", rc);
 
-	*val = 5;
+	*val = length_fuzz;
 	rc = usb_control_msg(udev, txpipe, RR3_SET_IR_PARAM,
 			     USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
 			     RR3_IR_IO_LENGTH_FUZZ, 0, val, len, HZ * 25);
 	dev_dbg(dev, "set ir parm len fuzz %d rc 0x%02x\n", *val, rc);
 
+	*val = (65536 - (minimum_pause * 2000)) / 256;
+	rc = usb_control_msg(udev, txpipe, RR3_SET_IR_PARAM,
+			     USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+			     RR3_IR_IO_MIN_PAUSE, 0, val, len, HZ * 25);
+	dev_dbg(dev, "set ir parm min pause %d rc 0x%02x\n", *val, rc);
+
+	*val = periods_measure_carrier;
+	rc = usb_control_msg(udev, txpipe, RR3_SET_IR_PARAM,
+			     USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
+			     RR3_IR_IO_PERIODS_MF, 0, val, len, HZ * 25);
+	dev_dbg(dev, "set ir parm periods measure carrier %d rc 0x%02x", *val,
+									rc);
+
 	*val = RR3_DRIVER_MAXLENS;
 	rc = usb_control_msg(udev, txpipe, RR3_SET_IR_PARAM,
 			     USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
@@ -889,7 +933,7 @@ static struct rc_dev *redrat3_init_rc_dev(struct redrat3_dev *rr3)
 	rc->allowed_protocols = RC_BIT_ALL;
 	rc->min_timeout = MS_TO_NS(RR3_RX_MIN_TIMEOUT);
 	rc->max_timeout = MS_TO_NS(RR3_RX_MAX_TIMEOUT);
-	rc->timeout = US_TO_NS(rr3->hw_timeout);
+	rc->timeout = US_TO_NS(redrat3_get_timeout(rr3));
 	rc->s_timeout = redrat3_set_timeout;
 	rc->tx_ir = redrat3_transmit_ir;
 	rc->s_tx_carrier = redrat3_set_tx_carrier;
@@ -970,10 +1014,8 @@ static int redrat3_dev_probe(struct usb_interface *intf,
 
 	/* set up bulk-in endpoint */
 	rr3->read_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!rr3->read_urb) {
-		dev_err(dev, "Read urb allocation failure\n");
+	if (!rr3->read_urb)
 		goto error;
-	}
 
 	rr3->ep_in = ep_in;
 	rr3->bulk_in_buf = usb_alloc_coherent(udev,
@@ -1000,9 +1042,6 @@ static int redrat3_dev_probe(struct usb_interface *intf,
 	if (retval < 0)
 		goto error;
 
-	/* store current hardware timeout, in µs */
-	rr3->hw_timeout = redrat3_get_timeout(rr3);
-
 	/* default.. will get overridden by any sends with a freq defined */
 	rr3->carrier = 38000;
 
diff --git a/drivers/media/rc/streamzap.c b/drivers/media/rc/streamzap.c
index 815243c65bc3..4004260a7c69 100644
--- a/drivers/media/rc/streamzap.c
+++ b/drivers/media/rc/streamzap.c
@@ -499,7 +499,7 @@ static int streamzap_resume(struct usb_interface *intf)
 	struct streamzap_ir *sz = usb_get_intfdata(intf);
 
 	if (usb_submit_urb(sz->urb_in, GFP_ATOMIC)) {
-		dev_err(sz->dev, "Error sumbiting urb\n");
+		dev_err(sz->dev, "Error submitting urb\n");
 		return -EIO;
 	}
 
diff --git a/drivers/media/spi/Kconfig b/drivers/media/spi/Kconfig
new file mode 100644
index 000000000000..a21f5a39a440
--- /dev/null
+++ b/drivers/media/spi/Kconfig
@@ -0,0 +1,14 @@
+if VIDEO_V4L2
+
+menu "SPI helper chips"
+	visible if !MEDIA_SUBDRV_AUTOSELECT || COMPILE_TEST
+
+config VIDEO_GS1662
+	tristate "Gennum Serializers video"
+	depends on SPI && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	---help---
+	  Enable the GS1662 driver which serializes video streams.
+
+endmenu
+
+endif
diff --git a/drivers/media/spi/Makefile b/drivers/media/spi/Makefile
new file mode 100644
index 000000000000..ea64013d16cc
--- /dev/null
+++ b/drivers/media/spi/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_VIDEO_GS1662) += gs1662.o
diff --git a/drivers/media/spi/gs1662.c b/drivers/media/spi/gs1662.c
new file mode 100644
index 000000000000..d76f36233f43
--- /dev/null
+++ b/drivers/media/spi/gs1662.c
@@ -0,0 +1,478 @@
+/*
+ * GS1662 device registration.
+ *
+ * Copyright (C) 2015-2016 Nexvision
+ * Author: Charles-Antoine Couret <charles-antoine.couret@nexvision.fr>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/spi/spi.h>
+#include <linux/platform_device.h>
+#include <linux/ctype.h>
+#include <linux/err.h>
+#include <linux/device.h>
+#include <linux/module.h>
+
+#include <linux/videodev2.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+#include <media/v4l2-dv-timings.h>
+#include <linux/v4l2-dv-timings.h>
+
+#define REG_STATUS			0x04
+#define REG_FORCE_FMT			0x06
+#define REG_LINES_PER_FRAME		0x12
+#define REG_WORDS_PER_LINE		0x13
+#define REG_WORDS_PER_ACT_LINE		0x14
+#define REG_ACT_LINES_PER_FRAME	0x15
+
+#define MASK_H_LOCK			0x001
+#define MASK_V_LOCK			0x002
+#define MASK_STD_LOCK			0x004
+#define MASK_FORCE_STD			0x020
+#define MASK_STD_STATUS		0x3E0
+
+#define GS_WIDTH_MIN			720
+#define GS_WIDTH_MAX			2048
+#define GS_HEIGHT_MIN			487
+#define GS_HEIGHT_MAX			1080
+#define GS_PIXELCLOCK_MIN		10519200
+#define GS_PIXELCLOCK_MAX		74250000
+
+struct gs {
+	struct spi_device *pdev;
+	struct v4l2_subdev sd;
+	struct v4l2_dv_timings current_timings;
+	int enabled;
+};
+
+struct gs_reg_fmt {
+	u16 reg_value;
+	struct v4l2_dv_timings format;
+};
+
+struct gs_reg_fmt_custom {
+	u16 reg_value;
+	__u32 width;
+	__u32 height;
+	__u64 pixelclock;
+	__u32 interlaced;
+};
+
+static const struct spi_device_id gs_id[] = {
+	{ "gs1662", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, gs_id);
+
+static const struct v4l2_dv_timings fmt_cap[] = {
+	V4L2_DV_BT_SDI_720X487I60,
+	V4L2_DV_BT_CEA_720X576P50,
+	V4L2_DV_BT_CEA_1280X720P24,
+	V4L2_DV_BT_CEA_1280X720P25,
+	V4L2_DV_BT_CEA_1280X720P30,
+	V4L2_DV_BT_CEA_1280X720P50,
+	V4L2_DV_BT_CEA_1280X720P60,
+	V4L2_DV_BT_CEA_1920X1080P24,
+	V4L2_DV_BT_CEA_1920X1080P25,
+	V4L2_DV_BT_CEA_1920X1080P30,
+	V4L2_DV_BT_CEA_1920X1080I50,
+	V4L2_DV_BT_CEA_1920X1080I60,
+};
+
+static const struct gs_reg_fmt reg_fmt[] = {
+	{ 0x00, V4L2_DV_BT_CEA_1280X720P60 },
+	{ 0x01, V4L2_DV_BT_CEA_1280X720P60 },
+	{ 0x02, V4L2_DV_BT_CEA_1280X720P30 },
+	{ 0x03, V4L2_DV_BT_CEA_1280X720P30 },
+	{ 0x04, V4L2_DV_BT_CEA_1280X720P50 },
+	{ 0x05, V4L2_DV_BT_CEA_1280X720P50 },
+	{ 0x06, V4L2_DV_BT_CEA_1280X720P25 },
+	{ 0x07, V4L2_DV_BT_CEA_1280X720P25 },
+	{ 0x08, V4L2_DV_BT_CEA_1280X720P24 },
+	{ 0x09, V4L2_DV_BT_CEA_1280X720P24 },
+	{ 0x0A, V4L2_DV_BT_CEA_1920X1080I60 },
+	{ 0x0B, V4L2_DV_BT_CEA_1920X1080P30 },
+
+	/* Default value: keep this field before 0xC */
+	{ 0x14, V4L2_DV_BT_CEA_1920X1080I50 },
+	{ 0x0C, V4L2_DV_BT_CEA_1920X1080I50 },
+	{ 0x0D, V4L2_DV_BT_CEA_1920X1080P25 },
+	{ 0x0E, V4L2_DV_BT_CEA_1920X1080P25 },
+	{ 0x10, V4L2_DV_BT_CEA_1920X1080P24 },
+	{ 0x12, V4L2_DV_BT_CEA_1920X1080P24 },
+	{ 0x16, V4L2_DV_BT_SDI_720X487I60 },
+	{ 0x19, V4L2_DV_BT_SDI_720X487I60 },
+	{ 0x18, V4L2_DV_BT_CEA_720X576P50 },
+	{ 0x1A, V4L2_DV_BT_CEA_720X576P50 },
+
+	/* Implement following timings before enable it.
+	 * Because of we don't have access to these theoretical timings yet.
+	 * Workaround: use functions to get and set registers for these formats.
+	 */
+#if 0
+	{ 0x0F, V4L2_DV_BT_XXX_1920X1080I25 }, /* SMPTE 274M */
+	{ 0x11, V4L2_DV_BT_XXX_1920X1080I24 }, /* SMPTE 274M */
+	{ 0x13, V4L2_DV_BT_XXX_1920X1080I25 }, /* SMPTE 274M */
+	{ 0x15, V4L2_DV_BT_XXX_1920X1035I60 }, /* SMPTE 260M */
+	{ 0x17, V4L2_DV_BT_SDI_720X507I60 }, /* SMPTE 125M */
+	{ 0x1B, V4L2_DV_BT_SDI_720X507I60 }, /* SMPTE 125M */
+	{ 0x1C, V4L2_DV_BT_XXX_2048X1080P25 }, /* SMPTE 428.1M */
+#endif
+};
+
+static const struct v4l2_dv_timings_cap gs_timings_cap = {
+	.type = V4L2_DV_BT_656_1120,
+	/* keep this initialization for compatibility with GCC < 4.4.6 */
+	.reserved = { 0 },
+	V4L2_INIT_BT_TIMINGS(GS_WIDTH_MIN, GS_WIDTH_MAX, GS_HEIGHT_MIN,
+			     GS_HEIGHT_MAX, GS_PIXELCLOCK_MIN,
+			     GS_PIXELCLOCK_MAX,
+			     V4L2_DV_BT_STD_CEA861 | V4L2_DV_BT_STD_SDI,
+			     V4L2_DV_BT_CAP_PROGRESSIVE
+			     | V4L2_DV_BT_CAP_INTERLACED)
+};
+
+static int gs_read_register(struct spi_device *spi, u16 addr, u16 *value)
+{
+	int ret;
+	u16 buf_addr = (0x8000 | (0x0FFF & addr));
+	u16 buf_value = 0;
+	struct spi_message msg;
+	struct spi_transfer tx[] = {
+		{
+			.tx_buf = &buf_addr,
+			.len = 2,
+			.delay_usecs = 1,
+		}, {
+			.rx_buf = &buf_value,
+			.len = 2,
+			.delay_usecs = 1,
+		},
+	};
+
+	spi_message_init(&msg);
+	spi_message_add_tail(&tx[0], &msg);
+	spi_message_add_tail(&tx[1], &msg);
+	ret = spi_sync(spi, &msg);
+
+	*value = buf_value;
+
+	return ret;
+}
+
+static int gs_write_register(struct spi_device *spi, u16 addr, u16 value)
+{
+	int ret;
+	u16 buf_addr = addr;
+	u16 buf_value = value;
+	struct spi_message msg;
+	struct spi_transfer tx[] = {
+		{
+			.tx_buf = &buf_addr,
+			.len = 2,
+			.delay_usecs = 1,
+		}, {
+			.tx_buf = &buf_value,
+			.len = 2,
+			.delay_usecs = 1,
+		},
+	};
+
+	spi_message_init(&msg);
+	spi_message_add_tail(&tx[0], &msg);
+	spi_message_add_tail(&tx[1], &msg);
+	ret = spi_sync(spi, &msg);
+
+	return ret;
+}
+
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+static int gs_g_register(struct v4l2_subdev *sd,
+		  struct v4l2_dbg_register *reg)
+{
+	struct spi_device *spi = v4l2_get_subdevdata(sd);
+	u16 val;
+	int ret;
+
+	ret = gs_read_register(spi, reg->reg & 0xFFFF, &val);
+	reg->val = val;
+	reg->size = 2;
+	return ret;
+}
+
+static int gs_s_register(struct v4l2_subdev *sd,
+		  const struct v4l2_dbg_register *reg)
+{
+	struct spi_device *spi = v4l2_get_subdevdata(sd);
+
+	return gs_write_register(spi, reg->reg & 0xFFFF, reg->val & 0xFFFF);
+}
+#endif
+
+static int gs_status_format(u16 status, struct v4l2_dv_timings *timings)
+{
+	int std = (status & MASK_STD_STATUS) >> 5;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(reg_fmt); i++) {
+		if (reg_fmt[i].reg_value == std) {
+			*timings = reg_fmt[i].format;
+			return 0;
+		}
+	}
+
+	return -ERANGE;
+}
+
+static u16 get_register_timings(struct v4l2_dv_timings *timings)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(reg_fmt); i++) {
+		if (v4l2_match_dv_timings(timings, &reg_fmt[i].format, 0,
+					  false))
+			return reg_fmt[i].reg_value | MASK_FORCE_STD;
+	}
+
+	return 0x0;
+}
+
+static inline struct gs *to_gs(struct v4l2_subdev *sd)
+{
+	return container_of(sd, struct gs, sd);
+}
+
+static int gs_s_dv_timings(struct v4l2_subdev *sd,
+		    struct v4l2_dv_timings *timings)
+{
+	struct gs *gs = to_gs(sd);
+	int reg_value;
+
+	reg_value = get_register_timings(timings);
+	if (reg_value == 0x0)
+		return -EINVAL;
+
+	gs->current_timings = *timings;
+	return 0;
+}
+
+static int gs_g_dv_timings(struct v4l2_subdev *sd,
+		    struct v4l2_dv_timings *timings)
+{
+	struct gs *gs = to_gs(sd);
+
+	*timings = gs->current_timings;
+	return 0;
+}
+
+static int gs_query_dv_timings(struct v4l2_subdev *sd,
+			struct v4l2_dv_timings *timings)
+{
+	struct gs *gs = to_gs(sd);
+	struct v4l2_dv_timings fmt;
+	u16 reg_value, i;
+	int ret;
+
+	if (gs->enabled)
+		return -EBUSY;
+
+	/*
+	 * Check if the component detect a line, a frame or something else
+	 * which looks like a video signal activity.
+	 */
+	for (i = 0; i < 4; i++) {
+		gs_read_register(gs->pdev, REG_LINES_PER_FRAME + i, &reg_value);
+		if (reg_value)
+			break;
+	}
+
+	/* If no register reports a video signal */
+	if (i >= 4)
+		return -ENOLINK;
+
+	gs_read_register(gs->pdev, REG_STATUS, &reg_value);
+	if (!(reg_value & MASK_H_LOCK) || !(reg_value & MASK_V_LOCK))
+		return -ENOLCK;
+	if (!(reg_value & MASK_STD_LOCK))
+		return -ERANGE;
+
+	ret = gs_status_format(reg_value, &fmt);
+
+	if (ret < 0)
+		return ret;
+
+	*timings = fmt;
+	return 0;
+}
+
+static int gs_enum_dv_timings(struct v4l2_subdev *sd,
+		       struct v4l2_enum_dv_timings *timings)
+{
+	if (timings->index >= ARRAY_SIZE(fmt_cap))
+		return -EINVAL;
+
+	if (timings->pad != 0)
+		return -EINVAL;
+
+	timings->timings = fmt_cap[timings->index];
+	return 0;
+}
+
+static int gs_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct gs *gs = to_gs(sd);
+	int reg_value;
+
+	if (gs->enabled == enable)
+		return 0;
+
+	gs->enabled = enable;
+
+	if (enable) {
+		/* To force the specific format */
+		reg_value = get_register_timings(&gs->current_timings);
+		return gs_write_register(gs->pdev, REG_FORCE_FMT, reg_value);
+	}
+
+	/* To renable auto-detection mode */
+	return gs_write_register(gs->pdev, REG_FORCE_FMT, 0x0);
+}
+
+static int gs_g_input_status(struct v4l2_subdev *sd, u32 *status)
+{
+	struct gs *gs = to_gs(sd);
+	u16 reg_value, i;
+	int ret;
+
+	/*
+	 * Check if the component detect a line, a frame or something else
+	 * which looks like a video signal activity.
+	 */
+	for (i = 0; i < 4; i++) {
+		ret = gs_read_register(gs->pdev,
+				       REG_LINES_PER_FRAME + i, &reg_value);
+		if (reg_value)
+			break;
+		if (ret) {
+			*status = V4L2_IN_ST_NO_POWER;
+			return ret;
+		}
+	}
+
+	/* If no register reports a video signal */
+	if (i >= 4)
+		*status |= V4L2_IN_ST_NO_SIGNAL;
+
+	ret = gs_read_register(gs->pdev, REG_STATUS, &reg_value);
+	if (!(reg_value & MASK_H_LOCK))
+		*status |=  V4L2_IN_ST_NO_H_LOCK;
+	if (!(reg_value & MASK_V_LOCK))
+		*status |=  V4L2_IN_ST_NO_V_LOCK;
+	if (!(reg_value & MASK_STD_LOCK))
+		*status |=  V4L2_IN_ST_NO_STD_LOCK;
+
+	return ret;
+}
+
+static int gs_dv_timings_cap(struct v4l2_subdev *sd,
+			     struct v4l2_dv_timings_cap *cap)
+{
+	if (cap->pad != 0)
+		return -EINVAL;
+
+	*cap = gs_timings_cap;
+	return 0;
+}
+
+/* V4L2 core operation handlers */
+static const struct v4l2_subdev_core_ops gs_core_ops = {
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+	.g_register = gs_g_register,
+	.s_register = gs_s_register,
+#endif
+};
+
+static const struct v4l2_subdev_video_ops gs_video_ops = {
+	.s_dv_timings = gs_s_dv_timings,
+	.g_dv_timings = gs_g_dv_timings,
+	.s_stream = gs_s_stream,
+	.g_input_status = gs_g_input_status,
+	.query_dv_timings = gs_query_dv_timings,
+};
+
+static const struct v4l2_subdev_pad_ops gs_pad_ops = {
+	.enum_dv_timings = gs_enum_dv_timings,
+	.dv_timings_cap = gs_dv_timings_cap,
+};
+
+/* V4L2 top level operation handlers */
+static const struct v4l2_subdev_ops gs_ops = {
+	.core = &gs_core_ops,
+	.video = &gs_video_ops,
+	.pad = &gs_pad_ops,
+};
+
+static int gs_probe(struct spi_device *spi)
+{
+	int ret;
+	struct gs *gs;
+	struct v4l2_subdev *sd;
+
+	gs = devm_kzalloc(&spi->dev, sizeof(struct gs), GFP_KERNEL);
+	if (!gs)
+		return -ENOMEM;
+
+	gs->pdev = spi;
+	sd = &gs->sd;
+
+	spi->mode = SPI_MODE_0;
+	spi->irq = -1;
+	spi->max_speed_hz = 10000000;
+	spi->bits_per_word = 16;
+	ret = spi_setup(spi);
+	v4l2_spi_subdev_init(sd, spi, &gs_ops);
+
+	gs->current_timings = reg_fmt[0].format;
+	gs->enabled = 0;
+
+	/* Set H_CONFIG to SMPTE timings */
+	gs_write_register(spi, 0x0, 0x300);
+
+	return ret;
+}
+
+static int gs_remove(struct spi_device *spi)
+{
+	struct v4l2_subdev *sd = spi_get_drvdata(spi);
+	struct gs *gs = to_gs(sd);
+
+	v4l2_device_unregister_subdev(sd);
+	kfree(gs);
+	return 0;
+}
+
+static struct spi_driver gs_driver = {
+	.driver = {
+		.name		= "gs1662",
+		.owner		= THIS_MODULE,
+	},
+
+	.probe		= gs_probe,
+	.remove		= gs_remove,
+	.id_table	= gs_id,
+};
+
+module_spi_driver(gs_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Charles-Antoine Couret <charles-antoine.couret@nexvision.fr>");
+MODULE_DESCRIPTION("Gennum GS1662 HD/SD-SDI Serializer driver");
diff --git a/drivers/media/tuners/fc0011.h b/drivers/media/tuners/fc0011.h
index 81bb568d6943..438cf897acd1 100644
--- a/drivers/media/tuners/fc0011.h
+++ b/drivers/media/tuners/fc0011.h
@@ -1,7 +1,6 @@
 #ifndef LINUX_FC0011_H_
 #define LINUX_FC0011_H_
 
-#include <linux/kconfig.h>
 #include "dvb_frontend.h"
 
 
diff --git a/drivers/media/tuners/fc0012.h b/drivers/media/tuners/fc0012.h
index 9ad32859bab0..4a23e418daf0 100644
--- a/drivers/media/tuners/fc0012.h
+++ b/drivers/media/tuners/fc0012.h
@@ -21,7 +21,6 @@
 #ifndef _FC0012_H_
 #define _FC0012_H_
 
-#include <linux/kconfig.h>
 #include "dvb_frontend.h"
 #include "fc001x-common.h"
 
diff --git a/drivers/media/tuners/fc0013.h b/drivers/media/tuners/fc0013.h
index e130bd7a3230..8c34105c9383 100644
--- a/drivers/media/tuners/fc0013.h
+++ b/drivers/media/tuners/fc0013.h
@@ -22,7 +22,6 @@
 #ifndef _FC0013_H_
 #define _FC0013_H_
 
-#include <linux/kconfig.h>
 #include "dvb_frontend.h"
 #include "fc001x-common.h"
 
diff --git a/drivers/media/tuners/max2165.h b/drivers/media/tuners/max2165.h
index 5054f01a78fb..aadd9fea59e4 100644
--- a/drivers/media/tuners/max2165.h
+++ b/drivers/media/tuners/max2165.h
@@ -22,8 +22,6 @@
 #ifndef __MAX2165_H__
 #define __MAX2165_H__
 
-#include <linux/kconfig.h>
-
 struct dvb_frontend;
 struct i2c_adapter;
 
diff --git a/drivers/media/tuners/mc44s803.h b/drivers/media/tuners/mc44s803.h
index b3e614be657d..6b40df339284 100644
--- a/drivers/media/tuners/mc44s803.h
+++ b/drivers/media/tuners/mc44s803.h
@@ -22,8 +22,6 @@
 #ifndef MC44S803_H
 #define MC44S803_H
 
-#include <linux/kconfig.h>
-
 struct dvb_frontend;
 struct i2c_adapter;
 
diff --git a/drivers/media/tuners/mt2063.c b/drivers/media/tuners/mt2063.c
index 7f0b9d5940db..dfec23743afe 100644
--- a/drivers/media/tuners/mt2063.c
+++ b/drivers/media/tuners/mt2063.c
@@ -2201,7 +2201,7 @@ static int mt2063_get_bandwidth(struct dvb_frontend *fe, u32 *bw)
 	return 0;
 }
 
-static struct dvb_tuner_ops mt2063_ops = {
+static const struct dvb_tuner_ops mt2063_ops = {
 	.info = {
 		 .name = "MT2063 Silicon Tuner",
 		 .frequency_min = 45000000,
diff --git a/drivers/media/tuners/mt20xx.c b/drivers/media/tuners/mt20xx.c
index 9e031040c13f..52da4671b0e0 100644
--- a/drivers/media/tuners/mt20xx.c
+++ b/drivers/media/tuners/mt20xx.c
@@ -363,7 +363,7 @@ static int mt2032_set_params(struct dvb_frontend *fe,
 	return ret;
 }
 
-static struct dvb_tuner_ops mt2032_tuner_ops = {
+static const struct dvb_tuner_ops mt2032_tuner_ops = {
 	.set_analog_params = mt2032_set_params,
 	.release           = microtune_release,
 	.get_frequency     = microtune_get_frequency,
@@ -563,7 +563,7 @@ static int mt2050_set_params(struct dvb_frontend *fe,
 	return ret;
 }
 
-static struct dvb_tuner_ops mt2050_tuner_ops = {
+static const struct dvb_tuner_ops mt2050_tuner_ops = {
 	.set_analog_params = mt2050_set_params,
 	.release           = microtune_release,
 	.get_frequency     = microtune_get_frequency,
diff --git a/drivers/media/tuners/mxl5005s.h b/drivers/media/tuners/mxl5005s.h
index 5764b12c5c7c..d842734f2dcd 100644
--- a/drivers/media/tuners/mxl5005s.h
+++ b/drivers/media/tuners/mxl5005s.h
@@ -23,8 +23,6 @@
 #ifndef __MXL5005S_H
 #define __MXL5005S_H
 
-#include <linux/kconfig.h>
-
 #include <linux/i2c.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/tuners/mxl5007t.c b/drivers/media/tuners/mxl5007t.c
index f4ae04c3328a..42569c6811e6 100644
--- a/drivers/media/tuners/mxl5007t.c
+++ b/drivers/media/tuners/mxl5007t.c
@@ -794,7 +794,7 @@ static int mxl5007t_release(struct dvb_frontend *fe)
 
 /* ------------------------------------------------------------------------- */
 
-static struct dvb_tuner_ops mxl5007t_tuner_ops = {
+static const struct dvb_tuner_ops mxl5007t_tuner_ops = {
 	.info = {
 		.name = "MaxLinear MxL5007T",
 	},
diff --git a/drivers/media/tuners/r820t.h b/drivers/media/tuners/r820t.h
index b1e5661af1c7..fdcab91405de 100644
--- a/drivers/media/tuners/r820t.h
+++ b/drivers/media/tuners/r820t.h
@@ -21,7 +21,6 @@
 #ifndef R820T_H
 #define R820T_H
 
-#include <linux/kconfig.h>
 #include "dvb_frontend.h"
 
 enum r820t_chip {
diff --git a/drivers/media/tuners/si2157.h b/drivers/media/tuners/si2157.h
index 5f1a60bf7ced..76807f5b3cf8 100644
--- a/drivers/media/tuners/si2157.h
+++ b/drivers/media/tuners/si2157.h
@@ -17,7 +17,6 @@
 #ifndef SI2157_H
 #define SI2157_H
 
-#include <linux/kconfig.h>
 #include <media/media-device.h>
 #include "dvb_frontend.h"
 
diff --git a/drivers/media/tuners/tda18212.h b/drivers/media/tuners/tda18212.h
index e58c9096d79c..6391dafd0c9d 100644
--- a/drivers/media/tuners/tda18212.h
+++ b/drivers/media/tuners/tda18212.h
@@ -21,7 +21,6 @@
 #ifndef TDA18212_H
 #define TDA18212_H
 
-#include <linux/kconfig.h>
 #include "dvb_frontend.h"
 
 struct tda18212_config {
diff --git a/drivers/media/tuners/tda18218.h b/drivers/media/tuners/tda18218.h
index 1eacb4f84e93..076b5f2e888d 100644
--- a/drivers/media/tuners/tda18218.h
+++ b/drivers/media/tuners/tda18218.h
@@ -21,7 +21,6 @@
 #ifndef TDA18218_H
 #define TDA18218_H
 
-#include <linux/kconfig.h>
 #include "dvb_frontend.h"
 
 struct tda18218_config {
diff --git a/drivers/media/tuners/tda18271-fe.c b/drivers/media/tuners/tda18271-fe.c
index f8620741bb5f..2d50e8b1dce1 100644
--- a/drivers/media/tuners/tda18271-fe.c
+++ b/drivers/media/tuners/tda18271-fe.c
@@ -18,11 +18,12 @@
     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
 
-#include <linux/delay.h>
-#include <linux/videodev2.h>
 #include "tda18271-priv.h"
 #include "tda8290.h"
 
+#include <linux/delay.h>
+#include <linux/videodev2.h>
+
 int tda18271_debug;
 module_param_named(debug, tda18271_debug, int, 0644);
 MODULE_PARM_DESC(debug, "set debug level "
@@ -646,7 +647,7 @@ static int tda18271_calc_rf_filter_curve(struct dvb_frontend *fe)
 	unsigned int i;
 	int ret;
 
-	tda_info("tda18271: performing RF tracking filter calibration\n");
+	tda_info("performing RF tracking filter calibration\n");
 
 	/* wait for die temperature stabilization */
 	msleep(200);
@@ -692,12 +693,12 @@ static int tda18271c2_rf_cal_init(struct dvb_frontend *fe)
 	if (tda_fail(ret))
 		goto fail;
 
-	tda_info("tda18271: RF tracking filter calibration complete\n");
+	tda_info("RF tracking filter calibration complete\n");
 
 	priv->cal_initialized = true;
 	goto end;
 fail:
-	tda_info("tda18271: RF tracking filter calibration failed!\n");
+	tda_info("RF tracking filter calibration failed!\n");
 end:
 	return ret;
 }
diff --git a/drivers/media/tuners/tda18271-priv.h b/drivers/media/tuners/tda18271-priv.h
index cc80f544af34..0bcc735a0427 100644
--- a/drivers/media/tuners/tda18271-priv.h
+++ b/drivers/media/tuners/tda18271-priv.h
@@ -21,6 +21,8 @@
 #ifndef __TDA18271_PRIV_H__
 #define __TDA18271_PRIV_H__
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/mutex.h>
diff --git a/drivers/media/tuners/tda827x.c b/drivers/media/tuners/tda827x.c
index edcb4a723aa1..5050ce9be423 100644
--- a/drivers/media/tuners/tda827x.c
+++ b/drivers/media/tuners/tda827x.c
@@ -818,7 +818,7 @@ static int tda827x_initial_sleep(struct dvb_frontend *fe)
 	return fe->ops.tuner_ops.sleep(fe);
 }
 
-static struct dvb_tuner_ops tda827xo_tuner_ops = {
+static const struct dvb_tuner_ops tda827xo_tuner_ops = {
 	.info = {
 		.name = "Philips TDA827X",
 		.frequency_min  =  55000000,
@@ -834,7 +834,7 @@ static struct dvb_tuner_ops tda827xo_tuner_ops = {
 	.get_bandwidth = tda827x_get_bandwidth,
 };
 
-static struct dvb_tuner_ops tda827xa_tuner_ops = {
+static const struct dvb_tuner_ops tda827xa_tuner_ops = {
 	.info = {
 		.name = "Philips TDA827XA",
 		.frequency_min  =  44000000,
diff --git a/drivers/media/tuners/tea5761.c b/drivers/media/tuners/tea5761.c
index bf78cb9fc52c..36b0b1e1d05b 100644
--- a/drivers/media/tuners/tea5761.c
+++ b/drivers/media/tuners/tea5761.c
@@ -301,7 +301,7 @@ static int tea5761_get_frequency(struct dvb_frontend *fe, u32 *frequency)
 	return 0;
 }
 
-static struct dvb_tuner_ops tea5761_tuner_ops = {
+static const struct dvb_tuner_ops tea5761_tuner_ops = {
 	.info = {
 		.name           = "tea5761", // Philips TEA5761HN FM Radio
 	},
diff --git a/drivers/media/tuners/tea5767.c b/drivers/media/tuners/tea5767.c
index 36e85d81acb2..d62a6d6b1f42 100644
--- a/drivers/media/tuners/tea5767.c
+++ b/drivers/media/tuners/tea5767.c
@@ -10,6 +10,8 @@
  * from their contributions on DScaler.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/i2c.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
@@ -370,17 +372,18 @@ int tea5767_autodetection(struct i2c_adapter* i2c_adap, u8 i2c_addr)
 {
 	struct tuner_i2c_props i2c = { .adap = i2c_adap, .addr = i2c_addr };
 	unsigned char buffer[7] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+
 	int rc;
 
 	if ((rc = tuner_i2c_xfer_recv(&i2c, buffer, 7))< 5) {
-		printk(KERN_WARNING "It is not a TEA5767. Received %i bytes.\n", rc);
+		pr_warn("It is not a TEA5767. Received %i bytes.\n", rc);
 		return -EINVAL;
 	}
 
 	/* If all bytes are the same then it's a TV tuner and not a tea5767 */
 	if (buffer[0] == buffer[1] && buffer[0] == buffer[2] &&
 	    buffer[0] == buffer[3] && buffer[0] == buffer[4]) {
-		printk(KERN_WARNING "All bytes are equal. It is not a TEA5767\n");
+		pr_warn("All bytes are equal. It is not a TEA5767\n");
 		return -EINVAL;
 	}
 
@@ -390,7 +393,7 @@ int tea5767_autodetection(struct i2c_adapter* i2c_adap, u8 i2c_addr)
 	 *  Byte 5: bit 7:0 : == 0
 	 */
 	if (((buffer[3] & 0x0f) != 0x00) || (buffer[4] != 0x00)) {
-		printk(KERN_WARNING "Chip ID is not zero. It is not a TEA5767\n");
+		pr_warn("Chip ID is not zero. It is not a TEA5767\n");
 		return -EINVAL;
 	}
 
@@ -423,7 +426,7 @@ static int tea5767_set_config (struct dvb_frontend *fe, void *priv_cfg)
 	return 0;
 }
 
-static struct dvb_tuner_ops tea5767_tuner_ops = {
+static const struct dvb_tuner_ops tea5767_tuner_ops = {
 	.info = {
 		.name           = "tea5767", // Philips TEA5767HN FM Radio
 	},
diff --git a/drivers/media/tuners/tuner-simple.c b/drivers/media/tuners/tuner-simple.c
index 8e9ce144da9a..9ba9582e7765 100644
--- a/drivers/media/tuners/tuner-simple.c
+++ b/drivers/media/tuners/tuner-simple.c
@@ -1035,7 +1035,7 @@ static int simple_get_bandwidth(struct dvb_frontend *fe, u32 *bandwidth)
 	return 0;
 }
 
-static struct dvb_tuner_ops simple_tuner_ops = {
+static const struct dvb_tuner_ops simple_tuner_ops = {
 	.init              = simple_init,
 	.sleep             = simple_sleep,
 	.set_analog_params = simple_set_params,
diff --git a/drivers/media/tuners/xc5000.h b/drivers/media/tuners/xc5000.h
index 00ba29e21fb9..336bd49eb09b 100644
--- a/drivers/media/tuners/xc5000.h
+++ b/drivers/media/tuners/xc5000.h
@@ -22,7 +22,6 @@
 #ifndef __XC5000_H__
 #define __XC5000_H__
 
-#include <linux/kconfig.h>
 #include <linux/firmware.h>
 
 struct dvb_frontend;
diff --git a/drivers/media/usb/airspy/airspy.c b/drivers/media/usb/airspy/airspy.c
index fe031b06935f..8251942bcd12 100644
--- a/drivers/media/usb/airspy/airspy.c
+++ b/drivers/media/usb/airspy/airspy.c
@@ -426,7 +426,6 @@ static int airspy_alloc_urbs(struct airspy *s)
 		dev_dbg(s->dev, "alloc urb=%d\n", i);
 		s->urb_list[i] = usb_alloc_urb(0, GFP_ATOMIC);
 		if (!s->urb_list[i]) {
-			dev_dbg(s->dev, "failed\n");
 			for (j = 0; j < i; j++)
 				usb_free_urb(s->urb_list[j]);
 			return -ENOMEM;
@@ -606,7 +605,7 @@ static void airspy_stop_streaming(struct vb2_queue *vq)
 	mutex_unlock(&s->v4l2_lock);
 }
 
-static struct vb2_ops airspy_vb2_ops = {
+static const struct vb2_ops airspy_vb2_ops = {
 	.queue_setup            = airspy_queue_setup,
 	.buf_queue              = airspy_buf_queue,
 	.start_streaming        = airspy_start_streaming,
diff --git a/drivers/media/usb/as102/as102_usb_drv.c b/drivers/media/usb/as102/as102_usb_drv.c
index 0e8030c071b8..68c3a80ce349 100644
--- a/drivers/media/usb/as102/as102_usb_drv.c
+++ b/drivers/media/usb/as102/as102_usb_drv.c
@@ -270,8 +270,6 @@ static int as102_alloc_usb_stream_buffer(struct as102_dev_t *dev)
 
 		urb = usb_alloc_urb(0, GFP_ATOMIC);
 		if (urb == NULL) {
-			dev_dbg(&dev->bus_adap.usb_dev->dev,
-				"%s: usb_alloc_urb failed\n", __func__);
 			as102_free_usb_stream_buffer(dev);
 			return -ENOMEM;
 		}
diff --git a/drivers/media/usb/au0828/au0828-input.c b/drivers/media/usb/au0828/au0828-input.c
index 3d6687f0407d..1e66e7828d8f 100644
--- a/drivers/media/usb/au0828/au0828-input.c
+++ b/drivers/media/usb/au0828/au0828-input.c
@@ -344,7 +344,8 @@ int au0828_rc_register(struct au0828_dev *dev)
 	rc->dev.parent = &dev->usbdev->dev;
 	rc->driver_name = "au0828-input";
 	rc->driver_type = RC_DRIVER_IR_RAW;
-	rc->allowed_protocols = RC_BIT_NEC | RC_BIT_RC5;
+	rc->allowed_protocols = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32 |
+								RC_BIT_RC5;
 
 	/* all done */
 	err = rc_register_device(rc);
diff --git a/drivers/media/usb/au0828/au0828-video.c b/drivers/media/usb/au0828/au0828-video.c
index 82b026985868..85dd9a8e83ff 100644
--- a/drivers/media/usb/au0828/au0828-video.c
+++ b/drivers/media/usb/au0828/au0828-video.c
@@ -245,7 +245,6 @@ static int au0828_init_isoc(struct au0828_dev *dev, int max_packets,
 	for (i = 0; i < dev->isoc_ctl.num_bufs; i++) {
 		urb = usb_alloc_urb(max_packets, GFP_KERNEL);
 		if (!urb) {
-			au0828_isocdbg("cannot alloc isoc_ctl.urb %i\n", i);
 			au0828_uninit_isoc(dev);
 			return -ENOMEM;
 		}
@@ -929,7 +928,7 @@ void au0828_stop_vbi_streaming(struct vb2_queue *vq)
 	del_timer_sync(&dev->vbi_timeout);
 }
 
-static struct vb2_ops au0828_video_qops = {
+static const struct vb2_ops au0828_video_qops = {
 	.queue_setup     = queue_setup,
 	.buf_prepare     = buffer_prepare,
 	.buf_queue       = buffer_queue,
diff --git a/drivers/media/usb/cpia2/cpia2_usb.c b/drivers/media/usb/cpia2/cpia2_usb.c
index c1aa1ab2ece9..13620cdf0599 100644
--- a/drivers/media/usb/cpia2/cpia2_usb.c
+++ b/drivers/media/usb/cpia2/cpia2_usb.c
@@ -662,7 +662,6 @@ static int submit_urbs(struct camera_data *cam)
 		}
 		urb = usb_alloc_urb(FRAMES_PER_DESC, GFP_KERNEL);
 		if (!urb) {
-			ERR("%s: usb_alloc_urb error!\n", __func__);
 			for (j = 0; j < i; j++)
 				usb_free_urb(cam->sbuf[j].urb);
 			return -ENOMEM;
diff --git a/drivers/media/usb/cx231xx/cx231xx-audio.c b/drivers/media/usb/cx231xx/cx231xx-audio.c
index a6a9508418f8..8263c4b0610b 100644
--- a/drivers/media/usb/cx231xx/cx231xx-audio.c
+++ b/drivers/media/usb/cx231xx/cx231xx-audio.c
@@ -293,7 +293,6 @@ static int cx231xx_init_audio_isoc(struct cx231xx *dev)
 		memset(dev->adev.transfer_buffer[i], 0x80, sb_size);
 		urb = usb_alloc_urb(CX231XX_ISO_NUM_AUDIO_PACKETS, GFP_ATOMIC);
 		if (!urb) {
-			dev_err(dev->dev, "usb_alloc_urb failed!\n");
 			for (j = 0; j < i; j++) {
 				usb_free_urb(dev->adev.urb[j]);
 				kfree(dev->adev.transfer_buffer[j]);
@@ -355,7 +354,6 @@ static int cx231xx_init_audio_bulk(struct cx231xx *dev)
 		memset(dev->adev.transfer_buffer[i], 0x80, sb_size);
 		urb = usb_alloc_urb(CX231XX_NUM_AUDIO_PACKETS, GFP_ATOMIC);
 		if (!urb) {
-			dev_err(dev->dev, "usb_alloc_urb failed!\n");
 			for (j = 0; j < i; j++) {
 				usb_free_urb(dev->adev.urb[j]);
 				kfree(dev->adev.transfer_buffer[j]);
@@ -637,7 +635,7 @@ static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs,
 	return vmalloc_to_page(pageptr);
 }
 
-static struct snd_pcm_ops snd_cx231xx_pcm_capture = {
+static const struct snd_pcm_ops snd_cx231xx_pcm_capture = {
 	.open = snd_cx231xx_capture_open,
 	.close = snd_cx231xx_pcm_close,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/media/usb/cx231xx/cx231xx-avcore.c b/drivers/media/usb/cx231xx/cx231xx-avcore.c
index 491913778bcc..2f52d66b4dae 100644
--- a/drivers/media/usb/cx231xx/cx231xx-avcore.c
+++ b/drivers/media/usb/cx231xx/cx231xx-avcore.c
@@ -1264,7 +1264,10 @@ int cx231xx_set_agc_analog_digital_mux_select(struct cx231xx *dev,
 				   dev->board.agc_analog_digital_select_gpio,
 				   analog_or_digital);
 
-	return status;
+	if (status < 0)
+		return status;
+
+	return 0;
 }
 
 int cx231xx_enable_i2c_port_3(struct cx231xx *dev, bool is_port_3)
diff --git a/drivers/media/usb/cx231xx/cx231xx-cards.c b/drivers/media/usb/cx231xx/cx231xx-cards.c
index c63248a18823..36bc25494319 100644
--- a/drivers/media/usb/cx231xx/cx231xx-cards.c
+++ b/drivers/media/usb/cx231xx/cx231xx-cards.c
@@ -486,7 +486,7 @@ struct cx231xx_board cx231xx_boards[] = {
 		.output_mode = OUT_MODE_VIP11,
 		.demod_xfer_mode = 0,
 		.ctl_pin_status_mask = 0xFFFFFFC4,
-		.agc_analog_digital_select_gpio = 0x00,	/* According with PV cxPolaris.inf file */
+		.agc_analog_digital_select_gpio = 0x1c,
 		.tuner_sif_gpio = -1,
 		.tuner_scl_gpio = -1,
 		.tuner_sda_gpio = -1,
@@ -1186,12 +1186,12 @@ static void cx231xx_unregister_media_device(struct cx231xx *dev)
 */
 void cx231xx_release_resources(struct cx231xx *dev)
 {
+	cx231xx_ir_exit(dev);
+
 	cx231xx_release_analog_resources(dev);
 
 	cx231xx_remove_from_devlist(dev);
 
-	cx231xx_ir_exit(dev);
-
 	/* Release I2C buses */
 	cx231xx_dev_uninit(dev);
 
diff --git a/drivers/media/usb/cx231xx/cx231xx-core.c b/drivers/media/usb/cx231xx/cx231xx-core.c
index 630f4fc5155f..8b099fe1d592 100644
--- a/drivers/media/usb/cx231xx/cx231xx-core.c
+++ b/drivers/media/usb/cx231xx/cx231xx-core.c
@@ -712,6 +712,7 @@ int cx231xx_set_mode(struct cx231xx *dev, enum cx231xx_mode set_mode)
 			break;
 		case CX231XX_BOARD_CNXT_RDE_253S:
 		case CX231XX_BOARD_CNXT_RDU_253S:
+		case CX231XX_BOARD_PV_PLAYTV_USB_HYBRID:
 			errCode = cx231xx_set_agc_analog_digital_mux_select(dev, 1);
 			break;
 		case CX231XX_BOARD_HAUPPAUGE_EXETER:
@@ -738,14 +739,21 @@ int cx231xx_set_mode(struct cx231xx *dev, enum cx231xx_mode set_mode)
 		case CX231XX_BOARD_PV_PLAYTV_USB_HYBRID:
 		case CX231XX_BOARD_HAUPPAUGE_USB2_FM_PAL:
 		case CX231XX_BOARD_HAUPPAUGE_USB2_FM_NTSC:
-		errCode = cx231xx_set_agc_analog_digital_mux_select(dev, 0);
+			errCode = cx231xx_set_agc_analog_digital_mux_select(dev, 0);
 			break;
 		default:
 			break;
 		}
 	}
 
-	return errCode ? -EINVAL : 0;
+	if (errCode < 0) {
+		dev_err(dev->dev, "Failed to set devmode to %s: error: %i",
+			dev->mode == CX231XX_DIGITAL_MODE ? "digital" : "analog",
+			errCode);
+		return errCode;
+	}
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(cx231xx_set_mode);
 
@@ -799,7 +807,7 @@ static void cx231xx_isoc_irq_callback(struct urb *urb)
 	case -ESHUTDOWN:
 		return;
 	default:		/* error */
-		cx231xx_isocdbg("urb completition error %d.\n", urb->status);
+		cx231xx_isocdbg("urb completion error %d.\n", urb->status);
 		break;
 	}
 
@@ -842,8 +850,11 @@ static void cx231xx_bulk_irq_callback(struct urb *urb)
 	case -ENOENT:
 	case -ESHUTDOWN:
 		return;
+	case -EPIPE:		/* stall */
+		cx231xx_isocdbg("urb completion error - device is stalled.\n");
+		return;
 	default:		/* error */
-		cx231xx_isocdbg("urb completition error %d.\n", urb->status);
+		cx231xx_isocdbg("urb completion error %d.\n", urb->status);
 		break;
 	}
 
@@ -867,6 +878,7 @@ void cx231xx_uninit_isoc(struct cx231xx *dev)
 	struct cx231xx_dmaqueue *dma_q = &dev->video_mode.vidq;
 	struct urb *urb;
 	int i;
+	bool broken_pipe = false;
 
 	cx231xx_isocdbg("cx231xx: called cx231xx_uninit_isoc\n");
 
@@ -886,12 +898,19 @@ void cx231xx_uninit_isoc(struct cx231xx *dev)
 						  transfer_buffer[i],
 						  urb->transfer_dma);
 			}
+			if (urb->status == -EPIPE) {
+				broken_pipe = true;
+			}
 			usb_free_urb(urb);
 			dev->video_mode.isoc_ctl.urb[i] = NULL;
 		}
 		dev->video_mode.isoc_ctl.transfer_buffer[i] = NULL;
 	}
 
+	if (broken_pipe) {
+		cx231xx_isocdbg("Reset endpoint to recover broken pipe.");
+		usb_reset_endpoint(dev->udev, dev->video_mode.end_point_addr);
+	}
 	kfree(dev->video_mode.isoc_ctl.urb);
 	kfree(dev->video_mode.isoc_ctl.transfer_buffer);
 	kfree(dma_q->p_left_data);
@@ -918,6 +937,7 @@ void cx231xx_uninit_bulk(struct cx231xx *dev)
 	struct cx231xx_dmaqueue *dma_q = &dev->video_mode.vidq;
 	struct urb *urb;
 	int i;
+	bool broken_pipe = false;
 
 	cx231xx_isocdbg("cx231xx: called cx231xx_uninit_bulk\n");
 
@@ -937,12 +957,19 @@ void cx231xx_uninit_bulk(struct cx231xx *dev)
 						transfer_buffer[i],
 						urb->transfer_dma);
 			}
+			if (urb->status == -EPIPE) {
+				broken_pipe = true;
+			}
 			usb_free_urb(urb);
 			dev->video_mode.bulk_ctl.urb[i] = NULL;
 		}
 		dev->video_mode.bulk_ctl.transfer_buffer[i] = NULL;
 	}
 
+	if (broken_pipe) {
+		cx231xx_isocdbg("Reset endpoint to recover broken pipe.");
+		usb_reset_endpoint(dev->udev, dev->video_mode.end_point_addr);
+	}
 	kfree(dev->video_mode.bulk_ctl.urb);
 	kfree(dev->video_mode.bulk_ctl.transfer_buffer);
 	kfree(dma_q->p_left_data);
@@ -1035,8 +1062,6 @@ int cx231xx_init_isoc(struct cx231xx *dev, int max_packets,
 	for (i = 0; i < dev->video_mode.isoc_ctl.num_bufs; i++) {
 		urb = usb_alloc_urb(max_packets, GFP_KERNEL);
 		if (!urb) {
-			dev_err(dev->dev,
-				"cannot alloc isoc_ctl.urb %i\n", i);
 			cx231xx_uninit_isoc(dev);
 			return -ENOMEM;
 		}
@@ -1172,8 +1197,6 @@ int cx231xx_init_bulk(struct cx231xx *dev, int max_packets,
 	for (i = 0; i < dev->video_mode.bulk_ctl.num_bufs; i++) {
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
-			dev_err(dev->dev,
-				"cannot alloc bulk_ctl.urb %i\n", i);
 			cx231xx_uninit_bulk(dev);
 			return -ENOMEM;
 		}
@@ -1301,15 +1324,29 @@ int cx231xx_dev_init(struct cx231xx *dev)
 	dev->i2c_bus[2].i2c_reserve = 0;
 
 	/* register I2C buses */
-	cx231xx_i2c_register(&dev->i2c_bus[0]);
-	cx231xx_i2c_register(&dev->i2c_bus[1]);
-	cx231xx_i2c_register(&dev->i2c_bus[2]);
+	errCode = cx231xx_i2c_register(&dev->i2c_bus[0]);
+	if (errCode < 0)
+		return errCode;
+	errCode = cx231xx_i2c_register(&dev->i2c_bus[1]);
+	if (errCode < 0)
+		return errCode;
+	errCode = cx231xx_i2c_register(&dev->i2c_bus[2]);
+	if (errCode < 0)
+		return errCode;
 
 	errCode = cx231xx_i2c_mux_create(dev);
+	if (errCode < 0) {
+		dev_err(dev->dev,
+			"%s: Failed to create I2C mux\n", __func__);
+		return errCode;
+	}
+	errCode = cx231xx_i2c_mux_register(dev, 0);
+	if (errCode < 0)
+		return errCode;
+
+	errCode = cx231xx_i2c_mux_register(dev, 1);
 	if (errCode < 0)
 		return errCode;
-	cx231xx_i2c_mux_register(dev, 0);
-	cx231xx_i2c_mux_register(dev, 1);
 
 	/* scan the real bus segments in the order of physical port numbers */
 	cx231xx_do_i2c_scan(dev, I2C_0);
@@ -1452,14 +1489,14 @@ int cx231xx_send_gpio_cmd(struct cx231xx *dev, u32 gpio_bit, u8 *gpio_val,
 	/* set request */
 	if (!request) {
 		if (direction)
-			ven_req.bRequest = VRT_GET_GPIO;	/* 0x8 gpio */
+			ven_req.bRequest = VRT_GET_GPIO;	/* 0x9 gpio */
 		else
-			ven_req.bRequest = VRT_SET_GPIO;	/* 0x9 gpio */
+			ven_req.bRequest = VRT_SET_GPIO;	/* 0x8 gpio */
 	} else {
 		if (direction)
-			ven_req.bRequest = VRT_GET_GPIE;	/* 0xa gpie */
+			ven_req.bRequest = VRT_GET_GPIE;	/* 0xb gpie */
 		else
-			ven_req.bRequest = VRT_SET_GPIE;	/* 0xb gpie */
+			ven_req.bRequest = VRT_SET_GPIE;	/* 0xa gpie */
 	}
 
 	/* set index value */
diff --git a/drivers/media/usb/cx231xx/cx231xx-dvb.c b/drivers/media/usb/cx231xx/cx231xx-dvb.c
index ab2fb9fa0cd1..1417515d30eb 100644
--- a/drivers/media/usb/cx231xx/cx231xx-dvb.c
+++ b/drivers/media/usb/cx231xx/cx231xx-dvb.c
@@ -65,6 +65,7 @@ struct cx231xx_dvb {
 	struct dmx_frontend fe_hw;
 	struct dmx_frontend fe_mem;
 	struct dvb_net net;
+	struct i2c_client *i2c_client_demod;
 	struct i2c_client *i2c_client_tuner;
 };
 
@@ -150,18 +151,6 @@ static struct tda18271_config pv_tda18271_config = {
 	.small_i2c = TDA18271_03_BYTE_CHUNK_INIT,
 };
 
-static const struct si2165_config hauppauge_930C_HD_1113xx_si2165_config = {
-	.i2c_addr	= 0x64,
-	.chip_mode	= SI2165_MODE_PLL_XTAL,
-	.ref_freq_Hz	= 16000000,
-};
-
-static const struct si2165_config pctv_quatro_stick_1114xx_si2165_config = {
-	.i2c_addr	= 0x64,
-	.chip_mode	= SI2165_MODE_PLL_EXT,
-	.ref_freq_Hz	= 24000000,
-};
-
 static struct lgdt3306a_config hauppauge_955q_lgdt3306a_config = {
 	.i2c_addr           = 0x59,
 	.qam_if_khz         = 4000,
@@ -586,8 +575,14 @@ static void unregister_dvb(struct cx231xx_dvb *dvb)
 	dvb->demux.dmx.remove_frontend(&dvb->demux.dmx, &dvb->fe_hw);
 	dvb_dmxdev_release(&dvb->dmxdev);
 	dvb_dmx_release(&dvb->demux);
-	client = dvb->i2c_client_tuner;
 	/* remove I2C tuner */
+	client = dvb->i2c_client_tuner;
+	if (client) {
+		module_put(client->dev.driver->owner);
+		i2c_unregister_device(client);
+	}
+	/* remove I2C demod */
+	client = dvb->i2c_client_demod;
 	if (client) {
 		module_put(client->dev.driver->owner);
 		i2c_unregister_device(client);
@@ -749,19 +744,38 @@ static int dvb_init(struct cx231xx *dev)
 		break;
 
 	case CX231XX_BOARD_HAUPPAUGE_930C_HD_1113xx:
+	{
+		struct i2c_client *client;
+		struct i2c_board_info info;
+		struct si2165_platform_data si2165_pdata;
 
-		dev->dvb->frontend = dvb_attach(si2165_attach,
-			&hauppauge_930C_HD_1113xx_si2165_config,
-			demod_i2c
-			);
+		/* attach demod */
+		memset(&si2165_pdata, 0, sizeof(si2165_pdata));
+		si2165_pdata.fe = &dev->dvb->frontend;
+		si2165_pdata.chip_mode = SI2165_MODE_PLL_XTAL,
+		si2165_pdata.ref_freq_Hz = 16000000,
 
-		if (dev->dvb->frontend == NULL) {
+		memset(&info, 0, sizeof(struct i2c_board_info));
+		strlcpy(info.type, "si2165", I2C_NAME_SIZE);
+		info.addr = 0x64;
+		info.platform_data = &si2165_pdata;
+		request_module(info.type);
+		client = i2c_new_device(demod_i2c, &info);
+		if (client == NULL || client->dev.driver == NULL || dev->dvb->frontend == NULL) {
 			dev_err(dev->dev,
 				"Failed to attach SI2165 front end\n");
 			result = -EINVAL;
 			goto out_free;
 		}
 
+		if (!try_module_get(client->dev.driver->owner)) {
+			i2c_unregister_device(client);
+			result = -ENODEV;
+			goto out_free;
+		}
+
+		dvb->i2c_client_demod = client;
+
 		dev->dvb->frontend->ops.i2c_gate_ctrl = NULL;
 
 		/* define general-purpose callback pointer */
@@ -774,27 +788,43 @@ static int dvb_init(struct cx231xx *dev)
 
 		dev->cx231xx_reset_analog_tuner = NULL;
 		break;
-
+	}
 	case CX231XX_BOARD_HAUPPAUGE_930C_HD_1114xx:
 	{
 		struct i2c_client *client;
 		struct i2c_board_info info;
+		struct si2165_platform_data si2165_pdata;
 		struct si2157_config si2157_config;
 
-		memset(&info, 0, sizeof(struct i2c_board_info));
+		/* attach demod */
+		memset(&si2165_pdata, 0, sizeof(si2165_pdata));
+		si2165_pdata.fe = &dev->dvb->frontend;
+		si2165_pdata.chip_mode = SI2165_MODE_PLL_EXT,
+		si2165_pdata.ref_freq_Hz = 24000000,
 
-		dev->dvb->frontend = dvb_attach(si2165_attach,
-			&pctv_quatro_stick_1114xx_si2165_config,
-			demod_i2c
-			);
-
-		if (dev->dvb->frontend == NULL) {
+		memset(&info, 0, sizeof(struct i2c_board_info));
+		strlcpy(info.type, "si2165", I2C_NAME_SIZE);
+		info.addr = 0x64;
+		info.platform_data = &si2165_pdata;
+		request_module(info.type);
+		client = i2c_new_device(demod_i2c, &info);
+		if (client == NULL || client->dev.driver == NULL || dev->dvb->frontend == NULL) {
 			dev_err(dev->dev,
 				"Failed to attach SI2165 front end\n");
 			result = -EINVAL;
 			goto out_free;
 		}
 
+		if (!try_module_get(client->dev.driver->owner)) {
+			i2c_unregister_device(client);
+			result = -ENODEV;
+			goto out_free;
+		}
+
+		dvb->i2c_client_demod = client;
+
+		memset(&info, 0, sizeof(struct i2c_board_info));
+
 		dev->dvb->frontend->ops.i2c_gate_ctrl = NULL;
 
 		/* define general-purpose callback pointer */
diff --git a/drivers/media/usb/cx231xx/cx231xx-i2c.c b/drivers/media/usb/cx231xx/cx231xx-i2c.c
index 473cd3433fe5..35e9acfe63d3 100644
--- a/drivers/media/usb/cx231xx/cx231xx-i2c.c
+++ b/drivers/media/usb/cx231xx/cx231xx-i2c.c
@@ -454,7 +454,7 @@ static u32 functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_SMBUS_EMUL | I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm cx231xx_algo = {
+static const struct i2c_algorithm cx231xx_algo = {
 	.master_xfer = cx231xx_i2c_xfer,
 	.functionality = functionality,
 };
@@ -608,7 +608,7 @@ struct i2c_adapter *cx231xx_get_i2c_adap(struct cx231xx *dev, int i2c_port)
 	case I2C_1_MUX_3:
 		return dev->muxc->adapter[1];
 	default:
-		return NULL;
+		BUG();
 	}
 }
 EXPORT_SYMBOL_GPL(cx231xx_get_i2c_adap);
diff --git a/drivers/media/usb/cx231xx/cx231xx-vbi.c b/drivers/media/usb/cx231xx/cx231xx-vbi.c
index 15bb573b78ac..76e901920f6f 100644
--- a/drivers/media/usb/cx231xx/cx231xx-vbi.c
+++ b/drivers/media/usb/cx231xx/cx231xx-vbi.c
@@ -442,8 +442,6 @@ int cx231xx_init_vbi_isoc(struct cx231xx *dev, int max_packets,
 
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
-			dev_err(dev->dev,
-				"cannot alloc bulk_ctl.urb %i\n", i);
 			cx231xx_uninit_vbi_isoc(dev);
 			return -ENOMEM;
 		}
diff --git a/drivers/media/usb/dvb-usb-v2/af9015.c b/drivers/media/usb/dvb-usb-v2/af9015.c
index 09e0f58f6bb7..941ceff9b268 100644
--- a/drivers/media/usb/dvb-usb-v2/af9015.c
+++ b/drivers/media/usb/dvb-usb-v2/af9015.c
@@ -1222,6 +1222,7 @@ static int af9015_rc_query(struct dvb_usb_device *d)
 
 	/* Only process key if canary killed */
 	if (buf[16] != 0xff && buf[0] != 0x01) {
+		enum rc_type proto;
 		dev_dbg(&d->udev->dev, "%s: key pressed %*ph\n",
 				__func__, 4, buf + 12);
 
@@ -1237,11 +1238,13 @@ static int af9015_rc_query(struct dvb_usb_device *d)
 				/* NEC */
 				state->rc_keycode = RC_SCANCODE_NEC(buf[12],
 								    buf[14]);
+				proto = RC_TYPE_NEC;
 			} else {
 				/* NEC extended*/
 				state->rc_keycode = RC_SCANCODE_NECX(buf[12] << 8 |
 								     buf[13],
 								     buf[14]);
+				proto = RC_TYPE_NECX;
 			}
 		} else {
 			/* 32 bit NEC */
@@ -1249,8 +1252,9 @@ static int af9015_rc_query(struct dvb_usb_device *d)
 							      buf[13] << 16 |
 							      buf[14] << 8  |
 							      buf[15]);
+			proto = RC_TYPE_NEC32;
 		}
-		rc_keydown(d->rc_dev, RC_TYPE_NEC, state->rc_keycode, 0);
+		rc_keydown(d->rc_dev, proto, state->rc_keycode, 0);
 	} else {
 		dev_dbg(&d->udev->dev, "%s: no key press\n", __func__);
 		/* Invalidate last keypress */
@@ -1317,7 +1321,7 @@ static int af9015_get_rc_config(struct dvb_usb_device *d, struct dvb_usb_rc *rc)
 	if (!rc->map_name)
 		rc->map_name = RC_MAP_EMPTY;
 
-	rc->allowed_protos = RC_BIT_NEC;
+	rc->allowed_protos = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32;
 	rc->query = af9015_rc_query;
 	rc->interval = 500;
 
diff --git a/drivers/media/usb/dvb-usb-v2/af9035.c b/drivers/media/usb/dvb-usb-v2/af9035.c
index ca018cd3fcd4..8961dd732522 100644
--- a/drivers/media/usb/dvb-usb-v2/af9035.c
+++ b/drivers/media/usb/dvb-usb-v2/af9035.c
@@ -1828,6 +1828,7 @@ static int af9035_rc_query(struct dvb_usb_device *d)
 {
 	struct usb_interface *intf = d->intf;
 	int ret;
+	enum rc_type proto;
 	u32 key;
 	u8 buf[4];
 	struct usb_req req = { CMD_IR_GET, 0, 0, NULL, 4, buf };
@@ -1842,19 +1843,22 @@ static int af9035_rc_query(struct dvb_usb_device *d)
 		if ((buf[0] + buf[1]) == 0xff) {
 			/* NEC standard 16bit */
 			key = RC_SCANCODE_NEC(buf[0], buf[2]);
+			proto = RC_TYPE_NEC;
 		} else {
 			/* NEC extended 24bit */
 			key = RC_SCANCODE_NECX(buf[0] << 8 | buf[1], buf[2]);
+			proto = RC_TYPE_NECX;
 		}
 	} else {
 		/* NEC full code 32bit */
 		key = RC_SCANCODE_NEC32(buf[0] << 24 | buf[1] << 16 |
 					buf[2] << 8  | buf[3]);
+		proto = RC_TYPE_NEC32;
 	}
 
 	dev_dbg(&intf->dev, "%*ph\n", 4, buf);
 
-	rc_keydown(d->rc_dev, RC_TYPE_NEC, key, 0);
+	rc_keydown(d->rc_dev, proto, key, 0);
 
 	return 0;
 
@@ -1889,7 +1893,8 @@ static int af9035_get_rc_config(struct dvb_usb_device *d, struct dvb_usb_rc *rc)
 		switch (tmp) {
 		case 0: /* NEC */
 		default:
-			rc->allowed_protos = RC_BIT_NEC;
+			rc->allowed_protos = RC_BIT_NEC | RC_BIT_NECX |
+								RC_BIT_NEC32;
 			break;
 		case 1: /* RC6 */
 			rc->allowed_protos = RC_BIT_RC6_MCE;
diff --git a/drivers/media/usb/dvb-usb-v2/az6007.c b/drivers/media/usb/dvb-usb-v2/az6007.c
index 935dbaa80ef0..50c07fe7dacb 100644
--- a/drivers/media/usb/dvb-usb-v2/az6007.c
+++ b/drivers/media/usb/dvb-usb-v2/az6007.c
@@ -208,6 +208,7 @@ static int az6007_rc_query(struct dvb_usb_device *d)
 {
 	struct az6007_device_state *st = d_to_priv(d);
 	unsigned code;
+	enum rc_type proto;
 
 	az6007_read(d, AZ6007_READ_IR, 0, 0, st->data, 10);
 
@@ -215,19 +216,23 @@ static int az6007_rc_query(struct dvb_usb_device *d)
 		return 0;
 
 	if ((st->data[3] ^ st->data[4]) == 0xff) {
-		if ((st->data[1] ^ st->data[2]) == 0xff)
+		if ((st->data[1] ^ st->data[2]) == 0xff) {
 			code = RC_SCANCODE_NEC(st->data[1], st->data[3]);
-		else
+			proto = RC_TYPE_NEC;
+		} else {
 			code = RC_SCANCODE_NECX(st->data[1] << 8 | st->data[2],
 						st->data[3]);
+			proto = RC_TYPE_NECX;
+		}
 	} else {
 		code = RC_SCANCODE_NEC32(st->data[1] << 24 |
 					 st->data[2] << 16 |
 					 st->data[3] << 8  |
 					 st->data[4]);
+		proto = RC_TYPE_NEC32;
 	}
 
-	rc_keydown(d->rc_dev, RC_TYPE_NEC, code, st->data[5]);
+	rc_keydown(d->rc_dev, proto, code, st->data[5]);
 
 	return 0;
 }
@@ -236,7 +241,7 @@ static int az6007_get_rc_config(struct dvb_usb_device *d, struct dvb_usb_rc *rc)
 {
 	pr_debug("Getting az6007 Remote Control properties\n");
 
-	rc->allowed_protos = RC_BIT_NEC;
+	rc->allowed_protos = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32;
 	rc->query          = az6007_rc_query;
 	rc->interval       = 400;
 
diff --git a/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c b/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c
index 3fbb2cd19f5e..a8e6624fbe83 100644
--- a/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c
+++ b/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c
@@ -82,8 +82,6 @@ static int dvb_usbv2_i2c_init(struct dvb_usb_device *d)
 	ret = i2c_add_adapter(&d->i2c_adap);
 	if (ret < 0) {
 		d->i2c_adap.algo = NULL;
-		dev_err(&d->udev->dev, "%s: i2c_add_adapter() failed=%d\n",
-				KBUILD_MODNAME, ret);
 		goto err;
 	}
 
diff --git a/drivers/media/usb/dvb-usb-v2/lmedm04.c b/drivers/media/usb/dvb-usb-v2/lmedm04.c
index 3721ee63b8fb..0e8fb89896c4 100644
--- a/drivers/media/usb/dvb-usb-v2/lmedm04.c
+++ b/drivers/media/usb/dvb-usb-v2/lmedm04.c
@@ -357,7 +357,8 @@ static void lme2510_int_response(struct urb *lme_urb)
 						ibuf[5]);
 
 			deb_info(1, "INT Key = 0x%08x", key);
-			rc_keydown(adap_to_d(adap)->rc_dev, RC_TYPE_NEC, key, 0);
+			rc_keydown(adap_to_d(adap)->rc_dev, RC_TYPE_NEC32, key,
+									0);
 			break;
 		case 0xbb:
 			switch (st->tuner_config) {
@@ -1242,7 +1243,7 @@ static int lme2510_get_stream_config(struct dvb_frontend *fe, u8 *ts_type,
 static int lme2510_get_rc_config(struct dvb_usb_device *d,
 	struct dvb_usb_rc *rc)
 {
-	rc->allowed_protos = RC_BIT_NEC;
+	rc->allowed_protos = RC_BIT_NEC32;
 	return 0;
 }
 
diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.h b/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.h
index 7065aca81252..e6eae9d88e9f 100644
--- a/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.h
+++ b/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.h
@@ -21,7 +21,6 @@
 #ifndef __MXL111SF_DEMOD_H__
 #define __MXL111SF_DEMOD_H__
 
-#include <linux/kconfig.h>
 #include "dvb_frontend.h"
 #include "mxl111sf.h"
 
diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.c b/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.c
index 7d16252dbb71..f141dcc55cc9 100644
--- a/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.c
+++ b/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.c
@@ -466,7 +466,7 @@ static int mxl111sf_tuner_release(struct dvb_frontend *fe)
 
 /* ------------------------------------------------------------------------- */
 
-static struct dvb_tuner_ops mxl111sf_tuner_tuner_ops = {
+static const struct dvb_tuner_ops mxl111sf_tuner_tuner_ops = {
 	.info = {
 		.name = "MaxLinear MxL111SF",
 #if 0
diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.h b/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.h
index 509b55071218..e96d9a444ed1 100644
--- a/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.h
+++ b/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.h
@@ -21,7 +21,6 @@
 #ifndef __MXL111SF_TUNER_H__
 #define __MXL111SF_TUNER_H__
 
-#include <linux/kconfig.h>
 #include "dvb_frontend.h"
 #include "mxl111sf.h"
 
diff --git a/drivers/media/usb/dvb-usb-v2/rtl28xxu.c b/drivers/media/usb/dvb-usb-v2/rtl28xxu.c
index 6643762a9ff7..c583c638e468 100644
--- a/drivers/media/usb/dvb-usb-v2/rtl28xxu.c
+++ b/drivers/media/usb/dvb-usb-v2/rtl28xxu.c
@@ -1631,22 +1631,27 @@ static int rtl2831u_rc_query(struct dvb_usb_device *d)
 		goto err;
 
 	if (buf[4] & 0x01) {
+		enum rc_type proto;
+
 		if (buf[2] == (u8) ~buf[3]) {
 			if (buf[0] == (u8) ~buf[1]) {
 				/* NEC standard (16 bit) */
 				rc_code = RC_SCANCODE_NEC(buf[0], buf[2]);
+				proto = RC_TYPE_NEC;
 			} else {
 				/* NEC extended (24 bit) */
 				rc_code = RC_SCANCODE_NECX(buf[0] << 8 | buf[1],
 							   buf[2]);
+				proto = RC_TYPE_NECX;
 			}
 		} else {
 			/* NEC full (32 bit) */
 			rc_code = RC_SCANCODE_NEC32(buf[0] << 24 | buf[1] << 16 |
 						    buf[2] << 8  | buf[3]);
+			proto = RC_TYPE_NEC32;
 		}
 
-		rc_keydown(d->rc_dev, RC_TYPE_NEC, rc_code, 0);
+		rc_keydown(d->rc_dev, proto, rc_code, 0);
 
 		ret = rtl28xxu_wr_reg(d, SYS_IRRC_SR, 1);
 		if (ret)
@@ -1668,7 +1673,7 @@ static int rtl2831u_get_rc_config(struct dvb_usb_device *d,
 		struct dvb_usb_rc *rc)
 {
 	rc->map_name = RC_MAP_EMPTY;
-	rc->allowed_protos = RC_BIT_NEC;
+	rc->allowed_protos = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32;
 	rc->query = rtl2831u_rc_query;
 	rc->interval = 400;
 
diff --git a/drivers/media/usb/dvb-usb/Kconfig b/drivers/media/usb/dvb-usb/Kconfig
index f03b0b70c901..959fa09dfd92 100644
--- a/drivers/media/usb/dvb-usb/Kconfig
+++ b/drivers/media/usb/dvb-usb/Kconfig
@@ -20,10 +20,20 @@ config DVB_USB_DEBUG
 	  Say Y if you want to enable debugging. See modinfo dvb-usb (and the
 	  appropriate drivers) for debug levels.
 
+config DVB_USB_DIB3000MC
+	tristate
+	depends on DVB_USB
+	select DVB_DIB3000MC
+	help
+	  This is a module with helper functions for accessing the
+	  DIB3000MC from USB DVB devices. It must be a separate module
+	  in case DVB_USB is built-in and DVB_DIB3000MC is a module,
+	  and gets selected automatically when needed.
+
 config DVB_USB_A800
 	tristate "AVerMedia AverTV DVB-T USB 2.0 (A800)"
 	depends on DVB_USB
-	select DVB_DIB3000MC
+	select DVB_USB_DIB3000MC
 	select DVB_PLL if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_MT2060 if MEDIA_SUBDRV_AUTOSELECT
 	help
@@ -34,6 +44,7 @@ config DVB_USB_DIBUSB_MB
 	depends on DVB_USB
 	select DVB_PLL if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_DIB3000MB
+	depends on DVB_DIB3000MC || !DVB_DIB3000MC
 	select MEDIA_TUNER_MT2060 if MEDIA_SUBDRV_AUTOSELECT
 	help
 	  Support for USB 1.1 and 2.0 DVB-T receivers based on reference designs made by
@@ -54,7 +65,7 @@ config DVB_USB_DIBUSB_MB_FAULTY
 config DVB_USB_DIBUSB_MC
 	tristate "DiBcom USB DVB-T devices (based on the DiB3000M-C/P) (see help for device list)"
 	depends on DVB_USB
-	select DVB_DIB3000MC
+	select DVB_USB_DIB3000MC
 	select MEDIA_TUNER_MT2060 if MEDIA_SUBDRV_AUTOSELECT
 	help
 	  Support for USB2.0 DVB-T receivers based on reference designs made by
@@ -72,7 +83,7 @@ config DVB_USB_DIB0700
 	select DVB_DIB7000P if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_DIB7000M if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_DIB8000 if MEDIA_SUBDRV_AUTOSELECT
-	select DVB_DIB3000MC if MEDIA_SUBDRV_AUTOSELECT
+	select DVB_USB_DIB3000MC if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_S5H1411 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_LGDT3305 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_TUNER_DIB0070 if MEDIA_SUBDRV_AUTOSELECT
@@ -99,7 +110,7 @@ config DVB_USB_UMT_010
 	tristate "HanfTek UMT-010 DVB-T USB2.0 support"
 	depends on DVB_USB
 	select DVB_PLL if MEDIA_SUBDRV_AUTOSELECT
-	select DVB_DIB3000MC
+	select DVB_USB_DIB3000MC
 	select MEDIA_TUNER_MT2060 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_MT352 if MEDIA_SUBDRV_AUTOSELECT
 	help
@@ -192,7 +203,7 @@ config DVB_USB_GP8PSK
 config DVB_USB_NOVA_T_USB2
 	tristate "Hauppauge WinTV-NOVA-T usb2 DVB-T USB2.0 support"
 	depends on DVB_USB
-	select DVB_DIB3000MC
+	select DVB_USB_DIB3000MC
 	select DVB_PLL if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_MT2060 if MEDIA_SUBDRV_AUTOSELECT
 	help
diff --git a/drivers/media/usb/dvb-usb/Makefile b/drivers/media/usb/dvb-usb/Makefile
index acdd1efd4e74..2a7b5a963acf 100644
--- a/drivers/media/usb/dvb-usb/Makefile
+++ b/drivers/media/usb/dvb-usb/Makefile
@@ -16,20 +16,23 @@ obj-$(CONFIG_DVB_USB_DTT200U) += dvb-usb-dtt200u.o
 
 dvb-usb-dibusb-common-objs := dibusb-common.o
 
+dvb-usb-dibusb-mc-common-objs := dibusb-mc-common.o
+obj-$(CONFIG_DVB_USB_DIB3000MC)	+= dvb-usb-dibusb-common.o dvb-usb-dibusb-mc-common.o
+
 dvb-usb-a800-objs := a800.o
-obj-$(CONFIG_DVB_USB_A800) += dvb-usb-dibusb-common.o dvb-usb-a800.o
+obj-$(CONFIG_DVB_USB_A800) += dvb-usb-a800.o
 
 dvb-usb-dibusb-mb-objs := dibusb-mb.o
 obj-$(CONFIG_DVB_USB_DIBUSB_MB) += dvb-usb-dibusb-common.o dvb-usb-dibusb-mb.o
 
 dvb-usb-dibusb-mc-objs := dibusb-mc.o
-obj-$(CONFIG_DVB_USB_DIBUSB_MC) += dvb-usb-dibusb-common.o dvb-usb-dibusb-mc.o
+obj-$(CONFIG_DVB_USB_DIBUSB_MC) += dvb-usb-dibusb-mc.o
 
 dvb-usb-nova-t-usb2-objs := nova-t-usb2.o
-obj-$(CONFIG_DVB_USB_NOVA_T_USB2) += dvb-usb-dibusb-common.o dvb-usb-nova-t-usb2.o
+obj-$(CONFIG_DVB_USB_NOVA_T_USB2) += dvb-usb-nova-t-usb2.o
 
 dvb-usb-umt-010-objs := umt-010.o
-obj-$(CONFIG_DVB_USB_UMT_010) += dvb-usb-dibusb-common.o dvb-usb-umt-010.o
+obj-$(CONFIG_DVB_USB_UMT_010) += dvb-usb-umt-010.o
 
 dvb-usb-m920x-objs := m920x.o
 obj-$(CONFIG_DVB_USB_M920X) += dvb-usb-m920x.o
diff --git a/drivers/media/usb/dvb-usb/dib0700_core.c b/drivers/media/usb/dvb-usb/dib0700_core.c
index bf890c3d9cda..f3196658fb70 100644
--- a/drivers/media/usb/dvb-usb/dib0700_core.c
+++ b/drivers/media/usb/dvb-usb/dib0700_core.c
@@ -710,7 +710,6 @@ static void dib0700_rc_urb_completion(struct urb *purb)
 
 	switch (d->props.rc.core.protocol) {
 	case RC_BIT_NEC:
-		protocol = RC_TYPE_NEC;
 		toggle = 0;
 
 		/* NEC protocol sends repeat code as 0 0 0 FF */
@@ -728,16 +727,19 @@ static void dib0700_rc_urb_completion(struct urb *purb)
 						     poll_reply->nec.not_system << 16 |
 						     poll_reply->nec.data       << 8  |
 						     poll_reply->nec.not_data);
+			protocol = RC_TYPE_NEC32;
 		} else if ((poll_reply->nec.system ^ poll_reply->nec.not_system) != 0xff) {
 			deb_data("NEC extended protocol\n");
 			keycode = RC_SCANCODE_NECX(poll_reply->nec.system << 8 |
 						    poll_reply->nec.not_system,
 						    poll_reply->nec.data);
 
+			protocol = RC_TYPE_NECX;
 		} else {
 			deb_data("NEC normal protocol\n");
 			keycode = RC_SCANCODE_NEC(poll_reply->nec.system,
 						   poll_reply->nec.data);
+			protocol = RC_TYPE_NEC;
 		}
 
 		break;
@@ -783,10 +785,8 @@ int dib0700_rc_setup(struct dvb_usb_device *d, struct usb_interface *intf)
 	/* Starting in firmware 1.20, the RC info is provided on a bulk pipe */
 
 	purb = usb_alloc_urb(0, GFP_KERNEL);
-	if (purb == NULL) {
-		err("rc usb alloc urb failed");
+	if (purb == NULL)
 		return -ENOMEM;
-	}
 
 	purb->transfer_buffer = kzalloc(RC_MSG_SIZE_V1_20, GFP_KERNEL);
 	if (purb->transfer_buffer == NULL) {
diff --git a/drivers/media/usb/dvb-usb/dibusb-common.c b/drivers/media/usb/dvb-usb/dibusb-common.c
index 6eea4e68891d..18ed3bfbb5e2 100644
--- a/drivers/media/usb/dvb-usb/dibusb-common.c
+++ b/drivers/media/usb/dvb-usb/dibusb-common.c
@@ -9,7 +9,6 @@
  * see Documentation/dvb/README.dvb-usb for more information
  */
 
-#include <linux/kconfig.h>
 #include "dibusb.h"
 
 /* Max transfer size done by I2C transfer functions */
@@ -184,164 +183,6 @@ int dibusb_read_eeprom_byte(struct dvb_usb_device *d, u8 offs, u8 *val)
 }
 EXPORT_SYMBOL(dibusb_read_eeprom_byte);
 
-#if IS_ENABLED(CONFIG_DVB_DIB3000MC)
-
-/* 3000MC/P stuff */
-// Config Adjacent channels  Perf -cal22
-static struct dibx000_agc_config dib3000p_mt2060_agc_config = {
-	.band_caps = BAND_VHF | BAND_UHF,
-	.setup     = (1 << 8) | (5 << 5) | (1 << 4) | (1 << 3) | (0 << 2) | (2 << 0),
-
-	.agc1_max = 48497,
-	.agc1_min = 23593,
-	.agc2_max = 46531,
-	.agc2_min = 24904,
-
-	.agc1_pt1 = 0x65,
-	.agc1_pt2 = 0x69,
-
-	.agc1_slope1 = 0x51,
-	.agc1_slope2 = 0x27,
-
-	.agc2_pt1 = 0,
-	.agc2_pt2 = 0x33,
-
-	.agc2_slope1 = 0x35,
-	.agc2_slope2 = 0x37,
-};
-
-static struct dib3000mc_config stk3000p_dib3000p_config = {
-	&dib3000p_mt2060_agc_config,
-
-	.max_time     = 0x196,
-	.ln_adc_level = 0x1cc7,
-
-	.output_mpeg2_in_188_bytes = 1,
-
-	.agc_command1 = 1,
-	.agc_command2 = 1,
-};
-
-static struct dibx000_agc_config dib3000p_panasonic_agc_config = {
-	.band_caps = BAND_VHF | BAND_UHF,
-	.setup     = (1 << 8) | (5 << 5) | (1 << 4) | (1 << 3) | (0 << 2) | (2 << 0),
-
-	.agc1_max = 56361,
-	.agc1_min = 22282,
-	.agc2_max = 47841,
-	.agc2_min = 36045,
-
-	.agc1_pt1 = 0x3b,
-	.agc1_pt2 = 0x6b,
-
-	.agc1_slope1 = 0x55,
-	.agc1_slope2 = 0x1d,
-
-	.agc2_pt1 = 0,
-	.agc2_pt2 = 0x0a,
-
-	.agc2_slope1 = 0x95,
-	.agc2_slope2 = 0x1e,
-};
-
-static struct dib3000mc_config mod3000p_dib3000p_config = {
-	&dib3000p_panasonic_agc_config,
-
-	.max_time     = 0x51,
-	.ln_adc_level = 0x1cc7,
-
-	.output_mpeg2_in_188_bytes = 1,
-
-	.agc_command1 = 1,
-	.agc_command2 = 1,
-};
-
-int dibusb_dib3000mc_frontend_attach(struct dvb_usb_adapter *adap)
-{
-	if (le16_to_cpu(adap->dev->udev->descriptor.idVendor) == USB_VID_LITEON &&
-	    le16_to_cpu(adap->dev->udev->descriptor.idProduct) ==
-			USB_PID_LITEON_DVB_T_WARM) {
-		msleep(1000);
-	}
-
-	adap->fe_adap[0].fe = dvb_attach(dib3000mc_attach,
-					 &adap->dev->i2c_adap,
-					 DEFAULT_DIB3000P_I2C_ADDRESS,
-					 &mod3000p_dib3000p_config);
-	if ((adap->fe_adap[0].fe) == NULL)
-		adap->fe_adap[0].fe = dvb_attach(dib3000mc_attach,
-						 &adap->dev->i2c_adap,
-						 DEFAULT_DIB3000MC_I2C_ADDRESS,
-						 &mod3000p_dib3000p_config);
-	if ((adap->fe_adap[0].fe) != NULL) {
-		if (adap->priv != NULL) {
-			struct dibusb_state *st = adap->priv;
-			st->ops.pid_parse = dib3000mc_pid_parse;
-			st->ops.pid_ctrl  = dib3000mc_pid_control;
-		}
-		return 0;
-	}
-	return -ENODEV;
-}
-EXPORT_SYMBOL(dibusb_dib3000mc_frontend_attach);
-
-static struct mt2060_config stk3000p_mt2060_config = {
-	0x60
-};
-
-int dibusb_dib3000mc_tuner_attach(struct dvb_usb_adapter *adap)
-{
-	struct dibusb_state *st = adap->priv;
-	u8 a,b;
-	u16 if1 = 1220;
-	struct i2c_adapter *tun_i2c;
-
-	// First IF calibration for Liteon Sticks
-	if (le16_to_cpu(adap->dev->udev->descriptor.idVendor) == USB_VID_LITEON &&
-	    le16_to_cpu(adap->dev->udev->descriptor.idProduct) == USB_PID_LITEON_DVB_T_WARM) {
-
-		dibusb_read_eeprom_byte(adap->dev,0x7E,&a);
-		dibusb_read_eeprom_byte(adap->dev,0x7F,&b);
-
-		if (a == 0x00)
-			if1 += b;
-		else if (a == 0x80)
-			if1 -= b;
-		else
-			warn("LITE-ON DVB-T: Strange IF1 calibration :%2X %2X\n", a, b);
-
-	} else if (le16_to_cpu(adap->dev->udev->descriptor.idVendor) == USB_VID_DIBCOM &&
-		   le16_to_cpu(adap->dev->udev->descriptor.idProduct) == USB_PID_DIBCOM_MOD3001_WARM) {
-		u8 desc;
-		dibusb_read_eeprom_byte(adap->dev, 7, &desc);
-		if (desc == 2) {
-			a = 127;
-			do {
-				dibusb_read_eeprom_byte(adap->dev, a, &desc);
-				a--;
-			} while (a > 7 && (desc == 0xff || desc == 0x00));
-			if (desc & 0x80)
-				if1 -= (0xff - desc);
-			else
-				if1 += desc;
-		}
-	}
-
-	tun_i2c = dib3000mc_get_tuner_i2c_master(adap->fe_adap[0].fe, 1);
-	if (dvb_attach(mt2060_attach, adap->fe_adap[0].fe, tun_i2c, &stk3000p_mt2060_config, if1) == NULL) {
-		/* not found - use panasonic pll parameters */
-		if (dvb_attach(dvb_pll_attach, adap->fe_adap[0].fe, 0x60, tun_i2c, DVB_PLL_ENV57H1XD5) == NULL)
-			return -ENOMEM;
-	} else {
-		st->mt2060_present = 1;
-		/* set the correct parameters for the dib3000p */
-		dib3000mc_set_config(adap->fe_adap[0].fe, &stk3000p_dib3000p_config);
-	}
-	return 0;
-}
-EXPORT_SYMBOL(dibusb_dib3000mc_tuner_attach);
-#endif
-
 /*
  * common remote control stuff
  */
diff --git a/drivers/media/usb/dvb-usb/dibusb-mc-common.c b/drivers/media/usb/dvb-usb/dibusb-mc-common.c
new file mode 100644
index 000000000000..d66f56cc46a5
--- /dev/null
+++ b/drivers/media/usb/dvb-usb/dibusb-mc-common.c
@@ -0,0 +1,168 @@
+/* Common methods for dibusb-based-receivers.
+ *
+ * Copyright (C) 2004-5 Patrick Boettcher (patrick.boettcher@desy.de)
+ *
+ *	This program is free software; you can redistribute it and/or modify it
+ *	under the terms of the GNU General Public License as published by the Free
+ *	Software Foundation, version 2.
+ *
+ * see Documentation/dvb/README.dvb-usb for more information
+ */
+
+#include <linux/kconfig.h>
+#include "dibusb.h"
+
+/* 3000MC/P stuff */
+// Config Adjacent channels  Perf -cal22
+static struct dibx000_agc_config dib3000p_mt2060_agc_config = {
+	.band_caps = BAND_VHF | BAND_UHF,
+	.setup     = (1 << 8) | (5 << 5) | (1 << 4) | (1 << 3) | (0 << 2) | (2 << 0),
+
+	.agc1_max = 48497,
+	.agc1_min = 23593,
+	.agc2_max = 46531,
+	.agc2_min = 24904,
+
+	.agc1_pt1 = 0x65,
+	.agc1_pt2 = 0x69,
+
+	.agc1_slope1 = 0x51,
+	.agc1_slope2 = 0x27,
+
+	.agc2_pt1 = 0,
+	.agc2_pt2 = 0x33,
+
+	.agc2_slope1 = 0x35,
+	.agc2_slope2 = 0x37,
+};
+
+static struct dib3000mc_config stk3000p_dib3000p_config = {
+	&dib3000p_mt2060_agc_config,
+
+	.max_time     = 0x196,
+	.ln_adc_level = 0x1cc7,
+
+	.output_mpeg2_in_188_bytes = 1,
+
+	.agc_command1 = 1,
+	.agc_command2 = 1,
+};
+
+static struct dibx000_agc_config dib3000p_panasonic_agc_config = {
+	.band_caps = BAND_VHF | BAND_UHF,
+	.setup     = (1 << 8) | (5 << 5) | (1 << 4) | (1 << 3) | (0 << 2) | (2 << 0),
+
+	.agc1_max = 56361,
+	.agc1_min = 22282,
+	.agc2_max = 47841,
+	.agc2_min = 36045,
+
+	.agc1_pt1 = 0x3b,
+	.agc1_pt2 = 0x6b,
+
+	.agc1_slope1 = 0x55,
+	.agc1_slope2 = 0x1d,
+
+	.agc2_pt1 = 0,
+	.agc2_pt2 = 0x0a,
+
+	.agc2_slope1 = 0x95,
+	.agc2_slope2 = 0x1e,
+};
+
+static struct dib3000mc_config mod3000p_dib3000p_config = {
+	&dib3000p_panasonic_agc_config,
+
+	.max_time     = 0x51,
+	.ln_adc_level = 0x1cc7,
+
+	.output_mpeg2_in_188_bytes = 1,
+
+	.agc_command1 = 1,
+	.agc_command2 = 1,
+};
+
+int dibusb_dib3000mc_frontend_attach(struct dvb_usb_adapter *adap)
+{
+	if (le16_to_cpu(adap->dev->udev->descriptor.idVendor) == USB_VID_LITEON &&
+	    le16_to_cpu(adap->dev->udev->descriptor.idProduct) ==
+			USB_PID_LITEON_DVB_T_WARM) {
+		msleep(1000);
+	}
+
+	adap->fe_adap[0].fe = dvb_attach(dib3000mc_attach,
+					 &adap->dev->i2c_adap,
+					 DEFAULT_DIB3000P_I2C_ADDRESS,
+					 &mod3000p_dib3000p_config);
+	if ((adap->fe_adap[0].fe) == NULL)
+		adap->fe_adap[0].fe = dvb_attach(dib3000mc_attach,
+						 &adap->dev->i2c_adap,
+						 DEFAULT_DIB3000MC_I2C_ADDRESS,
+						 &mod3000p_dib3000p_config);
+	if ((adap->fe_adap[0].fe) != NULL) {
+		if (adap->priv != NULL) {
+			struct dibusb_state *st = adap->priv;
+			st->ops.pid_parse = dib3000mc_pid_parse;
+			st->ops.pid_ctrl  = dib3000mc_pid_control;
+		}
+		return 0;
+	}
+	return -ENODEV;
+}
+EXPORT_SYMBOL(dibusb_dib3000mc_frontend_attach);
+
+static struct mt2060_config stk3000p_mt2060_config = {
+	0x60
+};
+
+int dibusb_dib3000mc_tuner_attach(struct dvb_usb_adapter *adap)
+{
+	struct dibusb_state *st = adap->priv;
+	u8 a,b;
+	u16 if1 = 1220;
+	struct i2c_adapter *tun_i2c;
+
+	// First IF calibration for Liteon Sticks
+	if (le16_to_cpu(adap->dev->udev->descriptor.idVendor) == USB_VID_LITEON &&
+	    le16_to_cpu(adap->dev->udev->descriptor.idProduct) == USB_PID_LITEON_DVB_T_WARM) {
+
+		dibusb_read_eeprom_byte(adap->dev,0x7E,&a);
+		dibusb_read_eeprom_byte(adap->dev,0x7F,&b);
+
+		if (a == 0x00)
+			if1 += b;
+		else if (a == 0x80)
+			if1 -= b;
+		else
+			warn("LITE-ON DVB-T: Strange IF1 calibration :%2X %2X\n", a, b);
+
+	} else if (le16_to_cpu(adap->dev->udev->descriptor.idVendor) == USB_VID_DIBCOM &&
+		   le16_to_cpu(adap->dev->udev->descriptor.idProduct) == USB_PID_DIBCOM_MOD3001_WARM) {
+		u8 desc;
+		dibusb_read_eeprom_byte(adap->dev, 7, &desc);
+		if (desc == 2) {
+			a = 127;
+			do {
+				dibusb_read_eeprom_byte(adap->dev, a, &desc);
+				a--;
+			} while (a > 7 && (desc == 0xff || desc == 0x00));
+			if (desc & 0x80)
+				if1 -= (0xff - desc);
+			else
+				if1 += desc;
+		}
+	}
+
+	tun_i2c = dib3000mc_get_tuner_i2c_master(adap->fe_adap[0].fe, 1);
+	if (dvb_attach(mt2060_attach, adap->fe_adap[0].fe, tun_i2c, &stk3000p_mt2060_config, if1) == NULL) {
+		/* not found - use panasonic pll parameters */
+		if (dvb_attach(dvb_pll_attach, adap->fe_adap[0].fe, 0x60, tun_i2c, DVB_PLL_ENV57H1XD5) == NULL)
+			return -ENOMEM;
+	} else {
+		st->mt2060_present = 1;
+		/* set the correct parameters for the dib3000p */
+		dib3000mc_set_config(adap->fe_adap[0].fe, &stk3000p_dib3000p_config);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(dibusb_dib3000mc_tuner_attach);
diff --git a/drivers/media/usb/dvb-usb/dtt200u.c b/drivers/media/usb/dvb-usb/dtt200u.c
index be633ece4194..d2a01b50af0d 100644
--- a/drivers/media/usb/dvb-usb/dtt200u.c
+++ b/drivers/media/usb/dvb-usb/dtt200u.c
@@ -62,18 +62,21 @@ static int dtt200u_rc_query(struct dvb_usb_device *d)
 
 	dvb_usb_generic_rw(d,&cmd,1,key,5,0);
 	if (key[0] == 1) {
+		enum rc_type proto = RC_TYPE_NEC;
+
 		scancode = key[1];
 		if ((u8) ~key[1] != key[2]) {
 			/* Extended NEC */
 			scancode = scancode << 8;
 			scancode |= key[2];
+			proto = RC_TYPE_NECX;
 		}
 		scancode = scancode << 8;
 		scancode |= key[3];
 
 		/* Check command checksum is ok */
 		if ((u8) ~key[3] == key[4])
-			rc_keydown(d->rc_dev, RC_TYPE_NEC, scancode, 0);
+			rc_keydown(d->rc_dev, proto, scancode, 0);
 		else
 			rc_keyup(d->rc_dev);
 	} else if (key[0] == 2) {
diff --git a/drivers/media/usb/em28xx/em28xx-audio.c b/drivers/media/usb/em28xx/em28xx-audio.c
index 49a5f9532bd8..e11fe46a547c 100644
--- a/drivers/media/usb/em28xx/em28xx-audio.c
+++ b/drivers/media/usb/em28xx/em28xx-audio.c
@@ -695,7 +695,7 @@ static int em28xx_cvol_new(struct snd_card *card, struct em28xx *dev,
 /*
  * register/unregister code and data
  */
-static struct snd_pcm_ops snd_em28xx_pcm_capture = {
+static const struct snd_pcm_ops snd_em28xx_pcm_capture = {
 	.open      = snd_em28xx_capture_open,
 	.close     = snd_em28xx_pcm_close,
 	.ioctl     = snd_pcm_lib_ioctl,
@@ -850,7 +850,6 @@ static int em28xx_audio_urb_init(struct em28xx *dev)
 
 		urb = usb_alloc_urb(npackets, GFP_ATOMIC);
 		if (!urb) {
-			em28xx_errdev("usb_alloc_urb failed!\n");
 			em28xx_audio_free_urb(dev);
 			return -ENOMEM;
 		}
diff --git a/drivers/media/usb/em28xx/em28xx-core.c b/drivers/media/usb/em28xx/em28xx-core.c
index 37456079f490..eebd5d7088d0 100644
--- a/drivers/media/usb/em28xx/em28xx-core.c
+++ b/drivers/media/usb/em28xx/em28xx-core.c
@@ -934,7 +934,6 @@ int em28xx_alloc_urbs(struct em28xx *dev, enum em28xx_mode mode, int xfer_bulk,
 	for (i = 0; i < usb_bufs->num_bufs; i++) {
 		urb = usb_alloc_urb(usb_bufs->num_packets, GFP_KERNEL);
 		if (!urb) {
-			em28xx_err("cannot alloc usb_ctl.urb %i\n", i);
 			em28xx_uninit_usb_xfer(dev, mode);
 			return -ENOMEM;
 		}
diff --git a/drivers/media/usb/em28xx/em28xx-i2c.c b/drivers/media/usb/em28xx/em28xx-i2c.c
index 1a9e1e556706..8b690ac908a4 100644
--- a/drivers/media/usb/em28xx/em28xx-i2c.c
+++ b/drivers/media/usb/em28xx/em28xx-i2c.c
@@ -855,7 +855,7 @@ static u32 functionality(struct i2c_adapter *i2c_adap)
 	return 0;
 }
 
-static struct i2c_algorithm em28xx_algo = {
+static const struct i2c_algorithm em28xx_algo = {
 	.master_xfer   = em28xx_i2c_xfer,
 	.functionality = functionality,
 };
diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c
index 7968695217f3..1f7fa059eb34 100644
--- a/drivers/media/usb/em28xx/em28xx-video.c
+++ b/drivers/media/usb/em28xx/em28xx-video.c
@@ -1204,7 +1204,7 @@ buffer_queue(struct vb2_buffer *vb)
 	spin_unlock_irqrestore(&dev->slock, flags);
 }
 
-static struct vb2_ops em28xx_video_qops = {
+static const struct vb2_ops em28xx_video_qops = {
 	.queue_setup    = queue_setup,
 	.buf_prepare    = buffer_prepare,
 	.buf_queue      = buffer_queue,
diff --git a/drivers/media/usb/go7007/go7007-i2c.c b/drivers/media/usb/go7007/go7007-i2c.c
index 55addfa855d4..c084bf794b56 100644
--- a/drivers/media/usb/go7007/go7007-i2c.c
+++ b/drivers/media/usb/go7007/go7007-i2c.c
@@ -191,7 +191,7 @@ static u32 go7007_functionality(struct i2c_adapter *adapter)
 	return I2C_FUNC_SMBUS_BYTE_DATA;
 }
 
-static struct i2c_algorithm go7007_algo = {
+static const struct i2c_algorithm go7007_algo = {
 	.smbus_xfer	= go7007_smbus_xfer,
 	.master_xfer	= go7007_i2c_master_xfer,
 	.functionality	= go7007_functionality,
diff --git a/drivers/media/usb/go7007/go7007-usb.c b/drivers/media/usb/go7007/go7007-usb.c
index 14d3f8c1ce4a..ed9bcaf08d5e 100644
--- a/drivers/media/usb/go7007/go7007-usb.c
+++ b/drivers/media/usb/go7007/go7007-usb.c
@@ -1032,7 +1032,7 @@ static u32 go7007_usb_functionality(struct i2c_adapter *adapter)
 	return (I2C_FUNC_SMBUS_EMUL) & ~I2C_FUNC_SMBUS_QUICK;
 }
 
-static struct i2c_algorithm go7007_usb_algo = {
+static const struct i2c_algorithm go7007_usb_algo = {
 	.master_xfer	= go7007_usb_i2c_master_xfer,
 	.functionality	= go7007_usb_functionality,
 };
diff --git a/drivers/media/usb/go7007/go7007-v4l2.c b/drivers/media/usb/go7007/go7007-v4l2.c
index af8458996d91..4eaba0c24629 100644
--- a/drivers/media/usb/go7007/go7007-v4l2.c
+++ b/drivers/media/usb/go7007/go7007-v4l2.c
@@ -477,7 +477,7 @@ static void go7007_stop_streaming(struct vb2_queue *q)
 		go7007_write_addr(go, 0x3c82, 0x000d);
 }
 
-static struct vb2_ops go7007_video_qops = {
+static const struct vb2_ops go7007_video_qops = {
 	.queue_setup    = go7007_queue_setup,
 	.buf_queue      = go7007_buf_queue,
 	.buf_prepare    = go7007_buf_prepare,
diff --git a/drivers/media/usb/go7007/snd-go7007.c b/drivers/media/usb/go7007/snd-go7007.c
index d22d7d574672..070871fb1fc4 100644
--- a/drivers/media/usb/go7007/snd-go7007.c
+++ b/drivers/media/usb/go7007/snd-go7007.c
@@ -198,7 +198,7 @@ static struct page *go7007_snd_pcm_page(struct snd_pcm_substream *substream,
 	return vmalloc_to_page(substream->runtime->dma_area + offset);
 }
 
-static struct snd_pcm_ops go7007_snd_capture_ops = {
+static const struct snd_pcm_ops go7007_snd_capture_ops = {
 	.open		= go7007_snd_capture_open,
 	.close		= go7007_snd_capture_close,
 	.ioctl		= snd_pcm_lib_ioctl,
diff --git a/drivers/media/usb/gspca/benq.c b/drivers/media/usb/gspca/benq.c
index 790baed33963..5fa67b78ad49 100644
--- a/drivers/media/usb/gspca/benq.c
+++ b/drivers/media/usb/gspca/benq.c
@@ -95,10 +95,8 @@ static int sd_start(struct gspca_dev *gspca_dev)
 #define SD_NPKT 32
 	for (n = 0; n < 4; n++) {
 		urb = usb_alloc_urb(SD_NPKT, GFP_KERNEL);
-		if (!urb) {
-			pr_err("usb_alloc_urb failed\n");
+		if (!urb)
 			return -ENOMEM;
-		}
 		gspca_dev->urb[n] = urb;
 		urb->transfer_buffer = usb_alloc_coherent(gspca_dev->dev,
 						SD_PKT_SZ * SD_NPKT,
diff --git a/drivers/media/usb/gspca/finepix.c b/drivers/media/usb/gspca/finepix.c
index 52bdb569760b..ae9a55d7bbbb 100644
--- a/drivers/media/usb/gspca/finepix.c
+++ b/drivers/media/usb/gspca/finepix.c
@@ -41,7 +41,6 @@ struct usb_fpix {
 	struct gspca_dev gspca_dev;	/* !! must be the first item */
 
 	struct work_struct work_struct;
-	struct workqueue_struct *work_thread;
 };
 
 /* Delay after which claim the next frame. If the delay is too small,
@@ -226,9 +225,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 	/* Again, reset bulk in endpoint */
 	usb_clear_halt(gspca_dev->dev, gspca_dev->urb[0]->pipe);
 
-	/* Start the workqueue function to do the streaming */
-	dev->work_thread = create_singlethread_workqueue(MODULE_NAME);
-	queue_work(dev->work_thread, &dev->work_struct);
+	schedule_work(&dev->work_struct);
 
 	return 0;
 }
@@ -241,9 +238,8 @@ static void sd_stop0(struct gspca_dev *gspca_dev)
 
 	/* wait for the work queue to terminate */
 	mutex_unlock(&gspca_dev->usb_lock);
-	destroy_workqueue(dev->work_thread);
+	flush_work(&dev->work_struct);
 	mutex_lock(&gspca_dev->usb_lock);
-	dev->work_thread = NULL;
 }
 
 /* Table of supported USB devices */
diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c
index b17bd7ebcb47..af2395a76d8b 100644
--- a/drivers/media/usb/gspca/gspca.c
+++ b/drivers/media/usb/gspca/gspca.c
@@ -795,10 +795,8 @@ static int create_urbs(struct gspca_dev *gspca_dev,
 
 	for (n = 0; n < nurbs; n++) {
 		urb = usb_alloc_urb(npkt, GFP_KERNEL);
-		if (!urb) {
-			pr_err("usb_alloc_urb failed\n");
+		if (!urb)
 			return -ENOMEM;
-		}
 		gspca_dev->urb[n] = urb;
 		urb->transfer_buffer = usb_alloc_coherent(gspca_dev->dev,
 						bsize,
diff --git a/drivers/media/usb/gspca/jl2005bcd.c b/drivers/media/usb/gspca/jl2005bcd.c
index 5b481fa43099..ac295f04bd18 100644
--- a/drivers/media/usb/gspca/jl2005bcd.c
+++ b/drivers/media/usb/gspca/jl2005bcd.c
@@ -45,7 +45,6 @@ struct sd {
 	const struct v4l2_pix_format *cap_mode;
 	/* Driver stuff */
 	struct work_struct work_struct;
-	struct workqueue_struct *work_thread;
 	u8 frame_brightness;
 	int block_size;	/* block size of camera */
 	int vga;	/* 1 if vga cam, 0 if cif cam */
@@ -477,9 +476,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		return -1;
 	}
 
-	/* Start the workqueue function to do the streaming */
-	sd->work_thread = create_singlethread_workqueue(MODULE_NAME);
-	queue_work(sd->work_thread, &sd->work_struct);
+	schedule_work(&sd->work_struct);
 
 	return 0;
 }
@@ -493,8 +490,7 @@ static void sd_stop0(struct gspca_dev *gspca_dev)
 	/* wait for the work queue to terminate */
 	mutex_unlock(&gspca_dev->usb_lock);
 	/* This waits for sq905c_dostream to finish */
-	destroy_workqueue(dev->work_thread);
-	dev->work_thread = NULL;
+	flush_work(&dev->work_struct);
 	mutex_lock(&gspca_dev->usb_lock);
 }
 
diff --git a/drivers/media/usb/gspca/konica.c b/drivers/media/usb/gspca/konica.c
index 0712b1bc90b4..40aaaa9c5f30 100644
--- a/drivers/media/usb/gspca/konica.c
+++ b/drivers/media/usb/gspca/konica.c
@@ -208,10 +208,8 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		packet_size =
 			le16_to_cpu(alt->endpoint[i].desc.wMaxPacketSize);
 		urb = usb_alloc_urb(SD_NPKT, GFP_KERNEL);
-		if (!urb) {
-			pr_err("usb_alloc_urb failed\n");
+		if (!urb)
 			return -ENOMEM;
-		}
 		gspca_dev->urb[n] = urb;
 		urb->transfer_buffer = usb_alloc_coherent(gspca_dev->dev,
 						packet_size * SD_NPKT,
diff --git a/drivers/media/usb/gspca/sonixj.c b/drivers/media/usb/gspca/sonixj.c
index fd1c8706d86a..d49d76ec1421 100644
--- a/drivers/media/usb/gspca/sonixj.c
+++ b/drivers/media/usb/gspca/sonixj.c
@@ -54,7 +54,6 @@ struct sd {
 	u32 exposure;
 
 	struct work_struct work;
-	struct workqueue_struct *work_thread;
 
 	u32 pktsz;			/* (used by pkt_scan) */
 	u16 npkt;
@@ -2485,7 +2484,6 @@ static int sd_start(struct gspca_dev *gspca_dev)
 
 	sd->pktsz = sd->npkt = 0;
 	sd->nchg = sd->short_mark = 0;
-	sd->work_thread = create_singlethread_workqueue(MODULE_NAME);
 
 	return gspca_dev->usb_err;
 }
@@ -2569,12 +2567,9 @@ static void sd_stop0(struct gspca_dev *gspca_dev)
 {
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	if (sd->work_thread != NULL) {
-		mutex_unlock(&gspca_dev->usb_lock);
-		destroy_workqueue(sd->work_thread);
-		mutex_lock(&gspca_dev->usb_lock);
-		sd->work_thread = NULL;
-	}
+	mutex_unlock(&gspca_dev->usb_lock);
+	flush_work(&sd->work);
+	mutex_lock(&gspca_dev->usb_lock);
 }
 
 static void do_autogain(struct gspca_dev *gspca_dev)
@@ -2785,7 +2780,7 @@ marker_found:
 				new_qual = QUALITY_MAX;
 			if (new_qual != sd->quality) {
 				sd->quality = new_qual;
-				queue_work(sd->work_thread, &sd->work);
+				schedule_work(&sd->work);
 			}
 		}
 	} else {
diff --git a/drivers/media/usb/gspca/vicam.c b/drivers/media/usb/gspca/vicam.c
index 103f6c4236b0..8860510c2f9c 100644
--- a/drivers/media/usb/gspca/vicam.c
+++ b/drivers/media/usb/gspca/vicam.c
@@ -47,7 +47,6 @@ MODULE_FIRMWARE(VICAM_FIRMWARE);
 struct sd {
 	struct gspca_dev gspca_dev;	/* !! must be the first item */
 	struct work_struct work_struct;
-	struct workqueue_struct *work_thread;
 };
 
 /* The vicam sensor has a resolution of 512 x 244, with I believe square
@@ -278,9 +277,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 	if (ret < 0)
 		return ret;
 
-	/* Start the workqueue function to do the streaming */
-	sd->work_thread = create_singlethread_workqueue(MODULE_NAME);
-	queue_work(sd->work_thread, &sd->work_struct);
+	schedule_work(&sd->work_struct);
 
 	return 0;
 }
@@ -294,8 +291,7 @@ static void sd_stop0(struct gspca_dev *gspca_dev)
 	/* wait for the work queue to terminate */
 	mutex_unlock(&gspca_dev->usb_lock);
 	/* This waits for vicam_dostream to finish */
-	destroy_workqueue(dev->work_thread);
-	dev->work_thread = NULL;
+	flush_work(&dev->work_struct);
 	mutex_lock(&gspca_dev->usb_lock);
 
 	if (gspca_dev->present)
diff --git a/drivers/media/usb/hackrf/hackrf.c b/drivers/media/usb/hackrf/hackrf.c
index b1e229a44192..d9a525260511 100644
--- a/drivers/media/usb/hackrf/hackrf.c
+++ b/drivers/media/usb/hackrf/hackrf.c
@@ -129,7 +129,7 @@ struct hackrf_dev {
 	struct list_head rx_buffer_list;
 	struct list_head tx_buffer_list;
 	spinlock_t buffer_list_lock; /* Protects buffer_list */
-	unsigned sequence;	     /* Buffer sequence counter */
+	unsigned int sequence;	     /* Buffer sequence counter */
 	unsigned int vb_full;        /* vb is full and packets dropped */
 	unsigned int vb_empty;       /* vb is empty and packets dropped */
 
@@ -691,7 +691,6 @@ static int hackrf_alloc_urbs(struct hackrf_dev *dev, bool rcv)
 		dev_dbg(dev->dev, "alloc urb=%d\n", i);
 		dev->urb_list[i] = usb_alloc_urb(0, GFP_ATOMIC);
 		if (!dev->urb_list[i]) {
-			dev_dbg(dev->dev, "failed\n");
 			for (j = 0; j < i; j++)
 				usb_free_urb(dev->urb_list[j]);
 			return -ENOMEM;
@@ -892,7 +891,7 @@ static void hackrf_stop_streaming(struct vb2_queue *vq)
 	mutex_unlock(&dev->v4l2_lock);
 }
 
-static struct vb2_ops hackrf_vb2_ops = {
+static const struct vb2_ops hackrf_vb2_ops = {
 	.queue_setup            = hackrf_queue_setup,
 	.buf_queue              = hackrf_buf_queue,
 	.start_streaming        = hackrf_start_streaming,
diff --git a/drivers/media/usb/hdpvr/hdpvr-i2c.c b/drivers/media/usb/hdpvr/hdpvr-i2c.c
index a38f58c4c6bf..9b641c4d4431 100644
--- a/drivers/media/usb/hdpvr/hdpvr-i2c.c
+++ b/drivers/media/usb/hdpvr/hdpvr-i2c.c
@@ -55,7 +55,7 @@ struct i2c_client *hdpvr_register_ir_rx_i2c(struct hdpvr_device *dev)
 	/* Our default information for ir-kbd-i2c.c to use */
 	init_data->ir_codes = RC_MAP_HAUPPAUGE;
 	init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
-	init_data->type = RC_BIT_RC5;
+	init_data->type = RC_BIT_RC5 | RC_BIT_RC6_MCE | RC_BIT_RC6_6A_32;
 	init_data->name = "HD-PVR";
 	init_data->polling_interval = 405; /* ms, duplicated from Windows */
 	hdpvr_ir_rx_i2c_board_info.platform_data = init_data;
@@ -180,7 +180,7 @@ static u32 hdpvr_functionality(struct i2c_adapter *adapter)
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm hdpvr_algo = {
+static const struct i2c_algorithm hdpvr_algo = {
 	.master_xfer   = hdpvr_transfer,
 	.functionality = hdpvr_functionality,
 };
diff --git a/drivers/media/usb/hdpvr/hdpvr-video.c b/drivers/media/usb/hdpvr/hdpvr-video.c
index 2a3a8b470555..474c11e1d495 100644
--- a/drivers/media/usb/hdpvr/hdpvr-video.c
+++ b/drivers/media/usb/hdpvr/hdpvr-video.c
@@ -10,7 +10,6 @@
  */
 
 #include <linux/kernel.h>
-#include <linux/kconfig.h>
 #include <linux/errno.h>
 #include <linux/init.h>
 #include <linux/slab.h>
@@ -155,10 +154,8 @@ int hdpvr_alloc_buffers(struct hdpvr_device *dev, uint count)
 		buf->dev = dev;
 
 		urb = usb_alloc_urb(0, GFP_KERNEL);
-		if (!urb) {
-			v4l2_err(&dev->v4l2_dev, "cannot allocate urb\n");
+		if (!urb)
 			goto exit_urb;
-		}
 		buf->urb = urb;
 
 		mem = usb_alloc_coherent(dev->udev, dev->bulk_in_size, GFP_KERNEL,
diff --git a/drivers/media/usb/msi2500/msi2500.c b/drivers/media/usb/msi2500/msi2500.c
index e7f167d44c61..bb3d31e2a0b5 100644
--- a/drivers/media/usb/msi2500/msi2500.c
+++ b/drivers/media/usb/msi2500/msi2500.c
@@ -509,7 +509,6 @@ static int msi2500_isoc_init(struct msi2500_dev *dev)
 	for (i = 0; i < MAX_ISO_BUFS; i++) {
 		urb = usb_alloc_urb(ISO_FRAMES_PER_DESC, GFP_KERNEL);
 		if (urb == NULL) {
-			dev_err(dev->dev, "Failed to allocate urb %d\n", i);
 			msi2500_isoc_cleanup(dev);
 			return -ENOMEM;
 		}
@@ -898,7 +897,7 @@ static void msi2500_stop_streaming(struct vb2_queue *vq)
 	mutex_unlock(&dev->v4l2_lock);
 }
 
-static struct vb2_ops msi2500_vb2_ops = {
+static const struct vb2_ops msi2500_vb2_ops = {
 	.queue_setup            = msi2500_queue_setup,
 	.buf_queue              = msi2500_buf_queue,
 	.start_streaming        = msi2500_start_streaming,
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw-internal.h b/drivers/media/usb/pvrusb2/pvrusb2-hdw-internal.h
index 60141b16d731..23473a21319c 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-hdw-internal.h
+++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw-internal.h
@@ -170,7 +170,6 @@ struct pvr2_hdw {
 	const struct pvr2_device_desc *hdw_desc;
 
 	/* Kernel worker thread handling */
-	struct workqueue_struct *workqueue;
 	struct work_struct workpoll;     /* Update driver state */
 
 	/* Video spigot */
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
index fe20fe4f2330..1eb4f7ba2967 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
@@ -2624,7 +2624,6 @@ struct pvr2_hdw *pvr2_hdw_create(struct usb_interface *intf,
 	if (cnt1 >= sizeof(hdw->name)) cnt1 = sizeof(hdw->name)-1;
 	hdw->name[cnt1] = 0;
 
-	hdw->workqueue = create_singlethread_workqueue(hdw->name);
 	INIT_WORK(&hdw->workpoll,pvr2_hdw_worker_poll);
 
 	pvr2_trace(PVR2_TRACE_INIT,"Driver unit number is %d, name is %s",
@@ -2651,11 +2650,7 @@ struct pvr2_hdw *pvr2_hdw_create(struct usb_interface *intf,
 		del_timer_sync(&hdw->decoder_stabilization_timer);
 		del_timer_sync(&hdw->encoder_run_timer);
 		del_timer_sync(&hdw->encoder_wait_timer);
-		if (hdw->workqueue) {
-			flush_workqueue(hdw->workqueue);
-			destroy_workqueue(hdw->workqueue);
-			hdw->workqueue = NULL;
-		}
+		flush_work(&hdw->workpoll);
 		usb_free_urb(hdw->ctl_read_urb);
 		usb_free_urb(hdw->ctl_write_urb);
 		kfree(hdw->ctl_read_buffer);
@@ -2712,11 +2707,7 @@ void pvr2_hdw_destroy(struct pvr2_hdw *hdw)
 {
 	if (!hdw) return;
 	pvr2_trace(PVR2_TRACE_INIT,"pvr2_hdw_destroy: hdw=%p",hdw);
-	if (hdw->workqueue) {
-		flush_workqueue(hdw->workqueue);
-		destroy_workqueue(hdw->workqueue);
-		hdw->workqueue = NULL;
-	}
+	flush_work(&hdw->workpoll);
 	del_timer_sync(&hdw->quiescent_timer);
 	del_timer_sync(&hdw->decoder_stabilization_timer);
 	del_timer_sync(&hdw->encoder_run_timer);
@@ -4443,7 +4434,7 @@ static void pvr2_hdw_quiescent_timeout(unsigned long data)
 	hdw->state_decoder_quiescent = !0;
 	trace_stbit("state_decoder_quiescent",hdw->state_decoder_quiescent);
 	hdw->state_stale = !0;
-	queue_work(hdw->workqueue,&hdw->workpoll);
+	schedule_work(&hdw->workpoll);
 }
 
 
@@ -4454,7 +4445,7 @@ static void pvr2_hdw_decoder_stabilization_timeout(unsigned long data)
 	hdw->state_decoder_ready = !0;
 	trace_stbit("state_decoder_ready", hdw->state_decoder_ready);
 	hdw->state_stale = !0;
-	queue_work(hdw->workqueue, &hdw->workpoll);
+	schedule_work(&hdw->workpoll);
 }
 
 
@@ -4465,7 +4456,7 @@ static void pvr2_hdw_encoder_wait_timeout(unsigned long data)
 	hdw->state_encoder_waitok = !0;
 	trace_stbit("state_encoder_waitok",hdw->state_encoder_waitok);
 	hdw->state_stale = !0;
-	queue_work(hdw->workqueue,&hdw->workpoll);
+	schedule_work(&hdw->workpoll);
 }
 
 
@@ -4477,7 +4468,7 @@ static void pvr2_hdw_encoder_run_timeout(unsigned long data)
 		hdw->state_encoder_runok = !0;
 		trace_stbit("state_encoder_runok",hdw->state_encoder_runok);
 		hdw->state_stale = !0;
-		queue_work(hdw->workqueue,&hdw->workpoll);
+		schedule_work(&hdw->workpoll);
 	}
 }
 
@@ -4991,7 +4982,7 @@ static void pvr2_hdw_state_sched(struct pvr2_hdw *hdw)
 	if (hdw->state_stale) return;
 	hdw->state_stale = !0;
 	trace_stbit("state_stale",hdw->state_stale);
-	queue_work(hdw->workqueue,&hdw->workpoll);
+	schedule_work(&hdw->workpoll);
 }
 
 
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c b/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c
index 14321d0a1833..6da5fb544817 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c
@@ -596,7 +596,8 @@ static void pvr2_i2c_register_ir(struct pvr2_hdw *hdw)
 	case PVR2_IR_SCHEME_24XXX_MCE: /* 24xxx MCE device */
 		init_data->ir_codes              = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
-		init_data->type                  = RC_BIT_RC5;
+		init_data->type                  = RC_BIT_RC5 | RC_BIT_RC6_MCE |
+							RC_BIT_RC6_6A_32;
 		init_data->name                  = hdw->hdw_desc->description;
 		/* IR Receiver */
 		info.addr          = 0x71;
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
index 81f788b7b242..2cc4d2b6f810 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
@@ -719,64 +719,85 @@ static int pvr2_cropcap(struct file *file, void *priv, struct v4l2_cropcap *cap)
 	return ret;
 }
 
-static int pvr2_g_crop(struct file *file, void *priv, struct v4l2_crop *crop)
+static int pvr2_g_selection(struct file *file, void *priv,
+			    struct v4l2_selection *sel)
 {
 	struct pvr2_v4l2_fh *fh = file->private_data;
 	struct pvr2_hdw *hdw = fh->channel.mc_head->hdw;
+	struct v4l2_cropcap cap;
 	int val = 0;
 	int ret;
 
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
-	ret = pvr2_ctrl_get_value(
-			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPL), &val);
-	if (ret != 0)
-		return -EINVAL;
-	crop->c.left = val;
-	ret = pvr2_ctrl_get_value(
-			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPT), &val);
-	if (ret != 0)
-		return -EINVAL;
-	crop->c.top = val;
-	ret = pvr2_ctrl_get_value(
-			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPW), &val);
-	if (ret != 0)
-		return -EINVAL;
-	crop->c.width = val;
-	ret = pvr2_ctrl_get_value(
-			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPH), &val);
-	if (ret != 0)
+
+	cap.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		ret = pvr2_ctrl_get_value(
+			  pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPL), &val);
+		if (ret != 0)
+			return -EINVAL;
+		sel->r.left = val;
+		ret = pvr2_ctrl_get_value(
+			  pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPT), &val);
+		if (ret != 0)
+			return -EINVAL;
+		sel->r.top = val;
+		ret = pvr2_ctrl_get_value(
+			  pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPW), &val);
+		if (ret != 0)
+			return -EINVAL;
+		sel->r.width = val;
+		ret = pvr2_ctrl_get_value(
+			  pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPH), &val);
+		if (ret != 0)
+			return -EINVAL;
+		sel->r.height = val;
+		break;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		ret = pvr2_hdw_get_cropcap(hdw, &cap);
+		sel->r = cap.defrect;
+		break;
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		ret = pvr2_hdw_get_cropcap(hdw, &cap);
+		sel->r = cap.bounds;
+		break;
+	default:
 		return -EINVAL;
-	crop->c.height = val;
-	return 0;
+	}
+	return ret;
 }
 
-static int pvr2_s_crop(struct file *file, void *priv, const struct v4l2_crop *crop)
+static int pvr2_s_selection(struct file *file, void *priv,
+			    struct v4l2_selection *sel)
 {
 	struct pvr2_v4l2_fh *fh = file->private_data;
 	struct pvr2_hdw *hdw = fh->channel.mc_head->hdw;
 	int ret;
 
-	if (crop->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+	if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE ||
+	    sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPL),
-			crop->c.left);
+			sel->r.left);
 	if (ret != 0)
 		return -EINVAL;
 	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPT),
-			crop->c.top);
+			sel->r.top);
 	if (ret != 0)
 		return -EINVAL;
 	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPW),
-			crop->c.width);
+			sel->r.width);
 	if (ret != 0)
 		return -EINVAL;
 	ret = pvr2_ctrl_set_value(
 			pvr2_hdw_get_ctrl_by_id(hdw, PVR2_CID_CROPH),
-			crop->c.height);
+			sel->r.height);
 	if (ret != 0)
 		return -EINVAL;
 	return 0;
@@ -798,8 +819,8 @@ static const struct v4l2_ioctl_ops pvr2_ioctl_ops = {
 	.vidioc_enumaudio		    = pvr2_enumaudio,
 	.vidioc_enum_input		    = pvr2_enum_input,
 	.vidioc_cropcap			    = pvr2_cropcap,
-	.vidioc_s_crop			    = pvr2_s_crop,
-	.vidioc_g_crop			    = pvr2_g_crop,
+	.vidioc_s_selection		    = pvr2_s_selection,
+	.vidioc_g_selection		    = pvr2_g_selection,
 	.vidioc_g_input			    = pvr2_g_input,
 	.vidioc_s_input			    = pvr2_s_input,
 	.vidioc_g_frequency		    = pvr2_g_frequency,
diff --git a/drivers/media/usb/pwc/pwc-if.c b/drivers/media/usb/pwc/pwc-if.c
index b51b27a3fd61..ff657644b6b3 100644
--- a/drivers/media/usb/pwc/pwc-if.c
+++ b/drivers/media/usb/pwc/pwc-if.c
@@ -410,7 +410,6 @@ retry:
 	for (i = 0; i < MAX_ISO_BUFS; i++) {
 		urb = usb_alloc_urb(ISO_FRAMES_PER_DESC, GFP_KERNEL);
 		if (urb == NULL) {
-			PWC_ERROR("Failed to allocate urb %d\n", i);
 			pwc_isoc_cleanup(pdev);
 			return -ENOMEM;
 		}
@@ -708,7 +707,7 @@ static void stop_streaming(struct vb2_queue *vq)
 	mutex_unlock(&pdev->v4l2_lock);
 }
 
-static struct vb2_ops pwc_vb_queue_ops = {
+static const struct vb2_ops pwc_vb_queue_ops = {
 	.queue_setup		= queue_setup,
 	.buf_init		= buffer_init,
 	.buf_prepare		= buffer_prepare,
diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c
index 43ba71a7d02b..c3a0e87066eb 100644
--- a/drivers/media/usb/s2255/s2255drv.c
+++ b/drivers/media/usb/s2255/s2255drv.c
@@ -717,7 +717,7 @@ static void buffer_queue(struct vb2_buffer *vb)
 static int start_streaming(struct vb2_queue *vq, unsigned int count);
 static void stop_streaming(struct vb2_queue *vq);
 
-static struct vb2_ops s2255_video_qops = {
+static const struct vb2_ops s2255_video_qops = {
 	.queue_setup = queue_setup,
 	.buf_prepare = buffer_prepare,
 	.buf_queue = buffer_queue,
@@ -2113,11 +2113,8 @@ static int s2255_start_readpipe(struct s2255_dev *dev)
 	pipe_info->state = 1;
 	pipe_info->err_count = 0;
 	pipe_info->stream_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!pipe_info->stream_urb) {
-		dev_err(&dev->udev->dev,
-			"ReadStream: Unable to alloc URB\n");
+	if (!pipe_info->stream_urb)
 		return -ENOMEM;
-	}
 	/* transfer buffer allocated in board_init */
 	usb_fill_bulk_urb(pipe_info->stream_urb, dev->udev,
 			  pipe,
@@ -2290,10 +2287,8 @@ static int s2255_probe(struct usb_interface *interface,
 	}
 
 	dev->fw_data->fw_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->fw_data->fw_urb) {
-		dev_err(&interface->dev, "out of memory!\n");
+	if (!dev->fw_data->fw_urb)
 		goto errorFWURB;
-	}
 
 	dev->fw_data->pfw_data = kzalloc(CHUNK_SIZE, GFP_KERNEL);
 	if (!dev->fw_data->pfw_data) {
diff --git a/drivers/media/usb/stk1160/stk1160-i2c.c b/drivers/media/usb/stk1160/stk1160-i2c.c
index 850cf285ada8..3f2517be02bb 100644
--- a/drivers/media/usb/stk1160/stk1160-i2c.c
+++ b/drivers/media/usb/stk1160/stk1160-i2c.c
@@ -235,7 +235,7 @@ static u32 functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm algo = {
+static const struct i2c_algorithm algo = {
 	.master_xfer   = stk1160_i2c_xfer,
 	.functionality = functionality,
 };
diff --git a/drivers/media/usb/stk1160/stk1160-v4l.c b/drivers/media/usb/stk1160/stk1160-v4l.c
index 5fab3bee8c74..a005d262392a 100644
--- a/drivers/media/usb/stk1160/stk1160-v4l.c
+++ b/drivers/media/usb/stk1160/stk1160-v4l.c
@@ -742,7 +742,7 @@ static void stop_streaming(struct vb2_queue *vq)
 	stk1160_stop_streaming(dev);
 }
 
-static struct vb2_ops stk1160_video_qops = {
+static const struct vb2_ops stk1160_video_qops = {
 	.queue_setup		= queue_setup,
 	.buf_queue		= buffer_queue,
 	.start_streaming	= start_streaming,
diff --git a/drivers/media/usb/stk1160/stk1160-video.c b/drivers/media/usb/stk1160/stk1160-video.c
index 6ecb0b48423f..ce8ebbe395a6 100644
--- a/drivers/media/usb/stk1160/stk1160-video.c
+++ b/drivers/media/usb/stk1160/stk1160-video.c
@@ -457,10 +457,8 @@ int stk1160_alloc_isoc(struct stk1160 *dev)
 	for (i = 0; i < num_bufs; i++) {
 
 		urb = usb_alloc_urb(max_packets, GFP_KERNEL);
-		if (!urb) {
-			stk1160_err("cannot alloc urb[%d]\n", i);
+		if (!urb)
 			goto free_i_bufs;
-		}
 		dev->isoc_ctl.urb[i] = urb;
 
 #ifndef CONFIG_DMA_NONCOHERENT
diff --git a/drivers/media/usb/stkwebcam/stk-webcam.c b/drivers/media/usb/stkwebcam/stk-webcam.c
index c21c4c004f97..db200c9d796d 100644
--- a/drivers/media/usb/stkwebcam/stk-webcam.c
+++ b/drivers/media/usb/stkwebcam/stk-webcam.c
@@ -452,10 +452,8 @@ static int stk_prepare_iso(struct stk_camera *dev)
 			STK_ERROR("isobuf data already allocated\n");
 		if (dev->isobufs[i].urb == NULL) {
 			urb = usb_alloc_urb(ISO_FRAMES_PER_DESC, GFP_KERNEL);
-			if (urb == NULL) {
-				STK_ERROR("Failed to allocate URB %d\n", i);
+			if (urb == NULL)
 				goto isobufs_out;
-			}
 			dev->isobufs[i].urb = urb;
 		} else {
 			STK_ERROR("Killing URB\n");
diff --git a/drivers/media/usb/tm6000/tm6000-alsa.c b/drivers/media/usb/tm6000/tm6000-alsa.c
index e21c7aacecb6..f16fbd1f9f51 100644
--- a/drivers/media/usb/tm6000/tm6000-alsa.c
+++ b/drivers/media/usb/tm6000/tm6000-alsa.c
@@ -388,7 +388,7 @@ static struct page *snd_pcm_get_vmalloc_page(struct snd_pcm_substream *subs,
 /*
  * operators
  */
-static struct snd_pcm_ops snd_tm6000_pcm_ops = {
+static const struct snd_pcm_ops snd_tm6000_pcm_ops = {
 	.open = snd_tm6000_pcm_open,
 	.close = snd_tm6000_close,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/media/usb/tm6000/tm6000-dvb.c b/drivers/media/usb/tm6000/tm6000-dvb.c
index 095f5db1a790..0426b210383b 100644
--- a/drivers/media/usb/tm6000/tm6000-dvb.c
+++ b/drivers/media/usb/tm6000/tm6000-dvb.c
@@ -129,10 +129,8 @@ static int tm6000_start_stream(struct tm6000_core *dev)
 	}
 
 	dvb->bulk_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (dvb->bulk_urb == NULL) {
-		printk(KERN_ERR "tm6000: couldn't allocate urb\n");
+	if (dvb->bulk_urb == NULL)
 		return -ENOMEM;
-	}
 
 	pipe = usb_rcvbulkpipe(dev->udev, dev->bulk_in.endp->desc.bEndpointAddress
 							  & USB_ENDPOINT_NUMBER_MASK);
diff --git a/drivers/media/usb/tm6000/tm6000-video.c b/drivers/media/usb/tm6000/tm6000-video.c
index fa5e8bda2ae4..dee7e7d3d47d 100644
--- a/drivers/media/usb/tm6000/tm6000-video.c
+++ b/drivers/media/usb/tm6000/tm6000-video.c
@@ -635,7 +635,6 @@ static int tm6000_prepare_isoc(struct tm6000_core *dev)
 	for (i = 0; i < dev->isoc_ctl.num_bufs; i++) {
 		urb = usb_alloc_urb(max_packets, GFP_KERNEL);
 		if (!urb) {
-			tm6000_err("cannot alloc isoc_ctl.urb %i\n", i);
 			tm6000_uninit_isoc(dev);
 			usb_free_urb(urb);
 			return -ENOMEM;
diff --git a/drivers/media/usb/ttusb-dec/ttusb_dec.c b/drivers/media/usb/ttusb-dec/ttusb_dec.c
index 4e36e24cb3a6..4e7671a3a1e4 100644
--- a/drivers/media/usb/ttusb-dec/ttusb_dec.c
+++ b/drivers/media/usb/ttusb-dec/ttusb_dec.c
@@ -206,7 +206,7 @@ static void ttusb_dec_set_model(struct ttusb_dec *dec,
 
 static void ttusb_dec_handle_irq( struct urb *urb)
 {
-	struct ttusb_dec * dec = urb->context;
+	struct ttusb_dec *dec = urb->context;
 	char *buffer = dec->irq_buffer;
 	int retval;
 
@@ -227,25 +227,31 @@ static void ttusb_dec_handle_irq( struct urb *urb)
 			goto exit;
 	}
 
-	if( (buffer[0] == 0x1) && (buffer[2] == 0x15) )  {
-		/* IR - Event */
-		/* this is an fact a bit too simple implementation;
+	if ((buffer[0] == 0x1) && (buffer[2] == 0x15))  {
+		/*
+		 * IR - Event
+		 *
+		 * this is an fact a bit too simple implementation;
 		 * the box also reports a keyrepeat signal
 		 * (with buffer[3] == 0x40) in an intervall of ~100ms.
 		 * But to handle this correctly we had to imlemenent some
 		 * kind of timer which signals a 'key up' event if no
 		 * keyrepeat signal is received for lets say 200ms.
 		 * this should/could be added later ...
-		 * for now lets report each signal as a key down and up*/
-		dprintk("%s:rc signal:%d\n", __func__, buffer[4]);
-		input_report_key(dec->rc_input_dev, rc_keys[buffer[4] - 1], 1);
-		input_sync(dec->rc_input_dev);
-		input_report_key(dec->rc_input_dev, rc_keys[buffer[4] - 1], 0);
-		input_sync(dec->rc_input_dev);
+		 * for now lets report each signal as a key down and up
+		 */
+		if (buffer[4] - 1 < ARRAY_SIZE(rc_keys)) {
+			dprintk("%s:rc signal:%d\n", __func__, buffer[4]);
+			input_report_key(dec->rc_input_dev, rc_keys[buffer[4] - 1], 1);
+			input_sync(dec->rc_input_dev);
+			input_report_key(dec->rc_input_dev, rc_keys[buffer[4] - 1], 0);
+			input_sync(dec->rc_input_dev);
+		}
 	}
 
-exit:	retval = usb_submit_urb(urb, GFP_ATOMIC);
-	if(retval)
+exit:
+	retval = usb_submit_urb(urb, GFP_ATOMIC);
+	if (retval)
 		printk("%s - usb_commit_urb failed with result: %d\n",
 			__func__, retval);
 }
diff --git a/drivers/media/usb/usbtv/usbtv-audio.c b/drivers/media/usb/usbtv/usbtv-audio.c
index 1965ff1b1f12..9db31db7d9ac 100644
--- a/drivers/media/usb/usbtv/usbtv-audio.c
+++ b/drivers/media/usb/usbtv/usbtv-audio.c
@@ -332,7 +332,7 @@ static snd_pcm_uframes_t snd_usbtv_pointer(struct snd_pcm_substream *substream)
 	return chip->snd_buffer_pos;
 }
 
-static struct snd_pcm_ops snd_usbtv_pcm_ops = {
+static const struct snd_pcm_ops snd_usbtv_pcm_ops = {
 	.open = snd_usbtv_pcm_open,
 	.close = snd_usbtv_pcm_close,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/media/usb/usbtv/usbtv-video.c b/drivers/media/usb/usbtv/usbtv-video.c
index 2a089756c988..6cbe4a245c9f 100644
--- a/drivers/media/usb/usbtv/usbtv-video.c
+++ b/drivers/media/usb/usbtv/usbtv-video.c
@@ -689,7 +689,7 @@ static void usbtv_stop_streaming(struct vb2_queue *vq)
 		usbtv_stop(usbtv);
 }
 
-static struct vb2_ops usbtv_vb2_ops = {
+static const struct vb2_ops usbtv_vb2_ops = {
 	.queue_setup = usbtv_queue_setup,
 	.buf_queue = usbtv_buf_queue,
 	.start_streaming = usbtv_start_streaming,
diff --git a/drivers/media/usb/usbvision/usbvision-core.c b/drivers/media/usb/usbvision/usbvision-core.c
index 52ac4391582c..c23bf73a68ea 100644
--- a/drivers/media/usb/usbvision/usbvision-core.c
+++ b/drivers/media/usb/usbvision/usbvision-core.c
@@ -2303,11 +2303,8 @@ int usbvision_init_isoc(struct usb_usbvision *usbvision)
 		struct urb *urb;
 
 		urb = usb_alloc_urb(USBVISION_URB_FRAMES, GFP_KERNEL);
-		if (urb == NULL) {
-			dev_err(&usbvision->dev->dev,
-				"%s: usb_alloc_urb() failed\n", __func__);
+		if (urb == NULL)
 			return -ENOMEM;
-		}
 		usbvision->sbuf[buf_idx].urb = urb;
 		usbvision->sbuf[buf_idx].data =
 			usb_alloc_coherent(usbvision->dev,
diff --git a/drivers/media/usb/uvc/uvc_queue.c b/drivers/media/usb/uvc/uvc_queue.c
index 773fefb52d7a..77edd206d345 100644
--- a/drivers/media/usb/uvc/uvc_queue.c
+++ b/drivers/media/usb/uvc/uvc_queue.c
@@ -177,7 +177,7 @@ static void uvc_stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irqrestore(&queue->irqlock, flags);
 }
 
-static struct vb2_ops uvc_queue_qops = {
+static const struct vb2_ops uvc_queue_qops = {
 	.queue_setup = uvc_queue_setup,
 	.buf_prepare = uvc_buffer_prepare,
 	.buf_queue = uvc_buffer_queue,
diff --git a/drivers/media/usb/zr364xx/zr364xx.c b/drivers/media/usb/zr364xx/zr364xx.c
index 7433ba5c4bad..cc128db85723 100644
--- a/drivers/media/usb/zr364xx/zr364xx.c
+++ b/drivers/media/usb/zr364xx/zr364xx.c
@@ -1045,10 +1045,8 @@ static int zr364xx_start_readpipe(struct zr364xx_camera *cam)
 	pipe_info->state = 1;
 	pipe_info->err_count = 0;
 	pipe_info->stream_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!pipe_info->stream_urb) {
-		dev_err(&cam->udev->dev, "ReadStream: Unable to alloc URB\n");
+	if (!pipe_info->stream_urb)
 		return -ENOMEM;
-	}
 	/* transfer buffer allocated in board_init */
 	usb_fill_bulk_urb(pipe_info->stream_urb, cam->udev,
 			  pipe,
diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c
index a4b224d92572..5bada202b2d3 100644
--- a/drivers/media/v4l2-core/v4l2-async.c
+++ b/drivers/media/v4l2-core/v4l2-async.c
@@ -119,13 +119,6 @@ static int v4l2_async_test_notify(struct v4l2_async_notifier *notifier,
 		return ret;
 	}
 
-	ret = v4l2_subdev_call(sd, core, registered_async);
-	if (ret < 0 && ret != -ENOIOCTLCMD) {
-		if (notifier->unbind)
-			notifier->unbind(notifier, sd, asd);
-		return ret;
-	}
-
 	if (list_empty(&notifier->waiting) && notifier->complete)
 		return notifier->complete(notifier);
 
diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c
index 5b808500e7e7..57cfe26a393f 100644
--- a/drivers/media/v4l2-core/v4l2-common.c
+++ b/drivers/media/v4l2-core/v4l2-common.c
@@ -291,7 +291,7 @@ struct v4l2_subdev *v4l2_spi_new_subdev(struct v4l2_device *v4l2_dev,
 error:
 	/* If we have a client but no subdev, then something went wrong and
 	   we must unregister the client. */
-	if (spi && sd == NULL)
+	if (!sd)
 		spi_unregister_device(spi);
 
 	return sd;
diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c
index f7abfad9ad23..adc2147fcff7 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls.c
@@ -361,6 +361,7 @@ const char * const *v4l2_ctrl_get_menu(u32 id)
 		"Scalable Baseline",
 		"Scalable High",
 		"Scalable High Intra",
+		"Stereo High",
 		"Multiview High",
 		NULL,
 	};
diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c
index e6da353b39bc..8be561ab2615 100644
--- a/drivers/media/v4l2-core/v4l2-dev.c
+++ b/drivers/media/v4l2-core/v4l2-dev.c
@@ -527,6 +527,7 @@ static void determine_valid_ioctls(struct video_device *vdev)
 	bool is_vbi = vdev->vfl_type == VFL_TYPE_VBI;
 	bool is_radio = vdev->vfl_type == VFL_TYPE_RADIO;
 	bool is_sdr = vdev->vfl_type == VFL_TYPE_SDR;
+	bool is_tch = vdev->vfl_type == VFL_TYPE_TOUCH;
 	bool is_rx = vdev->vfl_dir != VFL_DIR_TX;
 	bool is_tx = vdev->vfl_dir != VFL_DIR_RX;
 
@@ -573,7 +574,7 @@ static void determine_valid_ioctls(struct video_device *vdev)
 	if (ops->vidioc_enum_freq_bands || ops->vidioc_g_tuner || ops->vidioc_g_modulator)
 		set_bit(_IOC_NR(VIDIOC_ENUM_FREQ_BANDS), valid_ioctls);
 
-	if (is_vid) {
+	if (is_vid || is_tch) {
 		/* video specific ioctls */
 		if ((is_rx && (ops->vidioc_enum_fmt_vid_cap ||
 			       ops->vidioc_enum_fmt_vid_cap_mplane ||
@@ -662,7 +663,7 @@ static void determine_valid_ioctls(struct video_device *vdev)
 			set_bit(_IOC_NR(VIDIOC_TRY_FMT), valid_ioctls);
 	}
 
-	if (is_vid || is_vbi || is_sdr) {
+	if (is_vid || is_vbi || is_sdr || is_tch) {
 		/* ioctls valid for video, vbi or sdr */
 		SET_VALID_IOCTL(ops, VIDIOC_REQBUFS, vidioc_reqbufs);
 		SET_VALID_IOCTL(ops, VIDIOC_QUERYBUF, vidioc_querybuf);
@@ -675,7 +676,7 @@ static void determine_valid_ioctls(struct video_device *vdev)
 		SET_VALID_IOCTL(ops, VIDIOC_STREAMOFF, vidioc_streamoff);
 	}
 
-	if (is_vid || is_vbi) {
+	if (is_vid || is_vbi || is_tch) {
 		/* ioctls valid for video or vbi */
 		if (ops->vidioc_s_std)
 			set_bit(_IOC_NR(VIDIOC_ENUMSTD), valid_ioctls);
@@ -751,6 +752,10 @@ static int video_register_media_controller(struct video_device *vdev, int type)
 		intf_type = MEDIA_INTF_T_V4L_SWRADIO;
 		vdev->entity.function = MEDIA_ENT_F_IO_SWRADIO;
 		break;
+	case VFL_TYPE_TOUCH:
+		intf_type = MEDIA_INTF_T_V4L_TOUCH;
+		vdev->entity.function = MEDIA_ENT_F_IO_V4L;
+		break;
 	case VFL_TYPE_RADIO:
 		intf_type = MEDIA_INTF_T_V4L_RADIO;
 		/*
@@ -854,6 +859,9 @@ int __video_register_device(struct video_device *vdev, int type, int nr,
 		/* Use device name 'swradio' because 'sdr' was already taken. */
 		name_base = "swradio";
 		break;
+	case VFL_TYPE_TOUCH:
+		name_base = "v4l-touch";
+		break;
 	default:
 		printk(KERN_ERR "%s called with unknown type: %d\n",
 		       __func__, type);
diff --git a/drivers/media/v4l2-core/v4l2-device.c b/drivers/media/v4l2-core/v4l2-device.c
index 06fa5f1b2cff..62bbed76dbbc 100644
--- a/drivers/media/v4l2-core/v4l2-device.c
+++ b/drivers/media/v4l2-core/v4l2-device.c
@@ -160,12 +160,9 @@ int v4l2_device_register_subdev(struct v4l2_device *v4l2_dev,
 	int err;
 
 	/* Check for valid input */
-	if (v4l2_dev == NULL || sd == NULL || !sd->name[0])
+	if (!v4l2_dev || !sd || sd->v4l2_dev || !sd->name[0])
 		return -EINVAL;
 
-	/* Warn if we apparently re-register a subdev */
-	WARN_ON(sd->v4l2_dev != NULL);
-
 	/*
 	 * The reason to acquire the module here is to avoid unloading
 	 * a module of sub-device which is registered to a media
diff --git a/drivers/media/v4l2-core/v4l2-dv-timings.c b/drivers/media/v4l2-core/v4l2-dv-timings.c
index 889de0a32152..730a7c392c1d 100644
--- a/drivers/media/v4l2-core/v4l2-dv-timings.c
+++ b/drivers/media/v4l2-core/v4l2-dv-timings.c
@@ -306,7 +306,7 @@ void v4l2_print_dv_timings(const char *dev_prefix, const char *prefix,
 			(bt->polarities & V4L2_DV_VSYNC_POS_POL) ? "+" : "-",
 			bt->il_vsync, bt->il_vbackporch);
 	pr_info("%s: pixelclock: %llu\n", dev_prefix, bt->pixelclock);
-	pr_info("%s: flags (0x%x):%s%s%s%s%s%s\n", dev_prefix, bt->flags,
+	pr_info("%s: flags (0x%x):%s%s%s%s%s%s%s\n", dev_prefix, bt->flags,
 			(bt->flags & V4L2_DV_FL_REDUCED_BLANKING) ?
 			" REDUCED_BLANKING" : "",
 			((bt->flags & V4L2_DV_FL_REDUCED_BLANKING) &&
@@ -318,12 +318,15 @@ void v4l2_print_dv_timings(const char *dev_prefix, const char *prefix,
 			(bt->flags & V4L2_DV_FL_HALF_LINE) ?
 			" HALF_LINE" : "",
 			(bt->flags & V4L2_DV_FL_IS_CE_VIDEO) ?
-			" CE_VIDEO" : "");
-	pr_info("%s: standards (0x%x):%s%s%s%s\n", dev_prefix, bt->standards,
+			" CE_VIDEO" : "",
+			(bt->flags & V4L2_DV_FL_FIRST_FIELD_EXTRA_LINE) ?
+			" FIRST_FIELD_EXTRA_LINE" : "");
+	pr_info("%s: standards (0x%x):%s%s%s%s%s\n", dev_prefix, bt->standards,
 			(bt->standards & V4L2_DV_BT_STD_CEA861) ?  " CEA" : "",
 			(bt->standards & V4L2_DV_BT_STD_DMT) ?  " DMT" : "",
 			(bt->standards & V4L2_DV_BT_STD_CVT) ?  " CVT" : "",
-			(bt->standards & V4L2_DV_BT_STD_GTF) ?  " GTF" : "");
+			(bt->standards & V4L2_DV_BT_STD_GTF) ?  " GTF" : "",
+			(bt->standards & V4L2_DV_BT_STD_SDI) ?  " SDI" : "");
 }
 EXPORT_SYMBOL_GPL(v4l2_print_dv_timings);
 
diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
index 51a0fa144392..c52d94c018bb 100644
--- a/drivers/media/v4l2-core/v4l2-ioctl.c
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c
@@ -924,6 +924,7 @@ static int check_fmt(struct file *file, enum v4l2_buf_type type)
 	bool is_vid = vfd->vfl_type == VFL_TYPE_GRABBER;
 	bool is_vbi = vfd->vfl_type == VFL_TYPE_VBI;
 	bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR;
+	bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
 	bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
 	bool is_tx = vfd->vfl_dir != VFL_DIR_RX;
 
@@ -932,7 +933,7 @@ static int check_fmt(struct file *file, enum v4l2_buf_type type)
 
 	switch (type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		if (is_vid && is_rx &&
+		if ((is_vid || is_tch) && is_rx &&
 		    (ops->vidioc_g_fmt_vid_cap || ops->vidioc_g_fmt_vid_cap_mplane))
 			return 0;
 		break;
@@ -1243,6 +1244,10 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_SDR_FMT_CS8:		descr = "Complex S8"; break;
 	case V4L2_SDR_FMT_CS14LE:	descr = "Complex S14LE"; break;
 	case V4L2_SDR_FMT_RU12LE:	descr = "Real U12LE"; break;
+	case V4L2_TCH_FMT_DELTA_TD16:	descr = "16-bit signed deltas"; break;
+	case V4L2_TCH_FMT_DELTA_TD08:	descr = "8-bit signed deltas"; break;
+	case V4L2_TCH_FMT_TU16:		descr = "16-bit unsigned touch data"; break;
+	case V4L2_TCH_FMT_TU08:		descr = "8-bit unsigned touch data"; break;
 
 	default:
 		/* Compressed formats */
@@ -1309,13 +1314,14 @@ static int v4l_enum_fmt(const struct v4l2_ioctl_ops *ops,
 	struct video_device *vfd = video_devdata(file);
 	bool is_vid = vfd->vfl_type == VFL_TYPE_GRABBER;
 	bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR;
+	bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
 	bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
 	bool is_tx = vfd->vfl_dir != VFL_DIR_RX;
 	int ret = -EINVAL;
 
 	switch (p->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_enum_fmt_vid_cap))
+		if (unlikely(!is_rx || (!is_vid && !is_tch) || !ops->vidioc_enum_fmt_vid_cap))
 			break;
 		ret = ops->vidioc_enum_fmt_vid_cap(file, fh, arg);
 		break;
@@ -1362,6 +1368,7 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops,
 	struct video_device *vfd = video_devdata(file);
 	bool is_vid = vfd->vfl_type == VFL_TYPE_GRABBER;
 	bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR;
+	bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
 	bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
 	bool is_tx = vfd->vfl_dir != VFL_DIR_RX;
 	int ret;
@@ -1392,7 +1399,7 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops,
 
 	switch (p->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_g_fmt_vid_cap))
+		if (unlikely(!is_rx || (!is_vid && !is_tch) || !ops->vidioc_g_fmt_vid_cap))
 			break;
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
 		ret = ops->vidioc_g_fmt_vid_cap(file, fh, arg);
@@ -1451,6 +1458,21 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops,
 	return -EINVAL;
 }
 
+static void v4l_pix_format_touch(struct v4l2_pix_format *p)
+{
+	/*
+	 * The v4l2_pix_format structure contains fields that make no sense for
+	 * touch. Set them to default values in this case.
+	 */
+
+	p->field = V4L2_FIELD_NONE;
+	p->colorspace = V4L2_COLORSPACE_RAW;
+	p->flags = 0;
+	p->ycbcr_enc = 0;
+	p->quantization = 0;
+	p->xfer_func = 0;
+}
+
 static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops,
 				struct file *file, void *fh, void *arg)
 {
@@ -1458,6 +1480,7 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops,
 	struct video_device *vfd = video_devdata(file);
 	bool is_vid = vfd->vfl_type == VFL_TYPE_GRABBER;
 	bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR;
+	bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
 	bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
 	bool is_tx = vfd->vfl_dir != VFL_DIR_RX;
 	int ret;
@@ -1469,17 +1492,19 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops,
 
 	switch (p->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_s_fmt_vid_cap))
+		if (unlikely(!is_rx || (!is_vid && !is_tch) || !ops->vidioc_s_fmt_vid_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix);
 		ret = ops->vidioc_s_fmt_vid_cap(file, fh, arg);
 		/* just in case the driver zeroed it again */
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
+		if (is_tch)
+			v4l_pix_format_touch(&p->fmt.pix);
 		return ret;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 		if (unlikely(!is_rx || !is_vid || !ops->vidioc_s_fmt_vid_cap_mplane))
 			break;
-		CLEAR_AFTER_FIELD(p, fmt.pix_mp);
+		CLEAR_AFTER_FIELD(p, fmt.pix_mp.xfer_func);
 		return ops->vidioc_s_fmt_vid_cap_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
 		if (unlikely(!is_rx || !is_vid || !ops->vidioc_s_fmt_vid_overlay))
@@ -1507,7 +1532,7 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops,
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 		if (unlikely(!is_tx || !is_vid || !ops->vidioc_s_fmt_vid_out_mplane))
 			break;
-		CLEAR_AFTER_FIELD(p, fmt.pix_mp);
+		CLEAR_AFTER_FIELD(p, fmt.pix_mp.xfer_func);
 		return ops->vidioc_s_fmt_vid_out_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
 		if (unlikely(!is_tx || !is_vid || !ops->vidioc_s_fmt_vid_out_overlay))
@@ -1545,6 +1570,7 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops,
 	struct video_device *vfd = video_devdata(file);
 	bool is_vid = vfd->vfl_type == VFL_TYPE_GRABBER;
 	bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR;
+	bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
 	bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
 	bool is_tx = vfd->vfl_dir != VFL_DIR_RX;
 	int ret;
@@ -1553,7 +1579,7 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops,
 
 	switch (p->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_try_fmt_vid_cap))
+		if (unlikely(!is_rx || (!is_vid && !is_tch) || !ops->vidioc_try_fmt_vid_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix);
 		ret = ops->vidioc_try_fmt_vid_cap(file, fh, arg);
@@ -1563,7 +1589,7 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops,
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 		if (unlikely(!is_rx || !is_vid || !ops->vidioc_try_fmt_vid_cap_mplane))
 			break;
-		CLEAR_AFTER_FIELD(p, fmt.pix_mp);
+		CLEAR_AFTER_FIELD(p, fmt.pix_mp.xfer_func);
 		return ops->vidioc_try_fmt_vid_cap_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
 		if (unlikely(!is_rx || !is_vid || !ops->vidioc_try_fmt_vid_overlay))
@@ -1591,7 +1617,7 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops,
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 		if (unlikely(!is_tx || !is_vid || !ops->vidioc_try_fmt_vid_out_mplane))
 			break;
-		CLEAR_AFTER_FIELD(p, fmt.pix_mp);
+		CLEAR_AFTER_FIELD(p, fmt.pix_mp.xfer_func);
 		return ops->vidioc_try_fmt_vid_out_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
 		if (unlikely(!is_tx || !is_vid || !ops->vidioc_try_fmt_vid_out_overlay))
diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem.c
index 61d56c940f80..6bc27e7b2a33 100644
--- a/drivers/media/v4l2-core/v4l2-mem2mem.c
+++ b/drivers/media/v4l2-core/v4l2-mem2mem.c
@@ -76,9 +76,6 @@ static struct v4l2_m2m_queue_ctx *get_queue_ctx(struct v4l2_m2m_ctx *m2m_ctx,
 		return &m2m_ctx->cap_q_ctx;
 }
 
-/**
- * v4l2_m2m_get_vq() - return vb2_queue for the given type
- */
 struct vb2_queue *v4l2_m2m_get_vq(struct v4l2_m2m_ctx *m2m_ctx,
 				       enum v4l2_buf_type type)
 {
@@ -92,9 +89,6 @@ struct vb2_queue *v4l2_m2m_get_vq(struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL(v4l2_m2m_get_vq);
 
-/**
- * v4l2_m2m_next_buf() - return next buffer from the list of ready buffers
- */
 void *v4l2_m2m_next_buf(struct v4l2_m2m_queue_ctx *q_ctx)
 {
 	struct v4l2_m2m_buffer *b;
@@ -113,10 +107,6 @@ void *v4l2_m2m_next_buf(struct v4l2_m2m_queue_ctx *q_ctx)
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_next_buf);
 
-/**
- * v4l2_m2m_buf_remove() - take off a buffer from the list of ready buffers and
- * return it
- */
 void *v4l2_m2m_buf_remove(struct v4l2_m2m_queue_ctx *q_ctx)
 {
 	struct v4l2_m2m_buffer *b;
@@ -140,10 +130,6 @@ EXPORT_SYMBOL_GPL(v4l2_m2m_buf_remove);
  * Scheduling handlers
  */
 
-/**
- * v4l2_m2m_get_curr_priv() - return driver private data for the currently
- * running instance or NULL if no instance is running
- */
 void *v4l2_m2m_get_curr_priv(struct v4l2_m2m_dev *m2m_dev)
 {
 	unsigned long flags;
@@ -188,26 +174,6 @@ static void v4l2_m2m_try_run(struct v4l2_m2m_dev *m2m_dev)
 	m2m_dev->m2m_ops->device_run(m2m_dev->curr_ctx->priv);
 }
 
-/**
- * v4l2_m2m_try_schedule() - check whether an instance is ready to be added to
- * the pending job queue and add it if so.
- * @m2m_ctx:	m2m context assigned to the instance to be checked
- *
- * There are three basic requirements an instance has to meet to be able to run:
- * 1) at least one source buffer has to be queued,
- * 2) at least one destination buffer has to be queued,
- * 3) streaming has to be on.
- *
- * If a queue is buffered (for example a decoder hardware ringbuffer that has
- * to be drained before doing streamoff), allow scheduling without v4l2 buffers
- * on that queue.
- *
- * There may also be additional, custom requirements. In such case the driver
- * should supply a custom callback (job_ready in v4l2_m2m_ops) that should
- * return 1 if the instance is ready.
- * An example of the above could be an instance that requires more than one
- * src/dst buffer per transaction.
- */
 void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *m2m_ctx)
 {
 	struct v4l2_m2m_dev *m2m_dev;
@@ -311,18 +277,6 @@ static void v4l2_m2m_cancel_job(struct v4l2_m2m_ctx *m2m_ctx)
 	}
 }
 
-/**
- * v4l2_m2m_job_finish() - inform the framework that a job has been finished
- * and have it clean up
- *
- * Called by a driver to yield back the device after it has finished with it.
- * Should be called as soon as possible after reaching a state which allows
- * other instances to take control of the device.
- *
- * This function has to be called only after device_run() callback has been
- * called on the driver. To prevent recursion, it should not be called directly
- * from the device_run() callback though.
- */
 void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
 			 struct v4l2_m2m_ctx *m2m_ctx)
 {
@@ -350,9 +304,6 @@ void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
 }
 EXPORT_SYMBOL(v4l2_m2m_job_finish);
 
-/**
- * v4l2_m2m_reqbufs() - multi-queue-aware REQBUFS multiplexer
- */
 int v4l2_m2m_reqbufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		     struct v4l2_requestbuffers *reqbufs)
 {
@@ -370,11 +321,6 @@ int v4l2_m2m_reqbufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_reqbufs);
 
-/**
- * v4l2_m2m_querybuf() - multi-queue-aware QUERYBUF multiplexer
- *
- * See v4l2_m2m_mmap() documentation for details.
- */
 int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		      struct v4l2_buffer *buf)
 {
@@ -400,10 +346,6 @@ int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_querybuf);
 
-/**
- * v4l2_m2m_qbuf() - enqueue a source or destination buffer, depending on
- * the type
- */
 int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		  struct v4l2_buffer *buf)
 {
@@ -419,10 +361,6 @@ int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_qbuf);
 
-/**
- * v4l2_m2m_dqbuf() - dequeue a source or destination buffer, depending on
- * the type
- */
 int v4l2_m2m_dqbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		   struct v4l2_buffer *buf)
 {
@@ -433,10 +371,6 @@ int v4l2_m2m_dqbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_dqbuf);
 
-/**
- * v4l2_m2m_prepare_buf() - prepare a source or destination buffer, depending on
- * the type
- */
 int v4l2_m2m_prepare_buf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 			 struct v4l2_buffer *buf)
 {
@@ -452,10 +386,6 @@ int v4l2_m2m_prepare_buf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_prepare_buf);
 
-/**
- * v4l2_m2m_create_bufs() - create a source or destination buffer, depending
- * on the type
- */
 int v4l2_m2m_create_bufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 			 struct v4l2_create_buffers *create)
 {
@@ -466,10 +396,6 @@ int v4l2_m2m_create_bufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_create_bufs);
 
-/**
- * v4l2_m2m_expbuf() - export a source or destination buffer, depending on
- * the type
- */
 int v4l2_m2m_expbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		  struct v4l2_exportbuffer *eb)
 {
@@ -479,9 +405,7 @@ int v4l2_m2m_expbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 	return vb2_expbuf(vq, eb);
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_expbuf);
-/**
- * v4l2_m2m_streamon() - turn on streaming for a video queue
- */
+
 int v4l2_m2m_streamon(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		      enum v4l2_buf_type type)
 {
@@ -497,9 +421,6 @@ int v4l2_m2m_streamon(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_streamon);
 
-/**
- * v4l2_m2m_streamoff() - turn off streaming for a video queue
- */
 int v4l2_m2m_streamoff(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		       enum v4l2_buf_type type)
 {
@@ -540,14 +461,6 @@ int v4l2_m2m_streamoff(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_streamoff);
 
-/**
- * v4l2_m2m_poll() - poll replacement, for destination buffers only
- *
- * Call from the driver's poll() function. Will poll both queues. If a buffer
- * is available to dequeue (with dqbuf) from the source queue, this will
- * indicate that a non-blocking write can be performed, while read will be
- * returned in case of the destination queue.
- */
 unsigned int v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 			   struct poll_table_struct *wait)
 {
@@ -626,16 +539,6 @@ end:
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_poll);
 
-/**
- * v4l2_m2m_mmap() - source and destination queues-aware mmap multiplexer
- *
- * Call from driver's mmap() function. Will handle mmap() for both queues
- * seamlessly for videobuffer, which will receive normal per-queue offsets and
- * proper videobuf queue pointers. The differentiation is made outside videobuf
- * by adding a predefined offset to buffers from one of the queues and
- * subtracting it before passing it back to videobuf. Only drivers (and
- * thus applications) receive modified offsets.
- */
 int v4l2_m2m_mmap(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 			 struct vm_area_struct *vma)
 {
@@ -653,11 +556,6 @@ int v4l2_m2m_mmap(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL(v4l2_m2m_mmap);
 
-/**
- * v4l2_m2m_init() - initialize per-driver m2m data
- *
- * Usually called from driver's probe() function.
- */
 struct v4l2_m2m_dev *v4l2_m2m_init(const struct v4l2_m2m_ops *m2m_ops)
 {
 	struct v4l2_m2m_dev *m2m_dev;
@@ -679,26 +577,12 @@ struct v4l2_m2m_dev *v4l2_m2m_init(const struct v4l2_m2m_ops *m2m_ops)
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_init);
 
-/**
- * v4l2_m2m_release() - cleans up and frees a m2m_dev structure
- *
- * Usually called from driver's remove() function.
- */
 void v4l2_m2m_release(struct v4l2_m2m_dev *m2m_dev)
 {
 	kfree(m2m_dev);
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_release);
 
-/**
- * v4l2_m2m_ctx_init() - allocate and initialize a m2m context
- * @priv - driver's instance private data
- * @m2m_dev - a previously initialized m2m_dev struct
- * @vq_init - a callback for queue type-specific initialization function to be
- * used for initializing videobuf_queues
- *
- * Usually called from driver's open() function.
- */
 struct v4l2_m2m_ctx *v4l2_m2m_ctx_init(struct v4l2_m2m_dev *m2m_dev,
 		void *drv_priv,
 		int (*queue_init)(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq))
@@ -744,11 +628,6 @@ err:
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_ctx_init);
 
-/**
- * v4l2_m2m_ctx_release() - release m2m context
- *
- * Usually called from driver's release() function.
- */
 void v4l2_m2m_ctx_release(struct v4l2_m2m_ctx *m2m_ctx)
 {
 	/* wait until the current context is dequeued from job_queue */
@@ -761,11 +640,6 @@ void v4l2_m2m_ctx_release(struct v4l2_m2m_ctx *m2m_ctx)
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_ctx_release);
 
-/**
- * v4l2_m2m_buf_queue() - add a buffer to the proper ready buffers list.
- *
- * Call from buf_queue(), videobuf_queue_ops callback.
- */
 void v4l2_m2m_buf_queue(struct v4l2_m2m_ctx *m2m_ctx,
 		struct vb2_v4l2_buffer *vbuf)
 {
diff --git a/drivers/media/v4l2-core/videobuf2-core.c b/drivers/media/v4l2-core/videobuf2-core.c
index ca8ffeb56d72..21900202ff83 100644
--- a/drivers/media/v4l2-core/videobuf2-core.c
+++ b/drivers/media/v4l2-core/videobuf2-core.c
@@ -198,6 +198,7 @@ static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
 		q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 	void *mem_priv;
 	int plane;
+	int ret = -ENOMEM;
 
 	/*
 	 * Allocate memory for all planes in this buffer
@@ -209,8 +210,11 @@ static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
 		mem_priv = call_ptr_memop(vb, alloc,
 				q->alloc_devs[plane] ? : q->dev,
 				q->dma_attrs, size, dma_dir, q->gfp_flags);
-		if (IS_ERR_OR_NULL(mem_priv))
+		if (IS_ERR(mem_priv)) {
+			if (mem_priv)
+				ret = PTR_ERR(mem_priv);
 			goto free;
+		}
 
 		/* Associate allocator private data with this plane */
 		vb->planes[plane].mem_priv = mem_priv;
@@ -224,7 +228,7 @@ free:
 		vb->planes[plane - 1].mem_priv = NULL;
 	}
 
-	return -ENOMEM;
+	return ret;
 }
 
 /**
@@ -524,10 +528,6 @@ static int __vb2_queue_free(struct vb2_queue *q, unsigned int buffers)
 	return 0;
 }
 
-/**
- * vb2_buffer_in_use() - return true if the buffer is in use and
- * the queue cannot be freed (by the means of REQBUFS(0)) call
- */
 bool vb2_buffer_in_use(struct vb2_queue *q, struct vb2_buffer *vb)
 {
 	unsigned int plane;
@@ -560,16 +560,6 @@ static bool __buffers_in_use(struct vb2_queue *q)
 	return false;
 }
 
-/**
- * vb2_core_querybuf() - query video buffer information
- * @q:		videobuf queue
- * @index:	id number of the buffer
- * @pb:		buffer struct passed from userspace
- *
- * Should be called from vidioc_querybuf ioctl handler in driver.
- * The passed buffer should have been verified.
- * This function fills the relevant information for the userspace.
- */
 void vb2_core_querybuf(struct vb2_queue *q, unsigned int index, void *pb)
 {
 	call_void_bufop(q, fill_user_buffer, q->bufs[index], pb);
@@ -616,10 +606,6 @@ static int __verify_dmabuf_ops(struct vb2_queue *q)
 	return 0;
 }
 
-/**
- * vb2_verify_memory_type() - Check whether the memory type and buffer type
- * passed to a buffer operation are compatible with the queue.
- */
 int vb2_verify_memory_type(struct vb2_queue *q,
 		enum vb2_memory memory, unsigned int type)
 {
@@ -666,30 +652,6 @@ int vb2_verify_memory_type(struct vb2_queue *q,
 }
 EXPORT_SYMBOL(vb2_verify_memory_type);
 
-/**
- * vb2_core_reqbufs() - Initiate streaming
- * @q:		videobuf2 queue
- * @memory: memory type
- * @count: requested buffer count
- *
- * Should be called from vidioc_reqbufs ioctl handler of a driver.
- * This function:
- * 1) verifies streaming parameters passed from the userspace,
- * 2) sets up the queue,
- * 3) negotiates number of buffers and planes per buffer with the driver
- *    to be used during streaming,
- * 4) allocates internal buffer structures (struct vb2_buffer), according to
- *    the agreed parameters,
- * 5) for MMAP memory type, allocates actual video memory, using the
- *    memory handling/allocation routines provided during queue initialization
- *
- * If req->count is 0, all the memory will be freed instead.
- * If the queue has been allocated previously (by a previous vb2_reqbufs) call
- * and the queue is not busy, memory will be reallocated.
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_reqbufs handler in driver.
- */
 int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
 		unsigned int *count)
 {
@@ -815,22 +777,6 @@ int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
 }
 EXPORT_SYMBOL_GPL(vb2_core_reqbufs);
 
-/**
- * vb2_core_create_bufs() - Allocate buffers and any required auxiliary structs
- * @q:		videobuf2 queue
- * @memory: memory type
- * @count: requested buffer count
- * @parg: parameter passed to device driver
- *
- * Should be called from vidioc_create_bufs ioctl handler of a driver.
- * This function:
- * 1) verifies parameter sanity
- * 2) calls the .queue_setup() queue operation
- * 3) performs any necessary memory allocations
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_create_bufs handler in driver.
- */
 int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory,
 		unsigned int *count, unsigned requested_planes,
 		const unsigned requested_sizes[])
@@ -920,14 +866,6 @@ int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory,
 }
 EXPORT_SYMBOL_GPL(vb2_core_create_bufs);
 
-/**
- * vb2_plane_vaddr() - Return a kernel virtual address of a given plane
- * @vb:		vb2_buffer to which the plane in question belongs to
- * @plane_no:	plane number for which the address is to be returned
- *
- * This function returns a kernel virtual address of a given plane if
- * such a mapping exist, NULL otherwise.
- */
 void *vb2_plane_vaddr(struct vb2_buffer *vb, unsigned int plane_no)
 {
 	if (plane_no > vb->num_planes || !vb->planes[plane_no].mem_priv)
@@ -938,17 +876,6 @@ void *vb2_plane_vaddr(struct vb2_buffer *vb, unsigned int plane_no)
 }
 EXPORT_SYMBOL_GPL(vb2_plane_vaddr);
 
-/**
- * vb2_plane_cookie() - Return allocator specific cookie for the given plane
- * @vb:		vb2_buffer to which the plane in question belongs to
- * @plane_no:	plane number for which the cookie is to be returned
- *
- * This function returns an allocator specific cookie for a given plane if
- * available, NULL otherwise. The allocator should provide some simple static
- * inline function, which would convert this cookie to the allocator specific
- * type that can be used directly by the driver to access the buffer. This can
- * be for example physical address, pointer to scatter list or IOMMU mapping.
- */
 void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no)
 {
 	if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
@@ -958,26 +885,6 @@ void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no)
 }
 EXPORT_SYMBOL_GPL(vb2_plane_cookie);
 
-/**
- * vb2_buffer_done() - inform videobuf that an operation on a buffer is finished
- * @vb:		vb2_buffer returned from the driver
- * @state:	either VB2_BUF_STATE_DONE if the operation finished successfully,
- *		VB2_BUF_STATE_ERROR if the operation finished with an error or
- *		VB2_BUF_STATE_QUEUED if the driver wants to requeue buffers.
- *		If start_streaming fails then it should return buffers with state
- *		VB2_BUF_STATE_QUEUED to put them back into the queue.
- *
- * This function should be called by the driver after a hardware operation on
- * a buffer is finished and the buffer may be returned to userspace. The driver
- * cannot use this buffer anymore until it is queued back to it by videobuf
- * by the means of buf_queue callback. Only buffers previously queued to the
- * driver by buf_queue can be passed to this function.
- *
- * While streaming a buffer can only be returned in state DONE or ERROR.
- * The start_streaming op can also return them in case the DMA engine cannot
- * be started for some reason. In that case the buffers should be returned with
- * state QUEUED.
- */
 void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
 {
 	struct vb2_queue *q = vb->vb2_queue;
@@ -1036,18 +943,6 @@ void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
 }
 EXPORT_SYMBOL_GPL(vb2_buffer_done);
 
-/**
- * vb2_discard_done() - discard all buffers marked as DONE
- * @q:		videobuf2 queue
- *
- * This function is intended to be used with suspend/resume operations. It
- * discards all 'done' buffers as they would be too old to be requested after
- * resume.
- *
- * Drivers must stop the hardware and synchronize with interrupt handlers and/or
- * delayed works before calling this function to make sure no buffer will be
- * touched by the driver and/or hardware.
- */
 void vb2_discard_done(struct vb2_queue *q)
 {
 	struct vb2_buffer *vb;
@@ -1136,10 +1031,10 @@ static int __qbuf_userptr(struct vb2_buffer *vb, const void *pb)
 				q->alloc_devs[plane] ? : q->dev,
 				planes[plane].m.userptr,
 				planes[plane].length, dma_dir);
-		if (IS_ERR_OR_NULL(mem_priv)) {
+		if (IS_ERR(mem_priv)) {
 			dprintk(1, "failed acquiring userspace "
 						"memory for plane %d\n", plane);
-			ret = mem_priv ? PTR_ERR(mem_priv) : -EINVAL;
+			ret = PTR_ERR(mem_priv);
 			goto err;
 		}
 		vb->planes[plane].mem_priv = mem_priv;
@@ -1228,8 +1123,10 @@ static int __qbuf_dmabuf(struct vb2_buffer *vb, const void *pb)
 			planes[plane].length = dbuf->size;
 
 		if (planes[plane].length < vb->planes[plane].min_length) {
-			dprintk(1, "invalid dmabuf length for plane %d\n",
-				plane);
+			dprintk(1, "invalid dmabuf length %u for plane %d, "
+				"minimum length %u\n",
+				planes[plane].length, plane,
+				vb->planes[plane].min_length);
 			dma_buf_put(dbuf);
 			ret = -EINVAL;
 			goto err;
@@ -1271,9 +1168,10 @@ static int __qbuf_dmabuf(struct vb2_buffer *vb, const void *pb)
 		vb->planes[plane].mem_priv = mem_priv;
 	}
 
-	/* TODO: This pins the buffer(s) with  dma_buf_map_attachment()).. but
-	 * really we want to do this just before the DMA, not while queueing
-	 * the buffer(s)..
+	/*
+	 * This pins the buffer(s) with dma_buf_map_attachment()). It's done
+	 * here instead just before the DMA, while queueing the buffer(s) so
+	 * userspace knows sooner rather than later if the dma-buf map fails.
 	 */
 	for (plane = 0; plane < vb->num_planes; ++plane) {
 		ret = call_memop(vb, map_dmabuf, vb->planes[plane].mem_priv);
@@ -1377,22 +1275,6 @@ static int __buf_prepare(struct vb2_buffer *vb, const void *pb)
 	return ret;
 }
 
-/**
- * vb2_core_prepare_buf() - Pass ownership of a buffer from userspace
- *			to the kernel
- * @q:		videobuf2 queue
- * @index:	id number of the buffer
- * @pb:		buffer structure passed from userspace to vidioc_prepare_buf
- *		handler in driver
- *
- * Should be called from vidioc_prepare_buf ioctl handler of a driver.
- * The passed buffer should have been verified.
- * This function calls buf_prepare callback in the driver (if provided),
- * in which driver-specific buffer initialization can be performed,
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_prepare_buf handler in driver.
- */
 int vb2_core_prepare_buf(struct vb2_queue *q, unsigned int index, void *pb)
 {
 	struct vb2_buffer *vb;
@@ -1481,24 +1363,6 @@ static int vb2_start_streaming(struct vb2_queue *q)
 	return ret;
 }
 
-/**
- * vb2_core_qbuf() - Queue a buffer from userspace
- * @q:		videobuf2 queue
- * @index:	id number of the buffer
- * @pb:		buffer structure passed from userspace to vidioc_qbuf handler
- *		in driver
- *
- * Should be called from vidioc_qbuf ioctl handler of a driver.
- * The passed buffer should have been verified.
- * This function:
- * 1) if necessary, calls buf_prepare callback in the driver (if provided), in
- *    which driver-specific buffer initialization can be performed,
- * 2) if streaming is on, queues the buffer in driver by the means of buf_queue
- *    callback for processing.
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_qbuf handler in driver.
- */
 int vb2_core_qbuf(struct vb2_queue *q, unsigned int index, void *pb)
 {
 	struct vb2_buffer *vb;
@@ -1679,15 +1543,6 @@ static int __vb2_get_done_vb(struct vb2_queue *q, struct vb2_buffer **vb,
 	return ret;
 }
 
-/**
- * vb2_wait_for_all_buffers() - wait until all buffers are given back to vb2
- * @q:		videobuf2 queue
- *
- * This function will wait until all buffers that have been given to the driver
- * by buf_queue() are given back to vb2 with vb2_buffer_done(). It doesn't call
- * wait_prepare, wait_finish pair. It is intended to be called with all locks
- * taken, for example from stop_streaming() callback.
- */
 int vb2_wait_for_all_buffers(struct vb2_queue *q)
 {
 	if (!q->streaming) {
@@ -1725,27 +1580,6 @@ static void __vb2_dqbuf(struct vb2_buffer *vb)
 		}
 }
 
-/**
- * vb2_dqbuf() - Dequeue a buffer to the userspace
- * @q:		videobuf2 queue
- * @pb:		buffer structure passed from userspace to vidioc_dqbuf handler
- *		in driver
- * @nonblocking: if true, this call will not sleep waiting for a buffer if no
- *		 buffers ready for dequeuing are present. Normally the driver
- *		 would be passing (file->f_flags & O_NONBLOCK) here
- *
- * Should be called from vidioc_dqbuf ioctl handler of a driver.
- * The passed buffer should have been verified.
- * This function:
- * 1) calls buf_finish callback in the driver (if provided), in which
- *    driver can perform any additional operations that may be required before
- *    returning the buffer to userspace, such as cache sync,
- * 2) the buffer struct members are filled with relevant information for
- *    the userspace.
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_dqbuf handler in driver.
- */
 int vb2_core_dqbuf(struct vb2_queue *q, unsigned int *pindex, void *pb,
 		   bool nonblocking)
 {
@@ -1909,19 +1743,6 @@ int vb2_core_streamon(struct vb2_queue *q, unsigned int type)
 }
 EXPORT_SYMBOL_GPL(vb2_core_streamon);
 
-/**
- * vb2_queue_error() - signal a fatal error on the queue
- * @q:		videobuf2 queue
- *
- * Flag that a fatal unrecoverable error has occurred and wake up all processes
- * waiting on the queue. Polling will now set POLLERR and queuing and dequeuing
- * buffers will return -EIO.
- *
- * The error flag will be cleared when cancelling the queue, either from
- * vb2_streamoff or vb2_queue_release. Drivers should thus not call this
- * function before starting the stream, otherwise the error flag will remain set
- * until the queue is released when closing the device node.
- */
 void vb2_queue_error(struct vb2_queue *q)
 {
 	q->error = 1;
@@ -1984,19 +1805,6 @@ static int __find_plane_by_offset(struct vb2_queue *q, unsigned long off,
 	return -EINVAL;
 }
 
-/**
- * vb2_core_expbuf() - Export a buffer as a file descriptor
- * @q:		videobuf2 queue
- * @fd:		file descriptor associated with DMABUF (set by driver) *
- * @type:	buffer type
- * @index:	id number of the buffer
- * @plane:	index of the plane to be exported, 0 for single plane queues
- * @flags:	flags for newly created file, currently only O_CLOEXEC is
- *		supported, refer to manual of open syscall for more details
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_expbuf handler in driver.
- */
 int vb2_core_expbuf(struct vb2_queue *q, int *fd, unsigned int type,
 		unsigned int index, unsigned int plane, unsigned int flags)
 {
@@ -2068,25 +1876,6 @@ int vb2_core_expbuf(struct vb2_queue *q, int *fd, unsigned int type,
 }
 EXPORT_SYMBOL_GPL(vb2_core_expbuf);
 
-/**
- * vb2_mmap() - map video buffers into application address space
- * @q:		videobuf2 queue
- * @vma:	vma passed to the mmap file operation handler in the driver
- *
- * Should be called from mmap file operation handler of a driver.
- * This function maps one plane of one of the available video buffers to
- * userspace. To map whole video memory allocated on reqbufs, this function
- * has to be called once per each plane per each buffer previously allocated.
- *
- * When the userspace application calls mmap, it passes to it an offset returned
- * to it earlier by the means of vidioc_querybuf handler. That offset acts as
- * a "cookie", which is then used to identify the plane to be mapped.
- * This function finds a plane with a matching offset and a mapping is performed
- * by the means of a provided memory operation.
- *
- * The return values from this function are intended to be directly returned
- * from the mmap handler in driver.
- */
 int vb2_mmap(struct vb2_queue *q, struct vm_area_struct *vma)
 {
 	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
@@ -2188,17 +1977,6 @@ unsigned long vb2_get_unmapped_area(struct vb2_queue *q,
 EXPORT_SYMBOL_GPL(vb2_get_unmapped_area);
 #endif
 
-/**
- * vb2_core_queue_init() - initialize a videobuf2 queue
- * @q:		videobuf2 queue; this structure should be allocated in driver
- *
- * The vb2_queue structure should be allocated by the driver. The driver is
- * responsible of clearing it's content and setting initial values for some
- * required entries before calling this function.
- * q->ops, q->mem_ops, q->type and q->io_modes are mandatory. Please refer
- * to the struct vb2_queue description in include/media/videobuf2-core.h
- * for more information.
- */
 int vb2_core_queue_init(struct vb2_queue *q)
 {
 	/*
@@ -2228,14 +2006,6 @@ EXPORT_SYMBOL_GPL(vb2_core_queue_init);
 
 static int __vb2_init_fileio(struct vb2_queue *q, int read);
 static int __vb2_cleanup_fileio(struct vb2_queue *q);
-/**
- * vb2_core_queue_release() - stop streaming, release the queue and free memory
- * @q:		videobuf2 queue
- *
- * This function stops streaming and performs necessary clean ups, including
- * freeing video buffer memory. The driver is responsible for freeing
- * the vb2_queue structure itself.
- */
 void vb2_core_queue_release(struct vb2_queue *q)
 {
 	__vb2_cleanup_fileio(q);
@@ -2246,22 +2016,6 @@ void vb2_core_queue_release(struct vb2_queue *q)
 }
 EXPORT_SYMBOL_GPL(vb2_core_queue_release);
 
-/**
- * vb2_core_poll() - implements poll userspace operation
- * @q:		videobuf2 queue
- * @file:	file argument passed to the poll file operation handler
- * @wait:	wait argument passed to the poll file operation handler
- *
- * This function implements poll file operation handler for a driver.
- * For CAPTURE queues, if a buffer is ready to be dequeued, the userspace will
- * be informed that the file descriptor of a video device is available for
- * reading.
- * For OUTPUT queues, if a buffer is ready to be dequeued, the file descriptor
- * will be reported as available for writing.
- *
- * The return values from this function are intended to be directly returned
- * from poll handler in driver.
- */
 unsigned int vb2_core_poll(struct vb2_queue *q, struct file *file,
 		poll_table *wait)
 {
diff --git a/drivers/media/v4l2-core/videobuf2-dma-contig.c b/drivers/media/v4l2-core/videobuf2-dma-contig.c
index 59fa204b15f3..fb6a177be461 100644
--- a/drivers/media/v4l2-core/videobuf2-dma-contig.c
+++ b/drivers/media/v4l2-core/videobuf2-dma-contig.c
@@ -141,6 +141,9 @@ static void *vb2_dc_alloc(struct device *dev, unsigned long attrs,
 {
 	struct vb2_dc_buf *buf;
 
+	if (WARN_ON(!dev))
+		return ERR_PTR(-EINVAL);
+
 	buf = kzalloc(sizeof *buf, GFP_KERNEL);
 	if (!buf)
 		return ERR_PTR(-ENOMEM);
@@ -493,6 +496,9 @@ static void *vb2_dc_get_userptr(struct device *dev, unsigned long vaddr,
 		return ERR_PTR(-EINVAL);
 	}
 
+	if (WARN_ON(!dev))
+		return ERR_PTR(-EINVAL);
+
 	buf = kzalloc(sizeof *buf, GFP_KERNEL);
 	if (!buf)
 		return ERR_PTR(-ENOMEM);
@@ -673,6 +679,9 @@ static void *vb2_dc_attach_dmabuf(struct device *dev, struct dma_buf *dbuf,
 	if (dbuf->size < size)
 		return ERR_PTR(-EFAULT);
 
+	if (WARN_ON(!dev))
+		return ERR_PTR(-EINVAL);
+
 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
 	if (!buf)
 		return ERR_PTR(-ENOMEM);
diff --git a/drivers/media/v4l2-core/videobuf2-dma-sg.c b/drivers/media/v4l2-core/videobuf2-dma-sg.c
index bd82d709ee82..ecff8f492c4f 100644
--- a/drivers/media/v4l2-core/videobuf2-dma-sg.c
+++ b/drivers/media/v4l2-core/videobuf2-dma-sg.c
@@ -104,11 +104,12 @@ static void *vb2_dma_sg_alloc(struct device *dev, unsigned long dma_attrs,
 	int ret;
 	int num_pages;
 
-	if (WARN_ON(dev == NULL))
-		return NULL;
+	if (WARN_ON(!dev))
+		return ERR_PTR(-EINVAL);
+
 	buf = kzalloc(sizeof *buf, GFP_KERNEL);
 	if (!buf)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	buf->vaddr = NULL;
 	buf->dma_dir = dma_dir;
@@ -166,7 +167,7 @@ fail_pages_alloc:
 	kfree(buf->pages);
 fail_pages_array_alloc:
 	kfree(buf);
-	return NULL;
+	return ERR_PTR(-ENOMEM);
 }
 
 static void vb2_dma_sg_put(void *buf_priv)
@@ -224,9 +225,12 @@ static void *vb2_dma_sg_get_userptr(struct device *dev, unsigned long vaddr,
 	struct sg_table *sgt;
 	struct frame_vector *vec;
 
+	if (WARN_ON(!dev))
+		return ERR_PTR(-EINVAL);
+
 	buf = kzalloc(sizeof *buf, GFP_KERNEL);
 	if (!buf)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	buf->vaddr = NULL;
 	buf->dev = dev;
@@ -266,7 +270,7 @@ userptr_fail_sgtable:
 	vb2_destroy_framevec(vec);
 userptr_fail_pfnvec:
 	kfree(buf);
-	return NULL;
+	return ERR_PTR(-ENOMEM);
 }
 
 /*
@@ -606,6 +610,9 @@ static void *vb2_dma_sg_attach_dmabuf(struct device *dev, struct dma_buf *dbuf,
 	struct vb2_dma_sg_buf *buf;
 	struct dma_buf_attachment *dba;
 
+	if (WARN_ON(!dev))
+		return ERR_PTR(-EINVAL);
+
 	if (dbuf->size < size)
 		return ERR_PTR(-EFAULT);
 
diff --git a/drivers/media/v4l2-core/videobuf2-v4l2.c b/drivers/media/v4l2-core/videobuf2-v4l2.c
index 9cfbb6e4bc28..52ef8833f6b6 100644
--- a/drivers/media/v4l2-core/videobuf2-v4l2.c
+++ b/drivers/media/v4l2-core/videobuf2-v4l2.c
@@ -483,13 +483,6 @@ int vb2_querybuf(struct vb2_queue *q, struct v4l2_buffer *b)
 }
 EXPORT_SYMBOL(vb2_querybuf);
 
-/**
- * vb2_reqbufs() - Wrapper for vb2_core_reqbufs() that also verifies
- * the memory and type values.
- * @q:		videobuf2 queue
- * @req:	struct passed from userspace to vidioc_reqbufs handler
- *		in driver
- */
 int vb2_reqbufs(struct vb2_queue *q, struct v4l2_requestbuffers *req)
 {
 	int ret = vb2_verify_memory_type(q, req->memory, req->type);
@@ -498,21 +491,6 @@ int vb2_reqbufs(struct vb2_queue *q, struct v4l2_requestbuffers *req)
 }
 EXPORT_SYMBOL_GPL(vb2_reqbufs);
 
-/**
- * vb2_prepare_buf() - Pass ownership of a buffer from userspace to the kernel
- * @q:		videobuf2 queue
- * @b:		buffer structure passed from userspace to vidioc_prepare_buf
- *		handler in driver
- *
- * Should be called from vidioc_prepare_buf ioctl handler of a driver.
- * This function:
- * 1) verifies the passed buffer,
- * 2) calls buf_prepare callback in the driver (if provided), in which
- *    driver-specific buffer initialization can be performed,
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_prepare_buf handler in driver.
- */
 int vb2_prepare_buf(struct vb2_queue *q, struct v4l2_buffer *b)
 {
 	int ret;
@@ -528,13 +506,6 @@ int vb2_prepare_buf(struct vb2_queue *q, struct v4l2_buffer *b)
 }
 EXPORT_SYMBOL_GPL(vb2_prepare_buf);
 
-/**
- * vb2_create_bufs() - Wrapper for vb2_core_create_bufs() that also verifies
- * the memory and type values.
- * @q:		videobuf2 queue
- * @create:	creation parameters, passed from userspace to vidioc_create_bufs
- *		handler in driver
- */
 int vb2_create_bufs(struct vb2_queue *q, struct v4l2_create_buffers *create)
 {
 	unsigned requested_planes = 1;
@@ -586,23 +557,6 @@ int vb2_create_bufs(struct vb2_queue *q, struct v4l2_create_buffers *create)
 }
 EXPORT_SYMBOL_GPL(vb2_create_bufs);
 
-/**
- * vb2_qbuf() - Queue a buffer from userspace
- * @q:		videobuf2 queue
- * @b:		buffer structure passed from userspace to vidioc_qbuf handler
- *		in driver
- *
- * Should be called from vidioc_qbuf ioctl handler of a driver.
- * This function:
- * 1) verifies the passed buffer,
- * 2) if necessary, calls buf_prepare callback in the driver (if provided), in
- *    which driver-specific buffer initialization can be performed,
- * 3) if streaming is on, queues the buffer in driver by the means of buf_queue
- *    callback for processing.
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_qbuf handler in driver.
- */
 int vb2_qbuf(struct vb2_queue *q, struct v4l2_buffer *b)
 {
 	int ret;
@@ -617,27 +571,6 @@ int vb2_qbuf(struct vb2_queue *q, struct v4l2_buffer *b)
 }
 EXPORT_SYMBOL_GPL(vb2_qbuf);
 
-/**
- * vb2_dqbuf() - Dequeue a buffer to the userspace
- * @q:		videobuf2 queue
- * @b:		buffer structure passed from userspace to vidioc_dqbuf handler
- *		in driver
- * @nonblocking: if true, this call will not sleep waiting for a buffer if no
- *		 buffers ready for dequeuing are present. Normally the driver
- *		 would be passing (file->f_flags & O_NONBLOCK) here
- *
- * Should be called from vidioc_dqbuf ioctl handler of a driver.
- * This function:
- * 1) verifies the passed buffer,
- * 2) calls buf_finish callback in the driver (if provided), in which
- *    driver can perform any additional operations that may be required before
- *    returning the buffer to userspace, such as cache sync,
- * 3) the buffer struct members are filled with relevant information for
- *    the userspace.
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_dqbuf handler in driver.
- */
 int vb2_dqbuf(struct vb2_queue *q, struct v4l2_buffer *b, bool nonblocking)
 {
 	int ret;
@@ -664,19 +597,6 @@ int vb2_dqbuf(struct vb2_queue *q, struct v4l2_buffer *b, bool nonblocking)
 }
 EXPORT_SYMBOL_GPL(vb2_dqbuf);
 
-/**
- * vb2_streamon - start streaming
- * @q:		videobuf2 queue
- * @type:	type argument passed from userspace to vidioc_streamon handler
- *
- * Should be called from vidioc_streamon handler of a driver.
- * This function:
- * 1) verifies current state
- * 2) passes any previously queued buffers to the driver and starts streaming
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_streamon handler in the driver.
- */
 int vb2_streamon(struct vb2_queue *q, enum v4l2_buf_type type)
 {
 	if (vb2_fileio_is_active(q)) {
@@ -687,21 +607,6 @@ int vb2_streamon(struct vb2_queue *q, enum v4l2_buf_type type)
 }
 EXPORT_SYMBOL_GPL(vb2_streamon);
 
-/**
- * vb2_streamoff - stop streaming
- * @q:		videobuf2 queue
- * @type:	type argument passed from userspace to vidioc_streamoff handler
- *
- * Should be called from vidioc_streamoff handler of a driver.
- * This function:
- * 1) verifies current state,
- * 2) stop streaming and dequeues any queued buffers, including those previously
- *    passed to the driver (after waiting for the driver to finish).
- *
- * This call can be used for pausing playback.
- * The return values from this function are intended to be directly returned
- * from vidioc_streamoff handler in the driver
- */
 int vb2_streamoff(struct vb2_queue *q, enum v4l2_buf_type type)
 {
 	if (vb2_fileio_is_active(q)) {
@@ -712,15 +617,6 @@ int vb2_streamoff(struct vb2_queue *q, enum v4l2_buf_type type)
 }
 EXPORT_SYMBOL_GPL(vb2_streamoff);
 
-/**
- * vb2_expbuf() - Export a buffer as a file descriptor
- * @q:		videobuf2 queue
- * @eb:		export buffer structure passed from userspace to vidioc_expbuf
- *		handler in driver
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_expbuf handler in driver.
- */
 int vb2_expbuf(struct vb2_queue *q, struct v4l2_exportbuffer *eb)
 {
 	return vb2_core_expbuf(q, &eb->fd, eb->type, eb->index,
@@ -728,17 +624,6 @@ int vb2_expbuf(struct vb2_queue *q, struct v4l2_exportbuffer *eb)
 }
 EXPORT_SYMBOL_GPL(vb2_expbuf);
 
-/**
- * vb2_queue_init() - initialize a videobuf2 queue
- * @q:		videobuf2 queue; this structure should be allocated in driver
- *
- * The vb2_queue structure should be allocated by the driver. The driver is
- * responsible of clearing it's content and setting initial values for some
- * required entries before calling this function.
- * q->ops, q->mem_ops, q->type and q->io_modes are mandatory. Please refer
- * to the struct vb2_queue description in include/media/videobuf2-core.h
- * for more information.
- */
 int vb2_queue_init(struct vb2_queue *q)
 {
 	/*
@@ -779,39 +664,12 @@ int vb2_queue_init(struct vb2_queue *q)
 }
 EXPORT_SYMBOL_GPL(vb2_queue_init);
 
-/**
- * vb2_queue_release() - stop streaming, release the queue and free memory
- * @q:		videobuf2 queue
- *
- * This function stops streaming and performs necessary clean ups, including
- * freeing video buffer memory. The driver is responsible for freeing
- * the vb2_queue structure itself.
- */
 void vb2_queue_release(struct vb2_queue *q)
 {
 	vb2_core_queue_release(q);
 }
 EXPORT_SYMBOL_GPL(vb2_queue_release);
 
-/**
- * vb2_poll() - implements poll userspace operation
- * @q:		videobuf2 queue
- * @file:	file argument passed to the poll file operation handler
- * @wait:	wait argument passed to the poll file operation handler
- *
- * This function implements poll file operation handler for a driver.
- * For CAPTURE queues, if a buffer is ready to be dequeued, the userspace will
- * be informed that the file descriptor of a video device is available for
- * reading.
- * For OUTPUT queues, if a buffer is ready to be dequeued, the file descriptor
- * will be reported as available for writing.
- *
- * If the driver uses struct v4l2_fh, then vb2_poll() will also check for any
- * pending events.
- *
- * The return values from this function are intended to be directly returned
- * from poll handler in driver.
- */
 unsigned int vb2_poll(struct vb2_queue *q, struct file *file, poll_table *wait)
 {
 	struct video_device *vfd = video_devdata(file);
diff --git a/drivers/media/v4l2-core/videobuf2-vmalloc.c b/drivers/media/v4l2-core/videobuf2-vmalloc.c
index c2820a6e164d..ab3227b75c84 100644
--- a/drivers/media/v4l2-core/videobuf2-vmalloc.c
+++ b/drivers/media/v4l2-core/videobuf2-vmalloc.c
@@ -41,7 +41,7 @@ static void *vb2_vmalloc_alloc(struct device *dev, unsigned long attrs,
 
 	buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags);
 	if (!buf)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	buf->size = size;
 	buf->vaddr = vmalloc_user(buf->size);
@@ -53,7 +53,7 @@ static void *vb2_vmalloc_alloc(struct device *dev, unsigned long attrs,
 	if (!buf->vaddr) {
 		pr_debug("vmalloc of size %ld failed\n", buf->size);
 		kfree(buf);
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 	}
 
 	atomic_inc(&buf->refcount);
@@ -77,17 +77,20 @@ static void *vb2_vmalloc_get_userptr(struct device *dev, unsigned long vaddr,
 	struct vb2_vmalloc_buf *buf;
 	struct frame_vector *vec;
 	int n_pages, offset, i;
+	int ret = -ENOMEM;
 
 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
 	if (!buf)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	buf->dma_dir = dma_dir;
 	offset = vaddr & ~PAGE_MASK;
 	buf->size = size;
 	vec = vb2_create_framevec(vaddr, size, dma_dir == DMA_FROM_DEVICE);
-	if (IS_ERR(vec))
+	if (IS_ERR(vec)) {
+		ret = PTR_ERR(vec);
 		goto fail_pfnvec_create;
+	}
 	buf->vec = vec;
 	n_pages = frame_vector_count(vec);
 	if (frame_vector_to_pages(vec) < 0) {
@@ -117,7 +120,7 @@ fail_map:
 fail_pfnvec_create:
 	kfree(buf);
 
-	return NULL;
+	return ERR_PTR(ret);
 }
 
 static void vb2_vmalloc_put_userptr(void *buf_priv)
diff --git a/drivers/memory/atmel-ebi.c b/drivers/memory/atmel-ebi.c
index f87ad6f5d2dc..b5ed3bd082b5 100644
--- a/drivers/memory/atmel-ebi.c
+++ b/drivers/memory/atmel-ebi.c
@@ -410,10 +410,7 @@ static int at91sam9_ebi_init(struct at91_ebi *ebi)
 
 	field.reg = AT91SAM9_SMC_MODE(AT91SAM9_SMC_GENERIC);
 	fields->mode = devm_regmap_field_alloc(ebi->dev, ebi->smc, field);
-	if (IS_ERR(fields->mode))
-		return PTR_ERR(fields->mode);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(fields->mode);
 }
 
 static int sama5d3_ebi_init(struct at91_ebi *ebi)
@@ -441,10 +438,7 @@ static int sama5d3_ebi_init(struct at91_ebi *ebi)
 
 	field.reg = SAMA5_SMC_MODE(SAMA5_SMC_GENERIC);
 	fields->mode = devm_regmap_field_alloc(ebi->dev, ebi->smc, field);
-	if (IS_ERR(fields->mode))
-		return PTR_ERR(fields->mode);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(fields->mode);
 }
 
 static int at91_ebi_dev_setup(struct at91_ebi *ebi, struct device_node *np,
diff --git a/drivers/memory/atmel-sdramc.c b/drivers/memory/atmel-sdramc.c
index 53a341f3b305..12080b05e3e6 100644
--- a/drivers/memory/atmel-sdramc.c
+++ b/drivers/memory/atmel-sdramc.c
@@ -53,12 +53,10 @@ static const struct of_device_id atmel_ramc_of_match[] = {
 
 static int atmel_ramc_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *match;
 	const struct at91_ramc_caps *caps;
 	struct clk *clk;
 
-	match = of_match_device(atmel_ramc_of_match, &pdev->dev);
-	caps = match->data;
+	caps = of_device_get_match_data(&pdev->dev);
 
 	if (caps->has_ddrck) {
 		clk = devm_clk_get(&pdev->dev, "ddrck");
diff --git a/drivers/memory/of_memory.c b/drivers/memory/of_memory.c
index 9daf94bb8f27..568f05ed961a 100644
--- a/drivers/memory/of_memory.c
+++ b/drivers/memory/of_memory.c
@@ -16,6 +16,7 @@
 #include <linux/gfp.h>
 #include <memory/jedec_ddr.h>
 #include <linux/export.h>
+#include "of_memory.h"
 
 /**
  * of_get_min_tck() - extract min timing values for ddr
diff --git a/drivers/memory/omap-gpmc.c b/drivers/memory/omap-gpmc.c
index 869c83fb3c5d..5457c361ad58 100644
--- a/drivers/memory/omap-gpmc.c
+++ b/drivers/memory/omap-gpmc.c
@@ -350,8 +350,8 @@ static unsigned int gpmc_ps_to_ticks(unsigned int time_ps)
 	return (time_ps + tick_ps - 1) / tick_ps;
 }
 
-unsigned int gpmc_clk_ticks_to_ns(unsigned ticks, int cs,
-				  enum gpmc_clk_domain cd)
+static unsigned int gpmc_clk_ticks_to_ns(unsigned int ticks, int cs,
+					 enum gpmc_clk_domain cd)
 {
 	return ticks * gpmc_get_clk_period(cs, cd) / 1000;
 }
@@ -2143,9 +2143,7 @@ err_child_fail:
 	ret = -ENODEV;
 
 err_cs:
-	if (waitpin_desc)
-		gpiochip_free_own_desc(waitpin_desc);
-
+	gpiochip_free_own_desc(waitpin_desc);
 err:
 	gpmc_cs_free(cs);
 
@@ -2185,7 +2183,7 @@ static int gpmc_probe_dt(struct platform_device *pdev)
 	return 0;
 }
 
-static int gpmc_probe_dt_children(struct platform_device *pdev)
+static void gpmc_probe_dt_children(struct platform_device *pdev)
 {
 	int ret;
 	struct device_node *child;
@@ -2200,11 +2198,11 @@ static int gpmc_probe_dt_children(struct platform_device *pdev)
 		else
 			ret = gpmc_probe_generic_child(pdev, child);
 
-		if (ret)
-			return ret;
+		if (ret) {
+			dev_err(&pdev->dev, "failed to probe DT child '%s': %d\n",
+				child->name, ret);
+		}
 	}
-
-	return 0;
 }
 #else
 static int gpmc_probe_dt(struct platform_device *pdev)
@@ -2212,9 +2210,8 @@ static int gpmc_probe_dt(struct platform_device *pdev)
 	return 0;
 }
 
-static int gpmc_probe_dt_children(struct platform_device *pdev)
+static void gpmc_probe_dt_children(struct platform_device *pdev)
 {
-	return 0;
 }
 #endif /* CONFIG_OF */
 
@@ -2266,7 +2263,7 @@ static int gpmc_gpio_init(struct gpmc_device *gpmc)
 	gpmc->gpio_chip.get = gpmc_gpio_get;
 	gpmc->gpio_chip.base = -1;
 
-	ret = gpiochip_add(&gpmc->gpio_chip);
+	ret = devm_gpiochip_add_data(gpmc->dev, &gpmc->gpio_chip, NULL);
 	if (ret < 0) {
 		dev_err(gpmc->dev, "could not register gpio chip: %d\n", ret);
 		return ret;
@@ -2275,11 +2272,6 @@ static int gpmc_gpio_init(struct gpmc_device *gpmc)
 	return 0;
 }
 
-static void gpmc_gpio_exit(struct gpmc_device *gpmc)
-{
-	gpiochip_remove(&gpmc->gpio_chip);
-}
-
 static int gpmc_probe(struct platform_device *pdev)
 {
 	int rc;
@@ -2366,21 +2358,13 @@ static int gpmc_probe(struct platform_device *pdev)
 	rc = gpmc_setup_irq(gpmc);
 	if (rc) {
 		dev_err(gpmc->dev, "gpmc_setup_irq failed\n");
-		goto setup_irq_failed;
+		goto gpio_init_failed;
 	}
 
-	rc = gpmc_probe_dt_children(pdev);
-	if (rc < 0) {
-		dev_err(gpmc->dev, "failed to probe DT children\n");
-		goto dt_children_failed;
-	}
+	gpmc_probe_dt_children(pdev);
 
 	return 0;
 
-dt_children_failed:
-	gpmc_free_irq(gpmc);
-setup_irq_failed:
-	gpmc_gpio_exit(gpmc);
 gpio_init_failed:
 	gpmc_mem_exit();
 	pm_runtime_put_sync(&pdev->dev);
@@ -2394,7 +2378,6 @@ static int gpmc_remove(struct platform_device *pdev)
 	struct gpmc_device *gpmc = platform_get_drvdata(pdev);
 
 	gpmc_free_irq(gpmc);
-	gpmc_gpio_exit(gpmc);
 	gpmc_mem_exit();
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index 5537f8df8512..89c7ed16b4df 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -1865,8 +1865,8 @@ mpt_attach(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	snprintf(ioc->reset_work_q_name, MPT_KOBJ_NAME_LEN,
 		 "mpt_poll_%d", ioc->id);
-	ioc->reset_work_q =
-		create_singlethread_workqueue(ioc->reset_work_q_name);
+	ioc->reset_work_q = alloc_workqueue(ioc->reset_work_q_name,
+					    WQ_MEM_RECLAIM, 0);
 	if (!ioc->reset_work_q) {
 		printk(MYIOC_s_ERR_FMT "Insufficient memory to add adapter!\n",
 		    ioc->name);
@@ -1992,7 +1992,8 @@ mpt_attach(struct pci_dev *pdev, const struct pci_device_id *id)
 	INIT_LIST_HEAD(&ioc->fw_event_list);
 	spin_lock_init(&ioc->fw_event_lock);
 	snprintf(ioc->fw_event_q_name, MPT_KOBJ_NAME_LEN, "mpt/%d", ioc->id);
-	ioc->fw_event_q = create_singlethread_workqueue(ioc->fw_event_q_name);
+	ioc->fw_event_q = alloc_workqueue(ioc->fw_event_q_name,
+					  WQ_MEM_RECLAIM, 0);
 	if (!ioc->fw_event_q) {
 		printk(MYIOC_s_ERR_FMT "Insufficient memory to add adapter!\n",
 		    ioc->name);
diff --git a/drivers/message/fusion/mptfc.c b/drivers/message/fusion/mptfc.c
index d8bf84aef602..add6a3a6ef0d 100644
--- a/drivers/message/fusion/mptfc.c
+++ b/drivers/message/fusion/mptfc.c
@@ -1324,9 +1324,12 @@ mptfc_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	snprintf(ioc->fc_rescan_work_q_name, sizeof(ioc->fc_rescan_work_q_name),
 		 "mptfc_wq_%d", sh->host_no);
 	ioc->fc_rescan_work_q =
-		create_singlethread_workqueue(ioc->fc_rescan_work_q_name);
-	if (!ioc->fc_rescan_work_q)
+		alloc_ordered_workqueue(ioc->fc_rescan_work_q_name,
+					WQ_MEM_RECLAIM);
+	if (!ioc->fc_rescan_work_q) {
+		error = -ENOMEM;
 		goto out_mptfc_probe;
+	}
 
 	/*
 	 *  Pre-fetch FC port WWN and stuff...
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index 2d1fb6420592..c6df6442ba2b 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -50,7 +50,7 @@ config MFD_AS3711
 	  Support for the AS3711 PMIC from AMS
 
 config MFD_AS3722
-	bool "ams AS3722 Power Management IC"
+	tristate "ams AS3722 Power Management IC"
 	select MFD_CORE
 	select REGMAP_I2C
 	select REGMAP_IRQ
@@ -112,6 +112,16 @@ config MFD_BCM590XX
 	help
 	  Support for the BCM590xx PMUs from Broadcom
 
+config MFD_AC100
+	tristate "X-Powers AC100"
+	select MFD_CORE
+	depends on SUNXI_RSB
+	help
+	  If you say Y here you get support for the X-Powers AC100 audio codec
+	  IC.
+	  This driver include only the core APIs. You have to select individual
+	  components like codecs or RTC under the corresponding menus.
+
 config MFD_AXP20X
 	tristate
 	select MFD_CORE
@@ -281,6 +291,14 @@ config MFD_DLN2
 	  etc. must be enabled in order to use the functionality of
 	  the device.
 
+config MFD_EXYNOS_LPASS
+	tristate "Samsung Exynos SoC Low Power Audio Subsystem"
+	select MFD_CORE
+	select REGMAP_MMIO
+	help
+	  Select this option to enable support for Samsung Exynos Low Power
+	  Audio Subsystem.
+
 config MFD_MC13XXX
 	tristate
 	depends on (SPI_MASTER || I2C)
@@ -340,14 +358,6 @@ config MFD_HI655X_PMIC
 	help
 	  Select this option to enable Hisilicon hi655x series pmic driver.
 
-config HTC_EGPIO
-	bool "HTC EGPIO support"
-	depends on GPIOLIB && ARM
-	help
-	    This driver supports the CPLD egpio chip present on
-	    several HTC phones.  It provides basic support for input
-	    pins, output pins, and irqs.
-
 config HTC_PASIC3
 	tristate "HTC PASIC3 LED/DS1WM chip support"
 	select MFD_CORE
@@ -852,13 +862,13 @@ config MFD_RC5T583
 	  different functionality of the device.
 
 config MFD_RK808
-	tristate "Rockchip RK808 Power Management chip"
+	tristate "Rockchip RK808/RK818 Power Management Chip"
 	depends on I2C && OF
 	select MFD_CORE
 	select REGMAP_I2C
 	select REGMAP_IRQ
 	help
-	  If you say yes here you get support for the RK808
+	  If you say yes here you get support for the RK808 and RK818
 	  Power Management chips.
 	  This driver provides common support for accessing the device
 	  through I2C interface. The device supports multiple sub-devices
@@ -1214,6 +1224,7 @@ config MFD_TPS65217
 	depends on I2C
 	select MFD_CORE
 	select REGMAP_I2C
+	select IRQ_DOMAIN
 	help
 	  If you say yes here you get support for the TPS65217 series of
 	  Power Management / White LED chips.
@@ -1224,6 +1235,20 @@ config MFD_TPS65217
 	  This driver can also be built as a module.  If so, the module
 	  will be called tps65217.
 
+config MFD_TI_LP873X
+	tristate "TI LP873X Power Management IC"
+	depends on I2C
+	select MFD_CORE
+	select REGMAP_I2C
+	help
+	  If you say yes here then you get support for the LP873X series of
+	  Power Management Integrated Circuits (PMIC).
+	  These include voltage regulators, thermal protection, configurable
+	  General Purpose Outputs (GPO) that are used in portable devices.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called lp873x.
+
 config MFD_TPS65218
 	tristate "TI TPS65218 Power Management chips"
 	depends on I2C
@@ -1549,6 +1574,7 @@ config MFD_WM8350
 config MFD_WM8350_I2C
 	bool "Wolfson Microelectronics WM8350 with I2C"
 	select MFD_WM8350
+	select REGMAP_I2C
 	depends on I2C=y
 	help
 	  The WM8350 is an integrated audio and power management
diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile
index 2ba3ba35f745..9834e669d985 100644
--- a/drivers/mfd/Makefile
+++ b/drivers/mfd/Makefile
@@ -13,15 +13,17 @@ obj-$(CONFIG_MFD_BCM590XX)	+= bcm590xx.o
 obj-$(CONFIG_MFD_CROS_EC)	+= cros_ec.o
 obj-$(CONFIG_MFD_CROS_EC_I2C)	+= cros_ec_i2c.o
 obj-$(CONFIG_MFD_CROS_EC_SPI)	+= cros_ec_spi.o
+obj-$(CONFIG_MFD_EXYNOS_LPASS)	+= exynos-lpass.o
 
 rtsx_pci-objs			:= rtsx_pcr.o rts5209.o rts5229.o rtl8411.o rts5227.o rts5249.o
 obj-$(CONFIG_MFD_RTSX_PCI)	+= rtsx_pci.o
 obj-$(CONFIG_MFD_RTSX_USB)	+= rtsx_usb.o
 
-obj-$(CONFIG_HTC_EGPIO)		+= htc-egpio.o
 obj-$(CONFIG_HTC_PASIC3)	+= htc-pasic3.o
 obj-$(CONFIG_HTC_I2CPLD)	+= htc-i2cpld.o
 
+obj-$(CONFIG_MFD_TI_LP873X)	+= lp873x.o
+
 obj-$(CONFIG_MFD_DAVINCI_VOICECODEC)	+= davinci_voicecodec.o
 obj-$(CONFIG_MFD_DM355EVM_MSP)	+= dm355evm_msp.o
 obj-$(CONFIG_MFD_TI_AM335X_TSCADC)	+= ti_am335x_tscadc.o
@@ -113,6 +115,8 @@ obj-$(CONFIG_PMIC_DA9052)	+= da9052-irq.o
 obj-$(CONFIG_PMIC_DA9052)	+= da9052-core.o
 obj-$(CONFIG_MFD_DA9052_SPI)	+= da9052-spi.o
 obj-$(CONFIG_MFD_DA9052_I2C)	+= da9052-i2c.o
+
+obj-$(CONFIG_MFD_AC100)		+= ac100.o
 obj-$(CONFIG_MFD_AXP20X)	+= axp20x.o
 obj-$(CONFIG_MFD_AXP20X_I2C)	+= axp20x-i2c.o
 obj-$(CONFIG_MFD_AXP20X_RSB)	+= axp20x-rsb.o
diff --git a/drivers/mfd/ab8500-debugfs.c b/drivers/mfd/ab8500-debugfs.c
index 0aecd7bd35f8..acf6c00b14b9 100644
--- a/drivers/mfd/ab8500-debugfs.c
+++ b/drivers/mfd/ab8500-debugfs.c
@@ -153,14 +153,14 @@ static struct hwreg_cfg hwreg_cfg = {
 
 #define AB8500_NAME_STRING "ab8500"
 #define AB8500_ADC_NAME_STRING "gpadc"
-#define AB8500_NUM_BANKS 24
+#define AB8500_NUM_BANKS AB8500_DEBUG_FIELD_LAST
 
 #define AB8500_REV_REG 0x80
 
 static struct ab8500_prcmu_ranges *debug_ranges;
 
 static struct ab8500_prcmu_ranges ab8500_debug_ranges[AB8500_NUM_BANKS] = {
-	[0x0] = {
+	[AB8500_M_FSM_RANK] = {
 		.num_ranges = 0,
 		.range = NULL,
 	},
@@ -315,7 +315,7 @@ static struct ab8500_prcmu_ranges ab8500_debug_ranges[AB8500_NUM_BANKS] = {
 			},
 		},
 	},
-	[0x9] = {
+	[AB8500_RESERVED] = {
 		.num_ranges = 0,
 		.range = NULL,
 	},
@@ -386,24 +386,6 @@ static struct ab8500_prcmu_ranges ab8500_debug_ranges[AB8500_NUM_BANKS] = {
 			},
 		},
 	},
-	[AB8500_DEVELOPMENT] = {
-		.num_ranges = 1,
-		.range = (struct ab8500_reg_range[]) {
-			{
-				.first = 0x00,
-				.last = 0x00,
-			},
-		},
-	},
-	[AB8500_DEBUG] = {
-		.num_ranges = 1,
-		.range = (struct ab8500_reg_range[]) {
-			{
-				.first = 0x05,
-				.last = 0x07,
-			},
-		},
-	},
 	[AB8500_AUDIO] = {
 		.num_ranges = 1,
 		.range = (struct ab8500_reg_range[]) {
@@ -463,19 +445,29 @@ static struct ab8500_prcmu_ranges ab8500_debug_ranges[AB8500_NUM_BANKS] = {
 			},
 		},
 	},
-	[0x11] = {
-		.num_ranges = 0,
-		.range = NULL,
+	[AB8500_DEVELOPMENT] = {
+		.num_ranges = 1,
+		.range = (struct ab8500_reg_range[]) {
+			{
+				.first = 0x00,
+				.last = 0x00,
+			},
+		},
 	},
-	[0x12] = {
-		.num_ranges = 0,
-		.range = NULL,
+	[AB8500_DEBUG] = {
+		.num_ranges = 1,
+		.range = (struct ab8500_reg_range[]) {
+			{
+				.first = 0x05,
+				.last = 0x07,
+			},
+		},
 	},
-	[0x13] = {
+	[AB8500_PROD_TEST] = {
 		.num_ranges = 0,
 		.range = NULL,
 	},
-	[0x14] = {
+	[AB8500_STE_TEST] = {
 		.num_ranges = 0,
 		.range = NULL,
 	},
@@ -1382,60 +1374,6 @@ void ab8500_dump_all_banks(struct device *dev)
 	}
 }
 
-/* Space for 500 registers. */
-#define DUMP_MAX_REGS 700
-static struct ab8500_register_dump
-{
-	u8 bank;
-	u8 reg;
-	u8 value;
-} ab8500_complete_register_dump[DUMP_MAX_REGS];
-
-/* This shall only be called upon kernel panic! */
-void ab8500_dump_all_banks_to_mem(void)
-{
-	int i, r = 0;
-	u8 bank;
-	int err = 0;
-
-	pr_info("Saving all ABB registers for crash analysis.\n");
-
-	for (bank = 0; bank < AB8500_NUM_BANKS; bank++) {
-		for (i = 0; i < debug_ranges[bank].num_ranges; i++) {
-			u8 reg;
-
-			for (reg = debug_ranges[bank].range[i].first;
-			     reg <= debug_ranges[bank].range[i].last;
-			     reg++) {
-				u8 value;
-
-				err = prcmu_abb_read(bank, reg, &value, 1);
-
-				if (err < 0)
-					goto out;
-
-				ab8500_complete_register_dump[r].bank = bank;
-				ab8500_complete_register_dump[r].reg = reg;
-				ab8500_complete_register_dump[r].value = value;
-
-				r++;
-
-				if (r >= DUMP_MAX_REGS) {
-					pr_err("%s: too many register to dump!\n",
-						__func__);
-					err = -EINVAL;
-					goto out;
-				}
-			}
-		}
-	}
-out:
-	if (err >= 0)
-		pr_info("Saved all ABB registers.\n");
-	else
-		pr_info("Failed to save all ABB registers.\n");
-}
-
 static int ab8500_all_banks_open(struct inode *inode, struct file *file)
 {
 	struct seq_file *s;
@@ -1584,18 +1522,10 @@ static u32 num_interrupts[AB8500_MAX_NR_IRQS];
 static u32 num_wake_interrupts[AB8500_MAX_NR_IRQS];
 static int num_interrupt_lines;
 
-bool __attribute__((weak)) suspend_test_wake_cause_interrupt_is_mine(u32 my_int)
-{
-	return false;
-}
-
 void ab8500_debug_register_interrupt(int line)
 {
-	if (line < num_interrupt_lines) {
+	if (line < num_interrupt_lines)
 		num_interrupts[line]++;
-		if (suspend_test_wake_cause_interrupt_is_mine(irq_ab8500))
-			num_wake_interrupts[line]++;
-	}
 }
 
 static int ab8500_interrupts_print(struct seq_file *s, void *p)
diff --git a/drivers/mfd/ac100.c b/drivers/mfd/ac100.c
new file mode 100644
index 000000000000..9bc69cd7807d
--- /dev/null
+++ b/drivers/mfd/ac100.c
@@ -0,0 +1,137 @@
+/*
+ * MFD core driver for X-Powers' AC100 Audio Codec IC
+ *
+ * The AC100 is a highly integrated audio codec and RTC subsystem designed
+ * for mobile applications. It has 3 I2S/PCM interfaces, a 2 channel DAC,
+ * a 2 channel ADC with 5 inputs and a builtin mixer. The RTC subsystem has
+ * 3 clock outputs.
+ *
+ * The audio codec and RTC parts are completely separate, sharing only the
+ * host interface for access to its registers.
+ *
+ * Copyright (2016) Chen-Yu Tsai
+ *
+ * Author: Chen-Yu Tsai <wens@csie.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/core.h>
+#include <linux/mfd/ac100.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+#include <linux/sunxi-rsb.h>
+
+static const struct regmap_range ac100_writeable_ranges[] = {
+	regmap_reg_range(AC100_CHIP_AUDIO_RST, AC100_I2S_SR_CTRL),
+	regmap_reg_range(AC100_I2S1_CLK_CTRL, AC100_I2S1_MXR_GAIN),
+	regmap_reg_range(AC100_I2S2_CLK_CTRL, AC100_I2S2_MXR_GAIN),
+	regmap_reg_range(AC100_I2S3_CLK_CTRL, AC100_I2S3_SIG_PATH_CTRL),
+	regmap_reg_range(AC100_ADC_DIG_CTRL, AC100_ADC_VOL_CTRL),
+	regmap_reg_range(AC100_HMIC_CTRL1, AC100_HMIC_STATUS),
+	regmap_reg_range(AC100_DAC_DIG_CTRL, AC100_DAC_MXR_GAIN),
+	regmap_reg_range(AC100_ADC_APC_CTRL, AC100_LINEOUT_CTRL),
+	regmap_reg_range(AC100_ADC_DAP_L_CTRL, AC100_ADC_DAP_OPT),
+	regmap_reg_range(AC100_DAC_DAP_CTRL, AC100_DAC_DAP_OPT),
+	regmap_reg_range(AC100_ADC_DAP_ENA, AC100_DAC_DAP_ENA),
+	regmap_reg_range(AC100_SRC1_CTRL1, AC100_SRC1_CTRL2),
+	regmap_reg_range(AC100_SRC2_CTRL1, AC100_SRC2_CTRL2),
+	regmap_reg_range(AC100_CLK32K_ANALOG_CTRL, AC100_CLKOUT_CTRL3),
+	regmap_reg_range(AC100_RTC_RST, AC100_RTC_UPD),
+	regmap_reg_range(AC100_ALM_INT_ENA, AC100_ALM_INT_STA),
+	regmap_reg_range(AC100_ALM_SEC, AC100_RTC_GP(15)),
+};
+
+static const struct regmap_range ac100_volatile_ranges[] = {
+	regmap_reg_range(AC100_CHIP_AUDIO_RST, AC100_PLL_CTRL2),
+	regmap_reg_range(AC100_HMIC_STATUS, AC100_HMIC_STATUS),
+	regmap_reg_range(AC100_ADC_DAP_L_STA, AC100_ADC_DAP_L_STA),
+	regmap_reg_range(AC100_SRC1_CTRL1, AC100_SRC1_CTRL1),
+	regmap_reg_range(AC100_SRC1_CTRL3, AC100_SRC2_CTRL1),
+	regmap_reg_range(AC100_SRC2_CTRL3, AC100_SRC2_CTRL4),
+	regmap_reg_range(AC100_RTC_RST, AC100_RTC_RST),
+	regmap_reg_range(AC100_RTC_SEC, AC100_ALM_INT_STA),
+	regmap_reg_range(AC100_ALM_SEC, AC100_ALM_UPD),
+};
+
+static const struct regmap_access_table ac100_writeable_table = {
+	.yes_ranges	= ac100_writeable_ranges,
+	.n_yes_ranges	= ARRAY_SIZE(ac100_writeable_ranges),
+};
+
+static const struct regmap_access_table ac100_volatile_table = {
+	.yes_ranges	= ac100_volatile_ranges,
+	.n_yes_ranges	= ARRAY_SIZE(ac100_volatile_ranges),
+};
+
+static const struct regmap_config ac100_regmap_config = {
+	.reg_bits	= 8,
+	.val_bits	= 16,
+	.wr_table	= &ac100_writeable_table,
+	.volatile_table	= &ac100_volatile_table,
+	.max_register	= AC100_RTC_GP(15),
+	.cache_type	= REGCACHE_RBTREE,
+};
+
+static struct mfd_cell ac100_cells[] = {
+	{
+		.name		= "ac100-codec",
+		.of_compatible	= "x-powers,ac100-codec",
+	}, {
+		.name		= "ac100-rtc",
+		.of_compatible	= "x-powers,ac100-rtc",
+	},
+};
+
+static int ac100_rsb_probe(struct sunxi_rsb_device *rdev)
+{
+	struct ac100_dev *ac100;
+	int ret;
+
+	ac100 = devm_kzalloc(&rdev->dev, sizeof(*ac100), GFP_KERNEL);
+	if (!ac100)
+		return -ENOMEM;
+
+	ac100->dev = &rdev->dev;
+	sunxi_rsb_device_set_drvdata(rdev, ac100);
+
+	ac100->regmap = devm_regmap_init_sunxi_rsb(rdev, &ac100_regmap_config);
+	if (IS_ERR(ac100->regmap)) {
+		ret = PTR_ERR(ac100->regmap);
+		dev_err(ac100->dev, "regmap init failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_mfd_add_devices(ac100->dev, PLATFORM_DEVID_NONE, ac100_cells,
+				   ARRAY_SIZE(ac100_cells), NULL, 0, NULL);
+	if (ret) {
+		dev_err(ac100->dev, "failed to add MFD devices: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id ac100_of_match[] = {
+	{ .compatible = "x-powers,ac100" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ac100_of_match);
+
+static struct sunxi_rsb_driver ac100_rsb_driver = {
+	.driver = {
+		.name	= "ac100",
+		.of_match_table	= of_match_ptr(ac100_of_match),
+	},
+	.probe	= ac100_rsb_probe,
+};
+module_sunxi_rsb_driver(ac100_rsb_driver);
+
+MODULE_DESCRIPTION("Audio codec MFD core driver for AC100");
+MODULE_AUTHOR("Chen-Yu Tsai <wens@csie.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/act8945a.c b/drivers/mfd/act8945a.c
index 10c6d2da8822..a4024d91da01 100644
--- a/drivers/mfd/act8945a.c
+++ b/drivers/mfd/act8945a.c
@@ -23,6 +23,7 @@ static const struct mfd_cell act8945a_devs[] = {
 	},
 	{
 		.name = "act8945a-charger",
+		.of_compatible = "active-semi,act8945a-charger",
 	},
 };
 
diff --git a/drivers/mfd/altera-a10sr.c b/drivers/mfd/altera-a10sr.c
index c05aa4ff57fd..06e1f7fc5605 100644
--- a/drivers/mfd/altera-a10sr.c
+++ b/drivers/mfd/altera-a10sr.c
@@ -1,4 +1,8 @@
 /*
+ * Altera Arria10 DevKit System Resource MFD Driver
+ *
+ * Author: Thor Thayer <tthayer@opensource.altera.com>
+ *
  * Copyright Intel Corporation (C) 2014-2016. All Rights Reserved
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -20,7 +24,7 @@
 
 #include <linux/mfd/altera-a10sr.h>
 #include <linux/mfd/core.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 #include <linux/spi/spi.h>
 
@@ -94,7 +98,7 @@ static bool altr_a10sr_reg_volatile(struct device *dev, unsigned int reg)
 	}
 }
 
-const struct regmap_config altr_a10sr_regmap_config = {
+static const struct regmap_config altr_a10sr_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
 
@@ -152,7 +156,6 @@ static const struct of_device_id altr_a10sr_spi_of_match[] = {
 	{ .compatible = "altr,a10sr" },
 	{ },
 };
-MODULE_DEVICE_TABLE(of, altr_a10sr_spi_of_match);
 
 static struct spi_driver altr_a10sr_spi_driver = {
 	.probe = altr_a10sr_spi_probe,
@@ -161,9 +164,4 @@ static struct spi_driver altr_a10sr_spi_driver = {
 		.of_match_table = of_match_ptr(altr_a10sr_spi_of_match),
 	},
 };
-
-module_spi_driver(altr_a10sr_spi_driver);
-
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Thor Thayer <tthayer@opensource.altera.com>");
-MODULE_DESCRIPTION("Altera Arria10 DevKit System Resource MFD Driver");
+builtin_driver(altr_a10sr_spi_driver, spi_register_driver)
diff --git a/drivers/mfd/arizona-core.c b/drivers/mfd/arizona-core.c
index e4f97b3c824b..41767f7239bb 100644
--- a/drivers/mfd/arizona-core.c
+++ b/drivers/mfd/arizona-core.c
@@ -10,6 +10,7 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/gpio.h>
@@ -49,7 +50,15 @@ int arizona_clk32k_enable(struct arizona *arizona)
 		case ARIZONA_32KZ_MCLK1:
 			ret = pm_runtime_get_sync(arizona->dev);
 			if (ret != 0)
-				goto out;
+				goto err_ref;
+			ret = clk_prepare_enable(arizona->mclk[ARIZONA_MCLK1]);
+			if (ret != 0)
+				goto err_pm;
+			break;
+		case ARIZONA_32KZ_MCLK2:
+			ret = clk_prepare_enable(arizona->mclk[ARIZONA_MCLK2]);
+			if (ret != 0)
+				goto err_ref;
 			break;
 		}
 
@@ -58,7 +67,9 @@ int arizona_clk32k_enable(struct arizona *arizona)
 					 ARIZONA_CLK_32K_ENA);
 	}
 
-out:
+err_pm:
+	pm_runtime_put_sync(arizona->dev);
+err_ref:
 	if (ret != 0)
 		arizona->clk32k_ref--;
 
@@ -83,6 +94,10 @@ int arizona_clk32k_disable(struct arizona *arizona)
 		switch (arizona->pdata.clk32k_src) {
 		case ARIZONA_32KZ_MCLK1:
 			pm_runtime_put_sync(arizona->dev);
+			clk_disable_unprepare(arizona->mclk[ARIZONA_MCLK1]);
+			break;
+		case ARIZONA_32KZ_MCLK2:
+			clk_disable_unprepare(arizona->mclk[ARIZONA_MCLK2]);
 			break;
 		}
 	}
@@ -735,7 +750,7 @@ static int arizona_suspend(struct device *dev)
 	return 0;
 }
 
-static int arizona_suspend_late(struct device *dev)
+static int arizona_suspend_noirq(struct device *dev)
 {
 	struct arizona *arizona = dev_get_drvdata(dev);
 
@@ -759,7 +774,7 @@ static int arizona_resume(struct device *dev)
 {
 	struct arizona *arizona = dev_get_drvdata(dev);
 
-	dev_dbg(arizona->dev, "Late resume, reenabling IRQ\n");
+	dev_dbg(arizona->dev, "Resume, reenabling IRQ\n");
 	enable_irq(arizona->irq);
 
 	return 0;
@@ -771,10 +786,8 @@ const struct dev_pm_ops arizona_pm_ops = {
 			   arizona_runtime_resume,
 			   NULL)
 	SET_SYSTEM_SLEEP_PM_OPS(arizona_suspend, arizona_resume)
-#ifdef CONFIG_PM_SLEEP
-	.suspend_late = arizona_suspend_late,
-	.resume_noirq = arizona_resume_noirq,
-#endif
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(arizona_suspend_noirq,
+				      arizona_resume_noirq)
 };
 EXPORT_SYMBOL_GPL(arizona_pm_ops);
 
@@ -790,35 +803,25 @@ unsigned long arizona_of_get_type(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(arizona_of_get_type);
 
-int arizona_of_get_named_gpio(struct arizona *arizona, const char *prop,
-			      bool mandatory)
-{
-	int gpio;
-
-	gpio = of_get_named_gpio(arizona->dev->of_node, prop, 0);
-	if (gpio < 0) {
-		if (mandatory)
-			dev_err(arizona->dev,
-				"Mandatory DT gpio %s missing/malformed: %d\n",
-				prop, gpio);
-
-		gpio = 0;
-	}
-
-	return gpio;
-}
-EXPORT_SYMBOL_GPL(arizona_of_get_named_gpio);
-
 static int arizona_of_get_core_pdata(struct arizona *arizona)
 {
 	struct arizona_pdata *pdata = &arizona->pdata;
 	struct property *prop;
 	const __be32 *cur;
 	u32 val;
+	u32 pdm_val[ARIZONA_MAX_PDM_SPK];
 	int ret, i;
 	int count = 0;
 
-	pdata->reset = arizona_of_get_named_gpio(arizona, "wlf,reset", true);
+	pdata->reset = of_get_named_gpio(arizona->dev->of_node, "wlf,reset", 0);
+	if (pdata->reset == -EPROBE_DEFER) {
+		return pdata->reset;
+	} else if (pdata->reset < 0) {
+		dev_err(arizona->dev, "Reset GPIO missing/malformed: %d\n",
+			pdata->reset);
+
+		pdata->reset = 0;
+	}
 
 	ret = of_property_read_u32_array(arizona->dev->of_node,
 					 "wlf,gpio-defaults",
@@ -871,6 +874,35 @@ static int arizona_of_get_core_pdata(struct arizona *arizona)
 		count++;
 	}
 
+	count = 0;
+	of_property_for_each_u32(arizona->dev->of_node,
+				 "wlf,max-channels-clocked",
+				 prop, cur, val) {
+		if (count == ARRAY_SIZE(pdata->max_channels_clocked))
+			break;
+
+		pdata->max_channels_clocked[count] = val;
+		count++;
+	}
+
+	ret = of_property_read_u32_array(arizona->dev->of_node,
+					 "wlf,spk-fmt",
+					 pdm_val,
+					 ARRAY_SIZE(pdm_val));
+
+	if (ret >= 0)
+		for (count = 0; count < ARRAY_SIZE(pdata->spk_fmt); ++count)
+			pdata->spk_fmt[count] = pdm_val[count];
+
+	ret = of_property_read_u32_array(arizona->dev->of_node,
+					 "wlf,spk-mute",
+					 pdm_val,
+					 ARRAY_SIZE(pdm_val));
+
+	if (ret >= 0)
+		for (count = 0; count < ARRAY_SIZE(pdata->spk_mute); ++count)
+			pdata->spk_mute[count] = pdm_val[count];
+
 	return 0;
 }
 
@@ -1000,6 +1032,7 @@ static const struct mfd_cell wm8998_devs[] = {
 
 int arizona_dev_init(struct arizona *arizona)
 {
+	const char * const mclk_name[] = { "mclk1", "mclk2" };
 	struct device *dev = arizona->dev;
 	const char *type_name = NULL;
 	unsigned int reg, val, mask;
@@ -1010,11 +1043,24 @@ int arizona_dev_init(struct arizona *arizona)
 	dev_set_drvdata(arizona->dev, arizona);
 	mutex_init(&arizona->clk_lock);
 
-	if (dev_get_platdata(arizona->dev))
+	if (dev_get_platdata(arizona->dev)) {
 		memcpy(&arizona->pdata, dev_get_platdata(arizona->dev),
 		       sizeof(arizona->pdata));
-	else
-		arizona_of_get_core_pdata(arizona);
+	} else {
+		ret = arizona_of_get_core_pdata(arizona);
+		if (ret < 0)
+			return ret;
+	}
+
+	BUILD_BUG_ON(ARRAY_SIZE(arizona->mclk) != ARRAY_SIZE(mclk_name));
+	for (i = 0; i < ARRAY_SIZE(arizona->mclk); i++) {
+		arizona->mclk[i] = devm_clk_get(arizona->dev, mclk_name[i]);
+		if (IS_ERR(arizona->mclk[i])) {
+			dev_info(arizona->dev, "Failed to get %s: %ld\n",
+				 mclk_name[i], PTR_ERR(arizona->mclk[i]));
+			arizona->mclk[i] = NULL;
+		}
+	}
 
 	regcache_cache_only(arizona->regmap, true);
 
@@ -1035,7 +1081,7 @@ int arizona_dev_init(struct arizona *arizona)
 	default:
 		dev_err(arizona->dev, "Unknown device type %d\n",
 			arizona->type);
-		return -EINVAL;
+		return -ENODEV;
 	}
 
 	/* Mark DCVDD as external, LDO1 driver will clear if internal */
@@ -1121,6 +1167,7 @@ int arizona_dev_init(struct arizona *arizona)
 		break;
 	default:
 		dev_err(arizona->dev, "Unknown device ID: %x\n", reg);
+		ret = -ENODEV;
 		goto err_reset;
 	}
 
@@ -1280,12 +1327,14 @@ int arizona_dev_init(struct arizona *arizona)
 		break;
 	default:
 		dev_err(arizona->dev, "Unknown device ID %x\n", reg);
+		ret = -ENODEV;
 		goto err_reset;
 	}
 
 	if (!subdevs) {
 		dev_err(arizona->dev,
 			"No kernel support for device ID %x\n", reg);
+		ret = -ENODEV;
 		goto err_reset;
 	}
 
diff --git a/drivers/mfd/atmel-hlcdc.c b/drivers/mfd/atmel-hlcdc.c
index eca7ea69b81c..4b15b0840f16 100644
--- a/drivers/mfd/atmel-hlcdc.c
+++ b/drivers/mfd/atmel-hlcdc.c
@@ -50,8 +50,9 @@ static int regmap_atmel_hlcdc_reg_write(void *context, unsigned int reg,
 	if (reg <= ATMEL_HLCDC_DIS) {
 		u32 status;
 
-		readl_poll_timeout(hregmap->regs + ATMEL_HLCDC_SR, status,
-				   !(status & ATMEL_HLCDC_SIP), 1, 100);
+		readl_poll_timeout_atomic(hregmap->regs + ATMEL_HLCDC_SR,
+					  status, !(status & ATMEL_HLCDC_SIP),
+					  1, 100);
 	}
 
 	writel(val, hregmap->regs + reg);
diff --git a/drivers/mfd/axp20x-rsb.c b/drivers/mfd/axp20x-rsb.c
index a407527bcd09..a732cb50bcff 100644
--- a/drivers/mfd/axp20x-rsb.c
+++ b/drivers/mfd/axp20x-rsb.c
@@ -61,6 +61,7 @@ static int axp20x_rsb_remove(struct sunxi_rsb_device *rdev)
 
 static const struct of_device_id axp20x_rsb_of_match[] = {
 	{ .compatible = "x-powers,axp223", .data = (void *)AXP223_ID },
+	{ .compatible = "x-powers,axp806", .data = (void *)AXP806_ID },
 	{ .compatible = "x-powers,axp809", .data = (void *)AXP809_ID },
 	{ },
 };
diff --git a/drivers/mfd/axp20x.c b/drivers/mfd/axp20x.c
index fd80b0981f0f..ba130be32e61 100644
--- a/drivers/mfd/axp20x.c
+++ b/drivers/mfd/axp20x.c
@@ -38,6 +38,7 @@ static const char * const axp20x_model_names[] = {
 	"AXP221",
 	"AXP223",
 	"AXP288",
+	"AXP806",
 	"AXP809",
 };
 
@@ -129,6 +130,27 @@ static const struct regmap_access_table axp288_volatile_table = {
 	.n_yes_ranges	= ARRAY_SIZE(axp288_volatile_ranges),
 };
 
+static const struct regmap_range axp806_writeable_ranges[] = {
+	regmap_reg_range(AXP20X_DATACACHE(0), AXP20X_DATACACHE(3)),
+	regmap_reg_range(AXP806_PWR_OUT_CTRL1, AXP806_CLDO3_V_CTRL),
+	regmap_reg_range(AXP20X_IRQ1_EN, AXP20X_IRQ2_EN),
+	regmap_reg_range(AXP20X_IRQ1_STATE, AXP20X_IRQ2_STATE),
+};
+
+static const struct regmap_range axp806_volatile_ranges[] = {
+	regmap_reg_range(AXP20X_IRQ1_STATE, AXP20X_IRQ2_STATE),
+};
+
+static const struct regmap_access_table axp806_writeable_table = {
+	.yes_ranges	= axp806_writeable_ranges,
+	.n_yes_ranges	= ARRAY_SIZE(axp806_writeable_ranges),
+};
+
+static const struct regmap_access_table axp806_volatile_table = {
+	.yes_ranges	= axp806_volatile_ranges,
+	.n_yes_ranges	= ARRAY_SIZE(axp806_volatile_ranges),
+};
+
 static struct resource axp152_pek_resources[] = {
 	DEFINE_RES_IRQ_NAMED(AXP152_IRQ_PEK_RIS_EDGE, "PEK_DBR"),
 	DEFINE_RES_IRQ_NAMED(AXP152_IRQ_PEK_FAL_EDGE, "PEK_DBF"),
@@ -278,6 +300,15 @@ static const struct regmap_config axp288_regmap_config = {
 	.cache_type	= REGCACHE_RBTREE,
 };
 
+static const struct regmap_config axp806_regmap_config = {
+	.reg_bits	= 8,
+	.val_bits	= 8,
+	.wr_table	= &axp806_writeable_table,
+	.volatile_table	= &axp806_volatile_table,
+	.max_register	= AXP806_VREF_TEMP_WARN_L,
+	.cache_type	= REGCACHE_RBTREE,
+};
+
 #define INIT_REGMAP_IRQ(_variant, _irq, _off, _mask)			\
 	[_variant##_IRQ_##_irq] = { .reg_offset = (_off), .mask = BIT(_mask) }
 
@@ -409,6 +440,21 @@ static const struct regmap_irq axp288_regmap_irqs[] = {
 	INIT_REGMAP_IRQ(AXP288, BC_USB_CHNG,            5, 1),
 };
 
+static const struct regmap_irq axp806_regmap_irqs[] = {
+	INIT_REGMAP_IRQ(AXP806, DIE_TEMP_HIGH_LV1,	0, 0),
+	INIT_REGMAP_IRQ(AXP806, DIE_TEMP_HIGH_LV2,	0, 1),
+	INIT_REGMAP_IRQ(AXP806, DCDCA_V_LOW,		0, 3),
+	INIT_REGMAP_IRQ(AXP806, DCDCB_V_LOW,		0, 4),
+	INIT_REGMAP_IRQ(AXP806, DCDCC_V_LOW,		0, 5),
+	INIT_REGMAP_IRQ(AXP806, DCDCD_V_LOW,		0, 6),
+	INIT_REGMAP_IRQ(AXP806, DCDCE_V_LOW,		0, 7),
+	INIT_REGMAP_IRQ(AXP806, PWROK_LONG,		1, 0),
+	INIT_REGMAP_IRQ(AXP806, PWROK_SHORT,		1, 1),
+	INIT_REGMAP_IRQ(AXP806, WAKEUP,			1, 4),
+	INIT_REGMAP_IRQ(AXP806, PWROK_FALL,		1, 5),
+	INIT_REGMAP_IRQ(AXP806, PWROK_RISE,		1, 6),
+};
+
 static const struct regmap_irq axp809_regmap_irqs[] = {
 	INIT_REGMAP_IRQ(AXP809, ACIN_OVER_V,		0, 7),
 	INIT_REGMAP_IRQ(AXP809, ACIN_PLUGIN,		0, 6),
@@ -494,6 +540,18 @@ static const struct regmap_irq_chip axp288_regmap_irq_chip = {
 
 };
 
+static const struct regmap_irq_chip axp806_regmap_irq_chip = {
+	.name			= "axp806",
+	.status_base		= AXP20X_IRQ1_STATE,
+	.ack_base		= AXP20X_IRQ1_STATE,
+	.mask_base		= AXP20X_IRQ1_EN,
+	.mask_invert		= true,
+	.init_ack_masked	= true,
+	.irqs			= axp806_regmap_irqs,
+	.num_irqs		= ARRAY_SIZE(axp806_regmap_irqs),
+	.num_regs		= 2,
+};
+
 static const struct regmap_irq_chip axp809_regmap_irq_chip = {
 	.name			= "axp809",
 	.status_base		= AXP20X_IRQ1_STATE,
@@ -508,6 +566,9 @@ static const struct regmap_irq_chip axp809_regmap_irq_chip = {
 
 static struct mfd_cell axp20x_cells[] = {
 	{
+		.name		= "axp20x-gpio",
+		.of_compatible	= "x-powers,axp209-gpio",
+	}, {
 		.name		= "axp20x-pek",
 		.num_resources	= ARRAY_SIZE(axp20x_pek_resources),
 		.resources	= axp20x_pek_resources,
@@ -660,12 +721,20 @@ static struct mfd_cell axp288_cells[] = {
 	},
 };
 
+static struct mfd_cell axp806_cells[] = {
+	{
+		.id			= 2,
+		.name			= "axp20x-regulator",
+	},
+};
+
 static struct mfd_cell axp809_cells[] = {
 	{
 		.name			= "axp20x-pek",
 		.num_resources		= ARRAY_SIZE(axp809_pek_resources),
 		.resources		= axp809_pek_resources,
 	}, {
+		.id			= 1,
 		.name			= "axp20x-regulator",
 	},
 };
@@ -732,6 +801,12 @@ int axp20x_match_device(struct axp20x_dev *axp20x)
 		axp20x->regmap_cfg = &axp288_regmap_config;
 		axp20x->regmap_irq_chip = &axp288_regmap_irq_chip;
 		break;
+	case AXP806_ID:
+		axp20x->nr_cells = ARRAY_SIZE(axp806_cells);
+		axp20x->cells = axp806_cells;
+		axp20x->regmap_cfg = &axp806_regmap_config;
+		axp20x->regmap_irq_chip = &axp806_regmap_irq_chip;
+		break;
 	case AXP809_ID:
 		axp20x->nr_cells = ARRAY_SIZE(axp809_cells);
 		axp20x->cells = axp809_cells;
diff --git a/drivers/mfd/cros_ec.c b/drivers/mfd/cros_ec.c
index 0eee63542038..abd83424b498 100644
--- a/drivers/mfd/cros_ec.c
+++ b/drivers/mfd/cros_ec.c
@@ -23,6 +23,7 @@
 #include <linux/module.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/cros_ec.h>
+#include <asm/unaligned.h>
 
 #define CROS_EC_DEV_EC_INDEX 0
 #define CROS_EC_DEV_PD_INDEX 1
@@ -49,11 +50,28 @@ static const struct mfd_cell ec_pd_cell = {
 	.pdata_size = sizeof(pd_p),
 };
 
+static irqreturn_t ec_irq_thread(int irq, void *data)
+{
+	struct cros_ec_device *ec_dev = data;
+	int ret;
+
+	if (device_may_wakeup(ec_dev->dev))
+		pm_wakeup_event(ec_dev->dev, 0);
+
+	ret = cros_ec_get_next_event(ec_dev);
+	if (ret > 0)
+		blocking_notifier_call_chain(&ec_dev->event_notifier,
+					     0, ec_dev);
+	return IRQ_HANDLED;
+}
+
 int cros_ec_register(struct cros_ec_device *ec_dev)
 {
 	struct device *dev = ec_dev->dev;
 	int err = 0;
 
+	BLOCKING_INIT_NOTIFIER_HEAD(&ec_dev->event_notifier);
+
 	ec_dev->max_request = sizeof(struct ec_params_hello);
 	ec_dev->max_response = sizeof(struct ec_response_get_protocol_info);
 	ec_dev->max_passthru = 0;
@@ -70,13 +88,24 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 
 	cros_ec_query_all(ec_dev);
 
+	if (ec_dev->irq) {
+		err = request_threaded_irq(ec_dev->irq, NULL, ec_irq_thread,
+					   IRQF_TRIGGER_LOW | IRQF_ONESHOT,
+					   "chromeos-ec", ec_dev);
+		if (err) {
+			dev_err(dev, "Failed to request IRQ %d: %d",
+				ec_dev->irq, err);
+			return err;
+		}
+	}
+
 	err = mfd_add_devices(ec_dev->dev, PLATFORM_DEVID_AUTO, &ec_cell, 1,
 			      NULL, ec_dev->irq, NULL);
 	if (err) {
 		dev_err(dev,
 			"Failed to register Embedded Controller subdevice %d\n",
 			err);
-		return err;
+		goto fail_mfd;
 	}
 
 	if (ec_dev->max_passthru) {
@@ -94,7 +123,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 			dev_err(dev,
 				"Failed to register Power Delivery subdevice %d\n",
 				err);
-			return err;
+			goto fail_mfd;
 		}
 	}
 
@@ -103,13 +132,18 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 		if (err) {
 			mfd_remove_devices(dev);
 			dev_err(dev, "Failed to register sub-devices\n");
-			return err;
+			goto fail_mfd;
 		}
 	}
 
 	dev_info(dev, "Chrome EC device registered\n");
 
 	return 0;
+
+fail_mfd:
+	if (ec_dev->irq)
+		free_irq(ec_dev->irq, ec_dev);
+	return err;
 }
 EXPORT_SYMBOL(cros_ec_register);
 
@@ -136,13 +170,31 @@ int cros_ec_suspend(struct cros_ec_device *ec_dev)
 }
 EXPORT_SYMBOL(cros_ec_suspend);
 
+static void cros_ec_drain_events(struct cros_ec_device *ec_dev)
+{
+	while (cros_ec_get_next_event(ec_dev) > 0)
+		blocking_notifier_call_chain(&ec_dev->event_notifier,
+					     1, ec_dev);
+}
+
 int cros_ec_resume(struct cros_ec_device *ec_dev)
 {
 	enable_irq(ec_dev->irq);
 
+	/*
+	 * In some cases, we need to distinguish between events that occur
+	 * during suspend if the EC is not a wake source. For example,
+	 * keypresses during suspend should be discarded if it does not wake
+	 * the system.
+	 *
+	 * If the EC is not a wake source, drain the event queue and mark them
+	 * as "queued during suspend".
+	 */
 	if (ec_dev->wake_enabled) {
 		disable_irq_wake(ec_dev->irq);
 		ec_dev->wake_enabled = 0;
+	} else {
+		cros_ec_drain_events(ec_dev);
 	}
 
 	return 0;
diff --git a/drivers/mfd/cros_ec_spi.c b/drivers/mfd/cros_ec_spi.c
index ebe9b9477cb2..a518832ed5f5 100644
--- a/drivers/mfd/cros_ec_spi.c
+++ b/drivers/mfd/cros_ec_spi.c
@@ -366,7 +366,6 @@ static int cros_ec_spi_receive_response(struct cros_ec_device *ec_dev,
 static int cros_ec_pkt_xfer_spi(struct cros_ec_device *ec_dev,
 				struct cros_ec_command *ec_msg)
 {
-	struct ec_host_request *request;
 	struct ec_host_response *response;
 	struct cros_ec_spi *ec_spi = ec_dev->priv;
 	struct spi_transfer trans, trans_delay;
@@ -378,7 +377,6 @@ static int cros_ec_pkt_xfer_spi(struct cros_ec_device *ec_dev,
 	int ret = 0, final_ret;
 
 	len = cros_ec_prepare_tx(ec_dev, ec_msg);
-	request = (struct ec_host_request *)ec_dev->dout;
 	dev_dbg(ec_dev->dev, "prepared, len=%d\n", len);
 
 	/* If it's too soon to do another transaction, wait */
diff --git a/drivers/mfd/da9052-core.c b/drivers/mfd/da9052-core.c
index c0bf68a3e614..a88c2065d8ab 100644
--- a/drivers/mfd/da9052-core.c
+++ b/drivers/mfd/da9052-core.c
@@ -167,6 +167,7 @@ static bool da9052_reg_writeable(struct device *dev, unsigned int reg)
 	case DA9052_EVENT_B_REG:
 	case DA9052_EVENT_C_REG:
 	case DA9052_EVENT_D_REG:
+	case DA9052_FAULTLOG_REG:
 	case DA9052_IRQ_MASK_A_REG:
 	case DA9052_IRQ_MASK_B_REG:
 	case DA9052_IRQ_MASK_C_REG:
@@ -541,6 +542,52 @@ const struct regmap_config da9052_regmap_config = {
 };
 EXPORT_SYMBOL_GPL(da9052_regmap_config);
 
+static int da9052_clear_fault_log(struct da9052 *da9052)
+{
+	int ret = 0;
+	int fault_log = 0;
+
+	fault_log = da9052_reg_read(da9052, DA9052_FAULTLOG_REG);
+	if (fault_log < 0) {
+		dev_err(da9052->dev,
+			"Cannot read FAULT_LOG %d\n", fault_log);
+		return fault_log;
+	}
+
+	if (fault_log) {
+		if (fault_log & DA9052_FAULTLOG_TWDERROR)
+			dev_dbg(da9052->dev,
+				"Fault log entry detected: TWD_ERROR\n");
+		if (fault_log & DA9052_FAULTLOG_VDDFAULT)
+			dev_dbg(da9052->dev,
+				"Fault log entry detected: VDD_FAULT\n");
+		if (fault_log & DA9052_FAULTLOG_VDDSTART)
+			dev_dbg(da9052->dev,
+				"Fault log entry detected: VDD_START\n");
+		if (fault_log & DA9052_FAULTLOG_TEMPOVER)
+			dev_dbg(da9052->dev,
+				"Fault log entry detected: TEMP_OVER\n");
+		if (fault_log & DA9052_FAULTLOG_KEYSHUT)
+			dev_dbg(da9052->dev,
+				"Fault log entry detected: KEY_SHUT\n");
+		if (fault_log & DA9052_FAULTLOG_NSDSET)
+			dev_dbg(da9052->dev,
+				"Fault log entry detected: nSD_SHUT\n");
+		if (fault_log & DA9052_FAULTLOG_WAITSET)
+			dev_dbg(da9052->dev,
+				"Fault log entry detected: WAIT_SHUT\n");
+
+		ret = da9052_reg_write(da9052,
+					DA9052_FAULTLOG_REG,
+					0xFF);
+		if (ret < 0)
+			dev_err(da9052->dev,
+				"Cannot reset FAULT_LOG values %d\n", ret);
+	}
+
+	return ret;
+}
+
 int da9052_device_init(struct da9052 *da9052, u8 chip_id)
 {
 	struct da9052_pdata *pdata = dev_get_platdata(da9052->dev);
@@ -549,6 +596,10 @@ int da9052_device_init(struct da9052 *da9052, u8 chip_id)
 	mutex_init(&da9052->auxadc_lock);
 	init_completion(&da9052->done);
 
+	ret = da9052_clear_fault_log(da9052);
+	if (ret < 0)
+		dev_warn(da9052->dev, "Cannot clear FAULT_LOG\n");
+
 	if (pdata && pdata->init != NULL)
 		pdata->init(da9052);
 
diff --git a/drivers/mfd/da9063-core.c b/drivers/mfd/da9063-core.c
index af841c165787..6c2870d4e754 100644
--- a/drivers/mfd/da9063-core.c
+++ b/drivers/mfd/da9063-core.c
@@ -4,8 +4,8 @@
  * Copyright 2012 Dialog Semiconductors Ltd.
  * Copyright 2013 Philipp Zabel, Pengutronix
  *
- * Author: Krystian Garbaciak <krystian.garbaciak@diasemi.com>,
- *         Michal Hajduk <michal.hajduk@diasemi.com>
+ * Author: Krystian Garbaciak, Dialog Semiconductor
+ * Author: Michal Hajduk, Dialog Semiconductor
  *
  *  This program is free software; you can redistribute  it and/or modify it
  *  under  the terms of  the GNU General  Public License as published by the
@@ -242,5 +242,6 @@ void da9063_device_exit(struct da9063 *da9063)
 }
 
 MODULE_DESCRIPTION("PMIC driver for Dialog DA9063");
-MODULE_AUTHOR("Krystian Garbaciak <krystian.garbaciak@diasemi.com>, Michal Hajduk <michal.hajduk@diasemi.com>");
+MODULE_AUTHOR("Krystian Garbaciak");
+MODULE_AUTHOR("Michal Hajduk");
 MODULE_LICENSE("GPL");
diff --git a/drivers/mfd/da9063-i2c.c b/drivers/mfd/da9063-i2c.c
index 73901084945f..981805a2c521 100644
--- a/drivers/mfd/da9063-i2c.c
+++ b/drivers/mfd/da9063-i2c.c
@@ -3,7 +3,7 @@
  * Copyright 2012 Dialog Semiconductor Ltd.
  * Copyright 2013 Philipp Zabel, Pengutronix
  *
- * Author: Krystian Garbaciak <krystian.garbaciak@diasemi.com>
+ * Author: Krystian Garbaciak, Dialog Semiconductor
  *
  *  This program is free software; you can redistribute  it and/or modify it
  *  under  the terms of  the GNU General  Public License as published by the
diff --git a/drivers/mfd/da9063-irq.c b/drivers/mfd/da9063-irq.c
index 7e903fcb8813..207bbfe55449 100644
--- a/drivers/mfd/da9063-irq.c
+++ b/drivers/mfd/da9063-irq.c
@@ -3,7 +3,7 @@
  * Copyright 2012 Dialog Semiconductor Ltd.
  * Copyright 2013 Philipp Zabel, Pengutronix
  *
- * Author: Michal Hajduk <michal.hajduk@diasemi.com>
+ * Author: Michal Hajduk, Dialog Semiconductor
  *
  *  This program is free software; you can redistribute  it and/or modify it
  *  under  the terms of  the GNU General  Public License as published by the
diff --git a/drivers/mfd/db8500-prcmu.c b/drivers/mfd/db8500-prcmu.c
index 388e268b9bcf..ca38a6a14110 100644
--- a/drivers/mfd/db8500-prcmu.c
+++ b/drivers/mfd/db8500-prcmu.c
@@ -938,25 +938,6 @@ int db8500_prcmu_get_ddr_opp(void)
 	return readb(PRCM_DDR_SUBSYS_APE_MINBW);
 }
 
-/**
- * db8500_set_ddr_opp - set the appropriate DDR OPP
- * @opp: The new DDR operating point to which transition is to be made
- * Returns: 0 on success, non-zero on failure
- *
- * This function sets the operating point of the DDR.
- */
-static bool enable_set_ddr_opp;
-int db8500_prcmu_set_ddr_opp(u8 opp)
-{
-	if (opp < DDR_100_OPP || opp > DDR_25_OPP)
-		return -EINVAL;
-	/* Changing the DDR OPP can hang the hardware pre-v21 */
-	if (enable_set_ddr_opp)
-		writeb(opp, PRCM_DDR_SUBSYS_APE_MINBW);
-
-	return 0;
-}
-
 /* Divide the frequency of certain clocks by 2 for APE_50_PARTLY_25_OPP. */
 static void request_even_slower_clocks(bool enable)
 {
diff --git a/drivers/mfd/dm355evm_msp.c b/drivers/mfd/dm355evm_msp.c
index 270e19c0bba1..86eca614507b 100644
--- a/drivers/mfd/dm355evm_msp.c
+++ b/drivers/mfd/dm355evm_msp.c
@@ -209,7 +209,7 @@ static struct device *add_child(struct i2c_client *client, const char *name,
 		status = platform_device_add_data(pdev, pdata, pdata_len);
 		if (status < 0) {
 			dev_dbg(&pdev->dev, "can't add platform_data\n");
-			goto err;
+			goto put_device;
 		}
 	}
 
@@ -222,19 +222,20 @@ static struct device *add_child(struct i2c_client *client, const char *name,
 		status = platform_device_add_resources(pdev, &r, 1);
 		if (status < 0) {
 			dev_dbg(&pdev->dev, "can't add irq\n");
-			goto err;
+			goto put_device;
 		}
 	}
 
 	status = platform_device_add(pdev);
+	if (status)
+		goto put_device;
 
-err:
-	if (status < 0) {
-		platform_device_put(pdev);
-		dev_err(&client->dev, "can't add %s dev\n", name);
-		return ERR_PTR(status);
-	}
 	return &pdev->dev;
+
+put_device:
+	platform_device_put(pdev);
+	dev_err(&client->dev, "failed to add device %s\n", name);
+	return ERR_PTR(status);
 }
 
 static int add_children(struct i2c_client *client)
diff --git a/drivers/mfd/exynos-lpass.c b/drivers/mfd/exynos-lpass.c
new file mode 100644
index 000000000000..2e064fb8826f
--- /dev/null
+++ b/drivers/mfd/exynos-lpass.c
@@ -0,0 +1,185 @@
+/*
+ * Copyright (C) 2015 - 2016 Samsung Electronics Co., Ltd.
+ *
+ * Authors: Inha Song <ideal.song@samsung.com>
+ *          Sylwester Nawrocki <s.nawrocki@samsung.com>
+ *
+ * Samsung Exynos SoC series Low Power Audio Subsystem driver.
+ *
+ * This module provides regmap for the Top SFR region and instantiates
+ * devices for IP blocks like DMAC, I2S, UART.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ */
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/mfd/syscon.h>
+#include <linux/mfd/syscon/exynos5-pmu.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+/* LPASS Top register definitions */
+#define SFR_LPASS_CORE_SW_RESET		0x08
+#define  LPASS_SB_SW_RESET		BIT(11)
+#define  LPASS_UART_SW_RESET		BIT(10)
+#define  LPASS_PCM_SW_RESET		BIT(9)
+#define  LPASS_I2S_SW_RESET		BIT(8)
+#define  LPASS_WDT1_SW_RESET		BIT(4)
+#define  LPASS_WDT0_SW_RESET		BIT(3)
+#define  LPASS_TIMER_SW_RESET		BIT(2)
+#define  LPASS_MEM_SW_RESET		BIT(1)
+#define  LPASS_DMA_SW_RESET		BIT(0)
+
+#define SFR_LPASS_INTR_CA5_MASK		0x48
+#define SFR_LPASS_INTR_CPU_MASK		0x58
+#define  LPASS_INTR_APM			BIT(9)
+#define  LPASS_INTR_MIF			BIT(8)
+#define  LPASS_INTR_TIMER		BIT(7)
+#define  LPASS_INTR_DMA			BIT(6)
+#define  LPASS_INTR_GPIO		BIT(5)
+#define  LPASS_INTR_I2S			BIT(4)
+#define  LPASS_INTR_PCM			BIT(3)
+#define  LPASS_INTR_SLIMBUS		BIT(2)
+#define  LPASS_INTR_UART		BIT(1)
+#define  LPASS_INTR_SFR			BIT(0)
+
+struct exynos_lpass {
+	/* pointer to the Power Management Unit regmap */
+	struct regmap *pmu;
+	/* pointer to the LPASS TOP regmap */
+	struct regmap *top;
+};
+
+static void exynos_lpass_core_sw_reset(struct exynos_lpass *lpass, int mask)
+{
+	unsigned int val = 0;
+
+	regmap_read(lpass->top, SFR_LPASS_CORE_SW_RESET, &val);
+
+	val &= ~mask;
+	regmap_write(lpass->top, SFR_LPASS_CORE_SW_RESET, val);
+
+	usleep_range(100, 150);
+
+	val |= mask;
+	regmap_write(lpass->top, SFR_LPASS_CORE_SW_RESET, val);
+}
+
+static void exynos_lpass_enable(struct exynos_lpass *lpass)
+{
+	/* Unmask SFR, DMA and I2S interrupt */
+	regmap_write(lpass->top, SFR_LPASS_INTR_CA5_MASK,
+		     LPASS_INTR_SFR | LPASS_INTR_DMA | LPASS_INTR_I2S);
+
+	regmap_write(lpass->top, SFR_LPASS_INTR_CPU_MASK,
+		     LPASS_INTR_SFR | LPASS_INTR_DMA | LPASS_INTR_I2S);
+
+	/* Activate related PADs from retention state */
+	regmap_write(lpass->pmu, EXYNOS5433_PAD_RETENTION_AUD_OPTION,
+		     EXYNOS5433_PAD_INITIATE_WAKEUP_FROM_LOWPWR);
+
+	exynos_lpass_core_sw_reset(lpass, LPASS_I2S_SW_RESET);
+	exynos_lpass_core_sw_reset(lpass, LPASS_DMA_SW_RESET);
+	exynos_lpass_core_sw_reset(lpass, LPASS_MEM_SW_RESET);
+}
+
+static void exynos_lpass_disable(struct exynos_lpass *lpass)
+{
+	/* Mask any unmasked IP interrupt sources */
+	regmap_write(lpass->top, SFR_LPASS_INTR_CPU_MASK, 0);
+	regmap_write(lpass->top, SFR_LPASS_INTR_CA5_MASK, 0);
+
+	/* Deactivate related PADs from retention state */
+	regmap_write(lpass->pmu, EXYNOS5433_PAD_RETENTION_AUD_OPTION, 0);
+}
+
+static const struct regmap_config exynos_lpass_reg_conf = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0xfc,
+	.fast_io	= true,
+};
+
+static int exynos_lpass_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct exynos_lpass *lpass;
+	void __iomem *base_top;
+	struct resource *res;
+
+	lpass = devm_kzalloc(dev, sizeof(*lpass), GFP_KERNEL);
+	if (!lpass)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base_top = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base_top))
+		return PTR_ERR(base_top);
+
+	lpass->top = regmap_init_mmio(dev, base_top,
+					&exynos_lpass_reg_conf);
+	if (IS_ERR(lpass->top)) {
+		dev_err(dev, "LPASS top regmap initialization failed\n");
+		return PTR_ERR(lpass->top);
+	}
+
+	lpass->pmu = syscon_regmap_lookup_by_phandle(dev->of_node,
+						"samsung,pmu-syscon");
+	if (IS_ERR(lpass->pmu)) {
+		dev_err(dev, "Failed to lookup PMU regmap\n");
+		return PTR_ERR(lpass->pmu);
+	}
+
+	platform_set_drvdata(pdev, lpass);
+	exynos_lpass_enable(lpass);
+
+	return of_platform_populate(dev->of_node, NULL, NULL, dev);
+}
+
+static int __maybe_unused exynos_lpass_suspend(struct device *dev)
+{
+	struct exynos_lpass *lpass = dev_get_drvdata(dev);
+
+	exynos_lpass_disable(lpass);
+
+	return 0;
+}
+
+static int __maybe_unused exynos_lpass_resume(struct device *dev)
+{
+	struct exynos_lpass *lpass = dev_get_drvdata(dev);
+
+	exynos_lpass_enable(lpass);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(lpass_pm_ops, exynos_lpass_suspend,
+					exynos_lpass_resume);
+
+static const struct of_device_id exynos_lpass_of_match[] = {
+	{ .compatible = "samsung,exynos5433-lpass" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, exynos_lpass_of_match);
+
+static struct platform_driver exynos_lpass_driver = {
+	.driver = {
+		.name		= "exynos-lpass",
+		.pm		= &lpass_pm_ops,
+		.of_match_table	= exynos_lpass_of_match,
+	},
+	.probe	= exynos_lpass_probe,
+};
+module_platform_driver(exynos_lpass_driver);
+
+MODULE_DESCRIPTION("Samsung Low Power Audio Subsystem driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/intel-lpss-acpi.c b/drivers/mfd/intel-lpss-acpi.c
index 7ddc4a9563ea..6bf8d643d942 100644
--- a/drivers/mfd/intel-lpss-acpi.c
+++ b/drivers/mfd/intel-lpss-acpi.c
@@ -52,6 +52,18 @@ static const struct intel_lpss_platform_info bxt_i2c_info = {
 	.properties = bxt_i2c_properties,
 };
 
+static struct property_entry apl_i2c_properties[] = {
+	PROPERTY_ENTRY_U32("i2c-sda-hold-time-ns", 207),
+	PROPERTY_ENTRY_U32("i2c-sda-falling-time-ns", 171),
+	PROPERTY_ENTRY_U32("i2c-scl-falling-time-ns", 208),
+	{ },
+};
+
+static const struct intel_lpss_platform_info apl_i2c_info = {
+	.clk_rate = 133000000,
+	.properties = apl_i2c_properties,
+};
+
 static const struct acpi_device_id intel_lpss_acpi_ids[] = {
 	/* SPT */
 	{ "INT3446", (kernel_ulong_t)&spt_i2c_info },
@@ -61,7 +73,7 @@ static const struct acpi_device_id intel_lpss_acpi_ids[] = {
 	{ "80860ABC", (kernel_ulong_t)&bxt_info },
 	{ "80860AC2", (kernel_ulong_t)&bxt_info },
 	/* APL */
-	{ "80865AAC", (kernel_ulong_t)&bxt_i2c_info },
+	{ "80865AAC", (kernel_ulong_t)&apl_i2c_info },
 	{ "80865ABC", (kernel_ulong_t)&bxt_info },
 	{ "80865AC2", (kernel_ulong_t)&bxt_info },
 	{ }
diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c
index 1d79a3c9370f..3228fd182a99 100644
--- a/drivers/mfd/intel-lpss-pci.c
+++ b/drivers/mfd/intel-lpss-pci.c
@@ -111,6 +111,31 @@ static const struct intel_lpss_platform_info bxt_i2c_info = {
 	.properties = bxt_i2c_properties,
 };
 
+static struct property_entry apl_i2c_properties[] = {
+	PROPERTY_ENTRY_U32("i2c-sda-hold-time-ns", 207),
+	PROPERTY_ENTRY_U32("i2c-sda-falling-time-ns", 171),
+	PROPERTY_ENTRY_U32("i2c-scl-falling-time-ns", 208),
+	{ },
+};
+
+static const struct intel_lpss_platform_info apl_i2c_info = {
+	.clk_rate = 133000000,
+	.properties = apl_i2c_properties,
+};
+
+static const struct intel_lpss_platform_info kbl_info = {
+	.clk_rate = 120000000,
+};
+
+static const struct intel_lpss_platform_info kbl_uart_info = {
+	.clk_rate = 120000000,
+	.clk_con_id = "baudclk",
+};
+
+static const struct intel_lpss_platform_info kbl_i2c_info = {
+	.clk_rate = 133000000,
+};
+
 static const struct pci_device_id intel_lpss_pci_ids[] = {
 	/* BXT A-Step */
 	{ PCI_VDEVICE(INTEL, 0x0aac), (kernel_ulong_t)&bxt_i2c_info },
@@ -146,14 +171,14 @@ static const struct pci_device_id intel_lpss_pci_ids[] = {
 	{ PCI_VDEVICE(INTEL, 0x1aee), (kernel_ulong_t)&bxt_uart_info },
 
 	/* APL */
-	{ PCI_VDEVICE(INTEL, 0x5aac), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5aae), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab0), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab2), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab4), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab6), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5aba), (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5aac), (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5aae), (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab0), (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab2), (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab4), (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab6), (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab8), (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5aba), (kernel_ulong_t)&apl_i2c_info },
 	{ PCI_VDEVICE(INTEL, 0x5abc), (kernel_ulong_t)&bxt_uart_info },
 	{ PCI_VDEVICE(INTEL, 0x5abe), (kernel_ulong_t)&bxt_uart_info },
 	{ PCI_VDEVICE(INTEL, 0x5ac0), (kernel_ulong_t)&bxt_uart_info },
@@ -181,6 +206,16 @@ static const struct pci_device_id intel_lpss_pci_ids[] = {
 	{ PCI_VDEVICE(INTEL, 0xa160), (kernel_ulong_t)&spt_i2c_info },
 	{ PCI_VDEVICE(INTEL, 0xa161), (kernel_ulong_t)&spt_i2c_info },
 	{ PCI_VDEVICE(INTEL, 0xa166), (kernel_ulong_t)&spt_uart_info },
+	/* KBL-H */
+	{ PCI_VDEVICE(INTEL, 0xa2a7), (kernel_ulong_t)&kbl_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa2a8), (kernel_ulong_t)&kbl_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa2a9), (kernel_ulong_t)&kbl_info },
+	{ PCI_VDEVICE(INTEL, 0xa2aa), (kernel_ulong_t)&kbl_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e0), (kernel_ulong_t)&kbl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e1), (kernel_ulong_t)&kbl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e2), (kernel_ulong_t)&kbl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e3), (kernel_ulong_t)&kbl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e6), (kernel_ulong_t)&kbl_uart_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, intel_lpss_pci_ids);
diff --git a/drivers/mfd/intel_msic.c b/drivers/mfd/intel_msic.c
index 25d486c543cb..2017446c5b4b 100644
--- a/drivers/mfd/intel_msic.c
+++ b/drivers/mfd/intel_msic.c
@@ -12,7 +12,7 @@
 #include <linux/err.h>
 #include <linux/gpio.h>
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/intel_msic.h>
 #include <linux/platform_device.h>
@@ -449,9 +449,4 @@ static struct platform_driver intel_msic_driver = {
 		.name	= "intel_msic",
 	},
 };
-
-module_platform_driver(intel_msic_driver);
-
-MODULE_DESCRIPTION("Driver for Intel MSIC");
-MODULE_AUTHOR("Mika Westerberg <mika.westerberg@linux.intel.com>");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(intel_msic_driver);
diff --git a/drivers/mfd/intel_soc_pmic_bxtwc.c b/drivers/mfd/intel_soc_pmic_bxtwc.c
index b9428767e615..43e54b7e908f 100644
--- a/drivers/mfd/intel_soc_pmic_bxtwc.c
+++ b/drivers/mfd/intel_soc_pmic_bxtwc.c
@@ -47,6 +47,8 @@
 #define BXTWC_MIRQLVL1		0x4E0E
 #define BXTWC_MPWRTNIRQ		0x4E0F
 
+#define BXTWC_MIRQLVL1_MCHGR	BIT(5)
+
 #define BXTWC_MTHRM0IRQ		0x4E12
 #define BXTWC_MTHRM1IRQ		0x4E13
 #define BXTWC_MTHRM2IRQ		0x4E14
@@ -109,7 +111,7 @@ static const struct regmap_irq bxtwc_regmap_irqs_level2[] = {
 	REGMAP_IRQ_REG(BXTWC_THRM2_IRQ, 2, 0xff),
 	REGMAP_IRQ_REG(BXTWC_BCU_IRQ, 3, 0x1f),
 	REGMAP_IRQ_REG(BXTWC_ADC_IRQ, 4, 0xff),
-	REGMAP_IRQ_REG(BXTWC_CHGR0_IRQ, 5, 0x1f),
+	REGMAP_IRQ_REG(BXTWC_CHGR0_IRQ, 5, 0x3f),
 	REGMAP_IRQ_REG(BXTWC_CHGR1_IRQ, 6, 0x1f),
 	REGMAP_IRQ_REG(BXTWC_GPIO0_IRQ, 7, 0xff),
 	REGMAP_IRQ_REG(BXTWC_GPIO1_IRQ, 8, 0x3f),
@@ -143,6 +145,10 @@ static struct resource adc_resources[] = {
 	DEFINE_RES_IRQ_NAMED(BXTWC_ADC_IRQ, "ADC"),
 };
 
+static struct resource usbc_resources[] = {
+	DEFINE_RES_IRQ_NAMED(BXTWC_CHGR0_IRQ, "USBC"),
+};
+
 static struct resource charger_resources[] = {
 	DEFINE_RES_IRQ_NAMED(BXTWC_CHGR0_IRQ, "CHARGER"),
 	DEFINE_RES_IRQ_NAMED(BXTWC_CHGR1_IRQ, "CHARGER1"),
@@ -170,6 +176,11 @@ static struct mfd_cell bxt_wc_dev[] = {
 		.resources = thermal_resources,
 	},
 	{
+		.name = "bxt_wcove_usbc",
+		.num_resources = ARRAY_SIZE(usbc_resources),
+		.resources = usbc_resources,
+	},
+	{
 		.name = "bxt_wcove_ext_charger",
 		.num_resources = ARRAY_SIZE(charger_resources),
 		.resources = charger_resources,
@@ -403,6 +414,16 @@ static int bxtwc_probe(struct platform_device *pdev)
 		goto err_sysfs;
 	}
 
+	/*
+	 * There is known hw bug. Upon reset BIT 5 of register
+	 * BXTWC_CHGR_LVL1_IRQ is 0 which is the expected value. However,
+	 * later it's set to 1(masked) automatically by hardware. So we
+	 * have the software workaround here to unmaksed it in order to let
+	 * charger interrutp work.
+	 */
+	regmap_update_bits(pmic->regmap, BXTWC_MIRQLVL1,
+				BXTWC_MIRQLVL1_MCHGR, 0);
+
 	return 0;
 
 err_sysfs:
diff --git a/drivers/mfd/lp873x.c b/drivers/mfd/lp873x.c
new file mode 100644
index 000000000000..873c608e6a5d
--- /dev/null
+++ b/drivers/mfd/lp873x.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Author: Keerthy <j-keerthy@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/mfd/core.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+
+#include <linux/mfd/lp873x.h>
+
+static const struct regmap_config lp873x_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = LP873X_REG_MAX,
+};
+
+static const struct mfd_cell lp873x_cells[] = {
+	{ .name = "lp873x-regulator", },
+	{ .name = "lp873x-gpio", },
+};
+
+static int lp873x_probe(struct i2c_client *client,
+			const struct i2c_device_id *ids)
+{
+	struct lp873x *lp873;
+	int ret;
+	unsigned int otpid;
+
+	lp873 = devm_kzalloc(&client->dev, sizeof(*lp873), GFP_KERNEL);
+	if (!lp873)
+		return -ENOMEM;
+
+	lp873->dev = &client->dev;
+
+	lp873->regmap = devm_regmap_init_i2c(client, &lp873x_regmap_config);
+	if (IS_ERR(lp873->regmap)) {
+		ret = PTR_ERR(lp873->regmap);
+		dev_err(lp873->dev,
+			"Failed to initialize register map: %d\n", ret);
+		return ret;
+	}
+
+	ret = regmap_read(lp873->regmap, LP873X_REG_OTP_REV, &otpid);
+	if (ret) {
+		dev_err(lp873->dev, "Failed to read OTP ID\n");
+		return ret;
+	}
+
+	lp873->rev = otpid & LP873X_OTP_REV_OTP_ID;
+
+	i2c_set_clientdata(client, lp873);
+
+	ret = mfd_add_devices(lp873->dev, PLATFORM_DEVID_AUTO, lp873x_cells,
+			      ARRAY_SIZE(lp873x_cells), NULL, 0, NULL);
+
+	return ret;
+}
+
+static const struct of_device_id of_lp873x_match_table[] = {
+	{ .compatible = "ti,lp8733", },
+	{ .compatible = "ti,lp8732", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, of_lp873x_match_table);
+
+static const struct i2c_device_id lp873x_id_table[] = {
+	{ "lp873x", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(i2c, lp873x_id_table);
+
+static struct i2c_driver lp873x_driver = {
+	.driver	= {
+		.name	= "lp873x",
+		.of_match_table = of_lp873x_match_table,
+	},
+	.probe		= lp873x_probe,
+	.id_table	= lp873x_id_table,
+};
+module_i2c_driver(lp873x_driver);
+
+MODULE_AUTHOR("J Keerthy <j-keerthy@ti.com>");
+MODULE_DESCRIPTION("LP873X chip family Multi-Function Device driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/lpc_ich.c b/drivers/mfd/lpc_ich.c
index bd3aa4578346..c8dee47b45d9 100644
--- a/drivers/mfd/lpc_ich.c
+++ b/drivers/mfd/lpc_ich.c
@@ -984,6 +984,10 @@ static int lpc_ich_init_wdt(struct pci_dev *dev)
 	int ret;
 	struct resource *res;
 
+	/* If we have ACPI based watchdog use that instead */
+	if (acpi_has_watchdog())
+		return -ENODEV;
+
 	/* Setup power management base register */
 	pci_read_config_dword(dev, priv->abase, &base_addr_cfg);
 	base_addr = base_addr_cfg & 0x0000ff80;
diff --git a/drivers/mfd/max14577.c b/drivers/mfd/max14577.c
index 6c245128ab2e..6cbe96b28f42 100644
--- a/drivers/mfd/max14577.c
+++ b/drivers/mfd/max14577.c
@@ -3,7 +3,7 @@
  *
  * Copyright (C) 2014 Samsung Electronics
  * Chanwoo Choi <cw00.choi@samsung.com>
- * Krzysztof Kozlowski <k.kozlowski@samsung.com>
+ * Krzysztof Kozlowski <krzk@kernel.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -569,6 +569,6 @@ static void __exit max14577_i2c_exit(void)
 }
 module_exit(max14577_i2c_exit);
 
-MODULE_AUTHOR("Chanwoo Choi <cw00.choi@samsung.com>, Krzysztof Kozlowski <k.kozlowski@samsung.com>");
+MODULE_AUTHOR("Chanwoo Choi <cw00.choi@samsung.com>, Krzysztof Kozlowski <krzk@kernel.org>");
 MODULE_DESCRIPTION("Maxim 14577/77836 multi-function core driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/mfd/max8997-irq.c b/drivers/mfd/max8997-irq.c
index b95a46d79b9d..326f17b632a7 100644
--- a/drivers/mfd/max8997-irq.c
+++ b/drivers/mfd/max8997-irq.c
@@ -139,7 +139,7 @@ static void max8997_irq_sync_unlock(struct irq_data *data)
 	mutex_unlock(&max8997->irqlock);
 }
 
-static const inline struct max8997_irq_data *
+inline static const struct max8997_irq_data *
 irq_to_max8997_irq(struct max8997_dev *max8997, struct irq_data *data)
 {
 	return &max8997_irqs[data->hwirq];
diff --git a/drivers/mfd/omap-usb-host.c b/drivers/mfd/omap-usb-host.c
index 1d924d1533c0..7aab376ecb84 100644
--- a/drivers/mfd/omap-usb-host.c
+++ b/drivers/mfd/omap-usb-host.c
@@ -162,7 +162,7 @@ static const char * const port_modes[] = {
  * provided port mode string as per the port_modes table.
  * If no match is found it returns -ENODEV
  */
-static const int omap_usbhs_get_dt_port_mode(const char *mode)
+static int omap_usbhs_get_dt_port_mode(const char *mode)
 {
 	int i;
 
diff --git a/drivers/mfd/pm8921-core.c b/drivers/mfd/pm8921-core.c
index 1b7ec0870c2a..0e3a2ea25942 100644
--- a/drivers/mfd/pm8921-core.c
+++ b/drivers/mfd/pm8921-core.c
@@ -309,6 +309,7 @@ static const struct regmap_config ssbi_regmap_config = {
 };
 
 static const struct of_device_id pm8921_id_table[] = {
+	{ .compatible = "qcom,pm8018", },
 	{ .compatible = "qcom,pm8058", },
 	{ .compatible = "qcom,pm8921", },
 	{ }
diff --git a/drivers/mfd/qcom_rpm.c b/drivers/mfd/qcom_rpm.c
index 2e44323455dd..52fafea06067 100644
--- a/drivers/mfd/qcom_rpm.c
+++ b/drivers/mfd/qcom_rpm.c
@@ -21,6 +21,7 @@
 #include <linux/mfd/qcom_rpm.h>
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
+#include <linux/clk.h>
 
 #include <dt-bindings/mfd/qcom-rpm.h>
 
@@ -48,6 +49,7 @@ struct qcom_rpm {
 	struct regmap *ipc_regmap;
 	unsigned ipc_offset;
 	unsigned ipc_bit;
+	struct clk *ramclk;
 
 	struct completion ack;
 	struct mutex lock;
@@ -388,11 +390,62 @@ static const struct qcom_rpm_data ipq806x_template = {
 	.ack_sel_size = 7,
 };
 
+static const struct qcom_rpm_resource mdm9615_rpm_resource_table[] = {
+	[QCOM_RPM_CXO_CLK] =			{ 25, 9, 5, 1 },
+	[QCOM_RPM_SYS_FABRIC_CLK] =		{ 26, 10, 9, 1 },
+	[QCOM_RPM_DAYTONA_FABRIC_CLK] =		{ 27, 11, 11, 1 },
+	[QCOM_RPM_SFPB_CLK] =			{ 28, 12, 12, 1 },
+	[QCOM_RPM_CFPB_CLK] =			{ 29, 13, 13, 1 },
+	[QCOM_RPM_EBI1_CLK] =			{ 30, 14, 16, 1 },
+	[QCOM_RPM_APPS_FABRIC_HALT] =		{ 31, 15, 22, 2 },
+	[QCOM_RPM_APPS_FABRIC_MODE] =		{ 33, 16, 23, 3 },
+	[QCOM_RPM_APPS_FABRIC_IOCTL] =		{ 36, 17, 24, 1 },
+	[QCOM_RPM_APPS_FABRIC_ARB] =		{ 37, 18, 25, 27 },
+	[QCOM_RPM_PM8018_SMPS1] =		{ 64, 19, 30, 2 },
+	[QCOM_RPM_PM8018_SMPS2] =		{ 66, 21, 31, 2 },
+	[QCOM_RPM_PM8018_SMPS3] =		{ 68, 23, 32, 2 },
+	[QCOM_RPM_PM8018_SMPS4] =		{ 70, 25, 33, 2 },
+	[QCOM_RPM_PM8018_SMPS5] =		{ 72, 27, 34, 2 },
+	[QCOM_RPM_PM8018_LDO1] =		{ 74, 29, 35, 2 },
+	[QCOM_RPM_PM8018_LDO2] =		{ 76, 31, 36, 2 },
+	[QCOM_RPM_PM8018_LDO3] =		{ 78, 33, 37, 2 },
+	[QCOM_RPM_PM8018_LDO4] =		{ 80, 35, 38, 2 },
+	[QCOM_RPM_PM8018_LDO5] =		{ 82, 37, 39, 2 },
+	[QCOM_RPM_PM8018_LDO6] =		{ 84, 39, 40, 2 },
+	[QCOM_RPM_PM8018_LDO7] =		{ 86, 41, 41, 2 },
+	[QCOM_RPM_PM8018_LDO8] =		{ 88, 43, 42, 2 },
+	[QCOM_RPM_PM8018_LDO9] =		{ 90, 45, 43, 2 },
+	[QCOM_RPM_PM8018_LDO10] =		{ 92, 47, 44, 2 },
+	[QCOM_RPM_PM8018_LDO11] =		{ 94, 49, 45, 2 },
+	[QCOM_RPM_PM8018_LDO12] =		{ 96, 51, 46, 2 },
+	[QCOM_RPM_PM8018_LDO13] =		{ 98, 53, 47, 2 },
+	[QCOM_RPM_PM8018_LDO14] =		{ 100, 55, 48, 2 },
+	[QCOM_RPM_PM8018_LVS1] =		{ 102, 57, 49, 1 },
+	[QCOM_RPM_PM8018_NCP] =			{ 103, 58, 80, 2 },
+	[QCOM_RPM_CXO_BUFFERS] =		{ 105, 60, 81, 1 },
+	[QCOM_RPM_USB_OTG_SWITCH] =		{ 106, 61, 82, 1 },
+	[QCOM_RPM_HDMI_SWITCH] =		{ 107, 62, 83, 1 },
+	[QCOM_RPM_VOLTAGE_CORNER] =		{ 109, 64, 87, 1 },
+};
+
+static const struct qcom_rpm_data mdm9615_template = {
+	.version = 3,
+	.resource_table = mdm9615_rpm_resource_table,
+	.n_resources = ARRAY_SIZE(mdm9615_rpm_resource_table),
+	.req_ctx_off = 3,
+	.req_sel_off = 11,
+	.ack_ctx_off = 15,
+	.ack_sel_off = 23,
+	.req_sel_size = 4,
+	.ack_sel_size = 7,
+};
+
 static const struct of_device_id qcom_rpm_of_match[] = {
 	{ .compatible = "qcom,rpm-apq8064", .data = &apq8064_template },
 	{ .compatible = "qcom,rpm-msm8660", .data = &msm8660_template },
 	{ .compatible = "qcom,rpm-msm8960", .data = &msm8960_template },
 	{ .compatible = "qcom,rpm-ipq8064", .data = &ipq806x_template },
+	{ .compatible = "qcom,rpm-mdm9615", .data = &mdm9615_template },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, qcom_rpm_of_match);
@@ -501,6 +554,20 @@ static int qcom_rpm_probe(struct platform_device *pdev)
 	mutex_init(&rpm->lock);
 	init_completion(&rpm->ack);
 
+	/* Enable message RAM clock */
+	rpm->ramclk = devm_clk_get(&pdev->dev, "ram");
+	if (IS_ERR(rpm->ramclk)) {
+		ret = PTR_ERR(rpm->ramclk);
+		if (ret == -EPROBE_DEFER)
+			return ret;
+		/*
+		 * Fall through in all other cases, as the clock is
+		 * optional. (Does not exist on all platforms.)
+		 */
+		rpm->ramclk = NULL;
+	}
+	clk_prepare_enable(rpm->ramclk); /* Accepts NULL */
+
 	irq_ack = platform_get_irq_byname(pdev, "ack");
 	if (irq_ack < 0) {
 		dev_err(&pdev->dev, "required ack interrupt missing\n");
@@ -538,6 +605,7 @@ static int qcom_rpm_probe(struct platform_device *pdev)
 	}
 
 	rpm->ipc_regmap = syscon_node_to_regmap(syscon_np);
+	of_node_put(syscon_np);
 	if (IS_ERR(rpm->ipc_regmap))
 		return PTR_ERR(rpm->ipc_regmap);
 
@@ -620,7 +688,11 @@ static int qcom_rpm_probe(struct platform_device *pdev)
 
 static int qcom_rpm_remove(struct platform_device *pdev)
 {
+	struct qcom_rpm *rpm = dev_get_drvdata(&pdev->dev);
+
 	of_platform_depopulate(&pdev->dev);
+	clk_disable_unprepare(rpm->ramclk);
+
 	return 0;
 }
 
diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c
index 49d7f624fc94..0f8acc5882a4 100644
--- a/drivers/mfd/rk808.c
+++ b/drivers/mfd/rk808.c
@@ -1,11 +1,15 @@
 /*
- * MFD core driver for Rockchip RK808
+ * MFD core driver for Rockchip RK808/RK818
  *
  * Copyright (c) 2014, Fuzhou Rockchip Electronics Co., Ltd
  *
  * Author: Chris Zhong <zyw@rock-chips.com>
  * Author: Zhang Qing <zhangqing@rock-chips.com>
  *
+ * Copyright (C) 2016 PHYTEC Messtechnik GmbH
+ *
+ * Author: Wadim Egorov <w.egorov@phytec.de>
+ *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
@@ -21,6 +25,7 @@
 #include <linux/mfd/rk808.h>
 #include <linux/mfd/core.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/regmap.h>
 
 struct rk808_reg_data {
@@ -57,6 +62,14 @@ static bool rk808_is_volatile_reg(struct device *dev, unsigned int reg)
 	return false;
 }
 
+static const struct regmap_config rk818_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = RK818_USB_CTRL_REG,
+	.cache_type = REGCACHE_RBTREE,
+	.volatile_reg = rk808_is_volatile_reg,
+};
+
 static const struct regmap_config rk808_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
@@ -79,11 +92,21 @@ static const struct mfd_cell rk808s[] = {
 	{
 		.name = "rk808-rtc",
 		.num_resources = ARRAY_SIZE(rtc_resources),
-		.resources = &rtc_resources[0],
+		.resources = rtc_resources,
 	},
 };
 
-static const struct rk808_reg_data pre_init_reg[] = {
+static const struct mfd_cell rk818s[] = {
+	{ .name = "rk808-clkout", },
+	{ .name = "rk808-regulator", },
+	{
+		.name = "rk808-rtc",
+		.num_resources = ARRAY_SIZE(rtc_resources),
+		.resources = rtc_resources,
+	},
+};
+
+static const struct rk808_reg_data rk808_pre_init_reg[] = {
 	{ RK808_BUCK3_CONFIG_REG, BUCK_ILMIN_MASK,  BUCK_ILMIN_150MA },
 	{ RK808_BUCK4_CONFIG_REG, BUCK_ILMIN_MASK,  BUCK_ILMIN_200MA },
 	{ RK808_BOOST_CONFIG_REG, BOOST_ILMIN_MASK, BOOST_ILMIN_100MA },
@@ -94,6 +117,24 @@ static const struct rk808_reg_data pre_init_reg[] = {
 						    VB_LO_SEL_3500MV },
 };
 
+static const struct rk808_reg_data rk818_pre_init_reg[] = {
+	/* improve efficiency */
+	{ RK818_BUCK2_CONFIG_REG, BUCK2_RATE_MASK,  BUCK_ILMIN_250MA },
+	{ RK818_BUCK4_CONFIG_REG, BUCK_ILMIN_MASK,  BUCK_ILMIN_250MA },
+	{ RK818_BOOST_CONFIG_REG, BOOST_ILMIN_MASK, BOOST_ILMIN_100MA },
+	{ RK818_USB_CTRL_REG,	  RK818_USB_ILIM_SEL_MASK,
+						    RK818_USB_ILMIN_2000MA },
+	/* close charger when usb lower then 3.4V */
+	{ RK818_USB_CTRL_REG,	  RK818_USB_CHG_SD_VSEL_MASK,
+						    (0x7 << 4) },
+	/* no action when vref */
+	{ RK818_H5V_EN_REG,	  BIT(1),	    RK818_REF_RDY_CTRL },
+	/* enable HDMI 5V */
+	{ RK818_H5V_EN_REG,	  BIT(0),	    RK818_H5V_EN },
+	{ RK808_VB_MON_REG,	  MASK_ALL,	    VB_LO_ACT |
+						    VB_LO_SEL_3500MV },
+};
+
 static const struct regmap_irq rk808_irqs[] = {
 	/* INT_STS */
 	[RK808_IRQ_VOUT_LO] = {
@@ -136,6 +177,76 @@ static const struct regmap_irq rk808_irqs[] = {
 	},
 };
 
+static const struct regmap_irq rk818_irqs[] = {
+	/* INT_STS */
+	[RK818_IRQ_VOUT_LO] = {
+		.mask = RK818_IRQ_VOUT_LO_MSK,
+		.reg_offset = 0,
+	},
+	[RK818_IRQ_VB_LO] = {
+		.mask = RK818_IRQ_VB_LO_MSK,
+		.reg_offset = 0,
+	},
+	[RK818_IRQ_PWRON] = {
+		.mask = RK818_IRQ_PWRON_MSK,
+		.reg_offset = 0,
+	},
+	[RK818_IRQ_PWRON_LP] = {
+		.mask = RK818_IRQ_PWRON_LP_MSK,
+		.reg_offset = 0,
+	},
+	[RK818_IRQ_HOTDIE] = {
+		.mask = RK818_IRQ_HOTDIE_MSK,
+		.reg_offset = 0,
+	},
+	[RK818_IRQ_RTC_ALARM] = {
+		.mask = RK818_IRQ_RTC_ALARM_MSK,
+		.reg_offset = 0,
+	},
+	[RK818_IRQ_RTC_PERIOD] = {
+		.mask = RK818_IRQ_RTC_PERIOD_MSK,
+		.reg_offset = 0,
+	},
+	[RK818_IRQ_USB_OV] = {
+		.mask = RK818_IRQ_USB_OV_MSK,
+		.reg_offset = 0,
+	},
+
+	/* INT_STS2 */
+	[RK818_IRQ_PLUG_IN] = {
+		.mask = RK818_IRQ_PLUG_IN_MSK,
+		.reg_offset = 1,
+	},
+	[RK818_IRQ_PLUG_OUT] = {
+		.mask = RK818_IRQ_PLUG_OUT_MSK,
+		.reg_offset = 1,
+	},
+	[RK818_IRQ_CHG_OK] = {
+		.mask = RK818_IRQ_CHG_OK_MSK,
+		.reg_offset = 1,
+	},
+	[RK818_IRQ_CHG_TE] = {
+		.mask = RK818_IRQ_CHG_TE_MSK,
+		.reg_offset = 1,
+	},
+	[RK818_IRQ_CHG_TS1] = {
+		.mask = RK818_IRQ_CHG_TS1_MSK,
+		.reg_offset = 1,
+	},
+	[RK818_IRQ_TS2] = {
+		.mask = RK818_IRQ_TS2_MSK,
+		.reg_offset = 1,
+	},
+	[RK818_IRQ_CHG_CVTLIM] = {
+		.mask = RK818_IRQ_CHG_CVTLIM_MSK,
+		.reg_offset = 1,
+	},
+	[RK818_IRQ_DISCHG_ILIM] = {
+		.mask = RK818_IRQ_DISCHG_ILIM_MSK,
+		.reg_offset = 1,
+	},
+};
+
 static struct regmap_irq_chip rk808_irq_chip = {
 	.name = "rk808",
 	.irqs = rk808_irqs,
@@ -148,6 +259,18 @@ static struct regmap_irq_chip rk808_irq_chip = {
 	.init_ack_masked = true,
 };
 
+static struct regmap_irq_chip rk818_irq_chip = {
+	.name = "rk818",
+	.irqs = rk818_irqs,
+	.num_irqs = ARRAY_SIZE(rk818_irqs),
+	.num_regs = 2,
+	.irq_reg_stride = 2,
+	.status_base = RK818_INT_STS_REG1,
+	.mask_base = RK818_INT_STS_MSK_REG1,
+	.ack_base = RK818_INT_STS_REG1,
+	.init_ack_masked = true,
+};
+
 static struct i2c_client *rk808_i2c_client;
 static void rk808_device_shutdown(void)
 {
@@ -167,55 +290,100 @@ static void rk808_device_shutdown(void)
 		dev_err(&rk808_i2c_client->dev, "power off error!\n");
 }
 
+static const struct of_device_id rk808_of_match[] = {
+	{ .compatible = "rockchip,rk808" },
+	{ .compatible = "rockchip,rk818" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, rk808_of_match);
+
 static int rk808_probe(struct i2c_client *client,
 		       const struct i2c_device_id *id)
 {
 	struct device_node *np = client->dev.of_node;
 	struct rk808 *rk808;
+	const struct rk808_reg_data *pre_init_reg;
+	const struct mfd_cell *cells;
+	int nr_pre_init_regs;
+	int nr_cells;
 	int pm_off = 0;
 	int ret;
 	int i;
 
-	if (!client->irq) {
-		dev_err(&client->dev, "No interrupt support, no core IRQ\n");
-		return -EINVAL;
-	}
-
 	rk808 = devm_kzalloc(&client->dev, sizeof(*rk808), GFP_KERNEL);
 	if (!rk808)
 		return -ENOMEM;
 
-	rk808->regmap = devm_regmap_init_i2c(client, &rk808_regmap_config);
+	rk808->variant = i2c_smbus_read_word_data(client, RK808_ID_MSB);
+	if (rk808->variant < 0) {
+		dev_err(&client->dev, "Failed to read the chip id at 0x%02x\n",
+			RK808_ID_MSB);
+		return rk808->variant;
+	}
+
+	dev_dbg(&client->dev, "Chip id: 0x%x\n", (unsigned int)rk808->variant);
+
+	switch (rk808->variant) {
+	case RK808_ID:
+		rk808->regmap_cfg = &rk808_regmap_config;
+		rk808->regmap_irq_chip = &rk808_irq_chip;
+		pre_init_reg = rk808_pre_init_reg;
+		nr_pre_init_regs = ARRAY_SIZE(rk808_pre_init_reg);
+		cells = rk808s;
+		nr_cells = ARRAY_SIZE(rk808s);
+		break;
+	case RK818_ID:
+		rk808->regmap_cfg = &rk818_regmap_config;
+		rk808->regmap_irq_chip = &rk818_irq_chip;
+		pre_init_reg = rk818_pre_init_reg;
+		nr_pre_init_regs = ARRAY_SIZE(rk818_pre_init_reg);
+		cells = rk818s;
+		nr_cells = ARRAY_SIZE(rk818s);
+		break;
+	default:
+		dev_err(&client->dev, "Unsupported RK8XX ID %lu\n",
+			rk808->variant);
+		return -EINVAL;
+	}
+
+	rk808->i2c = client;
+	i2c_set_clientdata(client, rk808);
+
+	rk808->regmap = devm_regmap_init_i2c(client, rk808->regmap_cfg);
 	if (IS_ERR(rk808->regmap)) {
 		dev_err(&client->dev, "regmap initialization failed\n");
 		return PTR_ERR(rk808->regmap);
 	}
 
-	for (i = 0; i < ARRAY_SIZE(pre_init_reg); i++) {
-		ret = regmap_update_bits(rk808->regmap, pre_init_reg[i].addr,
-					 pre_init_reg[i].mask,
-					 pre_init_reg[i].value);
-		if (ret) {
-			dev_err(&client->dev,
-				"0x%x write err\n", pre_init_reg[i].addr);
-			return ret;
-		}
+	if (!client->irq) {
+		dev_err(&client->dev, "No interrupt support, no core IRQ\n");
+		return -EINVAL;
 	}
 
 	ret = regmap_add_irq_chip(rk808->regmap, client->irq,
 				  IRQF_ONESHOT, -1,
-				  &rk808_irq_chip, &rk808->irq_data);
+				  rk808->regmap_irq_chip, &rk808->irq_data);
 	if (ret) {
 		dev_err(&client->dev, "Failed to add irq_chip %d\n", ret);
 		return ret;
 	}
 
-	rk808->i2c = client;
-	i2c_set_clientdata(client, rk808);
+	for (i = 0; i < nr_pre_init_regs; i++) {
+		ret = regmap_update_bits(rk808->regmap,
+					pre_init_reg[i].addr,
+					pre_init_reg[i].mask,
+					pre_init_reg[i].value);
+		if (ret) {
+			dev_err(&client->dev,
+				"0x%x write err\n",
+				pre_init_reg[i].addr);
+			return ret;
+		}
+	}
 
-	ret = devm_mfd_add_devices(&client->dev, -1,
-				   rk808s, ARRAY_SIZE(rk808s), NULL, 0,
-				   regmap_irq_get_domain(rk808->irq_data));
+	ret = devm_mfd_add_devices(&client->dev, PLATFORM_DEVID_NONE,
+			      cells, nr_cells, NULL, 0,
+			      regmap_irq_get_domain(rk808->irq_data));
 	if (ret) {
 		dev_err(&client->dev, "failed to add MFD devices %d\n", ret);
 		goto err_irq;
@@ -245,14 +413,9 @@ static int rk808_remove(struct i2c_client *client)
 	return 0;
 }
 
-static const struct of_device_id rk808_of_match[] = {
-	{ .compatible = "rockchip,rk808" },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, rk808_of_match);
-
 static const struct i2c_device_id rk808_ids[] = {
 	{ "rk808" },
+	{ "rk818" },
 	{ },
 };
 MODULE_DEVICE_TABLE(i2c, rk808_ids);
@@ -272,4 +435,5 @@ module_i2c_driver(rk808_i2c_driver);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Chris Zhong <zyw@rock-chips.com>");
 MODULE_AUTHOR("Zhang Qing <zhangqing@rock-chips.com>");
-MODULE_DESCRIPTION("RK808 PMIC driver");
+MODULE_AUTHOR("Wadim Egorov <w.egorov@phytec.de>");
+MODULE_DESCRIPTION("RK808/RK818 PMIC driver");
diff --git a/drivers/mfd/rtsx_usb.c b/drivers/mfd/rtsx_usb.c
index dbd907d7170e..691dab791f7a 100644
--- a/drivers/mfd/rtsx_usb.c
+++ b/drivers/mfd/rtsx_usb.c
@@ -46,9 +46,6 @@ static void rtsx_usb_sg_timed_out(unsigned long data)
 
 	dev_dbg(&ucr->pusb_intf->dev, "%s: sg transfer timed out", __func__);
 	usb_sg_cancel(&ucr->current_sg);
-
-	/* we know the cancellation is caused by time-out */
-	ucr->current_sg.status = -ETIMEDOUT;
 }
 
 static int rtsx_usb_bulk_transfer_sglist(struct rtsx_ucr *ucr,
@@ -67,12 +64,15 @@ static int rtsx_usb_bulk_transfer_sglist(struct rtsx_ucr *ucr,
 	ucr->sg_timer.expires = jiffies + msecs_to_jiffies(timeout);
 	add_timer(&ucr->sg_timer);
 	usb_sg_wait(&ucr->current_sg);
-	del_timer_sync(&ucr->sg_timer);
+	if (!del_timer_sync(&ucr->sg_timer))
+		ret = -ETIMEDOUT;
+	else
+		ret = ucr->current_sg.status;
 
 	if (act_len)
 		*act_len = ucr->current_sg.bytes;
 
-	return ucr->current_sg.status;
+	return ret;
 }
 
 int rtsx_usb_transfer_data(struct rtsx_ucr *ucr, unsigned int pipe,
diff --git a/drivers/mfd/sm501.c b/drivers/mfd/sm501.c
index 65cd0d2a822a..40534352e574 100644
--- a/drivers/mfd/sm501.c
+++ b/drivers/mfd/sm501.c
@@ -1001,7 +1001,7 @@ static int sm501_gpio_output(struct gpio_chip *chip,
 	return 0;
 }
 
-static struct gpio_chip gpio_chip_template = {
+static const struct gpio_chip gpio_chip_template = {
 	.ngpio			= 32,
 	.direction_input	= sm501_gpio_input,
 	.direction_output	= sm501_gpio_output,
diff --git a/drivers/mfd/smsc-ece1099.c b/drivers/mfd/smsc-ece1099.c
index cd18c09827ef..1f40baf1234e 100644
--- a/drivers/mfd/smsc-ece1099.c
+++ b/drivers/mfd/smsc-ece1099.c
@@ -11,8 +11,7 @@
  *
  */
 
-#include <linux/module.h>
-#include <linux/moduleparam.h>
+#include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/i2c.h>
 #include <linux/gpio.h>
@@ -81,7 +80,6 @@ static const struct i2c_device_id smsc_i2c_id[] = {
 	{ "smscece1099", 0},
 	{},
 };
-MODULE_DEVICE_TABLE(i2c, smsc_i2c_id);
 
 static struct i2c_driver smsc_i2c_driver = {
 	.driver = {
@@ -90,9 +88,4 @@ static struct i2c_driver smsc_i2c_driver = {
 	.probe = smsc_i2c_probe,
 	.id_table = smsc_i2c_id,
 };
-
-module_i2c_driver(smsc_i2c_driver);
-
-MODULE_AUTHOR("Sourav Poddar <sourav.poddar@ti.com>");
-MODULE_DESCRIPTION("SMSC chip multi-function driver");
-MODULE_LICENSE("GPL v2");
+builtin_i2c_driver(smsc_i2c_driver);
diff --git a/drivers/mfd/stmpe-i2c.c b/drivers/mfd/stmpe-i2c.c
index c3f4aab53b07..863c39a3353c 100644
--- a/drivers/mfd/stmpe-i2c.c
+++ b/drivers/mfd/stmpe-i2c.c
@@ -57,6 +57,7 @@ static const struct of_device_id stmpe_of_match[] = {
 	{ .compatible = "st,stmpe610", .data = (void *)STMPE610, },
 	{ .compatible = "st,stmpe801", .data = (void *)STMPE801, },
 	{ .compatible = "st,stmpe811", .data = (void *)STMPE811, },
+	{ .compatible = "st,stmpe1600", .data = (void *)STMPE1600, },
 	{ .compatible = "st,stmpe1601", .data = (void *)STMPE1601, },
 	{ .compatible = "st,stmpe1801", .data = (void *)STMPE1801, },
 	{ .compatible = "st,stmpe2401", .data = (void *)STMPE2401, },
@@ -101,6 +102,7 @@ static const struct i2c_device_id stmpe_i2c_id[] = {
 	{ "stmpe610", STMPE610 },
 	{ "stmpe801", STMPE801 },
 	{ "stmpe811", STMPE811 },
+	{ "stmpe1600", STMPE1600 },
 	{ "stmpe1601", STMPE1601 },
 	{ "stmpe1801", STMPE1801 },
 	{ "stmpe2401", STMPE2401 },
diff --git a/drivers/mfd/stmpe.c b/drivers/mfd/stmpe.c
index 94c7cc02fdab..cfdae8a3d779 100644
--- a/drivers/mfd/stmpe.c
+++ b/drivers/mfd/stmpe.c
@@ -469,6 +469,8 @@ static const struct mfd_cell stmpe_ts_cell = {
 
 static const u8 stmpe811_regs[] = {
 	[STMPE_IDX_CHIP_ID]	= STMPE811_REG_CHIP_ID,
+	[STMPE_IDX_SYS_CTRL]	= STMPE811_REG_SYS_CTRL,
+	[STMPE_IDX_SYS_CTRL2]	= STMPE811_REG_SYS_CTRL2,
 	[STMPE_IDX_ICR_LSB]	= STMPE811_REG_INT_CTRL,
 	[STMPE_IDX_IER_LSB]	= STMPE811_REG_INT_EN,
 	[STMPE_IDX_ISR_MSB]	= STMPE811_REG_INT_STA,
@@ -481,7 +483,7 @@ static const u8 stmpe811_regs[] = {
 	[STMPE_IDX_GPAFR_U_MSB]	= STMPE811_REG_GPIO_AF,
 	[STMPE_IDX_IEGPIOR_LSB]	= STMPE811_REG_GPIO_INT_EN,
 	[STMPE_IDX_ISGPIOR_MSB]	= STMPE811_REG_GPIO_INT_STA,
-	[STMPE_IDX_GPEDR_MSB]	= STMPE811_REG_GPIO_ED,
+	[STMPE_IDX_GPEDR_LSB]	= STMPE811_REG_GPIO_ED,
 };
 
 static struct stmpe_variant_block stmpe811_blocks[] = {
@@ -511,7 +513,7 @@ static int stmpe811_enable(struct stmpe *stmpe, unsigned int blocks,
 	if (blocks & STMPE_BLOCK_TOUCHSCREEN)
 		mask |= STMPE811_SYS_CTRL2_TSC_OFF;
 
-	return __stmpe_set_bits(stmpe, STMPE811_REG_SYS_CTRL2, mask,
+	return __stmpe_set_bits(stmpe, stmpe->regs[STMPE_IDX_SYS_CTRL2], mask,
 				enable ? 0 : mask);
 }
 
@@ -551,25 +553,89 @@ static struct stmpe_variant_info stmpe610 = {
 };
 
 /*
+ * STMPE1600
+ * Compared to all others STMPE variant, LSB and MSB regs are located in this
+ * order :	LSB   addr
+ *		MSB   addr + 1
+ * As there is only 2 * 8bits registers for GPMR/GPSR/IEGPIOPR, CSB index is MSB registers
+ */
+
+static const u8 stmpe1600_regs[] = {
+	[STMPE_IDX_CHIP_ID]	= STMPE1600_REG_CHIP_ID,
+	[STMPE_IDX_SYS_CTRL]	= STMPE1600_REG_SYS_CTRL,
+	[STMPE_IDX_ICR_LSB]	= STMPE1600_REG_SYS_CTRL,
+	[STMPE_IDX_GPMR_LSB]	= STMPE1600_REG_GPMR_LSB,
+	[STMPE_IDX_GPMR_CSB]	= STMPE1600_REG_GPMR_MSB,
+	[STMPE_IDX_GPSR_LSB]	= STMPE1600_REG_GPSR_LSB,
+	[STMPE_IDX_GPSR_CSB]	= STMPE1600_REG_GPSR_MSB,
+	[STMPE_IDX_GPDR_LSB]	= STMPE1600_REG_GPDR_LSB,
+	[STMPE_IDX_GPDR_CSB]	= STMPE1600_REG_GPDR_MSB,
+	[STMPE_IDX_IEGPIOR_LSB]	= STMPE1600_REG_IEGPIOR_LSB,
+	[STMPE_IDX_IEGPIOR_CSB]	= STMPE1600_REG_IEGPIOR_MSB,
+	[STMPE_IDX_ISGPIOR_LSB]	= STMPE1600_REG_ISGPIOR_LSB,
+};
+
+static struct stmpe_variant_block stmpe1600_blocks[] = {
+	{
+		.cell	= &stmpe_gpio_cell,
+		.irq	= 0,
+		.block	= STMPE_BLOCK_GPIO,
+	},
+};
+
+static int stmpe1600_enable(struct stmpe *stmpe, unsigned int blocks,
+			   bool enable)
+{
+	if (blocks & STMPE_BLOCK_GPIO)
+		return 0;
+	else
+		return -EINVAL;
+}
+
+static struct stmpe_variant_info stmpe1600 = {
+	.name		= "stmpe1600",
+	.id_val		= STMPE1600_ID,
+	.id_mask	= 0xffff,
+	.num_gpios	= 16,
+	.af_bits	= 0,
+	.regs		= stmpe1600_regs,
+	.blocks		= stmpe1600_blocks,
+	.num_blocks	= ARRAY_SIZE(stmpe1600_blocks),
+	.num_irqs	= STMPE1600_NR_INTERNAL_IRQS,
+	.enable		= stmpe1600_enable,
+};
+
+/*
  * STMPE1601
  */
 
 static const u8 stmpe1601_regs[] = {
 	[STMPE_IDX_CHIP_ID]	= STMPE1601_REG_CHIP_ID,
+	[STMPE_IDX_SYS_CTRL]	= STMPE1601_REG_SYS_CTRL,
+	[STMPE_IDX_SYS_CTRL2]	= STMPE1601_REG_SYS_CTRL2,
 	[STMPE_IDX_ICR_LSB]	= STMPE1601_REG_ICR_LSB,
+	[STMPE_IDX_IER_MSB]	= STMPE1601_REG_IER_MSB,
 	[STMPE_IDX_IER_LSB]	= STMPE1601_REG_IER_LSB,
 	[STMPE_IDX_ISR_MSB]	= STMPE1601_REG_ISR_MSB,
 	[STMPE_IDX_GPMR_LSB]	= STMPE1601_REG_GPIO_MP_LSB,
+	[STMPE_IDX_GPMR_CSB]	= STMPE1601_REG_GPIO_MP_MSB,
 	[STMPE_IDX_GPSR_LSB]	= STMPE1601_REG_GPIO_SET_LSB,
+	[STMPE_IDX_GPSR_CSB]	= STMPE1601_REG_GPIO_SET_MSB,
 	[STMPE_IDX_GPCR_LSB]	= STMPE1601_REG_GPIO_CLR_LSB,
+	[STMPE_IDX_GPCR_CSB]	= STMPE1601_REG_GPIO_CLR_MSB,
 	[STMPE_IDX_GPDR_LSB]	= STMPE1601_REG_GPIO_SET_DIR_LSB,
+	[STMPE_IDX_GPDR_CSB]	= STMPE1601_REG_GPIO_SET_DIR_MSB,
+	[STMPE_IDX_GPEDR_LSB]	= STMPE1601_REG_GPIO_ED_LSB,
+	[STMPE_IDX_GPEDR_CSB]	= STMPE1601_REG_GPIO_ED_MSB,
 	[STMPE_IDX_GPRER_LSB]	= STMPE1601_REG_GPIO_RE_LSB,
+	[STMPE_IDX_GPRER_CSB]	= STMPE1601_REG_GPIO_RE_MSB,
 	[STMPE_IDX_GPFER_LSB]	= STMPE1601_REG_GPIO_FE_LSB,
+	[STMPE_IDX_GPFER_CSB]	= STMPE1601_REG_GPIO_FE_MSB,
 	[STMPE_IDX_GPPUR_LSB]	= STMPE1601_REG_GPIO_PU_LSB,
 	[STMPE_IDX_GPAFR_U_MSB]	= STMPE1601_REG_GPIO_AF_U_MSB,
 	[STMPE_IDX_IEGPIOR_LSB]	= STMPE1601_REG_INT_EN_GPIO_MASK_LSB,
+	[STMPE_IDX_IEGPIOR_CSB]	= STMPE1601_REG_INT_EN_GPIO_MASK_MSB,
 	[STMPE_IDX_ISGPIOR_MSB]	= STMPE1601_REG_INT_STA_GPIO_MSB,
-	[STMPE_IDX_GPEDR_MSB]	= STMPE1601_REG_GPIO_ED_MSB,
 };
 
 static struct stmpe_variant_block stmpe1601_blocks[] = {
@@ -640,13 +706,13 @@ static int stmpe1601_autosleep(struct stmpe *stmpe,
 		return timeout;
 	}
 
-	ret = __stmpe_set_bits(stmpe, STMPE1601_REG_SYS_CTRL2,
+	ret = __stmpe_set_bits(stmpe, stmpe->regs[STMPE_IDX_SYS_CTRL2],
 			STMPE1601_AUTOSLEEP_TIMEOUT_MASK,
 			timeout);
 	if (ret < 0)
 		return ret;
 
-	return __stmpe_set_bits(stmpe, STMPE1601_REG_SYS_CTRL2,
+	return __stmpe_set_bits(stmpe, stmpe->regs[STMPE_IDX_SYS_CTRL2],
 			STPME1601_AUTOSLEEP_ENABLE,
 			STPME1601_AUTOSLEEP_ENABLE);
 }
@@ -671,7 +737,7 @@ static int stmpe1601_enable(struct stmpe *stmpe, unsigned int blocks,
 	else
 		mask &= ~STMPE1601_SYS_CTRL_ENABLE_SPWM;
 
-	return __stmpe_set_bits(stmpe, STMPE1601_REG_SYS_CTRL, mask,
+	return __stmpe_set_bits(stmpe, stmpe->regs[STMPE_IDX_SYS_CTRL], mask,
 				enable ? mask : 0);
 }
 
@@ -710,18 +776,33 @@ static struct stmpe_variant_info stmpe1601 = {
  */
 static const u8 stmpe1801_regs[] = {
 	[STMPE_IDX_CHIP_ID]	= STMPE1801_REG_CHIP_ID,
+	[STMPE_IDX_SYS_CTRL]	= STMPE1801_REG_SYS_CTRL,
 	[STMPE_IDX_ICR_LSB]	= STMPE1801_REG_INT_CTRL_LOW,
 	[STMPE_IDX_IER_LSB]	= STMPE1801_REG_INT_EN_MASK_LOW,
 	[STMPE_IDX_ISR_LSB]	= STMPE1801_REG_INT_STA_LOW,
 	[STMPE_IDX_GPMR_LSB]	= STMPE1801_REG_GPIO_MP_LOW,
+	[STMPE_IDX_GPMR_CSB]	= STMPE1801_REG_GPIO_MP_MID,
+	[STMPE_IDX_GPMR_MSB]	= STMPE1801_REG_GPIO_MP_HIGH,
 	[STMPE_IDX_GPSR_LSB]	= STMPE1801_REG_GPIO_SET_LOW,
+	[STMPE_IDX_GPSR_CSB]	= STMPE1801_REG_GPIO_SET_MID,
+	[STMPE_IDX_GPSR_MSB]	= STMPE1801_REG_GPIO_SET_HIGH,
 	[STMPE_IDX_GPCR_LSB]	= STMPE1801_REG_GPIO_CLR_LOW,
+	[STMPE_IDX_GPCR_CSB]	= STMPE1801_REG_GPIO_CLR_MID,
+	[STMPE_IDX_GPCR_MSB]	= STMPE1801_REG_GPIO_CLR_HIGH,
 	[STMPE_IDX_GPDR_LSB]	= STMPE1801_REG_GPIO_SET_DIR_LOW,
+	[STMPE_IDX_GPDR_CSB]	= STMPE1801_REG_GPIO_SET_DIR_MID,
+	[STMPE_IDX_GPDR_MSB]	= STMPE1801_REG_GPIO_SET_DIR_HIGH,
 	[STMPE_IDX_GPRER_LSB]	= STMPE1801_REG_GPIO_RE_LOW,
+	[STMPE_IDX_GPRER_CSB]	= STMPE1801_REG_GPIO_RE_MID,
+	[STMPE_IDX_GPRER_MSB]	= STMPE1801_REG_GPIO_RE_HIGH,
 	[STMPE_IDX_GPFER_LSB]	= STMPE1801_REG_GPIO_FE_LOW,
+	[STMPE_IDX_GPFER_CSB]	= STMPE1801_REG_GPIO_FE_MID,
+	[STMPE_IDX_GPFER_MSB]	= STMPE1801_REG_GPIO_FE_HIGH,
 	[STMPE_IDX_GPPUR_LSB]	= STMPE1801_REG_GPIO_PULL_UP_LOW,
 	[STMPE_IDX_IEGPIOR_LSB]	= STMPE1801_REG_INT_EN_GPIO_MASK_LOW,
-	[STMPE_IDX_ISGPIOR_LSB]	= STMPE1801_REG_INT_STA_GPIO_LOW,
+	[STMPE_IDX_IEGPIOR_CSB]	= STMPE1801_REG_INT_EN_GPIO_MASK_MID,
+	[STMPE_IDX_IEGPIOR_MSB]	= STMPE1801_REG_INT_EN_GPIO_MASK_HIGH,
+	[STMPE_IDX_ISGPIOR_MSB]	= STMPE1801_REG_INT_STA_GPIO_HIGH,
 };
 
 static struct stmpe_variant_block stmpe1801_blocks[] = {
@@ -751,22 +832,31 @@ static int stmpe1801_enable(struct stmpe *stmpe, unsigned int blocks,
 				enable ? mask : 0);
 }
 
-static int stmpe1801_reset(struct stmpe *stmpe)
+static int stmpe_reset(struct stmpe *stmpe)
 {
+	u16 id_val = stmpe->variant->id_val;
 	unsigned long timeout;
 	int ret = 0;
+	u8 reset_bit;
+
+	if (id_val == STMPE811_ID)
+		/* STMPE801 and STMPE610 use bit 1 of SYS_CTRL register */
+		reset_bit = STMPE811_SYS_CTRL_RESET;
+	else
+		/* all other STMPE variant use bit 7 of SYS_CTRL register */
+		reset_bit = STMPE_SYS_CTRL_RESET;
 
-	ret = __stmpe_set_bits(stmpe, STMPE1801_REG_SYS_CTRL,
-		STMPE1801_MSK_SYS_CTRL_RESET, STMPE1801_MSK_SYS_CTRL_RESET);
+	ret = __stmpe_set_bits(stmpe, stmpe->regs[STMPE_IDX_SYS_CTRL],
+			       reset_bit, reset_bit);
 	if (ret < 0)
 		return ret;
 
 	timeout = jiffies + msecs_to_jiffies(100);
 	while (time_before(jiffies, timeout)) {
-		ret = __stmpe_reg_read(stmpe, STMPE1801_REG_SYS_CTRL);
+		ret = __stmpe_reg_read(stmpe, stmpe->regs[STMPE_IDX_SYS_CTRL]);
 		if (ret < 0)
 			return ret;
-		if (!(ret & STMPE1801_MSK_SYS_CTRL_RESET))
+		if (!(ret & reset_bit))
 			return 0;
 		usleep_range(100, 200);
 	}
@@ -794,20 +884,39 @@ static struct stmpe_variant_info stmpe1801 = {
 
 static const u8 stmpe24xx_regs[] = {
 	[STMPE_IDX_CHIP_ID]	= STMPE24XX_REG_CHIP_ID,
+	[STMPE_IDX_SYS_CTRL]	= STMPE24XX_REG_SYS_CTRL,
+	[STMPE_IDX_SYS_CTRL2]	= STMPE24XX_REG_SYS_CTRL2,
 	[STMPE_IDX_ICR_LSB]	= STMPE24XX_REG_ICR_LSB,
+	[STMPE_IDX_IER_MSB]	= STMPE24XX_REG_IER_MSB,
 	[STMPE_IDX_IER_LSB]	= STMPE24XX_REG_IER_LSB,
 	[STMPE_IDX_ISR_MSB]	= STMPE24XX_REG_ISR_MSB,
 	[STMPE_IDX_GPMR_LSB]	= STMPE24XX_REG_GPMR_LSB,
+	[STMPE_IDX_GPMR_CSB]	= STMPE24XX_REG_GPMR_CSB,
+	[STMPE_IDX_GPMR_MSB]	= STMPE24XX_REG_GPMR_MSB,
 	[STMPE_IDX_GPSR_LSB]	= STMPE24XX_REG_GPSR_LSB,
+	[STMPE_IDX_GPSR_CSB]	= STMPE24XX_REG_GPSR_CSB,
+	[STMPE_IDX_GPSR_MSB]	= STMPE24XX_REG_GPSR_MSB,
 	[STMPE_IDX_GPCR_LSB]	= STMPE24XX_REG_GPCR_LSB,
+	[STMPE_IDX_GPCR_CSB]	= STMPE24XX_REG_GPCR_CSB,
+	[STMPE_IDX_GPCR_MSB]	= STMPE24XX_REG_GPCR_MSB,
 	[STMPE_IDX_GPDR_LSB]	= STMPE24XX_REG_GPDR_LSB,
+	[STMPE_IDX_GPDR_CSB]	= STMPE24XX_REG_GPDR_CSB,
+	[STMPE_IDX_GPDR_MSB]	= STMPE24XX_REG_GPDR_MSB,
 	[STMPE_IDX_GPRER_LSB]	= STMPE24XX_REG_GPRER_LSB,
+	[STMPE_IDX_GPRER_CSB]	= STMPE24XX_REG_GPRER_CSB,
+	[STMPE_IDX_GPRER_MSB]	= STMPE24XX_REG_GPRER_MSB,
 	[STMPE_IDX_GPFER_LSB]	= STMPE24XX_REG_GPFER_LSB,
+	[STMPE_IDX_GPFER_CSB]	= STMPE24XX_REG_GPFER_CSB,
+	[STMPE_IDX_GPFER_MSB]	= STMPE24XX_REG_GPFER_MSB,
 	[STMPE_IDX_GPPUR_LSB]	= STMPE24XX_REG_GPPUR_LSB,
 	[STMPE_IDX_GPPDR_LSB]	= STMPE24XX_REG_GPPDR_LSB,
 	[STMPE_IDX_GPAFR_U_MSB]	= STMPE24XX_REG_GPAFR_U_MSB,
 	[STMPE_IDX_IEGPIOR_LSB]	= STMPE24XX_REG_IEGPIOR_LSB,
+	[STMPE_IDX_IEGPIOR_CSB]	= STMPE24XX_REG_IEGPIOR_CSB,
+	[STMPE_IDX_IEGPIOR_MSB]	= STMPE24XX_REG_IEGPIOR_MSB,
 	[STMPE_IDX_ISGPIOR_MSB]	= STMPE24XX_REG_ISGPIOR_MSB,
+	[STMPE_IDX_GPEDR_LSB]	= STMPE24XX_REG_GPEDR_LSB,
+	[STMPE_IDX_GPEDR_CSB]	= STMPE24XX_REG_GPEDR_CSB,
 	[STMPE_IDX_GPEDR_MSB]	= STMPE24XX_REG_GPEDR_MSB,
 };
 
@@ -840,7 +949,7 @@ static int stmpe24xx_enable(struct stmpe *stmpe, unsigned int blocks,
 	if (blocks & STMPE_BLOCK_KEYPAD)
 		mask |= STMPE24XX_SYS_CTRL_ENABLE_KPC;
 
-	return __stmpe_set_bits(stmpe, STMPE24XX_REG_SYS_CTRL, mask,
+	return __stmpe_set_bits(stmpe, stmpe->regs[STMPE_IDX_SYS_CTRL], mask,
 				enable ? mask : 0);
 }
 
@@ -893,6 +1002,7 @@ static struct stmpe_variant_info *stmpe_variant_info[STMPE_NBR_PARTS] = {
 	[STMPE610]	= &stmpe610,
 	[STMPE801]	= &stmpe801,
 	[STMPE811]	= &stmpe811,
+	[STMPE1600]	= &stmpe1600,
 	[STMPE1601]	= &stmpe1601,
 	[STMPE1801]	= &stmpe1801,
 	[STMPE2401]	= &stmpe2401,
@@ -919,7 +1029,8 @@ static irqreturn_t stmpe_irq(int irq, void *data)
 	int ret;
 	int i;
 
-	if (variant->id_val == STMPE801_ID) {
+	if (variant->id_val == STMPE801_ID ||
+	    variant->id_val == STMPE1600_ID) {
 		int base = irq_create_mapping(stmpe->domain, 0);
 
 		handle_nested_irq(base);
@@ -982,7 +1093,7 @@ static void stmpe_irq_sync_unlock(struct irq_data *data)
 			continue;
 
 		stmpe->oldier[i] = new;
-		stmpe_reg_write(stmpe, stmpe->regs[STMPE_IDX_IER_LSB] - i, new);
+		stmpe_reg_write(stmpe, stmpe->regs[STMPE_IDX_IER_LSB + i], new);
 	}
 
 	mutex_unlock(&stmpe->irq_lock);
@@ -1088,20 +1199,18 @@ static int stmpe_chip_init(struct stmpe *stmpe)
 	if (ret)
 		return ret;
 
-	if (id == STMPE1801_ID)	{
-		ret =  stmpe1801_reset(stmpe);
-		if (ret < 0)
-			return ret;
-	}
+	ret =  stmpe_reset(stmpe);
+	if (ret < 0)
+		return ret;
 
 	if (stmpe->irq >= 0) {
-		if (id == STMPE801_ID)
-			icr = STMPE801_REG_SYS_CTRL_INT_EN;
+		if (id == STMPE801_ID || id == STMPE1600_ID)
+			icr = STMPE_SYS_CTRL_INT_EN;
 		else
 			icr = STMPE_ICR_LSB_GIM;
 
-		/* STMPE801 doesn't support Edge interrupts */
-		if (id != STMPE801_ID) {
+		/* STMPE801 and STMPE1600 don't support Edge interrupts */
+		if (id != STMPE801_ID && id != STMPE1600_ID) {
 			if (irq_trigger == IRQF_TRIGGER_FALLING ||
 					irq_trigger == IRQF_TRIGGER_RISING)
 				icr |= STMPE_ICR_LSB_EDGE;
@@ -1109,8 +1218,8 @@ static int stmpe_chip_init(struct stmpe *stmpe)
 
 		if (irq_trigger == IRQF_TRIGGER_RISING ||
 				irq_trigger == IRQF_TRIGGER_HIGH) {
-			if (id == STMPE801_ID)
-				icr |= STMPE801_REG_SYS_CTRL_INT_HI;
+			if (id == STMPE801_ID || id == STMPE1600_ID)
+				icr |= STMPE_SYS_CTRL_INT_HI;
 			else
 				icr |= STMPE_ICR_LSB_HIGH;
 		}
diff --git a/drivers/mfd/stmpe.h b/drivers/mfd/stmpe.h
index 84adb46b3e2f..f7efdd8a5fc6 100644
--- a/drivers/mfd/stmpe.h
+++ b/drivers/mfd/stmpe.h
@@ -104,6 +104,10 @@ int stmpe_remove(struct stmpe *stmpe);
 #define STMPE_ICR_LSB_EDGE	(1 << 1)
 #define STMPE_ICR_LSB_GIM	(1 << 0)
 
+#define STMPE_SYS_CTRL_RESET	(1 << 7)
+#define STMPE_SYS_CTRL_INT_EN	(1 << 2)
+#define STMPE_SYS_CTRL_INT_HI	(1 << 0)
+
 /*
  * STMPE801
  */
@@ -119,13 +123,10 @@ int stmpe_remove(struct stmpe *stmpe);
 #define STMPE801_REG_GPIO_SET_PIN	0x11
 #define STMPE801_REG_GPIO_DIR		0x12
 
-#define STMPE801_REG_SYS_CTRL_RESET	(1 << 7)
-#define STMPE801_REG_SYS_CTRL_INT_EN	(1 << 2)
-#define STMPE801_REG_SYS_CTRL_INT_HI	(1 << 0)
-
 /*
  * STMPE811
  */
+#define STMPE811_ID			0x0811
 
 #define STMPE811_IRQ_TOUCH_DET		0
 #define STMPE811_IRQ_FIFO_TH		1
@@ -138,6 +139,7 @@ int stmpe_remove(struct stmpe *stmpe);
 #define STMPE811_NR_INTERNAL_IRQS	8
 
 #define STMPE811_REG_CHIP_ID		0x00
+#define STMPE811_REG_SYS_CTRL		0x03
 #define STMPE811_REG_SYS_CTRL2		0x04
 #define STMPE811_REG_SPI_CFG		0x08
 #define STMPE811_REG_INT_CTRL		0x09
@@ -154,12 +156,35 @@ int stmpe_remove(struct stmpe *stmpe);
 #define STMPE811_REG_GPIO_FE		0x16
 #define STMPE811_REG_GPIO_AF		0x17
 
+#define STMPE811_SYS_CTRL_RESET		(1 << 1)
+
 #define STMPE811_SYS_CTRL2_ADC_OFF	(1 << 0)
 #define STMPE811_SYS_CTRL2_TSC_OFF	(1 << 1)
 #define STMPE811_SYS_CTRL2_GPIO_OFF	(1 << 2)
 #define STMPE811_SYS_CTRL2_TS_OFF	(1 << 3)
 
 /*
+ * STMPE1600
+ */
+#define STMPE1600_ID			0x0016
+#define STMPE1600_NR_INTERNAL_IRQS	16
+
+#define STMPE1600_REG_CHIP_ID		0x00
+#define STMPE1600_REG_SYS_CTRL		0x03
+#define STMPE1600_REG_IEGPIOR_LSB	0x08
+#define STMPE1600_REG_IEGPIOR_MSB	0x09
+#define STMPE1600_REG_ISGPIOR_LSB	0x0A
+#define STMPE1600_REG_ISGPIOR_MSB	0x0B
+#define STMPE1600_REG_GPMR_LSB		0x10
+#define STMPE1600_REG_GPMR_MSB		0x11
+#define STMPE1600_REG_GPSR_LSB		0x12
+#define STMPE1600_REG_GPSR_MSB		0x13
+#define STMPE1600_REG_GPDR_LSB		0x14
+#define STMPE1600_REG_GPDR_MSB		0x15
+#define STMPE1600_REG_GPPIR_LSB		0x16
+#define STMPE1600_REG_GPPIR_MSB		0x17
+
+/*
  * STMPE1601
  */
 
@@ -175,19 +200,32 @@ int stmpe_remove(struct stmpe *stmpe);
 
 #define STMPE1601_REG_SYS_CTRL			0x02
 #define STMPE1601_REG_SYS_CTRL2			0x03
+#define STMPE1601_REG_ICR_MSB			0x10
 #define STMPE1601_REG_ICR_LSB			0x11
+#define STMPE1601_REG_IER_MSB			0x12
 #define STMPE1601_REG_IER_LSB			0x13
 #define STMPE1601_REG_ISR_MSB			0x14
-#define STMPE1601_REG_CHIP_ID			0x80
+#define STMPE1601_REG_ISR_LSB			0x15
+#define STMPE1601_REG_INT_EN_GPIO_MASK_MSB	0x16
 #define STMPE1601_REG_INT_EN_GPIO_MASK_LSB	0x17
 #define STMPE1601_REG_INT_STA_GPIO_MSB		0x18
-#define STMPE1601_REG_GPIO_MP_LSB		0x87
+#define STMPE1601_REG_INT_STA_GPIO_LSB		0x19
+#define STMPE1601_REG_CHIP_ID			0x80
+#define STMPE1601_REG_GPIO_SET_MSB		0x82
 #define STMPE1601_REG_GPIO_SET_LSB		0x83
+#define STMPE1601_REG_GPIO_CLR_MSB		0x84
 #define STMPE1601_REG_GPIO_CLR_LSB		0x85
+#define STMPE1601_REG_GPIO_MP_MSB		0x86
+#define STMPE1601_REG_GPIO_MP_LSB		0x87
+#define STMPE1601_REG_GPIO_SET_DIR_MSB		0x88
 #define STMPE1601_REG_GPIO_SET_DIR_LSB		0x89
 #define STMPE1601_REG_GPIO_ED_MSB		0x8A
+#define STMPE1601_REG_GPIO_ED_LSB		0x8B
+#define STMPE1601_REG_GPIO_RE_MSB		0x8C
 #define STMPE1601_REG_GPIO_RE_LSB		0x8D
+#define STMPE1601_REG_GPIO_FE_MSB		0x8E
 #define STMPE1601_REG_GPIO_FE_LSB		0x8F
+#define STMPE1601_REG_GPIO_PU_MSB		0x90
 #define STMPE1601_REG_GPIO_PU_LSB		0x91
 #define STMPE1601_REG_GPIO_AF_U_MSB		0x92
 
@@ -243,8 +281,6 @@ int stmpe_remove(struct stmpe *stmpe);
 #define STMPE1801_REG_GPIO_PULL_UP_MID		0x23
 #define STMPE1801_REG_GPIO_PULL_UP_HIGH		0x24
 
-#define STMPE1801_MSK_SYS_CTRL_RESET		(1 << 7)
-
 #define STMPE1801_MSK_INT_EN_KPC		(1 << 1)
 #define STMPE1801_MSK_INT_EN_GPIO		(1 << 3)
 
@@ -264,23 +300,48 @@ int stmpe_remove(struct stmpe *stmpe);
 #define STMPE24XX_NR_INTERNAL_IRQS	9
 
 #define STMPE24XX_REG_SYS_CTRL		0x02
+#define STMPE24XX_REG_SYS_CTRL2		0x03
+#define STMPE24XX_REG_ICR_MSB		0x10
 #define STMPE24XX_REG_ICR_LSB		0x11
+#define STMPE24XX_REG_IER_MSB		0x12
 #define STMPE24XX_REG_IER_LSB		0x13
 #define STMPE24XX_REG_ISR_MSB		0x14
-#define STMPE24XX_REG_CHIP_ID		0x80
+#define STMPE24XX_REG_ISR_LSB		0x15
+#define STMPE24XX_REG_IEGPIOR_MSB	0x16
+#define STMPE24XX_REG_IEGPIOR_CSB	0x17
 #define STMPE24XX_REG_IEGPIOR_LSB	0x18
 #define STMPE24XX_REG_ISGPIOR_MSB	0x19
-#define STMPE24XX_REG_GPMR_LSB		0xA4
+#define STMPE24XX_REG_ISGPIOR_CSB	0x1A
+#define STMPE24XX_REG_ISGPIOR_LSB	0x1B
+#define STMPE24XX_REG_CHIP_ID		0x80
+#define STMPE24XX_REG_GPSR_MSB		0x83
+#define STMPE24XX_REG_GPSR_CSB		0x84
 #define STMPE24XX_REG_GPSR_LSB		0x85
+#define STMPE24XX_REG_GPCR_MSB		0x86
+#define STMPE24XX_REG_GPCR_CSB		0x87
 #define STMPE24XX_REG_GPCR_LSB		0x88
+#define STMPE24XX_REG_GPDR_MSB		0x89
+#define STMPE24XX_REG_GPDR_CSB		0x8A
 #define STMPE24XX_REG_GPDR_LSB		0x8B
 #define STMPE24XX_REG_GPEDR_MSB		0x8C
+#define STMPE24XX_REG_GPEDR_CSB		0x8D
+#define STMPE24XX_REG_GPEDR_LSB		0x8E
+#define STMPE24XX_REG_GPRER_MSB		0x8F
+#define STMPE24XX_REG_GPRER_CSB		0x90
 #define STMPE24XX_REG_GPRER_LSB		0x91
+#define STMPE24XX_REG_GPFER_MSB		0x92
+#define STMPE24XX_REG_GPFER_CSB		0x93
 #define STMPE24XX_REG_GPFER_LSB		0x94
+#define STMPE24XX_REG_GPPUR_MSB		0x95
+#define STMPE24XX_REG_GPPUR_CSB		0x96
 #define STMPE24XX_REG_GPPUR_LSB		0x97
-#define STMPE24XX_REG_GPPDR_LSB		0x9a
+#define STMPE24XX_REG_GPPDR_MSB		0x98
+#define STMPE24XX_REG_GPPDR_CSB		0x99
+#define STMPE24XX_REG_GPPDR_LSB		0x9A
 #define STMPE24XX_REG_GPAFR_U_MSB	0x9B
-
+#define STMPE24XX_REG_GPMR_MSB		0xA2
+#define STMPE24XX_REG_GPMR_CSB		0xA3
+#define STMPE24XX_REG_GPMR_LSB		0xA4
 #define STMPE24XX_SYS_CTRL_ENABLE_GPIO		(1 << 3)
 #define STMPE24XX_SYSCON_ENABLE_PWM		(1 << 2)
 #define STMPE24XX_SYS_CTRL_ENABLE_KPC		(1 << 1)
diff --git a/drivers/mfd/sun6i-prcm.c b/drivers/mfd/sun6i-prcm.c
index 191173166d65..011fcc555945 100644
--- a/drivers/mfd/sun6i-prcm.c
+++ b/drivers/mfd/sun6i-prcm.c
@@ -9,7 +9,7 @@
  */
 
 #include <linux/mfd/core.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 
 struct prcm_data {
@@ -170,8 +170,4 @@ static struct platform_driver sun6i_prcm_driver = {
 	},
 	.probe = sun6i_prcm_probe,
 };
-module_platform_driver(sun6i_prcm_driver);
-
-MODULE_AUTHOR("Boris BREZILLON <boris.brezillon@free-electrons.com>");
-MODULE_DESCRIPTION("Allwinner sun6i PRCM driver");
-MODULE_LICENSE("GPL v2");
+builtin_platform_driver(sun6i_prcm_driver);
diff --git a/drivers/mfd/tps65217.c b/drivers/mfd/tps65217.c
index 049a6fcac651..9a4d8684dd32 100644
--- a/drivers/mfd/tps65217.c
+++ b/drivers/mfd/tps65217.c
@@ -15,22 +15,103 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/kernel.h>
 #include <linux/device.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
+#include <linux/err.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/i2c.h>
-#include <linux/slab.h>
-#include <linux/regmap.h>
-#include <linux/err.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
 
 #include <linux/mfd/core.h>
 #include <linux/mfd/tps65217.h>
 
-static const struct mfd_cell tps65217s[] = {
+static struct resource charger_resources[] = {
+	DEFINE_RES_IRQ_NAMED(TPS65217_IRQ_AC, "AC"),
+	DEFINE_RES_IRQ_NAMED(TPS65217_IRQ_USB, "USB"),
+};
+
+static struct resource pb_resources[] = {
+	DEFINE_RES_IRQ_NAMED(TPS65217_IRQ_PB, "PB"),
+};
+
+struct tps65217_irq {
+	int mask;
+	int interrupt;
+};
+
+static const struct tps65217_irq tps65217_irqs[] = {
+	[TPS65217_IRQ_PB] = {
+		.mask = TPS65217_INT_PBM,
+		.interrupt = TPS65217_INT_PBI,
+	},
+	[TPS65217_IRQ_AC] = {
+		.mask = TPS65217_INT_ACM,
+		.interrupt = TPS65217_INT_ACI,
+	},
+	[TPS65217_IRQ_USB] = {
+		.mask = TPS65217_INT_USBM,
+		.interrupt = TPS65217_INT_USBI,
+	},
+};
+
+static void tps65217_irq_lock(struct irq_data *data)
+{
+	struct tps65217 *tps = irq_data_get_irq_chip_data(data);
+
+	mutex_lock(&tps->irq_lock);
+}
+
+static void tps65217_irq_sync_unlock(struct irq_data *data)
+{
+	struct tps65217 *tps = irq_data_get_irq_chip_data(data);
+	int ret;
+
+	ret = tps65217_reg_write(tps, TPS65217_REG_INT, tps->irq_mask,
+				TPS65217_PROTECT_NONE);
+	if (ret != 0)
+		dev_err(tps->dev, "Failed to sync IRQ masks\n");
+
+	mutex_unlock(&tps->irq_lock);
+}
+
+static inline const struct tps65217_irq *
+irq_to_tps65217_irq(struct tps65217 *tps, struct irq_data *data)
+{
+	return &tps65217_irqs[data->hwirq];
+}
+
+static void tps65217_irq_enable(struct irq_data *data)
+{
+	struct tps65217 *tps = irq_data_get_irq_chip_data(data);
+	const struct tps65217_irq *irq_data = irq_to_tps65217_irq(tps, data);
+
+	tps->irq_mask &= ~irq_data->mask;
+}
+
+static void tps65217_irq_disable(struct irq_data *data)
+{
+	struct tps65217 *tps = irq_data_get_irq_chip_data(data);
+	const struct tps65217_irq *irq_data = irq_to_tps65217_irq(tps, data);
+
+	tps->irq_mask |= irq_data->mask;
+}
+
+static struct irq_chip tps65217_irq_chip = {
+	.irq_bus_lock		= tps65217_irq_lock,
+	.irq_bus_sync_unlock	= tps65217_irq_sync_unlock,
+	.irq_enable		= tps65217_irq_enable,
+	.irq_disable		= tps65217_irq_disable,
+};
+
+static struct mfd_cell tps65217s[] = {
 	{
 		.name = "tps65217-pmic",
 		.of_compatible = "ti,tps65217-pmic",
@@ -41,10 +122,96 @@ static const struct mfd_cell tps65217s[] = {
 	},
 	{
 		.name = "tps65217-charger",
+		.num_resources = ARRAY_SIZE(charger_resources),
+		.resources = charger_resources,
 		.of_compatible = "ti,tps65217-charger",
 	},
+	{
+		.name = "tps65217-pwrbutton",
+		.num_resources = ARRAY_SIZE(pb_resources),
+		.resources = pb_resources,
+		.of_compatible = "ti,tps65217-pwrbutton",
+	},
+};
+
+static irqreturn_t tps65217_irq_thread(int irq, void *data)
+{
+	struct tps65217 *tps = data;
+	unsigned int status;
+	bool handled = false;
+	int i;
+	int ret;
+
+	ret = tps65217_reg_read(tps, TPS65217_REG_INT, &status);
+	if (ret < 0) {
+		dev_err(tps->dev, "Failed to read IRQ status: %d\n",
+			ret);
+		return IRQ_NONE;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(tps65217_irqs); i++) {
+		if (status & tps65217_irqs[i].interrupt) {
+			handle_nested_irq(irq_find_mapping(tps->irq_domain, i));
+			handled = true;
+		}
+	}
+
+	if (handled)
+		return IRQ_HANDLED;
+
+	return IRQ_NONE;
+}
+
+static int tps65217_irq_map(struct irq_domain *h, unsigned int virq,
+			irq_hw_number_t hw)
+{
+	struct tps65217 *tps = h->host_data;
+
+	irq_set_chip_data(virq, tps);
+	irq_set_chip_and_handler(virq, &tps65217_irq_chip, handle_edge_irq);
+	irq_set_nested_thread(virq, 1);
+	irq_set_parent(virq, tps->irq);
+	irq_set_noprobe(virq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops tps65217_irq_domain_ops = {
+	.map = tps65217_irq_map,
 };
 
+static int tps65217_irq_init(struct tps65217 *tps, int irq)
+{
+	int ret;
+
+	mutex_init(&tps->irq_lock);
+	tps->irq = irq;
+
+	/* Mask all interrupt sources */
+	tps->irq_mask = (TPS65217_INT_RESERVEDM | TPS65217_INT_PBM
+			| TPS65217_INT_ACM | TPS65217_INT_USBM);
+	tps65217_reg_write(tps, TPS65217_REG_INT, tps->irq_mask,
+			TPS65217_PROTECT_NONE);
+
+	tps->irq_domain = irq_domain_add_linear(tps->dev->of_node,
+		TPS65217_NUM_IRQ, &tps65217_irq_domain_ops, tps);
+	if (!tps->irq_domain) {
+		dev_err(tps->dev, "Could not create IRQ domain\n");
+		return -ENOMEM;
+	}
+
+	ret = devm_request_threaded_irq(tps->dev, irq, NULL,
+					tps65217_irq_thread, IRQF_ONESHOT,
+					"tps65217-irq", tps);
+	if (ret) {
+		dev_err(tps->dev, "Failed to request IRQ %d: %d\n",
+			irq, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 /**
  * tps65217_reg_read: Read a single tps65217 register.
  *
@@ -149,11 +316,22 @@ int tps65217_clear_bits(struct tps65217 *tps, unsigned int reg,
 }
 EXPORT_SYMBOL_GPL(tps65217_clear_bits);
 
+static bool tps65217_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case TPS65217_REG_INT:
+		return true;
+	default:
+		return false;
+	}
+}
+
 static const struct regmap_config tps65217_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
 
 	.max_register = TPS65217_REG_MAX,
+	.volatile_reg = tps65217_volatile_reg,
 };
 
 static const struct of_device_id tps65217_of_match[] = {
@@ -205,8 +383,19 @@ static int tps65217_probe(struct i2c_client *client,
 		return ret;
 	}
 
+	if (client->irq) {
+		tps65217_irq_init(tps, client->irq);
+	} else {
+		int i;
+
+		/* Don't tell children about IRQ resources which won't fire */
+		for (i = 0; i < ARRAY_SIZE(tps65217s); i++)
+			tps65217s[i].num_resources = 0;
+	}
+
 	ret = devm_mfd_add_devices(tps->dev, -1, tps65217s,
-				   ARRAY_SIZE(tps65217s), NULL, 0, NULL);
+				   ARRAY_SIZE(tps65217s), NULL, 0,
+				   tps->irq_domain);
 	if (ret < 0) {
 		dev_err(tps->dev, "mfd_add_devices failed: %d\n", ret);
 		return ret;
diff --git a/drivers/mfd/tps65218.c b/drivers/mfd/tps65218.c
index 80b9dc363cd8..ba610adbdbff 100644
--- a/drivers/mfd/tps65218.c
+++ b/drivers/mfd/tps65218.c
@@ -219,6 +219,7 @@ static int tps65218_probe(struct i2c_client *client,
 	struct tps65218 *tps;
 	const struct of_device_id *match;
 	int ret;
+	unsigned int chipid;
 
 	match = of_match_device(of_tps65218_match_table, &client->dev);
 	if (!match) {
@@ -250,6 +251,14 @@ static int tps65218_probe(struct i2c_client *client,
 	if (ret < 0)
 		return ret;
 
+	ret = tps65218_reg_read(tps, TPS65218_REG_CHIPID, &chipid);
+	if (ret) {
+		dev_err(tps->dev, "Failed to read chipid: %d\n", ret);
+		return ret;
+	}
+
+	tps->rev = chipid & TPS65218_CHIPID_REV_MASK;
+
 	ret = of_platform_populate(client->dev.of_node, NULL, NULL,
 				   &client->dev);
 	if (ret < 0)
diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c
index a49d3db6d936..c64615dca2bd 100644
--- a/drivers/mfd/twl-core.c
+++ b/drivers/mfd/twl-core.c
@@ -30,7 +30,6 @@
 
 #include <linux/init.h>
 #include <linux/mutex.h>
-#include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 #include <linux/clk.h>
@@ -1258,7 +1257,6 @@ static const struct i2c_device_id twl_ids[] = {
 	{ "twl6032", TWL6030_CLASS | TWL6032_SUBCLASS }, /* "Phoenix lite" */
 	{ /* end of list */ },
 };
-MODULE_DEVICE_TABLE(i2c, twl_ids);
 
 /* One Client Driver , 4 Clients */
 static struct i2c_driver twl_driver = {
@@ -1267,9 +1265,4 @@ static struct i2c_driver twl_driver = {
 	.probe		= twl_probe,
 	.remove		= twl_remove,
 };
-
-module_i2c_driver(twl_driver);
-
-MODULE_AUTHOR("Texas Instruments, Inc.");
-MODULE_DESCRIPTION("I2C Core interface for TWL");
-MODULE_LICENSE("GPL");
+builtin_i2c_driver(twl_driver);
diff --git a/drivers/mfd/twl6040.c b/drivers/mfd/twl6040.c
index ab328ec49353..d66502d36ba0 100644
--- a/drivers/mfd/twl6040.c
+++ b/drivers/mfd/twl6040.c
@@ -609,6 +609,7 @@ static const struct regmap_config twl6040_regmap_config = {
 	.writeable_reg = twl6040_writeable_reg,
 
 	.cache_type = REGCACHE_RBTREE,
+	.use_single_rw = true,
 };
 
 static const struct regmap_irq twl6040_irqs[] = {
@@ -782,6 +783,11 @@ static int twl6040_probe(struct i2c_client *client,
 	cell->name = "twl6040-gpo";
 	children++;
 
+	/* PDM clock support  */
+	cell = &twl6040->cells[children];
+	cell->name = "twl6040-pdmclk";
+	children++;
+
 	/* The chip is powered down so mark regmap to cache only and dirty */
 	regcache_cache_only(twl6040->regmap, true);
 	regcache_mark_dirty(twl6040->regmap);
diff --git a/drivers/mfd/ucb1x00-core.c b/drivers/mfd/ucb1x00-core.c
index 9ab9ec47ea75..d6fb2e1a759a 100644
--- a/drivers/mfd/ucb1x00-core.c
+++ b/drivers/mfd/ucb1x00-core.c
@@ -446,10 +446,6 @@ static int ucb1x00_detect_irq(struct ucb1x00 *ucb)
 	unsigned long mask;
 
 	mask = probe_irq_on();
-	if (!mask) {
-		probe_irq_off(mask);
-		return NO_IRQ;
-	}
 
 	/*
 	 * Enable the ADC interrupt.
@@ -541,7 +537,7 @@ static int ucb1x00_probe(struct mcp *mcp)
 	ucb1x00_enable(ucb);
 	ucb->irq = ucb1x00_detect_irq(ucb);
 	ucb1x00_disable(ucb);
-	if (ucb->irq == NO_IRQ) {
+	if (!ucb->irq) {
 		dev_err(&ucb->dev, "IRQ probe failed\n");
 		ret = -ENODEV;
 		goto err_no_irq;
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index a216b4667742..64971baf11fa 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -345,16 +345,6 @@ config SENSORS_TSL2550
 	  This driver can also be built as a module.  If so, the module
 	  will be called tsl2550.
 
-config SENSORS_BH1780
-	tristate "ROHM BH1780GLI ambient light sensor"
-	depends on I2C && SYSFS
-	help
-	  If you say yes here you get support for the ROHM BH1780GLI
-	  ambient light sensor.
-
-	  This driver can also be built as a module.  If so, the module
-	  will be called bh1780gli.
-
 config SENSORS_BH1770
          tristate "BH1770GLC / SFH7770 combined ALS - Proximity sensor"
          depends on I2C
@@ -439,34 +429,6 @@ config ARM_CHARLCD
 	  line and the Linux version on the second line, but that's
 	  still useful.
 
-config BMP085
-	tristate
-	depends on SYSFS
-
-config BMP085_I2C
-	tristate "BMP085 digital pressure sensor on I2C"
-	select BMP085
-	select REGMAP_I2C
-	depends on I2C && SYSFS
-	help
-	  Say Y here if you want to support Bosch Sensortec's digital pressure
-	  sensor hooked to an I2C bus.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called bmp085-i2c.
-
-config BMP085_SPI
-	tristate "BMP085 digital pressure sensor on SPI"
-	select BMP085
-	select REGMAP_SPI
-	depends on SPI_MASTER && SYSFS
-	help
-	  Say Y here if you want to support Bosch Sensortec's digital pressure
-	  sensor hooked to an SPI bus.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called bmp085-spi.
-
 config PCH_PHUB
 	tristate "Intel EG20T PCH/LAPIS Semicon IOH(ML7213/ML7223/ML7831) PHUB"
 	select GENERIC_NET_UTILS
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index 4387ccb79e64..31983366090a 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -9,9 +9,6 @@ obj-$(CONFIG_AD525X_DPOT_SPI)	+= ad525x_dpot-spi.o
 obj-$(CONFIG_INTEL_MID_PTI)	+= pti.o
 obj-$(CONFIG_ATMEL_SSC)		+= atmel-ssc.o
 obj-$(CONFIG_ATMEL_TCLIB)	+= atmel_tclib.o
-obj-$(CONFIG_BMP085)		+= bmp085.o
-obj-$(CONFIG_BMP085_I2C)	+= bmp085-i2c.o
-obj-$(CONFIG_BMP085_SPI)	+= bmp085-spi.o
 obj-$(CONFIG_DUMMY_IRQ)		+= dummy-irq.o
 obj-$(CONFIG_ICS932S401)	+= ics932s401.o
 obj-$(CONFIG_LKDTM)		+= lkdtm.o
@@ -19,7 +16,6 @@ obj-$(CONFIG_TIFM_CORE)       	+= tifm_core.o
 obj-$(CONFIG_TIFM_7XX1)       	+= tifm_7xx1.o
 obj-$(CONFIG_PHANTOM)		+= phantom.o
 obj-$(CONFIG_QCOM_COINCELL)	+= qcom-coincell.o
-obj-$(CONFIG_SENSORS_BH1780)	+= bh1780gli.o
 obj-$(CONFIG_SENSORS_BH1770)	+= bh1770glc.o
 obj-$(CONFIG_SENSORS_APDS990X)	+= apds990x.o
 obj-$(CONFIG_SGI_IOC4)		+= ioc4.o
@@ -69,5 +65,6 @@ OBJCOPYFLAGS :=
 OBJCOPYFLAGS_lkdtm_rodata_objcopy.o := \
 			--set-section-flags .text=alloc,readonly \
 			--rename-section .text=.rodata
-$(obj)/lkdtm_rodata_objcopy.o: $(obj)/lkdtm_rodata.o
+targets += lkdtm_rodata.o lkdtm_rodata_objcopy.o
+$(obj)/lkdtm_rodata_objcopy.o: $(obj)/lkdtm_rodata.o FORCE
 	$(call if_changed,objcopy)
diff --git a/drivers/misc/bh1780gli.c b/drivers/misc/bh1780gli.c
deleted file mode 100644
index 7f90ce5a569a..000000000000
--- a/drivers/misc/bh1780gli.c
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * bh1780gli.c
- * ROHM Ambient Light Sensor Driver
- *
- * Copyright (C) 2010 Texas Instruments
- * Author: Hemanth V <hemanthv@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#include <linux/i2c.h>
-#include <linux/slab.h>
-#include <linux/mutex.h>
-#include <linux/platform_device.h>
-#include <linux/delay.h>
-#include <linux/module.h>
-#include <linux/of.h>
-
-#define BH1780_REG_CONTROL	0x80
-#define BH1780_REG_PARTID	0x8A
-#define BH1780_REG_MANFID	0x8B
-#define BH1780_REG_DLOW	0x8C
-#define BH1780_REG_DHIGH	0x8D
-
-#define BH1780_REVMASK		(0xf)
-#define BH1780_POWMASK		(0x3)
-#define BH1780_POFF		(0x0)
-#define BH1780_PON		(0x3)
-
-/* power on settling time in ms */
-#define BH1780_PON_DELAY	2
-
-struct bh1780_data {
-	struct i2c_client *client;
-	int power_state;
-	/* lock for sysfs operations */
-	struct mutex lock;
-};
-
-static int bh1780_write(struct bh1780_data *ddata, u8 reg, u8 val, char *msg)
-{
-	int ret = i2c_smbus_write_byte_data(ddata->client, reg, val);
-	if (ret < 0)
-		dev_err(&ddata->client->dev,
-			"i2c_smbus_write_byte_data failed error %d Register (%s)\n",
-			ret, msg);
-	return ret;
-}
-
-static int bh1780_read(struct bh1780_data *ddata, u8 reg, char *msg)
-{
-	int ret = i2c_smbus_read_byte_data(ddata->client, reg);
-	if (ret < 0)
-		dev_err(&ddata->client->dev,
-			"i2c_smbus_read_byte_data failed error %d Register (%s)\n",
-			ret, msg);
-	return ret;
-}
-
-static ssize_t bh1780_show_lux(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct bh1780_data *ddata = platform_get_drvdata(pdev);
-	int lsb, msb;
-
-	lsb = bh1780_read(ddata, BH1780_REG_DLOW, "DLOW");
-	if (lsb < 0)
-		return lsb;
-
-	msb = bh1780_read(ddata, BH1780_REG_DHIGH, "DHIGH");
-	if (msb < 0)
-		return msb;
-
-	return sprintf(buf, "%d\n", (msb << 8) | lsb);
-}
-
-static ssize_t bh1780_show_power_state(struct device *dev,
-					struct device_attribute *attr,
-					char *buf)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct bh1780_data *ddata = platform_get_drvdata(pdev);
-	int state;
-
-	state = bh1780_read(ddata, BH1780_REG_CONTROL, "CONTROL");
-	if (state < 0)
-		return state;
-
-	return sprintf(buf, "%d\n", state & BH1780_POWMASK);
-}
-
-static ssize_t bh1780_store_power_state(struct device *dev,
-					struct device_attribute *attr,
-					const char *buf, size_t count)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct bh1780_data *ddata = platform_get_drvdata(pdev);
-	unsigned long val;
-	int error;
-
-	error = kstrtoul(buf, 0, &val);
-	if (error)
-		return error;
-
-	if (val < BH1780_POFF || val > BH1780_PON)
-		return -EINVAL;
-
-	mutex_lock(&ddata->lock);
-
-	error = bh1780_write(ddata, BH1780_REG_CONTROL, val, "CONTROL");
-	if (error < 0) {
-		mutex_unlock(&ddata->lock);
-		return error;
-	}
-
-	msleep(BH1780_PON_DELAY);
-	ddata->power_state = val;
-	mutex_unlock(&ddata->lock);
-
-	return count;
-}
-
-static DEVICE_ATTR(lux, S_IRUGO, bh1780_show_lux, NULL);
-
-static DEVICE_ATTR(power_state, S_IWUSR | S_IRUGO,
-		bh1780_show_power_state, bh1780_store_power_state);
-
-static struct attribute *bh1780_attributes[] = {
-	&dev_attr_power_state.attr,
-	&dev_attr_lux.attr,
-	NULL
-};
-
-static const struct attribute_group bh1780_attr_group = {
-	.attrs = bh1780_attributes,
-};
-
-static int bh1780_probe(struct i2c_client *client,
-						const struct i2c_device_id *id)
-{
-	int ret;
-	struct bh1780_data *ddata;
-	struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent);
-
-	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE))
-		return -EIO;
-
-	ddata = devm_kzalloc(&client->dev, sizeof(struct bh1780_data),
-			     GFP_KERNEL);
-	if (ddata == NULL)
-		return -ENOMEM;
-
-	ddata->client = client;
-	i2c_set_clientdata(client, ddata);
-
-	ret = bh1780_read(ddata, BH1780_REG_PARTID, "PART ID");
-	if (ret < 0)
-		return ret;
-
-	dev_info(&client->dev, "Ambient Light Sensor, Rev : %d\n",
-			(ret & BH1780_REVMASK));
-
-	mutex_init(&ddata->lock);
-
-	return sysfs_create_group(&client->dev.kobj, &bh1780_attr_group);
-}
-
-static int bh1780_remove(struct i2c_client *client)
-{
-	sysfs_remove_group(&client->dev.kobj, &bh1780_attr_group);
-
-	return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int bh1780_suspend(struct device *dev)
-{
-	struct bh1780_data *ddata;
-	int state, ret;
-	struct i2c_client *client = to_i2c_client(dev);
-
-	ddata = i2c_get_clientdata(client);
-	state = bh1780_read(ddata, BH1780_REG_CONTROL, "CONTROL");
-	if (state < 0)
-		return state;
-
-	ddata->power_state = state & BH1780_POWMASK;
-
-	ret = bh1780_write(ddata, BH1780_REG_CONTROL, BH1780_POFF,
-				"CONTROL");
-
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-static int bh1780_resume(struct device *dev)
-{
-	struct bh1780_data *ddata;
-	int state, ret;
-	struct i2c_client *client = to_i2c_client(dev);
-
-	ddata = i2c_get_clientdata(client);
-	state = ddata->power_state;
-	ret = bh1780_write(ddata, BH1780_REG_CONTROL, state,
-				"CONTROL");
-
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-#endif /* CONFIG_PM_SLEEP */
-
-static SIMPLE_DEV_PM_OPS(bh1780_pm, bh1780_suspend, bh1780_resume);
-
-static const struct i2c_device_id bh1780_id[] = {
-	{ "bh1780", 0 },
-	{ },
-};
-
-MODULE_DEVICE_TABLE(i2c, bh1780_id);
-
-#ifdef CONFIG_OF
-static const struct of_device_id of_bh1780_match[] = {
-	{ .compatible = "rohm,bh1780gli", },
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, of_bh1780_match);
-#endif
-
-static struct i2c_driver bh1780_driver = {
-	.probe		= bh1780_probe,
-	.remove		= bh1780_remove,
-	.id_table	= bh1780_id,
-	.driver = {
-		.name = "bh1780",
-		.pm	= &bh1780_pm,
-		.of_match_table = of_match_ptr(of_bh1780_match),
-	},
-};
-
-module_i2c_driver(bh1780_driver);
-
-MODULE_DESCRIPTION("BH1780GLI Ambient Light Sensor Driver");
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Hemanth V <hemanthv@ti.com>");
diff --git a/drivers/misc/bmp085-i2c.c b/drivers/misc/bmp085-i2c.c
deleted file mode 100644
index f35c218aaa1a..000000000000
--- a/drivers/misc/bmp085-i2c.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright (c) 2012  Bosch Sensortec GmbH
- * Copyright (c) 2012  Unixphere AB
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#include <linux/module.h>
-#include <linux/i2c.h>
-#include <linux/err.h>
-#include "bmp085.h"
-
-#define BMP085_I2C_ADDRESS	0x77
-
-static const unsigned short normal_i2c[] = { BMP085_I2C_ADDRESS,
-							I2C_CLIENT_END };
-
-static int bmp085_i2c_detect(struct i2c_client *client,
-			     struct i2c_board_info *info)
-{
-	if (client->addr != BMP085_I2C_ADDRESS)
-		return -ENODEV;
-
-	return bmp085_detect(&client->dev);
-}
-
-static int bmp085_i2c_probe(struct i2c_client *client,
-				      const struct i2c_device_id *id)
-{
-	int err;
-	struct regmap *regmap = devm_regmap_init_i2c(client,
-						     &bmp085_regmap_config);
-
-	if (IS_ERR(regmap)) {
-		err = PTR_ERR(regmap);
-		dev_err(&client->dev, "Failed to init regmap: %d\n", err);
-		return err;
-	}
-
-	return bmp085_probe(&client->dev, regmap, client->irq);
-}
-
-static int bmp085_i2c_remove(struct i2c_client *client)
-{
-	return bmp085_remove(&client->dev);
-}
-
-static const struct i2c_device_id bmp085_id[] = {
-	{ BMP085_NAME, 0 },
-	{ "bmp180", 0 },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, bmp085_id);
-
-static struct i2c_driver bmp085_i2c_driver = {
-	.driver = {
-		.name	= BMP085_NAME,
-	},
-	.id_table	= bmp085_id,
-	.probe		= bmp085_i2c_probe,
-	.remove		= bmp085_i2c_remove,
-
-	.detect		= bmp085_i2c_detect,
-	.address_list	= normal_i2c
-};
-
-module_i2c_driver(bmp085_i2c_driver);
-
-MODULE_AUTHOR("Eric Andersson <eric.andersson@unixphere.com>");
-MODULE_DESCRIPTION("BMP085 I2C bus driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/misc/bmp085-spi.c b/drivers/misc/bmp085-spi.c
deleted file mode 100644
index 17ecbf95ff15..000000000000
--- a/drivers/misc/bmp085-spi.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2012  Bosch Sensortec GmbH
- * Copyright (c) 2012  Unixphere AB
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#include <linux/module.h>
-#include <linux/spi/spi.h>
-#include <linux/err.h>
-#include "bmp085.h"
-
-static int bmp085_spi_probe(struct spi_device *client)
-{
-	int err;
-	struct regmap *regmap;
-
-	client->bits_per_word = 8;
-	err = spi_setup(client);
-	if (err < 0) {
-		dev_err(&client->dev, "spi_setup failed!\n");
-		return err;
-	}
-
-	regmap = devm_regmap_init_spi(client, &bmp085_regmap_config);
-	if (IS_ERR(regmap)) {
-		err = PTR_ERR(regmap);
-		dev_err(&client->dev, "Failed to init regmap: %d\n", err);
-		return err;
-	}
-
-	return bmp085_probe(&client->dev, regmap, client->irq);
-}
-
-static int bmp085_spi_remove(struct spi_device *client)
-{
-	return bmp085_remove(&client->dev);
-}
-
-static const struct of_device_id bmp085_of_match[] = {
-	{ .compatible = "bosch,bmp085", },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, bmp085_of_match);
-
-static const struct spi_device_id bmp085_id[] = {
-	{ "bmp180", 0 },
-	{ "bmp181", 0 },
-	{ }
-};
-MODULE_DEVICE_TABLE(spi, bmp085_id);
-
-static struct spi_driver bmp085_spi_driver = {
-	.driver = {
-		.name	= BMP085_NAME,
-		.of_match_table = bmp085_of_match
-	},
-	.id_table	= bmp085_id,
-	.probe		= bmp085_spi_probe,
-	.remove		= bmp085_spi_remove
-};
-
-module_spi_driver(bmp085_spi_driver);
-
-MODULE_AUTHOR("Eric Andersson <eric.andersson@unixphere.com>");
-MODULE_DESCRIPTION("BMP085 SPI bus driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/misc/bmp085.c b/drivers/misc/bmp085.c
deleted file mode 100644
index 9b313f7810f5..000000000000
--- a/drivers/misc/bmp085.c
+++ /dev/null
@@ -1,506 +0,0 @@
-/*  Copyright (c) 2010  Christoph Mair <christoph.mair@gmail.com>
- *  Copyright (c) 2012  Bosch Sensortec GmbH
- *  Copyright (c) 2012  Unixphere AB
- *
- *  This driver supports the bmp085 and bmp18x digital barometric pressure
- *  and temperature sensors from Bosch Sensortec. The datasheets
- *  are available from their website:
- *  http://www.bosch-sensortec.com/content/language1/downloads/BST-BMP085-DS000-05.pdf
- *  http://www.bosch-sensortec.com/content/language1/downloads/BST-BMP180-DS000-07.pdf
- *
- *  A pressure measurement is issued by reading from pressure0_input.
- *  The return value ranges from 30000 to 110000 pascal with a resulution
- *  of 1 pascal (0.01 millibar) which enables measurements from 9000m above
- *  to 500m below sea level.
- *
- *  The temperature can be read from temp0_input. Values range from
- *  -400 to 850 representing the ambient temperature in degree celsius
- *  multiplied by 10.The resolution is 0.1 celsius.
- *
- *  Because ambient pressure is temperature dependent, a temperature
- *  measurement will be executed automatically even if the user is reading
- *  from pressure0_input. This happens if the last temperature measurement
- *  has been executed more then one second ago.
- *
- *  To decrease RMS noise from pressure measurements, the bmp085 can
- *  autonomously calculate the average of up to eight samples. This is
- *  set up by writing to the oversampling sysfs file. Accepted values
- *  are 0, 1, 2 and 3. 2^x when x is the value written to this file
- *  specifies the number of samples used to calculate the ambient pressure.
- *  RMS noise is specified with six pascal (without averaging) and decreases
- *  down to 3 pascal when using an oversampling setting of 3.
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#include <linux/module.h>
-#include <linux/device.h>
-#include <linux/slab.h>
-#include <linux/of.h>
-#include "bmp085.h"
-#include <linux/interrupt.h>
-#include <linux/completion.h>
-#include <linux/gpio.h>
-
-#define BMP085_CHIP_ID			0x55
-#define BMP085_CALIBRATION_DATA_START	0xAA
-#define BMP085_CALIBRATION_DATA_LENGTH	11	/* 16 bit values */
-#define BMP085_CHIP_ID_REG		0xD0
-#define BMP085_CTRL_REG			0xF4
-#define BMP085_TEMP_MEASUREMENT		0x2E
-#define BMP085_PRESSURE_MEASUREMENT	0x34
-#define BMP085_CONVERSION_REGISTER_MSB	0xF6
-#define BMP085_CONVERSION_REGISTER_LSB	0xF7
-#define BMP085_CONVERSION_REGISTER_XLSB	0xF8
-#define BMP085_TEMP_CONVERSION_TIME	5
-
-struct bmp085_calibration_data {
-	s16 AC1, AC2, AC3;
-	u16 AC4, AC5, AC6;
-	s16 B1, B2;
-	s16 MB, MC, MD;
-};
-
-struct bmp085_data {
-	struct	device *dev;
-	struct  regmap *regmap;
-	struct	mutex lock;
-	struct	bmp085_calibration_data calibration;
-	u8	oversampling_setting;
-	u32	raw_temperature;
-	u32	raw_pressure;
-	u32	temp_measurement_period;
-	unsigned long last_temp_measurement;
-	u8	chip_id;
-	s32	b6; /* calculated temperature correction coefficient */
-	int	irq;
-	struct	completion done;
-};
-
-static irqreturn_t bmp085_eoc_isr(int irq, void *devid)
-{
-	struct bmp085_data *data = devid;
-
-	complete(&data->done);
-
-	return IRQ_HANDLED;
-}
-
-static s32 bmp085_read_calibration_data(struct bmp085_data *data)
-{
-	u16 tmp[BMP085_CALIBRATION_DATA_LENGTH];
-	struct bmp085_calibration_data *cali = &(data->calibration);
-	s32 status = regmap_bulk_read(data->regmap,
-				BMP085_CALIBRATION_DATA_START, (u8 *)tmp,
-				(BMP085_CALIBRATION_DATA_LENGTH << 1));
-	if (status < 0)
-		return status;
-
-	cali->AC1 =  be16_to_cpu(tmp[0]);
-	cali->AC2 =  be16_to_cpu(tmp[1]);
-	cali->AC3 =  be16_to_cpu(tmp[2]);
-	cali->AC4 =  be16_to_cpu(tmp[3]);
-	cali->AC5 =  be16_to_cpu(tmp[4]);
-	cali->AC6 = be16_to_cpu(tmp[5]);
-	cali->B1 = be16_to_cpu(tmp[6]);
-	cali->B2 = be16_to_cpu(tmp[7]);
-	cali->MB = be16_to_cpu(tmp[8]);
-	cali->MC = be16_to_cpu(tmp[9]);
-	cali->MD = be16_to_cpu(tmp[10]);
-	return 0;
-}
-
-static s32 bmp085_update_raw_temperature(struct bmp085_data *data)
-{
-	u16 tmp;
-	s32 status;
-
-	mutex_lock(&data->lock);
-
-	init_completion(&data->done);
-
-	status = regmap_write(data->regmap, BMP085_CTRL_REG,
-			      BMP085_TEMP_MEASUREMENT);
-	if (status < 0) {
-		dev_err(data->dev,
-			"Error while requesting temperature measurement.\n");
-		goto exit;
-	}
-	wait_for_completion_timeout(&data->done, 1 + msecs_to_jiffies(
-					    BMP085_TEMP_CONVERSION_TIME));
-
-	status = regmap_bulk_read(data->regmap, BMP085_CONVERSION_REGISTER_MSB,
-				 &tmp, sizeof(tmp));
-	if (status < 0) {
-		dev_err(data->dev,
-			"Error while reading temperature measurement result\n");
-		goto exit;
-	}
-	data->raw_temperature = be16_to_cpu(tmp);
-	data->last_temp_measurement = jiffies;
-	status = 0;	/* everything ok, return 0 */
-
-exit:
-	mutex_unlock(&data->lock);
-	return status;
-}
-
-static s32 bmp085_update_raw_pressure(struct bmp085_data *data)
-{
-	u32 tmp = 0;
-	s32 status;
-
-	mutex_lock(&data->lock);
-
-	init_completion(&data->done);
-
-	status = regmap_write(data->regmap, BMP085_CTRL_REG,
-			BMP085_PRESSURE_MEASUREMENT +
-			(data->oversampling_setting << 6));
-	if (status < 0) {
-		dev_err(data->dev,
-			"Error while requesting pressure measurement.\n");
-		goto exit;
-	}
-
-	/* wait for the end of conversion */
-	wait_for_completion_timeout(&data->done, 1 + msecs_to_jiffies(
-					2+(3 << data->oversampling_setting)));
-	/* copy data into a u32 (4 bytes), but skip the first byte. */
-	status = regmap_bulk_read(data->regmap, BMP085_CONVERSION_REGISTER_MSB,
-				 ((u8 *)&tmp)+1, 3);
-	if (status < 0) {
-		dev_err(data->dev,
-			"Error while reading pressure measurement results\n");
-		goto exit;
-	}
-	data->raw_pressure = be32_to_cpu((tmp));
-	data->raw_pressure >>= (8-data->oversampling_setting);
-	status = 0;	/* everything ok, return 0 */
-
-exit:
-	mutex_unlock(&data->lock);
-	return status;
-}
-
-/*
- * This function starts the temperature measurement and returns the value
- * in tenth of a degree celsius.
- */
-static s32 bmp085_get_temperature(struct bmp085_data *data, int *temperature)
-{
-	struct bmp085_calibration_data *cali = &data->calibration;
-	long x1, x2;
-	int status;
-
-	status = bmp085_update_raw_temperature(data);
-	if (status < 0)
-		goto exit;
-
-	x1 = ((data->raw_temperature - cali->AC6) * cali->AC5) >> 15;
-	x2 = (cali->MC << 11) / (x1 + cali->MD);
-	data->b6 = x1 + x2 - 4000;
-	/* if NULL just update b6. Used for pressure only measurements */
-	if (temperature != NULL)
-		*temperature = (x1+x2+8) >> 4;
-
-exit:
-	return status;
-}
-
-/*
- * This function starts the pressure measurement and returns the value
- * in millibar. Since the pressure depends on the ambient temperature,
- * a temperature measurement is executed according to the given temperature
- * measurement period (default is 1 sec boundary). This period could vary
- * and needs to be adjusted according to the sensor environment, i.e. if big
- * temperature variations then the temperature needs to be read out often.
- */
-static s32 bmp085_get_pressure(struct bmp085_data *data, int *pressure)
-{
-	struct bmp085_calibration_data *cali = &data->calibration;
-	s32 x1, x2, x3, b3;
-	u32 b4, b7;
-	s32 p;
-	int status;
-
-	/* alt least every second force an update of the ambient temperature */
-	if ((data->last_temp_measurement == 0) ||
-	    time_is_before_jiffies(data->last_temp_measurement + 1*HZ)) {
-		status = bmp085_get_temperature(data, NULL);
-		if (status < 0)
-			return status;
-	}
-
-	status = bmp085_update_raw_pressure(data);
-	if (status < 0)
-		return status;
-
-	x1 = (data->b6 * data->b6) >> 12;
-	x1 *= cali->B2;
-	x1 >>= 11;
-
-	x2 = cali->AC2 * data->b6;
-	x2 >>= 11;
-
-	x3 = x1 + x2;
-
-	b3 = (((((s32)cali->AC1) * 4 + x3) << data->oversampling_setting) + 2);
-	b3 >>= 2;
-
-	x1 = (cali->AC3 * data->b6) >> 13;
-	x2 = (cali->B1 * ((data->b6 * data->b6) >> 12)) >> 16;
-	x3 = (x1 + x2 + 2) >> 2;
-	b4 = (cali->AC4 * (u32)(x3 + 32768)) >> 15;
-
-	b7 = ((u32)data->raw_pressure - b3) *
-					(50000 >> data->oversampling_setting);
-	p = ((b7 < 0x80000000) ? ((b7 << 1) / b4) : ((b7 / b4) * 2));
-
-	x1 = p >> 8;
-	x1 *= x1;
-	x1 = (x1 * 3038) >> 16;
-	x2 = (-7357 * p) >> 16;
-	p += (x1 + x2 + 3791) >> 4;
-
-	*pressure = p;
-
-	return 0;
-}
-
-/*
- * This function sets the chip-internal oversampling. Valid values are 0..3.
- * The chip will use 2^oversampling samples for internal averaging.
- * This influences the measurement time and the accuracy; larger values
- * increase both. The datasheet gives an overview on how measurement time,
- * accuracy and noise correlate.
- */
-static void bmp085_set_oversampling(struct bmp085_data *data,
-						unsigned char oversampling)
-{
-	if (oversampling > 3)
-		oversampling = 3;
-	data->oversampling_setting = oversampling;
-}
-
-/*
- * Returns the currently selected oversampling. Range: 0..3
- */
-static unsigned char bmp085_get_oversampling(struct bmp085_data *data)
-{
-	return data->oversampling_setting;
-}
-
-/* sysfs callbacks */
-static ssize_t set_oversampling(struct device *dev,
-				struct device_attribute *attr,
-				const char *buf, size_t count)
-{
-	struct bmp085_data *data = dev_get_drvdata(dev);
-	unsigned long oversampling;
-	int err = kstrtoul(buf, 10, &oversampling);
-
-	if (err == 0) {
-		mutex_lock(&data->lock);
-		bmp085_set_oversampling(data, oversampling);
-		mutex_unlock(&data->lock);
-		return count;
-	}
-
-	return err;
-}
-
-static ssize_t show_oversampling(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	struct bmp085_data *data = dev_get_drvdata(dev);
-
-	return sprintf(buf, "%u\n", bmp085_get_oversampling(data));
-}
-static DEVICE_ATTR(oversampling, S_IWUSR | S_IRUGO,
-					show_oversampling, set_oversampling);
-
-
-static ssize_t show_temperature(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	int temperature;
-	int status;
-	struct bmp085_data *data = dev_get_drvdata(dev);
-
-	status = bmp085_get_temperature(data, &temperature);
-	if (status < 0)
-		return status;
-	else
-		return sprintf(buf, "%d\n", temperature);
-}
-static DEVICE_ATTR(temp0_input, S_IRUGO, show_temperature, NULL);
-
-
-static ssize_t show_pressure(struct device *dev,
-			     struct device_attribute *attr, char *buf)
-{
-	int pressure;
-	int status;
-	struct bmp085_data *data = dev_get_drvdata(dev);
-
-	status = bmp085_get_pressure(data, &pressure);
-	if (status < 0)
-		return status;
-	else
-		return sprintf(buf, "%d\n", pressure);
-}
-static DEVICE_ATTR(pressure0_input, S_IRUGO, show_pressure, NULL);
-
-
-static struct attribute *bmp085_attributes[] = {
-	&dev_attr_temp0_input.attr,
-	&dev_attr_pressure0_input.attr,
-	&dev_attr_oversampling.attr,
-	NULL
-};
-
-static const struct attribute_group bmp085_attr_group = {
-	.attrs = bmp085_attributes,
-};
-
-int bmp085_detect(struct device *dev)
-{
-	struct bmp085_data *data = dev_get_drvdata(dev);
-	unsigned int id;
-	int ret;
-
-	ret = regmap_read(data->regmap, BMP085_CHIP_ID_REG, &id);
-	if (ret < 0)
-		return ret;
-
-	if (id != data->chip_id)
-		return -ENODEV;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(bmp085_detect);
-
-static void bmp085_get_of_properties(struct bmp085_data *data)
-{
-#ifdef CONFIG_OF
-	struct device_node *np = data->dev->of_node;
-	u32 prop;
-
-	if (!np)
-		return;
-
-	if (!of_property_read_u32(np, "chip-id", &prop))
-		data->chip_id = prop & 0xff;
-
-	if (!of_property_read_u32(np, "temp-measurement-period", &prop))
-		data->temp_measurement_period = (prop/100)*HZ;
-
-	if (!of_property_read_u32(np, "default-oversampling", &prop))
-		data->oversampling_setting = prop & 0xff;
-#endif
-}
-
-static int bmp085_init_client(struct bmp085_data *data)
-{
-	int status = bmp085_read_calibration_data(data);
-
-	if (status < 0)
-		return status;
-
-	/* default settings */
-	data->chip_id = BMP085_CHIP_ID;
-	data->last_temp_measurement = 0;
-	data->temp_measurement_period = 1*HZ;
-	data->oversampling_setting = 3;
-
-	bmp085_get_of_properties(data);
-
-	mutex_init(&data->lock);
-
-	return 0;
-}
-
-struct regmap_config bmp085_regmap_config = {
-	.reg_bits = 8,
-	.val_bits = 8
-};
-EXPORT_SYMBOL_GPL(bmp085_regmap_config);
-
-int bmp085_probe(struct device *dev, struct regmap *regmap, int irq)
-{
-	struct bmp085_data *data;
-	int err = 0;
-
-	data = kzalloc(sizeof(struct bmp085_data), GFP_KERNEL);
-	if (!data) {
-		err = -ENOMEM;
-		goto exit;
-	}
-
-	dev_set_drvdata(dev, data);
-	data->dev = dev;
-	data->regmap = regmap;
-	data->irq = irq;
-
-	if (data->irq > 0) {
-		err = devm_request_irq(dev, data->irq, bmp085_eoc_isr,
-					      IRQF_TRIGGER_RISING, "bmp085",
-					      data);
-		if (err < 0)
-			goto exit_free;
-	}
-
-	/* Initialize the BMP085 chip */
-	err = bmp085_init_client(data);
-	if (err < 0)
-		goto exit_free;
-
-	err = bmp085_detect(dev);
-	if (err < 0) {
-		dev_err(dev, "%s: chip_id failed!\n", BMP085_NAME);
-		goto exit_free;
-	}
-
-	/* Register sysfs hooks */
-	err = sysfs_create_group(&dev->kobj, &bmp085_attr_group);
-	if (err)
-		goto exit_free;
-
-	dev_info(dev, "Successfully initialized %s!\n", BMP085_NAME);
-
-	return 0;
-
-exit_free:
-	kfree(data);
-exit:
-	return err;
-}
-EXPORT_SYMBOL_GPL(bmp085_probe);
-
-int bmp085_remove(struct device *dev)
-{
-	struct bmp085_data *data = dev_get_drvdata(dev);
-
-	sysfs_remove_group(&data->dev->kobj, &bmp085_attr_group);
-	kfree(data);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(bmp085_remove);
-
-MODULE_AUTHOR("Christoph Mair <christoph.mair@gmail.com>");
-MODULE_DESCRIPTION("BMP085 driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/misc/bmp085.h b/drivers/misc/bmp085.h
deleted file mode 100644
index 8b8e3b1f5ca5..000000000000
--- a/drivers/misc/bmp085.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2012  Bosch Sensortec GmbH
- * Copyright (c) 2012  Unixphere AB
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#ifndef _BMP085_H
-#define _BMP085_H
-
-#include <linux/regmap.h>
-
-#define BMP085_NAME		"bmp085"
-
-extern struct regmap_config bmp085_regmap_config;
-
-int bmp085_probe(struct device *dev, struct regmap *regmap, int irq);
-int bmp085_remove(struct device *dev);
-int bmp085_detect(struct device *dev);
-
-#endif
diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
index bdee9a01ef35..c466ee2b0c97 100644
--- a/drivers/misc/cxl/context.c
+++ b/drivers/misc/cxl/context.c
@@ -90,8 +90,7 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master,
 	 */
 	mutex_lock(&afu->contexts_lock);
 	idr_preload(GFP_KERNEL);
-	i = idr_alloc(&ctx->afu->contexts_idr, ctx,
-		      ctx->afu->adapter->native->sl_ops->min_pe,
+	i = idr_alloc(&ctx->afu->contexts_idr, ctx, ctx->afu->adapter->min_pe,
 		      ctx->afu->num_procs, GFP_NOWAIT);
 	idr_preload_end();
 	mutex_unlock(&afu->contexts_lock);
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index de090533f18c..01d372aba131 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -162,7 +162,10 @@ static const cxl_p2n_reg_t CXL_PSL_WED_An     = {0x0A0};
 #define CXL_PSL_SPAP_V    0x0000000000000001ULL
 
 /****** CXL_PSL_Control ****************************************************/
-#define CXL_PSL_Control_tb 0x0000000000000001ULL
+#define CXL_PSL_Control_tb              (0x1ull << (63-63))
+#define CXL_PSL_Control_Fr              (0x1ull << (63-31))
+#define CXL_PSL_Control_Fs_MASK         (0x3ull << (63-29))
+#define CXL_PSL_Control_Fs_Complete     (0x3ull << (63-29))
 
 /****** CXL_PSL_DLCNTL *****************************************************/
 #define CXL_PSL_DLCNTL_D (0x1ull << (63-28))
@@ -561,7 +564,6 @@ struct cxl_service_layer_ops {
 	u64 (*timebase_read)(struct cxl *adapter);
 	int capi_mode;
 	bool needs_reset_before_disable;
-	int min_pe;
 };
 
 struct cxl_native {
@@ -603,6 +605,7 @@ struct cxl {
 	struct bin_attribute cxl_attr;
 	int adapter_num;
 	int user_irqs;
+	int min_pe;
 	u64 ps_size;
 	u16 psl_rev;
 	u16 base_image;
@@ -854,6 +857,7 @@ int cxl_register_one_irq(struct cxl *adapter, irq_handler_t handler,
 int cxl_check_error(struct cxl_afu *afu);
 int cxl_afu_slbia(struct cxl_afu *afu);
 int cxl_tlb_slb_invalidate(struct cxl *adapter);
+int cxl_data_cache_flush(struct cxl *adapter);
 int cxl_afu_disable(struct cxl_afu *afu);
 int cxl_psl_purge(struct cxl_afu *afu);
 
diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c
index 3bcdaee11ba1..a217a74ccc98 100644
--- a/drivers/misc/cxl/native.c
+++ b/drivers/misc/cxl/native.c
@@ -290,6 +290,37 @@ int cxl_tlb_slb_invalidate(struct cxl *adapter)
 	return 0;
 }
 
+int cxl_data_cache_flush(struct cxl *adapter)
+{
+	u64 reg;
+	unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
+
+	pr_devel("Flushing data cache\n");
+
+	reg = cxl_p1_read(adapter, CXL_PSL_Control);
+	reg |= CXL_PSL_Control_Fr;
+	cxl_p1_write(adapter, CXL_PSL_Control, reg);
+
+	reg = cxl_p1_read(adapter, CXL_PSL_Control);
+	while ((reg & CXL_PSL_Control_Fs_MASK) != CXL_PSL_Control_Fs_Complete) {
+		if (time_after_eq(jiffies, timeout)) {
+			dev_warn(&adapter->dev, "WARNING: cache flush timed out!\n");
+			return -EBUSY;
+		}
+
+		if (!cxl_ops->link_ok(adapter, NULL)) {
+			dev_warn(&adapter->dev, "WARNING: link down when flushing cache\n");
+			return -EIO;
+		}
+		cpu_relax();
+		reg = cxl_p1_read(adapter, CXL_PSL_Control);
+	}
+
+	reg &= ~CXL_PSL_Control_Fr;
+	cxl_p1_write(adapter, CXL_PSL_Control, reg);
+	return 0;
+}
+
 static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
 {
 	int rc;
@@ -924,7 +955,7 @@ static irqreturn_t native_irq_multiplexed(int irq, void *data)
 	return fail_psl_irq(afu, &irq_info);
 }
 
-void native_irq_wait(struct cxl_context *ctx)
+static void native_irq_wait(struct cxl_context *ctx)
 {
 	u64 dsisr;
 	int timeout = 1000;
diff --git a/drivers/misc/cxl/of.c b/drivers/misc/cxl/of.c
index edc458395f68..ec175ea5dfba 100644
--- a/drivers/misc/cxl/of.c
+++ b/drivers/misc/cxl/of.c
@@ -460,7 +460,7 @@ int cxl_of_probe(struct platform_device *pdev)
 	struct device_node *afu_np = NULL;
 	struct cxl *adapter = NULL;
 	int ret;
-	int slice, slice_ok;
+	int slice = 0, slice_ok = 0;
 
 	pr_devel("in %s\n", __func__);
 
@@ -476,13 +476,13 @@ int cxl_of_probe(struct platform_device *pdev)
 	}
 
 	/* init afu */
-	slice_ok = 0;
-	for (afu_np = NULL, slice = 0; (afu_np = of_get_next_child(np, afu_np)); slice++) {
+	for_each_child_of_node(np, afu_np) {
 		if ((ret = cxl_guest_init_afu(adapter, slice, afu_np)))
 			dev_err(&pdev->dev, "AFU %i failed to initialise: %i\n",
 				slice, ret);
 		else
 			slice_ok++;
+		slice++;
 	}
 
 	if (slice_ok == 0) {
@@ -490,8 +490,6 @@ int cxl_of_probe(struct platform_device *pdev)
 		adapter->slices = 0;
 	}
 
-	if (afu_np)
-		of_node_put(afu_np);
 	return 0;
 }
 
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index d152e2de8c93..7afad8477ad5 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -379,7 +379,7 @@ static int calc_capp_routing(struct pci_dev *dev, u64 *chipid, u64 *capp_unit_id
 
 static int init_implementation_adapter_psl_regs(struct cxl *adapter, struct pci_dev *dev)
 {
-	u64 psl_dsnctl;
+	u64 psl_dsnctl, psl_fircntl;
 	u64 chipid;
 	u64 capp_unit_id;
 	int rc;
@@ -398,8 +398,11 @@ static int init_implementation_adapter_psl_regs(struct cxl *adapter, struct pci_
 	cxl_p1_write(adapter, CXL_PSL_RESLCKTO, 0x20000000200ULL);
 	/* snoop write mask */
 	cxl_p1_write(adapter, CXL_PSL_SNWRALLOC, 0x00000000FFFFFFFFULL);
-	/* set fir_accum */
-	cxl_p1_write(adapter, CXL_PSL_FIR_CNTL, 0x0800000000000000ULL);
+	/* set fir_cntl to recommended value for production env */
+	psl_fircntl = (0x2ULL << (63-3)); /* ce_report */
+	psl_fircntl |= (0x1ULL << (63-6)); /* FIR_report */
+	psl_fircntl |= 0x1ULL; /* ce_thresh */
+	cxl_p1_write(adapter, CXL_PSL_FIR_CNTL, psl_fircntl);
 	/* for debugging with trace arrays */
 	cxl_p1_write(adapter, CXL_PSL_TRACE, 0x0000FF7C00000000ULL);
 
@@ -1236,6 +1239,9 @@ int cxl_pci_reset(struct cxl *adapter)
 
 	dev_info(&dev->dev, "CXL reset\n");
 
+	/* the adapter is about to be reset, so ignore errors */
+	cxl_data_cache_flush(adapter);
+
 	/* pcie_warm_reset requests a fundamental pci reset which includes a
 	 * PERST assert/deassert.  PERST triggers a loading of the image
 	 * if "user" or "factory" is selected in sysfs */
@@ -1521,16 +1527,17 @@ static const struct cxl_service_layer_ops xsl_ops = {
 	.write_timebase_ctrl = write_timebase_ctrl_xsl,
 	.timebase_read = timebase_read_xsl,
 	.capi_mode = OPAL_PHB_CAPI_MODE_DMA,
-	.min_pe = 1, /* Workaround for Mellanox CX4 HW bug */
 };
 
 static void set_sl_ops(struct cxl *adapter, struct pci_dev *dev)
 {
 	if (dev->vendor == PCI_VENDOR_ID_MELLANOX && dev->device == 0x1013) {
-		dev_info(&adapter->dev, "Device uses an XSL\n");
+		/* Mellanox CX-4 */
+		dev_info(&dev->dev, "Device uses an XSL\n");
 		adapter->native->sl_ops = &xsl_ops;
+		adapter->min_pe = 1; /* Workaround for CX-4 hardware bug */
 	} else {
-		dev_info(&adapter->dev, "Device uses a PSL\n");
+		dev_info(&dev->dev, "Device uses a PSL\n");
 		adapter->native->sl_ops = &psl_ops;
 	}
 }
diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
index dee8def1c193..3519acebfdab 100644
--- a/drivers/misc/cxl/vphb.c
+++ b/drivers/misc/cxl/vphb.c
@@ -221,7 +221,7 @@ int cxl_pci_vphb_add(struct cxl_afu *afu)
 	/* Setup the PHB using arch provided callback */
 	phb->ops = &cxl_pcie_pci_ops;
 	phb->cfg_addr = NULL;
-	phb->cfg_data = 0;
+	phb->cfg_data = NULL;
 	phb->private_data = afu;
 	phb->controller_ops = cxl_pci_controller_ops;
 
@@ -230,6 +230,11 @@ int cxl_pci_vphb_add(struct cxl_afu *afu)
 	if (phb->bus == NULL)
 		return -ENXIO;
 
+	/* Set release hook on root bus */
+	pci_set_host_bridge_release(to_pci_host_bridge(phb->bus->bridge),
+				    pcibios_free_controller_deferred,
+				    (void *) phb);
+
 	/* Claim resources. This might need some rework as well depending
 	 * whether we are doing probe-only or not, like assigning unassigned
 	 * resources etc...
@@ -256,7 +261,10 @@ void cxl_pci_vphb_remove(struct cxl_afu *afu)
 	afu->phb = NULL;
 
 	pci_remove_root_bus(phb->bus);
-	pcibios_free_controller(phb);
+	/*
+	 * We don't free phb here - that's handled by
+	 * pcibios_free_controller_deferred()
+	 */
 }
 
 static bool _cxl_pci_is_vphb_device(struct pci_controller *phb)
diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c
index 3cdf8e1ca0ad..051b14766ef9 100644
--- a/drivers/misc/eeprom/at24.c
+++ b/drivers/misc/eeprom/at24.c
@@ -593,6 +593,7 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	struct at24_data *at24;
 	int err;
 	unsigned i, num_addresses;
+	u8 test_byte;
 
 	if (client->dev.platform_data) {
 		chip = *(struct at24_platform_data *)client->dev.platform_data;
@@ -743,6 +744,18 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		}
 	}
 
+	i2c_set_clientdata(client, at24);
+
+	/*
+	 * Perform a one-byte test read to verify that the
+	 * chip is functional.
+	 */
+	err = at24_read(at24, 0, &test_byte, 1);
+	if (err) {
+		err = -ENODEV;
+		goto err_clients;
+	}
+
 	at24->nvmem_config.name = dev_name(&client->dev);
 	at24->nvmem_config.dev = &client->dev;
 	at24->nvmem_config.read_only = !writable;
@@ -764,8 +777,6 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		goto err_clients;
 	}
 
-	i2c_set_clientdata(client, at24);
-
 	dev_info(&client->dev, "%u byte %s EEPROM, %s, %u bytes/write\n",
 		chip.byte_len, client->name,
 		writable ? "writable" : "read-only", at24->write_max);
diff --git a/drivers/misc/eeprom/at25.c b/drivers/misc/eeprom/at25.c
index 2c6c7c8e3ead..5afe4cd16569 100644
--- a/drivers/misc/eeprom/at25.c
+++ b/drivers/misc/eeprom/at25.c
@@ -121,9 +121,8 @@ static int at25_ee_read(void *priv, unsigned int offset,
 	 * this chip is clocked very slowly
 	 */
 	status = spi_sync(at25->spi, &m);
-	dev_dbg(&at25->spi->dev,
-		"read %Zd bytes at %d --> %d\n",
-		count, offset, (int) status);
+	dev_dbg(&at25->spi->dev, "read %zu bytes at %d --> %zd\n",
+		count, offset, status);
 
 	mutex_unlock(&at25->lock);
 	return status;
@@ -167,8 +166,7 @@ static int at25_ee_write(void *priv, unsigned int off, void *val, size_t count)
 		*cp = AT25_WREN;
 		status = spi_write(at25->spi, cp, 1);
 		if (status < 0) {
-			dev_dbg(&at25->spi->dev, "WREN --> %d\n",
-					(int) status);
+			dev_dbg(&at25->spi->dev, "WREN --> %d\n", status);
 			break;
 		}
 
@@ -196,9 +194,8 @@ static int at25_ee_write(void *priv, unsigned int off, void *val, size_t count)
 		memcpy(cp, buf, segment);
 		status = spi_write(at25->spi, bounce,
 				segment + at25->addrlen + 1);
-		dev_dbg(&at25->spi->dev,
-				"write %u bytes at %u --> %d\n",
-				segment, offset, (int) status);
+		dev_dbg(&at25->spi->dev, "write %u bytes at %u --> %d\n",
+			segment, offset, status);
 		if (status < 0)
 			break;
 
@@ -225,8 +222,7 @@ static int at25_ee_write(void *priv, unsigned int off, void *val, size_t count)
 
 		if ((sr < 0) || (sr & AT25_SR_nRDY)) {
 			dev_err(&at25->spi->dev,
-				"write %d bytes offset %d, "
-				"timeout after %u msecs\n",
+				"write %u bytes offset %u, timeout after %u msecs\n",
 				segment, offset,
 				jiffies_to_msecs(jiffies -
 					(timeout - EE_TIMEOUT)));
@@ -368,9 +364,7 @@ static int at25_probe(struct spi_device *spi)
 		return PTR_ERR(at25->nvmem);
 
 	dev_info(&spi->dev, "%d %s %s eeprom%s, pagesize %u\n",
-		(chip.byte_len < 1024)
-			? chip.byte_len
-			: (chip.byte_len / 1024),
+		(chip.byte_len < 1024) ? chip.byte_len : (chip.byte_len / 1024),
 		(chip.byte_len < 1024) ? "Byte" : "KByte",
 		at25->chip.name,
 		(chip.flags & EE_READONLY) ? " (readonly)" : "",
diff --git a/drivers/misc/genwqe/card_base.c b/drivers/misc/genwqe/card_base.c
index a70b853fa2c9..6c1f49a85023 100644
--- a/drivers/misc/genwqe/card_base.c
+++ b/drivers/misc/genwqe/card_base.c
@@ -1351,6 +1351,19 @@ static struct pci_driver genwqe_driver = {
 };
 
 /**
+ * genwqe_devnode() - Set default access mode for genwqe devices.
+ *
+ * Default mode should be rw for everybody. Do not change default
+ * device name.
+ */
+static char *genwqe_devnode(struct device *dev, umode_t *mode)
+{
+	if (mode)
+		*mode = 0666;
+	return NULL;
+}
+
+/**
  * genwqe_init_module() - Driver registration and initialization
  */
 static int __init genwqe_init_module(void)
@@ -1363,6 +1376,8 @@ static int __init genwqe_init_module(void)
 		return -ENOMEM;
 	}
 
+	class_genwqe->devnode = genwqe_devnode;
+
 	debugfs_genwqe = debugfs_create_dir(GENWQE_DEVNAME, NULL);
 	if (!debugfs_genwqe) {
 		rc = -ENOMEM;
diff --git a/drivers/misc/genwqe/card_ddcb.c b/drivers/misc/genwqe/card_ddcb.c
index 353ee0cc733d..ddfeefe39540 100644
--- a/drivers/misc/genwqe/card_ddcb.c
+++ b/drivers/misc/genwqe/card_ddcb.c
@@ -1048,8 +1048,6 @@ static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
 			"[%s] **err: could not allocate DDCB **\n", __func__);
 		return -ENOMEM;
 	}
-	memset(queue->ddcb_vaddr, 0, queue_size);
-
 	queue->ddcb_req = kzalloc(sizeof(struct ddcb_requ *) *
 				  queue->ddcb_max, GFP_KERNEL);
 	if (!queue->ddcb_req) {
diff --git a/drivers/misc/genwqe/card_utils.c b/drivers/misc/genwqe/card_utils.c
index 222367cc8c81..8a679ecc8fd1 100644
--- a/drivers/misc/genwqe/card_utils.c
+++ b/drivers/misc/genwqe/card_utils.c
@@ -220,8 +220,8 @@ void *__genwqe_alloc_consistent(struct genwqe_dev *cd, size_t size,
 	if (get_order(size) > MAX_ORDER)
 		return NULL;
 
-	return dma_alloc_coherent(&cd->pci_dev->dev, size, dma_handle,
-				  GFP_KERNEL);
+	return dma_zalloc_coherent(&cd->pci_dev->dev, size, dma_handle,
+				   GFP_KERNEL);
 }
 
 void __genwqe_free_consistent(struct genwqe_dev *cd, size_t size,
diff --git a/drivers/misc/hpilo.c b/drivers/misc/hpilo.c
index d6a901cd4222..fea8ff40440f 100644
--- a/drivers/misc/hpilo.c
+++ b/drivers/misc/hpilo.c
@@ -688,7 +688,8 @@ static void ilo_unmap_device(struct pci_dev *pdev, struct ilo_hwinfo *hw)
 
 static int ilo_map_device(struct pci_dev *pdev, struct ilo_hwinfo *hw)
 {
-	int error = -ENOMEM;
+	int bar;
+	unsigned long off;
 
 	/* map the memory mapped i/o registers */
 	hw->mmio_vaddr = pci_iomap(pdev, 1, 0);
@@ -698,7 +699,15 @@ static int ilo_map_device(struct pci_dev *pdev, struct ilo_hwinfo *hw)
 	}
 
 	/* map the adapter shared memory region */
-	hw->ram_vaddr = pci_iomap(pdev, 2, max_ccb * ILOHW_CCB_SZ);
+	if (pdev->subsystem_device == 0x00E4) {
+		bar = 5;
+		/* Last 8k is reserved for CCBs */
+		off = pci_resource_len(pdev, bar) - 0x2000;
+	} else {
+		bar = 2;
+		off = 0;
+	}
+	hw->ram_vaddr = pci_iomap_range(pdev, bar, off, max_ccb * ILOHW_CCB_SZ);
 	if (hw->ram_vaddr == NULL) {
 		dev_err(&pdev->dev, "Error mapping shared mem\n");
 		goto mmio_free;
@@ -717,7 +726,7 @@ ram_free:
 mmio_free:
 	pci_iounmap(pdev, hw->mmio_vaddr);
 out:
-	return error;
+	return -ENOMEM;
 }
 
 static void ilo_remove(struct pci_dev *pdev)
@@ -899,7 +908,7 @@ static void __exit ilo_exit(void)
 	class_destroy(ilo_class);
 }
 
-MODULE_VERSION("1.4.1");
+MODULE_VERSION("1.5.0");
 MODULE_ALIAS(ILO_NAME);
 MODULE_DESCRIPTION(ILO_NAME);
 MODULE_AUTHOR("David Altobelli <david.altobelli@hpe.com>");
diff --git a/drivers/misc/ibmasm/ibmasmfs.c b/drivers/misc/ibmasm/ibmasmfs.c
index 9c677f3f3c26..520f58439080 100644
--- a/drivers/misc/ibmasm/ibmasmfs.c
+++ b/drivers/misc/ibmasm/ibmasmfs.c
@@ -144,7 +144,7 @@ static struct inode *ibmasmfs_make_inode(struct super_block *sb, int mode)
 	if (ret) {
 		ret->i_ino = get_next_ino();
 		ret->i_mode = mode;
-		ret->i_atime = ret->i_mtime = ret->i_ctime = CURRENT_TIME;
+		ret->i_atime = ret->i_mtime = ret->i_ctime = current_time(ret);
 	}
 	return ret;
 }
diff --git a/drivers/misc/lkdtm_rodata.c b/drivers/misc/lkdtm_rodata.c
index 166b1db3969f..3564477b8c2d 100644
--- a/drivers/misc/lkdtm_rodata.c
+++ b/drivers/misc/lkdtm_rodata.c
@@ -4,7 +4,7 @@
  */
 #include "lkdtm.h"
 
-void lkdtm_rodata_do_nothing(void)
+void notrace lkdtm_rodata_do_nothing(void)
 {
 	/* Does nothing. We just want an architecture agnostic "return". */
 }
diff --git a/drivers/misc/lkdtm_usercopy.c b/drivers/misc/lkdtm_usercopy.c
index 5a3fd76eec27..1dd611423d8b 100644
--- a/drivers/misc/lkdtm_usercopy.c
+++ b/drivers/misc/lkdtm_usercopy.c
@@ -9,7 +9,15 @@
 #include <linux/uaccess.h>
 #include <asm/cacheflush.h>
 
-static size_t cache_size = 1024;
+/*
+ * Many of the tests here end up using const sizes, but those would
+ * normally be ignored by hardened usercopy, so force the compiler
+ * into choosing the non-const path to make sure we trigger the
+ * hardened usercopy checks by added "unconst" to all the const copies,
+ * and making sure "cache_size" isn't optimized into a const.
+ */
+static volatile size_t unconst = 0;
+static volatile size_t cache_size = 1024;
 static struct kmem_cache *bad_cache;
 
 static const unsigned char test_text[] = "This is a test.\n";
@@ -49,7 +57,7 @@ static noinline void do_usercopy_stack(bool to_user, bool bad_frame)
 
 	/* This is a pointer to outside our current stack frame. */
 	if (bad_frame) {
-		bad_stack = do_usercopy_stack_callee((uintptr_t)bad_stack);
+		bad_stack = do_usercopy_stack_callee((uintptr_t)&bad_stack);
 	} else {
 		/* Put start address just inside stack. */
 		bad_stack = task_stack_page(current) + THREAD_SIZE;
@@ -67,14 +75,14 @@ static noinline void do_usercopy_stack(bool to_user, bool bad_frame)
 	if (to_user) {
 		pr_info("attempting good copy_to_user of local stack\n");
 		if (copy_to_user((void __user *)user_addr, good_stack,
-				 sizeof(good_stack))) {
+				 unconst + sizeof(good_stack))) {
 			pr_warn("copy_to_user failed unexpectedly?!\n");
 			goto free_user;
 		}
 
 		pr_info("attempting bad copy_to_user of distant stack\n");
 		if (copy_to_user((void __user *)user_addr, bad_stack,
-				 sizeof(good_stack))) {
+				 unconst + sizeof(good_stack))) {
 			pr_warn("copy_to_user failed, but lacked Oops\n");
 			goto free_user;
 		}
@@ -88,14 +96,14 @@ static noinline void do_usercopy_stack(bool to_user, bool bad_frame)
 
 		pr_info("attempting good copy_from_user of local stack\n");
 		if (copy_from_user(good_stack, (void __user *)user_addr,
-				   sizeof(good_stack))) {
+				   unconst + sizeof(good_stack))) {
 			pr_warn("copy_from_user failed unexpectedly?!\n");
 			goto free_user;
 		}
 
 		pr_info("attempting bad copy_from_user of distant stack\n");
 		if (copy_from_user(bad_stack, (void __user *)user_addr,
-				   sizeof(good_stack))) {
+				   unconst + sizeof(good_stack))) {
 			pr_warn("copy_from_user failed, but lacked Oops\n");
 			goto free_user;
 		}
@@ -109,7 +117,7 @@ static void do_usercopy_heap_size(bool to_user)
 {
 	unsigned long user_addr;
 	unsigned char *one, *two;
-	const size_t size = 1024;
+	size_t size = unconst + 1024;
 
 	one = kmalloc(size, GFP_KERNEL);
 	two = kmalloc(size, GFP_KERNEL);
@@ -285,13 +293,14 @@ void lkdtm_USERCOPY_KERNEL(void)
 
 	pr_info("attempting good copy_to_user from kernel rodata\n");
 	if (copy_to_user((void __user *)user_addr, test_text,
-			 sizeof(test_text))) {
+			 unconst + sizeof(test_text))) {
 		pr_warn("copy_to_user failed unexpectedly?!\n");
 		goto free_user;
 	}
 
 	pr_info("attempting bad copy_to_user from kernel text\n");
-	if (copy_to_user((void __user *)user_addr, vm_mmap, PAGE_SIZE)) {
+	if (copy_to_user((void __user *)user_addr, vm_mmap,
+			 unconst + PAGE_SIZE)) {
 		pr_warn("copy_to_user failed, but lacked Oops\n");
 		goto free_user;
 	}
diff --git a/drivers/misc/mei/amthif.c b/drivers/misc/mei/amthif.c
index a039a5df6f21..7ae89b4a21d5 100644
--- a/drivers/misc/mei/amthif.c
+++ b/drivers/misc/mei/amthif.c
@@ -47,7 +47,6 @@ const uuid_le mei_amthif_guid  = UUID_LE(0x12f80028, 0xb4b7, 0x4b2d,
 void mei_amthif_reset_params(struct mei_device *dev)
 {
 	/* reset iamthif parameters. */
-	dev->iamthif_current_cb = NULL;
 	dev->iamthif_canceled = false;
 	dev->iamthif_state = MEI_IAMTHIF_IDLE;
 	dev->iamthif_stall_timer = 0;
@@ -67,8 +66,12 @@ int mei_amthif_host_init(struct mei_device *dev, struct mei_me_client *me_cl)
 	struct mei_cl *cl = &dev->iamthif_cl;
 	int ret;
 
-	if (mei_cl_is_connected(cl))
-		return 0;
+	mutex_lock(&dev->device_lock);
+
+	if (mei_cl_is_connected(cl)) {
+		ret = 0;
+		goto out;
+	}
 
 	dev->iamthif_state = MEI_IAMTHIF_IDLE;
 
@@ -77,179 +80,37 @@ int mei_amthif_host_init(struct mei_device *dev, struct mei_me_client *me_cl)
 	ret = mei_cl_link(cl);
 	if (ret < 0) {
 		dev_err(dev->dev, "amthif: failed cl_link %d\n", ret);
-		return ret;
+		goto out;
 	}
 
 	ret = mei_cl_connect(cl, me_cl, NULL);
 
-	return ret;
-}
-
-/**
- * mei_amthif_read - read data from AMTHIF client
- *
- * @dev: the device structure
- * @file: pointer to file object
- * @ubuf: pointer to user data in user space
- * @length: data length to read
- * @offset: data read offset
- *
- * Locking: called under "dev->device_lock" lock
- *
- * Return:
- *  returned data length on success,
- *  zero if no data to read,
- *  negative on failure.
- */
-int mei_amthif_read(struct mei_device *dev, struct file *file,
-	       char __user *ubuf, size_t length, loff_t *offset)
-{
-	struct mei_cl *cl = file->private_data;
-	struct mei_cl_cb *cb;
-	int rets;
-	int wait_ret;
-
-	dev_dbg(dev->dev, "checking amthif data\n");
-	cb = mei_cl_read_cb(cl, file);
-
-	/* Check for if we can block or not*/
-	if (cb == NULL && file->f_flags & O_NONBLOCK)
-		return -EAGAIN;
-
-
-	dev_dbg(dev->dev, "waiting for amthif data\n");
-	while (cb == NULL) {
-		/* unlock the Mutex */
-		mutex_unlock(&dev->device_lock);
-
-		wait_ret = wait_event_interruptible(cl->rx_wait,
-					!list_empty(&cl->rd_completed) ||
-					!mei_cl_is_connected(cl));
-
-		/* Locking again the Mutex */
-		mutex_lock(&dev->device_lock);
-
-		if (wait_ret)
-			return -ERESTARTSYS;
-
-		if (!mei_cl_is_connected(cl)) {
-			rets = -EBUSY;
-			goto out;
-		}
-
-		cb = mei_cl_read_cb(cl, file);
-	}
-
-	if (cb->status) {
-		rets = cb->status;
-		dev_dbg(dev->dev, "read operation failed %d\n", rets);
-		goto free;
-	}
-
-	dev_dbg(dev->dev, "Got amthif data\n");
-	/* if the whole message will fit remove it from the list */
-	if (cb->buf_idx >= *offset && length >= (cb->buf_idx - *offset))
-		list_del_init(&cb->list);
-	else if (cb->buf_idx <= *offset) {
-		/* end of the message has been reached */
-		list_del_init(&cb->list);
-		rets = 0;
-		goto free;
-	}
-		/* else means that not full buffer will be read and do not
-		 * remove message from deletion list
-		 */
-
-	dev_dbg(dev->dev, "amthif cb->buf.size - %zu cb->buf_idx - %zu\n",
-		cb->buf.size, cb->buf_idx);
-
-	/* length is being truncated to PAGE_SIZE, however,
-	 * the buf_idx may point beyond */
-	length = min_t(size_t, length, (cb->buf_idx - *offset));
-
-	if (copy_to_user(ubuf, cb->buf.data + *offset, length)) {
-		dev_dbg(dev->dev, "failed to copy data to userland\n");
-		rets = -EFAULT;
-	} else {
-		rets = length;
-		if ((*offset + length) < cb->buf_idx) {
-			*offset += length;
-			goto out;
-		}
-	}
-free:
-	dev_dbg(dev->dev, "free amthif cb memory.\n");
-	*offset = 0;
-	mei_io_cb_free(cb);
 out:
-	return rets;
+	mutex_unlock(&dev->device_lock);
+	return ret;
 }
 
 /**
  * mei_amthif_read_start - queue message for sending read credential
  *
  * @cl: host client
- * @file: file pointer of message recipient
+ * @fp: file pointer of message recipient
  *
  * Return: 0 on success, <0 on failure.
  */
-static int mei_amthif_read_start(struct mei_cl *cl, const struct file *file)
+static int mei_amthif_read_start(struct mei_cl *cl, const struct file *fp)
 {
 	struct mei_device *dev = cl->dev;
 	struct mei_cl_cb *cb;
-	int rets;
-
-	cb = mei_io_cb_init(cl, MEI_FOP_READ, file);
-	if (!cb) {
-		rets = -ENOMEM;
-		goto err;
-	}
 
-	rets = mei_io_cb_alloc_buf(cb, mei_cl_mtu(cl));
-	if (rets)
-		goto err;
+	cb = mei_cl_enqueue_ctrl_wr_cb(cl, mei_cl_mtu(cl), MEI_FOP_READ, fp);
+	if (!cb)
+		return -ENOMEM;
 
-	list_add_tail(&cb->list, &dev->ctrl_wr_list.list);
+	cl->rx_flow_ctrl_creds++;
 
 	dev->iamthif_state = MEI_IAMTHIF_READING;
-	dev->iamthif_fp = cb->fp;
-	dev->iamthif_current_cb = cb;
-
-	return 0;
-err:
-	mei_io_cb_free(cb);
-	return rets;
-}
-
-/**
- * mei_amthif_send_cmd - send amthif command to the ME
- *
- * @cl: the host client
- * @cb: mei call back struct
- *
- * Return: 0 on success, <0 on failure.
- */
-static int mei_amthif_send_cmd(struct mei_cl *cl, struct mei_cl_cb *cb)
-{
-	struct mei_device *dev;
-	int ret;
-
-	if (!cl->dev || !cb)
-		return -ENODEV;
-
-	dev = cl->dev;
-
-	dev->iamthif_state = MEI_IAMTHIF_WRITING;
-	dev->iamthif_current_cb = cb;
-	dev->iamthif_fp = cb->fp;
-	dev->iamthif_canceled = false;
-
-	ret = mei_cl_write(cl, cb, false);
-	if (ret < 0)
-		return ret;
-
-	if (cb->completed)
-		cb->status = mei_amthif_read_start(cl, cb->fp);
+	cl->fp = cb->fp;
 
 	return 0;
 }
@@ -265,20 +126,32 @@ int mei_amthif_run_next_cmd(struct mei_device *dev)
 {
 	struct mei_cl *cl = &dev->iamthif_cl;
 	struct mei_cl_cb *cb;
+	int ret;
 
 	dev->iamthif_canceled = false;
-	dev->iamthif_state = MEI_IAMTHIF_IDLE;
-	dev->iamthif_fp = NULL;
 
 	dev_dbg(dev->dev, "complete amthif cmd_list cb.\n");
 
 	cb = list_first_entry_or_null(&dev->amthif_cmd_list.list,
 					typeof(*cb), list);
-	if (!cb)
+	if (!cb) {
+		dev->iamthif_state = MEI_IAMTHIF_IDLE;
+		cl->fp = NULL;
 		return 0;
+	}
 
 	list_del_init(&cb->list);
-	return mei_amthif_send_cmd(cl, cb);
+	dev->iamthif_state = MEI_IAMTHIF_WRITING;
+	cl->fp = cb->fp;
+
+	ret = mei_cl_write(cl, cb, false);
+	if (ret < 0)
+		return ret;
+
+	if (cb->completed)
+		cb->status = mei_amthif_read_start(cl, cb->fp);
+
+	return 0;
 }
 
 /**
@@ -299,8 +172,7 @@ int mei_amthif_write(struct mei_cl *cl, struct mei_cl_cb *cb)
 	/*
 	 * The previous request is still in processing, queue this one.
 	 */
-	if (dev->iamthif_state > MEI_IAMTHIF_IDLE &&
-	    dev->iamthif_state < MEI_IAMTHIF_READ_COMPLETE)
+	if (dev->iamthif_state != MEI_IAMTHIF_IDLE)
 		return 0;
 
 	return mei_amthif_run_next_cmd(dev);
@@ -309,7 +181,6 @@ int mei_amthif_write(struct mei_cl *cl, struct mei_cl_cb *cb)
 /**
  * mei_amthif_poll - the amthif poll function
  *
- * @dev: the device structure
  * @file: pointer to file structure
  * @wait: pointer to poll_table structure
  *
@@ -317,26 +188,19 @@ int mei_amthif_write(struct mei_cl *cl, struct mei_cl_cb *cb)
  *
  * Locking: called under "dev->device_lock" lock
  */
-
-unsigned int mei_amthif_poll(struct mei_device *dev,
-		struct file *file, poll_table *wait)
+unsigned int mei_amthif_poll(struct file *file, poll_table *wait)
 {
+	struct mei_cl *cl = file->private_data;
+	struct mei_cl_cb *cb = mei_cl_read_cb(cl, file);
 	unsigned int mask = 0;
 
-	poll_wait(file, &dev->iamthif_cl.rx_wait, wait);
-
-	if (dev->iamthif_state == MEI_IAMTHIF_READ_COMPLETE &&
-	    dev->iamthif_fp == file) {
-
+	poll_wait(file, &cl->rx_wait, wait);
+	if (cb)
 		mask |= POLLIN | POLLRDNORM;
-		mei_amthif_run_next_cmd(dev);
-	}
 
 	return mask;
 }
 
-
-
 /**
  * mei_amthif_irq_write - write iamthif command in irq thread context.
  *
@@ -393,7 +257,6 @@ int mei_amthif_irq_read_msg(struct mei_cl *cl,
 		return 0;
 
 	dev_dbg(dev->dev, "completed amthif read.\n ");
-	dev->iamthif_current_cb = NULL;
 	dev->iamthif_stall_timer = 0;
 
 	return 0;
@@ -409,115 +272,63 @@ void mei_amthif_complete(struct mei_cl *cl, struct mei_cl_cb *cb)
 {
 	struct mei_device *dev = cl->dev;
 
-	if (cb->fop_type == MEI_FOP_WRITE) {
+	dev_dbg(dev->dev, "completing amthif call back.\n");
+	switch (cb->fop_type) {
+	case MEI_FOP_WRITE:
 		if (!cb->status) {
 			dev->iamthif_stall_timer = MEI_IAMTHIF_STALL_TIMER;
+			mei_schedule_stall_timer(dev);
 			mei_io_cb_free(cb);
 			return;
 		}
-		/*
-		 * in case of error enqueue the write cb to complete read list
-		 * so it can be propagated to the reader
-		 */
-		list_add_tail(&cb->list, &cl->rd_completed);
-		wake_up_interruptible(&cl->rx_wait);
-		return;
-	}
+		dev->iamthif_state = MEI_IAMTHIF_IDLE;
+		cl->fp = NULL;
+		if (!dev->iamthif_canceled) {
+			/*
+			 * in case of error enqueue the write cb to complete
+			 * read list so it can be propagated to the reader
+			 */
+			list_add_tail(&cb->list, &cl->rd_completed);
+			wake_up_interruptible(&cl->rx_wait);
+		} else {
+			mei_io_cb_free(cb);
+		}
+		break;
+	case MEI_FOP_READ:
+		if (!dev->iamthif_canceled) {
+			list_add_tail(&cb->list, &cl->rd_completed);
+			dev_dbg(dev->dev, "amthif read completed\n");
+			wake_up_interruptible(&cl->rx_wait);
+		} else {
+			mei_io_cb_free(cb);
+		}
 
-	if (!dev->iamthif_canceled) {
-		dev->iamthif_state = MEI_IAMTHIF_READ_COMPLETE;
 		dev->iamthif_stall_timer = 0;
-		list_add_tail(&cb->list, &cl->rd_completed);
-		dev_dbg(dev->dev, "amthif read completed\n");
-	} else {
 		mei_amthif_run_next_cmd(dev);
+		break;
+	default:
+		WARN_ON(1);
 	}
-
-	dev_dbg(dev->dev, "completing amthif call back.\n");
-	wake_up_interruptible(&cl->rx_wait);
 }
 
 /**
  * mei_clear_list - removes all callbacks associated with file
  *		from mei_cb_list
  *
- * @dev: device structure.
  * @file: file structure
  * @mei_cb_list: callbacks list
  *
  * mei_clear_list is called to clear resources associated with file
  * when application calls close function or Ctrl-C was pressed
- *
- * Return: true if callback removed from the list, false otherwise
  */
-static bool mei_clear_list(struct mei_device *dev,
-		const struct file *file, struct list_head *mei_cb_list)
+static void mei_clear_list(const struct file *file,
+			   struct list_head *mei_cb_list)
 {
-	struct mei_cl *cl = &dev->iamthif_cl;
 	struct mei_cl_cb *cb, *next;
-	bool removed = false;
-
-	/* list all list member */
-	list_for_each_entry_safe(cb, next, mei_cb_list, list) {
-		/* check if list member associated with a file */
-		if (file == cb->fp) {
-			/* check if cb equal to current iamthif cb */
-			if (dev->iamthif_current_cb == cb) {
-				dev->iamthif_current_cb = NULL;
-				/* send flow control to iamthif client */
-				mei_hbm_cl_flow_control_req(dev, cl);
-			}
-			/* free all allocated buffers */
-			mei_io_cb_free(cb);
-			removed = true;
-		}
-	}
-	return removed;
-}
 
-/**
- * mei_clear_lists - removes all callbacks associated with file
- *
- * @dev: device structure
- * @file: file structure
- *
- * mei_clear_lists is called to clear resources associated with file
- * when application calls close function or Ctrl-C was pressed
- *
- * Return: true if callback removed from the list, false otherwise
- */
-static bool mei_clear_lists(struct mei_device *dev, const struct file *file)
-{
-	bool removed = false;
-	struct mei_cl *cl = &dev->iamthif_cl;
-
-	/* remove callbacks associated with a file */
-	mei_clear_list(dev, file, &dev->amthif_cmd_list.list);
-	if (mei_clear_list(dev, file, &cl->rd_completed))
-		removed = true;
-
-	mei_clear_list(dev, file, &dev->ctrl_rd_list.list);
-
-	if (mei_clear_list(dev, file, &dev->ctrl_wr_list.list))
-		removed = true;
-
-	if (mei_clear_list(dev, file, &dev->write_waiting_list.list))
-		removed = true;
-
-	if (mei_clear_list(dev, file, &dev->write_list.list))
-		removed = true;
-
-	/* check if iamthif_current_cb not NULL */
-	if (dev->iamthif_current_cb && !removed) {
-		/* check file and iamthif current cb association */
-		if (dev->iamthif_current_cb->fp == file) {
-			/* remove cb */
-			mei_io_cb_free(dev->iamthif_current_cb);
-			dev->iamthif_current_cb = NULL;
-			removed = true;
-		}
-	}
-	return removed;
+	list_for_each_entry_safe(cb, next, mei_cb_list, list)
+		if (file == cb->fp)
+			mei_io_cb_free(cb);
 }
 
 /**
@@ -530,23 +341,21 @@ static bool mei_clear_lists(struct mei_device *dev, const struct file *file)
 */
 int mei_amthif_release(struct mei_device *dev, struct file *file)
 {
+	struct mei_cl *cl = file->private_data;
+
 	if (dev->iamthif_open_count > 0)
 		dev->iamthif_open_count--;
 
-	if (dev->iamthif_fp == file &&
-	    dev->iamthif_state != MEI_IAMTHIF_IDLE) {
+	if (cl->fp == file && dev->iamthif_state != MEI_IAMTHIF_IDLE) {
 
 		dev_dbg(dev->dev, "amthif canceled iamthif state %d\n",
 		    dev->iamthif_state);
 		dev->iamthif_canceled = true;
-		if (dev->iamthif_state == MEI_IAMTHIF_READ_COMPLETE) {
-			dev_dbg(dev->dev, "run next amthif iamthif cb\n");
-			mei_amthif_run_next_cmd(dev);
-		}
 	}
 
-	if (mei_clear_lists(dev, file))
-		dev->iamthif_state = MEI_IAMTHIF_IDLE;
+	mei_clear_list(file, &dev->amthif_cmd_list.list);
+	mei_clear_list(file, &cl->rd_completed);
+	mei_clear_list(file, &dev->ctrl_rd_list.list);
 
 	return 0;
 }
diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c
index 1f33fea9299f..8cac7ef9ad0d 100644
--- a/drivers/misc/mei/bus.c
+++ b/drivers/misc/mei/bus.c
@@ -126,7 +126,8 @@ ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length)
 		goto out;
 
 	/* wait on event only if there is no other waiter */
-	if (list_empty(&cl->rd_completed) && !waitqueue_active(&cl->rx_wait)) {
+	/* synchronized under device mutex */
+	if (!waitqueue_active(&cl->rx_wait)) {
 
 		mutex_unlock(&bus->device_lock);
 
@@ -142,7 +143,7 @@ ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length)
 		mutex_lock(&bus->device_lock);
 
 		if (!mei_cl_is_connected(cl)) {
-			rets = -EBUSY;
+			rets = -ENODEV;
 			goto out;
 		}
 	}
@@ -234,7 +235,7 @@ static void mei_cl_bus_event_work(struct work_struct *work)
 	/* Prepare for the next read */
 	if (cldev->events_mask & BIT(MEI_CL_EVENT_RX)) {
 		mutex_lock(&bus->device_lock);
-		mei_cl_read_start(cldev->cl, 0, NULL);
+		mei_cl_read_start(cldev->cl, mei_cl_mtu(cldev->cl), NULL);
 		mutex_unlock(&bus->device_lock);
 	}
 }
@@ -324,7 +325,7 @@ int mei_cldev_register_event_cb(struct mei_cl_device *cldev,
 
 	if (cldev->events_mask & BIT(MEI_CL_EVENT_RX)) {
 		mutex_lock(&bus->device_lock);
-		ret = mei_cl_read_start(cldev->cl, 0, NULL);
+		ret = mei_cl_read_start(cldev->cl, mei_cl_mtu(cldev->cl), NULL);
 		mutex_unlock(&bus->device_lock);
 		if (ret && ret != -EBUSY)
 			return ret;
@@ -983,12 +984,10 @@ void mei_cl_bus_rescan_work(struct work_struct *work)
 		container_of(work, struct mei_device, bus_rescan_work);
 	struct mei_me_client *me_cl;
 
-	mutex_lock(&bus->device_lock);
 	me_cl = mei_me_cl_by_uuid(bus, &mei_amthif_guid);
 	if (me_cl)
 		mei_amthif_host_init(bus, me_cl);
 	mei_me_cl_put(me_cl);
-	mutex_unlock(&bus->device_lock);
 
 	mei_cl_bus_rescan(bus);
 }
diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c
index 641c1a566687..6fe02350578d 100644
--- a/drivers/misc/mei/client.c
+++ b/drivers/misc/mei/client.c
@@ -358,8 +358,9 @@ void mei_io_cb_free(struct mei_cl_cb *cb)
  *
  * Return: mei_cl_cb pointer or NULL;
  */
-struct mei_cl_cb *mei_io_cb_init(struct mei_cl *cl, enum mei_cb_file_ops type,
-				 const struct file *fp)
+static struct mei_cl_cb *mei_io_cb_init(struct mei_cl *cl,
+					enum mei_cb_file_ops type,
+					const struct file *fp)
 {
 	struct mei_cl_cb *cb;
 
@@ -420,32 +421,41 @@ static inline void mei_io_list_free(struct mei_cl_cb *list, struct mei_cl *cl)
 }
 
 /**
- * mei_io_cb_alloc_buf - allocate callback buffer
+ * mei_cl_alloc_cb - a convenient wrapper for allocating read cb
  *
- * @cb: io callback structure
+ * @cl: host client
  * @length: size of the buffer
+ * @type: operation type
+ * @fp: associated file pointer (might be NULL)
  *
- * Return: 0 on success
- *         -EINVAL if cb is NULL
- *         -ENOMEM if allocation failed
+ * Return: cb on success and NULL on failure
  */
-int mei_io_cb_alloc_buf(struct mei_cl_cb *cb, size_t length)
+struct mei_cl_cb *mei_cl_alloc_cb(struct mei_cl *cl, size_t length,
+				  enum mei_cb_file_ops fop_type,
+				  const struct file *fp)
 {
+	struct mei_cl_cb *cb;
+
+	cb = mei_io_cb_init(cl, fop_type, fp);
 	if (!cb)
-		return -EINVAL;
+		return NULL;
 
 	if (length == 0)
-		return 0;
+		return cb;
 
 	cb->buf.data = kmalloc(length, GFP_KERNEL);
-	if (!cb->buf.data)
-		return -ENOMEM;
+	if (!cb->buf.data) {
+		mei_io_cb_free(cb);
+		return NULL;
+	}
 	cb->buf.size = length;
-	return 0;
+
+	return cb;
 }
 
 /**
- * mei_cl_alloc_cb - a convenient wrapper for allocating read cb
+ * mei_cl_enqueue_ctrl_wr_cb - a convenient wrapper for allocating
+ *     and enqueuing of the control commands cb
  *
  * @cl: host client
  * @length: size of the buffer
@@ -453,22 +463,23 @@ int mei_io_cb_alloc_buf(struct mei_cl_cb *cb, size_t length)
  * @fp: associated file pointer (might be NULL)
  *
  * Return: cb on success and NULL on failure
+ * Locking: called under "dev->device_lock" lock
  */
-struct mei_cl_cb *mei_cl_alloc_cb(struct mei_cl *cl, size_t length,
-				  enum mei_cb_file_ops type,
-				  const struct file *fp)
+struct mei_cl_cb *mei_cl_enqueue_ctrl_wr_cb(struct mei_cl *cl, size_t length,
+					    enum mei_cb_file_ops fop_type,
+					    const struct file *fp)
 {
 	struct mei_cl_cb *cb;
 
-	cb = mei_io_cb_init(cl, type, fp);
-	if (!cb)
-		return NULL;
+	/* for RX always allocate at least client's mtu */
+	if (length)
+		length = max_t(size_t, length, mei_cl_mtu(cl));
 
-	if (mei_io_cb_alloc_buf(cb, length)) {
-		mei_io_cb_free(cb);
+	cb = mei_cl_alloc_cb(cl, length, fop_type, fp);
+	if (!cb)
 		return NULL;
-	}
 
+	list_add_tail(&cb->list, &cl->dev->ctrl_wr_list.list);
 	return cb;
 }
 
@@ -754,7 +765,8 @@ void mei_cl_set_disconnected(struct mei_cl *cl)
 	mei_io_list_flush(&dev->ctrl_rd_list, cl);
 	mei_io_list_flush(&dev->ctrl_wr_list, cl);
 	mei_cl_wake_all(cl);
-	cl->mei_flow_ctrl_creds = 0;
+	cl->rx_flow_ctrl_creds = 0;
+	cl->tx_flow_ctrl_creds = 0;
 	cl->timer_count = 0;
 
 	if (!cl->me_cl)
@@ -764,7 +776,7 @@ void mei_cl_set_disconnected(struct mei_cl *cl)
 		cl->me_cl->connect_count--;
 
 	if (cl->me_cl->connect_count == 0)
-		cl->me_cl->mei_flow_ctrl_creds = 0;
+		cl->me_cl->tx_flow_ctrl_creds = 0;
 
 	mei_me_cl_put(cl->me_cl);
 	cl->me_cl = NULL;
@@ -814,6 +826,7 @@ static int mei_cl_send_disconnect(struct mei_cl *cl, struct mei_cl_cb *cb)
 
 	list_move_tail(&cb->list, &dev->ctrl_rd_list.list);
 	cl->timer_count = MEI_CONNECT_TIMEOUT;
+	mei_schedule_stall_timer(dev);
 
 	return 0;
 }
@@ -867,13 +880,11 @@ static int __mei_cl_disconnect(struct mei_cl *cl)
 
 	cl->state = MEI_FILE_DISCONNECTING;
 
-	cb = mei_io_cb_init(cl, MEI_FOP_DISCONNECT, NULL);
-	rets = cb ? 0 : -ENOMEM;
-	if (rets)
+	cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_DISCONNECT, NULL);
+	if (!cb) {
+		rets = -ENOMEM;
 		goto out;
-
-	cl_dbg(dev, cl, "add disconnect cb to control write list\n");
-	list_add_tail(&cb->list, &dev->ctrl_wr_list.list);
+	}
 
 	if (mei_hbuf_acquire(dev)) {
 		rets = mei_cl_send_disconnect(cl, cb);
@@ -1001,6 +1012,7 @@ static int mei_cl_send_connect(struct mei_cl *cl, struct mei_cl_cb *cb)
 
 	list_move_tail(&cb->list, &dev->ctrl_rd_list.list);
 	cl->timer_count = MEI_CONNECT_TIMEOUT;
+	mei_schedule_stall_timer(dev);
 	return 0;
 }
 
@@ -1042,14 +1054,14 @@ int mei_cl_irq_connect(struct mei_cl *cl, struct mei_cl_cb *cb,
  *
  * @cl: host client
  * @me_cl: me client
- * @file: pointer to file structure
+ * @fp: pointer to file structure
  *
  * Locking: called under "dev->device_lock" lock
  *
  * Return: 0 on success, <0 on failure.
  */
 int mei_cl_connect(struct mei_cl *cl, struct mei_me_client *me_cl,
-		  const struct file *file)
+		   const struct file *fp)
 {
 	struct mei_device *dev;
 	struct mei_cl_cb *cb;
@@ -1076,12 +1088,11 @@ int mei_cl_connect(struct mei_cl *cl, struct mei_me_client *me_cl,
 		goto nortpm;
 	}
 
-	cb = mei_io_cb_init(cl, MEI_FOP_CONNECT, file);
-	rets = cb ? 0 : -ENOMEM;
-	if (rets)
+	cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_CONNECT, fp);
+	if (!cb) {
+		rets = -ENOMEM;
 		goto out;
-
-	list_add_tail(&cb->list, &dev->ctrl_wr_list.list);
+	}
 
 	/* run hbuf acquire last so we don't have to undo */
 	if (!mei_cl_is_other_connecting(cl) && mei_hbuf_acquire(dev)) {
@@ -1159,50 +1170,42 @@ err:
 	return ERR_PTR(ret);
 }
 
-
-
 /**
- * mei_cl_flow_ctrl_creds - checks flow_control credits for cl.
+ * mei_cl_tx_flow_ctrl_creds - checks flow_control credits for cl.
  *
  * @cl: host client
- * @fp: the file pointer associated with the pointer
  *
- * Return: 1 if mei_flow_ctrl_creds >0, 0 - otherwise.
+ * Return: 1 if tx_flow_ctrl_creds >0, 0 - otherwise.
  */
-static int mei_cl_flow_ctrl_creds(struct mei_cl *cl, const struct file *fp)
+static int mei_cl_tx_flow_ctrl_creds(struct mei_cl *cl)
 {
-	int rets;
-
 	if (WARN_ON(!cl || !cl->me_cl))
 		return -EINVAL;
 
-	if (cl->mei_flow_ctrl_creds > 0)
+	if (cl->tx_flow_ctrl_creds > 0)
 		return 1;
 
-	if (mei_cl_is_fixed_address(cl)) {
-		rets = mei_cl_read_start(cl, mei_cl_mtu(cl), fp);
-		if (rets && rets != -EBUSY)
-			return rets;
+	if (mei_cl_is_fixed_address(cl))
 		return 1;
-	}
 
 	if (mei_cl_is_single_recv_buf(cl)) {
-		if (cl->me_cl->mei_flow_ctrl_creds > 0)
+		if (cl->me_cl->tx_flow_ctrl_creds > 0)
 			return 1;
 	}
 	return 0;
 }
 
 /**
- * mei_cl_flow_ctrl_reduce - reduces flow_control.
+ * mei_cl_tx_flow_ctrl_creds_reduce - reduces transmit flow control credits
+ *   for a client
  *
- * @cl: private data of the file object
+ * @cl: host client
  *
  * Return:
  *	0 on success
  *	-EINVAL when ctrl credits are <= 0
  */
-static int mei_cl_flow_ctrl_reduce(struct mei_cl *cl)
+static int mei_cl_tx_flow_ctrl_creds_reduce(struct mei_cl *cl)
 {
 	if (WARN_ON(!cl || !cl->me_cl))
 		return -EINVAL;
@@ -1211,13 +1214,13 @@ static int mei_cl_flow_ctrl_reduce(struct mei_cl *cl)
 		return 0;
 
 	if (mei_cl_is_single_recv_buf(cl)) {
-		if (WARN_ON(cl->me_cl->mei_flow_ctrl_creds <= 0))
+		if (WARN_ON(cl->me_cl->tx_flow_ctrl_creds <= 0))
 			return -EINVAL;
-		cl->me_cl->mei_flow_ctrl_creds--;
+		cl->me_cl->tx_flow_ctrl_creds--;
 	} else {
-		if (WARN_ON(cl->mei_flow_ctrl_creds <= 0))
+		if (WARN_ON(cl->tx_flow_ctrl_creds <= 0))
 			return -EINVAL;
-		cl->mei_flow_ctrl_creds--;
+		cl->tx_flow_ctrl_creds--;
 	}
 	return 0;
 }
@@ -1292,7 +1295,7 @@ int mei_cl_irq_notify(struct mei_cl *cl, struct mei_cl_cb *cb,
  * mei_cl_notify_request - send notification stop/start request
  *
  * @cl: host client
- * @file: associate request with file
+ * @fp: associate request with file
  * @request: 1 for start or 0 for stop
  *
  * Locking: called under "dev->device_lock" lock
@@ -1300,7 +1303,7 @@ int mei_cl_irq_notify(struct mei_cl *cl, struct mei_cl_cb *cb,
  * Return: 0 on such and error otherwise.
  */
 int mei_cl_notify_request(struct mei_cl *cl,
-			  const struct file *file, u8 request)
+			  const struct file *fp, u8 request)
 {
 	struct mei_device *dev;
 	struct mei_cl_cb *cb;
@@ -1325,7 +1328,7 @@ int mei_cl_notify_request(struct mei_cl *cl,
 	}
 
 	fop_type = mei_cl_notify_req2fop(request);
-	cb = mei_io_cb_init(cl, fop_type, file);
+	cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, fop_type, fp);
 	if (!cb) {
 		rets = -ENOMEM;
 		goto out;
@@ -1336,9 +1339,7 @@ int mei_cl_notify_request(struct mei_cl *cl,
 			rets = -ENODEV;
 			goto out;
 		}
-		list_add_tail(&cb->list, &dev->ctrl_rd_list.list);
-	} else {
-		list_add_tail(&cb->list, &dev->ctrl_wr_list.list);
+		list_move_tail(&cb->list, &dev->ctrl_rd_list.list);
 	}
 
 	mutex_unlock(&dev->device_lock);
@@ -1436,25 +1437,6 @@ out:
 }
 
 /**
- * mei_cl_is_read_fc_cb - check if read cb is waiting for flow control
- *                        for given host client
- *
- * @cl: host client
- *
- * Return: true, if found at least one cb.
- */
-static bool mei_cl_is_read_fc_cb(struct mei_cl *cl)
-{
-	struct mei_device *dev = cl->dev;
-	struct mei_cl_cb *cb;
-
-	list_for_each_entry(cb, &dev->ctrl_wr_list.list, list)
-		if (cb->fop_type == MEI_FOP_READ && cb->cl == cl)
-			return true;
-	return false;
-}
-
-/**
  * mei_cl_read_start - the start read client message function.
  *
  * @cl: host client
@@ -1477,26 +1459,22 @@ int mei_cl_read_start(struct mei_cl *cl, size_t length, const struct file *fp)
 	if (!mei_cl_is_connected(cl))
 		return -ENODEV;
 
-	/* HW currently supports only one pending read */
-	if (!list_empty(&cl->rd_pending) || mei_cl_is_read_fc_cb(cl))
-		return -EBUSY;
-
 	if (!mei_me_cl_is_active(cl->me_cl)) {
 		cl_err(dev, cl, "no such me client\n");
 		return  -ENOTTY;
 	}
 
-	/* always allocate at least client max message */
-	length = max_t(size_t, length, mei_cl_mtu(cl));
-	cb = mei_cl_alloc_cb(cl, length, MEI_FOP_READ, fp);
+	if (mei_cl_is_fixed_address(cl) || cl == &dev->iamthif_cl)
+		return 0;
+
+	/* HW currently supports only one pending read */
+	if (cl->rx_flow_ctrl_creds)
+		return -EBUSY;
+
+	cb = mei_cl_enqueue_ctrl_wr_cb(cl, length, MEI_FOP_READ, fp);
 	if (!cb)
 		return -ENOMEM;
 
-	if (mei_cl_is_fixed_address(cl)) {
-		list_add_tail(&cb->list, &cl->rd_pending);
-		return 0;
-	}
-
 	rets = pm_runtime_get(dev->dev);
 	if (rets < 0 && rets != -EINPROGRESS) {
 		pm_runtime_put_noidle(dev->dev);
@@ -1504,16 +1482,15 @@ int mei_cl_read_start(struct mei_cl *cl, size_t length, const struct file *fp)
 		goto nortpm;
 	}
 
+	rets = 0;
 	if (mei_hbuf_acquire(dev)) {
 		rets = mei_hbm_cl_flow_control_req(dev, cl);
 		if (rets < 0)
 			goto out;
 
-		list_add_tail(&cb->list, &cl->rd_pending);
-	} else {
-		rets = 0;
-		list_add_tail(&cb->list, &dev->ctrl_wr_list.list);
+		list_move_tail(&cb->list, &cl->rd_pending);
 	}
+	cl->rx_flow_ctrl_creds++;
 
 out:
 	cl_dbg(dev, cl, "rpm: autosuspend\n");
@@ -1557,7 +1534,7 @@ int mei_cl_irq_write(struct mei_cl *cl, struct mei_cl_cb *cb,
 
 	first_chunk = cb->buf_idx == 0;
 
-	rets = first_chunk ? mei_cl_flow_ctrl_creds(cl, cb->fp) : 1;
+	rets = first_chunk ? mei_cl_tx_flow_ctrl_creds(cl) : 1;
 	if (rets < 0)
 		return rets;
 
@@ -1605,7 +1582,7 @@ int mei_cl_irq_write(struct mei_cl *cl, struct mei_cl_cb *cb,
 	cb->completed = mei_hdr.msg_complete == 1;
 
 	if (first_chunk) {
-		if (mei_cl_flow_ctrl_reduce(cl))
+		if (mei_cl_tx_flow_ctrl_creds_reduce(cl))
 			return -EIO;
 	}
 
@@ -1663,7 +1640,7 @@ int mei_cl_write(struct mei_cl *cl, struct mei_cl_cb *cb, bool blocking)
 	mei_hdr.msg_complete = 0;
 	mei_hdr.internal = cb->internal;
 
-	rets = mei_cl_flow_ctrl_creds(cl, cb->fp);
+	rets = mei_cl_tx_flow_ctrl_creds(cl);
 	if (rets < 0)
 		goto err;
 
@@ -1691,7 +1668,7 @@ int mei_cl_write(struct mei_cl *cl, struct mei_cl_cb *cb, bool blocking)
 	if (rets)
 		goto err;
 
-	rets = mei_cl_flow_ctrl_reduce(cl);
+	rets = mei_cl_tx_flow_ctrl_creds_reduce(cl);
 	if (rets)
 		goto err;
 
@@ -1761,6 +1738,9 @@ void mei_cl_complete(struct mei_cl *cl, struct mei_cl_cb *cb)
 
 	case MEI_FOP_READ:
 		list_add_tail(&cb->list, &cl->rd_completed);
+		if (!mei_cl_is_fixed_address(cl) &&
+		    !WARN_ON(!cl->rx_flow_ctrl_creds))
+			cl->rx_flow_ctrl_creds--;
 		if (!mei_cl_bus_rx_event(cl))
 			wake_up_interruptible(&cl->rx_wait);
 		break;
diff --git a/drivers/misc/mei/client.h b/drivers/misc/mei/client.h
index 0d7a3a1fef78..d2bfabecd882 100644
--- a/drivers/misc/mei/client.h
+++ b/drivers/misc/mei/client.h
@@ -82,11 +82,7 @@ static inline u8 mei_me_cl_ver(const struct mei_me_client *me_cl)
 /*
  * MEI IO Functions
  */
-struct mei_cl_cb *mei_io_cb_init(struct mei_cl *cl, enum mei_cb_file_ops type,
-				 const struct file *fp);
 void mei_io_cb_free(struct mei_cl_cb *priv_cb);
-int mei_io_cb_alloc_buf(struct mei_cl_cb *cb, size_t length);
-
 
 /**
  * mei_io_list_init - Sets up a queue list.
@@ -118,6 +114,9 @@ void mei_cl_read_cb_flush(const struct mei_cl *cl, const struct file *fp);
 struct mei_cl_cb *mei_cl_alloc_cb(struct mei_cl *cl, size_t length,
 				  enum mei_cb_file_ops type,
 				  const struct file *fp);
+struct mei_cl_cb *mei_cl_enqueue_ctrl_wr_cb(struct mei_cl *cl, size_t length,
+					    enum mei_cb_file_ops type,
+					    const struct file *fp);
 int mei_cl_flush_queues(struct mei_cl *cl, const struct file *fp);
 
 /*
diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c
index 085f3aafe6fa..dd7f15a65eed 100644
--- a/drivers/misc/mei/hbm.c
+++ b/drivers/misc/mei/hbm.c
@@ -161,6 +161,7 @@ void mei_hbm_cl_hdr(struct mei_cl *cl, u8 hbm_cmd, void *buf, size_t len)
  * @dev: the device structure
  * @cl: client
  * @hbm_cmd: host bus message command
+ * @buf: message buffer
  * @len: buffer length
  *
  * Return: 0 on success, <0 on failure.
@@ -276,6 +277,7 @@ int mei_hbm_start_req(struct mei_device *dev)
 
 	dev->hbm_state = MEI_HBM_STARTING;
 	dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT;
+	mei_schedule_stall_timer(dev);
 	return 0;
 }
 
@@ -311,6 +313,7 @@ static int mei_hbm_enum_clients_req(struct mei_device *dev)
 	}
 	dev->hbm_state = MEI_HBM_ENUM_CLIENTS;
 	dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT;
+	mei_schedule_stall_timer(dev);
 	return 0;
 }
 
@@ -339,7 +342,7 @@ static int mei_hbm_me_cl_add(struct mei_device *dev,
 
 	me_cl->props = res->client_properties;
 	me_cl->client_id = res->me_addr;
-	me_cl->mei_flow_ctrl_creds = 0;
+	me_cl->tx_flow_ctrl_creds = 0;
 
 	mei_me_cl_add(dev, me_cl);
 
@@ -561,6 +564,7 @@ static int mei_hbm_prop_req(struct mei_device *dev, unsigned long start_idx)
 	}
 
 	dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT;
+	mei_schedule_stall_timer(dev);
 
 	return 0;
 }
@@ -636,23 +640,22 @@ int mei_hbm_cl_flow_control_req(struct mei_device *dev, struct mei_cl *cl)
 }
 
 /**
- * mei_hbm_add_single_flow_creds - adds single buffer credentials.
+ * mei_hbm_add_single_tx_flow_ctrl_creds - adds single buffer credentials.
  *
  * @dev: the device structure
- * @flow: flow control.
+ * @fctrl: flow control response bus message
  *
  * Return: 0 on success, < 0 otherwise
  */
-static int mei_hbm_add_single_flow_creds(struct mei_device *dev,
-				  struct hbm_flow_control *flow)
+static int mei_hbm_add_single_tx_flow_ctrl_creds(struct mei_device *dev,
+						 struct hbm_flow_control *fctrl)
 {
 	struct mei_me_client *me_cl;
 	int rets;
 
-	me_cl = mei_me_cl_by_id(dev, flow->me_addr);
+	me_cl = mei_me_cl_by_id(dev, fctrl->me_addr);
 	if (!me_cl) {
-		dev_err(dev->dev, "no such me client %d\n",
-			flow->me_addr);
+		dev_err(dev->dev, "no such me client %d\n", fctrl->me_addr);
 		return -ENOENT;
 	}
 
@@ -661,9 +664,9 @@ static int mei_hbm_add_single_flow_creds(struct mei_device *dev,
 		goto out;
 	}
 
-	me_cl->mei_flow_ctrl_creds++;
+	me_cl->tx_flow_ctrl_creds++;
 	dev_dbg(dev->dev, "recv flow ctrl msg ME %d (single) creds = %d.\n",
-	    flow->me_addr, me_cl->mei_flow_ctrl_creds);
+		fctrl->me_addr, me_cl->tx_flow_ctrl_creds);
 
 	rets = 0;
 out:
@@ -675,24 +678,24 @@ out:
  * mei_hbm_cl_flow_control_res - flow control response from me
  *
  * @dev: the device structure
- * @flow_control: flow control response bus message
+ * @fctrl: flow control response bus message
  */
-static void mei_hbm_cl_flow_control_res(struct mei_device *dev,
-					struct hbm_flow_control *flow_control)
+static void mei_hbm_cl_tx_flow_ctrl_creds_res(struct mei_device *dev,
+					       struct hbm_flow_control *fctrl)
 {
 	struct mei_cl *cl;
 
-	if (!flow_control->host_addr) {
+	if (!fctrl->host_addr) {
 		/* single receive buffer */
-		mei_hbm_add_single_flow_creds(dev, flow_control);
+		mei_hbm_add_single_tx_flow_ctrl_creds(dev, fctrl);
 		return;
 	}
 
-	cl = mei_hbm_cl_find_by_cmd(dev, flow_control);
+	cl = mei_hbm_cl_find_by_cmd(dev, fctrl);
 	if (cl) {
-		cl->mei_flow_ctrl_creds++;
+		cl->tx_flow_ctrl_creds++;
 		cl_dbg(dev, cl, "flow control creds = %d.\n",
-				cl->mei_flow_ctrl_creds);
+				cl->tx_flow_ctrl_creds);
 	}
 }
 
@@ -871,10 +874,10 @@ static int mei_hbm_fw_disconnect_req(struct mei_device *dev,
 		cl->state = MEI_FILE_DISCONNECTING;
 		cl->timer_count = 0;
 
-		cb = mei_io_cb_init(cl, MEI_FOP_DISCONNECT_RSP, NULL);
+		cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_DISCONNECT_RSP,
+					       NULL);
 		if (!cb)
 			return -ENOMEM;
-		list_add_tail(&cb->list, &dev->ctrl_wr_list.list);
 	}
 	return 0;
 }
@@ -1022,7 +1025,7 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr)
 
 	struct mei_hbm_cl_cmd *cl_cmd;
 	struct hbm_client_connect_request *disconnect_req;
-	struct hbm_flow_control *flow_control;
+	struct hbm_flow_control *fctrl;
 
 	/* read the message to our buffer */
 	BUG_ON(hdr->length >= sizeof(dev->rd_msg_buf));
@@ -1102,8 +1105,8 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr)
 	case MEI_FLOW_CONTROL_CMD:
 		dev_dbg(dev->dev, "hbm: client flow control response: message received.\n");
 
-		flow_control = (struct hbm_flow_control *) mei_msg;
-		mei_hbm_cl_flow_control_res(dev, flow_control);
+		fctrl = (struct hbm_flow_control *)mei_msg;
+		mei_hbm_cl_tx_flow_ctrl_creds_res(dev, fctrl);
 		break;
 
 	case MEI_PG_ISOLATION_ENTRY_RES_CMD:
diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h
index 0dcb854b4bfc..7ad15d678878 100644
--- a/drivers/misc/mei/hw-me-regs.h
+++ b/drivers/misc/mei/hw-me-regs.h
@@ -125,6 +125,9 @@
 #define MEI_DEV_ID_BXT_M      0x1A9A  /* Broxton M */
 #define MEI_DEV_ID_APL_I      0x5A9A  /* Apollo Lake I */
 
+#define MEI_DEV_ID_KBP        0xA2BA  /* Kaby Point */
+#define MEI_DEV_ID_KBP_2      0xA2BB  /* Kaby Point 2 */
+
 /*
  * MEI HW Section
  */
diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c
index e2fb44cc5c37..56c2101e80ad 100644
--- a/drivers/misc/mei/hw-me.c
+++ b/drivers/misc/mei/hw-me.c
@@ -18,6 +18,7 @@
 
 #include <linux/kthread.h>
 #include <linux/interrupt.h>
+#include <linux/pm_runtime.h>
 
 #include "mei_dev.h"
 #include "hbm.h"
@@ -1063,6 +1064,8 @@ static int mei_me_hw_reset(struct mei_device *dev, bool intr_enable)
 		}
 	}
 
+	pm_runtime_set_active(dev->dev);
+
 	hcsr = mei_hcsr_read(dev);
 	/* H_RST may be found lit before reset is started,
 	 * for example if preceding reset flow hasn't completed.
@@ -1263,8 +1266,14 @@ static bool mei_me_fw_type_nm(struct pci_dev *pdev)
 static bool mei_me_fw_type_sps(struct pci_dev *pdev)
 {
 	u32 reg;
-	/* Read ME FW Status check for SPS Firmware */
-	pci_read_config_dword(pdev, PCI_CFG_HFS_1, &reg);
+	unsigned int devfn;
+
+	/*
+	 * Read ME FW Status register to check for SPS Firmware
+	 * The SPS FW is only signaled in pci function 0
+	 */
+	devfn = PCI_DEVFN(PCI_SLOT(pdev->devfn), 0);
+	pci_bus_read_config_dword(pdev->bus, devfn, PCI_CFG_HFS_1, &reg);
 	trace_mei_pci_cfg_read(&pdev->dev, "PCI_CFG_HFS_1", PCI_CFG_HFS_1, reg);
 	/* if bits [19:16] = 15, running SPS Firmware */
 	return (reg & 0xf0000) == 0xf0000;
diff --git a/drivers/misc/mei/hw-txe.c b/drivers/misc/mei/hw-txe.c
index 4a6c1b85f11e..e6e5e55a12ed 100644
--- a/drivers/misc/mei/hw-txe.c
+++ b/drivers/misc/mei/hw-txe.c
@@ -20,6 +20,7 @@
 #include <linux/delay.h>
 #include <linux/kthread.h>
 #include <linux/irqreturn.h>
+#include <linux/pm_runtime.h>
 
 #include <linux/mei.h>
 
@@ -935,6 +936,8 @@ static int mei_txe_hw_start(struct mei_device *dev)
 		return ret;
 	}
 
+	pm_runtime_set_active(dev->dev);
+
 	/* enable input ready interrupts:
 	 * SEC_IPC_HOST_INT_MASK.IPC_INPUT_READY_INT_MASK
 	 */
diff --git a/drivers/misc/mei/init.c b/drivers/misc/mei/init.c
index f7c8dfdb6a12..9a9c2484d107 100644
--- a/drivers/misc/mei/init.c
+++ b/drivers/misc/mei/init.c
@@ -94,7 +94,7 @@ void mei_cancel_work(struct mei_device *dev)
 	cancel_work_sync(&dev->reset_work);
 	cancel_work_sync(&dev->bus_rescan_work);
 
-	cancel_delayed_work(&dev->timer_work);
+	cancel_delayed_work_sync(&dev->timer_work);
 }
 EXPORT_SYMBOL_GPL(mei_cancel_work);
 
diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c
index 3831a7ba2531..5a4893ce9c24 100644
--- a/drivers/misc/mei/interrupt.c
+++ b/drivers/misc/mei/interrupt.c
@@ -102,26 +102,25 @@ int mei_cl_irq_read_msg(struct mei_cl *cl,
 {
 	struct mei_device *dev = cl->dev;
 	struct mei_cl_cb *cb;
-	unsigned char *buffer = NULL;
 	size_t buf_sz;
 
 	cb = list_first_entry_or_null(&cl->rd_pending, struct mei_cl_cb, list);
 	if (!cb) {
-		cl_err(dev, cl, "pending read cb not found\n");
-		goto out;
+		if (!mei_cl_is_fixed_address(cl)) {
+			cl_err(dev, cl, "pending read cb not found\n");
+			goto discard;
+		}
+		cb = mei_cl_alloc_cb(cl, mei_cl_mtu(cl), MEI_FOP_READ, cl->fp);
+		if (!cb)
+			goto discard;
+		list_add_tail(&cb->list, &cl->rd_pending);
 	}
 
 	if (!mei_cl_is_connected(cl)) {
 		cl_dbg(dev, cl, "not connected\n");
-		cb->status = -ENODEV;
-		goto out;
-	}
-
-	if (cb->buf.size == 0 || cb->buf.data == NULL) {
-		cl_err(dev, cl, "response buffer is not allocated.\n");
 		list_move_tail(&cb->list, &complete_list->list);
-		cb->status = -ENOMEM;
-		goto out;
+		cb->status = -ENODEV;
+		goto discard;
 	}
 
 	buf_sz = mei_hdr->length + cb->buf_idx;
@@ -132,25 +131,19 @@ int mei_cl_irq_read_msg(struct mei_cl *cl,
 
 		list_move_tail(&cb->list, &complete_list->list);
 		cb->status = -EMSGSIZE;
-		goto out;
+		goto discard;
 	}
 
 	if (cb->buf.size < buf_sz) {
 		cl_dbg(dev, cl, "message overflow. size %zu len %d idx %zu\n",
 			cb->buf.size, mei_hdr->length, cb->buf_idx);
-		buffer = krealloc(cb->buf.data, buf_sz, GFP_KERNEL);
 
-		if (!buffer) {
-			cb->status = -ENOMEM;
-			list_move_tail(&cb->list, &complete_list->list);
-			goto out;
-		}
-		cb->buf.data = buffer;
-		cb->buf.size = buf_sz;
+		list_move_tail(&cb->list, &complete_list->list);
+		cb->status = -EMSGSIZE;
+		goto discard;
 	}
 
-	buffer = cb->buf.data + cb->buf_idx;
-	mei_read_slots(dev, buffer, mei_hdr->length);
+	mei_read_slots(dev, cb->buf.data + cb->buf_idx, mei_hdr->length);
 
 	cb->buf_idx += mei_hdr->length;
 
@@ -162,10 +155,10 @@ int mei_cl_irq_read_msg(struct mei_cl *cl,
 		pm_request_autosuspend(dev->dev);
 	}
 
-out:
-	if (!buffer)
-		mei_irq_discard_msg(dev, mei_hdr);
+	return 0;
 
+discard:
+	mei_irq_discard_msg(dev, mei_hdr);
 	return 0;
 }
 
@@ -216,6 +209,9 @@ static int mei_cl_irq_read(struct mei_cl *cl, struct mei_cl_cb *cb,
 	int slots;
 	int ret;
 
+	if (!list_empty(&cl->rd_pending))
+		return 0;
+
 	msg_slots = mei_data2slots(sizeof(struct hbm_flow_control));
 	slots = mei_hbuf_empty_slots(dev);
 
@@ -463,6 +459,19 @@ static void mei_connect_timeout(struct mei_cl *cl)
 	mei_reset(dev);
 }
 
+#define MEI_STALL_TIMER_FREQ (2 * HZ)
+/**
+ * mei_schedule_stall_timer - re-arm stall_timer work
+ *
+ * Schedule stall timer
+ *
+ * @dev: the device structure
+ */
+void mei_schedule_stall_timer(struct mei_device *dev)
+{
+	schedule_delayed_work(&dev->timer_work, MEI_STALL_TIMER_FREQ);
+}
+
 /**
  * mei_timer - timer function.
  *
@@ -472,10 +481,9 @@ static void mei_connect_timeout(struct mei_cl *cl)
 void mei_timer(struct work_struct *work)
 {
 	struct mei_cl *cl;
-
 	struct mei_device *dev = container_of(work,
 					struct mei_device, timer_work.work);
-
+	bool reschedule_timer = false;
 
 	mutex_lock(&dev->device_lock);
 
@@ -490,6 +498,7 @@ void mei_timer(struct work_struct *work)
 				mei_reset(dev);
 				goto out;
 			}
+			reschedule_timer = true;
 		}
 	}
 
@@ -504,6 +513,7 @@ void mei_timer(struct work_struct *work)
 				mei_connect_timeout(cl);
 				goto out;
 			}
+			reschedule_timer = true;
 		}
 	}
 
@@ -514,19 +524,16 @@ void mei_timer(struct work_struct *work)
 		if (--dev->iamthif_stall_timer == 0) {
 			dev_err(dev->dev, "timer: amthif  hanged.\n");
 			mei_reset(dev);
-			dev->iamthif_canceled = false;
-			dev->iamthif_state = MEI_IAMTHIF_IDLE;
 
-			mei_io_cb_free(dev->iamthif_current_cb);
-			dev->iamthif_current_cb = NULL;
-
-			dev->iamthif_fp = NULL;
 			mei_amthif_run_next_cmd(dev);
+			goto out;
 		}
+		reschedule_timer = true;
 	}
 
 out:
-	if (dev->dev_state != MEI_DEV_DISABLED)
-		schedule_delayed_work(&dev->timer_work, 2 * HZ);
+	if (dev->dev_state != MEI_DEV_DISABLED && reschedule_timer)
+		mei_schedule_stall_timer(dev);
+
 	mutex_unlock(&dev->device_lock);
 }
diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c
index 52635b063873..fa50635512e8 100644
--- a/drivers/misc/mei/main.c
+++ b/drivers/misc/mei/main.c
@@ -71,6 +71,7 @@ static int mei_open(struct inode *inode, struct file *file)
 		goto err_unlock;
 	}
 
+	cl->fp = file;
 	file->private_data = cl;
 
 	mutex_unlock(&dev->device_lock);
@@ -138,9 +139,8 @@ static ssize_t mei_read(struct file *file, char __user *ubuf,
 	struct mei_cl *cl = file->private_data;
 	struct mei_device *dev;
 	struct mei_cl_cb *cb = NULL;
+	bool nonblock = !!(file->f_flags & O_NONBLOCK);
 	int rets;
-	int err;
-
 
 	if (WARN_ON(!cl || !cl->dev))
 		return -ENODEV;
@@ -164,11 +164,6 @@ static ssize_t mei_read(struct file *file, char __user *ubuf,
 		goto out;
 	}
 
-	if (cl == &dev->iamthif_cl) {
-		rets = mei_amthif_read(dev, file, ubuf, length, offset);
-		goto out;
-	}
-
 	cb = mei_cl_read_cb(cl, file);
 	if (cb)
 		goto copy_buffer;
@@ -176,24 +171,29 @@ static ssize_t mei_read(struct file *file, char __user *ubuf,
 	if (*offset > 0)
 		*offset = 0;
 
-	err = mei_cl_read_start(cl, length, file);
-	if (err && err != -EBUSY) {
-		cl_dbg(dev, cl, "mei start read failure status = %d\n", err);
-		rets = err;
+	rets = mei_cl_read_start(cl, length, file);
+	if (rets && rets != -EBUSY) {
+		cl_dbg(dev, cl, "mei start read failure status = %d\n", rets);
 		goto out;
 	}
 
-	if (list_empty(&cl->rd_completed) && !waitqueue_active(&cl->rx_wait)) {
-		if (file->f_flags & O_NONBLOCK) {
-			rets = -EAGAIN;
-			goto out;
-		}
+	if (nonblock) {
+		rets = -EAGAIN;
+		goto out;
+	}
 
+	if (rets == -EBUSY &&
+	    !mei_cl_enqueue_ctrl_wr_cb(cl, length, MEI_FOP_READ, file)) {
+		rets = -ENOMEM;
+		goto out;
+	}
+
+	do {
 		mutex_unlock(&dev->device_lock);
 
 		if (wait_event_interruptible(cl->rx_wait,
-				(!list_empty(&cl->rd_completed)) ||
-				(!mei_cl_is_connected(cl)))) {
+					     (!list_empty(&cl->rd_completed)) ||
+					     (!mei_cl_is_connected(cl)))) {
 
 			if (signal_pending(current))
 				return -EINTR;
@@ -202,16 +202,12 @@ static ssize_t mei_read(struct file *file, char __user *ubuf,
 
 		mutex_lock(&dev->device_lock);
 		if (!mei_cl_is_connected(cl)) {
-			rets = -EBUSY;
+			rets = -ENODEV;
 			goto out;
 		}
-	}
 
-	cb = mei_cl_read_cb(cl, file);
-	if (!cb) {
-		rets = 0;
-		goto out;
-	}
+		cb = mei_cl_read_cb(cl, file);
+	} while (!cb);
 
 copy_buffer:
 	/* now copy the data to user space */
@@ -609,24 +605,24 @@ static unsigned int mei_poll(struct file *file, poll_table *wait)
 		goto out;
 	}
 
-	if (cl == &dev->iamthif_cl) {
-		mask = mei_amthif_poll(dev, file, wait);
-		goto out;
-	}
-
 	if (notify_en) {
 		poll_wait(file, &cl->ev_wait, wait);
 		if (cl->notify_ev)
 			mask |= POLLPRI;
 	}
 
+	if (cl == &dev->iamthif_cl) {
+		mask |= mei_amthif_poll(file, wait);
+		goto out;
+	}
+
 	if (req_events & (POLLIN | POLLRDNORM)) {
 		poll_wait(file, &cl->rx_wait, wait);
 
 		if (!list_empty(&cl->rd_completed))
 			mask |= POLLIN | POLLRDNORM;
 		else
-			mei_cl_read_start(cl, 0, file);
+			mei_cl_read_start(cl, mei_cl_mtu(cl), file);
 	}
 
 out:
diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h
index e5e32503d4bc..1169fd9e7d02 100644
--- a/drivers/misc/mei/mei_dev.h
+++ b/drivers/misc/mei/mei_dev.h
@@ -80,18 +80,13 @@ const char *mei_dev_state_str(int state);
 enum iamthif_states {
 	MEI_IAMTHIF_IDLE,
 	MEI_IAMTHIF_WRITING,
-	MEI_IAMTHIF_FLOW_CONTROL,
 	MEI_IAMTHIF_READING,
-	MEI_IAMTHIF_READ_COMPLETE
 };
 
 enum mei_file_transaction_states {
 	MEI_IDLE,
 	MEI_WRITING,
 	MEI_WRITE_COMPLETE,
-	MEI_FLOW_CONTROL,
-	MEI_READING,
-	MEI_READ_COMPLETE
 };
 
 /**
@@ -146,7 +141,7 @@ struct mei_fw_status {
  * @refcnt: struct reference count
  * @props: client properties
  * @client_id: me client id
- * @mei_flow_ctrl_creds: flow control credits
+ * @tx_flow_ctrl_creds: flow control credits
  * @connect_count: number connections to this client
  * @bus_added: added to bus
  */
@@ -155,7 +150,7 @@ struct mei_me_client {
 	struct kref refcnt;
 	struct mei_client_properties props;
 	u8 client_id;
-	u8 mei_flow_ctrl_creds;
+	u8 tx_flow_ctrl_creds;
 	u8 connect_count;
 	u8 bus_added;
 };
@@ -202,10 +197,11 @@ struct mei_cl_cb {
  * @ev_async: event async notification
  * @status: connection status
  * @me_cl: fw client connected
+ * @fp: file associated with client
  * @host_client_id: host id
- * @mei_flow_ctrl_creds: transmit flow credentials
+ * @tx_flow_ctrl_creds: transmit flow credentials
+ * @rx_flow_ctrl_creds: receive flow credentials
  * @timer_count:  watchdog timer for operation completion
- * @reserved: reserved for alignment
  * @notify_en: notification - enabled/disabled
  * @notify_ev: pending notification event
  * @writing_state: state of the tx
@@ -225,10 +221,11 @@ struct mei_cl {
 	struct fasync_struct *ev_async;
 	int status;
 	struct mei_me_client *me_cl;
+	const struct file *fp;
 	u8 host_client_id;
-	u8 mei_flow_ctrl_creds;
+	u8 tx_flow_ctrl_creds;
+	u8 rx_flow_ctrl_creds;
 	u8 timer_count;
-	u8 reserved;
 	u8 notify_en;
 	u8 notify_ev;
 	enum mei_file_transaction_states writing_state;
@@ -400,9 +397,7 @@ const char *mei_pg_state_str(enum mei_pg_state state);
  * @override_fixed_address: force allow fixed address behavior
  *
  * @amthif_cmd_list : amthif list for cmd waiting
- * @iamthif_fp : file for current amthif operation
  * @iamthif_cl  : amthif host client
- * @iamthif_current_cb : amthif current operation callback
  * @iamthif_open_count : number of opened amthif connections
  * @iamthif_stall_timer : timer to detect amthif hang
  * @iamthif_state : amthif processor state
@@ -484,10 +479,7 @@ struct mei_device {
 
 	/* amthif list for cmd waiting */
 	struct mei_cl_cb amthif_cmd_list;
-	/* driver managed amthif list for reading completed amthif cmd data */
-	const struct file *iamthif_fp;
 	struct mei_cl iamthif_cl;
-	struct mei_cl_cb *iamthif_current_cb;
 	long iamthif_open_count;
 	u32 iamthif_stall_timer;
 	enum iamthif_states iamthif_state;
@@ -556,6 +548,7 @@ void mei_cancel_work(struct mei_device *dev);
  */
 
 void mei_timer(struct work_struct *work);
+void mei_schedule_stall_timer(struct mei_device *dev);
 int mei_irq_read_handler(struct mei_device *dev,
 		struct mei_cl_cb *cmpl_list, s32 *slots);
 
@@ -569,11 +562,7 @@ void mei_amthif_reset_params(struct mei_device *dev);
 
 int mei_amthif_host_init(struct mei_device *dev, struct mei_me_client *me_cl);
 
-int mei_amthif_read(struct mei_device *dev, struct file *file,
-		char __user *ubuf, size_t length, loff_t *offset);
-
-unsigned int mei_amthif_poll(struct mei_device *dev,
-		struct file *file, poll_table *wait);
+unsigned int mei_amthif_poll(struct file *file, poll_table *wait);
 
 int mei_amthif_release(struct mei_device *dev, struct file *file);
 
diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c
index 64e64da6da44..f3ffd883b232 100644
--- a/drivers/misc/mei/pci-me.c
+++ b/drivers/misc/mei/pci-me.c
@@ -85,12 +85,15 @@ static const struct pci_device_id mei_me_pci_tbl[] = {
 
 	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT, mei_me_pch8_cfg)},
 	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_2, mei_me_pch8_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H, mei_me_pch8_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H_2, mei_me_pch8_cfg)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H, mei_me_pch8_sps_cfg)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H_2, mei_me_pch8_sps_cfg)},
 
 	{MEI_PCI_DEVICE(MEI_DEV_ID_BXT_M, mei_me_pch8_cfg)},
 	{MEI_PCI_DEVICE(MEI_DEV_ID_APL_I, mei_me_pch8_cfg)},
 
+	{MEI_PCI_DEVICE(MEI_DEV_ID_KBP, mei_me_pch8_cfg)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_KBP_2, mei_me_pch8_cfg)},
+
 	/* required last entry */
 	{0, }
 };
@@ -217,8 +220,6 @@ static int mei_me_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_drvdata(pdev, dev);
 
-	schedule_delayed_work(&dev->timer_work, HZ);
-
 	/*
 	* For not wake-able HW runtime pm framework
 	* can't be used on pci device level.
@@ -400,6 +401,9 @@ static int mei_me_pm_runtime_suspend(struct device *device)
 
 	dev_dbg(&pdev->dev, "rpm: me: runtime suspend ret=%d\n", ret);
 
+	if (ret && ret != -EAGAIN)
+		schedule_work(&dev->reset_work);
+
 	return ret;
 }
 
@@ -423,6 +427,9 @@ static int mei_me_pm_runtime_resume(struct device *device)
 
 	dev_dbg(&pdev->dev, "rpm: me: runtime resume ret = %d\n", ret);
 
+	if (ret)
+		schedule_work(&dev->reset_work);
+
 	return ret;
 }
 
diff --git a/drivers/misc/mei/pci-txe.c b/drivers/misc/mei/pci-txe.c
index 30cc30683c07..58ffd30dcc91 100644
--- a/drivers/misc/mei/pci-txe.c
+++ b/drivers/misc/mei/pci-txe.c
@@ -347,6 +347,10 @@ static int mei_txe_pm_runtime_suspend(struct device *device)
 	dev_dbg(&pdev->dev, "rpm: txe: runtime suspend ret=%d\n", ret);
 
 	mutex_unlock(&dev->device_lock);
+
+	if (ret && ret != -EAGAIN)
+		schedule_work(&dev->reset_work);
+
 	return ret;
 }
 
@@ -372,6 +376,9 @@ static int mei_txe_pm_runtime_resume(struct device *device)
 
 	dev_dbg(&pdev->dev, "rpm: txe: runtime resume ret = %d\n", ret);
 
+	if (ret)
+		schedule_work(&dev->reset_work);
+
 	return ret;
 }
 
diff --git a/drivers/misc/mic/scif/scif_dma.c b/drivers/misc/mic/scif/scif_dma.c
index cd01a0efda6b..64d5760d069a 100644
--- a/drivers/misc/mic/scif/scif_dma.c
+++ b/drivers/misc/mic/scif/scif_dma.c
@@ -115,7 +115,6 @@ int scif_reserve_dma_chan(struct scif_endpt *ep)
  */
 static
 void __scif_rma_destroy_tcw(struct scif_mmu_notif *mmn,
-			    struct scif_endpt *ep,
 			    u64 start, u64 len)
 {
 	struct list_head *item, *tmp;
@@ -128,7 +127,6 @@ void __scif_rma_destroy_tcw(struct scif_mmu_notif *mmn,
 
 	list_for_each_safe(item, tmp, &mmn->tc_reg_list) {
 		window = list_entry(item, struct scif_window, list);
-		ep = (struct scif_endpt *)window->ep;
 		if (!len)
 			break;
 		start_va = window->va_for_temp;
@@ -146,7 +144,7 @@ static void scif_rma_destroy_tcw(struct scif_mmu_notif *mmn, u64 start, u64 len)
 	struct scif_endpt *ep = mmn->ep;
 
 	spin_lock(&ep->rma_info.tc_lock);
-	__scif_rma_destroy_tcw(mmn, ep, start, len);
+	__scif_rma_destroy_tcw(mmn, start, len);
 	spin_unlock(&ep->rma_info.tc_lock);
 }
 
@@ -169,7 +167,7 @@ static void __scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
 	spin_lock(&ep->rma_info.tc_lock);
 	list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
 		mmn = list_entry(item, struct scif_mmu_notif, list);
-		__scif_rma_destroy_tcw(mmn, ep, 0, ULONG_MAX);
+		__scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
 	}
 	spin_unlock(&ep->rma_info.tc_lock);
 }
diff --git a/drivers/misc/mic/scif/scif_mmap.c b/drivers/misc/mic/scif/scif_mmap.c
index 49cb8f7b4672..928211677079 100644
--- a/drivers/misc/mic/scif/scif_mmap.c
+++ b/drivers/misc/mic/scif/scif_mmap.c
@@ -552,7 +552,7 @@ static void scif_munmap(struct vm_area_struct *vma)
 {
 	struct scif_endpt *ep;
 	struct vma_pvt *vmapvt = vma->vm_private_data;
-	int nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	int nr_pages = vma_pages(vma);
 	s64 offset;
 	struct scif_rma_req req;
 	struct scif_window *window = NULL;
@@ -614,7 +614,7 @@ int scif_mmap(struct vm_area_struct *vma, scif_epd_t epd)
 	struct scif_window *window = NULL;
 	struct scif_endpt *ep = (struct scif_endpt *)epd;
 	s64 start_offset = vma->vm_pgoff << PAGE_SHIFT;
-	int nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	int nr_pages = vma_pages(vma);
 	int err;
 	struct vma_pvt *vmapvt;
 
diff --git a/drivers/misc/pch_phub.c b/drivers/misc/pch_phub.c
index 4810e039bbec..e42bdc90fa27 100644
--- a/drivers/misc/pch_phub.c
+++ b/drivers/misc/pch_phub.c
@@ -28,6 +28,7 @@
 #include <linux/if_ether.h>
 #include <linux/ctype.h>
 #include <linux/dmi.h>
+#include <linux/of.h>
 
 #define PHUB_STATUS 0x00		/* Status Register offset */
 #define PHUB_CONTROL 0x04		/* Control Register offset */
@@ -57,6 +58,7 @@
 
 /* CM-iTC */
 #define CLKCFG_UART_48MHZ			(1 << 16)
+#define CLKCFG_UART_25MHZ			(2 << 16)
 #define CLKCFG_BAUDDIV				(2 << 20)
 #define CLKCFG_PLL2VCO				(8 << 9)
 #define CLKCFG_UARTCLKSEL			(1 << 18)
@@ -711,6 +713,12 @@ static int pch_phub_probe(struct pci_dev *pdev,
 
 	if (id->driver_data == 1) { /* EG20T PCH */
 		const char *board_name;
+		unsigned int prefetch = 0x000affaa;
+
+		if (pdev->dev.of_node)
+			of_property_read_u32(pdev->dev.of_node,
+						  "intel,eg20t-prefetch",
+						  &prefetch);
 
 		ret = sysfs_create_file(&pdev->dev.kobj,
 					&dev_attr_pch_mac.attr);
@@ -736,11 +744,21 @@ static int pch_phub_probe(struct pci_dev *pdev,
 						CLKCFG_UART_MASK);
 
 		/* set the prefech value */
-		iowrite32(0x000affaa, chip->pch_phub_base_address + 0x14);
+		iowrite32(prefetch, chip->pch_phub_base_address + 0x14);
 		/* set the interrupt delay value */
 		iowrite32(0x25, chip->pch_phub_base_address + 0x44);
 		chip->pch_opt_rom_start_address = PCH_PHUB_ROM_START_ADDR_EG20T;
 		chip->pch_mac_start_address = PCH_PHUB_MAC_START_ADDR_EG20T;
+
+		/* quirk for MIPS Boston platform */
+		if (pdev->dev.of_node) {
+			if (of_machine_is_compatible("img,boston")) {
+				pch_phub_read_modify_write_reg(chip,
+					(unsigned int)CLKCFG_REG_OFFSET,
+					CLKCFG_UART_25MHZ,
+					CLKCFG_UART_MASK);
+			}
+		}
 	} else if (id->driver_data == 2) { /* ML7213 IOH */
 		ret = sysfs_create_bin_file(&pdev->dev.kobj, &pch_bin_attr);
 		if (ret)
diff --git a/drivers/misc/vmw_vmci/vmci_host.c b/drivers/misc/vmw_vmci/vmci_host.c
index 9ec262a52656..ec090105eb4b 100644
--- a/drivers/misc/vmw_vmci/vmci_host.c
+++ b/drivers/misc/vmw_vmci/vmci_host.c
@@ -381,18 +381,12 @@ static int vmci_host_do_send_datagram(struct vmci_host_dev *vmci_host_dev,
 		return -EINVAL;
 	}
 
-	dg = kmalloc(send_info.len, GFP_KERNEL);
-	if (!dg) {
+	dg = memdup_user((void __user *)(uintptr_t)send_info.addr,
+			 send_info.len);
+	if (IS_ERR(dg)) {
 		vmci_ioctl_err(
 			"cannot allocate memory to dispatch datagram\n");
-		return -ENOMEM;
-	}
-
-	if (copy_from_user(dg, (void __user *)(uintptr_t)send_info.addr,
-			   send_info.len)) {
-		vmci_ioctl_err("error getting datagram\n");
-		kfree(dg);
-		return -EFAULT;
+		return PTR_ERR(dg);
 	}
 
 	if (VMCI_DG_SIZE(dg) != send_info.len) {
diff --git a/drivers/mmc/card/block.c b/drivers/mmc/card/block.c
index 48a5dd740f3b..c3335112e68c 100644
--- a/drivers/mmc/card/block.c
+++ b/drivers/mmc/card/block.c
@@ -142,8 +142,6 @@ static inline void mmc_blk_clear_packed(struct mmc_queue_req *mqrq)
 {
 	struct mmc_packed *packed = mqrq->packed;
 
-	BUG_ON(!packed);
-
 	mqrq->cmd_type = MMC_PACKED_NONE;
 	packed->nr_entries = MMC_PACKED_NR_ZERO;
 	packed->idx_failure = MMC_PACKED_NR_IDX;
@@ -1443,8 +1441,6 @@ static int mmc_blk_packed_err_check(struct mmc_card *card,
 	int err, check, status;
 	u8 *ext_csd;
 
-	BUG_ON(!packed);
-
 	packed->retries--;
 	check = mmc_blk_err_check(card, areq);
 	err = get_card_status(card, &status, 0);
@@ -1673,6 +1669,18 @@ static u8 mmc_blk_prep_packed_list(struct mmc_queue *mq, struct request *req)
 	u8 max_packed_rw = 0;
 	u8 reqs = 0;
 
+	/*
+	 * We don't need to check packed for any further
+	 * operation of packed stuff as we set MMC_PACKED_NONE
+	 * and return zero for reqs if geting null packed. Also
+	 * we clean the flag of MMC_BLK_PACKED_CMD to avoid doing
+	 * it again when removing blk req.
+	 */
+	if (!mqrq->packed) {
+		md->flags &= (~MMC_BLK_PACKED_CMD);
+		goto no_packed;
+	}
+
 	if (!(md->flags & MMC_BLK_PACKED_CMD))
 		goto no_packed;
 
@@ -1726,6 +1734,7 @@ static u8 mmc_blk_prep_packed_list(struct mmc_queue *mq, struct request *req)
 			break;
 
 		if (req_op(next) == REQ_OP_DISCARD ||
+		    req_op(next) == REQ_OP_SECURE_ERASE ||
 		    req_op(next) == REQ_OP_FLUSH)
 			break;
 
@@ -1781,8 +1790,6 @@ static void mmc_blk_packed_hdr_wrq_prep(struct mmc_queue_req *mqrq,
 	u8 hdr_blocks;
 	u8 i = 1;
 
-	BUG_ON(!packed);
-
 	mqrq->cmd_type = MMC_PACKED_WRITE;
 	packed->blocks = 0;
 	packed->idx_failure = MMC_PACKED_NR_IDX;
@@ -1886,8 +1893,6 @@ static int mmc_blk_end_packed_req(struct mmc_queue_req *mq_rq)
 	int idx = packed->idx_failure, i = 0;
 	int ret = 0;
 
-	BUG_ON(!packed);
-
 	while (!list_empty(&packed->list)) {
 		prq = list_entry_rq(packed->list.next);
 		if (idx == i) {
@@ -1916,8 +1921,6 @@ static void mmc_blk_abort_packed_req(struct mmc_queue_req *mq_rq)
 	struct request *prq;
 	struct mmc_packed *packed = mq_rq->packed;
 
-	BUG_ON(!packed);
-
 	while (!list_empty(&packed->list)) {
 		prq = list_entry_rq(packed->list.next);
 		list_del_init(&prq->queuelist);
@@ -1934,8 +1937,6 @@ static void mmc_blk_revert_packed_req(struct mmc_queue *mq,
 	struct request_queue *q = mq->queue;
 	struct mmc_packed *packed = mq_rq->packed;
 
-	BUG_ON(!packed);
-
 	while (!list_empty(&packed->list)) {
 		prq = list_entry_rq(packed->list.prev);
 		if (prq->queuelist.prev != &packed->list) {
@@ -2143,13 +2144,14 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
 	return 0;
 }
 
-static int mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req)
+int mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req)
 {
 	int ret;
 	struct mmc_blk_data *md = mq->data;
 	struct mmc_card *card = md->queue.card;
 	struct mmc_host *host = card->host;
 	unsigned long flags;
+	bool req_is_special = mmc_req_is_special(req);
 
 	if (req && !mq->mqrq_prev->req)
 		/* claim host only for the first request */
@@ -2190,8 +2192,7 @@ static int mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req)
 	}
 
 out:
-	if ((!req && !(mq->flags & MMC_QUEUE_NEW_REQUEST)) ||
-	    mmc_req_is_special(req))
+	if ((!req && !(mq->flags & MMC_QUEUE_NEW_REQUEST)) || req_is_special)
 		/*
 		 * Release host when there are no more requests
 		 * and after special request(discard, flush) is done.
@@ -2264,7 +2265,6 @@ again:
 	if (ret)
 		goto err_putdisk;
 
-	md->queue.issue_fn = mmc_blk_issue_rq;
 	md->queue.data = md;
 
 	md->disk->major	= MMC_BLOCK_MAJOR;
@@ -2302,7 +2302,8 @@ again:
 	set_capacity(md->disk, size);
 
 	if (mmc_host_cmd23(card->host)) {
-		if (mmc_card_mmc(card) ||
+		if ((mmc_card_mmc(card) &&
+		     card->csd.mmca_vsn >= CSD_SPEC_VER_3) ||
 		    (mmc_card_sd(card) &&
 		     card->scr.cmds & SD_SCR_CMD23_SUPPORT))
 			md->flags |= MMC_BLK_CMD23;
diff --git a/drivers/mmc/card/block.h b/drivers/mmc/card/block.h
new file mode 100644
index 000000000000..cdabb2ee74be
--- /dev/null
+++ b/drivers/mmc/card/block.h
@@ -0,0 +1 @@
+int mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req);
diff --git a/drivers/mmc/card/mmc_test.c b/drivers/mmc/card/mmc_test.c
index c032eef45762..5a8dc5a76e0d 100644
--- a/drivers/mmc/card/mmc_test.c
+++ b/drivers/mmc/card/mmc_test.c
@@ -184,6 +184,29 @@ static int mmc_test_set_blksize(struct mmc_test_card *test, unsigned size)
 	return mmc_set_blocklen(test->card, size);
 }
 
+static bool mmc_test_card_cmd23(struct mmc_card *card)
+{
+	return mmc_card_mmc(card) ||
+	       (mmc_card_sd(card) && card->scr.cmds & SD_SCR_CMD23_SUPPORT);
+}
+
+static void mmc_test_prepare_sbc(struct mmc_test_card *test,
+				 struct mmc_request *mrq, unsigned int blocks)
+{
+	struct mmc_card *card = test->card;
+
+	if (!mrq->sbc || !mmc_host_cmd23(card->host) ||
+	    !mmc_test_card_cmd23(card) || !mmc_op_multi(mrq->cmd->opcode) ||
+	    (card->quirks & MMC_QUIRK_BLK_NO_CMD23)) {
+		mrq->sbc = NULL;
+		return;
+	}
+
+	mrq->sbc->opcode = MMC_SET_BLOCK_COUNT;
+	mrq->sbc->arg = blocks;
+	mrq->sbc->flags = MMC_RSP_R1 | MMC_CMD_AC;
+}
+
 /*
  * Fill in the mmc_request structure given a set of transfer parameters.
  */
@@ -221,6 +244,8 @@ static void mmc_test_prepare_mrq(struct mmc_test_card *test,
 	mrq->data->sg = sg;
 	mrq->data->sg_len = sg_len;
 
+	mmc_test_prepare_sbc(test, mrq, blocks);
+
 	mmc_set_data_timeout(mrq->data, test->card);
 }
 
@@ -693,6 +718,8 @@ static int mmc_test_check_result(struct mmc_test_card *test,
 
 	ret = 0;
 
+	if (mrq->sbc && mrq->sbc->error)
+		ret = mrq->sbc->error;
 	if (!ret && mrq->cmd->error)
 		ret = mrq->cmd->error;
 	if (!ret && mrq->data->error)
@@ -2278,6 +2305,245 @@ static int mmc_test_reset(struct mmc_test_card *test)
 	return RESULT_FAIL;
 }
 
+struct mmc_test_req {
+	struct mmc_request mrq;
+	struct mmc_command sbc;
+	struct mmc_command cmd;
+	struct mmc_command stop;
+	struct mmc_command status;
+	struct mmc_data data;
+};
+
+static struct mmc_test_req *mmc_test_req_alloc(void)
+{
+	struct mmc_test_req *rq = kzalloc(sizeof(*rq), GFP_KERNEL);
+
+	if (rq) {
+		rq->mrq.cmd = &rq->cmd;
+		rq->mrq.data = &rq->data;
+		rq->mrq.stop = &rq->stop;
+	}
+
+	return rq;
+}
+
+static int mmc_test_send_status(struct mmc_test_card *test,
+				struct mmc_command *cmd)
+{
+	memset(cmd, 0, sizeof(*cmd));
+
+	cmd->opcode = MMC_SEND_STATUS;
+	if (!mmc_host_is_spi(test->card->host))
+		cmd->arg = test->card->rca << 16;
+	cmd->flags = MMC_RSP_SPI_R2 | MMC_RSP_R1 | MMC_CMD_AC;
+
+	return mmc_wait_for_cmd(test->card->host, cmd, 0);
+}
+
+static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
+				     unsigned int dev_addr, int use_sbc,
+				     int repeat_cmd, int write, int use_areq)
+{
+	struct mmc_test_req *rq = mmc_test_req_alloc();
+	struct mmc_host *host = test->card->host;
+	struct mmc_test_area *t = &test->area;
+	struct mmc_async_req areq;
+	struct mmc_request *mrq;
+	unsigned long timeout;
+	bool expired = false;
+	int ret = 0, cmd_ret;
+	u32 status = 0;
+	int count = 0;
+
+	if (!rq)
+		return -ENOMEM;
+
+	mrq = &rq->mrq;
+	if (use_sbc)
+		mrq->sbc = &rq->sbc;
+	mrq->cap_cmd_during_tfr = true;
+
+	areq.mrq = mrq;
+	areq.err_check = mmc_test_check_result_async;
+
+	mmc_test_prepare_mrq(test, mrq, t->sg, t->sg_len, dev_addr, t->blocks,
+			     512, write);
+
+	if (use_sbc && t->blocks > 1 && !mrq->sbc) {
+		ret =  mmc_host_cmd23(host) ?
+		       RESULT_UNSUP_CARD :
+		       RESULT_UNSUP_HOST;
+		goto out_free;
+	}
+
+	/* Start ongoing data request */
+	if (use_areq) {
+		mmc_start_req(host, &areq, &ret);
+		if (ret)
+			goto out_free;
+	} else {
+		mmc_wait_for_req(host, mrq);
+	}
+
+	timeout = jiffies + msecs_to_jiffies(3000);
+	do {
+		count += 1;
+
+		/* Send status command while data transfer in progress */
+		cmd_ret = mmc_test_send_status(test, &rq->status);
+		if (cmd_ret)
+			break;
+
+		status = rq->status.resp[0];
+		if (status & R1_ERROR) {
+			cmd_ret = -EIO;
+			break;
+		}
+
+		if (mmc_is_req_done(host, mrq))
+			break;
+
+		expired = time_after(jiffies, timeout);
+		if (expired) {
+			pr_info("%s: timeout waiting for Tran state status %#x\n",
+				mmc_hostname(host), status);
+			cmd_ret = -ETIMEDOUT;
+			break;
+		}
+	} while (repeat_cmd && R1_CURRENT_STATE(status) != R1_STATE_TRAN);
+
+	/* Wait for data request to complete */
+	if (use_areq)
+		mmc_start_req(host, NULL, &ret);
+	else
+		mmc_wait_for_req_done(test->card->host, mrq);
+
+	/*
+	 * For cap_cmd_during_tfr request, upper layer must send stop if
+	 * required.
+	 */
+	if (mrq->data->stop && (mrq->data->error || !mrq->sbc)) {
+		if (ret)
+			mmc_wait_for_cmd(host, mrq->data->stop, 0);
+		else
+			ret = mmc_wait_for_cmd(host, mrq->data->stop, 0);
+	}
+
+	if (ret)
+		goto out_free;
+
+	if (cmd_ret) {
+		pr_info("%s: Send Status failed: status %#x, error %d\n",
+			mmc_hostname(test->card->host), status, cmd_ret);
+	}
+
+	ret = mmc_test_check_result(test, mrq);
+	if (ret)
+		goto out_free;
+
+	ret = mmc_test_wait_busy(test);
+	if (ret)
+		goto out_free;
+
+	if (repeat_cmd && (t->blocks + 1) << 9 > t->max_tfr)
+		pr_info("%s: %d commands completed during transfer of %u blocks\n",
+			mmc_hostname(test->card->host), count, t->blocks);
+
+	if (cmd_ret)
+		ret = cmd_ret;
+out_free:
+	kfree(rq);
+
+	return ret;
+}
+
+static int __mmc_test_cmds_during_tfr(struct mmc_test_card *test,
+				      unsigned long sz, int use_sbc, int write,
+				      int use_areq)
+{
+	struct mmc_test_area *t = &test->area;
+	int ret;
+
+	if (!(test->card->host->caps & MMC_CAP_CMD_DURING_TFR))
+		return RESULT_UNSUP_HOST;
+
+	ret = mmc_test_area_map(test, sz, 0, 0);
+	if (ret)
+		return ret;
+
+	ret = mmc_test_ongoing_transfer(test, t->dev_addr, use_sbc, 0, write,
+					use_areq);
+	if (ret)
+		return ret;
+
+	return mmc_test_ongoing_transfer(test, t->dev_addr, use_sbc, 1, write,
+					 use_areq);
+}
+
+static int mmc_test_cmds_during_tfr(struct mmc_test_card *test, int use_sbc,
+				    int write, int use_areq)
+{
+	struct mmc_test_area *t = &test->area;
+	unsigned long sz;
+	int ret;
+
+	for (sz = 512; sz <= t->max_tfr; sz += 512) {
+		ret = __mmc_test_cmds_during_tfr(test, sz, use_sbc, write,
+						 use_areq);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+/*
+ * Commands during read - no Set Block Count (CMD23).
+ */
+static int mmc_test_cmds_during_read(struct mmc_test_card *test)
+{
+	return mmc_test_cmds_during_tfr(test, 0, 0, 0);
+}
+
+/*
+ * Commands during write - no Set Block Count (CMD23).
+ */
+static int mmc_test_cmds_during_write(struct mmc_test_card *test)
+{
+	return mmc_test_cmds_during_tfr(test, 0, 1, 0);
+}
+
+/*
+ * Commands during read - use Set Block Count (CMD23).
+ */
+static int mmc_test_cmds_during_read_cmd23(struct mmc_test_card *test)
+{
+	return mmc_test_cmds_during_tfr(test, 1, 0, 0);
+}
+
+/*
+ * Commands during write - use Set Block Count (CMD23).
+ */
+static int mmc_test_cmds_during_write_cmd23(struct mmc_test_card *test)
+{
+	return mmc_test_cmds_during_tfr(test, 1, 1, 0);
+}
+
+/*
+ * Commands during non-blocking read - use Set Block Count (CMD23).
+ */
+static int mmc_test_cmds_during_read_cmd23_nonblock(struct mmc_test_card *test)
+{
+	return mmc_test_cmds_during_tfr(test, 1, 0, 1);
+}
+
+/*
+ * Commands during non-blocking write - use Set Block Count (CMD23).
+ */
+static int mmc_test_cmds_during_write_cmd23_nonblock(struct mmc_test_card *test)
+{
+	return mmc_test_cmds_during_tfr(test, 1, 1, 1);
+}
+
 static const struct mmc_test_case mmc_test_cases[] = {
 	{
 		.name = "Basic write (no data verification)",
@@ -2605,6 +2871,48 @@ static const struct mmc_test_case mmc_test_cases[] = {
 		.name = "Reset test",
 		.run = mmc_test_reset,
 	},
+
+	{
+		.name = "Commands during read - no Set Block Count (CMD23)",
+		.prepare = mmc_test_area_prepare,
+		.run = mmc_test_cmds_during_read,
+		.cleanup = mmc_test_area_cleanup,
+	},
+
+	{
+		.name = "Commands during write - no Set Block Count (CMD23)",
+		.prepare = mmc_test_area_prepare,
+		.run = mmc_test_cmds_during_write,
+		.cleanup = mmc_test_area_cleanup,
+	},
+
+	{
+		.name = "Commands during read - use Set Block Count (CMD23)",
+		.prepare = mmc_test_area_prepare,
+		.run = mmc_test_cmds_during_read_cmd23,
+		.cleanup = mmc_test_area_cleanup,
+	},
+
+	{
+		.name = "Commands during write - use Set Block Count (CMD23)",
+		.prepare = mmc_test_area_prepare,
+		.run = mmc_test_cmds_during_write_cmd23,
+		.cleanup = mmc_test_area_cleanup,
+	},
+
+	{
+		.name = "Commands during non-blocking read - use Set Block Count (CMD23)",
+		.prepare = mmc_test_area_prepare,
+		.run = mmc_test_cmds_during_read_cmd23_nonblock,
+		.cleanup = mmc_test_area_cleanup,
+	},
+
+	{
+		.name = "Commands during non-blocking write - use Set Block Count (CMD23)",
+		.prepare = mmc_test_area_prepare,
+		.run = mmc_test_cmds_during_write_cmd23_nonblock,
+		.cleanup = mmc_test_area_cleanup,
+	},
 };
 
 static DEFINE_MUTEX(mmc_test_lock);
diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c
index bf14642a576a..8037f73a109a 100644
--- a/drivers/mmc/card/queue.c
+++ b/drivers/mmc/card/queue.c
@@ -19,7 +19,9 @@
 
 #include <linux/mmc/card.h>
 #include <linux/mmc/host.h>
+
 #include "queue.h"
+#include "block.h"
 
 #define MMC_QUEUE_BOUNCESZ	65536
 
@@ -33,7 +35,8 @@ static int mmc_prep_request(struct request_queue *q, struct request *req)
 	/*
 	 * We only like normal block requests and discards.
 	 */
-	if (req->cmd_type != REQ_TYPE_FS && req_op(req) != REQ_OP_DISCARD) {
+	if (req->cmd_type != REQ_TYPE_FS && req_op(req) != REQ_OP_DISCARD &&
+	    req_op(req) != REQ_OP_SECURE_ERASE) {
 		blk_dump_rq_flags(req, "MMC bad request");
 		return BLKPREP_KILL;
 	}
@@ -64,8 +67,10 @@ static int mmc_queue_thread(void *d)
 		spin_unlock_irq(q->queue_lock);
 
 		if (req || mq->mqrq_prev->req) {
+			bool req_is_special = mmc_req_is_special(req);
+
 			set_current_state(TASK_RUNNING);
-			mq->issue_fn(mq, req);
+			mmc_blk_issue_rq(mq, req);
 			cond_resched();
 			if (mq->flags & MMC_QUEUE_NEW_REQUEST) {
 				mq->flags &= ~MMC_QUEUE_NEW_REQUEST;
@@ -79,7 +84,7 @@ static int mmc_queue_thread(void *d)
 			 * has been finished. Do not assign it to previous
 			 * request.
 			 */
-			if (mmc_req_is_special(req))
+			if (req_is_special)
 				mq->mqrq_cur->req = NULL;
 
 			mq->mqrq_prev->brq.mrq.data = NULL;
diff --git a/drivers/mmc/card/queue.h b/drivers/mmc/card/queue.h
index d62531124d54..3c15a75bae86 100644
--- a/drivers/mmc/card/queue.h
+++ b/drivers/mmc/card/queue.h
@@ -4,7 +4,9 @@
 static inline bool mmc_req_is_special(struct request *req)
 {
 	return req &&
-		(req_op(req) == REQ_OP_FLUSH || req_op(req) == REQ_OP_DISCARD);
+		(req_op(req) == REQ_OP_FLUSH ||
+		 req_op(req) == REQ_OP_DISCARD ||
+		 req_op(req) == REQ_OP_SECURE_ERASE);
 }
 
 struct request;
@@ -55,8 +57,6 @@ struct mmc_queue {
 	unsigned int		flags;
 #define MMC_QUEUE_SUSPENDED	(1 << 0)
 #define MMC_QUEUE_NEW_REQUEST	(1 << 1)
-
-	int			(*issue_fn)(struct mmc_queue *, struct request *);
 	void			*data;
 	struct request_queue	*queue;
 	struct mmc_queue_req	mqrq[2];
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index e55cde6d436d..2553d903a82b 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -58,6 +58,9 @@
  */
 #define MMC_BKOPS_MAX_TIMEOUT	(4 * 60 * 1000) /* max time to wait in ms */
 
+/* The max erase timeout, used when host->max_busy_timeout isn't specified */
+#define MMC_ERASE_TIMEOUT_MS	(60 * 1000) /* 60 s */
+
 static const unsigned freqs[] = { 400000, 300000, 200000, 100000 };
 
 /*
@@ -117,6 +120,24 @@ static inline void mmc_should_fail_request(struct mmc_host *host,
 
 #endif /* CONFIG_FAIL_MMC_REQUEST */
 
+static inline void mmc_complete_cmd(struct mmc_request *mrq)
+{
+	if (mrq->cap_cmd_during_tfr && !completion_done(&mrq->cmd_completion))
+		complete_all(&mrq->cmd_completion);
+}
+
+void mmc_command_done(struct mmc_host *host, struct mmc_request *mrq)
+{
+	if (!mrq->cap_cmd_during_tfr)
+		return;
+
+	mmc_complete_cmd(mrq);
+
+	pr_debug("%s: cmd done, tfr ongoing (CMD%u)\n",
+		 mmc_hostname(host), mrq->cmd->opcode);
+}
+EXPORT_SYMBOL(mmc_command_done);
+
 /**
  *	mmc_request_done - finish processing an MMC request
  *	@host: MMC host which completed request
@@ -143,6 +164,11 @@ void mmc_request_done(struct mmc_host *host, struct mmc_request *mrq)
 			cmd->retries = 0;
 	}
 
+	if (host->ongoing_mrq == mrq)
+		host->ongoing_mrq = NULL;
+
+	mmc_complete_cmd(mrq);
+
 	trace_mmc_request_done(host, mrq);
 
 	if (err && cmd->retries && !mmc_card_removed(host->card)) {
@@ -155,7 +181,8 @@ void mmc_request_done(struct mmc_host *host, struct mmc_request *mrq)
 	} else {
 		mmc_should_fail_request(host, mrq);
 
-		led_trigger_event(host->led, LED_OFF);
+		if (!host->ongoing_mrq)
+			led_trigger_event(host->led, LED_OFF);
 
 		if (mrq->sbc) {
 			pr_debug("%s: req done <CMD%u>: %d: %08x %08x %08x %08x\n",
@@ -220,6 +247,15 @@ static void __mmc_start_request(struct mmc_host *host, struct mmc_request *mrq)
 		}
 	}
 
+	if (mrq->cap_cmd_during_tfr) {
+		host->ongoing_mrq = mrq;
+		/*
+		 * Retry path could come through here without having waiting on
+		 * cmd_completion, so ensure it is reinitialised.
+		 */
+		reinit_completion(&mrq->cmd_completion);
+	}
+
 	trace_mmc_request_start(host, mrq);
 
 	host->ops->request(host, mrq);
@@ -386,6 +422,18 @@ static void mmc_wait_done(struct mmc_request *mrq)
 	complete(&mrq->completion);
 }
 
+static inline void mmc_wait_ongoing_tfr_cmd(struct mmc_host *host)
+{
+	struct mmc_request *ongoing_mrq = READ_ONCE(host->ongoing_mrq);
+
+	/*
+	 * If there is an ongoing transfer, wait for the command line to become
+	 * available.
+	 */
+	if (ongoing_mrq && !completion_done(&ongoing_mrq->cmd_completion))
+		wait_for_completion(&ongoing_mrq->cmd_completion);
+}
+
 /*
  *__mmc_start_data_req() - starts data request
  * @host: MMC host to start the request
@@ -393,17 +441,24 @@ static void mmc_wait_done(struct mmc_request *mrq)
  *
  * Sets the done callback to be called when request is completed by the card.
  * Starts data mmc request execution
+ * If an ongoing transfer is already in progress, wait for the command line
+ * to become available before sending another command.
  */
 static int __mmc_start_data_req(struct mmc_host *host, struct mmc_request *mrq)
 {
 	int err;
 
+	mmc_wait_ongoing_tfr_cmd(host);
+
 	mrq->done = mmc_wait_data_done;
 	mrq->host = host;
 
+	init_completion(&mrq->cmd_completion);
+
 	err = mmc_start_request(host, mrq);
 	if (err) {
 		mrq->cmd->error = err;
+		mmc_complete_cmd(mrq);
 		mmc_wait_data_done(mrq);
 	}
 
@@ -414,12 +469,17 @@ static int __mmc_start_req(struct mmc_host *host, struct mmc_request *mrq)
 {
 	int err;
 
+	mmc_wait_ongoing_tfr_cmd(host);
+
 	init_completion(&mrq->completion);
 	mrq->done = mmc_wait_done;
 
+	init_completion(&mrq->cmd_completion);
+
 	err = mmc_start_request(host, mrq);
 	if (err) {
 		mrq->cmd->error = err;
+		mmc_complete_cmd(mrq);
 		complete(&mrq->completion);
 	}
 
@@ -483,8 +543,7 @@ static int mmc_wait_for_data_req_done(struct mmc_host *host,
 	return err;
 }
 
-static void mmc_wait_for_req_done(struct mmc_host *host,
-				  struct mmc_request *mrq)
+void mmc_wait_for_req_done(struct mmc_host *host, struct mmc_request *mrq)
 {
 	struct mmc_command *cmd;
 
@@ -525,6 +584,28 @@ static void mmc_wait_for_req_done(struct mmc_host *host,
 
 	mmc_retune_release(host);
 }
+EXPORT_SYMBOL(mmc_wait_for_req_done);
+
+/**
+ *	mmc_is_req_done - Determine if a 'cap_cmd_during_tfr' request is done
+ *	@host: MMC host
+ *	@mrq: MMC request
+ *
+ *	mmc_is_req_done() is used with requests that have
+ *	mrq->cap_cmd_during_tfr = true. mmc_is_req_done() must be called after
+ *	starting a request and before waiting for it to complete. That is,
+ *	either in between calls to mmc_start_req(), or after mmc_wait_for_req()
+ *	and before mmc_wait_for_req_done(). If it is called at other times the
+ *	result is not meaningful.
+ */
+bool mmc_is_req_done(struct mmc_host *host, struct mmc_request *mrq)
+{
+	if (host->areq)
+		return host->context_info.is_done_rcv;
+	else
+		return completion_done(&mrq->completion);
+}
+EXPORT_SYMBOL(mmc_is_req_done);
 
 /**
  *	mmc_pre_req - Prepare for a new request
@@ -645,13 +726,18 @@ EXPORT_SYMBOL(mmc_start_req);
  *	@mrq: MMC request to start
  *
  *	Start a new MMC custom command request for a host, and wait
- *	for the command to complete. Does not attempt to parse the
- *	response.
+ *	for the command to complete. In the case of 'cap_cmd_during_tfr'
+ *	requests, the transfer is ongoing and the caller can issue further
+ *	commands that do not use the data lines, and then wait by calling
+ *	mmc_wait_for_req_done().
+ *	Does not attempt to parse the response.
  */
 void mmc_wait_for_req(struct mmc_host *host, struct mmc_request *mrq)
 {
 	__mmc_start_req(host, mrq);
-	mmc_wait_for_req_done(host, mrq);
+
+	if (!mrq->cap_cmd_during_tfr)
+		mmc_wait_for_req_done(host, mrq);
 }
 EXPORT_SYMBOL(mmc_wait_for_req);
 
@@ -2202,6 +2288,54 @@ out:
 	return err;
 }
 
+static unsigned int mmc_align_erase_size(struct mmc_card *card,
+					 unsigned int *from,
+					 unsigned int *to,
+					 unsigned int nr)
+{
+	unsigned int from_new = *from, nr_new = nr, rem;
+
+	/*
+	 * When the 'card->erase_size' is power of 2, we can use round_up/down()
+	 * to align the erase size efficiently.
+	 */
+	if (is_power_of_2(card->erase_size)) {
+		unsigned int temp = from_new;
+
+		from_new = round_up(temp, card->erase_size);
+		rem = from_new - temp;
+
+		if (nr_new > rem)
+			nr_new -= rem;
+		else
+			return 0;
+
+		nr_new = round_down(nr_new, card->erase_size);
+	} else {
+		rem = from_new % card->erase_size;
+		if (rem) {
+			rem = card->erase_size - rem;
+			from_new += rem;
+			if (nr_new > rem)
+				nr_new -= rem;
+			else
+				return 0;
+		}
+
+		rem = nr_new % card->erase_size;
+		if (rem)
+			nr_new -= rem;
+	}
+
+	if (nr_new == 0)
+		return 0;
+
+	*to = from_new + nr_new;
+	*from = from_new;
+
+	return nr_new;
+}
+
 /**
  * mmc_erase - erase sectors.
  * @card: card to erase
@@ -2240,26 +2374,12 @@ int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr,
 			return -EINVAL;
 	}
 
-	if (arg == MMC_ERASE_ARG) {
-		rem = from % card->erase_size;
-		if (rem) {
-			rem = card->erase_size - rem;
-			from += rem;
-			if (nr > rem)
-				nr -= rem;
-			else
-				return 0;
-		}
-		rem = nr % card->erase_size;
-		if (rem)
-			nr -= rem;
-	}
+	if (arg == MMC_ERASE_ARG)
+		nr = mmc_align_erase_size(card, &from, &to, nr);
 
 	if (nr == 0)
 		return 0;
 
-	to = from + nr;
-
 	if (to <= from)
 		return -EINVAL;
 
@@ -2352,6 +2472,8 @@ static unsigned int mmc_do_calc_max_discard(struct mmc_card *card,
 	struct mmc_host *host = card->host;
 	unsigned int max_discard, x, y, qty = 0, max_qty, min_qty, timeout;
 	unsigned int last_timeout = 0;
+	unsigned int max_busy_timeout = host->max_busy_timeout ?
+			host->max_busy_timeout : MMC_ERASE_TIMEOUT_MS;
 
 	if (card->erase_shift) {
 		max_qty = UINT_MAX >> card->erase_shift;
@@ -2374,15 +2496,15 @@ static unsigned int mmc_do_calc_max_discard(struct mmc_card *card,
 	 * matter what size of 'host->max_busy_timeout', but if the
 	 * 'host->max_busy_timeout' is large enough for more discard sectors,
 	 * then we can continue to increase the max discard sectors until we
-	 * get a balance value.
+	 * get a balance value. In cases when the 'host->max_busy_timeout'
+	 * isn't specified, use the default max erase timeout.
 	 */
 	do {
 		y = 0;
 		for (x = 1; x && x <= max_qty && max_qty - x >= qty; x <<= 1) {
 			timeout = mmc_erase_timeout(card, arg, qty + x);
 
-			if (qty + x > min_qty &&
-			    timeout > host->max_busy_timeout)
+			if (qty + x > min_qty && timeout > max_busy_timeout)
 				break;
 
 			if (timeout < last_timeout)
@@ -2427,9 +2549,6 @@ unsigned int mmc_calc_max_discard(struct mmc_card *card)
 	struct mmc_host *host = card->host;
 	unsigned int max_discard, max_trim;
 
-	if (!host->max_busy_timeout)
-		return UINT_MAX;
-
 	/*
 	 * Without erase_group_def set, MMC erase timeout depends on clock
 	 * frequence which can change.  In that case, the best choice is
@@ -2447,7 +2566,8 @@ unsigned int mmc_calc_max_discard(struct mmc_card *card)
 		max_discard = 0;
 	}
 	pr_debug("%s: calculated max. discard sectors %u for timeout %u ms\n",
-		 mmc_hostname(host), max_discard, host->max_busy_timeout);
+		mmc_hostname(host), max_discard, host->max_busy_timeout ?
+		host->max_busy_timeout : MMC_ERASE_TIMEOUT_MS);
 	return max_discard;
 }
 EXPORT_SYMBOL(mmc_calc_max_discard);
@@ -2456,7 +2576,8 @@ int mmc_set_blocklen(struct mmc_card *card, unsigned int blocklen)
 {
 	struct mmc_command cmd = {0};
 
-	if (mmc_card_blockaddr(card) || mmc_card_ddr52(card))
+	if (mmc_card_blockaddr(card) || mmc_card_ddr52(card) ||
+	    mmc_card_hs400(card) || mmc_card_hs400es(card))
 		return 0;
 
 	cmd.opcode = MMC_SET_BLOCKLEN;
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index f2d185cf8a8b..3486bc7fbb64 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -1029,6 +1029,10 @@ static int mmc_select_hs(struct mmc_card *card)
 		err = mmc_switch_status(card);
 	}
 
+	if (err)
+		pr_warn("%s: switch to high-speed failed, err:%d\n",
+			mmc_hostname(card->host), err);
+
 	return err;
 }
 
@@ -1265,11 +1269,8 @@ static int mmc_select_hs400es(struct mmc_card *card)
 
 	/* Switch card to HS mode */
 	err = mmc_select_hs(card);
-	if (err) {
-		pr_err("%s: switch to high-speed failed, err:%d\n",
-			mmc_hostname(host), err);
+	if (err)
 		goto out_err;
-	}
 
 	err = mmc_switch_status(card);
 	if (err)
diff --git a/drivers/mmc/core/pwrseq_simple.c b/drivers/mmc/core/pwrseq_simple.c
index 450d907c6e6c..1304160de168 100644
--- a/drivers/mmc/core/pwrseq_simple.c
+++ b/drivers/mmc/core/pwrseq_simple.c
@@ -16,6 +16,8 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/gpio/consumer.h>
+#include <linux/delay.h>
+#include <linux/property.h>
 
 #include <linux/mmc/host.h>
 
@@ -24,6 +26,7 @@
 struct mmc_pwrseq_simple {
 	struct mmc_pwrseq pwrseq;
 	bool clk_enabled;
+	u32 post_power_on_delay_ms;
 	struct clk *ext_clk;
 	struct gpio_descs *reset_gpios;
 };
@@ -64,6 +67,9 @@ static void mmc_pwrseq_simple_post_power_on(struct mmc_host *host)
 	struct mmc_pwrseq_simple *pwrseq = to_pwrseq_simple(host->pwrseq);
 
 	mmc_pwrseq_simple_set_gpios_value(pwrseq, 0);
+
+	if (pwrseq->post_power_on_delay_ms)
+		msleep(pwrseq->post_power_on_delay_ms);
 }
 
 static void mmc_pwrseq_simple_power_off(struct mmc_host *host)
@@ -111,6 +117,9 @@ static int mmc_pwrseq_simple_probe(struct platform_device *pdev)
 		return PTR_ERR(pwrseq->reset_gpios);
 	}
 
+	device_property_read_u32(dev, "post-power-on-delay-ms",
+				 &pwrseq->post_power_on_delay_ms);
+
 	pwrseq->pwrseq.dev = dev;
 	pwrseq->pwrseq.ops = &mmc_pwrseq_simple_ops;
 	pwrseq->pwrseq.owner = THIS_MODULE;
diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c
index 0123936241b0..73c762a28dfe 100644
--- a/drivers/mmc/core/sd.c
+++ b/drivers/mmc/core/sd.c
@@ -223,8 +223,7 @@ static int mmc_decode_scr(struct mmc_card *card)
 static int mmc_read_ssr(struct mmc_card *card)
 {
 	unsigned int au, es, et, eo;
-	int err, i;
-	u32 *ssr;
+	int i;
 
 	if (!(card->csd.cmdclass & CCC_APP_SPEC)) {
 		pr_warn("%s: card lacks mandatory SD Status function\n",
@@ -232,33 +231,27 @@ static int mmc_read_ssr(struct mmc_card *card)
 		return 0;
 	}
 
-	ssr = kmalloc(64, GFP_KERNEL);
-	if (!ssr)
-		return -ENOMEM;
-
-	err = mmc_app_sd_status(card, ssr);
-	if (err) {
+	if (mmc_app_sd_status(card, card->raw_ssr)) {
 		pr_warn("%s: problem reading SD Status register\n",
 			mmc_hostname(card->host));
-		err = 0;
-		goto out;
+		return 0;
 	}
 
 	for (i = 0; i < 16; i++)
-		ssr[i] = be32_to_cpu(ssr[i]);
+		card->raw_ssr[i] = be32_to_cpu(card->raw_ssr[i]);
 
 	/*
 	 * UNSTUFF_BITS only works with four u32s so we have to offset the
 	 * bitfield positions accordingly.
 	 */
-	au = UNSTUFF_BITS(ssr, 428 - 384, 4);
+	au = UNSTUFF_BITS(card->raw_ssr, 428 - 384, 4);
 	if (au) {
 		if (au <= 9 || card->scr.sda_spec3) {
 			card->ssr.au = sd_au_size[au];
-			es = UNSTUFF_BITS(ssr, 408 - 384, 16);
-			et = UNSTUFF_BITS(ssr, 402 - 384, 6);
+			es = UNSTUFF_BITS(card->raw_ssr, 408 - 384, 16);
+			et = UNSTUFF_BITS(card->raw_ssr, 402 - 384, 6);
 			if (es && et) {
-				eo = UNSTUFF_BITS(ssr, 400 - 384, 2);
+				eo = UNSTUFF_BITS(card->raw_ssr, 400 - 384, 2);
 				card->ssr.erase_timeout = (et * 1000) / es;
 				card->ssr.erase_offset = eo * 1000;
 			}
@@ -267,9 +260,8 @@ static int mmc_read_ssr(struct mmc_card *card)
 				mmc_hostname(card->host));
 		}
 	}
-out:
-	kfree(ssr);
-	return err;
+
+	return 0;
 }
 
 /*
@@ -666,6 +658,14 @@ MMC_DEV_ATTR(cid, "%08x%08x%08x%08x\n", card->raw_cid[0], card->raw_cid[1],
 MMC_DEV_ATTR(csd, "%08x%08x%08x%08x\n", card->raw_csd[0], card->raw_csd[1],
 	card->raw_csd[2], card->raw_csd[3]);
 MMC_DEV_ATTR(scr, "%08x%08x\n", card->raw_scr[0], card->raw_scr[1]);
+MMC_DEV_ATTR(ssr,
+	"%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x\n",
+		card->raw_ssr[0], card->raw_ssr[1], card->raw_ssr[2],
+		card->raw_ssr[3], card->raw_ssr[4], card->raw_ssr[5],
+		card->raw_ssr[6], card->raw_ssr[7], card->raw_ssr[8],
+		card->raw_ssr[9], card->raw_ssr[10], card->raw_ssr[11],
+		card->raw_ssr[12], card->raw_ssr[13], card->raw_ssr[14],
+		card->raw_ssr[15]);
 MMC_DEV_ATTR(date, "%02d/%04d\n", card->cid.month, card->cid.year);
 MMC_DEV_ATTR(erase_size, "%u\n", card->erase_size << 9);
 MMC_DEV_ATTR(preferred_erase_size, "%u\n", card->pref_erase << 9);
@@ -698,6 +698,7 @@ static struct attribute *sd_std_attrs[] = {
 	&dev_attr_cid.attr,
 	&dev_attr_csd.attr,
 	&dev_attr_scr.attr,
+	&dev_attr_ssr.attr,
 	&dev_attr_date.attr,
 	&dev_attr_erase_size.attr,
 	&dev_attr_preferred_erase_size.attr,
diff --git a/drivers/mmc/core/sdio_io.c b/drivers/mmc/core/sdio_io.c
index 78cb4d5d9d58..406e5f037e32 100644
--- a/drivers/mmc/core/sdio_io.c
+++ b/drivers/mmc/core/sdio_io.c
@@ -26,8 +26,8 @@
  */
 void sdio_claim_host(struct sdio_func *func)
 {
-	BUG_ON(!func);
-	BUG_ON(!func->card);
+	if (WARN_ON(!func))
+		return;
 
 	mmc_claim_host(func->card->host);
 }
@@ -42,8 +42,8 @@ EXPORT_SYMBOL_GPL(sdio_claim_host);
  */
 void sdio_release_host(struct sdio_func *func)
 {
-	BUG_ON(!func);
-	BUG_ON(!func->card);
+	if (WARN_ON(!func))
+		return;
 
 	mmc_release_host(func->card->host);
 }
@@ -62,8 +62,8 @@ int sdio_enable_func(struct sdio_func *func)
 	unsigned char reg;
 	unsigned long timeout;
 
-	BUG_ON(!func);
-	BUG_ON(!func->card);
+	if (!func)
+		return -EINVAL;
 
 	pr_debug("SDIO: Enabling device %s...\n", sdio_func_id(func));
 
@@ -112,8 +112,8 @@ int sdio_disable_func(struct sdio_func *func)
 	int ret;
 	unsigned char reg;
 
-	BUG_ON(!func);
-	BUG_ON(!func->card);
+	if (!func)
+		return -EINVAL;
 
 	pr_debug("SDIO: Disabling device %s...\n", sdio_func_id(func));
 
@@ -307,6 +307,9 @@ static int sdio_io_rw_ext_helper(struct sdio_func *func, int write,
 	unsigned max_blocks;
 	int ret;
 
+	if (!func || (func->num > 7))
+		return -EINVAL;
+
 	/* Do the bulk of the transfer using block mode (if supported). */
 	if (func->card->cccr.multi_block && (size > sdio_max_byte_size(func))) {
 		/* Blocks per command is limited by host count, host transfer
@@ -367,7 +370,10 @@ u8 sdio_readb(struct sdio_func *func, unsigned int addr, int *err_ret)
 	int ret;
 	u8 val;
 
-	BUG_ON(!func);
+	if (!func) {
+		*err_ret = -EINVAL;
+		return 0xFF;
+	}
 
 	if (err_ret)
 		*err_ret = 0;
@@ -398,7 +404,10 @@ void sdio_writeb(struct sdio_func *func, u8 b, unsigned int addr, int *err_ret)
 {
 	int ret;
 
-	BUG_ON(!func);
+	if (!func) {
+		*err_ret = -EINVAL;
+		return;
+	}
 
 	ret = mmc_io_rw_direct(func->card, 1, func->num, addr, b, NULL);
 	if (err_ret)
@@ -623,7 +632,10 @@ unsigned char sdio_f0_readb(struct sdio_func *func, unsigned int addr,
 	int ret;
 	unsigned char val;
 
-	BUG_ON(!func);
+	if (!func) {
+		*err_ret = -EINVAL;
+		return 0xFF;
+	}
 
 	if (err_ret)
 		*err_ret = 0;
@@ -658,7 +670,10 @@ void sdio_f0_writeb(struct sdio_func *func, unsigned char b, unsigned int addr,
 {
 	int ret;
 
-	BUG_ON(!func);
+	if (!func) {
+		*err_ret = -EINVAL;
+		return;
+	}
 
 	if ((addr < 0xF0 || addr > 0xFF) && (!mmc_card_lenient_fn0(func->card))) {
 		if (err_ret)
@@ -684,8 +699,8 @@ EXPORT_SYMBOL_GPL(sdio_f0_writeb);
  */
 mmc_pm_flag_t sdio_get_host_pm_caps(struct sdio_func *func)
 {
-	BUG_ON(!func);
-	BUG_ON(!func->card);
+	if (!func)
+		return 0;
 
 	return func->card->host->pm_caps;
 }
@@ -707,8 +722,8 @@ int sdio_set_host_pm_flags(struct sdio_func *func, mmc_pm_flag_t flags)
 {
 	struct mmc_host *host;
 
-	BUG_ON(!func);
-	BUG_ON(!func->card);
+	if (!func)
+		return -EINVAL;
 
 	host = func->card->host;
 
diff --git a/drivers/mmc/core/sdio_ops.c b/drivers/mmc/core/sdio_ops.c
index 34f6e8015306..90fe5545c677 100644
--- a/drivers/mmc/core/sdio_ops.c
+++ b/drivers/mmc/core/sdio_ops.c
@@ -24,8 +24,6 @@ int mmc_send_io_op_cond(struct mmc_host *host, u32 ocr, u32 *rocr)
 	struct mmc_command cmd = {0};
 	int i, err = 0;
 
-	BUG_ON(!host);
-
 	cmd.opcode = SD_IO_SEND_OP_COND;
 	cmd.arg = ocr;
 	cmd.flags = MMC_RSP_SPI_R4 | MMC_RSP_R4 | MMC_CMD_BCR;
@@ -71,8 +69,8 @@ static int mmc_io_rw_direct_host(struct mmc_host *host, int write, unsigned fn,
 	struct mmc_command cmd = {0};
 	int err;
 
-	BUG_ON(!host);
-	BUG_ON(fn > 7);
+	if (fn > 7)
+		return -EINVAL;
 
 	/* sanity check */
 	if (addr & ~0x1FFFF)
@@ -114,7 +112,6 @@ static int mmc_io_rw_direct_host(struct mmc_host *host, int write, unsigned fn,
 int mmc_io_rw_direct(struct mmc_card *card, int write, unsigned fn,
 	unsigned addr, u8 in, u8 *out)
 {
-	BUG_ON(!card);
 	return mmc_io_rw_direct_host(card->host, write, fn, addr, in, out);
 }
 
@@ -129,8 +126,6 @@ int mmc_io_rw_extended(struct mmc_card *card, int write, unsigned fn,
 	unsigned int nents, left_size, i;
 	unsigned int seg_size = card->host->max_seg_size;
 
-	BUG_ON(!card);
-	BUG_ON(fn > 7);
 	WARN_ON(blksz == 0);
 
 	/* sanity check */
diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c
index a56373c75983..8fa478c3b0db 100644
--- a/drivers/mmc/host/davinci_mmc.c
+++ b/drivers/mmc/host/davinci_mmc.c
@@ -1216,9 +1216,11 @@ static int __init davinci_mmcsd_probe(struct platform_device *pdev)
 	}
 
 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	irq = platform_get_irq(pdev, 0);
-	if (!r || irq == NO_IRQ)
+	if (!r)
 		return -ENODEV;
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
 
 	mem_size = resource_size(r);
 	mem = devm_request_mem_region(&pdev->dev, r->start, mem_size,
diff --git a/drivers/mmc/host/dw_mmc-exynos.c b/drivers/mmc/host/dw_mmc-exynos.c
index da0ef1765735..7ab3d749b5ae 100644
--- a/drivers/mmc/host/dw_mmc-exynos.c
+++ b/drivers/mmc/host/dw_mmc-exynos.c
@@ -225,8 +225,12 @@ static void dw_mci_exynos_config_hs400(struct dw_mci *host, u32 timing)
 	 * Not supported to configure register
 	 * related to HS400
 	 */
-	if (priv->ctrl_type < DW_MCI_TYPE_EXYNOS5420)
+	if (priv->ctrl_type < DW_MCI_TYPE_EXYNOS5420) {
+		if (timing == MMC_TIMING_MMC_HS400)
+			dev_warn(host->dev,
+				 "cannot configure HS400, unsupported chipset\n");
 		return;
+	}
 
 	dqs = priv->saved_dqs_en;
 	strobe = priv->saved_strobe_ctrl;
diff --git a/drivers/mmc/host/dw_mmc-k3.c b/drivers/mmc/host/dw_mmc-k3.c
index 8e9d886bfcda..624789496dce 100644
--- a/drivers/mmc/host/dw_mmc-k3.c
+++ b/drivers/mmc/host/dw_mmc-k3.c
@@ -131,11 +131,17 @@ static void dw_mci_hi6220_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 	host->bus_hz = clk_get_rate(host->biu_clk);
 }
 
+static int dw_mci_hi6220_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+{
+	return 0;
+}
+
 static const struct dw_mci_drv_data hi6220_data = {
 	.caps			= dw_mci_hi6220_caps,
 	.switch_voltage		= dw_mci_hi6220_switch_voltage,
 	.set_ios		= dw_mci_hi6220_set_ios,
 	.parse_dt		= dw_mci_hi6220_parse_dt,
+	.execute_tuning		= dw_mci_hi6220_execute_tuning,
 };
 
 static const struct of_device_id dw_mci_k3_match[] = {
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index 32380d5d4f6b..4fcbc4012ed0 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -61,6 +61,8 @@
 				 SDMMC_IDMAC_INT_FBE | SDMMC_IDMAC_INT_RI | \
 				 SDMMC_IDMAC_INT_TI)
 
+#define DESC_RING_BUF_SZ	PAGE_SIZE
+
 struct idmac_desc_64addr {
 	u32		des0;	/* Control Descriptor */
 
@@ -467,136 +469,6 @@ static void dw_mci_dmac_complete_dma(void *arg)
 	}
 }
 
-static void dw_mci_translate_sglist(struct dw_mci *host, struct mmc_data *data,
-				    unsigned int sg_len)
-{
-	unsigned int desc_len;
-	int i;
-
-	if (host->dma_64bit_address == 1) {
-		struct idmac_desc_64addr *desc_first, *desc_last, *desc;
-
-		desc_first = desc_last = desc = host->sg_cpu;
-
-		for (i = 0; i < sg_len; i++) {
-			unsigned int length = sg_dma_len(&data->sg[i]);
-
-			u64 mem_addr = sg_dma_address(&data->sg[i]);
-
-			for ( ; length ; desc++) {
-				desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
-					   length : DW_MCI_DESC_DATA_LENGTH;
-
-				length -= desc_len;
-
-				/*
-				 * Set the OWN bit and disable interrupts
-				 * for this descriptor
-				 */
-				desc->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC |
-							IDMAC_DES0_CH;
-
-				/* Buffer length */
-				IDMAC_64ADDR_SET_BUFFER1_SIZE(desc, desc_len);
-
-				/* Physical address to DMA to/from */
-				desc->des4 = mem_addr & 0xffffffff;
-				desc->des5 = mem_addr >> 32;
-
-				/* Update physical address for the next desc */
-				mem_addr += desc_len;
-
-				/* Save pointer to the last descriptor */
-				desc_last = desc;
-			}
-		}
-
-		/* Set first descriptor */
-		desc_first->des0 |= IDMAC_DES0_FD;
-
-		/* Set last descriptor */
-		desc_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC);
-		desc_last->des0 |= IDMAC_DES0_LD;
-
-	} else {
-		struct idmac_desc *desc_first, *desc_last, *desc;
-
-		desc_first = desc_last = desc = host->sg_cpu;
-
-		for (i = 0; i < sg_len; i++) {
-			unsigned int length = sg_dma_len(&data->sg[i]);
-
-			u32 mem_addr = sg_dma_address(&data->sg[i]);
-
-			for ( ; length ; desc++) {
-				desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
-					   length : DW_MCI_DESC_DATA_LENGTH;
-
-				length -= desc_len;
-
-				/*
-				 * Set the OWN bit and disable interrupts
-				 * for this descriptor
-				 */
-				desc->des0 = cpu_to_le32(IDMAC_DES0_OWN |
-							 IDMAC_DES0_DIC |
-							 IDMAC_DES0_CH);
-
-				/* Buffer length */
-				IDMAC_SET_BUFFER1_SIZE(desc, desc_len);
-
-				/* Physical address to DMA to/from */
-				desc->des2 = cpu_to_le32(mem_addr);
-
-				/* Update physical address for the next desc */
-				mem_addr += desc_len;
-
-				/* Save pointer to the last descriptor */
-				desc_last = desc;
-			}
-		}
-
-		/* Set first descriptor */
-		desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD);
-
-		/* Set last descriptor */
-		desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH |
-					       IDMAC_DES0_DIC));
-		desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD);
-	}
-
-	wmb(); /* drain writebuffer */
-}
-
-static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len)
-{
-	u32 temp;
-
-	dw_mci_translate_sglist(host, host->data, sg_len);
-
-	/* Make sure to reset DMA in case we did PIO before this */
-	dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET);
-	dw_mci_idmac_reset(host);
-
-	/* Select IDMAC interface */
-	temp = mci_readl(host, CTRL);
-	temp |= SDMMC_CTRL_USE_IDMAC;
-	mci_writel(host, CTRL, temp);
-
-	/* drain writebuffer */
-	wmb();
-
-	/* Enable the IDMAC */
-	temp = mci_readl(host, BMOD);
-	temp |= SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB;
-	mci_writel(host, BMOD, temp);
-
-	/* Start it running */
-	mci_writel(host, PLDMND, 1);
-
-	return 0;
-}
-
 static int dw_mci_idmac_init(struct dw_mci *host)
 {
 	int i;
@@ -604,7 +476,8 @@ static int dw_mci_idmac_init(struct dw_mci *host)
 	if (host->dma_64bit_address == 1) {
 		struct idmac_desc_64addr *p;
 		/* Number of descriptors in the ring buffer */
-		host->ring_size = PAGE_SIZE / sizeof(struct idmac_desc_64addr);
+		host->ring_size =
+			DESC_RING_BUF_SZ / sizeof(struct idmac_desc_64addr);
 
 		/* Forward link the descriptor list */
 		for (i = 0, p = host->sg_cpu; i < host->ring_size - 1;
@@ -630,7 +503,8 @@ static int dw_mci_idmac_init(struct dw_mci *host)
 	} else {
 		struct idmac_desc *p;
 		/* Number of descriptors in the ring buffer */
-		host->ring_size = PAGE_SIZE / sizeof(struct idmac_desc);
+		host->ring_size =
+			DESC_RING_BUF_SZ / sizeof(struct idmac_desc);
 
 		/* Forward link the descriptor list */
 		for (i = 0, p = host->sg_cpu;
@@ -671,6 +545,195 @@ static int dw_mci_idmac_init(struct dw_mci *host)
 	return 0;
 }
 
+static inline int dw_mci_prepare_desc64(struct dw_mci *host,
+					 struct mmc_data *data,
+					 unsigned int sg_len)
+{
+	unsigned int desc_len;
+	struct idmac_desc_64addr *desc_first, *desc_last, *desc;
+	unsigned long timeout;
+	int i;
+
+	desc_first = desc_last = desc = host->sg_cpu;
+
+	for (i = 0; i < sg_len; i++) {
+		unsigned int length = sg_dma_len(&data->sg[i]);
+
+		u64 mem_addr = sg_dma_address(&data->sg[i]);
+
+		for ( ; length ; desc++) {
+			desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
+				   length : DW_MCI_DESC_DATA_LENGTH;
+
+			length -= desc_len;
+
+			/*
+			 * Wait for the former clear OWN bit operation
+			 * of IDMAC to make sure that this descriptor
+			 * isn't still owned by IDMAC as IDMAC's write
+			 * ops and CPU's read ops are asynchronous.
+			 */
+			timeout = jiffies + msecs_to_jiffies(100);
+			while (readl(&desc->des0) & IDMAC_DES0_OWN) {
+				if (time_after(jiffies, timeout))
+					goto err_own_bit;
+				udelay(10);
+			}
+
+			/*
+			 * Set the OWN bit and disable interrupts
+			 * for this descriptor
+			 */
+			desc->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC |
+						IDMAC_DES0_CH;
+
+			/* Buffer length */
+			IDMAC_64ADDR_SET_BUFFER1_SIZE(desc, desc_len);
+
+			/* Physical address to DMA to/from */
+			desc->des4 = mem_addr & 0xffffffff;
+			desc->des5 = mem_addr >> 32;
+
+			/* Update physical address for the next desc */
+			mem_addr += desc_len;
+
+			/* Save pointer to the last descriptor */
+			desc_last = desc;
+		}
+	}
+
+	/* Set first descriptor */
+	desc_first->des0 |= IDMAC_DES0_FD;
+
+	/* Set last descriptor */
+	desc_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC);
+	desc_last->des0 |= IDMAC_DES0_LD;
+
+	return 0;
+err_own_bit:
+	/* restore the descriptor chain as it's polluted */
+	dev_dbg(host->dev, "desciptor is still owned by IDMAC.\n");
+	memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);
+	dw_mci_idmac_init(host);
+	return -EINVAL;
+}
+
+
+static inline int dw_mci_prepare_desc32(struct dw_mci *host,
+					 struct mmc_data *data,
+					 unsigned int sg_len)
+{
+	unsigned int desc_len;
+	struct idmac_desc *desc_first, *desc_last, *desc;
+	unsigned long timeout;
+	int i;
+
+	desc_first = desc_last = desc = host->sg_cpu;
+
+	for (i = 0; i < sg_len; i++) {
+		unsigned int length = sg_dma_len(&data->sg[i]);
+
+		u32 mem_addr = sg_dma_address(&data->sg[i]);
+
+		for ( ; length ; desc++) {
+			desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
+				   length : DW_MCI_DESC_DATA_LENGTH;
+
+			length -= desc_len;
+
+			/*
+			 * Wait for the former clear OWN bit operation
+			 * of IDMAC to make sure that this descriptor
+			 * isn't still owned by IDMAC as IDMAC's write
+			 * ops and CPU's read ops are asynchronous.
+			 */
+			timeout = jiffies + msecs_to_jiffies(100);
+			while (readl(&desc->des0) &
+			       cpu_to_le32(IDMAC_DES0_OWN)) {
+				if (time_after(jiffies, timeout))
+					goto err_own_bit;
+				udelay(10);
+			}
+
+			/*
+			 * Set the OWN bit and disable interrupts
+			 * for this descriptor
+			 */
+			desc->des0 = cpu_to_le32(IDMAC_DES0_OWN |
+						 IDMAC_DES0_DIC |
+						 IDMAC_DES0_CH);
+
+			/* Buffer length */
+			IDMAC_SET_BUFFER1_SIZE(desc, desc_len);
+
+			/* Physical address to DMA to/from */
+			desc->des2 = cpu_to_le32(mem_addr);
+
+			/* Update physical address for the next desc */
+			mem_addr += desc_len;
+
+			/* Save pointer to the last descriptor */
+			desc_last = desc;
+		}
+	}
+
+	/* Set first descriptor */
+	desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD);
+
+	/* Set last descriptor */
+	desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH |
+				       IDMAC_DES0_DIC));
+	desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD);
+
+	return 0;
+err_own_bit:
+	/* restore the descriptor chain as it's polluted */
+	dev_dbg(host->dev, "desciptor is still owned by IDMAC.\n");
+	memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);
+	dw_mci_idmac_init(host);
+	return -EINVAL;
+}
+
+static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len)
+{
+	u32 temp;
+	int ret;
+
+	if (host->dma_64bit_address == 1)
+		ret = dw_mci_prepare_desc64(host, host->data, sg_len);
+	else
+		ret = dw_mci_prepare_desc32(host, host->data, sg_len);
+
+	if (ret)
+		goto out;
+
+	/* drain writebuffer */
+	wmb();
+
+	/* Make sure to reset DMA in case we did PIO before this */
+	dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET);
+	dw_mci_idmac_reset(host);
+
+	/* Select IDMAC interface */
+	temp = mci_readl(host, CTRL);
+	temp |= SDMMC_CTRL_USE_IDMAC;
+	mci_writel(host, CTRL, temp);
+
+	/* drain writebuffer */
+	wmb();
+
+	/* Enable the IDMAC */
+	temp = mci_readl(host, BMOD);
+	temp |= SDMMC_IDMAC_ENABLE | SDMMC_IDMAC_FB;
+	mci_writel(host, BMOD, temp);
+
+	/* Start it running */
+	mci_writel(host, PLDMND, 1);
+
+out:
+	return ret;
+}
+
 static const struct dw_mci_dma_ops dw_mci_idmac_ops = {
 	.init = dw_mci_idmac_init,
 	.start = dw_mci_idmac_start_dma,
@@ -876,11 +939,8 @@ static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data)
 	 * MSIZE is '1',
 	 * if blksz is not a multiple of the FIFO width
 	 */
-	if (blksz % fifo_width) {
-		msize = 0;
-		rx_wmark = 1;
+	if (blksz % fifo_width)
 		goto done;
-	}
 
 	do {
 		if (!((blksz_depth % mszs[idx]) ||
@@ -998,8 +1058,10 @@ static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data)
 	spin_unlock_irqrestore(&host->irq_lock, irqflags);
 
 	if (host->dma_ops->start(host, sg_len)) {
-		/* We can't do DMA */
-		dev_err(host->dev, "%s: failed to start DMA.\n", __func__);
+		/* We can't do DMA, try PIO for this one */
+		dev_dbg(host->dev,
+			"%s: fall back to PIO mode for current transfer\n",
+			__func__);
 		return -ENODEV;
 	}
 
@@ -1112,11 +1174,12 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 
 		div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0;
 
-		dev_info(&slot->mmc->class_dev,
-			 "Bus speed (slot %d) = %dHz (slot req %dHz, actual %dHZ div = %d)\n",
-			 slot->id, host->bus_hz, clock,
-			 div ? ((host->bus_hz / div) >> 1) :
-			 host->bus_hz, div);
+		if (clock != slot->__clk_old || force_clkinit)
+			dev_info(&slot->mmc->class_dev,
+				 "Bus speed (slot %d) = %dHz (slot req %dHz, actual %dHZ div = %d)\n",
+				 slot->id, host->bus_hz, clock,
+				 div ? ((host->bus_hz / div) >> 1) :
+				 host->bus_hz, div);
 
 		/* disable clock */
 		mci_writel(host, CLKENA, 0);
@@ -1139,6 +1202,9 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 
 		/* inform CIU */
 		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
+
+		/* keep the last clock value that was requested from core */
+		slot->__clk_old = clock;
 	}
 
 	host->current_speed = clock;
@@ -1691,11 +1757,11 @@ static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data)
 				data->error = -ETIMEDOUT;
 			} else if (host->dir_status ==
 					DW_MCI_RECV_STATUS) {
-				data->error = -EIO;
+				data->error = -EILSEQ;
 			}
 		} else {
 			/* SDMMC_INT_SBE is included */
-			data->error = -EIO;
+			data->error = -EILSEQ;
 		}
 
 		dev_dbg(host->dev, "data error, status 0x%08x\n", status);
@@ -2523,47 +2589,6 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-#ifdef CONFIG_OF
-/* given a slot, find out the device node representing that slot */
-static struct device_node *dw_mci_of_find_slot_node(struct dw_mci_slot *slot)
-{
-	struct device *dev = slot->mmc->parent;
-	struct device_node *np;
-	const __be32 *addr;
-	int len;
-
-	if (!dev || !dev->of_node)
-		return NULL;
-
-	for_each_child_of_node(dev->of_node, np) {
-		addr = of_get_property(np, "reg", &len);
-		if (!addr || (len < sizeof(int)))
-			continue;
-		if (be32_to_cpup(addr) == slot->id)
-			return np;
-	}
-	return NULL;
-}
-
-static void dw_mci_slot_of_parse(struct dw_mci_slot *slot)
-{
-	struct device_node *np = dw_mci_of_find_slot_node(slot);
-
-	if (!np)
-		return;
-
-	if (of_property_read_bool(np, "disable-wp")) {
-		slot->mmc->caps2 |= MMC_CAP2_NO_WRITE_PROTECT;
-		dev_warn(slot->mmc->parent,
-			"Slot quirk 'disable-wp' is deprecated\n");
-	}
-}
-#else /* CONFIG_OF */
-static void dw_mci_slot_of_parse(struct dw_mci_slot *slot)
-{
-}
-#endif /* CONFIG_OF */
-
 static int dw_mci_init_slot(struct dw_mci *host, unsigned int id)
 {
 	struct mmc_host *mmc;
@@ -2626,8 +2651,6 @@ static int dw_mci_init_slot(struct dw_mci *host, unsigned int id)
 	if (host->pdata->caps2)
 		mmc->caps2 = host->pdata->caps2;
 
-	dw_mci_slot_of_parse(slot);
-
 	ret = mmc_of_parse(mmc);
 	if (ret)
 		goto err_host_allocated;
@@ -2732,7 +2755,8 @@ static void dw_mci_init_dma(struct dw_mci *host)
 		}
 
 		/* Alloc memory for sg translation */
-		host->sg_cpu = dmam_alloc_coherent(host->dev, PAGE_SIZE,
+		host->sg_cpu = dmam_alloc_coherent(host->dev,
+						   DESC_RING_BUF_SZ,
 						   &host->sg_dma, GFP_KERNEL);
 		if (!host->sg_cpu) {
 			dev_err(host->dev,
@@ -2915,6 +2939,13 @@ static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)
 	if (!pdata)
 		return ERR_PTR(-ENOMEM);
 
+	/* find reset controller when exist */
+	pdata->rstc = devm_reset_control_get_optional(dev, NULL);
+	if (IS_ERR(pdata->rstc)) {
+		if (PTR_ERR(pdata->rstc) == -EPROBE_DEFER)
+			return ERR_PTR(-EPROBE_DEFER);
+	}
+
 	/* find out number of slots supported */
 	of_property_read_u32(np, "num-slots", &pdata->num_slots);
 
@@ -2933,11 +2964,6 @@ static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)
 			return ERR_PTR(ret);
 	}
 
-	if (of_find_property(np, "supports-highspeed", NULL)) {
-		dev_info(dev, "supports-highspeed property is deprecated.\n");
-		pdata->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;
-	}
-
 	return pdata;
 }
 
@@ -2986,7 +3012,9 @@ int dw_mci_probe(struct dw_mci *host)
 
 	if (!host->pdata) {
 		host->pdata = dw_mci_parse_dt(host);
-		if (IS_ERR(host->pdata)) {
+		if (PTR_ERR(host->pdata) == -EPROBE_DEFER) {
+			return -EPROBE_DEFER;
+		} else if (IS_ERR(host->pdata)) {
 			dev_err(host->dev, "platform data not available\n");
 			return -EINVAL;
 		}
@@ -3040,6 +3068,12 @@ int dw_mci_probe(struct dw_mci *host)
 		}
 	}
 
+	if (!IS_ERR(host->pdata->rstc)) {
+		reset_control_assert(host->pdata->rstc);
+		usleep_range(10, 50);
+		reset_control_deassert(host->pdata->rstc);
+	}
+
 	setup_timer(&host->cmd11_timer,
 		    dw_mci_cmd11_timer, (unsigned long)host);
 
@@ -3189,13 +3223,14 @@ err_dmaunmap:
 	if (host->use_dma && host->dma_ops->exit)
 		host->dma_ops->exit(host);
 
+	if (!IS_ERR(host->pdata->rstc))
+		reset_control_assert(host->pdata->rstc);
+
 err_clk_ciu:
-	if (!IS_ERR(host->ciu_clk))
-		clk_disable_unprepare(host->ciu_clk);
+	clk_disable_unprepare(host->ciu_clk);
 
 err_clk_biu:
-	if (!IS_ERR(host->biu_clk))
-		clk_disable_unprepare(host->biu_clk);
+	clk_disable_unprepare(host->biu_clk);
 
 	return ret;
 }
@@ -3221,11 +3256,11 @@ void dw_mci_remove(struct dw_mci *host)
 	if (host->use_dma && host->dma_ops->exit)
 		host->dma_ops->exit(host);
 
-	if (!IS_ERR(host->ciu_clk))
-		clk_disable_unprepare(host->ciu_clk);
+	if (!IS_ERR(host->pdata->rstc))
+		reset_control_assert(host->pdata->rstc);
 
-	if (!IS_ERR(host->biu_clk))
-		clk_disable_unprepare(host->biu_clk);
+	clk_disable_unprepare(host->ciu_clk);
+	clk_disable_unprepare(host->biu_clk);
 }
 EXPORT_SYMBOL(dw_mci_remove);
 
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index 9e740bc232a8..e8cd2dec3263 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -249,6 +249,8 @@ extern int dw_mci_resume(struct dw_mci *host);
  * @queue_node: List node for placing this node in the @queue list of
  *	&struct dw_mci.
  * @clock: Clock rate configured by set_ios(). Protected by host->lock.
+ * @__clk_old: The last clock value that was requested from core.
+ *	Keeping track of this helps us to avoid spamming the console.
  * @flags: Random state bits associated with the slot.
  * @id: Number of this slot.
  * @sdio_id: Number of this slot in the SDIO interrupt registers.
@@ -263,6 +265,7 @@ struct dw_mci_slot {
 	struct list_head	queue_node;
 
 	unsigned int		clock;
+	unsigned int		__clk_old;
 
 	unsigned long		flags;
 #define DW_MMC_CARD_PRESENT	0
diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c
index 79905ce895ad..bbad309679cf 100644
--- a/drivers/mmc/host/moxart-mmc.c
+++ b/drivers/mmc/host/moxart-mmc.c
@@ -257,7 +257,7 @@ static void moxart_dma_complete(void *param)
 static void moxart_transfer_dma(struct mmc_data *data, struct moxart_host *host)
 {
 	u32 len, dir_data, dir_slave;
-	unsigned long dma_time;
+	long dma_time;
 	struct dma_async_tx_descriptor *desc = NULL;
 	struct dma_chan *dma_chan;
 
@@ -397,7 +397,8 @@ static void moxart_prepare_data(struct moxart_host *host)
 static void moxart_request(struct mmc_host *mmc, struct mmc_request *mrq)
 {
 	struct moxart_host *host = mmc_priv(mmc);
-	unsigned long pio_time, flags;
+	long pio_time;
+	unsigned long flags;
 	u32 status;
 
 	spin_lock_irqsave(&host->lock, flags);
diff --git a/drivers/mmc/host/omap.c b/drivers/mmc/host/omap.c
index f23d65eb070d..be3c49fa7382 100644
--- a/drivers/mmc/host/omap.c
+++ b/drivers/mmc/host/omap.c
@@ -1016,14 +1016,16 @@ mmc_omap_prepare_data(struct mmc_omap_host *host, struct mmc_request *req)
 
 		/* Only reconfigure if we have a different burst size */
 		if (*bp != burst) {
-			struct dma_slave_config cfg;
-
-			cfg.src_addr = host->phys_base + OMAP_MMC_REG(host, DATA);
-			cfg.dst_addr = host->phys_base + OMAP_MMC_REG(host, DATA);
-			cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
-			cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
-			cfg.src_maxburst = burst;
-			cfg.dst_maxburst = burst;
+			struct dma_slave_config cfg = {
+				.src_addr = host->phys_base +
+					    OMAP_MMC_REG(host, DATA),
+				.dst_addr = host->phys_base +
+					    OMAP_MMC_REG(host, DATA),
+				.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
+				.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
+				.src_maxburst = burst,
+				.dst_maxburst = burst,
+			};
 
 			if (dmaengine_slave_config(c, &cfg))
 				goto use_pio;
diff --git a/drivers/mmc/host/omap_hsmmc.c b/drivers/mmc/host/omap_hsmmc.c
index 24ebc9a8de89..5f2f24a7360d 100644
--- a/drivers/mmc/host/omap_hsmmc.c
+++ b/drivers/mmc/host/omap_hsmmc.c
@@ -1409,11 +1409,18 @@ static int omap_hsmmc_pre_dma_transfer(struct omap_hsmmc_host *host,
 static int omap_hsmmc_setup_dma_transfer(struct omap_hsmmc_host *host,
 					struct mmc_request *req)
 {
-	struct dma_slave_config cfg;
 	struct dma_async_tx_descriptor *tx;
 	int ret = 0, i;
 	struct mmc_data *data = req->data;
 	struct dma_chan *chan;
+	struct dma_slave_config cfg = {
+		.src_addr = host->mapbase + OMAP_HSMMC_DATA,
+		.dst_addr = host->mapbase + OMAP_HSMMC_DATA,
+		.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
+		.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
+		.src_maxburst = data->blksz / 4,
+		.dst_maxburst = data->blksz / 4,
+	};
 
 	/* Sanity check: all the SG entries must be aligned by block size. */
 	for (i = 0; i < data->sg_len; i++) {
@@ -1433,13 +1440,6 @@ static int omap_hsmmc_setup_dma_transfer(struct omap_hsmmc_host *host,
 
 	chan = omap_hsmmc_get_dma_chan(host, data);
 
-	cfg.src_addr = host->mapbase + OMAP_HSMMC_DATA;
-	cfg.dst_addr = host->mapbase + OMAP_HSMMC_DATA;
-	cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
-	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
-	cfg.src_maxburst = data->blksz / 4;
-	cfg.dst_maxburst = data->blksz / 4;
-
 	ret = dmaengine_slave_config(chan, &cfg);
 	if (ret)
 		return ret;
diff --git a/drivers/mmc/host/rtsx_pci_sdmmc.c b/drivers/mmc/host/rtsx_pci_sdmmc.c
index 396c9b7e4121..3ccaa1415f33 100644
--- a/drivers/mmc/host/rtsx_pci_sdmmc.c
+++ b/drivers/mmc/host/rtsx_pci_sdmmc.c
@@ -126,7 +126,7 @@ static int sd_response_type(struct mmc_command *cmd)
 		return SD_RSP_TYPE_R0;
 	case MMC_RSP_R1:
 		return SD_RSP_TYPE_R1;
-	case MMC_RSP_R1 & ~MMC_RSP_CRC:
+	case MMC_RSP_R1_NO_CRC:
 		return SD_RSP_TYPE_R1 | SD_NO_CHECK_CRC7;
 	case MMC_RSP_R1B:
 		return SD_RSP_TYPE_R1b;
diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c
index 6c71fc9f76c7..4106295527b9 100644
--- a/drivers/mmc/host/rtsx_usb_sdmmc.c
+++ b/drivers/mmc/host/rtsx_usb_sdmmc.c
@@ -324,7 +324,7 @@ static void sd_send_cmd_get_rsp(struct rtsx_usb_sdmmc *host,
 	case MMC_RSP_R1:
 		rsp_type = SD_RSP_TYPE_R1;
 		break;
-	case MMC_RSP_R1 & ~MMC_RSP_CRC:
+	case MMC_RSP_R1_NO_CRC:
 		rsp_type = SD_RSP_TYPE_R1 | SD_NO_CHECK_CRC7;
 		break;
 	case MMC_RSP_R1B:
diff --git a/drivers/mmc/host/sdhci-acpi.c b/drivers/mmc/host/sdhci-acpi.c
index 8fe0756c8e1e..81d4dc034793 100644
--- a/drivers/mmc/host/sdhci-acpi.c
+++ b/drivers/mmc/host/sdhci-acpi.c
@@ -275,7 +275,7 @@ static const struct sdhci_acpi_slot sdhci_acpi_slot_int_emmc = {
 	.chip    = &sdhci_acpi_chip_int,
 	.caps    = MMC_CAP_8_BIT_DATA | MMC_CAP_NONREMOVABLE |
 		   MMC_CAP_HW_RESET | MMC_CAP_1_8V_DDR |
-		   MMC_CAP_WAIT_WHILE_BUSY,
+		   MMC_CAP_CMD_DURING_TFR | MMC_CAP_WAIT_WHILE_BUSY,
 	.caps2   = MMC_CAP2_HC_ERASE_SZ,
 	.flags   = SDHCI_ACPI_RUNTIME_PM,
 	.quirks  = SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC,
diff --git a/drivers/mmc/host/sdhci-bcm-kona.c b/drivers/mmc/host/sdhci-bcm-kona.c
index e5c634bdfdd9..51dd2fd65000 100644
--- a/drivers/mmc/host/sdhci-bcm-kona.c
+++ b/drivers/mmc/host/sdhci-bcm-kona.c
@@ -253,12 +253,14 @@ static int sdhci_bcm_kona_probe(struct platform_device *pdev)
 		goto err_pltfm_free;
 	}
 
-	if (clk_set_rate(pltfm_priv->clk, host->mmc->f_max) != 0) {
+	ret = clk_set_rate(pltfm_priv->clk, host->mmc->f_max);
+	if (ret) {
 		dev_err(dev, "Failed to set rate core clock\n");
 		goto err_pltfm_free;
 	}
 
-	if (clk_prepare_enable(pltfm_priv->clk) != 0) {
+	ret = clk_prepare_enable(pltfm_priv->clk);
+	if (ret) {
 		dev_err(dev, "Failed to enable core clock\n");
 		goto err_pltfm_free;
 	}
diff --git a/drivers/mmc/host/sdhci-brcmstb.c b/drivers/mmc/host/sdhci-brcmstb.c
index cce10fe3e19e..159f6f64c68e 100644
--- a/drivers/mmc/host/sdhci-brcmstb.c
+++ b/drivers/mmc/host/sdhci-brcmstb.c
@@ -98,6 +98,8 @@ static int sdhci_brcmstb_probe(struct platform_device *pdev)
 	 * properties through mmc_of_parse().
 	 */
 	host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
+	if (of_device_is_compatible(pdev->dev.of_node, "brcm,bcm7425-sdhci"))
+		host->caps &= ~SDHCI_CAN_64BIT;
 	host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
 	host->caps1 &= ~(SDHCI_SUPPORT_SDR50 | SDHCI_SUPPORT_SDR104 |
 			SDHCI_SUPPORT_DDR50);
@@ -121,6 +123,7 @@ err_clk:
 
 static const struct of_device_id sdhci_brcm_of_match[] = {
 	{ .compatible = "brcm,bcm7425-sdhci" },
+	{ .compatible = "brcm,bcm7445-sdhci" },
 	{},
 };
 MODULE_DEVICE_TABLE(of, sdhci_brcm_of_match);
@@ -128,7 +131,6 @@ MODULE_DEVICE_TABLE(of, sdhci_brcm_of_match);
 static struct platform_driver sdhci_brcmstb_driver = {
 	.driver		= {
 		.name	= "sdhci-brcmstb",
-		.owner	= THIS_MODULE,
 		.pm	= &sdhci_brcmstb_pmops,
 		.of_match_table = of_match_ptr(sdhci_brcm_of_match),
 	},
diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c
index 99e0b334f9df..1f54fd8755c8 100644
--- a/drivers/mmc/host/sdhci-esdhc-imx.c
+++ b/drivers/mmc/host/sdhci-esdhc-imx.c
@@ -31,6 +31,7 @@
 #include "sdhci-pltfm.h"
 #include "sdhci-esdhc.h"
 
+#define ESDHC_SYS_CTRL_DTOCV_MASK	0x0f
 #define	ESDHC_CTRL_D3CD			0x08
 #define ESDHC_BURST_LEN_EN_INCR		(1 << 27)
 /* VENDOR SPEC register */
@@ -928,7 +929,8 @@ static unsigned int esdhc_get_max_timeout_count(struct sdhci_host *host)
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 	struct pltfm_imx_data *imx_data = sdhci_pltfm_priv(pltfm_host);
 
-	return esdhc_is_usdhc(imx_data) ? 1 << 28 : 1 << 27;
+	/* Doc Errata: the uSDHC actual maximum timeout count is 1 << 29 */
+	return esdhc_is_usdhc(imx_data) ? 1 << 29 : 1 << 27;
 }
 
 static void esdhc_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
@@ -937,7 +939,8 @@ static void esdhc_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 	struct pltfm_imx_data *imx_data = sdhci_pltfm_priv(pltfm_host);
 
 	/* use maximum timeout counter */
-	sdhci_writeb(host, esdhc_is_usdhc(imx_data) ? 0xF : 0xE,
+	esdhc_clrset_le(host, ESDHC_SYS_CTRL_DTOCV_MASK,
+			esdhc_is_usdhc(imx_data) ? 0xF : 0xE,
 			SDHCI_TIMEOUT_CONTROL);
 }
 
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index e0f193f7e3e5..da8e40af6f85 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -26,6 +26,7 @@
 #include <linux/phy/phy.h>
 #include <linux/regmap.h>
 #include "sdhci-pltfm.h"
+#include <linux/of.h>
 
 #define SDHCI_ARASAN_CLK_CTRL_OFFSET	0x2c
 #define SDHCI_ARASAN_VENDOR_REGISTER	0x78
@@ -35,6 +36,8 @@
 #define CLK_CTRL_TIMEOUT_MASK		(0xf << CLK_CTRL_TIMEOUT_SHIFT)
 #define CLK_CTRL_TIMEOUT_MIN_EXP	13
 
+#define PHY_CLK_TOO_SLOW_HZ		400000
+
 /*
  * On some SoCs the syscon area has a feature where the upper 16-bits of
  * each 32-bit register act as a write mask for the lower 16-bits.  This allows
@@ -65,10 +68,12 @@ struct sdhci_arasan_soc_ctl_field {
  * accessible via the syscon API.
  *
  * @baseclkfreq:	Where to find corecfg_baseclkfreq
+ * @clockmultiplier:	Where to find corecfg_clockmultiplier
  * @hiword_update:	If true, use HIWORD_UPDATE to access the syscon
  */
 struct sdhci_arasan_soc_ctl_map {
 	struct sdhci_arasan_soc_ctl_field	baseclkfreq;
+	struct sdhci_arasan_soc_ctl_field	clockmultiplier;
 	bool					hiword_update;
 };
 
@@ -77,6 +82,7 @@ struct sdhci_arasan_soc_ctl_map {
  * @host:		Pointer to the main SDHCI host structure.
  * @clk_ahb:		Pointer to the AHB clock
  * @phy:		Pointer to the generic phy
+ * @is_phy_on:		True if the PHY is on; false if not.
  * @sdcardclk_hw:	Struct for the clock we might provide to a PHY.
  * @sdcardclk:		Pointer to normal 'struct clock' for sdcardclk_hw.
  * @soc_ctl_base:	Pointer to regmap for syscon for soc_ctl registers.
@@ -86,16 +92,22 @@ struct sdhci_arasan_data {
 	struct sdhci_host *host;
 	struct clk	*clk_ahb;
 	struct phy	*phy;
+	bool		is_phy_on;
 
 	struct clk_hw	sdcardclk_hw;
 	struct clk      *sdcardclk;
 
 	struct regmap	*soc_ctl_base;
 	const struct sdhci_arasan_soc_ctl_map *soc_ctl_map;
+	unsigned int	quirks; /* Arasan deviations from spec */
+
+/* Controller does not have CD wired and will not function normally without */
+#define SDHCI_ARASAN_QUIRK_FORCE_CDTEST	BIT(0)
 };
 
 static const struct sdhci_arasan_soc_ctl_map rk3399_soc_ctl_map = {
 	.baseclkfreq = { .reg = 0xf000, .width = 8, .shift = 8 },
+	.clockmultiplier = { .reg = 0xf02c, .width = 8, .shift = 0},
 	.hiword_update = true,
 };
 
@@ -170,13 +182,47 @@ static void sdhci_arasan_set_clock(struct sdhci_host *host, unsigned int clock)
 	struct sdhci_arasan_data *sdhci_arasan = sdhci_pltfm_priv(pltfm_host);
 	bool ctrl_phy = false;
 
-	if (clock > MMC_HIGH_52_MAX_DTR && (!IS_ERR(sdhci_arasan->phy)))
-		ctrl_phy = true;
+	if (!IS_ERR(sdhci_arasan->phy)) {
+		if (!sdhci_arasan->is_phy_on && clock <= PHY_CLK_TOO_SLOW_HZ) {
+			/*
+			 * If PHY off, set clock to max speed and power PHY on.
+			 *
+			 * Although PHY docs apparently suggest power cycling
+			 * when changing the clock the PHY doesn't like to be
+			 * powered on while at low speeds like those used in ID
+			 * mode.  Even worse is powering the PHY on while the
+			 * clock is off.
+			 *
+			 * To workaround the PHY limitations, the best we can
+			 * do is to power it on at a faster speed and then slam
+			 * through low speeds without power cycling.
+			 */
+			sdhci_set_clock(host, host->max_clk);
+			spin_unlock_irq(&host->lock);
+			phy_power_on(sdhci_arasan->phy);
+			spin_lock_irq(&host->lock);
+			sdhci_arasan->is_phy_on = true;
+
+			/*
+			 * We'll now fall through to the below case with
+			 * ctrl_phy = false (so we won't turn off/on).  The
+			 * sdhci_set_clock() will set the real clock.
+			 */
+		} else if (clock > PHY_CLK_TOO_SLOW_HZ) {
+			/*
+			 * At higher clock speeds the PHY is fine being power
+			 * cycled and docs say you _should_ power cycle when
+			 * changing clock speeds.
+			 */
+			ctrl_phy = true;
+		}
+	}
 
-	if (ctrl_phy) {
+	if (ctrl_phy && sdhci_arasan->is_phy_on) {
 		spin_unlock_irq(&host->lock);
 		phy_power_off(sdhci_arasan->phy);
 		spin_lock_irq(&host->lock);
+		sdhci_arasan->is_phy_on = false;
 	}
 
 	sdhci_set_clock(host, clock);
@@ -185,6 +231,7 @@ static void sdhci_arasan_set_clock(struct sdhci_host *host, unsigned int clock)
 		spin_unlock_irq(&host->lock);
 		phy_power_on(sdhci_arasan->phy);
 		spin_lock_irq(&host->lock);
+		sdhci_arasan->is_phy_on = true;
 	}
 }
 
@@ -203,12 +250,27 @@ static void sdhci_arasan_hs400_enhanced_strobe(struct mmc_host *mmc,
 	writel(vendor, host->ioaddr + SDHCI_ARASAN_VENDOR_REGISTER);
 }
 
+void sdhci_arasan_reset(struct sdhci_host *host, u8 mask)
+{
+	u8 ctrl;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_arasan_data *sdhci_arasan = sdhci_pltfm_priv(pltfm_host);
+
+	sdhci_reset(host, mask);
+
+	if (sdhci_arasan->quirks & SDHCI_ARASAN_QUIRK_FORCE_CDTEST) {
+		ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
+		ctrl |= SDHCI_CTRL_CDTEST_INS | SDHCI_CTRL_CDTEST_EN;
+		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
+	}
+}
+
 static struct sdhci_ops sdhci_arasan_ops = {
 	.set_clock = sdhci_arasan_set_clock,
 	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
 	.get_timeout_clock = sdhci_arasan_get_timeout_clock,
 	.set_bus_width = sdhci_set_bus_width,
-	.reset = sdhci_reset,
+	.reset = sdhci_arasan_reset,
 	.set_uhs_signaling = sdhci_set_uhs_signaling,
 };
 
@@ -239,13 +301,14 @@ static int sdhci_arasan_suspend(struct device *dev)
 	if (ret)
 		return ret;
 
-	if (!IS_ERR(sdhci_arasan->phy)) {
+	if (!IS_ERR(sdhci_arasan->phy) && sdhci_arasan->is_phy_on) {
 		ret = phy_power_off(sdhci_arasan->phy);
 		if (ret) {
 			dev_err(dev, "Cannot power off phy.\n");
 			sdhci_resume_host(host);
 			return ret;
 		}
+		sdhci_arasan->is_phy_on = false;
 	}
 
 	clk_disable(pltfm_host->clk);
@@ -281,12 +344,13 @@ static int sdhci_arasan_resume(struct device *dev)
 		return ret;
 	}
 
-	if (!IS_ERR(sdhci_arasan->phy)) {
+	if (!IS_ERR(sdhci_arasan->phy) && host->mmc->actual_clock) {
 		ret = phy_power_on(sdhci_arasan->phy);
 		if (ret) {
 			dev_err(dev, "Cannot power on phy.\n");
 			return ret;
 		}
+		sdhci_arasan->is_phy_on = true;
 	}
 
 	return sdhci_resume_host(host);
@@ -338,6 +402,45 @@ static const struct clk_ops arasan_sdcardclk_ops = {
 };
 
 /**
+ * sdhci_arasan_update_clockmultiplier - Set corecfg_clockmultiplier
+ *
+ * The corecfg_clockmultiplier is supposed to contain clock multiplier
+ * value of programmable clock generator.
+ *
+ * NOTES:
+ * - Many existing devices don't seem to do this and work fine.  To keep
+ *   compatibility for old hardware where the device tree doesn't provide a
+ *   register map, this function is a noop if a soc_ctl_map hasn't been provided
+ *   for this platform.
+ * - The value of corecfg_clockmultiplier should sync with that of corresponding
+ *   value reading from sdhci_capability_register. So this function is called
+ *   once at probe time and never called again.
+ *
+ * @host:		The sdhci_host
+ */
+static void sdhci_arasan_update_clockmultiplier(struct sdhci_host *host,
+						u32 value)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_arasan_data *sdhci_arasan = sdhci_pltfm_priv(pltfm_host);
+	const struct sdhci_arasan_soc_ctl_map *soc_ctl_map =
+		sdhci_arasan->soc_ctl_map;
+
+	/* Having a map is optional */
+	if (!soc_ctl_map)
+		return;
+
+	/* If we have a map, we expect to have a syscon */
+	if (!sdhci_arasan->soc_ctl_base) {
+		pr_warn("%s: Have regmap, but no soc-ctl-syscon\n",
+			mmc_hostname(host->mmc));
+		return;
+	}
+
+	sdhci_arasan_syscon_write(host, &soc_ctl_map->clockmultiplier, value);
+}
+
+/**
  * sdhci_arasan_update_baseclkfreq - Set corecfg_baseclkfreq
  *
  * The corecfg_baseclkfreq is supposed to contain the MHz of clk_xin.  This
@@ -462,6 +565,7 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 	struct sdhci_host *host;
 	struct sdhci_pltfm_host *pltfm_host;
 	struct sdhci_arasan_data *sdhci_arasan;
+	struct device_node *np = pdev->dev.of_node;
 
 	host = sdhci_pltfm_init(pdev, &sdhci_arasan_pdata,
 				sizeof(*sdhci_arasan));
@@ -516,8 +620,16 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 	}
 
 	sdhci_get_of_property(pdev);
+
+	if (of_property_read_bool(np, "xlnx,fails-without-test-cd"))
+		sdhci_arasan->quirks |= SDHCI_ARASAN_QUIRK_FORCE_CDTEST;
+
 	pltfm_host->clk = clk_xin;
 
+	if (of_device_is_compatible(pdev->dev.of_node,
+				    "rockchip,rk3399-sdhci-5.1"))
+		sdhci_arasan_update_clockmultiplier(host, 0x0);
+
 	sdhci_arasan_update_baseclkfreq(host);
 
 	ret = sdhci_arasan_register_sdclk(sdhci_arasan, clk_xin, &pdev->dev);
@@ -547,12 +659,6 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 			goto unreg_clk;
 		}
 
-		ret = phy_power_on(sdhci_arasan->phy);
-		if (ret < 0) {
-			dev_err(&pdev->dev, "phy_power_on err.\n");
-			goto err_phy_power;
-		}
-
 		host->mmc_host_ops.hs400_enhanced_strobe =
 					sdhci_arasan_hs400_enhanced_strobe;
 	}
@@ -565,9 +671,6 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 
 err_add_host:
 	if (!IS_ERR(sdhci_arasan->phy))
-		phy_power_off(sdhci_arasan->phy);
-err_phy_power:
-	if (!IS_ERR(sdhci_arasan->phy))
 		phy_exit(sdhci_arasan->phy);
 unreg_clk:
 	sdhci_arasan_unregister_sdclk(&pdev->dev);
@@ -589,7 +692,8 @@ static int sdhci_arasan_remove(struct platform_device *pdev)
 	struct clk *clk_ahb = sdhci_arasan->clk_ahb;
 
 	if (!IS_ERR(sdhci_arasan->phy)) {
-		phy_power_off(sdhci_arasan->phy);
+		if (sdhci_arasan->is_phy_on)
+			phy_power_off(sdhci_arasan->phy);
 		phy_exit(sdhci_arasan->phy);
 	}
 
diff --git a/drivers/mmc/host/sdhci-of-esdhc.c b/drivers/mmc/host/sdhci-of-esdhc.c
index 239be2fde242..fb71c866eacc 100644
--- a/drivers/mmc/host/sdhci-of-esdhc.c
+++ b/drivers/mmc/host/sdhci-of-esdhc.c
@@ -583,7 +583,7 @@ static int sdhci_esdhc_probe(struct platform_device *pdev)
 
 	np = pdev->dev.of_node;
 
-	if (of_get_property(np, "little-endian", NULL))
+	if (of_property_read_bool(np, "little-endian"))
 		host = sdhci_pltfm_init(pdev, &sdhci_esdhc_le_pdata,
 					sizeof(struct sdhci_esdhc));
 	else
diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c
index 897cfd24ca2e..72a1f1f5180a 100644
--- a/drivers/mmc/host/sdhci-pci-core.c
+++ b/drivers/mmc/host/sdhci-pci-core.c
@@ -156,7 +156,7 @@ static void sdhci_pci_add_own_cd(struct sdhci_pci_slot *slot)
 	if (!gpio_is_valid(gpio))
 		return;
 
-	err = gpio_request(gpio, "sd_cd");
+	err = devm_gpio_request(&slot->chip->pdev->dev, gpio, "sd_cd");
 	if (err < 0)
 		goto out;
 
@@ -179,7 +179,7 @@ static void sdhci_pci_add_own_cd(struct sdhci_pci_slot *slot)
 	return;
 
 out_free:
-	gpio_free(gpio);
+	devm_gpio_free(&slot->chip->pdev->dev, gpio);
 out:
 	dev_warn(&slot->chip->pdev->dev, "failed to setup card detect wake up\n");
 }
@@ -188,8 +188,6 @@ static void sdhci_pci_remove_own_cd(struct sdhci_pci_slot *slot)
 {
 	if (slot->cd_irq >= 0)
 		free_irq(slot->cd_irq, slot);
-	if (gpio_is_valid(slot->cd_gpio))
-		gpio_free(slot->cd_gpio);
 }
 
 #else
@@ -356,6 +354,7 @@ static int byt_emmc_probe_slot(struct sdhci_pci_slot *slot)
 {
 	slot->host->mmc->caps |= MMC_CAP_8_BIT_DATA | MMC_CAP_NONREMOVABLE |
 				 MMC_CAP_HW_RESET | MMC_CAP_1_8V_DDR |
+				 MMC_CAP_CMD_DURING_TFR |
 				 MMC_CAP_WAIT_WHILE_BUSY;
 	slot->host->mmc->caps2 |= MMC_CAP2_HC_ERASE_SZ;
 	slot->hw_reset = sdhci_pci_int_hw_reset;
@@ -421,17 +420,30 @@ static const struct sdhci_pci_fixes sdhci_intel_byt_sd = {
 /* Define Host controllers for Intel Merrifield platform */
 #define INTEL_MRFLD_EMMC_0	0
 #define INTEL_MRFLD_EMMC_1	1
+#define INTEL_MRFLD_SD		2
+#define INTEL_MRFLD_SDIO	3
 
 static int intel_mrfld_mmc_probe_slot(struct sdhci_pci_slot *slot)
 {
-	if ((PCI_FUNC(slot->chip->pdev->devfn) != INTEL_MRFLD_EMMC_0) &&
-	    (PCI_FUNC(slot->chip->pdev->devfn) != INTEL_MRFLD_EMMC_1))
-		/* SD support is not ready yet */
+	unsigned int func = PCI_FUNC(slot->chip->pdev->devfn);
+
+	switch (func) {
+	case INTEL_MRFLD_EMMC_0:
+	case INTEL_MRFLD_EMMC_1:
+		slot->host->mmc->caps |= MMC_CAP_NONREMOVABLE |
+					 MMC_CAP_8_BIT_DATA |
+					 MMC_CAP_1_8V_DDR;
+		break;
+	case INTEL_MRFLD_SD:
+		slot->host->quirks2 |= SDHCI_QUIRK2_NO_1_8_V;
+		break;
+	case INTEL_MRFLD_SDIO:
+		slot->host->mmc->caps |= MMC_CAP_NONREMOVABLE |
+					 MMC_CAP_POWER_OFF_CARD;
+		break;
+	default:
 		return -ENODEV;
-
-	slot->host->mmc->caps |= MMC_CAP_8_BIT_DATA | MMC_CAP_NONREMOVABLE |
-				 MMC_CAP_1_8V_DDR;
-
+	}
 	return 0;
 }
 
@@ -1615,7 +1627,6 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot(
 
 	slot->chip = chip;
 	slot->host = host;
-	slot->pci_bar = bar;
 	slot->rst_n_gpio = -EINVAL;
 	slot->cd_gpio = -EINVAL;
 	slot->cd_idx = -1;
@@ -1643,27 +1654,22 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot(
 
 	host->irq = pdev->irq;
 
-	ret = pci_request_region(pdev, bar, mmc_hostname(host->mmc));
+	ret = pcim_iomap_regions(pdev, BIT(bar), mmc_hostname(host->mmc));
 	if (ret) {
 		dev_err(&pdev->dev, "cannot request region\n");
 		goto cleanup;
 	}
 
-	host->ioaddr = pci_ioremap_bar(pdev, bar);
-	if (!host->ioaddr) {
-		dev_err(&pdev->dev, "failed to remap registers\n");
-		ret = -ENOMEM;
-		goto release;
-	}
+	host->ioaddr = pcim_iomap_table(pdev)[bar];
 
 	if (chip->fixes && chip->fixes->probe_slot) {
 		ret = chip->fixes->probe_slot(slot);
 		if (ret)
-			goto unmap;
+			goto cleanup;
 	}
 
 	if (gpio_is_valid(slot->rst_n_gpio)) {
-		if (!gpio_request(slot->rst_n_gpio, "eMMC_reset")) {
+		if (!devm_gpio_request(&pdev->dev, slot->rst_n_gpio, "eMMC_reset")) {
 			gpio_direction_output(slot->rst_n_gpio, 1);
 			slot->host->mmc->caps |= MMC_CAP_HW_RESET;
 			slot->hw_reset = sdhci_pci_gpio_hw_reset;
@@ -1702,18 +1708,9 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot(
 	return slot;
 
 remove:
-	if (gpio_is_valid(slot->rst_n_gpio))
-		gpio_free(slot->rst_n_gpio);
-
 	if (chip->fixes && chip->fixes->remove_slot)
 		chip->fixes->remove_slot(slot, 0);
 
-unmap:
-	iounmap(host->ioaddr);
-
-release:
-	pci_release_region(pdev, bar);
-
 cleanup:
 	if (slot->data && slot->data->cleanup)
 		slot->data->cleanup(slot->data);
@@ -1738,17 +1735,12 @@ static void sdhci_pci_remove_slot(struct sdhci_pci_slot *slot)
 
 	sdhci_remove_host(slot->host, dead);
 
-	if (gpio_is_valid(slot->rst_n_gpio))
-		gpio_free(slot->rst_n_gpio);
-
 	if (slot->chip->fixes && slot->chip->fixes->remove_slot)
 		slot->chip->fixes->remove_slot(slot, dead);
 
 	if (slot->data && slot->data->cleanup)
 		slot->data->cleanup(slot->data);
 
-	pci_release_region(slot->chip->pdev, slot->pci_bar);
-
 	sdhci_free_host(slot->host);
 }
 
diff --git a/drivers/mmc/host/sdhci-pci.h b/drivers/mmc/host/sdhci-pci.h
index 7e0788712e1a..9c7c08b93223 100644
--- a/drivers/mmc/host/sdhci-pci.h
+++ b/drivers/mmc/host/sdhci-pci.h
@@ -72,7 +72,6 @@ struct sdhci_pci_slot {
 	struct sdhci_host	*host;
 	struct sdhci_pci_data	*data;
 
-	int			pci_bar;
 	int			rst_n_gpio;
 	int			cd_gpio;
 	int			cd_irq;
diff --git a/drivers/mmc/host/sdhci-pltfm.c b/drivers/mmc/host/sdhci-pltfm.c
index 1d17dcfc3ffb..ad49bfaf5bf8 100644
--- a/drivers/mmc/host/sdhci-pltfm.c
+++ b/drivers/mmc/host/sdhci-pltfm.c
@@ -156,13 +156,6 @@ struct sdhci_host *sdhci_pltfm_init(struct platform_device *pdev,
 		host->quirks2 = pdata->quirks2;
 	}
 
-	/*
-	 * Some platforms need to probe the controller to be able to
-	 * determine which caps should be used.
-	 */
-	if (host->ops && host->ops->platform_init)
-		host->ops->platform_init(host);
-
 	platform_set_drvdata(pdev, host);
 
 	return host;
diff --git a/drivers/mmc/host/sdhci-st.c b/drivers/mmc/host/sdhci-st.c
index c95ba83366a0..ed92ce729dde 100644
--- a/drivers/mmc/host/sdhci-st.c
+++ b/drivers/mmc/host/sdhci-st.c
@@ -28,6 +28,7 @@
 
 struct st_mmc_platform_data {
 	struct  reset_control *rstc;
+	struct  clk *icnclk;
 	void __iomem *top_ioaddr;
 };
 
@@ -353,7 +354,7 @@ static int sdhci_st_probe(struct platform_device *pdev)
 	struct sdhci_host *host;
 	struct st_mmc_platform_data *pdata;
 	struct sdhci_pltfm_host *pltfm_host;
-	struct clk *clk;
+	struct clk *clk, *icnclk;
 	int ret = 0;
 	u16 host_version;
 	struct resource *res;
@@ -365,6 +366,11 @@ static int sdhci_st_probe(struct platform_device *pdev)
 		return PTR_ERR(clk);
 	}
 
+	/* ICN clock isn't compulsory, but use it if it's provided. */
+	icnclk = devm_clk_get(&pdev->dev, "icn");
+	if (IS_ERR(icnclk))
+		icnclk = NULL;
+
 	rstc = devm_reset_control_get(&pdev->dev, NULL);
 	if (IS_ERR(rstc))
 		rstc = NULL;
@@ -389,6 +395,7 @@ static int sdhci_st_probe(struct platform_device *pdev)
 	}
 
 	clk_prepare_enable(clk);
+	clk_prepare_enable(icnclk);
 
 	/* Configure the FlashSS Top registers for setting eMMC TX/RX delay */
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
@@ -400,6 +407,7 @@ static int sdhci_st_probe(struct platform_device *pdev)
 	}
 
 	pltfm_host->clk = clk;
+	pdata->icnclk = icnclk;
 
 	/* Configure the Arasan HC inside the flashSS */
 	st_mmcss_cconfig(np, host);
@@ -422,6 +430,7 @@ static int sdhci_st_probe(struct platform_device *pdev)
 	return 0;
 
 err_out:
+	clk_disable_unprepare(icnclk);
 	clk_disable_unprepare(clk);
 err_of:
 	sdhci_pltfm_free(pdev);
@@ -442,6 +451,8 @@ static int sdhci_st_remove(struct platform_device *pdev)
 
 	ret = sdhci_pltfm_unregister(pdev);
 
+	clk_disable_unprepare(pdata->icnclk);
+
 	if (rstc)
 		reset_control_assert(rstc);
 
@@ -462,6 +473,7 @@ static int sdhci_st_suspend(struct device *dev)
 	if (pdata->rstc)
 		reset_control_assert(pdata->rstc);
 
+	clk_disable_unprepare(pdata->icnclk);
 	clk_disable_unprepare(pltfm_host->clk);
 out:
 	return ret;
@@ -475,6 +487,7 @@ static int sdhci_st_resume(struct device *dev)
 	struct device_node *np = dev->of_node;
 
 	clk_prepare_enable(pltfm_host->clk);
+	clk_prepare_enable(pdata->icnclk);
 
 	if (pdata->rstc)
 		reset_control_deassert(pdata->rstc);
diff --git a/drivers/mmc/host/sdhci-tegra.c b/drivers/mmc/host/sdhci-tegra.c
index 1e93dc4e303e..20b6ff5b4af1 100644
--- a/drivers/mmc/host/sdhci-tegra.c
+++ b/drivers/mmc/host/sdhci-tegra.c
@@ -391,6 +391,31 @@ static const struct sdhci_tegra_soc_data soc_data_tegra114 = {
 	.pdata = &sdhci_tegra114_pdata,
 };
 
+static const struct sdhci_pltfm_data sdhci_tegra124_pdata = {
+	.quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
+		  SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
+		  SDHCI_QUIRK_SINGLE_POWER_WRITE |
+		  SDHCI_QUIRK_NO_HISPD_BIT |
+		  SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC |
+		  SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
+		   /*
+		    * The TRM states that the SD/MMC controller found on
+		    * Tegra124 can address 34 bits (the maximum supported by
+		    * the Tegra memory controller), but tests show that DMA
+		    * to or from above 4 GiB doesn't work. This is possibly
+		    * caused by missing programming, though it's not obvious
+		    * what sequence is required. Mark 64-bit DMA broken for
+		    * now to fix this for existing users (e.g. Nyan boards).
+		    */
+		   SDHCI_QUIRK2_BROKEN_64_BIT_DMA,
+	.ops  = &tegra114_sdhci_ops,
+};
+
+static const struct sdhci_tegra_soc_data soc_data_tegra124 = {
+	.pdata = &sdhci_tegra124_pdata,
+};
+
 static const struct sdhci_pltfm_data sdhci_tegra210_pdata = {
 	.quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
 		  SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
@@ -408,7 +433,7 @@ static const struct sdhci_tegra_soc_data soc_data_tegra210 = {
 
 static const struct of_device_id sdhci_tegra_dt_match[] = {
 	{ .compatible = "nvidia,tegra210-sdhci", .data = &soc_data_tegra210 },
-	{ .compatible = "nvidia,tegra124-sdhci", .data = &soc_data_tegra114 },
+	{ .compatible = "nvidia,tegra124-sdhci", .data = &soc_data_tegra124 },
 	{ .compatible = "nvidia,tegra114-sdhci", .data = &soc_data_tegra114 },
 	{ .compatible = "nvidia,tegra30-sdhci", .data = &soc_data_tegra30 },
 	{ .compatible = "nvidia,tegra20-sdhci", .data = &soc_data_tegra20 },
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index cd65d474afa2..48055666c655 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -888,7 +888,8 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
 static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
 				    struct mmc_request *mrq)
 {
-	return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12);
+	return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
+	       !mrq->cap_cmd_during_tfr;
 }
 
 static void sdhci_set_transfer_mode(struct sdhci_host *host,
@@ -1031,9 +1032,18 @@ static void sdhci_finish_data(struct sdhci_host *host)
 			sdhci_do_reset(host, SDHCI_RESET_DATA);
 		}
 
-		/* Avoid triggering warning in sdhci_send_command() */
-		host->cmd = NULL;
-		sdhci_send_command(host, data->stop);
+		/*
+		 * 'cap_cmd_during_tfr' request must not use the command line
+		 * after mmc_command_done() has been called. It is upper layer's
+		 * responsibility to send the stop command if required.
+		 */
+		if (data->mrq->cap_cmd_during_tfr) {
+			sdhci_finish_mrq(host, data->mrq);
+		} else {
+			/* Avoid triggering warning in sdhci_send_command() */
+			host->cmd = NULL;
+			sdhci_send_command(host, data->stop);
+		}
 	} else {
 		sdhci_finish_mrq(host, data->mrq);
 	}
@@ -1165,6 +1175,9 @@ static void sdhci_finish_command(struct sdhci_host *host)
 		}
 	}
 
+	if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)
+		mmc_command_done(host->mmc, cmd->mrq);
+
 	/*
 	 * The host can send and interrupt when the busy state has
 	 * ended, allowing us to wait without wasting CPU cycles.
@@ -2062,7 +2075,7 @@ static int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
 
 		spin_unlock_irqrestore(&host->lock, flags);
 		/* Wait for Buffer Read Ready interrupt */
-		wait_event_interruptible_timeout(host->buf_ready_int,
+		wait_event_timeout(host->buf_ready_int,
 					(host->tuning_done == 1),
 					msecs_to_jiffies(50));
 		spin_lock_irqsave(&host->lock, flags);
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index 0411c9f36461..c722cd23205c 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -84,6 +84,8 @@
 #define   SDHCI_CTRL_ADMA32	0x10
 #define   SDHCI_CTRL_ADMA64	0x18
 #define   SDHCI_CTRL_8BITBUS	0x20
+#define  SDHCI_CTRL_CDTEST_INS	0x40
+#define  SDHCI_CTRL_CDTEST_EN	0x80
 
 #define SDHCI_POWER_CONTROL	0x29
 #define  SDHCI_POWER_ON		0x01
@@ -555,7 +557,6 @@ struct sdhci_ops {
 	void	(*set_uhs_signaling)(struct sdhci_host *host, unsigned int uhs);
 	void	(*hw_reset)(struct sdhci_host *host);
 	void    (*adma_workaround)(struct sdhci_host *host, u32 intmask);
-	void	(*platform_init)(struct sdhci_host *host);
 	void    (*card_event)(struct sdhci_host *host);
 	void	(*voltage_switch)(struct sdhci_host *host);
 	int	(*select_drive_strength)(struct sdhci_host *host,
diff --git a/drivers/mmc/host/sh_mobile_sdhi.c b/drivers/mmc/host/sh_mobile_sdhi.c
index c3b651bf89cb..49edff7fee49 100644
--- a/drivers/mmc/host/sh_mobile_sdhi.c
+++ b/drivers/mmc/host/sh_mobile_sdhi.c
@@ -94,6 +94,7 @@ static const struct of_device_id sh_mobile_sdhi_of_match[] = {
 	{ .compatible = "renesas,sdhi-r8a7793", .data = &of_rcar_gen2_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7794", .data = &of_rcar_gen2_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7795", .data = &of_rcar_gen3_compatible, },
+	{ .compatible = "renesas,sdhi-r8a7796", .data = &of_rcar_gen3_compatible, },
 	{},
 };
 MODULE_DEVICE_TABLE(of, sh_mobile_sdhi_of_match);
@@ -213,6 +214,13 @@ static void sh_mobile_sdhi_clk_disable(struct tmio_mmc_host *host)
 	clk_disable_unprepare(priv->clk);
 }
 
+static int sh_mobile_sdhi_card_busy(struct mmc_host *mmc)
+{
+	struct tmio_mmc_host *host = mmc_priv(mmc);
+
+	return !(sd_ctrl_read16_and_16_as_32(host, CTL_STATUS) & TMIO_STAT_DAT0);
+}
+
 static int sh_mobile_sdhi_start_signal_voltage_switch(struct mmc_host *mmc,
 						      struct mmc_ios *ios)
 {
@@ -369,7 +377,14 @@ static int sh_mobile_sdhi_probe(struct platform_device *pdev)
 	host->clk_update	= sh_mobile_sdhi_clk_update;
 	host->clk_disable	= sh_mobile_sdhi_clk_disable;
 	host->multi_io_quirk	= sh_mobile_sdhi_multi_io_quirk;
-	host->start_signal_voltage_switch = sh_mobile_sdhi_start_signal_voltage_switch;
+
+	/* SDR speeds are only available on Gen2+ */
+	if (mmc_data->flags & TMIO_MMC_MIN_RCAR2) {
+		/* card_busy caused issues on r8a73a4 (pre-Gen2) CD-less SDHI */
+		host->card_busy	= sh_mobile_sdhi_card_busy;
+		host->start_signal_voltage_switch =
+			sh_mobile_sdhi_start_signal_voltage_switch;
+	}
 
 	/* Orginally registers were 16 bit apart, could be 32 or 64 nowadays */
 	if (!host->bus_shift && resource_size(res) > 0x100) /* old way to determine the shift */
diff --git a/drivers/mmc/host/sunxi-mmc.c b/drivers/mmc/host/sunxi-mmc.c
index 2ee4c21ec55e..c0a5c676d0e8 100644
--- a/drivers/mmc/host/sunxi-mmc.c
+++ b/drivers/mmc/host/sunxi-mmc.c
@@ -72,6 +72,13 @@
 #define SDXC_REG_CHDA	(0x90)
 #define SDXC_REG_CBDA	(0x94)
 
+/* New registers introduced in A64 */
+#define SDXC_REG_A12A		0x058 /* SMC Auto Command 12 Register */
+#define SDXC_REG_SD_NTSR	0x05C /* SMC New Timing Set Register */
+#define SDXC_REG_DRV_DL		0x140 /* Drive Delay Control Register */
+#define SDXC_REG_SAMP_DL_REG	0x144 /* SMC sample delay control */
+#define SDXC_REG_DS_DL_REG	0x148 /* SMC data strobe delay control */
+
 #define mmc_readl(host, reg) \
 	readl((host)->reg_base + SDXC_##reg)
 #define mmc_writel(host, reg, value) \
@@ -217,21 +224,41 @@
 #define SDXC_CLK_50M_DDR	3
 #define SDXC_CLK_50M_DDR_8BIT	4
 
+#define SDXC_2X_TIMING_MODE	BIT(31)
+
+#define SDXC_CAL_START		BIT(15)
+#define SDXC_CAL_DONE		BIT(14)
+#define SDXC_CAL_DL_SHIFT	8
+#define SDXC_CAL_DL_SW_EN	BIT(7)
+#define SDXC_CAL_DL_SW_SHIFT	0
+#define SDXC_CAL_DL_MASK	0x3f
+
+#define SDXC_CAL_TIMEOUT	3	/* in seconds, 3s is enough*/
+
 struct sunxi_mmc_clk_delay {
 	u32 output;
 	u32 sample;
 };
 
 struct sunxi_idma_des {
-	u32	config;
-	u32	buf_size;
-	u32	buf_addr_ptr1;
-	u32	buf_addr_ptr2;
+	__le32 config;
+	__le32 buf_size;
+	__le32 buf_addr_ptr1;
+	__le32 buf_addr_ptr2;
+};
+
+struct sunxi_mmc_cfg {
+	u32 idma_des_size_bits;
+	const struct sunxi_mmc_clk_delay *clk_delays;
+
+	/* does the IP block support autocalibration? */
+	bool can_calibrate;
 };
 
 struct sunxi_mmc_host {
 	struct mmc_host	*mmc;
 	struct reset_control *reset;
+	const struct sunxi_mmc_cfg *cfg;
 
 	/* IO mapping base */
 	void __iomem	*reg_base;
@@ -241,7 +268,6 @@ struct sunxi_mmc_host {
 	struct clk	*clk_mmc;
 	struct clk	*clk_sample;
 	struct clk	*clk_output;
-	const struct sunxi_mmc_clk_delay *clk_delays;
 
 	/* irq */
 	spinlock_t	lock;
@@ -250,7 +276,6 @@ struct sunxi_mmc_host {
 	u32		sdio_imask;
 
 	/* dma */
-	u32		idma_des_size_bits;
 	dma_addr_t	sg_dma;
 	void		*sg_cpu;
 	bool		wait_dma;
@@ -322,25 +347,28 @@ static void sunxi_mmc_init_idma_des(struct sunxi_mmc_host *host,
 {
 	struct sunxi_idma_des *pdes = (struct sunxi_idma_des *)host->sg_cpu;
 	dma_addr_t next_desc = host->sg_dma;
-	int i, max_len = (1 << host->idma_des_size_bits);
+	int i, max_len = (1 << host->cfg->idma_des_size_bits);
 
 	for (i = 0; i < data->sg_len; i++) {
-		pdes[i].config = SDXC_IDMAC_DES0_CH | SDXC_IDMAC_DES0_OWN |
-				 SDXC_IDMAC_DES0_DIC;
+		pdes[i].config = cpu_to_le32(SDXC_IDMAC_DES0_CH |
+					     SDXC_IDMAC_DES0_OWN |
+					     SDXC_IDMAC_DES0_DIC);
 
 		if (data->sg[i].length == max_len)
 			pdes[i].buf_size = 0; /* 0 == max_len */
 		else
-			pdes[i].buf_size = data->sg[i].length;
+			pdes[i].buf_size = cpu_to_le32(data->sg[i].length);
 
 		next_desc += sizeof(struct sunxi_idma_des);
-		pdes[i].buf_addr_ptr1 = sg_dma_address(&data->sg[i]);
-		pdes[i].buf_addr_ptr2 = (u32)next_desc;
+		pdes[i].buf_addr_ptr1 =
+			cpu_to_le32(sg_dma_address(&data->sg[i]));
+		pdes[i].buf_addr_ptr2 = cpu_to_le32((u32)next_desc);
 	}
 
-	pdes[0].config |= SDXC_IDMAC_DES0_FD;
-	pdes[i - 1].config |= SDXC_IDMAC_DES0_LD | SDXC_IDMAC_DES0_ER;
-	pdes[i - 1].config &= ~SDXC_IDMAC_DES0_DIC;
+	pdes[0].config |= cpu_to_le32(SDXC_IDMAC_DES0_FD);
+	pdes[i - 1].config |= cpu_to_le32(SDXC_IDMAC_DES0_LD |
+					  SDXC_IDMAC_DES0_ER);
+	pdes[i - 1].config &= cpu_to_le32(~SDXC_IDMAC_DES0_DIC);
 	pdes[i - 1].buf_addr_ptr2 = 0;
 
 	/*
@@ -653,11 +681,84 @@ static int sunxi_mmc_oclk_onoff(struct sunxi_mmc_host *host, u32 oclk_en)
 	return 0;
 }
 
+static int sunxi_mmc_calibrate(struct sunxi_mmc_host *host, int reg_off)
+{
+	u32 reg = readl(host->reg_base + reg_off);
+	u32 delay;
+	unsigned long timeout;
+
+	if (!host->cfg->can_calibrate)
+		return 0;
+
+	reg &= ~(SDXC_CAL_DL_MASK << SDXC_CAL_DL_SW_SHIFT);
+	reg &= ~SDXC_CAL_DL_SW_EN;
+
+	writel(reg | SDXC_CAL_START, host->reg_base + reg_off);
+
+	dev_dbg(mmc_dev(host->mmc), "calibration started\n");
+
+	timeout = jiffies + HZ * SDXC_CAL_TIMEOUT;
+
+	while (!((reg = readl(host->reg_base + reg_off)) & SDXC_CAL_DONE)) {
+		if (time_before(jiffies, timeout))
+			cpu_relax();
+		else {
+			reg &= ~SDXC_CAL_START;
+			writel(reg, host->reg_base + reg_off);
+
+			return -ETIMEDOUT;
+		}
+	}
+
+	delay = (reg >> SDXC_CAL_DL_SHIFT) & SDXC_CAL_DL_MASK;
+
+	reg &= ~SDXC_CAL_START;
+	reg |= (delay << SDXC_CAL_DL_SW_SHIFT) | SDXC_CAL_DL_SW_EN;
+
+	writel(reg, host->reg_base + reg_off);
+
+	dev_dbg(mmc_dev(host->mmc), "calibration ended, reg is 0x%x\n", reg);
+
+	return 0;
+}
+
+static int sunxi_mmc_clk_set_phase(struct sunxi_mmc_host *host,
+				   struct mmc_ios *ios, u32 rate)
+{
+	int index;
+
+	if (!host->cfg->clk_delays)
+		return 0;
+
+	/* determine delays */
+	if (rate <= 400000) {
+		index = SDXC_CLK_400K;
+	} else if (rate <= 25000000) {
+		index = SDXC_CLK_25M;
+	} else if (rate <= 52000000) {
+		if (ios->timing != MMC_TIMING_UHS_DDR50 &&
+		    ios->timing != MMC_TIMING_MMC_DDR52) {
+			index = SDXC_CLK_50M;
+		} else if (ios->bus_width == MMC_BUS_WIDTH_8) {
+			index = SDXC_CLK_50M_DDR_8BIT;
+		} else {
+			index = SDXC_CLK_50M_DDR;
+		}
+	} else {
+		return -EINVAL;
+	}
+
+	clk_set_phase(host->clk_sample, host->cfg->clk_delays[index].sample);
+	clk_set_phase(host->clk_output, host->cfg->clk_delays[index].output);
+
+	return 0;
+}
+
 static int sunxi_mmc_clk_set_rate(struct sunxi_mmc_host *host,
 				  struct mmc_ios *ios)
 {
-	u32 rate, oclk_dly, rval, sclk_dly;
-	u32 clock = ios->clock;
+	long rate;
+	u32 rval, clock = ios->clock;
 	int ret;
 
 	/* 8 bit DDR requires a higher module clock */
@@ -666,13 +767,18 @@ static int sunxi_mmc_clk_set_rate(struct sunxi_mmc_host *host,
 		clock <<= 1;
 
 	rate = clk_round_rate(host->clk_mmc, clock);
-	dev_dbg(mmc_dev(host->mmc), "setting clk to %d, rounded %d\n",
+	if (rate < 0) {
+		dev_err(mmc_dev(host->mmc), "error rounding clk to %d: %ld\n",
+			clock, rate);
+		return rate;
+	}
+	dev_dbg(mmc_dev(host->mmc), "setting clk to %d, rounded %ld\n",
 		clock, rate);
 
 	/* setting clock rate */
 	ret = clk_set_rate(host->clk_mmc, rate);
 	if (ret) {
-		dev_err(mmc_dev(host->mmc), "error setting clk to %d: %d\n",
+		dev_err(mmc_dev(host->mmc), "error setting clk to %ld: %d\n",
 			rate, ret);
 		return ret;
 	}
@@ -692,31 +798,15 @@ static int sunxi_mmc_clk_set_rate(struct sunxi_mmc_host *host,
 	}
 	mmc_writel(host, REG_CLKCR, rval);
 
-	/* determine delays */
-	if (rate <= 400000) {
-		oclk_dly = host->clk_delays[SDXC_CLK_400K].output;
-		sclk_dly = host->clk_delays[SDXC_CLK_400K].sample;
-	} else if (rate <= 25000000) {
-		oclk_dly = host->clk_delays[SDXC_CLK_25M].output;
-		sclk_dly = host->clk_delays[SDXC_CLK_25M].sample;
-	} else if (rate <= 52000000) {
-		if (ios->timing != MMC_TIMING_UHS_DDR50 &&
-		    ios->timing != MMC_TIMING_MMC_DDR52) {
-			oclk_dly = host->clk_delays[SDXC_CLK_50M].output;
-			sclk_dly = host->clk_delays[SDXC_CLK_50M].sample;
-		} else if (ios->bus_width == MMC_BUS_WIDTH_8) {
-			oclk_dly = host->clk_delays[SDXC_CLK_50M_DDR_8BIT].output;
-			sclk_dly = host->clk_delays[SDXC_CLK_50M_DDR_8BIT].sample;
-		} else {
-			oclk_dly = host->clk_delays[SDXC_CLK_50M_DDR].output;
-			sclk_dly = host->clk_delays[SDXC_CLK_50M_DDR].sample;
-		}
-	} else {
-		return -EINVAL;
-	}
+	ret = sunxi_mmc_clk_set_phase(host, ios, rate);
+	if (ret)
+		return ret;
+
+	ret = sunxi_mmc_calibrate(host, SDXC_REG_SAMP_DL_REG);
+	if (ret)
+		return ret;
 
-	clk_set_phase(host->clk_sample, sclk_dly);
-	clk_set_phase(host->clk_output, oclk_dly);
+	/* TODO: enable calibrate on sdc2 SDXC_REG_DS_DL_REG of A64 */
 
 	return sunxi_mmc_oclk_onoff(host, 1);
 }
@@ -938,14 +1028,6 @@ static int sunxi_mmc_card_busy(struct mmc_host *mmc)
 	return !!(mmc_readl(host, REG_STAS) & SDXC_CARD_DATA_BUSY);
 }
 
-static const struct of_device_id sunxi_mmc_of_match[] = {
-	{ .compatible = "allwinner,sun4i-a10-mmc", },
-	{ .compatible = "allwinner,sun5i-a13-mmc", },
-	{ .compatible = "allwinner,sun9i-a80-mmc", },
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, sunxi_mmc_of_match);
-
 static struct mmc_host_ops sunxi_mmc_ops = {
 	.request	 = sunxi_mmc_request,
 	.set_ios	 = sunxi_mmc_set_ios,
@@ -974,21 +1056,54 @@ static const struct sunxi_mmc_clk_delay sun9i_mmc_clk_delays[] = {
 	[SDXC_CLK_50M_DDR_8BIT]	= { .output =  72, .sample =  72 },
 };
 
+static const struct sunxi_mmc_cfg sun4i_a10_cfg = {
+	.idma_des_size_bits = 13,
+	.clk_delays = NULL,
+	.can_calibrate = false,
+};
+
+static const struct sunxi_mmc_cfg sun5i_a13_cfg = {
+	.idma_des_size_bits = 16,
+	.clk_delays = NULL,
+	.can_calibrate = false,
+};
+
+static const struct sunxi_mmc_cfg sun7i_a20_cfg = {
+	.idma_des_size_bits = 16,
+	.clk_delays = sunxi_mmc_clk_delays,
+	.can_calibrate = false,
+};
+
+static const struct sunxi_mmc_cfg sun9i_a80_cfg = {
+	.idma_des_size_bits = 16,
+	.clk_delays = sun9i_mmc_clk_delays,
+	.can_calibrate = false,
+};
+
+static const struct sunxi_mmc_cfg sun50i_a64_cfg = {
+	.idma_des_size_bits = 16,
+	.clk_delays = NULL,
+	.can_calibrate = true,
+};
+
+static const struct of_device_id sunxi_mmc_of_match[] = {
+	{ .compatible = "allwinner,sun4i-a10-mmc", .data = &sun4i_a10_cfg },
+	{ .compatible = "allwinner,sun5i-a13-mmc", .data = &sun5i_a13_cfg },
+	{ .compatible = "allwinner,sun7i-a20-mmc", .data = &sun7i_a20_cfg },
+	{ .compatible = "allwinner,sun9i-a80-mmc", .data = &sun9i_a80_cfg },
+	{ .compatible = "allwinner,sun50i-a64-mmc", .data = &sun50i_a64_cfg },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, sunxi_mmc_of_match);
+
 static int sunxi_mmc_resource_request(struct sunxi_mmc_host *host,
 				      struct platform_device *pdev)
 {
-	struct device_node *np = pdev->dev.of_node;
 	int ret;
 
-	if (of_device_is_compatible(np, "allwinner,sun4i-a10-mmc"))
-		host->idma_des_size_bits = 13;
-	else
-		host->idma_des_size_bits = 16;
-
-	if (of_device_is_compatible(np, "allwinner,sun9i-a80-mmc"))
-		host->clk_delays = sun9i_mmc_clk_delays;
-	else
-		host->clk_delays = sunxi_mmc_clk_delays;
+	host->cfg = of_device_get_match_data(&pdev->dev);
+	if (!host->cfg)
+		return -EINVAL;
 
 	ret = mmc_regulator_get_supply(host->mmc);
 	if (ret) {
@@ -1014,16 +1129,18 @@ static int sunxi_mmc_resource_request(struct sunxi_mmc_host *host,
 		return PTR_ERR(host->clk_mmc);
 	}
 
-	host->clk_output = devm_clk_get(&pdev->dev, "output");
-	if (IS_ERR(host->clk_output)) {
-		dev_err(&pdev->dev, "Could not get output clock\n");
-		return PTR_ERR(host->clk_output);
-	}
+	if (host->cfg->clk_delays) {
+		host->clk_output = devm_clk_get(&pdev->dev, "output");
+		if (IS_ERR(host->clk_output)) {
+			dev_err(&pdev->dev, "Could not get output clock\n");
+			return PTR_ERR(host->clk_output);
+		}
 
-	host->clk_sample = devm_clk_get(&pdev->dev, "sample");
-	if (IS_ERR(host->clk_sample)) {
-		dev_err(&pdev->dev, "Could not get sample clock\n");
-		return PTR_ERR(host->clk_sample);
+		host->clk_sample = devm_clk_get(&pdev->dev, "sample");
+		if (IS_ERR(host->clk_sample)) {
+			dev_err(&pdev->dev, "Could not get sample clock\n");
+			return PTR_ERR(host->clk_sample);
+		}
 	}
 
 	host->reset = devm_reset_control_get_optional(&pdev->dev, "ahb");
@@ -1120,15 +1237,17 @@ static int sunxi_mmc_probe(struct platform_device *pdev)
 	mmc->max_blk_count	= 8192;
 	mmc->max_blk_size	= 4096;
 	mmc->max_segs		= PAGE_SIZE / sizeof(struct sunxi_idma_des);
-	mmc->max_seg_size	= (1 << host->idma_des_size_bits);
+	mmc->max_seg_size	= (1 << host->cfg->idma_des_size_bits);
 	mmc->max_req_size	= mmc->max_seg_size * mmc->max_segs;
 	/* 400kHz ~ 52MHz */
 	mmc->f_min		=   400000;
 	mmc->f_max		= 52000000;
 	mmc->caps	       |= MMC_CAP_MMC_HIGHSPEED | MMC_CAP_SD_HIGHSPEED |
-				  MMC_CAP_1_8V_DDR |
 				  MMC_CAP_ERASE | MMC_CAP_SDIO_IRQ;
 
+	if (host->cfg->clk_delays)
+		mmc->caps      |= MMC_CAP_1_8V_DDR;
+
 	ret = mmc_of_parse(mmc);
 	if (ret)
 		goto error_free_dma;
@@ -1160,6 +1279,8 @@ static int sunxi_mmc_remove(struct platform_device *pdev)
 	if (!IS_ERR(host->reset))
 		reset_control_assert(host->reset);
 
+	clk_disable_unprepare(host->clk_sample);
+	clk_disable_unprepare(host->clk_output);
 	clk_disable_unprepare(host->clk_mmc);
 	clk_disable_unprepare(host->clk_ahb);
 
diff --git a/drivers/mmc/host/tmio_mmc.h b/drivers/mmc/host/tmio_mmc.h
index 7f63ec05bdf4..8e126afd988c 100644
--- a/drivers/mmc/host/tmio_mmc.h
+++ b/drivers/mmc/host/tmio_mmc.h
@@ -79,6 +79,9 @@
 #define	CLK_CTL_DIV_MASK	0xff
 #define	CLK_CTL_SCLKEN		BIT(8)
 
+#define CARD_OPT_WIDTH8		BIT(13)
+#define CARD_OPT_WIDTH		BIT(15)
+
 #define TMIO_BBS		512		/* Boot block size */
 
 /* Definitions for values the CTRL_SDIO_STATUS register can take. */
@@ -158,6 +161,7 @@ struct tmio_mmc_host {
 	void (*clk_disable)(struct tmio_mmc_host *host);
 	int (*multi_io_quirk)(struct mmc_card *card,
 			      unsigned int direction, int blk_size);
+	int (*card_busy)(struct mmc_host *mmc);
 	int (*start_signal_voltage_switch)(struct mmc_host *mmc,
 					   struct mmc_ios *ios);
 };
diff --git a/drivers/mmc/host/tmio_mmc_pio.c b/drivers/mmc/host/tmio_mmc_pio.c
index 92467efc4e2c..700567603107 100644
--- a/drivers/mmc/host/tmio_mmc_pio.c
+++ b/drivers/mmc/host/tmio_mmc_pio.c
@@ -336,7 +336,9 @@ static int tmio_mmc_start_command(struct tmio_mmc_host *host, struct mmc_command
 
 	switch (mmc_resp_type(cmd)) {
 	case MMC_RSP_NONE: c |= RESP_NONE; break;
-	case MMC_RSP_R1:   c |= RESP_R1;   break;
+	case MMC_RSP_R1:
+	case MMC_RSP_R1_NO_CRC:
+			   c |= RESP_R1;   break;
 	case MMC_RSP_R1B:  c |= RESP_R1B;  break;
 	case MMC_RSP_R2:   c |= RESP_R2;   break;
 	case MMC_RSP_R3:   c |= RESP_R3;   break;
@@ -730,12 +732,13 @@ static int tmio_mmc_start_data(struct tmio_mmc_host *host,
 	pr_debug("setup data transfer: blocksize %08x  nr_blocks %d\n",
 		 data->blksz, data->blocks);
 
-	/* Some hardware cannot perform 2 byte requests in 4 bit mode */
-	if (host->mmc->ios.bus_width == MMC_BUS_WIDTH_4) {
+	/* Some hardware cannot perform 2 byte requests in 4/8 bit mode */
+	if (host->mmc->ios.bus_width == MMC_BUS_WIDTH_4 ||
+	    host->mmc->ios.bus_width == MMC_BUS_WIDTH_8) {
 		int blksz_2bytes = pdata->flags & TMIO_MMC_BLKSZ_2BYTES;
 
 		if (data->blksz < 2 || (data->blksz < 4 && !blksz_2bytes)) {
-			pr_err("%s: %d byte block unsupported in 4 bit mode\n",
+			pr_err("%s: %d byte block unsupported in 4/8 bit mode\n",
 			       mmc_hostname(host->mmc), data->blksz);
 			return -EINVAL;
 		}
@@ -857,14 +860,16 @@ static void tmio_mmc_power_off(struct tmio_mmc_host *host)
 static void tmio_mmc_set_bus_width(struct tmio_mmc_host *host,
 				unsigned char bus_width)
 {
-	switch (bus_width) {
-	case MMC_BUS_WIDTH_1:
-		sd_ctrl_write16(host, CTL_SD_MEM_CARD_OPT, 0x80e0);
-		break;
-	case MMC_BUS_WIDTH_4:
-		sd_ctrl_write16(host, CTL_SD_MEM_CARD_OPT, 0x00e0);
-		break;
-	}
+	u16 reg = sd_ctrl_read16(host, CTL_SD_MEM_CARD_OPT)
+				& ~(CARD_OPT_WIDTH | CARD_OPT_WIDTH8);
+
+	/* reg now applies to MMC_BUS_WIDTH_4 */
+	if (bus_width == MMC_BUS_WIDTH_1)
+		reg |= CARD_OPT_WIDTH;
+	else if (bus_width == MMC_BUS_WIDTH_8)
+		reg |= CARD_OPT_WIDTH8;
+
+	sd_ctrl_write16(host, CTL_SD_MEM_CARD_OPT, reg);
 }
 
 /* Set MMC clock / power.
@@ -960,20 +965,12 @@ static int tmio_multi_io_quirk(struct mmc_card *card,
 	return blk_size;
 }
 
-static int tmio_mmc_card_busy(struct mmc_host *mmc)
-{
-	struct tmio_mmc_host *host = mmc_priv(mmc);
-
-	return !(sd_ctrl_read16_and_16_as_32(host, CTL_STATUS) & TMIO_STAT_DAT0);
-}
-
 static struct mmc_host_ops tmio_mmc_ops = {
 	.request	= tmio_mmc_request,
 	.set_ios	= tmio_mmc_set_ios,
 	.get_ro         = tmio_mmc_get_ro,
 	.get_cd		= mmc_gpio_get_cd,
 	.enable_sdio_irq = tmio_mmc_enable_sdio_irq,
-	.card_busy	= tmio_mmc_card_busy,
 	.multi_io_quirk	= tmio_multi_io_quirk,
 };
 
@@ -1072,6 +1069,7 @@ int tmio_mmc_host_probe(struct tmio_mmc_host *_host,
 		goto host_free;
 	}
 
+	tmio_mmc_ops.card_busy = _host->card_busy;
 	tmio_mmc_ops.start_signal_voltage_switch = _host->start_signal_voltage_switch;
 	mmc->ops = &tmio_mmc_ops;
 
@@ -1089,6 +1087,15 @@ int tmio_mmc_host_probe(struct tmio_mmc_host *_host,
 				  !mmc_card_is_removable(mmc) ||
 				  mmc->slot.cd_irq >= 0);
 
+	/*
+	 * On Gen2+, eMMC with NONREMOVABLE currently fails because native
+	 * hotplug gets disabled. It seems RuntimePM related yet we need further
+	 * research. Since we are planning a PM overhaul anyway, let's enforce
+	 * for now the device being active by enabling native hotplug always.
+	 */
+	if (pdata->flags & TMIO_MMC_MIN_RCAR2)
+		_host->native_hotplug = true;
+
 	if (tmio_mmc_clk_enable(_host) < 0) {
 		mmc->f_max = pdata->hclk;
 		mmc->f_min = mmc->f_max / 512;
diff --git a/drivers/mmc/host/vub300.c b/drivers/mmc/host/vub300.c
index 1e819f98b94f..bb3e0d1dd355 100644
--- a/drivers/mmc/host/vub300.c
+++ b/drivers/mmc/host/vub300.c
@@ -2116,13 +2116,11 @@ static int vub300_probe(struct usb_interface *interface,
 	command_out_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!command_out_urb) {
 		retval = -ENOMEM;
-		dev_err(&udev->dev, "not enough memory for command_out_urb\n");
 		goto error0;
 	}
 	command_res_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!command_res_urb) {
 		retval = -ENOMEM;
-		dev_err(&udev->dev, "not enough memory for command_res_urb\n");
 		goto error1;
 	}
 	/* this also allocates memory for our VUB300 mmc host device */
diff --git a/drivers/mtd/maps/Kconfig b/drivers/mtd/maps/Kconfig
index 392f9eff5fb7..5bcc896a48c3 100644
--- a/drivers/mtd/maps/Kconfig
+++ b/drivers/mtd/maps/Kconfig
@@ -78,7 +78,7 @@ config MTD_PHYSMAP_OF_VERSATILE
 	bool "Support ARM Versatile physmap OF"
 	depends on MTD_PHYSMAP_OF
 	depends on MFD_SYSCON
-	default y if (ARCH_INTEGRATOR || ARCH_VERSATILE || REALVIEW_DT)
+	default y if (ARCH_INTEGRATOR || ARCH_VERSATILE || ARCH_REALVIEW)
 	help
 	  This provides some extra DT physmap parsing for the ARM Versatile
 	  platforms, basically to add a VPP (write protection) callback so
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index e3936b847c6b..d46e4adf6d2b 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -39,7 +39,6 @@
 #include <linux/gfp.h>
 #include <linux/slab.h>
 #include <linux/reboot.h>
-#include <linux/kconfig.h>
 #include <linux/leds.h>
 
 #include <linux/mtd/mtd.h>
@@ -376,6 +375,110 @@ static int mtd_reboot_notifier(struct notifier_block *n, unsigned long state,
 }
 
 /**
+ * mtd_wunit_to_pairing_info - get pairing information of a wunit
+ * @mtd: pointer to new MTD device info structure
+ * @wunit: write unit we are interested in
+ * @info: returned pairing information
+ *
+ * Retrieve pairing information associated to the wunit.
+ * This is mainly useful when dealing with MLC/TLC NANDs where pages can be
+ * paired together, and where programming a page may influence the page it is
+ * paired with.
+ * The notion of page is replaced by the term wunit (write-unit) to stay
+ * consistent with the ->writesize field.
+ *
+ * The @wunit argument can be extracted from an absolute offset using
+ * mtd_offset_to_wunit(). @info is filled with the pairing information attached
+ * to @wunit.
+ *
+ * From the pairing info the MTD user can find all the wunits paired with
+ * @wunit using the following loop:
+ *
+ * for (i = 0; i < mtd_pairing_groups(mtd); i++) {
+ *	info.pair = i;
+ *	mtd_pairing_info_to_wunit(mtd, &info);
+ *	...
+ * }
+ */
+int mtd_wunit_to_pairing_info(struct mtd_info *mtd, int wunit,
+			      struct mtd_pairing_info *info)
+{
+	int npairs = mtd_wunit_per_eb(mtd) / mtd_pairing_groups(mtd);
+
+	if (wunit < 0 || wunit >= npairs)
+		return -EINVAL;
+
+	if (mtd->pairing && mtd->pairing->get_info)
+		return mtd->pairing->get_info(mtd, wunit, info);
+
+	info->group = 0;
+	info->pair = wunit;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mtd_wunit_to_pairing_info);
+
+/**
+ * mtd_wunit_to_pairing_info - get wunit from pairing information
+ * @mtd: pointer to new MTD device info structure
+ * @info: pairing information struct
+ *
+ * Returns a positive number representing the wunit associated to the info
+ * struct, or a negative error code.
+ *
+ * This is the reverse of mtd_wunit_to_pairing_info(), and can help one to
+ * iterate over all wunits of a given pair (see mtd_wunit_to_pairing_info()
+ * doc).
+ *
+ * It can also be used to only program the first page of each pair (i.e.
+ * page attached to group 0), which allows one to use an MLC NAND in
+ * software-emulated SLC mode:
+ *
+ * info.group = 0;
+ * npairs = mtd_wunit_per_eb(mtd) / mtd_pairing_groups(mtd);
+ * for (info.pair = 0; info.pair < npairs; info.pair++) {
+ *	wunit = mtd_pairing_info_to_wunit(mtd, &info);
+ *	mtd_write(mtd, mtd_wunit_to_offset(mtd, blkoffs, wunit),
+ *		  mtd->writesize, &retlen, buf + (i * mtd->writesize));
+ * }
+ */
+int mtd_pairing_info_to_wunit(struct mtd_info *mtd,
+			      const struct mtd_pairing_info *info)
+{
+	int ngroups = mtd_pairing_groups(mtd);
+	int npairs = mtd_wunit_per_eb(mtd) / ngroups;
+
+	if (!info || info->pair < 0 || info->pair >= npairs ||
+	    info->group < 0 || info->group >= ngroups)
+		return -EINVAL;
+
+	if (mtd->pairing && mtd->pairing->get_wunit)
+		return mtd->pairing->get_wunit(mtd, info);
+
+	return info->pair;
+}
+EXPORT_SYMBOL_GPL(mtd_pairing_info_to_wunit);
+
+/**
+ * mtd_pairing_groups - get the number of pairing groups
+ * @mtd: pointer to new MTD device info structure
+ *
+ * Returns the number of pairing groups.
+ *
+ * This number is usually equal to the number of bits exposed by a single
+ * cell, and can be used in conjunction with mtd_pairing_info_to_wunit()
+ * to iterate over all pages of a given pair.
+ */
+int mtd_pairing_groups(struct mtd_info *mtd)
+{
+	if (!mtd->pairing || !mtd->pairing->ngroups)
+		return 1;
+
+	return mtd->pairing->ngroups;
+}
+EXPORT_SYMBOL_GPL(mtd_pairing_groups);
+
+/**
  *	add_mtd_device - register an MTD device
  *	@mtd: pointer to new MTD device info structure
  *
diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c
index 1f13e32556f8..fccdd49bb964 100644
--- a/drivers/mtd/mtdpart.c
+++ b/drivers/mtd/mtdpart.c
@@ -30,7 +30,6 @@
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/partitions.h>
 #include <linux/err.h>
-#include <linux/kconfig.h>
 
 #include "mtdcore.h"
 
@@ -317,6 +316,18 @@ static int part_block_markbad(struct mtd_info *mtd, loff_t ofs)
 	return res;
 }
 
+static int part_get_device(struct mtd_info *mtd)
+{
+	struct mtd_part *part = mtd_to_part(mtd);
+	return part->master->_get_device(part->master);
+}
+
+static void part_put_device(struct mtd_info *mtd)
+{
+	struct mtd_part *part = mtd_to_part(mtd);
+	part->master->_put_device(part->master);
+}
+
 static int part_ooblayout_ecc(struct mtd_info *mtd, int section,
 			      struct mtd_oob_region *oobregion)
 {
@@ -397,6 +408,7 @@ static struct mtd_part *allocate_partition(struct mtd_info *master,
 	slave->mtd.oobsize = master->oobsize;
 	slave->mtd.oobavail = master->oobavail;
 	slave->mtd.subpage_sft = master->subpage_sft;
+	slave->mtd.pairing = master->pairing;
 
 	slave->mtd.name = name;
 	slave->mtd.owner = master->owner;
@@ -463,6 +475,12 @@ static struct mtd_part *allocate_partition(struct mtd_info *master,
 		slave->mtd._block_isbad = part_block_isbad;
 	if (master->_block_markbad)
 		slave->mtd._block_markbad = part_block_markbad;
+
+	if (master->_get_device)
+		slave->mtd._get_device = part_get_device;
+	if (master->_put_device)
+		slave->mtd._put_device = part_put_device;
+
 	slave->mtd._erase = part_erase;
 	slave->master = master;
 	slave->offset = part->offset;
diff --git a/drivers/mtd/nand/Kconfig b/drivers/mtd/nand/Kconfig
index 21ff58099f3b..7b7a887b4709 100644
--- a/drivers/mtd/nand/Kconfig
+++ b/drivers/mtd/nand/Kconfig
@@ -88,11 +88,11 @@ config MTD_NAND_AMS_DELTA
 	  Support for NAND flash on Amstrad E3 (Delta).
 
 config MTD_NAND_OMAP2
-	tristate "NAND Flash device on OMAP2, OMAP3 and OMAP4"
-	depends on ARCH_OMAP2PLUS
+	tristate "NAND Flash device on OMAP2, OMAP3, OMAP4 and Keystone"
+	depends on (ARCH_OMAP2PLUS || ARCH_KEYSTONE)
 	help
-          Support for NAND flash on Texas Instruments OMAP2, OMAP3 and OMAP4
-	  platforms.
+          Support for NAND flash on Texas Instruments OMAP2, OMAP3, OMAP4
+	  and Keystone platforms.
 
 config MTD_NAND_OMAP_BCH
 	depends on MTD_NAND_OMAP2
@@ -428,7 +428,7 @@ config MTD_NAND_ORION
 
 config MTD_NAND_FSL_ELBC
 	tristate "NAND support for Freescale eLBC controllers"
-	depends on PPC
+	depends on FSL_SOC
 	select FSL_LBC
 	help
 	  Various Freescale chips, including the 8313, include a NAND Flash
@@ -438,7 +438,7 @@ config MTD_NAND_FSL_ELBC
 
 config MTD_NAND_FSL_IFC
 	tristate "NAND support for Freescale IFC controller"
-	depends on MTD_NAND && (FSL_SOC || ARCH_LAYERSCAPE)
+	depends on FSL_SOC || ARCH_LAYERSCAPE
 	select FSL_IFC
 	select MEMORY
 	help
diff --git a/drivers/mtd/nand/bf5xx_nand.c b/drivers/mtd/nand/bf5xx_nand.c
index 37da4236ab90..3962f55bd034 100644
--- a/drivers/mtd/nand/bf5xx_nand.c
+++ b/drivers/mtd/nand/bf5xx_nand.c
@@ -761,8 +761,7 @@ static int bf5xx_nand_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, info);
 
-	spin_lock_init(&info->controller.lock);
-	init_waitqueue_head(&info->controller.wq);
+	nand_hw_control_init(&info->controller);
 
 	info->device     = &pdev->dev;
 	info->platform   = plat;
diff --git a/drivers/mtd/nand/brcmnand/brcmnand.c b/drivers/mtd/nand/brcmnand/brcmnand.c
index 8eb2c64df38c..9d2424bfdbf5 100644
--- a/drivers/mtd/nand/brcmnand/brcmnand.c
+++ b/drivers/mtd/nand/brcmnand/brcmnand.c
@@ -1336,7 +1336,7 @@ static void brcmnand_cmdfunc(struct mtd_info *mtd, unsigned command,
 		u32 *flash_cache = (u32 *)ctrl->flash_cache;
 		int i;
 
-		brcmnand_soc_data_bus_prepare(ctrl->soc);
+		brcmnand_soc_data_bus_prepare(ctrl->soc, true);
 
 		/*
 		 * Must cache the FLASH_CACHE now, since changes in
@@ -1349,7 +1349,7 @@ static void brcmnand_cmdfunc(struct mtd_info *mtd, unsigned command,
 			 */
 			flash_cache[i] = be32_to_cpu(brcmnand_read_fc(ctrl, i));
 
-		brcmnand_soc_data_bus_unprepare(ctrl->soc);
+		brcmnand_soc_data_bus_unprepare(ctrl->soc, true);
 
 		/* Cleanup from HW quirk: restore SECTOR_SIZE_1K */
 		if (host->hwcfg.sector_size_1k)
@@ -1565,12 +1565,12 @@ static int brcmnand_read_by_pio(struct mtd_info *mtd, struct nand_chip *chip,
 		brcmnand_waitfunc(mtd, chip);
 
 		if (likely(buf)) {
-			brcmnand_soc_data_bus_prepare(ctrl->soc);
+			brcmnand_soc_data_bus_prepare(ctrl->soc, false);
 
 			for (j = 0; j < FC_WORDS; j++, buf++)
 				*buf = brcmnand_read_fc(ctrl, j);
 
-			brcmnand_soc_data_bus_unprepare(ctrl->soc);
+			brcmnand_soc_data_bus_unprepare(ctrl->soc, false);
 		}
 
 		if (oob)
@@ -1815,12 +1815,12 @@ static int brcmnand_write(struct mtd_info *mtd, struct nand_chip *chip,
 		(void)brcmnand_read_reg(ctrl, BRCMNAND_CMD_ADDRESS);
 
 		if (buf) {
-			brcmnand_soc_data_bus_prepare(ctrl->soc);
+			brcmnand_soc_data_bus_prepare(ctrl->soc, false);
 
 			for (j = 0; j < FC_WORDS; j++, buf++)
 				brcmnand_write_fc(ctrl, j, *buf);
 
-			brcmnand_soc_data_bus_unprepare(ctrl->soc);
+			brcmnand_soc_data_bus_unprepare(ctrl->soc, false);
 		} else if (oob) {
 			for (j = 0; j < FC_WORDS; j++)
 				brcmnand_write_fc(ctrl, j, 0xffffffff);
@@ -2370,8 +2370,7 @@ int brcmnand_probe(struct platform_device *pdev, struct brcmnand_soc *soc)
 
 	init_completion(&ctrl->done);
 	init_completion(&ctrl->dma_done);
-	spin_lock_init(&ctrl->controller.lock);
-	init_waitqueue_head(&ctrl->controller.wq);
+	nand_hw_control_init(&ctrl->controller);
 	INIT_LIST_HEAD(&ctrl->host_list);
 
 	/* NAND register range */
diff --git a/drivers/mtd/nand/brcmnand/brcmnand.h b/drivers/mtd/nand/brcmnand/brcmnand.h
index ef5eabba88e5..5c44cd4aba87 100644
--- a/drivers/mtd/nand/brcmnand/brcmnand.h
+++ b/drivers/mtd/nand/brcmnand/brcmnand.h
@@ -23,19 +23,22 @@ struct dev_pm_ops;
 struct brcmnand_soc {
 	bool (*ctlrdy_ack)(struct brcmnand_soc *soc);
 	void (*ctlrdy_set_enabled)(struct brcmnand_soc *soc, bool en);
-	void (*prepare_data_bus)(struct brcmnand_soc *soc, bool prepare);
+	void (*prepare_data_bus)(struct brcmnand_soc *soc, bool prepare,
+				 bool is_param);
 };
 
-static inline void brcmnand_soc_data_bus_prepare(struct brcmnand_soc *soc)
+static inline void brcmnand_soc_data_bus_prepare(struct brcmnand_soc *soc,
+						 bool is_param)
 {
 	if (soc && soc->prepare_data_bus)
-		soc->prepare_data_bus(soc, true);
+		soc->prepare_data_bus(soc, true, is_param);
 }
 
-static inline void brcmnand_soc_data_bus_unprepare(struct brcmnand_soc *soc)
+static inline void brcmnand_soc_data_bus_unprepare(struct brcmnand_soc *soc,
+						   bool is_param)
 {
 	if (soc && soc->prepare_data_bus)
-		soc->prepare_data_bus(soc, false);
+		soc->prepare_data_bus(soc, false, is_param);
 }
 
 static inline u32 brcmnand_readl(void __iomem *addr)
diff --git a/drivers/mtd/nand/brcmnand/iproc_nand.c b/drivers/mtd/nand/brcmnand/iproc_nand.c
index 585596c549b2..4c6ae113664d 100644
--- a/drivers/mtd/nand/brcmnand/iproc_nand.c
+++ b/drivers/mtd/nand/brcmnand/iproc_nand.c
@@ -74,7 +74,8 @@ static void iproc_nand_intc_set(struct brcmnand_soc *soc, bool en)
 	spin_unlock_irqrestore(&priv->idm_lock, flags);
 }
 
-static void iproc_nand_apb_access(struct brcmnand_soc *soc, bool prepare)
+static void iproc_nand_apb_access(struct brcmnand_soc *soc, bool prepare,
+				  bool is_param)
 {
 	struct iproc_nand_soc *priv =
 			container_of(soc, struct iproc_nand_soc, soc);
@@ -86,10 +87,19 @@ static void iproc_nand_apb_access(struct brcmnand_soc *soc, bool prepare)
 
 	val = brcmnand_readl(mmio);
 
-	if (prepare)
-		val |= IPROC_NAND_APB_LE_MODE;
-	else
+	/*
+	 * In the case of BE or when dealing with NAND data, alway configure
+	 * the APB bus to LE mode before accessing the FIFO and back to BE mode
+	 * after the access is done
+	 */
+	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) || !is_param) {
+		if (prepare)
+			val |= IPROC_NAND_APB_LE_MODE;
+		else
+			val &= ~IPROC_NAND_APB_LE_MODE;
+	} else { /* when in LE accessing the parameter page, keep APB in BE */
 		val &= ~IPROC_NAND_APB_LE_MODE;
+	}
 
 	brcmnand_writel(val, mmio);
 
diff --git a/drivers/mtd/nand/davinci_nand.c b/drivers/mtd/nand/davinci_nand.c
index cc07ba0f044d..27fa8b87cd5f 100644
--- a/drivers/mtd/nand/davinci_nand.c
+++ b/drivers/mtd/nand/davinci_nand.c
@@ -240,6 +240,9 @@ static void nand_davinci_hwctl_4bit(struct mtd_info *mtd, int mode)
 	unsigned long flags;
 	u32 val;
 
+	/* Reset ECC hardware */
+	davinci_nand_readl(info, NAND_4BIT_ECC1_OFFSET);
+
 	spin_lock_irqsave(&davinci_nand_lock, flags);
 
 	/* Start 4-bit ECC calculation for read/write */
diff --git a/drivers/mtd/nand/docg4.c b/drivers/mtd/nand/docg4.c
index 47316998017f..7af2a3cd949e 100644
--- a/drivers/mtd/nand/docg4.c
+++ b/drivers/mtd/nand/docg4.c
@@ -1249,8 +1249,7 @@ static void __init init_mtd_structs(struct mtd_info *mtd)
 	nand->options = NAND_BUSWIDTH_16 | NAND_NO_SUBPAGE_WRITE;
 	nand->IO_ADDR_R = nand->IO_ADDR_W = doc->virtadr + DOC_IOSPACE_DATA;
 	nand->controller = &nand->hwcontrol;
-	spin_lock_init(&nand->controller->lock);
-	init_waitqueue_head(&nand->controller->wq);
+	nand_hw_control_init(nand->controller);
 
 	/* methods */
 	nand->cmdfunc = docg4_command;
diff --git a/drivers/mtd/nand/fsl_elbc_nand.c b/drivers/mtd/nand/fsl_elbc_nand.c
index 60a88f24c6b3..113f76e59937 100644
--- a/drivers/mtd/nand/fsl_elbc_nand.c
+++ b/drivers/mtd/nand/fsl_elbc_nand.c
@@ -879,8 +879,7 @@ static int fsl_elbc_nand_probe(struct platform_device *pdev)
 		}
 		elbc_fcm_ctrl->counter++;
 
-		spin_lock_init(&elbc_fcm_ctrl->controller.lock);
-		init_waitqueue_head(&elbc_fcm_ctrl->controller.wq);
+		nand_hw_control_init(&elbc_fcm_ctrl->controller);
 		fsl_lbc_ctrl_dev->nand = elbc_fcm_ctrl;
 	} else {
 		elbc_fcm_ctrl = fsl_lbc_ctrl_dev->nand;
diff --git a/drivers/mtd/nand/fsl_ifc_nand.c b/drivers/mtd/nand/fsl_ifc_nand.c
index 4e9e5fd8faf3..0a177b1bfe3e 100644
--- a/drivers/mtd/nand/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/fsl_ifc_nand.c
@@ -987,8 +987,7 @@ static int fsl_ifc_nand_probe(struct platform_device *dev)
 		ifc_nand_ctrl->addr = NULL;
 		fsl_ifc_ctrl_dev->nand = ifc_nand_ctrl;
 
-		spin_lock_init(&ifc_nand_ctrl->controller.lock);
-		init_waitqueue_head(&ifc_nand_ctrl->controller.wq);
+		nand_hw_control_init(&ifc_nand_ctrl->controller);
 	} else {
 		ifc_nand_ctrl = fsl_ifc_ctrl_dev->nand;
 	}
diff --git a/drivers/mtd/nand/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/gpmi-nand/gpmi-nand.c
index 6e461560c6a8..6c062b8251d2 100644
--- a/drivers/mtd/nand/gpmi-nand/gpmi-nand.c
+++ b/drivers/mtd/nand/gpmi-nand/gpmi-nand.c
@@ -318,7 +318,8 @@ static int legacy_set_geometry(struct gpmi_nand_data *this)
 		return -EINVAL;
 	}
 
-	geo->page_size = mtd->writesize + mtd->oobsize;
+	geo->page_size = mtd->writesize + geo->metadata_size +
+		(geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
 	geo->payload_size = mtd->writesize;
 
 	/*
diff --git a/drivers/mtd/nand/jz4780_nand.c b/drivers/mtd/nand/jz4780_nand.c
index 175f67da25af..a39bb70175ee 100644
--- a/drivers/mtd/nand/jz4780_nand.c
+++ b/drivers/mtd/nand/jz4780_nand.c
@@ -368,9 +368,8 @@ static int jz4780_nand_probe(struct platform_device *pdev)
 	nfc->dev = dev;
 	nfc->num_banks = num_banks;
 
-	spin_lock_init(&nfc->controller.lock);
+	nand_hw_control_init(&nfc->controller);
 	INIT_LIST_HEAD(&nfc->chips);
-	init_waitqueue_head(&nfc->controller.wq);
 
 	ret = jz4780_nand_init_chips(nfc, pdev);
 	if (ret) {
diff --git a/drivers/mtd/nand/mtk_ecc.c b/drivers/mtd/nand/mtk_ecc.c
index 25a4fbd4d24a..d54f666417e1 100644
--- a/drivers/mtd/nand/mtk_ecc.c
+++ b/drivers/mtd/nand/mtk_ecc.c
@@ -366,7 +366,8 @@ int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config,
 		   u8 *data, u32 bytes)
 {
 	dma_addr_t addr;
-	u32 *p, len, i;
+	u8 *p;
+	u32 len, i, val;
 	int ret = 0;
 
 	addr = dma_map_single(ecc->dev, data, bytes, DMA_TO_DEVICE);
@@ -392,11 +393,14 @@ int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config,
 
 	/* Program ECC bytes to OOB: per sector oob = FDM + ECC + SPARE */
 	len = (config->strength * ECC_PARITY_BITS + 7) >> 3;
-	p = (u32 *)(data + bytes);
+	p = data + bytes;
 
 	/* write the parity bytes generated by the ECC back to the OOB region */
-	for (i = 0; i < len; i++)
-		p[i] = readl(ecc->regs + ECC_ENCPAR(i));
+	for (i = 0; i < len; i++) {
+		if ((i % 4) == 0)
+			val = readl(ecc->regs + ECC_ENCPAR(i / 4));
+		p[i] = (val >> ((i % 4) * 8)) & 0xff;
+	}
 timeout:
 
 	dma_unmap_single(ecc->dev, addr, bytes, DMA_TO_DEVICE);
diff --git a/drivers/mtd/nand/mtk_nand.c b/drivers/mtd/nand/mtk_nand.c
index ddaa2acb9dd7..5223a2182ee4 100644
--- a/drivers/mtd/nand/mtk_nand.c
+++ b/drivers/mtd/nand/mtk_nand.c
@@ -93,6 +93,9 @@
 #define		NFI_FSM_MASK		(0xf << 16)
 #define NFI_ADDRCNTR		(0x70)
 #define		CNTR_MASK		GENMASK(16, 12)
+#define		ADDRCNTR_SEC_SHIFT	(12)
+#define		ADDRCNTR_SEC(val) \
+		(((val) & CNTR_MASK) >> ADDRCNTR_SEC_SHIFT)
 #define NFI_STRADDR		(0x80)
 #define NFI_BYTELEN		(0x84)
 #define NFI_CSEL		(0x90)
@@ -699,7 +702,7 @@ static int mtk_nfc_do_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	}
 
 	ret = readl_poll_timeout_atomic(nfc->regs + NFI_ADDRCNTR, reg,
-					(reg & CNTR_MASK) >= chip->ecc.steps,
+					ADDRCNTR_SEC(reg) >= chip->ecc.steps,
 					10, MTK_TIMEOUT);
 	if (ret)
 		dev_err(dev, "hwecc write timeout\n");
@@ -902,7 +905,7 @@ static int mtk_nfc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 		dev_warn(nfc->dev, "read ahb/dma done timeout\n");
 
 	rc = readl_poll_timeout_atomic(nfc->regs + NFI_BYTELEN, reg,
-				       (reg & CNTR_MASK) >= sectors, 10,
+				       ADDRCNTR_SEC(reg) >= sectors, 10,
 				       MTK_TIMEOUT);
 	if (rc < 0) {
 		dev_err(nfc->dev, "subpage done timeout\n");
diff --git a/drivers/mtd/nand/mxc_nand.c b/drivers/mtd/nand/mxc_nand.c
index 5173fadc9a4e..d7f724b24fd7 100644
--- a/drivers/mtd/nand/mxc_nand.c
+++ b/drivers/mtd/nand/mxc_nand.c
@@ -152,6 +152,9 @@ struct mxc_nand_devtype_data {
 	void (*select_chip)(struct mtd_info *mtd, int chip);
 	int (*correct_data)(struct mtd_info *mtd, u_char *dat,
 			u_char *read_ecc, u_char *calc_ecc);
+	int (*setup_data_interface)(struct mtd_info *mtd,
+				    const struct nand_data_interface *conf,
+				    bool check_only);
 
 	/*
 	 * On i.MX21 the CONFIG2:INT bit cannot be read if interrupts are masked
@@ -943,7 +946,7 @@ static int mxc_v2_ooblayout_free(struct mtd_info *mtd, int section,
 	struct nand_chip *nand_chip = mtd_to_nand(mtd);
 	int stepsize = nand_chip->ecc.bytes == 9 ? 16 : 26;
 
-	if (section > nand_chip->ecc.steps)
+	if (section >= nand_chip->ecc.steps)
 		return -ERANGE;
 
 	if (!section) {
@@ -1012,6 +1015,82 @@ static void preset_v1(struct mtd_info *mtd)
 	writew(0x4, NFC_V1_V2_WRPROT);
 }
 
+static int mxc_nand_v2_setup_data_interface(struct mtd_info *mtd,
+					const struct nand_data_interface *conf,
+					bool check_only)
+{
+	struct nand_chip *nand_chip = mtd_to_nand(mtd);
+	struct mxc_nand_host *host = nand_get_controller_data(nand_chip);
+	int tRC_min_ns, tRC_ps, ret;
+	unsigned long rate, rate_round;
+	const struct nand_sdr_timings *timings;
+	u16 config1;
+
+	timings = nand_get_sdr_timings(conf);
+	if (IS_ERR(timings))
+		return -ENOTSUPP;
+
+	config1 = readw(NFC_V1_V2_CONFIG1);
+
+	tRC_min_ns = timings->tRC_min / 1000;
+	rate = 1000000000 / tRC_min_ns;
+
+	/*
+	 * For tRC < 30ns we have to use EDO mode. In this case the controller
+	 * does one access per clock cycle. Otherwise the controller does one
+	 * access in two clock cycles, thus we have to double the rate to the
+	 * controller.
+	 */
+	if (tRC_min_ns < 30) {
+		rate_round = clk_round_rate(host->clk, rate);
+		config1 |= NFC_V2_CONFIG1_ONE_CYCLE;
+		tRC_ps = 1000000000 / (rate_round / 1000);
+	} else {
+		rate *= 2;
+		rate_round = clk_round_rate(host->clk, rate);
+		config1 &= ~NFC_V2_CONFIG1_ONE_CYCLE;
+		tRC_ps = 1000000000 / (rate_round / 1000 / 2);
+	}
+
+	/*
+	 * The timing values compared against are from the i.MX25 Automotive
+	 * datasheet, Table 50. NFC Timing Parameters
+	 */
+	if (timings->tCLS_min > tRC_ps - 1000 ||
+	    timings->tCLH_min > tRC_ps - 2000 ||
+	    timings->tCS_min > tRC_ps - 1000 ||
+	    timings->tCH_min > tRC_ps - 2000 ||
+	    timings->tWP_min > tRC_ps - 1500 ||
+	    timings->tALS_min > tRC_ps ||
+	    timings->tALH_min > tRC_ps - 3000 ||
+	    timings->tDS_min > tRC_ps ||
+	    timings->tDH_min > tRC_ps - 5000 ||
+	    timings->tWC_min > 2 * tRC_ps ||
+	    timings->tWH_min > tRC_ps - 2500 ||
+	    timings->tRR_min > 6 * tRC_ps ||
+	    timings->tRP_min > 3 * tRC_ps / 2 ||
+	    timings->tRC_min > 2 * tRC_ps ||
+	    timings->tREH_min > (tRC_ps / 2) - 2500) {
+		dev_dbg(host->dev, "Timing out of bounds\n");
+		return -EINVAL;
+	}
+
+	if (check_only)
+		return 0;
+
+	ret = clk_set_rate(host->clk, rate);
+	if (ret)
+		return ret;
+
+	writew(config1, NFC_V1_V2_CONFIG1);
+
+	dev_dbg(host->dev, "Setting rate to %ldHz, %s mode\n", rate_round,
+		config1 & NFC_V2_CONFIG1_ONE_CYCLE ? "One cycle (EDO)" :
+		"normal");
+
+	return 0;
+}
+
 static void preset_v2(struct mtd_info *mtd)
 {
 	struct nand_chip *nand_chip = mtd_to_nand(mtd);
@@ -1239,6 +1318,57 @@ static void mxc_nand_command(struct mtd_info *mtd, unsigned command,
 	}
 }
 
+static int mxc_nand_onfi_set_features(struct mtd_info *mtd,
+				      struct nand_chip *chip, int addr,
+				      u8 *subfeature_param)
+{
+	struct nand_chip *nand_chip = mtd_to_nand(mtd);
+	struct mxc_nand_host *host = nand_get_controller_data(nand_chip);
+	int i;
+
+	if (!chip->onfi_version ||
+	    !(le16_to_cpu(chip->onfi_params.opt_cmd)
+	      & ONFI_OPT_CMD_SET_GET_FEATURES))
+		return -EINVAL;
+
+	host->buf_start = 0;
+
+	for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
+		chip->write_byte(mtd, subfeature_param[i]);
+
+	memcpy32_toio(host->main_area0, host->data_buf, mtd->writesize);
+	host->devtype_data->send_cmd(host, NAND_CMD_SET_FEATURES, false);
+	mxc_do_addr_cycle(mtd, addr, -1);
+	host->devtype_data->send_page(mtd, NFC_INPUT);
+
+	return 0;
+}
+
+static int mxc_nand_onfi_get_features(struct mtd_info *mtd,
+				      struct nand_chip *chip, int addr,
+				      u8 *subfeature_param)
+{
+	struct nand_chip *nand_chip = mtd_to_nand(mtd);
+	struct mxc_nand_host *host = nand_get_controller_data(nand_chip);
+	int i;
+
+	if (!chip->onfi_version ||
+	    !(le16_to_cpu(chip->onfi_params.opt_cmd)
+	      & ONFI_OPT_CMD_SET_GET_FEATURES))
+		return -EINVAL;
+
+	host->devtype_data->send_cmd(host, NAND_CMD_GET_FEATURES, false);
+	mxc_do_addr_cycle(mtd, addr, -1);
+	host->devtype_data->send_page(mtd, NFC_OUTPUT);
+	memcpy32_fromio(host->data_buf, host->main_area0, 512);
+	host->buf_start = 0;
+
+	for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
+		*subfeature_param++ = chip->read_byte(mtd);
+
+	return 0;
+}
+
 /*
  * The generic flash bbt decriptors overlap with our ecc
  * hardware, so define some i.MX specific ones.
@@ -1327,6 +1457,7 @@ static const struct mxc_nand_devtype_data imx25_nand_devtype_data = {
 	.ooblayout = &mxc_v2_ooblayout_ops,
 	.select_chip = mxc_nand_select_chip_v2,
 	.correct_data = mxc_nand_correct_data_v2_v3,
+	.setup_data_interface = mxc_nand_v2_setup_data_interface,
 	.irqpending_quirk = 0,
 	.needs_ip = 0,
 	.regs_offset = 0x1e00,
@@ -1434,7 +1565,7 @@ static const struct platform_device_id mxcnd_devtype[] = {
 };
 MODULE_DEVICE_TABLE(platform, mxcnd_devtype);
 
-#ifdef CONFIG_OF_MTD
+#ifdef CONFIG_OF
 static const struct of_device_id mxcnd_dt_ids[] = {
 	{
 		.compatible = "fsl,imx21-nand",
@@ -1513,6 +1644,8 @@ static int mxcnd_probe(struct platform_device *pdev)
 	this->read_word = mxc_nand_read_word;
 	this->write_buf = mxc_nand_write_buf;
 	this->read_buf = mxc_nand_read_buf;
+	this->onfi_set_features = mxc_nand_onfi_set_features;
+	this->onfi_get_features = mxc_nand_onfi_get_features;
 
 	host->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(host->clk))
@@ -1533,6 +1666,8 @@ static int mxcnd_probe(struct platform_device *pdev)
 	if (err < 0)
 		return err;
 
+	this->setup_data_interface = host->devtype_data->setup_data_interface;
+
 	if (host->devtype_data->needs_ip) {
 		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 		host->regs_ip = devm_ioremap_resource(&pdev->dev, res);
diff --git a/drivers/mtd/nand/nand_base.c b/drivers/mtd/nand/nand_base.c
index 77533f7f2429..e5718e5ecf92 100644
--- a/drivers/mtd/nand/nand_base.c
+++ b/drivers/mtd/nand/nand_base.c
@@ -745,7 +745,10 @@ static void nand_command_lp(struct mtd_info *mtd, unsigned int command,
 				column >>= 1;
 			chip->cmd_ctrl(mtd, column, ctrl);
 			ctrl &= ~NAND_CTRL_CHANGE;
-			chip->cmd_ctrl(mtd, column >> 8, ctrl);
+
+			/* Only output a single addr cycle for 8bits opcodes. */
+			if (!nand_opcode_8bits(command))
+				chip->cmd_ctrl(mtd, column >> 8, ctrl);
 		}
 		if (page_addr != -1) {
 			chip->cmd_ctrl(mtd, page_addr, ctrl);
@@ -948,6 +951,172 @@ static int nand_wait(struct mtd_info *mtd, struct nand_chip *chip)
 }
 
 /**
+ * nand_reset_data_interface - Reset data interface and timings
+ * @chip: The NAND chip
+ *
+ * Reset the Data interface and timings to ONFI mode 0.
+ *
+ * Returns 0 for success or negative error code otherwise.
+ */
+static int nand_reset_data_interface(struct nand_chip *chip)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	const struct nand_data_interface *conf;
+	int ret;
+
+	if (!chip->setup_data_interface)
+		return 0;
+
+	/*
+	 * The ONFI specification says:
+	 * "
+	 * To transition from NV-DDR or NV-DDR2 to the SDR data
+	 * interface, the host shall use the Reset (FFh) command
+	 * using SDR timing mode 0. A device in any timing mode is
+	 * required to recognize Reset (FFh) command issued in SDR
+	 * timing mode 0.
+	 * "
+	 *
+	 * Configure the data interface in SDR mode and set the
+	 * timings to timing mode 0.
+	 */
+
+	conf = nand_get_default_data_interface();
+	ret = chip->setup_data_interface(mtd, conf, false);
+	if (ret)
+		pr_err("Failed to configure data interface to SDR timing mode 0\n");
+
+	return ret;
+}
+
+/**
+ * nand_setup_data_interface - Setup the best data interface and timings
+ * @chip: The NAND chip
+ *
+ * Find and configure the best data interface and NAND timings supported by
+ * the chip and the driver.
+ * First tries to retrieve supported timing modes from ONFI information,
+ * and if the NAND chip does not support ONFI, relies on the
+ * ->onfi_timing_mode_default specified in the nand_ids table.
+ *
+ * Returns 0 for success or negative error code otherwise.
+ */
+static int nand_setup_data_interface(struct nand_chip *chip)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int ret;
+
+	if (!chip->setup_data_interface || !chip->data_interface)
+		return 0;
+
+	/*
+	 * Ensure the timing mode has been changed on the chip side
+	 * before changing timings on the controller side.
+	 */
+	if (chip->onfi_version) {
+		u8 tmode_param[ONFI_SUBFEATURE_PARAM_LEN] = {
+			chip->onfi_timing_mode_default,
+		};
+
+		ret = chip->onfi_set_features(mtd, chip,
+				ONFI_FEATURE_ADDR_TIMING_MODE,
+				tmode_param);
+		if (ret)
+			goto err;
+	}
+
+	ret = chip->setup_data_interface(mtd, chip->data_interface, false);
+err:
+	return ret;
+}
+
+/**
+ * nand_init_data_interface - find the best data interface and timings
+ * @chip: The NAND chip
+ *
+ * Find the best data interface and NAND timings supported by the chip
+ * and the driver.
+ * First tries to retrieve supported timing modes from ONFI information,
+ * and if the NAND chip does not support ONFI, relies on the
+ * ->onfi_timing_mode_default specified in the nand_ids table. After this
+ * function nand_chip->data_interface is initialized with the best timing mode
+ * available.
+ *
+ * Returns 0 for success or negative error code otherwise.
+ */
+static int nand_init_data_interface(struct nand_chip *chip)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int modes, mode, ret;
+
+	if (!chip->setup_data_interface)
+		return 0;
+
+	/*
+	 * First try to identify the best timings from ONFI parameters and
+	 * if the NAND does not support ONFI, fallback to the default ONFI
+	 * timing mode.
+	 */
+	modes = onfi_get_async_timing_mode(chip);
+	if (modes == ONFI_TIMING_MODE_UNKNOWN) {
+		if (!chip->onfi_timing_mode_default)
+			return 0;
+
+		modes = GENMASK(chip->onfi_timing_mode_default, 0);
+	}
+
+	chip->data_interface = kzalloc(sizeof(*chip->data_interface),
+				       GFP_KERNEL);
+	if (!chip->data_interface)
+		return -ENOMEM;
+
+	for (mode = fls(modes) - 1; mode >= 0; mode--) {
+		ret = onfi_init_data_interface(chip, chip->data_interface,
+					       NAND_SDR_IFACE, mode);
+		if (ret)
+			continue;
+
+		ret = chip->setup_data_interface(mtd, chip->data_interface,
+						 true);
+		if (!ret) {
+			chip->onfi_timing_mode_default = mode;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static void nand_release_data_interface(struct nand_chip *chip)
+{
+	kfree(chip->data_interface);
+}
+
+/**
+ * nand_reset - Reset and initialize a NAND device
+ * @chip: The NAND chip
+ *
+ * Returns 0 for success or negative error code otherwise
+ */
+int nand_reset(struct nand_chip *chip)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int ret;
+
+	ret = nand_reset_data_interface(chip);
+	if (ret)
+		return ret;
+
+	chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+
+	ret = nand_setup_data_interface(chip);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+/**
  * __nand_unlock - [REPLACEABLE] unlocks specified locked blocks
  * @mtd: mtd info
  * @ofs: offset to start unlock from
@@ -1025,7 +1194,7 @@ int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 	 * some operation can also clear the bit 7 of status register
 	 * eg. erase/program a locked block
 	 */
-	chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+	nand_reset(chip);
 
 	/* Check, if it is write protected */
 	if (nand_check_wp(mtd)) {
@@ -1084,7 +1253,7 @@ int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 	 * some operation can also clear the bit 7 of status register
 	 * eg. erase/program a locked block
 	 */
-	chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+	nand_reset(chip);
 
 	/* Check, if it is write protected */
 	if (nand_check_wp(mtd)) {
@@ -2162,7 +2331,7 @@ static int nand_do_read_oob(struct mtd_info *mtd, loff_t from,
 static int nand_read_oob(struct mtd_info *mtd, loff_t from,
 			 struct mtd_oob_ops *ops)
 {
-	int ret = -ENOTSUPP;
+	int ret;
 
 	ops->retlen = 0;
 
@@ -2173,24 +2342,18 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from,
 		return -EINVAL;
 	}
 
-	nand_get_device(mtd, FL_READING);
-
-	switch (ops->mode) {
-	case MTD_OPS_PLACE_OOB:
-	case MTD_OPS_AUTO_OOB:
-	case MTD_OPS_RAW:
-		break;
+	if (ops->mode != MTD_OPS_PLACE_OOB &&
+	    ops->mode != MTD_OPS_AUTO_OOB &&
+	    ops->mode != MTD_OPS_RAW)
+		return -ENOTSUPP;
 
-	default:
-		goto out;
-	}
+	nand_get_device(mtd, FL_READING);
 
 	if (!ops->datbuf)
 		ret = nand_do_read_oob(mtd, from, ops);
 	else
 		ret = nand_do_read_ops(mtd, from, ops);
 
-out:
 	nand_release_device(mtd);
 	return ret;
 }
@@ -2788,7 +2951,7 @@ static int nand_do_write_oob(struct mtd_info *mtd, loff_t to,
 	 * if we don't do this. I have no clue why, but I seem to have 'fixed'
 	 * it in the doc2000 driver in August 1999.  dwmw2.
 	 */
-	chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+	nand_reset(chip);
 
 	/* Check, if it is write protected */
 	if (nand_check_wp(mtd)) {
@@ -3191,8 +3354,7 @@ static void nand_set_defaults(struct nand_chip *chip, int busw)
 
 	if (!chip->controller) {
 		chip->controller = &chip->hwcontrol;
-		spin_lock_init(&chip->controller->lock);
-		init_waitqueue_head(&chip->controller->wq);
+		nand_hw_control_init(chip->controller);
 	}
 
 }
@@ -3829,7 +3991,7 @@ static struct nand_flash_dev *nand_get_flash_type(struct mtd_info *mtd,
 	 * Reset the chip, required by some chips (e.g. Micron MT29FxGxxxxx)
 	 * after power-up.
 	 */
-	chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+	nand_reset(chip);
 
 	/* Send the command for reading device ID */
 	chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
@@ -4113,6 +4275,9 @@ static int nand_dt_init(struct nand_chip *chip)
 	if (ecc_step > 0)
 		chip->ecc.size = ecc_step;
 
+	if (of_property_read_bool(dn, "nand-ecc-maximize"))
+		chip->ecc.options |= NAND_ECC_MAXIMIZE;
+
 	return 0;
 }
 
@@ -4141,6 +4306,15 @@ int nand_scan_ident(struct mtd_info *mtd, int maxchips,
 	if (!mtd->name && mtd->dev.parent)
 		mtd->name = dev_name(mtd->dev.parent);
 
+	if ((!chip->cmdfunc || !chip->select_chip) && !chip->cmd_ctrl) {
+		/*
+		 * Default functions assigned for chip_select() and
+		 * cmdfunc() both expect cmd_ctrl() to be populated,
+		 * so we need to check that that's the case
+		 */
+		pr_err("chip.cmd_ctrl() callback is not provided");
+		return -EINVAL;
+	}
 	/* Set the default functions */
 	nand_set_defaults(chip, chip->options & NAND_BUSWIDTH_16);
 
@@ -4155,13 +4329,17 @@ int nand_scan_ident(struct mtd_info *mtd, int maxchips,
 		return PTR_ERR(type);
 	}
 
+	ret = nand_init_data_interface(chip);
+	if (ret)
+		return ret;
+
 	chip->select_chip(mtd, -1);
 
 	/* Check for a chip array */
 	for (i = 1; i < maxchips; i++) {
 		chip->select_chip(mtd, i);
 		/* See comment in nand_get_flash_type for reset */
-		chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+		nand_reset(chip);
 		/* Send the command for reading device ID */
 		chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
 		/* Read manufacturer and device IDs */
@@ -4221,6 +4399,7 @@ static int nand_set_ecc_soft_ops(struct mtd_info *mtd)
 		ecc->write_page_raw = nand_write_page_raw;
 		ecc->read_oob = nand_read_oob_std;
 		ecc->write_oob = nand_write_oob_std;
+
 		/*
 		* Board driver should supply ecc.size and ecc.strength
 		* values to select how many bits are correctable.
@@ -4243,6 +4422,25 @@ static int nand_set_ecc_soft_ops(struct mtd_info *mtd)
 			}
 
 			mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops);
+
+		}
+
+		/*
+		 * We can only maximize ECC config when the default layout is
+		 * used, otherwise we don't know how many bytes can really be
+		 * used.
+		 */
+		if (mtd->ooblayout == &nand_ooblayout_lp_ops &&
+		    ecc->options & NAND_ECC_MAXIMIZE) {
+			int steps, bytes;
+
+			/* Always prefer 1k blocks over 512bytes ones */
+			ecc->size = 1024;
+			steps = mtd->writesize / ecc->size;
+
+			/* Reserve 2 bytes for the BBM */
+			bytes = (mtd->oobsize - 2) / steps;
+			ecc->strength = bytes * 8 / fls(8 * ecc->size);
 		}
 
 		/* See nand_bch_init() for details. */
@@ -4601,18 +4799,16 @@ int nand_scan(struct mtd_info *mtd, int maxchips)
 EXPORT_SYMBOL(nand_scan);
 
 /**
- * nand_release - [NAND Interface] Free resources held by the NAND device
- * @mtd: MTD device structure
+ * nand_cleanup - [NAND Interface] Free resources held by the NAND device
+ * @chip: NAND chip object
  */
-void nand_release(struct mtd_info *mtd)
+void nand_cleanup(struct nand_chip *chip)
 {
-	struct nand_chip *chip = mtd_to_nand(mtd);
-
 	if (chip->ecc.mode == NAND_ECC_SOFT &&
 	    chip->ecc.algo == NAND_ECC_BCH)
 		nand_bch_free((struct nand_bch_control *)chip->ecc.priv);
 
-	mtd_device_unregister(mtd);
+	nand_release_data_interface(chip);
 
 	/* Free bad block table memory */
 	kfree(chip->bbt);
@@ -4624,6 +4820,18 @@ void nand_release(struct mtd_info *mtd)
 			& NAND_BBT_DYNAMICSTRUCT)
 		kfree(chip->badblock_pattern);
 }
+EXPORT_SYMBOL_GPL(nand_cleanup);
+
+/**
+ * nand_release - [NAND Interface] Unregister the MTD device and free resources
+ *		  held by the NAND device
+ * @mtd: MTD device structure
+ */
+void nand_release(struct mtd_info *mtd)
+{
+	mtd_device_unregister(mtd);
+	nand_cleanup(mtd_to_nand(mtd));
+}
 EXPORT_SYMBOL_GPL(nand_release);
 
 MODULE_LICENSE("GPL");
diff --git a/drivers/mtd/nand/nand_bbt.c b/drivers/mtd/nand/nand_bbt.c
index 2fbb523df066..7695efea65f2 100644
--- a/drivers/mtd/nand/nand_bbt.c
+++ b/drivers/mtd/nand/nand_bbt.c
@@ -605,6 +605,100 @@ static void search_read_bbts(struct mtd_info *mtd, uint8_t *buf,
 }
 
 /**
+ * get_bbt_block - Get the first valid eraseblock suitable to store a BBT
+ * @this: the NAND device
+ * @td: the BBT description
+ * @md: the mirror BBT descriptor
+ * @chip: the CHIP selector
+ *
+ * This functions returns a positive block number pointing a valid eraseblock
+ * suitable to store a BBT (i.e. in the range reserved for BBT), or -ENOSPC if
+ * all blocks are already used of marked bad. If td->pages[chip] was already
+ * pointing to a valid block we re-use it, otherwise we search for the next
+ * valid one.
+ */
+static int get_bbt_block(struct nand_chip *this, struct nand_bbt_descr *td,
+			 struct nand_bbt_descr *md, int chip)
+{
+	int startblock, dir, page, numblocks, i;
+
+	/*
+	 * There was already a version of the table, reuse the page. This
+	 * applies for absolute placement too, as we have the page number in
+	 * td->pages.
+	 */
+	if (td->pages[chip] != -1)
+		return td->pages[chip] >>
+				(this->bbt_erase_shift - this->page_shift);
+
+	numblocks = (int)(this->chipsize >> this->bbt_erase_shift);
+	if (!(td->options & NAND_BBT_PERCHIP))
+		numblocks *= this->numchips;
+
+	/*
+	 * Automatic placement of the bad block table. Search direction
+	 * top -> down?
+	 */
+	if (td->options & NAND_BBT_LASTBLOCK) {
+		startblock = numblocks * (chip + 1) - 1;
+		dir = -1;
+	} else {
+		startblock = chip * numblocks;
+		dir = 1;
+	}
+
+	for (i = 0; i < td->maxblocks; i++) {
+		int block = startblock + dir * i;
+
+		/* Check, if the block is bad */
+		switch (bbt_get_entry(this, block)) {
+		case BBT_BLOCK_WORN:
+		case BBT_BLOCK_FACTORY_BAD:
+			continue;
+		}
+
+		page = block << (this->bbt_erase_shift - this->page_shift);
+
+		/* Check, if the block is used by the mirror table */
+		if (!md || md->pages[chip] != page)
+			return block;
+	}
+
+	return -ENOSPC;
+}
+
+/**
+ * mark_bbt_block_bad - Mark one of the block reserved for BBT bad
+ * @this: the NAND device
+ * @td: the BBT description
+ * @chip: the CHIP selector
+ * @block: the BBT block to mark
+ *
+ * Blocks reserved for BBT can become bad. This functions is an helper to mark
+ * such blocks as bad. It takes care of updating the in-memory BBT, marking the
+ * block as bad using a bad block marker and invalidating the associated
+ * td->pages[] entry.
+ */
+static void mark_bbt_block_bad(struct nand_chip *this,
+			       struct nand_bbt_descr *td,
+			       int chip, int block)
+{
+	struct mtd_info *mtd = nand_to_mtd(this);
+	loff_t to;
+	int res;
+
+	bbt_mark_entry(this, block, BBT_BLOCK_WORN);
+
+	to = (loff_t)block << this->bbt_erase_shift;
+	res = this->block_markbad(mtd, to);
+	if (res)
+		pr_warn("nand_bbt: error %d while marking block %d bad\n",
+			res, block);
+
+	td->pages[chip] = -1;
+}
+
+/**
  * write_bbt - [GENERIC] (Re)write the bad block table
  * @mtd: MTD device structure
  * @buf: temporary buffer
@@ -621,7 +715,7 @@ static int write_bbt(struct mtd_info *mtd, uint8_t *buf,
 	struct nand_chip *this = mtd_to_nand(mtd);
 	struct erase_info einfo;
 	int i, res, chip = 0;
-	int bits, startblock, dir, page, offs, numblocks, sft, sftmsk;
+	int bits, page, offs, numblocks, sft, sftmsk;
 	int nrchips, pageoffs, ooboffs;
 	uint8_t msk[4];
 	uint8_t rcode = td->reserved_block_code;
@@ -652,46 +746,21 @@ static int write_bbt(struct mtd_info *mtd, uint8_t *buf,
 	}
 
 	/* Loop through the chips */
-	for (; chip < nrchips; chip++) {
-		/*
-		 * There was already a version of the table, reuse the page
-		 * This applies for absolute placement too, as we have the
-		 * page nr. in td->pages.
-		 */
-		if (td->pages[chip] != -1) {
-			page = td->pages[chip];
-			goto write;
+	while (chip < nrchips) {
+		int block;
+
+		block = get_bbt_block(this, td, md, chip);
+		if (block < 0) {
+			pr_err("No space left to write bad block table\n");
+			res = block;
+			goto outerr;
 		}
 
 		/*
-		 * Automatic placement of the bad block table. Search direction
-		 * top -> down?
+		 * get_bbt_block() returns a block number, shift the value to
+		 * get a page number.
 		 */
-		if (td->options & NAND_BBT_LASTBLOCK) {
-			startblock = numblocks * (chip + 1) - 1;
-			dir = -1;
-		} else {
-			startblock = chip * numblocks;
-			dir = 1;
-		}
-
-		for (i = 0; i < td->maxblocks; i++) {
-			int block = startblock + dir * i;
-			/* Check, if the block is bad */
-			switch (bbt_get_entry(this, block)) {
-			case BBT_BLOCK_WORN:
-			case BBT_BLOCK_FACTORY_BAD:
-				continue;
-			}
-			page = block <<
-				(this->bbt_erase_shift - this->page_shift);
-			/* Check, if the block is used by the mirror table */
-			if (!md || md->pages[chip] != page)
-				goto write;
-		}
-		pr_err("No space left to write bad block table\n");
-		return -ENOSPC;
-	write:
+		page = block << (this->bbt_erase_shift - this->page_shift);
 
 		/* Set up shift count and masks for the flash table */
 		bits = td->options & NAND_BBT_NRBITS_MSK;
@@ -787,20 +856,28 @@ static int write_bbt(struct mtd_info *mtd, uint8_t *buf,
 		einfo.addr = to;
 		einfo.len = 1 << this->bbt_erase_shift;
 		res = nand_erase_nand(mtd, &einfo, 1);
-		if (res < 0)
-			goto outerr;
+		if (res < 0) {
+			pr_warn("nand_bbt: error while erasing BBT block %d\n",
+				res);
+			mark_bbt_block_bad(this, td, chip, block);
+			continue;
+		}
 
 		res = scan_write_bbt(mtd, to, len, buf,
 				td->options & NAND_BBT_NO_OOB ? NULL :
 				&buf[len]);
-		if (res < 0)
-			goto outerr;
+		if (res < 0) {
+			pr_warn("nand_bbt: error while writing BBT block %d\n",
+				res);
+			mark_bbt_block_bad(this, td, chip, block);
+			continue;
+		}
 
 		pr_info("Bad block table written to 0x%012llx, version 0x%02X\n",
 			 (unsigned long long)to, td->version[chip]);
 
 		/* Mark it as used */
-		td->pages[chip] = page;
+		td->pages[chip++] = page;
 	}
 	return 0;
 
diff --git a/drivers/mtd/nand/nand_timings.c b/drivers/mtd/nand/nand_timings.c
index e81470a8ac67..13a587407be3 100644
--- a/drivers/mtd/nand/nand_timings.c
+++ b/drivers/mtd/nand/nand_timings.c
@@ -13,228 +13,246 @@
 #include <linux/export.h>
 #include <linux/mtd/nand.h>
 
-static const struct nand_sdr_timings onfi_sdr_timings[] = {
+static const struct nand_data_interface onfi_sdr_timings[] = {
 	/* Mode 0 */
 	{
-		.tADL_min = 200000,
-		.tALH_min = 20000,
-		.tALS_min = 50000,
-		.tAR_min = 25000,
-		.tCEA_max = 100000,
-		.tCEH_min = 20000,
-		.tCH_min = 20000,
-		.tCHZ_max = 100000,
-		.tCLH_min = 20000,
-		.tCLR_min = 20000,
-		.tCLS_min = 50000,
-		.tCOH_min = 0,
-		.tCS_min = 70000,
-		.tDH_min = 20000,
-		.tDS_min = 40000,
-		.tFEAT_max = 1000000,
-		.tIR_min = 10000,
-		.tITC_max = 1000000,
-		.tRC_min = 100000,
-		.tREA_max = 40000,
-		.tREH_min = 30000,
-		.tRHOH_min = 0,
-		.tRHW_min = 200000,
-		.tRHZ_max = 200000,
-		.tRLOH_min = 0,
-		.tRP_min = 50000,
-		.tRST_max = 250000000000ULL,
-		.tWB_max = 200000,
-		.tRR_min = 40000,
-		.tWC_min = 100000,
-		.tWH_min = 30000,
-		.tWHR_min = 120000,
-		.tWP_min = 50000,
-		.tWW_min = 100000,
+		.type = NAND_SDR_IFACE,
+		.timings.sdr = {
+			.tADL_min = 400000,
+			.tALH_min = 20000,
+			.tALS_min = 50000,
+			.tAR_min = 25000,
+			.tCEA_max = 100000,
+			.tCEH_min = 20000,
+			.tCH_min = 20000,
+			.tCHZ_max = 100000,
+			.tCLH_min = 20000,
+			.tCLR_min = 20000,
+			.tCLS_min = 50000,
+			.tCOH_min = 0,
+			.tCS_min = 70000,
+			.tDH_min = 20000,
+			.tDS_min = 40000,
+			.tFEAT_max = 1000000,
+			.tIR_min = 10000,
+			.tITC_max = 1000000,
+			.tRC_min = 100000,
+			.tREA_max = 40000,
+			.tREH_min = 30000,
+			.tRHOH_min = 0,
+			.tRHW_min = 200000,
+			.tRHZ_max = 200000,
+			.tRLOH_min = 0,
+			.tRP_min = 50000,
+			.tRR_min = 40000,
+			.tRST_max = 250000000000ULL,
+			.tWB_max = 200000,
+			.tWC_min = 100000,
+			.tWH_min = 30000,
+			.tWHR_min = 120000,
+			.tWP_min = 50000,
+			.tWW_min = 100000,
+		},
 	},
 	/* Mode 1 */
 	{
-		.tADL_min = 100000,
-		.tALH_min = 10000,
-		.tALS_min = 25000,
-		.tAR_min = 10000,
-		.tCEA_max = 45000,
-		.tCEH_min = 20000,
-		.tCH_min = 10000,
-		.tCHZ_max = 50000,
-		.tCLH_min = 10000,
-		.tCLR_min = 10000,
-		.tCLS_min = 25000,
-		.tCOH_min = 15000,
-		.tCS_min = 35000,
-		.tDH_min = 10000,
-		.tDS_min = 20000,
-		.tFEAT_max = 1000000,
-		.tIR_min = 0,
-		.tITC_max = 1000000,
-		.tRC_min = 50000,
-		.tREA_max = 30000,
-		.tREH_min = 15000,
-		.tRHOH_min = 15000,
-		.tRHW_min = 100000,
-		.tRHZ_max = 100000,
-		.tRLOH_min = 0,
-		.tRP_min = 25000,
-		.tRR_min = 20000,
-		.tRST_max = 500000000,
-		.tWB_max = 100000,
-		.tWC_min = 45000,
-		.tWH_min = 15000,
-		.tWHR_min = 80000,
-		.tWP_min = 25000,
-		.tWW_min = 100000,
+		.type = NAND_SDR_IFACE,
+		.timings.sdr = {
+			.tADL_min = 400000,
+			.tALH_min = 10000,
+			.tALS_min = 25000,
+			.tAR_min = 10000,
+			.tCEA_max = 45000,
+			.tCEH_min = 20000,
+			.tCH_min = 10000,
+			.tCHZ_max = 50000,
+			.tCLH_min = 10000,
+			.tCLR_min = 10000,
+			.tCLS_min = 25000,
+			.tCOH_min = 15000,
+			.tCS_min = 35000,
+			.tDH_min = 10000,
+			.tDS_min = 20000,
+			.tFEAT_max = 1000000,
+			.tIR_min = 0,
+			.tITC_max = 1000000,
+			.tRC_min = 50000,
+			.tREA_max = 30000,
+			.tREH_min = 15000,
+			.tRHOH_min = 15000,
+			.tRHW_min = 100000,
+			.tRHZ_max = 100000,
+			.tRLOH_min = 0,
+			.tRP_min = 25000,
+			.tRR_min = 20000,
+			.tRST_max = 500000000,
+			.tWB_max = 100000,
+			.tWC_min = 45000,
+			.tWH_min = 15000,
+			.tWHR_min = 80000,
+			.tWP_min = 25000,
+			.tWW_min = 100000,
+		},
 	},
 	/* Mode 2 */
 	{
-		.tADL_min = 100000,
-		.tALH_min = 10000,
-		.tALS_min = 15000,
-		.tAR_min = 10000,
-		.tCEA_max = 30000,
-		.tCEH_min = 20000,
-		.tCH_min = 10000,
-		.tCHZ_max = 50000,
-		.tCLH_min = 10000,
-		.tCLR_min = 10000,
-		.tCLS_min = 15000,
-		.tCOH_min = 15000,
-		.tCS_min = 25000,
-		.tDH_min = 5000,
-		.tDS_min = 15000,
-		.tFEAT_max = 1000000,
-		.tIR_min = 0,
-		.tITC_max = 1000000,
-		.tRC_min = 35000,
-		.tREA_max = 25000,
-		.tREH_min = 15000,
-		.tRHOH_min = 15000,
-		.tRHW_min = 100000,
-		.tRHZ_max = 100000,
-		.tRLOH_min = 0,
-		.tRR_min = 20000,
-		.tRST_max = 500000000,
-		.tWB_max = 100000,
-		.tRP_min = 17000,
-		.tWC_min = 35000,
-		.tWH_min = 15000,
-		.tWHR_min = 80000,
-		.tWP_min = 17000,
-		.tWW_min = 100000,
+		.type = NAND_SDR_IFACE,
+		.timings.sdr = {
+			.tADL_min = 400000,
+			.tALH_min = 10000,
+			.tALS_min = 15000,
+			.tAR_min = 10000,
+			.tCEA_max = 30000,
+			.tCEH_min = 20000,
+			.tCH_min = 10000,
+			.tCHZ_max = 50000,
+			.tCLH_min = 10000,
+			.tCLR_min = 10000,
+			.tCLS_min = 15000,
+			.tCOH_min = 15000,
+			.tCS_min = 25000,
+			.tDH_min = 5000,
+			.tDS_min = 15000,
+			.tFEAT_max = 1000000,
+			.tIR_min = 0,
+			.tITC_max = 1000000,
+			.tRC_min = 35000,
+			.tREA_max = 25000,
+			.tREH_min = 15000,
+			.tRHOH_min = 15000,
+			.tRHW_min = 100000,
+			.tRHZ_max = 100000,
+			.tRLOH_min = 0,
+			.tRR_min = 20000,
+			.tRST_max = 500000000,
+			.tWB_max = 100000,
+			.tRP_min = 17000,
+			.tWC_min = 35000,
+			.tWH_min = 15000,
+			.tWHR_min = 80000,
+			.tWP_min = 17000,
+			.tWW_min = 100000,
+		},
 	},
 	/* Mode 3 */
 	{
-		.tADL_min = 100000,
-		.tALH_min = 5000,
-		.tALS_min = 10000,
-		.tAR_min = 10000,
-		.tCEA_max = 25000,
-		.tCEH_min = 20000,
-		.tCH_min = 5000,
-		.tCHZ_max = 50000,
-		.tCLH_min = 5000,
-		.tCLR_min = 10000,
-		.tCLS_min = 10000,
-		.tCOH_min = 15000,
-		.tCS_min = 25000,
-		.tDH_min = 5000,
-		.tDS_min = 10000,
-		.tFEAT_max = 1000000,
-		.tIR_min = 0,
-		.tITC_max = 1000000,
-		.tRC_min = 30000,
-		.tREA_max = 20000,
-		.tREH_min = 10000,
-		.tRHOH_min = 15000,
-		.tRHW_min = 100000,
-		.tRHZ_max = 100000,
-		.tRLOH_min = 0,
-		.tRP_min = 15000,
-		.tRR_min = 20000,
-		.tRST_max = 500000000,
-		.tWB_max = 100000,
-		.tWC_min = 30000,
-		.tWH_min = 10000,
-		.tWHR_min = 80000,
-		.tWP_min = 15000,
-		.tWW_min = 100000,
+		.type = NAND_SDR_IFACE,
+		.timings.sdr = {
+			.tADL_min = 400000,
+			.tALH_min = 5000,
+			.tALS_min = 10000,
+			.tAR_min = 10000,
+			.tCEA_max = 25000,
+			.tCEH_min = 20000,
+			.tCH_min = 5000,
+			.tCHZ_max = 50000,
+			.tCLH_min = 5000,
+			.tCLR_min = 10000,
+			.tCLS_min = 10000,
+			.tCOH_min = 15000,
+			.tCS_min = 25000,
+			.tDH_min = 5000,
+			.tDS_min = 10000,
+			.tFEAT_max = 1000000,
+			.tIR_min = 0,
+			.tITC_max = 1000000,
+			.tRC_min = 30000,
+			.tREA_max = 20000,
+			.tREH_min = 10000,
+			.tRHOH_min = 15000,
+			.tRHW_min = 100000,
+			.tRHZ_max = 100000,
+			.tRLOH_min = 0,
+			.tRP_min = 15000,
+			.tRR_min = 20000,
+			.tRST_max = 500000000,
+			.tWB_max = 100000,
+			.tWC_min = 30000,
+			.tWH_min = 10000,
+			.tWHR_min = 80000,
+			.tWP_min = 15000,
+			.tWW_min = 100000,
+		},
 	},
 	/* Mode 4 */
 	{
-		.tADL_min = 70000,
-		.tALH_min = 5000,
-		.tALS_min = 10000,
-		.tAR_min = 10000,
-		.tCEA_max = 25000,
-		.tCEH_min = 20000,
-		.tCH_min = 5000,
-		.tCHZ_max = 30000,
-		.tCLH_min = 5000,
-		.tCLR_min = 10000,
-		.tCLS_min = 10000,
-		.tCOH_min = 15000,
-		.tCS_min = 20000,
-		.tDH_min = 5000,
-		.tDS_min = 10000,
-		.tFEAT_max = 1000000,
-		.tIR_min = 0,
-		.tITC_max = 1000000,
-		.tRC_min = 25000,
-		.tREA_max = 20000,
-		.tREH_min = 10000,
-		.tRHOH_min = 15000,
-		.tRHW_min = 100000,
-		.tRHZ_max = 100000,
-		.tRLOH_min = 5000,
-		.tRP_min = 12000,
-		.tRR_min = 20000,
-		.tRST_max = 500000000,
-		.tWB_max = 100000,
-		.tWC_min = 25000,
-		.tWH_min = 10000,
-		.tWHR_min = 80000,
-		.tWP_min = 12000,
-		.tWW_min = 100000,
+		.type = NAND_SDR_IFACE,
+		.timings.sdr = {
+			.tADL_min = 400000,
+			.tALH_min = 5000,
+			.tALS_min = 10000,
+			.tAR_min = 10000,
+			.tCEA_max = 25000,
+			.tCEH_min = 20000,
+			.tCH_min = 5000,
+			.tCHZ_max = 30000,
+			.tCLH_min = 5000,
+			.tCLR_min = 10000,
+			.tCLS_min = 10000,
+			.tCOH_min = 15000,
+			.tCS_min = 20000,
+			.tDH_min = 5000,
+			.tDS_min = 10000,
+			.tFEAT_max = 1000000,
+			.tIR_min = 0,
+			.tITC_max = 1000000,
+			.tRC_min = 25000,
+			.tREA_max = 20000,
+			.tREH_min = 10000,
+			.tRHOH_min = 15000,
+			.tRHW_min = 100000,
+			.tRHZ_max = 100000,
+			.tRLOH_min = 5000,
+			.tRP_min = 12000,
+			.tRR_min = 20000,
+			.tRST_max = 500000000,
+			.tWB_max = 100000,
+			.tWC_min = 25000,
+			.tWH_min = 10000,
+			.tWHR_min = 80000,
+			.tWP_min = 12000,
+			.tWW_min = 100000,
+		},
 	},
 	/* Mode 5 */
 	{
-		.tADL_min = 70000,
-		.tALH_min = 5000,
-		.tALS_min = 10000,
-		.tAR_min = 10000,
-		.tCEA_max = 25000,
-		.tCEH_min = 20000,
-		.tCH_min = 5000,
-		.tCHZ_max = 30000,
-		.tCLH_min = 5000,
-		.tCLR_min = 10000,
-		.tCLS_min = 10000,
-		.tCOH_min = 15000,
-		.tCS_min = 15000,
-		.tDH_min = 5000,
-		.tDS_min = 7000,
-		.tFEAT_max = 1000000,
-		.tIR_min = 0,
-		.tITC_max = 1000000,
-		.tRC_min = 20000,
-		.tREA_max = 16000,
-		.tREH_min = 7000,
-		.tRHOH_min = 15000,
-		.tRHW_min = 100000,
-		.tRHZ_max = 100000,
-		.tRLOH_min = 5000,
-		.tRP_min = 10000,
-		.tRR_min = 20000,
-		.tRST_max = 500000000,
-		.tWB_max = 100000,
-		.tWC_min = 20000,
-		.tWH_min = 7000,
-		.tWHR_min = 80000,
-		.tWP_min = 10000,
-		.tWW_min = 100000,
+		.type = NAND_SDR_IFACE,
+		.timings.sdr = {
+			.tADL_min = 400000,
+			.tALH_min = 5000,
+			.tALS_min = 10000,
+			.tAR_min = 10000,
+			.tCEA_max = 25000,
+			.tCEH_min = 20000,
+			.tCH_min = 5000,
+			.tCHZ_max = 30000,
+			.tCLH_min = 5000,
+			.tCLR_min = 10000,
+			.tCLS_min = 10000,
+			.tCOH_min = 15000,
+			.tCS_min = 15000,
+			.tDH_min = 5000,
+			.tDS_min = 7000,
+			.tFEAT_max = 1000000,
+			.tIR_min = 0,
+			.tITC_max = 1000000,
+			.tRC_min = 20000,
+			.tREA_max = 16000,
+			.tREH_min = 7000,
+			.tRHOH_min = 15000,
+			.tRHW_min = 100000,
+			.tRHZ_max = 100000,
+			.tRLOH_min = 5000,
+			.tRP_min = 10000,
+			.tRR_min = 20000,
+			.tRST_max = 500000000,
+			.tWB_max = 100000,
+			.tWC_min = 20000,
+			.tWH_min = 7000,
+			.tWHR_min = 80000,
+			.tWP_min = 10000,
+			.tWW_min = 100000,
+		},
 	},
 };
 
@@ -248,6 +266,46 @@ const struct nand_sdr_timings *onfi_async_timing_mode_to_sdr_timings(int mode)
 	if (mode < 0 || mode >= ARRAY_SIZE(onfi_sdr_timings))
 		return ERR_PTR(-EINVAL);
 
-	return &onfi_sdr_timings[mode];
+	return &onfi_sdr_timings[mode].timings.sdr;
 }
 EXPORT_SYMBOL(onfi_async_timing_mode_to_sdr_timings);
+
+/**
+ * onfi_init_data_interface - [NAND Interface] Initialize a data interface from
+ * given ONFI mode
+ * @iface: The data interface to be initialized
+ * @mode: The ONFI timing mode
+ */
+int onfi_init_data_interface(struct nand_chip *chip,
+			     struct nand_data_interface *iface,
+			     enum nand_data_interface_type type,
+			     int timing_mode)
+{
+	if (type != NAND_SDR_IFACE)
+		return -EINVAL;
+
+	if (timing_mode < 0 || timing_mode >= ARRAY_SIZE(onfi_sdr_timings))
+		return -EINVAL;
+
+	*iface = onfi_sdr_timings[timing_mode];
+
+	/*
+	 * TODO: initialize timings that cannot be deduced from timing mode:
+	 * tR, tPROG, tCCS, ...
+	 * These information are part of the ONFI parameter page.
+	 */
+
+	return 0;
+}
+EXPORT_SYMBOL(onfi_init_data_interface);
+
+/**
+ * nand_get_default_data_interface - [NAND Interface] Retrieve NAND
+ * data interface for mode 0. This is used as default timing after
+ * reset.
+ */
+const struct nand_data_interface *nand_get_default_data_interface(void)
+{
+	return &onfi_sdr_timings[0];
+}
+EXPORT_SYMBOL(nand_get_default_data_interface);
diff --git a/drivers/mtd/nand/ndfc.c b/drivers/mtd/nand/ndfc.c
index 218c789ca7ab..28e6118362f7 100644
--- a/drivers/mtd/nand/ndfc.c
+++ b/drivers/mtd/nand/ndfc.c
@@ -218,8 +218,7 @@ static int ndfc_probe(struct platform_device *ofdev)
 	ndfc = &ndfc_ctrl[cs];
 	ndfc->chip_select = cs;
 
-	spin_lock_init(&ndfc->ndfc_control.lock);
-	init_waitqueue_head(&ndfc->ndfc_control.wq);
+	nand_hw_control_init(&ndfc->ndfc_control);
 	ndfc->ofdev = ofdev;
 	dev_set_drvdata(&ofdev->dev, ndfc);
 
diff --git a/drivers/mtd/nand/omap2.c b/drivers/mtd/nand/omap2.c
index a59361c36f40..5513bfd9cdc9 100644
--- a/drivers/mtd/nand/omap2.c
+++ b/drivers/mtd/nand/omap2.c
@@ -2169,7 +2169,7 @@ scan_tail:
 	return 0;
 
 return_error:
-	if (info->dma)
+	if (!IS_ERR_OR_NULL(info->dma))
 		dma_release_channel(info->dma);
 	if (nand_chip->ecc.priv) {
 		nand_bch_free(nand_chip->ecc.priv);
diff --git a/drivers/mtd/nand/pxa3xx_nand.c b/drivers/mtd/nand/pxa3xx_nand.c
index 436dd6dc11f4..b121bf4ed73a 100644
--- a/drivers/mtd/nand/pxa3xx_nand.c
+++ b/drivers/mtd/nand/pxa3xx_nand.c
@@ -1810,8 +1810,7 @@ static int alloc_nand_resource(struct platform_device *pdev)
 		chip->cmdfunc		= nand_cmdfunc;
 	}
 
-	spin_lock_init(&chip->controller->lock);
-	init_waitqueue_head(&chip->controller->wq);
+	nand_hw_control_init(chip->controller);
 	info->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(info->clk)) {
 		dev_err(&pdev->dev, "failed to get nand clock\n");
diff --git a/drivers/mtd/nand/qcom_nandc.c b/drivers/mtd/nand/qcom_nandc.c
index de7d28e62d4e..57d483ac5765 100644
--- a/drivers/mtd/nand/qcom_nandc.c
+++ b/drivers/mtd/nand/qcom_nandc.c
@@ -1957,8 +1957,7 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
 	INIT_LIST_HEAD(&nandc->desc_list);
 	INIT_LIST_HEAD(&nandc->host_list);
 
-	spin_lock_init(&nandc->controller.lock);
-	init_waitqueue_head(&nandc->controller.wq);
+	nand_hw_control_init(&nandc->controller);
 
 	return 0;
 }
diff --git a/drivers/mtd/nand/s3c2410.c b/drivers/mtd/nand/s3c2410.c
index d9309cf0ce2e..d459c19d78de 100644
--- a/drivers/mtd/nand/s3c2410.c
+++ b/drivers/mtd/nand/s3c2410.c
@@ -180,7 +180,7 @@ struct s3c2410_nand_info {
 
 	enum s3c_cpu_type		cpu_type;
 
-#ifdef CONFIG_CPU_FREQ
+#ifdef CONFIG_ARM_S3C24XX_CPUFREQ
 	struct notifier_block	freq_transition;
 #endif
 };
@@ -701,7 +701,7 @@ static void s3c2440_nand_write_buf(struct mtd_info *mtd, const u_char *buf,
 
 /* cpufreq driver support */
 
-#ifdef CONFIG_CPU_FREQ
+#ifdef CONFIG_ARM_S3C24XX_CPUFREQ
 
 static int s3c2410_nand_cpufreq_transition(struct notifier_block *nb,
 					  unsigned long val, void *data)
@@ -977,8 +977,7 @@ static int s3c24xx_nand_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, info);
 
-	spin_lock_init(&info->controller.lock);
-	init_waitqueue_head(&info->controller.wq);
+	nand_hw_control_init(&info->controller);
 
 	/* get the clock source and enable it */
 
diff --git a/drivers/mtd/nand/sh_flctl.c b/drivers/mtd/nand/sh_flctl.c
index 6fa3bcd59769..442ce619b3b6 100644
--- a/drivers/mtd/nand/sh_flctl.c
+++ b/drivers/mtd/nand/sh_flctl.c
@@ -397,7 +397,7 @@ static int flctl_dma_fifo0_transfer(struct sh_flctl *flctl, unsigned long *buf,
 	struct dma_chan *chan;
 	enum dma_transfer_direction tr_dir;
 	dma_addr_t dma_addr;
-	dma_cookie_t cookie = -EINVAL;
+	dma_cookie_t cookie;
 	uint32_t reg;
 	int ret;
 
@@ -423,6 +423,12 @@ static int flctl_dma_fifo0_transfer(struct sh_flctl *flctl, unsigned long *buf,
 		desc->callback = flctl_dma_complete;
 		desc->callback_param = flctl;
 		cookie = dmaengine_submit(desc);
+		if (dma_submit_error(cookie)) {
+			ret = dma_submit_error(cookie);
+			dev_warn(&flctl->pdev->dev,
+				 "DMA submit failed, falling back to PIO\n");
+			goto out;
+		}
 
 		dma_async_issue_pending(chan);
 	} else {
diff --git a/drivers/mtd/nand/sunxi_nand.c b/drivers/mtd/nand/sunxi_nand.c
index e414b31b71c1..8b8470c4e6d0 100644
--- a/drivers/mtd/nand/sunxi_nand.c
+++ b/drivers/mtd/nand/sunxi_nand.c
@@ -1572,14 +1572,22 @@ static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
 #define sunxi_nand_lookup_timing(l, p, c) \
 			_sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
 
-static int sunxi_nand_chip_set_timings(struct sunxi_nand_chip *chip,
-				       const struct nand_sdr_timings *timings)
+static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd,
+					const struct nand_data_interface *conf,
+					bool check_only)
 {
+	struct nand_chip *nand = mtd_to_nand(mtd);
+	struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
 	struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
+	const struct nand_sdr_timings *timings;
 	u32 min_clk_period = 0;
 	s32 tWB, tADL, tWHR, tRHW, tCAD;
 	long real_clk_rate;
 
+	timings = nand_get_sdr_timings(conf);
+	if (IS_ERR(timings))
+		return -ENOTSUPP;
+
 	/* T1 <=> tCLS */
 	if (timings->tCLS_min > min_clk_period)
 		min_clk_period = timings->tCLS_min;
@@ -1679,6 +1687,9 @@ static int sunxi_nand_chip_set_timings(struct sunxi_nand_chip *chip,
 		return tRHW;
 	}
 
+	if (check_only)
+		return 0;
+
 	/*
 	 * TODO: according to ONFI specs this value only applies for DDR NAND,
 	 * but Allwinner seems to set this to 0x7. Mimic them for now.
@@ -1712,44 +1723,6 @@ static int sunxi_nand_chip_set_timings(struct sunxi_nand_chip *chip,
 	return 0;
 }
 
-static int sunxi_nand_chip_init_timings(struct sunxi_nand_chip *chip,
-					struct device_node *np)
-{
-	struct mtd_info *mtd = nand_to_mtd(&chip->nand);
-	const struct nand_sdr_timings *timings;
-	int ret;
-	int mode;
-
-	mode = onfi_get_async_timing_mode(&chip->nand);
-	if (mode == ONFI_TIMING_MODE_UNKNOWN) {
-		mode = chip->nand.onfi_timing_mode_default;
-	} else {
-		uint8_t feature[ONFI_SUBFEATURE_PARAM_LEN] = {};
-		int i;
-
-		mode = fls(mode) - 1;
-		if (mode < 0)
-			mode = 0;
-
-		feature[0] = mode;
-		for (i = 0; i < chip->nsels; i++) {
-			chip->nand.select_chip(mtd, i);
-			ret = chip->nand.onfi_set_features(mtd,	&chip->nand,
-						ONFI_FEATURE_ADDR_TIMING_MODE,
-						feature);
-			chip->nand.select_chip(mtd, -1);
-			if (ret)
-				return ret;
-		}
-	}
-
-	timings = onfi_async_timing_mode_to_sdr_timings(mode);
-	if (IS_ERR(timings))
-		return PTR_ERR(timings);
-
-	return sunxi_nand_chip_set_timings(chip, timings);
-}
-
 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
 				    struct mtd_oob_region *oobregion)
 {
@@ -1814,6 +1787,35 @@ static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
 	int ret;
 	int i;
 
+	if (ecc->options & NAND_ECC_MAXIMIZE) {
+		int bytes;
+
+		ecc->size = 1024;
+		nsectors = mtd->writesize / ecc->size;
+
+		/* Reserve 2 bytes for the BBM */
+		bytes = (mtd->oobsize - 2) / nsectors;
+
+		/* 4 non-ECC bytes are added before each ECC bytes section */
+		bytes -= 4;
+
+		/* and bytes has to be even. */
+		if (bytes % 2)
+			bytes--;
+
+		ecc->strength = bytes * 8 / fls(8 * ecc->size);
+
+		for (i = 0; i < ARRAY_SIZE(strengths); i++) {
+			if (strengths[i] > ecc->strength)
+				break;
+		}
+
+		if (!i)
+			ecc->strength = 0;
+		else
+			ecc->strength = strengths[i - 1];
+	}
+
 	if (ecc->size != 512 && ecc->size != 1024)
 		return -EINVAL;
 
@@ -1975,7 +1977,6 @@ static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
 				struct device_node *np)
 {
-	const struct nand_sdr_timings *timings;
 	struct sunxi_nand_chip *chip;
 	struct mtd_info *mtd;
 	struct nand_chip *nand;
@@ -2065,25 +2066,11 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
 	nand->read_buf = sunxi_nfc_read_buf;
 	nand->write_buf = sunxi_nfc_write_buf;
 	nand->read_byte = sunxi_nfc_read_byte;
+	nand->setup_data_interface = sunxi_nfc_setup_data_interface;
 
 	mtd = nand_to_mtd(nand);
 	mtd->dev.parent = dev;
 
-	timings = onfi_async_timing_mode_to_sdr_timings(0);
-	if (IS_ERR(timings)) {
-		ret = PTR_ERR(timings);
-		dev_err(dev,
-			"could not retrieve timings for ONFI mode 0: %d\n",
-			ret);
-		return ret;
-	}
-
-	ret = sunxi_nand_chip_set_timings(chip, timings);
-	if (ret) {
-		dev_err(dev, "could not configure chip timings: %d\n", ret);
-		return ret;
-	}
-
 	ret = nand_scan_ident(mtd, nsels, NULL);
 	if (ret)
 		return ret;
@@ -2096,12 +2083,6 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
 
 	nand->options |= NAND_SUBPAGE_READ;
 
-	ret = sunxi_nand_chip_init_timings(chip, np);
-	if (ret) {
-		dev_err(dev, "could not configure chip timings: %d\n", ret);
-		return ret;
-	}
-
 	ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
 	if (ret) {
 		dev_err(dev, "ECC init failed: %d\n", ret);
@@ -2175,8 +2156,7 @@ static int sunxi_nfc_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	nfc->dev = dev;
-	spin_lock_init(&nfc->controller.lock);
-	init_waitqueue_head(&nfc->controller.wq);
+	nand_hw_control_init(&nfc->controller);
 	INIT_LIST_HEAD(&nfc->chips);
 
 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/mtd/nand/txx9ndfmc.c b/drivers/mtd/nand/txx9ndfmc.c
index 04d63f56baa4..0a14fda2e41b 100644
--- a/drivers/mtd/nand/txx9ndfmc.c
+++ b/drivers/mtd/nand/txx9ndfmc.c
@@ -303,8 +303,7 @@ static int __init txx9ndfmc_probe(struct platform_device *dev)
 	dev_info(&dev->dev, "CLK:%ldMHz HOLD:%d SPW:%d\n",
 		 (gbusclk + 500000) / 1000000, hold, spw);
 
-	spin_lock_init(&drvdata->hw_control.lock);
-	init_waitqueue_head(&drvdata->hw_control.wq);
+	nand_hw_control_init(&drvdata->hw_control);
 
 	platform_set_drvdata(dev, drvdata);
 	txx9ndfmc_initialize(dev);
diff --git a/drivers/mtd/ubi/attach.c b/drivers/mtd/ubi/attach.c
index 903becd31410..93ceea4f27d5 100644
--- a/drivers/mtd/ubi/attach.c
+++ b/drivers/mtd/ubi/attach.c
@@ -91,9 +91,132 @@
 
 static int self_check_ai(struct ubi_device *ubi, struct ubi_attach_info *ai);
 
-/* Temporary variables used during scanning */
-static struct ubi_ec_hdr *ech;
-static struct ubi_vid_hdr *vidh;
+#define AV_FIND		BIT(0)
+#define AV_ADD		BIT(1)
+#define AV_FIND_OR_ADD	(AV_FIND | AV_ADD)
+
+/**
+ * find_or_add_av - internal function to find a volume, add a volume or do
+ *		    both (find and add if missing).
+ * @ai: attaching information
+ * @vol_id: the requested volume ID
+ * @flags: a combination of the %AV_FIND and %AV_ADD flags describing the
+ *	   expected operation. If only %AV_ADD is set, -EEXIST is returned
+ *	   if the volume already exists. If only %AV_FIND is set, NULL is
+ *	   returned if the volume does not exist. And if both flags are
+ *	   set, the helper first tries to find an existing volume, and if
+ *	   it does not exist it creates a new one.
+ * @created: in value used to inform the caller whether it"s a newly created
+ *	     volume or not.
+ *
+ * This function returns a pointer to a volume description or an ERR_PTR if
+ * the operation failed. It can also return NULL if only %AV_FIND is set and
+ * the volume does not exist.
+ */
+static struct ubi_ainf_volume *find_or_add_av(struct ubi_attach_info *ai,
+					      int vol_id, unsigned int flags,
+					      bool *created)
+{
+	struct ubi_ainf_volume *av;
+	struct rb_node **p = &ai->volumes.rb_node, *parent = NULL;
+
+	/* Walk the volume RB-tree to look if this volume is already present */
+	while (*p) {
+		parent = *p;
+		av = rb_entry(parent, struct ubi_ainf_volume, rb);
+
+		if (vol_id == av->vol_id) {
+			*created = false;
+
+			if (!(flags & AV_FIND))
+				return ERR_PTR(-EEXIST);
+
+			return av;
+		}
+
+		if (vol_id > av->vol_id)
+			p = &(*p)->rb_left;
+		else
+			p = &(*p)->rb_right;
+	}
+
+	if (!(flags & AV_ADD))
+		return NULL;
+
+	/* The volume is absent - add it */
+	av = kzalloc(sizeof(*av), GFP_KERNEL);
+	if (!av)
+		return ERR_PTR(-ENOMEM);
+
+	av->vol_id = vol_id;
+
+	if (vol_id > ai->highest_vol_id)
+		ai->highest_vol_id = vol_id;
+
+	rb_link_node(&av->rb, parent, p);
+	rb_insert_color(&av->rb, &ai->volumes);
+	ai->vols_found += 1;
+	*created = true;
+	dbg_bld("added volume %d", vol_id);
+	return av;
+}
+
+/**
+ * ubi_find_or_add_av - search for a volume in the attaching information and
+ *			add one if it does not exist.
+ * @ai: attaching information
+ * @vol_id: the requested volume ID
+ * @created: whether the volume has been created or not
+ *
+ * This function returns a pointer to the new volume description or an
+ * ERR_PTR if the operation failed.
+ */
+static struct ubi_ainf_volume *ubi_find_or_add_av(struct ubi_attach_info *ai,
+						  int vol_id, bool *created)
+{
+	return find_or_add_av(ai, vol_id, AV_FIND_OR_ADD, created);
+}
+
+/**
+ * ubi_alloc_aeb - allocate an aeb element
+ * @ai: attaching information
+ * @pnum: physical eraseblock number
+ * @ec: erase counter of the physical eraseblock
+ *
+ * Allocate an aeb object and initialize the pnum and ec information.
+ * vol_id and lnum are set to UBI_UNKNOWN, and the other fields are
+ * initialized to zero.
+ * Note that the element is not added in any list or RB tree.
+ */
+struct ubi_ainf_peb *ubi_alloc_aeb(struct ubi_attach_info *ai, int pnum,
+				   int ec)
+{
+	struct ubi_ainf_peb *aeb;
+
+	aeb = kmem_cache_zalloc(ai->aeb_slab_cache, GFP_KERNEL);
+	if (!aeb)
+		return NULL;
+
+	aeb->pnum = pnum;
+	aeb->ec = ec;
+	aeb->vol_id = UBI_UNKNOWN;
+	aeb->lnum = UBI_UNKNOWN;
+
+	return aeb;
+}
+
+/**
+ * ubi_free_aeb - free an aeb element
+ * @ai: attaching information
+ * @aeb: the element to free
+ *
+ * Free an aeb object. The caller must have removed the element from any list
+ * or RB tree.
+ */
+void ubi_free_aeb(struct ubi_attach_info *ai, struct ubi_ainf_peb *aeb)
+{
+	kmem_cache_free(ai->aeb_slab_cache, aeb);
+}
 
 /**
  * add_to_list - add physical eraseblock to a list.
@@ -131,14 +254,12 @@ static int add_to_list(struct ubi_attach_info *ai, int pnum, int vol_id,
 	} else
 		BUG();
 
-	aeb = kmem_cache_alloc(ai->aeb_slab_cache, GFP_KERNEL);
+	aeb = ubi_alloc_aeb(ai, pnum, ec);
 	if (!aeb)
 		return -ENOMEM;
 
-	aeb->pnum = pnum;
 	aeb->vol_id = vol_id;
 	aeb->lnum = lnum;
-	aeb->ec = ec;
 	if (to_head)
 		list_add(&aeb->u.list, list);
 	else
@@ -163,13 +284,11 @@ static int add_corrupted(struct ubi_attach_info *ai, int pnum, int ec)
 
 	dbg_bld("add to corrupted: PEB %d, EC %d", pnum, ec);
 
-	aeb = kmem_cache_alloc(ai->aeb_slab_cache, GFP_KERNEL);
+	aeb = ubi_alloc_aeb(ai, pnum, ec);
 	if (!aeb)
 		return -ENOMEM;
 
 	ai->corr_peb_count += 1;
-	aeb->pnum = pnum;
-	aeb->ec = ec;
 	list_add(&aeb->u.list, &ai->corr);
 	return 0;
 }
@@ -192,14 +311,12 @@ static int add_fastmap(struct ubi_attach_info *ai, int pnum,
 {
 	struct ubi_ainf_peb *aeb;
 
-	aeb = kmem_cache_alloc(ai->aeb_slab_cache, GFP_KERNEL);
+	aeb = ubi_alloc_aeb(ai, pnum, ec);
 	if (!aeb)
 		return -ENOMEM;
 
-	aeb->pnum = pnum;
-	aeb->vol_id = be32_to_cpu(vidh->vol_id);
-	aeb->sqnum = be64_to_cpu(vidh->sqnum);
-	aeb->ec = ec;
+	aeb->vol_id = be32_to_cpu(vid_hdr->vol_id);
+	aeb->sqnum = be64_to_cpu(vid_hdr->sqnum);
 	list_add(&aeb->u.list, &ai->fastmap);
 
 	dbg_bld("add to fastmap list: PEB %d, vol_id %d, sqnum: %llu", pnum,
@@ -294,44 +411,20 @@ static struct ubi_ainf_volume *add_volume(struct ubi_attach_info *ai,
 					  const struct ubi_vid_hdr *vid_hdr)
 {
 	struct ubi_ainf_volume *av;
-	struct rb_node **p = &ai->volumes.rb_node, *parent = NULL;
+	bool created;
 
 	ubi_assert(vol_id == be32_to_cpu(vid_hdr->vol_id));
 
-	/* Walk the volume RB-tree to look if this volume is already present */
-	while (*p) {
-		parent = *p;
-		av = rb_entry(parent, struct ubi_ainf_volume, rb);
-
-		if (vol_id == av->vol_id)
-			return av;
-
-		if (vol_id > av->vol_id)
-			p = &(*p)->rb_left;
-		else
-			p = &(*p)->rb_right;
-	}
-
-	/* The volume is absent - add it */
-	av = kmalloc(sizeof(struct ubi_ainf_volume), GFP_KERNEL);
-	if (!av)
-		return ERR_PTR(-ENOMEM);
+	av = ubi_find_or_add_av(ai, vol_id, &created);
+	if (IS_ERR(av) || !created)
+		return av;
 
-	av->highest_lnum = av->leb_count = 0;
-	av->vol_id = vol_id;
-	av->root = RB_ROOT;
 	av->used_ebs = be32_to_cpu(vid_hdr->used_ebs);
 	av->data_pad = be32_to_cpu(vid_hdr->data_pad);
 	av->compat = vid_hdr->compat;
 	av->vol_type = vid_hdr->vol_type == UBI_VID_DYNAMIC ? UBI_DYNAMIC_VOLUME
 							    : UBI_STATIC_VOLUME;
-	if (vol_id > ai->highest_vol_id)
-		ai->highest_vol_id = vol_id;
 
-	rb_link_node(&av->rb, parent, p);
-	rb_insert_color(&av->rb, &ai->volumes);
-	ai->vols_found += 1;
-	dbg_bld("added volume %d", vol_id);
 	return av;
 }
 
@@ -360,7 +453,7 @@ int ubi_compare_lebs(struct ubi_device *ubi, const struct ubi_ainf_peb *aeb,
 {
 	int len, err, second_is_newer, bitflips = 0, corrupted = 0;
 	uint32_t data_crc, crc;
-	struct ubi_vid_hdr *vh = NULL;
+	struct ubi_vid_io_buf *vidb = NULL;
 	unsigned long long sqnum2 = be64_to_cpu(vid_hdr->sqnum);
 
 	if (sqnum2 == aeb->sqnum) {
@@ -403,12 +496,12 @@ int ubi_compare_lebs(struct ubi_device *ubi, const struct ubi_ainf_peb *aeb,
 			return bitflips << 1;
 		}
 
-		vh = ubi_zalloc_vid_hdr(ubi, GFP_KERNEL);
-		if (!vh)
+		vidb = ubi_alloc_vid_buf(ubi, GFP_KERNEL);
+		if (!vidb)
 			return -ENOMEM;
 
 		pnum = aeb->pnum;
-		err = ubi_io_read_vid_hdr(ubi, pnum, vh, 0);
+		err = ubi_io_read_vid_hdr(ubi, pnum, vidb, 0);
 		if (err) {
 			if (err == UBI_IO_BITFLIPS)
 				bitflips = 1;
@@ -422,7 +515,7 @@ int ubi_compare_lebs(struct ubi_device *ubi, const struct ubi_ainf_peb *aeb,
 			}
 		}
 
-		vid_hdr = vh;
+		vid_hdr = ubi_get_vid_hdr(vidb);
 	}
 
 	/* Read the data of the copy and check the CRC */
@@ -448,7 +541,7 @@ int ubi_compare_lebs(struct ubi_device *ubi, const struct ubi_ainf_peb *aeb,
 	}
 	mutex_unlock(&ubi->buf_mutex);
 
-	ubi_free_vid_hdr(ubi, vh);
+	ubi_free_vid_buf(vidb);
 
 	if (second_is_newer)
 		dbg_bld("second PEB %d is newer, copy_flag is set", pnum);
@@ -460,7 +553,7 @@ int ubi_compare_lebs(struct ubi_device *ubi, const struct ubi_ainf_peb *aeb,
 out_unlock:
 	mutex_unlock(&ubi->buf_mutex);
 out_free_vidh:
-	ubi_free_vid_hdr(ubi, vh);
+	ubi_free_vid_buf(vidb);
 	return err;
 }
 
@@ -605,12 +698,10 @@ int ubi_add_to_av(struct ubi_device *ubi, struct ubi_attach_info *ai, int pnum,
 	if (err)
 		return err;
 
-	aeb = kmem_cache_alloc(ai->aeb_slab_cache, GFP_KERNEL);
+	aeb = ubi_alloc_aeb(ai, pnum, ec);
 	if (!aeb)
 		return -ENOMEM;
 
-	aeb->ec = ec;
-	aeb->pnum = pnum;
 	aeb->vol_id = vol_id;
 	aeb->lnum = lnum;
 	aeb->scrub = bitflips;
@@ -629,6 +720,21 @@ int ubi_add_to_av(struct ubi_device *ubi, struct ubi_attach_info *ai, int pnum,
 }
 
 /**
+ * ubi_add_av - add volume to the attaching information.
+ * @ai: attaching information
+ * @vol_id: the requested volume ID
+ *
+ * This function returns a pointer to the new volume description or an
+ * ERR_PTR if the operation failed.
+ */
+struct ubi_ainf_volume *ubi_add_av(struct ubi_attach_info *ai, int vol_id)
+{
+	bool created;
+
+	return find_or_add_av(ai, vol_id, AV_ADD, &created);
+}
+
+/**
  * ubi_find_av - find volume in the attaching information.
  * @ai: attaching information
  * @vol_id: the requested volume ID
@@ -639,24 +745,15 @@ int ubi_add_to_av(struct ubi_device *ubi, struct ubi_attach_info *ai, int pnum,
 struct ubi_ainf_volume *ubi_find_av(const struct ubi_attach_info *ai,
 				    int vol_id)
 {
-	struct ubi_ainf_volume *av;
-	struct rb_node *p = ai->volumes.rb_node;
-
-	while (p) {
-		av = rb_entry(p, struct ubi_ainf_volume, rb);
-
-		if (vol_id == av->vol_id)
-			return av;
-
-		if (vol_id > av->vol_id)
-			p = p->rb_left;
-		else
-			p = p->rb_right;
-	}
+	bool created;
 
-	return NULL;
+	return find_or_add_av((struct ubi_attach_info *)ai, vol_id, AV_FIND,
+			      &created);
 }
 
+static void destroy_av(struct ubi_attach_info *ai, struct ubi_ainf_volume *av,
+		       struct list_head *list);
+
 /**
  * ubi_remove_av - delete attaching information about a volume.
  * @ai: attaching information
@@ -664,19 +761,10 @@ struct ubi_ainf_volume *ubi_find_av(const struct ubi_attach_info *ai,
  */
 void ubi_remove_av(struct ubi_attach_info *ai, struct ubi_ainf_volume *av)
 {
-	struct rb_node *rb;
-	struct ubi_ainf_peb *aeb;
-
 	dbg_bld("remove attaching information about volume %d", av->vol_id);
 
-	while ((rb = rb_first(&av->root))) {
-		aeb = rb_entry(rb, struct ubi_ainf_peb, u.rb);
-		rb_erase(&aeb->u.rb, &av->root);
-		list_add_tail(&aeb->u.list, &ai->erase);
-	}
-
 	rb_erase(&av->rb, &ai->volumes);
-	kfree(av);
+	destroy_av(ai, av, &ai->erase);
 	ai->vols_found -= 1;
 }
 
@@ -866,6 +954,9 @@ static bool vol_ignored(int vol_id)
 static int scan_peb(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		    int pnum, bool fast)
 {
+	struct ubi_ec_hdr *ech = ai->ech;
+	struct ubi_vid_io_buf *vidb = ai->vidb;
+	struct ubi_vid_hdr *vidh = ubi_get_vid_hdr(vidb);
 	long long ec;
 	int err, bitflips = 0, vol_id = -1, ec_err = 0;
 
@@ -963,7 +1054,7 @@ static int scan_peb(struct ubi_device *ubi, struct ubi_attach_info *ai,
 
 	/* OK, we've done with the EC header, let's look at the VID header */
 
-	err = ubi_io_read_vid_hdr(ubi, pnum, vidh, 0);
+	err = ubi_io_read_vid_hdr(ubi, pnum, vidb, 0);
 	if (err < 0)
 		return err;
 	switch (err) {
@@ -1191,10 +1282,12 @@ static int late_analysis(struct ubi_device *ubi, struct ubi_attach_info *ai)
  * destroy_av - free volume attaching information.
  * @av: volume attaching information
  * @ai: attaching information
+ * @list: put the aeb elements in there if !NULL, otherwise free them
  *
  * This function destroys the volume attaching information.
  */
-static void destroy_av(struct ubi_attach_info *ai, struct ubi_ainf_volume *av)
+static void destroy_av(struct ubi_attach_info *ai, struct ubi_ainf_volume *av,
+		       struct list_head *list)
 {
 	struct ubi_ainf_peb *aeb;
 	struct rb_node *this = av->root.rb_node;
@@ -1214,7 +1307,10 @@ static void destroy_av(struct ubi_attach_info *ai, struct ubi_ainf_volume *av)
 					this->rb_right = NULL;
 			}
 
-			kmem_cache_free(ai->aeb_slab_cache, aeb);
+			if (list)
+				list_add_tail(&aeb->u.list, list);
+			else
+				ubi_free_aeb(ai, aeb);
 		}
 	}
 	kfree(av);
@@ -1232,23 +1328,23 @@ static void destroy_ai(struct ubi_attach_info *ai)
 
 	list_for_each_entry_safe(aeb, aeb_tmp, &ai->alien, u.list) {
 		list_del(&aeb->u.list);
-		kmem_cache_free(ai->aeb_slab_cache, aeb);
+		ubi_free_aeb(ai, aeb);
 	}
 	list_for_each_entry_safe(aeb, aeb_tmp, &ai->erase, u.list) {
 		list_del(&aeb->u.list);
-		kmem_cache_free(ai->aeb_slab_cache, aeb);
+		ubi_free_aeb(ai, aeb);
 	}
 	list_for_each_entry_safe(aeb, aeb_tmp, &ai->corr, u.list) {
 		list_del(&aeb->u.list);
-		kmem_cache_free(ai->aeb_slab_cache, aeb);
+		ubi_free_aeb(ai, aeb);
 	}
 	list_for_each_entry_safe(aeb, aeb_tmp, &ai->free, u.list) {
 		list_del(&aeb->u.list);
-		kmem_cache_free(ai->aeb_slab_cache, aeb);
+		ubi_free_aeb(ai, aeb);
 	}
 	list_for_each_entry_safe(aeb, aeb_tmp, &ai->fastmap, u.list) {
 		list_del(&aeb->u.list);
-		kmem_cache_free(ai->aeb_slab_cache, aeb);
+		ubi_free_aeb(ai, aeb);
 	}
 
 	/* Destroy the volume RB-tree */
@@ -1269,7 +1365,7 @@ static void destroy_ai(struct ubi_attach_info *ai)
 					rb->rb_right = NULL;
 			}
 
-			destroy_av(ai, av);
+			destroy_av(ai, av, NULL);
 		}
 	}
 
@@ -1297,12 +1393,12 @@ static int scan_all(struct ubi_device *ubi, struct ubi_attach_info *ai,
 
 	err = -ENOMEM;
 
-	ech = kzalloc(ubi->ec_hdr_alsize, GFP_KERNEL);
-	if (!ech)
+	ai->ech = kzalloc(ubi->ec_hdr_alsize, GFP_KERNEL);
+	if (!ai->ech)
 		return err;
 
-	vidh = ubi_zalloc_vid_hdr(ubi, GFP_KERNEL);
-	if (!vidh)
+	ai->vidb = ubi_alloc_vid_buf(ubi, GFP_KERNEL);
+	if (!ai->vidb)
 		goto out_ech;
 
 	for (pnum = start; pnum < ubi->peb_count; pnum++) {
@@ -1351,15 +1447,15 @@ static int scan_all(struct ubi_device *ubi, struct ubi_attach_info *ai,
 	if (err)
 		goto out_vidh;
 
-	ubi_free_vid_hdr(ubi, vidh);
-	kfree(ech);
+	ubi_free_vid_buf(ai->vidb);
+	kfree(ai->ech);
 
 	return 0;
 
 out_vidh:
-	ubi_free_vid_hdr(ubi, vidh);
+	ubi_free_vid_buf(ai->vidb);
 out_ech:
-	kfree(ech);
+	kfree(ai->ech);
 	return err;
 }
 
@@ -1411,12 +1507,12 @@ static int scan_fast(struct ubi_device *ubi, struct ubi_attach_info **ai)
 	if (!scan_ai)
 		goto out;
 
-	ech = kzalloc(ubi->ec_hdr_alsize, GFP_KERNEL);
-	if (!ech)
+	scan_ai->ech = kzalloc(ubi->ec_hdr_alsize, GFP_KERNEL);
+	if (!scan_ai->ech)
 		goto out_ai;
 
-	vidh = ubi_zalloc_vid_hdr(ubi, GFP_KERNEL);
-	if (!vidh)
+	scan_ai->vidb = ubi_alloc_vid_buf(ubi, GFP_KERNEL);
+	if (!scan_ai->vidb)
 		goto out_ech;
 
 	for (pnum = 0; pnum < UBI_FM_MAX_START; pnum++) {
@@ -1428,8 +1524,8 @@ static int scan_fast(struct ubi_device *ubi, struct ubi_attach_info **ai)
 			goto out_vidh;
 	}
 
-	ubi_free_vid_hdr(ubi, vidh);
-	kfree(ech);
+	ubi_free_vid_buf(scan_ai->vidb);
+	kfree(scan_ai->ech);
 
 	if (scan_ai->force_full_scan)
 		err = UBI_NO_FASTMAP;
@@ -1449,9 +1545,9 @@ static int scan_fast(struct ubi_device *ubi, struct ubi_attach_info **ai)
 	return err;
 
 out_vidh:
-	ubi_free_vid_hdr(ubi, vidh);
+	ubi_free_vid_buf(scan_ai->vidb);
 out_ech:
-	kfree(ech);
+	kfree(scan_ai->ech);
 out_ai:
 	destroy_ai(scan_ai);
 out:
@@ -1573,6 +1669,8 @@ out_ai:
  */
 static int self_check_ai(struct ubi_device *ubi, struct ubi_attach_info *ai)
 {
+	struct ubi_vid_io_buf *vidb = ai->vidb;
+	struct ubi_vid_hdr *vidh = ubi_get_vid_hdr(vidb);
 	int pnum, err, vols_found = 0;
 	struct rb_node *rb1, *rb2;
 	struct ubi_ainf_volume *av;
@@ -1708,7 +1806,7 @@ static int self_check_ai(struct ubi_device *ubi, struct ubi_attach_info *ai)
 
 			last_aeb = aeb;
 
-			err = ubi_io_read_vid_hdr(ubi, aeb->pnum, vidh, 1);
+			err = ubi_io_read_vid_hdr(ubi, aeb->pnum, vidb, 1);
 			if (err && err != UBI_IO_BITFLIPS) {
 				ubi_err(ubi, "VID header is not OK (%d)",
 					err);
diff --git a/drivers/mtd/ubi/block.c b/drivers/mtd/ubi/block.c
index ebf46ad2d513..d1e6931c132f 100644
--- a/drivers/mtd/ubi/block.c
+++ b/drivers/mtd/ubi/block.c
@@ -351,7 +351,6 @@ static int ubiblock_init_request(void *data, struct request *req,
 static struct blk_mq_ops ubiblock_mq_ops = {
 	.queue_rq       = ubiblock_queue_rq,
 	.init_request	= ubiblock_init_request,
-	.map_queue      = blk_mq_map_queue,
 };
 
 static DEFINE_IDR(ubiblock_minor_idr);
diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c
index 0680516bb472..85d54f37e28f 100644
--- a/drivers/mtd/ubi/build.c
+++ b/drivers/mtd/ubi/build.c
@@ -574,7 +574,7 @@ void ubi_free_internal_volumes(struct ubi_device *ubi)
 
 	for (i = ubi->vtbl_slots;
 	     i < ubi->vtbl_slots + UBI_INT_VOL_COUNT; i++) {
-		kfree(ubi->volumes[i]->eba_tbl);
+		ubi_eba_replace_table(ubi->volumes[i], NULL);
 		kfree(ubi->volumes[i]);
 	}
 }
diff --git a/drivers/mtd/ubi/cdev.c b/drivers/mtd/ubi/cdev.c
index ee2b74d1d1b5..45c329694a5e 100644
--- a/drivers/mtd/ubi/cdev.c
+++ b/drivers/mtd/ubi/cdev.c
@@ -416,7 +416,7 @@ static long vol_cdev_ioctl(struct file *file, unsigned int cmd,
 		}
 
 		rsvd_bytes = (long long)vol->reserved_pebs *
-					ubi->leb_size-vol->data_pad;
+					vol->usable_leb_size;
 		if (bytes < 0 || bytes > rsvd_bytes) {
 			err = -EINVAL;
 			break;
@@ -454,7 +454,7 @@ static long vol_cdev_ioctl(struct file *file, unsigned int cmd,
 
 		/* Validate the request */
 		err = -EINVAL;
-		if (req.lnum < 0 || req.lnum >= vol->reserved_pebs ||
+		if (!ubi_leb_valid(vol, req.lnum) ||
 		    req.bytes < 0 || req.bytes > vol->usable_leb_size)
 			break;
 
@@ -485,7 +485,7 @@ static long vol_cdev_ioctl(struct file *file, unsigned int cmd,
 			break;
 		}
 
-		if (lnum < 0 || lnum >= vol->reserved_pebs) {
+		if (!ubi_leb_valid(vol, lnum)) {
 			err = -EINVAL;
 			break;
 		}
diff --git a/drivers/mtd/ubi/eba.c b/drivers/mtd/ubi/eba.c
index ebf517271d29..95c4048a371e 100644
--- a/drivers/mtd/ubi/eba.c
+++ b/drivers/mtd/ubi/eba.c
@@ -50,6 +50,30 @@
 #define EBA_RESERVED_PEBS 1
 
 /**
+ * struct ubi_eba_entry - structure encoding a single LEB -> PEB association
+ * @pnum: the physical eraseblock number attached to the LEB
+ *
+ * This structure is encoding a LEB -> PEB association. Note that the LEB
+ * number is not stored here, because it is the index used to access the
+ * entries table.
+ */
+struct ubi_eba_entry {
+	int pnum;
+};
+
+/**
+ * struct ubi_eba_table - LEB -> PEB association information
+ * @entries: the LEB to PEB mapping (one entry per LEB).
+ *
+ * This structure is private to the EBA logic and should be kept here.
+ * It is encoding the LEB to PEB association table, and is subject to
+ * changes.
+ */
+struct ubi_eba_table {
+	struct ubi_eba_entry *entries;
+};
+
+/**
  * next_sqnum - get next sequence number.
  * @ubi: UBI device description object
  *
@@ -84,6 +108,110 @@ static int ubi_get_compat(const struct ubi_device *ubi, int vol_id)
 }
 
 /**
+ * ubi_eba_get_ldesc - get information about a LEB
+ * @vol: volume description object
+ * @lnum: logical eraseblock number
+ * @ldesc: the LEB descriptor to fill
+ *
+ * Used to query information about a specific LEB.
+ * It is currently only returning the physical position of the LEB, but will be
+ * extended to provide more information.
+ */
+void ubi_eba_get_ldesc(struct ubi_volume *vol, int lnum,
+		       struct ubi_eba_leb_desc *ldesc)
+{
+	ldesc->lnum = lnum;
+	ldesc->pnum = vol->eba_tbl->entries[lnum].pnum;
+}
+
+/**
+ * ubi_eba_create_table - allocate a new EBA table and initialize it with all
+ *			  LEBs unmapped
+ * @vol: volume containing the EBA table to copy
+ * @nentries: number of entries in the table
+ *
+ * Allocate a new EBA table and initialize it with all LEBs unmapped.
+ * Returns a valid pointer if it succeed, an ERR_PTR() otherwise.
+ */
+struct ubi_eba_table *ubi_eba_create_table(struct ubi_volume *vol,
+					   int nentries)
+{
+	struct ubi_eba_table *tbl;
+	int err = -ENOMEM;
+	int i;
+
+	tbl = kzalloc(sizeof(*tbl), GFP_KERNEL);
+	if (!tbl)
+		return ERR_PTR(-ENOMEM);
+
+	tbl->entries = kmalloc_array(nentries, sizeof(*tbl->entries),
+				     GFP_KERNEL);
+	if (!tbl->entries)
+		goto err;
+
+	for (i = 0; i < nentries; i++)
+		tbl->entries[i].pnum = UBI_LEB_UNMAPPED;
+
+	return tbl;
+
+err:
+	kfree(tbl->entries);
+	kfree(tbl);
+
+	return ERR_PTR(err);
+}
+
+/**
+ * ubi_eba_destroy_table - destroy an EBA table
+ * @tbl: the table to destroy
+ *
+ * Destroy an EBA table.
+ */
+void ubi_eba_destroy_table(struct ubi_eba_table *tbl)
+{
+	if (!tbl)
+		return;
+
+	kfree(tbl->entries);
+	kfree(tbl);
+}
+
+/**
+ * ubi_eba_copy_table - copy the EBA table attached to vol into another table
+ * @vol: volume containing the EBA table to copy
+ * @dst: destination
+ * @nentries: number of entries to copy
+ *
+ * Copy the EBA table stored in vol into the one pointed by dst.
+ */
+void ubi_eba_copy_table(struct ubi_volume *vol, struct ubi_eba_table *dst,
+			int nentries)
+{
+	struct ubi_eba_table *src;
+	int i;
+
+	ubi_assert(dst && vol && vol->eba_tbl);
+
+	src = vol->eba_tbl;
+
+	for (i = 0; i < nentries; i++)
+		dst->entries[i].pnum = src->entries[i].pnum;
+}
+
+/**
+ * ubi_eba_replace_table - assign a new EBA table to a volume
+ * @vol: volume containing the EBA table to copy
+ * @tbl: new EBA table
+ *
+ * Assign a new EBA table to the volume and release the old one.
+ */
+void ubi_eba_replace_table(struct ubi_volume *vol, struct ubi_eba_table *tbl)
+{
+	ubi_eba_destroy_table(vol->eba_tbl);
+	vol->eba_tbl = tbl;
+}
+
+/**
  * ltree_lookup - look up the lock tree.
  * @ubi: UBI device description object
  * @vol_id: volume ID
@@ -312,6 +440,18 @@ static void leb_write_unlock(struct ubi_device *ubi, int vol_id, int lnum)
 }
 
 /**
+ * ubi_eba_is_mapped - check if a LEB is mapped.
+ * @vol: volume description object
+ * @lnum: logical eraseblock number
+ *
+ * This function returns true if the LEB is mapped, false otherwise.
+ */
+bool ubi_eba_is_mapped(struct ubi_volume *vol, int lnum)
+{
+	return vol->eba_tbl->entries[lnum].pnum >= 0;
+}
+
+/**
  * ubi_eba_unmap_leb - un-map logical eraseblock.
  * @ubi: UBI device description object
  * @vol: volume description object
@@ -333,7 +473,7 @@ int ubi_eba_unmap_leb(struct ubi_device *ubi, struct ubi_volume *vol,
 	if (err)
 		return err;
 
-	pnum = vol->eba_tbl[lnum];
+	pnum = vol->eba_tbl->entries[lnum].pnum;
 	if (pnum < 0)
 		/* This logical eraseblock is already unmapped */
 		goto out_unlock;
@@ -341,7 +481,7 @@ int ubi_eba_unmap_leb(struct ubi_device *ubi, struct ubi_volume *vol,
 	dbg_eba("erase LEB %d:%d, PEB %d", vol_id, lnum, pnum);
 
 	down_read(&ubi->fm_eba_sem);
-	vol->eba_tbl[lnum] = UBI_LEB_UNMAPPED;
+	vol->eba_tbl->entries[lnum].pnum = UBI_LEB_UNMAPPED;
 	up_read(&ubi->fm_eba_sem);
 	err = ubi_wl_put_peb(ubi, vol_id, lnum, pnum, 0);
 
@@ -373,6 +513,7 @@ int ubi_eba_read_leb(struct ubi_device *ubi, struct ubi_volume *vol, int lnum,
 		     void *buf, int offset, int len, int check)
 {
 	int err, pnum, scrub = 0, vol_id = vol->vol_id;
+	struct ubi_vid_io_buf *vidb;
 	struct ubi_vid_hdr *vid_hdr;
 	uint32_t uninitialized_var(crc);
 
@@ -380,7 +521,7 @@ int ubi_eba_read_leb(struct ubi_device *ubi, struct ubi_volume *vol, int lnum,
 	if (err)
 		return err;
 
-	pnum = vol->eba_tbl[lnum];
+	pnum = vol->eba_tbl->entries[lnum].pnum;
 	if (pnum < 0) {
 		/*
 		 * The logical eraseblock is not mapped, fill the whole buffer
@@ -403,13 +544,15 @@ int ubi_eba_read_leb(struct ubi_device *ubi, struct ubi_volume *vol, int lnum,
 
 retry:
 	if (check) {
-		vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_NOFS);
-		if (!vid_hdr) {
+		vidb = ubi_alloc_vid_buf(ubi, GFP_NOFS);
+		if (!vidb) {
 			err = -ENOMEM;
 			goto out_unlock;
 		}
 
-		err = ubi_io_read_vid_hdr(ubi, pnum, vid_hdr, 1);
+		vid_hdr = ubi_get_vid_hdr(vidb);
+
+		err = ubi_io_read_vid_hdr(ubi, pnum, vidb, 1);
 		if (err && err != UBI_IO_BITFLIPS) {
 			if (err > 0) {
 				/*
@@ -455,7 +598,7 @@ retry:
 		ubi_assert(len == be32_to_cpu(vid_hdr->data_size));
 
 		crc = be32_to_cpu(vid_hdr->data_crc);
-		ubi_free_vid_hdr(ubi, vid_hdr);
+		ubi_free_vid_buf(vidb);
 	}
 
 	err = ubi_io_read_data(ubi, buf, pnum, offset, len);
@@ -492,7 +635,7 @@ retry:
 	return err;
 
 out_free:
-	ubi_free_vid_hdr(ubi, vid_hdr);
+	ubi_free_vid_buf(vidb);
 out_unlock:
 	leb_read_unlock(ubi, vol_id, lnum);
 	return err;
@@ -554,49 +697,47 @@ int ubi_eba_read_leb_sg(struct ubi_device *ubi, struct ubi_volume *vol,
 }
 
 /**
- * recover_peb - recover from write failure.
- * @ubi: UBI device description object
+ * try_recover_peb - try to recover from write failure.
+ * @vol: volume description object
  * @pnum: the physical eraseblock to recover
- * @vol_id: volume ID
  * @lnum: logical eraseblock number
  * @buf: data which was not written because of the write failure
  * @offset: offset of the failed write
  * @len: how many bytes should have been written
+ * @vidb: VID buffer
+ * @retry: whether the caller should retry in case of failure
  *
  * This function is called in case of a write failure and moves all good data
  * from the potentially bad physical eraseblock to a good physical eraseblock.
  * This function also writes the data which was not written due to the failure.
- * Returns new physical eraseblock number in case of success, and a negative
- * error code in case of failure.
+ * Returns 0 in case of success, and a negative error code in case of failure.
+ * In case of failure, the %retry parameter is set to false if this is a fatal
+ * error (retrying won't help), and true otherwise.
  */
-static int recover_peb(struct ubi_device *ubi, int pnum, int vol_id, int lnum,
-		       const void *buf, int offset, int len)
+static int try_recover_peb(struct ubi_volume *vol, int pnum, int lnum,
+			   const void *buf, int offset, int len,
+			   struct ubi_vid_io_buf *vidb, bool *retry)
 {
-	int err, idx = vol_id2idx(ubi, vol_id), new_pnum, data_size, tries = 0;
-	struct ubi_volume *vol = ubi->volumes[idx];
+	struct ubi_device *ubi = vol->ubi;
 	struct ubi_vid_hdr *vid_hdr;
+	int new_pnum, err, vol_id = vol->vol_id, data_size;
 	uint32_t crc;
 
-	vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_NOFS);
-	if (!vid_hdr)
-		return -ENOMEM;
+	*retry = false;
 
-retry:
 	new_pnum = ubi_wl_get_peb(ubi);
 	if (new_pnum < 0) {
-		ubi_free_vid_hdr(ubi, vid_hdr);
-		up_read(&ubi->fm_eba_sem);
-		return new_pnum;
+		err = new_pnum;
+		goto out_put;
 	}
 
 	ubi_msg(ubi, "recover PEB %d, move data to PEB %d",
 		pnum, new_pnum);
 
-	err = ubi_io_read_vid_hdr(ubi, pnum, vid_hdr, 1);
+	err = ubi_io_read_vid_hdr(ubi, pnum, vidb, 1);
 	if (err && err != UBI_IO_BITFLIPS) {
 		if (err > 0)
 			err = -EIO;
-		up_read(&ubi->fm_eba_sem);
 		goto out_put;
 	}
 
@@ -608,12 +749,12 @@ retry:
 	/* Read everything before the area where the write failure happened */
 	if (offset > 0) {
 		err = ubi_io_read_data(ubi, ubi->peb_buf, pnum, 0, offset);
-		if (err && err != UBI_IO_BITFLIPS) {
-			up_read(&ubi->fm_eba_sem);
+		if (err && err != UBI_IO_BITFLIPS)
 			goto out_unlock;
-		}
 	}
 
+	*retry = true;
+
 	memcpy(ubi->peb_buf + offset, buf, len);
 
 	data_size = offset + len;
@@ -622,50 +763,140 @@ retry:
 	vid_hdr->copy_flag = 1;
 	vid_hdr->data_size = cpu_to_be32(data_size);
 	vid_hdr->data_crc = cpu_to_be32(crc);
-	err = ubi_io_write_vid_hdr(ubi, new_pnum, vid_hdr);
-	if (err) {
-		mutex_unlock(&ubi->buf_mutex);
-		up_read(&ubi->fm_eba_sem);
-		goto write_error;
-	}
+	err = ubi_io_write_vid_hdr(ubi, new_pnum, vidb);
+	if (err)
+		goto out_unlock;
 
 	err = ubi_io_write_data(ubi, ubi->peb_buf, new_pnum, 0, data_size);
-	if (err) {
-		mutex_unlock(&ubi->buf_mutex);
-		up_read(&ubi->fm_eba_sem);
-		goto write_error;
-	}
 
+out_unlock:
 	mutex_unlock(&ubi->buf_mutex);
-	ubi_free_vid_hdr(ubi, vid_hdr);
 
-	vol->eba_tbl[lnum] = new_pnum;
+	if (!err)
+		vol->eba_tbl->entries[lnum].pnum = new_pnum;
+
+out_put:
 	up_read(&ubi->fm_eba_sem);
-	ubi_wl_put_peb(ubi, vol_id, lnum, pnum, 1);
 
-	ubi_msg(ubi, "data was successfully recovered");
-	return 0;
+	if (!err) {
+		ubi_wl_put_peb(ubi, vol_id, lnum, pnum, 1);
+		ubi_msg(ubi, "data was successfully recovered");
+	} else if (new_pnum >= 0) {
+		/*
+		 * Bad luck? This physical eraseblock is bad too? Crud. Let's
+		 * try to get another one.
+		 */
+		ubi_wl_put_peb(ubi, vol_id, lnum, new_pnum, 1);
+		ubi_warn(ubi, "failed to write to PEB %d", new_pnum);
+	}
 
-out_unlock:
-	mutex_unlock(&ubi->buf_mutex);
-out_put:
-	ubi_wl_put_peb(ubi, vol_id, lnum, new_pnum, 1);
-	ubi_free_vid_hdr(ubi, vid_hdr);
 	return err;
+}
 
-write_error:
-	/*
-	 * Bad luck? This physical eraseblock is bad too? Crud. Let's try to
-	 * get another one.
-	 */
-	ubi_warn(ubi, "failed to write to PEB %d", new_pnum);
-	ubi_wl_put_peb(ubi, vol_id, lnum, new_pnum, 1);
-	if (++tries > UBI_IO_RETRIES) {
-		ubi_free_vid_hdr(ubi, vid_hdr);
-		return err;
+/**
+ * recover_peb - recover from write failure.
+ * @ubi: UBI device description object
+ * @pnum: the physical eraseblock to recover
+ * @vol_id: volume ID
+ * @lnum: logical eraseblock number
+ * @buf: data which was not written because of the write failure
+ * @offset: offset of the failed write
+ * @len: how many bytes should have been written
+ *
+ * This function is called in case of a write failure and moves all good data
+ * from the potentially bad physical eraseblock to a good physical eraseblock.
+ * This function also writes the data which was not written due to the failure.
+ * Returns 0 in case of success, and a negative error code in case of failure.
+ * This function tries %UBI_IO_RETRIES before giving up.
+ */
+static int recover_peb(struct ubi_device *ubi, int pnum, int vol_id, int lnum,
+		       const void *buf, int offset, int len)
+{
+	int err, idx = vol_id2idx(ubi, vol_id), tries;
+	struct ubi_volume *vol = ubi->volumes[idx];
+	struct ubi_vid_io_buf *vidb;
+
+	vidb = ubi_alloc_vid_buf(ubi, GFP_NOFS);
+	if (!vidb)
+		return -ENOMEM;
+
+	for (tries = 0; tries <= UBI_IO_RETRIES; tries++) {
+		bool retry;
+
+		err = try_recover_peb(vol, pnum, lnum, buf, offset, len, vidb,
+				      &retry);
+		if (!err || !retry)
+			break;
+
+		ubi_msg(ubi, "try again");
 	}
-	ubi_msg(ubi, "try again");
-	goto retry;
+
+	ubi_free_vid_buf(vidb);
+
+	return err;
+}
+
+/**
+ * try_write_vid_and_data - try to write VID header and data to a new PEB.
+ * @vol: volume description object
+ * @lnum: logical eraseblock number
+ * @vidb: the VID buffer to write
+ * @buf: buffer containing the data
+ * @offset: where to start writing data
+ * @len: how many bytes should be written
+ *
+ * This function tries to write VID header and data belonging to logical
+ * eraseblock @lnum of volume @vol to a new physical eraseblock. Returns zero
+ * in case of success and a negative error code in case of failure.
+ * In case of error, it is possible that something was still written to the
+ * flash media, but may be some garbage.
+ */
+static int try_write_vid_and_data(struct ubi_volume *vol, int lnum,
+				  struct ubi_vid_io_buf *vidb, const void *buf,
+				  int offset, int len)
+{
+	struct ubi_device *ubi = vol->ubi;
+	int pnum, opnum, err, vol_id = vol->vol_id;
+
+	pnum = ubi_wl_get_peb(ubi);
+	if (pnum < 0) {
+		err = pnum;
+		goto out_put;
+	}
+
+	opnum = vol->eba_tbl->entries[lnum].pnum;
+
+	dbg_eba("write VID hdr and %d bytes at offset %d of LEB %d:%d, PEB %d",
+		len, offset, vol_id, lnum, pnum);
+
+	err = ubi_io_write_vid_hdr(ubi, pnum, vidb);
+	if (err) {
+		ubi_warn(ubi, "failed to write VID header to LEB %d:%d, PEB %d",
+			 vol_id, lnum, pnum);
+		goto out_put;
+	}
+
+	if (len) {
+		err = ubi_io_write_data(ubi, buf, pnum, offset, len);
+		if (err) {
+			ubi_warn(ubi,
+				 "failed to write %d bytes at offset %d of LEB %d:%d, PEB %d",
+				 len, offset, vol_id, lnum, pnum);
+			goto out_put;
+		}
+	}
+
+	vol->eba_tbl->entries[lnum].pnum = pnum;
+
+out_put:
+	up_read(&ubi->fm_eba_sem);
+
+	if (err && pnum >= 0)
+		err = ubi_wl_put_peb(ubi, vol_id, lnum, pnum, 1);
+	else if (!err && opnum >= 0)
+		err = ubi_wl_put_peb(ubi, vol_id, lnum, opnum, 0);
+
+	return err;
 }
 
 /**
@@ -681,11 +912,13 @@ write_error:
  * @vol. Returns zero in case of success and a negative error code in case
  * of failure. In case of error, it is possible that something was still
  * written to the flash media, but may be some garbage.
+ * This function retries %UBI_IO_RETRIES times before giving up.
  */
 int ubi_eba_write_leb(struct ubi_device *ubi, struct ubi_volume *vol, int lnum,
 		      const void *buf, int offset, int len)
 {
-	int err, pnum, tries = 0, vol_id = vol->vol_id;
+	int err, pnum, tries, vol_id = vol->vol_id;
+	struct ubi_vid_io_buf *vidb;
 	struct ubi_vid_hdr *vid_hdr;
 
 	if (ubi->ro_mode)
@@ -695,7 +928,7 @@ int ubi_eba_write_leb(struct ubi_device *ubi, struct ubi_volume *vol, int lnum,
 	if (err)
 		return err;
 
-	pnum = vol->eba_tbl[lnum];
+	pnum = vol->eba_tbl->entries[lnum].pnum;
 	if (pnum >= 0) {
 		dbg_eba("write %d bytes at offset %d of LEB %d:%d, PEB %d",
 			len, offset, vol_id, lnum, pnum);
@@ -706,23 +939,23 @@ int ubi_eba_write_leb(struct ubi_device *ubi, struct ubi_volume *vol, int lnum,
 			if (err == -EIO && ubi->bad_allowed)
 				err = recover_peb(ubi, pnum, vol_id, lnum, buf,
 						  offset, len);
-			if (err)
-				ubi_ro_mode(ubi);
 		}
-		leb_write_unlock(ubi, vol_id, lnum);
-		return err;
+
+		goto out;
 	}
 
 	/*
 	 * The logical eraseblock is not mapped. We have to get a free physical
 	 * eraseblock and write the volume identifier header there first.
 	 */
-	vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_NOFS);
-	if (!vid_hdr) {
+	vidb = ubi_alloc_vid_buf(ubi, GFP_NOFS);
+	if (!vidb) {
 		leb_write_unlock(ubi, vol_id, lnum);
 		return -ENOMEM;
 	}
 
+	vid_hdr = ubi_get_vid_hdr(vidb);
+
 	vid_hdr->vol_type = UBI_VID_DYNAMIC;
 	vid_hdr->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
 	vid_hdr->vol_id = cpu_to_be32(vol_id);
@@ -730,67 +963,30 @@ int ubi_eba_write_leb(struct ubi_device *ubi, struct ubi_volume *vol, int lnum,
 	vid_hdr->compat = ubi_get_compat(ubi, vol_id);
 	vid_hdr->data_pad = cpu_to_be32(vol->data_pad);
 
-retry:
-	pnum = ubi_wl_get_peb(ubi);
-	if (pnum < 0) {
-		ubi_free_vid_hdr(ubi, vid_hdr);
-		leb_write_unlock(ubi, vol_id, lnum);
-		up_read(&ubi->fm_eba_sem);
-		return pnum;
-	}
-
-	dbg_eba("write VID hdr and %d bytes at offset %d of LEB %d:%d, PEB %d",
-		len, offset, vol_id, lnum, pnum);
-
-	err = ubi_io_write_vid_hdr(ubi, pnum, vid_hdr);
-	if (err) {
-		ubi_warn(ubi, "failed to write VID header to LEB %d:%d, PEB %d",
-			 vol_id, lnum, pnum);
-		up_read(&ubi->fm_eba_sem);
-		goto write_error;
-	}
+	for (tries = 0; tries <= UBI_IO_RETRIES; tries++) {
+		err = try_write_vid_and_data(vol, lnum, vidb, buf, offset, len);
+		if (err != -EIO || !ubi->bad_allowed)
+			break;
 
-	if (len) {
-		err = ubi_io_write_data(ubi, buf, pnum, offset, len);
-		if (err) {
-			ubi_warn(ubi, "failed to write %d bytes at offset %d of LEB %d:%d, PEB %d",
-				 len, offset, vol_id, lnum, pnum);
-			up_read(&ubi->fm_eba_sem);
-			goto write_error;
-		}
+		/*
+		 * Fortunately, this is the first write operation to this
+		 * physical eraseblock, so just put it and request a new one.
+		 * We assume that if this physical eraseblock went bad, the
+		 * erase code will handle that.
+		 */
+		vid_hdr->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
+		ubi_msg(ubi, "try another PEB");
 	}
 
-	vol->eba_tbl[lnum] = pnum;
-	up_read(&ubi->fm_eba_sem);
-
-	leb_write_unlock(ubi, vol_id, lnum);
-	ubi_free_vid_hdr(ubi, vid_hdr);
-	return 0;
+	ubi_free_vid_buf(vidb);
 
-write_error:
-	if (err != -EIO || !ubi->bad_allowed) {
+out:
+	if (err)
 		ubi_ro_mode(ubi);
-		leb_write_unlock(ubi, vol_id, lnum);
-		ubi_free_vid_hdr(ubi, vid_hdr);
-		return err;
-	}
 
-	/*
-	 * Fortunately, this is the first write operation to this physical
-	 * eraseblock, so just put it and request a new one. We assume that if
-	 * this physical eraseblock went bad, the erase code will handle that.
-	 */
-	err = ubi_wl_put_peb(ubi, vol_id, lnum, pnum, 1);
-	if (err || ++tries > UBI_IO_RETRIES) {
-		ubi_ro_mode(ubi);
-		leb_write_unlock(ubi, vol_id, lnum);
-		ubi_free_vid_hdr(ubi, vid_hdr);
-		return err;
-	}
+	leb_write_unlock(ubi, vol_id, lnum);
 
-	vid_hdr->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
-	ubi_msg(ubi, "try another PEB");
-	goto retry;
+	return err;
 }
 
 /**
@@ -818,7 +1014,8 @@ write_error:
 int ubi_eba_write_leb_st(struct ubi_device *ubi, struct ubi_volume *vol,
 			 int lnum, const void *buf, int len, int used_ebs)
 {
-	int err, pnum, tries = 0, data_size = len, vol_id = vol->vol_id;
+	int err, tries, data_size = len, vol_id = vol->vol_id;
+	struct ubi_vid_io_buf *vidb;
 	struct ubi_vid_hdr *vid_hdr;
 	uint32_t crc;
 
@@ -831,15 +1028,15 @@ int ubi_eba_write_leb_st(struct ubi_device *ubi, struct ubi_volume *vol,
 	else
 		ubi_assert(!(len & (ubi->min_io_size - 1)));
 
-	vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_NOFS);
-	if (!vid_hdr)
+	vidb = ubi_alloc_vid_buf(ubi, GFP_NOFS);
+	if (!vidb)
 		return -ENOMEM;
 
+	vid_hdr = ubi_get_vid_hdr(vidb);
+
 	err = leb_write_lock(ubi, vol_id, lnum);
-	if (err) {
-		ubi_free_vid_hdr(ubi, vid_hdr);
-		return err;
-	}
+	if (err)
+		goto out;
 
 	vid_hdr->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
 	vid_hdr->vol_id = cpu_to_be32(vol_id);
@@ -853,66 +1050,26 @@ int ubi_eba_write_leb_st(struct ubi_device *ubi, struct ubi_volume *vol,
 	vid_hdr->used_ebs = cpu_to_be32(used_ebs);
 	vid_hdr->data_crc = cpu_to_be32(crc);
 
-retry:
-	pnum = ubi_wl_get_peb(ubi);
-	if (pnum < 0) {
-		ubi_free_vid_hdr(ubi, vid_hdr);
-		leb_write_unlock(ubi, vol_id, lnum);
-		up_read(&ubi->fm_eba_sem);
-		return pnum;
-	}
+	ubi_assert(vol->eba_tbl->entries[lnum].pnum < 0);
 
-	dbg_eba("write VID hdr and %d bytes at LEB %d:%d, PEB %d, used_ebs %d",
-		len, vol_id, lnum, pnum, used_ebs);
+	for (tries = 0; tries <= UBI_IO_RETRIES; tries++) {
+		err = try_write_vid_and_data(vol, lnum, vidb, buf, 0, len);
+		if (err != -EIO || !ubi->bad_allowed)
+			break;
 
-	err = ubi_io_write_vid_hdr(ubi, pnum, vid_hdr);
-	if (err) {
-		ubi_warn(ubi, "failed to write VID header to LEB %d:%d, PEB %d",
-			 vol_id, lnum, pnum);
-		up_read(&ubi->fm_eba_sem);
-		goto write_error;
+		vid_hdr->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
+		ubi_msg(ubi, "try another PEB");
 	}
 
-	err = ubi_io_write_data(ubi, buf, pnum, 0, len);
-	if (err) {
-		ubi_warn(ubi, "failed to write %d bytes of data to PEB %d",
-			 len, pnum);
-		up_read(&ubi->fm_eba_sem);
-		goto write_error;
-	}
-
-	ubi_assert(vol->eba_tbl[lnum] < 0);
-	vol->eba_tbl[lnum] = pnum;
-	up_read(&ubi->fm_eba_sem);
+	if (err)
+		ubi_ro_mode(ubi);
 
 	leb_write_unlock(ubi, vol_id, lnum);
-	ubi_free_vid_hdr(ubi, vid_hdr);
-	return 0;
-
-write_error:
-	if (err != -EIO || !ubi->bad_allowed) {
-		/*
-		 * This flash device does not admit of bad eraseblocks or
-		 * something nasty and unexpected happened. Switch to read-only
-		 * mode just in case.
-		 */
-		ubi_ro_mode(ubi);
-		leb_write_unlock(ubi, vol_id, lnum);
-		ubi_free_vid_hdr(ubi, vid_hdr);
-		return err;
-	}
 
-	err = ubi_wl_put_peb(ubi, vol_id, lnum, pnum, 1);
-	if (err || ++tries > UBI_IO_RETRIES) {
-		ubi_ro_mode(ubi);
-		leb_write_unlock(ubi, vol_id, lnum);
-		ubi_free_vid_hdr(ubi, vid_hdr);
-		return err;
-	}
+out:
+	ubi_free_vid_buf(vidb);
 
-	vid_hdr->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
-	ubi_msg(ubi, "try another PEB");
-	goto retry;
+	return err;
 }
 
 /*
@@ -935,7 +1092,8 @@ write_error:
 int ubi_eba_atomic_leb_change(struct ubi_device *ubi, struct ubi_volume *vol,
 			      int lnum, const void *buf, int len)
 {
-	int err, pnum, old_pnum, tries = 0, vol_id = vol->vol_id;
+	int err, tries, vol_id = vol->vol_id;
+	struct ubi_vid_io_buf *vidb;
 	struct ubi_vid_hdr *vid_hdr;
 	uint32_t crc;
 
@@ -953,10 +1111,12 @@ int ubi_eba_atomic_leb_change(struct ubi_device *ubi, struct ubi_volume *vol,
 		return ubi_eba_write_leb(ubi, vol, lnum, NULL, 0, 0);
 	}
 
-	vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_NOFS);
-	if (!vid_hdr)
+	vidb = ubi_alloc_vid_buf(ubi, GFP_NOFS);
+	if (!vidb)
 		return -ENOMEM;
 
+	vid_hdr = ubi_get_vid_hdr(vidb);
+
 	mutex_lock(&ubi->alc_mutex);
 	err = leb_write_lock(ubi, vol_id, lnum);
 	if (err)
@@ -974,70 +1134,31 @@ int ubi_eba_atomic_leb_change(struct ubi_device *ubi, struct ubi_volume *vol,
 	vid_hdr->copy_flag = 1;
 	vid_hdr->data_crc = cpu_to_be32(crc);
 
-retry:
-	pnum = ubi_wl_get_peb(ubi);
-	if (pnum < 0) {
-		err = pnum;
-		up_read(&ubi->fm_eba_sem);
-		goto out_leb_unlock;
-	}
-
-	dbg_eba("change LEB %d:%d, PEB %d, write VID hdr to PEB %d",
-		vol_id, lnum, vol->eba_tbl[lnum], pnum);
+	dbg_eba("change LEB %d:%d", vol_id, lnum);
 
-	err = ubi_io_write_vid_hdr(ubi, pnum, vid_hdr);
-	if (err) {
-		ubi_warn(ubi, "failed to write VID header to LEB %d:%d, PEB %d",
-			 vol_id, lnum, pnum);
-		up_read(&ubi->fm_eba_sem);
-		goto write_error;
-	}
+	for (tries = 0; tries <= UBI_IO_RETRIES; tries++) {
+		err = try_write_vid_and_data(vol, lnum, vidb, buf, 0, len);
+		if (err != -EIO || !ubi->bad_allowed)
+			break;
 
-	err = ubi_io_write_data(ubi, buf, pnum, 0, len);
-	if (err) {
-		ubi_warn(ubi, "failed to write %d bytes of data to PEB %d",
-			 len, pnum);
-		up_read(&ubi->fm_eba_sem);
-		goto write_error;
+		vid_hdr->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
+		ubi_msg(ubi, "try another PEB");
 	}
 
-	old_pnum = vol->eba_tbl[lnum];
-	vol->eba_tbl[lnum] = pnum;
-	up_read(&ubi->fm_eba_sem);
-
-	if (old_pnum >= 0) {
-		err = ubi_wl_put_peb(ubi, vol_id, lnum, old_pnum, 0);
-		if (err)
-			goto out_leb_unlock;
-	}
+	/*
+	 * This flash device does not admit of bad eraseblocks or
+	 * something nasty and unexpected happened. Switch to read-only
+	 * mode just in case.
+	 */
+	if (err)
+		ubi_ro_mode(ubi);
 
-out_leb_unlock:
 	leb_write_unlock(ubi, vol_id, lnum);
+
 out_mutex:
 	mutex_unlock(&ubi->alc_mutex);
-	ubi_free_vid_hdr(ubi, vid_hdr);
+	ubi_free_vid_buf(vidb);
 	return err;
-
-write_error:
-	if (err != -EIO || !ubi->bad_allowed) {
-		/*
-		 * This flash device does not admit of bad eraseblocks or
-		 * something nasty and unexpected happened. Switch to read-only
-		 * mode just in case.
-		 */
-		ubi_ro_mode(ubi);
-		goto out_leb_unlock;
-	}
-
-	err = ubi_wl_put_peb(ubi, vol_id, lnum, pnum, 1);
-	if (err || ++tries > UBI_IO_RETRIES) {
-		ubi_ro_mode(ubi);
-		goto out_leb_unlock;
-	}
-
-	vid_hdr->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
-	ubi_msg(ubi, "try another PEB");
-	goto retry;
 }
 
 /**
@@ -1082,12 +1203,15 @@ static int is_error_sane(int err)
  *   o a negative error code in case of failure.
  */
 int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
-		     struct ubi_vid_hdr *vid_hdr)
+		     struct ubi_vid_io_buf *vidb)
 {
 	int err, vol_id, lnum, data_size, aldata_size, idx;
+	struct ubi_vid_hdr *vid_hdr = ubi_get_vid_hdr(vidb);
 	struct ubi_volume *vol;
 	uint32_t crc;
 
+	ubi_assert(rwsem_is_locked(&ubi->fm_eba_sem));
+
 	vol_id = be32_to_cpu(vid_hdr->vol_id);
 	lnum = be32_to_cpu(vid_hdr->lnum);
 
@@ -1142,9 +1266,9 @@ int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
 	 * probably waiting on @ubi->move_mutex. No need to continue the work,
 	 * cancel it.
 	 */
-	if (vol->eba_tbl[lnum] != from) {
+	if (vol->eba_tbl->entries[lnum].pnum != from) {
 		dbg_wl("LEB %d:%d is no longer mapped to PEB %d, mapped to PEB %d, cancel",
-		       vol_id, lnum, from, vol->eba_tbl[lnum]);
+		       vol_id, lnum, from, vol->eba_tbl->entries[lnum].pnum);
 		err = MOVE_CANCEL_RACE;
 		goto out_unlock_leb;
 	}
@@ -1196,7 +1320,7 @@ int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
 	}
 	vid_hdr->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
 
-	err = ubi_io_write_vid_hdr(ubi, to, vid_hdr);
+	err = ubi_io_write_vid_hdr(ubi, to, vidb);
 	if (err) {
 		if (err == -EIO)
 			err = MOVE_TARGET_WR_ERR;
@@ -1206,7 +1330,7 @@ int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
 	cond_resched();
 
 	/* Read the VID header back and check if it was written correctly */
-	err = ubi_io_read_vid_hdr(ubi, to, vid_hdr, 1);
+	err = ubi_io_read_vid_hdr(ubi, to, vidb, 1);
 	if (err) {
 		if (err != UBI_IO_BITFLIPS) {
 			ubi_warn(ubi, "error %d while reading VID header back from PEB %d",
@@ -1229,10 +1353,8 @@ int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
 		cond_resched();
 	}
 
-	ubi_assert(vol->eba_tbl[lnum] == from);
-	down_read(&ubi->fm_eba_sem);
-	vol->eba_tbl[lnum] = to;
-	up_read(&ubi->fm_eba_sem);
+	ubi_assert(vol->eba_tbl->entries[lnum].pnum == from);
+	vol->eba_tbl->entries[lnum].pnum = to;
 
 out_unlock_buf:
 	mutex_unlock(&ubi->buf_mutex);
@@ -1388,7 +1510,7 @@ out_free:
  */
 int ubi_eba_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
 {
-	int i, j, err, num_volumes;
+	int i, err, num_volumes;
 	struct ubi_ainf_volume *av;
 	struct ubi_volume *vol;
 	struct ubi_ainf_peb *aeb;
@@ -1404,35 +1526,39 @@ int ubi_eba_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
 	num_volumes = ubi->vtbl_slots + UBI_INT_VOL_COUNT;
 
 	for (i = 0; i < num_volumes; i++) {
+		struct ubi_eba_table *tbl;
+
 		vol = ubi->volumes[i];
 		if (!vol)
 			continue;
 
 		cond_resched();
 
-		vol->eba_tbl = kmalloc(vol->reserved_pebs * sizeof(int),
-				       GFP_KERNEL);
-		if (!vol->eba_tbl) {
-			err = -ENOMEM;
+		tbl = ubi_eba_create_table(vol, vol->reserved_pebs);
+		if (IS_ERR(tbl)) {
+			err = PTR_ERR(tbl);
 			goto out_free;
 		}
 
-		for (j = 0; j < vol->reserved_pebs; j++)
-			vol->eba_tbl[j] = UBI_LEB_UNMAPPED;
+		ubi_eba_replace_table(vol, tbl);
 
 		av = ubi_find_av(ai, idx2vol_id(ubi, i));
 		if (!av)
 			continue;
 
 		ubi_rb_for_each_entry(rb, aeb, &av->root, u.rb) {
-			if (aeb->lnum >= vol->reserved_pebs)
+			if (aeb->lnum >= vol->reserved_pebs) {
 				/*
 				 * This may happen in case of an unclean reboot
 				 * during re-size.
 				 */
 				ubi_move_aeb_to_list(av, aeb, &ai->erase);
-			else
-				vol->eba_tbl[aeb->lnum] = aeb->pnum;
+			} else {
+				struct ubi_eba_entry *entry;
+
+				entry = &vol->eba_tbl->entries[aeb->lnum];
+				entry->pnum = aeb->pnum;
+			}
 		}
 	}
 
@@ -1469,8 +1595,7 @@ out_free:
 	for (i = 0; i < num_volumes; i++) {
 		if (!ubi->volumes[i])
 			continue;
-		kfree(ubi->volumes[i]->eba_tbl);
-		ubi->volumes[i]->eba_tbl = NULL;
+		ubi_eba_replace_table(ubi->volumes[i], NULL);
 	}
 	return err;
 }
diff --git a/drivers/mtd/ubi/fastmap-wl.c b/drivers/mtd/ubi/fastmap-wl.c
index 30d3999dddba..4f0bd6b4422a 100644
--- a/drivers/mtd/ubi/fastmap-wl.c
+++ b/drivers/mtd/ubi/fastmap-wl.c
@@ -262,6 +262,8 @@ static struct ubi_wl_entry *get_peb_for_wl(struct ubi_device *ubi)
 	struct ubi_fm_pool *pool = &ubi->fm_wl_pool;
 	int pnum;
 
+	ubi_assert(rwsem_is_locked(&ubi->fm_eba_sem));
+
 	if (pool->used == pool->size) {
 		/* We cannot update the fastmap here because this
 		 * function is called in atomic context.
@@ -303,7 +305,7 @@ int ubi_ensure_anchor_pebs(struct ubi_device *ubi)
 
 	wrk->anchor = 1;
 	wrk->func = &wear_leveling_worker;
-	schedule_ubi_work(ubi, wrk);
+	__schedule_ubi_work(ubi, wrk);
 	return 0;
 }
 
@@ -344,7 +346,7 @@ int ubi_wl_put_fm_peb(struct ubi_device *ubi, struct ubi_wl_entry *fm_e,
 	spin_unlock(&ubi->wl_lock);
 
 	vol_id = lnum ? UBI_FM_DATA_VOLUME_ID : UBI_FM_SB_VOLUME_ID;
-	return schedule_erase(ubi, e, vol_id, lnum, torture);
+	return schedule_erase(ubi, e, vol_id, lnum, torture, true);
 }
 
 /**
diff --git a/drivers/mtd/ubi/fastmap.c b/drivers/mtd/ubi/fastmap.c
index 48eb55f344eb..d6384d965788 100644
--- a/drivers/mtd/ubi/fastmap.c
+++ b/drivers/mtd/ubi/fastmap.c
@@ -110,21 +110,23 @@ size_t ubi_calc_fm_size(struct ubi_device *ubi)
  * Returns a new struct ubi_vid_hdr on success.
  * NULL indicates out of memory.
  */
-static struct ubi_vid_hdr *new_fm_vhdr(struct ubi_device *ubi, int vol_id)
+static struct ubi_vid_io_buf *new_fm_vbuf(struct ubi_device *ubi, int vol_id)
 {
-	struct ubi_vid_hdr *new;
+	struct ubi_vid_io_buf *new;
+	struct ubi_vid_hdr *vh;
 
-	new = ubi_zalloc_vid_hdr(ubi, GFP_KERNEL);
+	new = ubi_alloc_vid_buf(ubi, GFP_KERNEL);
 	if (!new)
 		goto out;
 
-	new->vol_type = UBI_VID_DYNAMIC;
-	new->vol_id = cpu_to_be32(vol_id);
+	vh = ubi_get_vid_hdr(new);
+	vh->vol_type = UBI_VID_DYNAMIC;
+	vh->vol_id = cpu_to_be32(vol_id);
 
 	/* UBI implementations without fastmap support have to delete the
 	 * fastmap.
 	 */
-	new->compat = UBI_COMPAT_DELETE;
+	vh->compat = UBI_COMPAT_DELETE;
 
 out:
 	return new;
@@ -145,12 +147,10 @@ static int add_aeb(struct ubi_attach_info *ai, struct list_head *list,
 {
 	struct ubi_ainf_peb *aeb;
 
-	aeb = kmem_cache_alloc(ai->aeb_slab_cache, GFP_KERNEL);
+	aeb = ubi_alloc_aeb(ai, pnum, ec);
 	if (!aeb)
 		return -ENOMEM;
 
-	aeb->pnum = pnum;
-	aeb->ec = ec;
 	aeb->lnum = -1;
 	aeb->scrub = scrub;
 	aeb->copy_flag = aeb->sqnum = 0;
@@ -186,40 +186,19 @@ static struct ubi_ainf_volume *add_vol(struct ubi_attach_info *ai, int vol_id,
 				       int last_eb_bytes)
 {
 	struct ubi_ainf_volume *av;
-	struct rb_node **p = &ai->volumes.rb_node, *parent = NULL;
-
-	while (*p) {
-		parent = *p;
-		av = rb_entry(parent, struct ubi_ainf_volume, rb);
-
-		if (vol_id > av->vol_id)
-			p = &(*p)->rb_left;
-		else if (vol_id < av->vol_id)
-			p = &(*p)->rb_right;
-		else
-			return ERR_PTR(-EINVAL);
-	}
 
-	av = kmalloc(sizeof(struct ubi_ainf_volume), GFP_KERNEL);
-	if (!av)
-		goto out;
+	av = ubi_add_av(ai, vol_id);
+	if (IS_ERR(av))
+		return av;
 
-	av->highest_lnum = av->leb_count = av->used_ebs = 0;
-	av->vol_id = vol_id;
 	av->data_pad = data_pad;
 	av->last_data_size = last_eb_bytes;
 	av->compat = 0;
 	av->vol_type = vol_type;
-	av->root = RB_ROOT;
 	if (av->vol_type == UBI_STATIC_VOLUME)
 		av->used_ebs = used_ebs;
 
 	dbg_bld("found volume (ID %i)", vol_id);
-
-	rb_link_node(&av->rb, parent, p);
-	rb_insert_color(&av->rb, &ai->volumes);
-
-out:
 	return av;
 }
 
@@ -297,7 +276,7 @@ static int update_vol(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		 */
 		if (aeb->pnum == new_aeb->pnum) {
 			ubi_assert(aeb->lnum == new_aeb->lnum);
-			kmem_cache_free(ai->aeb_slab_cache, new_aeb);
+			ubi_free_aeb(ai, new_aeb);
 
 			return 0;
 		}
@@ -308,13 +287,10 @@ static int update_vol(struct ubi_device *ubi, struct ubi_attach_info *ai,
 
 		/* new_aeb is newer */
 		if (cmp_res & 1) {
-			victim = kmem_cache_alloc(ai->aeb_slab_cache,
-				GFP_KERNEL);
+			victim = ubi_alloc_aeb(ai, aeb->ec, aeb->pnum);
 			if (!victim)
 				return -ENOMEM;
 
-			victim->ec = aeb->ec;
-			victim->pnum = aeb->pnum;
 			list_add_tail(&victim->u.list, &ai->erase);
 
 			if (av->highest_lnum == be32_to_cpu(new_vh->lnum))
@@ -328,7 +304,8 @@ static int update_vol(struct ubi_device *ubi, struct ubi_attach_info *ai,
 			aeb->pnum = new_aeb->pnum;
 			aeb->copy_flag = new_vh->copy_flag;
 			aeb->scrub = new_aeb->scrub;
-			kmem_cache_free(ai->aeb_slab_cache, new_aeb);
+			aeb->sqnum = new_aeb->sqnum;
+			ubi_free_aeb(ai, new_aeb);
 
 		/* new_aeb is older */
 		} else {
@@ -370,41 +347,24 @@ static int process_pool_aeb(struct ubi_device *ubi, struct ubi_attach_info *ai,
 			    struct ubi_vid_hdr *new_vh,
 			    struct ubi_ainf_peb *new_aeb)
 {
-	struct ubi_ainf_volume *av, *tmp_av = NULL;
-	struct rb_node **p = &ai->volumes.rb_node, *parent = NULL;
-	int found = 0;
+	int vol_id = be32_to_cpu(new_vh->vol_id);
+	struct ubi_ainf_volume *av;
 
-	if (be32_to_cpu(new_vh->vol_id) == UBI_FM_SB_VOLUME_ID ||
-		be32_to_cpu(new_vh->vol_id) == UBI_FM_DATA_VOLUME_ID) {
-		kmem_cache_free(ai->aeb_slab_cache, new_aeb);
+	if (vol_id == UBI_FM_SB_VOLUME_ID || vol_id == UBI_FM_DATA_VOLUME_ID) {
+		ubi_free_aeb(ai, new_aeb);
 
 		return 0;
 	}
 
 	/* Find the volume this SEB belongs to */
-	while (*p) {
-		parent = *p;
-		tmp_av = rb_entry(parent, struct ubi_ainf_volume, rb);
-
-		if (be32_to_cpu(new_vh->vol_id) > tmp_av->vol_id)
-			p = &(*p)->rb_left;
-		else if (be32_to_cpu(new_vh->vol_id) < tmp_av->vol_id)
-			p = &(*p)->rb_right;
-		else {
-			found = 1;
-			break;
-		}
-	}
-
-	if (found)
-		av = tmp_av;
-	else {
+	av = ubi_find_av(ai, vol_id);
+	if (!av) {
 		ubi_err(ubi, "orphaned volume in fastmap pool!");
-		kmem_cache_free(ai->aeb_slab_cache, new_aeb);
+		ubi_free_aeb(ai, new_aeb);
 		return UBI_BAD_FASTMAP;
 	}
 
-	ubi_assert(be32_to_cpu(new_vh->vol_id) == av->vol_id);
+	ubi_assert(vol_id == av->vol_id);
 
 	return update_vol(ubi, ai, av, new_vh, new_aeb);
 }
@@ -423,16 +383,12 @@ static void unmap_peb(struct ubi_attach_info *ai, int pnum)
 	struct rb_node *node, *node2;
 	struct ubi_ainf_peb *aeb;
 
-	for (node = rb_first(&ai->volumes); node; node = rb_next(node)) {
-		av = rb_entry(node, struct ubi_ainf_volume, rb);
-
-		for (node2 = rb_first(&av->root); node2;
-		     node2 = rb_next(node2)) {
-			aeb = rb_entry(node2, struct ubi_ainf_peb, u.rb);
+	ubi_rb_for_each_entry(node, av, &ai->volumes, rb) {
+		ubi_rb_for_each_entry(node2, aeb, &av->root, u.rb) {
 			if (aeb->pnum == pnum) {
 				rb_erase(&aeb->u.rb, &av->root);
 				av->leb_count--;
-				kmem_cache_free(ai->aeb_slab_cache, aeb);
+				ubi_free_aeb(ai, aeb);
 				return;
 			}
 		}
@@ -455,6 +411,7 @@ static int scan_pool(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		     __be32 *pebs, int pool_size, unsigned long long *max_sqnum,
 		     struct list_head *free)
 {
+	struct ubi_vid_io_buf *vb;
 	struct ubi_vid_hdr *vh;
 	struct ubi_ec_hdr *ech;
 	struct ubi_ainf_peb *new_aeb;
@@ -464,12 +421,14 @@ static int scan_pool(struct ubi_device *ubi, struct ubi_attach_info *ai,
 	if (!ech)
 		return -ENOMEM;
 
-	vh = ubi_zalloc_vid_hdr(ubi, GFP_KERNEL);
-	if (!vh) {
+	vb = ubi_alloc_vid_buf(ubi, GFP_KERNEL);
+	if (!vb) {
 		kfree(ech);
 		return -ENOMEM;
 	}
 
+	vh = ubi_get_vid_hdr(vb);
+
 	dbg_bld("scanning fastmap pool: size = %i", pool_size);
 
 	/*
@@ -510,15 +469,16 @@ static int scan_pool(struct ubi_device *ubi, struct ubi_attach_info *ai,
 			goto out;
 		}
 
-		err = ubi_io_read_vid_hdr(ubi, pnum, vh, 0);
+		err = ubi_io_read_vid_hdr(ubi, pnum, vb, 0);
 		if (err == UBI_IO_FF || err == UBI_IO_FF_BITFLIPS) {
 			unsigned long long ec = be64_to_cpu(ech->ec);
 			unmap_peb(ai, pnum);
 			dbg_bld("Adding PEB to free: %i", pnum);
+
 			if (err == UBI_IO_FF_BITFLIPS)
-				add_aeb(ai, free, pnum, ec, 1);
-			else
-				add_aeb(ai, free, pnum, ec, 0);
+				scrub = 1;
+
+			add_aeb(ai, free, pnum, ec, scrub);
 			continue;
 		} else if (err == 0 || err == UBI_IO_BITFLIPS) {
 			dbg_bld("Found non empty PEB:%i in pool", pnum);
@@ -526,15 +486,12 @@ static int scan_pool(struct ubi_device *ubi, struct ubi_attach_info *ai,
 			if (err == UBI_IO_BITFLIPS)
 				scrub = 1;
 
-			new_aeb = kmem_cache_alloc(ai->aeb_slab_cache,
-						   GFP_KERNEL);
+			new_aeb = ubi_alloc_aeb(ai, pnum, be64_to_cpu(ech->ec));
 			if (!new_aeb) {
 				ret = -ENOMEM;
 				goto out;
 			}
 
-			new_aeb->ec = be64_to_cpu(ech->ec);
-			new_aeb->pnum = pnum;
 			new_aeb->lnum = be32_to_cpu(vh->lnum);
 			new_aeb->sqnum = be64_to_cpu(vh->sqnum);
 			new_aeb->copy_flag = vh->copy_flag;
@@ -558,7 +515,7 @@ static int scan_pool(struct ubi_device *ubi, struct ubi_attach_info *ai,
 	}
 
 out:
-	ubi_free_vid_hdr(ubi, vh);
+	ubi_free_vid_buf(vb);
 	kfree(ech);
 	return ret;
 }
@@ -841,11 +798,11 @@ fail_bad:
 fail:
 	list_for_each_entry_safe(tmp_aeb, _tmp_aeb, &used, u.list) {
 		list_del(&tmp_aeb->u.list);
-		kmem_cache_free(ai->aeb_slab_cache, tmp_aeb);
+		ubi_free_aeb(ai, tmp_aeb);
 	}
 	list_for_each_entry_safe(tmp_aeb, _tmp_aeb, &free, u.list) {
 		list_del(&tmp_aeb->u.list);
-		kmem_cache_free(ai->aeb_slab_cache, tmp_aeb);
+		ubi_free_aeb(ai, tmp_aeb);
 	}
 
 	return ret;
@@ -886,6 +843,7 @@ int ubi_scan_fastmap(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		     struct ubi_attach_info *scan_ai)
 {
 	struct ubi_fm_sb *fmsb, *fmsb2;
+	struct ubi_vid_io_buf *vb;
 	struct ubi_vid_hdr *vh;
 	struct ubi_ec_hdr *ech;
 	struct ubi_fastmap_layout *fm;
@@ -919,7 +877,7 @@ int ubi_scan_fastmap(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		goto out;
 	}
 
-	ret = ubi_io_read(ubi, fmsb, fm_anchor, ubi->leb_start, sizeof(*fmsb));
+	ret = ubi_io_read_data(ubi, fmsb, fm_anchor, 0, sizeof(*fmsb));
 	if (ret && ret != UBI_IO_BITFLIPS)
 		goto free_fm_sb;
 	else if (ret == UBI_IO_BITFLIPS)
@@ -961,12 +919,14 @@ int ubi_scan_fastmap(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		goto free_fm_sb;
 	}
 
-	vh = ubi_zalloc_vid_hdr(ubi, GFP_KERNEL);
-	if (!vh) {
+	vb = ubi_alloc_vid_buf(ubi, GFP_KERNEL);
+	if (!vb) {
 		ret = -ENOMEM;
 		goto free_hdr;
 	}
 
+	vh = ubi_get_vid_hdr(vb);
+
 	for (i = 0; i < used_blocks; i++) {
 		int image_seq;
 
@@ -1009,7 +969,7 @@ int ubi_scan_fastmap(struct ubi_device *ubi, struct ubi_attach_info *ai,
 			goto free_hdr;
 		}
 
-		ret = ubi_io_read_vid_hdr(ubi, pnum, vh, 0);
+		ret = ubi_io_read_vid_hdr(ubi, pnum, vb, 0);
 		if (ret && ret != UBI_IO_BITFLIPS) {
 			ubi_err(ubi, "unable to read fastmap block# %i (PEB: %i)",
 				i, pnum);
@@ -1037,8 +997,8 @@ int ubi_scan_fastmap(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		if (sqnum < be64_to_cpu(vh->sqnum))
 			sqnum = be64_to_cpu(vh->sqnum);
 
-		ret = ubi_io_read(ubi, ubi->fm_buf + (ubi->leb_size * i), pnum,
-				  ubi->leb_start, ubi->leb_size);
+		ret = ubi_io_read_data(ubi, ubi->fm_buf + (ubi->leb_size * i),
+				       pnum, 0, ubi->leb_size);
 		if (ret && ret != UBI_IO_BITFLIPS) {
 			ubi_err(ubi, "unable to read fastmap block# %i (PEB: %i, "
 				"err: %i)", i, pnum, ret);
@@ -1099,7 +1059,7 @@ int ubi_scan_fastmap(struct ubi_device *ubi, struct ubi_attach_info *ai,
 	ubi->fm_disabled = 0;
 	ubi->fast_attach = 1;
 
-	ubi_free_vid_hdr(ubi, vh);
+	ubi_free_vid_buf(vb);
 	kfree(ech);
 out:
 	up_write(&ubi->fm_protect);
@@ -1108,7 +1068,7 @@ out:
 	return ret;
 
 free_hdr:
-	ubi_free_vid_hdr(ubi, vh);
+	ubi_free_vid_buf(vb);
 	kfree(ech);
 free_fm_sb:
 	kfree(fmsb);
@@ -1136,6 +1096,7 @@ static int ubi_write_fastmap(struct ubi_device *ubi,
 	struct ubi_fm_eba *feba;
 	struct ubi_wl_entry *wl_e;
 	struct ubi_volume *vol;
+	struct ubi_vid_io_buf *avbuf, *dvbuf;
 	struct ubi_vid_hdr *avhdr, *dvhdr;
 	struct ubi_work *ubi_wrk;
 	struct rb_node *tmp_rb;
@@ -1146,18 +1107,21 @@ static int ubi_write_fastmap(struct ubi_device *ubi,
 	fm_raw = ubi->fm_buf;
 	memset(ubi->fm_buf, 0, ubi->fm_size);
 
-	avhdr = new_fm_vhdr(ubi, UBI_FM_SB_VOLUME_ID);
-	if (!avhdr) {
+	avbuf = new_fm_vbuf(ubi, UBI_FM_SB_VOLUME_ID);
+	if (!avbuf) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
-	dvhdr = new_fm_vhdr(ubi, UBI_FM_DATA_VOLUME_ID);
-	if (!dvhdr) {
+	dvbuf = new_fm_vbuf(ubi, UBI_FM_DATA_VOLUME_ID);
+	if (!dvbuf) {
 		ret = -ENOMEM;
 		goto out_kfree;
 	}
 
+	avhdr = ubi_get_vid_hdr(avbuf);
+	dvhdr = ubi_get_vid_hdr(dvbuf);
+
 	seen_pebs = init_seen(ubi);
 	if (IS_ERR(seen_pebs)) {
 		ret = PTR_ERR(seen_pebs);
@@ -1306,8 +1270,12 @@ static int ubi_write_fastmap(struct ubi_device *ubi,
 		fm_pos += sizeof(*feba) + (sizeof(__be32) * vol->reserved_pebs);
 		ubi_assert(fm_pos <= ubi->fm_size);
 
-		for (j = 0; j < vol->reserved_pebs; j++)
-			feba->pnum[j] = cpu_to_be32(vol->eba_tbl[j]);
+		for (j = 0; j < vol->reserved_pebs; j++) {
+			struct ubi_eba_leb_desc ldesc;
+
+			ubi_eba_get_ldesc(vol, j, &ldesc);
+			feba->pnum[j] = cpu_to_be32(ldesc.pnum);
+		}
 
 		feba->reserved_pebs = cpu_to_be32(j);
 		feba->magic = cpu_to_be32(UBI_FM_EBA_MAGIC);
@@ -1322,7 +1290,7 @@ static int ubi_write_fastmap(struct ubi_device *ubi,
 	spin_unlock(&ubi->volumes_lock);
 
 	dbg_bld("writing fastmap SB to PEB %i", new_fm->e[0]->pnum);
-	ret = ubi_io_write_vid_hdr(ubi, new_fm->e[0]->pnum, avhdr);
+	ret = ubi_io_write_vid_hdr(ubi, new_fm->e[0]->pnum, avbuf);
 	if (ret) {
 		ubi_err(ubi, "unable to write vid_hdr to fastmap SB!");
 		goto out_kfree;
@@ -1343,7 +1311,7 @@ static int ubi_write_fastmap(struct ubi_device *ubi,
 		dvhdr->lnum = cpu_to_be32(i);
 		dbg_bld("writing fastmap data to PEB %i sqnum %llu",
 			new_fm->e[i]->pnum, be64_to_cpu(dvhdr->sqnum));
-		ret = ubi_io_write_vid_hdr(ubi, new_fm->e[i]->pnum, dvhdr);
+		ret = ubi_io_write_vid_hdr(ubi, new_fm->e[i]->pnum, dvbuf);
 		if (ret) {
 			ubi_err(ubi, "unable to write vid_hdr to PEB %i!",
 				new_fm->e[i]->pnum);
@@ -1352,8 +1320,8 @@ static int ubi_write_fastmap(struct ubi_device *ubi,
 	}
 
 	for (i = 0; i < new_fm->used_blocks; i++) {
-		ret = ubi_io_write(ubi, fm_raw + (i * ubi->leb_size),
-			new_fm->e[i]->pnum, ubi->leb_start, ubi->leb_size);
+		ret = ubi_io_write_data(ubi, fm_raw + (i * ubi->leb_size),
+					new_fm->e[i]->pnum, 0, ubi->leb_size);
 		if (ret) {
 			ubi_err(ubi, "unable to write fastmap to PEB %i!",
 				new_fm->e[i]->pnum);
@@ -1368,8 +1336,8 @@ static int ubi_write_fastmap(struct ubi_device *ubi,
 	dbg_bld("fastmap written!");
 
 out_kfree:
-	ubi_free_vid_hdr(ubi, avhdr);
-	ubi_free_vid_hdr(ubi, dvhdr);
+	ubi_free_vid_buf(avbuf);
+	ubi_free_vid_buf(dvbuf);
 	free_seen(seen_pebs);
 out:
 	return ret;
@@ -1439,7 +1407,8 @@ static int invalidate_fastmap(struct ubi_device *ubi)
 	int ret;
 	struct ubi_fastmap_layout *fm;
 	struct ubi_wl_entry *e;
-	struct ubi_vid_hdr *vh = NULL;
+	struct ubi_vid_io_buf *vb = NULL;
+	struct ubi_vid_hdr *vh;
 
 	if (!ubi->fm)
 		return 0;
@@ -1451,10 +1420,12 @@ static int invalidate_fastmap(struct ubi_device *ubi)
 	if (!fm)
 		goto out;
 
-	vh = new_fm_vhdr(ubi, UBI_FM_SB_VOLUME_ID);
-	if (!vh)
+	vb = new_fm_vbuf(ubi, UBI_FM_SB_VOLUME_ID);
+	if (!vb)
 		goto out_free_fm;
 
+	vh = ubi_get_vid_hdr(vb);
+
 	ret = -ENOSPC;
 	e = ubi_wl_get_fm_peb(ubi, 1);
 	if (!e)
@@ -1465,7 +1436,7 @@ static int invalidate_fastmap(struct ubi_device *ubi)
 	 * to scanning mode.
 	 */
 	vh->sqnum = cpu_to_be64(ubi_next_sqnum(ubi));
-	ret = ubi_io_write_vid_hdr(ubi, e->pnum, vh);
+	ret = ubi_io_write_vid_hdr(ubi, e->pnum, vb);
 	if (ret < 0) {
 		ubi_wl_put_fm_peb(ubi, e, 0, 0);
 		goto out_free_fm;
@@ -1477,7 +1448,7 @@ static int invalidate_fastmap(struct ubi_device *ubi)
 	ubi->fm = fm;
 
 out:
-	ubi_free_vid_hdr(ubi, vh);
+	ubi_free_vid_buf(vb);
 	return ret;
 
 out_free_fm:
@@ -1522,22 +1493,30 @@ int ubi_update_fastmap(struct ubi_device *ubi)
 	struct ubi_wl_entry *tmp_e;
 
 	down_write(&ubi->fm_protect);
+	down_write(&ubi->work_sem);
+	down_write(&ubi->fm_eba_sem);
 
 	ubi_refill_pools(ubi);
 
 	if (ubi->ro_mode || ubi->fm_disabled) {
+		up_write(&ubi->fm_eba_sem);
+		up_write(&ubi->work_sem);
 		up_write(&ubi->fm_protect);
 		return 0;
 	}
 
 	ret = ubi_ensure_anchor_pebs(ubi);
 	if (ret) {
+		up_write(&ubi->fm_eba_sem);
+		up_write(&ubi->work_sem);
 		up_write(&ubi->fm_protect);
 		return ret;
 	}
 
 	new_fm = kzalloc(sizeof(*new_fm), GFP_KERNEL);
 	if (!new_fm) {
+		up_write(&ubi->fm_eba_sem);
+		up_write(&ubi->work_sem);
 		up_write(&ubi->fm_protect);
 		return -ENOMEM;
 	}
@@ -1646,16 +1625,14 @@ int ubi_update_fastmap(struct ubi_device *ubi)
 		new_fm->e[0] = tmp_e;
 	}
 
-	down_write(&ubi->work_sem);
-	down_write(&ubi->fm_eba_sem);
 	ret = ubi_write_fastmap(ubi, new_fm);
-	up_write(&ubi->fm_eba_sem);
-	up_write(&ubi->work_sem);
 
 	if (ret)
 		goto err;
 
 out_unlock:
+	up_write(&ubi->fm_eba_sem);
+	up_write(&ubi->work_sem);
 	up_write(&ubi->fm_protect);
 	kfree(old_fm);
 	return ret;
diff --git a/drivers/mtd/ubi/io.c b/drivers/mtd/ubi/io.c
index ff8cafe1e5cd..b6fb8f945c21 100644
--- a/drivers/mtd/ubi/io.c
+++ b/drivers/mtd/ubi/io.c
@@ -502,6 +502,7 @@ static int nor_erase_prepare(struct ubi_device *ubi, int pnum)
 	loff_t addr;
 	uint32_t data = 0;
 	struct ubi_ec_hdr ec_hdr;
+	struct ubi_vid_io_buf vidb;
 
 	/*
 	 * Note, we cannot generally define VID header buffers on stack,
@@ -528,7 +529,10 @@ static int nor_erase_prepare(struct ubi_device *ubi, int pnum)
 			goto error;
 	}
 
-	err = ubi_io_read_vid_hdr(ubi, pnum, &vid_hdr, 0);
+	ubi_init_vid_buf(ubi, &vidb, &vid_hdr);
+	ubi_assert(&vid_hdr == ubi_get_vid_hdr(&vidb));
+
+	err = ubi_io_read_vid_hdr(ubi, pnum, &vidb, 0);
 	if (err != UBI_IO_BAD_HDR_EBADMSG && err != UBI_IO_BAD_HDR &&
 	    err != UBI_IO_FF){
 		addr += ubi->vid_hdr_aloffset;
@@ -995,12 +999,11 @@ bad:
  * ubi_io_read_vid_hdr - read and check a volume identifier header.
  * @ubi: UBI device description object
  * @pnum: physical eraseblock number to read from
- * @vid_hdr: &struct ubi_vid_hdr object where to store the read volume
- * identifier header
+ * @vidb: the volume identifier buffer to store data in
  * @verbose: be verbose if the header is corrupted or wasn't found
  *
  * This function reads the volume identifier header from physical eraseblock
- * @pnum and stores it in @vid_hdr. It also checks CRC checksum of the read
+ * @pnum and stores it in @vidb. It also checks CRC checksum of the read
  * volume identifier header. The error codes are the same as in
  * 'ubi_io_read_ec_hdr()'.
  *
@@ -1008,16 +1011,16 @@ bad:
  * 'ubi_io_read_ec_hdr()', so refer commentaries in 'ubi_io_read_ec_hdr()'.
  */
 int ubi_io_read_vid_hdr(struct ubi_device *ubi, int pnum,
-			struct ubi_vid_hdr *vid_hdr, int verbose)
+			struct ubi_vid_io_buf *vidb, int verbose)
 {
 	int err, read_err;
 	uint32_t crc, magic, hdr_crc;
-	void *p;
+	struct ubi_vid_hdr *vid_hdr = ubi_get_vid_hdr(vidb);
+	void *p = vidb->buffer;
 
 	dbg_io("read VID header from PEB %d", pnum);
 	ubi_assert(pnum >= 0 &&  pnum < ubi->peb_count);
 
-	p = (char *)vid_hdr - ubi->vid_hdr_shift;
 	read_err = ubi_io_read(ubi, p, pnum, ubi->vid_hdr_aloffset,
 			  ubi->vid_hdr_shift + UBI_VID_HDR_SIZE);
 	if (read_err && read_err != UBI_IO_BITFLIPS && !mtd_is_eccerr(read_err))
@@ -1080,23 +1083,24 @@ int ubi_io_read_vid_hdr(struct ubi_device *ubi, int pnum,
  * ubi_io_write_vid_hdr - write a volume identifier header.
  * @ubi: UBI device description object
  * @pnum: the physical eraseblock number to write to
- * @vid_hdr: the volume identifier header to write
+ * @vidb: the volume identifier buffer to write
  *
  * This function writes the volume identifier header described by @vid_hdr to
  * physical eraseblock @pnum. This function automatically fills the
- * @vid_hdr->magic and the @vid_hdr->version fields, as well as calculates
- * header CRC checksum and stores it at vid_hdr->hdr_crc.
+ * @vidb->hdr->magic and the @vidb->hdr->version fields, as well as calculates
+ * header CRC checksum and stores it at vidb->hdr->hdr_crc.
  *
  * This function returns zero in case of success and a negative error code in
  * case of failure. If %-EIO is returned, the physical eraseblock probably went
  * bad.
  */
 int ubi_io_write_vid_hdr(struct ubi_device *ubi, int pnum,
-			 struct ubi_vid_hdr *vid_hdr)
+			 struct ubi_vid_io_buf *vidb)
 {
+	struct ubi_vid_hdr *vid_hdr = ubi_get_vid_hdr(vidb);
 	int err;
 	uint32_t crc;
-	void *p;
+	void *p = vidb->buffer;
 
 	dbg_io("write VID header to PEB %d", pnum);
 	ubi_assert(pnum >= 0 &&  pnum < ubi->peb_count);
@@ -1117,7 +1121,6 @@ int ubi_io_write_vid_hdr(struct ubi_device *ubi, int pnum,
 	if (ubi_dbg_power_cut(ubi, POWER_CUT_VID_WRITE))
 		return -EROFS;
 
-	p = (char *)vid_hdr - ubi->vid_hdr_shift;
 	err = ubi_io_write(ubi, p, pnum, ubi->vid_hdr_aloffset,
 			   ubi->vid_hdr_alsize);
 	return err;
@@ -1283,17 +1286,19 @@ static int self_check_peb_vid_hdr(const struct ubi_device *ubi, int pnum)
 {
 	int err;
 	uint32_t crc, hdr_crc;
+	struct ubi_vid_io_buf *vidb;
 	struct ubi_vid_hdr *vid_hdr;
 	void *p;
 
 	if (!ubi_dbg_chk_io(ubi))
 		return 0;
 
-	vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_NOFS);
-	if (!vid_hdr)
+	vidb = ubi_alloc_vid_buf(ubi, GFP_NOFS);
+	if (!vidb)
 		return -ENOMEM;
 
-	p = (char *)vid_hdr - ubi->vid_hdr_shift;
+	vid_hdr = ubi_get_vid_hdr(vidb);
+	p = vidb->buffer;
 	err = ubi_io_read(ubi, p, pnum, ubi->vid_hdr_aloffset,
 			  ubi->vid_hdr_alsize);
 	if (err && err != UBI_IO_BITFLIPS && !mtd_is_eccerr(err))
@@ -1314,7 +1319,7 @@ static int self_check_peb_vid_hdr(const struct ubi_device *ubi, int pnum)
 	err = self_check_vid_hdr(ubi, pnum, vid_hdr);
 
 exit:
-	ubi_free_vid_hdr(ubi, vid_hdr);
+	ubi_free_vid_buf(vidb);
 	return err;
 }
 
diff --git a/drivers/mtd/ubi/kapi.c b/drivers/mtd/ubi/kapi.c
index a9e2cef7c95c..88b1897aeb40 100644
--- a/drivers/mtd/ubi/kapi.c
+++ b/drivers/mtd/ubi/kapi.c
@@ -538,7 +538,7 @@ int ubi_leb_write(struct ubi_volume_desc *desc, int lnum, const void *buf,
 	if (desc->mode == UBI_READONLY || vol->vol_type == UBI_STATIC_VOLUME)
 		return -EROFS;
 
-	if (lnum < 0 || lnum >= vol->reserved_pebs || offset < 0 || len < 0 ||
+	if (!ubi_leb_valid(vol, lnum) || offset < 0 || len < 0 ||
 	    offset + len > vol->usable_leb_size ||
 	    offset & (ubi->min_io_size - 1) || len & (ubi->min_io_size - 1))
 		return -EINVAL;
@@ -583,7 +583,7 @@ int ubi_leb_change(struct ubi_volume_desc *desc, int lnum, const void *buf,
 	if (desc->mode == UBI_READONLY || vol->vol_type == UBI_STATIC_VOLUME)
 		return -EROFS;
 
-	if (lnum < 0 || lnum >= vol->reserved_pebs || len < 0 ||
+	if (!ubi_leb_valid(vol, lnum) || len < 0 ||
 	    len > vol->usable_leb_size || len & (ubi->min_io_size - 1))
 		return -EINVAL;
 
@@ -620,7 +620,7 @@ int ubi_leb_erase(struct ubi_volume_desc *desc, int lnum)
 	if (desc->mode == UBI_READONLY || vol->vol_type == UBI_STATIC_VOLUME)
 		return -EROFS;
 
-	if (lnum < 0 || lnum >= vol->reserved_pebs)
+	if (!ubi_leb_valid(vol, lnum))
 		return -EINVAL;
 
 	if (vol->upd_marker)
@@ -680,7 +680,7 @@ int ubi_leb_unmap(struct ubi_volume_desc *desc, int lnum)
 	if (desc->mode == UBI_READONLY || vol->vol_type == UBI_STATIC_VOLUME)
 		return -EROFS;
 
-	if (lnum < 0 || lnum >= vol->reserved_pebs)
+	if (!ubi_leb_valid(vol, lnum))
 		return -EINVAL;
 
 	if (vol->upd_marker)
@@ -716,13 +716,13 @@ int ubi_leb_map(struct ubi_volume_desc *desc, int lnum)
 	if (desc->mode == UBI_READONLY || vol->vol_type == UBI_STATIC_VOLUME)
 		return -EROFS;
 
-	if (lnum < 0 || lnum >= vol->reserved_pebs)
+	if (!ubi_leb_valid(vol, lnum))
 		return -EINVAL;
 
 	if (vol->upd_marker)
 		return -EBADF;
 
-	if (vol->eba_tbl[lnum] >= 0)
+	if (ubi_eba_is_mapped(vol, lnum))
 		return -EBADMSG;
 
 	return ubi_eba_write_leb(ubi, vol, lnum, NULL, 0, 0);
@@ -751,13 +751,13 @@ int ubi_is_mapped(struct ubi_volume_desc *desc, int lnum)
 
 	dbg_gen("test LEB %d:%d", vol->vol_id, lnum);
 
-	if (lnum < 0 || lnum >= vol->reserved_pebs)
+	if (!ubi_leb_valid(vol, lnum))
 		return -EINVAL;
 
 	if (vol->upd_marker)
 		return -EBADF;
 
-	return vol->eba_tbl[lnum] >= 0;
+	return ubi_eba_is_mapped(vol, lnum);
 }
 EXPORT_SYMBOL_GPL(ubi_is_mapped);
 
diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h
index b616a115c9d3..697dbcba7371 100644
--- a/drivers/mtd/ubi/ubi.h
+++ b/drivers/mtd/ubi/ubi.h
@@ -167,6 +167,17 @@ enum {
 };
 
 /**
+ * struct ubi_vid_io_buf - VID buffer used to read/write VID info to/from the
+ *			   flash.
+ * @hdr: a pointer to the VID header stored in buffer
+ * @buffer: underlying buffer
+ */
+struct ubi_vid_io_buf {
+	struct ubi_vid_hdr *hdr;
+	void *buffer;
+};
+
+/**
  * struct ubi_wl_entry - wear-leveling entry.
  * @u.rb: link in the corresponding (free/used) RB-tree
  * @u.list: link in the protection queue
@@ -267,6 +278,21 @@ struct ubi_fm_pool {
 };
 
 /**
+ * struct ubi_eba_leb_desc - EBA logical eraseblock descriptor
+ * @lnum: the logical eraseblock number
+ * @pnum: the physical eraseblock where the LEB can be found
+ *
+ * This structure is here to hide EBA's internal from other part of the
+ * UBI implementation.
+ *
+ * One can query the position of a LEB by calling ubi_eba_get_ldesc().
+ */
+struct ubi_eba_leb_desc {
+	int lnum;
+	int pnum;
+};
+
+/**
  * struct ubi_volume - UBI volume description data structure.
  * @dev: device object to make use of the the Linux device model
  * @cdev: character device object to create character device
@@ -344,7 +370,7 @@ struct ubi_volume {
 	long long upd_received;
 	void *upd_buf;
 
-	int *eba_tbl;
+	struct ubi_eba_table *eba_tbl;
 	unsigned int checked:1;
 	unsigned int corrupted:1;
 	unsigned int upd_marker:1;
@@ -724,6 +750,8 @@ struct ubi_ainf_volume {
  * @ec_sum: a temporary variable used when calculating @mean_ec
  * @ec_count: a temporary variable used when calculating @mean_ec
  * @aeb_slab_cache: slab cache for &struct ubi_ainf_peb objects
+ * @ech: temporary EC header. Only available during scan
+ * @vidh: temporary VID buffer. Only available during scan
  *
  * This data structure contains the result of attaching an MTD device and may
  * be used by other UBI sub-systems to build final UBI data structures, further
@@ -752,6 +780,8 @@ struct ubi_attach_info {
 	uint64_t ec_sum;
 	int ec_count;
 	struct kmem_cache *aeb_slab_cache;
+	struct ubi_ec_hdr *ech;
+	struct ubi_vid_io_buf *vidb;
 };
 
 /**
@@ -792,8 +822,12 @@ extern struct mutex ubi_devices_mutex;
 extern struct blocking_notifier_head ubi_notifiers;
 
 /* attach.c */
+struct ubi_ainf_peb *ubi_alloc_aeb(struct ubi_attach_info *ai, int pnum,
+				   int ec);
+void ubi_free_aeb(struct ubi_attach_info *ai, struct ubi_ainf_peb *aeb);
 int ubi_add_to_av(struct ubi_device *ubi, struct ubi_attach_info *ai, int pnum,
 		  int ec, const struct ubi_vid_hdr *vid_hdr, int bitflips);
+struct ubi_ainf_volume *ubi_add_av(struct ubi_attach_info *ai, int vol_id);
 struct ubi_ainf_volume *ubi_find_av(const struct ubi_attach_info *ai,
 				    int vol_id);
 void ubi_remove_av(struct ubi_attach_info *ai, struct ubi_ainf_volume *av);
@@ -835,7 +869,21 @@ void ubi_update_reserved(struct ubi_device *ubi);
 void ubi_calculate_reserved(struct ubi_device *ubi);
 int ubi_check_pattern(const void *buf, uint8_t patt, int size);
 
+static inline bool ubi_leb_valid(struct ubi_volume *vol, int lnum)
+{
+	return lnum >= 0 && lnum < vol->reserved_pebs;
+}
+
 /* eba.c */
+struct ubi_eba_table *ubi_eba_create_table(struct ubi_volume *vol,
+					   int nentries);
+void ubi_eba_destroy_table(struct ubi_eba_table *tbl);
+void ubi_eba_copy_table(struct ubi_volume *vol, struct ubi_eba_table *dst,
+			int nentries);
+void ubi_eba_replace_table(struct ubi_volume *vol, struct ubi_eba_table *tbl);
+void ubi_eba_get_ldesc(struct ubi_volume *vol, int lnum,
+		       struct ubi_eba_leb_desc *ldesc);
+bool ubi_eba_is_mapped(struct ubi_volume *vol, int lnum);
 int ubi_eba_unmap_leb(struct ubi_device *ubi, struct ubi_volume *vol,
 		      int lnum);
 int ubi_eba_read_leb(struct ubi_device *ubi, struct ubi_volume *vol, int lnum,
@@ -850,7 +898,7 @@ int ubi_eba_write_leb_st(struct ubi_device *ubi, struct ubi_volume *vol,
 int ubi_eba_atomic_leb_change(struct ubi_device *ubi, struct ubi_volume *vol,
 			      int lnum, const void *buf, int len);
 int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
-		     struct ubi_vid_hdr *vid_hdr);
+		     struct ubi_vid_io_buf *vidb);
 int ubi_eba_init(struct ubi_device *ubi, struct ubi_attach_info *ai);
 unsigned long long ubi_next_sqnum(struct ubi_device *ubi);
 int self_check_eba(struct ubi_device *ubi, struct ubi_attach_info *ai_fastmap,
@@ -885,9 +933,9 @@ int ubi_io_read_ec_hdr(struct ubi_device *ubi, int pnum,
 int ubi_io_write_ec_hdr(struct ubi_device *ubi, int pnum,
 			struct ubi_ec_hdr *ec_hdr);
 int ubi_io_read_vid_hdr(struct ubi_device *ubi, int pnum,
-			struct ubi_vid_hdr *vid_hdr, int verbose);
+			struct ubi_vid_io_buf *vidb, int verbose);
 int ubi_io_write_vid_hdr(struct ubi_device *ubi, int pnum,
-			 struct ubi_vid_hdr *vid_hdr);
+			 struct ubi_vid_io_buf *vidb);
 
 /* build.c */
 int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
@@ -1008,44 +1056,68 @@ static inline void ubi_move_aeb_to_list(struct ubi_ainf_volume *av,
 }
 
 /**
- * ubi_zalloc_vid_hdr - allocate a volume identifier header object.
- * @ubi: UBI device description object
- * @gfp_flags: GFP flags to allocate with
- *
- * This function returns a pointer to the newly allocated and zero-filled
- * volume identifier header object in case of success and %NULL in case of
- * failure.
+ * ubi_init_vid_buf - Initialize a VID buffer
+ * @ubi: the UBI device
+ * @vidb: the VID buffer to initialize
+ * @buf: the underlying buffer
  */
-static inline struct ubi_vid_hdr *
-ubi_zalloc_vid_hdr(const struct ubi_device *ubi, gfp_t gfp_flags)
+static inline void ubi_init_vid_buf(const struct ubi_device *ubi,
+				    struct ubi_vid_io_buf *vidb,
+				    void *buf)
 {
-	void *vid_hdr;
+	if (buf)
+		memset(buf, 0, ubi->vid_hdr_alsize);
 
-	vid_hdr = kzalloc(ubi->vid_hdr_alsize, gfp_flags);
-	if (!vid_hdr)
+	vidb->buffer = buf;
+	vidb->hdr = buf + ubi->vid_hdr_shift;
+}
+
+/**
+ * ubi_init_vid_buf - Allocate a VID buffer
+ * @ubi: the UBI device
+ * @gfp_flags: GFP flags to use for the allocation
+ */
+static inline struct ubi_vid_io_buf *
+ubi_alloc_vid_buf(const struct ubi_device *ubi, gfp_t gfp_flags)
+{
+	struct ubi_vid_io_buf *vidb;
+	void *buf;
+
+	vidb = kzalloc(sizeof(*vidb), gfp_flags);
+	if (!vidb)
+		return NULL;
+
+	buf = kmalloc(ubi->vid_hdr_alsize, gfp_flags);
+	if (!buf) {
+		kfree(vidb);
 		return NULL;
+	}
 
-	/*
-	 * VID headers may be stored at un-aligned flash offsets, so we shift
-	 * the pointer.
-	 */
-	return vid_hdr + ubi->vid_hdr_shift;
+	ubi_init_vid_buf(ubi, vidb, buf);
+
+	return vidb;
 }
 
 /**
- * ubi_free_vid_hdr - free a volume identifier header object.
- * @ubi: UBI device description object
- * @vid_hdr: the object to free
+ * ubi_free_vid_buf - Free a VID buffer
+ * @vidb: the VID buffer to free
  */
-static inline void ubi_free_vid_hdr(const struct ubi_device *ubi,
-				    struct ubi_vid_hdr *vid_hdr)
+static inline void ubi_free_vid_buf(struct ubi_vid_io_buf *vidb)
 {
-	void *p = vid_hdr;
-
-	if (!p)
+	if (!vidb)
 		return;
 
-	kfree(p - ubi->vid_hdr_shift);
+	kfree(vidb->buffer);
+	kfree(vidb);
+}
+
+/**
+ * ubi_get_vid_hdr - Get the VID header attached to a VID buffer
+ * @vidb: VID buffer
+ */
+static inline struct ubi_vid_hdr *ubi_get_vid_hdr(struct ubi_vid_io_buf *vidb)
+{
+	return vidb->hdr;
 }
 
 /*
diff --git a/drivers/mtd/ubi/vmt.c b/drivers/mtd/ubi/vmt.c
index 0138f526474a..7ac78c13dd1c 100644
--- a/drivers/mtd/ubi/vmt.c
+++ b/drivers/mtd/ubi/vmt.c
@@ -138,7 +138,7 @@ static void vol_release(struct device *dev)
 {
 	struct ubi_volume *vol = container_of(dev, struct ubi_volume, dev);
 
-	kfree(vol->eba_tbl);
+	ubi_eba_replace_table(vol, NULL);
 	kfree(vol);
 }
 
@@ -158,6 +158,7 @@ int ubi_create_volume(struct ubi_device *ubi, struct ubi_mkvol_req *req)
 	int i, err, vol_id = req->vol_id, do_free = 1;
 	struct ubi_volume *vol;
 	struct ubi_vtbl_record vtbl_rec;
+	struct ubi_eba_table *eba_tbl = NULL;
 	dev_t dev;
 
 	if (ubi->ro_mode)
@@ -241,14 +242,13 @@ int ubi_create_volume(struct ubi_device *ubi, struct ubi_mkvol_req *req)
 	if (err)
 		goto out_acc;
 
-	vol->eba_tbl = kmalloc(vol->reserved_pebs * sizeof(int), GFP_KERNEL);
-	if (!vol->eba_tbl) {
-		err = -ENOMEM;
+	eba_tbl = ubi_eba_create_table(vol, vol->reserved_pebs);
+	if (IS_ERR(eba_tbl)) {
+		err = PTR_ERR(eba_tbl);
 		goto out_acc;
 	}
 
-	for (i = 0; i < vol->reserved_pebs; i++)
-		vol->eba_tbl[i] = UBI_LEB_UNMAPPED;
+	ubi_eba_replace_table(vol, eba_tbl);
 
 	if (vol->vol_type == UBI_DYNAMIC_VOLUME) {
 		vol->used_ebs = vol->reserved_pebs;
@@ -329,7 +329,7 @@ out_cdev:
 	cdev_del(&vol->cdev);
 out_mapping:
 	if (do_free)
-		kfree(vol->eba_tbl);
+		ubi_eba_destroy_table(eba_tbl);
 out_acc:
 	spin_lock(&ubi->volumes_lock);
 	ubi->rsvd_pebs -= vol->reserved_pebs;
@@ -427,10 +427,11 @@ out_unlock:
  */
 int ubi_resize_volume(struct ubi_volume_desc *desc, int reserved_pebs)
 {
-	int i, err, pebs, *new_mapping;
+	int i, err, pebs;
 	struct ubi_volume *vol = desc->vol;
 	struct ubi_device *ubi = vol->ubi;
 	struct ubi_vtbl_record vtbl_rec;
+	struct ubi_eba_table *new_eba_tbl = NULL;
 	int vol_id = vol->vol_id;
 
 	if (ubi->ro_mode)
@@ -450,12 +451,9 @@ int ubi_resize_volume(struct ubi_volume_desc *desc, int reserved_pebs)
 	if (reserved_pebs == vol->reserved_pebs)
 		return 0;
 
-	new_mapping = kmalloc(reserved_pebs * sizeof(int), GFP_KERNEL);
-	if (!new_mapping)
-		return -ENOMEM;
-
-	for (i = 0; i < reserved_pebs; i++)
-		new_mapping[i] = UBI_LEB_UNMAPPED;
+	new_eba_tbl = ubi_eba_create_table(vol, reserved_pebs);
+	if (IS_ERR(new_eba_tbl))
+		return PTR_ERR(new_eba_tbl);
 
 	spin_lock(&ubi->volumes_lock);
 	if (vol->ref_count > 1) {
@@ -481,10 +479,8 @@ int ubi_resize_volume(struct ubi_volume_desc *desc, int reserved_pebs)
 		}
 		ubi->avail_pebs -= pebs;
 		ubi->rsvd_pebs += pebs;
-		for (i = 0; i < vol->reserved_pebs; i++)
-			new_mapping[i] = vol->eba_tbl[i];
-		kfree(vol->eba_tbl);
-		vol->eba_tbl = new_mapping;
+		ubi_eba_copy_table(vol, new_eba_tbl, vol->reserved_pebs);
+		ubi_eba_replace_table(vol, new_eba_tbl);
 		spin_unlock(&ubi->volumes_lock);
 	}
 
@@ -498,10 +494,8 @@ int ubi_resize_volume(struct ubi_volume_desc *desc, int reserved_pebs)
 		ubi->rsvd_pebs += pebs;
 		ubi->avail_pebs -= pebs;
 		ubi_update_reserved(ubi);
-		for (i = 0; i < reserved_pebs; i++)
-			new_mapping[i] = vol->eba_tbl[i];
-		kfree(vol->eba_tbl);
-		vol->eba_tbl = new_mapping;
+		ubi_eba_copy_table(vol, new_eba_tbl, reserved_pebs);
+		ubi_eba_replace_table(vol, new_eba_tbl);
 		spin_unlock(&ubi->volumes_lock);
 	}
 
@@ -543,7 +537,7 @@ out_acc:
 		spin_unlock(&ubi->volumes_lock);
 	}
 out_free:
-	kfree(new_mapping);
+	kfree(new_eba_tbl);
 	return err;
 }
 
diff --git a/drivers/mtd/ubi/vtbl.c b/drivers/mtd/ubi/vtbl.c
index d85c19762160..263743e7b741 100644
--- a/drivers/mtd/ubi/vtbl.c
+++ b/drivers/mtd/ubi/vtbl.c
@@ -299,15 +299,18 @@ static int create_vtbl(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		       int copy, void *vtbl)
 {
 	int err, tries = 0;
+	struct ubi_vid_io_buf *vidb;
 	struct ubi_vid_hdr *vid_hdr;
 	struct ubi_ainf_peb *new_aeb;
 
 	dbg_gen("create volume table (copy #%d)", copy + 1);
 
-	vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_KERNEL);
-	if (!vid_hdr)
+	vidb = ubi_alloc_vid_buf(ubi, GFP_KERNEL);
+	if (!vidb)
 		return -ENOMEM;
 
+	vid_hdr = ubi_get_vid_hdr(vidb);
+
 retry:
 	new_aeb = ubi_early_get_peb(ubi, ai);
 	if (IS_ERR(new_aeb)) {
@@ -324,7 +327,7 @@ retry:
 	vid_hdr->sqnum = cpu_to_be64(++ai->max_sqnum);
 
 	/* The EC header is already there, write the VID header */
-	err = ubi_io_write_vid_hdr(ubi, new_aeb->pnum, vid_hdr);
+	err = ubi_io_write_vid_hdr(ubi, new_aeb->pnum, vidb);
 	if (err)
 		goto write_error;
 
@@ -338,8 +341,8 @@ retry:
 	 * of this LEB as it will be deleted and freed in 'ubi_add_to_av()'.
 	 */
 	err = ubi_add_to_av(ubi, ai, new_aeb->pnum, new_aeb->ec, vid_hdr, 0);
-	kmem_cache_free(ai->aeb_slab_cache, new_aeb);
-	ubi_free_vid_hdr(ubi, vid_hdr);
+	ubi_free_aeb(ai, new_aeb);
+	ubi_free_vid_buf(vidb);
 	return err;
 
 write_error:
@@ -351,9 +354,9 @@ write_error:
 		list_add(&new_aeb->u.list, &ai->erase);
 		goto retry;
 	}
-	kmem_cache_free(ai->aeb_slab_cache, new_aeb);
+	ubi_free_aeb(ai, new_aeb);
 out_free:
-	ubi_free_vid_hdr(ubi, vid_hdr);
+	ubi_free_vid_buf(vidb);
 	return err;
 
 }
diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c
index f4533266d7b2..b5b8cd6f481c 100644
--- a/drivers/mtd/ubi/wl.c
+++ b/drivers/mtd/ubi/wl.c
@@ -580,7 +580,7 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk,
  * failure.
  */
 static int schedule_erase(struct ubi_device *ubi, struct ubi_wl_entry *e,
-			  int vol_id, int lnum, int torture)
+			  int vol_id, int lnum, int torture, bool nested)
 {
 	struct ubi_work *wl_wrk;
 
@@ -599,7 +599,10 @@ static int schedule_erase(struct ubi_device *ubi, struct ubi_wl_entry *e,
 	wl_wrk->lnum = lnum;
 	wl_wrk->torture = torture;
 
-	schedule_ubi_work(ubi, wl_wrk);
+	if (nested)
+		__schedule_ubi_work(ubi, wl_wrk);
+	else
+		schedule_ubi_work(ubi, wl_wrk);
 	return 0;
 }
 
@@ -644,11 +647,12 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 				int shutdown)
 {
 	int err, scrubbing = 0, torture = 0, protect = 0, erroneous = 0;
-	int vol_id = -1, lnum = -1;
+	int erase = 0, keep = 0, vol_id = -1, lnum = -1;
 #ifdef CONFIG_MTD_UBI_FASTMAP
 	int anchor = wrk->anchor;
 #endif
 	struct ubi_wl_entry *e1, *e2;
+	struct ubi_vid_io_buf *vidb;
 	struct ubi_vid_hdr *vid_hdr;
 	int dst_leb_clean = 0;
 
@@ -656,10 +660,13 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 	if (shutdown)
 		return 0;
 
-	vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_NOFS);
-	if (!vid_hdr)
+	vidb = ubi_alloc_vid_buf(ubi, GFP_NOFS);
+	if (!vidb)
 		return -ENOMEM;
 
+	vid_hdr = ubi_get_vid_hdr(vidb);
+
+	down_read(&ubi->fm_eba_sem);
 	mutex_lock(&ubi->move_mutex);
 	spin_lock(&ubi->wl_lock);
 	ubi_assert(!ubi->move_from && !ubi->move_to);
@@ -753,7 +760,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 	 * which is being moved was unmapped.
 	 */
 
-	err = ubi_io_read_vid_hdr(ubi, e1->pnum, vid_hdr, 0);
+	err = ubi_io_read_vid_hdr(ubi, e1->pnum, vidb, 0);
 	if (err && err != UBI_IO_BITFLIPS) {
 		dst_leb_clean = 1;
 		if (err == UBI_IO_FF) {
@@ -780,6 +787,16 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 			       e1->pnum);
 			scrubbing = 1;
 			goto out_not_moved;
+		} else if (ubi->fast_attach && err == UBI_IO_BAD_HDR_EBADMSG) {
+			/*
+			 * While a full scan would detect interrupted erasures
+			 * at attach time we can face them here when attached from
+			 * Fastmap.
+			 */
+			dbg_wl("PEB %d has ECC errors, maybe from an interrupted erasure",
+			       e1->pnum);
+			erase = 1;
+			goto out_not_moved;
 		}
 
 		ubi_err(ubi, "error %d while reading VID header from PEB %d",
@@ -790,7 +807,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 	vol_id = be32_to_cpu(vid_hdr->vol_id);
 	lnum = be32_to_cpu(vid_hdr->lnum);
 
-	err = ubi_eba_copy_leb(ubi, e1->pnum, e2->pnum, vid_hdr);
+	err = ubi_eba_copy_leb(ubi, e1->pnum, e2->pnum, vidb);
 	if (err) {
 		if (err == MOVE_CANCEL_RACE) {
 			/*
@@ -815,6 +832,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 			 * Target PEB had bit-flips or write error - torture it.
 			 */
 			torture = 1;
+			keep = 1;
 			goto out_not_moved;
 		}
 
@@ -847,7 +865,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 	if (scrubbing)
 		ubi_msg(ubi, "scrubbed PEB %d (LEB %d:%d), data moved to PEB %d",
 			e1->pnum, vol_id, lnum, e2->pnum);
-	ubi_free_vid_hdr(ubi, vid_hdr);
+	ubi_free_vid_buf(vidb);
 
 	spin_lock(&ubi->wl_lock);
 	if (!ubi->move_to_put) {
@@ -879,6 +897,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 
 	dbg_wl("done");
 	mutex_unlock(&ubi->move_mutex);
+	up_read(&ubi->fm_eba_sem);
 	return 0;
 
 	/*
@@ -901,7 +920,7 @@ out_not_moved:
 		ubi->erroneous_peb_count += 1;
 	} else if (scrubbing)
 		wl_tree_add(e1, &ubi->scrub);
-	else
+	else if (keep)
 		wl_tree_add(e1, &ubi->used);
 	if (dst_leb_clean) {
 		wl_tree_add(e2, &ubi->free);
@@ -913,7 +932,7 @@ out_not_moved:
 	ubi->wl_scheduled = 0;
 	spin_unlock(&ubi->wl_lock);
 
-	ubi_free_vid_hdr(ubi, vid_hdr);
+	ubi_free_vid_buf(vidb);
 	if (dst_leb_clean) {
 		ensure_wear_leveling(ubi, 1);
 	} else {
@@ -922,7 +941,14 @@ out_not_moved:
 			goto out_ro;
 	}
 
+	if (erase) {
+		err = do_sync_erase(ubi, e1, vol_id, lnum, 1);
+		if (err)
+			goto out_ro;
+	}
+
 	mutex_unlock(&ubi->move_mutex);
+	up_read(&ubi->fm_eba_sem);
 	return 0;
 
 out_error:
@@ -937,13 +963,14 @@ out_error:
 	ubi->move_to_put = ubi->wl_scheduled = 0;
 	spin_unlock(&ubi->wl_lock);
 
-	ubi_free_vid_hdr(ubi, vid_hdr);
+	ubi_free_vid_buf(vidb);
 	wl_entry_destroy(ubi, e1);
 	wl_entry_destroy(ubi, e2);
 
 out_ro:
 	ubi_ro_mode(ubi);
 	mutex_unlock(&ubi->move_mutex);
+	up_read(&ubi->fm_eba_sem);
 	ubi_assert(err != 0);
 	return err < 0 ? err : -EIO;
 
@@ -951,7 +978,8 @@ out_cancel:
 	ubi->wl_scheduled = 0;
 	spin_unlock(&ubi->wl_lock);
 	mutex_unlock(&ubi->move_mutex);
-	ubi_free_vid_hdr(ubi, vid_hdr);
+	up_read(&ubi->fm_eba_sem);
+	ubi_free_vid_buf(vidb);
 	return 0;
 }
 
@@ -1073,7 +1101,7 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk)
 		int err1;
 
 		/* Re-schedule the LEB for erasure */
-		err1 = schedule_erase(ubi, e, vol_id, lnum, 0);
+		err1 = schedule_erase(ubi, e, vol_id, lnum, 0, false);
 		if (err1) {
 			wl_entry_destroy(ubi, e);
 			err = err1;
@@ -1254,7 +1282,7 @@ retry:
 	}
 	spin_unlock(&ubi->wl_lock);
 
-	err = schedule_erase(ubi, e, vol_id, lnum, torture);
+	err = schedule_erase(ubi, e, vol_id, lnum, torture, false);
 	if (err) {
 		spin_lock(&ubi->wl_lock);
 		wl_tree_add(e, &ubi->used);
@@ -1545,7 +1573,7 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
 		e->pnum = aeb->pnum;
 		e->ec = aeb->ec;
 		ubi->lookuptbl[e->pnum] = e;
-		if (schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0)) {
+		if (schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0, false)) {
 			wl_entry_destroy(ubi, e);
 			goto out_free;
 		}
@@ -1624,7 +1652,7 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
 			e->ec = aeb->ec;
 			ubi_assert(!ubi->lookuptbl[e->pnum]);
 			ubi->lookuptbl[e->pnum] = e;
-			if (schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0)) {
+			if (schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0, false)) {
 				wl_entry_destroy(ubi, e);
 				goto out_free;
 			}
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index 0c5415b05ea9..95c32f2d7601 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -149,6 +149,8 @@ config IPVLAN
     tristate "IP-VLAN support"
     depends on INET
     depends on IPV6
+    depends on NETFILTER
+    depends on NET_L3_MASTER_DEV
     ---help---
       This allows one to create virtual devices off of a main interface
       and packets will be delivered based on the dest L3 (IPv6/IPv4 addr)
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 1f276fa30ba6..5fa36ebc0640 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -152,7 +152,7 @@ module_param(lacp_rate, charp, 0);
 MODULE_PARM_DESC(lacp_rate, "LACPDU tx rate to request from 802.3ad partner; "
 			    "0 for slow, 1 for fast");
 module_param(ad_select, charp, 0);
-MODULE_PARM_DESC(ad_select, "803.ad aggregation selection logic; "
+MODULE_PARM_DESC(ad_select, "802.3ad aggregation selection logic; "
 			    "0 for stable (default), 1 for bandwidth, "
 			    "2 for count");
 module_param(min_links, int, 0);
@@ -471,9 +471,9 @@ static int bond_check_dev_link(struct bonding *bond,
 		/* Yes, the mii is overlaid on the ifreq.ifr_ifru */
 		strncpy(ifr.ifr_name, slave_dev->name, IFNAMSIZ);
 		mii = if_mii(&ifr);
-		if (IOCTL(slave_dev, &ifr, SIOCGMIIPHY) == 0) {
+		if (ioctl(slave_dev, &ifr, SIOCGMIIPHY) == 0) {
 			mii->reg_num = MII_BMSR;
-			if (IOCTL(slave_dev, &ifr, SIOCGMIIREG) == 0)
+			if (ioctl(slave_dev, &ifr, SIOCGMIIREG) == 0)
 				return mii->val_out & BMSR_LSTATUS;
 		}
 	}
@@ -1341,9 +1341,10 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
 			    slave_dev->name);
 	}
 
-	/* already enslaved */
-	if (slave_dev->flags & IFF_SLAVE) {
-		netdev_dbg(bond_dev, "Error: Device was already enslaved\n");
+	/* already in-use? */
+	if (netdev_is_rx_handler_busy(slave_dev)) {
+		netdev_err(bond_dev,
+			   "Error: Device is in use and cannot be enslaved\n");
 		return -EBUSY;
 	}
 
@@ -4627,7 +4628,7 @@ static int bond_init(struct net_device *bond_dev)
 
 	netdev_dbg(bond_dev, "Begin bond_init\n");
 
-	bond->wq = create_singlethread_workqueue(bond_dev->name);
+	bond->wq = alloc_ordered_workqueue(bond_dev->name, WQ_MEM_RECLAIM);
 	if (!bond->wq)
 		return -ENOMEM;
 
diff --git a/drivers/net/can/dev.c b/drivers/net/can/dev.c
index e21f7cc5ae4d..8d6208c0b400 100644
--- a/drivers/net/can/dev.c
+++ b/drivers/net/can/dev.c
@@ -21,6 +21,7 @@
 #include <linux/slab.h>
 #include <linux/netdevice.h>
 #include <linux/if_arp.h>
+#include <linux/workqueue.h>
 #include <linux/can.h>
 #include <linux/can/dev.h>
 #include <linux/can/skb.h>
@@ -501,9 +502,8 @@ EXPORT_SYMBOL_GPL(can_free_echo_skb);
 /*
  * CAN device restart for bus-off recovery
  */
-static void can_restart(unsigned long data)
+static void can_restart(struct net_device *dev)
 {
-	struct net_device *dev = (struct net_device *)data;
 	struct can_priv *priv = netdev_priv(dev);
 	struct net_device_stats *stats = &dev->stats;
 	struct sk_buff *skb;
@@ -543,6 +543,14 @@ restart:
 		netdev_err(dev, "Error %d during restart", err);
 }
 
+static void can_restart_work(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct can_priv *priv = container_of(dwork, struct can_priv, restart_work);
+
+	can_restart(priv->dev);
+}
+
 int can_restart_now(struct net_device *dev)
 {
 	struct can_priv *priv = netdev_priv(dev);
@@ -556,8 +564,8 @@ int can_restart_now(struct net_device *dev)
 	if (priv->state != CAN_STATE_BUS_OFF)
 		return -EBUSY;
 
-	/* Runs as soon as possible in the timer context */
-	mod_timer(&priv->restart_timer, jiffies);
+	cancel_delayed_work_sync(&priv->restart_work);
+	can_restart(dev);
 
 	return 0;
 }
@@ -578,8 +586,8 @@ void can_bus_off(struct net_device *dev)
 	netif_carrier_off(dev);
 
 	if (priv->restart_ms)
-		mod_timer(&priv->restart_timer,
-			  jiffies + (priv->restart_ms * HZ) / 1000);
+		schedule_delayed_work(&priv->restart_work,
+				      msecs_to_jiffies(priv->restart_ms));
 }
 EXPORT_SYMBOL_GPL(can_bus_off);
 
@@ -688,6 +696,7 @@ struct net_device *alloc_candev(int sizeof_priv, unsigned int echo_skb_max)
 		return NULL;
 
 	priv = netdev_priv(dev);
+	priv->dev = dev;
 
 	if (echo_skb_max) {
 		priv->echo_skb_max = echo_skb_max;
@@ -697,7 +706,7 @@ struct net_device *alloc_candev(int sizeof_priv, unsigned int echo_skb_max)
 
 	priv->state = CAN_STATE_STOPPED;
 
-	init_timer(&priv->restart_timer);
+	INIT_DELAYED_WORK(&priv->restart_work, can_restart_work);
 
 	return dev;
 }
@@ -778,8 +787,6 @@ int open_candev(struct net_device *dev)
 	if (!netif_carrier_ok(dev))
 		netif_carrier_on(dev);
 
-	setup_timer(&priv->restart_timer, can_restart, (unsigned long)dev);
-
 	return 0;
 }
 EXPORT_SYMBOL_GPL(open_candev);
@@ -794,7 +801,7 @@ void close_candev(struct net_device *dev)
 {
 	struct can_priv *priv = netdev_priv(dev);
 
-	del_timer_sync(&priv->restart_timer);
+	cancel_delayed_work_sync(&priv->restart_work);
 	can_flush_echo_skb(dev);
 }
 EXPORT_SYMBOL_GPL(close_candev);
diff --git a/drivers/net/can/flexcan.c b/drivers/net/can/flexcan.c
index 41c0fc9f3b14..16f7cadda5c3 100644
--- a/drivers/net/can/flexcan.c
+++ b/drivers/net/can/flexcan.c
@@ -1268,11 +1268,10 @@ static int __maybe_unused flexcan_suspend(struct device *device)
 	struct flexcan_priv *priv = netdev_priv(dev);
 	int err;
 
-	err = flexcan_chip_disable(priv);
-	if (err)
-		return err;
-
 	if (netif_running(dev)) {
+		err = flexcan_chip_disable(priv);
+		if (err)
+			return err;
 		netif_stop_queue(dev);
 		netif_device_detach(dev);
 	}
@@ -1285,13 +1284,17 @@ static int __maybe_unused flexcan_resume(struct device *device)
 {
 	struct net_device *dev = dev_get_drvdata(device);
 	struct flexcan_priv *priv = netdev_priv(dev);
+	int err;
 
 	priv->can.state = CAN_STATE_ERROR_ACTIVE;
 	if (netif_running(dev)) {
 		netif_device_attach(dev);
 		netif_start_queue(dev);
+		err = flexcan_chip_enable(priv);
+		if (err)
+			return err;
 	}
-	return flexcan_chip_enable(priv);
+	return 0;
 }
 
 static SIMPLE_DEV_PM_OPS(flexcan_pm_ops, flexcan_suspend, flexcan_resume);
diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c
index 2d1d22eec750..368bb0710d8f 100644
--- a/drivers/net/can/ifi_canfd/ifi_canfd.c
+++ b/drivers/net/can/ifi_canfd/ifi_canfd.c
@@ -81,6 +81,10 @@
 #define IFI_CANFD_TIME_SET_TIMEA_4_12_6_6	BIT(15)
 
 #define IFI_CANFD_TDELAY			0x1c
+#define IFI_CANFD_TDELAY_DEFAULT		0xb
+#define IFI_CANFD_TDELAY_MASK			0x3fff
+#define IFI_CANFD_TDELAY_ABS			BIT(14)
+#define IFI_CANFD_TDELAY_EN			BIT(15)
 
 #define IFI_CANFD_ERROR				0x20
 #define IFI_CANFD_ERROR_TX_OFFSET		0
@@ -641,7 +645,7 @@ static void ifi_canfd_set_bittiming(struct net_device *ndev)
 	struct ifi_canfd_priv *priv = netdev_priv(ndev);
 	const struct can_bittiming *bt = &priv->can.bittiming;
 	const struct can_bittiming *dbt = &priv->can.data_bittiming;
-	u16 brp, sjw, tseg1, tseg2;
+	u16 brp, sjw, tseg1, tseg2, tdc;
 
 	/* Configure bit timing */
 	brp = bt->brp - 2;
@@ -664,6 +668,11 @@ static void ifi_canfd_set_bittiming(struct net_device *ndev)
 	       (brp << IFI_CANFD_TIME_PRESCALE_OFF) |
 	       (sjw << IFI_CANFD_TIME_SJW_OFF_7_9_8_8),
 	       priv->base + IFI_CANFD_FTIME);
+
+	/* Configure transmitter delay */
+	tdc = (dbt->brp * (dbt->phase_seg1 + 1)) & IFI_CANFD_TDELAY_MASK;
+	writel(IFI_CANFD_TDELAY_EN | IFI_CANFD_TDELAY_ABS | tdc,
+	       priv->base + IFI_CANFD_TDELAY);
 }
 
 static void ifi_canfd_set_filter(struct net_device *ndev, const u32 id,
diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c
index 71f0e791355b..b3d02759c226 100644
--- a/drivers/net/can/usb/ems_usb.c
+++ b/drivers/net/can/usb/ems_usb.c
@@ -600,7 +600,6 @@ static int ems_usb_start(struct ems_usb *dev)
 		/* create a URB, and a buffer for it */
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
-			netdev_err(netdev, "No memory left for URBs\n");
 			err = -ENOMEM;
 			break;
 		}
@@ -752,10 +751,8 @@ static netdev_tx_t ems_usb_start_xmit(struct sk_buff *skb, struct net_device *ne
 
 	/* create a URB, and a buffer for it, and copy the data to the URB */
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!urb) {
-		netdev_err(netdev, "No memory left for URBs\n");
+	if (!urb)
 		goto nomem;
-	}
 
 	buf = usb_alloc_coherent(dev->udev, size, GFP_ATOMIC, &urb->transfer_dma);
 	if (!buf) {
@@ -1007,10 +1004,8 @@ static int ems_usb_probe(struct usb_interface *intf,
 		dev->tx_contexts[i].echo_index = MAX_TX_URBS;
 
 	dev->intr_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->intr_urb) {
-		dev_err(&intf->dev, "Couldn't alloc intr URB\n");
+	if (!dev->intr_urb)
 		goto cleanup_candev;
-	}
 
 	dev->intr_in_buffer = kzalloc(INTR_IN_BUFFER_SIZE, GFP_KERNEL);
 	if (!dev->intr_in_buffer)
diff --git a/drivers/net/can/usb/esd_usb2.c b/drivers/net/can/usb/esd_usb2.c
index 784a9002fbb9..be928ce62d32 100644
--- a/drivers/net/can/usb/esd_usb2.c
+++ b/drivers/net/can/usb/esd_usb2.c
@@ -558,8 +558,6 @@ static int esd_usb2_setup_rx_urbs(struct esd_usb2 *dev)
 		/* create a URB, and a buffer for it */
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
-			dev_warn(dev->udev->dev.parent,
-				 "No memory left for URBs\n");
 			err = -ENOMEM;
 			break;
 		}
@@ -730,7 +728,6 @@ static netdev_tx_t esd_usb2_start_xmit(struct sk_buff *skb,
 	/* create a URB, and a buffer for it, and copy the data to the URB */
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
 	if (!urb) {
-		netdev_err(netdev, "No memory left for URBs\n");
 		stats->tx_dropped++;
 		dev_kfree_skb(skb);
 		goto nourbmem;
diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c
index 6f0cbc38782e..77e3cc06a30c 100644
--- a/drivers/net/can/usb/gs_usb.c
+++ b/drivers/net/can/usb/gs_usb.c
@@ -493,10 +493,8 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb,
 
 	/* create a URB, and a buffer for it */
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!urb) {
-		netdev_err(netdev, "No memory left for URB\n");
+	if (!urb)
 		goto nomem_urb;
-	}
 
 	hf = usb_alloc_coherent(dev->udev, sizeof(*hf), GFP_ATOMIC,
 				&urb->transfer_dma);
@@ -600,11 +598,8 @@ static int gs_can_open(struct net_device *netdev)
 
 			/* alloc rx urb */
 			urb = usb_alloc_urb(0, GFP_KERNEL);
-			if (!urb) {
-				netdev_err(netdev,
-					   "No memory left for URB\n");
+			if (!urb)
 				return -ENOMEM;
-			}
 
 			/* alloc rx buffer */
 			buf = usb_alloc_coherent(dev->udev,
diff --git a/drivers/net/can/usb/kvaser_usb.c b/drivers/net/can/usb/kvaser_usb.c
index 6f1f3b675ff5..d51e0c401b48 100644
--- a/drivers/net/can/usb/kvaser_usb.c
+++ b/drivers/net/can/usb/kvaser_usb.c
@@ -787,10 +787,8 @@ static int kvaser_usb_simple_msg_async(struct kvaser_usb_net_priv *priv,
 	int err;
 
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!urb) {
-		netdev_err(netdev, "No memory left for URBs\n");
+	if (!urb)
 		return -ENOMEM;
-	}
 
 	buf = kmalloc(sizeof(struct kvaser_msg), GFP_ATOMIC);
 	if (!buf) {
@@ -1393,8 +1391,6 @@ static int kvaser_usb_setup_rx_urbs(struct kvaser_usb *dev)
 
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
-			dev_warn(dev->udev->dev.parent,
-				 "No memory left for URBs\n");
 			err = -ENOMEM;
 			break;
 		}
@@ -1670,7 +1666,6 @@ static netdev_tx_t kvaser_usb_start_xmit(struct sk_buff *skb,
 
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
 	if (!urb) {
-		netdev_err(netdev, "No memory left for URBs\n");
 		stats->tx_dropped++;
 		dev_kfree_skb(skb);
 		return NETDEV_TX_OK;
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c
index bfb91d8fa460..c06382cdfdfe 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c
@@ -399,7 +399,6 @@ static int peak_usb_start(struct peak_usb_device *dev)
 		/* create a URB, and a buffer for it, to receive usb messages */
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
-			netdev_err(netdev, "No memory left for URBs\n");
 			err = -ENOMEM;
 			break;
 		}
@@ -454,7 +453,6 @@ static int peak_usb_start(struct peak_usb_device *dev)
 		/* create a URB and a buffer for it, to transmit usb messages */
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
-			netdev_err(netdev, "No memory left for URBs\n");
 			err = -ENOMEM;
 			break;
 		}
@@ -651,10 +649,8 @@ static int peak_usb_restart(struct peak_usb_device *dev)
 
 	/* first allocate a urb to handle the asynchronous steps */
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!urb) {
-		netdev_err(dev->netdev, "no memory left for urb\n");
+	if (!urb)
 		return -ENOMEM;
-	}
 
 	/* also allocate enough space for the commands to send */
 	buf = kmalloc(PCAN_USB_MAX_CMD_LEN, GFP_ATOMIC);
diff --git a/drivers/net/can/usb/usb_8dev.c b/drivers/net/can/usb/usb_8dev.c
index a731720f1d13..108a30e15097 100644
--- a/drivers/net/can/usb/usb_8dev.c
+++ b/drivers/net/can/usb/usb_8dev.c
@@ -623,10 +623,8 @@ static netdev_tx_t usb_8dev_start_xmit(struct sk_buff *skb,
 
 	/* create a URB, and a buffer for it, and copy the data to the URB */
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!urb) {
-		netdev_err(netdev, "No memory left for URBs\n");
+	if (!urb)
 		goto nomem;
-	}
 
 	buf = usb_alloc_coherent(priv->udev, size, GFP_ATOMIC,
 				 &urb->transfer_dma);
@@ -748,7 +746,6 @@ static int usb_8dev_start(struct usb_8dev_priv *priv)
 		/* create a URB, and a buffer for it */
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
-			netdev_err(netdev, "No memory left for URBs\n");
 			err = -ENOMEM;
 			break;
 		}
diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig
index 8f4544394f44..065984670ff1 100644
--- a/drivers/net/dsa/Kconfig
+++ b/drivers/net/dsa/Kconfig
@@ -16,6 +16,7 @@ config NET_DSA_BCM_SF2
 	select FIXED_PHY
 	select BCM7XXX_PHY
 	select MDIO_BCM_UNIMAC
+	select B53
 	---help---
 	  This enables support for the Broadcom Starfighter 2 Ethernet
 	  switch chips.
@@ -24,4 +25,13 @@ source "drivers/net/dsa/b53/Kconfig"
 
 source "drivers/net/dsa/mv88e6xxx/Kconfig"
 
+config NET_DSA_QCA8K
+	tristate "Qualcomm Atheros QCA8K Ethernet switch family support"
+	depends on NET_DSA
+	select NET_DSA_TAG_QCA
+	select REGMAP
+	---help---
+	  This enables support for the Qualcomm Atheros QCA8K Ethernet
+	  switch chips.
+
 endmenu
diff --git a/drivers/net/dsa/Makefile b/drivers/net/dsa/Makefile
index ca1e71b853a6..8346e4f9737a 100644
--- a/drivers/net/dsa/Makefile
+++ b/drivers/net/dsa/Makefile
@@ -1,5 +1,6 @@
 obj-$(CONFIG_NET_DSA_MV88E6060) += mv88e6060.o
 obj-$(CONFIG_NET_DSA_BCM_SF2)	+= bcm_sf2.o
+obj-$(CONFIG_NET_DSA_QCA8K)	+= qca8k.o
 
 obj-y				+= b53/
 obj-y				+= mv88e6xxx/
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index bda37d336736..7717b19dc806 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -167,6 +167,65 @@ static const struct b53_mib_desc b53_mibs[] = {
 
 #define B53_MIBS_SIZE	ARRAY_SIZE(b53_mibs)
 
+static const struct b53_mib_desc b53_mibs_58xx[] = {
+	{ 8, 0x00, "TxOctets" },
+	{ 4, 0x08, "TxDropPkts" },
+	{ 4, 0x0c, "TxQPKTQ0" },
+	{ 4, 0x10, "TxBroadcastPkts" },
+	{ 4, 0x14, "TxMulticastPkts" },
+	{ 4, 0x18, "TxUnicastPKts" },
+	{ 4, 0x1c, "TxCollisions" },
+	{ 4, 0x20, "TxSingleCollision" },
+	{ 4, 0x24, "TxMultipleCollision" },
+	{ 4, 0x28, "TxDeferredCollision" },
+	{ 4, 0x2c, "TxLateCollision" },
+	{ 4, 0x30, "TxExcessiveCollision" },
+	{ 4, 0x34, "TxFrameInDisc" },
+	{ 4, 0x38, "TxPausePkts" },
+	{ 4, 0x3c, "TxQPKTQ1" },
+	{ 4, 0x40, "TxQPKTQ2" },
+	{ 4, 0x44, "TxQPKTQ3" },
+	{ 4, 0x48, "TxQPKTQ4" },
+	{ 4, 0x4c, "TxQPKTQ5" },
+	{ 8, 0x50, "RxOctets" },
+	{ 4, 0x58, "RxUndersizePkts" },
+	{ 4, 0x5c, "RxPausePkts" },
+	{ 4, 0x60, "RxPkts64Octets" },
+	{ 4, 0x64, "RxPkts65to127Octets" },
+	{ 4, 0x68, "RxPkts128to255Octets" },
+	{ 4, 0x6c, "RxPkts256to511Octets" },
+	{ 4, 0x70, "RxPkts512to1023Octets" },
+	{ 4, 0x74, "RxPkts1024toMaxPktsOctets" },
+	{ 4, 0x78, "RxOversizePkts" },
+	{ 4, 0x7c, "RxJabbers" },
+	{ 4, 0x80, "RxAlignmentErrors" },
+	{ 4, 0x84, "RxFCSErrors" },
+	{ 8, 0x88, "RxGoodOctets" },
+	{ 4, 0x90, "RxDropPkts" },
+	{ 4, 0x94, "RxUnicastPkts" },
+	{ 4, 0x98, "RxMulticastPkts" },
+	{ 4, 0x9c, "RxBroadcastPkts" },
+	{ 4, 0xa0, "RxSAChanges" },
+	{ 4, 0xa4, "RxFragments" },
+	{ 4, 0xa8, "RxJumboPkt" },
+	{ 4, 0xac, "RxSymblErr" },
+	{ 4, 0xb0, "InRangeErrCount" },
+	{ 4, 0xb4, "OutRangeErrCount" },
+	{ 4, 0xb8, "EEELpiEvent" },
+	{ 4, 0xbc, "EEELpiDuration" },
+	{ 4, 0xc0, "RxDiscard" },
+	{ 4, 0xc8, "TxQPKTQ6" },
+	{ 4, 0xcc, "TxQPKTQ7" },
+	{ 4, 0xd0, "TxPkts64Octets" },
+	{ 4, 0xd4, "TxPkts65to127Octets" },
+	{ 4, 0xd8, "TxPkts128to255Octets" },
+	{ 4, 0xdc, "TxPkts256to511Ocets" },
+	{ 4, 0xe0, "TxPkts512to1023Ocets" },
+	{ 4, 0xe4, "TxPkts1024toMaxPktOcets" },
+};
+
+#define B53_MIBS_58XX_SIZE	ARRAY_SIZE(b53_mibs_58xx)
+
 static int b53_do_vlan_op(struct b53_device *dev, u8 op)
 {
 	unsigned int i;
@@ -418,7 +477,7 @@ static int b53_fast_age_vlan(struct b53_device *dev, u16 vid)
 
 static void b53_imp_vlan_setup(struct dsa_switch *ds, int cpu_port)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	unsigned int i;
 	u16 pvlan;
 
@@ -436,7 +495,7 @@ static void b53_imp_vlan_setup(struct dsa_switch *ds, int cpu_port)
 static int b53_enable_port(struct dsa_switch *ds, int port,
 			   struct phy_device *phy)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	unsigned int cpu_port = dev->cpu_port;
 	u16 pvlan;
 
@@ -461,7 +520,7 @@ static int b53_enable_port(struct dsa_switch *ds, int port,
 static void b53_disable_port(struct dsa_switch *ds, int port,
 			     struct phy_device *phy)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	u8 reg;
 
 	/* Disable Tx/Rx for the port */
@@ -570,7 +629,7 @@ static int b53_switch_reset(struct b53_device *dev)
 
 static int b53_phy_read16(struct dsa_switch *ds, int addr, int reg)
 {
-	struct b53_device *priv = ds_to_priv(ds);
+	struct b53_device *priv = ds->priv;
 	u16 value = 0;
 	int ret;
 
@@ -585,7 +644,7 @@ static int b53_phy_read16(struct dsa_switch *ds, int addr, int reg)
 
 static int b53_phy_write16(struct dsa_switch *ds, int addr, int reg, u16 val)
 {
-	struct b53_device *priv = ds_to_priv(ds);
+	struct b53_device *priv = ds->priv;
 
 	if (priv->ops->phy_write16)
 		return priv->ops->phy_write16(priv, addr, reg, val);
@@ -635,6 +694,8 @@ static const struct b53_mib_desc *b53_get_mib(struct b53_device *dev)
 		return b53_mibs_65;
 	else if (is63xx(dev))
 		return b53_mibs_63xx;
+	else if (is58xx(dev))
+		return b53_mibs_58xx;
 	else
 		return b53_mibs;
 }
@@ -645,13 +706,15 @@ static unsigned int b53_get_mib_size(struct b53_device *dev)
 		return B53_MIBS_65_SIZE;
 	else if (is63xx(dev))
 		return B53_MIBS_63XX_SIZE;
+	else if (is58xx(dev))
+		return B53_MIBS_58XX_SIZE;
 	else
 		return B53_MIBS_SIZE;
 }
 
 static void b53_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	const struct b53_mib_desc *mibs = b53_get_mib(dev);
 	unsigned int mib_size = b53_get_mib_size(dev);
 	unsigned int i;
@@ -664,7 +727,7 @@ static void b53_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
 static void b53_get_ethtool_stats(struct dsa_switch *ds, int port,
 				  uint64_t *data)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	const struct b53_mib_desc *mibs = b53_get_mib(dev);
 	unsigned int mib_size = b53_get_mib_size(dev);
 	const struct b53_mib_desc *s;
@@ -696,19 +759,14 @@ static void b53_get_ethtool_stats(struct dsa_switch *ds, int port,
 
 static int b53_get_sset_count(struct dsa_switch *ds)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 
 	return b53_get_mib_size(dev);
 }
 
-static int b53_set_addr(struct dsa_switch *ds, u8 *addr)
-{
-	return 0;
-}
-
 static int b53_setup(struct dsa_switch *ds)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	unsigned int port;
 	int ret;
 
@@ -739,7 +797,7 @@ static int b53_setup(struct dsa_switch *ds)
 static void b53_adjust_link(struct dsa_switch *ds, int port,
 			    struct phy_device *phydev)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	u8 rgmii_ctrl = 0, reg = 0, off;
 
 	if (!phy_is_pseudo_fixed_link(phydev))
@@ -873,7 +931,7 @@ static int b53_vlan_prepare(struct dsa_switch *ds, int port,
 			    const struct switchdev_obj_port_vlan *vlan,
 			    struct switchdev_trans *trans)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 
 	if ((is5325(dev) || is5365(dev)) && vlan->vid_begin == 0)
 		return -EOPNOTSUPP;
@@ -890,7 +948,7 @@ static void b53_vlan_add(struct dsa_switch *ds, int port,
 			 const struct switchdev_obj_port_vlan *vlan,
 			 struct switchdev_trans *trans)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
 	bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID;
 	unsigned int cpu_port = dev->cpu_port;
@@ -924,7 +982,7 @@ static void b53_vlan_add(struct dsa_switch *ds, int port,
 static int b53_vlan_del(struct dsa_switch *ds, int port,
 			const struct switchdev_obj_port_vlan *vlan)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
 	unsigned int cpu_port = dev->cpu_port;
 	struct b53_vlan *vl;
@@ -970,7 +1028,7 @@ static int b53_vlan_dump(struct dsa_switch *ds, int port,
 			 struct switchdev_obj_port_vlan *vlan,
 			 int (*cb)(struct switchdev_obj *obj))
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	u16 vid, vid_start = 0, pvid;
 	struct b53_vlan *vl;
 	int err = 0;
@@ -1129,7 +1187,7 @@ static int b53_fdb_prepare(struct dsa_switch *ds, int port,
 			   const struct switchdev_obj_port_fdb *fdb,
 			   struct switchdev_trans *trans)
 {
-	struct b53_device *priv = ds_to_priv(ds);
+	struct b53_device *priv = ds->priv;
 
 	/* 5325 and 5365 require some more massaging, but could
 	 * be supported eventually
@@ -1144,7 +1202,7 @@ static void b53_fdb_add(struct dsa_switch *ds, int port,
 			const struct switchdev_obj_port_fdb *fdb,
 			struct switchdev_trans *trans)
 {
-	struct b53_device *priv = ds_to_priv(ds);
+	struct b53_device *priv = ds->priv;
 
 	if (b53_arl_op(priv, 0, port, fdb->addr, fdb->vid, true))
 		pr_err("%s: failed to add MAC address\n", __func__);
@@ -1153,7 +1211,7 @@ static void b53_fdb_add(struct dsa_switch *ds, int port,
 static int b53_fdb_del(struct dsa_switch *ds, int port,
 		       const struct switchdev_obj_port_fdb *fdb)
 {
-	struct b53_device *priv = ds_to_priv(ds);
+	struct b53_device *priv = ds->priv;
 
 	return b53_arl_op(priv, 0, port, fdb->addr, fdb->vid, false);
 }
@@ -1212,7 +1270,7 @@ static int b53_fdb_dump(struct dsa_switch *ds, int port,
 			struct switchdev_obj_port_fdb *fdb,
 			int (*cb)(struct switchdev_obj *obj))
 {
-	struct b53_device *priv = ds_to_priv(ds);
+	struct b53_device *priv = ds->priv;
 	struct net_device *dev = ds->ports[port].netdev;
 	struct b53_arl_entry results[2];
 	unsigned int count = 0;
@@ -1251,10 +1309,22 @@ static int b53_fdb_dump(struct dsa_switch *ds, int port,
 static int b53_br_join(struct dsa_switch *ds, int port,
 		       struct net_device *bridge)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
+	s8 cpu_port = ds->dst->cpu_port;
 	u16 pvlan, reg;
 	unsigned int i;
 
+	/* Make this port leave the all VLANs join since we will have proper
+	 * VLAN entries from now on
+	 */
+	if (is58xx(dev)) {
+		b53_read16(dev, B53_VLAN_PAGE, B53_JOIN_ALL_VLAN_EN, &reg);
+		reg &= ~BIT(port);
+		if ((reg & BIT(cpu_port)) == BIT(cpu_port))
+			reg &= ~BIT(cpu_port);
+		b53_write16(dev, B53_VLAN_PAGE, B53_JOIN_ALL_VLAN_EN, reg);
+	}
+
 	dev->ports[port].bridge_dev = bridge;
 	b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), &pvlan);
 
@@ -1284,9 +1354,10 @@ static int b53_br_join(struct dsa_switch *ds, int port,
 
 static void b53_br_leave(struct dsa_switch *ds, int port)
 {
-	struct b53_device *dev = ds_to_priv(ds);
+	struct b53_device *dev = ds->priv;
 	struct net_device *bridge = dev->ports[port].bridge_dev;
 	struct b53_vlan *vl = &dev->vlans[0];
+	s8 cpu_port = ds->dst->cpu_port;
 	unsigned int i;
 	u16 pvlan, reg, pvid;
 
@@ -1316,22 +1387,27 @@ static void b53_br_leave(struct dsa_switch *ds, int port)
 	else
 		pvid = 0;
 
-	b53_get_vlan_entry(dev, pvid, vl);
-	vl->members |= BIT(port) | BIT(dev->cpu_port);
-	vl->untag |= BIT(port) | BIT(dev->cpu_port);
-	b53_set_vlan_entry(dev, pvid, vl);
+	/* Make this port join all VLANs without VLAN entries */
+	if (is58xx(dev)) {
+		b53_read16(dev, B53_VLAN_PAGE, B53_JOIN_ALL_VLAN_EN, &reg);
+		reg |= BIT(port);
+		if (!(reg & BIT(cpu_port)))
+			reg |= BIT(cpu_port);
+		b53_write16(dev, B53_VLAN_PAGE, B53_JOIN_ALL_VLAN_EN, reg);
+	} else {
+		b53_get_vlan_entry(dev, pvid, vl);
+		vl->members |= BIT(port) | BIT(dev->cpu_port);
+		vl->untag |= BIT(port) | BIT(dev->cpu_port);
+		b53_set_vlan_entry(dev, pvid, vl);
+	}
 }
 
-static void b53_br_set_stp_state(struct dsa_switch *ds, int port,
-				 u8 state)
+static void b53_br_set_stp_state(struct dsa_switch *ds, int port, u8 state)
 {
-	struct b53_device *dev = ds_to_priv(ds);
-	u8 hw_state, cur_hw_state;
+	struct b53_device *dev = ds->priv;
+	u8 hw_state;
 	u8 reg;
 
-	b53_read8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(port), &reg);
-	cur_hw_state = reg & PORT_CTRL_STP_STATE_MASK;
-
 	switch (state) {
 	case BR_STATE_DISABLED:
 		hw_state = PORT_CTRL_DIS_STATE;
@@ -1353,30 +1429,28 @@ static void b53_br_set_stp_state(struct dsa_switch *ds, int port,
 		return;
 	}
 
-	/* Fast-age ARL entries if we are moving a port from Learning or
-	 * Forwarding (cur_hw_state) state to Disabled, Blocking or Listening
-	 * state (hw_state)
-	 */
-	if (cur_hw_state != hw_state) {
-		if (cur_hw_state >= PORT_CTRL_LEARN_STATE &&
-		    hw_state <= PORT_CTRL_LISTEN_STATE) {
-			if (b53_fast_age_port(dev, port)) {
-				dev_err(ds->dev, "fast ageing failed\n");
-				return;
-			}
-		}
-	}
-
 	b53_read8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(port), &reg);
 	reg &= ~PORT_CTRL_STP_STATE_MASK;
 	reg |= hw_state;
 	b53_write8(dev, B53_CTRL_PAGE, B53_PORT_CTRL(port), reg);
 }
 
-static struct dsa_switch_driver b53_switch_ops = {
-	.tag_protocol		= DSA_TAG_PROTO_NONE,
+static void b53_br_fast_age(struct dsa_switch *ds, int port)
+{
+	struct b53_device *dev = ds->priv;
+
+	if (b53_fast_age_port(dev, port))
+		dev_err(ds->dev, "fast ageing failed\n");
+}
+
+static enum dsa_tag_protocol b53_get_tag_protocol(struct dsa_switch *ds)
+{
+	return DSA_TAG_PROTO_NONE;
+}
+
+static struct dsa_switch_ops b53_switch_ops = {
+	.get_tag_protocol	= b53_get_tag_protocol,
 	.setup			= b53_setup,
-	.set_addr		= b53_set_addr,
 	.get_strings		= b53_get_strings,
 	.get_ethtool_stats	= b53_get_ethtool_stats,
 	.get_sset_count		= b53_get_sset_count,
@@ -1388,6 +1462,7 @@ static struct dsa_switch_driver b53_switch_ops = {
 	.port_bridge_join	= b53_br_join,
 	.port_bridge_leave	= b53_br_leave,
 	.port_stp_state_set	= b53_br_set_stp_state,
+	.port_fast_age		= b53_br_fast_age,
 	.port_vlan_filtering	= b53_vlan_filtering,
 	.port_vlan_prepare	= b53_vlan_prepare,
 	.port_vlan_add		= b53_vlan_add,
@@ -1593,11 +1668,22 @@ static const struct b53_chip_data b53_switch_chips[] = {
 		.jumbo_pm_reg = B53_JUMBO_PORT_MASK,
 		.jumbo_size_reg = B53_JUMBO_MAX_SIZE,
 	},
+	{
+		.chip_id = BCM7445_DEVICE_ID,
+		.dev_name = "BCM7445",
+		.vlans	= 4096,
+		.enabled_ports = 0x1ff,
+		.arl_entries = 4,
+		.cpu_port = B53_CPU_PORT,
+		.vta_regs = B53_VTA_REGS,
+		.duplex_reg = B53_DUPLEX_STAT_GE,
+		.jumbo_pm_reg = B53_JUMBO_PORT_MASK,
+		.jumbo_size_reg = B53_JUMBO_MAX_SIZE,
+	},
 };
 
 static int b53_switch_init(struct b53_device *dev)
 {
-	struct dsa_switch *ds = dev->ds;
 	unsigned int i;
 	int ret;
 
@@ -1613,7 +1699,6 @@ static int b53_switch_init(struct b53_device *dev)
 			dev->vta_regs[1] = chip->vta_regs[1];
 			dev->vta_regs[2] = chip->vta_regs[2];
 			dev->jumbo_pm_reg = chip->jumbo_pm_reg;
-			ds->drv = &b53_switch_ops;
 			dev->cpu_port = chip->cpu_port;
 			dev->num_vlans = chip->vlans;
 			dev->num_arl_entries = chip->arl_entries;
@@ -1681,7 +1766,8 @@ static int b53_switch_init(struct b53_device *dev)
 	return 0;
 }
 
-struct b53_device *b53_switch_alloc(struct device *base, struct b53_io_ops *ops,
+struct b53_device *b53_switch_alloc(struct device *base,
+				    const struct b53_io_ops *ops,
 				    void *priv)
 {
 	struct dsa_switch *ds;
@@ -1700,6 +1786,7 @@ struct b53_device *b53_switch_alloc(struct device *base, struct b53_io_ops *ops,
 	dev->ds = ds;
 	dev->priv = priv;
 	dev->ops = ops;
+	ds->ops = &b53_switch_ops;
 	mutex_init(&dev->reg_mutex);
 	mutex_init(&dev->stats_mutex);
 
diff --git a/drivers/net/dsa/b53/b53_mdio.c b/drivers/net/dsa/b53/b53_mdio.c
index aa87c3fffdac..477a16b5660a 100644
--- a/drivers/net/dsa/b53/b53_mdio.c
+++ b/drivers/net/dsa/b53/b53_mdio.c
@@ -267,7 +267,7 @@ static int b53_mdio_phy_write16(struct b53_device *dev, int addr, int reg,
 	return mdiobus_write_nested(bus, addr, reg, value);
 }
 
-static struct b53_io_ops b53_mdio_ops = {
+static const struct b53_io_ops b53_mdio_ops = {
 	.read8 = b53_mdio_read8,
 	.read16 = b53_mdio_read16,
 	.read32 = b53_mdio_read32,
diff --git a/drivers/net/dsa/b53/b53_mmap.c b/drivers/net/dsa/b53/b53_mmap.c
index 77ffc4312808..76fb8552c9d9 100644
--- a/drivers/net/dsa/b53/b53_mmap.c
+++ b/drivers/net/dsa/b53/b53_mmap.c
@@ -17,7 +17,6 @@
  */
 
 #include <linux/kernel.h>
-#include <linux/kconfig.h>
 #include <linux/module.h>
 #include <linux/io.h>
 #include <linux/platform_device.h>
@@ -208,7 +207,7 @@ static int b53_mmap_write64(struct b53_device *dev, u8 page, u8 reg,
 	return 0;
 }
 
-static struct b53_io_ops b53_mmap_ops = {
+static const struct b53_io_ops b53_mmap_ops = {
 	.read8 = b53_mmap_read8,
 	.read16 = b53_mmap_read16,
 	.read32 = b53_mmap_read32,
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index 835a744f206e..f192a673caba 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -60,6 +60,7 @@ enum {
 	BCM53018_DEVICE_ID = 0x53018,
 	BCM53019_DEVICE_ID = 0x53019,
 	BCM58XX_DEVICE_ID = 0x5800,
+	BCM7445_DEVICE_ID = 0x7445,
 };
 
 #define B53_N_PORTS	9
@@ -174,6 +175,12 @@ static inline int is5301x(struct b53_device *dev)
 		dev->chip_id == BCM53019_DEVICE_ID;
 }
 
+static inline int is58xx(struct b53_device *dev)
+{
+	return dev->chip_id == BCM58XX_DEVICE_ID ||
+		dev->chip_id == BCM7445_DEVICE_ID;
+}
+
 #define B53_CPU_PORT_25	5
 #define B53_CPU_PORT	8
 
@@ -182,7 +189,8 @@ static inline int is_cpu_port(struct b53_device *dev, int port)
 	return dev->cpu_port;
 }
 
-struct b53_device *b53_switch_alloc(struct device *base, struct b53_io_ops *ops,
+struct b53_device *b53_switch_alloc(struct device *base,
+				    const struct b53_io_ops *ops,
 				    void *priv);
 
 int b53_switch_detect(struct b53_device *dev);
@@ -364,7 +372,6 @@ static inline void b53_arl_from_entry(u64 *mac_vid, u32 *fwd_entry,
 
 #ifdef CONFIG_BCM47XX
 
-#include <linux/version.h>
 #include <linux/bcm47xx_nvram.h>
 #include <bcm47xx_board.h>
 static inline int b53_switch_get_reset_gpio(struct b53_device *dev)
diff --git a/drivers/net/dsa/b53/b53_regs.h b/drivers/net/dsa/b53/b53_regs.h
index 8f12bddd5dc9..dac0af4e2cd0 100644
--- a/drivers/net/dsa/b53/b53_regs.h
+++ b/drivers/net/dsa/b53/b53_regs.h
@@ -258,7 +258,7 @@
  * BCM5325 and BCM5365 share most definitions below
  */
 #define B53_ARLTBL_MAC_VID_ENTRY(n)	(0x10 * (n))
-#define   ARLTBL_MAC_MASK		0xffffffffffff
+#define   ARLTBL_MAC_MASK		0xffffffffffffULL
 #define   ARLTBL_VID_S			48
 #define   ARLTBL_VID_MASK_25		0xff
 #define   ARLTBL_VID_MASK		0xfff
@@ -309,6 +309,9 @@
 /* Port VLAN mask (16 bit) IMP port is always 8, also on 5325 & co */
 #define B53_PVLAN_PORT_MASK(i)		((i) * 2)
 
+/* Join all VLANs register (16 bit) */
+#define B53_JOIN_ALL_VLAN_EN		0x50
+
 /*************************************************************************
  * 802.1Q Page Registers
  *************************************************************************/
diff --git a/drivers/net/dsa/b53/b53_spi.c b/drivers/net/dsa/b53/b53_spi.c
index 2bda0b5f1578..f89f5308a99b 100644
--- a/drivers/net/dsa/b53/b53_spi.c
+++ b/drivers/net/dsa/b53/b53_spi.c
@@ -270,7 +270,7 @@ static int b53_spi_write64(struct b53_device *dev, u8 page, u8 reg, u64 value)
 	return spi_write(spi, txbuf, sizeof(txbuf));
 }
 
-static struct b53_io_ops b53_spi_ops = {
+static const struct b53_io_ops b53_spi_ops = {
 	.read8 = b53_spi_read8,
 	.read16 = b53_spi_read16,
 	.read32 = b53_spi_read32,
@@ -317,8 +317,6 @@ static int b53_spi_remove(struct spi_device *spi)
 static struct spi_driver b53_spi_driver = {
 	.driver = {
 		.name	= "b53-switch",
-		.bus	= &spi_bus_type,
-		.owner	= THIS_MODULE,
 	},
 	.probe	= b53_spi_probe,
 	.remove	= b53_spi_remove,
diff --git a/drivers/net/dsa/b53/b53_srab.c b/drivers/net/dsa/b53/b53_srab.c
index 3e2d4a5fcd5a..8a62b6a69703 100644
--- a/drivers/net/dsa/b53/b53_srab.c
+++ b/drivers/net/dsa/b53/b53_srab.c
@@ -344,7 +344,7 @@ err:
 	return ret;
 }
 
-static struct b53_io_ops b53_srab_ops = {
+static const struct b53_io_ops b53_srab_ops = {
 	.read8 = b53_srab_read8,
 	.read16 = b53_srab_read16,
 	.read32 = b53_srab_read32,
diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
index b2b838724a9b..e218887f18b7 100644
--- a/drivers/net/dsa/bcm_sf2.c
+++ b/drivers/net/dsa/bcm_sf2.c
@@ -29,130 +29,21 @@
 #include <linux/brcmphy.h>
 #include <linux/etherdevice.h>
 #include <net/switchdev.h>
+#include <linux/platform_data/b53.h>
 
 #include "bcm_sf2.h"
 #include "bcm_sf2_regs.h"
+#include "b53/b53_priv.h"
+#include "b53/b53_regs.h"
 
-/* String, offset, and register size in bytes if different from 4 bytes */
-static const struct bcm_sf2_hw_stats bcm_sf2_mib[] = {
-	{ "TxOctets",		0x000, 8	},
-	{ "TxDropPkts",		0x020		},
-	{ "TxQPKTQ0",		0x030		},
-	{ "TxBroadcastPkts",	0x040		},
-	{ "TxMulticastPkts",	0x050		},
-	{ "TxUnicastPKts",	0x060		},
-	{ "TxCollisions",	0x070		},
-	{ "TxSingleCollision",	0x080		},
-	{ "TxMultipleCollision", 0x090		},
-	{ "TxDeferredCollision", 0x0a0		},
-	{ "TxLateCollision",	0x0b0		},
-	{ "TxExcessiveCollision", 0x0c0		},
-	{ "TxFrameInDisc",	0x0d0		},
-	{ "TxPausePkts",	0x0e0		},
-	{ "TxQPKTQ1",		0x0f0		},
-	{ "TxQPKTQ2",		0x100		},
-	{ "TxQPKTQ3",		0x110		},
-	{ "TxQPKTQ4",		0x120		},
-	{ "TxQPKTQ5",		0x130		},
-	{ "RxOctets",		0x140, 8	},
-	{ "RxUndersizePkts",	0x160		},
-	{ "RxPausePkts",	0x170		},
-	{ "RxPkts64Octets",	0x180		},
-	{ "RxPkts65to127Octets", 0x190		},
-	{ "RxPkts128to255Octets", 0x1a0		},
-	{ "RxPkts256to511Octets", 0x1b0		},
-	{ "RxPkts512to1023Octets", 0x1c0	},
-	{ "RxPkts1024toMaxPktsOctets", 0x1d0	},
-	{ "RxOversizePkts",	0x1e0		},
-	{ "RxJabbers",		0x1f0		},
-	{ "RxAlignmentErrors",	0x200		},
-	{ "RxFCSErrors",	0x210		},
-	{ "RxGoodOctets",	0x220, 8	},
-	{ "RxDropPkts",		0x240		},
-	{ "RxUnicastPkts",	0x250		},
-	{ "RxMulticastPkts",	0x260		},
-	{ "RxBroadcastPkts",	0x270		},
-	{ "RxSAChanges",	0x280		},
-	{ "RxFragments",	0x290		},
-	{ "RxJumboPkt",		0x2a0		},
-	{ "RxSymblErr",		0x2b0		},
-	{ "InRangeErrCount",	0x2c0		},
-	{ "OutRangeErrCount",	0x2d0		},
-	{ "EEELpiEvent",	0x2e0		},
-	{ "EEELpiDuration",	0x2f0		},
-	{ "RxDiscard",		0x300, 8	},
-	{ "TxQPKTQ6",		0x320		},
-	{ "TxQPKTQ7",		0x330		},
-	{ "TxPkts64Octets",	0x340		},
-	{ "TxPkts65to127Octets", 0x350		},
-	{ "TxPkts128to255Octets", 0x360		},
-	{ "TxPkts256to511Ocets", 0x370		},
-	{ "TxPkts512to1023Ocets", 0x380		},
-	{ "TxPkts1024toMaxPktOcets", 0x390	},
-};
-
-#define BCM_SF2_STATS_SIZE	ARRAY_SIZE(bcm_sf2_mib)
-
-static void bcm_sf2_sw_get_strings(struct dsa_switch *ds,
-				   int port, uint8_t *data)
-{
-	unsigned int i;
-
-	for (i = 0; i < BCM_SF2_STATS_SIZE; i++)
-		memcpy(data + i * ETH_GSTRING_LEN,
-		       bcm_sf2_mib[i].string, ETH_GSTRING_LEN);
-}
-
-static void bcm_sf2_sw_get_ethtool_stats(struct dsa_switch *ds,
-					 int port, uint64_t *data)
-{
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-	const struct bcm_sf2_hw_stats *s;
-	unsigned int i;
-	u64 val = 0;
-	u32 offset;
-
-	mutex_lock(&priv->stats_mutex);
-
-	/* Now fetch the per-port counters */
-	for (i = 0; i < BCM_SF2_STATS_SIZE; i++) {
-		s = &bcm_sf2_mib[i];
-
-		/* Do a latched 64-bit read if needed */
-		offset = s->reg + CORE_P_MIB_OFFSET(port);
-		if (s->sizeof_stat == 8)
-			val = core_readq(priv, offset);
-		else
-			val = core_readl(priv, offset);
-
-		data[i] = (u64)val;
-	}
-
-	mutex_unlock(&priv->stats_mutex);
-}
-
-static int bcm_sf2_sw_get_sset_count(struct dsa_switch *ds)
-{
-	return BCM_SF2_STATS_SIZE;
-}
-
-static const char *bcm_sf2_sw_drv_probe(struct device *dsa_dev,
-					struct device *host_dev, int sw_addr,
-					void **_priv)
+static enum dsa_tag_protocol bcm_sf2_sw_get_tag_protocol(struct dsa_switch *ds)
 {
-	struct bcm_sf2_priv *priv;
-
-	priv = devm_kzalloc(dsa_dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return NULL;
-	*_priv = priv;
-
-	return "Broadcom Starfighter 2";
+	return DSA_TAG_PROTO_BRCM;
 }
 
 static void bcm_sf2_imp_vlan_setup(struct dsa_switch *ds, int cpu_port)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	unsigned int i;
 	u32 reg;
 
@@ -172,7 +63,7 @@ static void bcm_sf2_imp_vlan_setup(struct dsa_switch *ds, int cpu_port)
 
 static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	u32 reg, val;
 
 	/* Enable the port memories */
@@ -237,7 +128,7 @@ static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port)
 
 static void bcm_sf2_eee_enable_set(struct dsa_switch *ds, int port, bool enable)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	u32 reg;
 
 	reg = core_readl(priv, CORE_EEE_EN_CTRL);
@@ -250,7 +141,7 @@ static void bcm_sf2_eee_enable_set(struct dsa_switch *ds, int port, bool enable)
 
 static void bcm_sf2_gphy_enable_set(struct dsa_switch *ds, bool enable)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	u32 reg;
 
 	reg = reg_readl(priv, REG_SPHY_CNTRL);
@@ -324,7 +215,7 @@ static inline void bcm_sf2_port_intr_disable(struct bcm_sf2_priv *priv,
 static int bcm_sf2_port_setup(struct dsa_switch *ds, int port,
 			      struct phy_device *phy)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	s8 cpu_port = ds->dst[ds->index].cpu_port;
 	u32 reg;
 
@@ -365,7 +256,7 @@ static int bcm_sf2_port_setup(struct dsa_switch *ds, int port,
 	reg = core_readl(priv, CORE_PORT_VLAN_CTL_PORT(port));
 	reg &= ~PORT_VLAN_CTRL_MASK;
 	reg |= (1 << port);
-	reg |= priv->port_sts[port].vlan_ctl_mask;
+	reg |= priv->dev->ports[port].vlan_ctl_mask;
 	core_writel(priv, reg, CORE_PORT_VLAN_CTL_PORT(port));
 
 	bcm_sf2_imp_vlan_setup(ds, cpu_port);
@@ -380,7 +271,7 @@ static int bcm_sf2_port_setup(struct dsa_switch *ds, int port,
 static void bcm_sf2_port_disable(struct dsa_switch *ds, int port,
 				 struct phy_device *phy)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	u32 off, reg;
 
 	if (priv->wol_ports_mask & (1 << port))
@@ -412,7 +303,7 @@ static void bcm_sf2_port_disable(struct dsa_switch *ds, int port,
 static int bcm_sf2_eee_init(struct dsa_switch *ds, int port,
 			    struct phy_device *phy)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct ethtool_eee *p = &priv->port_sts[port].eee;
 	int ret;
 
@@ -430,7 +321,7 @@ static int bcm_sf2_eee_init(struct dsa_switch *ds, int port,
 static int bcm_sf2_sw_get_eee(struct dsa_switch *ds, int port,
 			      struct ethtool_eee *e)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct ethtool_eee *p = &priv->port_sts[port].eee;
 	u32 reg;
 
@@ -445,7 +336,7 @@ static int bcm_sf2_sw_set_eee(struct dsa_switch *ds, int port,
 			      struct phy_device *phydev,
 			      struct ethtool_eee *e)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct ethtool_eee *p = &priv->port_sts[port].eee;
 
 	p->eee_enabled = e->eee_enabled;
@@ -461,469 +352,6 @@ static int bcm_sf2_sw_set_eee(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static int bcm_sf2_fast_age_op(struct bcm_sf2_priv *priv)
-{
-	unsigned int timeout = 1000;
-	u32 reg;
-
-	reg = core_readl(priv, CORE_FAST_AGE_CTRL);
-	reg |= EN_AGE_PORT | EN_AGE_VLAN | EN_AGE_DYNAMIC | FAST_AGE_STR_DONE;
-	core_writel(priv, reg, CORE_FAST_AGE_CTRL);
-
-	do {
-		reg = core_readl(priv, CORE_FAST_AGE_CTRL);
-		if (!(reg & FAST_AGE_STR_DONE))
-			break;
-
-		cpu_relax();
-	} while (timeout--);
-
-	if (!timeout)
-		return -ETIMEDOUT;
-
-	core_writel(priv, 0, CORE_FAST_AGE_CTRL);
-
-	return 0;
-}
-
-/* Fast-ageing of ARL entries for a given port, equivalent to an ARL
- * flush for that port.
- */
-static int bcm_sf2_sw_fast_age_port(struct dsa_switch *ds, int port)
-{
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-
-	core_writel(priv, port, CORE_FAST_AGE_PORT);
-
-	return bcm_sf2_fast_age_op(priv);
-}
-
-static int bcm_sf2_sw_fast_age_vlan(struct bcm_sf2_priv *priv, u16 vid)
-{
-	core_writel(priv, vid, CORE_FAST_AGE_VID);
-
-	return bcm_sf2_fast_age_op(priv);
-}
-
-static int bcm_sf2_vlan_op_wait(struct bcm_sf2_priv *priv)
-{
-	unsigned int timeout = 10;
-	u32 reg;
-
-	do {
-		reg = core_readl(priv, CORE_ARLA_VTBL_RWCTRL);
-		if (!(reg & ARLA_VTBL_STDN))
-			return 0;
-
-		usleep_range(1000, 2000);
-	} while (timeout--);
-
-	return -ETIMEDOUT;
-}
-
-static int bcm_sf2_vlan_op(struct bcm_sf2_priv *priv, u8 op)
-{
-	core_writel(priv, ARLA_VTBL_STDN | op, CORE_ARLA_VTBL_RWCTRL);
-
-	return bcm_sf2_vlan_op_wait(priv);
-}
-
-static void bcm_sf2_set_vlan_entry(struct bcm_sf2_priv *priv, u16 vid,
-				   struct bcm_sf2_vlan *vlan)
-{
-	int ret;
-
-	core_writel(priv, vid & VTBL_ADDR_INDEX_MASK, CORE_ARLA_VTBL_ADDR);
-	core_writel(priv, vlan->untag << UNTAG_MAP_SHIFT | vlan->members,
-		    CORE_ARLA_VTBL_ENTRY);
-
-	ret = bcm_sf2_vlan_op(priv, ARLA_VTBL_CMD_WRITE);
-	if (ret)
-		pr_err("failed to write VLAN entry\n");
-}
-
-static int bcm_sf2_get_vlan_entry(struct bcm_sf2_priv *priv, u16 vid,
-				  struct bcm_sf2_vlan *vlan)
-{
-	u32 entry;
-	int ret;
-
-	core_writel(priv, vid & VTBL_ADDR_INDEX_MASK, CORE_ARLA_VTBL_ADDR);
-
-	ret = bcm_sf2_vlan_op(priv, ARLA_VTBL_CMD_READ);
-	if (ret)
-		return ret;
-
-	entry = core_readl(priv, CORE_ARLA_VTBL_ENTRY);
-	vlan->members = entry & FWD_MAP_MASK;
-	vlan->untag = (entry >> UNTAG_MAP_SHIFT) & UNTAG_MAP_MASK;
-
-	return 0;
-}
-
-static int bcm_sf2_sw_br_join(struct dsa_switch *ds, int port,
-			      struct net_device *bridge)
-{
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-	s8 cpu_port = ds->dst->cpu_port;
-	unsigned int i;
-	u32 reg, p_ctl;
-
-	/* Make this port leave the all VLANs join since we will have proper
-	 * VLAN entries from now on
-	 */
-	reg = core_readl(priv, CORE_JOIN_ALL_VLAN_EN);
-	reg &= ~BIT(port);
-	if ((reg & BIT(cpu_port)) == BIT(cpu_port))
-		reg &= ~BIT(cpu_port);
-	core_writel(priv, reg, CORE_JOIN_ALL_VLAN_EN);
-
-	priv->port_sts[port].bridge_dev = bridge;
-	p_ctl = core_readl(priv, CORE_PORT_VLAN_CTL_PORT(port));
-
-	for (i = 0; i < priv->hw_params.num_ports; i++) {
-		if (priv->port_sts[i].bridge_dev != bridge)
-			continue;
-
-		/* Add this local port to the remote port VLAN control
-		 * membership and update the remote port bitmask
-		 */
-		reg = core_readl(priv, CORE_PORT_VLAN_CTL_PORT(i));
-		reg |= 1 << port;
-		core_writel(priv, reg, CORE_PORT_VLAN_CTL_PORT(i));
-		priv->port_sts[i].vlan_ctl_mask = reg;
-
-		p_ctl |= 1 << i;
-	}
-
-	/* Configure the local port VLAN control membership to include
-	 * remote ports and update the local port bitmask
-	 */
-	core_writel(priv, p_ctl, CORE_PORT_VLAN_CTL_PORT(port));
-	priv->port_sts[port].vlan_ctl_mask = p_ctl;
-
-	return 0;
-}
-
-static void bcm_sf2_sw_br_leave(struct dsa_switch *ds, int port)
-{
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-	struct net_device *bridge = priv->port_sts[port].bridge_dev;
-	s8 cpu_port = ds->dst->cpu_port;
-	unsigned int i;
-	u32 reg, p_ctl;
-
-	p_ctl = core_readl(priv, CORE_PORT_VLAN_CTL_PORT(port));
-
-	for (i = 0; i < priv->hw_params.num_ports; i++) {
-		/* Don't touch the remaining ports */
-		if (priv->port_sts[i].bridge_dev != bridge)
-			continue;
-
-		reg = core_readl(priv, CORE_PORT_VLAN_CTL_PORT(i));
-		reg &= ~(1 << port);
-		core_writel(priv, reg, CORE_PORT_VLAN_CTL_PORT(i));
-		priv->port_sts[port].vlan_ctl_mask = reg;
-
-		/* Prevent self removal to preserve isolation */
-		if (port != i)
-			p_ctl &= ~(1 << i);
-	}
-
-	core_writel(priv, p_ctl, CORE_PORT_VLAN_CTL_PORT(port));
-	priv->port_sts[port].vlan_ctl_mask = p_ctl;
-	priv->port_sts[port].bridge_dev = NULL;
-
-	/* Make this port join all VLANs without VLAN entries */
-	reg = core_readl(priv, CORE_JOIN_ALL_VLAN_EN);
-	reg |= BIT(port);
-	if (!(reg & BIT(cpu_port)))
-		reg |= BIT(cpu_port);
-	core_writel(priv, reg, CORE_JOIN_ALL_VLAN_EN);
-}
-
-static void bcm_sf2_sw_br_set_stp_state(struct dsa_switch *ds, int port,
-					u8 state)
-{
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-	u8 hw_state, cur_hw_state;
-	u32 reg;
-
-	reg = core_readl(priv, CORE_G_PCTL_PORT(port));
-	cur_hw_state = reg & (G_MISTP_STATE_MASK << G_MISTP_STATE_SHIFT);
-
-	switch (state) {
-	case BR_STATE_DISABLED:
-		hw_state = G_MISTP_DIS_STATE;
-		break;
-	case BR_STATE_LISTENING:
-		hw_state = G_MISTP_LISTEN_STATE;
-		break;
-	case BR_STATE_LEARNING:
-		hw_state = G_MISTP_LEARN_STATE;
-		break;
-	case BR_STATE_FORWARDING:
-		hw_state = G_MISTP_FWD_STATE;
-		break;
-	case BR_STATE_BLOCKING:
-		hw_state = G_MISTP_BLOCK_STATE;
-		break;
-	default:
-		pr_err("%s: invalid STP state: %d\n", __func__, state);
-		return;
-	}
-
-	/* Fast-age ARL entries if we are moving a port from Learning or
-	 * Forwarding (cur_hw_state) state to Disabled, Blocking or Listening
-	 * state (hw_state)
-	 */
-	if (cur_hw_state != hw_state) {
-		if (cur_hw_state >= G_MISTP_LEARN_STATE &&
-		    hw_state <= G_MISTP_LISTEN_STATE) {
-			if (bcm_sf2_sw_fast_age_port(ds, port)) {
-				pr_err("%s: fast-ageing failed\n", __func__);
-				return;
-			}
-		}
-	}
-
-	reg = core_readl(priv, CORE_G_PCTL_PORT(port));
-	reg &= ~(G_MISTP_STATE_MASK << G_MISTP_STATE_SHIFT);
-	reg |= hw_state;
-	core_writel(priv, reg, CORE_G_PCTL_PORT(port));
-}
-
-/* Address Resolution Logic routines */
-static int bcm_sf2_arl_op_wait(struct bcm_sf2_priv *priv)
-{
-	unsigned int timeout = 10;
-	u32 reg;
-
-	do {
-		reg = core_readl(priv, CORE_ARLA_RWCTL);
-		if (!(reg & ARL_STRTDN))
-			return 0;
-
-		usleep_range(1000, 2000);
-	} while (timeout--);
-
-	return -ETIMEDOUT;
-}
-
-static int bcm_sf2_arl_rw_op(struct bcm_sf2_priv *priv, unsigned int op)
-{
-	u32 cmd;
-
-	if (op > ARL_RW)
-		return -EINVAL;
-
-	cmd = core_readl(priv, CORE_ARLA_RWCTL);
-	cmd &= ~IVL_SVL_SELECT;
-	cmd |= ARL_STRTDN;
-	if (op)
-		cmd |= ARL_RW;
-	else
-		cmd &= ~ARL_RW;
-	core_writel(priv, cmd, CORE_ARLA_RWCTL);
-
-	return bcm_sf2_arl_op_wait(priv);
-}
-
-static int bcm_sf2_arl_read(struct bcm_sf2_priv *priv, u64 mac,
-			    u16 vid, struct bcm_sf2_arl_entry *ent, u8 *idx,
-			    bool is_valid)
-{
-	unsigned int i;
-	int ret;
-
-	ret = bcm_sf2_arl_op_wait(priv);
-	if (ret)
-		return ret;
-
-	/* Read the 4 bins */
-	for (i = 0; i < 4; i++) {
-		u64 mac_vid;
-		u32 fwd_entry;
-
-		mac_vid = core_readq(priv, CORE_ARLA_MACVID_ENTRY(i));
-		fwd_entry = core_readl(priv, CORE_ARLA_FWD_ENTRY(i));
-		bcm_sf2_arl_to_entry(ent, mac_vid, fwd_entry);
-
-		if (ent->is_valid && is_valid) {
-			*idx = i;
-			return 0;
-		}
-
-		/* This is the MAC we just deleted */
-		if (!is_valid && (mac_vid & mac))
-			return 0;
-	}
-
-	return -ENOENT;
-}
-
-static int bcm_sf2_arl_op(struct bcm_sf2_priv *priv, int op, int port,
-			  const unsigned char *addr, u16 vid, bool is_valid)
-{
-	struct bcm_sf2_arl_entry ent;
-	u32 fwd_entry;
-	u64 mac, mac_vid = 0;
-	u8 idx = 0;
-	int ret;
-
-	/* Convert the array into a 64-bit MAC */
-	mac = bcm_sf2_mac_to_u64(addr);
-
-	/* Perform a read for the given MAC and VID */
-	core_writeq(priv, mac, CORE_ARLA_MAC);
-	core_writel(priv, vid, CORE_ARLA_VID);
-
-	/* Issue a read operation for this MAC */
-	ret = bcm_sf2_arl_rw_op(priv, 1);
-	if (ret)
-		return ret;
-
-	ret = bcm_sf2_arl_read(priv, mac, vid, &ent, &idx, is_valid);
-	/* If this is a read, just finish now */
-	if (op)
-		return ret;
-
-	/* We could not find a matching MAC, so reset to a new entry */
-	if (ret) {
-		fwd_entry = 0;
-		idx = 0;
-	}
-
-	memset(&ent, 0, sizeof(ent));
-	ent.port = port;
-	ent.is_valid = is_valid;
-	ent.vid = vid;
-	ent.is_static = true;
-	memcpy(ent.mac, addr, ETH_ALEN);
-	bcm_sf2_arl_from_entry(&mac_vid, &fwd_entry, &ent);
-
-	core_writeq(priv, mac_vid, CORE_ARLA_MACVID_ENTRY(idx));
-	core_writel(priv, fwd_entry, CORE_ARLA_FWD_ENTRY(idx));
-
-	ret = bcm_sf2_arl_rw_op(priv, 0);
-	if (ret)
-		return ret;
-
-	/* Re-read the entry to check */
-	return bcm_sf2_arl_read(priv, mac, vid, &ent, &idx, is_valid);
-}
-
-static int bcm_sf2_sw_fdb_prepare(struct dsa_switch *ds, int port,
-				  const struct switchdev_obj_port_fdb *fdb,
-				  struct switchdev_trans *trans)
-{
-	/* We do not need to do anything specific here yet */
-	return 0;
-}
-
-static void bcm_sf2_sw_fdb_add(struct dsa_switch *ds, int port,
-			       const struct switchdev_obj_port_fdb *fdb,
-			       struct switchdev_trans *trans)
-{
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-
-	if (bcm_sf2_arl_op(priv, 0, port, fdb->addr, fdb->vid, true))
-		pr_err("%s: failed to add MAC address\n", __func__);
-}
-
-static int bcm_sf2_sw_fdb_del(struct dsa_switch *ds, int port,
-			      const struct switchdev_obj_port_fdb *fdb)
-{
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-
-	return bcm_sf2_arl_op(priv, 0, port, fdb->addr, fdb->vid, false);
-}
-
-static int bcm_sf2_arl_search_wait(struct bcm_sf2_priv *priv)
-{
-	unsigned timeout = 1000;
-	u32 reg;
-
-	do {
-		reg = core_readl(priv, CORE_ARLA_SRCH_CTL);
-		if (!(reg & ARLA_SRCH_STDN))
-			return 0;
-
-		if (reg & ARLA_SRCH_VLID)
-			return 0;
-
-		usleep_range(1000, 2000);
-	} while (timeout--);
-
-	return -ETIMEDOUT;
-}
-
-static void bcm_sf2_arl_search_rd(struct bcm_sf2_priv *priv, u8 idx,
-				  struct bcm_sf2_arl_entry *ent)
-{
-	u64 mac_vid;
-	u32 fwd_entry;
-
-	mac_vid = core_readq(priv, CORE_ARLA_SRCH_RSLT_MACVID(idx));
-	fwd_entry = core_readl(priv, CORE_ARLA_SRCH_RSLT(idx));
-	bcm_sf2_arl_to_entry(ent, mac_vid, fwd_entry);
-}
-
-static int bcm_sf2_sw_fdb_copy(struct net_device *dev, int port,
-			       const struct bcm_sf2_arl_entry *ent,
-			       struct switchdev_obj_port_fdb *fdb,
-			       int (*cb)(struct switchdev_obj *obj))
-{
-	if (!ent->is_valid)
-		return 0;
-
-	if (port != ent->port)
-		return 0;
-
-	ether_addr_copy(fdb->addr, ent->mac);
-	fdb->vid = ent->vid;
-	fdb->ndm_state = ent->is_static ? NUD_NOARP : NUD_REACHABLE;
-
-	return cb(&fdb->obj);
-}
-
-static int bcm_sf2_sw_fdb_dump(struct dsa_switch *ds, int port,
-			       struct switchdev_obj_port_fdb *fdb,
-			       int (*cb)(struct switchdev_obj *obj))
-{
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-	struct net_device *dev = ds->ports[port].netdev;
-	struct bcm_sf2_arl_entry results[2];
-	unsigned int count = 0;
-	int ret;
-
-	/* Start search operation */
-	core_writel(priv, ARLA_SRCH_STDN, CORE_ARLA_SRCH_CTL);
-
-	do {
-		ret = bcm_sf2_arl_search_wait(priv);
-		if (ret)
-			return ret;
-
-		/* Read both entries, then return their values back */
-		bcm_sf2_arl_search_rd(priv, 0, &results[0]);
-		ret = bcm_sf2_sw_fdb_copy(dev, port, &results[0], fdb, cb);
-		if (ret)
-			return ret;
-
-		bcm_sf2_arl_search_rd(priv, 1, &results[1]);
-		ret = bcm_sf2_sw_fdb_copy(dev, port, &results[1], fdb, cb);
-		if (ret)
-			return ret;
-
-		if (!results[0].is_valid && !results[1].is_valid)
-			break;
-
-	} while (count++ < CORE_ARLA_NUM_ENTRIES);
-
-	return 0;
-}
-
 static int bcm_sf2_sw_indir_rw(struct bcm_sf2_priv *priv, int op, int addr,
 			       int regnum, u16 val)
 {
@@ -1036,12 +464,10 @@ static int bcm_sf2_sw_rst(struct bcm_sf2_priv *priv)
 
 static void bcm_sf2_intr_disable(struct bcm_sf2_priv *priv)
 {
-	intrl2_0_writel(priv, 0xffffffff, INTRL2_CPU_MASK_SET);
+	intrl2_0_mask_set(priv, 0xffffffff);
 	intrl2_0_writel(priv, 0xffffffff, INTRL2_CPU_CLEAR);
-	intrl2_0_writel(priv, 0, INTRL2_CPU_MASK_CLEAR);
-	intrl2_1_writel(priv, 0xffffffff, INTRL2_CPU_MASK_SET);
+	intrl2_1_mask_set(priv, 0xffffffff);
 	intrl2_1_writel(priv, 0xffffffff, INTRL2_CPU_CLEAR);
-	intrl2_1_writel(priv, 0, INTRL2_CPU_MASK_CLEAR);
 }
 
 static void bcm_sf2_identify_ports(struct bcm_sf2_priv *priv,
@@ -1082,7 +508,7 @@ static void bcm_sf2_identify_ports(struct bcm_sf2_priv *priv,
 
 static int bcm_sf2_mdio_register(struct dsa_switch *ds)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct device_node *dn;
 	static int index;
 	int err;
@@ -1146,14 +572,9 @@ static void bcm_sf2_mdio_unregister(struct bcm_sf2_priv *priv)
 		of_node_put(priv->master_mii_dn);
 }
 
-static int bcm_sf2_sw_set_addr(struct dsa_switch *ds, u8 *addr)
-{
-	return 0;
-}
-
 static u32 bcm_sf2_sw_get_phy_flags(struct dsa_switch *ds, int port)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 
 	/* The BCM7xxx PHY driver expects to find the integrated PHY revision
 	 * in bits 15:8 and the patch level in bits 7:0 which is exactly what
@@ -1166,7 +587,7 @@ static u32 bcm_sf2_sw_get_phy_flags(struct dsa_switch *ds, int port)
 static void bcm_sf2_sw_adjust_link(struct dsa_switch *ds, int port,
 				   struct phy_device *phydev)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	u32 id_mode_dis = 0, port_mode;
 	const char *str = NULL;
 	u32 reg;
@@ -1246,7 +667,7 @@ force_link:
 static void bcm_sf2_sw_fixed_link_update(struct dsa_switch *ds, int port,
 					 struct fixed_phy_status *status)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	u32 duplex, pause;
 	u32 reg;
 
@@ -1298,7 +719,7 @@ static void bcm_sf2_sw_fixed_link_update(struct dsa_switch *ds, int port,
 
 static int bcm_sf2_sw_suspend(struct dsa_switch *ds)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	unsigned int port;
 
 	bcm_sf2_intr_disable(priv);
@@ -1318,7 +739,7 @@ static int bcm_sf2_sw_suspend(struct dsa_switch *ds)
 
 static int bcm_sf2_sw_resume(struct dsa_switch *ds)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	unsigned int port;
 	int ret;
 
@@ -1345,7 +766,7 @@ static void bcm_sf2_sw_get_wol(struct dsa_switch *ds, int port,
 			       struct ethtool_wolinfo *wol)
 {
 	struct net_device *p = ds->dst[ds->index].master_netdev;
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct ethtool_wolinfo pwol;
 
 	/* Get the parent device WoL settings */
@@ -1368,7 +789,7 @@ static int bcm_sf2_sw_set_wol(struct dsa_switch *ds, int port,
 			      struct ethtool_wolinfo *wol)
 {
 	struct net_device *p = ds->dst[ds->index].master_netdev;
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	s8 cpu_port = ds->dst[ds->index].cpu_port;
 	struct ethtool_wolinfo pwol;
 
@@ -1393,43 +814,32 @@ static int bcm_sf2_sw_set_wol(struct dsa_switch *ds, int port,
 	return p->ethtool_ops->set_wol(p, wol);
 }
 
-static void bcm_sf2_enable_vlan(struct bcm_sf2_priv *priv, bool enable)
+static int bcm_sf2_vlan_op_wait(struct bcm_sf2_priv *priv)
 {
-	u32 mgmt, vc0, vc1, vc4, vc5;
+	unsigned int timeout = 10;
+	u32 reg;
 
-	mgmt = core_readl(priv, CORE_SWMODE);
-	vc0 = core_readl(priv, CORE_VLAN_CTRL0);
-	vc1 = core_readl(priv, CORE_VLAN_CTRL1);
-	vc4 = core_readl(priv, CORE_VLAN_CTRL4);
-	vc5 = core_readl(priv, CORE_VLAN_CTRL5);
+	do {
+		reg = core_readl(priv, CORE_ARLA_VTBL_RWCTRL);
+		if (!(reg & ARLA_VTBL_STDN))
+			return 0;
 
-	mgmt &= ~SW_FWDG_MODE;
+		usleep_range(1000, 2000);
+	} while (timeout--);
 
-	if (enable) {
-		vc0 |= VLAN_EN | VLAN_LEARN_MODE_IVL;
-		vc1 |= EN_RSV_MCAST_UNTAG | EN_RSV_MCAST_FWDMAP;
-		vc4 &= ~(INGR_VID_CHK_MASK << INGR_VID_CHK_SHIFT);
-		vc4 |= INGR_VID_CHK_DROP;
-		vc5 |= DROP_VTABLE_MISS | EN_VID_FFF_FWD;
-	} else {
-		vc0 &= ~(VLAN_EN | VLAN_LEARN_MODE_IVL);
-		vc1 &= ~(EN_RSV_MCAST_UNTAG | EN_RSV_MCAST_FWDMAP);
-		vc4 &= ~(INGR_VID_CHK_MASK << INGR_VID_CHK_SHIFT);
-		vc5 &= ~(DROP_VTABLE_MISS | EN_VID_FFF_FWD);
-		vc4 |= INGR_VID_CHK_VID_VIOL_IMP;
-	}
+	return -ETIMEDOUT;
+}
+
+static int bcm_sf2_vlan_op(struct bcm_sf2_priv *priv, u8 op)
+{
+	core_writel(priv, ARLA_VTBL_STDN | op, CORE_ARLA_VTBL_RWCTRL);
 
-	core_writel(priv, vc0, CORE_VLAN_CTRL0);
-	core_writel(priv, vc1, CORE_VLAN_CTRL1);
-	core_writel(priv, 0, CORE_VLAN_CTRL3);
-	core_writel(priv, vc4, CORE_VLAN_CTRL4);
-	core_writel(priv, vc5, CORE_VLAN_CTRL5);
-	core_writel(priv, mgmt, CORE_SWMODE);
+	return bcm_sf2_vlan_op_wait(priv);
 }
 
 static void bcm_sf2_sw_configure_vlan(struct dsa_switch *ds)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	unsigned int port;
 
 	/* Clear all VLANs */
@@ -1443,162 +853,199 @@ static void bcm_sf2_sw_configure_vlan(struct dsa_switch *ds)
 	}
 }
 
-static int bcm_sf2_sw_vlan_filtering(struct dsa_switch *ds, int port,
-				     bool vlan_filtering)
+static int bcm_sf2_sw_setup(struct dsa_switch *ds)
 {
+	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
+	unsigned int port;
+
+	/* Enable all valid ports and disable those unused */
+	for (port = 0; port < priv->hw_params.num_ports; port++) {
+		/* IMP port receives special treatment */
+		if ((1 << port) & ds->enabled_port_mask)
+			bcm_sf2_port_setup(ds, port, NULL);
+		else if (dsa_is_cpu_port(ds, port))
+			bcm_sf2_imp_setup(ds, port);
+		else
+			bcm_sf2_port_disable(ds, port, NULL);
+	}
+
+	bcm_sf2_sw_configure_vlan(ds);
+
 	return 0;
 }
 
-static int bcm_sf2_sw_vlan_prepare(struct dsa_switch *ds, int port,
-				   const struct switchdev_obj_port_vlan *vlan,
-				   struct switchdev_trans *trans)
+/* The SWITCH_CORE register space is managed by b53 but operates on a page +
+ * register basis so we need to translate that into an address that the
+ * bus-glue understands.
+ */
+#define SF2_PAGE_REG_MKADDR(page, reg)	((page) << 10 | (reg) << 2)
+
+static int bcm_sf2_core_read8(struct b53_device *dev, u8 page, u8 reg,
+			      u8 *val)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
+	struct bcm_sf2_priv *priv = dev->priv;
 
-	bcm_sf2_enable_vlan(priv, true);
+	*val = core_readl(priv, SF2_PAGE_REG_MKADDR(page, reg));
 
 	return 0;
 }
 
-static void bcm_sf2_sw_vlan_add(struct dsa_switch *ds, int port,
-				const struct switchdev_obj_port_vlan *vlan,
-				struct switchdev_trans *trans)
+static int bcm_sf2_core_read16(struct b53_device *dev, u8 page, u8 reg,
+			       u16 *val)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
-	bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID;
-	s8 cpu_port = ds->dst->cpu_port;
-	struct bcm_sf2_vlan *vl;
-	u16 vid;
+	struct bcm_sf2_priv *priv = dev->priv;
 
-	for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid) {
-		vl = &priv->vlans[vid];
+	*val = core_readl(priv, SF2_PAGE_REG_MKADDR(page, reg));
 
-		bcm_sf2_get_vlan_entry(priv, vid, vl);
+	return 0;
+}
 
-		vl->members |= BIT(port) | BIT(cpu_port);
-		if (untagged)
-			vl->untag |= BIT(port) | BIT(cpu_port);
-		else
-			vl->untag &= ~(BIT(port) | BIT(cpu_port));
+static int bcm_sf2_core_read32(struct b53_device *dev, u8 page, u8 reg,
+			       u32 *val)
+{
+	struct bcm_sf2_priv *priv = dev->priv;
 
-		bcm_sf2_set_vlan_entry(priv, vid, vl);
-		bcm_sf2_sw_fast_age_vlan(priv, vid);
-	}
+	*val = core_readl(priv, SF2_PAGE_REG_MKADDR(page, reg));
 
-	if (pvid) {
-		core_writel(priv, vlan->vid_end, CORE_DEFAULT_1Q_TAG_P(port));
-		core_writel(priv, vlan->vid_end,
-			    CORE_DEFAULT_1Q_TAG_P(cpu_port));
-		bcm_sf2_sw_fast_age_vlan(priv, vid);
-	}
+	return 0;
 }
 
-static int bcm_sf2_sw_vlan_del(struct dsa_switch *ds, int port,
-			       const struct switchdev_obj_port_vlan *vlan)
+static int bcm_sf2_core_read64(struct b53_device *dev, u8 page, u8 reg,
+			       u64 *val)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
-	s8 cpu_port = ds->dst->cpu_port;
-	struct bcm_sf2_vlan *vl;
-	u16 vid, pvid;
-	int ret;
+	struct bcm_sf2_priv *priv = dev->priv;
 
-	pvid = core_readl(priv, CORE_DEFAULT_1Q_TAG_P(port));
+	*val = core_readq(priv, SF2_PAGE_REG_MKADDR(page, reg));
 
-	for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid) {
-		vl = &priv->vlans[vid];
-
-		ret = bcm_sf2_get_vlan_entry(priv, vid, vl);
-		if (ret)
-			return ret;
-
-		vl->members &= ~BIT(port);
-		if ((vl->members & BIT(cpu_port)) == BIT(cpu_port))
-			vl->members = 0;
-		if (pvid == vid)
-			pvid = 0;
-		if (untagged) {
-			vl->untag &= ~BIT(port);
-			if ((vl->untag & BIT(port)) == BIT(cpu_port))
-				vl->untag = 0;
-		}
+	return 0;
+}
 
-		bcm_sf2_set_vlan_entry(priv, vid, vl);
-		bcm_sf2_sw_fast_age_vlan(priv, vid);
-	}
+static int bcm_sf2_core_write8(struct b53_device *dev, u8 page, u8 reg,
+			       u8 value)
+{
+	struct bcm_sf2_priv *priv = dev->priv;
 
-	core_writel(priv, pvid, CORE_DEFAULT_1Q_TAG_P(port));
-	core_writel(priv, pvid, CORE_DEFAULT_1Q_TAG_P(cpu_port));
-	bcm_sf2_sw_fast_age_vlan(priv, vid);
+	core_writel(priv, value, SF2_PAGE_REG_MKADDR(page, reg));
 
 	return 0;
 }
 
-static int bcm_sf2_sw_vlan_dump(struct dsa_switch *ds, int port,
-				struct switchdev_obj_port_vlan *vlan,
-				int (*cb)(struct switchdev_obj *obj))
+static int bcm_sf2_core_write16(struct b53_device *dev, u8 page, u8 reg,
+				u16 value)
 {
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-	struct bcm_sf2_port_status *p = &priv->port_sts[port];
-	struct bcm_sf2_vlan *vl;
-	u16 vid, pvid;
-	int err = 0;
+	struct bcm_sf2_priv *priv = dev->priv;
 
-	pvid = core_readl(priv, CORE_DEFAULT_1Q_TAG_P(port));
+	core_writel(priv, value, SF2_PAGE_REG_MKADDR(page, reg));
 
-	for (vid = 0; vid < VLAN_N_VID; vid++) {
-		vl = &priv->vlans[vid];
+	return 0;
+}
 
-		if (!(vl->members & BIT(port)))
-			continue;
+static int bcm_sf2_core_write32(struct b53_device *dev, u8 page, u8 reg,
+				u32 value)
+{
+	struct bcm_sf2_priv *priv = dev->priv;
 
-		vlan->vid_begin = vlan->vid_end = vid;
-		vlan->flags = 0;
+	core_writel(priv, value, SF2_PAGE_REG_MKADDR(page, reg));
 
-		if (vl->untag & BIT(port))
-			vlan->flags |= BRIDGE_VLAN_INFO_UNTAGGED;
-		if (p->pvid == vid)
-			vlan->flags |= BRIDGE_VLAN_INFO_PVID;
+	return 0;
+}
 
-		err = cb(&vlan->obj);
-		if (err)
-			break;
-	}
+static int bcm_sf2_core_write64(struct b53_device *dev, u8 page, u8 reg,
+				u64 value)
+{
+	struct bcm_sf2_priv *priv = dev->priv;
 
-	return err;
+	core_writeq(priv, value, SF2_PAGE_REG_MKADDR(page, reg));
+
+	return 0;
 }
 
-static int bcm_sf2_sw_setup(struct dsa_switch *ds)
+static struct b53_io_ops bcm_sf2_io_ops = {
+	.read8	= bcm_sf2_core_read8,
+	.read16	= bcm_sf2_core_read16,
+	.read32	= bcm_sf2_core_read32,
+	.read48	= bcm_sf2_core_read64,
+	.read64	= bcm_sf2_core_read64,
+	.write8	= bcm_sf2_core_write8,
+	.write16 = bcm_sf2_core_write16,
+	.write32 = bcm_sf2_core_write32,
+	.write48 = bcm_sf2_core_write64,
+	.write64 = bcm_sf2_core_write64,
+};
+
+static int bcm_sf2_sw_probe(struct platform_device *pdev)
 {
 	const char *reg_names[BCM_SF2_REGS_NUM] = BCM_SF2_REGS_NAME;
-	struct bcm_sf2_priv *priv = ds_to_priv(ds);
-	struct device_node *dn;
+	struct device_node *dn = pdev->dev.of_node;
+	struct b53_platform_data *pdata;
+	struct bcm_sf2_priv *priv;
+	struct b53_device *dev;
+	struct dsa_switch *ds;
 	void __iomem **base;
-	unsigned int port;
+	struct resource *r;
 	unsigned int i;
 	u32 reg, rev;
 	int ret;
 
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	dev = b53_switch_alloc(&pdev->dev, &bcm_sf2_io_ops, priv);
+	if (!dev)
+		return -ENOMEM;
+
+	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+	if (!pdata)
+		return -ENOMEM;
+
+	/* Auto-detection using standard registers will not work, so
+	 * provide an indication of what kind of device we are for
+	 * b53_common to work with
+	 */
+	pdata->chip_id = BCM7445_DEVICE_ID;
+	dev->pdata = pdata;
+
+	priv->dev = dev;
+	ds = dev->ds;
+
+	/* Override the parts that are non-standard wrt. normal b53 devices */
+	ds->ops->get_tag_protocol = bcm_sf2_sw_get_tag_protocol;
+	ds->ops->setup = bcm_sf2_sw_setup;
+	ds->ops->get_phy_flags = bcm_sf2_sw_get_phy_flags;
+	ds->ops->adjust_link = bcm_sf2_sw_adjust_link;
+	ds->ops->fixed_link_update = bcm_sf2_sw_fixed_link_update;
+	ds->ops->suspend = bcm_sf2_sw_suspend;
+	ds->ops->resume = bcm_sf2_sw_resume;
+	ds->ops->get_wol = bcm_sf2_sw_get_wol;
+	ds->ops->set_wol = bcm_sf2_sw_set_wol;
+	ds->ops->port_enable = bcm_sf2_port_setup;
+	ds->ops->port_disable = bcm_sf2_port_disable;
+	ds->ops->get_eee = bcm_sf2_sw_get_eee;
+	ds->ops->set_eee = bcm_sf2_sw_set_eee;
+
+	/* Avoid having DSA free our slave MDIO bus (checking for
+	 * ds->slave_mii_bus and ds->ops->phy_read being non-NULL)
+	 */
+	ds->ops->phy_read = NULL;
+
+	dev_set_drvdata(&pdev->dev, priv);
+
 	spin_lock_init(&priv->indir_lock);
 	mutex_init(&priv->stats_mutex);
 
-	/* All the interesting properties are at the parent device_node
-	 * level
-	 */
-	dn = ds->cd->of_node->parent;
-	bcm_sf2_identify_ports(priv, ds->cd->of_node);
+	bcm_sf2_identify_ports(priv, dn->child);
 
 	priv->irq0 = irq_of_parse_and_map(dn, 0);
 	priv->irq1 = irq_of_parse_and_map(dn, 1);
 
 	base = &priv->core;
 	for (i = 0; i < BCM_SF2_REGS_NUM; i++) {
-		*base = of_iomap(dn, i);
-		if (*base == NULL) {
+		r = platform_get_resource(pdev, IORESOURCE_MEM, i);
+		*base = devm_ioremap_resource(&pdev->dev, r);
+		if (IS_ERR(*base)) {
 			pr_err("unable to find register: %s\n", reg_names[i]);
-			ret = -ENOMEM;
-			goto out_unmap;
+			return PTR_ERR(*base);
 		}
 		base++;
 	}
@@ -1606,30 +1053,30 @@ static int bcm_sf2_sw_setup(struct dsa_switch *ds)
 	ret = bcm_sf2_sw_rst(priv);
 	if (ret) {
 		pr_err("unable to software reset switch: %d\n", ret);
-		goto out_unmap;
+		return ret;
 	}
 
 	ret = bcm_sf2_mdio_register(ds);
 	if (ret) {
 		pr_err("failed to register MDIO bus\n");
-		goto out_unmap;
+		return ret;
 	}
 
 	/* Disable all interrupts and request them */
 	bcm_sf2_intr_disable(priv);
 
-	ret = request_irq(priv->irq0, bcm_sf2_switch_0_isr, 0,
-			  "switch_0", priv);
+	ret = devm_request_irq(&pdev->dev, priv->irq0, bcm_sf2_switch_0_isr, 0,
+			       "switch_0", priv);
 	if (ret < 0) {
 		pr_err("failed to request switch_0 IRQ\n");
 		goto out_mdio;
 	}
 
-	ret = request_irq(priv->irq1, bcm_sf2_switch_1_isr, 0,
-			  "switch_1", priv);
+	ret = devm_request_irq(&pdev->dev, priv->irq1, bcm_sf2_switch_1_isr, 0,
+			       "switch_1", priv);
 	if (ret < 0) {
 		pr_err("failed to request switch_1 IRQ\n");
-		goto out_free_irq0;
+		goto out_mdio;
 	}
 
 	/* Reset the MIB counters */
@@ -1649,19 +1096,6 @@ static int bcm_sf2_sw_setup(struct dsa_switch *ds)
 				 &priv->hw_params.num_gphy))
 		priv->hw_params.num_gphy = 1;
 
-	/* Enable all valid ports and disable those unused */
-	for (port = 0; port < priv->hw_params.num_ports; port++) {
-		/* IMP port receives special treatment */
-		if ((1 << port) & ds->enabled_port_mask)
-			bcm_sf2_port_setup(ds, port, NULL);
-		else if (dsa_is_cpu_port(ds, port))
-			bcm_sf2_imp_setup(ds, port);
-		else
-			bcm_sf2_port_disable(ds, port, NULL);
-	}
-
-	bcm_sf2_sw_configure_vlan(ds);
-
 	rev = reg_readl(priv, REG_SWITCH_REVISION);
 	priv->hw_params.top_rev = (rev >> SWITCH_TOP_REV_SHIFT) &
 					SWITCH_TOP_REV_MASK;
@@ -1670,6 +1104,10 @@ static int bcm_sf2_sw_setup(struct dsa_switch *ds)
 	rev = reg_readl(priv, REG_PHY_REVISION);
 	priv->hw_params.gphy_rev = rev & PHY_REVISION_MASK;
 
+	ret = b53_switch_register(dev);
+	if (ret)
+		goto out_mdio;
+
 	pr_info("Starfighter 2 top: %x.%02x, core: %x.%02x base: 0x%p, IRQs: %d, %d\n",
 		priv->hw_params.top_rev >> 8, priv->hw_params.top_rev & 0xff,
 		priv->hw_params.core_rev >> 8, priv->hw_params.core_rev & 0xff,
@@ -1677,66 +1115,60 @@ static int bcm_sf2_sw_setup(struct dsa_switch *ds)
 
 	return 0;
 
-out_free_irq0:
-	free_irq(priv->irq0, priv);
 out_mdio:
 	bcm_sf2_mdio_unregister(priv);
-out_unmap:
-	base = &priv->core;
-	for (i = 0; i < BCM_SF2_REGS_NUM; i++) {
-		if (*base)
-			iounmap(*base);
-		base++;
-	}
 	return ret;
 }
 
-static struct dsa_switch_driver bcm_sf2_switch_driver = {
-	.tag_protocol		= DSA_TAG_PROTO_BRCM,
-	.probe			= bcm_sf2_sw_drv_probe,
-	.setup			= bcm_sf2_sw_setup,
-	.set_addr		= bcm_sf2_sw_set_addr,
-	.get_phy_flags		= bcm_sf2_sw_get_phy_flags,
-	.get_strings		= bcm_sf2_sw_get_strings,
-	.get_ethtool_stats	= bcm_sf2_sw_get_ethtool_stats,
-	.get_sset_count		= bcm_sf2_sw_get_sset_count,
-	.adjust_link		= bcm_sf2_sw_adjust_link,
-	.fixed_link_update	= bcm_sf2_sw_fixed_link_update,
-	.suspend		= bcm_sf2_sw_suspend,
-	.resume			= bcm_sf2_sw_resume,
-	.get_wol		= bcm_sf2_sw_get_wol,
-	.set_wol		= bcm_sf2_sw_set_wol,
-	.port_enable		= bcm_sf2_port_setup,
-	.port_disable		= bcm_sf2_port_disable,
-	.get_eee		= bcm_sf2_sw_get_eee,
-	.set_eee		= bcm_sf2_sw_set_eee,
-	.port_bridge_join	= bcm_sf2_sw_br_join,
-	.port_bridge_leave	= bcm_sf2_sw_br_leave,
-	.port_stp_state_set	= bcm_sf2_sw_br_set_stp_state,
-	.port_fdb_prepare	= bcm_sf2_sw_fdb_prepare,
-	.port_fdb_add		= bcm_sf2_sw_fdb_add,
-	.port_fdb_del		= bcm_sf2_sw_fdb_del,
-	.port_fdb_dump		= bcm_sf2_sw_fdb_dump,
-	.port_vlan_filtering	= bcm_sf2_sw_vlan_filtering,
-	.port_vlan_prepare	= bcm_sf2_sw_vlan_prepare,
-	.port_vlan_add		= bcm_sf2_sw_vlan_add,
-	.port_vlan_del		= bcm_sf2_sw_vlan_del,
-	.port_vlan_dump		= bcm_sf2_sw_vlan_dump,
-};
-
-static int __init bcm_sf2_init(void)
+static int bcm_sf2_sw_remove(struct platform_device *pdev)
 {
-	register_switch_driver(&bcm_sf2_switch_driver);
+	struct bcm_sf2_priv *priv = platform_get_drvdata(pdev);
+
+	/* Disable all ports and interrupts */
+	priv->wol_ports_mask = 0;
+	bcm_sf2_sw_suspend(priv->dev->ds);
+	dsa_unregister_switch(priv->dev->ds);
+	bcm_sf2_mdio_unregister(priv);
 
 	return 0;
 }
-module_init(bcm_sf2_init);
 
-static void __exit bcm_sf2_exit(void)
+#ifdef CONFIG_PM_SLEEP
+static int bcm_sf2_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct bcm_sf2_priv *priv = platform_get_drvdata(pdev);
+
+	return dsa_switch_suspend(priv->dev->ds);
+}
+
+static int bcm_sf2_resume(struct device *dev)
 {
-	unregister_switch_driver(&bcm_sf2_switch_driver);
+	struct platform_device *pdev = to_platform_device(dev);
+	struct bcm_sf2_priv *priv = platform_get_drvdata(pdev);
+
+	return dsa_switch_resume(priv->dev->ds);
 }
-module_exit(bcm_sf2_exit);
+#endif /* CONFIG_PM_SLEEP */
+
+static SIMPLE_DEV_PM_OPS(bcm_sf2_pm_ops,
+			 bcm_sf2_suspend, bcm_sf2_resume);
+
+static const struct of_device_id bcm_sf2_of_match[] = {
+	{ .compatible = "brcm,bcm7445-switch-v4.0" },
+	{ /* sentinel */ },
+};
+
+static struct platform_driver bcm_sf2_driver = {
+	.probe	= bcm_sf2_sw_probe,
+	.remove	= bcm_sf2_sw_remove,
+	.driver = {
+		.name = "brcm-sf2",
+		.of_match_table = bcm_sf2_of_match,
+		.pm = &bcm_sf2_pm_ops,
+	},
+};
+module_platform_driver(bcm_sf2_driver);
 
 MODULE_AUTHOR("Broadcom Corporation");
 MODULE_DESCRIPTION("Driver for Broadcom Starfighter 2 ethernet switch chip");
diff --git a/drivers/net/dsa/bcm_sf2.h b/drivers/net/dsa/bcm_sf2.h
index 463bed8cbe4c..44692673e1d5 100644
--- a/drivers/net/dsa/bcm_sf2.h
+++ b/drivers/net/dsa/bcm_sf2.h
@@ -26,6 +26,7 @@
 #include <net/dsa.h>
 
 #include "bcm_sf2_regs.h"
+#include "b53/b53_priv.h"
 
 struct bcm_sf2_hw_params {
 	u16	top_rev;
@@ -49,72 +50,8 @@ struct bcm_sf2_port_status {
 	unsigned int link;
 
 	struct ethtool_eee eee;
-
-	u32 vlan_ctl_mask;
-	u16 pvid;
-
-	struct net_device *bridge_dev;
-};
-
-struct bcm_sf2_arl_entry {
-	u8 port;
-	u8 mac[ETH_ALEN];
-	u16 vid;
-	u8 is_valid:1;
-	u8 is_age:1;
-	u8 is_static:1;
 };
 
-struct bcm_sf2_vlan {
-	u16 members;
-	u16 untag;
-};
-
-static inline void bcm_sf2_mac_from_u64(u64 src, u8 *dst)
-{
-	unsigned int i;
-
-	for (i = 0; i < ETH_ALEN; i++)
-		dst[ETH_ALEN - 1 - i] = (src >> (8 * i)) & 0xff;
-}
-
-static inline u64 bcm_sf2_mac_to_u64(const u8 *src)
-{
-	unsigned int i;
-	u64 dst = 0;
-
-	for (i = 0; i < ETH_ALEN; i++)
-		dst |= (u64)src[ETH_ALEN - 1 - i] << (8 * i);
-
-	return dst;
-}
-
-static inline void bcm_sf2_arl_to_entry(struct bcm_sf2_arl_entry *ent,
-					u64 mac_vid, u32 fwd_entry)
-{
-	memset(ent, 0, sizeof(*ent));
-	ent->port = fwd_entry & PORTID_MASK;
-	ent->is_valid = !!(fwd_entry & ARL_VALID);
-	ent->is_age = !!(fwd_entry & ARL_AGE);
-	ent->is_static = !!(fwd_entry & ARL_STATIC);
-	bcm_sf2_mac_from_u64(mac_vid, ent->mac);
-	ent->vid = mac_vid >> VID_SHIFT;
-}
-
-static inline void bcm_sf2_arl_from_entry(u64 *mac_vid, u32 *fwd_entry,
-					  const struct bcm_sf2_arl_entry *ent)
-{
-	*mac_vid = bcm_sf2_mac_to_u64(ent->mac);
-	*mac_vid |= (u64)(ent->vid & VID_MASK) << VID_SHIFT;
-	*fwd_entry = ent->port & PORTID_MASK;
-	if (ent->is_valid)
-		*fwd_entry |= ARL_VALID;
-	if (ent->is_static)
-		*fwd_entry |= ARL_STATIC;
-	if (ent->is_age)
-		*fwd_entry |= ARL_AGE;
-}
-
 struct bcm_sf2_priv {
 	/* Base registers, keep those in order with BCM_SF2_REGS_NAME */
 	void __iomem			*core;
@@ -134,6 +71,9 @@ struct bcm_sf2_priv {
 	u32				irq1_stat;
 	u32				irq1_mask;
 
+	/* Backing b53_device */
+	struct b53_device		*dev;
+
 	/* Mutex protecting access to the MIB counters */
 	struct mutex			stats_mutex;
 
@@ -155,16 +95,14 @@ struct bcm_sf2_priv {
 	struct device_node		*master_mii_dn;
 	struct mii_bus			*slave_mii_bus;
 	struct mii_bus			*master_mii_bus;
-
-	/* Cache of programmed VLANs */
-	struct bcm_sf2_vlan		vlans[VLAN_N_VID];
 };
 
-struct bcm_sf2_hw_stats {
-	const char	*string;
-	u16		reg;
-	u8		sizeof_stat;
-};
+static inline struct bcm_sf2_priv *bcm_sf2_to_priv(struct dsa_switch *ds)
+{
+	struct b53_device *dev = ds->priv;
+
+	return dev->priv;
+}
 
 #define SF2_IO_MACRO(name) \
 static inline u32 name##_readl(struct bcm_sf2_priv *priv, u32 off)	\
@@ -205,8 +143,8 @@ static inline void name##_writeq(struct bcm_sf2_priv *priv, u64 val,	\
 static inline void intrl2_##which##_mask_clear(struct bcm_sf2_priv *priv, \
 						u32 mask)		\
 {									\
-	intrl2_##which##_writel(priv, mask, INTRL2_CPU_MASK_CLEAR);	\
 	priv->irq##which##_mask &= ~(mask);				\
+	intrl2_##which##_writel(priv, mask, INTRL2_CPU_MASK_CLEAR);	\
 }									\
 static inline void intrl2_##which##_mask_set(struct bcm_sf2_priv *priv, \
 						u32 mask)		\
diff --git a/drivers/net/dsa/bcm_sf2_regs.h b/drivers/net/dsa/bcm_sf2_regs.h
index 9f2a9cb42074..838fe373cd6f 100644
--- a/drivers/net/dsa/bcm_sf2_regs.h
+++ b/drivers/net/dsa/bcm_sf2_regs.h
@@ -115,14 +115,6 @@
 #define  RX_BCST_EN			(1 << 2)
 #define  RX_MCST_EN			(1 << 3)
 #define  RX_UCST_EN			(1 << 4)
-#define  G_MISTP_STATE_SHIFT		5
-#define  G_MISTP_NO_STP			(0 << G_MISTP_STATE_SHIFT)
-#define  G_MISTP_DIS_STATE		(1 << G_MISTP_STATE_SHIFT)
-#define  G_MISTP_BLOCK_STATE		(2 << G_MISTP_STATE_SHIFT)
-#define  G_MISTP_LISTEN_STATE		(3 << G_MISTP_STATE_SHIFT)
-#define  G_MISTP_LEARN_STATE		(4 << G_MISTP_STATE_SHIFT)
-#define  G_MISTP_FWD_STATE		(5 << G_MISTP_STATE_SHIFT)
-#define  G_MISTP_STATE_MASK		0x7
 
 #define CORE_SWMODE			0x0002c
 #define  SW_FWDG_MODE			(1 << 0)
@@ -205,75 +197,11 @@
 #define  BRCM_HDR_EN_P5			(1 << 1)
 #define  BRCM_HDR_EN_P7			(1 << 2)
 
-#define CORE_BRCM_HDR_CTRL2		0x0828
-
-#define CORE_HL_PRTC_CTRL		0x0940
-#define  ARP_EN				(1 << 0)
-#define  RARP_EN			(1 << 1)
-#define  DHCP_EN			(1 << 2)
-#define  ICMPV4_EN			(1 << 3)
-#define  ICMPV6_EN			(1 << 4)
-#define  ICMPV6_FWD_MODE		(1 << 5)
-#define  IGMP_DIP_EN			(1 << 8)
-#define  IGMP_RPTLVE_EN			(1 << 9)
-#define  IGMP_RTPLVE_FWD_MODE		(1 << 10)
-#define  IGMP_QRY_EN			(1 << 11)
-#define  IGMP_QRY_FWD_MODE		(1 << 12)
-#define  IGMP_UKN_EN			(1 << 13)
-#define  IGMP_UKN_FWD_MODE		(1 << 14)
-#define  MLD_RPTDONE_EN			(1 << 15)
-#define  MLD_RPTDONE_FWD_MODE		(1 << 16)
-#define  MLD_QRY_EN			(1 << 17)
-#define  MLD_QRY_FWD_MODE		(1 << 18)
-
 #define CORE_RST_MIB_CNT_EN		0x0950
 
 #define CORE_BRCM_HDR_RX_DIS		0x0980
 #define CORE_BRCM_HDR_TX_DIS		0x0988
 
-#define CORE_ARLA_NUM_ENTRIES		1024
-
-#define CORE_ARLA_RWCTL			0x1400
-#define  ARL_RW				(1 << 0)
-#define  IVL_SVL_SELECT			(1 << 6)
-#define  ARL_STRTDN			(1 << 7)
-
-#define CORE_ARLA_MAC			0x1408
-#define CORE_ARLA_VID			0x1420
-#define  ARLA_VIDTAB_INDX_MASK		0x1fff
-
-#define CORE_ARLA_MACVID0		0x1440
-#define  MAC_MASK			0xffffffffff
-#define  VID_SHIFT			48
-#define  VID_MASK			0xfff
-
-#define CORE_ARLA_FWD_ENTRY0		0x1460
-#define  PORTID_MASK			0x1ff
-#define  ARL_CON_SHIFT			9
-#define  ARL_CON_MASK			0x3
-#define  ARL_PRI_SHIFT			11
-#define  ARL_PRI_MASK			0x7
-#define  ARL_AGE			(1 << 14)
-#define  ARL_STATIC			(1 << 15)
-#define  ARL_VALID			(1 << 16)
-
-#define CORE_ARLA_MACVID_ENTRY(x)	(CORE_ARLA_MACVID0 + ((x) * 0x40))
-#define CORE_ARLA_FWD_ENTRY(x)		(CORE_ARLA_FWD_ENTRY0 + ((x) * 0x40))
-
-#define CORE_ARLA_SRCH_CTL		0x1540
-#define  ARLA_SRCH_VLID			(1 << 0)
-#define  IVL_SVL_SELECT			(1 << 6)
-#define  ARLA_SRCH_STDN			(1 << 7)
-
-#define CORE_ARLA_SRCH_ADR		0x1544
-#define  ARLA_SRCH_ADR_VALID		(1 << 15)
-
-#define CORE_ARLA_SRCH_RSLT_0_MACVID	0x1580
-#define CORE_ARLA_SRCH_RSLT_0		0x15a0
-
-#define CORE_ARLA_SRCH_RSLT_MACVID(x)	(CORE_ARLA_SRCH_RSLT_0_MACVID + ((x) * 0x40))
-#define CORE_ARLA_SRCH_RSLT(x)		(CORE_ARLA_SRCH_RSLT_0 + ((x) * 0x40))
-
 #define CORE_ARLA_VTBL_RWCTRL		0x1600
 #define  ARLA_VTBL_CMD_WRITE		0
 #define  ARLA_VTBL_CMD_READ		1
@@ -297,59 +225,9 @@
 #define  P_TXQ_PSM_VDD(x)		(P_TXQ_PSM_VDD_MASK << \
 					((x) * P_TXQ_PSM_VDD_SHIFT))
 
-#define	CORE_P0_MIB_OFFSET		0x8000
-#define P_MIB_SIZE			0x400
-#define CORE_P_MIB_OFFSET(x)		(CORE_P0_MIB_OFFSET + (x) * P_MIB_SIZE)
-
 #define CORE_PORT_VLAN_CTL_PORT(x)	(0xc400 + ((x) * 0x8))
 #define  PORT_VLAN_CTRL_MASK		0x1ff
 
-#define CORE_VLAN_CTRL0			0xd000
-#define  CHANGE_1P_VID_INNER		(1 << 0)
-#define  CHANGE_1P_VID_OUTER		(1 << 1)
-#define  CHANGE_1Q_VID			(1 << 3)
-#define  VLAN_LEARN_MODE_SVL		(0 << 5)
-#define  VLAN_LEARN_MODE_IVL		(3 << 5)
-#define  VLAN_EN			(1 << 7)
-
-#define CORE_VLAN_CTRL1			0xd004
-#define  EN_RSV_MCAST_FWDMAP		(1 << 2)
-#define  EN_RSV_MCAST_UNTAG		(1 << 3)
-#define  EN_IPMC_BYPASS_FWDMAP		(1 << 5)
-#define  EN_IPMC_BYPASS_UNTAG		(1 << 6)
-
-#define CORE_VLAN_CTRL2			0xd008
-#define  EN_MIIM_BYPASS_V_FWDMAP	(1 << 2)
-#define  EN_GMRP_GVRP_V_FWDMAP		(1 << 5)
-#define  EN_GMRP_GVRP_UNTAG_MAP		(1 << 6)
-
-#define CORE_VLAN_CTRL3			0xd00c
-#define  EN_DROP_NON1Q_MASK		0x1ff
-
-#define CORE_VLAN_CTRL4			0xd014
-#define  RESV_MCAST_FLOOD		(1 << 1)
-#define  EN_DOUBLE_TAG_MASK		0x3
-#define  EN_DOUBLE_TAG_SHIFT		2
-#define  EN_MGE_REV_GMRP		(1 << 4)
-#define  EN_MGE_REV_GVRP		(1 << 5)
-#define  INGR_VID_CHK_SHIFT		6
-#define  INGR_VID_CHK_MASK		0x3
-#define  INGR_VID_CHK_FWD		(0 << INGR_VID_CHK_SHIFT)
-#define  INGR_VID_CHK_DROP		(1 << INGR_VID_CHK_SHIFT)
-#define  INGR_VID_CHK_NO_CHK		(2 << INGR_VID_CHK_SHIFT)
-#define  INGR_VID_CHK_VID_VIOL_IMP	(3 << INGR_VID_CHK_SHIFT)
-
-#define CORE_VLAN_CTRL5			0xd018
-#define  EN_CPU_RX_BYP_INNER_CRCCHCK	(1 << 0)
-#define  EN_VID_FFF_FWD			(1 << 2)
-#define  DROP_VTABLE_MISS		(1 << 3)
-#define  EGRESS_DIR_FRM_BYP_TRUNK_EN	(1 << 4)
-#define  PRESV_NON1Q			(1 << 6)
-
-#define CORE_VLAN_CTRL6			0xd01c
-#define  STRICT_SFD_DETECT		(1 << 0)
-#define  DIS_ARL_BUST_LMIT		(1 << 4)
-
 #define CORE_DEFAULT_1Q_TAG_P(x)	(0xd040 + ((x) * 8))
 #define  CFI_SHIFT			12
 #define  PRI_SHIFT			13
diff --git a/drivers/net/dsa/mv88e6060.c b/drivers/net/dsa/mv88e6060.c
index e36b40886bd8..7ce36dbd9b62 100644
--- a/drivers/net/dsa/mv88e6060.c
+++ b/drivers/net/dsa/mv88e6060.c
@@ -19,7 +19,7 @@
 
 static int reg_read(struct dsa_switch *ds, int addr, int reg)
 {
-	struct mv88e6060_priv *priv = ds_to_priv(ds);
+	struct mv88e6060_priv *priv = ds->priv;
 
 	return mdiobus_read_nested(priv->bus, priv->sw_addr + addr, reg);
 }
@@ -37,7 +37,7 @@ static int reg_read(struct dsa_switch *ds, int addr, int reg)
 
 static int reg_write(struct dsa_switch *ds, int addr, int reg, u16 val)
 {
-	struct mv88e6060_priv *priv = ds_to_priv(ds);
+	struct mv88e6060_priv *priv = ds->priv;
 
 	return mdiobus_write_nested(priv->bus, priv->sw_addr + addr, reg, val);
 }
@@ -69,6 +69,11 @@ static const char *mv88e6060_get_name(struct mii_bus *bus, int sw_addr)
 	return NULL;
 }
 
+static enum dsa_tag_protocol mv88e6060_get_tag_protocol(struct dsa_switch *ds)
+{
+	return DSA_TAG_PROTO_TRAILER;
+}
+
 static const char *mv88e6060_drv_probe(struct device *dsa_dev,
 				       struct device *host_dev, int sw_addr,
 				       void **_priv)
@@ -247,8 +252,8 @@ mv88e6060_phy_write(struct dsa_switch *ds, int port, int regnum, u16 val)
 	return reg_write(ds, addr, regnum, val);
 }
 
-static struct dsa_switch_driver mv88e6060_switch_driver = {
-	.tag_protocol	= DSA_TAG_PROTO_TRAILER,
+static struct dsa_switch_ops mv88e6060_switch_ops = {
+	.get_tag_protocol = mv88e6060_get_tag_protocol,
 	.probe		= mv88e6060_drv_probe,
 	.setup		= mv88e6060_setup,
 	.set_addr	= mv88e6060_set_addr,
@@ -258,14 +263,14 @@ static struct dsa_switch_driver mv88e6060_switch_driver = {
 
 static int __init mv88e6060_init(void)
 {
-	register_switch_driver(&mv88e6060_switch_driver);
+	register_switch_driver(&mv88e6060_switch_ops);
 	return 0;
 }
 module_init(mv88e6060_init);
 
 static void __exit mv88e6060_cleanup(void)
 {
-	unregister_switch_driver(&mv88e6060_switch_driver);
+	unregister_switch_driver(&mv88e6060_switch_ops);
 }
 module_exit(mv88e6060_cleanup);
 
diff --git a/drivers/net/dsa/mv88e6xxx/Kconfig b/drivers/net/dsa/mv88e6xxx/Kconfig
index 490bc06f993e..486668813e15 100644
--- a/drivers/net/dsa/mv88e6xxx/Kconfig
+++ b/drivers/net/dsa/mv88e6xxx/Kconfig
@@ -2,6 +2,18 @@ config NET_DSA_MV88E6XXX
 	tristate "Marvell 88E6xxx Ethernet switch fabric support"
 	depends on NET_DSA
 	select NET_DSA_TAG_EDSA
+	select NET_DSA_TAG_DSA
 	help
 	  This driver adds support for most of the Marvell 88E6xxx models of
 	  Ethernet switch chips, except 88E6060.
+
+config NET_DSA_MV88E6XXX_GLOBAL2
+	bool "Switch Global 2 Registers support"
+	default y
+	depends on NET_DSA_MV88E6XXX
+	help
+	  This registers set at internal SMI address 0x1C provides extended
+	  features like EEPROM interface, trunking, cross-chip setup, etc.
+
+	  It is required on most chips. If the chip you compile the support for
+	  doesn't have such registers set, say N here. In doubt, say Y.
diff --git a/drivers/net/dsa/mv88e6xxx/Makefile b/drivers/net/dsa/mv88e6xxx/Makefile
index 6e29a75ee2f7..10ce820daa48 100644
--- a/drivers/net/dsa/mv88e6xxx/Makefile
+++ b/drivers/net/dsa/mv88e6xxx/Makefile
@@ -1 +1,4 @@
-obj-$(CONFIG_NET_DSA_MV88E6XXX) += chip.o
+obj-$(CONFIG_NET_DSA_MV88E6XXX) += mv88e6xxx.o
+mv88e6xxx-objs := chip.o
+mv88e6xxx-objs += global1.o
+mv88e6xxx-$(CONFIG_NET_DSA_MV88E6XXX_GLOBAL2) += global2.o
diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index d36aedde8cb9..883fd9809dd2 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -29,7 +29,10 @@
 #include <linux/phy.h>
 #include <net/dsa.h>
 #include <net/switchdev.h>
+
 #include "mv88e6xxx.h"
+#include "global1.h"
+#include "global2.h"
 
 static void assert_reg_lock(struct mv88e6xxx_chip *chip)
 {
@@ -95,7 +98,7 @@ static int mv88e6xxx_smi_single_chip_write(struct mv88e6xxx_chip *chip,
 	return 0;
 }
 
-static const struct mv88e6xxx_ops mv88e6xxx_smi_single_chip_ops = {
+static const struct mv88e6xxx_bus_ops mv88e6xxx_smi_single_chip_ops = {
 	.read = mv88e6xxx_smi_single_chip_read,
 	.write = mv88e6xxx_smi_single_chip_write,
 };
@@ -177,13 +180,12 @@ static int mv88e6xxx_smi_multi_chip_write(struct mv88e6xxx_chip *chip,
 	return 0;
 }
 
-static const struct mv88e6xxx_ops mv88e6xxx_smi_multi_chip_ops = {
+static const struct mv88e6xxx_bus_ops mv88e6xxx_smi_multi_chip_ops = {
 	.read = mv88e6xxx_smi_multi_chip_read,
 	.write = mv88e6xxx_smi_multi_chip_write,
 };
 
-static int mv88e6xxx_read(struct mv88e6xxx_chip *chip,
-			  int addr, int reg, u16 *val)
+int mv88e6xxx_read(struct mv88e6xxx_chip *chip, int addr, int reg, u16 *val)
 {
 	int err;
 
@@ -199,8 +201,7 @@ static int mv88e6xxx_read(struct mv88e6xxx_chip *chip,
 	return 0;
 }
 
-static int mv88e6xxx_write(struct mv88e6xxx_chip *chip,
-			   int addr, int reg, u16 val)
+int mv88e6xxx_write(struct mv88e6xxx_chip *chip, int addr, int reg, u16 val)
 {
 	int err;
 
@@ -216,89 +217,172 @@ static int mv88e6xxx_write(struct mv88e6xxx_chip *chip,
 	return 0;
 }
 
-/* Indirect write to single pointer-data register with an Update bit */
-static int mv88e6xxx_update(struct mv88e6xxx_chip *chip, int addr, int reg,
-			    u16 update)
+static int mv88e6xxx_port_read(struct mv88e6xxx_chip *chip, int port, int reg,
+			       u16 *val)
 {
-	u16 val;
-	int i, err;
+	int addr = chip->info->port_base_addr + port;
 
-	/* Wait until the previous operation is completed */
-	for (i = 0; i < 16; ++i) {
-		err = mv88e6xxx_read(chip, addr, reg, &val);
-		if (err)
-			return err;
+	return mv88e6xxx_read(chip, addr, reg, val);
+}
 
-		if (!(val & BIT(15)))
-			break;
+static int mv88e6xxx_port_write(struct mv88e6xxx_chip *chip, int port, int reg,
+				u16 val)
+{
+	int addr = chip->info->port_base_addr + port;
+
+	return mv88e6xxx_write(chip, addr, reg, val);
+}
+
+static int mv88e6xxx_phy_read(struct mv88e6xxx_chip *chip, int phy,
+			      int reg, u16 *val)
+{
+	int addr = phy; /* PHY devices addresses start at 0x0 */
+
+	if (!chip->info->ops->phy_read)
+		return -EOPNOTSUPP;
+
+	return chip->info->ops->phy_read(chip, addr, reg, val);
+}
+
+static int mv88e6xxx_phy_write(struct mv88e6xxx_chip *chip, int phy,
+			       int reg, u16 val)
+{
+	int addr = phy; /* PHY devices addresses start at 0x0 */
+
+	if (!chip->info->ops->phy_write)
+		return -EOPNOTSUPP;
+
+	return chip->info->ops->phy_write(chip, addr, reg, val);
+}
+
+static int mv88e6xxx_phy_page_get(struct mv88e6xxx_chip *chip, int phy, u8 page)
+{
+	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_PHY_PAGE))
+		return -EOPNOTSUPP;
+
+	return mv88e6xxx_phy_write(chip, phy, PHY_PAGE, page);
+}
+
+static void mv88e6xxx_phy_page_put(struct mv88e6xxx_chip *chip, int phy)
+{
+	int err;
+
+	/* Restore PHY page Copper 0x0 for access via the registered MDIO bus */
+	err = mv88e6xxx_phy_write(chip, phy, PHY_PAGE, PHY_PAGE_COPPER);
+	if (unlikely(err)) {
+		dev_err(chip->dev, "failed to restore PHY %d page Copper (%d)\n",
+			phy, err);
 	}
+}
+
+static int mv88e6xxx_phy_page_read(struct mv88e6xxx_chip *chip, int phy,
+				   u8 page, int reg, u16 *val)
+{
+	int err;
 
-	if (i == 16)
-		return -ETIMEDOUT;
+	/* There is no paging for registers 22 */
+	if (reg == PHY_PAGE)
+		return -EINVAL;
 
-	/* Set the Update bit to trigger a write operation */
-	val = BIT(15) | update;
+	err = mv88e6xxx_phy_page_get(chip, phy, page);
+	if (!err) {
+		err = mv88e6xxx_phy_read(chip, phy, reg, val);
+		mv88e6xxx_phy_page_put(chip, phy);
+	}
 
-	return mv88e6xxx_write(chip, addr, reg, val);
+	return err;
 }
 
-static int _mv88e6xxx_reg_read(struct mv88e6xxx_chip *chip, int addr, int reg)
+static int mv88e6xxx_phy_page_write(struct mv88e6xxx_chip *chip, int phy,
+				    u8 page, int reg, u16 val)
 {
-	u16 val;
 	int err;
 
-	err = mv88e6xxx_read(chip, addr, reg, &val);
-	if (err)
-		return err;
+	/* There is no paging for registers 22 */
+	if (reg == PHY_PAGE)
+		return -EINVAL;
 
-	return val;
+	err = mv88e6xxx_phy_page_get(chip, phy, page);
+	if (!err) {
+		err = mv88e6xxx_phy_write(chip, phy, PHY_PAGE, page);
+		mv88e6xxx_phy_page_put(chip, phy);
+	}
+
+	return err;
 }
 
-static int _mv88e6xxx_reg_write(struct mv88e6xxx_chip *chip, int addr,
-				int reg, u16 val)
+static int mv88e6xxx_serdes_read(struct mv88e6xxx_chip *chip, int reg, u16 *val)
 {
-	return mv88e6xxx_write(chip, addr, reg, val);
+	return mv88e6xxx_phy_page_read(chip, ADDR_SERDES, SERDES_PAGE_FIBER,
+				       reg, val);
 }
 
-static int mv88e6xxx_mdio_read_direct(struct mv88e6xxx_chip *chip,
-				      int addr, int regnum)
+static int mv88e6xxx_serdes_write(struct mv88e6xxx_chip *chip, int reg, u16 val)
 {
-	if (addr >= 0)
-		return _mv88e6xxx_reg_read(chip, addr, regnum);
-	return 0xffff;
+	return mv88e6xxx_phy_page_write(chip, ADDR_SERDES, SERDES_PAGE_FIBER,
+					reg, val);
 }
 
-static int mv88e6xxx_mdio_write_direct(struct mv88e6xxx_chip *chip,
-				       int addr, int regnum, u16 val)
+int mv88e6xxx_wait(struct mv88e6xxx_chip *chip, int addr, int reg, u16 mask)
 {
-	if (addr >= 0)
-		return _mv88e6xxx_reg_write(chip, addr, regnum, val);
-	return 0;
+	int i;
+
+	for (i = 0; i < 16; i++) {
+		u16 val;
+		int err;
+
+		err = mv88e6xxx_read(chip, addr, reg, &val);
+		if (err)
+			return err;
+
+		if (!(val & mask))
+			return 0;
+
+		usleep_range(1000, 2000);
+	}
+
+	dev_err(chip->dev, "Timeout while waiting for switch\n");
+	return -ETIMEDOUT;
+}
+
+/* Indirect write to single pointer-data register with an Update bit */
+int mv88e6xxx_update(struct mv88e6xxx_chip *chip, int addr, int reg, u16 update)
+{
+	u16 val;
+	int err;
+
+	/* Wait until the previous operation is completed */
+	err = mv88e6xxx_wait(chip, addr, reg, BIT(15));
+	if (err)
+		return err;
+
+	/* Set the Update bit to trigger a write operation */
+	val = BIT(15) | update;
+
+	return mv88e6xxx_write(chip, addr, reg, val);
 }
 
 static int mv88e6xxx_ppu_disable(struct mv88e6xxx_chip *chip)
 {
-	int ret;
-	unsigned long timeout;
+	u16 val;
+	int i, err;
 
-	ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_CONTROL);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_read(chip, GLOBAL_CONTROL, &val);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_CONTROL,
-				   ret & ~GLOBAL_CONTROL_PPU_ENABLE);
-	if (ret)
-		return ret;
+	err = mv88e6xxx_g1_write(chip, GLOBAL_CONTROL,
+				 val & ~GLOBAL_CONTROL_PPU_ENABLE);
+	if (err)
+		return err;
 
-	timeout = jiffies + 1 * HZ;
-	while (time_before(jiffies, timeout)) {
-		ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATUS);
-		if (ret < 0)
-			return ret;
+	for (i = 0; i < 16; i++) {
+		err = mv88e6xxx_g1_read(chip, GLOBAL_STATUS, &val);
+		if (err)
+			return err;
 
 		usleep_range(1000, 2000);
-		if ((ret & GLOBAL_STATUS_PPU_MASK) !=
-		    GLOBAL_STATUS_PPU_POLLING)
+		if ((val & GLOBAL_STATUS_PPU_MASK) != GLOBAL_STATUS_PPU_POLLING)
 			return 0;
 	}
 
@@ -307,27 +391,25 @@ static int mv88e6xxx_ppu_disable(struct mv88e6xxx_chip *chip)
 
 static int mv88e6xxx_ppu_enable(struct mv88e6xxx_chip *chip)
 {
-	int ret, err;
-	unsigned long timeout;
+	u16 val;
+	int i, err;
 
-	ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_CONTROL);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_read(chip, GLOBAL_CONTROL, &val);
+	if (err)
+		return err;
 
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_CONTROL,
-				   ret | GLOBAL_CONTROL_PPU_ENABLE);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_CONTROL,
+				 val | GLOBAL_CONTROL_PPU_ENABLE);
 	if (err)
 		return err;
 
-	timeout = jiffies + 1 * HZ;
-	while (time_before(jiffies, timeout)) {
-		ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATUS);
-		if (ret < 0)
-			return ret;
+	for (i = 0; i < 16; i++) {
+		err = mv88e6xxx_g1_read(chip, GLOBAL_STATUS, &val);
+		if (err)
+			return err;
 
 		usleep_range(1000, 2000);
-		if ((ret & GLOBAL_STATUS_PPU_MASK) ==
-		    GLOBAL_STATUS_PPU_POLLING)
+		if ((val & GLOBAL_STATUS_PPU_MASK) == GLOBAL_STATUS_PPU_POLLING)
 			return 0;
 	}
 
@@ -400,32 +482,37 @@ static void mv88e6xxx_ppu_state_init(struct mv88e6xxx_chip *chip)
 	chip->ppu_timer.function = mv88e6xxx_ppu_reenable_timer;
 }
 
-static int mv88e6xxx_mdio_read_ppu(struct mv88e6xxx_chip *chip, int addr,
-				   int regnum)
+static void mv88e6xxx_ppu_state_destroy(struct mv88e6xxx_chip *chip)
 {
-	int ret;
+	del_timer_sync(&chip->ppu_timer);
+}
+
+static int mv88e6xxx_phy_ppu_read(struct mv88e6xxx_chip *chip, int addr,
+				  int reg, u16 *val)
+{
+	int err;
 
-	ret = mv88e6xxx_ppu_access_get(chip);
-	if (ret >= 0) {
-		ret = _mv88e6xxx_reg_read(chip, addr, regnum);
+	err = mv88e6xxx_ppu_access_get(chip);
+	if (!err) {
+		err = mv88e6xxx_read(chip, addr, reg, val);
 		mv88e6xxx_ppu_access_put(chip);
 	}
 
-	return ret;
+	return err;
 }
 
-static int mv88e6xxx_mdio_write_ppu(struct mv88e6xxx_chip *chip, int addr,
-				    int regnum, u16 val)
+static int mv88e6xxx_phy_ppu_write(struct mv88e6xxx_chip *chip, int addr,
+				   int reg, u16 val)
 {
-	int ret;
+	int err;
 
-	ret = mv88e6xxx_ppu_access_get(chip);
-	if (ret >= 0) {
-		ret = _mv88e6xxx_reg_write(chip, addr, regnum, val);
+	err = mv88e6xxx_ppu_access_get(chip);
+	if (!err) {
+		err = mv88e6xxx_write(chip, addr, reg, val);
 		mv88e6xxx_ppu_access_put(chip);
 	}
 
-	return ret;
+	return err;
 }
 
 static bool mv88e6xxx_6065_family(struct mv88e6xxx_chip *chip)
@@ -468,21 +555,6 @@ static bool mv88e6xxx_6352_family(struct mv88e6xxx_chip *chip)
 	return chip->info->family == MV88E6XXX_FAMILY_6352;
 }
 
-static unsigned int mv88e6xxx_num_databases(struct mv88e6xxx_chip *chip)
-{
-	return chip->info->num_databases;
-}
-
-static bool mv88e6xxx_has_fid_reg(struct mv88e6xxx_chip *chip)
-{
-	/* Does the device have dedicated FID registers for ATU and VTU ops? */
-	if (mv88e6xxx_6097_family(chip) || mv88e6xxx_6165_family(chip) ||
-	    mv88e6xxx_6351_family(chip) || mv88e6xxx_6352_family(chip))
-		return true;
-
-	return false;
-}
-
 /* We expect the switch to perform auto negotiation if there is a real
  * phy. However, in the case of a fixed link phy, we force the port
  * settings from the fixed link settings.
@@ -490,24 +562,24 @@ static bool mv88e6xxx_has_fid_reg(struct mv88e6xxx_chip *chip)
 static void mv88e6xxx_adjust_link(struct dsa_switch *ds, int port,
 				  struct phy_device *phydev)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	u32 reg;
-	int ret;
+	struct mv88e6xxx_chip *chip = ds->priv;
+	u16 reg;
+	int err;
 
 	if (!phy_is_pseudo_fixed_link(phydev))
 		return;
 
 	mutex_lock(&chip->reg_lock);
 
-	ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_PCS_CTRL);
-	if (ret < 0)
+	err = mv88e6xxx_port_read(chip, port, PORT_PCS_CTRL, &reg);
+	if (err)
 		goto out;
 
-	reg = ret & ~(PORT_PCS_CTRL_LINK_UP |
-		      PORT_PCS_CTRL_FORCE_LINK |
-		      PORT_PCS_CTRL_DUPLEX_FULL |
-		      PORT_PCS_CTRL_FORCE_DUPLEX |
-		      PORT_PCS_CTRL_UNFORCED);
+	reg &= ~(PORT_PCS_CTRL_LINK_UP |
+		 PORT_PCS_CTRL_FORCE_LINK |
+		 PORT_PCS_CTRL_DUPLEX_FULL |
+		 PORT_PCS_CTRL_FORCE_DUPLEX |
+		 PORT_PCS_CTRL_UNFORCED);
 
 	reg |= PORT_PCS_CTRL_FORCE_LINK;
 	if (phydev->link)
@@ -536,7 +608,7 @@ static void mv88e6xxx_adjust_link(struct dsa_switch *ds, int port,
 		reg |= PORT_PCS_CTRL_DUPLEX_FULL;
 
 	if ((mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip)) &&
-	    (port >= chip->info->num_ports - 2)) {
+	    (port >= mv88e6xxx_num_ports(chip) - 2)) {
 		if (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID)
 			reg |= PORT_PCS_CTRL_RGMII_DELAY_RXCLK;
 		if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID)
@@ -545,7 +617,7 @@ static void mv88e6xxx_adjust_link(struct dsa_switch *ds, int port,
 			reg |= (PORT_PCS_CTRL_RGMII_DELAY_RXCLK |
 				PORT_PCS_CTRL_RGMII_DELAY_TXCLK);
 	}
-	_mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_PCS_CTRL, reg);
+	mv88e6xxx_port_write(chip, port, PORT_PCS_CTRL, reg);
 
 out:
 	mutex_unlock(&chip->reg_lock);
@@ -553,12 +625,12 @@ out:
 
 static int _mv88e6xxx_stats_wait(struct mv88e6xxx_chip *chip)
 {
-	int ret;
-	int i;
+	u16 val;
+	int i, err;
 
 	for (i = 0; i < 10; i++) {
-		ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATS_OP);
-		if ((ret & GLOBAL_STATS_OP_BUSY) == 0)
+		err = mv88e6xxx_g1_read(chip, GLOBAL_STATS_OP, &val);
+		if ((val & GLOBAL_STATS_OP_BUSY) == 0)
 			return 0;
 	}
 
@@ -567,55 +639,52 @@ static int _mv88e6xxx_stats_wait(struct mv88e6xxx_chip *chip)
 
 static int _mv88e6xxx_stats_snapshot(struct mv88e6xxx_chip *chip, int port)
 {
-	int ret;
+	int err;
 
 	if (mv88e6xxx_6320_family(chip) || mv88e6xxx_6352_family(chip))
 		port = (port + 1) << 5;
 
 	/* Snapshot the hardware statistics counters for this port. */
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_STATS_OP,
-				   GLOBAL_STATS_OP_CAPTURE_PORT |
-				   GLOBAL_STATS_OP_HIST_RX_TX | port);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_write(chip, GLOBAL_STATS_OP,
+				 GLOBAL_STATS_OP_CAPTURE_PORT |
+				 GLOBAL_STATS_OP_HIST_RX_TX | port);
+	if (err)
+		return err;
 
 	/* Wait for the snapshotting to complete. */
-	ret = _mv88e6xxx_stats_wait(chip);
-	if (ret < 0)
-		return ret;
-
-	return 0;
+	return _mv88e6xxx_stats_wait(chip);
 }
 
 static void _mv88e6xxx_stats_read(struct mv88e6xxx_chip *chip,
 				  int stat, u32 *val)
 {
-	u32 _val;
-	int ret;
+	u32 value;
+	u16 reg;
+	int err;
 
 	*val = 0;
 
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_STATS_OP,
-				   GLOBAL_STATS_OP_READ_CAPTURED |
-				   GLOBAL_STATS_OP_HIST_RX_TX | stat);
-	if (ret < 0)
+	err = mv88e6xxx_g1_write(chip, GLOBAL_STATS_OP,
+				 GLOBAL_STATS_OP_READ_CAPTURED |
+				 GLOBAL_STATS_OP_HIST_RX_TX | stat);
+	if (err)
 		return;
 
-	ret = _mv88e6xxx_stats_wait(chip);
-	if (ret < 0)
+	err = _mv88e6xxx_stats_wait(chip);
+	if (err)
 		return;
 
-	ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATS_COUNTER_32);
-	if (ret < 0)
+	err = mv88e6xxx_g1_read(chip, GLOBAL_STATS_COUNTER_32, &reg);
+	if (err)
 		return;
 
-	_val = ret << 16;
+	value = reg << 16;
 
-	ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_STATS_COUNTER_01);
-	if (ret < 0)
+	err = mv88e6xxx_g1_read(chip, GLOBAL_STATS_COUNTER_01, &reg);
+	if (err)
 		return;
 
-	*val = _val | ret;
+	*val = value | reg;
 }
 
 static struct mv88e6xxx_hw_stat mv88e6xxx_hw_stats[] = {
@@ -705,22 +774,22 @@ static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_chip *chip,
 {
 	u32 low;
 	u32 high = 0;
-	int ret;
+	int err;
+	u16 reg;
 	u64 value;
 
 	switch (s->type) {
 	case PORT:
-		ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), s->reg);
-		if (ret < 0)
+		err = mv88e6xxx_port_read(chip, port, s->reg, &reg);
+		if (err)
 			return UINT64_MAX;
 
-		low = ret;
+		low = reg;
 		if (s->sizeof_stat == 4) {
-			ret = _mv88e6xxx_reg_read(chip, REG_PORT(port),
-						  s->reg + 1);
-			if (ret < 0)
+			err = mv88e6xxx_port_read(chip, port, s->reg + 1, &reg);
+			if (err)
 				return UINT64_MAX;
-			high = ret;
+			high = reg;
 		}
 		break;
 	case BANK0:
@@ -736,7 +805,7 @@ static uint64_t _mv88e6xxx_get_ethtool_stat(struct mv88e6xxx_chip *chip,
 static void mv88e6xxx_get_strings(struct dsa_switch *ds, int port,
 				  uint8_t *data)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	struct mv88e6xxx_hw_stat *stat;
 	int i, j;
 
@@ -752,7 +821,7 @@ static void mv88e6xxx_get_strings(struct dsa_switch *ds, int port,
 
 static int mv88e6xxx_get_sset_count(struct dsa_switch *ds)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	struct mv88e6xxx_hw_stat *stat;
 	int i, j;
 
@@ -767,7 +836,7 @@ static int mv88e6xxx_get_sset_count(struct dsa_switch *ds)
 static void mv88e6xxx_get_ethtool_stats(struct dsa_switch *ds, int port,
 					uint64_t *data)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	struct mv88e6xxx_hw_stat *stat;
 	int ret;
 	int i, j;
@@ -798,7 +867,9 @@ static int mv88e6xxx_get_regs_len(struct dsa_switch *ds, int port)
 static void mv88e6xxx_get_regs(struct dsa_switch *ds, int port,
 			       struct ethtool_regs *regs, void *_p)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
+	int err;
+	u16 reg;
 	u16 *p = _p;
 	int i;
 
@@ -809,170 +880,106 @@ static void mv88e6xxx_get_regs(struct dsa_switch *ds, int port,
 	mutex_lock(&chip->reg_lock);
 
 	for (i = 0; i < 32; i++) {
-		int ret;
 
-		ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), i);
-		if (ret >= 0)
-			p[i] = ret;
+		err = mv88e6xxx_port_read(chip, port, i, &reg);
+		if (!err)
+			p[i] = reg;
 	}
 
 	mutex_unlock(&chip->reg_lock);
 }
 
-static int _mv88e6xxx_wait(struct mv88e6xxx_chip *chip, int reg, int offset,
-			   u16 mask)
-{
-	unsigned long timeout = jiffies + HZ / 10;
-
-	while (time_before(jiffies, timeout)) {
-		int ret;
-
-		ret = _mv88e6xxx_reg_read(chip, reg, offset);
-		if (ret < 0)
-			return ret;
-		if (!(ret & mask))
-			return 0;
-
-		usleep_range(1000, 2000);
-	}
-	return -ETIMEDOUT;
-}
-
-static int mv88e6xxx_mdio_wait(struct mv88e6xxx_chip *chip)
-{
-	return _mv88e6xxx_wait(chip, REG_GLOBAL2, GLOBAL2_SMI_OP,
-			       GLOBAL2_SMI_OP_BUSY);
-}
-
 static int _mv88e6xxx_atu_wait(struct mv88e6xxx_chip *chip)
 {
-	return _mv88e6xxx_wait(chip, REG_GLOBAL, GLOBAL_ATU_OP,
-			       GLOBAL_ATU_OP_BUSY);
-}
-
-static int mv88e6xxx_mdio_read_indirect(struct mv88e6xxx_chip *chip,
-					int addr, int regnum)
-{
-	int ret;
-
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_SMI_OP,
-				   GLOBAL2_SMI_OP_22_READ | (addr << 5) |
-				   regnum);
-	if (ret < 0)
-		return ret;
-
-	ret = mv88e6xxx_mdio_wait(chip);
-	if (ret < 0)
-		return ret;
-
-	ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL2, GLOBAL2_SMI_DATA);
-
-	return ret;
-}
-
-static int mv88e6xxx_mdio_write_indirect(struct mv88e6xxx_chip *chip,
-					 int addr, int regnum, u16 val)
-{
-	int ret;
-
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_SMI_DATA, val);
-	if (ret < 0)
-		return ret;
-
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL2, GLOBAL2_SMI_OP,
-				   GLOBAL2_SMI_OP_22_WRITE | (addr << 5) |
-				   regnum);
-
-	return mv88e6xxx_mdio_wait(chip);
+	return mv88e6xxx_g1_wait(chip, GLOBAL_ATU_OP, GLOBAL_ATU_OP_BUSY);
 }
 
 static int mv88e6xxx_get_eee(struct dsa_switch *ds, int port,
 			     struct ethtool_eee *e)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	int reg;
+	struct mv88e6xxx_chip *chip = ds->priv;
+	u16 reg;
+	int err;
 
 	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEE))
 		return -EOPNOTSUPP;
 
 	mutex_lock(&chip->reg_lock);
 
-	reg = mv88e6xxx_mdio_read_indirect(chip, port, 16);
-	if (reg < 0)
+	err = mv88e6xxx_phy_read(chip, port, 16, &reg);
+	if (err)
 		goto out;
 
 	e->eee_enabled = !!(reg & 0x0200);
 	e->tx_lpi_enabled = !!(reg & 0x0100);
 
-	reg = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_STATUS);
-	if (reg < 0)
+	err = mv88e6xxx_port_read(chip, port, PORT_STATUS, &reg);
+	if (err)
 		goto out;
 
 	e->eee_active = !!(reg & PORT_STATUS_EEE);
-	reg = 0;
-
 out:
 	mutex_unlock(&chip->reg_lock);
-	return reg;
+
+	return err;
 }
 
 static int mv88e6xxx_set_eee(struct dsa_switch *ds, int port,
 			     struct phy_device *phydev, struct ethtool_eee *e)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	int reg;
-	int ret;
+	struct mv88e6xxx_chip *chip = ds->priv;
+	u16 reg;
+	int err;
 
 	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEE))
 		return -EOPNOTSUPP;
 
 	mutex_lock(&chip->reg_lock);
 
-	ret = mv88e6xxx_mdio_read_indirect(chip, port, 16);
-	if (ret < 0)
+	err = mv88e6xxx_phy_read(chip, port, 16, &reg);
+	if (err)
 		goto out;
 
-	reg = ret & ~0x0300;
+	reg &= ~0x0300;
 	if (e->eee_enabled)
 		reg |= 0x0200;
 	if (e->tx_lpi_enabled)
 		reg |= 0x0100;
 
-	ret = mv88e6xxx_mdio_write_indirect(chip, port, 16, reg);
+	err = mv88e6xxx_phy_write(chip, port, 16, reg);
 out:
 	mutex_unlock(&chip->reg_lock);
 
-	return ret;
+	return err;
 }
 
 static int _mv88e6xxx_atu_cmd(struct mv88e6xxx_chip *chip, u16 fid, u16 cmd)
 {
-	int ret;
+	u16 val;
+	int err;
 
-	if (mv88e6xxx_has_fid_reg(chip)) {
-		ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_ATU_FID,
-					   fid);
-		if (ret < 0)
-			return ret;
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G1_ATU_FID)) {
+		err = mv88e6xxx_g1_write(chip, GLOBAL_ATU_FID, fid);
+		if (err)
+			return err;
 	} else if (mv88e6xxx_num_databases(chip) == 256) {
 		/* ATU DBNum[7:4] are located in ATU Control 15:12 */
-		ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_ATU_CONTROL);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_g1_read(chip, GLOBAL_ATU_CONTROL, &val);
+		if (err)
+			return err;
 
-		ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_ATU_CONTROL,
-					   (ret & 0xfff) |
-					   ((fid << 8) & 0xf000));
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_g1_write(chip, GLOBAL_ATU_CONTROL,
+					 (val & 0xfff) | ((fid << 8) & 0xf000));
+		if (err)
+			return err;
 
 		/* ATU DBNum[3:0] are located in ATU Operation 3:0 */
 		cmd |= fid & 0xf;
 	}
 
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_ATU_OP, cmd);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_write(chip, GLOBAL_ATU_OP, cmd);
+	if (err)
+		return err;
 
 	return _mv88e6xxx_atu_wait(chip);
 }
@@ -997,7 +1004,7 @@ static int _mv88e6xxx_atu_data_write(struct mv88e6xxx_chip *chip,
 		data |= (entry->portv_trunkid << shift) & mask;
 	}
 
-	return _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_ATU_DATA, data);
+	return mv88e6xxx_g1_write(chip, GLOBAL_ATU_DATA, data);
 }
 
 static int _mv88e6xxx_atu_flush_move(struct mv88e6xxx_chip *chip,
@@ -1073,57 +1080,45 @@ static int _mv88e6xxx_port_state(struct mv88e6xxx_chip *chip, int port,
 				 u8 state)
 {
 	struct dsa_switch *ds = chip->ds;
-	int reg, ret = 0;
+	u16 reg;
+	int err;
 	u8 oldstate;
 
-	reg = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_CONTROL);
-	if (reg < 0)
-		return reg;
+	err = mv88e6xxx_port_read(chip, port, PORT_CONTROL, &reg);
+	if (err)
+		return err;
 
 	oldstate = reg & PORT_CONTROL_STATE_MASK;
 
-	if (oldstate != state) {
-		/* Flush forwarding database if we're moving a port
-		 * from Learning or Forwarding state to Disabled or
-		 * Blocking or Listening state.
-		 */
-		if ((oldstate == PORT_CONTROL_STATE_LEARNING ||
-		     oldstate == PORT_CONTROL_STATE_FORWARDING) &&
-		    (state == PORT_CONTROL_STATE_DISABLED ||
-		     state == PORT_CONTROL_STATE_BLOCKING)) {
-			ret = _mv88e6xxx_atu_remove(chip, 0, port, false);
-			if (ret)
-				return ret;
-		}
+	reg &= ~PORT_CONTROL_STATE_MASK;
+	reg |= state;
 
-		reg = (reg & ~PORT_CONTROL_STATE_MASK) | state;
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_CONTROL,
-					   reg);
-		if (ret)
-			return ret;
+	err = mv88e6xxx_port_write(chip, port, PORT_CONTROL, reg);
+	if (err)
+		return err;
 
-		netdev_dbg(ds->ports[port].netdev, "PortState %s (was %s)\n",
-			   mv88e6xxx_port_state_names[state],
-			   mv88e6xxx_port_state_names[oldstate]);
-	}
+	netdev_dbg(ds->ports[port].netdev, "PortState %s (was %s)\n",
+		   mv88e6xxx_port_state_names[state],
+		   mv88e6xxx_port_state_names[oldstate]);
 
-	return ret;
+	return 0;
 }
 
 static int _mv88e6xxx_port_based_vlan_map(struct mv88e6xxx_chip *chip, int port)
 {
 	struct net_device *bridge = chip->ports[port].bridge_dev;
-	const u16 mask = (1 << chip->info->num_ports) - 1;
+	const u16 mask = (1 << mv88e6xxx_num_ports(chip)) - 1;
 	struct dsa_switch *ds = chip->ds;
 	u16 output_ports = 0;
-	int reg;
+	u16 reg;
+	int err;
 	int i;
 
 	/* allow CPU port or DSA link(s) to send frames to every port */
 	if (dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)) {
 		output_ports = mask;
 	} else {
-		for (i = 0; i < chip->info->num_ports; ++i) {
+		for (i = 0; i < mv88e6xxx_num_ports(chip); ++i) {
 			/* allow sending frames to every group member */
 			if (bridge && chip->ports[i].bridge_dev == bridge)
 				output_ports |= BIT(i);
@@ -1137,20 +1132,20 @@ static int _mv88e6xxx_port_based_vlan_map(struct mv88e6xxx_chip *chip, int port)
 	/* prevent frames from going back out of the port they came in on */
 	output_ports &= ~BIT(port);
 
-	reg = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_BASE_VLAN);
-	if (reg < 0)
-		return reg;
+	err = mv88e6xxx_port_read(chip, port, PORT_BASE_VLAN, &reg);
+	if (err)
+		return err;
 
 	reg &= ~mask;
 	reg |= output_ports & mask;
 
-	return _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_BASE_VLAN, reg);
+	return mv88e6xxx_port_write(chip, port, PORT_BASE_VLAN, reg);
 }
 
 static void mv88e6xxx_port_stp_state_set(struct dsa_switch *ds, int port,
 					 u8 state)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int stp_state;
 	int err;
 
@@ -1181,27 +1176,39 @@ static void mv88e6xxx_port_stp_state_set(struct dsa_switch *ds, int port,
 			   mv88e6xxx_port_state_names[stp_state]);
 }
 
+static void mv88e6xxx_port_fast_age(struct dsa_switch *ds, int port)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+	int err;
+
+	mutex_lock(&chip->reg_lock);
+	err = _mv88e6xxx_atu_remove(chip, 0, port, false);
+	mutex_unlock(&chip->reg_lock);
+
+	if (err)
+		netdev_err(ds->ports[port].netdev, "failed to flush ATU\n");
+}
+
 static int _mv88e6xxx_port_pvid(struct mv88e6xxx_chip *chip, int port,
 				u16 *new, u16 *old)
 {
 	struct dsa_switch *ds = chip->ds;
-	u16 pvid;
-	int ret;
+	u16 pvid, reg;
+	int err;
 
-	ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_DEFAULT_VLAN);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_port_read(chip, port, PORT_DEFAULT_VLAN, &reg);
+	if (err)
+		return err;
 
-	pvid = ret & PORT_DEFAULT_VLAN_MASK;
+	pvid = reg & PORT_DEFAULT_VLAN_MASK;
 
 	if (new) {
-		ret &= ~PORT_DEFAULT_VLAN_MASK;
-		ret |= *new & PORT_DEFAULT_VLAN_MASK;
+		reg &= ~PORT_DEFAULT_VLAN_MASK;
+		reg |= *new & PORT_DEFAULT_VLAN_MASK;
 
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_DEFAULT_VLAN, ret);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_DEFAULT_VLAN, reg);
+		if (err)
+			return err;
 
 		netdev_dbg(ds->ports[port].netdev,
 			   "DefaultVID %d (was %d)\n", *new, pvid);
@@ -1227,17 +1234,16 @@ static int _mv88e6xxx_port_pvid_set(struct mv88e6xxx_chip *chip,
 
 static int _mv88e6xxx_vtu_wait(struct mv88e6xxx_chip *chip)
 {
-	return _mv88e6xxx_wait(chip, REG_GLOBAL, GLOBAL_VTU_OP,
-			       GLOBAL_VTU_OP_BUSY);
+	return mv88e6xxx_g1_wait(chip, GLOBAL_VTU_OP, GLOBAL_VTU_OP_BUSY);
 }
 
 static int _mv88e6xxx_vtu_cmd(struct mv88e6xxx_chip *chip, u16 op)
 {
-	int ret;
+	int err;
 
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_OP, op);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_write(chip, GLOBAL_VTU_OP, op);
+	if (err)
+		return err;
 
 	return _mv88e6xxx_vtu_wait(chip);
 }
@@ -1254,23 +1260,21 @@ static int _mv88e6xxx_vtu_stu_flush(struct mv88e6xxx_chip *chip)
 }
 
 static int _mv88e6xxx_vtu_stu_data_read(struct mv88e6xxx_chip *chip,
-					struct mv88e6xxx_vtu_stu_entry *entry,
+					struct mv88e6xxx_vtu_entry *entry,
 					unsigned int nibble_offset)
 {
 	u16 regs[3];
-	int i;
-	int ret;
+	int i, err;
 
 	for (i = 0; i < 3; ++i) {
-		ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
-					  GLOBAL_VTU_DATA_0_3 + i);
-		if (ret < 0)
-			return ret;
+		u16 *reg = &regs[i];
 
-		regs[i] = ret;
+		err = mv88e6xxx_g1_read(chip, GLOBAL_VTU_DATA_0_3 + i, reg);
+		if (err)
+			return err;
 	}
 
-	for (i = 0; i < chip->info->num_ports; ++i) {
+	for (i = 0; i < mv88e6xxx_num_ports(chip); ++i) {
 		unsigned int shift = (i % 4) * 4 + nibble_offset;
 		u16 reg = regs[i / 4];
 
@@ -1281,26 +1285,25 @@ static int _mv88e6xxx_vtu_stu_data_read(struct mv88e6xxx_chip *chip,
 }
 
 static int mv88e6xxx_vtu_data_read(struct mv88e6xxx_chip *chip,
-				   struct mv88e6xxx_vtu_stu_entry *entry)
+				   struct mv88e6xxx_vtu_entry *entry)
 {
 	return _mv88e6xxx_vtu_stu_data_read(chip, entry, 0);
 }
 
 static int mv88e6xxx_stu_data_read(struct mv88e6xxx_chip *chip,
-				   struct mv88e6xxx_vtu_stu_entry *entry)
+				   struct mv88e6xxx_vtu_entry *entry)
 {
 	return _mv88e6xxx_vtu_stu_data_read(chip, entry, 2);
 }
 
 static int _mv88e6xxx_vtu_stu_data_write(struct mv88e6xxx_chip *chip,
-					 struct mv88e6xxx_vtu_stu_entry *entry,
+					 struct mv88e6xxx_vtu_entry *entry,
 					 unsigned int nibble_offset)
 {
 	u16 regs[3] = { 0 };
-	int i;
-	int ret;
+	int i, err;
 
-	for (i = 0; i < chip->info->num_ports; ++i) {
+	for (i = 0; i < mv88e6xxx_num_ports(chip); ++i) {
 		unsigned int shift = (i % 4) * 4 + nibble_offset;
 		u8 data = entry->data[i];
 
@@ -1308,86 +1311,85 @@ static int _mv88e6xxx_vtu_stu_data_write(struct mv88e6xxx_chip *chip,
 	}
 
 	for (i = 0; i < 3; ++i) {
-		ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL,
-					   GLOBAL_VTU_DATA_0_3 + i, regs[i]);
-		if (ret < 0)
-			return ret;
+		u16 reg = regs[i];
+
+		err = mv88e6xxx_g1_write(chip, GLOBAL_VTU_DATA_0_3 + i, reg);
+		if (err)
+			return err;
 	}
 
 	return 0;
 }
 
 static int mv88e6xxx_vtu_data_write(struct mv88e6xxx_chip *chip,
-				    struct mv88e6xxx_vtu_stu_entry *entry)
+				    struct mv88e6xxx_vtu_entry *entry)
 {
 	return _mv88e6xxx_vtu_stu_data_write(chip, entry, 0);
 }
 
 static int mv88e6xxx_stu_data_write(struct mv88e6xxx_chip *chip,
-				    struct mv88e6xxx_vtu_stu_entry *entry)
+				    struct mv88e6xxx_vtu_entry *entry)
 {
 	return _mv88e6xxx_vtu_stu_data_write(chip, entry, 2);
 }
 
 static int _mv88e6xxx_vtu_vid_write(struct mv88e6xxx_chip *chip, u16 vid)
 {
-	return _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_VID,
-				    vid & GLOBAL_VTU_VID_MASK);
+	return mv88e6xxx_g1_write(chip, GLOBAL_VTU_VID,
+				  vid & GLOBAL_VTU_VID_MASK);
 }
 
 static int _mv88e6xxx_vtu_getnext(struct mv88e6xxx_chip *chip,
-				  struct mv88e6xxx_vtu_stu_entry *entry)
+				  struct mv88e6xxx_vtu_entry *entry)
 {
-	struct mv88e6xxx_vtu_stu_entry next = { 0 };
-	int ret;
+	struct mv88e6xxx_vtu_entry next = { 0 };
+	u16 val;
+	int err;
 
-	ret = _mv88e6xxx_vtu_wait(chip);
-	if (ret < 0)
-		return ret;
+	err = _mv88e6xxx_vtu_wait(chip);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_vtu_cmd(chip, GLOBAL_VTU_OP_VTU_GET_NEXT);
-	if (ret < 0)
-		return ret;
+	err = _mv88e6xxx_vtu_cmd(chip, GLOBAL_VTU_OP_VTU_GET_NEXT);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_VTU_VID);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_read(chip, GLOBAL_VTU_VID, &val);
+	if (err)
+		return err;
 
-	next.vid = ret & GLOBAL_VTU_VID_MASK;
-	next.valid = !!(ret & GLOBAL_VTU_VID_VALID);
+	next.vid = val & GLOBAL_VTU_VID_MASK;
+	next.valid = !!(val & GLOBAL_VTU_VID_VALID);
 
 	if (next.valid) {
-		ret = mv88e6xxx_vtu_data_read(chip, &next);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_vtu_data_read(chip, &next);
+		if (err)
+			return err;
 
-		if (mv88e6xxx_has_fid_reg(chip)) {
-			ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
-						  GLOBAL_VTU_FID);
-			if (ret < 0)
-				return ret;
+		if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G1_VTU_FID)) {
+			err = mv88e6xxx_g1_read(chip, GLOBAL_VTU_FID, &val);
+			if (err)
+				return err;
 
-			next.fid = ret & GLOBAL_VTU_FID_MASK;
+			next.fid = val & GLOBAL_VTU_FID_MASK;
 		} else if (mv88e6xxx_num_databases(chip) == 256) {
 			/* VTU DBNum[7:4] are located in VTU Operation 11:8, and
 			 * VTU DBNum[3:0] are located in VTU Operation 3:0
 			 */
-			ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
-						  GLOBAL_VTU_OP);
-			if (ret < 0)
-				return ret;
+			err = mv88e6xxx_g1_read(chip, GLOBAL_VTU_OP, &val);
+			if (err)
+				return err;
 
-			next.fid = (ret & 0xf00) >> 4;
-			next.fid |= ret & 0xf;
+			next.fid = (val & 0xf00) >> 4;
+			next.fid |= val & 0xf;
 		}
 
 		if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_STU)) {
-			ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
-						  GLOBAL_VTU_SID);
-			if (ret < 0)
-				return ret;
+			err = mv88e6xxx_g1_read(chip, GLOBAL_VTU_SID, &val);
+			if (err)
+				return err;
 
-			next.sid = ret & GLOBAL_VTU_SID_MASK;
+			next.sid = val & GLOBAL_VTU_SID_MASK;
 		}
 	}
 
@@ -1399,8 +1401,8 @@ static int mv88e6xxx_port_vlan_dump(struct dsa_switch *ds, int port,
 				    struct switchdev_obj_port_vlan *vlan,
 				    int (*cb)(struct switchdev_obj *obj))
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	struct mv88e6xxx_vtu_stu_entry next;
+	struct mv88e6xxx_chip *chip = ds->priv;
+	struct mv88e6xxx_vtu_entry next;
 	u16 pvid;
 	int err;
 
@@ -1451,38 +1453,36 @@ unlock:
 }
 
 static int _mv88e6xxx_vtu_loadpurge(struct mv88e6xxx_chip *chip,
-				    struct mv88e6xxx_vtu_stu_entry *entry)
+				    struct mv88e6xxx_vtu_entry *entry)
 {
 	u16 op = GLOBAL_VTU_OP_VTU_LOAD_PURGE;
 	u16 reg = 0;
-	int ret;
+	int err;
 
-	ret = _mv88e6xxx_vtu_wait(chip);
-	if (ret < 0)
-		return ret;
+	err = _mv88e6xxx_vtu_wait(chip);
+	if (err)
+		return err;
 
 	if (!entry->valid)
 		goto loadpurge;
 
 	/* Write port member tags */
-	ret = mv88e6xxx_vtu_data_write(chip, entry);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_vtu_data_write(chip, entry);
+	if (err)
+		return err;
 
 	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_STU)) {
 		reg = entry->sid & GLOBAL_VTU_SID_MASK;
-		ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_SID,
-					   reg);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_g1_write(chip, GLOBAL_VTU_SID, reg);
+		if (err)
+			return err;
 	}
 
-	if (mv88e6xxx_has_fid_reg(chip)) {
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G1_VTU_FID)) {
 		reg = entry->fid & GLOBAL_VTU_FID_MASK;
-		ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_FID,
-					   reg);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_g1_write(chip, GLOBAL_VTU_FID, reg);
+		if (err)
+			return err;
 	} else if (mv88e6xxx_num_databases(chip) == 256) {
 		/* VTU DBNum[7:4] are located in VTU Operation 11:8, and
 		 * VTU DBNum[3:0] are located in VTU Operation 3:0
@@ -1494,48 +1494,49 @@ static int _mv88e6xxx_vtu_loadpurge(struct mv88e6xxx_chip *chip,
 	reg = GLOBAL_VTU_VID_VALID;
 loadpurge:
 	reg |= entry->vid & GLOBAL_VTU_VID_MASK;
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_VID, reg);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_write(chip, GLOBAL_VTU_VID, reg);
+	if (err)
+		return err;
 
 	return _mv88e6xxx_vtu_cmd(chip, op);
 }
 
 static int _mv88e6xxx_stu_getnext(struct mv88e6xxx_chip *chip, u8 sid,
-				  struct mv88e6xxx_vtu_stu_entry *entry)
+				  struct mv88e6xxx_vtu_entry *entry)
 {
-	struct mv88e6xxx_vtu_stu_entry next = { 0 };
-	int ret;
+	struct mv88e6xxx_vtu_entry next = { 0 };
+	u16 val;
+	int err;
 
-	ret = _mv88e6xxx_vtu_wait(chip);
-	if (ret < 0)
-		return ret;
+	err = _mv88e6xxx_vtu_wait(chip);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_SID,
-				   sid & GLOBAL_VTU_SID_MASK);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_write(chip, GLOBAL_VTU_SID,
+				 sid & GLOBAL_VTU_SID_MASK);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_vtu_cmd(chip, GLOBAL_VTU_OP_STU_GET_NEXT);
-	if (ret < 0)
-		return ret;
+	err = _mv88e6xxx_vtu_cmd(chip, GLOBAL_VTU_OP_STU_GET_NEXT);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_VTU_SID);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_read(chip, GLOBAL_VTU_SID, &val);
+	if (err)
+		return err;
 
-	next.sid = ret & GLOBAL_VTU_SID_MASK;
+	next.sid = val & GLOBAL_VTU_SID_MASK;
 
-	ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_VTU_VID);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_read(chip, GLOBAL_VTU_VID, &val);
+	if (err)
+		return err;
 
-	next.valid = !!(ret & GLOBAL_VTU_VID_VALID);
+	next.valid = !!(val & GLOBAL_VTU_VID_VALID);
 
 	if (next.valid) {
-		ret = mv88e6xxx_stu_data_read(chip, &next);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_stu_data_read(chip, &next);
+		if (err)
+			return err;
 	}
 
 	*entry = next;
@@ -1543,33 +1544,33 @@ static int _mv88e6xxx_stu_getnext(struct mv88e6xxx_chip *chip, u8 sid,
 }
 
 static int _mv88e6xxx_stu_loadpurge(struct mv88e6xxx_chip *chip,
-				    struct mv88e6xxx_vtu_stu_entry *entry)
+				    struct mv88e6xxx_vtu_entry *entry)
 {
 	u16 reg = 0;
-	int ret;
+	int err;
 
-	ret = _mv88e6xxx_vtu_wait(chip);
-	if (ret < 0)
-		return ret;
+	err = _mv88e6xxx_vtu_wait(chip);
+	if (err)
+		return err;
 
 	if (!entry->valid)
 		goto loadpurge;
 
 	/* Write port states */
-	ret = mv88e6xxx_stu_data_write(chip, entry);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_stu_data_write(chip, entry);
+	if (err)
+		return err;
 
 	reg = GLOBAL_VTU_VID_VALID;
 loadpurge:
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_VID, reg);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_write(chip, GLOBAL_VTU_VID, reg);
+	if (err)
+		return err;
 
 	reg = entry->sid & GLOBAL_VTU_SID_MASK;
-	ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_VTU_SID, reg);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_write(chip, GLOBAL_VTU_SID, reg);
+	if (err)
+		return err;
 
 	return _mv88e6xxx_vtu_cmd(chip, GLOBAL_VTU_OP_STU_LOAD_PURGE);
 }
@@ -1580,7 +1581,8 @@ static int _mv88e6xxx_port_fid(struct mv88e6xxx_chip *chip, int port,
 	struct dsa_switch *ds = chip->ds;
 	u16 upper_mask;
 	u16 fid;
-	int ret;
+	u16 reg;
+	int err;
 
 	if (mv88e6xxx_num_databases(chip) == 4096)
 		upper_mask = 0xff;
@@ -1590,37 +1592,35 @@ static int _mv88e6xxx_port_fid(struct mv88e6xxx_chip *chip, int port,
 		return -EOPNOTSUPP;
 
 	/* Port's default FID bits 3:0 are located in reg 0x06, offset 12 */
-	ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_BASE_VLAN);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_port_read(chip, port, PORT_BASE_VLAN, &reg);
+	if (err)
+		return err;
 
-	fid = (ret & PORT_BASE_VLAN_FID_3_0_MASK) >> 12;
+	fid = (reg & PORT_BASE_VLAN_FID_3_0_MASK) >> 12;
 
 	if (new) {
-		ret &= ~PORT_BASE_VLAN_FID_3_0_MASK;
-		ret |= (*new << 12) & PORT_BASE_VLAN_FID_3_0_MASK;
+		reg &= ~PORT_BASE_VLAN_FID_3_0_MASK;
+		reg |= (*new << 12) & PORT_BASE_VLAN_FID_3_0_MASK;
 
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_BASE_VLAN,
-					   ret);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_BASE_VLAN, reg);
+		if (err)
+			return err;
 	}
 
 	/* Port's default FID bits 11:4 are located in reg 0x05, offset 0 */
-	ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_CONTROL_1);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_port_read(chip, port, PORT_CONTROL_1, &reg);
+	if (err)
+		return err;
 
-	fid |= (ret & upper_mask) << 4;
+	fid |= (reg & upper_mask) << 4;
 
 	if (new) {
-		ret &= ~upper_mask;
-		ret |= (*new >> 4) & upper_mask;
+		reg &= ~upper_mask;
+		reg |= (*new >> 4) & upper_mask;
 
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_CONTROL_1,
-					   ret);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_CONTROL_1, reg);
+		if (err)
+			return err;
 
 		netdev_dbg(ds->ports[port].netdev,
 			   "FID %d (was %d)\n", *new, fid);
@@ -1647,13 +1647,13 @@ static int _mv88e6xxx_port_fid_set(struct mv88e6xxx_chip *chip,
 static int _mv88e6xxx_fid_new(struct mv88e6xxx_chip *chip, u16 *fid)
 {
 	DECLARE_BITMAP(fid_bitmap, MV88E6XXX_N_FID);
-	struct mv88e6xxx_vtu_stu_entry vlan;
+	struct mv88e6xxx_vtu_entry vlan;
 	int i, err;
 
 	bitmap_zero(fid_bitmap, MV88E6XXX_N_FID);
 
 	/* Set every FID bit used by the (un)bridged ports */
-	for (i = 0; i < chip->info->num_ports; ++i) {
+	for (i = 0; i < mv88e6xxx_num_ports(chip); ++i) {
 		err = _mv88e6xxx_port_fid_get(chip, i, fid);
 		if (err)
 			return err;
@@ -1689,10 +1689,10 @@ static int _mv88e6xxx_fid_new(struct mv88e6xxx_chip *chip, u16 *fid)
 }
 
 static int _mv88e6xxx_vtu_new(struct mv88e6xxx_chip *chip, u16 vid,
-			      struct mv88e6xxx_vtu_stu_entry *entry)
+			      struct mv88e6xxx_vtu_entry *entry)
 {
 	struct dsa_switch *ds = chip->ds;
-	struct mv88e6xxx_vtu_stu_entry vlan = {
+	struct mv88e6xxx_vtu_entry vlan = {
 		.valid = true,
 		.vid = vid,
 	};
@@ -1703,14 +1703,14 @@ static int _mv88e6xxx_vtu_new(struct mv88e6xxx_chip *chip, u16 vid,
 		return err;
 
 	/* exclude all ports except the CPU and DSA ports */
-	for (i = 0; i < chip->info->num_ports; ++i)
+	for (i = 0; i < mv88e6xxx_num_ports(chip); ++i)
 		vlan.data[i] = dsa_is_cpu_port(ds, i) || dsa_is_dsa_port(ds, i)
 			? GLOBAL_VTU_DATA_MEMBER_TAG_UNMODIFIED
 			: GLOBAL_VTU_DATA_MEMBER_TAG_NON_MEMBER;
 
 	if (mv88e6xxx_6097_family(chip) || mv88e6xxx_6165_family(chip) ||
 	    mv88e6xxx_6351_family(chip) || mv88e6xxx_6352_family(chip)) {
-		struct mv88e6xxx_vtu_stu_entry vstp;
+		struct mv88e6xxx_vtu_entry vstp;
 
 		/* Adding a VTU entry requires a valid STU entry. As VSTP is not
 		 * implemented, only one STU entry is needed to cover all VTU
@@ -1737,7 +1737,7 @@ static int _mv88e6xxx_vtu_new(struct mv88e6xxx_chip *chip, u16 vid,
 }
 
 static int _mv88e6xxx_vtu_get(struct mv88e6xxx_chip *chip, u16 vid,
-			      struct mv88e6xxx_vtu_stu_entry *entry, bool creat)
+			      struct mv88e6xxx_vtu_entry *entry, bool creat)
 {
 	int err;
 
@@ -1768,8 +1768,8 @@ static int _mv88e6xxx_vtu_get(struct mv88e6xxx_chip *chip, u16 vid,
 static int mv88e6xxx_port_check_hw_vlan(struct dsa_switch *ds, int port,
 					u16 vid_begin, u16 vid_end)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	struct mv88e6xxx_vtu_stu_entry vlan;
+	struct mv88e6xxx_chip *chip = ds->priv;
+	struct mv88e6xxx_vtu_entry vlan;
 	int i, err;
 
 	if (!vid_begin)
@@ -1792,7 +1792,7 @@ static int mv88e6xxx_port_check_hw_vlan(struct dsa_switch *ds, int port,
 		if (vlan.vid > vid_end)
 			break;
 
-		for (i = 0; i < chip->info->num_ports; ++i) {
+		for (i = 0; i < mv88e6xxx_num_ports(chip); ++i) {
 			if (dsa_is_dsa_port(ds, i) || dsa_is_cpu_port(ds, i))
 				continue;
 
@@ -1829,29 +1829,29 @@ static const char * const mv88e6xxx_port_8021q_mode_names[] = {
 static int mv88e6xxx_port_vlan_filtering(struct dsa_switch *ds, int port,
 					 bool vlan_filtering)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	u16 old, new = vlan_filtering ? PORT_CONTROL_2_8021Q_SECURE :
 		PORT_CONTROL_2_8021Q_DISABLED;
-	int ret;
+	u16 reg;
+	int err;
 
 	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_VTU))
 		return -EOPNOTSUPP;
 
 	mutex_lock(&chip->reg_lock);
 
-	ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_CONTROL_2);
-	if (ret < 0)
+	err = mv88e6xxx_port_read(chip, port, PORT_CONTROL_2, &reg);
+	if (err)
 		goto unlock;
 
-	old = ret & PORT_CONTROL_2_8021Q_MASK;
+	old = reg & PORT_CONTROL_2_8021Q_MASK;
 
 	if (new != old) {
-		ret &= ~PORT_CONTROL_2_8021Q_MASK;
-		ret |= new & PORT_CONTROL_2_8021Q_MASK;
+		reg &= ~PORT_CONTROL_2_8021Q_MASK;
+		reg |= new & PORT_CONTROL_2_8021Q_MASK;
 
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_CONTROL_2,
-					   ret);
-		if (ret < 0)
+		err = mv88e6xxx_port_write(chip, port, PORT_CONTROL_2, reg);
+		if (err)
 			goto unlock;
 
 		netdev_dbg(ds->ports[port].netdev, "802.1Q Mode %s (was %s)\n",
@@ -1859,11 +1859,11 @@ static int mv88e6xxx_port_vlan_filtering(struct dsa_switch *ds, int port,
 			   mv88e6xxx_port_8021q_mode_names[old]);
 	}
 
-	ret = 0;
+	err = 0;
 unlock:
 	mutex_unlock(&chip->reg_lock);
 
-	return ret;
+	return err;
 }
 
 static int
@@ -1871,7 +1871,7 @@ mv88e6xxx_port_vlan_prepare(struct dsa_switch *ds, int port,
 			    const struct switchdev_obj_port_vlan *vlan,
 			    struct switchdev_trans *trans)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
 
 	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_VTU))
@@ -1894,7 +1894,7 @@ mv88e6xxx_port_vlan_prepare(struct dsa_switch *ds, int port,
 static int _mv88e6xxx_port_vlan_add(struct mv88e6xxx_chip *chip, int port,
 				    u16 vid, bool untagged)
 {
-	struct mv88e6xxx_vtu_stu_entry vlan;
+	struct mv88e6xxx_vtu_entry vlan;
 	int err;
 
 	err = _mv88e6xxx_vtu_get(chip, vid, &vlan, true);
@@ -1912,7 +1912,7 @@ static void mv88e6xxx_port_vlan_add(struct dsa_switch *ds, int port,
 				    const struct switchdev_obj_port_vlan *vlan,
 				    struct switchdev_trans *trans)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
 	bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID;
 	u16 vid;
@@ -1939,7 +1939,7 @@ static int _mv88e6xxx_port_vlan_del(struct mv88e6xxx_chip *chip,
 				    int port, u16 vid)
 {
 	struct dsa_switch *ds = chip->ds;
-	struct mv88e6xxx_vtu_stu_entry vlan;
+	struct mv88e6xxx_vtu_entry vlan;
 	int i, err;
 
 	err = _mv88e6xxx_vtu_get(chip, vid, &vlan, false);
@@ -1954,7 +1954,7 @@ static int _mv88e6xxx_port_vlan_del(struct mv88e6xxx_chip *chip,
 
 	/* keep the VLAN unless all ports are excluded */
 	vlan.valid = false;
-	for (i = 0; i < chip->info->num_ports; ++i) {
+	for (i = 0; i < mv88e6xxx_num_ports(chip); ++i) {
 		if (dsa_is_cpu_port(ds, i) || dsa_is_dsa_port(ds, i))
 			continue;
 
@@ -1974,7 +1974,7 @@ static int _mv88e6xxx_port_vlan_del(struct mv88e6xxx_chip *chip,
 static int mv88e6xxx_port_vlan_del(struct dsa_switch *ds, int port,
 				   const struct switchdev_obj_port_vlan *vlan)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	u16 pvid, vid;
 	int err = 0;
 
@@ -2008,14 +2008,13 @@ unlock:
 static int _mv88e6xxx_atu_mac_write(struct mv88e6xxx_chip *chip,
 				    const unsigned char *addr)
 {
-	int i, ret;
+	int i, err;
 
 	for (i = 0; i < 3; i++) {
-		ret = _mv88e6xxx_reg_write(
-			chip, REG_GLOBAL, GLOBAL_ATU_MAC_01 + i,
-			(addr[i * 2] << 8) | addr[i * 2 + 1]);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_g1_write(chip, GLOBAL_ATU_MAC_01 + i,
+					 (addr[i * 2] << 8) | addr[i * 2 + 1]);
+		if (err)
+			return err;
 	}
 
 	return 0;
@@ -2024,15 +2023,16 @@ static int _mv88e6xxx_atu_mac_write(struct mv88e6xxx_chip *chip,
 static int _mv88e6xxx_atu_mac_read(struct mv88e6xxx_chip *chip,
 				   unsigned char *addr)
 {
-	int i, ret;
+	u16 val;
+	int i, err;
 
 	for (i = 0; i < 3; i++) {
-		ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL,
-					  GLOBAL_ATU_MAC_01 + i);
-		if (ret < 0)
-			return ret;
-		addr[i * 2] = ret >> 8;
-		addr[i * 2 + 1] = ret & 0xff;
+		err = mv88e6xxx_g1_read(chip, GLOBAL_ATU_MAC_01 + i, &val);
+		if (err)
+			return err;
+
+		addr[i * 2] = val >> 8;
+		addr[i * 2 + 1] = val & 0xff;
 	}
 
 	return 0;
@@ -2058,12 +2058,48 @@ static int _mv88e6xxx_atu_load(struct mv88e6xxx_chip *chip,
 	return _mv88e6xxx_atu_cmd(chip, entry->fid, GLOBAL_ATU_OP_LOAD_DB);
 }
 
-static int _mv88e6xxx_port_fdb_load(struct mv88e6xxx_chip *chip, int port,
-				    const unsigned char *addr, u16 vid,
-				    u8 state)
+static int _mv88e6xxx_atu_getnext(struct mv88e6xxx_chip *chip, u16 fid,
+				  struct mv88e6xxx_atu_entry *entry);
+
+static int mv88e6xxx_atu_get(struct mv88e6xxx_chip *chip, int fid,
+			     const u8 *addr, struct mv88e6xxx_atu_entry *entry)
 {
-	struct mv88e6xxx_atu_entry entry = { 0 };
-	struct mv88e6xxx_vtu_stu_entry vlan;
+	struct mv88e6xxx_atu_entry next;
+	int err;
+
+	eth_broadcast_addr(next.mac);
+
+	err = _mv88e6xxx_atu_mac_write(chip, next.mac);
+	if (err)
+		return err;
+
+	do {
+		err = _mv88e6xxx_atu_getnext(chip, fid, &next);
+		if (err)
+			return err;
+
+		if (next.state == GLOBAL_ATU_DATA_STATE_UNUSED)
+			break;
+
+		if (ether_addr_equal(next.mac, addr)) {
+			*entry = next;
+			return 0;
+		}
+	} while (!is_broadcast_ether_addr(next.mac));
+
+	memset(entry, 0, sizeof(*entry));
+	entry->fid = fid;
+	ether_addr_copy(entry->mac, addr);
+
+	return 0;
+}
+
+static int mv88e6xxx_port_db_load_purge(struct mv88e6xxx_chip *chip, int port,
+					const unsigned char *addr, u16 vid,
+					u8 state)
+{
+	struct mv88e6xxx_vtu_entry vlan;
+	struct mv88e6xxx_atu_entry entry;
 	int err;
 
 	/* Null VLAN ID corresponds to the port private database */
@@ -2074,12 +2110,18 @@ static int _mv88e6xxx_port_fdb_load(struct mv88e6xxx_chip *chip, int port,
 	if (err)
 		return err;
 
-	entry.fid = vlan.fid;
-	entry.state = state;
-	ether_addr_copy(entry.mac, addr);
-	if (state != GLOBAL_ATU_DATA_STATE_UNUSED) {
-		entry.trunk = false;
-		entry.portv_trunkid = BIT(port);
+	err = mv88e6xxx_atu_get(chip, vlan.fid, addr, &entry);
+	if (err)
+		return err;
+
+	/* Purge the ATU entry only if no port is using it anymore */
+	if (state == GLOBAL_ATU_DATA_STATE_UNUSED) {
+		entry.portv_trunkid &= ~BIT(port);
+		if (!entry.portv_trunkid)
+			entry.state = GLOBAL_ATU_DATA_STATE_UNUSED;
+	} else {
+		entry.portv_trunkid |= BIT(port);
+		entry.state = state;
 	}
 
 	return _mv88e6xxx_atu_load(chip, &entry);
@@ -2099,61 +2141,59 @@ static void mv88e6xxx_port_fdb_add(struct dsa_switch *ds, int port,
 				   const struct switchdev_obj_port_fdb *fdb,
 				   struct switchdev_trans *trans)
 {
-	int state = is_multicast_ether_addr(fdb->addr) ?
-		GLOBAL_ATU_DATA_STATE_MC_STATIC :
-		GLOBAL_ATU_DATA_STATE_UC_STATIC;
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 
 	mutex_lock(&chip->reg_lock);
-	if (_mv88e6xxx_port_fdb_load(chip, port, fdb->addr, fdb->vid, state))
-		netdev_err(ds->ports[port].netdev,
-			   "failed to load MAC address\n");
+	if (mv88e6xxx_port_db_load_purge(chip, port, fdb->addr, fdb->vid,
+					 GLOBAL_ATU_DATA_STATE_UC_STATIC))
+		netdev_err(ds->ports[port].netdev, "failed to load unicast MAC address\n");
 	mutex_unlock(&chip->reg_lock);
 }
 
 static int mv88e6xxx_port_fdb_del(struct dsa_switch *ds, int port,
 				  const struct switchdev_obj_port_fdb *fdb)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	int ret;
+	struct mv88e6xxx_chip *chip = ds->priv;
+	int err;
 
 	mutex_lock(&chip->reg_lock);
-	ret = _mv88e6xxx_port_fdb_load(chip, port, fdb->addr, fdb->vid,
-				       GLOBAL_ATU_DATA_STATE_UNUSED);
+	err = mv88e6xxx_port_db_load_purge(chip, port, fdb->addr, fdb->vid,
+					   GLOBAL_ATU_DATA_STATE_UNUSED);
 	mutex_unlock(&chip->reg_lock);
 
-	return ret;
+	return err;
 }
 
 static int _mv88e6xxx_atu_getnext(struct mv88e6xxx_chip *chip, u16 fid,
 				  struct mv88e6xxx_atu_entry *entry)
 {
 	struct mv88e6xxx_atu_entry next = { 0 };
-	int ret;
+	u16 val;
+	int err;
 
 	next.fid = fid;
 
-	ret = _mv88e6xxx_atu_wait(chip);
-	if (ret < 0)
-		return ret;
+	err = _mv88e6xxx_atu_wait(chip);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_atu_cmd(chip, fid, GLOBAL_ATU_OP_GET_NEXT_DB);
-	if (ret < 0)
-		return ret;
+	err = _mv88e6xxx_atu_cmd(chip, fid, GLOBAL_ATU_OP_GET_NEXT_DB);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_atu_mac_read(chip, next.mac);
-	if (ret < 0)
-		return ret;
+	err = _mv88e6xxx_atu_mac_read(chip, next.mac);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, GLOBAL_ATU_DATA);
-	if (ret < 0)
-		return ret;
+	err = mv88e6xxx_g1_read(chip, GLOBAL_ATU_DATA, &val);
+	if (err)
+		return err;
 
-	next.state = ret & GLOBAL_ATU_DATA_STATE_MASK;
+	next.state = val & GLOBAL_ATU_DATA_STATE_MASK;
 	if (next.state != GLOBAL_ATU_DATA_STATE_UNUSED) {
 		unsigned int mask, shift;
 
-		if (ret & GLOBAL_ATU_DATA_TRUNK) {
+		if (val & GLOBAL_ATU_DATA_TRUNK) {
 			next.trunk = true;
 			mask = GLOBAL_ATU_DATA_TRUNK_ID_MASK;
 			shift = GLOBAL_ATU_DATA_TRUNK_ID_SHIFT;
@@ -2163,17 +2203,17 @@ static int _mv88e6xxx_atu_getnext(struct mv88e6xxx_chip *chip, u16 fid,
 			shift = GLOBAL_ATU_DATA_PORT_VECTOR_SHIFT;
 		}
 
-		next.portv_trunkid = (ret & mask) >> shift;
+		next.portv_trunkid = (val & mask) >> shift;
 	}
 
 	*entry = next;
 	return 0;
 }
 
-static int _mv88e6xxx_port_fdb_dump_one(struct mv88e6xxx_chip *chip,
-					u16 fid, u16 vid, int port,
-					struct switchdev_obj_port_fdb *fdb,
-					int (*cb)(struct switchdev_obj *obj))
+static int mv88e6xxx_port_db_dump_fid(struct mv88e6xxx_chip *chip,
+				      u16 fid, u16 vid, int port,
+				      struct switchdev_obj *obj,
+				      int (*cb)(struct switchdev_obj *obj))
 {
 	struct mv88e6xxx_atu_entry addr = {
 		.mac = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff },
@@ -2187,72 +2227,98 @@ static int _mv88e6xxx_port_fdb_dump_one(struct mv88e6xxx_chip *chip,
 	do {
 		err = _mv88e6xxx_atu_getnext(chip, fid, &addr);
 		if (err)
-			break;
+			return err;
 
 		if (addr.state == GLOBAL_ATU_DATA_STATE_UNUSED)
 			break;
 
-		if (!addr.trunk && addr.portv_trunkid & BIT(port)) {
-			bool is_static = addr.state ==
-				(is_multicast_ether_addr(addr.mac) ?
-				 GLOBAL_ATU_DATA_STATE_MC_STATIC :
-				 GLOBAL_ATU_DATA_STATE_UC_STATIC);
+		if (addr.trunk || (addr.portv_trunkid & BIT(port)) == 0)
+			continue;
+
+		if (obj->id == SWITCHDEV_OBJ_ID_PORT_FDB) {
+			struct switchdev_obj_port_fdb *fdb;
+
+			if (!is_unicast_ether_addr(addr.mac))
+				continue;
 
+			fdb = SWITCHDEV_OBJ_PORT_FDB(obj);
 			fdb->vid = vid;
 			ether_addr_copy(fdb->addr, addr.mac);
-			fdb->ndm_state = is_static ? NUD_NOARP : NUD_REACHABLE;
+			if (addr.state == GLOBAL_ATU_DATA_STATE_UC_STATIC)
+				fdb->ndm_state = NUD_NOARP;
+			else
+				fdb->ndm_state = NUD_REACHABLE;
+		} else if (obj->id == SWITCHDEV_OBJ_ID_PORT_MDB) {
+			struct switchdev_obj_port_mdb *mdb;
 
-			err = cb(&fdb->obj);
-			if (err)
-				break;
+			if (!is_multicast_ether_addr(addr.mac))
+				continue;
+
+			mdb = SWITCHDEV_OBJ_PORT_MDB(obj);
+			mdb->vid = vid;
+			ether_addr_copy(mdb->addr, addr.mac);
+		} else {
+			return -EOPNOTSUPP;
 		}
+
+		err = cb(obj);
+		if (err)
+			return err;
 	} while (!is_broadcast_ether_addr(addr.mac));
 
 	return err;
 }
 
-static int mv88e6xxx_port_fdb_dump(struct dsa_switch *ds, int port,
-				   struct switchdev_obj_port_fdb *fdb,
-				   int (*cb)(struct switchdev_obj *obj))
+static int mv88e6xxx_port_db_dump(struct mv88e6xxx_chip *chip, int port,
+				  struct switchdev_obj *obj,
+				  int (*cb)(struct switchdev_obj *obj))
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	struct mv88e6xxx_vtu_stu_entry vlan = {
+	struct mv88e6xxx_vtu_entry vlan = {
 		.vid = GLOBAL_VTU_VID_MASK, /* all ones */
 	};
 	u16 fid;
 	int err;
 
-	mutex_lock(&chip->reg_lock);
-
 	/* Dump port's default Filtering Information Database (VLAN ID 0) */
 	err = _mv88e6xxx_port_fid_get(chip, port, &fid);
 	if (err)
-		goto unlock;
+		return err;
 
-	err = _mv88e6xxx_port_fdb_dump_one(chip, fid, 0, port, fdb, cb);
+	err = mv88e6xxx_port_db_dump_fid(chip, fid, 0, port, obj, cb);
 	if (err)
-		goto unlock;
+		return err;
 
 	/* Dump VLANs' Filtering Information Databases */
 	err = _mv88e6xxx_vtu_vid_write(chip, vlan.vid);
 	if (err)
-		goto unlock;
+		return err;
 
 	do {
 		err = _mv88e6xxx_vtu_getnext(chip, &vlan);
 		if (err)
-			break;
+			return err;
 
 		if (!vlan.valid)
 			break;
 
-		err = _mv88e6xxx_port_fdb_dump_one(chip, vlan.fid, vlan.vid,
-						   port, fdb, cb);
+		err = mv88e6xxx_port_db_dump_fid(chip, vlan.fid, vlan.vid, port,
+						 obj, cb);
 		if (err)
-			break;
+			return err;
 	} while (vlan.vid < GLOBAL_VTU_VID_MASK);
 
-unlock:
+	return err;
+}
+
+static int mv88e6xxx_port_fdb_dump(struct dsa_switch *ds, int port,
+				   struct switchdev_obj_port_fdb *fdb,
+				   int (*cb)(struct switchdev_obj *obj))
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+	int err;
+
+	mutex_lock(&chip->reg_lock);
+	err = mv88e6xxx_port_db_dump(chip, port, &fdb->obj, cb);
 	mutex_unlock(&chip->reg_lock);
 
 	return err;
@@ -2261,7 +2327,7 @@ unlock:
 static int mv88e6xxx_port_bridge_join(struct dsa_switch *ds, int port,
 				      struct net_device *bridge)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int i, err = 0;
 
 	mutex_lock(&chip->reg_lock);
@@ -2269,7 +2335,7 @@ static int mv88e6xxx_port_bridge_join(struct dsa_switch *ds, int port,
 	/* Assign the bridge and remap each port's VLANTable */
 	chip->ports[port].bridge_dev = bridge;
 
-	for (i = 0; i < chip->info->num_ports; ++i) {
+	for (i = 0; i < mv88e6xxx_num_ports(chip); ++i) {
 		if (chip->ports[i].bridge_dev == bridge) {
 			err = _mv88e6xxx_port_based_vlan_map(chip, i);
 			if (err)
@@ -2284,7 +2350,7 @@ static int mv88e6xxx_port_bridge_join(struct dsa_switch *ds, int port,
 
 static void mv88e6xxx_port_bridge_leave(struct dsa_switch *ds, int port)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	struct net_device *bridge = chip->ports[port].bridge_dev;
 	int i;
 
@@ -2293,7 +2359,7 @@ static void mv88e6xxx_port_bridge_leave(struct dsa_switch *ds, int port)
 	/* Unassign the bridge and remap each port's VLANTable */
 	chip->ports[port].bridge_dev = NULL;
 
-	for (i = 0; i < chip->info->num_ports; ++i)
+	for (i = 0; i < mv88e6xxx_num_ports(chip); ++i)
 		if (i == port || chip->ports[i].bridge_dev == bridge)
 			if (_mv88e6xxx_port_based_vlan_map(chip, i))
 				netdev_warn(ds->ports[i].netdev,
@@ -2302,57 +2368,26 @@ static void mv88e6xxx_port_bridge_leave(struct dsa_switch *ds, int port)
 	mutex_unlock(&chip->reg_lock);
 }
 
-static int _mv88e6xxx_mdio_page_write(struct mv88e6xxx_chip *chip,
-				      int port, int page, int reg, int val)
-{
-	int ret;
-
-	ret = mv88e6xxx_mdio_write_indirect(chip, port, 0x16, page);
-	if (ret < 0)
-		goto restore_page_0;
-
-	ret = mv88e6xxx_mdio_write_indirect(chip, port, reg, val);
-restore_page_0:
-	mv88e6xxx_mdio_write_indirect(chip, port, 0x16, 0x0);
-
-	return ret;
-}
-
-static int _mv88e6xxx_mdio_page_read(struct mv88e6xxx_chip *chip,
-				     int port, int page, int reg)
-{
-	int ret;
-
-	ret = mv88e6xxx_mdio_write_indirect(chip, port, 0x16, page);
-	if (ret < 0)
-		goto restore_page_0;
-
-	ret = mv88e6xxx_mdio_read_indirect(chip, port, reg);
-restore_page_0:
-	mv88e6xxx_mdio_write_indirect(chip, port, 0x16, 0x0);
-
-	return ret;
-}
-
 static int mv88e6xxx_switch_reset(struct mv88e6xxx_chip *chip)
 {
 	bool ppu_active = mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU_ACTIVE);
 	u16 is_reset = (ppu_active ? 0x8800 : 0xc800);
 	struct gpio_desc *gpiod = chip->reset;
 	unsigned long timeout;
-	int ret;
+	u16 reg;
+	int err;
 	int i;
 
 	/* Set all ports to the disabled state. */
-	for (i = 0; i < chip->info->num_ports; i++) {
-		ret = _mv88e6xxx_reg_read(chip, REG_PORT(i), PORT_CONTROL);
-		if (ret < 0)
-			return ret;
+	for (i = 0; i < mv88e6xxx_num_ports(chip); i++) {
+		err = mv88e6xxx_port_read(chip, i, PORT_CONTROL, &reg);
+		if (err)
+			return err;
 
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(i), PORT_CONTROL,
-					   ret & 0xfffc);
-		if (ret)
-			return ret;
+		err = mv88e6xxx_port_write(chip, i, PORT_CONTROL,
+					   reg & 0xfffc);
+		if (err)
+			return err;
 	}
 
 	/* Wait for transmit queues to drain. */
@@ -2371,65 +2406,53 @@ static int mv88e6xxx_switch_reset(struct mv88e6xxx_chip *chip)
 	 * through global registers 0x18 and 0x19.
 	 */
 	if (ppu_active)
-		ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, 0x04, 0xc000);
+		err = mv88e6xxx_g1_write(chip, 0x04, 0xc000);
 	else
-		ret = _mv88e6xxx_reg_write(chip, REG_GLOBAL, 0x04, 0xc400);
-	if (ret)
-		return ret;
+		err = mv88e6xxx_g1_write(chip, 0x04, 0xc400);
+	if (err)
+		return err;
 
 	/* Wait up to one second for reset to complete. */
 	timeout = jiffies + 1 * HZ;
 	while (time_before(jiffies, timeout)) {
-		ret = _mv88e6xxx_reg_read(chip, REG_GLOBAL, 0x00);
-		if (ret < 0)
-			return ret;
+		err = mv88e6xxx_g1_read(chip, 0x00, &reg);
+		if (err)
+			return err;
 
-		if ((ret & is_reset) == is_reset)
+		if ((reg & is_reset) == is_reset)
 			break;
 		usleep_range(1000, 2000);
 	}
 	if (time_after(jiffies, timeout))
-		ret = -ETIMEDOUT;
+		err = -ETIMEDOUT;
 	else
-		ret = 0;
+		err = 0;
 
-	return ret;
+	return err;
 }
 
-static int mv88e6xxx_power_on_serdes(struct mv88e6xxx_chip *chip)
+static int mv88e6xxx_serdes_power_on(struct mv88e6xxx_chip *chip)
 {
-	int ret;
+	u16 val;
+	int err;
 
-	ret = _mv88e6xxx_mdio_page_read(chip, REG_FIBER_SERDES,
-					PAGE_FIBER_SERDES, MII_BMCR);
-	if (ret < 0)
-		return ret;
+	/* Clear Power Down bit */
+	err = mv88e6xxx_serdes_read(chip, MII_BMCR, &val);
+	if (err)
+		return err;
 
-	if (ret & BMCR_PDOWN) {
-		ret &= ~BMCR_PDOWN;
-		ret = _mv88e6xxx_mdio_page_write(chip, REG_FIBER_SERDES,
-						 PAGE_FIBER_SERDES, MII_BMCR,
-						 ret);
+	if (val & BMCR_PDOWN) {
+		val &= ~BMCR_PDOWN;
+		err = mv88e6xxx_serdes_write(chip, MII_BMCR, val);
 	}
 
-	return ret;
-}
-
-static int mv88e6xxx_port_read(struct mv88e6xxx_chip *chip, int port,
-			       int reg, u16 *val)
-{
-	int addr = chip->info->port_base_addr + port;
-
-	if (port >= chip->info->num_ports)
-		return -EINVAL;
-
-	return mv88e6xxx_read(chip, addr, reg, val);
+	return err;
 }
 
 static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 {
 	struct dsa_switch *ds = chip->ds;
-	int ret;
+	int err;
 	u16 reg;
 
 	if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
@@ -2442,7 +2465,7 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 		 * and all DSA ports to their maximum bandwidth and
 		 * full duplex.
 		 */
-		reg = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_PCS_CTRL);
+		err = mv88e6xxx_port_read(chip, port, PORT_PCS_CTRL, &reg);
 		if (dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)) {
 			reg &= ~PORT_PCS_CTRL_UNFORCED;
 			reg |= PORT_PCS_CTRL_FORCE_LINK |
@@ -2457,10 +2480,9 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 			reg |= PORT_PCS_CTRL_UNFORCED;
 		}
 
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_PCS_CTRL, reg);
-		if (ret)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_PCS_CTRL, reg);
+		if (err)
+			return err;
 	}
 
 	/* Port Control: disable Drop-on-Unlock, disable Drop-on-Lock,
@@ -2486,28 +2508,13 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 		PORT_CONTROL_USE_TAG | PORT_CONTROL_USE_IP |
 		PORT_CONTROL_STATE_FORWARDING;
 	if (dsa_is_cpu_port(ds, port)) {
-		if (mv88e6xxx_6095_family(chip) || mv88e6xxx_6185_family(chip))
-			reg |= PORT_CONTROL_DSA_TAG;
-		if (mv88e6xxx_6352_family(chip) ||
-		    mv88e6xxx_6351_family(chip) ||
-		    mv88e6xxx_6165_family(chip) ||
-		    mv88e6xxx_6097_family(chip) ||
-		    mv88e6xxx_6320_family(chip)) {
+		if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_EDSA))
 			reg |= PORT_CONTROL_FRAME_ETHER_TYPE_DSA |
-				PORT_CONTROL_FORWARD_UNKNOWN |
 				PORT_CONTROL_FORWARD_UNKNOWN_MC;
-		}
-
-		if (mv88e6xxx_6352_family(chip) ||
-		    mv88e6xxx_6351_family(chip) ||
-		    mv88e6xxx_6165_family(chip) ||
-		    mv88e6xxx_6097_family(chip) ||
-		    mv88e6xxx_6095_family(chip) ||
-		    mv88e6xxx_6065_family(chip) ||
-		    mv88e6xxx_6185_family(chip) ||
-		    mv88e6xxx_6320_family(chip)) {
-			reg |= PORT_CONTROL_EGRESS_ADD_TAG;
-		}
+		else
+			reg |= PORT_CONTROL_DSA_TAG;
+		reg |= PORT_CONTROL_EGRESS_ADD_TAG |
+			PORT_CONTROL_FORWARD_UNKNOWN;
 	}
 	if (dsa_is_dsa_port(ds, port)) {
 		if (mv88e6xxx_6095_family(chip) ||
@@ -2526,26 +2533,25 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 				PORT_CONTROL_FORWARD_UNKNOWN_MC;
 	}
 	if (reg) {
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_CONTROL, reg);
-		if (ret)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_CONTROL, reg);
+		if (err)
+			return err;
 	}
 
 	/* If this port is connected to a SerDes, make sure the SerDes is not
 	 * powered down.
 	 */
-	if (mv88e6xxx_6352_family(chip)) {
-		ret = _mv88e6xxx_reg_read(chip, REG_PORT(port), PORT_STATUS);
-		if (ret < 0)
-			return ret;
-		ret &= PORT_STATUS_CMODE_MASK;
-		if ((ret == PORT_STATUS_CMODE_100BASE_X) ||
-		    (ret == PORT_STATUS_CMODE_1000BASE_X) ||
-		    (ret == PORT_STATUS_CMODE_SGMII)) {
-			ret = mv88e6xxx_power_on_serdes(chip);
-			if (ret < 0)
-				return ret;
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_SERDES)) {
+		err = mv88e6xxx_port_read(chip, port, PORT_STATUS, &reg);
+		if (err)
+			return err;
+		reg &= PORT_STATUS_CMODE_MASK;
+		if ((reg == PORT_STATUS_CMODE_100BASE_X) ||
+		    (reg == PORT_STATUS_CMODE_1000BASE_X) ||
+		    (reg == PORT_STATUS_CMODE_SGMII)) {
+			err = mv88e6xxx_serdes_power_on(chip);
+			if (err < 0)
+				return err;
 		}
 	}
 
@@ -2579,10 +2585,9 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 	reg |= PORT_CONTROL_2_8021Q_DISABLED;
 
 	if (reg) {
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_CONTROL_2, reg);
-		if (ret)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_CONTROL_2, reg);
+		if (err)
+			return err;
 	}
 
 	/* Port Association Vector: when learning source addresses
@@ -2595,16 +2600,14 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 	if (dsa_is_cpu_port(ds, port))
 		reg = 0;
 
-	ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_ASSOC_VECTOR,
-				   reg);
-	if (ret)
-		return ret;
+	err = mv88e6xxx_port_write(chip, port, PORT_ASSOC_VECTOR, reg);
+	if (err)
+		return err;
 
 	/* Egress rate control 2: disable egress rate control. */
-	ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_RATE_CONTROL_2,
-				   0x0000);
-	if (ret)
-		return ret;
+	err = mv88e6xxx_port_write(chip, port, PORT_RATE_CONTROL_2, 0x0000);
+	if (err)
+		return err;
 
 	if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
 	    mv88e6xxx_6165_family(chip) || mv88e6xxx_6097_family(chip) ||
@@ -2613,107 +2616,108 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 		 * be paused for by the remote end or the period of
 		 * time that this port can pause the remote end.
 		 */
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_PAUSE_CTRL, 0x0000);
-		if (ret)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_PAUSE_CTRL, 0x0000);
+		if (err)
+			return err;
 
 		/* Port ATU control: disable limiting the number of
 		 * address database entries that this port is allowed
 		 * to use.
 		 */
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_ATU_CONTROL, 0x0000);
+		err = mv88e6xxx_port_write(chip, port, PORT_ATU_CONTROL,
+					   0x0000);
 		/* Priority Override: disable DA, SA and VTU priority
 		 * override.
 		 */
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_PRI_OVERRIDE, 0x0000);
-		if (ret)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_PRI_OVERRIDE,
+					   0x0000);
+		if (err)
+			return err;
 
 		/* Port Ethertype: use the Ethertype DSA Ethertype
 		 * value.
 		 */
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_ETH_TYPE, ETH_P_EDSA);
-		if (ret)
-			return ret;
+		if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_EDSA)) {
+			err = mv88e6xxx_port_write(chip, port, PORT_ETH_TYPE,
+						   ETH_P_EDSA);
+			if (err)
+				return err;
+		}
+
 		/* Tag Remap: use an identity 802.1p prio -> switch
 		 * prio mapping.
 		 */
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_TAG_REGMAP_0123, 0x3210);
-		if (ret)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_TAG_REGMAP_0123,
+					   0x3210);
+		if (err)
+			return err;
 
 		/* Tag Remap 2: use an identity 802.1p prio -> switch
 		 * prio mapping.
 		 */
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_TAG_REGMAP_4567, 0x7654);
-		if (ret)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_TAG_REGMAP_4567,
+					   0x7654);
+		if (err)
+			return err;
 	}
 
+	/* Rate Control: disable ingress rate limiting. */
 	if (mv88e6xxx_6352_family(chip) || mv88e6xxx_6351_family(chip) ||
 	    mv88e6xxx_6165_family(chip) || mv88e6xxx_6097_family(chip) ||
-	    mv88e6xxx_6185_family(chip) || mv88e6xxx_6095_family(chip) ||
 	    mv88e6xxx_6320_family(chip)) {
-		/* Rate Control: disable ingress rate limiting. */
-		ret = _mv88e6xxx_reg_write(chip, REG_PORT(port),
-					   PORT_RATE_CONTROL, 0x0001);
-		if (ret)
-			return ret;
+		err = mv88e6xxx_port_write(chip, port, PORT_RATE_CONTROL,
+					   0x0001);
+		if (err)
+			return err;
+	} else if (mv88e6xxx_6185_family(chip) || mv88e6xxx_6095_family(chip)) {
+		err = mv88e6xxx_port_write(chip, port, PORT_RATE_CONTROL,
+					   0x0000);
+		if (err)
+			return err;
 	}
 
 	/* Port Control 1: disable trunking, disable sending
 	 * learning messages to this port.
 	 */
-	ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_CONTROL_1,
-				   0x0000);
-	if (ret)
-		return ret;
+	err = mv88e6xxx_port_write(chip, port, PORT_CONTROL_1, 0x0000);
+	if (err)
+		return err;
 
 	/* Port based VLAN map: give each port the same default address
 	 * database, and allow bidirectional communication between the
 	 * CPU and DSA port(s), and the other ports.
 	 */
-	ret = _mv88e6xxx_port_fid_set(chip, port, 0);
-	if (ret)
-		return ret;
+	err = _mv88e6xxx_port_fid_set(chip, port, 0);
+	if (err)
+		return err;
 
-	ret = _mv88e6xxx_port_based_vlan_map(chip, port);
-	if (ret)
-		return ret;
+	err = _mv88e6xxx_port_based_vlan_map(chip, port);
+	if (err)
+		return err;
 
 	/* Default VLAN ID and priority: don't set a default VLAN
 	 * ID, and set the default packet priority to zero.
 	 */
-	ret = _mv88e6xxx_reg_write(chip, REG_PORT(port), PORT_DEFAULT_VLAN,
-				   0x0000);
-	if (ret)
-		return ret;
-
-	return 0;
+	return mv88e6xxx_port_write(chip, port, PORT_DEFAULT_VLAN, 0x0000);
 }
 
-static int mv88e6xxx_g1_set_switch_mac(struct mv88e6xxx_chip *chip, u8 *addr)
+int mv88e6xxx_g1_set_switch_mac(struct mv88e6xxx_chip *chip, u8 *addr)
 {
 	int err;
 
-	err = mv88e6xxx_write(chip, REG_GLOBAL, GLOBAL_MAC_01,
-			      (addr[0] << 8) | addr[1]);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_MAC_01, (addr[0] << 8) | addr[1]);
 	if (err)
 		return err;
 
-	err = mv88e6xxx_write(chip, REG_GLOBAL, GLOBAL_MAC_23,
-			      (addr[2] << 8) | addr[3]);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_MAC_23, (addr[2] << 8) | addr[3]);
 	if (err)
 		return err;
 
-	return mv88e6xxx_write(chip, REG_GLOBAL, GLOBAL_MAC_45,
-			       (addr[4] << 8) | addr[5]);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_MAC_45, (addr[4] << 8) | addr[5]);
+	if (err)
+		return err;
+
+	return 0;
 }
 
 static int mv88e6xxx_g1_set_age_time(struct mv88e6xxx_chip *chip,
@@ -2732,7 +2736,7 @@ static int mv88e6xxx_g1_set_age_time(struct mv88e6xxx_chip *chip,
 	/* Round to nearest multiple of coeff */
 	age_time = (msecs + coeff / 2) / coeff;
 
-	err = mv88e6xxx_read(chip, REG_GLOBAL, GLOBAL_ATU_CONTROL, &val);
+	err = mv88e6xxx_g1_read(chip, GLOBAL_ATU_CONTROL, &val);
 	if (err)
 		return err;
 
@@ -2740,13 +2744,13 @@ static int mv88e6xxx_g1_set_age_time(struct mv88e6xxx_chip *chip,
 	val &= ~0xff0;
 	val |= age_time << 4;
 
-	return mv88e6xxx_write(chip, REG_GLOBAL, GLOBAL_ATU_CONTROL, val);
+	return mv88e6xxx_g1_write(chip, GLOBAL_ATU_CONTROL, val);
 }
 
 static int mv88e6xxx_set_ageing_time(struct dsa_switch *ds,
 				     unsigned int ageing_time)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
 
 	mutex_lock(&chip->reg_lock);
@@ -2771,7 +2775,7 @@ static int mv88e6xxx_g1_setup(struct mv88e6xxx_chip *chip)
 	    mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU_ACTIVE))
 		reg |= GLOBAL_CONTROL_PPU_ENABLE;
 
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_CONTROL, reg);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_CONTROL, reg);
 	if (err)
 		return err;
 
@@ -2781,15 +2785,14 @@ static int mv88e6xxx_g1_setup(struct mv88e6xxx_chip *chip)
 	reg = upstream_port << GLOBAL_MONITOR_CONTROL_INGRESS_SHIFT |
 		upstream_port << GLOBAL_MONITOR_CONTROL_EGRESS_SHIFT |
 		upstream_port << GLOBAL_MONITOR_CONTROL_ARP_SHIFT;
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_MONITOR_CONTROL,
-				   reg);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_MONITOR_CONTROL, reg);
 	if (err)
 		return err;
 
 	/* Disable remote management, and set the switch's DSA device number. */
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_CONTROL_2,
-				   GLOBAL_CONTROL_2_MULTIPLE_CASCADE |
-				   (ds->index & 0x1f));
+	err = mv88e6xxx_g1_write(chip, GLOBAL_CONTROL_2,
+				 GLOBAL_CONTROL_2_MULTIPLE_CASCADE |
+				 (ds->index & 0x1f));
 	if (err)
 		return err;
 
@@ -2802,8 +2805,8 @@ static int mv88e6xxx_g1_setup(struct mv88e6xxx_chip *chip)
 	 * enable address learn messages to be sent to all message
 	 * ports.
 	 */
-	err = mv88e6xxx_write(chip, REG_GLOBAL, GLOBAL_ATU_CONTROL,
-			      GLOBAL_ATU_CONTROL_LEARN2ALL);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_ATU_CONTROL,
+				 GLOBAL_ATU_CONTROL_LEARN2ALL);
 	if (err)
 		return err;
 
@@ -2817,39 +2820,39 @@ static int mv88e6xxx_g1_setup(struct mv88e6xxx_chip *chip)
 		return err;
 
 	/* Configure the IP ToS mapping registers. */
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_0, 0x0000);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_IP_PRI_0, 0x0000);
 	if (err)
 		return err;
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_1, 0x0000);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_IP_PRI_1, 0x0000);
 	if (err)
 		return err;
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_2, 0x5555);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_IP_PRI_2, 0x5555);
 	if (err)
 		return err;
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_3, 0x5555);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_IP_PRI_3, 0x5555);
 	if (err)
 		return err;
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_4, 0xaaaa);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_IP_PRI_4, 0xaaaa);
 	if (err)
 		return err;
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_5, 0xaaaa);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_IP_PRI_5, 0xaaaa);
 	if (err)
 		return err;
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_6, 0xffff);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_IP_PRI_6, 0xffff);
 	if (err)
 		return err;
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IP_PRI_7, 0xffff);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_IP_PRI_7, 0xffff);
 	if (err)
 		return err;
 
 	/* Configure the IEEE 802.1p priority mapping register. */
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_IEEE_PRI, 0xfa41);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_IEEE_PRI, 0xfa41);
 	if (err)
 		return err;
 
 	/* Clear the statistics counters for all ports */
-	err = _mv88e6xxx_reg_write(chip, REG_GLOBAL, GLOBAL_STATS_OP,
-				   GLOBAL_STATS_OP_FLUSH_ALL);
+	err = mv88e6xxx_g1_write(chip, GLOBAL_STATS_OP,
+				 GLOBAL_STATS_OP_FLUSH_ALL);
 	if (err)
 		return err;
 
@@ -2861,277 +2864,9 @@ static int mv88e6xxx_g1_setup(struct mv88e6xxx_chip *chip)
 	return 0;
 }
 
-static int mv88e6xxx_g2_device_mapping_write(struct mv88e6xxx_chip *chip,
-					     int target, int port)
-{
-	u16 val = (target << 8) | (port & 0xf);
-
-	return mv88e6xxx_update(chip, REG_GLOBAL2, GLOBAL2_DEVICE_MAPPING, val);
-}
-
-static int mv88e6xxx_g2_set_device_mapping(struct mv88e6xxx_chip *chip)
-{
-	int target, port;
-	int err;
-
-	/* Initialize the routing port to the 32 possible target devices */
-	for (target = 0; target < 32; ++target) {
-		port = 0xf;
-
-		if (target < DSA_MAX_SWITCHES) {
-			port = chip->ds->rtable[target];
-			if (port == DSA_RTABLE_NONE)
-				port = 0xf;
-		}
-
-		err = mv88e6xxx_g2_device_mapping_write(chip, target, port);
-		if (err)
-			break;
-	}
-
-	return err;
-}
-
-static int mv88e6xxx_g2_trunk_mask_write(struct mv88e6xxx_chip *chip, int num,
-					 bool hask, u16 mask)
-{
-	const u16 port_mask = BIT(chip->info->num_ports) - 1;
-	u16 val = (num << 12) | (mask & port_mask);
-
-	if (hask)
-		val |= GLOBAL2_TRUNK_MASK_HASK;
-
-	return mv88e6xxx_update(chip, REG_GLOBAL2, GLOBAL2_TRUNK_MASK, val);
-}
-
-static int mv88e6xxx_g2_trunk_mapping_write(struct mv88e6xxx_chip *chip, int id,
-					    u16 map)
-{
-	const u16 port_mask = BIT(chip->info->num_ports) - 1;
-	u16 val = (id << 11) | (map & port_mask);
-
-	return mv88e6xxx_update(chip, REG_GLOBAL2, GLOBAL2_TRUNK_MAPPING, val);
-}
-
-static int mv88e6xxx_g2_clear_trunk(struct mv88e6xxx_chip *chip)
-{
-	const u16 port_mask = BIT(chip->info->num_ports) - 1;
-	int i, err;
-
-	/* Clear all eight possible Trunk Mask vectors */
-	for (i = 0; i < 8; ++i) {
-		err = mv88e6xxx_g2_trunk_mask_write(chip, i, false, port_mask);
-		if (err)
-			return err;
-	}
-
-	/* Clear all sixteen possible Trunk ID routing vectors */
-	for (i = 0; i < 16; ++i) {
-		err = mv88e6xxx_g2_trunk_mapping_write(chip, i, 0);
-		if (err)
-			return err;
-	}
-
-	return 0;
-}
-
-static int mv88e6xxx_g2_clear_irl(struct mv88e6xxx_chip *chip)
-{
-	int port, err;
-
-	/* Init all Ingress Rate Limit resources of all ports */
-	for (port = 0; port < chip->info->num_ports; ++port) {
-		/* XXX newer chips (like 88E6390) have different 2-bit ops */
-		err = mv88e6xxx_write(chip, REG_GLOBAL2, GLOBAL2_IRL_CMD,
-				      GLOBAL2_IRL_CMD_OP_INIT_ALL |
-				      (port << 8));
-		if (err)
-			break;
-
-		/* Wait for the operation to complete */
-		err = _mv88e6xxx_wait(chip, REG_GLOBAL2, GLOBAL2_IRL_CMD,
-				      GLOBAL2_IRL_CMD_BUSY);
-		if (err)
-			break;
-	}
-
-	return err;
-}
-
-/* Indirect write to the Switch MAC/WoL/WoF register */
-static int mv88e6xxx_g2_switch_mac_write(struct mv88e6xxx_chip *chip,
-					 unsigned int pointer, u8 data)
-{
-	u16 val = (pointer << 8) | data;
-
-	return mv88e6xxx_update(chip, REG_GLOBAL2, GLOBAL2_SWITCH_MAC, val);
-}
-
-static int mv88e6xxx_g2_set_switch_mac(struct mv88e6xxx_chip *chip, u8 *addr)
-{
-	int i, err;
-
-	for (i = 0; i < 6; i++) {
-		err = mv88e6xxx_g2_switch_mac_write(chip, i, addr[i]);
-		if (err)
-			break;
-	}
-
-	return err;
-}
-
-static int mv88e6xxx_g2_pot_write(struct mv88e6xxx_chip *chip, int pointer,
-				  u8 data)
-{
-	u16 val = (pointer << 8) | (data & 0x7);
-
-	return mv88e6xxx_update(chip, REG_GLOBAL2, GLOBAL2_PRIO_OVERRIDE, val);
-}
-
-static int mv88e6xxx_g2_clear_pot(struct mv88e6xxx_chip *chip)
-{
-	int i, err;
-
-	/* Clear all sixteen possible Priority Override entries */
-	for (i = 0; i < 16; i++) {
-		err = mv88e6xxx_g2_pot_write(chip, i, 0);
-		if (err)
-			break;
-	}
-
-	return err;
-}
-
-static int mv88e6xxx_g2_eeprom_wait(struct mv88e6xxx_chip *chip)
-{
-	return _mv88e6xxx_wait(chip, REG_GLOBAL2, GLOBAL2_EEPROM_CMD,
-			       GLOBAL2_EEPROM_CMD_BUSY |
-			       GLOBAL2_EEPROM_CMD_RUNNING);
-}
-
-static int mv88e6xxx_g2_eeprom_cmd(struct mv88e6xxx_chip *chip, u16 cmd)
-{
-	int err;
-
-	err = mv88e6xxx_write(chip, REG_GLOBAL2, GLOBAL2_EEPROM_CMD, cmd);
-	if (err)
-		return err;
-
-	return mv88e6xxx_g2_eeprom_wait(chip);
-}
-
-static int mv88e6xxx_g2_eeprom_read16(struct mv88e6xxx_chip *chip,
-				      u8 addr, u16 *data)
-{
-	u16 cmd = GLOBAL2_EEPROM_CMD_OP_READ | addr;
-	int err;
-
-	err = mv88e6xxx_g2_eeprom_wait(chip);
-	if (err)
-		return err;
-
-	err = mv88e6xxx_g2_eeprom_cmd(chip, cmd);
-	if (err)
-		return err;
-
-	return mv88e6xxx_read(chip, REG_GLOBAL2, GLOBAL2_EEPROM_DATA, data);
-}
-
-static int mv88e6xxx_g2_eeprom_write16(struct mv88e6xxx_chip *chip,
-				       u8 addr, u16 data)
-{
-	u16 cmd = GLOBAL2_EEPROM_CMD_OP_WRITE | addr;
-	int err;
-
-	err = mv88e6xxx_g2_eeprom_wait(chip);
-	if (err)
-		return err;
-
-	err = mv88e6xxx_write(chip, REG_GLOBAL2, GLOBAL2_EEPROM_DATA, data);
-	if (err)
-		return err;
-
-	return mv88e6xxx_g2_eeprom_cmd(chip, cmd);
-}
-
-static int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip)
-{
-	u16 reg;
-	int err;
-
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_2X)) {
-		/* Consider the frames with reserved multicast destination
-		 * addresses matching 01:80:c2:00:00:2x as MGMT.
-		 */
-		err = mv88e6xxx_write(chip, REG_GLOBAL2, GLOBAL2_MGMT_EN_2X,
-				      0xffff);
-		if (err)
-			return err;
-	}
-
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_0X)) {
-		/* Consider the frames with reserved multicast destination
-		 * addresses matching 01:80:c2:00:00:0x as MGMT.
-		 */
-		err = mv88e6xxx_write(chip, REG_GLOBAL2, GLOBAL2_MGMT_EN_0X,
-				      0xffff);
-		if (err)
-			return err;
-	}
-
-	/* Ignore removed tag data on doubly tagged packets, disable
-	 * flow control messages, force flow control priority to the
-	 * highest, and send all special multicast frames to the CPU
-	 * port at the highest priority.
-	 */
-	reg = GLOBAL2_SWITCH_MGMT_FORCE_FLOW_CTRL_PRI | (0x7 << 4);
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_0X) ||
-	    mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_2X))
-		reg |= GLOBAL2_SWITCH_MGMT_RSVD2CPU | 0x7;
-	err = mv88e6xxx_write(chip, REG_GLOBAL2, GLOBAL2_SWITCH_MGMT, reg);
-	if (err)
-		return err;
-
-	/* Program the DSA routing table. */
-	err = mv88e6xxx_g2_set_device_mapping(chip);
-	if (err)
-		return err;
-
-	/* Clear all trunk masks and mapping. */
-	err = mv88e6xxx_g2_clear_trunk(chip);
-	if (err)
-		return err;
-
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_IRL)) {
-		/* Disable ingress rate limiting by resetting all per port
-		 * ingress rate limit resources to their initial state.
-		 */
-		err = mv88e6xxx_g2_clear_irl(chip);
-			if (err)
-				return err;
-	}
-
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_PVT)) {
-		/* Initialize Cross-chip Port VLAN Table to reset defaults */
-		err = mv88e6xxx_write(chip, REG_GLOBAL2, GLOBAL2_PVT_ADDR,
-				      GLOBAL2_PVT_ADDR_OP_INIT_ONES);
-		if (err)
-			return err;
-	}
-
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_POT)) {
-		/* Clear the priority override table. */
-		err = mv88e6xxx_g2_clear_pot(chip);
-		if (err)
-			return err;
-	}
-
-	return 0;
-}
-
 static int mv88e6xxx_setup(struct dsa_switch *ds)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
 	int i;
 
@@ -3145,7 +2880,7 @@ static int mv88e6xxx_setup(struct dsa_switch *ds)
 		goto unlock;
 
 	/* Setup Switch Port Registers */
-	for (i = 0; i < chip->info->num_ports; i++) {
+	for (i = 0; i < mv88e6xxx_num_ports(chip); i++) {
 		err = mv88e6xxx_setup_port(chip, i);
 		if (err)
 			goto unlock;
@@ -3171,98 +2906,48 @@ unlock:
 
 static int mv88e6xxx_set_addr(struct dsa_switch *ds, u8 *addr)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
 
-	mutex_lock(&chip->reg_lock);
-
-	/* Has an indirect Switch MAC/WoL/WoF register in Global 2? */
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_SWITCH_MAC))
-		err = mv88e6xxx_g2_set_switch_mac(chip, addr);
-	else
-		err = mv88e6xxx_g1_set_switch_mac(chip, addr);
-
-	mutex_unlock(&chip->reg_lock);
-
-	return err;
-}
-
-static int mv88e6xxx_mdio_page_read(struct dsa_switch *ds, int port, int page,
-				    int reg)
-{
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	int ret;
-
-	mutex_lock(&chip->reg_lock);
-	ret = _mv88e6xxx_mdio_page_read(chip, port, page, reg);
-	mutex_unlock(&chip->reg_lock);
-
-	return ret;
-}
-
-static int mv88e6xxx_mdio_page_write(struct dsa_switch *ds, int port, int page,
-				     int reg, int val)
-{
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	int ret;
+	if (!chip->info->ops->set_switch_mac)
+		return -EOPNOTSUPP;
 
 	mutex_lock(&chip->reg_lock);
-	ret = _mv88e6xxx_mdio_page_write(chip, port, page, reg, val);
+	err = chip->info->ops->set_switch_mac(chip, addr);
 	mutex_unlock(&chip->reg_lock);
 
-	return ret;
-}
-
-static int mv88e6xxx_port_to_mdio_addr(struct mv88e6xxx_chip *chip, int port)
-{
-	if (port >= 0 && port < chip->info->num_ports)
-		return port;
-	return -EINVAL;
+	return err;
 }
 
-static int mv88e6xxx_mdio_read(struct mii_bus *bus, int port, int regnum)
+static int mv88e6xxx_mdio_read(struct mii_bus *bus, int phy, int reg)
 {
 	struct mv88e6xxx_chip *chip = bus->priv;
-	int addr = mv88e6xxx_port_to_mdio_addr(chip, port);
-	int ret;
+	u16 val;
+	int err;
 
-	if (addr < 0)
+	if (phy >= mv88e6xxx_num_ports(chip))
 		return 0xffff;
 
 	mutex_lock(&chip->reg_lock);
-
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU))
-		ret = mv88e6xxx_mdio_read_ppu(chip, addr, regnum);
-	else if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_SMI_PHY))
-		ret = mv88e6xxx_mdio_read_indirect(chip, addr, regnum);
-	else
-		ret = mv88e6xxx_mdio_read_direct(chip, addr, regnum);
-
+	err = mv88e6xxx_phy_read(chip, phy, reg, &val);
 	mutex_unlock(&chip->reg_lock);
-	return ret;
+
+	return err ? err : val;
 }
 
-static int mv88e6xxx_mdio_write(struct mii_bus *bus, int port, int regnum,
-				u16 val)
+static int mv88e6xxx_mdio_write(struct mii_bus *bus, int phy, int reg, u16 val)
 {
 	struct mv88e6xxx_chip *chip = bus->priv;
-	int addr = mv88e6xxx_port_to_mdio_addr(chip, port);
-	int ret;
+	int err;
 
-	if (addr < 0)
+	if (phy >= mv88e6xxx_num_ports(chip))
 		return 0xffff;
 
 	mutex_lock(&chip->reg_lock);
-
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU))
-		ret = mv88e6xxx_mdio_write_ppu(chip, addr, regnum, val);
-	else if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_SMI_PHY))
-		ret = mv88e6xxx_mdio_write_indirect(chip, addr, regnum, val);
-	else
-		ret = mv88e6xxx_mdio_write_direct(chip, addr, regnum, val);
-
+	err = mv88e6xxx_phy_write(chip, phy, reg, val);
 	mutex_unlock(&chip->reg_lock);
-	return ret;
+
+	return err;
 }
 
 static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
@@ -3272,9 +2957,6 @@ static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
 	struct mii_bus *bus;
 	int err;
 
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU))
-		mv88e6xxx_ppu_state_init(chip);
-
 	if (np)
 		chip->mdio_np = of_get_child_by_name(np, "mdio");
 
@@ -3329,69 +3011,70 @@ static void mv88e6xxx_mdio_unregister(struct mv88e6xxx_chip *chip)
 
 static int mv88e61xx_get_temp(struct dsa_switch *ds, int *temp)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
+	u16 val;
 	int ret;
-	int val;
 
 	*temp = 0;
 
 	mutex_lock(&chip->reg_lock);
 
-	ret = mv88e6xxx_mdio_write_direct(chip, 0x0, 0x16, 0x6);
+	ret = mv88e6xxx_phy_write(chip, 0x0, 0x16, 0x6);
 	if (ret < 0)
 		goto error;
 
 	/* Enable temperature sensor */
-	ret = mv88e6xxx_mdio_read_direct(chip, 0x0, 0x1a);
+	ret = mv88e6xxx_phy_read(chip, 0x0, 0x1a, &val);
 	if (ret < 0)
 		goto error;
 
-	ret = mv88e6xxx_mdio_write_direct(chip, 0x0, 0x1a, ret | (1 << 5));
+	ret = mv88e6xxx_phy_write(chip, 0x0, 0x1a, val | (1 << 5));
 	if (ret < 0)
 		goto error;
 
 	/* Wait for temperature to stabilize */
 	usleep_range(10000, 12000);
 
-	val = mv88e6xxx_mdio_read_direct(chip, 0x0, 0x1a);
-	if (val < 0) {
-		ret = val;
+	ret = mv88e6xxx_phy_read(chip, 0x0, 0x1a, &val);
+	if (ret < 0)
 		goto error;
-	}
 
 	/* Disable temperature sensor */
-	ret = mv88e6xxx_mdio_write_direct(chip, 0x0, 0x1a, ret & ~(1 << 5));
+	ret = mv88e6xxx_phy_write(chip, 0x0, 0x1a, val & ~(1 << 5));
 	if (ret < 0)
 		goto error;
 
 	*temp = ((val & 0x1f) - 5) * 5;
 
 error:
-	mv88e6xxx_mdio_write_direct(chip, 0x0, 0x16, 0x0);
+	mv88e6xxx_phy_write(chip, 0x0, 0x16, 0x0);
 	mutex_unlock(&chip->reg_lock);
 	return ret;
 }
 
 static int mv88e63xx_get_temp(struct dsa_switch *ds, int *temp)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int phy = mv88e6xxx_6320_family(chip) ? 3 : 0;
+	u16 val;
 	int ret;
 
 	*temp = 0;
 
-	ret = mv88e6xxx_mdio_page_read(ds, phy, 6, 27);
+	mutex_lock(&chip->reg_lock);
+	ret = mv88e6xxx_phy_page_read(chip, phy, 6, 27, &val);
+	mutex_unlock(&chip->reg_lock);
 	if (ret < 0)
 		return ret;
 
-	*temp = (ret & 0xff) - 25;
+	*temp = (val & 0xff) - 25;
 
 	return 0;
 }
 
 static int mv88e6xxx_get_temp(struct dsa_switch *ds, int *temp)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 
 	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_TEMP))
 		return -EOPNOTSUPP;
@@ -3404,8 +3087,9 @@ static int mv88e6xxx_get_temp(struct dsa_switch *ds, int *temp)
 
 static int mv88e6xxx_get_temp_limit(struct dsa_switch *ds, int *temp)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int phy = mv88e6xxx_6320_family(chip) ? 3 : 0;
+	u16 val;
 	int ret;
 
 	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_TEMP_LIMIT))
@@ -3413,36 +3097,45 @@ static int mv88e6xxx_get_temp_limit(struct dsa_switch *ds, int *temp)
 
 	*temp = 0;
 
-	ret = mv88e6xxx_mdio_page_read(ds, phy, 6, 26);
+	mutex_lock(&chip->reg_lock);
+	ret = mv88e6xxx_phy_page_read(chip, phy, 6, 26, &val);
+	mutex_unlock(&chip->reg_lock);
 	if (ret < 0)
 		return ret;
 
-	*temp = (((ret >> 8) & 0x1f) * 5) - 25;
+	*temp = (((val >> 8) & 0x1f) * 5) - 25;
 
 	return 0;
 }
 
 static int mv88e6xxx_set_temp_limit(struct dsa_switch *ds, int temp)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int phy = mv88e6xxx_6320_family(chip) ? 3 : 0;
-	int ret;
+	u16 val;
+	int err;
 
 	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_TEMP_LIMIT))
 		return -EOPNOTSUPP;
 
-	ret = mv88e6xxx_mdio_page_read(ds, phy, 6, 26);
-	if (ret < 0)
-		return ret;
+	mutex_lock(&chip->reg_lock);
+	err = mv88e6xxx_phy_page_read(chip, phy, 6, 26, &val);
+	if (err)
+		goto unlock;
 	temp = clamp_val(DIV_ROUND_CLOSEST(temp, 5) + 5, 0, 0x1f);
-	return mv88e6xxx_mdio_page_write(ds, phy, 6, 26,
-					 (ret & 0xe0ff) | (temp << 8));
+	err = mv88e6xxx_phy_page_write(chip, phy, 6, 26,
+				       (val & 0xe0ff) | (temp << 8));
+unlock:
+	mutex_unlock(&chip->reg_lock);
+
+	return err;
 }
 
 static int mv88e6xxx_get_temp_alarm(struct dsa_switch *ds, bool *alarm)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int phy = mv88e6xxx_6320_family(chip) ? 3 : 0;
+	u16 val;
 	int ret;
 
 	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_TEMP_LIMIT))
@@ -3450,11 +3143,13 @@ static int mv88e6xxx_get_temp_alarm(struct dsa_switch *ds, bool *alarm)
 
 	*alarm = false;
 
-	ret = mv88e6xxx_mdio_page_read(ds, phy, 6, 26);
+	mutex_lock(&chip->reg_lock);
+	ret = mv88e6xxx_phy_page_read(chip, phy, 6, 26, &val);
+	mutex_unlock(&chip->reg_lock);
 	if (ret < 0)
 		return ret;
 
-	*alarm = !!(ret & 0x40);
+	*alarm = !!(val & 0x40);
 
 	return 0;
 }
@@ -3462,74 +3157,22 @@ static int mv88e6xxx_get_temp_alarm(struct dsa_switch *ds, bool *alarm)
 
 static int mv88e6xxx_get_eeprom_len(struct dsa_switch *ds)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 
 	return chip->eeprom_len;
 }
 
-static int mv88e6xxx_get_eeprom16(struct mv88e6xxx_chip *chip,
-				  struct ethtool_eeprom *eeprom, u8 *data)
-{
-	unsigned int offset = eeprom->offset;
-	unsigned int len = eeprom->len;
-	u16 val;
-	int err;
-
-	eeprom->len = 0;
-
-	if (offset & 1) {
-		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
-		if (err)
-			return err;
-
-		*data++ = (val >> 8) & 0xff;
-
-		offset++;
-		len--;
-		eeprom->len++;
-	}
-
-	while (len >= 2) {
-		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
-		if (err)
-			return err;
-
-		*data++ = val & 0xff;
-		*data++ = (val >> 8) & 0xff;
-
-		offset += 2;
-		len -= 2;
-		eeprom->len += 2;
-	}
-
-	if (len) {
-		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
-		if (err)
-			return err;
-
-		*data++ = val & 0xff;
-
-		offset++;
-		len--;
-		eeprom->len++;
-	}
-
-	return 0;
-}
-
 static int mv88e6xxx_get_eeprom(struct dsa_switch *ds,
 				struct ethtool_eeprom *eeprom, u8 *data)
 {
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
 
-	mutex_lock(&chip->reg_lock);
-
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_EEPROM16))
-		err = mv88e6xxx_get_eeprom16(chip, eeprom, data);
-	else
-		err = -EOPNOTSUPP;
+	if (!chip->info->ops->get_eeprom)
+		return -EOPNOTSUPP;
 
+	mutex_lock(&chip->reg_lock);
+	err = chip->info->ops->get_eeprom(chip, eeprom, data);
 	mutex_unlock(&chip->reg_lock);
 
 	if (err)
@@ -3540,92 +3183,138 @@ static int mv88e6xxx_get_eeprom(struct dsa_switch *ds,
 	return 0;
 }
 
-static int mv88e6xxx_set_eeprom16(struct mv88e6xxx_chip *chip,
-				  struct ethtool_eeprom *eeprom, u8 *data)
+static int mv88e6xxx_set_eeprom(struct dsa_switch *ds,
+				struct ethtool_eeprom *eeprom, u8 *data)
 {
-	unsigned int offset = eeprom->offset;
-	unsigned int len = eeprom->len;
-	u16 val;
+	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
 
-	/* Ensure the RO WriteEn bit is set */
-	err = mv88e6xxx_read(chip, REG_GLOBAL2, GLOBAL2_EEPROM_CMD, &val);
-	if (err)
-		return err;
+	if (!chip->info->ops->set_eeprom)
+		return -EOPNOTSUPP;
 
-	if (!(val & GLOBAL2_EEPROM_CMD_WRITE_EN))
-		return -EROFS;
+	if (eeprom->magic != 0xc3ec4951)
+		return -EINVAL;
 
-	eeprom->len = 0;
+	mutex_lock(&chip->reg_lock);
+	err = chip->info->ops->set_eeprom(chip, eeprom, data);
+	mutex_unlock(&chip->reg_lock);
 
-	if (offset & 1) {
-		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
-		if (err)
-			return err;
+	return err;
+}
 
-		val = (*data++ << 8) | (val & 0xff);
+static const struct mv88e6xxx_ops mv88e6085_ops = {
+	.set_switch_mac = mv88e6xxx_g1_set_switch_mac,
+	.phy_read = mv88e6xxx_phy_ppu_read,
+	.phy_write = mv88e6xxx_phy_ppu_write,
+};
 
-		err = mv88e6xxx_g2_eeprom_write16(chip, offset >> 1, val);
-		if (err)
-			return err;
+static const struct mv88e6xxx_ops mv88e6095_ops = {
+	.set_switch_mac = mv88e6xxx_g1_set_switch_mac,
+	.phy_read = mv88e6xxx_phy_ppu_read,
+	.phy_write = mv88e6xxx_phy_ppu_write,
+};
 
-		offset++;
-		len--;
-		eeprom->len++;
-	}
+static const struct mv88e6xxx_ops mv88e6123_ops = {
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_read,
+	.phy_write = mv88e6xxx_write,
+};
 
-	while (len >= 2) {
-		val = *data++;
-		val |= *data++ << 8;
+static const struct mv88e6xxx_ops mv88e6131_ops = {
+	.set_switch_mac = mv88e6xxx_g1_set_switch_mac,
+	.phy_read = mv88e6xxx_phy_ppu_read,
+	.phy_write = mv88e6xxx_phy_ppu_write,
+};
 
-		err = mv88e6xxx_g2_eeprom_write16(chip, offset >> 1, val);
-		if (err)
-			return err;
+static const struct mv88e6xxx_ops mv88e6161_ops = {
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_read,
+	.phy_write = mv88e6xxx_write,
+};
 
-		offset += 2;
-		len -= 2;
-		eeprom->len += 2;
-	}
+static const struct mv88e6xxx_ops mv88e6165_ops = {
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_read,
+	.phy_write = mv88e6xxx_write,
+};
 
-	if (len) {
-		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
-		if (err)
-			return err;
+static const struct mv88e6xxx_ops mv88e6171_ops = {
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
-		val = (val & 0xff00) | *data++;
+static const struct mv88e6xxx_ops mv88e6172_ops = {
+	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
+	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
-		err = mv88e6xxx_g2_eeprom_write16(chip, offset >> 1, val);
-		if (err)
-			return err;
+static const struct mv88e6xxx_ops mv88e6175_ops = {
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
-		offset++;
-		len--;
-		eeprom->len++;
-	}
+static const struct mv88e6xxx_ops mv88e6176_ops = {
+	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
+	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
-	return 0;
-}
+static const struct mv88e6xxx_ops mv88e6185_ops = {
+	.set_switch_mac = mv88e6xxx_g1_set_switch_mac,
+	.phy_read = mv88e6xxx_phy_ppu_read,
+	.phy_write = mv88e6xxx_phy_ppu_write,
+};
 
-static int mv88e6xxx_set_eeprom(struct dsa_switch *ds,
-				struct ethtool_eeprom *eeprom, u8 *data)
-{
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
-	int err;
+static const struct mv88e6xxx_ops mv88e6240_ops = {
+	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
+	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
-	if (eeprom->magic != 0xc3ec4951)
-		return -EINVAL;
+static const struct mv88e6xxx_ops mv88e6320_ops = {
+	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
+	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
-	mutex_lock(&chip->reg_lock);
+static const struct mv88e6xxx_ops mv88e6321_ops = {
+	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
+	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_EEPROM16))
-		err = mv88e6xxx_set_eeprom16(chip, eeprom, data);
-	else
-		err = -EOPNOTSUPP;
+static const struct mv88e6xxx_ops mv88e6350_ops = {
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
-	mutex_unlock(&chip->reg_lock);
+static const struct mv88e6xxx_ops mv88e6351_ops = {
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
-	return err;
-}
+static const struct mv88e6xxx_ops mv88e6352_ops = {
+	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
+	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read,
+	.phy_write = mv88e6xxx_g2_smi_phy_write,
+};
 
 static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 	[MV88E6085] = {
@@ -3635,8 +3324,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 10,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6097,
+		.ops = &mv88e6085_ops,
 	},
 
 	[MV88E6095] = {
@@ -3646,8 +3337,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 256,
 		.num_ports = 11,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6095,
+		.ops = &mv88e6095_ops,
 	},
 
 	[MV88E6123] = {
@@ -3657,8 +3350,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 3,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6165,
+		.ops = &mv88e6123_ops,
 	},
 
 	[MV88E6131] = {
@@ -3668,8 +3363,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 256,
 		.num_ports = 8,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6185,
+		.ops = &mv88e6131_ops,
 	},
 
 	[MV88E6161] = {
@@ -3679,8 +3376,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 6,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6165,
+		.ops = &mv88e6161_ops,
 	},
 
 	[MV88E6165] = {
@@ -3690,8 +3389,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 6,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6165,
+		.ops = &mv88e6165_ops,
 	},
 
 	[MV88E6171] = {
@@ -3701,8 +3402,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6351,
+		.ops = &mv88e6171_ops,
 	},
 
 	[MV88E6172] = {
@@ -3712,8 +3415,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6352,
+		.ops = &mv88e6172_ops,
 	},
 
 	[MV88E6175] = {
@@ -3723,8 +3428,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6351,
+		.ops = &mv88e6175_ops,
 	},
 
 	[MV88E6176] = {
@@ -3734,8 +3441,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6352,
+		.ops = &mv88e6176_ops,
 	},
 
 	[MV88E6185] = {
@@ -3745,8 +3454,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 256,
 		.num_ports = 10,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6185,
+		.ops = &mv88e6185_ops,
 	},
 
 	[MV88E6240] = {
@@ -3756,8 +3467,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6352,
+		.ops = &mv88e6240_ops,
 	},
 
 	[MV88E6320] = {
@@ -3767,8 +3480,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6320,
+		.ops = &mv88e6320_ops,
 	},
 
 	[MV88E6321] = {
@@ -3778,8 +3493,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6320,
+		.ops = &mv88e6321_ops,
 	},
 
 	[MV88E6350] = {
@@ -3789,8 +3506,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6351,
+		.ops = &mv88e6350_ops,
 	},
 
 	[MV88E6351] = {
@@ -3800,8 +3519,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6351,
+		.ops = &mv88e6351_ops,
 	},
 
 	[MV88E6352] = {
@@ -3811,8 +3532,10 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.num_databases = 4096,
 		.num_ports = 7,
 		.port_base_addr = 0x10,
+		.global1_addr = 0x1b,
 		.age_time_coeff = 15000,
 		.flags = MV88E6XXX_FLAGS_FAMILY_6352,
+		.ops = &mv88e6352_ops,
 	},
 };
 
@@ -3850,6 +3573,10 @@ static int mv88e6xxx_detect(struct mv88e6xxx_chip *chip)
 	/* Update the compatible info with the probed one */
 	chip->info = info;
 
+	err = mv88e6xxx_g2_require(chip);
+	if (err)
+		return err;
+
 	dev_info(chip->dev, "switch 0x%x detected: %s, revision %u\n",
 		 chip->info->prod_num, chip->info->name, rev);
 
@@ -3871,6 +3598,18 @@ static struct mv88e6xxx_chip *mv88e6xxx_alloc_chip(struct device *dev)
 	return chip;
 }
 
+static void mv88e6xxx_phy_init(struct mv88e6xxx_chip *chip)
+{
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU))
+		mv88e6xxx_ppu_state_init(chip);
+}
+
+static void mv88e6xxx_phy_destroy(struct mv88e6xxx_chip *chip)
+{
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_PPU))
+		mv88e6xxx_ppu_state_destroy(chip);
+}
+
 static int mv88e6xxx_smi_init(struct mv88e6xxx_chip *chip,
 			      struct mii_bus *bus, int sw_addr)
 {
@@ -3880,7 +3619,7 @@ static int mv88e6xxx_smi_init(struct mv88e6xxx_chip *chip,
 
 	if (sw_addr == 0)
 		chip->smi_ops = &mv88e6xxx_smi_single_chip_ops;
-	else if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_MULTI_CHIP))
+	else if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_MULTI_CHIP))
 		chip->smi_ops = &mv88e6xxx_smi_multi_chip_ops;
 	else
 		return -EINVAL;
@@ -3891,6 +3630,16 @@ static int mv88e6xxx_smi_init(struct mv88e6xxx_chip *chip,
 	return 0;
 }
 
+static enum dsa_tag_protocol mv88e6xxx_get_tag_protocol(struct dsa_switch *ds)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_EDSA))
+		return DSA_TAG_PROTO_EDSA;
+
+	return DSA_TAG_PROTO_DSA;
+}
+
 static const char *mv88e6xxx_drv_probe(struct device *dsa_dev,
 				       struct device *host_dev, int sw_addr,
 				       void **priv)
@@ -3918,6 +3667,8 @@ static const char *mv88e6xxx_drv_probe(struct device *dsa_dev,
 	if (err)
 		goto free;
 
+	mv88e6xxx_phy_init(chip);
+
 	err = mv88e6xxx_mdio_register(chip, NULL);
 	if (err)
 		goto free;
@@ -3931,9 +3682,61 @@ free:
 	return NULL;
 }
 
-static struct dsa_switch_driver mv88e6xxx_switch_driver = {
-	.tag_protocol		= DSA_TAG_PROTO_EDSA,
+static int mv88e6xxx_port_mdb_prepare(struct dsa_switch *ds, int port,
+				      const struct switchdev_obj_port_mdb *mdb,
+				      struct switchdev_trans *trans)
+{
+	/* We don't need any dynamic resource from the kernel (yet),
+	 * so skip the prepare phase.
+	 */
+
+	return 0;
+}
+
+static void mv88e6xxx_port_mdb_add(struct dsa_switch *ds, int port,
+				   const struct switchdev_obj_port_mdb *mdb,
+				   struct switchdev_trans *trans)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+
+	mutex_lock(&chip->reg_lock);
+	if (mv88e6xxx_port_db_load_purge(chip, port, mdb->addr, mdb->vid,
+					 GLOBAL_ATU_DATA_STATE_MC_STATIC))
+		netdev_err(ds->ports[port].netdev, "failed to load multicast MAC address\n");
+	mutex_unlock(&chip->reg_lock);
+}
+
+static int mv88e6xxx_port_mdb_del(struct dsa_switch *ds, int port,
+				  const struct switchdev_obj_port_mdb *mdb)
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+	int err;
+
+	mutex_lock(&chip->reg_lock);
+	err = mv88e6xxx_port_db_load_purge(chip, port, mdb->addr, mdb->vid,
+					   GLOBAL_ATU_DATA_STATE_UNUSED);
+	mutex_unlock(&chip->reg_lock);
+
+	return err;
+}
+
+static int mv88e6xxx_port_mdb_dump(struct dsa_switch *ds, int port,
+				   struct switchdev_obj_port_mdb *mdb,
+				   int (*cb)(struct switchdev_obj *obj))
+{
+	struct mv88e6xxx_chip *chip = ds->priv;
+	int err;
+
+	mutex_lock(&chip->reg_lock);
+	err = mv88e6xxx_port_db_dump(chip, port, &mdb->obj, cb);
+	mutex_unlock(&chip->reg_lock);
+
+	return err;
+}
+
+static struct dsa_switch_ops mv88e6xxx_switch_ops = {
 	.probe			= mv88e6xxx_drv_probe,
+	.get_tag_protocol	= mv88e6xxx_get_tag_protocol,
 	.setup			= mv88e6xxx_setup,
 	.set_addr		= mv88e6xxx_set_addr,
 	.adjust_link		= mv88e6xxx_adjust_link,
@@ -3957,6 +3760,7 @@ static struct dsa_switch_driver mv88e6xxx_switch_driver = {
 	.port_bridge_join	= mv88e6xxx_port_bridge_join,
 	.port_bridge_leave	= mv88e6xxx_port_bridge_leave,
 	.port_stp_state_set	= mv88e6xxx_port_stp_state_set,
+	.port_fast_age		= mv88e6xxx_port_fast_age,
 	.port_vlan_filtering	= mv88e6xxx_port_vlan_filtering,
 	.port_vlan_prepare	= mv88e6xxx_port_vlan_prepare,
 	.port_vlan_add		= mv88e6xxx_port_vlan_add,
@@ -3966,6 +3770,10 @@ static struct dsa_switch_driver mv88e6xxx_switch_driver = {
 	.port_fdb_add           = mv88e6xxx_port_fdb_add,
 	.port_fdb_del           = mv88e6xxx_port_fdb_del,
 	.port_fdb_dump          = mv88e6xxx_port_fdb_dump,
+	.port_mdb_prepare       = mv88e6xxx_port_mdb_prepare,
+	.port_mdb_add           = mv88e6xxx_port_mdb_add,
+	.port_mdb_del           = mv88e6xxx_port_mdb_del,
+	.port_mdb_dump          = mv88e6xxx_port_mdb_dump,
 };
 
 static int mv88e6xxx_register_switch(struct mv88e6xxx_chip *chip,
@@ -3980,7 +3788,7 @@ static int mv88e6xxx_register_switch(struct mv88e6xxx_chip *chip,
 
 	ds->dev = dev;
 	ds->priv = chip;
-	ds->drv = &mv88e6xxx_switch_driver;
+	ds->ops = &mv88e6xxx_switch_ops;
 
 	dev_set_drvdata(dev, ds);
 
@@ -4019,11 +3827,13 @@ static int mv88e6xxx_probe(struct mdio_device *mdiodev)
 	if (err)
 		return err;
 
+	mv88e6xxx_phy_init(chip);
+
 	chip->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_ASIS);
 	if (IS_ERR(chip->reset))
 		return PTR_ERR(chip->reset);
 
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_EEPROM16) &&
+	if (chip->info->ops->get_eeprom &&
 	    !of_property_read_u32(np, "eeprom-length", &eeprom_len))
 		chip->eeprom_len = eeprom_len;
 
@@ -4043,8 +3853,9 @@ static int mv88e6xxx_probe(struct mdio_device *mdiodev)
 static void mv88e6xxx_remove(struct mdio_device *mdiodev)
 {
 	struct dsa_switch *ds = dev_get_drvdata(&mdiodev->dev);
-	struct mv88e6xxx_chip *chip = ds_to_priv(ds);
+	struct mv88e6xxx_chip *chip = ds->priv;
 
+	mv88e6xxx_phy_destroy(chip);
 	mv88e6xxx_unregister_switch(chip);
 	mv88e6xxx_mdio_unregister(chip);
 }
@@ -4070,7 +3881,7 @@ static struct mdio_driver mv88e6xxx_driver = {
 
 static int __init mv88e6xxx_init(void)
 {
-	register_switch_driver(&mv88e6xxx_switch_driver);
+	register_switch_driver(&mv88e6xxx_switch_ops);
 	return mdio_driver_register(&mv88e6xxx_driver);
 }
 module_init(mv88e6xxx_init);
@@ -4078,7 +3889,7 @@ module_init(mv88e6xxx_init);
 static void __exit mv88e6xxx_cleanup(void)
 {
 	mdio_driver_unregister(&mv88e6xxx_driver);
-	unregister_switch_driver(&mv88e6xxx_switch_driver);
+	unregister_switch_driver(&mv88e6xxx_switch_ops);
 }
 module_exit(mv88e6xxx_cleanup);
 
diff --git a/drivers/net/dsa/mv88e6xxx/global1.c b/drivers/net/dsa/mv88e6xxx/global1.c
new file mode 100644
index 000000000000..d358720b6c2d
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/global1.c
@@ -0,0 +1,34 @@
+/*
+ * Marvell 88E6xxx Switch Global (1) Registers support
+ *
+ * Copyright (c) 2008 Marvell Semiconductor
+ *
+ * Copyright (c) 2016 Vivien Didelot <vivien.didelot@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include "mv88e6xxx.h"
+#include "global1.h"
+
+int mv88e6xxx_g1_read(struct mv88e6xxx_chip *chip, int reg, u16 *val)
+{
+	int addr = chip->info->global1_addr;
+
+	return mv88e6xxx_read(chip, addr, reg, val);
+}
+
+int mv88e6xxx_g1_write(struct mv88e6xxx_chip *chip, int reg, u16 val)
+{
+	int addr = chip->info->global1_addr;
+
+	return mv88e6xxx_write(chip, addr, reg, val);
+}
+
+int mv88e6xxx_g1_wait(struct mv88e6xxx_chip *chip, int reg, u16 mask)
+{
+	return mv88e6xxx_wait(chip, chip->info->global1_addr, reg, mask);
+}
diff --git a/drivers/net/dsa/mv88e6xxx/global1.h b/drivers/net/dsa/mv88e6xxx/global1.h
new file mode 100644
index 000000000000..62291e6fe3a3
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/global1.h
@@ -0,0 +1,23 @@
+/*
+ * Marvell 88E6xxx Switch Global (1) Registers support
+ *
+ * Copyright (c) 2008 Marvell Semiconductor
+ *
+ * Copyright (c) 2016 Vivien Didelot <vivien.didelot@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef _MV88E6XXX_GLOBAL1_H
+#define _MV88E6XXX_GLOBAL1_H
+
+#include "mv88e6xxx.h"
+
+int mv88e6xxx_g1_read(struct mv88e6xxx_chip *chip, int reg, u16 *val);
+int mv88e6xxx_g1_write(struct mv88e6xxx_chip *chip, int reg, u16 val);
+int mv88e6xxx_g1_wait(struct mv88e6xxx_chip *chip, int reg, u16 mask);
+
+#endif /* _MV88E6XXX_GLOBAL1_H */
diff --git a/drivers/net/dsa/mv88e6xxx/global2.c b/drivers/net/dsa/mv88e6xxx/global2.c
new file mode 100644
index 000000000000..cf686e7506a9
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/global2.c
@@ -0,0 +1,491 @@
+/*
+ * Marvell 88E6xxx Switch Global 2 Registers support (device address 0x1C)
+ *
+ * Copyright (c) 2008 Marvell Semiconductor
+ *
+ * Copyright (c) 2016 Vivien Didelot <vivien.didelot@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include "mv88e6xxx.h"
+#include "global2.h"
+
+#define ADDR_GLOBAL2	0x1c
+
+static int mv88e6xxx_g2_read(struct mv88e6xxx_chip *chip, int reg, u16 *val)
+{
+	return mv88e6xxx_read(chip, ADDR_GLOBAL2, reg, val);
+}
+
+static int mv88e6xxx_g2_write(struct mv88e6xxx_chip *chip, int reg, u16 val)
+{
+	return mv88e6xxx_write(chip, ADDR_GLOBAL2, reg, val);
+}
+
+static int mv88e6xxx_g2_update(struct mv88e6xxx_chip *chip, int reg, u16 update)
+{
+	return mv88e6xxx_update(chip, ADDR_GLOBAL2, reg, update);
+}
+
+static int mv88e6xxx_g2_wait(struct mv88e6xxx_chip *chip, int reg, u16 mask)
+{
+	return mv88e6xxx_wait(chip, ADDR_GLOBAL2, reg, mask);
+}
+
+/* Offset 0x06: Device Mapping Table register */
+
+static int mv88e6xxx_g2_device_mapping_write(struct mv88e6xxx_chip *chip,
+					     int target, int port)
+{
+	u16 val = (target << 8) | (port & 0xf);
+
+	return mv88e6xxx_g2_update(chip, GLOBAL2_DEVICE_MAPPING, val);
+}
+
+static int mv88e6xxx_g2_set_device_mapping(struct mv88e6xxx_chip *chip)
+{
+	int target, port;
+	int err;
+
+	/* Initialize the routing port to the 32 possible target devices */
+	for (target = 0; target < 32; ++target) {
+		port = 0xf;
+
+		if (target < DSA_MAX_SWITCHES) {
+			port = chip->ds->rtable[target];
+			if (port == DSA_RTABLE_NONE)
+				port = 0xf;
+		}
+
+		err = mv88e6xxx_g2_device_mapping_write(chip, target, port);
+		if (err)
+			break;
+	}
+
+	return err;
+}
+
+/* Offset 0x07: Trunk Mask Table register */
+
+static int mv88e6xxx_g2_trunk_mask_write(struct mv88e6xxx_chip *chip, int num,
+					 bool hask, u16 mask)
+{
+	const u16 port_mask = BIT(mv88e6xxx_num_ports(chip)) - 1;
+	u16 val = (num << 12) | (mask & port_mask);
+
+	if (hask)
+		val |= GLOBAL2_TRUNK_MASK_HASK;
+
+	return mv88e6xxx_g2_update(chip, GLOBAL2_TRUNK_MASK, val);
+}
+
+/* Offset 0x08: Trunk Mapping Table register */
+
+static int mv88e6xxx_g2_trunk_mapping_write(struct mv88e6xxx_chip *chip, int id,
+					    u16 map)
+{
+	const u16 port_mask = BIT(mv88e6xxx_num_ports(chip)) - 1;
+	u16 val = (id << 11) | (map & port_mask);
+
+	return mv88e6xxx_g2_update(chip, GLOBAL2_TRUNK_MAPPING, val);
+}
+
+static int mv88e6xxx_g2_clear_trunk(struct mv88e6xxx_chip *chip)
+{
+	const u16 port_mask = BIT(mv88e6xxx_num_ports(chip)) - 1;
+	int i, err;
+
+	/* Clear all eight possible Trunk Mask vectors */
+	for (i = 0; i < 8; ++i) {
+		err = mv88e6xxx_g2_trunk_mask_write(chip, i, false, port_mask);
+		if (err)
+			return err;
+	}
+
+	/* Clear all sixteen possible Trunk ID routing vectors */
+	for (i = 0; i < 16; ++i) {
+		err = mv88e6xxx_g2_trunk_mapping_write(chip, i, 0);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+/* Offset 0x09: Ingress Rate Command register
+ * Offset 0x0A: Ingress Rate Data register
+ */
+
+static int mv88e6xxx_g2_clear_irl(struct mv88e6xxx_chip *chip)
+{
+	int port, err;
+
+	/* Init all Ingress Rate Limit resources of all ports */
+	for (port = 0; port < mv88e6xxx_num_ports(chip); ++port) {
+		/* XXX newer chips (like 88E6390) have different 2-bit ops */
+		err = mv88e6xxx_g2_write(chip, GLOBAL2_IRL_CMD,
+					 GLOBAL2_IRL_CMD_OP_INIT_ALL |
+					 (port << 8));
+		if (err)
+			break;
+
+		/* Wait for the operation to complete */
+		err = mv88e6xxx_g2_wait(chip, GLOBAL2_IRL_CMD,
+					GLOBAL2_IRL_CMD_BUSY);
+		if (err)
+			break;
+	}
+
+	return err;
+}
+
+/* Offset 0x0D: Switch MAC/WoL/WoF register */
+
+static int mv88e6xxx_g2_switch_mac_write(struct mv88e6xxx_chip *chip,
+					 unsigned int pointer, u8 data)
+{
+	u16 val = (pointer << 8) | data;
+
+	return mv88e6xxx_g2_update(chip, GLOBAL2_SWITCH_MAC, val);
+}
+
+int mv88e6xxx_g2_set_switch_mac(struct mv88e6xxx_chip *chip, u8 *addr)
+{
+	int i, err;
+
+	for (i = 0; i < 6; i++) {
+		err = mv88e6xxx_g2_switch_mac_write(chip, i, addr[i]);
+		if (err)
+			break;
+	}
+
+	return err;
+}
+
+/* Offset 0x0F: Priority Override Table */
+
+static int mv88e6xxx_g2_pot_write(struct mv88e6xxx_chip *chip, int pointer,
+				  u8 data)
+{
+	u16 val = (pointer << 8) | (data & 0x7);
+
+	return mv88e6xxx_g2_update(chip, GLOBAL2_PRIO_OVERRIDE, val);
+}
+
+static int mv88e6xxx_g2_clear_pot(struct mv88e6xxx_chip *chip)
+{
+	int i, err;
+
+	/* Clear all sixteen possible Priority Override entries */
+	for (i = 0; i < 16; i++) {
+		err = mv88e6xxx_g2_pot_write(chip, i, 0);
+		if (err)
+			break;
+	}
+
+	return err;
+}
+
+/* Offset 0x14: EEPROM Command
+ * Offset 0x15: EEPROM Data
+ */
+
+static int mv88e6xxx_g2_eeprom_wait(struct mv88e6xxx_chip *chip)
+{
+	return mv88e6xxx_g2_wait(chip, GLOBAL2_EEPROM_CMD,
+				 GLOBAL2_EEPROM_CMD_BUSY |
+				 GLOBAL2_EEPROM_CMD_RUNNING);
+}
+
+static int mv88e6xxx_g2_eeprom_cmd(struct mv88e6xxx_chip *chip, u16 cmd)
+{
+	int err;
+
+	err = mv88e6xxx_g2_write(chip, GLOBAL2_EEPROM_CMD, cmd);
+	if (err)
+		return err;
+
+	return mv88e6xxx_g2_eeprom_wait(chip);
+}
+
+static int mv88e6xxx_g2_eeprom_read16(struct mv88e6xxx_chip *chip,
+				      u8 addr, u16 *data)
+{
+	u16 cmd = GLOBAL2_EEPROM_CMD_OP_READ | addr;
+	int err;
+
+	err = mv88e6xxx_g2_eeprom_wait(chip);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g2_eeprom_cmd(chip, cmd);
+	if (err)
+		return err;
+
+	return mv88e6xxx_g2_read(chip, GLOBAL2_EEPROM_DATA, data);
+}
+
+static int mv88e6xxx_g2_eeprom_write16(struct mv88e6xxx_chip *chip,
+				       u8 addr, u16 data)
+{
+	u16 cmd = GLOBAL2_EEPROM_CMD_OP_WRITE | addr;
+	int err;
+
+	err = mv88e6xxx_g2_eeprom_wait(chip);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g2_write(chip, GLOBAL2_EEPROM_DATA, data);
+	if (err)
+		return err;
+
+	return mv88e6xxx_g2_eeprom_cmd(chip, cmd);
+}
+
+int mv88e6xxx_g2_get_eeprom16(struct mv88e6xxx_chip *chip,
+			      struct ethtool_eeprom *eeprom, u8 *data)
+{
+	unsigned int offset = eeprom->offset;
+	unsigned int len = eeprom->len;
+	u16 val;
+	int err;
+
+	eeprom->len = 0;
+
+	if (offset & 1) {
+		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
+		if (err)
+			return err;
+
+		*data++ = (val >> 8) & 0xff;
+
+		offset++;
+		len--;
+		eeprom->len++;
+	}
+
+	while (len >= 2) {
+		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
+		if (err)
+			return err;
+
+		*data++ = val & 0xff;
+		*data++ = (val >> 8) & 0xff;
+
+		offset += 2;
+		len -= 2;
+		eeprom->len += 2;
+	}
+
+	if (len) {
+		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
+		if (err)
+			return err;
+
+		*data++ = val & 0xff;
+
+		offset++;
+		len--;
+		eeprom->len++;
+	}
+
+	return 0;
+}
+
+int mv88e6xxx_g2_set_eeprom16(struct mv88e6xxx_chip *chip,
+			      struct ethtool_eeprom *eeprom, u8 *data)
+{
+	unsigned int offset = eeprom->offset;
+	unsigned int len = eeprom->len;
+	u16 val;
+	int err;
+
+	/* Ensure the RO WriteEn bit is set */
+	err = mv88e6xxx_g2_read(chip, GLOBAL2_EEPROM_CMD, &val);
+	if (err)
+		return err;
+
+	if (!(val & GLOBAL2_EEPROM_CMD_WRITE_EN))
+		return -EROFS;
+
+	eeprom->len = 0;
+
+	if (offset & 1) {
+		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
+		if (err)
+			return err;
+
+		val = (*data++ << 8) | (val & 0xff);
+
+		err = mv88e6xxx_g2_eeprom_write16(chip, offset >> 1, val);
+		if (err)
+			return err;
+
+		offset++;
+		len--;
+		eeprom->len++;
+	}
+
+	while (len >= 2) {
+		val = *data++;
+		val |= *data++ << 8;
+
+		err = mv88e6xxx_g2_eeprom_write16(chip, offset >> 1, val);
+		if (err)
+			return err;
+
+		offset += 2;
+		len -= 2;
+		eeprom->len += 2;
+	}
+
+	if (len) {
+		err = mv88e6xxx_g2_eeprom_read16(chip, offset >> 1, &val);
+		if (err)
+			return err;
+
+		val = (val & 0xff00) | *data++;
+
+		err = mv88e6xxx_g2_eeprom_write16(chip, offset >> 1, val);
+		if (err)
+			return err;
+
+		offset++;
+		len--;
+		eeprom->len++;
+	}
+
+	return 0;
+}
+
+/* Offset 0x18: SMI PHY Command Register
+ * Offset 0x19: SMI PHY Data Register
+ */
+
+static int mv88e6xxx_g2_smi_phy_wait(struct mv88e6xxx_chip *chip)
+{
+	return mv88e6xxx_g2_wait(chip, GLOBAL2_SMI_PHY_CMD,
+				 GLOBAL2_SMI_PHY_CMD_BUSY);
+}
+
+static int mv88e6xxx_g2_smi_phy_cmd(struct mv88e6xxx_chip *chip, u16 cmd)
+{
+	int err;
+
+	err = mv88e6xxx_g2_write(chip, GLOBAL2_SMI_PHY_CMD, cmd);
+	if (err)
+		return err;
+
+	return mv88e6xxx_g2_smi_phy_wait(chip);
+}
+
+int mv88e6xxx_g2_smi_phy_read(struct mv88e6xxx_chip *chip, int addr, int reg,
+			      u16 *val)
+{
+	u16 cmd = GLOBAL2_SMI_PHY_CMD_OP_22_READ_DATA | (addr << 5) | reg;
+	int err;
+
+	err = mv88e6xxx_g2_smi_phy_wait(chip);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g2_smi_phy_cmd(chip, cmd);
+	if (err)
+		return err;
+
+	return mv88e6xxx_g2_read(chip, GLOBAL2_SMI_PHY_DATA, val);
+}
+
+int mv88e6xxx_g2_smi_phy_write(struct mv88e6xxx_chip *chip, int addr, int reg,
+			       u16 val)
+{
+	u16 cmd = GLOBAL2_SMI_PHY_CMD_OP_22_WRITE_DATA | (addr << 5) | reg;
+	int err;
+
+	err = mv88e6xxx_g2_smi_phy_wait(chip);
+	if (err)
+		return err;
+
+	err = mv88e6xxx_g2_write(chip, GLOBAL2_SMI_PHY_DATA, val);
+	if (err)
+		return err;
+
+	return mv88e6xxx_g2_smi_phy_cmd(chip, cmd);
+}
+
+int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip)
+{
+	u16 reg;
+	int err;
+
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_2X)) {
+		/* Consider the frames with reserved multicast destination
+		 * addresses matching 01:80:c2:00:00:2x as MGMT.
+		 */
+		err = mv88e6xxx_g2_write(chip, GLOBAL2_MGMT_EN_2X, 0xffff);
+		if (err)
+			return err;
+	}
+
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_0X)) {
+		/* Consider the frames with reserved multicast destination
+		 * addresses matching 01:80:c2:00:00:0x as MGMT.
+		 */
+		err = mv88e6xxx_g2_write(chip, GLOBAL2_MGMT_EN_0X, 0xffff);
+		if (err)
+			return err;
+	}
+
+	/* Ignore removed tag data on doubly tagged packets, disable
+	 * flow control messages, force flow control priority to the
+	 * highest, and send all special multicast frames to the CPU
+	 * port at the highest priority.
+	 */
+	reg = GLOBAL2_SWITCH_MGMT_FORCE_FLOW_CTRL_PRI | (0x7 << 4);
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_0X) ||
+	    mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_2X))
+		reg |= GLOBAL2_SWITCH_MGMT_RSVD2CPU | 0x7;
+	err = mv88e6xxx_g2_write(chip, GLOBAL2_SWITCH_MGMT, reg);
+	if (err)
+		return err;
+
+	/* Program the DSA routing table. */
+	err = mv88e6xxx_g2_set_device_mapping(chip);
+	if (err)
+		return err;
+
+	/* Clear all trunk masks and mapping. */
+	err = mv88e6xxx_g2_clear_trunk(chip);
+	if (err)
+		return err;
+
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_IRL)) {
+		/* Disable ingress rate limiting by resetting all per port
+		 * ingress rate limit resources to their initial state.
+		 */
+		err = mv88e6xxx_g2_clear_irl(chip);
+			if (err)
+				return err;
+	}
+
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_PVT)) {
+		/* Initialize Cross-chip Port VLAN Table to reset defaults */
+		err = mv88e6xxx_g2_write(chip, GLOBAL2_PVT_ADDR,
+					 GLOBAL2_PVT_ADDR_OP_INIT_ONES);
+		if (err)
+			return err;
+	}
+
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_POT)) {
+		/* Clear the priority override table. */
+		err = mv88e6xxx_g2_clear_pot(chip);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/dsa/mv88e6xxx/global2.h b/drivers/net/dsa/mv88e6xxx/global2.h
new file mode 100644
index 000000000000..c4bb9035ee3a
--- /dev/null
+++ b/drivers/net/dsa/mv88e6xxx/global2.h
@@ -0,0 +1,88 @@
+/*
+ * Marvell 88E6xxx Switch Global 2 Registers support (device address 0x1C)
+ *
+ * Copyright (c) 2008 Marvell Semiconductor
+ *
+ * Copyright (c) 2016 Vivien Didelot <vivien.didelot@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef _MV88E6XXX_GLOBAL2_H
+#define _MV88E6XXX_GLOBAL2_H
+
+#include "mv88e6xxx.h"
+
+#ifdef CONFIG_NET_DSA_MV88E6XXX_GLOBAL2
+
+static inline int mv88e6xxx_g2_require(struct mv88e6xxx_chip *chip)
+{
+	return 0;
+}
+
+int mv88e6xxx_g2_smi_phy_read(struct mv88e6xxx_chip *chip, int addr, int reg,
+			      u16 *val);
+int mv88e6xxx_g2_smi_phy_write(struct mv88e6xxx_chip *chip, int addr, int reg,
+			       u16 val);
+int mv88e6xxx_g2_set_switch_mac(struct mv88e6xxx_chip *chip, u8 *addr);
+int mv88e6xxx_g2_get_eeprom16(struct mv88e6xxx_chip *chip,
+			      struct ethtool_eeprom *eeprom, u8 *data);
+int mv88e6xxx_g2_set_eeprom16(struct mv88e6xxx_chip *chip,
+			      struct ethtool_eeprom *eeprom, u8 *data);
+int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip);
+
+#else /* !CONFIG_NET_DSA_MV88E6XXX_GLOBAL2 */
+
+static inline int mv88e6xxx_g2_require(struct mv88e6xxx_chip *chip)
+{
+	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_GLOBAL2)) {
+		dev_err(chip->dev, "this chip requires CONFIG_NET_DSA_MV88E6XXX_GLOBAL2 enabled\n");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static inline int mv88e6xxx_g2_smi_phy_read(struct mv88e6xxx_chip *chip,
+					    int addr, int reg, u16 *val)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mv88e6xxx_g2_smi_phy_write(struct mv88e6xxx_chip *chip,
+					     int addr, int reg, u16 val)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mv88e6xxx_g2_set_switch_mac(struct mv88e6xxx_chip *chip,
+					      u8 *addr)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mv88e6xxx_g2_get_eeprom16(struct mv88e6xxx_chip *chip,
+					    struct ethtool_eeprom *eeprom,
+					    u8 *data)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mv88e6xxx_g2_set_eeprom16(struct mv88e6xxx_chip *chip,
+					    struct ethtool_eeprom *eeprom,
+					    u8 *data)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip)
+{
+	return -EOPNOTSUPP;
+}
+
+#endif /* CONFIG_NET_DSA_MV88E6XXX_GLOBAL2 */
+
+#endif /* _MV88E6XXX_GLOBAL2_H */
diff --git a/drivers/net/dsa/mv88e6xxx/mv88e6xxx.h b/drivers/net/dsa/mv88e6xxx/mv88e6xxx.h
index 48d6ea77f9bd..e572121c196e 100644
--- a/drivers/net/dsa/mv88e6xxx/mv88e6xxx.h
+++ b/drivers/net/dsa/mv88e6xxx/mv88e6xxx.h
@@ -30,11 +30,13 @@
 #define SMI_CMD_OP_45_READ_DATA_INC	((3 << 10) | SMI_CMD_BUSY)
 #define SMI_DATA		0x01
 
-/* Fiber/SERDES Registers are located at SMI address F, page 1 */
-#define REG_FIBER_SERDES	0x0f
-#define PAGE_FIBER_SERDES	0x01
+/* PHY Registers */
+#define PHY_PAGE		0x16
+#define PHY_PAGE_COPPER		0x00
+
+#define ADDR_SERDES		0x0f
+#define SERDES_PAGE_FIBER	0x01
 
-#define REG_PORT(p)		(0x10 + (p))
 #define PORT_STATUS		0x00
 #define PORT_STATUS_PAUSE_EN	BIT(15)
 #define PORT_STATUS_MY_PAUSE	BIT(14)
@@ -157,7 +159,6 @@
 #define PORT_TAG_REGMAP_0123	0x18
 #define PORT_TAG_REGMAP_4567	0x19
 
-#define REG_GLOBAL		0x1b
 #define GLOBAL_STATUS		0x00
 #define GLOBAL_STATUS_PPU_STATE BIT(15) /* 6351 and 6171 */
 /* Two bits for 6165, 6185 etc */
@@ -169,8 +170,8 @@
 #define GLOBAL_MAC_01		0x01
 #define GLOBAL_MAC_23		0x02
 #define GLOBAL_MAC_45		0x03
-#define GLOBAL_ATU_FID		0x01	/* 6097 6165 6351 6352 */
-#define GLOBAL_VTU_FID		0x02	/* 6097 6165 6351 6352 */
+#define GLOBAL_ATU_FID		0x01
+#define GLOBAL_VTU_FID		0x02
 #define GLOBAL_VTU_FID_MASK	0xfff
 #define GLOBAL_VTU_SID		0x03	/* 6097 6165 6351 6352 */
 #define GLOBAL_VTU_SID_MASK	0x3f
@@ -275,7 +276,6 @@
 #define GLOBAL_STATS_COUNTER_32	0x1e
 #define GLOBAL_STATS_COUNTER_01	0x1f
 
-#define REG_GLOBAL2		0x1c
 #define GLOBAL2_INT_SOURCE	0x00
 #define GLOBAL2_INT_MASK	0x01
 #define GLOBAL2_MGMT_EN_2X	0x02
@@ -329,17 +329,16 @@
 #define GLOBAL2_EEPROM_DATA	0x15
 #define GLOBAL2_PTP_AVB_OP	0x16
 #define GLOBAL2_PTP_AVB_DATA	0x17
-#define GLOBAL2_SMI_OP		0x18
-#define GLOBAL2_SMI_OP_BUSY		BIT(15)
-#define GLOBAL2_SMI_OP_CLAUSE_22	BIT(12)
-#define GLOBAL2_SMI_OP_22_WRITE		((1 << 10) | GLOBAL2_SMI_OP_BUSY | \
-					 GLOBAL2_SMI_OP_CLAUSE_22)
-#define GLOBAL2_SMI_OP_22_READ		((2 << 10) | GLOBAL2_SMI_OP_BUSY | \
-					 GLOBAL2_SMI_OP_CLAUSE_22)
-#define GLOBAL2_SMI_OP_45_WRITE_ADDR	((0 << 10) | GLOBAL2_SMI_OP_BUSY)
-#define GLOBAL2_SMI_OP_45_WRITE_DATA	((1 << 10) | GLOBAL2_SMI_OP_BUSY)
-#define GLOBAL2_SMI_OP_45_READ_DATA	((2 << 10) | GLOBAL2_SMI_OP_BUSY)
-#define GLOBAL2_SMI_DATA	0x19
+#define GLOBAL2_SMI_PHY_CMD			0x18
+#define GLOBAL2_SMI_PHY_CMD_BUSY		BIT(15)
+#define GLOBAL2_SMI_PHY_CMD_MODE_22		BIT(12)
+#define GLOBAL2_SMI_PHY_CMD_OP_22_WRITE_DATA	((0x1 << 10) | \
+						 GLOBAL2_SMI_PHY_CMD_MODE_22 | \
+						 GLOBAL2_SMI_PHY_CMD_BUSY)
+#define GLOBAL2_SMI_PHY_CMD_OP_22_READ_DATA	((0x2 << 10) | \
+						 GLOBAL2_SMI_PHY_CMD_MODE_22 | \
+						 GLOBAL2_SMI_PHY_CMD_BUSY)
+#define GLOBAL2_SMI_PHY_DATA			0x19
 #define GLOBAL2_SCRATCH_MISC	0x1a
 #define GLOBAL2_SCRATCH_BUSY		BIT(15)
 #define GLOBAL2_SCRATCH_REGISTER_SHIFT	8
@@ -384,10 +383,36 @@ enum mv88e6xxx_family {
 };
 
 enum mv88e6xxx_cap {
+	/* Two different tag protocols can be used by the driver. All
+	 * switches support DSA, but only later generations support
+	 * EDSA.
+	 */
+	MV88E6XXX_CAP_EDSA,
+
 	/* Energy Efficient Ethernet.
 	 */
 	MV88E6XXX_CAP_EEE,
 
+	/* Multi-chip Addressing Mode.
+	 * Some chips respond to only 2 registers of its own SMI device address
+	 * when it is non-zero, and use indirect access to internal registers.
+	 */
+	MV88E6XXX_CAP_SMI_CMD,		/* (0x00) SMI Command */
+	MV88E6XXX_CAP_SMI_DATA,		/* (0x01) SMI Data */
+
+	/* PHY Registers.
+	 */
+	MV88E6XXX_CAP_PHY_PAGE,		/* (0x16) Page Register */
+
+	/* Fiber/SERDES Registers (SMI address F).
+	 */
+	MV88E6XXX_CAP_SERDES,
+
+	/* Switch Global (1) Registers.
+	 */
+	MV88E6XXX_CAP_G1_ATU_FID,	/* (0x01) ATU FID Register */
+	MV88E6XXX_CAP_G1_VTU_FID,	/* (0x02) VTU FID Register */
+
 	/* Switch Global 2 Registers.
 	 * The device contains a second set of global 16-bit registers.
 	 */
@@ -398,16 +423,7 @@ enum mv88e6xxx_cap {
 	MV88E6XXX_CAP_G2_IRL_DATA,	/* (0x0a) Ingress Rate Data */
 	MV88E6XXX_CAP_G2_PVT_ADDR,	/* (0x0b) Cross Chip Port VLAN Addr */
 	MV88E6XXX_CAP_G2_PVT_DATA,	/* (0x0c) Cross Chip Port VLAN Data */
-	MV88E6XXX_CAP_G2_SWITCH_MAC,	/* (0x0d) Switch MAC/WoL/WoF */
 	MV88E6XXX_CAP_G2_POT,		/* (0x0f) Priority Override Table */
-	MV88E6XXX_CAP_G2_EEPROM_CMD,	/* (0x14) EEPROM Command */
-	MV88E6XXX_CAP_G2_EEPROM_DATA,	/* (0x15) EEPROM Data */
-
-	/* Multi-chip Addressing Mode.
-	 * Some chips require an indirect SMI access when their SMI device
-	 * address is not zero. See SMI_CMD and SMI_DATA.
-	 */
-	MV88E6XXX_CAP_MULTI_CHIP,
 
 	/* PHY Polling Unit.
 	 * See GLOBAL_CONTROL_PPU_ENABLE and GLOBAL_STATUS_PPU_POLLING.
@@ -415,12 +431,6 @@ enum mv88e6xxx_cap {
 	MV88E6XXX_CAP_PPU,
 	MV88E6XXX_CAP_PPU_ACTIVE,
 
-	/* SMI PHY Command and Data registers.
-	 * This requires an indirect access to PHY registers through
-	 * GLOBAL2_SMI_OP, otherwise direct access to PHY registers is done.
-	 */
-	MV88E6XXX_CAP_SMI_PHY,
-
 	/* Per VLAN Spanning Tree Unit (STU).
 	 * The Port State database, if present, is accessed through VTU
 	 * operations and dedicated SID registers. See GLOBAL_VTU_SID.
@@ -440,130 +450,148 @@ enum mv88e6xxx_cap {
 };
 
 /* Bitmask of capabilities */
-#define MV88E6XXX_FLAG_EEE		BIT(MV88E6XXX_CAP_EEE)
-#define MV88E6XXX_FLAG_GLOBAL2		BIT(MV88E6XXX_CAP_GLOBAL2)
-#define MV88E6XXX_FLAG_G2_MGMT_EN_2X	BIT(MV88E6XXX_CAP_G2_MGMT_EN_2X)
-#define MV88E6XXX_FLAG_G2_MGMT_EN_0X	BIT(MV88E6XXX_CAP_G2_MGMT_EN_0X)
-#define MV88E6XXX_FLAG_G2_IRL_CMD	BIT(MV88E6XXX_CAP_G2_IRL_CMD)
-#define MV88E6XXX_FLAG_G2_IRL_DATA	BIT(MV88E6XXX_CAP_G2_IRL_DATA)
-#define MV88E6XXX_FLAG_G2_PVT_ADDR	BIT(MV88E6XXX_CAP_G2_PVT_ADDR)
-#define MV88E6XXX_FLAG_G2_PVT_DATA	BIT(MV88E6XXX_CAP_G2_PVT_DATA)
-#define MV88E6XXX_FLAG_G2_SWITCH_MAC	BIT(MV88E6XXX_CAP_G2_SWITCH_MAC)
-#define MV88E6XXX_FLAG_G2_POT		BIT(MV88E6XXX_CAP_G2_POT)
-#define MV88E6XXX_FLAG_G2_EEPROM_CMD	BIT(MV88E6XXX_CAP_G2_EEPROM_CMD)
-#define MV88E6XXX_FLAG_G2_EEPROM_DATA	BIT(MV88E6XXX_CAP_G2_EEPROM_DATA)
-#define MV88E6XXX_FLAG_MULTI_CHIP	BIT(MV88E6XXX_CAP_MULTI_CHIP)
-#define MV88E6XXX_FLAG_PPU		BIT(MV88E6XXX_CAP_PPU)
-#define MV88E6XXX_FLAG_PPU_ACTIVE	BIT(MV88E6XXX_CAP_PPU_ACTIVE)
-#define MV88E6XXX_FLAG_SMI_PHY		BIT(MV88E6XXX_CAP_SMI_PHY)
-#define MV88E6XXX_FLAG_STU		BIT(MV88E6XXX_CAP_STU)
-#define MV88E6XXX_FLAG_TEMP		BIT(MV88E6XXX_CAP_TEMP)
-#define MV88E6XXX_FLAG_TEMP_LIMIT	BIT(MV88E6XXX_CAP_TEMP_LIMIT)
-#define MV88E6XXX_FLAG_VTU		BIT(MV88E6XXX_CAP_VTU)
-
-/* EEPROM Programming via Global2 with 16-bit data */
-#define MV88E6XXX_FLAGS_EEPROM16	\
-	(MV88E6XXX_FLAG_G2_EEPROM_CMD |	\
-	 MV88E6XXX_FLAG_G2_EEPROM_DATA)
+#define MV88E6XXX_FLAG_EDSA		BIT_ULL(MV88E6XXX_CAP_EDSA)
+#define MV88E6XXX_FLAG_EEE		BIT_ULL(MV88E6XXX_CAP_EEE)
+
+#define MV88E6XXX_FLAG_SMI_CMD		BIT_ULL(MV88E6XXX_CAP_SMI_CMD)
+#define MV88E6XXX_FLAG_SMI_DATA		BIT_ULL(MV88E6XXX_CAP_SMI_DATA)
+
+#define MV88E6XXX_FLAG_PHY_PAGE		BIT_ULL(MV88E6XXX_CAP_PHY_PAGE)
+
+#define MV88E6XXX_FLAG_SERDES		BIT_ULL(MV88E6XXX_CAP_SERDES)
+
+#define MV88E6XXX_FLAG_G1_ATU_FID	BIT_ULL(MV88E6XXX_CAP_G1_ATU_FID)
+#define MV88E6XXX_FLAG_G1_VTU_FID	BIT_ULL(MV88E6XXX_CAP_G1_VTU_FID)
+
+#define MV88E6XXX_FLAG_GLOBAL2		BIT_ULL(MV88E6XXX_CAP_GLOBAL2)
+#define MV88E6XXX_FLAG_G2_MGMT_EN_2X	BIT_ULL(MV88E6XXX_CAP_G2_MGMT_EN_2X)
+#define MV88E6XXX_FLAG_G2_MGMT_EN_0X	BIT_ULL(MV88E6XXX_CAP_G2_MGMT_EN_0X)
+#define MV88E6XXX_FLAG_G2_IRL_CMD	BIT_ULL(MV88E6XXX_CAP_G2_IRL_CMD)
+#define MV88E6XXX_FLAG_G2_IRL_DATA	BIT_ULL(MV88E6XXX_CAP_G2_IRL_DATA)
+#define MV88E6XXX_FLAG_G2_PVT_ADDR	BIT_ULL(MV88E6XXX_CAP_G2_PVT_ADDR)
+#define MV88E6XXX_FLAG_G2_PVT_DATA	BIT_ULL(MV88E6XXX_CAP_G2_PVT_DATA)
+#define MV88E6XXX_FLAG_G2_POT		BIT_ULL(MV88E6XXX_CAP_G2_POT)
+
+#define MV88E6XXX_FLAG_PPU		BIT_ULL(MV88E6XXX_CAP_PPU)
+#define MV88E6XXX_FLAG_PPU_ACTIVE	BIT_ULL(MV88E6XXX_CAP_PPU_ACTIVE)
+#define MV88E6XXX_FLAG_STU		BIT_ULL(MV88E6XXX_CAP_STU)
+#define MV88E6XXX_FLAG_TEMP		BIT_ULL(MV88E6XXX_CAP_TEMP)
+#define MV88E6XXX_FLAG_TEMP_LIMIT	BIT_ULL(MV88E6XXX_CAP_TEMP_LIMIT)
+#define MV88E6XXX_FLAG_VTU		BIT_ULL(MV88E6XXX_CAP_VTU)
 
 /* Ingress Rate Limit unit */
 #define MV88E6XXX_FLAGS_IRL		\
 	(MV88E6XXX_FLAG_G2_IRL_CMD |	\
 	 MV88E6XXX_FLAG_G2_IRL_DATA)
 
+/* Multi-chip Addressing Mode */
+#define MV88E6XXX_FLAGS_MULTI_CHIP	\
+	(MV88E6XXX_FLAG_SMI_CMD |	\
+	 MV88E6XXX_FLAG_SMI_DATA)
+
 /* Cross-chip Port VLAN Table */
 #define MV88E6XXX_FLAGS_PVT		\
 	(MV88E6XXX_FLAG_G2_PVT_ADDR |	\
 	 MV88E6XXX_FLAG_G2_PVT_DATA)
 
+/* Fiber/SERDES Registers at SMI address F, page 1 */
+#define MV88E6XXX_FLAGS_SERDES		\
+	(MV88E6XXX_FLAG_PHY_PAGE |	\
+	 MV88E6XXX_FLAG_SERDES)
+
 #define MV88E6XXX_FLAGS_FAMILY_6095	\
 	(MV88E6XXX_FLAG_GLOBAL2 |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_MULTI_CHIP |	\
 	 MV88E6XXX_FLAG_PPU |		\
-	 MV88E6XXX_FLAG_VTU)
+	 MV88E6XXX_FLAG_VTU |		\
+	 MV88E6XXX_FLAGS_MULTI_CHIP)
 
 #define MV88E6XXX_FLAGS_FAMILY_6097	\
-	(MV88E6XXX_FLAG_GLOBAL2 |	\
+	(MV88E6XXX_FLAG_G1_ATU_FID |	\
+	 MV88E6XXX_FLAG_G1_VTU_FID |	\
+	 MV88E6XXX_FLAG_GLOBAL2 |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
 	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAG_MULTI_CHIP |	\
 	 MV88E6XXX_FLAG_PPU |		\
 	 MV88E6XXX_FLAG_STU |		\
 	 MV88E6XXX_FLAG_VTU |		\
 	 MV88E6XXX_FLAGS_IRL |		\
+	 MV88E6XXX_FLAGS_MULTI_CHIP |	\
 	 MV88E6XXX_FLAGS_PVT)
 
 #define MV88E6XXX_FLAGS_FAMILY_6165	\
-	(MV88E6XXX_FLAG_GLOBAL2 |	\
+	(MV88E6XXX_FLAG_G1_ATU_FID |	\
+	 MV88E6XXX_FLAG_G1_VTU_FID |	\
+	 MV88E6XXX_FLAG_GLOBAL2 |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_G2_SWITCH_MAC |	\
 	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAG_MULTI_CHIP |	\
 	 MV88E6XXX_FLAG_STU |		\
 	 MV88E6XXX_FLAG_TEMP |		\
 	 MV88E6XXX_FLAG_VTU |		\
 	 MV88E6XXX_FLAGS_IRL |		\
+	 MV88E6XXX_FLAGS_MULTI_CHIP |	\
 	 MV88E6XXX_FLAGS_PVT)
 
 #define MV88E6XXX_FLAGS_FAMILY_6185	\
 	(MV88E6XXX_FLAG_GLOBAL2 |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_MULTI_CHIP |	\
+	 MV88E6XXX_FLAGS_MULTI_CHIP |	\
 	 MV88E6XXX_FLAG_PPU |		\
 	 MV88E6XXX_FLAG_VTU)
 
 #define MV88E6XXX_FLAGS_FAMILY_6320	\
-	(MV88E6XXX_FLAG_EEE |		\
+	(MV88E6XXX_FLAG_EDSA |		\
+	 MV88E6XXX_FLAG_EEE |		\
 	 MV88E6XXX_FLAG_GLOBAL2 |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_G2_SWITCH_MAC |	\
 	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAG_MULTI_CHIP |	\
 	 MV88E6XXX_FLAG_PPU_ACTIVE |	\
-	 MV88E6XXX_FLAG_SMI_PHY |	\
 	 MV88E6XXX_FLAG_TEMP |		\
 	 MV88E6XXX_FLAG_TEMP_LIMIT |	\
 	 MV88E6XXX_FLAG_VTU |		\
-	 MV88E6XXX_FLAGS_EEPROM16 |	\
 	 MV88E6XXX_FLAGS_IRL |		\
+	 MV88E6XXX_FLAGS_MULTI_CHIP |	\
 	 MV88E6XXX_FLAGS_PVT)
 
 #define MV88E6XXX_FLAGS_FAMILY_6351	\
-	(MV88E6XXX_FLAG_GLOBAL2 |	\
+	(MV88E6XXX_FLAG_EDSA |		\
+	 MV88E6XXX_FLAG_G1_ATU_FID |	\
+	 MV88E6XXX_FLAG_G1_VTU_FID |	\
+	 MV88E6XXX_FLAG_GLOBAL2 |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_G2_SWITCH_MAC |	\
 	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAG_MULTI_CHIP |	\
 	 MV88E6XXX_FLAG_PPU_ACTIVE |	\
-	 MV88E6XXX_FLAG_SMI_PHY |	\
 	 MV88E6XXX_FLAG_STU |		\
 	 MV88E6XXX_FLAG_TEMP |		\
 	 MV88E6XXX_FLAG_VTU |		\
 	 MV88E6XXX_FLAGS_IRL |		\
+	 MV88E6XXX_FLAGS_MULTI_CHIP |	\
 	 MV88E6XXX_FLAGS_PVT)
 
 #define MV88E6XXX_FLAGS_FAMILY_6352	\
-	(MV88E6XXX_FLAG_EEE |		\
+	(MV88E6XXX_FLAG_EDSA |		\
+	 MV88E6XXX_FLAG_EEE |		\
+	 MV88E6XXX_FLAG_G1_ATU_FID |	\
+	 MV88E6XXX_FLAG_G1_VTU_FID |	\
 	 MV88E6XXX_FLAG_GLOBAL2 |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
 	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_G2_SWITCH_MAC |	\
 	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAG_MULTI_CHIP |	\
 	 MV88E6XXX_FLAG_PPU_ACTIVE |	\
-	 MV88E6XXX_FLAG_SMI_PHY |	\
 	 MV88E6XXX_FLAG_STU |		\
 	 MV88E6XXX_FLAG_TEMP |		\
 	 MV88E6XXX_FLAG_TEMP_LIMIT |	\
 	 MV88E6XXX_FLAG_VTU |		\
-	 MV88E6XXX_FLAGS_EEPROM16 |	\
 	 MV88E6XXX_FLAGS_IRL |		\
-	 MV88E6XXX_FLAGS_PVT)
+	 MV88E6XXX_FLAGS_MULTI_CHIP |	\
+	 MV88E6XXX_FLAGS_PVT |		\
+	 MV88E6XXX_FLAGS_SERDES)
+
+struct mv88e6xxx_ops;
 
 struct mv88e6xxx_info {
 	enum mv88e6xxx_family family;
@@ -572,8 +600,10 @@ struct mv88e6xxx_info {
 	unsigned int num_databases;
 	unsigned int num_ports;
 	unsigned int port_base_addr;
+	unsigned int global1_addr;
 	unsigned int age_time_coeff;
-	unsigned long flags;
+	unsigned long long flags;
+	const struct mv88e6xxx_ops *ops;
 };
 
 struct mv88e6xxx_atu_entry {
@@ -584,18 +614,15 @@ struct mv88e6xxx_atu_entry {
 	u8	mac[ETH_ALEN];
 };
 
-struct mv88e6xxx_vtu_stu_entry {
-	/* VTU only */
+struct mv88e6xxx_vtu_entry {
 	u16	vid;
 	u16	fid;
-
-	/* VTU and STU */
 	u8	sid;
 	bool	valid;
 	u8	data[DSA_MAX_PORTS];
 };
 
-struct mv88e6xxx_ops;
+struct mv88e6xxx_bus_ops;
 
 struct mv88e6xxx_priv_port {
 	struct net_device *bridge_dev;
@@ -616,13 +643,14 @@ struct mv88e6xxx_chip {
 	/* The MII bus and the address on the bus that is used to
 	 * communication with the switch
 	 */
-	const struct mv88e6xxx_ops *smi_ops;
+	const struct mv88e6xxx_bus_ops *smi_ops;
 	struct mii_bus *bus;
 	int sw_addr;
 
 	/* Handles automatic disabling and re-enabling of the PHY
 	 * polling unit.
 	 */
+	const struct mv88e6xxx_bus_ops *phy_ops;
 	struct mutex		ppu_mutex;
 	int			ppu_disabled;
 	struct work_struct	ppu_work;
@@ -651,11 +679,25 @@ struct mv88e6xxx_chip {
 	struct mii_bus *mdio_bus;
 };
 
-struct mv88e6xxx_ops {
+struct mv88e6xxx_bus_ops {
 	int (*read)(struct mv88e6xxx_chip *chip, int addr, int reg, u16 *val);
 	int (*write)(struct mv88e6xxx_chip *chip, int addr, int reg, u16 val);
 };
 
+struct mv88e6xxx_ops {
+	int (*get_eeprom)(struct mv88e6xxx_chip *chip,
+			  struct ethtool_eeprom *eeprom, u8 *data);
+	int (*set_eeprom)(struct mv88e6xxx_chip *chip,
+			  struct ethtool_eeprom *eeprom, u8 *data);
+
+	int (*set_switch_mac)(struct mv88e6xxx_chip *chip, u8 *addr);
+
+	int (*phy_read)(struct mv88e6xxx_chip *chip, int addr, int reg,
+			u16 *val);
+	int (*phy_write)(struct mv88e6xxx_chip *chip, int addr, int reg,
+			 u16 val);
+};
+
 enum stat_type {
 	BANK0,
 	BANK1,
@@ -675,4 +717,20 @@ static inline bool mv88e6xxx_has(struct mv88e6xxx_chip *chip,
 	return (chip->info->flags & flags) == flags;
 }
 
+static inline unsigned int mv88e6xxx_num_databases(struct mv88e6xxx_chip *chip)
+{
+	return chip->info->num_databases;
+}
+
+static inline unsigned int mv88e6xxx_num_ports(struct mv88e6xxx_chip *chip)
+{
+	return chip->info->num_ports;
+}
+
+int mv88e6xxx_read(struct mv88e6xxx_chip *chip, int addr, int reg, u16 *val);
+int mv88e6xxx_write(struct mv88e6xxx_chip *chip, int addr, int reg, u16 val);
+int mv88e6xxx_update(struct mv88e6xxx_chip *chip, int addr, int reg,
+		     u16 update);
+int mv88e6xxx_wait(struct mv88e6xxx_chip *chip, int addr, int reg, u16 mask);
+
 #endif
diff --git a/drivers/net/dsa/qca8k.c b/drivers/net/dsa/qca8k.c
new file mode 100644
index 000000000000..b3df70d07ff6
--- /dev/null
+++ b/drivers/net/dsa/qca8k.c
@@ -0,0 +1,1040 @@
+/*
+ * Copyright (C) 2009 Felix Fietkau <nbd@nbd.name>
+ * Copyright (C) 2011-2012 Gabor Juhos <juhosg@openwrt.org>
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2016 John Crispin <john@phrozen.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/phy.h>
+#include <linux/netdevice.h>
+#include <net/dsa.h>
+#include <net/switchdev.h>
+#include <linux/of_net.h>
+#include <linux/of_platform.h>
+#include <linux/if_bridge.h>
+#include <linux/mdio.h>
+#include <linux/etherdevice.h>
+
+#include "qca8k.h"
+
+#define MIB_DESC(_s, _o, _n)	\
+	{			\
+		.size = (_s),	\
+		.offset = (_o),	\
+		.name = (_n),	\
+	}
+
+static const struct qca8k_mib_desc ar8327_mib[] = {
+	MIB_DESC(1, 0x00, "RxBroad"),
+	MIB_DESC(1, 0x04, "RxPause"),
+	MIB_DESC(1, 0x08, "RxMulti"),
+	MIB_DESC(1, 0x0c, "RxFcsErr"),
+	MIB_DESC(1, 0x10, "RxAlignErr"),
+	MIB_DESC(1, 0x14, "RxRunt"),
+	MIB_DESC(1, 0x18, "RxFragment"),
+	MIB_DESC(1, 0x1c, "Rx64Byte"),
+	MIB_DESC(1, 0x20, "Rx128Byte"),
+	MIB_DESC(1, 0x24, "Rx256Byte"),
+	MIB_DESC(1, 0x28, "Rx512Byte"),
+	MIB_DESC(1, 0x2c, "Rx1024Byte"),
+	MIB_DESC(1, 0x30, "Rx1518Byte"),
+	MIB_DESC(1, 0x34, "RxMaxByte"),
+	MIB_DESC(1, 0x38, "RxTooLong"),
+	MIB_DESC(2, 0x3c, "RxGoodByte"),
+	MIB_DESC(2, 0x44, "RxBadByte"),
+	MIB_DESC(1, 0x4c, "RxOverFlow"),
+	MIB_DESC(1, 0x50, "Filtered"),
+	MIB_DESC(1, 0x54, "TxBroad"),
+	MIB_DESC(1, 0x58, "TxPause"),
+	MIB_DESC(1, 0x5c, "TxMulti"),
+	MIB_DESC(1, 0x60, "TxUnderRun"),
+	MIB_DESC(1, 0x64, "Tx64Byte"),
+	MIB_DESC(1, 0x68, "Tx128Byte"),
+	MIB_DESC(1, 0x6c, "Tx256Byte"),
+	MIB_DESC(1, 0x70, "Tx512Byte"),
+	MIB_DESC(1, 0x74, "Tx1024Byte"),
+	MIB_DESC(1, 0x78, "Tx1518Byte"),
+	MIB_DESC(1, 0x7c, "TxMaxByte"),
+	MIB_DESC(1, 0x80, "TxOverSize"),
+	MIB_DESC(2, 0x84, "TxByte"),
+	MIB_DESC(1, 0x8c, "TxCollision"),
+	MIB_DESC(1, 0x90, "TxAbortCol"),
+	MIB_DESC(1, 0x94, "TxMultiCol"),
+	MIB_DESC(1, 0x98, "TxSingleCol"),
+	MIB_DESC(1, 0x9c, "TxExcDefer"),
+	MIB_DESC(1, 0xa0, "TxDefer"),
+	MIB_DESC(1, 0xa4, "TxLateCol"),
+};
+
+/* The 32bit switch registers are accessed indirectly. To achieve this we need
+ * to set the page of the register. Track the last page that was set to reduce
+ * mdio writes
+ */
+static u16 qca8k_current_page = 0xffff;
+
+static void
+qca8k_split_addr(u32 regaddr, u16 *r1, u16 *r2, u16 *page)
+{
+	regaddr >>= 1;
+	*r1 = regaddr & 0x1e;
+
+	regaddr >>= 5;
+	*r2 = regaddr & 0x7;
+
+	regaddr >>= 3;
+	*page = regaddr & 0x3ff;
+}
+
+static u32
+qca8k_mii_read32(struct mii_bus *bus, int phy_id, u32 regnum)
+{
+	u32 val;
+	int ret;
+
+	ret = bus->read(bus, phy_id, regnum);
+	if (ret >= 0) {
+		val = ret;
+		ret = bus->read(bus, phy_id, regnum + 1);
+		val |= ret << 16;
+	}
+
+	if (ret < 0) {
+		dev_err_ratelimited(&bus->dev,
+				    "failed to read qca8k 32bit register\n");
+		return ret;
+	}
+
+	return val;
+}
+
+static void
+qca8k_mii_write32(struct mii_bus *bus, int phy_id, u32 regnum, u32 val)
+{
+	u16 lo, hi;
+	int ret;
+
+	lo = val & 0xffff;
+	hi = (u16)(val >> 16);
+
+	ret = bus->write(bus, phy_id, regnum, lo);
+	if (ret >= 0)
+		ret = bus->write(bus, phy_id, regnum + 1, hi);
+	if (ret < 0)
+		dev_err_ratelimited(&bus->dev,
+				    "failed to write qca8k 32bit register\n");
+}
+
+static void
+qca8k_set_page(struct mii_bus *bus, u16 page)
+{
+	if (page == qca8k_current_page)
+		return;
+
+	if (bus->write(bus, 0x18, 0, page) < 0)
+		dev_err_ratelimited(&bus->dev,
+				    "failed to set qca8k page\n");
+	qca8k_current_page = page;
+}
+
+static u32
+qca8k_read(struct qca8k_priv *priv, u32 reg)
+{
+	u16 r1, r2, page;
+	u32 val;
+
+	qca8k_split_addr(reg, &r1, &r2, &page);
+
+	mutex_lock_nested(&priv->bus->mdio_lock, MDIO_MUTEX_NESTED);
+
+	qca8k_set_page(priv->bus, page);
+	val = qca8k_mii_read32(priv->bus, 0x10 | r2, r1);
+
+	mutex_unlock(&priv->bus->mdio_lock);
+
+	return val;
+}
+
+static void
+qca8k_write(struct qca8k_priv *priv, u32 reg, u32 val)
+{
+	u16 r1, r2, page;
+
+	qca8k_split_addr(reg, &r1, &r2, &page);
+
+	mutex_lock_nested(&priv->bus->mdio_lock, MDIO_MUTEX_NESTED);
+
+	qca8k_set_page(priv->bus, page);
+	qca8k_mii_write32(priv->bus, 0x10 | r2, r1, val);
+
+	mutex_unlock(&priv->bus->mdio_lock);
+}
+
+static u32
+qca8k_rmw(struct qca8k_priv *priv, u32 reg, u32 mask, u32 val)
+{
+	u16 r1, r2, page;
+	u32 ret;
+
+	qca8k_split_addr(reg, &r1, &r2, &page);
+
+	mutex_lock_nested(&priv->bus->mdio_lock, MDIO_MUTEX_NESTED);
+
+	qca8k_set_page(priv->bus, page);
+	ret = qca8k_mii_read32(priv->bus, 0x10 | r2, r1);
+	ret &= ~mask;
+	ret |= val;
+	qca8k_mii_write32(priv->bus, 0x10 | r2, r1, ret);
+
+	mutex_unlock(&priv->bus->mdio_lock);
+
+	return ret;
+}
+
+static void
+qca8k_reg_set(struct qca8k_priv *priv, u32 reg, u32 val)
+{
+	qca8k_rmw(priv, reg, 0, val);
+}
+
+static void
+qca8k_reg_clear(struct qca8k_priv *priv, u32 reg, u32 val)
+{
+	qca8k_rmw(priv, reg, val, 0);
+}
+
+static int
+qca8k_regmap_read(void *ctx, uint32_t reg, uint32_t *val)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ctx;
+
+	*val = qca8k_read(priv, reg);
+
+	return 0;
+}
+
+static int
+qca8k_regmap_write(void *ctx, uint32_t reg, uint32_t val)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ctx;
+
+	qca8k_write(priv, reg, val);
+
+	return 0;
+}
+
+static const struct regmap_range qca8k_readable_ranges[] = {
+	regmap_reg_range(0x0000, 0x00e4), /* Global control */
+	regmap_reg_range(0x0100, 0x0168), /* EEE control */
+	regmap_reg_range(0x0200, 0x0270), /* Parser control */
+	regmap_reg_range(0x0400, 0x0454), /* ACL */
+	regmap_reg_range(0x0600, 0x0718), /* Lookup */
+	regmap_reg_range(0x0800, 0x0b70), /* QM */
+	regmap_reg_range(0x0c00, 0x0c80), /* PKT */
+	regmap_reg_range(0x0e00, 0x0e98), /* L3 */
+	regmap_reg_range(0x1000, 0x10ac), /* MIB - Port0 */
+	regmap_reg_range(0x1100, 0x11ac), /* MIB - Port1 */
+	regmap_reg_range(0x1200, 0x12ac), /* MIB - Port2 */
+	regmap_reg_range(0x1300, 0x13ac), /* MIB - Port3 */
+	regmap_reg_range(0x1400, 0x14ac), /* MIB - Port4 */
+	regmap_reg_range(0x1500, 0x15ac), /* MIB - Port5 */
+	regmap_reg_range(0x1600, 0x16ac), /* MIB - Port6 */
+
+};
+
+static struct regmap_access_table qca8k_readable_table = {
+	.yes_ranges = qca8k_readable_ranges,
+	.n_yes_ranges = ARRAY_SIZE(qca8k_readable_ranges),
+};
+
+static struct regmap_config qca8k_regmap_config = {
+	.reg_bits = 16,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = 0x16ac, /* end MIB - Port6 range */
+	.reg_read = qca8k_regmap_read,
+	.reg_write = qca8k_regmap_write,
+	.rd_table = &qca8k_readable_table,
+};
+
+static int
+qca8k_busy_wait(struct qca8k_priv *priv, u32 reg, u32 mask)
+{
+	unsigned long timeout;
+
+	timeout = jiffies + msecs_to_jiffies(20);
+
+	/* loop until the busy flag has cleared */
+	do {
+		u32 val = qca8k_read(priv, reg);
+		int busy = val & mask;
+
+		if (!busy)
+			break;
+		cond_resched();
+	} while (!time_after_eq(jiffies, timeout));
+
+	return time_after_eq(jiffies, timeout);
+}
+
+static void
+qca8k_fdb_read(struct qca8k_priv *priv, struct qca8k_fdb *fdb)
+{
+	u32 reg[4];
+	int i;
+
+	/* load the ARL table into an array */
+	for (i = 0; i < 4; i++)
+		reg[i] = qca8k_read(priv, QCA8K_REG_ATU_DATA0 + (i * 4));
+
+	/* vid - 83:72 */
+	fdb->vid = (reg[2] >> QCA8K_ATU_VID_S) & QCA8K_ATU_VID_M;
+	/* aging - 67:64 */
+	fdb->aging = reg[2] & QCA8K_ATU_STATUS_M;
+	/* portmask - 54:48 */
+	fdb->port_mask = (reg[1] >> QCA8K_ATU_PORT_S) & QCA8K_ATU_PORT_M;
+	/* mac - 47:0 */
+	fdb->mac[0] = (reg[1] >> QCA8K_ATU_ADDR0_S) & 0xff;
+	fdb->mac[1] = reg[1] & 0xff;
+	fdb->mac[2] = (reg[0] >> QCA8K_ATU_ADDR2_S) & 0xff;
+	fdb->mac[3] = (reg[0] >> QCA8K_ATU_ADDR3_S) & 0xff;
+	fdb->mac[4] = (reg[0] >> QCA8K_ATU_ADDR4_S) & 0xff;
+	fdb->mac[5] = reg[0] & 0xff;
+}
+
+static void
+qca8k_fdb_write(struct qca8k_priv *priv, u16 vid, u8 port_mask, const u8 *mac,
+		u8 aging)
+{
+	u32 reg[3] = { 0 };
+	int i;
+
+	/* vid - 83:72 */
+	reg[2] = (vid & QCA8K_ATU_VID_M) << QCA8K_ATU_VID_S;
+	/* aging - 67:64 */
+	reg[2] |= aging & QCA8K_ATU_STATUS_M;
+	/* portmask - 54:48 */
+	reg[1] = (port_mask & QCA8K_ATU_PORT_M) << QCA8K_ATU_PORT_S;
+	/* mac - 47:0 */
+	reg[1] |= mac[0] << QCA8K_ATU_ADDR0_S;
+	reg[1] |= mac[1];
+	reg[0] |= mac[2] << QCA8K_ATU_ADDR2_S;
+	reg[0] |= mac[3] << QCA8K_ATU_ADDR3_S;
+	reg[0] |= mac[4] << QCA8K_ATU_ADDR4_S;
+	reg[0] |= mac[5];
+
+	/* load the array into the ARL table */
+	for (i = 0; i < 3; i++)
+		qca8k_write(priv, QCA8K_REG_ATU_DATA0 + (i * 4), reg[i]);
+}
+
+static int
+qca8k_fdb_access(struct qca8k_priv *priv, enum qca8k_fdb_cmd cmd, int port)
+{
+	u32 reg;
+
+	/* Set the command and FDB index */
+	reg = QCA8K_ATU_FUNC_BUSY;
+	reg |= cmd;
+	if (port >= 0) {
+		reg |= QCA8K_ATU_FUNC_PORT_EN;
+		reg |= (port & QCA8K_ATU_FUNC_PORT_M) << QCA8K_ATU_FUNC_PORT_S;
+	}
+
+	/* Write the function register triggering the table access */
+	qca8k_write(priv, QCA8K_REG_ATU_FUNC, reg);
+
+	/* wait for completion */
+	if (qca8k_busy_wait(priv, QCA8K_REG_ATU_FUNC, QCA8K_ATU_FUNC_BUSY))
+		return -1;
+
+	/* Check for table full violation when adding an entry */
+	if (cmd == QCA8K_FDB_LOAD) {
+		reg = qca8k_read(priv, QCA8K_REG_ATU_FUNC);
+		if (reg & QCA8K_ATU_FUNC_FULL)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int
+qca8k_fdb_next(struct qca8k_priv *priv, struct qca8k_fdb *fdb, int port)
+{
+	int ret;
+
+	qca8k_fdb_write(priv, fdb->vid, fdb->port_mask, fdb->mac, fdb->aging);
+	ret = qca8k_fdb_access(priv, QCA8K_FDB_NEXT, port);
+	if (ret >= 0)
+		qca8k_fdb_read(priv, fdb);
+
+	return ret;
+}
+
+static int
+qca8k_fdb_add(struct qca8k_priv *priv, const u8 *mac, u16 port_mask,
+	      u16 vid, u8 aging)
+{
+	int ret;
+
+	mutex_lock(&priv->reg_mutex);
+	qca8k_fdb_write(priv, vid, port_mask, mac, aging);
+	ret = qca8k_fdb_access(priv, QCA8K_FDB_LOAD, -1);
+	mutex_unlock(&priv->reg_mutex);
+
+	return ret;
+}
+
+static int
+qca8k_fdb_del(struct qca8k_priv *priv, const u8 *mac, u16 port_mask, u16 vid)
+{
+	int ret;
+
+	mutex_lock(&priv->reg_mutex);
+	qca8k_fdb_write(priv, vid, port_mask, mac, 0);
+	ret = qca8k_fdb_access(priv, QCA8K_FDB_PURGE, -1);
+	mutex_unlock(&priv->reg_mutex);
+
+	return ret;
+}
+
+static void
+qca8k_fdb_flush(struct qca8k_priv *priv)
+{
+	mutex_lock(&priv->reg_mutex);
+	qca8k_fdb_access(priv, QCA8K_FDB_FLUSH, -1);
+	mutex_unlock(&priv->reg_mutex);
+}
+
+static void
+qca8k_mib_init(struct qca8k_priv *priv)
+{
+	mutex_lock(&priv->reg_mutex);
+	qca8k_reg_set(priv, QCA8K_REG_MIB, QCA8K_MIB_FLUSH | QCA8K_MIB_BUSY);
+	qca8k_busy_wait(priv, QCA8K_REG_MIB, QCA8K_MIB_BUSY);
+	qca8k_reg_set(priv, QCA8K_REG_MIB, QCA8K_MIB_CPU_KEEP);
+	qca8k_write(priv, QCA8K_REG_MODULE_EN, QCA8K_MODULE_EN_MIB);
+	mutex_unlock(&priv->reg_mutex);
+}
+
+static int
+qca8k_set_pad_ctrl(struct qca8k_priv *priv, int port, int mode)
+{
+	u32 reg;
+
+	switch (port) {
+	case 0:
+		reg = QCA8K_REG_PORT0_PAD_CTRL;
+		break;
+	case 6:
+		reg = QCA8K_REG_PORT6_PAD_CTRL;
+		break;
+	default:
+		pr_err("Can't set PAD_CTRL on port %d\n", port);
+		return -EINVAL;
+	}
+
+	/* Configure a port to be directly connected to an external
+	 * PHY or MAC.
+	 */
+	switch (mode) {
+	case PHY_INTERFACE_MODE_RGMII:
+		qca8k_write(priv, reg,
+			    QCA8K_PORT_PAD_RGMII_EN |
+			    QCA8K_PORT_PAD_RGMII_TX_DELAY(3) |
+			    QCA8K_PORT_PAD_RGMII_RX_DELAY(3));
+
+		/* According to the datasheet, RGMII delay is enabled through
+		 * PORT5_PAD_CTRL for all ports, rather than individual port
+		 * registers
+		 */
+		qca8k_write(priv, QCA8K_REG_PORT5_PAD_CTRL,
+			    QCA8K_PORT_PAD_RGMII_RX_DELAY_EN);
+		break;
+	case PHY_INTERFACE_MODE_SGMII:
+		qca8k_write(priv, reg, QCA8K_PORT_PAD_SGMII_EN);
+		break;
+	default:
+		pr_err("xMII mode %d not supported\n", mode);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void
+qca8k_port_set_status(struct qca8k_priv *priv, int port, int enable)
+{
+	u32 mask = QCA8K_PORT_STATUS_TXMAC;
+
+	/* Port 0 and 6 have no internal PHY */
+	if ((port > 0) && (port < 6))
+		mask |= QCA8K_PORT_STATUS_LINK_AUTO;
+
+	if (enable)
+		qca8k_reg_set(priv, QCA8K_REG_PORT_STATUS(port), mask);
+	else
+		qca8k_reg_clear(priv, QCA8K_REG_PORT_STATUS(port), mask);
+}
+
+static int
+qca8k_setup(struct dsa_switch *ds)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	int ret, i, phy_mode = -1;
+
+	/* Make sure that port 0 is the cpu port */
+	if (!dsa_is_cpu_port(ds, 0)) {
+		pr_err("port 0 is not the CPU port\n");
+		return -EINVAL;
+	}
+
+	mutex_init(&priv->reg_mutex);
+
+	/* Start by setting up the register mapping */
+	priv->regmap = devm_regmap_init(ds->dev, NULL, priv,
+					&qca8k_regmap_config);
+	if (IS_ERR(priv->regmap))
+		pr_warn("regmap initialization failed");
+
+	/* Initialize CPU port pad mode (xMII type, delays...) */
+	phy_mode = of_get_phy_mode(ds->ports[ds->dst->cpu_port].dn);
+	if (phy_mode < 0) {
+		pr_err("Can't find phy-mode for master device\n");
+		return phy_mode;
+	}
+	ret = qca8k_set_pad_ctrl(priv, QCA8K_CPU_PORT, phy_mode);
+	if (ret < 0)
+		return ret;
+
+	/* Enable CPU Port */
+	qca8k_reg_set(priv, QCA8K_REG_GLOBAL_FW_CTRL0,
+		      QCA8K_GLOBAL_FW_CTRL0_CPU_PORT_EN);
+	qca8k_port_set_status(priv, QCA8K_CPU_PORT, 1);
+	priv->port_sts[QCA8K_CPU_PORT].enabled = 1;
+
+	/* Enable MIB counters */
+	qca8k_mib_init(priv);
+
+	/* Enable QCA header mode on the cpu port */
+	qca8k_write(priv, QCA8K_REG_PORT_HDR_CTRL(QCA8K_CPU_PORT),
+		    QCA8K_PORT_HDR_CTRL_ALL << QCA8K_PORT_HDR_CTRL_TX_S |
+		    QCA8K_PORT_HDR_CTRL_ALL << QCA8K_PORT_HDR_CTRL_RX_S);
+
+	/* Disable forwarding by default on all ports */
+	for (i = 0; i < QCA8K_NUM_PORTS; i++)
+		qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(i),
+			  QCA8K_PORT_LOOKUP_MEMBER, 0);
+
+	/* Disable MAC by default on all user ports */
+	for (i = 1; i < QCA8K_NUM_PORTS; i++)
+		if (ds->enabled_port_mask & BIT(i))
+			qca8k_port_set_status(priv, i, 0);
+
+	/* Forward all unknown frames to CPU port for Linux processing */
+	qca8k_write(priv, QCA8K_REG_GLOBAL_FW_CTRL1,
+		    BIT(0) << QCA8K_GLOBAL_FW_CTRL1_IGMP_DP_S |
+		    BIT(0) << QCA8K_GLOBAL_FW_CTRL1_BC_DP_S |
+		    BIT(0) << QCA8K_GLOBAL_FW_CTRL1_MC_DP_S |
+		    BIT(0) << QCA8K_GLOBAL_FW_CTRL1_UC_DP_S);
+
+	/* Setup connection between CPU port & user ports */
+	for (i = 0; i < DSA_MAX_PORTS; i++) {
+		/* CPU port gets connected to all user ports of the switch */
+		if (dsa_is_cpu_port(ds, i)) {
+			qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(QCA8K_CPU_PORT),
+				  QCA8K_PORT_LOOKUP_MEMBER,
+				  ds->enabled_port_mask);
+		}
+
+		/* Invividual user ports get connected to CPU port only */
+		if (ds->enabled_port_mask & BIT(i)) {
+			int shift = 16 * (i % 2);
+
+			qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(i),
+				  QCA8K_PORT_LOOKUP_MEMBER,
+				  BIT(QCA8K_CPU_PORT));
+
+			/* Enable ARP Auto-learning by default */
+			qca8k_reg_set(priv, QCA8K_PORT_LOOKUP_CTRL(i),
+				      QCA8K_PORT_LOOKUP_LEARN);
+
+			/* For port based vlans to work we need to set the
+			 * default egress vid
+			 */
+			qca8k_rmw(priv, QCA8K_EGRESS_VLAN(i),
+				  0xffff << shift, 1 << shift);
+			qca8k_write(priv, QCA8K_REG_PORT_VLAN_CTRL0(i),
+				    QCA8K_PORT_VLAN_CVID(1) |
+				    QCA8K_PORT_VLAN_SVID(1));
+		}
+	}
+
+	/* Flush the FDB table */
+	qca8k_fdb_flush(priv);
+
+	return 0;
+}
+
+static int
+qca8k_phy_read(struct dsa_switch *ds, int phy, int regnum)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+
+	return mdiobus_read(priv->bus, phy, regnum);
+}
+
+static int
+qca8k_phy_write(struct dsa_switch *ds, int phy, int regnum, u16 val)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+
+	return mdiobus_write(priv->bus, phy, regnum, val);
+}
+
+static void
+qca8k_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ar8327_mib); i++)
+		strncpy(data + i * ETH_GSTRING_LEN, ar8327_mib[i].name,
+			ETH_GSTRING_LEN);
+}
+
+static void
+qca8k_get_ethtool_stats(struct dsa_switch *ds, int port,
+			uint64_t *data)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	const struct qca8k_mib_desc *mib;
+	u32 reg, i;
+	u64 hi;
+
+	for (i = 0; i < ARRAY_SIZE(ar8327_mib); i++) {
+		mib = &ar8327_mib[i];
+		reg = QCA8K_PORT_MIB_COUNTER(port) + mib->offset;
+
+		data[i] = qca8k_read(priv, reg);
+		if (mib->size == 2) {
+			hi = qca8k_read(priv, reg + 4);
+			data[i] |= hi << 32;
+		}
+	}
+}
+
+static int
+qca8k_get_sset_count(struct dsa_switch *ds)
+{
+	return ARRAY_SIZE(ar8327_mib);
+}
+
+static void
+qca8k_eee_enable_set(struct dsa_switch *ds, int port, bool enable)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	u32 lpi_en = QCA8K_REG_EEE_CTRL_LPI_EN(port);
+	u32 reg;
+
+	mutex_lock(&priv->reg_mutex);
+	reg = qca8k_read(priv, QCA8K_REG_EEE_CTRL);
+	if (enable)
+		reg |= lpi_en;
+	else
+		reg &= ~lpi_en;
+	qca8k_write(priv, QCA8K_REG_EEE_CTRL, reg);
+	mutex_unlock(&priv->reg_mutex);
+}
+
+static int
+qca8k_eee_init(struct dsa_switch *ds, int port,
+	       struct phy_device *phy)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	struct ethtool_eee *p = &priv->port_sts[port].eee;
+	int ret;
+
+	p->supported = (SUPPORTED_1000baseT_Full | SUPPORTED_100baseT_Full);
+
+	ret = phy_init_eee(phy, 0);
+	if (ret)
+		return ret;
+
+	qca8k_eee_enable_set(ds, port, true);
+
+	return 0;
+}
+
+static int
+qca8k_set_eee(struct dsa_switch *ds, int port,
+	      struct phy_device *phydev,
+	      struct ethtool_eee *e)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	struct ethtool_eee *p = &priv->port_sts[port].eee;
+	int ret = 0;
+
+	p->eee_enabled = e->eee_enabled;
+
+	if (e->eee_enabled) {
+		p->eee_enabled = qca8k_eee_init(ds, port, phydev);
+		if (!p->eee_enabled)
+			ret = -EOPNOTSUPP;
+	}
+	qca8k_eee_enable_set(ds, port, p->eee_enabled);
+
+	return ret;
+}
+
+static int
+qca8k_get_eee(struct dsa_switch *ds, int port,
+	      struct ethtool_eee *e)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	struct ethtool_eee *p = &priv->port_sts[port].eee;
+	struct net_device *netdev = ds->ports[port].netdev;
+	int ret;
+
+	ret = phy_ethtool_get_eee(netdev->phydev, p);
+	if (!ret)
+		e->eee_active =
+			!!(p->supported & p->advertised & p->lp_advertised);
+	else
+		e->eee_active = 0;
+
+	e->eee_enabled = p->eee_enabled;
+
+	return ret;
+}
+
+static void
+qca8k_port_stp_state_set(struct dsa_switch *ds, int port, u8 state)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	u32 stp_state;
+
+	switch (state) {
+	case BR_STATE_DISABLED:
+		stp_state = QCA8K_PORT_LOOKUP_STATE_DISABLED;
+		break;
+	case BR_STATE_BLOCKING:
+		stp_state = QCA8K_PORT_LOOKUP_STATE_BLOCKING;
+		break;
+	case BR_STATE_LISTENING:
+		stp_state = QCA8K_PORT_LOOKUP_STATE_LISTENING;
+		break;
+	case BR_STATE_LEARNING:
+		stp_state = QCA8K_PORT_LOOKUP_STATE_LEARNING;
+		break;
+	case BR_STATE_FORWARDING:
+	default:
+		stp_state = QCA8K_PORT_LOOKUP_STATE_FORWARD;
+		break;
+	}
+
+	qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(port),
+		  QCA8K_PORT_LOOKUP_STATE_MASK, stp_state);
+}
+
+static int
+qca8k_port_bridge_join(struct dsa_switch *ds, int port,
+		       struct net_device *bridge)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	int port_mask = BIT(QCA8K_CPU_PORT);
+	int i;
+
+	priv->port_sts[port].bridge_dev = bridge;
+
+	for (i = 1; i < QCA8K_NUM_PORTS; i++) {
+		if (priv->port_sts[i].bridge_dev != bridge)
+			continue;
+		/* Add this port to the portvlan mask of the other ports
+		 * in the bridge
+		 */
+		qca8k_reg_set(priv,
+			      QCA8K_PORT_LOOKUP_CTRL(i),
+			      BIT(port));
+		if (i != port)
+			port_mask |= BIT(i);
+	}
+	/* Add all other ports to this ports portvlan mask */
+	qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(port),
+		  QCA8K_PORT_LOOKUP_MEMBER, port_mask);
+
+	return 0;
+}
+
+static void
+qca8k_port_bridge_leave(struct dsa_switch *ds, int port)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	int i;
+
+	for (i = 1; i < QCA8K_NUM_PORTS; i++) {
+		if (priv->port_sts[i].bridge_dev !=
+		    priv->port_sts[port].bridge_dev)
+			continue;
+		/* Remove this port to the portvlan mask of the other ports
+		 * in the bridge
+		 */
+		qca8k_reg_clear(priv,
+				QCA8K_PORT_LOOKUP_CTRL(i),
+				BIT(port));
+	}
+	priv->port_sts[port].bridge_dev = NULL;
+	/* Set the cpu port to be the only one in the portvlan mask of
+	 * this port
+	 */
+	qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(port),
+		  QCA8K_PORT_LOOKUP_MEMBER, BIT(QCA8K_CPU_PORT));
+}
+
+static int
+qca8k_port_enable(struct dsa_switch *ds, int port,
+		  struct phy_device *phy)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+
+	qca8k_port_set_status(priv, port, 1);
+	priv->port_sts[port].enabled = 1;
+
+	return 0;
+}
+
+static void
+qca8k_port_disable(struct dsa_switch *ds, int port,
+		   struct phy_device *phy)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+
+	qca8k_port_set_status(priv, port, 0);
+	priv->port_sts[port].enabled = 0;
+}
+
+static int
+qca8k_port_fdb_insert(struct qca8k_priv *priv, const u8 *addr,
+		      u16 port_mask, u16 vid)
+{
+	/* Set the vid to the port vlan id if no vid is set */
+	if (!vid)
+		vid = 1;
+
+	return qca8k_fdb_add(priv, addr, port_mask, vid,
+			     QCA8K_ATU_STATUS_STATIC);
+}
+
+static int
+qca8k_port_fdb_prepare(struct dsa_switch *ds, int port,
+		       const struct switchdev_obj_port_fdb *fdb,
+		       struct switchdev_trans *trans)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+
+	/* The FDB table for static and auto learned entries is the same. We
+	 * need to reserve an entry with no port_mask set to make sure that
+	 * when port_fdb_add is called an entry is still available. Otherwise
+	 * the last free entry might have been used up by auto learning
+	 */
+	return qca8k_port_fdb_insert(priv, fdb->addr, 0, fdb->vid);
+}
+
+static void
+qca8k_port_fdb_add(struct dsa_switch *ds, int port,
+		   const struct switchdev_obj_port_fdb *fdb,
+		   struct switchdev_trans *trans)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	u16 port_mask = BIT(port);
+
+	/* Update the FDB entry adding the port_mask */
+	qca8k_port_fdb_insert(priv, fdb->addr, port_mask, fdb->vid);
+}
+
+static int
+qca8k_port_fdb_del(struct dsa_switch *ds, int port,
+		   const struct switchdev_obj_port_fdb *fdb)
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	u16 port_mask = BIT(port);
+	u16 vid = fdb->vid;
+
+	if (!vid)
+		vid = 1;
+
+	return qca8k_fdb_del(priv, fdb->addr, port_mask, vid);
+}
+
+static int
+qca8k_port_fdb_dump(struct dsa_switch *ds, int port,
+		    struct switchdev_obj_port_fdb *fdb,
+		    int (*cb)(struct switchdev_obj *obj))
+{
+	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
+	struct qca8k_fdb _fdb = { 0 };
+	int cnt = QCA8K_NUM_FDB_RECORDS;
+	int ret = 0;
+
+	mutex_lock(&priv->reg_mutex);
+	while (cnt-- && !qca8k_fdb_next(priv, &_fdb, port)) {
+		if (!_fdb.aging)
+			break;
+
+		ether_addr_copy(fdb->addr, _fdb.mac);
+		fdb->vid = _fdb.vid;
+		if (_fdb.aging == QCA8K_ATU_STATUS_STATIC)
+			fdb->ndm_state = NUD_NOARP;
+		else
+			fdb->ndm_state = NUD_REACHABLE;
+
+		ret = cb(&fdb->obj);
+		if (ret)
+			break;
+	}
+	mutex_unlock(&priv->reg_mutex);
+
+	return 0;
+}
+
+static enum dsa_tag_protocol
+qca8k_get_tag_protocol(struct dsa_switch *ds)
+{
+	return DSA_TAG_PROTO_QCA;
+}
+
+static struct dsa_switch_ops qca8k_switch_ops = {
+	.get_tag_protocol	= qca8k_get_tag_protocol,
+	.setup			= qca8k_setup,
+	.get_strings		= qca8k_get_strings,
+	.phy_read		= qca8k_phy_read,
+	.phy_write		= qca8k_phy_write,
+	.get_ethtool_stats	= qca8k_get_ethtool_stats,
+	.get_sset_count		= qca8k_get_sset_count,
+	.get_eee		= qca8k_get_eee,
+	.set_eee		= qca8k_set_eee,
+	.port_enable		= qca8k_port_enable,
+	.port_disable		= qca8k_port_disable,
+	.port_stp_state_set	= qca8k_port_stp_state_set,
+	.port_bridge_join	= qca8k_port_bridge_join,
+	.port_bridge_leave	= qca8k_port_bridge_leave,
+	.port_fdb_prepare	= qca8k_port_fdb_prepare,
+	.port_fdb_add		= qca8k_port_fdb_add,
+	.port_fdb_del		= qca8k_port_fdb_del,
+	.port_fdb_dump		= qca8k_port_fdb_dump,
+};
+
+static int
+qca8k_sw_probe(struct mdio_device *mdiodev)
+{
+	struct qca8k_priv *priv;
+	u32 id;
+
+	/* allocate the private data struct so that we can probe the switches
+	 * ID register
+	 */
+	priv = devm_kzalloc(&mdiodev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->bus = mdiodev->bus;
+
+	/* read the switches ID register */
+	id = qca8k_read(priv, QCA8K_REG_MASK_CTRL);
+	id >>= QCA8K_MASK_CTRL_ID_S;
+	id &= QCA8K_MASK_CTRL_ID_M;
+	if (id != QCA8K_ID_QCA8337)
+		return -ENODEV;
+
+	priv->ds = devm_kzalloc(&mdiodev->dev, sizeof(*priv->ds), GFP_KERNEL);
+	if (!priv->ds)
+		return -ENOMEM;
+
+	priv->ds->priv = priv;
+	priv->ds->dev = &mdiodev->dev;
+	priv->ds->ops = &qca8k_switch_ops;
+	mutex_init(&priv->reg_mutex);
+	dev_set_drvdata(&mdiodev->dev, priv);
+
+	return dsa_register_switch(priv->ds, priv->ds->dev->of_node);
+}
+
+static void
+qca8k_sw_remove(struct mdio_device *mdiodev)
+{
+	struct qca8k_priv *priv = dev_get_drvdata(&mdiodev->dev);
+	int i;
+
+	for (i = 0; i < QCA8K_NUM_PORTS; i++)
+		qca8k_port_set_status(priv, i, 0);
+
+	dsa_unregister_switch(priv->ds);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static void
+qca8k_set_pm(struct qca8k_priv *priv, int enable)
+{
+	int i;
+
+	for (i = 0; i < QCA8K_NUM_PORTS; i++) {
+		if (!priv->port_sts[i].enabled)
+			continue;
+
+		qca8k_port_set_status(priv, i, enable);
+	}
+}
+
+static int qca8k_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct qca8k_priv *priv = platform_get_drvdata(pdev);
+
+	qca8k_set_pm(priv, 0);
+
+	return dsa_switch_suspend(priv->ds);
+}
+
+static int qca8k_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct qca8k_priv *priv = platform_get_drvdata(pdev);
+
+	qca8k_set_pm(priv, 1);
+
+	return dsa_switch_resume(priv->ds);
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static SIMPLE_DEV_PM_OPS(qca8k_pm_ops,
+			 qca8k_suspend, qca8k_resume);
+
+static const struct of_device_id qca8k_of_match[] = {
+	{ .compatible = "qca,qca8337" },
+	{ /* sentinel */ },
+};
+
+static struct mdio_driver qca8kmdio_driver = {
+	.probe  = qca8k_sw_probe,
+	.remove = qca8k_sw_remove,
+	.mdiodrv.driver = {
+		.name = "qca8k",
+		.of_match_table = qca8k_of_match,
+		.pm = &qca8k_pm_ops,
+	},
+};
+
+mdio_module_driver(qca8kmdio_driver);
+
+MODULE_AUTHOR("Mathieu Olivari, John Crispin <john@phrozen.org>");
+MODULE_DESCRIPTION("Driver for QCA8K ethernet switch family");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:qca8k");
diff --git a/drivers/net/dsa/qca8k.h b/drivers/net/dsa/qca8k.h
new file mode 100644
index 000000000000..201464719531
--- /dev/null
+++ b/drivers/net/dsa/qca8k.h
@@ -0,0 +1,185 @@
+/*
+ * Copyright (C) 2009 Felix Fietkau <nbd@nbd.name>
+ * Copyright (C) 2011-2012 Gabor Juhos <juhosg@openwrt.org>
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __QCA8K_H
+#define __QCA8K_H
+
+#include <linux/delay.h>
+#include <linux/regmap.h>
+
+#define QCA8K_NUM_PORTS					7
+
+#define PHY_ID_QCA8337					0x004dd036
+#define QCA8K_ID_QCA8337				0x13
+
+#define QCA8K_NUM_FDB_RECORDS				2048
+
+#define QCA8K_CPU_PORT					0
+
+/* Global control registers */
+#define QCA8K_REG_MASK_CTRL				0x000
+#define   QCA8K_MASK_CTRL_ID_M				0xff
+#define   QCA8K_MASK_CTRL_ID_S				8
+#define QCA8K_REG_PORT0_PAD_CTRL			0x004
+#define QCA8K_REG_PORT5_PAD_CTRL			0x008
+#define QCA8K_REG_PORT6_PAD_CTRL			0x00c
+#define   QCA8K_PORT_PAD_RGMII_EN			BIT(26)
+#define   QCA8K_PORT_PAD_RGMII_TX_DELAY(x)		\
+						((0x8 + (x & 0x3)) << 22)
+#define   QCA8K_PORT_PAD_RGMII_RX_DELAY(x)		\
+						((0x10 + (x & 0x3)) << 20)
+#define   QCA8K_PORT_PAD_RGMII_RX_DELAY_EN		BIT(24)
+#define   QCA8K_PORT_PAD_SGMII_EN			BIT(7)
+#define QCA8K_REG_MODULE_EN				0x030
+#define   QCA8K_MODULE_EN_MIB				BIT(0)
+#define QCA8K_REG_MIB					0x034
+#define   QCA8K_MIB_FLUSH				BIT(24)
+#define   QCA8K_MIB_CPU_KEEP				BIT(20)
+#define   QCA8K_MIB_BUSY				BIT(17)
+#define QCA8K_GOL_MAC_ADDR0				0x60
+#define QCA8K_GOL_MAC_ADDR1				0x64
+#define QCA8K_REG_PORT_STATUS(_i)			(0x07c + (_i) * 4)
+#define   QCA8K_PORT_STATUS_SPEED			GENMASK(2, 0)
+#define   QCA8K_PORT_STATUS_SPEED_S			0
+#define   QCA8K_PORT_STATUS_TXMAC			BIT(2)
+#define   QCA8K_PORT_STATUS_RXMAC			BIT(3)
+#define   QCA8K_PORT_STATUS_TXFLOW			BIT(4)
+#define   QCA8K_PORT_STATUS_RXFLOW			BIT(5)
+#define   QCA8K_PORT_STATUS_DUPLEX			BIT(6)
+#define   QCA8K_PORT_STATUS_LINK_UP			BIT(8)
+#define   QCA8K_PORT_STATUS_LINK_AUTO			BIT(9)
+#define   QCA8K_PORT_STATUS_LINK_PAUSE			BIT(10)
+#define QCA8K_REG_PORT_HDR_CTRL(_i)			(0x9c + (_i * 4))
+#define   QCA8K_PORT_HDR_CTRL_RX_MASK			GENMASK(3, 2)
+#define   QCA8K_PORT_HDR_CTRL_RX_S			2
+#define   QCA8K_PORT_HDR_CTRL_TX_MASK			GENMASK(1, 0)
+#define   QCA8K_PORT_HDR_CTRL_TX_S			0
+#define   QCA8K_PORT_HDR_CTRL_ALL			2
+#define   QCA8K_PORT_HDR_CTRL_MGMT			1
+#define   QCA8K_PORT_HDR_CTRL_NONE			0
+
+/* EEE control registers */
+#define QCA8K_REG_EEE_CTRL				0x100
+#define  QCA8K_REG_EEE_CTRL_LPI_EN(_i)			((_i + 1) * 2)
+
+/* ACL registers */
+#define QCA8K_REG_PORT_VLAN_CTRL0(_i)			(0x420 + (_i * 8))
+#define   QCA8K_PORT_VLAN_CVID(x)			(x << 16)
+#define   QCA8K_PORT_VLAN_SVID(x)			x
+#define QCA8K_REG_PORT_VLAN_CTRL1(_i)			(0x424 + (_i * 8))
+#define QCA8K_REG_IPV4_PRI_BASE_ADDR			0x470
+#define QCA8K_REG_IPV4_PRI_ADDR_MASK			0x474
+
+/* Lookup registers */
+#define QCA8K_REG_ATU_DATA0				0x600
+#define   QCA8K_ATU_ADDR2_S				24
+#define   QCA8K_ATU_ADDR3_S				16
+#define   QCA8K_ATU_ADDR4_S				8
+#define QCA8K_REG_ATU_DATA1				0x604
+#define   QCA8K_ATU_PORT_M				0x7f
+#define   QCA8K_ATU_PORT_S				16
+#define   QCA8K_ATU_ADDR0_S				8
+#define QCA8K_REG_ATU_DATA2				0x608
+#define   QCA8K_ATU_VID_M				0xfff
+#define   QCA8K_ATU_VID_S				8
+#define   QCA8K_ATU_STATUS_M				0xf
+#define   QCA8K_ATU_STATUS_STATIC			0xf
+#define QCA8K_REG_ATU_FUNC				0x60c
+#define   QCA8K_ATU_FUNC_BUSY				BIT(31)
+#define   QCA8K_ATU_FUNC_PORT_EN			BIT(14)
+#define   QCA8K_ATU_FUNC_MULTI_EN			BIT(13)
+#define   QCA8K_ATU_FUNC_FULL				BIT(12)
+#define   QCA8K_ATU_FUNC_PORT_M				0xf
+#define   QCA8K_ATU_FUNC_PORT_S				8
+#define QCA8K_REG_GLOBAL_FW_CTRL0			0x620
+#define   QCA8K_GLOBAL_FW_CTRL0_CPU_PORT_EN		BIT(10)
+#define QCA8K_REG_GLOBAL_FW_CTRL1			0x624
+#define   QCA8K_GLOBAL_FW_CTRL1_IGMP_DP_S		24
+#define   QCA8K_GLOBAL_FW_CTRL1_BC_DP_S			16
+#define   QCA8K_GLOBAL_FW_CTRL1_MC_DP_S			8
+#define   QCA8K_GLOBAL_FW_CTRL1_UC_DP_S			0
+#define QCA8K_PORT_LOOKUP_CTRL(_i)			(0x660 + (_i) * 0xc)
+#define   QCA8K_PORT_LOOKUP_MEMBER			GENMASK(6, 0)
+#define   QCA8K_PORT_LOOKUP_STATE_MASK			GENMASK(18, 16)
+#define   QCA8K_PORT_LOOKUP_STATE_DISABLED		(0 << 16)
+#define   QCA8K_PORT_LOOKUP_STATE_BLOCKING		(1 << 16)
+#define   QCA8K_PORT_LOOKUP_STATE_LISTENING		(2 << 16)
+#define   QCA8K_PORT_LOOKUP_STATE_LEARNING		(3 << 16)
+#define   QCA8K_PORT_LOOKUP_STATE_FORWARD		(4 << 16)
+#define   QCA8K_PORT_LOOKUP_STATE			GENMASK(18, 16)
+#define   QCA8K_PORT_LOOKUP_LEARN			BIT(20)
+
+/* Pkt edit registers */
+#define QCA8K_EGRESS_VLAN(x)				(0x0c70 + (4 * (x / 2)))
+
+/* L3 registers */
+#define QCA8K_HROUTER_CONTROL				0xe00
+#define   QCA8K_HROUTER_CONTROL_GLB_LOCKTIME_M		GENMASK(17, 16)
+#define   QCA8K_HROUTER_CONTROL_GLB_LOCKTIME_S		16
+#define   QCA8K_HROUTER_CONTROL_ARP_AGE_MODE		1
+#define QCA8K_HROUTER_PBASED_CONTROL1			0xe08
+#define QCA8K_HROUTER_PBASED_CONTROL2			0xe0c
+#define QCA8K_HNAT_CONTROL				0xe38
+
+/* MIB registers */
+#define QCA8K_PORT_MIB_COUNTER(_i)			(0x1000 + (_i) * 0x100)
+
+/* QCA specific MII registers */
+#define MII_ATH_MMD_ADDR				0x0d
+#define MII_ATH_MMD_DATA				0x0e
+
+enum {
+	QCA8K_PORT_SPEED_10M = 0,
+	QCA8K_PORT_SPEED_100M = 1,
+	QCA8K_PORT_SPEED_1000M = 2,
+	QCA8K_PORT_SPEED_ERR = 3,
+};
+
+enum qca8k_fdb_cmd {
+	QCA8K_FDB_FLUSH	= 1,
+	QCA8K_FDB_LOAD = 2,
+	QCA8K_FDB_PURGE = 3,
+	QCA8K_FDB_NEXT = 6,
+	QCA8K_FDB_SEARCH = 7,
+};
+
+struct ar8xxx_port_status {
+	struct ethtool_eee eee;
+	struct net_device *bridge_dev;
+	int enabled;
+};
+
+struct qca8k_priv {
+	struct regmap *regmap;
+	struct mii_bus *bus;
+	struct ar8xxx_port_status port_sts[QCA8K_NUM_PORTS];
+	struct dsa_switch *ds;
+	struct mutex reg_mutex;
+};
+
+struct qca8k_mib_desc {
+	unsigned int size;
+	unsigned int offset;
+	const char *name;
+};
+
+struct qca8k_fdb {
+	u16 vid;
+	u8 port_mask;
+	u8 aging;
+	u8 mac[6];
+};
+
+#endif /* __QCA8K_H */
diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c
index 25c55ab05c7d..9133e7926da5 100644
--- a/drivers/net/ethernet/3com/3c59x.c
+++ b/drivers/net/ethernet/3com/3c59x.c
@@ -3089,7 +3089,7 @@ static void set_rx_mode(struct net_device *dev)
 	iowrite16(new_mode, ioaddr + EL3_CMD);
 }
 
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
 /* Setup the card so that it can receive frames with an 802.1q VLAN tag.
    Note that this must be done after each RxReset due to some backwards
    compatibility logic in the Cyclone and Tornado ASICs */
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index 2ffd63463299..8cc7467b6c1f 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -24,6 +24,7 @@ source "drivers/net/ethernet/agere/Kconfig"
 source "drivers/net/ethernet/allwinner/Kconfig"
 source "drivers/net/ethernet/alteon/Kconfig"
 source "drivers/net/ethernet/altera/Kconfig"
+source "drivers/net/ethernet/amazon/Kconfig"
 source "drivers/net/ethernet/amd/Kconfig"
 source "drivers/net/ethernet/apm/Kconfig"
 source "drivers/net/ethernet/apple/Kconfig"
diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile
index 1d349e9aa9a6..a09423df83f2 100644
--- a/drivers/net/ethernet/Makefile
+++ b/drivers/net/ethernet/Makefile
@@ -10,6 +10,7 @@ obj-$(CONFIG_NET_VENDOR_AGERE) += agere/
 obj-$(CONFIG_NET_VENDOR_ALLWINNER) += allwinner/
 obj-$(CONFIG_NET_VENDOR_ALTEON) += alteon/
 obj-$(CONFIG_ALTERA_TSE) += altera/
+obj-$(CONFIG_NET_VENDOR_AMAZON) += amazon/
 obj-$(CONFIG_NET_VENDOR_AMD) += amd/
 obj-$(CONFIG_NET_XGENE) += apm/
 obj-$(CONFIG_NET_VENDOR_APPLE) += apple/
diff --git a/drivers/net/ethernet/adaptec/starfire.c b/drivers/net/ethernet/adaptec/starfire.c
index 1d1069641d81..8af2c88d5b33 100644
--- a/drivers/net/ethernet/adaptec/starfire.c
+++ b/drivers/net/ethernet/adaptec/starfire.c
@@ -66,7 +66,7 @@
  */
 #define ZEROCOPY
 
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
 #define VLAN_SUPPORT
 #endif
 
diff --git a/drivers/net/ethernet/adi/bfin_mac.c b/drivers/net/ethernet/adi/bfin_mac.c
index 38eaea18da23..00f9ee3fc3e5 100644
--- a/drivers/net/ethernet/adi/bfin_mac.c
+++ b/drivers/net/ethernet/adi/bfin_mac.c
@@ -192,8 +192,8 @@ static int desc_list_init(struct net_device *dev)
 			goto init_error;
 
 		skb_reserve(new_skb, NET_IP_ALIGN);
-		/* Invidate the data cache of skb->data range when it is write back
-		 * cache. It will prevent overwritting the new data from DMA
+		/* Invalidate the data cache of skb->data range when it is write back
+		 * cache. It will prevent overwriting the new data from DMA
 		 */
 		blackfin_dcache_invalidate_range((unsigned long)new_skb->head,
 					 (unsigned long)new_skb->end);
@@ -1205,7 +1205,7 @@ static void bfin_mac_rx(struct bfin_mac_local *lp)
 	}
 	/* reserve 2 bytes for RXDWA padding */
 	skb_reserve(new_skb, NET_IP_ALIGN);
-	/* Invidate the data cache of skb->data range when it is write back
+	/* Invalidate the data cache of skb->data range when it is write back
 	 * cache. It will prevent overwritting the new data from DMA
 	 */
 	blackfin_dcache_invalidate_range((unsigned long)new_skb->head,
@@ -1599,7 +1599,7 @@ static int bfin_mac_probe(struct platform_device *pdev)
 	*(__le16 *) (&(ndev->dev_addr[4])) = cpu_to_le16((u16) bfin_read_EMAC_ADDRHI());
 
 	/* probe mac */
-	/*todo: how to proble? which is revision_register */
+	/*todo: how to probe? which is revision_register */
 	bfin_write_EMAC_ADDRLO(0x12345678);
 	if (bfin_read_EMAC_ADDRLO() != 0x12345678) {
 		dev_err(&pdev->dev, "Cannot detect Blackfin on-chip ethernet MAC controller!\n");
diff --git a/drivers/net/ethernet/aeroflex/greth.c b/drivers/net/ethernet/aeroflex/greth.c
index bca07c5c94bd..f8df8248035e 100644
--- a/drivers/net/ethernet/aeroflex/greth.c
+++ b/drivers/net/ethernet/aeroflex/greth.c
@@ -1105,27 +1105,6 @@ static void greth_set_msglevel(struct net_device *dev, u32 value)
 	struct greth_private *greth = netdev_priv(dev);
 	greth->msg_enable = value;
 }
-static int greth_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-	struct greth_private *greth = netdev_priv(dev);
-	struct phy_device *phy = greth->phy;
-
-	if (!phy)
-		return -ENODEV;
-
-	return phy_ethtool_gset(phy, cmd);
-}
-
-static int greth_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-	struct greth_private *greth = netdev_priv(dev);
-	struct phy_device *phy = greth->phy;
-
-	if (!phy)
-		return -ENODEV;
-
-	return phy_ethtool_sset(phy, cmd);
-}
 
 static int greth_get_regs_len(struct net_device *dev)
 {
@@ -1157,12 +1136,12 @@ static void greth_get_regs(struct net_device *dev, struct ethtool_regs *regs, vo
 static const struct ethtool_ops greth_ethtool_ops = {
 	.get_msglevel		= greth_get_msglevel,
 	.set_msglevel		= greth_set_msglevel,
-	.get_settings		= greth_get_settings,
-	.set_settings		= greth_set_settings,
 	.get_drvinfo		= greth_get_drvinfo,
 	.get_regs_len           = greth_get_regs_len,
 	.get_regs               = greth_get_regs,
 	.get_link		= ethtool_op_get_link,
+	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
+	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
 };
 
 static struct net_device_ops greth_netdev_ops = {
@@ -1224,7 +1203,7 @@ static int greth_mdio_write(struct mii_bus *bus, int phy, int reg, u16 val)
 static void greth_link_change(struct net_device *dev)
 {
 	struct greth_private *greth = netdev_priv(dev);
-	struct phy_device *phydev = greth->phy;
+	struct phy_device *phydev = dev->phydev;
 	unsigned long flags;
 	int status_change = 0;
 	u32 ctrl;
@@ -1307,7 +1286,6 @@ static int greth_mdio_probe(struct net_device *dev)
 	greth->link = 0;
 	greth->speed = 0;
 	greth->duplex = -1;
-	greth->phy = phy;
 
 	return 0;
 }
@@ -1325,6 +1303,7 @@ static int greth_mdio_init(struct greth_private *greth)
 {
 	int ret;
 	unsigned long timeout;
+	struct net_device *ndev = greth->netdev;
 
 	greth->mdio = mdiobus_alloc();
 	if (!greth->mdio) {
@@ -1349,15 +1328,16 @@ static int greth_mdio_init(struct greth_private *greth)
 		goto unreg_mdio;
 	}
 
-	phy_start(greth->phy);
+	phy_start(ndev->phydev);
 
 	/* If Ethernet debug link is used make autoneg happen right away */
 	if (greth->edcl && greth_edcl == 1) {
-		phy_start_aneg(greth->phy);
+		phy_start_aneg(ndev->phydev);
 		timeout = jiffies + 6*HZ;
-		while (!phy_aneg_done(greth->phy) && time_before(jiffies, timeout)) {
+		while (!phy_aneg_done(ndev->phydev) &&
+		       time_before(jiffies, timeout)) {
 		}
-		phy_read_status(greth->phy);
+		phy_read_status(ndev->phydev);
 		greth_link_change(greth->netdev);
 	}
 
@@ -1569,8 +1549,8 @@ static int greth_of_remove(struct platform_device *of_dev)
 
 	dma_free_coherent(&of_dev->dev, 1024, greth->tx_bd_base, greth->tx_bd_base_phys);
 
-	if (greth->phy)
-		phy_stop(greth->phy);
+	if (ndev->phydev)
+		phy_stop(ndev->phydev);
 	mdiobus_unregister(greth->mdio);
 
 	unregister_netdev(ndev);
diff --git a/drivers/net/ethernet/aeroflex/greth.h b/drivers/net/ethernet/aeroflex/greth.h
index 92dd918e4a83..9c07140a5d8d 100644
--- a/drivers/net/ethernet/aeroflex/greth.h
+++ b/drivers/net/ethernet/aeroflex/greth.h
@@ -123,7 +123,6 @@ struct greth_private {
 	struct napi_struct napi;
 	spinlock_t devlock;
 
-	struct phy_device *phy;
 	struct mii_bus *mdio;
 	unsigned int link;
 	unsigned int speed;
diff --git a/drivers/net/ethernet/agere/et131x.c b/drivers/net/ethernet/agere/et131x.c
index c83ebae73d91..906683851c7d 100644
--- a/drivers/net/ethernet/agere/et131x.c
+++ b/drivers/net/ethernet/agere/et131x.c
@@ -2961,7 +2961,7 @@ static void et131x_get_drvinfo(struct net_device *netdev,
 		sizeof(info->bus_info));
 }
 
-static struct ethtool_ops et131x_ethtool_ops = {
+static const struct ethtool_ops et131x_ethtool_ops = {
 	.get_drvinfo	= et131x_get_drvinfo,
 	.get_regs_len	= et131x_get_regs_len,
 	.get_regs	= et131x_get_regs,
diff --git a/drivers/net/ethernet/amazon/Kconfig b/drivers/net/ethernet/amazon/Kconfig
new file mode 100644
index 000000000000..99b30353541a
--- /dev/null
+++ b/drivers/net/ethernet/amazon/Kconfig
@@ -0,0 +1,27 @@
+#
+# Amazon network device configuration
+#
+
+config NET_VENDOR_AMAZON
+	bool "Amazon Devices"
+	default y
+	---help---
+	  If you have a network (Ethernet) device belonging to this class, say Y.
+
+	  Note that the answer to this question doesn't directly affect the
+	  kernel: saying N will just cause the configurator to skip all
+	  the questions about Amazon devices. If you say Y, you will be asked
+	  for your specific device in the following questions.
+
+if NET_VENDOR_AMAZON
+
+config ENA_ETHERNET
+	tristate "Elastic Network Adapter (ENA) support"
+	depends on (PCI_MSI && X86)
+	---help---
+	  This driver supports Elastic Network Adapter (ENA)"
+
+	  To compile this driver as a module, choose M here.
+	  The module will be called ena.
+
+endif #NET_VENDOR_AMAZON
diff --git a/drivers/net/ethernet/amazon/Makefile b/drivers/net/ethernet/amazon/Makefile
new file mode 100644
index 000000000000..8e0b73f60d51
--- /dev/null
+++ b/drivers/net/ethernet/amazon/Makefile
@@ -0,0 +1,5 @@
+#
+# Makefile for the Amazon network device drivers.
+#
+
+obj-$(CONFIG_ENA_ETHERNET) += ena/
diff --git a/drivers/net/ethernet/amazon/ena/Makefile b/drivers/net/ethernet/amazon/ena/Makefile
new file mode 100644
index 000000000000..eaeeae06c5d9
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for the Elastic Network Adapter (ENA) device drivers.
+#
+
+obj-$(CONFIG_ENA_ETHERNET) += ena.o
+
+ena-y := ena_netdev.o ena_com.o ena_eth_com.o ena_ethtool.o
diff --git a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h
new file mode 100644
index 000000000000..a46e749bf226
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h
@@ -0,0 +1,973 @@
+/*
+ * Copyright 2015 - 2016 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef _ENA_ADMIN_H_
+#define _ENA_ADMIN_H_
+
+enum ena_admin_aq_opcode {
+	ENA_ADMIN_CREATE_SQ	= 1,
+
+	ENA_ADMIN_DESTROY_SQ	= 2,
+
+	ENA_ADMIN_CREATE_CQ	= 3,
+
+	ENA_ADMIN_DESTROY_CQ	= 4,
+
+	ENA_ADMIN_GET_FEATURE	= 8,
+
+	ENA_ADMIN_SET_FEATURE	= 9,
+
+	ENA_ADMIN_GET_STATS	= 11,
+};
+
+enum ena_admin_aq_completion_status {
+	ENA_ADMIN_SUCCESS			= 0,
+
+	ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE	= 1,
+
+	ENA_ADMIN_BAD_OPCODE			= 2,
+
+	ENA_ADMIN_UNSUPPORTED_OPCODE		= 3,
+
+	ENA_ADMIN_MALFORMED_REQUEST		= 4,
+
+	/* Additional status is provided in ACQ entry extended_status */
+	ENA_ADMIN_ILLEGAL_PARAMETER		= 5,
+
+	ENA_ADMIN_UNKNOWN_ERROR			= 6,
+};
+
+enum ena_admin_aq_feature_id {
+	ENA_ADMIN_DEVICE_ATTRIBUTES		= 1,
+
+	ENA_ADMIN_MAX_QUEUES_NUM		= 2,
+
+	ENA_ADMIN_RSS_HASH_FUNCTION		= 10,
+
+	ENA_ADMIN_STATELESS_OFFLOAD_CONFIG	= 11,
+
+	ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG	= 12,
+
+	ENA_ADMIN_MTU				= 14,
+
+	ENA_ADMIN_RSS_HASH_INPUT		= 18,
+
+	ENA_ADMIN_INTERRUPT_MODERATION		= 20,
+
+	ENA_ADMIN_AENQ_CONFIG			= 26,
+
+	ENA_ADMIN_LINK_CONFIG			= 27,
+
+	ENA_ADMIN_HOST_ATTR_CONFIG		= 28,
+
+	ENA_ADMIN_FEATURES_OPCODE_NUM		= 32,
+};
+
+enum ena_admin_placement_policy_type {
+	/* descriptors and headers are in host memory */
+	ENA_ADMIN_PLACEMENT_POLICY_HOST	= 1,
+
+	/* descriptors and headers are in device memory (a.k.a Low Latency
+	 * Queue)
+	 */
+	ENA_ADMIN_PLACEMENT_POLICY_DEV	= 3,
+};
+
+enum ena_admin_link_types {
+	ENA_ADMIN_LINK_SPEED_1G		= 0x1,
+
+	ENA_ADMIN_LINK_SPEED_2_HALF_G	= 0x2,
+
+	ENA_ADMIN_LINK_SPEED_5G		= 0x4,
+
+	ENA_ADMIN_LINK_SPEED_10G	= 0x8,
+
+	ENA_ADMIN_LINK_SPEED_25G	= 0x10,
+
+	ENA_ADMIN_LINK_SPEED_40G	= 0x20,
+
+	ENA_ADMIN_LINK_SPEED_50G	= 0x40,
+
+	ENA_ADMIN_LINK_SPEED_100G	= 0x80,
+
+	ENA_ADMIN_LINK_SPEED_200G	= 0x100,
+
+	ENA_ADMIN_LINK_SPEED_400G	= 0x200,
+};
+
+enum ena_admin_completion_policy_type {
+	/* completion queue entry for each sq descriptor */
+	ENA_ADMIN_COMPLETION_POLICY_DESC		= 0,
+
+	/* completion queue entry upon request in sq descriptor */
+	ENA_ADMIN_COMPLETION_POLICY_DESC_ON_DEMAND	= 1,
+
+	/* current queue head pointer is updated in OS memory upon sq
+	 * descriptor request
+	 */
+	ENA_ADMIN_COMPLETION_POLICY_HEAD_ON_DEMAND	= 2,
+
+	/* current queue head pointer is updated in OS memory for each sq
+	 * descriptor
+	 */
+	ENA_ADMIN_COMPLETION_POLICY_HEAD		= 3,
+};
+
+/* basic stats return ena_admin_basic_stats while extanded stats return a
+ * buffer (string format) with additional statistics per queue and per
+ * device id
+ */
+enum ena_admin_get_stats_type {
+	ENA_ADMIN_GET_STATS_TYPE_BASIC		= 0,
+
+	ENA_ADMIN_GET_STATS_TYPE_EXTENDED	= 1,
+};
+
+enum ena_admin_get_stats_scope {
+	ENA_ADMIN_SPECIFIC_QUEUE	= 0,
+
+	ENA_ADMIN_ETH_TRAFFIC		= 1,
+};
+
+struct ena_admin_aq_common_desc {
+	/* 11:0 : command_id
+	 * 15:12 : reserved12
+	 */
+	u16 command_id;
+
+	/* as appears in ena_admin_aq_opcode */
+	u8 opcode;
+
+	/* 0 : phase
+	 * 1 : ctrl_data - control buffer address valid
+	 * 2 : ctrl_data_indirect - control buffer address
+	 *    points to list of pages with addresses of control
+	 *    buffers
+	 * 7:3 : reserved3
+	 */
+	u8 flags;
+};
+
+/* used in ena_admin_aq_entry. Can point directly to control data, or to a
+ * page list chunk. Used also at the end of indirect mode page list chunks,
+ * for chaining.
+ */
+struct ena_admin_ctrl_buff_info {
+	u32 length;
+
+	struct ena_common_mem_addr address;
+};
+
+struct ena_admin_sq {
+	u16 sq_idx;
+
+	/* 4:0 : reserved
+	 * 7:5 : sq_direction - 0x1 - Tx; 0x2 - Rx
+	 */
+	u8 sq_identity;
+
+	u8 reserved1;
+};
+
+struct ena_admin_aq_entry {
+	struct ena_admin_aq_common_desc aq_common_descriptor;
+
+	union {
+		u32 inline_data_w1[3];
+
+		struct ena_admin_ctrl_buff_info control_buffer;
+	} u;
+
+	u32 inline_data_w4[12];
+};
+
+struct ena_admin_acq_common_desc {
+	/* command identifier to associate it with the aq descriptor
+	 * 11:0 : command_id
+	 * 15:12 : reserved12
+	 */
+	u16 command;
+
+	u8 status;
+
+	/* 0 : phase
+	 * 7:1 : reserved1
+	 */
+	u8 flags;
+
+	u16 extended_status;
+
+	/* serves as a hint what AQ entries can be revoked */
+	u16 sq_head_indx;
+};
+
+struct ena_admin_acq_entry {
+	struct ena_admin_acq_common_desc acq_common_descriptor;
+
+	u32 response_specific_data[14];
+};
+
+struct ena_admin_aq_create_sq_cmd {
+	struct ena_admin_aq_common_desc aq_common_descriptor;
+
+	/* 4:0 : reserved0_w1
+	 * 7:5 : sq_direction - 0x1 - Tx, 0x2 - Rx
+	 */
+	u8 sq_identity;
+
+	u8 reserved8_w1;
+
+	/* 3:0 : placement_policy - Describing where the SQ
+	 *    descriptor ring and the SQ packet headers reside:
+	 *    0x1 - descriptors and headers are in OS memory,
+	 *    0x3 - descriptors and headers in device memory
+	 *    (a.k.a Low Latency Queue)
+	 * 6:4 : completion_policy - Describing what policy
+	 *    to use for generation completion entry (cqe) in
+	 *    the CQ associated with this SQ: 0x0 - cqe for each
+	 *    sq descriptor, 0x1 - cqe upon request in sq
+	 *    descriptor, 0x2 - current queue head pointer is
+	 *    updated in OS memory upon sq descriptor request
+	 *    0x3 - current queue head pointer is updated in OS
+	 *    memory for each sq descriptor
+	 * 7 : reserved15_w1
+	 */
+	u8 sq_caps_2;
+
+	/* 0 : is_physically_contiguous - Described if the
+	 *    queue ring memory is allocated in physical
+	 *    contiguous pages or split.
+	 * 7:1 : reserved17_w1
+	 */
+	u8 sq_caps_3;
+
+	/* associated completion queue id. This CQ must be created prior to
+	 *    SQ creation
+	 */
+	u16 cq_idx;
+
+	/* submission queue depth in entries */
+	u16 sq_depth;
+
+	/* SQ physical base address in OS memory. This field should not be
+	 * used for Low Latency queues. Has to be page aligned.
+	 */
+	struct ena_common_mem_addr sq_ba;
+
+	/* specifies queue head writeback location in OS memory. Valid if
+	 * completion_policy is set to completion_policy_head_on_demand or
+	 * completion_policy_head. Has to be cache aligned
+	 */
+	struct ena_common_mem_addr sq_head_writeback;
+
+	u32 reserved0_w7;
+
+	u32 reserved0_w8;
+};
+
+enum ena_admin_sq_direction {
+	ENA_ADMIN_SQ_DIRECTION_TX	= 1,
+
+	ENA_ADMIN_SQ_DIRECTION_RX	= 2,
+};
+
+struct ena_admin_acq_create_sq_resp_desc {
+	struct ena_admin_acq_common_desc acq_common_desc;
+
+	u16 sq_idx;
+
+	u16 reserved;
+
+	/* queue doorbell address as an offset to PCIe MMIO REG BAR */
+	u32 sq_doorbell_offset;
+
+	/* low latency queue ring base address as an offset to PCIe MMIO
+	 * LLQ_MEM BAR
+	 */
+	u32 llq_descriptors_offset;
+
+	/* low latency queue headers' memory as an offset to PCIe MMIO
+	 * LLQ_MEM BAR
+	 */
+	u32 llq_headers_offset;
+};
+
+struct ena_admin_aq_destroy_sq_cmd {
+	struct ena_admin_aq_common_desc aq_common_descriptor;
+
+	struct ena_admin_sq sq;
+};
+
+struct ena_admin_acq_destroy_sq_resp_desc {
+	struct ena_admin_acq_common_desc acq_common_desc;
+};
+
+struct ena_admin_aq_create_cq_cmd {
+	struct ena_admin_aq_common_desc aq_common_descriptor;
+
+	/* 4:0 : reserved5
+	 * 5 : interrupt_mode_enabled - if set, cq operates
+	 *    in interrupt mode, otherwise - polling
+	 * 7:6 : reserved6
+	 */
+	u8 cq_caps_1;
+
+	/* 4:0 : cq_entry_size_words - size of CQ entry in
+	 *    32-bit words, valid values: 4, 8.
+	 * 7:5 : reserved7
+	 */
+	u8 cq_caps_2;
+
+	/* completion queue depth in # of entries. must be power of 2 */
+	u16 cq_depth;
+
+	/* msix vector assigned to this cq */
+	u32 msix_vector;
+
+	/* cq physical base address in OS memory. CQ must be physically
+	 * contiguous
+	 */
+	struct ena_common_mem_addr cq_ba;
+};
+
+struct ena_admin_acq_create_cq_resp_desc {
+	struct ena_admin_acq_common_desc acq_common_desc;
+
+	u16 cq_idx;
+
+	/* actual cq depth in number of entries */
+	u16 cq_actual_depth;
+
+	u32 numa_node_register_offset;
+
+	u32 cq_head_db_register_offset;
+
+	u32 cq_interrupt_unmask_register_offset;
+};
+
+struct ena_admin_aq_destroy_cq_cmd {
+	struct ena_admin_aq_common_desc aq_common_descriptor;
+
+	u16 cq_idx;
+
+	u16 reserved1;
+};
+
+struct ena_admin_acq_destroy_cq_resp_desc {
+	struct ena_admin_acq_common_desc acq_common_desc;
+};
+
+/* ENA AQ Get Statistics command. Extended statistics are placed in control
+ * buffer pointed by AQ entry
+ */
+struct ena_admin_aq_get_stats_cmd {
+	struct ena_admin_aq_common_desc aq_common_descriptor;
+
+	union {
+		/* command specific inline data */
+		u32 inline_data_w1[3];
+
+		struct ena_admin_ctrl_buff_info control_buffer;
+	} u;
+
+	/* stats type as defined in enum ena_admin_get_stats_type */
+	u8 type;
+
+	/* stats scope defined in enum ena_admin_get_stats_scope */
+	u8 scope;
+
+	u16 reserved3;
+
+	/* queue id. used when scope is specific_queue */
+	u16 queue_idx;
+
+	/* device id, value 0xFFFF means mine. only privileged device can get
+	 *    stats of other device
+	 */
+	u16 device_id;
+};
+
+/* Basic Statistics Command. */
+struct ena_admin_basic_stats {
+	u32 tx_bytes_low;
+
+	u32 tx_bytes_high;
+
+	u32 tx_pkts_low;
+
+	u32 tx_pkts_high;
+
+	u32 rx_bytes_low;
+
+	u32 rx_bytes_high;
+
+	u32 rx_pkts_low;
+
+	u32 rx_pkts_high;
+
+	u32 rx_drops_low;
+
+	u32 rx_drops_high;
+};
+
+struct ena_admin_acq_get_stats_resp {
+	struct ena_admin_acq_common_desc acq_common_desc;
+
+	struct ena_admin_basic_stats basic_stats;
+};
+
+struct ena_admin_get_set_feature_common_desc {
+	/* 1:0 : select - 0x1 - current value; 0x3 - default
+	 *    value
+	 * 7:3 : reserved3
+	 */
+	u8 flags;
+
+	/* as appears in ena_admin_aq_feature_id */
+	u8 feature_id;
+
+	u16 reserved16;
+};
+
+struct ena_admin_device_attr_feature_desc {
+	u32 impl_id;
+
+	u32 device_version;
+
+	/* bitmap of ena_admin_aq_feature_id */
+	u32 supported_features;
+
+	u32 reserved3;
+
+	/* Indicates how many bits are used physical address access. */
+	u32 phys_addr_width;
+
+	/* Indicates how many bits are used virtual address access. */
+	u32 virt_addr_width;
+
+	/* unicast MAC address (in Network byte order) */
+	u8 mac_addr[6];
+
+	u8 reserved7[2];
+
+	u32 max_mtu;
+};
+
+struct ena_admin_queue_feature_desc {
+	/* including LLQs */
+	u32 max_sq_num;
+
+	u32 max_sq_depth;
+
+	u32 max_cq_num;
+
+	u32 max_cq_depth;
+
+	u32 max_llq_num;
+
+	u32 max_llq_depth;
+
+	u32 max_header_size;
+
+	/* Maximum Descriptors number, including meta descriptor, allowed for
+	 *    a single Tx packet
+	 */
+	u16 max_packet_tx_descs;
+
+	/* Maximum Descriptors number allowed for a single Rx packet */
+	u16 max_packet_rx_descs;
+};
+
+struct ena_admin_set_feature_mtu_desc {
+	/* exclude L2 */
+	u32 mtu;
+};
+
+struct ena_admin_set_feature_host_attr_desc {
+	/* host OS info base address in OS memory. host info is 4KB of
+	 * physically contiguous
+	 */
+	struct ena_common_mem_addr os_info_ba;
+
+	/* host debug area base address in OS memory. debug area must be
+	 * physically contiguous
+	 */
+	struct ena_common_mem_addr debug_ba;
+
+	/* debug area size */
+	u32 debug_area_size;
+};
+
+struct ena_admin_feature_intr_moder_desc {
+	/* interrupt delay granularity in usec */
+	u16 intr_delay_resolution;
+
+	u16 reserved;
+};
+
+struct ena_admin_get_feature_link_desc {
+	/* Link speed in Mb */
+	u32 speed;
+
+	/* bit field of enum ena_admin_link types */
+	u32 supported;
+
+	/* 0 : autoneg
+	 * 1 : duplex - Full Duplex
+	 * 31:2 : reserved2
+	 */
+	u32 flags;
+};
+
+struct ena_admin_feature_aenq_desc {
+	/* bitmask for AENQ groups the device can report */
+	u32 supported_groups;
+
+	/* bitmask for AENQ groups to report */
+	u32 enabled_groups;
+};
+
+struct ena_admin_feature_offload_desc {
+	/* 0 : TX_L3_csum_ipv4
+	 * 1 : TX_L4_ipv4_csum_part - The checksum field
+	 *    should be initialized with pseudo header checksum
+	 * 2 : TX_L4_ipv4_csum_full
+	 * 3 : TX_L4_ipv6_csum_part - The checksum field
+	 *    should be initialized with pseudo header checksum
+	 * 4 : TX_L4_ipv6_csum_full
+	 * 5 : tso_ipv4
+	 * 6 : tso_ipv6
+	 * 7 : tso_ecn
+	 */
+	u32 tx;
+
+	/* Receive side supported stateless offload
+	 * 0 : RX_L3_csum_ipv4 - IPv4 checksum
+	 * 1 : RX_L4_ipv4_csum - TCP/UDP/IPv4 checksum
+	 * 2 : RX_L4_ipv6_csum - TCP/UDP/IPv6 checksum
+	 * 3 : RX_hash - Hash calculation
+	 */
+	u32 rx_supported;
+
+	u32 rx_enabled;
+};
+
+enum ena_admin_hash_functions {
+	ENA_ADMIN_TOEPLITZ	= 1,
+
+	ENA_ADMIN_CRC32		= 2,
+};
+
+struct ena_admin_feature_rss_flow_hash_control {
+	u32 keys_num;
+
+	u32 reserved;
+
+	u32 key[10];
+};
+
+struct ena_admin_feature_rss_flow_hash_function {
+	/* 7:0 : funcs - bitmask of ena_admin_hash_functions */
+	u32 supported_func;
+
+	/* 7:0 : selected_func - bitmask of
+	 *    ena_admin_hash_functions
+	 */
+	u32 selected_func;
+
+	/* initial value */
+	u32 init_val;
+};
+
+/* RSS flow hash protocols */
+enum ena_admin_flow_hash_proto {
+	ENA_ADMIN_RSS_TCP4	= 0,
+
+	ENA_ADMIN_RSS_UDP4	= 1,
+
+	ENA_ADMIN_RSS_TCP6	= 2,
+
+	ENA_ADMIN_RSS_UDP6	= 3,
+
+	ENA_ADMIN_RSS_IP4	= 4,
+
+	ENA_ADMIN_RSS_IP6	= 5,
+
+	ENA_ADMIN_RSS_IP4_FRAG	= 6,
+
+	ENA_ADMIN_RSS_NOT_IP	= 7,
+
+	ENA_ADMIN_RSS_PROTO_NUM	= 16,
+};
+
+/* RSS flow hash fields */
+enum ena_admin_flow_hash_fields {
+	/* Ethernet Dest Addr */
+	ENA_ADMIN_RSS_L2_DA	= 0,
+
+	/* Ethernet Src Addr */
+	ENA_ADMIN_RSS_L2_SA	= 1,
+
+	/* ipv4/6 Dest Addr */
+	ENA_ADMIN_RSS_L3_DA	= 2,
+
+	/* ipv4/6 Src Addr */
+	ENA_ADMIN_RSS_L3_SA	= 5,
+
+	/* tcp/udp Dest Port */
+	ENA_ADMIN_RSS_L4_DP	= 6,
+
+	/* tcp/udp Src Port */
+	ENA_ADMIN_RSS_L4_SP	= 7,
+};
+
+struct ena_admin_proto_input {
+	/* flow hash fields (bitwise according to ena_admin_flow_hash_fields) */
+	u16 fields;
+
+	u16 reserved2;
+};
+
+struct ena_admin_feature_rss_hash_control {
+	struct ena_admin_proto_input supported_fields[ENA_ADMIN_RSS_PROTO_NUM];
+
+	struct ena_admin_proto_input selected_fields[ENA_ADMIN_RSS_PROTO_NUM];
+
+	struct ena_admin_proto_input reserved2[ENA_ADMIN_RSS_PROTO_NUM];
+
+	struct ena_admin_proto_input reserved3[ENA_ADMIN_RSS_PROTO_NUM];
+};
+
+struct ena_admin_feature_rss_flow_hash_input {
+	/* supported hash input sorting
+	 * 1 : L3_sort - support swap L3 addresses if DA is
+	 *    smaller than SA
+	 * 2 : L4_sort - support swap L4 ports if DP smaller
+	 *    SP
+	 */
+	u16 supported_input_sort;
+
+	/* enabled hash input sorting
+	 * 1 : enable_L3_sort - enable swap L3 addresses if
+	 *    DA smaller than SA
+	 * 2 : enable_L4_sort - enable swap L4 ports if DP
+	 *    smaller than SP
+	 */
+	u16 enabled_input_sort;
+};
+
+enum ena_admin_os_type {
+	ENA_ADMIN_OS_LINUX	= 1,
+
+	ENA_ADMIN_OS_WIN	= 2,
+
+	ENA_ADMIN_OS_DPDK	= 3,
+
+	ENA_ADMIN_OS_FREEBSD	= 4,
+
+	ENA_ADMIN_OS_IPXE	= 5,
+};
+
+struct ena_admin_host_info {
+	/* defined in enum ena_admin_os_type */
+	u32 os_type;
+
+	/* os distribution string format */
+	u8 os_dist_str[128];
+
+	/* OS distribution numeric format */
+	u32 os_dist;
+
+	/* kernel version string format */
+	u8 kernel_ver_str[32];
+
+	/* Kernel version numeric format */
+	u32 kernel_ver;
+
+	/* 7:0 : major
+	 * 15:8 : minor
+	 * 23:16 : sub_minor
+	 */
+	u32 driver_version;
+
+	/* features bitmap */
+	u32 supported_network_features[4];
+};
+
+struct ena_admin_rss_ind_table_entry {
+	u16 cq_idx;
+
+	u16 reserved;
+};
+
+struct ena_admin_feature_rss_ind_table {
+	/* min supported table size (2^min_size) */
+	u16 min_size;
+
+	/* max supported table size (2^max_size) */
+	u16 max_size;
+
+	/* table size (2^size) */
+	u16 size;
+
+	u16 reserved;
+
+	/* index of the inline entry. 0xFFFFFFFF means invalid */
+	u32 inline_index;
+
+	/* used for updating single entry, ignored when setting the entire
+	 * table through the control buffer.
+	 */
+	struct ena_admin_rss_ind_table_entry inline_entry;
+};
+
+struct ena_admin_get_feat_cmd {
+	struct ena_admin_aq_common_desc aq_common_descriptor;
+
+	struct ena_admin_ctrl_buff_info control_buffer;
+
+	struct ena_admin_get_set_feature_common_desc feat_common;
+
+	u32 raw[11];
+};
+
+struct ena_admin_get_feat_resp {
+	struct ena_admin_acq_common_desc acq_common_desc;
+
+	union {
+		u32 raw[14];
+
+		struct ena_admin_device_attr_feature_desc dev_attr;
+
+		struct ena_admin_queue_feature_desc max_queue;
+
+		struct ena_admin_feature_aenq_desc aenq;
+
+		struct ena_admin_get_feature_link_desc link;
+
+		struct ena_admin_feature_offload_desc offload;
+
+		struct ena_admin_feature_rss_flow_hash_function flow_hash_func;
+
+		struct ena_admin_feature_rss_flow_hash_input flow_hash_input;
+
+		struct ena_admin_feature_rss_ind_table ind_table;
+
+		struct ena_admin_feature_intr_moder_desc intr_moderation;
+	} u;
+};
+
+struct ena_admin_set_feat_cmd {
+	struct ena_admin_aq_common_desc aq_common_descriptor;
+
+	struct ena_admin_ctrl_buff_info control_buffer;
+
+	struct ena_admin_get_set_feature_common_desc feat_common;
+
+	union {
+		u32 raw[11];
+
+		/* mtu size */
+		struct ena_admin_set_feature_mtu_desc mtu;
+
+		/* host attributes */
+		struct ena_admin_set_feature_host_attr_desc host_attr;
+
+		/* AENQ configuration */
+		struct ena_admin_feature_aenq_desc aenq;
+
+		/* rss flow hash function */
+		struct ena_admin_feature_rss_flow_hash_function flow_hash_func;
+
+		/* rss flow hash input */
+		struct ena_admin_feature_rss_flow_hash_input flow_hash_input;
+
+		/* rss indirection table */
+		struct ena_admin_feature_rss_ind_table ind_table;
+	} u;
+};
+
+struct ena_admin_set_feat_resp {
+	struct ena_admin_acq_common_desc acq_common_desc;
+
+	union {
+		u32 raw[14];
+	} u;
+};
+
+struct ena_admin_aenq_common_desc {
+	u16 group;
+
+	u16 syndrom;
+
+	/* 0 : phase */
+	u8 flags;
+
+	u8 reserved1[3];
+
+	u32 timestamp_low;
+
+	u32 timestamp_high;
+};
+
+/* asynchronous event notification groups */
+enum ena_admin_aenq_group {
+	ENA_ADMIN_LINK_CHANGE		= 0,
+
+	ENA_ADMIN_FATAL_ERROR		= 1,
+
+	ENA_ADMIN_WARNING		= 2,
+
+	ENA_ADMIN_NOTIFICATION		= 3,
+
+	ENA_ADMIN_KEEP_ALIVE		= 4,
+
+	ENA_ADMIN_AENQ_GROUPS_NUM	= 5,
+};
+
+enum ena_admin_aenq_notification_syndrom {
+	ENA_ADMIN_SUSPEND	= 0,
+
+	ENA_ADMIN_RESUME	= 1,
+};
+
+struct ena_admin_aenq_entry {
+	struct ena_admin_aenq_common_desc aenq_common_desc;
+
+	/* command specific inline data */
+	u32 inline_data_w4[12];
+};
+
+struct ena_admin_aenq_link_change_desc {
+	struct ena_admin_aenq_common_desc aenq_common_desc;
+
+	/* 0 : link_status */
+	u32 flags;
+};
+
+struct ena_admin_ena_mmio_req_read_less_resp {
+	u16 req_id;
+
+	u16 reg_off;
+
+	/* value is valid when poll is cleared */
+	u32 reg_val;
+};
+
+/* aq_common_desc */
+#define ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0)
+#define ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK BIT(0)
+#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT 1
+#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK BIT(1)
+#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT 2
+#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK BIT(2)
+
+/* sq */
+#define ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT 5
+#define ENA_ADMIN_SQ_SQ_DIRECTION_MASK GENMASK(7, 5)
+
+/* acq_common_desc */
+#define ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0)
+#define ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK BIT(0)
+
+/* aq_create_sq_cmd */
+#define ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT 5
+#define ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK GENMASK(7, 5)
+#define ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK GENMASK(3, 0)
+#define ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT 4
+#define ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK GENMASK(6, 4)
+#define ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK BIT(0)
+
+/* aq_create_cq_cmd */
+#define ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT 5
+#define ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK BIT(5)
+#define ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0)
+
+/* get_set_feature_common_desc */
+#define ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK GENMASK(1, 0)
+
+/* get_feature_link_desc */
+#define ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK BIT(0)
+#define ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT 1
+#define ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK BIT(1)
+
+/* feature_offload_desc */
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK BIT(0)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT 1
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK BIT(1)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT 2
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK BIT(2)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT 3
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK BIT(3)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT 4
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK BIT(4)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT 5
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK BIT(5)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT 6
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK BIT(6)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT 7
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK BIT(7)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK BIT(0)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT 1
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK BIT(1)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT 2
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK BIT(2)
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT 3
+#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK BIT(3)
+
+/* feature_rss_flow_hash_function */
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK GENMASK(7, 0)
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK GENMASK(7, 0)
+
+/* feature_rss_flow_hash_input */
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT 1
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK BIT(1)
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT 2
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK BIT(2)
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT 1
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK BIT(1)
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT 2
+#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK BIT(2)
+
+/* host_info */
+#define ENA_ADMIN_HOST_INFO_MAJOR_MASK GENMASK(7, 0)
+#define ENA_ADMIN_HOST_INFO_MINOR_SHIFT 8
+#define ENA_ADMIN_HOST_INFO_MINOR_MASK GENMASK(15, 8)
+#define ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT 16
+#define ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK GENMASK(23, 16)
+
+/* aenq_common_desc */
+#define ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK BIT(0)
+
+/* aenq_link_change_desc */
+#define ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK BIT(0)
+
+#endif /*_ENA_ADMIN_H_ */
diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c
new file mode 100644
index 000000000000..3066d9c99984
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_com.c
@@ -0,0 +1,2666 @@
+/*
+ * Copyright 2015 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "ena_com.h"
+
+/*****************************************************************************/
+/*****************************************************************************/
+
+/* Timeout in micro-sec */
+#define ADMIN_CMD_TIMEOUT_US (1000000)
+
+#define ENA_ASYNC_QUEUE_DEPTH 4
+#define ENA_ADMIN_QUEUE_DEPTH 32
+
+#define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \
+		ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \
+		| (ENA_COMMON_SPEC_VERSION_MINOR))
+
+#define ENA_CTRL_MAJOR		0
+#define ENA_CTRL_MINOR		0
+#define ENA_CTRL_SUB_MINOR	1
+
+#define MIN_ENA_CTRL_VER \
+	(((ENA_CTRL_MAJOR) << \
+	(ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \
+	((ENA_CTRL_MINOR) << \
+	(ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \
+	(ENA_CTRL_SUB_MINOR))
+
+#define ENA_DMA_ADDR_TO_UINT32_LOW(x)	((u32)((u64)(x)))
+#define ENA_DMA_ADDR_TO_UINT32_HIGH(x)	((u32)(((u64)(x)) >> 32))
+
+#define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF
+
+/*****************************************************************************/
+/*****************************************************************************/
+/*****************************************************************************/
+
+enum ena_cmd_status {
+	ENA_CMD_SUBMITTED,
+	ENA_CMD_COMPLETED,
+	/* Abort - canceled by the driver */
+	ENA_CMD_ABORTED,
+};
+
+struct ena_comp_ctx {
+	struct completion wait_event;
+	struct ena_admin_acq_entry *user_cqe;
+	u32 comp_size;
+	enum ena_cmd_status status;
+	/* status from the device */
+	u8 comp_status;
+	u8 cmd_opcode;
+	bool occupied;
+};
+
+struct ena_com_stats_ctx {
+	struct ena_admin_aq_get_stats_cmd get_cmd;
+	struct ena_admin_acq_get_stats_resp get_resp;
+};
+
+static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
+				       struct ena_common_mem_addr *ena_addr,
+				       dma_addr_t addr)
+{
+	if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {
+		pr_err("dma address has more bits that the device supports\n");
+		return -EINVAL;
+	}
+
+	ena_addr->mem_addr_low = (u32)addr;
+	ena_addr->mem_addr_high = (u64)addr >> 32;
+
+	return 0;
+}
+
+static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue)
+{
+	struct ena_com_admin_sq *sq = &queue->sq;
+	u16 size = ADMIN_SQ_SIZE(queue->q_depth);
+
+	sq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &sq->dma_addr,
+					  GFP_KERNEL);
+
+	if (!sq->entries) {
+		pr_err("memory allocation failed");
+		return -ENOMEM;
+	}
+
+	sq->head = 0;
+	sq->tail = 0;
+	sq->phase = 1;
+
+	sq->db_addr = NULL;
+
+	return 0;
+}
+
+static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
+{
+	struct ena_com_admin_cq *cq = &queue->cq;
+	u16 size = ADMIN_CQ_SIZE(queue->q_depth);
+
+	cq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &cq->dma_addr,
+					  GFP_KERNEL);
+
+	if (!cq->entries) {
+		pr_err("memory allocation failed");
+		return -ENOMEM;
+	}
+
+	cq->head = 0;
+	cq->phase = 1;
+
+	return 0;
+}
+
+static int ena_com_admin_init_aenq(struct ena_com_dev *dev,
+				   struct ena_aenq_handlers *aenq_handlers)
+{
+	struct ena_com_aenq *aenq = &dev->aenq;
+	u32 addr_low, addr_high, aenq_caps;
+	u16 size;
+
+	dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
+	size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH);
+	aenq->entries = dma_zalloc_coherent(dev->dmadev, size, &aenq->dma_addr,
+					    GFP_KERNEL);
+
+	if (!aenq->entries) {
+		pr_err("memory allocation failed");
+		return -ENOMEM;
+	}
+
+	aenq->head = aenq->q_depth;
+	aenq->phase = 1;
+
+	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr);
+	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr);
+
+	writel(addr_low, dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF);
+	writel(addr_high, dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF);
+
+	aenq_caps = 0;
+	aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
+	aenq_caps |= (sizeof(struct ena_admin_aenq_entry)
+		      << ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
+		     ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
+	writel(aenq_caps, dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF);
+
+	if (unlikely(!aenq_handlers)) {
+		pr_err("aenq handlers pointer is NULL\n");
+		return -EINVAL;
+	}
+
+	aenq->aenq_handlers = aenq_handlers;
+
+	return 0;
+}
+
+static inline void comp_ctxt_release(struct ena_com_admin_queue *queue,
+				     struct ena_comp_ctx *comp_ctx)
+{
+	comp_ctx->occupied = false;
+	atomic_dec(&queue->outstanding_cmds);
+}
+
+static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue,
+					  u16 command_id, bool capture)
+{
+	if (unlikely(command_id >= queue->q_depth)) {
+		pr_err("command id is larger than the queue size. cmd_id: %u queue size %d\n",
+		       command_id, queue->q_depth);
+		return NULL;
+	}
+
+	if (unlikely(queue->comp_ctx[command_id].occupied && capture)) {
+		pr_err("Completion context is occupied\n");
+		return NULL;
+	}
+
+	if (capture) {
+		atomic_inc(&queue->outstanding_cmds);
+		queue->comp_ctx[command_id].occupied = true;
+	}
+
+	return &queue->comp_ctx[command_id];
+}
+
+static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
+						       struct ena_admin_aq_entry *cmd,
+						       size_t cmd_size_in_bytes,
+						       struct ena_admin_acq_entry *comp,
+						       size_t comp_size_in_bytes)
+{
+	struct ena_comp_ctx *comp_ctx;
+	u16 tail_masked, cmd_id;
+	u16 queue_size_mask;
+	u16 cnt;
+
+	queue_size_mask = admin_queue->q_depth - 1;
+
+	tail_masked = admin_queue->sq.tail & queue_size_mask;
+
+	/* In case of queue FULL */
+	cnt = admin_queue->sq.tail - admin_queue->sq.head;
+	if (cnt >= admin_queue->q_depth) {
+		pr_debug("admin queue is FULL (tail %d head %d depth: %d)\n",
+			 admin_queue->sq.tail, admin_queue->sq.head,
+			 admin_queue->q_depth);
+		admin_queue->stats.out_of_space++;
+		return ERR_PTR(-ENOSPC);
+	}
+
+	cmd_id = admin_queue->curr_cmd_id;
+
+	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &
+		ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
+
+	cmd->aq_common_descriptor.command_id |= cmd_id &
+		ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
+
+	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);
+	if (unlikely(!comp_ctx))
+		return ERR_PTR(-EINVAL);
+
+	comp_ctx->status = ENA_CMD_SUBMITTED;
+	comp_ctx->comp_size = (u32)comp_size_in_bytes;
+	comp_ctx->user_cqe = comp;
+	comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
+
+	reinit_completion(&comp_ctx->wait_event);
+
+	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);
+
+	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &
+		queue_size_mask;
+
+	admin_queue->sq.tail++;
+	admin_queue->stats.submitted_cmd++;
+
+	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))
+		admin_queue->sq.phase = !admin_queue->sq.phase;
+
+	writel(admin_queue->sq.tail, admin_queue->sq.db_addr);
+
+	return comp_ctx;
+}
+
+static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
+{
+	size_t size = queue->q_depth * sizeof(struct ena_comp_ctx);
+	struct ena_comp_ctx *comp_ctx;
+	u16 i;
+
+	queue->comp_ctx = devm_kzalloc(queue->q_dmadev, size, GFP_KERNEL);
+	if (unlikely(!queue->comp_ctx)) {
+		pr_err("memory allocation failed");
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < queue->q_depth; i++) {
+		comp_ctx = get_comp_ctxt(queue, i, false);
+		if (comp_ctx)
+			init_completion(&comp_ctx->wait_event);
+	}
+
+	return 0;
+}
+
+static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
+						     struct ena_admin_aq_entry *cmd,
+						     size_t cmd_size_in_bytes,
+						     struct ena_admin_acq_entry *comp,
+						     size_t comp_size_in_bytes)
+{
+	unsigned long flags;
+	struct ena_comp_ctx *comp_ctx;
+
+	spin_lock_irqsave(&admin_queue->q_lock, flags);
+	if (unlikely(!admin_queue->running_state)) {
+		spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+		return ERR_PTR(-ENODEV);
+	}
+	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,
+					      cmd_size_in_bytes,
+					      comp,
+					      comp_size_in_bytes);
+	if (unlikely(IS_ERR(comp_ctx)))
+		admin_queue->running_state = false;
+	spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+
+	return comp_ctx;
+}
+
+static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
+			      struct ena_com_create_io_ctx *ctx,
+			      struct ena_com_io_sq *io_sq)
+{
+	size_t size;
+	int dev_node = 0;
+
+	memset(&io_sq->desc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
+
+	io_sq->desc_entry_size =
+		(io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
+		sizeof(struct ena_eth_io_tx_desc) :
+		sizeof(struct ena_eth_io_rx_desc);
+
+	size = io_sq->desc_entry_size * io_sq->q_depth;
+
+	if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
+		dev_node = dev_to_node(ena_dev->dmadev);
+		set_dev_node(ena_dev->dmadev, ctx->numa_node);
+		io_sq->desc_addr.virt_addr =
+			dma_zalloc_coherent(ena_dev->dmadev, size,
+					    &io_sq->desc_addr.phys_addr,
+					    GFP_KERNEL);
+		set_dev_node(ena_dev->dmadev, dev_node);
+		if (!io_sq->desc_addr.virt_addr) {
+			io_sq->desc_addr.virt_addr =
+				dma_zalloc_coherent(ena_dev->dmadev, size,
+						    &io_sq->desc_addr.phys_addr,
+						    GFP_KERNEL);
+		}
+	} else {
+		dev_node = dev_to_node(ena_dev->dmadev);
+		set_dev_node(ena_dev->dmadev, ctx->numa_node);
+		io_sq->desc_addr.virt_addr =
+			devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
+		set_dev_node(ena_dev->dmadev, dev_node);
+		if (!io_sq->desc_addr.virt_addr) {
+			io_sq->desc_addr.virt_addr =
+				devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
+		}
+	}
+
+	if (!io_sq->desc_addr.virt_addr) {
+		pr_err("memory allocation failed");
+		return -ENOMEM;
+	}
+
+	io_sq->tail = 0;
+	io_sq->next_to_comp = 0;
+	io_sq->phase = 1;
+
+	return 0;
+}
+
+static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
+			      struct ena_com_create_io_ctx *ctx,
+			      struct ena_com_io_cq *io_cq)
+{
+	size_t size;
+	int prev_node = 0;
+
+	memset(&io_cq->cdesc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
+
+	/* Use the basic completion descriptor for Rx */
+	io_cq->cdesc_entry_size_in_bytes =
+		(io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
+		sizeof(struct ena_eth_io_tx_cdesc) :
+		sizeof(struct ena_eth_io_rx_cdesc_base);
+
+	size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
+
+	prev_node = dev_to_node(ena_dev->dmadev);
+	set_dev_node(ena_dev->dmadev, ctx->numa_node);
+	io_cq->cdesc_addr.virt_addr =
+		dma_zalloc_coherent(ena_dev->dmadev, size,
+				    &io_cq->cdesc_addr.phys_addr, GFP_KERNEL);
+	set_dev_node(ena_dev->dmadev, prev_node);
+	if (!io_cq->cdesc_addr.virt_addr) {
+		io_cq->cdesc_addr.virt_addr =
+			dma_zalloc_coherent(ena_dev->dmadev, size,
+					    &io_cq->cdesc_addr.phys_addr,
+					    GFP_KERNEL);
+	}
+
+	if (!io_cq->cdesc_addr.virt_addr) {
+		pr_err("memory allocation failed");
+		return -ENOMEM;
+	}
+
+	io_cq->phase = 1;
+	io_cq->head = 0;
+
+	return 0;
+}
+
+static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
+						   struct ena_admin_acq_entry *cqe)
+{
+	struct ena_comp_ctx *comp_ctx;
+	u16 cmd_id;
+
+	cmd_id = cqe->acq_common_descriptor.command &
+		ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
+
+	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);
+	if (unlikely(!comp_ctx)) {
+		pr_err("comp_ctx is NULL. Changing the admin queue running state\n");
+		admin_queue->running_state = false;
+		return;
+	}
+
+	comp_ctx->status = ENA_CMD_COMPLETED;
+	comp_ctx->comp_status = cqe->acq_common_descriptor.status;
+
+	if (comp_ctx->user_cqe)
+		memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size);
+
+	if (!admin_queue->polling)
+		complete(&comp_ctx->wait_event);
+}
+
+static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)
+{
+	struct ena_admin_acq_entry *cqe = NULL;
+	u16 comp_num = 0;
+	u16 head_masked;
+	u8 phase;
+
+	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);
+	phase = admin_queue->cq.phase;
+
+	cqe = &admin_queue->cq.entries[head_masked];
+
+	/* Go over all the completions */
+	while ((cqe->acq_common_descriptor.flags &
+			ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
+		/* Do not read the rest of the completion entry before the
+		 * phase bit was validated
+		 */
+		rmb();
+		ena_com_handle_single_admin_completion(admin_queue, cqe);
+
+		head_masked++;
+		comp_num++;
+		if (unlikely(head_masked == admin_queue->q_depth)) {
+			head_masked = 0;
+			phase = !phase;
+		}
+
+		cqe = &admin_queue->cq.entries[head_masked];
+	}
+
+	admin_queue->cq.head += comp_num;
+	admin_queue->cq.phase = phase;
+	admin_queue->sq.head += comp_num;
+	admin_queue->stats.completed_cmd += comp_num;
+}
+
+static int ena_com_comp_status_to_errno(u8 comp_status)
+{
+	if (unlikely(comp_status != 0))
+		pr_err("admin command failed[%u]\n", comp_status);
+
+	if (unlikely(comp_status > ENA_ADMIN_UNKNOWN_ERROR))
+		return -EINVAL;
+
+	switch (comp_status) {
+	case ENA_ADMIN_SUCCESS:
+		return 0;
+	case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
+		return -ENOMEM;
+	case ENA_ADMIN_UNSUPPORTED_OPCODE:
+		return -EPERM;
+	case ENA_ADMIN_BAD_OPCODE:
+	case ENA_ADMIN_MALFORMED_REQUEST:
+	case ENA_ADMIN_ILLEGAL_PARAMETER:
+	case ENA_ADMIN_UNKNOWN_ERROR:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx,
+						     struct ena_com_admin_queue *admin_queue)
+{
+	unsigned long flags;
+	u32 start_time;
+	int ret;
+
+	start_time = ((u32)jiffies_to_usecs(jiffies));
+
+	while (comp_ctx->status == ENA_CMD_SUBMITTED) {
+		if ((((u32)jiffies_to_usecs(jiffies)) - start_time) >
+		    ADMIN_CMD_TIMEOUT_US) {
+			pr_err("Wait for completion (polling) timeout\n");
+			/* ENA didn't have any completion */
+			spin_lock_irqsave(&admin_queue->q_lock, flags);
+			admin_queue->stats.no_completion++;
+			admin_queue->running_state = false;
+			spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+
+			ret = -ETIME;
+			goto err;
+		}
+
+		spin_lock_irqsave(&admin_queue->q_lock, flags);
+		ena_com_handle_admin_completion(admin_queue);
+		spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+
+		msleep(100);
+	}
+
+	if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
+		pr_err("Command was aborted\n");
+		spin_lock_irqsave(&admin_queue->q_lock, flags);
+		admin_queue->stats.aborted_cmd++;
+		spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+		ret = -ENODEV;
+		goto err;
+	}
+
+	WARN(comp_ctx->status != ENA_CMD_COMPLETED, "Invalid comp status %d\n",
+	     comp_ctx->status);
+
+	ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
+err:
+	comp_ctxt_release(admin_queue, comp_ctx);
+	return ret;
+}
+
+static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
+							struct ena_com_admin_queue *admin_queue)
+{
+	unsigned long flags;
+	int ret;
+
+	wait_for_completion_timeout(&comp_ctx->wait_event,
+				    usecs_to_jiffies(ADMIN_CMD_TIMEOUT_US));
+
+	/* In case the command wasn't completed find out the root cause.
+	 * There might be 2 kinds of errors
+	 * 1) No completion (timeout reached)
+	 * 2) There is completion but the device didn't get any msi-x interrupt.
+	 */
+	if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) {
+		spin_lock_irqsave(&admin_queue->q_lock, flags);
+		ena_com_handle_admin_completion(admin_queue);
+		admin_queue->stats.no_completion++;
+		spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+
+		if (comp_ctx->status == ENA_CMD_COMPLETED)
+			pr_err("The ena device have completion but the driver didn't receive any MSI-X interrupt (cmd %d)\n",
+			       comp_ctx->cmd_opcode);
+		else
+			pr_err("The ena device doesn't send any completion for the admin cmd %d status %d\n",
+			       comp_ctx->cmd_opcode, comp_ctx->status);
+
+		admin_queue->running_state = false;
+		ret = -ETIME;
+		goto err;
+	}
+
+	ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
+err:
+	comp_ctxt_release(admin_queue, comp_ctx);
+	return ret;
+}
+
+/* This method read the hardware device register through posting writes
+ * and waiting for response
+ * On timeout the function will return ENA_MMIO_READ_TIMEOUT
+ */
+static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
+{
+	struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
+	volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp =
+		mmio_read->read_resp;
+	u32 mmio_read_reg, ret;
+	unsigned long flags;
+	int i;
+
+	might_sleep();
+
+	/* If readless is disabled, perform regular read */
+	if (!mmio_read->readless_supported)
+		return readl(ena_dev->reg_bar + offset);
+
+	spin_lock_irqsave(&mmio_read->lock, flags);
+	mmio_read->seq_num++;
+
+	read_resp->req_id = mmio_read->seq_num + 0xDEAD;
+	mmio_read_reg = (offset << ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &
+			ENA_REGS_MMIO_REG_READ_REG_OFF_MASK;
+	mmio_read_reg |= mmio_read->seq_num &
+			ENA_REGS_MMIO_REG_READ_REQ_ID_MASK;
+
+	/* make sure read_resp->req_id get updated before the hw can write
+	 * there
+	 */
+	wmb();
+
+	writel(mmio_read_reg, ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF);
+
+	for (i = 0; i < ENA_REG_READ_TIMEOUT; i++) {
+		if (read_resp->req_id == mmio_read->seq_num)
+			break;
+
+		udelay(1);
+	}
+
+	if (unlikely(i == ENA_REG_READ_TIMEOUT)) {
+		pr_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n",
+		       mmio_read->seq_num, offset, read_resp->req_id,
+		       read_resp->reg_off);
+		ret = ENA_MMIO_READ_TIMEOUT;
+		goto err;
+	}
+
+	if (read_resp->reg_off != offset) {
+		pr_err("Read failure: wrong offset provided");
+		ret = ENA_MMIO_READ_TIMEOUT;
+	} else {
+		ret = read_resp->reg_val;
+	}
+err:
+	spin_unlock_irqrestore(&mmio_read->lock, flags);
+
+	return ret;
+}
+
+/* There are two types to wait for completion.
+ * Polling mode - wait until the completion is available.
+ * Async mode - wait on wait queue until the completion is ready
+ * (or the timeout expired).
+ * It is expected that the IRQ called ena_com_handle_admin_completion
+ * to mark the completions.
+ */
+static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
+					     struct ena_com_admin_queue *admin_queue)
+{
+	if (admin_queue->polling)
+		return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
+								 admin_queue);
+
+	return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx,
+							    admin_queue);
+}
+
+static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
+				 struct ena_com_io_sq *io_sq)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_admin_aq_destroy_sq_cmd destroy_cmd;
+	struct ena_admin_acq_destroy_sq_resp_desc destroy_resp;
+	u8 direction;
+	int ret;
+
+	memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
+
+	if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
+		direction = ENA_ADMIN_SQ_DIRECTION_TX;
+	else
+		direction = ENA_ADMIN_SQ_DIRECTION_RX;
+
+	destroy_cmd.sq.sq_identity |= (direction <<
+		ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) &
+		ENA_ADMIN_SQ_SQ_DIRECTION_MASK;
+
+	destroy_cmd.sq.sq_idx = io_sq->idx;
+	destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ;
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&destroy_cmd,
+					    sizeof(destroy_cmd),
+					    (struct ena_admin_acq_entry *)&destroy_resp,
+					    sizeof(destroy_resp));
+
+	if (unlikely(ret && (ret != -ENODEV)))
+		pr_err("failed to destroy io sq error: %d\n", ret);
+
+	return ret;
+}
+
+static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
+				  struct ena_com_io_sq *io_sq,
+				  struct ena_com_io_cq *io_cq)
+{
+	size_t size;
+
+	if (io_cq->cdesc_addr.virt_addr) {
+		size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
+
+		dma_free_coherent(ena_dev->dmadev, size,
+				  io_cq->cdesc_addr.virt_addr,
+				  io_cq->cdesc_addr.phys_addr);
+
+		io_cq->cdesc_addr.virt_addr = NULL;
+	}
+
+	if (io_sq->desc_addr.virt_addr) {
+		size = io_sq->desc_entry_size * io_sq->q_depth;
+
+		if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
+			dma_free_coherent(ena_dev->dmadev, size,
+					  io_sq->desc_addr.virt_addr,
+					  io_sq->desc_addr.phys_addr);
+		else
+			devm_kfree(ena_dev->dmadev, io_sq->desc_addr.virt_addr);
+
+		io_sq->desc_addr.virt_addr = NULL;
+	}
+}
+
+static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout,
+				u16 exp_state)
+{
+	u32 val, i;
+
+	for (i = 0; i < timeout; i++) {
+		val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
+
+		if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) {
+			pr_err("Reg read timeout occurred\n");
+			return -ETIME;
+		}
+
+		if ((val & ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==
+			exp_state)
+			return 0;
+
+		/* The resolution of the timeout is 100ms */
+		msleep(100);
+	}
+
+	return -ETIME;
+}
+
+static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev,
+					       enum ena_admin_aq_feature_id feature_id)
+{
+	u32 feature_mask = 1 << feature_id;
+
+	/* Device attributes is always supported */
+	if ((feature_id != ENA_ADMIN_DEVICE_ATTRIBUTES) &&
+	    !(ena_dev->supported_features & feature_mask))
+		return false;
+
+	return true;
+}
+
+static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
+				  struct ena_admin_get_feat_resp *get_resp,
+				  enum ena_admin_aq_feature_id feature_id,
+				  dma_addr_t control_buf_dma_addr,
+				  u32 control_buff_size)
+{
+	struct ena_com_admin_queue *admin_queue;
+	struct ena_admin_get_feat_cmd get_cmd;
+	int ret;
+
+	if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
+		pr_info("Feature %d isn't supported\n", feature_id);
+		return -EPERM;
+	}
+
+	memset(&get_cmd, 0x0, sizeof(get_cmd));
+	admin_queue = &ena_dev->admin_queue;
+
+	get_cmd.aq_common_descriptor.opcode = ENA_ADMIN_GET_FEATURE;
+
+	if (control_buff_size)
+		get_cmd.aq_common_descriptor.flags =
+			ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
+	else
+		get_cmd.aq_common_descriptor.flags = 0;
+
+	ret = ena_com_mem_addr_set(ena_dev,
+				   &get_cmd.control_buffer.address,
+				   control_buf_dma_addr);
+	if (unlikely(ret)) {
+		pr_err("memory address set failed\n");
+		return ret;
+	}
+
+	get_cmd.control_buffer.length = control_buff_size;
+
+	get_cmd.feat_common.feature_id = feature_id;
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)
+					    &get_cmd,
+					    sizeof(get_cmd),
+					    (struct ena_admin_acq_entry *)
+					    get_resp,
+					    sizeof(*get_resp));
+
+	if (unlikely(ret))
+		pr_err("Failed to submit get_feature command %d error: %d\n",
+		       feature_id, ret);
+
+	return ret;
+}
+
+static int ena_com_get_feature(struct ena_com_dev *ena_dev,
+			       struct ena_admin_get_feat_resp *get_resp,
+			       enum ena_admin_aq_feature_id feature_id)
+{
+	return ena_com_get_feature_ex(ena_dev,
+				      get_resp,
+				      feature_id,
+				      0,
+				      0);
+}
+
+static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+
+	rss->hash_key =
+		dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
+				    &rss->hash_key_dma_addr, GFP_KERNEL);
+
+	if (unlikely(!rss->hash_key))
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void ena_com_hash_key_destroy(struct ena_com_dev *ena_dev)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+
+	if (rss->hash_key)
+		dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
+				  rss->hash_key, rss->hash_key_dma_addr);
+	rss->hash_key = NULL;
+}
+
+static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+
+	rss->hash_ctrl =
+		dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
+				    &rss->hash_ctrl_dma_addr, GFP_KERNEL);
+
+	if (unlikely(!rss->hash_ctrl))
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+
+	if (rss->hash_ctrl)
+		dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
+				  rss->hash_ctrl, rss->hash_ctrl_dma_addr);
+	rss->hash_ctrl = NULL;
+}
+
+static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev,
+					   u16 log_size)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_get_feat_resp get_resp;
+	size_t tbl_size;
+	int ret;
+
+	ret = ena_com_get_feature(ena_dev, &get_resp,
+				  ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
+	if (unlikely(ret))
+		return ret;
+
+	if ((get_resp.u.ind_table.min_size > log_size) ||
+	    (get_resp.u.ind_table.max_size < log_size)) {
+		pr_err("indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n",
+		       1 << log_size, 1 << get_resp.u.ind_table.min_size,
+		       1 << get_resp.u.ind_table.max_size);
+		return -EINVAL;
+	}
+
+	tbl_size = (1ULL << log_size) *
+		sizeof(struct ena_admin_rss_ind_table_entry);
+
+	rss->rss_ind_tbl =
+		dma_zalloc_coherent(ena_dev->dmadev, tbl_size,
+				    &rss->rss_ind_tbl_dma_addr, GFP_KERNEL);
+	if (unlikely(!rss->rss_ind_tbl))
+		goto mem_err1;
+
+	tbl_size = (1ULL << log_size) * sizeof(u16);
+	rss->host_rss_ind_tbl =
+		devm_kzalloc(ena_dev->dmadev, tbl_size, GFP_KERNEL);
+	if (unlikely(!rss->host_rss_ind_tbl))
+		goto mem_err2;
+
+	rss->tbl_log_size = log_size;
+
+	return 0;
+
+mem_err2:
+	tbl_size = (1ULL << log_size) *
+		sizeof(struct ena_admin_rss_ind_table_entry);
+
+	dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
+			  rss->rss_ind_tbl_dma_addr);
+	rss->rss_ind_tbl = NULL;
+mem_err1:
+	rss->tbl_log_size = 0;
+	return -ENOMEM;
+}
+
+static void ena_com_indirect_table_destroy(struct ena_com_dev *ena_dev)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+	size_t tbl_size = (1ULL << rss->tbl_log_size) *
+		sizeof(struct ena_admin_rss_ind_table_entry);
+
+	if (rss->rss_ind_tbl)
+		dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
+				  rss->rss_ind_tbl_dma_addr);
+	rss->rss_ind_tbl = NULL;
+
+	if (rss->host_rss_ind_tbl)
+		devm_kfree(ena_dev->dmadev, rss->host_rss_ind_tbl);
+	rss->host_rss_ind_tbl = NULL;
+}
+
+static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
+				struct ena_com_io_sq *io_sq, u16 cq_idx)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_admin_aq_create_sq_cmd create_cmd;
+	struct ena_admin_acq_create_sq_resp_desc cmd_completion;
+	u8 direction;
+	int ret;
+
+	memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_sq_cmd));
+
+	create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ;
+
+	if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
+		direction = ENA_ADMIN_SQ_DIRECTION_TX;
+	else
+		direction = ENA_ADMIN_SQ_DIRECTION_RX;
+
+	create_cmd.sq_identity |= (direction <<
+		ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) &
+		ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK;
+
+	create_cmd.sq_caps_2 |= io_sq->mem_queue_type &
+		ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK;
+
+	create_cmd.sq_caps_2 |= (ENA_ADMIN_COMPLETION_POLICY_DESC <<
+		ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) &
+		ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK;
+
+	create_cmd.sq_caps_3 |=
+		ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
+
+	create_cmd.cq_idx = cq_idx;
+	create_cmd.sq_depth = io_sq->q_depth;
+
+	if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
+		ret = ena_com_mem_addr_set(ena_dev,
+					   &create_cmd.sq_ba,
+					   io_sq->desc_addr.phys_addr);
+		if (unlikely(ret)) {
+			pr_err("memory address set failed\n");
+			return ret;
+		}
+	}
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&create_cmd,
+					    sizeof(create_cmd),
+					    (struct ena_admin_acq_entry *)&cmd_completion,
+					    sizeof(cmd_completion));
+	if (unlikely(ret)) {
+		pr_err("Failed to create IO SQ. error: %d\n", ret);
+		return ret;
+	}
+
+	io_sq->idx = cmd_completion.sq_idx;
+
+	io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
+		(uintptr_t)cmd_completion.sq_doorbell_offset);
+
+	if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+		io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar
+				+ cmd_completion.llq_headers_offset);
+
+		io_sq->desc_addr.pbuf_dev_addr =
+			(u8 __iomem *)((uintptr_t)ena_dev->mem_bar +
+			cmd_completion.llq_descriptors_offset);
+	}
+
+	pr_debug("created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth);
+
+	return ret;
+}
+
+static int ena_com_ind_tbl_convert_to_device(struct ena_com_dev *ena_dev)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_com_io_sq *io_sq;
+	u16 qid;
+	int i;
+
+	for (i = 0; i < 1 << rss->tbl_log_size; i++) {
+		qid = rss->host_rss_ind_tbl[i];
+		if (qid >= ENA_TOTAL_NUM_QUEUES)
+			return -EINVAL;
+
+		io_sq = &ena_dev->io_sq_queues[qid];
+
+		if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX)
+			return -EINVAL;
+
+		rss->rss_ind_tbl[i].cq_idx = io_sq->idx;
+	}
+
+	return 0;
+}
+
+static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev)
+{
+	u16 dev_idx_to_host_tbl[ENA_TOTAL_NUM_QUEUES] = { (u16)-1 };
+	struct ena_rss *rss = &ena_dev->rss;
+	u8 idx;
+	u16 i;
+
+	for (i = 0; i < ENA_TOTAL_NUM_QUEUES; i++)
+		dev_idx_to_host_tbl[ena_dev->io_sq_queues[i].idx] = i;
+
+	for (i = 0; i < 1 << rss->tbl_log_size; i++) {
+		if (rss->rss_ind_tbl[i].cq_idx > ENA_TOTAL_NUM_QUEUES)
+			return -EINVAL;
+		idx = (u8)rss->rss_ind_tbl[i].cq_idx;
+
+		if (dev_idx_to_host_tbl[idx] > ENA_TOTAL_NUM_QUEUES)
+			return -EINVAL;
+
+		rss->host_rss_ind_tbl[i] = dev_idx_to_host_tbl[idx];
+	}
+
+	return 0;
+}
+
+static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev)
+{
+	size_t size;
+
+	size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS;
+
+	ena_dev->intr_moder_tbl =
+		devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
+	if (!ena_dev->intr_moder_tbl)
+		return -ENOMEM;
+
+	ena_com_config_default_interrupt_moderation_table(ena_dev);
+
+	return 0;
+}
+
+static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
+						 u16 intr_delay_resolution)
+{
+	struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
+	unsigned int i;
+
+	if (!intr_delay_resolution) {
+		pr_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
+		intr_delay_resolution = 1;
+	}
+	ena_dev->intr_delay_resolution = intr_delay_resolution;
+
+	/* update Rx */
+	for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++)
+		intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution;
+
+	/* update Tx */
+	ena_dev->intr_moder_tx_interval /= intr_delay_resolution;
+}
+
+/*****************************************************************************/
+/*******************************      API       ******************************/
+/*****************************************************************************/
+
+int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
+				  struct ena_admin_aq_entry *cmd,
+				  size_t cmd_size,
+				  struct ena_admin_acq_entry *comp,
+				  size_t comp_size)
+{
+	struct ena_comp_ctx *comp_ctx;
+	int ret;
+
+	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
+					    comp, comp_size);
+	if (unlikely(IS_ERR(comp_ctx))) {
+		pr_err("Failed to submit command [%ld]\n", PTR_ERR(comp_ctx));
+		return PTR_ERR(comp_ctx);
+	}
+
+	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
+	if (unlikely(ret)) {
+		if (admin_queue->running_state)
+			pr_err("Failed to process command. ret = %d\n", ret);
+		else
+			pr_debug("Failed to process command. ret = %d\n", ret);
+	}
+	return ret;
+}
+
+int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
+			 struct ena_com_io_cq *io_cq)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_admin_aq_create_cq_cmd create_cmd;
+	struct ena_admin_acq_create_cq_resp_desc cmd_completion;
+	int ret;
+
+	memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_cq_cmd));
+
+	create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ;
+
+	create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) &
+		ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
+	create_cmd.cq_caps_1 |=
+		ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK;
+
+	create_cmd.msix_vector = io_cq->msix_vector;
+	create_cmd.cq_depth = io_cq->q_depth;
+
+	ret = ena_com_mem_addr_set(ena_dev,
+				   &create_cmd.cq_ba,
+				   io_cq->cdesc_addr.phys_addr);
+	if (unlikely(ret)) {
+		pr_err("memory address set failed\n");
+		return ret;
+	}
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&create_cmd,
+					    sizeof(create_cmd),
+					    (struct ena_admin_acq_entry *)&cmd_completion,
+					    sizeof(cmd_completion));
+	if (unlikely(ret)) {
+		pr_err("Failed to create IO CQ. error: %d\n", ret);
+		return ret;
+	}
+
+	io_cq->idx = cmd_completion.cq_idx;
+
+	io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
+		cmd_completion.cq_interrupt_unmask_register_offset);
+
+	if (cmd_completion.cq_head_db_register_offset)
+		io_cq->cq_head_db_reg =
+			(u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
+			cmd_completion.cq_head_db_register_offset);
+
+	if (cmd_completion.numa_node_register_offset)
+		io_cq->numa_node_cfg_reg =
+			(u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
+			cmd_completion.numa_node_register_offset);
+
+	pr_debug("created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth);
+
+	return ret;
+}
+
+int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid,
+			    struct ena_com_io_sq **io_sq,
+			    struct ena_com_io_cq **io_cq)
+{
+	if (qid >= ENA_TOTAL_NUM_QUEUES) {
+		pr_err("Invalid queue number %d but the max is %d\n", qid,
+		       ENA_TOTAL_NUM_QUEUES);
+		return -EINVAL;
+	}
+
+	*io_sq = &ena_dev->io_sq_queues[qid];
+	*io_cq = &ena_dev->io_cq_queues[qid];
+
+	return 0;
+}
+
+void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_comp_ctx *comp_ctx;
+	u16 i;
+
+	if (!admin_queue->comp_ctx)
+		return;
+
+	for (i = 0; i < admin_queue->q_depth; i++) {
+		comp_ctx = get_comp_ctxt(admin_queue, i, false);
+		if (unlikely(!comp_ctx))
+			break;
+
+		comp_ctx->status = ENA_CMD_ABORTED;
+
+		complete(&comp_ctx->wait_event);
+	}
+}
+
+void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	unsigned long flags;
+
+	spin_lock_irqsave(&admin_queue->q_lock, flags);
+	while (atomic_read(&admin_queue->outstanding_cmds) != 0) {
+		spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+		msleep(20);
+		spin_lock_irqsave(&admin_queue->q_lock, flags);
+	}
+	spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+}
+
+int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
+			  struct ena_com_io_cq *io_cq)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_admin_aq_destroy_cq_cmd destroy_cmd;
+	struct ena_admin_acq_destroy_cq_resp_desc destroy_resp;
+	int ret;
+
+	memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
+
+	destroy_cmd.cq_idx = io_cq->idx;
+	destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ;
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&destroy_cmd,
+					    sizeof(destroy_cmd),
+					    (struct ena_admin_acq_entry *)&destroy_resp,
+					    sizeof(destroy_resp));
+
+	if (unlikely(ret && (ret != -ENODEV)))
+		pr_err("Failed to destroy IO CQ. error: %d\n", ret);
+
+	return ret;
+}
+
+bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev)
+{
+	return ena_dev->admin_queue.running_state;
+}
+
+void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	unsigned long flags;
+
+	spin_lock_irqsave(&admin_queue->q_lock, flags);
+	ena_dev->admin_queue.running_state = state;
+	spin_unlock_irqrestore(&admin_queue->q_lock, flags);
+}
+
+void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev)
+{
+	u16 depth = ena_dev->aenq.q_depth;
+
+	WARN(ena_dev->aenq.head != depth, "Invalid AENQ state\n");
+
+	/* Init head_db to mark that all entries in the queue
+	 * are initially available
+	 */
+	writel(depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
+}
+
+int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
+{
+	struct ena_com_admin_queue *admin_queue;
+	struct ena_admin_set_feat_cmd cmd;
+	struct ena_admin_set_feat_resp resp;
+	struct ena_admin_get_feat_resp get_resp;
+	int ret;
+
+	ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG);
+	if (ret) {
+		pr_info("Can't get aenq configuration\n");
+		return ret;
+	}
+
+	if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) {
+		pr_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n",
+			get_resp.u.aenq.supported_groups, groups_flag);
+		return -EPERM;
+	}
+
+	memset(&cmd, 0x0, sizeof(cmd));
+	admin_queue = &ena_dev->admin_queue;
+
+	cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
+	cmd.aq_common_descriptor.flags = 0;
+	cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG;
+	cmd.u.aenq.enabled_groups = groups_flag;
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&cmd,
+					    sizeof(cmd),
+					    (struct ena_admin_acq_entry *)&resp,
+					    sizeof(resp));
+
+	if (unlikely(ret))
+		pr_err("Failed to config AENQ ret: %d\n", ret);
+
+	return ret;
+}
+
+int ena_com_get_dma_width(struct ena_com_dev *ena_dev)
+{
+	u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
+	int width;
+
+	if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) {
+		pr_err("Reg read timeout occurred\n");
+		return -ETIME;
+	}
+
+	width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
+		ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
+
+	pr_debug("ENA dma width: %d\n", width);
+
+	if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) {
+		pr_err("DMA width illegal value: %d\n", width);
+		return -EINVAL;
+	}
+
+	ena_dev->dma_addr_bits = width;
+
+	return width;
+}
+
+int ena_com_validate_version(struct ena_com_dev *ena_dev)
+{
+	u32 ver;
+	u32 ctrl_ver;
+	u32 ctrl_ver_masked;
+
+	/* Make sure the ENA version and the controller version are at least
+	 * as the driver expects
+	 */
+	ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_VERSION_OFF);
+	ctrl_ver = ena_com_reg_bar_read32(ena_dev,
+					  ENA_REGS_CONTROLLER_VERSION_OFF);
+
+	if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) ||
+		     (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) {
+		pr_err("Reg read timeout occurred\n");
+		return -ETIME;
+	}
+
+	pr_info("ena device version: %d.%d\n",
+		(ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >>
+			ENA_REGS_VERSION_MAJOR_VERSION_SHIFT,
+		ver & ENA_REGS_VERSION_MINOR_VERSION_MASK);
+
+	if (ver < MIN_ENA_VER) {
+		pr_err("ENA version is lower than the minimal version the driver supports\n");
+		return -1;
+	}
+
+	pr_info("ena controller version: %d.%d.%d implementation version %d\n",
+		(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >>
+			ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
+		(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) >>
+			ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,
+		(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),
+		(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >>
+			ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);
+
+	ctrl_ver_masked =
+		(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |
+		(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |
+		(ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);
+
+	/* Validate the ctrl version without the implementation ID */
+	if (ctrl_ver_masked < MIN_ENA_CTRL_VER) {
+		pr_err("ENA ctrl version is lower than the minimal ctrl version the driver supports\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_com_admin_cq *cq = &admin_queue->cq;
+	struct ena_com_admin_sq *sq = &admin_queue->sq;
+	struct ena_com_aenq *aenq = &ena_dev->aenq;
+	u16 size;
+
+	if (admin_queue->comp_ctx)
+		devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);
+	admin_queue->comp_ctx = NULL;
+	size = ADMIN_SQ_SIZE(admin_queue->q_depth);
+	if (sq->entries)
+		dma_free_coherent(ena_dev->dmadev, size, sq->entries,
+				  sq->dma_addr);
+	sq->entries = NULL;
+
+	size = ADMIN_CQ_SIZE(admin_queue->q_depth);
+	if (cq->entries)
+		dma_free_coherent(ena_dev->dmadev, size, cq->entries,
+				  cq->dma_addr);
+	cq->entries = NULL;
+
+	size = ADMIN_AENQ_SIZE(aenq->q_depth);
+	if (ena_dev->aenq.entries)
+		dma_free_coherent(ena_dev->dmadev, size, aenq->entries,
+				  aenq->dma_addr);
+	aenq->entries = NULL;
+}
+
+void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling)
+{
+	ena_dev->admin_queue.polling = polling;
+}
+
+int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev)
+{
+	struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
+
+	spin_lock_init(&mmio_read->lock);
+	mmio_read->read_resp =
+		dma_zalloc_coherent(ena_dev->dmadev,
+				    sizeof(*mmio_read->read_resp),
+				    &mmio_read->read_resp_dma_addr, GFP_KERNEL);
+	if (unlikely(!mmio_read->read_resp))
+		return -ENOMEM;
+
+	ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
+
+	mmio_read->read_resp->req_id = 0x0;
+	mmio_read->seq_num = 0x0;
+	mmio_read->readless_supported = true;
+
+	return 0;
+}
+
+void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported)
+{
+	struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
+
+	mmio_read->readless_supported = readless_supported;
+}
+
+void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev)
+{
+	struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
+
+	writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
+	writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
+
+	dma_free_coherent(ena_dev->dmadev, sizeof(*mmio_read->read_resp),
+			  mmio_read->read_resp, mmio_read->read_resp_dma_addr);
+
+	mmio_read->read_resp = NULL;
+}
+
+void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev)
+{
+	struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
+	u32 addr_low, addr_high;
+
+	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr);
+	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr);
+
+	writel(addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
+	writel(addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
+}
+
+int ena_com_admin_init(struct ena_com_dev *ena_dev,
+		       struct ena_aenq_handlers *aenq_handlers,
+		       bool init_spinlock)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high;
+	int ret;
+
+	dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
+
+	if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) {
+		pr_err("Reg read timeout occurred\n");
+		return -ETIME;
+	}
+
+	if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) {
+		pr_err("Device isn't ready, abort com init\n");
+		return -ENODEV;
+	}
+
+	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;
+
+	admin_queue->q_dmadev = ena_dev->dmadev;
+	admin_queue->polling = false;
+	admin_queue->curr_cmd_id = 0;
+
+	atomic_set(&admin_queue->outstanding_cmds, 0);
+
+	if (init_spinlock)
+		spin_lock_init(&admin_queue->q_lock);
+
+	ret = ena_com_init_comp_ctxt(admin_queue);
+	if (ret)
+		goto error;
+
+	ret = ena_com_admin_init_sq(admin_queue);
+	if (ret)
+		goto error;
+
+	ret = ena_com_admin_init_cq(admin_queue);
+	if (ret)
+		goto error;
+
+	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
+		ENA_REGS_AQ_DB_OFF);
+
+	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);
+	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);
+
+	writel(addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF);
+	writel(addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF);
+
+	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);
+	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);
+
+	writel(addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF);
+	writel(addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF);
+
+	aq_caps = 0;
+	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
+	aq_caps |= (sizeof(struct ena_admin_aq_entry) <<
+			ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &
+			ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;
+
+	acq_caps = 0;
+	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;
+	acq_caps |= (sizeof(struct ena_admin_acq_entry) <<
+		ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
+		ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
+
+	writel(aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF);
+	writel(acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF);
+	ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers);
+	if (ret)
+		goto error;
+
+	admin_queue->running_state = true;
+
+	return 0;
+error:
+	ena_com_admin_destroy(ena_dev);
+
+	return ret;
+}
+
+int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
+			    struct ena_com_create_io_ctx *ctx)
+{
+	struct ena_com_io_sq *io_sq;
+	struct ena_com_io_cq *io_cq;
+	int ret;
+
+	if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) {
+		pr_err("Qid (%d) is bigger than max num of queues (%d)\n",
+		       ctx->qid, ENA_TOTAL_NUM_QUEUES);
+		return -EINVAL;
+	}
+
+	io_sq = &ena_dev->io_sq_queues[ctx->qid];
+	io_cq = &ena_dev->io_cq_queues[ctx->qid];
+
+	memset(io_sq, 0x0, sizeof(struct ena_com_io_sq));
+	memset(io_cq, 0x0, sizeof(struct ena_com_io_cq));
+
+	/* Init CQ */
+	io_cq->q_depth = ctx->queue_size;
+	io_cq->direction = ctx->direction;
+	io_cq->qid = ctx->qid;
+
+	io_cq->msix_vector = ctx->msix_vector;
+
+	io_sq->q_depth = ctx->queue_size;
+	io_sq->direction = ctx->direction;
+	io_sq->qid = ctx->qid;
+
+	io_sq->mem_queue_type = ctx->mem_queue_type;
+
+	if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
+		/* header length is limited to 8 bits */
+		io_sq->tx_max_header_size =
+			min_t(u32, ena_dev->tx_max_header_size, SZ_256);
+
+	ret = ena_com_init_io_sq(ena_dev, ctx, io_sq);
+	if (ret)
+		goto error;
+	ret = ena_com_init_io_cq(ena_dev, ctx, io_cq);
+	if (ret)
+		goto error;
+
+	ret = ena_com_create_io_cq(ena_dev, io_cq);
+	if (ret)
+		goto error;
+
+	ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx);
+	if (ret)
+		goto destroy_io_cq;
+
+	return 0;
+
+destroy_io_cq:
+	ena_com_destroy_io_cq(ena_dev, io_cq);
+error:
+	ena_com_io_queue_free(ena_dev, io_sq, io_cq);
+	return ret;
+}
+
+void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid)
+{
+	struct ena_com_io_sq *io_sq;
+	struct ena_com_io_cq *io_cq;
+
+	if (qid >= ENA_TOTAL_NUM_QUEUES) {
+		pr_err("Qid (%d) is bigger than max num of queues (%d)\n", qid,
+		       ENA_TOTAL_NUM_QUEUES);
+		return;
+	}
+
+	io_sq = &ena_dev->io_sq_queues[qid];
+	io_cq = &ena_dev->io_cq_queues[qid];
+
+	ena_com_destroy_io_sq(ena_dev, io_sq);
+	ena_com_destroy_io_cq(ena_dev, io_cq);
+
+	ena_com_io_queue_free(ena_dev, io_sq, io_cq);
+}
+
+int ena_com_get_link_params(struct ena_com_dev *ena_dev,
+			    struct ena_admin_get_feat_resp *resp)
+{
+	return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG);
+}
+
+int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
+			      struct ena_com_dev_get_features_ctx *get_feat_ctx)
+{
+	struct ena_admin_get_feat_resp get_resp;
+	int rc;
+
+	rc = ena_com_get_feature(ena_dev, &get_resp,
+				 ENA_ADMIN_DEVICE_ATTRIBUTES);
+	if (rc)
+		return rc;
+
+	memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr,
+	       sizeof(get_resp.u.dev_attr));
+	ena_dev->supported_features = get_resp.u.dev_attr.supported_features;
+
+	rc = ena_com_get_feature(ena_dev, &get_resp,
+				 ENA_ADMIN_MAX_QUEUES_NUM);
+	if (rc)
+		return rc;
+
+	memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue,
+	       sizeof(get_resp.u.max_queue));
+	ena_dev->tx_max_header_size = get_resp.u.max_queue.max_header_size;
+
+	rc = ena_com_get_feature(ena_dev, &get_resp,
+				 ENA_ADMIN_AENQ_CONFIG);
+	if (rc)
+		return rc;
+
+	memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq,
+	       sizeof(get_resp.u.aenq));
+
+	rc = ena_com_get_feature(ena_dev, &get_resp,
+				 ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
+	if (rc)
+		return rc;
+
+	memcpy(&get_feat_ctx->offload, &get_resp.u.offload,
+	       sizeof(get_resp.u.offload));
+
+	return 0;
+}
+
+void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev)
+{
+	ena_com_handle_admin_completion(&ena_dev->admin_queue);
+}
+
+/* ena_handle_specific_aenq_event:
+ * return the handler that is relevant to the specific event group
+ */
+static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev,
+						     u16 group)
+{
+	struct ena_aenq_handlers *aenq_handlers = dev->aenq.aenq_handlers;
+
+	if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group])
+		return aenq_handlers->handlers[group];
+
+	return aenq_handlers->unimplemented_handler;
+}
+
+/* ena_aenq_intr_handler:
+ * handles the aenq incoming events.
+ * pop events from the queue and apply the specific handler
+ */
+void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
+{
+	struct ena_admin_aenq_entry *aenq_e;
+	struct ena_admin_aenq_common_desc *aenq_common;
+	struct ena_com_aenq *aenq  = &dev->aenq;
+	ena_aenq_handler handler_cb;
+	u16 masked_head, processed = 0;
+	u8 phase;
+
+	masked_head = aenq->head & (aenq->q_depth - 1);
+	phase = aenq->phase;
+	aenq_e = &aenq->entries[masked_head]; /* Get first entry */
+	aenq_common = &aenq_e->aenq_common_desc;
+
+	/* Go over all the events */
+	while ((aenq_common->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) ==
+	       phase) {
+		pr_debug("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n",
+			 aenq_common->group, aenq_common->syndrom,
+			 (u64)aenq_common->timestamp_low +
+				 ((u64)aenq_common->timestamp_high << 32));
+
+		/* Handle specific event*/
+		handler_cb = ena_com_get_specific_aenq_cb(dev,
+							  aenq_common->group);
+		handler_cb(data, aenq_e); /* call the actual event handler*/
+
+		/* Get next event entry */
+		masked_head++;
+		processed++;
+
+		if (unlikely(masked_head == aenq->q_depth)) {
+			masked_head = 0;
+			phase = !phase;
+		}
+		aenq_e = &aenq->entries[masked_head];
+		aenq_common = &aenq_e->aenq_common_desc;
+	}
+
+	aenq->head += processed;
+	aenq->phase = phase;
+
+	/* Don't update aenq doorbell if there weren't any processed events */
+	if (!processed)
+		return;
+
+	/* write the aenq doorbell after all AENQ descriptors were read */
+	mb();
+	writel((u32)aenq->head, dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
+}
+
+int ena_com_dev_reset(struct ena_com_dev *ena_dev)
+{
+	u32 stat, timeout, cap, reset_val;
+	int rc;
+
+	stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
+	cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
+
+	if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) ||
+		     (cap == ENA_MMIO_READ_TIMEOUT))) {
+		pr_err("Reg read32 timeout occurred\n");
+		return -ETIME;
+	}
+
+	if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) {
+		pr_err("Device isn't ready, can't reset device\n");
+		return -EINVAL;
+	}
+
+	timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
+			ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
+	if (timeout == 0) {
+		pr_err("Invalid timeout value\n");
+		return -EINVAL;
+	}
+
+	/* start reset */
+	reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK;
+	writel(reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
+
+	/* Write again the MMIO read request address */
+	ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
+
+	rc = wait_for_reset_state(ena_dev, timeout,
+				  ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
+	if (rc != 0) {
+		pr_err("Reset indication didn't turn on\n");
+		return rc;
+	}
+
+	/* reset done */
+	writel(0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
+	rc = wait_for_reset_state(ena_dev, timeout, 0);
+	if (rc != 0) {
+		pr_err("Reset indication didn't turn off\n");
+		return rc;
+	}
+
+	return 0;
+}
+
+static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
+			     struct ena_com_stats_ctx *ctx,
+			     enum ena_admin_get_stats_type type)
+{
+	struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd;
+	struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp;
+	struct ena_com_admin_queue *admin_queue;
+	int ret;
+
+	admin_queue = &ena_dev->admin_queue;
+
+	get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS;
+	get_cmd->aq_common_descriptor.flags = 0;
+	get_cmd->type = type;
+
+	ret =  ena_com_execute_admin_command(admin_queue,
+					     (struct ena_admin_aq_entry *)get_cmd,
+					     sizeof(*get_cmd),
+					     (struct ena_admin_acq_entry *)get_resp,
+					     sizeof(*get_resp));
+
+	if (unlikely(ret))
+		pr_err("Failed to get stats. error: %d\n", ret);
+
+	return ret;
+}
+
+int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
+				struct ena_admin_basic_stats *stats)
+{
+	struct ena_com_stats_ctx ctx;
+	int ret;
+
+	memset(&ctx, 0x0, sizeof(ctx));
+	ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC);
+	if (likely(ret == 0))
+		memcpy(stats, &ctx.get_resp.basic_stats,
+		       sizeof(ctx.get_resp.basic_stats));
+
+	return ret;
+}
+
+int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
+{
+	struct ena_com_admin_queue *admin_queue;
+	struct ena_admin_set_feat_cmd cmd;
+	struct ena_admin_set_feat_resp resp;
+	int ret;
+
+	if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
+		pr_info("Feature %d isn't supported\n", ENA_ADMIN_MTU);
+		return -EPERM;
+	}
+
+	memset(&cmd, 0x0, sizeof(cmd));
+	admin_queue = &ena_dev->admin_queue;
+
+	cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
+	cmd.aq_common_descriptor.flags = 0;
+	cmd.feat_common.feature_id = ENA_ADMIN_MTU;
+	cmd.u.mtu.mtu = mtu;
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&cmd,
+					    sizeof(cmd),
+					    (struct ena_admin_acq_entry *)&resp,
+					    sizeof(resp));
+
+	if (unlikely(ret))
+		pr_err("Failed to set mtu %d. error: %d\n", mtu, ret);
+
+	return ret;
+}
+
+int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
+				 struct ena_admin_feature_offload_desc *offload)
+{
+	int ret;
+	struct ena_admin_get_feat_resp resp;
+
+	ret = ena_com_get_feature(ena_dev, &resp,
+				  ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
+	if (unlikely(ret)) {
+		pr_err("Failed to get offload capabilities %d\n", ret);
+		return ret;
+	}
+
+	memcpy(offload, &resp.u.offload, sizeof(resp.u.offload));
+
+	return 0;
+}
+
+int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_set_feat_cmd cmd;
+	struct ena_admin_set_feat_resp resp;
+	struct ena_admin_get_feat_resp get_resp;
+	int ret;
+
+	if (!ena_com_check_supported_feature_id(ena_dev,
+						ENA_ADMIN_RSS_HASH_FUNCTION)) {
+		pr_info("Feature %d isn't supported\n",
+			ENA_ADMIN_RSS_HASH_FUNCTION);
+		return -EPERM;
+	}
+
+	/* Validate hash function is supported */
+	ret = ena_com_get_feature(ena_dev, &get_resp,
+				  ENA_ADMIN_RSS_HASH_FUNCTION);
+	if (unlikely(ret))
+		return ret;
+
+	if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) {
+		pr_err("Func hash %d isn't supported by device, abort\n",
+		       rss->hash_func);
+		return -EPERM;
+	}
+
+	memset(&cmd, 0x0, sizeof(cmd));
+
+	cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
+	cmd.aq_common_descriptor.flags =
+		ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
+	cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION;
+	cmd.u.flow_hash_func.init_val = rss->hash_init_val;
+	cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func;
+
+	ret = ena_com_mem_addr_set(ena_dev,
+				   &cmd.control_buffer.address,
+				   rss->hash_key_dma_addr);
+	if (unlikely(ret)) {
+		pr_err("memory address set failed\n");
+		return ret;
+	}
+
+	cmd.control_buffer.length = sizeof(*rss->hash_key);
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&cmd,
+					    sizeof(cmd),
+					    (struct ena_admin_acq_entry *)&resp,
+					    sizeof(resp));
+	if (unlikely(ret)) {
+		pr_err("Failed to set hash function %d. error: %d\n",
+		       rss->hash_func, ret);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
+			       enum ena_admin_hash_functions func,
+			       const u8 *key, u16 key_len, u32 init_val)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_get_feat_resp get_resp;
+	struct ena_admin_feature_rss_flow_hash_control *hash_key =
+		rss->hash_key;
+	int rc;
+
+	/* Make sure size is a mult of DWs */
+	if (unlikely(key_len & 0x3))
+		return -EINVAL;
+
+	rc = ena_com_get_feature_ex(ena_dev, &get_resp,
+				    ENA_ADMIN_RSS_HASH_FUNCTION,
+				    rss->hash_key_dma_addr,
+				    sizeof(*rss->hash_key));
+	if (unlikely(rc))
+		return rc;
+
+	if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) {
+		pr_err("Flow hash function %d isn't supported\n", func);
+		return -EPERM;
+	}
+
+	switch (func) {
+	case ENA_ADMIN_TOEPLITZ:
+		if (key_len > sizeof(hash_key->key)) {
+			pr_err("key len (%hu) is bigger than the max supported (%zu)\n",
+			       key_len, sizeof(hash_key->key));
+			return -EINVAL;
+		}
+
+		memcpy(hash_key->key, key, key_len);
+		rss->hash_init_val = init_val;
+		hash_key->keys_num = key_len >> 2;
+		break;
+	case ENA_ADMIN_CRC32:
+		rss->hash_init_val = init_val;
+		break;
+	default:
+		pr_err("Invalid hash function (%d)\n", func);
+		return -EINVAL;
+	}
+
+	rc = ena_com_set_hash_function(ena_dev);
+
+	/* Restore the old function */
+	if (unlikely(rc))
+		ena_com_get_hash_function(ena_dev, NULL, NULL);
+
+	return rc;
+}
+
+int ena_com_get_hash_function(struct ena_com_dev *ena_dev,
+			      enum ena_admin_hash_functions *func,
+			      u8 *key)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_get_feat_resp get_resp;
+	struct ena_admin_feature_rss_flow_hash_control *hash_key =
+		rss->hash_key;
+	int rc;
+
+	rc = ena_com_get_feature_ex(ena_dev, &get_resp,
+				    ENA_ADMIN_RSS_HASH_FUNCTION,
+				    rss->hash_key_dma_addr,
+				    sizeof(*rss->hash_key));
+	if (unlikely(rc))
+		return rc;
+
+	rss->hash_func = get_resp.u.flow_hash_func.selected_func;
+	if (func)
+		*func = rss->hash_func;
+
+	if (key)
+		memcpy(key, hash_key->key, (size_t)(hash_key->keys_num) << 2);
+
+	return 0;
+}
+
+int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev,
+			  enum ena_admin_flow_hash_proto proto,
+			  u16 *fields)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_get_feat_resp get_resp;
+	int rc;
+
+	rc = ena_com_get_feature_ex(ena_dev, &get_resp,
+				    ENA_ADMIN_RSS_HASH_INPUT,
+				    rss->hash_ctrl_dma_addr,
+				    sizeof(*rss->hash_ctrl));
+	if (unlikely(rc))
+		return rc;
+
+	if (fields)
+		*fields = rss->hash_ctrl->selected_fields[proto].fields;
+
+	return 0;
+}
+
+int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
+	struct ena_admin_set_feat_cmd cmd;
+	struct ena_admin_set_feat_resp resp;
+	int ret;
+
+	if (!ena_com_check_supported_feature_id(ena_dev,
+						ENA_ADMIN_RSS_HASH_INPUT)) {
+		pr_info("Feature %d isn't supported\n", ENA_ADMIN_RSS_HASH_INPUT);
+		return -EPERM;
+	}
+
+	memset(&cmd, 0x0, sizeof(cmd));
+
+	cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
+	cmd.aq_common_descriptor.flags =
+		ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
+	cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT;
+	cmd.u.flow_hash_input.enabled_input_sort =
+		ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK |
+		ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK;
+
+	ret = ena_com_mem_addr_set(ena_dev,
+				   &cmd.control_buffer.address,
+				   rss->hash_ctrl_dma_addr);
+	if (unlikely(ret)) {
+		pr_err("memory address set failed\n");
+		return ret;
+	}
+	cmd.control_buffer.length = sizeof(*hash_ctrl);
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&cmd,
+					    sizeof(cmd),
+					    (struct ena_admin_acq_entry *)&resp,
+					    sizeof(resp));
+	if (unlikely(ret))
+		pr_err("Failed to set hash input. error: %d\n", ret);
+
+	return ret;
+}
+
+int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_feature_rss_hash_control *hash_ctrl =
+		rss->hash_ctrl;
+	u16 available_fields = 0;
+	int rc, i;
+
+	/* Get the supported hash input */
+	rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL);
+	if (unlikely(rc))
+		return rc;
+
+	hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields =
+		ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
+		ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
+
+	hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields =
+		ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
+		ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
+
+	hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields =
+		ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
+		ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
+
+	hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields =
+		ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
+		ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
+
+	hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields =
+		ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
+
+	hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields =
+		ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
+
+	hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
+		ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
+
+	hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
+		ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
+
+	for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
+		available_fields = hash_ctrl->selected_fields[i].fields &
+				hash_ctrl->supported_fields[i].fields;
+		if (available_fields != hash_ctrl->selected_fields[i].fields) {
+			pr_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
+			       i, hash_ctrl->supported_fields[i].fields,
+			       hash_ctrl->selected_fields[i].fields);
+			return -EPERM;
+		}
+	}
+
+	rc = ena_com_set_hash_ctrl(ena_dev);
+
+	/* In case of failure, restore the old hash ctrl */
+	if (unlikely(rc))
+		ena_com_get_hash_ctrl(ena_dev, 0, NULL);
+
+	return rc;
+}
+
+int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
+			   enum ena_admin_flow_hash_proto proto,
+			   u16 hash_fields)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
+	u16 supported_fields;
+	int rc;
+
+	if (proto >= ENA_ADMIN_RSS_PROTO_NUM) {
+		pr_err("Invalid proto num (%u)\n", proto);
+		return -EINVAL;
+	}
+
+	/* Get the ctrl table */
+	rc = ena_com_get_hash_ctrl(ena_dev, proto, NULL);
+	if (unlikely(rc))
+		return rc;
+
+	/* Make sure all the fields are supported */
+	supported_fields = hash_ctrl->supported_fields[proto].fields;
+	if ((hash_fields & supported_fields) != hash_fields) {
+		pr_err("proto %d doesn't support the required fields %x. supports only: %x\n",
+		       proto, hash_fields, supported_fields);
+	}
+
+	hash_ctrl->selected_fields[proto].fields = hash_fields;
+
+	rc = ena_com_set_hash_ctrl(ena_dev);
+
+	/* In case of failure, restore the old hash ctrl */
+	if (unlikely(rc))
+		ena_com_get_hash_ctrl(ena_dev, 0, NULL);
+
+	return 0;
+}
+
+int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev,
+				      u16 entry_idx, u16 entry_value)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+
+	if (unlikely(entry_idx >= (1 << rss->tbl_log_size)))
+		return -EINVAL;
+
+	if (unlikely((entry_value > ENA_TOTAL_NUM_QUEUES)))
+		return -EINVAL;
+
+	rss->host_rss_ind_tbl[entry_idx] = entry_value;
+
+	return 0;
+}
+
+int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
+{
+	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_set_feat_cmd cmd;
+	struct ena_admin_set_feat_resp resp;
+	int ret;
+
+	if (!ena_com_check_supported_feature_id(
+		    ena_dev, ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
+		pr_info("Feature %d isn't supported\n",
+			ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
+		return -EPERM;
+	}
+
+	ret = ena_com_ind_tbl_convert_to_device(ena_dev);
+	if (ret) {
+		pr_err("Failed to convert host indirection table to device table\n");
+		return ret;
+	}
+
+	memset(&cmd, 0x0, sizeof(cmd));
+
+	cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
+	cmd.aq_common_descriptor.flags =
+		ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
+	cmd.feat_common.feature_id = ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG;
+	cmd.u.ind_table.size = rss->tbl_log_size;
+	cmd.u.ind_table.inline_index = 0xFFFFFFFF;
+
+	ret = ena_com_mem_addr_set(ena_dev,
+				   &cmd.control_buffer.address,
+				   rss->rss_ind_tbl_dma_addr);
+	if (unlikely(ret)) {
+		pr_err("memory address set failed\n");
+		return ret;
+	}
+
+	cmd.control_buffer.length = (1ULL << rss->tbl_log_size) *
+		sizeof(struct ena_admin_rss_ind_table_entry);
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&cmd,
+					    sizeof(cmd),
+					    (struct ena_admin_acq_entry *)&resp,
+					    sizeof(resp));
+
+	if (unlikely(ret))
+		pr_err("Failed to set indirect table. error: %d\n", ret);
+
+	return ret;
+}
+
+int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl)
+{
+	struct ena_rss *rss = &ena_dev->rss;
+	struct ena_admin_get_feat_resp get_resp;
+	u32 tbl_size;
+	int i, rc;
+
+	tbl_size = (1ULL << rss->tbl_log_size) *
+		sizeof(struct ena_admin_rss_ind_table_entry);
+
+	rc = ena_com_get_feature_ex(ena_dev, &get_resp,
+				    ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG,
+				    rss->rss_ind_tbl_dma_addr,
+				    tbl_size);
+	if (unlikely(rc))
+		return rc;
+
+	if (!ind_tbl)
+		return 0;
+
+	rc = ena_com_ind_tbl_convert_from_device(ena_dev);
+	if (unlikely(rc))
+		return rc;
+
+	for (i = 0; i < (1 << rss->tbl_log_size); i++)
+		ind_tbl[i] = rss->host_rss_ind_tbl[i];
+
+	return 0;
+}
+
+int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 indr_tbl_log_size)
+{
+	int rc;
+
+	memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
+
+	rc = ena_com_indirect_table_allocate(ena_dev, indr_tbl_log_size);
+	if (unlikely(rc))
+		goto err_indr_tbl;
+
+	rc = ena_com_hash_key_allocate(ena_dev);
+	if (unlikely(rc))
+		goto err_hash_key;
+
+	rc = ena_com_hash_ctrl_init(ena_dev);
+	if (unlikely(rc))
+		goto err_hash_ctrl;
+
+	return 0;
+
+err_hash_ctrl:
+	ena_com_hash_key_destroy(ena_dev);
+err_hash_key:
+	ena_com_indirect_table_destroy(ena_dev);
+err_indr_tbl:
+
+	return rc;
+}
+
+void ena_com_rss_destroy(struct ena_com_dev *ena_dev)
+{
+	ena_com_indirect_table_destroy(ena_dev);
+	ena_com_hash_key_destroy(ena_dev);
+	ena_com_hash_ctrl_destroy(ena_dev);
+
+	memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
+}
+
+int ena_com_allocate_host_info(struct ena_com_dev *ena_dev)
+{
+	struct ena_host_attribute *host_attr = &ena_dev->host_attr;
+
+	host_attr->host_info =
+		dma_zalloc_coherent(ena_dev->dmadev, SZ_4K,
+				    &host_attr->host_info_dma_addr, GFP_KERNEL);
+	if (unlikely(!host_attr->host_info))
+		return -ENOMEM;
+
+	return 0;
+}
+
+int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev,
+				u32 debug_area_size)
+{
+	struct ena_host_attribute *host_attr = &ena_dev->host_attr;
+
+	host_attr->debug_area_virt_addr =
+		dma_zalloc_coherent(ena_dev->dmadev, debug_area_size,
+				    &host_attr->debug_area_dma_addr, GFP_KERNEL);
+	if (unlikely(!host_attr->debug_area_virt_addr)) {
+		host_attr->debug_area_size = 0;
+		return -ENOMEM;
+	}
+
+	host_attr->debug_area_size = debug_area_size;
+
+	return 0;
+}
+
+void ena_com_delete_host_info(struct ena_com_dev *ena_dev)
+{
+	struct ena_host_attribute *host_attr = &ena_dev->host_attr;
+
+	if (host_attr->host_info) {
+		dma_free_coherent(ena_dev->dmadev, SZ_4K, host_attr->host_info,
+				  host_attr->host_info_dma_addr);
+		host_attr->host_info = NULL;
+	}
+}
+
+void ena_com_delete_debug_area(struct ena_com_dev *ena_dev)
+{
+	struct ena_host_attribute *host_attr = &ena_dev->host_attr;
+
+	if (host_attr->debug_area_virt_addr) {
+		dma_free_coherent(ena_dev->dmadev, host_attr->debug_area_size,
+				  host_attr->debug_area_virt_addr,
+				  host_attr->debug_area_dma_addr);
+		host_attr->debug_area_virt_addr = NULL;
+	}
+}
+
+int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
+{
+	struct ena_host_attribute *host_attr = &ena_dev->host_attr;
+	struct ena_com_admin_queue *admin_queue;
+	struct ena_admin_set_feat_cmd cmd;
+	struct ena_admin_set_feat_resp resp;
+
+	int ret;
+
+	if (!ena_com_check_supported_feature_id(ena_dev,
+						ENA_ADMIN_HOST_ATTR_CONFIG)) {
+		pr_warn("Set host attribute isn't supported\n");
+		return -EPERM;
+	}
+
+	memset(&cmd, 0x0, sizeof(cmd));
+	admin_queue = &ena_dev->admin_queue;
+
+	cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
+	cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG;
+
+	ret = ena_com_mem_addr_set(ena_dev,
+				   &cmd.u.host_attr.debug_ba,
+				   host_attr->debug_area_dma_addr);
+	if (unlikely(ret)) {
+		pr_err("memory address set failed\n");
+		return ret;
+	}
+
+	ret = ena_com_mem_addr_set(ena_dev,
+				   &cmd.u.host_attr.os_info_ba,
+				   host_attr->host_info_dma_addr);
+	if (unlikely(ret)) {
+		pr_err("memory address set failed\n");
+		return ret;
+	}
+
+	cmd.u.host_attr.debug_area_size = host_attr->debug_area_size;
+
+	ret = ena_com_execute_admin_command(admin_queue,
+					    (struct ena_admin_aq_entry *)&cmd,
+					    sizeof(cmd),
+					    (struct ena_admin_acq_entry *)&resp,
+					    sizeof(resp));
+
+	if (unlikely(ret))
+		pr_err("Failed to set host attributes: %d\n", ret);
+
+	return ret;
+}
+
+/* Interrupt moderation */
+bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
+{
+	return ena_com_check_supported_feature_id(ena_dev,
+						  ENA_ADMIN_INTERRUPT_MODERATION);
+}
+
+int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
+						      u32 tx_coalesce_usecs)
+{
+	if (!ena_dev->intr_delay_resolution) {
+		pr_err("Illegal interrupt delay granularity value\n");
+		return -EFAULT;
+	}
+
+	ena_dev->intr_moder_tx_interval = tx_coalesce_usecs /
+		ena_dev->intr_delay_resolution;
+
+	return 0;
+}
+
+int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
+						      u32 rx_coalesce_usecs)
+{
+	if (!ena_dev->intr_delay_resolution) {
+		pr_err("Illegal interrupt delay granularity value\n");
+		return -EFAULT;
+	}
+
+	/* We use LOWEST entry of moderation table for storing
+	 * nonadaptive interrupt coalescing values
+	 */
+	ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
+		rx_coalesce_usecs / ena_dev->intr_delay_resolution;
+
+	return 0;
+}
+
+void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev)
+{
+	if (ena_dev->intr_moder_tbl)
+		devm_kfree(ena_dev->dmadev, ena_dev->intr_moder_tbl);
+	ena_dev->intr_moder_tbl = NULL;
+}
+
+int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
+{
+	struct ena_admin_get_feat_resp get_resp;
+	u16 delay_resolution;
+	int rc;
+
+	rc = ena_com_get_feature(ena_dev, &get_resp,
+				 ENA_ADMIN_INTERRUPT_MODERATION);
+
+	if (rc) {
+		if (rc == -EPERM) {
+			pr_info("Feature %d isn't supported\n",
+				ENA_ADMIN_INTERRUPT_MODERATION);
+			rc = 0;
+		} else {
+			pr_err("Failed to get interrupt moderation admin cmd. rc: %d\n",
+			       rc);
+		}
+
+		/* no moderation supported, disable adaptive support */
+		ena_com_disable_adaptive_moderation(ena_dev);
+		return rc;
+	}
+
+	rc = ena_com_init_interrupt_moderation_table(ena_dev);
+	if (rc)
+		goto err;
+
+	/* if moderation is supported by device we set adaptive moderation */
+	delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution;
+	ena_com_update_intr_delay_resolution(ena_dev, delay_resolution);
+	ena_com_enable_adaptive_moderation(ena_dev);
+
+	return 0;
+err:
+	ena_com_destroy_interrupt_moderation(ena_dev);
+	return rc;
+}
+
+void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
+{
+	struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
+
+	if (!intr_moder_tbl)
+		return;
+
+	intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
+		ENA_INTR_LOWEST_USECS;
+	intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval =
+		ENA_INTR_LOWEST_PKTS;
+	intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval =
+		ENA_INTR_LOWEST_BYTES;
+
+	intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval =
+		ENA_INTR_LOW_USECS;
+	intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval =
+		ENA_INTR_LOW_PKTS;
+	intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval =
+		ENA_INTR_LOW_BYTES;
+
+	intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval =
+		ENA_INTR_MID_USECS;
+	intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval =
+		ENA_INTR_MID_PKTS;
+	intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval =
+		ENA_INTR_MID_BYTES;
+
+	intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval =
+		ENA_INTR_HIGH_USECS;
+	intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval =
+		ENA_INTR_HIGH_PKTS;
+	intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval =
+		ENA_INTR_HIGH_BYTES;
+
+	intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval =
+		ENA_INTR_HIGHEST_USECS;
+	intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval =
+		ENA_INTR_HIGHEST_PKTS;
+	intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval =
+		ENA_INTR_HIGHEST_BYTES;
+}
+
+unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
+{
+	return ena_dev->intr_moder_tx_interval;
+}
+
+unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
+{
+	struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
+
+	if (intr_moder_tbl)
+		return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval;
+
+	return 0;
+}
+
+void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
+					enum ena_intr_moder_level level,
+					struct ena_intr_moder_entry *entry)
+{
+	struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
+
+	if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
+		return;
+
+	intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval;
+	if (ena_dev->intr_delay_resolution)
+		intr_moder_tbl[level].intr_moder_interval /=
+			ena_dev->intr_delay_resolution;
+	intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval;
+
+	/* use hardcoded value until ethtool supports bytecount parameter */
+	if (entry->bytes_per_interval != ENA_INTR_BYTE_COUNT_NOT_SUPPORTED)
+		intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval;
+}
+
+void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
+				       enum ena_intr_moder_level level,
+				       struct ena_intr_moder_entry *entry)
+{
+	struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
+
+	if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
+		return;
+
+	entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval;
+	if (ena_dev->intr_delay_resolution)
+		entry->intr_moder_interval *= ena_dev->intr_delay_resolution;
+	entry->pkts_per_interval =
+	intr_moder_tbl[level].pkts_per_interval;
+	entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval;
+}
diff --git a/drivers/net/ethernet/amazon/ena/ena_com.h b/drivers/net/ethernet/amazon/ena/ena_com.h
new file mode 100644
index 000000000000..509d7b8e15ab
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_com.h
@@ -0,0 +1,1038 @@
+/*
+ * Copyright 2015 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef ENA_COM
+#define ENA_COM
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/gfp.h>
+#include <linux/sched.h>
+#include <linux/sizes.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+#include <linux/wait.h>
+
+#include "ena_common_defs.h"
+#include "ena_admin_defs.h"
+#include "ena_eth_io_defs.h"
+#include "ena_regs_defs.h"
+
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#define ENA_MAX_NUM_IO_QUEUES		128U
+/* We need to queues for each IO (on for Tx and one for Rx) */
+#define ENA_TOTAL_NUM_QUEUES		(2 * (ENA_MAX_NUM_IO_QUEUES))
+
+#define ENA_MAX_HANDLERS 256
+
+#define ENA_MAX_PHYS_ADDR_SIZE_BITS 48
+
+/* Unit in usec */
+#define ENA_REG_READ_TIMEOUT 200000
+
+#define ADMIN_SQ_SIZE(depth)	((depth) * sizeof(struct ena_admin_aq_entry))
+#define ADMIN_CQ_SIZE(depth)	((depth) * sizeof(struct ena_admin_acq_entry))
+#define ADMIN_AENQ_SIZE(depth)	((depth) * sizeof(struct ena_admin_aenq_entry))
+
+/*****************************************************************************/
+/*****************************************************************************/
+/* ENA adaptive interrupt moderation settings */
+
+#define ENA_INTR_LOWEST_USECS           (0)
+#define ENA_INTR_LOWEST_PKTS            (3)
+#define ENA_INTR_LOWEST_BYTES           (2 * 1524)
+
+#define ENA_INTR_LOW_USECS              (32)
+#define ENA_INTR_LOW_PKTS               (12)
+#define ENA_INTR_LOW_BYTES              (16 * 1024)
+
+#define ENA_INTR_MID_USECS              (80)
+#define ENA_INTR_MID_PKTS               (48)
+#define ENA_INTR_MID_BYTES              (64 * 1024)
+
+#define ENA_INTR_HIGH_USECS             (128)
+#define ENA_INTR_HIGH_PKTS              (96)
+#define ENA_INTR_HIGH_BYTES             (128 * 1024)
+
+#define ENA_INTR_HIGHEST_USECS          (192)
+#define ENA_INTR_HIGHEST_PKTS           (128)
+#define ENA_INTR_HIGHEST_BYTES          (192 * 1024)
+
+#define ENA_INTR_INITIAL_TX_INTERVAL_USECS		196
+#define ENA_INTR_INITIAL_RX_INTERVAL_USECS		4
+#define ENA_INTR_DELAY_OLD_VALUE_WEIGHT			6
+#define ENA_INTR_DELAY_NEW_VALUE_WEIGHT			4
+#define ENA_INTR_MODER_LEVEL_STRIDE			2
+#define ENA_INTR_BYTE_COUNT_NOT_SUPPORTED		0xFFFFFF
+
+enum ena_intr_moder_level {
+	ENA_INTR_MODER_LOWEST = 0,
+	ENA_INTR_MODER_LOW,
+	ENA_INTR_MODER_MID,
+	ENA_INTR_MODER_HIGH,
+	ENA_INTR_MODER_HIGHEST,
+	ENA_INTR_MAX_NUM_OF_LEVELS,
+};
+
+struct ena_intr_moder_entry {
+	unsigned int intr_moder_interval;
+	unsigned int pkts_per_interval;
+	unsigned int bytes_per_interval;
+};
+
+enum queue_direction {
+	ENA_COM_IO_QUEUE_DIRECTION_TX,
+	ENA_COM_IO_QUEUE_DIRECTION_RX
+};
+
+struct ena_com_buf {
+	dma_addr_t paddr; /**< Buffer physical address */
+	u16 len; /**< Buffer length in bytes */
+};
+
+struct ena_com_rx_buf_info {
+	u16 len;
+	u16 req_id;
+};
+
+struct ena_com_io_desc_addr {
+	u8 __iomem *pbuf_dev_addr; /* LLQ address */
+	u8 *virt_addr;
+	dma_addr_t phys_addr;
+};
+
+struct ena_com_tx_meta {
+	u16 mss;
+	u16 l3_hdr_len;
+	u16 l3_hdr_offset;
+	u16 l4_hdr_len; /* In words */
+};
+
+struct ena_com_io_cq {
+	struct ena_com_io_desc_addr cdesc_addr;
+
+	/* Interrupt unmask register */
+	u32 __iomem *unmask_reg;
+
+	/* The completion queue head doorbell register */
+	u32 __iomem *cq_head_db_reg;
+
+	/* numa configuration register (for TPH) */
+	u32 __iomem *numa_node_cfg_reg;
+
+	/* The value to write to the above register to unmask
+	 * the interrupt of this queue
+	 */
+	u32 msix_vector;
+
+	enum queue_direction direction;
+
+	/* holds the number of cdesc of the current packet */
+	u16 cur_rx_pkt_cdesc_count;
+	/* save the firt cdesc idx of the current packet */
+	u16 cur_rx_pkt_cdesc_start_idx;
+
+	u16 q_depth;
+	/* Caller qid */
+	u16 qid;
+
+	/* Device queue index */
+	u16 idx;
+	u16 head;
+	u16 last_head_update;
+	u8 phase;
+	u8 cdesc_entry_size_in_bytes;
+
+} ____cacheline_aligned;
+
+struct ena_com_io_sq {
+	struct ena_com_io_desc_addr desc_addr;
+
+	u32 __iomem *db_addr;
+	u8 __iomem *header_addr;
+
+	enum queue_direction direction;
+	enum ena_admin_placement_policy_type mem_queue_type;
+
+	u32 msix_vector;
+	struct ena_com_tx_meta cached_tx_meta;
+
+	u16 q_depth;
+	u16 qid;
+
+	u16 idx;
+	u16 tail;
+	u16 next_to_comp;
+	u32 tx_max_header_size;
+	u8 phase;
+	u8 desc_entry_size;
+	u8 dma_addr_bits;
+} ____cacheline_aligned;
+
+struct ena_com_admin_cq {
+	struct ena_admin_acq_entry *entries;
+	dma_addr_t dma_addr;
+
+	u16 head;
+	u8 phase;
+};
+
+struct ena_com_admin_sq {
+	struct ena_admin_aq_entry *entries;
+	dma_addr_t dma_addr;
+
+	u32 __iomem *db_addr;
+
+	u16 head;
+	u16 tail;
+	u8 phase;
+
+};
+
+struct ena_com_stats_admin {
+	u32 aborted_cmd;
+	u32 submitted_cmd;
+	u32 completed_cmd;
+	u32 out_of_space;
+	u32 no_completion;
+};
+
+struct ena_com_admin_queue {
+	void *q_dmadev;
+	spinlock_t q_lock; /* spinlock for the admin queue */
+	struct ena_comp_ctx *comp_ctx;
+	u16 q_depth;
+	struct ena_com_admin_cq cq;
+	struct ena_com_admin_sq sq;
+
+	/* Indicate if the admin queue should poll for completion */
+	bool polling;
+
+	u16 curr_cmd_id;
+
+	/* Indicate that the ena was initialized and can
+	 * process new admin commands
+	 */
+	bool running_state;
+
+	/* Count the number of outstanding admin commands */
+	atomic_t outstanding_cmds;
+
+	struct ena_com_stats_admin stats;
+};
+
+struct ena_aenq_handlers;
+
+struct ena_com_aenq {
+	u16 head;
+	u8 phase;
+	struct ena_admin_aenq_entry *entries;
+	dma_addr_t dma_addr;
+	u16 q_depth;
+	struct ena_aenq_handlers *aenq_handlers;
+};
+
+struct ena_com_mmio_read {
+	struct ena_admin_ena_mmio_req_read_less_resp *read_resp;
+	dma_addr_t read_resp_dma_addr;
+	u16 seq_num;
+	bool readless_supported;
+	/* spin lock to ensure a single outstanding read */
+	spinlock_t lock;
+};
+
+struct ena_rss {
+	/* Indirect table */
+	u16 *host_rss_ind_tbl;
+	struct ena_admin_rss_ind_table_entry *rss_ind_tbl;
+	dma_addr_t rss_ind_tbl_dma_addr;
+	u16 tbl_log_size;
+
+	/* Hash key */
+	enum ena_admin_hash_functions hash_func;
+	struct ena_admin_feature_rss_flow_hash_control *hash_key;
+	dma_addr_t hash_key_dma_addr;
+	u32 hash_init_val;
+
+	/* Flow Control */
+	struct ena_admin_feature_rss_hash_control *hash_ctrl;
+	dma_addr_t hash_ctrl_dma_addr;
+
+};
+
+struct ena_host_attribute {
+	/* Debug area */
+	u8 *debug_area_virt_addr;
+	dma_addr_t debug_area_dma_addr;
+	u32 debug_area_size;
+
+	/* Host information */
+	struct ena_admin_host_info *host_info;
+	dma_addr_t host_info_dma_addr;
+};
+
+/* Each ena_dev is a PCI function. */
+struct ena_com_dev {
+	struct ena_com_admin_queue admin_queue;
+	struct ena_com_aenq aenq;
+	struct ena_com_io_cq io_cq_queues[ENA_TOTAL_NUM_QUEUES];
+	struct ena_com_io_sq io_sq_queues[ENA_TOTAL_NUM_QUEUES];
+	u8 __iomem *reg_bar;
+	void __iomem *mem_bar;
+	void *dmadev;
+
+	enum ena_admin_placement_policy_type tx_mem_queue_type;
+	u32 tx_max_header_size;
+	u16 stats_func; /* Selected function for extended statistic dump */
+	u16 stats_queue; /* Selected queue for extended statistic dump */
+
+	struct ena_com_mmio_read mmio_read;
+
+	struct ena_rss rss;
+	u32 supported_features;
+	u32 dma_addr_bits;
+
+	struct ena_host_attribute host_attr;
+	bool adaptive_coalescing;
+	u16 intr_delay_resolution;
+	u32 intr_moder_tx_interval;
+	struct ena_intr_moder_entry *intr_moder_tbl;
+};
+
+struct ena_com_dev_get_features_ctx {
+	struct ena_admin_queue_feature_desc max_queues;
+	struct ena_admin_device_attr_feature_desc dev_attr;
+	struct ena_admin_feature_aenq_desc aenq;
+	struct ena_admin_feature_offload_desc offload;
+};
+
+struct ena_com_create_io_ctx {
+	enum ena_admin_placement_policy_type mem_queue_type;
+	enum queue_direction direction;
+	int numa_node;
+	u32 msix_vector;
+	u16 queue_size;
+	u16 qid;
+};
+
+typedef void (*ena_aenq_handler)(void *data,
+	struct ena_admin_aenq_entry *aenq_e);
+
+/* Holds aenq handlers. Indexed by AENQ event group */
+struct ena_aenq_handlers {
+	ena_aenq_handler handlers[ENA_MAX_HANDLERS];
+	ena_aenq_handler unimplemented_handler;
+};
+
+/*****************************************************************************/
+/*****************************************************************************/
+
+/* ena_com_mmio_reg_read_request_init - Init the mmio reg read mechanism
+ * @ena_dev: ENA communication layer struct
+ *
+ * Initialize the register read mechanism.
+ *
+ * @note: This method must be the first stage in the initialization sequence.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev);
+
+/* ena_com_set_mmio_read_mode - Enable/disable the mmio reg read mechanism
+ * @ena_dev: ENA communication layer struct
+ * @readless_supported: readless mode (enable/disable)
+ */
+void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev,
+				bool readless_supported);
+
+/* ena_com_mmio_reg_read_request_write_dev_addr - Write the mmio reg read return
+ * value physical address.
+ * @ena_dev: ENA communication layer struct
+ */
+void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev);
+
+/* ena_com_mmio_reg_read_request_destroy - Destroy the mmio reg read mechanism
+ * @ena_dev: ENA communication layer struct
+ */
+void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev);
+
+/* ena_com_admin_init - Init the admin and the async queues
+ * @ena_dev: ENA communication layer struct
+ * @aenq_handlers: Those handlers to be called upon event.
+ * @init_spinlock: Indicate if this method should init the admin spinlock or
+ * the spinlock was init before (for example, in a case of FLR).
+ *
+ * Initialize the admin submission and completion queues.
+ * Initialize the asynchronous events notification queues.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_admin_init(struct ena_com_dev *ena_dev,
+		       struct ena_aenq_handlers *aenq_handlers,
+		       bool init_spinlock);
+
+/* ena_com_admin_destroy - Destroy the admin and the async events queues.
+ * @ena_dev: ENA communication layer struct
+ *
+ * @note: Before calling this method, the caller must validate that the device
+ * won't send any additional admin completions/aenq.
+ * To achieve that, a FLR is recommended.
+ */
+void ena_com_admin_destroy(struct ena_com_dev *ena_dev);
+
+/* ena_com_dev_reset - Perform device FLR to the device.
+ * @ena_dev: ENA communication layer struct
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_dev_reset(struct ena_com_dev *ena_dev);
+
+/* ena_com_create_io_queue - Create io queue.
+ * @ena_dev: ENA communication layer struct
+ * @ctx - create context structure
+ *
+ * Create the submission and the completion queues.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
+			    struct ena_com_create_io_ctx *ctx);
+
+/* ena_com_destroy_io_queue - Destroy IO queue with the queue id - qid.
+ * @ena_dev: ENA communication layer struct
+ * @qid - the caller virtual queue id.
+ */
+void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid);
+
+/* ena_com_get_io_handlers - Return the io queue handlers
+ * @ena_dev: ENA communication layer struct
+ * @qid - the caller virtual queue id.
+ * @io_sq - IO submission queue handler
+ * @io_cq - IO completion queue handler.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid,
+			    struct ena_com_io_sq **io_sq,
+			    struct ena_com_io_cq **io_cq);
+
+/* ena_com_admin_aenq_enable - ENAble asynchronous event notifications
+ * @ena_dev: ENA communication layer struct
+ *
+ * After this method, aenq event can be received via AENQ.
+ */
+void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev);
+
+/* ena_com_set_admin_running_state - Set the state of the admin queue
+ * @ena_dev: ENA communication layer struct
+ *
+ * Change the state of the admin queue (enable/disable)
+ */
+void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state);
+
+/* ena_com_get_admin_running_state - Get the admin queue state
+ * @ena_dev: ENA communication layer struct
+ *
+ * Retrieve the state of the admin queue (enable/disable)
+ *
+ * @return - current polling mode (enable/disable)
+ */
+bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev);
+
+/* ena_com_set_admin_polling_mode - Set the admin completion queue polling mode
+ * @ena_dev: ENA communication layer struct
+ * @polling: ENAble/Disable polling mode
+ *
+ * Set the admin completion mode.
+ */
+void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling);
+
+/* ena_com_set_admin_polling_mode - Get the admin completion queue polling mode
+ * @ena_dev: ENA communication layer struct
+ *
+ * Get the admin completion mode.
+ * If polling mode is on, ena_com_execute_admin_command will perform a
+ * polling on the admin completion queue for the commands completion,
+ * otherwise it will wait on wait event.
+ *
+ * @return state
+ */
+bool ena_com_get_ena_admin_polling_mode(struct ena_com_dev *ena_dev);
+
+/* ena_com_admin_q_comp_intr_handler - admin queue interrupt handler
+ * @ena_dev: ENA communication layer struct
+ *
+ * This method go over the admin completion queue and wake up all the pending
+ * threads that wait on the commands wait event.
+ *
+ * @note: Should be called after MSI-X interrupt.
+ */
+void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev);
+
+/* ena_com_aenq_intr_handler - AENQ interrupt handler
+ * @ena_dev: ENA communication layer struct
+ *
+ * This method go over the async event notification queue and call the proper
+ * aenq handler.
+ */
+void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data);
+
+/* ena_com_abort_admin_commands - Abort all the outstanding admin commands.
+ * @ena_dev: ENA communication layer struct
+ *
+ * This method aborts all the outstanding admin commands.
+ * The caller should then call ena_com_wait_for_abort_completion to make sure
+ * all the commands were completed.
+ */
+void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev);
+
+/* ena_com_wait_for_abort_completion - Wait for admin commands abort.
+ * @ena_dev: ENA communication layer struct
+ *
+ * This method wait until all the outstanding admin commands will be completed.
+ */
+void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev);
+
+/* ena_com_validate_version - Validate the device parameters
+ * @ena_dev: ENA communication layer struct
+ *
+ * This method validate the device parameters are the same as the saved
+ * parameters in ena_dev.
+ * This method is useful after device reset, to validate the device mac address
+ * and the device offloads are the same as before the reset.
+ *
+ * @return - 0 on success negative value otherwise.
+ */
+int ena_com_validate_version(struct ena_com_dev *ena_dev);
+
+/* ena_com_get_link_params - Retrieve physical link parameters.
+ * @ena_dev: ENA communication layer struct
+ * @resp: Link parameters
+ *
+ * Retrieve the physical link parameters,
+ * like speed, auto-negotiation and full duplex support.
+ *
+ * @return - 0 on Success negative value otherwise.
+ */
+int ena_com_get_link_params(struct ena_com_dev *ena_dev,
+			    struct ena_admin_get_feat_resp *resp);
+
+/* ena_com_get_dma_width - Retrieve physical dma address width the device
+ * supports.
+ * @ena_dev: ENA communication layer struct
+ *
+ * Retrieve the maximum physical address bits the device can handle.
+ *
+ * @return: > 0 on Success and negative value otherwise.
+ */
+int ena_com_get_dma_width(struct ena_com_dev *ena_dev);
+
+/* ena_com_set_aenq_config - Set aenq groups configurations
+ * @ena_dev: ENA communication layer struct
+ * @groups flag: bit fields flags of enum ena_admin_aenq_group.
+ *
+ * Configure which aenq event group the driver would like to receive.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag);
+
+/* ena_com_get_dev_attr_feat - Get device features
+ * @ena_dev: ENA communication layer struct
+ * @get_feat_ctx: returned context that contain the get features.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
+			      struct ena_com_dev_get_features_ctx *get_feat_ctx);
+
+/* ena_com_get_dev_basic_stats - Get device basic statistics
+ * @ena_dev: ENA communication layer struct
+ * @stats: stats return value
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
+				struct ena_admin_basic_stats *stats);
+
+/* ena_com_set_dev_mtu - Configure the device mtu.
+ * @ena_dev: ENA communication layer struct
+ * @mtu: mtu value
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu);
+
+/* ena_com_get_offload_settings - Retrieve the device offloads capabilities
+ * @ena_dev: ENA communication layer struct
+ * @offlad: offload return value
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
+				 struct ena_admin_feature_offload_desc *offload);
+
+/* ena_com_rss_init - Init RSS
+ * @ena_dev: ENA communication layer struct
+ * @log_size: indirection log size
+ *
+ * Allocate RSS/RFS resources.
+ * The caller then can configure rss using ena_com_set_hash_function,
+ * ena_com_set_hash_ctrl and ena_com_indirect_table_set.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 log_size);
+
+/* ena_com_rss_destroy - Destroy rss
+ * @ena_dev: ENA communication layer struct
+ *
+ * Free all the RSS/RFS resources.
+ */
+void ena_com_rss_destroy(struct ena_com_dev *ena_dev);
+
+/* ena_com_fill_hash_function - Fill RSS hash function
+ * @ena_dev: ENA communication layer struct
+ * @func: The hash function (Toeplitz or crc)
+ * @key: Hash key (for toeplitz hash)
+ * @key_len: key length (max length 10 DW)
+ * @init_val: initial value for the hash function
+ *
+ * Fill the ena_dev resources with the desire hash function, hash key, key_len
+ * and key initial value (if needed by the hash function).
+ * To flush the key into the device the caller should call
+ * ena_com_set_hash_function.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
+			       enum ena_admin_hash_functions func,
+			       const u8 *key, u16 key_len, u32 init_val);
+
+/* ena_com_set_hash_function - Flush the hash function and it dependencies to
+ * the device.
+ * @ena_dev: ENA communication layer struct
+ *
+ * Flush the hash function and it dependencies (key, key length and
+ * initial value) if needed.
+ *
+ * @note: Prior to this method the caller should call ena_com_fill_hash_function
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_set_hash_function(struct ena_com_dev *ena_dev);
+
+/* ena_com_get_hash_function - Retrieve the hash function and the hash key
+ * from the device.
+ * @ena_dev: ENA communication layer struct
+ * @func: hash function
+ * @key: hash key
+ *
+ * Retrieve the hash function and the hash key from the device.
+ *
+ * @note: If the caller called ena_com_fill_hash_function but didn't flash
+ * it to the device, the new configuration will be lost.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_get_hash_function(struct ena_com_dev *ena_dev,
+			      enum ena_admin_hash_functions *func,
+			      u8 *key);
+
+/* ena_com_fill_hash_ctrl - Fill RSS hash control
+ * @ena_dev: ENA communication layer struct.
+ * @proto: The protocol to configure.
+ * @hash_fields: bit mask of ena_admin_flow_hash_fields
+ *
+ * Fill the ena_dev resources with the desire hash control (the ethernet
+ * fields that take part of the hash) for a specific protocol.
+ * To flush the hash control to the device, the caller should call
+ * ena_com_set_hash_ctrl.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
+			   enum ena_admin_flow_hash_proto proto,
+			   u16 hash_fields);
+
+/* ena_com_set_hash_ctrl - Flush the hash control resources to the device.
+ * @ena_dev: ENA communication layer struct
+ *
+ * Flush the hash control (the ethernet fields that take part of the hash)
+ *
+ * @note: Prior to this method the caller should call ena_com_fill_hash_ctrl.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev);
+
+/* ena_com_get_hash_ctrl - Retrieve the hash control from the device.
+ * @ena_dev: ENA communication layer struct
+ * @proto: The protocol to retrieve.
+ * @fields: bit mask of ena_admin_flow_hash_fields.
+ *
+ * Retrieve the hash control from the device.
+ *
+ * @note, If the caller called ena_com_fill_hash_ctrl but didn't flash
+ * it to the device, the new configuration will be lost.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev,
+			  enum ena_admin_flow_hash_proto proto,
+			  u16 *fields);
+
+/* ena_com_set_default_hash_ctrl - Set the hash control to a default
+ * configuration.
+ * @ena_dev: ENA communication layer struct
+ *
+ * Fill the ena_dev resources with the default hash control configuration.
+ * To flush the hash control to the device, the caller should call
+ * ena_com_set_hash_ctrl.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev);
+
+/* ena_com_indirect_table_fill_entry - Fill a single entry in the RSS
+ * indirection table
+ * @ena_dev: ENA communication layer struct.
+ * @entry_idx - indirection table entry.
+ * @entry_value - redirection value
+ *
+ * Fill a single entry of the RSS indirection table in the ena_dev resources.
+ * To flush the indirection table to the device, the called should call
+ * ena_com_indirect_table_set.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev,
+				      u16 entry_idx, u16 entry_value);
+
+/* ena_com_indirect_table_set - Flush the indirection table to the device.
+ * @ena_dev: ENA communication layer struct
+ *
+ * Flush the indirection hash control to the device.
+ * Prior to this method the caller should call ena_com_indirect_table_fill_entry
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_indirect_table_set(struct ena_com_dev *ena_dev);
+
+/* ena_com_indirect_table_get - Retrieve the indirection table from the device.
+ * @ena_dev: ENA communication layer struct
+ * @ind_tbl: indirection table
+ *
+ * Retrieve the RSS indirection table from the device.
+ *
+ * @note: If the caller called ena_com_indirect_table_fill_entry but didn't flash
+ * it to the device, the new configuration will be lost.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl);
+
+/* ena_com_allocate_host_info - Allocate host info resources.
+ * @ena_dev: ENA communication layer struct
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_allocate_host_info(struct ena_com_dev *ena_dev);
+
+/* ena_com_allocate_debug_area - Allocate debug area.
+ * @ena_dev: ENA communication layer struct
+ * @debug_area_size - debug area size.
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev,
+				u32 debug_area_size);
+
+/* ena_com_delete_debug_area - Free the debug area resources.
+ * @ena_dev: ENA communication layer struct
+ *
+ * Free the allocate debug area.
+ */
+void ena_com_delete_debug_area(struct ena_com_dev *ena_dev);
+
+/* ena_com_delete_host_info - Free the host info resources.
+ * @ena_dev: ENA communication layer struct
+ *
+ * Free the allocate host info.
+ */
+void ena_com_delete_host_info(struct ena_com_dev *ena_dev);
+
+/* ena_com_set_host_attributes - Update the device with the host
+ * attributes (debug area and host info) base address.
+ * @ena_dev: ENA communication layer struct
+ *
+ * @return: 0 on Success and negative value otherwise.
+ */
+int ena_com_set_host_attributes(struct ena_com_dev *ena_dev);
+
+/* ena_com_create_io_cq - Create io completion queue.
+ * @ena_dev: ENA communication layer struct
+ * @io_cq - io completion queue handler
+
+ * Create IO completion queue.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
+			 struct ena_com_io_cq *io_cq);
+
+/* ena_com_destroy_io_cq - Destroy io completion queue.
+ * @ena_dev: ENA communication layer struct
+ * @io_cq - io completion queue handler
+
+ * Destroy IO completion queue.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
+			  struct ena_com_io_cq *io_cq);
+
+/* ena_com_execute_admin_command - Execute admin command
+ * @admin_queue: admin queue.
+ * @cmd: the admin command to execute.
+ * @cmd_size: the command size.
+ * @cmd_completion: command completion return value.
+ * @cmd_comp_size: command completion size.
+
+ * Submit an admin command and then wait until the device will return a
+ * completion.
+ * The completion will be copyed into cmd_comp.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
+				  struct ena_admin_aq_entry *cmd,
+				  size_t cmd_size,
+				  struct ena_admin_acq_entry *cmd_comp,
+				  size_t cmd_comp_size);
+
+/* ena_com_init_interrupt_moderation - Init interrupt moderation
+ * @ena_dev: ENA communication layer struct
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev);
+
+/* ena_com_destroy_interrupt_moderation - Destroy interrupt moderation resources
+ * @ena_dev: ENA communication layer struct
+ */
+void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev);
+
+/* ena_com_interrupt_moderation_supported - Return if interrupt moderation
+ * capability is supported by the device.
+ *
+ * @return - supported or not.
+ */
+bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev);
+
+/* ena_com_config_default_interrupt_moderation_table - Restore the interrupt
+ * moderation table back to the default parameters.
+ * @ena_dev: ENA communication layer struct
+ */
+void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev);
+
+/* ena_com_update_nonadaptive_moderation_interval_tx - Update the
+ * non-adaptive interval in Tx direction.
+ * @ena_dev: ENA communication layer struct
+ * @tx_coalesce_usecs: Interval in usec.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
+						      u32 tx_coalesce_usecs);
+
+/* ena_com_update_nonadaptive_moderation_interval_rx - Update the
+ * non-adaptive interval in Rx direction.
+ * @ena_dev: ENA communication layer struct
+ * @rx_coalesce_usecs: Interval in usec.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
+						      u32 rx_coalesce_usecs);
+
+/* ena_com_get_nonadaptive_moderation_interval_tx - Retrieve the
+ * non-adaptive interval in Tx direction.
+ * @ena_dev: ENA communication layer struct
+ *
+ * @return - interval in usec
+ */
+unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev);
+
+/* ena_com_get_nonadaptive_moderation_interval_rx - Retrieve the
+ * non-adaptive interval in Rx direction.
+ * @ena_dev: ENA communication layer struct
+ *
+ * @return - interval in usec
+ */
+unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev);
+
+/* ena_com_init_intr_moderation_entry - Update a single entry in the interrupt
+ * moderation table.
+ * @ena_dev: ENA communication layer struct
+ * @level: Interrupt moderation table level
+ * @entry: Entry value
+ *
+ * Update a single entry in the interrupt moderation table.
+ */
+void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
+					enum ena_intr_moder_level level,
+					struct ena_intr_moder_entry *entry);
+
+/* ena_com_get_intr_moderation_entry - Init ena_intr_moder_entry.
+ * @ena_dev: ENA communication layer struct
+ * @level: Interrupt moderation table level
+ * @entry: Entry to fill.
+ *
+ * Initialize the entry according to the adaptive interrupt moderation table.
+ */
+void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
+				       enum ena_intr_moder_level level,
+				       struct ena_intr_moder_entry *entry);
+
+static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev)
+{
+	return ena_dev->adaptive_coalescing;
+}
+
+static inline void ena_com_enable_adaptive_moderation(struct ena_com_dev *ena_dev)
+{
+	ena_dev->adaptive_coalescing = true;
+}
+
+static inline void ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_dev)
+{
+	ena_dev->adaptive_coalescing = false;
+}
+
+/* ena_com_calculate_interrupt_delay - Calculate new interrupt delay
+ * @ena_dev: ENA communication layer struct
+ * @pkts: Number of packets since the last update
+ * @bytes: Number of bytes received since the last update.
+ * @smoothed_interval: Returned interval
+ * @moder_tbl_idx: Current table level as input update new level as return
+ * value.
+ */
+static inline void ena_com_calculate_interrupt_delay(struct ena_com_dev *ena_dev,
+						     unsigned int pkts,
+						     unsigned int bytes,
+						     unsigned int *smoothed_interval,
+						     unsigned int *moder_tbl_idx)
+{
+	enum ena_intr_moder_level curr_moder_idx, new_moder_idx;
+	struct ena_intr_moder_entry *curr_moder_entry;
+	struct ena_intr_moder_entry *pred_moder_entry;
+	struct ena_intr_moder_entry *new_moder_entry;
+	struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
+	unsigned int interval;
+
+	/* We apply adaptive moderation on Rx path only.
+	 * Tx uses static interrupt moderation.
+	 */
+	if (!pkts || !bytes)
+		/* Tx interrupt, or spurious interrupt,
+		 * in both cases we just use same delay values
+		 */
+		return;
+
+	curr_moder_idx = (enum ena_intr_moder_level)(*moder_tbl_idx);
+	if (unlikely(curr_moder_idx >= ENA_INTR_MAX_NUM_OF_LEVELS)) {
+		pr_err("Wrong moderation index %u\n", curr_moder_idx);
+		return;
+	}
+
+	curr_moder_entry = &intr_moder_tbl[curr_moder_idx];
+	new_moder_idx = curr_moder_idx;
+
+	if (curr_moder_idx == ENA_INTR_MODER_LOWEST) {
+		if ((pkts > curr_moder_entry->pkts_per_interval) ||
+		    (bytes > curr_moder_entry->bytes_per_interval))
+			new_moder_idx =
+				(enum ena_intr_moder_level)(curr_moder_idx + ENA_INTR_MODER_LEVEL_STRIDE);
+	} else {
+		pred_moder_entry = &intr_moder_tbl[curr_moder_idx - ENA_INTR_MODER_LEVEL_STRIDE];
+
+		if ((pkts <= pred_moder_entry->pkts_per_interval) ||
+		    (bytes <= pred_moder_entry->bytes_per_interval))
+			new_moder_idx =
+				(enum ena_intr_moder_level)(curr_moder_idx - ENA_INTR_MODER_LEVEL_STRIDE);
+		else if ((pkts > curr_moder_entry->pkts_per_interval) ||
+			 (bytes > curr_moder_entry->bytes_per_interval)) {
+			if (curr_moder_idx != ENA_INTR_MODER_HIGHEST)
+				new_moder_idx =
+					(enum ena_intr_moder_level)(curr_moder_idx + ENA_INTR_MODER_LEVEL_STRIDE);
+		}
+	}
+	new_moder_entry = &intr_moder_tbl[new_moder_idx];
+
+	interval = new_moder_entry->intr_moder_interval;
+	*smoothed_interval = (
+		(interval * ENA_INTR_DELAY_NEW_VALUE_WEIGHT +
+		ENA_INTR_DELAY_OLD_VALUE_WEIGHT * (*smoothed_interval)) + 5) /
+		10;
+
+	*moder_tbl_idx = new_moder_idx;
+}
+
+/* ena_com_update_intr_reg - Prepare interrupt register
+ * @intr_reg: interrupt register to update.
+ * @rx_delay_interval: Rx interval in usecs
+ * @tx_delay_interval: Tx interval in usecs
+ * @unmask: unask enable/disable
+ *
+ * Prepare interrupt update register with the supplied parameters.
+ */
+static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg,
+					   u32 rx_delay_interval,
+					   u32 tx_delay_interval,
+					   bool unmask)
+{
+	intr_reg->intr_control = 0;
+	intr_reg->intr_control |= rx_delay_interval &
+		ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK;
+
+	intr_reg->intr_control |=
+		(tx_delay_interval << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT)
+		& ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK;
+
+	if (unmask)
+		intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK;
+}
+
+#endif /* !(ENA_COM) */
diff --git a/drivers/net/ethernet/amazon/ena/ena_common_defs.h b/drivers/net/ethernet/amazon/ena/ena_common_defs.h
new file mode 100644
index 000000000000..bb8d73676eab
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_common_defs.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2015 - 2016 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef _ENA_COMMON_H_
+#define _ENA_COMMON_H_
+
+#define ENA_COMMON_SPEC_VERSION_MAJOR	0 /*  */
+#define ENA_COMMON_SPEC_VERSION_MINOR	10 /*  */
+
+/* ENA operates with 48-bit memory addresses. ena_mem_addr_t */
+struct ena_common_mem_addr {
+	u32 mem_addr_low;
+
+	u16 mem_addr_high;
+
+	/* MBZ */
+	u16 reserved16;
+};
+
+#endif /*_ENA_COMMON_H_ */
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.c b/drivers/net/ethernet/amazon/ena/ena_eth_com.c
new file mode 100644
index 000000000000..539c536464a5
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.c
@@ -0,0 +1,501 @@
+/*
+ * Copyright 2015 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "ena_eth_com.h"
+
+static inline struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc(
+	struct ena_com_io_cq *io_cq)
+{
+	struct ena_eth_io_rx_cdesc_base *cdesc;
+	u16 expected_phase, head_masked;
+	u16 desc_phase;
+
+	head_masked = io_cq->head & (io_cq->q_depth - 1);
+	expected_phase = io_cq->phase;
+
+	cdesc = (struct ena_eth_io_rx_cdesc_base *)(io_cq->cdesc_addr.virt_addr
+			+ (head_masked * io_cq->cdesc_entry_size_in_bytes));
+
+	desc_phase = (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >>
+			ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT;
+
+	if (desc_phase != expected_phase)
+		return NULL;
+
+	return cdesc;
+}
+
+static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq)
+{
+	io_cq->head++;
+
+	/* Switch phase bit in case of wrap around */
+	if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0))
+		io_cq->phase ^= 1;
+}
+
+static inline void *get_sq_desc(struct ena_com_io_sq *io_sq)
+{
+	u16 tail_masked;
+	u32 offset;
+
+	tail_masked = io_sq->tail & (io_sq->q_depth - 1);
+
+	offset = tail_masked * io_sq->desc_entry_size;
+
+	return (void *)((uintptr_t)io_sq->desc_addr.virt_addr + offset);
+}
+
+static inline void ena_com_copy_curr_sq_desc_to_dev(struct ena_com_io_sq *io_sq)
+{
+	u16 tail_masked = io_sq->tail & (io_sq->q_depth - 1);
+	u32 offset = tail_masked * io_sq->desc_entry_size;
+
+	/* In case this queue isn't a LLQ */
+	if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
+		return;
+
+	memcpy_toio(io_sq->desc_addr.pbuf_dev_addr + offset,
+		    io_sq->desc_addr.virt_addr + offset,
+		    io_sq->desc_entry_size);
+}
+
+static inline void ena_com_sq_update_tail(struct ena_com_io_sq *io_sq)
+{
+	io_sq->tail++;
+
+	/* Switch phase bit in case of wrap around */
+	if (unlikely((io_sq->tail & (io_sq->q_depth - 1)) == 0))
+		io_sq->phase ^= 1;
+}
+
+static inline int ena_com_write_header(struct ena_com_io_sq *io_sq,
+				       u8 *head_src, u16 header_len)
+{
+	u16 tail_masked = io_sq->tail & (io_sq->q_depth - 1);
+	u8 __iomem *dev_head_addr =
+		io_sq->header_addr + (tail_masked * io_sq->tx_max_header_size);
+
+	if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
+		return 0;
+
+	if (unlikely(!io_sq->header_addr)) {
+		pr_err("Push buffer header ptr is NULL\n");
+		return -EINVAL;
+	}
+
+	memcpy_toio(dev_head_addr, head_src, header_len);
+
+	return 0;
+}
+
+static inline struct ena_eth_io_rx_cdesc_base *
+	ena_com_rx_cdesc_idx_to_ptr(struct ena_com_io_cq *io_cq, u16 idx)
+{
+	idx &= (io_cq->q_depth - 1);
+	return (struct ena_eth_io_rx_cdesc_base *)
+		((uintptr_t)io_cq->cdesc_addr.virt_addr +
+		idx * io_cq->cdesc_entry_size_in_bytes);
+}
+
+static inline u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
+					   u16 *first_cdesc_idx)
+{
+	struct ena_eth_io_rx_cdesc_base *cdesc;
+	u16 count = 0, head_masked;
+	u32 last = 0;
+
+	do {
+		cdesc = ena_com_get_next_rx_cdesc(io_cq);
+		if (!cdesc)
+			break;
+
+		ena_com_cq_inc_head(io_cq);
+		count++;
+		last = (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >>
+			ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT;
+	} while (!last);
+
+	if (last) {
+		*first_cdesc_idx = io_cq->cur_rx_pkt_cdesc_start_idx;
+		count += io_cq->cur_rx_pkt_cdesc_count;
+
+		head_masked = io_cq->head & (io_cq->q_depth - 1);
+
+		io_cq->cur_rx_pkt_cdesc_count = 0;
+		io_cq->cur_rx_pkt_cdesc_start_idx = head_masked;
+
+		pr_debug("ena q_id: %d packets were completed. first desc idx %u descs# %d\n",
+			 io_cq->qid, *first_cdesc_idx, count);
+	} else {
+		io_cq->cur_rx_pkt_cdesc_count += count;
+		count = 0;
+	}
+
+	return count;
+}
+
+static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq,
+					     struct ena_com_tx_ctx *ena_tx_ctx)
+{
+	int rc;
+
+	if (ena_tx_ctx->meta_valid) {
+		rc = memcmp(&io_sq->cached_tx_meta,
+			    &ena_tx_ctx->ena_meta,
+			    sizeof(struct ena_com_tx_meta));
+
+		if (unlikely(rc != 0))
+			return true;
+	}
+
+	return false;
+}
+
+static inline void ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq,
+							 struct ena_com_tx_ctx *ena_tx_ctx)
+{
+	struct ena_eth_io_tx_meta_desc *meta_desc = NULL;
+	struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
+
+	meta_desc = get_sq_desc(io_sq);
+	memset(meta_desc, 0x0, sizeof(struct ena_eth_io_tx_meta_desc));
+
+	meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_DESC_MASK;
+
+	meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK;
+
+	/* bits 0-9 of the mss */
+	meta_desc->word2 |= (ena_meta->mss <<
+		ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT) &
+		ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK;
+	/* bits 10-13 of the mss */
+	meta_desc->len_ctrl |= ((ena_meta->mss >> 10) <<
+		ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT) &
+		ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK;
+
+	/* Extended meta desc */
+	meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK;
+	meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
+	meta_desc->len_ctrl |= (io_sq->phase <<
+		ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) &
+		ENA_ETH_IO_TX_META_DESC_PHASE_MASK;
+
+	meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_FIRST_MASK;
+	meta_desc->word2 |= ena_meta->l3_hdr_len &
+		ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK;
+	meta_desc->word2 |= (ena_meta->l3_hdr_offset <<
+		ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT) &
+		ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK;
+
+	meta_desc->word2 |= (ena_meta->l4_hdr_len <<
+		ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) &
+		ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK;
+
+	meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
+
+	/* Cached the meta desc */
+	memcpy(&io_sq->cached_tx_meta, ena_meta,
+	       sizeof(struct ena_com_tx_meta));
+
+	ena_com_copy_curr_sq_desc_to_dev(io_sq);
+	ena_com_sq_update_tail(io_sq);
+}
+
+static inline void ena_com_rx_set_flags(struct ena_com_rx_ctx *ena_rx_ctx,
+					struct ena_eth_io_rx_cdesc_base *cdesc)
+{
+	ena_rx_ctx->l3_proto = cdesc->status &
+		ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK;
+	ena_rx_ctx->l4_proto =
+		(cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) >>
+		ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT;
+	ena_rx_ctx->l3_csum_err =
+		(cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK) >>
+		ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT;
+	ena_rx_ctx->l4_csum_err =
+		(cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK) >>
+		ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT;
+	ena_rx_ctx->hash = cdesc->hash;
+	ena_rx_ctx->frag =
+		(cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK) >>
+		ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT;
+
+	pr_debug("ena_rx_ctx->l3_proto %d ena_rx_ctx->l4_proto %d\nena_rx_ctx->l3_csum_err %d ena_rx_ctx->l4_csum_err %d\nhash frag %d frag: %d cdesc_status: %x\n",
+		 ena_rx_ctx->l3_proto, ena_rx_ctx->l4_proto,
+		 ena_rx_ctx->l3_csum_err, ena_rx_ctx->l4_csum_err,
+		 ena_rx_ctx->hash, ena_rx_ctx->frag, cdesc->status);
+}
+
+/*****************************************************************************/
+/*****************************     API      **********************************/
+/*****************************************************************************/
+
+int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
+		       struct ena_com_tx_ctx *ena_tx_ctx,
+		       int *nb_hw_desc)
+{
+	struct ena_eth_io_tx_desc *desc = NULL;
+	struct ena_com_buf *ena_bufs = ena_tx_ctx->ena_bufs;
+	void *push_header = ena_tx_ctx->push_header;
+	u16 header_len = ena_tx_ctx->header_len;
+	u16 num_bufs = ena_tx_ctx->num_bufs;
+	int total_desc, i, rc;
+	bool have_meta;
+	u64 addr_hi;
+
+	WARN(io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_TX, "wrong Q type");
+
+	/* num_bufs +1 for potential meta desc */
+	if (ena_com_sq_empty_space(io_sq) < (num_bufs + 1)) {
+		pr_err("Not enough space in the tx queue\n");
+		return -ENOMEM;
+	}
+
+	if (unlikely(header_len > io_sq->tx_max_header_size)) {
+		pr_err("header size is too large %d max header: %d\n",
+		       header_len, io_sq->tx_max_header_size);
+		return -EINVAL;
+	}
+
+	/* start with pushing the header (if needed) */
+	rc = ena_com_write_header(io_sq, push_header, header_len);
+	if (unlikely(rc))
+		return rc;
+
+	have_meta = ena_tx_ctx->meta_valid && ena_com_meta_desc_changed(io_sq,
+			ena_tx_ctx);
+	if (have_meta)
+		ena_com_create_and_store_tx_meta_desc(io_sq, ena_tx_ctx);
+
+	/* If the caller doesn't want send packets */
+	if (unlikely(!num_bufs && !header_len)) {
+		*nb_hw_desc = have_meta ? 0 : 1;
+		return 0;
+	}
+
+	desc = get_sq_desc(io_sq);
+	memset(desc, 0x0, sizeof(struct ena_eth_io_tx_desc));
+
+	/* Set first desc when we don't have meta descriptor */
+	if (!have_meta)
+		desc->len_ctrl |= ENA_ETH_IO_TX_DESC_FIRST_MASK;
+
+	desc->buff_addr_hi_hdr_sz |= (header_len <<
+		ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT) &
+		ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK;
+	desc->len_ctrl |= (io_sq->phase << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) &
+		ENA_ETH_IO_TX_DESC_PHASE_MASK;
+
+	desc->len_ctrl |= ENA_ETH_IO_TX_DESC_COMP_REQ_MASK;
+
+	/* Bits 0-9 */
+	desc->meta_ctrl |= (ena_tx_ctx->req_id <<
+		ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT) &
+		ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK;
+
+	desc->meta_ctrl |= (ena_tx_ctx->df <<
+		ENA_ETH_IO_TX_DESC_DF_SHIFT) &
+		ENA_ETH_IO_TX_DESC_DF_MASK;
+
+	/* Bits 10-15 */
+	desc->len_ctrl |= ((ena_tx_ctx->req_id >> 10) <<
+		ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT) &
+		ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK;
+
+	if (ena_tx_ctx->meta_valid) {
+		desc->meta_ctrl |= (ena_tx_ctx->tso_enable <<
+			ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT) &
+			ENA_ETH_IO_TX_DESC_TSO_EN_MASK;
+		desc->meta_ctrl |= ena_tx_ctx->l3_proto &
+			ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK;
+		desc->meta_ctrl |= (ena_tx_ctx->l4_proto <<
+			ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT) &
+			ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK;
+		desc->meta_ctrl |= (ena_tx_ctx->l3_csum_enable <<
+			ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT) &
+			ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK;
+		desc->meta_ctrl |= (ena_tx_ctx->l4_csum_enable <<
+			ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT) &
+			ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK;
+		desc->meta_ctrl |= (ena_tx_ctx->l4_csum_partial <<
+			ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT) &
+			ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK;
+	}
+
+	for (i = 0; i < num_bufs; i++) {
+		/* The first desc share the same desc as the header */
+		if (likely(i != 0)) {
+			ena_com_copy_curr_sq_desc_to_dev(io_sq);
+			ena_com_sq_update_tail(io_sq);
+
+			desc = get_sq_desc(io_sq);
+			memset(desc, 0x0, sizeof(struct ena_eth_io_tx_desc));
+
+			desc->len_ctrl |= (io_sq->phase <<
+				ENA_ETH_IO_TX_DESC_PHASE_SHIFT) &
+				ENA_ETH_IO_TX_DESC_PHASE_MASK;
+		}
+
+		desc->len_ctrl |= ena_bufs->len &
+			ENA_ETH_IO_TX_DESC_LENGTH_MASK;
+
+		addr_hi = ((ena_bufs->paddr &
+			GENMASK_ULL(io_sq->dma_addr_bits - 1, 32)) >> 32);
+
+		desc->buff_addr_lo = (u32)ena_bufs->paddr;
+		desc->buff_addr_hi_hdr_sz |= addr_hi &
+			ENA_ETH_IO_TX_DESC_ADDR_HI_MASK;
+		ena_bufs++;
+	}
+
+	/* set the last desc indicator */
+	desc->len_ctrl |= ENA_ETH_IO_TX_DESC_LAST_MASK;
+
+	ena_com_copy_curr_sq_desc_to_dev(io_sq);
+
+	ena_com_sq_update_tail(io_sq);
+
+	total_desc = max_t(u16, num_bufs, 1);
+	total_desc += have_meta ? 1 : 0;
+
+	*nb_hw_desc = total_desc;
+	return 0;
+}
+
+int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
+		   struct ena_com_io_sq *io_sq,
+		   struct ena_com_rx_ctx *ena_rx_ctx)
+{
+	struct ena_com_rx_buf_info *ena_buf = &ena_rx_ctx->ena_bufs[0];
+	struct ena_eth_io_rx_cdesc_base *cdesc = NULL;
+	u16 cdesc_idx = 0;
+	u16 nb_hw_desc;
+	u16 i;
+
+	WARN(io_cq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX, "wrong Q type");
+
+	nb_hw_desc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx);
+	if (nb_hw_desc == 0) {
+		ena_rx_ctx->descs = nb_hw_desc;
+		return 0;
+	}
+
+	pr_debug("fetch rx packet: queue %d completed desc: %d\n", io_cq->qid,
+		 nb_hw_desc);
+
+	if (unlikely(nb_hw_desc > ena_rx_ctx->max_bufs)) {
+		pr_err("Too many RX cdescs (%d) > MAX(%d)\n", nb_hw_desc,
+		       ena_rx_ctx->max_bufs);
+		return -ENOSPC;
+	}
+
+	for (i = 0; i < nb_hw_desc; i++) {
+		cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx + i);
+
+		ena_buf->len = cdesc->length;
+		ena_buf->req_id = cdesc->req_id;
+		ena_buf++;
+	}
+
+	/* Update SQ head ptr */
+	io_sq->next_to_comp += nb_hw_desc;
+
+	pr_debug("[%s][QID#%d] Updating SQ head to: %d\n", __func__, io_sq->qid,
+		 io_sq->next_to_comp);
+
+	/* Get rx flags from the last pkt */
+	ena_com_rx_set_flags(ena_rx_ctx, cdesc);
+
+	ena_rx_ctx->descs = nb_hw_desc;
+	return 0;
+}
+
+int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
+			       struct ena_com_buf *ena_buf,
+			       u16 req_id)
+{
+	struct ena_eth_io_rx_desc *desc;
+
+	WARN(io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX, "wrong Q type");
+
+	if (unlikely(ena_com_sq_empty_space(io_sq) == 0))
+		return -ENOSPC;
+
+	desc = get_sq_desc(io_sq);
+	memset(desc, 0x0, sizeof(struct ena_eth_io_rx_desc));
+
+	desc->length = ena_buf->len;
+
+	desc->ctrl |= ENA_ETH_IO_RX_DESC_FIRST_MASK;
+	desc->ctrl |= ENA_ETH_IO_RX_DESC_LAST_MASK;
+	desc->ctrl |= io_sq->phase & ENA_ETH_IO_RX_DESC_PHASE_MASK;
+	desc->ctrl |= ENA_ETH_IO_RX_DESC_COMP_REQ_MASK;
+
+	desc->req_id = req_id;
+
+	desc->buff_addr_lo = (u32)ena_buf->paddr;
+	desc->buff_addr_hi =
+		((ena_buf->paddr & GENMASK_ULL(io_sq->dma_addr_bits - 1, 32)) >> 32);
+
+	ena_com_sq_update_tail(io_sq);
+
+	return 0;
+}
+
+int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id)
+{
+	u8 expected_phase, cdesc_phase;
+	struct ena_eth_io_tx_cdesc *cdesc;
+	u16 masked_head;
+
+	masked_head = io_cq->head & (io_cq->q_depth - 1);
+	expected_phase = io_cq->phase;
+
+	cdesc = (struct ena_eth_io_tx_cdesc *)
+		((uintptr_t)io_cq->cdesc_addr.virt_addr +
+		(masked_head * io_cq->cdesc_entry_size_in_bytes));
+
+	/* When the current completion descriptor phase isn't the same as the
+	 * expected, it mean that the device still didn't update
+	 * this completion.
+	 */
+	cdesc_phase = cdesc->flags & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
+	if (cdesc_phase != expected_phase)
+		return -EAGAIN;
+
+	ena_com_cq_inc_head(io_cq);
+
+	*req_id = cdesc->req_id;
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.h b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
new file mode 100644
index 000000000000..bb53c3a4f8e9
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2015 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef ENA_ETH_COM_H_
+#define ENA_ETH_COM_H_
+
+#include "ena_com.h"
+
+/* head update threshold in units of (queue size / ENA_COMP_HEAD_THRESH) */
+#define ENA_COMP_HEAD_THRESH 4
+
+struct ena_com_tx_ctx {
+	struct ena_com_tx_meta ena_meta;
+	struct ena_com_buf *ena_bufs;
+	/* For LLQ, header buffer - pushed to the device mem space */
+	void *push_header;
+
+	enum ena_eth_io_l3_proto_index l3_proto;
+	enum ena_eth_io_l4_proto_index l4_proto;
+	u16 num_bufs;
+	u16 req_id;
+	/* For regular queue, indicate the size of the header
+	 * For LLQ, indicate the size of the pushed buffer
+	 */
+	u16 header_len;
+
+	u8 meta_valid;
+	u8 tso_enable;
+	u8 l3_csum_enable;
+	u8 l4_csum_enable;
+	u8 l4_csum_partial;
+	u8 df; /* Don't fragment */
+};
+
+struct ena_com_rx_ctx {
+	struct ena_com_rx_buf_info *ena_bufs;
+	enum ena_eth_io_l3_proto_index l3_proto;
+	enum ena_eth_io_l4_proto_index l4_proto;
+	bool l3_csum_err;
+	bool l4_csum_err;
+	/* fragmented packet */
+	bool frag;
+	u32 hash;
+	u16 descs;
+	int max_bufs;
+};
+
+int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
+		       struct ena_com_tx_ctx *ena_tx_ctx,
+		       int *nb_hw_desc);
+
+int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
+		   struct ena_com_io_sq *io_sq,
+		   struct ena_com_rx_ctx *ena_rx_ctx);
+
+int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
+			       struct ena_com_buf *ena_buf,
+			       u16 req_id);
+
+int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id);
+
+static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,
+				       struct ena_eth_io_intr_reg *intr_reg)
+{
+	writel(intr_reg->intr_control, io_cq->unmask_reg);
+}
+
+static inline int ena_com_sq_empty_space(struct ena_com_io_sq *io_sq)
+{
+	u16 tail, next_to_comp, cnt;
+
+	next_to_comp = io_sq->next_to_comp;
+	tail = io_sq->tail;
+	cnt = tail - next_to_comp;
+
+	return io_sq->q_depth - 1 - cnt;
+}
+
+static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
+{
+	u16 tail;
+
+	tail = io_sq->tail;
+
+	pr_debug("write submission queue doorbell for queue: %d tail: %d\n",
+		 io_sq->qid, tail);
+
+	writel(tail, io_sq->db_addr);
+
+	return 0;
+}
+
+static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq)
+{
+	u16 unreported_comp, head;
+	bool need_update;
+
+	head = io_cq->head;
+	unreported_comp = head - io_cq->last_head_update;
+	need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH);
+
+	if (io_cq->cq_head_db_reg && need_update) {
+		pr_debug("Write completion queue doorbell for queue %d: head: %d\n",
+			 io_cq->qid, head);
+		writel(head, io_cq->cq_head_db_reg);
+		io_cq->last_head_update = head;
+	}
+
+	return 0;
+}
+
+static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq,
+					    u8 numa_node)
+{
+	struct ena_eth_io_numa_node_cfg_reg numa_cfg;
+
+	if (!io_cq->numa_node_cfg_reg)
+		return;
+
+	numa_cfg.numa_cfg = (numa_node & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK)
+		| ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK;
+
+	writel(numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg);
+}
+
+static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem)
+{
+	io_sq->next_to_comp += elem;
+}
+
+#endif /* ENA_ETH_COM_H_ */
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_io_defs.h b/drivers/net/ethernet/amazon/ena/ena_eth_io_defs.h
new file mode 100644
index 000000000000..f320c58793a5
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_io_defs.h
@@ -0,0 +1,416 @@
+/*
+ * Copyright 2015 - 2016 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef _ENA_ETH_IO_H_
+#define _ENA_ETH_IO_H_
+
+enum ena_eth_io_l3_proto_index {
+	ENA_ETH_IO_L3_PROTO_UNKNOWN	= 0,
+
+	ENA_ETH_IO_L3_PROTO_IPV4	= 8,
+
+	ENA_ETH_IO_L3_PROTO_IPV6	= 11,
+
+	ENA_ETH_IO_L3_PROTO_FCOE	= 21,
+
+	ENA_ETH_IO_L3_PROTO_ROCE	= 22,
+};
+
+enum ena_eth_io_l4_proto_index {
+	ENA_ETH_IO_L4_PROTO_UNKNOWN		= 0,
+
+	ENA_ETH_IO_L4_PROTO_TCP			= 12,
+
+	ENA_ETH_IO_L4_PROTO_UDP			= 13,
+
+	ENA_ETH_IO_L4_PROTO_ROUTEABLE_ROCE	= 23,
+};
+
+struct ena_eth_io_tx_desc {
+	/* 15:0 : length - Buffer length in bytes, must
+	 *    include any packet trailers that the ENA supposed
+	 *    to update like End-to-End CRC, Authentication GMAC
+	 *    etc. This length must not include the
+	 *    'Push_Buffer' length. This length must not include
+	 *    the 4-byte added in the end for 802.3 Ethernet FCS
+	 * 21:16 : req_id_hi - Request ID[15:10]
+	 * 22 : reserved22 - MBZ
+	 * 23 : meta_desc - MBZ
+	 * 24 : phase
+	 * 25 : reserved1 - MBZ
+	 * 26 : first - Indicates first descriptor in
+	 *    transaction
+	 * 27 : last - Indicates last descriptor in
+	 *    transaction
+	 * 28 : comp_req - Indicates whether completion
+	 *    should be posted, after packet is transmitted.
+	 *    Valid only for first descriptor
+	 * 30:29 : reserved29 - MBZ
+	 * 31 : reserved31 - MBZ
+	 */
+	u32 len_ctrl;
+
+	/* 3:0 : l3_proto_idx - L3 protocol. This field
+	 *    required when l3_csum_en,l3_csum or tso_en are set.
+	 * 4 : DF - IPv4 DF, must be 0 if packet is IPv4 and
+	 *    DF flags of the IPv4 header is 0. Otherwise must
+	 *    be set to 1
+	 * 6:5 : reserved5
+	 * 7 : tso_en - Enable TSO, For TCP only.
+	 * 12:8 : l4_proto_idx - L4 protocol. This field need
+	 *    to be set when l4_csum_en or tso_en are set.
+	 * 13 : l3_csum_en - enable IPv4 header checksum.
+	 * 14 : l4_csum_en - enable TCP/UDP checksum.
+	 * 15 : ethernet_fcs_dis - when set, the controller
+	 *    will not append the 802.3 Ethernet Frame Check
+	 *    Sequence to the packet
+	 * 16 : reserved16
+	 * 17 : l4_csum_partial - L4 partial checksum. when
+	 *    set to 0, the ENA calculates the L4 checksum,
+	 *    where the Destination Address required for the
+	 *    TCP/UDP pseudo-header is taken from the actual
+	 *    packet L3 header. when set to 1, the ENA doesn't
+	 *    calculate the sum of the pseudo-header, instead,
+	 *    the checksum field of the L4 is used instead. When
+	 *    TSO enabled, the checksum of the pseudo-header
+	 *    must not include the tcp length field. L4 partial
+	 *    checksum should be used for IPv6 packet that
+	 *    contains Routing Headers.
+	 * 20:18 : reserved18 - MBZ
+	 * 21 : reserved21 - MBZ
+	 * 31:22 : req_id_lo - Request ID[9:0]
+	 */
+	u32 meta_ctrl;
+
+	u32 buff_addr_lo;
+
+	/* address high and header size
+	 * 15:0 : addr_hi - Buffer Pointer[47:32]
+	 * 23:16 : reserved16_w2
+	 * 31:24 : header_length - Header length. For Low
+	 *    Latency Queues, this fields indicates the number
+	 *    of bytes written to the headers' memory. For
+	 *    normal queues, if packet is TCP or UDP, and longer
+	 *    than max_header_size, then this field should be
+	 *    set to the sum of L4 header offset and L4 header
+	 *    size(without options), otherwise, this field
+	 *    should be set to 0. For both modes, this field
+	 *    must not exceed the max_header_size.
+	 *    max_header_size value is reported by the Max
+	 *    Queues Feature descriptor
+	 */
+	u32 buff_addr_hi_hdr_sz;
+};
+
+struct ena_eth_io_tx_meta_desc {
+	/* 9:0 : req_id_lo - Request ID[9:0]
+	 * 11:10 : reserved10 - MBZ
+	 * 12 : reserved12 - MBZ
+	 * 13 : reserved13 - MBZ
+	 * 14 : ext_valid - if set, offset fields in Word2
+	 *    are valid Also MSS High in Word 0 and bits [31:24]
+	 *    in Word 3
+	 * 15 : reserved15
+	 * 19:16 : mss_hi
+	 * 20 : eth_meta_type - 0: Tx Metadata Descriptor, 1:
+	 *    Extended Metadata Descriptor
+	 * 21 : meta_store - Store extended metadata in queue
+	 *    cache
+	 * 22 : reserved22 - MBZ
+	 * 23 : meta_desc - MBO
+	 * 24 : phase
+	 * 25 : reserved25 - MBZ
+	 * 26 : first - Indicates first descriptor in
+	 *    transaction
+	 * 27 : last - Indicates last descriptor in
+	 *    transaction
+	 * 28 : comp_req - Indicates whether completion
+	 *    should be posted, after packet is transmitted.
+	 *    Valid only for first descriptor
+	 * 30:29 : reserved29 - MBZ
+	 * 31 : reserved31 - MBZ
+	 */
+	u32 len_ctrl;
+
+	/* 5:0 : req_id_hi
+	 * 31:6 : reserved6 - MBZ
+	 */
+	u32 word1;
+
+	/* 7:0 : l3_hdr_len
+	 * 15:8 : l3_hdr_off
+	 * 21:16 : l4_hdr_len_in_words - counts the L4 header
+	 *    length in words. there is an explicit assumption
+	 *    that L4 header appears right after L3 header and
+	 *    L4 offset is based on l3_hdr_off+l3_hdr_len
+	 * 31:22 : mss_lo
+	 */
+	u32 word2;
+
+	u32 reserved;
+};
+
+struct ena_eth_io_tx_cdesc {
+	/* Request ID[15:0] */
+	u16 req_id;
+
+	u8 status;
+
+	/* flags
+	 * 0 : phase
+	 * 7:1 : reserved1
+	 */
+	u8 flags;
+
+	u16 sub_qid;
+
+	u16 sq_head_idx;
+};
+
+struct ena_eth_io_rx_desc {
+	/* In bytes. 0 means 64KB */
+	u16 length;
+
+	/* MBZ */
+	u8 reserved2;
+
+	/* 0 : phase
+	 * 1 : reserved1 - MBZ
+	 * 2 : first - Indicates first descriptor in
+	 *    transaction
+	 * 3 : last - Indicates last descriptor in transaction
+	 * 4 : comp_req
+	 * 5 : reserved5 - MBO
+	 * 7:6 : reserved6 - MBZ
+	 */
+	u8 ctrl;
+
+	u16 req_id;
+
+	/* MBZ */
+	u16 reserved6;
+
+	u32 buff_addr_lo;
+
+	u16 buff_addr_hi;
+
+	/* MBZ */
+	u16 reserved16_w3;
+};
+
+/* 4-word format Note: all ethernet parsing information are valid only when
+ * last=1
+ */
+struct ena_eth_io_rx_cdesc_base {
+	/* 4:0 : l3_proto_idx
+	 * 6:5 : src_vlan_cnt
+	 * 7 : reserved7 - MBZ
+	 * 12:8 : l4_proto_idx
+	 * 13 : l3_csum_err - when set, either the L3
+	 *    checksum error detected, or, the controller didn't
+	 *    validate the checksum. This bit is valid only when
+	 *    l3_proto_idx indicates IPv4 packet
+	 * 14 : l4_csum_err - when set, either the L4
+	 *    checksum error detected, or, the controller didn't
+	 *    validate the checksum. This bit is valid only when
+	 *    l4_proto_idx indicates TCP/UDP packet, and,
+	 *    ipv4_frag is not set
+	 * 15 : ipv4_frag - Indicates IPv4 fragmented packet
+	 * 23:16 : reserved16
+	 * 24 : phase
+	 * 25 : l3_csum2 - second checksum engine result
+	 * 26 : first - Indicates first descriptor in
+	 *    transaction
+	 * 27 : last - Indicates last descriptor in
+	 *    transaction
+	 * 29:28 : reserved28
+	 * 30 : buffer - 0: Metadata descriptor. 1: Buffer
+	 *    Descriptor was used
+	 * 31 : reserved31
+	 */
+	u32 status;
+
+	u16 length;
+
+	u16 req_id;
+
+	/* 32-bit hash result */
+	u32 hash;
+
+	u16 sub_qid;
+
+	u16 reserved;
+};
+
+/* 8-word format */
+struct ena_eth_io_rx_cdesc_ext {
+	struct ena_eth_io_rx_cdesc_base base;
+
+	u32 buff_addr_lo;
+
+	u16 buff_addr_hi;
+
+	u16 reserved16;
+
+	u32 reserved_w6;
+
+	u32 reserved_w7;
+};
+
+struct ena_eth_io_intr_reg {
+	/* 14:0 : rx_intr_delay
+	 * 29:15 : tx_intr_delay
+	 * 30 : intr_unmask
+	 * 31 : reserved
+	 */
+	u32 intr_control;
+};
+
+struct ena_eth_io_numa_node_cfg_reg {
+	/* 7:0 : numa
+	 * 30:8 : reserved
+	 * 31 : enabled
+	 */
+	u32 numa_cfg;
+};
+
+/* tx_desc */
+#define ENA_ETH_IO_TX_DESC_LENGTH_MASK GENMASK(15, 0)
+#define ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT 16
+#define ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK GENMASK(21, 16)
+#define ENA_ETH_IO_TX_DESC_META_DESC_SHIFT 23
+#define ENA_ETH_IO_TX_DESC_META_DESC_MASK BIT(23)
+#define ENA_ETH_IO_TX_DESC_PHASE_SHIFT 24
+#define ENA_ETH_IO_TX_DESC_PHASE_MASK BIT(24)
+#define ENA_ETH_IO_TX_DESC_FIRST_SHIFT 26
+#define ENA_ETH_IO_TX_DESC_FIRST_MASK BIT(26)
+#define ENA_ETH_IO_TX_DESC_LAST_SHIFT 27
+#define ENA_ETH_IO_TX_DESC_LAST_MASK BIT(27)
+#define ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT 28
+#define ENA_ETH_IO_TX_DESC_COMP_REQ_MASK BIT(28)
+#define ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK GENMASK(3, 0)
+#define ENA_ETH_IO_TX_DESC_DF_SHIFT 4
+#define ENA_ETH_IO_TX_DESC_DF_MASK BIT(4)
+#define ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT 7
+#define ENA_ETH_IO_TX_DESC_TSO_EN_MASK BIT(7)
+#define ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT 8
+#define ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK GENMASK(12, 8)
+#define ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT 13
+#define ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK BIT(13)
+#define ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT 14
+#define ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK BIT(14)
+#define ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT 15
+#define ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK BIT(15)
+#define ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT 17
+#define ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK BIT(17)
+#define ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT 22
+#define ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK GENMASK(31, 22)
+#define ENA_ETH_IO_TX_DESC_ADDR_HI_MASK GENMASK(15, 0)
+#define ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT 24
+#define ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK GENMASK(31, 24)
+
+/* tx_meta_desc */
+#define ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK GENMASK(9, 0)
+#define ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT 14
+#define ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK BIT(14)
+#define ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT 16
+#define ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK GENMASK(19, 16)
+#define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT 20
+#define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK BIT(20)
+#define ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT 21
+#define ENA_ETH_IO_TX_META_DESC_META_STORE_MASK BIT(21)
+#define ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT 23
+#define ENA_ETH_IO_TX_META_DESC_META_DESC_MASK BIT(23)
+#define ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT 24
+#define ENA_ETH_IO_TX_META_DESC_PHASE_MASK BIT(24)
+#define ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT 26
+#define ENA_ETH_IO_TX_META_DESC_FIRST_MASK BIT(26)
+#define ENA_ETH_IO_TX_META_DESC_LAST_SHIFT 27
+#define ENA_ETH_IO_TX_META_DESC_LAST_MASK BIT(27)
+#define ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT 28
+#define ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK BIT(28)
+#define ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK GENMASK(5, 0)
+#define ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK GENMASK(7, 0)
+#define ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT 8
+#define ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK GENMASK(15, 8)
+#define ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT 16
+#define ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK GENMASK(21, 16)
+#define ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT 22
+#define ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK GENMASK(31, 22)
+
+/* tx_cdesc */
+#define ENA_ETH_IO_TX_CDESC_PHASE_MASK BIT(0)
+
+/* rx_desc */
+#define ENA_ETH_IO_RX_DESC_PHASE_MASK BIT(0)
+#define ENA_ETH_IO_RX_DESC_FIRST_SHIFT 2
+#define ENA_ETH_IO_RX_DESC_FIRST_MASK BIT(2)
+#define ENA_ETH_IO_RX_DESC_LAST_SHIFT 3
+#define ENA_ETH_IO_RX_DESC_LAST_MASK BIT(3)
+#define ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT 4
+#define ENA_ETH_IO_RX_DESC_COMP_REQ_MASK BIT(4)
+
+/* rx_cdesc_base */
+#define ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK GENMASK(4, 0)
+#define ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT 5
+#define ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK GENMASK(6, 5)
+#define ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT 8
+#define ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK GENMASK(12, 8)
+#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT 13
+#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK BIT(13)
+#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT 14
+#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK BIT(14)
+#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT 15
+#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK BIT(15)
+#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT 24
+#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK BIT(24)
+#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT 25
+#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK BIT(25)
+#define ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT 26
+#define ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK BIT(26)
+#define ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT 27
+#define ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK BIT(27)
+#define ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT 30
+#define ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK BIT(30)
+
+/* intr_reg */
+#define ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK GENMASK(14, 0)
+#define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT 15
+#define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK GENMASK(29, 15)
+#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT 30
+#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK BIT(30)
+
+/* numa_node_cfg_reg */
+#define ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK GENMASK(7, 0)
+#define ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT 31
+#define ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK BIT(31)
+
+#endif /*_ENA_ETH_IO_H_ */
diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c
new file mode 100644
index 000000000000..67b2338f8fb3
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c
@@ -0,0 +1,895 @@
+/*
+ * Copyright 2015 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/pci.h>
+
+#include "ena_netdev.h"
+
+struct ena_stats {
+	char name[ETH_GSTRING_LEN];
+	int stat_offset;
+};
+
+#define ENA_STAT_ENA_COM_ENTRY(stat) { \
+	.name = #stat, \
+	.stat_offset = offsetof(struct ena_com_stats_admin, stat) \
+}
+
+#define ENA_STAT_ENTRY(stat, stat_type) { \
+	.name = #stat, \
+	.stat_offset = offsetof(struct ena_stats_##stat_type, stat) \
+}
+
+#define ENA_STAT_RX_ENTRY(stat) \
+	ENA_STAT_ENTRY(stat, rx)
+
+#define ENA_STAT_TX_ENTRY(stat) \
+	ENA_STAT_ENTRY(stat, tx)
+
+#define ENA_STAT_GLOBAL_ENTRY(stat) \
+	ENA_STAT_ENTRY(stat, dev)
+
+static const struct ena_stats ena_stats_global_strings[] = {
+	ENA_STAT_GLOBAL_ENTRY(tx_timeout),
+	ENA_STAT_GLOBAL_ENTRY(io_suspend),
+	ENA_STAT_GLOBAL_ENTRY(io_resume),
+	ENA_STAT_GLOBAL_ENTRY(wd_expired),
+	ENA_STAT_GLOBAL_ENTRY(interface_up),
+	ENA_STAT_GLOBAL_ENTRY(interface_down),
+	ENA_STAT_GLOBAL_ENTRY(admin_q_pause),
+};
+
+static const struct ena_stats ena_stats_tx_strings[] = {
+	ENA_STAT_TX_ENTRY(cnt),
+	ENA_STAT_TX_ENTRY(bytes),
+	ENA_STAT_TX_ENTRY(queue_stop),
+	ENA_STAT_TX_ENTRY(queue_wakeup),
+	ENA_STAT_TX_ENTRY(dma_mapping_err),
+	ENA_STAT_TX_ENTRY(linearize),
+	ENA_STAT_TX_ENTRY(linearize_failed),
+	ENA_STAT_TX_ENTRY(napi_comp),
+	ENA_STAT_TX_ENTRY(tx_poll),
+	ENA_STAT_TX_ENTRY(doorbells),
+	ENA_STAT_TX_ENTRY(prepare_ctx_err),
+	ENA_STAT_TX_ENTRY(missing_tx_comp),
+	ENA_STAT_TX_ENTRY(bad_req_id),
+};
+
+static const struct ena_stats ena_stats_rx_strings[] = {
+	ENA_STAT_RX_ENTRY(cnt),
+	ENA_STAT_RX_ENTRY(bytes),
+	ENA_STAT_RX_ENTRY(refil_partial),
+	ENA_STAT_RX_ENTRY(bad_csum),
+	ENA_STAT_RX_ENTRY(page_alloc_fail),
+	ENA_STAT_RX_ENTRY(skb_alloc_fail),
+	ENA_STAT_RX_ENTRY(dma_mapping_err),
+	ENA_STAT_RX_ENTRY(bad_desc_num),
+	ENA_STAT_RX_ENTRY(rx_copybreak_pkt),
+};
+
+static const struct ena_stats ena_stats_ena_com_strings[] = {
+	ENA_STAT_ENA_COM_ENTRY(aborted_cmd),
+	ENA_STAT_ENA_COM_ENTRY(submitted_cmd),
+	ENA_STAT_ENA_COM_ENTRY(completed_cmd),
+	ENA_STAT_ENA_COM_ENTRY(out_of_space),
+	ENA_STAT_ENA_COM_ENTRY(no_completion),
+};
+
+#define ENA_STATS_ARRAY_GLOBAL	ARRAY_SIZE(ena_stats_global_strings)
+#define ENA_STATS_ARRAY_TX	ARRAY_SIZE(ena_stats_tx_strings)
+#define ENA_STATS_ARRAY_RX	ARRAY_SIZE(ena_stats_rx_strings)
+#define ENA_STATS_ARRAY_ENA_COM	ARRAY_SIZE(ena_stats_ena_com_strings)
+
+static void ena_safe_update_stat(u64 *src, u64 *dst,
+				 struct u64_stats_sync *syncp)
+{
+	unsigned int start;
+
+	do {
+		start = u64_stats_fetch_begin_irq(syncp);
+		*(dst) = *src;
+	} while (u64_stats_fetch_retry_irq(syncp, start));
+}
+
+static void ena_queue_stats(struct ena_adapter *adapter, u64 **data)
+{
+	const struct ena_stats *ena_stats;
+	struct ena_ring *ring;
+
+	u64 *ptr;
+	int i, j;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		/* Tx stats */
+		ring = &adapter->tx_ring[i];
+
+		for (j = 0; j < ENA_STATS_ARRAY_TX; j++) {
+			ena_stats = &ena_stats_tx_strings[j];
+
+			ptr = (u64 *)((uintptr_t)&ring->tx_stats +
+				(uintptr_t)ena_stats->stat_offset);
+
+			ena_safe_update_stat(ptr, (*data)++, &ring->syncp);
+		}
+
+		/* Rx stats */
+		ring = &adapter->rx_ring[i];
+
+		for (j = 0; j < ENA_STATS_ARRAY_RX; j++) {
+			ena_stats = &ena_stats_rx_strings[j];
+
+			ptr = (u64 *)((uintptr_t)&ring->rx_stats +
+				(uintptr_t)ena_stats->stat_offset);
+
+			ena_safe_update_stat(ptr, (*data)++, &ring->syncp);
+		}
+	}
+}
+
+static void ena_dev_admin_queue_stats(struct ena_adapter *adapter, u64 **data)
+{
+	const struct ena_stats *ena_stats;
+	u32 *ptr;
+	int i;
+
+	for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) {
+		ena_stats = &ena_stats_ena_com_strings[i];
+
+		ptr = (u32 *)((uintptr_t)&adapter->ena_dev->admin_queue.stats +
+			(uintptr_t)ena_stats->stat_offset);
+
+		*(*data)++ = *ptr;
+	}
+}
+
+static void ena_get_ethtool_stats(struct net_device *netdev,
+				  struct ethtool_stats *stats,
+				  u64 *data)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	const struct ena_stats *ena_stats;
+	u64 *ptr;
+	int i;
+
+	for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) {
+		ena_stats = &ena_stats_global_strings[i];
+
+		ptr = (u64 *)((uintptr_t)&adapter->dev_stats +
+			(uintptr_t)ena_stats->stat_offset);
+
+		ena_safe_update_stat(ptr, data++, &adapter->syncp);
+	}
+
+	ena_queue_stats(adapter, &data);
+	ena_dev_admin_queue_stats(adapter, &data);
+}
+
+int ena_get_sset_count(struct net_device *netdev, int sset)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+
+	if (sset != ETH_SS_STATS)
+		return -EOPNOTSUPP;
+
+	return  adapter->num_queues * (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX)
+		+ ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM;
+}
+
+static void ena_queue_strings(struct ena_adapter *adapter, u8 **data)
+{
+	const struct ena_stats *ena_stats;
+	int i, j;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		/* Tx stats */
+		for (j = 0; j < ENA_STATS_ARRAY_TX; j++) {
+			ena_stats = &ena_stats_tx_strings[j];
+
+			snprintf(*data, ETH_GSTRING_LEN,
+				 "queue_%u_tx_%s", i, ena_stats->name);
+			 (*data) += ETH_GSTRING_LEN;
+		}
+		/* Rx stats */
+		for (j = 0; j < ENA_STATS_ARRAY_RX; j++) {
+			ena_stats = &ena_stats_rx_strings[j];
+
+			snprintf(*data, ETH_GSTRING_LEN,
+				 "queue_%u_rx_%s", i, ena_stats->name);
+			(*data) += ETH_GSTRING_LEN;
+		}
+	}
+}
+
+static void ena_com_dev_strings(u8 **data)
+{
+	const struct ena_stats *ena_stats;
+	int i;
+
+	for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) {
+		ena_stats = &ena_stats_ena_com_strings[i];
+
+		snprintf(*data, ETH_GSTRING_LEN,
+			 "ena_admin_q_%s", ena_stats->name);
+		(*data) += ETH_GSTRING_LEN;
+	}
+}
+
+static void ena_get_strings(struct net_device *netdev, u32 sset, u8 *data)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	const struct ena_stats *ena_stats;
+	int i;
+
+	if (sset != ETH_SS_STATS)
+		return;
+
+	for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) {
+		ena_stats = &ena_stats_global_strings[i];
+
+		memcpy(data, ena_stats->name, ETH_GSTRING_LEN);
+		data += ETH_GSTRING_LEN;
+	}
+
+	ena_queue_strings(adapter, &data);
+	ena_com_dev_strings(&data);
+}
+
+static int ena_get_link_ksettings(struct net_device *netdev,
+				  struct ethtool_link_ksettings *link_ksettings)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	struct ena_admin_get_feature_link_desc *link;
+	struct ena_admin_get_feat_resp feat_resp;
+	int rc;
+
+	rc = ena_com_get_link_params(ena_dev, &feat_resp);
+	if (rc)
+		return rc;
+
+	link = &feat_resp.u.link;
+	link_ksettings->base.speed = link->speed;
+
+	if (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) {
+		ethtool_link_ksettings_add_link_mode(link_ksettings,
+						     supported, Autoneg);
+		ethtool_link_ksettings_add_link_mode(link_ksettings,
+						     supported, Autoneg);
+	}
+
+	link_ksettings->base.autoneg =
+		(link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) ?
+		AUTONEG_ENABLE : AUTONEG_DISABLE;
+
+	link_ksettings->base.duplex = DUPLEX_FULL;
+
+	return 0;
+}
+
+static int ena_get_coalesce(struct net_device *net_dev,
+			    struct ethtool_coalesce *coalesce)
+{
+	struct ena_adapter *adapter = netdev_priv(net_dev);
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	struct ena_intr_moder_entry intr_moder_entry;
+
+	if (!ena_com_interrupt_moderation_supported(ena_dev)) {
+		/* the devie doesn't support interrupt moderation */
+		return -EOPNOTSUPP;
+	}
+	coalesce->tx_coalesce_usecs =
+		ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) /
+			ena_dev->intr_delay_resolution;
+	if (!ena_com_get_adaptive_moderation_enabled(ena_dev)) {
+		coalesce->rx_coalesce_usecs =
+			ena_com_get_nonadaptive_moderation_interval_rx(ena_dev)
+			/ ena_dev->intr_delay_resolution;
+	} else {
+		ena_com_get_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_LOWEST, &intr_moder_entry);
+		coalesce->rx_coalesce_usecs_low = intr_moder_entry.intr_moder_interval;
+		coalesce->rx_max_coalesced_frames_low = intr_moder_entry.pkts_per_interval;
+
+		ena_com_get_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_MID, &intr_moder_entry);
+		coalesce->rx_coalesce_usecs = intr_moder_entry.intr_moder_interval;
+		coalesce->rx_max_coalesced_frames = intr_moder_entry.pkts_per_interval;
+
+		ena_com_get_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_HIGHEST, &intr_moder_entry);
+		coalesce->rx_coalesce_usecs_high = intr_moder_entry.intr_moder_interval;
+		coalesce->rx_max_coalesced_frames_high = intr_moder_entry.pkts_per_interval;
+	}
+	coalesce->use_adaptive_rx_coalesce =
+		ena_com_get_adaptive_moderation_enabled(ena_dev);
+
+	return 0;
+}
+
+static void ena_update_tx_rings_intr_moderation(struct ena_adapter *adapter)
+{
+	unsigned int val;
+	int i;
+
+	val = ena_com_get_nonadaptive_moderation_interval_tx(adapter->ena_dev);
+
+	for (i = 0; i < adapter->num_queues; i++)
+		adapter->tx_ring[i].smoothed_interval = val;
+}
+
+static int ena_set_coalesce(struct net_device *net_dev,
+			    struct ethtool_coalesce *coalesce)
+{
+	struct ena_adapter *adapter = netdev_priv(net_dev);
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	struct ena_intr_moder_entry intr_moder_entry;
+	int rc;
+
+	if (!ena_com_interrupt_moderation_supported(ena_dev)) {
+		/* the devie doesn't support interrupt moderation */
+		return -EOPNOTSUPP;
+	}
+
+	if (coalesce->rx_coalesce_usecs_irq ||
+	    coalesce->rx_max_coalesced_frames_irq ||
+	    coalesce->tx_coalesce_usecs_irq ||
+	    coalesce->tx_max_coalesced_frames ||
+	    coalesce->tx_max_coalesced_frames_irq ||
+	    coalesce->stats_block_coalesce_usecs ||
+	    coalesce->use_adaptive_tx_coalesce ||
+	    coalesce->pkt_rate_low ||
+	    coalesce->tx_coalesce_usecs_low ||
+	    coalesce->tx_max_coalesced_frames_low ||
+	    coalesce->pkt_rate_high ||
+	    coalesce->tx_coalesce_usecs_high ||
+	    coalesce->tx_max_coalesced_frames_high ||
+	    coalesce->rate_sample_interval)
+		return -EINVAL;
+
+	rc = ena_com_update_nonadaptive_moderation_interval_tx(ena_dev,
+							       coalesce->tx_coalesce_usecs);
+	if (rc)
+		return rc;
+
+	ena_update_tx_rings_intr_moderation(adapter);
+
+	if (ena_com_get_adaptive_moderation_enabled(ena_dev)) {
+		if (!coalesce->use_adaptive_rx_coalesce) {
+			ena_com_disable_adaptive_moderation(ena_dev);
+			rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev,
+									       coalesce->rx_coalesce_usecs);
+			return rc;
+		}
+	} else { /* was in non-adaptive mode */
+		if (coalesce->use_adaptive_rx_coalesce) {
+			ena_com_enable_adaptive_moderation(ena_dev);
+		} else {
+			rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev,
+									       coalesce->rx_coalesce_usecs);
+			return rc;
+		}
+	}
+
+	intr_moder_entry.intr_moder_interval = coalesce->rx_coalesce_usecs_low;
+	intr_moder_entry.pkts_per_interval = coalesce->rx_max_coalesced_frames_low;
+	intr_moder_entry.bytes_per_interval = ENA_INTR_BYTE_COUNT_NOT_SUPPORTED;
+	ena_com_init_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_LOWEST, &intr_moder_entry);
+
+	intr_moder_entry.intr_moder_interval = coalesce->rx_coalesce_usecs;
+	intr_moder_entry.pkts_per_interval = coalesce->rx_max_coalesced_frames;
+	intr_moder_entry.bytes_per_interval = ENA_INTR_BYTE_COUNT_NOT_SUPPORTED;
+	ena_com_init_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_MID, &intr_moder_entry);
+
+	intr_moder_entry.intr_moder_interval = coalesce->rx_coalesce_usecs_high;
+	intr_moder_entry.pkts_per_interval = coalesce->rx_max_coalesced_frames_high;
+	intr_moder_entry.bytes_per_interval = ENA_INTR_BYTE_COUNT_NOT_SUPPORTED;
+	ena_com_init_intr_moderation_entry(adapter->ena_dev, ENA_INTR_MODER_HIGHEST, &intr_moder_entry);
+
+	return 0;
+}
+
+static u32 ena_get_msglevel(struct net_device *netdev)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+
+	return adapter->msg_enable;
+}
+
+static void ena_set_msglevel(struct net_device *netdev, u32 value)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+
+	adapter->msg_enable = value;
+}
+
+static void ena_get_drvinfo(struct net_device *dev,
+			    struct ethtool_drvinfo *info)
+{
+	struct ena_adapter *adapter = netdev_priv(dev);
+
+	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
+	strlcpy(info->bus_info, pci_name(adapter->pdev),
+		sizeof(info->bus_info));
+}
+
+static void ena_get_ringparam(struct net_device *netdev,
+			      struct ethtool_ringparam *ring)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	struct ena_ring *tx_ring = &adapter->tx_ring[0];
+	struct ena_ring *rx_ring = &adapter->rx_ring[0];
+
+	ring->rx_max_pending = rx_ring->ring_size;
+	ring->tx_max_pending = tx_ring->ring_size;
+	ring->rx_pending = rx_ring->ring_size;
+	ring->tx_pending = tx_ring->ring_size;
+}
+
+static u32 ena_flow_hash_to_flow_type(u16 hash_fields)
+{
+	u32 data = 0;
+
+	if (hash_fields & ENA_ADMIN_RSS_L2_DA)
+		data |= RXH_L2DA;
+
+	if (hash_fields & ENA_ADMIN_RSS_L3_DA)
+		data |= RXH_IP_DST;
+
+	if (hash_fields & ENA_ADMIN_RSS_L3_SA)
+		data |= RXH_IP_SRC;
+
+	if (hash_fields & ENA_ADMIN_RSS_L4_DP)
+		data |= RXH_L4_B_2_3;
+
+	if (hash_fields & ENA_ADMIN_RSS_L4_SP)
+		data |= RXH_L4_B_0_1;
+
+	return data;
+}
+
+static u16 ena_flow_data_to_flow_hash(u32 hash_fields)
+{
+	u16 data = 0;
+
+	if (hash_fields & RXH_L2DA)
+		data |= ENA_ADMIN_RSS_L2_DA;
+
+	if (hash_fields & RXH_IP_DST)
+		data |= ENA_ADMIN_RSS_L3_DA;
+
+	if (hash_fields & RXH_IP_SRC)
+		data |= ENA_ADMIN_RSS_L3_SA;
+
+	if (hash_fields & RXH_L4_B_2_3)
+		data |= ENA_ADMIN_RSS_L4_DP;
+
+	if (hash_fields & RXH_L4_B_0_1)
+		data |= ENA_ADMIN_RSS_L4_SP;
+
+	return data;
+}
+
+static int ena_get_rss_hash(struct ena_com_dev *ena_dev,
+			    struct ethtool_rxnfc *cmd)
+{
+	enum ena_admin_flow_hash_proto proto;
+	u16 hash_fields;
+	int rc;
+
+	cmd->data = 0;
+
+	switch (cmd->flow_type) {
+	case TCP_V4_FLOW:
+		proto = ENA_ADMIN_RSS_TCP4;
+		break;
+	case UDP_V4_FLOW:
+		proto = ENA_ADMIN_RSS_UDP4;
+		break;
+	case TCP_V6_FLOW:
+		proto = ENA_ADMIN_RSS_TCP6;
+		break;
+	case UDP_V6_FLOW:
+		proto = ENA_ADMIN_RSS_UDP6;
+		break;
+	case IPV4_FLOW:
+		proto = ENA_ADMIN_RSS_IP4;
+		break;
+	case IPV6_FLOW:
+		proto = ENA_ADMIN_RSS_IP6;
+		break;
+	case ETHER_FLOW:
+		proto = ENA_ADMIN_RSS_NOT_IP;
+		break;
+	case AH_V4_FLOW:
+	case ESP_V4_FLOW:
+	case AH_V6_FLOW:
+	case ESP_V6_FLOW:
+	case SCTP_V4_FLOW:
+	case AH_ESP_V4_FLOW:
+		return -EOPNOTSUPP;
+	default:
+		return -EINVAL;
+	}
+
+	rc = ena_com_get_hash_ctrl(ena_dev, proto, &hash_fields);
+	if (rc) {
+		/* If device don't have permission, return unsupported */
+		if (rc == -EPERM)
+			rc = -EOPNOTSUPP;
+		return rc;
+	}
+
+	cmd->data = ena_flow_hash_to_flow_type(hash_fields);
+
+	return 0;
+}
+
+static int ena_set_rss_hash(struct ena_com_dev *ena_dev,
+			    struct ethtool_rxnfc *cmd)
+{
+	enum ena_admin_flow_hash_proto proto;
+	u16 hash_fields;
+
+	switch (cmd->flow_type) {
+	case TCP_V4_FLOW:
+		proto = ENA_ADMIN_RSS_TCP4;
+		break;
+	case UDP_V4_FLOW:
+		proto = ENA_ADMIN_RSS_UDP4;
+		break;
+	case TCP_V6_FLOW:
+		proto = ENA_ADMIN_RSS_TCP6;
+		break;
+	case UDP_V6_FLOW:
+		proto = ENA_ADMIN_RSS_UDP6;
+		break;
+	case IPV4_FLOW:
+		proto = ENA_ADMIN_RSS_IP4;
+		break;
+	case IPV6_FLOW:
+		proto = ENA_ADMIN_RSS_IP6;
+		break;
+	case ETHER_FLOW:
+		proto = ENA_ADMIN_RSS_NOT_IP;
+		break;
+	case AH_V4_FLOW:
+	case ESP_V4_FLOW:
+	case AH_V6_FLOW:
+	case ESP_V6_FLOW:
+	case SCTP_V4_FLOW:
+	case AH_ESP_V4_FLOW:
+		return -EOPNOTSUPP;
+	default:
+		return -EINVAL;
+	}
+
+	hash_fields = ena_flow_data_to_flow_hash(cmd->data);
+
+	return ena_com_fill_hash_ctrl(ena_dev, proto, hash_fields);
+}
+
+static int ena_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	int rc = 0;
+
+	switch (info->cmd) {
+	case ETHTOOL_SRXFH:
+		rc = ena_set_rss_hash(adapter->ena_dev, info);
+		break;
+	case ETHTOOL_SRXCLSRLDEL:
+	case ETHTOOL_SRXCLSRLINS:
+	default:
+		netif_err(adapter, drv, netdev,
+			  "Command parameter %d is not supported\n", info->cmd);
+		rc = -EOPNOTSUPP;
+	}
+
+	return (rc == -EPERM) ? -EOPNOTSUPP : rc;
+}
+
+static int ena_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
+			 u32 *rules)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	int rc = 0;
+
+	switch (info->cmd) {
+	case ETHTOOL_GRXRINGS:
+		info->data = adapter->num_queues;
+		rc = 0;
+		break;
+	case ETHTOOL_GRXFH:
+		rc = ena_get_rss_hash(adapter->ena_dev, info);
+		break;
+	case ETHTOOL_GRXCLSRLCNT:
+	case ETHTOOL_GRXCLSRULE:
+	case ETHTOOL_GRXCLSRLALL:
+	default:
+		netif_err(adapter, drv, netdev,
+			  "Command parameter %d is not supported\n", info->cmd);
+		rc = -EOPNOTSUPP;
+	}
+
+	return (rc == -EPERM) ? -EOPNOTSUPP : rc;
+}
+
+static u32 ena_get_rxfh_indir_size(struct net_device *netdev)
+{
+	return ENA_RX_RSS_TABLE_SIZE;
+}
+
+static u32 ena_get_rxfh_key_size(struct net_device *netdev)
+{
+	return ENA_HASH_KEY_SIZE;
+}
+
+static int ena_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
+			u8 *hfunc)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	enum ena_admin_hash_functions ena_func;
+	u8 func;
+	int rc;
+
+	rc = ena_com_indirect_table_get(adapter->ena_dev, indir);
+	if (rc)
+		return rc;
+
+	rc = ena_com_get_hash_function(adapter->ena_dev, &ena_func, key);
+	if (rc)
+		return rc;
+
+	switch (ena_func) {
+	case ENA_ADMIN_TOEPLITZ:
+		func = ETH_RSS_HASH_TOP;
+		break;
+	case ENA_ADMIN_CRC32:
+		func = ETH_RSS_HASH_XOR;
+		break;
+	default:
+		netif_err(adapter, drv, netdev,
+			  "Command parameter is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (hfunc)
+		*hfunc = func;
+
+	return rc;
+}
+
+static int ena_set_rxfh(struct net_device *netdev, const u32 *indir,
+			const u8 *key, const u8 hfunc)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	enum ena_admin_hash_functions func;
+	int rc, i;
+
+	if (indir) {
+		for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
+			rc = ena_com_indirect_table_fill_entry(ena_dev,
+							       ENA_IO_RXQ_IDX(indir[i]),
+							       i);
+			if (unlikely(rc)) {
+				netif_err(adapter, drv, netdev,
+					  "Cannot fill indirect table (index is too large)\n");
+				return rc;
+			}
+		}
+
+		rc = ena_com_indirect_table_set(ena_dev);
+		if (rc) {
+			netif_err(adapter, drv, netdev,
+				  "Cannot set indirect table\n");
+			return rc == -EPERM ? -EOPNOTSUPP : rc;
+		}
+	}
+
+	switch (hfunc) {
+	case ETH_RSS_HASH_TOP:
+		func = ENA_ADMIN_TOEPLITZ;
+		break;
+	case ETH_RSS_HASH_XOR:
+		func = ENA_ADMIN_CRC32;
+		break;
+	default:
+		netif_err(adapter, drv, netdev, "Unsupported hfunc %d\n",
+			  hfunc);
+		return -EOPNOTSUPP;
+	}
+
+	if (key) {
+		rc = ena_com_fill_hash_function(ena_dev, func, key,
+						ENA_HASH_KEY_SIZE,
+						0xFFFFFFFF);
+		if (unlikely(rc)) {
+			netif_err(adapter, drv, netdev, "Cannot fill key\n");
+			return rc == -EPERM ? -EOPNOTSUPP : rc;
+		}
+	}
+
+	return 0;
+}
+
+static void ena_get_channels(struct net_device *netdev,
+			     struct ethtool_channels *channels)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+
+	channels->max_rx = ENA_MAX_NUM_IO_QUEUES;
+	channels->max_tx = ENA_MAX_NUM_IO_QUEUES;
+	channels->max_other = 0;
+	channels->max_combined = 0;
+	channels->rx_count = adapter->num_queues;
+	channels->tx_count = adapter->num_queues;
+	channels->other_count = 0;
+	channels->combined_count = 0;
+}
+
+static int ena_get_tunable(struct net_device *netdev,
+			   const struct ethtool_tunable *tuna, void *data)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	int ret = 0;
+
+	switch (tuna->id) {
+	case ETHTOOL_RX_COPYBREAK:
+		*(u32 *)data = adapter->rx_copybreak;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int ena_set_tunable(struct net_device *netdev,
+			   const struct ethtool_tunable *tuna,
+			   const void *data)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	int ret = 0;
+	u32 len;
+
+	switch (tuna->id) {
+	case ETHTOOL_RX_COPYBREAK:
+		len = *(u32 *)data;
+		if (len > adapter->netdev->mtu) {
+			ret = -EINVAL;
+			break;
+		}
+		adapter->rx_copybreak = len;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static const struct ethtool_ops ena_ethtool_ops = {
+	.get_link_ksettings	= ena_get_link_ksettings,
+	.get_drvinfo		= ena_get_drvinfo,
+	.get_msglevel		= ena_get_msglevel,
+	.set_msglevel		= ena_set_msglevel,
+	.get_link		= ethtool_op_get_link,
+	.get_coalesce		= ena_get_coalesce,
+	.set_coalesce		= ena_set_coalesce,
+	.get_ringparam		= ena_get_ringparam,
+	.get_sset_count         = ena_get_sset_count,
+	.get_strings		= ena_get_strings,
+	.get_ethtool_stats      = ena_get_ethtool_stats,
+	.get_rxnfc		= ena_get_rxnfc,
+	.set_rxnfc		= ena_set_rxnfc,
+	.get_rxfh_indir_size    = ena_get_rxfh_indir_size,
+	.get_rxfh_key_size	= ena_get_rxfh_key_size,
+	.get_rxfh		= ena_get_rxfh,
+	.set_rxfh		= ena_set_rxfh,
+	.get_channels		= ena_get_channels,
+	.get_tunable		= ena_get_tunable,
+	.set_tunable		= ena_set_tunable,
+};
+
+void ena_set_ethtool_ops(struct net_device *netdev)
+{
+	netdev->ethtool_ops = &ena_ethtool_ops;
+}
+
+static void ena_dump_stats_ex(struct ena_adapter *adapter, u8 *buf)
+{
+	struct net_device *netdev = adapter->netdev;
+	u8 *strings_buf;
+	u64 *data_buf;
+	int strings_num;
+	int i, rc;
+
+	strings_num = ena_get_sset_count(netdev, ETH_SS_STATS);
+	if (strings_num <= 0) {
+		netif_err(adapter, drv, netdev, "Can't get stats num\n");
+		return;
+	}
+
+	strings_buf = devm_kzalloc(&adapter->pdev->dev,
+				   strings_num * ETH_GSTRING_LEN,
+				   GFP_ATOMIC);
+	if (!strings_buf) {
+		netif_err(adapter, drv, netdev,
+			  "failed to alloc strings_buf\n");
+		return;
+	}
+
+	data_buf = devm_kzalloc(&adapter->pdev->dev,
+				strings_num * sizeof(u64),
+				GFP_ATOMIC);
+	if (!data_buf) {
+		netif_err(adapter, drv, netdev,
+			  "failed to allocate data buf\n");
+		devm_kfree(&adapter->pdev->dev, strings_buf);
+		return;
+	}
+
+	ena_get_strings(netdev, ETH_SS_STATS, strings_buf);
+	ena_get_ethtool_stats(netdev, NULL, data_buf);
+
+	/* If there is a buffer, dump stats, otherwise print them to dmesg */
+	if (buf)
+		for (i = 0; i < strings_num; i++) {
+			rc = snprintf(buf, ETH_GSTRING_LEN + sizeof(u64),
+				      "%s %llu\n",
+				      strings_buf + i * ETH_GSTRING_LEN,
+				      data_buf[i]);
+			buf += rc;
+		}
+	else
+		for (i = 0; i < strings_num; i++)
+			netif_err(adapter, drv, netdev, "%s: %llu\n",
+				  strings_buf + i * ETH_GSTRING_LEN,
+				  data_buf[i]);
+
+	devm_kfree(&adapter->pdev->dev, strings_buf);
+	devm_kfree(&adapter->pdev->dev, data_buf);
+}
+
+void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf)
+{
+	if (!buf)
+		return;
+
+	ena_dump_stats_ex(adapter, buf);
+}
+
+void ena_dump_stats_to_dmesg(struct ena_adapter *adapter)
+{
+	ena_dump_stats_ex(adapter, NULL);
+}
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
new file mode 100644
index 000000000000..bfeaec5bd7b9
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
@@ -0,0 +1,3272 @@
+/*
+ * Copyright 2015 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#ifdef CONFIG_RFS_ACCEL
+#include <linux/cpu_rmap.h>
+#endif /* CONFIG_RFS_ACCEL */
+#include <linux/ethtool.h>
+#include <linux/if_vlan.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/numa.h>
+#include <linux/pci.h>
+#include <linux/utsname.h>
+#include <linux/version.h>
+#include <linux/vmalloc.h>
+#include <net/ip.h>
+
+#include "ena_netdev.h"
+#include "ena_pci_id_tbl.h"
+
+static char version[] = DEVICE_NAME " v" DRV_MODULE_VERSION "\n";
+
+MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
+MODULE_DESCRIPTION(DEVICE_NAME);
+MODULE_LICENSE("GPL");
+MODULE_VERSION(DRV_MODULE_VERSION);
+
+/* Time in jiffies before concluding the transmitter is hung. */
+#define TX_TIMEOUT  (5 * HZ)
+
+#define ENA_NAPI_BUDGET 64
+
+#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \
+		NETIF_MSG_TX_DONE | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR)
+static int debug = -1;
+module_param(debug, int, 0);
+MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
+
+static struct ena_aenq_handlers aenq_handlers;
+
+static struct workqueue_struct *ena_wq;
+
+MODULE_DEVICE_TABLE(pci, ena_pci_tbl);
+
+static int ena_rss_init_default(struct ena_adapter *adapter);
+
+static void ena_tx_timeout(struct net_device *dev)
+{
+	struct ena_adapter *adapter = netdev_priv(dev);
+
+	u64_stats_update_begin(&adapter->syncp);
+	adapter->dev_stats.tx_timeout++;
+	u64_stats_update_end(&adapter->syncp);
+
+	netif_err(adapter, tx_err, dev, "Transmit time out\n");
+
+	/* Change the state of the device to trigger reset */
+	set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+}
+
+static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu)
+{
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++)
+		adapter->rx_ring[i].mtu = mtu;
+}
+
+static int ena_change_mtu(struct net_device *dev, int new_mtu)
+{
+	struct ena_adapter *adapter = netdev_priv(dev);
+	int ret;
+
+	if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) {
+		netif_err(adapter, drv, dev,
+			  "Invalid MTU setting. new_mtu: %d\n", new_mtu);
+
+		return -EINVAL;
+	}
+
+	ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
+	if (!ret) {
+		netif_dbg(adapter, drv, dev, "set MTU to %d\n", new_mtu);
+		update_rx_ring_mtu(adapter, new_mtu);
+		dev->mtu = new_mtu;
+	} else {
+		netif_err(adapter, drv, dev, "Failed to set MTU to %d\n",
+			  new_mtu);
+	}
+
+	return ret;
+}
+
+static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter)
+{
+#ifdef CONFIG_RFS_ACCEL
+	u32 i;
+	int rc;
+
+	adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_queues);
+	if (!adapter->netdev->rx_cpu_rmap)
+		return -ENOMEM;
+	for (i = 0; i < adapter->num_queues; i++) {
+		int irq_idx = ENA_IO_IRQ_IDX(i);
+
+		rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,
+				      adapter->msix_entries[irq_idx].vector);
+		if (rc) {
+			free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
+			adapter->netdev->rx_cpu_rmap = NULL;
+			return rc;
+		}
+	}
+#endif /* CONFIG_RFS_ACCEL */
+	return 0;
+}
+
+static void ena_init_io_rings_common(struct ena_adapter *adapter,
+				     struct ena_ring *ring, u16 qid)
+{
+	ring->qid = qid;
+	ring->pdev = adapter->pdev;
+	ring->dev = &adapter->pdev->dev;
+	ring->netdev = adapter->netdev;
+	ring->napi = &adapter->ena_napi[qid].napi;
+	ring->adapter = adapter;
+	ring->ena_dev = adapter->ena_dev;
+	ring->per_napi_packets = 0;
+	ring->per_napi_bytes = 0;
+	ring->cpu = 0;
+	u64_stats_init(&ring->syncp);
+}
+
+static void ena_init_io_rings(struct ena_adapter *adapter)
+{
+	struct ena_com_dev *ena_dev;
+	struct ena_ring *txr, *rxr;
+	int i;
+
+	ena_dev = adapter->ena_dev;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		txr = &adapter->tx_ring[i];
+		rxr = &adapter->rx_ring[i];
+
+		/* TX/RX common ring state */
+		ena_init_io_rings_common(adapter, txr, i);
+		ena_init_io_rings_common(adapter, rxr, i);
+
+		/* TX specific ring state */
+		txr->ring_size = adapter->tx_ring_size;
+		txr->tx_max_header_size = ena_dev->tx_max_header_size;
+		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
+		txr->sgl_size = adapter->max_tx_sgl_size;
+		txr->smoothed_interval =
+			ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
+
+		/* RX specific ring state */
+		rxr->ring_size = adapter->rx_ring_size;
+		rxr->rx_copybreak = adapter->rx_copybreak;
+		rxr->sgl_size = adapter->max_rx_sgl_size;
+		rxr->smoothed_interval =
+			ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
+	}
+}
+
+/* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors)
+ * @adapter: network interface device structure
+ * @qid: queue index
+ *
+ * Return 0 on success, negative on failure
+ */
+static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
+{
+	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
+	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
+	int size, i, node;
+
+	if (tx_ring->tx_buffer_info) {
+		netif_err(adapter, ifup,
+			  adapter->netdev, "tx_buffer_info info is not NULL");
+		return -EEXIST;
+	}
+
+	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
+	node = cpu_to_node(ena_irq->cpu);
+
+	tx_ring->tx_buffer_info = vzalloc_node(size, node);
+	if (!tx_ring->tx_buffer_info) {
+		tx_ring->tx_buffer_info = vzalloc(size);
+		if (!tx_ring->tx_buffer_info)
+			return -ENOMEM;
+	}
+
+	size = sizeof(u16) * tx_ring->ring_size;
+	tx_ring->free_tx_ids = vzalloc_node(size, node);
+	if (!tx_ring->free_tx_ids) {
+		tx_ring->free_tx_ids = vzalloc(size);
+		if (!tx_ring->free_tx_ids) {
+			vfree(tx_ring->tx_buffer_info);
+			return -ENOMEM;
+		}
+	}
+
+	/* Req id ring for TX out of order completions */
+	for (i = 0; i < tx_ring->ring_size; i++)
+		tx_ring->free_tx_ids[i] = i;
+
+	/* Reset tx statistics */
+	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));
+
+	tx_ring->next_to_use = 0;
+	tx_ring->next_to_clean = 0;
+	tx_ring->cpu = ena_irq->cpu;
+	return 0;
+}
+
+/* ena_free_tx_resources - Free I/O Tx Resources per Queue
+ * @adapter: network interface device structure
+ * @qid: queue index
+ *
+ * Free all transmit software resources
+ */
+static void ena_free_tx_resources(struct ena_adapter *adapter, int qid)
+{
+	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
+
+	vfree(tx_ring->tx_buffer_info);
+	tx_ring->tx_buffer_info = NULL;
+
+	vfree(tx_ring->free_tx_ids);
+	tx_ring->free_tx_ids = NULL;
+}
+
+/* ena_setup_all_tx_resources - allocate I/O Tx queues resources for All queues
+ * @adapter: private structure
+ *
+ * Return 0 on success, negative on failure
+ */
+static int ena_setup_all_tx_resources(struct ena_adapter *adapter)
+{
+	int i, rc = 0;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		rc = ena_setup_tx_resources(adapter, i);
+		if (rc)
+			goto err_setup_tx;
+	}
+
+	return 0;
+
+err_setup_tx:
+
+	netif_err(adapter, ifup, adapter->netdev,
+		  "Tx queue %d: allocation failed\n", i);
+
+	/* rewind the index freeing the rings as we go */
+	while (i--)
+		ena_free_tx_resources(adapter, i);
+	return rc;
+}
+
+/* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues
+ * @adapter: board private structure
+ *
+ * Free all transmit software resources
+ */
+static void ena_free_all_io_tx_resources(struct ena_adapter *adapter)
+{
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++)
+		ena_free_tx_resources(adapter, i);
+}
+
+/* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
+ * @adapter: network interface device structure
+ * @qid: queue index
+ *
+ * Returns 0 on success, negative on failure
+ */
+static int ena_setup_rx_resources(struct ena_adapter *adapter,
+				  u32 qid)
+{
+	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
+	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
+	int size, node;
+
+	if (rx_ring->rx_buffer_info) {
+		netif_err(adapter, ifup, adapter->netdev,
+			  "rx_buffer_info is not NULL");
+		return -EEXIST;
+	}
+
+	/* alloc extra element so in rx path
+	 * we can always prefetch rx_info + 1
+	 */
+	size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1);
+	node = cpu_to_node(ena_irq->cpu);
+
+	rx_ring->rx_buffer_info = vzalloc_node(size, node);
+	if (!rx_ring->rx_buffer_info) {
+		rx_ring->rx_buffer_info = vzalloc(size);
+		if (!rx_ring->rx_buffer_info)
+			return -ENOMEM;
+	}
+
+	/* Reset rx statistics */
+	memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
+
+	rx_ring->next_to_clean = 0;
+	rx_ring->next_to_use = 0;
+	rx_ring->cpu = ena_irq->cpu;
+
+	return 0;
+}
+
+/* ena_free_rx_resources - Free I/O Rx Resources
+ * @adapter: network interface device structure
+ * @qid: queue index
+ *
+ * Free all receive software resources
+ */
+static void ena_free_rx_resources(struct ena_adapter *adapter,
+				  u32 qid)
+{
+	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
+
+	vfree(rx_ring->rx_buffer_info);
+	rx_ring->rx_buffer_info = NULL;
+}
+
+/* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
+ * @adapter: board private structure
+ *
+ * Return 0 on success, negative on failure
+ */
+static int ena_setup_all_rx_resources(struct ena_adapter *adapter)
+{
+	int i, rc = 0;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		rc = ena_setup_rx_resources(adapter, i);
+		if (rc)
+			goto err_setup_rx;
+	}
+
+	return 0;
+
+err_setup_rx:
+
+	netif_err(adapter, ifup, adapter->netdev,
+		  "Rx queue %d: allocation failed\n", i);
+
+	/* rewind the index freeing the rings as we go */
+	while (i--)
+		ena_free_rx_resources(adapter, i);
+	return rc;
+}
+
+/* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues
+ * @adapter: board private structure
+ *
+ * Free all receive software resources
+ */
+static void ena_free_all_io_rx_resources(struct ena_adapter *adapter)
+{
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++)
+		ena_free_rx_resources(adapter, i);
+}
+
+static inline int ena_alloc_rx_page(struct ena_ring *rx_ring,
+				    struct ena_rx_buffer *rx_info, gfp_t gfp)
+{
+	struct ena_com_buf *ena_buf;
+	struct page *page;
+	dma_addr_t dma;
+
+	/* if previous allocated page is not used */
+	if (unlikely(rx_info->page))
+		return 0;
+
+	page = alloc_page(gfp);
+	if (unlikely(!page)) {
+		u64_stats_update_begin(&rx_ring->syncp);
+		rx_ring->rx_stats.page_alloc_fail++;
+		u64_stats_update_end(&rx_ring->syncp);
+		return -ENOMEM;
+	}
+
+	dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE,
+			   DMA_FROM_DEVICE);
+	if (unlikely(dma_mapping_error(rx_ring->dev, dma))) {
+		u64_stats_update_begin(&rx_ring->syncp);
+		rx_ring->rx_stats.dma_mapping_err++;
+		u64_stats_update_end(&rx_ring->syncp);
+
+		__free_page(page);
+		return -EIO;
+	}
+	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
+		  "alloc page %p, rx_info %p\n", page, rx_info);
+
+	rx_info->page = page;
+	rx_info->page_offset = 0;
+	ena_buf = &rx_info->ena_buf;
+	ena_buf->paddr = dma;
+	ena_buf->len = PAGE_SIZE;
+
+	return 0;
+}
+
+static void ena_free_rx_page(struct ena_ring *rx_ring,
+			     struct ena_rx_buffer *rx_info)
+{
+	struct page *page = rx_info->page;
+	struct ena_com_buf *ena_buf = &rx_info->ena_buf;
+
+	if (unlikely(!page)) {
+		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
+			   "Trying to free unallocated buffer\n");
+		return;
+	}
+
+	dma_unmap_page(rx_ring->dev, ena_buf->paddr, PAGE_SIZE,
+		       DMA_FROM_DEVICE);
+
+	__free_page(page);
+	rx_info->page = NULL;
+}
+
+static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
+{
+	u16 next_to_use;
+	u32 i;
+	int rc;
+
+	next_to_use = rx_ring->next_to_use;
+
+	for (i = 0; i < num; i++) {
+		struct ena_rx_buffer *rx_info =
+			&rx_ring->rx_buffer_info[next_to_use];
+
+		rc = ena_alloc_rx_page(rx_ring, rx_info,
+				       __GFP_COLD | GFP_ATOMIC | __GFP_COMP);
+		if (unlikely(rc < 0)) {
+			netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
+				   "failed to alloc buffer for rx queue %d\n",
+				   rx_ring->qid);
+			break;
+		}
+		rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
+						&rx_info->ena_buf,
+						next_to_use);
+		if (unlikely(rc)) {
+			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
+				   "failed to add buffer for rx queue %d\n",
+				   rx_ring->qid);
+			break;
+		}
+		next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
+						   rx_ring->ring_size);
+	}
+
+	if (unlikely(i < num)) {
+		u64_stats_update_begin(&rx_ring->syncp);
+		rx_ring->rx_stats.refil_partial++;
+		u64_stats_update_end(&rx_ring->syncp);
+		netdev_warn(rx_ring->netdev,
+			    "refilled rx qid %d with only %d buffers (from %d)\n",
+			    rx_ring->qid, i, num);
+	}
+
+	if (likely(i)) {
+		/* Add memory barrier to make sure the desc were written before
+		 * issue a doorbell
+		 */
+		wmb();
+		ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
+	}
+
+	rx_ring->next_to_use = next_to_use;
+
+	return i;
+}
+
+static void ena_free_rx_bufs(struct ena_adapter *adapter,
+			     u32 qid)
+{
+	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
+	u32 i;
+
+	for (i = 0; i < rx_ring->ring_size; i++) {
+		struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
+
+		if (rx_info->page)
+			ena_free_rx_page(rx_ring, rx_info);
+	}
+}
+
+/* ena_refill_all_rx_bufs - allocate all queues Rx buffers
+ * @adapter: board private structure
+ *
+ */
+static void ena_refill_all_rx_bufs(struct ena_adapter *adapter)
+{
+	struct ena_ring *rx_ring;
+	int i, rc, bufs_num;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		rx_ring = &adapter->rx_ring[i];
+		bufs_num = rx_ring->ring_size - 1;
+		rc = ena_refill_rx_bufs(rx_ring, bufs_num);
+
+		if (unlikely(rc != bufs_num))
+			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
+				   "refilling Queue %d failed. allocated %d buffers from: %d\n",
+				   i, rc, bufs_num);
+	}
+}
+
+static void ena_free_all_rx_bufs(struct ena_adapter *adapter)
+{
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++)
+		ena_free_rx_bufs(adapter, i);
+}
+
+/* ena_free_tx_bufs - Free Tx Buffers per Queue
+ * @tx_ring: TX ring for which buffers be freed
+ */
+static void ena_free_tx_bufs(struct ena_ring *tx_ring)
+{
+	u32 i;
+
+	for (i = 0; i < tx_ring->ring_size; i++) {
+		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
+		struct ena_com_buf *ena_buf;
+		int nr_frags;
+		int j;
+
+		if (!tx_info->skb)
+			continue;
+
+		netdev_notice(tx_ring->netdev,
+			      "free uncompleted tx skb qid %d idx 0x%x\n",
+			      tx_ring->qid, i);
+
+		ena_buf = tx_info->bufs;
+		dma_unmap_single(tx_ring->dev,
+				 ena_buf->paddr,
+				 ena_buf->len,
+				 DMA_TO_DEVICE);
+
+		/* unmap remaining mapped pages */
+		nr_frags = tx_info->num_of_bufs - 1;
+		for (j = 0; j < nr_frags; j++) {
+			ena_buf++;
+			dma_unmap_page(tx_ring->dev,
+				       ena_buf->paddr,
+				       ena_buf->len,
+				       DMA_TO_DEVICE);
+		}
+
+		dev_kfree_skb_any(tx_info->skb);
+	}
+	netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
+						  tx_ring->qid));
+}
+
+static void ena_free_all_tx_bufs(struct ena_adapter *adapter)
+{
+	struct ena_ring *tx_ring;
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		tx_ring = &adapter->tx_ring[i];
+		ena_free_tx_bufs(tx_ring);
+	}
+}
+
+static void ena_destroy_all_tx_queues(struct ena_adapter *adapter)
+{
+	u16 ena_qid;
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		ena_qid = ENA_IO_TXQ_IDX(i);
+		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
+	}
+}
+
+static void ena_destroy_all_rx_queues(struct ena_adapter *adapter)
+{
+	u16 ena_qid;
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		ena_qid = ENA_IO_RXQ_IDX(i);
+		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
+	}
+}
+
+static void ena_destroy_all_io_queues(struct ena_adapter *adapter)
+{
+	ena_destroy_all_tx_queues(adapter);
+	ena_destroy_all_rx_queues(adapter);
+}
+
+static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
+{
+	struct ena_tx_buffer *tx_info = NULL;
+
+	if (likely(req_id < tx_ring->ring_size)) {
+		tx_info = &tx_ring->tx_buffer_info[req_id];
+		if (likely(tx_info->skb))
+			return 0;
+	}
+
+	if (tx_info)
+		netif_err(tx_ring->adapter, tx_done, tx_ring->netdev,
+			  "tx_info doesn't have valid skb\n");
+	else
+		netif_err(tx_ring->adapter, tx_done, tx_ring->netdev,
+			  "Invalid req_id: %hu\n", req_id);
+
+	u64_stats_update_begin(&tx_ring->syncp);
+	tx_ring->tx_stats.bad_req_id++;
+	u64_stats_update_end(&tx_ring->syncp);
+
+	/* Trigger device reset */
+	set_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags);
+	return -EFAULT;
+}
+
+static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
+{
+	struct netdev_queue *txq;
+	bool above_thresh;
+	u32 tx_bytes = 0;
+	u32 total_done = 0;
+	u16 next_to_clean;
+	u16 req_id;
+	int tx_pkts = 0;
+	int rc;
+
+	next_to_clean = tx_ring->next_to_clean;
+	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);
+
+	while (tx_pkts < budget) {
+		struct ena_tx_buffer *tx_info;
+		struct sk_buff *skb;
+		struct ena_com_buf *ena_buf;
+		int i, nr_frags;
+
+		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
+						&req_id);
+		if (rc)
+			break;
+
+		rc = validate_tx_req_id(tx_ring, req_id);
+		if (rc)
+			break;
+
+		tx_info = &tx_ring->tx_buffer_info[req_id];
+		skb = tx_info->skb;
+
+		/* prefetch skb_end_pointer() to speedup skb_shinfo(skb) */
+		prefetch(&skb->end);
+
+		tx_info->skb = NULL;
+		tx_info->last_jiffies = 0;
+
+		if (likely(tx_info->num_of_bufs != 0)) {
+			ena_buf = tx_info->bufs;
+
+			dma_unmap_single(tx_ring->dev,
+					 dma_unmap_addr(ena_buf, paddr),
+					 dma_unmap_len(ena_buf, len),
+					 DMA_TO_DEVICE);
+
+			/* unmap remaining mapped pages */
+			nr_frags = tx_info->num_of_bufs - 1;
+			for (i = 0; i < nr_frags; i++) {
+				ena_buf++;
+				dma_unmap_page(tx_ring->dev,
+					       dma_unmap_addr(ena_buf, paddr),
+					       dma_unmap_len(ena_buf, len),
+					       DMA_TO_DEVICE);
+			}
+		}
+
+		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
+			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,
+			  skb);
+
+		tx_bytes += skb->len;
+		dev_kfree_skb(skb);
+		tx_pkts++;
+		total_done += tx_info->tx_descs;
+
+		tx_ring->free_tx_ids[next_to_clean] = req_id;
+		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
+						     tx_ring->ring_size);
+	}
+
+	tx_ring->next_to_clean = next_to_clean;
+	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
+	ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
+
+	netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
+
+	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
+		  "tx_poll: q %d done. total pkts: %d\n",
+		  tx_ring->qid, tx_pkts);
+
+	/* need to make the rings circular update visible to
+	 * ena_start_xmit() before checking for netif_queue_stopped().
+	 */
+	smp_mb();
+
+	above_thresh = ena_com_sq_empty_space(tx_ring->ena_com_io_sq) >
+		ENA_TX_WAKEUP_THRESH;
+	if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) {
+		__netif_tx_lock(txq, smp_processor_id());
+		above_thresh = ena_com_sq_empty_space(tx_ring->ena_com_io_sq) >
+			ENA_TX_WAKEUP_THRESH;
+		if (netif_tx_queue_stopped(txq) && above_thresh) {
+			netif_tx_wake_queue(txq);
+			u64_stats_update_begin(&tx_ring->syncp);
+			tx_ring->tx_stats.queue_wakeup++;
+			u64_stats_update_end(&tx_ring->syncp);
+		}
+		__netif_tx_unlock(txq);
+	}
+
+	tx_ring->per_napi_bytes += tx_bytes;
+	tx_ring->per_napi_packets += tx_pkts;
+
+	return tx_pkts;
+}
+
+static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
+				  struct ena_com_rx_buf_info *ena_bufs,
+				  u32 descs,
+				  u16 *next_to_clean)
+{
+	struct sk_buff *skb;
+	struct ena_rx_buffer *rx_info =
+		&rx_ring->rx_buffer_info[*next_to_clean];
+	u32 len;
+	u32 buf = 0;
+	void *va;
+
+	len = ena_bufs[0].len;
+	if (unlikely(!rx_info->page)) {
+		netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+			  "Page is NULL\n");
+		return NULL;
+	}
+
+	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
+		  "rx_info %p page %p\n",
+		  rx_info, rx_info->page);
+
+	/* save virt address of first buffer */
+	va = page_address(rx_info->page) + rx_info->page_offset;
+	prefetch(va + NET_IP_ALIGN);
+
+	if (len <= rx_ring->rx_copybreak) {
+		skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
+						rx_ring->rx_copybreak);
+		if (unlikely(!skb)) {
+			u64_stats_update_begin(&rx_ring->syncp);
+			rx_ring->rx_stats.skb_alloc_fail++;
+			u64_stats_update_end(&rx_ring->syncp);
+			netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+				  "Failed to allocate skb\n");
+			return NULL;
+		}
+
+		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
+			  "rx allocated small packet. len %d. data_len %d\n",
+			  skb->len, skb->data_len);
+
+		/* sync this buffer for CPU use */
+		dma_sync_single_for_cpu(rx_ring->dev,
+					dma_unmap_addr(&rx_info->ena_buf, paddr),
+					len,
+					DMA_FROM_DEVICE);
+		skb_copy_to_linear_data(skb, va, len);
+		dma_sync_single_for_device(rx_ring->dev,
+					   dma_unmap_addr(&rx_info->ena_buf, paddr),
+					   len,
+					   DMA_FROM_DEVICE);
+
+		skb_put(skb, len);
+		skb->protocol = eth_type_trans(skb, rx_ring->netdev);
+		*next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,
+						     rx_ring->ring_size);
+		return skb;
+	}
+
+	skb = napi_get_frags(rx_ring->napi);
+	if (unlikely(!skb)) {
+		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
+			  "Failed allocating skb\n");
+		u64_stats_update_begin(&rx_ring->syncp);
+		rx_ring->rx_stats.skb_alloc_fail++;
+		u64_stats_update_end(&rx_ring->syncp);
+		return NULL;
+	}
+
+	do {
+		dma_unmap_page(rx_ring->dev,
+			       dma_unmap_addr(&rx_info->ena_buf, paddr),
+			       PAGE_SIZE, DMA_FROM_DEVICE);
+
+		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page,
+				rx_info->page_offset, len, PAGE_SIZE);
+
+		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
+			  "rx skb updated. len %d. data_len %d\n",
+			  skb->len, skb->data_len);
+
+		rx_info->page = NULL;
+		*next_to_clean =
+			ENA_RX_RING_IDX_NEXT(*next_to_clean,
+					     rx_ring->ring_size);
+		if (likely(--descs == 0))
+			break;
+		rx_info = &rx_ring->rx_buffer_info[*next_to_clean];
+		len = ena_bufs[++buf].len;
+	} while (1);
+
+	return skb;
+}
+
+/* ena_rx_checksum - indicate in skb if hw indicated a good cksum
+ * @adapter: structure containing adapter specific data
+ * @ena_rx_ctx: received packet context/metadata
+ * @skb: skb currently being received and modified
+ */
+static inline void ena_rx_checksum(struct ena_ring *rx_ring,
+				   struct ena_com_rx_ctx *ena_rx_ctx,
+				   struct sk_buff *skb)
+{
+	/* Rx csum disabled */
+	if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) {
+		skb->ip_summed = CHECKSUM_NONE;
+		return;
+	}
+
+	/* For fragmented packets the checksum isn't valid */
+	if (ena_rx_ctx->frag) {
+		skb->ip_summed = CHECKSUM_NONE;
+		return;
+	}
+
+	/* if IP and error */
+	if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
+		     (ena_rx_ctx->l3_csum_err))) {
+		/* ipv4 checksum error */
+		skb->ip_summed = CHECKSUM_NONE;
+		u64_stats_update_begin(&rx_ring->syncp);
+		rx_ring->rx_stats.bad_csum++;
+		u64_stats_update_end(&rx_ring->syncp);
+		netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+			  "RX IPv4 header checksum error\n");
+		return;
+	}
+
+	/* if TCP/UDP */
+	if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
+		   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) {
+		if (unlikely(ena_rx_ctx->l4_csum_err)) {
+			/* TCP/UDP checksum error */
+			u64_stats_update_begin(&rx_ring->syncp);
+			rx_ring->rx_stats.bad_csum++;
+			u64_stats_update_end(&rx_ring->syncp);
+			netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+				  "RX L4 checksum error\n");
+			skb->ip_summed = CHECKSUM_NONE;
+			return;
+		}
+
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+	}
+}
+
+static void ena_set_rx_hash(struct ena_ring *rx_ring,
+			    struct ena_com_rx_ctx *ena_rx_ctx,
+			    struct sk_buff *skb)
+{
+	enum pkt_hash_types hash_type;
+
+	if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) {
+		if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
+			   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)))
+
+			hash_type = PKT_HASH_TYPE_L4;
+		else
+			hash_type = PKT_HASH_TYPE_NONE;
+
+		/* Override hash type if the packet is fragmented */
+		if (ena_rx_ctx->frag)
+			hash_type = PKT_HASH_TYPE_NONE;
+
+		skb_set_hash(skb, ena_rx_ctx->hash, hash_type);
+	}
+}
+
+/* ena_clean_rx_irq - Cleanup RX irq
+ * @rx_ring: RX ring to clean
+ * @napi: napi handler
+ * @budget: how many packets driver is allowed to clean
+ *
+ * Returns the number of cleaned buffers.
+ */
+static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
+			    u32 budget)
+{
+	u16 next_to_clean = rx_ring->next_to_clean;
+	u32 res_budget, work_done;
+
+	struct ena_com_rx_ctx ena_rx_ctx;
+	struct ena_adapter *adapter;
+	struct sk_buff *skb;
+	int refill_required;
+	int refill_threshold;
+	int rc = 0;
+	int total_len = 0;
+	int rx_copybreak_pkt = 0;
+
+	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
+		  "%s qid %d\n", __func__, rx_ring->qid);
+	res_budget = budget;
+
+	do {
+		ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
+		ena_rx_ctx.max_bufs = rx_ring->sgl_size;
+		ena_rx_ctx.descs = 0;
+		rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
+				    rx_ring->ena_com_io_sq,
+				    &ena_rx_ctx);
+		if (unlikely(rc))
+			goto error;
+
+		if (unlikely(ena_rx_ctx.descs == 0))
+			break;
+
+		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
+			  "rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
+			  rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
+			  ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
+
+		/* allocate skb and fill it */
+		skb = ena_rx_skb(rx_ring, rx_ring->ena_bufs, ena_rx_ctx.descs,
+				 &next_to_clean);
+
+		/* exit if we failed to retrieve a buffer */
+		if (unlikely(!skb)) {
+			next_to_clean = ENA_RX_RING_IDX_ADD(next_to_clean,
+							    ena_rx_ctx.descs,
+							    rx_ring->ring_size);
+			break;
+		}
+
+		ena_rx_checksum(rx_ring, &ena_rx_ctx, skb);
+
+		ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb);
+
+		skb_record_rx_queue(skb, rx_ring->qid);
+
+		if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak) {
+			total_len += rx_ring->ena_bufs[0].len;
+			rx_copybreak_pkt++;
+			napi_gro_receive(napi, skb);
+		} else {
+			total_len += skb->len;
+			napi_gro_frags(napi);
+		}
+
+		res_budget--;
+	} while (likely(res_budget));
+
+	work_done = budget - res_budget;
+	rx_ring->per_napi_bytes += total_len;
+	rx_ring->per_napi_packets += work_done;
+	u64_stats_update_begin(&rx_ring->syncp);
+	rx_ring->rx_stats.bytes += total_len;
+	rx_ring->rx_stats.cnt += work_done;
+	rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt;
+	u64_stats_update_end(&rx_ring->syncp);
+
+	rx_ring->next_to_clean = next_to_clean;
+
+	refill_required = ena_com_sq_empty_space(rx_ring->ena_com_io_sq);
+	refill_threshold = rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER;
+
+	/* Optimization, try to batch new rx buffers */
+	if (refill_required > refill_threshold) {
+		ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
+		ena_refill_rx_bufs(rx_ring, refill_required);
+	}
+
+	return work_done;
+
+error:
+	adapter = netdev_priv(rx_ring->netdev);
+
+	u64_stats_update_begin(&rx_ring->syncp);
+	rx_ring->rx_stats.bad_desc_num++;
+	u64_stats_update_end(&rx_ring->syncp);
+
+	/* Too many desc from the device. Trigger reset */
+	set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+
+	return 0;
+}
+
+inline void ena_adjust_intr_moderation(struct ena_ring *rx_ring,
+				       struct ena_ring *tx_ring)
+{
+	/* We apply adaptive moderation on Rx path only.
+	 * Tx uses static interrupt moderation.
+	 */
+	ena_com_calculate_interrupt_delay(rx_ring->ena_dev,
+					  rx_ring->per_napi_packets,
+					  rx_ring->per_napi_bytes,
+					  &rx_ring->smoothed_interval,
+					  &rx_ring->moder_tbl_idx);
+
+	/* Reset per napi packets/bytes */
+	tx_ring->per_napi_packets = 0;
+	tx_ring->per_napi_bytes = 0;
+	rx_ring->per_napi_packets = 0;
+	rx_ring->per_napi_bytes = 0;
+}
+
+static inline void ena_update_ring_numa_node(struct ena_ring *tx_ring,
+					     struct ena_ring *rx_ring)
+{
+	int cpu = get_cpu();
+	int numa_node;
+
+	/* Check only one ring since the 2 rings are running on the same cpu */
+	if (likely(tx_ring->cpu == cpu))
+		goto out;
+
+	numa_node = cpu_to_node(cpu);
+	put_cpu();
+
+	if (numa_node != NUMA_NO_NODE) {
+		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);
+		ena_com_update_numa_node(rx_ring->ena_com_io_cq, numa_node);
+	}
+
+	tx_ring->cpu = cpu;
+	rx_ring->cpu = cpu;
+
+	return;
+out:
+	put_cpu();
+}
+
+static int ena_io_poll(struct napi_struct *napi, int budget)
+{
+	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
+	struct ena_ring *tx_ring, *rx_ring;
+	struct ena_eth_io_intr_reg intr_reg;
+
+	u32 tx_work_done;
+	u32 rx_work_done;
+	int tx_budget;
+	int napi_comp_call = 0;
+	int ret;
+
+	tx_ring = ena_napi->tx_ring;
+	rx_ring = ena_napi->rx_ring;
+
+	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
+
+	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {
+		napi_complete_done(napi, 0);
+		return 0;
+	}
+
+	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);
+	rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget);
+
+	if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {
+		napi_complete_done(napi, rx_work_done);
+
+		napi_comp_call = 1;
+		/* Tx and Rx share the same interrupt vector */
+		if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
+			ena_adjust_intr_moderation(rx_ring, tx_ring);
+
+		/* Update intr register: rx intr delay, tx intr delay and
+		 * interrupt unmask
+		 */
+		ena_com_update_intr_reg(&intr_reg,
+					rx_ring->smoothed_interval,
+					tx_ring->smoothed_interval,
+					true);
+
+		/* It is a shared MSI-X. Tx and Rx CQ have pointer to it.
+		 * So we use one of them to reach the intr reg
+		 */
+		ena_com_unmask_intr(rx_ring->ena_com_io_cq, &intr_reg);
+
+		ena_update_ring_numa_node(tx_ring, rx_ring);
+
+		ret = rx_work_done;
+	} else {
+		ret = budget;
+	}
+
+	u64_stats_update_begin(&tx_ring->syncp);
+	tx_ring->tx_stats.napi_comp += napi_comp_call;
+	tx_ring->tx_stats.tx_poll++;
+	u64_stats_update_end(&tx_ring->syncp);
+
+	return ret;
+}
+
+static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)data;
+
+	ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
+
+	/* Don't call the aenq handler before probe is done */
+	if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)))
+		ena_com_aenq_intr_handler(adapter->ena_dev, data);
+
+	return IRQ_HANDLED;
+}
+
+/* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx
+ * @irq: interrupt number
+ * @data: pointer to a network interface private napi device structure
+ */
+static irqreturn_t ena_intr_msix_io(int irq, void *data)
+{
+	struct ena_napi *ena_napi = data;
+
+	napi_schedule(&ena_napi->napi);
+
+	return IRQ_HANDLED;
+}
+
+static int ena_enable_msix(struct ena_adapter *adapter, int num_queues)
+{
+	int i, msix_vecs, rc;
+
+	if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
+		netif_err(adapter, probe, adapter->netdev,
+			  "Error, MSI-X is already enabled\n");
+		return -EPERM;
+	}
+
+	/* Reserved the max msix vectors we might need */
+	msix_vecs = ENA_MAX_MSIX_VEC(num_queues);
+
+	netif_dbg(adapter, probe, adapter->netdev,
+		  "trying to enable MSI-X, vectors %d\n", msix_vecs);
+
+	adapter->msix_entries = vzalloc(msix_vecs * sizeof(struct msix_entry));
+
+	if (!adapter->msix_entries)
+		return -ENOMEM;
+
+	for (i = 0; i < msix_vecs; i++)
+		adapter->msix_entries[i].entry = i;
+
+	rc = pci_enable_msix(adapter->pdev, adapter->msix_entries, msix_vecs);
+	if (rc != 0) {
+		netif_err(adapter, probe, adapter->netdev,
+			  "Failed to enable MSI-X, vectors %d rc %d\n",
+			  msix_vecs, rc);
+		return -ENOSPC;
+	}
+
+	netif_dbg(adapter, probe, adapter->netdev, "enable MSI-X, vectors %d\n",
+		  msix_vecs);
+
+	if (msix_vecs >= 1) {
+		if (ena_init_rx_cpu_rmap(adapter))
+			netif_warn(adapter, probe, adapter->netdev,
+				   "Failed to map IRQs to CPUs\n");
+	}
+
+	adapter->msix_vecs = msix_vecs;
+	set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags);
+
+	return 0;
+}
+
+static void ena_setup_mgmnt_intr(struct ena_adapter *adapter)
+{
+	u32 cpu;
+
+	snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
+		 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
+		 pci_name(adapter->pdev));
+	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler =
+		ena_intr_msix_mgmnt;
+	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
+	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
+		adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector;
+	cpu = cpumask_first(cpu_online_mask);
+	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;
+	cpumask_set_cpu(cpu,
+			&adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask);
+}
+
+static void ena_setup_io_intr(struct ena_adapter *adapter)
+{
+	struct net_device *netdev;
+	int irq_idx, i, cpu;
+
+	netdev = adapter->netdev;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		irq_idx = ENA_IO_IRQ_IDX(i);
+		cpu = i % num_online_cpus();
+
+		snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
+			 "%s-Tx-Rx-%d", netdev->name, i);
+		adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;
+		adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];
+		adapter->irq_tbl[irq_idx].vector =
+			adapter->msix_entries[irq_idx].vector;
+		adapter->irq_tbl[irq_idx].cpu = cpu;
+
+		cpumask_set_cpu(cpu,
+				&adapter->irq_tbl[irq_idx].affinity_hint_mask);
+	}
+}
+
+static int ena_request_mgmnt_irq(struct ena_adapter *adapter)
+{
+	unsigned long flags = 0;
+	struct ena_irq *irq;
+	int rc;
+
+	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
+	rc = request_irq(irq->vector, irq->handler, flags, irq->name,
+			 irq->data);
+	if (rc) {
+		netif_err(adapter, probe, adapter->netdev,
+			  "failed to request admin irq\n");
+		return rc;
+	}
+
+	netif_dbg(adapter, probe, adapter->netdev,
+		  "set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",
+		  irq->affinity_hint_mask.bits[0], irq->vector);
+
+	irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
+
+	return rc;
+}
+
+static int ena_request_io_irq(struct ena_adapter *adapter)
+{
+	unsigned long flags = 0;
+	struct ena_irq *irq;
+	int rc = 0, i, k;
+
+	if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
+		netif_err(adapter, ifup, adapter->netdev,
+			  "Failed to request I/O IRQ: MSI-X is not enabled\n");
+		return -EINVAL;
+	}
+
+	for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
+		irq = &adapter->irq_tbl[i];
+		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
+				 irq->data);
+		if (rc) {
+			netif_err(adapter, ifup, adapter->netdev,
+				  "Failed to request I/O IRQ. index %d rc %d\n",
+				   i, rc);
+			goto err;
+		}
+
+		netif_dbg(adapter, ifup, adapter->netdev,
+			  "set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",
+			  i, irq->affinity_hint_mask.bits[0], irq->vector);
+
+		irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
+	}
+
+	return rc;
+
+err:
+	for (k = ENA_IO_IRQ_FIRST_IDX; k < i; k++) {
+		irq = &adapter->irq_tbl[k];
+		free_irq(irq->vector, irq->data);
+	}
+
+	return rc;
+}
+
+static void ena_free_mgmnt_irq(struct ena_adapter *adapter)
+{
+	struct ena_irq *irq;
+
+	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
+	synchronize_irq(irq->vector);
+	irq_set_affinity_hint(irq->vector, NULL);
+	free_irq(irq->vector, irq->data);
+}
+
+static void ena_free_io_irq(struct ena_adapter *adapter)
+{
+	struct ena_irq *irq;
+	int i;
+
+#ifdef CONFIG_RFS_ACCEL
+	if (adapter->msix_vecs >= 1) {
+		free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
+		adapter->netdev->rx_cpu_rmap = NULL;
+	}
+#endif /* CONFIG_RFS_ACCEL */
+
+	for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
+		irq = &adapter->irq_tbl[i];
+		irq_set_affinity_hint(irq->vector, NULL);
+		free_irq(irq->vector, irq->data);
+	}
+}
+
+static void ena_disable_msix(struct ena_adapter *adapter)
+{
+	if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags))
+		pci_disable_msix(adapter->pdev);
+
+	if (adapter->msix_entries)
+		vfree(adapter->msix_entries);
+	adapter->msix_entries = NULL;
+}
+
+static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
+{
+	int i;
+
+	if (!netif_running(adapter->netdev))
+		return;
+
+	for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++)
+		synchronize_irq(adapter->irq_tbl[i].vector);
+}
+
+static void ena_del_napi(struct ena_adapter *adapter)
+{
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++)
+		netif_napi_del(&adapter->ena_napi[i].napi);
+}
+
+static void ena_init_napi(struct ena_adapter *adapter)
+{
+	struct ena_napi *napi;
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		napi = &adapter->ena_napi[i];
+
+		netif_napi_add(adapter->netdev,
+			       &adapter->ena_napi[i].napi,
+			       ena_io_poll,
+			       ENA_NAPI_BUDGET);
+		napi->rx_ring = &adapter->rx_ring[i];
+		napi->tx_ring = &adapter->tx_ring[i];
+		napi->qid = i;
+	}
+}
+
+static void ena_napi_disable_all(struct ena_adapter *adapter)
+{
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++)
+		napi_disable(&adapter->ena_napi[i].napi);
+}
+
+static void ena_napi_enable_all(struct ena_adapter *adapter)
+{
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++)
+		napi_enable(&adapter->ena_napi[i].napi);
+}
+
+static void ena_restore_ethtool_params(struct ena_adapter *adapter)
+{
+	adapter->tx_usecs = 0;
+	adapter->rx_usecs = 0;
+	adapter->tx_frames = 1;
+	adapter->rx_frames = 1;
+}
+
+/* Configure the Rx forwarding */
+static int ena_rss_configure(struct ena_adapter *adapter)
+{
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	int rc;
+
+	/* In case the RSS table wasn't initialized by probe */
+	if (!ena_dev->rss.tbl_log_size) {
+		rc = ena_rss_init_default(adapter);
+		if (rc && (rc != -EPERM)) {
+			netif_err(adapter, ifup, adapter->netdev,
+				  "Failed to init RSS rc: %d\n", rc);
+			return rc;
+		}
+	}
+
+	/* Set indirect table */
+	rc = ena_com_indirect_table_set(ena_dev);
+	if (unlikely(rc && rc != -EPERM))
+		return rc;
+
+	/* Configure hash function (if supported) */
+	rc = ena_com_set_hash_function(ena_dev);
+	if (unlikely(rc && (rc != -EPERM)))
+		return rc;
+
+	/* Configure hash inputs (if supported) */
+	rc = ena_com_set_hash_ctrl(ena_dev);
+	if (unlikely(rc && (rc != -EPERM)))
+		return rc;
+
+	return 0;
+}
+
+static int ena_up_complete(struct ena_adapter *adapter)
+{
+	int rc, i;
+
+	rc = ena_rss_configure(adapter);
+	if (rc)
+		return rc;
+
+	ena_init_napi(adapter);
+
+	ena_change_mtu(adapter->netdev, adapter->netdev->mtu);
+
+	ena_refill_all_rx_bufs(adapter);
+
+	/* enable transmits */
+	netif_tx_start_all_queues(adapter->netdev);
+
+	ena_restore_ethtool_params(adapter);
+
+	ena_napi_enable_all(adapter);
+
+	/* schedule napi in case we had pending packets
+	 * from the last time we disable napi
+	 */
+	for (i = 0; i < adapter->num_queues; i++)
+		napi_schedule(&adapter->ena_napi[i].napi);
+
+	return 0;
+}
+
+static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
+{
+	struct ena_com_create_io_ctx ctx = { 0 };
+	struct ena_com_dev *ena_dev;
+	struct ena_ring *tx_ring;
+	u32 msix_vector;
+	u16 ena_qid;
+	int rc;
+
+	ena_dev = adapter->ena_dev;
+
+	tx_ring = &adapter->tx_ring[qid];
+	msix_vector = ENA_IO_IRQ_IDX(qid);
+	ena_qid = ENA_IO_TXQ_IDX(qid);
+
+	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
+	ctx.qid = ena_qid;
+	ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
+	ctx.msix_vector = msix_vector;
+	ctx.queue_size = adapter->tx_ring_size;
+	ctx.numa_node = cpu_to_node(tx_ring->cpu);
+
+	rc = ena_com_create_io_queue(ena_dev, &ctx);
+	if (rc) {
+		netif_err(adapter, ifup, adapter->netdev,
+			  "Failed to create I/O TX queue num %d rc: %d\n",
+			  qid, rc);
+		return rc;
+	}
+
+	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
+				     &tx_ring->ena_com_io_sq,
+				     &tx_ring->ena_com_io_cq);
+	if (rc) {
+		netif_err(adapter, ifup, adapter->netdev,
+			  "Failed to get TX queue handlers. TX queue num %d rc: %d\n",
+			  qid, rc);
+		ena_com_destroy_io_queue(ena_dev, ena_qid);
+	}
+
+	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
+	return rc;
+}
+
+static int ena_create_all_io_tx_queues(struct ena_adapter *adapter)
+{
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	int rc, i;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		rc = ena_create_io_tx_queue(adapter, i);
+		if (rc)
+			goto create_err;
+	}
+
+	return 0;
+
+create_err:
+	while (i--)
+		ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
+
+	return rc;
+}
+
+static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
+{
+	struct ena_com_dev *ena_dev;
+	struct ena_com_create_io_ctx ctx = { 0 };
+	struct ena_ring *rx_ring;
+	u32 msix_vector;
+	u16 ena_qid;
+	int rc;
+
+	ena_dev = adapter->ena_dev;
+
+	rx_ring = &adapter->rx_ring[qid];
+	msix_vector = ENA_IO_IRQ_IDX(qid);
+	ena_qid = ENA_IO_RXQ_IDX(qid);
+
+	ctx.qid = ena_qid;
+	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
+	ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
+	ctx.msix_vector = msix_vector;
+	ctx.queue_size = adapter->rx_ring_size;
+	ctx.numa_node = cpu_to_node(rx_ring->cpu);
+
+	rc = ena_com_create_io_queue(ena_dev, &ctx);
+	if (rc) {
+		netif_err(adapter, ifup, adapter->netdev,
+			  "Failed to create I/O RX queue num %d rc: %d\n",
+			  qid, rc);
+		return rc;
+	}
+
+	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
+				     &rx_ring->ena_com_io_sq,
+				     &rx_ring->ena_com_io_cq);
+	if (rc) {
+		netif_err(adapter, ifup, adapter->netdev,
+			  "Failed to get RX queue handlers. RX queue num %d rc: %d\n",
+			  qid, rc);
+		ena_com_destroy_io_queue(ena_dev, ena_qid);
+	}
+
+	ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
+
+	return rc;
+}
+
+static int ena_create_all_io_rx_queues(struct ena_adapter *adapter)
+{
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	int rc, i;
+
+	for (i = 0; i < adapter->num_queues; i++) {
+		rc = ena_create_io_rx_queue(adapter, i);
+		if (rc)
+			goto create_err;
+	}
+
+	return 0;
+
+create_err:
+	while (i--)
+		ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
+
+	return rc;
+}
+
+static int ena_up(struct ena_adapter *adapter)
+{
+	int rc;
+
+	netdev_dbg(adapter->netdev, "%s\n", __func__);
+
+	ena_setup_io_intr(adapter);
+
+	rc = ena_request_io_irq(adapter);
+	if (rc)
+		goto err_req_irq;
+
+	/* allocate transmit descriptors */
+	rc = ena_setup_all_tx_resources(adapter);
+	if (rc)
+		goto err_setup_tx;
+
+	/* allocate receive descriptors */
+	rc = ena_setup_all_rx_resources(adapter);
+	if (rc)
+		goto err_setup_rx;
+
+	/* Create TX queues */
+	rc = ena_create_all_io_tx_queues(adapter);
+	if (rc)
+		goto err_create_tx_queues;
+
+	/* Create RX queues */
+	rc = ena_create_all_io_rx_queues(adapter);
+	if (rc)
+		goto err_create_rx_queues;
+
+	rc = ena_up_complete(adapter);
+	if (rc)
+		goto err_up;
+
+	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
+		netif_carrier_on(adapter->netdev);
+
+	u64_stats_update_begin(&adapter->syncp);
+	adapter->dev_stats.interface_up++;
+	u64_stats_update_end(&adapter->syncp);
+
+	set_bit(ENA_FLAG_DEV_UP, &adapter->flags);
+
+	return rc;
+
+err_up:
+	ena_destroy_all_rx_queues(adapter);
+err_create_rx_queues:
+	ena_destroy_all_tx_queues(adapter);
+err_create_tx_queues:
+	ena_free_all_io_rx_resources(adapter);
+err_setup_rx:
+	ena_free_all_io_tx_resources(adapter);
+err_setup_tx:
+	ena_free_io_irq(adapter);
+err_req_irq:
+
+	return rc;
+}
+
+static void ena_down(struct ena_adapter *adapter)
+{
+	netif_info(adapter, ifdown, adapter->netdev, "%s\n", __func__);
+
+	clear_bit(ENA_FLAG_DEV_UP, &adapter->flags);
+
+	u64_stats_update_begin(&adapter->syncp);
+	adapter->dev_stats.interface_down++;
+	u64_stats_update_end(&adapter->syncp);
+
+	/* After this point the napi handler won't enable the tx queue */
+	ena_napi_disable_all(adapter);
+	netif_carrier_off(adapter->netdev);
+	netif_tx_disable(adapter->netdev);
+
+	/* After destroy the queue there won't be any new interrupts */
+	ena_destroy_all_io_queues(adapter);
+
+	ena_disable_io_intr_sync(adapter);
+	ena_free_io_irq(adapter);
+	ena_del_napi(adapter);
+
+	ena_free_all_tx_bufs(adapter);
+	ena_free_all_rx_bufs(adapter);
+	ena_free_all_io_tx_resources(adapter);
+	ena_free_all_io_rx_resources(adapter);
+}
+
+/* ena_open - Called when a network interface is made active
+ * @netdev: network interface device structure
+ *
+ * Returns 0 on success, negative value on failure
+ *
+ * The open entry point is called when a network interface is made
+ * active by the system (IFF_UP).  At this point all resources needed
+ * for transmit and receive operations are allocated, the interrupt
+ * handler is registered with the OS, the watchdog timer is started,
+ * and the stack is notified that the interface is ready.
+ */
+static int ena_open(struct net_device *netdev)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	int rc;
+
+	/* Notify the stack of the actual queue counts. */
+	rc = netif_set_real_num_tx_queues(netdev, adapter->num_queues);
+	if (rc) {
+		netif_err(adapter, ifup, netdev, "Can't set num tx queues\n");
+		return rc;
+	}
+
+	rc = netif_set_real_num_rx_queues(netdev, adapter->num_queues);
+	if (rc) {
+		netif_err(adapter, ifup, netdev, "Can't set num rx queues\n");
+		return rc;
+	}
+
+	rc = ena_up(adapter);
+	if (rc)
+		return rc;
+
+	return rc;
+}
+
+/* ena_close - Disables a network interface
+ * @netdev: network interface device structure
+ *
+ * Returns 0, this is not allowed to fail
+ *
+ * The close entry point is called when an interface is de-activated
+ * by the OS.  The hardware is still under the drivers control, but
+ * needs to be disabled.  A global MAC reset is issued to stop the
+ * hardware, and all transmit and receive resources are freed.
+ */
+static int ena_close(struct net_device *netdev)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+
+	netif_dbg(adapter, ifdown, netdev, "%s\n", __func__);
+
+	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
+		ena_down(adapter);
+
+	return 0;
+}
+
+static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct sk_buff *skb)
+{
+	u32 mss = skb_shinfo(skb)->gso_size;
+	struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
+	u8 l4_protocol = 0;
+
+	if ((skb->ip_summed == CHECKSUM_PARTIAL) || mss) {
+		ena_tx_ctx->l4_csum_enable = 1;
+		if (mss) {
+			ena_tx_ctx->tso_enable = 1;
+			ena_meta->l4_hdr_len = tcp_hdr(skb)->doff;
+			ena_tx_ctx->l4_csum_partial = 0;
+		} else {
+			ena_tx_ctx->tso_enable = 0;
+			ena_meta->l4_hdr_len = 0;
+			ena_tx_ctx->l4_csum_partial = 1;
+		}
+
+		switch (ip_hdr(skb)->version) {
+		case IPVERSION:
+			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
+			if (ip_hdr(skb)->frag_off & htons(IP_DF))
+				ena_tx_ctx->df = 1;
+			if (mss)
+				ena_tx_ctx->l3_csum_enable = 1;
+			l4_protocol = ip_hdr(skb)->protocol;
+			break;
+		case 6:
+			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
+			l4_protocol = ipv6_hdr(skb)->nexthdr;
+			break;
+		default:
+			break;
+		}
+
+		if (l4_protocol == IPPROTO_TCP)
+			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
+		else
+			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
+
+		ena_meta->mss = mss;
+		ena_meta->l3_hdr_len = skb_network_header_len(skb);
+		ena_meta->l3_hdr_offset = skb_network_offset(skb);
+		ena_tx_ctx->meta_valid = 1;
+
+	} else {
+		ena_tx_ctx->meta_valid = 0;
+	}
+}
+
+static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,
+				       struct sk_buff *skb)
+{
+	int num_frags, header_len, rc;
+
+	num_frags = skb_shinfo(skb)->nr_frags;
+	header_len = skb_headlen(skb);
+
+	if (num_frags < tx_ring->sgl_size)
+		return 0;
+
+	if ((num_frags == tx_ring->sgl_size) &&
+	    (header_len < tx_ring->tx_max_header_size))
+		return 0;
+
+	u64_stats_update_begin(&tx_ring->syncp);
+	tx_ring->tx_stats.linearize++;
+	u64_stats_update_end(&tx_ring->syncp);
+
+	rc = skb_linearize(skb);
+	if (unlikely(rc)) {
+		u64_stats_update_begin(&tx_ring->syncp);
+		tx_ring->tx_stats.linearize_failed++;
+		u64_stats_update_end(&tx_ring->syncp);
+	}
+
+	return rc;
+}
+
+/* Called with netif_tx_lock. */
+static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct ena_adapter *adapter = netdev_priv(dev);
+	struct ena_tx_buffer *tx_info;
+	struct ena_com_tx_ctx ena_tx_ctx;
+	struct ena_ring *tx_ring;
+	struct netdev_queue *txq;
+	struct ena_com_buf *ena_buf;
+	void *push_hdr;
+	u32 len, last_frag;
+	u16 next_to_use;
+	u16 req_id;
+	u16 push_len;
+	u16 header_len;
+	dma_addr_t dma;
+	int qid, rc, nb_hw_desc;
+	int i = -1;
+
+	netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
+	/*  Determine which tx ring we will be placed on */
+	qid = skb_get_queue_mapping(skb);
+	tx_ring = &adapter->tx_ring[qid];
+	txq = netdev_get_tx_queue(dev, qid);
+
+	rc = ena_check_and_linearize_skb(tx_ring, skb);
+	if (unlikely(rc))
+		goto error_drop_packet;
+
+	skb_tx_timestamp(skb);
+	len = skb_headlen(skb);
+
+	next_to_use = tx_ring->next_to_use;
+	req_id = tx_ring->free_tx_ids[next_to_use];
+	tx_info = &tx_ring->tx_buffer_info[req_id];
+	tx_info->num_of_bufs = 0;
+
+	WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
+	ena_buf = tx_info->bufs;
+	tx_info->skb = skb;
+
+	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+		/* prepared the push buffer */
+		push_len = min_t(u32, len, tx_ring->tx_max_header_size);
+		header_len = push_len;
+		push_hdr = skb->data;
+	} else {
+		push_len = 0;
+		header_len = min_t(u32, len, tx_ring->tx_max_header_size);
+		push_hdr = NULL;
+	}
+
+	netif_dbg(adapter, tx_queued, dev,
+		  "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
+		  push_hdr, push_len);
+
+	if (len > push_len) {
+		dma = dma_map_single(tx_ring->dev, skb->data + push_len,
+				     len - push_len, DMA_TO_DEVICE);
+		if (dma_mapping_error(tx_ring->dev, dma))
+			goto error_report_dma_error;
+
+		ena_buf->paddr = dma;
+		ena_buf->len = len - push_len;
+
+		ena_buf++;
+		tx_info->num_of_bufs++;
+	}
+
+	last_frag = skb_shinfo(skb)->nr_frags;
+
+	for (i = 0; i < last_frag; i++) {
+		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+
+		len = skb_frag_size(frag);
+		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
+				       DMA_TO_DEVICE);
+		if (dma_mapping_error(tx_ring->dev, dma))
+			goto error_report_dma_error;
+
+		ena_buf->paddr = dma;
+		ena_buf->len = len;
+		ena_buf++;
+	}
+
+	tx_info->num_of_bufs += last_frag;
+
+	memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
+	ena_tx_ctx.ena_bufs = tx_info->bufs;
+	ena_tx_ctx.push_header = push_hdr;
+	ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
+	ena_tx_ctx.req_id = req_id;
+	ena_tx_ctx.header_len = header_len;
+
+	/* set flags and meta data */
+	ena_tx_csum(&ena_tx_ctx, skb);
+
+	/* prepare the packet's descriptors to dma engine */
+	rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx,
+				&nb_hw_desc);
+
+	if (unlikely(rc)) {
+		netif_err(adapter, tx_queued, dev,
+			  "failed to prepare tx bufs\n");
+		u64_stats_update_begin(&tx_ring->syncp);
+		tx_ring->tx_stats.queue_stop++;
+		tx_ring->tx_stats.prepare_ctx_err++;
+		u64_stats_update_end(&tx_ring->syncp);
+		netif_tx_stop_queue(txq);
+		goto error_unmap_dma;
+	}
+
+	netdev_tx_sent_queue(txq, skb->len);
+
+	u64_stats_update_begin(&tx_ring->syncp);
+	tx_ring->tx_stats.cnt++;
+	tx_ring->tx_stats.bytes += skb->len;
+	u64_stats_update_end(&tx_ring->syncp);
+
+	tx_info->tx_descs = nb_hw_desc;
+	tx_info->last_jiffies = jiffies;
+
+	tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
+		tx_ring->ring_size);
+
+	/* This WMB is aimed to:
+	 * 1 - perform smp barrier before reading next_to_completion
+	 * 2 - make sure the desc were written before trigger DB
+	 */
+	wmb();
+
+	/* stop the queue when no more space available, the packet can have up
+	 * to sgl_size + 2. one for the meta descriptor and one for header
+	 * (if the header is larger than tx_max_header_size).
+	 */
+	if (unlikely(ena_com_sq_empty_space(tx_ring->ena_com_io_sq) <
+		     (tx_ring->sgl_size + 2))) {
+		netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n",
+			  __func__, qid);
+
+		netif_tx_stop_queue(txq);
+		u64_stats_update_begin(&tx_ring->syncp);
+		tx_ring->tx_stats.queue_stop++;
+		u64_stats_update_end(&tx_ring->syncp);
+
+		/* There is a rare condition where this function decide to
+		 * stop the queue but meanwhile clean_tx_irq updates
+		 * next_to_completion and terminates.
+		 * The queue will remain stopped forever.
+		 * To solve this issue this function perform rmb, check
+		 * the wakeup condition and wake up the queue if needed.
+		 */
+		smp_rmb();
+
+		if (ena_com_sq_empty_space(tx_ring->ena_com_io_sq)
+				> ENA_TX_WAKEUP_THRESH) {
+			netif_tx_wake_queue(txq);
+			u64_stats_update_begin(&tx_ring->syncp);
+			tx_ring->tx_stats.queue_wakeup++;
+			u64_stats_update_end(&tx_ring->syncp);
+		}
+	}
+
+	if (netif_xmit_stopped(txq) || !skb->xmit_more) {
+		/* trigger the dma engine */
+		ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
+		u64_stats_update_begin(&tx_ring->syncp);
+		tx_ring->tx_stats.doorbells++;
+		u64_stats_update_end(&tx_ring->syncp);
+	}
+
+	return NETDEV_TX_OK;
+
+error_report_dma_error:
+	u64_stats_update_begin(&tx_ring->syncp);
+	tx_ring->tx_stats.dma_mapping_err++;
+	u64_stats_update_end(&tx_ring->syncp);
+	netdev_warn(adapter->netdev, "failed to map skb\n");
+
+	tx_info->skb = NULL;
+
+error_unmap_dma:
+	if (i >= 0) {
+		/* save value of frag that failed */
+		last_frag = i;
+
+		/* start back at beginning and unmap skb */
+		tx_info->skb = NULL;
+		ena_buf = tx_info->bufs;
+		dma_unmap_single(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
+				 dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
+
+		/* unmap remaining mapped pages */
+		for (i = 0; i < last_frag; i++) {
+			ena_buf++;
+			dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
+				       dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
+		}
+	}
+
+error_drop_packet:
+
+	dev_kfree_skb(skb);
+	return NETDEV_TX_OK;
+}
+
+#ifdef CONFIG_NET_POLL_CONTROLLER
+static void ena_netpoll(struct net_device *netdev)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	int i;
+
+	for (i = 0; i < adapter->num_queues; i++)
+		napi_schedule(&adapter->ena_napi[i].napi);
+}
+#endif /* CONFIG_NET_POLL_CONTROLLER */
+
+static u16 ena_select_queue(struct net_device *dev, struct sk_buff *skb,
+			    void *accel_priv, select_queue_fallback_t fallback)
+{
+	u16 qid;
+	/* we suspect that this is good for in--kernel network services that
+	 * want to loop incoming skb rx to tx in normal user generated traffic,
+	 * most probably we will not get to this
+	 */
+	if (skb_rx_queue_recorded(skb))
+		qid = skb_get_rx_queue(skb);
+	else
+		qid = fallback(dev, skb);
+
+	return qid;
+}
+
+static void ena_config_host_info(struct ena_com_dev *ena_dev)
+{
+	struct ena_admin_host_info *host_info;
+	int rc;
+
+	/* Allocate only the host info */
+	rc = ena_com_allocate_host_info(ena_dev);
+	if (rc) {
+		pr_err("Cannot allocate host info\n");
+		return;
+	}
+
+	host_info = ena_dev->host_attr.host_info;
+
+	host_info->os_type = ENA_ADMIN_OS_LINUX;
+	host_info->kernel_ver = LINUX_VERSION_CODE;
+	strncpy(host_info->kernel_ver_str, utsname()->version,
+		sizeof(host_info->kernel_ver_str) - 1);
+	host_info->os_dist = 0;
+	strncpy(host_info->os_dist_str, utsname()->release,
+		sizeof(host_info->os_dist_str) - 1);
+	host_info->driver_version =
+		(DRV_MODULE_VER_MAJOR) |
+		(DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
+		(DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
+
+	rc = ena_com_set_host_attributes(ena_dev);
+	if (rc) {
+		if (rc == -EPERM)
+			pr_warn("Cannot set host attributes\n");
+		else
+			pr_err("Cannot set host attributes\n");
+
+		goto err;
+	}
+
+	return;
+
+err:
+	ena_com_delete_host_info(ena_dev);
+}
+
+static void ena_config_debug_area(struct ena_adapter *adapter)
+{
+	u32 debug_area_size;
+	int rc, ss_count;
+
+	ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS);
+	if (ss_count <= 0) {
+		netif_err(adapter, drv, adapter->netdev,
+			  "SS count is negative\n");
+		return;
+	}
+
+	/* allocate 32 bytes for each string and 64bit for the value */
+	debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count;
+
+	rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size);
+	if (rc) {
+		pr_err("Cannot allocate debug area\n");
+		return;
+	}
+
+	rc = ena_com_set_host_attributes(adapter->ena_dev);
+	if (rc) {
+		if (rc == -EPERM)
+			netif_warn(adapter, drv, adapter->netdev,
+				   "Cannot set host attributes\n");
+		else
+			netif_err(adapter, drv, adapter->netdev,
+				  "Cannot set host attributes\n");
+		goto err;
+	}
+
+	return;
+err:
+	ena_com_delete_debug_area(adapter->ena_dev);
+}
+
+static struct rtnl_link_stats64 *ena_get_stats64(struct net_device *netdev,
+						 struct rtnl_link_stats64 *stats)
+{
+	struct ena_adapter *adapter = netdev_priv(netdev);
+	struct ena_admin_basic_stats ena_stats;
+	int rc;
+
+	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
+		return NULL;
+
+	rc = ena_com_get_dev_basic_stats(adapter->ena_dev, &ena_stats);
+	if (rc)
+		return NULL;
+
+	stats->tx_bytes = ((u64)ena_stats.tx_bytes_high << 32) |
+		ena_stats.tx_bytes_low;
+	stats->rx_bytes = ((u64)ena_stats.rx_bytes_high << 32) |
+		ena_stats.rx_bytes_low;
+
+	stats->rx_packets = ((u64)ena_stats.rx_pkts_high << 32) |
+		ena_stats.rx_pkts_low;
+	stats->tx_packets = ((u64)ena_stats.tx_pkts_high << 32) |
+		ena_stats.tx_pkts_low;
+
+	stats->rx_dropped = ((u64)ena_stats.rx_drops_high << 32) |
+		ena_stats.rx_drops_low;
+
+	stats->multicast = 0;
+	stats->collisions = 0;
+
+	stats->rx_length_errors = 0;
+	stats->rx_crc_errors = 0;
+	stats->rx_frame_errors = 0;
+	stats->rx_fifo_errors = 0;
+	stats->rx_missed_errors = 0;
+	stats->tx_window_errors = 0;
+
+	stats->rx_errors = 0;
+	stats->tx_errors = 0;
+
+	return stats;
+}
+
+static const struct net_device_ops ena_netdev_ops = {
+	.ndo_open		= ena_open,
+	.ndo_stop		= ena_close,
+	.ndo_start_xmit		= ena_start_xmit,
+	.ndo_select_queue	= ena_select_queue,
+	.ndo_get_stats64	= ena_get_stats64,
+	.ndo_tx_timeout		= ena_tx_timeout,
+	.ndo_change_mtu		= ena_change_mtu,
+	.ndo_set_mac_address	= NULL,
+	.ndo_validate_addr	= eth_validate_addr,
+#ifdef CONFIG_NET_POLL_CONTROLLER
+	.ndo_poll_controller	= ena_netpoll,
+#endif /* CONFIG_NET_POLL_CONTROLLER */
+};
+
+static void ena_device_io_suspend(struct work_struct *work)
+{
+	struct ena_adapter *adapter =
+		container_of(work, struct ena_adapter, suspend_io_task);
+	struct net_device *netdev = adapter->netdev;
+
+	/* ena_napi_disable_all disables only the IO handling.
+	 * We are still subject to AENQ keep alive watchdog.
+	 */
+	u64_stats_update_begin(&adapter->syncp);
+	adapter->dev_stats.io_suspend++;
+	u64_stats_update_begin(&adapter->syncp);
+	ena_napi_disable_all(adapter);
+	netif_tx_lock(netdev);
+	netif_device_detach(netdev);
+	netif_tx_unlock(netdev);
+}
+
+static void ena_device_io_resume(struct work_struct *work)
+{
+	struct ena_adapter *adapter =
+		container_of(work, struct ena_adapter, resume_io_task);
+	struct net_device *netdev = adapter->netdev;
+
+	u64_stats_update_begin(&adapter->syncp);
+	adapter->dev_stats.io_resume++;
+	u64_stats_update_end(&adapter->syncp);
+
+	netif_device_attach(netdev);
+	ena_napi_enable_all(adapter);
+}
+
+static int ena_device_validate_params(struct ena_adapter *adapter,
+				      struct ena_com_dev_get_features_ctx *get_feat_ctx)
+{
+	struct net_device *netdev = adapter->netdev;
+	int rc;
+
+	rc = ether_addr_equal(get_feat_ctx->dev_attr.mac_addr,
+			      adapter->mac_addr);
+	if (!rc) {
+		netif_err(adapter, drv, netdev,
+			  "Error, mac address are different\n");
+		return -EINVAL;
+	}
+
+	if ((get_feat_ctx->max_queues.max_cq_num < adapter->num_queues) ||
+	    (get_feat_ctx->max_queues.max_sq_num < adapter->num_queues)) {
+		netif_err(adapter, drv, netdev,
+			  "Error, device doesn't support enough queues\n");
+		return -EINVAL;
+	}
+
+	if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) {
+		netif_err(adapter, drv, netdev,
+			  "Error, device max mtu is smaller than netdev MTU\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,
+			   struct ena_com_dev_get_features_ctx *get_feat_ctx,
+			   bool *wd_state)
+{
+	struct device *dev = &pdev->dev;
+	bool readless_supported;
+	u32 aenq_groups;
+	int dma_width;
+	int rc;
+
+	rc = ena_com_mmio_reg_read_request_init(ena_dev);
+	if (rc) {
+		dev_err(dev, "failed to init mmio read less\n");
+		return rc;
+	}
+
+	/* The PCIe configuration space revision id indicate if mmio reg
+	 * read is disabled
+	 */
+	readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);
+	ena_com_set_mmio_read_mode(ena_dev, readless_supported);
+
+	rc = ena_com_dev_reset(ena_dev);
+	if (rc) {
+		dev_err(dev, "Can not reset device\n");
+		goto err_mmio_read_less;
+	}
+
+	rc = ena_com_validate_version(ena_dev);
+	if (rc) {
+		dev_err(dev, "device version is too low\n");
+		goto err_mmio_read_less;
+	}
+
+	dma_width = ena_com_get_dma_width(ena_dev);
+	if (dma_width < 0) {
+		dev_err(dev, "Invalid dma width value %d", dma_width);
+		rc = dma_width;
+		goto err_mmio_read_less;
+	}
+
+	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_width));
+	if (rc) {
+		dev_err(dev, "pci_set_dma_mask failed 0x%x\n", rc);
+		goto err_mmio_read_less;
+	}
+
+	rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(dma_width));
+	if (rc) {
+		dev_err(dev, "err_pci_set_consistent_dma_mask failed 0x%x\n",
+			rc);
+		goto err_mmio_read_less;
+	}
+
+	/* ENA admin level init */
+	rc = ena_com_admin_init(ena_dev, &aenq_handlers, true);
+	if (rc) {
+		dev_err(dev,
+			"Can not initialize ena admin queue with device\n");
+		goto err_mmio_read_less;
+	}
+
+	/* To enable the msix interrupts the driver needs to know the number
+	 * of queues. So the driver uses polling mode to retrieve this
+	 * information
+	 */
+	ena_com_set_admin_polling_mode(ena_dev, true);
+
+	/* Get Device Attributes*/
+	rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
+	if (rc) {
+		dev_err(dev, "Cannot get attribute for ena device rc=%d\n", rc);
+		goto err_admin_init;
+	}
+
+	/* Try to turn all the available aenq groups */
+	aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
+		BIT(ENA_ADMIN_FATAL_ERROR) |
+		BIT(ENA_ADMIN_WARNING) |
+		BIT(ENA_ADMIN_NOTIFICATION) |
+		BIT(ENA_ADMIN_KEEP_ALIVE);
+
+	aenq_groups &= get_feat_ctx->aenq.supported_groups;
+
+	rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
+	if (rc) {
+		dev_err(dev, "Cannot configure aenq groups rc= %d\n", rc);
+		goto err_admin_init;
+	}
+
+	*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
+
+	ena_config_host_info(ena_dev);
+
+	return 0;
+
+err_admin_init:
+	ena_com_admin_destroy(ena_dev);
+err_mmio_read_less:
+	ena_com_mmio_reg_read_request_destroy(ena_dev);
+
+	return rc;
+}
+
+static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter,
+						    int io_vectors)
+{
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	struct device *dev = &adapter->pdev->dev;
+	int rc;
+
+	rc = ena_enable_msix(adapter, io_vectors);
+	if (rc) {
+		dev_err(dev, "Can not reserve msix vectors\n");
+		return rc;
+	}
+
+	ena_setup_mgmnt_intr(adapter);
+
+	rc = ena_request_mgmnt_irq(adapter);
+	if (rc) {
+		dev_err(dev, "Can not setup management interrupts\n");
+		goto err_disable_msix;
+	}
+
+	ena_com_set_admin_polling_mode(ena_dev, false);
+
+	ena_com_admin_aenq_enable(ena_dev);
+
+	return 0;
+
+err_disable_msix:
+	ena_disable_msix(adapter);
+
+	return rc;
+}
+
+static void ena_fw_reset_device(struct work_struct *work)
+{
+	struct ena_com_dev_get_features_ctx get_feat_ctx;
+	struct ena_adapter *adapter =
+		container_of(work, struct ena_adapter, reset_task);
+	struct net_device *netdev = adapter->netdev;
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	struct pci_dev *pdev = adapter->pdev;
+	bool dev_up, wd_state;
+	int rc;
+
+	del_timer_sync(&adapter->timer_service);
+
+	rtnl_lock();
+
+	dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
+	ena_com_set_admin_running_state(ena_dev, false);
+
+	/* After calling ena_close the tx queues and the napi
+	 * are disabled so no one can interfere or touch the
+	 * data structures
+	 */
+	ena_close(netdev);
+
+	rc = ena_com_dev_reset(ena_dev);
+	if (rc) {
+		dev_err(&pdev->dev, "Device reset failed\n");
+		goto err;
+	}
+
+	ena_free_mgmnt_irq(adapter);
+
+	ena_disable_msix(adapter);
+
+	ena_com_abort_admin_commands(ena_dev);
+
+	ena_com_wait_for_abort_completion(ena_dev);
+
+	ena_com_admin_destroy(ena_dev);
+
+	ena_com_mmio_reg_read_request_destroy(ena_dev);
+
+	/* Finish with the destroy part. Start the init part */
+
+	rc = ena_device_init(ena_dev, adapter->pdev, &get_feat_ctx, &wd_state);
+	if (rc) {
+		dev_err(&pdev->dev, "Can not initialize device\n");
+		goto err;
+	}
+	adapter->wd_state = wd_state;
+
+	rc = ena_device_validate_params(adapter, &get_feat_ctx);
+	if (rc) {
+		dev_err(&pdev->dev, "Validation of device parameters failed\n");
+		goto err_device_destroy;
+	}
+
+	rc = ena_enable_msix_and_set_admin_interrupts(adapter,
+						      adapter->num_queues);
+	if (rc) {
+		dev_err(&pdev->dev, "Enable MSI-X failed\n");
+		goto err_device_destroy;
+	}
+	/* If the interface was up before the reset bring it up */
+	if (dev_up) {
+		rc = ena_up(adapter);
+		if (rc) {
+			dev_err(&pdev->dev, "Failed to create I/O queues\n");
+			goto err_disable_msix;
+		}
+	}
+
+	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
+
+	rtnl_unlock();
+
+	dev_err(&pdev->dev, "Device reset completed successfully\n");
+
+	return;
+err_disable_msix:
+	ena_free_mgmnt_irq(adapter);
+	ena_disable_msix(adapter);
+err_device_destroy:
+	ena_com_admin_destroy(ena_dev);
+err:
+	rtnl_unlock();
+
+	dev_err(&pdev->dev,
+		"Reset attempt failed. Can not reset the device\n");
+}
+
+static void check_for_missing_tx_completions(struct ena_adapter *adapter)
+{
+	struct ena_tx_buffer *tx_buf;
+	unsigned long last_jiffies;
+	struct ena_ring *tx_ring;
+	int i, j, budget;
+	u32 missed_tx;
+
+	/* Make sure the driver doesn't turn the device in other process */
+	smp_rmb();
+
+	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
+		return;
+
+	budget = ENA_MONITORED_TX_QUEUES;
+
+	for (i = adapter->last_monitored_tx_qid; i < adapter->num_queues; i++) {
+		tx_ring = &adapter->tx_ring[i];
+
+		for (j = 0; j < tx_ring->ring_size; j++) {
+			tx_buf = &tx_ring->tx_buffer_info[j];
+			last_jiffies = tx_buf->last_jiffies;
+			if (unlikely(last_jiffies && time_is_before_jiffies(last_jiffies + TX_TIMEOUT))) {
+				netif_notice(adapter, tx_err, adapter->netdev,
+					     "Found a Tx that wasn't completed on time, qid %d, index %d.\n",
+					     tx_ring->qid, j);
+
+				u64_stats_update_begin(&tx_ring->syncp);
+				missed_tx = tx_ring->tx_stats.missing_tx_comp++;
+				u64_stats_update_end(&tx_ring->syncp);
+
+				/* Clear last jiffies so the lost buffer won't
+				 * be counted twice.
+				 */
+				tx_buf->last_jiffies = 0;
+
+				if (unlikely(missed_tx > MAX_NUM_OF_TIMEOUTED_PACKETS)) {
+					netif_err(adapter, tx_err, adapter->netdev,
+						  "The number of lost tx completion is above the threshold (%d > %d). Reset the device\n",
+						  missed_tx, MAX_NUM_OF_TIMEOUTED_PACKETS);
+					set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+				}
+			}
+		}
+
+		budget--;
+		if (!budget)
+			break;
+	}
+
+	adapter->last_monitored_tx_qid = i % adapter->num_queues;
+}
+
+/* Check for keep alive expiration */
+static void check_for_missing_keep_alive(struct ena_adapter *adapter)
+{
+	unsigned long keep_alive_expired;
+
+	if (!adapter->wd_state)
+		return;
+
+	keep_alive_expired = round_jiffies(adapter->last_keep_alive_jiffies
+					   + ENA_DEVICE_KALIVE_TIMEOUT);
+	if (unlikely(time_is_before_jiffies(keep_alive_expired))) {
+		netif_err(adapter, drv, adapter->netdev,
+			  "Keep alive watchdog timeout.\n");
+		u64_stats_update_begin(&adapter->syncp);
+		adapter->dev_stats.wd_expired++;
+		u64_stats_update_end(&adapter->syncp);
+		set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+	}
+}
+
+static void check_for_admin_com_state(struct ena_adapter *adapter)
+{
+	if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) {
+		netif_err(adapter, drv, adapter->netdev,
+			  "ENA admin queue is not in running state!\n");
+		u64_stats_update_begin(&adapter->syncp);
+		adapter->dev_stats.admin_q_pause++;
+		u64_stats_update_end(&adapter->syncp);
+		set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+	}
+}
+
+static void ena_update_host_info(struct ena_admin_host_info *host_info,
+				 struct net_device *netdev)
+{
+	host_info->supported_network_features[0] =
+		netdev->features & GENMASK_ULL(31, 0);
+	host_info->supported_network_features[1] =
+		(netdev->features & GENMASK_ULL(63, 32)) >> 32;
+}
+
+static void ena_timer_service(unsigned long data)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)data;
+	u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr;
+	struct ena_admin_host_info *host_info =
+		adapter->ena_dev->host_attr.host_info;
+
+	check_for_missing_keep_alive(adapter);
+
+	check_for_admin_com_state(adapter);
+
+	check_for_missing_tx_completions(adapter);
+
+	if (debug_area)
+		ena_dump_stats_to_buf(adapter, debug_area);
+
+	if (host_info)
+		ena_update_host_info(host_info, adapter->netdev);
+
+	if (unlikely(test_and_clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
+		netif_err(adapter, drv, adapter->netdev,
+			  "Trigger reset is on\n");
+		ena_dump_stats_to_dmesg(adapter);
+		queue_work(ena_wq, &adapter->reset_task);
+		return;
+	}
+
+	/* Reset the timer */
+	mod_timer(&adapter->timer_service, jiffies + HZ);
+}
+
+static int ena_calc_io_queue_num(struct pci_dev *pdev,
+				 struct ena_com_dev *ena_dev,
+				 struct ena_com_dev_get_features_ctx *get_feat_ctx)
+{
+	int io_sq_num, io_queue_num;
+
+	/* In case of LLQ use the llq number in the get feature cmd */
+	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+		io_sq_num = get_feat_ctx->max_queues.max_llq_num;
+
+		if (io_sq_num == 0) {
+			dev_err(&pdev->dev,
+				"Trying to use LLQ but llq_num is 0. Fall back into regular queues\n");
+
+			ena_dev->tx_mem_queue_type =
+				ENA_ADMIN_PLACEMENT_POLICY_HOST;
+			io_sq_num = get_feat_ctx->max_queues.max_sq_num;
+		}
+	} else {
+		io_sq_num = get_feat_ctx->max_queues.max_sq_num;
+	}
+
+	io_queue_num = min_t(int, num_possible_cpus(), ENA_MAX_NUM_IO_QUEUES);
+	io_queue_num = min_t(int, io_queue_num, io_sq_num);
+	io_queue_num = min_t(int, io_queue_num,
+			     get_feat_ctx->max_queues.max_cq_num);
+	/* 1 IRQ for for mgmnt and 1 IRQs for each IO direction */
+	io_queue_num = min_t(int, io_queue_num, pci_msix_vec_count(pdev) - 1);
+	if (unlikely(!io_queue_num)) {
+		dev_err(&pdev->dev, "The device doesn't have io queues\n");
+		return -EFAULT;
+	}
+
+	return io_queue_num;
+}
+
+static void ena_set_push_mode(struct pci_dev *pdev, struct ena_com_dev *ena_dev,
+			      struct ena_com_dev_get_features_ctx *get_feat_ctx)
+{
+	bool has_mem_bar;
+
+	has_mem_bar = pci_select_bars(pdev, IORESOURCE_MEM) & BIT(ENA_MEM_BAR);
+
+	/* Enable push mode if device supports LLQ */
+	if (has_mem_bar && (get_feat_ctx->max_queues.max_llq_num > 0))
+		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;
+	else
+		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
+}
+
+static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
+				 struct net_device *netdev)
+{
+	netdev_features_t dev_features = 0;
+
+	/* Set offload features */
+	if (feat->offload.tx &
+		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
+		dev_features |= NETIF_F_IP_CSUM;
+
+	if (feat->offload.tx &
+		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)
+		dev_features |= NETIF_F_IPV6_CSUM;
+
+	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
+		dev_features |= NETIF_F_TSO;
+
+	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)
+		dev_features |= NETIF_F_TSO6;
+
+	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK)
+		dev_features |= NETIF_F_TSO_ECN;
+
+	if (feat->offload.rx_supported &
+		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
+		dev_features |= NETIF_F_RXCSUM;
+
+	if (feat->offload.rx_supported &
+		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
+		dev_features |= NETIF_F_RXCSUM;
+
+	netdev->features =
+		dev_features |
+		NETIF_F_SG |
+		NETIF_F_NTUPLE |
+		NETIF_F_RXHASH |
+		NETIF_F_HIGHDMA;
+
+	netdev->hw_features |= netdev->features;
+	netdev->vlan_features |= netdev->features;
+}
+
+static void ena_set_conf_feat_params(struct ena_adapter *adapter,
+				     struct ena_com_dev_get_features_ctx *feat)
+{
+	struct net_device *netdev = adapter->netdev;
+
+	/* Copy mac address */
+	if (!is_valid_ether_addr(feat->dev_attr.mac_addr)) {
+		eth_hw_addr_random(netdev);
+		ether_addr_copy(adapter->mac_addr, netdev->dev_addr);
+	} else {
+		ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr);
+		ether_addr_copy(netdev->dev_addr, adapter->mac_addr);
+	}
+
+	/* Set offload features */
+	ena_set_dev_offloads(feat, netdev);
+
+	adapter->max_mtu = feat->dev_attr.max_mtu;
+}
+
+static int ena_rss_init_default(struct ena_adapter *adapter)
+{
+	struct ena_com_dev *ena_dev = adapter->ena_dev;
+	struct device *dev = &adapter->pdev->dev;
+	int rc, i;
+	u32 val;
+
+	rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
+	if (unlikely(rc)) {
+		dev_err(dev, "Cannot init indirect table\n");
+		goto err_rss_init;
+	}
+
+	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
+		val = ethtool_rxfh_indir_default(i, adapter->num_queues);
+		rc = ena_com_indirect_table_fill_entry(ena_dev, i,
+						       ENA_IO_RXQ_IDX(val));
+		if (unlikely(rc && (rc != -EPERM))) {
+			dev_err(dev, "Cannot fill indirect table\n");
+			goto err_fill_indir;
+		}
+	}
+
+	rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL,
+					ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
+	if (unlikely(rc && (rc != -EPERM))) {
+		dev_err(dev, "Cannot fill hash function\n");
+		goto err_fill_indir;
+	}
+
+	rc = ena_com_set_default_hash_ctrl(ena_dev);
+	if (unlikely(rc && (rc != -EPERM))) {
+		dev_err(dev, "Cannot fill hash control\n");
+		goto err_fill_indir;
+	}
+
+	return 0;
+
+err_fill_indir:
+	ena_com_rss_destroy(ena_dev);
+err_rss_init:
+
+	return rc;
+}
+
+static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
+{
+	int release_bars;
+
+	release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
+	pci_release_selected_regions(pdev, release_bars);
+}
+
+static int ena_calc_queue_size(struct pci_dev *pdev,
+			       struct ena_com_dev *ena_dev,
+			       u16 *max_tx_sgl_size,
+			       u16 *max_rx_sgl_size,
+			       struct ena_com_dev_get_features_ctx *get_feat_ctx)
+{
+	u32 queue_size = ENA_DEFAULT_RING_SIZE;
+
+	queue_size = min_t(u32, queue_size,
+			   get_feat_ctx->max_queues.max_cq_depth);
+	queue_size = min_t(u32, queue_size,
+			   get_feat_ctx->max_queues.max_sq_depth);
+
+	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
+		queue_size = min_t(u32, queue_size,
+				   get_feat_ctx->max_queues.max_llq_depth);
+
+	queue_size = rounddown_pow_of_two(queue_size);
+
+	if (unlikely(!queue_size)) {
+		dev_err(&pdev->dev, "Invalid queue size\n");
+		return -EFAULT;
+	}
+
+	*max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
+				 get_feat_ctx->max_queues.max_packet_tx_descs);
+	*max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
+				 get_feat_ctx->max_queues.max_packet_rx_descs);
+
+	return queue_size;
+}
+
+/* ena_probe - Device Initialization Routine
+ * @pdev: PCI device information struct
+ * @ent: entry in ena_pci_tbl
+ *
+ * Returns 0 on success, negative on failure
+ *
+ * ena_probe initializes an adapter identified by a pci_dev structure.
+ * The OS initialization, configuring of the adapter private structure,
+ * and a hardware reset occur.
+ */
+static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	struct ena_com_dev_get_features_ctx get_feat_ctx;
+	static int version_printed;
+	struct net_device *netdev;
+	struct ena_adapter *adapter;
+	struct ena_com_dev *ena_dev = NULL;
+	static int adapters_found;
+	int io_queue_num, bars, rc;
+	int queue_size;
+	u16 tx_sgl_size = 0;
+	u16 rx_sgl_size = 0;
+	bool wd_state;
+
+	dev_dbg(&pdev->dev, "%s\n", __func__);
+
+	if (version_printed++ == 0)
+		dev_info(&pdev->dev, "%s", version);
+
+	rc = pci_enable_device_mem(pdev);
+	if (rc) {
+		dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
+		return rc;
+	}
+
+	pci_set_master(pdev);
+
+	ena_dev = vzalloc(sizeof(*ena_dev));
+	if (!ena_dev) {
+		rc = -ENOMEM;
+		goto err_disable_device;
+	}
+
+	bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
+	rc = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
+	if (rc) {
+		dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
+			rc);
+		goto err_free_ena_dev;
+	}
+
+	ena_dev->reg_bar = ioremap(pci_resource_start(pdev, ENA_REG_BAR),
+				   pci_resource_len(pdev, ENA_REG_BAR));
+	if (!ena_dev->reg_bar) {
+		dev_err(&pdev->dev, "failed to remap regs bar\n");
+		rc = -EFAULT;
+		goto err_free_region;
+	}
+
+	ena_dev->dmadev = &pdev->dev;
+
+	rc = ena_device_init(ena_dev, pdev, &get_feat_ctx, &wd_state);
+	if (rc) {
+		dev_err(&pdev->dev, "ena device init failed\n");
+		if (rc == -ETIME)
+			rc = -EPROBE_DEFER;
+		goto err_free_region;
+	}
+
+	ena_set_push_mode(pdev, ena_dev, &get_feat_ctx);
+
+	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+		ena_dev->mem_bar = ioremap_wc(pci_resource_start(pdev, ENA_MEM_BAR),
+					      pci_resource_len(pdev, ENA_MEM_BAR));
+		if (!ena_dev->mem_bar) {
+			rc = -EFAULT;
+			goto err_device_destroy;
+		}
+	}
+
+	/* initial Tx interrupt delay, Assumes 1 usec granularity.
+	* Updated during device initialization with the real granularity
+	*/
+	ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;
+	io_queue_num = ena_calc_io_queue_num(pdev, ena_dev, &get_feat_ctx);
+	queue_size = ena_calc_queue_size(pdev, ena_dev, &tx_sgl_size,
+					 &rx_sgl_size, &get_feat_ctx);
+	if ((queue_size <= 0) || (io_queue_num <= 0)) {
+		rc = -EFAULT;
+		goto err_device_destroy;
+	}
+
+	dev_info(&pdev->dev, "creating %d io queues. queue size: %d\n",
+		 io_queue_num, queue_size);
+
+	/* dev zeroed in init_etherdev */
+	netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), io_queue_num);
+	if (!netdev) {
+		dev_err(&pdev->dev, "alloc_etherdev_mq failed\n");
+		rc = -ENOMEM;
+		goto err_device_destroy;
+	}
+
+	SET_NETDEV_DEV(netdev, &pdev->dev);
+
+	adapter = netdev_priv(netdev);
+	pci_set_drvdata(pdev, adapter);
+
+	adapter->ena_dev = ena_dev;
+	adapter->netdev = netdev;
+	adapter->pdev = pdev;
+
+	ena_set_conf_feat_params(adapter, &get_feat_ctx);
+
+	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
+
+	adapter->tx_ring_size = queue_size;
+	adapter->rx_ring_size = queue_size;
+
+	adapter->max_tx_sgl_size = tx_sgl_size;
+	adapter->max_rx_sgl_size = rx_sgl_size;
+
+	adapter->num_queues = io_queue_num;
+	adapter->last_monitored_tx_qid = 0;
+
+	adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;
+	adapter->wd_state = wd_state;
+
+	snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);
+
+	rc = ena_com_init_interrupt_moderation(adapter->ena_dev);
+	if (rc) {
+		dev_err(&pdev->dev,
+			"Failed to query interrupt moderation feature\n");
+		goto err_netdev_destroy;
+	}
+	ena_init_io_rings(adapter);
+
+	netdev->netdev_ops = &ena_netdev_ops;
+	netdev->watchdog_timeo = TX_TIMEOUT;
+	ena_set_ethtool_ops(netdev);
+
+	netdev->priv_flags |= IFF_UNICAST_FLT;
+
+	u64_stats_init(&adapter->syncp);
+
+	rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num);
+	if (rc) {
+		dev_err(&pdev->dev,
+			"Failed to enable and set the admin interrupts\n");
+		goto err_worker_destroy;
+	}
+	rc = ena_rss_init_default(adapter);
+	if (rc && (rc != -EPERM)) {
+		dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);
+		goto err_free_msix;
+	}
+
+	ena_config_debug_area(adapter);
+
+	memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len);
+
+	netif_carrier_off(netdev);
+
+	rc = register_netdev(netdev);
+	if (rc) {
+		dev_err(&pdev->dev, "Cannot register net device\n");
+		goto err_rss;
+	}
+
+	INIT_WORK(&adapter->suspend_io_task, ena_device_io_suspend);
+	INIT_WORK(&adapter->resume_io_task, ena_device_io_resume);
+	INIT_WORK(&adapter->reset_task, ena_fw_reset_device);
+
+	adapter->last_keep_alive_jiffies = jiffies;
+
+	init_timer(&adapter->timer_service);
+	adapter->timer_service.expires = round_jiffies(jiffies + HZ);
+	adapter->timer_service.function = ena_timer_service;
+	adapter->timer_service.data = (unsigned long)adapter;
+
+	add_timer(&adapter->timer_service);
+
+	dev_info(&pdev->dev, "%s found at mem %lx, mac addr %pM Queues %d\n",
+		 DEVICE_NAME, (long)pci_resource_start(pdev, 0),
+		 netdev->dev_addr, io_queue_num);
+
+	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
+
+	adapters_found++;
+
+	return 0;
+
+err_rss:
+	ena_com_delete_debug_area(ena_dev);
+	ena_com_rss_destroy(ena_dev);
+err_free_msix:
+	ena_com_dev_reset(ena_dev);
+	ena_free_mgmnt_irq(adapter);
+	ena_disable_msix(adapter);
+err_worker_destroy:
+	ena_com_destroy_interrupt_moderation(ena_dev);
+	del_timer(&adapter->timer_service);
+	cancel_work_sync(&adapter->suspend_io_task);
+	cancel_work_sync(&adapter->resume_io_task);
+err_netdev_destroy:
+	free_netdev(netdev);
+err_device_destroy:
+	ena_com_delete_host_info(ena_dev);
+	ena_com_admin_destroy(ena_dev);
+err_free_region:
+	ena_release_bars(ena_dev, pdev);
+err_free_ena_dev:
+	vfree(ena_dev);
+err_disable_device:
+	pci_disable_device(pdev);
+	return rc;
+}
+
+/*****************************************************************************/
+static int ena_sriov_configure(struct pci_dev *dev, int numvfs)
+{
+	int rc;
+
+	if (numvfs > 0) {
+		rc = pci_enable_sriov(dev, numvfs);
+		if (rc != 0) {
+			dev_err(&dev->dev,
+				"pci_enable_sriov failed to enable: %d vfs with the error: %d\n",
+				numvfs, rc);
+			return rc;
+		}
+
+		return numvfs;
+	}
+
+	if (numvfs == 0) {
+		pci_disable_sriov(dev);
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+/*****************************************************************************/
+/*****************************************************************************/
+
+/* ena_remove - Device Removal Routine
+ * @pdev: PCI device information struct
+ *
+ * ena_remove is called by the PCI subsystem to alert the driver
+ * that it should release a PCI device.
+ */
+static void ena_remove(struct pci_dev *pdev)
+{
+	struct ena_adapter *adapter = pci_get_drvdata(pdev);
+	struct ena_com_dev *ena_dev;
+	struct net_device *netdev;
+
+	if (!adapter)
+		/* This device didn't load properly and it's resources
+		 * already released, nothing to do
+		 */
+		return;
+
+	ena_dev = adapter->ena_dev;
+	netdev = adapter->netdev;
+
+#ifdef CONFIG_RFS_ACCEL
+	if ((adapter->msix_vecs >= 1) && (netdev->rx_cpu_rmap)) {
+		free_irq_cpu_rmap(netdev->rx_cpu_rmap);
+		netdev->rx_cpu_rmap = NULL;
+	}
+#endif /* CONFIG_RFS_ACCEL */
+
+	unregister_netdev(netdev);
+	del_timer_sync(&adapter->timer_service);
+
+	cancel_work_sync(&adapter->reset_task);
+
+	cancel_work_sync(&adapter->suspend_io_task);
+
+	cancel_work_sync(&adapter->resume_io_task);
+
+	ena_com_dev_reset(ena_dev);
+
+	ena_free_mgmnt_irq(adapter);
+
+	ena_disable_msix(adapter);
+
+	free_netdev(netdev);
+
+	ena_com_mmio_reg_read_request_destroy(ena_dev);
+
+	ena_com_abort_admin_commands(ena_dev);
+
+	ena_com_wait_for_abort_completion(ena_dev);
+
+	ena_com_admin_destroy(ena_dev);
+
+	ena_com_rss_destroy(ena_dev);
+
+	ena_com_delete_debug_area(ena_dev);
+
+	ena_com_delete_host_info(ena_dev);
+
+	ena_release_bars(ena_dev, pdev);
+
+	pci_disable_device(pdev);
+
+	ena_com_destroy_interrupt_moderation(ena_dev);
+
+	vfree(ena_dev);
+}
+
+static struct pci_driver ena_pci_driver = {
+	.name		= DRV_MODULE_NAME,
+	.id_table	= ena_pci_tbl,
+	.probe		= ena_probe,
+	.remove		= ena_remove,
+	.sriov_configure = ena_sriov_configure,
+};
+
+static int __init ena_init(void)
+{
+	pr_info("%s", version);
+
+	ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME);
+	if (!ena_wq) {
+		pr_err("Failed to create workqueue\n");
+		return -ENOMEM;
+	}
+
+	return pci_register_driver(&ena_pci_driver);
+}
+
+static void __exit ena_cleanup(void)
+{
+	pci_unregister_driver(&ena_pci_driver);
+
+	if (ena_wq) {
+		destroy_workqueue(ena_wq);
+		ena_wq = NULL;
+	}
+}
+
+/******************************************************************************
+ ******************************** AENQ Handlers *******************************
+ *****************************************************************************/
+/* ena_update_on_link_change:
+ * Notify the network interface about the change in link status
+ */
+static void ena_update_on_link_change(void *adapter_data,
+				      struct ena_admin_aenq_entry *aenq_e)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
+	struct ena_admin_aenq_link_change_desc *aenq_desc =
+		(struct ena_admin_aenq_link_change_desc *)aenq_e;
+	int status = aenq_desc->flags &
+		ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
+
+	if (status) {
+		netdev_dbg(adapter->netdev, "%s\n", __func__);
+		set_bit(ENA_FLAG_LINK_UP, &adapter->flags);
+		netif_carrier_on(adapter->netdev);
+	} else {
+		clear_bit(ENA_FLAG_LINK_UP, &adapter->flags);
+		netif_carrier_off(adapter->netdev);
+	}
+}
+
+static void ena_keep_alive_wd(void *adapter_data,
+			      struct ena_admin_aenq_entry *aenq_e)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
+
+	adapter->last_keep_alive_jiffies = jiffies;
+}
+
+static void ena_notification(void *adapter_data,
+			     struct ena_admin_aenq_entry *aenq_e)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
+
+	WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
+	     "Invalid group(%x) expected %x\n",
+	     aenq_e->aenq_common_desc.group,
+	     ENA_ADMIN_NOTIFICATION);
+
+	switch (aenq_e->aenq_common_desc.syndrom) {
+	case ENA_ADMIN_SUSPEND:
+		/* Suspend just the IO queues.
+		 * We deliberately don't suspend admin so the timer and
+		 * the keep_alive events should remain.
+		 */
+		queue_work(ena_wq, &adapter->suspend_io_task);
+		break;
+	case ENA_ADMIN_RESUME:
+		queue_work(ena_wq, &adapter->resume_io_task);
+		break;
+	default:
+		netif_err(adapter, drv, adapter->netdev,
+			  "Invalid aenq notification link state %d\n",
+			  aenq_e->aenq_common_desc.syndrom);
+	}
+}
+
+/* This handler will called for unknown event group or unimplemented handlers*/
+static void unimplemented_aenq_handler(void *data,
+				       struct ena_admin_aenq_entry *aenq_e)
+{
+	struct ena_adapter *adapter = (struct ena_adapter *)data;
+
+	netif_err(adapter, drv, adapter->netdev,
+		  "Unknown event was received or event with unimplemented handler\n");
+}
+
+static struct ena_aenq_handlers aenq_handlers = {
+	.handlers = {
+		[ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
+		[ENA_ADMIN_NOTIFICATION] = ena_notification,
+		[ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
+	},
+	.unimplemented_handler = unimplemented_aenq_handler
+};
+
+module_init(ena_init);
+module_exit(ena_cleanup);
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.h b/drivers/net/ethernet/amazon/ena/ena_netdev.h
new file mode 100644
index 000000000000..69d7e9ed5bc8
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.h
@@ -0,0 +1,324 @@
+/*
+ * Copyright 2015 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef ENA_H
+#define ENA_H
+
+#include <linux/bitops.h>
+#include <linux/etherdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/interrupt.h>
+#include <linux/netdevice.h>
+#include <linux/skbuff.h>
+
+#include "ena_com.h"
+#include "ena_eth_com.h"
+
+#define DRV_MODULE_VER_MAJOR	1
+#define DRV_MODULE_VER_MINOR	0
+#define DRV_MODULE_VER_SUBMINOR 2
+
+#define DRV_MODULE_NAME		"ena"
+#ifndef DRV_MODULE_VERSION
+#define DRV_MODULE_VERSION \
+	__stringify(DRV_MODULE_VER_MAJOR) "."	\
+	__stringify(DRV_MODULE_VER_MINOR) "."	\
+	__stringify(DRV_MODULE_VER_SUBMINOR)
+#endif
+
+#define DEVICE_NAME	"Elastic Network Adapter (ENA)"
+
+/* 1 for AENQ + ADMIN */
+#define ENA_MAX_MSIX_VEC(io_queues)	(1 + (io_queues))
+
+#define ENA_REG_BAR			0
+#define ENA_MEM_BAR			2
+#define ENA_BAR_MASK (BIT(ENA_REG_BAR) | BIT(ENA_MEM_BAR))
+
+#define ENA_DEFAULT_RING_SIZE	(1024)
+
+#define ENA_TX_WAKEUP_THRESH		(MAX_SKB_FRAGS + 2)
+#define ENA_DEFAULT_RX_COPYBREAK	(128 - NET_IP_ALIGN)
+
+/* limit the buffer size to 600 bytes to handle MTU changes from very
+ * small to very large, in which case the number of buffers per packet
+ * could exceed ENA_PKT_MAX_BUFS
+ */
+#define ENA_DEFAULT_MIN_RX_BUFF_ALLOC_SIZE 600
+
+#define ENA_MIN_MTU		128
+
+#define ENA_NAME_MAX_LEN	20
+#define ENA_IRQNAME_SIZE	40
+
+#define ENA_PKT_MAX_BUFS	19
+
+#define ENA_RX_RSS_TABLE_LOG_SIZE  7
+#define ENA_RX_RSS_TABLE_SIZE	(1 << ENA_RX_RSS_TABLE_LOG_SIZE)
+
+#define ENA_HASH_KEY_SIZE	40
+
+/* The number of tx packet completions that will be handled each NAPI poll
+ * cycle is ring_size / ENA_TX_POLL_BUDGET_DIVIDER.
+ */
+#define ENA_TX_POLL_BUDGET_DIVIDER	4
+
+/* Refill Rx queue when number of available descriptors is below
+ * QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER
+ */
+#define ENA_RX_REFILL_THRESH_DIVIDER	8
+
+/* Number of queues to check for missing queues per timer service */
+#define ENA_MONITORED_TX_QUEUES	4
+/* Max timeout packets before device reset */
+#define MAX_NUM_OF_TIMEOUTED_PACKETS 32
+
+#define ENA_TX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1))
+
+#define ENA_RX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1))
+#define ENA_RX_RING_IDX_ADD(idx, n, ring_size) \
+	(((idx) + (n)) & ((ring_size) - 1))
+
+#define ENA_IO_TXQ_IDX(q)	(2 * (q))
+#define ENA_IO_RXQ_IDX(q)	(2 * (q) + 1)
+
+#define ENA_MGMNT_IRQ_IDX		0
+#define ENA_IO_IRQ_FIRST_IDX		1
+#define ENA_IO_IRQ_IDX(q)		(ENA_IO_IRQ_FIRST_IDX + (q))
+
+/* ENA device should send keep alive msg every 1 sec.
+ * We wait for 3 sec just to be on the safe side.
+ */
+#define ENA_DEVICE_KALIVE_TIMEOUT	(3 * HZ)
+
+#define ENA_MMIO_DISABLE_REG_READ	BIT(0)
+
+struct ena_irq {
+	irq_handler_t handler;
+	void *data;
+	int cpu;
+	u32 vector;
+	cpumask_t affinity_hint_mask;
+	char name[ENA_IRQNAME_SIZE];
+};
+
+struct ena_napi {
+	struct napi_struct napi ____cacheline_aligned;
+	struct ena_ring *tx_ring;
+	struct ena_ring *rx_ring;
+	u32 qid;
+};
+
+struct ena_tx_buffer {
+	struct sk_buff *skb;
+	/* num of ena desc for this specific skb
+	 * (includes data desc and metadata desc)
+	 */
+	u32 tx_descs;
+	/* num of buffers used by this skb */
+	u32 num_of_bufs;
+	/* Save the last jiffies to detect missing tx packets */
+	unsigned long last_jiffies;
+	struct ena_com_buf bufs[ENA_PKT_MAX_BUFS];
+} ____cacheline_aligned;
+
+struct ena_rx_buffer {
+	struct sk_buff *skb;
+	struct page *page;
+	u32 page_offset;
+	struct ena_com_buf ena_buf;
+} ____cacheline_aligned;
+
+struct ena_stats_tx {
+	u64 cnt;
+	u64 bytes;
+	u64 queue_stop;
+	u64 prepare_ctx_err;
+	u64 queue_wakeup;
+	u64 dma_mapping_err;
+	u64 linearize;
+	u64 linearize_failed;
+	u64 napi_comp;
+	u64 tx_poll;
+	u64 doorbells;
+	u64 missing_tx_comp;
+	u64 bad_req_id;
+};
+
+struct ena_stats_rx {
+	u64 cnt;
+	u64 bytes;
+	u64 refil_partial;
+	u64 bad_csum;
+	u64 page_alloc_fail;
+	u64 skb_alloc_fail;
+	u64 dma_mapping_err;
+	u64 bad_desc_num;
+	u64 rx_copybreak_pkt;
+};
+
+struct ena_ring {
+	/* Holds the empty requests for TX out of order completions */
+	u16 *free_tx_ids;
+	union {
+		struct ena_tx_buffer *tx_buffer_info;
+		struct ena_rx_buffer *rx_buffer_info;
+	};
+
+	/* cache ptr to avoid using the adapter */
+	struct device *dev;
+	struct pci_dev *pdev;
+	struct napi_struct *napi;
+	struct net_device *netdev;
+	struct ena_com_dev *ena_dev;
+	struct ena_adapter *adapter;
+	struct ena_com_io_cq *ena_com_io_cq;
+	struct ena_com_io_sq *ena_com_io_sq;
+
+	u16 next_to_use;
+	u16 next_to_clean;
+	u16 rx_copybreak;
+	u16 qid;
+	u16 mtu;
+	u16 sgl_size;
+
+	/* The maximum header length the device can handle */
+	u8 tx_max_header_size;
+
+	/* cpu for TPH */
+	int cpu;
+	 /* number of tx/rx_buffer_info's entries */
+	int ring_size;
+
+	enum ena_admin_placement_policy_type tx_mem_queue_type;
+
+	struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS];
+	u32  smoothed_interval;
+	u32  per_napi_packets;
+	u32  per_napi_bytes;
+	enum ena_intr_moder_level moder_tbl_idx;
+	struct u64_stats_sync syncp;
+	union {
+		struct ena_stats_tx tx_stats;
+		struct ena_stats_rx rx_stats;
+	};
+} ____cacheline_aligned;
+
+struct ena_stats_dev {
+	u64 tx_timeout;
+	u64 io_suspend;
+	u64 io_resume;
+	u64 wd_expired;
+	u64 interface_up;
+	u64 interface_down;
+	u64 admin_q_pause;
+};
+
+enum ena_flags_t {
+	ENA_FLAG_DEVICE_RUNNING,
+	ENA_FLAG_DEV_UP,
+	ENA_FLAG_LINK_UP,
+	ENA_FLAG_MSIX_ENABLED,
+	ENA_FLAG_TRIGGER_RESET
+};
+
+/* adapter specific private data structure */
+struct ena_adapter {
+	struct ena_com_dev *ena_dev;
+	/* OS defined structs */
+	struct net_device *netdev;
+	struct pci_dev *pdev;
+
+	/* rx packets that shorter that this len will be copied to the skb
+	 * header
+	 */
+	u32 rx_copybreak;
+	u32 max_mtu;
+
+	int num_queues;
+
+	struct msix_entry *msix_entries;
+	int msix_vecs;
+
+	u32 tx_usecs, rx_usecs; /* interrupt moderation */
+	u32 tx_frames, rx_frames; /* interrupt moderation */
+
+	u32 tx_ring_size;
+	u32 rx_ring_size;
+
+	u32 msg_enable;
+
+	u16 max_tx_sgl_size;
+	u16 max_rx_sgl_size;
+
+	u8 mac_addr[ETH_ALEN];
+
+	char name[ENA_NAME_MAX_LEN];
+
+	unsigned long flags;
+	/* TX */
+	struct ena_ring tx_ring[ENA_MAX_NUM_IO_QUEUES]
+		____cacheline_aligned_in_smp;
+
+	/* RX */
+	struct ena_ring rx_ring[ENA_MAX_NUM_IO_QUEUES]
+		____cacheline_aligned_in_smp;
+
+	struct ena_napi ena_napi[ENA_MAX_NUM_IO_QUEUES];
+
+	struct ena_irq irq_tbl[ENA_MAX_MSIX_VEC(ENA_MAX_NUM_IO_QUEUES)];
+
+	/* timer service */
+	struct work_struct reset_task;
+	struct work_struct suspend_io_task;
+	struct work_struct resume_io_task;
+	struct timer_list timer_service;
+
+	bool wd_state;
+	unsigned long last_keep_alive_jiffies;
+
+	struct u64_stats_sync syncp;
+	struct ena_stats_dev dev_stats;
+
+	/* last queue index that was checked for uncompleted tx packets */
+	u32 last_monitored_tx_qid;
+};
+
+void ena_set_ethtool_ops(struct net_device *netdev);
+
+void ena_dump_stats_to_dmesg(struct ena_adapter *adapter);
+
+void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf);
+
+int ena_get_sset_count(struct net_device *netdev, int sset);
+
+#endif /* !(ENA_H) */
diff --git a/drivers/net/ethernet/amazon/ena/ena_pci_id_tbl.h b/drivers/net/ethernet/amazon/ena/ena_pci_id_tbl.h
new file mode 100644
index 000000000000..f80d2a47fa94
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_pci_id_tbl.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2015 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef ENA_PCI_ID_TBL_H_
+#define ENA_PCI_ID_TBL_H_
+
+#ifndef PCI_VENDOR_ID_AMAZON
+#define PCI_VENDOR_ID_AMAZON 0x1d0f
+#endif
+
+#ifndef PCI_DEV_ID_ENA_PF
+#define PCI_DEV_ID_ENA_PF	0x0ec2
+#endif
+
+#ifndef PCI_DEV_ID_ENA_LLQ_PF
+#define PCI_DEV_ID_ENA_LLQ_PF	0x1ec2
+#endif
+
+#ifndef PCI_DEV_ID_ENA_VF
+#define PCI_DEV_ID_ENA_VF	0xec20
+#endif
+
+#ifndef PCI_DEV_ID_ENA_LLQ_VF
+#define PCI_DEV_ID_ENA_LLQ_VF	0xec21
+#endif
+
+#define ENA_PCI_ID_TABLE_ENTRY(devid) \
+	{PCI_DEVICE(PCI_VENDOR_ID_AMAZON, devid)},
+
+static const struct pci_device_id ena_pci_tbl[] = {
+	ENA_PCI_ID_TABLE_ENTRY(PCI_DEV_ID_ENA_PF)
+	ENA_PCI_ID_TABLE_ENTRY(PCI_DEV_ID_ENA_LLQ_PF)
+	ENA_PCI_ID_TABLE_ENTRY(PCI_DEV_ID_ENA_VF)
+	ENA_PCI_ID_TABLE_ENTRY(PCI_DEV_ID_ENA_LLQ_VF)
+	{ }
+};
+
+#endif /* ENA_PCI_ID_TBL_H_ */
diff --git a/drivers/net/ethernet/amazon/ena/ena_regs_defs.h b/drivers/net/ethernet/amazon/ena/ena_regs_defs.h
new file mode 100644
index 000000000000..26097a2b6030
--- /dev/null
+++ b/drivers/net/ethernet/amazon/ena/ena_regs_defs.h
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2015 - 2016 Amazon.com, Inc. or its affiliates.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef _ENA_REGS_H_
+#define _ENA_REGS_H_
+
+/* ena_registers offsets */
+#define ENA_REGS_VERSION_OFF		0x0
+#define ENA_REGS_CONTROLLER_VERSION_OFF		0x4
+#define ENA_REGS_CAPS_OFF		0x8
+#define ENA_REGS_CAPS_EXT_OFF		0xc
+#define ENA_REGS_AQ_BASE_LO_OFF		0x10
+#define ENA_REGS_AQ_BASE_HI_OFF		0x14
+#define ENA_REGS_AQ_CAPS_OFF		0x18
+#define ENA_REGS_ACQ_BASE_LO_OFF		0x20
+#define ENA_REGS_ACQ_BASE_HI_OFF		0x24
+#define ENA_REGS_ACQ_CAPS_OFF		0x28
+#define ENA_REGS_AQ_DB_OFF		0x2c
+#define ENA_REGS_ACQ_TAIL_OFF		0x30
+#define ENA_REGS_AENQ_CAPS_OFF		0x34
+#define ENA_REGS_AENQ_BASE_LO_OFF		0x38
+#define ENA_REGS_AENQ_BASE_HI_OFF		0x3c
+#define ENA_REGS_AENQ_HEAD_DB_OFF		0x40
+#define ENA_REGS_AENQ_TAIL_OFF		0x44
+#define ENA_REGS_INTR_MASK_OFF		0x4c
+#define ENA_REGS_DEV_CTL_OFF		0x54
+#define ENA_REGS_DEV_STS_OFF		0x58
+#define ENA_REGS_MMIO_REG_READ_OFF		0x5c
+#define ENA_REGS_MMIO_RESP_LO_OFF		0x60
+#define ENA_REGS_MMIO_RESP_HI_OFF		0x64
+#define ENA_REGS_RSS_IND_ENTRY_UPDATE_OFF		0x68
+
+/* version register */
+#define ENA_REGS_VERSION_MINOR_VERSION_MASK		0xff
+#define ENA_REGS_VERSION_MAJOR_VERSION_SHIFT		8
+#define ENA_REGS_VERSION_MAJOR_VERSION_MASK		0xff00
+
+/* controller_version register */
+#define ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK		0xff
+#define ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT		8
+#define ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK		0xff00
+#define ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT		16
+#define ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK		0xff0000
+#define ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT		24
+#define ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK		0xff000000
+
+/* caps register */
+#define ENA_REGS_CAPS_CONTIGUOUS_QUEUE_REQUIRED_MASK		0x1
+#define ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT		1
+#define ENA_REGS_CAPS_RESET_TIMEOUT_MASK		0x3e
+#define ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT		8
+#define ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK		0xff00
+
+/* aq_caps register */
+#define ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK		0xffff
+#define ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT		16
+#define ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK		0xffff0000
+
+/* acq_caps register */
+#define ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK		0xffff
+#define ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT		16
+#define ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK		0xffff0000
+
+/* aenq_caps register */
+#define ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK		0xffff
+#define ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT		16
+#define ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK		0xffff0000
+
+/* dev_ctl register */
+#define ENA_REGS_DEV_CTL_DEV_RESET_MASK		0x1
+#define ENA_REGS_DEV_CTL_AQ_RESTART_SHIFT		1
+#define ENA_REGS_DEV_CTL_AQ_RESTART_MASK		0x2
+#define ENA_REGS_DEV_CTL_QUIESCENT_SHIFT		2
+#define ENA_REGS_DEV_CTL_QUIESCENT_MASK		0x4
+#define ENA_REGS_DEV_CTL_IO_RESUME_SHIFT		3
+#define ENA_REGS_DEV_CTL_IO_RESUME_MASK		0x8
+
+/* dev_sts register */
+#define ENA_REGS_DEV_STS_READY_MASK		0x1
+#define ENA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_SHIFT		1
+#define ENA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_MASK		0x2
+#define ENA_REGS_DEV_STS_AQ_RESTART_FINISHED_SHIFT		2
+#define ENA_REGS_DEV_STS_AQ_RESTART_FINISHED_MASK		0x4
+#define ENA_REGS_DEV_STS_RESET_IN_PROGRESS_SHIFT		3
+#define ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK		0x8
+#define ENA_REGS_DEV_STS_RESET_FINISHED_SHIFT		4
+#define ENA_REGS_DEV_STS_RESET_FINISHED_MASK		0x10
+#define ENA_REGS_DEV_STS_FATAL_ERROR_SHIFT		5
+#define ENA_REGS_DEV_STS_FATAL_ERROR_MASK		0x20
+#define ENA_REGS_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_SHIFT		6
+#define ENA_REGS_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_MASK		0x40
+#define ENA_REGS_DEV_STS_QUIESCENT_STATE_ACHIEVED_SHIFT		7
+#define ENA_REGS_DEV_STS_QUIESCENT_STATE_ACHIEVED_MASK		0x80
+
+/* mmio_reg_read register */
+#define ENA_REGS_MMIO_REG_READ_REQ_ID_MASK		0xffff
+#define ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT		16
+#define ENA_REGS_MMIO_REG_READ_REG_OFF_MASK		0xffff0000
+
+/* rss_ind_entry_update register */
+#define ENA_REGS_RSS_IND_ENTRY_UPDATE_INDEX_MASK		0xffff
+#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_SHIFT		16
+#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_MASK		0xffff0000
+
+#endif /*_ENA_REGS_H_ */
diff --git a/drivers/net/ethernet/amd/7990.c b/drivers/net/ethernet/amd/7990.c
index dcf2a1f3643d..dc57f2759f44 100644
--- a/drivers/net/ethernet/amd/7990.c
+++ b/drivers/net/ethernet/amd/7990.c
@@ -45,14 +45,14 @@
 #define WRITERDP(lp, x)	out_be16(lp->base + LANCE_RDP, (x))
 #define READRDP(lp)	in_be16(lp->base + LANCE_RDP)
 
-#if defined(CONFIG_HPLANCE) || defined(CONFIG_HPLANCE_MODULE)
+#if IS_ENABLED(CONFIG_HPLANCE)
 #include "hplance.h"
 
 #undef WRITERAP
 #undef WRITERDP
 #undef READRDP
 
-#if defined(CONFIG_MVME147_NET) || defined(CONFIG_MVME147_NET_MODULE)
+#if IS_ENABLED(CONFIG_MVME147_NET)
 
 /* Lossage Factor Nine, Mr Sulu. */
 #define WRITERAP(lp, x)	(lp->writerap(lp, x))
@@ -86,7 +86,7 @@ static inline __u16 READRDP(struct lance_private *lp)
 }
 
 #endif
-#endif /* CONFIG_HPLANCE || CONFIG_HPLANCE_MODULE */
+#endif /* IS_ENABLED(CONFIG_HPLANCE) */
 
 /* debugging output macros, various flavours */
 /* #define TEST_HITS */
diff --git a/drivers/net/ethernet/amd/amd8111e.c b/drivers/net/ethernet/amd/amd8111e.c
index 94960055fa1f..f92cc97151ec 100644
--- a/drivers/net/ethernet/amd/amd8111e.c
+++ b/drivers/net/ethernet/amd/amd8111e.c
@@ -89,7 +89,7 @@ Revision History:
 #include <asm/byteorder.h>
 #include <asm/uaccess.h>
 
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
 #define AMD8111E_VLAN_TAG_USED 1
 #else
 #define AMD8111E_VLAN_TAG_USED 0
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
index a9b2709567ec..7f9216db026f 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
@@ -1708,9 +1708,9 @@ static const struct net_device_ops xgbe_netdev_ops = {
 	.ndo_set_features	= xgbe_set_features,
 };
 
-struct net_device_ops *xgbe_get_netdev_ops(void)
+const struct net_device_ops *xgbe_get_netdev_ops(void)
 {
-	return (struct net_device_ops *)&xgbe_netdev_ops;
+	return &xgbe_netdev_ops;
 }
 
 static void xgbe_rx_refresh(struct xgbe_channel *channel)
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
index 11d9f0c5b78b..4007b429c80c 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
@@ -623,7 +623,7 @@ static const struct ethtool_ops xgbe_ethtool_ops = {
 	.get_ts_info = xgbe_get_ts_info,
 };
 
-struct ethtool_ops *xgbe_get_ethtool_ops(void)
+const struct ethtool_ops *xgbe_get_ethtool_ops(void)
 {
-	return (struct ethtool_ops *)&xgbe_ethtool_ops;
+	return &xgbe_ethtool_ops;
 }
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-main.c b/drivers/net/ethernet/amd/xgbe/xgbe-main.c
index 3eee3201b58f..9de078819aa6 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-main.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-main.c
@@ -861,9 +861,15 @@ static int xgbe_resume(struct device *dev)
 	pdata->lpm_ctrl &= ~MDIO_CTRL1_LPOWER;
 	XMDIO_WRITE(pdata, MDIO_MMD_PCS, MDIO_CTRL1, pdata->lpm_ctrl);
 
-	if (netif_running(netdev))
+	if (netif_running(netdev)) {
 		ret = xgbe_powerup(netdev, XGMAC_DRIVER_CONTEXT);
 
+		/* Schedule a restart in case the link or phy state changed
+		 * while we were powered down.
+		 */
+		schedule_work(&pdata->restart_work);
+	}
+
 	DBGPR("<--xgbe_resume\n");
 
 	return ret;
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h
index 98d9d63c4353..5dd17dcea2f8 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe.h
+++ b/drivers/net/ethernet/amd/xgbe/xgbe.h
@@ -956,8 +956,9 @@ struct xgbe_prv_data {
 void xgbe_init_function_ptrs_dev(struct xgbe_hw_if *);
 void xgbe_init_function_ptrs_phy(struct xgbe_phy_if *);
 void xgbe_init_function_ptrs_desc(struct xgbe_desc_if *);
-struct net_device_ops *xgbe_get_netdev_ops(void);
-struct ethtool_ops *xgbe_get_ethtool_ops(void);
+const struct net_device_ops *xgbe_get_netdev_ops(void);
+const struct ethtool_ops *xgbe_get_ethtool_ops(void);
+
 #ifdef CONFIG_AMD_XGBE_DCB
 const struct dcbnl_rtnl_ops *xgbe_get_dcbnl_ops(void);
 #endif
diff --git a/drivers/net/ethernet/apm/xgene/Kconfig b/drivers/net/ethernet/apm/xgene/Kconfig
index 300e3b5c54e0..afccb033177b 100644
--- a/drivers/net/ethernet/apm/xgene/Kconfig
+++ b/drivers/net/ethernet/apm/xgene/Kconfig
@@ -4,6 +4,7 @@ config NET_XGENE
 	depends on ARCH_XGENE || COMPILE_TEST
 	select PHYLIB
 	select MDIO_XGENE
+	select GPIOLIB
 	help
 	  This is the Ethernet driver for the on-chip ethernet interface on the
 	  APM X-Gene SoC.
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_cle.c b/drivers/net/ethernet/apm/xgene/xgene_enet_cle.c
index 472c0fb3f4c4..23d72af83d82 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_cle.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_cle.c
@@ -32,12 +32,19 @@ static void xgene_cle_sband_to_hw(u8 frag, enum xgene_cle_prot_version ver,
 		SET_VAL(SB_HDRLEN, len);
 }
 
-static void xgene_cle_idt_to_hw(u32 dstqid, u32 fpsel,
+static void xgene_cle_idt_to_hw(struct xgene_enet_pdata *pdata,
+				u32 dstqid, u32 fpsel,
 				u32 nfpsel, u32 *idt_reg)
 {
-	*idt_reg =  SET_VAL(IDT_DSTQID, dstqid) |
-		    SET_VAL(IDT_FPSEL, fpsel) |
-		    SET_VAL(IDT_NFPSEL, nfpsel);
+	if (pdata->enet_id == XGENE_ENET1) {
+		*idt_reg = SET_VAL(IDT_DSTQID, dstqid) |
+			   SET_VAL(IDT_FPSEL1, fpsel)  |
+			   SET_VAL(IDT_NFPSEL1, nfpsel);
+	} else {
+		*idt_reg = SET_VAL(IDT_DSTQID, dstqid) |
+			   SET_VAL(IDT_FPSEL, fpsel)   |
+			   SET_VAL(IDT_NFPSEL, nfpsel);
+	}
 }
 
 static void xgene_cle_dbptr_to_hw(struct xgene_enet_pdata *pdata,
@@ -344,7 +351,7 @@ static int xgene_cle_set_rss_idt(struct xgene_enet_pdata *pdata)
 		nfpsel = 0;
 		idt_reg = 0;
 
-		xgene_cle_idt_to_hw(dstqid, fpsel, nfpsel, &idt_reg);
+		xgene_cle_idt_to_hw(pdata, dstqid, fpsel, nfpsel, &idt_reg);
 		ret = xgene_cle_dram_wr(&pdata->cle, &idt_reg, 1, i,
 					RSS_IDT, CLE_CMD_WR);
 		if (ret)
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_cle.h b/drivers/net/ethernet/apm/xgene/xgene_enet_cle.h
index 33c5f6b25824..9ac9f8e145ec 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_cle.h
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_cle.h
@@ -196,9 +196,13 @@ enum xgene_cle_ptree_dbptrs {
 #define IDT_DSTQID_POS		0
 #define IDT_DSTQID_LEN		12
 #define IDT_FPSEL_POS		12
-#define IDT_FPSEL_LEN		4
-#define IDT_NFPSEL_POS		16
-#define IDT_NFPSEL_LEN		4
+#define IDT_FPSEL_LEN		5
+#define IDT_NFPSEL_POS		17
+#define IDT_NFPSEL_LEN		5
+#define IDT_FPSEL1_POS		12
+#define IDT_FPSEL1_LEN		4
+#define IDT_NFPSEL1_POS		16
+#define IDT_NFPSEL1_LEN		4
 
 struct xgene_cle_ptree_branch {
 	bool valid;
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_ethtool.c b/drivers/net/ethernet/apm/xgene/xgene_enet_ethtool.c
index 22a7b26ca1d6..d372d4235c81 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_ethtool.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_ethtool.c
@@ -54,55 +54,68 @@ static void xgene_get_drvinfo(struct net_device *ndev,
 	sprintf(info->bus_info, "%s", pdev->name);
 }
 
-static int xgene_get_settings(struct net_device *ndev, struct ethtool_cmd *cmd)
+static int xgene_get_link_ksettings(struct net_device *ndev,
+				    struct ethtool_link_ksettings *cmd)
 {
 	struct xgene_enet_pdata *pdata = netdev_priv(ndev);
-	struct phy_device *phydev = pdata->phy_dev;
+	struct phy_device *phydev = ndev->phydev;
+	u32 supported;
 
 	if (pdata->phy_mode == PHY_INTERFACE_MODE_RGMII) {
 		if (phydev == NULL)
 			return -ENODEV;
 
-		return phy_ethtool_gset(phydev, cmd);
+		return phy_ethtool_ksettings_get(phydev, cmd);
 	} else if (pdata->phy_mode == PHY_INTERFACE_MODE_SGMII) {
 		if (pdata->mdio_driver) {
 			if (!phydev)
 				return -ENODEV;
 
-			return phy_ethtool_gset(phydev, cmd);
+			return phy_ethtool_ksettings_get(phydev, cmd);
 		}
 
-		cmd->supported = SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg |
-				 SUPPORTED_MII;
-		cmd->advertising = cmd->supported;
-		ethtool_cmd_speed_set(cmd, SPEED_1000);
-		cmd->duplex = DUPLEX_FULL;
-		cmd->port = PORT_MII;
-		cmd->transceiver = XCVR_INTERNAL;
-		cmd->autoneg = AUTONEG_ENABLE;
+		supported = SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg |
+			SUPPORTED_MII;
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.supported,
+			supported);
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.advertising,
+			supported);
+
+		cmd->base.speed = SPEED_1000;
+		cmd->base.duplex = DUPLEX_FULL;
+		cmd->base.port = PORT_MII;
+		cmd->base.autoneg = AUTONEG_ENABLE;
 	} else {
-		cmd->supported = SUPPORTED_10000baseT_Full | SUPPORTED_FIBRE;
-		cmd->advertising = cmd->supported;
-		ethtool_cmd_speed_set(cmd, SPEED_10000);
-		cmd->duplex = DUPLEX_FULL;
-		cmd->port = PORT_FIBRE;
-		cmd->transceiver = XCVR_INTERNAL;
-		cmd->autoneg = AUTONEG_DISABLE;
+		supported = SUPPORTED_10000baseT_Full | SUPPORTED_FIBRE;
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.supported,
+			supported);
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.advertising,
+			supported);
+
+		cmd->base.speed = SPEED_10000;
+		cmd->base.duplex = DUPLEX_FULL;
+		cmd->base.port = PORT_FIBRE;
+		cmd->base.autoneg = AUTONEG_DISABLE;
 	}
 
 	return 0;
 }
 
-static int xgene_set_settings(struct net_device *ndev, struct ethtool_cmd *cmd)
+static int xgene_set_link_ksettings(struct net_device *ndev,
+				    const struct ethtool_link_ksettings *cmd)
 {
 	struct xgene_enet_pdata *pdata = netdev_priv(ndev);
-	struct phy_device *phydev = pdata->phy_dev;
+	struct phy_device *phydev = ndev->phydev;
 
 	if (pdata->phy_mode == PHY_INTERFACE_MODE_RGMII) {
 		if (!phydev)
 			return -ENODEV;
 
-		return phy_ethtool_sset(phydev, cmd);
+		return phy_ethtool_ksettings_set(phydev, cmd);
 	}
 
 	if (pdata->phy_mode == PHY_INTERFACE_MODE_SGMII) {
@@ -110,7 +123,7 @@ static int xgene_set_settings(struct net_device *ndev, struct ethtool_cmd *cmd)
 			if (!phydev)
 				return -ENODEV;
 
-			return phy_ethtool_sset(phydev, cmd);
+			return phy_ethtool_ksettings_set(phydev, cmd);
 		}
 	}
 
@@ -152,12 +165,12 @@ static void xgene_get_ethtool_stats(struct net_device *ndev,
 
 static const struct ethtool_ops xgene_ethtool_ops = {
 	.get_drvinfo = xgene_get_drvinfo,
-	.get_settings = xgene_get_settings,
-	.set_settings = xgene_set_settings,
 	.get_link = ethtool_op_get_link,
 	.get_strings = xgene_get_strings,
 	.get_sset_count = xgene_get_sset_count,
-	.get_ethtool_stats = xgene_get_ethtool_stats
+	.get_ethtool_stats = xgene_get_ethtool_stats,
+	.get_link_ksettings = xgene_get_link_ksettings,
+	.set_link_ksettings = xgene_set_link_ksettings,
 };
 
 void xgene_enet_set_ethtool_ops(struct net_device *ndev)
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c b/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c
index 37a0f463b8de..c481f104a8fe 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c
@@ -713,7 +713,7 @@ static void xgene_enet_adjust_link(struct net_device *ndev)
 {
 	struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 	const struct xgene_mac_ops *mac_ops = pdata->mac_ops;
-	struct phy_device *phydev = pdata->phy_dev;
+	struct phy_device *phydev = ndev->phydev;
 
 	if (phydev->link) {
 		if (pdata->phy_speed != phydev->speed) {
@@ -761,31 +761,25 @@ int xgene_enet_phy_connect(struct net_device *ndev)
 	if (dev->of_node) {
 		for (i = 0 ; i < 2; i++) {
 			np = of_parse_phandle(dev->of_node, "phy-handle", i);
-			if (np)
+			phy_dev = of_phy_connect(ndev, np,
+						 &xgene_enet_adjust_link,
+						 0, pdata->phy_mode);
+			of_node_put(np);
+			if (phy_dev)
 				break;
 		}
 
-		if (!np) {
-			netdev_dbg(ndev, "No phy-handle found in DT\n");
-			return -ENODEV;
-		}
-
-		phy_dev = of_phy_connect(ndev, np, &xgene_enet_adjust_link,
-					 0, pdata->phy_mode);
-		of_node_put(np);
 		if (!phy_dev) {
 			netdev_err(ndev, "Could not connect to PHY\n");
 			return -ENODEV;
 		}
-
-		pdata->phy_dev = phy_dev;
 	} else {
 #ifdef CONFIG_ACPI
 		struct acpi_device *adev = acpi_phy_find_device(dev);
 		if (adev)
-			pdata->phy_dev =  adev->driver_data;
-
-		phy_dev = pdata->phy_dev;
+			phy_dev = adev->driver_data;
+		else
+			phy_dev = NULL;
 
 		if (!phy_dev ||
 		    phy_connect_direct(ndev, phy_dev, &xgene_enet_adjust_link,
@@ -793,6 +787,8 @@ int xgene_enet_phy_connect(struct net_device *ndev)
 			netdev_err(ndev, "Could not connect to PHY\n");
 			return  -ENODEV;
 		}
+#else
+		return -ENODEV;
 #endif
 	}
 
@@ -851,8 +847,6 @@ static int xgene_mdiobus_register(struct xgene_enet_pdata *pdata,
 	if (!phy)
 		return -EIO;
 
-	pdata->phy_dev = phy;
-
 	return ret;
 }
 
@@ -892,14 +886,18 @@ int xgene_enet_mdio_config(struct xgene_enet_pdata *pdata)
 
 void xgene_enet_phy_disconnect(struct xgene_enet_pdata *pdata)
 {
-	if (pdata->phy_dev)
-		phy_disconnect(pdata->phy_dev);
+	struct net_device *ndev = pdata->ndev;
+
+	if (ndev->phydev)
+		phy_disconnect(ndev->phydev);
 }
 
 void xgene_enet_mdio_remove(struct xgene_enet_pdata *pdata)
 {
-	if (pdata->phy_dev)
-		phy_disconnect(pdata->phy_dev);
+	struct net_device *ndev = pdata->ndev;
+
+	if (ndev->phydev)
+		phy_disconnect(ndev->phydev);
 
 	mdiobus_unregister(pdata->mdio_bus);
 	mdiobus_free(pdata->mdio_bus);
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_hw.h b/drivers/net/ethernet/apm/xgene/xgene_enet_hw.h
index 179a44dceb29..8456337a237d 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_hw.h
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_hw.h
@@ -124,6 +124,12 @@ enum xgene_enet_rm {
 #define MAC_READ_REG_OFFSET		0x0c
 #define MAC_COMMAND_DONE_REG_OFFSET	0x10
 
+#define PCS_ADDR_REG_OFFSET		0x00
+#define PCS_COMMAND_REG_OFFSET		0x04
+#define PCS_WRITE_REG_OFFSET		0x08
+#define PCS_READ_REG_OFFSET		0x0c
+#define PCS_COMMAND_DONE_REG_OFFSET	0x10
+
 #define MII_MGMT_CONFIG_ADDR		0x20
 #define MII_MGMT_COMMAND_ADDR		0x24
 #define MII_MGMT_ADDRESS_ADDR		0x28
@@ -231,6 +237,8 @@ enum xgene_enet_rm {
 #define TCPHDR_LEN			6
 #define IPHDR_POS			6
 #define IPHDR_LEN			6
+#define MSS_POS				20
+#define MSS_LEN				2
 #define EC_POS				22	/* Enable checksum */
 #define EC_LEN				1
 #define ET_POS				23	/* Enable TSO */
@@ -247,6 +255,11 @@ enum xgene_enet_rm {
 
 #define LAST_BUFFER			(0x7800ULL << BUFDATALEN_POS)
 
+#define TSO_MSS0_POS			0
+#define TSO_MSS0_LEN			14
+#define TSO_MSS1_POS			16
+#define TSO_MSS1_LEN			14
+
 struct xgene_enet_raw_desc {
 	__le64 m0;
 	__le64 m1;
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
index d1d6b5eeb613..429f18fc5503 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
@@ -19,6 +19,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/gpio.h>
 #include "xgene_enet_main.h"
 #include "xgene_enet_hw.h"
 #include "xgene_enet_sgmac.h"
@@ -72,7 +73,6 @@ static int xgene_enet_refill_bufpool(struct xgene_enet_desc_ring *buf_pool,
 		skb = netdev_alloc_skb_ip_align(ndev, len);
 		if (unlikely(!skb))
 			return -ENOMEM;
-		buf_pool->rx_skb[tail] = skb;
 
 		dma_addr = dma_map_single(dev, skb->data, len, DMA_FROM_DEVICE);
 		if (dma_mapping_error(dev, dma_addr)) {
@@ -81,6 +81,8 @@ static int xgene_enet_refill_bufpool(struct xgene_enet_desc_ring *buf_pool,
 			return -EINVAL;
 		}
 
+		buf_pool->rx_skb[tail] = skb;
+
 		raw_desc->m1 = cpu_to_le64(SET_VAL(DATAADDR, dma_addr) |
 					   SET_VAL(BUFDATALEN, bufdatalen) |
 					   SET_BIT(COHERENT));
@@ -102,12 +104,21 @@ static u8 xgene_enet_hdr_len(const void *data)
 
 static void xgene_enet_delete_bufpool(struct xgene_enet_desc_ring *buf_pool)
 {
+	struct device *dev = ndev_to_dev(buf_pool->ndev);
+	struct xgene_enet_raw_desc16 *raw_desc;
+	dma_addr_t dma_addr;
 	int i;
 
 	/* Free up the buffers held by hardware */
 	for (i = 0; i < buf_pool->slots; i++) {
-		if (buf_pool->rx_skb[i])
+		if (buf_pool->rx_skb[i]) {
 			dev_kfree_skb_any(buf_pool->rx_skb[i]);
+
+			raw_desc = &buf_pool->raw_desc16[i];
+			dma_addr = GET_VAL(DATAADDR, le64_to_cpu(raw_desc->m1));
+			dma_unmap_single(dev, dma_addr, XGENE_ENET_MAX_MTU,
+					 DMA_FROM_DEVICE);
+		}
 	}
 }
 
@@ -126,6 +137,7 @@ static irqreturn_t xgene_enet_rx_irq(const int irq, void *data)
 static int xgene_enet_tx_completion(struct xgene_enet_desc_ring *cp_ring,
 				    struct xgene_enet_raw_desc *raw_desc)
 {
+	struct xgene_enet_pdata *pdata = netdev_priv(cp_ring->ndev);
 	struct sk_buff *skb;
 	struct device *dev;
 	skb_frag_t *frag;
@@ -133,6 +145,7 @@ static int xgene_enet_tx_completion(struct xgene_enet_desc_ring *cp_ring,
 	u16 skb_index;
 	u8 status;
 	int i, ret = 0;
+	u8 mss_index;
 
 	skb_index = GET_VAL(USERINFO, le64_to_cpu(raw_desc->m0));
 	skb = cp_ring->cp_skb[skb_index];
@@ -149,6 +162,13 @@ static int xgene_enet_tx_completion(struct xgene_enet_desc_ring *cp_ring,
 			       DMA_TO_DEVICE);
 	}
 
+	if (GET_BIT(ET, le64_to_cpu(raw_desc->m3))) {
+		mss_index = GET_VAL(MSS, le64_to_cpu(raw_desc->m3));
+		spin_lock(&pdata->mss_lock);
+		pdata->mss_refcnt[mss_index]--;
+		spin_unlock(&pdata->mss_lock);
+	}
+
 	/* Checking for error */
 	status = GET_VAL(LERR, le64_to_cpu(raw_desc->m0));
 	if (unlikely(status > 2)) {
@@ -167,15 +187,53 @@ static int xgene_enet_tx_completion(struct xgene_enet_desc_ring *cp_ring,
 	return ret;
 }
 
-static u64 xgene_enet_work_msg(struct sk_buff *skb)
+static int xgene_enet_setup_mss(struct net_device *ndev, u32 mss)
+{
+	struct xgene_enet_pdata *pdata = netdev_priv(ndev);
+	bool mss_index_found = false;
+	int mss_index;
+	int i;
+
+	spin_lock(&pdata->mss_lock);
+
+	/* Reuse the slot if MSS matches */
+	for (i = 0; !mss_index_found && i < NUM_MSS_REG; i++) {
+		if (pdata->mss[i] == mss) {
+			pdata->mss_refcnt[i]++;
+			mss_index = i;
+			mss_index_found = true;
+		}
+	}
+
+	/* Overwrite the slot with ref_count = 0 */
+	for (i = 0; !mss_index_found && i < NUM_MSS_REG; i++) {
+		if (!pdata->mss_refcnt[i]) {
+			pdata->mss_refcnt[i]++;
+			pdata->mac_ops->set_mss(pdata, mss, i);
+			pdata->mss[i] = mss;
+			mss_index = i;
+			mss_index_found = true;
+		}
+	}
+
+	spin_unlock(&pdata->mss_lock);
+
+	/* No slots with ref_count = 0 available, return busy */
+	if (!mss_index_found)
+		return -EBUSY;
+
+	return mss_index;
+}
+
+static int xgene_enet_work_msg(struct sk_buff *skb, u64 *hopinfo)
 {
 	struct net_device *ndev = skb->dev;
 	struct iphdr *iph;
 	u8 l3hlen = 0, l4hlen = 0;
 	u8 ethhdr, proto = 0, csum_enable = 0;
-	u64 hopinfo = 0;
 	u32 hdr_len, mss = 0;
 	u32 i, len, nr_frags;
+	int mss_index;
 
 	ethhdr = xgene_enet_hdr_len(skb->data);
 
@@ -215,7 +273,11 @@ static u64 xgene_enet_work_msg(struct sk_buff *skb)
 			if (!mss || ((skb->len - hdr_len) <= mss))
 				goto out;
 
-			hopinfo |= SET_BIT(ET);
+			mss_index = xgene_enet_setup_mss(ndev, mss);
+			if (unlikely(mss_index < 0))
+				return -EBUSY;
+
+			*hopinfo |= SET_BIT(ET) | SET_VAL(MSS, mss_index);
 		}
 	} else if (iph->protocol == IPPROTO_UDP) {
 		l4hlen = UDP_HDR_SIZE;
@@ -223,15 +285,15 @@ static u64 xgene_enet_work_msg(struct sk_buff *skb)
 	}
 out:
 	l3hlen = ip_hdrlen(skb) >> 2;
-	hopinfo |= SET_VAL(TCPHDR, l4hlen) |
-		  SET_VAL(IPHDR, l3hlen) |
-		  SET_VAL(ETHHDR, ethhdr) |
-		  SET_VAL(EC, csum_enable) |
-		  SET_VAL(IS, proto) |
-		  SET_BIT(IC) |
-		  SET_BIT(TYPE_ETH_WORK_MESSAGE);
-
-	return hopinfo;
+	*hopinfo |= SET_VAL(TCPHDR, l4hlen) |
+		    SET_VAL(IPHDR, l3hlen) |
+		    SET_VAL(ETHHDR, ethhdr) |
+		    SET_VAL(EC, csum_enable) |
+		    SET_VAL(IS, proto) |
+		    SET_BIT(IC) |
+		    SET_BIT(TYPE_ETH_WORK_MESSAGE);
+
+	return 0;
 }
 
 static u16 xgene_enet_encode_len(u16 len)
@@ -271,20 +333,22 @@ static int xgene_enet_setup_tx_desc(struct xgene_enet_desc_ring *tx_ring,
 	dma_addr_t dma_addr, pbuf_addr, *frag_dma_addr;
 	skb_frag_t *frag;
 	u16 tail = tx_ring->tail;
-	u64 hopinfo;
+	u64 hopinfo = 0;
 	u32 len, hw_len;
 	u8 ll = 0, nv = 0, idx = 0;
 	bool split = false;
 	u32 size, offset, ell_bytes = 0;
 	u32 i, fidx, nr_frags, count = 1;
+	int ret;
 
 	raw_desc = &tx_ring->raw_desc[tail];
 	tail = (tail + 1) & (tx_ring->slots - 1);
 	memset(raw_desc, 0, sizeof(struct xgene_enet_raw_desc));
 
-	hopinfo = xgene_enet_work_msg(skb);
-	if (!hopinfo)
-		return -EINVAL;
+	ret = xgene_enet_work_msg(skb, &hopinfo);
+	if (ret)
+		return ret;
+
 	raw_desc->m3 = cpu_to_le64(SET_VAL(HENQNUM, tx_ring->dst_ring_num) |
 				   hopinfo);
 
@@ -424,6 +488,9 @@ static netdev_tx_t xgene_enet_start_xmit(struct sk_buff *skb,
 		return NETDEV_TX_OK;
 
 	count = xgene_enet_setup_tx_desc(tx_ring, skb);
+	if (count == -EBUSY)
+		return NETDEV_TX_BUSY;
+
 	if (count <= 0) {
 		dev_kfree_skb_any(skb);
 		return NETDEV_TX_OK;
@@ -452,7 +519,6 @@ static int xgene_enet_rx_frame(struct xgene_enet_desc_ring *rx_ring,
 			       struct xgene_enet_raw_desc *raw_desc)
 {
 	struct net_device *ndev;
-	struct xgene_enet_pdata *pdata;
 	struct device *dev;
 	struct xgene_enet_desc_ring *buf_pool;
 	u32 datalen, skb_index;
@@ -461,7 +527,6 @@ static int xgene_enet_rx_frame(struct xgene_enet_desc_ring *rx_ring,
 	int ret = 0;
 
 	ndev = rx_ring->ndev;
-	pdata = netdev_priv(ndev);
 	dev = ndev_to_dev(rx_ring->ndev);
 	buf_pool = rx_ring->buf_pool;
 
@@ -739,8 +804,8 @@ static int xgene_enet_open(struct net_device *ndev)
 	if (ret)
 		return ret;
 
-	if (pdata->phy_dev) {
-		phy_start(pdata->phy_dev);
+	if (ndev->phydev) {
+		phy_start(ndev->phydev);
 	} else {
 		schedule_delayed_work(&pdata->link_work, PHY_POLL_LINK_OFF);
 		netif_carrier_off(ndev);
@@ -763,8 +828,8 @@ static int xgene_enet_close(struct net_device *ndev)
 	mac_ops->tx_disable(pdata);
 	mac_ops->rx_disable(pdata);
 
-	if (pdata->phy_dev)
-		phy_stop(pdata->phy_dev);
+	if (ndev->phydev)
+		phy_stop(ndev->phydev);
 	else
 		cancel_delayed_work_sync(&pdata->link_work);
 
@@ -1312,6 +1377,18 @@ static int xgene_enet_check_phy_handle(struct xgene_enet_pdata *pdata)
 	return 0;
 }
 
+static void xgene_enet_gpiod_get(struct xgene_enet_pdata *pdata)
+{
+	struct device *dev = &pdata->pdev->dev;
+
+	if (pdata->phy_mode != PHY_INTERFACE_MODE_XGMII)
+		return;
+
+	pdata->sfp_rdy = gpiod_get(dev, "rxlos", GPIOD_IN);
+	if (IS_ERR(pdata->sfp_rdy))
+		pdata->sfp_rdy = gpiod_get(dev, "sfp", GPIOD_IN);
+}
+
 static int xgene_enet_get_resources(struct xgene_enet_pdata *pdata)
 {
 	struct platform_device *pdev;
@@ -1401,6 +1478,8 @@ static int xgene_enet_get_resources(struct xgene_enet_pdata *pdata)
 	if (ret)
 		return ret;
 
+	xgene_enet_gpiod_get(pdata);
+
 	pdata->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(pdata->clk)) {
 		/* Firmware may have set up the clock already. */
@@ -1425,6 +1504,7 @@ static int xgene_enet_get_resources(struct xgene_enet_pdata *pdata)
 	} else {
 		pdata->mcx_mac_addr = base_addr + BLOCK_AXG_MAC_OFFSET;
 		pdata->mcx_mac_csr_addr = base_addr + BLOCK_AXG_MAC_CSR_OFFSET;
+		pdata->pcs_addr = base_addr + BLOCK_PCS_OFFSET;
 	}
 	pdata->rx_buff_cnt = NUM_PKT_BUF;
 
@@ -1454,10 +1534,8 @@ static int xgene_enet_init_hw(struct xgene_enet_pdata *pdata)
 		buf_pool = pdata->rx_ring[i]->buf_pool;
 		xgene_enet_init_bufpool(buf_pool);
 		ret = xgene_enet_refill_bufpool(buf_pool, pdata->rx_buff_cnt);
-		if (ret) {
-			xgene_enet_delete_desc_rings(pdata);
-			return ret;
-		}
+		if (ret)
+			goto err;
 	}
 
 	dst_ring_num = xgene_enet_dst_ring_num(pdata->rx_ring[0]);
@@ -1474,7 +1552,7 @@ static int xgene_enet_init_hw(struct xgene_enet_pdata *pdata)
 		ret = pdata->cle_ops->cle_init(pdata);
 		if (ret) {
 			netdev_err(ndev, "Preclass Tree init error\n");
-			return ret;
+			goto err;
 		}
 	} else {
 		pdata->port_ops->cle_bypass(pdata, dst_ring_num, buf_pool->id);
@@ -1484,6 +1562,10 @@ static int xgene_enet_init_hw(struct xgene_enet_pdata *pdata)
 	pdata->mac_ops->init(pdata);
 
 	return ret;
+
+err:
+	xgene_enet_delete_desc_rings(pdata);
+	return ret;
 }
 
 static void xgene_enet_setup_ops(struct xgene_enet_pdata *pdata)
@@ -1631,8 +1713,8 @@ static int xgene_enet_probe(struct platform_device *pdev)
 	}
 #endif
 	if (!pdata->enet_id) {
-		free_netdev(ndev);
-		return -ENODEV;
+		ret = -ENODEV;
+		goto err;
 	}
 
 	ret = xgene_enet_get_resources(pdata);
@@ -1643,7 +1725,7 @@ static int xgene_enet_probe(struct platform_device *pdev)
 
 	if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
 		ndev->features |= NETIF_F_TSO;
-		pdata->mss = XGENE_ENET_MSS;
+		spin_lock_init(&pdata->mss_lock);
 	}
 	ndev->hw_features = ndev->features;
 
@@ -1655,7 +1737,7 @@ static int xgene_enet_probe(struct platform_device *pdev)
 
 	ret = xgene_enet_init_hw(pdata);
 	if (ret)
-		goto err_netdev;
+		goto err;
 
 	link_state = pdata->mac_ops->link_state;
 	if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
@@ -1665,21 +1747,32 @@ static int xgene_enet_probe(struct platform_device *pdev)
 			ret = xgene_enet_mdio_config(pdata);
 		else
 			INIT_DELAYED_WORK(&pdata->link_work, link_state);
+
+		if (ret)
+			goto err1;
 	}
-	if (ret)
-		goto err;
 
 	xgene_enet_napi_add(pdata);
 	ret = register_netdev(ndev);
 	if (ret) {
 		netdev_err(ndev, "Failed to register netdev\n");
-		goto err;
+		goto err2;
 	}
 
 	return 0;
 
-err_netdev:
-	unregister_netdev(ndev);
+err2:
+	/*
+	 * If necessary, free_netdev() will call netif_napi_del() and undo
+	 * the effects of xgene_enet_napi_add()'s calls to netif_napi_add().
+	 */
+
+	if (pdata->mdio_driver)
+		xgene_enet_phy_disconnect(pdata);
+	else if (pdata->phy_mode == PHY_INTERFACE_MODE_RGMII)
+		xgene_enet_mdio_remove(pdata);
+err1:
+	xgene_enet_delete_desc_rings(pdata);
 err:
 	free_netdev(ndev);
 	return ret;
@@ -1688,11 +1781,9 @@ err:
 static int xgene_enet_remove(struct platform_device *pdev)
 {
 	struct xgene_enet_pdata *pdata;
-	const struct xgene_mac_ops *mac_ops;
 	struct net_device *ndev;
 
 	pdata = platform_get_drvdata(pdev);
-	mac_ops = pdata->mac_ops;
 	ndev = pdata->ndev;
 
 	rtnl_lock();
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.h b/drivers/net/ethernet/apm/xgene/xgene_enet_main.h
index 217546e5714a..0cda58f5a840 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.h
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.h
@@ -47,7 +47,7 @@
 #define NUM_PKT_BUF	64
 #define NUM_BUFPOOL	32
 #define MAX_EXP_BUFFS	256
-#define XGENE_ENET_MSS	1448
+#define NUM_MSS_REG	4
 #define XGENE_MIN_ENET_FRAME_SIZE	60
 
 #define XGENE_MAX_ENET_IRQ	16
@@ -143,7 +143,7 @@ struct xgene_mac_ops {
 	void (*rx_disable)(struct xgene_enet_pdata *pdata);
 	void (*set_speed)(struct xgene_enet_pdata *pdata);
 	void (*set_mac_addr)(struct xgene_enet_pdata *pdata);
-	void (*set_mss)(struct xgene_enet_pdata *pdata);
+	void (*set_mss)(struct xgene_enet_pdata *pdata, u16 mss, u8 index);
 	void (*link_state)(struct work_struct *work);
 };
 
@@ -174,7 +174,6 @@ struct xgene_cle_ops {
 struct xgene_enet_pdata {
 	struct net_device *ndev;
 	struct mii_bus *mdio_bus;
-	struct phy_device *phy_dev;
 	int phy_speed;
 	struct clk *clk;
 	struct platform_device *pdev;
@@ -196,6 +195,7 @@ struct xgene_enet_pdata {
 	void __iomem *mcx_mac_addr;
 	void __iomem *mcx_mac_csr_addr;
 	void __iomem *base_addr;
+	void __iomem *pcs_addr;
 	void __iomem *ring_csr_addr;
 	void __iomem *ring_cmd_addr;
 	int phy_mode;
@@ -212,10 +212,13 @@ struct xgene_enet_pdata {
 	u8 eth_bufnum;
 	u8 bp_bufnum;
 	u16 ring_num;
-	u32 mss;
+	u32 mss[NUM_MSS_REG];
+	u32 mss_refcnt[NUM_MSS_REG];
+	spinlock_t mss_lock;  /* mss lock */
 	u8 tx_delay;
 	u8 rx_delay;
 	bool mdio_driver;
+	struct gpio_desc *sfp_rdy;
 };
 
 struct xgene_indirect_ctl {
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.c b/drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.c
index 9c6ad0dce00f..6475f383ba83 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.c
@@ -18,6 +18,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/of_gpio.h>
+#include <linux/gpio.h>
 #include "xgene_enet_main.h"
 #include "xgene_enet_hw.h"
 #include "xgene_enet_xgmac.h"
@@ -84,6 +86,21 @@ static void xgene_enet_wr_mac(struct xgene_enet_pdata *pdata,
 			   wr_addr);
 }
 
+static void xgene_enet_wr_pcs(struct xgene_enet_pdata *pdata,
+			      u32 wr_addr, u32 wr_data)
+{
+	void __iomem *addr, *wr, *cmd, *cmd_done;
+
+	addr = pdata->pcs_addr + PCS_ADDR_REG_OFFSET;
+	wr = pdata->pcs_addr + PCS_WRITE_REG_OFFSET;
+	cmd = pdata->pcs_addr + PCS_COMMAND_REG_OFFSET;
+	cmd_done = pdata->pcs_addr + PCS_COMMAND_DONE_REG_OFFSET;
+
+	if (!xgene_enet_wr_indirect(addr, wr, cmd, cmd_done, wr_addr, wr_data))
+		netdev_err(pdata->ndev, "PCS write failed, addr: %04x\n",
+			   wr_addr);
+}
+
 static void xgene_enet_rd_csr(struct xgene_enet_pdata *pdata,
 			      u32 offset, u32 *val)
 {
@@ -122,6 +139,7 @@ static bool xgene_enet_rd_indirect(void __iomem *addr, void __iomem *rd,
 
 	return true;
 }
+
 static void xgene_enet_rd_mac(struct xgene_enet_pdata *pdata,
 			      u32 rd_addr, u32 *rd_data)
 {
@@ -137,6 +155,25 @@ static void xgene_enet_rd_mac(struct xgene_enet_pdata *pdata,
 			   rd_addr);
 }
 
+static bool xgene_enet_rd_pcs(struct xgene_enet_pdata *pdata,
+			      u32 rd_addr, u32 *rd_data)
+{
+	void __iomem *addr, *rd, *cmd, *cmd_done;
+	bool success;
+
+	addr = pdata->pcs_addr + PCS_ADDR_REG_OFFSET;
+	rd = pdata->pcs_addr + PCS_READ_REG_OFFSET;
+	cmd = pdata->pcs_addr + PCS_COMMAND_REG_OFFSET;
+	cmd_done = pdata->pcs_addr + PCS_COMMAND_DONE_REG_OFFSET;
+
+	success = xgene_enet_rd_indirect(addr, rd, cmd, cmd_done, rd_addr, rd_data);
+	if (!success)
+		netdev_err(pdata->ndev, "PCS read failed, addr: %04x\n",
+			   rd_addr);
+
+	return success;
+}
+
 static int xgene_enet_ecc_init(struct xgene_enet_pdata *pdata)
 {
 	struct net_device *ndev = pdata->ndev;
@@ -171,6 +208,17 @@ static void xgene_xgmac_reset(struct xgene_enet_pdata *pdata)
 	xgene_enet_wr_mac(pdata, AXGMAC_CONFIG_0, 0);
 }
 
+static void xgene_pcs_reset(struct xgene_enet_pdata *pdata)
+{
+	u32 data;
+
+	if (!xgene_enet_rd_pcs(pdata, PCS_CONTROL_1, &data))
+		return;
+
+	xgene_enet_wr_pcs(pdata, PCS_CONTROL_1, data | PCS_CTRL_PCS_RST);
+	xgene_enet_wr_pcs(pdata, PCS_CONTROL_1, data & ~PCS_CTRL_PCS_RST);
+}
+
 static void xgene_xgmac_set_mac_addr(struct xgene_enet_pdata *pdata)
 {
 	u32 addr0, addr1;
@@ -184,9 +232,22 @@ static void xgene_xgmac_set_mac_addr(struct xgene_enet_pdata *pdata)
 	xgene_enet_wr_mac(pdata, HSTMACADR_MSW_ADDR, addr1);
 }
 
-static void xgene_xgmac_set_mss(struct xgene_enet_pdata *pdata)
+static void xgene_xgmac_set_mss(struct xgene_enet_pdata *pdata,
+				u16 mss, u8 index)
 {
-	xgene_enet_wr_csr(pdata, XG_TSIF_MSS_REG0_ADDR, pdata->mss);
+	u8 offset;
+	u32 data;
+
+	offset = (index < 2) ? 0 : 4;
+	xgene_enet_rd_csr(pdata, XG_TSIF_MSS_REG0_ADDR + offset, &data);
+
+	if (!(index & 0x1))
+		data = SET_VAL(TSO_MSS1, data >> TSO_MSS1_POS) |
+			SET_VAL(TSO_MSS0, mss);
+	else
+		data = SET_VAL(TSO_MSS1, mss) | SET_VAL(TSO_MSS0, data);
+
+	xgene_enet_wr_csr(pdata, XG_TSIF_MSS_REG0_ADDR + offset, data);
 }
 
 static u32 xgene_enet_link_status(struct xgene_enet_pdata *pdata)
@@ -210,18 +271,17 @@ static void xgene_xgmac_init(struct xgene_enet_pdata *pdata)
 	xgene_enet_wr_mac(pdata, AXGMAC_CONFIG_1, data);
 
 	xgene_xgmac_set_mac_addr(pdata);
-	xgene_xgmac_set_mss(pdata);
 
 	xgene_enet_rd_csr(pdata, XG_RSIF_CONFIG_REG_ADDR, &data);
 	data |= CFG_RSIF_FPBUFF_TIMEOUT_EN;
 	xgene_enet_wr_csr(pdata, XG_RSIF_CONFIG_REG_ADDR, data);
 
-	xgene_enet_wr_csr(pdata, XG_CFG_BYPASS_ADDR, RESUME_TX);
-	xgene_enet_wr_csr(pdata, XGENET_RX_DV_GATE_REG_0_ADDR, 0);
 	xgene_enet_rd_csr(pdata, XG_ENET_SPARE_CFG_REG_ADDR, &data);
 	data |= BIT(12);
 	xgene_enet_wr_csr(pdata, XG_ENET_SPARE_CFG_REG_ADDR, data);
 	xgene_enet_wr_csr(pdata, XG_ENET_SPARE_CFG_REG_1_ADDR, 0x82);
+	xgene_enet_wr_csr(pdata, XGENET_RX_DV_GATE_REG_0_ADDR, 0);
+	xgene_enet_wr_csr(pdata, XG_CFG_BYPASS_ADDR, RESUME_TX);
 }
 
 static void xgene_xgmac_rx_enable(struct xgene_enet_pdata *pdata)
@@ -359,14 +419,17 @@ static void xgene_enet_link_state(struct work_struct *work)
 {
 	struct xgene_enet_pdata *pdata = container_of(to_delayed_work(work),
 					 struct xgene_enet_pdata, link_work);
+	struct gpio_desc *sfp_rdy = pdata->sfp_rdy;
 	struct net_device *ndev = pdata->ndev;
 	u32 link_status, poll_interval;
 
 	link_status = xgene_enet_link_status(pdata);
+	if (link_status && !IS_ERR(sfp_rdy) && !gpiod_get_value(sfp_rdy))
+		link_status = 0;
+
 	if (link_status) {
 		if (!netif_carrier_ok(ndev)) {
 			netif_carrier_on(ndev);
-			xgene_xgmac_init(pdata);
 			xgene_xgmac_rx_enable(pdata);
 			xgene_xgmac_tx_enable(pdata);
 			netdev_info(ndev, "Link is Up - 10Gbps\n");
@@ -380,6 +443,8 @@ static void xgene_enet_link_state(struct work_struct *work)
 			netdev_info(ndev, "Link is Down\n");
 		}
 		poll_interval = PHY_POLL_LINK_OFF;
+
+		xgene_pcs_reset(pdata);
 	}
 
 	schedule_delayed_work(&pdata->link_work, poll_interval);
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.h b/drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.h
index f1ea485f916b..360ccbd95566 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.h
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.h
@@ -24,6 +24,7 @@
 #define X2_BLOCK_ETH_MAC_CSR_OFFSET	0x3000
 #define BLOCK_AXG_MAC_OFFSET		0x0800
 #define BLOCK_AXG_MAC_CSR_OFFSET	0x2000
+#define BLOCK_PCS_OFFSET		0x3800
 
 #define XGENET_CONFIG_REG_ADDR		0x20
 #define XGENET_SRST_ADDR		0x00
@@ -72,6 +73,9 @@
 #define XG_MCX_ICM_CONFIG0_REG_0_ADDR	0x00e0
 #define XG_MCX_ICM_CONFIG2_REG_0_ADDR	0x00e8
 
+#define PCS_CONTROL_1			0x0000
+#define PCS_CTRL_PCS_RST		BIT(15)
+
 extern const struct xgene_mac_ops xgene_xgmac_ops;
 extern const struct xgene_port_ops xgene_xgport_ops;
 
diff --git a/drivers/net/ethernet/arc/emac_main.c b/drivers/net/ethernet/arc/emac_main.c
index 4bff0f3040df..b0da9693f28a 100644
--- a/drivers/net/ethernet/arc/emac_main.c
+++ b/drivers/net/ethernet/arc/emac_main.c
@@ -771,8 +771,10 @@ int arc_emac_probe(struct net_device *ndev, int interface)
 	priv->dev = dev;
 
 	priv->regs = devm_ioremap_resource(dev, &res_regs);
-	if (IS_ERR(priv->regs))
-		return PTR_ERR(priv->regs);
+	if (IS_ERR(priv->regs)) {
+		err = PTR_ERR(priv->regs);
+		goto out_put_node;
+	}
 
 	dev_dbg(dev, "Registers base address is 0x%p\n", priv->regs);
 
diff --git a/drivers/net/ethernet/arc/emac_mdio.c b/drivers/net/ethernet/arc/emac_mdio.c
index 058460bdd5a6..a22403c688c9 100644
--- a/drivers/net/ethernet/arc/emac_mdio.c
+++ b/drivers/net/ethernet/arc/emac_mdio.c
@@ -104,7 +104,7 @@ static int arc_mdio_write(struct mii_bus *bus, int phy_addr,
  * @bus: points to the mii_bus structure
  * Description: reset the MII bus
  */
-int arc_mdio_reset(struct mii_bus *bus)
+static int arc_mdio_reset(struct mii_bus *bus)
 {
 	struct arc_emac_priv *priv = bus->priv;
 	struct arc_emac_mdio_bus_data *data = &priv->bus_data;
diff --git a/drivers/net/ethernet/atheros/alx/alx.h b/drivers/net/ethernet/atheros/alx/alx.h
index 8fc93c5f6abc..6cac919272ea 100644
--- a/drivers/net/ethernet/atheros/alx/alx.h
+++ b/drivers/net/ethernet/atheros/alx/alx.h
@@ -76,11 +76,19 @@ enum alx_device_quirks {
 	ALX_DEV_QUIRK_MSI_INTX_DISABLE_BUG = BIT(0),
 };
 
+#define ALX_FLAG_USING_MSIX	BIT(0)
+#define ALX_FLAG_USING_MSI	BIT(1)
+
 struct alx_priv {
 	struct net_device *dev;
 
 	struct alx_hw hw;
 
+	/* msi-x vectors */
+	int num_vec;
+	struct msix_entry *msix_entries;
+	char irq_lbl[IFNAMSIZ + 8];
+
 	/* all descriptor memory */
 	struct {
 		dma_addr_t dma;
@@ -105,7 +113,7 @@ struct alx_priv {
 
 	u16 msg_enable;
 
-	bool msi;
+	int flags;
 
 	/* protects hw.stats */
 	spinlock_t stats_lock;
diff --git a/drivers/net/ethernet/atheros/alx/hw.c b/drivers/net/ethernet/atheros/alx/hw.c
index 1fe35e453d43..6ac40b0003a3 100644
--- a/drivers/net/ethernet/atheros/alx/hw.c
+++ b/drivers/net/ethernet/atheros/alx/hw.c
@@ -1031,6 +1031,20 @@ void alx_configure_basic(struct alx_hw *hw)
 	alx_write_mem32(hw, ALX_WRR, val);
 }
 
+void alx_mask_msix(struct alx_hw *hw, int index, bool mask)
+{
+	u32 reg, val;
+
+	reg = ALX_MSIX_ENTRY_BASE + index * PCI_MSIX_ENTRY_SIZE +
+		PCI_MSIX_ENTRY_VECTOR_CTRL;
+
+	val = mask ? PCI_MSIX_ENTRY_CTRL_MASKBIT : 0;
+
+	alx_write_mem32(hw, reg, val);
+	alx_post_write(hw);
+}
+
+
 bool alx_get_phy_info(struct alx_hw *hw)
 {
 	u16  devs1, devs2;
diff --git a/drivers/net/ethernet/atheros/alx/hw.h b/drivers/net/ethernet/atheros/alx/hw.h
index f289c05f5cb4..0191477ace51 100644
--- a/drivers/net/ethernet/atheros/alx/hw.h
+++ b/drivers/net/ethernet/atheros/alx/hw.h
@@ -562,6 +562,7 @@ int alx_reset_mac(struct alx_hw *hw);
 void alx_set_macaddr(struct alx_hw *hw, const u8 *addr);
 bool alx_phy_configured(struct alx_hw *hw);
 void alx_configure_basic(struct alx_hw *hw);
+void alx_mask_msix(struct alx_hw *hw, int index, bool mask);
 void alx_disable_rss(struct alx_hw *hw);
 bool alx_get_phy_info(struct alx_hw *hw);
 void alx_update_hw_stats(struct alx_hw *hw);
diff --git a/drivers/net/ethernet/atheros/alx/main.c b/drivers/net/ethernet/atheros/alx/main.c
index 6453148d066a..c0f84b73574d 100644
--- a/drivers/net/ethernet/atheros/alx/main.c
+++ b/drivers/net/ethernet/atheros/alx/main.c
@@ -51,6 +51,9 @@
 
 const char alx_drv_name[] = "alx";
 
+static bool msix = false;
+module_param(msix, bool, 0);
+MODULE_PARM_DESC(msix, "Enable msi-x interrupt support");
 
 static void alx_free_txbuf(struct alx_priv *alx, int entry)
 {
@@ -292,32 +295,29 @@ static int alx_poll(struct napi_struct *napi, int budget)
 	napi_complete(&alx->napi);
 
 	/* enable interrupt */
-	spin_lock_irqsave(&alx->irq_lock, flags);
-	alx->int_mask |= ALX_ISR_TX_Q0 | ALX_ISR_RX_Q0;
-	alx_write_mem32(hw, ALX_IMR, alx->int_mask);
-	spin_unlock_irqrestore(&alx->irq_lock, flags);
+	if (alx->flags & ALX_FLAG_USING_MSIX) {
+		alx_mask_msix(hw, 1, false);
+	} else {
+		spin_lock_irqsave(&alx->irq_lock, flags);
+		alx->int_mask |= ALX_ISR_TX_Q0 | ALX_ISR_RX_Q0;
+		alx_write_mem32(hw, ALX_IMR, alx->int_mask);
+		spin_unlock_irqrestore(&alx->irq_lock, flags);
+	}
 
 	alx_post_write(hw);
 
 	return work;
 }
 
-static irqreturn_t alx_intr_handle(struct alx_priv *alx, u32 intr)
+static bool alx_intr_handle_misc(struct alx_priv *alx, u32 intr)
 {
 	struct alx_hw *hw = &alx->hw;
-	bool write_int_mask = false;
-
-	spin_lock(&alx->irq_lock);
-
-	/* ACK interrupt */
-	alx_write_mem32(hw, ALX_ISR, intr | ALX_ISR_DIS);
-	intr &= alx->int_mask;
 
 	if (intr & ALX_ISR_FATAL) {
 		netif_warn(alx, hw, alx->dev,
 			   "fatal interrupt 0x%x, resetting\n", intr);
 		alx_schedule_reset(alx);
-		goto out;
+		return true;
 	}
 
 	if (intr & ALX_ISR_ALERT)
@@ -329,19 +329,32 @@ static irqreturn_t alx_intr_handle(struct alx_priv *alx, u32 intr)
 		 * is cleared, the interrupt status could be cleared.
 		 */
 		alx->int_mask &= ~ALX_ISR_PHY;
-		write_int_mask = true;
+		alx_write_mem32(hw, ALX_IMR, alx->int_mask);
 		alx_schedule_link_check(alx);
 	}
 
+	return false;
+}
+
+static irqreturn_t alx_intr_handle(struct alx_priv *alx, u32 intr)
+{
+	struct alx_hw *hw = &alx->hw;
+
+	spin_lock(&alx->irq_lock);
+
+	/* ACK interrupt */
+	alx_write_mem32(hw, ALX_ISR, intr | ALX_ISR_DIS);
+	intr &= alx->int_mask;
+
+	if (alx_intr_handle_misc(alx, intr))
+		goto out;
+
 	if (intr & (ALX_ISR_TX_Q0 | ALX_ISR_RX_Q0)) {
 		napi_schedule(&alx->napi);
 		/* mask rx/tx interrupt, enable them when napi complete */
 		alx->int_mask &= ~ALX_ISR_ALL_QUEUES;
-		write_int_mask = true;
-	}
-
-	if (write_int_mask)
 		alx_write_mem32(hw, ALX_IMR, alx->int_mask);
+	}
 
 	alx_write_mem32(hw, ALX_ISR, 0);
 
@@ -350,6 +363,46 @@ static irqreturn_t alx_intr_handle(struct alx_priv *alx, u32 intr)
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t alx_intr_msix_ring(int irq, void *data)
+{
+	struct alx_priv *alx = data;
+	struct alx_hw *hw = &alx->hw;
+
+	/* mask interrupt to ACK chip */
+	alx_mask_msix(hw, 1, true);
+	/* clear interrupt status */
+	alx_write_mem32(hw, ALX_ISR, (ALX_ISR_TX_Q0 | ALX_ISR_RX_Q0));
+
+	napi_schedule(&alx->napi);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t alx_intr_msix_misc(int irq, void *data)
+{
+	struct alx_priv *alx = data;
+	struct alx_hw *hw = &alx->hw;
+	u32 intr;
+
+	/* mask interrupt to ACK chip */
+	alx_mask_msix(hw, 0, true);
+
+	/* read interrupt status */
+	intr = alx_read_mem32(hw, ALX_ISR);
+	intr &= (alx->int_mask & ~ALX_ISR_ALL_QUEUES);
+
+	if (alx_intr_handle_misc(alx, intr))
+		return IRQ_HANDLED;
+
+	/* clear interrupt status */
+	alx_write_mem32(hw, ALX_ISR, intr);
+
+	/* enable interrupt again */
+	alx_mask_msix(hw, 0, false);
+
+	return IRQ_HANDLED;
+}
+
 static irqreturn_t alx_intr_msi(int irq, void *data)
 {
 	struct alx_priv *alx = data;
@@ -614,31 +667,136 @@ static void alx_free_rings(struct alx_priv *alx)
 static void alx_config_vector_mapping(struct alx_priv *alx)
 {
 	struct alx_hw *hw = &alx->hw;
+	u32 tbl = 0;
+
+	if (alx->flags & ALX_FLAG_USING_MSIX) {
+		tbl |= 1 << ALX_MSI_MAP_TBL1_TXQ0_SHIFT;
+		tbl |= 1 << ALX_MSI_MAP_TBL1_RXQ0_SHIFT;
+	}
 
-	alx_write_mem32(hw, ALX_MSI_MAP_TBL1, 0);
+	alx_write_mem32(hw, ALX_MSI_MAP_TBL1, tbl);
 	alx_write_mem32(hw, ALX_MSI_MAP_TBL2, 0);
 	alx_write_mem32(hw, ALX_MSI_ID_MAP, 0);
 }
 
+static bool alx_enable_msix(struct alx_priv *alx)
+{
+	int i, err, num_vec = 2;
+
+	alx->msix_entries = kcalloc(num_vec, sizeof(struct msix_entry),
+				    GFP_KERNEL);
+	if (!alx->msix_entries) {
+		netdev_warn(alx->dev, "Allocation of msix entries failed!\n");
+		return false;
+	}
+
+	for (i = 0; i < num_vec; i++)
+		alx->msix_entries[i].entry = i;
+
+	err = pci_enable_msix(alx->hw.pdev, alx->msix_entries, num_vec);
+	if (err) {
+		kfree(alx->msix_entries);
+		netdev_warn(alx->dev, "Enabling MSI-X interrupts failed!\n");
+		return false;
+	}
+
+	alx->num_vec = num_vec;
+	return true;
+}
+
+static int alx_request_msix(struct alx_priv *alx)
+{
+	struct net_device *netdev = alx->dev;
+	int i, err, vector = 0, free_vector = 0;
+
+	err = request_irq(alx->msix_entries[0].vector, alx_intr_msix_misc,
+			  0, netdev->name, alx);
+	if (err)
+		goto out_err;
+
+	vector++;
+	sprintf(alx->irq_lbl, "%s-TxRx-0", netdev->name);
+
+	err = request_irq(alx->msix_entries[vector].vector,
+			  alx_intr_msix_ring, 0, alx->irq_lbl, alx);
+		if (err)
+			goto out_free;
+
+	return 0;
+
+out_free:
+	free_irq(alx->msix_entries[free_vector++].vector, alx);
+
+	vector--;
+	for (i = 0; i < vector; i++)
+		free_irq(alx->msix_entries[free_vector++].vector, alx);
+
+out_err:
+	return err;
+}
+
+static void alx_init_intr(struct alx_priv *alx, bool msix)
+{
+	if (msix) {
+		if (alx_enable_msix(alx))
+			alx->flags |= ALX_FLAG_USING_MSIX;
+	}
+
+	if (!(alx->flags & ALX_FLAG_USING_MSIX)) {
+		alx->num_vec = 1;
+
+		if (!pci_enable_msi(alx->hw.pdev))
+			alx->flags |= ALX_FLAG_USING_MSI;
+	}
+}
+
+static void alx_disable_advanced_intr(struct alx_priv *alx)
+{
+	if (alx->flags & ALX_FLAG_USING_MSIX) {
+		kfree(alx->msix_entries);
+		pci_disable_msix(alx->hw.pdev);
+		alx->flags &= ~ALX_FLAG_USING_MSIX;
+	}
+
+	if (alx->flags & ALX_FLAG_USING_MSI) {
+		pci_disable_msi(alx->hw.pdev);
+		alx->flags &= ~ALX_FLAG_USING_MSI;
+	}
+}
+
 static void alx_irq_enable(struct alx_priv *alx)
 {
 	struct alx_hw *hw = &alx->hw;
+	int i;
 
 	/* level-1 interrupt switch */
 	alx_write_mem32(hw, ALX_ISR, 0);
 	alx_write_mem32(hw, ALX_IMR, alx->int_mask);
 	alx_post_write(hw);
+
+	if (alx->flags & ALX_FLAG_USING_MSIX)
+		/* enable all msix irqs */
+		for (i = 0; i < alx->num_vec; i++)
+			alx_mask_msix(hw, i, false);
 }
 
 static void alx_irq_disable(struct alx_priv *alx)
 {
 	struct alx_hw *hw = &alx->hw;
+	int i;
 
 	alx_write_mem32(hw, ALX_ISR, ALX_ISR_DIS);
 	alx_write_mem32(hw, ALX_IMR, 0);
 	alx_post_write(hw);
 
-	synchronize_irq(alx->hw.pdev->irq);
+	if (alx->flags & ALX_FLAG_USING_MSIX) {
+		for (i = 0; i < alx->num_vec; i++) {
+			alx_mask_msix(hw, i, true);
+			synchronize_irq(alx->msix_entries[i].vector);
+		}
+	} else {
+		synchronize_irq(alx->hw.pdev->irq);
+	}
 }
 
 static int alx_request_irq(struct alx_priv *alx)
@@ -650,9 +808,18 @@ static int alx_request_irq(struct alx_priv *alx)
 
 	msi_ctrl = (hw->imt >> 1) << ALX_MSI_RETRANS_TM_SHIFT;
 
-	if (!pci_enable_msi(alx->hw.pdev)) {
-		alx->msi = true;
+	if (alx->flags & ALX_FLAG_USING_MSIX) {
+		alx_write_mem32(hw, ALX_MSI_RETRANS_TIMER, msi_ctrl);
+		err = alx_request_msix(alx);
+		if (!err)
+			goto out;
+
+		/* msix request failed, realloc resources */
+		alx_disable_advanced_intr(alx);
+		alx_init_intr(alx, false);
+	}
 
+	if (alx->flags & ALX_FLAG_USING_MSI) {
 		alx_write_mem32(hw, ALX_MSI_RETRANS_TIMER,
 				msi_ctrl | ALX_MSI_MASK_SEL_LINE);
 		err = request_irq(pdev->irq, alx_intr_msi, 0,
@@ -660,6 +827,7 @@ static int alx_request_irq(struct alx_priv *alx)
 		if (!err)
 			goto out;
 		/* fall back to legacy interrupt */
+		alx->flags &= ~ALX_FLAG_USING_MSI;
 		pci_disable_msi(alx->hw.pdev);
 	}
 
@@ -669,19 +837,25 @@ static int alx_request_irq(struct alx_priv *alx)
 out:
 	if (!err)
 		alx_config_vector_mapping(alx);
+	else
+		netdev_err(alx->dev, "IRQ registration failed!\n");
 	return err;
 }
 
 static void alx_free_irq(struct alx_priv *alx)
 {
 	struct pci_dev *pdev = alx->hw.pdev;
+	int i;
 
-	free_irq(pdev->irq, alx);
-
-	if (alx->msi) {
-		pci_disable_msi(alx->hw.pdev);
-		alx->msi = false;
+	if (alx->flags & ALX_FLAG_USING_MSIX) {
+		/* we have only 2 vectors without multi queue support */
+		for (i = 0; i < 2; i++)
+			free_irq(alx->msix_entries[i].vector, alx);
+	} else {
+		free_irq(pdev->irq, alx);
 	}
+
+	alx_disable_advanced_intr(alx);
 }
 
 static int alx_identify_hw(struct alx_priv *alx)
@@ -847,12 +1021,14 @@ static int __alx_open(struct alx_priv *alx, bool resume)
 {
 	int err;
 
+	alx_init_intr(alx, msix);
+
 	if (!resume)
 		netif_carrier_off(alx->dev);
 
 	err = alx_alloc_rings(alx);
 	if (err)
-		return err;
+		goto out_disable_adv_intr;
 
 	alx_configure(alx);
 
@@ -873,6 +1049,8 @@ static int __alx_open(struct alx_priv *alx, bool resume)
 
 out_free_rings:
 	alx_free_rings(alx);
+out_disable_adv_intr:
+	alx_disable_advanced_intr(alx);
 	return err;
 }
 
@@ -993,6 +1171,18 @@ static void alx_reset(struct work_struct *work)
 	rtnl_unlock();
 }
 
+static int alx_tpd_req(struct sk_buff *skb)
+{
+	int num;
+
+	num = skb_shinfo(skb)->nr_frags + 1;
+	/* we need one extra descriptor for LSOv2 */
+	if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
+		num++;
+
+	return num;
+}
+
 static int alx_tx_csum(struct sk_buff *skb, struct alx_txd *first)
 {
 	u8 cso, css;
@@ -1012,6 +1202,45 @@ static int alx_tx_csum(struct sk_buff *skb, struct alx_txd *first)
 	return 0;
 }
 
+static int alx_tso(struct sk_buff *skb, struct alx_txd *first)
+{
+	int err;
+
+	if (skb->ip_summed != CHECKSUM_PARTIAL)
+		return 0;
+
+	if (!skb_is_gso(skb))
+		return 0;
+
+	err = skb_cow_head(skb, 0);
+	if (err < 0)
+		return err;
+
+	if (skb->protocol == htons(ETH_P_IP)) {
+		struct iphdr *iph = ip_hdr(skb);
+
+		iph->check = 0;
+		tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
+							 0, IPPROTO_TCP, 0);
+		first->word1 |= 1 << TPD_IPV4_SHIFT;
+	} else if (skb_is_gso_v6(skb)) {
+		ipv6_hdr(skb)->payload_len = 0;
+		tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
+						       &ipv6_hdr(skb)->daddr,
+						       0, IPPROTO_TCP, 0);
+		/* LSOv2: the first TPD only provides the packet length */
+		first->adrl.l.pkt_len = skb->len;
+		first->word1 |= 1 << TPD_LSO_V2_SHIFT;
+	}
+
+	first->word1 |= 1 << TPD_LSO_EN_SHIFT;
+	first->word1 |= (skb_transport_offset(skb) &
+			 TPD_L4HDROFFSET_MASK) << TPD_L4HDROFFSET_SHIFT;
+	first->word1 |= (skb_shinfo(skb)->gso_size &
+			 TPD_MSS_MASK) << TPD_MSS_SHIFT;
+	return 1;
+}
+
 static int alx_map_tx_skb(struct alx_priv *alx, struct sk_buff *skb)
 {
 	struct alx_tx_queue *txq = &alx->txq;
@@ -1022,6 +1251,16 @@ static int alx_map_tx_skb(struct alx_priv *alx, struct sk_buff *skb)
 	first_tpd = &txq->tpd[txq->write_idx];
 	tpd = first_tpd;
 
+	if (tpd->word1 & (1 << TPD_LSO_V2_SHIFT)) {
+		if (++txq->write_idx == alx->tx_ringsz)
+			txq->write_idx = 0;
+
+		tpd = &txq->tpd[txq->write_idx];
+		tpd->len = first_tpd->len;
+		tpd->vlan_tag = first_tpd->vlan_tag;
+		tpd->word1 = first_tpd->word1;
+	}
+
 	maplen = skb_headlen(skb);
 	dma = dma_map_single(&alx->hw.pdev->dev, skb->data, maplen,
 			     DMA_TO_DEVICE);
@@ -1082,9 +1321,9 @@ static netdev_tx_t alx_start_xmit(struct sk_buff *skb,
 	struct alx_priv *alx = netdev_priv(netdev);
 	struct alx_tx_queue *txq = &alx->txq;
 	struct alx_txd *first;
-	int tpdreq = skb_shinfo(skb)->nr_frags + 1;
+	int tso;
 
-	if (alx_tpd_avail(alx) < tpdreq) {
+	if (alx_tpd_avail(alx) < alx_tpd_req(skb)) {
 		netif_stop_queue(alx->dev);
 		goto drop;
 	}
@@ -1092,7 +1331,10 @@ static netdev_tx_t alx_start_xmit(struct sk_buff *skb,
 	first = &txq->tpd[txq->write_idx];
 	memset(first, 0, sizeof(*first));
 
-	if (alx_tx_csum(skb, first))
+	tso = alx_tso(skb, first);
+	if (tso < 0)
+		goto drop;
+	else if (!tso && alx_tx_csum(skb, first))
 		goto drop;
 
 	if (alx_map_tx_skb(alx, skb) < 0)
@@ -1172,7 +1414,10 @@ static void alx_poll_controller(struct net_device *netdev)
 {
 	struct alx_priv *alx = netdev_priv(netdev);
 
-	if (alx->msi)
+	if (alx->flags & ALX_FLAG_USING_MSIX) {
+		alx_intr_msix_misc(0, alx);
+		alx_intr_msix_ring(0, alx);
+	} else if (alx->flags & ALX_FLAG_USING_MSI)
 		alx_intr_msi(0, alx);
 	else
 		alx_intr_legacy(0, alx);
@@ -1351,7 +1596,10 @@ static int alx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		}
 	}
 
-	netdev->hw_features = NETIF_F_SG | NETIF_F_HW_CSUM;
+	netdev->hw_features = NETIF_F_SG |
+			      NETIF_F_HW_CSUM |
+			      NETIF_F_TSO |
+			      NETIF_F_TSO6;
 
 	if (alx_get_perm_macaddr(hw, hw->perm_addr)) {
 		dev_warn(&pdev->dev,
@@ -1545,6 +1793,8 @@ static const struct pci_device_id alx_pci_tbl[] = {
 	  .driver_data = ALX_DEV_QUIRK_MSI_INTX_DISABLE_BUG },
 	{ PCI_VDEVICE(ATTANSIC, ALX_DEV_ID_E2400),
 	  .driver_data = ALX_DEV_QUIRK_MSI_INTX_DISABLE_BUG },
+	{ PCI_VDEVICE(ATTANSIC, ALX_DEV_ID_E2500),
+	  .driver_data = ALX_DEV_QUIRK_MSI_INTX_DISABLE_BUG },
 	{ PCI_VDEVICE(ATTANSIC, ALX_DEV_ID_AR8162),
 	  .driver_data = ALX_DEV_QUIRK_MSI_INTX_DISABLE_BUG },
 	{ PCI_VDEVICE(ATTANSIC, ALX_DEV_ID_AR8171) },
diff --git a/drivers/net/ethernet/atheros/alx/reg.h b/drivers/net/ethernet/atheros/alx/reg.h
index 0959e6824cb6..1fc2d852249f 100644
--- a/drivers/net/ethernet/atheros/alx/reg.h
+++ b/drivers/net/ethernet/atheros/alx/reg.h
@@ -38,6 +38,7 @@
 #define ALX_DEV_ID_AR8161				0x1091
 #define ALX_DEV_ID_E2200				0xe091
 #define ALX_DEV_ID_E2400				0xe0a1
+#define ALX_DEV_ID_E2500				0xe0b1
 #define ALX_DEV_ID_AR8162				0x1090
 #define ALX_DEV_ID_AR8171				0x10A1
 #define ALX_DEV_ID_AR8172				0x10A0
diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c
index 74f0a37c4eb6..17aa33c5567d 100644
--- a/drivers/net/ethernet/broadcom/b44.c
+++ b/drivers/net/ethernet/broadcom/b44.c
@@ -1486,7 +1486,7 @@ static int b44_open(struct net_device *dev)
 	b44_enable_ints(bp);
 
 	if (bp->flags & B44_FLAG_EXTERNAL_PHY)
-		phy_start(bp->phydev);
+		phy_start(dev->phydev);
 
 	netif_start_queue(dev);
 out:
@@ -1651,7 +1651,7 @@ static int b44_close(struct net_device *dev)
 	netif_stop_queue(dev);
 
 	if (bp->flags & B44_FLAG_EXTERNAL_PHY)
-		phy_stop(bp->phydev);
+		phy_stop(dev->phydev);
 
 	napi_disable(&bp->napi);
 
@@ -1832,90 +1832,100 @@ static int b44_nway_reset(struct net_device *dev)
 	return r;
 }
 
-static int b44_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int b44_get_link_ksettings(struct net_device *dev,
+				  struct ethtool_link_ksettings *cmd)
 {
 	struct b44 *bp = netdev_priv(dev);
+	u32 supported, advertising;
 
 	if (bp->flags & B44_FLAG_EXTERNAL_PHY) {
-		BUG_ON(!bp->phydev);
-		return phy_ethtool_gset(bp->phydev, cmd);
+		BUG_ON(!dev->phydev);
+		return phy_ethtool_ksettings_get(dev->phydev, cmd);
 	}
 
-	cmd->supported = (SUPPORTED_Autoneg);
-	cmd->supported |= (SUPPORTED_100baseT_Half |
-			  SUPPORTED_100baseT_Full |
-			  SUPPORTED_10baseT_Half |
-			  SUPPORTED_10baseT_Full |
-			  SUPPORTED_MII);
+	supported = (SUPPORTED_Autoneg);
+	supported |= (SUPPORTED_100baseT_Half |
+		      SUPPORTED_100baseT_Full |
+		      SUPPORTED_10baseT_Half |
+		      SUPPORTED_10baseT_Full |
+		      SUPPORTED_MII);
 
-	cmd->advertising = 0;
+	advertising = 0;
 	if (bp->flags & B44_FLAG_ADV_10HALF)
-		cmd->advertising |= ADVERTISED_10baseT_Half;
+		advertising |= ADVERTISED_10baseT_Half;
 	if (bp->flags & B44_FLAG_ADV_10FULL)
-		cmd->advertising |= ADVERTISED_10baseT_Full;
+		advertising |= ADVERTISED_10baseT_Full;
 	if (bp->flags & B44_FLAG_ADV_100HALF)
-		cmd->advertising |= ADVERTISED_100baseT_Half;
+		advertising |= ADVERTISED_100baseT_Half;
 	if (bp->flags & B44_FLAG_ADV_100FULL)
-		cmd->advertising |= ADVERTISED_100baseT_Full;
-	cmd->advertising |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
-	ethtool_cmd_speed_set(cmd, ((bp->flags & B44_FLAG_100_BASE_T) ?
-				    SPEED_100 : SPEED_10));
-	cmd->duplex = (bp->flags & B44_FLAG_FULL_DUPLEX) ?
+		advertising |= ADVERTISED_100baseT_Full;
+	advertising |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
+	cmd->base.speed = (bp->flags & B44_FLAG_100_BASE_T) ?
+		SPEED_100 : SPEED_10;
+	cmd->base.duplex = (bp->flags & B44_FLAG_FULL_DUPLEX) ?
 		DUPLEX_FULL : DUPLEX_HALF;
-	cmd->port = 0;
-	cmd->phy_address = bp->phy_addr;
-	cmd->transceiver = (bp->flags & B44_FLAG_EXTERNAL_PHY) ?
-		XCVR_EXTERNAL : XCVR_INTERNAL;
-	cmd->autoneg = (bp->flags & B44_FLAG_FORCE_LINK) ?
+	cmd->base.port = 0;
+	cmd->base.phy_address = bp->phy_addr;
+	cmd->base.autoneg = (bp->flags & B44_FLAG_FORCE_LINK) ?
 		AUTONEG_DISABLE : AUTONEG_ENABLE;
-	if (cmd->autoneg == AUTONEG_ENABLE)
-		cmd->advertising |= ADVERTISED_Autoneg;
+	if (cmd->base.autoneg == AUTONEG_ENABLE)
+		advertising |= ADVERTISED_Autoneg;
+
+	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
+						supported);
+	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
+						advertising);
+
 	if (!netif_running(dev)){
-		ethtool_cmd_speed_set(cmd, 0);
-		cmd->duplex = 0xff;
+		cmd->base.speed = 0;
+		cmd->base.duplex = 0xff;
 	}
-	cmd->maxtxpkt = 0;
-	cmd->maxrxpkt = 0;
+
 	return 0;
 }
 
-static int b44_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int b44_set_link_ksettings(struct net_device *dev,
+				  const struct ethtool_link_ksettings *cmd)
 {
 	struct b44 *bp = netdev_priv(dev);
 	u32 speed;
 	int ret;
+	u32 advertising;
 
 	if (bp->flags & B44_FLAG_EXTERNAL_PHY) {
-		BUG_ON(!bp->phydev);
+		BUG_ON(!dev->phydev);
 		spin_lock_irq(&bp->lock);
 		if (netif_running(dev))
 			b44_setup_phy(bp);
 
-		ret = phy_ethtool_sset(bp->phydev, cmd);
+		ret = phy_ethtool_ksettings_set(dev->phydev, cmd);
 
 		spin_unlock_irq(&bp->lock);
 
 		return ret;
 	}
 
-	speed = ethtool_cmd_speed(cmd);
+	speed = cmd->base.speed;
+
+	ethtool_convert_link_mode_to_legacy_u32(&advertising,
+						cmd->link_modes.advertising);
 
 	/* We do not support gigabit. */
-	if (cmd->autoneg == AUTONEG_ENABLE) {
-		if (cmd->advertising &
+	if (cmd->base.autoneg == AUTONEG_ENABLE) {
+		if (advertising &
 		    (ADVERTISED_1000baseT_Half |
 		     ADVERTISED_1000baseT_Full))
 			return -EINVAL;
 	} else if ((speed != SPEED_100 &&
 		    speed != SPEED_10) ||
-		   (cmd->duplex != DUPLEX_HALF &&
-		    cmd->duplex != DUPLEX_FULL)) {
+		   (cmd->base.duplex != DUPLEX_HALF &&
+		    cmd->base.duplex != DUPLEX_FULL)) {
 			return -EINVAL;
 	}
 
 	spin_lock_irq(&bp->lock);
 
-	if (cmd->autoneg == AUTONEG_ENABLE) {
+	if (cmd->base.autoneg == AUTONEG_ENABLE) {
 		bp->flags &= ~(B44_FLAG_FORCE_LINK |
 			       B44_FLAG_100_BASE_T |
 			       B44_FLAG_FULL_DUPLEX |
@@ -1923,19 +1933,19 @@ static int b44_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 			       B44_FLAG_ADV_10FULL |
 			       B44_FLAG_ADV_100HALF |
 			       B44_FLAG_ADV_100FULL);
-		if (cmd->advertising == 0) {
+		if (advertising == 0) {
 			bp->flags |= (B44_FLAG_ADV_10HALF |
 				      B44_FLAG_ADV_10FULL |
 				      B44_FLAG_ADV_100HALF |
 				      B44_FLAG_ADV_100FULL);
 		} else {
-			if (cmd->advertising & ADVERTISED_10baseT_Half)
+			if (advertising & ADVERTISED_10baseT_Half)
 				bp->flags |= B44_FLAG_ADV_10HALF;
-			if (cmd->advertising & ADVERTISED_10baseT_Full)
+			if (advertising & ADVERTISED_10baseT_Full)
 				bp->flags |= B44_FLAG_ADV_10FULL;
-			if (cmd->advertising & ADVERTISED_100baseT_Half)
+			if (advertising & ADVERTISED_100baseT_Half)
 				bp->flags |= B44_FLAG_ADV_100HALF;
-			if (cmd->advertising & ADVERTISED_100baseT_Full)
+			if (advertising & ADVERTISED_100baseT_Full)
 				bp->flags |= B44_FLAG_ADV_100FULL;
 		}
 	} else {
@@ -1943,7 +1953,7 @@ static int b44_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 		bp->flags &= ~(B44_FLAG_100_BASE_T | B44_FLAG_FULL_DUPLEX);
 		if (speed == SPEED_100)
 			bp->flags |= B44_FLAG_100_BASE_T;
-		if (cmd->duplex == DUPLEX_FULL)
+		if (cmd->base.duplex == DUPLEX_FULL)
 			bp->flags |= B44_FLAG_FULL_DUPLEX;
 	}
 
@@ -2110,8 +2120,6 @@ static int b44_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
 
 static const struct ethtool_ops b44_ethtool_ops = {
 	.get_drvinfo		= b44_get_drvinfo,
-	.get_settings		= b44_get_settings,
-	.set_settings		= b44_set_settings,
 	.nway_reset		= b44_nway_reset,
 	.get_link		= ethtool_op_get_link,
 	.get_wol		= b44_get_wol,
@@ -2125,6 +2133,8 @@ static const struct ethtool_ops b44_ethtool_ops = {
 	.get_strings		= b44_get_strings,
 	.get_sset_count		= b44_get_sset_count,
 	.get_ethtool_stats	= b44_get_ethtool_stats,
+	.get_link_ksettings	= b44_get_link_ksettings,
+	.set_link_ksettings	= b44_set_link_ksettings,
 };
 
 static int b44_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
@@ -2137,8 +2147,8 @@ static int b44_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 
 	spin_lock_irq(&bp->lock);
 	if (bp->flags & B44_FLAG_EXTERNAL_PHY) {
-		BUG_ON(!bp->phydev);
-		err = phy_mii_ioctl(bp->phydev, ifr, cmd);
+		BUG_ON(!dev->phydev);
+		err = phy_mii_ioctl(dev->phydev, ifr, cmd);
 	} else {
 		err = generic_mii_ioctl(&bp->mii_if, if_mii(ifr), cmd, NULL);
 	}
@@ -2206,7 +2216,7 @@ static const struct net_device_ops b44_netdev_ops = {
 static void b44_adjust_link(struct net_device *dev)
 {
 	struct b44 *bp = netdev_priv(dev);
-	struct phy_device *phydev = bp->phydev;
+	struct phy_device *phydev = dev->phydev;
 	bool status_changed = 0;
 
 	BUG_ON(!phydev);
@@ -2303,7 +2313,6 @@ static int b44_register_phy_one(struct b44 *bp)
 			      SUPPORTED_MII);
 	phydev->advertising = phydev->supported;
 
-	bp->phydev = phydev;
 	bp->old_link = 0;
 	bp->phy_addr = phydev->mdio.addr;
 
@@ -2323,9 +2332,10 @@ err_out:
 
 static void b44_unregister_phy_one(struct b44 *bp)
 {
+	struct net_device *dev = bp->dev;
 	struct mii_bus *mii_bus = bp->mii_bus;
 
-	phy_disconnect(bp->phydev);
+	phy_disconnect(dev->phydev);
 	mdiobus_unregister(mii_bus);
 	mdiobus_free(mii_bus);
 }
diff --git a/drivers/net/ethernet/broadcom/b44.h b/drivers/net/ethernet/broadcom/b44.h
index 65d88d7c5581..89d2cf341163 100644
--- a/drivers/net/ethernet/broadcom/b44.h
+++ b/drivers/net/ethernet/broadcom/b44.h
@@ -404,7 +404,6 @@ struct b44 {
 	u32			tx_pending;
 	u8			phy_addr;
 	u8			force_copybreak;
-	struct phy_device	*phydev;
 	struct mii_bus		*mii_bus;
 	int			old_link;
 	struct mii_if_info	mii_if;
diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
index 6c8bc5fadac7..ae364c74baf3 100644
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
@@ -791,7 +791,7 @@ static void bcm_enet_adjust_phy_link(struct net_device *dev)
 	int status_changed;
 
 	priv = netdev_priv(dev);
-	phydev = priv->phydev;
+	phydev = dev->phydev;
 	status_changed = 0;
 
 	if (priv->old_link != phydev->link) {
@@ -913,7 +913,6 @@ static int bcm_enet_open(struct net_device *dev)
 		priv->old_link = 0;
 		priv->old_duplex = -1;
 		priv->old_pause = -1;
-		priv->phydev = phydev;
 	}
 
 	/* mask all interrupts and request them */
@@ -1085,7 +1084,7 @@ static int bcm_enet_open(struct net_device *dev)
 			 ENETDMAC_IRMASK, priv->tx_chan);
 
 	if (priv->has_phy)
-		phy_start(priv->phydev);
+		phy_start(phydev);
 	else
 		bcm_enet_adjust_link(dev);
 
@@ -1127,7 +1126,7 @@ out_freeirq:
 	free_irq(dev->irq, dev);
 
 out_phy_disconnect:
-	phy_disconnect(priv->phydev);
+	phy_disconnect(phydev);
 
 	return ret;
 }
@@ -1190,7 +1189,7 @@ static int bcm_enet_stop(struct net_device *dev)
 	netif_stop_queue(dev);
 	napi_disable(&priv->napi);
 	if (priv->has_phy)
-		phy_stop(priv->phydev);
+		phy_stop(dev->phydev);
 	del_timer_sync(&priv->rx_timeout);
 
 	/* mask all interrupts */
@@ -1234,10 +1233,8 @@ static int bcm_enet_stop(struct net_device *dev)
 	free_irq(dev->irq, dev);
 
 	/* release phy */
-	if (priv->has_phy) {
-		phy_disconnect(priv->phydev);
-		priv->phydev = NULL;
-	}
+	if (priv->has_phy)
+		phy_disconnect(dev->phydev);
 
 	return 0;
 }
@@ -1437,64 +1434,68 @@ static int bcm_enet_nway_reset(struct net_device *dev)
 
 	priv = netdev_priv(dev);
 	if (priv->has_phy) {
-		if (!priv->phydev)
+		if (!dev->phydev)
 			return -ENODEV;
-		return genphy_restart_aneg(priv->phydev);
+		return genphy_restart_aneg(dev->phydev);
 	}
 
 	return -EOPNOTSUPP;
 }
 
-static int bcm_enet_get_settings(struct net_device *dev,
-				 struct ethtool_cmd *cmd)
+static int bcm_enet_get_link_ksettings(struct net_device *dev,
+				       struct ethtool_link_ksettings *cmd)
 {
 	struct bcm_enet_priv *priv;
+	u32 supported, advertising;
 
 	priv = netdev_priv(dev);
 
-	cmd->maxrxpkt = 0;
-	cmd->maxtxpkt = 0;
-
 	if (priv->has_phy) {
-		if (!priv->phydev)
+		if (!dev->phydev)
 			return -ENODEV;
-		return phy_ethtool_gset(priv->phydev, cmd);
+		return phy_ethtool_ksettings_get(dev->phydev, cmd);
 	} else {
-		cmd->autoneg = 0;
-		ethtool_cmd_speed_set(cmd, ((priv->force_speed_100)
-					    ? SPEED_100 : SPEED_10));
-		cmd->duplex = (priv->force_duplex_full) ?
+		cmd->base.autoneg = 0;
+		cmd->base.speed = (priv->force_speed_100) ?
+			SPEED_100 : SPEED_10;
+		cmd->base.duplex = (priv->force_duplex_full) ?
 			DUPLEX_FULL : DUPLEX_HALF;
-		cmd->supported = ADVERTISED_10baseT_Half  |
+		supported = ADVERTISED_10baseT_Half |
 			ADVERTISED_10baseT_Full |
 			ADVERTISED_100baseT_Half |
 			ADVERTISED_100baseT_Full;
-		cmd->advertising = 0;
-		cmd->port = PORT_MII;
-		cmd->transceiver = XCVR_EXTERNAL;
+		advertising = 0;
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.supported, supported);
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.advertising, advertising);
+		cmd->base.port = PORT_MII;
 	}
 	return 0;
 }
 
-static int bcm_enet_set_settings(struct net_device *dev,
-				 struct ethtool_cmd *cmd)
+static int bcm_enet_set_link_ksettings(struct net_device *dev,
+				       const struct ethtool_link_ksettings *cmd)
 {
 	struct bcm_enet_priv *priv;
 
 	priv = netdev_priv(dev);
 	if (priv->has_phy) {
-		if (!priv->phydev)
+		if (!dev->phydev)
 			return -ENODEV;
-		return phy_ethtool_sset(priv->phydev, cmd);
+		return phy_ethtool_ksettings_set(dev->phydev, cmd);
 	} else {
 
-		if (cmd->autoneg ||
-		    (cmd->speed != SPEED_100 && cmd->speed != SPEED_10) ||
-		    cmd->port != PORT_MII)
+		if (cmd->base.autoneg ||
+		    (cmd->base.speed != SPEED_100 &&
+		     cmd->base.speed != SPEED_10) ||
+		    cmd->base.port != PORT_MII)
 			return -EINVAL;
 
-		priv->force_speed_100 = (cmd->speed == SPEED_100) ? 1 : 0;
-		priv->force_duplex_full = (cmd->duplex == DUPLEX_FULL) ? 1 : 0;
+		priv->force_speed_100 =
+			(cmd->base.speed == SPEED_100) ? 1 : 0;
+		priv->force_duplex_full =
+			(cmd->base.duplex == DUPLEX_FULL) ? 1 : 0;
 
 		if (netif_running(dev))
 			bcm_enet_adjust_link(dev);
@@ -1588,14 +1589,14 @@ static const struct ethtool_ops bcm_enet_ethtool_ops = {
 	.get_sset_count		= bcm_enet_get_sset_count,
 	.get_ethtool_stats      = bcm_enet_get_ethtool_stats,
 	.nway_reset		= bcm_enet_nway_reset,
-	.get_settings		= bcm_enet_get_settings,
-	.set_settings		= bcm_enet_set_settings,
 	.get_drvinfo		= bcm_enet_get_drvinfo,
 	.get_link		= ethtool_op_get_link,
 	.get_ringparam		= bcm_enet_get_ringparam,
 	.set_ringparam		= bcm_enet_set_ringparam,
 	.get_pauseparam		= bcm_enet_get_pauseparam,
 	.set_pauseparam		= bcm_enet_set_pauseparam,
+	.get_link_ksettings	= bcm_enet_get_link_ksettings,
+	.set_link_ksettings	= bcm_enet_set_link_ksettings,
 };
 
 static int bcm_enet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
@@ -1604,9 +1605,9 @@ static int bcm_enet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 
 	priv = netdev_priv(dev);
 	if (priv->has_phy) {
-		if (!priv->phydev)
+		if (!dev->phydev)
 			return -ENODEV;
-		return phy_mii_ioctl(priv->phydev, rq, cmd);
+		return phy_mii_ioctl(dev->phydev, rq, cmd);
 	} else {
 		struct mii_if_info mii;
 
diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.h b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
index f55af4310085..0a1b7b2e55bd 100644
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.h
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
@@ -290,7 +290,6 @@ struct bcm_enet_priv {
 
 	/* used when a phy is connected (phylib used) */
 	struct mii_bus *mii_bus;
-	struct phy_device *phydev;
 	int old_link;
 	int old_duplex;
 	int old_pause;
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
index b2d30863caeb..c3354b9941d1 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.c
+++ b/drivers/net/ethernet/broadcom/bcmsysport.c
@@ -58,8 +58,8 @@ BCM_SYSPORT_IO_MACRO(topctrl, SYS_PORT_TOPCTRL_OFFSET);
 static inline void intrl2_##which##_mask_clear(struct bcm_sysport_priv *priv, \
 						u32 mask)		\
 {									\
-	intrl2_##which##_writel(priv, mask, INTRL2_CPU_MASK_CLEAR);	\
 	priv->irq##which##_mask &= ~(mask);				\
+	intrl2_##which##_writel(priv, mask, INTRL2_CPU_MASK_CLEAR);	\
 }									\
 static inline void intrl2_##which##_mask_set(struct bcm_sysport_priv *priv, \
 						u32 mask)		\
@@ -1692,7 +1692,7 @@ static int bcm_sysport_stop(struct net_device *dev)
 	return 0;
 }
 
-static struct ethtool_ops bcm_sysport_ethtool_ops = {
+static const struct ethtool_ops bcm_sysport_ethtool_ops = {
 	.get_drvinfo		= bcm_sysport_get_drvinfo,
 	.get_msglevel		= bcm_sysport_get_msglvl,
 	.set_msglevel		= bcm_sysport_set_msglvl,
diff --git a/drivers/net/ethernet/broadcom/bgmac-bcma.c b/drivers/net/ethernet/broadcom/bgmac-bcma.c
index 9a9745c4047c..c16ec3a51876 100644
--- a/drivers/net/ethernet/broadcom/bgmac-bcma.c
+++ b/drivers/net/ethernet/broadcom/bgmac-bcma.c
@@ -92,6 +92,7 @@ MODULE_DEVICE_TABLE(bcma, bgmac_bcma_tbl);
 /* http://bcm-v4.sipsolutions.net/mac-gbit/gmac/chipattach */
 static int bgmac_probe(struct bcma_device *core)
 {
+	struct bcma_chipinfo *ci = &core->bus->chipinfo;
 	struct ssb_sprom *sprom = &core->bus->sprom;
 	struct mii_bus *mii_bus;
 	struct bgmac *bgmac;
@@ -157,9 +158,10 @@ static int bgmac_probe(struct bcma_device *core)
 	dev_info(bgmac->dev, "Found PHY addr: %d%s\n", bgmac->phyaddr,
 		 bgmac->phyaddr == BGMAC_PHY_NOREGS ? " (NOREGS)" : "");
 
-	if (!bgmac_is_bcm4707_family(core)) {
+	if (!bgmac_is_bcm4707_family(core) &&
+	    !(ci->id == BCMA_CHIP_ID_BCM53573 && core->core_unit == 1)) {
 		mii_bus = bcma_mdio_mii_register(core, bgmac->phyaddr);
-		if (!IS_ERR(mii_bus)) {
+		if (IS_ERR(mii_bus)) {
 			err = PTR_ERR(mii_bus);
 			goto err;
 		}
@@ -230,6 +232,21 @@ static int bgmac_probe(struct bcma_device *core)
 		bgmac->feature_flags |= BGMAC_FEAT_NO_RESET;
 		bgmac->feature_flags |= BGMAC_FEAT_FORCE_SPEED_2500;
 		break;
+	case BCMA_CHIP_ID_BCM53573:
+		bgmac->feature_flags |= BGMAC_FEAT_CLKCTLST;
+		bgmac->feature_flags |= BGMAC_FEAT_SET_RXQ_CLK;
+		if (ci->pkg == BCMA_PKG_ID_BCM47189)
+			bgmac->feature_flags |= BGMAC_FEAT_IOST_ATTACHED;
+		if (core->core_unit == 0) {
+			bgmac->feature_flags |= BGMAC_FEAT_CC4_IF_SW_TYPE;
+			if (ci->pkg == BCMA_PKG_ID_BCM47189)
+				bgmac->feature_flags |=
+					BGMAC_FEAT_CC4_IF_SW_TYPE_RGMII;
+		} else if (core->core_unit == 1) {
+			bgmac->feature_flags |= BGMAC_FEAT_IRQ_ID_OOB_6;
+			bgmac->feature_flags |= BGMAC_FEAT_CC7_IF_TYPE_RGMII;
+		}
+		break;
 	default:
 		bgmac->feature_flags |= BGMAC_FEAT_CLKCTLST;
 		bgmac->feature_flags |= BGMAC_FEAT_SET_RXQ_CLK;
diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
index c4751ece76f6..856379cbb402 100644
--- a/drivers/net/ethernet/broadcom/bgmac.c
+++ b/drivers/net/ethernet/broadcom/bgmac.c
@@ -932,7 +932,8 @@ static void bgmac_chip_reset(struct bgmac *bgmac)
 			et_swtype <<= 4;
 			sw_type = et_swtype;
 		} else if (bgmac->feature_flags & BGMAC_FEAT_SW_TYPE_EPHYRMII) {
-			sw_type = BGMAC_CHIPCTL_1_SW_TYPE_EPHYRMII;
+			sw_type = BGMAC_CHIPCTL_1_IF_TYPE_RMII |
+				  BGMAC_CHIPCTL_1_SW_TYPE_EPHYRMII;
 		} else if (bgmac->feature_flags & BGMAC_FEAT_SW_TYPE_RGMII) {
 			sw_type = BGMAC_CHIPCTL_1_IF_TYPE_RGMII |
 				  BGMAC_CHIPCTL_1_SW_TYPE_RGMII;
@@ -940,6 +941,27 @@ static void bgmac_chip_reset(struct bgmac *bgmac)
 		bgmac_cco_ctl_maskset(bgmac, 1, ~(BGMAC_CHIPCTL_1_IF_TYPE_MASK |
 						  BGMAC_CHIPCTL_1_SW_TYPE_MASK),
 				      sw_type);
+	} else if (bgmac->feature_flags & BGMAC_FEAT_CC4_IF_SW_TYPE) {
+		u32 sw_type = BGMAC_CHIPCTL_4_IF_TYPE_MII |
+			      BGMAC_CHIPCTL_4_SW_TYPE_EPHY;
+		u8 et_swtype = 0;
+		char buf[4];
+
+		if (bcm47xx_nvram_getenv("et_swtype", buf, sizeof(buf)) > 0) {
+			if (kstrtou8(buf, 0, &et_swtype))
+				dev_err(bgmac->dev, "Failed to parse et_swtype (%s)\n",
+					buf);
+			sw_type = (et_swtype & 0x0f) << 12;
+		} else if (bgmac->feature_flags & BGMAC_FEAT_CC4_IF_SW_TYPE_RGMII) {
+			sw_type = BGMAC_CHIPCTL_4_IF_TYPE_RGMII |
+				  BGMAC_CHIPCTL_4_SW_TYPE_RGMII;
+		}
+		bgmac_cco_ctl_maskset(bgmac, 4, ~(BGMAC_CHIPCTL_4_IF_TYPE_MASK |
+						  BGMAC_CHIPCTL_4_SW_TYPE_MASK),
+				      sw_type);
+	} else if (bgmac->feature_flags & BGMAC_FEAT_CC7_IF_TYPE_RGMII) {
+		bgmac_cco_ctl_maskset(bgmac, 7, ~BGMAC_CHIPCTL_7_IF_TYPE_MASK,
+				      BGMAC_CHIPCTL_7_IF_TYPE_RGMII);
 	}
 
 	if (iost & BGMAC_BCMA_IOST_ATTACHED && !bgmac->has_robosw)
@@ -1024,7 +1046,7 @@ static void bgmac_enable(struct bgmac *bgmac)
 
 	mode = (bgmac_read(bgmac, BGMAC_DEV_STATUS) & BGMAC_DS_MM_MASK) >>
 		BGMAC_DS_MM_SHIFT;
-	if (bgmac->feature_flags & BGMAC_FEAT_CLKCTLST || mode != 0)
+	if (!(bgmac->feature_flags & BGMAC_FEAT_CLKCTLST) || mode != 0)
 		bgmac_set(bgmac, BCMA_CLKCTLST, BCMA_CLKCTLST_FORCEHT);
 	if (bgmac->feature_flags & BGMAC_FEAT_CLKCTLST && mode == 2)
 		bgmac_cco_ctl_maskset(bgmac, 1, ~0,
@@ -1467,6 +1489,10 @@ int bgmac_enet_probe(struct bgmac *info)
 	 */
 	bgmac_clk_enable(bgmac, 0);
 
+	/* This seems to be fixing IRQ by assigning OOB #6 to the core */
+	if (bgmac->feature_flags & BGMAC_FEAT_IRQ_ID_OOB_6)
+		bgmac_idm_write(bgmac, BCMA_OOB_SEL_OUT_A30, 0x86);
+
 	bgmac_chip_reset(bgmac);
 
 	err = bgmac_dma_alloc(bgmac);
diff --git a/drivers/net/ethernet/broadcom/bgmac.h b/drivers/net/ethernet/broadcom/bgmac.h
index 24a250267b88..80836b4c9f38 100644
--- a/drivers/net/ethernet/broadcom/bgmac.h
+++ b/drivers/net/ethernet/broadcom/bgmac.h
@@ -369,6 +369,21 @@
 #define BGMAC_CHIPCTL_1_SW_TYPE_RGMII		0x000000C0
 #define BGMAC_CHIPCTL_1_RXC_DLL_BYPASS		0x00010000
 
+#define BGMAC_CHIPCTL_4_IF_TYPE_MASK		0x00003000
+#define BGMAC_CHIPCTL_4_IF_TYPE_RMII		0x00000000
+#define BGMAC_CHIPCTL_4_IF_TYPE_MII		0x00001000
+#define BGMAC_CHIPCTL_4_IF_TYPE_RGMII		0x00002000
+#define BGMAC_CHIPCTL_4_SW_TYPE_MASK		0x0000C000
+#define BGMAC_CHIPCTL_4_SW_TYPE_EPHY		0x00000000
+#define BGMAC_CHIPCTL_4_SW_TYPE_EPHYMII		0x00004000
+#define BGMAC_CHIPCTL_4_SW_TYPE_EPHYRMII	0x00008000
+#define BGMAC_CHIPCTL_4_SW_TYPE_RGMII		0x0000C000
+
+#define BGMAC_CHIPCTL_7_IF_TYPE_MASK		0x000000C0
+#define BGMAC_CHIPCTL_7_IF_TYPE_RMII		0x00000000
+#define BGMAC_CHIPCTL_7_IF_TYPE_MII		0x00000040
+#define BGMAC_CHIPCTL_7_IF_TYPE_RGMII		0x00000080
+
 #define BGMAC_WEIGHT	64
 
 #define ETHER_MAX_LEN   1518
@@ -390,6 +405,10 @@
 #define BGMAC_FEAT_NO_CLR_MIB		BIT(13)
 #define BGMAC_FEAT_FORCE_SPEED_2500	BIT(14)
 #define BGMAC_FEAT_CMDCFG_SR_REV4	BIT(15)
+#define BGMAC_FEAT_IRQ_ID_OOB_6		BIT(16)
+#define BGMAC_FEAT_CC4_IF_SW_TYPE	BIT(17)
+#define BGMAC_FEAT_CC4_IF_SW_TYPE_RGMII	BIT(18)
+#define BGMAC_FEAT_CC7_IF_TYPE_RGMII	BIT(19)
 
 struct bgmac_slot_info {
 	union {
diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
index 8fc3f3c137f8..27f11a5d5fe2 100644
--- a/drivers/net/ethernet/broadcom/bnx2.c
+++ b/drivers/net/ethernet/broadcom/bnx2.c
@@ -50,7 +50,7 @@
 #include <linux/log2.h>
 #include <linux/aer.h>
 
-#if defined(CONFIG_CNIC) || defined(CONFIG_CNIC_MODULE)
+#if IS_ENABLED(CONFIG_CNIC)
 #define BCM_CNIC 1
 #include "cnic_if.h"
 #endif
@@ -6356,10 +6356,6 @@ bnx2_open(struct net_device *dev)
 	struct bnx2 *bp = netdev_priv(dev);
 	int rc;
 
-	rc = bnx2_request_firmware(bp);
-	if (rc < 0)
-		goto out;
-
 	netif_carrier_off(dev);
 
 	bnx2_disable_int(bp);
@@ -6428,7 +6424,6 @@ open_err:
 	bnx2_free_irq(bp);
 	bnx2_free_mem(bp);
 	bnx2_del_napi(bp);
-	bnx2_release_firmware(bp);
 	goto out;
 }
 
@@ -8575,6 +8570,12 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_drvdata(pdev, dev);
 
+	rc = bnx2_request_firmware(bp);
+	if (rc < 0)
+		goto error;
+
+
+	bnx2_reset_chip(bp, BNX2_DRV_MSG_CODE_RESET);
 	memcpy(dev->dev_addr, bp->mac_addr, ETH_ALEN);
 
 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG |
@@ -8607,6 +8608,7 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	return 0;
 
 error:
+	bnx2_release_firmware(bp);
 	pci_iounmap(pdev, bp->regview);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h
index 0e68fadecfdb..243cb9748d35 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h
@@ -492,7 +492,8 @@ int __bnx2x_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
 int bnx2x_get_vf_config(struct net_device *dev, int vf,
 			struct ifla_vf_info *ivi);
 int bnx2x_set_vf_mac(struct net_device *dev, int queue, u8 *mac);
-int bnx2x_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos);
+int bnx2x_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
+		      __be16 vlan_proto);
 
 /* select_queue callback */
 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
index 97e892511666..20fe6a8c35c1 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
@@ -772,6 +772,11 @@ void bnx2x_fw_dump_lvl(struct bnx2x *bp, const char *lvl)
 		(bp->common.bc_ver & 0xff00) >> 8,
 		(bp->common.bc_ver & 0xff));
 
+	if (pci_channel_offline(bp->pdev)) {
+		BNX2X_ERR("Cannot dump MCP info while in PCI error\n");
+		return;
+	}
+
 	val = REG_RD(bp, MCP_REG_MCPR_CPU_PROGRAM_COUNTER);
 	if (val == REG_RD(bp, MCP_REG_MCPR_CPU_PROGRAM_COUNTER))
 		BNX2X_ERR("%s" "MCP PC at 0x%x\n", lvl, val);
@@ -9415,10 +9420,16 @@ unload_error:
 	/* Release IRQs */
 	bnx2x_free_irq(bp);
 
-	/* Reset the chip */
-	rc = bnx2x_reset_hw(bp, reset_code);
-	if (rc)
-		BNX2X_ERR("HW_RESET failed\n");
+	/* Reset the chip, unless PCI function is offline. If we reach this
+	 * point following a PCI error handling, it means device is really
+	 * in a bad state and we're about to remove it, so reset the chip
+	 * is not a good idea.
+	 */
+	if (!pci_channel_offline(bp->pdev)) {
+		rc = bnx2x_reset_hw(bp, reset_code);
+		if (rc)
+			BNX2X_ERR("HW_RESET failed\n");
+	}
 
 	/* Report UNLOAD_DONE to MCP */
 	bnx2x_send_unload_done(bp, keep_link);
@@ -12552,41 +12563,64 @@ static int bnx2x_close(struct net_device *dev)
 	return 0;
 }
 
+struct bnx2x_mcast_list_elem_group
+{
+	struct list_head mcast_group_link;
+	struct bnx2x_mcast_list_elem mcast_elems[];
+};
+
+#define MCAST_ELEMS_PER_PG \
+	((PAGE_SIZE - sizeof(struct bnx2x_mcast_list_elem_group)) / \
+	sizeof(struct bnx2x_mcast_list_elem))
+
+static void bnx2x_free_mcast_macs_list(struct list_head *mcast_group_list)
+{
+	struct bnx2x_mcast_list_elem_group *current_mcast_group;
+
+	while (!list_empty(mcast_group_list)) {
+		current_mcast_group = list_first_entry(mcast_group_list,
+				      struct bnx2x_mcast_list_elem_group,
+				      mcast_group_link);
+		list_del(&current_mcast_group->mcast_group_link);
+		free_page((unsigned long)current_mcast_group);
+	}
+}
+
 static int bnx2x_init_mcast_macs_list(struct bnx2x *bp,
-				      struct bnx2x_mcast_ramrod_params *p)
+				      struct bnx2x_mcast_ramrod_params *p,
+				      struct list_head *mcast_group_list)
 {
-	int mc_count = netdev_mc_count(bp->dev);
-	struct bnx2x_mcast_list_elem *mc_mac =
-		kcalloc(mc_count, sizeof(*mc_mac), GFP_ATOMIC);
+	struct bnx2x_mcast_list_elem *mc_mac;
 	struct netdev_hw_addr *ha;
-
-	if (!mc_mac)
-		return -ENOMEM;
+	struct bnx2x_mcast_list_elem_group *current_mcast_group = NULL;
+	int mc_count = netdev_mc_count(bp->dev);
+	int offset = 0;
 
 	INIT_LIST_HEAD(&p->mcast_list);
-
 	netdev_for_each_mc_addr(ha, bp->dev) {
+		if (!offset) {
+			current_mcast_group =
+				(struct bnx2x_mcast_list_elem_group *)
+				__get_free_page(GFP_ATOMIC);
+			if (!current_mcast_group) {
+				bnx2x_free_mcast_macs_list(mcast_group_list);
+				BNX2X_ERR("Failed to allocate mc MAC list\n");
+				return -ENOMEM;
+			}
+			list_add(&current_mcast_group->mcast_group_link,
+				 mcast_group_list);
+		}
+		mc_mac = &current_mcast_group->mcast_elems[offset];
 		mc_mac->mac = bnx2x_mc_addr(ha);
 		list_add_tail(&mc_mac->link, &p->mcast_list);
-		mc_mac++;
+		offset++;
+		if (offset == MCAST_ELEMS_PER_PG)
+			offset = 0;
 	}
-
 	p->mcast_list_len = mc_count;
-
 	return 0;
 }
 
-static void bnx2x_free_mcast_macs_list(
-	struct bnx2x_mcast_ramrod_params *p)
-{
-	struct bnx2x_mcast_list_elem *mc_mac =
-		list_first_entry(&p->mcast_list, struct bnx2x_mcast_list_elem,
-				 link);
-
-	WARN_ON(!mc_mac);
-	kfree(mc_mac);
-}
-
 /**
  * bnx2x_set_uc_list - configure a new unicast MACs list.
  *
@@ -12632,8 +12666,9 @@ static int bnx2x_set_uc_list(struct bnx2x *bp)
 				 BNX2X_UC_LIST_MAC, &ramrod_flags);
 }
 
-static int bnx2x_set_mc_list(struct bnx2x *bp)
+static int bnx2x_set_mc_list_e1x(struct bnx2x *bp)
 {
+	LIST_HEAD(mcast_group_list);
 	struct net_device *dev = bp->dev;
 	struct bnx2x_mcast_ramrod_params rparam = {NULL};
 	int rc = 0;
@@ -12649,12 +12684,9 @@ static int bnx2x_set_mc_list(struct bnx2x *bp)
 
 	/* then, configure a new MACs list */
 	if (netdev_mc_count(dev)) {
-		rc = bnx2x_init_mcast_macs_list(bp, &rparam);
-		if (rc) {
-			BNX2X_ERR("Failed to create multicast MACs list: %d\n",
-				  rc);
+		rc = bnx2x_init_mcast_macs_list(bp, &rparam, &mcast_group_list);
+		if (rc)
 			return rc;
-		}
 
 		/* Now add the new MACs */
 		rc = bnx2x_config_mcast(bp, &rparam,
@@ -12663,7 +12695,44 @@ static int bnx2x_set_mc_list(struct bnx2x *bp)
 			BNX2X_ERR("Failed to set a new multicast configuration: %d\n",
 				  rc);
 
-		bnx2x_free_mcast_macs_list(&rparam);
+		bnx2x_free_mcast_macs_list(&mcast_group_list);
+	}
+
+	return rc;
+}
+
+static int bnx2x_set_mc_list(struct bnx2x *bp)
+{
+	LIST_HEAD(mcast_group_list);
+	struct bnx2x_mcast_ramrod_params rparam = {NULL};
+	struct net_device *dev = bp->dev;
+	int rc = 0;
+
+	/* On older adapters, we need to flush and re-add filters */
+	if (CHIP_IS_E1x(bp))
+		return bnx2x_set_mc_list_e1x(bp);
+
+	rparam.mcast_obj = &bp->mcast_obj;
+
+	if (netdev_mc_count(dev)) {
+		rc = bnx2x_init_mcast_macs_list(bp, &rparam, &mcast_group_list);
+		if (rc)
+			return rc;
+
+		/* Override the curently configured set of mc filters */
+		rc = bnx2x_config_mcast(bp, &rparam,
+					BNX2X_MCAST_CMD_SET);
+		if (rc < 0)
+			BNX2X_ERR("Failed to set a new multicast configuration: %d\n",
+				  rc);
+
+		bnx2x_free_mcast_macs_list(&mcast_group_list);
+	} else {
+		/* If no mc addresses are required, flush the configuration */
+		rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
+		if (rc)
+			BNX2X_ERR("Failed to clear multicast configuration %d\n",
+				  rc);
 	}
 
 	return rc;
@@ -13203,13 +13272,22 @@ static int bnx2x_init_dev(struct bnx2x *bp, struct pci_dev *pdev,
 		NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_GRO |
 		NETIF_F_RXHASH | NETIF_F_HW_VLAN_CTAG_TX;
 	if (!chip_is_e1x) {
-		dev->hw_features |= NETIF_F_GSO_GRE | NETIF_F_GSO_UDP_TUNNEL |
-				    NETIF_F_GSO_IPXIP4;
+		dev->hw_features |= NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM |
+				    NETIF_F_GSO_IPXIP4 |
+				    NETIF_F_GSO_UDP_TUNNEL |
+				    NETIF_F_GSO_UDP_TUNNEL_CSUM |
+				    NETIF_F_GSO_PARTIAL;
+
 		dev->hw_enc_features =
 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
 			NETIF_F_TSO | NETIF_F_TSO_ECN | NETIF_F_TSO6 |
 			NETIF_F_GSO_IPXIP4 |
-			NETIF_F_GSO_GRE | NETIF_F_GSO_UDP_TUNNEL;
+			NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM |
+			NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_UDP_TUNNEL_CSUM |
+			NETIF_F_GSO_PARTIAL;
+
+		dev->gso_partial_features = NETIF_F_GSO_GRE_CSUM |
+					    NETIF_F_GSO_UDP_TUNNEL_CSUM;
 	}
 
 	dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c
index ff702a707a91..cea6bdcde33f 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c
@@ -2600,8 +2600,29 @@ struct bnx2x_mcast_mac_elem {
 	u8 pad[2]; /* For a natural alignment of the following buffer */
 };
 
+struct bnx2x_mcast_bin_elem {
+	struct list_head link;
+	int bin;
+	int type; /* BNX2X_MCAST_CMD_SET_{ADD, DEL} */
+};
+
+union bnx2x_mcast_elem {
+	struct bnx2x_mcast_bin_elem bin_elem;
+	struct bnx2x_mcast_mac_elem mac_elem;
+};
+
+struct bnx2x_mcast_elem_group {
+	struct list_head mcast_group_link;
+	union bnx2x_mcast_elem mcast_elems[];
+};
+
+#define MCAST_MAC_ELEMS_PER_PG \
+	((PAGE_SIZE - sizeof(struct bnx2x_mcast_elem_group)) / \
+	sizeof(union bnx2x_mcast_elem))
+
 struct bnx2x_pending_mcast_cmd {
 	struct list_head link;
+	struct list_head group_head;
 	int type; /* BNX2X_MCAST_CMD_X */
 	union {
 		struct list_head macs_head;
@@ -2609,6 +2630,11 @@ struct bnx2x_pending_mcast_cmd {
 		int next_bin; /* Needed for RESTORE flow with aprox match */
 	} data;
 
+	bool set_convert; /* in case type == BNX2X_MCAST_CMD_SET, this is set
+			   * when macs_head had been converted to a list of
+			   * bnx2x_mcast_bin_elem.
+			   */
+
 	bool done; /* set to true, when the command has been handled,
 		    * practically used in 57712 handling only, where one pending
 		    * command may be handled in a few operations. As long as for
@@ -2627,53 +2653,93 @@ static int bnx2x_mcast_wait(struct bnx2x *bp,
 	return 0;
 }
 
+static void bnx2x_free_groups(struct list_head *mcast_group_list)
+{
+	struct bnx2x_mcast_elem_group *current_mcast_group;
+
+	while (!list_empty(mcast_group_list)) {
+		current_mcast_group = list_first_entry(mcast_group_list,
+				      struct bnx2x_mcast_elem_group,
+				      mcast_group_link);
+		list_del(&current_mcast_group->mcast_group_link);
+		free_page((unsigned long)current_mcast_group);
+	}
+}
+
 static int bnx2x_mcast_enqueue_cmd(struct bnx2x *bp,
 				   struct bnx2x_mcast_obj *o,
 				   struct bnx2x_mcast_ramrod_params *p,
 				   enum bnx2x_mcast_cmd cmd)
 {
-	int total_sz;
 	struct bnx2x_pending_mcast_cmd *new_cmd;
-	struct bnx2x_mcast_mac_elem *cur_mac = NULL;
 	struct bnx2x_mcast_list_elem *pos;
-	int macs_list_len = ((cmd == BNX2X_MCAST_CMD_ADD) ?
-			     p->mcast_list_len : 0);
+	struct bnx2x_mcast_elem_group *elem_group;
+	struct bnx2x_mcast_mac_elem *mac_elem;
+	int total_elems = 0, macs_list_len = 0, offset = 0;
+
+	/* When adding MACs we'll need to store their values */
+	if (cmd == BNX2X_MCAST_CMD_ADD || cmd == BNX2X_MCAST_CMD_SET)
+		macs_list_len = p->mcast_list_len;
 
 	/* If the command is empty ("handle pending commands only"), break */
 	if (!p->mcast_list_len)
 		return 0;
 
-	total_sz = sizeof(*new_cmd) +
-		macs_list_len * sizeof(struct bnx2x_mcast_mac_elem);
-
 	/* Add mcast is called under spin_lock, thus calling with GFP_ATOMIC */
-	new_cmd = kzalloc(total_sz, GFP_ATOMIC);
-
+	new_cmd = kzalloc(sizeof(*new_cmd), GFP_ATOMIC);
 	if (!new_cmd)
 		return -ENOMEM;
 
-	DP(BNX2X_MSG_SP, "About to enqueue a new %d command. macs_list_len=%d\n",
-	   cmd, macs_list_len);
-
 	INIT_LIST_HEAD(&new_cmd->data.macs_head);
-
+	INIT_LIST_HEAD(&new_cmd->group_head);
 	new_cmd->type = cmd;
 	new_cmd->done = false;
 
+	DP(BNX2X_MSG_SP, "About to enqueue a new %d command. macs_list_len=%d\n",
+	   cmd, macs_list_len);
+
 	switch (cmd) {
 	case BNX2X_MCAST_CMD_ADD:
-		cur_mac = (struct bnx2x_mcast_mac_elem *)
-			  ((u8 *)new_cmd + sizeof(*new_cmd));
-
-		/* Push the MACs of the current command into the pending command
-		 * MACs list: FIFO
+	case BNX2X_MCAST_CMD_SET:
+		/* For a set command, we need to allocate sufficient memory for
+		 * all the bins, since we can't analyze at this point how much
+		 * memory would be required.
 		 */
+		total_elems = macs_list_len;
+		if (cmd == BNX2X_MCAST_CMD_SET) {
+			if (total_elems < BNX2X_MCAST_BINS_NUM)
+				total_elems = BNX2X_MCAST_BINS_NUM;
+		}
+		while (total_elems > 0) {
+			elem_group = (struct bnx2x_mcast_elem_group *)
+				     __get_free_page(GFP_ATOMIC | __GFP_ZERO);
+			if (!elem_group) {
+				bnx2x_free_groups(&new_cmd->group_head);
+				kfree(new_cmd);
+				return -ENOMEM;
+			}
+			total_elems -= MCAST_MAC_ELEMS_PER_PG;
+			list_add_tail(&elem_group->mcast_group_link,
+				      &new_cmd->group_head);
+		}
+		elem_group = list_first_entry(&new_cmd->group_head,
+					      struct bnx2x_mcast_elem_group,
+					      mcast_group_link);
 		list_for_each_entry(pos, &p->mcast_list, link) {
-			memcpy(cur_mac->mac, pos->mac, ETH_ALEN);
-			list_add_tail(&cur_mac->link, &new_cmd->data.macs_head);
-			cur_mac++;
+			mac_elem = &elem_group->mcast_elems[offset].mac_elem;
+			memcpy(mac_elem->mac, pos->mac, ETH_ALEN);
+			/* Push the MACs of the current command into the pending
+			 * command MACs list: FIFO
+			 */
+			list_add_tail(&mac_elem->link,
+				      &new_cmd->data.macs_head);
+			offset++;
+			if (offset == MCAST_MAC_ELEMS_PER_PG) {
+				offset = 0;
+				elem_group = list_next_entry(elem_group,
+							     mcast_group_link);
+			}
 		}
-
 		break;
 
 	case BNX2X_MCAST_CMD_DEL:
@@ -2771,7 +2837,8 @@ static void bnx2x_mcast_set_one_rule_e2(struct bnx2x *bp,
 	u8 rx_tx_add_flag = bnx2x_mcast_get_rx_tx_flag(o);
 	int bin;
 
-	if ((cmd == BNX2X_MCAST_CMD_ADD) || (cmd == BNX2X_MCAST_CMD_RESTORE))
+	if ((cmd == BNX2X_MCAST_CMD_ADD) || (cmd == BNX2X_MCAST_CMD_RESTORE) ||
+	    (cmd == BNX2X_MCAST_CMD_SET_ADD))
 		rx_tx_add_flag |= ETH_MULTICAST_RULES_CMD_IS_ADD;
 
 	data->rules[idx].cmd_general_data |= rx_tx_add_flag;
@@ -2797,6 +2864,16 @@ static void bnx2x_mcast_set_one_rule_e2(struct bnx2x *bp,
 		bin = cfg_data->bin;
 		break;
 
+	case BNX2X_MCAST_CMD_SET_ADD:
+		bin = cfg_data->bin;
+		BIT_VEC64_SET_BIT(o->registry.aprox_match.vec, bin);
+		break;
+
+	case BNX2X_MCAST_CMD_SET_DEL:
+		bin = cfg_data->bin;
+		BIT_VEC64_CLEAR_BIT(o->registry.aprox_match.vec, bin);
+		break;
+
 	default:
 		BNX2X_ERR("Unknown command: %d\n", cmd);
 		return;
@@ -2932,6 +3009,110 @@ static inline void bnx2x_mcast_hdl_pending_restore_e2(struct bnx2x *bp,
 		cmd_pos->data.next_bin++;
 }
 
+static void
+bnx2x_mcast_hdl_pending_set_e2_convert(struct bnx2x *bp,
+				       struct bnx2x_mcast_obj *o,
+				       struct bnx2x_pending_mcast_cmd *cmd_pos)
+{
+	u64 cur[BNX2X_MCAST_VEC_SZ], req[BNX2X_MCAST_VEC_SZ];
+	struct bnx2x_mcast_mac_elem *pmac_pos, *pmac_pos_n;
+	struct bnx2x_mcast_bin_elem *p_item;
+	struct bnx2x_mcast_elem_group *elem_group;
+	int cnt = 0, mac_cnt = 0, offset = 0, i;
+
+	memset(req, 0, sizeof(u64) * BNX2X_MCAST_VEC_SZ);
+	memcpy(cur, o->registry.aprox_match.vec,
+	       sizeof(u64) * BNX2X_MCAST_VEC_SZ);
+
+	/* Fill `current' with the required set of bins to configure */
+	list_for_each_entry_safe(pmac_pos, pmac_pos_n, &cmd_pos->data.macs_head,
+				 link) {
+		int bin = bnx2x_mcast_bin_from_mac(pmac_pos->mac);
+
+		DP(BNX2X_MSG_SP, "Set contains %pM mcast MAC\n",
+		   pmac_pos->mac);
+
+		BIT_VEC64_SET_BIT(req, bin);
+		list_del(&pmac_pos->link);
+		mac_cnt++;
+	}
+
+	/* We no longer have use for the MACs; Need to re-use memory for
+	 * a list that will be used to configure bins.
+	 */
+	cmd_pos->set_convert = true;
+	INIT_LIST_HEAD(&cmd_pos->data.macs_head);
+	elem_group = list_first_entry(&cmd_pos->group_head,
+				      struct bnx2x_mcast_elem_group,
+				      mcast_group_link);
+	for (i = 0; i < BNX2X_MCAST_BINS_NUM; i++) {
+		bool b_current = !!BIT_VEC64_TEST_BIT(cur, i);
+		bool b_required = !!BIT_VEC64_TEST_BIT(req, i);
+
+		if (b_current == b_required)
+			continue;
+
+		p_item = &elem_group->mcast_elems[offset].bin_elem;
+		p_item->bin = i;
+		p_item->type = b_required ? BNX2X_MCAST_CMD_SET_ADD
+					  : BNX2X_MCAST_CMD_SET_DEL;
+		list_add_tail(&p_item->link , &cmd_pos->data.macs_head);
+		cnt++;
+		offset++;
+		if (offset == MCAST_MAC_ELEMS_PER_PG) {
+			offset = 0;
+			elem_group = list_next_entry(elem_group,
+						     mcast_group_link);
+		}
+	}
+
+	/* We now definitely know how many commands are hiding here.
+	 * Also need to correct the disruption we've added to guarantee this
+	 * would be enqueued.
+	 */
+	o->total_pending_num -= (o->max_cmd_len + mac_cnt);
+	o->total_pending_num += cnt;
+
+	DP(BNX2X_MSG_SP, "o->total_pending_num=%d\n", o->total_pending_num);
+}
+
+static void
+bnx2x_mcast_hdl_pending_set_e2(struct bnx2x *bp,
+			       struct bnx2x_mcast_obj *o,
+			       struct bnx2x_pending_mcast_cmd *cmd_pos,
+			       int *cnt)
+{
+	union bnx2x_mcast_config_data cfg_data = {NULL};
+	struct bnx2x_mcast_bin_elem *p_item, *p_item_n;
+
+	/* This is actually a 2-part scheme - it starts by converting the MACs
+	 * into a list of bins to be added/removed, and correcting the numbers
+	 * on the object. this is now allowed, as we're now sure that all
+	 * previous configured requests have already applied.
+	 * The second part is actually adding rules for the newly introduced
+	 * entries [like all the rest of the hdl_pending functions].
+	 */
+	if (!cmd_pos->set_convert)
+		bnx2x_mcast_hdl_pending_set_e2_convert(bp, o, cmd_pos);
+
+	list_for_each_entry_safe(p_item, p_item_n, &cmd_pos->data.macs_head,
+				 link) {
+		cfg_data.bin = (u8)p_item->bin;
+		o->set_one_rule(bp, o, *cnt, &cfg_data, p_item->type);
+		(*cnt)++;
+
+		list_del(&p_item->link);
+
+		/* Break if we reached the maximum number of rules. */
+		if (*cnt >= o->max_cmd_len)
+			break;
+	}
+
+	/* if no more MACs to configure - we are done */
+	if (list_empty(&cmd_pos->data.macs_head))
+		cmd_pos->done = true;
+}
+
 static inline int bnx2x_mcast_handle_pending_cmds_e2(struct bnx2x *bp,
 				struct bnx2x_mcast_ramrod_params *p)
 {
@@ -2955,6 +3136,10 @@ static inline int bnx2x_mcast_handle_pending_cmds_e2(struct bnx2x *bp,
 							   &cnt);
 			break;
 
+		case BNX2X_MCAST_CMD_SET:
+			bnx2x_mcast_hdl_pending_set_e2(bp, o, cmd_pos, &cnt);
+			break;
+
 		default:
 			BNX2X_ERR("Unknown command: %d\n", cmd_pos->type);
 			return -EINVAL;
@@ -2965,6 +3150,7 @@ static inline int bnx2x_mcast_handle_pending_cmds_e2(struct bnx2x *bp,
 		 */
 		if (cmd_pos->done) {
 			list_del(&cmd_pos->link);
+			bnx2x_free_groups(&cmd_pos->group_head);
 			kfree(cmd_pos);
 		}
 
@@ -3095,6 +3281,19 @@ static int bnx2x_mcast_validate_e2(struct bnx2x *bp,
 		o->set_registry_size(o, reg_sz + p->mcast_list_len);
 		break;
 
+	case BNX2X_MCAST_CMD_SET:
+		/* We can only learn how many commands would actually be used
+		 * when this is being configured. So for now, simply guarantee
+		 * the command will be enqueued [to refrain from adding logic
+		 * that handles this and THEN learns it needs several ramrods].
+		 * Just like for ADD/Cont, the mcast_list_len might be an over
+		 * estimation; or even more so, since we don't take into
+		 * account the possibility of removal of existing bins.
+		 */
+		o->set_registry_size(o, reg_sz + p->mcast_list_len);
+		o->total_pending_num += o->max_cmd_len;
+		break;
+
 	default:
 		BNX2X_ERR("Unknown command: %d\n", cmd);
 		return -EINVAL;
@@ -3108,12 +3307,16 @@ static int bnx2x_mcast_validate_e2(struct bnx2x *bp,
 
 static void bnx2x_mcast_revert_e2(struct bnx2x *bp,
 				      struct bnx2x_mcast_ramrod_params *p,
-				      int old_num_bins)
+				  int old_num_bins,
+				  enum bnx2x_mcast_cmd cmd)
 {
 	struct bnx2x_mcast_obj *o = p->mcast_obj;
 
 	o->set_registry_size(o, old_num_bins);
 	o->total_pending_num -= p->mcast_list_len;
+
+	if (cmd == BNX2X_MCAST_CMD_SET)
+		o->total_pending_num -= o->max_cmd_len;
 }
 
 /**
@@ -3223,9 +3426,11 @@ static int bnx2x_mcast_setup_e2(struct bnx2x *bp,
 		bnx2x_mcast_refresh_registry_e2(bp, o);
 
 	/* If CLEAR_ONLY was requested - don't send a ramrod and clear
-	 * RAMROD_PENDING status immediately.
+	 * RAMROD_PENDING status immediately. due to the SET option, it's also
+	 * possible that after evaluating the differences there's no need for
+	 * a ramrod. In that case, we can skip it as well.
 	 */
-	if (test_bit(RAMROD_DRV_CLR_ONLY, &p->ramrod_flags)) {
+	if (test_bit(RAMROD_DRV_CLR_ONLY, &p->ramrod_flags) || !cnt) {
 		raw->clear_pending(raw);
 		return 0;
 	} else {
@@ -3253,6 +3458,11 @@ static int bnx2x_mcast_validate_e1h(struct bnx2x *bp,
 				    struct bnx2x_mcast_ramrod_params *p,
 				    enum bnx2x_mcast_cmd cmd)
 {
+	if (cmd == BNX2X_MCAST_CMD_SET) {
+		BNX2X_ERR("Can't use `set' command on e1h!\n");
+		return -EINVAL;
+	}
+
 	/* Mark, that there is a work to do */
 	if ((cmd == BNX2X_MCAST_CMD_DEL) || (cmd == BNX2X_MCAST_CMD_RESTORE))
 		p->mcast_list_len = 1;
@@ -3262,7 +3472,8 @@ static int bnx2x_mcast_validate_e1h(struct bnx2x *bp,
 
 static void bnx2x_mcast_revert_e1h(struct bnx2x *bp,
 				       struct bnx2x_mcast_ramrod_params *p,
-				       int old_num_bins)
+				       int old_num_bins,
+				       enum bnx2x_mcast_cmd cmd)
 {
 	/* Do nothing */
 }
@@ -3372,6 +3583,11 @@ static int bnx2x_mcast_validate_e1(struct bnx2x *bp,
 	struct bnx2x_mcast_obj *o = p->mcast_obj;
 	int reg_sz = o->get_registry_size(o);
 
+	if (cmd == BNX2X_MCAST_CMD_SET) {
+		BNX2X_ERR("Can't use `set' command on e1!\n");
+		return -EINVAL;
+	}
+
 	switch (cmd) {
 	/* DEL command deletes all currently configured MACs */
 	case BNX2X_MCAST_CMD_DEL:
@@ -3422,7 +3638,8 @@ static int bnx2x_mcast_validate_e1(struct bnx2x *bp,
 
 static void bnx2x_mcast_revert_e1(struct bnx2x *bp,
 				      struct bnx2x_mcast_ramrod_params *p,
-				      int old_num_macs)
+				   int old_num_macs,
+				   enum bnx2x_mcast_cmd cmd)
 {
 	struct bnx2x_mcast_obj *o = p->mcast_obj;
 
@@ -3572,6 +3789,7 @@ static inline int bnx2x_mcast_handle_pending_cmds_e1(
 	}
 
 	list_del(&cmd_pos->link);
+	bnx2x_free_groups(&cmd_pos->group_head);
 	kfree(cmd_pos);
 
 	return cnt;
@@ -3816,7 +4034,7 @@ error_exit2:
 	r->clear_pending(r);
 
 error_exit1:
-	o->revert(bp, p, old_reg_size);
+	o->revert(bp, p, old_reg_size, cmd);
 
 	return rc;
 }
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.h
index 4048fc594cce..0bf2fd470819 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.h
@@ -536,6 +536,15 @@ enum bnx2x_mcast_cmd {
 	BNX2X_MCAST_CMD_CONT,
 	BNX2X_MCAST_CMD_DEL,
 	BNX2X_MCAST_CMD_RESTORE,
+
+	/* Following this, multicast configuration should equal to approx
+	 * the set of MACs provided [i.e., remove all else].
+	 * The two sub-commands are used internally to decide whether a given
+	 * bin is to be added or removed
+	 */
+	BNX2X_MCAST_CMD_SET,
+	BNX2X_MCAST_CMD_SET_ADD,
+	BNX2X_MCAST_CMD_SET_DEL,
 };
 
 struct bnx2x_mcast_obj {
@@ -635,7 +644,8 @@ struct bnx2x_mcast_obj {
 	 */
 	void (*revert)(struct bnx2x *bp,
 		       struct bnx2x_mcast_ramrod_params *p,
-		       int old_num_bins);
+		       int old_num_bins,
+		       enum bnx2x_mcast_cmd cmd);
 
 	int (*get_registry_size)(struct bnx2x_mcast_obj *o);
 	void (*set_registry_size)(struct bnx2x_mcast_obj *o, int n);
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
index 632daff117d3..3f77d0863543 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
@@ -573,17 +573,6 @@ int bnx2x_vf_mcast(struct bnx2x *bp, struct bnx2x_virtf *vf,
 		}
 	}
 
-	/* clear existing mcasts */
-	mcast.mcast_list_len = vf->mcast_list_len;
-	vf->mcast_list_len = mc_num;
-	rc = bnx2x_config_mcast(bp, &mcast, BNX2X_MCAST_CMD_DEL);
-	if (rc) {
-		BNX2X_ERR("Failed to remove multicasts\n");
-		kfree(mc);
-		return rc;
-	}
-
-	/* update mcast list on the ramrod params */
 	if (mc_num) {
 		INIT_LIST_HEAD(&mcast.mcast_list);
 		for (i = 0; i < mc_num; i++) {
@@ -594,12 +583,18 @@ int bnx2x_vf_mcast(struct bnx2x *bp, struct bnx2x_virtf *vf,
 
 		/* add new mcasts */
 		mcast.mcast_list_len = mc_num;
-		rc = bnx2x_config_mcast(bp, &mcast, BNX2X_MCAST_CMD_ADD);
+		rc = bnx2x_config_mcast(bp, &mcast, BNX2X_MCAST_CMD_SET);
 		if (rc)
-			BNX2X_ERR("Faled to add multicasts\n");
-		kfree(mc);
+			BNX2X_ERR("Faled to set multicasts\n");
+	} else {
+		/* clear existing mcasts */
+		rc = bnx2x_config_mcast(bp, &mcast, BNX2X_MCAST_CMD_DEL);
+		if (rc)
+			BNX2X_ERR("Failed to remove multicasts\n");
 	}
 
+	kfree(mc);
+
 	return rc;
 }
 
@@ -1583,7 +1578,6 @@ int bnx2x_iov_nic_init(struct bnx2x *bp)
 		 *  It needs to be initialized here so that it can be safely
 		 *  handled by a subsequent FLR flow.
 		 */
-		vf->mcast_list_len = 0;
 		bnx2x_init_mcast_obj(bp, &vf->mcast_obj, 0xFF,
 				     0xFF, 0xFF, 0xFF,
 				     bnx2x_vf_sp(bp, vf, mcast_rdata),
@@ -2527,7 +2521,8 @@ void bnx2x_pf_set_vfs_vlan(struct bnx2x *bp)
 	for_each_vf(bp, vfidx) {
 		bulletin = BP_VF_BULLETIN(bp, vfidx);
 		if (bulletin->valid_bitmap & (1 << VLAN_VALID))
-			bnx2x_set_vf_vlan(bp->dev, vfidx, bulletin->vlan, 0);
+			bnx2x_set_vf_vlan(bp->dev, vfidx, bulletin->vlan, 0,
+					  htons(ETH_P_8021Q));
 	}
 }
 
@@ -2787,7 +2782,8 @@ static int bnx2x_set_vf_vlan_filter(struct bnx2x *bp, struct bnx2x_virtf *vf,
 	return 0;
 }
 
-int bnx2x_set_vf_vlan(struct net_device *dev, int vfidx, u16 vlan, u8 qos)
+int bnx2x_set_vf_vlan(struct net_device *dev, int vfidx, u16 vlan, u8 qos,
+		      __be16 vlan_proto)
 {
 	struct pf_vf_bulletin_content *bulletin = NULL;
 	struct bnx2x *bp = netdev_priv(dev);
@@ -2802,6 +2798,9 @@ int bnx2x_set_vf_vlan(struct net_device *dev, int vfidx, u16 vlan, u8 qos)
 		return -EINVAL;
 	}
 
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
 	DP(BNX2X_MSG_IOV, "configuring VF %d with VLAN %d qos %d\n",
 	   vfidx, vlan, 0);
 
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h
index 670a581ffabc..7a6d406f4c11 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h
@@ -195,7 +195,6 @@ struct bnx2x_virtf {
 	int leading_rss;
 
 	/* MCAST object */
-	int mcast_list_len;
 	struct bnx2x_mcast_obj		mcast_obj;
 
 	/* RSS configuration object */
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index 2cf79100c9cb..a9f9f3738022 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -32,6 +32,7 @@
 #include <linux/mii.h>
 #include <linux/if.h>
 #include <linux/if_vlan.h>
+#include <linux/rtc.h>
 #include <net/ip.h>
 #include <net/tcp.h>
 #include <net/udp.h>
@@ -93,50 +94,49 @@ enum board_idx {
 	BCM57404_NPAR,
 	BCM57406_NPAR,
 	BCM57407_SFP,
+	BCM57407_NPAR,
 	BCM57414_NPAR,
 	BCM57416_NPAR,
-	BCM57304_VF,
-	BCM57404_VF,
-	BCM57414_VF,
-	BCM57314_VF,
+	NETXTREME_E_VF,
+	NETXTREME_C_VF,
 };
 
 /* indexed by enum above */
 static const struct {
 	char *name;
 } board_info[] = {
-	{ "Broadcom BCM57301 NetXtreme-C Single-port 10Gb Ethernet" },
-	{ "Broadcom BCM57302 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57304 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
+	{ "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
+	{ "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
+	{ "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 	{ "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM58700 Nitro 4-port 1Gb/2.5Gb/10Gb Ethernet" },
-	{ "Broadcom BCM57311 NetXtreme-C Single-port 10Gb Ethernet" },
-	{ "Broadcom BCM57312 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57402 NetXtreme-E Dual-port 10Gb Ethernet" },
-	{ "Broadcom BCM57404 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57406 NetXtreme-E Dual-port 10GBase-T Ethernet" },
+	{ "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
+	{ "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
+	{ "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
+	{ "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
+	{ "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
+	{ "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 	{ "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57407 NetXtreme-E Dual-port 10GBase-T Ethernet" },
-	{ "Broadcom BCM57412 NetXtreme-E Dual-port 10Gb Ethernet" },
-	{ "Broadcom BCM57414 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57416 NetXtreme-E Dual-port 10GBase-T Ethernet" },
-	{ "Broadcom BCM57417 NetXtreme-E Dual-port 10GBase-T Ethernet" },
+	{ "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
+	{ "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
+	{ "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
+	{ "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
+	{ "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 	{ "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57314 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
-	{ "Broadcom BCM57417 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57416 NetXtreme-E Dual-port 10Gb Ethernet" },
+	{ "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
+	{ "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
+	{ "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 	{ "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 	{ "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57407 NetXtreme-E Dual-port 25Gb Ethernet" },
+	{ "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
+	{ "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 	{ "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 	{ "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57304 NetXtreme-C Ethernet Virtual Function" },
-	{ "Broadcom BCM57404 NetXtreme-E Ethernet Virtual Function" },
-	{ "Broadcom BCM57414 NetXtreme-E Ethernet Virtual Function" },
-	{ "Broadcom BCM57314 NetXtreme-E Ethernet Virtual Function" },
+	{ "Broadcom NetXtreme-E Ethernet Virtual Function" },
+	{ "Broadcom NetXtreme-C Ethernet Virtual Function" },
 };
 
 static const struct pci_device_id bnxt_pci_tbl[] = {
+	{ PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 	{ PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 	{ PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
@@ -160,13 +160,19 @@ static const struct pci_device_id bnxt_pci_tbl[] = {
 	{ PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
+	{ PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
+	{ PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
+	{ PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
+	{ PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 #ifdef CONFIG_BNXT_SRIOV
-	{ PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = BCM57304_VF },
-	{ PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = BCM57404_VF },
-	{ PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = BCM57414_VF },
-	{ PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = BCM57314_VF },
+	{ PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
+	{ PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
+	{ PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
+	{ PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
+	{ PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
+	{ PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 #endif
 	{ 0 }
 };
@@ -189,8 +195,7 @@ static const u16 bnxt_async_events_arr[] = {
 
 static bool bnxt_vf_pciid(enum board_idx idx)
 {
-	return (idx == BCM57304_VF || idx == BCM57404_VF ||
-		idx == BCM57314_VF || idx == BCM57414_VF);
+	return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF);
 }
 
 #define DB_CP_REARM_FLAGS	(DB_KEY_CP | DB_IDX_VALID)
@@ -353,8 +358,8 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		push_len = (length + sizeof(*tx_push) + 7) / 8;
 		if (push_len > 16) {
 			__iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
-			__iowrite64_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
-					 push_len - 16);
+			__iowrite32_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
+					 (push_len - 16) << 1);
 		} else {
 			__iowrite64_copy(txr->tx_doorbell, tx_push_buf,
 					 push_len);
@@ -3419,10 +3424,10 @@ static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
 
 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
 	if (set_rss) {
-		vnic->hash_type = BNXT_RSS_HASH_TYPE_FLAG_IPV4 |
-				 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV4 |
-				 BNXT_RSS_HASH_TYPE_FLAG_IPV6 |
-				 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV6;
+		vnic->hash_type = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
+				  VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
+				  VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
+				  VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
 
 		req.hash_type = cpu_to_le32(vnic->hash_type);
 
@@ -4156,6 +4161,11 @@ int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 	if (rc)
 		goto hwrm_func_qcaps_exit;
 
+	bp->tx_push_thresh = 0;
+	if (resp->flags &
+	    cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
+		bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
+
 	if (BNXT_PF(bp)) {
 		struct bnxt_pf_info *pf = &bp->pf;
 
@@ -4187,12 +4197,6 @@ int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 		struct bnxt_vf_info *vf = &bp->vf;
 
 		vf->fw_fid = le16_to_cpu(resp->fid);
-		memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
-		if (is_valid_ether_addr(vf->mac_addr))
-			/* overwrite netdev dev_adr with admin VF MAC */
-			memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
-		else
-			random_ether_addr(bp->dev->dev_addr);
 
 		vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
 		vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
@@ -4204,14 +4208,21 @@ int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 		vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
 		vf->max_vnics = le16_to_cpu(resp->max_vnics);
 		vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
+
+		memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
+		mutex_unlock(&bp->hwrm_cmd_lock);
+
+		if (is_valid_ether_addr(vf->mac_addr)) {
+			/* overwrite netdev dev_adr with admin VF MAC */
+			memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
+		} else {
+			random_ether_addr(bp->dev->dev_addr);
+			rc = bnxt_approve_mac(bp, bp->dev->dev_addr);
+		}
+		return rc;
 #endif
 	}
 
-	bp->tx_push_thresh = 0;
-	if (resp->flags &
-	    cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
-		bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
-
 hwrm_func_qcaps_exit:
 	mutex_unlock(&bp->hwrm_cmd_lock);
 	return rc;
@@ -4249,6 +4260,9 @@ static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
 	if (bp->max_tc > BNXT_MAX_QUEUE)
 		bp->max_tc = BNXT_MAX_QUEUE;
 
+	if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
+		bp->max_tc = 1;
+
 	qptr = &resp->queue_id0;
 	for (i = 0; i < bp->max_tc; i++) {
 		bp->q_info[i].queue_id = *qptr++;
@@ -4307,6 +4321,31 @@ hwrm_ver_get_exit:
 	return rc;
 }
 
+int bnxt_hwrm_fw_set_time(struct bnxt *bp)
+{
+#if IS_ENABLED(CONFIG_RTC_LIB)
+	struct hwrm_fw_set_time_input req = {0};
+	struct rtc_time tm;
+	struct timeval tv;
+
+	if (bp->hwrm_spec_code < 0x10400)
+		return -EOPNOTSUPP;
+
+	do_gettimeofday(&tv);
+	rtc_time_to_tm(tv.tv_sec, &tm);
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
+	req.year = cpu_to_le16(1900 + tm.tm_year);
+	req.month = 1 + tm.tm_mon;
+	req.day = tm.tm_mday;
+	req.hour = tm.tm_hour;
+	req.minute = tm.tm_min;
+	req.second = tm.tm_sec;
+	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+#else
+	return -EOPNOTSUPP;
+#endif
+}
+
 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
 {
 	int rc;
@@ -6804,6 +6843,8 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (rc)
 		goto init_err;
 
+	bnxt_hwrm_fw_set_time(bp);
+
 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
 			   NETIF_F_TSO | NETIF_F_TSO6 |
 			   NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index 23e04a6142fb..51b164a0e844 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -11,10 +11,10 @@
 #define BNXT_H
 
 #define DRV_MODULE_NAME		"bnxt_en"
-#define DRV_MODULE_VERSION	"1.3.0"
+#define DRV_MODULE_VERSION	"1.5.0"
 
 #define DRV_VER_MAJ	1
-#define DRV_VER_MIN	3
+#define DRV_VER_MIN	5
 #define DRV_VER_UPD	0
 
 struct tx_bd {
@@ -106,11 +106,11 @@ struct tx_cmp {
 	 #define CMP_TYPE_REMOTE_DRIVER_REQ			 34
 	 #define CMP_TYPE_REMOTE_DRIVER_RESP			 36
 	 #define CMP_TYPE_ERROR_STATUS				 48
-	 #define CMPL_BASE_TYPE_STAT_EJECT			 (0x1aUL << 0)
-	 #define CMPL_BASE_TYPE_HWRM_DONE			 (0x20UL << 0)
-	 #define CMPL_BASE_TYPE_HWRM_FWD_REQ			 (0x22UL << 0)
-	 #define CMPL_BASE_TYPE_HWRM_FWD_RESP			 (0x24UL << 0)
-	 #define CMPL_BASE_TYPE_HWRM_ASYNC_EVENT		 (0x2eUL << 0)
+	 #define CMPL_BASE_TYPE_STAT_EJECT			 0x1aUL
+	 #define CMPL_BASE_TYPE_HWRM_DONE			 0x20UL
+	 #define CMPL_BASE_TYPE_HWRM_FWD_REQ			 0x22UL
+	 #define CMPL_BASE_TYPE_HWRM_FWD_RESP			 0x24UL
+	 #define CMPL_BASE_TYPE_HWRM_ASYNC_EVENT		 0x2eUL
 
 	#define TX_CMP_FLAGS_ERROR				(1 << 6)
 	#define TX_CMP_FLAGS_PUSH				(1 << 7)
@@ -389,11 +389,6 @@ struct rx_tpa_end_cmp_ext {
 
 #define INVALID_HW_RING_ID	((u16)-1)
 
-#define BNXT_RSS_HASH_TYPE_FLAG_IPV4		0x01
-#define BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV4	0x02
-#define BNXT_RSS_HASH_TYPE_FLAG_IPV6		0x04
-#define BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV6	0x08
-
 /* The hardware supports certain page sizes.  Use the supported page sizes
  * to allocate the rings.
  */
@@ -418,7 +413,7 @@ struct rx_tpa_end_cmp_ext {
 
 #define BNXT_RX_PAGE_SIZE (1 << BNXT_RX_PAGE_SHIFT)
 
-#define BNXT_MIN_PKT_SIZE	45
+#define BNXT_MIN_PKT_SIZE	52
 
 #define BNXT_NUM_TESTS(bp)	0
 
@@ -1225,6 +1220,7 @@ int bnxt_hwrm_set_coal(struct bnxt *);
 int bnxt_hwrm_func_qcaps(struct bnxt *);
 int bnxt_hwrm_set_pause(struct bnxt *);
 int bnxt_hwrm_set_link_setting(struct bnxt *, bool, bool);
+int bnxt_hwrm_fw_set_time(struct bnxt *);
 int bnxt_open_nic(struct bnxt *, bool, bool);
 int bnxt_close_nic(struct bnxt *, bool, bool);
 int bnxt_get_max_rings(struct bnxt *, int *, int *, bool);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index b83e17403d6c..a7e04ff4eaed 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -21,6 +21,8 @@
 #include "bnxt_nvm_defs.h"	/* NVRAM content constant and structure defs */
 #include "bnxt_fw_hdr.h"	/* Firmware hdr constant and structure defs */
 #define FLASH_NVRAM_TIMEOUT	((HWRM_CMD_TIMEOUT) * 100)
+#define FLASH_PACKAGE_TIMEOUT	((HWRM_CMD_TIMEOUT) * 200)
+#define INSTALL_PACKAGE_TIMEOUT	((HWRM_CMD_TIMEOUT) * 200)
 
 static char *bnxt_get_pkgver(struct net_device *dev, char *buf, size_t buflen);
 
@@ -346,7 +348,7 @@ static void bnxt_get_channels(struct net_device *dev,
 	int max_rx_rings, max_tx_rings, tcs;
 
 	bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, true);
-	channel->max_combined = max_rx_rings;
+	channel->max_combined = max_t(int, max_rx_rings, max_tx_rings);
 
 	if (bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, false)) {
 		max_rx_rings = 0;
@@ -404,8 +406,8 @@ static int bnxt_set_channels(struct net_device *dev,
 	if (tcs > 1)
 		max_tx_rings /= tcs;
 
-	if (sh && (channel->combined_count > max_rx_rings ||
-		   channel->combined_count > max_tx_rings))
+	if (sh &&
+	    channel->combined_count > max_t(int, max_rx_rings, max_tx_rings))
 		return -ENOMEM;
 
 	if (!sh && (channel->rx_count > max_rx_rings ||
@@ -428,8 +430,10 @@ static int bnxt_set_channels(struct net_device *dev,
 
 	if (sh) {
 		bp->flags |= BNXT_FLAG_SHARED_RINGS;
-		bp->rx_nr_rings = channel->combined_count;
-		bp->tx_nr_rings_per_tc = channel->combined_count;
+		bp->rx_nr_rings = min_t(int, channel->combined_count,
+					max_rx_rings);
+		bp->tx_nr_rings_per_tc = min_t(int, channel->combined_count,
+					       max_tx_rings);
 	} else {
 		bp->flags &= ~BNXT_FLAG_SHARED_RINGS;
 		bp->rx_nr_rings = channel->rx_count;
@@ -1028,6 +1032,10 @@ static u32 bnxt_get_link(struct net_device *dev)
 	return bp->link_info.link_up;
 }
 
+static int bnxt_find_nvram_item(struct net_device *dev, u16 type, u16 ordinal,
+				u16 ext, u16 *index, u32 *item_length,
+				u32 *data_length);
+
 static int bnxt_flash_nvram(struct net_device *dev,
 			    u16 dir_type,
 			    u16 dir_ordinal,
@@ -1179,7 +1187,6 @@ static int bnxt_flash_firmware(struct net_device *dev,
 			   (unsigned long)calculated_crc);
 		return -EINVAL;
 	}
-	/* TODO: Validate digital signature (RSA-encrypted SHA-256 hash) here */
 	rc = bnxt_flash_nvram(dev, dir_type, BNX_DIR_ORDINAL_FIRST,
 			      0, 0, fw_data, fw_size);
 	if (rc == 0)	/* Firmware update successful */
@@ -1188,6 +1195,57 @@ static int bnxt_flash_firmware(struct net_device *dev,
 	return rc;
 }
 
+static int bnxt_flash_microcode(struct net_device *dev,
+				u16 dir_type,
+				const u8 *fw_data,
+				size_t fw_size)
+{
+	struct bnxt_ucode_trailer *trailer;
+	u32 calculated_crc;
+	u32 stored_crc;
+	int rc = 0;
+
+	if (fw_size < sizeof(struct bnxt_ucode_trailer)) {
+		netdev_err(dev, "Invalid microcode file size: %u\n",
+			   (unsigned int)fw_size);
+		return -EINVAL;
+	}
+	trailer = (struct bnxt_ucode_trailer *)(fw_data + (fw_size -
+						sizeof(*trailer)));
+	if (trailer->sig != cpu_to_le32(BNXT_UCODE_TRAILER_SIGNATURE)) {
+		netdev_err(dev, "Invalid microcode trailer signature: %08X\n",
+			   le32_to_cpu(trailer->sig));
+		return -EINVAL;
+	}
+	if (le16_to_cpu(trailer->dir_type) != dir_type) {
+		netdev_err(dev, "Expected microcode type: %d, read: %d\n",
+			   dir_type, le16_to_cpu(trailer->dir_type));
+		return -EINVAL;
+	}
+	if (le16_to_cpu(trailer->trailer_length) <
+		sizeof(struct bnxt_ucode_trailer)) {
+		netdev_err(dev, "Invalid microcode trailer length: %d\n",
+			   le16_to_cpu(trailer->trailer_length));
+		return -EINVAL;
+	}
+
+	/* Confirm the CRC32 checksum of the file: */
+	stored_crc = le32_to_cpu(*(__le32 *)(fw_data + fw_size -
+					     sizeof(stored_crc)));
+	calculated_crc = ~crc32(~0, fw_data, fw_size - sizeof(stored_crc));
+	if (calculated_crc != stored_crc) {
+		netdev_err(dev,
+			   "CRC32 (%08lX) does not match calculated: %08lX\n",
+			   (unsigned long)stored_crc,
+			   (unsigned long)calculated_crc);
+		return -EINVAL;
+	}
+	rc = bnxt_flash_nvram(dev, dir_type, BNX_DIR_ORDINAL_FIRST,
+			      0, 0, fw_data, fw_size);
+
+	return rc;
+}
+
 static bool bnxt_dir_type_is_ape_bin_format(u16 dir_type)
 {
 	switch (dir_type) {
@@ -1206,7 +1264,7 @@ static bool bnxt_dir_type_is_ape_bin_format(u16 dir_type)
 	return false;
 }
 
-static bool bnxt_dir_type_is_unprotected_exec_format(u16 dir_type)
+static bool bnxt_dir_type_is_other_exec_format(u16 dir_type)
 {
 	switch (dir_type) {
 	case BNX_DIR_TYPE_AVS:
@@ -1227,7 +1285,7 @@ static bool bnxt_dir_type_is_unprotected_exec_format(u16 dir_type)
 static bool bnxt_dir_type_is_executable(u16 dir_type)
 {
 	return bnxt_dir_type_is_ape_bin_format(dir_type) ||
-		bnxt_dir_type_is_unprotected_exec_format(dir_type);
+		bnxt_dir_type_is_other_exec_format(dir_type);
 }
 
 static int bnxt_flash_firmware_from_file(struct net_device *dev,
@@ -1237,10 +1295,6 @@ static int bnxt_flash_firmware_from_file(struct net_device *dev,
 	const struct firmware  *fw;
 	int			rc;
 
-	if (dir_type != BNX_DIR_TYPE_UPDATE &&
-	    bnxt_dir_type_is_executable(dir_type) == false)
-		return -EINVAL;
-
 	rc = request_firmware(&fw, filename, &dev->dev);
 	if (rc != 0) {
 		netdev_err(dev, "Error %d requesting firmware file: %s\n",
@@ -1249,6 +1303,8 @@ static int bnxt_flash_firmware_from_file(struct net_device *dev,
 	}
 	if (bnxt_dir_type_is_ape_bin_format(dir_type) == true)
 		rc = bnxt_flash_firmware(dev, dir_type, fw->data, fw->size);
+	else if (bnxt_dir_type_is_other_exec_format(dir_type) == true)
+		rc = bnxt_flash_microcode(dev, dir_type, fw->data, fw->size);
 	else
 		rc = bnxt_flash_nvram(dev, dir_type, BNX_DIR_ORDINAL_FIRST,
 				      0, 0, fw->data, fw->size);
@@ -1257,10 +1313,83 @@ static int bnxt_flash_firmware_from_file(struct net_device *dev,
 }
 
 static int bnxt_flash_package_from_file(struct net_device *dev,
-					char *filename)
+					char *filename, u32 install_type)
 {
-	netdev_err(dev, "packages are not yet supported\n");
-	return -EINVAL;
+	struct bnxt *bp = netdev_priv(dev);
+	struct hwrm_nvm_install_update_output *resp = bp->hwrm_cmd_resp_addr;
+	struct hwrm_nvm_install_update_input install = {0};
+	const struct firmware *fw;
+	u32 item_len;
+	u16 index;
+	int rc;
+
+	bnxt_hwrm_fw_set_time(bp);
+
+	if (bnxt_find_nvram_item(dev, BNX_DIR_TYPE_UPDATE,
+				 BNX_DIR_ORDINAL_FIRST, BNX_DIR_EXT_NONE,
+				 &index, &item_len, NULL) != 0) {
+		netdev_err(dev, "PKG update area not created in nvram\n");
+		return -ENOBUFS;
+	}
+
+	rc = request_firmware(&fw, filename, &dev->dev);
+	if (rc != 0) {
+		netdev_err(dev, "PKG error %d requesting file: %s\n",
+			   rc, filename);
+		return rc;
+	}
+
+	if (fw->size > item_len) {
+		netdev_err(dev, "PKG insufficient update area in nvram: %lu",
+			   (unsigned long)fw->size);
+		rc = -EFBIG;
+	} else {
+		dma_addr_t dma_handle;
+		u8 *kmem;
+		struct hwrm_nvm_modify_input modify = {0};
+
+		bnxt_hwrm_cmd_hdr_init(bp, &modify, HWRM_NVM_MODIFY, -1, -1);
+
+		modify.dir_idx = cpu_to_le16(index);
+		modify.len = cpu_to_le32(fw->size);
+
+		kmem = dma_alloc_coherent(&bp->pdev->dev, fw->size,
+					  &dma_handle, GFP_KERNEL);
+		if (!kmem) {
+			netdev_err(dev,
+				   "dma_alloc_coherent failure, length = %u\n",
+				   (unsigned int)fw->size);
+			rc = -ENOMEM;
+		} else {
+			memcpy(kmem, fw->data, fw->size);
+			modify.host_src_addr = cpu_to_le64(dma_handle);
+
+			rc = hwrm_send_message(bp, &modify, sizeof(modify),
+					       FLASH_PACKAGE_TIMEOUT);
+			dma_free_coherent(&bp->pdev->dev, fw->size, kmem,
+					  dma_handle);
+		}
+	}
+	release_firmware(fw);
+	if (rc)
+		return rc;
+
+	if ((install_type & 0xffff) == 0)
+		install_type >>= 16;
+	bnxt_hwrm_cmd_hdr_init(bp, &install, HWRM_NVM_INSTALL_UPDATE, -1, -1);
+	install.install_type = cpu_to_le32(install_type);
+
+	rc = hwrm_send_message(bp, &install, sizeof(install),
+			       INSTALL_PACKAGE_TIMEOUT);
+	if (rc)
+		return -EOPNOTSUPP;
+
+	if (resp->result) {
+		netdev_err(dev, "PKG install error = %d, problem_item = %d\n",
+			   (s8)resp->result, (int)resp->problem_item);
+		return -ENOPKG;
+	}
+	return 0;
 }
 
 static int bnxt_flash_device(struct net_device *dev,
@@ -1271,8 +1400,10 @@ static int bnxt_flash_device(struct net_device *dev,
 		return -EINVAL;
 	}
 
-	if (flash->region == ETHTOOL_FLASH_ALL_REGIONS)
-		return bnxt_flash_package_from_file(dev, flash->data);
+	if (flash->region == ETHTOOL_FLASH_ALL_REGIONS ||
+	    flash->region > 0xffff)
+		return bnxt_flash_package_from_file(dev, flash->data,
+						    flash->region);
 
 	return bnxt_flash_firmware_from_file(dev, flash->region, flash->data);
 }
@@ -1516,7 +1647,7 @@ static int bnxt_set_eeprom(struct net_device *dev,
 
 	/* Create or re-write an NVM item: */
 	if (bnxt_dir_type_is_executable(type) == true)
-		return -EINVAL;
+		return -EOPNOTSUPP;
 	ext = eeprom->magic & 0xffff;
 	ordinal = eeprom->offset >> 16;
 	attr = eeprom->offset & 0xffff;
@@ -1718,6 +1849,25 @@ static int bnxt_get_module_eeprom(struct net_device *dev,
 	return rc;
 }
 
+static int bnxt_nway_reset(struct net_device *dev)
+{
+	int rc = 0;
+
+	struct bnxt *bp = netdev_priv(dev);
+	struct bnxt_link_info *link_info = &bp->link_info;
+
+	if (!BNXT_SINGLE_PF(bp))
+		return -EOPNOTSUPP;
+
+	if (!(link_info->autoneg & BNXT_AUTONEG_SPEED))
+		return -EINVAL;
+
+	if (netif_running(dev))
+		rc = bnxt_hwrm_set_link_setting(bp, true, false);
+
+	return rc;
+}
+
 const struct ethtool_ops bnxt_ethtool_ops = {
 	.get_link_ksettings	= bnxt_get_link_ksettings,
 	.set_link_ksettings	= bnxt_set_link_ksettings,
@@ -1750,4 +1900,5 @@ const struct ethtool_ops bnxt_ethtool_ops = {
 	.set_eee		= bnxt_set_eee,
 	.get_module_info	= bnxt_get_module_info,
 	.get_module_eeprom	= bnxt_get_module_eeprom,
+	.nway_reset		= bnxt_nway_reset
 };
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_fw_hdr.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_fw_hdr.h
index 82bf44ab811b..cad30ddc6936 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_fw_hdr.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_fw_hdr.h
@@ -11,6 +11,7 @@
 #define __BNXT_FW_HDR_H__
 
 #define BNXT_FIRMWARE_BIN_SIGNATURE     0x1a4d4342	/* "BCM"+0x1a */
+#define BNXT_UCODE_TRAILER_SIGNATURE	0x726c7254	/* "Trlr" */
 
 enum SUPPORTED_FAMILY {
 	DEVICE_5702_3_4_FAMILY,		/* 0  - Denali, Vinson, K2 */
@@ -85,7 +86,7 @@ enum SUPPORTED_MEDIA {
 
 struct bnxt_fw_header {
 	__le32 signature;	/* constains the constant value of
-				 * BNXT_Firmware_Bin_Signatures
+				 * BNXT_FIRMWARE_BIN_SIGNATURE
 				 */
 	u8 flags;		/* reserved for ChiMP use */
 	u8 code_type;		/* enum SUPPORTED_CODE */
@@ -102,4 +103,17 @@ struct bnxt_fw_header {
 	u8 major_ver;
 };
 
+/* Microcode and pre-boot software/firmware trailer: */
+struct bnxt_ucode_trailer {
+	u8 rsa_sig[256];
+	__le16 flags;
+	u8 version_format;
+	u8 version_length;
+	u8 version[16];
+	__le16 dir_type;
+	__le16 trailer_length;
+	__le32 sig;		/* BNXT_UCODE_TRAILER_SIGNATURE */
+	__le32 chksum;		/* CRC-32 */
+};
+
 #endif
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
index 517567f6d651..04a96cc3498a 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
@@ -39,7 +39,7 @@ struct eject_cmpl {
 	__le16 type;
 	#define EJECT_CMPL_TYPE_MASK				    0x3fUL
 	#define EJECT_CMPL_TYPE_SFT				    0
-	#define EJECT_CMPL_TYPE_STAT_EJECT			   (0x1aUL << 0)
+	#define EJECT_CMPL_TYPE_STAT_EJECT			   0x1aUL
 	__le16 len;
 	__le32 opaque;
 	__le32 v;
@@ -52,7 +52,7 @@ struct hwrm_cmpl {
 	__le16 type;
 	#define HWRM_CMPL_TYPE_MASK				    0x3fUL
 	#define HWRM_CMPL_TYPE_SFT				    0
-	#define HWRM_CMPL_TYPE_HWRM_DONE			   (0x20UL << 0)
+	#define HWRM_CMPL_TYPE_HWRM_DONE			   0x20UL
 	__le16 sequence_id;
 	__le32 unused_1;
 	__le32 v;
@@ -65,7 +65,7 @@ struct hwrm_fwd_req_cmpl {
 	__le16 req_len_type;
 	#define HWRM_FWD_REQ_CMPL_TYPE_MASK			    0x3fUL
 	#define HWRM_FWD_REQ_CMPL_TYPE_SFT			    0
-	#define HWRM_FWD_REQ_CMPL_TYPE_HWRM_FWD_REQ		   (0x22UL << 0)
+	#define HWRM_FWD_REQ_CMPL_TYPE_HWRM_FWD_REQ		   0x22UL
 	#define HWRM_FWD_REQ_CMPL_REQ_LEN_MASK			    0xffc0UL
 	#define HWRM_FWD_REQ_CMPL_REQ_LEN_SFT			    6
 	__le16 source_id;
@@ -81,7 +81,7 @@ struct hwrm_fwd_resp_cmpl {
 	__le16 type;
 	#define HWRM_FWD_RESP_CMPL_TYPE_MASK			    0x3fUL
 	#define HWRM_FWD_RESP_CMPL_TYPE_SFT			    0
-	#define HWRM_FWD_RESP_CMPL_TYPE_HWRM_FWD_RESP		   (0x24UL << 0)
+	#define HWRM_FWD_RESP_CMPL_TYPE_HWRM_FWD_RESP		   0x24UL
 	__le16 source_id;
 	__le16 resp_len;
 	__le16 unused_1;
@@ -96,25 +96,26 @@ struct hwrm_async_event_cmpl {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_TYPE_MASK		    0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_TYPE_SFT			    0
-	#define HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT       (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT       0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE (0x0UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE    (0x1UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE  (0x2UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE  (0x3UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED (0x4UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED (0x5UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE (0x6UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE (0x7UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD   (0x10UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD     (0x11UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD     (0x20UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD       (0x21UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR		   (0x30UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE (0x31UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE (0x32UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE      (0x33UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR	   (0xffUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE 0x0UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE    0x1UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE  0x2UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE  0x3UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED 0x4UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED 0x5UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE 0x6UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE 0x7UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD   0x10UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD     0x11UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_FLR_PROC_CMPLT 0x12UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD     0x20UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD       0x21UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR		   0x30UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE 0x31UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE 0x32UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE      0x33UL
+	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR	   0xffUL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_V			    0x1UL
@@ -130,9 +131,9 @@ struct hwrm_async_event_cmpl_link_status_change {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_MASK 0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_SFT  0
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_ID_LINK_STATUS_CHANGE (0x0UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_ID_LINK_STATUS_CHANGE 0x0UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_V	    0x1UL
@@ -156,9 +157,9 @@ struct hwrm_async_event_cmpl_link_mtu_change {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_TYPE_MASK    0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_TYPE_SFT     0
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_EVENT_ID_LINK_MTU_CHANGE (0x1UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_EVENT_ID_LINK_MTU_CHANGE 0x1UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_V	    0x1UL
@@ -176,9 +177,9 @@ struct hwrm_async_event_cmpl_link_speed_change {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_TYPE_MASK  0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_TYPE_SFT   0
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_ID_LINK_SPEED_CHANGE (0x2UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_ID_LINK_SPEED_CHANGE 0x2UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_V	    0x1UL
@@ -200,8 +201,7 @@ struct hwrm_async_event_cmpl_link_speed_change {
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_40GB (0x190UL << 1)
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_50GB (0x1f4UL << 1)
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_100GB (0x3e8UL << 1)
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_10MB (0xffffUL << 1)
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_LAST    HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_10MB
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_LAST    HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_100GB
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_PORT_ID_MASK 0xffff0000UL
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_PORT_ID_SFT 16
 };
@@ -211,9 +211,9 @@ struct hwrm_async_event_cmpl_dcb_config_change {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_TYPE_MASK  0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_TYPE_SFT   0
-	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_ID_DCB_CONFIG_CHANGE (0x3UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_ID_DCB_CONFIG_CHANGE 0x3UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_V	    0x1UL
@@ -231,9 +231,9 @@ struct hwrm_async_event_cmpl_port_conn_not_allowed {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_MASK 0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_SFT 0
-	#define HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_ID_PORT_CONN_NOT_ALLOWED (0x4UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_ID_PORT_CONN_NOT_ALLOWED 0x4UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_V      0x1UL
@@ -258,9 +258,9 @@ struct hwrm_async_event_cmpl_link_speed_cfg_not_allowed {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_TYPE_MASK 0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_TYPE_SFT 0
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED (0x5UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED 0x5UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_V 0x1UL
@@ -278,9 +278,9 @@ struct hwrm_async_event_cmpl_link_speed_cfg_change {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_MASK 0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_SFT 0
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_ID_LINK_SPEED_CFG_CHANGE (0x6UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_ID_LINK_SPEED_CFG_CHANGE 0x6UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_V      0x1UL
@@ -300,9 +300,9 @@ struct hwrm_async_event_cmpl_func_drvr_unload {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_TYPE_MASK   0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_TYPE_SFT    0
-	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_EVENT_ID_FUNC_DRVR_UNLOAD (0x10UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_EVENT_ID_FUNC_DRVR_UNLOAD 0x10UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_V	    0x1UL
@@ -320,9 +320,9 @@ struct hwrm_async_event_cmpl_func_drvr_load {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_TYPE_MASK     0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_TYPE_SFT      0
-	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_EVENT_ID_FUNC_DRVR_LOAD (0x11UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_EVENT_ID_FUNC_DRVR_LOAD 0x11UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_V		    0x1UL
@@ -340,9 +340,9 @@ struct hwrm_async_event_cmpl_pf_drvr_unload {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_MASK     0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_SFT      0
-	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_ID_PF_DRVR_UNLOAD (0x20UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_ID_PF_DRVR_UNLOAD 0x20UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_V		    0x1UL
@@ -362,9 +362,9 @@ struct hwrm_async_event_cmpl_pf_drvr_load {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_LOAD_TYPE_MASK       0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_LOAD_TYPE_SFT	    0
-	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_LOAD_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_LOAD_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_LOAD_EVENT_ID_PF_DRVR_LOAD (0x21UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_LOAD_EVENT_ID_PF_DRVR_LOAD 0x21UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_PF_DRVR_LOAD_V		    0x1UL
@@ -384,9 +384,9 @@ struct hwrm_async_event_cmpl_vf_flr {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_TYPE_MASK		    0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_TYPE_SFT		    0
-	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_EVENT_ID_VF_FLR      (0x30UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_EVENT_ID_VF_FLR      0x30UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_VF_FLR_V			    0x1UL
@@ -404,9 +404,9 @@ struct hwrm_async_event_cmpl_vf_mac_addr_change {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_TYPE_MASK 0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_TYPE_SFT  0
-	#define HWRM_ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_EVENT_ID_VF_MAC_ADDR_CHANGE (0x31UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_EVENT_ID_VF_MAC_ADDR_CHANGE 0x31UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_V	    0x1UL
@@ -424,9 +424,9 @@ struct hwrm_async_event_cmpl_pf_vf_comm_status_change {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_TYPE_MASK 0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_TYPE_SFT 0
-	#define HWRM_ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_EVENT_ID_PF_VF_COMM_STATUS_CHANGE (0x32UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_EVENT_ID_PF_VF_COMM_STATUS_CHANGE 0x32UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_V   0x1UL
@@ -443,9 +443,9 @@ struct hwrm_async_event_cmpl_vf_cfg_change {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_MASK      0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_SFT       0
-	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_VF_CFG_CHANGE (0x33UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_VF_CFG_CHANGE 0x33UL
 	__le32 event_data2;
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_V		    0x1UL
@@ -465,15 +465,15 @@ struct hwrm_async_event_cmpl_hwrm_error {
 	__le16 type;
 	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_MASK	    0x3fUL
 	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_SFT	    0
-	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_HWRM_ASYNC_EVENT (0x2eUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_HWRM_ASYNC_EVENT 0x2eUL
 	__le16 event_id;
-	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_ID_HWRM_ERROR (0xffUL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_ID_HWRM_ERROR 0xffUL
 	__le32 event_data2;
 	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_MASK 0xffUL
 	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_SFT 0
-	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_WARNING (0x0UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_NONFATAL (0x1UL << 0)
-	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_FATAL (0x2UL << 0)
+	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_WARNING 0x0UL
+	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_NONFATAL 0x1UL
+	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_FATAL 0x2UL
 	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_LAST    HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_FATAL
 	u8 opaque_v;
 	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_V		    0x1UL
@@ -485,12 +485,12 @@ struct hwrm_async_event_cmpl_hwrm_error {
 	#define HWRM_ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA1_TIMESTAMP 0x1UL
 };
 
-/* HW Resource Manager Specification 1.3.0 */
+/* HW Resource Manager Specification 1.5.1 */
 #define HWRM_VERSION_MAJOR	1
-#define HWRM_VERSION_MINOR	3
-#define HWRM_VERSION_UPDATE	0
+#define HWRM_VERSION_MINOR	5
+#define HWRM_VERSION_UPDATE	1
 
-#define HWRM_VERSION_STR	"1.3.0"
+#define HWRM_VERSION_STR	"1.5.1"
 /*
  * Following is the signature for HWRM message field that indicates not
  * applicable (All F's). Need to cast it the size of the field if needed.
@@ -556,8 +556,8 @@ struct cmd_nums {
 	#define HWRM_QUEUE_QPORTCFG				   (0x30UL)
 	#define HWRM_QUEUE_QCFG				   (0x31UL)
 	#define HWRM_QUEUE_CFG					   (0x32UL)
-	#define HWRM_QUEUE_BUFFERS_QCFG			   (0x33UL)
-	#define HWRM_QUEUE_BUFFERS_CFG				   (0x34UL)
+	#define RESERVED2					   (0x33UL)
+	#define RESERVED3					   (0x34UL)
 	#define HWRM_QUEUE_PFCENABLE_QCFG			   (0x35UL)
 	#define HWRM_QUEUE_PFCENABLE_CFG			   (0x36UL)
 	#define HWRM_QUEUE_PRI2COS_QCFG			   (0x37UL)
@@ -574,6 +574,7 @@ struct cmd_nums {
 	#define HWRM_VNIC_RSS_QCFG				   (0x47UL)
 	#define HWRM_VNIC_PLCMODES_CFG				   (0x48UL)
 	#define HWRM_VNIC_PLCMODES_QCFG			   (0x49UL)
+	#define HWRM_VNIC_QCAPS				   (0x4aUL)
 	#define HWRM_RING_ALLOC				   (0x50UL)
 	#define HWRM_RING_FREE					   (0x51UL)
 	#define HWRM_RING_CMPL_RING_QAGGINT_PARAMS		   (0x52UL)
@@ -581,13 +582,15 @@ struct cmd_nums {
 	#define HWRM_RING_RESET				   (0x5eUL)
 	#define HWRM_RING_GRP_ALLOC				   (0x60UL)
 	#define HWRM_RING_GRP_FREE				   (0x61UL)
+	#define RESERVED5					   (0x64UL)
+	#define RESERVED6					   (0x65UL)
 	#define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC			   (0x70UL)
 	#define HWRM_VNIC_RSS_COS_LB_CTX_FREE			   (0x71UL)
 	#define HWRM_CFA_L2_FILTER_ALLOC			   (0x90UL)
 	#define HWRM_CFA_L2_FILTER_FREE			   (0x91UL)
 	#define HWRM_CFA_L2_FILTER_CFG				   (0x92UL)
 	#define HWRM_CFA_L2_SET_RX_MASK			   (0x93UL)
-	#define RESERVED3					   (0x94UL)
+	#define RESERVED4					   (0x94UL)
 	#define HWRM_CFA_TUNNEL_FILTER_ALLOC			   (0x95UL)
 	#define HWRM_CFA_TUNNEL_FILTER_FREE			   (0x96UL)
 	#define HWRM_CFA_ENCAP_RECORD_ALLOC			   (0x97UL)
@@ -607,6 +610,8 @@ struct cmd_nums {
 	#define HWRM_STAT_CTX_CLR_STATS			   (0xb3UL)
 	#define HWRM_FW_RESET					   (0xc0UL)
 	#define HWRM_FW_QSTATUS				   (0xc1UL)
+	#define HWRM_FW_SET_TIME				   (0xc8UL)
+	#define HWRM_FW_GET_TIME				   (0xc9UL)
 	#define HWRM_EXEC_FWD_RESP				   (0xd0UL)
 	#define HWRM_REJECT_FWD_RESP				   (0xd1UL)
 	#define HWRM_FWD_RESP					   (0xd2UL)
@@ -615,11 +620,13 @@ struct cmd_nums {
 	#define HWRM_WOL_FILTER_ALLOC				   (0xf0UL)
 	#define HWRM_WOL_FILTER_FREE				   (0xf1UL)
 	#define HWRM_WOL_FILTER_QCFG				   (0xf2UL)
+	#define HWRM_WOL_REASON_QCFG				   (0xf3UL)
 	#define HWRM_DBG_READ_DIRECT				   (0xff10UL)
 	#define HWRM_DBG_READ_INDIRECT				   (0xff11UL)
 	#define HWRM_DBG_WRITE_DIRECT				   (0xff12UL)
 	#define HWRM_DBG_WRITE_INDIRECT			   (0xff13UL)
 	#define HWRM_DBG_DUMP					   (0xff14UL)
+	#define HWRM_NVM_INSTALL_UPDATE			   (0xfff3UL)
 	#define HWRM_NVM_MODIFY				   (0xfff4UL)
 	#define HWRM_NVM_VERIFY_UPDATE				   (0xfff5UL)
 	#define HWRM_NVM_GET_DEV_INFO				   (0xfff6UL)
@@ -824,7 +831,9 @@ struct hwrm_ver_get_output {
 	u8 netctrl_fw_min;
 	u8 netctrl_fw_bld;
 	u8 netctrl_fw_rsvd;
-	__le32 reserved1;
+	__le32 dev_caps_cfg;
+	#define VER_GET_RESP_DEV_CAPS_CFG_SECURE_FW_UPD_SUPPORTED  0x1UL
+	#define VER_GET_RESP_DEV_CAPS_CFG_FW_DCBX_AGENT_SUPPORTED  0x2UL
 	u8 roce_fw_maj;
 	u8 roce_fw_min;
 	u8 roce_fw_bld;
@@ -839,9 +848,9 @@ struct hwrm_ver_get_output {
 	u8 chip_metal;
 	u8 chip_bond_id;
 	u8 chip_platform_type;
-	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_ASIC		   (0x0UL << 0)
-	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_FPGA		   (0x1UL << 0)
-	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_PALLADIUM	   (0x2UL << 0)
+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_ASIC		   0x0UL
+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_FPGA		   0x1UL
+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_PALLADIUM	   0x2UL
 	__le16 max_req_win_len;
 	__le16 max_resp_len;
 	__le16 def_req_timeout;
@@ -863,10 +872,10 @@ struct hwrm_func_reset_input {
 	#define FUNC_RESET_REQ_ENABLES_VF_ID_VALID		    0x1UL
 	__le16 vf_id;
 	u8 func_reset_level;
-	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETALL	   (0x0UL << 0)
-	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETME	   (0x1UL << 0)
-	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETCHILDREN     (0x2UL << 0)
-	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETVF	   (0x3UL << 0)
+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETALL	   0x0UL
+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETME	   0x1UL
+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETCHILDREN     0x2UL
+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETVF	   0x3UL
 	u8 unused_0;
 };
 
@@ -1028,6 +1037,10 @@ struct hwrm_func_qcaps_output {
 	#define FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED	    0x10UL
 	#define FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED       0x20UL
 	#define FUNC_QCAPS_RESP_FLAGS_WOL_BMP_SUPPORTED	    0x40UL
+	#define FUNC_QCAPS_RESP_FLAGS_TX_RING_RL_SUPPORTED	    0x80UL
+	#define FUNC_QCAPS_RESP_FLAGS_TX_BW_CFG_SUPPORTED	    0x100UL
+	#define FUNC_QCAPS_RESP_FLAGS_VF_TX_RING_RL_SUPPORTED      0x200UL
+	#define FUNC_QCAPS_RESP_FLAGS_VF_BW_CFG_SUPPORTED	    0x400UL
 	u8 mac_address[6];
 	__le16 max_rsscos_ctx;
 	__le16 max_cmpl_rings;
@@ -1047,9 +1060,8 @@ struct hwrm_func_qcaps_output {
 	__le32 max_mcast_filters;
 	__le32 max_flow_id;
 	__le32 max_hw_ring_grps;
+	__le16 max_sp_tx_rings;
 	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
 	u8 valid;
 };
 
@@ -1077,6 +1089,7 @@ struct hwrm_func_qcfg_output {
 	__le16 flags;
 	#define FUNC_QCFG_RESP_FLAGS_OOB_WOL_MAGICPKT_ENABLED      0x1UL
 	#define FUNC_QCFG_RESP_FLAGS_OOB_WOL_BMP_ENABLED	    0x2UL
+	#define FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED	    0x4UL
 	u8 mac_address[6];
 	__le16 pci_id;
 	__le16 alloc_rsscos_ctx;
@@ -1089,29 +1102,46 @@ struct hwrm_func_qcfg_output {
 	__le16 mru;
 	__le16 stat_ctx_id;
 	u8 port_partition_type;
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_SPF		   (0x0UL << 0)
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_MPFS	   (0x1UL << 0)
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0	   (0x2UL << 0)
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5	   (0x3UL << 0)
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0	   (0x4UL << 0)
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN	   (0xffUL << 0)
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_SPF		   0x0UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_MPFS	   0x1UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0	   0x2UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5	   0x3UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0	   0x4UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN	   0xffUL
 	u8 unused_0;
 	__le16 dflt_vnic_id;
 	u8 unused_1;
 	u8 unused_2;
 	__le32 min_bw;
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_MASK		    0xfffffffUL
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_SFT		    0
+	#define FUNC_QCFG_RESP_MIN_BW_RSVD			    0x10000000UL
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MASK	    0xe0000000UL
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_SFT	    29
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MBPS	   (0x0UL << 29)
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_LAST    FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 max_bw;
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_MASK		    0xfffffffUL
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_SFT		    0
+	#define FUNC_QCFG_RESP_MAX_BW_RSVD			    0x10000000UL
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MASK	    0xe0000000UL
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_SFT	    29
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MBPS	   (0x0UL << 29)
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_LAST    FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 evb_mode;
-	#define FUNC_QCFG_RESP_EVB_MODE_NO_EVB			   (0x0UL << 0)
-	#define FUNC_QCFG_RESP_EVB_MODE_VEB			   (0x1UL << 0)
-	#define FUNC_QCFG_RESP_EVB_MODE_VEPA			   (0x2UL << 0)
+	#define FUNC_QCFG_RESP_EVB_MODE_NO_EVB			   0x0UL
+	#define FUNC_QCFG_RESP_EVB_MODE_VEB			   0x1UL
+	#define FUNC_QCFG_RESP_EVB_MODE_VEPA			   0x2UL
 	u8 unused_3;
-	__le16 unused_4;
+	__le16 alloc_vfs;
 	__le32 alloc_mcast_filters;
 	__le32 alloc_hw_ring_grps;
-	u8 unused_5;
-	u8 unused_6;
-	u8 unused_7;
+	__le16 alloc_sp_tx_rings;
+	u8 unused_4;
 	u8 valid;
 };
 
@@ -1171,18 +1201,36 @@ struct hwrm_func_cfg_input {
 	__le16 dflt_vlan;
 	__be32 dflt_ip_addr[4];
 	__le32 min_bw;
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_MASK		    0xfffffffUL
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_SFT		    0
+	#define FUNC_CFG_REQ_MIN_BW_RSVD			    0x10000000UL
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_MASK		    0xe0000000UL
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_SFT		    29
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_MBPS		   (0x0UL << 29)
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_PERCENT1_100    (0x1UL << 29)
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_INVALID	   (0x7UL << 29)
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_LAST    FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 max_bw;
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_MASK		    0xfffffffUL
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_SFT		    0
+	#define FUNC_CFG_REQ_MAX_BW_RSVD			    0x10000000UL
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_MASK		    0xe0000000UL
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_SFT		    29
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_MBPS		   (0x0UL << 29)
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_PERCENT1_100    (0x1UL << 29)
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_INVALID	   (0x7UL << 29)
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_LAST    FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_INVALID
 	__le16 async_event_cr;
 	u8 vlan_antispoof_mode;
-	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_NOCHECK	   (0x0UL << 0)
-	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN    (0x1UL << 0)
-	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_IF_VLANDNE (0x2UL << 0)
-	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_OR_OVERRIDE_VLAN (0x3UL << 0)
+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_NOCHECK	   0x0UL
+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN    0x1UL
+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_IF_VLANDNE 0x2UL
+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_OR_OVERRIDE_VLAN 0x3UL
 	u8 allowed_vlan_pris;
 	u8 evb_mode;
-	#define FUNC_CFG_REQ_EVB_MODE_NO_EVB			   (0x0UL << 0)
-	#define FUNC_CFG_REQ_EVB_MODE_VEB			   (0x1UL << 0)
-	#define FUNC_CFG_REQ_EVB_MODE_VEPA			   (0x2UL << 0)
+	#define FUNC_CFG_REQ_EVB_MODE_NO_EVB			   0x0UL
+	#define FUNC_CFG_REQ_EVB_MODE_VEB			   0x1UL
+	#define FUNC_CFG_REQ_EVB_MODE_VEPA			   0x2UL
 	u8 unused_2;
 	__le16 num_mcast_filters;
 };
@@ -1341,16 +1389,16 @@ struct hwrm_func_drv_rgtr_input {
 	#define FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD		    0x8UL
 	#define FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD	    0x10UL
 	__le16 os_type;
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_UNKNOWN		   (0x0UL << 0)
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_OTHER		   (0x1UL << 0)
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_MSDOS		   (0xeUL << 0)
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WINDOWS		   (0x12UL << 0)
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_SOLARIS		   (0x1dUL << 0)
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX		   (0x24UL << 0)
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_FREEBSD		   (0x2aUL << 0)
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_ESXI			   (0x68UL << 0)
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN864		   (0x73UL << 0)
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN2012R2		   (0x74UL << 0)
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_UNKNOWN		   0x0UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_OTHER		   0x1UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_MSDOS		   0xeUL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WINDOWS		   0x12UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_SOLARIS		   0x1dUL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX		   0x24UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_FREEBSD		   0x2aUL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_ESXI			   0x68UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN864		   0x73UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN2012R2		   0x74UL
 	u8 ver_maj;
 	u8 ver_min;
 	u8 ver_upd;
@@ -1415,13 +1463,13 @@ struct hwrm_func_buf_rgtr_input {
 	__le16 vf_id;
 	__le16 req_buf_num_pages;
 	__le16 req_buf_page_size;
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_16B	   (0x4UL << 0)
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4K		   (0xcUL << 0)
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_8K		   (0xdUL << 0)
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_64K	   (0x10UL << 0)
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_2M		   (0x15UL << 0)
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4M		   (0x16UL << 0)
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_1G		   (0x1eUL << 0)
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_16B	   0x4UL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4K		   0xcUL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_8K		   0xdUL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_64K	   0x10UL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_2M		   0x15UL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4M		   0x16UL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_1G		   0x1eUL
 	__le16 req_buf_len;
 	__le16 resp_buf_len;
 	u8 unused_0;
@@ -1473,16 +1521,16 @@ struct hwrm_func_drv_qver_output {
 	__le16 seq_id;
 	__le16 resp_len;
 	__le16 os_type;
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_UNKNOWN		   (0x0UL << 0)
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_OTHER		   (0x1UL << 0)
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_MSDOS		   (0xeUL << 0)
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_WINDOWS		   (0x12UL << 0)
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_SOLARIS		   (0x1dUL << 0)
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_LINUX		   (0x24UL << 0)
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_FREEBSD		   (0x2aUL << 0)
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_ESXI		   (0x68UL << 0)
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN864		   (0x73UL << 0)
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN2012R2		   (0x74UL << 0)
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_UNKNOWN		   0x0UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_OTHER		   0x1UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_MSDOS		   0xeUL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_WINDOWS		   0x12UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_SOLARIS		   0x1dUL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_LINUX		   0x24UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_FREEBSD		   0x2aUL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_ESXI		   0x68UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN864		   0x73UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN2012R2		   0x74UL
 	u8 ver_maj;
 	u8 ver_min;
 	u8 ver_upd;
@@ -1528,44 +1576,44 @@ struct hwrm_port_phy_cfg_input {
 	#define PORT_PHY_CFG_REQ_ENABLES_TX_LPI_TIMER		    0x400UL
 	__le16 port_id;
 	__le16 force_link_speed;
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100MB	   (0x1UL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_1GB		   (0xaUL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2GB		   (0x14UL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2_5GB	   (0x19UL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB		   (0x64UL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_20GB		   (0xc8UL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB		   (0xfaUL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB		   (0x190UL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB		   (0x1f4UL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB	   (0x3e8UL << 0)
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10MB		   (0xffffUL << 0)
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100MB	   0x1UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_1GB		   0xaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2GB		   0x14UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2_5GB	   0x19UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB		   0x64UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_20GB		   0xc8UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB		   0xfaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB		   0x190UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB		   0x1f4UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB	   0x3e8UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10MB		   0xffffUL
 	u8 auto_mode;
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_NONE		   (0x0UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_ALL_SPEEDS		   (0x1UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_SPEED		   (0x2UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_OR_BELOW	   (0x3UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK		   (0x4UL << 0)
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_NONE		   0x0UL
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_ALL_SPEEDS		   0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_SPEED		   0x2UL
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_OR_BELOW	   0x3UL
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK		   0x4UL
 	u8 auto_duplex;
-	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_HALF		   (0x0UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_FULL		   (0x1UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH		   (0x2UL << 0)
+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_HALF		   0x0UL
+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_FULL		   0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH		   0x2UL
 	u8 auto_pause;
 	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_TX			    0x1UL
 	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_RX			    0x2UL
 	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE	    0x4UL
 	u8 unused_0;
 	__le16 auto_link_speed;
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100MB		   (0x1UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_1GB		   (0xaUL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2GB		   (0x14UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2_5GB		   (0x19UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10GB		   (0x64UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_20GB		   (0xc8UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_25GB		   (0xfaUL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_40GB		   (0x190UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_50GB		   (0x1f4UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100GB		   (0x3e8UL << 0)
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10MB		   (0xffffUL << 0)
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100MB		   0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_1GB		   0xaUL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2GB		   0x14UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2_5GB		   0x19UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10GB		   0x64UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_20GB		   0xc8UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_25GB		   0xfaUL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_40GB		   0x190UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_50GB		   0x1f4UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100GB		   0x3e8UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10MB		   0xffffUL
 	__le16 auto_link_speed_mask;
 	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MBHD      0x1UL
 	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MB	    0x2UL
@@ -1582,12 +1630,12 @@ struct hwrm_port_phy_cfg_input {
 	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MBHD       0x1000UL
 	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MB	    0x2000UL
 	u8 wirespeed;
-	#define PORT_PHY_CFG_REQ_WIRESPEED_OFF			   (0x0UL << 0)
-	#define PORT_PHY_CFG_REQ_WIRESPEED_ON			   (0x1UL << 0)
+	#define PORT_PHY_CFG_REQ_WIRESPEED_OFF			   0x0UL
+	#define PORT_PHY_CFG_REQ_WIRESPEED_ON			   0x1UL
 	u8 lpbk;
-	#define PORT_PHY_CFG_REQ_LPBK_NONE			   (0x0UL << 0)
-	#define PORT_PHY_CFG_REQ_LPBK_LOCAL			   (0x1UL << 0)
-	#define PORT_PHY_CFG_REQ_LPBK_REMOTE			   (0x2UL << 0)
+	#define PORT_PHY_CFG_REQ_LPBK_NONE			   0x0UL
+	#define PORT_PHY_CFG_REQ_LPBK_LOCAL			   0x1UL
+	#define PORT_PHY_CFG_REQ_LPBK_REMOTE			   0x2UL
 	u8 force_pause;
 	#define PORT_PHY_CFG_REQ_FORCE_PAUSE_TX		    0x1UL
 	#define PORT_PHY_CFG_REQ_FORCE_PAUSE_RX		    0x2UL
@@ -1641,25 +1689,25 @@ struct hwrm_port_phy_qcfg_output {
 	__le16 seq_id;
 	__le16 resp_len;
 	u8 link;
-	#define PORT_PHY_QCFG_RESP_LINK_NO_LINK		   (0x0UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SIGNAL			   (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_LINK			   (0x2UL << 0)
+	#define PORT_PHY_QCFG_RESP_LINK_NO_LINK		   0x0UL
+	#define PORT_PHY_QCFG_RESP_LINK_SIGNAL			   0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_LINK			   0x2UL
 	u8 unused_0;
 	__le16 link_speed;
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100MB		   (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_1GB		   (0xaUL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2GB		   (0x14UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2_5GB		   (0x19UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10GB		   (0x64UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_20GB		   (0xc8UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_25GB		   (0xfaUL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_40GB		   (0x190UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_50GB		   (0x1f4UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100GB		   (0x3e8UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10MB		   (0xffffUL << 0)
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100MB		   0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_1GB		   0xaUL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2GB		   0x14UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2_5GB		   0x19UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10GB		   0x64UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_20GB		   0xc8UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_25GB		   0xfaUL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_40GB		   0x190UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_50GB		   0x1f4UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100GB		   0x3e8UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10MB		   0xffffUL
 	u8 duplex;
-	#define PORT_PHY_QCFG_RESP_DUPLEX_HALF			   (0x0UL << 0)
-	#define PORT_PHY_QCFG_RESP_DUPLEX_FULL			   (0x1UL << 0)
+	#define PORT_PHY_QCFG_RESP_DUPLEX_HALF			   0x0UL
+	#define PORT_PHY_QCFG_RESP_DUPLEX_FULL			   0x1UL
 	u8 pause;
 	#define PORT_PHY_QCFG_RESP_PAUSE_TX			    0x1UL
 	#define PORT_PHY_QCFG_RESP_PAUSE_RX			    0x2UL
@@ -1679,39 +1727,39 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10MBHD	    0x1000UL
 	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10MB		    0x2000UL
 	__le16 force_link_speed;
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100MB	   (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_1GB	   (0xaUL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2GB	   (0x14UL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2_5GB	   (0x19UL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10GB	   (0x64UL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_20GB	   (0xc8UL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_25GB	   (0xfaUL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_40GB	   (0x190UL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_50GB	   (0x1f4UL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100GB	   (0x3e8UL << 0)
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10MB	   (0xffffUL << 0)
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100MB	   0x1UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_1GB	   0xaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2GB	   0x14UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2_5GB	   0x19UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10GB	   0x64UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_20GB	   0xc8UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_25GB	   0xfaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_40GB	   0x190UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_50GB	   0x1f4UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100GB	   0x3e8UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10MB	   0xffffUL
 	u8 auto_mode;
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_NONE		   (0x0UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ALL_SPEEDS	   (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_SPEED		   (0x2UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_OR_BELOW	   (0x3UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_SPEED_MASK	   (0x4UL << 0)
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_NONE		   0x0UL
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ALL_SPEEDS	   0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_SPEED		   0x2UL
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_OR_BELOW	   0x3UL
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_SPEED_MASK	   0x4UL
 	u8 auto_pause;
 	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_TX		    0x1UL
 	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_RX		    0x2UL
 	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_AUTONEG_PAUSE	    0x4UL
 	__le16 auto_link_speed;
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100MB	   (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_1GB		   (0xaUL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2GB		   (0x14UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2_5GB	   (0x19UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10GB	   (0x64UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_20GB	   (0xc8UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_25GB	   (0xfaUL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_40GB	   (0x190UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_50GB	   (0x1f4UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100GB	   (0x3e8UL << 0)
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10MB	   (0xffffUL << 0)
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100MB	   0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_1GB		   0xaUL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2GB		   0x14UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2_5GB	   0x19UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10GB	   0x64UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_20GB	   0xc8UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_25GB	   0xfaUL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_40GB	   0x190UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_50GB	   0x1f4UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100GB	   0x3e8UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10MB	   0xffffUL
 	__le16 auto_link_speed_mask;
 	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100MBHD    0x1UL
 	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100MB      0x2UL
@@ -1728,46 +1776,46 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10MBHD     0x1000UL
 	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10MB       0x2000UL
 	u8 wirespeed;
-	#define PORT_PHY_QCFG_RESP_WIRESPEED_OFF		   (0x0UL << 0)
-	#define PORT_PHY_QCFG_RESP_WIRESPEED_ON		   (0x1UL << 0)
+	#define PORT_PHY_QCFG_RESP_WIRESPEED_OFF		   0x0UL
+	#define PORT_PHY_QCFG_RESP_WIRESPEED_ON		   0x1UL
 	u8 lpbk;
-	#define PORT_PHY_QCFG_RESP_LPBK_NONE			   (0x0UL << 0)
-	#define PORT_PHY_QCFG_RESP_LPBK_LOCAL			   (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_LPBK_REMOTE			   (0x2UL << 0)
+	#define PORT_PHY_QCFG_RESP_LPBK_NONE			   0x0UL
+	#define PORT_PHY_QCFG_RESP_LPBK_LOCAL			   0x1UL
+	#define PORT_PHY_QCFG_RESP_LPBK_REMOTE			   0x2UL
 	u8 force_pause;
 	#define PORT_PHY_QCFG_RESP_FORCE_PAUSE_TX		    0x1UL
 	#define PORT_PHY_QCFG_RESP_FORCE_PAUSE_RX		    0x2UL
 	u8 module_status;
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NONE		   (0x0UL << 0)
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX	   (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG       (0x2UL << 0)
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN	   (0x3UL << 0)
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTINSERTED      (0x4UL << 0)
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTAPPLICABLE    (0xffUL << 0)
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NONE		   0x0UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX	   0x1UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG       0x2UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN	   0x3UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTINSERTED      0x4UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTAPPLICABLE    0xffUL
 	__le32 preemphasis;
 	u8 phy_maj;
 	u8 phy_min;
 	u8 phy_bld;
 	u8 phy_type;
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_UNKNOWN		   (0x0UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASECR		   (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR4		   (0x2UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASELR		   (0x3UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASESR		   (0x4UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR2		   (0x5UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKX		   (0x6UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR		   (0x7UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASET		   (0x8UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASETE		   (0x9UL << 0)
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_SGMIIEXTPHY	   (0xaUL << 0)
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_UNKNOWN		   0x0UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASECR		   0x1UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR4		   0x2UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASELR		   0x3UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASESR		   0x4UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR2		   0x5UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKX		   0x6UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR		   0x7UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASET		   0x8UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASETE		   0x9UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_SGMIIEXTPHY	   0xaUL
 	u8 media_type;
-	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_UNKNOWN		   (0x0UL << 0)
-	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_TP		   (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_DAC		   (0x2UL << 0)
-	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_FIBRE		   (0x3UL << 0)
+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_UNKNOWN		   0x0UL
+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_TP		   0x1UL
+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_DAC		   0x2UL
+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_FIBRE		   0x3UL
 	u8 xcvr_pkg_type;
-	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_INTERNAL    (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_EXTERNAL    (0x2UL << 0)
+	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_INTERNAL    0x1UL
+	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_EXTERNAL    0x2UL
 	u8 eee_config_phy_addr;
 	#define PORT_PHY_QCFG_RESP_PHY_ADDR_MASK		    0x1fUL
 	#define PORT_PHY_QCFG_RESP_PHY_ADDR_SFT		    0
@@ -1796,11 +1844,11 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10MBHD  0x1000UL
 	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10MB    0x2000UL
 	u8 link_partner_adv_auto_mode;
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_NONE (0x0UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ALL_SPEEDS (0x1UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ONE_SPEED (0x2UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ONE_OR_BELOW (0x3UL << 0)
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK (0x4UL << 0)
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_NONE 0x0UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ALL_SPEEDS 0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ONE_SPEED 0x2UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ONE_OR_BELOW 0x3UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK 0x4UL
 	u8 link_partner_adv_pause;
 	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_PAUSE_TX       0x1UL
 	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_PAUSE_RX       0x2UL
@@ -1859,7 +1907,7 @@ struct hwrm_port_mac_cfg_input {
 	__le64 resp_addr;
 	__le32 flags;
 	#define PORT_MAC_CFG_REQ_FLAGS_MATCH_LINK		    0x1UL
-	#define PORT_MAC_CFG_REQ_FLAGS_COS_ASSIGNMENT_ENABLE       0x2UL
+	#define PORT_MAC_CFG_REQ_FLAGS_VLAN_PRI2COS_ENABLE	    0x2UL
 	#define PORT_MAC_CFG_REQ_FLAGS_TUNNEL_PRI2COS_ENABLE       0x4UL
 	#define PORT_MAC_CFG_REQ_FLAGS_IP_DSCP2COS_ENABLE	    0x8UL
 	#define PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_ENABLE    0x10UL
@@ -1868,28 +1916,50 @@ struct hwrm_port_mac_cfg_input {
 	#define PORT_MAC_CFG_REQ_FLAGS_PTP_TX_TS_CAPTURE_DISABLE   0x80UL
 	#define PORT_MAC_CFG_REQ_FLAGS_OOB_WOL_ENABLE		    0x100UL
 	#define PORT_MAC_CFG_REQ_FLAGS_OOB_WOL_DISABLE		    0x200UL
+	#define PORT_MAC_CFG_REQ_FLAGS_VLAN_PRI2COS_DISABLE	    0x400UL
+	#define PORT_MAC_CFG_REQ_FLAGS_TUNNEL_PRI2COS_DISABLE      0x800UL
+	#define PORT_MAC_CFG_REQ_FLAGS_IP_DSCP2COS_DISABLE	    0x1000UL
 	__le32 enables;
 	#define PORT_MAC_CFG_REQ_ENABLES_IPG			    0x1UL
 	#define PORT_MAC_CFG_REQ_ENABLES_LPBK			    0x2UL
-	#define PORT_MAC_CFG_REQ_ENABLES_IVLAN_PRI2COS_MAP_PRI     0x4UL
-	#define PORT_MAC_CFG_REQ_ENABLES_LCOS_MAP_PRI		    0x8UL
+	#define PORT_MAC_CFG_REQ_ENABLES_VLAN_PRI2COS_MAP_PRI      0x4UL
+	#define PORT_MAC_CFG_REQ_ENABLES_RESERVED1		    0x8UL
 	#define PORT_MAC_CFG_REQ_ENABLES_TUNNEL_PRI2COS_MAP_PRI    0x10UL
 	#define PORT_MAC_CFG_REQ_ENABLES_DSCP2COS_MAP_PRI	    0x20UL
 	#define PORT_MAC_CFG_REQ_ENABLES_RX_TS_CAPTURE_PTP_MSG_TYPE 0x40UL
 	#define PORT_MAC_CFG_REQ_ENABLES_TX_TS_CAPTURE_PTP_MSG_TYPE 0x80UL
+	#define PORT_MAC_CFG_REQ_ENABLES_COS_FIELD_CFG		    0x100UL
 	__le16 port_id;
 	u8 ipg;
 	u8 lpbk;
-	#define PORT_MAC_CFG_REQ_LPBK_NONE			   (0x0UL << 0)
-	#define PORT_MAC_CFG_REQ_LPBK_LOCAL			   (0x1UL << 0)
-	#define PORT_MAC_CFG_REQ_LPBK_REMOTE			   (0x2UL << 0)
-	u8 ivlan_pri2cos_map_pri;
-	u8 lcos_map_pri;
+	#define PORT_MAC_CFG_REQ_LPBK_NONE			   0x0UL
+	#define PORT_MAC_CFG_REQ_LPBK_LOCAL			   0x1UL
+	#define PORT_MAC_CFG_REQ_LPBK_REMOTE			   0x2UL
+	u8 vlan_pri2cos_map_pri;
+	u8 reserved1;
 	u8 tunnel_pri2cos_map_pri;
 	u8 dscp2pri_map_pri;
 	__le16 rx_ts_capture_ptp_msg_type;
 	__le16 tx_ts_capture_ptp_msg_type;
-	__le32 unused_0;
+	u8 cos_field_cfg;
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_RSVD1		    0x1UL
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_MASK   0x6UL
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_SFT    1
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_INNERMOST (0x0UL << 1)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_OUTER (0x1UL << 1)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_OUTERMOST (0x2UL << 1)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED (0x3UL << 1)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_LAST    PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_MASK 0x18UL
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_SFT  3
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_INNERMOST (0x0UL << 3)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTER (0x1UL << 3)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTERMOST (0x2UL << 3)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED (0x3UL << 3)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_LAST    PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_MASK    0xe0UL
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_SFT     5
+	u8 unused_0[3];
 };
 
 /* Output (16 bytes) */
@@ -1902,9 +1972,9 @@ struct hwrm_port_mac_cfg_output {
 	__le16 mtu;
 	u8 ipg;
 	u8 lpbk;
-	#define PORT_MAC_CFG_RESP_LPBK_NONE			   (0x0UL << 0)
-	#define PORT_MAC_CFG_RESP_LPBK_LOCAL			   (0x1UL << 0)
-	#define PORT_MAC_CFG_RESP_LPBK_REMOTE			   (0x2UL << 0)
+	#define PORT_MAC_CFG_RESP_LPBK_NONE			   0x0UL
+	#define PORT_MAC_CFG_RESP_LPBK_LOCAL			   0x1UL
+	#define PORT_MAC_CFG_RESP_LPBK_REMOTE			   0x2UL
 	u8 unused_0;
 	u8 valid;
 };
@@ -2163,8 +2233,8 @@ struct hwrm_queue_qportcfg_input {
 	__le64 resp_addr;
 	__le32 flags;
 	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH			    0x1UL
-	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_TX		   (0x0UL << 0)
-	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_RX		   (0x1UL << 0)
+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_TX		   0x0UL
+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_RX		   0x1UL
 	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_LAST    QUEUE_QPORTCFG_REQ_FLAGS_PATH_RX
 	__le16 port_id;
 	__le16 unused_0;
@@ -2179,50 +2249,51 @@ struct hwrm_queue_qportcfg_output {
 	u8 max_configurable_queues;
 	u8 max_configurable_lossless_queues;
 	u8 queue_cfg_allowed;
-	u8 queue_buffers_cfg_allowed;
+	u8 queue_cfg_info;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG	    0x1UL
 	u8 queue_pfcenable_cfg_allowed;
 	u8 queue_pri2cos_cfg_allowed;
 	u8 queue_cos2bw_cfg_allowed;
 	u8 queue_id0;
 	u8 queue_id0_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSY (0x0UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS (0x1UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN (0xffUL << 0)
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS 0x1UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN 0xffUL
 	u8 queue_id1;
 	u8 queue_id1_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSY (0x0UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSLESS (0x1UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN (0xffUL << 0)
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSLESS 0x1UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN 0xffUL
 	u8 queue_id2;
 	u8 queue_id2_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSY (0x0UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSLESS (0x1UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN (0xffUL << 0)
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSLESS 0x1UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN 0xffUL
 	u8 queue_id3;
 	u8 queue_id3_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSY (0x0UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSLESS (0x1UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN (0xffUL << 0)
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSLESS 0x1UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN 0xffUL
 	u8 queue_id4;
 	u8 queue_id4_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSY (0x0UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSLESS (0x1UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN (0xffUL << 0)
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSLESS 0x1UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN 0xffUL
 	u8 queue_id5;
 	u8 queue_id5_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSY (0x0UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSLESS (0x1UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN (0xffUL << 0)
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSLESS 0x1UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN 0xffUL
 	u8 queue_id6;
 	u8 queue_id6_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSY (0x0UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSLESS (0x1UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN (0xffUL << 0)
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSLESS 0x1UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN 0xffUL
 	u8 queue_id7;
 	u8 queue_id7_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSY (0x0UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSLESS (0x1UL << 0)
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN (0xffUL << 0)
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSLESS 0x1UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN 0xffUL
 	u8 valid;
 };
 
@@ -2235,19 +2306,21 @@ struct hwrm_queue_cfg_input {
 	__le16 target_id;
 	__le64 resp_addr;
 	__le32 flags;
-	#define QUEUE_CFG_REQ_FLAGS_PATH			    0x1UL
-	#define QUEUE_CFG_REQ_FLAGS_PATH_TX			   (0x0UL << 0)
-	#define QUEUE_CFG_REQ_FLAGS_PATH_RX			   (0x1UL << 0)
-	#define QUEUE_CFG_REQ_FLAGS_PATH_LAST    QUEUE_CFG_REQ_FLAGS_PATH_RX
+	#define QUEUE_CFG_REQ_FLAGS_PATH_MASK			    0x3UL
+	#define QUEUE_CFG_REQ_FLAGS_PATH_SFT			    0
+	#define QUEUE_CFG_REQ_FLAGS_PATH_TX			   0x0UL
+	#define QUEUE_CFG_REQ_FLAGS_PATH_RX			   0x1UL
+	#define QUEUE_CFG_REQ_FLAGS_PATH_BIDIR			   0x2UL
+	#define QUEUE_CFG_REQ_FLAGS_PATH_LAST    QUEUE_CFG_REQ_FLAGS_PATH_BIDIR
 	__le32 enables;
 	#define QUEUE_CFG_REQ_ENABLES_DFLT_LEN			    0x1UL
 	#define QUEUE_CFG_REQ_ENABLES_SERVICE_PROFILE		    0x2UL
 	__le32 queue_id;
 	__le32 dflt_len;
 	u8 service_profile;
-	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSY		   (0x0UL << 0)
-	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSLESS		   (0x1UL << 0)
-	#define QUEUE_CFG_REQ_SERVICE_PROFILE_UNKNOWN		   (0xffUL << 0)
+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSY		   0x0UL
+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSLESS		   0x1UL
+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_UNKNOWN		   0xffUL
 	u8 unused_0[7];
 };
 
@@ -2264,50 +2337,6 @@ struct hwrm_queue_cfg_output {
 	u8 valid;
 };
 
-/* hwrm_queue_buffers_cfg */
-/* Input (56 bytes) */
-struct hwrm_queue_buffers_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define QUEUE_BUFFERS_CFG_REQ_FLAGS_PATH		    0x1UL
-	#define QUEUE_BUFFERS_CFG_REQ_FLAGS_PATH_TX		   (0x0UL << 0)
-	#define QUEUE_BUFFERS_CFG_REQ_FLAGS_PATH_RX		   (0x1UL << 0)
-	#define QUEUE_BUFFERS_CFG_REQ_FLAGS_PATH_LAST    QUEUE_BUFFERS_CFG_REQ_FLAGS_PATH_RX
-	__le32 enables;
-	#define QUEUE_BUFFERS_CFG_REQ_ENABLES_RESERVED		    0x1UL
-	#define QUEUE_BUFFERS_CFG_REQ_ENABLES_SHARED		    0x2UL
-	#define QUEUE_BUFFERS_CFG_REQ_ENABLES_XOFF		    0x4UL
-	#define QUEUE_BUFFERS_CFG_REQ_ENABLES_XON		    0x8UL
-	#define QUEUE_BUFFERS_CFG_REQ_ENABLES_FULL		    0x10UL
-	#define QUEUE_BUFFERS_CFG_REQ_ENABLES_NOTFULL		    0x20UL
-	#define QUEUE_BUFFERS_CFG_REQ_ENABLES_MAX		    0x40UL
-	__le32 queue_id;
-	__le32 reserved;
-	__le32 shared;
-	__le32 xoff;
-	__le32 xon;
-	__le32 full;
-	__le32 notfull;
-	__le32 max;
-};
-
-/* Output (16 bytes) */
-struct hwrm_queue_buffers_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
 /* hwrm_queue_pfcenable_cfg */
 /* Input (24 bytes) */
 struct hwrm_queue_pfcenable_cfg_input {
@@ -2351,12 +2380,22 @@ struct hwrm_queue_pri2cos_cfg_input {
 	__le16 target_id;
 	__le64 resp_addr;
 	__le32 flags;
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH		    0x1UL
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_MASK		    0x3UL
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_SFT		    0
 	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_TX		   (0x0UL << 0)
 	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_RX		   (0x1UL << 0)
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_LAST    QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_RX
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_IVLAN		    0x2UL
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_BIDIR		   (0x2UL << 0)
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_LAST    QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_BIDIR
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_IVLAN		    0x4UL
 	__le32 enables;
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI0_COS_QUEUE_ID    0x1UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI1_COS_QUEUE_ID    0x2UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI2_COS_QUEUE_ID    0x4UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI3_COS_QUEUE_ID    0x8UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI4_COS_QUEUE_ID    0x10UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI5_COS_QUEUE_ID    0x20UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI6_COS_QUEUE_ID    0x40UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI7_COS_QUEUE_ID    0x80UL
 	u8 port_id;
 	u8 pri0_cos_queue_id;
 	u8 pri1_cos_queue_id;
@@ -2404,82 +2443,226 @@ struct hwrm_queue_cos2bw_cfg_input {
 	u8 queue_id0;
 	u8 unused_0;
 	__le32 queue_id0_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 queue_id0_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 queue_id0_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_SP      (0x0UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_ETS     (0x1UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_RESERVED_FIRST (0x2UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_RESERVED_LAST (0xffUL << 0)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_SP      0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_RESERVED_FIRST 0x2UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_RESERVED_LAST 0xffUL
 	u8 queue_id0_pri_lvl;
 	u8 queue_id0_bw_weight;
 	u8 queue_id1;
 	__le32 queue_id1_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 queue_id1_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 queue_id1_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_SP      (0x0UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_ETS     (0x1UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_RESERVED_FIRST (0x2UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_RESERVED_LAST (0xffUL << 0)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_SP      0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_RESERVED_FIRST 0x2UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_RESERVED_LAST 0xffUL
 	u8 queue_id1_pri_lvl;
 	u8 queue_id1_bw_weight;
 	u8 queue_id2;
 	__le32 queue_id2_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 queue_id2_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 queue_id2_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_SP      (0x0UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_ETS     (0x1UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_RESERVED_FIRST (0x2UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_RESERVED_LAST (0xffUL << 0)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_SP      0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_RESERVED_FIRST 0x2UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_RESERVED_LAST 0xffUL
 	u8 queue_id2_pri_lvl;
 	u8 queue_id2_bw_weight;
 	u8 queue_id3;
 	__le32 queue_id3_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 queue_id3_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 queue_id3_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_SP      (0x0UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_ETS     (0x1UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_RESERVED_FIRST (0x2UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_RESERVED_LAST (0xffUL << 0)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_SP      0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_RESERVED_FIRST 0x2UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_RESERVED_LAST 0xffUL
 	u8 queue_id3_pri_lvl;
 	u8 queue_id3_bw_weight;
 	u8 queue_id4;
 	__le32 queue_id4_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 queue_id4_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 queue_id4_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_SP      (0x0UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_ETS     (0x1UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_RESERVED_FIRST (0x2UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_RESERVED_LAST (0xffUL << 0)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_SP      0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_RESERVED_FIRST 0x2UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_RESERVED_LAST 0xffUL
 	u8 queue_id4_pri_lvl;
 	u8 queue_id4_bw_weight;
 	u8 queue_id5;
 	__le32 queue_id5_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 queue_id5_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 queue_id5_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_SP      (0x0UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_ETS     (0x1UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_RESERVED_FIRST (0x2UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_RESERVED_LAST (0xffUL << 0)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_SP      0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_RESERVED_FIRST 0x2UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_RESERVED_LAST 0xffUL
 	u8 queue_id5_pri_lvl;
 	u8 queue_id5_bw_weight;
 	u8 queue_id6;
 	__le32 queue_id6_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 queue_id6_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 queue_id6_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_SP      (0x0UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_ETS     (0x1UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_RESERVED_FIRST (0x2UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_RESERVED_LAST (0xffUL << 0)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_SP      0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_RESERVED_FIRST 0x2UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_RESERVED_LAST 0xffUL
 	u8 queue_id6_pri_lvl;
 	u8 queue_id6_bw_weight;
 	u8 queue_id7;
 	__le32 queue_id7_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID
 	__le32 queue_id7_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_MASK 0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_SFT 0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_RSVD	    0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_SFT 29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MBPS (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 queue_id7_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_SP      (0x0UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_ETS     (0x1UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_RESERVED_FIRST (0x2UL << 0)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_RESERVED_LAST (0xffUL << 0)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_SP      0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_RESERVED_FIRST 0x2UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_RESERVED_LAST 0xffUL
 	u8 queue_id7_pri_lvl;
 	u8 queue_id7_bw_weight;
 	u8 unused_1[5];
@@ -2563,6 +2746,7 @@ struct hwrm_vnic_cfg_input {
 	#define VNIC_CFG_REQ_FLAGS_BD_STALL_MODE		    0x4UL
 	#define VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE		    0x8UL
 	#define VNIC_CFG_REQ_FLAGS_ROCE_ONLY_VNIC_MODE		    0x10UL
+	#define VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE		    0x20UL
 	__le32 enables;
 	#define VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP		    0x1UL
 	#define VNIC_CFG_REQ_ENABLES_RSS_RULE			    0x2UL
@@ -2615,18 +2799,18 @@ struct hwrm_vnic_tpa_cfg_input {
 	#define VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN		    0x8UL
 	__le16 vnic_id;
 	__le16 max_agg_segs;
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_1		   (0x0UL << 0)
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_2		   (0x1UL << 0)
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_4		   (0x2UL << 0)
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_8		   (0x3UL << 0)
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_MAX		   (0x1fUL << 0)
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_1		   0x0UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_2		   0x1UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_4		   0x2UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_8		   0x3UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_MAX		   0x1fUL
 	__le16 max_aggs;
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_1			   (0x0UL << 0)
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_2			   (0x1UL << 0)
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_4			   (0x2UL << 0)
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_8			   (0x3UL << 0)
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_16			   (0x4UL << 0)
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_MAX			   (0x7UL << 0)
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_1			   0x0UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_2			   0x1UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_4			   0x2UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_8			   0x3UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_16			   0x4UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_MAX			   0x7UL
 	u8 unused_0;
 	u8 unused_1;
 	__le32 max_agg_timer;
@@ -2780,15 +2964,15 @@ struct hwrm_ring_alloc_input {
 	__le64 resp_addr;
 	__le32 enables;
 	#define RING_ALLOC_REQ_ENABLES_RESERVED1		    0x1UL
-	#define RING_ALLOC_REQ_ENABLES_RESERVED2		    0x2UL
+	#define RING_ALLOC_REQ_ENABLES_RING_ARB_CFG		    0x2UL
 	#define RING_ALLOC_REQ_ENABLES_RESERVED3		    0x4UL
 	#define RING_ALLOC_REQ_ENABLES_STAT_CTX_ID_VALID	    0x8UL
 	#define RING_ALLOC_REQ_ENABLES_RESERVED4		    0x10UL
 	#define RING_ALLOC_REQ_ENABLES_MAX_BW_VALID		    0x20UL
 	u8 ring_type;
-	#define RING_ALLOC_REQ_RING_TYPE_CMPL			   (0x0UL << 0)
-	#define RING_ALLOC_REQ_RING_TYPE_TX			   (0x1UL << 0)
-	#define RING_ALLOC_REQ_RING_TYPE_RX			   (0x2UL << 0)
+	#define RING_ALLOC_REQ_RING_TYPE_CMPL			   0x0UL
+	#define RING_ALLOC_REQ_RING_TYPE_TX			   0x1UL
+	#define RING_ALLOC_REQ_RING_TYPE_RX			   0x2UL
 	u8 unused_0;
 	__le16 unused_1;
 	__le64 page_tbl_addr;
@@ -2804,18 +2988,36 @@ struct hwrm_ring_alloc_input {
 	u8 unused_4;
 	u8 unused_5;
 	__le32 reserved1;
-	__le16 reserved2;
+	__le16 ring_arb_cfg;
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_MASK	    0xfUL
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SFT	    0
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SP	   (0x1UL << 0)
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ	   (0x2UL << 0)
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_LAST    RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ
+	#define RING_ALLOC_REQ_RING_ARB_CFG_RSVD_MASK		    0xf0UL
+	#define RING_ALLOC_REQ_RING_ARB_CFG_RSVD_SFT		    4
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_MASK  0xff00UL
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_SFT   8
 	u8 unused_6;
 	u8 unused_7;
 	__le32 reserved3;
 	__le32 stat_ctx_id;
 	__le32 reserved4;
 	__le32 max_bw;
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_MASK		    0xfffffffUL
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_SFT		    0
+	#define RING_ALLOC_REQ_MAX_BW_RSVD			    0x10000000UL
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MASK	    0xe0000000UL
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_SFT	    29
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MBPS	   (0x0UL << 29)
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_LAST    RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_INVALID
 	u8 int_mode;
-	#define RING_ALLOC_REQ_INT_MODE_LEGACY			   (0x0UL << 0)
-	#define RING_ALLOC_REQ_INT_MODE_RSVD			   (0x1UL << 0)
-	#define RING_ALLOC_REQ_INT_MODE_MSIX			   (0x2UL << 0)
-	#define RING_ALLOC_REQ_INT_MODE_POLL			   (0x3UL << 0)
+	#define RING_ALLOC_REQ_INT_MODE_LEGACY			   0x0UL
+	#define RING_ALLOC_REQ_INT_MODE_RSVD			   0x1UL
+	#define RING_ALLOC_REQ_INT_MODE_MSIX			   0x2UL
+	#define RING_ALLOC_REQ_INT_MODE_POLL			   0x3UL
 	u8 unused_8[3];
 };
 
@@ -2842,9 +3044,9 @@ struct hwrm_ring_free_input {
 	__le16 target_id;
 	__le64 resp_addr;
 	u8 ring_type;
-	#define RING_FREE_REQ_RING_TYPE_CMPL			   (0x0UL << 0)
-	#define RING_FREE_REQ_RING_TYPE_TX			   (0x1UL << 0)
-	#define RING_FREE_REQ_RING_TYPE_RX			   (0x2UL << 0)
+	#define RING_FREE_REQ_RING_TYPE_CMPL			   0x0UL
+	#define RING_FREE_REQ_RING_TYPE_TX			   0x1UL
+	#define RING_FREE_REQ_RING_TYPE_RX			   0x2UL
 	u8 unused_0;
 	__le16 ring_id;
 	__le32 unused_1;
@@ -2942,9 +3144,9 @@ struct hwrm_ring_reset_input {
 	__le16 target_id;
 	__le64 resp_addr;
 	u8 ring_type;
-	#define RING_RESET_REQ_RING_TYPE_CMPL			   (0x0UL << 0)
-	#define RING_RESET_REQ_RING_TYPE_TX			   (0x1UL << 0)
-	#define RING_RESET_REQ_RING_TYPE_RX			   (0x2UL << 0)
+	#define RING_RESET_REQ_RING_TYPE_CMPL			   0x0UL
+	#define RING_RESET_REQ_RING_TYPE_TX			   0x1UL
+	#define RING_RESET_REQ_RING_TYPE_RX			   0x2UL
 	u8 unused_0;
 	__le16 ring_id;
 	__le32 unused_1;
@@ -3068,36 +3270,36 @@ struct hwrm_cfa_l2_filter_alloc_input {
 	__le16 t_l2_ivlan;
 	__le16 t_l2_ivlan_mask;
 	u8 src_type;
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_NPORT		   (0x0UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_PF		   (0x1UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VF		   (0x2UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VNIC		   (0x3UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_KONG		   (0x4UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_APE		   (0x5UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_BONO		   (0x6UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_TANG		   (0x7UL << 0)
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_NPORT		   0x0UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_PF		   0x1UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VF		   0x2UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VNIC		   0x3UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_KONG		   0x4UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_APE		   0x5UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_BONO		   0x6UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_TANG		   0x7UL
 	u8 unused_6;
 	__le32 src_id;
 	u8 tunnel_type;
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL     (0x0UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN	   (0x1UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE	   (0x2UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE	   (0x3UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP	   (0x4UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE	   (0x5UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS	   (0x6UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT	   (0x7UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE	   (0x8UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL     (0xffUL << 0)
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL     0x0UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN	   0x1UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE	   0x2UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE	   0x3UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP	   0x4UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE	   0x5UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS	   0x6UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT	   0x7UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE	   0x8UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL     0xffUL
 	u8 unused_7;
 	__le16 dst_id;
 	__le16 mirror_vnic_id;
 	u8 pri_hint;
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER	   (0x0UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_ABOVE_FILTER     (0x1UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_BELOW_FILTER     (0x2UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MAX		   (0x3UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN		   (0x4UL << 0)
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER	   0x0UL
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_ABOVE_FILTER     0x1UL
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_BELOW_FILTER     0x2UL
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MAX		   0x3UL
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN		   0x4UL
 	u8 unused_8;
 	__le32 unused_9;
 	__le64 l2_filter_id_hint;
@@ -3246,16 +3448,16 @@ struct hwrm_cfa_tunnel_filter_alloc_input {
 	u8 l3_addr_type;
 	u8 t_l3_addr_type;
 	u8 tunnel_type;
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL (0x0UL << 0)
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN     (0x1UL << 0)
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE     (0x2UL << 0)
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE     (0x3UL << 0)
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP      (0x4UL << 0)
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE    (0x5UL << 0)
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS      (0x6UL << 0)
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT       (0x7UL << 0)
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE     (0x8UL << 0)
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL (0xffUL << 0)
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL 0x0UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN     0x1UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE     0x2UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE     0x3UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP      0x4UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE    0x5UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS      0x6UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT       0x7UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE     0x8UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL 0xffUL
 	u8 unused_0;
 	__le32 vni;
 	__le32 dst_vnic_id;
@@ -3311,14 +3513,14 @@ struct hwrm_cfa_encap_record_alloc_input {
 	__le32 flags;
 	#define CFA_ENCAP_RECORD_ALLOC_REQ_FLAGS_LOOPBACK	    0x1UL
 	u8 encap_type;
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN       (0x1UL << 0)
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_NVGRE       (0x2UL << 0)
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_L2GRE       (0x3UL << 0)
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPIP	   (0x4UL << 0)
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_GENEVE      (0x5UL << 0)
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_MPLS	   (0x6UL << 0)
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VLAN	   (0x7UL << 0)
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPGRE       (0x8UL << 0)
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN       0x1UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_NVGRE       0x2UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_L2GRE       0x3UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPIP	   0x4UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_GENEVE      0x5UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_MPLS	   0x6UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VLAN	   0x7UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPGRE       0x8UL
 	u8 unused_0;
 	__le16 unused_1;
 	__le32 encap_data[16];
@@ -3397,32 +3599,32 @@ struct hwrm_cfa_ntuple_filter_alloc_input {
 	u8 src_macaddr[6];
 	__be16 ethertype;
 	u8 ip_addr_type;
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN  (0x0UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4     (0x4UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6     (0x6UL << 0)
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN  0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4     0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6     0x6UL
 	u8 ip_protocol;
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN   (0x0UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP       (0x6UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP       (0x11UL << 0)
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN   0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP       0x6UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP       0x11UL
 	__le16 dst_id;
 	__le16 mirror_vnic_id;
 	u8 tunnel_type;
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL (0x0UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN     (0x1UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE     (0x2UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE     (0x3UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP      (0x4UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE    (0x5UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS      (0x6UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT       (0x7UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE     (0x8UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL (0xffUL << 0)
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL 0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN     0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE     0x2UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE     0x3UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP      0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE    0x5UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS      0x6UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT       0x7UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE     0x8UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL 0xffUL
 	u8 pri_hint;
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER    (0x0UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_ABOVE	   (0x1UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_BELOW	   (0x2UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_HIGHEST      (0x3UL << 0)
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST       (0x4UL << 0)
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER    0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_ABOVE	   0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_BELOW	   0x2UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_HIGHEST      0x3UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST       0x4UL
 	__be32 src_ipaddr[4];
 	__be32 src_ipaddr_mask[4];
 	__be32 dst_ipaddr[4];
@@ -3511,8 +3713,8 @@ struct hwrm_tunnel_dst_port_query_input {
 	__le16 target_id;
 	__le64 resp_addr;
 	u8 tunnel_type;
-	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN       (0x1UL << 0)
-	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_GENEVE      (0x5UL << 0)
+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN       0x1UL
+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_GENEVE      0x5UL
 	u8 unused_0[7];
 };
 
@@ -3539,8 +3741,8 @@ struct hwrm_tunnel_dst_port_alloc_input {
 	__le16 target_id;
 	__le64 resp_addr;
 	u8 tunnel_type;
-	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN       (0x1UL << 0)
-	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE      (0x5UL << 0)
+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN       0x1UL
+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE      0x5UL
 	u8 unused_0;
 	__be16 tunnel_dst_port_val;
 	__le32 unused_1;
@@ -3570,8 +3772,8 @@ struct hwrm_tunnel_dst_port_free_input {
 	__le16 target_id;
 	__le64 resp_addr;
 	u8 tunnel_type;
-	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN	   (0x1UL << 0)
-	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE       (0x5UL << 0)
+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN	   0x1UL
+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE       0x5UL
 	u8 unused_0;
 	__le16 tunnel_dst_port_id;
 	__le32 unused_1;
@@ -3720,15 +3922,15 @@ struct hwrm_fw_reset_input {
 	__le16 target_id;
 	__le64 resp_addr;
 	u8 embedded_proc_type;
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_BOOT		   (0x0UL << 0)
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_MGMT		   (0x1UL << 0)
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_NETCTRL	   (0x2UL << 0)
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_ROCE		   (0x3UL << 0)
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_RSVD		   (0x4UL << 0)
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_BOOT		   0x0UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_MGMT		   0x1UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_NETCTRL	   0x2UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_ROCE		   0x3UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_RSVD		   0x4UL
 	u8 selfrst_status;
-	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTNONE	   (0x0UL << 0)
-	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP	   (0x1UL << 0)
-	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTPCIERST	   (0x2UL << 0)
+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTNONE	   0x0UL
+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP	   0x1UL
+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTPCIERST	   0x2UL
 	__le16 unused_0[3];
 };
 
@@ -3739,9 +3941,9 @@ struct hwrm_fw_reset_output {
 	__le16 seq_id;
 	__le16 resp_len;
 	u8 selfrst_status;
-	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTNONE	   (0x0UL << 0)
-	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTASAP	   (0x1UL << 0)
-	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTPCIERST       (0x2UL << 0)
+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTNONE	   0x0UL
+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTASAP	   0x1UL
+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTPCIERST       0x2UL
 	u8 unused_0;
 	__le16 unused_1;
 	u8 unused_2;
@@ -3759,11 +3961,11 @@ struct hwrm_fw_qstatus_input {
 	__le16 target_id;
 	__le64 resp_addr;
 	u8 embedded_proc_type;
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_BOOT		   (0x0UL << 0)
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_MGMT		   (0x1UL << 0)
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_NETCTRL	   (0x2UL << 0)
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_ROCE		   (0x3UL << 0)
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_RSVD		   (0x4UL << 0)
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_BOOT		   0x0UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_MGMT		   0x1UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_NETCTRL	   0x2UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_ROCE		   0x3UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_RSVD		   0x4UL
 	u8 unused_0[7];
 };
 
@@ -3774,9 +3976,9 @@ struct hwrm_fw_qstatus_output {
 	__le16 seq_id;
 	__le16 resp_len;
 	u8 selfrst_status;
-	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTNONE	   (0x0UL << 0)
-	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTASAP	   (0x1UL << 0)
-	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTPCIERST     (0x2UL << 0)
+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTNONE	   0x0UL
+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTASAP	   0x1UL
+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTPCIERST     0x2UL
 	u8 unused_0;
 	__le16 unused_1;
 	u8 unused_2;
@@ -3785,6 +3987,42 @@ struct hwrm_fw_qstatus_output {
 	u8 valid;
 };
 
+/* hwrm_fw_set_time */
+/* Input (32 bytes) */
+struct hwrm_fw_set_time_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le16 year;
+	#define FW_SET_TIME_REQ_YEAR_UNKNOWN			   0x0UL
+	u8 month;
+	u8 day;
+	u8 hour;
+	u8 minute;
+	u8 second;
+	u8 unused_0;
+	__le16 millisecond;
+	__le16 zone;
+	#define FW_SET_TIME_REQ_ZONE_UTC			   0x0UL
+	#define FW_SET_TIME_REQ_ZONE_UNKNOWN			   0xffffUL
+	__le32 unused_1;
+};
+
+/* Output (16 bytes) */
+struct hwrm_fw_set_time_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le32 unused_0;
+	u8 unused_1;
+	u8 unused_2;
+	u8 unused_3;
+	u8 valid;
+};
+
 /* hwrm_exec_fwd_resp */
 /* Input (128 bytes) */
 struct hwrm_exec_fwd_resp_input {
@@ -3921,32 +4159,6 @@ struct hwrm_temp_monitor_query_output {
 	u8 valid;
 };
 
-/* hwrm_nvm_raw_write_blk */
-/* Input (32 bytes) */
-struct hwrm_nvm_raw_write_blk_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 host_src_addr;
-	__le32 dest_addr;
-	__le32 len;
-};
-
-/* Output (16 bytes) */
-struct hwrm_nvm_raw_write_blk_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
 /* hwrm_nvm_read */
 /* Input (40 bytes) */
 struct hwrm_nvm_read_input {
@@ -4132,9 +4344,9 @@ struct hwrm_nvm_find_dir_entry_input {
 	u8 opt_ordinal;
 	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_MASK	    0x3UL
 	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_SFT		    0
-	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_EQ		   (0x0UL << 0)
-	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GE		   (0x1UL << 0)
-	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GT		   (0x2UL << 0)
+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_EQ		   0x0UL
+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GE		   0x1UL
+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GT		   0x2UL
 	u8 unused_1[3];
 };
 
@@ -4266,4 +4478,41 @@ struct hwrm_nvm_verify_update_output {
 	u8 valid;
 };
 
+/* hwrm_nvm_install_update */
+/* Input (24 bytes) */
+struct hwrm_nvm_install_update_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le32 install_type;
+	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_NORMAL	   0x0UL
+	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_ALL	   0xffffffffUL
+	__le32 unused_0;
+};
+
+/* Output (24 bytes) */
+struct hwrm_nvm_install_update_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le64 installed_items;
+	u8 result;
+	#define NVM_INSTALL_UPDATE_RESP_RESULT_SUCCESS		   0x0UL
+	u8 problem_item;
+	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_NONE	   0x0UL
+	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_PACKAGE      0xffUL
+	u8 reset_required;
+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_NONE       0x0UL
+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_PCI	   0x1UL
+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_POWER      0x2UL
+	u8 unused_0;
+	u8 unused_1;
+	u8 unused_2;
+	u8 unused_3;
+	u8 valid;
+};
+
 #endif
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
index 50d2007a2640..ec6cd18842c3 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
@@ -19,6 +19,45 @@
 #include "bnxt_ethtool.h"
 
 #ifdef CONFIG_BNXT_SRIOV
+static int bnxt_hwrm_fwd_async_event_cmpl(struct bnxt *bp,
+					  struct bnxt_vf_info *vf, u16 event_id)
+{
+	struct hwrm_fwd_async_event_cmpl_output *resp = bp->hwrm_cmd_resp_addr;
+	struct hwrm_fwd_async_event_cmpl_input req = {0};
+	struct hwrm_async_event_cmpl *async_cmpl;
+	int rc = 0;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FWD_ASYNC_EVENT_CMPL, -1, -1);
+	if (vf)
+		req.encap_async_event_target_id = cpu_to_le16(vf->fw_fid);
+	else
+		/* broadcast this async event to all VFs */
+		req.encap_async_event_target_id = cpu_to_le16(0xffff);
+	async_cmpl = (struct hwrm_async_event_cmpl *)req.encap_async_event_cmpl;
+	async_cmpl->type =
+		cpu_to_le16(HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT);
+	async_cmpl->event_id = cpu_to_le16(event_id);
+
+	mutex_lock(&bp->hwrm_cmd_lock);
+	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+
+	if (rc) {
+		netdev_err(bp->dev, "hwrm_fwd_async_event_cmpl failed. rc:%d\n",
+			   rc);
+		goto fwd_async_event_cmpl_exit;
+	}
+
+	if (resp->error_code) {
+		netdev_err(bp->dev, "hwrm_fwd_async_event_cmpl error %d\n",
+			   resp->error_code);
+		rc = -1;
+	}
+
+fwd_async_event_cmpl_exit:
+	mutex_unlock(&bp->hwrm_cmd_lock);
+	return rc;
+}
+
 static int bnxt_vf_ndo_prep(struct bnxt *bp, int vf_id)
 {
 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
@@ -135,7 +174,8 @@ int bnxt_set_vf_mac(struct net_device *dev, int vf_id, u8 *mac)
 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 }
 
-int bnxt_set_vf_vlan(struct net_device *dev, int vf_id, u16 vlan_id, u8 qos)
+int bnxt_set_vf_vlan(struct net_device *dev, int vf_id, u16 vlan_id, u8 qos,
+		     __be16 vlan_proto)
 {
 	struct hwrm_func_cfg_input req = {0};
 	struct bnxt *bp = netdev_priv(dev);
@@ -146,6 +186,9 @@ int bnxt_set_vf_vlan(struct net_device *dev, int vf_id, u16 vlan_id, u8 qos)
 	if (bp->hwrm_spec_code < 0x10201)
 		return -ENOTSUPP;
 
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
 	rc = bnxt_vf_ndo_prep(bp, vf_id);
 	if (rc)
 		return rc;
@@ -243,8 +286,9 @@ int bnxt_set_vf_link_state(struct net_device *dev, int vf_id, int link)
 		rc = -EINVAL;
 		break;
 	}
-	/* CHIMP TODO: send msg to VF to update new link state */
-
+	if (vf->flags & (BNXT_VF_LINK_UP | BNXT_VF_LINK_FORCED))
+		rc = bnxt_hwrm_fwd_async_event_cmpl(bp, vf,
+			HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE);
 	return rc;
 }
 
@@ -525,46 +569,6 @@ err_out1:
 	return rc;
 }
 
-static int bnxt_hwrm_fwd_async_event_cmpl(struct bnxt *bp,
-					  struct bnxt_vf_info *vf,
-					  u16 event_id)
-{
-	int rc = 0;
-	struct hwrm_fwd_async_event_cmpl_input req = {0};
-	struct hwrm_fwd_async_event_cmpl_output *resp = bp->hwrm_cmd_resp_addr;
-	struct hwrm_async_event_cmpl *async_cmpl;
-
-	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FWD_ASYNC_EVENT_CMPL, -1, -1);
-	if (vf)
-		req.encap_async_event_target_id = cpu_to_le16(vf->fw_fid);
-	else
-		/* broadcast this async event to all VFs */
-		req.encap_async_event_target_id = cpu_to_le16(0xffff);
-	async_cmpl = (struct hwrm_async_event_cmpl *)req.encap_async_event_cmpl;
-	async_cmpl->type =
-		cpu_to_le16(HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT);
-	async_cmpl->event_id = cpu_to_le16(event_id);
-
-	mutex_lock(&bp->hwrm_cmd_lock);
-	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
-
-	if (rc) {
-		netdev_err(bp->dev, "hwrm_fwd_async_event_cmpl failed. rc:%d\n",
-			   rc);
-		goto fwd_async_event_cmpl_exit;
-	}
-
-	if (resp->error_code) {
-		netdev_err(bp->dev, "hwrm_fwd_async_event_cmpl error %d\n",
-			   resp->error_code);
-		rc = -1;
-	}
-
-fwd_async_event_cmpl_exit:
-	mutex_unlock(&bp->hwrm_cmd_lock);
-	return rc;
-}
-
 void bnxt_sriov_disable(struct bnxt *bp)
 {
 	u16 num_vfs = pci_num_vf(bp->pdev);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h
index 0392670ab49c..1ab72e4820af 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.h
@@ -12,7 +12,7 @@
 
 int bnxt_get_vf_config(struct net_device *, int, struct ifla_vf_info *);
 int bnxt_set_vf_mac(struct net_device *, int, u8 *);
-int bnxt_set_vf_vlan(struct net_device *, int, u16, u8);
+int bnxt_set_vf_vlan(struct net_device *, int, u16, u8, __be16);
 int bnxt_set_vf_bw(struct net_device *, int, int, int);
 int bnxt_set_vf_link_state(struct net_device *, int, int);
 int bnxt_set_vf_spoofchk(struct net_device *, int, bool);
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
index 8d4f8495dbb3..4464bc5db934 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
@@ -450,28 +450,32 @@ static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv,
 			genet_dma_ring_regs[r]);
 }
 
-static int bcmgenet_get_settings(struct net_device *dev,
-				 struct ethtool_cmd *cmd)
+static int bcmgenet_get_link_ksettings(struct net_device *dev,
+				       struct ethtool_link_ksettings *cmd)
 {
+	struct bcmgenet_priv *priv = netdev_priv(dev);
+
 	if (!netif_running(dev))
 		return -EINVAL;
 
-	if (!dev->phydev)
+	if (!priv->phydev)
 		return -ENODEV;
 
-	return phy_ethtool_gset(dev->phydev, cmd);
+	return phy_ethtool_ksettings_get(priv->phydev, cmd);
 }
 
-static int bcmgenet_set_settings(struct net_device *dev,
-				 struct ethtool_cmd *cmd)
+static int bcmgenet_set_link_ksettings(struct net_device *dev,
+				       const struct ethtool_link_ksettings *cmd)
 {
+	struct bcmgenet_priv *priv = netdev_priv(dev);
+
 	if (!netif_running(dev))
 		return -EINVAL;
 
-	if (!dev->phydev)
+	if (!priv->phydev)
 		return -ENODEV;
 
-	return phy_ethtool_sset(dev->phydev, cmd);
+	return phy_ethtool_ksettings_set(priv->phydev, cmd);
 }
 
 static int bcmgenet_set_rx_csum(struct net_device *dev,
@@ -937,7 +941,7 @@ static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_eee *e)
 	e->eee_active = p->eee_active;
 	e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER);
 
-	return phy_ethtool_get_eee(dev->phydev, e);
+	return phy_ethtool_get_eee(priv->phydev, e);
 }
 
 static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_eee *e)
@@ -954,7 +958,7 @@ static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_eee *e)
 	if (!p->eee_enabled) {
 		bcmgenet_eee_enable_set(dev, false);
 	} else {
-		ret = phy_init_eee(dev->phydev, 0);
+		ret = phy_init_eee(priv->phydev, 0);
 		if (ret) {
 			netif_err(priv, hw, dev, "EEE initialization failed\n");
 			return ret;
@@ -964,21 +968,21 @@ static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_eee *e)
 		bcmgenet_eee_enable_set(dev, true);
 	}
 
-	return phy_ethtool_set_eee(dev->phydev, e);
+	return phy_ethtool_set_eee(priv->phydev, e);
 }
 
 static int bcmgenet_nway_reset(struct net_device *dev)
 {
-	return genphy_restart_aneg(dev->phydev);
+	struct bcmgenet_priv *priv = netdev_priv(dev);
+
+	return genphy_restart_aneg(priv->phydev);
 }
 
 /* standard ethtool support functions. */
-static struct ethtool_ops bcmgenet_ethtool_ops = {
+static const struct ethtool_ops bcmgenet_ethtool_ops = {
 	.get_strings		= bcmgenet_get_strings,
 	.get_sset_count		= bcmgenet_get_sset_count,
 	.get_ethtool_stats	= bcmgenet_get_ethtool_stats,
-	.get_settings		= bcmgenet_get_settings,
-	.set_settings		= bcmgenet_set_settings,
 	.get_drvinfo		= bcmgenet_get_drvinfo,
 	.get_link		= ethtool_op_get_link,
 	.get_msglevel		= bcmgenet_get_msglevel,
@@ -990,19 +994,20 @@ static struct ethtool_ops bcmgenet_ethtool_ops = {
 	.nway_reset		= bcmgenet_nway_reset,
 	.get_coalesce		= bcmgenet_get_coalesce,
 	.set_coalesce		= bcmgenet_set_coalesce,
+	.get_link_ksettings	= bcmgenet_get_link_ksettings,
+	.set_link_ksettings	= bcmgenet_set_link_ksettings,
 };
 
 /* Power down the unimac, based on mode. */
 static int bcmgenet_power_down(struct bcmgenet_priv *priv,
 				enum bcmgenet_power_mode mode)
 {
-	struct net_device *ndev = priv->dev;
 	int ret = 0;
 	u32 reg;
 
 	switch (mode) {
 	case GENET_POWER_CABLE_SENSE:
-		phy_detach(ndev->phydev);
+		phy_detach(priv->phydev);
 		break;
 
 	case GENET_POWER_WOL_MAGIC:
@@ -1063,6 +1068,7 @@ static void bcmgenet_power_up(struct bcmgenet_priv *priv,
 /* ioctl handle special commands that are not present in ethtool. */
 static int bcmgenet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 {
+	struct bcmgenet_priv *priv = netdev_priv(dev);
 	int val = 0;
 
 	if (!netif_running(dev))
@@ -1072,10 +1078,10 @@ static int bcmgenet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 	case SIOCGMIIPHY:
 	case SIOCGMIIREG:
 	case SIOCSMIIREG:
-		if (!dev->phydev)
+		if (!priv->phydev)
 			val = -ENODEV;
 		else
-			val = phy_mii_ioctl(dev->phydev, rq, cmd);
+			val = phy_mii_ioctl(priv->phydev, rq, cmd);
 		break;
 
 	default:
@@ -2458,7 +2464,6 @@ static void bcmgenet_irq_task(struct work_struct *work)
 {
 	struct bcmgenet_priv *priv = container_of(
 			work, struct bcmgenet_priv, bcmgenet_irq_work);
-	struct net_device *ndev = priv->dev;
 
 	netif_dbg(priv, intr, priv->dev, "%s\n", __func__);
 
@@ -2471,7 +2476,7 @@ static void bcmgenet_irq_task(struct work_struct *work)
 
 	/* Link UP/DOWN event */
 	if (priv->irq0_stat & UMAC_IRQ_LINK_EVENT) {
-		phy_mac_interrupt(ndev->phydev,
+		phy_mac_interrupt(priv->phydev,
 				  !!(priv->irq0_stat & UMAC_IRQ_LINK_UP));
 		priv->irq0_stat &= ~UMAC_IRQ_LINK_EVENT;
 	}
@@ -2664,128 +2669,6 @@ static void bcmgenet_enable_dma(struct bcmgenet_priv *priv, u32 dma_ctrl)
 	bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
 }
 
-static bool bcmgenet_hfb_is_filter_enabled(struct bcmgenet_priv *priv,
-					   u32 f_index)
-{
-	u32 offset;
-	u32 reg;
-
-	offset = HFB_FLT_ENABLE_V3PLUS + (f_index < 32) * sizeof(u32);
-	reg = bcmgenet_hfb_reg_readl(priv, offset);
-	return !!(reg & (1 << (f_index % 32)));
-}
-
-static void bcmgenet_hfb_enable_filter(struct bcmgenet_priv *priv, u32 f_index)
-{
-	u32 offset;
-	u32 reg;
-
-	offset = HFB_FLT_ENABLE_V3PLUS + (f_index < 32) * sizeof(u32);
-	reg = bcmgenet_hfb_reg_readl(priv, offset);
-	reg |= (1 << (f_index % 32));
-	bcmgenet_hfb_reg_writel(priv, reg, offset);
-}
-
-static void bcmgenet_hfb_set_filter_rx_queue_mapping(struct bcmgenet_priv *priv,
-						     u32 f_index, u32 rx_queue)
-{
-	u32 offset;
-	u32 reg;
-
-	offset = f_index / 8;
-	reg = bcmgenet_rdma_readl(priv, DMA_INDEX2RING_0 + offset);
-	reg &= ~(0xF << (4 * (f_index % 8)));
-	reg |= ((rx_queue & 0xF) << (4 * (f_index % 8)));
-	bcmgenet_rdma_writel(priv, reg, DMA_INDEX2RING_0 + offset);
-}
-
-static void bcmgenet_hfb_set_filter_length(struct bcmgenet_priv *priv,
-					   u32 f_index, u32 f_length)
-{
-	u32 offset;
-	u32 reg;
-
-	offset = HFB_FLT_LEN_V3PLUS +
-		 ((priv->hw_params->hfb_filter_cnt - 1 - f_index) / 4) *
-		 sizeof(u32);
-	reg = bcmgenet_hfb_reg_readl(priv, offset);
-	reg &= ~(0xFF << (8 * (f_index % 4)));
-	reg |= ((f_length & 0xFF) << (8 * (f_index % 4)));
-	bcmgenet_hfb_reg_writel(priv, reg, offset);
-}
-
-static int bcmgenet_hfb_find_unused_filter(struct bcmgenet_priv *priv)
-{
-	u32 f_index;
-
-	for (f_index = 0; f_index < priv->hw_params->hfb_filter_cnt; f_index++)
-		if (!bcmgenet_hfb_is_filter_enabled(priv, f_index))
-			return f_index;
-
-	return -ENOMEM;
-}
-
-/* bcmgenet_hfb_add_filter
- *
- * Add new filter to Hardware Filter Block to match and direct Rx traffic to
- * desired Rx queue.
- *
- * f_data is an array of unsigned 32-bit integers where each 32-bit integer
- * provides filter data for 2 bytes (4 nibbles) of Rx frame:
- *
- * bits 31:20 - unused
- * bit  19    - nibble 0 match enable
- * bit  18    - nibble 1 match enable
- * bit  17    - nibble 2 match enable
- * bit  16    - nibble 3 match enable
- * bits 15:12 - nibble 0 data
- * bits 11:8  - nibble 1 data
- * bits 7:4   - nibble 2 data
- * bits 3:0   - nibble 3 data
- *
- * Example:
- * In order to match:
- * - Ethernet frame type = 0x0800 (IP)
- * - IP version field = 4
- * - IP protocol field = 0x11 (UDP)
- *
- * The following filter is needed:
- * u32 hfb_filter_ipv4_udp[] = {
- *   Rx frame offset 0x00: 0x00000000, 0x00000000, 0x00000000, 0x00000000,
- *   Rx frame offset 0x08: 0x00000000, 0x00000000, 0x000F0800, 0x00084000,
- *   Rx frame offset 0x10: 0x00000000, 0x00000000, 0x00000000, 0x00030011,
- * };
- *
- * To add the filter to HFB and direct the traffic to Rx queue 0, call:
- * bcmgenet_hfb_add_filter(priv, hfb_filter_ipv4_udp,
- *                         ARRAY_SIZE(hfb_filter_ipv4_udp), 0);
- */
-int bcmgenet_hfb_add_filter(struct bcmgenet_priv *priv, u32 *f_data,
-			    u32 f_length, u32 rx_queue)
-{
-	int f_index;
-	u32 i;
-
-	f_index = bcmgenet_hfb_find_unused_filter(priv);
-	if (f_index < 0)
-		return -ENOMEM;
-
-	if (f_length > priv->hw_params->hfb_filter_size)
-		return -EINVAL;
-
-	for (i = 0; i < f_length; i++)
-		bcmgenet_hfb_writel(priv, f_data[i],
-			(f_index * priv->hw_params->hfb_filter_size + i) *
-			sizeof(u32));
-
-	bcmgenet_hfb_set_filter_length(priv, f_index, 2 * f_length);
-	bcmgenet_hfb_set_filter_rx_queue_mapping(priv, f_index, rx_queue);
-	bcmgenet_hfb_enable_filter(priv, f_index);
-	bcmgenet_hfb_reg_writel(priv, 0x1, HFB_CTRL);
-
-	return 0;
-}
-
 /* bcmgenet_hfb_clear
  *
  * Clear Hardware Filter Block and disable all filtering.
@@ -2833,7 +2716,7 @@ static void bcmgenet_netif_start(struct net_device *dev)
 	/* Monitor link interrupts now */
 	bcmgenet_link_intr_enable(priv);
 
-	phy_start(dev->phydev);
+	phy_start(priv->phydev);
 }
 
 static int bcmgenet_open(struct net_device *dev)
@@ -2932,7 +2815,7 @@ static void bcmgenet_netif_stop(struct net_device *dev)
 	struct bcmgenet_priv *priv = netdev_priv(dev);
 
 	netif_tx_stop_all_queues(dev);
-	phy_stop(dev->phydev);
+	phy_stop(priv->phydev);
 	bcmgenet_intr_disable(priv);
 	bcmgenet_disable_rx_napi(priv);
 	bcmgenet_disable_tx_napi(priv);
@@ -2958,7 +2841,7 @@ static int bcmgenet_close(struct net_device *dev)
 	bcmgenet_netif_stop(dev);
 
 	/* Really kill the PHY state machine and disconnect from it */
-	phy_disconnect(dev->phydev);
+	phy_disconnect(priv->phydev);
 
 	/* Disable MAC receive */
 	umac_enable_set(priv, CMD_RX_EN, false);
@@ -3517,7 +3400,7 @@ static int bcmgenet_suspend(struct device *d)
 
 	bcmgenet_netif_stop(dev);
 
-	phy_suspend(dev->phydev);
+	phy_suspend(priv->phydev);
 
 	netif_device_detach(dev);
 
@@ -3581,7 +3464,7 @@ static int bcmgenet_resume(struct device *d)
 	if (priv->wolopts)
 		clk_disable_unprepare(priv->clk_wol);
 
-	phy_init_hw(dev->phydev);
+	phy_init_hw(priv->phydev);
 	/* Speed settings must be restored */
 	bcmgenet_mii_config(priv->dev);
 
@@ -3614,7 +3497,7 @@ static int bcmgenet_resume(struct device *d)
 
 	netif_device_attach(dev);
 
-	phy_resume(dev->phydev);
+	phy_resume(priv->phydev);
 
 	if (priv->eee.eee_enabled)
 		bcmgenet_eee_enable_set(dev, true);
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h
index 0f0868c56f05..1e2dc34d331a 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h
@@ -597,6 +597,7 @@ struct bcmgenet_priv {
 
 	/* MDIO bus variables */
 	wait_queue_head_t wq;
+	struct phy_device *phydev;
 	bool internal_phy;
 	struct device_node *phy_dn;
 	struct device_node *mdio_dn;
diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c
index e907acd81da9..457c3bc8cfff 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmmii.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c
@@ -86,7 +86,7 @@ static int bcmgenet_mii_write(struct mii_bus *bus, int phy_id,
 void bcmgenet_mii_setup(struct net_device *dev)
 {
 	struct bcmgenet_priv *priv = netdev_priv(dev);
-	struct phy_device *phydev = dev->phydev;
+	struct phy_device *phydev = priv->phydev;
 	u32 reg, cmd_bits = 0;
 	bool status_changed = false;
 
@@ -183,9 +183,9 @@ void bcmgenet_mii_reset(struct net_device *dev)
 	if (GENET_IS_V4(priv))
 		return;
 
-	if (dev->phydev) {
-		phy_init_hw(dev->phydev);
-		phy_start_aneg(dev->phydev);
+	if (priv->phydev) {
+		phy_init_hw(priv->phydev);
+		phy_start_aneg(priv->phydev);
 	}
 }
 
@@ -236,7 +236,6 @@ static void bcmgenet_internal_phy_setup(struct net_device *dev)
 
 static void bcmgenet_moca_phy_setup(struct bcmgenet_priv *priv)
 {
-	struct net_device *ndev = priv->dev;
 	u32 reg;
 
 	/* Speed settings are set in bcmgenet_mii_setup() */
@@ -245,14 +244,14 @@ static void bcmgenet_moca_phy_setup(struct bcmgenet_priv *priv)
 	bcmgenet_sys_writel(priv, reg, SYS_PORT_CTRL);
 
 	if (priv->hw_params->flags & GENET_HAS_MOCA_LINK_DET)
-		fixed_phy_set_link_update(ndev->phydev,
+		fixed_phy_set_link_update(priv->phydev,
 					  bcmgenet_fixed_phy_link_update);
 }
 
 int bcmgenet_mii_config(struct net_device *dev)
 {
 	struct bcmgenet_priv *priv = netdev_priv(dev);
-	struct phy_device *phydev = dev->phydev;
+	struct phy_device *phydev = priv->phydev;
 	struct device *kdev = &priv->pdev->dev;
 	const char *phy_name = NULL;
 	u32 id_mode_dis = 0;
@@ -303,7 +302,7 @@ int bcmgenet_mii_config(struct net_device *dev)
 		 * capabilities, use that knowledge to also configure the
 		 * Reverse MII interface correctly.
 		 */
-		if ((phydev->supported & PHY_BASIC_FEATURES) ==
+		if ((priv->phydev->supported & PHY_BASIC_FEATURES) ==
 				PHY_BASIC_FEATURES)
 			port_ctrl = PORT_MODE_EXT_RVMII_25;
 		else
@@ -372,7 +371,7 @@ int bcmgenet_mii_probe(struct net_device *dev)
 			return -ENODEV;
 		}
 	} else {
-		phydev = dev->phydev;
+		phydev = priv->phydev;
 		phydev->dev_flags = phy_flags;
 
 		ret = phy_connect_direct(dev, phydev, bcmgenet_mii_setup,
@@ -383,6 +382,8 @@ int bcmgenet_mii_probe(struct net_device *dev)
 		}
 	}
 
+	priv->phydev = phydev;
+
 	/* Configure port multiplexer based on what the probed PHY device since
 	 * reading the 'max-speed' property determines the maximum supported
 	 * PHY speed which is needed for bcmgenet_mii_config() to configure
@@ -390,7 +391,7 @@ int bcmgenet_mii_probe(struct net_device *dev)
 	 */
 	ret = bcmgenet_mii_config(dev);
 	if (ret) {
-		phy_disconnect(phydev);
+		phy_disconnect(priv->phydev);
 		return ret;
 	}
 
@@ -400,7 +401,7 @@ int bcmgenet_mii_probe(struct net_device *dev)
 	 * Ethernet MAC ISRs
 	 */
 	if (priv->internal_phy)
-		phydev->irq = PHY_IGNORE_INTERRUPT;
+		priv->phydev->irq = PHY_IGNORE_INTERRUPT;
 
 	return 0;
 }
@@ -605,6 +606,7 @@ static int bcmgenet_mii_pd_init(struct bcmgenet_priv *priv)
 
 	}
 
+	priv->phydev = phydev;
 	priv->phy_interface = pd->phy_interface;
 
 	return 0;
diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c
index ff300f7cf529..a927a730da10 100644
--- a/drivers/net/ethernet/broadcom/tg3.c
+++ b/drivers/net/ethernet/broadcom/tg3.c
@@ -12079,95 +12079,107 @@ static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
 	return ret;
 }
 
-static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int tg3_get_link_ksettings(struct net_device *dev,
+				  struct ethtool_link_ksettings *cmd)
 {
 	struct tg3 *tp = netdev_priv(dev);
+	u32 supported, advertising;
 
 	if (tg3_flag(tp, USE_PHYLIB)) {
 		struct phy_device *phydev;
 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
 			return -EAGAIN;
 		phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
-		return phy_ethtool_gset(phydev, cmd);
+		return phy_ethtool_ksettings_get(phydev, cmd);
 	}
 
-	cmd->supported = (SUPPORTED_Autoneg);
+	supported = (SUPPORTED_Autoneg);
 
 	if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
-		cmd->supported |= (SUPPORTED_1000baseT_Half |
-				   SUPPORTED_1000baseT_Full);
+		supported |= (SUPPORTED_1000baseT_Half |
+			      SUPPORTED_1000baseT_Full);
 
 	if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
-		cmd->supported |= (SUPPORTED_100baseT_Half |
-				  SUPPORTED_100baseT_Full |
-				  SUPPORTED_10baseT_Half |
-				  SUPPORTED_10baseT_Full |
-				  SUPPORTED_TP);
-		cmd->port = PORT_TP;
+		supported |= (SUPPORTED_100baseT_Half |
+			      SUPPORTED_100baseT_Full |
+			      SUPPORTED_10baseT_Half |
+			      SUPPORTED_10baseT_Full |
+			      SUPPORTED_TP);
+		cmd->base.port = PORT_TP;
 	} else {
-		cmd->supported |= SUPPORTED_FIBRE;
-		cmd->port = PORT_FIBRE;
+		supported |= SUPPORTED_FIBRE;
+		cmd->base.port = PORT_FIBRE;
 	}
+	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
+						supported);
 
-	cmd->advertising = tp->link_config.advertising;
+	advertising = tp->link_config.advertising;
 	if (tg3_flag(tp, PAUSE_AUTONEG)) {
 		if (tp->link_config.flowctrl & FLOW_CTRL_RX) {
 			if (tp->link_config.flowctrl & FLOW_CTRL_TX) {
-				cmd->advertising |= ADVERTISED_Pause;
+				advertising |= ADVERTISED_Pause;
 			} else {
-				cmd->advertising |= ADVERTISED_Pause |
-						    ADVERTISED_Asym_Pause;
+				advertising |= ADVERTISED_Pause |
+					ADVERTISED_Asym_Pause;
 			}
 		} else if (tp->link_config.flowctrl & FLOW_CTRL_TX) {
-			cmd->advertising |= ADVERTISED_Asym_Pause;
+			advertising |= ADVERTISED_Asym_Pause;
 		}
 	}
+	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
+						advertising);
+
 	if (netif_running(dev) && tp->link_up) {
-		ethtool_cmd_speed_set(cmd, tp->link_config.active_speed);
-		cmd->duplex = tp->link_config.active_duplex;
-		cmd->lp_advertising = tp->link_config.rmt_adv;
+		cmd->base.speed = tp->link_config.active_speed;
+		cmd->base.duplex = tp->link_config.active_duplex;
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.lp_advertising,
+			tp->link_config.rmt_adv);
+
 		if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
 			if (tp->phy_flags & TG3_PHYFLG_MDIX_STATE)
-				cmd->eth_tp_mdix = ETH_TP_MDI_X;
+				cmd->base.eth_tp_mdix = ETH_TP_MDI_X;
 			else
-				cmd->eth_tp_mdix = ETH_TP_MDI;
+				cmd->base.eth_tp_mdix = ETH_TP_MDI;
 		}
 	} else {
-		ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN);
-		cmd->duplex = DUPLEX_UNKNOWN;
-		cmd->eth_tp_mdix = ETH_TP_MDI_INVALID;
-	}
-	cmd->phy_address = tp->phy_addr;
-	cmd->transceiver = XCVR_INTERNAL;
-	cmd->autoneg = tp->link_config.autoneg;
-	cmd->maxtxpkt = 0;
-	cmd->maxrxpkt = 0;
+		cmd->base.speed = SPEED_UNKNOWN;
+		cmd->base.duplex = DUPLEX_UNKNOWN;
+		cmd->base.eth_tp_mdix = ETH_TP_MDI_INVALID;
+	}
+	cmd->base.phy_address = tp->phy_addr;
+	cmd->base.autoneg = tp->link_config.autoneg;
 	return 0;
 }
 
-static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int tg3_set_link_ksettings(struct net_device *dev,
+				  const struct ethtool_link_ksettings *cmd)
 {
 	struct tg3 *tp = netdev_priv(dev);
-	u32 speed = ethtool_cmd_speed(cmd);
+	u32 speed = cmd->base.speed;
+	u32 advertising;
 
 	if (tg3_flag(tp, USE_PHYLIB)) {
 		struct phy_device *phydev;
 		if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
 			return -EAGAIN;
 		phydev = mdiobus_get_phy(tp->mdio_bus, tp->phy_addr);
-		return phy_ethtool_sset(phydev, cmd);
+		return phy_ethtool_ksettings_set(phydev, cmd);
 	}
 
-	if (cmd->autoneg != AUTONEG_ENABLE &&
-	    cmd->autoneg != AUTONEG_DISABLE)
+	if (cmd->base.autoneg != AUTONEG_ENABLE &&
+	    cmd->base.autoneg != AUTONEG_DISABLE)
 		return -EINVAL;
 
-	if (cmd->autoneg == AUTONEG_DISABLE &&
-	    cmd->duplex != DUPLEX_FULL &&
-	    cmd->duplex != DUPLEX_HALF)
+	if (cmd->base.autoneg == AUTONEG_DISABLE &&
+	    cmd->base.duplex != DUPLEX_FULL &&
+	    cmd->base.duplex != DUPLEX_HALF)
 		return -EINVAL;
 
-	if (cmd->autoneg == AUTONEG_ENABLE) {
+	ethtool_convert_link_mode_to_legacy_u32(&advertising,
+						cmd->link_modes.advertising);
+
+	if (cmd->base.autoneg == AUTONEG_ENABLE) {
 		u32 mask = ADVERTISED_Autoneg |
 			   ADVERTISED_Pause |
 			   ADVERTISED_Asym_Pause;
@@ -12185,7 +12197,7 @@ static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 		else
 			mask |= ADVERTISED_FIBRE;
 
-		if (cmd->advertising & ~mask)
+		if (advertising & ~mask)
 			return -EINVAL;
 
 		mask &= (ADVERTISED_1000baseT_Half |
@@ -12195,13 +12207,13 @@ static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 			 ADVERTISED_10baseT_Half |
 			 ADVERTISED_10baseT_Full);
 
-		cmd->advertising &= mask;
+		advertising &= mask;
 	} else {
 		if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES) {
 			if (speed != SPEED_1000)
 				return -EINVAL;
 
-			if (cmd->duplex != DUPLEX_FULL)
+			if (cmd->base.duplex != DUPLEX_FULL)
 				return -EINVAL;
 		} else {
 			if (speed != SPEED_100 &&
@@ -12212,16 +12224,16 @@ static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 
 	tg3_full_lock(tp, 0);
 
-	tp->link_config.autoneg = cmd->autoneg;
-	if (cmd->autoneg == AUTONEG_ENABLE) {
-		tp->link_config.advertising = (cmd->advertising |
+	tp->link_config.autoneg = cmd->base.autoneg;
+	if (cmd->base.autoneg == AUTONEG_ENABLE) {
+		tp->link_config.advertising = (advertising |
 					      ADVERTISED_Autoneg);
 		tp->link_config.speed = SPEED_UNKNOWN;
 		tp->link_config.duplex = DUPLEX_UNKNOWN;
 	} else {
 		tp->link_config.advertising = 0;
 		tp->link_config.speed = speed;
-		tp->link_config.duplex = cmd->duplex;
+		tp->link_config.duplex = cmd->base.duplex;
 	}
 
 	tp->phy_flags |= TG3_PHYFLG_USER_CONFIGURED;
@@ -12552,10 +12564,6 @@ static int tg3_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
 				info->data = TG3_RSS_MAX_NUM_QS;
 		}
 
-		/* The first interrupt vector only
-		 * handles link interrupts.
-		 */
-		info->data -= 1;
 		return 0;
 
 	default:
@@ -14014,7 +14022,9 @@ static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
 	}
 
 	if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
+	    (!ec->rx_coalesce_usecs) ||
 	    (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
+	    (!ec->tx_coalesce_usecs) ||
 	    (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
 	    (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
 	    (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
@@ -14025,16 +14035,6 @@ static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
 	    (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
 		return -EINVAL;
 
-	/* No rx interrupts will be generated if both are zero */
-	if ((ec->rx_coalesce_usecs == 0) &&
-	    (ec->rx_max_coalesced_frames == 0))
-		return -EINVAL;
-
-	/* No tx interrupts will be generated if both are zero */
-	if ((ec->tx_coalesce_usecs == 0) &&
-	    (ec->tx_max_coalesced_frames == 0))
-		return -EINVAL;
-
 	/* Only copy relevant parameters, ignore all others. */
 	tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
 	tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
@@ -14106,8 +14106,6 @@ static int tg3_get_eee(struct net_device *dev, struct ethtool_eee *edata)
 }
 
 static const struct ethtool_ops tg3_ethtool_ops = {
-	.get_settings		= tg3_get_settings,
-	.set_settings		= tg3_set_settings,
 	.get_drvinfo		= tg3_get_drvinfo,
 	.get_regs_len		= tg3_get_regs_len,
 	.get_regs		= tg3_get_regs,
@@ -14140,6 +14138,8 @@ static const struct ethtool_ops tg3_ethtool_ops = {
 	.get_ts_info		= tg3_get_ts_info,
 	.get_eee		= tg3_get_eee,
 	.set_eee		= tg3_set_eee,
+	.get_link_ksettings	= tg3_get_link_ksettings,
+	.set_link_ksettings	= tg3_set_link_ksettings,
 };
 
 static struct rtnl_link_stats64 *tg3_get_stats64(struct net_device *dev,
@@ -18134,14 +18134,14 @@ static pci_ers_result_t tg3_io_error_detected(struct pci_dev *pdev,
 
 	rtnl_lock();
 
-	/* We needn't recover from permanent error */
-	if (state == pci_channel_io_frozen)
-		tp->pcierr_recovery = true;
-
 	/* We probably don't have netdev yet */
 	if (!netdev || !netif_running(netdev))
 		goto done;
 
+	/* We needn't recover from permanent error */
+	if (state == pci_channel_io_frozen)
+		tp->pcierr_recovery = true;
+
 	tg3_phy_stop(tp);
 
 	tg3_netif_stop(tp);
@@ -18238,7 +18238,7 @@ static void tg3_io_resume(struct pci_dev *pdev)
 
 	rtnl_lock();
 
-	if (!netif_running(netdev))
+	if (!netdev || !netif_running(netdev))
 		goto done;
 
 	tg3_full_lock(tp, 0);
diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c
index 771cc267f217..f9df4b5ae90e 100644
--- a/drivers/net/ethernet/brocade/bna/bnad.c
+++ b/drivers/net/ethernet/brocade/bna/bnad.c
@@ -54,9 +54,7 @@ MODULE_PARM_DESC(bna_debugfs_enable, "Enables debugfs feature, default=1,"
  * Global variables
  */
 static u32 bnad_rxqs_per_cq = 2;
-static u32 bna_id;
-static struct mutex bnad_list_mutex;
-static LIST_HEAD(bnad_list);
+static atomic_t bna_id;
 static const u8 bnad_bcast_addr[] __aligned(2) =
 	{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
@@ -76,23 +74,6 @@ do {								\
 	(_res_info)->res_u.mem_info.len = (_size);		\
 } while (0)
 
-static void
-bnad_add_to_list(struct bnad *bnad)
-{
-	mutex_lock(&bnad_list_mutex);
-	list_add_tail(&bnad->list_entry, &bnad_list);
-	bnad->id = bna_id++;
-	mutex_unlock(&bnad_list_mutex);
-}
-
-static void
-bnad_remove_from_list(struct bnad *bnad)
-{
-	mutex_lock(&bnad_list_mutex);
-	list_del(&bnad->list_entry);
-	mutex_unlock(&bnad_list_mutex);
-}
-
 /*
  * Reinitialize completions in CQ, once Rx is taken down
  */
@@ -3573,14 +3554,12 @@ bnad_lock_init(struct bnad *bnad)
 {
 	spin_lock_init(&bnad->bna_lock);
 	mutex_init(&bnad->conf_mutex);
-	mutex_init(&bnad_list_mutex);
 }
 
 static void
 bnad_lock_uninit(struct bnad *bnad)
 {
 	mutex_destroy(&bnad->conf_mutex);
-	mutex_destroy(&bnad_list_mutex);
 }
 
 /* PCI Initialization */
@@ -3653,7 +3632,7 @@ bnad_pci_probe(struct pci_dev *pdev,
 	}
 	bnad = netdev_priv(netdev);
 	bnad_lock_init(bnad);
-	bnad_add_to_list(bnad);
+	bnad->id = atomic_inc_return(&bna_id) - 1;
 
 	mutex_lock(&bnad->conf_mutex);
 	/*
@@ -3807,7 +3786,6 @@ pci_uninit:
 	bnad_pci_uninit(pdev);
 unlock_mutex:
 	mutex_unlock(&bnad->conf_mutex);
-	bnad_remove_from_list(bnad);
 	bnad_lock_uninit(bnad);
 	free_netdev(netdev);
 	return err;
@@ -3845,7 +3823,6 @@ bnad_pci_remove(struct pci_dev *pdev)
 	bnad_disable_msix(bnad);
 	bnad_pci_uninit(pdev);
 	mutex_unlock(&bnad->conf_mutex);
-	bnad_remove_from_list(bnad);
 	bnad_lock_uninit(bnad);
 	/* Remove the debugfs node for this bnad */
 	kfree(bnad->regdata);
diff --git a/drivers/net/ethernet/brocade/bna/bnad.h b/drivers/net/ethernet/brocade/bna/bnad.h
index f4ed816b93ee..46f7b842b39c 100644
--- a/drivers/net/ethernet/brocade/bna/bnad.h
+++ b/drivers/net/ethernet/brocade/bna/bnad.h
@@ -288,7 +288,6 @@ struct bnad_rx_unmap_q {
 struct bnad {
 	struct net_device	*netdev;
 	u32			id;
-	struct list_head	list_entry;
 
 	/* Data path */
 	struct bnad_tx_info tx_info[BNAD_MAX_TX];
diff --git a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c
index 0e4fdc3dd729..31f61a744d66 100644
--- a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c
+++ b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c
@@ -31,15 +31,10 @@
 #define BNAD_NUM_TXF_COUNTERS 12
 #define BNAD_NUM_RXF_COUNTERS 10
 #define BNAD_NUM_CQ_COUNTERS (3 + 5)
-#define BNAD_NUM_RXQ_COUNTERS 6
+#define BNAD_NUM_RXQ_COUNTERS 7
 #define BNAD_NUM_TXQ_COUNTERS 5
 
-#define BNAD_ETHTOOL_STATS_NUM						\
-	(sizeof(struct rtnl_link_stats64) / sizeof(u64) +	\
-	sizeof(struct bnad_drv_stats) / sizeof(u64) +		\
-	offsetof(struct bfi_enet_stats, rxf_stats[0]) / sizeof(u64))
-
-static const char *bnad_net_stats_strings[BNAD_ETHTOOL_STATS_NUM] = {
+static const char *bnad_net_stats_strings[] = {
 	"rx_packets",
 	"tx_packets",
 	"rx_bytes",
@@ -50,22 +45,10 @@ static const char *bnad_net_stats_strings[BNAD_ETHTOOL_STATS_NUM] = {
 	"tx_dropped",
 	"multicast",
 	"collisions",
-
 	"rx_length_errors",
-	"rx_over_errors",
 	"rx_crc_errors",
 	"rx_frame_errors",
-	"rx_fifo_errors",
-	"rx_missed_errors",
-
-	"tx_aborted_errors",
-	"tx_carrier_errors",
 	"tx_fifo_errors",
-	"tx_heartbeat_errors",
-	"tx_window_errors",
-
-	"rx_compressed",
-	"tx_compressed",
 
 	"netif_queue_stop",
 	"netif_queue_wakeup",
@@ -254,6 +237,8 @@ static const char *bnad_net_stats_strings[BNAD_ETHTOOL_STATS_NUM] = {
 	"fc_tx_fid_parity_errors",
 };
 
+#define BNAD_ETHTOOL_STATS_NUM	ARRAY_SIZE(bnad_net_stats_strings)
+
 static int
 bnad_get_settings(struct net_device *netdev, struct ethtool_cmd *cmd)
 {
@@ -658,6 +643,8 @@ bnad_get_strings(struct net_device *netdev, u32 stringset, u8 *string)
 				string += ETH_GSTRING_LEN;
 				sprintf(string, "rxq%d_allocbuf_failed", q_num);
 				string += ETH_GSTRING_LEN;
+				sprintf(string, "rxq%d_mapbuf_failed", q_num);
+				string += ETH_GSTRING_LEN;
 				sprintf(string, "rxq%d_producer_index", q_num);
 				string += ETH_GSTRING_LEN;
 				sprintf(string, "rxq%d_consumer_index", q_num);
@@ -678,6 +665,9 @@ bnad_get_strings(struct net_device *netdev, u32 stringset, u8 *string)
 					sprintf(string, "rxq%d_allocbuf_failed",
 								q_num);
 					string += ETH_GSTRING_LEN;
+					sprintf(string, "rxq%d_mapbuf_failed",
+						q_num);
+					string += ETH_GSTRING_LEN;
 					sprintf(string, "rxq%d_producer_index",
 								q_num);
 					string += ETH_GSTRING_LEN;
@@ -854,9 +844,9 @@ bnad_get_ethtool_stats(struct net_device *netdev, struct ethtool_stats *stats,
 		       u64 *buf)
 {
 	struct bnad *bnad = netdev_priv(netdev);
-	int i, j, bi;
+	int i, j, bi = 0;
 	unsigned long flags;
-	struct rtnl_link_stats64 *net_stats64;
+	struct rtnl_link_stats64 net_stats64;
 	u64 *stats64;
 	u32 bmap;
 
@@ -871,14 +861,25 @@ bnad_get_ethtool_stats(struct net_device *netdev, struct ethtool_stats *stats,
 	 * under the same lock
 	 */
 	spin_lock_irqsave(&bnad->bna_lock, flags);
-	bi = 0;
-	memset(buf, 0, stats->n_stats * sizeof(u64));
-
-	net_stats64 = (struct rtnl_link_stats64 *)buf;
-	bnad_netdev_qstats_fill(bnad, net_stats64);
-	bnad_netdev_hwstats_fill(bnad, net_stats64);
 
-	bi = sizeof(*net_stats64) / sizeof(u64);
+	memset(&net_stats64, 0, sizeof(net_stats64));
+	bnad_netdev_qstats_fill(bnad, &net_stats64);
+	bnad_netdev_hwstats_fill(bnad, &net_stats64);
+
+	buf[bi++] = net_stats64.rx_packets;
+	buf[bi++] = net_stats64.tx_packets;
+	buf[bi++] = net_stats64.rx_bytes;
+	buf[bi++] = net_stats64.tx_bytes;
+	buf[bi++] = net_stats64.rx_errors;
+	buf[bi++] = net_stats64.tx_errors;
+	buf[bi++] = net_stats64.rx_dropped;
+	buf[bi++] = net_stats64.tx_dropped;
+	buf[bi++] = net_stats64.multicast;
+	buf[bi++] = net_stats64.collisions;
+	buf[bi++] = net_stats64.rx_length_errors;
+	buf[bi++] = net_stats64.rx_crc_errors;
+	buf[bi++] = net_stats64.rx_frame_errors;
+	buf[bi++] = net_stats64.tx_fifo_errors;
 
 	/* Get netif_queue_stopped from stack */
 	bnad->stats.drv_stats.netif_queue_stopped = netif_queue_stopped(netdev);
diff --git a/drivers/net/ethernet/cadence/macb.c b/drivers/net/ethernet/cadence/macb.c
index 89c0cfa9719f..b32444a3ed79 100644
--- a/drivers/net/ethernet/cadence/macb.c
+++ b/drivers/net/ethernet/cadence/macb.c
@@ -541,6 +541,14 @@ static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb)
 	}
 }
 
+static inline void macb_set_addr(struct macb_dma_desc *desc, dma_addr_t addr)
+{
+	desc->addr = (u32)addr;
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	desc->addrh = (u32)(addr >> 32);
+#endif
+}
+
 static void macb_tx_error_task(struct work_struct *work)
 {
 	struct macb_queue	*queue = container_of(work, struct macb_queue,
@@ -621,14 +629,17 @@ static void macb_tx_error_task(struct work_struct *work)
 
 	/* Set end of TX queue */
 	desc = macb_tx_desc(queue, 0);
-	desc->addr = 0;
+	macb_set_addr(desc, 0);
 	desc->ctrl = MACB_BIT(TX_USED);
 
 	/* Make descriptor updates visible to hardware */
 	wmb();
 
 	/* Reinitialize the TX desc queue */
-	queue_writel(queue, TBQP, queue->tx_ring_dma);
+	queue_writel(queue, TBQP, (u32)(queue->tx_ring_dma));
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	queue_writel(queue, TBQPH, (u32)(queue->tx_ring_dma >> 32));
+#endif
 	/* Make TX ring reflect state of hardware */
 	queue->tx_head = 0;
 	queue->tx_tail = 0;
@@ -750,7 +761,7 @@ static void gem_rx_refill(struct macb *bp)
 
 			if (entry == RX_RING_SIZE - 1)
 				paddr |= MACB_BIT(RX_WRAP);
-			bp->rx_ring[entry].addr = paddr;
+			macb_set_addr(&(bp->rx_ring[entry]), paddr);
 			bp->rx_ring[entry].ctrl = 0;
 
 			/* properly align Ethernet header */
@@ -798,7 +809,9 @@ static int gem_rx(struct macb *bp, int budget)
 	int			count = 0;
 
 	while (count < budget) {
-		u32 addr, ctrl;
+		u32 ctrl;
+		dma_addr_t addr;
+		bool rxused;
 
 		entry = macb_rx_ring_wrap(bp->rx_tail);
 		desc = &bp->rx_ring[entry];
@@ -806,10 +819,14 @@ static int gem_rx(struct macb *bp, int budget)
 		/* Make hw descriptor updates visible to CPU */
 		rmb();
 
-		addr = desc->addr;
+		rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false;
+		addr = MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+		addr |= ((u64)(desc->addrh) << 32);
+#endif
 		ctrl = desc->ctrl;
 
-		if (!(addr & MACB_BIT(RX_USED)))
+		if (!rxused)
 			break;
 
 		bp->rx_tail++;
@@ -835,7 +852,6 @@ static int gem_rx(struct macb *bp, int budget)
 		netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);
 
 		skb_put(skb, len);
-		addr = MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, addr));
 		dma_unmap_single(&bp->pdev->dev, addr,
 				 bp->rx_buffer_size, DMA_FROM_DEVICE);
 
@@ -1299,7 +1315,7 @@ static unsigned int macb_tx_map(struct macb *bp,
 			ctrl |= MACB_BIT(TX_WRAP);
 
 		/* Set TX buffer descriptor */
-		desc->addr = tx_skb->mapping;
+		macb_set_addr(desc, tx_skb->mapping);
 		/* desc->addr must be visible to hardware before clearing
 		 * 'TX_USED' bit in desc->ctrl.
 		 */
@@ -1323,6 +1339,24 @@ dma_error:
 	return 0;
 }
 
+static inline int macb_clear_csum(struct sk_buff *skb)
+{
+	/* no change for packets without checksum offloading */
+	if (skb->ip_summed != CHECKSUM_PARTIAL)
+		return 0;
+
+	/* make sure we can modify the header */
+	if (unlikely(skb_cow_head(skb, 0)))
+		return -1;
+
+	/* initialize checksum field
+	 * This is required - at least for Zynq, which otherwise calculates
+	 * wrong UDP header checksums for UDP packets with UDP data len <=2
+	 */
+	*(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0;
+	return 0;
+}
+
 static int macb_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	u16 queue_index = skb_get_queue_mapping(skb);
@@ -1362,6 +1396,11 @@ static int macb_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		return NETDEV_TX_BUSY;
 	}
 
+	if (macb_clear_csum(skb)) {
+		dev_kfree_skb_any(skb);
+		goto unlock;
+	}
+
 	/* Map socket buffer for DMA transfer */
 	if (!macb_tx_map(bp, queue, skb)) {
 		dev_kfree_skb_any(skb);
@@ -1422,6 +1461,9 @@ static void gem_free_rx_buffers(struct macb *bp)
 
 		desc = &bp->rx_ring[i];
 		addr = MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+		addr |= ((u64)(desc->addrh) << 32);
+#endif
 		dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,
 				 DMA_FROM_DEVICE);
 		dev_kfree_skb_any(skb);
@@ -1547,7 +1589,7 @@ static void gem_init_rings(struct macb *bp)
 
 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
 		for (i = 0; i < TX_RING_SIZE; i++) {
-			queue->tx_ring[i].addr = 0;
+			macb_set_addr(&(queue->tx_ring[i]), 0);
 			queue->tx_ring[i].ctrl = MACB_BIT(TX_USED);
 		}
 		queue->tx_ring[TX_RING_SIZE - 1].ctrl |= MACB_BIT(TX_WRAP);
@@ -1694,6 +1736,10 @@ static void macb_configure_dma(struct macb *bp)
 			dmacfg |= GEM_BIT(TXCOEN);
 		else
 			dmacfg &= ~GEM_BIT(TXCOEN);
+
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+		dmacfg |= GEM_BIT(ADDR64);
+#endif
 		netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n",
 			   dmacfg);
 		gem_writel(bp, DMACFG, dmacfg);
@@ -1739,9 +1785,15 @@ static void macb_init_hw(struct macb *bp)
 	macb_configure_dma(bp);
 
 	/* Initialize TX and RX buffers */
-	macb_writel(bp, RBQP, bp->rx_ring_dma);
+	macb_writel(bp, RBQP, (u32)(bp->rx_ring_dma));
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	macb_writel(bp, RBQPH, (u32)(bp->rx_ring_dma >> 32));
+#endif
 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
-		queue_writel(queue, TBQP, queue->tx_ring_dma);
+		queue_writel(queue, TBQP, (u32)(queue->tx_ring_dma));
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+		queue_writel(queue, TBQPH, (u32)(queue->tx_ring_dma >> 32));
+#endif
 
 		/* Enable interrupts */
 		queue_writel(queue, IER,
@@ -2303,7 +2355,8 @@ static void macb_probe_queues(void __iomem *mem,
 }
 
 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
-			 struct clk **hclk, struct clk **tx_clk)
+			 struct clk **hclk, struct clk **tx_clk,
+			 struct clk **rx_clk)
 {
 	int err;
 
@@ -2325,6 +2378,10 @@ static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
 	if (IS_ERR(*tx_clk))
 		*tx_clk = NULL;
 
+	*rx_clk = devm_clk_get(&pdev->dev, "rx_clk");
+	if (IS_ERR(*rx_clk))
+		*rx_clk = NULL;
+
 	err = clk_prepare_enable(*pclk);
 	if (err) {
 		dev_err(&pdev->dev, "failed to enable pclk (%u)\n", err);
@@ -2343,8 +2400,17 @@ static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
 		goto err_disable_hclk;
 	}
 
+	err = clk_prepare_enable(*rx_clk);
+	if (err) {
+		dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err);
+		goto err_disable_txclk;
+	}
+
 	return 0;
 
+err_disable_txclk:
+	clk_disable_unprepare(*tx_clk);
+
 err_disable_hclk:
 	clk_disable_unprepare(*hclk);
 
@@ -2379,6 +2445,9 @@ static int macb_init(struct platform_device *pdev)
 			queue->IDR  = GEM_IDR(hw_q - 1);
 			queue->IMR  = GEM_IMR(hw_q - 1);
 			queue->TBQP = GEM_TBQP(hw_q - 1);
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+			queue->TBQPH = GEM_TBQPH(hw_q -1);
+#endif
 		} else {
 			/* queue0 uses legacy registers */
 			queue->ISR  = MACB_ISR;
@@ -2386,6 +2455,9 @@ static int macb_init(struct platform_device *pdev)
 			queue->IDR  = MACB_IDR;
 			queue->IMR  = MACB_IMR;
 			queue->TBQP = MACB_TBQP;
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+			queue->TBQPH = MACB_TBQPH;
+#endif
 		}
 
 		/* get irq: here we use the linux queue index, not the hardware
@@ -2728,12 +2800,14 @@ static const struct net_device_ops at91ether_netdev_ops = {
 };
 
 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk,
-			      struct clk **hclk, struct clk **tx_clk)
+			      struct clk **hclk, struct clk **tx_clk,
+			      struct clk **rx_clk)
 {
 	int err;
 
 	*hclk = NULL;
 	*tx_clk = NULL;
+	*rx_clk = NULL;
 
 	*pclk = devm_clk_get(&pdev->dev, "ether_clk");
 	if (IS_ERR(*pclk))
@@ -2857,13 +2931,13 @@ MODULE_DEVICE_TABLE(of, macb_dt_ids);
 static int macb_probe(struct platform_device *pdev)
 {
 	int (*clk_init)(struct platform_device *, struct clk **,
-			struct clk **, struct clk **)
+			struct clk **, struct clk **,  struct clk **)
 					      = macb_clk_init;
 	int (*init)(struct platform_device *) = macb_init;
 	struct device_node *np = pdev->dev.of_node;
 	struct device_node *phy_node;
 	const struct macb_config *macb_config = NULL;
-	struct clk *pclk, *hclk = NULL, *tx_clk = NULL;
+	struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL;
 	unsigned int queue_mask, num_queues;
 	struct macb_platform_data *pdata;
 	bool native_io;
@@ -2891,7 +2965,7 @@ static int macb_probe(struct platform_device *pdev)
 		}
 	}
 
-	err = clk_init(pdev, &pclk, &hclk, &tx_clk);
+	err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk);
 	if (err)
 		return err;
 
@@ -2927,6 +3001,7 @@ static int macb_probe(struct platform_device *pdev)
 	bp->pclk = pclk;
 	bp->hclk = hclk;
 	bp->tx_clk = tx_clk;
+	bp->rx_clk = rx_clk;
 	if (macb_config)
 		bp->jumbo_max_len = macb_config->jumbo_max_len;
 
@@ -2935,6 +3010,11 @@ static int macb_probe(struct platform_device *pdev)
 		bp->wol |= MACB_WOL_HAS_MAGIC_PACKET;
 	device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET);
 
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	if (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1)) > GEM_DBW32)
+		dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));
+#endif
+
 	spin_lock_init(&bp->lock);
 
 	/* setup capabilities */
@@ -2945,7 +3025,7 @@ static int macb_probe(struct platform_device *pdev)
 	dev->irq = platform_get_irq(pdev, 0);
 	if (dev->irq < 0) {
 		err = dev->irq;
-		goto err_disable_clocks;
+		goto err_out_free_netdev;
 	}
 
 	mac = of_get_mac_address(np);
@@ -3020,6 +3100,7 @@ err_disable_clocks:
 	clk_disable_unprepare(tx_clk);
 	clk_disable_unprepare(hclk);
 	clk_disable_unprepare(pclk);
+	clk_disable_unprepare(rx_clk);
 
 	return err;
 }
@@ -3036,6 +3117,7 @@ static int macb_remove(struct platform_device *pdev)
 		if (dev->phydev)
 			phy_disconnect(dev->phydev);
 		mdiobus_unregister(bp->mii_bus);
+		dev->phydev = NULL;
 		mdiobus_free(bp->mii_bus);
 
 		/* Shutdown the PHY if there is a GPIO reset */
@@ -3046,6 +3128,7 @@ static int macb_remove(struct platform_device *pdev)
 		clk_disable_unprepare(bp->tx_clk);
 		clk_disable_unprepare(bp->hclk);
 		clk_disable_unprepare(bp->pclk);
+		clk_disable_unprepare(bp->rx_clk);
 		free_netdev(dev);
 	}
 
@@ -3069,6 +3152,7 @@ static int __maybe_unused macb_suspend(struct device *dev)
 		clk_disable_unprepare(bp->tx_clk);
 		clk_disable_unprepare(bp->hclk);
 		clk_disable_unprepare(bp->pclk);
+		clk_disable_unprepare(bp->rx_clk);
 	}
 
 	return 0;
@@ -3088,6 +3172,7 @@ static int __maybe_unused macb_resume(struct device *dev)
 		clk_prepare_enable(bp->pclk);
 		clk_prepare_enable(bp->hclk);
 		clk_prepare_enable(bp->tx_clk);
+		clk_prepare_enable(bp->rx_clk);
 	}
 
 	netif_device_attach(netdev);
diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h
index 36893d8958d4..8bed4b52fef5 100644
--- a/drivers/net/ethernet/cadence/macb.h
+++ b/drivers/net/ethernet/cadence/macb.h
@@ -66,6 +66,8 @@
 #define MACB_USRIO		0x00c0
 #define MACB_WOL		0x00c4
 #define MACB_MID		0x00fc
+#define MACB_TBQPH		0x04C8
+#define MACB_RBQPH		0x04D4
 
 /* GEM register offsets. */
 #define GEM_NCFGR		0x0004 /* Network Config */
@@ -139,6 +141,7 @@
 
 #define GEM_ISR(hw_q)		(0x0400 + ((hw_q) << 2))
 #define GEM_TBQP(hw_q)		(0x0440 + ((hw_q) << 2))
+#define GEM_TBQPH(hw_q)		(0x04C8)
 #define GEM_RBQP(hw_q)		(0x0480 + ((hw_q) << 2))
 #define GEM_IER(hw_q)		(0x0600 + ((hw_q) << 2))
 #define GEM_IDR(hw_q)		(0x0620 + ((hw_q) << 2))
@@ -249,6 +252,8 @@
 #define GEM_RXBS_SIZE		8
 #define GEM_DDRP_OFFSET		24 /* disc_when_no_ahb */
 #define GEM_DDRP_SIZE		1
+#define GEM_ADDR64_OFFSET	30 /* Address bus width - 64b or 32b */
+#define GEM_ADDR64_SIZE		1
 
 
 /* Bitfields in NSR */
@@ -403,11 +408,11 @@
 #define MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII	0x00000004
 #define MACB_CAPS_NO_GIGABIT_HALF		0x00000008
 #define MACB_CAPS_USRIO_DISABLED		0x00000010
+#define MACB_CAPS_JUMBO				0x00000020
 #define MACB_CAPS_FIFO_MODE			0x10000000
 #define MACB_CAPS_GIGABIT_MODE_AVAILABLE	0x20000000
 #define MACB_CAPS_SG_DISABLED			0x40000000
 #define MACB_CAPS_MACB_IS_GEM			0x80000000
-#define MACB_CAPS_JUMBO				0x00000010
 
 /* Bit manipulation macros */
 #define MACB_BIT(name)					\
@@ -474,6 +479,10 @@
 struct macb_dma_desc {
 	u32	addr;
 	u32	ctrl;
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	u32     addrh;
+	u32     resvd;
+#endif
 };
 
 /* DMA descriptor bitfields */
@@ -763,7 +772,8 @@ struct macb_config {
 	u32			caps;
 	unsigned int		dma_burst_length;
 	int	(*clk_init)(struct platform_device *pdev, struct clk **pclk,
-			    struct clk **hclk, struct clk **tx_clk);
+			    struct clk **hclk, struct clk **tx_clk,
+			    struct clk **rx_clk);
 	int	(*init)(struct platform_device *pdev);
 	int	jumbo_max_len;
 };
@@ -777,6 +787,7 @@ struct macb_queue {
 	unsigned int		IDR;
 	unsigned int		IMR;
 	unsigned int		TBQP;
+	unsigned int		TBQPH;
 
 	unsigned int		tx_head, tx_tail;
 	struct macb_dma_desc	*tx_ring;
@@ -809,6 +820,7 @@ struct macb {
 	struct clk		*pclk;
 	struct clk		*hclk;
 	struct clk		*tx_clk;
+	struct clk		*rx_clk;
 	struct net_device	*dev;
 	struct napi_struct	napi;
 	struct net_device_stats	stats;
diff --git a/drivers/net/ethernet/cavium/Kconfig b/drivers/net/ethernet/cavium/Kconfig
index 0ef232d3331e..92f411c9f0df 100644
--- a/drivers/net/ethernet/cavium/Kconfig
+++ b/drivers/net/ethernet/cavium/Kconfig
@@ -36,10 +36,20 @@ config	THUNDER_NIC_BGX
 	depends on 64BIT
 	select PHYLIB
 	select MDIO_THUNDER
+	select THUNDER_NIC_RGX
 	---help---
 	  This driver supports programming and controlling of MAC
 	  interface from NIC physical function driver.
 
+config	THUNDER_NIC_RGX
+	tristate "Thunder MAC interface driver (RGX)"
+	depends on 64BIT
+	select PHYLIB
+	select MDIO_THUNDER
+	---help---
+	  This driver supports configuring XCV block of RGX interface
+	  present on CN81XX chip.
+
 config LIQUIDIO
 	tristate "Cavium LiquidIO support"
 	depends on 64BIT
@@ -48,7 +58,7 @@ config LIQUIDIO
 	select LIBCRC32C
 	---help---
 	  This driver supports Cavium LiquidIO Intelligent Server Adapters
-	  based on CN66XX and CN68XX chips.
+	  based on CN66XX, CN68XX and CN23XX chips.
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called liquidio.  This is recommended.
diff --git a/drivers/net/ethernet/cavium/liquidio/Makefile b/drivers/net/ethernet/cavium/liquidio/Makefile
index 2f366806835d..5a27b2a44039 100644
--- a/drivers/net/ethernet/cavium/liquidio/Makefile
+++ b/drivers/net/ethernet/cavium/liquidio/Makefile
@@ -3,14 +3,16 @@
 #
 obj-$(CONFIG_LIQUIDIO) += liquidio.o
 
-liquidio-objs := lio_main.o  \
-	      lio_ethtool.o      \
-	      request_manager.o  \
-	      response_manager.o \
-	      octeon_device.o    \
-	      cn66xx_device.o    \
-	      cn68xx_device.o    \
-	      octeon_mem_ops.o   \
-	      octeon_droq.o      \
-	      octeon_console.o   \
-	      octeon_nic.o
+liquidio-$(CONFIG_LIQUIDIO) += lio_ethtool.o \
+			lio_core.o         \
+			request_manager.o  \
+			response_manager.o \
+			octeon_device.o    \
+			cn66xx_device.o    \
+			cn68xx_device.o    \
+			cn23xx_pf_device.o \
+			octeon_mem_ops.o   \
+			octeon_droq.o      \
+			octeon_nic.o
+
+liquidio-objs := lio_main.o octeon_console.o $(liquidio-y)
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c
new file mode 100644
index 000000000000..bddb198c0b74
--- /dev/null
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c
@@ -0,0 +1,1237 @@
+/**********************************************************************
+* Author: Cavium, Inc.
+*
+* Contact: support@cavium.com
+*          Please include "LiquidIO" in the subject.
+*
+* Copyright (c) 2003-2015 Cavium, Inc.
+*
+* This file is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License, Version 2, as
+* published by the Free Software Foundation.
+*
+* This file is distributed in the hope that it will be useful, but
+* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
+* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
+* NONINFRINGEMENT.  See the GNU General Public License for more
+* details.
+*
+* This file may also be available under a different license from Cavium.
+* Contact Cavium, Inc. for more information
+**********************************************************************/
+
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/vmalloc.h>
+#include "liquidio_common.h"
+#include "octeon_droq.h"
+#include "octeon_iq.h"
+#include "response_manager.h"
+#include "octeon_device.h"
+#include "cn23xx_pf_device.h"
+#include "octeon_main.h"
+
+#define RESET_NOTDONE 0
+#define RESET_DONE 1
+
+/* Change the value of SLI Packet Input Jabber Register to allow
+ * VXLAN TSO packets which can be 64424 bytes, exceeding the
+ * MAX_GSO_SIZE we supplied to the kernel
+ */
+#define CN23XX_INPUT_JABBER 64600
+
+#define LIOLUT_RING_DISTRIBUTION 9
+const int liolut_num_vfs_to_rings_per_vf[LIOLUT_RING_DISTRIBUTION] = {
+	0, 8, 4, 2, 2, 2, 1, 1, 1
+};
+
+void cn23xx_dump_pf_initialized_regs(struct octeon_device *oct)
+{
+	int i = 0;
+	u32 regval = 0;
+	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
+
+	/*In cn23xx_soft_reset*/
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%llx\n",
+		"CN23XX_WIN_WR_MASK_REG", CVM_CAST64(CN23XX_WIN_WR_MASK_REG),
+		CVM_CAST64(octeon_read_csr64(oct, CN23XX_WIN_WR_MASK_REG)));
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"CN23XX_SLI_SCRATCH1", CVM_CAST64(CN23XX_SLI_SCRATCH1),
+		CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_SCRATCH1)));
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"CN23XX_RST_SOFT_RST", CN23XX_RST_SOFT_RST,
+		lio_pci_readq(oct, CN23XX_RST_SOFT_RST));
+
+	/*In cn23xx_set_dpi_regs*/
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"CN23XX_DPI_DMA_CONTROL", CN23XX_DPI_DMA_CONTROL,
+		lio_pci_readq(oct, CN23XX_DPI_DMA_CONTROL));
+
+	for (i = 0; i < 6; i++) {
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_DPI_DMA_ENG_ENB", i,
+			CN23XX_DPI_DMA_ENG_ENB(i),
+			lio_pci_readq(oct, CN23XX_DPI_DMA_ENG_ENB(i)));
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_DPI_DMA_ENG_BUF", i,
+			CN23XX_DPI_DMA_ENG_BUF(i),
+			lio_pci_readq(oct, CN23XX_DPI_DMA_ENG_BUF(i)));
+	}
+
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n", "CN23XX_DPI_CTL",
+		CN23XX_DPI_CTL, lio_pci_readq(oct, CN23XX_DPI_CTL));
+
+	/*In cn23xx_setup_pcie_mps and cn23xx_setup_pcie_mrrs */
+	pci_read_config_dword(oct->pci_dev, CN23XX_CONFIG_PCIE_DEVCTL, &regval);
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"CN23XX_CONFIG_PCIE_DEVCTL",
+		CVM_CAST64(CN23XX_CONFIG_PCIE_DEVCTL), CVM_CAST64(regval));
+
+	dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+		"CN23XX_DPI_SLI_PRTX_CFG", oct->pcie_port,
+		CN23XX_DPI_SLI_PRTX_CFG(oct->pcie_port),
+		lio_pci_readq(oct, CN23XX_DPI_SLI_PRTX_CFG(oct->pcie_port)));
+
+	/*In cn23xx_specific_regs_setup */
+	dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+		"CN23XX_SLI_S2M_PORTX_CTL", oct->pcie_port,
+		CVM_CAST64(CN23XX_SLI_S2M_PORTX_CTL(oct->pcie_port)),
+		CVM_CAST64(octeon_read_csr64(
+			oct, CN23XX_SLI_S2M_PORTX_CTL(oct->pcie_port))));
+
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"CN23XX_SLI_RING_RST", CVM_CAST64(CN23XX_SLI_PKT_IOQ_RING_RST),
+		(u64)octeon_read_csr64(oct, CN23XX_SLI_PKT_IOQ_RING_RST));
+
+	/*In cn23xx_setup_global_mac_regs*/
+	for (i = 0; i < CN23XX_MAX_MACS; i++) {
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_PKT_MAC_RINFO64", i,
+			CVM_CAST64(CN23XX_SLI_PKT_MAC_RINFO64(i, oct->pf_num)),
+			CVM_CAST64(octeon_read_csr64
+				(oct, CN23XX_SLI_PKT_MAC_RINFO64
+					(i, oct->pf_num))));
+	}
+
+	/*In cn23xx_setup_global_input_regs*/
+	for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++) {
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_IQ_PKT_CONTROL64", i,
+			CVM_CAST64(CN23XX_SLI_IQ_PKT_CONTROL64(i)),
+			CVM_CAST64(octeon_read_csr64
+				(oct, CN23XX_SLI_IQ_PKT_CONTROL64(i))));
+	}
+
+	/*In cn23xx_setup_global_output_regs*/
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"CN23XX_SLI_OQ_WMARK", CVM_CAST64(CN23XX_SLI_OQ_WMARK),
+		CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_OQ_WMARK)));
+
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_OQ_PKT_CONTROL", i,
+			CVM_CAST64(CN23XX_SLI_OQ_PKT_CONTROL(i)),
+			CVM_CAST64(octeon_read_csr(
+				oct, CN23XX_SLI_OQ_PKT_CONTROL(i))));
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_OQ_PKT_INT_LEVELS", i,
+			CVM_CAST64(CN23XX_SLI_OQ_PKT_INT_LEVELS(i)),
+			CVM_CAST64(octeon_read_csr64(
+				oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(i))));
+	}
+
+	/*In cn23xx_enable_interrupt and cn23xx_disable_interrupt*/
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"cn23xx->intr_enb_reg64",
+		CVM_CAST64((long)(cn23xx->intr_enb_reg64)),
+		CVM_CAST64(readq(cn23xx->intr_enb_reg64)));
+
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"cn23xx->intr_sum_reg64",
+		CVM_CAST64((long)(cn23xx->intr_sum_reg64)),
+		CVM_CAST64(readq(cn23xx->intr_sum_reg64)));
+
+	/*In cn23xx_setup_iq_regs*/
+	for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++) {
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_IQ_BASE_ADDR64", i,
+			CVM_CAST64(CN23XX_SLI_IQ_BASE_ADDR64(i)),
+			CVM_CAST64(octeon_read_csr64(
+				oct, CN23XX_SLI_IQ_BASE_ADDR64(i))));
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_IQ_SIZE", i,
+			CVM_CAST64(CN23XX_SLI_IQ_SIZE(i)),
+			CVM_CAST64(octeon_read_csr
+				(oct, CN23XX_SLI_IQ_SIZE(i))));
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_IQ_DOORBELL", i,
+			CVM_CAST64(CN23XX_SLI_IQ_DOORBELL(i)),
+			CVM_CAST64(octeon_read_csr64(
+				oct, CN23XX_SLI_IQ_DOORBELL(i))));
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_IQ_INSTR_COUNT64", i,
+			CVM_CAST64(CN23XX_SLI_IQ_INSTR_COUNT64(i)),
+			CVM_CAST64(octeon_read_csr64(
+				oct, CN23XX_SLI_IQ_INSTR_COUNT64(i))));
+	}
+
+	/*In cn23xx_setup_oq_regs*/
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_OQ_BASE_ADDR64", i,
+			CVM_CAST64(CN23XX_SLI_OQ_BASE_ADDR64(i)),
+			CVM_CAST64(octeon_read_csr64(
+				oct, CN23XX_SLI_OQ_BASE_ADDR64(i))));
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_OQ_SIZE", i,
+			CVM_CAST64(CN23XX_SLI_OQ_SIZE(i)),
+			CVM_CAST64(octeon_read_csr
+				(oct, CN23XX_SLI_OQ_SIZE(i))));
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_OQ_BUFF_INFO_SIZE", i,
+			CVM_CAST64(CN23XX_SLI_OQ_BUFF_INFO_SIZE(i)),
+			CVM_CAST64(octeon_read_csr(
+				oct, CN23XX_SLI_OQ_BUFF_INFO_SIZE(i))));
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_OQ_PKTS_SENT", i,
+			CVM_CAST64(CN23XX_SLI_OQ_PKTS_SENT(i)),
+			CVM_CAST64(octeon_read_csr64(
+				oct, CN23XX_SLI_OQ_PKTS_SENT(i))));
+		dev_dbg(&oct->pci_dev->dev, "%s(%d)[%llx] : 0x%016llx\n",
+			"CN23XX_SLI_OQ_PKTS_CREDIT", i,
+			CVM_CAST64(CN23XX_SLI_OQ_PKTS_CREDIT(i)),
+			CVM_CAST64(octeon_read_csr64(
+				oct, CN23XX_SLI_OQ_PKTS_CREDIT(i))));
+	}
+
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"CN23XX_SLI_PKT_TIME_INT",
+		CVM_CAST64(CN23XX_SLI_PKT_TIME_INT),
+		CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_PKT_TIME_INT)));
+	dev_dbg(&oct->pci_dev->dev, "%s[%llx] : 0x%016llx\n",
+		"CN23XX_SLI_PKT_CNT_INT",
+		CVM_CAST64(CN23XX_SLI_PKT_CNT_INT),
+		CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_PKT_CNT_INT)));
+}
+
+static int cn23xx_pf_soft_reset(struct octeon_device *oct)
+{
+	octeon_write_csr64(oct, CN23XX_WIN_WR_MASK_REG, 0xFF);
+
+	dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: BIST enabled for CN23XX soft reset\n",
+		oct->octeon_id);
+
+	octeon_write_csr64(oct, CN23XX_SLI_SCRATCH1, 0x1234ULL);
+
+	/* Initiate chip-wide soft reset */
+	lio_pci_readq(oct, CN23XX_RST_SOFT_RST);
+	lio_pci_writeq(oct, 1, CN23XX_RST_SOFT_RST);
+
+	/* Wait for 100ms as Octeon resets. */
+	mdelay(100);
+
+	if (octeon_read_csr64(oct, CN23XX_SLI_SCRATCH1) == 0x1234ULL) {
+		dev_err(&oct->pci_dev->dev, "OCTEON[%d]: Soft reset failed\n",
+			oct->octeon_id);
+		return 1;
+	}
+
+	dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: Reset completed\n",
+		oct->octeon_id);
+
+	/* restore the  reset value*/
+	octeon_write_csr64(oct, CN23XX_WIN_WR_MASK_REG, 0xFF);
+
+	return 0;
+}
+
+static void cn23xx_enable_error_reporting(struct octeon_device *oct)
+{
+	u32 regval;
+	u32 uncorrectable_err_mask, corrtable_err_status;
+
+	pci_read_config_dword(oct->pci_dev, CN23XX_CONFIG_PCIE_DEVCTL, &regval);
+	if (regval & CN23XX_CONFIG_PCIE_DEVCTL_MASK) {
+		uncorrectable_err_mask = 0;
+		corrtable_err_status = 0;
+		pci_read_config_dword(oct->pci_dev,
+				      CN23XX_CONFIG_PCIE_UNCORRECT_ERR_MASK,
+				      &uncorrectable_err_mask);
+		pci_read_config_dword(oct->pci_dev,
+				      CN23XX_CONFIG_PCIE_CORRECT_ERR_STATUS,
+				      &corrtable_err_status);
+		dev_err(&oct->pci_dev->dev, "PCI-E Fatal error detected;\n"
+				 "\tdev_ctl_status_reg = 0x%08x\n"
+				 "\tuncorrectable_error_mask_reg = 0x%08x\n"
+				 "\tcorrectable_error_status_reg = 0x%08x\n",
+			    regval, uncorrectable_err_mask,
+			    corrtable_err_status);
+	}
+
+	regval |= 0xf; /* Enable Link error reporting */
+
+	dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: Enabling PCI-E error reporting..\n",
+		oct->octeon_id);
+	pci_write_config_dword(oct->pci_dev, CN23XX_CONFIG_PCIE_DEVCTL, regval);
+}
+
+static u32 cn23xx_coprocessor_clock(struct octeon_device *oct)
+{
+	/* Bits 29:24 of RST_BOOT[PNR_MUL] holds the ref.clock MULTIPLIER
+	 * for SLI.
+	 */
+
+	/* TBD: get the info in Hand-shake */
+	return (((lio_pci_readq(oct, CN23XX_RST_BOOT) >> 24) & 0x3f) * 50);
+}
+
+u32 cn23xx_pf_get_oq_ticks(struct octeon_device *oct, u32 time_intr_in_us)
+{
+	/* This gives the SLI clock per microsec */
+	u32 oqticks_per_us = cn23xx_coprocessor_clock(oct);
+
+	oct->pfvf_hsword.coproc_tics_per_us = oqticks_per_us;
+
+	/* This gives the clock cycles per millisecond */
+	oqticks_per_us *= 1000;
+
+	/* This gives the oq ticks (1024 core clock cycles) per millisecond */
+	oqticks_per_us /= 1024;
+
+	/* time_intr is in microseconds. The next 2 steps gives the oq ticks
+	 *  corressponding to time_intr.
+	 */
+	oqticks_per_us *= time_intr_in_us;
+	oqticks_per_us /= 1000;
+
+	return oqticks_per_us;
+}
+
+static void cn23xx_setup_global_mac_regs(struct octeon_device *oct)
+{
+	u64 reg_val;
+	u16 mac_no = oct->pcie_port;
+	u16 pf_num = oct->pf_num;
+
+	/* programming SRN and TRS for each MAC(0..3)  */
+
+	dev_dbg(&oct->pci_dev->dev, "%s:Using pcie port %d\n",
+		__func__, mac_no);
+	/* By default, mapping all 64 IOQs to  a single MACs */
+
+	reg_val =
+	    octeon_read_csr64(oct, CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num));
+
+	if (oct->rev_id == OCTEON_CN23XX_REV_1_1) {
+		/* setting SRN <6:0>  */
+		reg_val = pf_num * CN23XX_MAX_RINGS_PER_PF_PASS_1_1;
+	} else {
+		/* setting SRN <6:0>  */
+		reg_val = pf_num * CN23XX_MAX_RINGS_PER_PF;
+	}
+
+	/* setting TRS <23:16> */
+	reg_val = reg_val |
+		  (oct->sriov_info.trs << CN23XX_PKT_MAC_CTL_RINFO_TRS_BIT_POS);
+	/* write these settings to MAC register */
+	octeon_write_csr64(oct, CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num),
+			   reg_val);
+
+	dev_dbg(&oct->pci_dev->dev, "SLI_PKT_MAC(%d)_PF(%d)_RINFO : 0x%016llx\n",
+		mac_no, pf_num, (u64)octeon_read_csr64
+		(oct, CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num)));
+}
+
+static int cn23xx_reset_io_queues(struct octeon_device *oct)
+{
+	int ret_val = 0;
+	u64 d64;
+	u32 q_no, srn, ern;
+	u32 loop = 1000;
+
+	srn = oct->sriov_info.pf_srn;
+	ern = srn + oct->sriov_info.num_pf_rings;
+
+	/*As per HRM reg description, s/w cant write 0 to ENB. */
+	/*to make the queue off, need to set the RST bit. */
+
+	/* Reset the Enable bit for all the 64 IQs.  */
+	for (q_no = srn; q_no < ern; q_no++) {
+		/* set RST bit to 1. This bit applies to both IQ and OQ */
+		d64 = octeon_read_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
+		d64 = d64 | CN23XX_PKT_INPUT_CTL_RST;
+		octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), d64);
+	}
+
+	/*wait until the RST bit is clear or the RST and quite bits are set*/
+	for (q_no = srn; q_no < ern; q_no++) {
+		u64 reg_val = octeon_read_csr64(oct,
+					CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
+		while ((READ_ONCE(reg_val) & CN23XX_PKT_INPUT_CTL_RST) &&
+		       !(READ_ONCE(reg_val) & CN23XX_PKT_INPUT_CTL_QUIET) &&
+		       loop--) {
+			WRITE_ONCE(reg_val, octeon_read_csr64(
+			    oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no)));
+		}
+		if (!loop) {
+			dev_err(&oct->pci_dev->dev,
+				"clearing the reset reg failed or setting the quiet reg failed for qno: %u\n",
+				q_no);
+			return -1;
+		}
+		WRITE_ONCE(reg_val, READ_ONCE(reg_val) &
+			~CN23XX_PKT_INPUT_CTL_RST);
+		octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
+				   READ_ONCE(reg_val));
+
+		WRITE_ONCE(reg_val, octeon_read_csr64(
+			   oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no)));
+		if (READ_ONCE(reg_val) & CN23XX_PKT_INPUT_CTL_RST) {
+			dev_err(&oct->pci_dev->dev,
+				"clearing the reset failed for qno: %u\n",
+				q_no);
+			ret_val = -1;
+		}
+	}
+
+	return ret_val;
+}
+
+static int cn23xx_pf_setup_global_input_regs(struct octeon_device *oct)
+{
+	u32 q_no, ern, srn;
+	u64 pf_num;
+	u64 intr_threshold, reg_val;
+	struct octeon_instr_queue *iq;
+	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
+
+	pf_num = oct->pf_num;
+
+	srn = oct->sriov_info.pf_srn;
+	ern = srn + oct->sriov_info.num_pf_rings;
+
+	if (cn23xx_reset_io_queues(oct))
+		return -1;
+
+	/** Set the MAC_NUM and PVF_NUM in IQ_PKT_CONTROL reg
+	* for all queues.Only PF can set these bits.
+	* bits 29:30 indicate the MAC num.
+	* bits 32:47 indicate the PVF num.
+	*/
+	for (q_no = 0; q_no < ern; q_no++) {
+		reg_val = oct->pcie_port << CN23XX_PKT_INPUT_CTL_MAC_NUM_POS;
+		reg_val |= pf_num << CN23XX_PKT_INPUT_CTL_PF_NUM_POS;
+
+		octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
+				   reg_val);
+	}
+
+	/* Select ES, RO, NS, RDSIZE,DPTR Fomat#0 for
+	 * pf queues
+	 */
+	for (q_no = srn; q_no < ern; q_no++) {
+		void __iomem *inst_cnt_reg;
+
+		iq = oct->instr_queue[q_no];
+		if (iq)
+			inst_cnt_reg = iq->inst_cnt_reg;
+		else
+			inst_cnt_reg = (u8 *)oct->mmio[0].hw_addr +
+				       CN23XX_SLI_IQ_INSTR_COUNT64(q_no);
+
+		reg_val =
+		    octeon_read_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
+
+		reg_val |= CN23XX_PKT_INPUT_CTL_MASK;
+
+		octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
+				   reg_val);
+
+		/* Set WMARK level for triggering PI_INT */
+		/* intr_threshold = CN23XX_DEF_IQ_INTR_THRESHOLD & */
+		intr_threshold = CFG_GET_IQ_INTR_PKT(cn23xx->conf) &
+				 CN23XX_PKT_IN_DONE_WMARK_MASK;
+
+		writeq((readq(inst_cnt_reg) &
+			~(CN23XX_PKT_IN_DONE_WMARK_MASK <<
+			  CN23XX_PKT_IN_DONE_WMARK_BIT_POS)) |
+		       (intr_threshold << CN23XX_PKT_IN_DONE_WMARK_BIT_POS),
+		       inst_cnt_reg);
+	}
+	return 0;
+}
+
+static void cn23xx_pf_setup_global_output_regs(struct octeon_device *oct)
+{
+	u32 reg_val;
+	u32 q_no, ern, srn;
+	u64 time_threshold;
+
+	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
+
+	srn = oct->sriov_info.pf_srn;
+	ern = srn + oct->sriov_info.num_pf_rings;
+
+	if (CFG_GET_IS_SLI_BP_ON(cn23xx->conf)) {
+		octeon_write_csr64(oct, CN23XX_SLI_OQ_WMARK, 32);
+	} else {
+		/** Set Output queue watermark to 0 to disable backpressure */
+		octeon_write_csr64(oct, CN23XX_SLI_OQ_WMARK, 0);
+	}
+
+	for (q_no = srn; q_no < ern; q_no++) {
+		reg_val = octeon_read_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(q_no));
+
+		/* set IPTR & DPTR */
+		reg_val |=
+		    (CN23XX_PKT_OUTPUT_CTL_IPTR | CN23XX_PKT_OUTPUT_CTL_DPTR);
+
+		/* reset BMODE */
+		reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_BMODE);
+
+		/* No Relaxed Ordering, No Snoop, 64-bit Byte swap
+		 * for Output Queue ScatterList
+		 * reset ROR_P, NSR_P
+		 */
+		reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ROR_P);
+		reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_NSR_P);
+
+#ifdef __LITTLE_ENDIAN_BITFIELD
+		reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ES_P);
+#else
+		reg_val |= (CN23XX_PKT_OUTPUT_CTL_ES_P);
+#endif
+		/* No Relaxed Ordering, No Snoop, 64-bit Byte swap
+		 * for Output Queue Data
+		 * reset ROR, NSR
+		 */
+		reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ROR);
+		reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_NSR);
+		/* set the ES bit */
+		reg_val |= (CN23XX_PKT_OUTPUT_CTL_ES);
+
+		/* write all the selected settings */
+		octeon_write_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(q_no), reg_val);
+
+		/* Enabling these interrupt in oct->fn_list.enable_interrupt()
+		 * routine which called after IOQ init.
+		 * Set up interrupt packet and time thresholds
+		 * for all the OQs
+		 */
+		time_threshold = cn23xx_pf_get_oq_ticks(
+		    oct, (u32)CFG_GET_OQ_INTR_TIME(cn23xx->conf));
+
+		octeon_write_csr64(oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(q_no),
+				   (CFG_GET_OQ_INTR_PKT(cn23xx->conf) |
+				    (time_threshold << 32)));
+	}
+
+	/** Setting the water mark level for pko back pressure **/
+	writeq(0x40, (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OQ_WMARK);
+
+	/** Disabling setting OQs in reset when ring has no dorebells
+	  * enabling this will cause of head of line blocking
+	  */
+	/* Do it only for pass1.1. and pass1.2 */
+	if ((oct->rev_id == OCTEON_CN23XX_REV_1_0) ||
+	    (oct->rev_id == OCTEON_CN23XX_REV_1_1))
+		writeq(readq((u8 *)oct->mmio[0].hw_addr +
+				     CN23XX_SLI_GBL_CONTROL) | 0x2,
+		       (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_GBL_CONTROL);
+
+	/** Enable channel-level backpressure */
+	if (oct->pf_num)
+		writeq(0xffffffffffffffffULL,
+		       (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OUT_BP_EN2_W1S);
+	else
+		writeq(0xffffffffffffffffULL,
+		       (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OUT_BP_EN_W1S);
+}
+
+static int cn23xx_setup_pf_device_regs(struct octeon_device *oct)
+{
+	cn23xx_enable_error_reporting(oct);
+
+	/* program the MAC(0..3)_RINFO before setting up input/output regs */
+	cn23xx_setup_global_mac_regs(oct);
+
+	if (cn23xx_pf_setup_global_input_regs(oct))
+		return -1;
+
+	cn23xx_pf_setup_global_output_regs(oct);
+
+	/* Default error timeout value should be 0x200000 to avoid host hang
+	 * when reads invalid register
+	 */
+	octeon_write_csr64(oct, CN23XX_SLI_WINDOW_CTL,
+			   CN23XX_SLI_WINDOW_CTL_DEFAULT);
+
+	/* set SLI_PKT_IN_JABBER to handle large VXLAN packets */
+	octeon_write_csr64(oct, CN23XX_SLI_PKT_IN_JABBER, CN23XX_INPUT_JABBER);
+	return 0;
+}
+
+static void cn23xx_setup_iq_regs(struct octeon_device *oct, u32 iq_no)
+{
+	struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
+	u64 pkt_in_done;
+
+	iq_no += oct->sriov_info.pf_srn;
+
+	/* Write the start of the input queue's ring and its size  */
+	octeon_write_csr64(oct, CN23XX_SLI_IQ_BASE_ADDR64(iq_no),
+			   iq->base_addr_dma);
+	octeon_write_csr(oct, CN23XX_SLI_IQ_SIZE(iq_no), iq->max_count);
+
+	/* Remember the doorbell & instruction count register addr
+	 * for this queue
+	 */
+	iq->doorbell_reg =
+	    (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_IQ_DOORBELL(iq_no);
+	iq->inst_cnt_reg =
+	    (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_IQ_INSTR_COUNT64(iq_no);
+	dev_dbg(&oct->pci_dev->dev, "InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p\n",
+		iq_no, iq->doorbell_reg, iq->inst_cnt_reg);
+
+	/* Store the current instruction counter (used in flush_iq
+	 * calculation)
+	 */
+	pkt_in_done = readq(iq->inst_cnt_reg);
+
+	if (oct->msix_on) {
+		/* Set CINT_ENB to enable IQ interrupt   */
+		writeq((pkt_in_done | CN23XX_INTR_CINT_ENB),
+		       iq->inst_cnt_reg);
+	} else {
+		/* Clear the count by writing back what we read, but don't
+		 * enable interrupts
+		 */
+		writeq(pkt_in_done, iq->inst_cnt_reg);
+	}
+
+	iq->reset_instr_cnt = 0;
+}
+
+static void cn23xx_setup_oq_regs(struct octeon_device *oct, u32 oq_no)
+{
+	u32 reg_val;
+	struct octeon_droq *droq = oct->droq[oq_no];
+	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
+	u64 time_threshold;
+	u64 cnt_threshold;
+
+	oq_no += oct->sriov_info.pf_srn;
+
+	octeon_write_csr64(oct, CN23XX_SLI_OQ_BASE_ADDR64(oq_no),
+			   droq->desc_ring_dma);
+	octeon_write_csr(oct, CN23XX_SLI_OQ_SIZE(oq_no), droq->max_count);
+
+	octeon_write_csr(oct, CN23XX_SLI_OQ_BUFF_INFO_SIZE(oq_no),
+			 (droq->buffer_size | (OCT_RH_SIZE << 16)));
+
+	/* Get the mapped address of the pkt_sent and pkts_credit regs */
+	droq->pkts_sent_reg =
+	    (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OQ_PKTS_SENT(oq_no);
+	droq->pkts_credit_reg =
+	    (u8 *)oct->mmio[0].hw_addr + CN23XX_SLI_OQ_PKTS_CREDIT(oq_no);
+
+	if (!oct->msix_on) {
+		/* Enable this output queue to generate Packet Timer Interrupt
+		 */
+		reg_val =
+		    octeon_read_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(oq_no));
+		reg_val |= CN23XX_PKT_OUTPUT_CTL_TENB;
+		octeon_write_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(oq_no),
+				 reg_val);
+
+		/* Enable this output queue to generate Packet Count Interrupt
+		 */
+		reg_val =
+		    octeon_read_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(oq_no));
+		reg_val |= CN23XX_PKT_OUTPUT_CTL_CENB;
+		octeon_write_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(oq_no),
+				 reg_val);
+	} else {
+		time_threshold = cn23xx_pf_get_oq_ticks(
+		    oct, (u32)CFG_GET_OQ_INTR_TIME(cn23xx->conf));
+		cnt_threshold = (u32)CFG_GET_OQ_INTR_PKT(cn23xx->conf);
+
+		octeon_write_csr64(
+		    oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(oq_no),
+		    ((time_threshold << 32 | cnt_threshold)));
+	}
+}
+
+static int cn23xx_enable_io_queues(struct octeon_device *oct)
+{
+	u64 reg_val;
+	u32 srn, ern, q_no;
+	u32 loop = 1000;
+
+	srn = oct->sriov_info.pf_srn;
+	ern = srn + oct->num_iqs;
+
+	for (q_no = srn; q_no < ern; q_no++) {
+		/* set the corresponding IQ IS_64B bit */
+		if (oct->io_qmask.iq64B & BIT_ULL(q_no - srn)) {
+			reg_val = octeon_read_csr64(
+			    oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
+			reg_val = reg_val | CN23XX_PKT_INPUT_CTL_IS_64B;
+			octeon_write_csr64(
+			    oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), reg_val);
+		}
+
+		/* set the corresponding IQ ENB bit */
+		if (oct->io_qmask.iq & BIT_ULL(q_no - srn)) {
+			/* IOQs are in reset by default in PEM2 mode,
+			 * clearing reset bit
+			 */
+			reg_val = octeon_read_csr64(
+			    oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
+
+			if (reg_val & CN23XX_PKT_INPUT_CTL_RST) {
+				while ((reg_val & CN23XX_PKT_INPUT_CTL_RST) &&
+				       !(reg_val &
+					 CN23XX_PKT_INPUT_CTL_QUIET) &&
+				       loop--) {
+					reg_val = octeon_read_csr64(
+					    oct,
+					    CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
+				}
+				if (!loop) {
+					dev_err(&oct->pci_dev->dev,
+						"clearing the reset reg failed or setting the quiet reg failed for qno: %u\n",
+						q_no);
+					return -1;
+				}
+				reg_val = reg_val & ~CN23XX_PKT_INPUT_CTL_RST;
+				octeon_write_csr64(
+				    oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
+				    reg_val);
+
+				reg_val = octeon_read_csr64(
+				    oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
+				if (reg_val & CN23XX_PKT_INPUT_CTL_RST) {
+					dev_err(&oct->pci_dev->dev,
+						"clearing the reset failed for qno: %u\n",
+						q_no);
+					return -1;
+				}
+			}
+			reg_val = octeon_read_csr64(
+			    oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
+			reg_val = reg_val | CN23XX_PKT_INPUT_CTL_RING_ENB;
+			octeon_write_csr64(
+			    oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), reg_val);
+		}
+	}
+	for (q_no = srn; q_no < ern; q_no++) {
+		u32 reg_val;
+		/* set the corresponding OQ ENB bit */
+		if (oct->io_qmask.oq & BIT_ULL(q_no - srn)) {
+			reg_val = octeon_read_csr(
+			    oct, CN23XX_SLI_OQ_PKT_CONTROL(q_no));
+			reg_val = reg_val | CN23XX_PKT_OUTPUT_CTL_RING_ENB;
+			octeon_write_csr(oct, CN23XX_SLI_OQ_PKT_CONTROL(q_no),
+					 reg_val);
+		}
+	}
+	return 0;
+}
+
+static void cn23xx_disable_io_queues(struct octeon_device *oct)
+{
+	int q_no, loop;
+	u64 d64;
+	u32 d32;
+	u32 srn, ern;
+
+	srn = oct->sriov_info.pf_srn;
+	ern = srn + oct->num_iqs;
+
+	/*** Disable Input Queues. ***/
+	for (q_no = srn; q_no < ern; q_no++) {
+		loop = HZ;
+
+		/* start the Reset for a particular ring */
+		WRITE_ONCE(d64, octeon_read_csr64(
+			   oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no)));
+		WRITE_ONCE(d64, READ_ONCE(d64) &
+					(~(CN23XX_PKT_INPUT_CTL_RING_ENB)));
+		WRITE_ONCE(d64, READ_ONCE(d64) | CN23XX_PKT_INPUT_CTL_RST);
+		octeon_write_csr64(oct, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
+				   READ_ONCE(d64));
+
+		/* Wait until hardware indicates that the particular IQ
+		 * is out of reset.
+		 */
+		WRITE_ONCE(d64, octeon_read_csr64(
+					oct, CN23XX_SLI_PKT_IOQ_RING_RST));
+		while (!(READ_ONCE(d64) & BIT_ULL(q_no)) && loop--) {
+			WRITE_ONCE(d64, octeon_read_csr64(
+					oct, CN23XX_SLI_PKT_IOQ_RING_RST));
+			schedule_timeout_uninterruptible(1);
+		}
+
+		/* Reset the doorbell register for this Input Queue. */
+		octeon_write_csr(oct, CN23XX_SLI_IQ_DOORBELL(q_no), 0xFFFFFFFF);
+		while (octeon_read_csr64(oct, CN23XX_SLI_IQ_DOORBELL(q_no)) &&
+		       loop--) {
+			schedule_timeout_uninterruptible(1);
+		}
+	}
+
+	/*** Disable Output Queues. ***/
+	for (q_no = srn; q_no < ern; q_no++) {
+		loop = HZ;
+
+		/* Wait until hardware indicates that the particular IQ
+		 * is out of reset.It given that SLI_PKT_RING_RST is
+		 * common for both IQs and OQs
+		 */
+		WRITE_ONCE(d64, octeon_read_csr64(
+					oct, CN23XX_SLI_PKT_IOQ_RING_RST));
+		while (!(READ_ONCE(d64) & BIT_ULL(q_no)) && loop--) {
+			WRITE_ONCE(d64, octeon_read_csr64(
+					oct, CN23XX_SLI_PKT_IOQ_RING_RST));
+			schedule_timeout_uninterruptible(1);
+		}
+
+		/* Reset the doorbell register for this Output Queue. */
+		octeon_write_csr(oct, CN23XX_SLI_OQ_PKTS_CREDIT(q_no),
+				 0xFFFFFFFF);
+		while (octeon_read_csr64(oct,
+					 CN23XX_SLI_OQ_PKTS_CREDIT(q_no)) &&
+		       loop--) {
+			schedule_timeout_uninterruptible(1);
+		}
+
+		/* clear the SLI_PKT(0..63)_CNTS[CNT] reg value */
+		WRITE_ONCE(d32, octeon_read_csr(
+					oct, CN23XX_SLI_OQ_PKTS_SENT(q_no)));
+		octeon_write_csr(oct, CN23XX_SLI_OQ_PKTS_SENT(q_no),
+				 READ_ONCE(d32));
+	}
+}
+
+static u64 cn23xx_pf_msix_interrupt_handler(void *dev)
+{
+	struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev;
+	struct octeon_device *oct = ioq_vector->oct_dev;
+	u64 pkts_sent;
+	u64 ret = 0;
+	struct octeon_droq *droq = oct->droq[ioq_vector->droq_index];
+
+	dev_dbg(&oct->pci_dev->dev, "In %s octeon_dev @ %p\n", __func__, oct);
+
+	if (!droq) {
+		dev_err(&oct->pci_dev->dev, "23XX bringup FIXME: oct pfnum:%d ioq_vector->ioq_num :%d droq is NULL\n",
+			oct->pf_num, ioq_vector->ioq_num);
+		return 0;
+	}
+
+	pkts_sent = readq(droq->pkts_sent_reg);
+
+	/* If our device has interrupted, then proceed. Also check
+	 * for all f's if interrupt was triggered on an error
+	 * and the PCI read fails.
+	 */
+	if (!pkts_sent || (pkts_sent == 0xFFFFFFFFFFFFFFFFULL))
+		return ret;
+
+	/* Write count reg in sli_pkt_cnts to clear these int.*/
+	if ((pkts_sent & CN23XX_INTR_PO_INT) ||
+	    (pkts_sent & CN23XX_INTR_PI_INT)) {
+		if (pkts_sent & CN23XX_INTR_PO_INT)
+			ret |= MSIX_PO_INT;
+	}
+
+	if (pkts_sent & CN23XX_INTR_PI_INT)
+		/* We will clear the count when we update the read_index. */
+		ret |= MSIX_PI_INT;
+
+	/* Never need to handle msix mbox intr for pf. They arrive on the last
+	 * msix
+	 */
+	return ret;
+}
+
+static irqreturn_t cn23xx_interrupt_handler(void *dev)
+{
+	struct octeon_device *oct = (struct octeon_device *)dev;
+	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
+	u64 intr64;
+
+	dev_dbg(&oct->pci_dev->dev, "In %s octeon_dev @ %p\n", __func__, oct);
+	intr64 = readq(cn23xx->intr_sum_reg64);
+
+	oct->int_status = 0;
+
+	if (intr64 & CN23XX_INTR_ERR)
+		dev_err(&oct->pci_dev->dev, "OCTEON[%d]: Error Intr: 0x%016llx\n",
+			oct->octeon_id, CVM_CAST64(intr64));
+
+	if (oct->msix_on != LIO_FLAG_MSIX_ENABLED) {
+		if (intr64 & CN23XX_INTR_PKT_DATA)
+			oct->int_status |= OCT_DEV_INTR_PKT_DATA;
+	}
+
+	if (intr64 & (CN23XX_INTR_DMA0_FORCE))
+		oct->int_status |= OCT_DEV_INTR_DMA0_FORCE;
+	if (intr64 & (CN23XX_INTR_DMA1_FORCE))
+		oct->int_status |= OCT_DEV_INTR_DMA1_FORCE;
+
+	/* Clear the current interrupts */
+	writeq(intr64, cn23xx->intr_sum_reg64);
+
+	return IRQ_HANDLED;
+}
+
+static void cn23xx_bar1_idx_setup(struct octeon_device *oct, u64 core_addr,
+				  u32 idx, int valid)
+{
+	u64 bar1;
+	u64 reg_adr;
+
+	if (!valid) {
+		reg_adr = lio_pci_readq(
+			oct, CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
+		WRITE_ONCE(bar1, reg_adr);
+		lio_pci_writeq(oct, (READ_ONCE(bar1) & 0xFFFFFFFEULL),
+			       CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
+		reg_adr = lio_pci_readq(
+			oct, CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
+		WRITE_ONCE(bar1, reg_adr);
+		return;
+	}
+
+	/*  The PEM(0..3)_BAR1_INDEX(0..15)[ADDR_IDX]<23:4> stores
+	 *  bits <41:22> of the Core Addr
+	 */
+	lio_pci_writeq(oct, (((core_addr >> 22) << 4) | PCI_BAR1_MASK),
+		       CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
+
+	WRITE_ONCE(bar1, lio_pci_readq(
+		   oct, CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx)));
+}
+
+static void cn23xx_bar1_idx_write(struct octeon_device *oct, u32 idx, u32 mask)
+{
+	lio_pci_writeq(oct, mask,
+		       CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
+}
+
+static u32 cn23xx_bar1_idx_read(struct octeon_device *oct, u32 idx)
+{
+	return (u32)lio_pci_readq(
+	    oct, CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
+}
+
+/* always call with lock held */
+static u32 cn23xx_update_read_index(struct octeon_instr_queue *iq)
+{
+	u32 new_idx;
+	u32 last_done;
+	u32 pkt_in_done = readl(iq->inst_cnt_reg);
+
+	last_done = pkt_in_done - iq->pkt_in_done;
+	iq->pkt_in_done = pkt_in_done;
+
+	/* Modulo of the new index with the IQ size will give us
+	 * the new index.  The iq->reset_instr_cnt is always zero for
+	 * cn23xx, so no extra adjustments are needed.
+	 */
+	new_idx = (iq->octeon_read_index +
+		   (u32)(last_done & CN23XX_PKT_IN_DONE_CNT_MASK)) %
+		  iq->max_count;
+
+	return new_idx;
+}
+
+static void cn23xx_enable_pf_interrupt(struct octeon_device *oct, u8 intr_flag)
+{
+	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
+	u64 intr_val = 0;
+
+	/*  Divide the single write to multiple writes based on the flag. */
+	/* Enable Interrupt */
+	if (intr_flag == OCTEON_ALL_INTR) {
+		writeq(cn23xx->intr_mask64, cn23xx->intr_enb_reg64);
+	} else if (intr_flag & OCTEON_OUTPUT_INTR) {
+		intr_val = readq(cn23xx->intr_enb_reg64);
+		intr_val |= CN23XX_INTR_PKT_DATA;
+		writeq(intr_val, cn23xx->intr_enb_reg64);
+	}
+}
+
+static void cn23xx_disable_pf_interrupt(struct octeon_device *oct, u8 intr_flag)
+{
+	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
+	u64 intr_val = 0;
+
+	/* Disable Interrupts */
+	if (intr_flag == OCTEON_ALL_INTR) {
+		writeq(0, cn23xx->intr_enb_reg64);
+	} else if (intr_flag & OCTEON_OUTPUT_INTR) {
+		intr_val = readq(cn23xx->intr_enb_reg64);
+		intr_val &= ~CN23XX_INTR_PKT_DATA;
+		writeq(intr_val, cn23xx->intr_enb_reg64);
+	}
+}
+
+static void cn23xx_get_pcie_qlmport(struct octeon_device *oct)
+{
+	oct->pcie_port = (octeon_read_csr(oct, CN23XX_SLI_MAC_NUMBER)) & 0xff;
+
+	dev_dbg(&oct->pci_dev->dev, "OCTEON: CN23xx uses PCIE Port %d\n",
+		oct->pcie_port);
+}
+
+static void cn23xx_get_pf_num(struct octeon_device *oct)
+{
+	u32 fdl_bit = 0;
+
+	/** Read Function Dependency Link reg to get the function number */
+	pci_read_config_dword(oct->pci_dev, CN23XX_PCIE_SRIOV_FDL, &fdl_bit);
+	oct->pf_num = ((fdl_bit >> CN23XX_PCIE_SRIOV_FDL_BIT_POS) &
+		       CN23XX_PCIE_SRIOV_FDL_MASK);
+}
+
+static void cn23xx_setup_reg_address(struct octeon_device *oct)
+{
+	u8 __iomem *bar0_pciaddr = oct->mmio[0].hw_addr;
+	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
+
+	oct->reg_list.pci_win_wr_addr_hi =
+	    (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_ADDR_HI);
+	oct->reg_list.pci_win_wr_addr_lo =
+	    (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_ADDR_LO);
+	oct->reg_list.pci_win_wr_addr =
+	    (u64 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_ADDR64);
+
+	oct->reg_list.pci_win_rd_addr_hi =
+	    (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_ADDR_HI);
+	oct->reg_list.pci_win_rd_addr_lo =
+	    (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_ADDR_LO);
+	oct->reg_list.pci_win_rd_addr =
+	    (u64 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_ADDR64);
+
+	oct->reg_list.pci_win_wr_data_hi =
+	    (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_DATA_HI);
+	oct->reg_list.pci_win_wr_data_lo =
+	    (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_DATA_LO);
+	oct->reg_list.pci_win_wr_data =
+	    (u64 __iomem *)(bar0_pciaddr + CN23XX_WIN_WR_DATA64);
+
+	oct->reg_list.pci_win_rd_data_hi =
+	    (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_DATA_HI);
+	oct->reg_list.pci_win_rd_data_lo =
+	    (u32 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_DATA_LO);
+	oct->reg_list.pci_win_rd_data =
+	    (u64 __iomem *)(bar0_pciaddr + CN23XX_WIN_RD_DATA64);
+
+	cn23xx_get_pcie_qlmport(oct);
+
+	cn23xx->intr_mask64 = CN23XX_INTR_MASK;
+	if (!oct->msix_on)
+		cn23xx->intr_mask64 |= CN23XX_INTR_PKT_TIME;
+	if (oct->rev_id >= OCTEON_CN23XX_REV_1_1)
+		cn23xx->intr_mask64 |= CN23XX_INTR_VF_MBOX;
+
+	cn23xx->intr_sum_reg64 =
+	    bar0_pciaddr +
+	    CN23XX_SLI_MAC_PF_INT_SUM64(oct->pcie_port, oct->pf_num);
+	cn23xx->intr_enb_reg64 =
+	    bar0_pciaddr +
+	    CN23XX_SLI_MAC_PF_INT_ENB64(oct->pcie_port, oct->pf_num);
+}
+
+static int cn23xx_sriov_config(struct octeon_device *oct)
+{
+	u32 total_rings;
+	struct octeon_cn23xx_pf *cn23xx = (struct octeon_cn23xx_pf *)oct->chip;
+	/* num_vfs is already filled for us */
+	u32 pf_srn, num_pf_rings;
+
+	cn23xx->conf =
+	    (struct octeon_config *)oct_get_config_info(oct, LIO_23XX);
+	switch (oct->rev_id) {
+	case OCTEON_CN23XX_REV_1_0:
+		total_rings = CN23XX_MAX_RINGS_PER_PF_PASS_1_0;
+		break;
+	case OCTEON_CN23XX_REV_1_1:
+		total_rings = CN23XX_MAX_RINGS_PER_PF_PASS_1_1;
+		break;
+	default:
+		total_rings = CN23XX_MAX_RINGS_PER_PF;
+		break;
+	}
+	if (!oct->sriov_info.num_pf_rings) {
+		if (total_rings > num_present_cpus())
+			num_pf_rings = num_present_cpus();
+		else
+			num_pf_rings = total_rings;
+	} else {
+		num_pf_rings = oct->sriov_info.num_pf_rings;
+
+		if (num_pf_rings > total_rings) {
+			dev_warn(&oct->pci_dev->dev,
+				 "num_queues_per_pf requested %u is more than available rings. Reducing to %u\n",
+				 num_pf_rings, total_rings);
+			num_pf_rings = total_rings;
+		}
+	}
+
+	total_rings = num_pf_rings;
+	/* the first ring of the pf */
+	pf_srn = total_rings - num_pf_rings;
+
+	oct->sriov_info.trs = total_rings;
+	oct->sriov_info.pf_srn = pf_srn;
+	oct->sriov_info.num_pf_rings = num_pf_rings;
+	dev_dbg(&oct->pci_dev->dev, "trs:%d pf_srn:%d num_pf_rings:%d\n",
+		oct->sriov_info.trs, oct->sriov_info.pf_srn,
+		oct->sriov_info.num_pf_rings);
+	return 0;
+}
+
+int setup_cn23xx_octeon_pf_device(struct octeon_device *oct)
+{
+	if (octeon_map_pci_barx(oct, 0, 0))
+		return 1;
+
+	if (octeon_map_pci_barx(oct, 1, MAX_BAR1_IOREMAP_SIZE)) {
+		dev_err(&oct->pci_dev->dev, "%s CN23XX BAR1 map failed\n",
+			__func__);
+		octeon_unmap_pci_barx(oct, 0);
+		return 1;
+	}
+
+	cn23xx_get_pf_num(oct);
+
+	if (cn23xx_sriov_config(oct)) {
+		octeon_unmap_pci_barx(oct, 0);
+		octeon_unmap_pci_barx(oct, 1);
+		return 1;
+	}
+
+	octeon_write_csr64(oct, CN23XX_SLI_MAC_CREDIT_CNT, 0x3F802080802080ULL);
+
+	oct->fn_list.setup_iq_regs = cn23xx_setup_iq_regs;
+	oct->fn_list.setup_oq_regs = cn23xx_setup_oq_regs;
+	oct->fn_list.process_interrupt_regs = cn23xx_interrupt_handler;
+	oct->fn_list.msix_interrupt_handler = cn23xx_pf_msix_interrupt_handler;
+
+	oct->fn_list.soft_reset = cn23xx_pf_soft_reset;
+	oct->fn_list.setup_device_regs = cn23xx_setup_pf_device_regs;
+	oct->fn_list.update_iq_read_idx = cn23xx_update_read_index;
+
+	oct->fn_list.bar1_idx_setup = cn23xx_bar1_idx_setup;
+	oct->fn_list.bar1_idx_write = cn23xx_bar1_idx_write;
+	oct->fn_list.bar1_idx_read = cn23xx_bar1_idx_read;
+
+	oct->fn_list.enable_interrupt = cn23xx_enable_pf_interrupt;
+	oct->fn_list.disable_interrupt = cn23xx_disable_pf_interrupt;
+
+	oct->fn_list.enable_io_queues = cn23xx_enable_io_queues;
+	oct->fn_list.disable_io_queues = cn23xx_disable_io_queues;
+
+	cn23xx_setup_reg_address(oct);
+
+	oct->coproc_clock_rate = 1000000ULL * cn23xx_coprocessor_clock(oct);
+
+	return 0;
+}
+
+int validate_cn23xx_pf_config_info(struct octeon_device *oct,
+				   struct octeon_config *conf23xx)
+{
+	if (CFG_GET_IQ_MAX_Q(conf23xx) > CN23XX_MAX_INPUT_QUEUES) {
+		dev_err(&oct->pci_dev->dev, "%s: Num IQ (%d) exceeds Max (%d)\n",
+			__func__, CFG_GET_IQ_MAX_Q(conf23xx),
+			CN23XX_MAX_INPUT_QUEUES);
+		return 1;
+	}
+
+	if (CFG_GET_OQ_MAX_Q(conf23xx) > CN23XX_MAX_OUTPUT_QUEUES) {
+		dev_err(&oct->pci_dev->dev, "%s: Num OQ (%d) exceeds Max (%d)\n",
+			__func__, CFG_GET_OQ_MAX_Q(conf23xx),
+			CN23XX_MAX_OUTPUT_QUEUES);
+		return 1;
+	}
+
+	if (CFG_GET_IQ_INSTR_TYPE(conf23xx) != OCTEON_32BYTE_INSTR &&
+	    CFG_GET_IQ_INSTR_TYPE(conf23xx) != OCTEON_64BYTE_INSTR) {
+		dev_err(&oct->pci_dev->dev, "%s: Invalid instr type for IQ\n",
+			__func__);
+		return 1;
+	}
+
+	if (!(CFG_GET_OQ_INFO_PTR(conf23xx)) ||
+	    !(CFG_GET_OQ_REFILL_THRESHOLD(conf23xx))) {
+		dev_err(&oct->pci_dev->dev, "%s: Invalid parameter for OQ\n",
+			__func__);
+		return 1;
+	}
+
+	if (!(CFG_GET_OQ_INTR_TIME(conf23xx))) {
+		dev_err(&oct->pci_dev->dev, "%s: Invalid parameter for OQ\n",
+			__func__);
+		return 1;
+	}
+
+	return 0;
+}
+
+void cn23xx_dump_iq_regs(struct octeon_device *oct)
+{
+	u32 regval, q_no;
+
+	dev_dbg(&oct->pci_dev->dev, "SLI_IQ_DOORBELL_0 [0x%x]: 0x%016llx\n",
+		CN23XX_SLI_IQ_DOORBELL(0),
+		CVM_CAST64(octeon_read_csr64
+			(oct, CN23XX_SLI_IQ_DOORBELL(0))));
+
+	dev_dbg(&oct->pci_dev->dev, "SLI_IQ_BASEADDR_0 [0x%x]: 0x%016llx\n",
+		CN23XX_SLI_IQ_BASE_ADDR64(0),
+		CVM_CAST64(octeon_read_csr64
+			(oct, CN23XX_SLI_IQ_BASE_ADDR64(0))));
+
+	dev_dbg(&oct->pci_dev->dev, "SLI_IQ_FIFO_RSIZE_0 [0x%x]: 0x%016llx\n",
+		CN23XX_SLI_IQ_SIZE(0),
+		CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_IQ_SIZE(0))));
+
+	dev_dbg(&oct->pci_dev->dev, "SLI_CTL_STATUS [0x%x]: 0x%016llx\n",
+		CN23XX_SLI_CTL_STATUS,
+		CVM_CAST64(octeon_read_csr64(oct, CN23XX_SLI_CTL_STATUS)));
+
+	for (q_no = 0; q_no < CN23XX_MAX_INPUT_QUEUES; q_no++) {
+		dev_dbg(&oct->pci_dev->dev, "SLI_PKT[%d]_INPUT_CTL [0x%x]: 0x%016llx\n",
+			q_no, CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
+			CVM_CAST64(octeon_read_csr64
+				(oct,
+					CN23XX_SLI_IQ_PKT_CONTROL64(q_no))));
+	}
+
+	pci_read_config_dword(oct->pci_dev, CN23XX_CONFIG_PCIE_DEVCTL, &regval);
+	dev_dbg(&oct->pci_dev->dev, "Config DevCtl [0x%x]: 0x%08x\n",
+		CN23XX_CONFIG_PCIE_DEVCTL, regval);
+
+	dev_dbg(&oct->pci_dev->dev, "SLI_PRT[%d]_CFG [0x%llx]: 0x%016llx\n",
+		oct->pcie_port, CN23XX_DPI_SLI_PRTX_CFG(oct->pcie_port),
+		CVM_CAST64(lio_pci_readq(
+			oct, CN23XX_DPI_SLI_PRTX_CFG(oct->pcie_port))));
+
+	dev_dbg(&oct->pci_dev->dev, "SLI_S2M_PORT[%d]_CTL [0x%x]: 0x%016llx\n",
+		oct->pcie_port, CN23XX_SLI_S2M_PORTX_CTL(oct->pcie_port),
+		CVM_CAST64(octeon_read_csr64(
+			oct, CN23XX_SLI_S2M_PORTX_CTL(oct->pcie_port))));
+}
+
+int cn23xx_fw_loaded(struct octeon_device *oct)
+{
+	u64 val;
+
+	val = octeon_read_csr64(oct, CN23XX_SLI_SCRATCH1);
+	return (val >> 1) & 1ULL;
+}
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h
new file mode 100644
index 000000000000..21b5c9051967
--- /dev/null
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h
@@ -0,0 +1,59 @@
+/**********************************************************************
+* Author: Cavium, Inc.
+*
+* Contact: support@cavium.com
+*          Please include "LiquidIO" in the subject.
+*
+* Copyright (c) 2003-2015 Cavium, Inc.
+*
+* This file is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License, Version 2, as
+* published by the Free Software Foundation.
+*
+* This file is distributed in the hope that it will be useful, but
+* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
+* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
+* NONINFRINGEMENT.  See the GNU General Public License for more
+* details.
+*
+* This file may also be available under a different license from Cavium.
+* Contact Cavium, Inc. for more information
+**********************************************************************/
+
+/*! \file  cn23xx_device.h
+ * \brief Host Driver: Routines that perform CN23XX specific operations.
+*/
+
+#ifndef __CN23XX_PF_DEVICE_H__
+#define __CN23XX_PF_DEVICE_H__
+
+#include "cn23xx_pf_regs.h"
+
+/* Register address and configuration for a CN23XX devices.
+ * If device specific changes need to be made then add a struct to include
+ * device specific fields as shown in the commented section
+ */
+struct octeon_cn23xx_pf {
+	/** PCI interrupt summary register */
+	u8 __iomem *intr_sum_reg64;
+
+	/** PCI interrupt enable register */
+	u8 __iomem *intr_enb_reg64;
+
+	/** The PCI interrupt mask used by interrupt handler */
+	u64 intr_mask64;
+
+	struct octeon_config *conf;
+};
+
+int setup_cn23xx_octeon_pf_device(struct octeon_device *oct);
+
+int validate_cn23xx_pf_config_info(struct octeon_device *oct,
+				   struct octeon_config *conf23xx);
+
+u32 cn23xx_pf_get_oq_ticks(struct octeon_device *oct, u32 time_intr_in_us);
+
+void cn23xx_dump_pf_initialized_regs(struct octeon_device *oct);
+
+int cn23xx_fw_loaded(struct octeon_device *oct);
+#endif
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h
new file mode 100644
index 000000000000..03d79d95ab75
--- /dev/null
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h
@@ -0,0 +1,604 @@
+/**********************************************************************
+* Author: Cavium, Inc.
+*
+* Contact: support@cavium.com
+*          Please include "LiquidIO" in the subject.
+*
+* Copyright (c) 2003-2015 Cavium, Inc.
+*
+* This file is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License, Version 2, as
+* published by the Free Software Foundation.
+*
+* This file is distributed in the hope that it will be useful, but
+* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
+* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
+* NONINFRINGEMENT.  See the GNU General Public License for more
+* details.
+*
+* This file may also be available under a different license from Cavium.
+* Contact Cavium, Inc. for more information
+**********************************************************************/
+
+/*! \file cn23xx_regs.h
+ * \brief Host Driver: Register Address and Register Mask values for
+ * Octeon CN23XX devices.
+*/
+
+#ifndef __CN23XX_PF_REGS_H__
+#define __CN23XX_PF_REGS_H__
+
+#define     CN23XX_CONFIG_VENDOR_ID	0x00
+#define     CN23XX_CONFIG_DEVICE_ID	0x02
+
+#define     CN23XX_CONFIG_XPANSION_BAR             0x38
+
+#define     CN23XX_CONFIG_MSIX_CAP		   0x50
+#define     CN23XX_CONFIG_MSIX_LMSI		   0x54
+#define     CN23XX_CONFIG_MSIX_UMSI		   0x58
+#define     CN23XX_CONFIG_MSIX_MSIMD		   0x5C
+#define     CN23XX_CONFIG_MSIX_MSIMM		   0x60
+#define     CN23XX_CONFIG_MSIX_MSIMP		   0x64
+
+#define     CN23XX_CONFIG_PCIE_CAP                 0x70
+#define     CN23XX_CONFIG_PCIE_DEVCAP              0x74
+#define     CN23XX_CONFIG_PCIE_DEVCTL              0x78
+#define     CN23XX_CONFIG_PCIE_LINKCAP             0x7C
+#define     CN23XX_CONFIG_PCIE_LINKCTL             0x80
+#define     CN23XX_CONFIG_PCIE_SLOTCAP             0x84
+#define     CN23XX_CONFIG_PCIE_SLOTCTL             0x88
+#define     CN23XX_CONFIG_PCIE_DEVCTL2             0x98
+#define     CN23XX_CONFIG_PCIE_LINKCTL2            0xA0
+#define     CN23XX_CONFIG_PCIE_UNCORRECT_ERR_MASK  0x108
+#define     CN23XX_CONFIG_PCIE_CORRECT_ERR_STATUS  0x110
+#define     CN23XX_CONFIG_PCIE_DEVCTL_MASK         0x00040000
+
+#define     CN23XX_PCIE_SRIOV_FDL		   0x188
+#define     CN23XX_PCIE_SRIOV_FDL_BIT_POS	   0x10
+#define     CN23XX_PCIE_SRIOV_FDL_MASK		   0xFF
+
+#define     CN23XX_CONFIG_PCIE_FLTMSK              0x720
+
+#define     CN23XX_CONFIG_SRIOV_VFDEVID            0x190
+
+#define     CN23XX_CONFIG_SRIOV_BAR_START	   0x19C
+#define     CN23XX_CONFIG_SRIOV_BARX(i)		\
+		(CN23XX_CONFIG_SRIOV_BAR_START + (i * 4))
+#define     CN23XX_CONFIG_SRIOV_BAR_PF		   0x08
+#define     CN23XX_CONFIG_SRIOV_BAR_64BIT	   0x04
+#define     CN23XX_CONFIG_SRIOV_BAR_IO		   0x01
+
+/* ##############  BAR0 Registers ################ */
+
+#define    CN23XX_SLI_CTL_PORT_START               0x286E0
+#define    CN23XX_PORT_OFFSET                      0x10
+
+#define    CN23XX_SLI_CTL_PORT(p)                  \
+		(CN23XX_SLI_CTL_PORT_START + ((p) * CN23XX_PORT_OFFSET))
+
+/* 2 scatch registers (64-bit)  */
+#define    CN23XX_SLI_WINDOW_CTL                   0x282E0
+#define    CN23XX_SLI_SCRATCH1                     0x283C0
+#define    CN23XX_SLI_SCRATCH2                     0x283D0
+#define    CN23XX_SLI_WINDOW_CTL_DEFAULT           0x200000ULL
+
+/* 1 registers (64-bit)  - SLI_CTL_STATUS */
+#define    CN23XX_SLI_CTL_STATUS                   0x28570
+
+/* SLI Packet Input Jabber Register (64 bit register)
+ * <31:0> for Byte count for limiting sizes of packet sizes
+ * that are allowed for sli packet inbound packets.
+ * the default value is 0xFA00(=64000).
+ */
+#define    CN23XX_SLI_PKT_IN_JABBER                0x29170
+/* The input jabber is used to determine the TSO max size.
+ * Due to H/W limitation, this need to be reduced to 60000
+ * in order to to H/W TSO and avoid the WQE malfarmation
+ * PKO_BUG_24989_WQE_LEN
+ */
+#define    CN23XX_DEFAULT_INPUT_JABBER             0xEA60 /*60000*/
+
+#define    CN23XX_WIN_WR_ADDR_LO                   0x20000
+#define    CN23XX_WIN_WR_ADDR_HI                   0x20004
+#define    CN23XX_WIN_WR_ADDR64                    CN23XX_WIN_WR_ADDR_LO
+
+#define    CN23XX_WIN_RD_ADDR_LO                   0x20010
+#define    CN23XX_WIN_RD_ADDR_HI                   0x20014
+#define    CN23XX_WIN_RD_ADDR64                    CN23XX_WIN_RD_ADDR_LO
+
+#define    CN23XX_WIN_WR_DATA_LO                   0x20020
+#define    CN23XX_WIN_WR_DATA_HI                   0x20024
+#define    CN23XX_WIN_WR_DATA64                    CN23XX_WIN_WR_DATA_LO
+
+#define    CN23XX_WIN_RD_DATA_LO                   0x20040
+#define    CN23XX_WIN_RD_DATA_HI                   0x20044
+#define    CN23XX_WIN_RD_DATA64                    CN23XX_WIN_RD_DATA_LO
+
+#define    CN23XX_WIN_WR_MASK_LO                   0x20030
+#define    CN23XX_WIN_WR_MASK_HI                   0x20034
+#define    CN23XX_WIN_WR_MASK_REG                  CN23XX_WIN_WR_MASK_LO
+#define    CN23XX_SLI_MAC_CREDIT_CNT               0x23D70
+
+/* 4 registers (64-bit) for mapping IOQs to MACs(PEMs)-
+ * SLI_PKT_MAC(0..3)_PF(0..1)_RINFO
+ */
+#define    CN23XX_SLI_PKT_MAC_RINFO_START64       0x29030
+
+/*1 register (64-bit) to determine whether IOQs are in reset. */
+#define    CN23XX_SLI_PKT_IOQ_RING_RST            0x291E0
+
+/* Each Input Queue register is at a 16-byte Offset in BAR0 */
+#define    CN23XX_IQ_OFFSET                       0x20000
+
+#define    CN23XX_MAC_RINFO_OFFSET                0x20
+#define    CN23XX_PF_RINFO_OFFSET                 0x10
+
+#define CN23XX_SLI_PKT_MAC_RINFO64(mac, pf)		\
+		(CN23XX_SLI_PKT_MAC_RINFO_START64 +     \
+		 ((mac) * CN23XX_MAC_RINFO_OFFSET) +	\
+		 ((pf) * CN23XX_PF_RINFO_OFFSET))
+
+/** mask for total rings, setting TRS to base */
+#define    CN23XX_PKT_MAC_CTL_RINFO_TRS               BIT_ULL(16)
+/** mask for starting ring number: setting SRN <6:0> = 0x7F */
+#define    CN23XX_PKT_MAC_CTL_RINFO_SRN               (0x7F)
+
+/* Starting bit of the TRS field in CN23XX_SLI_PKT_MAC_RINFO64 register */
+#define    CN23XX_PKT_MAC_CTL_RINFO_TRS_BIT_POS     16
+/* Starting bit of SRN field in CN23XX_SLI_PKT_MAC_RINFO64 register */
+#define    CN23XX_PKT_MAC_CTL_RINFO_SRN_BIT_POS     0
+/* Starting bit of RPVF field in CN23XX_SLI_PKT_MAC_RINFO64 register */
+#define    CN23XX_PKT_MAC_CTL_RINFO_RPVF_BIT_POS     32
+/* Starting bit of NVFS field in CN23XX_SLI_PKT_MAC_RINFO64 register */
+#define    CN23XX_PKT_MAC_CTL_RINFO_NVFS_BIT_POS     48
+
+/*###################### REQUEST QUEUE #########################*/
+
+/* 64 registers for Input Queue Instr Count - SLI_PKT_IN_DONE0_CNTS */
+#define    CN23XX_SLI_IQ_INSTR_COUNT_START64     0x10040
+
+/* 64 registers for Input Queues Start Addr - SLI_PKT0_INSTR_BADDR */
+#define    CN23XX_SLI_IQ_BASE_ADDR_START64       0x10010
+
+/* 64 registers for Input Doorbell - SLI_PKT0_INSTR_BAOFF_DBELL */
+#define    CN23XX_SLI_IQ_DOORBELL_START          0x10020
+
+/* 64 registers for Input Queue size - SLI_PKT0_INSTR_FIFO_RSIZE */
+#define    CN23XX_SLI_IQ_SIZE_START              0x10030
+
+/* 64 registers (64-bit) - ES, RO, NS, Arbitration for Input Queue Data &
+ * gather list fetches. SLI_PKT(0..63)_INPUT_CONTROL.
+ */
+#define    CN23XX_SLI_IQ_PKT_CONTROL_START64    0x10000
+
+/*------- Request Queue Macros ---------*/
+#define    CN23XX_SLI_IQ_PKT_CONTROL64(iq)          \
+		(CN23XX_SLI_IQ_PKT_CONTROL_START64 + ((iq) * CN23XX_IQ_OFFSET))
+
+#define    CN23XX_SLI_IQ_BASE_ADDR64(iq)          \
+		(CN23XX_SLI_IQ_BASE_ADDR_START64 + ((iq) * CN23XX_IQ_OFFSET))
+
+#define    CN23XX_SLI_IQ_SIZE(iq)                 \
+		(CN23XX_SLI_IQ_SIZE_START + ((iq) * CN23XX_IQ_OFFSET))
+
+#define    CN23XX_SLI_IQ_DOORBELL(iq)             \
+		(CN23XX_SLI_IQ_DOORBELL_START + ((iq) * CN23XX_IQ_OFFSET))
+
+#define    CN23XX_SLI_IQ_INSTR_COUNT64(iq)          \
+		(CN23XX_SLI_IQ_INSTR_COUNT_START64 + ((iq) * CN23XX_IQ_OFFSET))
+
+/*------------------ Masks ----------------*/
+#define    CN23XX_PKT_INPUT_CTL_VF_NUM                  BIT_ULL(32)
+#define    CN23XX_PKT_INPUT_CTL_MAC_NUM                 BIT(29)
+/* Number of instructions to be read in one MAC read request.
+ * setting to Max value(4)
+ */
+#define    CN23XX_PKT_INPUT_CTL_RDSIZE                  (3 << 25)
+#define    CN23XX_PKT_INPUT_CTL_IS_64B                  BIT(24)
+#define    CN23XX_PKT_INPUT_CTL_RST                     BIT(23)
+#define    CN23XX_PKT_INPUT_CTL_QUIET                   BIT(28)
+#define    CN23XX_PKT_INPUT_CTL_RING_ENB                BIT(22)
+#define    CN23XX_PKT_INPUT_CTL_DATA_NS                 BIT(8)
+#define    CN23XX_PKT_INPUT_CTL_DATA_ES_64B_SWAP        BIT(6)
+#define    CN23XX_PKT_INPUT_CTL_DATA_RO                 BIT(5)
+#define    CN23XX_PKT_INPUT_CTL_USE_CSR                 BIT(4)
+#define    CN23XX_PKT_INPUT_CTL_GATHER_NS               BIT(3)
+#define    CN23XX_PKT_INPUT_CTL_GATHER_ES_64B_SWAP      (2)
+#define    CN23XX_PKT_INPUT_CTL_GATHER_RO               (1)
+
+/** Rings per Virtual Function **/
+#define    CN23XX_PKT_INPUT_CTL_RPVF_MASK               (0x3F)
+#define    CN23XX_PKT_INPUT_CTL_RPVF_POS                (48)
+/** These bits[47:44] select the Physical function number within the MAC */
+#define    CN23XX_PKT_INPUT_CTL_PF_NUM_MASK             (0x7)
+#define    CN23XX_PKT_INPUT_CTL_PF_NUM_POS              (45)
+/** These bits[43:32] select the function number within the PF */
+#define    CN23XX_PKT_INPUT_CTL_VF_NUM_MASK             (0x1FFF)
+#define    CN23XX_PKT_INPUT_CTL_VF_NUM_POS              (32)
+#define    CN23XX_PKT_INPUT_CTL_MAC_NUM_MASK            (0x3)
+#define    CN23XX_PKT_INPUT_CTL_MAC_NUM_POS             (29)
+#define    CN23XX_PKT_IN_DONE_WMARK_MASK                (0xFFFFULL)
+#define    CN23XX_PKT_IN_DONE_WMARK_BIT_POS             (32)
+#define    CN23XX_PKT_IN_DONE_CNT_MASK                  (0x00000000FFFFFFFFULL)
+
+#ifdef __LITTLE_ENDIAN_BITFIELD
+#define    CN23XX_PKT_INPUT_CTL_MASK				\
+		(CN23XX_PKT_INPUT_CTL_RDSIZE		|	\
+		 CN23XX_PKT_INPUT_CTL_DATA_ES_64B_SWAP	|	\
+		 CN23XX_PKT_INPUT_CTL_USE_CSR)
+#else
+#define    CN23XX_PKT_INPUT_CTL_MASK				\
+		(CN23XX_PKT_INPUT_CTL_RDSIZE		|	\
+		 CN23XX_PKT_INPUT_CTL_DATA_ES_64B_SWAP	|	\
+		 CN23XX_PKT_INPUT_CTL_USE_CSR		|	\
+		 CN23XX_PKT_INPUT_CTL_GATHER_ES_64B_SWAP)
+#endif
+
+/** Masks for SLI_PKT_IN_DONE(0..63)_CNTS Register */
+#define    CN23XX_IN_DONE_CNTS_PI_INT               BIT_ULL(62)
+#define    CN23XX_IN_DONE_CNTS_CINT_ENB             BIT_ULL(48)
+
+/*############################ OUTPUT QUEUE #########################*/
+
+/* 64 registers for Output queue control - SLI_PKT(0..63)_OUTPUT_CONTROL */
+#define    CN23XX_SLI_OQ_PKT_CONTROL_START       0x10050
+
+/* 64 registers for Output queue buffer and info size - SLI_PKT0_OUT_SIZE */
+#define    CN23XX_SLI_OQ0_BUFF_INFO_SIZE         0x10060
+
+/* 64 registers for Output Queue Start Addr - SLI_PKT0_SLIST_BADDR */
+#define    CN23XX_SLI_OQ_BASE_ADDR_START64       0x10070
+
+/* 64 registers for Output Queue Packet Credits - SLI_PKT0_SLIST_BAOFF_DBELL */
+#define    CN23XX_SLI_OQ_PKT_CREDITS_START       0x10080
+
+/* 64 registers for Output Queue size - SLI_PKT0_SLIST_FIFO_RSIZE */
+#define    CN23XX_SLI_OQ_SIZE_START              0x10090
+
+/* 64 registers for Output Queue Packet Count - SLI_PKT0_CNTS */
+#define    CN23XX_SLI_OQ_PKT_SENT_START          0x100B0
+
+/* 64 registers for Output Queue INT Levels - SLI_PKT0_INT_LEVELS */
+#define    CN23XX_SLI_OQ_PKT_INT_LEVELS_START64   0x100A0
+
+/* Each Output Queue register is at a 16-byte Offset in BAR0 */
+#define    CN23XX_OQ_OFFSET                      0x20000
+
+/* 1 (64-bit register) for Output Queue backpressure across all rings. */
+#define    CN23XX_SLI_OQ_WMARK                   0x29180
+
+/* Global pkt control register */
+#define    CN23XX_SLI_GBL_CONTROL                0x29210
+
+/* Backpressure enable register for PF0  */
+#define    CN23XX_SLI_OUT_BP_EN_W1S              0x29260
+
+/* Backpressure enable register for PF1  */
+#define    CN23XX_SLI_OUT_BP_EN2_W1S             0x29270
+
+/* Backpressure disable register for PF0  */
+#define    CN23XX_SLI_OUT_BP_EN_W1C              0x29280
+
+/* Backpressure disable register for PF1  */
+#define    CN23XX_SLI_OUT_BP_EN2_W1C             0x29290
+
+/*------- Output Queue Macros ---------*/
+
+#define    CN23XX_SLI_OQ_PKT_CONTROL(oq)          \
+		(CN23XX_SLI_OQ_PKT_CONTROL_START + ((oq) * CN23XX_OQ_OFFSET))
+
+#define    CN23XX_SLI_OQ_BASE_ADDR64(oq)          \
+		(CN23XX_SLI_OQ_BASE_ADDR_START64 + ((oq) * CN23XX_OQ_OFFSET))
+
+#define    CN23XX_SLI_OQ_SIZE(oq)                 \
+		(CN23XX_SLI_OQ_SIZE_START + ((oq) * CN23XX_OQ_OFFSET))
+
+#define    CN23XX_SLI_OQ_BUFF_INFO_SIZE(oq)                 \
+		(CN23XX_SLI_OQ0_BUFF_INFO_SIZE + ((oq) * CN23XX_OQ_OFFSET))
+
+#define    CN23XX_SLI_OQ_PKTS_SENT(oq)            \
+		(CN23XX_SLI_OQ_PKT_SENT_START + ((oq) * CN23XX_OQ_OFFSET))
+
+#define    CN23XX_SLI_OQ_PKTS_CREDIT(oq)          \
+		(CN23XX_SLI_OQ_PKT_CREDITS_START + ((oq) * CN23XX_OQ_OFFSET))
+
+#define    CN23XX_SLI_OQ_PKT_INT_LEVELS(oq)		\
+		(CN23XX_SLI_OQ_PKT_INT_LEVELS_START64 +	\
+		 ((oq) * CN23XX_OQ_OFFSET))
+
+/*Macro's for accessing CNT and TIME separately from INT_LEVELS*/
+#define    CN23XX_SLI_OQ_PKT_INT_LEVELS_CNT(oq)		\
+		(CN23XX_SLI_OQ_PKT_INT_LEVELS_START64 + \
+		 ((oq) * CN23XX_OQ_OFFSET))
+
+#define    CN23XX_SLI_OQ_PKT_INT_LEVELS_TIME(oq)	\
+		(CN23XX_SLI_OQ_PKT_INT_LEVELS_START64 +	\
+		 ((oq) * CN23XX_OQ_OFFSET) + 4)
+
+/*------------------ Masks ----------------*/
+#define    CN23XX_PKT_OUTPUT_CTL_TENB                  BIT(13)
+#define    CN23XX_PKT_OUTPUT_CTL_CENB                  BIT(12)
+#define    CN23XX_PKT_OUTPUT_CTL_IPTR                  BIT(11)
+#define    CN23XX_PKT_OUTPUT_CTL_ES                    BIT(9)
+#define    CN23XX_PKT_OUTPUT_CTL_NSR                   BIT(8)
+#define    CN23XX_PKT_OUTPUT_CTL_ROR                   BIT(7)
+#define    CN23XX_PKT_OUTPUT_CTL_DPTR                  BIT(6)
+#define    CN23XX_PKT_OUTPUT_CTL_BMODE                 BIT(5)
+#define    CN23XX_PKT_OUTPUT_CTL_ES_P                  BIT(3)
+#define    CN23XX_PKT_OUTPUT_CTL_NSR_P                 BIT(2)
+#define    CN23XX_PKT_OUTPUT_CTL_ROR_P                 BIT(1)
+#define    CN23XX_PKT_OUTPUT_CTL_RING_ENB              BIT(0)
+
+/*######################### Mailbox Reg Macros ########################*/
+#define    CN23XX_SLI_PKT_MBOX_INT_START             0x10210
+#define    CN23XX_SLI_PKT_PF_VF_MBOX_SIG_START       0x10200
+#define    CN23XX_SLI_MAC_PF_MBOX_INT_START          0x27380
+
+#define    CN23XX_SLI_MBOX_OFFSET		     0x20000
+#define    CN23XX_SLI_MBOX_SIG_IDX_OFFSET	     0x8
+
+#define    CN23XX_SLI_PKT_MBOX_INT(q)          \
+		(CN23XX_SLI_PKT_MBOX_INT_START + ((q) * CN23XX_SLI_MBOX_OFFSET))
+
+#define    CN23XX_SLI_PKT_PF_VF_MBOX_SIG(q, idx)		\
+		(CN23XX_SLI_PKT_PF_VF_MBOX_SIG_START +		\
+		 ((q) * CN23XX_SLI_MBOX_OFFSET +		\
+		  (idx) * CN23XX_SLI_MBOX_SIG_IDX_OFFSET))
+
+#define    CN23XX_SLI_MAC_PF_MBOX_INT(mac, pf)		\
+		(CN23XX_SLI_MAC_PF_MBOX_INT_START +	\
+		 ((mac) * CN23XX_MAC_INT_OFFSET +	\
+		  (pf) * CN23XX_PF_INT_OFFSET))
+
+/*######################### DMA Counters #########################*/
+
+/* 2 registers (64-bit) - DMA Count - 1 for each DMA counter 0/1. */
+#define    CN23XX_DMA_CNT_START                   0x28400
+
+/* 2 registers (64-bit) - DMA Timer 0/1, contains DMA timer values */
+/* SLI_DMA_0_TIM */
+#define    CN23XX_DMA_TIM_START                   0x28420
+
+/* 2 registers (64-bit) - DMA count & Time Interrupt threshold -
+ * SLI_DMA_0_INT_LEVEL
+ */
+#define    CN23XX_DMA_INT_LEVEL_START             0x283E0
+
+/* Each DMA register is at a 16-byte Offset in BAR0 */
+#define    CN23XX_DMA_OFFSET                      0x10
+
+/*---------- DMA Counter Macros ---------*/
+#define    CN23XX_DMA_CNT(dq)                      \
+		(CN23XX_DMA_CNT_START + ((dq) * CN23XX_DMA_OFFSET))
+
+#define    CN23XX_DMA_INT_LEVEL(dq)                \
+		(CN23XX_DMA_INT_LEVEL_START + ((dq) * CN23XX_DMA_OFFSET))
+
+#define    CN23XX_DMA_PKT_INT_LEVEL(dq)            \
+		(CN23XX_DMA_INT_LEVEL_START + ((dq) * CN23XX_DMA_OFFSET))
+
+#define    CN23XX_DMA_TIME_INT_LEVEL(dq)           \
+		(CN23XX_DMA_INT_LEVEL_START + 4 + ((dq) * CN23XX_DMA_OFFSET))
+
+#define    CN23XX_DMA_TIM(dq)                     \
+		(CN23XX_DMA_TIM_START + ((dq) * CN23XX_DMA_OFFSET))
+
+/*######################## MSIX TABLE #########################*/
+
+#define	CN23XX_MSIX_TABLE_ADDR_START		0x0
+#define	CN23XX_MSIX_TABLE_DATA_START		0x8
+
+#define	CN23XX_MSIX_TABLE_SIZE			0x10
+#define	CN23XX_MSIX_TABLE_ENTRIES		0x41
+
+#define CN23XX_MSIX_ENTRY_VECTOR_CTL	BIT_ULL(32)
+
+#define	CN23XX_MSIX_TABLE_ADDR(idx)		\
+	(CN23XX_MSIX_TABLE_ADDR_START + ((idx) * CN23XX_MSIX_TABLE_SIZE))
+
+#define	CN23XX_MSIX_TABLE_DATA(idx)		\
+	(CN23XX_MSIX_TABLE_DATA_START + ((idx) * CN23XX_MSIX_TABLE_SIZE))
+
+/*######################## INTERRUPTS #########################*/
+#define CN23XX_MAC_INT_OFFSET   0x20
+#define CN23XX_PF_INT_OFFSET    0x10
+
+/* 1 register (64-bit) for Interrupt Summary */
+#define    CN23XX_SLI_INT_SUM64            0x27000
+
+/* 4 registers (64-bit) for Interrupt Enable for each Port */
+#define    CN23XX_SLI_INT_ENB64            0x27080
+
+#define    CN23XX_SLI_MAC_PF_INT_SUM64(mac, pf)			\
+		(CN23XX_SLI_INT_SUM64 +				\
+		 ((mac) * CN23XX_MAC_INT_OFFSET) +		\
+		 ((pf) * CN23XX_PF_INT_OFFSET))
+
+#define    CN23XX_SLI_MAC_PF_INT_ENB64(mac, pf)		\
+		(CN23XX_SLI_INT_ENB64 +			\
+		 ((mac) * CN23XX_MAC_INT_OFFSET) +	\
+		 ((pf) * CN23XX_PF_INT_OFFSET))
+
+/* 1 register (64-bit) to indicate which Output Queue reached pkt threshold */
+#define    CN23XX_SLI_PKT_CNT_INT                0x29130
+
+/* 1 register (64-bit) to indicate which Output Queue reached time threshold */
+#define    CN23XX_SLI_PKT_TIME_INT               0x29140
+
+/*------------------ Interrupt Masks ----------------*/
+
+#define    CN23XX_INTR_PO_INT			BIT_ULL(63)
+#define    CN23XX_INTR_PI_INT			BIT_ULL(62)
+#define    CN23XX_INTR_MBOX_INT			BIT_ULL(61)
+#define    CN23XX_INTR_RESEND			BIT_ULL(60)
+
+#define    CN23XX_INTR_CINT_ENB                 BIT_ULL(48)
+#define    CN23XX_INTR_MBOX_ENB                 BIT(0)
+
+#define    CN23XX_INTR_RML_TIMEOUT_ERR           (1)
+
+#define    CN23XX_INTR_MIO_INT                   BIT(1)
+
+#define    CN23XX_INTR_RESERVED1                 (3 << 2)
+
+#define    CN23XX_INTR_PKT_COUNT                 BIT(4)
+#define    CN23XX_INTR_PKT_TIME                  BIT(5)
+
+#define    CN23XX_INTR_RESERVED2                 (3 << 6)
+
+#define    CN23XX_INTR_M0UPB0_ERR                BIT(8)
+#define    CN23XX_INTR_M0UPWI_ERR                BIT(9)
+#define    CN23XX_INTR_M0UNB0_ERR                BIT(10)
+#define    CN23XX_INTR_M0UNWI_ERR                BIT(11)
+
+#define    CN23XX_INTR_RESERVED3                 (0xFFFFFULL << 12)
+
+#define    CN23XX_INTR_DMA0_FORCE                BIT_ULL(32)
+#define    CN23XX_INTR_DMA1_FORCE                BIT_ULL(33)
+
+#define    CN23XX_INTR_DMA0_COUNT                BIT_ULL(34)
+#define    CN23XX_INTR_DMA1_COUNT                BIT_ULL(35)
+
+#define    CN23XX_INTR_DMA0_TIME                 BIT_ULL(36)
+#define    CN23XX_INTR_DMA1_TIME                 BIT_ULL(37)
+
+#define    CN23XX_INTR_RESERVED4                 (0x7FFFFULL << 38)
+
+#define    CN23XX_INTR_VF_MBOX                   BIT_ULL(57)
+#define    CN23XX_INTR_DMAVF_ERR                 BIT_ULL(58)
+#define    CN23XX_INTR_DMAPF_ERR                 BIT_ULL(59)
+
+#define    CN23XX_INTR_PKTVF_ERR                 BIT_ULL(60)
+#define    CN23XX_INTR_PKTPF_ERR                 BIT_ULL(61)
+#define    CN23XX_INTR_PPVF_ERR                  BIT_ULL(62)
+#define    CN23XX_INTR_PPPF_ERR                  BIT_ULL(63)
+
+#define    CN23XX_INTR_DMA0_DATA                 (CN23XX_INTR_DMA0_TIME)
+#define    CN23XX_INTR_DMA1_DATA                 (CN23XX_INTR_DMA1_TIME)
+
+#define    CN23XX_INTR_DMA_DATA                  \
+		(CN23XX_INTR_DMA0_DATA | CN23XX_INTR_DMA1_DATA)
+
+/* By fault only TIME based */
+#define    CN23XX_INTR_PKT_DATA                  (CN23XX_INTR_PKT_TIME)
+/* For both COUNT and TIME based */
+/* #define    CN23XX_INTR_PKT_DATA                  \
+ * (CN23XX_INTR_PKT_COUNT | CN23XX_INTR_PKT_TIME)
+ */
+
+/* Sum of interrupts for all PCI-Express Data Interrupts */
+#define    CN23XX_INTR_PCIE_DATA                 \
+		(CN23XX_INTR_DMA_DATA | CN23XX_INTR_PKT_DAT)
+
+/* Sum of interrupts for error events */
+#define    CN23XX_INTR_ERR			\
+		(CN23XX_INTR_M0UPB0_ERR	|	\
+		 CN23XX_INTR_M0UPWI_ERR	|	\
+		 CN23XX_INTR_M0UNB0_ERR	|	\
+		 CN23XX_INTR_M0UNWI_ERR	|	\
+		 CN23XX_INTR_DMAVF_ERR	|	\
+		 CN23XX_INTR_DMAPF_ERR	|	\
+		 CN23XX_INTR_PKTPF_ERR	|	\
+		 CN23XX_INTR_PPPF_ERR	|	\
+		 CN23XX_INTR_PPVF_ERR)
+
+/* Programmed Mask for Interrupt Sum */
+#define    CN23XX_INTR_MASK			\
+		(CN23XX_INTR_DMA_DATA	|	\
+		 CN23XX_INTR_DMA0_FORCE	|	\
+		 CN23XX_INTR_DMA1_FORCE	|	\
+		 CN23XX_INTR_MIO_INT	|	\
+		 CN23XX_INTR_ERR)
+
+/* 4 Registers (64 - bit) */
+#define    CN23XX_SLI_S2M_PORT_CTL_START         0x23D80
+#define    CN23XX_SLI_S2M_PORTX_CTL(port)	\
+		(CN23XX_SLI_S2M_PORT_CTL_START + (port * 0x10))
+
+#define    CN23XX_SLI_MAC_NUMBER                 0x20050
+
+/** PEM(0..3)_BAR1_INDEX(0..15)address is defined as
+ *  addr = (0x00011800C0000100  |port <<24 |idx <<3 )
+ *  Here, port is PEM(0..3) & idx is INDEX(0..15)
+ */
+#define    CN23XX_PEM_BAR1_INDEX_START             0x00011800C0000100ULL
+#define    CN23XX_PEM_OFFSET                       24
+#define    CN23XX_BAR1_INDEX_OFFSET                3
+
+#define    CN23XX_PEM_BAR1_INDEX_REG(port, idx)		\
+		(CN23XX_PEM_BAR1_INDEX_START + ((port) << CN23XX_PEM_OFFSET) + \
+		 ((idx) << CN23XX_BAR1_INDEX_OFFSET))
+
+/*############################ DPI #########################*/
+
+/* 1 register (64-bit) - provides DMA Enable */
+#define    CN23XX_DPI_CTL                 0x0001df0000000040ULL
+
+/* 1 register (64-bit) - Controls the DMA IO Operation */
+#define    CN23XX_DPI_DMA_CONTROL         0x0001df0000000048ULL
+
+/* 1 register (64-bit) - Provides DMA Instr'n Queue Enable  */
+#define    CN23XX_DPI_REQ_GBL_ENB         0x0001df0000000050ULL
+
+/* 1 register (64-bit) - DPI_REQ_ERR_RSP
+ * Indicates which Instr'n Queue received error response from the IO sub-system
+ */
+#define    CN23XX_DPI_REQ_ERR_RSP         0x0001df0000000058ULL
+
+/* 1 register (64-bit) - DPI_REQ_ERR_RST
+ * Indicates which Instr'n Queue dropped an Instr'n
+ */
+#define    CN23XX_DPI_REQ_ERR_RST         0x0001df0000000060ULL
+
+/* 6 register (64-bit) - DPI_DMA_ENG(0..5)_EN
+ * Provides DMA Engine Queue Enable
+ */
+#define    CN23XX_DPI_DMA_ENG0_ENB        0x0001df0000000080ULL
+#define    CN23XX_DPI_DMA_ENG_ENB(eng) (CN23XX_DPI_DMA_ENG0_ENB + (eng * 8))
+
+/* 8 register (64-bit) - DPI_DMA(0..7)_REQQ_CTL
+ * Provides control bits for transaction on 8 Queues
+ */
+#define    CN23XX_DPI_DMA_REQQ0_CTL       0x0001df0000000180ULL
+#define    CN23XX_DPI_DMA_REQQ_CTL(q_no)	\
+		(CN23XX_DPI_DMA_REQQ0_CTL + (q_no * 8))
+
+/* 6 register (64-bit) - DPI_ENG(0..5)_BUF
+ * Provides DMA Engine FIFO (Queue) Size
+ */
+#define    CN23XX_DPI_DMA_ENG0_BUF        0x0001df0000000880ULL
+#define    CN23XX_DPI_DMA_ENG_BUF(eng)   \
+		(CN23XX_DPI_DMA_ENG0_BUF + (eng * 8))
+
+/* 4 Registers (64-bit) */
+#define    CN23XX_DPI_SLI_PRT_CFG_START   0x0001df0000000900ULL
+#define    CN23XX_DPI_SLI_PRTX_CFG(port)        \
+		(CN23XX_DPI_SLI_PRT_CFG_START + (port * 0x8))
+
+/* Masks for DPI_DMA_CONTROL Register */
+#define    CN23XX_DPI_DMA_COMMIT_MODE     BIT_ULL(58)
+#define    CN23XX_DPI_DMA_PKT_EN          BIT_ULL(56)
+#define    CN23XX_DPI_DMA_ENB             (0x0FULL << 48)
+/* Set the DMA Control, to update packet count not byte count sent by DMA,
+ * when we use Interrupt Coalescing (CA mode)
+ */
+#define    CN23XX_DPI_DMA_O_ADD1          BIT(19)
+/*selecting 64-bit Byte Swap Mode */
+#define    CN23XX_DPI_DMA_O_ES            BIT(15)
+#define    CN23XX_DPI_DMA_O_MODE          BIT(14)
+
+#define    CN23XX_DPI_DMA_CTL_MASK			\
+		(CN23XX_DPI_DMA_COMMIT_MODE	|	\
+		 CN23XX_DPI_DMA_PKT_EN		|	\
+		 CN23XX_DPI_DMA_O_ES		|	\
+		 CN23XX_DPI_DMA_O_MODE)
+
+/*############################ RST #########################*/
+
+#define    CN23XX_RST_BOOT            0x0001180006001600ULL
+#define    CN23XX_RST_SOFT_RST        0x0001180006001680ULL
+
+#define    CN23XX_LMC0_RESET_CTL               0x0001180088000180ULL
+#define    CN23XX_LMC0_RESET_CTL_DDR3RST_MASK  0x0000000000000001ULL
+
+#endif
diff --git a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c
index c03d37016a48..e779af88621b 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c
@@ -338,7 +338,7 @@ void lio_cn6xxx_setup_oq_regs(struct octeon_device *oct, u32 oq_no)
 	octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, intr);
 }
 
-void lio_cn6xxx_enable_io_queues(struct octeon_device *oct)
+int lio_cn6xxx_enable_io_queues(struct octeon_device *oct)
 {
 	u32 mask;
 
@@ -353,6 +353,8 @@ void lio_cn6xxx_enable_io_queues(struct octeon_device *oct)
 	mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB);
 	mask |= oct->io_qmask.oq;
 	octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, mask);
+
+	return 0;
 }
 
 void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
@@ -418,36 +420,6 @@ void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
 		octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT, d32);
 }
 
-void lio_cn6xxx_reinit_regs(struct octeon_device *oct)
-{
-	int i;
-
-	for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
-		if (!(oct->io_qmask.iq & (1ULL << i)))
-			continue;
-		oct->fn_list.setup_iq_regs(oct, i);
-	}
-
-	for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
-		if (!(oct->io_qmask.oq & (1ULL << i)))
-			continue;
-		oct->fn_list.setup_oq_regs(oct, i);
-	}
-
-	oct->fn_list.setup_device_regs(oct);
-
-	oct->fn_list.enable_interrupt(oct->chip);
-
-	oct->fn_list.enable_io_queues(oct);
-
-	/* for (i = 0; i < oct->num_oqs; i++) { */
-	for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
-		if (!(oct->io_qmask.oq & (1ULL << i)))
-			continue;
-		writel(oct->droq[i]->max_count, oct->droq[i]->pkts_credit_reg);
-	}
-}
-
 void
 lio_cn6xxx_bar1_idx_setup(struct octeon_device *oct,
 			  u64 core_addr,
@@ -507,18 +479,20 @@ lio_cn6xxx_update_read_index(struct octeon_instr_queue *iq)
 	return new_idx;
 }
 
-void lio_cn6xxx_enable_interrupt(void *chip)
+void lio_cn6xxx_enable_interrupt(struct octeon_device *oct,
+				 u8 unused __attribute__((unused)))
 {
-	struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)chip;
+	struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
 	u64 mask = cn6xxx->intr_mask64 | CN6XXX_INTR_DMA0_FORCE;
 
 	/* Enable Interrupt */
 	writeq(mask, cn6xxx->intr_enb_reg64);
 }
 
-void lio_cn6xxx_disable_interrupt(void *chip)
+void lio_cn6xxx_disable_interrupt(struct octeon_device *oct,
+				  u8 unused __attribute__((unused)))
 {
-	struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)chip;
+	struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
 
 	/* Disable Interrupts */
 	writeq(0, cn6xxx->intr_enb_reg64);
@@ -714,7 +688,6 @@ int lio_setup_cn66xx_octeon_device(struct octeon_device *oct)
 
 	oct->fn_list.soft_reset = lio_cn6xxx_soft_reset;
 	oct->fn_list.setup_device_regs = lio_cn6xxx_setup_device_regs;
-	oct->fn_list.reinit_regs = lio_cn6xxx_reinit_regs;
 	oct->fn_list.update_iq_read_idx = lio_cn6xxx_update_read_index;
 
 	oct->fn_list.bar1_idx_setup = lio_cn6xxx_bar1_idx_setup;
diff --git a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h
index 28c47224221a..a40a91394079 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h
+++ b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.h
@@ -80,18 +80,17 @@ void lio_cn6xxx_setup_global_input_regs(struct octeon_device *oct);
 void lio_cn6xxx_setup_global_output_regs(struct octeon_device *oct);
 void lio_cn6xxx_setup_iq_regs(struct octeon_device *oct, u32 iq_no);
 void lio_cn6xxx_setup_oq_regs(struct octeon_device *oct, u32 oq_no);
-void lio_cn6xxx_enable_io_queues(struct octeon_device *oct);
+int lio_cn6xxx_enable_io_queues(struct octeon_device *oct);
 void lio_cn6xxx_disable_io_queues(struct octeon_device *oct);
 irqreturn_t lio_cn6xxx_process_interrupt_regs(void *dev);
-void lio_cn6xxx_reinit_regs(struct octeon_device *oct);
 void lio_cn6xxx_bar1_idx_setup(struct octeon_device *oct, u64 core_addr,
 			       u32 idx, int valid);
 void lio_cn6xxx_bar1_idx_write(struct octeon_device *oct, u32 idx, u32 mask);
 u32 lio_cn6xxx_bar1_idx_read(struct octeon_device *oct, u32 idx);
 u32
 lio_cn6xxx_update_read_index(struct octeon_instr_queue *iq);
-void lio_cn6xxx_enable_interrupt(void *chip);
-void lio_cn6xxx_disable_interrupt(void *chip);
+void lio_cn6xxx_enable_interrupt(struct octeon_device *oct, u8 unused);
+void lio_cn6xxx_disable_interrupt(struct octeon_device *oct, u8 unused);
 void cn6xxx_get_pcie_qlmport(struct octeon_device *oct);
 void lio_cn6xxx_setup_reg_address(struct octeon_device *oct, void *chip,
 				  struct octeon_reg_list *reg_list);
diff --git a/drivers/net/ethernet/cavium/liquidio/cn68xx_device.c b/drivers/net/ethernet/cavium/liquidio/cn68xx_device.c
index 29755bc68f12..dbf3566ead53 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn68xx_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/cn68xx_device.c
@@ -148,7 +148,6 @@ int lio_setup_cn68xx_octeon_device(struct octeon_device *oct)
 	oct->fn_list.process_interrupt_regs = lio_cn6xxx_process_interrupt_regs;
 	oct->fn_list.soft_reset = lio_cn68xx_soft_reset;
 	oct->fn_list.setup_device_regs = lio_cn68xx_setup_device_regs;
-	oct->fn_list.reinit_regs = lio_cn6xxx_reinit_regs;
 	oct->fn_list.update_iq_read_idx = lio_cn6xxx_update_read_index;
 
 	oct->fn_list.bar1_idx_setup = lio_cn6xxx_bar1_idx_setup;
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_core.c b/drivers/net/ethernet/cavium/liquidio/lio_core.c
new file mode 100644
index 000000000000..201eddb3013a
--- /dev/null
+++ b/drivers/net/ethernet/cavium/liquidio/lio_core.c
@@ -0,0 +1,266 @@
+/**********************************************************************
+* Author: Cavium, Inc.
+*
+* Contact: support@cavium.com
+*          Please include "LiquidIO" in the subject.
+*
+* Copyright (c) 2003-2015 Cavium, Inc.
+*
+* This file is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License, Version 2, as
+* published by the Free Software Foundation.
+*
+* This file is distributed in the hope that it will be useful, but
+* AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
+* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
+* NONINFRINGEMENT.  See the GNU General Public License for more
+* details.
+*
+* This file may also be available under a different license from Cavium.
+* Contact Cavium, Inc. for more information
+**********************************************************************/
+#include <linux/pci.h>
+#include <linux/if_vlan.h>
+#include "liquidio_common.h"
+#include "octeon_droq.h"
+#include "octeon_iq.h"
+#include "response_manager.h"
+#include "octeon_device.h"
+#include "octeon_nic.h"
+#include "octeon_main.h"
+#include "octeon_network.h"
+
+int liquidio_set_feature(struct net_device *netdev, int cmd, u16 param1)
+{
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct = lio->oct_dev;
+	struct octnic_ctrl_pkt nctrl;
+	int ret = 0;
+
+	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
+
+	nctrl.ncmd.u64 = 0;
+	nctrl.ncmd.s.cmd = cmd;
+	nctrl.ncmd.s.param1 = param1;
+	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
+	nctrl.wait_time = 100;
+	nctrl.netpndev = (u64)netdev;
+	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
+
+	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
+	if (ret < 0) {
+		dev_err(&oct->pci_dev->dev, "Feature change failed in core (ret: 0x%x)\n",
+			ret);
+	}
+	return ret;
+}
+
+void octeon_report_tx_completion_to_bql(void *txq, unsigned int pkts_compl,
+					unsigned int bytes_compl)
+{
+	struct netdev_queue *netdev_queue = txq;
+
+	netdev_tx_completed_queue(netdev_queue, pkts_compl, bytes_compl);
+}
+
+void octeon_update_tx_completion_counters(void *buf, int reqtype,
+					  unsigned int *pkts_compl,
+					  unsigned int *bytes_compl)
+{
+	struct octnet_buf_free_info *finfo;
+	struct sk_buff *skb = NULL;
+	struct octeon_soft_command *sc;
+
+	switch (reqtype) {
+	case REQTYPE_NORESP_NET:
+	case REQTYPE_NORESP_NET_SG:
+		finfo = buf;
+		skb = finfo->skb;
+		break;
+
+	case REQTYPE_RESP_NET_SG:
+	case REQTYPE_RESP_NET:
+		sc = buf;
+		skb = sc->callback_arg;
+		break;
+
+	default:
+		return;
+	}
+
+	(*pkts_compl)++;
+/*TODO, Use some other pound define to suggest
+ * the fact that iqs are not tied to netdevs
+ * and can take traffic from different netdevs
+ * hence bql reporting is done per packet
+ * than in bulk. Usage of NO_NAPI in txq completion is
+ * a little confusing
+ */
+	*bytes_compl += skb->len;
+}
+
+void octeon_report_sent_bytes_to_bql(void *buf, int reqtype)
+{
+	struct octnet_buf_free_info *finfo;
+	struct sk_buff *skb;
+	struct octeon_soft_command *sc;
+	struct netdev_queue *txq;
+
+	switch (reqtype) {
+	case REQTYPE_NORESP_NET:
+	case REQTYPE_NORESP_NET_SG:
+		finfo = buf;
+		skb = finfo->skb;
+		break;
+
+	case REQTYPE_RESP_NET_SG:
+	case REQTYPE_RESP_NET:
+		sc = buf;
+		skb = sc->callback_arg;
+		break;
+
+	default:
+		return;
+	}
+
+	txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb));
+	netdev_tx_sent_queue(txq, skb->len);
+}
+
+void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr)
+{
+	struct octnic_ctrl_pkt *nctrl = (struct octnic_ctrl_pkt *)nctrl_ptr;
+	struct net_device *netdev = (struct net_device *)nctrl->netpndev;
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct = lio->oct_dev;
+	u8 *mac;
+
+	switch (nctrl->ncmd.s.cmd) {
+	case OCTNET_CMD_CHANGE_DEVFLAGS:
+	case OCTNET_CMD_SET_MULTI_LIST:
+		break;
+
+	case OCTNET_CMD_CHANGE_MACADDR:
+		mac = ((u8 *)&nctrl->udd[0]) + 2;
+		netif_info(lio, probe, lio->netdev,
+			   "MACAddr changed to %2.2x:%2.2x:%2.2x:%2.2x:%2.2x:%2.2x\n",
+			   mac[0], mac[1],
+			   mac[2], mac[3],
+			   mac[4], mac[5]);
+		break;
+
+	case OCTNET_CMD_CHANGE_MTU:
+		/* If command is successful, change the MTU. */
+		netif_info(lio, probe, lio->netdev, "MTU Changed from %d to %d\n",
+			   netdev->mtu, nctrl->ncmd.s.param1);
+		dev_info(&oct->pci_dev->dev, "%s MTU Changed from %d to %d\n",
+			 netdev->name, netdev->mtu,
+			 nctrl->ncmd.s.param1);
+		netdev->mtu = nctrl->ncmd.s.param1;
+		queue_delayed_work(lio->link_status_wq.wq,
+				   &lio->link_status_wq.wk.work, 0);
+		break;
+
+	case OCTNET_CMD_GPIO_ACCESS:
+		netif_info(lio, probe, lio->netdev, "LED Flashing visual identification\n");
+
+		break;
+
+	case OCTNET_CMD_ID_ACTIVE:
+		netif_info(lio, probe, lio->netdev, "LED Flashing visual identification\n");
+
+		break;
+
+	case OCTNET_CMD_LRO_ENABLE:
+		dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name);
+		break;
+
+	case OCTNET_CMD_LRO_DISABLE:
+		dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n",
+			 netdev->name);
+		break;
+
+	case OCTNET_CMD_VERBOSE_ENABLE:
+		dev_info(&oct->pci_dev->dev, "%s Firmware debug enabled\n",
+			 netdev->name);
+		break;
+
+	case OCTNET_CMD_VERBOSE_DISABLE:
+		dev_info(&oct->pci_dev->dev, "%s Firmware debug disabled\n",
+			 netdev->name);
+		break;
+
+	case OCTNET_CMD_ENABLE_VLAN_FILTER:
+		dev_info(&oct->pci_dev->dev, "%s VLAN filter enabled\n",
+			 netdev->name);
+		break;
+
+	case OCTNET_CMD_ADD_VLAN_FILTER:
+		dev_info(&oct->pci_dev->dev, "%s VLAN filter %d added\n",
+			 netdev->name, nctrl->ncmd.s.param1);
+		break;
+
+	case OCTNET_CMD_DEL_VLAN_FILTER:
+		dev_info(&oct->pci_dev->dev, "%s VLAN filter %d removed\n",
+			 netdev->name, nctrl->ncmd.s.param1);
+		break;
+
+	case OCTNET_CMD_SET_SETTINGS:
+		dev_info(&oct->pci_dev->dev, "%s settings changed\n",
+			 netdev->name);
+
+		break;
+
+	/* Case to handle "OCTNET_CMD_TNL_RX_CSUM_CTL"
+	 * Command passed by NIC driver
+	 */
+	case OCTNET_CMD_TNL_RX_CSUM_CTL:
+		if (nctrl->ncmd.s.param1 == OCTNET_CMD_RXCSUM_ENABLE) {
+			netif_info(lio, probe, lio->netdev,
+				   "RX Checksum Offload Enabled\n");
+		} else if (nctrl->ncmd.s.param1 ==
+			   OCTNET_CMD_RXCSUM_DISABLE) {
+			netif_info(lio, probe, lio->netdev,
+				   "RX Checksum Offload Disabled\n");
+		}
+		break;
+
+		/* Case to handle "OCTNET_CMD_TNL_TX_CSUM_CTL"
+		 * Command passed by NIC driver
+		 */
+	case OCTNET_CMD_TNL_TX_CSUM_CTL:
+		if (nctrl->ncmd.s.param1 == OCTNET_CMD_TXCSUM_ENABLE) {
+			netif_info(lio, probe, lio->netdev,
+				   "TX Checksum Offload Enabled\n");
+		} else if (nctrl->ncmd.s.param1 ==
+			   OCTNET_CMD_TXCSUM_DISABLE) {
+			netif_info(lio, probe, lio->netdev,
+				   "TX Checksum Offload Disabled\n");
+		}
+		break;
+
+		/* Case to handle "OCTNET_CMD_VXLAN_PORT_CONFIG"
+		 * Command passed by NIC driver
+		 */
+	case OCTNET_CMD_VXLAN_PORT_CONFIG:
+		if (nctrl->ncmd.s.more == OCTNET_CMD_VXLAN_PORT_ADD) {
+			netif_info(lio, probe, lio->netdev,
+				   "VxLAN Destination UDP PORT:%d ADDED\n",
+				   nctrl->ncmd.s.param1);
+		} else if (nctrl->ncmd.s.more ==
+			   OCTNET_CMD_VXLAN_PORT_DEL) {
+			netif_info(lio, probe, lio->netdev,
+				   "VxLAN Destination UDP PORT:%d DELETED\n",
+				   nctrl->ncmd.s.param1);
+		}
+		break;
+
+	case OCTNET_CMD_SET_FLOW_CTL:
+		netif_info(lio, probe, lio->netdev, "Set RX/TX flow control parameters\n");
+		break;
+
+	default:
+		dev_err(&oct->pci_dev->dev, "%s Unknown cmd %d\n", __func__,
+			nctrl->ncmd.s.cmd);
+	}
+}
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
index 289eb8907922..f163e0abbeb2 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
@@ -32,6 +32,7 @@
 #include "octeon_network.h"
 #include "cn66xx_regs.h"
 #include "cn66xx_device.h"
+#include "cn23xx_pf_device.h"
 
 static int octnet_get_link_stats(struct net_device *netdev);
 
@@ -75,6 +76,7 @@ enum {
 
 #define ARRAY_LENGTH(a) (sizeof(a) / sizeof((a)[0]))
 #define OCT_ETHTOOL_REGDUMP_LEN  4096
+#define OCT_ETHTOOL_REGDUMP_LEN_23XX  (4096 * 11)
 #define OCT_ETHTOOL_REGSVER  1
 
 /* statistics of PF */
@@ -188,6 +190,10 @@ static const char oct_droq_stats_strings[][ETH_GSTRING_LEN] = {
 	"buffer_alloc_failure",
 };
 
+/* LiquidIO driver private flags */
+static const char oct_priv_flags_strings[][ETH_GSTRING_LEN] = {
+};
+
 #define OCTNIC_NCMD_AUTONEG_ON  0x1
 #define OCTNIC_NCMD_PHY_ON      0x2
 
@@ -259,6 +265,13 @@ lio_ethtool_get_channels(struct net_device *dev,
 		max_tx = CFG_GET_IQ_MAX_Q(conf6x);
 		rx_count = CFG_GET_NUM_RXQS_NIC_IF(conf6x, lio->ifidx);
 		tx_count = CFG_GET_NUM_TXQS_NIC_IF(conf6x, lio->ifidx);
+	} else if (OCTEON_CN23XX_PF(oct)) {
+		struct octeon_config *conf23 = CHIP_FIELD(oct, cn23xx_pf, conf);
+
+		max_rx = CFG_GET_OQ_MAX_Q(conf23);
+		max_tx = CFG_GET_IQ_MAX_Q(conf23);
+		rx_count = CFG_GET_NUM_RXQS_NIC_IF(conf23, lio->ifidx);
+		tx_count = CFG_GET_NUM_TXQS_NIC_IF(conf23, lio->ifidx);
 	}
 
 	channel->max_rx = max_rx;
@@ -290,18 +303,16 @@ lio_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom,
 	struct lio *lio = GET_LIO(netdev);
 	struct octeon_device *oct_dev = lio->oct_dev;
 	struct octeon_board_info *board_info;
-	int len;
 
-	if (eeprom->offset != 0)
+	if (eeprom->offset)
 		return -EINVAL;
 
 	eeprom->magic = oct_dev->pci_dev->vendor;
 	board_info = (struct octeon_board_info *)(&oct_dev->boardinfo);
-	len =
-		sprintf((char *)bytes,
-			"boardname:%s serialnum:%s maj:%lld min:%lld\n",
-			board_info->name, board_info->serial_number,
-			board_info->major, board_info->minor);
+	sprintf((char *)bytes,
+		"boardname:%s serialnum:%s maj:%lld min:%lld\n",
+		board_info->name, board_info->serial_number,
+		board_info->major, board_info->minor);
 
 	return 0;
 }
@@ -333,6 +344,32 @@ static int octnet_gpio_access(struct net_device *netdev, int addr, int val)
 	return 0;
 }
 
+static int octnet_id_active(struct net_device *netdev, int val)
+{
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct = lio->oct_dev;
+	struct octnic_ctrl_pkt nctrl;
+	int ret = 0;
+
+	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
+
+	nctrl.ncmd.u64 = 0;
+	nctrl.ncmd.s.cmd = OCTNET_CMD_ID_ACTIVE;
+	nctrl.ncmd.s.param1 = val;
+	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
+	nctrl.wait_time = 100;
+	nctrl.netpndev = (u64)netdev;
+	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
+
+	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
+	if (ret < 0) {
+		dev_err(&oct->pci_dev->dev, "Failed to configure gpio value\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 /* Callback for when mdio command response arrives
  */
 static void octnet_mdio_resp_callback(struct octeon_device *oct,
@@ -406,7 +443,7 @@ octnet_mdio45_access(struct lio *lio, int op, int loc, int *value)
 		dev_err(&oct_dev->pci_dev->dev,
 			"octnet_mdio45_access instruction failed status: %x\n",
 			retval);
-		retval =  -EBUSY;
+		retval = -EBUSY;
 	} else {
 		/* Sleep on a wait queue till the cond flag indicates that the
 		 * response arrived
@@ -476,6 +513,11 @@ static int lio_set_phys_id(struct net_device *netdev,
 						   &value);
 			if (ret)
 				return ret;
+		} else if (oct->chip_id == OCTEON_CN23XX_PF_VID) {
+			octnet_id_active(netdev, LED_IDENTIFICATION_ON);
+
+			/* returns 0 since updates are asynchronous */
+			return 0;
 		} else {
 			return -EINVAL;
 		}
@@ -521,7 +563,10 @@ static int lio_set_phys_id(struct net_device *netdev,
 						   &lio->phy_beacon_val);
 			if (ret)
 				return ret;
+		} else if (oct->chip_id == OCTEON_CN23XX_PF_VID) {
+			octnet_id_active(netdev, LED_IDENTIFICATION_OFF);
 
+			return 0;
 		} else {
 			return -EINVAL;
 		}
@@ -550,6 +595,13 @@ lio_ethtool_get_ringparam(struct net_device *netdev,
 		rx_max_pending = CN6XXX_MAX_OQ_DESCRIPTORS;
 		rx_pending = CFG_GET_NUM_RX_DESCS_NIC_IF(conf6x, lio->ifidx);
 		tx_pending = CFG_GET_NUM_TX_DESCS_NIC_IF(conf6x, lio->ifidx);
+	} else if (OCTEON_CN23XX_PF(oct)) {
+		struct octeon_config *conf23 = CHIP_FIELD(oct, cn23xx_pf, conf);
+
+		tx_max_pending = CN23XX_MAX_IQ_DESCRIPTORS;
+		rx_max_pending = CN23XX_MAX_OQ_DESCRIPTORS;
+		rx_pending = CFG_GET_NUM_RX_DESCS_NIC_IF(conf23, lio->ifidx);
+		tx_pending = CFG_GET_NUM_TX_DESCS_NIC_IF(conf23, lio->ifidx);
 	}
 
 	if (lio->mtu > OCTNET_DEFAULT_FRM_SIZE - OCTNET_FRM_HEADER_SIZE) {
@@ -610,6 +662,69 @@ lio_get_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause)
 	pause->rx_pause = oct->rx_pause;
 }
 
+static int
+lio_set_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause)
+{
+	/* Notes: Not supporting any auto negotiation in these
+	 * drivers.
+	 */
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct = lio->oct_dev;
+	struct octnic_ctrl_pkt nctrl;
+	struct oct_link_info *linfo = &lio->linfo;
+
+	int ret = 0;
+
+	if (oct->chip_id != OCTEON_CN23XX_PF_VID)
+		return -EINVAL;
+
+	if (linfo->link.s.duplex == 0) {
+		/*no flow control for half duplex*/
+		if (pause->rx_pause || pause->tx_pause)
+			return -EINVAL;
+	}
+
+	/*do not support autoneg of link flow control*/
+	if (pause->autoneg == AUTONEG_ENABLE)
+		return -EINVAL;
+
+	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
+
+	nctrl.ncmd.u64 = 0;
+	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_FLOW_CTL;
+	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
+	nctrl.wait_time = 100;
+	nctrl.netpndev = (u64)netdev;
+	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
+
+	if (pause->rx_pause) {
+		/*enable rx pause*/
+		nctrl.ncmd.s.param1 = 1;
+	} else {
+		/*disable rx pause*/
+		nctrl.ncmd.s.param1 = 0;
+	}
+
+	if (pause->tx_pause) {
+		/*enable tx pause*/
+		nctrl.ncmd.s.param2 = 1;
+	} else {
+		/*disable tx pause*/
+		nctrl.ncmd.s.param2 = 0;
+	}
+
+	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
+	if (ret < 0) {
+		dev_err(&oct->pci_dev->dev, "Failed to set pause parameter\n");
+		return -EINVAL;
+	}
+
+	oct->rx_pause = pause->rx_pause;
+	oct->tx_pause = pause->tx_pause;
+
+	return 0;
+}
+
 static void
 lio_get_ethtool_stats(struct net_device *netdev,
 		      struct ethtool_stats *stats  __attribute__((unused)),
@@ -877,6 +992,27 @@ lio_get_ethtool_stats(struct net_device *netdev,
 	}
 }
 
+static void lio_get_priv_flags_strings(struct lio *lio, u8 *data)
+{
+	struct octeon_device *oct_dev = lio->oct_dev;
+	int i;
+
+	switch (oct_dev->chip_id) {
+	case OCTEON_CN23XX_PF_VID:
+		for (i = 0; i < ARRAY_SIZE(oct_priv_flags_strings); i++) {
+			sprintf(data, "%s", oct_priv_flags_strings[i]);
+			data += ETH_GSTRING_LEN;
+		}
+		break;
+	case OCTEON_CN68XX:
+	case OCTEON_CN66XX:
+		break;
+	default:
+		netif_info(lio, drv, lio->netdev, "Unknown Chip !!\n");
+		break;
+	}
+}
+
 static void lio_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
 {
 	struct lio *lio = GET_LIO(netdev);
@@ -916,12 +1052,31 @@ static void lio_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
 		}
 		break;
 
+	case ETH_SS_PRIV_FLAGS:
+		lio_get_priv_flags_strings(lio, data);
+		break;
 	default:
 		netif_info(lio, drv, lio->netdev, "Unknown Stringset !!\n");
 		break;
 	}
 }
 
+static int lio_get_priv_flags_ss_count(struct lio *lio)
+{
+	struct octeon_device *oct_dev = lio->oct_dev;
+
+	switch (oct_dev->chip_id) {
+	case OCTEON_CN23XX_PF_VID:
+		return ARRAY_SIZE(oct_priv_flags_strings);
+	case OCTEON_CN68XX:
+	case OCTEON_CN66XX:
+		return -EOPNOTSUPP;
+	default:
+		netif_info(lio, drv, lio->netdev, "Unknown Chip !!\n");
+		return -EOPNOTSUPP;
+	}
+}
+
 static int lio_get_sset_count(struct net_device *netdev, int sset)
 {
 	struct lio *lio = GET_LIO(netdev);
@@ -932,6 +1087,8 @@ static int lio_get_sset_count(struct net_device *netdev, int sset)
 		return (ARRAY_SIZE(oct_stats_strings) +
 			ARRAY_SIZE(oct_iq_stats_strings) * oct_dev->num_iqs +
 			ARRAY_SIZE(oct_droq_stats_strings) * oct_dev->num_oqs);
+	case ETH_SS_PRIV_FLAGS:
+		return lio_get_priv_flags_ss_count(lio);
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -948,6 +1105,16 @@ static int lio_get_intr_coalesce(struct net_device *netdev,
 	intrmod_cfg = &oct->intrmod;
 
 	switch (oct->chip_id) {
+	case OCTEON_CN23XX_PF_VID:
+		if (!intrmod_cfg->rx_enable) {
+			intr_coal->rx_coalesce_usecs = intrmod_cfg->rx_usecs;
+			intr_coal->rx_max_coalesced_frames =
+				intrmod_cfg->rx_frames;
+		}
+		if (!intrmod_cfg->tx_enable)
+			intr_coal->tx_max_coalesced_frames =
+				intrmod_cfg->tx_frames;
+		break;
 	case OCTEON_CN68XX:
 	case OCTEON_CN66XX: {
 		struct octeon_cn6xxx *cn6xxx =
@@ -983,7 +1150,15 @@ static int lio_get_intr_coalesce(struct net_device *netdev,
 		intr_coal->rx_coalesce_usecs_low =
 			intrmod_cfg->rx_mintmr_trigger;
 		intr_coal->rx_max_coalesced_frames_low =
-			intrmod_cfg->rx_mincnt_trigger;
+		    intrmod_cfg->rx_mincnt_trigger;
+	}
+	if (OCTEON_CN23XX_PF(oct) &&
+	    (intrmod_cfg->tx_enable)) {
+		intr_coal->use_adaptive_tx_coalesce = intrmod_cfg->tx_enable;
+		intr_coal->tx_max_coalesced_frames_high =
+		    intrmod_cfg->tx_maxcnt_trigger;
+		intr_coal->tx_max_coalesced_frames_low =
+		    intrmod_cfg->tx_mincnt_trigger;
 	}
 	return 0;
 }
@@ -1060,11 +1235,11 @@ static void
 octnet_nic_stats_callback(struct octeon_device *oct_dev,
 			  u32 status, void *ptr)
 {
-	struct octeon_soft_command  *sc = (struct octeon_soft_command  *)ptr;
-	struct oct_nic_stats_resp *resp = (struct oct_nic_stats_resp *)
-		sc->virtrptr;
-	struct oct_nic_stats_ctrl *ctrl = (struct oct_nic_stats_ctrl *)
-		sc->ctxptr;
+	struct octeon_soft_command *sc = (struct octeon_soft_command *)ptr;
+	struct oct_nic_stats_resp *resp =
+	    (struct oct_nic_stats_resp *)sc->virtrptr;
+	struct oct_nic_stats_ctrl *ctrl =
+	    (struct oct_nic_stats_ctrl *)sc->ctxptr;
 	struct nic_rx_stats *rsp_rstats = &resp->stats.fromwire;
 	struct nic_tx_stats *rsp_tstats = &resp->stats.fromhost;
 
@@ -1314,14 +1489,35 @@ oct_cfg_rx_intrcnt(struct lio *lio, struct ethtool_coalesce *intr_coal)
 		CFG_SET_OQ_INTR_PKT(cn6xxx->conf, rx_max_coalesced_frames);
 		break;
 	}
+	case OCTEON_CN23XX_PF_VID: {
+		int q_no;
+
+		if (!intr_coal->rx_max_coalesced_frames)
+			rx_max_coalesced_frames = oct->intrmod.rx_frames;
+		else
+			rx_max_coalesced_frames =
+			    intr_coal->rx_max_coalesced_frames;
+		for (q_no = 0; q_no < oct->num_oqs; q_no++) {
+			q_no += oct->sriov_info.pf_srn;
+			octeon_write_csr64(
+			    oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(q_no),
+			    (octeon_read_csr64(
+				 oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(q_no)) &
+			     (0x3fffff00000000UL)) |
+				rx_max_coalesced_frames);
+			/*consider setting resend bit*/
+		}
+		oct->intrmod.rx_frames = rx_max_coalesced_frames;
+		break;
+	}
 	default:
 		return -EINVAL;
 	}
 	return 0;
 }
 
-static int oct_cfg_rx_intrtime(struct lio *lio, struct ethtool_coalesce
-			       *intr_coal)
+static int oct_cfg_rx_intrtime(struct lio *lio,
+			       struct ethtool_coalesce *intr_coal)
 {
 	struct octeon_device *oct = lio->oct_dev;
 	u32 time_threshold, rx_coalesce_usecs;
@@ -1346,6 +1542,27 @@ static int oct_cfg_rx_intrtime(struct lio *lio, struct ethtool_coalesce
 		CFG_SET_OQ_INTR_TIME(cn6xxx->conf, rx_coalesce_usecs);
 		break;
 	}
+	case OCTEON_CN23XX_PF_VID: {
+		u64 time_threshold;
+		int q_no;
+
+		if (!intr_coal->rx_coalesce_usecs)
+			rx_coalesce_usecs = oct->intrmod.rx_usecs;
+		else
+			rx_coalesce_usecs = intr_coal->rx_coalesce_usecs;
+		time_threshold =
+		    cn23xx_pf_get_oq_ticks(oct, (u32)rx_coalesce_usecs);
+		for (q_no = 0; q_no < oct->num_oqs; q_no++) {
+			q_no += oct->sriov_info.pf_srn;
+			octeon_write_csr64(oct,
+					   CN23XX_SLI_OQ_PKT_INT_LEVELS(q_no),
+					   (oct->intrmod.rx_frames |
+					    (time_threshold << 32)));
+			/*consider writing to resend bit here*/
+		}
+		oct->intrmod.rx_usecs = rx_coalesce_usecs;
+		break;
+	}
 	default:
 		return -EINVAL;
 	}
@@ -1358,12 +1575,37 @@ oct_cfg_tx_intrcnt(struct lio *lio, struct ethtool_coalesce *intr_coal
 		   __attribute__((unused)))
 {
 	struct octeon_device *oct = lio->oct_dev;
+	u32 iq_intr_pkt;
+	void __iomem *inst_cnt_reg;
+	u64 val;
 
 	/* Config Cnt based interrupt values */
 	switch (oct->chip_id) {
 	case OCTEON_CN68XX:
 	case OCTEON_CN66XX:
 		break;
+	case OCTEON_CN23XX_PF_VID: {
+		int q_no;
+
+		if (!intr_coal->tx_max_coalesced_frames)
+			iq_intr_pkt = CN23XX_DEF_IQ_INTR_THRESHOLD &
+				      CN23XX_PKT_IN_DONE_WMARK_MASK;
+		else
+			iq_intr_pkt = intr_coal->tx_max_coalesced_frames &
+				      CN23XX_PKT_IN_DONE_WMARK_MASK;
+		for (q_no = 0; q_no < oct->num_iqs; q_no++) {
+			inst_cnt_reg = (oct->instr_queue[q_no])->inst_cnt_reg;
+			val = readq(inst_cnt_reg);
+			/*clear wmark and count.dont want to write count back*/
+			val = (val & 0xFFFF000000000000ULL) |
+			      ((u64)iq_intr_pkt
+			       << CN23XX_PKT_IN_DONE_WMARK_BIT_POS);
+			writeq(val, inst_cnt_reg);
+			/*consider setting resend bit*/
+		}
+		oct->intrmod.tx_frames = iq_intr_pkt;
+		break;
+	}
 	default:
 		return -EINVAL;
 	}
@@ -1399,6 +1641,8 @@ static int lio_set_intr_coalesce(struct net_device *netdev,
 			return -EINVAL;
 		}
 		break;
+	case OCTEON_CN23XX_PF_VID:
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -1541,9 +1785,237 @@ static int lio_nway_reset(struct net_device *netdev)
 }
 
 /* Return register dump len. */
-static int lio_get_regs_len(struct net_device *dev __attribute__((unused)))
+static int lio_get_regs_len(struct net_device *dev)
 {
-	return OCT_ETHTOOL_REGDUMP_LEN;
+	struct lio *lio = GET_LIO(dev);
+	struct octeon_device *oct = lio->oct_dev;
+
+	switch (oct->chip_id) {
+	case OCTEON_CN23XX_PF_VID:
+		return OCT_ETHTOOL_REGDUMP_LEN_23XX;
+	default:
+		return OCT_ETHTOOL_REGDUMP_LEN;
+	}
+}
+
+static int cn23xx_read_csr_reg(char *s, struct octeon_device *oct)
+{
+	u32 reg;
+	u8 pf_num = oct->pf_num;
+	int len = 0;
+	int i;
+
+	/* PCI  Window Registers */
+
+	len += sprintf(s + len, "\n\t Octeon CSR Registers\n\n");
+
+	/*0x29030 or 0x29040*/
+	reg = CN23XX_SLI_PKT_MAC_RINFO64(oct->pcie_port, oct->pf_num);
+	len += sprintf(s + len,
+		       "\n[%08x] (SLI_PKT_MAC%d_PF%d_RINFO): %016llx\n",
+		       reg, oct->pcie_port, oct->pf_num,
+		       (u64)octeon_read_csr64(oct, reg));
+
+	/*0x27080 or 0x27090*/
+	reg = CN23XX_SLI_MAC_PF_INT_ENB64(oct->pcie_port, oct->pf_num);
+	len +=
+	    sprintf(s + len, "\n[%08x] (SLI_MAC%d_PF%d_INT_ENB): %016llx\n",
+		    reg, oct->pcie_port, oct->pf_num,
+		    (u64)octeon_read_csr64(oct, reg));
+
+	/*0x27000 or 0x27010*/
+	reg = CN23XX_SLI_MAC_PF_INT_SUM64(oct->pcie_port, oct->pf_num);
+	len +=
+	    sprintf(s + len, "\n[%08x] (SLI_MAC%d_PF%d_INT_SUM): %016llx\n",
+		    reg, oct->pcie_port, oct->pf_num,
+		    (u64)octeon_read_csr64(oct, reg));
+
+	/*0x29120*/
+	reg = 0x29120;
+	len += sprintf(s + len, "\n[%08x] (SLI_PKT_MEM_CTL): %016llx\n", reg,
+		       (u64)octeon_read_csr64(oct, reg));
+
+	/*0x27300*/
+	reg = 0x27300 + oct->pcie_port * CN23XX_MAC_INT_OFFSET +
+	      (oct->pf_num) * CN23XX_PF_INT_OFFSET;
+	len += sprintf(
+	    s + len, "\n[%08x] (SLI_MAC%d_PF%d_PKT_VF_INT): %016llx\n", reg,
+	    oct->pcie_port, oct->pf_num, (u64)octeon_read_csr64(oct, reg));
+
+	/*0x27200*/
+	reg = 0x27200 + oct->pcie_port * CN23XX_MAC_INT_OFFSET +
+	      (oct->pf_num) * CN23XX_PF_INT_OFFSET;
+	len += sprintf(s + len,
+		       "\n[%08x] (SLI_MAC%d_PF%d_PP_VF_INT): %016llx\n",
+		       reg, oct->pcie_port, oct->pf_num,
+		       (u64)octeon_read_csr64(oct, reg));
+
+	/*29130*/
+	reg = CN23XX_SLI_PKT_CNT_INT;
+	len += sprintf(s + len, "\n[%08x] (SLI_PKT_CNT_INT): %016llx\n", reg,
+		       (u64)octeon_read_csr64(oct, reg));
+
+	/*0x29140*/
+	reg = CN23XX_SLI_PKT_TIME_INT;
+	len += sprintf(s + len, "\n[%08x] (SLI_PKT_TIME_INT): %016llx\n", reg,
+		       (u64)octeon_read_csr64(oct, reg));
+
+	/*0x29160*/
+	reg = 0x29160;
+	len += sprintf(s + len, "\n[%08x] (SLI_PKT_INT): %016llx\n", reg,
+		       (u64)octeon_read_csr64(oct, reg));
+
+	/*0x29180*/
+	reg = CN23XX_SLI_OQ_WMARK;
+	len += sprintf(s + len, "\n[%08x] (SLI_PKT_OUTPUT_WMARK): %016llx\n",
+		       reg, (u64)octeon_read_csr64(oct, reg));
+
+	/*0x291E0*/
+	reg = CN23XX_SLI_PKT_IOQ_RING_RST;
+	len += sprintf(s + len, "\n[%08x] (SLI_PKT_RING_RST): %016llx\n", reg,
+		       (u64)octeon_read_csr64(oct, reg));
+
+	/*0x29210*/
+	reg = CN23XX_SLI_GBL_CONTROL;
+	len += sprintf(s + len,
+		       "\n[%08x] (SLI_PKT_GBL_CONTROL): %016llx\n", reg,
+		       (u64)octeon_read_csr64(oct, reg));
+
+	/*0x29220*/
+	reg = 0x29220;
+	len += sprintf(s + len, "\n[%08x] (SLI_PKT_BIST_STATUS): %016llx\n",
+		       reg, (u64)octeon_read_csr64(oct, reg));
+
+	/*PF only*/
+	if (pf_num == 0) {
+		/*0x29260*/
+		reg = CN23XX_SLI_OUT_BP_EN_W1S;
+		len += sprintf(s + len,
+			       "\n[%08x] (SLI_PKT_OUT_BP_EN_W1S):  %016llx\n",
+			       reg, (u64)octeon_read_csr64(oct, reg));
+	} else if (pf_num == 1) {
+		/*0x29270*/
+		reg = CN23XX_SLI_OUT_BP_EN2_W1S;
+		len += sprintf(s + len,
+			       "\n[%08x] (SLI_PKT_OUT_BP_EN2_W1S): %016llx\n",
+			       reg, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		reg = CN23XX_SLI_OQ_BUFF_INFO_SIZE(i);
+		len +=
+		    sprintf(s + len, "\n[%08x] (SLI_PKT%d_OUT_SIZE): %016llx\n",
+			    reg, i, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	/*0x10040*/
+	for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++) {
+		reg = CN23XX_SLI_IQ_INSTR_COUNT64(i);
+		len += sprintf(s + len,
+			       "\n[%08x] (SLI_PKT_IN_DONE%d_CNTS): %016llx\n",
+			       reg, i, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	/*0x10080*/
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		reg = CN23XX_SLI_OQ_PKTS_CREDIT(i);
+		len += sprintf(s + len,
+			       "\n[%08x] (SLI_PKT%d_SLIST_BAOFF_DBELL): %016llx\n",
+			       reg, i, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	/*0x10090*/
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		reg = CN23XX_SLI_OQ_SIZE(i);
+		len += sprintf(
+		    s + len, "\n[%08x] (SLI_PKT%d_SLIST_FIFO_RSIZE): %016llx\n",
+		    reg, i, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	/*0x10050*/
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		reg = CN23XX_SLI_OQ_PKT_CONTROL(i);
+		len += sprintf(
+			s + len,
+			"\n[%08x] (SLI_PKT%d__OUTPUT_CONTROL): %016llx\n",
+			reg, i, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	/*0x10070*/
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		reg = CN23XX_SLI_OQ_BASE_ADDR64(i);
+		len += sprintf(s + len,
+			       "\n[%08x] (SLI_PKT%d_SLIST_BADDR): %016llx\n",
+			       reg, i, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	/*0x100a0*/
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		reg = CN23XX_SLI_OQ_PKT_INT_LEVELS(i);
+		len += sprintf(s + len,
+			       "\n[%08x] (SLI_PKT%d_INT_LEVELS): %016llx\n",
+			       reg, i, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	/*0x100b0*/
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		reg = CN23XX_SLI_OQ_PKTS_SENT(i);
+		len += sprintf(s + len, "\n[%08x] (SLI_PKT%d_CNTS): %016llx\n",
+			       reg, i, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	/*0x100c0*/
+	for (i = 0; i < CN23XX_MAX_OUTPUT_QUEUES; i++) {
+		reg = 0x100c0 + i * CN23XX_OQ_OFFSET;
+		len += sprintf(s + len,
+			       "\n[%08x] (SLI_PKT%d_ERROR_INFO): %016llx\n",
+			       reg, i, (u64)octeon_read_csr64(oct, reg));
+
+		/*0x10000*/
+		for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++) {
+			reg = CN23XX_SLI_IQ_PKT_CONTROL64(i);
+			len += sprintf(
+				s + len,
+				"\n[%08x] (SLI_PKT%d_INPUT_CONTROL): %016llx\n",
+				reg, i, (u64)octeon_read_csr64(oct, reg));
+		}
+
+		/*0x10010*/
+		for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++) {
+			reg = CN23XX_SLI_IQ_BASE_ADDR64(i);
+			len += sprintf(
+			    s + len,
+			    "\n[%08x] (SLI_PKT%d_INSTR_BADDR): %016llx\n", reg,
+			    i, (u64)octeon_read_csr64(oct, reg));
+		}
+
+		/*0x10020*/
+		for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++) {
+			reg = CN23XX_SLI_IQ_DOORBELL(i);
+			len += sprintf(
+			    s + len,
+			    "\n[%08x] (SLI_PKT%d_INSTR_BAOFF_DBELL): %016llx\n",
+			    reg, i, (u64)octeon_read_csr64(oct, reg));
+		}
+
+		/*0x10030*/
+		for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++) {
+			reg = CN23XX_SLI_IQ_SIZE(i);
+			len += sprintf(
+			    s + len,
+			    "\n[%08x] (SLI_PKT%d_INSTR_FIFO_RSIZE): %016llx\n",
+			    reg, i, (u64)octeon_read_csr64(oct, reg));
+		}
+
+		/*0x10040*/
+		for (i = 0; i < CN23XX_MAX_INPUT_QUEUES; i++)
+			reg = CN23XX_SLI_IQ_INSTR_COUNT64(i);
+		len += sprintf(s + len,
+			       "\n[%08x] (SLI_PKT_IN_DONE%d_CNTS): %016llx\n",
+			       reg, i, (u64)octeon_read_csr64(oct, reg));
+	}
+
+	return len;
 }
 
 static int cn6xxx_read_csr_reg(char *s, struct octeon_device *oct)
@@ -1688,6 +2160,10 @@ static void lio_get_regs(struct net_device *dev,
 	regs->version = OCT_ETHTOOL_REGSVER;
 
 	switch (oct->chip_id) {
+	case OCTEON_CN23XX_PF_VID:
+		memset(regbuf, 0, OCT_ETHTOOL_REGDUMP_LEN_23XX);
+		len += cn23xx_read_csr_reg(regbuf + len, oct);
+		break;
 	case OCTEON_CN68XX:
 	case OCTEON_CN66XX:
 		memset(regbuf, 0, OCT_ETHTOOL_REGDUMP_LEN);
@@ -1729,6 +2205,7 @@ static const struct ethtool_ops lio_ethtool_ops = {
 	.get_strings		= lio_get_strings,
 	.get_ethtool_stats	= lio_get_ethtool_stats,
 	.get_pauseparam		= lio_get_pauseparam,
+	.set_pauseparam		= lio_set_pauseparam,
 	.get_regs_len		= lio_get_regs_len,
 	.get_regs		= lio_get_regs,
 	.get_msglevel		= lio_get_msglevel,
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c
index 20d6942edf40..afc6f9dc8119 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c
@@ -21,11 +21,10 @@
 **********************************************************************/
 #include <linux/version.h>
 #include <linux/pci.h>
-#include <linux/net_tstamp.h>
-#include <linux/if_vlan.h>
 #include <linux/firmware.h>
 #include <linux/ptp_clock_kernel.h>
 #include <net/vxlan.h>
+#include <linux/kthread.h>
 #include "liquidio_common.h"
 #include "octeon_droq.h"
 #include "octeon_iq.h"
@@ -37,6 +36,7 @@
 #include "cn66xx_regs.h"
 #include "cn66xx_device.h"
 #include "cn68xx_device.h"
+#include "cn23xx_pf_device.h"
 #include "liquidio_image.h"
 
 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
@@ -52,11 +52,6 @@ module_param(ddr_timeout, int, 0644);
 MODULE_PARM_DESC(ddr_timeout,
 		 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
 
-static u32 console_bitmask;
-module_param(console_bitmask, int, 0644);
-MODULE_PARM_DESC(console_bitmask,
-		 "Bitmask indicating which consoles have debug output redirected to syslog.");
-
 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
 
 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count)  \
@@ -102,6 +97,14 @@ struct liquidio_if_cfg_resp {
 	u64 status;
 };
 
+struct liquidio_rx_ctl_context {
+	int octeon_id;
+
+	wait_queue_head_t wc;
+
+	int cond;
+};
+
 struct oct_link_status_resp {
 	u64 rh;
 	struct oct_link_info link_info;
@@ -139,7 +142,8 @@ union tx_info {
 #define OCTNIC_MAX_SG  (MAX_SKB_FRAGS)
 
 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
-#define OCTNIC_GSO_MAX_SIZE (GSO_MAX_SIZE - OCTNIC_GSO_MAX_HEADER_SIZE)
+#define OCTNIC_GSO_MAX_SIZE                                                    \
+	(CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
 
 /** Structure of a node in list of gather components maintained by
  * NIC driver for each network device.
@@ -162,27 +166,6 @@ struct octnic_gather {
 	u64 sg_dma_ptr;
 };
 
-/** This structure is used by NIC driver to store information required
- * to free the sk_buff when the packet has been fetched by Octeon.
- * Bytes offset below assume worst-case of a 64-bit system.
- */
-struct octnet_buf_free_info {
-	/** Bytes 1-8.  Pointer to network device private structure. */
-	struct lio *lio;
-
-	/** Bytes 9-16.  Pointer to sk_buff. */
-	struct sk_buff *skb;
-
-	/** Bytes 17-24.  Pointer to gather list. */
-	struct octnic_gather *g;
-
-	/** Bytes 25-32. Physical address of skb->data or gather list. */
-	u64 dptr;
-
-	/** Bytes 33-47. Piggybacked soft command, if any */
-	struct octeon_soft_command *sc;
-};
-
 struct handshake {
 	struct completion init;
 	struct completion started;
@@ -198,6 +181,7 @@ struct octeon_device_priv {
 };
 
 static int octeon_device_init(struct octeon_device *);
+static int liquidio_stop(struct net_device *netdev);
 static void liquidio_remove(struct pci_dev *pdev);
 static int liquidio_probe(struct pci_dev *pdev,
 			  const struct pci_device_id *ent);
@@ -219,6 +203,20 @@ static void octeon_droq_bh(unsigned long pdev)
 			continue;
 		reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 							  MAX_PACKET_BUDGET);
+		lio_enable_irq(oct->droq[q_no], NULL);
+
+		if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
+			/* set time and cnt interrupt thresholds for this DROQ
+			 * for NAPI
+			 */
+			int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
+
+			octeon_write_csr64(
+			    oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
+			    0x5700000040ULL);
+			octeon_write_csr64(
+			    oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
+		}
 	}
 
 	if (reschedule)
@@ -252,76 +250,6 @@ static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 	return pkt_cnt;
 }
 
-void octeon_report_tx_completion_to_bql(void *txq, unsigned int pkts_compl,
-					unsigned int bytes_compl)
-{
-	struct netdev_queue *netdev_queue = txq;
-
-	netdev_tx_completed_queue(netdev_queue, pkts_compl, bytes_compl);
-}
-
-void octeon_update_tx_completion_counters(void *buf, int reqtype,
-					  unsigned int *pkts_compl,
-					  unsigned int *bytes_compl)
-{
-	struct octnet_buf_free_info *finfo;
-	struct sk_buff *skb = NULL;
-	struct octeon_soft_command *sc;
-
-	switch (reqtype) {
-	case REQTYPE_NORESP_NET:
-	case REQTYPE_NORESP_NET_SG:
-		finfo = buf;
-		skb = finfo->skb;
-		break;
-
-	case REQTYPE_RESP_NET_SG:
-	case REQTYPE_RESP_NET:
-		sc = buf;
-		skb = sc->callback_arg;
-		break;
-
-	default:
-		return;
-	}
-
-	(*pkts_compl)++;
-	*bytes_compl += skb->len;
-}
-
-void octeon_report_sent_bytes_to_bql(void *buf, int reqtype)
-{
-	struct octnet_buf_free_info *finfo;
-	struct sk_buff *skb;
-	struct octeon_soft_command *sc;
-	struct netdev_queue *txq;
-
-	switch (reqtype) {
-	case REQTYPE_NORESP_NET:
-	case REQTYPE_NORESP_NET_SG:
-		finfo = buf;
-		skb = finfo->skb;
-		break;
-
-	case REQTYPE_RESP_NET_SG:
-	case REQTYPE_RESP_NET:
-		sc = buf;
-		skb = sc->callback_arg;
-		break;
-
-	default:
-		return;
-	}
-
-	txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb));
-	netdev_tx_sent_queue(txq, skb->len);
-}
-
-int octeon_console_debug_enabled(u32 console)
-{
-	return (console_bitmask >> (console)) & 0x1;
-}
-
 /**
  * \brief Forces all IO queues off on a given device
  * @param oct Pointer to Octeon device
@@ -441,7 +369,7 @@ static void stop_pci_io(struct octeon_device *oct)
 	pci_disable_device(oct->pci_dev);
 
 	/* Disable interrupts  */
-	oct->fn_list.disable_interrupt(oct->chip);
+	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
 
 	pcierror_quiesce_device(oct);
 
@@ -570,6 +498,9 @@ static const struct pci_device_id liquidio_pci_tbl[] = {
 	{       /* 66xx */
 		PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 	},
+	{       /* 23xx pf */
+		PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
+	},
 	{
 		0, 0, 0, 0, 0, 0, 0
 	}
@@ -587,7 +518,6 @@ static struct pci_driver liquidio_pci_driver = {
 	.suspend	= liquidio_suspend,
 	.resume		= liquidio_resume,
 #endif
-
 };
 
 /**
@@ -936,6 +866,52 @@ static void print_link_info(struct net_device *netdev)
 }
 
 /**
+ * \brief Routine to notify MTU change
+ * @param work work_struct data structure
+ */
+static void octnet_link_status_change(struct work_struct *work)
+{
+	struct cavium_wk *wk = (struct cavium_wk *)work;
+	struct lio *lio = (struct lio *)wk->ctxptr;
+
+	rtnl_lock();
+	call_netdevice_notifiers(NETDEV_CHANGEMTU, lio->netdev);
+	rtnl_unlock();
+}
+
+/**
+ * \brief Sets up the mtu status change work
+ * @param netdev network device
+ */
+static inline int setup_link_status_change_wq(struct net_device *netdev)
+{
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct = lio->oct_dev;
+
+	lio->link_status_wq.wq = alloc_workqueue("link-status",
+						 WQ_MEM_RECLAIM, 0);
+	if (!lio->link_status_wq.wq) {
+		dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
+		return -1;
+	}
+	INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
+			  octnet_link_status_change);
+	lio->link_status_wq.wk.ctxptr = lio;
+
+	return 0;
+}
+
+static inline void cleanup_link_status_change_wq(struct net_device *netdev)
+{
+	struct lio *lio = GET_LIO(netdev);
+
+	if (lio->link_status_wq.wq) {
+		cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
+		destroy_workqueue(lio->link_status_wq.wq);
+	}
+}
+
+/**
  * \brief Update link status
  * @param netdev network device
  * @param ls link status structure
@@ -973,8 +949,6 @@ static void update_txq_status(struct octeon_device *oct, int iq_num)
 	struct lio *lio;
 	struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
 
-	/*octeon_update_iq_read_idx(oct, iq);*/
-
 	netdev = oct->props[iq->ifidx].netdev;
 
 	/* This is needed because the first IQ does not have
@@ -1002,12 +976,32 @@ static void update_txq_status(struct octeon_device *oct, int iq_num)
 	}
 }
 
+static
+int liquidio_schedule_msix_droq_pkt_handler(struct octeon_droq *droq, u64 ret)
+{
+	struct octeon_device *oct = droq->oct_dev;
+	struct octeon_device_priv *oct_priv =
+	    (struct octeon_device_priv *)oct->priv;
+
+	if (droq->ops.poll_mode) {
+		droq->ops.napi_fn(droq);
+	} else {
+		if (ret & MSIX_PO_INT) {
+			tasklet_schedule(&oct_priv->droq_tasklet);
+			return 1;
+		}
+		/* this will be flushed periodically by check iq db */
+		if (ret & MSIX_PI_INT)
+			return 0;
+	}
+	return 0;
+}
+
 /**
  * \brief Droq packet processor sceduler
  * @param oct octeon device
  */
-static
-void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
+static void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
 {
 	struct octeon_device_priv *oct_priv =
 		(struct octeon_device_priv *)oct->priv;
@@ -1032,19 +1026,36 @@ void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
 	}
 }
 
+static irqreturn_t
+liquidio_msix_intr_handler(int irq __attribute__((unused)), void *dev)
+{
+	u64 ret;
+	struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev;
+	struct octeon_device *oct = ioq_vector->oct_dev;
+	struct octeon_droq *droq = oct->droq[ioq_vector->droq_index];
+
+	ret = oct->fn_list.msix_interrupt_handler(ioq_vector);
+
+	if ((ret & MSIX_PO_INT) || (ret & MSIX_PI_INT))
+		liquidio_schedule_msix_droq_pkt_handler(droq, ret);
+
+	return IRQ_HANDLED;
+}
+
 /**
  * \brief Interrupt handler for octeon
  * @param irq unused
  * @param dev octeon device
  */
 static
-irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev)
+irqreturn_t liquidio_legacy_intr_handler(int irq __attribute__((unused)),
+					 void *dev)
 {
 	struct octeon_device *oct = (struct octeon_device *)dev;
 	irqreturn_t ret;
 
 	/* Disable our interrupts for the duration of ISR */
-	oct->fn_list.disable_interrupt(oct->chip);
+	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
 
 	ret = oct->fn_list.process_interrupt_regs(oct);
 
@@ -1053,7 +1064,7 @@ irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev)
 
 	/* Re-enable our interrupts  */
 	if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
-		oct->fn_list.enable_interrupt(oct->chip);
+		oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
 
 	return ret;
 }
@@ -1067,22 +1078,204 @@ irqreturn_t liquidio_intr_handler(int irq __attribute__((unused)), void *dev)
 static int octeon_setup_interrupt(struct octeon_device *oct)
 {
 	int irqret, err;
+	struct msix_entry *msix_entries;
+	int i;
+	int num_ioq_vectors;
+	int num_alloc_ioq_vectors;
 
-	err = pci_enable_msi(oct->pci_dev);
-	if (err)
-		dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
-			 err);
-	else
-		oct->flags |= LIO_FLAG_MSI_ENABLED;
-
-	irqret = request_irq(oct->pci_dev->irq, liquidio_intr_handler,
-			     IRQF_SHARED, "octeon", oct);
-	if (irqret) {
-		if (oct->flags & LIO_FLAG_MSI_ENABLED)
-			pci_disable_msi(oct->pci_dev);
-		dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
-			irqret);
-		return 1;
+	if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
+		oct->num_msix_irqs = oct->sriov_info.num_pf_rings;
+		/* one non ioq interrupt for handling sli_mac_pf_int_sum */
+		oct->num_msix_irqs += 1;
+
+		oct->msix_entries = kcalloc(
+		    oct->num_msix_irqs, sizeof(struct msix_entry), GFP_KERNEL);
+		if (!oct->msix_entries)
+			return 1;
+
+		msix_entries = (struct msix_entry *)oct->msix_entries;
+		/*Assumption is that pf msix vectors start from pf srn to pf to
+		 * trs and not from 0. if not change this code
+		 */
+		for (i = 0; i < oct->num_msix_irqs - 1; i++)
+			msix_entries[i].entry = oct->sriov_info.pf_srn + i;
+		msix_entries[oct->num_msix_irqs - 1].entry =
+		    oct->sriov_info.trs;
+		num_alloc_ioq_vectors = pci_enable_msix_range(
+						oct->pci_dev, msix_entries,
+						oct->num_msix_irqs,
+						oct->num_msix_irqs);
+		if (num_alloc_ioq_vectors < 0) {
+			dev_err(&oct->pci_dev->dev, "unable to Allocate MSI-X interrupts\n");
+			kfree(oct->msix_entries);
+			oct->msix_entries = NULL;
+			return 1;
+		}
+		dev_dbg(&oct->pci_dev->dev, "OCTEON: Enough MSI-X interrupts are allocated...\n");
+
+		num_ioq_vectors = oct->num_msix_irqs;
+
+		/** For PF, there is one non-ioq interrupt handler */
+		num_ioq_vectors -= 1;
+		irqret = request_irq(msix_entries[num_ioq_vectors].vector,
+				     liquidio_legacy_intr_handler, 0, "octeon",
+				     oct);
+		if (irqret) {
+			dev_err(&oct->pci_dev->dev,
+				"OCTEON: Request_irq failed for MSIX interrupt Error: %d\n",
+				irqret);
+			pci_disable_msix(oct->pci_dev);
+			kfree(oct->msix_entries);
+			oct->msix_entries = NULL;
+			return 1;
+		}
+
+		for (i = 0; i < num_ioq_vectors; i++) {
+			irqret = request_irq(msix_entries[i].vector,
+					     liquidio_msix_intr_handler, 0,
+					     "octeon", &oct->ioq_vector[i]);
+			if (irqret) {
+				dev_err(&oct->pci_dev->dev,
+					"OCTEON: Request_irq failed for MSIX interrupt Error: %d\n",
+					irqret);
+				/** Freeing the non-ioq irq vector here . */
+				free_irq(msix_entries[num_ioq_vectors].vector,
+					 oct);
+
+				while (i) {
+					i--;
+					/** clearing affinity mask. */
+					irq_set_affinity_hint(
+						msix_entries[i].vector, NULL);
+					free_irq(msix_entries[i].vector,
+						 &oct->ioq_vector[i]);
+				}
+				pci_disable_msix(oct->pci_dev);
+				kfree(oct->msix_entries);
+				oct->msix_entries = NULL;
+				return 1;
+			}
+			oct->ioq_vector[i].vector = msix_entries[i].vector;
+			/* assign the cpu mask for this msix interrupt vector */
+			irq_set_affinity_hint(
+					msix_entries[i].vector,
+					(&oct->ioq_vector[i].affinity_mask));
+		}
+		dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: MSI-X enabled\n",
+			oct->octeon_id);
+	} else {
+		err = pci_enable_msi(oct->pci_dev);
+		if (err)
+			dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
+				 err);
+		else
+			oct->flags |= LIO_FLAG_MSI_ENABLED;
+
+		irqret = request_irq(oct->pci_dev->irq,
+				     liquidio_legacy_intr_handler, IRQF_SHARED,
+				     "octeon", oct);
+		if (irqret) {
+			if (oct->flags & LIO_FLAG_MSI_ENABLED)
+				pci_disable_msi(oct->pci_dev);
+			dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
+				irqret);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static int liquidio_watchdog(void *param)
+{
+	u64 wdog;
+	u16 mask_of_stuck_cores = 0;
+	u16 mask_of_crashed_cores = 0;
+	int core_num;
+	u8 core_is_stuck[LIO_MAX_CORES];
+	u8 core_crashed[LIO_MAX_CORES];
+	struct octeon_device *oct = param;
+
+	memset(core_is_stuck, 0, sizeof(core_is_stuck));
+	memset(core_crashed, 0, sizeof(core_crashed));
+
+	while (!kthread_should_stop()) {
+		mask_of_crashed_cores =
+		    (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
+
+		for (core_num = 0; core_num < LIO_MAX_CORES; core_num++) {
+			if (!core_is_stuck[core_num]) {
+				wdog = lio_pci_readq(oct, CIU3_WDOG(core_num));
+
+				/* look at watchdog state field */
+				wdog &= CIU3_WDOG_MASK;
+				if (wdog) {
+					/* this watchdog timer has expired */
+					core_is_stuck[core_num] =
+						LIO_MONITOR_WDOG_EXPIRE;
+					mask_of_stuck_cores |= (1 << core_num);
+				}
+			}
+
+			if (!core_crashed[core_num])
+				core_crashed[core_num] =
+				    (mask_of_crashed_cores >> core_num) & 1;
+		}
+
+		if (mask_of_stuck_cores) {
+			for (core_num = 0; core_num < LIO_MAX_CORES;
+			     core_num++) {
+				if (core_is_stuck[core_num] == 1) {
+					dev_err(&oct->pci_dev->dev,
+						"ERROR: Octeon core %d is stuck!\n",
+						core_num);
+					/* 2 means we have printk'd  an error
+					 * so no need to repeat the same printk
+					 */
+					core_is_stuck[core_num] =
+						LIO_MONITOR_CORE_STUCK_MSGD;
+				}
+			}
+		}
+
+		if (mask_of_crashed_cores) {
+			for (core_num = 0; core_num < LIO_MAX_CORES;
+			     core_num++) {
+				if (core_crashed[core_num] == 1) {
+					dev_err(&oct->pci_dev->dev,
+						"ERROR: Octeon core %d crashed!  See oct-fwdump for details.\n",
+						core_num);
+					/* 2 means we have printk'd  an error
+					 * so no need to repeat the same printk
+					 */
+					core_crashed[core_num] =
+						LIO_MONITOR_CORE_STUCK_MSGD;
+				}
+			}
+		}
+#ifdef CONFIG_MODULE_UNLOAD
+		if (mask_of_stuck_cores || mask_of_crashed_cores) {
+			/* make module refcount=0 so that rmmod will work */
+			long refcount;
+
+			refcount = module_refcount(THIS_MODULE);
+
+			while (refcount > 0) {
+				module_put(THIS_MODULE);
+				refcount = module_refcount(THIS_MODULE);
+			}
+
+			/* compensate for and withstand an unlikely (but still
+			 * possible) race condition
+			 */
+			while (refcount < 0) {
+				try_module_get(THIS_MODULE);
+				refcount = module_refcount(THIS_MODULE);
+			}
+		}
+#endif
+		/* sleep for two seconds */
+		set_current_state(TASK_INTERRUPTIBLE);
+		schedule_timeout(2 * HZ);
 	}
 
 	return 0;
@@ -1107,6 +1300,9 @@ liquidio_probe(struct pci_dev *pdev,
 		return -ENOMEM;
 	}
 
+	if (pdev->device == OCTEON_CN23XX_PF_VID)
+		oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
+
 	dev_info(&pdev->dev, "Initializing device %x:%x.\n",
 		 (u32)pdev->vendor, (u32)pdev->device);
 
@@ -1130,6 +1326,30 @@ liquidio_probe(struct pci_dev *pdev,
 		return -ENOMEM;
 	}
 
+	if (OCTEON_CN23XX_PF(oct_dev)) {
+		u64 scratch1;
+		u8 bus, device, function;
+
+		scratch1 = octeon_read_csr64(oct_dev, CN23XX_SLI_SCRATCH1);
+		if (!(scratch1 & 4ULL)) {
+			/* Bit 2 of SLI_SCRATCH_1 is a flag that indicates that
+			 * the lio watchdog kernel thread is running for this
+			 * NIC.  Each NIC gets one watchdog kernel thread.
+			 */
+			scratch1 |= 4ULL;
+			octeon_write_csr64(oct_dev, CN23XX_SLI_SCRATCH1,
+					   scratch1);
+
+			bus = pdev->bus->number;
+			device = PCI_SLOT(pdev->devfn);
+			function = PCI_FUNC(pdev->devfn);
+			oct_dev->watchdog_task = kthread_create(
+			    liquidio_watchdog, oct_dev,
+			    "liowd/%02hhx:%02hhx.%hhx", bus, device, function);
+			wake_up_process(oct_dev->watchdog_task);
+		}
+	}
+
 	oct_dev->rx_pause = 1;
 	oct_dev->tx_pause = 1;
 
@@ -1146,6 +1366,7 @@ liquidio_probe(struct pci_dev *pdev,
 static void octeon_destroy_resources(struct octeon_device *oct)
 {
 	int i;
+	struct msix_entry *msix_entries;
 	struct octeon_device_priv *oct_priv =
 		(struct octeon_device_priv *)oct->priv;
 
@@ -1190,21 +1411,40 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 			dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
 
 		/* Disable interrupts  */
-		oct->fn_list.disable_interrupt(oct->chip);
+		oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
+
+		if (oct->msix_on) {
+			msix_entries = (struct msix_entry *)oct->msix_entries;
+			for (i = 0; i < oct->num_msix_irqs - 1; i++) {
+				/* clear the affinity_cpumask */
+				irq_set_affinity_hint(msix_entries[i].vector,
+						      NULL);
+				free_irq(msix_entries[i].vector,
+					 &oct->ioq_vector[i]);
+			}
+			/* non-iov vector's argument is oct struct */
+			free_irq(msix_entries[i].vector, oct);
 
-		/* Release the interrupt line */
-		free_irq(oct->pci_dev->irq, oct);
+			pci_disable_msix(oct->pci_dev);
+			kfree(oct->msix_entries);
+			oct->msix_entries = NULL;
+		} else {
+			/* Release the interrupt line */
+			free_irq(oct->pci_dev->irq, oct);
 
-		if (oct->flags & LIO_FLAG_MSI_ENABLED)
-			pci_disable_msi(oct->pci_dev);
+			if (oct->flags & LIO_FLAG_MSI_ENABLED)
+				pci_disable_msi(oct->pci_dev);
+		}
 
-		/* fallthrough */
+		if (OCTEON_CN23XX_PF(oct))
+			octeon_free_ioq_vector(oct);
+	/* fallthrough */
 	case OCT_DEV_IN_RESET:
 	case OCT_DEV_DROQ_INIT_DONE:
 		/*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
 		mdelay(100);
 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
-			if (!(oct->io_qmask.oq & (1ULL << i)))
+			if (!(oct->io_qmask.oq & BIT_ULL(i)))
 				continue;
 			octeon_delete_droq(oct, i);
 		}
@@ -1226,16 +1466,15 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 		octeon_delete_response_list(oct);
 
 		/* fallthrough */
-	case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
-		octeon_free_sc_buffer_pool(oct);
-
-		/* fallthrough */
 	case OCT_DEV_INSTR_QUEUE_INIT_DONE:
 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
-			if (!(oct->io_qmask.iq & (1ULL << i)))
+			if (!(oct->io_qmask.iq & BIT_ULL(i)))
 				continue;
 			octeon_delete_instr_queue(oct, i);
 		}
+		/* fallthrough */
+	case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
+		octeon_free_sc_buffer_pool(oct);
 
 		/* fallthrough */
 	case OCT_DEV_DISPATCH_INIT_DONE:
@@ -1244,9 +1483,9 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 
 		/* fallthrough */
 	case OCT_DEV_PCI_MAP_DONE:
-
 		/* Soft reset the octeon device before exiting */
-		oct->fn_list.soft_reset(oct);
+		if ((!OCTEON_CN23XX_PF(oct)) || !oct->octeon_id)
+			oct->fn_list.soft_reset(oct);
 
 		octeon_unmap_pci_barx(oct, 0);
 		octeon_unmap_pci_barx(oct, 1);
@@ -1264,23 +1503,89 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 }
 
 /**
+ * \brief Callback for rx ctrl
+ * @param status status of request
+ * @param buf pointer to resp structure
+ */
+static void rx_ctl_callback(struct octeon_device *oct,
+			    u32 status,
+			    void *buf)
+{
+	struct octeon_soft_command *sc = (struct octeon_soft_command *)buf;
+	struct liquidio_rx_ctl_context *ctx;
+
+	ctx  = (struct liquidio_rx_ctl_context *)sc->ctxptr;
+
+	oct = lio_get_device(ctx->octeon_id);
+	if (status)
+		dev_err(&oct->pci_dev->dev, "rx ctl instruction failed. Status: %llx\n",
+			CVM_CAST64(status));
+	WRITE_ONCE(ctx->cond, 1);
+
+	/* This barrier is required to be sure that the response has been
+	 * written fully before waking up the handler
+	 */
+	wmb();
+
+	wake_up_interruptible(&ctx->wc);
+}
+
+/**
  * \brief Send Rx control command
  * @param lio per-network private data
  * @param start_stop whether to start or stop
  */
 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
 {
-	struct octnic_ctrl_pkt nctrl;
+	struct octeon_soft_command *sc;
+	struct liquidio_rx_ctl_context *ctx;
+	union octnet_cmd *ncmd;
+	int ctx_size = sizeof(struct liquidio_rx_ctl_context);
+	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
+	int retval;
 
-	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
+	if (oct->props[lio->ifidx].rx_on == start_stop)
+		return;
 
-	nctrl.ncmd.s.cmd = OCTNET_CMD_RX_CTL;
-	nctrl.ncmd.s.param1 = start_stop;
-	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
-	nctrl.netpndev = (u64)lio->netdev;
+	sc = (struct octeon_soft_command *)
+		octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
+					  16, ctx_size);
 
-	if (octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl) < 0)
+	ncmd = (union octnet_cmd *)sc->virtdptr;
+	ctx  = (struct liquidio_rx_ctl_context *)sc->ctxptr;
+
+	WRITE_ONCE(ctx->cond, 0);
+	ctx->octeon_id = lio_get_device_id(oct);
+	init_waitqueue_head(&ctx->wc);
+
+	ncmd->u64 = 0;
+	ncmd->s.cmd = OCTNET_CMD_RX_CTL;
+	ncmd->s.param1 = start_stop;
+
+	octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
+
+	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
+
+	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
+				    OPCODE_NIC_CMD, 0, 0, 0);
+
+	sc->callback = rx_ctl_callback;
+	sc->callback_arg = sc;
+	sc->wait_time = 5000;
+
+	retval = octeon_send_soft_command(oct, sc);
+	if (retval == IQ_SEND_FAILED) {
 		netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
+	} else {
+		/* Sleep on a wait queue till the cond flag indicates that the
+		 * response arrived or timed-out.
+		 */
+		if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR)
+			return;
+		oct->props[lio->ifidx].rx_on = start_stop;
+	}
+
+	octeon_free_soft_command(oct, sc);
 }
 
 /**
@@ -1307,21 +1612,24 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
 
 	dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
 
-	send_rx_ctrl_cmd(lio, 0);
-
 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
-		txqs_stop(netdev);
+		liquidio_stop(netdev);
 
 	if (oct->props[lio->ifidx].napi_enabled == 1) {
 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
 			napi_disable(napi);
 
 		oct->props[lio->ifidx].napi_enabled = 0;
+
+		if (OCTEON_CN23XX_PF(oct))
+			oct->droq[0]->ops.poll_mode = 0;
 	}
 
 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
 		unregister_netdev(netdev);
 
+	cleanup_link_status_change_wq(netdev);
+
 	delete_glists(lio);
 
 	free_netdev(netdev);
@@ -1374,6 +1682,9 @@ static void liquidio_remove(struct pci_dev *pdev)
 
 	dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
 
+	if (oct_dev->watchdog_task)
+		kthread_stop(oct_dev->watchdog_task);
+
 	if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
 		liquidio_stop_nic_module(oct_dev);
 
@@ -1417,6 +1728,12 @@ static int octeon_chip_specific_setup(struct octeon_device *oct)
 		s = "CN66XX";
 		break;
 
+	case OCTEON_CN23XX_PCIID_PF:
+		oct->chip_id = OCTEON_CN23XX_PF_VID;
+		ret = setup_cn23xx_octeon_pf_device(oct);
+		s = "CN23XX";
+		break;
+
 	default:
 		s = "?";
 		dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
@@ -1867,7 +2184,7 @@ static void if_cfg_callback(struct octeon_device *oct,
 	struct liquidio_if_cfg_context *ctx;
 
 	resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
-	ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
+	ctx = (struct liquidio_if_cfg_context *)sc->ctxptr;
 
 	oct = lio_get_device(ctx->octeon_id);
 	if (resp->status)
@@ -2060,11 +2377,14 @@ static void napi_schedule_wrapper(void *param)
  */
 static void liquidio_napi_drv_callback(void *arg)
 {
+	struct octeon_device *oct;
 	struct octeon_droq *droq = arg;
 	int this_cpu = smp_processor_id();
 
-	if (droq->cpu_id == this_cpu) {
-		napi_schedule(&droq->napi);
+	oct = droq->oct_dev;
+
+	if (OCTEON_CN23XX_PF(oct) || droq->cpu_id == this_cpu) {
+		napi_schedule_irqoff(&droq->napi);
 	} else {
 		struct call_single_data *csd = &droq->csd;
 
@@ -2173,17 +2493,15 @@ static inline int setup_io_queues(struct octeon_device *octeon_dev,
 						   lio->ifidx), NULL);
 		if (retval) {
 			dev_err(&octeon_dev->pci_dev->dev,
-				" %s : Runtime DROQ(RxQ) creation failed.\n",
+				"%s : Runtime DROQ(RxQ) creation failed.\n",
 				__func__);
 			return 1;
 		}
 
 		droq = octeon_dev->droq[q_no];
 		napi = &droq->napi;
-		dev_dbg(&octeon_dev->pci_dev->dev,
-			"netif_napi_add netdev:%llx oct:%llx\n",
-			(u64)netdev,
-			(u64)octeon_dev);
+		dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx pf_num:%d\n",
+			(u64)netdev, (u64)octeon_dev, octeon_dev->pf_num);
 		netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
 
 		/* designate a CPU for this droq */
@@ -2195,6 +2513,14 @@ static inline int setup_io_queues(struct octeon_device *octeon_dev,
 		octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
 	}
 
+	if (OCTEON_CN23XX_PF(octeon_dev)) {
+		/* 23XX PF can receive control messages (via the first PF-owned
+		 * droq) from the firmware even if the ethX interface is down,
+		 * so that's why poll_mode must be off for the first droq.
+		 */
+		octeon_dev->droq[0]->ops.poll_mode = 0;
+	}
+
 	/* set up IQs. */
 	for (q = 0; q < lio->linfo.num_txpciq; q++) {
 		num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
@@ -2235,7 +2561,7 @@ static void octnet_poll_check_txq_status(struct work_struct *work)
  * \brief Sets up the txq poll check
  * @param netdev network device
  */
-static inline void setup_tx_poll_fn(struct net_device *netdev)
+static inline int setup_tx_poll_fn(struct net_device *netdev)
 {
 	struct lio *lio = GET_LIO(netdev);
 	struct octeon_device *oct = lio->oct_dev;
@@ -2244,21 +2570,24 @@ static inline void setup_tx_poll_fn(struct net_device *netdev)
 						WQ_MEM_RECLAIM, 0);
 	if (!lio->txq_status_wq.wq) {
 		dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
-		return;
+		return -1;
 	}
 	INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
 			  octnet_poll_check_txq_status);
 	lio->txq_status_wq.wk.ctxptr = lio;
 	queue_delayed_work(lio->txq_status_wq.wq,
 			   &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
+	return 0;
 }
 
 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
 {
 	struct lio *lio = GET_LIO(netdev);
 
-	cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
-	destroy_workqueue(lio->txq_status_wq.wq);
+	if (lio->txq_status_wq.wq) {
+		cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
+		destroy_workqueue(lio->txq_status_wq.wq);
+	}
 }
 
 /**
@@ -2276,24 +2605,34 @@ static int liquidio_open(struct net_device *netdev)
 			napi_enable(napi);
 
 		oct->props[lio->ifidx].napi_enabled = 1;
+
+		if (OCTEON_CN23XX_PF(oct))
+			oct->droq[0]->ops.poll_mode = 1;
 	}
 
 	oct_ptp_open(netdev);
 
 	ifstate_set(lio, LIO_IFSTATE_RUNNING);
 
-	setup_tx_poll_fn(netdev);
-
-	start_txq(netdev);
+	/* Ready for link status updates */
+	lio->intf_open = 1;
 
 	netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
 
+	if (OCTEON_CN23XX_PF(oct)) {
+		if (!oct->msix_on)
+			if (setup_tx_poll_fn(netdev))
+				return -1;
+	} else {
+		if (setup_tx_poll_fn(netdev))
+			return -1;
+	}
+
+	start_txq(netdev);
+
 	/* tell Octeon to start forwarding packets to host */
 	send_rx_ctrl_cmd(lio, 1);
 
-	/* Ready for link status updates */
-	lio->intf_open = 1;
-
 	dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
 		 netdev->name);
 
@@ -2328,7 +2667,12 @@ static int liquidio_stop(struct net_device *netdev)
 	/* Now it should be safe to tell Octeon that nic interface is down. */
 	send_rx_ctrl_cmd(lio, 0);
 
-	cleanup_tx_poll_fn(netdev);
+	if (OCTEON_CN23XX_PF(oct)) {
+		if (!oct->msix_on)
+			cleanup_tx_poll_fn(netdev);
+	} else {
+		cleanup_tx_poll_fn(netdev);
+	}
 
 	if (lio->ptp_clock) {
 		ptp_clock_unregister(lio->ptp_clock);
@@ -2340,143 +2684,6 @@ static int liquidio_stop(struct net_device *netdev)
 	return 0;
 }
 
-void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr)
-{
-	struct octnic_ctrl_pkt *nctrl = (struct octnic_ctrl_pkt *)nctrl_ptr;
-	struct net_device *netdev = (struct net_device *)nctrl->netpndev;
-	struct lio *lio = GET_LIO(netdev);
-	struct octeon_device *oct = lio->oct_dev;
-	u8 *mac;
-
-	switch (nctrl->ncmd.s.cmd) {
-	case OCTNET_CMD_CHANGE_DEVFLAGS:
-	case OCTNET_CMD_SET_MULTI_LIST:
-		break;
-
-	case OCTNET_CMD_CHANGE_MACADDR:
-		mac = ((u8 *)&nctrl->udd[0]) + 2;
-		netif_info(lio, probe, lio->netdev,
-			   "%s %2.2x:%2.2x:%2.2x:%2.2x:%2.2x:%2.2x\n",
-			   "MACAddr changed to", mac[0], mac[1],
-			   mac[2], mac[3], mac[4], mac[5]);
-		break;
-
-	case OCTNET_CMD_CHANGE_MTU:
-		/* If command is successful, change the MTU. */
-		netif_info(lio, probe, lio->netdev, " MTU Changed from %d to %d\n",
-			   netdev->mtu, nctrl->ncmd.s.param1);
-		dev_info(&oct->pci_dev->dev, "%s MTU Changed from %d to %d\n",
-			 netdev->name, netdev->mtu,
-			 nctrl->ncmd.s.param1);
-		rtnl_lock();
-		netdev->mtu = nctrl->ncmd.s.param1;
-		call_netdevice_notifiers(NETDEV_CHANGEMTU, netdev);
-		rtnl_unlock();
-		break;
-
-	case OCTNET_CMD_GPIO_ACCESS:
-		netif_info(lio, probe, lio->netdev, "LED Flashing visual identification\n");
-
-		break;
-
-	case OCTNET_CMD_LRO_ENABLE:
-		dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name);
-		break;
-
-	case OCTNET_CMD_LRO_DISABLE:
-		dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n",
-			 netdev->name);
-		break;
-
-	case OCTNET_CMD_VERBOSE_ENABLE:
-		dev_info(&oct->pci_dev->dev, "%s LRO Enabled\n", netdev->name);
-		break;
-
-	case OCTNET_CMD_VERBOSE_DISABLE:
-		dev_info(&oct->pci_dev->dev, "%s LRO Disabled\n",
-			 netdev->name);
-		break;
-
-	case OCTNET_CMD_ENABLE_VLAN_FILTER:
-		dev_info(&oct->pci_dev->dev, "%s VLAN filter enabled\n",
-			 netdev->name);
-		break;
-
-	case OCTNET_CMD_ADD_VLAN_FILTER:
-		dev_info(&oct->pci_dev->dev, "%s VLAN filter %d added\n",
-			 netdev->name, nctrl->ncmd.s.param1);
-		break;
-
-	case OCTNET_CMD_DEL_VLAN_FILTER:
-		dev_info(&oct->pci_dev->dev, "%s VLAN filter %d removed\n",
-			 netdev->name, nctrl->ncmd.s.param1);
-		break;
-
-	case OCTNET_CMD_SET_SETTINGS:
-		dev_info(&oct->pci_dev->dev, "%s settings changed\n",
-			 netdev->name);
-
-		break;
-		/* Case to handle "OCTNET_CMD_TNL_RX_CSUM_CTL"
-		 * Command passed by NIC driver
-		 */
-	case OCTNET_CMD_TNL_RX_CSUM_CTL:
-		if (nctrl->ncmd.s.param1 == OCTNET_CMD_RXCSUM_ENABLE) {
-			netif_info(lio, probe, lio->netdev,
-				   "%s RX Checksum Offload Enabled\n",
-				   netdev->name);
-		} else if (nctrl->ncmd.s.param1 ==
-			   OCTNET_CMD_RXCSUM_DISABLE) {
-			netif_info(lio, probe, lio->netdev,
-				   "%s RX Checksum Offload Disabled\n",
-				   netdev->name);
-		}
-		break;
-
-		/* Case to handle "OCTNET_CMD_TNL_TX_CSUM_CTL"
-		 * Command passed by NIC driver
-		 */
-	case OCTNET_CMD_TNL_TX_CSUM_CTL:
-		if (nctrl->ncmd.s.param1 == OCTNET_CMD_TXCSUM_ENABLE) {
-			netif_info(lio, probe, lio->netdev,
-				   "%s TX Checksum Offload Enabled\n",
-				   netdev->name);
-		} else if (nctrl->ncmd.s.param1 ==
-			   OCTNET_CMD_TXCSUM_DISABLE) {
-			netif_info(lio, probe, lio->netdev,
-				   "%s TX Checksum Offload Disabled\n",
-				   netdev->name);
-		}
-		break;
-
-		/* Case to handle "OCTNET_CMD_VXLAN_PORT_CONFIG"
-		 * Command passed by NIC driver
-		 */
-	case OCTNET_CMD_VXLAN_PORT_CONFIG:
-		if (nctrl->ncmd.s.more == OCTNET_CMD_VXLAN_PORT_ADD) {
-			netif_info(lio, probe, lio->netdev,
-				   "%s VxLAN Destination UDP PORT:%d ADDED\n",
-				   netdev->name,
-				   nctrl->ncmd.s.param1);
-		} else if (nctrl->ncmd.s.more ==
-			   OCTNET_CMD_VXLAN_PORT_DEL) {
-			netif_info(lio, probe, lio->netdev,
-				   "%s VxLAN Destination UDP PORT:%d DELETED\n",
-				   netdev->name,
-				   nctrl->ncmd.s.param1);
-		}
-		break;
-
-	case OCTNET_CMD_SET_FLOW_CTL:
-		netif_info(lio, probe, lio->netdev, "Set RX/TX flow control parameters\n");
-		break;
-
-	default:
-		dev_err(&oct->pci_dev->dev, "%s Unknown cmd %d\n", __func__,
-			nctrl->ncmd.s.cmd);
-	}
-}
-
 /**
  * \brief Converts a mask based on net device flags
  * @param netdev network device
@@ -2817,8 +3024,7 @@ static void handle_timestamp(struct octeon_device *oct,
  */
 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
 					 struct octnic_data_pkt *ndata,
-					 struct octnet_buf_free_info *finfo,
-					 int xmit_more)
+					 struct octnet_buf_free_info *finfo)
 {
 	int retval;
 	struct octeon_soft_command *sc;
@@ -2846,9 +3052,15 @@ static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
 	sc->callback_arg = finfo->skb;
 	sc->iq_no = ndata->q_no;
 
-	len = (u32)((struct octeon_instr_ih2 *)(&sc->cmd.cmd2.ih2))->dlengsz;
+	if (OCTEON_CN23XX_PF(oct))
+		len = (u32)((struct octeon_instr_ih3 *)
+			    (&sc->cmd.cmd3.ih3))->dlengsz;
+	else
+		len = (u32)((struct octeon_instr_ih2 *)
+			    (&sc->cmd.cmd2.ih2))->dlengsz;
+
+	ring_doorbell = 1;
 
-	ring_doorbell = !xmit_more;
 	retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
 				     sc, len, ndata->reqtype);
 
@@ -2881,7 +3093,7 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 	union tx_info *tx_info;
 	int status = 0;
 	int q_idx = 0, iq_no = 0;
-	int xmit_more, j;
+	int j;
 	u64 dptr = 0;
 	u32 tag = 0;
 
@@ -2980,7 +3192,10 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 			return NETDEV_TX_BUSY;
 		}
 
-		ndata.cmd.cmd2.dptr = dptr;
+		if (OCTEON_CN23XX_PF(oct))
+			ndata.cmd.cmd3.dptr = dptr;
+		else
+			ndata.cmd.cmd2.dptr = dptr;
 		finfo->dptr = dptr;
 		ndata.reqtype = REQTYPE_NORESP_NET;
 
@@ -3055,15 +3270,23 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 					   g->sg_size, DMA_TO_DEVICE);
 		dptr = g->sg_dma_ptr;
 
-		ndata.cmd.cmd2.dptr = dptr;
+		if (OCTEON_CN23XX_PF(oct))
+			ndata.cmd.cmd3.dptr = dptr;
+		else
+			ndata.cmd.cmd2.dptr = dptr;
 		finfo->dptr = dptr;
 		finfo->g = g;
 
 		ndata.reqtype = REQTYPE_NORESP_NET_SG;
 	}
 
-	irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
-	tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
+	if (OCTEON_CN23XX_PF(oct)) {
+		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
+		tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
+	} else {
+		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
+		tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
+	}
 
 	if (skb_shinfo(skb)->gso_size) {
 		tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
@@ -3077,12 +3300,10 @@ static int liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
 		irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
 	}
 
-	xmit_more = skb->xmit_more;
-
 	if (unlikely(cmdsetup.s.timestamp))
-		status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
+		status = send_nic_timestamp_pkt(oct, &ndata, finfo);
 	else
-		status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
+		status = octnet_send_nic_data_pkt(oct, &ndata);
 	if (status == IQ_SEND_FAILED)
 		goto lio_xmit_failed;
 
@@ -3190,8 +3411,8 @@ static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
  *                              OCTNET_CMD_RXCSUM_DISABLE
  * @returns                     SUCCESS or FAILURE
  */
-int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
-				u8 rx_cmd)
+static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
+				       u8 rx_cmd)
 {
 	struct lio *lio = GET_LIO(netdev);
 	struct octeon_device *oct = lio->oct_dev;
@@ -3249,31 +3470,6 @@ static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
 	return ret;
 }
 
-int liquidio_set_feature(struct net_device *netdev, int cmd, u16 param1)
-{
-	struct lio *lio = GET_LIO(netdev);
-	struct octeon_device *oct = lio->oct_dev;
-	struct octnic_ctrl_pkt nctrl;
-	int ret = 0;
-
-	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
-
-	nctrl.ncmd.u64 = 0;
-	nctrl.ncmd.s.cmd = cmd;
-	nctrl.ncmd.s.param1 = param1;
-	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
-	nctrl.wait_time = 100;
-	nctrl.netpndev = (u64)netdev;
-	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
-
-	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
-	if (ret < 0) {
-		dev_err(&oct->pci_dev->dev, "Feature change failed in core (ret: 0x%x)\n",
-			ret);
-	}
-	return ret;
-}
-
 /** \brief Net device fix features
  * @param netdev  pointer to network device
  * @param request features requested
@@ -3492,8 +3688,9 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 	union oct_nic_if_cfg if_cfg;
 	unsigned int base_queue;
 	unsigned int gmx_port_id;
-	u32 resp_size, ctx_size;
+	u32 resp_size, ctx_size, data_size;
 	u32 ifidx_or_pfnum;
+	struct lio_version *vdata;
 
 	/* This is to handle link status changes */
 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
@@ -3515,21 +3712,37 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 	for (i = 0; i < octeon_dev->ifcount; i++) {
 		resp_size = sizeof(struct liquidio_if_cfg_resp);
 		ctx_size = sizeof(struct liquidio_if_cfg_context);
+		data_size = sizeof(struct lio_version);
 		sc = (struct octeon_soft_command *)
-			octeon_alloc_soft_command(octeon_dev, 0,
+			octeon_alloc_soft_command(octeon_dev, data_size,
 						  resp_size, ctx_size);
 		resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
 		ctx  = (struct liquidio_if_cfg_context *)sc->ctxptr;
+		vdata = (struct lio_version *)sc->virtdptr;
+
+		*((u64 *)vdata) = 0;
+		vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
+		vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
+		vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
 
-		num_iqueues =
-			CFG_GET_NUM_TXQS_NIC_IF(octeon_get_conf(octeon_dev), i);
-		num_oqueues =
-			CFG_GET_NUM_RXQS_NIC_IF(octeon_get_conf(octeon_dev), i);
-		base_queue =
-			CFG_GET_BASE_QUE_NIC_IF(octeon_get_conf(octeon_dev), i);
-		gmx_port_id =
-			CFG_GET_GMXID_NIC_IF(octeon_get_conf(octeon_dev), i);
-		ifidx_or_pfnum = i;
+		if (OCTEON_CN23XX_PF(octeon_dev)) {
+			num_iqueues = octeon_dev->sriov_info.num_pf_rings;
+			num_oqueues = octeon_dev->sriov_info.num_pf_rings;
+			base_queue = octeon_dev->sriov_info.pf_srn;
+
+			gmx_port_id = octeon_dev->pf_num;
+			ifidx_or_pfnum = octeon_dev->pf_num;
+		} else {
+			num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
+						octeon_get_conf(octeon_dev), i);
+			num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
+						octeon_get_conf(octeon_dev), i);
+			base_queue = CFG_GET_BASE_QUE_NIC_IF(
+						octeon_get_conf(octeon_dev), i);
+			gmx_port_id = CFG_GET_GMXID_NIC_IF(
+						octeon_get_conf(octeon_dev), i);
+			ifidx_or_pfnum = i;
+		}
 
 		dev_dbg(&octeon_dev->pci_dev->dev,
 			"requesting config for interface %d, iqs %d, oqs %d\n",
@@ -3566,7 +3779,11 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 		/* Sleep on a wait queue till the cond flag indicates that the
 		 * response arrived or timed-out.
 		 */
-		sleep_cond(&ctx->wc, &ctx->cond);
+		if (sleep_cond(&ctx->wc, &ctx->cond) == -EINTR) {
+			dev_err(&octeon_dev->pci_dev->dev, "Wait interrupted\n");
+			goto setup_nic_wait_intr;
+		}
+
 		retval = resp->status;
 		if (retval) {
 			dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
@@ -3633,12 +3850,16 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 
 		lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
 
-		lio->dev_capability = NETIF_F_HIGHDMA
-				| NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM
-				| NETIF_F_SG | NETIF_F_RXCSUM
-				| NETIF_F_GRO
-				| NETIF_F_TSO | NETIF_F_TSO6
-				| NETIF_F_LRO;
+		if (OCTEON_CN23XX_PF(octeon_dev) ||
+		    OCTEON_CN6XXX(octeon_dev)) {
+			lio->dev_capability = NETIF_F_HIGHDMA
+					      | NETIF_F_IP_CSUM
+					      | NETIF_F_IPV6_CSUM
+					      | NETIF_F_SG | NETIF_F_RXCSUM
+					      | NETIF_F_GRO
+					      | NETIF_F_TSO | NETIF_F_TSO6
+					      | NETIF_F_LRO;
+		}
 		netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
 
 		/*  Copy of transmit encapsulation capabilities:
@@ -3713,7 +3934,10 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 
 		/* Register ethtool support */
 		liquidio_set_ethtool_ops(netdev);
-		octeon_dev->priv_flags = 0x0;
+		if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
+			octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
+		else
+			octeon_dev->priv_flags = 0x0;
 
 		if (netdev->features & NETIF_F_LRO)
 			liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
@@ -3725,6 +3949,9 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 			liquidio_set_feature(netdev,
 					     OCTNET_CMD_VERBOSE_ENABLE, 0);
 
+		if (setup_link_status_change_wq(netdev))
+			goto setup_nic_dev_fail;
+
 		/* Register the network device with the OS */
 		if (register_netdev(netdev)) {
 			dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
@@ -3760,6 +3987,8 @@ setup_nic_dev_fail:
 
 	octeon_free_soft_command(octeon_dev, sc);
 
+setup_nic_wait_intr:
+
 	while (i--) {
 		dev_err(&octeon_dev->pci_dev->dev,
 			"NIC ifidx:%d Setup failed\n", i);
@@ -3789,8 +4018,7 @@ static int liquidio_init_nic_module(struct octeon_device *oct)
 	/* run port_config command for each port */
 	oct->ifcount = num_nic_ports;
 
-	memset(oct->props, 0,
-	       sizeof(struct octdev_props) * num_nic_ports);
+	memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
 
 	for (i = 0; i < MAX_OCTEON_LINKS; i++)
 		oct->props[i].gmxport = -1;
@@ -3806,7 +4034,7 @@ static int liquidio_init_nic_module(struct octeon_device *oct)
 	/* Initialize interrupt moderation params */
 	intrmod_cfg = &((struct octeon_device *)oct)->intrmod;
 	intrmod_cfg->rx_enable = 1;
-	intrmod_cfg->check_intrvl =   LIO_INTRMOD_CHECK_INTERVAL;
+	intrmod_cfg->check_intrvl = LIO_INTRMOD_CHECK_INTERVAL;
 	intrmod_cfg->maxpkt_ratethr = LIO_INTRMOD_MAXPKT_RATETHR;
 	intrmod_cfg->minpkt_ratethr = LIO_INTRMOD_MINPKT_RATETHR;
 	intrmod_cfg->rx_maxcnt_trigger = LIO_INTRMOD_RXMAXCNT_TRIGGER;
@@ -3818,6 +4046,7 @@ static int liquidio_init_nic_module(struct octeon_device *oct)
 	intrmod_cfg->tx_mincnt_trigger = LIO_INTRMOD_TXMINCNT_TRIGGER;
 	intrmod_cfg->rx_frames = CFG_GET_OQ_INTR_PKT(octeon_get_conf(oct));
 	intrmod_cfg->rx_usecs = CFG_GET_OQ_INTR_TIME(octeon_get_conf(oct));
+	intrmod_cfg->tx_frames = CFG_GET_IQ_INTR_PKT(octeon_get_conf(oct));
 	dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
 
 	return retval;
@@ -3880,6 +4109,7 @@ static void nic_starter(struct work_struct *work)
 static int octeon_device_init(struct octeon_device *octeon_dev)
 {
 	int j, ret;
+	int fw_loaded = 0;
 	char bootcmd[] = "\n";
 	struct octeon_device_priv *oct_priv =
 		(struct octeon_device_priv *)octeon_dev->priv;
@@ -3901,9 +4131,23 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 
 	octeon_dev->app_mode = CVM_DRV_INVALID_APP;
 
-	/* Do a soft reset of the Octeon device. */
-	if (octeon_dev->fn_list.soft_reset(octeon_dev))
+	if (OCTEON_CN23XX_PF(octeon_dev)) {
+		if (!cn23xx_fw_loaded(octeon_dev)) {
+			fw_loaded = 0;
+			/* Do a soft reset of the Octeon device. */
+			if (octeon_dev->fn_list.soft_reset(octeon_dev))
+				return 1;
+			/* things might have changed */
+			if (!cn23xx_fw_loaded(octeon_dev))
+				fw_loaded = 0;
+			else
+				fw_loaded = 1;
+		} else {
+			fw_loaded = 1;
+		}
+	} else if (octeon_dev->fn_list.soft_reset(octeon_dev)) {
 		return 1;
+	}
 
 	/* Initialize the dispatch mechanism used to push packets arriving on
 	 * Octeon Output queues.
@@ -3925,6 +4169,22 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 
 	octeon_set_io_queues_off(octeon_dev);
 
+	if (OCTEON_CN23XX_PF(octeon_dev)) {
+		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
+		if (ret) {
+			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
+			return ret;
+		}
+	}
+
+	/* Initialize soft command buffer pool
+	 */
+	if (octeon_setup_sc_buffer_pool(octeon_dev)) {
+		dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
+		return 1;
+	}
+	atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
+
 	/*  Setup the data structures that manage this Octeon's Input queues. */
 	if (octeon_setup_instr_queues(octeon_dev)) {
 		dev_err(&octeon_dev->pci_dev->dev,
@@ -3936,14 +4196,6 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 	}
 	atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
 
-	/* Initialize soft command buffer pool
-	 */
-	if (octeon_setup_sc_buffer_pool(octeon_dev)) {
-		dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
-		return 1;
-	}
-	atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
-
 	/* Initialize lists to manage the requests of different types that
 	 * arrive from user & kernel applications for this octeon device.
 	 */
@@ -3963,15 +4215,23 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 
 	atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
 
-	/* The input and output queue registers were setup earlier (the queues
-	 * were not enabled). Any additional registers that need to be
-	 * programmed should be done now.
-	 */
-	ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
-	if (ret) {
-		dev_err(&octeon_dev->pci_dev->dev,
-			"Failed to configure device registers\n");
-		return ret;
+	if (OCTEON_CN23XX_PF(octeon_dev)) {
+		if (octeon_allocate_ioq_vector(octeon_dev)) {
+			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
+			return 1;
+		}
+
+	} else {
+		/* The input and output queue registers were setup earlier (the
+		 * queues were not enabled). Any additional registers
+		 * that need to be programmed should be done now.
+		 */
+		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
+		if (ret) {
+			dev_err(&octeon_dev->pci_dev->dev,
+				"Failed to configure device registers\n");
+			return ret;
+		}
 	}
 
 	/* Initialize the tasklet that handles output queue packet processing.*/
@@ -3985,63 +4245,76 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 		return 1;
 
 	/* Enable Octeon device interrupts */
-	octeon_dev->fn_list.enable_interrupt(octeon_dev->chip);
+	octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
 
 	/* Enable the input and output queues for this Octeon device */
-	octeon_dev->fn_list.enable_io_queues(octeon_dev);
+	ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
+	if (ret) {
+		dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
+		return ret;
+	}
 
 	atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
 
-	dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
-
-	if (ddr_timeout == 0)
-		dev_info(&octeon_dev->pci_dev->dev, "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
+	if ((!OCTEON_CN23XX_PF(octeon_dev)) || !fw_loaded) {
+		dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
+		if (!ddr_timeout) {
+			dev_info(&octeon_dev->pci_dev->dev,
+				 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
+		}
 
-	schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
+		schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
 
-	/* Wait for the octeon to initialize DDR after the soft-reset. */
-	while (ddr_timeout == 0) {
-		set_current_state(TASK_INTERRUPTIBLE);
-		if (schedule_timeout(HZ / 10)) {
-			/* user probably pressed Control-C */
+		/* Wait for the octeon to initialize DDR after the soft-reset.*/
+		while (!ddr_timeout) {
+			set_current_state(TASK_INTERRUPTIBLE);
+			if (schedule_timeout(HZ / 10)) {
+				/* user probably pressed Control-C */
+				return 1;
+			}
+		}
+		ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
+		if (ret) {
+			dev_err(&octeon_dev->pci_dev->dev,
+				"DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
+				ret);
 			return 1;
 		}
-	}
-	ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
-	if (ret) {
-		dev_err(&octeon_dev->pci_dev->dev,
-			"DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
-			ret);
-		return 1;
-	}
 
-	if (octeon_wait_for_bootloader(octeon_dev, 1000) != 0) {
-		dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
-		return 1;
-	}
+		if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
+			dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
+			return 1;
+		}
 
-	/* Divert uboot to take commands from host instead. */
-	ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
+		/* Divert uboot to take commands from host instead. */
+		ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
 
-	dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
-	ret = octeon_init_consoles(octeon_dev);
-	if (ret) {
-		dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
-		return 1;
-	}
-	ret = octeon_add_console(octeon_dev, 0);
-	if (ret) {
-		dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
-		return 1;
-	}
+		dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
+		ret = octeon_init_consoles(octeon_dev);
+		if (ret) {
+			dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
+			return 1;
+		}
+		ret = octeon_add_console(octeon_dev, 0);
+		if (ret) {
+			dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
+			return 1;
+		}
 
-	atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
+		atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
 
-	dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
-	ret = load_firmware(octeon_dev);
-	if (ret) {
-		dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
-		return 1;
+		dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
+		ret = load_firmware(octeon_dev);
+		if (ret) {
+			dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
+			return 1;
+		}
+		/* set bit 1 of SLI_SCRATCH_1 to indicate that firmware is
+		 * loaded
+		 */
+		if (OCTEON_CN23XX_PF(octeon_dev))
+			octeon_write_csr64(octeon_dev, CN23XX_SLI_SCRATCH1,
+					   2ULL);
 	}
 
 	handshake[octeon_dev->octeon_id].init_ok = 1;
@@ -4057,7 +4330,6 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 		       octeon_dev->droq[j]->pkts_credit_reg);
 
 	/* Packets can start arriving on the output queues from this point. */
-
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
index 199a8b9c7dc5..0d990accb65e 100644
--- a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
+++ b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
@@ -30,10 +30,24 @@
 
 #include "octeon_config.h"
 
-#define LIQUIDIO_BASE_VERSION   "1.4"
-#define LIQUIDIO_MICRO_VERSION  ".1"
 #define LIQUIDIO_PACKAGE ""
-#define LIQUIDIO_VERSION  "1.4.1"
+#define LIQUIDIO_BASE_MAJOR_VERSION 1
+#define LIQUIDIO_BASE_MINOR_VERSION 4
+#define LIQUIDIO_BASE_MICRO_VERSION 1
+#define LIQUIDIO_BASE_VERSION   __stringify(LIQUIDIO_BASE_MAJOR_VERSION) "." \
+				__stringify(LIQUIDIO_BASE_MINOR_VERSION)
+#define LIQUIDIO_MICRO_VERSION  "." __stringify(LIQUIDIO_BASE_MICRO_VERSION)
+#define LIQUIDIO_VERSION        LIQUIDIO_PACKAGE \
+				__stringify(LIQUIDIO_BASE_MAJOR_VERSION) "." \
+				__stringify(LIQUIDIO_BASE_MINOR_VERSION) \
+				"." __stringify(LIQUIDIO_BASE_MICRO_VERSION)
+
+struct lio_version {
+	u16  major;
+	u16  minor;
+	u16  micro;
+	u16  reserved;
+};
 
 #define CONTROL_IQ 0
 /** Tag types used by Octeon cores in its work. */
@@ -218,6 +232,9 @@ static inline void add_sg_size(struct octeon_sg_entry *sg_entry,
 #define   OCTNET_CMD_ADD_VLAN_FILTER  0x17
 #define   OCTNET_CMD_DEL_VLAN_FILTER  0x18
 #define   OCTNET_CMD_VXLAN_PORT_CONFIG 0x19
+
+#define   OCTNET_CMD_ID_ACTIVE         0x1a
+
 #define   OCTNET_CMD_VXLAN_PORT_ADD    0x0
 #define   OCTNET_CMD_VXLAN_PORT_DEL    0x1
 #define   OCTNET_CMD_RXCSUM_ENABLE     0x0
@@ -296,6 +313,13 @@ union octnet_cmd {
 
 #define   OCTNET_CMD_SIZE     (sizeof(union octnet_cmd))
 
+/*pkiih3 + irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */
+#define LIO_SOFTCMDRESP_IH2       40
+#define LIO_SOFTCMDRESP_IH3       (40 + 8)
+
+#define LIO_PCICMD_O2             24
+#define LIO_PCICMD_O3             (24 + 8)
+
 /* Instruction Header(DPI) - for OCTEON-III models */
 struct  octeon_instr_ih3 {
 #ifdef __BIG_ENDIAN_BITFIELD
@@ -814,6 +838,8 @@ struct oct_link_stats {
 #define VITESSE_PHY_GPIO_DRIVEOFF 0x4
 #define VITESSE_PHY_GPIO_HIGH     0x2
 #define VITESSE_PHY_GPIO_LOW      0x3
+#define LED_IDENTIFICATION_ON     0x1
+#define LED_IDENTIFICATION_OFF    0x0
 
 struct oct_mdio_cmd {
 	u64 op;
@@ -832,7 +858,7 @@ struct oct_mdio_cmd {
 /* intrmod: max. packets to trigger interrupt */
 #define LIO_INTRMOD_RXMAXCNT_TRIGGER	384
 /* intrmod: min. packets to trigger interrupt */
-#define LIO_INTRMOD_RXMINCNT_TRIGGER	1
+#define LIO_INTRMOD_RXMINCNT_TRIGGER	0
 /* intrmod: max. time to trigger interrupt */
 #define LIO_INTRMOD_RXMAXTMR_TRIGGER	128
 /* 66xx:intrmod: min. time to trigger interrupt
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_config.h b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
index b3396e3a8bab..c76556809ed1 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_config.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
@@ -64,6 +64,34 @@
 #define   DEFAULT_NUM_NIC_PORTS_68XX   4
 #define   DEFAULT_NUM_NIC_PORTS_68XX_210NV  2
 
+/* CN23xx  IQ configuration macros */
+#define   CN23XX_MAX_RINGS_PER_PF_PASS_1_0 12
+#define   CN23XX_MAX_RINGS_PER_PF_PASS_1_1 32
+#define   CN23XX_MAX_RINGS_PER_PF          64
+
+#define   CN23XX_MAX_INPUT_QUEUES	CN23XX_MAX_RINGS_PER_PF
+#define   CN23XX_MAX_IQ_DESCRIPTORS	2048
+#define   CN23XX_DB_MIN                 1
+#define   CN23XX_DB_MAX                 8
+#define   CN23XX_DB_TIMEOUT             1
+
+#define   CN23XX_MAX_OUTPUT_QUEUES	CN23XX_MAX_RINGS_PER_PF
+#define   CN23XX_MAX_OQ_DESCRIPTORS	2048
+#define   CN23XX_OQ_BUF_SIZE		1536
+#define   CN23XX_OQ_PKTSPER_INTR	128
+/*#define CAVIUM_ONLY_CN23XX_RX_PERF*/
+#define   CN23XX_OQ_REFIL_THRESHOLD	128
+
+#define   CN23XX_OQ_INTR_PKT		64
+#define   CN23XX_OQ_INTR_TIME		100
+#define   DEFAULT_NUM_NIC_PORTS_23XX	1
+
+#define   CN23XX_CFG_IO_QUEUES		CN23XX_MAX_RINGS_PER_PF
+/* PEMs count */
+#define   CN23XX_MAX_MACS		4
+
+#define   CN23XX_DEF_IQ_INTR_THRESHOLD	32
+#define   CN23XX_DEF_IQ_INTR_BYTE_THRESHOLD   (64 * 1024)
 /* common OCTEON configuration macros */
 #define   CN6XXX_CFG_IO_QUEUES         32
 #define   OCTEON_32BYTE_INSTR          32
@@ -92,6 +120,9 @@
 #define CFG_GET_IQ_DB_MIN(cfg)                   ((cfg)->iq.db_min)
 #define CFG_GET_IQ_DB_TIMEOUT(cfg)               ((cfg)->iq.db_timeout)
 
+#define CFG_GET_IQ_INTR_PKT(cfg)                 ((cfg)->iq.iq_intr_pkt)
+#define CFG_SET_IQ_INTR_PKT(cfg, val)            (cfg)->iq.iq_intr_pkt = val
+
 #define CFG_GET_OQ_MAX_Q(cfg)                    ((cfg)->oq.max_oqs)
 #define CFG_GET_OQ_INFO_PTR(cfg)                 ((cfg)->oq.info_ptr)
 #define CFG_GET_OQ_PKTS_PER_INTR(cfg)            ((cfg)->oq.pkts_per_intr)
@@ -140,19 +171,24 @@
 enum lio_card_type {
 	LIO_210SV = 0, /* Two port, 66xx */
 	LIO_210NV,     /* Two port, 68xx */
-	LIO_410NV      /* Four port, 68xx */
+	LIO_410NV,     /* Four port, 68xx */
+	LIO_23XX       /* 23xx */
 };
 
 #define LIO_210SV_NAME "210sv"
 #define LIO_210NV_NAME "210nv"
 #define LIO_410NV_NAME "410nv"
+#define LIO_23XX_NAME  "23xx"
 
 /** Structure to define the configuration attributes for each Input queue.
  *  Applicable to all Octeon processors
  **/
 struct octeon_iq_config {
 #ifdef __BIG_ENDIAN_BITFIELD
-	u64 reserved:32;
+	u64 reserved:16;
+
+	/** Tx interrupt packets. Applicable to 23xx only */
+	u64 iq_intr_pkt:16;
 
 	/** Minimum ticks to wait before checking for pending instructions. */
 	u64 db_timeout:16;
@@ -192,7 +228,10 @@ struct octeon_iq_config {
 	/** Minimum ticks to wait before checking for pending instructions. */
 	u64 db_timeout:16;
 
-	u64 reserved:32;
+	/** Tx interrupt packets. Applicable to 23xx only */
+	u64 iq_intr_pkt:16;
+
+	u64 reserved:16;
 #endif
 };
 
@@ -416,11 +455,15 @@ struct octeon_config {
 #define DISPATCH_LIST_SIZE                      BIT(OPCODE_MASK_BITS)
 
 /* Maximum number of Octeon Instruction (command) queues */
-#define MAX_OCTEON_INSTR_QUEUES(oct)         CN6XXX_MAX_INPUT_QUEUES
-/* Maximum number of Octeon Output queues */
-#define MAX_OCTEON_OUTPUT_QUEUES(oct)         CN6XXX_MAX_OUTPUT_QUEUES
+#define MAX_OCTEON_INSTR_QUEUES(oct)		\
+		(OCTEON_CN23XX_PF(oct) ? CN23XX_MAX_INPUT_QUEUES : \
+					CN6XXX_MAX_INPUT_QUEUES)
 
-#define MAX_POSSIBLE_OCTEON_INSTR_QUEUES       CN6XXX_MAX_INPUT_QUEUES
-#define MAX_POSSIBLE_OCTEON_OUTPUT_QUEUES      CN6XXX_MAX_OUTPUT_QUEUES
+/* Maximum number of Octeon Instruction (command) queues */
+#define MAX_OCTEON_OUTPUT_QUEUES(oct)		\
+		(OCTEON_CN23XX_PF(oct) ? CN23XX_MAX_OUTPUT_QUEUES : \
+					CN6XXX_MAX_OUTPUT_QUEUES)
 
+#define MAX_POSSIBLE_OCTEON_INSTR_QUEUES	CN23XX_MAX_INPUT_QUEUES
+#define MAX_POSSIBLE_OCTEON_OUTPUT_QUEUES	CN23XX_MAX_OUTPUT_QUEUES
 #endif /* __OCTEON_CONFIG_H__  */
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_console.c b/drivers/net/ethernet/cavium/liquidio/octeon_console.c
index bbb50ea66f16..01a50f3b0c8e 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_console.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_console.c
@@ -25,12 +25,13 @@
  */
 #include <linux/pci.h>
 #include <linux/netdevice.h>
+#include <linux/crc32.h>
 #include "liquidio_common.h"
 #include "octeon_droq.h"
 #include "octeon_iq.h"
 #include "response_manager.h"
 #include "octeon_device.h"
-#include "octeon_main.h"
+#include "liquidio_image.h"
 #include "octeon_mem_ops.h"
 
 static void octeon_remote_lock(void);
@@ -40,6 +41,10 @@ static u64 cvmx_bootmem_phy_named_block_find(struct octeon_device *oct,
 					     u32 flags);
 static int octeon_console_read(struct octeon_device *oct, u32 console_num,
 			       char *buffer, u32 buf_size);
+static u32 console_bitmask;
+module_param(console_bitmask, int, 0644);
+MODULE_PARM_DESC(console_bitmask,
+		 "Bitmask indicating which consoles have debug output redirected to syslog.");
 
 #define MIN(a, b) min((a), (b))
 #define CAST_ULL(v) ((u64)(v))
@@ -177,6 +182,15 @@ struct octeon_pci_console_desc {
 	__cvmx_bootmem_desc_get(oct, addr,                               \
 		offsetof(struct cvmx_bootmem_named_block_desc, field),   \
 		SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, field))
+/**
+ * \brief determines if a given console has debug enabled.
+ * @param console console to check
+ * @returns  1 = enabled. 0 otherwise
+ */
+static int octeon_console_debug_enabled(u32 console)
+{
+	return (console_bitmask >> (console)) & 0x1;
+}
 
 /**
  * This function is the implementation of the get macros defined
@@ -709,3 +723,104 @@ static int octeon_console_read(struct octeon_device *oct, u32 console_num,
 
 	return bytes_to_read;
 }
+
+#define FBUF_SIZE	(4 * 1024 * 1024)
+u8 fbuf[FBUF_SIZE];
+
+int octeon_download_firmware(struct octeon_device *oct, const u8 *data,
+			     size_t size)
+{
+	int ret = 0;
+	u8 *p = fbuf;
+	u32 crc32_result;
+	u64 load_addr;
+	u32 image_len;
+	struct octeon_firmware_file_header *h;
+	u32 i, rem;
+
+	if (size < sizeof(struct octeon_firmware_file_header)) {
+		dev_err(&oct->pci_dev->dev, "Firmware file too small (%d < %d).\n",
+			(u32)size,
+			(u32)sizeof(struct octeon_firmware_file_header));
+		return -EINVAL;
+	}
+
+	h = (struct octeon_firmware_file_header *)data;
+
+	if (be32_to_cpu(h->magic) != LIO_NIC_MAGIC) {
+		dev_err(&oct->pci_dev->dev, "Unrecognized firmware file.\n");
+		return -EINVAL;
+	}
+
+	crc32_result = crc32((unsigned int)~0, data,
+			     sizeof(struct octeon_firmware_file_header) -
+			     sizeof(u32)) ^ ~0U;
+	if (crc32_result != be32_to_cpu(h->crc32)) {
+		dev_err(&oct->pci_dev->dev, "Firmware CRC mismatch (0x%08x != 0x%08x).\n",
+			crc32_result, be32_to_cpu(h->crc32));
+		return -EINVAL;
+	}
+
+	if (strncmp(LIQUIDIO_PACKAGE, h->version, strlen(LIQUIDIO_PACKAGE))) {
+		dev_err(&oct->pci_dev->dev, "Unmatched firmware package type. Expected %s, got %s.\n",
+			LIQUIDIO_PACKAGE, h->version);
+		return -EINVAL;
+	}
+
+	if (memcmp(LIQUIDIO_BASE_VERSION, h->version + strlen(LIQUIDIO_PACKAGE),
+		   strlen(LIQUIDIO_BASE_VERSION))) {
+		dev_err(&oct->pci_dev->dev, "Unmatched firmware version. Expected %s.x, got %s.\n",
+			LIQUIDIO_BASE_VERSION,
+			h->version + strlen(LIQUIDIO_PACKAGE));
+		return -EINVAL;
+	}
+
+	if (be32_to_cpu(h->num_images) > LIO_MAX_IMAGES) {
+		dev_err(&oct->pci_dev->dev, "Too many images in firmware file (%d).\n",
+			be32_to_cpu(h->num_images));
+		return -EINVAL;
+	}
+
+	dev_info(&oct->pci_dev->dev, "Firmware version: %s\n", h->version);
+	snprintf(oct->fw_info.liquidio_firmware_version, 32, "LIQUIDIO: %s",
+		 h->version);
+
+	data += sizeof(struct octeon_firmware_file_header);
+
+	dev_info(&oct->pci_dev->dev, "%s: Loading %d images\n", __func__,
+		 be32_to_cpu(h->num_images));
+	/* load all images */
+	for (i = 0; i < be32_to_cpu(h->num_images); i++) {
+		load_addr = be64_to_cpu(h->desc[i].addr);
+		image_len = be32_to_cpu(h->desc[i].len);
+
+		dev_info(&oct->pci_dev->dev, "Loading firmware %d at %llx\n",
+			 image_len, load_addr);
+
+		/* Write in 4MB chunks*/
+		rem = image_len;
+
+		while (rem) {
+			if (rem < FBUF_SIZE)
+				size = rem;
+			else
+				size = FBUF_SIZE;
+
+			memcpy(p, data, size);
+
+			/* download the image */
+			octeon_pci_write_core_mem(oct, load_addr, p, (u32)size);
+
+			data += size;
+			rem -= (u32)size;
+			load_addr += size;
+		}
+	}
+	dev_info(&oct->pci_dev->dev, "Writing boot command: %s\n",
+		 h->bootcmd);
+
+	/* Invoke the bootcmd */
+	ret = octeon_console_send_cmd(oct, h->bootcmd, 50);
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.c b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
index 0eb504a4379a..586b68899b06 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
@@ -20,7 +20,6 @@
 * Contact Cavium, Inc. for more information
 **********************************************************************/
 #include <linux/pci.h>
-#include <linux/crc32.h>
 #include <linux/netdevice.h>
 #include <linux/vmalloc.h>
 #include "liquidio_common.h"
@@ -32,8 +31,7 @@
 #include "octeon_network.h"
 #include "cn66xx_regs.h"
 #include "cn66xx_device.h"
-#include "liquidio_image.h"
-#include "octeon_mem_ops.h"
+#include "cn23xx_pf_device.h"
 
 /** Default configuration
  *  for CN66XX OCTEON Models.
@@ -420,6 +418,108 @@ static struct octeon_config default_cn68xx_210nv_conf = {
 	,
 };
 
+static struct octeon_config default_cn23xx_conf = {
+	.card_type                              = LIO_23XX,
+	.card_name                              = LIO_23XX_NAME,
+	/** IQ attributes */
+	.iq = {
+		.max_iqs		= CN23XX_CFG_IO_QUEUES,
+		.pending_list_size	= (CN23XX_MAX_IQ_DESCRIPTORS *
+					   CN23XX_CFG_IO_QUEUES),
+		.instr_type		= OCTEON_64BYTE_INSTR,
+		.db_min			= CN23XX_DB_MIN,
+		.db_timeout		= CN23XX_DB_TIMEOUT,
+		.iq_intr_pkt		= CN23XX_DEF_IQ_INTR_THRESHOLD,
+	},
+
+	/** OQ attributes */
+	.oq = {
+		.max_oqs		= CN23XX_CFG_IO_QUEUES,
+		.info_ptr		= OCTEON_OQ_INFOPTR_MODE,
+		.pkts_per_intr	= CN23XX_OQ_PKTSPER_INTR,
+		.refill_threshold	= CN23XX_OQ_REFIL_THRESHOLD,
+		.oq_intr_pkt	= CN23XX_OQ_INTR_PKT,
+		.oq_intr_time	= CN23XX_OQ_INTR_TIME,
+	},
+
+	.num_nic_ports				= DEFAULT_NUM_NIC_PORTS_23XX,
+	.num_def_rx_descs			= CN23XX_MAX_OQ_DESCRIPTORS,
+	.num_def_tx_descs			= CN23XX_MAX_IQ_DESCRIPTORS,
+	.def_rx_buf_size			= CN23XX_OQ_BUF_SIZE,
+
+	/* For ethernet interface 0:  Port cfg Attributes */
+	.nic_if_cfg[0] = {
+		/* Max Txqs: Half for each of the two ports :max_iq/2 */
+		.max_txqs			= MAX_TXQS_PER_INTF,
+
+		/* Actual configured value. Range could be: 1...max_txqs */
+		.num_txqs			= DEF_TXQS_PER_INTF,
+
+		/* Max Rxqs: Half for each of the two ports :max_oq/2  */
+		.max_rxqs			= MAX_RXQS_PER_INTF,
+
+		/* Actual configured value. Range could be: 1...max_rxqs */
+		.num_rxqs			= DEF_RXQS_PER_INTF,
+
+		/* Num of desc for rx rings */
+		.num_rx_descs			= CN23XX_MAX_OQ_DESCRIPTORS,
+
+		/* Num of desc for tx rings */
+		.num_tx_descs			= CN23XX_MAX_IQ_DESCRIPTORS,
+
+		/* SKB size, We need not change buf size even for Jumbo frames.
+		 * Octeon can send jumbo frames in 4 consecutive descriptors,
+		 */
+		.rx_buf_size			= CN23XX_OQ_BUF_SIZE,
+
+		.base_queue			= BASE_QUEUE_NOT_REQUESTED,
+
+		.gmx_port_id			= 0,
+	},
+
+	.nic_if_cfg[1] = {
+		/* Max Txqs: Half for each of the two ports :max_iq/2 */
+		.max_txqs			= MAX_TXQS_PER_INTF,
+
+		/* Actual configured value. Range could be: 1...max_txqs */
+		.num_txqs			= DEF_TXQS_PER_INTF,
+
+		/* Max Rxqs: Half for each of the two ports :max_oq/2  */
+		.max_rxqs			= MAX_RXQS_PER_INTF,
+
+		/* Actual configured value. Range could be: 1...max_rxqs */
+		.num_rxqs			= DEF_RXQS_PER_INTF,
+
+		/* Num of desc for rx rings */
+		.num_rx_descs			= CN23XX_MAX_OQ_DESCRIPTORS,
+
+		/* Num of desc for tx rings */
+		.num_tx_descs			= CN23XX_MAX_IQ_DESCRIPTORS,
+
+		/* SKB size, We need not change buf size even for Jumbo frames.
+		 * Octeon can send jumbo frames in 4 consecutive descriptors,
+		 */
+		.rx_buf_size			= CN23XX_OQ_BUF_SIZE,
+
+		.base_queue			= BASE_QUEUE_NOT_REQUESTED,
+
+		.gmx_port_id			= 1,
+	},
+
+	.misc					= {
+		/* Host driver link query interval */
+		.oct_link_query_interval	= 100,
+
+		/* Octeon link query interval */
+		.host_link_query_interval	= 500,
+
+		.enable_sli_oq_bp		= 0,
+
+		/* Control queue group */
+		.ctrlq_grp			= 1,
+	}
+};
+
 enum {
 	OCTEON_CONFIG_TYPE_DEFAULT = 0,
 	NUM_OCTEON_CONFS,
@@ -487,6 +587,8 @@ static void *__retrieve_octeon_config_info(struct octeon_device *oct,
 		} else if ((oct->chip_id == OCTEON_CN68XX) &&
 			   (card_type == LIO_410NV)) {
 			ret =  (void *)&default_cn68xx_conf;
+		} else if (oct->chip_id == OCTEON_CN23XX_PF_VID) {
+			ret =  (void *)&default_cn23xx_conf;
 		}
 		break;
 	default:
@@ -501,7 +603,8 @@ static int __verify_octeon_config_info(struct octeon_device *oct, void *conf)
 	case OCTEON_CN66XX:
 	case OCTEON_CN68XX:
 		return lio_validate_cn6xxx_config_info(oct, conf);
-
+	case OCTEON_CN23XX_PF_VID:
+		return 0;
 	default:
 		break;
 	}
@@ -541,107 +644,6 @@ static char *get_oct_app_string(u32 app_mode)
 	return oct_dev_app_str[CVM_DRV_INVALID_APP - CVM_DRV_APP_START];
 }
 
-u8 fbuf[4 * 1024 * 1024];
-
-int octeon_download_firmware(struct octeon_device *oct, const u8 *data,
-			     size_t size)
-{
-	int ret = 0;
-	u8 *p = fbuf;
-	u32 crc32_result;
-	u64 load_addr;
-	u32 image_len;
-	struct octeon_firmware_file_header *h;
-	u32 i, rem, base_len = strlen(LIQUIDIO_BASE_VERSION);
-	char *base;
-
-	if (size < sizeof(struct octeon_firmware_file_header)) {
-		dev_err(&oct->pci_dev->dev, "Firmware file too small (%d < %d).\n",
-			(u32)size,
-			(u32)sizeof(struct octeon_firmware_file_header));
-		return -EINVAL;
-	}
-
-	h = (struct octeon_firmware_file_header *)data;
-
-	if (be32_to_cpu(h->magic) != LIO_NIC_MAGIC) {
-		dev_err(&oct->pci_dev->dev, "Unrecognized firmware file.\n");
-		return -EINVAL;
-	}
-
-	crc32_result = crc32((unsigned int)~0, data,
-			     sizeof(struct octeon_firmware_file_header) -
-			     sizeof(u32)) ^ ~0U;
-	if (crc32_result != be32_to_cpu(h->crc32)) {
-		dev_err(&oct->pci_dev->dev, "Firmware CRC mismatch (0x%08x != 0x%08x).\n",
-			crc32_result, be32_to_cpu(h->crc32));
-		return -EINVAL;
-	}
-
-	if (strncmp(LIQUIDIO_PACKAGE, h->version, strlen(LIQUIDIO_PACKAGE))) {
-		dev_err(&oct->pci_dev->dev, "Unmatched firmware package type. Expected %s, got %s.\n",
-			LIQUIDIO_PACKAGE, h->version);
-		return -EINVAL;
-	}
-
-	base = h->version + strlen(LIQUIDIO_PACKAGE);
-	ret = memcmp(LIQUIDIO_BASE_VERSION, base, base_len);
-	if (ret) {
-		dev_err(&oct->pci_dev->dev, "Unmatched firmware version. Expected %s.x, got %s.\n",
-			LIQUIDIO_BASE_VERSION, base);
-		return -EINVAL;
-	}
-
-	if (be32_to_cpu(h->num_images) > LIO_MAX_IMAGES) {
-		dev_err(&oct->pci_dev->dev, "Too many images in firmware file (%d).\n",
-			be32_to_cpu(h->num_images));
-		return -EINVAL;
-	}
-
-	dev_info(&oct->pci_dev->dev, "Firmware version: %s\n", h->version);
-	snprintf(oct->fw_info.liquidio_firmware_version, 32, "LIQUIDIO: %s",
-		 h->version);
-
-	data += sizeof(struct octeon_firmware_file_header);
-
-	dev_info(&oct->pci_dev->dev, "%s: Loading %d images\n", __func__,
-		 be32_to_cpu(h->num_images));
-	/* load all images */
-	for (i = 0; i < be32_to_cpu(h->num_images); i++) {
-		load_addr = be64_to_cpu(h->desc[i].addr);
-		image_len = be32_to_cpu(h->desc[i].len);
-
-		dev_info(&oct->pci_dev->dev, "Loading firmware %d at %llx\n",
-			 image_len, load_addr);
-
-		/* Write in 4MB chunks*/
-		rem = image_len;
-
-		while (rem) {
-			if (rem < (4 * 1024 * 1024))
-				size = rem;
-			else
-				size = 4 * 1024 * 1024;
-
-			memcpy(p, data, size);
-
-			/* download the image */
-			octeon_pci_write_core_mem(oct, load_addr, p, (u32)size);
-
-			data += size;
-			rem -= (u32)size;
-			load_addr += size;
-		}
-	}
-	dev_info(&oct->pci_dev->dev, "Writing boot command: %s\n",
-		 h->bootcmd);
-
-	/* Invoke the bootcmd */
-	ret = octeon_console_send_cmd(oct, h->bootcmd, 50);
-
-	return 0;
-}
-
 void octeon_free_device_mem(struct octeon_device *oct)
 {
 	int i;
@@ -676,6 +678,9 @@ static struct octeon_device *octeon_allocate_device_mem(u32 pci_id,
 		configsize = sizeof(struct octeon_cn6xxx);
 		break;
 
+	case OCTEON_CN23XX_PF_VID:
+		configsize = sizeof(struct octeon_cn23xx_pf);
+		break;
 	default:
 		pr_err("%s: Unknown PCI Device: 0x%x\n",
 		       __func__,
@@ -741,6 +746,45 @@ struct octeon_device *octeon_allocate_device(u32 pci_id,
 	return oct;
 }
 
+int
+octeon_allocate_ioq_vector(struct octeon_device  *oct)
+{
+	int i, num_ioqs = 0;
+	struct octeon_ioq_vector *ioq_vector;
+	int cpu_num;
+	int size;
+
+	if (OCTEON_CN23XX_PF(oct))
+		num_ioqs = oct->sriov_info.num_pf_rings;
+	size = sizeof(struct octeon_ioq_vector) * num_ioqs;
+
+	oct->ioq_vector = vmalloc(size);
+	if (!oct->ioq_vector)
+		return 1;
+	memset(oct->ioq_vector, 0, size);
+	for (i = 0; i < num_ioqs; i++) {
+		ioq_vector		= &oct->ioq_vector[i];
+		ioq_vector->oct_dev	= oct;
+		ioq_vector->iq_index	= i;
+		ioq_vector->droq_index	= i;
+
+		cpu_num = i % num_online_cpus();
+		cpumask_set_cpu(cpu_num, &ioq_vector->affinity_mask);
+
+		if (oct->chip_id == OCTEON_CN23XX_PF_VID)
+			ioq_vector->ioq_num	= i + oct->sriov_info.pf_srn;
+		else
+			ioq_vector->ioq_num	= i;
+	}
+	return 0;
+}
+
+void
+octeon_free_ioq_vector(struct octeon_device *oct)
+{
+	vfree(oct->ioq_vector);
+}
+
 /* this function is only for setting up the first queue */
 int octeon_setup_instr_queues(struct octeon_device *oct)
 {
@@ -749,10 +793,12 @@ int octeon_setup_instr_queues(struct octeon_device *oct)
 	union oct_txpciq txpciq;
 	int numa_node = cpu_to_node(iq_no % num_online_cpus());
 
-	/* this causes queue 0 to be default queue */
 	if (OCTEON_CN6XXX(oct))
 		num_descs =
 			CFG_GET_NUM_DEF_TX_DESCS(CHIP_FIELD(oct, cn6xxx, conf));
+	else if (OCTEON_CN23XX_PF(oct))
+		num_descs = CFG_GET_NUM_DEF_TX_DESCS(CHIP_FIELD(oct, cn23xx_pf,
+								conf));
 
 	oct->num_iqs = 0;
 
@@ -769,6 +815,7 @@ int octeon_setup_instr_queues(struct octeon_device *oct)
 	oct->instr_queue[0]->ifidx = 0;
 	txpciq.u64 = 0;
 	txpciq.s.q_no = iq_no;
+	txpciq.s.pkind = oct->pfvf_hsword.pkind;
 	txpciq.s.use_qpg = 0;
 	txpciq.s.qpg = 0;
 	if (octeon_init_instr_queue(oct, txpciq, num_descs)) {
@@ -788,14 +835,17 @@ int octeon_setup_output_queues(struct octeon_device *oct)
 	u32 oq_no = 0;
 	int numa_node = cpu_to_node(oq_no % num_online_cpus());
 
-	/* this causes queue 0 to be default queue */
 	if (OCTEON_CN6XXX(oct)) {
 		num_descs =
 			CFG_GET_NUM_DEF_RX_DESCS(CHIP_FIELD(oct, cn6xxx, conf));
 		desc_size =
 			CFG_GET_DEF_RX_BUF_SIZE(CHIP_FIELD(oct, cn6xxx, conf));
+	} else if (OCTEON_CN23XX_PF(oct)) {
+		num_descs = CFG_GET_NUM_DEF_RX_DESCS(CHIP_FIELD(oct, cn23xx_pf,
+								conf));
+		desc_size = CFG_GET_DEF_RX_BUF_SIZE(CHIP_FIELD(oct, cn23xx_pf,
+							       conf));
 	}
-
 	oct->num_oqs = 0;
 	oct->droq[0] = vmalloc_node(sizeof(*oct->droq[0]), numa_node);
 	if (!oct->droq[0])
@@ -812,10 +862,10 @@ int octeon_setup_output_queues(struct octeon_device *oct)
 
 void octeon_set_io_queues_off(struct octeon_device *oct)
 {
-	/* Disable the i/p and o/p queues for this Octeon. */
-
-	octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
-	octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
+	if (OCTEON_CN6XXX(oct)) {
+		octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
+		octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
+	}
 }
 
 void octeon_set_droq_pkt_op(struct octeon_device *oct,
@@ -825,14 +875,16 @@ void octeon_set_droq_pkt_op(struct octeon_device *oct,
 	u32 reg_val = 0;
 
 	/* Disable the i/p and o/p queues for this Octeon. */
-	reg_val = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB);
+	if (OCTEON_CN6XXX(oct)) {
+		reg_val = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB);
 
-	if (enable)
-		reg_val = reg_val | (1 << q_no);
-	else
-		reg_val = reg_val & (~(1 << q_no));
+		if (enable)
+			reg_val = reg_val | (1 << q_no);
+		else
+			reg_val = reg_val & (~(1 << q_no));
 
-	octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, reg_val);
+		octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, reg_val);
+	}
 }
 
 int octeon_init_dispatch_list(struct octeon_device *oct)
@@ -1019,6 +1071,9 @@ int octeon_core_drv_init(struct octeon_recv_info *recv_info, void *buf)
 	if (OCTEON_CN6XXX(oct))
 		num_nic_ports =
 			CFG_GET_NUM_NIC_PORTS(CHIP_FIELD(oct, cn6xxx, conf));
+	else if (OCTEON_CN23XX_PF(oct))
+		num_nic_ports =
+			CFG_GET_NUM_NIC_PORTS(CHIP_FIELD(oct, cn23xx_pf, conf));
 
 	if (atomic_read(&oct->status) >= OCT_DEV_RUNNING) {
 		dev_err(&oct->pci_dev->dev, "Received CORE OK when device state is 0x%x\n",
@@ -1046,6 +1101,12 @@ int octeon_core_drv_init(struct octeon_recv_info *recv_info, void *buf)
 	}
 	oct->fw_info.app_cap_flags = recv_pkt->rh.r_core_drv_init.app_cap_flags;
 	oct->fw_info.app_mode = (u32)recv_pkt->rh.r_core_drv_init.app_mode;
+	oct->pfvf_hsword.app_mode = (u32)recv_pkt->rh.r_core_drv_init.app_mode;
+
+	oct->pfvf_hsword.pkind = recv_pkt->rh.r_core_drv_init.pkind;
+
+	for (i = 0; i < oct->num_iqs; i++)
+		oct->instr_queue[i]->txpciq.s.pkind = oct->pfvf_hsword.pkind;
 
 	atomic_set(&oct->status, OCT_DEV_CORE_OK);
 
@@ -1108,8 +1169,10 @@ struct octeon_config *octeon_get_conf(struct octeon_device *oct)
 	if (OCTEON_CN6XXX(oct)) {
 		default_oct_conf =
 			(struct octeon_config *)(CHIP_FIELD(oct, cn6xxx, conf));
+	} else if (OCTEON_CN23XX_PF(oct)) {
+		default_oct_conf = (struct octeon_config *)
+			(CHIP_FIELD(oct, cn23xx_pf, conf));
 	}
-
 	return default_oct_conf;
 }
 
@@ -1141,7 +1204,9 @@ u64 lio_pci_readq(struct octeon_device *oct, u64 addr)
 	 * So write MSB first
 	 */
 	addrhi = (addr >> 32);
-	if ((oct->chip_id == OCTEON_CN66XX) || (oct->chip_id == OCTEON_CN68XX))
+	if ((oct->chip_id == OCTEON_CN66XX) ||
+	    (oct->chip_id == OCTEON_CN68XX) ||
+	    (oct->chip_id == OCTEON_CN23XX_PF_VID))
 		addrhi |= 0x00060000;
 	writel(addrhi, oct->reg_list.pci_win_rd_addr_hi);
 
@@ -1185,8 +1250,15 @@ int octeon_mem_access_ok(struct octeon_device *oct)
 	u64 lmc0_reset_ctl;
 
 	/* Check to make sure a DDR interface is enabled */
-	lmc0_reset_ctl = lio_pci_readq(oct, CN6XXX_LMC0_RESET_CTL);
-	access_okay = (lmc0_reset_ctl & CN6XXX_LMC0_RESET_CTL_DDR3RST_MASK);
+	if (OCTEON_CN23XX_PF(oct)) {
+		lmc0_reset_ctl = lio_pci_readq(oct, CN23XX_LMC0_RESET_CTL);
+		access_okay =
+			(lmc0_reset_ctl & CN23XX_LMC0_RESET_CTL_DDR3RST_MASK);
+	} else {
+		lmc0_reset_ctl = lio_pci_readq(oct, CN6XXX_LMC0_RESET_CTL);
+		access_okay =
+			(lmc0_reset_ctl & CN6XXX_LMC0_RESET_CTL_DDR3RST_MASK);
+	}
 
 	return access_okay ? 0 : 1;
 }
@@ -1226,3 +1298,39 @@ int lio_get_device_id(void *dev)
 			return octeon_dev->octeon_id;
 	return -1;
 }
+
+void lio_enable_irq(struct octeon_droq *droq, struct octeon_instr_queue *iq)
+{
+	u64 instr_cnt;
+	struct octeon_device *oct = NULL;
+
+	/* the whole thing needs to be atomic, ideally */
+	if (droq) {
+		spin_lock_bh(&droq->lock);
+		writel(droq->pkt_count, droq->pkts_sent_reg);
+		droq->pkt_count = 0;
+		spin_unlock_bh(&droq->lock);
+		oct = droq->oct_dev;
+	}
+	if (iq) {
+		spin_lock_bh(&iq->lock);
+		writel(iq->pkt_in_done, iq->inst_cnt_reg);
+		iq->pkt_in_done = 0;
+		spin_unlock_bh(&iq->lock);
+		oct = iq->oct_dev;
+	}
+	/*write resend. Writing RESEND in SLI_PKTX_CNTS should be enough
+	 *to trigger tx interrupts as well, if they are pending.
+	 */
+	if (oct && OCTEON_CN23XX_PF(oct)) {
+		if (droq)
+			writeq(CN23XX_INTR_RESEND, droq->pkts_sent_reg);
+		/*we race with firmrware here. read and write the IN_DONE_CNTS*/
+		else if (iq) {
+			instr_cnt =  readq(iq->inst_cnt_reg);
+			writeq(((instr_cnt & 0xFFFFFFFF00000000ULL) |
+				CN23XX_INTR_RESEND),
+			       iq->inst_cnt_reg);
+		}
+	}
+}
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.h b/drivers/net/ethernet/cavium/liquidio/octeon_device.h
index 01edfb404346..da15c2ae9330 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_device.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.h
@@ -30,13 +30,19 @@
 /** PCI VendorId Device Id */
 #define  OCTEON_CN68XX_PCIID          0x91177d
 #define  OCTEON_CN66XX_PCIID          0x92177d
-
+#define  OCTEON_CN23XX_PCIID_PF       0x9702177d
 /** Driver identifies chips by these Ids, created by clubbing together
  *  DeviceId+RevisionId; Where Revision Id is not used to distinguish
  *  between chips, a value of 0 is used for revision id.
  */
 #define  OCTEON_CN68XX                0x0091
 #define  OCTEON_CN66XX                0x0092
+#define  OCTEON_CN23XX_PF_VID         0x9702
+
+/**RevisionId for the chips */
+#define  OCTEON_CN23XX_REV_1_0        0x00
+#define  OCTEON_CN23XX_REV_1_1        0x01
+#define  OCTEON_CN23XX_REV_2_0        0x80
 
 /** Endian-swap modes supported by Octeon. */
 enum octeon_pci_swap_mode {
@@ -46,6 +52,9 @@ enum octeon_pci_swap_mode {
 	OCTEON_PCI_32BIT_LW_SWAP = 3
 };
 
+#define  OCTEON_OUTPUT_INTR   (2)
+#define  OCTEON_ALL_INTR      0xff
+
 /*---------------   PCI BAR1 index registers -------------*/
 
 /* BAR1 Mask */
@@ -198,9 +207,9 @@ struct octeon_fn_list {
 	void (*setup_oq_regs)(struct octeon_device *, u32);
 
 	irqreturn_t (*process_interrupt_regs)(void *);
+	u64 (*msix_interrupt_handler)(void *);
 	int (*soft_reset)(struct octeon_device *);
 	int (*setup_device_regs)(struct octeon_device *);
-	void (*reinit_regs)(struct octeon_device *);
 	void (*bar1_idx_setup)(struct octeon_device *, u64, u32, int);
 	void (*bar1_idx_write)(struct octeon_device *, u32, u32);
 	u32 (*bar1_idx_read)(struct octeon_device *, u32);
@@ -209,10 +218,10 @@ struct octeon_fn_list {
 	void (*enable_oq_pkt_time_intr)(struct octeon_device *, u32);
 	void (*disable_oq_pkt_time_intr)(struct octeon_device *, u32);
 
-	void (*enable_interrupt)(void *);
-	void (*disable_interrupt)(void *);
+	void (*enable_interrupt)(struct octeon_device *, u8);
+	void (*disable_interrupt)(struct octeon_device *, u8);
 
-	void (*enable_io_queues)(struct octeon_device *);
+	int (*enable_io_queues)(struct octeon_device *);
 	void (*disable_io_queues)(struct octeon_device *);
 };
 
@@ -266,11 +275,72 @@ struct octdev_props {
 	/* Each interface in the Octeon device has a network
 	 * device pointer (used for OS specific calls).
 	 */
+	int    rx_on;
 	int    napi_enabled;
 	int    gmxport;
 	struct net_device *netdev;
 };
 
+#define LIO_FLAG_MSIX_ENABLED	0x1
+#define MSIX_PO_INT		0x1
+#define MSIX_PI_INT		0x2
+
+struct octeon_pf_vf_hs_word {
+#ifdef __LITTLE_ENDIAN_BITFIELD
+	/** PKIND value assigned for the DPI interface */
+	u64        pkind : 8;
+
+	/** OCTEON core clock multiplier   */
+	u64        core_tics_per_us : 16;
+
+	/** OCTEON coprocessor clock multiplier  */
+	u64        coproc_tics_per_us : 16;
+
+	/** app that currently running on OCTEON  */
+	u64        app_mode : 8;
+
+	/** RESERVED */
+	u64 reserved : 16;
+
+#else
+
+	/** RESERVED */
+	u64 reserved : 16;
+
+	/** app that currently running on OCTEON  */
+	u64        app_mode : 8;
+
+	/** OCTEON coprocessor clock multiplier  */
+	u64        coproc_tics_per_us : 16;
+
+	/** OCTEON core clock multiplier   */
+	u64        core_tics_per_us : 16;
+
+	/** PKIND value assigned for the DPI interface */
+	u64        pkind : 8;
+#endif
+};
+
+struct octeon_sriov_info {
+	/* Actual rings left for PF device */
+	u32	num_pf_rings;
+
+	/* SRN of PF usable IO queues   */
+	u32	pf_srn;
+	/* total pf rings */
+	u32	trs;
+
+};
+
+struct octeon_ioq_vector {
+	struct octeon_device   *oct_dev;
+	int		        iq_index;
+	int		        droq_index;
+	int			vector;
+	struct cpumask		affinity_mask;
+	u32			ioq_num;
+};
+
 /** The Octeon device.
  *  Each Octeon device has this structure to represent all its
  *  components.
@@ -296,7 +366,7 @@ struct octeon_device {
 	/** Octeon Chip type. */
 	u16 chip_id;
 	u16 rev_id;
-
+	u16 pf_num;
 	/** This device's id - set by the driver. */
 	u32 octeon_id;
 
@@ -305,7 +375,6 @@ struct octeon_device {
 
 	u16 flags;
 #define LIO_FLAG_MSI_ENABLED                  (u32)(1 << 1)
-#define LIO_FLAG_MSIX_ENABLED                 (u32)(1 << 2)
 
 	/** The state of this device */
 	atomic_t status;
@@ -395,6 +464,19 @@ struct octeon_device {
 
 	void *priv;
 
+	int num_msix_irqs;
+
+	void *msix_entries;
+
+	struct octeon_sriov_info sriov_info;
+
+	struct octeon_pf_vf_hs_word pfvf_hsword;
+
+	int msix_on;
+
+	/** IOq information of it's corresponding MSI-X interrupt. */
+	struct octeon_ioq_vector    *ioq_vector;
+
 	int rx_pause;
 	int tx_pause;
 
@@ -402,12 +484,15 @@ struct octeon_device {
 
 	/* private flags to control driver-specific features through ethtool */
 	u32 priv_flags;
+
+	void *watchdog_task;
 };
 
 #define  OCT_DRV_ONLINE 1
 #define  OCT_DRV_OFFLINE 2
 #define  OCTEON_CN6XXX(oct)           ((oct->chip_id == OCTEON_CN66XX) || \
 				       (oct->chip_id == OCTEON_CN68XX))
+#define  OCTEON_CN23XX_PF(oct)        (oct->chip_id == OCTEON_CN23XX_PF_VID)
 #define CHIP_FIELD(oct, TYPE, field)             \
 	(((struct octeon_ ## TYPE  *)(oct->chip))->field)
 
@@ -661,13 +746,24 @@ void *oct_get_config_info(struct octeon_device *oct, u16 card_type);
  */
 struct octeon_config *octeon_get_conf(struct octeon_device *oct);
 
+void octeon_free_ioq_vector(struct octeon_device *oct);
+int octeon_allocate_ioq_vector(struct octeon_device  *oct);
+void lio_enable_irq(struct octeon_droq *droq, struct octeon_instr_queue *iq);
+
 /* LiquidIO driver pivate flags */
 enum {
 	OCT_PRIV_FLAG_TX_BYTES = 0, /* Tx interrupts by pending byte count */
 };
 
-static inline void lio_set_priv_flag(struct octeon_device *octdev, u32 flag,
-				     u32 val)
+#define OCT_PRIV_FLAG_DEFAULT 0x0
+
+static inline u32 lio_get_priv_flag(struct octeon_device *octdev, u32 flag)
+{
+	return !!(octdev->priv_flags & (0x1 << flag));
+}
+
+static inline void lio_set_priv_flag(struct octeon_device *octdev,
+				     u32 flag, u32 val)
 {
 	if (val)
 		octdev->priv_flags |= (0x1 << flag);
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_droq.c b/drivers/net/ethernet/cavium/liquidio/octeon_droq.c
index e0afe4c1fd01..f60e5320daf4 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_droq.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_droq.c
@@ -31,6 +31,7 @@
 #include "octeon_network.h"
 #include "cn66xx_regs.h"
 #include "cn66xx_device.h"
+#include "cn23xx_pf_device.h"
 
 #define     CVM_MIN(d1, d2)           (((d1) < (d2)) ? (d1) : (d2))
 #define     CVM_MAX(d1, d2)           (((d1) > (d2)) ? (d1) : (d2))
@@ -92,22 +93,25 @@ static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
 	return fn_arg;
 }
 
-/** Check for packets on Droq. This function should be called with
- * lock held.
+/** Check for packets on Droq. This function should be called with lock held.
  *  @param  droq - Droq on which count is checked.
  *  @return Returns packet count.
  */
 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
 {
 	u32 pkt_count = 0;
+	u32 last_count;
 
 	pkt_count = readl(droq->pkts_sent_reg);
-	if (pkt_count) {
-		atomic_add(pkt_count, &droq->pkts_pending);
-		writel(pkt_count, droq->pkts_sent_reg);
-	}
 
-	return pkt_count;
+	last_count = pkt_count - droq->pkt_count;
+	droq->pkt_count = pkt_count;
+
+	/* we shall write to cnts  at napi irq enable or end of droq tasklet */
+	if (last_count)
+		atomic_add(last_count, &droq->pkts_pending);
+
+	return last_count;
 }
 
 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
@@ -259,6 +263,11 @@ int octeon_init_droq(struct octeon_device *oct,
 		c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
 		c_refill_threshold =
 			(u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
+	} else if (OCTEON_CN23XX_PF(oct)) {
+		struct octeon_config *conf23 = CHIP_FIELD(oct, cn23xx_pf, conf);
+
+		c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
+		c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
 	} else {
 		return 1;
 	}
@@ -564,7 +573,7 @@ octeon_droq_dispatch_pkt(struct octeon_device *oct,
 			(unsigned int)rh->r.opcode,
 			(unsigned int)rh->r.subcode);
 		droq->stats.dropped_nodispatch++;
-	}                       /* else (dispatch_fn ... */
+	}
 
 	return cnt;
 }
@@ -735,16 +744,20 @@ octeon_droq_process_packets(struct octeon_device *oct,
 	u32 pkt_count = 0, pkts_processed = 0;
 	struct list_head *tmp, *tmp2;
 
+	/* Grab the droq lock */
+	spin_lock(&droq->lock);
+
+	octeon_droq_check_hw_for_pkts(droq);
 	pkt_count = atomic_read(&droq->pkts_pending);
-	if (!pkt_count)
+
+	if (!pkt_count) {
+		spin_unlock(&droq->lock);
 		return 0;
+	}
 
 	if (pkt_count > budget)
 		pkt_count = budget;
 
-	/* Grab the droq lock */
-	spin_lock(&droq->lock);
-
 	pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count);
 
 	atomic_sub(pkts_processed, &droq->pkts_pending);
@@ -789,6 +802,8 @@ octeon_droq_process_poll_pkts(struct octeon_device *oct,
 	spin_lock(&droq->lock);
 
 	while (total_pkts_processed < budget) {
+		octeon_droq_check_hw_for_pkts(droq);
+
 		pkts_available =
 			CVM_MIN((budget - total_pkts_processed),
 				(u32)(atomic_read(&droq->pkts_pending)));
@@ -803,8 +818,6 @@ octeon_droq_process_poll_pkts(struct octeon_device *oct,
 		atomic_sub(pkts_processed, &droq->pkts_pending);
 
 		total_pkts_processed += pkts_processed;
-
-		octeon_droq_check_hw_for_pkts(droq);
 	}
 
 	spin_unlock(&droq->lock);
@@ -874,8 +887,11 @@ octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd,
 			return 0;
 		}
 		break;
+		case OCTEON_CN23XX_PF_VID: {
+			lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
+		}
+		break;
 		}
-
 		return 0;
 	}
 
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_droq.h b/drivers/net/ethernet/cavium/liquidio/octeon_droq.h
index 5a6fb9113bbd..5be002d5dba4 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_droq.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_droq.h
@@ -261,6 +261,8 @@ struct octeon_droq {
 
 	u32 q_no;
 
+	u32 pkt_count;
+
 	struct octeon_droq_ops ops;
 
 	struct octeon_device *oct_dev;
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
index ff4b1d6f007b..e4d426ba18dc 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
@@ -88,6 +88,8 @@ struct octeon_instr_queue {
 	/** A spinlock to protect while posting on the ring.  */
 	spinlock_t post_lock;
 
+	u32 pkt_in_done;
+
 	/** A spinlock to protect access to the input ring.*/
 	spinlock_t iq_flush_running_lock;
 
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_main.h b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
index bc14e4c27332..366298f7bcb2 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_main.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
@@ -38,12 +38,26 @@
 
 #define DRV_NAME "LiquidIO"
 
-/**
- * \brief determines if a given console has debug enabled.
- * @param console console to check
- * @returns  1 = enabled. 0 otherwise
+/** This structure is used by NIC driver to store information required
+ * to free the sk_buff when the packet has been fetched by Octeon.
+ * Bytes offset below assume worst-case of a 64-bit system.
  */
-int octeon_console_debug_enabled(u32 console);
+struct octnet_buf_free_info {
+	/** Bytes 1-8.  Pointer to network device private structure. */
+	struct lio *lio;
+
+	/** Bytes 9-16.  Pointer to sk_buff. */
+	struct sk_buff *skb;
+
+	/** Bytes 17-24.  Pointer to gather list. */
+	struct octnic_gather *g;
+
+	/** Bytes 25-32. Physical address of skb->data or gather list. */
+	u64 dptr;
+
+	/** Bytes 33-47. Piggybacked soft command, if any */
+	struct octeon_soft_command *sc;
+};
 
 /* BQL-related functions */
 void octeon_report_sent_bytes_to_bql(void *buf, int reqtype);
@@ -167,22 +181,26 @@ cnnic_numa_alloc_aligned_dma(u32 size,
 #define cnnic_free_aligned_dma(pci_dev, ptr, size, orig_ptr, dma_addr) \
 		free_pages(orig_ptr, get_order(size))
 
-static inline void
+static inline int
 sleep_cond(wait_queue_head_t *wait_queue, int *condition)
 {
+	int errno = 0;
 	wait_queue_t we;
 
 	init_waitqueue_entry(&we, current);
 	add_wait_queue(wait_queue, &we);
 	while (!(READ_ONCE(*condition))) {
 		set_current_state(TASK_INTERRUPTIBLE);
-		if (signal_pending(current))
+		if (signal_pending(current)) {
+			errno = -EINTR;
 			goto out;
+		}
 		schedule();
 	}
 out:
 	set_current_state(TASK_RUNNING);
 	remove_wait_queue(wait_queue, &we);
+	return errno;
 }
 
 static inline void
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.c b/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.c
index 95a4bbedf557..0dc081a99b30 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_mem_ops.c
@@ -19,7 +19,6 @@
  * This file may also be available under a different license from Cavium.
  * Contact Cavium, Inc. for more information
  **********************************************************************/
-#include <linux/pci.h>
 #include <linux/netdevice.h>
 #include "liquidio_common.h"
 #include "octeon_droq.h"
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_network.h b/drivers/net/ethernet/cavium/liquidio/octeon_network.h
index fb820dc7fcb7..e5d1debd05ad 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_network.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_network.h
@@ -26,8 +26,6 @@
 
 #ifndef __OCTEON_NETWORK_H__
 #define __OCTEON_NETWORK_H__
-#include <linux/version.h>
-#include <linux/dma-mapping.h>
 #include <linux/ptp_clock_kernel.h>
 
 #define LIO_MAX_MTU_SIZE (OCTNET_MAX_FRM_SIZE - OCTNET_FRM_HEADER_SIZE)
@@ -124,11 +122,21 @@ struct lio {
 
 	/* work queue for  txq status */
 	struct cavium_wq	txq_status_wq;
+
+	/* work queue for  link status */
+	struct cavium_wq	link_status_wq;
+
 };
 
 #define LIO_SIZE         (sizeof(struct lio))
 #define GET_LIO(netdev)  ((struct lio *)netdev_priv(netdev))
 
+#define CIU3_WDOG(c)                 (0x1010000020000ULL + (c << 3))
+#define CIU3_WDOG_MASK               12ULL
+#define LIO_MONITOR_WDOG_EXPIRE      1
+#define LIO_MONITOR_CORE_STUCK_MSGD  2
+#define LIO_MAX_CORES                12
+
 /**
  * \brief Enable or disable feature
  * @param netdev    pointer to network device
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_nic.c b/drivers/net/ethernet/cavium/liquidio/octeon_nic.c
index 166727be928f..40ac1fe88956 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_nic.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_nic.c
@@ -19,7 +19,6 @@
  * This file may also be available under a different license from Cavium.
  * Contact Cavium, Inc. for more information
  **********************************************************************/
-#include <linux/interrupt.h>
 #include <linux/pci.h>
 #include <linux/netdevice.h>
 #include "liquidio_common.h"
@@ -36,6 +35,7 @@ octeon_alloc_soft_command_resp(struct octeon_device    *oct,
 			       u32		       rdatasize)
 {
 	struct octeon_soft_command *sc;
+	struct octeon_instr_ih3  *ih3;
 	struct octeon_instr_ih2  *ih2;
 	struct octeon_instr_irh *irh;
 	struct octeon_instr_rdp *rdp;
@@ -52,10 +52,19 @@ octeon_alloc_soft_command_resp(struct octeon_device    *oct,
 	/* Add in the response related fields. Opcode and Param are already
 	 * there.
 	 */
-	ih2      = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
-	rdp     = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
-	irh     = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
-	ih2->fsz = 40; /* irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */
+	if (OCTEON_CN23XX_PF(oct)) {
+		ih3      = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
+		rdp     = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
+		irh     = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
+		/*pkiih3 + irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */
+		ih3->fsz = LIO_SOFTCMDRESP_IH3;
+	} else {
+		ih2      = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
+		rdp     = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
+		irh     = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
+		/* irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */
+		ih2->fsz = LIO_SOFTCMDRESP_IH2;
+	}
 
 	irh->rflag = 1; /* a response is required */
 
@@ -64,7 +73,10 @@ octeon_alloc_soft_command_resp(struct octeon_device    *oct,
 
 	*sc->status_word = COMPLETION_WORD_INIT;
 
-	sc->cmd.cmd2.rptr =  sc->dmarptr;
+	if (OCTEON_CN23XX_PF(oct))
+		sc->cmd.cmd3.rptr =  sc->dmarptr;
+	else
+		sc->cmd.cmd2.rptr =  sc->dmarptr;
 
 	sc->wait_time = 1000;
 	sc->timeout = jiffies + sc->wait_time;
@@ -73,12 +85,9 @@ octeon_alloc_soft_command_resp(struct octeon_device    *oct,
 }
 
 int octnet_send_nic_data_pkt(struct octeon_device *oct,
-			     struct octnic_data_pkt *ndata,
-			     u32 xmit_more)
+			     struct octnic_data_pkt *ndata)
 {
-	int ring_doorbell;
-
-	ring_doorbell = !xmit_more;
+	int ring_doorbell = 1;
 
 	return octeon_send_command(oct, ndata->q_no, ring_doorbell, &ndata->cmd,
 				   ndata->buf, ndata->datasize,
@@ -183,8 +192,8 @@ octnet_send_nic_ctrl_pkt(struct octeon_device *oct,
 	retval = octeon_send_soft_command(oct, sc);
 	if (retval == IQ_SEND_FAILED) {
 		octeon_free_soft_command(oct, sc);
-		dev_err(&oct->pci_dev->dev, "%s soft command:%d send failed status: %x\n",
-			__func__, nctrl->ncmd.s.cmd, retval);
+		dev_err(&oct->pci_dev->dev, "%s pf_num:%d soft command:%d send failed status: %x\n",
+			__func__, oct->pf_num, nctrl->ncmd.s.cmd, retval);
 		spin_unlock_bh(&oct->cmd_resp_wqlock);
 		return -1;
 	}
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
index b71a2bbe4bee..4b8da67b995f 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
@@ -138,7 +138,7 @@ octnet_prepare_pci_cmd_o2(struct octeon_device *oct,
 	/* assume that rflag is cleared so therefore front data will only have
 	 * irh and ossp[0], ossp[1] for a total of 32 bytes
 	 */
-	ih2->fsz = 24;
+	ih2->fsz = LIO_PCICMD_O2;
 
 	ih2->tagtype = ORDERED_TAG;
 	ih2->grp = DEFAULT_POW_GRP;
@@ -196,7 +196,7 @@ octnet_prepare_pci_cmd_o3(struct octeon_device *oct,
 	 */
 	ih3->pkind       = oct->instr_queue[setup->s.iq_no]->txpciq.s.pkind;
 	/*PKI IH*/
-	ih3->fsz = 24 + 8;
+	ih3->fsz = LIO_PCICMD_O3;
 
 	if (!setup->s.gather) {
 		ih3->dlengsz = setup->s.u.datasize;
@@ -278,7 +278,7 @@ octeon_alloc_soft_command_resp(struct octeon_device    *oct,
  * queue should be stopped, and IQ_SEND_OK if it sent okay.
  */
 int octnet_send_nic_data_pkt(struct octeon_device *oct,
-			     struct octnic_data_pkt *ndata, u32 xmit_more);
+			     struct octnic_data_pkt *ndata);
 
 /** Send a NIC control packet to the device
  * @param oct - octeon device pointer
diff --git a/drivers/net/ethernet/cavium/liquidio/request_manager.c b/drivers/net/ethernet/cavium/liquidio/request_manager.c
index d32492f185ff..90866bb50033 100644
--- a/drivers/net/ethernet/cavium/liquidio/request_manager.c
+++ b/drivers/net/ethernet/cavium/liquidio/request_manager.c
@@ -30,6 +30,7 @@
 #include "octeon_main.h"
 #include "octeon_network.h"
 #include "cn66xx_device.h"
+#include "cn23xx_pf_device.h"
 
 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count)  \
 	(octeon_dev_ptr->instr_queue[iq_no]->stats.field += count)
@@ -71,7 +72,8 @@ int octeon_init_instr_queue(struct octeon_device *oct,
 
 	if (OCTEON_CN6XXX(oct))
 		conf = &(CFG_GET_IQ_CFG(CHIP_FIELD(oct, cn6xxx, conf)));
-
+	else if (OCTEON_CN23XX_PF(oct))
+		conf = &(CFG_GET_IQ_CFG(CHIP_FIELD(oct, cn23xx_pf, conf)));
 	if (!conf) {
 		dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n",
 			oct->chip_id);
@@ -88,6 +90,7 @@ int octeon_init_instr_queue(struct octeon_device *oct,
 	q_size = (u32)conf->instr_type * num_descs;
 
 	iq = oct->instr_queue[iq_no];
+
 	iq->oct_dev = oct;
 
 	set_dev_node(&oct->pci_dev->dev, numa_node);
@@ -181,6 +184,9 @@ int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)
 	if (OCTEON_CN6XXX(oct))
 		desc_size =
 		    CFG_GET_IQ_INSTR_TYPE(CHIP_FIELD(oct, cn6xxx, conf));
+	else if (OCTEON_CN23XX_PF(oct))
+		desc_size =
+		    CFG_GET_IQ_INSTR_TYPE(CHIP_FIELD(oct, cn23xx_pf, conf));
 
 	vfree(iq->request_list);
 
@@ -383,7 +389,12 @@ lio_process_iq_request_list(struct octeon_device *oct,
 		case REQTYPE_SOFT_COMMAND:
 			sc = buf;
 
-			irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
+			if (OCTEON_CN23XX_PF(oct))
+				irh = (struct octeon_instr_irh *)
+					&sc->cmd.cmd3.irh;
+			else
+				irh = (struct octeon_instr_irh *)
+					&sc->cmd.cmd2.irh;
 			if (irh->rflag) {
 				/* We're expecting a response from Octeon.
 				 * It's up to lio_process_ordered_list() to
@@ -499,6 +510,7 @@ static void __check_db_timeout(struct octeon_device *oct, u64 iq_no)
 
 	if (!oct)
 		return;
+
 	iq = oct->instr_queue[iq_no];
 	if (!iq)
 		return;
@@ -514,6 +526,8 @@ static void __check_db_timeout(struct octeon_device *oct, u64 iq_no)
 
 	/* Flush the instruction queue */
 	octeon_flush_iq(oct, iq, 1, 0);
+
+	lio_enable_irq(NULL, iq);
 }
 
 /* Called by the Poll thread at regular intervals to check the instruction
@@ -580,6 +594,8 @@ octeon_prepare_soft_command(struct octeon_device *oct,
 {
 	struct octeon_config *oct_cfg;
 	struct octeon_instr_ih2 *ih2;
+	struct octeon_instr_ih3 *ih3;
+	struct octeon_instr_pki_ih3 *pki_ih3;
 	struct octeon_instr_irh *irh;
 	struct octeon_instr_rdp *rdp;
 
@@ -588,36 +604,88 @@ octeon_prepare_soft_command(struct octeon_device *oct,
 
 	oct_cfg = octeon_get_conf(oct);
 
-	ih2          = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
-	ih2->tagtype = ATOMIC_TAG;
-	ih2->tag     = LIO_CONTROL;
-	ih2->raw     = 1;
-	ih2->grp     = CFG_GET_CTRL_Q_GRP(oct_cfg);
-
-	if (sc->datasize) {
-		ih2->dlengsz = sc->datasize;
-		ih2->rs = 1;
-	}
-
-	irh            = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
-	irh->opcode    = opcode;
-	irh->subcode   = subcode;
-
-	/* opcode/subcode specific parameters (ossp) */
-	irh->ossp       = irh_ossp;
-	sc->cmd.cmd2.ossp[0] = ossp0;
-	sc->cmd.cmd2.ossp[1] = ossp1;
-
-	if (sc->rdatasize) {
-		rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
-		rdp->pcie_port = oct->pcie_port;
-		rdp->rlen      = sc->rdatasize;
+	if (OCTEON_CN23XX_PF(oct)) {
+		ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
+
+		ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind;
+
+		pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3;
+
+		pki_ih3->w           = 1;
+		pki_ih3->raw         = 1;
+		pki_ih3->utag        = 1;
+		pki_ih3->uqpg        =
+			oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg;
+		pki_ih3->utt         = 1;
+		pki_ih3->tag     = LIO_CONTROL;
+		pki_ih3->tagtype = ATOMIC_TAG;
+		pki_ih3->qpg         =
+			oct->instr_queue[sc->iq_no]->txpciq.s.qpg;
+		pki_ih3->pm          = 0x7;
+		pki_ih3->sl          = 8;
+
+		if (sc->datasize)
+			ih3->dlengsz = sc->datasize;
+
+		irh            = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
+		irh->opcode    = opcode;
+		irh->subcode   = subcode;
+
+		/* opcode/subcode specific parameters (ossp) */
+		irh->ossp       = irh_ossp;
+		sc->cmd.cmd3.ossp[0] = ossp0;
+		sc->cmd.cmd3.ossp[1] = ossp1;
+
+		if (sc->rdatasize) {
+			rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
+			rdp->pcie_port = oct->pcie_port;
+			rdp->rlen      = sc->rdatasize;
+
+			irh->rflag =  1;
+			/*PKI IH3*/
+			/* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */
+			ih3->fsz    = LIO_SOFTCMDRESP_IH3;
+		} else {
+			irh->rflag =  0;
+			/*PKI IH3*/
+			/* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */
+			ih3->fsz    = LIO_PCICMD_O3;
+		}
 
-		irh->rflag =  1;
-		ih2->fsz   = 40; /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
 	} else {
-		irh->rflag =  0;
-		ih2->fsz   = 24; /* irh + ossp[0] + ossp[1] = 24 bytes */
+		ih2          = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
+		ih2->tagtype = ATOMIC_TAG;
+		ih2->tag     = LIO_CONTROL;
+		ih2->raw     = 1;
+		ih2->grp     = CFG_GET_CTRL_Q_GRP(oct_cfg);
+
+		if (sc->datasize) {
+			ih2->dlengsz = sc->datasize;
+			ih2->rs = 1;
+		}
+
+		irh            = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
+		irh->opcode    = opcode;
+		irh->subcode   = subcode;
+
+		/* opcode/subcode specific parameters (ossp) */
+		irh->ossp       = irh_ossp;
+		sc->cmd.cmd2.ossp[0] = ossp0;
+		sc->cmd.cmd2.ossp[1] = ossp1;
+
+		if (sc->rdatasize) {
+			rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
+			rdp->pcie_port = oct->pcie_port;
+			rdp->rlen      = sc->rdatasize;
+
+			irh->rflag =  1;
+			/* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
+			ih2->fsz   = LIO_SOFTCMDRESP_IH2;
+		} else {
+			irh->rflag =  0;
+			/* irh + ossp[0] + ossp[1] = 24 bytes */
+			ih2->fsz   = LIO_PCICMD_O2;
+		}
 	}
 }
 
@@ -625,23 +693,39 @@ int octeon_send_soft_command(struct octeon_device *oct,
 			     struct octeon_soft_command *sc)
 {
 	struct octeon_instr_ih2 *ih2;
+	struct octeon_instr_ih3 *ih3;
 	struct octeon_instr_irh *irh;
 	u32 len;
 
-	ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
-	if (ih2->dlengsz) {
-		WARN_ON(!sc->dmadptr);
-		sc->cmd.cmd2.dptr = sc->dmadptr;
-	}
-	irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
-	if (irh->rflag) {
-		WARN_ON(!sc->dmarptr);
-		WARN_ON(!sc->status_word);
-		*sc->status_word = COMPLETION_WORD_INIT;
-
-		sc->cmd.cmd2.rptr = sc->dmarptr;
+	if (OCTEON_CN23XX_PF(oct)) {
+		ih3 =  (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
+		if (ih3->dlengsz) {
+			WARN_ON(!sc->dmadptr);
+			sc->cmd.cmd3.dptr = sc->dmadptr;
+		}
+		irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
+		if (irh->rflag) {
+			WARN_ON(!sc->dmarptr);
+			WARN_ON(!sc->status_word);
+			*sc->status_word = COMPLETION_WORD_INIT;
+			sc->cmd.cmd3.rptr = sc->dmarptr;
+		}
+		len = (u32)ih3->dlengsz;
+	} else {
+		ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
+		if (ih2->dlengsz) {
+			WARN_ON(!sc->dmadptr);
+			sc->cmd.cmd2.dptr = sc->dmadptr;
+		}
+		irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
+		if (irh->rflag) {
+			WARN_ON(!sc->dmarptr);
+			WARN_ON(!sc->status_word);
+			*sc->status_word = COMPLETION_WORD_INIT;
+			sc->cmd.cmd2.rptr = sc->dmarptr;
+		}
+		len = (u32)ih2->dlengsz;
 	}
-	len = (u32)ih2->dlengsz;
 
 	if (sc->wait_time)
 		sc->timeout = jiffies + sc->wait_time;
diff --git a/drivers/net/ethernet/cavium/liquidio/response_manager.c b/drivers/net/ethernet/cavium/liquidio/response_manager.c
index 709049e36627..be52178d8cb6 100644
--- a/drivers/net/ethernet/cavium/liquidio/response_manager.c
+++ b/drivers/net/ethernet/cavium/liquidio/response_manager.c
@@ -91,8 +91,13 @@ int lio_process_ordered_list(struct octeon_device *octeon_dev,
 
 		sc = (struct octeon_soft_command *)ordered_sc_list->
 		    head.next;
-		rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
-		rptr = sc->cmd.cmd2.rptr;
+		if (OCTEON_CN23XX_PF(octeon_dev)) {
+			rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
+			rptr = sc->cmd.cmd3.rptr;
+		} else {
+			rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
+			rptr = sc->cmd.cmd2.rptr;
+		}
 
 		status = OCTEON_REQUEST_PENDING;
 
diff --git a/drivers/net/ethernet/cavium/thunder/Makefile b/drivers/net/ethernet/cavium/thunder/Makefile
index 5c4615ccaa14..6b4d4add7353 100644
--- a/drivers/net/ethernet/cavium/thunder/Makefile
+++ b/drivers/net/ethernet/cavium/thunder/Makefile
@@ -2,6 +2,7 @@
 # Makefile for Cavium's Thunder ethernet device
 #
 
+obj-$(CONFIG_THUNDER_NIC_RGX) += thunder_xcv.o
 obj-$(CONFIG_THUNDER_NIC_BGX) += thunder_bgx.o
 obj-$(CONFIG_THUNDER_NIC_PF) += nicpf.o
 obj-$(CONFIG_THUNDER_NIC_VF) += nicvf.o
diff --git a/drivers/net/ethernet/cavium/thunder/nic.h b/drivers/net/ethernet/cavium/thunder/nic.h
index 83025bb4737c..30426109711c 100644
--- a/drivers/net/ethernet/cavium/thunder/nic.h
+++ b/drivers/net/ethernet/cavium/thunder/nic.h
@@ -20,6 +20,17 @@
 #define	PCI_DEVICE_ID_THUNDER_NIC_VF		0xA034
 #define	PCI_DEVICE_ID_THUNDER_BGX		0xA026
 
+/* Subsystem device IDs */
+#define PCI_SUBSYS_DEVID_88XX_NIC_PF		0xA11E
+#define PCI_SUBSYS_DEVID_81XX_NIC_PF		0xA21E
+#define PCI_SUBSYS_DEVID_83XX_NIC_PF		0xA31E
+
+#define PCI_SUBSYS_DEVID_88XX_PASS1_NIC_VF	0xA11E
+#define PCI_SUBSYS_DEVID_88XX_NIC_VF		0xA134
+#define PCI_SUBSYS_DEVID_81XX_NIC_VF		0xA234
+#define PCI_SUBSYS_DEVID_83XX_NIC_VF		0xA334
+
+
 /* PCI BAR nos */
 #define	PCI_CFG_REG_BAR_NUM		0
 #define	PCI_MSIX_REG_BAR_NUM		4
@@ -41,40 +52,8 @@
 /* Max pkinds */
 #define	NIC_MAX_PKIND			16
 
-/* Rx Channels */
-/* Receive channel configuration in TNS bypass mode
- * Below is configuration in TNS bypass mode
- * BGX0-LMAC0-CHAN0 - VNIC CHAN0
- * BGX0-LMAC1-CHAN0 - VNIC CHAN16
- * ...
- * BGX1-LMAC0-CHAN0 - VNIC CHAN128
- * ...
- * BGX1-LMAC3-CHAN0 - VNIC CHAN174
- */
-#define	NIC_INTF_COUNT			2  /* Interfaces btw VNIC and TNS/BGX */
-#define	NIC_CHANS_PER_INF		128
-#define	NIC_MAX_CHANS			(NIC_INTF_COUNT * NIC_CHANS_PER_INF)
-#define	NIC_CPI_COUNT			2048 /* No of channel parse indices */
-
-/* TNS bypass mode: 1-1 mapping between VNIC and BGX:LMAC */
-#define NIC_MAX_BGX			MAX_BGX_PER_CN88XX
-#define	NIC_CPI_PER_BGX			(NIC_CPI_COUNT / NIC_MAX_BGX)
-#define	NIC_MAX_CPI_PER_LMAC		64 /* Max when CPI_ALG is IP diffserv */
-#define	NIC_RSSI_PER_BGX		(NIC_RSSI_COUNT / NIC_MAX_BGX)
-
-/* Tx scheduling */
-#define	NIC_MAX_TL4			1024
-#define	NIC_MAX_TL4_SHAPERS		256 /* 1 shaper for 4 TL4s */
-#define	NIC_MAX_TL3			256
-#define	NIC_MAX_TL3_SHAPERS		64  /* 1 shaper for 4 TL3s */
-#define	NIC_MAX_TL2			64
-#define	NIC_MAX_TL2_SHAPERS		2  /* 1 shaper for 32 TL2s */
-#define	NIC_MAX_TL1			2
-
-/* TNS bypass mode */
-#define	NIC_TL2_PER_BGX			32
-#define	NIC_TL4_PER_BGX			(NIC_MAX_TL4 / NIC_MAX_BGX)
-#define	NIC_TL4_PER_LMAC		(NIC_MAX_TL4 / NIC_CHANS_PER_INF)
+/* Max when CPI_ALG is IP diffserv */
+#define	NIC_MAX_CPI_PER_LMAC		64
 
 /* NIC VF Interrupts */
 #define	NICVF_INTR_CQ			0
@@ -148,7 +127,6 @@ struct nicvf_cq_poll {
 	struct	napi_struct napi;
 };
 
-#define	NIC_RSSI_COUNT			4096 /* Total no of RSS indices */
 #define NIC_MAX_RSS_HASH_BITS		8
 #define NIC_MAX_RSS_IDR_TBL_SIZE	(1 << NIC_MAX_RSS_HASH_BITS)
 #define RSS_HASH_KEY_SIZE		5 /* 320 bit key */
@@ -273,12 +251,14 @@ struct nicvf {
 	struct net_device	*netdev;
 	struct pci_dev		*pdev;
 	void __iomem		*reg_base;
+#define	MAX_QUEUES_PER_QSET			8
 	struct queue_set	*qs;
 	struct nicvf_cq_poll	*napi[8];
 	u8			vf_id;
 	u8			sqs_id;
 	bool                    sqs_mode;
 	bool			hw_tso;
+	bool			t88;
 
 	/* Receive buffer alloc */
 	u32			rb_page_offset;
@@ -325,7 +305,7 @@ struct nicvf {
 	bool			msix_enabled;
 	u8			num_vec;
 	struct msix_entry	msix_entries[NIC_VF_MSIX_VECTORS];
-	char			irq_name[NIC_VF_MSIX_VECTORS][20];
+	char			irq_name[NIC_VF_MSIX_VECTORS][IFNAMSIZ + 15];
 	bool			irq_allocated[NIC_VF_MSIX_VECTORS];
 	cpumask_var_t		affinity_mask[NIC_VF_MSIX_VECTORS];
 
@@ -368,6 +348,7 @@ struct nicvf {
 #define	NIC_MBOX_MSG_PNICVF_PTR		0x14	/* Get primary qset nicvf ptr */
 #define	NIC_MBOX_MSG_SNICVF_PTR		0x15	/* Send sqet nicvf ptr to PVF */
 #define	NIC_MBOX_MSG_LOOPBACK		0x16	/* Set interface in loopback */
+#define	NIC_MBOX_MSG_RESET_STAT_COUNTER 0x17	/* Reset statistics counters */
 #define	NIC_MBOX_MSG_CFG_DONE		0xF0	/* VF configuration done */
 #define	NIC_MBOX_MSG_SHUTDOWN		0xF1	/* VF is being shutdown */
 
@@ -484,6 +465,31 @@ struct set_loopback {
 	bool  enable;
 };
 
+/* Reset statistics counters */
+struct reset_stat_cfg {
+	u8    msg;
+	/* Bitmap to select NIC_PF_VNIC(vf_id)_RX_STAT(0..13) */
+	u16   rx_stat_mask;
+	/* Bitmap to select NIC_PF_VNIC(vf_id)_TX_STAT(0..4) */
+	u8    tx_stat_mask;
+	/* Bitmap to select NIC_PF_QS(0..127)_RQ(0..7)_STAT(0..1)
+	 * bit14, bit15 NIC_PF_QS(vf_id)_RQ7_STAT(0..1)
+	 * bit12, bit13 NIC_PF_QS(vf_id)_RQ6_STAT(0..1)
+	 * ..
+	 * bit2, bit3 NIC_PF_QS(vf_id)_RQ1_STAT(0..1)
+	 * bit0, bit1 NIC_PF_QS(vf_id)_RQ0_STAT(0..1)
+	 */
+	u16   rq_stat_mask;
+	/* Bitmap to select NIC_PF_QS(0..127)_SQ(0..7)_STAT(0..1)
+	 * bit14, bit15 NIC_PF_QS(vf_id)_SQ7_STAT(0..1)
+	 * bit12, bit13 NIC_PF_QS(vf_id)_SQ6_STAT(0..1)
+	 * ..
+	 * bit2, bit3 NIC_PF_QS(vf_id)_SQ1_STAT(0..1)
+	 * bit0, bit1 NIC_PF_QS(vf_id)_SQ0_STAT(0..1)
+	 */
+	u16   sq_stat_mask;
+};
+
 /* 128 bit shared memory between PF and each VF */
 union nic_mbx {
 	struct { u8 msg; }	msg;
@@ -501,6 +507,7 @@ union nic_mbx {
 	struct sqs_alloc        sqs_alloc;
 	struct nicvf_ptr	nicvf;
 	struct set_loopback	lbk;
+	struct reset_stat_cfg	reset_stat;
 };
 
 #define NIC_NODE_ID_MASK	0x03
@@ -514,7 +521,14 @@ static inline int nic_get_node_id(struct pci_dev *pdev)
 
 static inline bool pass1_silicon(struct pci_dev *pdev)
 {
-	return pdev->revision < 8;
+	return (pdev->revision < 8) &&
+		(pdev->subsystem_device == PCI_SUBSYS_DEVID_88XX_NIC_PF);
+}
+
+static inline bool pass2_silicon(struct pci_dev *pdev)
+{
+	return (pdev->revision >= 8) &&
+		(pdev->subsystem_device == PCI_SUBSYS_DEVID_88XX_NIC_PF);
 }
 
 int nicvf_set_real_num_queues(struct net_device *netdev,
diff --git a/drivers/net/ethernet/cavium/thunder/nic_main.c b/drivers/net/ethernet/cavium/thunder/nic_main.c
index 16ed20357c5c..2bbf4cbf08b2 100644
--- a/drivers/net/ethernet/cavium/thunder/nic_main.c
+++ b/drivers/net/ethernet/cavium/thunder/nic_main.c
@@ -20,8 +20,25 @@
 #define DRV_NAME	"thunder-nic"
 #define DRV_VERSION	"1.0"
 
+struct hw_info {
+	u8		bgx_cnt;
+	u8		chans_per_lmac;
+	u8		chans_per_bgx; /* Rx/Tx chans */
+	u8		chans_per_rgx;
+	u8		chans_per_lbk;
+	u16		cpi_cnt;
+	u16		rssi_cnt;
+	u16		rss_ind_tbl_size;
+	u16		tl4_cnt;
+	u16		tl3_cnt;
+	u8		tl2_cnt;
+	u8		tl1_cnt;
+	bool		tl1_per_bgx; /* TL1 per BGX or per LMAC */
+};
+
 struct nicpf {
 	struct pci_dev		*pdev;
+	struct hw_info          *hw;
 	u8			node;
 	unsigned int		flags;
 	u8			num_vf_en;      /* No of VF enabled */
@@ -36,22 +53,22 @@ struct nicpf {
 #define	NIC_SET_VF_LMAC_MAP(bgx, lmac)	(((bgx & 0xF) << 4) | (lmac & 0xF))
 #define	NIC_GET_BGX_FROM_VF_LMAC_MAP(map)	((map >> 4) & 0xF)
 #define	NIC_GET_LMAC_FROM_VF_LMAC_MAP(map)	(map & 0xF)
-	u8			vf_lmac_map[MAX_LMAC];
+	u8			*vf_lmac_map;
 	struct delayed_work     dwork;
 	struct workqueue_struct *check_link;
-	u8			link[MAX_LMAC];
-	u8			duplex[MAX_LMAC];
-	u32			speed[MAX_LMAC];
+	u8			*link;
+	u8			*duplex;
+	u32			*speed;
 	u16			cpi_base[MAX_NUM_VFS_SUPPORTED];
 	u16			rssi_base[MAX_NUM_VFS_SUPPORTED];
-	u16			rss_ind_tbl_size;
 	bool			mbx_lock[MAX_NUM_VFS_SUPPORTED];
 
 	/* MSI-X */
 	bool			msix_enabled;
 	u8			num_vec;
-	struct msix_entry	msix_entries[NIC_PF_MSIX_VECTORS];
+	struct msix_entry	*msix_entries;
 	bool			irq_allocated[NIC_PF_MSIX_VECTORS];
+	char			irq_name[NIC_PF_MSIX_VECTORS][20];
 };
 
 /* Supported devices */
@@ -89,9 +106,22 @@ static u64 nic_reg_read(struct nicpf *nic, u64 offset)
 /* PF -> VF mailbox communication APIs */
 static void nic_enable_mbx_intr(struct nicpf *nic)
 {
-	/* Enable mailbox interrupt for all 128 VFs */
-	nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S, ~0ull);
-	nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S + sizeof(u64), ~0ull);
+	int vf_cnt = pci_sriov_get_totalvfs(nic->pdev);
+
+#define INTR_MASK(vfs) ((vfs < 64) ? (BIT_ULL(vfs) - 1) : (~0ull))
+
+	/* Clear it, to avoid spurious interrupts (if any) */
+	nic_reg_write(nic, NIC_PF_MAILBOX_INT, INTR_MASK(vf_cnt));
+
+	/* Enable mailbox interrupt for all VFs */
+	nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S, INTR_MASK(vf_cnt));
+	/* One mailbox intr enable reg per 64 VFs */
+	if (vf_cnt > 64) {
+		nic_reg_write(nic, NIC_PF_MAILBOX_INT + sizeof(u64),
+			      INTR_MASK(vf_cnt - 64));
+		nic_reg_write(nic, NIC_PF_MAILBOX_ENA_W1S + sizeof(u64),
+			      INTR_MASK(vf_cnt - 64));
+	}
 }
 
 static void nic_clear_mbx_intr(struct nicpf *nic, int vf, int mbx_reg)
@@ -144,7 +174,7 @@ static void nic_mbx_send_ready(struct nicpf *nic, int vf)
 
 	mbx.nic_cfg.tns_mode = NIC_TNS_BYPASS_MODE;
 
-	if (vf < MAX_LMAC) {
+	if (vf < nic->num_vf_en) {
 		bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]);
 		lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]);
 
@@ -155,7 +185,7 @@ static void nic_mbx_send_ready(struct nicpf *nic, int vf)
 	mbx.nic_cfg.sqs_mode = (vf >= nic->num_vf_en) ? true : false;
 	mbx.nic_cfg.node_id = nic->node;
 
-	mbx.nic_cfg.loopback_supported = vf < MAX_LMAC;
+	mbx.nic_cfg.loopback_supported = vf < nic->num_vf_en;
 
 	nic_send_msg_to_vf(nic, vf, &mbx);
 }
@@ -248,14 +278,27 @@ static int nic_update_hw_frs(struct nicpf *nic, int new_frs, int vf)
 /* Set minimum transmit packet size */
 static void nic_set_tx_pkt_pad(struct nicpf *nic, int size)
 {
-	int lmac;
+	int lmac, max_lmac;
+	u16 sdevid;
 	u64 lmac_cfg;
 
-	/* Max value that can be set is 60 */
-	if (size > 60)
-		size = 60;
+	/* There is a issue in HW where-in while sending GSO sized
+	 * pkts as part of TSO, if pkt len falls below this size
+	 * NIC will zero PAD packet and also updates IP total length.
+	 * Hence set this value to lessthan min pkt size of MAC+IP+TCP
+	 * headers, BGX will do the padding to transmit 64 byte pkt.
+	 */
+	if (size > 52)
+		size = 52;
+
+	pci_read_config_word(nic->pdev, PCI_SUBSYSTEM_ID, &sdevid);
+	/* 81xx's RGX has only one LMAC */
+	if (sdevid == PCI_SUBSYS_DEVID_81XX_NIC_PF)
+		max_lmac = ((nic->hw->bgx_cnt - 1) * MAX_LMAC_PER_BGX) + 1;
+	else
+		max_lmac = nic->hw->bgx_cnt * MAX_LMAC_PER_BGX;
 
-	for (lmac = 0; lmac < (MAX_BGX_PER_CN88XX * MAX_LMAC_PER_BGX); lmac++) {
+	for (lmac = 0; lmac < max_lmac; lmac++) {
 		lmac_cfg = nic_reg_read(nic, NIC_PF_LMAC_0_7_CFG | (lmac << 3));
 		lmac_cfg &= ~(0xF << 2);
 		lmac_cfg |= ((size / 4) << 2);
@@ -275,7 +318,7 @@ static void nic_set_lmac_vf_mapping(struct nicpf *nic)
 
 	nic->num_vf_en = 0;
 
-	for (bgx = 0; bgx < NIC_MAX_BGX; bgx++) {
+	for (bgx = 0; bgx < nic->hw->bgx_cnt; bgx++) {
 		if (!(bgx_map & (1 << bgx)))
 			continue;
 		lmac_cnt = bgx_get_lmac_count(nic->node, bgx);
@@ -295,28 +338,125 @@ static void nic_set_lmac_vf_mapping(struct nicpf *nic)
 			nic_reg_write(nic,
 				      NIC_PF_LMAC_0_7_CREDIT + (lmac * 8),
 				      lmac_credit);
+
+		/* On CN81XX there are only 8 VFs but max possible no of
+		 * interfaces are 9.
+		 */
+		if (nic->num_vf_en >= pci_sriov_get_totalvfs(nic->pdev)) {
+			nic->num_vf_en = pci_sriov_get_totalvfs(nic->pdev);
+			break;
+		}
 	}
 }
 
+static void nic_free_lmacmem(struct nicpf *nic)
+{
+	kfree(nic->vf_lmac_map);
+	kfree(nic->link);
+	kfree(nic->duplex);
+	kfree(nic->speed);
+}
+
+static int nic_get_hw_info(struct nicpf *nic)
+{
+	u8 max_lmac;
+	u16 sdevid;
+	struct hw_info *hw = nic->hw;
+
+	pci_read_config_word(nic->pdev, PCI_SUBSYSTEM_ID, &sdevid);
+
+	switch (sdevid) {
+	case PCI_SUBSYS_DEVID_88XX_NIC_PF:
+		hw->bgx_cnt = MAX_BGX_PER_CN88XX;
+		hw->chans_per_lmac = 16;
+		hw->chans_per_bgx = 128;
+		hw->cpi_cnt = 2048;
+		hw->rssi_cnt = 4096;
+		hw->rss_ind_tbl_size = NIC_MAX_RSS_IDR_TBL_SIZE;
+		hw->tl3_cnt = 256;
+		hw->tl2_cnt = 64;
+		hw->tl1_cnt = 2;
+		hw->tl1_per_bgx = true;
+		break;
+	case PCI_SUBSYS_DEVID_81XX_NIC_PF:
+		hw->bgx_cnt = MAX_BGX_PER_CN81XX;
+		hw->chans_per_lmac = 8;
+		hw->chans_per_bgx = 32;
+		hw->chans_per_rgx = 8;
+		hw->chans_per_lbk = 24;
+		hw->cpi_cnt = 512;
+		hw->rssi_cnt = 256;
+		hw->rss_ind_tbl_size = 32; /* Max RSSI / Max interfaces */
+		hw->tl3_cnt = 64;
+		hw->tl2_cnt = 16;
+		hw->tl1_cnt = 10;
+		hw->tl1_per_bgx = false;
+		break;
+	case PCI_SUBSYS_DEVID_83XX_NIC_PF:
+		hw->bgx_cnt = MAX_BGX_PER_CN83XX;
+		hw->chans_per_lmac = 8;
+		hw->chans_per_bgx = 32;
+		hw->chans_per_lbk = 64;
+		hw->cpi_cnt = 2048;
+		hw->rssi_cnt = 1024;
+		hw->rss_ind_tbl_size = 64; /* Max RSSI / Max interfaces */
+		hw->tl3_cnt = 256;
+		hw->tl2_cnt = 64;
+		hw->tl1_cnt = 18;
+		hw->tl1_per_bgx = false;
+		break;
+	}
+	hw->tl4_cnt = MAX_QUEUES_PER_QSET * pci_sriov_get_totalvfs(nic->pdev);
+
+	/* Allocate memory for LMAC tracking elements */
+	max_lmac = hw->bgx_cnt * MAX_LMAC_PER_BGX;
+	nic->vf_lmac_map = kmalloc_array(max_lmac, sizeof(u8), GFP_KERNEL);
+	if (!nic->vf_lmac_map)
+		goto error;
+	nic->link = kmalloc_array(max_lmac, sizeof(u8), GFP_KERNEL);
+	if (!nic->link)
+		goto error;
+	nic->duplex = kmalloc_array(max_lmac, sizeof(u8), GFP_KERNEL);
+	if (!nic->duplex)
+		goto error;
+	nic->speed = kmalloc_array(max_lmac, sizeof(u32), GFP_KERNEL);
+	if (!nic->speed)
+		goto error;
+	return 0;
+
+error:
+	nic_free_lmacmem(nic);
+	return -ENOMEM;
+}
+
 #define BGX0_BLOCK 8
 #define BGX1_BLOCK 9
 
-static void nic_init_hw(struct nicpf *nic)
+static int nic_init_hw(struct nicpf *nic)
 {
-	int i;
+	int i, err;
 	u64 cqm_cfg;
 
+	/* Get HW capability info */
+	err = nic_get_hw_info(nic);
+	if (err)
+		return err;
+
 	/* Enable NIC HW block */
 	nic_reg_write(nic, NIC_PF_CFG, 0x3);
 
 	/* Enable backpressure */
 	nic_reg_write(nic, NIC_PF_BP_CFG, (1ULL << 6) | 0x03);
 
-	/* Disable TNS mode on both interfaces */
-	nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG,
-		      (NIC_TNS_BYPASS_MODE << 7) | BGX0_BLOCK);
-	nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG | (1 << 8),
-		      (NIC_TNS_BYPASS_MODE << 7) | BGX1_BLOCK);
+	/* TNS and TNS bypass modes are present only on 88xx */
+	if (nic->pdev->subsystem_device == PCI_SUBSYS_DEVID_88XX_NIC_PF) {
+		/* Disable TNS mode on both interfaces */
+		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG,
+			      (NIC_TNS_BYPASS_MODE << 7) | BGX0_BLOCK);
+		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG | (1 << 8),
+			      (NIC_TNS_BYPASS_MODE << 7) | BGX1_BLOCK);
+	}
+
 	nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG,
 		      (1ULL << 63) | BGX0_BLOCK);
 	nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG + (1 << 8),
@@ -346,11 +486,14 @@ static void nic_init_hw(struct nicpf *nic)
 	cqm_cfg = nic_reg_read(nic, NIC_PF_CQM_CFG);
 	if (cqm_cfg < NICPF_CQM_MIN_DROP_LEVEL)
 		nic_reg_write(nic, NIC_PF_CQM_CFG, NICPF_CQM_MIN_DROP_LEVEL);
+
+	return 0;
 }
 
 /* Channel parse index configuration */
 static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg)
 {
+	struct hw_info *hw = nic->hw;
 	u32 vnic, bgx, lmac, chan;
 	u32 padd, cpi_count = 0;
 	u64 cpi_base, cpi, rssi_base, rssi;
@@ -360,9 +503,9 @@ static void nic_config_cpi(struct nicpf *nic, struct cpi_cfg_msg *cfg)
 	bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]);
 	lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]);
 
-	chan = (lmac * MAX_BGX_CHANS_PER_LMAC) + (bgx * NIC_CHANS_PER_INF);
-	cpi_base = (lmac * NIC_MAX_CPI_PER_LMAC) + (bgx * NIC_CPI_PER_BGX);
-	rssi_base = (lmac * nic->rss_ind_tbl_size) + (bgx * NIC_RSSI_PER_BGX);
+	chan = (lmac * hw->chans_per_lmac) + (bgx * hw->chans_per_bgx);
+	cpi_base = vnic * NIC_MAX_CPI_PER_LMAC;
+	rssi_base = vnic * hw->rss_ind_tbl_size;
 
 	/* Rx channel configuration */
 	nic_reg_write(nic, NIC_PF_CHAN_0_255_RX_BP_CFG | (chan << 3),
@@ -434,7 +577,7 @@ static void nic_send_rss_size(struct nicpf *nic, int vf)
 	msg = (u64 *)&mbx;
 
 	mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
-	mbx.rss_size.ind_tbl_size = nic->rss_ind_tbl_size;
+	mbx.rss_size.ind_tbl_size = nic->hw->rss_ind_tbl_size;
 	nic_send_msg_to_vf(nic, vf, &mbx);
 }
 
@@ -481,7 +624,7 @@ static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg)
 /* 4 level transmit side scheduler configutation
  * for TNS bypass mode
  *
- * Sample configuration for SQ0
+ * Sample configuration for SQ0 on 88xx
  * VNIC0-SQ0 -> TL4(0)   -> TL3[0]   -> TL2[0]  -> TL1[0] -> BGX0
  * VNIC1-SQ0 -> TL4(8)   -> TL3[2]   -> TL2[0]  -> TL1[0] -> BGX0
  * VNIC2-SQ0 -> TL4(16)  -> TL3[4]   -> TL2[1]  -> TL1[0] -> BGX0
@@ -494,6 +637,7 @@ static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg)
 static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic,
 			       struct sq_cfg_msg *sq)
 {
+	struct hw_info *hw = nic->hw;
 	u32 bgx, lmac, chan;
 	u32 tl2, tl3, tl4;
 	u32 rr_quantum;
@@ -512,21 +656,28 @@ static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic,
 	/* 24 bytes for FCS, IPG and preamble */
 	rr_quantum = ((NIC_HW_MAX_FRS + 24) / 4);
 
-	if (!sq->sqs_mode) {
-		tl4 = (lmac * NIC_TL4_PER_LMAC) + (bgx * NIC_TL4_PER_BGX);
-	} else {
-		for (svf = 0; svf < MAX_SQS_PER_VF; svf++) {
-			if (nic->vf_sqs[pqs_vnic][svf] == vnic)
-				break;
+	/* For 88xx 0-511 TL4 transmits via BGX0 and
+	 * 512-1023 TL4s transmit via BGX1.
+	 */
+	if (hw->tl1_per_bgx) {
+		tl4 = bgx * (hw->tl4_cnt / hw->bgx_cnt);
+		if (!sq->sqs_mode) {
+			tl4 += (lmac * MAX_QUEUES_PER_QSET);
+		} else {
+			for (svf = 0; svf < MAX_SQS_PER_VF; svf++) {
+				if (nic->vf_sqs[pqs_vnic][svf] == vnic)
+					break;
+			}
+			tl4 += (MAX_LMAC_PER_BGX * MAX_QUEUES_PER_QSET);
+			tl4 += (lmac * MAX_QUEUES_PER_QSET * MAX_SQS_PER_VF);
+			tl4 += (svf * MAX_QUEUES_PER_QSET);
 		}
-		tl4 = (MAX_LMAC_PER_BGX * NIC_TL4_PER_LMAC);
-		tl4 += (lmac * NIC_TL4_PER_LMAC * MAX_SQS_PER_VF);
-		tl4 += (svf * NIC_TL4_PER_LMAC);
-		tl4 += (bgx * NIC_TL4_PER_BGX);
+	} else {
+		tl4 = (vnic * MAX_QUEUES_PER_QSET);
 	}
 	tl4 += sq_idx;
 
-	tl3 = tl4 / (NIC_MAX_TL4 / NIC_MAX_TL3);
+	tl3 = tl4 / (hw->tl4_cnt / hw->tl3_cnt);
 	nic_reg_write(nic, NIC_PF_QSET_0_127_SQ_0_7_CFG2 |
 		      ((u64)vnic << NIC_QS_ID_SHIFT) |
 		      ((u32)sq_idx << NIC_Q_NUM_SHIFT), tl4);
@@ -534,8 +685,19 @@ static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic,
 		      ((u64)vnic << 27) | ((u32)sq_idx << 24) | rr_quantum);
 
 	nic_reg_write(nic, NIC_PF_TL3_0_255_CFG | (tl3 << 3), rr_quantum);
-	chan = (lmac * MAX_BGX_CHANS_PER_LMAC) + (bgx * NIC_CHANS_PER_INF);
-	nic_reg_write(nic, NIC_PF_TL3_0_255_CHAN | (tl3 << 3), chan);
+
+	/* On 88xx 0-127 channels are for BGX0 and
+	 * 127-255 channels for BGX1.
+	 *
+	 * On 81xx/83xx TL3_CHAN reg should be configured with channel
+	 * within LMAC i.e 0-7 and not the actual channel number like on 88xx
+	 */
+	chan = (lmac * hw->chans_per_lmac) + (bgx * hw->chans_per_bgx);
+	if (hw->tl1_per_bgx)
+		nic_reg_write(nic, NIC_PF_TL3_0_255_CHAN | (tl3 << 3), chan);
+	else
+		nic_reg_write(nic, NIC_PF_TL3_0_255_CHAN | (tl3 << 3), 0);
+
 	/* Enable backpressure on the channel */
 	nic_reg_write(nic, NIC_PF_CHAN_0_255_TX_CFG | (chan << 3), 1);
 
@@ -544,6 +706,16 @@ static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic,
 	nic_reg_write(nic, NIC_PF_TL2_0_63_CFG | (tl2 << 3), rr_quantum);
 	/* No priorities as of now */
 	nic_reg_write(nic, NIC_PF_TL2_0_63_PRI | (tl2 << 3), 0x00);
+
+	/* Unlike 88xx where TL2s 0-31 transmits to TL1 '0' and rest to TL1 '1'
+	 * on 81xx/83xx TL2 needs to be configured to transmit to one of the
+	 * possible LMACs.
+	 *
+	 * This register doesn't exist on 88xx.
+	 */
+	if (!hw->tl1_per_bgx)
+		nic_reg_write(nic, NIC_PF_TL2_LMAC | (tl2 << 3),
+			      lmac + (bgx * MAX_LMAC_PER_BGX));
 }
 
 /* Send primary nicvf pointer to secondary QS's VF */
@@ -615,7 +787,7 @@ static int nic_config_loopback(struct nicpf *nic, struct set_loopback *lbk)
 {
 	int bgx_idx, lmac_idx;
 
-	if (lbk->vf_id > MAX_LMAC)
+	if (lbk->vf_id >= nic->num_vf_en)
 		return -1;
 
 	bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lbk->vf_id]);
@@ -626,6 +798,67 @@ static int nic_config_loopback(struct nicpf *nic, struct set_loopback *lbk)
 	return 0;
 }
 
+/* Reset statistics counters */
+static int nic_reset_stat_counters(struct nicpf *nic,
+				   int vf, struct reset_stat_cfg *cfg)
+{
+	int i, stat, qnum;
+	u64 reg_addr;
+
+	for (i = 0; i < RX_STATS_ENUM_LAST; i++) {
+		if (cfg->rx_stat_mask & BIT(i)) {
+			reg_addr = NIC_PF_VNIC_0_127_RX_STAT_0_13 |
+				   (vf << NIC_QS_ID_SHIFT) |
+				   (i << 3);
+			nic_reg_write(nic, reg_addr, 0);
+		}
+	}
+
+	for (i = 0; i < TX_STATS_ENUM_LAST; i++) {
+		if (cfg->tx_stat_mask & BIT(i)) {
+			reg_addr = NIC_PF_VNIC_0_127_TX_STAT_0_4 |
+				   (vf << NIC_QS_ID_SHIFT) |
+				   (i << 3);
+			nic_reg_write(nic, reg_addr, 0);
+		}
+	}
+
+	for (i = 0; i <= 15; i++) {
+		qnum = i >> 1;
+		stat = i & 1 ? 1 : 0;
+		reg_addr = (vf << NIC_QS_ID_SHIFT) |
+			   (qnum << NIC_Q_NUM_SHIFT) | (stat << 3);
+		if (cfg->rq_stat_mask & BIT(i)) {
+			reg_addr |= NIC_PF_QSET_0_127_RQ_0_7_STAT_0_1;
+			nic_reg_write(nic, reg_addr, 0);
+		}
+		if (cfg->sq_stat_mask & BIT(i)) {
+			reg_addr |= NIC_PF_QSET_0_127_SQ_0_7_STAT_0_1;
+			nic_reg_write(nic, reg_addr, 0);
+		}
+	}
+	return 0;
+}
+
+static void nic_enable_tunnel_parsing(struct nicpf *nic, int vf)
+{
+	u64 prot_def = (IPV6_PROT << 32) | (IPV4_PROT << 16) | ET_PROT;
+	u64 vxlan_prot_def = (IPV6_PROT_DEF << 32) |
+			      (IPV4_PROT_DEF) << 16 | ET_PROT_DEF;
+
+	/* Configure tunnel parsing parameters */
+	nic_reg_write(nic, NIC_PF_RX_GENEVE_DEF,
+		      (1ULL << 63 | UDP_GENEVE_PORT_NUM));
+	nic_reg_write(nic, NIC_PF_RX_GENEVE_PROT_DEF,
+		      ((7ULL << 61) | prot_def));
+	nic_reg_write(nic, NIC_PF_RX_NVGRE_PROT_DEF,
+		      ((7ULL << 61) | prot_def));
+	nic_reg_write(nic, NIC_PF_RX_VXLAN_DEF_0_1,
+		      ((1ULL << 63) | UDP_VXLAN_PORT_NUM));
+	nic_reg_write(nic, NIC_PF_RX_VXLAN_PROT_DEF,
+		      ((0xfULL << 60) | vxlan_prot_def));
+}
+
 static void nic_enable_vf(struct nicpf *nic, int vf, bool enable)
 {
 	int bgx, lmac;
@@ -664,18 +897,17 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
 		mbx_addr += sizeof(u64);
 	}
 
-	dev_dbg(&nic->pdev->dev, "%s: Mailbox msg %d from VF%d\n",
+	dev_dbg(&nic->pdev->dev, "%s: Mailbox msg 0x%02x from VF%d\n",
 		__func__, mbx.msg.msg, vf);
 	switch (mbx.msg.msg) {
 	case NIC_MBOX_MSG_READY:
 		nic_mbx_send_ready(nic, vf);
-		if (vf < MAX_LMAC) {
+		if (vf < nic->num_vf_en) {
 			nic->link[vf] = 0;
 			nic->duplex[vf] = 0;
 			nic->speed[vf] = 0;
 		}
-		ret = 1;
-		break;
+		goto unlock;
 	case NIC_MBOX_MSG_QS_CFG:
 		reg_addr = NIC_PF_QSET_0_127_CFG |
 			   (mbx.qs.num << NIC_QS_ID_SHIFT);
@@ -693,6 +925,15 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
 			   (mbx.rq.qs_num << NIC_QS_ID_SHIFT) |
 			   (mbx.rq.rq_num << NIC_Q_NUM_SHIFT);
 		nic_reg_write(nic, reg_addr, mbx.rq.cfg);
+		/* Enable CQE_RX2_S extension in CQE_RX descriptor.
+		 * This gets appended by default on 81xx/83xx chips,
+		 * for consistency enabling the same on 88xx pass2
+		 * where this is introduced.
+		 */
+		if (pass2_silicon(nic->pdev))
+			nic_reg_write(nic, NIC_PF_RX_CFG, 0x01);
+		if (!pass1_silicon(nic->pdev))
+			nic_enable_tunnel_parsing(nic, vf);
 		break;
 	case NIC_MBOX_MSG_RQ_BP_CFG:
 		reg_addr = NIC_PF_QSET_0_127_RQ_0_7_BP_CFG |
@@ -717,8 +958,10 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
 		nic_tx_channel_cfg(nic, mbx.qs.num, &mbx.sq);
 		break;
 	case NIC_MBOX_MSG_SET_MAC:
-		if (vf >= nic->num_vf_en)
+		if (vf >= nic->num_vf_en) {
+			ret = -1; /* NACK */
 			break;
+		}
 		lmac = mbx.mac.vf_id;
 		bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]);
 		lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]);
@@ -767,25 +1010,38 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
 	case NIC_MBOX_MSG_LOOPBACK:
 		ret = nic_config_loopback(nic, &mbx.lbk);
 		break;
+	case NIC_MBOX_MSG_RESET_STAT_COUNTER:
+		ret = nic_reset_stat_counters(nic, vf, &mbx.reset_stat);
+		break;
 	default:
 		dev_err(&nic->pdev->dev,
 			"Invalid msg from VF%d, msg 0x%x\n", vf, mbx.msg.msg);
 		break;
 	}
 
-	if (!ret)
+	if (!ret) {
 		nic_mbx_send_ack(nic, vf);
-	else if (mbx.msg.msg != NIC_MBOX_MSG_READY)
+	} else if (mbx.msg.msg != NIC_MBOX_MSG_READY) {
+		dev_err(&nic->pdev->dev, "NACK for MBOX 0x%02x from VF %d\n",
+			mbx.msg.msg, vf);
 		nic_mbx_send_nack(nic, vf);
+	}
 unlock:
 	nic->mbx_lock[vf] = false;
 }
 
-static void nic_mbx_intr_handler (struct nicpf *nic, int mbx)
+static irqreturn_t nic_mbx_intr_handler(int irq, void *nic_irq)
 {
+	struct nicpf *nic = (struct nicpf *)nic_irq;
+	int mbx;
 	u64 intr;
 	u8  vf, vf_per_mbx_reg = 64;
 
+	if (irq == nic->msix_entries[NIC_PF_INTR_ID_MBOX0].vector)
+		mbx = 0;
+	else
+		mbx = 1;
+
 	intr = nic_reg_read(nic, NIC_PF_MAILBOX_INT + (mbx << 3));
 	dev_dbg(&nic->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr);
 	for (vf = 0; vf < vf_per_mbx_reg; vf++) {
@@ -797,23 +1053,6 @@ static void nic_mbx_intr_handler (struct nicpf *nic, int mbx)
 			nic_clear_mbx_intr(nic, vf, mbx);
 		}
 	}
-}
-
-static irqreturn_t nic_mbx0_intr_handler (int irq, void *nic_irq)
-{
-	struct nicpf *nic = (struct nicpf *)nic_irq;
-
-	nic_mbx_intr_handler(nic, 0);
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t nic_mbx1_intr_handler (int irq, void *nic_irq)
-{
-	struct nicpf *nic = (struct nicpf *)nic_irq;
-
-	nic_mbx_intr_handler(nic, 1);
-
 	return IRQ_HANDLED;
 }
 
@@ -821,7 +1060,13 @@ static int nic_enable_msix(struct nicpf *nic)
 {
 	int i, ret;
 
-	nic->num_vec = NIC_PF_MSIX_VECTORS;
+	nic->num_vec = pci_msix_vec_count(nic->pdev);
+
+	nic->msix_entries = kmalloc_array(nic->num_vec,
+					  sizeof(struct msix_entry),
+					  GFP_KERNEL);
+	if (!nic->msix_entries)
+		return -ENOMEM;
 
 	for (i = 0; i < nic->num_vec; i++)
 		nic->msix_entries[i].entry = i;
@@ -829,8 +1074,9 @@ static int nic_enable_msix(struct nicpf *nic)
 	ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec);
 	if (ret) {
 		dev_err(&nic->pdev->dev,
-			"Request for #%d msix vectors failed\n",
-			   nic->num_vec);
+			"Request for #%d msix vectors failed, returned %d\n",
+			   nic->num_vec, ret);
+		kfree(nic->msix_entries);
 		return ret;
 	}
 
@@ -842,6 +1088,7 @@ static void nic_disable_msix(struct nicpf *nic)
 {
 	if (nic->msix_enabled) {
 		pci_disable_msix(nic->pdev);
+		kfree(nic->msix_entries);
 		nic->msix_enabled = 0;
 		nic->num_vec = 0;
 	}
@@ -860,27 +1107,26 @@ static void nic_free_all_interrupts(struct nicpf *nic)
 
 static int nic_register_interrupts(struct nicpf *nic)
 {
-	int ret;
+	int i, ret;
 
 	/* Enable MSI-X */
 	ret = nic_enable_msix(nic);
 	if (ret)
 		return ret;
 
-	/* Register mailbox interrupt handlers */
-	ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX0].vector,
-			  nic_mbx0_intr_handler, 0, "NIC Mbox0", nic);
-	if (ret)
-		goto fail;
-
-	nic->irq_allocated[NIC_PF_INTR_ID_MBOX0] = true;
+	/* Register mailbox interrupt handler */
+	for (i = NIC_PF_INTR_ID_MBOX0; i < nic->num_vec; i++) {
+		sprintf(nic->irq_name[i],
+			"NICPF Mbox%d", (i - NIC_PF_INTR_ID_MBOX0));
 
-	ret = request_irq(nic->msix_entries[NIC_PF_INTR_ID_MBOX1].vector,
-			  nic_mbx1_intr_handler, 0, "NIC Mbox1", nic);
-	if (ret)
-		goto fail;
+		ret = request_irq(nic->msix_entries[i].vector,
+				  nic_mbx_intr_handler, 0,
+				  nic->irq_name[i], nic);
+		if (ret)
+			goto fail;
 
-	nic->irq_allocated[NIC_PF_INTR_ID_MBOX1] = true;
+		nic->irq_allocated[i] = true;
+	}
 
 	/* Enable mailbox interrupt */
 	nic_enable_mbx_intr(nic);
@@ -889,6 +1135,7 @@ static int nic_register_interrupts(struct nicpf *nic)
 fail:
 	dev_err(&nic->pdev->dev, "Request irq failed\n");
 	nic_free_all_interrupts(nic);
+	nic_disable_msix(nic);
 	return ret;
 }
 
@@ -903,6 +1150,12 @@ static int nic_num_sqs_en(struct nicpf *nic, int vf_en)
 	int pos, sqs_per_vf = MAX_SQS_PER_VF_SINGLE_NODE;
 	u16 total_vf;
 
+	/* Secondary Qsets are needed only if CPU count is
+	 * morethan MAX_QUEUES_PER_QSET.
+	 */
+	if (num_online_cpus() <= MAX_QUEUES_PER_QSET)
+		return 0;
+
 	/* Check if its a multi-node environment */
 	if (nr_node_ids > 1)
 		sqs_per_vf = MAX_SQS_PER_VF;
@@ -1008,6 +1261,12 @@ static int nic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (!nic)
 		return -ENOMEM;
 
+	nic->hw = devm_kzalloc(dev, sizeof(struct hw_info), GFP_KERNEL);
+	if (!nic->hw) {
+		devm_kfree(dev, nic);
+		return -ENOMEM;
+	}
+
 	pci_set_drvdata(pdev, nic);
 
 	nic->pdev = pdev;
@@ -1047,13 +1306,12 @@ static int nic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	nic->node = nic_get_node_id(pdev);
 
-	nic_set_lmac_vf_mapping(nic);
-
 	/* Initialize hardware */
-	nic_init_hw(nic);
+	err = nic_init_hw(nic);
+	if (err)
+		goto err_release_regions;
 
-	/* Set RSS TBL size for each VF */
-	nic->rss_ind_tbl_size = NIC_MAX_RSS_IDR_TBL_SIZE;
+	nic_set_lmac_vf_mapping(nic);
 
 	/* Register interrupts */
 	err = nic_register_interrupts(nic);
@@ -1086,6 +1344,9 @@ err_unregister_interrupts:
 err_release_regions:
 	pci_release_regions(pdev);
 err_disable_device:
+	nic_free_lmacmem(nic);
+	devm_kfree(dev, nic->hw);
+	devm_kfree(dev, nic);
 	pci_disable_device(pdev);
 	pci_set_drvdata(pdev, NULL);
 	return err;
@@ -1106,6 +1367,11 @@ static void nic_remove(struct pci_dev *pdev)
 
 	nic_unregister_interrupts(nic);
 	pci_release_regions(pdev);
+
+	nic_free_lmacmem(nic);
+	devm_kfree(&pdev->dev, nic->hw);
+	devm_kfree(&pdev->dev, nic);
+
 	pci_disable_device(pdev);
 	pci_set_drvdata(pdev, NULL);
 }
diff --git a/drivers/net/ethernet/cavium/thunder/nic_reg.h b/drivers/net/ethernet/cavium/thunder/nic_reg.h
index afb10e326b4f..edf779f5a227 100644
--- a/drivers/net/ethernet/cavium/thunder/nic_reg.h
+++ b/drivers/net/ethernet/cavium/thunder/nic_reg.h
@@ -36,6 +36,20 @@
 #define   NIC_PF_MAILBOX_ENA_W1C		(0x0450)
 #define   NIC_PF_MAILBOX_ENA_W1S		(0x0470)
 #define   NIC_PF_RX_ETYPE_0_7			(0x0500)
+#define   NIC_PF_RX_GENEVE_DEF			(0x0580)
+#define    UDP_GENEVE_PORT_NUM				0x17C1ULL
+#define   NIC_PF_RX_GENEVE_PROT_DEF		(0x0588)
+#define    IPV6_PROT					0x86DDULL
+#define    IPV4_PROT					0x800ULL
+#define    ET_PROT					0x6558ULL
+#define   NIC_PF_RX_NVGRE_PROT_DEF		(0x0598)
+#define   NIC_PF_RX_VXLAN_DEF_0_1		(0x05A0)
+#define    UDP_VXLAN_PORT_NUM				0x12B5
+#define   NIC_PF_RX_VXLAN_PROT_DEF		(0x05B0)
+#define    IPV6_PROT_DEF				0x2ULL
+#define    IPV4_PROT_DEF				0x1ULL
+#define    ET_PROT_DEF					0x3ULL
+#define   NIC_PF_RX_CFG				(0x05D0)
 #define   NIC_PF_PKIND_0_15_CFG			(0x0600)
 #define   NIC_PF_ECC0_FLIP0			(0x1000)
 #define   NIC_PF_ECC1_FLIP0			(0x1008)
@@ -103,6 +117,7 @@
 #define   NIC_PF_SW_SYNC_RX_DONE		(0x490008)
 #define   NIC_PF_TL2_0_63_CFG			(0x500000)
 #define   NIC_PF_TL2_0_63_PRI			(0x520000)
+#define   NIC_PF_TL2_LMAC			(0x540000)
 #define   NIC_PF_TL2_0_63_SH_STATUS		(0x580000)
 #define   NIC_PF_TL3A_0_63_CFG			(0x5F0000)
 #define   NIC_PF_TL3_0_255_CFG			(0x600000)
@@ -170,7 +185,6 @@
 #define   NIC_QSET_SQ_0_7_DOOR			(0x010838)
 #define   NIC_QSET_SQ_0_7_STATUS		(0x010840)
 #define   NIC_QSET_SQ_0_7_DEBUG			(0x010848)
-#define   NIC_QSET_SQ_0_7_CNM_CHG		(0x010860)
 #define   NIC_QSET_SQ_0_7_STAT_0_1		(0x010900)
 
 #define   NIC_QSET_RBDR_0_1_CFG			(0x010C00)
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
index d2d8ef270142..ad4fddb55421 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
@@ -382,7 +382,10 @@ static void nicvf_get_regs(struct net_device *dev,
 		p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_DOOR, q);
 		p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_STATUS, q);
 		p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_DEBUG, q);
-		p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CNM_CHG, q);
+		/* Padding, was NIC_QSET_SQ_0_7_CNM_CHG, which
+		 * produces bus errors when read
+		 */
+		p[i++] = 0;
 		p[i++] = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1, q);
 		reg_offset = NIC_QSET_SQ_0_7_STAT_0_1 | (1 << 3);
 		p[i++] = nicvf_queue_reg_read(nic, reg_offset, q);
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
index a19e73f11d73..45a13f718863 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
@@ -29,10 +29,20 @@
 static const struct pci_device_id nicvf_id_table[] = {
 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
 			 PCI_DEVICE_ID_THUNDER_NIC_VF,
-			 PCI_VENDOR_ID_CAVIUM, 0xA134) },
+			 PCI_VENDOR_ID_CAVIUM,
+			 PCI_SUBSYS_DEVID_88XX_NIC_VF) },
 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
 			 PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF,
-			 PCI_VENDOR_ID_CAVIUM, 0xA11E) },
+			 PCI_VENDOR_ID_CAVIUM,
+			 PCI_SUBSYS_DEVID_88XX_PASS1_NIC_VF) },
+	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
+			 PCI_DEVICE_ID_THUNDER_NIC_VF,
+			 PCI_VENDOR_ID_CAVIUM,
+			 PCI_SUBSYS_DEVID_81XX_NIC_VF) },
+	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
+			 PCI_DEVICE_ID_THUNDER_NIC_VF,
+			 PCI_VENDOR_ID_CAVIUM,
+			 PCI_SUBSYS_DEVID_83XX_NIC_VF) },
 	{ 0, }  /* end of table */
 };
 
@@ -134,15 +144,19 @@ int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
 
 	/* Wait for previous message to be acked, timeout 2sec */
 	while (!nic->pf_acked) {
-		if (nic->pf_nacked)
+		if (nic->pf_nacked) {
+			netdev_err(nic->netdev,
+				   "PF NACK to mbox msg 0x%02x from VF%d\n",
+				   (mbx->msg.msg & 0xFF), nic->vf_id);
 			return -EINVAL;
+		}
 		msleep(sleep);
 		if (nic->pf_acked)
 			break;
 		timeout -= sleep;
 		if (!timeout) {
 			netdev_err(nic->netdev,
-				   "PF didn't ack to mbox msg %d from VF%d\n",
+				   "PF didn't ACK to mbox msg 0x%02x from VF%d\n",
 				   (mbx->msg.msg & 0xFF), nic->vf_id);
 			return -EBUSY;
 		}
@@ -352,13 +366,7 @@ static int nicvf_rss_init(struct nicvf *nic)
 
 	rss->enable = true;
 
-	/* Using the HW reset value for now */
-	rss->key[0] = 0xFEED0BADFEED0BADULL;
-	rss->key[1] = 0xFEED0BADFEED0BADULL;
-	rss->key[2] = 0xFEED0BADFEED0BADULL;
-	rss->key[3] = 0xFEED0BADFEED0BADULL;
-	rss->key[4] = 0xFEED0BADFEED0BADULL;
-
+	netdev_rss_key_fill(rss->key, RSS_HASH_KEY_SIZE * sizeof(u64));
 	nicvf_set_rss_key(nic);
 
 	rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
@@ -507,12 +515,15 @@ static int nicvf_init_resources(struct nicvf *nic)
 
 static void nicvf_snd_pkt_handler(struct net_device *netdev,
 				  struct cmp_queue *cq,
-				  struct cqe_send_t *cqe_tx, int cqe_type)
+				  struct cqe_send_t *cqe_tx,
+				  int cqe_type, int budget,
+				  unsigned int *tx_pkts, unsigned int *tx_bytes)
 {
 	struct sk_buff *skb = NULL;
 	struct nicvf *nic = netdev_priv(netdev);
 	struct snd_queue *sq;
 	struct sq_hdr_subdesc *hdr;
+	struct sq_hdr_subdesc *tso_sqe;
 
 	sq = &nic->qs->sq[cqe_tx->sq_idx];
 
@@ -527,17 +538,23 @@ static void nicvf_snd_pkt_handler(struct net_device *netdev,
 
 	nicvf_check_cqe_tx_errs(nic, cq, cqe_tx);
 	skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr];
-	/* For TSO offloaded packets only one SQE will have a valid SKB */
 	if (skb) {
+		/* Check for dummy descriptor used for HW TSO offload on 88xx */
+		if (hdr->dont_send) {
+			/* Get actual TSO descriptors and free them */
+			tso_sqe =
+			 (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2);
+			nicvf_put_sq_desc(sq, tso_sqe->subdesc_cnt + 1);
+		}
 		nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
 		prefetch(skb);
-		dev_consume_skb_any(skb);
+		(*tx_pkts)++;
+		*tx_bytes += skb->len;
+		napi_consume_skb(skb, budget);
 		sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL;
 	} else {
-		/* In case of HW TSO, HW sends a CQE for each segment of a TSO
-		 * packet instead of a single CQE for the whole TSO packet
-		 * transmitted. Each of this CQE points to the same SQE, so
-		 * avoid freeing same SQE multiple times.
+		/* In case of SW TSO on 88xx, only last segment will have
+		 * a SKB attached, so just free SQEs here.
 		 */
 		if (!nic->hw_tso)
 			nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
@@ -648,6 +665,7 @@ static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
 	struct cmp_queue *cq = &qs->cq[cq_idx];
 	struct cqe_rx_t *cq_desc;
 	struct netdev_queue *txq;
+	unsigned int tx_pkts = 0, tx_bytes = 0;
 
 	spin_lock_bh(&cq->lock);
 loop:
@@ -686,7 +704,8 @@ loop:
 		break;
 		case CQE_TYPE_SEND:
 			nicvf_snd_pkt_handler(netdev, cq,
-					      (void *)cq_desc, CQE_TYPE_SEND);
+					      (void *)cq_desc, CQE_TYPE_SEND,
+					      budget, &tx_pkts, &tx_bytes);
 			tx_done++;
 		break;
 		case CQE_TYPE_INVALID:
@@ -715,6 +734,9 @@ done:
 		netdev = nic->pnicvf->netdev;
 		txq = netdev_get_tx_queue(netdev,
 					  nicvf_netdev_qidx(nic, cq_idx));
+		if (tx_pkts)
+			netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
+
 		nic = nic->pnicvf;
 		if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) {
 			netif_tx_start_queue(txq);
@@ -928,16 +950,19 @@ static int nicvf_register_interrupts(struct nicvf *nic)
 	int vector;
 
 	for_each_cq_irq(irq)
-		sprintf(nic->irq_name[irq], "NICVF%d CQ%d",
-			nic->vf_id, irq);
+		sprintf(nic->irq_name[irq], "%s-rxtx-%d",
+			nic->pnicvf->netdev->name,
+			nicvf_netdev_qidx(nic, irq));
 
 	for_each_sq_irq(irq)
-		sprintf(nic->irq_name[irq], "NICVF%d SQ%d",
-			nic->vf_id, irq - NICVF_INTR_ID_SQ);
+		sprintf(nic->irq_name[irq], "%s-sq-%d",
+			nic->pnicvf->netdev->name,
+			nicvf_netdev_qidx(nic, irq - NICVF_INTR_ID_SQ));
 
 	for_each_rbdr_irq(irq)
-		sprintf(nic->irq_name[irq], "NICVF%d RBDR%d",
-			nic->vf_id, irq - NICVF_INTR_ID_RBDR);
+		sprintf(nic->irq_name[irq], "%s-rbdr-%d",
+			nic->pnicvf->netdev->name,
+			nic->sqs_mode ? (nic->sqs_id + 1) : 0);
 
 	/* Register CQ interrupts */
 	for (irq = 0; irq < nic->qs->cq_cnt; irq++) {
@@ -961,8 +986,9 @@ static int nicvf_register_interrupts(struct nicvf *nic)
 	}
 
 	/* Register QS error interrupt */
-	sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR],
-		"NICVF%d Qset error", nic->vf_id);
+	sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR], "%s-qset-err-%d",
+		nic->pnicvf->netdev->name,
+		nic->sqs_mode ? (nic->sqs_id + 1) : 0);
 	irq = NICVF_INTR_ID_QS_ERR;
 	ret = request_irq(nic->msix_entries[irq].vector,
 			  nicvf_qs_err_intr_handler,
@@ -1141,6 +1167,9 @@ int nicvf_stop(struct net_device *netdev)
 
 	netif_tx_disable(netdev);
 
+	for (qidx = 0; qidx < netdev->num_tx_queues; qidx++)
+		netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx));
+
 	/* Free resources */
 	nicvf_config_data_transfer(nic, false);
 
@@ -1191,7 +1220,7 @@ int nicvf_open(struct net_device *netdev)
 	}
 
 	/* Check if we got MAC address from PF or else generate a radom MAC */
-	if (is_zero_ether_addr(netdev->dev_addr)) {
+	if (!nic->sqs_mode && is_zero_ether_addr(netdev->dev_addr)) {
 		eth_hw_addr_random(netdev);
 		nicvf_hw_set_mac_addr(nic, netdev);
 	}
@@ -1502,6 +1531,7 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct net_device *netdev;
 	struct nicvf *nic;
 	int    err, qcount;
+	u16    sdevid;
 
 	err = pci_enable_device(pdev);
 	if (err) {
@@ -1527,14 +1557,13 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto err_release_regions;
 	}
 
-	qcount = MAX_CMP_QUEUES_PER_QS;
+	qcount = netif_get_num_default_rss_queues();
 
 	/* Restrict multiqset support only for host bound VFs */
 	if (pdev->is_virtfn) {
 		/* Set max number of queues per VF */
-		qcount = roundup(num_online_cpus(), MAX_CMP_QUEUES_PER_QS);
-		qcount = min(qcount,
-			     (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS);
+		qcount = min_t(int, num_online_cpus(),
+			       (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS);
 	}
 
 	netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount);
@@ -1575,6 +1604,10 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (!pass1_silicon(nic->pdev))
 		nic->hw_tso = true;
 
+	pci_read_config_word(nic->pdev, PCI_SUBSYSTEM_ID, &sdevid);
+	if (sdevid == 0xA134)
+		nic->t88 = true;
+
 	/* Check if this VF is in QS only mode */
 	if (nic->sqs_mode)
 		return 0;
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c
index 0ff8e60deccb..a4fc50155881 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c
@@ -479,6 +479,16 @@ void nicvf_config_vlan_stripping(struct nicvf *nic, netdev_features_t features)
 					      NIC_QSET_RQ_GEN_CFG, 0, rq_cfg);
 }
 
+static void nicvf_reset_rcv_queue_stats(struct nicvf *nic)
+{
+	union nic_mbx mbx = {};
+
+	/* Reset all RXQ's stats */
+	mbx.reset_stat.msg = NIC_MBOX_MSG_RESET_STAT_COUNTER;
+	mbx.reset_stat.rq_stat_mask = 0xFFFF;
+	nicvf_send_msg_to_pf(nic, &mbx);
+}
+
 /* Configures receive queue */
 static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
 				   int qidx, bool enable)
@@ -762,10 +772,10 @@ int nicvf_set_qset_resources(struct nicvf *nic)
 	nic->qs = qs;
 
 	/* Set count of each queue */
-	qs->rbdr_cnt = RBDR_CNT;
-	qs->rq_cnt = RCV_QUEUE_CNT;
-	qs->sq_cnt = SND_QUEUE_CNT;
-	qs->cq_cnt = CMP_QUEUE_CNT;
+	qs->rbdr_cnt = DEFAULT_RBDR_CNT;
+	qs->rq_cnt = min_t(u8, MAX_RCV_QUEUES_PER_QS, num_online_cpus());
+	qs->sq_cnt = min_t(u8, MAX_SND_QUEUES_PER_QS, num_online_cpus());
+	qs->cq_cnt = max_t(u8, qs->rq_cnt, qs->sq_cnt);
 
 	/* Set queue lengths */
 	qs->rbdr_len = RCV_BUF_COUNT;
@@ -812,6 +822,11 @@ int nicvf_config_data_transfer(struct nicvf *nic, bool enable)
 		nicvf_free_resources(nic);
 	}
 
+	/* Reset RXQ's stats.
+	 * SQ's stats will get reset automatically once SQ is reset.
+	 */
+	nicvf_reset_rcv_queue_stats(nic);
+
 	return 0;
 }
 
@@ -938,6 +953,8 @@ static int nicvf_tso_count_subdescs(struct sk_buff *skb)
 	return num_edescs + sh->gso_segs;
 }
 
+#define POST_CQE_DESC_COUNT 2
+
 /* Get the number of SQ descriptors needed to xmit this skb */
 static int nicvf_sq_subdesc_required(struct nicvf *nic, struct sk_buff *skb)
 {
@@ -948,6 +965,10 @@ static int nicvf_sq_subdesc_required(struct nicvf *nic, struct sk_buff *skb)
 		return subdesc_cnt;
 	}
 
+	/* Dummy descriptors to get TSO pkt completion notification */
+	if (nic->t88 && nic->hw_tso && skb_shinfo(skb)->gso_size)
+		subdesc_cnt += POST_CQE_DESC_COUNT;
+
 	if (skb_shinfo(skb)->nr_frags)
 		subdesc_cnt += skb_shinfo(skb)->nr_frags;
 
@@ -965,14 +986,21 @@ nicvf_sq_add_hdr_subdesc(struct nicvf *nic, struct snd_queue *sq, int qentry,
 	struct sq_hdr_subdesc *hdr;
 
 	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
-	sq->skbuff[qentry] = (u64)skb;
-
 	memset(hdr, 0, SND_QUEUE_DESC_SIZE);
 	hdr->subdesc_type = SQ_DESC_TYPE_HEADER;
-	/* Enable notification via CQE after processing SQE */
-	hdr->post_cqe = 1;
-	/* No of subdescriptors following this */
-	hdr->subdesc_cnt = subdesc_cnt;
+
+	if (nic->t88 && nic->hw_tso && skb_shinfo(skb)->gso_size) {
+		/* post_cqe = 0, to avoid HW posting a CQE for every TSO
+		 * segment transmitted on 88xx.
+		 */
+		hdr->subdesc_cnt = subdesc_cnt - POST_CQE_DESC_COUNT;
+	} else {
+		sq->skbuff[qentry] = (u64)skb;
+		/* Enable notification via CQE after processing SQE */
+		hdr->post_cqe = 1;
+		/* No of subdescriptors following this */
+		hdr->subdesc_cnt = subdesc_cnt;
+	}
 	hdr->tot_len = len;
 
 	/* Offload checksum calculation to HW */
@@ -1023,6 +1051,55 @@ static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry,
 	gather->addr = data;
 }
 
+/* Add HDR + IMMEDIATE subdescriptors right after descriptors of a TSO
+ * packet so that a CQE is posted as a notifation for transmission of
+ * TSO packet.
+ */
+static inline void nicvf_sq_add_cqe_subdesc(struct snd_queue *sq, int qentry,
+					    int tso_sqe, struct sk_buff *skb)
+{
+	struct sq_imm_subdesc *imm;
+	struct sq_hdr_subdesc *hdr;
+
+	sq->skbuff[qentry] = (u64)skb;
+
+	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry);
+	memset(hdr, 0, SND_QUEUE_DESC_SIZE);
+	hdr->subdesc_type = SQ_DESC_TYPE_HEADER;
+	/* Enable notification via CQE after processing SQE */
+	hdr->post_cqe = 1;
+	/* There is no packet to transmit here */
+	hdr->dont_send = 1;
+	hdr->subdesc_cnt = POST_CQE_DESC_COUNT - 1;
+	hdr->tot_len = 1;
+	/* Actual TSO header SQE index, needed for cleanup */
+	hdr->rsvd2 = tso_sqe;
+
+	qentry = nicvf_get_nxt_sqentry(sq, qentry);
+	imm = (struct sq_imm_subdesc *)GET_SQ_DESC(sq, qentry);
+	memset(imm, 0, SND_QUEUE_DESC_SIZE);
+	imm->subdesc_type = SQ_DESC_TYPE_IMMEDIATE;
+	imm->len = 1;
+}
+
+static inline void nicvf_sq_doorbell(struct nicvf *nic, struct sk_buff *skb,
+				     int sq_num, int desc_cnt)
+{
+	struct netdev_queue *txq;
+
+	txq = netdev_get_tx_queue(nic->pnicvf->netdev,
+				  skb_get_queue_mapping(skb));
+
+	netdev_tx_sent_queue(txq, skb->len);
+
+	/* make sure all memory stores are done before ringing doorbell */
+	smp_wmb();
+
+	/* Inform HW to xmit all TSO segments */
+	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR,
+			      sq_num, desc_cnt);
+}
+
 /* Segment a TSO packet into 'gso_size' segments and append
  * them to SQ for transfer
  */
@@ -1082,12 +1159,8 @@ static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq,
 	/* Save SKB in the last segment for freeing */
 	sq->skbuff[hdr_qentry] = (u64)skb;
 
-	/* make sure all memory stores are done before ringing doorbell */
-	smp_wmb();
+	nicvf_sq_doorbell(nic, skb, sq_num, desc_cnt);
 
-	/* Inform HW to xmit all TSO segments */
-	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR,
-			      sq_num, desc_cnt);
 	nic->drv_stats.tx_tso++;
 	return 1;
 }
@@ -1096,7 +1169,7 @@ static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq,
 int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb)
 {
 	int i, size;
-	int subdesc_cnt;
+	int subdesc_cnt, tso_sqe = 0;
 	int sq_num, qentry;
 	struct queue_set *qs;
 	struct snd_queue *sq;
@@ -1131,6 +1204,7 @@ int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb)
 	/* Add SQ header subdesc */
 	nicvf_sq_add_hdr_subdesc(nic, sq, qentry, subdesc_cnt - 1,
 				 skb, skb->len);
+	tso_sqe = qentry;
 
 	/* Add SQ gather subdescs */
 	qentry = nicvf_get_nxt_sqentry(sq, qentry);
@@ -1154,12 +1228,13 @@ int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb)
 	}
 
 doorbell:
-	/* make sure all memory stores are done before ringing doorbell */
-	smp_wmb();
+	if (nic->t88 && skb_shinfo(skb)->gso_size) {
+		qentry = nicvf_get_nxt_sqentry(sq, qentry);
+		nicvf_sq_add_cqe_subdesc(sq, qentry, tso_sqe, skb);
+	}
+
+	nicvf_sq_doorbell(nic, skb, sq_num, subdesc_cnt);
 
-	/* Inform HW to xmit new packet */
-	nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR,
-			      sq_num, subdesc_cnt);
 	return 1;
 
 append_fail:
@@ -1184,13 +1259,23 @@ struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
 	int frag;
 	int payload_len = 0;
 	struct sk_buff *skb = NULL;
-	struct sk_buff *skb_frag = NULL;
-	struct sk_buff *prev_frag = NULL;
+	struct page *page;
+	int offset;
 	u16 *rb_lens = NULL;
 	u64 *rb_ptrs = NULL;
 
 	rb_lens = (void *)cqe_rx + (3 * sizeof(u64));
-	rb_ptrs = (void *)cqe_rx + (6 * sizeof(u64));
+	/* Except 88xx pass1 on all other chips CQE_RX2_S is added to
+	 * CQE_RX at word6, hence buffer pointers move by word
+	 *
+	 * Use existing 'hw_tso' flag which will be set for all chips
+	 * except 88xx pass1 instead of a additional cache line
+	 * access (or miss) by using pci dev's revision.
+	 */
+	if (!nic->hw_tso)
+		rb_ptrs = (void *)cqe_rx + (6 * sizeof(u64));
+	else
+		rb_ptrs = (void *)cqe_rx + (7 * sizeof(u64));
 
 	netdev_dbg(nic->netdev, "%s rb_cnt %d rb0_ptr %llx rb0_sz %d\n",
 		   __func__, cqe_rx->rb_cnt, cqe_rx->rb0_ptr, cqe_rx->rb0_sz);
@@ -1208,22 +1293,10 @@ struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx)
 			skb_put(skb, payload_len);
 		} else {
 			/* Add fragments */
-			skb_frag = nicvf_rb_ptr_to_skb(nic, *rb_ptrs,
-						       payload_len);
-			if (!skb_frag) {
-				dev_kfree_skb(skb);
-				return NULL;
-			}
-
-			if (!skb_shinfo(skb)->frag_list)
-				skb_shinfo(skb)->frag_list = skb_frag;
-			else
-				prev_frag->next = skb_frag;
-
-			prev_frag = skb_frag;
-			skb->len += payload_len;
-			skb->data_len += payload_len;
-			skb_frag->len = payload_len;
+			page = virt_to_page(phys_to_virt(*rb_ptrs));
+			offset = phys_to_virt(*rb_ptrs) - page_address(page);
+			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
+					offset, payload_len, RCV_FRAG_LEN);
 		}
 		/* Next buffer pointer */
 		rb_ptrs++;
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.h b/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
index 6673e1133523..869f3386028b 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
@@ -57,10 +57,7 @@
 #define CMP_QUEUE_SIZE6		6ULL /* 64K entries */
 
 /* Default queue count per QS, its lengths and threshold values */
-#define RBDR_CNT		1
-#define RCV_QUEUE_CNT		8
-#define SND_QUEUE_CNT		8
-#define CMP_QUEUE_CNT		8 /* Max of RCV and SND qcount */
+#define DEFAULT_RBDR_CNT	1
 
 #define SND_QSIZE		SND_QUEUE_SIZE2
 #define SND_QUEUE_LEN		(1ULL << (SND_QSIZE + 10))
diff --git a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c
index 63a39ac97d53..8bbaedbb7b94 100644
--- a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c
+++ b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c
@@ -28,6 +28,9 @@ struct lmac {
 	struct bgx		*bgx;
 	int			dmac;
 	u8			mac[ETH_ALEN];
+	u8                      lmac_type;
+	u8                      lane_to_sds;
+	bool                    use_training;
 	bool			link_up;
 	int			lmacid; /* ID within BGX */
 	int			lmacid_bd; /* ID on board */
@@ -43,14 +46,13 @@ struct lmac {
 
 struct bgx {
 	u8			bgx_id;
-	u8			qlm_mode;
 	struct	lmac		lmac[MAX_LMAC_PER_BGX];
 	int			lmac_count;
-	int                     lmac_type;
-	int                     lane_to_sds;
-	int			use_training;
+	u8			max_lmac;
 	void __iomem		*reg_base;
 	struct pci_dev		*pdev;
+	bool                    is_dlm;
+	bool                    is_rgx;
 };
 
 static struct bgx *bgx_vnic[MAX_BGX_THUNDER];
@@ -61,6 +63,7 @@ static int bgx_xaui_check_link(struct lmac *lmac);
 /* Supported devices */
 static const struct pci_device_id bgx_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_BGX) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_RGX) },
 	{ 0, }  /* end of table */
 };
 
@@ -124,8 +127,8 @@ unsigned bgx_get_map(int node)
 	int i;
 	unsigned map = 0;
 
-	for (i = 0; i < MAX_BGX_PER_CN88XX; i++) {
-		if (bgx_vnic[(node * MAX_BGX_PER_CN88XX) + i])
+	for (i = 0; i < MAX_BGX_PER_NODE; i++) {
+		if (bgx_vnic[(node * MAX_BGX_PER_NODE) + i])
 			map |= (1 << i);
 	}
 
@@ -138,7 +141,7 @@ int bgx_get_lmac_count(int node, int bgx_idx)
 {
 	struct bgx *bgx;
 
-	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	bgx = bgx_vnic[(node * MAX_BGX_PER_NODE) + bgx_idx];
 	if (bgx)
 		return bgx->lmac_count;
 
@@ -153,7 +156,7 @@ void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status)
 	struct bgx *bgx;
 	struct lmac *lmac;
 
-	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	bgx = bgx_vnic[(node * MAX_BGX_PER_NODE) + bgx_idx];
 	if (!bgx)
 		return;
 
@@ -166,7 +169,7 @@ EXPORT_SYMBOL(bgx_get_lmac_link_state);
 
 const u8 *bgx_get_lmac_mac(int node, int bgx_idx, int lmacid)
 {
-	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_NODE) + bgx_idx];
 
 	if (bgx)
 		return bgx->lmac[lmacid].mac;
@@ -177,7 +180,7 @@ EXPORT_SYMBOL(bgx_get_lmac_mac);
 
 void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const u8 *mac)
 {
-	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_NODE) + bgx_idx];
 
 	if (!bgx)
 		return;
@@ -188,11 +191,13 @@ EXPORT_SYMBOL(bgx_set_lmac_mac);
 
 void bgx_lmac_rx_tx_enable(int node, int bgx_idx, int lmacid, bool enable)
 {
-	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	struct bgx *bgx = bgx_vnic[(node * MAX_BGX_PER_NODE) + bgx_idx];
+	struct lmac *lmac;
 	u64 cfg;
 
 	if (!bgx)
 		return;
+	lmac = &bgx->lmac[lmacid];
 
 	cfg = bgx_reg_read(bgx, lmacid, BGX_CMRX_CFG);
 	if (enable)
@@ -200,6 +205,9 @@ void bgx_lmac_rx_tx_enable(int node, int bgx_idx, int lmacid, bool enable)
 	else
 		cfg &= ~(CMR_PKT_RX_EN | CMR_PKT_TX_EN);
 	bgx_reg_write(bgx, lmacid, BGX_CMRX_CFG, cfg);
+
+	if (bgx->is_rgx)
+		xcv_setup_link(enable ? lmac->link_up : 0, lmac->last_speed);
 }
 EXPORT_SYMBOL(bgx_lmac_rx_tx_enable);
 
@@ -266,9 +274,12 @@ static void bgx_sgmii_change_link_state(struct lmac *lmac)
 
 	port_cfg = bgx_reg_read(bgx, lmac->lmacid, BGX_GMP_GMI_PRTX_CFG);
 
-	/* renable lmac */
+	/* Re-enable lmac */
 	cmr_cfg |= CMR_EN;
 	bgx_reg_write(bgx, lmac->lmacid, BGX_CMRX_CFG, cmr_cfg);
+
+	if (bgx->is_rgx && (cmr_cfg & (CMR_PKT_RX_EN | CMR_PKT_TX_EN)))
+		xcv_setup_link(lmac->link_up, lmac->last_speed);
 }
 
 static void bgx_lmac_handler(struct net_device *netdev)
@@ -314,7 +325,7 @@ u64 bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx)
 {
 	struct bgx *bgx;
 
-	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	bgx = bgx_vnic[(node * MAX_BGX_PER_NODE) + bgx_idx];
 	if (!bgx)
 		return 0;
 
@@ -328,7 +339,7 @@ u64 bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx)
 {
 	struct bgx *bgx;
 
-	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	bgx = bgx_vnic[(node * MAX_BGX_PER_NODE) + bgx_idx];
 	if (!bgx)
 		return 0;
 
@@ -356,7 +367,7 @@ void bgx_lmac_internal_loopback(int node, int bgx_idx,
 	struct lmac *lmac;
 	u64    cfg;
 
-	bgx = bgx_vnic[(node * MAX_BGX_PER_CN88XX) + bgx_idx];
+	bgx = bgx_vnic[(node * MAX_BGX_PER_NODE) + bgx_idx];
 	if (!bgx)
 		return;
 
@@ -379,8 +390,9 @@ void bgx_lmac_internal_loopback(int node, int bgx_idx,
 }
 EXPORT_SYMBOL(bgx_lmac_internal_loopback);
 
-static int bgx_lmac_sgmii_init(struct bgx *bgx, int lmacid)
+static int bgx_lmac_sgmii_init(struct bgx *bgx, struct lmac *lmac)
 {
+	int lmacid = lmac->lmacid;
 	u64 cfg;
 
 	bgx_reg_modify(bgx, lmacid, BGX_GMP_GMI_TXX_THRESH, 0x30);
@@ -409,18 +421,29 @@ static int bgx_lmac_sgmii_init(struct bgx *bgx, int lmacid)
 	cfg |= (PCS_MRX_CTL_RST_AN | PCS_MRX_CTL_AN_EN);
 	bgx_reg_write(bgx, lmacid, BGX_GMP_PCS_MRX_CTL, cfg);
 
-	if (bgx_poll_reg(bgx, lmacid, BGX_GMP_PCS_MRX_STATUS,
-			 PCS_MRX_STATUS_AN_CPT, false)) {
-		dev_err(&bgx->pdev->dev, "BGX AN_CPT not completed\n");
-		return -1;
+	if (lmac->lmac_type == BGX_MODE_QSGMII) {
+		/* Disable disparity check for QSGMII */
+		cfg = bgx_reg_read(bgx, lmacid, BGX_GMP_PCS_MISCX_CTL);
+		cfg &= ~PCS_MISC_CTL_DISP_EN;
+		bgx_reg_write(bgx, lmacid, BGX_GMP_PCS_MISCX_CTL, cfg);
+		return 0;
+	}
+
+	if (lmac->lmac_type == BGX_MODE_SGMII) {
+		if (bgx_poll_reg(bgx, lmacid, BGX_GMP_PCS_MRX_STATUS,
+				 PCS_MRX_STATUS_AN_CPT, false)) {
+			dev_err(&bgx->pdev->dev, "BGX AN_CPT not completed\n");
+			return -1;
+		}
 	}
 
 	return 0;
 }
 
-static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
+static int bgx_lmac_xaui_init(struct bgx *bgx, struct lmac *lmac)
 {
 	u64 cfg;
+	int lmacid = lmac->lmacid;
 
 	/* Reset SPU */
 	bgx_reg_modify(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_RESET);
@@ -436,12 +459,14 @@ static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
 
 	bgx_reg_modify(bgx, lmacid, BGX_SPUX_CONTROL1, SPU_CTL_LOW_POWER);
 	/* Set interleaved running disparity for RXAUI */
-	if (bgx->lmac_type != BGX_MODE_RXAUI)
-		bgx_reg_modify(bgx, lmacid,
-			       BGX_SPUX_MISC_CONTROL, SPU_MISC_CTL_RX_DIS);
-	else
+	if (lmac->lmac_type == BGX_MODE_RXAUI)
 		bgx_reg_modify(bgx, lmacid, BGX_SPUX_MISC_CONTROL,
-			       SPU_MISC_CTL_RX_DIS | SPU_MISC_CTL_INTLV_RDISP);
+			       SPU_MISC_CTL_INTLV_RDISP);
+
+	/* Clear receive packet disable */
+	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_MISC_CONTROL);
+	cfg &= ~SPU_MISC_CTL_RX_DIS;
+	bgx_reg_write(bgx, lmacid, BGX_SPUX_MISC_CONTROL, cfg);
 
 	/* clear all interrupts */
 	cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_RX_INT);
@@ -451,7 +476,7 @@ static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
 	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT);
 	bgx_reg_write(bgx, lmacid, BGX_SPUX_INT, cfg);
 
-	if (bgx->use_training) {
+	if (lmac->use_training) {
 		bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LP_CUP, 0x00);
 		bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LD_CUP, 0x00);
 		bgx_reg_write(bgx, lmacid, BGX_SPUX_BR_PMD_LD_REP, 0x00);
@@ -474,9 +499,9 @@ static int bgx_lmac_xaui_init(struct bgx *bgx, int lmacid, int lmac_type)
 	bgx_reg_write(bgx, lmacid, BGX_SPUX_AN_CONTROL, cfg);
 
 	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_AN_ADV);
-	if (bgx->lmac_type == BGX_MODE_10G_KR)
+	if (lmac->lmac_type == BGX_MODE_10G_KR)
 		cfg |= (1 << 23);
-	else if (bgx->lmac_type == BGX_MODE_40G_KR)
+	else if (lmac->lmac_type == BGX_MODE_40G_KR)
 		cfg |= (1 << 24);
 	else
 		cfg &= ~((1 << 23) | (1 << 24));
@@ -511,11 +536,10 @@ static int bgx_xaui_check_link(struct lmac *lmac)
 {
 	struct bgx *bgx = lmac->bgx;
 	int lmacid = lmac->lmacid;
-	int lmac_type = bgx->lmac_type;
+	int lmac_type = lmac->lmac_type;
 	u64 cfg;
 
-	bgx_reg_modify(bgx, lmacid, BGX_SPUX_MISC_CONTROL, SPU_MISC_CTL_RX_DIS);
-	if (bgx->use_training) {
+	if (lmac->use_training) {
 		cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT);
 		if (!(cfg & (1ull << 13))) {
 			cfg = (1ull << 13) | (1ull << 14);
@@ -556,7 +580,7 @@ static int bgx_xaui_check_link(struct lmac *lmac)
 			       BGX_SPUX_STATUS2, SPU_STATUS2_RCVFLT);
 	if (bgx_reg_read(bgx, lmacid, BGX_SPUX_STATUS2) & SPU_STATUS2_RCVFLT) {
 		dev_err(&bgx->pdev->dev, "Receive fault, retry training\n");
-		if (bgx->use_training) {
+		if (lmac->use_training) {
 			cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_INT);
 			if (!(cfg & (1ull << 13))) {
 				cfg = (1ull << 13) | (1ull << 14);
@@ -584,11 +608,6 @@ static int bgx_xaui_check_link(struct lmac *lmac)
 		return -1;
 	}
 
-	/* Clear receive packet disable */
-	cfg = bgx_reg_read(bgx, lmacid, BGX_SPUX_MISC_CONTROL);
-	cfg &= ~SPU_MISC_CTL_RX_DIS;
-	bgx_reg_write(bgx, lmacid, BGX_SPUX_MISC_CONTROL, cfg);
-
 	/* Check for MAC RX faults */
 	cfg = bgx_reg_read(bgx, lmacid, BGX_SMUX_RX_CTL);
 	/* 0 - Link is okay, 1 - Local fault, 2 - Remote fault */
@@ -599,7 +618,7 @@ static int bgx_xaui_check_link(struct lmac *lmac)
 	/* Rx local/remote fault seen.
 	 * Do lmac reinit to see if condition recovers
 	 */
-	bgx_lmac_xaui_init(bgx, lmacid, bgx->lmac_type);
+	bgx_lmac_xaui_init(bgx, lmac);
 
 	return -1;
 }
@@ -623,7 +642,7 @@ static void bgx_poll_for_link(struct work_struct *work)
 	if ((spu_link & SPU_STATUS1_RCV_LNK) &&
 	    !(smu_link & SMU_RX_CTL_STATUS)) {
 		lmac->link_up = 1;
-		if (lmac->bgx->lmac_type == BGX_MODE_XLAUI)
+		if (lmac->lmac_type == BGX_MODE_XLAUI)
 			lmac->last_speed = 40000;
 		else
 			lmac->last_speed = 10000;
@@ -649,6 +668,16 @@ static void bgx_poll_for_link(struct work_struct *work)
 	queue_delayed_work(lmac->check_link, &lmac->dwork, HZ * 2);
 }
 
+static int phy_interface_mode(u8 lmac_type)
+{
+	if (lmac_type == BGX_MODE_QSGMII)
+		return PHY_INTERFACE_MODE_QSGMII;
+	if (lmac_type == BGX_MODE_RGMII)
+		return PHY_INTERFACE_MODE_RGMII;
+
+	return PHY_INTERFACE_MODE_SGMII;
+}
+
 static int bgx_lmac_enable(struct bgx *bgx, u8 lmacid)
 {
 	struct lmac *lmac;
@@ -657,13 +686,15 @@ static int bgx_lmac_enable(struct bgx *bgx, u8 lmacid)
 	lmac = &bgx->lmac[lmacid];
 	lmac->bgx = bgx;
 
-	if (bgx->lmac_type == BGX_MODE_SGMII) {
+	if ((lmac->lmac_type == BGX_MODE_SGMII) ||
+	    (lmac->lmac_type == BGX_MODE_QSGMII) ||
+	    (lmac->lmac_type == BGX_MODE_RGMII)) {
 		lmac->is_sgmii = 1;
-		if (bgx_lmac_sgmii_init(bgx, lmacid))
+		if (bgx_lmac_sgmii_init(bgx, lmac))
 			return -1;
 	} else {
 		lmac->is_sgmii = 0;
-		if (bgx_lmac_xaui_init(bgx, lmacid, bgx->lmac_type))
+		if (bgx_lmac_xaui_init(bgx, lmac))
 			return -1;
 	}
 
@@ -685,10 +716,10 @@ static int bgx_lmac_enable(struct bgx *bgx, u8 lmacid)
 	/* Restore default cfg, incase low level firmware changed it */
 	bgx_reg_write(bgx, lmacid, BGX_CMRX_RX_DMAC_CTL, 0x03);
 
-	if ((bgx->lmac_type != BGX_MODE_XFI) &&
-	    (bgx->lmac_type != BGX_MODE_XLAUI) &&
-	    (bgx->lmac_type != BGX_MODE_40G_KR) &&
-	    (bgx->lmac_type != BGX_MODE_10G_KR)) {
+	if ((lmac->lmac_type != BGX_MODE_XFI) &&
+	    (lmac->lmac_type != BGX_MODE_XLAUI) &&
+	    (lmac->lmac_type != BGX_MODE_40G_KR) &&
+	    (lmac->lmac_type != BGX_MODE_10G_KR)) {
 		if (!lmac->phydev)
 			return -ENODEV;
 
@@ -696,7 +727,7 @@ static int bgx_lmac_enable(struct bgx *bgx, u8 lmacid)
 
 		if (phy_connect_direct(&lmac->netdev, lmac->phydev,
 				       bgx_lmac_handler,
-				       PHY_INTERFACE_MODE_SGMII))
+				       phy_interface_mode(lmac->lmac_type)))
 			return -ENODEV;
 
 		phy_start_aneg(lmac->phydev);
@@ -753,76 +784,19 @@ static void bgx_lmac_disable(struct bgx *bgx, u8 lmacid)
 
 	bgx_flush_dmac_addrs(bgx, lmacid);
 
-	if ((bgx->lmac_type != BGX_MODE_XFI) &&
-	    (bgx->lmac_type != BGX_MODE_XLAUI) &&
-	    (bgx->lmac_type != BGX_MODE_40G_KR) &&
-	    (bgx->lmac_type != BGX_MODE_10G_KR) && lmac->phydev)
+	if ((lmac->lmac_type != BGX_MODE_XFI) &&
+	    (lmac->lmac_type != BGX_MODE_XLAUI) &&
+	    (lmac->lmac_type != BGX_MODE_40G_KR) &&
+	    (lmac->lmac_type != BGX_MODE_10G_KR) && lmac->phydev)
 		phy_disconnect(lmac->phydev);
 
 	lmac->phydev = NULL;
 }
 
-static void bgx_set_num_ports(struct bgx *bgx)
-{
-	u64 lmac_count;
-
-	switch (bgx->qlm_mode) {
-	case QLM_MODE_SGMII:
-		bgx->lmac_count = 4;
-		bgx->lmac_type = BGX_MODE_SGMII;
-		bgx->lane_to_sds = 0;
-		break;
-	case QLM_MODE_XAUI_1X4:
-		bgx->lmac_count = 1;
-		bgx->lmac_type = BGX_MODE_XAUI;
-		bgx->lane_to_sds = 0xE4;
-			break;
-	case QLM_MODE_RXAUI_2X2:
-		bgx->lmac_count = 2;
-		bgx->lmac_type = BGX_MODE_RXAUI;
-		bgx->lane_to_sds = 0xE4;
-			break;
-	case QLM_MODE_XFI_4X1:
-		bgx->lmac_count = 4;
-		bgx->lmac_type = BGX_MODE_XFI;
-		bgx->lane_to_sds = 0;
-		break;
-	case QLM_MODE_XLAUI_1X4:
-		bgx->lmac_count = 1;
-		bgx->lmac_type = BGX_MODE_XLAUI;
-		bgx->lane_to_sds = 0xE4;
-		break;
-	case QLM_MODE_10G_KR_4X1:
-		bgx->lmac_count = 4;
-		bgx->lmac_type = BGX_MODE_10G_KR;
-		bgx->lane_to_sds = 0;
-		bgx->use_training = 1;
-		break;
-	case QLM_MODE_40G_KR4_1X4:
-		bgx->lmac_count = 1;
-		bgx->lmac_type = BGX_MODE_40G_KR;
-		bgx->lane_to_sds = 0xE4;
-		bgx->use_training = 1;
-		break;
-	default:
-		bgx->lmac_count = 0;
-		break;
-	}
-
-	/* Check if low level firmware has programmed LMAC count
-	 * based on board type, if yes consider that otherwise
-	 * the default static values
-	 */
-	lmac_count = bgx_reg_read(bgx, 0, BGX_CMR_RX_LMACS) & 0x7;
-	if (lmac_count != 4)
-		bgx->lmac_count = lmac_count;
-}
-
 static void bgx_init_hw(struct bgx *bgx)
 {
 	int i;
-
-	bgx_set_num_ports(bgx);
+	struct lmac *lmac;
 
 	bgx_reg_modify(bgx, 0, BGX_CMR_GLOBAL_CFG, CMR_GLOBAL_CFG_FCS_STRIP);
 	if (bgx_reg_read(bgx, 0, BGX_CMR_BIST_STATUS))
@@ -830,17 +804,9 @@ static void bgx_init_hw(struct bgx *bgx)
 
 	/* Set lmac type and lane2serdes mapping */
 	for (i = 0; i < bgx->lmac_count; i++) {
-		if (bgx->lmac_type == BGX_MODE_RXAUI) {
-			if (i)
-				bgx->lane_to_sds = 0x0e;
-			else
-				bgx->lane_to_sds = 0x04;
-			bgx_reg_write(bgx, i, BGX_CMRX_CFG,
-				      (bgx->lmac_type << 8) | bgx->lane_to_sds);
-			continue;
-		}
+		lmac = &bgx->lmac[i];
 		bgx_reg_write(bgx, i, BGX_CMRX_CFG,
-			      (bgx->lmac_type << 8) | (bgx->lane_to_sds + i));
+			      (lmac->lmac_type << 8) | lmac->lane_to_sds);
 		bgx->lmac[i].lmacid_bd = lmac_count;
 		lmac_count++;
 	}
@@ -863,55 +829,212 @@ static void bgx_init_hw(struct bgx *bgx)
 		bgx_reg_write(bgx, 0, BGX_CMR_RX_STREERING + (i * 8), 0x00);
 }
 
-static void bgx_get_qlm_mode(struct bgx *bgx)
+static u8 bgx_get_lane2sds_cfg(struct bgx *bgx, struct lmac *lmac)
+{
+	return (u8)(bgx_reg_read(bgx, lmac->lmacid, BGX_CMRX_CFG) & 0xFF);
+}
+
+static void bgx_print_qlm_mode(struct bgx *bgx, u8 lmacid)
 {
 	struct device *dev = &bgx->pdev->dev;
-	int lmac_type;
-	int train_en;
+	struct lmac *lmac;
+	char str[20];
+	u8 dlm;
 
-	/* Read LMAC0 type to figure out QLM mode
-	 * This is configured by low level firmware
-	 */
-	lmac_type = bgx_reg_read(bgx, 0, BGX_CMRX_CFG);
-	lmac_type = (lmac_type >> 8) & 0x07;
+	if (lmacid > bgx->max_lmac)
+		return;
 
-	train_en = bgx_reg_read(bgx, 0, BGX_SPUX_BR_PMD_CRTL) &
-				SPU_PMD_CRTL_TRAIN_EN;
+	lmac = &bgx->lmac[lmacid];
+	dlm = (lmacid / 2) + (bgx->bgx_id * 2);
+	if (!bgx->is_dlm)
+		sprintf(str, "BGX%d QLM mode", bgx->bgx_id);
+	else
+		sprintf(str, "BGX%d DLM%d mode", bgx->bgx_id, dlm);
 
-	switch (lmac_type) {
+	switch (lmac->lmac_type) {
 	case BGX_MODE_SGMII:
-		bgx->qlm_mode = QLM_MODE_SGMII;
-		dev_info(dev, "BGX%d QLM mode: SGMII\n", bgx->bgx_id);
+		dev_info(dev, "%s: SGMII\n", (char *)str);
 		break;
 	case BGX_MODE_XAUI:
-		bgx->qlm_mode = QLM_MODE_XAUI_1X4;
-		dev_info(dev, "BGX%d QLM mode: XAUI\n", bgx->bgx_id);
+		dev_info(dev, "%s: XAUI\n", (char *)str);
 		break;
 	case BGX_MODE_RXAUI:
-		bgx->qlm_mode = QLM_MODE_RXAUI_2X2;
-		dev_info(dev, "BGX%d QLM mode: RXAUI\n", bgx->bgx_id);
+		dev_info(dev, "%s: RXAUI\n", (char *)str);
 		break;
 	case BGX_MODE_XFI:
-		if (!train_en) {
-			bgx->qlm_mode = QLM_MODE_XFI_4X1;
-			dev_info(dev, "BGX%d QLM mode: XFI\n", bgx->bgx_id);
-		} else {
-			bgx->qlm_mode = QLM_MODE_10G_KR_4X1;
-			dev_info(dev, "BGX%d QLM mode: 10G_KR\n", bgx->bgx_id);
-		}
+		if (!lmac->use_training)
+			dev_info(dev, "%s: XFI\n", (char *)str);
+		else
+			dev_info(dev, "%s: 10G_KR\n", (char *)str);
 		break;
 	case BGX_MODE_XLAUI:
-		if (!train_en) {
-			bgx->qlm_mode = QLM_MODE_XLAUI_1X4;
-			dev_info(dev, "BGX%d QLM mode: XLAUI\n", bgx->bgx_id);
-		} else {
-			bgx->qlm_mode = QLM_MODE_40G_KR4_1X4;
-			dev_info(dev, "BGX%d QLM mode: 40G_KR4\n", bgx->bgx_id);
-		}
+		if (!lmac->use_training)
+			dev_info(dev, "%s: XLAUI\n", (char *)str);
+		else
+			dev_info(dev, "%s: 40G_KR4\n", (char *)str);
+		break;
+	case BGX_MODE_QSGMII:
+		if ((lmacid == 0) &&
+		    (bgx_get_lane2sds_cfg(bgx, lmac) != lmacid))
+			return;
+		if ((lmacid == 2) &&
+		    (bgx_get_lane2sds_cfg(bgx, lmac) == lmacid))
+			return;
+		dev_info(dev, "%s: QSGMII\n", (char *)str);
+		break;
+	case BGX_MODE_RGMII:
+		dev_info(dev, "%s: RGMII\n", (char *)str);
+		break;
+	case BGX_MODE_INVALID:
+		/* Nothing to do */
+		break;
+	}
+}
+
+static void lmac_set_lane2sds(struct bgx *bgx, struct lmac *lmac)
+{
+	switch (lmac->lmac_type) {
+	case BGX_MODE_SGMII:
+	case BGX_MODE_XFI:
+		lmac->lane_to_sds = lmac->lmacid;
+		break;
+	case BGX_MODE_XAUI:
+	case BGX_MODE_XLAUI:
+	case BGX_MODE_RGMII:
+		lmac->lane_to_sds = 0xE4;
+		break;
+	case BGX_MODE_RXAUI:
+		lmac->lane_to_sds = (lmac->lmacid) ? 0xE : 0x4;
+		break;
+	case BGX_MODE_QSGMII:
+		/* There is no way to determine if DLM0/2 is QSGMII or
+		 * DLM1/3 is configured to QSGMII as bootloader will
+		 * configure all LMACs, so take whatever is configured
+		 * by low level firmware.
+		 */
+		lmac->lane_to_sds = bgx_get_lane2sds_cfg(bgx, lmac);
 		break;
 	default:
-		bgx->qlm_mode = QLM_MODE_SGMII;
-		dev_info(dev, "BGX%d QLM default mode: SGMII\n", bgx->bgx_id);
+		lmac->lane_to_sds = 0;
+		break;
+	}
+}
+
+static void lmac_set_training(struct bgx *bgx, struct lmac *lmac, int lmacid)
+{
+	if ((lmac->lmac_type != BGX_MODE_10G_KR) &&
+	    (lmac->lmac_type != BGX_MODE_40G_KR)) {
+		lmac->use_training = 0;
+		return;
+	}
+
+	lmac->use_training = bgx_reg_read(bgx, lmacid, BGX_SPUX_BR_PMD_CRTL) &
+							SPU_PMD_CRTL_TRAIN_EN;
+}
+
+static void bgx_set_lmac_config(struct bgx *bgx, u8 idx)
+{
+	struct lmac *lmac;
+	struct lmac *olmac;
+	u64 cmr_cfg;
+	u8 lmac_type;
+	u8 lane_to_sds;
+
+	lmac = &bgx->lmac[idx];
+
+	if (!bgx->is_dlm || bgx->is_rgx) {
+		/* Read LMAC0 type to figure out QLM mode
+		 * This is configured by low level firmware
+		 */
+		cmr_cfg = bgx_reg_read(bgx, 0, BGX_CMRX_CFG);
+		lmac->lmac_type = (cmr_cfg >> 8) & 0x07;
+		if (bgx->is_rgx)
+			lmac->lmac_type = BGX_MODE_RGMII;
+		lmac_set_training(bgx, lmac, 0);
+		lmac_set_lane2sds(bgx, lmac);
+		return;
+	}
+
+	/* On 81xx BGX can be split across 2 DLMs
+	 * firmware programs lmac_type of LMAC0 and LMAC2
+	 */
+	if ((idx == 0) || (idx == 2)) {
+		cmr_cfg = bgx_reg_read(bgx, idx, BGX_CMRX_CFG);
+		lmac_type = (u8)((cmr_cfg >> 8) & 0x07);
+		lane_to_sds = (u8)(cmr_cfg & 0xFF);
+		/* Check if config is not reset value */
+		if ((lmac_type == 0) && (lane_to_sds == 0xE4))
+			lmac->lmac_type = BGX_MODE_INVALID;
+		else
+			lmac->lmac_type = lmac_type;
+		lmac_set_training(bgx, lmac, lmac->lmacid);
+		lmac_set_lane2sds(bgx, lmac);
+
+		/* Set LMAC type of other lmac on same DLM i.e LMAC 1/3 */
+		olmac = &bgx->lmac[idx + 1];
+		olmac->lmac_type = lmac->lmac_type;
+		lmac_set_training(bgx, olmac, olmac->lmacid);
+		lmac_set_lane2sds(bgx, olmac);
+	}
+}
+
+static bool is_dlm0_in_bgx_mode(struct bgx *bgx)
+{
+	struct lmac *lmac;
+
+	if (!bgx->is_dlm)
+		return true;
+
+	lmac = &bgx->lmac[0];
+	if (lmac->lmac_type == BGX_MODE_INVALID)
+		return false;
+
+	return true;
+}
+
+static void bgx_get_qlm_mode(struct bgx *bgx)
+{
+	struct lmac *lmac;
+	struct lmac *lmac01;
+	struct lmac *lmac23;
+	u8  idx;
+
+	/* Init all LMAC's type to invalid */
+	for (idx = 0; idx < bgx->max_lmac; idx++) {
+		lmac = &bgx->lmac[idx];
+		lmac->lmacid = idx;
+		lmac->lmac_type = BGX_MODE_INVALID;
+		lmac->use_training = false;
+	}
+
+	/* It is assumed that low level firmware sets this value */
+	bgx->lmac_count = bgx_reg_read(bgx, 0, BGX_CMR_RX_LMACS) & 0x7;
+	if (bgx->lmac_count > bgx->max_lmac)
+		bgx->lmac_count = bgx->max_lmac;
+
+	for (idx = 0; idx < bgx->max_lmac; idx++)
+		bgx_set_lmac_config(bgx, idx);
+
+	if (!bgx->is_dlm || bgx->is_rgx) {
+		bgx_print_qlm_mode(bgx, 0);
+		return;
+	}
+
+	if (bgx->lmac_count) {
+		bgx_print_qlm_mode(bgx, 0);
+		bgx_print_qlm_mode(bgx, 2);
+	}
+
+	/* If DLM0 is not in BGX mode then LMAC0/1 have
+	 * to be configured with serdes lanes of DLM1
+	 */
+	if (is_dlm0_in_bgx_mode(bgx) || (bgx->lmac_count > 2))
+		return;
+	for (idx = 0; idx < bgx->lmac_count; idx++) {
+		lmac01 = &bgx->lmac[idx];
+		lmac23 = &bgx->lmac[idx + 2];
+		lmac01->lmac_type = lmac23->lmac_type;
+		lmac01->lane_to_sds = lmac23->lane_to_sds;
 	}
 }
 
@@ -1042,7 +1165,7 @@ static int bgx_init_of_phy(struct bgx *bgx)
 		}
 
 		lmac++;
-		if (lmac == MAX_LMAC_PER_BGX) {
+		if (lmac == bgx->max_lmac) {
 			of_node_put(node);
 			break;
 		}
@@ -1087,6 +1210,7 @@ static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct device *dev = &pdev->dev;
 	struct bgx *bgx = NULL;
 	u8 lmac;
+	u16 sdevid;
 
 	bgx = devm_kzalloc(dev, sizeof(*bgx), GFP_KERNEL);
 	if (!bgx)
@@ -1115,10 +1239,30 @@ static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		err = -ENOMEM;
 		goto err_release_regions;
 	}
-	bgx->bgx_id = (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24) & 1;
-	bgx->bgx_id += nic_get_node_id(pdev) * MAX_BGX_PER_CN88XX;
 
-	bgx_vnic[bgx->bgx_id] = bgx;
+	pci_read_config_word(pdev, PCI_DEVICE_ID, &sdevid);
+	if (sdevid != PCI_DEVICE_ID_THUNDER_RGX) {
+		bgx->bgx_id =
+		    (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24) & 1;
+		bgx->bgx_id += nic_get_node_id(pdev) * MAX_BGX_PER_NODE;
+		bgx->max_lmac = MAX_LMAC_PER_BGX;
+		bgx_vnic[bgx->bgx_id] = bgx;
+	} else {
+		bgx->is_rgx = true;
+		bgx->max_lmac = 1;
+		bgx->bgx_id = MAX_BGX_PER_CN81XX - 1;
+		bgx_vnic[bgx->bgx_id] = bgx;
+		xcv_init_hw();
+	}
+
+	/* On 81xx all are DLMs and on 83xx there are 3 BGX QLMs and one
+	 * BGX i.e BGX2 can be split across 2 DLMs.
+	 */
+	pci_read_config_word(pdev, PCI_SUBSYSTEM_ID, &sdevid);
+	if ((sdevid == PCI_SUBSYS_DEVID_81XX_BGX) ||
+	    ((sdevid == PCI_SUBSYS_DEVID_83XX_BGX) && (bgx->bgx_id == 2)))
+		bgx->is_dlm = true;
+
 	bgx_get_qlm_mode(bgx);
 
 	err = bgx_init_phy(bgx);
@@ -1133,6 +1277,8 @@ static int bgx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		if (err) {
 			dev_err(dev, "BGX%d failed to enable lmac%d\n",
 				bgx->bgx_id, lmac);
+			while (lmac)
+				bgx_lmac_disable(bgx, --lmac);
 			goto err_enable;
 		}
 	}
diff --git a/drivers/net/ethernet/cavium/thunder/thunder_bgx.h b/drivers/net/ethernet/cavium/thunder/thunder_bgx.h
index 42010d2e5ddf..d59c71e4a000 100644
--- a/drivers/net/ethernet/cavium/thunder/thunder_bgx.h
+++ b/drivers/net/ethernet/cavium/thunder/thunder_bgx.h
@@ -9,8 +9,20 @@
 #ifndef THUNDER_BGX_H
 #define THUNDER_BGX_H
 
-#define    MAX_BGX_THUNDER			8 /* Max 4 nodes, 2 per node */
+/* PCI device ID */
+#define	PCI_DEVICE_ID_THUNDER_BGX		0xA026
+#define	PCI_DEVICE_ID_THUNDER_RGX		0xA054
+
+/* Subsystem device IDs */
+#define PCI_SUBSYS_DEVID_88XX_BGX		0xA126
+#define PCI_SUBSYS_DEVID_81XX_BGX		0xA226
+#define PCI_SUBSYS_DEVID_83XX_BGX		0xA326
+
+#define    MAX_BGX_THUNDER			8 /* Max 2 nodes, 4 per node */
 #define    MAX_BGX_PER_CN88XX			2
+#define    MAX_BGX_PER_CN81XX			3 /* 2 BGXs + 1 RGX */
+#define    MAX_BGX_PER_CN83XX			4
+#define    MAX_BGX_PER_NODE			4
 #define    MAX_LMAC_PER_BGX			4
 #define    MAX_BGX_CHANS_PER_LMAC		16
 #define    MAX_DMAC_PER_LMAC			8
@@ -18,8 +30,6 @@
 
 #define    MAX_DMAC_PER_LMAC_TNS_BYPASS_MODE	2
 
-#define    MAX_LMAC	(MAX_BGX_PER_CN88XX * MAX_LMAC_PER_BGX)
-
 /* Registers */
 #define BGX_CMRX_CFG			0x00
 #define  CMR_PKT_TX_EN				BIT_ULL(13)
@@ -136,6 +146,7 @@
 #define BGX_GMP_PCS_ANX_AN_RESULTS	0x30020
 #define BGX_GMP_PCS_SGM_AN_ADV		0x30068
 #define BGX_GMP_PCS_MISCX_CTL		0x30078
+#define  PCS_MISC_CTL_DISP_EN			BIT_ULL(13)
 #define  PCS_MISC_CTL_GMX_ENO			BIT_ULL(11)
 #define  PCS_MISC_CTL_SAMP_PT_MASK	0x7Full
 #define BGX_GMP_GMI_PRTX_CFG		0x38020
@@ -194,6 +205,9 @@ void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const u8 *mac);
 void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status);
 void bgx_lmac_internal_loopback(int node, int bgx_idx,
 				int lmac_idx, bool enable);
+void xcv_init_hw(void);
+void xcv_setup_link(bool link_up, int link_speed);
+
 u64 bgx_get_rx_stats(int node, int bgx_idx, int lmac, int idx);
 u64 bgx_get_tx_stats(int node, int bgx_idx, int lmac, int idx);
 #define BGX_RX_STATS_COUNT 11
@@ -213,16 +227,9 @@ enum LMAC_TYPE {
 	BGX_MODE_XLAUI = 4, /* 4 lanes, 10.3125 Gbaud */
 	BGX_MODE_10G_KR = 3,/* 1 lane, 10.3125 Gbaud */
 	BGX_MODE_40G_KR = 4,/* 4 lanes, 10.3125 Gbaud */
-};
-
-enum qlm_mode {
-	QLM_MODE_SGMII,         /* SGMII, each lane independent */
-	QLM_MODE_XAUI_1X4,      /* 1 XAUI or DXAUI, 4 lanes */
-	QLM_MODE_RXAUI_2X2,     /* 2 RXAUI, 2 lanes each */
-	QLM_MODE_XFI_4X1,       /* 4 XFI, 1 lane each */
-	QLM_MODE_XLAUI_1X4,     /* 1 XLAUI, 4 lanes each */
-	QLM_MODE_10G_KR_4X1,    /* 4 10GBASE-KR, 1 lane each */
-	QLM_MODE_40G_KR4_1X4,   /* 1 40GBASE-KR4, 4 lanes each */
+	BGX_MODE_RGMII = 5,
+	BGX_MODE_QSGMII = 6,
+	BGX_MODE_INVALID = 7,
 };
 
 #endif /* THUNDER_BGX_H */
diff --git a/drivers/net/ethernet/cavium/thunder/thunder_xcv.c b/drivers/net/ethernet/cavium/thunder/thunder_xcv.c
new file mode 100644
index 000000000000..67befedef709
--- /dev/null
+++ b/drivers/net/ethernet/cavium/thunder/thunder_xcv.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright (C) 2016 Cavium, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+#include <linux/acpi.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/phy.h>
+#include <linux/of.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+
+#include "nic.h"
+#include "thunder_bgx.h"
+
+#define DRV_NAME	"thunder-xcv"
+#define DRV_VERSION	"1.0"
+
+/* Register offsets */
+#define XCV_RESET		0x00
+#define   PORT_EN		BIT_ULL(63)
+#define   CLK_RESET		BIT_ULL(15)
+#define   DLL_RESET		BIT_ULL(11)
+#define   COMP_EN		BIT_ULL(7)
+#define   TX_PKT_RESET		BIT_ULL(3)
+#define   TX_DATA_RESET		BIT_ULL(2)
+#define   RX_PKT_RESET		BIT_ULL(1)
+#define   RX_DATA_RESET		BIT_ULL(0)
+#define XCV_DLL_CTL		0x10
+#define   CLKRX_BYP		BIT_ULL(23)
+#define   CLKTX_BYP		BIT_ULL(15)
+#define XCV_COMP_CTL		0x20
+#define   DRV_BYP		BIT_ULL(63)
+#define XCV_CTL			0x30
+#define XCV_INT			0x40
+#define XCV_INT_W1S		0x48
+#define XCV_INT_ENA_W1C		0x50
+#define XCV_INT_ENA_W1S		0x58
+#define XCV_INBND_STATUS	0x80
+#define XCV_BATCH_CRD_RET	0x100
+
+struct xcv {
+	void __iomem		*reg_base;
+	struct pci_dev		*pdev;
+};
+
+static struct xcv *xcv;
+
+/* Supported devices */
+static const struct pci_device_id xcv_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xA056) },
+	{ 0, }  /* end of table */
+};
+
+MODULE_AUTHOR("Cavium Inc");
+MODULE_DESCRIPTION("Cavium Thunder RGX/XCV Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(DRV_VERSION);
+MODULE_DEVICE_TABLE(pci, xcv_id_table);
+
+void xcv_init_hw(void)
+{
+	u64  cfg;
+
+	/* Take DLL out of reset */
+	cfg = readq_relaxed(xcv->reg_base + XCV_RESET);
+	cfg &= ~DLL_RESET;
+	writeq_relaxed(cfg, xcv->reg_base + XCV_RESET);
+
+	/* Take clock tree out of reset */
+	cfg = readq_relaxed(xcv->reg_base + XCV_RESET);
+	cfg &= ~CLK_RESET;
+	writeq_relaxed(cfg, xcv->reg_base + XCV_RESET);
+	/* Wait for DLL to lock */
+	msleep(1);
+
+	/* Configure DLL - enable or bypass
+	 * TX no bypass, RX bypass
+	 */
+	cfg = readq_relaxed(xcv->reg_base + XCV_DLL_CTL);
+	cfg &= ~0xFF03;
+	cfg |= CLKRX_BYP;
+	writeq_relaxed(cfg, xcv->reg_base + XCV_DLL_CTL);
+
+	/* Enable compensation controller and force the
+	 * write to be visible to HW by readig back.
+	 */
+	cfg = readq_relaxed(xcv->reg_base + XCV_RESET);
+	cfg |= COMP_EN;
+	writeq_relaxed(cfg, xcv->reg_base + XCV_RESET);
+	readq_relaxed(xcv->reg_base + XCV_RESET);
+	/* Wait for compensation state machine to lock */
+	msleep(10);
+
+	/* enable the XCV block */
+	cfg = readq_relaxed(xcv->reg_base + XCV_RESET);
+	cfg |= PORT_EN;
+	writeq_relaxed(cfg, xcv->reg_base + XCV_RESET);
+
+	cfg = readq_relaxed(xcv->reg_base + XCV_RESET);
+	cfg |= CLK_RESET;
+	writeq_relaxed(cfg, xcv->reg_base + XCV_RESET);
+}
+EXPORT_SYMBOL(xcv_init_hw);
+
+void xcv_setup_link(bool link_up, int link_speed)
+{
+	u64  cfg;
+	int speed = 2;
+
+	if (!xcv) {
+		dev_err(&xcv->pdev->dev,
+			"XCV init not done, probe may have failed\n");
+		return;
+	}
+
+	if (link_speed == 100)
+		speed = 1;
+	else if (link_speed == 10)
+		speed = 0;
+
+	if (link_up) {
+		/* set operating speed */
+		cfg = readq_relaxed(xcv->reg_base + XCV_CTL);
+		cfg &= ~0x03;
+		cfg |= speed;
+		writeq_relaxed(cfg, xcv->reg_base + XCV_CTL);
+
+		/* Reset datapaths */
+		cfg = readq_relaxed(xcv->reg_base + XCV_RESET);
+		cfg |= TX_DATA_RESET | RX_DATA_RESET;
+		writeq_relaxed(cfg, xcv->reg_base + XCV_RESET);
+
+		/* Enable the packet flow */
+		cfg = readq_relaxed(xcv->reg_base + XCV_RESET);
+		cfg |= TX_PKT_RESET | RX_PKT_RESET;
+		writeq_relaxed(cfg, xcv->reg_base + XCV_RESET);
+
+		/* Return credits to RGX */
+		writeq_relaxed(0x01, xcv->reg_base + XCV_BATCH_CRD_RET);
+	} else {
+		/* Disable packet flow */
+		cfg = readq_relaxed(xcv->reg_base + XCV_RESET);
+		cfg &= ~(TX_PKT_RESET | RX_PKT_RESET);
+		writeq_relaxed(cfg, xcv->reg_base + XCV_RESET);
+		readq_relaxed(xcv->reg_base + XCV_RESET);
+	}
+}
+EXPORT_SYMBOL(xcv_setup_link);
+
+static int xcv_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	int err;
+	struct device *dev = &pdev->dev;
+
+	xcv = devm_kzalloc(dev, sizeof(struct xcv), GFP_KERNEL);
+	if (!xcv)
+		return -ENOMEM;
+	xcv->pdev = pdev;
+
+	pci_set_drvdata(pdev, xcv);
+
+	err = pci_enable_device(pdev);
+	if (err) {
+		dev_err(dev, "Failed to enable PCI device\n");
+		goto err_kfree;
+	}
+
+	err = pci_request_regions(pdev, DRV_NAME);
+	if (err) {
+		dev_err(dev, "PCI request regions failed 0x%x\n", err);
+		goto err_disable_device;
+	}
+
+	/* MAP configuration registers */
+	xcv->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
+	if (!xcv->reg_base) {
+		dev_err(dev, "XCV: Cannot map CSR memory space, aborting\n");
+		err = -ENOMEM;
+		goto err_release_regions;
+	}
+
+	return 0;
+
+err_release_regions:
+	pci_release_regions(pdev);
+err_disable_device:
+	pci_disable_device(pdev);
+err_kfree:
+	devm_kfree(dev, xcv);
+	xcv = NULL;
+	return err;
+}
+
+static void xcv_remove(struct pci_dev *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	if (xcv) {
+		devm_kfree(dev, xcv);
+		xcv = NULL;
+	}
+
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+}
+
+static struct pci_driver xcv_driver = {
+	.name = DRV_NAME,
+	.id_table = xcv_id_table,
+	.probe = xcv_probe,
+	.remove = xcv_remove,
+};
+
+static int __init xcv_init_module(void)
+{
+	pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
+
+	return pci_register_driver(&xcv_driver);
+}
+
+static void __exit xcv_cleanup_module(void)
+{
+	pci_unregister_driver(&xcv_driver);
+}
+
+module_init(xcv_init_module);
+module_exit(xcv_cleanup_module);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/Makefile b/drivers/net/ethernet/chelsio/cxgb4/Makefile
index ace0ab98d0f1..c6b71f656992 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/Makefile
+++ b/drivers/net/ethernet/chelsio/cxgb4/Makefile
@@ -4,7 +4,7 @@
 
 obj-$(CONFIG_CHELSIO_T4) += cxgb4.o
 
-cxgb4-objs := cxgb4_main.o l2t.o t4_hw.o sge.o clip_tbl.o cxgb4_ethtool.o
+cxgb4-objs := cxgb4_main.o l2t.o t4_hw.o sge.o clip_tbl.o cxgb4_ethtool.o cxgb4_uld.o sched.o cxgb4_filter.o cxgb4_tc_u32.o
 cxgb4-$(CONFIG_CHELSIO_T4_DCB) +=  cxgb4_dcb.o
 cxgb4-$(CONFIG_CHELSIO_T4_FCOE) +=  cxgb4_fcoe.o
 cxgb4-$(CONFIG_DEBUG_FS) += cxgb4_debugfs.o
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index 2e2aa9fec9bb..2125903043fb 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -1,7 +1,7 @@
 /*
  * This file is part of the Chelsio T4 Ethernet driver for Linux.
  *
- * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -53,6 +53,8 @@
 #include "cxgb4_uld.h"
 
 #define CH_WARN(adap, fmt, ...) dev_warn(adap->pdev_dev, fmt, ## __VA_ARGS__)
+extern struct list_head adapter_list;
+extern struct mutex uld_mutex;
 
 enum {
 	MAX_NPORTS	= 4,     /* max # of ports */
@@ -338,14 +340,17 @@ struct adapter_params {
 	enum chip_type chip;               /* chip code */
 	struct arch_specific_params arch;  /* chip specific params */
 	unsigned char offload;
+	unsigned char crypto;		/* HW capability for crypto */
 
 	unsigned char bypass;
 
 	unsigned int ofldq_wr_cred;
 	bool ulptx_memwrite_dsgl;          /* use of T5 DSGL allowed */
 
+	unsigned int nsched_cls;          /* number of traffic classes */
 	unsigned int max_ordird_qp;       /* Max read depth per RDMA QP */
 	unsigned int max_ird_adapter;     /* Max read depth per adapter */
+	bool fr_nsmr_tpte_wr_support;	  /* FW support for FR_NSMR_TPTE_WR */
 };
 
 /* State needed to monitor the forward progress of SGE Ingress DMA activities
@@ -403,7 +408,6 @@ struct fw_info {
 	struct fw_hdr fw_hdr;
 };
 
-
 struct trace_params {
 	u32 data[TRACE_LEN / 4];
 	u32 mask[TRACE_LEN / 4];
@@ -419,8 +423,8 @@ struct link_config {
 	unsigned short supported;        /* link capabilities */
 	unsigned short advertising;      /* advertised capabilities */
 	unsigned short lp_advertising;   /* peer advertised capabilities */
-	unsigned short requested_speed;  /* speed user has requested */
-	unsigned short speed;            /* actual link speed */
+	unsigned int   requested_speed;  /* speed user has requested */
+	unsigned int   speed;            /* actual link speed */
 	unsigned char  requested_fc;     /* flow control user has requested */
 	unsigned char  fc;               /* actual link flow control */
 	unsigned char  autoneg;          /* autonegotiating? */
@@ -434,11 +438,6 @@ enum {
 	MAX_ETH_QSETS = 32,           /* # of Ethernet Tx/Rx queue sets */
 	MAX_OFLD_QSETS = 16,          /* # of offload Tx, iscsi Rx queue sets */
 	MAX_CTRL_QUEUES = NCHAN,      /* # of control Tx queues */
-	MAX_RDMA_QUEUES = NCHAN,      /* # of streaming RDMA Rx queues */
-	MAX_RDMA_CIQS = 32,        /* # of  RDMA concentrator IQs */
-
-	/* # of streaming iSCSIT Rx queues */
-	MAX_ISCSIT_QUEUES = MAX_OFLD_QSETS,
 };
 
 enum {
@@ -455,8 +454,7 @@ enum {
 enum {
 	INGQ_EXTRAS = 2,        /* firmware event queue and */
 				/*   forwarded interrupts */
-	MAX_INGQ = MAX_ETH_QSETS + MAX_OFLD_QSETS + MAX_RDMA_QUEUES +
-		   MAX_RDMA_CIQS + MAX_ISCSIT_QUEUES + INGQ_EXTRAS,
+	MAX_INGQ = MAX_ETH_QSETS + INGQ_EXTRAS,
 };
 
 struct adapter;
@@ -493,6 +491,7 @@ struct port_info {
 #endif /* CONFIG_CHELSIO_T4_FCOE */
 	bool rxtstamp;  /* Enable TS */
 	struct hwtstamp_config tstamp_config;
+	struct sched_table *sched_tbl;
 };
 
 struct dentry;
@@ -510,6 +509,10 @@ enum {                                 /* adapter flags */
 	FW_OFLD_CONN       = (1 << 9),
 };
 
+enum {
+	ULP_CRYPTO_LOOKASIDE = 1 << 0,
+};
+
 struct rx_sw_desc;
 
 struct sge_fl {                     /* SGE free-buffer queue state */
@@ -680,17 +683,24 @@ struct sge_ctrl_txq {               /* state for an SGE control Tx queue */
 	u8 full;                    /* the Tx ring is full */
 } ____cacheline_aligned_in_smp;
 
+struct sge_uld_rxq_info {
+	char name[IFNAMSIZ];	/* name of ULD driver */
+	struct sge_ofld_rxq *uldrxq; /* Rxq's for ULD */
+	u16 *msix_tbl;		/* msix_tbl for uld */
+	u16 *rspq_id;		/* response queue id's of rxq */
+	u16 nrxq;		/* # of ingress uld queues */
+	u16 nciq;		/* # of completion queues */
+	u8 uld;			/* uld type */
+};
+
 struct sge {
 	struct sge_eth_txq ethtxq[MAX_ETH_QSETS];
 	struct sge_ofld_txq ofldtxq[MAX_OFLD_QSETS];
 	struct sge_ctrl_txq ctrlq[MAX_CTRL_QUEUES];
 
 	struct sge_eth_rxq ethrxq[MAX_ETH_QSETS];
-	struct sge_ofld_rxq iscsirxq[MAX_OFLD_QSETS];
-	struct sge_ofld_rxq iscsitrxq[MAX_ISCSIT_QUEUES];
-	struct sge_ofld_rxq rdmarxq[MAX_RDMA_QUEUES];
-	struct sge_ofld_rxq rdmaciq[MAX_RDMA_CIQS];
 	struct sge_rspq fw_evtq ____cacheline_aligned_in_smp;
+	struct sge_uld_rxq_info **uld_rxq_info;
 
 	struct sge_rspq intrq ____cacheline_aligned_in_smp;
 	spinlock_t intrq_lock;
@@ -698,14 +708,8 @@ struct sge {
 	u16 max_ethqsets;           /* # of available Ethernet queue sets */
 	u16 ethqsets;               /* # of active Ethernet queue sets */
 	u16 ethtxq_rover;           /* Tx queue to clean up next */
-	u16 iscsiqsets;              /* # of active iSCSI queue sets */
-	u16 niscsitq;               /* # of available iSCST Rx queues */
-	u16 rdmaqs;                 /* # of available RDMA Rx queues */
-	u16 rdmaciqs;               /* # of available RDMA concentrator IQs */
-	u16 iscsi_rxq[MAX_OFLD_QSETS];
-	u16 iscsit_rxq[MAX_ISCSIT_QUEUES];
-	u16 rdma_rxq[MAX_RDMA_QUEUES];
-	u16 rdma_ciq[MAX_RDMA_CIQS];
+	u16 ofldqsets;              /* # of active ofld queue sets */
+	u16 nqs_per_uld;	    /* # of Rx queues per ULD */
 	u16 timer_val[SGE_NTIMERS];
 	u8 counter_val[SGE_NCOUNTERS];
 	u32 fl_pg_order;            /* large page allocation size */
@@ -729,10 +733,7 @@ struct sge {
 };
 
 #define for_each_ethrxq(sge, i) for (i = 0; i < (sge)->ethqsets; i++)
-#define for_each_iscsirxq(sge, i) for (i = 0; i < (sge)->iscsiqsets; i++)
-#define for_each_iscsitrxq(sge, i) for (i = 0; i < (sge)->niscsitq; i++)
-#define for_each_rdmarxq(sge, i) for (i = 0; i < (sge)->rdmaqs; i++)
-#define for_each_rdmaciq(sge, i) for (i = 0; i < (sge)->rdmaciqs; i++)
+#define for_each_ofldtxq(sge, i) for (i = 0; i < (sge)->ofldqsets; i++)
 
 struct l2t_data;
 
@@ -757,6 +758,23 @@ struct hash_mac_addr {
 	u8 addr[ETH_ALEN];
 };
 
+struct uld_msix_bmap {
+	unsigned long *msix_bmap;
+	unsigned int mapsize;
+	spinlock_t lock; /* lock for acquiring bitmap */
+};
+
+struct uld_msix_info {
+	unsigned short vec;
+	char desc[IFNAMSIZ + 10];
+	unsigned int idx;
+};
+
+struct vf_info {
+	unsigned char vf_mac_addr[ETH_ALEN];
+	bool pf_set_mac;
+};
+
 struct adapter {
 	void __iomem *regs;
 	void __iomem *bar2;
@@ -767,6 +785,7 @@ struct adapter {
 	unsigned int mbox;
 	unsigned int pf;
 	unsigned int flags;
+	unsigned int adap_idx;
 	enum chip_type chip;
 
 	int msg_enable;
@@ -779,6 +798,9 @@ struct adapter {
 		unsigned short vec;
 		char desc[IFNAMSIZ + 10];
 	} msix_info[MAX_INGQ + 1];
+	struct uld_msix_info *msix_info_ulds; /* msix info for uld's */
+	struct uld_msix_bmap msix_bmap_ulds; /* msix bitmap for all uld */
+	int msi_idx;
 
 	struct doorbell_stats db_stats;
 	struct sge sge;
@@ -786,6 +808,9 @@ struct adapter {
 	struct net_device *port[MAX_NPORTS];
 	u8 chan_map[NCHAN];                   /* channel -> port map */
 
+	struct vf_info *vfinfo;
+	u8 num_vfs;
+
 	u32 filter_mode;
 	unsigned int l2t_start;
 	unsigned int l2t_end;
@@ -793,7 +818,10 @@ struct adapter {
 	unsigned int clipt_start;
 	unsigned int clipt_end;
 	struct clip_tbl *clipt;
+	struct cxgb4_uld_info *uld;
 	void *uld_handle[CXGB4_ULD_MAX];
+	unsigned int num_uld;
+	unsigned int num_ofld_uld;
 	struct list_head list_node;
 	struct list_head rcu_node;
 	struct list_head mac_hlist; /* list of MAC addresses in MPS Hash */
@@ -813,6 +841,8 @@ struct adapter {
 #define T4_OS_LOG_MBOX_CMDS 256
 	struct mbox_cmd_log *mbox_log;
 
+	struct mutex uld_mutex;
+
 	struct dentry *debugfs_root;
 	bool use_bd;     /* Use SGE Back Door intfc for reading SGE Contexts */
 	bool trace_rss;	/* 1 implies that different RSS flit per filter is
@@ -822,6 +852,58 @@ struct adapter {
 
 	spinlock_t stats_lock;
 	spinlock_t win0_lock ____cacheline_aligned_in_smp;
+
+	/* TC u32 offload */
+	struct cxgb4_tc_u32_table *tc_u32;
+};
+
+/* Support for "sched-class" command to allow a TX Scheduling Class to be
+ * programmed with various parameters.
+ */
+struct ch_sched_params {
+	s8   type;                     /* packet or flow */
+	union {
+		struct {
+			s8   level;    /* scheduler hierarchy level */
+			s8   mode;     /* per-class or per-flow */
+			s8   rateunit; /* bit or packet rate */
+			s8   ratemode; /* %port relative or kbps absolute */
+			s8   channel;  /* scheduler channel [0..N] */
+			s8   class;    /* scheduler class [0..N] */
+			s32  minrate;  /* minimum rate */
+			s32  maxrate;  /* maximum rate */
+			s16  weight;   /* percent weight */
+			s16  pktsize;  /* average packet size */
+		} params;
+	} u;
+};
+
+enum {
+	SCHED_CLASS_TYPE_PACKET = 0,    /* class type */
+};
+
+enum {
+	SCHED_CLASS_LEVEL_CL_RL = 0,    /* class rate limiter */
+};
+
+enum {
+	SCHED_CLASS_MODE_CLASS = 0,     /* per-class scheduling */
+};
+
+enum {
+	SCHED_CLASS_RATEUNIT_BITS = 0,  /* bit rate scheduling */
+};
+
+enum {
+	SCHED_CLASS_RATEMODE_ABS = 1,   /* Kb/s */
+};
+
+/* Support for "sched_queue" command to allow one or more NIC TX Queues
+ * to be bound to a TX Scheduling Class.
+ */
+struct ch_sched_queue {
+	s8   queue;    /* queue index */
+	s8   class;    /* class index */
 };
 
 /* Defined bit width of user definable filter tuples
@@ -947,11 +1029,47 @@ enum {
 	VLAN_REWRITE
 };
 
+/* Host shadow copy of ingress filter entry.  This is in host native format
+ * and doesn't match the ordering or bit order, etc. of the hardware of the
+ * firmware command.  The use of bit-field structure elements is purely to
+ * remind ourselves of the field size limitations and save memory in the case
+ * where the filter table is large.
+ */
+struct filter_entry {
+	/* Administrative fields for filter. */
+	u32 valid:1;            /* filter allocated and valid */
+	u32 locked:1;           /* filter is administratively locked */
+
+	u32 pending:1;          /* filter action is pending firmware reply */
+	u32 smtidx:8;           /* Source MAC Table index for smac */
+	struct filter_ctx *ctx; /* Caller's completion hook */
+	struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
+	struct net_device *dev; /* Associated net device */
+	u32 tid;                /* This will store the actual tid */
+
+	/* The filter itself.  Most of this is a straight copy of information
+	 * provided by the extended ioctl().  Some fields are translated to
+	 * internal forms -- for instance the Ingress Queue ID passed in from
+	 * the ioctl() is translated into the Absolute Ingress Queue ID.
+	 */
+	struct ch_filter_specification fs;
+};
+
 static inline int is_offload(const struct adapter *adap)
 {
 	return adap->params.offload;
 }
 
+static inline int is_pci_uld(const struct adapter *adap)
+{
+	return adap->params.crypto;
+}
+
+static inline int is_uld(const struct adapter *adap)
+{
+	return (adap->params.offload || adap->params.crypto);
+}
+
 static inline u32 t4_read_reg(struct adapter *adap, u32 reg_addr)
 {
 	return readl(adap->regs + reg_addr);
@@ -1178,6 +1296,8 @@ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
 int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
 			  struct net_device *dev, unsigned int iqid,
 			  unsigned int cmplqid);
+int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid,
+			unsigned int cmplqid);
 int t4_sge_alloc_ofld_txq(struct adapter *adap, struct sge_ofld_txq *txq,
 			  struct net_device *dev, unsigned int iqid);
 irqreturn_t t4_sge_intr_msix(int irq, void *cookie);
@@ -1185,8 +1305,6 @@ int t4_sge_init(struct adapter *adap);
 void t4_sge_start(struct adapter *adap);
 void t4_sge_stop(struct adapter *adap);
 int cxgb_busy_poll(struct napi_struct *napi);
-int cxgb4_set_rspq_intr_params(struct sge_rspq *q, unsigned int us,
-			       unsigned int cnt);
 void cxgb4_set_ethtool_ops(struct net_device *netdev);
 int cxgb4_write_rss(const struct port_info *pi, const u16 *queues);
 extern int dbfifo_int_thresh;
@@ -1289,6 +1407,18 @@ static inline int hash_mac_addr(const u8 *addr)
 	return a & 0x3f;
 }
 
+int cxgb4_set_rspq_intr_params(struct sge_rspq *q, unsigned int us,
+			       unsigned int cnt);
+static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
+			     unsigned int us, unsigned int cnt,
+			     unsigned int size, unsigned int iqe_size)
+{
+	q->adap = adap;
+	cxgb4_set_rspq_intr_params(q, us, cnt);
+	q->iqe_len = iqe_size;
+	q->size = size;
+}
+
 void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
 		       unsigned int data_reg, const u32 *vals,
 		       unsigned int nregs, unsigned int start_idx);
@@ -1514,6 +1644,9 @@ void t4_get_trace_filter(struct adapter *adapter, struct trace_params *tp,
 			 int filter_index, int *enabled);
 int t4_fwaddrspace_write(struct adapter *adap, unsigned int mbox,
 			 u32 addr, u32 val);
+int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
+		    int rateunit, int ratemode, int channel, int class,
+		    int minrate, int maxrate, int weight, int pktsize);
 void t4_sge_decode_idma_state(struct adapter *adapter, int state);
 void t4_free_mem(void *addr);
 void t4_idma_monitor_init(struct adapter *adapter,
@@ -1521,4 +1654,11 @@ void t4_idma_monitor_init(struct adapter *adapter,
 void t4_idma_monitor(struct adapter *adapter,
 		     struct sge_idma_monitor_state *idma,
 		     int hz, int ticks);
+int t4_set_vf_mac_acl(struct adapter *adapter, unsigned int vf,
+		      unsigned int naddr, u8 *addr);
+void t4_uld_mem_free(struct adapter *adap);
+int t4_uld_mem_alloc(struct adapter *adap);
+void t4_uld_clean_up(struct adapter *adap);
+void t4_register_netevent_notifier(void);
+void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, struct sge_fl *fl);
 #endif /* __CXGB4_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index 91fb50850fff..20455d082cb8 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -2432,17 +2432,11 @@ static int sge_qinfo_show(struct seq_file *seq, void *v)
 {
 	struct adapter *adap = seq->private;
 	int eth_entries = DIV_ROUND_UP(adap->sge.ethqsets, 4);
-	int iscsi_entries = DIV_ROUND_UP(adap->sge.iscsiqsets, 4);
-	int iscsit_entries = DIV_ROUND_UP(adap->sge.niscsitq, 4);
-	int rdma_entries = DIV_ROUND_UP(adap->sge.rdmaqs, 4);
-	int ciq_entries = DIV_ROUND_UP(adap->sge.rdmaciqs, 4);
+	int ofld_entries = DIV_ROUND_UP(adap->sge.ofldqsets, 4);
 	int ctrl_entries = DIV_ROUND_UP(MAX_CTRL_QUEUES, 4);
 	int i, r = (uintptr_t)v - 1;
-	int iscsi_idx = r - eth_entries;
-	int iscsit_idx = iscsi_idx - iscsi_entries;
-	int rdma_idx = iscsit_idx - iscsit_entries;
-	int ciq_idx = rdma_idx - rdma_entries;
-	int ctrl_idx =  ciq_idx - ciq_entries;
+	int ofld_idx = r - eth_entries;
+	int ctrl_idx =  ofld_idx - ofld_entries;
 	int fq_idx =  ctrl_idx - ctrl_entries;
 
 	if (r)
@@ -2518,119 +2512,17 @@ do { \
 		RL("FLLow:", fl.low);
 		RL("FLStarving:", fl.starving);
 
-	} else if (iscsi_idx < iscsi_entries) {
-		const struct sge_ofld_rxq *rx =
-			&adap->sge.iscsirxq[iscsi_idx * 4];
+	} else if (ofld_idx < ofld_entries) {
 		const struct sge_ofld_txq *tx =
-			&adap->sge.ofldtxq[iscsi_idx * 4];
-		int n = min(4, adap->sge.iscsiqsets - 4 * iscsi_idx);
+			&adap->sge.ofldtxq[ofld_idx * 4];
+		int n = min(4, adap->sge.ofldqsets - 4 * ofld_idx);
 
-		S("QType:", "iSCSI");
+		S("QType:", "OFLD-Txq");
 		T("TxQ ID:", q.cntxt_id);
 		T("TxQ size:", q.size);
 		T("TxQ inuse:", q.in_use);
 		T("TxQ CIDX:", q.cidx);
 		T("TxQ PIDX:", q.pidx);
-		R("RspQ ID:", rspq.abs_id);
-		R("RspQ size:", rspq.size);
-		R("RspQE size:", rspq.iqe_len);
-		R("RspQ CIDX:", rspq.cidx);
-		R("RspQ Gen:", rspq.gen);
-		S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq));
-		S3("u", "Intr pktcnt:",
-		   adap->sge.counter_val[rx[i].rspq.pktcnt_idx]);
-		R("FL ID:", fl.cntxt_id);
-		R("FL size:", fl.size - 8);
-		R("FL pend:", fl.pend_cred);
-		R("FL avail:", fl.avail);
-		R("FL PIDX:", fl.pidx);
-		R("FL CIDX:", fl.cidx);
-		RL("RxPackets:", stats.pkts);
-		RL("RxImmPkts:", stats.imm);
-		RL("RxNoMem:", stats.nomem);
-		RL("FLAllocErr:", fl.alloc_failed);
-		RL("FLLrgAlcErr:", fl.large_alloc_failed);
-		RL("FLMapErr:", fl.mapping_err);
-		RL("FLLow:", fl.low);
-		RL("FLStarving:", fl.starving);
-
-	} else if (iscsit_idx < iscsit_entries) {
-		const struct sge_ofld_rxq *rx =
-			&adap->sge.iscsitrxq[iscsit_idx * 4];
-		int n = min(4, adap->sge.niscsitq - 4 * iscsit_idx);
-
-		S("QType:", "iSCSIT");
-		R("RspQ ID:", rspq.abs_id);
-		R("RspQ size:", rspq.size);
-		R("RspQE size:", rspq.iqe_len);
-		R("RspQ CIDX:", rspq.cidx);
-		R("RspQ Gen:", rspq.gen);
-		S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq));
-		S3("u", "Intr pktcnt:",
-		   adap->sge.counter_val[rx[i].rspq.pktcnt_idx]);
-		R("FL ID:", fl.cntxt_id);
-		R("FL size:", fl.size - 8);
-		R("FL pend:", fl.pend_cred);
-		R("FL avail:", fl.avail);
-		R("FL PIDX:", fl.pidx);
-		R("FL CIDX:", fl.cidx);
-		RL("RxPackets:", stats.pkts);
-		RL("RxImmPkts:", stats.imm);
-		RL("RxNoMem:", stats.nomem);
-		RL("FLAllocErr:", fl.alloc_failed);
-		RL("FLLrgAlcErr:", fl.large_alloc_failed);
-		RL("FLMapErr:", fl.mapping_err);
-		RL("FLLow:", fl.low);
-		RL("FLStarving:", fl.starving);
-
-	} else if (rdma_idx < rdma_entries) {
-		const struct sge_ofld_rxq *rx =
-				&adap->sge.rdmarxq[rdma_idx * 4];
-		int n = min(4, adap->sge.rdmaqs - 4 * rdma_idx);
-
-		S("QType:", "RDMA-CPL");
-		S("Interface:",
-		  rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A");
-		R("RspQ ID:", rspq.abs_id);
-		R("RspQ size:", rspq.size);
-		R("RspQE size:", rspq.iqe_len);
-		R("RspQ CIDX:", rspq.cidx);
-		R("RspQ Gen:", rspq.gen);
-		S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq));
-		S3("u", "Intr pktcnt:",
-		   adap->sge.counter_val[rx[i].rspq.pktcnt_idx]);
-		R("FL ID:", fl.cntxt_id);
-		R("FL size:", fl.size - 8);
-		R("FL pend:", fl.pend_cred);
-		R("FL avail:", fl.avail);
-		R("FL PIDX:", fl.pidx);
-		R("FL CIDX:", fl.cidx);
-		RL("RxPackets:", stats.pkts);
-		RL("RxImmPkts:", stats.imm);
-		RL("RxNoMem:", stats.nomem);
-		RL("FLAllocErr:", fl.alloc_failed);
-		RL("FLLrgAlcErr:", fl.large_alloc_failed);
-		RL("FLMapErr:", fl.mapping_err);
-		RL("FLLow:", fl.low);
-		RL("FLStarving:", fl.starving);
-
-	} else if (ciq_idx < ciq_entries) {
-		const struct sge_ofld_rxq *rx = &adap->sge.rdmaciq[ciq_idx * 4];
-		int n = min(4, adap->sge.rdmaciqs - 4 * ciq_idx);
-
-		S("QType:", "RDMA-CIQ");
-		S("Interface:",
-		  rx[i].rspq.netdev ? rx[i].rspq.netdev->name : "N/A");
-		R("RspQ ID:", rspq.abs_id);
-		R("RspQ size:", rspq.size);
-		R("RspQE size:", rspq.iqe_len);
-		R("RspQ CIDX:", rspq.cidx);
-		R("RspQ Gen:", rspq.gen);
-		S3("u", "Intr delay:", qtimer_val(adap, &rx[i].rspq));
-		S3("u", "Intr pktcnt:",
-		   adap->sge.counter_val[rx[i].rspq.pktcnt_idx]);
-		RL("RxAN:", stats.an);
-		RL("RxNoMem:", stats.nomem);
 
 	} else if (ctrl_idx < ctrl_entries) {
 		const struct sge_ctrl_txq *tx = &adap->sge.ctrlq[ctrl_idx * 4];
@@ -2672,10 +2564,7 @@ do { \
 static int sge_queue_entries(const struct adapter *adap)
 {
 	return DIV_ROUND_UP(adap->sge.ethqsets, 4) +
-	       DIV_ROUND_UP(adap->sge.iscsiqsets, 4) +
-	       DIV_ROUND_UP(adap->sge.niscsitq, 4) +
-	       DIV_ROUND_UP(adap->sge.rdmaqs, 4) +
-	       DIV_ROUND_UP(adap->sge.rdmaciqs, 4) +
+	       DIV_ROUND_UP(adap->sge.ofldqsets, 4) +
 	       DIV_ROUND_UP(MAX_CTRL_QUEUES, 4) + 1;
 }
 
@@ -2859,12 +2748,6 @@ static void add_debugfs_mem(struct adapter *adap, const char *name,
 				 size_mb << 20);
 }
 
-static int blocked_fl_open(struct inode *inode, struct file *file)
-{
-	file->private_data = inode->i_private;
-	return 0;
-}
-
 static ssize_t blocked_fl_read(struct file *filp, char __user *ubuf,
 			       size_t count, loff_t *ppos)
 {
@@ -2908,7 +2791,7 @@ static ssize_t blocked_fl_write(struct file *filp, const char __user *ubuf,
 
 static const struct file_operations blocked_fl_fops = {
 	.owner   = THIS_MODULE,
-	.open    = blocked_fl_open,
+	.open    = simple_open,
 	.read    = blocked_fl_read,
 	.write   = blocked_fl_write,
 	.llseek  = generic_file_llseek,
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c
new file mode 100644
index 000000000000..10736738ff30
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c
@@ -0,0 +1,721 @@
+/*
+ * This file is part of the Chelsio T4 Ethernet driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "cxgb4.h"
+#include "t4_regs.h"
+#include "l2t.h"
+#include "t4fw_api.h"
+#include "cxgb4_filter.h"
+
+static inline bool is_field_set(u32 val, u32 mask)
+{
+	return val || mask;
+}
+
+static inline bool unsupported(u32 conf, u32 conf_mask, u32 val, u32 mask)
+{
+	return !(conf & conf_mask) && is_field_set(val, mask);
+}
+
+/* Validate filter spec against configuration done on the card. */
+static int validate_filter(struct net_device *dev,
+			   struct ch_filter_specification *fs)
+{
+	struct adapter *adapter = netdev2adap(dev);
+	u32 fconf, iconf;
+
+	/* Check for unconfigured fields being used. */
+	fconf = adapter->params.tp.vlan_pri_map;
+	iconf = adapter->params.tp.ingress_config;
+
+	if (unsupported(fconf, FCOE_F, fs->val.fcoe, fs->mask.fcoe) ||
+	    unsupported(fconf, PORT_F, fs->val.iport, fs->mask.iport) ||
+	    unsupported(fconf, TOS_F, fs->val.tos, fs->mask.tos) ||
+	    unsupported(fconf, ETHERTYPE_F, fs->val.ethtype,
+			fs->mask.ethtype) ||
+	    unsupported(fconf, MACMATCH_F, fs->val.macidx, fs->mask.macidx) ||
+	    unsupported(fconf, MPSHITTYPE_F, fs->val.matchtype,
+			fs->mask.matchtype) ||
+	    unsupported(fconf, FRAGMENTATION_F, fs->val.frag, fs->mask.frag) ||
+	    unsupported(fconf, PROTOCOL_F, fs->val.proto, fs->mask.proto) ||
+	    unsupported(fconf, VNIC_ID_F, fs->val.pfvf_vld,
+			fs->mask.pfvf_vld) ||
+	    unsupported(fconf, VNIC_ID_F, fs->val.ovlan_vld,
+			fs->mask.ovlan_vld) ||
+	    unsupported(fconf, VLAN_F, fs->val.ivlan_vld, fs->mask.ivlan_vld))
+		return -EOPNOTSUPP;
+
+	/* T4 inconveniently uses the same FT_VNIC_ID_W bits for both the Outer
+	 * VLAN Tag and PF/VF/VFvld fields based on VNIC_F being set
+	 * in TP_INGRESS_CONFIG.  Hense the somewhat crazy checks
+	 * below.  Additionally, since the T4 firmware interface also
+	 * carries that overlap, we need to translate any PF/VF
+	 * specification into that internal format below.
+	 */
+	if (is_field_set(fs->val.pfvf_vld, fs->mask.pfvf_vld) &&
+	    is_field_set(fs->val.ovlan_vld, fs->mask.ovlan_vld))
+		return -EOPNOTSUPP;
+	if (unsupported(iconf, VNIC_F, fs->val.pfvf_vld, fs->mask.pfvf_vld) ||
+	    (is_field_set(fs->val.ovlan_vld, fs->mask.ovlan_vld) &&
+	     (iconf & VNIC_F)))
+		return -EOPNOTSUPP;
+	if (fs->val.pf > 0x7 || fs->val.vf > 0x7f)
+		return -ERANGE;
+	fs->mask.pf &= 0x7;
+	fs->mask.vf &= 0x7f;
+
+	/* If the user is requesting that the filter action loop
+	 * matching packets back out one of our ports, make sure that
+	 * the egress port is in range.
+	 */
+	if (fs->action == FILTER_SWITCH &&
+	    fs->eport >= adapter->params.nports)
+		return -ERANGE;
+
+	/* Don't allow various trivially obvious bogus out-of-range values... */
+	if (fs->val.iport >= adapter->params.nports)
+		return -ERANGE;
+
+	/* T4 doesn't support removing VLAN Tags for loop back filters. */
+	if (is_t4(adapter->params.chip) &&
+	    fs->action == FILTER_SWITCH &&
+	    (fs->newvlan == VLAN_REMOVE ||
+	     fs->newvlan == VLAN_REWRITE))
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
+static int get_filter_steerq(struct net_device *dev,
+			     struct ch_filter_specification *fs)
+{
+	struct adapter *adapter = netdev2adap(dev);
+	int iq;
+
+	/* If the user has requested steering matching Ingress Packets
+	 * to a specific Queue Set, we need to make sure it's in range
+	 * for the port and map that into the Absolute Queue ID of the
+	 * Queue Set's Response Queue.
+	 */
+	if (!fs->dirsteer) {
+		if (fs->iq)
+			return -EINVAL;
+		iq = 0;
+	} else {
+		struct port_info *pi = netdev_priv(dev);
+
+		/* If the iq id is greater than the number of qsets,
+		 * then assume it is an absolute qid.
+		 */
+		if (fs->iq < pi->nqsets)
+			iq = adapter->sge.ethrxq[pi->first_qset +
+						 fs->iq].rspq.abs_id;
+		else
+			iq = fs->iq;
+	}
+
+	return iq;
+}
+
+static int cxgb4_set_ftid(struct tid_info *t, int fidx, int family)
+{
+	spin_lock_bh(&t->ftid_lock);
+
+	if (test_bit(fidx, t->ftid_bmap)) {
+		spin_unlock_bh(&t->ftid_lock);
+		return -EBUSY;
+	}
+
+	if (family == PF_INET)
+		__set_bit(fidx, t->ftid_bmap);
+	else
+		bitmap_allocate_region(t->ftid_bmap, fidx, 2);
+
+	spin_unlock_bh(&t->ftid_lock);
+	return 0;
+}
+
+static void cxgb4_clear_ftid(struct tid_info *t, int fidx, int family)
+{
+	spin_lock_bh(&t->ftid_lock);
+	if (family == PF_INET)
+		__clear_bit(fidx, t->ftid_bmap);
+	else
+		bitmap_release_region(t->ftid_bmap, fidx, 2);
+	spin_unlock_bh(&t->ftid_lock);
+}
+
+/* Delete the filter at a specified index. */
+static int del_filter_wr(struct adapter *adapter, int fidx)
+{
+	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
+	struct fw_filter_wr *fwr;
+	struct sk_buff *skb;
+	unsigned int len;
+
+	len = sizeof(*fwr);
+
+	skb = alloc_skb(len, GFP_KERNEL);
+	if (!skb)
+		return -ENOMEM;
+
+	fwr = (struct fw_filter_wr *)__skb_put(skb, len);
+	t4_mk_filtdelwr(f->tid, fwr, adapter->sge.fw_evtq.abs_id);
+
+	/* Mark the filter as "pending" and ship off the Filter Work Request.
+	 * When we get the Work Request Reply we'll clear the pending status.
+	 */
+	f->pending = 1;
+	t4_mgmt_tx(adapter, skb);
+	return 0;
+}
+
+/* Send a Work Request to write the filter at a specified index.  We construct
+ * a Firmware Filter Work Request to have the work done and put the indicated
+ * filter into "pending" mode which will prevent any further actions against
+ * it till we get a reply from the firmware on the completion status of the
+ * request.
+ */
+int set_filter_wr(struct adapter *adapter, int fidx)
+{
+	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
+	struct fw_filter_wr *fwr;
+	struct sk_buff *skb;
+
+	skb = alloc_skb(sizeof(*fwr), GFP_KERNEL);
+	if (!skb)
+		return -ENOMEM;
+
+	/* If the new filter requires loopback Destination MAC and/or VLAN
+	 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
+	 * the filter.
+	 */
+	if (f->fs.newdmac || f->fs.newvlan) {
+		/* allocate L2T entry for new filter */
+		f->l2t = t4_l2t_alloc_switching(adapter, f->fs.vlan,
+						f->fs.eport, f->fs.dmac);
+		if (!f->l2t) {
+			kfree_skb(skb);
+			return -ENOMEM;
+		}
+	}
+
+	fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
+	memset(fwr, 0, sizeof(*fwr));
+
+	/* It would be nice to put most of the following in t4_hw.c but most
+	 * of the work is translating the cxgbtool ch_filter_specification
+	 * into the Work Request and the definition of that structure is
+	 * currently in cxgbtool.h which isn't appropriate to pull into the
+	 * common code.  We may eventually try to come up with a more neutral
+	 * filter specification structure but for now it's easiest to simply
+	 * put this fairly direct code in line ...
+	 */
+	fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
+	fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr) / 16));
+	fwr->tid_to_iq =
+		htonl(FW_FILTER_WR_TID_V(f->tid) |
+		      FW_FILTER_WR_RQTYPE_V(f->fs.type) |
+		      FW_FILTER_WR_NOREPLY_V(0) |
+		      FW_FILTER_WR_IQ_V(f->fs.iq));
+	fwr->del_filter_to_l2tix =
+		htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
+		      FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
+		      FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
+		      FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
+		      FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
+		      FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
+		      FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
+		      FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
+		      FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
+					     f->fs.newvlan == VLAN_REWRITE) |
+		      FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
+					    f->fs.newvlan == VLAN_REWRITE) |
+		      FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
+		      FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
+		      FW_FILTER_WR_PRIO_V(f->fs.prio) |
+		      FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
+	fwr->ethtype = htons(f->fs.val.ethtype);
+	fwr->ethtypem = htons(f->fs.mask.ethtype);
+	fwr->frag_to_ovlan_vldm =
+		(FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
+		 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
+		 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
+		 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
+		 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
+		 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
+	fwr->smac_sel = 0;
+	fwr->rx_chan_rx_rpl_iq =
+		htons(FW_FILTER_WR_RX_CHAN_V(0) |
+		      FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
+	fwr->maci_to_matchtypem =
+		htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
+		      FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
+		      FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
+		      FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
+		      FW_FILTER_WR_PORT_V(f->fs.val.iport) |
+		      FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
+		      FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
+		      FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
+	fwr->ptcl = f->fs.val.proto;
+	fwr->ptclm = f->fs.mask.proto;
+	fwr->ttyp = f->fs.val.tos;
+	fwr->ttypm = f->fs.mask.tos;
+	fwr->ivlan = htons(f->fs.val.ivlan);
+	fwr->ivlanm = htons(f->fs.mask.ivlan);
+	fwr->ovlan = htons(f->fs.val.ovlan);
+	fwr->ovlanm = htons(f->fs.mask.ovlan);
+	memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
+	memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
+	memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
+	memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
+	fwr->lp = htons(f->fs.val.lport);
+	fwr->lpm = htons(f->fs.mask.lport);
+	fwr->fp = htons(f->fs.val.fport);
+	fwr->fpm = htons(f->fs.mask.fport);
+	if (f->fs.newsmac)
+		memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
+
+	/* Mark the filter as "pending" and ship off the Filter Work Request.
+	 * When we get the Work Request Reply we'll clear the pending status.
+	 */
+	f->pending = 1;
+	set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
+	t4_ofld_send(adapter, skb);
+	return 0;
+}
+
+/* Return an error number if the indicated filter isn't writable ... */
+int writable_filter(struct filter_entry *f)
+{
+	if (f->locked)
+		return -EPERM;
+	if (f->pending)
+		return -EBUSY;
+
+	return 0;
+}
+
+/* Delete the filter at the specified index (if valid).  The checks for all
+ * the common problems with doing this like the filter being locked, currently
+ * pending in another operation, etc.
+ */
+int delete_filter(struct adapter *adapter, unsigned int fidx)
+{
+	struct filter_entry *f;
+	int ret;
+
+	if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
+		return -EINVAL;
+
+	f = &adapter->tids.ftid_tab[fidx];
+	ret = writable_filter(f);
+	if (ret)
+		return ret;
+	if (f->valid)
+		return del_filter_wr(adapter, fidx);
+
+	return 0;
+}
+
+/* Clear a filter and release any of its resources that we own.  This also
+ * clears the filter's "pending" status.
+ */
+void clear_filter(struct adapter *adap, struct filter_entry *f)
+{
+	/* If the new or old filter have loopback rewriteing rules then we'll
+	 * need to free any existing Layer Two Table (L2T) entries of the old
+	 * filter rule.  The firmware will handle freeing up any Source MAC
+	 * Table (SMT) entries used for rewriting Source MAC Addresses in
+	 * loopback rules.
+	 */
+	if (f->l2t)
+		cxgb4_l2t_release(f->l2t);
+
+	/* The zeroing of the filter rule below clears the filter valid,
+	 * pending, locked flags, l2t pointer, etc. so it's all we need for
+	 * this operation.
+	 */
+	memset(f, 0, sizeof(*f));
+}
+
+void clear_all_filters(struct adapter *adapter)
+{
+	unsigned int i;
+
+	if (adapter->tids.ftid_tab) {
+		struct filter_entry *f = &adapter->tids.ftid_tab[0];
+		unsigned int max_ftid = adapter->tids.nftids +
+					adapter->tids.nsftids;
+
+		for (i = 0; i < max_ftid; i++, f++)
+			if (f->valid || f->pending)
+				clear_filter(adapter, f);
+	}
+}
+
+/* Fill up default masks for set match fields. */
+static void fill_default_mask(struct ch_filter_specification *fs)
+{
+	unsigned int lip = 0, lip_mask = 0;
+	unsigned int fip = 0, fip_mask = 0;
+	unsigned int i;
+
+	if (fs->val.iport && !fs->mask.iport)
+		fs->mask.iport |= ~0;
+	if (fs->val.fcoe && !fs->mask.fcoe)
+		fs->mask.fcoe |= ~0;
+	if (fs->val.matchtype && !fs->mask.matchtype)
+		fs->mask.matchtype |= ~0;
+	if (fs->val.macidx && !fs->mask.macidx)
+		fs->mask.macidx |= ~0;
+	if (fs->val.ethtype && !fs->mask.ethtype)
+		fs->mask.ethtype |= ~0;
+	if (fs->val.ivlan && !fs->mask.ivlan)
+		fs->mask.ivlan |= ~0;
+	if (fs->val.ovlan && !fs->mask.ovlan)
+		fs->mask.ovlan |= ~0;
+	if (fs->val.frag && !fs->mask.frag)
+		fs->mask.frag |= ~0;
+	if (fs->val.tos && !fs->mask.tos)
+		fs->mask.tos |= ~0;
+	if (fs->val.proto && !fs->mask.proto)
+		fs->mask.proto |= ~0;
+
+	for (i = 0; i < ARRAY_SIZE(fs->val.lip); i++) {
+		lip |= fs->val.lip[i];
+		lip_mask |= fs->mask.lip[i];
+		fip |= fs->val.fip[i];
+		fip_mask |= fs->mask.fip[i];
+	}
+
+	if (lip && !lip_mask)
+		memset(fs->mask.lip, ~0, sizeof(fs->mask.lip));
+
+	if (fip && !fip_mask)
+		memset(fs->mask.fip, ~0, sizeof(fs->mask.lip));
+
+	if (fs->val.lport && !fs->mask.lport)
+		fs->mask.lport = ~0;
+	if (fs->val.fport && !fs->mask.fport)
+		fs->mask.fport = ~0;
+}
+
+/* Check a Chelsio Filter Request for validity, convert it into our internal
+ * format and send it to the hardware.  Return 0 on success, an error number
+ * otherwise.  We attach any provided filter operation context to the internal
+ * filter specification in order to facilitate signaling completion of the
+ * operation.
+ */
+int __cxgb4_set_filter(struct net_device *dev, int filter_id,
+		       struct ch_filter_specification *fs,
+		       struct filter_ctx *ctx)
+{
+	struct adapter *adapter = netdev2adap(dev);
+	unsigned int max_fidx, fidx;
+	struct filter_entry *f;
+	u32 iconf;
+	int iq, ret;
+
+	max_fidx = adapter->tids.nftids;
+	if (filter_id != (max_fidx + adapter->tids.nsftids - 1) &&
+	    filter_id >= max_fidx)
+		return -E2BIG;
+
+	fill_default_mask(fs);
+
+	ret = validate_filter(dev, fs);
+	if (ret)
+		return ret;
+
+	iq = get_filter_steerq(dev, fs);
+	if (iq < 0)
+		return iq;
+
+	/* IPv6 filters occupy four slots and must be aligned on
+	 * four-slot boundaries.  IPv4 filters only occupy a single
+	 * slot and have no alignment requirements but writing a new
+	 * IPv4 filter into the middle of an existing IPv6 filter
+	 * requires clearing the old IPv6 filter and hence we prevent
+	 * insertion.
+	 */
+	if (fs->type == 0) { /* IPv4 */
+		/* If our IPv4 filter isn't being written to a
+		 * multiple of four filter index and there's an IPv6
+		 * filter at the multiple of 4 base slot, then we
+		 * prevent insertion.
+		 */
+		fidx = filter_id & ~0x3;
+		if (fidx != filter_id &&
+		    adapter->tids.ftid_tab[fidx].fs.type) {
+			f = &adapter->tids.ftid_tab[fidx];
+			if (f->valid) {
+				dev_err(adapter->pdev_dev,
+					"Invalid location. IPv6 requires 4 slots and is occupying slots %u to %u\n",
+					fidx, fidx + 3);
+				return -EINVAL;
+			}
+		}
+	} else { /* IPv6 */
+		/* Ensure that the IPv6 filter is aligned on a
+		 * multiple of 4 boundary.
+		 */
+		if (filter_id & 0x3) {
+			dev_err(adapter->pdev_dev,
+				"Invalid location. IPv6 must be aligned on a 4-slot boundary\n");
+			return -EINVAL;
+		}
+
+		/* Check all except the base overlapping IPv4 filter slots. */
+		for (fidx = filter_id + 1; fidx < filter_id + 4; fidx++) {
+			f = &adapter->tids.ftid_tab[fidx];
+			if (f->valid) {
+				dev_err(adapter->pdev_dev,
+					"Invalid location.  IPv6 requires 4 slots and an IPv4 filter exists at %u\n",
+					fidx);
+				return -EINVAL;
+			}
+		}
+	}
+
+	/* Check to make sure that provided filter index is not
+	 * already in use by someone else
+	 */
+	f = &adapter->tids.ftid_tab[filter_id];
+	if (f->valid)
+		return -EBUSY;
+
+	fidx = filter_id + adapter->tids.ftid_base;
+	ret = cxgb4_set_ftid(&adapter->tids, filter_id,
+			     fs->type ? PF_INET6 : PF_INET);
+	if (ret)
+		return ret;
+
+	/* Check to make sure the filter requested is writable ... */
+	ret = writable_filter(f);
+	if (ret) {
+		/* Clear the bits we have set above */
+		cxgb4_clear_ftid(&adapter->tids, filter_id,
+				 fs->type ? PF_INET6 : PF_INET);
+		return ret;
+	}
+
+	/* Clear out any old resources being used by the filter before
+	 * we start constructing the new filter.
+	 */
+	if (f->valid)
+		clear_filter(adapter, f);
+
+	/* Convert the filter specification into our internal format.
+	 * We copy the PF/VF specification into the Outer VLAN field
+	 * here so the rest of the code -- including the interface to
+	 * the firmware -- doesn't have to constantly do these checks.
+	 */
+	f->fs = *fs;
+	f->fs.iq = iq;
+	f->dev = dev;
+
+	iconf = adapter->params.tp.ingress_config;
+	if (iconf & VNIC_F) {
+		f->fs.val.ovlan = (fs->val.pf << 13) | fs->val.vf;
+		f->fs.mask.ovlan = (fs->mask.pf << 13) | fs->mask.vf;
+		f->fs.val.ovlan_vld = fs->val.pfvf_vld;
+		f->fs.mask.ovlan_vld = fs->mask.pfvf_vld;
+	}
+
+	/* Attempt to set the filter.  If we don't succeed, we clear
+	 * it and return the failure.
+	 */
+	f->ctx = ctx;
+	f->tid = fidx; /* Save the actual tid */
+	ret = set_filter_wr(adapter, filter_id);
+	if (ret) {
+		cxgb4_clear_ftid(&adapter->tids, filter_id,
+				 fs->type ? PF_INET6 : PF_INET);
+		clear_filter(adapter, f);
+	}
+
+	return ret;
+}
+
+/* Check a delete filter request for validity and send it to the hardware.
+ * Return 0 on success, an error number otherwise.  We attach any provided
+ * filter operation context to the internal filter specification in order to
+ * facilitate signaling completion of the operation.
+ */
+int __cxgb4_del_filter(struct net_device *dev, int filter_id,
+		       struct filter_ctx *ctx)
+{
+	struct adapter *adapter = netdev2adap(dev);
+	struct filter_entry *f;
+	unsigned int max_fidx;
+	int ret;
+
+	max_fidx = adapter->tids.nftids;
+	if (filter_id != (max_fidx + adapter->tids.nsftids - 1) &&
+	    filter_id >= max_fidx)
+		return -E2BIG;
+
+	f = &adapter->tids.ftid_tab[filter_id];
+	ret = writable_filter(f);
+	if (ret)
+		return ret;
+
+	if (f->valid) {
+		f->ctx = ctx;
+		cxgb4_clear_ftid(&adapter->tids, filter_id,
+				 f->fs.type ? PF_INET6 : PF_INET);
+		return del_filter_wr(adapter, filter_id);
+	}
+
+	/* If the caller has passed in a Completion Context then we need to
+	 * mark it as a successful completion so they don't stall waiting
+	 * for it.
+	 */
+	if (ctx) {
+		ctx->result = 0;
+		complete(&ctx->completion);
+	}
+	return ret;
+}
+
+int cxgb4_set_filter(struct net_device *dev, int filter_id,
+		     struct ch_filter_specification *fs)
+{
+	struct filter_ctx ctx;
+	int ret;
+
+	init_completion(&ctx.completion);
+
+	ret = __cxgb4_set_filter(dev, filter_id, fs, &ctx);
+	if (ret)
+		goto out;
+
+	/* Wait for reply */
+	ret = wait_for_completion_timeout(&ctx.completion, 10 * HZ);
+	if (!ret)
+		return -ETIMEDOUT;
+
+	ret = ctx.result;
+out:
+	return ret;
+}
+
+int cxgb4_del_filter(struct net_device *dev, int filter_id)
+{
+	struct filter_ctx ctx;
+	int ret;
+
+	init_completion(&ctx.completion);
+
+	ret = __cxgb4_del_filter(dev, filter_id, &ctx);
+	if (ret)
+		goto out;
+
+	/* Wait for reply */
+	ret = wait_for_completion_timeout(&ctx.completion, 10 * HZ);
+	if (!ret)
+		return -ETIMEDOUT;
+
+	ret = ctx.result;
+out:
+	return ret;
+}
+
+/* Handle a filter write/deletion reply. */
+void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
+{
+	unsigned int tid = GET_TID(rpl);
+	struct filter_entry *f = NULL;
+	unsigned int max_fidx;
+	int idx;
+
+	max_fidx = adap->tids.nftids + adap->tids.nsftids;
+	/* Get the corresponding filter entry for this tid */
+	if (adap->tids.ftid_tab) {
+		/* Check this in normal filter region */
+		idx = tid - adap->tids.ftid_base;
+		if (idx >= max_fidx)
+			return;
+		f = &adap->tids.ftid_tab[idx];
+		if (f->tid != tid)
+			return;
+	}
+
+	/* We found the filter entry for this tid */
+	if (f) {
+		unsigned int ret = TCB_COOKIE_G(rpl->cookie);
+		struct filter_ctx *ctx;
+
+		/* Pull off any filter operation context attached to the
+		 * filter.
+		 */
+		ctx = f->ctx;
+		f->ctx = NULL;
+
+		if (ret == FW_FILTER_WR_FLT_DELETED) {
+			/* Clear the filter when we get confirmation from the
+			 * hardware that the filter has been deleted.
+			 */
+			clear_filter(adap, f);
+			if (ctx)
+				ctx->result = 0;
+		} else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
+			dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
+				idx);
+			clear_filter(adap, f);
+			if (ctx)
+				ctx->result = -ENOMEM;
+		} else if (ret == FW_FILTER_WR_FLT_ADDED) {
+			f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
+			f->pending = 0;  /* asynchronous setup completed */
+			f->valid = 1;
+			if (ctx) {
+				ctx->result = 0;
+				ctx->tid = idx;
+			}
+		} else {
+			/* Something went wrong.  Issue a warning about the
+			 * problem and clear everything out.
+			 */
+			dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
+				idx, ret);
+			clear_filter(adap, f);
+			if (ctx)
+				ctx->result = -EINVAL;
+		}
+		if (ctx)
+			complete(&ctx->completion);
+	}
+}
diff --git a/drivers/infiniband/hw/cxgb4/user.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.h
index 295f422b9a3a..23742cb1c69f 100644
--- a/drivers/infiniband/hw/cxgb4/user.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.h
@@ -1,5 +1,7 @@
 /*
- * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
+ * This file is part of the Chelsio T4 Ethernet driver for Linux.
+ *
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -29,52 +31,18 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-#ifndef __C4IW_USER_H__
-#define __C4IW_USER_H__
-
-#define C4IW_UVERBS_ABI_VERSION	3
 
-/*
- * Make sure that all structs defined in this file remain laid out so
- * that they pack the same way on 32-bit and 64-bit architectures (to
- * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __u64
- * instead.
- */
-struct c4iw_create_cq_resp {
-	__u64 key;
-	__u64 gts_key;
-	__u64 memsize;
-	__u32 cqid;
-	__u32 size;
-	__u32 qid_mask;
-	__u32 reserved; /* explicit padding (optional for i386) */
-};
+#ifndef __CXGB4_FILTER_H
+#define __CXGB4_FILTER_H
 
+#include "t4_msg.h"
 
-enum {
-	C4IW_QPF_ONCHIP = (1<<0)
-};
+void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl);
+void clear_filter(struct adapter *adap, struct filter_entry *f);
 
-struct c4iw_create_qp_resp {
-	__u64 ma_sync_key;
-	__u64 sq_key;
-	__u64 rq_key;
-	__u64 sq_db_gts_key;
-	__u64 rq_db_gts_key;
-	__u64 sq_memsize;
-	__u64 rq_memsize;
-	__u32 sqid;
-	__u32 rqid;
-	__u32 sq_size;
-	__u32 rq_size;
-	__u32 qid_mask;
-	__u32 flags;
-};
+int set_filter_wr(struct adapter *adapter, int fidx);
+int delete_filter(struct adapter *adapter, unsigned int fidx);
 
-struct c4iw_alloc_ucontext_resp {
-	__u64 status_page_key;
-	__u32 status_page_size;
-	__u32 reserved; /* explicit padding (optional for i386) */
-};
-#endif
+int writable_filter(struct filter_entry *f);
+void clear_all_filters(struct adapter *adapter);
+#endif /* __CXGB4_FILTER_H */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index c45de49dc963..f320497368f4 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -1,7 +1,7 @@
 /*
  * This file is part of the Chelsio T4 Ethernet driver for Linux.
  *
- * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -67,6 +67,7 @@
 #include <linux/crash_dump.h>
 
 #include "cxgb4.h"
+#include "cxgb4_filter.h"
 #include "t4_regs.h"
 #include "t4_values.h"
 #include "t4_msg.h"
@@ -76,6 +77,8 @@
 #include "cxgb4_debugfs.h"
 #include "clip_tbl.h"
 #include "l2t.h"
+#include "sched.h"
+#include "cxgb4_tc_u32.h"
 
 char cxgb4_driver_name[] = KBUILD_MODNAME;
 
@@ -86,30 +89,6 @@ char cxgb4_driver_name[] = KBUILD_MODNAME;
 const char cxgb4_driver_version[] = DRV_VERSION;
 #define DRV_DESC "Chelsio T4/T5/T6 Network Driver"
 
-/* Host shadow copy of ingress filter entry.  This is in host native format
- * and doesn't match the ordering or bit order, etc. of the hardware of the
- * firmware command.  The use of bit-field structure elements is purely to
- * remind ourselves of the field size limitations and save memory in the case
- * where the filter table is large.
- */
-struct filter_entry {
-	/* Administrative fields for filter.
-	 */
-	u32 valid:1;            /* filter allocated and valid */
-	u32 locked:1;           /* filter is administratively locked */
-
-	u32 pending:1;          /* filter action is pending firmware reply */
-	u32 smtidx:8;           /* Source MAC Table index for smac */
-	struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
-
-	/* The filter itself.  Most of this is a straight copy of information
-	 * provided by the extended ioctl().  Some fields are translated to
-	 * internal forms -- for instance the Ingress Queue ID passed in from
-	 * the ioctl() is translated into the Absolute Ingress Queue ID.
-	 */
-	struct ch_filter_specification fs;
-};
-
 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 			 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
@@ -223,13 +202,8 @@ MODULE_PARM_DESC(select_queue,
 
 static struct dentry *cxgb4_debugfs_root;
 
-static LIST_HEAD(adapter_list);
-static DEFINE_MUTEX(uld_mutex);
-/* Adapter list to be accessed from atomic context */
-static LIST_HEAD(adap_rcu_list);
-static DEFINE_SPINLOCK(adap_rcu_lock);
-static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
-static const char *const uld_str[] = { "RDMA", "iSCSI", "iSCSIT" };
+LIST_HEAD(adapter_list);
+DEFINE_MUTEX(uld_mutex);
 
 static void link_report(struct net_device *dev)
 {
@@ -303,11 +277,9 @@ static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
 			txq->dcb_prio = value;
 	}
 }
-#endif /* CONFIG_CHELSIO_T4_DCB */
 
-int cxgb4_dcb_enabled(const struct net_device *dev)
+static int cxgb4_dcb_enabled(const struct net_device *dev)
 {
-#ifdef CONFIG_CHELSIO_T4_DCB
 	struct port_info *pi = netdev_priv(dev);
 
 	if (!pi->dcb.enabled)
@@ -315,11 +287,8 @@ int cxgb4_dcb_enabled(const struct net_device *dev)
 
 	return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
 		(pi->dcb.state == CXGB4_DCB_STATE_HOST));
-#else
-	return 0;
-#endif
 }
-EXPORT_SYMBOL(cxgb4_dcb_enabled);
+#endif /* CONFIG_CHELSIO_T4_DCB */
 
 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 {
@@ -531,66 +500,6 @@ static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
 }
 #endif /* CONFIG_CHELSIO_T4_DCB */
 
-/* Clear a filter and release any of its resources that we own.  This also
- * clears the filter's "pending" status.
- */
-static void clear_filter(struct adapter *adap, struct filter_entry *f)
-{
-	/* If the new or old filter have loopback rewriteing rules then we'll
-	 * need to free any existing Layer Two Table (L2T) entries of the old
-	 * filter rule.  The firmware will handle freeing up any Source MAC
-	 * Table (SMT) entries used for rewriting Source MAC Addresses in
-	 * loopback rules.
-	 */
-	if (f->l2t)
-		cxgb4_l2t_release(f->l2t);
-
-	/* The zeroing of the filter rule below clears the filter valid,
-	 * pending, locked flags, l2t pointer, etc. so it's all we need for
-	 * this operation.
-	 */
-	memset(f, 0, sizeof(*f));
-}
-
-/* Handle a filter write/deletion reply.
- */
-static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
-{
-	unsigned int idx = GET_TID(rpl);
-	unsigned int nidx = idx - adap->tids.ftid_base;
-	unsigned int ret;
-	struct filter_entry *f;
-
-	if (idx >= adap->tids.ftid_base && nidx <
-	   (adap->tids.nftids + adap->tids.nsftids)) {
-		idx = nidx;
-		ret = TCB_COOKIE_G(rpl->cookie);
-		f = &adap->tids.ftid_tab[idx];
-
-		if (ret == FW_FILTER_WR_FLT_DELETED) {
-			/* Clear the filter when we get confirmation from the
-			 * hardware that the filter has been deleted.
-			 */
-			clear_filter(adap, f);
-		} else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
-			dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
-				idx);
-			clear_filter(adap, f);
-		} else if (ret == FW_FILTER_WR_FLT_ADDED) {
-			f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
-			f->pending = 0;  /* asynchronous setup completed */
-			f->valid = 1;
-		} else {
-			/* Something went wrong.  Issue a warning about the
-			 * problem and clear everything out.
-			 */
-			dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
-				idx, ret);
-			clear_filter(adap, f);
-		}
-	}
-}
-
 /* Response queue handler for the FW event queue.
  */
 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
@@ -677,56 +586,6 @@ out:
 	return 0;
 }
 
-/* Flush the aggregated lro sessions */
-static void uldrx_flush_handler(struct sge_rspq *q)
-{
-	if (ulds[q->uld].lro_flush)
-		ulds[q->uld].lro_flush(&q->lro_mgr);
-}
-
-/**
- *	uldrx_handler - response queue handler for ULD queues
- *	@q: the response queue that received the packet
- *	@rsp: the response queue descriptor holding the offload message
- *	@gl: the gather list of packet fragments
- *
- *	Deliver an ingress offload packet to a ULD.  All processing is done by
- *	the ULD, we just maintain statistics.
- */
-static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
-			 const struct pkt_gl *gl)
-{
-	struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
-	int ret;
-
-	/* FW can send CPLs encapsulated in a CPL_FW4_MSG.
-	 */
-	if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
-	    ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
-		rsp += 2;
-
-	if (q->flush_handler)
-		ret = ulds[q->uld].lro_rx_handler(q->adap->uld_handle[q->uld],
-						  rsp, gl, &q->lro_mgr,
-						  &q->napi);
-	else
-		ret = ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld],
-					      rsp, gl);
-
-	if (ret) {
-		rxq->stats.nomem++;
-		return -1;
-	}
-
-	if (gl == NULL)
-		rxq->stats.imm++;
-	else if (gl == CXGB4_MSG_AN)
-		rxq->stats.an++;
-	else
-		rxq->stats.pkts++;
-	return 0;
-}
-
 static void disable_msi(struct adapter *adapter)
 {
 	if (adapter->flags & USING_MSIX) {
@@ -778,30 +637,12 @@ static void name_msix_vecs(struct adapter *adap)
 			snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 				 d->name, i);
 	}
-
-	/* offload queues */
-	for_each_iscsirxq(&adap->sge, i)
-		snprintf(adap->msix_info[msi_idx++].desc, n, "%s-iscsi%d",
-			 adap->port[0]->name, i);
-
-	for_each_iscsitrxq(&adap->sge, i)
-		snprintf(adap->msix_info[msi_idx++].desc, n, "%s-iSCSIT%d",
-			 adap->port[0]->name, i);
-
-	for_each_rdmarxq(&adap->sge, i)
-		snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
-			 adap->port[0]->name, i);
-
-	for_each_rdmaciq(&adap->sge, i)
-		snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
-			 adap->port[0]->name, i);
 }
 
 static int request_msix_queue_irqs(struct adapter *adap)
 {
 	struct sge *s = &adap->sge;
-	int err, ethqidx, iscsiqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
-	int iscsitqidx = 0;
+	int err, ethqidx;
 	int msi_index = 2;
 
 	err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
@@ -818,57 +659,9 @@ static int request_msix_queue_irqs(struct adapter *adap)
 			goto unwind;
 		msi_index++;
 	}
-	for_each_iscsirxq(s, iscsiqidx) {
-		err = request_irq(adap->msix_info[msi_index].vec,
-				  t4_sge_intr_msix, 0,
-				  adap->msix_info[msi_index].desc,
-				  &s->iscsirxq[iscsiqidx].rspq);
-		if (err)
-			goto unwind;
-		msi_index++;
-	}
-	for_each_iscsitrxq(s, iscsitqidx) {
-		err = request_irq(adap->msix_info[msi_index].vec,
-				  t4_sge_intr_msix, 0,
-				  adap->msix_info[msi_index].desc,
-				  &s->iscsitrxq[iscsitqidx].rspq);
-		if (err)
-			goto unwind;
-		msi_index++;
-	}
-	for_each_rdmarxq(s, rdmaqidx) {
-		err = request_irq(adap->msix_info[msi_index].vec,
-				  t4_sge_intr_msix, 0,
-				  adap->msix_info[msi_index].desc,
-				  &s->rdmarxq[rdmaqidx].rspq);
-		if (err)
-			goto unwind;
-		msi_index++;
-	}
-	for_each_rdmaciq(s, rdmaciqqidx) {
-		err = request_irq(adap->msix_info[msi_index].vec,
-				  t4_sge_intr_msix, 0,
-				  adap->msix_info[msi_index].desc,
-				  &s->rdmaciq[rdmaciqqidx].rspq);
-		if (err)
-			goto unwind;
-		msi_index++;
-	}
 	return 0;
 
 unwind:
-	while (--rdmaciqqidx >= 0)
-		free_irq(adap->msix_info[--msi_index].vec,
-			 &s->rdmaciq[rdmaciqqidx].rspq);
-	while (--rdmaqidx >= 0)
-		free_irq(adap->msix_info[--msi_index].vec,
-			 &s->rdmarxq[rdmaqidx].rspq);
-	while (--iscsitqidx >= 0)
-		free_irq(adap->msix_info[--msi_index].vec,
-			 &s->iscsitrxq[iscsitqidx].rspq);
-	while (--iscsiqidx >= 0)
-		free_irq(adap->msix_info[--msi_index].vec,
-			 &s->iscsirxq[iscsiqidx].rspq);
 	while (--ethqidx >= 0)
 		free_irq(adap->msix_info[--msi_index].vec,
 			 &s->ethrxq[ethqidx].rspq);
@@ -884,16 +677,6 @@ static void free_msix_queue_irqs(struct adapter *adap)
 	free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 	for_each_ethrxq(s, i)
 		free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
-	for_each_iscsirxq(s, i)
-		free_irq(adap->msix_info[msi_index++].vec,
-			 &s->iscsirxq[i].rspq);
-	for_each_iscsitrxq(s, i)
-		free_irq(adap->msix_info[msi_index++].vec,
-			 &s->iscsitrxq[i].rspq);
-	for_each_rdmarxq(s, i)
-		free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
-	for_each_rdmaciq(s, i)
-		free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
 }
 
 /**
@@ -1032,28 +815,30 @@ static void enable_rx(struct adapter *adap)
 	}
 }
 
-static int alloc_ofld_rxqs(struct adapter *adap, struct sge_ofld_rxq *q,
-			   unsigned int nq, unsigned int per_chan, int msi_idx,
-			   u16 *ids, bool lro)
+
+static int setup_fw_sge_queues(struct adapter *adap)
 {
-	int i, err;
+	struct sge *s = &adap->sge;
+	int err = 0;
+
+	bitmap_zero(s->starving_fl, s->egr_sz);
+	bitmap_zero(s->txq_maperr, s->egr_sz);
 
-	for (i = 0; i < nq; i++, q++) {
-		if (msi_idx > 0)
-			msi_idx++;
-		err = t4_sge_alloc_rxq(adap, &q->rspq, false,
-				       adap->port[i / per_chan],
-				       msi_idx, q->fl.size ? &q->fl : NULL,
-				       uldrx_handler,
-				       lro ? uldrx_flush_handler : NULL,
-				       0);
+	if (adap->flags & USING_MSIX)
+		adap->msi_idx = 1;         /* vector 0 is for non-queue interrupts */
+	else {
+		err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
+				       NULL, NULL, NULL, -1);
 		if (err)
 			return err;
-		memset(&q->stats, 0, sizeof(q->stats));
-		if (ids)
-			ids[i] = q->rspq.abs_id;
+		adap->msi_idx = -((int)s->intrq.abs_id + 1);
 	}
-	return 0;
+
+	err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
+			       adap->msi_idx, NULL, fwevtq_handler, NULL, -1);
+	if (err)
+		t4_free_sge_resources(adap);
+	return err;
 }
 
 /**
@@ -1066,41 +851,10 @@ static int alloc_ofld_rxqs(struct adapter *adap, struct sge_ofld_rxq *q,
  */
 static int setup_sge_queues(struct adapter *adap)
 {
-	int err, msi_idx, i, j;
+	int err, i, j;
 	struct sge *s = &adap->sge;
-
-	bitmap_zero(s->starving_fl, s->egr_sz);
-	bitmap_zero(s->txq_maperr, s->egr_sz);
-
-	if (adap->flags & USING_MSIX)
-		msi_idx = 1;         /* vector 0 is for non-queue interrupts */
-	else {
-		err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
-				       NULL, NULL, NULL, -1);
-		if (err)
-			return err;
-		msi_idx = -((int)s->intrq.abs_id + 1);
-	}
-
-	/* NOTE: If you add/delete any Ingress/Egress Queue allocations in here,
-	 * don't forget to update the following which need to be
-	 * synchronized to and changes here.
-	 *
-	 * 1. The calculations of MAX_INGQ in cxgb4.h.
-	 *
-	 * 2. Update enable_msix/name_msix_vecs/request_msix_queue_irqs
-	 *    to accommodate any new/deleted Ingress Queues
-	 *    which need MSI-X Vectors.
-	 *
-	 * 3. Update sge_qinfo_show() to include information on the
-	 *    new/deleted queues.
-	 */
-	err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
-			       msi_idx, NULL, fwevtq_handler, NULL, -1);
-	if (err) {
-freeout:	t4_free_sge_resources(adap);
-		return err;
-	}
+	struct sge_uld_rxq_info *rxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA];
+	unsigned int cmplqid = 0;
 
 	for_each_port(adap, i) {
 		struct net_device *dev = adap->port[i];
@@ -1109,10 +863,10 @@ freeout:	t4_free_sge_resources(adap);
 		struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
 
 		for (j = 0; j < pi->nqsets; j++, q++) {
-			if (msi_idx > 0)
-				msi_idx++;
+			if (adap->msi_idx > 0)
+				adap->msi_idx++;
 			err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
-					       msi_idx, &q->fl,
+					       adap->msi_idx, &q->fl,
 					       t4_ethrx_handler,
 					       NULL,
 					       t4_get_mps_bg_map(adap,
@@ -1131,8 +885,8 @@ freeout:	t4_free_sge_resources(adap);
 		}
 	}
 
-	j = s->iscsiqsets / adap->params.nports; /* iscsi queues per channel */
-	for_each_iscsirxq(s, i) {
+	j = s->ofldqsets / adap->params.nports; /* iscsi queues per channel */
+	for_each_ofldtxq(s, i) {
 		err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i],
 					    adap->port[i / j],
 					    s->fw_evtq.cntxt_id);
@@ -1140,30 +894,15 @@ freeout:	t4_free_sge_resources(adap);
 			goto freeout;
 	}
 
-#define ALLOC_OFLD_RXQS(firstq, nq, per_chan, ids, lro) do { \
-	err = alloc_ofld_rxqs(adap, firstq, nq, per_chan, msi_idx, ids, lro); \
-	if (err) \
-		goto freeout; \
-	if (msi_idx > 0) \
-		msi_idx += nq; \
-} while (0)
-
-	ALLOC_OFLD_RXQS(s->iscsirxq, s->iscsiqsets, j, s->iscsi_rxq, false);
-	ALLOC_OFLD_RXQS(s->iscsitrxq, s->niscsitq, j, s->iscsit_rxq, true);
-	ALLOC_OFLD_RXQS(s->rdmarxq, s->rdmaqs, 1, s->rdma_rxq, false);
-	j = s->rdmaciqs / adap->params.nports; /* rdmaq queues per channel */
-	ALLOC_OFLD_RXQS(s->rdmaciq, s->rdmaciqs, j, s->rdma_ciq, false);
-
-#undef ALLOC_OFLD_RXQS
-
 	for_each_port(adap, i) {
-		/*
-		 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
+		/* Note that cmplqid below is 0 if we don't
 		 * have RDMA queues, and that's the right value.
 		 */
+		if (rxq_info)
+			cmplqid	= rxq_info->uldrxq[i].rspq.cntxt_id;
+
 		err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
-					    s->fw_evtq.cntxt_id,
-					    s->rdmarxq[i].rspq.cntxt_id);
+					    s->fw_evtq.cntxt_id, cmplqid);
 		if (err)
 			goto freeout;
 	}
@@ -1174,6 +913,9 @@ freeout:	t4_free_sge_resources(adap);
 		     RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
 		     QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
 	return 0;
+freeout:
+	t4_free_sge_resources(adap);
+	return err;
 }
 
 /*
@@ -1197,151 +939,6 @@ void t4_free_mem(void *addr)
 	kvfree(addr);
 }
 
-/* Send a Work Request to write the filter at a specified index.  We construct
- * a Firmware Filter Work Request to have the work done and put the indicated
- * filter into "pending" mode which will prevent any further actions against
- * it till we get a reply from the firmware on the completion status of the
- * request.
- */
-static int set_filter_wr(struct adapter *adapter, int fidx)
-{
-	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
-	struct sk_buff *skb;
-	struct fw_filter_wr *fwr;
-	unsigned int ftid;
-
-	skb = alloc_skb(sizeof(*fwr), GFP_KERNEL);
-	if (!skb)
-		return -ENOMEM;
-
-	/* If the new filter requires loopback Destination MAC and/or VLAN
-	 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
-	 * the filter.
-	 */
-	if (f->fs.newdmac || f->fs.newvlan) {
-		/* allocate L2T entry for new filter */
-		f->l2t = t4_l2t_alloc_switching(adapter, f->fs.vlan,
-						f->fs.eport, f->fs.dmac);
-		if (f->l2t == NULL) {
-			kfree_skb(skb);
-			return -ENOMEM;
-		}
-	}
-
-	ftid = adapter->tids.ftid_base + fidx;
-
-	fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
-	memset(fwr, 0, sizeof(*fwr));
-
-	/* It would be nice to put most of the following in t4_hw.c but most
-	 * of the work is translating the cxgbtool ch_filter_specification
-	 * into the Work Request and the definition of that structure is
-	 * currently in cxgbtool.h which isn't appropriate to pull into the
-	 * common code.  We may eventually try to come up with a more neutral
-	 * filter specification structure but for now it's easiest to simply
-	 * put this fairly direct code in line ...
-	 */
-	fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
-	fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16));
-	fwr->tid_to_iq =
-		htonl(FW_FILTER_WR_TID_V(ftid) |
-		      FW_FILTER_WR_RQTYPE_V(f->fs.type) |
-		      FW_FILTER_WR_NOREPLY_V(0) |
-		      FW_FILTER_WR_IQ_V(f->fs.iq));
-	fwr->del_filter_to_l2tix =
-		htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
-		      FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
-		      FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
-		      FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
-		      FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
-		      FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
-		      FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
-		      FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
-		      FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
-					     f->fs.newvlan == VLAN_REWRITE) |
-		      FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
-					    f->fs.newvlan == VLAN_REWRITE) |
-		      FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
-		      FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
-		      FW_FILTER_WR_PRIO_V(f->fs.prio) |
-		      FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
-	fwr->ethtype = htons(f->fs.val.ethtype);
-	fwr->ethtypem = htons(f->fs.mask.ethtype);
-	fwr->frag_to_ovlan_vldm =
-		(FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
-		 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
-		 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
-		 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
-		 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
-		 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
-	fwr->smac_sel = 0;
-	fwr->rx_chan_rx_rpl_iq =
-		htons(FW_FILTER_WR_RX_CHAN_V(0) |
-		      FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
-	fwr->maci_to_matchtypem =
-		htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
-		      FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
-		      FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
-		      FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
-		      FW_FILTER_WR_PORT_V(f->fs.val.iport) |
-		      FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
-		      FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
-		      FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
-	fwr->ptcl = f->fs.val.proto;
-	fwr->ptclm = f->fs.mask.proto;
-	fwr->ttyp = f->fs.val.tos;
-	fwr->ttypm = f->fs.mask.tos;
-	fwr->ivlan = htons(f->fs.val.ivlan);
-	fwr->ivlanm = htons(f->fs.mask.ivlan);
-	fwr->ovlan = htons(f->fs.val.ovlan);
-	fwr->ovlanm = htons(f->fs.mask.ovlan);
-	memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
-	memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
-	memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
-	memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
-	fwr->lp = htons(f->fs.val.lport);
-	fwr->lpm = htons(f->fs.mask.lport);
-	fwr->fp = htons(f->fs.val.fport);
-	fwr->fpm = htons(f->fs.mask.fport);
-	if (f->fs.newsmac)
-		memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
-
-	/* Mark the filter as "pending" and ship off the Filter Work Request.
-	 * When we get the Work Request Reply we'll clear the pending status.
-	 */
-	f->pending = 1;
-	set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
-	t4_ofld_send(adapter, skb);
-	return 0;
-}
-
-/* Delete the filter at a specified index.
- */
-static int del_filter_wr(struct adapter *adapter, int fidx)
-{
-	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
-	struct sk_buff *skb;
-	struct fw_filter_wr *fwr;
-	unsigned int len, ftid;
-
-	len = sizeof(*fwr);
-	ftid = adapter->tids.ftid_base + fidx;
-
-	skb = alloc_skb(len, GFP_KERNEL);
-	if (!skb)
-		return -ENOMEM;
-
-	fwr = (struct fw_filter_wr *)__skb_put(skb, len);
-	t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
-
-	/* Mark the filter as "pending" and ship off the Filter Work Request.
-	 * When we get the Work Request Reply we'll clear the pending status.
-	 */
-	f->pending = 1;
-	t4_mgmt_tx(adapter, skb);
-	return 0;
-}
-
 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
 			     void *accel_priv, select_queue_fallback_t fallback)
 {
@@ -1723,19 +1320,22 @@ EXPORT_SYMBOL(cxgb4_remove_tid);
  */
 static int tid_init(struct tid_info *t)
 {
-	size_t size;
-	unsigned int stid_bmap_size;
-	unsigned int natids = t->natids;
 	struct adapter *adap = container_of(t, struct adapter, tids);
+	unsigned int max_ftids = t->nftids + t->nsftids;
+	unsigned int natids = t->natids;
+	unsigned int stid_bmap_size;
+	unsigned int ftid_bmap_size;
+	size_t size;
 
 	stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
+	ftid_bmap_size = BITS_TO_LONGS(t->nftids);
 	size = t->ntids * sizeof(*t->tid_tab) +
 	       natids * sizeof(*t->atid_tab) +
 	       t->nstids * sizeof(*t->stid_tab) +
 	       t->nsftids * sizeof(*t->stid_tab) +
 	       stid_bmap_size * sizeof(long) +
-	       t->nftids * sizeof(*t->ftid_tab) +
-	       t->nsftids * sizeof(*t->ftid_tab);
+	       max_ftids * sizeof(*t->ftid_tab) +
+	       ftid_bmap_size * sizeof(long);
 
 	t->tid_tab = t4_alloc_mem(size);
 	if (!t->tid_tab)
@@ -1745,8 +1345,10 @@ static int tid_init(struct tid_info *t)
 	t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
 	t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
 	t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
+	t->ftid_bmap = (unsigned long *)&t->ftid_tab[max_ftids];
 	spin_lock_init(&t->stid_lock);
 	spin_lock_init(&t->atid_lock);
+	spin_lock_init(&t->ftid_lock);
 
 	t->stids_in_use = 0;
 	t->sftids_in_use = 0;
@@ -1761,12 +1363,16 @@ static int tid_init(struct tid_info *t)
 			t->atid_tab[natids - 1].next = &t->atid_tab[natids];
 		t->afree = t->atid_tab;
 	}
-	bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
-	/* Reserve stid 0 for T4/T5 adapters */
-	if (!t->stid_base &&
-	    (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5))
-		__set_bit(0, t->stid_bmap);
 
+	if (is_offload(adap)) {
+		bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
+		/* Reserve stid 0 for T4/T5 adapters */
+		if (!t->stid_base &&
+		    CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
+			__set_bit(0, t->stid_bmap);
+	}
+
+	bitmap_zero(t->ftid_bmap, t->nftids);
 	return 0;
 }
 
@@ -2316,7 +1922,7 @@ static void disable_dbs(struct adapter *adap)
 
 	for_each_ethrxq(&adap->sge, i)
 		disable_txq_db(&adap->sge.ethtxq[i].q);
-	for_each_iscsirxq(&adap->sge, i)
+	for_each_ofldtxq(&adap->sge, i)
 		disable_txq_db(&adap->sge.ofldtxq[i].q);
 	for_each_port(adap, i)
 		disable_txq_db(&adap->sge.ctrlq[i].q);
@@ -2328,7 +1934,7 @@ static void enable_dbs(struct adapter *adap)
 
 	for_each_ethrxq(&adap->sge, i)
 		enable_txq_db(adap, &adap->sge.ethtxq[i].q);
-	for_each_iscsirxq(&adap->sge, i)
+	for_each_ofldtxq(&adap->sge, i)
 		enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
 	for_each_port(adap, i)
 		enable_txq_db(adap, &adap->sge.ctrlq[i].q);
@@ -2336,9 +1942,10 @@ static void enable_dbs(struct adapter *adap)
 
 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
 {
-	if (adap->uld_handle[CXGB4_ULD_RDMA])
-		ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
-				cmd);
+	enum cxgb4_uld type = CXGB4_ULD_RDMA;
+
+	if (adap->uld && adap->uld[type].handle)
+		adap->uld[type].control(adap->uld[type].handle, cmd);
 }
 
 static void process_db_full(struct work_struct *work)
@@ -2392,13 +1999,14 @@ out:
 	if (ret)
 		CH_WARN(adap, "DB drop recovery failed.\n");
 }
+
 static void recover_all_queues(struct adapter *adap)
 {
 	int i;
 
 	for_each_ethrxq(&adap->sge, i)
 		sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
-	for_each_iscsirxq(&adap->sge, i)
+	for_each_ofldtxq(&adap->sge, i)
 		sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
 	for_each_port(adap, i)
 		sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
@@ -2463,94 +2071,12 @@ void t4_db_dropped(struct adapter *adap)
 	queue_work(adap->workq, &adap->db_drop_task);
 }
 
-static void uld_attach(struct adapter *adap, unsigned int uld)
-{
-	void *handle;
-	struct cxgb4_lld_info lli;
-	unsigned short i;
-
-	lli.pdev = adap->pdev;
-	lli.pf = adap->pf;
-	lli.l2t = adap->l2t;
-	lli.tids = &adap->tids;
-	lli.ports = adap->port;
-	lli.vr = &adap->vres;
-	lli.mtus = adap->params.mtus;
-	if (uld == CXGB4_ULD_RDMA) {
-		lli.rxq_ids = adap->sge.rdma_rxq;
-		lli.ciq_ids = adap->sge.rdma_ciq;
-		lli.nrxq = adap->sge.rdmaqs;
-		lli.nciq = adap->sge.rdmaciqs;
-	} else if (uld == CXGB4_ULD_ISCSI) {
-		lli.rxq_ids = adap->sge.iscsi_rxq;
-		lli.nrxq = adap->sge.iscsiqsets;
-	} else if (uld == CXGB4_ULD_ISCSIT) {
-		lli.rxq_ids = adap->sge.iscsit_rxq;
-		lli.nrxq = adap->sge.niscsitq;
-	}
-	lli.ntxq = adap->sge.iscsiqsets;
-	lli.nchan = adap->params.nports;
-	lli.nports = adap->params.nports;
-	lli.wr_cred = adap->params.ofldq_wr_cred;
-	lli.adapter_type = adap->params.chip;
-	lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
-	lli.iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK_A);
-	lli.iscsi_pgsz_order = t4_read_reg(adap, ULP_RX_ISCSI_PSZ_A);
-	lli.iscsi_llimit = t4_read_reg(adap, ULP_RX_ISCSI_LLIMIT_A);
-	lli.iscsi_ppm = &adap->iscsi_ppm;
-	lli.cclk_ps = 1000000000 / adap->params.vpd.cclk;
-	lli.udb_density = 1 << adap->params.sge.eq_qpp;
-	lli.ucq_density = 1 << adap->params.sge.iq_qpp;
-	lli.filt_mode = adap->params.tp.vlan_pri_map;
-	/* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
-	for (i = 0; i < NCHAN; i++)
-		lli.tx_modq[i] = i;
-	lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
-	lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
-	lli.fw_vers = adap->params.fw_vers;
-	lli.dbfifo_int_thresh = dbfifo_int_thresh;
-	lli.sge_ingpadboundary = adap->sge.fl_align;
-	lli.sge_egrstatuspagesize = adap->sge.stat_len;
-	lli.sge_pktshift = adap->sge.pktshift;
-	lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
-	lli.max_ordird_qp = adap->params.max_ordird_qp;
-	lli.max_ird_adapter = adap->params.max_ird_adapter;
-	lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
-	lli.nodeid = dev_to_node(adap->pdev_dev);
-
-	handle = ulds[uld].add(&lli);
-	if (IS_ERR(handle)) {
-		dev_warn(adap->pdev_dev,
-			 "could not attach to the %s driver, error %ld\n",
-			 uld_str[uld], PTR_ERR(handle));
-		return;
-	}
-
-	adap->uld_handle[uld] = handle;
-
+void t4_register_netevent_notifier(void)
+{
 	if (!netevent_registered) {
 		register_netevent_notifier(&cxgb4_netevent_nb);
 		netevent_registered = true;
 	}
-
-	if (adap->flags & FULL_INIT_DONE)
-		ulds[uld].state_change(handle, CXGB4_STATE_UP);
-}
-
-static void attach_ulds(struct adapter *adap)
-{
-	unsigned int i;
-
-	spin_lock(&adap_rcu_lock);
-	list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
-	spin_unlock(&adap_rcu_lock);
-
-	mutex_lock(&uld_mutex);
-	list_add_tail(&adap->list_node, &adapter_list);
-	for (i = 0; i < CXGB4_ULD_MAX; i++)
-		if (ulds[i].add)
-			uld_attach(adap, i);
-	mutex_unlock(&uld_mutex);
 }
 
 static void detach_ulds(struct adapter *adap)
@@ -2560,20 +2086,16 @@ static void detach_ulds(struct adapter *adap)
 	mutex_lock(&uld_mutex);
 	list_del(&adap->list_node);
 	for (i = 0; i < CXGB4_ULD_MAX; i++)
-		if (adap->uld_handle[i]) {
-			ulds[i].state_change(adap->uld_handle[i],
+		if (adap->uld && adap->uld[i].handle) {
+			adap->uld[i].state_change(adap->uld[i].handle,
 					     CXGB4_STATE_DETACH);
-			adap->uld_handle[i] = NULL;
+			adap->uld[i].handle = NULL;
 		}
 	if (netevent_registered && list_empty(&adapter_list)) {
 		unregister_netevent_notifier(&cxgb4_netevent_nb);
 		netevent_registered = false;
 	}
 	mutex_unlock(&uld_mutex);
-
-	spin_lock(&adap_rcu_lock);
-	list_del_rcu(&adap->rcu_node);
-	spin_unlock(&adap_rcu_lock);
 }
 
 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
@@ -2582,61 +2104,12 @@ static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
 
 	mutex_lock(&uld_mutex);
 	for (i = 0; i < CXGB4_ULD_MAX; i++)
-		if (adap->uld_handle[i])
-			ulds[i].state_change(adap->uld_handle[i], new_state);
+		if (adap->uld && adap->uld[i].handle)
+			adap->uld[i].state_change(adap->uld[i].handle,
+						  new_state);
 	mutex_unlock(&uld_mutex);
 }
 
-/**
- *	cxgb4_register_uld - register an upper-layer driver
- *	@type: the ULD type
- *	@p: the ULD methods
- *
- *	Registers an upper-layer driver with this driver and notifies the ULD
- *	about any presently available devices that support its type.  Returns
- *	%-EBUSY if a ULD of the same type is already registered.
- */
-int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
-{
-	int ret = 0;
-	struct adapter *adap;
-
-	if (type >= CXGB4_ULD_MAX)
-		return -EINVAL;
-	mutex_lock(&uld_mutex);
-	if (ulds[type].add) {
-		ret = -EBUSY;
-		goto out;
-	}
-	ulds[type] = *p;
-	list_for_each_entry(adap, &adapter_list, list_node)
-		uld_attach(adap, type);
-out:	mutex_unlock(&uld_mutex);
-	return ret;
-}
-EXPORT_SYMBOL(cxgb4_register_uld);
-
-/**
- *	cxgb4_unregister_uld - unregister an upper-layer driver
- *	@type: the ULD type
- *
- *	Unregisters an existing upper-layer driver.
- */
-int cxgb4_unregister_uld(enum cxgb4_uld type)
-{
-	struct adapter *adap;
-
-	if (type >= CXGB4_ULD_MAX)
-		return -EINVAL;
-	mutex_lock(&uld_mutex);
-	list_for_each_entry(adap, &adapter_list, list_node)
-		adap->uld_handle[type] = NULL;
-	ulds[type].add = NULL;
-	mutex_unlock(&uld_mutex);
-	return 0;
-}
-EXPORT_SYMBOL(cxgb4_unregister_uld);
-
 #if IS_ENABLED(CONFIG_IPV6)
 static int cxgb4_inet6addr_handler(struct notifier_block *this,
 				   unsigned long event, void *data)
@@ -2741,7 +2214,6 @@ static int cxgb_up(struct adapter *adap)
 				  adap->msix_info[0].desc, adap);
 		if (err)
 			goto irq_err;
-
 		err = request_msix_queue_irqs(adap);
 		if (err) {
 			free_irq(adap->msix_info[0].vec, adap);
@@ -2819,40 +2291,6 @@ static int cxgb_close(struct net_device *dev)
 	return t4_enable_vi(adapter, adapter->pf, pi->viid, false, false);
 }
 
-/* Return an error number if the indicated filter isn't writable ...
- */
-static int writable_filter(struct filter_entry *f)
-{
-	if (f->locked)
-		return -EPERM;
-	if (f->pending)
-		return -EBUSY;
-
-	return 0;
-}
-
-/* Delete the filter at the specified index (if valid).  The checks for all
- * the common problems with doing this like the filter being locked, currently
- * pending in another operation, etc.
- */
-static int delete_filter(struct adapter *adapter, unsigned int fidx)
-{
-	struct filter_entry *f;
-	int ret;
-
-	if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
-		return -EINVAL;
-
-	f = &adapter->tids.ftid_tab[fidx];
-	ret = writable_filter(f);
-	if (ret)
-		return ret;
-	if (f->valid)
-		return del_filter_wr(adapter, fidx);
-
-	return 0;
-}
-
 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
 		__be32 sip, __be16 sport, __be16 vlan,
 		unsigned int queue, unsigned char port, unsigned char mask)
@@ -2922,7 +2360,6 @@ EXPORT_SYMBOL(cxgb4_create_server_filter);
 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
 		unsigned int queue, bool ipv6)
 {
-	int ret;
 	struct filter_entry *f;
 	struct adapter *adap;
 
@@ -2936,11 +2373,7 @@ int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
 	/* Unlock the filter */
 	f->locked = 0;
 
-	ret = delete_filter(adap, stid);
-	if (ret)
-		return ret;
-
-	return 0;
+	return delete_filter(adap, stid);
 }
 EXPORT_SYMBOL(cxgb4_remove_server_filter);
 
@@ -3078,6 +2511,85 @@ static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
 	return ret;
 }
 
+#ifdef CONFIG_PCI_IOV
+static int dummy_open(struct net_device *dev)
+{
+	/* Turn carrier off since we don't have to transmit anything on this
+	 * interface.
+	 */
+	netif_carrier_off(dev);
+	return 0;
+}
+
+/* Fill MAC address that will be assigned by the FW */
+static void fill_vf_station_mac_addr(struct adapter *adap)
+{
+	unsigned int i;
+	u8 hw_addr[ETH_ALEN], macaddr[ETH_ALEN];
+	int err;
+	u8 *na;
+	u16 a, b;
+
+	err = t4_get_raw_vpd_params(adap, &adap->params.vpd);
+	if (!err) {
+		na = adap->params.vpd.na;
+		for (i = 0; i < ETH_ALEN; i++)
+			hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 +
+				      hex2val(na[2 * i + 1]));
+		a = (hw_addr[0] << 8) | hw_addr[1];
+		b = (hw_addr[1] << 8) | hw_addr[2];
+		a ^= b;
+		a |= 0x0200;    /* locally assigned Ethernet MAC address */
+		a &= ~0x0100;   /* not a multicast Ethernet MAC address */
+		macaddr[0] = a >> 8;
+		macaddr[1] = a & 0xff;
+
+		for (i = 2; i < 5; i++)
+			macaddr[i] = hw_addr[i + 1];
+
+		for (i = 0; i < adap->num_vfs; i++) {
+			macaddr[5] = adap->pf * 16 + i;
+			ether_addr_copy(adap->vfinfo[i].vf_mac_addr, macaddr);
+		}
+	}
+}
+
+static int cxgb_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
+{
+	struct port_info *pi = netdev_priv(dev);
+	struct adapter *adap = pi->adapter;
+	int ret;
+
+	/* verify MAC addr is valid */
+	if (!is_valid_ether_addr(mac)) {
+		dev_err(pi->adapter->pdev_dev,
+			"Invalid Ethernet address %pM for VF %d\n",
+			mac, vf);
+		return -EINVAL;
+	}
+
+	dev_info(pi->adapter->pdev_dev,
+		 "Setting MAC %pM on VF %d\n", mac, vf);
+	ret = t4_set_vf_mac_acl(adap, vf + 1, 1, mac);
+	if (!ret)
+		ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, mac);
+	return ret;
+}
+
+static int cxgb_get_vf_config(struct net_device *dev,
+			      int vf, struct ifla_vf_info *ivi)
+{
+	struct port_info *pi = netdev_priv(dev);
+	struct adapter *adap = pi->adapter;
+
+	if (vf >= adap->num_vfs)
+		return -EINVAL;
+	ivi->vf = vf;
+	ether_addr_copy(ivi->mac, adap->vfinfo[vf].vf_mac_addr);
+	return 0;
+}
+#endif
+
 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
 {
 	int ret;
@@ -3114,6 +2626,116 @@ static void cxgb_netpoll(struct net_device *dev)
 }
 #endif
 
+static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
+{
+	struct port_info *pi = netdev_priv(dev);
+	struct adapter *adap = pi->adapter;
+	struct sched_class *e;
+	struct ch_sched_params p;
+	struct ch_sched_queue qe;
+	u32 req_rate;
+	int err = 0;
+
+	if (!can_sched(dev))
+		return -ENOTSUPP;
+
+	if (index < 0 || index > pi->nqsets - 1)
+		return -EINVAL;
+
+	if (!(adap->flags & FULL_INIT_DONE)) {
+		dev_err(adap->pdev_dev,
+			"Failed to rate limit on queue %d. Link Down?\n",
+			index);
+		return -EINVAL;
+	}
+
+	/* Convert from Mbps to Kbps */
+	req_rate = rate << 10;
+
+	/* Max rate is 10 Gbps */
+	if (req_rate >= SCHED_MAX_RATE_KBPS) {
+		dev_err(adap->pdev_dev,
+			"Invalid rate %u Mbps, Max rate is %u Gbps\n",
+			rate, SCHED_MAX_RATE_KBPS);
+		return -ERANGE;
+	}
+
+	/* First unbind the queue from any existing class */
+	memset(&qe, 0, sizeof(qe));
+	qe.queue = index;
+	qe.class = SCHED_CLS_NONE;
+
+	err = cxgb4_sched_class_unbind(dev, (void *)(&qe), SCHED_QUEUE);
+	if (err) {
+		dev_err(adap->pdev_dev,
+			"Unbinding Queue %d on port %d fail. Err: %d\n",
+			index, pi->port_id, err);
+		return err;
+	}
+
+	/* Queue already unbound */
+	if (!req_rate)
+		return 0;
+
+	/* Fetch any available unused or matching scheduling class */
+	memset(&p, 0, sizeof(p));
+	p.type = SCHED_CLASS_TYPE_PACKET;
+	p.u.params.level    = SCHED_CLASS_LEVEL_CL_RL;
+	p.u.params.mode     = SCHED_CLASS_MODE_CLASS;
+	p.u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS;
+	p.u.params.ratemode = SCHED_CLASS_RATEMODE_ABS;
+	p.u.params.channel  = pi->tx_chan;
+	p.u.params.class    = SCHED_CLS_NONE;
+	p.u.params.minrate  = 0;
+	p.u.params.maxrate  = req_rate;
+	p.u.params.weight   = 0;
+	p.u.params.pktsize  = dev->mtu;
+
+	e = cxgb4_sched_class_alloc(dev, &p);
+	if (!e)
+		return -ENOMEM;
+
+	/* Bind the queue to a scheduling class */
+	memset(&qe, 0, sizeof(qe));
+	qe.queue = index;
+	qe.class = e->idx;
+
+	err = cxgb4_sched_class_bind(dev, (void *)(&qe), SCHED_QUEUE);
+	if (err)
+		dev_err(adap->pdev_dev,
+			"Queue rate limiting failed. Err: %d\n", err);
+	return err;
+}
+
+static int cxgb_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
+			 struct tc_to_netdev *tc)
+{
+	struct port_info *pi = netdev2pinfo(dev);
+	struct adapter *adap = netdev2adap(dev);
+
+	if (!(adap->flags & FULL_INIT_DONE)) {
+		dev_err(adap->pdev_dev,
+			"Failed to setup tc on port %d. Link Down?\n",
+			pi->port_id);
+		return -EINVAL;
+	}
+
+	if (TC_H_MAJ(handle) == TC_H_MAJ(TC_H_INGRESS) &&
+	    tc->type == TC_SETUP_CLSU32) {
+		switch (tc->cls_u32->command) {
+		case TC_CLSU32_NEW_KNODE:
+		case TC_CLSU32_REPLACE_KNODE:
+			return cxgb4_config_knode(dev, proto, tc->cls_u32);
+		case TC_CLSU32_DELETE_KNODE:
+			return cxgb4_delete_knode(dev, proto, tc->cls_u32);
+		default:
+			return -EOPNOTSUPP;
+		}
+	}
+
+	return -EOPNOTSUPP;
+}
+
 static const struct net_device_ops cxgb4_netdev_ops = {
 	.ndo_open             = cxgb_open,
 	.ndo_stop             = cxgb_close,
@@ -3136,7 +2758,31 @@ static const struct net_device_ops cxgb4_netdev_ops = {
 #ifdef CONFIG_NET_RX_BUSY_POLL
 	.ndo_busy_poll        = cxgb_busy_poll,
 #endif
+	.ndo_set_tx_maxrate   = cxgb_set_tx_maxrate,
+	.ndo_setup_tc         = cxgb_setup_tc,
+};
 
+#ifdef CONFIG_PCI_IOV
+static const struct net_device_ops cxgb4_mgmt_netdev_ops = {
+	.ndo_open             = dummy_open,
+	.ndo_set_vf_mac       = cxgb_set_vf_mac,
+	.ndo_get_vf_config    = cxgb_get_vf_config,
+};
+#endif
+
+static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
+{
+	struct adapter *adapter = netdev2adap(dev);
+
+	strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver));
+	strlcpy(info->version, cxgb4_driver_version,
+		sizeof(info->version));
+	strlcpy(info->bus_info, pci_name(adapter->pdev),
+		sizeof(info->bus_info));
+}
+
+static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = {
+	.get_drvinfo       = get_drvinfo,
 };
 
 void t4_fatal_err(struct adapter *adap)
@@ -3979,6 +3625,12 @@ static int adap_init0(struct adapter *adap)
 	adap->clipt_start = val[0];
 	adap->clipt_end = val[1];
 
+	/* We don't yet have a PARAMs calls to retrieve the number of Traffic
+	 * Classes supported by the hardware/firmware so we hard code it here
+	 * for now.
+	 */
+	adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16;
+
 	/* query params related to active filter region */
 	params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
 	params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
@@ -4016,6 +3668,12 @@ static int adap_init0(struct adapter *adap)
 		adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
 	}
 
+	/* See if FW supports FW_RI_FR_NSMR_TPTE_WR work request */
+	params[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR);
+	ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+			      1, params, val);
+	adap->params.fr_nsmr_tpte_wr_support = (ret == 0 && val[0] != 0);
+
 	/*
 	 * Get device capabilities so we can determine what resources we need
 	 * to manage.
@@ -4067,6 +3725,7 @@ static int adap_init0(struct adapter *adap)
 		adap->params.ofldq_wr_cred = val[5];
 
 		adap->params.offload = 1;
+		adap->num_ofld_uld += 1;
 	}
 	if (caps_cmd.rdmacaps) {
 		params[0] = FW_PARAM_PFVF(STAG_START);
@@ -4119,6 +3778,7 @@ static int adap_init0(struct adapter *adap)
 			 "max_ordird_qp %d max_ird_adapter %d\n",
 			 adap->params.max_ordird_qp,
 			 adap->params.max_ird_adapter);
+		adap->num_ofld_uld += 2;
 	}
 	if (caps_cmd.iscsicaps) {
 		params[0] = FW_PARAM_PFVF(ISCSI_START);
@@ -4129,6 +3789,13 @@ static int adap_init0(struct adapter *adap)
 			goto bye;
 		adap->vres.iscsi.start = val[0];
 		adap->vres.iscsi.size = val[1] - val[0] + 1;
+		/* LIO target and cxgb4i initiaitor */
+		adap->num_ofld_uld += 2;
+	}
+	if (caps_cmd.cryptocaps) {
+		/* Should query params here...TODO */
+		adap->params.crypto |= ULP_CRYPTO_LOOKASIDE;
+		adap->num_uld += 1;
 	}
 #undef FW_PARAM_PFVF
 #undef FW_PARAM_DEV
@@ -4305,20 +3972,17 @@ static const struct pci_error_handlers cxgb4_eeh = {
 	.resume         = eeh_resume,
 };
 
+/* Return true if the Link Configuration supports "High Speeds" (those greater
+ * than 1Gb/s).
+ */
 static inline bool is_x_10g_port(const struct link_config *lc)
 {
-	return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
-	       (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
-}
+	unsigned int speeds, high_speeds;
 
-static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
-			     unsigned int us, unsigned int cnt,
-			     unsigned int size, unsigned int iqe_size)
-{
-	q->adap = adap;
-	cxgb4_set_rspq_intr_params(q, us, cnt);
-	q->iqe_len = iqe_size;
-	q->size = size;
+	speeds = FW_PORT_CAP_SPEED_V(FW_PORT_CAP_SPEED_G(lc->supported));
+	high_speeds = speeds & ~(FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G);
+
+	return high_speeds != 0;
 }
 
 /*
@@ -4333,7 +3997,16 @@ static void cfg_queues(struct adapter *adap)
 #ifndef CONFIG_CHELSIO_T4_DCB
 	int q10g = 0;
 #endif
-	int ciq_size;
+
+	/* Reduce memory usage in kdump environment, disable all offload.
+	 */
+	if (is_kdump_kernel()) {
+		adap->params.offload = 0;
+		adap->params.crypto = 0;
+	} else if (is_uld(adap) && t4_uld_mem_alloc(adap)) {
+		adap->params.offload = 0;
+		adap->params.crypto = 0;
+	}
 
 	for_each_port(adap, i)
 		n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
@@ -4365,11 +4038,6 @@ static void cfg_queues(struct adapter *adap)
 	if (q10g > netif_get_num_default_rss_queues())
 		q10g = netif_get_num_default_rss_queues();
 
-	/* Reduce memory usage in kdump environment, disable all offload.
-	 */
-	if (is_kdump_kernel())
-		adap->params.offload = 0;
-
 	for_each_port(adap, i) {
 		struct port_info *pi = adap2pinfo(adap, i);
 
@@ -4382,33 +4050,18 @@ static void cfg_queues(struct adapter *adap)
 	s->ethqsets = qidx;
 	s->max_ethqsets = qidx;   /* MSI-X may lower it later */
 
-	if (is_offload(adap)) {
+	if (is_uld(adap)) {
 		/*
 		 * For offload we use 1 queue/channel if all ports are up to 1G,
 		 * otherwise we divide all available queues amongst the channels
 		 * capped by the number of available cores.
 		 */
 		if (n10g) {
-			i = min_t(int, ARRAY_SIZE(s->iscsirxq),
-				  num_online_cpus());
-			s->iscsiqsets = roundup(i, adap->params.nports);
-		} else
-			s->iscsiqsets = adap->params.nports;
-		/* For RDMA one Rx queue per channel suffices */
-		s->rdmaqs = adap->params.nports;
-		/* Try and allow at least 1 CIQ per cpu rounding down
-		 * to the number of ports, with a minimum of 1 per port.
-		 * A 2 port card in a 6 cpu system: 6 CIQs, 3 / port.
-		 * A 4 port card in a 6 cpu system: 4 CIQs, 1 / port.
-		 * A 4 port card in a 2 cpu system: 4 CIQs, 1 / port.
-		 */
-		s->rdmaciqs = min_t(int, MAX_RDMA_CIQS, num_online_cpus());
-		s->rdmaciqs = (s->rdmaciqs / adap->params.nports) *
-				adap->params.nports;
-		s->rdmaciqs = max_t(int, s->rdmaciqs, adap->params.nports);
-
-		if (!is_t4(adap->params.chip))
-			s->niscsitq = s->iscsiqsets;
+			i = num_online_cpus();
+			s->ofldqsets = roundup(i, adap->params.nports);
+		} else {
+			s->ofldqsets = adap->params.nports;
+		}
 	}
 
 	for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
@@ -4427,47 +4080,8 @@ static void cfg_queues(struct adapter *adap)
 	for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
 		s->ofldtxq[i].q.size = 1024;
 
-	for (i = 0; i < ARRAY_SIZE(s->iscsirxq); i++) {
-		struct sge_ofld_rxq *r = &s->iscsirxq[i];
-
-		init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
-		r->rspq.uld = CXGB4_ULD_ISCSI;
-		r->fl.size = 72;
-	}
-
-	if (!is_t4(adap->params.chip)) {
-		for (i = 0; i < ARRAY_SIZE(s->iscsitrxq); i++) {
-			struct sge_ofld_rxq *r = &s->iscsitrxq[i];
-
-			init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
-			r->rspq.uld = CXGB4_ULD_ISCSIT;
-			r->fl.size = 72;
-		}
-	}
-
-	for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
-		struct sge_ofld_rxq *r = &s->rdmarxq[i];
-
-		init_rspq(adap, &r->rspq, 5, 1, 511, 64);
-		r->rspq.uld = CXGB4_ULD_RDMA;
-		r->fl.size = 72;
-	}
-
-	ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
-	if (ciq_size > SGE_MAX_IQ_SIZE) {
-		CH_WARN(adap, "CIQ size too small for available IQs\n");
-		ciq_size = SGE_MAX_IQ_SIZE;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
-		struct sge_ofld_rxq *r = &s->rdmaciq[i];
-
-		init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
-		r->rspq.uld = CXGB4_ULD_RDMA;
-	}
-
 	init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
-	init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
+	init_rspq(adap, &s->intrq, 0, 1, 512, 64);
 }
 
 /*
@@ -4498,42 +4112,90 @@ static void reduce_ethqs(struct adapter *adap, int n)
 	}
 }
 
+static int get_msix_info(struct adapter *adap)
+{
+	struct uld_msix_info *msix_info;
+	unsigned int max_ingq = 0;
+
+	if (is_offload(adap))
+		max_ingq += MAX_OFLD_QSETS * adap->num_ofld_uld;
+	if (is_pci_uld(adap))
+		max_ingq += MAX_OFLD_QSETS * adap->num_uld;
+
+	if (!max_ingq)
+		goto out;
+
+	msix_info = kcalloc(max_ingq, sizeof(*msix_info), GFP_KERNEL);
+	if (!msix_info)
+		return -ENOMEM;
+
+	adap->msix_bmap_ulds.msix_bmap = kcalloc(BITS_TO_LONGS(max_ingq),
+						 sizeof(long), GFP_KERNEL);
+	if (!adap->msix_bmap_ulds.msix_bmap) {
+		kfree(msix_info);
+		return -ENOMEM;
+	}
+	spin_lock_init(&adap->msix_bmap_ulds.lock);
+	adap->msix_info_ulds = msix_info;
+out:
+	return 0;
+}
+
+static void free_msix_info(struct adapter *adap)
+{
+	if (!(adap->num_uld && adap->num_ofld_uld))
+		return;
+
+	kfree(adap->msix_info_ulds);
+	kfree(adap->msix_bmap_ulds.msix_bmap);
+}
+
 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
 #define EXTRA_VECS 2
 
 static int enable_msix(struct adapter *adap)
 {
-	int ofld_need = 0;
-	int i, want, need, allocated;
+	int ofld_need = 0, uld_need = 0;
+	int i, j, want, need, allocated;
 	struct sge *s = &adap->sge;
 	unsigned int nchan = adap->params.nports;
 	struct msix_entry *entries;
+	int max_ingq = MAX_INGQ;
 
-	entries = kmalloc(sizeof(*entries) * (MAX_INGQ + 1),
+	if (is_pci_uld(adap))
+		max_ingq += (MAX_OFLD_QSETS * adap->num_uld);
+	if (is_offload(adap))
+		max_ingq += (MAX_OFLD_QSETS * adap->num_ofld_uld);
+	entries = kmalloc(sizeof(*entries) * (max_ingq + 1),
 			  GFP_KERNEL);
 	if (!entries)
 		return -ENOMEM;
 
-	for (i = 0; i < MAX_INGQ + 1; ++i)
+	/* map for msix */
+	if (get_msix_info(adap)) {
+		adap->params.offload = 0;
+		adap->params.crypto = 0;
+	}
+
+	for (i = 0; i < max_ingq + 1; ++i)
 		entries[i].entry = i;
 
 	want = s->max_ethqsets + EXTRA_VECS;
 	if (is_offload(adap)) {
-		want += s->rdmaqs + s->rdmaciqs + s->iscsiqsets	+
-			s->niscsitq;
-		/* need nchan for each possible ULD */
-		if (is_t4(adap->params.chip))
-			ofld_need = 3 * nchan;
-		else
-			ofld_need = 4 * nchan;
+		want += adap->num_ofld_uld * s->ofldqsets;
+		ofld_need = adap->num_ofld_uld * nchan;
+	}
+	if (is_pci_uld(adap)) {
+		want += adap->num_uld * s->ofldqsets;
+		uld_need = adap->num_uld * nchan;
 	}
 #ifdef CONFIG_CHELSIO_T4_DCB
 	/* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
 	 * each port.
 	 */
-	need = 8 * adap->params.nports + EXTRA_VECS + ofld_need;
+	need = 8 * adap->params.nports + EXTRA_VECS + ofld_need + uld_need;
 #else
-	need = adap->params.nports + EXTRA_VECS + ofld_need;
+	need = adap->params.nports + EXTRA_VECS + ofld_need + uld_need;
 #endif
 	allocated = pci_enable_msix_range(adap->pdev, entries, need, want);
 	if (allocated < 0) {
@@ -4547,33 +4209,31 @@ static int enable_msix(struct adapter *adap)
 	 * Every group gets its minimum requirement and NIC gets top
 	 * priority for leftovers.
 	 */
-	i = allocated - EXTRA_VECS - ofld_need;
+	i = allocated - EXTRA_VECS - ofld_need - uld_need;
 	if (i < s->max_ethqsets) {
 		s->max_ethqsets = i;
 		if (i < s->ethqsets)
 			reduce_ethqs(adap, i);
 	}
-	if (is_offload(adap)) {
-		if (allocated < want) {
-			s->rdmaqs = nchan;
-			s->rdmaciqs = nchan;
+	if (is_uld(adap)) {
+		if (allocated < want)
+			s->nqs_per_uld = nchan;
+		else
+			s->nqs_per_uld = s->ofldqsets;
+	}
 
-			if (!is_t4(adap->params.chip))
-				s->niscsitq = nchan;
+	for (i = 0; i < (s->max_ethqsets + EXTRA_VECS); ++i)
+		adap->msix_info[i].vec = entries[i].vector;
+	if (is_uld(adap)) {
+		for (j = 0 ; i < allocated; ++i, j++) {
+			adap->msix_info_ulds[j].vec = entries[i].vector;
+			adap->msix_info_ulds[j].idx = i;
 		}
-
-		/* leftovers go to OFLD */
-		i = allocated - EXTRA_VECS - s->max_ethqsets -
-		    s->rdmaqs - s->rdmaciqs - s->niscsitq;
-		s->iscsiqsets = (i / nchan) * nchan;  /* round down */
-
+		adap->msix_bmap_ulds.mapsize = j;
 	}
-	for (i = 0; i < allocated; ++i)
-		adap->msix_info[i].vec = entries[i].vector;
 	dev_info(adap->pdev_dev, "%d MSI-X vectors allocated, "
-		 "nic %d iscsi %d rdma cpl %d rdma ciq %d\n",
-		 allocated, s->max_ethqsets, s->iscsiqsets, s->rdmaqs,
-		 s->rdmaciqs);
+		 "nic %d per uld %d\n",
+		 allocated, s->max_ethqsets, s->nqs_per_uld);
 
 	kfree(entries);
 	return 0;
@@ -4756,8 +4416,12 @@ static void print_port_info(const struct net_device *dev)
 		bufp += sprintf(bufp, "1000/");
 	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
 		bufp += sprintf(bufp, "10G/");
+	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_25G)
+		bufp += sprintf(bufp, "25G/");
 	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
 		bufp += sprintf(bufp, "40G/");
+	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100G)
+		bufp += sprintf(bufp, "100G/");
 	if (bufp != buf)
 		--bufp;
 	sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
@@ -4783,7 +4447,9 @@ static void free_some_resources(struct adapter *adapter)
 	unsigned int i;
 
 	t4_free_mem(adapter->l2t);
+	t4_cleanup_sched(adapter);
 	t4_free_mem(adapter->tids.tid_tab);
+	cxgb4_cleanup_tc_u32(adapter);
 	kfree(adapter->sge.egr_map);
 	kfree(adapter->sge.ingr_map);
 	kfree(adapter->sge.starving_fl);
@@ -4834,21 +4500,59 @@ static int get_chip_type(struct pci_dev *pdev, u32 pl_rev)
 }
 
 #ifdef CONFIG_PCI_IOV
+static void dummy_setup(struct net_device *dev)
+{
+	dev->type = ARPHRD_NONE;
+	dev->mtu = 0;
+	dev->hard_header_len = 0;
+	dev->addr_len = 0;
+	dev->tx_queue_len = 0;
+	dev->flags |= IFF_NOARP;
+	dev->priv_flags |= IFF_NO_QUEUE;
+
+	/* Initialize the device structure. */
+	dev->netdev_ops = &cxgb4_mgmt_netdev_ops;
+	dev->ethtool_ops = &cxgb4_mgmt_ethtool_ops;
+	dev->destructor = free_netdev;
+}
+
+static int config_mgmt_dev(struct pci_dev *pdev)
+{
+	struct adapter *adap = pci_get_drvdata(pdev);
+	struct net_device *netdev;
+	struct port_info *pi;
+	char name[IFNAMSIZ];
+	int err;
+
+	snprintf(name, IFNAMSIZ, "mgmtpf%d%d", adap->adap_idx, adap->pf);
+	netdev = alloc_netdev(0, name, NET_NAME_UNKNOWN, dummy_setup);
+	if (!netdev)
+		return -ENOMEM;
+
+	pi = netdev_priv(netdev);
+	pi->adapter = adap;
+	SET_NETDEV_DEV(netdev, &pdev->dev);
+
+	adap->port[0] = netdev;
+
+	err = register_netdev(adap->port[0]);
+	if (err) {
+		pr_info("Unable to register VF mgmt netdev %s\n", name);
+		free_netdev(adap->port[0]);
+		adap->port[0] = NULL;
+		return err;
+	}
+	return 0;
+}
+
 static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
 {
+	struct adapter *adap = pci_get_drvdata(pdev);
 	int err = 0;
 	int current_vfs = pci_num_vf(pdev);
 	u32 pcie_fw;
-	void __iomem *regs;
 
-	regs = pci_ioremap_bar(pdev, 0);
-	if (!regs) {
-		dev_err(&pdev->dev, "cannot map device registers\n");
-		return -ENOMEM;
-	}
-
-	pcie_fw = readl(regs + PCIE_FW_A);
-	iounmap(regs);
+	pcie_fw = readl(adap->regs + PCIE_FW_A);
 	/* Check if cxgb4 is the MASTER and fw is initialized */
 	if (!(pcie_fw & PCIE_FW_INIT_F) ||
 	    !(pcie_fw & PCIE_FW_MASTER_VLD_F) ||
@@ -4875,6 +4579,14 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
 	 */
 	if (!num_vfs) {
 		pci_disable_sriov(pdev);
+		if (adap->port[0]) {
+			unregister_netdev(adap->port[0]);
+			adap->port[0] = NULL;
+		}
+		/* free VF resources */
+		kfree(adap->vfinfo);
+		adap->vfinfo = NULL;
+		adap->num_vfs = 0;
 		return num_vfs;
 	}
 
@@ -4882,7 +4594,17 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
 		err = pci_enable_sriov(pdev, num_vfs);
 		if (err)
 			return err;
+
+		adap->num_vfs = num_vfs;
+		err = config_mgmt_dev(pdev);
+		if (err)
+			return err;
 	}
+
+	adap->vfinfo = kcalloc(adap->num_vfs,
+			       sizeof(struct vf_info), GFP_KERNEL);
+	if (adap->vfinfo)
+		fill_vf_station_mac_addr(adap);
 	return num_vfs;
 }
 #endif
@@ -4893,9 +4615,11 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct port_info *pi;
 	bool highdma = false;
 	struct adapter *adapter = NULL;
+	struct net_device *netdev;
 	void __iomem *regs;
 	u32 whoami, pl_rev;
 	enum chip_type chip;
+	static int adap_idx = 1;
 
 	printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
 
@@ -4930,7 +4654,9 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ?
 		SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
 	if (func != ent->driver_data) {
+#ifndef CONFIG_PCI_IOV
 		iounmap(regs);
+#endif
 		pci_disable_device(pdev);
 		pci_save_state(pdev);        /* to restore SR-IOV later */
 		goto sriov;
@@ -4962,6 +4688,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		err = -ENOMEM;
 		goto out_unmap_bar0;
 	}
+	adap_idx++;
 
 	adapter->workq = create_singlethread_workqueue("cxgb4");
 	if (!adapter->workq) {
@@ -5048,8 +4775,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 			      T6_STATMODE_V(0)));
 
 	for_each_port(adapter, i) {
-		struct net_device *netdev;
-
 		netdev = alloc_etherdev_mq(sizeof(struct port_info),
 					   MAX_ETH_QSETS);
 		if (!netdev) {
@@ -5069,7 +4794,8 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 			NETIF_F_RXCSUM | NETIF_F_RXHASH |
-			NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
+			NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
+			NETIF_F_HW_TC;
 		if (highdma)
 			netdev->hw_features |= NETIF_F_HIGHDMA;
 		netdev->features |= netdev->hw_features;
@@ -5143,10 +4869,26 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		}
 	}
 #endif
-	if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
+
+	for_each_port(adapter, i) {
+		pi = adap2pinfo(adapter, i);
+		pi->sched_tbl = t4_init_sched(adapter->params.nsched_cls);
+		if (!pi->sched_tbl)
+			dev_warn(&pdev->dev,
+				 "could not activate scheduling on port %d\n",
+				 i);
+	}
+
+	if (tid_init(&adapter->tids) < 0) {
 		dev_warn(&pdev->dev, "could not allocate TID table, "
 			 "continuing\n");
 		adapter->params.offload = 0;
+	} else {
+		adapter->tc_u32 = cxgb4_init_tc_u32(adapter,
+						    CXGB4_MAX_LINK_HANDLE);
+		if (!adapter->tc_u32)
+			dev_warn(&pdev->dev,
+				 "could not offload tc u32, continuing\n");
 	}
 
 	if (is_offload(adapter)) {
@@ -5168,8 +4910,11 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* See what interrupts we'll be using */
 	if (msi > 1 && enable_msix(adapter) == 0)
 		adapter->flags |= USING_MSIX;
-	else if (msi > 0 && pci_enable_msi(pdev) == 0)
+	else if (msi > 0 && pci_enable_msi(pdev) == 0) {
 		adapter->flags |= USING_MSI;
+		if (msi > 1)
+			free_msix_info(adapter);
+	}
 
 	/* check for PCI Express bandwidth capabiltites */
 	cxgb4_check_pcie_caps(adapter);
@@ -5213,10 +4958,15 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* PCIe EEH recovery on powerpc platforms needs fundamental reset */
 	pdev->needs_freset = 1;
 
-	if (is_offload(adapter))
-		attach_ulds(adapter);
+	if (is_uld(adapter)) {
+		mutex_lock(&uld_mutex);
+		list_add_tail(&adapter->list_node, &adapter_list);
+		mutex_unlock(&uld_mutex);
+	}
 
 	print_adapter_info(adapter);
+	setup_fw_sge_queues(adapter);
+	return 0;
 
 sriov:
 #ifdef CONFIG_PCI_IOV
@@ -5230,11 +4980,48 @@ sriov:
 				 "instantiated %u virtual functions\n",
 				 num_vf[func]);
 	}
-#endif
+
+	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
+	if (!adapter) {
+		err = -ENOMEM;
+		goto free_pci_region;
+	}
+
+	adapter->pdev = pdev;
+	adapter->pdev_dev = &pdev->dev;
+	adapter->name = pci_name(pdev);
+	adapter->mbox = func;
+	adapter->pf = func;
+	adapter->regs = regs;
+	adapter->adap_idx = adap_idx;
+	adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) +
+				    (sizeof(struct mbox_cmd) *
+				     T4_OS_LOG_MBOX_CMDS),
+				    GFP_KERNEL);
+	if (!adapter->mbox_log) {
+		err = -ENOMEM;
+		goto free_adapter;
+	}
+	pci_set_drvdata(pdev, adapter);
 	return 0;
 
+ free_adapter:
+	kfree(adapter);
+ free_pci_region:
+	iounmap(regs);
+	pci_disable_sriov(pdev);
+	pci_release_regions(pdev);
+	return err;
+#else
+	return 0;
+#endif
+
  out_free_dev:
 	free_some_resources(adapter);
+	if (adapter->flags & USING_MSIX)
+		free_msix_info(adapter);
+	if (adapter->num_uld || adapter->num_ofld_uld)
+		t4_uld_mem_free(adapter);
  out_unmap_bar:
 	if (!is_t4(adapter->params.chip))
 		iounmap(adapter->bar2);
@@ -5258,12 +5045,12 @@ static void remove_one(struct pci_dev *pdev)
 {
 	struct adapter *adapter = pci_get_drvdata(pdev);
 
-#ifdef CONFIG_PCI_IOV
-	pci_disable_sriov(pdev);
-
-#endif
+	if (!adapter) {
+		pci_release_regions(pdev);
+		return;
+	}
 
-	if (adapter) {
+	if (adapter->pf == 4) {
 		int i;
 
 		/* Tear down per-adapter Work Queue first since it can contain
@@ -5271,7 +5058,7 @@ static void remove_one(struct pci_dev *pdev)
 		 */
 		destroy_workqueue(adapter->workq);
 
-		if (is_offload(adapter))
+		if (is_uld(adapter))
 			detach_ulds(adapter);
 
 		disable_interrupts(adapter);
@@ -5285,17 +5072,15 @@ static void remove_one(struct pci_dev *pdev)
 		/* If we allocated filters, free up state associated with any
 		 * valid filters ...
 		 */
-		if (adapter->tids.ftid_tab) {
-			struct filter_entry *f = &adapter->tids.ftid_tab[0];
-			for (i = 0; i < (adapter->tids.nftids +
-					adapter->tids.nsftids); i++, f++)
-				if (f->valid)
-					clear_filter(adapter, f);
-		}
+		clear_all_filters(adapter);
 
 		if (adapter->flags & FULL_INIT_DONE)
 			cxgb_down(adapter);
 
+		if (adapter->flags & USING_MSIX)
+			free_msix_info(adapter);
+		if (adapter->num_uld || adapter->num_ofld_uld)
+			t4_uld_mem_free(adapter);
 		free_some_resources(adapter);
 #if IS_ENABLED(CONFIG_IPV6)
 		t4_cleanup_clip_tbl(adapter);
@@ -5312,8 +5097,64 @@ static void remove_one(struct pci_dev *pdev)
 		kfree(adapter->mbox_log);
 		synchronize_rcu();
 		kfree(adapter);
-	} else
+	}
+#ifdef CONFIG_PCI_IOV
+	else {
+		if (adapter->port[0])
+			unregister_netdev(adapter->port[0]);
+		iounmap(adapter->regs);
+		kfree(adapter->vfinfo);
+		kfree(adapter);
+		pci_disable_sriov(pdev);
+		pci_release_regions(pdev);
+	}
+#endif
+}
+
+/* "Shutdown" quiesces the device, stopping Ingress Packet and Interrupt
+ * delivery.  This is essentially a stripped down version of the PCI remove()
+ * function where we do the minimal amount of work necessary to shutdown any
+ * further activity.
+ */
+static void shutdown_one(struct pci_dev *pdev)
+{
+	struct adapter *adapter = pci_get_drvdata(pdev);
+
+	/* As with remove_one() above (see extended comment), we only want do
+	 * do cleanup on PCI Devices which went all the way through init_one()
+	 * ...
+	 */
+	if (!adapter) {
+		pci_release_regions(pdev);
+		return;
+	}
+
+	if (adapter->pf == 4) {
+		int i;
+
+		for_each_port(adapter, i)
+			if (adapter->port[i]->reg_state == NETREG_REGISTERED)
+				cxgb_close(adapter->port[i]);
+
+		t4_uld_clean_up(adapter);
+		disable_interrupts(adapter);
+		disable_msi(adapter);
+
+		t4_sge_stop(adapter);
+		if (adapter->flags & FW_OK)
+			t4_fw_bye(adapter, adapter->mbox);
+	}
+#ifdef CONFIG_PCI_IOV
+	else {
+		if (adapter->port[0])
+			unregister_netdev(adapter->port[0]);
+		iounmap(adapter->regs);
+		kfree(adapter->vfinfo);
+		kfree(adapter);
+		pci_disable_sriov(pdev);
 		pci_release_regions(pdev);
+	}
+#endif
 }
 
 static struct pci_driver cxgb4_driver = {
@@ -5321,7 +5162,7 @@ static struct pci_driver cxgb4_driver = {
 	.id_table = cxgb4_pci_tbl,
 	.probe    = init_one,
 	.remove   = remove_one,
-	.shutdown = remove_one,
+	.shutdown = shutdown_one,
 #ifdef CONFIG_PCI_IOV
 	.sriov_configure = cxgb4_iov_configure,
 #endif
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c
new file mode 100644
index 000000000000..49d2debb334e
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c
@@ -0,0 +1,483 @@
+/*
+ * This file is part of the Chelsio T4 Ethernet driver for Linux.
+ *
+ * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <net/tc_act/tc_gact.h>
+#include <net/tc_act/tc_mirred.h>
+
+#include "cxgb4.h"
+#include "cxgb4_tc_u32_parse.h"
+#include "cxgb4_tc_u32.h"
+
+/* Fill ch_filter_specification with parsed match value/mask pair. */
+static int fill_match_fields(struct adapter *adap,
+			     struct ch_filter_specification *fs,
+			     struct tc_cls_u32_offload *cls,
+			     const struct cxgb4_match_field *entry,
+			     bool next_header)
+{
+	unsigned int i, j;
+	u32 val, mask;
+	int off, err;
+	bool found;
+
+	for (i = 0; i < cls->knode.sel->nkeys; i++) {
+		off = cls->knode.sel->keys[i].off;
+		val = cls->knode.sel->keys[i].val;
+		mask = cls->knode.sel->keys[i].mask;
+
+		if (next_header) {
+			/* For next headers, parse only keys with offmask */
+			if (!cls->knode.sel->keys[i].offmask)
+				continue;
+		} else {
+			/* For the remaining, parse only keys without offmask */
+			if (cls->knode.sel->keys[i].offmask)
+				continue;
+		}
+
+		found = false;
+
+		for (j = 0; entry[j].val; j++) {
+			if (off == entry[j].off) {
+				found = true;
+				err = entry[j].val(fs, val, mask);
+				if (err)
+					return err;
+				break;
+			}
+		}
+
+		if (!found)
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Fill ch_filter_specification with parsed action. */
+static int fill_action_fields(struct adapter *adap,
+			      struct ch_filter_specification *fs,
+			      struct tc_cls_u32_offload *cls)
+{
+	unsigned int num_actions = 0;
+	const struct tc_action *a;
+	struct tcf_exts *exts;
+	LIST_HEAD(actions);
+
+	exts = cls->knode.exts;
+	if (tc_no_actions(exts))
+		return -EINVAL;
+
+	tcf_exts_to_list(exts, &actions);
+	list_for_each_entry(a, &actions, list) {
+		/* Don't allow more than one action per rule. */
+		if (num_actions)
+			return -EINVAL;
+
+		/* Drop in hardware. */
+		if (is_tcf_gact_shot(a)) {
+			fs->action = FILTER_DROP;
+			num_actions++;
+			continue;
+		}
+
+		/* Re-direct to specified port in hardware. */
+		if (is_tcf_mirred_redirect(a)) {
+			struct net_device *n_dev;
+			unsigned int i, index;
+			bool found = false;
+
+			index = tcf_mirred_ifindex(a);
+			for_each_port(adap, i) {
+				n_dev = adap->port[i];
+				if (index == n_dev->ifindex) {
+					fs->action = FILTER_SWITCH;
+					fs->eport = i;
+					found = true;
+					break;
+				}
+			}
+
+			/* Interface doesn't belong to any port of
+			 * the underlying hardware.
+			 */
+			if (!found)
+				return -EINVAL;
+
+			num_actions++;
+			continue;
+		}
+
+		/* Un-supported action. */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int cxgb4_config_knode(struct net_device *dev, __be16 protocol,
+		       struct tc_cls_u32_offload *cls)
+{
+	const struct cxgb4_match_field *start, *link_start = NULL;
+	struct adapter *adapter = netdev2adap(dev);
+	struct ch_filter_specification fs;
+	struct cxgb4_tc_u32_table *t;
+	struct cxgb4_link *link;
+	unsigned int filter_id;
+	u32 uhtid, link_uhtid;
+	bool is_ipv6 = false;
+	int ret;
+
+	if (!can_tc_u32_offload(dev))
+		return -EOPNOTSUPP;
+
+	if (protocol != htons(ETH_P_IP) && protocol != htons(ETH_P_IPV6))
+		return -EOPNOTSUPP;
+
+	/* Fetch the location to insert the filter. */
+	filter_id = cls->knode.handle & 0xFFFFF;
+
+	if (filter_id > adapter->tids.nftids) {
+		dev_err(adapter->pdev_dev,
+			"Location %d out of range for insertion. Max: %d\n",
+			filter_id, adapter->tids.nftids);
+		return -ERANGE;
+	}
+
+	t = adapter->tc_u32;
+	uhtid = TC_U32_USERHTID(cls->knode.handle);
+	link_uhtid = TC_U32_USERHTID(cls->knode.link_handle);
+
+	/* Ensure that uhtid is either root u32 (i.e. 0x800)
+	 * or a a valid linked bucket.
+	 */
+	if (uhtid != 0x800 && uhtid >= t->size)
+		return -EINVAL;
+
+	/* Ensure link handle uhtid is sane, if specified. */
+	if (link_uhtid >= t->size)
+		return -EINVAL;
+
+	memset(&fs, 0, sizeof(fs));
+
+	if (protocol == htons(ETH_P_IPV6)) {
+		start = cxgb4_ipv6_fields;
+		is_ipv6 = true;
+	} else {
+		start = cxgb4_ipv4_fields;
+		is_ipv6 = false;
+	}
+
+	if (uhtid != 0x800) {
+		/* Link must exist from root node before insertion. */
+		if (!t->table[uhtid - 1].link_handle)
+			return -EINVAL;
+
+		/* Link must have a valid supported next header. */
+		link_start = t->table[uhtid - 1].match_field;
+		if (!link_start)
+			return -EINVAL;
+	}
+
+	/* Parse links and record them for subsequent jumps to valid
+	 * next headers.
+	 */
+	if (link_uhtid) {
+		const struct cxgb4_next_header *next;
+		bool found = false;
+		unsigned int i, j;
+		u32 val, mask;
+		int off;
+
+		if (t->table[link_uhtid - 1].link_handle) {
+			dev_err(adapter->pdev_dev,
+				"Link handle exists for: 0x%x\n",
+				link_uhtid);
+			return -EINVAL;
+		}
+
+		next = is_ipv6 ? cxgb4_ipv6_jumps : cxgb4_ipv4_jumps;
+
+		/* Try to find matches that allow jumps to next header. */
+		for (i = 0; next[i].jump; i++) {
+			if (next[i].offoff != cls->knode.sel->offoff ||
+			    next[i].shift != cls->knode.sel->offshift ||
+			    next[i].mask != cls->knode.sel->offmask ||
+			    next[i].offset != cls->knode.sel->off)
+				continue;
+
+			/* Found a possible candidate.  Find a key that
+			 * matches the corresponding offset, value, and
+			 * mask to jump to next header.
+			 */
+			for (j = 0; j < cls->knode.sel->nkeys; j++) {
+				off = cls->knode.sel->keys[j].off;
+				val = cls->knode.sel->keys[j].val;
+				mask = cls->knode.sel->keys[j].mask;
+
+				if (next[i].match_off == off &&
+				    next[i].match_val == val &&
+				    next[i].match_mask == mask) {
+					found = true;
+					break;
+				}
+			}
+
+			if (!found)
+				continue; /* Try next candidate. */
+
+			/* Candidate to jump to next header found.
+			 * Translate all keys to internal specification
+			 * and store them in jump table. This spec is copied
+			 * later to set the actual filters.
+			 */
+			ret = fill_match_fields(adapter, &fs, cls,
+						start, false);
+			if (ret)
+				goto out;
+
+			link = &t->table[link_uhtid - 1];
+			link->match_field = next[i].jump;
+			link->link_handle = cls->knode.handle;
+			memcpy(&link->fs, &fs, sizeof(fs));
+			break;
+		}
+
+		/* No candidate found to jump to next header. */
+		if (!found)
+			return -EINVAL;
+
+		return 0;
+	}
+
+	/* Fill ch_filter_specification match fields to be shipped to hardware.
+	 * Copy the linked spec (if any) first.  And then update the spec as
+	 * needed.
+	 */
+	if (uhtid != 0x800 && t->table[uhtid - 1].link_handle) {
+		/* Copy linked ch_filter_specification */
+		memcpy(&fs, &t->table[uhtid - 1].fs, sizeof(fs));
+		ret = fill_match_fields(adapter, &fs, cls,
+					link_start, true);
+		if (ret)
+			goto out;
+	}
+
+	ret = fill_match_fields(adapter, &fs, cls, start, false);
+	if (ret)
+		goto out;
+
+	/* Fill ch_filter_specification action fields to be shipped to
+	 * hardware.
+	 */
+	ret = fill_action_fields(adapter, &fs, cls);
+	if (ret)
+		goto out;
+
+	/* The filter spec has been completely built from the info
+	 * provided from u32.  We now set some default fields in the
+	 * spec for sanity.
+	 */
+
+	/* Match only packets coming from the ingress port where this
+	 * filter will be created.
+	 */
+	fs.val.iport = netdev2pinfo(dev)->port_id;
+	fs.mask.iport = ~0;
+
+	/* Enable filter hit counts. */
+	fs.hitcnts = 1;
+
+	/* Set type of filter - IPv6 or IPv4 */
+	fs.type = is_ipv6 ? 1 : 0;
+
+	/* Set the filter */
+	ret = cxgb4_set_filter(dev, filter_id, &fs);
+	if (ret)
+		goto out;
+
+	/* If this is a linked bucket, then set the corresponding
+	 * entry in the bitmap to mark it as belonging to this linked
+	 * bucket.
+	 */
+	if (uhtid != 0x800 && t->table[uhtid - 1].link_handle)
+		set_bit(filter_id, t->table[uhtid - 1].tid_map);
+
+out:
+	return ret;
+}
+
+int cxgb4_delete_knode(struct net_device *dev, __be16 protocol,
+		       struct tc_cls_u32_offload *cls)
+{
+	struct adapter *adapter = netdev2adap(dev);
+	unsigned int filter_id, max_tids, i, j;
+	struct cxgb4_link *link = NULL;
+	struct cxgb4_tc_u32_table *t;
+	u32 handle, uhtid;
+	int ret;
+
+	if (!can_tc_u32_offload(dev))
+		return -EOPNOTSUPP;
+
+	/* Fetch the location to delete the filter. */
+	filter_id = cls->knode.handle & 0xFFFFF;
+
+	if (filter_id > adapter->tids.nftids) {
+		dev_err(adapter->pdev_dev,
+			"Location %d out of range for deletion. Max: %d\n",
+			filter_id, adapter->tids.nftids);
+		return -ERANGE;
+	}
+
+	t = adapter->tc_u32;
+	handle = cls->knode.handle;
+	uhtid = TC_U32_USERHTID(cls->knode.handle);
+
+	/* Ensure that uhtid is either root u32 (i.e. 0x800)
+	 * or a a valid linked bucket.
+	 */
+	if (uhtid != 0x800 && uhtid >= t->size)
+		return -EINVAL;
+
+	/* Delete the specified filter */
+	if (uhtid != 0x800) {
+		link = &t->table[uhtid - 1];
+		if (!link->link_handle)
+			return -EINVAL;
+
+		if (!test_bit(filter_id, link->tid_map))
+			return -EINVAL;
+	}
+
+	ret = cxgb4_del_filter(dev, filter_id);
+	if (ret)
+		goto out;
+
+	if (link)
+		clear_bit(filter_id, link->tid_map);
+
+	/* If a link is being deleted, then delete all filters
+	 * associated with the link.
+	 */
+	max_tids = adapter->tids.nftids;
+	for (i = 0; i < t->size; i++) {
+		link = &t->table[i];
+
+		if (link->link_handle == handle) {
+			for (j = 0; j < max_tids; j++) {
+				if (!test_bit(j, link->tid_map))
+					continue;
+
+				ret = __cxgb4_del_filter(dev, j, NULL);
+				if (ret)
+					goto out;
+
+				clear_bit(j, link->tid_map);
+			}
+
+			/* Clear the link state */
+			link->match_field = NULL;
+			link->link_handle = 0;
+			memset(&link->fs, 0, sizeof(link->fs));
+			break;
+		}
+	}
+
+out:
+	return ret;
+}
+
+void cxgb4_cleanup_tc_u32(struct adapter *adap)
+{
+	struct cxgb4_tc_u32_table *t;
+	unsigned int i;
+
+	if (!adap->tc_u32)
+		return;
+
+	/* Free up all allocated memory. */
+	t = adap->tc_u32;
+	for (i = 0; i < t->size; i++) {
+		struct cxgb4_link *link = &t->table[i];
+
+		t4_free_mem(link->tid_map);
+	}
+	t4_free_mem(adap->tc_u32);
+}
+
+struct cxgb4_tc_u32_table *cxgb4_init_tc_u32(struct adapter *adap,
+					     unsigned int size)
+{
+	struct cxgb4_tc_u32_table *t;
+	unsigned int i;
+
+	if (!size)
+		return NULL;
+
+	t = t4_alloc_mem(sizeof(*t) +
+			 (size * sizeof(struct cxgb4_link)));
+	if (!t)
+		return NULL;
+
+	t->size = size;
+
+	for (i = 0; i < t->size; i++) {
+		struct cxgb4_link *link = &t->table[i];
+		unsigned int bmap_size;
+		unsigned int max_tids;
+
+		max_tids = adap->tids.nftids;
+		bmap_size = BITS_TO_LONGS(max_tids);
+		link->tid_map = t4_alloc_mem(sizeof(unsigned long) * bmap_size);
+		if (!link->tid_map)
+			goto out_no_mem;
+		bitmap_zero(link->tid_map, max_tids);
+	}
+
+	return t;
+
+out_no_mem:
+	for (i = 0; i < t->size; i++) {
+		struct cxgb4_link *link = &t->table[i];
+
+		if (link->tid_map)
+			t4_free_mem(link->tid_map);
+	}
+
+	if (t)
+		t4_free_mem(t);
+
+	return NULL;
+}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h
new file mode 100644
index 000000000000..6bdc885eff22
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h
@@ -0,0 +1,57 @@
+/*
+ * This file is part of the Chelsio T4 Ethernet driver for Linux.
+ *
+ * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __CXGB4_TC_U32_H
+#define __CXGB4_TC_U32_H
+
+#include <net/pkt_cls.h>
+
+#define CXGB4_MAX_LINK_HANDLE 32
+
+static inline bool can_tc_u32_offload(struct net_device *dev)
+{
+	struct adapter *adap = netdev2adap(dev);
+
+	return (dev->features & NETIF_F_HW_TC) && adap->tc_u32 ? true : false;
+}
+
+int cxgb4_config_knode(struct net_device *dev, __be16 protocol,
+		       struct tc_cls_u32_offload *cls);
+int cxgb4_delete_knode(struct net_device *dev, __be16 protocol,
+		       struct tc_cls_u32_offload *cls);
+
+void cxgb4_cleanup_tc_u32(struct adapter *adapter);
+struct cxgb4_tc_u32_table *cxgb4_init_tc_u32(struct adapter *adap,
+					     unsigned int size);
+#endif /* __CXGB4_TC_U32_H */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32_parse.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32_parse.h
new file mode 100644
index 000000000000..a4b99edcc339
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32_parse.h
@@ -0,0 +1,294 @@
+/*
+ * This file is part of the Chelsio T4 Ethernet driver for Linux.
+ *
+ * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __CXGB4_TC_U32_PARSE_H
+#define __CXGB4_TC_U32_PARSE_H
+
+struct cxgb4_match_field {
+	int off; /* Offset from the beginning of the header to match */
+	/* Fill the value/mask pair in the spec if matched */
+	int (*val)(struct ch_filter_specification *f, u32 val, u32 mask);
+};
+
+/* IPv4 match fields */
+static inline int cxgb4_fill_ipv4_tos(struct ch_filter_specification *f,
+				      u32 val, u32 mask)
+{
+	f->val.tos  = (ntohl(val)  >> 16) & 0x000000FF;
+	f->mask.tos = (ntohl(mask) >> 16) & 0x000000FF;
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv4_frag(struct ch_filter_specification *f,
+				       u32 val, u32 mask)
+{
+	u32 mask_val;
+	u8 frag_val;
+
+	frag_val = (ntohl(val) >> 13) & 0x00000007;
+	mask_val = ntohl(mask) & 0x0000FFFF;
+
+	if (frag_val == 0x1 && mask_val != 0x3FFF) { /* MF set */
+		f->val.frag = 1;
+		f->mask.frag = 1;
+	} else if (frag_val == 0x2 && mask_val != 0x3FFF) { /* DF set */
+		f->val.frag = 0;
+		f->mask.frag = 1;
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv4_proto(struct ch_filter_specification *f,
+					u32 val, u32 mask)
+{
+	f->val.proto  = (ntohl(val)  >> 16) & 0x000000FF;
+	f->mask.proto = (ntohl(mask) >> 16) & 0x000000FF;
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv4_src_ip(struct ch_filter_specification *f,
+					 u32 val, u32 mask)
+{
+	memcpy(&f->val.fip[0],  &val,  sizeof(u32));
+	memcpy(&f->mask.fip[0], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv4_dst_ip(struct ch_filter_specification *f,
+					 u32 val, u32 mask)
+{
+	memcpy(&f->val.lip[0],  &val,  sizeof(u32));
+	memcpy(&f->mask.lip[0], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static const struct cxgb4_match_field cxgb4_ipv4_fields[] = {
+	{ .off = 0,  .val = cxgb4_fill_ipv4_tos },
+	{ .off = 4,  .val = cxgb4_fill_ipv4_frag },
+	{ .off = 8,  .val = cxgb4_fill_ipv4_proto },
+	{ .off = 12, .val = cxgb4_fill_ipv4_src_ip },
+	{ .off = 16, .val = cxgb4_fill_ipv4_dst_ip },
+	{ .val = NULL }
+};
+
+/* IPv6 match fields */
+static inline int cxgb4_fill_ipv6_tos(struct ch_filter_specification *f,
+				      u32 val, u32 mask)
+{
+	f->val.tos  = (ntohl(val)  >> 20) & 0x000000FF;
+	f->mask.tos = (ntohl(mask) >> 20) & 0x000000FF;
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv6_proto(struct ch_filter_specification *f,
+					u32 val, u32 mask)
+{
+	f->val.proto  = (ntohl(val)  >> 8) & 0x000000FF;
+	f->mask.proto = (ntohl(mask) >> 8) & 0x000000FF;
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv6_src_ip0(struct ch_filter_specification *f,
+					  u32 val, u32 mask)
+{
+	memcpy(&f->val.fip[0],  &val,  sizeof(u32));
+	memcpy(&f->mask.fip[0], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv6_src_ip1(struct ch_filter_specification *f,
+					  u32 val, u32 mask)
+{
+	memcpy(&f->val.fip[4],  &val,  sizeof(u32));
+	memcpy(&f->mask.fip[4], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv6_src_ip2(struct ch_filter_specification *f,
+					  u32 val, u32 mask)
+{
+	memcpy(&f->val.fip[8],  &val,  sizeof(u32));
+	memcpy(&f->mask.fip[8], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv6_src_ip3(struct ch_filter_specification *f,
+					  u32 val, u32 mask)
+{
+	memcpy(&f->val.fip[12],  &val,  sizeof(u32));
+	memcpy(&f->mask.fip[12], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv6_dst_ip0(struct ch_filter_specification *f,
+					  u32 val, u32 mask)
+{
+	memcpy(&f->val.lip[0],  &val,  sizeof(u32));
+	memcpy(&f->mask.lip[0], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv6_dst_ip1(struct ch_filter_specification *f,
+					  u32 val, u32 mask)
+{
+	memcpy(&f->val.lip[4],  &val,  sizeof(u32));
+	memcpy(&f->mask.lip[4], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv6_dst_ip2(struct ch_filter_specification *f,
+					  u32 val, u32 mask)
+{
+	memcpy(&f->val.lip[8],  &val,  sizeof(u32));
+	memcpy(&f->mask.lip[8], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static inline int cxgb4_fill_ipv6_dst_ip3(struct ch_filter_specification *f,
+					  u32 val, u32 mask)
+{
+	memcpy(&f->val.lip[12],  &val,  sizeof(u32));
+	memcpy(&f->mask.lip[12], &mask, sizeof(u32));
+
+	return 0;
+}
+
+static const struct cxgb4_match_field cxgb4_ipv6_fields[] = {
+	{ .off = 0,  .val = cxgb4_fill_ipv6_tos },
+	{ .off = 4,  .val = cxgb4_fill_ipv6_proto },
+	{ .off = 8,  .val = cxgb4_fill_ipv6_src_ip0 },
+	{ .off = 12, .val = cxgb4_fill_ipv6_src_ip1 },
+	{ .off = 16, .val = cxgb4_fill_ipv6_src_ip2 },
+	{ .off = 20, .val = cxgb4_fill_ipv6_src_ip3 },
+	{ .off = 24, .val = cxgb4_fill_ipv6_dst_ip0 },
+	{ .off = 28, .val = cxgb4_fill_ipv6_dst_ip1 },
+	{ .off = 32, .val = cxgb4_fill_ipv6_dst_ip2 },
+	{ .off = 36, .val = cxgb4_fill_ipv6_dst_ip3 },
+	{ .val = NULL }
+};
+
+/* TCP/UDP match */
+static inline int cxgb4_fill_l4_ports(struct ch_filter_specification *f,
+				      u32 val, u32 mask)
+{
+	f->val.fport  = ntohl(val)  >> 16;
+	f->mask.fport = ntohl(mask) >> 16;
+	f->val.lport  = ntohl(val)  & 0x0000FFFF;
+	f->mask.lport = ntohl(mask) & 0x0000FFFF;
+
+	return 0;
+};
+
+static const struct cxgb4_match_field cxgb4_tcp_fields[] = {
+	{ .off = 0, .val = cxgb4_fill_l4_ports },
+	{ .val = NULL }
+};
+
+static const struct cxgb4_match_field cxgb4_udp_fields[] = {
+	{ .off = 0, .val = cxgb4_fill_l4_ports },
+	{ .val = NULL }
+};
+
+struct cxgb4_next_header {
+	unsigned int offset; /* Offset to next header */
+	/* offset, shift, and mask added to offset above
+	 * to get to next header.  Useful when using a header
+	 * field's value to jump to next header such as IHL field
+	 * in IPv4 header.
+	 */
+	unsigned int offoff;
+	u32 shift;
+	u32 mask;
+	/* match criteria to make this jump */
+	unsigned int match_off;
+	u32 match_val;
+	u32 match_mask;
+	/* location of jump to make */
+	const struct cxgb4_match_field *jump;
+};
+
+/* Accept a rule with a jump to transport layer header based on IHL field in
+ * IPv4 header.
+ */
+static const struct cxgb4_next_header cxgb4_ipv4_jumps[] = {
+	{ .offset = 0, .offoff = 0, .shift = 6, .mask = 0xF,
+	  .match_off = 8, .match_val = 0x600, .match_mask = 0xFF00,
+	  .jump = cxgb4_tcp_fields },
+	{ .offset = 0, .offoff = 0, .shift = 6, .mask = 0xF,
+	  .match_off = 8, .match_val = 0x1100, .match_mask = 0xFF00,
+	  .jump = cxgb4_udp_fields },
+	{ .jump = NULL }
+};
+
+/* Accept a rule with a jump directly past the 40 Bytes of IPv6 fixed header
+ * to get to transport layer header.
+ */
+static const struct cxgb4_next_header cxgb4_ipv6_jumps[] = {
+	{ .offset = 0x28, .offoff = 0, .shift = 0, .mask = 0,
+	  .match_off = 4, .match_val = 0x60000, .match_mask = 0xFF0000,
+	  .jump = cxgb4_tcp_fields },
+	{ .offset = 0x28, .offoff = 0, .shift = 0, .mask = 0,
+	  .match_off = 4, .match_val = 0x110000, .match_mask = 0xFF0000,
+	  .jump = cxgb4_udp_fields },
+	{ .jump = NULL }
+};
+
+struct cxgb4_link {
+	const struct cxgb4_match_field *match_field;  /* Next header */
+	struct ch_filter_specification fs; /* Match spec associated with link */
+	u32 link_handle;         /* Knode handle associated with the link */
+	unsigned long *tid_map;  /* Bitmap for filter tids */
+};
+
+struct cxgb4_tc_u32_table {
+	unsigned int size;          /* number of entries in table */
+	struct cxgb4_link table[0]; /* Jump table */
+};
+#endif /* __CXGB4_TC_U32_PARSE_H */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
new file mode 100644
index 000000000000..0945fa49a5dd
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
@@ -0,0 +1,697 @@
+/*
+ * cxgb4_uld.c:Chelsio Upper Layer Driver Interface for T4/T5/T6 SGE management
+ *
+ * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ *  Written by: Atul Gupta (atul.gupta@chelsio.com)
+ *  Written by: Hariprasad Shenai (hariprasad@chelsio.com)
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/debugfs.h>
+#include <linux/export.h>
+#include <linux/list.h>
+#include <linux/skbuff.h>
+#include <linux/pci.h>
+
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+#include "t4_regs.h"
+#include "t4fw_api.h"
+#include "t4_msg.h"
+
+#define for_each_uldrxq(m, i) for (i = 0; i < ((m)->nrxq + (m)->nciq); i++)
+
+static int get_msix_idx_from_bmap(struct adapter *adap)
+{
+	struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
+	unsigned long flags;
+	unsigned int msix_idx;
+
+	spin_lock_irqsave(&bmap->lock, flags);
+	msix_idx = find_first_zero_bit(bmap->msix_bmap, bmap->mapsize);
+	if (msix_idx < bmap->mapsize) {
+		__set_bit(msix_idx, bmap->msix_bmap);
+	} else {
+		spin_unlock_irqrestore(&bmap->lock, flags);
+		return -ENOSPC;
+	}
+
+	spin_unlock_irqrestore(&bmap->lock, flags);
+	return msix_idx;
+}
+
+static void free_msix_idx_in_bmap(struct adapter *adap, unsigned int msix_idx)
+{
+	struct uld_msix_bmap *bmap = &adap->msix_bmap_ulds;
+	unsigned long flags;
+
+	spin_lock_irqsave(&bmap->lock, flags);
+	 __clear_bit(msix_idx, bmap->msix_bmap);
+	spin_unlock_irqrestore(&bmap->lock, flags);
+}
+
+/* Flush the aggregated lro sessions */
+static void uldrx_flush_handler(struct sge_rspq *q)
+{
+	struct adapter *adap = q->adap;
+
+	if (adap->uld[q->uld].lro_flush)
+		adap->uld[q->uld].lro_flush(&q->lro_mgr);
+}
+
+/**
+ *	uldrx_handler - response queue handler for ULD queues
+ *	@q: the response queue that received the packet
+ *	@rsp: the response queue descriptor holding the offload message
+ *	@gl: the gather list of packet fragments
+ *
+ *	Deliver an ingress offload packet to a ULD.  All processing is done by
+ *	the ULD, we just maintain statistics.
+ */
+static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
+			 const struct pkt_gl *gl)
+{
+	struct adapter *adap = q->adap;
+	struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
+	int ret;
+
+	/* FW can send CPLs encapsulated in a CPL_FW4_MSG */
+	if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
+	    ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
+		rsp += 2;
+
+	if (q->flush_handler)
+		ret = adap->uld[q->uld].lro_rx_handler(adap->uld[q->uld].handle,
+				rsp, gl, &q->lro_mgr,
+				&q->napi);
+	else
+		ret = adap->uld[q->uld].rx_handler(adap->uld[q->uld].handle,
+				rsp, gl);
+
+	if (ret) {
+		rxq->stats.nomem++;
+		return -1;
+	}
+
+	if (!gl)
+		rxq->stats.imm++;
+	else if (gl == CXGB4_MSG_AN)
+		rxq->stats.an++;
+	else
+		rxq->stats.pkts++;
+	return 0;
+}
+
+static int alloc_uld_rxqs(struct adapter *adap,
+			  struct sge_uld_rxq_info *rxq_info,
+			  unsigned int nq, unsigned int offset, bool lro)
+{
+	struct sge *s = &adap->sge;
+	struct sge_ofld_rxq *q = rxq_info->uldrxq + offset;
+	unsigned short *ids = rxq_info->rspq_id + offset;
+	unsigned int per_chan = nq / adap->params.nports;
+	unsigned int bmap_idx = 0;
+	int i, err, msi_idx;
+
+	if (adap->flags & USING_MSIX)
+		msi_idx = 1;
+	else
+		msi_idx = -((int)s->intrq.abs_id + 1);
+
+	for (i = 0; i < nq; i++, q++) {
+		if (msi_idx >= 0) {
+			bmap_idx = get_msix_idx_from_bmap(adap);
+			msi_idx = adap->msix_info_ulds[bmap_idx].idx;
+		}
+		err = t4_sge_alloc_rxq(adap, &q->rspq, false,
+				       adap->port[i / per_chan],
+				       msi_idx,
+				       q->fl.size ? &q->fl : NULL,
+				       uldrx_handler,
+				       lro ? uldrx_flush_handler : NULL,
+				       0);
+		if (err)
+			goto freeout;
+		if (msi_idx >= 0)
+			rxq_info->msix_tbl[i + offset] = bmap_idx;
+		memset(&q->stats, 0, sizeof(q->stats));
+		if (ids)
+			ids[i] = q->rspq.abs_id;
+	}
+	return 0;
+freeout:
+	q = rxq_info->uldrxq + offset;
+	for ( ; i; i--, q++) {
+		if (q->rspq.desc)
+			free_rspq_fl(adap, &q->rspq,
+				     q->fl.size ? &q->fl : NULL);
+	}
+
+	/* We need to free rxq also in case of ciq allocation failure */
+	if (offset) {
+		q = rxq_info->uldrxq + offset;
+		for ( ; i; i--, q++) {
+			if (q->rspq.desc)
+				free_rspq_fl(adap, &q->rspq,
+					     q->fl.size ? &q->fl : NULL);
+		}
+	}
+	return err;
+}
+
+static int
+setup_sge_queues_uld(struct adapter *adap, unsigned int uld_type, bool lro)
+{
+	struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+	int i, ret = 0;
+
+	if (adap->flags & USING_MSIX) {
+		rxq_info->msix_tbl = kcalloc((rxq_info->nrxq + rxq_info->nciq),
+					     sizeof(unsigned short),
+					     GFP_KERNEL);
+		if (!rxq_info->msix_tbl)
+			return -ENOMEM;
+	}
+
+	ret = !(!alloc_uld_rxqs(adap, rxq_info, rxq_info->nrxq, 0, lro) &&
+		 !alloc_uld_rxqs(adap, rxq_info, rxq_info->nciq,
+				 rxq_info->nrxq, lro));
+
+	/* Tell uP to route control queue completions to rdma rspq */
+	if (adap->flags & FULL_INIT_DONE &&
+	    !ret && uld_type == CXGB4_ULD_RDMA) {
+		struct sge *s = &adap->sge;
+		unsigned int cmplqid;
+		u32 param, cmdop;
+
+		cmdop = FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL;
+		for_each_port(adap, i) {
+			cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id;
+			param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
+				 FW_PARAMS_PARAM_X_V(cmdop) |
+				 FW_PARAMS_PARAM_YZ_V(s->ctrlq[i].q.cntxt_id));
+			ret = t4_set_params(adap, adap->mbox, adap->pf,
+					    0, 1, &param, &cmplqid);
+		}
+	}
+	return ret;
+}
+
+static void t4_free_uld_rxqs(struct adapter *adap, int n,
+			     struct sge_ofld_rxq *q)
+{
+	for ( ; n; n--, q++) {
+		if (q->rspq.desc)
+			free_rspq_fl(adap, &q->rspq,
+				     q->fl.size ? &q->fl : NULL);
+	}
+}
+
+static void free_sge_queues_uld(struct adapter *adap, unsigned int uld_type)
+{
+	struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+
+	if (adap->flags & FULL_INIT_DONE && uld_type == CXGB4_ULD_RDMA) {
+		struct sge *s = &adap->sge;
+		u32 param, cmdop, cmplqid = 0;
+		int i;
+
+		cmdop = FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL;
+		for_each_port(adap, i) {
+			param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
+				 FW_PARAMS_PARAM_X_V(cmdop) |
+				 FW_PARAMS_PARAM_YZ_V(s->ctrlq[i].q.cntxt_id));
+			t4_set_params(adap, adap->mbox, adap->pf,
+				      0, 1, &param, &cmplqid);
+		}
+	}
+
+	if (rxq_info->nciq)
+		t4_free_uld_rxqs(adap, rxq_info->nciq,
+				 rxq_info->uldrxq + rxq_info->nrxq);
+	t4_free_uld_rxqs(adap, rxq_info->nrxq, rxq_info->uldrxq);
+	if (adap->flags & USING_MSIX)
+		kfree(rxq_info->msix_tbl);
+}
+
+static int cfg_queues_uld(struct adapter *adap, unsigned int uld_type,
+			  const struct cxgb4_uld_info *uld_info)
+{
+	struct sge *s = &adap->sge;
+	struct sge_uld_rxq_info *rxq_info;
+	int i, nrxq, ciq_size;
+
+	rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);
+	if (!rxq_info)
+		return -ENOMEM;
+
+	if (adap->flags & USING_MSIX && uld_info->nrxq > s->nqs_per_uld) {
+		i = s->nqs_per_uld;
+		rxq_info->nrxq = roundup(i, adap->params.nports);
+	} else {
+		i = min_t(int, uld_info->nrxq,
+			  num_online_cpus());
+		rxq_info->nrxq = roundup(i, adap->params.nports);
+	}
+	if (!uld_info->ciq) {
+		rxq_info->nciq = 0;
+	} else  {
+		if (adap->flags & USING_MSIX)
+			rxq_info->nciq = min_t(int, s->nqs_per_uld,
+					       num_online_cpus());
+		else
+			rxq_info->nciq = min_t(int, MAX_OFLD_QSETS,
+					       num_online_cpus());
+		rxq_info->nciq = ((rxq_info->nciq / adap->params.nports) *
+				  adap->params.nports);
+		rxq_info->nciq = max_t(int, rxq_info->nciq,
+				       adap->params.nports);
+	}
+
+	nrxq = rxq_info->nrxq + rxq_info->nciq; /* total rxq's */
+	rxq_info->uldrxq = kcalloc(nrxq, sizeof(struct sge_ofld_rxq),
+				   GFP_KERNEL);
+	if (!rxq_info->uldrxq) {
+		kfree(rxq_info);
+		return -ENOMEM;
+	}
+
+	rxq_info->rspq_id = kcalloc(nrxq, sizeof(unsigned short), GFP_KERNEL);
+	if (!rxq_info->rspq_id) {
+		kfree(rxq_info->uldrxq);
+		kfree(rxq_info);
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < rxq_info->nrxq; i++) {
+		struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
+
+		init_rspq(adap, &r->rspq, 5, 1, uld_info->rxq_size, 64);
+		r->rspq.uld = uld_type;
+		r->fl.size = 72;
+	}
+
+	ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
+	if (ciq_size > SGE_MAX_IQ_SIZE) {
+		dev_warn(adap->pdev_dev, "CIQ size too small for available IQs\n");
+		ciq_size = SGE_MAX_IQ_SIZE;
+	}
+
+	for (i = rxq_info->nrxq; i < nrxq; i++) {
+		struct sge_ofld_rxq *r = &rxq_info->uldrxq[i];
+
+		init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
+		r->rspq.uld = uld_type;
+	}
+
+	memcpy(rxq_info->name, uld_info->name, IFNAMSIZ);
+	adap->sge.uld_rxq_info[uld_type] = rxq_info;
+
+	return 0;
+}
+
+static void free_queues_uld(struct adapter *adap, unsigned int uld_type)
+{
+	struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+
+	kfree(rxq_info->rspq_id);
+	kfree(rxq_info->uldrxq);
+	kfree(rxq_info);
+}
+
+static int
+request_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
+{
+	struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+	int err = 0;
+	unsigned int idx, bmap_idx;
+
+	for_each_uldrxq(rxq_info, idx) {
+		bmap_idx = rxq_info->msix_tbl[idx];
+		err = request_irq(adap->msix_info_ulds[bmap_idx].vec,
+				  t4_sge_intr_msix, 0,
+				  adap->msix_info_ulds[bmap_idx].desc,
+				  &rxq_info->uldrxq[idx].rspq);
+		if (err)
+			goto unwind;
+	}
+	return 0;
+unwind:
+	while (idx-- > 0) {
+		bmap_idx = rxq_info->msix_tbl[idx];
+		free_msix_idx_in_bmap(adap, bmap_idx);
+		free_irq(adap->msix_info_ulds[bmap_idx].vec,
+			 &rxq_info->uldrxq[idx].rspq);
+	}
+	return err;
+}
+
+static void
+free_msix_queue_irqs_uld(struct adapter *adap, unsigned int uld_type)
+{
+	struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+	unsigned int idx, bmap_idx;
+
+	for_each_uldrxq(rxq_info, idx) {
+		bmap_idx = rxq_info->msix_tbl[idx];
+
+		free_msix_idx_in_bmap(adap, bmap_idx);
+		free_irq(adap->msix_info_ulds[bmap_idx].vec,
+			 &rxq_info->uldrxq[idx].rspq);
+	}
+}
+
+static void name_msix_vecs_uld(struct adapter *adap, unsigned int uld_type)
+{
+	struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+	int n = sizeof(adap->msix_info_ulds[0].desc);
+	unsigned int idx, bmap_idx;
+
+	for_each_uldrxq(rxq_info, idx) {
+		bmap_idx = rxq_info->msix_tbl[idx];
+
+		snprintf(adap->msix_info_ulds[bmap_idx].desc, n, "%s-%s%d",
+			 adap->port[0]->name, rxq_info->name, idx);
+	}
+}
+
+static void enable_rx(struct adapter *adap, struct sge_rspq *q)
+{
+	if (!q)
+		return;
+
+	if (q->handler) {
+		cxgb_busy_poll_init_lock(q);
+		napi_enable(&q->napi);
+	}
+	/* 0-increment GTS to start the timer and enable interrupts */
+	t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
+		     SEINTARM_V(q->intr_params) |
+		     INGRESSQID_V(q->cntxt_id));
+}
+
+static void quiesce_rx(struct adapter *adap, struct sge_rspq *q)
+{
+	if (q && q->handler) {
+		napi_disable(&q->napi);
+		local_bh_disable();
+		while (!cxgb_poll_lock_napi(q))
+			mdelay(1);
+		local_bh_enable();
+	}
+}
+
+static void enable_rx_uld(struct adapter *adap, unsigned int uld_type)
+{
+	struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+	int idx;
+
+	for_each_uldrxq(rxq_info, idx)
+		enable_rx(adap, &rxq_info->uldrxq[idx].rspq);
+}
+
+static void quiesce_rx_uld(struct adapter *adap, unsigned int uld_type)
+{
+	struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+	int idx;
+
+	for_each_uldrxq(rxq_info, idx)
+		quiesce_rx(adap, &rxq_info->uldrxq[idx].rspq);
+}
+
+static void uld_queue_init(struct adapter *adap, unsigned int uld_type,
+			   struct cxgb4_lld_info *lli)
+{
+	struct sge_uld_rxq_info *rxq_info = adap->sge.uld_rxq_info[uld_type];
+
+	lli->rxq_ids = rxq_info->rspq_id;
+	lli->nrxq = rxq_info->nrxq;
+	lli->ciq_ids = rxq_info->rspq_id + rxq_info->nrxq;
+	lli->nciq = rxq_info->nciq;
+}
+
+int t4_uld_mem_alloc(struct adapter *adap)
+{
+	struct sge *s = &adap->sge;
+
+	adap->uld = kcalloc(CXGB4_ULD_MAX, sizeof(*adap->uld), GFP_KERNEL);
+	if (!adap->uld)
+		return -ENOMEM;
+
+	s->uld_rxq_info = kzalloc(CXGB4_ULD_MAX *
+				  sizeof(struct sge_uld_rxq_info *),
+				  GFP_KERNEL);
+	if (!s->uld_rxq_info)
+		goto err_uld;
+
+	return 0;
+err_uld:
+	kfree(adap->uld);
+	return -ENOMEM;
+}
+
+void t4_uld_mem_free(struct adapter *adap)
+{
+	struct sge *s = &adap->sge;
+
+	kfree(s->uld_rxq_info);
+	kfree(adap->uld);
+}
+
+void t4_uld_clean_up(struct adapter *adap)
+{
+	struct sge_uld_rxq_info *rxq_info;
+	unsigned int i;
+
+	if (!adap->uld)
+		return;
+	for (i = 0; i < CXGB4_ULD_MAX; i++) {
+		if (!adap->uld[i].handle)
+			continue;
+		rxq_info = adap->sge.uld_rxq_info[i];
+		if (adap->flags & FULL_INIT_DONE)
+			quiesce_rx_uld(adap, i);
+		if (adap->flags & USING_MSIX)
+			free_msix_queue_irqs_uld(adap, i);
+		free_sge_queues_uld(adap, i);
+		free_queues_uld(adap, i);
+	}
+}
+
+static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld)
+{
+	int i;
+
+	lld->pdev = adap->pdev;
+	lld->pf = adap->pf;
+	lld->l2t = adap->l2t;
+	lld->tids = &adap->tids;
+	lld->ports = adap->port;
+	lld->vr = &adap->vres;
+	lld->mtus = adap->params.mtus;
+	lld->ntxq = adap->sge.ofldqsets;
+	lld->nchan = adap->params.nports;
+	lld->nports = adap->params.nports;
+	lld->wr_cred = adap->params.ofldq_wr_cred;
+	lld->iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
+	lld->iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK_A);
+	lld->iscsi_pgsz_order = t4_read_reg(adap, ULP_RX_ISCSI_PSZ_A);
+	lld->iscsi_llimit = t4_read_reg(adap, ULP_RX_ISCSI_LLIMIT_A);
+	lld->iscsi_ppm = &adap->iscsi_ppm;
+	lld->adapter_type = adap->params.chip;
+	lld->cclk_ps = 1000000000 / adap->params.vpd.cclk;
+	lld->udb_density = 1 << adap->params.sge.eq_qpp;
+	lld->ucq_density = 1 << adap->params.sge.iq_qpp;
+	lld->filt_mode = adap->params.tp.vlan_pri_map;
+	/* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
+	for (i = 0; i < NCHAN; i++)
+		lld->tx_modq[i] = i;
+	lld->gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
+	lld->db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
+	lld->fw_vers = adap->params.fw_vers;
+	lld->dbfifo_int_thresh = dbfifo_int_thresh;
+	lld->sge_ingpadboundary = adap->sge.fl_align;
+	lld->sge_egrstatuspagesize = adap->sge.stat_len;
+	lld->sge_pktshift = adap->sge.pktshift;
+	lld->enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
+	lld->max_ordird_qp = adap->params.max_ordird_qp;
+	lld->max_ird_adapter = adap->params.max_ird_adapter;
+	lld->ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
+	lld->nodeid = dev_to_node(adap->pdev_dev);
+	lld->fr_nsmr_tpte_wr_support = adap->params.fr_nsmr_tpte_wr_support;
+}
+
+static void uld_attach(struct adapter *adap, unsigned int uld)
+{
+	void *handle;
+	struct cxgb4_lld_info lli;
+
+	uld_init(adap, &lli);
+	uld_queue_init(adap, uld, &lli);
+
+	handle = adap->uld[uld].add(&lli);
+	if (IS_ERR(handle)) {
+		dev_warn(adap->pdev_dev,
+			 "could not attach to the %s driver, error %ld\n",
+			 adap->uld[uld].name, PTR_ERR(handle));
+		return;
+	}
+
+	adap->uld[uld].handle = handle;
+	t4_register_netevent_notifier();
+
+	if (adap->flags & FULL_INIT_DONE)
+		adap->uld[uld].state_change(handle, CXGB4_STATE_UP);
+}
+
+/**
+ *	cxgb4_register_uld - register an upper-layer driver
+ *	@type: the ULD type
+ *	@p: the ULD methods
+ *
+ *	Registers an upper-layer driver with this driver and notifies the ULD
+ *	about any presently available devices that support its type.  Returns
+ *	%-EBUSY if a ULD of the same type is already registered.
+ */
+int cxgb4_register_uld(enum cxgb4_uld type,
+		       const struct cxgb4_uld_info *p)
+{
+	int ret = 0;
+	unsigned int adap_idx = 0;
+	struct adapter *adap;
+
+	if (type >= CXGB4_ULD_MAX)
+		return -EINVAL;
+
+	mutex_lock(&uld_mutex);
+	list_for_each_entry(adap, &adapter_list, list_node) {
+		if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) ||
+		    (type != CXGB4_ULD_CRYPTO && !is_offload(adap)))
+			continue;
+		if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip))
+			continue;
+		ret = cfg_queues_uld(adap, type, p);
+		if (ret)
+			goto out;
+		ret = setup_sge_queues_uld(adap, type, p->lro);
+		if (ret)
+			goto free_queues;
+		if (adap->flags & USING_MSIX) {
+			name_msix_vecs_uld(adap, type);
+			ret = request_msix_queue_irqs_uld(adap, type);
+			if (ret)
+				goto free_rxq;
+		}
+		if (adap->flags & FULL_INIT_DONE)
+			enable_rx_uld(adap, type);
+		if (adap->uld[type].add) {
+			ret = -EBUSY;
+			goto free_irq;
+		}
+		adap->uld[type] = *p;
+		uld_attach(adap, type);
+		adap_idx++;
+	}
+	mutex_unlock(&uld_mutex);
+	return 0;
+
+free_irq:
+	if (adap->flags & FULL_INIT_DONE)
+		quiesce_rx_uld(adap, type);
+	if (adap->flags & USING_MSIX)
+		free_msix_queue_irqs_uld(adap, type);
+free_rxq:
+	free_sge_queues_uld(adap, type);
+free_queues:
+	free_queues_uld(adap, type);
+out:
+
+	list_for_each_entry(adap, &adapter_list, list_node) {
+		if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) ||
+		    (type != CXGB4_ULD_CRYPTO && !is_offload(adap)))
+			continue;
+		if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip))
+			continue;
+		if (!adap_idx)
+			break;
+		adap->uld[type].handle = NULL;
+		adap->uld[type].add = NULL;
+		if (adap->flags & FULL_INIT_DONE)
+			quiesce_rx_uld(adap, type);
+		if (adap->flags & USING_MSIX)
+			free_msix_queue_irqs_uld(adap, type);
+		free_sge_queues_uld(adap, type);
+		free_queues_uld(adap, type);
+		adap_idx--;
+	}
+	mutex_unlock(&uld_mutex);
+	return ret;
+}
+EXPORT_SYMBOL(cxgb4_register_uld);
+
+/**
+ *	cxgb4_unregister_uld - unregister an upper-layer driver
+ *	@type: the ULD type
+ *
+ *	Unregisters an existing upper-layer driver.
+ */
+int cxgb4_unregister_uld(enum cxgb4_uld type)
+{
+	struct adapter *adap;
+
+	if (type >= CXGB4_ULD_MAX)
+		return -EINVAL;
+
+	mutex_lock(&uld_mutex);
+	list_for_each_entry(adap, &adapter_list, list_node) {
+		if ((type == CXGB4_ULD_CRYPTO && !is_pci_uld(adap)) ||
+		    (type != CXGB4_ULD_CRYPTO && !is_offload(adap)))
+			continue;
+		if (type == CXGB4_ULD_ISCSIT && is_t4(adap->params.chip))
+			continue;
+		adap->uld[type].handle = NULL;
+		adap->uld[type].add = NULL;
+		if (adap->flags & FULL_INIT_DONE)
+			quiesce_rx_uld(adap, type);
+		if (adap->flags & USING_MSIX)
+			free_msix_queue_irqs_uld(adap, type);
+		free_sge_queues_uld(adap, type);
+		free_queues_uld(adap, type);
+	}
+	mutex_unlock(&uld_mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL(cxgb4_unregister_uld);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
index f3c58aaa932d..2996793b1aaa 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
@@ -1,7 +1,7 @@
 /*
  * This file is part of the Chelsio T4 Ethernet driver for Linux.
  *
- * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -32,8 +32,8 @@
  * SOFTWARE.
  */
 
-#ifndef __CXGB4_OFLD_H
-#define __CXGB4_OFLD_H
+#ifndef __CXGB4_ULD_H
+#define __CXGB4_ULD_H
 
 #include <linux/cache.h>
 #include <linux/spinlock.h>
@@ -42,6 +42,8 @@
 #include <linux/atomic.h>
 #include "cxgb4.h"
 
+#define MAX_ULD_QSETS 16
+
 /* CPL message priority levels */
 enum {
 	CPL_PRIORITY_DATA     = 0,  /* data messages */
@@ -104,6 +106,7 @@ struct tid_info {
 	unsigned int atid_base;
 
 	struct filter_entry *ftid_tab;
+	unsigned long *ftid_bmap;
 	unsigned int nftids;
 	unsigned int ftid_base;
 	unsigned int aftid_base;
@@ -124,6 +127,8 @@ struct tid_info {
 	atomic_t tids_in_use;
 	/* TIDs in the HASH */
 	atomic_t hash_tids_in_use;
+	/* lock for setting/clearing filter bitmap */
+	spinlock_t ftid_lock;
 };
 
 static inline void *lookup_tid(const struct tid_info *t, unsigned int tid)
@@ -183,15 +188,38 @@ int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
 			       unsigned int queue, bool ipv6);
 
+/* Filter operation context to allow callers of cxgb4_set_filter() and
+ * cxgb4_del_filter() to wait for an asynchronous completion.
+ */
+struct filter_ctx {
+	struct completion completion;	/* completion rendezvous */
+	void *closure;			/* caller's opaque information */
+	int result;			/* result of operation */
+	u32 tid;			/* to store tid */
+};
+
+struct ch_filter_specification;
+
+int __cxgb4_set_filter(struct net_device *dev, int filter_id,
+		       struct ch_filter_specification *fs,
+		       struct filter_ctx *ctx);
+int __cxgb4_del_filter(struct net_device *dev, int filter_id,
+		       struct filter_ctx *ctx);
+int cxgb4_set_filter(struct net_device *dev, int filter_id,
+		     struct ch_filter_specification *fs);
+int cxgb4_del_filter(struct net_device *dev, int filter_id);
+
 static inline void set_wr_txq(struct sk_buff *skb, int prio, int queue)
 {
 	skb_set_queue_mapping(skb, (queue << 1) | prio);
 }
 
 enum cxgb4_uld {
+	CXGB4_ULD_INIT,
 	CXGB4_ULD_RDMA,
 	CXGB4_ULD_ISCSI,
 	CXGB4_ULD_ISCSIT,
+	CXGB4_ULD_CRYPTO,
 	CXGB4_ULD_MAX
 };
 
@@ -280,10 +308,16 @@ struct cxgb4_lld_info {
 	unsigned int iscsi_llimit;	     /* chip's iscsi region llimit */
 	void **iscsi_ppm;		     /* iscsi page pod manager */
 	int nodeid;			     /* device numa node id */
+	bool fr_nsmr_tpte_wr_support;	     /* FW supports FR_NSMR_TPTE_WR */
 };
 
 struct cxgb4_uld_info {
 	const char *name;
+	void *handle;
+	unsigned int nrxq;
+	unsigned int rxq_size;
+	bool ciq;
+	bool lro;
 	void *(*add)(const struct cxgb4_lld_info *p);
 	int (*rx_handler)(void *handle, const __be64 *rsp,
 			  const struct pkt_gl *gl);
@@ -330,4 +364,4 @@ int cxgb4_bar2_sge_qregs(struct net_device *dev,
 			 u64 *pbar2_qoffset,
 			 unsigned int *pbar2_qid);
 
-#endif  /* !__CXGB4_OFLD_H */
+#endif  /* !__CXGB4_ULD_H */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sched.c b/drivers/net/ethernet/chelsio/cxgb4/sched.c
new file mode 100644
index 000000000000..539de764bbd3
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/cxgb4/sched.c
@@ -0,0 +1,556 @@
+/*
+ * This file is part of the Chelsio T4 Ethernet driver for Linux.
+ *
+ * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+
+#include "cxgb4.h"
+#include "sched.h"
+
+/* Spinlock must be held by caller */
+static int t4_sched_class_fw_cmd(struct port_info *pi,
+				 struct ch_sched_params *p,
+				 enum sched_fw_ops op)
+{
+	struct adapter *adap = pi->adapter;
+	struct sched_table *s = pi->sched_tbl;
+	struct sched_class *e;
+	int err = 0;
+
+	e = &s->tab[p->u.params.class];
+	switch (op) {
+	case SCHED_FW_OP_ADD:
+		err = t4_sched_params(adap, p->type,
+				      p->u.params.level, p->u.params.mode,
+				      p->u.params.rateunit,
+				      p->u.params.ratemode,
+				      p->u.params.channel, e->idx,
+				      p->u.params.minrate, p->u.params.maxrate,
+				      p->u.params.weight, p->u.params.pktsize);
+		break;
+	default:
+		err = -ENOTSUPP;
+		break;
+	}
+
+	return err;
+}
+
+/* Spinlock must be held by caller */
+static int t4_sched_bind_unbind_op(struct port_info *pi, void *arg,
+				   enum sched_bind_type type, bool bind)
+{
+	struct adapter *adap = pi->adapter;
+	u32 fw_mnem, fw_class, fw_param;
+	unsigned int pf = adap->pf;
+	unsigned int vf = 0;
+	int err = 0;
+
+	switch (type) {
+	case SCHED_QUEUE: {
+		struct sched_queue_entry *qe;
+
+		qe = (struct sched_queue_entry *)arg;
+
+		/* Create a template for the FW_PARAMS_CMD mnemonic and
+		 * value (TX Scheduling Class in this case).
+		 */
+		fw_mnem = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
+			   FW_PARAMS_PARAM_X_V(
+				   FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
+		fw_class = bind ? qe->param.class : FW_SCHED_CLS_NONE;
+		fw_param = (fw_mnem | FW_PARAMS_PARAM_YZ_V(qe->cntxt_id));
+
+		pf = adap->pf;
+		vf = 0;
+		break;
+	}
+	default:
+		err = -ENOTSUPP;
+		goto out;
+	}
+
+	err = t4_set_params(adap, adap->mbox, pf, vf, 1, &fw_param, &fw_class);
+
+out:
+	return err;
+}
+
+static struct sched_class *t4_sched_queue_lookup(struct port_info *pi,
+						 const unsigned int qid,
+						 int *index)
+{
+	struct sched_table *s = pi->sched_tbl;
+	struct sched_class *e, *end;
+	struct sched_class *found = NULL;
+	int i;
+
+	/* Look for a class with matching bound queue parameters */
+	end = &s->tab[s->sched_size];
+	for (e = &s->tab[0]; e != end; ++e) {
+		struct sched_queue_entry *qe;
+
+		i = 0;
+		if (e->state == SCHED_STATE_UNUSED)
+			continue;
+
+		list_for_each_entry(qe, &e->queue_list, list) {
+			if (qe->cntxt_id == qid) {
+				found = e;
+				if (index)
+					*index = i;
+				break;
+			}
+			i++;
+		}
+
+		if (found)
+			break;
+	}
+
+	return found;
+}
+
+static int t4_sched_queue_unbind(struct port_info *pi, struct ch_sched_queue *p)
+{
+	struct adapter *adap = pi->adapter;
+	struct sched_class *e;
+	struct sched_queue_entry *qe = NULL;
+	struct sge_eth_txq *txq;
+	unsigned int qid;
+	int index = -1;
+	int err = 0;
+
+	if (p->queue < 0 || p->queue >= pi->nqsets)
+		return -ERANGE;
+
+	txq = &adap->sge.ethtxq[pi->first_qset + p->queue];
+	qid = txq->q.cntxt_id;
+
+	/* Find the existing class that the queue is bound to */
+	e = t4_sched_queue_lookup(pi, qid, &index);
+	if (e && index >= 0) {
+		int i = 0;
+
+		spin_lock(&e->lock);
+		list_for_each_entry(qe, &e->queue_list, list) {
+			if (i == index)
+				break;
+			i++;
+		}
+		err = t4_sched_bind_unbind_op(pi, (void *)qe, SCHED_QUEUE,
+					      false);
+		if (err) {
+			spin_unlock(&e->lock);
+			goto out;
+		}
+
+		list_del(&qe->list);
+		t4_free_mem(qe);
+		if (atomic_dec_and_test(&e->refcnt)) {
+			e->state = SCHED_STATE_UNUSED;
+			memset(&e->info, 0, sizeof(e->info));
+		}
+		spin_unlock(&e->lock);
+	}
+out:
+	return err;
+}
+
+static int t4_sched_queue_bind(struct port_info *pi, struct ch_sched_queue *p)
+{
+	struct adapter *adap = pi->adapter;
+	struct sched_table *s = pi->sched_tbl;
+	struct sched_class *e;
+	struct sched_queue_entry *qe = NULL;
+	struct sge_eth_txq *txq;
+	unsigned int qid;
+	int err = 0;
+
+	if (p->queue < 0 || p->queue >= pi->nqsets)
+		return -ERANGE;
+
+	qe = t4_alloc_mem(sizeof(struct sched_queue_entry));
+	if (!qe)
+		return -ENOMEM;
+
+	txq = &adap->sge.ethtxq[pi->first_qset + p->queue];
+	qid = txq->q.cntxt_id;
+
+	/* Unbind queue from any existing class */
+	err = t4_sched_queue_unbind(pi, p);
+	if (err)
+		goto out;
+
+	/* Bind queue to specified class */
+	memset(qe, 0, sizeof(*qe));
+	qe->cntxt_id = qid;
+	memcpy(&qe->param, p, sizeof(qe->param));
+
+	e = &s->tab[qe->param.class];
+	spin_lock(&e->lock);
+	err = t4_sched_bind_unbind_op(pi, (void *)qe, SCHED_QUEUE, true);
+	if (err) {
+		t4_free_mem(qe);
+		spin_unlock(&e->lock);
+		goto out;
+	}
+
+	list_add_tail(&qe->list, &e->queue_list);
+	atomic_inc(&e->refcnt);
+	spin_unlock(&e->lock);
+out:
+	return err;
+}
+
+static void t4_sched_class_unbind_all(struct port_info *pi,
+				      struct sched_class *e,
+				      enum sched_bind_type type)
+{
+	if (!e)
+		return;
+
+	switch (type) {
+	case SCHED_QUEUE: {
+		struct sched_queue_entry *qe;
+
+		list_for_each_entry(qe, &e->queue_list, list)
+			t4_sched_queue_unbind(pi, &qe->param);
+		break;
+	}
+	default:
+		break;
+	}
+}
+
+static int t4_sched_class_bind_unbind_op(struct port_info *pi, void *arg,
+					 enum sched_bind_type type, bool bind)
+{
+	int err = 0;
+
+	if (!arg)
+		return -EINVAL;
+
+	switch (type) {
+	case SCHED_QUEUE: {
+		struct ch_sched_queue *qe = (struct ch_sched_queue *)arg;
+
+		if (bind)
+			err = t4_sched_queue_bind(pi, qe);
+		else
+			err = t4_sched_queue_unbind(pi, qe);
+		break;
+	}
+	default:
+		err = -ENOTSUPP;
+		break;
+	}
+
+	return err;
+}
+
+/**
+ * cxgb4_sched_class_bind - Bind an entity to a scheduling class
+ * @dev: net_device pointer
+ * @arg: Entity opaque data
+ * @type: Entity type (Queue)
+ *
+ * Binds an entity (queue) to a scheduling class.  If the entity
+ * is bound to another class, it will be unbound from the other class
+ * and bound to the class specified in @arg.
+ */
+int cxgb4_sched_class_bind(struct net_device *dev, void *arg,
+			   enum sched_bind_type type)
+{
+	struct port_info *pi = netdev2pinfo(dev);
+	struct sched_table *s;
+	int err = 0;
+	u8 class_id;
+
+	if (!can_sched(dev))
+		return -ENOTSUPP;
+
+	if (!arg)
+		return -EINVAL;
+
+	switch (type) {
+	case SCHED_QUEUE: {
+		struct ch_sched_queue *qe = (struct ch_sched_queue *)arg;
+
+		class_id = qe->class;
+		break;
+	}
+	default:
+		return -ENOTSUPP;
+	}
+
+	if (!valid_class_id(dev, class_id))
+		return -EINVAL;
+
+	if (class_id == SCHED_CLS_NONE)
+		return -ENOTSUPP;
+
+	s = pi->sched_tbl;
+	write_lock(&s->rw_lock);
+	err = t4_sched_class_bind_unbind_op(pi, arg, type, true);
+	write_unlock(&s->rw_lock);
+
+	return err;
+}
+
+/**
+ * cxgb4_sched_class_unbind - Unbind an entity from a scheduling class
+ * @dev: net_device pointer
+ * @arg: Entity opaque data
+ * @type: Entity type (Queue)
+ *
+ * Unbinds an entity (queue) from a scheduling class.
+ */
+int cxgb4_sched_class_unbind(struct net_device *dev, void *arg,
+			     enum sched_bind_type type)
+{
+	struct port_info *pi = netdev2pinfo(dev);
+	struct sched_table *s;
+	int err = 0;
+	u8 class_id;
+
+	if (!can_sched(dev))
+		return -ENOTSUPP;
+
+	if (!arg)
+		return -EINVAL;
+
+	switch (type) {
+	case SCHED_QUEUE: {
+		struct ch_sched_queue *qe = (struct ch_sched_queue *)arg;
+
+		class_id = qe->class;
+		break;
+	}
+	default:
+		return -ENOTSUPP;
+	}
+
+	if (!valid_class_id(dev, class_id))
+		return -EINVAL;
+
+	s = pi->sched_tbl;
+	write_lock(&s->rw_lock);
+	err = t4_sched_class_bind_unbind_op(pi, arg, type, false);
+	write_unlock(&s->rw_lock);
+
+	return err;
+}
+
+/* If @p is NULL, fetch any available unused class */
+static struct sched_class *t4_sched_class_lookup(struct port_info *pi,
+						const struct ch_sched_params *p)
+{
+	struct sched_table *s = pi->sched_tbl;
+	struct sched_class *e, *end;
+	struct sched_class *found = NULL;
+
+	if (!p) {
+		/* Get any available unused class */
+		end = &s->tab[s->sched_size];
+		for (e = &s->tab[0]; e != end; ++e) {
+			if (e->state == SCHED_STATE_UNUSED) {
+				found = e;
+				break;
+			}
+		}
+	} else {
+		/* Look for a class with matching scheduling parameters */
+		struct ch_sched_params info;
+		struct ch_sched_params tp;
+
+		memset(&info, 0, sizeof(info));
+		memset(&tp, 0, sizeof(tp));
+
+		memcpy(&tp, p, sizeof(tp));
+		/* Don't try to match class parameter */
+		tp.u.params.class = SCHED_CLS_NONE;
+
+		end = &s->tab[s->sched_size];
+		for (e = &s->tab[0]; e != end; ++e) {
+			if (e->state == SCHED_STATE_UNUSED)
+				continue;
+
+			memset(&info, 0, sizeof(info));
+			memcpy(&info, &e->info, sizeof(info));
+			/* Don't try to match class parameter */
+			info.u.params.class = SCHED_CLS_NONE;
+
+			if ((info.type == tp.type) &&
+			    (!memcmp(&info.u.params, &tp.u.params,
+				     sizeof(info.u.params)))) {
+				found = e;
+				break;
+			}
+		}
+	}
+
+	return found;
+}
+
+static struct sched_class *t4_sched_class_alloc(struct port_info *pi,
+						struct ch_sched_params *p)
+{
+	struct sched_table *s = pi->sched_tbl;
+	struct sched_class *e;
+	u8 class_id;
+	int err;
+
+	if (!p)
+		return NULL;
+
+	class_id = p->u.params.class;
+
+	/* Only accept search for existing class with matching params
+	 * or allocation of new class with specified params
+	 */
+	if (class_id != SCHED_CLS_NONE)
+		return NULL;
+
+	write_lock(&s->rw_lock);
+	/* See if there's an exisiting class with same
+	 * requested sched params
+	 */
+	e = t4_sched_class_lookup(pi, p);
+	if (!e) {
+		struct ch_sched_params np;
+
+		/* Fetch any available unused class */
+		e = t4_sched_class_lookup(pi, NULL);
+		if (!e)
+			goto out;
+
+		memset(&np, 0, sizeof(np));
+		memcpy(&np, p, sizeof(np));
+		np.u.params.class = e->idx;
+
+		spin_lock(&e->lock);
+		/* New class */
+		err = t4_sched_class_fw_cmd(pi, &np, SCHED_FW_OP_ADD);
+		if (err) {
+			spin_unlock(&e->lock);
+			e = NULL;
+			goto out;
+		}
+		memcpy(&e->info, &np, sizeof(e->info));
+		atomic_set(&e->refcnt, 0);
+		e->state = SCHED_STATE_ACTIVE;
+		spin_unlock(&e->lock);
+	}
+
+out:
+	write_unlock(&s->rw_lock);
+	return e;
+}
+
+/**
+ * cxgb4_sched_class_alloc - allocate a scheduling class
+ * @dev: net_device pointer
+ * @p: new scheduling class to create.
+ *
+ * Returns pointer to the scheduling class created.  If @p is NULL, then
+ * it allocates and returns any available unused scheduling class. If a
+ * scheduling class with matching @p is found, then the matching class is
+ * returned.
+ */
+struct sched_class *cxgb4_sched_class_alloc(struct net_device *dev,
+					    struct ch_sched_params *p)
+{
+	struct port_info *pi = netdev2pinfo(dev);
+	u8 class_id;
+
+	if (!can_sched(dev))
+		return NULL;
+
+	class_id = p->u.params.class;
+	if (!valid_class_id(dev, class_id))
+		return NULL;
+
+	return t4_sched_class_alloc(pi, p);
+}
+
+static void t4_sched_class_free(struct port_info *pi, struct sched_class *e)
+{
+	t4_sched_class_unbind_all(pi, e, SCHED_QUEUE);
+}
+
+struct sched_table *t4_init_sched(unsigned int sched_size)
+{
+	struct sched_table *s;
+	unsigned int i;
+
+	s = t4_alloc_mem(sizeof(*s) + sched_size * sizeof(struct sched_class));
+	if (!s)
+		return NULL;
+
+	s->sched_size = sched_size;
+	rwlock_init(&s->rw_lock);
+
+	for (i = 0; i < s->sched_size; i++) {
+		memset(&s->tab[i], 0, sizeof(struct sched_class));
+		s->tab[i].idx = i;
+		s->tab[i].state = SCHED_STATE_UNUSED;
+		INIT_LIST_HEAD(&s->tab[i].queue_list);
+		spin_lock_init(&s->tab[i].lock);
+		atomic_set(&s->tab[i].refcnt, 0);
+	}
+	return s;
+}
+
+void t4_cleanup_sched(struct adapter *adap)
+{
+	struct sched_table *s;
+	unsigned int i;
+
+	for_each_port(adap, i) {
+		struct port_info *pi = netdev2pinfo(adap->port[i]);
+
+		s = pi->sched_tbl;
+		for (i = 0; i < s->sched_size; i++) {
+			struct sched_class *e;
+
+			write_lock(&s->rw_lock);
+			e = &s->tab[i];
+			if (e->state == SCHED_STATE_ACTIVE)
+				t4_sched_class_free(pi, e);
+			write_unlock(&s->rw_lock);
+		}
+		t4_free_mem(s);
+	}
+}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sched.h b/drivers/net/ethernet/chelsio/cxgb4/sched.h
new file mode 100644
index 000000000000..77b2b3fd9021
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/cxgb4/sched.h
@@ -0,0 +1,110 @@
+/*
+ * This file is part of the Chelsio T4 Ethernet driver for Linux.
+ *
+ * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __CXGB4_SCHED_H
+#define __CXGB4_SCHED_H
+
+#include <linux/spinlock.h>
+#include <linux/atomic.h>
+
+#define SCHED_CLS_NONE 0xff
+
+#define FW_SCHED_CLS_NONE 0xffffffff
+
+/* Max rate that can be set to a scheduling class is 10 Gbps */
+#define SCHED_MAX_RATE_KBPS 10000000U
+
+enum {
+	SCHED_STATE_ACTIVE,
+	SCHED_STATE_UNUSED,
+};
+
+enum sched_fw_ops {
+	SCHED_FW_OP_ADD,
+};
+
+enum sched_bind_type {
+	SCHED_QUEUE,
+};
+
+struct sched_queue_entry {
+	struct list_head list;
+	unsigned int cntxt_id;
+	struct ch_sched_queue param;
+};
+
+struct sched_class {
+	u8 state;
+	u8 idx;
+	struct ch_sched_params info;
+	struct list_head queue_list;
+	spinlock_t lock; /* Per class lock */
+	atomic_t refcnt;
+};
+
+struct sched_table {      /* per port scheduling table */
+	u8 sched_size;
+	rwlock_t rw_lock; /* Table lock */
+	struct sched_class tab[0];
+};
+
+static inline bool can_sched(struct net_device *dev)
+{
+	struct port_info *pi = netdev2pinfo(dev);
+
+	return !pi->sched_tbl ? false : true;
+}
+
+static inline bool valid_class_id(struct net_device *dev, u8 class_id)
+{
+	struct port_info *pi = netdev2pinfo(dev);
+
+	if ((class_id > pi->sched_tbl->sched_size - 1) &&
+	    (class_id != SCHED_CLS_NONE))
+		return false;
+
+	return true;
+}
+
+int cxgb4_sched_class_bind(struct net_device *dev, void *arg,
+			   enum sched_bind_type type);
+int cxgb4_sched_class_unbind(struct net_device *dev, void *arg,
+			     enum sched_bind_type type);
+
+struct sched_class *cxgb4_sched_class_alloc(struct net_device *dev,
+					    struct ch_sched_params *p);
+
+struct sched_table *t4_init_sched(unsigned int size);
+void t4_cleanup_sched(struct adapter *adap);
+#endif  /* __CXGB4_SCHED_H */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index ad3552df0545..1e74fd6085df 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -2860,6 +2860,18 @@ int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
 	return 0;
 }
 
+int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid,
+			unsigned int cmplqid)
+{
+	u32 param, val;
+
+	param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
+		 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL) |
+		 FW_PARAMS_PARAM_YZ_V(eqid));
+	val = cmplqid;
+	return t4_set_params(adap, adap->mbox, adap->pf, 0, 1, &param, &val);
+}
+
 int t4_sge_alloc_ofld_txq(struct adapter *adap, struct sge_ofld_txq *txq,
 			  struct net_device *dev, unsigned int iqid)
 {
@@ -2928,8 +2940,8 @@ static void free_txq(struct adapter *adap, struct sge_txq *q)
 	q->desc = NULL;
 }
 
-static void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq,
-			 struct sge_fl *fl)
+void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq,
+		  struct sge_fl *fl)
 {
 	struct sge *s = &adap->sge;
 	unsigned int fl_id = fl ? fl->cntxt_id : 0xffff;
@@ -3014,12 +3026,6 @@ void t4_free_sge_resources(struct adapter *adap)
 		}
 	}
 
-	/* clean up RDMA and iSCSI Rx queues */
-	t4_free_ofld_rxqs(adap, adap->sge.iscsiqsets, adap->sge.iscsirxq);
-	t4_free_ofld_rxqs(adap, adap->sge.niscsitq, adap->sge.iscsitrxq);
-	t4_free_ofld_rxqs(adap, adap->sge.rdmaqs, adap->sge.rdmarxq);
-	t4_free_ofld_rxqs(adap, adap->sge.rdmaciqs, adap->sge.rdmaciq);
-
 	/* clean up offload Tx queues */
 	for (i = 0; i < ARRAY_SIZE(adap->sge.ofldtxq); i++) {
 		struct sge_ofld_txq *q = &adap->sge.ofldtxq[i];
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
index dc92c80a75f4..20dec85da63d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
@@ -1,7 +1,7 @@
 /*
  * This file is part of the Chelsio T4 Ethernet driver for Linux.
  *
- * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
+ * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -2729,7 +2729,7 @@ int t4_get_raw_vpd_params(struct adapter *adapter, struct vpd_params *p)
 
 out:
 	vfree(vpd);
-	return ret;
+	return ret < 0 ? ret : 0;
 }
 
 /**
@@ -3627,7 +3627,8 @@ void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf)
 }
 
 #define ADVERT_MASK (FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G |\
-		     FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_40G | \
+		     FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_25G | \
+		     FW_PORT_CAP_SPEED_40G | FW_PORT_CAP_SPEED_100G | \
 		     FW_PORT_CAP_ANEG)
 
 /**
@@ -7196,8 +7197,12 @@ void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
 		speed = 1000;
 	else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G))
 		speed = 10000;
+	else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G))
+		speed = 25000;
 	else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G))
 		speed = 40000;
+	else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G))
+		speed = 100000;
 
 	lc = &pi->link_cfg;
 
@@ -8264,3 +8269,73 @@ void t4_idma_monitor(struct adapter *adapter,
 		t4_sge_decode_idma_state(adapter, idma->idma_state[i]);
 	}
 }
+
+/**
+ *	t4_set_vf_mac - Set MAC address for the specified VF
+ *	@adapter: The adapter
+ *	@vf: one of the VFs instantiated by the specified PF
+ *	@naddr: the number of MAC addresses
+ *	@addr: the MAC address(es) to be set to the specified VF
+ */
+int t4_set_vf_mac_acl(struct adapter *adapter, unsigned int vf,
+		      unsigned int naddr, u8 *addr)
+{
+	struct fw_acl_mac_cmd cmd;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_MAC_CMD) |
+				    FW_CMD_REQUEST_F |
+				    FW_CMD_WRITE_F |
+				    FW_ACL_MAC_CMD_PFN_V(adapter->pf) |
+				    FW_ACL_MAC_CMD_VFN_V(vf));
+
+	/* Note: Do not enable the ACL */
+	cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd));
+	cmd.nmac = naddr;
+
+	switch (adapter->pf) {
+	case 3:
+		memcpy(cmd.macaddr3, addr, sizeof(cmd.macaddr3));
+		break;
+	case 2:
+		memcpy(cmd.macaddr2, addr, sizeof(cmd.macaddr2));
+		break;
+	case 1:
+		memcpy(cmd.macaddr1, addr, sizeof(cmd.macaddr1));
+		break;
+	case 0:
+		memcpy(cmd.macaddr0, addr, sizeof(cmd.macaddr0));
+		break;
+	}
+
+	return t4_wr_mbox(adapter, adapter->mbox, &cmd, sizeof(cmd), &cmd);
+}
+
+int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
+		    int rateunit, int ratemode, int channel, int class,
+		    int minrate, int maxrate, int weight, int pktsize)
+{
+	struct fw_sched_cmd cmd;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.op_to_write = cpu_to_be32(FW_CMD_OP_V(FW_SCHED_CMD) |
+				      FW_CMD_REQUEST_F |
+				      FW_CMD_WRITE_F);
+	cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
+
+	cmd.u.params.sc = FW_SCHED_SC_PARAMS;
+	cmd.u.params.type = type;
+	cmd.u.params.level = level;
+	cmd.u.params.mode = mode;
+	cmd.u.params.ch = channel;
+	cmd.u.params.cl = class;
+	cmd.u.params.unit = rateunit;
+	cmd.u.params.rate = ratemode;
+	cmd.u.params.min = cpu_to_be32(minrate);
+	cmd.u.params.max = cpu_to_be32(maxrate);
+	cmd.u.params.weight = cpu_to_be16(weight);
+	cmd.u.params.pktsize = cpu_to_be16(pktsize);
+
+	return t4_wr_mbox_meat(adapter, adapter->mbox, &cmd, sizeof(cmd),
+			       NULL, 1);
+}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
index e0ebe1378cb2..fba3b2ad382d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
@@ -61,6 +61,7 @@ enum {
 	CPL_ABORT_REQ_RSS     = 0x2B,
 	CPL_ABORT_RPL_RSS     = 0x2D,
 
+	CPL_RX_PHYS_ADDR      = 0x30,
 	CPL_CLOSE_CON_RPL     = 0x32,
 	CPL_ISCSI_HDR         = 0x33,
 	CPL_RDMA_CQE          = 0x35,
@@ -83,6 +84,10 @@ enum {
 	CPL_PASS_OPEN_REQ6    = 0x81,
 	CPL_ACT_OPEN_REQ6     = 0x83,
 
+	CPL_TX_TLS_PDU     =    0x88,
+	CPL_TX_SEC_PDU        = 0x8A,
+	CPL_TX_TLS_ACK        = 0x8B,
+
 	CPL_RDMA_TERMINATE    = 0xA2,
 	CPL_RDMA_WRITE        = 0xA4,
 	CPL_SGE_EGR_UPDATE    = 0xA5,
@@ -94,6 +99,8 @@ enum {
 	CPL_FW4_PLD           = 0xC1,
 	CPL_FW4_ACK           = 0xC3,
 
+	CPL_RX_PHYS_DSGL      = 0xD0,
+
 	CPL_FW6_MSG           = 0xE0,
 	CPL_FW6_PLD           = 0xE1,
 	CPL_TX_PKT_LSO        = 0xED,
@@ -1362,6 +1369,15 @@ struct ulptx_idata {
 	__be32 len;
 };
 
+struct ulp_txpkt {
+	__be32 cmd_dest;
+	__be32 len;
+};
+
+#define ULPTX_CMD_S    24
+#define ULPTX_CMD_M    0xFF
+#define ULPTX_CMD_V(x) ((x) << ULPTX_CMD_S)
+
 #define ULPTX_NSGE_S    0
 #define ULPTX_NSGE_V(x) ((x) << ULPTX_NSGE_S)
 
@@ -1369,6 +1385,22 @@ struct ulptx_idata {
 #define ULPTX_MORE_V(x)	((x) << ULPTX_MORE_S)
 #define ULPTX_MORE_F	ULPTX_MORE_V(1U)
 
+#define ULP_TXPKT_DEST_S    16
+#define ULP_TXPKT_DEST_M    0x3
+#define ULP_TXPKT_DEST_V(x) ((x) << ULP_TXPKT_DEST_S)
+
+#define ULP_TXPKT_FID_S     4
+#define ULP_TXPKT_FID_M     0x7ff
+#define ULP_TXPKT_FID_V(x)  ((x) << ULP_TXPKT_FID_S)
+
+#define ULP_TXPKT_RO_S      3
+#define ULP_TXPKT_RO_V(x) ((x) << ULP_TXPKT_RO_S)
+#define ULP_TXPKT_RO_F ULP_TXPKT_RO_V(1U)
+
+#define ULP_TX_SC_MORE_S 23
+#define ULP_TX_SC_MORE_V(x) ((x) << ULP_TX_SC_MORE_S)
+#define ULP_TX_SC_MORE_F  ULP_TX_SC_MORE_V(1U)
+
 struct ulp_mem_io {
 	WR_HDR;
 	__be32 cmd;
@@ -1406,4 +1438,409 @@ struct ulp_mem_io {
 #define ULP_MEMIO_DATA_LEN_S    0
 #define ULP_MEMIO_DATA_LEN_V(x) ((x) << ULP_MEMIO_DATA_LEN_S)
 
+#define ULPTX_NSGE_S    0
+#define ULPTX_NSGE_M    0xFFFF
+#define ULPTX_NSGE_V(x) ((x) << ULPTX_NSGE_S)
+#define ULPTX_NSGE_G(x) (((x) >> ULPTX_NSGE_S) & ULPTX_NSGE_M)
+
+struct ulptx_sc_memrd {
+	__be32 cmd_to_len;
+	__be32 addr;
+};
+
+#define ULP_TXPKT_DATAMODIFY_S       23
+#define ULP_TXPKT_DATAMODIFY_M       0x1
+#define ULP_TXPKT_DATAMODIFY_V(x)    ((x) << ULP_TXPKT_DATAMODIFY_S)
+#define ULP_TXPKT_DATAMODIFY_G(x)    \
+	(((x) >> ULP_TXPKT_DATAMODIFY_S) & ULP_TXPKT_DATAMODIFY__M)
+#define ULP_TXPKT_DATAMODIFY_F       ULP_TXPKT_DATAMODIFY_V(1U)
+
+#define ULP_TXPKT_CHANNELID_S        22
+#define ULP_TXPKT_CHANNELID_M        0x1
+#define ULP_TXPKT_CHANNELID_V(x)     ((x) << ULP_TXPKT_CHANNELID_S)
+#define ULP_TXPKT_CHANNELID_G(x)     \
+	(((x) >> ULP_TXPKT_CHANNELID_S) & ULP_TXPKT_CHANNELID_M)
+#define ULP_TXPKT_CHANNELID_F        ULP_TXPKT_CHANNELID_V(1U)
+
+#define SCMD_SEQ_NO_CTRL_S      29
+#define SCMD_SEQ_NO_CTRL_M      0x3
+#define SCMD_SEQ_NO_CTRL_V(x)   ((x) << SCMD_SEQ_NO_CTRL_S)
+#define SCMD_SEQ_NO_CTRL_G(x)   \
+	(((x) >> SCMD_SEQ_NO_CTRL_S) & SCMD_SEQ_NO_CTRL_M)
+
+/* StsFieldPrsnt- Status field at the end of the TLS PDU */
+#define SCMD_STATUS_PRESENT_S   28
+#define SCMD_STATUS_PRESENT_M   0x1
+#define SCMD_STATUS_PRESENT_V(x)    ((x) << SCMD_STATUS_PRESENT_S)
+#define SCMD_STATUS_PRESENT_G(x)    \
+	(((x) >> SCMD_STATUS_PRESENT_S) & SCMD_STATUS_PRESENT_M)
+#define SCMD_STATUS_PRESENT_F   SCMD_STATUS_PRESENT_V(1U)
+
+/* ProtoVersion - Protocol Version 0: 1.2, 1:1.1, 2:DTLS, 3:Generic,
+ * 3-15: Reserved.
+ */
+#define SCMD_PROTO_VERSION_S    24
+#define SCMD_PROTO_VERSION_M    0xf
+#define SCMD_PROTO_VERSION_V(x) ((x) << SCMD_PROTO_VERSION_S)
+#define SCMD_PROTO_VERSION_G(x) \
+	(((x) >> SCMD_PROTO_VERSION_S) & SCMD_PROTO_VERSION_M)
+
+/* EncDecCtrl - Encryption/Decryption Control. 0: Encrypt, 1: Decrypt */
+#define SCMD_ENC_DEC_CTRL_S     23
+#define SCMD_ENC_DEC_CTRL_M     0x1
+#define SCMD_ENC_DEC_CTRL_V(x)  ((x) << SCMD_ENC_DEC_CTRL_S)
+#define SCMD_ENC_DEC_CTRL_G(x)  \
+	(((x) >> SCMD_ENC_DEC_CTRL_S) & SCMD_ENC_DEC_CTRL_M)
+#define SCMD_ENC_DEC_CTRL_F SCMD_ENC_DEC_CTRL_V(1U)
+
+/* CipherAuthSeqCtrl - Cipher Authentication Sequence Control. */
+#define SCMD_CIPH_AUTH_SEQ_CTRL_S       22
+#define SCMD_CIPH_AUTH_SEQ_CTRL_M       0x1
+#define SCMD_CIPH_AUTH_SEQ_CTRL_V(x)    \
+	((x) << SCMD_CIPH_AUTH_SEQ_CTRL_S)
+#define SCMD_CIPH_AUTH_SEQ_CTRL_G(x)    \
+	(((x) >> SCMD_CIPH_AUTH_SEQ_CTRL_S) & SCMD_CIPH_AUTH_SEQ_CTRL_M)
+#define SCMD_CIPH_AUTH_SEQ_CTRL_F   SCMD_CIPH_AUTH_SEQ_CTRL_V(1U)
+
+/* CiphMode -  Cipher Mode. 0: NOP, 1:AES-CBC, 2:AES-GCM, 3:AES-CTR,
+ * 4:Generic-AES, 5-15: Reserved.
+ */
+#define SCMD_CIPH_MODE_S    18
+#define SCMD_CIPH_MODE_M    0xf
+#define SCMD_CIPH_MODE_V(x) ((x) << SCMD_CIPH_MODE_S)
+#define SCMD_CIPH_MODE_G(x) \
+	(((x) >> SCMD_CIPH_MODE_S) & SCMD_CIPH_MODE_M)
+
+/* AuthMode - Auth Mode. 0: NOP, 1:SHA1, 2:SHA2-224, 3:SHA2-256
+ * 4-15: Reserved
+ */
+#define SCMD_AUTH_MODE_S    14
+#define SCMD_AUTH_MODE_M    0xf
+#define SCMD_AUTH_MODE_V(x) ((x) << SCMD_AUTH_MODE_S)
+#define SCMD_AUTH_MODE_G(x) \
+	(((x) >> SCMD_AUTH_MODE_S) & SCMD_AUTH_MODE_M)
+
+/* HmacCtrl - HMAC Control. 0:NOP, 1:No truncation, 2:Support HMAC Truncation
+ * per RFC 4366, 3:IPSec 96 bits, 4-7:Reserved
+ */
+#define SCMD_HMAC_CTRL_S    11
+#define SCMD_HMAC_CTRL_M    0x7
+#define SCMD_HMAC_CTRL_V(x) ((x) << SCMD_HMAC_CTRL_S)
+#define SCMD_HMAC_CTRL_G(x) \
+	(((x) >> SCMD_HMAC_CTRL_S) & SCMD_HMAC_CTRL_M)
+
+/* IvSize - IV size in units of 2 bytes */
+#define SCMD_IV_SIZE_S  7
+#define SCMD_IV_SIZE_M  0xf
+#define SCMD_IV_SIZE_V(x)   ((x) << SCMD_IV_SIZE_S)
+#define SCMD_IV_SIZE_G(x)   \
+	(((x) >> SCMD_IV_SIZE_S) & SCMD_IV_SIZE_M)
+
+/* NumIVs - Number of IVs */
+#define SCMD_NUM_IVS_S  0
+#define SCMD_NUM_IVS_M  0x7f
+#define SCMD_NUM_IVS_V(x)   ((x) << SCMD_NUM_IVS_S)
+#define SCMD_NUM_IVS_G(x)   \
+	(((x) >> SCMD_NUM_IVS_S) & SCMD_NUM_IVS_M)
+
+/* EnbDbgId - If this is enabled upper 20 (63:44) bits if SeqNumber
+ * (below) are used as Cid (connection id for debug status), these
+ * bits are padded to zero for forming the 64 bit
+ * sequence number for TLS
+ */
+#define SCMD_ENB_DBGID_S  31
+#define SCMD_ENB_DBGID_M  0x1
+#define SCMD_ENB_DBGID_V(x)   ((x) << SCMD_ENB_DBGID_S)
+#define SCMD_ENB_DBGID_G(x)   \
+	(((x) >> SCMD_ENB_DBGID_S) & SCMD_ENB_DBGID_M)
+
+/* IV generation in SW. */
+#define SCMD_IV_GEN_CTRL_S      30
+#define SCMD_IV_GEN_CTRL_M      0x1
+#define SCMD_IV_GEN_CTRL_V(x)   ((x) << SCMD_IV_GEN_CTRL_S)
+#define SCMD_IV_GEN_CTRL_G(x)   \
+	(((x) >> SCMD_IV_GEN_CTRL_S) & SCMD_IV_GEN_CTRL_M)
+#define SCMD_IV_GEN_CTRL_F  SCMD_IV_GEN_CTRL_V(1U)
+
+/* More frags */
+#define SCMD_MORE_FRAGS_S   20
+#define SCMD_MORE_FRAGS_M   0x1
+#define SCMD_MORE_FRAGS_V(x)    ((x) << SCMD_MORE_FRAGS_S)
+#define SCMD_MORE_FRAGS_G(x)    (((x) >> SCMD_MORE_FRAGS_S) & SCMD_MORE_FRAGS_M)
+
+/*last frag */
+#define SCMD_LAST_FRAG_S    19
+#define SCMD_LAST_FRAG_M    0x1
+#define SCMD_LAST_FRAG_V(x) ((x) << SCMD_LAST_FRAG_S)
+#define SCMD_LAST_FRAG_G(x) (((x) >> SCMD_LAST_FRAG_S) & SCMD_LAST_FRAG_M)
+
+/* TlsCompPdu */
+#define SCMD_TLS_COMPPDU_S    18
+#define SCMD_TLS_COMPPDU_M    0x1
+#define SCMD_TLS_COMPPDU_V(x) ((x) << SCMD_TLS_COMPPDU_S)
+#define SCMD_TLS_COMPPDU_G(x) (((x) >> SCMD_TLS_COMPPDU_S) & SCMD_TLS_COMPPDU_M)
+
+/* KeyCntxtInline - Key context inline after the scmd  OR PayloadOnly*/
+#define SCMD_KEY_CTX_INLINE_S   17
+#define SCMD_KEY_CTX_INLINE_M   0x1
+#define SCMD_KEY_CTX_INLINE_V(x)    ((x) << SCMD_KEY_CTX_INLINE_S)
+#define SCMD_KEY_CTX_INLINE_G(x)    \
+	(((x) >> SCMD_KEY_CTX_INLINE_S) & SCMD_KEY_CTX_INLINE_M)
+#define SCMD_KEY_CTX_INLINE_F   SCMD_KEY_CTX_INLINE_V(1U)
+
+/* TLSFragEnable - 0: Host created TLS PDUs, 1: TLS Framgmentation in ASIC */
+#define SCMD_TLS_FRAG_ENABLE_S  16
+#define SCMD_TLS_FRAG_ENABLE_M  0x1
+#define SCMD_TLS_FRAG_ENABLE_V(x)   ((x) << SCMD_TLS_FRAG_ENABLE_S)
+#define SCMD_TLS_FRAG_ENABLE_G(x)   \
+	(((x) >> SCMD_TLS_FRAG_ENABLE_S) & SCMD_TLS_FRAG_ENABLE_M)
+#define SCMD_TLS_FRAG_ENABLE_F  SCMD_TLS_FRAG_ENABLE_V(1U)
+
+/* MacOnly - Only send the MAC and discard PDU. This is valid for hash only
+ * modes, in this case TLS_TX  will drop the PDU and only
+ * send back the MAC bytes.
+ */
+#define SCMD_MAC_ONLY_S 15
+#define SCMD_MAC_ONLY_M 0x1
+#define SCMD_MAC_ONLY_V(x)  ((x) << SCMD_MAC_ONLY_S)
+#define SCMD_MAC_ONLY_G(x)  \
+	(((x) >> SCMD_MAC_ONLY_S) & SCMD_MAC_ONLY_M)
+#define SCMD_MAC_ONLY_F SCMD_MAC_ONLY_V(1U)
+
+/* AadIVDrop - Drop the AAD and IV fields. Useful in protocols
+ * which have complex AAD and IV formations Eg:AES-CCM
+ */
+#define SCMD_AADIVDROP_S 14
+#define SCMD_AADIVDROP_M 0x1
+#define SCMD_AADIVDROP_V(x)  ((x) << SCMD_AADIVDROP_S)
+#define SCMD_AADIVDROP_G(x)  \
+	(((x) >> SCMD_AADIVDROP_S) & SCMD_AADIVDROP_M)
+#define SCMD_AADIVDROP_F SCMD_AADIVDROP_V(1U)
+
+/* HdrLength - Length of all headers excluding TLS header
+ * present before start of crypto PDU/payload.
+ */
+#define SCMD_HDR_LEN_S  0
+#define SCMD_HDR_LEN_M  0x3fff
+#define SCMD_HDR_LEN_V(x)   ((x) << SCMD_HDR_LEN_S)
+#define SCMD_HDR_LEN_G(x)   \
+	(((x) >> SCMD_HDR_LEN_S) & SCMD_HDR_LEN_M)
+
+struct cpl_tx_sec_pdu {
+	__be32 op_ivinsrtofst;
+	__be32 pldlen;
+	__be32 aadstart_cipherstop_hi;
+	__be32 cipherstop_lo_authinsert;
+	__be32 seqno_numivs;
+	__be32 ivgen_hdrlen;
+	__be64 scmd1;
+};
+
+#define CPL_TX_SEC_PDU_OPCODE_S     24
+#define CPL_TX_SEC_PDU_OPCODE_M     0xff
+#define CPL_TX_SEC_PDU_OPCODE_V(x)  ((x) << CPL_TX_SEC_PDU_OPCODE_S)
+#define CPL_TX_SEC_PDU_OPCODE_G(x)  \
+	(((x) >> CPL_TX_SEC_PDU_OPCODE_S) & CPL_TX_SEC_PDU_OPCODE_M)
+
+/* RX Channel Id */
+#define CPL_TX_SEC_PDU_RXCHID_S  22
+#define CPL_TX_SEC_PDU_RXCHID_M  0x1
+#define CPL_TX_SEC_PDU_RXCHID_V(x)   ((x) << CPL_TX_SEC_PDU_RXCHID_S)
+#define CPL_TX_SEC_PDU_RXCHID_G(x)   \
+	(((x) >> CPL_TX_SEC_PDU_RXCHID_S) & CPL_TX_SEC_PDU_RXCHID_M)
+#define CPL_TX_SEC_PDU_RXCHID_F  CPL_TX_SEC_PDU_RXCHID_V(1U)
+
+/* Ack Follows */
+#define CPL_TX_SEC_PDU_ACKFOLLOWS_S  21
+#define CPL_TX_SEC_PDU_ACKFOLLOWS_M  0x1
+#define CPL_TX_SEC_PDU_ACKFOLLOWS_V(x)   ((x) << CPL_TX_SEC_PDU_ACKFOLLOWS_S)
+#define CPL_TX_SEC_PDU_ACKFOLLOWS_G(x)   \
+	(((x) >> CPL_TX_SEC_PDU_ACKFOLLOWS_S) & CPL_TX_SEC_PDU_ACKFOLLOWS_M)
+#define CPL_TX_SEC_PDU_ACKFOLLOWS_F  CPL_TX_SEC_PDU_ACKFOLLOWS_V(1U)
+
+/* Loopback bit in cpl_tx_sec_pdu */
+#define CPL_TX_SEC_PDU_ULPTXLPBK_S  20
+#define CPL_TX_SEC_PDU_ULPTXLPBK_M  0x1
+#define CPL_TX_SEC_PDU_ULPTXLPBK_V(x)   ((x) << CPL_TX_SEC_PDU_ULPTXLPBK_S)
+#define CPL_TX_SEC_PDU_ULPTXLPBK_G(x)   \
+	(((x) >> CPL_TX_SEC_PDU_ULPTXLPBK_S) & CPL_TX_SEC_PDU_ULPTXLPBK_M)
+#define CPL_TX_SEC_PDU_ULPTXLPBK_F  CPL_TX_SEC_PDU_ULPTXLPBK_V(1U)
+
+/* Length of cpl header encapsulated */
+#define CPL_TX_SEC_PDU_CPLLEN_S     16
+#define CPL_TX_SEC_PDU_CPLLEN_M     0xf
+#define CPL_TX_SEC_PDU_CPLLEN_V(x)  ((x) << CPL_TX_SEC_PDU_CPLLEN_S)
+#define CPL_TX_SEC_PDU_CPLLEN_G(x)  \
+	(((x) >> CPL_TX_SEC_PDU_CPLLEN_S) & CPL_TX_SEC_PDU_CPLLEN_M)
+
+/* PlaceHolder */
+#define CPL_TX_SEC_PDU_PLACEHOLDER_S    10
+#define CPL_TX_SEC_PDU_PLACEHOLDER_M    0x1
+#define CPL_TX_SEC_PDU_PLACEHOLDER_V(x) ((x) << CPL_TX_SEC_PDU_PLACEHOLDER_S)
+#define CPL_TX_SEC_PDU_PLACEHOLDER_G(x) \
+	(((x) >> CPL_TX_SEC_PDU_PLACEHOLDER_S) & \
+	 CPL_TX_SEC_PDU_PLACEHOLDER_M)
+
+/* IvInsrtOffset: Insertion location for IV */
+#define CPL_TX_SEC_PDU_IVINSRTOFST_S    0
+#define CPL_TX_SEC_PDU_IVINSRTOFST_M    0x3ff
+#define CPL_TX_SEC_PDU_IVINSRTOFST_V(x) ((x) << CPL_TX_SEC_PDU_IVINSRTOFST_S)
+#define CPL_TX_SEC_PDU_IVINSRTOFST_G(x) \
+	(((x) >> CPL_TX_SEC_PDU_IVINSRTOFST_S) & \
+	 CPL_TX_SEC_PDU_IVINSRTOFST_M)
+
+/* AadStartOffset: Offset in bytes for AAD start from
+ * the first byte following the pkt headers (0-255 bytes)
+ */
+#define CPL_TX_SEC_PDU_AADSTART_S   24
+#define CPL_TX_SEC_PDU_AADSTART_M   0xff
+#define CPL_TX_SEC_PDU_AADSTART_V(x)    ((x) << CPL_TX_SEC_PDU_AADSTART_S)
+#define CPL_TX_SEC_PDU_AADSTART_G(x)    \
+	(((x) >> CPL_TX_SEC_PDU_AADSTART_S) & \
+	 CPL_TX_SEC_PDU_AADSTART_M)
+
+/* AadStopOffset: offset in bytes for AAD stop/end from the first byte following
+ * the pkt headers (0-511 bytes)
+ */
+#define CPL_TX_SEC_PDU_AADSTOP_S    15
+#define CPL_TX_SEC_PDU_AADSTOP_M    0x1ff
+#define CPL_TX_SEC_PDU_AADSTOP_V(x) ((x) << CPL_TX_SEC_PDU_AADSTOP_S)
+#define CPL_TX_SEC_PDU_AADSTOP_G(x) \
+	(((x) >> CPL_TX_SEC_PDU_AADSTOP_S) & CPL_TX_SEC_PDU_AADSTOP_M)
+
+/* CipherStartOffset: offset in bytes for encryption/decryption start from the
+ * first byte following the pkt headers (0-1023 bytes)
+ */
+#define CPL_TX_SEC_PDU_CIPHERSTART_S    5
+#define CPL_TX_SEC_PDU_CIPHERSTART_M    0x3ff
+#define CPL_TX_SEC_PDU_CIPHERSTART_V(x) ((x) << CPL_TX_SEC_PDU_CIPHERSTART_S)
+#define CPL_TX_SEC_PDU_CIPHERSTART_G(x) \
+	(((x) >> CPL_TX_SEC_PDU_CIPHERSTART_S) & \
+	 CPL_TX_SEC_PDU_CIPHERSTART_M)
+
+/* CipherStopOffset: offset in bytes for encryption/decryption end
+ * from end of the payload of this command (0-511 bytes)
+ */
+#define CPL_TX_SEC_PDU_CIPHERSTOP_HI_S      0
+#define CPL_TX_SEC_PDU_CIPHERSTOP_HI_M      0x1f
+#define CPL_TX_SEC_PDU_CIPHERSTOP_HI_V(x)   \
+	((x) << CPL_TX_SEC_PDU_CIPHERSTOP_HI_S)
+#define CPL_TX_SEC_PDU_CIPHERSTOP_HI_G(x)   \
+	(((x) >> CPL_TX_SEC_PDU_CIPHERSTOP_HI_S) & \
+	 CPL_TX_SEC_PDU_CIPHERSTOP_HI_M)
+
+#define CPL_TX_SEC_PDU_CIPHERSTOP_LO_S      28
+#define CPL_TX_SEC_PDU_CIPHERSTOP_LO_M      0xf
+#define CPL_TX_SEC_PDU_CIPHERSTOP_LO_V(x)   \
+	((x) << CPL_TX_SEC_PDU_CIPHERSTOP_LO_S)
+#define CPL_TX_SEC_PDU_CIPHERSTOP_LO_G(x)   \
+	(((x) >> CPL_TX_SEC_PDU_CIPHERSTOP_LO_S) & \
+	 CPL_TX_SEC_PDU_CIPHERSTOP_LO_M)
+
+/* AuthStartOffset: offset in bytes for authentication start from
+ * the first byte following the pkt headers (0-1023)
+ */
+#define CPL_TX_SEC_PDU_AUTHSTART_S  18
+#define CPL_TX_SEC_PDU_AUTHSTART_M  0x3ff
+#define CPL_TX_SEC_PDU_AUTHSTART_V(x)   ((x) << CPL_TX_SEC_PDU_AUTHSTART_S)
+#define CPL_TX_SEC_PDU_AUTHSTART_G(x)   \
+	(((x) >> CPL_TX_SEC_PDU_AUTHSTART_S) & \
+	 CPL_TX_SEC_PDU_AUTHSTART_M)
+
+/* AuthStopOffset: offset in bytes for authentication
+ * end from end of the payload of this command (0-511 Bytes)
+ */
+#define CPL_TX_SEC_PDU_AUTHSTOP_S   9
+#define CPL_TX_SEC_PDU_AUTHSTOP_M   0x1ff
+#define CPL_TX_SEC_PDU_AUTHSTOP_V(x)    ((x) << CPL_TX_SEC_PDU_AUTHSTOP_S)
+#define CPL_TX_SEC_PDU_AUTHSTOP_G(x)    \
+	(((x) >> CPL_TX_SEC_PDU_AUTHSTOP_S) & \
+	 CPL_TX_SEC_PDU_AUTHSTOP_M)
+
+/* AuthInsrtOffset: offset in bytes for authentication insertion
+ * from end of the payload of this command (0-511 bytes)
+ */
+#define CPL_TX_SEC_PDU_AUTHINSERT_S 0
+#define CPL_TX_SEC_PDU_AUTHINSERT_M 0x1ff
+#define CPL_TX_SEC_PDU_AUTHINSERT_V(x)  ((x) << CPL_TX_SEC_PDU_AUTHINSERT_S)
+#define CPL_TX_SEC_PDU_AUTHINSERT_G(x)  \
+	(((x) >> CPL_TX_SEC_PDU_AUTHINSERT_S) & \
+	 CPL_TX_SEC_PDU_AUTHINSERT_M)
+
+struct cpl_rx_phys_dsgl {
+	__be32 op_to_tid;
+	__be32 pcirlxorder_to_noofsgentr;
+	struct rss_header rss_hdr_int;
+};
+
+#define CPL_RX_PHYS_DSGL_OPCODE_S       24
+#define CPL_RX_PHYS_DSGL_OPCODE_M       0xff
+#define CPL_RX_PHYS_DSGL_OPCODE_V(x)    ((x) << CPL_RX_PHYS_DSGL_OPCODE_S)
+#define CPL_RX_PHYS_DSGL_OPCODE_G(x)    \
+	(((x) >> CPL_RX_PHYS_DSGL_OPCODE_S) & CPL_RX_PHYS_DSGL_OPCODE_M)
+
+#define CPL_RX_PHYS_DSGL_ISRDMA_S       23
+#define CPL_RX_PHYS_DSGL_ISRDMA_M       0x1
+#define CPL_RX_PHYS_DSGL_ISRDMA_V(x)    ((x) << CPL_RX_PHYS_DSGL_ISRDMA_S)
+#define CPL_RX_PHYS_DSGL_ISRDMA_G(x)    \
+	(((x) >> CPL_RX_PHYS_DSGL_ISRDMA_S) & CPL_RX_PHYS_DSGL_ISRDMA_M)
+#define CPL_RX_PHYS_DSGL_ISRDMA_F       CPL_RX_PHYS_DSGL_ISRDMA_V(1U)
+
+#define CPL_RX_PHYS_DSGL_RSVD1_S        20
+#define CPL_RX_PHYS_DSGL_RSVD1_M        0x7
+#define CPL_RX_PHYS_DSGL_RSVD1_V(x)     ((x) << CPL_RX_PHYS_DSGL_RSVD1_S)
+#define CPL_RX_PHYS_DSGL_RSVD1_G(x)     \
+	(((x) >> CPL_RX_PHYS_DSGL_RSVD1_S) & \
+	 CPL_RX_PHYS_DSGL_RSVD1_M)
+
+#define CPL_RX_PHYS_DSGL_PCIRLXORDER_S          31
+#define CPL_RX_PHYS_DSGL_PCIRLXORDER_M          0x1
+#define CPL_RX_PHYS_DSGL_PCIRLXORDER_V(x)       \
+	((x) << CPL_RX_PHYS_DSGL_PCIRLXORDER_S)
+#define CPL_RX_PHYS_DSGL_PCIRLXORDER_G(x)       \
+	(((x) >> CPL_RX_PHYS_DSGL_PCIRLXORDER_S) & \
+	 CPL_RX_PHYS_DSGL_PCIRLXORDER_M)
+#define CPL_RX_PHYS_DSGL_PCIRLXORDER_F  CPL_RX_PHYS_DSGL_PCIRLXORDER_V(1U)
+
+#define CPL_RX_PHYS_DSGL_PCINOSNOOP_S           30
+#define CPL_RX_PHYS_DSGL_PCINOSNOOP_M           0x1
+#define CPL_RX_PHYS_DSGL_PCINOSNOOP_V(x)        \
+	((x) << CPL_RX_PHYS_DSGL_PCINOSNOOP_S)
+#define CPL_RX_PHYS_DSGL_PCINOSNOOP_G(x)        \
+	(((x) >> CPL_RX_PHYS_DSGL_PCINOSNOOP_S) & \
+	 CPL_RX_PHYS_DSGL_PCINOSNOOP_M)
+
+#define CPL_RX_PHYS_DSGL_PCINOSNOOP_F   CPL_RX_PHYS_DSGL_PCINOSNOOP_V(1U)
+
+#define CPL_RX_PHYS_DSGL_PCITPHNTENB_S          29
+#define CPL_RX_PHYS_DSGL_PCITPHNTENB_M          0x1
+#define CPL_RX_PHYS_DSGL_PCITPHNTENB_V(x)       \
+	((x) << CPL_RX_PHYS_DSGL_PCITPHNTENB_S)
+#define CPL_RX_PHYS_DSGL_PCITPHNTENB_G(x)       \
+	(((x) >> CPL_RX_PHYS_DSGL_PCITPHNTENB_S) & \
+	 CPL_RX_PHYS_DSGL_PCITPHNTENB_M)
+#define CPL_RX_PHYS_DSGL_PCITPHNTENB_F  CPL_RX_PHYS_DSGL_PCITPHNTENB_V(1U)
+
+#define CPL_RX_PHYS_DSGL_PCITPHNT_S     27
+#define CPL_RX_PHYS_DSGL_PCITPHNT_M     0x3
+#define CPL_RX_PHYS_DSGL_PCITPHNT_V(x)  ((x) << CPL_RX_PHYS_DSGL_PCITPHNT_S)
+#define CPL_RX_PHYS_DSGL_PCITPHNT_G(x)  \
+	(((x) >> CPL_RX_PHYS_DSGL_PCITPHNT_S) & \
+	 CPL_RX_PHYS_DSGL_PCITPHNT_M)
+
+#define CPL_RX_PHYS_DSGL_DCAID_S        16
+#define CPL_RX_PHYS_DSGL_DCAID_M        0x7ff
+#define CPL_RX_PHYS_DSGL_DCAID_V(x)     ((x) << CPL_RX_PHYS_DSGL_DCAID_S)
+#define CPL_RX_PHYS_DSGL_DCAID_G(x)     \
+	(((x) >> CPL_RX_PHYS_DSGL_DCAID_S) & \
+	 CPL_RX_PHYS_DSGL_DCAID_M)
+
+#define CPL_RX_PHYS_DSGL_NOOFSGENTR_S           0
+#define CPL_RX_PHYS_DSGL_NOOFSGENTR_M           0xffff
+#define CPL_RX_PHYS_DSGL_NOOFSGENTR_V(x)        \
+	((x) << CPL_RX_PHYS_DSGL_NOOFSGENTR_S)
+#define CPL_RX_PHYS_DSGL_NOOFSGENTR_G(x)        \
+	(((x) >> CPL_RX_PHYS_DSGL_NOOFSGENTR_S) & \
+	 CPL_RX_PHYS_DSGL_NOOFSGENTR_M)
+
 #endif  /* __T4_MSG_H */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
index a89b30720e38..8d9e4b7a8e84 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
@@ -1,7 +1,7 @@
 /*
  * This file is part of the Chelsio T4 Ethernet driver for Linux.
  *
- * Copyright (c) 2009-2014 Chelsio Communications, Inc. All rights reserved.
+ * Copyright (c) 2009-2016 Chelsio Communications, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -100,8 +100,10 @@ enum fw_wr_opcodes {
 	FW_RI_RECV_WR                  = 0x17,
 	FW_RI_BIND_MW_WR               = 0x18,
 	FW_RI_FR_NSMR_WR               = 0x19,
+	FW_RI_FR_NSMR_TPTE_WR	       = 0x20,
 	FW_RI_INV_LSTAG_WR             = 0x1a,
 	FW_ISCSI_TX_DATA_WR	       = 0x45,
+	FW_CRYPTO_LOOKASIDE_WR         = 0X6d,
 	FW_LASTC2E_WR                  = 0x70
 };
 
@@ -680,6 +682,7 @@ enum fw_cmd_opcodes {
 	FW_RSS_IND_TBL_CMD             = 0x20,
 	FW_RSS_GLB_CONFIG_CMD          = 0x22,
 	FW_RSS_VI_CONFIG_CMD           = 0x23,
+	FW_SCHED_CMD                   = 0x24,
 	FW_DEVLOG_CMD                  = 0x25,
 	FW_CLIP_CMD                    = 0x28,
 	FW_LASTC2E_CMD                 = 0x40,
@@ -1060,7 +1063,7 @@ struct fw_caps_config_cmd {
 	__be16 niccaps;
 	__be16 ofldcaps;
 	__be16 rdmacaps;
-	__be16 r4;
+	__be16 cryptocaps;
 	__be16 iscsicaps;
 	__be16 fcoecaps;
 	__be32 cfcsum;
@@ -1119,6 +1122,7 @@ enum fw_params_param_dev {
 	FW_PARAMS_PARAM_DEV_MAXIRD_ADAPTER = 0x14, /* max supported adap IRD */
 	FW_PARAMS_PARAM_DEV_ULPTX_MEMWRITE_DSGL = 0x17,
 	FW_PARAMS_PARAM_DEV_FWCACHE = 0x18,
+	FW_PARAMS_PARAM_DEV_RI_FR_NSMR_TPTE_WR	= 0x1C,
 };
 
 /*
@@ -2265,6 +2269,12 @@ enum fw_port_cap {
 	FW_PORT_CAP_802_3_ASM_DIR	= 0x8000,
 };
 
+#define FW_PORT_CAP_SPEED_S     0
+#define FW_PORT_CAP_SPEED_M     0x3f
+#define FW_PORT_CAP_SPEED_V(x)  ((x) << FW_PORT_CAP_SPEED_S)
+#define FW_PORT_CAP_SPEED_G(x) \
+	(((x) >> FW_PORT_CAP_SPEED_S) & FW_PORT_CAP_SPEED_M)
+
 enum fw_port_mdi {
 	FW_PORT_CAP_MDI_UNCHANGED,
 	FW_PORT_CAP_MDI_AUTO,
@@ -2961,6 +2971,41 @@ struct fw_rss_vi_config_cmd {
 #define FW_RSS_VI_CONFIG_CMD_UDPEN_V(x)	((x) << FW_RSS_VI_CONFIG_CMD_UDPEN_S)
 #define FW_RSS_VI_CONFIG_CMD_UDPEN_F	FW_RSS_VI_CONFIG_CMD_UDPEN_V(1U)
 
+enum fw_sched_sc {
+	FW_SCHED_SC_PARAMS		= 1,
+};
+
+struct fw_sched_cmd {
+	__be32 op_to_write;
+	__be32 retval_len16;
+	union fw_sched {
+		struct fw_sched_config {
+			__u8   sc;
+			__u8   type;
+			__u8   minmaxen;
+			__u8   r3[5];
+			__u8   nclasses[4];
+			__be32 r4;
+		} config;
+		struct fw_sched_params {
+			__u8   sc;
+			__u8   type;
+			__u8   level;
+			__u8   mode;
+			__u8   unit;
+			__u8   rate;
+			__u8   ch;
+			__u8   cl;
+			__be32 min;
+			__be32 max;
+			__be16 weight;
+			__be16 pktsize;
+			__be16 burstsize;
+			__be16 r4;
+		} params;
+	} u;
+};
+
 struct fw_clip_cmd {
 	__be32 op_to_write;
 	__be32 alloc_to_len16;
@@ -3249,4 +3294,127 @@ struct fw_devlog_cmd {
 #define PCIE_FW_PF_DEVLOG_MEMTYPE_G(x) \
 	(((x) >> PCIE_FW_PF_DEVLOG_MEMTYPE_S) & PCIE_FW_PF_DEVLOG_MEMTYPE_M)
 
+#define MAX_IMM_OFLD_TX_DATA_WR_LEN (0xff + sizeof(struct fw_ofld_tx_data_wr))
+
+struct fw_crypto_lookaside_wr {
+	__be32 op_to_cctx_size;
+	__be32 len16_pkd;
+	__be32 session_id;
+	__be32 rx_chid_to_rx_q_id;
+	__be32 key_addr;
+	__be32 pld_size_hash_size;
+	__be64 cookie;
+};
+
+#define FW_CRYPTO_LOOKASIDE_WR_OPCODE_S 24
+#define FW_CRYPTO_LOOKASIDE_WR_OPCODE_M 0xff
+#define FW_CRYPTO_LOOKASIDE_WR_OPCODE_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_OPCODE_S)
+#define FW_CRYPTO_LOOKASIDE_WR_OPCODE_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_OPCODE_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_OPCODE_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_COMPL_S 23
+#define FW_CRYPTO_LOOKASIDE_WR_COMPL_M 0x1
+#define FW_CRYPTO_LOOKASIDE_WR_COMPL_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_COMPL_S)
+#define FW_CRYPTO_LOOKASIDE_WR_COMPL_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_COMPL_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_COMPL_M)
+#define FW_CRYPTO_LOOKASIDE_WR_COMPL_F FW_CRYPTO_LOOKASIDE_WR_COMPL_V(1U)
+
+#define FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_S 15
+#define FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_M 0xff
+#define FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_S)
+#define FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_S 5
+#define FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_M 0x3
+#define FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_S)
+#define FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_S 0
+#define FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_M 0x1f
+#define FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_S)
+#define FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_LEN16_S 0
+#define FW_CRYPTO_LOOKASIDE_WR_LEN16_M 0xff
+#define FW_CRYPTO_LOOKASIDE_WR_LEN16_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_LEN16_S)
+#define FW_CRYPTO_LOOKASIDE_WR_LEN16_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_LEN16_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_LEN16_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_RX_CHID_S 29
+#define FW_CRYPTO_LOOKASIDE_WR_RX_CHID_M 0x3
+#define FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_RX_CHID_S)
+#define FW_CRYPTO_LOOKASIDE_WR_RX_CHID_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_RX_CHID_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_RX_CHID_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_LCB_S  27
+#define FW_CRYPTO_LOOKASIDE_WR_LCB_M  0x3
+#define FW_CRYPTO_LOOKASIDE_WR_LCB_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_LCB_S)
+#define FW_CRYPTO_LOOKASIDE_WR_LCB_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_LCB_S) & FW_CRYPTO_LOOKASIDE_WR_LCB_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_PHASH_S 25
+#define FW_CRYPTO_LOOKASIDE_WR_PHASH_M 0x3
+#define FW_CRYPTO_LOOKASIDE_WR_PHASH_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_PHASH_S)
+#define FW_CRYPTO_LOOKASIDE_WR_PHASH_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_PHASH_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_PHASH_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_IV_S   23
+#define FW_CRYPTO_LOOKASIDE_WR_IV_M   0x3
+#define FW_CRYPTO_LOOKASIDE_WR_IV_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_IV_S)
+#define FW_CRYPTO_LOOKASIDE_WR_IV_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_IV_S) & FW_CRYPTO_LOOKASIDE_WR_IV_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_S 10
+#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_M 0x3
+#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_TX_CH_S)
+#define FW_CRYPTO_LOOKASIDE_WR_TX_CH_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_TX_CH_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_TX_CH_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_S 0
+#define FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_M 0x3ff
+#define FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_S)
+#define FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_S 24
+#define FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_M 0xff
+#define FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_S)
+#define FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_M)
+
+#define FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_S 17
+#define FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_M 0x7f
+#define FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(x) \
+	((x) << FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_S)
+#define FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_G(x) \
+	(((x) >> FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_S) & \
+	 FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_M)
+
 #endif /* _T4FW_INTERFACE_H_ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
index e116bb8d1729..100b2cc064a3 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
@@ -2378,7 +2378,7 @@ static void size_nports_qsets(struct adapter *adapter)
 	 */
 	pmask_nports = hweight32(adapter->params.vfres.pmask);
 	if (pmask_nports < adapter->params.nports) {
-		dev_warn(adapter->pdev_dev, "only using %d of %d provissioned"
+		dev_warn(adapter->pdev_dev, "only using %d of %d provisioned"
 			 " virtual interfaces; limited by Port Access Rights"
 			 " mask %#x\n", pmask_nports, adapter->params.nports,
 			 adapter->params.vfres.pmask);
@@ -2777,6 +2777,7 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev,
 	struct adapter *adapter;
 	struct port_info *pi;
 	struct net_device *netdev;
+	unsigned int pf;
 
 	/*
 	 * Print our driver banner the first time we're called to initialize a
@@ -2903,8 +2904,11 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev,
 	 * Allocate our "adapter ports" and stitch everything together.
 	 */
 	pmask = adapter->params.vfres.pmask;
+	pf = t4vf_get_pf_from_vf(adapter);
 	for_each_port(adapter, pidx) {
 		int port_id, viid;
+		u8 mac[ETH_ALEN];
+		unsigned int naddr = 1;
 
 		/*
 		 * We simplistically allocate our virtual interfaces
@@ -2975,6 +2979,26 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev,
 				pidx);
 			goto err_free_dev;
 		}
+
+		err = t4vf_get_vf_mac_acl(adapter, pf, &naddr, mac);
+		if (err) {
+			dev_err(&pdev->dev,
+				"unable to determine MAC ACL address, "
+				"continuing anyway.. (status %d)\n", err);
+		} else if (naddr && adapter->params.vfres.nvi == 1) {
+			struct sockaddr addr;
+
+			ether_addr_copy(addr.sa_data, mac);
+			err = cxgb4vf_set_mac_addr(netdev, &addr);
+			if (err) {
+				dev_err(&pdev->dev,
+					"unable to set MAC address %pM\n",
+					mac);
+				goto err_free_dev;
+			}
+			dev_info(&pdev->dev,
+				 "Using assigned MAC ACL: %pM\n", mac);
+		}
 	}
 
 	/* See what interrupts we'll be using.  If we've been configured to
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
index c8fd4f8fe1fa..f3ed9ce99e5e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
@@ -1648,14 +1648,15 @@ int t4vf_ethrx_handler(struct sge_rspq *rspq, const __be64 *rsp,
 
 	if (csum_ok && !pkt->err_vec &&
 	    (be32_to_cpu(pkt->l2info) & (RXF_UDP_F | RXF_TCP_F))) {
-		if (!pkt->ip_frag)
+		if (!pkt->ip_frag) {
 			skb->ip_summed = CHECKSUM_UNNECESSARY;
-		else {
+			rxq->stats.rx_cso++;
+		} else if (pkt->l2info & htonl(RXF_IP_F)) {
 			__sum16 c = (__force __sum16)pkt->csum;
 			skb->csum = csum_unfold(c);
 			skb->ip_summed = CHECKSUM_COMPLETE;
+			rxq->stats.rx_cso++;
 		}
-		rxq->stats.rx_cso++;
 	} else
 		skb_checksum_none_assert(skb);
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
index 8ee541431e8b..b3903fe411aa 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
@@ -108,8 +108,8 @@ struct link_config {
 	unsigned int   supported;        /* link capabilities */
 	unsigned int   advertising;      /* advertised capabilities */
 	unsigned short lp_advertising;   /* peer advertised capabilities */
-	unsigned short requested_speed;  /* speed user has requested */
-	unsigned short speed;            /* actual link speed */
+	unsigned int   requested_speed;  /* speed user has requested */
+	unsigned int   speed;            /* actual link speed */
 	unsigned char  requested_fc;     /* flow control user has requested */
 	unsigned char  fc;               /* actual link flow control */
 	unsigned char  autoneg;          /* autonegotiating? */
@@ -271,10 +271,17 @@ static inline bool is_10g_port(const struct link_config *lc)
 	return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
 }
 
+/* Return true if the Link Configuration supports "High Speeds" (those greater
+ * than 1Gb/s).
+ */
 static inline bool is_x_10g_port(const struct link_config *lc)
 {
-	return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
-		(lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
+	unsigned int speeds, high_speeds;
+
+	speeds = FW_PORT_CAP_SPEED_V(FW_PORT_CAP_SPEED_G(lc->supported));
+	high_speeds = speeds & ~(FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G);
+
+	return high_speeds != 0;
 }
 
 static inline unsigned int core_ticks_per_usec(const struct adapter *adapter)
@@ -347,6 +354,7 @@ int t4vf_bar2_sge_qregs(struct adapter *adapter,
 			u64 *pbar2_qoffset,
 			unsigned int *pbar2_qid);
 
+unsigned int t4vf_get_pf_from_vf(struct adapter *);
 int t4vf_get_sge_params(struct adapter *);
 int t4vf_get_vpd_params(struct adapter *);
 int t4vf_get_dev_params(struct adapter *);
@@ -381,5 +389,7 @@ int t4vf_eth_eq_free(struct adapter *, unsigned int);
 
 int t4vf_handle_fw_rpl(struct adapter *, const __be64 *);
 int t4vf_prep_adapter(struct adapter *);
+int t4vf_get_vf_mac_acl(struct adapter *adapter, unsigned int pf,
+			unsigned int *naddr, u8 *addr);
 
 #endif /* __T4VF_COMMON_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
index 427bfa71388b..e98248f00fef 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
@@ -314,8 +314,9 @@ int t4vf_wr_mbox_core(struct adapter *adapter, const void *cmd, int size,
 }
 
 #define ADVERT_MASK (FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G |\
-		     FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_40G | \
-		     FW_PORT_CAP_SPEED_100G | FW_PORT_CAP_ANEG)
+		     FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_25G | \
+		     FW_PORT_CAP_SPEED_40G | FW_PORT_CAP_SPEED_100G | \
+		     FW_PORT_CAP_ANEG)
 
 /**
  *	init_link_config - initialize a link's SW state
@@ -639,6 +640,15 @@ int t4vf_bar2_sge_qregs(struct adapter *adapter,
 	return 0;
 }
 
+unsigned int t4vf_get_pf_from_vf(struct adapter *adapter)
+{
+	u32 whoami;
+
+	whoami = t4_read_reg(adapter, T4VF_PL_BASE_ADDR + PL_VF_WHOAMI_A);
+	return (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ?
+			SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami));
+}
+
 /**
  *	t4vf_get_sge_params - retrieve adapter Scatter gather Engine parameters
  *	@adapter: the adapter
@@ -716,7 +726,6 @@ int t4vf_get_sge_params(struct adapter *adapter)
 	 * read.
 	 */
 	if (!is_t4(adapter->params.chip)) {
-		u32 whoami;
 		unsigned int pf, s_hps, s_qpp;
 
 		params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) |
@@ -740,11 +749,7 @@ int t4vf_get_sge_params(struct adapter *adapter)
 		 * register we just read. Do it once here so other code in
 		 * the driver can just use it.
 		 */
-		whoami = t4_read_reg(adapter,
-				     T4VF_PL_BASE_ADDR + PL_VF_WHOAMI_A);
-		pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ?
-			SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
-
+		pf = t4vf_get_pf_from_vf(adapter);
 		s_hps = (HOSTPAGESIZEPF0_S +
 			 (HOSTPAGESIZEPF1_S - HOSTPAGESIZEPF0_S) * pf);
 		sge_params->sge_vf_hps =
@@ -1712,8 +1717,12 @@ int t4vf_handle_fw_rpl(struct adapter *adapter, const __be64 *rpl)
 			speed = 1000;
 		else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G))
 			speed = 10000;
+		else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G))
+			speed = 25000;
 		else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G))
 			speed = 40000;
+		else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G))
+			speed = 100000;
 
 		/*
 		 * Scan all of our "ports" (Virtual Interfaces) looking for
@@ -1807,3 +1816,50 @@ int t4vf_prep_adapter(struct adapter *adapter)
 
 	return 0;
 }
+
+/**
+ *	t4vf_get_vf_mac_acl - Get the MAC address to be set to
+ *			      the VI of this VF.
+ *	@adapter: The adapter
+ *	@pf: The pf associated with vf
+ *	@naddr: the number of ACL MAC addresses returned in addr
+ *	@addr: Placeholder for MAC addresses
+ *
+ *	Find the MAC address to be set to the VF's VI. The requested MAC address
+ *	is from the host OS via callback in the PF driver.
+ */
+int t4vf_get_vf_mac_acl(struct adapter *adapter, unsigned int pf,
+			unsigned int *naddr, u8 *addr)
+{
+	struct fw_acl_mac_cmd cmd;
+	int ret;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_MAC_CMD) |
+				    FW_CMD_REQUEST_F |
+				    FW_CMD_READ_F);
+	cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd));
+	ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &cmd);
+	if (ret)
+		return ret;
+
+	if (cmd.nmac < *naddr)
+		*naddr = cmd.nmac;
+
+	switch (pf) {
+	case 3:
+		memcpy(addr, cmd.macaddr3, sizeof(cmd.macaddr3));
+		break;
+	case 2:
+		memcpy(addr, cmd.macaddr2, sizeof(cmd.macaddr2));
+		break;
+	case 1:
+		memcpy(addr, cmd.macaddr1, sizeof(cmd.macaddr1));
+		break;
+	case 0:
+		memcpy(addr, cmd.macaddr0, sizeof(cmd.macaddr0));
+		break;
+	}
+
+	return ret;
+}
diff --git a/drivers/net/ethernet/chelsio/libcxgb/Makefile b/drivers/net/ethernet/chelsio/libcxgb/Makefile
index 2362230ef4fe..2534e30a1560 100644
--- a/drivers/net/ethernet/chelsio/libcxgb/Makefile
+++ b/drivers/net/ethernet/chelsio/libcxgb/Makefile
@@ -1,3 +1,5 @@
+ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
+
 obj-$(CONFIG_CHELSIO_LIB) += libcxgb.o
 
-libcxgb-y := libcxgb_ppm.o
+libcxgb-y := libcxgb_ppm.o libcxgb_cm.o
diff --git a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c
new file mode 100644
index 000000000000..0f0de5b63622
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.c
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *	  copyright notice, this list of conditions and the following
+ *	  disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *	  copyright notice, this list of conditions and the following
+ *	  disclaimer in the documentation and/or other materials
+ *	  provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/tcp.h>
+#include <linux/ipv6.h>
+#include <net/route.h>
+#include <net/ip6_route.h>
+
+#include "libcxgb_cm.h"
+
+void
+cxgb_get_4tuple(struct cpl_pass_accept_req *req, enum chip_type type,
+		int *iptype, __u8 *local_ip, __u8 *peer_ip,
+		__be16 *local_port, __be16 *peer_port)
+{
+	int eth_len = (CHELSIO_CHIP_VERSION(type) <= CHELSIO_T5) ?
+		      ETH_HDR_LEN_G(be32_to_cpu(req->hdr_len)) :
+		      T6_ETH_HDR_LEN_G(be32_to_cpu(req->hdr_len));
+	int ip_len = (CHELSIO_CHIP_VERSION(type) <= CHELSIO_T5) ?
+		     IP_HDR_LEN_G(be32_to_cpu(req->hdr_len)) :
+		     T6_IP_HDR_LEN_G(be32_to_cpu(req->hdr_len));
+	struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len);
+	struct ipv6hdr *ip6 = (struct ipv6hdr *)((u8 *)(req + 1) + eth_len);
+	struct tcphdr *tcp = (struct tcphdr *)
+			     ((u8 *)(req + 1) + eth_len + ip_len);
+
+	if (ip->version == 4) {
+		pr_debug("%s saddr 0x%x daddr 0x%x sport %u dport %u\n",
+			 __func__, ntohl(ip->saddr), ntohl(ip->daddr),
+			 ntohs(tcp->source), ntohs(tcp->dest));
+		*iptype = 4;
+		memcpy(peer_ip, &ip->saddr, 4);
+		memcpy(local_ip, &ip->daddr, 4);
+	} else {
+		pr_debug("%s saddr %pI6 daddr %pI6 sport %u dport %u\n",
+			 __func__, ip6->saddr.s6_addr, ip6->daddr.s6_addr,
+			 ntohs(tcp->source), ntohs(tcp->dest));
+		*iptype = 6;
+		memcpy(peer_ip, ip6->saddr.s6_addr, 16);
+		memcpy(local_ip, ip6->daddr.s6_addr, 16);
+	}
+	*peer_port = tcp->source;
+	*local_port = tcp->dest;
+}
+EXPORT_SYMBOL(cxgb_get_4tuple);
+
+static bool
+cxgb_our_interface(struct cxgb4_lld_info *lldi,
+		   struct net_device *(*get_real_dev)(struct net_device *),
+		   struct net_device *egress_dev)
+{
+	int i;
+
+	egress_dev = get_real_dev(egress_dev);
+	for (i = 0; i < lldi->nports; i++)
+		if (lldi->ports[i] == egress_dev)
+			return true;
+	return false;
+}
+
+struct dst_entry *
+cxgb_find_route(struct cxgb4_lld_info *lldi,
+		struct net_device *(*get_real_dev)(struct net_device *),
+		__be32 local_ip, __be32 peer_ip, __be16 local_port,
+		__be16 peer_port, u8 tos)
+{
+	struct rtable *rt;
+	struct flowi4 fl4;
+	struct neighbour *n;
+
+	rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip,
+				   peer_port, local_port, IPPROTO_TCP,
+				   tos, 0);
+	if (IS_ERR(rt))
+		return NULL;
+	n = dst_neigh_lookup(&rt->dst, &peer_ip);
+	if (!n)
+		return NULL;
+	if (!cxgb_our_interface(lldi, get_real_dev, n->dev) &&
+	    !(n->dev->flags & IFF_LOOPBACK)) {
+		neigh_release(n);
+		dst_release(&rt->dst);
+		return NULL;
+	}
+	neigh_release(n);
+	return &rt->dst;
+}
+EXPORT_SYMBOL(cxgb_find_route);
+
+struct dst_entry *
+cxgb_find_route6(struct cxgb4_lld_info *lldi,
+		 struct net_device *(*get_real_dev)(struct net_device *),
+		 __u8 *local_ip, __u8 *peer_ip, __be16 local_port,
+		 __be16 peer_port, u8 tos, __u32 sin6_scope_id)
+{
+	struct dst_entry *dst = NULL;
+
+	if (IS_ENABLED(CONFIG_IPV6)) {
+		struct flowi6 fl6;
+
+		memset(&fl6, 0, sizeof(fl6));
+		memcpy(&fl6.daddr, peer_ip, 16);
+		memcpy(&fl6.saddr, local_ip, 16);
+		if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
+			fl6.flowi6_oif = sin6_scope_id;
+		dst = ip6_route_output(&init_net, NULL, &fl6);
+		if (!dst)
+			goto out;
+		if (!cxgb_our_interface(lldi, get_real_dev,
+					ip6_dst_idev(dst)->dev) &&
+		    !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK)) {
+			dst_release(dst);
+			dst = NULL;
+		}
+	}
+
+out:
+	return dst;
+}
+EXPORT_SYMBOL(cxgb_find_route6);
diff --git a/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h
new file mode 100644
index 000000000000..515b94ff9080
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/libcxgb/libcxgb_cm.h
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *	  copyright notice, this list of conditions and the following
+ *	  disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *	  copyright notice, this list of conditions and the following
+ *	  disclaimer in the documentation and/or other materials
+ *	  provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __LIBCXGB_CM_H__
+#define __LIBCXGB_CM_H__
+
+
+#include <net/tcp.h>
+
+#include <cxgb4.h>
+#include <t4_msg.h>
+#include <l2t.h>
+
+void
+cxgb_get_4tuple(struct cpl_pass_accept_req *, enum chip_type,
+		int *, __u8 *, __u8 *, __be16 *, __be16 *);
+struct dst_entry *
+cxgb_find_route(struct cxgb4_lld_info *,
+		struct net_device *(*)(struct net_device *),
+		__be32, __be32, __be16,	__be16, u8);
+struct dst_entry *
+cxgb_find_route6(struct cxgb4_lld_info *,
+		 struct net_device *(*)(struct net_device *),
+		 __u8 *, __u8 *, __be16, __be16, u8, __u32);
+
+/* Returns whether a CPL status conveys negative advice.
+ */
+static inline bool cxgb_is_neg_adv(unsigned int status)
+{
+	return status == CPL_ERR_RTX_NEG_ADVICE ||
+	       status == CPL_ERR_PERSIST_NEG_ADVICE ||
+	       status == CPL_ERR_KEEPALV_NEG_ADVICE;
+}
+
+static inline void
+cxgb_best_mtu(const unsigned short *mtus, unsigned short mtu,
+	      unsigned int *idx, int use_ts, int ipv6)
+{
+	unsigned short hdr_size = (ipv6 ?
+				   sizeof(struct ipv6hdr) :
+				   sizeof(struct iphdr)) +
+				  sizeof(struct tcphdr) +
+				  (use_ts ?
+				   round_up(TCPOLEN_TIMESTAMP, 4) : 0);
+	unsigned short data_size = mtu - hdr_size;
+
+	cxgb4_best_aligned_mtu(mtus, hdr_size, data_size, 8, idx);
+}
+
+static inline u32 cxgb_compute_wscale(u32 win)
+{
+	u32 wscale = 0;
+
+	while (wscale < 14 && (65535 << wscale) < win)
+		wscale++;
+	return wscale;
+}
+
+static inline void
+cxgb_mk_tid_release(struct sk_buff *skb, u32 len, u32 tid, u16 chan)
+{
+	struct cpl_tid_release *req;
+
+	req = (struct cpl_tid_release *)__skb_put(skb, len);
+	memset(req, 0, len);
+
+	INIT_TP_WR(req, tid);
+	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
+	set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
+}
+
+static inline void
+cxgb_mk_close_con_req(struct sk_buff *skb, u32 len, u32 tid, u16 chan,
+		      void *handle, arp_err_handler_t handler)
+{
+	struct cpl_close_con_req *req;
+
+	req = (struct cpl_close_con_req *)__skb_put(skb, len);
+	memset(req, 0, len);
+
+	INIT_TP_WR(req, tid);
+	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
+	set_wr_txq(skb, CPL_PRIORITY_DATA, chan);
+	t4_set_arp_err_handler(skb, handle, handler);
+}
+
+static inline void
+cxgb_mk_abort_req(struct sk_buff *skb, u32 len, u32 tid, u16 chan,
+		  void *handle, arp_err_handler_t handler)
+{
+	struct cpl_abort_req *req;
+
+	req = (struct cpl_abort_req *)__skb_put(skb, len);
+	memset(req, 0, len);
+
+	INIT_TP_WR(req, tid);
+	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
+	req->cmd = CPL_ABORT_SEND_RST;
+	set_wr_txq(skb, CPL_PRIORITY_DATA, chan);
+	t4_set_arp_err_handler(skb, handle, handler);
+}
+
+static inline void
+cxgb_mk_abort_rpl(struct sk_buff *skb, u32 len, u32 tid, u16 chan)
+{
+	struct cpl_abort_rpl *rpl;
+
+	rpl = (struct cpl_abort_rpl *)__skb_put(skb, len);
+	memset(rpl, 0, len);
+
+	INIT_TP_WR(rpl, tid);
+	OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
+	rpl->cmd = CPL_ABORT_NO_RST;
+	set_wr_txq(skb, CPL_PRIORITY_DATA, chan);
+}
+
+static inline void
+cxgb_mk_rx_data_ack(struct sk_buff *skb, u32 len, u32 tid, u16 chan,
+		    u32 credit_dack)
+{
+	struct cpl_rx_data_ack *req;
+
+	req = (struct cpl_rx_data_ack *)__skb_put(skb, len);
+	memset(req, 0, len);
+
+	INIT_TP_WR(req, tid);
+	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid));
+	req->credit_dack = cpu_to_be32(credit_dack);
+	set_wr_txq(skb, CPL_PRIORITY_ACK, chan);
+}
+#endif
diff --git a/drivers/net/ethernet/davicom/dm9000.c b/drivers/net/ethernet/davicom/dm9000.c
index 1471e16ba719..f45385f5c6e5 100644
--- a/drivers/net/ethernet/davicom/dm9000.c
+++ b/drivers/net/ethernet/davicom/dm9000.c
@@ -1299,6 +1299,7 @@ static int
 dm9000_open(struct net_device *dev)
 {
 	struct board_info *db = netdev_priv(dev);
+	unsigned int irq_flags = irq_get_trigger_type(dev->irq);
 
 	if (netif_msg_ifup(db))
 		dev_dbg(db->dev, "enabling %s\n", dev->name);
@@ -1306,9 +1307,11 @@ dm9000_open(struct net_device *dev)
 	/* If there is no IRQ type specified, tell the user that this is a
 	 * problem
 	 */
-	if (irq_get_trigger_type(dev->irq) == IRQF_TRIGGER_NONE)
+	if (irq_flags == IRQF_TRIGGER_NONE)
 		dev_warn(db->dev, "WARNING: no IRQ resource flags set.\n");
 
+	irq_flags |= IRQF_SHARED;
+
 	/* GPIO0 on pre-activate PHY, Reg 1F is not set by reset */
 	iow(db, DM9000_GPR, 0);	/* REG_1F bit0 activate phyxcer */
 	mdelay(1); /* delay needs by DM9000B */
@@ -1316,8 +1319,7 @@ dm9000_open(struct net_device *dev)
 	/* Initialize DM9000 board */
 	dm9000_init_dm9000(dev);
 
-	if (request_irq(dev->irq, dm9000_interrupt, IRQF_SHARED,
-			dev->name, dev))
+	if (request_irq(dev->irq, dm9000_interrupt, irq_flags, dev->name, dev))
 		return -EAGAIN;
 	/* Now that we have an interrupt handler hooked up we can unmask
 	 * our interrupts
diff --git a/drivers/net/ethernet/dec/tulip/de4x5.c b/drivers/net/ethernet/dec/tulip/de4x5.c
index f0e9e2ef62a0..6620fc861c47 100644
--- a/drivers/net/ethernet/dec/tulip/de4x5.c
+++ b/drivers/net/ethernet/dec/tulip/de4x5.c
@@ -1966,7 +1966,7 @@ SetMulticastFilter(struct net_device *dev)
     } else if (lp->setup_f == HASH_PERF) {   /* Hash Filtering */
 	netdev_for_each_mc_addr(ha, dev) {
 		crc = ether_crc_le(ETH_ALEN, ha->addr);
-		hashcode = crc & HASH_BITS;  /* hashcode is 9 LSb of CRC */
+		hashcode = crc & DE4X5_HASH_BITS;  /* hashcode is 9 LSb of CRC */
 
 		byte = hashcode >> 3;        /* bit[3-8] -> byte in filter */
 		bit = 1 << (hashcode & 0x07);/* bit[0-2] -> bit in byte */
@@ -5043,7 +5043,7 @@ build_setup_frame(struct net_device *dev, int mode)
 	    *(pa + i) = dev->dev_addr[i];                 /* Host address */
 	    if (i & 0x01) pa += 2;
 	}
-	*(lp->setup_frame + (HASH_TABLE_LEN >> 3) - 3) = 0x80;
+	*(lp->setup_frame + (DE4X5_HASH_TABLE_LEN >> 3) - 3) = 0x80;
     } else {
 	for (i=0; i<ETH_ALEN; i++) { /* Host address */
 	    *(pa + (i&1)) = dev->dev_addr[i];
diff --git a/drivers/net/ethernet/dec/tulip/de4x5.h b/drivers/net/ethernet/dec/tulip/de4x5.h
index ec756eba397b..1bfdc9b117f6 100644
--- a/drivers/net/ethernet/dec/tulip/de4x5.h
+++ b/drivers/net/ethernet/dec/tulip/de4x5.h
@@ -860,8 +860,8 @@
 #define PCI  0
 #define EISA 1
 
-#define HASH_TABLE_LEN   512       /* Bits */
-#define HASH_BITS        0x01ff    /* 9 LS bits */
+#define DE4X5_HASH_TABLE_LEN   512       /* Bits */
+#define DE4X5_HASH_BITS        0x01ff    /* 9 LS bits */
 
 #define SETUP_FRAME_LEN  192       /* Bytes */
 #define IMPERF_PA_OFFSET 156       /* Bytes */
diff --git a/drivers/net/ethernet/dlink/sundance.c b/drivers/net/ethernet/dlink/sundance.c
index 58c6338a839e..79d80090eac8 100644
--- a/drivers/net/ethernet/dlink/sundance.c
+++ b/drivers/net/ethernet/dlink/sundance.c
@@ -867,7 +867,7 @@ static int netdev_open(struct net_device *dev)
 
 	/* Initialize other registers. */
 	__set_mac_addr(dev);
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
 	iowrite16(dev->mtu + 18, ioaddr + MaxFrameSize);
 #else
 	iowrite16(dev->mtu + 14, ioaddr + MaxFrameSize);
diff --git a/drivers/net/ethernet/emulex/benet/be.h b/drivers/net/ethernet/emulex/benet/be.h
index 4555e041ef69..6cfa63a5e9b4 100644
--- a/drivers/net/ethernet/emulex/benet/be.h
+++ b/drivers/net/ethernet/emulex/benet/be.h
@@ -37,7 +37,7 @@
 #include "be_hw.h"
 #include "be_roce.h"
 
-#define DRV_VER			"11.0.0.0"
+#define DRV_VER			"11.1.0.0"
 #define DRV_NAME		"be2net"
 #define BE_NAME			"Emulex BladeEngine2"
 #define BE3_NAME		"Emulex BladeEngine3"
@@ -399,13 +399,13 @@ enum vf_state {
 #define BE_FLAGS_PHY_MISCONFIGURED		BIT(10)
 #define BE_FLAGS_ERR_DETECTION_SCHEDULED	BIT(11)
 #define BE_FLAGS_OS2BMC				BIT(12)
+#define BE_FLAGS_TRY_RECOVERY			BIT(13)
 
 #define BE_UC_PMAC_COUNT			30
 #define BE_VF_UC_PMAC_COUNT			2
 
 #define MAX_ERR_RECOVERY_RETRY_COUNT		3
 #define ERR_DETECTION_DELAY			1000
-#define ERR_RECOVERY_RETRY_DELAY		30000
 
 /* Ethtool set_dump flags */
 #define LANCER_INITIATE_FW_DUMP			0x1
@@ -508,6 +508,70 @@ struct be_wrb_params {
 	u16 lso_mss;	/* MSS for LSO */
 };
 
+struct be_eth_addr {
+	unsigned char mac[ETH_ALEN];
+};
+
+#define BE_SEC	1000			/* in msec */
+#define BE_MIN	(60 * BE_SEC)		/* in msec */
+#define BE_HOUR	(60 * BE_MIN)		/* in msec */
+
+#define ERR_RECOVERY_MAX_RETRY_COUNT		3
+#define ERR_RECOVERY_DETECTION_DELAY		BE_SEC
+#define ERR_RECOVERY_RETRY_DELAY		(30 * BE_SEC)
+
+/* UE-detection-duration in BEx/Skyhawk:
+ * All PFs must wait for this duration after they detect UE before reading
+ * SLIPORT_SEMAPHORE register. At the end of this duration, the Firmware
+ * guarantees that the SLIPORT_SEMAPHORE register is updated to indicate
+ * if the UE is recoverable.
+ */
+#define ERR_RECOVERY_UE_DETECT_DURATION			BE_SEC
+
+/* Initial idle time (in msec) to elapse after driver load,
+ * before UE recovery is allowed.
+ */
+#define ERR_IDLE_HR			24
+#define ERR_RECOVERY_IDLE_TIME		(ERR_IDLE_HR * BE_HOUR)
+
+/* Time interval (in msec) after which UE recovery can be repeated */
+#define ERR_INTERVAL_HR			72
+#define ERR_RECOVERY_INTERVAL		(ERR_INTERVAL_HR * BE_HOUR)
+
+/* BEx/SH UE recovery state machine */
+enum {
+	ERR_RECOVERY_ST_NONE = 0,		/* No Recovery */
+	ERR_RECOVERY_ST_DETECT = 1,		/* UE detection duration */
+	ERR_RECOVERY_ST_RESET = 2,		/* Reset Phase (PF0 only) */
+	ERR_RECOVERY_ST_PRE_POLL = 3,		/* Pre-Poll Phase (all PFs) */
+	ERR_RECOVERY_ST_REINIT = 4		/* Re-initialize Phase */
+};
+
+struct be_error_recovery {
+	/* Lancer error recovery variables */
+	u8 recovery_retries;
+
+	/* BEx/Skyhawk error recovery variables */
+	u8 recovery_state;
+	u16 ue_to_reset_time;		/* Time after UE, to soft reset
+					 * the chip - PF0 only
+					 */
+	u16 ue_to_poll_time;		/* Time after UE, to Restart Polling
+					 * of SLIPORT_SEMAPHORE reg
+					 */
+	u16 last_err_code;
+	bool recovery_supported;
+	unsigned long probe_time;
+	unsigned long last_recovery_time;
+
+	/* Common to both Lancer & BEx/SH error recovery */
+	u32 resched_delay;
+	struct delayed_work err_detection_work;
+};
+
+/* Ethtool priv_flags */
+#define	BE_DISABLE_TPE_RECOVERY	0x1
+
 struct be_adapter {
 	struct pci_dev *pdev;
 	struct net_device *netdev;
@@ -523,7 +587,7 @@ struct be_adapter {
 	struct be_dma_mem mbox_mem_alloced;
 
 	struct be_mcc_obj mcc_obj;
-	spinlock_t mcc_lock;	/* For serializing mcc cmds to BE card */
+	struct mutex mcc_lock;	/* For serializing mcc cmds to BE card */
 	spinlock_t mcc_cq_lock;
 
 	u16 cfg_num_rx_irqs;		/* configured via set-channels */
@@ -556,7 +620,6 @@ struct be_adapter {
 	struct delayed_work work;
 	u16 work_counter;
 
-	struct delayed_work be_err_detection_work;
 	u8 recovery_retries;
 	u8 err_flags;
 	bool pcicfg_mapped;	/* pcicfg obtained via pci_iomap() */
@@ -570,9 +633,15 @@ struct be_adapter {
 	int if_handle;		/* Used to configure filtering */
 	u32 if_flags;		/* Interface filtering flags */
 	u32 *pmac_id;		/* MAC addr handle used by BE card */
+	struct be_eth_addr *uc_list;/* list of uc-addrs programmed (not perm) */
 	u32 uc_macs;		/* Count of secondary UC MAC programmed */
+	struct be_eth_addr *mc_list;/* list of mcast addrs programmed */
+	u32 mc_count;
 	unsigned long vids[BITS_TO_LONGS(VLAN_N_VID)];
 	u16 vlans_added;
+	bool update_uc_list;
+	bool update_mc_list;
+	struct mutex rx_filter_lock;/* For protecting vids[] & mc/uc_list[] */
 
 	u32 beacon_state;	/* for set_phys_id */
 
@@ -624,6 +693,18 @@ struct be_adapter {
 	u32 fat_dump_len;
 	u16 serial_num[CNTL_SERIAL_NUM_WORDS];
 	u8 phy_state; /* state of sfp optics (functional, faulted, etc.,) */
+	u8 dev_mac[ETH_ALEN];
+	u32 priv_flags; /* ethtool get/set_priv_flags() */
+	struct be_error_recovery error_recovery;
+};
+
+/* Used for defered FW config cmds. Add fields to this struct as reqd */
+struct be_cmd_work {
+	struct work_struct work;
+	struct be_adapter *adapter;
+	union {
+		__be16 vxlan_port;
+	} info;
 };
 
 #define be_physfn(adapter)		(!adapter->virtfn)
@@ -848,6 +929,9 @@ static inline bool is_ipv4_pkt(struct sk_buff *skb)
 	return skb->protocol == htons(ETH_P_IP) && ip_hdr(skb)->version == 4;
 }
 
+#define be_error_recovering(adapter)	\
+		(adapter->flags & BE_FLAGS_TRY_RECOVERY)
+
 #define BE_ERROR_EEH		1
 #define BE_ERROR_UE		BIT(1)
 #define BE_ERROR_FW		BIT(2)
diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c
index 2cc11756859f..1fb5d7239254 100644
--- a/drivers/net/ethernet/emulex/benet/be_cmds.c
+++ b/drivers/net/ethernet/emulex/benet/be_cmds.c
@@ -92,6 +92,11 @@ static struct be_cmd_priv_map cmd_priv_map[] = {
 		CMD_SUBSYSTEM_COMMON,
 		BE_PRIV_DEVCFG | BE_PRIV_VHADM
 	},
+	{
+		OPCODE_COMMON_GET_EXT_FAT_CAPABILITIES,
+		CMD_SUBSYSTEM_COMMON,
+		BE_PRIV_DEVCFG
+	}
 };
 
 static bool be_cmd_allowed(struct be_adapter *adapter, u8 opcode, u8 subsystem)
@@ -571,7 +576,7 @@ int be_process_mcc(struct be_adapter *adapter)
 /* Wait till no more pending mcc requests are present */
 static int be_mcc_wait_compl(struct be_adapter *adapter)
 {
-#define mcc_timeout		120000 /* 12s timeout */
+#define mcc_timeout		12000 /* 12s timeout */
 	int i, status = 0;
 	struct be_mcc_obj *mcc_obj = &adapter->mcc_obj;
 
@@ -585,7 +590,7 @@ static int be_mcc_wait_compl(struct be_adapter *adapter)
 
 		if (atomic_read(&mcc_obj->q.used) == 0)
 			break;
-		udelay(100);
+		usleep_range(500, 1000);
 	}
 	if (i == mcc_timeout) {
 		dev_err(&adapter->pdev->dev, "FW not responding\n");
@@ -705,7 +710,7 @@ static int be_mbox_notify_wait(struct be_adapter *adapter)
 	return 0;
 }
 
-static u16 be_POST_stage_get(struct be_adapter *adapter)
+u16 be_POST_stage_get(struct be_adapter *adapter)
 {
 	u32 sem;
 
@@ -863,7 +868,7 @@ static bool use_mcc(struct be_adapter *adapter)
 static int be_cmd_lock(struct be_adapter *adapter)
 {
 	if (use_mcc(adapter)) {
-		spin_lock_bh(&adapter->mcc_lock);
+		mutex_lock(&adapter->mcc_lock);
 		return 0;
 	} else {
 		return mutex_lock_interruptible(&adapter->mbox_lock);
@@ -874,7 +879,7 @@ static int be_cmd_lock(struct be_adapter *adapter)
 static void be_cmd_unlock(struct be_adapter *adapter)
 {
 	if (use_mcc(adapter))
-		spin_unlock_bh(&adapter->mcc_lock);
+		return mutex_unlock(&adapter->mcc_lock);
 	else
 		return mutex_unlock(&adapter->mbox_lock);
 }
@@ -1044,7 +1049,7 @@ int be_cmd_mac_addr_query(struct be_adapter *adapter, u8 *mac_addr,
 	struct be_cmd_req_mac_query *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1073,7 +1078,7 @@ int be_cmd_mac_addr_query(struct be_adapter *adapter, u8 *mac_addr,
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1085,7 +1090,7 @@ int be_cmd_pmac_add(struct be_adapter *adapter, u8 *mac_addr,
 	struct be_cmd_req_pmac_add *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1110,7 +1115,7 @@ int be_cmd_pmac_add(struct be_adapter *adapter, u8 *mac_addr,
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 
 	 if (status == MCC_STATUS_UNAUTHORIZED_REQUEST)
 		status = -EPERM;
@@ -1128,7 +1133,7 @@ int be_cmd_pmac_del(struct be_adapter *adapter, u32 if_id, int pmac_id, u32 dom)
 	if (pmac_id == -1)
 		return 0;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1148,7 +1153,7 @@ int be_cmd_pmac_del(struct be_adapter *adapter, u32 if_id, int pmac_id, u32 dom)
 	status = be_mcc_notify_wait(adapter);
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1411,7 +1416,7 @@ int be_cmd_rxq_create(struct be_adapter *adapter,
 	struct be_dma_mem *q_mem = &rxq->dma_mem;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1441,7 +1446,7 @@ int be_cmd_rxq_create(struct be_adapter *adapter,
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1505,7 +1510,7 @@ int be_cmd_rxq_destroy(struct be_adapter *adapter, struct be_queue_info *q)
 	struct be_cmd_req_q_destroy *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1522,7 +1527,7 @@ int be_cmd_rxq_destroy(struct be_adapter *adapter, struct be_queue_info *q)
 	q->created = false;
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1590,7 +1595,7 @@ int be_cmd_get_stats(struct be_adapter *adapter, struct be_dma_mem *nonemb_cmd)
 	struct be_cmd_req_hdr *hdr;
 	int status = 0;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1618,7 +1623,7 @@ int be_cmd_get_stats(struct be_adapter *adapter, struct be_dma_mem *nonemb_cmd)
 	adapter->stats_cmd_sent = true;
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1634,7 +1639,7 @@ int lancer_cmd_get_pport_stats(struct be_adapter *adapter,
 			    CMD_SUBSYSTEM_ETH))
 		return -EPERM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1657,7 +1662,7 @@ int lancer_cmd_get_pport_stats(struct be_adapter *adapter,
 	adapter->stats_cmd_sent = true;
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1694,7 +1699,7 @@ int be_cmd_link_status_query(struct be_adapter *adapter, u16 *link_speed,
 	struct be_cmd_req_link_status *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	if (link_status)
 		*link_status = LINK_DOWN;
@@ -1733,7 +1738,7 @@ int be_cmd_link_status_query(struct be_adapter *adapter, u16 *link_speed,
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1744,7 +1749,7 @@ int be_cmd_get_die_temperature(struct be_adapter *adapter)
 	struct be_cmd_req_get_cntl_addnl_attribs *req;
 	int status = 0;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1759,7 +1764,7 @@ int be_cmd_get_die_temperature(struct be_adapter *adapter)
 
 	status = be_mcc_notify(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1808,7 +1813,7 @@ int be_cmd_get_fat_dump(struct be_adapter *adapter, u32 buf_len, void *buf)
 	if (!get_fat_cmd.va)
 		return -ENOMEM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	while (total_size) {
 		buf_size = min(total_size, (u32)60*1024);
@@ -1848,7 +1853,7 @@ int be_cmd_get_fat_dump(struct be_adapter *adapter, u32 buf_len, void *buf)
 err:
 	dma_free_coherent(&adapter->pdev->dev, get_fat_cmd.size,
 			  get_fat_cmd.va, get_fat_cmd.dma);
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1859,7 +1864,7 @@ int be_cmd_get_fw_ver(struct be_adapter *adapter)
 	struct be_cmd_req_get_fw_version *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1882,7 +1887,7 @@ int be_cmd_get_fw_ver(struct be_adapter *adapter)
 			sizeof(adapter->fw_on_flash));
 	}
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1896,7 +1901,7 @@ static int __be_cmd_modify_eqd(struct be_adapter *adapter,
 	struct be_cmd_req_modify_eq_delay *req;
 	int status = 0, i;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1919,7 +1924,7 @@ static int __be_cmd_modify_eqd(struct be_adapter *adapter,
 
 	status = be_mcc_notify(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1946,7 +1951,7 @@ int be_cmd_vlan_config(struct be_adapter *adapter, u32 if_id, u16 *vtag_array,
 	struct be_cmd_req_vlan_config *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1968,7 +1973,7 @@ int be_cmd_vlan_config(struct be_adapter *adapter, u32 if_id, u16 *vtag_array,
 
 	status = be_mcc_notify_wait(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -1979,7 +1984,7 @@ static int __be_cmd_rx_filter(struct be_adapter *adapter, u32 flags, u32 value)
 	struct be_cmd_req_rx_filter *req = mem->va;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -1996,8 +2001,7 @@ static int __be_cmd_rx_filter(struct be_adapter *adapter, u32 flags, u32 value)
 	req->if_flags = (value == ON) ? req->if_flags_mask : 0;
 
 	if (flags & BE_IF_FLAGS_MULTICAST) {
-		struct netdev_hw_addr *ha;
-		int i = 0;
+		int i;
 
 		/* Reset mcast promisc mode if already set by setting mask
 		 * and not setting flags field
@@ -2005,14 +2009,15 @@ static int __be_cmd_rx_filter(struct be_adapter *adapter, u32 flags, u32 value)
 		req->if_flags_mask |=
 			cpu_to_le32(BE_IF_FLAGS_MCAST_PROMISCUOUS &
 				    be_if_cap_flags(adapter));
-		req->mcast_num = cpu_to_le32(netdev_mc_count(adapter->netdev));
-		netdev_for_each_mc_addr(ha, adapter->netdev)
-			memcpy(req->mcast_mac[i++].byte, ha->addr, ETH_ALEN);
+		req->mcast_num = cpu_to_le32(adapter->mc_count);
+		for (i = 0; i < adapter->mc_count; i++)
+			ether_addr_copy(req->mcast_mac[i].byte,
+					adapter->mc_list[i].mac);
 	}
 
 	status = be_mcc_notify_wait(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2043,7 +2048,7 @@ int be_cmd_set_flow_control(struct be_adapter *adapter, u32 tx_fc, u32 rx_fc)
 			    CMD_SUBSYSTEM_COMMON))
 		return -EPERM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -2063,7 +2068,7 @@ int be_cmd_set_flow_control(struct be_adapter *adapter, u32 tx_fc, u32 rx_fc)
 	status = be_mcc_notify_wait(adapter);
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 
 	if (base_status(status) == MCC_STATUS_FEATURE_NOT_SUPPORTED)
 		return  -EOPNOTSUPP;
@@ -2082,7 +2087,7 @@ int be_cmd_get_flow_control(struct be_adapter *adapter, u32 *tx_fc, u32 *rx_fc)
 			    CMD_SUBSYSTEM_COMMON))
 		return -EPERM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -2105,7 +2110,7 @@ int be_cmd_get_flow_control(struct be_adapter *adapter, u32 *tx_fc, u32 *rx_fc)
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2186,7 +2191,7 @@ int be_cmd_rss_config(struct be_adapter *adapter, u8 *rsstable,
 	if (!(be_if_cap_flags(adapter) & BE_IF_FLAGS_RSS))
 		return 0;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -2211,7 +2216,7 @@ int be_cmd_rss_config(struct be_adapter *adapter, u8 *rsstable,
 
 	status = be_mcc_notify_wait(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2223,7 +2228,7 @@ int be_cmd_set_beacon_state(struct be_adapter *adapter, u8 port_num,
 	struct be_cmd_req_enable_disable_beacon *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -2244,7 +2249,7 @@ int be_cmd_set_beacon_state(struct be_adapter *adapter, u8 port_num,
 	status = be_mcc_notify_wait(adapter);
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2255,7 +2260,7 @@ int be_cmd_get_beacon_state(struct be_adapter *adapter, u8 port_num, u32 *state)
 	struct be_cmd_req_get_beacon_state *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -2279,7 +2284,7 @@ int be_cmd_get_beacon_state(struct be_adapter *adapter, u8 port_num, u32 *state)
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2303,7 +2308,7 @@ int be_cmd_read_port_transceiver_data(struct be_adapter *adapter,
 		return -ENOMEM;
 	}
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -2325,7 +2330,7 @@ int be_cmd_read_port_transceiver_data(struct be_adapter *adapter,
 		memcpy(data, resp->page_data, PAGE_DATA_LEN);
 	}
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
 	return status;
 }
@@ -2342,7 +2347,7 @@ static int lancer_cmd_write_object(struct be_adapter *adapter,
 	void *ctxt = NULL;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 	adapter->flash_status = 0;
 
 	wrb = wrb_from_mccq(adapter);
@@ -2384,7 +2389,7 @@ static int lancer_cmd_write_object(struct be_adapter *adapter,
 	if (status)
 		goto err_unlock;
 
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 
 	if (!wait_for_completion_timeout(&adapter->et_cmd_compl,
 					 msecs_to_jiffies(60000)))
@@ -2403,7 +2408,7 @@ static int lancer_cmd_write_object(struct be_adapter *adapter,
 	return status;
 
 err_unlock:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2457,7 +2462,7 @@ static int lancer_cmd_delete_object(struct be_adapter *adapter,
 	struct be_mcc_wrb *wrb;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -2475,7 +2480,7 @@ static int lancer_cmd_delete_object(struct be_adapter *adapter,
 
 	status = be_mcc_notify_wait(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2488,7 +2493,7 @@ int lancer_cmd_read_object(struct be_adapter *adapter, struct be_dma_mem *cmd,
 	struct lancer_cmd_resp_read_object *resp;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -2522,7 +2527,7 @@ int lancer_cmd_read_object(struct be_adapter *adapter, struct be_dma_mem *cmd,
 	}
 
 err_unlock:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2534,7 +2539,7 @@ static int be_cmd_write_flashrom(struct be_adapter *adapter,
 	struct be_cmd_write_flashrom *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 	adapter->flash_status = 0;
 
 	wrb = wrb_from_mccq(adapter);
@@ -2559,7 +2564,7 @@ static int be_cmd_write_flashrom(struct be_adapter *adapter,
 	if (status)
 		goto err_unlock;
 
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 
 	if (!wait_for_completion_timeout(&adapter->et_cmd_compl,
 					 msecs_to_jiffies(40000)))
@@ -2570,7 +2575,7 @@ static int be_cmd_write_flashrom(struct be_adapter *adapter,
 	return status;
 
 err_unlock:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2581,7 +2586,7 @@ static int be_cmd_get_flash_crc(struct be_adapter *adapter, u8 *flashed_crc,
 	struct be_mcc_wrb *wrb;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -2608,7 +2613,7 @@ static int be_cmd_get_flash_crc(struct be_adapter *adapter, u8 *flashed_crc,
 		memcpy(flashed_crc, req->crc, 4);
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -2723,6 +2728,26 @@ static int be_flash(struct be_adapter *adapter, const u8 *img,
 	return 0;
 }
 
+#define NCSI_UPDATE_LOG	"NCSI section update is not supported in FW ver %s\n"
+static bool be_fw_ncsi_supported(char *ver)
+{
+	int v1[4] = {3, 102, 148, 0}; /* Min ver that supports NCSI FW */
+	int v2[4];
+	int i;
+
+	if (sscanf(ver, "%d.%d.%d.%d", &v2[0], &v2[1], &v2[2], &v2[3]) != 4)
+		return false;
+
+	for (i = 0; i < 4; i++) {
+		if (v1[i] < v2[i])
+			return true;
+		else if (v1[i] > v2[i])
+			return false;
+	}
+
+	return true;
+}
+
 /* For BE2, BE3 and BE3-R */
 static int be_flash_BEx(struct be_adapter *adapter,
 			const struct firmware *fw,
@@ -2800,8 +2825,10 @@ static int be_flash_BEx(struct be_adapter *adapter,
 			continue;
 
 		if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
-		    memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
+		    !be_fw_ncsi_supported(adapter->fw_ver)) {
+			dev_info(dev, NCSI_UPDATE_LOG, adapter->fw_ver);
 			continue;
+		}
 
 		if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
 		    !phy_flashing_required(adapter))
@@ -3192,7 +3219,7 @@ int be_cmd_enable_magic_wol(struct be_adapter *adapter, u8 *mac,
 	struct be_cmd_req_acpi_wol_magic_config *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3209,7 +3236,7 @@ int be_cmd_enable_magic_wol(struct be_adapter *adapter, u8 *mac,
 	status = be_mcc_notify_wait(adapter);
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3224,7 +3251,7 @@ int be_cmd_set_loopback(struct be_adapter *adapter, u8 port_num,
 			    CMD_SUBSYSTEM_LOWLEVEL))
 		return -EPERM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3247,7 +3274,7 @@ int be_cmd_set_loopback(struct be_adapter *adapter, u8 port_num,
 	if (status)
 		goto err_unlock;
 
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 
 	if (!wait_for_completion_timeout(&adapter->et_cmd_compl,
 					 msecs_to_jiffies(SET_LB_MODE_TIMEOUT)))
@@ -3256,7 +3283,7 @@ int be_cmd_set_loopback(struct be_adapter *adapter, u8 port_num,
 	return status;
 
 err_unlock:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3273,7 +3300,7 @@ int be_cmd_loopback_test(struct be_adapter *adapter, u32 port_num,
 			    CMD_SUBSYSTEM_LOWLEVEL))
 		return -EPERM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3299,7 +3326,7 @@ int be_cmd_loopback_test(struct be_adapter *adapter, u32 port_num,
 	if (status)
 		goto err;
 
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 
 	wait_for_completion(&adapter->et_cmd_compl);
 	resp = embedded_payload(wrb);
@@ -3307,7 +3334,7 @@ int be_cmd_loopback_test(struct be_adapter *adapter, u32 port_num,
 
 	return status;
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3323,7 +3350,7 @@ int be_cmd_ddr_dma_test(struct be_adapter *adapter, u64 pattern,
 			    CMD_SUBSYSTEM_LOWLEVEL))
 		return -EPERM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3357,7 +3384,7 @@ int be_cmd_ddr_dma_test(struct be_adapter *adapter, u64 pattern,
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3368,7 +3395,7 @@ int be_cmd_get_seeprom_data(struct be_adapter *adapter,
 	struct be_cmd_req_seeprom_read *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3384,7 +3411,7 @@ int be_cmd_get_seeprom_data(struct be_adapter *adapter,
 	status = be_mcc_notify_wait(adapter);
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3399,7 +3426,7 @@ int be_cmd_get_phy_info(struct be_adapter *adapter)
 			    CMD_SUBSYSTEM_COMMON))
 		return -EPERM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3444,7 +3471,7 @@ int be_cmd_get_phy_info(struct be_adapter *adapter)
 	}
 	dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3454,7 +3481,7 @@ static int be_cmd_set_qos(struct be_adapter *adapter, u32 bps, u32 domain)
 	struct be_cmd_req_set_qos *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3474,7 +3501,7 @@ static int be_cmd_set_qos(struct be_adapter *adapter, u32 bps, u32 domain)
 	status = be_mcc_notify_wait(adapter);
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3522,6 +3549,11 @@ int be_cmd_get_cntl_attributes(struct be_adapter *adapter)
 		for (i = 0; i < CNTL_SERIAL_NUM_WORDS; i++)
 			adapter->serial_num[i] = le32_to_cpu(serial_num[i]) &
 				(BIT_MASK(16) - 1);
+		/* For BEx, since GET_FUNC_CONFIG command is not
+		 * supported, we read funcnum here as a workaround.
+		 */
+		if (BEx_chip(adapter))
+			adapter->pf_num = attribs->hba_attribs.pci_funcnum;
 	}
 
 err:
@@ -3581,7 +3613,7 @@ int be_cmd_get_fn_privileges(struct be_adapter *adapter, u32 *privilege,
 	struct be_cmd_req_get_fn_privileges *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3613,7 +3645,7 @@ int be_cmd_get_fn_privileges(struct be_adapter *adapter, u32 *privilege,
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3625,7 +3657,7 @@ int be_cmd_set_fn_privileges(struct be_adapter *adapter, u32 privileges,
 	struct be_cmd_req_set_fn_privileges *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3645,7 +3677,7 @@ int be_cmd_set_fn_privileges(struct be_adapter *adapter, u32 privileges,
 
 	status = be_mcc_notify_wait(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3677,7 +3709,7 @@ int be_cmd_get_mac_from_list(struct be_adapter *adapter, u8 *mac,
 		return -ENOMEM;
 	}
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3741,7 +3773,7 @@ int be_cmd_get_mac_from_list(struct be_adapter *adapter, u8 *mac,
 	}
 
 out:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	dma_free_coherent(&adapter->pdev->dev, get_mac_list_cmd.size,
 			  get_mac_list_cmd.va, get_mac_list_cmd.dma);
 	return status;
@@ -3801,7 +3833,7 @@ int be_cmd_set_mac_list(struct be_adapter *adapter, u8 *mac_array,
 	if (!cmd.va)
 		return -ENOMEM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3823,7 +3855,7 @@ int be_cmd_set_mac_list(struct be_adapter *adapter, u8 *mac_array,
 
 err:
 	dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3859,7 +3891,7 @@ int be_cmd_set_hsw_config(struct be_adapter *adapter, u16 pvid,
 			    CMD_SUBSYSTEM_COMMON))
 		return -EPERM;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3900,7 +3932,7 @@ int be_cmd_set_hsw_config(struct be_adapter *adapter, u16 pvid,
 	status = be_mcc_notify_wait(adapter);
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -3914,7 +3946,7 @@ int be_cmd_get_hsw_config(struct be_adapter *adapter, u16 *pvid,
 	int status;
 	u16 vid;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -3961,7 +3993,7 @@ int be_cmd_get_hsw_config(struct be_adapter *adapter, u16 *pvid,
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -4127,6 +4159,10 @@ int be_cmd_get_ext_fat_capabilites(struct be_adapter *adapter,
 	struct be_cmd_req_get_ext_fat_caps *req;
 	int status;
 
+	if (!be_cmd_allowed(adapter, OPCODE_COMMON_GET_EXT_FAT_CAPABILITIES,
+			    CMD_SUBSYSTEM_COMMON))
+		return -EPERM;
+
 	if (mutex_lock_interruptible(&adapter->mbox_lock))
 		return -1;
 
@@ -4138,7 +4174,7 @@ int be_cmd_get_ext_fat_capabilites(struct be_adapter *adapter,
 
 	req = cmd->va;
 	be_wrb_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_COMMON,
-			       OPCODE_COMMON_GET_EXT_FAT_CAPABILITES,
+			       OPCODE_COMMON_GET_EXT_FAT_CAPABILITIES,
 			       cmd->size, wrb, cmd);
 	req->parameter_type = cpu_to_le32(1);
 
@@ -4156,7 +4192,7 @@ int be_cmd_set_ext_fat_capabilites(struct be_adapter *adapter,
 	struct be_cmd_req_set_ext_fat_caps *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -4167,12 +4203,12 @@ int be_cmd_set_ext_fat_capabilites(struct be_adapter *adapter,
 	req = cmd->va;
 	memcpy(&req->set_params, configs, sizeof(struct be_fat_conf_params));
 	be_wrb_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_COMMON,
-			       OPCODE_COMMON_SET_EXT_FAT_CAPABILITES,
+			       OPCODE_COMMON_SET_EXT_FAT_CAPABILITIES,
 			       cmd->size, wrb, cmd);
 
 	status = be_mcc_notify_wait(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -4373,7 +4409,7 @@ err:
 }
 
 /* This routine returns a list of all the NIC PF_nums in the adapter */
-u16 be_get_nic_pf_num_list(u8 *buf, u32 desc_count, u16 *nic_pf_nums)
+static u16 be_get_nic_pf_num_list(u8 *buf, u32 desc_count, u16 *nic_pf_nums)
 {
 	struct be_res_desc_hdr *hdr = (struct be_res_desc_hdr *)buf;
 	struct be_pcie_res_desc *pcie = NULL;
@@ -4525,7 +4561,7 @@ static int be_cmd_set_profile_config(struct be_adapter *adapter, void *desc,
 }
 
 /* Mark all fields invalid */
-void be_reset_nic_desc(struct be_nic_res_desc *nic)
+static void be_reset_nic_desc(struct be_nic_res_desc *nic)
 {
 	memset(nic, 0, sizeof(*nic));
 	nic->unicast_mac_count = 0xFFFF;
@@ -4650,7 +4686,7 @@ int be_cmd_manage_iface(struct be_adapter *adapter, u32 iface, u8 op)
 	if (iface == 0xFFFFFFFF)
 		return -1;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -4667,7 +4703,7 @@ int be_cmd_manage_iface(struct be_adapter *adapter, u32 iface, u8 op)
 
 	status = be_mcc_notify_wait(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -4701,7 +4737,7 @@ int be_cmd_get_if_id(struct be_adapter *adapter, struct be_vf_cfg *vf_cfg,
 	struct be_cmd_resp_get_iface_list *resp;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -4722,7 +4758,7 @@ int be_cmd_get_if_id(struct be_adapter *adapter, struct be_vf_cfg *vf_cfg,
 	}
 
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -4816,7 +4852,7 @@ int be_cmd_enable_vf(struct be_adapter *adapter, u8 domain)
 	if (BEx_chip(adapter))
 		return 0;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -4834,7 +4870,7 @@ int be_cmd_enable_vf(struct be_adapter *adapter, u8 domain)
 	req->enable = 1;
 	status = be_mcc_notify_wait(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -4898,14 +4934,15 @@ err:
 	return status;
 }
 
-int __be_cmd_set_logical_link_config(struct be_adapter *adapter,
-				     int link_state, int version, u8 domain)
+static int
+__be_cmd_set_logical_link_config(struct be_adapter *adapter,
+				 int link_state, int version, u8 domain)
 {
 	struct be_mcc_wrb *wrb;
 	struct be_cmd_req_set_ll_link *req;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -4931,7 +4968,7 @@ int __be_cmd_set_logical_link_config(struct be_adapter *adapter,
 
 	status = be_mcc_notify_wait(adapter);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 
@@ -4940,7 +4977,7 @@ int be_cmd_set_logical_link_config(struct be_adapter *adapter,
 {
 	int status;
 
-	if (BEx_chip(adapter))
+	if (BE2_chip(adapter))
 		return -EOPNOTSUPP;
 
 	status = __be_cmd_set_logical_link_config(adapter, link_state,
@@ -4954,6 +4991,57 @@ int be_cmd_set_logical_link_config(struct be_adapter *adapter,
 							  1, domain);
 	return status;
 }
+
+int be_cmd_set_features(struct be_adapter *adapter)
+{
+	struct be_cmd_resp_set_features *resp;
+	struct be_cmd_req_set_features *req;
+	struct be_mcc_wrb *wrb;
+	int status;
+
+	if (mutex_lock_interruptible(&adapter->mcc_lock))
+		return -1;
+
+	wrb = wrb_from_mccq(adapter);
+	if (!wrb) {
+		status = -EBUSY;
+		goto err;
+	}
+
+	req = embedded_payload(wrb);
+
+	be_wrb_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_COMMON,
+			       OPCODE_COMMON_SET_FEATURES,
+			       sizeof(*req), wrb, NULL);
+
+	req->features = cpu_to_le32(BE_FEATURE_UE_RECOVERY);
+	req->parameter_len = cpu_to_le32(sizeof(struct be_req_ue_recovery));
+	req->parameter.req.uer = cpu_to_le32(BE_UE_RECOVERY_UER_MASK);
+
+	status = be_mcc_notify_wait(adapter);
+	if (status)
+		goto err;
+
+	resp = embedded_payload(wrb);
+
+	adapter->error_recovery.ue_to_poll_time =
+		le16_to_cpu(resp->parameter.resp.ue2rp);
+	adapter->error_recovery.ue_to_reset_time =
+		le16_to_cpu(resp->parameter.resp.ue2sr);
+	adapter->error_recovery.recovery_supported = true;
+err:
+	/* Checking "MCC_STATUS_INVALID_LENGTH" for SKH as FW
+	 * returns this error in older firmware versions
+	 */
+	if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
+	    base_status(status) == MCC_STATUS_INVALID_LENGTH)
+		dev_info(&adapter->pdev->dev,
+			 "Adapter does not support HW error recovery\n");
+
+	mutex_unlock(&adapter->mcc_lock);
+	return status;
+}
+
 int be_roce_mcc_cmd(void *netdev_handle, void *wrb_payload,
 		    int wrb_payload_size, u16 *cmd_status, u16 *ext_status)
 {
@@ -4964,7 +5052,7 @@ int be_roce_mcc_cmd(void *netdev_handle, void *wrb_payload,
 	struct be_cmd_resp_hdr *resp;
 	int status;
 
-	spin_lock_bh(&adapter->mcc_lock);
+	mutex_lock(&adapter->mcc_lock);
 
 	wrb = wrb_from_mccq(adapter);
 	if (!wrb) {
@@ -4987,7 +5075,7 @@ int be_roce_mcc_cmd(void *netdev_handle, void *wrb_payload,
 	memcpy(wrb_payload, resp, sizeof(*resp) + resp->response_length);
 	be_dws_le_to_cpu(wrb_payload, sizeof(*resp) + resp->response_length);
 err:
-	spin_unlock_bh(&adapter->mcc_lock);
+	mutex_unlock(&adapter->mcc_lock);
 	return status;
 }
 EXPORT_SYMBOL(be_roce_mcc_cmd);
diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.h b/drivers/net/ethernet/emulex/benet/be_cmds.h
index 0d6be224a787..09da2d82c2f0 100644
--- a/drivers/net/ethernet/emulex/benet/be_cmds.h
+++ b/drivers/net/ethernet/emulex/benet/be_cmds.h
@@ -58,7 +58,8 @@ enum mcc_base_status {
 	MCC_STATUS_INSUFFICIENT_BUFFER = 4,
 	MCC_STATUS_UNAUTHORIZED_REQUEST = 5,
 	MCC_STATUS_NOT_SUPPORTED = 66,
-	MCC_STATUS_FEATURE_NOT_SUPPORTED = 68
+	MCC_STATUS_FEATURE_NOT_SUPPORTED = 68,
+	MCC_STATUS_INVALID_LENGTH = 116
 };
 
 /* Additional status */
@@ -294,8 +295,8 @@ struct be_mcc_mailbox {
 #define OPCODE_COMMON_GET_PHY_DETAILS			102
 #define OPCODE_COMMON_SET_DRIVER_FUNCTION_CAP		103
 #define OPCODE_COMMON_GET_CNTL_ADDITIONAL_ATTRIBUTES	121
-#define OPCODE_COMMON_GET_EXT_FAT_CAPABILITES		125
-#define OPCODE_COMMON_SET_EXT_FAT_CAPABILITES		126
+#define OPCODE_COMMON_GET_EXT_FAT_CAPABILITIES		125
+#define OPCODE_COMMON_SET_EXT_FAT_CAPABILITIES		126
 #define OPCODE_COMMON_GET_MAC_LIST			147
 #define OPCODE_COMMON_SET_MAC_LIST			148
 #define OPCODE_COMMON_GET_HSW_CONFIG			152
@@ -308,6 +309,7 @@ struct be_mcc_mailbox {
 #define OPCODE_COMMON_READ_OBJECT			171
 #define OPCODE_COMMON_WRITE_OBJECT			172
 #define OPCODE_COMMON_DELETE_OBJECT			174
+#define OPCODE_COMMON_SET_FEATURES			191
 #define OPCODE_COMMON_MANAGE_IFACE_FILTERS		193
 #define OPCODE_COMMON_GET_IFACE_LIST			194
 #define OPCODE_COMMON_ENABLE_DISABLE_VF			196
@@ -1718,7 +1720,11 @@ struct mgmt_hba_attribs {
 	u32 rsvd2[55];
 	u8 rsvd3[3];
 	u8 phy_port;
-	u32 rsvd4[13];
+	u32 rsvd4[15];
+	u8 rsvd5[2];
+	u8 pci_funcnum;
+	u8 rsvd6;
+	u32 rsvd7[6];
 } __packed;
 
 struct mgmt_controller_attrib {
@@ -2315,6 +2321,41 @@ struct be_cmd_resp_get_iface_list {
 	struct be_if_desc if_desc;
 };
 
+/************** Set Features *******************/
+#define	BE_FEATURE_UE_RECOVERY		0x10
+#define	BE_UE_RECOVERY_UER_MASK		0x1
+
+struct be_req_ue_recovery {
+	u32	uer;
+	u32	rsvd;
+};
+
+struct be_cmd_req_set_features {
+	struct be_cmd_req_hdr hdr;
+	u32 features;
+	u32 parameter_len;
+	union {
+		struct be_req_ue_recovery req;
+		u32 rsvd[2];
+	} parameter;
+};
+
+struct be_resp_ue_recovery {
+	u32 uer;
+	u16 ue2rp;
+	u16 ue2sr;
+};
+
+struct be_cmd_resp_set_features {
+	struct be_cmd_resp_hdr hdr;
+	u32 features;
+	u32 parameter_len;
+	union {
+		struct be_resp_ue_recovery resp;
+		u32 rsvd[2];
+	} parameter;
+};
+
 /*************** Set logical link ********************/
 #define PLINK_ENABLE            BIT(0)
 #define PLINK_TRACK             BIT(8)
@@ -2343,6 +2384,7 @@ struct be_cmd_req_manage_iface_filters {
 	u32 cap_control_flags;
 } __packed;
 
+u16 be_POST_stage_get(struct be_adapter *adapter);
 int be_pci_fnum_get(struct be_adapter *adapter);
 int be_fw_wait_ready(struct be_adapter *adapter);
 int be_cmd_mac_addr_query(struct be_adapter *adapter, u8 *mac_addr,
@@ -2470,3 +2512,4 @@ int be_cmd_manage_iface(struct be_adapter *adapter, u32 iface, u8 op);
 int be_cmd_set_sriov_config(struct be_adapter *adapter,
 			    struct be_resources res, u16 num_vfs,
 			    struct be_resources *vft_res);
+int be_cmd_set_features(struct be_adapter *adapter);
diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c
index 50e7be5da50c..0a48a31225e6 100644
--- a/drivers/net/ethernet/emulex/benet/be_ethtool.c
+++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c
@@ -421,6 +421,10 @@ static void be_get_ethtool_stats(struct net_device *netdev,
 	}
 }
 
+static const char be_priv_flags[][ETH_GSTRING_LEN] = {
+	"disable-tpe-recovery"
+};
+
 static void be_get_stat_strings(struct net_device *netdev, uint32_t stringset,
 				uint8_t *data)
 {
@@ -454,6 +458,10 @@ static void be_get_stat_strings(struct net_device *netdev, uint32_t stringset,
 			data += ETH_GSTRING_LEN;
 		}
 		break;
+	case ETH_SS_PRIV_FLAGS:
+		for (i = 0; i < ARRAY_SIZE(be_priv_flags); i++)
+			strcpy(data + i * ETH_GSTRING_LEN, be_priv_flags[i]);
+		break;
 	}
 }
 
@@ -468,6 +476,8 @@ static int be_get_sset_count(struct net_device *netdev, int stringset)
 		return ETHTOOL_STATS_NUM +
 			adapter->num_rx_qs * ETHTOOL_RXSTATS_NUM +
 			adapter->num_tx_qs * ETHTOOL_TXSTATS_NUM;
+	case ETH_SS_PRIV_FLAGS:
+		return ARRAY_SIZE(be_priv_flags);
 	default:
 		return -EINVAL;
 	}
@@ -1360,6 +1370,34 @@ err:
 	return be_cmd_status(status);
 }
 
+static u32 be_get_priv_flags(struct net_device *netdev)
+{
+	struct be_adapter *adapter = netdev_priv(netdev);
+
+	return adapter->priv_flags;
+}
+
+static int be_set_priv_flags(struct net_device *netdev, u32 flags)
+{
+	struct be_adapter *adapter = netdev_priv(netdev);
+	bool tpe_old = !!(adapter->priv_flags & BE_DISABLE_TPE_RECOVERY);
+	bool tpe_new = !!(flags & BE_DISABLE_TPE_RECOVERY);
+
+	if (tpe_old != tpe_new) {
+		if (tpe_new) {
+			adapter->priv_flags |= BE_DISABLE_TPE_RECOVERY;
+			dev_info(&adapter->pdev->dev,
+				 "HW error recovery is disabled\n");
+		} else {
+			adapter->priv_flags &= ~BE_DISABLE_TPE_RECOVERY;
+			dev_info(&adapter->pdev->dev,
+				 "HW error recovery is enabled\n");
+		}
+	}
+
+	return 0;
+}
+
 const struct ethtool_ops be_ethtool_ops = {
 	.get_settings = be_get_settings,
 	.get_drvinfo = be_get_drvinfo,
@@ -1373,6 +1411,8 @@ const struct ethtool_ops be_ethtool_ops = {
 	.get_ringparam = be_get_ringparam,
 	.get_pauseparam = be_get_pauseparam,
 	.set_pauseparam = be_set_pauseparam,
+	.set_priv_flags = be_set_priv_flags,
+	.get_priv_flags = be_get_priv_flags,
 	.get_strings = be_get_stat_strings,
 	.set_phys_id = be_set_phys_id,
 	.set_dump = be_set_dump,
diff --git a/drivers/net/ethernet/emulex/benet/be_hw.h b/drivers/net/ethernet/emulex/benet/be_hw.h
index c684bb32b487..36e4232ed6b8 100644
--- a/drivers/net/ethernet/emulex/benet/be_hw.h
+++ b/drivers/net/ethernet/emulex/benet/be_hw.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005-2016 Broadcom.
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -32,18 +32,23 @@
 #define MPU_EP_CONTROL 		0
 
 /********** MPU semphore: used for SH & BE  *************/
+#define SLIPORT_SOFTRESET_OFFSET		0x5c	/* CSR BAR offset */
 #define SLIPORT_SEMAPHORE_OFFSET_BEx		0xac  /* CSR BAR offset */
 #define SLIPORT_SEMAPHORE_OFFSET_SH		0x94  /* PCI-CFG offset */
 #define POST_STAGE_MASK				0x0000FFFF
 #define POST_ERR_MASK				0x1
 #define POST_ERR_SHIFT				31
+#define POST_ERR_RECOVERY_CODE_MASK		0xFFF
+
+/* Soft Reset register masks */
+#define SLIPORT_SOFTRESET_SR_MASK		0x00000080	/* SR bit */
 
 /* MPU semphore POST stage values */
 #define POST_STAGE_AWAITING_HOST_RDY 	0x1 /* FW awaiting goahead from host */
 #define POST_STAGE_HOST_RDY 		0x2 /* Host has given go-ahed to FW */
 #define POST_STAGE_BE_RESET		0x3 /* Host wants to reset chip */
 #define POST_STAGE_ARMFW_RDY		0xc000	/* FW is done with POST */
-
+#define POST_STAGE_RECOVERABLE_ERR	0xE000	/* Recoverable err detected */
 
 /* Lancer SLIPORT registers */
 #define SLIPORT_STATUS_OFFSET		0x404
diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c
index 874c7539a79d..cece8a08edca 100644
--- a/drivers/net/ethernet/emulex/benet/be_main.c
+++ b/drivers/net/ethernet/emulex/benet/be_main.c
@@ -41,6 +41,11 @@ static ushort rx_frag_size = 2048;
 module_param(rx_frag_size, ushort, S_IRUGO);
 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
 
+/* Per-module error detection/recovery workq shared across all functions.
+ * Each function schedules its own work request on this shared workq.
+ */
+static struct workqueue_struct *be_err_recovery_workq;
+
 static const struct pci_device_id be_dev_ids[] = {
 	{ PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 	{ PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
@@ -53,6 +58,10 @@ static const struct pci_device_id be_dev_ids[] = {
 	{ 0 }
 };
 MODULE_DEVICE_TABLE(pci, be_dev_ids);
+
+/* Workqueue used by all functions for defering cmd calls to the adapter */
+static struct workqueue_struct *be_wq;
+
 /* UE Status Low CSR */
 static const char * const ue_status_low_desc[] = {
 	"CEV",
@@ -260,6 +269,38 @@ void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 	iowrite32(val, adapter->db + DB_CQ_OFFSET);
 }
 
+static int be_dev_mac_add(struct be_adapter *adapter, u8 *mac)
+{
+	int i;
+
+	/* Check if mac has already been added as part of uc-list */
+	for (i = 0; i < adapter->uc_macs; i++) {
+		if (ether_addr_equal((u8 *)&adapter->uc_list[i * ETH_ALEN],
+				     mac)) {
+			/* mac already added, skip addition */
+			adapter->pmac_id[0] = adapter->pmac_id[i + 1];
+			return 0;
+		}
+	}
+
+	return be_cmd_pmac_add(adapter, mac, adapter->if_handle,
+			       &adapter->pmac_id[0], 0);
+}
+
+static void be_dev_mac_del(struct be_adapter *adapter, int pmac_id)
+{
+	int i;
+
+	/* Skip deletion if the programmed mac is
+	 * being used in uc-list
+	 */
+	for (i = 0; i < adapter->uc_macs; i++) {
+		if (adapter->pmac_id[i + 1] == pmac_id)
+			return;
+	}
+	be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0);
+}
+
 static int be_mac_addr_set(struct net_device *netdev, void *p)
 {
 	struct be_adapter *adapter = netdev_priv(netdev);
@@ -267,7 +308,7 @@ static int be_mac_addr_set(struct net_device *netdev, void *p)
 	struct sockaddr *addr = p;
 	int status;
 	u8 mac[ETH_ALEN];
-	u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
+	u32 old_pmac_id = adapter->pmac_id[0];
 
 	if (!is_valid_ether_addr(addr->sa_data))
 		return -EADDRNOTAVAIL;
@@ -275,7 +316,7 @@ static int be_mac_addr_set(struct net_device *netdev, void *p)
 	/* Proceed further only if, User provided MAC is different
 	 * from active MAC
 	 */
-	if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
+	if (ether_addr_equal(addr->sa_data, adapter->dev_mac))
 		return 0;
 
 	/* if device is not running, copy MAC to netdev->dev_addr */
@@ -288,23 +329,22 @@ static int be_mac_addr_set(struct net_device *netdev, void *p)
 	 * FILTMGMT privilege. This failure is OK, only if the PF programmed
 	 * the MAC for the VF.
 	 */
-	status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
-				 adapter->if_handle, &adapter->pmac_id[0], 0);
+	mutex_lock(&adapter->rx_filter_lock);
+	status = be_dev_mac_add(adapter, (u8 *)addr->sa_data);
 	if (!status) {
-		curr_pmac_id = adapter->pmac_id[0];
 
 		/* Delete the old programmed MAC. This call may fail if the
 		 * old MAC was already deleted by the PF driver.
 		 */
 		if (adapter->pmac_id[0] != old_pmac_id)
-			be_cmd_pmac_del(adapter, adapter->if_handle,
-					old_pmac_id, 0);
+			be_dev_mac_del(adapter, old_pmac_id);
 	}
 
+	mutex_unlock(&adapter->rx_filter_lock);
 	/* Decide if the new MAC is successfully activated only after
 	 * querying the FW
 	 */
-	status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
+	status = be_cmd_get_active_mac(adapter, adapter->pmac_id[0], mac,
 				       adapter->if_handle, true, 0);
 	if (status)
 		goto err;
@@ -317,6 +357,7 @@ static int be_mac_addr_set(struct net_device *netdev, void *p)
 		goto err;
 	}
 done:
+	ether_addr_copy(adapter->dev_mac, addr->sa_data);
 	ether_addr_copy(netdev->dev_addr, addr->sa_data);
 	dev_info(dev, "MAC address changed to %pM\n", addr->sa_data);
 	return 0;
@@ -683,14 +724,24 @@ void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 	netdev_info(netdev, "Link is %s\n", link_status ? "Up" : "Down");
 }
 
+static int be_gso_hdr_len(struct sk_buff *skb)
+{
+	if (skb->encapsulation)
+		return skb_inner_transport_offset(skb) +
+		       inner_tcp_hdrlen(skb);
+	return skb_transport_offset(skb) + tcp_hdrlen(skb);
+}
+
 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
 {
 	struct be_tx_stats *stats = tx_stats(txo);
-	u64 tx_pkts = skb_shinfo(skb)->gso_segs ? : 1;
+	u32 tx_pkts = skb_shinfo(skb)->gso_segs ? : 1;
+	/* Account for headers which get duplicated in TSO pkt */
+	u32 dup_hdr_len = tx_pkts > 1 ? be_gso_hdr_len(skb) * (tx_pkts - 1) : 0;
 
 	u64_stats_update_begin(&stats->sync);
 	stats->tx_reqs++;
-	stats->tx_bytes += skb->len;
+	stats->tx_bytes += skb->len + dup_hdr_len;
 	stats->tx_pkts += tx_pkts;
 	if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL)
 		stats->tx_vxlan_offload_pkts += tx_pkts;
@@ -1420,13 +1471,18 @@ static int be_vid_config(struct be_adapter *adapter)
 	u16 num = 0, i = 0;
 	int status = 0;
 
-	/* No need to further configure vids if in promiscuous mode */
-	if (be_in_all_promisc(adapter))
+	/* No need to change the VLAN state if the I/F is in promiscuous */
+	if (adapter->netdev->flags & IFF_PROMISC)
 		return 0;
 
 	if (adapter->vlans_added > be_max_vlans(adapter))
 		return be_set_vlan_promisc(adapter);
 
+	if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
+		status = be_clear_vlan_promisc(adapter);
+		if (status)
+			return status;
+	}
 	/* Construct VLAN Table to give to HW */
 	for_each_set_bit(i, adapter->vids, VLAN_N_VID)
 		vids[num++] = cpu_to_le16(i);
@@ -1439,8 +1495,6 @@ static int be_vid_config(struct be_adapter *adapter)
 		    addl_status(status) ==
 				MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
 			return be_set_vlan_promisc(adapter);
-	} else if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
-		status = be_clear_vlan_promisc(adapter);
 	}
 	return status;
 }
@@ -1450,46 +1504,45 @@ static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
 	struct be_adapter *adapter = netdev_priv(netdev);
 	int status = 0;
 
+	mutex_lock(&adapter->rx_filter_lock);
+
 	/* Packets with VID 0 are always received by Lancer by default */
 	if (lancer_chip(adapter) && vid == 0)
-		return status;
+		goto done;
 
 	if (test_bit(vid, adapter->vids))
-		return status;
+		goto done;
 
 	set_bit(vid, adapter->vids);
 	adapter->vlans_added++;
 
 	status = be_vid_config(adapter);
-	if (status) {
-		adapter->vlans_added--;
-		clear_bit(vid, adapter->vids);
-	}
-
+done:
+	mutex_unlock(&adapter->rx_filter_lock);
 	return status;
 }
 
 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
 {
 	struct be_adapter *adapter = netdev_priv(netdev);
+	int status = 0;
+
+	mutex_lock(&adapter->rx_filter_lock);
 
 	/* Packets with VID 0 are always received by Lancer by default */
 	if (lancer_chip(adapter) && vid == 0)
-		return 0;
+		goto done;
 
 	if (!test_bit(vid, adapter->vids))
-		return 0;
+		goto done;
 
 	clear_bit(vid, adapter->vids);
 	adapter->vlans_added--;
 
-	return be_vid_config(adapter);
-}
-
-static void be_clear_all_promisc(struct be_adapter *adapter)
-{
-	be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, OFF);
-	adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS;
+	status = be_vid_config(adapter);
+done:
+	mutex_unlock(&adapter->rx_filter_lock);
+	return status;
 }
 
 static void be_set_all_promisc(struct be_adapter *adapter)
@@ -1510,75 +1563,226 @@ static void be_set_mc_promisc(struct be_adapter *adapter)
 		adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS;
 }
 
-static void be_set_mc_list(struct be_adapter *adapter)
+static void be_set_uc_promisc(struct be_adapter *adapter)
 {
 	int status;
 
-	status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
+	if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS)
+		return;
+
+	status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, ON);
 	if (!status)
-		adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
-	else
+		adapter->if_flags |= BE_IF_FLAGS_PROMISCUOUS;
+}
+
+static void be_clear_uc_promisc(struct be_adapter *adapter)
+{
+	int status;
+
+	if (!(adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS))
+		return;
+
+	status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, OFF);
+	if (!status)
+		adapter->if_flags &= ~BE_IF_FLAGS_PROMISCUOUS;
+}
+
+/* The below 2 functions are the callback args for __dev_mc_sync/dev_uc_sync().
+ * We use a single callback function for both sync and unsync. We really don't
+ * add/remove addresses through this callback. But, we use it to detect changes
+ * to the uc/mc lists. The entire uc/mc list is programmed in be_set_rx_mode().
+ */
+static int be_uc_list_update(struct net_device *netdev,
+			     const unsigned char *addr)
+{
+	struct be_adapter *adapter = netdev_priv(netdev);
+
+	adapter->update_uc_list = true;
+	return 0;
+}
+
+static int be_mc_list_update(struct net_device *netdev,
+			     const unsigned char *addr)
+{
+	struct be_adapter *adapter = netdev_priv(netdev);
+
+	adapter->update_mc_list = true;
+	return 0;
+}
+
+static void be_set_mc_list(struct be_adapter *adapter)
+{
+	struct net_device *netdev = adapter->netdev;
+	struct netdev_hw_addr *ha;
+	bool mc_promisc = false;
+	int status;
+
+	netif_addr_lock_bh(netdev);
+	__dev_mc_sync(netdev, be_mc_list_update, be_mc_list_update);
+
+	if (netdev->flags & IFF_PROMISC) {
+		adapter->update_mc_list = false;
+	} else if (netdev->flags & IFF_ALLMULTI ||
+		   netdev_mc_count(netdev) > be_max_mc(adapter)) {
+		/* Enable multicast promisc if num configured exceeds
+		 * what we support
+		 */
+		mc_promisc = true;
+		adapter->update_mc_list = false;
+	} else if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) {
+		/* Update mc-list unconditionally if the iface was previously
+		 * in mc-promisc mode and now is out of that mode.
+		 */
+		adapter->update_mc_list = true;
+	}
+
+	if (adapter->update_mc_list) {
+		int i = 0;
+
+		/* cache the mc-list in adapter */
+		netdev_for_each_mc_addr(ha, netdev) {
+			ether_addr_copy(adapter->mc_list[i].mac, ha->addr);
+			i++;
+		}
+		adapter->mc_count = netdev_mc_count(netdev);
+	}
+	netif_addr_unlock_bh(netdev);
+
+	if (mc_promisc) {
 		be_set_mc_promisc(adapter);
+	} else if (adapter->update_mc_list) {
+		status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
+		if (!status)
+			adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
+		else
+			be_set_mc_promisc(adapter);
+
+		adapter->update_mc_list = false;
+	}
+}
+
+static void be_clear_mc_list(struct be_adapter *adapter)
+{
+	struct net_device *netdev = adapter->netdev;
+
+	__dev_mc_unsync(netdev, NULL);
+	be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, OFF);
+	adapter->mc_count = 0;
+}
+
+static int be_uc_mac_add(struct be_adapter *adapter, int uc_idx)
+{
+	if (ether_addr_equal((u8 *)&adapter->uc_list[uc_idx * ETH_ALEN],
+			     adapter->dev_mac)) {
+		adapter->pmac_id[uc_idx + 1] = adapter->pmac_id[0];
+		return 0;
+	}
+
+	return be_cmd_pmac_add(adapter,
+			       (u8 *)&adapter->uc_list[uc_idx * ETH_ALEN],
+			       adapter->if_handle,
+			       &adapter->pmac_id[uc_idx + 1], 0);
+}
+
+static void be_uc_mac_del(struct be_adapter *adapter, int pmac_id)
+{
+	if (pmac_id == adapter->pmac_id[0])
+		return;
+
+	be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0);
 }
 
 static void be_set_uc_list(struct be_adapter *adapter)
 {
+	struct net_device *netdev = adapter->netdev;
 	struct netdev_hw_addr *ha;
-	int i = 1; /* First slot is claimed by the Primary MAC */
+	bool uc_promisc = false;
+	int curr_uc_macs = 0, i;
 
-	for (; adapter->uc_macs > 0; adapter->uc_macs--, i++)
-		be_cmd_pmac_del(adapter, adapter->if_handle,
-				adapter->pmac_id[i], 0);
+	netif_addr_lock_bh(netdev);
+	__dev_uc_sync(netdev, be_uc_list_update, be_uc_list_update);
 
-	if (netdev_uc_count(adapter->netdev) > be_max_uc(adapter)) {
-		be_set_all_promisc(adapter);
-		return;
+	if (netdev->flags & IFF_PROMISC) {
+		adapter->update_uc_list = false;
+	} else if (netdev_uc_count(netdev) > (be_max_uc(adapter) - 1)) {
+		uc_promisc = true;
+		adapter->update_uc_list = false;
+	}  else if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) {
+		/* Update uc-list unconditionally if the iface was previously
+		 * in uc-promisc mode and now is out of that mode.
+		 */
+		adapter->update_uc_list = true;
+	}
+
+	if (adapter->update_uc_list) {
+		i = 1; /* First slot is claimed by the Primary MAC */
+
+		/* cache the uc-list in adapter array */
+		netdev_for_each_uc_addr(ha, netdev) {
+			ether_addr_copy(adapter->uc_list[i].mac, ha->addr);
+			i++;
+		}
+		curr_uc_macs = netdev_uc_count(netdev);
 	}
+	netif_addr_unlock_bh(netdev);
+
+	if (uc_promisc) {
+		be_set_uc_promisc(adapter);
+	} else if (adapter->update_uc_list) {
+		be_clear_uc_promisc(adapter);
+
+		for (i = 0; i < adapter->uc_macs; i++)
+			be_uc_mac_del(adapter, adapter->pmac_id[i + 1]);
 
-	netdev_for_each_uc_addr(ha, adapter->netdev) {
-		adapter->uc_macs++; /* First slot is for Primary MAC */
-		be_cmd_pmac_add(adapter, (u8 *)ha->addr, adapter->if_handle,
-				&adapter->pmac_id[adapter->uc_macs], 0);
+		for (i = 0; i < curr_uc_macs; i++)
+			be_uc_mac_add(adapter, i);
+		adapter->uc_macs = curr_uc_macs;
+		adapter->update_uc_list = false;
 	}
 }
 
 static void be_clear_uc_list(struct be_adapter *adapter)
 {
+	struct net_device *netdev = adapter->netdev;
 	int i;
 
-	for (i = 1; i < (adapter->uc_macs + 1); i++)
-		be_cmd_pmac_del(adapter, adapter->if_handle,
-				adapter->pmac_id[i], 0);
+	__dev_uc_unsync(netdev, NULL);
+	for (i = 0; i < adapter->uc_macs; i++)
+		be_uc_mac_del(adapter, adapter->pmac_id[i + 1]);
+
 	adapter->uc_macs = 0;
 }
 
-static void be_set_rx_mode(struct net_device *netdev)
+static void __be_set_rx_mode(struct be_adapter *adapter)
 {
-	struct be_adapter *adapter = netdev_priv(netdev);
+	struct net_device *netdev = adapter->netdev;
+
+	mutex_lock(&adapter->rx_filter_lock);
 
 	if (netdev->flags & IFF_PROMISC) {
-		be_set_all_promisc(adapter);
-		return;
+		if (!be_in_all_promisc(adapter))
+			be_set_all_promisc(adapter);
+	} else if (be_in_all_promisc(adapter)) {
+		/* We need to re-program the vlan-list or clear
+		 * vlan-promisc mode (if needed) when the interface
+		 * comes out of promisc mode.
+		 */
+		be_vid_config(adapter);
 	}
 
-	/* Interface was previously in promiscuous mode; disable it */
-	if (be_in_all_promisc(adapter)) {
-		be_clear_all_promisc(adapter);
-		if (adapter->vlans_added)
-			be_vid_config(adapter);
-	}
+	be_set_uc_list(adapter);
+	be_set_mc_list(adapter);
 
-	/* Enable multicast promisc if num configured exceeds what we support */
-	if (netdev->flags & IFF_ALLMULTI ||
-	    netdev_mc_count(netdev) > be_max_mc(adapter)) {
-		be_set_mc_promisc(adapter);
-		return;
-	}
+	mutex_unlock(&adapter->rx_filter_lock);
+}
 
-	if (netdev_uc_count(netdev) != adapter->uc_macs)
-		be_set_uc_list(adapter);
+static void be_work_set_rx_mode(struct work_struct *work)
+{
+	struct be_cmd_work *cmd_work =
+				container_of(work, struct be_cmd_work, work);
 
-	be_set_mc_list(adapter);
+	__be_set_rx_mode(cmd_work->adapter);
+	kfree(cmd_work);
 }
 
 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
@@ -1701,7 +1905,8 @@ static int be_clear_vf_tvt(struct be_adapter *adapter, int vf)
 	return 0;
 }
 
-static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
+static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
+			  __be16 vlan_proto)
 {
 	struct be_adapter *adapter = netdev_priv(netdev);
 	struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
@@ -1713,6 +1918,9 @@ static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
 	if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
 		return -EINVAL;
 
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
 	if (vlan || qos) {
 		vlan |= qos << VLAN_PRIO_SHIFT;
 		status = be_set_vf_tvt(adapter, vf, vlan);
@@ -3220,9 +3428,7 @@ void be_detect_error(struct be_adapter *adapter)
 		 */
 
 		if (ue_lo || ue_hi) {
-			dev_err(dev,
-				"Unrecoverable Error detected in the adapter");
-			dev_err(dev, "Please reboot server to recover");
+			dev_err(dev, "Error detected in the adapter");
 			if (skyhawk_chip(adapter))
 				be_set_error(adapter, BE_ERROR_UE);
 
@@ -3425,10 +3631,9 @@ static void be_rx_qs_destroy(struct be_adapter *adapter)
 
 static void be_disable_if_filters(struct be_adapter *adapter)
 {
-	be_cmd_pmac_del(adapter, adapter->if_handle,
-			adapter->pmac_id[0], 0);
-
+	be_dev_mac_del(adapter, adapter->pmac_id[0]);
 	be_clear_uc_list(adapter);
+	be_clear_mc_list(adapter);
 
 	/* The IFACE flags are enabled in the open path and cleared
 	 * in the close path. When a VF gets detached from the host and
@@ -3462,6 +3667,11 @@ static int be_close(struct net_device *netdev)
 	if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
 		return 0;
 
+	/* Before attempting cleanup ensure all the pending cmds in the
+	 * config_wq have finished execution
+	 */
+	flush_workqueue(be_wq);
+
 	be_disable_if_filters(adapter);
 
 	if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
@@ -3576,17 +3786,16 @@ static int be_enable_if_filters(struct be_adapter *adapter)
 
 	/* For BE3 VFs, the PF programs the initial MAC address */
 	if (!(BEx_chip(adapter) && be_virtfn(adapter))) {
-		status = be_cmd_pmac_add(adapter, adapter->netdev->dev_addr,
-					 adapter->if_handle,
-					 &adapter->pmac_id[0], 0);
+		status = be_dev_mac_add(adapter, adapter->netdev->dev_addr);
 		if (status)
 			return status;
+		ether_addr_copy(adapter->dev_mac, adapter->netdev->dev_addr);
 	}
 
 	if (adapter->vlans_added)
 		be_vid_config(adapter);
 
-	be_set_rx_mode(adapter->netdev);
+	__be_set_rx_mode(adapter);
 
 	return 0;
 }
@@ -3759,8 +3968,13 @@ static void be_cancel_worker(struct be_adapter *adapter)
 
 static void be_cancel_err_detection(struct be_adapter *adapter)
 {
+	struct be_error_recovery *err_rec = &adapter->error_recovery;
+
+	if (!be_err_recovery_workq)
+		return;
+
 	if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) {
-		cancel_delayed_work_sync(&adapter->be_err_detection_work);
+		cancel_delayed_work_sync(&err_rec->err_detection_work);
 		adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED;
 	}
 }
@@ -3860,6 +4074,20 @@ static void be_calculate_vf_res(struct be_adapter *adapter, u16 num_vfs,
 		vft_res->max_mcc_count = res.max_mcc_count / (num_vfs + 1);
 }
 
+static void be_if_destroy(struct be_adapter *adapter)
+{
+	be_cmd_if_destroy(adapter, adapter->if_handle,  0);
+
+	kfree(adapter->pmac_id);
+	adapter->pmac_id = NULL;
+
+	kfree(adapter->mc_list);
+	adapter->mc_list = NULL;
+
+	kfree(adapter->uc_list);
+	adapter->uc_list = NULL;
+}
+
 static int be_clear(struct be_adapter *adapter)
 {
 	struct pci_dev *pdev = adapter->pdev;
@@ -3867,6 +4095,8 @@ static int be_clear(struct be_adapter *adapter)
 
 	be_cancel_worker(adapter);
 
+	flush_workqueue(be_wq);
+
 	if (sriov_enabled(adapter))
 		be_vf_clear(adapter);
 
@@ -3884,10 +4114,8 @@ static int be_clear(struct be_adapter *adapter)
 	}
 
 	be_disable_vxlan_offloads(adapter);
-	kfree(adapter->pmac_id);
-	adapter->pmac_id = NULL;
 
-	be_cmd_if_destroy(adapter, adapter->if_handle,  0);
+	be_if_destroy(adapter);
 
 	be_clear_queues(adapter);
 
@@ -4151,7 +4379,7 @@ static void be_setup_init(struct be_adapter *adapter)
  * for distribution between the VFs. This self-imposed limit will determine the
  * no: of VFs for which RSS can be enabled.
  */
-void be_calculate_pf_pool_rss_tables(struct be_adapter *adapter)
+static void be_calculate_pf_pool_rss_tables(struct be_adapter *adapter)
 {
 	struct be_port_resources port_res = {0};
 	u8 rss_tables_on_port;
@@ -4341,14 +4569,29 @@ static int be_mac_setup(struct be_adapter *adapter)
 
 static void be_schedule_worker(struct be_adapter *adapter)
 {
-	schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
+	queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000));
 	adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
 }
 
+static void be_destroy_err_recovery_workq(void)
+{
+	if (!be_err_recovery_workq)
+		return;
+
+	flush_workqueue(be_err_recovery_workq);
+	destroy_workqueue(be_err_recovery_workq);
+	be_err_recovery_workq = NULL;
+}
+
 static void be_schedule_err_detection(struct be_adapter *adapter, u32 delay)
 {
-	schedule_delayed_work(&adapter->be_err_detection_work,
-			      msecs_to_jiffies(delay));
+	struct be_error_recovery *err_rec = &adapter->error_recovery;
+
+	if (!be_err_recovery_workq)
+		return;
+
+	queue_delayed_work(be_err_recovery_workq, &err_rec->err_detection_work,
+			   msecs_to_jiffies(delay));
 	adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED;
 }
 
@@ -4393,6 +4636,22 @@ static int be_if_create(struct be_adapter *adapter)
 	u32 cap_flags = be_if_cap_flags(adapter);
 	int status;
 
+	/* alloc required memory for other filtering fields */
+	adapter->pmac_id = kcalloc(be_max_uc(adapter),
+				   sizeof(*adapter->pmac_id), GFP_KERNEL);
+	if (!adapter->pmac_id)
+		return -ENOMEM;
+
+	adapter->mc_list = kcalloc(be_max_mc(adapter),
+				   sizeof(*adapter->mc_list), GFP_KERNEL);
+	if (!adapter->mc_list)
+		return -ENOMEM;
+
+	adapter->uc_list = kcalloc(be_max_uc(adapter),
+				   sizeof(*adapter->uc_list), GFP_KERNEL);
+	if (!adapter->uc_list)
+		return -ENOMEM;
+
 	if (adapter->cfg_num_rx_irqs == 1)
 		cap_flags &= ~(BE_IF_FLAGS_DEFQ_RSS | BE_IF_FLAGS_RSS);
 
@@ -4401,7 +4660,10 @@ static int be_if_create(struct be_adapter *adapter)
 	status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
 				  &adapter->if_handle, 0);
 
-	return status;
+	if (status)
+		return status;
+
+	return 0;
 }
 
 int be_update_queues(struct be_adapter *adapter)
@@ -4458,10 +4720,15 @@ static inline int fw_major_num(const char *fw_ver)
 	return fw_major;
 }
 
-/* If any VFs are already enabled don't FLR the PF */
+/* If it is error recovery, FLR the PF
+ * Else if any VFs are already enabled don't FLR the PF
+ */
 static bool be_reset_required(struct be_adapter *adapter)
 {
-	return pci_num_vf(adapter->pdev) ? false : true;
+	if (be_error_recovering(adapter))
+		return true;
+	else
+		return pci_num_vf(adapter->pdev) == 0;
 }
 
 /* Wait for the FW to be ready and perform the required initialization */
@@ -4473,6 +4740,9 @@ static int be_func_init(struct be_adapter *adapter)
 	if (status)
 		return status;
 
+	/* FW is now ready; clear errors to allow cmds/doorbell */
+	be_clear_error(adapter, BE_CLEAR_ALL);
+
 	if (be_reset_required(adapter)) {
 		status = be_cmd_reset_function(adapter);
 		if (status)
@@ -4480,9 +4750,6 @@ static int be_func_init(struct be_adapter *adapter)
 
 		/* Wait for interrupts to quiesce after an FLR */
 		msleep(100);
-
-		/* We can clear all errors when function reset succeeds */
-		be_clear_error(adapter, BE_CLEAR_ALL);
 	}
 
 	/* Tell FW we're ready to fire cmds */
@@ -4530,11 +4797,6 @@ static int be_setup(struct be_adapter *adapter)
 	if (status)
 		goto err;
 
-	adapter->pmac_id = kcalloc(be_max_uc(adapter),
-				   sizeof(*adapter->pmac_id), GFP_KERNEL);
-	if (!adapter->pmac_id)
-		return -ENOMEM;
-
 	status = be_msix_enable(adapter);
 	if (status)
 		goto err;
@@ -4595,6 +4857,9 @@ static int be_setup(struct be_adapter *adapter)
 	if (!status && be_pause_supported(adapter))
 		adapter->phy.fc_autoneg = 1;
 
+	if (be_physfn(adapter) && !lancer_chip(adapter))
+		be_cmd_set_features(adapter);
+
 	be_schedule_worker(adapter);
 	adapter->flags |= BE_FLAGS_SETUP_DONE;
 	return 0;
@@ -4728,6 +4993,23 @@ static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
 				       0, 0, nlflags, filter_mask, NULL);
 }
 
+static struct be_cmd_work *be_alloc_work(struct be_adapter *adapter,
+					 void (*func)(struct work_struct *))
+{
+	struct be_cmd_work *work;
+
+	work = kzalloc(sizeof(*work), GFP_ATOMIC);
+	if (!work) {
+		dev_err(&adapter->pdev->dev,
+			"be_work memory allocation failed\n");
+		return NULL;
+	}
+
+	INIT_WORK(&work->work, func);
+	work->adapter = adapter;
+	return work;
+}
+
 /* VxLAN offload Notes:
  *
  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
@@ -4742,23 +5024,19 @@ static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
  * adds more than one port, disable offloads and don't re-enable them again
  * until after all the tunnels are removed.
  */
-static void be_add_vxlan_port(struct net_device *netdev,
-			      struct udp_tunnel_info *ti)
+static void be_work_add_vxlan_port(struct work_struct *work)
 {
-	struct be_adapter *adapter = netdev_priv(netdev);
+	struct be_cmd_work *cmd_work =
+				container_of(work, struct be_cmd_work, work);
+	struct be_adapter *adapter = cmd_work->adapter;
+	struct net_device *netdev = adapter->netdev;
 	struct device *dev = &adapter->pdev->dev;
-	__be16 port = ti->port;
+	__be16 port = cmd_work->info.vxlan_port;
 	int status;
 
-	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
-		return;
-
-	if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter))
-		return;
-
 	if (adapter->vxlan_port == port && adapter->vxlan_port_count) {
 		adapter->vxlan_port_aliases++;
-		return;
+		goto done;
 	}
 
 	if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
@@ -4770,7 +5048,7 @@ static void be_add_vxlan_port(struct net_device *netdev,
 	}
 
 	if (adapter->vxlan_port_count++ >= 1)
-		return;
+		goto done;
 
 	status = be_cmd_manage_iface(adapter, adapter->if_handle,
 				     OP_CONVERT_NORMAL_TO_TUNNEL);
@@ -4795,29 +5073,26 @@ static void be_add_vxlan_port(struct net_device *netdev,
 
 	dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
 		 be16_to_cpu(port));
-	return;
+	goto done;
 err:
 	be_disable_vxlan_offloads(adapter);
+done:
+	kfree(cmd_work);
 }
 
-static void be_del_vxlan_port(struct net_device *netdev,
-			      struct udp_tunnel_info *ti)
+static void be_work_del_vxlan_port(struct work_struct *work)
 {
-	struct be_adapter *adapter = netdev_priv(netdev);
-	__be16 port = ti->port;
-
-	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
-		return;
-
-	if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter))
-		return;
+	struct be_cmd_work *cmd_work =
+				container_of(work, struct be_cmd_work, work);
+	struct be_adapter *adapter = cmd_work->adapter;
+	__be16 port = cmd_work->info.vxlan_port;
 
 	if (adapter->vxlan_port != port)
 		goto done;
 
 	if (adapter->vxlan_port_aliases) {
 		adapter->vxlan_port_aliases--;
-		return;
+		goto out;
 	}
 
 	be_disable_vxlan_offloads(adapter);
@@ -4827,6 +5102,40 @@ static void be_del_vxlan_port(struct net_device *netdev,
 		 be16_to_cpu(port));
 done:
 	adapter->vxlan_port_count--;
+out:
+	kfree(cmd_work);
+}
+
+static void be_cfg_vxlan_port(struct net_device *netdev,
+			      struct udp_tunnel_info *ti,
+			      void (*func)(struct work_struct *))
+{
+	struct be_adapter *adapter = netdev_priv(netdev);
+	struct be_cmd_work *cmd_work;
+
+	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+		return;
+
+	if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter))
+		return;
+
+	cmd_work = be_alloc_work(adapter, func);
+	if (cmd_work) {
+		cmd_work->info.vxlan_port = ti->port;
+		queue_work(be_wq, &cmd_work->work);
+	}
+}
+
+static void be_del_vxlan_port(struct net_device *netdev,
+			      struct udp_tunnel_info *ti)
+{
+	be_cfg_vxlan_port(netdev, ti, be_work_del_vxlan_port);
+}
+
+static void be_add_vxlan_port(struct net_device *netdev,
+			      struct udp_tunnel_info *ti)
+{
+	be_cfg_vxlan_port(netdev, ti, be_work_add_vxlan_port);
 }
 
 static netdev_features_t be_features_check(struct sk_buff *skb,
@@ -4891,6 +5200,16 @@ static int be_get_phys_port_id(struct net_device *dev,
 	return 0;
 }
 
+static void be_set_rx_mode(struct net_device *dev)
+{
+	struct be_adapter *adapter = netdev_priv(dev);
+	struct be_cmd_work *work;
+
+	work = be_alloc_work(adapter, be_work_set_rx_mode);
+	if (work)
+		queue_work(be_wq, &work->work);
+}
+
 static const struct net_device_ops be_netdev_ops = {
 	.ndo_open		= be_open,
 	.ndo_stop		= be_close,
@@ -4984,13 +5303,145 @@ static int be_resume(struct be_adapter *adapter)
 	return 0;
 }
 
+static void be_soft_reset(struct be_adapter *adapter)
+{
+	u32 val;
+
+	dev_info(&adapter->pdev->dev, "Initiating chip soft reset\n");
+	val = ioread32(adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET);
+	val |= SLIPORT_SOFTRESET_SR_MASK;
+	iowrite32(val, adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET);
+}
+
+static bool be_err_is_recoverable(struct be_adapter *adapter)
+{
+	struct be_error_recovery *err_rec = &adapter->error_recovery;
+	unsigned long initial_idle_time =
+		msecs_to_jiffies(ERR_RECOVERY_IDLE_TIME);
+	unsigned long recovery_interval =
+		msecs_to_jiffies(ERR_RECOVERY_INTERVAL);
+	u16 ue_err_code;
+	u32 val;
+
+	val = be_POST_stage_get(adapter);
+	if ((val & POST_STAGE_RECOVERABLE_ERR) != POST_STAGE_RECOVERABLE_ERR)
+		return false;
+	ue_err_code = val & POST_ERR_RECOVERY_CODE_MASK;
+	if (ue_err_code == 0)
+		return false;
+
+	dev_err(&adapter->pdev->dev, "Recoverable HW error code: 0x%x\n",
+		ue_err_code);
+
+	if (jiffies - err_rec->probe_time <= initial_idle_time) {
+		dev_err(&adapter->pdev->dev,
+			"Cannot recover within %lu sec from driver load\n",
+			jiffies_to_msecs(initial_idle_time) / MSEC_PER_SEC);
+		return false;
+	}
+
+	if (err_rec->last_recovery_time &&
+	    (jiffies - err_rec->last_recovery_time <= recovery_interval)) {
+		dev_err(&adapter->pdev->dev,
+			"Cannot recover within %lu sec from last recovery\n",
+			jiffies_to_msecs(recovery_interval) / MSEC_PER_SEC);
+		return false;
+	}
+
+	if (ue_err_code == err_rec->last_err_code) {
+		dev_err(&adapter->pdev->dev,
+			"Cannot recover from a consecutive TPE error\n");
+		return false;
+	}
+
+	err_rec->last_recovery_time = jiffies;
+	err_rec->last_err_code = ue_err_code;
+	return true;
+}
+
+static int be_tpe_recover(struct be_adapter *adapter)
+{
+	struct be_error_recovery *err_rec = &adapter->error_recovery;
+	int status = -EAGAIN;
+	u32 val;
+
+	switch (err_rec->recovery_state) {
+	case ERR_RECOVERY_ST_NONE:
+		err_rec->recovery_state = ERR_RECOVERY_ST_DETECT;
+		err_rec->resched_delay = ERR_RECOVERY_UE_DETECT_DURATION;
+		break;
+
+	case ERR_RECOVERY_ST_DETECT:
+		val = be_POST_stage_get(adapter);
+		if ((val & POST_STAGE_RECOVERABLE_ERR) !=
+		    POST_STAGE_RECOVERABLE_ERR) {
+			dev_err(&adapter->pdev->dev,
+				"Unrecoverable HW error detected: 0x%x\n", val);
+			status = -EINVAL;
+			err_rec->resched_delay = 0;
+			break;
+		}
+
+		dev_err(&adapter->pdev->dev, "Recoverable HW error detected\n");
+
+		/* Only PF0 initiates Chip Soft Reset. But PF0 must wait UE2SR
+		 * milliseconds before it checks for final error status in
+		 * SLIPORT_SEMAPHORE to determine if recovery criteria is met.
+		 * If it does, then PF0 initiates a Soft Reset.
+		 */
+		if (adapter->pf_num == 0) {
+			err_rec->recovery_state = ERR_RECOVERY_ST_RESET;
+			err_rec->resched_delay = err_rec->ue_to_reset_time -
+					ERR_RECOVERY_UE_DETECT_DURATION;
+			break;
+		}
+
+		err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL;
+		err_rec->resched_delay = err_rec->ue_to_poll_time -
+					ERR_RECOVERY_UE_DETECT_DURATION;
+		break;
+
+	case ERR_RECOVERY_ST_RESET:
+		if (!be_err_is_recoverable(adapter)) {
+			dev_err(&adapter->pdev->dev,
+				"Failed to meet recovery criteria\n");
+			status = -EIO;
+			err_rec->resched_delay = 0;
+			break;
+		}
+		be_soft_reset(adapter);
+		err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL;
+		err_rec->resched_delay = err_rec->ue_to_poll_time -
+					err_rec->ue_to_reset_time;
+		break;
+
+	case ERR_RECOVERY_ST_PRE_POLL:
+		err_rec->recovery_state = ERR_RECOVERY_ST_REINIT;
+		err_rec->resched_delay = 0;
+		status = 0;			/* done */
+		break;
+
+	default:
+		status = -EINVAL;
+		err_rec->resched_delay = 0;
+		break;
+	}
+
+	return status;
+}
+
 static int be_err_recover(struct be_adapter *adapter)
 {
 	int status;
 
-	/* Error recovery is supported only Lancer as of now */
-	if (!lancer_chip(adapter))
-		return -EIO;
+	if (!lancer_chip(adapter)) {
+		if (!adapter->error_recovery.recovery_supported ||
+		    adapter->priv_flags & BE_DISABLE_TPE_RECOVERY)
+			return -EIO;
+		status = be_tpe_recover(adapter);
+		if (status)
+			goto err;
+	}
 
 	/* Wait for adapter to reach quiescent state before
 	 * destroying queues
@@ -4999,59 +5450,74 @@ static int be_err_recover(struct be_adapter *adapter)
 	if (status)
 		goto err;
 
+	adapter->flags |= BE_FLAGS_TRY_RECOVERY;
+
 	be_cleanup(adapter);
 
 	status = be_resume(adapter);
 	if (status)
 		goto err;
 
-	return 0;
+	adapter->flags &= ~BE_FLAGS_TRY_RECOVERY;
+
 err:
 	return status;
 }
 
 static void be_err_detection_task(struct work_struct *work)
 {
+	struct be_error_recovery *err_rec =
+			container_of(work, struct be_error_recovery,
+				     err_detection_work.work);
 	struct be_adapter *adapter =
-				container_of(work, struct be_adapter,
-					     be_err_detection_work.work);
+			container_of(err_rec, struct be_adapter,
+				     error_recovery);
+	u32 resched_delay = ERR_RECOVERY_DETECTION_DELAY;
 	struct device *dev = &adapter->pdev->dev;
 	int recovery_status;
-	int delay = ERR_DETECTION_DELAY;
 
 	be_detect_error(adapter);
-
-	if (be_check_error(adapter, BE_ERROR_HW))
-		recovery_status = be_err_recover(adapter);
-	else
+	if (!be_check_error(adapter, BE_ERROR_HW))
 		goto reschedule_task;
 
+	recovery_status = be_err_recover(adapter);
 	if (!recovery_status) {
-		adapter->recovery_retries = 0;
+		err_rec->recovery_retries = 0;
+		err_rec->recovery_state = ERR_RECOVERY_ST_NONE;
 		dev_info(dev, "Adapter recovery successful\n");
 		goto reschedule_task;
-	} else if (be_virtfn(adapter)) {
+	} else if (!lancer_chip(adapter) && err_rec->resched_delay) {
+		/* BEx/SH recovery state machine */
+		if (adapter->pf_num == 0 &&
+		    err_rec->recovery_state > ERR_RECOVERY_ST_DETECT)
+			dev_err(&adapter->pdev->dev,
+				"Adapter recovery in progress\n");
+		resched_delay = err_rec->resched_delay;
+		goto reschedule_task;
+	} else if (lancer_chip(adapter) && be_virtfn(adapter)) {
 		/* For VFs, check if PF have allocated resources
 		 * every second.
 		 */
 		dev_err(dev, "Re-trying adapter recovery\n");
 		goto reschedule_task;
-	} else if (adapter->recovery_retries++ <
-		   MAX_ERR_RECOVERY_RETRY_COUNT) {
+	} else if (lancer_chip(adapter) && err_rec->recovery_retries++ <
+		   ERR_RECOVERY_MAX_RETRY_COUNT) {
 		/* In case of another error during recovery, it takes 30 sec
 		 * for adapter to come out of error. Retry error recovery after
 		 * this time interval.
 		 */
 		dev_err(&adapter->pdev->dev, "Re-trying adapter recovery\n");
-		delay = ERR_RECOVERY_RETRY_DELAY;
+		resched_delay = ERR_RECOVERY_RETRY_DELAY;
 		goto reschedule_task;
 	} else {
 		dev_err(dev, "Adapter recovery failed\n");
+		dev_err(dev, "Please reboot server to recover\n");
 	}
 
 	return;
+
 reschedule_task:
-	be_schedule_err_detection(adapter, delay);
+	be_schedule_err_detection(adapter, resched_delay);
 }
 
 static void be_log_sfp_info(struct be_adapter *adapter)
@@ -5116,7 +5582,7 @@ static void be_worker(struct work_struct *work)
 
 reschedule:
 	adapter->work_counter++;
-	schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
+	queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000));
 }
 
 static void be_unmap_pci_bars(struct be_adapter *adapter)
@@ -5256,14 +5722,18 @@ static int be_drv_init(struct be_adapter *adapter)
 	}
 
 	mutex_init(&adapter->mbox_lock);
-	spin_lock_init(&adapter->mcc_lock);
+	mutex_init(&adapter->mcc_lock);
+	mutex_init(&adapter->rx_filter_lock);
 	spin_lock_init(&adapter->mcc_cq_lock);
 	init_completion(&adapter->et_cmd_compl);
 
 	pci_save_state(adapter->pdev);
 
 	INIT_DELAYED_WORK(&adapter->work, be_worker);
-	INIT_DELAYED_WORK(&adapter->be_err_detection_work,
+
+	adapter->error_recovery.recovery_state = ERR_RECOVERY_ST_NONE;
+	adapter->error_recovery.resched_delay = 0;
+	INIT_DELAYED_WORK(&adapter->error_recovery.err_detection_work,
 			  be_err_detection_task);
 
 	adapter->rx_fc = true;
@@ -5298,6 +5768,9 @@ static void be_remove(struct pci_dev *pdev)
 
 	be_clear(adapter);
 
+	if (!pci_vfs_assigned(adapter->pdev))
+		be_cmd_reset_function(adapter);
+
 	/* tell fw we're done with firing cmds */
 	be_cmd_fw_clean(adapter);
 
@@ -5454,6 +5927,7 @@ static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
 	be_roce_dev_add(adapter);
 
 	be_schedule_err_detection(adapter, ERR_DETECTION_DELAY);
+	adapter->error_recovery.probe_time = jiffies;
 
 	/* On Die temperature not supported for VF. */
 	if (be_physfn(adapter) && IS_ENABLED(CONFIG_BE2NET_HWMON)) {
@@ -5699,6 +6173,8 @@ static struct pci_driver be_driver = {
 
 static int __init be_init_module(void)
 {
+	int status;
+
 	if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
 	    rx_frag_size != 2048) {
 		printk(KERN_WARNING DRV_NAME
@@ -5712,12 +6188,33 @@ static int __init be_init_module(void)
 		pr_info(DRV_NAME " : Use sysfs method to enable VFs\n");
 	}
 
-	return pci_register_driver(&be_driver);
+	be_wq = create_singlethread_workqueue("be_wq");
+	if (!be_wq) {
+		pr_warn(DRV_NAME "workqueue creation failed\n");
+		return -1;
+	}
+
+	be_err_recovery_workq =
+		create_singlethread_workqueue("be_err_recover");
+	if (!be_err_recovery_workq)
+		pr_warn(DRV_NAME "Could not create error recovery workqueue\n");
+
+	status = pci_register_driver(&be_driver);
+	if (status) {
+		destroy_workqueue(be_wq);
+		be_destroy_err_recovery_workq();
+	}
+	return status;
 }
 module_init(be_init_module);
 
 static void __exit be_exit_module(void)
 {
 	pci_unregister_driver(&be_driver);
+
+	be_destroy_err_recovery_workq();
+
+	if (be_wq)
+		destroy_workqueue(be_wq);
 }
 module_exit(be_exit_module);
diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c
index 36361f8bf894..262587240c86 100644
--- a/drivers/net/ethernet/faraday/ftgmac100.c
+++ b/drivers/net/ethernet/faraday/ftgmac100.c
@@ -60,6 +60,8 @@ struct ftgmac100 {
 	struct ftgmac100_descs *descs;
 	dma_addr_t descs_dma_addr;
 
+	struct page *rx_pages[RX_QUEUE_ENTRIES];
+
 	unsigned int rx_pointer;
 	unsigned int tx_clean_pointer;
 	unsigned int tx_pointer;
@@ -77,6 +79,9 @@ struct ftgmac100 {
 	int int_mask_all;
 	bool use_ncsi;
 	bool enabled;
+
+	u32 rxdes0_edorr_mask;
+	u32 txdes0_edotr_mask;
 };
 
 static int ftgmac100_alloc_rx_page(struct ftgmac100 *priv,
@@ -257,10 +262,11 @@ static bool ftgmac100_rxdes_packet_ready(struct ftgmac100_rxdes *rxdes)
 	return rxdes->rxdes0 & cpu_to_le32(FTGMAC100_RXDES0_RXPKT_RDY);
 }
 
-static void ftgmac100_rxdes_set_dma_own(struct ftgmac100_rxdes *rxdes)
+static void ftgmac100_rxdes_set_dma_own(const struct ftgmac100 *priv,
+					struct ftgmac100_rxdes *rxdes)
 {
 	/* clear status bits */
-	rxdes->rxdes0 &= cpu_to_le32(FTGMAC100_RXDES0_EDORR);
+	rxdes->rxdes0 &= cpu_to_le32(priv->rxdes0_edorr_mask);
 }
 
 static bool ftgmac100_rxdes_rx_error(struct ftgmac100_rxdes *rxdes)
@@ -298,9 +304,10 @@ static bool ftgmac100_rxdes_multicast(struct ftgmac100_rxdes *rxdes)
 	return rxdes->rxdes0 & cpu_to_le32(FTGMAC100_RXDES0_MULTICAST);
 }
 
-static void ftgmac100_rxdes_set_end_of_ring(struct ftgmac100_rxdes *rxdes)
+static void ftgmac100_rxdes_set_end_of_ring(const struct ftgmac100 *priv,
+					    struct ftgmac100_rxdes *rxdes)
 {
-	rxdes->rxdes0 |= cpu_to_le32(FTGMAC100_RXDES0_EDORR);
+	rxdes->rxdes0 |= cpu_to_le32(priv->rxdes0_edorr_mask);
 }
 
 static void ftgmac100_rxdes_set_dma_addr(struct ftgmac100_rxdes *rxdes,
@@ -341,18 +348,27 @@ static bool ftgmac100_rxdes_ipcs_err(struct ftgmac100_rxdes *rxdes)
 	return rxdes->rxdes1 & cpu_to_le32(FTGMAC100_RXDES1_IP_CHKSUM_ERR);
 }
 
+static inline struct page **ftgmac100_rxdes_page_slot(struct ftgmac100 *priv,
+						      struct ftgmac100_rxdes *rxdes)
+{
+	return &priv->rx_pages[rxdes - priv->descs->rxdes];
+}
+
 /*
  * rxdes2 is not used by hardware. We use it to keep track of page.
  * Since hardware does not touch it, we can skip cpu_to_le32()/le32_to_cpu().
  */
-static void ftgmac100_rxdes_set_page(struct ftgmac100_rxdes *rxdes, struct page *page)
+static void ftgmac100_rxdes_set_page(struct ftgmac100 *priv,
+				     struct ftgmac100_rxdes *rxdes,
+				     struct page *page)
 {
-	rxdes->rxdes2 = (unsigned int)page;
+	*ftgmac100_rxdes_page_slot(priv, rxdes) = page;
 }
 
-static struct page *ftgmac100_rxdes_get_page(struct ftgmac100_rxdes *rxdes)
+static struct page *ftgmac100_rxdes_get_page(struct ftgmac100 *priv,
+					     struct ftgmac100_rxdes *rxdes)
 {
-	return (struct page *)rxdes->rxdes2;
+	return *ftgmac100_rxdes_page_slot(priv, rxdes);
 }
 
 /******************************************************************************
@@ -382,7 +398,7 @@ ftgmac100_rx_locate_first_segment(struct ftgmac100 *priv)
 		if (ftgmac100_rxdes_first_segment(rxdes))
 			return rxdes;
 
-		ftgmac100_rxdes_set_dma_own(rxdes);
+		ftgmac100_rxdes_set_dma_own(priv, rxdes);
 		ftgmac100_rx_pointer_advance(priv);
 		rxdes = ftgmac100_current_rxdes(priv);
 	}
@@ -453,7 +469,7 @@ static void ftgmac100_rx_drop_packet(struct ftgmac100 *priv)
 		if (ftgmac100_rxdes_last_segment(rxdes))
 			done = true;
 
-		ftgmac100_rxdes_set_dma_own(rxdes);
+		ftgmac100_rxdes_set_dma_own(priv, rxdes);
 		ftgmac100_rx_pointer_advance(priv);
 		rxdes = ftgmac100_current_rxdes(priv);
 	} while (!done && ftgmac100_rxdes_packet_ready(rxdes));
@@ -501,7 +517,7 @@ static bool ftgmac100_rx_packet(struct ftgmac100 *priv, int *processed)
 
 	do {
 		dma_addr_t map = ftgmac100_rxdes_get_dma_addr(rxdes);
-		struct page *page = ftgmac100_rxdes_get_page(rxdes);
+		struct page *page = ftgmac100_rxdes_get_page(priv, rxdes);
 		unsigned int size;
 
 		dma_unmap_page(priv->dev, map, RX_BUF_SIZE, DMA_FROM_DEVICE);
@@ -545,10 +561,11 @@ static bool ftgmac100_rx_packet(struct ftgmac100 *priv, int *processed)
 /******************************************************************************
  * internal functions (transmit descriptor)
  *****************************************************************************/
-static void ftgmac100_txdes_reset(struct ftgmac100_txdes *txdes)
+static void ftgmac100_txdes_reset(const struct ftgmac100 *priv,
+				  struct ftgmac100_txdes *txdes)
 {
 	/* clear all except end of ring bit */
-	txdes->txdes0 &= cpu_to_le32(FTGMAC100_TXDES0_EDOTR);
+	txdes->txdes0 &= cpu_to_le32(priv->txdes0_edotr_mask);
 	txdes->txdes1 = 0;
 	txdes->txdes2 = 0;
 	txdes->txdes3 = 0;
@@ -569,9 +586,10 @@ static void ftgmac100_txdes_set_dma_own(struct ftgmac100_txdes *txdes)
 	txdes->txdes0 |= cpu_to_le32(FTGMAC100_TXDES0_TXDMA_OWN);
 }
 
-static void ftgmac100_txdes_set_end_of_ring(struct ftgmac100_txdes *txdes)
+static void ftgmac100_txdes_set_end_of_ring(const struct ftgmac100 *priv,
+					    struct ftgmac100_txdes *txdes)
 {
-	txdes->txdes0 |= cpu_to_le32(FTGMAC100_TXDES0_EDOTR);
+	txdes->txdes0 |= cpu_to_le32(priv->txdes0_edotr_mask);
 }
 
 static void ftgmac100_txdes_set_first_segment(struct ftgmac100_txdes *txdes)
@@ -690,7 +708,7 @@ static bool ftgmac100_tx_complete_packet(struct ftgmac100 *priv)
 
 	dev_kfree_skb(skb);
 
-	ftgmac100_txdes_reset(txdes);
+	ftgmac100_txdes_reset(priv, txdes);
 
 	ftgmac100_tx_clean_pointer_advance(priv);
 
@@ -779,9 +797,9 @@ static int ftgmac100_alloc_rx_page(struct ftgmac100 *priv,
 		return -ENOMEM;
 	}
 
-	ftgmac100_rxdes_set_page(rxdes, page);
+	ftgmac100_rxdes_set_page(priv, rxdes, page);
 	ftgmac100_rxdes_set_dma_addr(rxdes, map);
-	ftgmac100_rxdes_set_dma_own(rxdes);
+	ftgmac100_rxdes_set_dma_own(priv, rxdes);
 	return 0;
 }
 
@@ -791,7 +809,7 @@ static void ftgmac100_free_buffers(struct ftgmac100 *priv)
 
 	for (i = 0; i < RX_QUEUE_ENTRIES; i++) {
 		struct ftgmac100_rxdes *rxdes = &priv->descs->rxdes[i];
-		struct page *page = ftgmac100_rxdes_get_page(rxdes);
+		struct page *page = ftgmac100_rxdes_get_page(priv, rxdes);
 		dma_addr_t map = ftgmac100_rxdes_get_dma_addr(rxdes);
 
 		if (!page)
@@ -828,7 +846,8 @@ static int ftgmac100_alloc_buffers(struct ftgmac100 *priv)
 		return -ENOMEM;
 
 	/* initialize RX ring */
-	ftgmac100_rxdes_set_end_of_ring(&priv->descs->rxdes[RX_QUEUE_ENTRIES - 1]);
+	ftgmac100_rxdes_set_end_of_ring(priv,
+					&priv->descs->rxdes[RX_QUEUE_ENTRIES - 1]);
 
 	for (i = 0; i < RX_QUEUE_ENTRIES; i++) {
 		struct ftgmac100_rxdes *rxdes = &priv->descs->rxdes[i];
@@ -838,7 +857,8 @@ static int ftgmac100_alloc_buffers(struct ftgmac100 *priv)
 	}
 
 	/* initialize TX ring */
-	ftgmac100_txdes_set_end_of_ring(&priv->descs->txdes[TX_QUEUE_ENTRIES - 1]);
+	ftgmac100_txdes_set_end_of_ring(priv,
+					&priv->descs->txdes[TX_QUEUE_ENTRIES - 1]);
 	return 0;
 
 err:
@@ -1055,14 +1075,12 @@ static int ftgmac100_poll(struct napi_struct *napi, int budget)
 	}
 
 	if (status & priv->int_mask_all & (FTGMAC100_INT_NO_RXBUF |
-			FTGMAC100_INT_RPKT_LOST | FTGMAC100_INT_AHB_ERR |
-			FTGMAC100_INT_PHYSTS_CHG)) {
+			FTGMAC100_INT_RPKT_LOST | FTGMAC100_INT_AHB_ERR)) {
 		if (net_ratelimit())
-			netdev_info(netdev, "[ISR] = 0x%x: %s%s%s%s\n", status,
+			netdev_info(netdev, "[ISR] = 0x%x: %s%s%s\n", status,
 				    status & FTGMAC100_INT_NO_RXBUF ? "NO_RXBUF " : "",
 				    status & FTGMAC100_INT_RPKT_LOST ? "RPKT_LOST " : "",
-				    status & FTGMAC100_INT_AHB_ERR ? "AHB_ERR " : "",
-				    status & FTGMAC100_INT_PHYSTS_CHG ? "PHYSTS_CHG" : "");
+				    status & FTGMAC100_INT_AHB_ERR ? "AHB_ERR " : "");
 
 		if (status & FTGMAC100_INT_NO_RXBUF) {
 			/* RX buffer unavailable */
@@ -1092,6 +1110,7 @@ static int ftgmac100_poll(struct napi_struct *napi, int budget)
 static int ftgmac100_open(struct net_device *netdev)
 {
 	struct ftgmac100 *priv = netdev_priv(netdev);
+	unsigned int status;
 	int err;
 
 	err = ftgmac100_alloc_buffers(priv);
@@ -1117,6 +1136,11 @@ static int ftgmac100_open(struct net_device *netdev)
 
 	ftgmac100_init_hw(priv);
 	ftgmac100_start_hw(priv, priv->use_ncsi ? 100 : 10);
+
+	/* Clear stale interrupts */
+	status = ioread32(priv->base + FTGMAC100_OFFSET_ISR);
+	iowrite32(status, priv->base + FTGMAC100_OFFSET_ISR);
+
 	if (netdev->phydev)
 		phy_start(netdev->phydev);
 	else if (priv->use_ncsi)
@@ -1166,6 +1190,8 @@ static int ftgmac100_stop(struct net_device *netdev)
 	napi_disable(&priv->napi);
 	if (netdev->phydev)
 		phy_stop(netdev->phydev);
+	else if (priv->use_ncsi)
+		ncsi_stop_dev(priv->ndev);
 
 	ftgmac100_stop_hw(priv);
 	free_irq(priv->irq, netdev);
@@ -1226,12 +1252,21 @@ static int ftgmac100_setup_mdio(struct net_device *netdev)
 	struct ftgmac100 *priv = netdev_priv(netdev);
 	struct platform_device *pdev = to_platform_device(priv->dev);
 	int i, err = 0;
+	u32 reg;
 
 	/* initialize mdio bus */
 	priv->mii_bus = mdiobus_alloc();
 	if (!priv->mii_bus)
 		return -EIO;
 
+	if (of_machine_is_compatible("aspeed,ast2400") ||
+	    of_machine_is_compatible("aspeed,ast2500")) {
+		/* This driver supports the old MDIO interface */
+		reg = ioread32(priv->base + FTGMAC100_OFFSET_REVR);
+		reg &= ~FTGMAC100_REVR_NEW_MDIO_INTERFACE;
+		iowrite32(reg, priv->base + FTGMAC100_OFFSET_REVR);
+	};
+
 	priv->mii_bus->name = "ftgmac100_mdio";
 	snprintf(priv->mii_bus->id, MII_BUS_ID_SIZE, "%s-%d",
 		 pdev->name, pdev->id);
@@ -1355,9 +1390,18 @@ static int ftgmac100_probe(struct platform_device *pdev)
 			      FTGMAC100_INT_XPKT_ETH |
 			      FTGMAC100_INT_XPKT_LOST |
 			      FTGMAC100_INT_AHB_ERR |
-			      FTGMAC100_INT_PHYSTS_CHG |
 			      FTGMAC100_INT_RPKT_BUF |
 			      FTGMAC100_INT_NO_RXBUF);
+
+	if (of_machine_is_compatible("aspeed,ast2400") ||
+	    of_machine_is_compatible("aspeed,ast2500")) {
+		priv->rxdes0_edorr_mask = BIT(30);
+		priv->txdes0_edotr_mask = BIT(30);
+	} else {
+		priv->rxdes0_edorr_mask = BIT(15);
+		priv->txdes0_edotr_mask = BIT(15);
+	}
+
 	if (pdev->dev.of_node &&
 	    of_get_property(pdev->dev.of_node, "use-ncsi", NULL)) {
 		if (!IS_ENABLED(CONFIG_NET_NCSI)) {
@@ -1367,7 +1411,6 @@ static int ftgmac100_probe(struct platform_device *pdev)
 
 		dev_info(&pdev->dev, "Using NCSI interface\n");
 		priv->use_ncsi = true;
-		priv->int_mask_all &= ~FTGMAC100_INT_PHYSTS_CHG;
 		priv->ndev = ncsi_register_dev(netdev, ftgmac100_ncsi_handler);
 		if (!priv->ndev)
 			goto err_ncsi_dev;
diff --git a/drivers/net/ethernet/faraday/ftgmac100.h b/drivers/net/ethernet/faraday/ftgmac100.h
index 13408d448b05..a7ce0ac8858a 100644
--- a/drivers/net/ethernet/faraday/ftgmac100.h
+++ b/drivers/net/ethernet/faraday/ftgmac100.h
@@ -134,6 +134,11 @@
 #define FTGMAC100_DMAFIFOS_TXDMA_REQ		(1 << 31)
 
 /*
+ * Feature Register
+ */
+#define FTGMAC100_REVR_NEW_MDIO_INTERFACE	BIT(31)
+
+/*
  * Receive buffer size register
  */
 #define FTGMAC100_RBSR_SIZE(x)		((x) & 0x3fff)
@@ -152,6 +157,7 @@
 #define FTGMAC100_MACCR_FULLDUP		(1 << 8)
 #define FTGMAC100_MACCR_GIGA_MODE	(1 << 9)
 #define FTGMAC100_MACCR_CRC_APD		(1 << 10)
+#define FTGMAC100_MACCR_PHY_LINK_LEVEL	(1 << 11)
 #define FTGMAC100_MACCR_RX_RUNT		(1 << 12)
 #define FTGMAC100_MACCR_JUMBO_LF	(1 << 13)
 #define FTGMAC100_MACCR_RX_ALL		(1 << 14)
@@ -189,7 +195,6 @@ struct ftgmac100_txdes {
 } __attribute__ ((aligned(16)));
 
 #define FTGMAC100_TXDES0_TXBUF_SIZE(x)	((x) & 0x3fff)
-#define FTGMAC100_TXDES0_EDOTR		(1 << 15)
 #define FTGMAC100_TXDES0_CRC_ERR	(1 << 19)
 #define FTGMAC100_TXDES0_LTS		(1 << 28)
 #define FTGMAC100_TXDES0_FTS		(1 << 29)
@@ -215,7 +220,6 @@ struct ftgmac100_rxdes {
 } __attribute__ ((aligned(16)));
 
 #define FTGMAC100_RXDES0_VDBC		0x3fff
-#define FTGMAC100_RXDES0_EDORR		(1 << 15)
 #define FTGMAC100_RXDES0_MULTICAST	(1 << 16)
 #define FTGMAC100_RXDES0_BROADCAST	(1 << 17)
 #define FTGMAC100_RXDES0_RX_ERR		(1 << 18)
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index 01f7e811739b..48a033e64423 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -89,10 +89,10 @@ static struct platform_device_id fec_devtype[] = {
 		.driver_data = 0,
 	}, {
 		.name = "imx25-fec",
-		.driver_data = FEC_QUIRK_USE_GASKET | FEC_QUIRK_HAS_RACC,
+		.driver_data = FEC_QUIRK_USE_GASKET,
 	}, {
 		.name = "imx27-fec",
-		.driver_data = FEC_QUIRK_HAS_RACC,
+		.driver_data = 0,
 	}, {
 		.name = "imx28-fec",
 		.driver_data = FEC_QUIRK_ENET_MAC | FEC_QUIRK_SWAP_FRAME |
@@ -180,6 +180,7 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address");
 /* FEC receive acceleration */
 #define FEC_RACC_IPDIS		(1 << 1)
 #define FEC_RACC_PRODIS		(1 << 2)
+#define FEC_RACC_SHIFT16	BIT(7)
 #define FEC_RACC_OPTIONS	(FEC_RACC_IPDIS | FEC_RACC_PRODIS)
 
 /*
@@ -912,13 +913,11 @@ fec_restart(struct net_device *ndev)
 	 * enet-mac reset will reset mac address registers too,
 	 * so need to reconfigure it.
 	 */
-	if (fep->quirks & FEC_QUIRK_ENET_MAC) {
-		memcpy(&temp_mac, ndev->dev_addr, ETH_ALEN);
-		writel((__force u32)cpu_to_be32(temp_mac[0]),
-		       fep->hwp + FEC_ADDR_LOW);
-		writel((__force u32)cpu_to_be32(temp_mac[1]),
-		       fep->hwp + FEC_ADDR_HIGH);
-	}
+	memcpy(&temp_mac, ndev->dev_addr, ETH_ALEN);
+	writel((__force u32)cpu_to_be32(temp_mac[0]),
+	       fep->hwp + FEC_ADDR_LOW);
+	writel((__force u32)cpu_to_be32(temp_mac[1]),
+	       fep->hwp + FEC_ADDR_HIGH);
 
 	/* Clear any outstanding interrupt. */
 	writel(0xffffffff, fep->hwp + FEC_IEVENT);
@@ -945,9 +944,11 @@ fec_restart(struct net_device *ndev)
 
 #if !defined(CONFIG_M5272)
 	if (fep->quirks & FEC_QUIRK_HAS_RACC) {
-		/* set RX checksum */
 		val = readl(fep->hwp + FEC_RACC);
+		/* align IP header */
+		val |= FEC_RACC_SHIFT16;
 		if (fep->csum_flags & FLAG_RX_CSUM_ENABLED)
+			/* set RX checksum */
 			val |= FEC_RACC_OPTIONS;
 		else
 			val &= ~FEC_RACC_OPTIONS;
@@ -1428,6 +1429,12 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
 		prefetch(skb->data - NET_IP_ALIGN);
 		skb_put(skb, pkt_len - 4);
 		data = skb->data;
+
+#if !defined(CONFIG_M5272)
+		if (fep->quirks & FEC_QUIRK_HAS_RACC)
+			data = skb_pull_inline(skb, 2);
+#endif
+
 		if (!is_copybreak && need_swap)
 			swap_buffer(data, pkt_len);
 
@@ -2887,7 +2894,7 @@ fec_enet_close(struct net_device *ndev)
  * this kind of feature?).
  */
 
-#define HASH_BITS	6		/* #bits in hash */
+#define FEC_HASH_BITS	6		/* #bits in hash */
 #define CRC32_POLY	0xEDB88320
 
 static void set_multicast_list(struct net_device *ndev)
@@ -2935,10 +2942,10 @@ static void set_multicast_list(struct net_device *ndev)
 			}
 		}
 
-		/* only upper 6 bits (HASH_BITS) are used
+		/* only upper 6 bits (FEC_HASH_BITS) are used
 		 * which point to specific bit in he hash registers
 		 */
-		hash = (crc >> (32 - HASH_BITS)) & 0x3f;
+		hash = (crc >> (32 - FEC_HASH_BITS)) & 0x3f;
 
 		if (hash > 31) {
 			tmp = readl(fep->hwp + FEC_GRP_HASH_TABLE_HIGH);
diff --git a/drivers/net/ethernet/freescale/fman/Makefile b/drivers/net/ethernet/freescale/fman/Makefile
index 51fd2e6c1b84..60491779e49f 100644
--- a/drivers/net/ethernet/freescale/fman/Makefile
+++ b/drivers/net/ethernet/freescale/fman/Makefile
@@ -1,7 +1,9 @@
 subdir-ccflags-y +=  -I$(srctree)/drivers/net/ethernet/freescale/fman
 
-obj-y		+= fsl_fman.o fsl_fman_mac.o fsl_mac.o
+obj-$(CONFIG_FSL_FMAN) += fsl_fman.o
+obj-$(CONFIG_FSL_FMAN) += fsl_fman_port.o
+obj-$(CONFIG_FSL_FMAN) += fsl_mac.o
 
-fsl_fman-objs	:= fman_muram.o fman.o fman_sp.o fman_port.o
-fsl_fman_mac-objs := fman_dtsec.o fman_memac.o fman_tgec.o
-fsl_mac-objs += mac.o
+fsl_fman-objs	:= fman_muram.o fman.o fman_sp.o
+fsl_fman_port-objs := fman_port.o
+fsl_mac-objs:= mac.o fman_dtsec.o fman_memac.o fman_tgec.o
diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c
index 1de2e1e51c2b..dafd9e1baba2 100644
--- a/drivers/net/ethernet/freescale/fman/fman.c
+++ b/drivers/net/ethernet/freescale/fman/fman.c
@@ -618,7 +618,7 @@ struct fman {
 	unsigned long cam_offset;
 	size_t cam_size;
 	/* Fifo in MURAM */
-	int fifo_offset;
+	unsigned long fifo_offset;
 	size_t fifo_size;
 
 	u32 liodn_base[64];
@@ -2036,7 +2036,7 @@ static int fman_init(struct fman *fman)
 	/* allocate MURAM for FIFO according to total size */
 	fman->fifo_offset = fman_muram_alloc(fman->muram,
 					     fman->state->total_fifo_size);
-	if (IS_ERR_VALUE(fman->cam_offset)) {
+	if (IS_ERR_VALUE(fman->fifo_offset)) {
 		free_init_resources(fman);
 		dev_err(fman->dev, "%s: MURAM alloc for BMI FIFO failed\n",
 			__func__);
@@ -2115,6 +2115,7 @@ void fman_register_intr(struct fman *fman, enum fman_event_modules module,
 	fman->intr_mng[event].isr_cb = isr_cb;
 	fman->intr_mng[event].src_handle = src_arg;
 }
+EXPORT_SYMBOL(fman_register_intr);
 
 /**
  * fman_unregister_intr
@@ -2138,6 +2139,7 @@ void fman_unregister_intr(struct fman *fman, enum fman_event_modules module,
 	fman->intr_mng[event].isr_cb = NULL;
 	fman->intr_mng[event].src_handle = NULL;
 }
+EXPORT_SYMBOL(fman_unregister_intr);
 
 /**
  * fman_set_port_params
@@ -2241,6 +2243,7 @@ return_err:
 	spin_unlock_irqrestore(&fman->spinlock, flags);
 	return err;
 }
+EXPORT_SYMBOL(fman_set_port_params);
 
 /**
  * fman_reset_mac
@@ -2310,6 +2313,7 @@ int fman_reset_mac(struct fman *fman, u8 mac_id)
 
 	return 0;
 }
+EXPORT_SYMBOL(fman_reset_mac);
 
 /**
  * fman_set_mac_max_frame
@@ -2327,8 +2331,7 @@ int fman_set_mac_max_frame(struct fman *fman, u8 mac_id, u16 mfl)
 	 * or equal to the port's max
 	 */
 	if ((!fman->state->port_mfl[mac_id]) ||
-	    (fman->state->port_mfl[mac_id] &&
-	    (mfl <= fman->state->port_mfl[mac_id]))) {
+	    (mfl <= fman->state->port_mfl[mac_id])) {
 		fman->state->mac_mfl[mac_id] = mfl;
 	} else {
 		dev_warn(fman->dev, "%s: MAC max_frame_length is larger than Port max_frame_length\n",
@@ -2337,6 +2340,7 @@ int fman_set_mac_max_frame(struct fman *fman, u8 mac_id, u16 mfl)
 	}
 	return 0;
 }
+EXPORT_SYMBOL(fman_set_mac_max_frame);
 
 /**
  * fman_get_clock_freq
@@ -2363,6 +2367,7 @@ u32 fman_get_bmi_max_fifo_size(struct fman *fman)
 {
 	return fman->state->bmi_max_fifo_size;
 }
+EXPORT_SYMBOL(fman_get_bmi_max_fifo_size);
 
 /**
  * fman_get_revision
@@ -2384,6 +2389,7 @@ void fman_get_revision(struct fman *fman, struct fman_rev_info *rev_info)
 				FPM_REV1_MAJOR_SHIFT);
 	rev_info->minor = tmp & FPM_REV1_MINOR_MASK;
 }
+EXPORT_SYMBOL(fman_get_revision);
 
 /**
  * fman_get_qman_channel_id
@@ -2419,6 +2425,7 @@ u32 fman_get_qman_channel_id(struct fman *fman, u32 port_id)
 
 	return fman->state->qman_channel_base + i;
 }
+EXPORT_SYMBOL(fman_get_qman_channel_id);
 
 /**
  * fman_get_mem_region
@@ -2432,6 +2439,7 @@ struct resource *fman_get_mem_region(struct fman *fman)
 {
 	return fman->state->res;
 }
+EXPORT_SYMBOL(fman_get_mem_region);
 
 /* Bootargs defines */
 /* Extra headroom for RX buffers - Default, min and max */
@@ -2453,7 +2461,7 @@ struct resource *fman_get_mem_region(struct fman *fman)
  * particular forwarding scenarios that add extra headers to the
  * forwarded frame.
  */
-int fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
+static int fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
 module_param(fsl_fm_rx_extra_headroom, int, 0);
 MODULE_PARM_DESC(fsl_fm_rx_extra_headroom, "Extra headroom for Rx buffers");
 
@@ -2466,7 +2474,7 @@ MODULE_PARM_DESC(fsl_fm_rx_extra_headroom, "Extra headroom for Rx buffers");
  * Could be overridden once, at boot-time, via the
  * fm_set_max_frm() callback.
  */
-int fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
+static int fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
 module_param(fsl_fm_max_frm, int, 0);
 MODULE_PARM_DESC(fsl_fm_max_frm, "Maximum frame size, across all interfaces");
 
@@ -2538,6 +2546,7 @@ struct fman *fman_bind(struct device *fm_dev)
 {
 	return (struct fman *)(dev_get_drvdata(get_device(fm_dev)));
 }
+EXPORT_SYMBOL(fman_bind);
 
 static irqreturn_t fman_err_irq(int irq, void *handle)
 {
@@ -2727,8 +2736,8 @@ static struct fman *read_dts_node(struct platform_device *of_dev)
 	struct fman *fman;
 	struct device_node *fm_node, *muram_node;
 	struct resource *res;
-	const u32 *u32_prop;
-	int lenp, err, irq;
+	u32 val, range[2];
+	int err, irq;
 	struct clk *clk;
 	u32 clk_rate;
 	phys_addr_t phys_base_addr;
@@ -2740,16 +2749,13 @@ static struct fman *read_dts_node(struct platform_device *of_dev)
 
 	fm_node = of_node_get(of_dev->dev.of_node);
 
-	u32_prop = (const u32 *)of_get_property(fm_node, "cell-index", &lenp);
-	if (!u32_prop) {
-		dev_err(&of_dev->dev, "%s: of_get_property(%s, cell-index) failed\n",
+	err = of_property_read_u32(fm_node, "cell-index", &val);
+	if (err) {
+		dev_err(&of_dev->dev, "%s: failed to read cell-index for %s\n",
 			__func__, fm_node->full_name);
 		goto fman_node_put;
 	}
-	if (WARN_ON(lenp != sizeof(u32)))
-		goto fman_node_put;
-
-	fman->dts_params.id = (u8)fdt32_to_cpu(u32_prop[0]);
+	fman->dts_params.id = (u8)val;
 
 	/* Get the FM interrupt */
 	res = platform_get_resource(of_dev, IORESOURCE_IRQ, 0);
@@ -2796,18 +2802,15 @@ static struct fman *read_dts_node(struct platform_device *of_dev)
 	/* Rounding to MHz */
 	fman->dts_params.clk_freq = DIV_ROUND_UP(clk_rate, 1000000);
 
-	u32_prop = (const u32 *)of_get_property(fm_node,
-						"fsl,qman-channel-range",
-						&lenp);
-	if (!u32_prop) {
-		dev_err(&of_dev->dev, "%s: of_get_property(%s, fsl,qman-channel-range) failed\n",
+	err = of_property_read_u32_array(fm_node, "fsl,qman-channel-range",
+					 &range[0], 2);
+	if (err) {
+		dev_err(&of_dev->dev, "%s: failed to read fsl,qman-channel-range for %s\n",
 			__func__, fm_node->full_name);
 		goto fman_node_put;
 	}
-	if (WARN_ON(lenp != sizeof(u32) * 2))
-		goto fman_node_put;
-	fman->dts_params.qman_channel_base = fdt32_to_cpu(u32_prop[0]);
-	fman->dts_params.num_of_qman_channels = fdt32_to_cpu(u32_prop[1]);
+	fman->dts_params.qman_channel_base = range[0];
+	fman->dts_params.num_of_qman_channels = range[1];
 
 	/* Get the MURAM base address and size */
 	muram_node = of_find_matching_node(fm_node, fman_muram_match);
@@ -2858,7 +2861,7 @@ static struct fman *read_dts_node(struct platform_device *of_dev)
 
 	fman->dts_params.base_addr =
 		devm_ioremap(&of_dev->dev, phys_base_addr, mem_size);
-	if (fman->dts_params.base_addr == 0) {
+	if (!fman->dts_params.base_addr) {
 		dev_err(&of_dev->dev, "%s: devm_ioremap() failed\n", __func__);
 		goto fman_free;
 	}
@@ -2930,7 +2933,7 @@ static const struct of_device_id fman_match[] = {
 	{}
 };
 
-MODULE_DEVICE_TABLE(of, fm_match);
+MODULE_DEVICE_TABLE(of, fman_match);
 
 static struct platform_driver fman_driver = {
 	.driver = {
@@ -2940,4 +2943,25 @@ static struct platform_driver fman_driver = {
 	.probe = fman_probe,
 };
 
-builtin_platform_driver(fman_driver);
+static int __init fman_load(void)
+{
+	int err;
+
+	pr_debug("FSL DPAA FMan driver\n");
+
+	err = platform_driver_register(&fman_driver);
+	if (err < 0)
+		pr_err("Error, platform_driver_register() = %d\n", err);
+
+	return err;
+}
+module_init(fman_load);
+
+static void __exit fman_unload(void)
+{
+	platform_driver_unregister(&fman_driver);
+}
+module_exit(fman_unload);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Freescale DPAA Frame Manager driver");
diff --git a/drivers/net/ethernet/freescale/fman/fman_mac.h b/drivers/net/ethernet/freescale/fman/fman_mac.h
index 8ddeedbcef9c..dd6d0526f6c1 100644
--- a/drivers/net/ethernet/freescale/fman/fman_mac.h
+++ b/drivers/net/ethernet/freescale/fman/fman_mac.h
@@ -191,10 +191,6 @@ struct fman_mac_params {
 	u16 max_speed;
 	/* A handle to the FM object this port related to */
 	void *fm;
-	/* MDIO exceptions interrupt source - not valid for all
-	 * MACs; MUST be set to 'NO_IRQ' for MACs that don't have
-	 * mdio-irq, or for polling
-	 */
 	void *dev_id; /* device cookie used by the exception cbs */
 	fman_mac_exception_cb *event_cb;    /* MDIO Events Callback Routine */
 	fman_mac_exception_cb *exception_cb;/* Exception Callback Routine */
diff --git a/drivers/net/ethernet/freescale/fman/fman_memac.c b/drivers/net/ethernet/freescale/fman/fman_memac.c
index 45e98fd8b79e..53ef51e3bd9e 100644
--- a/drivers/net/ethernet/freescale/fman/fman_memac.c
+++ b/drivers/net/ethernet/freescale/fman/fman_memac.c
@@ -507,6 +507,9 @@ static void setup_sgmii_internal_phy(struct fman_mac *memac,
 {
 	u16 tmp_reg16;
 
+	if (WARN_ON(!memac->pcsphy))
+		return;
+
 	/* SGMII mode */
 	tmp_reg16 = IF_MODE_SGMII_EN;
 	if (!fixed_link)
@@ -1151,7 +1154,8 @@ struct fman_mac *memac_config(struct fman_mac_params *params)
 	/* Save FMan revision */
 	fman_get_revision(memac->fm, &memac->fm_rev_info);
 
-	if (memac->phy_if == PHY_INTERFACE_MODE_SGMII) {
+	if (memac->phy_if == PHY_INTERFACE_MODE_SGMII ||
+	    memac->phy_if == PHY_INTERFACE_MODE_QSGMII) {
 		if (!params->internal_phy_node) {
 			pr_err("PCS PHY node is not available\n");
 			memac_free(memac);
diff --git a/drivers/net/ethernet/freescale/fman/fman_muram.c b/drivers/net/ethernet/freescale/fman/fman_muram.c
index 47394c45b6e8..5ec94d243da0 100644
--- a/drivers/net/ethernet/freescale/fman/fman_muram.c
+++ b/drivers/net/ethernet/freescale/fman/fman_muram.c
@@ -150,7 +150,8 @@ unsigned long fman_muram_alloc(struct muram_info *muram, size_t size)
  *
  * Free an allocated memory from FM-MURAM partition.
  */
-void fman_muram_free_mem(struct muram_info *muram, unsigned long offset, size_t size)
+void fman_muram_free_mem(struct muram_info *muram, unsigned long offset,
+			 size_t size)
 {
 	unsigned long addr = fman_muram_offset_to_vbase(muram, offset);
 
diff --git a/drivers/net/ethernet/freescale/fman/fman_muram.h b/drivers/net/ethernet/freescale/fman/fman_muram.h
index 889649ad8931..453bf849eee1 100644
--- a/drivers/net/ethernet/freescale/fman/fman_muram.h
+++ b/drivers/net/ethernet/freescale/fman/fman_muram.h
@@ -46,6 +46,7 @@ unsigned long fman_muram_offset_to_vbase(struct muram_info *muram,
 
 unsigned long fman_muram_alloc(struct muram_info *muram, size_t size);
 
-void fman_muram_free_mem(struct muram_info *muram, unsigned long offset, size_t size);
+void fman_muram_free_mem(struct muram_info *muram, unsigned long offset,
+			 size_t size);
 
 #endif /* __FM_MURAM_EXT */
diff --git a/drivers/net/ethernet/freescale/fman/fman_port.c b/drivers/net/ethernet/freescale/fman/fman_port.c
index 70c198d072dc..9f3bb50a2365 100644
--- a/drivers/net/ethernet/freescale/fman/fman_port.c
+++ b/drivers/net/ethernet/freescale/fman/fman_port.c
@@ -1477,7 +1477,8 @@ EXPORT_SYMBOL(fman_port_cfg_buf_prefix_content);
  */
 int fman_port_disable(struct fman_port *port)
 {
-	u32 __iomem *bmi_cfg_reg, *bmi_status_reg, tmp;
+	u32 __iomem *bmi_cfg_reg, *bmi_status_reg;
+	u32 tmp;
 	bool rx_port, failure = false;
 	int count;
 
@@ -1553,7 +1554,8 @@ EXPORT_SYMBOL(fman_port_disable);
  */
 int fman_port_enable(struct fman_port *port)
 {
-	u32 __iomem *bmi_cfg_reg, tmp;
+	u32 __iomem *bmi_cfg_reg;
+	u32 tmp;
 	bool rx_port;
 
 	if (!is_init_done(port->cfg))
@@ -1623,7 +1625,7 @@ static int fman_port_probe(struct platform_device *of_dev)
 	struct device_node *fm_node, *port_node;
 	struct resource res;
 	struct resource *dev_res;
-	const u32 *u32_prop;
+	u32 val;
 	int err = 0, lenp;
 	enum fman_port_type port_type;
 	u16 port_speed;
@@ -1652,28 +1654,20 @@ static int fman_port_probe(struct platform_device *of_dev)
 		goto return_err;
 	}
 
-	u32_prop = (const u32 *)of_get_property(port_node, "cell-index", &lenp);
-	if (!u32_prop) {
-		dev_err(port->dev, "%s: of_get_property(%s, cell-index) failed\n",
+	err = of_property_read_u32(port_node, "cell-index", &val);
+	if (err) {
+		dev_err(port->dev, "%s: reading cell-index for %s failed\n",
 			__func__, port_node->full_name);
 		err = -EINVAL;
 		goto return_err;
 	}
-	if (WARN_ON(lenp != sizeof(u32))) {
-		err = -EINVAL;
-		goto return_err;
-	}
-	port_id = (u8)fdt32_to_cpu(u32_prop[0]);
-
+	port_id = (u8)val;
 	port->dts_params.id = port_id;
 
 	if (of_device_is_compatible(port_node, "fsl,fman-v3-port-tx")) {
 		port_type = FMAN_PORT_TYPE_TX;
 		port_speed = 1000;
-		u32_prop = (const u32 *)of_get_property(port_node,
-							"fsl,fman-10g-port",
-							&lenp);
-		if (u32_prop)
+		if (of_find_property(port_node, "fsl,fman-10g-port", &lenp))
 			port_speed = 10000;
 
 	} else if (of_device_is_compatible(port_node, "fsl,fman-v2-port-tx")) {
@@ -1686,9 +1680,7 @@ static int fman_port_probe(struct platform_device *of_dev)
 	} else if (of_device_is_compatible(port_node, "fsl,fman-v3-port-rx")) {
 		port_type = FMAN_PORT_TYPE_RX;
 		port_speed = 1000;
-		u32_prop = (const u32 *)of_get_property(port_node,
-						  "fsl,fman-10g-port", &lenp);
-		if (u32_prop)
+		if (of_find_property(port_node, "fsl,fman-10g-port", &lenp))
 			port_speed = 10000;
 
 	} else if (of_device_is_compatible(port_node, "fsl,fman-v2-port-rx")) {
@@ -1743,7 +1735,7 @@ static int fman_port_probe(struct platform_device *of_dev)
 
 	port->dts_params.base_addr = devm_ioremap(port->dev, res.start,
 						  resource_size(&res));
-	if (port->dts_params.base_addr == 0)
+	if (!port->dts_params.base_addr)
 		dev_err(port->dev, "%s: devm_ioremap() failed\n", __func__);
 
 	dev_set_drvdata(&of_dev->dev, port);
@@ -1775,4 +1767,25 @@ static struct platform_driver fman_port_driver = {
 	.probe = fman_port_probe,
 };
 
-builtin_platform_driver(fman_port_driver);
+static int __init fman_port_load(void)
+{
+	int err;
+
+	pr_debug("FSL DPAA FMan driver\n");
+
+	err = platform_driver_register(&fman_port_driver);
+	if (err < 0)
+		pr_err("Error, platform_driver_register() = %d\n", err);
+
+	return err;
+}
+module_init(fman_port_load);
+
+static void __exit fman_port_unload(void)
+{
+	platform_driver_unregister(&fman_port_driver);
+}
+module_exit(fman_port_unload);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Freescale DPAA Frame Manager Port driver");
diff --git a/drivers/net/ethernet/freescale/fman/fman_sp.c b/drivers/net/ethernet/freescale/fman/fman_sp.c
index f9e7aa385cba..248f5bcca468 100644
--- a/drivers/net/ethernet/freescale/fman/fman_sp.c
+++ b/drivers/net/ethernet/freescale/fman/fman_sp.c
@@ -80,6 +80,7 @@ void fman_sp_set_buf_pools_in_asc_order_of_buf_sizes(struct fman_ext_pools
 		}
 	}
 }
+EXPORT_SYMBOL(fman_sp_set_buf_pools_in_asc_order_of_buf_sizes);
 
 int fman_sp_build_buffer_struct(struct fman_sp_int_context_data_copy *
 				int_context_data_copy,
@@ -164,3 +165,5 @@ int fman_sp_build_buffer_struct(struct fman_sp_int_context_data_copy *
 
 	return 0;
 }
+EXPORT_SYMBOL(fman_sp_build_buffer_struct);
+
diff --git a/drivers/net/ethernet/freescale/fman/mac.c b/drivers/net/ethernet/freescale/fman/mac.c
index e33d9d24c1db..8fe6b3e253fa 100644
--- a/drivers/net/ethernet/freescale/fman/mac.c
+++ b/drivers/net/ethernet/freescale/fman/mac.c
@@ -469,9 +469,9 @@ static void adjust_link_memac(struct net_device *net_dev)
 /* Initializes driver's PHY state, and attaches to the PHY.
  * Returns 0 on success.
  */
-static int init_phy(struct net_device *net_dev,
-		    struct mac_device *mac_dev,
-		    void (*adj_lnk)(struct net_device *))
+static struct phy_device *init_phy(struct net_device *net_dev,
+				   struct mac_device *mac_dev,
+				   void (*adj_lnk)(struct net_device *))
 {
 	struct phy_device	*phy_dev;
 	struct mac_priv_s	*priv = mac_dev->priv;
@@ -480,7 +480,7 @@ static int init_phy(struct net_device *net_dev,
 				 priv->phy_if);
 	if (!phy_dev) {
 		netdev_err(net_dev, "Could not connect to PHY\n");
-		return -ENODEV;
+		return NULL;
 	}
 
 	/* Remove any features not supported by the controller */
@@ -493,23 +493,23 @@ static int init_phy(struct net_device *net_dev,
 
 	mac_dev->phy_dev = phy_dev;
 
-	return 0;
+	return phy_dev;
 }
 
-static int dtsec_init_phy(struct net_device *net_dev,
-			  struct mac_device *mac_dev)
+static struct phy_device *dtsec_init_phy(struct net_device *net_dev,
+					 struct mac_device *mac_dev)
 {
 	return init_phy(net_dev, mac_dev, &adjust_link_dtsec);
 }
 
-static int tgec_init_phy(struct net_device *net_dev,
-			 struct mac_device *mac_dev)
+static struct phy_device *tgec_init_phy(struct net_device *net_dev,
+					struct mac_device *mac_dev)
 {
 	return init_phy(net_dev, mac_dev, adjust_link_void);
 }
 
-static int memac_init_phy(struct net_device *net_dev,
-			  struct mac_device *mac_dev)
+static struct phy_device *memac_init_phy(struct net_device *net_dev,
+					 struct mac_device *mac_dev)
 {
 	return init_phy(net_dev, mac_dev, &adjust_link_memac);
 }
@@ -583,31 +583,6 @@ static void setup_memac(struct mac_device *mac_dev)
 
 static DEFINE_MUTEX(eth_lock);
 
-static const char phy_str[][11] = {
-	[PHY_INTERFACE_MODE_MII]		= "mii",
-	[PHY_INTERFACE_MODE_GMII]		= "gmii",
-	[PHY_INTERFACE_MODE_SGMII]		= "sgmii",
-	[PHY_INTERFACE_MODE_TBI]		= "tbi",
-	[PHY_INTERFACE_MODE_RMII]		= "rmii",
-	[PHY_INTERFACE_MODE_RGMII]		= "rgmii",
-	[PHY_INTERFACE_MODE_RGMII_ID]		= "rgmii-id",
-	[PHY_INTERFACE_MODE_RGMII_RXID]	= "rgmii-rxid",
-	[PHY_INTERFACE_MODE_RGMII_TXID]	= "rgmii-txid",
-	[PHY_INTERFACE_MODE_RTBI]		= "rtbi",
-	[PHY_INTERFACE_MODE_XGMII]		= "xgmii"
-};
-
-static phy_interface_t __pure __attribute__((nonnull)) str2phy(const char *str)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(phy_str); i++)
-		if (strcmp(str, phy_str[i]) == 0)
-			return (phy_interface_t)i;
-
-	return PHY_INTERFACE_MODE_MII;
-}
-
 static const u16 phy2speed[] = {
 	[PHY_INTERFACE_MODE_MII]		= SPEED_100,
 	[PHY_INTERFACE_MODE_GMII]		= SPEED_1000,
@@ -678,7 +653,7 @@ MODULE_DEVICE_TABLE(of, mac_match);
 
 static int mac_probe(struct platform_device *_of_dev)
 {
-	int			 err, i, lenp, nph;
+	int			 err, i, nph;
 	struct device		*dev;
 	struct device_node	*mac_node, *dev_node;
 	struct mac_device	*mac_dev;
@@ -686,9 +661,9 @@ static int mac_probe(struct platform_device *_of_dev)
 	struct resource		 res;
 	struct mac_priv_s	*priv;
 	const u8		*mac_addr;
-	const char		*char_prop;
-	const u32		*u32_prop;
+	u32			 val;
 	u8			fman_id;
+	int			phy_if;
 
 	dev = &_of_dev->dev;
 	mac_node = dev->of_node;
@@ -749,16 +724,15 @@ static int mac_probe(struct platform_device *_of_dev)
 	}
 
 	/* Get the FMan cell-index */
-	u32_prop = of_get_property(dev_node, "cell-index", &lenp);
-	if (!u32_prop) {
-		dev_err(dev, "of_get_property(%s, cell-index) failed\n",
+	err = of_property_read_u32(dev_node, "cell-index", &val);
+	if (err) {
+		dev_err(dev, "failed to read cell-index for %s\n",
 			dev_node->full_name);
 		err = -EINVAL;
 		goto _return_of_node_put;
 	}
-	WARN_ON(lenp != sizeof(u32));
 	/* cell-index 0 => FMan id 1 */
-	fman_id = (u8)(fdt32_to_cpu(u32_prop[0]) + 1);
+	fman_id = (u8)(val + 1);
 
 	priv->fman = fman_bind(&of_dev->dev);
 	if (!priv->fman) {
@@ -805,15 +779,14 @@ static int mac_probe(struct platform_device *_of_dev)
 	}
 
 	/* Get the cell-index */
-	u32_prop = of_get_property(mac_node, "cell-index", &lenp);
-	if (!u32_prop) {
-		dev_err(dev, "of_get_property(%s, cell-index) failed\n",
+	err = of_property_read_u32(mac_node, "cell-index", &val);
+	if (err) {
+		dev_err(dev, "failed to read cell-index for %s\n",
 			mac_node->full_name);
 		err = -EINVAL;
 		goto _return_dev_set_drvdata;
 	}
-	WARN_ON(lenp != sizeof(u32));
-	priv->cell_index = (u8)fdt32_to_cpu(u32_prop[0]);
+	priv->cell_index = (u8)val;
 
 	/* Get the MAC address */
 	mac_addr = of_get_mac_address(mac_node);
@@ -870,16 +843,14 @@ static int mac_probe(struct platform_device *_of_dev)
 	}
 
 	/* Get the PHY connection type */
-	char_prop = (const char *)of_get_property(mac_node,
-						  "phy-connection-type", NULL);
-	if (!char_prop) {
+	phy_if = of_get_phy_mode(mac_node);
+	if (phy_if < 0) {
 		dev_warn(dev,
-			 "of_get_property(%s, phy-connection-type) failed. Defaulting to MII\n",
+			 "of_get_phy_mode() for %s failed. Defaulting to SGMII\n",
 			 mac_node->full_name);
-		priv->phy_if = PHY_INTERFACE_MODE_MII;
-	} else {
-		priv->phy_if = str2phy(char_prop);
+		phy_if = PHY_INTERFACE_MODE_SGMII;
 	}
+	priv->phy_if = phy_if;
 
 	priv->speed		= phy2speed[priv->phy_if];
 	priv->max_speed		= priv->speed;
diff --git a/drivers/net/ethernet/freescale/fman/mac.h b/drivers/net/ethernet/freescale/fman/mac.h
index 0211cc9a46d6..d7313f0c5135 100644
--- a/drivers/net/ethernet/freescale/fman/mac.h
+++ b/drivers/net/ethernet/freescale/fman/mac.h
@@ -58,7 +58,8 @@ struct mac_device {
 	bool tx_pause_active;
 	bool promisc;
 
-	int (*init_phy)(struct net_device *net_dev, struct mac_device *mac_dev);
+	struct phy_device *(*init_phy)(struct net_device *net_dev,
+				       struct mac_device *mac_dev);
 	int (*init)(struct mac_device *mac_dev);
 	int (*start)(struct mac_device *mac_dev);
 	int (*stop)(struct mac_device *mac_dev);
diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
index 61fd486c50bb..dc120c148d97 100644
--- a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
+++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
@@ -60,6 +60,9 @@ module_param(fs_enet_debug, int, 0);
 MODULE_PARM_DESC(fs_enet_debug,
 		 "Freescale bitmapped debugging message enable value");
 
+#define RX_RING_SIZE	32
+#define TX_RING_SIZE	64
+
 #ifdef CONFIG_NET_POLL_CONTROLLER
 static void fs_enet_netpoll(struct net_device *dev);
 #endif
@@ -79,8 +82,8 @@ static void skb_align(struct sk_buff *skb, int align)
 		skb_reserve(skb, align - off);
 }
 
-/* NAPI receive function */
-static int fs_enet_rx_napi(struct napi_struct *napi, int budget)
+/* NAPI function */
+static int fs_enet_napi(struct napi_struct *napi, int budget)
 {
 	struct fs_enet_private *fep = container_of(napi, struct fs_enet_private, napi);
 	struct net_device *dev = fep->ndev;
@@ -90,9 +93,102 @@ static int fs_enet_rx_napi(struct napi_struct *napi, int budget)
 	int received = 0;
 	u16 pkt_len, sc;
 	int curidx;
+	int dirtyidx, do_wake, do_restart;
+	int tx_left = TX_RING_SIZE;
 
-	if (budget <= 0)
-		return received;
+	spin_lock(&fep->tx_lock);
+	bdp = fep->dirty_tx;
+
+	/* clear status bits for napi*/
+	(*fep->ops->napi_clear_event)(dev);
+
+	do_wake = do_restart = 0;
+	while (((sc = CBDR_SC(bdp)) & BD_ENET_TX_READY) == 0 && tx_left) {
+		dirtyidx = bdp - fep->tx_bd_base;
+
+		if (fep->tx_free == fep->tx_ring)
+			break;
+
+		skb = fep->tx_skbuff[dirtyidx];
+
+		/*
+		 * Check for errors.
+		 */
+		if (sc & (BD_ENET_TX_HB | BD_ENET_TX_LC |
+			  BD_ENET_TX_RL | BD_ENET_TX_UN | BD_ENET_TX_CSL)) {
+
+			if (sc & BD_ENET_TX_HB)	/* No heartbeat */
+				fep->stats.tx_heartbeat_errors++;
+			if (sc & BD_ENET_TX_LC)	/* Late collision */
+				fep->stats.tx_window_errors++;
+			if (sc & BD_ENET_TX_RL)	/* Retrans limit */
+				fep->stats.tx_aborted_errors++;
+			if (sc & BD_ENET_TX_UN)	/* Underrun */
+				fep->stats.tx_fifo_errors++;
+			if (sc & BD_ENET_TX_CSL)	/* Carrier lost */
+				fep->stats.tx_carrier_errors++;
+
+			if (sc & (BD_ENET_TX_LC | BD_ENET_TX_RL | BD_ENET_TX_UN)) {
+				fep->stats.tx_errors++;
+				do_restart = 1;
+			}
+		} else
+			fep->stats.tx_packets++;
+
+		if (sc & BD_ENET_TX_READY) {
+			dev_warn(fep->dev,
+				 "HEY! Enet xmit interrupt and TX_READY.\n");
+		}
+
+		/*
+		 * Deferred means some collisions occurred during transmit,
+		 * but we eventually sent the packet OK.
+		 */
+		if (sc & BD_ENET_TX_DEF)
+			fep->stats.collisions++;
+
+		/* unmap */
+		if (fep->mapped_as_page[dirtyidx])
+			dma_unmap_page(fep->dev, CBDR_BUFADDR(bdp),
+				       CBDR_DATLEN(bdp), DMA_TO_DEVICE);
+		else
+			dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
+					 CBDR_DATLEN(bdp), DMA_TO_DEVICE);
+
+		/*
+		 * Free the sk buffer associated with this last transmit.
+		 */
+		if (skb) {
+			dev_kfree_skb(skb);
+			fep->tx_skbuff[dirtyidx] = NULL;
+		}
+
+		/*
+		 * Update pointer to next buffer descriptor to be transmitted.
+		 */
+		if ((sc & BD_ENET_TX_WRAP) == 0)
+			bdp++;
+		else
+			bdp = fep->tx_bd_base;
+
+		/*
+		 * Since we have freed up a buffer, the ring is no longer
+		 * full.
+		 */
+		if (++fep->tx_free == MAX_SKB_FRAGS)
+			do_wake = 1;
+		tx_left--;
+	}
+
+	fep->dirty_tx = bdp;
+
+	if (do_restart)
+		(*fep->ops->tx_restart)(dev);
+
+	spin_unlock(&fep->tx_lock);
+
+	if (do_wake)
+		netif_wake_queue(dev);
 
 	/*
 	 * First, grab all of the stats for the incoming packet.
@@ -100,10 +196,8 @@ static int fs_enet_rx_napi(struct napi_struct *napi, int budget)
 	 */
 	bdp = fep->cur_rx;
 
-	/* clear RX status bits for napi*/
-	(*fep->ops->napi_clear_rx_event)(dev);
-
-	while (((sc = CBDR_SC(bdp)) & BD_ENET_RX_EMPTY) == 0) {
+	while (((sc = CBDR_SC(bdp)) & BD_ENET_RX_EMPTY) == 0 &&
+	       received < budget) {
 		curidx = bdp - fep->rx_bd_base;
 
 		/*
@@ -132,21 +226,10 @@ static int fs_enet_rx_napi(struct napi_struct *napi, int budget)
 			if (sc & BD_ENET_RX_OV)
 				fep->stats.rx_crc_errors++;
 
-			skb = fep->rx_skbuff[curidx];
-
-			dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
-				L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
-				DMA_FROM_DEVICE);
-
-			skbn = skb;
-
+			skbn = fep->rx_skbuff[curidx];
 		} else {
 			skb = fep->rx_skbuff[curidx];
 
-			dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
-				L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
-				DMA_FROM_DEVICE);
-
 			/*
 			 * Process the incoming frame.
 			 */
@@ -162,12 +245,30 @@ static int fs_enet_rx_napi(struct napi_struct *napi, int budget)
 					skb_copy_from_linear_data(skb,
 						      skbn->data, pkt_len);
 					swap(skb, skbn);
+					dma_sync_single_for_cpu(fep->dev,
+						CBDR_BUFADDR(bdp),
+						L1_CACHE_ALIGN(pkt_len),
+						DMA_FROM_DEVICE);
 				}
 			} else {
 				skbn = netdev_alloc_skb(dev, ENET_RX_FRSIZE);
 
-				if (skbn)
+				if (skbn) {
+					dma_addr_t dma;
+
 					skb_align(skbn, ENET_RX_ALIGN);
+
+					dma_unmap_single(fep->dev,
+						CBDR_BUFADDR(bdp),
+						L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
+						DMA_FROM_DEVICE);
+
+					dma = dma_map_single(fep->dev,
+						skbn->data,
+						L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
+						DMA_FROM_DEVICE);
+					CBDW_BUFADDR(bdp, dma);
+				}
 			}
 
 			if (skbn != NULL) {
@@ -182,9 +283,6 @@ static int fs_enet_rx_napi(struct napi_struct *napi, int budget)
 		}
 
 		fep->rx_skbuff[curidx] = skbn;
-		CBDW_BUFADDR(bdp, dma_map_single(fep->dev, skbn->data,
-			     L1_CACHE_ALIGN(PKT_MAXBUF_SIZE),
-			     DMA_FROM_DEVICE));
 		CBDW_DATLEN(bdp, 0);
 		CBDW_SC(bdp, (sc & ~BD_ENET_RX_STATS) | BD_ENET_RX_EMPTY);
 
@@ -197,134 +295,19 @@ static int fs_enet_rx_napi(struct napi_struct *napi, int budget)
 			bdp = fep->rx_bd_base;
 
 		(*fep->ops->rx_bd_done)(dev);
-
-		if (received >= budget)
-			break;
 	}
 
 	fep->cur_rx = bdp;
 
-	if (received < budget) {
+	if (received < budget && tx_left) {
 		/* done */
 		napi_complete(napi);
-		(*fep->ops->napi_enable_rx)(dev);
-	}
-	return received;
-}
-
-static int fs_enet_tx_napi(struct napi_struct *napi, int budget)
-{
-	struct fs_enet_private *fep = container_of(napi, struct fs_enet_private,
-						   napi_tx);
-	struct net_device *dev = fep->ndev;
-	cbd_t __iomem *bdp;
-	struct sk_buff *skb;
-	int dirtyidx, do_wake, do_restart;
-	u16 sc;
-	int has_tx_work = 0;
-
-	spin_lock(&fep->tx_lock);
-	bdp = fep->dirty_tx;
-
-	/* clear TX status bits for napi*/
-	(*fep->ops->napi_clear_tx_event)(dev);
-
-	do_wake = do_restart = 0;
-	while (((sc = CBDR_SC(bdp)) & BD_ENET_TX_READY) == 0) {
-		dirtyidx = bdp - fep->tx_bd_base;
-
-		if (fep->tx_free == fep->tx_ring)
-			break;
-
-		skb = fep->tx_skbuff[dirtyidx];
-
-		/*
-		 * Check for errors.
-		 */
-		if (sc & (BD_ENET_TX_HB | BD_ENET_TX_LC |
-			  BD_ENET_TX_RL | BD_ENET_TX_UN | BD_ENET_TX_CSL)) {
-
-			if (sc & BD_ENET_TX_HB)	/* No heartbeat */
-				fep->stats.tx_heartbeat_errors++;
-			if (sc & BD_ENET_TX_LC)	/* Late collision */
-				fep->stats.tx_window_errors++;
-			if (sc & BD_ENET_TX_RL)	/* Retrans limit */
-				fep->stats.tx_aborted_errors++;
-			if (sc & BD_ENET_TX_UN)	/* Underrun */
-				fep->stats.tx_fifo_errors++;
-			if (sc & BD_ENET_TX_CSL)	/* Carrier lost */
-				fep->stats.tx_carrier_errors++;
-
-			if (sc & (BD_ENET_TX_LC | BD_ENET_TX_RL | BD_ENET_TX_UN)) {
-				fep->stats.tx_errors++;
-				do_restart = 1;
-			}
-		} else
-			fep->stats.tx_packets++;
-
-		if (sc & BD_ENET_TX_READY) {
-			dev_warn(fep->dev,
-				 "HEY! Enet xmit interrupt and TX_READY.\n");
-		}
-
-		/*
-		 * Deferred means some collisions occurred during transmit,
-		 * but we eventually sent the packet OK.
-		 */
-		if (sc & BD_ENET_TX_DEF)
-			fep->stats.collisions++;
-
-		/* unmap */
-		if (fep->mapped_as_page[dirtyidx])
-			dma_unmap_page(fep->dev, CBDR_BUFADDR(bdp),
-				       CBDR_DATLEN(bdp), DMA_TO_DEVICE);
-		else
-			dma_unmap_single(fep->dev, CBDR_BUFADDR(bdp),
-					 CBDR_DATLEN(bdp), DMA_TO_DEVICE);
-
-		/*
-		 * Free the sk buffer associated with this last transmit.
-		 */
-		if (skb) {
-			dev_kfree_skb(skb);
-			fep->tx_skbuff[dirtyidx] = NULL;
-		}
-
-		/*
-		 * Update pointer to next buffer descriptor to be transmitted.
-		 */
-		if ((sc & BD_ENET_TX_WRAP) == 0)
-			bdp++;
-		else
-			bdp = fep->tx_bd_base;
-
-		/*
-		 * Since we have freed up a buffer, the ring is no longer
-		 * full.
-		 */
-		if (++fep->tx_free >= MAX_SKB_FRAGS)
-			do_wake = 1;
-		has_tx_work = 1;
-	}
-
-	fep->dirty_tx = bdp;
-
-	if (do_restart)
-		(*fep->ops->tx_restart)(dev);
+		(*fep->ops->napi_enable)(dev);
 
-	if (!has_tx_work) {
-		napi_complete(napi);
-		(*fep->ops->napi_enable_tx)(dev);
+		return received;
 	}
 
-	spin_unlock(&fep->tx_lock);
-
-	if (do_wake)
-		netif_wake_queue(dev);
-
-	if (has_tx_work)
-		return budget;
-	return 0;
+	return budget;
 }
 
 /*
@@ -350,18 +333,18 @@ fs_enet_interrupt(int irq, void *dev_id)
 		nr++;
 
 		int_clr_events = int_events;
-		int_clr_events &= ~fep->ev_napi_rx;
+		int_clr_events &= ~fep->ev_napi;
 
 		(*fep->ops->clear_int_events)(dev, int_clr_events);
 
 		if (int_events & fep->ev_err)
 			(*fep->ops->ev_error)(dev, int_events);
 
-		if (int_events & fep->ev_rx) {
+		if (int_events & fep->ev) {
 			napi_ok = napi_schedule_prep(&fep->napi);
 
-			(*fep->ops->napi_disable_rx)(dev);
-			(*fep->ops->clear_int_events)(dev, fep->ev_napi_rx);
+			(*fep->ops->napi_disable)(dev);
+			(*fep->ops->clear_int_events)(dev, fep->ev_napi);
 
 			/* NOTE: it is possible for FCCs in NAPI mode    */
 			/* to submit a spurious interrupt while in poll  */
@@ -369,17 +352,6 @@ fs_enet_interrupt(int irq, void *dev_id)
 				__napi_schedule(&fep->napi);
 		}
 
-		if (int_events & fep->ev_tx) {
-			napi_ok = napi_schedule_prep(&fep->napi_tx);
-
-			(*fep->ops->napi_disable_tx)(dev);
-			(*fep->ops->clear_int_events)(dev, fep->ev_napi_tx);
-
-			/* NOTE: it is possible for FCCs in NAPI mode    */
-			/* to submit a spurious interrupt while in poll  */
-			if (napi_ok)
-				__napi_schedule(&fep->napi_tx);
-		}
 	}
 
 	handled = nr > 0;
@@ -659,7 +631,8 @@ static void fs_timeout(struct net_device *dev)
 	}
 
 	phy_start(dev->phydev);
-	wake = fep->tx_free && !(CBDR_SC(fep->cur_tx) & BD_ENET_TX_READY);
+	wake = fep->tx_free >= MAX_SKB_FRAGS &&
+	       !(CBDR_SC(fep->cur_tx) & BD_ENET_TX_READY);
 	spin_unlock_irqrestore(&fep->lock, flags);
 
 	if (wake)
@@ -751,11 +724,10 @@ static int fs_enet_open(struct net_device *dev)
 	int err;
 
 	/* to initialize the fep->cur_rx,... */
-	/* not doing this, will cause a crash in fs_enet_rx_napi */
+	/* not doing this, will cause a crash in fs_enet_napi */
 	fs_init_bds(fep->ndev);
 
 	napi_enable(&fep->napi);
-	napi_enable(&fep->napi_tx);
 
 	/* Install our interrupt handler. */
 	r = request_irq(fep->interrupt, fs_enet_interrupt, IRQF_SHARED,
@@ -763,7 +735,6 @@ static int fs_enet_open(struct net_device *dev)
 	if (r != 0) {
 		dev_err(fep->dev, "Could not allocate FS_ENET IRQ!");
 		napi_disable(&fep->napi);
-		napi_disable(&fep->napi_tx);
 		return -EINVAL;
 	}
 
@@ -771,7 +742,6 @@ static int fs_enet_open(struct net_device *dev)
 	if (err) {
 		free_irq(fep->interrupt, dev);
 		napi_disable(&fep->napi);
-		napi_disable(&fep->napi_tx);
 		return err;
 	}
 	phy_start(dev->phydev);
@@ -789,7 +759,6 @@ static int fs_enet_close(struct net_device *dev)
 	netif_stop_queue(dev);
 	netif_carrier_off(dev);
 	napi_disable(&fep->napi);
-	napi_disable(&fep->napi_tx);
 	phy_stop(dev->phydev);
 
 	spin_lock_irqsave(&fep->lock, flags);
@@ -861,6 +830,44 @@ static void fs_set_msglevel(struct net_device *dev, u32 value)
 	fep->msg_enable = value;
 }
 
+static int fs_get_tunable(struct net_device *dev,
+			  const struct ethtool_tunable *tuna, void *data)
+{
+	struct fs_enet_private *fep = netdev_priv(dev);
+	struct fs_platform_info *fpi = fep->fpi;
+	int ret = 0;
+
+	switch (tuna->id) {
+	case ETHTOOL_RX_COPYBREAK:
+		*(u32 *)data = fpi->rx_copybreak;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int fs_set_tunable(struct net_device *dev,
+			  const struct ethtool_tunable *tuna, const void *data)
+{
+	struct fs_enet_private *fep = netdev_priv(dev);
+	struct fs_platform_info *fpi = fep->fpi;
+	int ret = 0;
+
+	switch (tuna->id) {
+	case ETHTOOL_RX_COPYBREAK:
+		fpi->rx_copybreak = *(u32 *)data;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
 static const struct ethtool_ops fs_ethtool_ops = {
 	.get_drvinfo = fs_get_drvinfo,
 	.get_regs_len = fs_get_regs_len,
@@ -872,6 +879,8 @@ static const struct ethtool_ops fs_ethtool_ops = {
 	.get_ts_info = ethtool_op_get_ts_info,
 	.get_link_ksettings = phy_ethtool_get_link_ksettings,
 	.set_link_ksettings = phy_ethtool_set_link_ksettings,
+	.get_tunable = fs_get_tunable,
+	.set_tunable = fs_set_tunable,
 };
 
 static int fs_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
@@ -939,8 +948,8 @@ static int fs_enet_probe(struct platform_device *ofdev)
 		fpi->cp_command = *data;
 	}
 
-	fpi->rx_ring = 32;
-	fpi->tx_ring = 64;
+	fpi->rx_ring = RX_RING_SIZE;
+	fpi->tx_ring = TX_RING_SIZE;
 	fpi->rx_copybreak = 240;
 	fpi->napi_weight = 17;
 	fpi->phy_node = of_parse_phandle(ofdev->dev.of_node, "phy-handle", 0);
@@ -1024,8 +1033,7 @@ static int fs_enet_probe(struct platform_device *ofdev)
 
 	ndev->netdev_ops = &fs_enet_netdev_ops;
 	ndev->watchdog_timeo = 2 * HZ;
-	netif_napi_add(ndev, &fep->napi, fs_enet_rx_napi, fpi->napi_weight);
-	netif_tx_napi_add(ndev, &fep->napi_tx, fs_enet_tx_napi, 2);
+	netif_napi_add(ndev, &fep->napi, fs_enet_napi, fpi->napi_weight);
 
 	ndev->ethtool_ops = &fs_ethtool_ops;
 
diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet.h b/drivers/net/ethernet/freescale/fs_enet/fs_enet.h
index e29f54a35210..fee24c822fad 100644
--- a/drivers/net/ethernet/freescale/fs_enet/fs_enet.h
+++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet.h
@@ -81,12 +81,9 @@ struct fs_ops {
 	void (*adjust_link)(struct net_device *dev);
 	void (*restart)(struct net_device *dev);
 	void (*stop)(struct net_device *dev);
-	void (*napi_clear_rx_event)(struct net_device *dev);
-	void (*napi_enable_rx)(struct net_device *dev);
-	void (*napi_disable_rx)(struct net_device *dev);
-	void (*napi_clear_tx_event)(struct net_device *dev);
-	void (*napi_enable_tx)(struct net_device *dev);
-	void (*napi_disable_tx)(struct net_device *dev);
+	void (*napi_clear_event)(struct net_device *dev);
+	void (*napi_enable)(struct net_device *dev);
+	void (*napi_disable)(struct net_device *dev);
 	void (*rx_bd_done)(struct net_device *dev);
 	void (*tx_kickstart)(struct net_device *dev);
 	u32 (*get_int_events)(struct net_device *dev);
@@ -122,7 +119,6 @@ struct phy_info {
 
 struct fs_enet_private {
 	struct napi_struct napi;
-	struct napi_struct napi_tx;
 	struct device *dev;	/* pointer back to the device (must be initialized first) */
 	struct net_device *ndev;
 	spinlock_t lock;	/* during all ops except TX pckt processing */
@@ -152,10 +148,8 @@ struct fs_enet_private {
 	int oldduplex, oldspeed, oldlink;	/* current settings */
 
 	/* event masks */
-	u32 ev_napi_rx;		/* mask of NAPI rx events */
-	u32 ev_napi_tx;		/* mask of NAPI rx events */
-	u32 ev_rx;		/* rx event mask          */
-	u32 ev_tx;		/* tx event mask          */
+	u32 ev_napi;		/* mask of NAPI events */
+	u32 ev;			/* event mask          */
 	u32 ev_err;		/* error event mask       */
 
 	u16 bd_rx_empty;	/* mask of BD rx empty	  */
diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c b/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c
index d71761a34022..120c758f5d01 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c
@@ -90,7 +90,7 @@ static int do_pd_setup(struct fs_enet_private *fep)
 	int ret = -EINVAL;
 
 	fep->interrupt = irq_of_parse_and_map(ofdev->dev.of_node, 0);
-	if (fep->interrupt == NO_IRQ)
+	if (!fep->interrupt)
 		goto out;
 
 	fep->fcc.fccp = of_iomap(ofdev->dev.of_node, 0);
@@ -124,10 +124,8 @@ out:
 	return ret;
 }
 
-#define FCC_NAPI_RX_EVENT_MSK	(FCC_ENET_RXF | FCC_ENET_RXB)
-#define FCC_NAPI_TX_EVENT_MSK	(FCC_ENET_TXB)
-#define FCC_RX_EVENT		(FCC_ENET_RXF)
-#define FCC_TX_EVENT		(FCC_ENET_TXB)
+#define FCC_NAPI_EVENT_MSK	(FCC_ENET_RXF | FCC_ENET_RXB | FCC_ENET_TXB)
+#define FCC_EVENT		(FCC_ENET_RXF | FCC_ENET_TXB)
 #define FCC_ERR_EVENT_MSK	(FCC_ENET_TXE)
 
 static int setup_data(struct net_device *dev)
@@ -137,10 +135,8 @@ static int setup_data(struct net_device *dev)
 	if (do_pd_setup(fep) != 0)
 		return -EINVAL;
 
-	fep->ev_napi_rx = FCC_NAPI_RX_EVENT_MSK;
-	fep->ev_napi_tx = FCC_NAPI_TX_EVENT_MSK;
-	fep->ev_rx = FCC_RX_EVENT;
-	fep->ev_tx = FCC_TX_EVENT;
+	fep->ev_napi = FCC_NAPI_EVENT_MSK;
+	fep->ev = FCC_EVENT;
 	fep->ev_err = FCC_ERR_EVENT_MSK;
 
 	return 0;
@@ -424,52 +420,28 @@ static void stop(struct net_device *dev)
 	fs_cleanup_bds(dev);
 }
 
-static void napi_clear_rx_event(struct net_device *dev)
+static void napi_clear_event_fs(struct net_device *dev)
 {
 	struct fs_enet_private *fep = netdev_priv(dev);
 	fcc_t __iomem *fccp = fep->fcc.fccp;
 
-	W16(fccp, fcc_fcce, FCC_NAPI_RX_EVENT_MSK);
+	W16(fccp, fcc_fcce, FCC_NAPI_EVENT_MSK);
 }
 
-static void napi_enable_rx(struct net_device *dev)
+static void napi_enable_fs(struct net_device *dev)
 {
 	struct fs_enet_private *fep = netdev_priv(dev);
 	fcc_t __iomem *fccp = fep->fcc.fccp;
 
-	S16(fccp, fcc_fccm, FCC_NAPI_RX_EVENT_MSK);
+	S16(fccp, fcc_fccm, FCC_NAPI_EVENT_MSK);
 }
 
-static void napi_disable_rx(struct net_device *dev)
+static void napi_disable_fs(struct net_device *dev)
 {
 	struct fs_enet_private *fep = netdev_priv(dev);
 	fcc_t __iomem *fccp = fep->fcc.fccp;
 
-	C16(fccp, fcc_fccm, FCC_NAPI_RX_EVENT_MSK);
-}
-
-static void napi_clear_tx_event(struct net_device *dev)
-{
-	struct fs_enet_private *fep = netdev_priv(dev);
-	fcc_t __iomem *fccp = fep->fcc.fccp;
-
-	W16(fccp, fcc_fcce, FCC_NAPI_TX_EVENT_MSK);
-}
-
-static void napi_enable_tx(struct net_device *dev)
-{
-	struct fs_enet_private *fep = netdev_priv(dev);
-	fcc_t __iomem *fccp = fep->fcc.fccp;
-
-	S16(fccp, fcc_fccm, FCC_NAPI_TX_EVENT_MSK);
-}
-
-static void napi_disable_tx(struct net_device *dev)
-{
-	struct fs_enet_private *fep = netdev_priv(dev);
-	fcc_t __iomem *fccp = fep->fcc.fccp;
-
-	C16(fccp, fcc_fccm, FCC_NAPI_TX_EVENT_MSK);
+	C16(fccp, fcc_fccm, FCC_NAPI_EVENT_MSK);
 }
 
 static void rx_bd_done(struct net_device *dev)
@@ -595,12 +567,9 @@ const struct fs_ops fs_fcc_ops = {
 	.set_multicast_list	= set_multicast_list,
 	.restart		= restart,
 	.stop			= stop,
-	.napi_clear_rx_event	= napi_clear_rx_event,
-	.napi_enable_rx		= napi_enable_rx,
-	.napi_disable_rx	= napi_disable_rx,
-	.napi_clear_tx_event	= napi_clear_tx_event,
-	.napi_enable_tx		= napi_enable_tx,
-	.napi_disable_tx	= napi_disable_tx,
+	.napi_clear_event	= napi_clear_event_fs,
+	.napi_enable		= napi_enable_fs,
+	.napi_disable		= napi_disable_fs,
 	.rx_bd_done		= rx_bd_done,
 	.tx_kickstart		= tx_kickstart,
 	.get_int_events		= get_int_events,
diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c
index 35a318ed3a62..777beffa1e1e 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c
@@ -99,7 +99,7 @@ static int do_pd_setup(struct fs_enet_private *fep)
 	struct platform_device *ofdev = to_platform_device(fep->dev);
 
 	fep->interrupt = irq_of_parse_and_map(ofdev->dev.of_node, 0);
-	if (fep->interrupt == NO_IRQ)
+	if (!fep->interrupt)
 		return -EINVAL;
 
 	fep->fec.fecp = of_iomap(ofdev->dev.of_node, 0);
@@ -109,10 +109,8 @@ static int do_pd_setup(struct fs_enet_private *fep)
 	return 0;
 }
 
-#define FEC_NAPI_RX_EVENT_MSK	(FEC_ENET_RXF | FEC_ENET_RXB)
-#define FEC_NAPI_TX_EVENT_MSK	(FEC_ENET_TXF)
-#define FEC_RX_EVENT		(FEC_ENET_RXF)
-#define FEC_TX_EVENT		(FEC_ENET_TXF)
+#define FEC_NAPI_EVENT_MSK	(FEC_ENET_RXF | FEC_ENET_RXB | FEC_ENET_TXF)
+#define FEC_EVENT		(FEC_ENET_RXF | FEC_ENET_TXF)
 #define FEC_ERR_EVENT_MSK	(FEC_ENET_HBERR | FEC_ENET_BABR | \
 				 FEC_ENET_BABT | FEC_ENET_EBERR)
 
@@ -126,10 +124,8 @@ static int setup_data(struct net_device *dev)
 	fep->fec.hthi = 0;
 	fep->fec.htlo = 0;
 
-	fep->ev_napi_rx = FEC_NAPI_RX_EVENT_MSK;
-	fep->ev_napi_tx = FEC_NAPI_TX_EVENT_MSK;
-	fep->ev_rx = FEC_RX_EVENT;
-	fep->ev_tx = FEC_TX_EVENT;
+	fep->ev_napi = FEC_NAPI_EVENT_MSK;
+	fep->ev = FEC_EVENT;
 	fep->ev_err = FEC_ERR_EVENT_MSK;
 
 	return 0;
@@ -396,52 +392,28 @@ static void stop(struct net_device *dev)
 	}
 }
 
-static void napi_clear_rx_event(struct net_device *dev)
+static void napi_clear_event_fs(struct net_device *dev)
 {
 	struct fs_enet_private *fep = netdev_priv(dev);
 	struct fec __iomem *fecp = fep->fec.fecp;
 
-	FW(fecp, ievent, FEC_NAPI_RX_EVENT_MSK);
+	FW(fecp, ievent, FEC_NAPI_EVENT_MSK);
 }
 
-static void napi_enable_rx(struct net_device *dev)
+static void napi_enable_fs(struct net_device *dev)
 {
 	struct fs_enet_private *fep = netdev_priv(dev);
 	struct fec __iomem *fecp = fep->fec.fecp;
 
-	FS(fecp, imask, FEC_NAPI_RX_EVENT_MSK);
+	FS(fecp, imask, FEC_NAPI_EVENT_MSK);
 }
 
-static void napi_disable_rx(struct net_device *dev)
+static void napi_disable_fs(struct net_device *dev)
 {
 	struct fs_enet_private *fep = netdev_priv(dev);
 	struct fec __iomem *fecp = fep->fec.fecp;
 
-	FC(fecp, imask, FEC_NAPI_RX_EVENT_MSK);
-}
-
-static void napi_clear_tx_event(struct net_device *dev)
-{
-	struct fs_enet_private *fep = netdev_priv(dev);
-	struct fec __iomem *fecp = fep->fec.fecp;
-
-	FW(fecp, ievent, FEC_NAPI_TX_EVENT_MSK);
-}
-
-static void napi_enable_tx(struct net_device *dev)
-{
-	struct fs_enet_private *fep = netdev_priv(dev);
-	struct fec __iomem *fecp = fep->fec.fecp;
-
-	FS(fecp, imask, FEC_NAPI_TX_EVENT_MSK);
-}
-
-static void napi_disable_tx(struct net_device *dev)
-{
-	struct fs_enet_private *fep = netdev_priv(dev);
-	struct fec __iomem *fecp = fep->fec.fecp;
-
-	FC(fecp, imask, FEC_NAPI_TX_EVENT_MSK);
+	FC(fecp, imask, FEC_NAPI_EVENT_MSK);
 }
 
 static void rx_bd_done(struct net_device *dev)
@@ -513,12 +485,9 @@ const struct fs_ops fs_fec_ops = {
 	.set_multicast_list	= set_multicast_list,
 	.restart		= restart,
 	.stop			= stop,
-	.napi_clear_rx_event	= napi_clear_rx_event,
-	.napi_enable_rx		= napi_enable_rx,
-	.napi_disable_rx	= napi_disable_rx,
-	.napi_clear_tx_event	= napi_clear_tx_event,
-	.napi_enable_tx		= napi_enable_tx,
-	.napi_disable_tx	= napi_disable_tx,
+	.napi_clear_event	= napi_clear_event_fs,
+	.napi_enable		= napi_enable_fs,
+	.napi_disable		= napi_disable_fs,
 	.rx_bd_done		= rx_bd_done,
 	.tx_kickstart		= tx_kickstart,
 	.get_int_events		= get_int_events,
diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-scc.c b/drivers/net/ethernet/freescale/fs_enet/mac-scc.c
index e8b9c33d35b4..15abd37d70e3 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mac-scc.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mac-scc.c
@@ -99,7 +99,7 @@ static int do_pd_setup(struct fs_enet_private *fep)
 	struct platform_device *ofdev = to_platform_device(fep->dev);
 
 	fep->interrupt = irq_of_parse_and_map(ofdev->dev.of_node, 0);
-	if (fep->interrupt == NO_IRQ)
+	if (!fep->interrupt)
 		return -EINVAL;
 
 	fep->scc.sccp = of_iomap(ofdev->dev.of_node, 0);
@@ -115,10 +115,8 @@ static int do_pd_setup(struct fs_enet_private *fep)
 	return 0;
 }
 
-#define SCC_NAPI_RX_EVENT_MSK	(SCCE_ENET_RXF | SCCE_ENET_RXB)
-#define SCC_NAPI_TX_EVENT_MSK	(SCCE_ENET_TXB)
-#define SCC_RX_EVENT		(SCCE_ENET_RXF)
-#define SCC_TX_EVENT		(SCCE_ENET_TXB)
+#define SCC_NAPI_EVENT_MSK	(SCCE_ENET_RXF | SCCE_ENET_RXB | SCCE_ENET_TXB)
+#define SCC_EVENT		(SCCE_ENET_RXF | SCCE_ENET_TXB)
 #define SCC_ERR_EVENT_MSK	(SCCE_ENET_TXE | SCCE_ENET_BSY)
 
 static int setup_data(struct net_device *dev)
@@ -130,10 +128,8 @@ static int setup_data(struct net_device *dev)
 	fep->scc.hthi = 0;
 	fep->scc.htlo = 0;
 
-	fep->ev_napi_rx = SCC_NAPI_RX_EVENT_MSK;
-	fep->ev_napi_tx = SCC_NAPI_TX_EVENT_MSK;
-	fep->ev_rx = SCC_RX_EVENT;
-	fep->ev_tx = SCC_TX_EVENT | SCCE_ENET_TXE;
+	fep->ev_napi = SCC_NAPI_EVENT_MSK;
+	fep->ev = SCC_EVENT | SCCE_ENET_TXE;
 	fep->ev_err = SCC_ERR_EVENT_MSK;
 
 	return 0;
@@ -379,52 +375,28 @@ static void stop(struct net_device *dev)
 	fs_cleanup_bds(dev);
 }
 
-static void napi_clear_rx_event(struct net_device *dev)
+static void napi_clear_event_fs(struct net_device *dev)
 {
 	struct fs_enet_private *fep = netdev_priv(dev);
 	scc_t __iomem *sccp = fep->scc.sccp;
 
-	W16(sccp, scc_scce, SCC_NAPI_RX_EVENT_MSK);
+	W16(sccp, scc_scce, SCC_NAPI_EVENT_MSK);
 }
 
-static void napi_enable_rx(struct net_device *dev)
+static void napi_enable_fs(struct net_device *dev)
 {
 	struct fs_enet_private *fep = netdev_priv(dev);
 	scc_t __iomem *sccp = fep->scc.sccp;
 
-	S16(sccp, scc_sccm, SCC_NAPI_RX_EVENT_MSK);
+	S16(sccp, scc_sccm, SCC_NAPI_EVENT_MSK);
 }
 
-static void napi_disable_rx(struct net_device *dev)
+static void napi_disable_fs(struct net_device *dev)
 {
 	struct fs_enet_private *fep = netdev_priv(dev);
 	scc_t __iomem *sccp = fep->scc.sccp;
 
-	C16(sccp, scc_sccm, SCC_NAPI_RX_EVENT_MSK);
-}
-
-static void napi_clear_tx_event(struct net_device *dev)
-{
-	struct fs_enet_private *fep = netdev_priv(dev);
-	scc_t __iomem *sccp = fep->scc.sccp;
-
-	W16(sccp, scc_scce, SCC_NAPI_TX_EVENT_MSK);
-}
-
-static void napi_enable_tx(struct net_device *dev)
-{
-	struct fs_enet_private *fep = netdev_priv(dev);
-	scc_t __iomem *sccp = fep->scc.sccp;
-
-	S16(sccp, scc_sccm, SCC_NAPI_TX_EVENT_MSK);
-}
-
-static void napi_disable_tx(struct net_device *dev)
-{
-	struct fs_enet_private *fep = netdev_priv(dev);
-	scc_t __iomem *sccp = fep->scc.sccp;
-
-	C16(sccp, scc_sccm, SCC_NAPI_TX_EVENT_MSK);
+	C16(sccp, scc_sccm, SCC_NAPI_EVENT_MSK);
 }
 
 static void rx_bd_done(struct net_device *dev)
@@ -497,12 +469,9 @@ const struct fs_ops fs_scc_ops = {
 	.set_multicast_list	= set_multicast_list,
 	.restart		= restart,
 	.stop			= stop,
-	.napi_clear_rx_event	= napi_clear_rx_event,
-	.napi_enable_rx		= napi_enable_rx,
-	.napi_disable_rx	= napi_disable_rx,
-	.napi_clear_tx_event	= napi_clear_tx_event,
-	.napi_enable_tx		= napi_enable_tx,
-	.napi_disable_tx	= napi_disable_tx,
+	.napi_clear_event	= napi_clear_event_fs,
+	.napi_enable		= napi_enable_fs,
+	.napi_disable		= napi_disable_fs,
 	.rx_bd_done		= rx_bd_done,
 	.tx_kickstart		= tx_kickstart,
 	.get_int_events		= get_int_events,
diff --git a/drivers/net/ethernet/freescale/fsl_pq_mdio.c b/drivers/net/ethernet/freescale/fsl_pq_mdio.c
index f3c63dce1e30..446c7b374ff5 100644
--- a/drivers/net/ethernet/freescale/fsl_pq_mdio.c
+++ b/drivers/net/ethernet/freescale/fsl_pq_mdio.c
@@ -195,7 +195,7 @@ static int fsl_pq_mdio_reset(struct mii_bus *bus)
 	return 0;
 }
 
-#if defined(CONFIG_GIANFAR) || defined(CONFIG_GIANFAR_MODULE)
+#if IS_ENABLED(CONFIG_GIANFAR)
 /*
  * Return the TBIPA address, starting from the address
  * of the mapped GFAR MDIO registers (struct gfar)
@@ -228,7 +228,7 @@ static uint32_t __iomem *get_etsec_tbipa(void __iomem *p)
 }
 #endif
 
-#if defined(CONFIG_UCC_GETH) || defined(CONFIG_UCC_GETH_MODULE)
+#if IS_ENABLED(CONFIG_UCC_GETH)
 /*
  * Return the TBIPAR address for a QE MDIO node, starting from the address
  * of the mapped MII registers (struct fsl_pq_mii)
@@ -306,7 +306,7 @@ static void ucc_configure(phys_addr_t start, phys_addr_t end)
 #endif
 
 static const struct of_device_id fsl_pq_mdio_match[] = {
-#if defined(CONFIG_GIANFAR) || defined(CONFIG_GIANFAR_MODULE)
+#if IS_ENABLED(CONFIG_GIANFAR)
 	{
 		.compatible = "fsl,gianfar-tbi",
 		.data = &(struct fsl_pq_mdio_data) {
@@ -344,7 +344,7 @@ static const struct of_device_id fsl_pq_mdio_match[] = {
 		},
 	},
 #endif
-#if defined(CONFIG_UCC_GETH) || defined(CONFIG_UCC_GETH_MODULE)
+#if IS_ENABLED(CONFIG_UCC_GETH)
 	{
 		.compatible = "fsl,ucc-mdio",
 		.data = &(struct fsl_pq_mdio_data) {
diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c
index d20935dc8399..4b4f5bc0e279 100644
--- a/drivers/net/ethernet/freescale/gianfar.c
+++ b/drivers/net/ethernet/freescale/gianfar.c
@@ -2922,17 +2922,25 @@ static bool gfar_add_rx_frag(struct gfar_rx_buff *rxb, u32 lstatus,
 {
 	unsigned int size = lstatus & BD_LENGTH_MASK;
 	struct page *page = rxb->page;
+	bool last = !!(lstatus & BD_LFLAG(RXBD_LAST));
 
 	/* Remove the FCS from the packet length */
-	if (likely(lstatus & BD_LFLAG(RXBD_LAST)))
+	if (last)
 		size -= ETH_FCS_LEN;
 
-	if (likely(first))
+	if (likely(first)) {
 		skb_put(skb, size);
-	else
-		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
-				rxb->page_offset + RXBUF_ALIGNMENT,
-				size, GFAR_RXB_TRUESIZE);
+	} else {
+		/* the last fragments' length contains the full frame length */
+		if (last)
+			size -= skb->len;
+
+		/* in case the last fragment consisted only of the FCS */
+		if (size > 0)
+			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
+					rxb->page_offset + RXBUF_ALIGNMENT,
+					size, GFAR_RXB_TRUESIZE);
+	}
 
 	/* try reuse page */
 	if (unlikely(page_count(page) != 1))
diff --git a/drivers/net/ethernet/freescale/gianfar.h b/drivers/net/ethernet/freescale/gianfar.h
index 373fd094f2f3..6e8a9c8467b9 100644
--- a/drivers/net/ethernet/freescale/gianfar.h
+++ b/drivers/net/ethernet/freescale/gianfar.h
@@ -100,7 +100,8 @@ extern const char gfar_driver_version[];
 #define DEFAULT_RX_LFC_THR  16
 #define DEFAULT_LFC_PTVVAL  4
 
-#define GFAR_RXB_SIZE 1536
+/* prevent fragmenation by HW in DSA environments */
+#define GFAR_RXB_SIZE roundup(1536 + 8, 64)
 #define GFAR_SKBFRAG_SIZE (RXBUF_ALIGNMENT + GFAR_RXB_SIZE \
 			  + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
 #define GFAR_RXB_TRUESIZE 2048
diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c
index 5bf1ade28315..186ef8f16c80 100644
--- a/drivers/net/ethernet/freescale/ucc_geth.c
+++ b/drivers/net/ethernet/freescale/ucc_geth.c
@@ -3756,7 +3756,7 @@ static int ucc_geth_probe(struct platform_device* ofdev)
 			return -EINVAL;
 		}
 		if ((*prop < QE_CLK_NONE) || (*prop > QE_CLK24)) {
-			pr_err("invalid rx-clock propperty\n");
+			pr_err("invalid rx-clock property\n");
 			return -EINVAL;
 		}
 		ug_info->uf_info.rx_clock = *prop;
diff --git a/drivers/net/ethernet/freescale/xgmac_mdio.c b/drivers/net/ethernet/freescale/xgmac_mdio.c
index 7b8fe866f603..e03b30c60dcf 100644
--- a/drivers/net/ethernet/freescale/xgmac_mdio.c
+++ b/drivers/net/ethernet/freescale/xgmac_mdio.c
@@ -271,11 +271,8 @@ static int xgmac_mdio_probe(struct platform_device *pdev)
 		goto err_ioremap;
 	}
 
-	if (of_get_property(pdev->dev.of_node,
-			    "little-endian", NULL))
-		priv->is_little_endian = true;
-	else
-		priv->is_little_endian = false;
+	priv->is_little_endian = of_property_read_bool(pdev->dev.of_node,
+						       "little-endian");
 
 	ret = of_mdiobus_register(bus, np);
 	if (ret) {
diff --git a/drivers/net/ethernet/hisilicon/hip04_eth.c b/drivers/net/ethernet/hisilicon/hip04_eth.c
index 0c4afe95ef54..39778892b3b3 100644
--- a/drivers/net/ethernet/hisilicon/hip04_eth.c
+++ b/drivers/net/ethernet/hisilicon/hip04_eth.c
@@ -600,7 +600,7 @@ static irqreturn_t hip04_mac_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-enum hrtimer_restart tx_done(struct hrtimer *hrtimer)
+static enum hrtimer_restart tx_done(struct hrtimer *hrtimer)
 {
 	struct hip04_priv *priv;
 
@@ -755,13 +755,13 @@ static void hip04_get_drvinfo(struct net_device *netdev,
 	strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version));
 }
 
-static struct ethtool_ops hip04_ethtool_ops = {
+static const struct ethtool_ops hip04_ethtool_ops = {
 	.get_coalesce		= hip04_get_coalesce,
 	.set_coalesce		= hip04_set_coalesce,
 	.get_drvinfo		= hip04_get_drvinfo,
 };
 
-static struct net_device_ops hip04_netdev_ops = {
+static const struct net_device_ops hip04_netdev_ops = {
 	.ndo_open		= hip04_mac_open,
 	.ndo_stop		= hip04_mac_stop,
 	.ndo_get_stats		= hip04_get_stats,
diff --git a/drivers/net/ethernet/hisilicon/hisi_femac.c b/drivers/net/ethernet/hisilicon/hisi_femac.c
index b5d7ad0252a0..ced185962ef8 100644
--- a/drivers/net/ethernet/hisilicon/hisi_femac.c
+++ b/drivers/net/ethernet/hisilicon/hisi_femac.c
@@ -699,7 +699,7 @@ static int hisi_femac_net_ioctl(struct net_device *dev,
 	return phy_mii_ioctl(dev->phydev, ifreq, cmd);
 }
 
-static struct ethtool_ops hisi_femac_ethtools_ops = {
+static const struct ethtool_ops hisi_femac_ethtools_ops = {
 	.get_link		= ethtool_op_get_link,
 	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
 	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
@@ -940,8 +940,8 @@ static int hisi_femac_drv_remove(struct platform_device *pdev)
 }
 
 #ifdef CONFIG_PM
-int hisi_femac_drv_suspend(struct platform_device *pdev,
-			   pm_message_t state)
+static int hisi_femac_drv_suspend(struct platform_device *pdev,
+				  pm_message_t state)
 {
 	struct net_device *ndev = platform_get_drvdata(pdev);
 	struct hisi_femac_priv *priv = netdev_priv(ndev);
@@ -957,7 +957,7 @@ int hisi_femac_drv_suspend(struct platform_device *pdev,
 	return 0;
 }
 
-int hisi_femac_drv_resume(struct platform_device *pdev)
+static int hisi_femac_drv_resume(struct platform_device *pdev)
 {
 	struct net_device *ndev = platform_get_drvdata(pdev);
 	struct hisi_femac_priv *priv = netdev_priv(ndev);
diff --git a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c
index 275618bb4646..e69a6bed31a9 100644
--- a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c
+++ b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c
@@ -750,7 +750,7 @@ static const struct net_device_ops hix5hd2_netdev_ops = {
 	.ndo_set_mac_address	= hix5hd2_net_set_mac_address,
 };
 
-static struct ethtool_ops hix5hd2_ethtools_ops = {
+static const struct ethtool_ops hix5hd2_ethtools_ops = {
 	.get_link		= ethtool_op_get_link,
 	.get_link_ksettings     = phy_ethtool_get_link_ksettings,
 	.set_link_ksettings     = phy_ethtool_set_link_ksettings,
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c
index 1235c7f2564b..1e1eb92998fb 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c
@@ -17,7 +17,7 @@ static const struct mac_stats_string g_gmac_stats_string[] = {
 	{"gmac_rx_octets_total_ok", MAC_STATS_FIELD_OFF(rx_good_bytes)},
 	{"gmac_rx_octets_bad", MAC_STATS_FIELD_OFF(rx_bad_bytes)},
 	{"gmac_rx_uc_pkts", MAC_STATS_FIELD_OFF(rx_uc_pkts)},
-	{"gamc_rx_mc_pkts", MAC_STATS_FIELD_OFF(rx_mc_pkts)},
+	{"gmac_rx_mc_pkts", MAC_STATS_FIELD_OFF(rx_mc_pkts)},
 	{"gmac_rx_bc_pkts", MAC_STATS_FIELD_OFF(rx_bc_pkts)},
 	{"gmac_rx_pkts_64octets", MAC_STATS_FIELD_OFF(rx_64bytes)},
 	{"gmac_rx_pkts_65to127", MAC_STATS_FIELD_OFF(rx_65to127)},
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
index 5c8afe1a5ccb..22e141005cd9 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
@@ -126,7 +126,7 @@ void hns_mac_adjust_link(struct hns_mac_cb *mac_cb, int speed, int duplex)
 			(enum mac_speed)speed, duplex);
 		if (ret) {
 			dev_err(mac_cb->dev,
-				"adjust_link failed,%s mac%d ret = %#x!\n",
+				"adjust_link failed, %s mac%d ret = %#x!\n",
 				mac_cb->dsaf_dev->ae_dev.name,
 				mac_cb->mac_id, ret);
 			return;
@@ -149,7 +149,7 @@ static int hns_mac_get_inner_port_num(struct hns_mac_cb *mac_cb,
 	if (mac_cb->dsaf_dev->dsaf_mode <= DSAF_MODE_ENABLE) {
 		if (mac_cb->mac_id != DSAF_MAX_PORT_NUM) {
 			dev_err(mac_cb->dev,
-				"input invalid,%s mac%d vmid%d !\n",
+				"input invalid, %s mac%d vmid%d !\n",
 				mac_cb->dsaf_dev->ae_dev.name,
 				mac_cb->mac_id, vmid);
 			return -EINVAL;
@@ -157,19 +157,19 @@ static int hns_mac_get_inner_port_num(struct hns_mac_cb *mac_cb,
 	} else if (mac_cb->dsaf_dev->dsaf_mode < DSAF_MODE_MAX) {
 		if (mac_cb->mac_id >= DSAF_MAX_PORT_NUM) {
 			dev_err(mac_cb->dev,
-				"input invalid,%s mac%d vmid%d!\n",
+				"input invalid, %s mac%d vmid%d!\n",
 				mac_cb->dsaf_dev->ae_dev.name,
 				mac_cb->mac_id, vmid);
 			return -EINVAL;
 		}
 	} else {
-		dev_err(mac_cb->dev, "dsaf mode invalid,%s mac%d!\n",
+		dev_err(mac_cb->dev, "dsaf mode invalid, %s mac%d!\n",
 			mac_cb->dsaf_dev->ae_dev.name, mac_cb->mac_id);
 		return -EINVAL;
 	}
 
 	if (vmid >= mac_cb->dsaf_dev->rcb_common[0]->max_vfn) {
-		dev_err(mac_cb->dev, "input invalid,%s mac%d vmid%d !\n",
+		dev_err(mac_cb->dev, "input invalid, %s mac%d vmid%d !\n",
 			mac_cb->dsaf_dev->ae_dev.name, mac_cb->mac_id, vmid);
 		return -EINVAL;
 	}
@@ -196,7 +196,7 @@ static int hns_mac_get_inner_port_num(struct hns_mac_cb *mac_cb,
 		tmp_port = vmid;
 		break;
 	default:
-		dev_err(mac_cb->dev, "dsaf mode invalid,%s mac%d!\n",
+		dev_err(mac_cb->dev, "dsaf mode invalid, %s mac%d!\n",
 			mac_cb->dsaf_dev->ae_dev.name, mac_cb->mac_id);
 		return -EINVAL;
 	}
@@ -275,7 +275,7 @@ int hns_mac_set_multi(struct hns_mac_cb *mac_cb,
 			ret = hns_dsaf_add_mac_mc_port(dsaf_dev, &mac_entry);
 		if (ret) {
 			dev_err(dsaf_dev->dev,
-				"set mac mc port failed,%s mac%d ret = %#x!\n",
+				"set mac mc port failed, %s mac%d ret = %#x!\n",
 				mac_cb->dsaf_dev->ae_dev.name,
 				mac_cb->mac_id, ret);
 			return ret;
@@ -305,7 +305,7 @@ int hns_mac_del_mac(struct hns_mac_cb *mac_cb, u32 vfn, char *mac)
 		old_mac = &mac_cb->addr_entry_idx[vfn];
 	} else {
 		dev_err(mac_cb->dev,
-			"vf queue is too large,%s mac%d queue = %#x!\n",
+			"vf queue is too large, %s mac%d queue = %#x!\n",
 			mac_cb->dsaf_dev->ae_dev.name, mac_cb->mac_id, vfn);
 		return -EINVAL;
 	}
@@ -547,7 +547,7 @@ int hns_mac_set_autoneg(struct hns_mac_cb *mac_cb, u8 enable)
 	struct mac_driver *mac_ctrl_drv = hns_mac_get_drv(mac_cb);
 
 	if (mac_cb->phy_if == PHY_INTERFACE_MODE_XGMII && enable) {
-		dev_err(mac_cb->dev, "enable autoneg is not allowed!");
+		dev_err(mac_cb->dev, "enabling autoneg is not allowed!\n");
 		return -ENOTSUPP;
 	}
 
@@ -571,7 +571,7 @@ int hns_mac_set_pauseparam(struct hns_mac_cb *mac_cb, u32 rx_en, u32 tx_en)
 
 	if (mac_cb->mac_type == HNAE_PORT_DEBUG) {
 		if (is_ver1 && (tx_en || rx_en)) {
-			dev_err(mac_cb->dev, "macv1 cann't enable tx/rx_pause!");
+			dev_err(mac_cb->dev, "macv1 can't enable tx/rx_pause!\n");
 			return -EINVAL;
 		}
 	}
@@ -644,21 +644,6 @@ hns_mac_phy_parse_addr(struct device *dev, struct fwnode_handle *fwnode)
 	return addr;
 }
 
-static int hns_mac_phydev_match(struct device *dev, void *fwnode)
-{
-	return dev->fwnode == fwnode;
-}
-
-static struct
-platform_device *hns_mac_find_platform_device(struct fwnode_handle *fwnode)
-{
-	struct device *dev;
-
-	dev = bus_find_device(&platform_bus_type, NULL,
-			      fwnode, hns_mac_phydev_match);
-	return dev ? to_platform_device(dev) : NULL;
-}
-
 static int
 hns_mac_register_phydev(struct mii_bus *mdio, struct hns_mac_cb *mac_cb,
 			u32 addr)
@@ -684,8 +669,7 @@ hns_mac_register_phydev(struct mii_bus *mdio, struct hns_mac_cb *mac_cb,
 	if (!phy || IS_ERR(phy))
 		return -EIO;
 
-	if (mdio->irq)
-		phy->irq = mdio->irq[addr];
+	phy->irq = mdio->irq[addr];
 
 	/* All data is now stored in the phy struct;
 	 * register it
@@ -725,7 +709,7 @@ static void hns_mac_register_phy(struct hns_mac_cb *mac_cb)
 		return;
 
 	/* dev address in adev */
-	pdev = hns_mac_find_platform_device(acpi_fwnode_handle(args.adev));
+	pdev = hns_dsaf_find_platform_device(acpi_fwnode_handle(args.adev));
 	mii_bus = platform_get_drvdata(pdev);
 	rc = hns_mac_register_phydev(mii_bus, mac_cb, addr);
 	if (!rc)
@@ -942,7 +926,7 @@ int hns_mac_get_cfg(struct dsaf_device *dsaf_dev, struct hns_mac_cb *mac_cb)
 	ret = hns_mac_get_mode(mac_cb->phy_if);
 	if (ret < 0) {
 		dev_err(dsaf_dev->dev,
-			"hns_mac_get_mode failed,mac%d ret = %#x!\n",
+			"hns_mac_get_mode failed, mac%d ret = %#x!\n",
 			mac_cb->mac_id, ret);
 		return ret;
 	}
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c
index afb5daa3721d..8e5b3f51b47b 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c
@@ -18,6 +18,7 @@
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
+#include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/vmalloc.h>
 
@@ -115,10 +116,8 @@ int hns_dsaf_get_cfg(struct dsaf_device *dsaf_dev)
 
 			dsaf_dev->sc_base = devm_ioremap_resource(&pdev->dev,
 								  res);
-			if (IS_ERR(dsaf_dev->sc_base)) {
-				dev_err(dsaf_dev->dev, "subctrl can not map!\n");
+			if (IS_ERR(dsaf_dev->sc_base))
 				return PTR_ERR(dsaf_dev->sc_base);
-			}
 
 			res = platform_get_resource(pdev, IORESOURCE_MEM,
 						    res_idx++);
@@ -129,10 +128,8 @@ int hns_dsaf_get_cfg(struct dsaf_device *dsaf_dev)
 
 			dsaf_dev->sds_base = devm_ioremap_resource(&pdev->dev,
 								   res);
-			if (IS_ERR(dsaf_dev->sds_base)) {
-				dev_err(dsaf_dev->dev, "serdes-ctrl can not map!\n");
+			if (IS_ERR(dsaf_dev->sds_base))
 				return PTR_ERR(dsaf_dev->sds_base);
-			}
 		} else {
 			dsaf_dev->sub_ctrl = syscon;
 		}
@@ -147,10 +144,8 @@ int hns_dsaf_get_cfg(struct dsaf_device *dsaf_dev)
 		}
 	}
 	dsaf_dev->ppe_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(dsaf_dev->ppe_base)) {
-		dev_err(dsaf_dev->dev, "ppe-base resource can not map!\n");
+	if (IS_ERR(dsaf_dev->ppe_base))
 		return PTR_ERR(dsaf_dev->ppe_base);
-	}
 	dsaf_dev->ppe_paddr = res->start;
 
 	if (!HNS_DSAF_IS_DEBUG(dsaf_dev)) {
@@ -166,10 +161,8 @@ int hns_dsaf_get_cfg(struct dsaf_device *dsaf_dev)
 			}
 		}
 		dsaf_dev->io_base = devm_ioremap_resource(&pdev->dev, res);
-		if (IS_ERR(dsaf_dev->io_base)) {
-			dev_err(dsaf_dev->dev, "dsaf-base resource can not map!\n");
+		if (IS_ERR(dsaf_dev->io_base))
 			return PTR_ERR(dsaf_dev->io_base);
-		}
 	}
 
 	ret = device_property_read_u32(dsaf_dev->dev, "desc-num", &desc_num);
@@ -2781,6 +2774,110 @@ static struct platform_driver g_dsaf_driver = {
 
 module_platform_driver(g_dsaf_driver);
 
+/**
+ * hns_dsaf_roce_reset - reset dsaf and roce
+ * @dsaf_fwnode: Pointer to framework node for the dasf
+ * @enable: false - request reset , true - drop reset
+ * retuen 0 - success , negative -fail
+ */
+int hns_dsaf_roce_reset(struct fwnode_handle *dsaf_fwnode, bool dereset)
+{
+	struct dsaf_device *dsaf_dev;
+	struct platform_device *pdev;
+	u32 mp;
+	u32 sl;
+	u32 credit;
+	int i;
+	const u32 port_map[DSAF_ROCE_CREDIT_CHN][DSAF_ROCE_CHAN_MODE_NUM] = {
+		{DSAF_ROCE_PORT_0, DSAF_ROCE_PORT_0, DSAF_ROCE_PORT_0},
+		{DSAF_ROCE_PORT_1, DSAF_ROCE_PORT_0, DSAF_ROCE_PORT_0},
+		{DSAF_ROCE_PORT_2, DSAF_ROCE_PORT_1, DSAF_ROCE_PORT_0},
+		{DSAF_ROCE_PORT_3, DSAF_ROCE_PORT_1, DSAF_ROCE_PORT_0},
+		{DSAF_ROCE_PORT_4, DSAF_ROCE_PORT_2, DSAF_ROCE_PORT_1},
+		{DSAF_ROCE_PORT_4, DSAF_ROCE_PORT_2, DSAF_ROCE_PORT_1},
+		{DSAF_ROCE_PORT_5, DSAF_ROCE_PORT_3, DSAF_ROCE_PORT_1},
+		{DSAF_ROCE_PORT_5, DSAF_ROCE_PORT_3, DSAF_ROCE_PORT_1},
+	};
+	const u32 sl_map[DSAF_ROCE_CREDIT_CHN][DSAF_ROCE_CHAN_MODE_NUM] = {
+		{DSAF_ROCE_SL_0, DSAF_ROCE_SL_0, DSAF_ROCE_SL_0},
+		{DSAF_ROCE_SL_0, DSAF_ROCE_SL_1, DSAF_ROCE_SL_1},
+		{DSAF_ROCE_SL_0, DSAF_ROCE_SL_0, DSAF_ROCE_SL_2},
+		{DSAF_ROCE_SL_0, DSAF_ROCE_SL_1, DSAF_ROCE_SL_3},
+		{DSAF_ROCE_SL_0, DSAF_ROCE_SL_0, DSAF_ROCE_SL_0},
+		{DSAF_ROCE_SL_1, DSAF_ROCE_SL_1, DSAF_ROCE_SL_1},
+		{DSAF_ROCE_SL_0, DSAF_ROCE_SL_0, DSAF_ROCE_SL_2},
+		{DSAF_ROCE_SL_1, DSAF_ROCE_SL_1, DSAF_ROCE_SL_3},
+	};
+
+	/* find the platform device corresponding to fwnode */
+	if (is_of_node(dsaf_fwnode)) {
+		pdev = of_find_device_by_node(to_of_node(dsaf_fwnode));
+	} else if (is_acpi_device_node(dsaf_fwnode)) {
+		pdev = hns_dsaf_find_platform_device(dsaf_fwnode);
+	} else {
+		pr_err("fwnode is neither OF or ACPI type\n");
+		return -EINVAL;
+	}
+
+	/* check if we were a success in fetching pdev */
+	if (!pdev) {
+		pr_err("couldn't find platform device for node\n");
+		return -ENODEV;
+	}
+
+	/* retrieve the dsaf_device from the driver data */
+	dsaf_dev = dev_get_drvdata(&pdev->dev);
+	if (!dsaf_dev) {
+		dev_err(&pdev->dev, "dsaf_dev is NULL\n");
+		return -ENODEV;
+	}
+
+	/* now, make sure we are running on compatible SoC */
+	if (AE_IS_VER1(dsaf_dev->dsaf_ver)) {
+		dev_err(dsaf_dev->dev, "%s v1 chip doesn't support RoCE!\n",
+			dsaf_dev->ae_dev.name);
+		return -ENODEV;
+	}
+
+	/* do reset or de-reset according to the flag */
+	if (!dereset) {
+		/* reset rocee-channels in dsaf and rocee */
+		dsaf_dev->misc_op->hns_dsaf_srst_chns(dsaf_dev, DSAF_CHNS_MASK,
+						      false);
+		dsaf_dev->misc_op->hns_dsaf_roce_srst(dsaf_dev, false);
+	} else {
+		/* configure dsaf tx roce correspond to port map and sl map */
+		mp = dsaf_read_dev(dsaf_dev, DSAF_ROCE_PORT_MAP_REG);
+		for (i = 0; i < DSAF_ROCE_CREDIT_CHN; i++)
+			dsaf_set_field(mp, 7 << i * 3, i * 3,
+				       port_map[i][DSAF_ROCE_6PORT_MODE]);
+		dsaf_set_field(mp, 3 << i * 3, i * 3, 0);
+		dsaf_write_dev(dsaf_dev, DSAF_ROCE_PORT_MAP_REG, mp);
+
+		sl = dsaf_read_dev(dsaf_dev, DSAF_ROCE_SL_MAP_REG);
+		for (i = 0; i < DSAF_ROCE_CREDIT_CHN; i++)
+			dsaf_set_field(sl, 3 << i * 2, i * 2,
+				       sl_map[i][DSAF_ROCE_6PORT_MODE]);
+		dsaf_write_dev(dsaf_dev, DSAF_ROCE_SL_MAP_REG, sl);
+
+		/* de-reset rocee-channels in dsaf and rocee */
+		dsaf_dev->misc_op->hns_dsaf_srst_chns(dsaf_dev, DSAF_CHNS_MASK,
+						      true);
+		msleep(SRST_TIME_INTERVAL);
+		dsaf_dev->misc_op->hns_dsaf_roce_srst(dsaf_dev, true);
+
+		/* enable dsaf channel rocee credit */
+		credit = dsaf_read_dev(dsaf_dev, DSAF_SBM_ROCEE_CFG_REG_REG);
+		dsaf_set_bit(credit, DSAF_SBM_ROCEE_CFG_CRD_EN_B, 0);
+		dsaf_write_dev(dsaf_dev, DSAF_SBM_ROCEE_CFG_REG_REG, credit);
+
+		dsaf_set_bit(credit, DSAF_SBM_ROCEE_CFG_CRD_EN_B, 1);
+		dsaf_write_dev(dsaf_dev, DSAF_SBM_ROCEE_CFG_REG_REG, credit);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(hns_dsaf_roce_reset);
+
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
 MODULE_DESCRIPTION("HNS DSAF driver");
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.h
index 1daf018d9071..35df187e66f1 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.h
@@ -43,6 +43,32 @@ struct hns_mac_cb;
 #define DSAF_PRIO_NR	8
 #define DSAF_REG_PER_ZONE	3
 
+#define DSAF_ROCE_CREDIT_CHN	8
+#define DSAF_ROCE_CHAN_MODE	3
+
+enum dsaf_roce_port_mode {
+	DSAF_ROCE_6PORT_MODE,
+	DSAF_ROCE_4PORT_MODE,
+	DSAF_ROCE_2PORT_MODE,
+	DSAF_ROCE_CHAN_MODE_NUM,
+};
+
+enum dsaf_roce_port_num {
+	DSAF_ROCE_PORT_0,
+	DSAF_ROCE_PORT_1,
+	DSAF_ROCE_PORT_2,
+	DSAF_ROCE_PORT_3,
+	DSAF_ROCE_PORT_4,
+	DSAF_ROCE_PORT_5,
+};
+
+enum dsaf_roce_qos_sl {
+	DSAF_ROCE_SL_0,
+	DSAF_ROCE_SL_1,
+	DSAF_ROCE_SL_2,
+	DSAF_ROCE_SL_3,
+};
+
 #define DSAF_STATS_READ(p, offset) (*((u64 *)((u8 *)(p) + (offset))))
 #define HNS_DSAF_IS_DEBUG(dev) (dev->dsaf_mode == DSAF_MODE_DISABLE_SP)
 
@@ -279,7 +305,7 @@ struct dsaf_misc_op {
 	void (*cpld_reset_led)(struct hns_mac_cb *mac_cb);
 	int (*cpld_set_led_id)(struct hns_mac_cb *mac_cb,
 			       enum hnae_led_state status);
-	/* reset seris function, it will be reset if the dereseet is 0 */
+	/* reset series function, it will be reset if the dereset is 0 */
 	void (*dsaf_reset)(struct dsaf_device *dsaf_dev, bool dereset);
 	void (*xge_srst)(struct dsaf_device *dsaf_dev, u32 port, bool dereset);
 	void (*xge_core_srst)(struct dsaf_device *dsaf_dev, u32 port,
@@ -287,6 +313,9 @@ struct dsaf_misc_op {
 	void (*ge_srst)(struct dsaf_device *dsaf_dev, u32 port, bool dereset);
 	void (*ppe_srst)(struct dsaf_device *dsaf_dev, u32 port, bool dereset);
 	void (*ppe_comm_srst)(struct dsaf_device *dsaf_dev, bool dereset);
+	void (*hns_dsaf_srst_chns)(struct dsaf_device *dsaf_dev, u32 msk,
+				   bool dereset);
+	void (*hns_dsaf_roce_srst)(struct dsaf_device *dsaf_dev, bool dereset);
 
 	phy_interface_t (*get_phy_if)(struct hns_mac_cb *mac_cb);
 	int (*get_sfp_prsnt)(struct hns_mac_cb *mac_cb, int *sfp_prsnt);
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
index 611b67b6f450..67accce1d33d 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
@@ -26,6 +26,8 @@ enum _dsm_rst_type {
 	HNS_XGE_CORE_RESET_FUNC = 0x3,
 	HNS_XGE_RESET_FUNC      = 0x4,
 	HNS_GE_RESET_FUNC       = 0x5,
+	HNS_DSAF_CHN_RESET_FUNC = 0x6,
+	HNS_ROCE_RESET_FUNC     = 0x7,
 };
 
 const u8 hns_dsaf_acpi_dsm_uuid[] = {
@@ -231,6 +233,66 @@ static void hns_dsaf_xge_core_srst_by_port(struct dsaf_device *dsaf_dev,
 	dsaf_write_sub(dsaf_dev, reg_addr, reg_val);
 }
 
+/**
+ * hns_dsaf_srst_chns - reset dsaf channels
+ * @dsaf_dev: dsaf device struct pointer
+ * @msk: xbar channels mask value:
+ * bit0-5 for xge0-5
+ * bit6-11 for ppe0-5
+ * bit12-17 for roce0-5
+ * bit18-19 for com/dfx
+ * @enable: false - request reset , true - drop reset
+ */
+void hns_dsaf_srst_chns(struct dsaf_device *dsaf_dev, u32 msk, bool dereset)
+{
+	u32 reg_addr;
+
+	if (!dereset)
+		reg_addr = DSAF_SUB_SC_DSAF_RESET_REQ_REG;
+	else
+		reg_addr = DSAF_SUB_SC_DSAF_RESET_DREQ_REG;
+
+	dsaf_write_sub(dsaf_dev, reg_addr, msk);
+}
+
+/**
+ * hns_dsaf_srst_chns - reset dsaf channels
+ * @dsaf_dev: dsaf device struct pointer
+ * @msk: xbar channels mask value:
+ * bit0-5 for xge0-5
+ * bit6-11 for ppe0-5
+ * bit12-17 for roce0-5
+ * bit18-19 for com/dfx
+ * @enable: false - request reset , true - drop reset
+ */
+void
+hns_dsaf_srst_chns_acpi(struct dsaf_device *dsaf_dev, u32 msk, bool dereset)
+{
+	hns_dsaf_acpi_srst_by_port(dsaf_dev, HNS_OP_RESET_FUNC,
+				   HNS_DSAF_CHN_RESET_FUNC,
+				   msk, dereset);
+}
+
+void hns_dsaf_roce_srst(struct dsaf_device *dsaf_dev, bool dereset)
+{
+	if (!dereset) {
+		dsaf_write_sub(dsaf_dev, DSAF_SUB_SC_ROCEE_RESET_REQ_REG, 1);
+	} else {
+		dsaf_write_sub(dsaf_dev,
+			       DSAF_SUB_SC_ROCEE_CLK_DIS_REG, 1);
+		dsaf_write_sub(dsaf_dev,
+			       DSAF_SUB_SC_ROCEE_RESET_DREQ_REG, 1);
+		msleep(20);
+		dsaf_write_sub(dsaf_dev, DSAF_SUB_SC_ROCEE_CLK_EN_REG, 1);
+	}
+}
+
+void hns_dsaf_roce_srst_acpi(struct dsaf_device *dsaf_dev, bool dereset)
+{
+	hns_dsaf_acpi_srst_by_port(dsaf_dev, HNS_OP_RESET_FUNC,
+				   HNS_ROCE_RESET_FUNC, 0, dereset);
+}
+
 static void
 hns_dsaf_xge_core_srst_by_port_acpi(struct dsaf_device *dsaf_dev,
 				    u32 port, bool dereset)
@@ -539,6 +601,8 @@ struct dsaf_misc_op *hns_misc_op_get(struct dsaf_device *dsaf_dev)
 		misc_op->ge_srst = hns_dsaf_ge_srst_by_port;
 		misc_op->ppe_srst = hns_ppe_srst_by_port;
 		misc_op->ppe_comm_srst = hns_ppe_com_srst;
+		misc_op->hns_dsaf_srst_chns = hns_dsaf_srst_chns;
+		misc_op->hns_dsaf_roce_srst = hns_dsaf_roce_srst;
 
 		misc_op->get_phy_if = hns_mac_get_phy_if;
 		misc_op->get_sfp_prsnt = hns_mac_get_sfp_prsnt;
@@ -555,6 +619,8 @@ struct dsaf_misc_op *hns_misc_op_get(struct dsaf_device *dsaf_dev)
 		misc_op->ge_srst = hns_dsaf_ge_srst_by_port_acpi;
 		misc_op->ppe_srst = hns_ppe_srst_by_port_acpi;
 		misc_op->ppe_comm_srst = hns_ppe_com_srst;
+		misc_op->hns_dsaf_srst_chns = hns_dsaf_srst_chns_acpi;
+		misc_op->hns_dsaf_roce_srst = hns_dsaf_roce_srst_acpi;
 
 		misc_op->get_phy_if = hns_mac_get_phy_if_acpi;
 		misc_op->get_sfp_prsnt = hns_mac_get_sfp_prsnt;
@@ -567,3 +633,18 @@ struct dsaf_misc_op *hns_misc_op_get(struct dsaf_device *dsaf_dev)
 
 	return (void *)misc_op;
 }
+
+static int hns_dsaf_dev_match(struct device *dev, void *fwnode)
+{
+	return dev->fwnode == fwnode;
+}
+
+struct
+platform_device *hns_dsaf_find_platform_device(struct fwnode_handle *fwnode)
+{
+	struct device *dev;
+
+	dev = bus_find_device(&platform_bus_type, NULL,
+			      fwnode, hns_dsaf_dev_match);
+	return dev ? to_platform_device(dev) : NULL;
+}
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.h
index f06bb03d47a6..310e80261366 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.h
@@ -34,5 +34,6 @@
 #define DSAF_LED_ANCHOR_B 5
 
 struct dsaf_misc_op *hns_misc_op_get(struct dsaf_device *dsaf_dev);
-
+struct
+platform_device *hns_dsaf_find_platform_device(struct fwnode_handle *fwnode);
 #endif
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
index ff8b6a468b24..6ea872287307 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
@@ -328,9 +328,10 @@ static void hns_ppe_init_hw(struct hns_ppe_cb *ppe_cb)
 static void hns_ppe_uninit_hw(struct hns_ppe_cb *ppe_cb)
 {
 	u32 port;
-	struct dsaf_device *dsaf_dev = ppe_cb->ppe_common_cb->dsaf_dev;
 
 	if (ppe_cb->ppe_common_cb) {
+		struct dsaf_device *dsaf_dev = ppe_cb->ppe_common_cb->dsaf_dev;
+
 		port = ppe_cb->index;
 		dsaf_dev->misc_op->ppe_srst(dsaf_dev, port, 0);
 	}
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
index 235f74444b1d..4b8b803822d1 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
@@ -77,6 +77,12 @@
 #define DSAF_SUB_SC_PPE_RESET_DREQ_REG			0xA4C
 #define DSAF_SUB_SC_RCB_PPE_COM_RESET_REQ_REG		0xA88
 #define DSAF_SUB_SC_RCB_PPE_COM_RESET_DREQ_REG		0xA8C
+#define DSAF_SUB_SC_DSAF_RESET_REQ_REG			0xAA8
+#define DSAF_SUB_SC_DSAF_RESET_DREQ_REG			0xAAC
+#define DSAF_SUB_SC_ROCEE_RESET_REQ_REG			0xA50
+#define DSAF_SUB_SC_ROCEE_RESET_DREQ_REG		0xA54
+#define DSAF_SUB_SC_ROCEE_CLK_DIS_REG			0x32C
+#define DSAF_SUB_SC_ROCEE_CLK_EN_REG			0x328
 #define DSAF_SUB_SC_LIGHT_MODULE_DETECT_EN_REG		0x2060
 #define DSAF_SUB_SC_TCAM_MBIST_EN_REG			0x2300
 #define DSAF_SUB_SC_DSAF_CLK_ST_REG			0x5300
@@ -133,6 +139,8 @@
 #define DSAF_ROCEE_INT_STS_0_REG	0x200
 #define DSAFV2_SERDES_LBK_0_REG         0x220
 #define DSAF_PAUSE_CFG_REG		0x240
+#define DSAF_ROCE_PORT_MAP_REG		0x2A0
+#define DSAF_ROCE_SL_MAP_REG		0x2A4
 #define DSAF_PPE_QID_CFG_0_REG		0x300
 #define DSAF_SW_PORT_TYPE_0_REG		0x320
 #define DSAF_STP_PORT_TYPE_0_REG	0x340
@@ -178,6 +186,7 @@
 #define DSAF_SBM_BP_CFG_2_XGE_REG_0_REG		0x200C
 #define DSAF_SBM_BP_CFG_2_PPE_REG_0_REG		0x230C
 #define DSAF_SBM_BP_CFG_2_ROCEE_REG_0_REG	0x260C
+#define DSAF_SBM_ROCEE_CFG_REG_REG		0x2380
 #define DSAFV2_SBM_BP_CFG_2_ROCEE_REG_0_REG	0x238C
 #define DSAF_SBM_FREE_CNT_0_0_REG		0x2010
 #define DSAF_SBM_FREE_CNT_1_0_REG		0x2014
@@ -796,6 +805,9 @@
 #define DSAFV2_SBM_CFG4_RESET_BUF_NUM_NO_PFC_S 9
 #define DSAFV2_SBM_CFG4_RESET_BUF_NUM_NO_PFC_M (((1ULL << 9) - 1) << 9)
 
+#define DSAF_CHNS_MASK			0x3f000
+#define DSAF_SBM_ROCEE_CFG_CRD_EN_B	2
+#define SRST_TIME_INTERVAL		20
 #define DSAFV2_SBM_CFG2_ROCEE_SET_BUF_NUM_S 0
 #define DSAFV2_SBM_CFG2_ROCEE_SET_BUF_NUM_M (((1ULL << 8) - 1) << 0)
 #define DSAFV2_SBM_CFG2_ROCEE_RESET_BUF_NUM_S 8
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.c b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
index d7e1f8c7ae92..059aaeda46b1 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
@@ -994,10 +994,10 @@ static void hns_nic_adjust_link(struct net_device *ndev)
 	struct hnae_handle *h = priv->ae_handle;
 	int state = 1;
 
-	if (priv->phy) {
+	if (ndev->phydev) {
 		h->dev->ops->adjust_link(h, ndev->phydev->speed,
 					 ndev->phydev->duplex);
-		state = priv->phy->link;
+		state = ndev->phydev->link;
 	}
 	state = state && h->dev->ops->get_status(h);
 
@@ -1022,7 +1022,6 @@ static void hns_nic_adjust_link(struct net_device *ndev)
  */
 int hns_nic_init_phy(struct net_device *ndev, struct hnae_handle *h)
 {
-	struct hns_nic_priv *priv = netdev_priv(ndev);
 	struct phy_device *phy_dev = h->phy_dev;
 	int ret;
 
@@ -1046,8 +1045,6 @@ int hns_nic_init_phy(struct net_device *ndev, struct hnae_handle *h)
 	if (h->phy_if == PHY_INTERFACE_MODE_XGMII)
 		phy_dev->autoneg = false;
 
-	priv->phy = phy_dev;
-
 	return 0;
 }
 
@@ -1224,8 +1221,8 @@ static int hns_nic_net_up(struct net_device *ndev)
 	if (ret)
 		goto out_start_err;
 
-	if (priv->phy)
-		phy_start(priv->phy);
+	if (ndev->phydev)
+		phy_start(ndev->phydev);
 
 	clear_bit(NIC_STATE_DOWN, &priv->state);
 	(void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ);
@@ -1259,8 +1256,8 @@ static void hns_nic_net_down(struct net_device *ndev)
 	netif_tx_disable(ndev);
 	priv->link = 0;
 
-	if (priv->phy)
-		phy_stop(priv->phy);
+	if (ndev->phydev)
+		phy_stop(ndev->phydev);
 
 	ops = priv->ae_handle->dev->ops;
 
@@ -1359,8 +1356,7 @@ static void hns_nic_net_timeout(struct net_device *ndev)
 static int hns_nic_do_ioctl(struct net_device *netdev, struct ifreq *ifr,
 			    int cmd)
 {
-	struct hns_nic_priv *priv = netdev_priv(netdev);
-	struct phy_device *phy_dev = priv->phy;
+	struct phy_device *phy_dev = netdev->phydev;
 
 	if (!netif_running(netdev))
 		return -EINVAL;
@@ -2017,9 +2013,8 @@ static int hns_nic_dev_remove(struct platform_device *pdev)
 		hns_nic_uninit_ring_data(priv);
 	priv->ring_data = NULL;
 
-	if (priv->phy)
-		phy_disconnect(priv->phy);
-	priv->phy = NULL;
+	if (ndev->phydev)
+		phy_disconnect(ndev->phydev);
 
 	if (!IS_ERR_OR_NULL(priv->ae_handle))
 		hnae_put_handle(priv->ae_handle);
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.h b/drivers/net/ethernet/hisilicon/hns/hns_enet.h
index 44bb3015eed3..5b412de350aa 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_enet.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.h
@@ -59,7 +59,6 @@ struct hns_nic_priv {
 	u32 port_id;
 	int phy_mode;
 	int phy_led_val;
-	struct phy_device *phy;
 	struct net_device *netdev;
 	struct device *dev;
 	struct hnae_handle *ae_handle;
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
index ab33487a5321..47e59bbfd061 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
@@ -48,9 +48,9 @@ static u32 hns_nic_get_link(struct net_device *net_dev)
 
 	h = priv->ae_handle;
 
-	if (priv->phy) {
-		if (!genphy_read_status(priv->phy))
-			link_stat = priv->phy->link;
+	if (net_dev->phydev) {
+		if (!genphy_read_status(net_dev->phydev))
+			link_stat = net_dev->phydev->link;
 		else
 			link_stat = 0;
 	}
@@ -64,15 +64,14 @@ static u32 hns_nic_get_link(struct net_device *net_dev)
 }
 
 static void hns_get_mdix_mode(struct net_device *net_dev,
-			      struct ethtool_cmd *cmd)
+			      struct ethtool_link_ksettings *cmd)
 {
 	int mdix_ctrl, mdix, retval, is_resolved;
-	struct hns_nic_priv *priv = netdev_priv(net_dev);
-	struct phy_device *phy_dev = priv->phy;
+	struct phy_device *phy_dev = net_dev->phydev;
 
 	if (!phy_dev || !phy_dev->mdio.bus) {
-		cmd->eth_tp_mdix_ctrl = ETH_TP_MDI_INVALID;
-		cmd->eth_tp_mdix = ETH_TP_MDI_INVALID;
+		cmd->base.eth_tp_mdix_ctrl = ETH_TP_MDI_INVALID;
+		cmd->base.eth_tp_mdix = ETH_TP_MDI_INVALID;
 		return;
 	}
 
@@ -89,35 +88,35 @@ static void hns_get_mdix_mode(struct net_device *net_dev,
 
 	switch (mdix_ctrl) {
 	case 0x0:
-		cmd->eth_tp_mdix_ctrl = ETH_TP_MDI;
+		cmd->base.eth_tp_mdix_ctrl = ETH_TP_MDI;
 		break;
 	case 0x1:
-		cmd->eth_tp_mdix_ctrl = ETH_TP_MDI_X;
+		cmd->base.eth_tp_mdix_ctrl = ETH_TP_MDI_X;
 		break;
 	case 0x3:
-		cmd->eth_tp_mdix_ctrl = ETH_TP_MDI_AUTO;
+		cmd->base.eth_tp_mdix_ctrl = ETH_TP_MDI_AUTO;
 		break;
 	default:
-		cmd->eth_tp_mdix_ctrl = ETH_TP_MDI_INVALID;
+		cmd->base.eth_tp_mdix_ctrl = ETH_TP_MDI_INVALID;
 		break;
 	}
 
 	if (!is_resolved)
-		cmd->eth_tp_mdix = ETH_TP_MDI_INVALID;
+		cmd->base.eth_tp_mdix = ETH_TP_MDI_INVALID;
 	else if (mdix)
-		cmd->eth_tp_mdix = ETH_TP_MDI_X;
+		cmd->base.eth_tp_mdix = ETH_TP_MDI_X;
 	else
-		cmd->eth_tp_mdix = ETH_TP_MDI;
+		cmd->base.eth_tp_mdix = ETH_TP_MDI;
 }
 
 /**
- *hns_nic_get_settings - implement ethtool get settings
+ *hns_nic_get_link_ksettings - implement ethtool get link ksettings
  *@net_dev: net_device
- *@cmd: ethtool_cmd
+ *@cmd: ethtool_link_ksettings
  *retuen 0 - success , negative --fail
  */
-static int hns_nic_get_settings(struct net_device *net_dev,
-				struct ethtool_cmd *cmd)
+static int hns_nic_get_link_ksettings(struct net_device *net_dev,
+				      struct ethtool_link_ksettings *cmd)
 {
 	struct hns_nic_priv *priv = netdev_priv(net_dev);
 	struct hnae_handle *h;
@@ -125,6 +124,7 @@ static int hns_nic_get_settings(struct net_device *net_dev,
 	int ret;
 	u8 duplex;
 	u16 speed;
+	u32 supported, advertising;
 
 	if (!priv || !priv->ae_handle)
 		return -ESRCH;
@@ -139,38 +139,43 @@ static int hns_nic_get_settings(struct net_device *net_dev,
 		return -EINVAL;
 	}
 
+	ethtool_convert_link_mode_to_legacy_u32(&supported,
+						cmd->link_modes.supported);
+	ethtool_convert_link_mode_to_legacy_u32(&advertising,
+						cmd->link_modes.advertising);
+
 	/* When there is no phy, autoneg is off. */
-	cmd->autoneg = false;
-	ethtool_cmd_speed_set(cmd, speed);
-	cmd->duplex = duplex;
+	cmd->base.autoneg = false;
+	cmd->base.cmd = speed;
+	cmd->base.duplex = duplex;
 
-	if (priv->phy)
-		(void)phy_ethtool_gset(priv->phy, cmd);
+	if (net_dev->phydev)
+		(void)phy_ethtool_ksettings_get(net_dev->phydev, cmd);
 
 	link_stat = hns_nic_get_link(net_dev);
 	if (!link_stat) {
-		ethtool_cmd_speed_set(cmd, (u32)SPEED_UNKNOWN);
-		cmd->duplex = DUPLEX_UNKNOWN;
+		cmd->base.speed = (u32)SPEED_UNKNOWN;
+		cmd->base.duplex = DUPLEX_UNKNOWN;
 	}
 
-	if (cmd->autoneg)
-		cmd->advertising |= ADVERTISED_Autoneg;
+	if (cmd->base.autoneg)
+		advertising |= ADVERTISED_Autoneg;
 
-	cmd->supported |= h->if_support;
+	supported |= h->if_support;
 	if (h->phy_if == PHY_INTERFACE_MODE_SGMII) {
-		cmd->supported |= SUPPORTED_TP;
-		cmd->advertising |= ADVERTISED_1000baseT_Full;
+		supported |= SUPPORTED_TP;
+		advertising |= ADVERTISED_1000baseT_Full;
 	} else if (h->phy_if == PHY_INTERFACE_MODE_XGMII) {
-		cmd->supported |= SUPPORTED_FIBRE;
-		cmd->advertising |= ADVERTISED_10000baseKR_Full;
+		supported |= SUPPORTED_FIBRE;
+		advertising |= ADVERTISED_10000baseKR_Full;
 	}
 
 	switch (h->media_type) {
 	case HNAE_MEDIA_TYPE_FIBER:
-		cmd->port = PORT_FIBRE;
+		cmd->base.port = PORT_FIBRE;
 		break;
 	case HNAE_MEDIA_TYPE_COPPER:
-		cmd->port = PORT_TP;
+		cmd->base.port = PORT_TP;
 		break;
 	case HNAE_MEDIA_TYPE_UNKNOWN:
 	default:
@@ -178,23 +183,27 @@ static int hns_nic_get_settings(struct net_device *net_dev,
 	}
 
 	if (!(AE_IS_VER1(priv->enet_ver) && h->port_type == HNAE_PORT_DEBUG))
-		cmd->supported |= SUPPORTED_Pause;
+		supported |= SUPPORTED_Pause;
+
+	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
+						supported);
+	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
+						advertising);
 
-	cmd->transceiver = XCVR_EXTERNAL;
-	cmd->mdio_support = (ETH_MDIO_SUPPORTS_C45 | ETH_MDIO_SUPPORTS_C22);
+	cmd->base.mdio_support = ETH_MDIO_SUPPORTS_C45 | ETH_MDIO_SUPPORTS_C22;
 	hns_get_mdix_mode(net_dev, cmd);
 
 	return 0;
 }
 
 /**
- *hns_nic_set_settings - implement ethtool set settings
+ *hns_nic_set_link_settings - implement ethtool set link ksettings
  *@net_dev: net_device
- *@cmd: ethtool_cmd
+ *@cmd: ethtool_link_ksettings
  *retuen 0 - success , negative --fail
  */
-static int hns_nic_set_settings(struct net_device *net_dev,
-				struct ethtool_cmd *cmd)
+static int hns_nic_set_link_ksettings(struct net_device *net_dev,
+				      const struct ethtool_link_ksettings *cmd)
 {
 	struct hns_nic_priv *priv = netdev_priv(net_dev);
 	struct hnae_handle *h;
@@ -208,24 +217,25 @@ static int hns_nic_set_settings(struct net_device *net_dev,
 		return -ENODEV;
 
 	h = priv->ae_handle;
-	speed = ethtool_cmd_speed(cmd);
+	speed = cmd->base.speed;
 
 	if (h->phy_if == PHY_INTERFACE_MODE_XGMII) {
-		if (cmd->autoneg == AUTONEG_ENABLE || speed != SPEED_10000 ||
-		    cmd->duplex != DUPLEX_FULL)
+		if (cmd->base.autoneg == AUTONEG_ENABLE ||
+		    speed != SPEED_10000 ||
+		    cmd->base.duplex != DUPLEX_FULL)
 			return -EINVAL;
 	} else if (h->phy_if == PHY_INTERFACE_MODE_SGMII) {
-		if (!priv->phy && cmd->autoneg == AUTONEG_ENABLE)
+		if (!net_dev->phydev && cmd->base.autoneg == AUTONEG_ENABLE)
 			return -EINVAL;
 
-		if (speed == SPEED_1000 && cmd->duplex == DUPLEX_HALF)
+		if (speed == SPEED_1000 && cmd->base.duplex == DUPLEX_HALF)
 			return -EINVAL;
-		if (priv->phy)
-			return phy_ethtool_sset(priv->phy, cmd);
+		if (net_dev->phydev)
+			return phy_ethtool_ksettings_set(net_dev->phydev, cmd);
 
 		if ((speed != SPEED_10 && speed != SPEED_100 &&
-		     speed != SPEED_1000) || (cmd->duplex != DUPLEX_HALF &&
-		     cmd->duplex != DUPLEX_FULL))
+		     speed != SPEED_1000) || (cmd->base.duplex != DUPLEX_HALF &&
+		     cmd->base.duplex != DUPLEX_FULL))
 			return -EINVAL;
 	} else {
 		netdev_err(net_dev, "Not supported!");
@@ -233,7 +243,7 @@ static int hns_nic_set_settings(struct net_device *net_dev,
 	}
 
 	if (h->dev->ops->adjust_link) {
-		h->dev->ops->adjust_link(h, (int)speed, cmd->duplex);
+		h->dev->ops->adjust_link(h, (int)speed, cmd->base.duplex);
 		return 0;
 	}
 
@@ -305,7 +315,7 @@ static int __lb_setup(struct net_device *ndev,
 {
 	int ret = 0;
 	struct hns_nic_priv *priv = netdev_priv(ndev);
-	struct phy_device *phy_dev = priv->phy;
+	struct phy_device *phy_dev = ndev->phydev;
 	struct hnae_handle *h = priv->ae_handle;
 
 	switch (loop) {
@@ -910,7 +920,7 @@ void hns_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
 		memcpy(buff, hns_nic_test_strs[MAC_INTERNALLOOP_SERDES],
 		       ETH_GSTRING_LEN);
 		buff += ETH_GSTRING_LEN;
-		if ((priv->phy) && (!priv->phy->is_c45))
+		if ((netdev->phydev) && (!netdev->phydev->is_c45))
 			memcpy(buff, hns_nic_test_strs[MAC_INTERNALLOOP_PHY],
 			       ETH_GSTRING_LEN);
 
@@ -996,7 +1006,7 @@ int hns_get_sset_count(struct net_device *netdev, int stringset)
 		if (priv->ae_handle->phy_if == PHY_INTERFACE_MODE_XGMII)
 			cnt--;
 
-		if ((!priv->phy) || (priv->phy->is_c45))
+		if ((!netdev->phydev) || (netdev->phydev->is_c45))
 			cnt--;
 
 		return cnt;
@@ -1015,8 +1025,7 @@ int hns_get_sset_count(struct net_device *netdev, int stringset)
 int hns_phy_led_set(struct net_device *netdev, int value)
 {
 	int retval;
-	struct hns_nic_priv *priv = netdev_priv(netdev);
-	struct phy_device *phy_dev = priv->phy;
+	struct phy_device *phy_dev = netdev->phydev;
 
 	retval = phy_write(phy_dev, HNS_PHY_PAGE_REG, HNS_PHY_PAGE_LED);
 	retval |= phy_write(phy_dev, HNS_LED_FC_REG, value);
@@ -1039,7 +1048,7 @@ int hns_set_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state)
 {
 	struct hns_nic_priv *priv = netdev_priv(netdev);
 	struct hnae_handle *h = priv->ae_handle;
-	struct phy_device *phy_dev = priv->phy;
+	struct phy_device *phy_dev = netdev->phydev;
 	int ret;
 
 	if (phy_dev)
@@ -1159,8 +1168,7 @@ static int hns_get_regs_len(struct net_device *net_dev)
 static int hns_nic_nway_reset(struct net_device *netdev)
 {
 	int ret = 0;
-	struct hns_nic_priv *priv = netdev_priv(netdev);
-	struct phy_device *phy = priv->phy;
+	struct phy_device *phy = netdev->phydev;
 
 	if (netif_running(netdev)) {
 		if (phy)
@@ -1264,11 +1272,9 @@ static int hns_get_rxnfc(struct net_device *netdev,
 	return 0;
 }
 
-static struct ethtool_ops hns_ethtool_ops = {
+static const struct ethtool_ops hns_ethtool_ops = {
 	.get_drvinfo = hns_nic_get_drvinfo,
 	.get_link  = hns_nic_get_link,
-	.get_settings  = hns_nic_get_settings,
-	.set_settings  = hns_nic_set_settings,
 	.get_ringparam = hns_get_ringparam,
 	.get_pauseparam = hns_get_pauseparam,
 	.set_pauseparam = hns_set_pauseparam,
@@ -1288,6 +1294,8 @@ static struct ethtool_ops hns_ethtool_ops = {
 	.get_rxfh = hns_get_rss,
 	.set_rxfh = hns_set_rss,
 	.get_rxnfc = hns_get_rxnfc,
+	.get_link_ksettings  = hns_nic_get_link_ksettings,
+	.set_link_ksettings  = hns_nic_set_link_ksettings,
 };
 
 void hns_ethtool_set_ops(struct net_device *ndev)
diff --git a/drivers/net/ethernet/i825xx/82596.c b/drivers/net/ethernet/i825xx/82596.c
index befb4ac3e2b0..ce235b776793 100644
--- a/drivers/net/ethernet/i825xx/82596.c
+++ b/drivers/net/ethernet/i825xx/82596.c
@@ -89,10 +89,10 @@ static char version[] __initdata =
 #define DEB(x,y)	if (i596_debug & (x)) y
 
 
-#if defined(CONFIG_MVME16x_NET) || defined(CONFIG_MVME16x_NET_MODULE)
+#if IS_ENABLED(CONFIG_MVME16x_NET)
 #define ENABLE_MVME16x_NET
 #endif
-#if defined(CONFIG_BVME6000_NET) || defined(CONFIG_BVME6000_NET_MODULE)
+#if IS_ENABLED(CONFIG_BVME6000_NET)
 #define ENABLE_BVME6000_NET
 #endif
 
diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c
index 4c9771d57d6e..8f139197f1aa 100644
--- a/drivers/net/ethernet/ibm/emac/core.c
+++ b/drivers/net/ethernet/ibm/emac/core.c
@@ -977,7 +977,37 @@ static void emac_set_multicast_list(struct net_device *ndev)
 		dev->mcast_pending = 1;
 		return;
 	}
+
+	mutex_lock(&dev->link_lock);
 	__emac_set_multicast_list(dev);
+	mutex_unlock(&dev->link_lock);
+}
+
+static int emac_set_mac_address(struct net_device *ndev, void *sa)
+{
+	struct emac_instance *dev = netdev_priv(ndev);
+	struct sockaddr *addr = sa;
+	struct emac_regs __iomem *p = dev->emacp;
+
+	if (!is_valid_ether_addr(addr->sa_data))
+	       return -EADDRNOTAVAIL;
+
+	mutex_lock(&dev->link_lock);
+
+	memcpy(ndev->dev_addr, addr->sa_data, ndev->addr_len);
+
+	emac_rx_disable(dev);
+	emac_tx_disable(dev);
+	out_be32(&p->iahr, (ndev->dev_addr[0] << 8) | ndev->dev_addr[1]);
+	out_be32(&p->ialr, (ndev->dev_addr[2] << 24) |
+		(ndev->dev_addr[3] << 16) | (ndev->dev_addr[4] << 8) |
+		ndev->dev_addr[5]);
+	emac_tx_enable(dev);
+	emac_rx_enable(dev);
+
+	mutex_unlock(&dev->link_lock);
+
+	return 0;
 }
 
 static int emac_resize_rx_ring(struct emac_instance *dev, int new_mtu)
@@ -2686,7 +2716,7 @@ static const struct net_device_ops emac_netdev_ops = {
 	.ndo_do_ioctl		= emac_ioctl,
 	.ndo_tx_timeout		= emac_tx_timeout,
 	.ndo_validate_addr	= eth_validate_addr,
-	.ndo_set_mac_address	= eth_mac_addr,
+	.ndo_set_mac_address	= emac_set_mac_address,
 	.ndo_start_xmit		= emac_start_xmit,
 	.ndo_change_mtu		= eth_change_mtu,
 };
@@ -2699,7 +2729,7 @@ static const struct net_device_ops emac_gige_netdev_ops = {
 	.ndo_do_ioctl		= emac_ioctl,
 	.ndo_tx_timeout		= emac_tx_timeout,
 	.ndo_validate_addr	= eth_validate_addr,
-	.ndo_set_mac_address	= eth_mac_addr,
+	.ndo_set_mac_address	= emac_set_mac_address,
 	.ndo_start_xmit		= emac_start_xmit_sg,
 	.ndo_change_mtu		= emac_change_mtu,
 };
@@ -2750,7 +2780,7 @@ static int emac_probe(struct platform_device *ofdev)
 	/* Get interrupts. EMAC irq is mandatory, WOL irq is optional */
 	dev->emac_irq = irq_of_parse_and_map(np, 0);
 	dev->wol_irq = irq_of_parse_and_map(np, 1);
-	if (dev->emac_irq == NO_IRQ) {
+	if (!dev->emac_irq) {
 		printk(KERN_ERR "%s: Can't map main interrupt\n", np->full_name);
 		goto err_free;
 	}
@@ -2913,9 +2943,9 @@ static int emac_probe(struct platform_device *ofdev)
  err_reg_unmap:
 	iounmap(dev->emacp);
  err_irq_unmap:
-	if (dev->wol_irq != NO_IRQ)
+	if (dev->wol_irq)
 		irq_dispose_mapping(dev->wol_irq);
-	if (dev->emac_irq != NO_IRQ)
+	if (dev->emac_irq)
 		irq_dispose_mapping(dev->emac_irq);
  err_free:
 	free_netdev(ndev);
@@ -2957,9 +2987,9 @@ static int emac_remove(struct platform_device *ofdev)
 	emac_dbg_unregister(dev);
 	iounmap(dev->emacp);
 
-	if (dev->wol_irq != NO_IRQ)
+	if (dev->wol_irq)
 		irq_dispose_mapping(dev->wol_irq);
-	if (dev->emac_irq != NO_IRQ)
+	if (dev->emac_irq)
 		irq_dispose_mapping(dev->emac_irq);
 
 	free_netdev(dev->ndev);
diff --git a/drivers/net/ethernet/ibm/emac/mal.c b/drivers/net/ethernet/ibm/emac/mal.c
index fdb5cdb3cd15..aaf6fec566b5 100644
--- a/drivers/net/ethernet/ibm/emac/mal.c
+++ b/drivers/net/ethernet/ibm/emac/mal.c
@@ -597,9 +597,8 @@ static int mal_probe(struct platform_device *ofdev)
 		mal->rxde_irq = irq_of_parse_and_map(ofdev->dev.of_node, 4);
 	}
 
-	if (mal->txeob_irq == NO_IRQ || mal->rxeob_irq == NO_IRQ ||
-	    mal->serr_irq == NO_IRQ || mal->txde_irq == NO_IRQ ||
-	    mal->rxde_irq == NO_IRQ) {
+	if (!mal->txeob_irq || !mal->rxeob_irq || !mal->serr_irq ||
+	    !mal->txde_irq  || !mal->rxde_irq) {
 		printk(KERN_ERR
 		       "mal%d: failed to map interrupts !\n", index);
 		err = -ENODEV;
diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index 88f3c85fb04a..bfe17d9c022d 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -203,7 +203,8 @@ static void free_long_term_buff(struct ibmvnic_adapter *adapter,
 	struct device *dev = &adapter->vdev->dev;
 
 	dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);
-	send_request_unmap(adapter, ltb->map_id);
+	if (!adapter->failover)
+		send_request_unmap(adapter, ltb->map_id);
 }
 
 static int alloc_rx_pool(struct ibmvnic_adapter *adapter,
@@ -522,7 +523,8 @@ static int ibmvnic_close(struct net_device *netdev)
 	for (i = 0; i < adapter->req_rx_queues; i++)
 		napi_disable(&adapter->napi[i]);
 
-	netif_tx_stop_all_queues(netdev);
+	if (!adapter->failover)
+		netif_tx_stop_all_queues(netdev);
 
 	if (adapter->bounce_buffer) {
 		if (!dma_mapping_error(dev, adapter->bounce_buffer_dma)) {
@@ -1422,7 +1424,7 @@ static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)
 		scrq = adapter->tx_scrq[i];
 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
 
-		if (scrq->irq == NO_IRQ) {
+		if (!scrq->irq) {
 			rc = -EINVAL;
 			dev_err(dev, "Error mapping irq\n");
 			goto req_tx_irq_failed;
@@ -1442,7 +1444,7 @@ static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)
 	for (i = 0; i < adapter->req_rx_queues; i++) {
 		scrq = adapter->rx_scrq[i];
 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
-		if (scrq->irq == NO_IRQ) {
+		if (!scrq->irq) {
 			rc = -EINVAL;
 			dev_err(dev, "Error mapping irq\n");
 			goto req_rx_irq_failed;
@@ -2777,12 +2779,6 @@ static void handle_control_ras_rsp(union ibmvnic_crq *crq,
 	}
 }
 
-static int ibmvnic_fw_comp_open(struct inode *inode, struct file *file)
-{
-	file->private_data = inode->i_private;
-	return 0;
-}
-
 static ssize_t trace_read(struct file *file, char __user *user_buf, size_t len,
 			  loff_t *ppos)
 {
@@ -2834,7 +2830,7 @@ static ssize_t trace_read(struct file *file, char __user *user_buf, size_t len,
 
 static const struct file_operations trace_ops = {
 	.owner		= THIS_MODULE,
-	.open		= ibmvnic_fw_comp_open,
+	.open		= simple_open,
 	.read		= trace_read,
 };
 
@@ -2884,7 +2880,7 @@ static ssize_t paused_write(struct file *file, const char __user *user_buf,
 
 static const struct file_operations paused_ops = {
 	.owner		= THIS_MODULE,
-	.open		= ibmvnic_fw_comp_open,
+	.open		= simple_open,
 	.read		= paused_read,
 	.write		= paused_write,
 };
@@ -2932,7 +2928,7 @@ static ssize_t tracing_write(struct file *file, const char __user *user_buf,
 
 static const struct file_operations tracing_ops = {
 	.owner		= THIS_MODULE,
-	.open		= ibmvnic_fw_comp_open,
+	.open		= simple_open,
 	.read		= tracing_read,
 	.write		= tracing_write,
 };
@@ -2985,7 +2981,7 @@ static ssize_t error_level_write(struct file *file, const char __user *user_buf,
 
 static const struct file_operations error_level_ops = {
 	.owner		= THIS_MODULE,
-	.open		= ibmvnic_fw_comp_open,
+	.open		= simple_open,
 	.read		= error_level_read,
 	.write		= error_level_write,
 };
@@ -3036,7 +3032,7 @@ static ssize_t trace_level_write(struct file *file, const char __user *user_buf,
 
 static const struct file_operations trace_level_ops = {
 	.owner		= THIS_MODULE,
-	.open		= ibmvnic_fw_comp_open,
+	.open		= simple_open,
 	.read		= trace_level_read,
 	.write		= trace_level_write,
 };
@@ -3089,7 +3085,7 @@ static ssize_t trace_buff_size_write(struct file *file,
 
 static const struct file_operations trace_size_ops = {
 	.owner		= THIS_MODULE,
-	.open		= ibmvnic_fw_comp_open,
+	.open		= simple_open,
 	.read		= trace_buff_size_read,
 	.write		= trace_buff_size_write,
 };
@@ -3280,6 +3276,10 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
 			rc = ibmvnic_send_crq_init(adapter);
 			if (rc)
 				dev_err(dev, "Error sending init rc=%ld\n", rc);
+		} else if (gen_crq->cmd == IBMVNIC_DEVICE_FAILOVER) {
+			dev_info(dev, "Backing device failover detected\n");
+			netif_carrier_off(netdev);
+			adapter->failover = true;
 		} else {
 			/* The adapter lost the connection */
 			dev_err(dev, "Virtual Adapter failed (rc=%d)\n",
@@ -3615,8 +3615,18 @@ static void handle_crq_init_rsp(struct work_struct *work)
 	struct device *dev = &adapter->vdev->dev;
 	struct net_device *netdev = adapter->netdev;
 	unsigned long timeout = msecs_to_jiffies(30000);
+	bool restart = false;
 	int rc;
 
+	if (adapter->failover) {
+		release_sub_crqs(adapter);
+		if (netif_running(netdev)) {
+			netif_tx_disable(netdev);
+			ibmvnic_close(netdev);
+			restart = true;
+		}
+	}
+
 	send_version_xchg(adapter);
 	reinit_completion(&adapter->init_done);
 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {
@@ -3645,6 +3655,17 @@ static void handle_crq_init_rsp(struct work_struct *work)
 
 	netdev->real_num_tx_queues = adapter->req_tx_queues;
 
+	if (adapter->failover) {
+		adapter->failover = false;
+		if (restart) {
+			rc = ibmvnic_open(netdev);
+			if (rc)
+				goto restart_failed;
+		}
+		netif_carrier_on(netdev);
+		return;
+	}
+
 	rc = register_netdev(netdev);
 	if (rc) {
 		dev_err(dev,
@@ -3655,6 +3676,8 @@ static void handle_crq_init_rsp(struct work_struct *work)
 
 	return;
 
+restart_failed:
+	dev_err(dev, "Failed to restart ibmvnic, rc=%d\n", rc);
 register_failed:
 	release_sub_crqs(adapter);
 task_failed:
@@ -3692,6 +3715,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
 	dev_set_drvdata(&dev->dev, netdev);
 	adapter->vdev = dev;
 	adapter->netdev = netdev;
+	adapter->failover = false;
 
 	ether_addr_copy(adapter->mac_addr, mac_addr_p);
 	ether_addr_copy(netdev->dev_addr, adapter->mac_addr);
@@ -3721,6 +3745,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
 	if (dma_mapping_error(&dev->dev, adapter->stats_token)) {
 		if (!firmware_has_feature(FW_FEATURE_CMO))
 			dev_err(&dev->dev, "Couldn't map stats buffer\n");
+		rc = -ENOMEM;
 		goto free_crq;
 	}
 
diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h
index e82898fd518e..bfc84c7d0e11 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.h
+++ b/drivers/net/ethernet/ibm/ibmvnic.h
@@ -830,6 +830,7 @@ enum ibmvfc_crq_format {
 	IBMVNIC_CRQ_INIT                 = 0x01,
 	IBMVNIC_CRQ_INIT_COMPLETE        = 0x02,
 	IBMVNIC_PARTITION_MIGRATED       = 0x06,
+	IBMVNIC_DEVICE_FAILOVER          = 0x08,
 };
 
 struct ibmvnic_crq_queue {
@@ -1047,4 +1048,5 @@ struct ibmvnic_adapter {
 	u8 map_id;
 
 	struct work_struct vnic_crq_init;
+	bool failover;
 };
diff --git a/drivers/net/ethernet/intel/e1000e/82571.c b/drivers/net/ethernet/intel/e1000e/82571.c
index 7fd4d54599e4..6b03c8553e59 100644
--- a/drivers/net/ethernet/intel/e1000e/82571.c
+++ b/drivers/net/ethernet/intel/e1000e/82571.c
@@ -2032,7 +2032,8 @@ const struct e1000_info e1000_82574_info = {
 				  | FLAG2_DISABLE_ASPM_L0S
 				  | FLAG2_DISABLE_ASPM_L1
 				  | FLAG2_NO_DISABLE_RX
-				  | FLAG2_DMA_BURST,
+				  | FLAG2_DMA_BURST
+				  | FLAG2_CHECK_SYSTIM_OVERFLOW,
 	.pba			= 32,
 	.max_hw_frame_size	= DEFAULT_JUMBO,
 	.get_variants		= e1000_get_variants_82571,
@@ -2053,7 +2054,8 @@ const struct e1000_info e1000_82583_info = {
 				  | FLAG_HAS_CTRLEXT_ON_LOAD,
 	.flags2			= FLAG2_DISABLE_ASPM_L0S
 				  | FLAG2_DISABLE_ASPM_L1
-				  | FLAG2_NO_DISABLE_RX,
+				  | FLAG2_NO_DISABLE_RX
+				  | FLAG2_CHECK_SYSTIM_OVERFLOW,
 	.pba			= 32,
 	.max_hw_frame_size	= DEFAULT_JUMBO,
 	.get_variants		= e1000_get_variants_82571,
diff --git a/drivers/net/ethernet/intel/e1000e/e1000.h b/drivers/net/ethernet/intel/e1000e/e1000.h
index ef96cd11d6d2..879cca47b021 100644
--- a/drivers/net/ethernet/intel/e1000e/e1000.h
+++ b/drivers/net/ethernet/intel/e1000e/e1000.h
@@ -452,6 +452,7 @@ s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca);
 #define FLAG2_PCIM2PCI_ARBITER_WA         BIT(11)
 #define FLAG2_DFLT_CRC_STRIPPING          BIT(12)
 #define FLAG2_CHECK_RX_HWTSTAMP           BIT(13)
+#define FLAG2_CHECK_SYSTIM_OVERFLOW       BIT(14)
 
 #define E1000_RX_DESC_PS(R, i)	    \
 	(&(((union e1000_rx_desc_packet_split *)((R).desc))[i]))
diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c
index 3e11322d8d58..f3aaca743ea3 100644
--- a/drivers/net/ethernet/intel/e1000e/ich8lan.c
+++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c
@@ -5885,7 +5885,8 @@ const struct e1000_info e1000_pch_lpt_info = {
 				  | FLAG_HAS_JUMBO_FRAMES
 				  | FLAG_APME_IN_WUC,
 	.flags2			= FLAG2_HAS_PHY_STATS
-				  | FLAG2_HAS_EEE,
+				  | FLAG2_HAS_EEE
+				  | FLAG2_CHECK_SYSTIM_OVERFLOW,
 	.pba			= 26,
 	.max_hw_frame_size	= 9022,
 	.get_variants		= e1000_get_variants_ich8lan,
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
index 02f443958f31..7017281ba2dc 100644
--- a/drivers/net/ethernet/intel/e1000e/netdev.c
+++ b/drivers/net/ethernet/intel/e1000e/netdev.c
@@ -4303,6 +4303,42 @@ void e1000e_reinit_locked(struct e1000_adapter *adapter)
 }
 
 /**
+ * e1000e_sanitize_systim - sanitize raw cycle counter reads
+ * @hw: pointer to the HW structure
+ * @systim: cycle_t value read, sanitized and returned
+ *
+ * Errata for 82574/82583 possible bad bits read from SYSTIMH/L:
+ * check to see that the time is incrementing at a reasonable
+ * rate and is a multiple of incvalue.
+ **/
+static cycle_t e1000e_sanitize_systim(struct e1000_hw *hw, cycle_t systim)
+{
+	u64 time_delta, rem, temp;
+	cycle_t systim_next;
+	u32 incvalue;
+	int i;
+
+	incvalue = er32(TIMINCA) & E1000_TIMINCA_INCVALUE_MASK;
+	for (i = 0; i < E1000_MAX_82574_SYSTIM_REREADS; i++) {
+		/* latch SYSTIMH on read of SYSTIML */
+		systim_next = (cycle_t)er32(SYSTIML);
+		systim_next |= (cycle_t)er32(SYSTIMH) << 32;
+
+		time_delta = systim_next - systim;
+		temp = time_delta;
+		/* VMWare users have seen incvalue of zero, don't div / 0 */
+		rem = incvalue ? do_div(temp, incvalue) : (time_delta != 0);
+
+		systim = systim_next;
+
+		if ((time_delta < E1000_82574_SYSTIM_EPSILON) && (rem == 0))
+			break;
+	}
+
+	return systim;
+}
+
+/**
  * e1000e_cyclecounter_read - read raw cycle counter (used by time counter)
  * @cc: cyclecounter structure
  **/
@@ -4312,7 +4348,7 @@ static cycle_t e1000e_cyclecounter_read(const struct cyclecounter *cc)
 						     cc);
 	struct e1000_hw *hw = &adapter->hw;
 	u32 systimel, systimeh;
-	cycle_t systim, systim_next;
+	cycle_t systim;
 	/* SYSTIMH latching upon SYSTIML read does not work well.
 	 * This means that if SYSTIML overflows after we read it but before
 	 * we read SYSTIMH, the value of SYSTIMH has been incremented and we
@@ -4335,33 +4371,9 @@ static cycle_t e1000e_cyclecounter_read(const struct cyclecounter *cc)
 	systim = (cycle_t)systimel;
 	systim |= (cycle_t)systimeh << 32;
 
-	if ((hw->mac.type == e1000_82574) || (hw->mac.type == e1000_82583)) {
-		u64 time_delta, rem, temp;
-		u32 incvalue;
-		int i;
-
-		/* errata for 82574/82583 possible bad bits read from SYSTIMH/L
-		 * check to see that the time is incrementing at a reasonable
-		 * rate and is a multiple of incvalue
-		 */
-		incvalue = er32(TIMINCA) & E1000_TIMINCA_INCVALUE_MASK;
-		for (i = 0; i < E1000_MAX_82574_SYSTIM_REREADS; i++) {
-			/* latch SYSTIMH on read of SYSTIML */
-			systim_next = (cycle_t)er32(SYSTIML);
-			systim_next |= (cycle_t)er32(SYSTIMH) << 32;
-
-			time_delta = systim_next - systim;
-			temp = time_delta;
-			/* VMWare users have seen incvalue of zero, don't div / 0 */
-			rem = incvalue ? do_div(temp, incvalue) : (time_delta != 0);
-
-			systim = systim_next;
+	if (adapter->flags2 & FLAG2_CHECK_SYSTIM_OVERFLOW)
+		systim = e1000e_sanitize_systim(hw, systim);
 
-			if ((time_delta < E1000_82574_SYSTIM_EPSILON) &&
-			    (rem == 0))
-				break;
-		}
-	}
 	return systim;
 }
 
diff --git a/drivers/net/ethernet/intel/e1000e/ptp.c b/drivers/net/ethernet/intel/e1000e/ptp.c
index 2e1b17ad52a3..ad03763e009a 100644
--- a/drivers/net/ethernet/intel/e1000e/ptp.c
+++ b/drivers/net/ethernet/intel/e1000e/ptp.c
@@ -334,7 +334,7 @@ void e1000e_ptp_init(struct e1000_adapter *adapter)
 	if (IS_ERR(adapter->ptp_clock)) {
 		adapter->ptp_clock = NULL;
 		e_err("ptp_clock_register failed\n");
-	} else {
+	} else if (adapter->ptp_clock) {
 		e_info("registered PHC clock\n");
 	}
 }
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k.h b/drivers/net/ethernet/intel/fm10k/fm10k.h
index c4cf08dcf5af..4d19e46f7c55 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k.h
@@ -240,9 +240,7 @@ struct fm10k_iov_data {
 	struct fm10k_vf_info	vf_info[0];
 };
 
-#define fm10k_vxlan_port_for_each(vp, intfc) \
-	list_for_each_entry(vp, &(intfc)->vxlan_port, list)
-struct fm10k_vxlan_port {
+struct fm10k_udp_port {
 	struct list_head	list;
 	sa_family_t		sa_family;
 	__be16			port;
@@ -335,8 +333,9 @@ struct fm10k_intfc {
 	u32 reta[FM10K_RETA_SIZE];
 	u32 rssrk[FM10K_RSSRK_SIZE];
 
-	/* VXLAN port tracking information */
+	/* UDP encapsulation port tracking information */
 	struct list_head vxlan_port;
+	struct list_head geneve_port;
 
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *dbg_intfc;
@@ -458,7 +457,7 @@ __be16 fm10k_tx_encap_offload(struct sk_buff *skb);
 netdev_tx_t fm10k_xmit_frame_ring(struct sk_buff *skb,
 				  struct fm10k_ring *tx_ring);
 void fm10k_tx_timeout_reset(struct fm10k_intfc *interface);
-u64 fm10k_get_tx_pending(struct fm10k_ring *ring);
+u64 fm10k_get_tx_pending(struct fm10k_ring *ring, bool in_sw);
 bool fm10k_check_tx_hang(struct fm10k_ring *tx_ring);
 void fm10k_alloc_rx_buffers(struct fm10k_ring *rx_ring, u16 cleaned_count);
 
@@ -496,7 +495,6 @@ int fm10k_close(struct net_device *netdev);
 
 /* Ethtool */
 void fm10k_set_ethtool_ops(struct net_device *dev);
-u32 fm10k_get_reta_size(struct net_device *netdev);
 void fm10k_write_reta(struct fm10k_intfc *interface, const u32 *indir);
 
 /* IOV */
@@ -509,7 +507,7 @@ int fm10k_iov_configure(struct pci_dev *pdev, int num_vfs);
 s32 fm10k_iov_update_pvid(struct fm10k_intfc *interface, u16 glort, u16 pvid);
 int fm10k_ndo_set_vf_mac(struct net_device *netdev, int vf_idx, u8 *mac);
 int fm10k_ndo_set_vf_vlan(struct net_device *netdev,
-			  int vf_idx, u16 vid, u8 qos);
+			  int vf_idx, u16 vid, u8 qos, __be16 vlan_proto);
 int fm10k_ndo_set_vf_bw(struct net_device *netdev, int vf_idx, int rate,
 			int unused);
 int fm10k_ndo_get_vf_config(struct net_device *netdev,
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_common.c b/drivers/net/ethernet/intel/fm10k/fm10k_common.c
index d6baaea8bc7c..dd95ac4f4c64 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_common.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_common.c
@@ -207,6 +207,9 @@ s32 fm10k_disable_queues_generic(struct fm10k_hw *hw, u16 q_cnt)
 	/* clear tx_ready to prevent any false hits for reset */
 	hw->mac.tx_ready = false;
 
+	if (FM10K_REMOVED(hw->hw_addr))
+		return 0;
+
 	/* clear the enable bit for all rings */
 	for (i = 0; i < q_cnt; i++) {
 		reg = fm10k_read_reg(hw, FM10K_TXDCTL(i));
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_common.h b/drivers/net/ethernet/intel/fm10k/fm10k_common.h
index 50f71e997448..d51f9c7a47ff 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_common.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_common.h
@@ -34,7 +34,7 @@ u32 fm10k_read_reg(struct fm10k_hw *hw, int reg);
 /* write operations, indexed using DWORDS */
 #define fm10k_write_reg(hw, reg, val) \
 do { \
-	u32 __iomem *hw_addr = ACCESS_ONCE((hw)->hw_addr); \
+	u32 __iomem *hw_addr = READ_ONCE((hw)->hw_addr); \
 	if (!FM10K_REMOVED(hw_addr)) \
 		writel((val), &hw_addr[(reg)]); \
 } while (0)
@@ -42,7 +42,7 @@ do { \
 /* Switch register write operations, index using DWORDS */
 #define fm10k_write_sw_reg(hw, reg, val) \
 do { \
-	u32 __iomem *sw_addr = ACCESS_ONCE((hw)->sw_addr); \
+	u32 __iomem *sw_addr = READ_ONCE((hw)->sw_addr); \
 	if (!FM10K_REMOVED(sw_addr)) \
 		writel((val), &sw_addr[(reg)]); \
 } while (0)
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c
index c04cbe9c9f7c..5241e0873397 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c
@@ -966,7 +966,7 @@ static int fm10k_set_priv_flags(struct net_device *netdev, u32 priv_flags)
 	return 0;
 }
 
-u32 fm10k_get_reta_size(struct net_device __always_unused *netdev)
+static u32 fm10k_get_reta_size(struct net_device __always_unused *netdev)
 {
 	return FM10K_RETA_SIZE * FM10K_RETA_ENTRIES_PER_REG;
 }
@@ -1182,6 +1182,7 @@ static const struct ethtool_ops fm10k_ethtool_ops = {
 	.set_rxfh		= fm10k_set_rssh,
 	.get_channels		= fm10k_get_channels,
 	.set_channels		= fm10k_set_channels,
+	.get_ts_info		= ethtool_op_get_ts_info,
 };
 
 void fm10k_set_ethtool_ops(struct net_device *dev)
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_iov.c b/drivers/net/ethernet/intel/fm10k/fm10k_iov.c
index 47f0743ec03b..5f4dac0d36ef 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_iov.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_iov.c
@@ -51,7 +51,7 @@ s32 fm10k_iov_event(struct fm10k_intfc *interface)
 	int i;
 
 	/* if there is no iov_data then there is no mailbox to process */
-	if (!ACCESS_ONCE(interface->iov_data))
+	if (!READ_ONCE(interface->iov_data))
 		return 0;
 
 	rcu_read_lock();
@@ -99,7 +99,7 @@ s32 fm10k_iov_mbx(struct fm10k_intfc *interface)
 	int i;
 
 	/* if there is no iov_data then there is no mailbox to process */
-	if (!ACCESS_ONCE(interface->iov_data))
+	if (!READ_ONCE(interface->iov_data))
 		return 0;
 
 	rcu_read_lock();
@@ -445,7 +445,7 @@ int fm10k_ndo_set_vf_mac(struct net_device *netdev, int vf_idx, u8 *mac)
 }
 
 int fm10k_ndo_set_vf_vlan(struct net_device *netdev, int vf_idx, u16 vid,
-			  u8 qos)
+			  u8 qos, __be16 vlan_proto)
 {
 	struct fm10k_intfc *interface = netdev_priv(netdev);
 	struct fm10k_iov_data *iov_data = interface->iov_data;
@@ -460,6 +460,10 @@ int fm10k_ndo_set_vf_vlan(struct net_device *netdev, int vf_idx, u16 vid,
 	if (qos || (vid > (VLAN_VID_MASK - 1)))
 		return -EINVAL;
 
+	/* VF VLAN Protocol part to default is unsupported */
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
 	vf_info = &iov_data->vf_info[vf_idx];
 
 	/* exit if there is nothing to do */
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_main.c b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
index e9767b6366a8..5de937852436 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_main.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
@@ -56,7 +56,8 @@ static int __init fm10k_init_module(void)
 	pr_info("%s\n", fm10k_copyright);
 
 	/* create driver workqueue */
-	fm10k_workqueue = alloc_workqueue("fm10k", WQ_MEM_RECLAIM, 0);
+	fm10k_workqueue = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0,
+					  fm10k_driver_name);
 
 	fm10k_dbg_init();
 
@@ -651,11 +652,11 @@ static int fm10k_clean_rx_irq(struct fm10k_q_vector *q_vector,
 static struct ethhdr *fm10k_port_is_vxlan(struct sk_buff *skb)
 {
 	struct fm10k_intfc *interface = netdev_priv(skb->dev);
-	struct fm10k_vxlan_port *vxlan_port;
+	struct fm10k_udp_port *vxlan_port;
 
 	/* we can only offload a vxlan if we recognize it as such */
 	vxlan_port = list_first_entry_or_null(&interface->vxlan_port,
-					      struct fm10k_vxlan_port, list);
+					      struct fm10k_udp_port, list);
 
 	if (!vxlan_port)
 		return NULL;
@@ -1128,13 +1129,24 @@ static u64 fm10k_get_tx_completed(struct fm10k_ring *ring)
 	return ring->stats.packets;
 }
 
-u64 fm10k_get_tx_pending(struct fm10k_ring *ring)
+/**
+ * fm10k_get_tx_pending - how many Tx descriptors not processed
+ * @ring: the ring structure
+ * @in_sw: is tx_pending being checked in SW or in HW?
+ */
+u64 fm10k_get_tx_pending(struct fm10k_ring *ring, bool in_sw)
 {
 	struct fm10k_intfc *interface = ring->q_vector->interface;
 	struct fm10k_hw *hw = &interface->hw;
+	u32 head, tail;
 
-	u32 head = fm10k_read_reg(hw, FM10K_TDH(ring->reg_idx));
-	u32 tail = fm10k_read_reg(hw, FM10K_TDT(ring->reg_idx));
+	if (likely(in_sw)) {
+		head = ring->next_to_clean;
+		tail = ring->next_to_use;
+	} else {
+		head = fm10k_read_reg(hw, FM10K_TDH(ring->reg_idx));
+		tail = fm10k_read_reg(hw, FM10K_TDT(ring->reg_idx));
+	}
 
 	return ((head <= tail) ? tail : tail + ring->count) - head;
 }
@@ -1143,7 +1155,7 @@ bool fm10k_check_tx_hang(struct fm10k_ring *tx_ring)
 {
 	u32 tx_done = fm10k_get_tx_completed(tx_ring);
 	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
-	u32 tx_pending = fm10k_get_tx_pending(tx_ring);
+	u32 tx_pending = fm10k_get_tx_pending(tx_ring, true);
 
 	clear_check_for_tx_hang(tx_ring);
 
@@ -1397,7 +1409,7 @@ static void fm10k_update_itr(struct fm10k_ring_container *ring_container)
 	 * that the calculation will never get below a 1. The bit shift
 	 * accounts for changes in the ITR due to PCIe link speed.
 	 */
-	itr_round = ACCESS_ONCE(ring_container->itr_scale) + 8;
+	itr_round = READ_ONCE(ring_container->itr_scale) + 8;
 	avg_wire_size += BIT(itr_round) - 1;
 	avg_wire_size >>= itr_round;
 
@@ -1473,7 +1485,7 @@ static int fm10k_poll(struct napi_struct *napi, int budget)
 	/* re-enable the q_vector */
 	fm10k_qv_enable(q_vector);
 
-	return 0;
+	return min(work_done, budget - 1);
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
index 20a5bbe3f536..05629381be6b 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
@@ -384,129 +384,171 @@ static void fm10k_request_glort_range(struct fm10k_intfc *interface)
 }
 
 /**
- * fm10k_del_vxlan_port_all
+ * fm10k_free_udp_port_info
  * @interface: board private structure
  *
- * This function frees the entire vxlan_port list
+ * This function frees both geneve_port and vxlan_port structures
  **/
-static void fm10k_del_vxlan_port_all(struct fm10k_intfc *interface)
+static void fm10k_free_udp_port_info(struct fm10k_intfc *interface)
 {
-	struct fm10k_vxlan_port *vxlan_port;
-
-	/* flush all entries from list */
-	vxlan_port = list_first_entry_or_null(&interface->vxlan_port,
-					      struct fm10k_vxlan_port, list);
-	while (vxlan_port) {
-		list_del(&vxlan_port->list);
-		kfree(vxlan_port);
-		vxlan_port = list_first_entry_or_null(&interface->vxlan_port,
-						      struct fm10k_vxlan_port,
-						      list);
+	struct fm10k_udp_port *port;
+
+	/* flush all entries from vxlan list */
+	port = list_first_entry_or_null(&interface->vxlan_port,
+					struct fm10k_udp_port, list);
+	while (port) {
+		list_del(&port->list);
+		kfree(port);
+		port = list_first_entry_or_null(&interface->vxlan_port,
+						struct fm10k_udp_port,
+						list);
+	}
+
+	/* flush all entries from geneve list */
+	port = list_first_entry_or_null(&interface->geneve_port,
+					struct fm10k_udp_port, list);
+	while (port) {
+		list_del(&port->list);
+		kfree(port);
+		port = list_first_entry_or_null(&interface->vxlan_port,
+						struct fm10k_udp_port,
+						list);
 	}
 }
 
 /**
- * fm10k_restore_vxlan_port
+ * fm10k_restore_udp_port_info
  * @interface: board private structure
  *
- * This function restores the value in the tunnel_cfg register after reset
+ * This function restores the value in the tunnel_cfg register(s) after reset
  **/
-static void fm10k_restore_vxlan_port(struct fm10k_intfc *interface)
+static void fm10k_restore_udp_port_info(struct fm10k_intfc *interface)
 {
 	struct fm10k_hw *hw = &interface->hw;
-	struct fm10k_vxlan_port *vxlan_port;
+	struct fm10k_udp_port *port;
 
 	/* only the PF supports configuring tunnels */
 	if (hw->mac.type != fm10k_mac_pf)
 		return;
 
-	vxlan_port = list_first_entry_or_null(&interface->vxlan_port,
-					      struct fm10k_vxlan_port, list);
+	port = list_first_entry_or_null(&interface->vxlan_port,
+					struct fm10k_udp_port, list);
 
 	/* restore tunnel configuration register */
 	fm10k_write_reg(hw, FM10K_TUNNEL_CFG,
-			(vxlan_port ? ntohs(vxlan_port->port) : 0) |
+			(port ? ntohs(port->port) : 0) |
 			(ETH_P_TEB << FM10K_TUNNEL_CFG_NVGRE_SHIFT));
+
+	port = list_first_entry_or_null(&interface->geneve_port,
+					struct fm10k_udp_port, list);
+
+	/* restore Geneve tunnel configuration register */
+	fm10k_write_reg(hw, FM10K_TUNNEL_CFG_GENEVE,
+			(port ? ntohs(port->port) : 0));
+}
+
+static struct fm10k_udp_port *
+fm10k_remove_tunnel_port(struct list_head *ports,
+			 struct udp_tunnel_info *ti)
+{
+	struct fm10k_udp_port *port;
+
+	list_for_each_entry(port, ports, list) {
+		if ((port->port == ti->port) &&
+		    (port->sa_family == ti->sa_family)) {
+			list_del(&port->list);
+			return port;
+		}
+	}
+
+	return NULL;
+}
+
+static void fm10k_insert_tunnel_port(struct list_head *ports,
+				     struct udp_tunnel_info *ti)
+{
+	struct fm10k_udp_port *port;
+
+	/* remove existing port entry from the list so that the newest items
+	 * are always at the tail of the list.
+	 */
+	port = fm10k_remove_tunnel_port(ports, ti);
+	if (!port) {
+		port = kmalloc(sizeof(*port), GFP_ATOMIC);
+		if  (!port)
+			return;
+		port->port = ti->port;
+		port->sa_family = ti->sa_family;
+	}
+
+	list_add_tail(&port->list, ports);
 }
 
 /**
- * fm10k_add_vxlan_port
+ * fm10k_udp_tunnel_add
  * @netdev: network interface device structure
  * @ti: Tunnel endpoint information
  *
- * This function is called when a new VXLAN interface has added a new port
- * number to the range that is currently in use for VXLAN.  The new port
- * number is always added to the tail so that the port number list should
- * match the order in which the ports were allocated.  The head of the list
- * is always used as the VXLAN port number for offloads.
+ * This function is called when a new UDP tunnel port has been added.
+ * Due to hardware restrictions, only one port per type can be offloaded at
+ * once.
  **/
-static void fm10k_add_vxlan_port(struct net_device *dev,
+static void fm10k_udp_tunnel_add(struct net_device *dev,
 				 struct udp_tunnel_info *ti)
 {
 	struct fm10k_intfc *interface = netdev_priv(dev);
-	struct fm10k_vxlan_port *vxlan_port;
 
-	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
-		return;
 	/* only the PF supports configuring tunnels */
 	if (interface->hw.mac.type != fm10k_mac_pf)
 		return;
 
-	/* existing ports are pulled out so our new entry is always last */
-	fm10k_vxlan_port_for_each(vxlan_port, interface) {
-		if ((vxlan_port->port == ti->port) &&
-		    (vxlan_port->sa_family == ti->sa_family)) {
-			list_del(&vxlan_port->list);
-			goto insert_tail;
-		}
-	}
-
-	/* allocate memory to track ports */
-	vxlan_port = kmalloc(sizeof(*vxlan_port), GFP_ATOMIC);
-	if (!vxlan_port)
+	switch (ti->type) {
+	case UDP_TUNNEL_TYPE_VXLAN:
+		fm10k_insert_tunnel_port(&interface->vxlan_port, ti);
+		break;
+	case UDP_TUNNEL_TYPE_GENEVE:
+		fm10k_insert_tunnel_port(&interface->geneve_port, ti);
+		break;
+	default:
 		return;
-	vxlan_port->port = ti->port;
-	vxlan_port->sa_family = ti->sa_family;
-
-insert_tail:
-	/* add new port value to list */
-	list_add_tail(&vxlan_port->list, &interface->vxlan_port);
+	}
 
-	fm10k_restore_vxlan_port(interface);
+	fm10k_restore_udp_port_info(interface);
 }
 
 /**
- * fm10k_del_vxlan_port
+ * fm10k_udp_tunnel_del
  * @netdev: network interface device structure
  * @ti: Tunnel endpoint information
  *
- * This function is called when a new VXLAN interface has freed a port
- * number from the range that is currently in use for VXLAN.  The freed
- * port is removed from the list and the new head is used to determine
- * the port number for offloads.
+ * This function is called when a new UDP tunnel port is deleted. The freed
+ * port will be removed from the list, then we reprogram the offloaded port
+ * based on the head of the list.
  **/
-static void fm10k_del_vxlan_port(struct net_device *dev,
+static void fm10k_udp_tunnel_del(struct net_device *dev,
 				 struct udp_tunnel_info *ti)
 {
 	struct fm10k_intfc *interface = netdev_priv(dev);
-	struct fm10k_vxlan_port *vxlan_port;
+	struct fm10k_udp_port *port = NULL;
 
-	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
-		return;
 	if (interface->hw.mac.type != fm10k_mac_pf)
 		return;
 
-	/* find the port in the list and free it */
-	fm10k_vxlan_port_for_each(vxlan_port, interface) {
-		if ((vxlan_port->port == ti->port) &&
-		    (vxlan_port->sa_family == ti->sa_family)) {
-			list_del(&vxlan_port->list);
-			kfree(vxlan_port);
-			break;
-		}
+	switch (ti->type) {
+	case UDP_TUNNEL_TYPE_VXLAN:
+		port = fm10k_remove_tunnel_port(&interface->vxlan_port, ti);
+		break;
+	case UDP_TUNNEL_TYPE_GENEVE:
+		port = fm10k_remove_tunnel_port(&interface->geneve_port, ti);
+		break;
+	default:
+		return;
 	}
 
-	fm10k_restore_vxlan_port(interface);
+	/* if we did remove a port we need to free its memory */
+	kfree(port);
+
+	fm10k_restore_udp_port_info(interface);
 }
 
 /**
@@ -555,7 +597,6 @@ int fm10k_open(struct net_device *netdev)
 	if (err)
 		goto err_set_queues;
 
-	/* update VXLAN port configuration */
 	udp_tunnel_get_rx_info(netdev);
 
 	fm10k_up(interface);
@@ -591,7 +632,7 @@ int fm10k_close(struct net_device *netdev)
 
 	fm10k_qv_free_irq(interface);
 
-	fm10k_del_vxlan_port_all(interface);
+	fm10k_free_udp_port_info(interface);
 
 	fm10k_free_all_tx_resources(interface);
 	fm10k_free_all_rx_resources(interface);
@@ -1055,7 +1096,7 @@ void fm10k_restore_rx_state(struct fm10k_intfc *interface)
 	interface->xcast_mode = xcast_mode;
 
 	/* Restore tunnel configuration */
-	fm10k_restore_vxlan_port(interface);
+	fm10k_restore_udp_port_info(interface);
 }
 
 void fm10k_reset_rx_state(struct fm10k_intfc *interface)
@@ -1098,7 +1139,7 @@ static struct rtnl_link_stats64 *fm10k_get_stats64(struct net_device *netdev,
 	rcu_read_lock();
 
 	for (i = 0; i < interface->num_rx_queues; i++) {
-		ring = ACCESS_ONCE(interface->rx_ring[i]);
+		ring = READ_ONCE(interface->rx_ring[i]);
 
 		if (!ring)
 			continue;
@@ -1114,7 +1155,7 @@ static struct rtnl_link_stats64 *fm10k_get_stats64(struct net_device *netdev,
 	}
 
 	for (i = 0; i < interface->num_tx_queues; i++) {
-		ring = ACCESS_ONCE(interface->tx_ring[i]);
+		ring = READ_ONCE(interface->tx_ring[i]);
 
 		if (!ring)
 			continue;
@@ -1299,7 +1340,7 @@ static void *fm10k_dfwd_add_station(struct net_device *dev,
 static void fm10k_dfwd_del_station(struct net_device *dev, void *priv)
 {
 	struct fm10k_intfc *interface = netdev_priv(dev);
-	struct fm10k_l2_accel *l2_accel = ACCESS_ONCE(interface->l2_accel);
+	struct fm10k_l2_accel *l2_accel = READ_ONCE(interface->l2_accel);
 	struct fm10k_dglort_cfg dglort = { 0 };
 	struct fm10k_hw *hw = &interface->hw;
 	struct net_device *sdev = priv;
@@ -1375,8 +1416,8 @@ static const struct net_device_ops fm10k_netdev_ops = {
 	.ndo_set_vf_vlan	= fm10k_ndo_set_vf_vlan,
 	.ndo_set_vf_rate	= fm10k_ndo_set_vf_bw,
 	.ndo_get_vf_config	= fm10k_ndo_get_vf_config,
-	.ndo_udp_tunnel_add	= fm10k_add_vxlan_port,
-	.ndo_udp_tunnel_del	= fm10k_del_vxlan_port,
+	.ndo_udp_tunnel_add	= fm10k_udp_tunnel_add,
+	.ndo_udp_tunnel_del	= fm10k_udp_tunnel_del,
 	.ndo_dfwd_add_station	= fm10k_dfwd_add_station,
 	.ndo_dfwd_del_station	= fm10k_dfwd_del_station,
 #ifdef CONFIG_NET_POLL_CONTROLLER
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
index 774a5654bf42..b1a2f8437d59 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
@@ -62,7 +62,7 @@ u16 fm10k_read_pci_cfg_word(struct fm10k_hw *hw, u32 reg)
 
 u32 fm10k_read_reg(struct fm10k_hw *hw, int reg)
 {
-	u32 __iomem *hw_addr = ACCESS_ONCE(hw->hw_addr);
+	u32 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
 	u32 value = 0;
 
 	if (FM10K_REMOVED(hw_addr))
@@ -133,7 +133,7 @@ static void fm10k_detach_subtask(struct fm10k_intfc *interface)
 	/* check the real address space to see if we've recovered */
 	hw_addr = READ_ONCE(interface->uc_addr);
 	value = readl(hw_addr);
-	if ((~value)) {
+	if (~value) {
 		interface->hw.hw_addr = interface->uc_addr;
 		netif_device_attach(netdev);
 		interface->flags |= FM10K_FLAG_RESET_REQUESTED;
@@ -734,15 +734,15 @@ static void fm10k_configure_rx_ring(struct fm10k_intfc *interface,
 	u64 rdba = ring->dma;
 	struct fm10k_hw *hw = &interface->hw;
 	u32 size = ring->count * sizeof(union fm10k_rx_desc);
-	u32 rxqctl = FM10K_RXQCTL_ENABLE | FM10K_RXQCTL_PF;
-	u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
+	u32 rxqctl, rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 	u32 srrctl = FM10K_SRRCTL_BUFFER_CHAINING_EN;
 	u32 rxint = FM10K_INT_MAP_DISABLE;
 	u8 rx_pause = interface->rx_pause;
 	u8 reg_idx = ring->reg_idx;
 
 	/* disable queue to avoid issues while updating state */
-	fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), 0);
+	rxqctl = fm10k_read_reg(hw, FM10K_RXQCTL(reg_idx));
+	rxqctl &= ~FM10K_RXQCTL_ENABLE;
 	fm10k_write_flush(hw);
 
 	/* possible poll here to verify ring resources have been cleaned */
@@ -797,6 +797,8 @@ static void fm10k_configure_rx_ring(struct fm10k_intfc *interface,
 	fm10k_write_reg(hw, FM10K_RXINT(reg_idx), rxint);
 
 	/* enable queue */
+	rxqctl = fm10k_read_reg(hw, FM10K_RXQCTL(reg_idx));
+	rxqctl |= FM10K_RXQCTL_ENABLE;
 	fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl);
 
 	/* place buffers on ring for receive data */
@@ -1699,7 +1701,7 @@ void fm10k_down(struct fm10k_intfc *interface)
 
 		/* start checking at the last ring to have pending Tx */
 		for (; i < interface->num_tx_queues; i++)
-			if (fm10k_get_tx_pending(interface->tx_ring[i]))
+			if (fm10k_get_tx_pending(interface->tx_ring[i], false))
 				break;
 
 		/* if all the queues are drained, we can break now */
@@ -1835,8 +1837,9 @@ static int fm10k_sw_init(struct fm10k_intfc *interface,
 	interface->tx_itr = FM10K_TX_ITR_DEFAULT;
 	interface->rx_itr = FM10K_ITR_ADAPTIVE | FM10K_RX_ITR_DEFAULT;
 
-	/* initialize vxlan_port list */
+	/* initialize udp port lists */
 	INIT_LIST_HEAD(&interface->vxlan_port);
+	INIT_LIST_HEAD(&interface->geneve_port);
 
 	netdev_rss_key_fill(rss_key, sizeof(rss_key));
 	memcpy(interface->rssrk, rss_key, sizeof(rss_key));
@@ -1950,9 +1953,18 @@ static int fm10k_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct fm10k_intfc *interface;
 	int err;
 
+	if (pdev->error_state != pci_channel_io_normal) {
+		dev_err(&pdev->dev,
+			"PCI device still in an error state. Unable to load...\n");
+		return -EIO;
+	}
+
 	err = pci_enable_device_mem(pdev);
-	if (err)
+	if (err) {
+		dev_err(&pdev->dev,
+			"PCI enable device failed: %d\n", err);
 		return err;
+	}
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
 	if (err)
@@ -2275,7 +2287,7 @@ static pci_ers_result_t fm10k_io_slot_reset(struct pci_dev *pdev)
 {
 	pci_ers_result_t result;
 
-	if (pci_enable_device_mem(pdev)) {
+	if (pci_reenable_device(pdev)) {
 		dev_err(&pdev->dev,
 			"Cannot re-enable PCI device after reset.\n");
 		result = PCI_ERS_RESULT_DISCONNECT;
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c
index 682299dd0ce4..23fb319fd2a0 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c
@@ -867,10 +867,6 @@ static s32 fm10k_iov_assign_default_mac_vlan_pf(struct fm10k_hw *hw,
 	vf_q_idx = fm10k_vf_queue_index(hw, vf_idx);
 	qmap_idx = qmap_stride * vf_idx;
 
-	/* MAP Tx queue back to 0 temporarily, and disable it */
-	fm10k_write_reg(hw, FM10K_TQMAP(qmap_idx), 0);
-	fm10k_write_reg(hw, FM10K_TXDCTL(vf_q_idx), 0);
-
 	/* Determine correct default VLAN ID. The FM10K_VLAN_OVERRIDE bit is
 	 * used here to indicate to the VF that it will not have privilege to
 	 * write VLAN_TABLE. All policy is enforced on the PF but this allows
@@ -886,9 +882,35 @@ static s32 fm10k_iov_assign_default_mac_vlan_pf(struct fm10k_hw *hw,
 	fm10k_tlv_attr_put_mac_vlan(msg, FM10K_MAC_VLAN_MSG_DEFAULT_MAC,
 				    vf_info->mac, vf_vid);
 
-	/* load onto outgoing mailbox, ignore any errors on enqueue */
-	if (vf_info->mbx.ops.enqueue_tx)
-		vf_info->mbx.ops.enqueue_tx(hw, &vf_info->mbx, msg);
+	/* Configure Queue control register with new VLAN ID. The TXQCTL
+	 * register is RO from the VF, so the PF must do this even in the
+	 * case of notifying the VF of a new VID via the mailbox.
+	 */
+	txqctl = ((u32)vf_vid << FM10K_TXQCTL_VID_SHIFT) &
+		 FM10K_TXQCTL_VID_MASK;
+	txqctl |= (vf_idx << FM10K_TXQCTL_TC_SHIFT) |
+		  FM10K_TXQCTL_VF | vf_idx;
+
+	for (i = 0; i < queues_per_pool; i++)
+		fm10k_write_reg(hw, FM10K_TXQCTL(vf_q_idx + i), txqctl);
+
+	/* try loading a message onto outgoing mailbox first */
+	if (vf_info->mbx.ops.enqueue_tx) {
+		err = vf_info->mbx.ops.enqueue_tx(hw, &vf_info->mbx, msg);
+		if (err != FM10K_MBX_ERR_NO_MBX)
+			return err;
+		err = 0;
+	}
+
+	/* If we aren't connected to a mailbox, this is most likely because
+	 * the VF driver is not running. It should thus be safe to re-map
+	 * queues and use the registers to pass the MAC address so that the VF
+	 * driver gets correct information during its initialization.
+	 */
+
+	/* MAP Tx queue back to 0 temporarily, and disable it */
+	fm10k_write_reg(hw, FM10K_TQMAP(qmap_idx), 0);
+	fm10k_write_reg(hw, FM10K_TXDCTL(vf_q_idx), 0);
 
 	/* verify ring has disabled before modifying base address registers */
 	txdctl = fm10k_read_reg(hw, FM10K_TXDCTL(vf_q_idx));
@@ -927,16 +949,6 @@ static s32 fm10k_iov_assign_default_mac_vlan_pf(struct fm10k_hw *hw,
 						   FM10K_TDLEN_ITR_SCALE_SHIFT);
 
 err_out:
-	/* configure Queue control register */
-	txqctl = ((u32)vf_vid << FM10K_TXQCTL_VID_SHIFT) &
-		 FM10K_TXQCTL_VID_MASK;
-	txqctl |= (vf_idx << FM10K_TXQCTL_TC_SHIFT) |
-		  FM10K_TXQCTL_VF | vf_idx;
-
-	/* assign VLAN ID */
-	for (i = 0; i < queues_per_pool; i++)
-		fm10k_write_reg(hw, FM10K_TXQCTL(vf_q_idx + i), txqctl);
-
 	/* restore the queue back to VF ownership */
 	fm10k_write_reg(hw, FM10K_TQMAP(qmap_idx), vf_q_idx);
 	return err;
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_type.h b/drivers/net/ethernet/intel/fm10k/fm10k_type.h
index f4e75c498287..6bb16c13d9d6 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_type.h
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_type.h
@@ -154,6 +154,7 @@ struct fm10k_hw;
 #define FM10K_DGLORTDEC_INNERRSS_ENABLE		0x08000000
 #define FM10K_TUNNEL_CFG	0x0040
 #define FM10K_TUNNEL_CFG_NVGRE_SHIFT		16
+#define FM10K_TUNNEL_CFG_GENEVE	0x0041
 #define FM10K_SWPRI_MAP(_n)	((_n) + 0x0050)
 #define FM10K_SWPRI_MAX		16
 #define FM10K_RSSRK(_n, _m)	(((_n) * 0x10) + (_m) + 0x0800)
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index 2a882916b4f6..2030d7c1dc94 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -65,76 +65,72 @@
 #include "i40e_dcb.h"
 
 /* Useful i40e defaults */
-#define I40E_MAX_VEB          16
-
-#define I40E_MAX_NUM_DESCRIPTORS      4096
-#define I40E_MAX_CSR_SPACE (4 * 1024 * 1024 - 64 * 1024)
-#define I40E_DEFAULT_NUM_DESCRIPTORS  512
-#define I40E_REQ_DESCRIPTOR_MULTIPLE  32
-#define I40E_MIN_NUM_DESCRIPTORS      64
-#define I40E_MIN_MSIX                 2
-#define I40E_DEFAULT_NUM_VMDQ_VSI     8 /* max 256 VSIs */
-#define I40E_MIN_VSI_ALLOC            51 /* LAN, ATR, FCOE, 32 VF, 16 VMDQ */
+#define I40E_MAX_VEB			16
+
+#define I40E_MAX_NUM_DESCRIPTORS	4096
+#define I40E_MAX_CSR_SPACE		(4 * 1024 * 1024 - 64 * 1024)
+#define I40E_DEFAULT_NUM_DESCRIPTORS	512
+#define I40E_REQ_DESCRIPTOR_MULTIPLE	32
+#define I40E_MIN_NUM_DESCRIPTORS	64
+#define I40E_MIN_MSIX			2
+#define I40E_DEFAULT_NUM_VMDQ_VSI	8 /* max 256 VSIs */
+#define I40E_MIN_VSI_ALLOC		83 /* LAN, ATR, FCOE, 64 VF */
 /* max 16 qps */
 #define i40e_default_queues_per_vmdq(pf) \
 		(((pf)->flags & I40E_FLAG_RSS_AQ_CAPABLE) ? 4 : 1)
-#define I40E_DEFAULT_QUEUES_PER_VF    4
-#define I40E_DEFAULT_QUEUES_PER_TC    1 /* should be a power of 2 */
+#define I40E_DEFAULT_QUEUES_PER_VF	4
+#define I40E_DEFAULT_QUEUES_PER_TC	1 /* should be a power of 2 */
 #define i40e_pf_get_max_q_per_tc(pf) \
 		(((pf)->flags & I40E_FLAG_128_QP_RSS_CAPABLE) ? 128 : 64)
-#define I40E_FDIR_RING                0
-#define I40E_FDIR_RING_COUNT          32
+#define I40E_FDIR_RING			0
+#define I40E_FDIR_RING_COUNT		32
 #ifdef I40E_FCOE
-#define I40E_DEFAULT_FCOE             8 /* default number of QPs for FCoE */
-#define I40E_MINIMUM_FCOE             1 /* minimum number of QPs for FCoE */
+#define I40E_DEFAULT_FCOE		8 /* default number of QPs for FCoE */
+#define I40E_MINIMUM_FCOE		1 /* minimum number of QPs for FCoE */
 #endif /* I40E_FCOE */
-#define I40E_MAX_AQ_BUF_SIZE          4096
-#define I40E_AQ_LEN                   256
-#define I40E_AQ_WORK_LIMIT            66 /* max number of VFs + a little */
-#define I40E_MAX_USER_PRIORITY        8
-#define I40E_DEFAULT_MSG_ENABLE       4
-#define I40E_QUEUE_WAIT_RETRY_LIMIT   10
-#define I40E_INT_NAME_STR_LEN        (IFNAMSIZ + 16)
+#define I40E_MAX_AQ_BUF_SIZE		4096
+#define I40E_AQ_LEN			256
+#define I40E_AQ_WORK_LIMIT		66 /* max number of VFs + a little */
+#define I40E_MAX_USER_PRIORITY		8
+#define I40E_DEFAULT_MSG_ENABLE		4
+#define I40E_QUEUE_WAIT_RETRY_LIMIT	10
+#define I40E_INT_NAME_STR_LEN		(IFNAMSIZ + 16)
 
 /* Ethtool Private Flags */
-#define	I40E_PRIV_FLAGS_MFP_FLAG		BIT(0)
-#define	I40E_PRIV_FLAGS_LINKPOLL_FLAG		BIT(1)
+#define I40E_PRIV_FLAGS_MFP_FLAG		BIT(0)
+#define I40E_PRIV_FLAGS_LINKPOLL_FLAG		BIT(1)
 #define I40E_PRIV_FLAGS_FD_ATR			BIT(2)
 #define I40E_PRIV_FLAGS_VEB_STATS		BIT(3)
 #define I40E_PRIV_FLAGS_HW_ATR_EVICT		BIT(4)
 #define I40E_PRIV_FLAGS_TRUE_PROMISC_SUPPORT	BIT(5)
 
-#define I40E_NVM_VERSION_LO_SHIFT  0
-#define I40E_NVM_VERSION_LO_MASK   (0xff << I40E_NVM_VERSION_LO_SHIFT)
-#define I40E_NVM_VERSION_HI_SHIFT  12
-#define I40E_NVM_VERSION_HI_MASK   (0xf << I40E_NVM_VERSION_HI_SHIFT)
-#define I40E_OEM_VER_BUILD_MASK    0xffff
-#define I40E_OEM_VER_PATCH_MASK    0xff
-#define I40E_OEM_VER_BUILD_SHIFT   8
-#define I40E_OEM_VER_SHIFT         24
+#define I40E_NVM_VERSION_LO_SHIFT	0
+#define I40E_NVM_VERSION_LO_MASK	(0xff << I40E_NVM_VERSION_LO_SHIFT)
+#define I40E_NVM_VERSION_HI_SHIFT	12
+#define I40E_NVM_VERSION_HI_MASK	(0xf << I40E_NVM_VERSION_HI_SHIFT)
+#define I40E_OEM_VER_BUILD_MASK		0xffff
+#define I40E_OEM_VER_PATCH_MASK		0xff
+#define I40E_OEM_VER_BUILD_SHIFT	8
+#define I40E_OEM_VER_SHIFT		24
 #define I40E_PHY_DEBUG_ALL \
 	(I40E_AQ_PHY_DEBUG_DISABLE_LINK_FW | \
 	I40E_AQ_PHY_DEBUG_DISABLE_ALL_LINK_FW)
 
 /* The values in here are decimal coded as hex as is the case in the NVM map*/
-#define I40E_CURRENT_NVM_VERSION_HI 0x2
-#define I40E_CURRENT_NVM_VERSION_LO 0x40
+#define I40E_CURRENT_NVM_VERSION_HI	0x2
+#define I40E_CURRENT_NVM_VERSION_LO	0x40
 
-/* magic for getting defines into strings */
-#define STRINGIFY(foo)  #foo
-#define XSTRINGIFY(bar) STRINGIFY(bar)
-
-#define I40E_RX_DESC(R, i)			\
+#define I40E_RX_DESC(R, i)	\
 	(&(((union i40e_32byte_rx_desc *)((R)->desc))[i]))
-#define I40E_TX_DESC(R, i)			\
+#define I40E_TX_DESC(R, i)	\
 	(&(((struct i40e_tx_desc *)((R)->desc))[i]))
-#define I40E_TX_CTXTDESC(R, i)			\
+#define I40E_TX_CTXTDESC(R, i)	\
 	(&(((struct i40e_tx_context_desc *)((R)->desc))[i]))
-#define I40E_TX_FDIRDESC(R, i)			\
+#define I40E_TX_FDIRDESC(R, i)	\
 	(&(((struct i40e_filter_program_desc *)((R)->desc))[i]))
 
 /* default to trying for four seconds */
-#define I40E_TRY_LINK_TIMEOUT (4 * HZ)
+#define I40E_TRY_LINK_TIMEOUT	(4 * HZ)
 
 /**
  * i40e_is_mac_710 - Return true if MAC is X710/XL710
@@ -199,9 +195,9 @@ struct i40e_lump_tracking {
 #define I40E_FDIR_BUFFER_HEAD_ROOM	32
 #define I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR (I40E_FDIR_BUFFER_HEAD_ROOM * 4)
 
-#define I40E_HKEY_ARRAY_SIZE ((I40E_PFQF_HKEY_MAX_INDEX + 1) * 4)
-#define I40E_HLUT_ARRAY_SIZE ((I40E_PFQF_HLUT_MAX_INDEX + 1) * 4)
-#define I40E_VF_HLUT_ARRAY_SIZE ((I40E_VFQF_HLUT1_MAX_INDEX + 1) * 4)
+#define I40E_HKEY_ARRAY_SIZE	((I40E_PFQF_HKEY_MAX_INDEX + 1) * 4)
+#define I40E_HLUT_ARRAY_SIZE	((I40E_PFQF_HLUT_MAX_INDEX + 1) * 4)
+#define I40E_VF_HLUT_ARRAY_SIZE	((I40E_VFQF_HLUT1_MAX_INDEX + 1) * 4)
 
 enum i40e_fd_stat_idx {
 	I40E_FD_STAT_ATR,
@@ -387,8 +383,8 @@ struct i40e_pf {
 	struct mutex switch_mutex;
 	u16 lan_vsi;       /* our default LAN VSI */
 	u16 lan_veb;       /* initial relay, if exists */
-#define I40E_NO_VEB   0xffff
-#define I40E_NO_VSI   0xffff
+#define I40E_NO_VEB	0xffff
+#define I40E_NO_VSI	0xffff
 	u16 next_vsi;      /* Next unallocated VSI - 0-based! */
 	struct i40e_vsi **vsi;
 	struct i40e_veb *veb[I40E_MAX_VEB];
@@ -423,8 +419,8 @@ struct i40e_pf {
 	 */
 	u16 dcbx_cap;
 
-	u32	fcoe_hmc_filt_num;
-	u32	fcoe_hmc_cntx_num;
+	u32 fcoe_hmc_filt_num;
+	u32 fcoe_hmc_cntx_num;
 	struct i40e_filter_control_settings filter_settings;
 
 	struct ptp_clock *ptp_clock;
@@ -470,10 +466,10 @@ struct i40e_mac_filter {
 struct i40e_veb {
 	struct i40e_pf *pf;
 	u16 idx;
-	u16 veb_idx;           /* index of VEB parent */
+	u16 veb_idx;		/* index of VEB parent */
 	u16 seid;
 	u16 uplink_seid;
-	u16 stats_idx;           /* index of VEB parent */
+	u16 stats_idx;		/* index of VEB parent */
 	u8  enabled_tc;
 	u16 bridge_mode;	/* Bridge Mode (VEB/VEPA) */
 	u16 flags;
@@ -534,12 +530,13 @@ struct i40e_vsi {
 	u32  promisc_threshold;
 
 	u16 work_limit;
-	u16 int_rate_limit;  /* value in usecs */
+	u16 int_rate_limit;	/* value in usecs */
+
+	u16 rss_table_size;	/* HW RSS table size */
+	u16 rss_size;		/* Allocated RSS queues */
+	u8  *rss_hkey_user;	/* User configured hash keys */
+	u8  *rss_lut_user;	/* User configured lookup table entries */
 
-	u16 rss_table_size; /* HW RSS table size */
-	u16 rss_size;       /* Allocated RSS queues */
-	u8  *rss_hkey_user; /* User configured hash keys */
-	u8  *rss_lut_user;  /* User configured lookup table entries */
 
 	u16 max_frame;
 	u16 rx_buf_len;
@@ -550,14 +547,14 @@ struct i40e_vsi {
 	int base_vector;
 	bool irqs_ready;
 
-	u16 seid;            /* HW index of this VSI (absolute index) */
-	u16 id;              /* VSI number */
+	u16 seid;		/* HW index of this VSI (absolute index) */
+	u16 id;			/* VSI number */
 	u16 uplink_seid;
 
-	u16 base_queue;      /* vsi's first queue in hw array */
-	u16 alloc_queue_pairs; /* Allocated Tx/Rx queues */
-	u16 req_queue_pairs; /* User requested queue pairs */
-	u16 num_queue_pairs; /* Used tx and rx pairs */
+	u16 base_queue;		/* vsi's first queue in hw array */
+	u16 alloc_queue_pairs;	/* Allocated Tx/Rx queues */
+	u16 req_queue_pairs;	/* User requested queue pairs */
+	u16 num_queue_pairs;	/* Used tx and rx pairs */
 	u16 num_desc;
 	enum i40e_vsi_type type;  /* VSI type, e.g., LAN, FCoE, etc */
 	s16 vf_id;		/* Virtual function ID for SRIOV VSIs */
@@ -576,19 +573,16 @@ struct i40e_vsi {
 	/* TC BW limit max quanta within VSI */
 	u8  bw_ets_max_quanta[I40E_MAX_TRAFFIC_CLASS];
 
-	struct i40e_pf *back;  /* Backreference to associated PF */
-	u16 idx;               /* index in pf->vsi[] */
-	u16 veb_idx;           /* index of VEB parent */
-	struct kobject *kobj;  /* sysfs object */
-	bool current_isup;     /* Sync 'link up' logging */
+	struct i40e_pf *back;	/* Backreference to associated PF */
+	u16 idx;		/* index in pf->vsi[] */
+	u16 veb_idx;		/* index of VEB parent */
+	struct kobject *kobj;	/* sysfs object */
+	bool current_isup;	/* Sync 'link up' logging */
 
 	void *priv;	/* client driver data reference. */
 
 	/* VSI specific handlers */
 	irqreturn_t (*irq_handler)(int irq, void *data);
-
-	/* current rxnfc data */
-	struct ethtool_rxnfc rxnfc; /* current rss hash opts */
 } ____cacheline_internodealigned_in_smp;
 
 struct i40e_netdev_priv {
@@ -707,6 +701,8 @@ void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags);
 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags);
 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size);
 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size);
+void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
+		       u16 rss_table_size, u16 rss_size);
 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id);
 void i40e_update_stats(struct i40e_vsi *vsi);
 void i40e_update_eth_stats(struct i40e_vsi *vsi);
@@ -714,8 +710,6 @@ struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi);
 int i40e_fetch_switch_configuration(struct i40e_pf *pf,
 				    bool printconfig);
 
-int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
-			     struct i40e_pf *pf, bool add);
 int i40e_add_del_fdir(struct i40e_vsi *vsi,
 		      struct i40e_fdir_filter *input, bool add);
 void i40e_fdir_check_and_reenable(struct i40e_pf *pf);
diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h b/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h
index 11cf1a5ebccf..67e396b2b347 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h
@@ -204,6 +204,9 @@ enum i40e_admin_queue_opc {
 	i40e_aqc_opc_suspend_port_tx				= 0x041B,
 	i40e_aqc_opc_resume_port_tx				= 0x041C,
 	i40e_aqc_opc_configure_partition_bw			= 0x041D,
+	/* hmc */
+	i40e_aqc_opc_query_hmc_resource_profile	= 0x0500,
+	i40e_aqc_opc_set_hmc_resource_profile	= 0x0501,
 
 	/* phy commands*/
 	i40e_aqc_opc_get_phy_abilities		= 0x0600,
@@ -450,13 +453,15 @@ I40E_CHECK_CMD_LENGTH(i40e_aqc_cppm_configuration);
 /* Set ARP Proxy command / response (indirect 0x0104) */
 struct i40e_aqc_arp_proxy_data {
 	__le16	command_flags;
-#define I40E_AQ_ARP_INIT_IPV4	0x0008
-#define I40E_AQ_ARP_UNSUP_CTL	0x0010
-#define I40E_AQ_ARP_ENA		0x0020
-#define I40E_AQ_ARP_ADD_IPV4	0x0040
-#define I40E_AQ_ARP_DEL_IPV4	0x0080
+#define I40E_AQ_ARP_INIT_IPV4	0x0800
+#define I40E_AQ_ARP_UNSUP_CTL	0x1000
+#define I40E_AQ_ARP_ENA		0x2000
+#define I40E_AQ_ARP_ADD_IPV4	0x4000
+#define I40E_AQ_ARP_DEL_IPV4	0x8000
 	__le16	table_id;
-	__le32	pfpm_proxyfc;
+	__le32	enabled_offloads;
+#define I40E_AQ_ARP_DIRECTED_OFFLOAD_ENABLE	0x00000020
+#define I40E_AQ_ARP_OFFLOAD_ENABLE		0x00000800
 	__le32	ip_addr;
 	u8	mac_addr[6];
 	u8	reserved[2];
@@ -471,17 +476,19 @@ struct i40e_aqc_ns_proxy_data {
 	__le16	table_idx_ipv6_0;
 	__le16	table_idx_ipv6_1;
 	__le16	control;
-#define I40E_AQ_NS_PROXY_ADD_0		0x0100
-#define I40E_AQ_NS_PROXY_DEL_0		0x0200
-#define I40E_AQ_NS_PROXY_ADD_1		0x0400
-#define I40E_AQ_NS_PROXY_DEL_1		0x0800
-#define I40E_AQ_NS_PROXY_ADD_IPV6_0	0x1000
-#define I40E_AQ_NS_PROXY_DEL_IPV6_0	0x2000
-#define I40E_AQ_NS_PROXY_ADD_IPV6_1	0x4000
-#define I40E_AQ_NS_PROXY_DEL_IPV6_1	0x8000
-#define I40E_AQ_NS_PROXY_COMMAND_SEQ	0x0001
-#define I40E_AQ_NS_PROXY_INIT_IPV6_TBL	0x0002
-#define I40E_AQ_NS_PROXY_INIT_MAC_TBL	0x0004
+#define I40E_AQ_NS_PROXY_ADD_0		0x0001
+#define I40E_AQ_NS_PROXY_DEL_0		0x0002
+#define I40E_AQ_NS_PROXY_ADD_1		0x0004
+#define I40E_AQ_NS_PROXY_DEL_1		0x0008
+#define I40E_AQ_NS_PROXY_ADD_IPV6_0	0x0010
+#define I40E_AQ_NS_PROXY_DEL_IPV6_0	0x0020
+#define I40E_AQ_NS_PROXY_ADD_IPV6_1	0x0040
+#define I40E_AQ_NS_PROXY_DEL_IPV6_1	0x0080
+#define I40E_AQ_NS_PROXY_COMMAND_SEQ	0x0100
+#define I40E_AQ_NS_PROXY_INIT_IPV6_TBL	0x0200
+#define I40E_AQ_NS_PROXY_INIT_MAC_TBL	0x0400
+#define I40E_AQ_NS_PROXY_OFFLOAD_ENABLE	0x0800
+#define I40E_AQ_NS_PROXY_DIRECTED_OFFLOAD_ENABLE	0x1000
 	u8	mac_addr_0[6];
 	u8	mac_addr_1[6];
 	u8	local_mac_addr[6];
@@ -1582,6 +1589,24 @@ struct i40e_aqc_configure_partition_bw_data {
 
 I40E_CHECK_STRUCT_LEN(0x22, i40e_aqc_configure_partition_bw_data);
 
+/* Get and set the active HMC resource profile and status.
+ * (direct 0x0500) and (direct 0x0501)
+ */
+struct i40e_aq_get_set_hmc_resource_profile {
+	u8	pm_profile;
+	u8	pe_vf_enabled;
+	u8	reserved[14];
+};
+
+I40E_CHECK_CMD_LENGTH(i40e_aq_get_set_hmc_resource_profile);
+
+enum i40e_aq_hmc_profile {
+	/* I40E_HMC_PROFILE_NO_CHANGE	= 0, reserved */
+	I40E_HMC_PROFILE_DEFAULT	= 1,
+	I40E_HMC_PROFILE_FAVOR_VF	= 2,
+	I40E_HMC_PROFILE_EQUAL		= 3,
+};
+
 /* Get PHY Abilities (indirect 0x0600) uses the generic indirect struct */
 
 /* set in param0 for get phy abilities to report qualified modules */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.c b/drivers/net/ethernet/intel/i40e/i40e_client.c
index e1370c556a3c..250db0b244b7 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_client.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_client.c
@@ -148,6 +148,11 @@ i40e_notify_client_of_vf_msg(struct i40e_vsi *vsi, u32 vf_id, u8 *msg, u16 len)
 					"Cannot locate client instance virtual channel receive routine\n");
 				continue;
 			}
+			if (!test_bit(__I40E_CLIENT_INSTANCE_OPENED,
+				      &cdev->state)) {
+				dev_dbg(&vsi->back->pdev->dev, "Client is not open, abort virtchnl_receive\n");
+				continue;
+			}
 			cdev->client->ops->virtchnl_receive(&cdev->lan_info,
 							    cdev->client,
 							    vf_id, msg, len);
@@ -181,6 +186,11 @@ void i40e_notify_client_of_l2_param_changes(struct i40e_vsi *vsi)
 					"Cannot locate client instance l2_param_change routine\n");
 				continue;
 			}
+			if (!test_bit(__I40E_CLIENT_INSTANCE_OPENED,
+				      &cdev->state)) {
+				dev_dbg(&vsi->back->pdev->dev, "Client is not open, abort l2 param change\n");
+				continue;
+			}
 			cdev->lan_info.params = params;
 			cdev->client->ops->l2_param_change(&cdev->lan_info,
 							   cdev->client,
@@ -199,6 +209,7 @@ void i40e_notify_client_of_l2_param_changes(struct i40e_vsi *vsi)
 void i40e_notify_client_of_netdev_open(struct i40e_vsi *vsi)
 {
 	struct i40e_client_instance *cdev;
+	int ret = 0;
 
 	if (!vsi)
 		return;
@@ -211,7 +222,14 @@ void i40e_notify_client_of_netdev_open(struct i40e_vsi *vsi)
 					"Cannot locate client instance open routine\n");
 				continue;
 			}
-			cdev->client->ops->open(&cdev->lan_info, cdev->client);
+			if (!(test_bit(__I40E_CLIENT_INSTANCE_OPENED,
+				       &cdev->state))) {
+				ret = cdev->client->ops->open(&cdev->lan_info,
+							      cdev->client);
+				if (!ret)
+					set_bit(__I40E_CLIENT_INSTANCE_OPENED,
+						&cdev->state);
+			}
 		}
 	}
 	mutex_unlock(&i40e_client_instance_mutex);
@@ -298,6 +316,11 @@ void i40e_notify_client_of_vf_reset(struct i40e_pf *pf, u32 vf_id)
 					"Cannot locate client instance VF reset routine\n");
 				continue;
 			}
+			if (!test_bit(__I40E_CLIENT_INSTANCE_OPENED,
+				      &cdev->state)) {
+				dev_dbg(&pf->pdev->dev, "Client is not open, abort vf-reset\n");
+				continue;
+			}
 			cdev->client->ops->vf_reset(&cdev->lan_info,
 						    cdev->client, vf_id);
 		}
@@ -328,6 +351,11 @@ void i40e_notify_client_of_vf_enable(struct i40e_pf *pf, u32 num_vfs)
 					"Cannot locate client instance VF enable routine\n");
 				continue;
 			}
+			if (!test_bit(__I40E_CLIENT_INSTANCE_OPENED,
+				      &cdev->state)) {
+				dev_dbg(&pf->pdev->dev, "Client is not open, abort vf-enable\n");
+				continue;
+			}
 			cdev->client->ops->vf_enable(&cdev->lan_info,
 						     cdev->client, num_vfs);
 		}
@@ -362,6 +390,11 @@ int i40e_vf_client_capable(struct i40e_pf *pf, u32 vf_id,
 					"Cannot locate client instance VF capability routine\n");
 				continue;
 			}
+			if (!test_bit(__I40E_CLIENT_INSTANCE_OPENED,
+				      &cdev->state)) {
+				dev_dbg(&pf->pdev->dev, "Client is not open, abort vf-capable\n");
+				continue;
+			}
 			capable = cdev->client->ops->vf_capable(&cdev->lan_info,
 								cdev->client,
 								vf_id);
@@ -407,12 +440,14 @@ struct i40e_vsi *i40e_vsi_lookup(struct i40e_pf *pf,
  * i40e_client_add_instance - add a client instance struct to the instance list
  * @pf: pointer to the board struct
  * @client: pointer to a client struct in the client list.
+ * @existing: if there was already an existing instance
  *
- * Returns cdev ptr on success, NULL on failure
+ * Returns cdev ptr on success or if already exists, NULL on failure
  **/
 static
 struct i40e_client_instance *i40e_client_add_instance(struct i40e_pf *pf,
-						      struct i40e_client *client)
+						     struct i40e_client *client,
+						     bool *existing)
 {
 	struct i40e_client_instance *cdev;
 	struct netdev_hw_addr *mac = NULL;
@@ -421,7 +456,7 @@ struct i40e_client_instance *i40e_client_add_instance(struct i40e_pf *pf,
 	mutex_lock(&i40e_client_instance_mutex);
 	list_for_each_entry(cdev, &i40e_client_instances, list) {
 		if ((cdev->lan_info.pf == pf) && (cdev->client == client)) {
-			cdev = NULL;
+			*existing = true;
 			goto out;
 		}
 	}
@@ -505,6 +540,7 @@ void i40e_client_subtask(struct i40e_pf *pf)
 {
 	struct i40e_client_instance *cdev;
 	struct i40e_client *client;
+	bool existing = false;
 	int ret = 0;
 
 	if (!(pf->flags & I40E_FLAG_SERVICE_CLIENT_REQUESTED))
@@ -528,19 +564,27 @@ void i40e_client_subtask(struct i40e_pf *pf)
 			/* check if L2 VSI is up, if not we are not ready */
 			if (test_bit(__I40E_DOWN, &pf->vsi[pf->lan_vsi]->state))
 				continue;
+		} else {
+			dev_warn(&pf->pdev->dev, "This client %s is being instanciated at probe\n",
+				 client->name);
 		}
 
 		/* Add the client instance to the instance list */
-		cdev = i40e_client_add_instance(pf, client);
+		cdev = i40e_client_add_instance(pf, client, &existing);
 		if (!cdev)
 			continue;
 
-		/* Also up the ref_cnt of no. of instances of this client */
-		atomic_inc(&client->ref_cnt);
-		dev_info(&pf->pdev->dev, "Added instance of Client %s to PF%d bus=0x%02x func=0x%02x\n",
-			 client->name, pf->hw.pf_id,
-			 pf->hw.bus.device, pf->hw.bus.func);
+		if (!existing) {
+			/* Also up the ref_cnt for no. of instances of this
+			 * client.
+			 */
+			atomic_inc(&client->ref_cnt);
+			dev_info(&pf->pdev->dev, "Added instance of Client %s to PF%d bus=0x%02x func=0x%02x\n",
+				 client->name, pf->hw.pf_id,
+				 pf->hw.bus.device, pf->hw.bus.func);
+		}
 
+		mutex_lock(&i40e_client_instance_mutex);
 		/* Send an Open request to the client */
 		atomic_inc(&cdev->ref_cnt);
 		if (client->ops && client->ops->open)
@@ -550,10 +594,12 @@ void i40e_client_subtask(struct i40e_pf *pf)
 			set_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state);
 		} else {
 			/* remove client instance */
+			mutex_unlock(&i40e_client_instance_mutex);
 			i40e_client_del_instance(pf, client);
 			atomic_dec(&client->ref_cnt);
 			continue;
 		}
+		mutex_unlock(&i40e_client_instance_mutex);
 	}
 	mutex_unlock(&i40e_client_mutex);
 }
@@ -588,7 +634,8 @@ int i40e_lan_add_device(struct i40e_pf *pf)
 		 pf->hw.pf_id, pf->hw.bus.device, pf->hw.bus.func);
 
 	/* Since in some cases register may have happened before a device gets
-	 * added, we can schedule a subtask to go initiate the clients.
+	 * added, we can schedule a subtask to go initiate the clients if
+	 * they can be launched at probe time.
 	 */
 	pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
 	i40e_service_event_schedule(pf);
@@ -635,7 +682,7 @@ int i40e_lan_del_device(struct i40e_pf *pf)
 static int i40e_client_release(struct i40e_client *client)
 {
 	struct i40e_client_instance *cdev, *tmp;
-	struct i40e_pf *pf = NULL;
+	struct i40e_pf *pf;
 	int ret = 0;
 
 	LIST_HEAD(cdevs_tmp);
@@ -645,12 +692,12 @@ static int i40e_client_release(struct i40e_client *client)
 		if (strncmp(cdev->client->name, client->name,
 			    I40E_CLIENT_STR_LENGTH))
 			continue;
+		pf = (struct i40e_pf *)cdev->lan_info.pf;
 		if (test_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state)) {
 			if (atomic_read(&cdev->ref_cnt) > 0) {
 				ret = I40E_ERR_NOT_READY;
 				goto out;
 			}
-			pf = (struct i40e_pf *)cdev->lan_info.pf;
 			if (client->ops && client->ops->close)
 				client->ops->close(&cdev->lan_info, client,
 						   false);
@@ -662,8 +709,7 @@ static int i40e_client_release(struct i40e_client *client)
 				 client->name, pf->hw.pf_id);
 		}
 		/* delete the client instance from the list */
-		list_del(&cdev->list);
-		list_add(&cdev->list, &cdevs_tmp);
+		list_move(&cdev->list, &cdevs_tmp);
 		atomic_dec(&client->ref_cnt);
 		dev_info(&pf->pdev->dev, "Deleted client instance of Client %s\n",
 			 client->name);
@@ -792,7 +838,8 @@ static int i40e_client_setup_qvlist(struct i40e_info *ldev,
 			wr32(hw, I40E_PFINT_AEQCTL, reg);
 		}
 	}
-
+	/* Mitigate sync problems with iwarp VF driver */
+	i40e_flush(hw);
 	return 0;
 err:
 	kfree(ldev->qvlist_info);
@@ -990,7 +1037,6 @@ int i40e_unregister_client(struct i40e_client *client)
 	if (!i40e_client_is_registered(client)) {
 		pr_info("i40e: Client %s has not been registered\n",
 			client->name);
-		mutex_unlock(&i40e_client_mutex);
 		ret = -ENODEV;
 		goto out;
 	}
diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.h b/drivers/net/ethernet/intel/i40e/i40e_client.h
index a4601d97fb24..38a6c36a6a0e 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_client.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_client.h
@@ -36,9 +36,9 @@
 #define I40E_CLIENT_VERSION_MINOR 01
 #define I40E_CLIENT_VERSION_BUILD 00
 #define I40E_CLIENT_VERSION_STR     \
-	XSTRINGIFY(I40E_CLIENT_VERSION_MAJOR) "." \
-	XSTRINGIFY(I40E_CLIENT_VERSION_MINOR) "." \
-	XSTRINGIFY(I40E_CLIENT_VERSION_BUILD)
+	__stringify(I40E_CLIENT_VERSION_MAJOR) "." \
+	__stringify(I40E_CLIENT_VERSION_MINOR) "." \
+	__stringify(I40E_CLIENT_VERSION_BUILD)
 
 struct i40e_client_version {
 	u8 major;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
index 05cf9a719bab..0c1875b5b16d 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
@@ -1054,6 +1054,7 @@ static ssize_t i40e_dbg_command_write(struct file *filp,
 			struct i40e_dcbx_config *r_cfg =
 						&pf->hw.remote_dcbx_config;
 			int i, ret;
+			u32 switch_id;
 
 			bw_data = kzalloc(sizeof(
 				    struct i40e_aqc_query_port_ets_config_resp),
@@ -1063,8 +1064,12 @@ static ssize_t i40e_dbg_command_write(struct file *filp,
 				goto command_write_done;
 			}
 
+			vsi = pf->vsi[pf->lan_vsi];
+			switch_id =
+				vsi->info.switch_id & I40E_AQ_VSI_SW_ID_MASK;
+
 			ret = i40e_aq_query_port_ets_config(&pf->hw,
-							    pf->mac_seid,
+							    switch_id,
 							    bw_data, NULL);
 			if (ret) {
 				dev_info(&pf->pdev->dev,
@@ -1425,84 +1430,6 @@ static ssize_t i40e_dbg_command_write(struct file *filp,
 		buff = NULL;
 		kfree(desc);
 		desc = NULL;
-	} else if ((strncmp(cmd_buf, "add fd_filter", 13) == 0) ||
-		   (strncmp(cmd_buf, "rem fd_filter", 13) == 0)) {
-		struct i40e_fdir_filter fd_data;
-		u16 packet_len, i, j = 0;
-		char *asc_packet;
-		u8 *raw_packet;
-		bool add = false;
-		int ret;
-
-		if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
-			goto command_write_done;
-
-		if (strncmp(cmd_buf, "add", 3) == 0)
-			add = true;
-
-		if (add && (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED))
-			goto command_write_done;
-
-		asc_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE,
-				     GFP_KERNEL);
-		if (!asc_packet)
-			goto command_write_done;
-
-		raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE,
-				     GFP_KERNEL);
-
-		if (!raw_packet) {
-			kfree(asc_packet);
-			asc_packet = NULL;
-			goto command_write_done;
-		}
-
-		cnt = sscanf(&cmd_buf[13],
-			     "%hx %2hhx %2hhx %hx %2hhx %2hhx %hx %x %hd %511s",
-			     &fd_data.q_index,
-			     &fd_data.flex_off, &fd_data.pctype,
-			     &fd_data.dest_vsi, &fd_data.dest_ctl,
-			     &fd_data.fd_status, &fd_data.cnt_index,
-			     &fd_data.fd_id, &packet_len, asc_packet);
-		if (cnt != 10) {
-			dev_info(&pf->pdev->dev,
-				 "program fd_filter: bad command string, cnt=%d\n",
-				 cnt);
-			kfree(asc_packet);
-			asc_packet = NULL;
-			kfree(raw_packet);
-			goto command_write_done;
-		}
-
-		/* fix packet length if user entered 0 */
-		if (packet_len == 0)
-			packet_len = I40E_FDIR_MAX_RAW_PACKET_SIZE;
-
-		/* make sure to check the max as well */
-		packet_len = min_t(u16,
-				   packet_len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
-
-		for (i = 0; i < packet_len; i++) {
-			cnt = sscanf(&asc_packet[j], "%2hhx ", &raw_packet[i]);
-			if (!cnt)
-				break;
-			j += 3;
-		}
-		dev_info(&pf->pdev->dev, "FD raw packet dump\n");
-		print_hex_dump(KERN_INFO, "FD raw packet: ",
-			       DUMP_PREFIX_OFFSET, 16, 1,
-			       raw_packet, packet_len, true);
-		ret = i40e_program_fdir_filter(&fd_data, raw_packet, pf, add);
-		if (!ret) {
-			dev_info(&pf->pdev->dev, "Filter command send Status : Success\n");
-		} else {
-			dev_info(&pf->pdev->dev,
-				 "Filter command send failed %d\n", ret);
-		}
-		kfree(raw_packet);
-		raw_packet = NULL;
-		kfree(asc_packet);
-		asc_packet = NULL;
 	} else if (strncmp(cmd_buf, "fd current cnt", 14) == 0) {
 		dev_info(&pf->pdev->dev, "FD current total filter count for this interface: %d\n",
 			 i40e_get_current_fd_count(pf));
@@ -1727,8 +1654,6 @@ static ssize_t i40e_dbg_command_write(struct file *filp,
 		dev_info(&pf->pdev->dev, "  globr\n");
 		dev_info(&pf->pdev->dev, "  send aq_cmd <flags> <opcode> <datalen> <retval> <cookie_h> <cookie_l> <param0> <param1> <param2> <param3>\n");
 		dev_info(&pf->pdev->dev, "  send indirect aq_cmd <flags> <opcode> <datalen> <retval> <cookie_h> <cookie_l> <param0> <param1> <param2> <param3> <buffer_len>\n");
-		dev_info(&pf->pdev->dev, "  add fd_filter <dest q_index> <flex_off> <pctype> <dest_vsi> <dest_ctl> <fd_status> <cnt_index> <fd_id> <packet_len> <packet>\n");
-		dev_info(&pf->pdev->dev, "  rem fd_filter <dest q_index> <flex_off> <pctype> <dest_vsi> <dest_ctl> <fd_status> <cnt_index> <fd_id> <packet_len> <packet>\n");
 		dev_info(&pf->pdev->dev, "  fd current cnt");
 		dev_info(&pf->pdev->dev, "  lldp start\n");
 		dev_info(&pf->pdev->dev, "  lldp stop\n");
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index c912e041d102..92bc8846f1ba 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -1560,13 +1560,13 @@ static void i40e_get_strings(struct net_device *netdev, u32 stringset,
 		}
 #endif
 		for (i = 0; i < vsi->num_queue_pairs; i++) {
-			snprintf(p, ETH_GSTRING_LEN, "tx-%u.tx_packets", i);
+			snprintf(p, ETH_GSTRING_LEN, "tx-%d.tx_packets", i);
 			p += ETH_GSTRING_LEN;
-			snprintf(p, ETH_GSTRING_LEN, "tx-%u.tx_bytes", i);
+			snprintf(p, ETH_GSTRING_LEN, "tx-%d.tx_bytes", i);
 			p += ETH_GSTRING_LEN;
-			snprintf(p, ETH_GSTRING_LEN, "rx-%u.rx_packets", i);
+			snprintf(p, ETH_GSTRING_LEN, "rx-%d.rx_packets", i);
 			p += ETH_GSTRING_LEN;
-			snprintf(p, ETH_GSTRING_LEN, "rx-%u.rx_bytes", i);
+			snprintf(p, ETH_GSTRING_LEN, "rx-%d.rx_bytes", i);
 			p += ETH_GSTRING_LEN;
 		}
 		if (vsi != pf->vsi[pf->lan_vsi] || pf->hw.partition_id != 1)
@@ -1581,16 +1581,16 @@ static void i40e_get_strings(struct net_device *netdev, u32 stringset,
 			}
 			for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 				snprintf(p, ETH_GSTRING_LEN,
-					 "veb.tc_%u_tx_packets", i);
+					 "veb.tc_%d_tx_packets", i);
 				p += ETH_GSTRING_LEN;
 				snprintf(p, ETH_GSTRING_LEN,
-					 "veb.tc_%u_tx_bytes", i);
+					 "veb.tc_%d_tx_bytes", i);
 				p += ETH_GSTRING_LEN;
 				snprintf(p, ETH_GSTRING_LEN,
-					 "veb.tc_%u_rx_packets", i);
+					 "veb.tc_%d_rx_packets", i);
 				p += ETH_GSTRING_LEN;
 				snprintf(p, ETH_GSTRING_LEN,
-					 "veb.tc_%u_rx_bytes", i);
+					 "veb.tc_%d_rx_bytes", i);
 				p += ETH_GSTRING_LEN;
 			}
 		}
@@ -1601,23 +1601,23 @@ static void i40e_get_strings(struct net_device *netdev, u32 stringset,
 		}
 		for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
 			snprintf(p, ETH_GSTRING_LEN,
-				 "port.tx_priority_%u_xon", i);
+				 "port.tx_priority_%d_xon", i);
 			p += ETH_GSTRING_LEN;
 			snprintf(p, ETH_GSTRING_LEN,
-				 "port.tx_priority_%u_xoff", i);
+				 "port.tx_priority_%d_xoff", i);
 			p += ETH_GSTRING_LEN;
 		}
 		for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
 			snprintf(p, ETH_GSTRING_LEN,
-				 "port.rx_priority_%u_xon", i);
+				 "port.rx_priority_%d_xon", i);
 			p += ETH_GSTRING_LEN;
 			snprintf(p, ETH_GSTRING_LEN,
-				 "port.rx_priority_%u_xoff", i);
+				 "port.rx_priority_%d_xoff", i);
 			p += ETH_GSTRING_LEN;
 		}
 		for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
 			snprintf(p, ETH_GSTRING_LEN,
-				 "port.rx_priority_%u_xon_2_xoff", i);
+				 "port.rx_priority_%d_xon_2_xoff", i);
 			p += ETH_GSTRING_LEN;
 		}
 		/* BUG_ON(p - data != I40E_STATS_LEN * ETH_GSTRING_LEN); */
@@ -1970,11 +1970,22 @@ static int i40e_set_phys_id(struct net_device *netdev,
  * 125us (8000 interrupts per second) == ITR(62)
  */
 
+/**
+ * __i40e_get_coalesce - get per-queue coalesce settings
+ * @netdev: the netdev to check
+ * @ec: ethtool coalesce data structure
+ * @queue: which queue to pick
+ *
+ * Gets the per-queue settings for coalescence. Specifically Rx and Tx usecs
+ * are per queue. If queue is <0 then we default to queue 0 as the
+ * representative value.
+ **/
 static int __i40e_get_coalesce(struct net_device *netdev,
 			       struct ethtool_coalesce *ec,
 			       int queue)
 {
 	struct i40e_netdev_priv *np = netdev_priv(netdev);
+	struct i40e_ring *rx_ring, *tx_ring;
 	struct i40e_vsi *vsi = np->vsi;
 
 	ec->tx_max_coalesced_frames_irq = vsi->work_limit;
@@ -1989,14 +2000,18 @@ static int __i40e_get_coalesce(struct net_device *netdev,
 		return -EINVAL;
 	}
 
-	if (ITR_IS_DYNAMIC(vsi->rx_rings[queue]->rx_itr_setting))
+	rx_ring = vsi->rx_rings[queue];
+	tx_ring = vsi->tx_rings[queue];
+
+	if (ITR_IS_DYNAMIC(rx_ring->rx_itr_setting))
 		ec->use_adaptive_rx_coalesce = 1;
 
-	if (ITR_IS_DYNAMIC(vsi->tx_rings[queue]->tx_itr_setting))
+	if (ITR_IS_DYNAMIC(tx_ring->tx_itr_setting))
 		ec->use_adaptive_tx_coalesce = 1;
 
-	ec->rx_coalesce_usecs = vsi->rx_rings[queue]->rx_itr_setting & ~I40E_ITR_DYNAMIC;
-	ec->tx_coalesce_usecs = vsi->tx_rings[queue]->tx_itr_setting & ~I40E_ITR_DYNAMIC;
+	ec->rx_coalesce_usecs = rx_ring->rx_itr_setting & ~I40E_ITR_DYNAMIC;
+	ec->tx_coalesce_usecs = tx_ring->tx_itr_setting & ~I40E_ITR_DYNAMIC;
+
 
 	/* we use the _usecs_high to store/set the interrupt rate limit
 	 * that the hardware supports, that almost but not quite
@@ -2010,18 +2025,44 @@ static int __i40e_get_coalesce(struct net_device *netdev,
 	return 0;
 }
 
+/**
+ * i40e_get_coalesce - get a netdev's coalesce settings
+ * @netdev: the netdev to check
+ * @ec: ethtool coalesce data structure
+ *
+ * Gets the coalesce settings for a particular netdev. Note that if user has
+ * modified per-queue settings, this only guarantees to represent queue 0. See
+ * __i40e_get_coalesce for more details.
+ **/
 static int i40e_get_coalesce(struct net_device *netdev,
 			     struct ethtool_coalesce *ec)
 {
 	return __i40e_get_coalesce(netdev, ec, -1);
 }
 
+/**
+ * i40e_get_per_queue_coalesce - gets coalesce settings for particular queue
+ * @netdev: netdev structure
+ * @ec: ethtool's coalesce settings
+ * @queue: the particular queue to read
+ *
+ * Will read a specific queue's coalesce settings
+ **/
 static int i40e_get_per_queue_coalesce(struct net_device *netdev, u32 queue,
 				       struct ethtool_coalesce *ec)
 {
 	return __i40e_get_coalesce(netdev, ec, queue);
 }
 
+/**
+ * i40e_set_itr_per_queue - set ITR values for specific queue
+ * @vsi: the VSI to set values for
+ * @ec: coalesce settings from ethtool
+ * @queue: the queue to modify
+ *
+ * Change the ITR settings for a specific queue.
+ **/
+
 static void i40e_set_itr_per_queue(struct i40e_vsi *vsi,
 				   struct ethtool_coalesce *ec,
 				   int queue)
@@ -2060,6 +2101,14 @@ static void i40e_set_itr_per_queue(struct i40e_vsi *vsi,
 	i40e_flush(hw);
 }
 
+/**
+ * __i40e_set_coalesce - set coalesce settings for particular queue
+ * @netdev: the netdev to change
+ * @ec: ethtool coalesce settings
+ * @queue: the queue to change
+ *
+ * Sets the coalesce settings for a particular queue.
+ **/
 static int __i40e_set_coalesce(struct net_device *netdev,
 			       struct ethtool_coalesce *ec,
 			       int queue)
@@ -2120,12 +2169,27 @@ static int __i40e_set_coalesce(struct net_device *netdev,
 	return 0;
 }
 
+/**
+ * i40e_set_coalesce - set coalesce settings for every queue on the netdev
+ * @netdev: the netdev to change
+ * @ec: ethtool coalesce settings
+ *
+ * This will set each queue to the same coalesce settings.
+ **/
 static int i40e_set_coalesce(struct net_device *netdev,
 			     struct ethtool_coalesce *ec)
 {
 	return __i40e_set_coalesce(netdev, ec, -1);
 }
 
+/**
+ * i40e_set_per_queue_coalesce - set specific queue's coalesce settings
+ * @netdev: the netdev to change
+ * @ec: ethtool's coalesce settings
+ * @queue: the queue to change
+ *
+ * Sets the specified queue's coalesce settings.
+ **/
 static int i40e_set_per_queue_coalesce(struct net_device *netdev, u32 queue,
 				       struct ethtool_coalesce *ec)
 {
@@ -2141,41 +2205,72 @@ static int i40e_set_per_queue_coalesce(struct net_device *netdev, u32 queue,
  **/
 static int i40e_get_rss_hash_opts(struct i40e_pf *pf, struct ethtool_rxnfc *cmd)
 {
+	struct i40e_hw *hw = &pf->hw;
+	u8 flow_pctype = 0;
+	u64 i_set = 0;
+
 	cmd->data = 0;
 
-	if (pf->vsi[pf->lan_vsi]->rxnfc.data != 0) {
-		cmd->data = pf->vsi[pf->lan_vsi]->rxnfc.data;
-		cmd->flow_type = pf->vsi[pf->lan_vsi]->rxnfc.flow_type;
-		return 0;
-	}
-	/* Report default options for RSS on i40e */
 	switch (cmd->flow_type) {
 	case TCP_V4_FLOW:
+		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
+		break;
 	case UDP_V4_FLOW:
-		cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
-	/* fall through to add IP fields */
+		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
+		break;
+	case TCP_V6_FLOW:
+		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
+		break;
+	case UDP_V6_FLOW:
+		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
+		break;
 	case SCTP_V4_FLOW:
 	case AH_ESP_V4_FLOW:
 	case AH_V4_FLOW:
 	case ESP_V4_FLOW:
 	case IPV4_FLOW:
-		cmd->data |= RXH_IP_SRC | RXH_IP_DST;
-		break;
-	case TCP_V6_FLOW:
-	case UDP_V6_FLOW:
-		cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
-	/* fall through to add IP fields */
 	case SCTP_V6_FLOW:
 	case AH_ESP_V6_FLOW:
 	case AH_V6_FLOW:
 	case ESP_V6_FLOW:
 	case IPV6_FLOW:
+		/* Default is src/dest for IP, no matter the L4 hashing */
 		cmd->data |= RXH_IP_SRC | RXH_IP_DST;
 		break;
 	default:
 		return -EINVAL;
 	}
 
+	/* Read flow based hash input set register */
+	if (flow_pctype) {
+		i_set = (u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0,
+					      flow_pctype)) |
+			((u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1,
+					       flow_pctype)) << 32);
+	}
+
+	/* Process bits of hash input set */
+	if (i_set) {
+		if (i_set & I40E_L4_SRC_MASK)
+			cmd->data |= RXH_L4_B_0_1;
+		if (i_set & I40E_L4_DST_MASK)
+			cmd->data |= RXH_L4_B_2_3;
+
+		if (cmd->flow_type == TCP_V4_FLOW ||
+		    cmd->flow_type == UDP_V4_FLOW) {
+			if (i_set & I40E_L3_SRC_MASK)
+				cmd->data |= RXH_IP_SRC;
+			if (i_set & I40E_L3_DST_MASK)
+				cmd->data |= RXH_IP_DST;
+		} else if (cmd->flow_type == TCP_V6_FLOW ||
+			  cmd->flow_type == UDP_V6_FLOW) {
+			if (i_set & I40E_L3_V6_SRC_MASK)
+				cmd->data |= RXH_IP_SRC;
+			if (i_set & I40E_L3_V6_DST_MASK)
+				cmd->data |= RXH_IP_DST;
+		}
+	}
+
 	return 0;
 }
 
@@ -2318,6 +2413,51 @@ static int i40e_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,
 }
 
 /**
+ * i40e_get_rss_hash_bits - Read RSS Hash bits from register
+ * @nfc: pointer to user request
+ * @i_setc bits currently set
+ *
+ * Returns value of bits to be set per user request
+ **/
+static u64 i40e_get_rss_hash_bits(struct ethtool_rxnfc *nfc, u64 i_setc)
+{
+	u64 i_set = i_setc;
+	u64 src_l3 = 0, dst_l3 = 0;
+
+	if (nfc->data & RXH_L4_B_0_1)
+		i_set |= I40E_L4_SRC_MASK;
+	else
+		i_set &= ~I40E_L4_SRC_MASK;
+	if (nfc->data & RXH_L4_B_2_3)
+		i_set |= I40E_L4_DST_MASK;
+	else
+		i_set &= ~I40E_L4_DST_MASK;
+
+	if (nfc->flow_type == TCP_V6_FLOW || nfc->flow_type == UDP_V6_FLOW) {
+		src_l3 = I40E_L3_V6_SRC_MASK;
+		dst_l3 = I40E_L3_V6_DST_MASK;
+	} else if (nfc->flow_type == TCP_V4_FLOW ||
+		  nfc->flow_type == UDP_V4_FLOW) {
+		src_l3 = I40E_L3_SRC_MASK;
+		dst_l3 = I40E_L3_DST_MASK;
+	} else {
+		/* Any other flow type are not supported here */
+		return i_set;
+	}
+
+	if (nfc->data & RXH_IP_SRC)
+		i_set |= src_l3;
+	else
+		i_set &= ~src_l3;
+	if (nfc->data & RXH_IP_DST)
+		i_set |= dst_l3;
+	else
+		i_set &= ~dst_l3;
+
+	return i_set;
+}
+
+/**
  * i40e_set_rss_hash_opt - Enable/Disable flow types for RSS hash
  * @pf: pointer to the physical function struct
  * @cmd: ethtool rxnfc command
@@ -2329,6 +2469,8 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc)
 	struct i40e_hw *hw = &pf->hw;
 	u64 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
 		   ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
+	u8 flow_pctype = 0;
+	u64 i_set, i_setc;
 
 	/* RSS does not support anything other than hashing
 	 * to queues on src and dst IPs and ports
@@ -2337,75 +2479,39 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc)
 			  RXH_L4_B_0_1 | RXH_L4_B_2_3))
 		return -EINVAL;
 
-	/* We need at least the IP SRC and DEST fields for hashing */
-	if (!(nfc->data & RXH_IP_SRC) ||
-	    !(nfc->data & RXH_IP_DST))
-		return -EINVAL;
-
 	switch (nfc->flow_type) {
 	case TCP_V4_FLOW:
-		switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
-		case 0:
-			return -EINVAL;
-		case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
-			if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
-				hena |=
-			   BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
-
-			hena |= BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
-			break;
-		default:
-			return -EINVAL;
-		}
+		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
+		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+			hena |=
+			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
 		break;
 	case TCP_V6_FLOW:
-		switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
-		case 0:
-			return -EINVAL;
-		case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
-			if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
-				hena |=
-			   BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
-
-			hena |= BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
-			break;
-		default:
-			return -EINVAL;
-		}
+		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
+		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+			hena |=
+			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
+		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+			hena |=
+			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
 		break;
 	case UDP_V4_FLOW:
-		switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
-		case 0:
-			return -EINVAL;
-		case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
-			if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
-				hena |=
-			    BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
-			    BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
-
-			hena |= (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
-				 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4));
-			break;
-		default:
-			return -EINVAL;
-		}
+		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
+		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+			hena |=
+			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
+			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
+
+		hena |= BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4);
 		break;
 	case UDP_V6_FLOW:
-		switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
-		case 0:
-			return -EINVAL;
-		case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
-			if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
-				hena |=
-			    BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
-			    BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
-
-			hena |= (BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
-				 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6));
-			break;
-		default:
-			return -EINVAL;
-		}
+		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
+		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+			hena |=
+			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
+			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
+
+		hena |= BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6);
 		break;
 	case AH_ESP_V4_FLOW:
 	case AH_V4_FLOW:
@@ -2437,13 +2543,23 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc)
 		return -EINVAL;
 	}
 
+	if (flow_pctype) {
+		i_setc = (u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0,
+					       flow_pctype)) |
+			((u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1,
+					       flow_pctype)) << 32);
+		i_set = i40e_get_rss_hash_bits(nfc, i_setc);
+		i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_pctype),
+				  (u32)i_set);
+		i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_pctype),
+				  (u32)(i_set >> 32));
+		hena |= BIT_ULL(flow_pctype);
+	}
+
 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
 	i40e_flush(hw);
 
-	/* Save setting for future output/update */
-	pf->vsi[pf->lan_vsi]->rxnfc = *nfc;
-
 	return 0;
 }
 
@@ -2744,11 +2860,15 @@ static void i40e_get_channels(struct net_device *dev,
 static int i40e_set_channels(struct net_device *dev,
 			      struct ethtool_channels *ch)
 {
+	const u8 drop = I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET;
 	struct i40e_netdev_priv *np = netdev_priv(dev);
 	unsigned int count = ch->combined_count;
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
+	struct i40e_fdir_filter *rule;
+	struct hlist_node *node2;
 	int new_count;
+	int err = 0;
 
 	/* We do not support setting channels for any other VSI at present */
 	if (vsi->type != I40E_VSI_MAIN)
@@ -2766,6 +2886,26 @@ static int i40e_set_channels(struct net_device *dev,
 	if (count > i40e_max_channels(vsi))
 		return -EINVAL;
 
+	/* verify that the number of channels does not invalidate any current
+	 * flow director rules
+	 */
+	hlist_for_each_entry_safe(rule, node2,
+				  &pf->fdir_filter_list, fdir_node) {
+		if (rule->dest_ctl != drop && count <= rule->q_index) {
+			dev_warn(&pf->pdev->dev,
+				 "Existing user defined filter %d assigns flow to queue %d\n",
+				 rule->fd_id, rule->q_index);
+			err = -EINVAL;
+		}
+	}
+
+	if (err) {
+		dev_err(&pf->pdev->dev,
+			"Existing filter rules must be deleted to reduce combined channel count to %d\n",
+			count);
+		return err;
+	}
+
 	/* update feature limits from largest to smallest supported values */
 	/* TODO: Flow director limit, DCB etc */
 
@@ -2846,15 +2986,13 @@ static int i40e_set_rxfh(struct net_device *netdev, const u32 *indir,
 {
 	struct i40e_netdev_priv *np = netdev_priv(netdev);
 	struct i40e_vsi *vsi = np->vsi;
+	struct i40e_pf *pf = vsi->back;
 	u8 *seed = NULL;
 	u16 i;
 
 	if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
 		return -EOPNOTSUPP;
 
-	if (!indir)
-		return 0;
-
 	if (key) {
 		if (!vsi->rss_hkey_user) {
 			vsi->rss_hkey_user = kzalloc(I40E_HKEY_ARRAY_SIZE,
@@ -2872,8 +3010,12 @@ static int i40e_set_rxfh(struct net_device *netdev, const u32 *indir,
 	}
 
 	/* Each 32 bits pointed by 'indir' is stored with a lut entry */
-	for (i = 0; i < I40E_HLUT_ARRAY_SIZE; i++)
-		vsi->rss_lut_user[i] = (u8)(indir[i]);
+	if (indir)
+		for (i = 0; i < I40E_HLUT_ARRAY_SIZE; i++)
+			vsi->rss_lut_user[i] = (u8)(indir[i]);
+	else
+		i40e_fill_rss_lut(pf, vsi->rss_lut_user, I40E_HLUT_ARRAY_SIZE,
+				  vsi->rss_size);
 
 	return i40e_config_rss(vsi, seed, vsi->rss_lut_user,
 			       I40E_HLUT_ARRAY_SIZE);
@@ -2943,6 +3085,9 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 	} else {
 		pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 		pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED;
+
+		/* flush current ATR settings */
+		set_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);
 	}
 
 	if ((flags & I40E_PRIV_FLAGS_VEB_STATS) &&
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 81c99e1be708..ac1faee2a5b8 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -41,7 +41,7 @@ static const char i40e_driver_string[] =
 
 #define DRV_VERSION_MAJOR 1
 #define DRV_VERSION_MINOR 6
-#define DRV_VERSION_BUILD 11
+#define DRV_VERSION_BUILD 16
 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
 	     __stringify(DRV_VERSION_MINOR) "." \
 	     __stringify(DRV_VERSION_BUILD)    DRV_KERN
@@ -57,8 +57,6 @@ static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
 static int i40e_setup_misc_vector(struct i40e_pf *pf);
 static void i40e_determine_queue_usage(struct i40e_pf *pf);
 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
-static void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
-			      u16 rss_table_size, u16 rss_size);
 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
 
@@ -527,6 +525,7 @@ void i40e_pf_reset_stats(struct i40e_pf *pf)
 			pf->veb[i]->stat_offsets_loaded = false;
 		}
 	}
+	pf->hw_csum_rx_error = 0;
 }
 
 /**
@@ -1316,7 +1315,7 @@ static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
 	element.vlan_tag = 0;
 	/* ...and some firmware does it this way. */
 	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
-			I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
+			I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
 	i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
 }
 
@@ -1909,7 +1908,7 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
 			ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
 			if (f->vlan == I40E_VLAN_ANY) {
 				del_list[num_del].vlan_tag = 0;
-				cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
+				cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
 			} else {
 				del_list[num_del].vlan_tag =
 					cpu_to_le16((u16)(f->vlan));
@@ -4554,23 +4553,38 @@ static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
  **/
 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
 {
+	int i, tc_unused = 0;
 	u8 num_tc = 0;
-	int i;
+	u8 ret = 0;
 
 	/* Scan the ETS Config Priority Table to find
 	 * traffic class enabled for a given priority
-	 * and use the traffic class index to get the
-	 * number of traffic classes enabled
+	 * and create a bitmask of enabled TCs
 	 */
-	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
-		if (dcbcfg->etscfg.prioritytable[i] > num_tc)
-			num_tc = dcbcfg->etscfg.prioritytable[i];
-	}
+	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
+		num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
 
-	/* Traffic class index starts from zero so
-	 * increment to return the actual count
+	/* Now scan the bitmask to check for
+	 * contiguous TCs starting with TC0
 	 */
-	return num_tc + 1;
+	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
+		if (num_tc & BIT(i)) {
+			if (!tc_unused) {
+				ret++;
+			} else {
+				pr_err("Non-contiguous TC - Disabling DCB\n");
+				return 1;
+			}
+		} else {
+			tc_unused = 1;
+		}
+	}
+
+	/* There is always at least TC0 */
+	if (!ret)
+		ret = 1;
+
+	return ret;
 }
 
 /**
@@ -4601,7 +4615,7 @@ static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
-	u8 i, enabled_tc;
+	u8 i, enabled_tc = 1;
 	u8 num_tc = 0;
 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
 
@@ -4619,8 +4633,6 @@ static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
 	else
 		return 1; /* Only TC0 */
 
-	/* At least have TC0 */
-	enabled_tc = (enabled_tc ? enabled_tc : 0x1);
 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 		if (enabled_tc & BIT(i))
 			num_tc++;
@@ -5098,9 +5110,13 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf)
 				       DCB_CAP_DCBX_VER_IEEE;
 
 			pf->flags |= I40E_FLAG_DCB_CAPABLE;
-			/* Enable DCB tagging only when more than one TC */
+			/* Enable DCB tagging only when more than one TC
+			 * or explicitly disable if only one TC
+			 */
 			if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
 				pf->flags |= I40E_FLAG_DCB_ENABLED;
+			else
+				pf->flags &= ~I40E_FLAG_DCB_ENABLED;
 			dev_dbg(&pf->pdev->dev,
 				"DCBX offload is supported for this PF.\n");
 		}
@@ -5226,7 +5242,7 @@ static int i40e_up_complete(struct i40e_vsi *vsi)
 		/* reset fd counters */
 		pf->fd_add_err = pf->fd_atr_cnt = 0;
 		if (pf->fd_tcp_rule > 0) {
-			pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
+			pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED;
 			if (I40E_DEBUG_FD & pf->hw.debug_mask)
 				dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 exist\n");
 			pf->fd_tcp_rule = 0;
@@ -5416,7 +5432,6 @@ int i40e_open(struct net_device *netdev)
 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
 
 	udp_tunnel_get_rx_info(netdev);
-	i40e_notify_client_of_netdev_open(vsi);
 
 	return 0;
 }
@@ -5702,7 +5717,7 @@ static int i40e_handle_lldp_event(struct i40e_pf *pf,
 	u8 type;
 
 	/* Not DCB capable or capability disabled */
-	if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
+	if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
 		return ret;
 
 	/* Ignore if event is not for Nearest Bridge */
@@ -5924,13 +5939,17 @@ void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
 				dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
 		}
 	}
-	/* Wait for some more space to be available to turn on ATR */
+
+	/* Wait for some more space to be available to turn on ATR. We also
+	 * must check that no existing ntuple rules for TCP are in effect
+	 */
 	if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) {
 		if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
-		    (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) {
+		    (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED) &&
+		    (pf->fd_tcp_rule == 0)) {
 			pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 			if (I40E_DEBUG_FD & pf->hw.debug_mask)
-				dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table now\n");
+				dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
 		}
 	}
 
@@ -5961,9 +5980,6 @@ static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
 	int fd_room;
 	int reg;
 
-	if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)))
-		return;
-
 	if (!time_after(jiffies, pf->fd_flush_timestamp +
 				 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
 		return;
@@ -5983,7 +5999,7 @@ static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
 	}
 
 	pf->fd_flush_timestamp = jiffies;
-	pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
+	pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED;
 	/* flush all filters */
 	wr32(&pf->hw, I40E_PFQF_CTL_1,
 	     I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
@@ -6003,7 +6019,7 @@ static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
 		/* replay sideband filters */
 		i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
 		if (!disable_atr)
-			pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
+			pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 		clear_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);
 		if (I40E_DEBUG_FD & pf->hw.debug_mask)
 			dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
@@ -6037,9 +6053,6 @@ static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
 	if (test_bit(__I40E_DOWN, &pf->state))
 		return;
 
-	if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)))
-		return;
-
 	if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state))
 		i40e_fdir_flush_and_replay(pf);
 
@@ -7139,9 +7152,9 @@ static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
 			pf->pending_udp_bitmap &= ~BIT_ULL(i);
 			port = pf->udp_ports[i].index;
 			if (port)
-				ret = i40e_aq_add_udp_tunnel(hw, ntohs(port),
-						     pf->udp_ports[i].type,
-						     NULL, NULL);
+				ret = i40e_aq_add_udp_tunnel(hw, port,
+							pf->udp_ports[i].type,
+							NULL, NULL);
 			else
 				ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
 
@@ -7628,7 +7641,6 @@ static int i40e_init_msix(struct i40e_pf *pf)
 			vectors_left--;
 		} else {
 			pf->num_fdsb_msix = 0;
-			pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
 		}
 	}
 
@@ -7648,6 +7660,8 @@ static int i40e_init_msix(struct i40e_pf *pf)
 #endif
 	/* can we reserve enough for iWARP? */
 	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
+		iwarp_requested = pf->num_iwarp_msix;
+
 		if (!vectors_left)
 			pf->num_iwarp_msix = 0;
 		else if (vectors_left < pf->num_iwarp_msix)
@@ -7661,18 +7675,23 @@ static int i40e_init_msix(struct i40e_pf *pf)
 		int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
 		int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
 
-		/* if we're short on vectors for what's desired, we limit
-		 * the queues per vmdq.  If this is still more than are
-		 * available, the user will need to change the number of
-		 * queues/vectors used by the PF later with the ethtool
-		 * channels command
-		 */
-		if (vmdq_vecs < vmdq_vecs_wanted)
-			pf->num_vmdq_qps = 1;
-		pf->num_vmdq_msix = pf->num_vmdq_qps;
+		if (!vectors_left) {
+			pf->num_vmdq_msix = 0;
+			pf->num_vmdq_qps = 0;
+		} else {
+			/* if we're short on vectors for what's desired, we limit
+			 * the queues per vmdq.  If this is still more than are
+			 * available, the user will need to change the number of
+			 * queues/vectors used by the PF later with the ethtool
+			 * channels command
+			 */
+			if (vmdq_vecs < vmdq_vecs_wanted)
+				pf->num_vmdq_qps = 1;
+			pf->num_vmdq_msix = pf->num_vmdq_qps;
 
-		v_budget += vmdq_vecs;
-		vectors_left -= vmdq_vecs;
+			v_budget += vmdq_vecs;
+			vectors_left -= vmdq_vecs;
+		}
 	}
 
 	pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
@@ -7684,21 +7703,6 @@ static int i40e_init_msix(struct i40e_pf *pf)
 		pf->msix_entries[i].entry = i;
 	v_actual = i40e_reserve_msix_vectors(pf, v_budget);
 
-	if (v_actual != v_budget) {
-		/* If we have limited resources, we will start with no vectors
-		 * for the special features and then allocate vectors to some
-		 * of these features based on the policy and at the end disable
-		 * the features that did not get any vectors.
-		 */
-		iwarp_requested = pf->num_iwarp_msix;
-		pf->num_iwarp_msix = 0;
-#ifdef I40E_FCOE
-		pf->num_fcoe_qps = 0;
-		pf->num_fcoe_msix = 0;
-#endif
-		pf->num_vmdq_msix = 0;
-	}
-
 	if (v_actual < I40E_MIN_MSIX) {
 		pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
 		kfree(pf->msix_entries);
@@ -7712,9 +7716,16 @@ static int i40e_init_msix(struct i40e_pf *pf)
 		pf->num_lan_qps = 1;
 		pf->num_lan_msix = 1;
 
-	} else if (v_actual != v_budget) {
+	} else if (!vectors_left) {
+		/* If we have limited resources, we will start with no vectors
+		 * for the special features and then allocate vectors to some
+		 * of these features based on the policy and at the end disable
+		 * the features that did not get any vectors.
+		 */
 		int vec;
 
+		dev_info(&pf->pdev->dev,
+			 "MSI-X vector limit reached, attempting to redistribute vectors\n");
 		/* reserve the misc vector */
 		vec = v_actual - 1;
 
@@ -7722,7 +7733,10 @@ static int i40e_init_msix(struct i40e_pf *pf)
 		pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
 		pf->num_vmdq_vsis = 1;
 		pf->num_vmdq_qps = 1;
-		pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
+#ifdef I40E_FCOE
+		pf->num_fcoe_qps = 0;
+		pf->num_fcoe_msix = 0;
+#endif
 
 		/* partition out the remaining vectors */
 		switch (vec) {
@@ -7754,9 +7768,14 @@ static int i40e_init_msix(struct i40e_pf *pf)
 				pf->num_vmdq_vsis = min_t(int, (vec / 2),
 						  I40E_DEFAULT_NUM_VMDQ_VSI);
 			}
+			if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
+				pf->num_fdsb_msix = 1;
+				vec--;
+			}
 			pf->num_lan_msix = min_t(int,
 			       (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
 							      pf->num_lan_msix);
+			pf->num_lan_qps = pf->num_lan_msix;
 #ifdef I40E_FCOE
 			/* give one vector to FCoE */
 			if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
@@ -7768,6 +7787,11 @@ static int i40e_init_msix(struct i40e_pf *pf)
 		}
 	}
 
+	if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
+	    (pf->num_fdsb_msix == 0)) {
+		dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
+		pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
+	}
 	if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
 	    (pf->num_vmdq_msix == 0)) {
 		dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
@@ -7786,6 +7810,13 @@ static int i40e_init_msix(struct i40e_pf *pf)
 		pf->flags &= ~I40E_FLAG_FCOE_ENABLED;
 	}
 #endif
+	i40e_debug(&pf->hw, I40E_DEBUG_INIT,
+		   "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
+		   pf->num_lan_msix,
+		   pf->num_vmdq_msix * pf->num_vmdq_vsis,
+		   pf->num_fdsb_msix,
+		   pf->num_iwarp_msix);
+
 	return v_actual;
 }
 
@@ -7882,6 +7913,7 @@ static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
 #endif
 				       I40E_FLAG_RSS_ENABLED	|
 				       I40E_FLAG_DCB_CAPABLE	|
+				       I40E_FLAG_DCB_ENABLED	|
 				       I40E_FLAG_SRIOV_ENABLED	|
 				       I40E_FLAG_FD_SB_ENABLED	|
 				       I40E_FLAG_FD_ATR_ENABLED	|
@@ -7971,72 +8003,34 @@ static int i40e_setup_misc_vector(struct i40e_pf *pf)
 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
 			      u8 *lut, u16 lut_size)
 {
-	struct i40e_aqc_get_set_rss_key_data rss_key;
 	struct i40e_pf *pf = vsi->back;
 	struct i40e_hw *hw = &pf->hw;
-	bool pf_lut = false;
-	u8 *rss_lut;
-	int ret, i;
-
-	memcpy(&rss_key, seed, sizeof(rss_key));
-
-	rss_lut = kzalloc(pf->rss_table_size, GFP_KERNEL);
-	if (!rss_lut)
-		return -ENOMEM;
-
-	/* Populate the LUT with max no. of queues in round robin fashion */
-	for (i = 0; i < vsi->rss_table_size; i++)
-		rss_lut[i] = i % vsi->rss_size;
+	int ret = 0;
 
-	ret = i40e_aq_set_rss_key(hw, vsi->id, &rss_key);
-	if (ret) {
-		dev_info(&pf->pdev->dev,
-			 "Cannot set RSS key, err %s aq_err %s\n",
-			 i40e_stat_str(&pf->hw, ret),
-			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
-		goto config_rss_aq_out;
+	if (seed) {
+		struct i40e_aqc_get_set_rss_key_data *seed_dw =
+			(struct i40e_aqc_get_set_rss_key_data *)seed;
+		ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
+		if (ret) {
+			dev_info(&pf->pdev->dev,
+				 "Cannot set RSS key, err %s aq_err %s\n",
+				 i40e_stat_str(hw, ret),
+				 i40e_aq_str(hw, hw->aq.asq_last_status));
+			return ret;
+		}
 	}
+	if (lut) {
+		bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
 
-	if (vsi->type == I40E_VSI_MAIN)
-		pf_lut = true;
-
-	ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, rss_lut,
-				  vsi->rss_table_size);
-	if (ret)
-		dev_info(&pf->pdev->dev,
-			 "Cannot set RSS lut, err %s aq_err %s\n",
-			 i40e_stat_str(&pf->hw, ret),
-			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
-
-config_rss_aq_out:
-	kfree(rss_lut);
-	return ret;
-}
-
-/**
- * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
- * @vsi: VSI structure
- **/
-static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
-{
-	u8 seed[I40E_HKEY_ARRAY_SIZE];
-	struct i40e_pf *pf = vsi->back;
-	u8 *lut;
-	int ret;
-
-	if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE))
-		return 0;
-
-	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
-	if (!lut)
-		return -ENOMEM;
-
-	i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
-	netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
-	vsi->rss_size = min_t(int, pf->alloc_rss_size, vsi->num_queue_pairs);
-	ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
-	kfree(lut);
-
+		ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
+		if (ret) {
+			dev_info(&pf->pdev->dev,
+				 "Cannot set RSS lut, err %s aq_err %s\n",
+				 i40e_stat_str(hw, ret),
+				 i40e_aq_str(hw, hw->aq.asq_last_status));
+			return ret;
+		}
+	}
 	return ret;
 }
 
@@ -8087,6 +8081,46 @@ static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
 }
 
 /**
+ * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
+ * @vsi: VSI structure
+ **/
+static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
+{
+	u8 seed[I40E_HKEY_ARRAY_SIZE];
+	struct i40e_pf *pf = vsi->back;
+	u8 *lut;
+	int ret;
+
+	if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE))
+		return 0;
+
+	if (!vsi->rss_size)
+		vsi->rss_size = min_t(int, pf->alloc_rss_size,
+				      vsi->num_queue_pairs);
+	if (!vsi->rss_size)
+		return -EINVAL;
+
+	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
+	if (!lut)
+		return -ENOMEM;
+	/* Use the user configured hash keys and lookup table if there is one,
+	 * otherwise use default
+	 */
+	if (vsi->rss_lut_user)
+		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
+	else
+		i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
+	if (vsi->rss_hkey_user)
+		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
+	else
+		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
+	ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
+	kfree(lut);
+
+	return ret;
+}
+
+/**
  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
  * @vsi: Pointer to vsi structure
  * @seed: RSS hash seed
@@ -8224,8 +8258,8 @@ int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
  * @rss_table_size: Lookup table size
  * @rss_size: Range of queue number for hashing
  */
-static void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
-			      u16 rss_table_size, u16 rss_size)
+void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
+		       u16 rss_table_size, u16 rss_size)
 {
 	u16 i;
 
@@ -8266,6 +8300,8 @@ static int i40e_pf_config_rss(struct i40e_pf *pf)
 	if (!vsi->rss_size)
 		vsi->rss_size = min_t(int, pf->alloc_rss_size,
 				      vsi->num_queue_pairs);
+	if (!vsi->rss_size)
+		return -EINVAL;
 
 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
 	if (!lut)
@@ -8590,7 +8626,6 @@ static int i40e_sw_init(struct i40e_pf *pf)
 			     I40E_FLAG_WB_ON_ITR_CAPABLE |
 			     I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE |
 			     I40E_FLAG_NO_PCI_LINK_CHECK |
-			     I40E_FLAG_100M_SGMII_CAPABLE |
 			     I40E_FLAG_USE_SET_LLDP_MIB |
 			     I40E_FLAG_GENEVE_OFFLOAD_CAPABLE;
 	} else if ((pf->hw.aq.api_maj_ver > 1) ||
@@ -8665,18 +8700,40 @@ bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
 		/* reset fd counters */
 		pf->fd_add_err = pf->fd_atr_cnt = pf->fd_tcp_rule = 0;
 		pf->fdir_pf_active_filters = 0;
-		pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
-		if (I40E_DEBUG_FD & pf->hw.debug_mask)
-			dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
 		/* if ATR was auto disabled it can be re-enabled. */
 		if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
-		    (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
+		    (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) {
 			pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
+			if (I40E_DEBUG_FD & pf->hw.debug_mask)
+				dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
+		}
 	}
 	return need_reset;
 }
 
 /**
+ * i40e_clear_rss_lut - clear the rx hash lookup table
+ * @vsi: the VSI being configured
+ **/
+static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
+{
+	struct i40e_pf *pf = vsi->back;
+	struct i40e_hw *hw = &pf->hw;
+	u16 vf_id = vsi->vf_id;
+	u8 i;
+
+	if (vsi->type == I40E_VSI_MAIN) {
+		for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
+			wr32(hw, I40E_PFQF_HLUT(i), 0);
+	} else if (vsi->type == I40E_VSI_SRIOV) {
+		for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
+			i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
+	} else {
+		dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
+	}
+}
+
+/**
  * i40e_set_features - set the netdev feature flags
  * @netdev: ptr to the netdev being adjusted
  * @features: the feature set that the stack is suggesting
@@ -8689,6 +8746,12 @@ static int i40e_set_features(struct net_device *netdev,
 	struct i40e_pf *pf = vsi->back;
 	bool need_reset;
 
+	if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
+		i40e_pf_config_rss(pf);
+	else if (!(features & NETIF_F_RXHASH) &&
+		 netdev->features & NETIF_F_RXHASH)
+		i40e_clear_rss_lut(vsi);
+
 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
 		i40e_vlan_stripping_enable(vsi);
 	else
@@ -10488,6 +10551,7 @@ static void i40e_determine_queue_usage(struct i40e_pf *pf)
 			       I40E_FLAG_FD_SB_ENABLED	|
 			       I40E_FLAG_FD_ATR_ENABLED	|
 			       I40E_FLAG_DCB_CAPABLE	|
+			       I40E_FLAG_DCB_ENABLED	|
 			       I40E_FLAG_SRIOV_ENABLED	|
 			       I40E_FLAG_VMDQ_ENABLED);
 	} else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
@@ -10511,7 +10575,8 @@ static void i40e_determine_queue_usage(struct i40e_pf *pf)
 		/* Not enough queues for all TCs */
 		if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
 		    (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
-			pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
+			pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
+					I40E_FLAG_DCB_ENABLED);
 			dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
 		}
 		pf->num_lan_qps = max_t(int, pf->rss_size_max,
@@ -10908,7 +10973,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	err = i40e_init_pf_dcb(pf);
 	if (err) {
 		dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
-		pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
+		pf->flags &= ~(I40E_FLAG_DCB_CAPABLE & I40E_FLAG_DCB_ENABLED);
 		/* Continue without DCB enabled */
 	}
 #endif /* CONFIG_I40E_DCB */
@@ -11288,11 +11353,7 @@ static void i40e_remove(struct pci_dev *pdev)
 	}
 
 	/* shutdown the adminq */
-	ret_code = i40e_shutdown_adminq(hw);
-	if (ret_code)
-		dev_warn(&pdev->dev,
-			 "Failed to destroy the Admin Queue resources: %d\n",
-			 ret_code);
+	i40e_shutdown_adminq(hw);
 
 	/* destroy the locks only once, here */
 	mutex_destroy(&hw->aq.arq_mutex);
@@ -11339,6 +11400,12 @@ static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
 
 	dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
 
+	if (!pf) {
+		dev_info(&pdev->dev,
+			 "Cannot recover - error happened during device probe\n");
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
 	/* shutdown all operations */
 	if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
 		rtnl_lock();
@@ -11561,7 +11628,8 @@ static int __init i40e_init_module(void)
 	 * it can't be any worse than using the system workqueue which
 	 * was already single threaded
 	 */
-	i40e_wq = create_singlethread_workqueue(i40e_driver_name);
+	i40e_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
+				  i40e_driver_name);
 	if (!i40e_wq) {
 		pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
 		return -ENOMEM;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
index ed39cbad24bd..f1feceab758a 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
@@ -669,7 +669,7 @@ void i40e_ptp_init(struct i40e_pf *pf)
 		pf->ptp_clock = NULL;
 		dev_err(&pf->pdev->dev, "%s: ptp_clock_register failed\n",
 			__func__);
-	} else {
+	} else if (pf->ptp_clock) {
 		struct timespec64 ts;
 		u32 regval;
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index df7ecc9578c9..6287bf63c43c 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -40,6 +40,69 @@ static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
 }
 
 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
+/**
+ * i40e_fdir - Generate a Flow Director descriptor based on fdata
+ * @tx_ring: Tx ring to send buffer on
+ * @fdata: Flow director filter data
+ * @add: Indicate if we are adding a rule or deleting one
+ *
+ **/
+static void i40e_fdir(struct i40e_ring *tx_ring,
+		      struct i40e_fdir_filter *fdata, bool add)
+{
+	struct i40e_filter_program_desc *fdir_desc;
+	struct i40e_pf *pf = tx_ring->vsi->back;
+	u32 flex_ptype, dtype_cmd;
+	u16 i;
+
+	/* grab the next descriptor */
+	i = tx_ring->next_to_use;
+	fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
+
+	i++;
+	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
+
+	flex_ptype = I40E_TXD_FLTR_QW0_QINDEX_MASK &
+		     (fdata->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT);
+
+	flex_ptype |= I40E_TXD_FLTR_QW0_FLEXOFF_MASK &
+		      (fdata->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT);
+
+	flex_ptype |= I40E_TXD_FLTR_QW0_PCTYPE_MASK &
+		      (fdata->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
+
+	/* Use LAN VSI Id if not programmed by user */
+	flex_ptype |= I40E_TXD_FLTR_QW0_DEST_VSI_MASK &
+		      ((u32)(fdata->dest_vsi ? : pf->vsi[pf->lan_vsi]->id) <<
+		       I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT);
+
+	dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
+
+	dtype_cmd |= add ?
+		     I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
+		     I40E_TXD_FLTR_QW1_PCMD_SHIFT :
+		     I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
+		     I40E_TXD_FLTR_QW1_PCMD_SHIFT;
+
+	dtype_cmd |= I40E_TXD_FLTR_QW1_DEST_MASK &
+		     (fdata->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT);
+
+	dtype_cmd |= I40E_TXD_FLTR_QW1_FD_STATUS_MASK &
+		     (fdata->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT);
+
+	if (fdata->cnt_index) {
+		dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
+		dtype_cmd |= I40E_TXD_FLTR_QW1_CNTINDEX_MASK &
+			     ((u32)fdata->cnt_index <<
+			      I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT);
+	}
+
+	fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
+	fdir_desc->rsvd = cpu_to_le32(0);
+	fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
+	fdir_desc->fd_id = cpu_to_le32(fdata->fd_id);
+}
+
 #define I40E_FD_CLEAN_DELAY 10
 /**
  * i40e_program_fdir_filter - Program a Flow Director filter
@@ -48,14 +111,13 @@ static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
  * @pf: The PF pointer
  * @add: True for add/update, False for remove
  **/
-int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
-			     struct i40e_pf *pf, bool add)
+static int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data,
+				    u8 *raw_packet, struct i40e_pf *pf,
+				    bool add)
 {
-	struct i40e_filter_program_desc *fdir_desc;
 	struct i40e_tx_buffer *tx_buf, *first;
 	struct i40e_tx_desc *tx_desc;
 	struct i40e_ring *tx_ring;
-	unsigned int fpt, dcc;
 	struct i40e_vsi *vsi;
 	struct device *dev;
 	dma_addr_t dma;
@@ -92,56 +154,8 @@ int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
 
 	/* grab the next descriptor */
 	i = tx_ring->next_to_use;
-	fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
 	first = &tx_ring->tx_bi[i];
-	memset(first, 0, sizeof(struct i40e_tx_buffer));
-
-	tx_ring->next_to_use = ((i + 1) < tx_ring->count) ? i + 1 : 0;
-
-	fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
-	      I40E_TXD_FLTR_QW0_QINDEX_MASK;
-
-	fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
-	       I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
-
-	fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
-	       I40E_TXD_FLTR_QW0_PCTYPE_MASK;
-
-	/* Use LAN VSI Id if not programmed by user */
-	if (fdir_data->dest_vsi == 0)
-		fpt |= (pf->vsi[pf->lan_vsi]->id) <<
-		       I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
-	else
-		fpt |= ((u32)fdir_data->dest_vsi <<
-			I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
-		       I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
-
-	dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
-
-	if (add)
-		dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
-		       I40E_TXD_FLTR_QW1_PCMD_SHIFT;
-	else
-		dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
-		       I40E_TXD_FLTR_QW1_PCMD_SHIFT;
-
-	dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
-	       I40E_TXD_FLTR_QW1_DEST_MASK;
-
-	dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
-	       I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
-
-	if (fdir_data->cnt_index != 0) {
-		dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
-		dcc |= ((u32)fdir_data->cnt_index <<
-			I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
-			I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
-	}
-
-	fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
-	fdir_desc->rsvd = cpu_to_le32(0);
-	fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
-	fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
+	i40e_fdir(tx_ring, fdir_data, add);
 
 	/* Now program a dummy descriptor */
 	i = tx_ring->next_to_use;
@@ -282,18 +296,18 @@ static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
 
 	if (add) {
 		pf->fd_tcp_rule++;
-		if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
-			if (I40E_DEBUG_FD & pf->hw.debug_mask)
-				dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
-			pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
-		}
+		if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
+		    I40E_DEBUG_FD & pf->hw.debug_mask)
+			dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
+		pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED;
 	} else {
 		pf->fd_tcp_rule = (pf->fd_tcp_rule > 0) ?
 				  (pf->fd_tcp_rule - 1) : 0;
 		if (pf->fd_tcp_rule == 0) {
-			pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
-			if (I40E_DEBUG_FD & pf->hw.debug_mask)
+			if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
+			    I40E_DEBUG_FD & pf->hw.debug_mask)
 				dev_info(&pf->pdev->dev, "ATR re-enabled due to no sideband TCP/IPv4 rules\n");
+			pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 		}
 	}
 
@@ -532,7 +546,10 @@ static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
 					    struct i40e_tx_buffer *tx_buffer)
 {
 	if (tx_buffer->skb) {
-		dev_kfree_skb_any(tx_buffer->skb);
+		if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
+			kfree(tx_buffer->raw_buf);
+		else
+			dev_kfree_skb_any(tx_buffer->skb);
 		if (dma_unmap_len(tx_buffer, len))
 			dma_unmap_single(ring->dev,
 					 dma_unmap_addr(tx_buffer, dma),
@@ -545,9 +562,6 @@ static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
 			       DMA_TO_DEVICE);
 	}
 
-	if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
-		kfree(tx_buffer->raw_buf);
-
 	tx_buffer->next_to_watch = NULL;
 	tx_buffer->skb = NULL;
 	dma_unmap_len_set(tx_buffer, len, 0);
@@ -584,8 +598,7 @@ void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
 		return;
 
 	/* cleanup Tx queue statistics */
-	netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
-						  tx_ring->queue_index));
+	netdev_tx_reset_queue(txring_txq(tx_ring));
 }
 
 /**
@@ -754,8 +767,8 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
 			tx_ring->arm_wb = true;
 	}
 
-	netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
-						      tx_ring->queue_index),
+	/* notify netdev of completed buffers */
+	netdev_tx_completed_queue(txring_txq(tx_ring),
 				  total_packets, total_bytes);
 
 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
@@ -1864,6 +1877,15 @@ static u32 i40e_buildreg_itr(const int type, const u16 itr)
 
 /* a small macro to shorten up some long lines */
 #define INTREG I40E_PFINT_DYN_CTLN
+static inline int get_rx_itr_enabled(struct i40e_vsi *vsi, int idx)
+{
+	return !!(vsi->rx_rings[idx]->rx_itr_setting);
+}
+
+static inline int get_tx_itr_enabled(struct i40e_vsi *vsi, int idx)
+{
+	return !!(vsi->tx_rings[idx]->tx_itr_setting);
+}
 
 /**
  * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt
@@ -1879,6 +1901,7 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 	u32 rxval, txval;
 	int vector;
 	int idx = q_vector->v_idx;
+	int rx_itr_setting, tx_itr_setting;
 
 	vector = (q_vector->v_idx + vsi->base_vector);
 
@@ -1887,18 +1910,21 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 	 */
 	rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
 
+	rx_itr_setting = get_rx_itr_enabled(vsi, idx);
+	tx_itr_setting = get_tx_itr_enabled(vsi, idx);
+
 	if (q_vector->itr_countdown > 0 ||
-	    (!ITR_IS_DYNAMIC(vsi->rx_rings[idx]->rx_itr_setting) &&
-	     !ITR_IS_DYNAMIC(vsi->tx_rings[idx]->tx_itr_setting))) {
+	    (!ITR_IS_DYNAMIC(rx_itr_setting) &&
+	     !ITR_IS_DYNAMIC(tx_itr_setting))) {
 		goto enable_int;
 	}
 
-	if (ITR_IS_DYNAMIC(vsi->rx_rings[idx]->rx_itr_setting)) {
+	if (ITR_IS_DYNAMIC(tx_itr_setting)) {
 		rx = i40e_set_new_dynamic_itr(&q_vector->rx);
 		rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr);
 	}
 
-	if (ITR_IS_DYNAMIC(vsi->tx_rings[idx]->tx_itr_setting)) {
+	if (ITR_IS_DYNAMIC(tx_itr_setting)) {
 		tx = i40e_set_new_dynamic_itr(&q_vector->tx);
 		txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr);
 	}
@@ -2621,9 +2647,7 @@ bool __i40e_chk_linearize(struct sk_buff *skb)
 		return false;
 
 	/* We need to walk through the list and validate that each group
-	 * of 6 fragments totals at least gso_size.  However we don't need
-	 * to perform such validation on the last 6 since the last 6 cannot
-	 * inherit any data from a descriptor after them.
+	 * of 6 fragments totals at least gso_size.
 	 */
 	nr_frags -= I40E_MAX_BUFFER_TXD - 2;
 	frag = &skb_shinfo(skb)->frags[0];
@@ -2654,8 +2678,7 @@ bool __i40e_chk_linearize(struct sk_buff *skb)
 		if (sum < 0)
 			return true;
 
-		/* use pre-decrement to avoid processing last fragment */
-		if (!--nr_frags)
+		if (!nr_frags--)
 			break;
 
 		sum -= skb_frag_size(stale++);
@@ -2787,9 +2810,7 @@ static inline void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
 
 	tx_ring->next_to_use = i;
 
-	netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
-						 tx_ring->queue_index),
-						 first->bytecount);
+	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
 	i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
 
 	/* Algorithm to optimize tail and RS bit setting:
@@ -2814,13 +2835,11 @@ static inline void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
 	 * trigger a force WB.
 	 */
 	if (skb->xmit_more  &&
-	    !netif_xmit_stopped(netdev_get_tx_queue(tx_ring->netdev,
-						    tx_ring->queue_index))) {
+	    !netif_xmit_stopped(txring_txq(tx_ring))) {
 		tx_ring->flags |= I40E_TXR_FLAGS_LAST_XMIT_MORE_SET;
 		tail_bump = false;
 	} else if (!skb->xmit_more &&
-		   !netif_xmit_stopped(netdev_get_tx_queue(tx_ring->netdev,
-						       tx_ring->queue_index)) &&
+		   !netif_xmit_stopped(txring_txq(tx_ring)) &&
 		   (!(tx_ring->flags & I40E_TXR_FLAGS_LAST_XMIT_MORE_SET)) &&
 		   (tx_ring->packet_stride < WB_STRIDE) &&
 		   (desc_count < WB_STRIDE)) {
@@ -2840,10 +2859,9 @@ static inline void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
 						  I40E_TXD_QW1_CMD_SHIFT);
 
 	/* notify HW of packet */
-	if (!tail_bump)
+	if (!tail_bump) {
 		prefetchw(tx_desc + 1);
-
-	if (tail_bump) {
+	} else {
 		/* Force memory writes to complete before letting h/w
 		 * know there are new descriptors to fetch.  (Only
 		 * applicable for weak-ordered memory model archs,
@@ -2852,7 +2870,6 @@ static inline void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
 		wmb();
 		writel(i, tx_ring->tail);
 	}
-
 	return;
 
 dma_error:
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index b78c810d1835..508840585645 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -463,4 +463,13 @@ static inline bool i40e_rx_is_fcoe(u16 ptype)
 	return (ptype >= I40E_RX_PTYPE_L2_FCOE_PAY3) &&
 	       (ptype <= I40E_RX_PTYPE_L2_FCOE_VFT_FCOTHER);
 }
+
+/**
+ * txring_txq - Find the netdev Tx ring based on the i40e Tx ring
+ * @ring: Tx ring to find the netdev equivalent of
+ **/
+static inline struct netdev_queue *txring_txq(const struct i40e_ring *ring)
+{
+	return netdev_get_tx_queue(ring->netdev, ring->queue_index);
+}
 #endif /* _I40E_TXRX_H_ */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl.h b/drivers/net/ethernet/intel/i40e/i40e_virtchnl.h
index c92a3bdee229..f861d3109d1a 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl.h
@@ -163,6 +163,7 @@ struct i40e_virtchnl_vsi_resource {
 #define I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING	0x00020000
 #define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2	0x00040000
 #define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF		0X00080000
+#define I40E_VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM	0X00100000
 
 struct i40e_virtchnl_vf_resource {
 	u16 num_vsis;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
index 6fcbf764f32b..54b8ee2583f1 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
@@ -502,8 +502,16 @@ static int i40e_config_vsi_tx_queue(struct i40e_vf *vf, u16 vsi_id,
 	u32 qtx_ctl;
 	int ret = 0;
 
+	if (!i40e_vc_isvalid_vsi_id(vf, info->vsi_id)) {
+		ret = -ENOENT;
+		goto error_context;
+	}
 	pf_queue_id = i40e_vc_get_pf_queue_id(vf, vsi_id, vsi_queue_id);
 	vsi = i40e_find_vsi_from_id(pf, vsi_id);
+	if (!vsi) {
+		ret = -ENOENT;
+		goto error_context;
+	}
 
 	/* clear the context structure first */
 	memset(&tx_ctx, 0, sizeof(struct i40e_hmc_obj_txq));
@@ -991,7 +999,10 @@ complete_reset:
 		i40e_enable_vf_mappings(vf);
 		set_bit(I40E_VF_STAT_ACTIVE, &vf->vf_states);
 		clear_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
-		i40e_notify_client_of_vf_reset(pf, abs_vf_id);
+		/* Do not notify the client during VF init */
+		if (vf->pf->num_alloc_vfs)
+			i40e_notify_client_of_vf_reset(pf, abs_vf_id);
+		vf->num_vlan = 0;
 	}
 	/* tell the VF the reset is done */
 	wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_id), I40E_VFR_VFACTIVE);
@@ -1089,7 +1100,6 @@ int i40e_alloc_vfs(struct i40e_pf *pf, u16 num_alloc_vfs)
 			goto err_iov;
 		}
 	}
-	i40e_notify_client_of_vf_enable(pf, num_alloc_vfs);
 	/* allocate memory */
 	vfs = kcalloc(num_alloc_vfs, sizeof(struct i40e_vf), GFP_KERNEL);
 	if (!vfs) {
@@ -1113,6 +1123,8 @@ int i40e_alloc_vfs(struct i40e_pf *pf, u16 num_alloc_vfs)
 	}
 	pf->num_alloc_vfs = num_alloc_vfs;
 
+	i40e_notify_client_of_vf_enable(pf, num_alloc_vfs);
+
 err_alloc:
 	if (ret)
 		i40e_free_vfs(pf);
@@ -1472,7 +1484,8 @@ static int i40e_vc_config_promiscuous_mode_msg(struct i40e_vf *vf,
 
 	vsi = i40e_find_vsi_from_id(pf, info->vsi_id);
 	if (!test_bit(I40E_VF_STAT_ACTIVE, &vf->vf_states) ||
-	    !i40e_vc_isvalid_vsi_id(vf, info->vsi_id)) {
+	    !i40e_vc_isvalid_vsi_id(vf, info->vsi_id) ||
+	    !vsi) {
 		aq_ret = I40E_ERR_PARAM;
 		goto error_param;
 	}
@@ -2213,8 +2226,8 @@ static int i40e_vc_iwarp_qvmap_msg(struct i40e_vf *vf, u8 *msg, u16 msglen,
 error_param:
 	/* send the response to the VF */
 	return i40e_vc_send_resp_to_vf(vf,
-			       config ? I40E_VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP :
-			       I40E_VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP,
+			       config ? I40E_VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP :
+			       I40E_VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP,
 			       aq_ret);
 }
 
@@ -2314,6 +2327,7 @@ err:
 	/* send the response back to the VF */
 	aq_ret = i40e_vc_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_GET_RSS_HENA_CAPS,
 					aq_ret, (u8 *)vrh, len);
+	kfree(vrh);
 	return aq_ret;
 }
 
@@ -2742,11 +2756,12 @@ error_param:
  * @vf_id: VF identifier
  * @vlan_id: mac address
  * @qos: priority setting
+ * @vlan_proto: vlan protocol
  *
  * program VF vlan id and/or qos
  **/
-int i40e_ndo_set_vf_port_vlan(struct net_device *netdev,
-			      int vf_id, u16 vlan_id, u8 qos)
+int i40e_ndo_set_vf_port_vlan(struct net_device *netdev, int vf_id,
+			      u16 vlan_id, u8 qos, __be16 vlan_proto)
 {
 	u16 vlanprio = vlan_id | (qos << I40E_VLAN_PRIORITY_SHIFT);
 	struct i40e_netdev_priv *np = netdev_priv(netdev);
@@ -2769,6 +2784,12 @@ int i40e_ndo_set_vf_port_vlan(struct net_device *netdev,
 		goto error_pvid;
 	}
 
+	if (vlan_proto != htons(ETH_P_8021Q)) {
+		dev_err(&pf->pdev->dev, "VF VLAN protocol is not supported\n");
+		ret = -EPROTONOSUPPORT;
+		goto error_pvid;
+	}
+
 	vf = &(pf->vf[vf_id]);
 	vsi = pf->vsi[vf->lan_vsi_idx];
 	if (!test_bit(I40E_VF_STAT_INIT, &vf->vf_states)) {
@@ -2995,6 +3016,7 @@ int i40e_ndo_get_vf_config(struct net_device *netdev,
 	else
 		ivi->linkstate = IFLA_VF_LINK_STATE_DISABLE;
 	ivi->spoofchk = vf->spoofchk;
+	ivi->trusted = vf->trusted;
 	ret = 0;
 
 error_param:
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h
index 875174141451..4012d069939a 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h
@@ -129,8 +129,8 @@ void i40e_vc_notify_vf_reset(struct i40e_vf *vf);
 
 /* VF configuration related iplink handlers */
 int i40e_ndo_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac);
-int i40e_ndo_set_vf_port_vlan(struct net_device *netdev,
-			      int vf_id, u16 vlan_id, u8 qos);
+int i40e_ndo_set_vf_port_vlan(struct net_device *netdev, int vf_id,
+			      u16 vlan_id, u8 qos, __be16 vlan_proto);
 int i40e_ndo_set_vf_bw(struct net_device *netdev, int vf_id, int min_tx_rate,
 		       int max_tx_rate);
 int i40e_ndo_set_vf_trust(struct net_device *netdev, int vf_id, bool setting);
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h b/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h
index 3114dcfa1724..40b0eafd0c71 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h
@@ -204,6 +204,9 @@ enum i40e_admin_queue_opc {
 	i40e_aqc_opc_suspend_port_tx				= 0x041B,
 	i40e_aqc_opc_resume_port_tx				= 0x041C,
 	i40e_aqc_opc_configure_partition_bw			= 0x041D,
+	/* hmc */
+	i40e_aqc_opc_query_hmc_resource_profile	= 0x0500,
+	i40e_aqc_opc_set_hmc_resource_profile	= 0x0501,
 
 	/* phy commands*/
 	i40e_aqc_opc_get_phy_abilities		= 0x0600,
@@ -447,13 +450,15 @@ I40E_CHECK_CMD_LENGTH(i40e_aqc_cppm_configuration);
 /* Set ARP Proxy command / response (indirect 0x0104) */
 struct i40e_aqc_arp_proxy_data {
 	__le16	command_flags;
-#define I40E_AQ_ARP_INIT_IPV4	0x0008
-#define I40E_AQ_ARP_UNSUP_CTL	0x0010
-#define I40E_AQ_ARP_ENA		0x0020
-#define I40E_AQ_ARP_ADD_IPV4	0x0040
-#define I40E_AQ_ARP_DEL_IPV4	0x0080
+#define I40E_AQ_ARP_INIT_IPV4	0x0800
+#define I40E_AQ_ARP_UNSUP_CTL	0x1000
+#define I40E_AQ_ARP_ENA		0x2000
+#define I40E_AQ_ARP_ADD_IPV4	0x4000
+#define I40E_AQ_ARP_DEL_IPV4	0x8000
 	__le16	table_id;
-	__le32	pfpm_proxyfc;
+	__le32	enabled_offloads;
+#define I40E_AQ_ARP_DIRECTED_OFFLOAD_ENABLE	0x00000020
+#define I40E_AQ_ARP_OFFLOAD_ENABLE		0x00000800
 	__le32	ip_addr;
 	u8	mac_addr[6];
 	u8	reserved[2];
@@ -468,17 +473,19 @@ struct i40e_aqc_ns_proxy_data {
 	__le16	table_idx_ipv6_0;
 	__le16	table_idx_ipv6_1;
 	__le16	control;
-#define I40E_AQ_NS_PROXY_ADD_0		0x0100
-#define I40E_AQ_NS_PROXY_DEL_0		0x0200
-#define I40E_AQ_NS_PROXY_ADD_1		0x0400
-#define I40E_AQ_NS_PROXY_DEL_1		0x0800
-#define I40E_AQ_NS_PROXY_ADD_IPV6_0	0x1000
-#define I40E_AQ_NS_PROXY_DEL_IPV6_0	0x2000
-#define I40E_AQ_NS_PROXY_ADD_IPV6_1	0x4000
-#define I40E_AQ_NS_PROXY_DEL_IPV6_1	0x8000
-#define I40E_AQ_NS_PROXY_COMMAND_SEQ	0x0001
-#define I40E_AQ_NS_PROXY_INIT_IPV6_TBL	0x0002
-#define I40E_AQ_NS_PROXY_INIT_MAC_TBL	0x0004
+#define I40E_AQ_NS_PROXY_ADD_0		0x0001
+#define I40E_AQ_NS_PROXY_DEL_0		0x0002
+#define I40E_AQ_NS_PROXY_ADD_1		0x0004
+#define I40E_AQ_NS_PROXY_DEL_1		0x0008
+#define I40E_AQ_NS_PROXY_ADD_IPV6_0	0x0010
+#define I40E_AQ_NS_PROXY_DEL_IPV6_0	0x0020
+#define I40E_AQ_NS_PROXY_ADD_IPV6_1	0x0040
+#define I40E_AQ_NS_PROXY_DEL_IPV6_1	0x0080
+#define I40E_AQ_NS_PROXY_COMMAND_SEQ	0x0100
+#define I40E_AQ_NS_PROXY_INIT_IPV6_TBL	0x0200
+#define I40E_AQ_NS_PROXY_INIT_MAC_TBL	0x0400
+#define I40E_AQ_NS_PROXY_OFFLOAD_ENABLE	0x0800
+#define I40E_AQ_NS_PROXY_DIRECTED_OFFLOAD_ENABLE	0x1000
 	u8	mac_addr_0[6];
 	u8	mac_addr_1[6];
 	u8	local_mac_addr[6];
@@ -1579,6 +1586,24 @@ struct i40e_aqc_configure_partition_bw_data {
 
 I40E_CHECK_STRUCT_LEN(0x22, i40e_aqc_configure_partition_bw_data);
 
+/* Get and set the active HMC resource profile and status.
+ * (direct 0x0500) and (direct 0x0501)
+ */
+struct i40e_aq_get_set_hmc_resource_profile {
+	u8	pm_profile;
+	u8	pe_vf_enabled;
+	u8	reserved[14];
+};
+
+I40E_CHECK_CMD_LENGTH(i40e_aq_get_set_hmc_resource_profile);
+
+enum i40e_aq_hmc_profile {
+	/* I40E_HMC_PROFILE_NO_CHANGE	= 0, reserved */
+	I40E_HMC_PROFILE_DEFAULT	= 1,
+	I40E_HMC_PROFILE_FAVOR_VF	= 2,
+	I40E_HMC_PROFILE_EQUAL		= 3,
+};
+
 /* Get PHY Abilities (indirect 0x0600) uses the generic indirect struct */
 
 /* set in param0 for get phy abilities to report qualified modules */
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_common.c b/drivers/net/ethernet/intel/i40evf/i40e_common.c
index 4db0c0326185..7953c13451b9 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_common.c
@@ -302,7 +302,6 @@ void i40evf_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void *desc,
 		   void *buffer, u16 buf_len)
 {
 	struct i40e_aq_desc *aq_desc = (struct i40e_aq_desc *)desc;
-	u16 len = le16_to_cpu(aq_desc->datalen);
 	u8 *buf = (u8 *)buffer;
 	u16 i = 0;
 
@@ -326,6 +325,8 @@ void i40evf_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void *desc,
 		   le32_to_cpu(aq_desc->params.external.addr_low));
 
 	if ((buffer != NULL) && (aq_desc->datalen != 0)) {
+		u16 len = le16_to_cpu(aq_desc->datalen);
+
 		i40e_debug(hw, mask, "AQ CMD Buffer:\n");
 		if (buf_len < len)
 			len = buf_len;
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
index a579193b2c21..75f2a2cdd738 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
@@ -51,7 +51,10 @@ static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
 					    struct i40e_tx_buffer *tx_buffer)
 {
 	if (tx_buffer->skb) {
-		dev_kfree_skb_any(tx_buffer->skb);
+		if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
+			kfree(tx_buffer->raw_buf);
+		else
+			dev_kfree_skb_any(tx_buffer->skb);
 		if (dma_unmap_len(tx_buffer, len))
 			dma_unmap_single(ring->dev,
 					 dma_unmap_addr(tx_buffer, dma),
@@ -64,9 +67,6 @@ static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
 			       DMA_TO_DEVICE);
 	}
 
-	if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
-		kfree(tx_buffer->raw_buf);
-
 	tx_buffer->next_to_watch = NULL;
 	tx_buffer->skb = NULL;
 	dma_unmap_len_set(tx_buffer, len, 0);
@@ -103,8 +103,7 @@ void i40evf_clean_tx_ring(struct i40e_ring *tx_ring)
 		return;
 
 	/* cleanup Tx queue statistics */
-	netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
-						  tx_ring->queue_index));
+	netdev_tx_reset_queue(txring_txq(tx_ring));
 }
 
 /**
@@ -273,8 +272,8 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
 			tx_ring->arm_wb = true;
 	}
 
-	netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
-						      tx_ring->queue_index),
+	/* notify netdev of completed buffers */
+	netdev_tx_completed_queue(txring_txq(tx_ring),
 				  total_packets, total_bytes);
 
 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
@@ -1312,6 +1311,19 @@ static u32 i40e_buildreg_itr(const int type, const u16 itr)
 
 /* a small macro to shorten up some long lines */
 #define INTREG I40E_VFINT_DYN_CTLN1
+static inline int get_rx_itr_enabled(struct i40e_vsi *vsi, int idx)
+{
+	struct i40evf_adapter *adapter = vsi->back;
+
+	return !!(adapter->rx_rings[idx].rx_itr_setting);
+}
+
+static inline int get_tx_itr_enabled(struct i40e_vsi *vsi, int idx)
+{
+	struct i40evf_adapter *adapter = vsi->back;
+
+	return !!(adapter->tx_rings[idx].tx_itr_setting);
+}
 
 /**
  * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt
@@ -1326,6 +1338,8 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 	bool rx = false, tx = false;
 	u32 rxval, txval;
 	int vector;
+	int idx = q_vector->v_idx;
+	int rx_itr_setting, tx_itr_setting;
 
 	vector = (q_vector->v_idx + vsi->base_vector);
 
@@ -1334,18 +1348,21 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 	 */
 	rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
 
+	rx_itr_setting = get_rx_itr_enabled(vsi, idx);
+	tx_itr_setting = get_tx_itr_enabled(vsi, idx);
+
 	if (q_vector->itr_countdown > 0 ||
-	    (!ITR_IS_DYNAMIC(vsi->rx_itr_setting) &&
-	     !ITR_IS_DYNAMIC(vsi->tx_itr_setting))) {
+	    (!ITR_IS_DYNAMIC(rx_itr_setting) &&
+	     !ITR_IS_DYNAMIC(tx_itr_setting))) {
 		goto enable_int;
 	}
 
-	if (ITR_IS_DYNAMIC(vsi->rx_itr_setting)) {
+	if (ITR_IS_DYNAMIC(rx_itr_setting)) {
 		rx = i40e_set_new_dynamic_itr(&q_vector->rx);
 		rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr);
 	}
 
-	if (ITR_IS_DYNAMIC(vsi->tx_itr_setting)) {
+	if (ITR_IS_DYNAMIC(tx_itr_setting)) {
 		tx = i40e_set_new_dynamic_itr(&q_vector->tx);
 		txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr);
 	}
@@ -1832,9 +1849,7 @@ bool __i40evf_chk_linearize(struct sk_buff *skb)
 		return false;
 
 	/* We need to walk through the list and validate that each group
-	 * of 6 fragments totals at least gso_size.  However we don't need
-	 * to perform such validation on the last 6 since the last 6 cannot
-	 * inherit any data from a descriptor after them.
+	 * of 6 fragments totals at least gso_size.
 	 */
 	nr_frags -= I40E_MAX_BUFFER_TXD - 2;
 	frag = &skb_shinfo(skb)->frags[0];
@@ -1865,8 +1880,7 @@ bool __i40evf_chk_linearize(struct sk_buff *skb)
 		if (sum < 0)
 			return true;
 
-		/* use pre-decrement to avoid processing last fragment */
-		if (!--nr_frags)
+		if (!nr_frags--)
 			break;
 
 		sum -= skb_frag_size(stale++);
@@ -2015,9 +2029,7 @@ static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
 
 	tx_ring->next_to_use = i;
 
-	netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
-						 tx_ring->queue_index),
-						 first->bytecount);
+	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
 	i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
 
 	/* Algorithm to optimize tail and RS bit setting:
@@ -2042,13 +2054,11 @@ static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
 	 * trigger a force WB.
 	 */
 	if (skb->xmit_more  &&
-	    !netif_xmit_stopped(netdev_get_tx_queue(tx_ring->netdev,
-						    tx_ring->queue_index))) {
+	    !netif_xmit_stopped(txring_txq(tx_ring))) {
 		tx_ring->flags |= I40E_TXR_FLAGS_LAST_XMIT_MORE_SET;
 		tail_bump = false;
 	} else if (!skb->xmit_more &&
-		   !netif_xmit_stopped(netdev_get_tx_queue(tx_ring->netdev,
-						       tx_ring->queue_index)) &&
+		   !netif_xmit_stopped(txring_txq(tx_ring)) &&
 		   (!(tx_ring->flags & I40E_TXR_FLAGS_LAST_XMIT_MORE_SET)) &&
 		   (tx_ring->packet_stride < WB_STRIDE) &&
 		   (desc_count < WB_STRIDE)) {
@@ -2068,10 +2078,9 @@ static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
 						  I40E_TXD_QW1_CMD_SHIFT);
 
 	/* notify HW of packet */
-	if (!tail_bump)
+	if (!tail_bump) {
 		prefetchw(tx_desc + 1);
-
-	if (tail_bump) {
+	} else {
 		/* Force memory writes to complete before letting h/w
 		 * know there are new descriptors to fetch.  (Only
 		 * applicable for weak-ordered memory model archs,
@@ -2080,7 +2089,6 @@ static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
 		wmb();
 		writel(i, tx_ring->tail);
 	}
-
 	return;
 
 dma_error:
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
index 0112277e5882..abcdecabbc56 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
@@ -287,6 +287,14 @@ struct i40e_ring {
 	u8 dcb_tc;			/* Traffic class of ring */
 	u8 __iomem *tail;
 
+	/* high bit set means dynamic, use accessors routines to read/write.
+	 * hardware only supports 2us resolution for the ITR registers.
+	 * these values always store the USER setting, and must be converted
+	 * before programming to a register.
+	 */
+	u16 rx_itr_setting;
+	u16 tx_itr_setting;
+
 	u16 count;			/* Number of descriptors */
 	u16 reg_idx;			/* HW register index of the ring */
 	u16 rx_buf_len;
@@ -445,4 +453,13 @@ static inline bool i40e_rx_is_fcoe(u16 ptype)
 	return (ptype >= I40E_RX_PTYPE_L2_FCOE_PAY3) &&
 	       (ptype <= I40E_RX_PTYPE_L2_FCOE_VFT_FCOTHER);
 }
+
+/**
+ * txring_txq - Find the netdev Tx ring based on the i40e Tx ring
+ * @ring: Tx ring to find the netdev equivalent of
+ **/
+static inline struct netdev_queue *txring_txq(const struct i40e_ring *ring)
+{
+	return netdev_get_tx_queue(ring->netdev, ring->queue_index);
+}
 #endif /* _I40E_TXRX_H_ */
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_virtchnl.h b/drivers/net/ethernet/intel/i40evf/i40e_virtchnl.h
index f04ce6cb70dc..bd691ad86673 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_virtchnl.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_virtchnl.h
@@ -160,6 +160,7 @@ struct i40e_virtchnl_vsi_resource {
 #define I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING	0x00020000
 #define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2	0x00040000
 #define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF		0X00080000
+#define I40E_VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM	0X00100000
 
 struct i40e_virtchnl_vf_resource {
 	u16 num_vsis;
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf.h b/drivers/net/ethernet/intel/i40evf/i40evf.h
index 76ed97db28e2..c5fd724313c7 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf.h
+++ b/drivers/net/ethernet/intel/i40evf/i40evf.h
@@ -59,32 +59,25 @@ struct i40e_vsi {
 	unsigned long state;
 	int base_vector;
 	u16 work_limit;
-	/* high bit set means dynamic, use accessor routines to read/write.
-	 * hardware only supports 2us resolution for the ITR registers.
-	 * these values always store the USER setting, and must be converted
-	 * before programming to a register.
-	 */
-	u16 rx_itr_setting;
-	u16 tx_itr_setting;
 	u16 qs_handle;
 };
 
 /* How many Rx Buffers do we bundle into one write to the hardware ? */
 #define I40EVF_RX_BUFFER_WRITE	16	/* Must be power of 2 */
-#define I40EVF_DEFAULT_TXD   512
-#define I40EVF_DEFAULT_RXD   512
-#define I40EVF_MAX_TXD       4096
-#define I40EVF_MIN_TXD       64
-#define I40EVF_MAX_RXD       4096
-#define I40EVF_MIN_RXD       64
-#define I40EVF_REQ_DESCRIPTOR_MULTIPLE  32
+#define I40EVF_DEFAULT_TXD	512
+#define I40EVF_DEFAULT_RXD	512
+#define I40EVF_MAX_TXD		4096
+#define I40EVF_MIN_TXD		64
+#define I40EVF_MAX_RXD		4096
+#define I40EVF_MIN_RXD		64
+#define I40EVF_REQ_DESCRIPTOR_MULTIPLE	32
 
 /* Supported Rx Buffer Sizes */
-#define I40EVF_RXBUFFER_2048  2048
-#define I40EVF_MAX_RXBUFFER   16384  /* largest size for single descriptor */
-#define I40EVF_MAX_AQ_BUF_SIZE    4096
-#define I40EVF_AQ_LEN             32
-#define I40EVF_AQ_MAX_ERR         20 /* times to try before resetting AQ */
+#define I40EVF_RXBUFFER_2048	2048
+#define I40EVF_MAX_RXBUFFER	16384  /* largest size for single descriptor */
+#define I40EVF_MAX_AQ_BUF_SIZE	4096
+#define I40EVF_AQ_LEN		32
+#define I40EVF_AQ_MAX_ERR	20 /* times to try before resetting AQ */
 
 #define MAXIMUM_ETHERNET_VLAN_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)
 
@@ -111,7 +104,7 @@ struct i40e_q_vector {
 	u8 num_ringpairs;	/* total number of ring pairs in vector */
 #define ITR_COUNTDOWN_START 100
 	u8 itr_countdown;	/* when 0 or 1 update ITR */
-	int v_idx;	  /* vector index in list */
+	int v_idx;	/* vector index in list */
 	char name[IFNAMSIZ + 9];
 	bool arm_wb_state;
 	cpumask_var_t affinity_mask;
@@ -129,11 +122,11 @@ struct i40e_q_vector {
 	((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \
 	(R)->next_to_clean - (R)->next_to_use - 1)
 
-#define I40EVF_RX_DESC_ADV(R, i)	    \
+#define I40EVF_RX_DESC_ADV(R, i)	\
 	(&(((union i40e_adv_rx_desc *)((R).desc))[i]))
-#define I40EVF_TX_DESC_ADV(R, i)	    \
+#define I40EVF_TX_DESC_ADV(R, i)	\
 	(&(((union i40e_adv_tx_desc *)((R).desc))[i]))
-#define I40EVF_TX_CTXTDESC_ADV(R, i)	    \
+#define I40EVF_TX_CTXTDESC_ADV(R, i)	\
 	(&(((struct i40e_adv_tx_context_desc *)((R).desc))[i]))
 
 #define OTHER_VECTOR 1
@@ -204,22 +197,25 @@ struct i40evf_adapter {
 	struct msix_entry *msix_entries;
 
 	u32 flags;
-#define I40EVF_FLAG_RX_CSUM_ENABLED              BIT(0)
-#define I40EVF_FLAG_IMIR_ENABLED                 BIT(5)
-#define I40EVF_FLAG_MQ_CAPABLE                   BIT(6)
-#define I40EVF_FLAG_NEED_LINK_UPDATE             BIT(7)
-#define I40EVF_FLAG_PF_COMMS_FAILED              BIT(8)
-#define I40EVF_FLAG_RESET_PENDING                BIT(9)
-#define I40EVF_FLAG_RESET_NEEDED                 BIT(10)
+#define I40EVF_FLAG_RX_CSUM_ENABLED		BIT(0)
+#define I40EVF_FLAG_IN_NETPOLL			BIT(4)
+#define I40EVF_FLAG_IMIR_ENABLED		BIT(5)
+#define I40EVF_FLAG_MQ_CAPABLE			BIT(6)
+#define I40EVF_FLAG_NEED_LINK_UPDATE		BIT(7)
+#define I40EVF_FLAG_PF_COMMS_FAILED		BIT(8)
+#define I40EVF_FLAG_RESET_PENDING		BIT(9)
+#define I40EVF_FLAG_RESET_NEEDED		BIT(10)
 #define I40EVF_FLAG_WB_ON_ITR_CAPABLE		BIT(11)
 #define I40EVF_FLAG_OUTER_UDP_CSUM_CAPABLE	BIT(12)
 #define I40EVF_FLAG_ADDR_SET_BY_PF		BIT(13)
+#define I40EVF_FLAG_SERVICE_CLIENT_REQUESTED	BIT(14)
 #define I40EVF_FLAG_PROMISC_ON			BIT(15)
 #define I40EVF_FLAG_ALLMULTI_ON			BIT(16)
 /* duplicates for common code */
-#define I40E_FLAG_FDIR_ATR_ENABLED		 0
-#define I40E_FLAG_DCB_ENABLED			 0
-#define I40E_FLAG_RX_CSUM_ENABLED                I40EVF_FLAG_RX_CSUM_ENABLED
+#define I40E_FLAG_FDIR_ATR_ENABLED		0
+#define I40E_FLAG_DCB_ENABLED			0
+#define I40E_FLAG_IN_NETPOLL			I40EVF_FLAG_IN_NETPOLL
+#define I40E_FLAG_RX_CSUM_ENABLED		I40EVF_FLAG_RX_CSUM_ENABLED
 #define I40E_FLAG_WB_ON_ITR_CAPABLE		I40EVF_FLAG_WB_ON_ITR_CAPABLE
 #define I40E_FLAG_OUTER_UDP_CSUM_CAPABLE	I40EVF_FLAG_OUTER_UDP_CSUM_CAPABLE
 	/* flags for admin queue service task */
@@ -233,7 +229,7 @@ struct i40evf_adapter {
 #define I40EVF_FLAG_AQ_CONFIGURE_QUEUES		BIT(6)
 #define I40EVF_FLAG_AQ_MAP_VECTORS		BIT(7)
 #define I40EVF_FLAG_AQ_HANDLE_RESET		BIT(8)
-#define I40EVF_FLAG_AQ_CONFIGURE_RSS		BIT(9)	/* direct AQ config */
+#define I40EVF_FLAG_AQ_CONFIGURE_RSS		BIT(9) /* direct AQ config */
 #define I40EVF_FLAG_AQ_GET_CONFIG		BIT(10)
 /* Newer style, RSS done by the PF so we can ignore hardware vagaries. */
 #define I40EVF_FLAG_AQ_GET_HENA			BIT(11)
@@ -258,6 +254,7 @@ struct i40evf_adapter {
 	struct work_struct watchdog_task;
 	bool netdev_registered;
 	bool link_up;
+	enum i40e_aq_link_speed link_speed;
 	enum i40e_virtchnl_ops current_op;
 #define CLIENT_ENABLED(_a) ((_a)->vf_res ? \
 			    (_a)->vf_res->vf_offload_flags & \
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
index c9c202f6c521..a9940154eead 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
@@ -74,13 +74,33 @@ static const struct i40evf_stats i40evf_gstrings_stats[] = {
 static int i40evf_get_settings(struct net_device *netdev,
 			       struct ethtool_cmd *ecmd)
 {
-	/* In the future the VF will be able to query the PF for
-	 * some information - for now use a dummy value
-	 */
+	struct i40evf_adapter *adapter = netdev_priv(netdev);
+
 	ecmd->supported = 0;
 	ecmd->autoneg = AUTONEG_DISABLE;
 	ecmd->transceiver = XCVR_DUMMY1;
 	ecmd->port = PORT_NONE;
+	/* Set speed and duplex */
+	switch (adapter->link_speed) {
+	case I40E_LINK_SPEED_40GB:
+		ethtool_cmd_speed_set(ecmd, SPEED_40000);
+		break;
+	case I40E_LINK_SPEED_20GB:
+		ethtool_cmd_speed_set(ecmd, SPEED_20000);
+		break;
+	case I40E_LINK_SPEED_10GB:
+		ethtool_cmd_speed_set(ecmd, SPEED_10000);
+		break;
+	case I40E_LINK_SPEED_1GB:
+		ethtool_cmd_speed_set(ecmd, SPEED_1000);
+		break;
+	case I40E_LINK_SPEED_100MB:
+		ethtool_cmd_speed_set(ecmd, SPEED_100);
+		break;
+	default:
+		break;
+	}
+	ecmd->duplex = DUPLEX_FULL;
 
 	return 0;
 }
@@ -276,93 +296,207 @@ static int i40evf_set_ringparam(struct net_device *netdev,
 }
 
 /**
- * i40evf_get_coalesce - Get interrupt coalescing settings
- * @netdev: network interface device structure
- * @ec: ethtool coalesce structure
+ * __i40evf_get_coalesce - get per-queue coalesce settings
+ * @netdev: the netdev to check
+ * @ec: ethtool coalesce data structure
+ * @queue: which queue to pick
  *
- * Returns current coalescing settings. This is referred to elsewhere in the
- * driver as Interrupt Throttle Rate, as this is how the hardware describes
- * this functionality.
+ * Gets the per-queue settings for coalescence. Specifically Rx and Tx usecs
+ * are per queue. If queue is <0 then we default to queue 0 as the
+ * representative value.
  **/
-static int i40evf_get_coalesce(struct net_device *netdev,
-			       struct ethtool_coalesce *ec)
+static int __i40evf_get_coalesce(struct net_device *netdev,
+				 struct ethtool_coalesce *ec,
+				 int queue)
 {
 	struct i40evf_adapter *adapter = netdev_priv(netdev);
 	struct i40e_vsi *vsi = &adapter->vsi;
+	struct i40e_ring *rx_ring, *tx_ring;
 
 	ec->tx_max_coalesced_frames = vsi->work_limit;
 	ec->rx_max_coalesced_frames = vsi->work_limit;
 
-	if (ITR_IS_DYNAMIC(vsi->rx_itr_setting))
+	/* Rx and Tx usecs per queue value. If user doesn't specify the
+	 * queue, return queue 0's value to represent.
+	 */
+	if (queue < 0)
+		queue = 0;
+	else if (queue >= adapter->num_active_queues)
+		return -EINVAL;
+
+	rx_ring = &adapter->rx_rings[queue];
+	tx_ring = &adapter->tx_rings[queue];
+
+	if (ITR_IS_DYNAMIC(rx_ring->rx_itr_setting))
 		ec->use_adaptive_rx_coalesce = 1;
 
-	if (ITR_IS_DYNAMIC(vsi->tx_itr_setting))
+	if (ITR_IS_DYNAMIC(tx_ring->tx_itr_setting))
 		ec->use_adaptive_tx_coalesce = 1;
 
-	ec->rx_coalesce_usecs = vsi->rx_itr_setting & ~I40E_ITR_DYNAMIC;
-	ec->tx_coalesce_usecs = vsi->tx_itr_setting & ~I40E_ITR_DYNAMIC;
+	ec->rx_coalesce_usecs = rx_ring->rx_itr_setting & ~I40E_ITR_DYNAMIC;
+	ec->tx_coalesce_usecs = tx_ring->tx_itr_setting & ~I40E_ITR_DYNAMIC;
 
 	return 0;
 }
 
 /**
- * i40evf_set_coalesce - Set interrupt coalescing settings
+ * i40evf_get_coalesce - Get interrupt coalescing settings
  * @netdev: network interface device structure
  * @ec: ethtool coalesce structure
  *
- * Change current coalescing settings.
+ * Returns current coalescing settings. This is referred to elsewhere in the
+ * driver as Interrupt Throttle Rate, as this is how the hardware describes
+ * this functionality. Note that if per-queue settings have been modified this
+ * only represents the settings of queue 0.
  **/
-static int i40evf_set_coalesce(struct net_device *netdev,
+static int i40evf_get_coalesce(struct net_device *netdev,
 			       struct ethtool_coalesce *ec)
 {
-	struct i40evf_adapter *adapter = netdev_priv(netdev);
-	struct i40e_hw *hw = &adapter->hw;
+	return __i40evf_get_coalesce(netdev, ec, -1);
+}
+
+/**
+ * i40evf_get_per_queue_coalesce - get coalesce values for specific queue
+ * @netdev: netdev to read
+ * @ec: coalesce settings from ethtool
+ * @queue: the queue to read
+ *
+ * Read specific queue's coalesce settings.
+ **/
+static int i40evf_get_per_queue_coalesce(struct net_device *netdev,
+					 u32 queue,
+					 struct ethtool_coalesce *ec)
+{
+	return __i40evf_get_coalesce(netdev, ec, queue);
+}
+
+/**
+ * i40evf_set_itr_per_queue - set ITR values for specific queue
+ * @vsi: the VSI to set values for
+ * @ec: coalesce settings from ethtool
+ * @queue: the queue to modify
+ *
+ * Change the ITR settings for a specific queue.
+ **/
+static void i40evf_set_itr_per_queue(struct i40evf_adapter *adapter,
+				     struct ethtool_coalesce *ec,
+				     int queue)
+{
 	struct i40e_vsi *vsi = &adapter->vsi;
+	struct i40e_hw *hw = &adapter->hw;
 	struct i40e_q_vector *q_vector;
+	u16 vector;
+
+	adapter->rx_rings[queue].rx_itr_setting = ec->rx_coalesce_usecs;
+	adapter->tx_rings[queue].tx_itr_setting = ec->tx_coalesce_usecs;
+
+	if (ec->use_adaptive_rx_coalesce)
+		adapter->rx_rings[queue].rx_itr_setting |= I40E_ITR_DYNAMIC;
+	else
+		adapter->rx_rings[queue].rx_itr_setting &= ~I40E_ITR_DYNAMIC;
+
+	if (ec->use_adaptive_tx_coalesce)
+		adapter->tx_rings[queue].tx_itr_setting |= I40E_ITR_DYNAMIC;
+	else
+		adapter->tx_rings[queue].tx_itr_setting &= ~I40E_ITR_DYNAMIC;
+
+	q_vector = adapter->rx_rings[queue].q_vector;
+	q_vector->rx.itr = ITR_TO_REG(adapter->rx_rings[queue].rx_itr_setting);
+	vector = vsi->base_vector + q_vector->v_idx;
+	wr32(hw, I40E_VFINT_ITRN1(I40E_RX_ITR, vector - 1), q_vector->rx.itr);
+
+	q_vector = adapter->tx_rings[queue].q_vector;
+	q_vector->tx.itr = ITR_TO_REG(adapter->tx_rings[queue].tx_itr_setting);
+	vector = vsi->base_vector + q_vector->v_idx;
+	wr32(hw, I40E_VFINT_ITRN1(I40E_TX_ITR, vector - 1), q_vector->tx.itr);
+
+	i40e_flush(hw);
+}
+
+/**
+ * __i40evf_set_coalesce - set coalesce settings for particular queue
+ * @netdev: the netdev to change
+ * @ec: ethtool coalesce settings
+ * @queue: the queue to change
+ *
+ * Sets the coalesce settings for a particular queue.
+ **/
+static int __i40evf_set_coalesce(struct net_device *netdev,
+				 struct ethtool_coalesce *ec,
+				 int queue)
+{
+	struct i40evf_adapter *adapter = netdev_priv(netdev);
+	struct i40e_vsi *vsi = &adapter->vsi;
 	int i;
 
 	if (ec->tx_max_coalesced_frames_irq || ec->rx_max_coalesced_frames_irq)
 		vsi->work_limit = ec->tx_max_coalesced_frames_irq;
 
-	if ((ec->rx_coalesce_usecs >= (I40E_MIN_ITR << 1)) &&
-	    (ec->rx_coalesce_usecs <= (I40E_MAX_ITR << 1)))
-		vsi->rx_itr_setting = ec->rx_coalesce_usecs;
-
-	else
+	if (ec->rx_coalesce_usecs == 0) {
+		if (ec->use_adaptive_rx_coalesce)
+			netif_info(adapter, drv, netdev, "rx-usecs=0, need to disable adaptive-rx for a complete disable\n");
+	} else if ((ec->rx_coalesce_usecs < (I40E_MIN_ITR << 1)) ||
+		   (ec->rx_coalesce_usecs > (I40E_MAX_ITR << 1))) {
+		netif_info(adapter, drv, netdev, "Invalid value, rx-usecs range is 0-8160\n");
 		return -EINVAL;
+	}
 
-	if ((ec->tx_coalesce_usecs >= (I40E_MIN_ITR << 1)) &&
-	    (ec->tx_coalesce_usecs <= (I40E_MAX_ITR << 1)))
-		vsi->tx_itr_setting = ec->tx_coalesce_usecs;
-	else if (ec->use_adaptive_tx_coalesce)
-		vsi->tx_itr_setting = (I40E_ITR_DYNAMIC |
-				       ITR_REG_TO_USEC(I40E_ITR_RX_DEF));
 	else
+	if (ec->tx_coalesce_usecs == 0) {
+		if (ec->use_adaptive_tx_coalesce)
+			netif_info(adapter, drv, netdev, "tx-usecs=0, need to disable adaptive-tx for a complete disable\n");
+	} else if ((ec->tx_coalesce_usecs < (I40E_MIN_ITR << 1)) ||
+		   (ec->tx_coalesce_usecs > (I40E_MAX_ITR << 1))) {
+		netif_info(adapter, drv, netdev, "Invalid value, tx-usecs range is 0-8160\n");
 		return -EINVAL;
+	}
 
-	if (ec->use_adaptive_rx_coalesce)
-		vsi->rx_itr_setting |= I40E_ITR_DYNAMIC;
-	else
-		vsi->rx_itr_setting &= ~I40E_ITR_DYNAMIC;
-
-	if (ec->use_adaptive_tx_coalesce)
-		vsi->tx_itr_setting |= I40E_ITR_DYNAMIC;
-	else
-		vsi->tx_itr_setting &= ~I40E_ITR_DYNAMIC;
-
-	for (i = 0; i < adapter->num_msix_vectors - NONQ_VECS; i++) {
-		q_vector = &adapter->q_vectors[i];
-		q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
-		wr32(hw, I40E_VFINT_ITRN1(0, i), q_vector->rx.itr);
-		q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
-		wr32(hw, I40E_VFINT_ITRN1(1, i), q_vector->tx.itr);
-		i40e_flush(hw);
+	/* Rx and Tx usecs has per queue value. If user doesn't specify the
+	 * queue, apply to all queues.
+	 */
+	if (queue < 0) {
+		for (i = 0; i < adapter->num_active_queues; i++)
+			i40evf_set_itr_per_queue(adapter, ec, i);
+	} else if (queue < adapter->num_active_queues) {
+		i40evf_set_itr_per_queue(adapter, ec, queue);
+	} else {
+		netif_info(adapter, drv, netdev, "Invalid queue value, queue range is 0 - %d\n",
+			   adapter->num_active_queues - 1);
+		return -EINVAL;
 	}
 
 	return 0;
 }
 
 /**
+ * i40evf_set_coalesce - Set interrupt coalescing settings
+ * @netdev: network interface device structure
+ * @ec: ethtool coalesce structure
+ *
+ * Change current coalescing settings for every queue.
+ **/
+static int i40evf_set_coalesce(struct net_device *netdev,
+			       struct ethtool_coalesce *ec)
+{
+	return __i40evf_set_coalesce(netdev, ec, -1);
+}
+
+/**
+ * i40evf_set_per_queue_coalesce - set specific queue's coalesce settings
+ * @netdev: the netdev to change
+ * @ec: ethtool's coalesce settings
+ * @queue: the queue to modify
+ *
+ * Modifies a specific queue's coalesce settings.
+ */
+static int i40evf_set_per_queue_coalesce(struct net_device *netdev,
+					 u32 queue,
+					 struct ethtool_coalesce *ec)
+{
+	return __i40evf_set_coalesce(netdev, ec, queue);
+}
+
+/**
  * i40evf_get_rxnfc - command to get RX flow classification rules
  * @netdev: network interface device structure
  * @cmd: ethtool rxnfc command
@@ -513,6 +647,8 @@ static const struct ethtool_ops i40evf_ethtool_ops = {
 	.set_msglevel		= i40evf_set_msglevel,
 	.get_coalesce		= i40evf_get_coalesce,
 	.set_coalesce		= i40evf_set_coalesce,
+	.get_per_queue_coalesce = i40evf_get_per_queue_coalesce,
+	.set_per_queue_coalesce = i40evf_set_per_queue_coalesce,
 	.get_rxnfc		= i40evf_get_rxnfc,
 	.get_rxfh_indir_size	= i40evf_get_rxfh_indir_size,
 	.get_rxfh		= i40evf_get_rxfh,
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_main.c b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
index 600fb9c4a7f0..14372810fc27 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_main.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
@@ -38,7 +38,7 @@ static const char i40evf_driver_string[] =
 
 #define DRV_VERSION_MAJOR 1
 #define DRV_VERSION_MINOR 6
-#define DRV_VERSION_BUILD 11
+#define DRV_VERSION_BUILD 16
 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
 	     __stringify(DRV_VERSION_MINOR) "." \
 	     __stringify(DRV_VERSION_BUILD) \
@@ -370,6 +370,7 @@ i40evf_map_vector_to_rxq(struct i40evf_adapter *adapter, int v_idx, int r_idx)
 {
 	struct i40e_q_vector *q_vector = &adapter->q_vectors[v_idx];
 	struct i40e_ring *rx_ring = &adapter->rx_rings[r_idx];
+	struct i40e_hw *hw = &adapter->hw;
 
 	rx_ring->q_vector = q_vector;
 	rx_ring->next = q_vector->rx.ring;
@@ -377,7 +378,10 @@ i40evf_map_vector_to_rxq(struct i40evf_adapter *adapter, int v_idx, int r_idx)
 	q_vector->rx.ring = rx_ring;
 	q_vector->rx.count++;
 	q_vector->rx.latency_range = I40E_LOW_LATENCY;
+	q_vector->rx.itr = ITR_TO_REG(rx_ring->rx_itr_setting);
+	q_vector->ring_mask |= BIT(r_idx);
 	q_vector->itr_countdown = ITR_COUNTDOWN_START;
+	wr32(hw, I40E_VFINT_ITRN1(I40E_RX_ITR, v_idx - 1), q_vector->rx.itr);
 }
 
 /**
@@ -391,6 +395,7 @@ i40evf_map_vector_to_txq(struct i40evf_adapter *adapter, int v_idx, int t_idx)
 {
 	struct i40e_q_vector *q_vector = &adapter->q_vectors[v_idx];
 	struct i40e_ring *tx_ring = &adapter->tx_rings[t_idx];
+	struct i40e_hw *hw = &adapter->hw;
 
 	tx_ring->q_vector = q_vector;
 	tx_ring->next = q_vector->tx.ring;
@@ -398,9 +403,10 @@ i40evf_map_vector_to_txq(struct i40evf_adapter *adapter, int v_idx, int t_idx)
 	q_vector->tx.ring = tx_ring;
 	q_vector->tx.count++;
 	q_vector->tx.latency_range = I40E_LOW_LATENCY;
+	q_vector->tx.itr = ITR_TO_REG(tx_ring->tx_itr_setting);
 	q_vector->itr_countdown = ITR_COUNTDOWN_START;
 	q_vector->num_ringpairs++;
-	q_vector->ring_mask |= BIT(t_idx);
+	wr32(hw, I40E_VFINT_ITRN1(I40E_TX_ITR, v_idx - 1), q_vector->tx.itr);
 }
 
 /**
@@ -1007,7 +1013,7 @@ static void i40evf_configure(struct i40evf_adapter *adapter)
  * i40evf_up_complete - Finish the last steps of bringing up a connection
  * @adapter: board private structure
  **/
-static int i40evf_up_complete(struct i40evf_adapter *adapter)
+static void i40evf_up_complete(struct i40evf_adapter *adapter)
 {
 	adapter->state = __I40EVF_RUNNING;
 	clear_bit(__I40E_DOWN, &adapter->vsi.state);
@@ -1016,7 +1022,6 @@ static int i40evf_up_complete(struct i40evf_adapter *adapter)
 
 	adapter->aq_required |= I40EVF_FLAG_AQ_ENABLE_QUEUES;
 	mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
-	return 0;
 }
 
 /**
@@ -1037,6 +1042,7 @@ void i40evf_down(struct i40evf_adapter *adapter)
 
 	netif_carrier_off(netdev);
 	netif_tx_disable(netdev);
+	adapter->link_up = false;
 	i40evf_napi_disable_all(adapter);
 	i40evf_irq_disable(adapter);
 
@@ -1154,6 +1160,7 @@ static int i40evf_alloc_queues(struct i40evf_adapter *adapter)
 		tx_ring->netdev = adapter->netdev;
 		tx_ring->dev = &adapter->pdev->dev;
 		tx_ring->count = adapter->tx_desc_count;
+		tx_ring->tx_itr_setting = (I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF);
 		if (adapter->flags & I40E_FLAG_WB_ON_ITR_CAPABLE)
 			tx_ring->flags |= I40E_TXR_FLAGS_WB_ON_ITR;
 
@@ -1162,6 +1169,7 @@ static int i40evf_alloc_queues(struct i40evf_adapter *adapter)
 		rx_ring->netdev = adapter->netdev;
 		rx_ring->dev = &adapter->pdev->dev;
 		rx_ring->count = adapter->rx_desc_count;
+		rx_ring->rx_itr_setting = (I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF);
 	}
 
 	return 0;
@@ -1420,7 +1428,9 @@ int i40evf_init_interrupt_scheme(struct i40evf_adapter *adapter)
 {
 	int err;
 
+	rtnl_lock();
 	err = i40evf_set_interrupt_capability(adapter);
+	rtnl_unlock();
 	if (err) {
 		dev_err(&adapter->pdev->dev,
 			"Unable to setup interrupt capabilities\n");
@@ -1729,6 +1739,7 @@ static void i40evf_reset_task(struct work_struct *work)
 			set_bit(__I40E_DOWN, &adapter->vsi.state);
 			netif_carrier_off(netdev);
 			netif_tx_disable(netdev);
+			adapter->link_up = false;
 			i40evf_napi_disable_all(adapter);
 			i40evf_irq_disable(adapter);
 			i40evf_free_traffic_irqs(adapter);
@@ -1767,6 +1778,7 @@ continue_reset:
 	if (netif_running(adapter->netdev)) {
 		netif_carrier_off(netdev);
 		netif_tx_stop_all_queues(netdev);
+		adapter->link_up = false;
 		i40evf_napi_disable_all(adapter);
 	}
 	i40evf_irq_disable(adapter);
@@ -1781,8 +1793,7 @@ continue_reset:
 	i40evf_free_all_tx_resources(adapter);
 
 	/* kill and reinit the admin queue */
-	if (i40evf_shutdown_adminq(hw))
-		dev_warn(&adapter->pdev->dev, "Failed to shut down adminq\n");
+	i40evf_shutdown_adminq(hw);
 	adapter->current_op = I40E_VIRTCHNL_OP_UNKNOWN;
 	err = i40evf_init_adminq(hw);
 	if (err)
@@ -1802,6 +1813,8 @@ continue_reset:
 	}
 	adapter->aq_required |= I40EVF_FLAG_AQ_ADD_MAC_FILTER;
 	adapter->aq_required |= I40EVF_FLAG_AQ_ADD_VLAN_FILTER;
+	/* Open RDMA Client again */
+	adapter->aq_required |= I40EVF_FLAG_SERVICE_CLIENT_REQUESTED;
 	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 	i40evf_misc_irq_enable(adapter);
 
@@ -1820,9 +1833,7 @@ continue_reset:
 
 		i40evf_configure(adapter);
 
-		err = i40evf_up_complete(adapter);
-		if (err)
-			goto reset_err;
+		i40evf_up_complete(adapter);
 
 		i40evf_irq_enable(adapter, true);
 	} else {
@@ -2052,9 +2063,7 @@ static int i40evf_open(struct net_device *netdev)
 	i40evf_add_filter(adapter, adapter->hw.mac.addr);
 	i40evf_configure(adapter);
 
-	err = i40evf_up_complete(adapter);
-	if (err)
-		goto err_req_irq;
+	i40evf_up_complete(adapter);
 
 	i40evf_irq_enable(adapter, true);
 
@@ -2268,10 +2277,6 @@ int i40evf_process_config(struct i40evf_adapter *adapter)
 	adapter->vsi.back = adapter;
 	adapter->vsi.base_vector = 1;
 	adapter->vsi.work_limit = I40E_DEFAULT_IRQ_WORK;
-	adapter->vsi.rx_itr_setting = (I40E_ITR_DYNAMIC |
-				       ITR_REG_TO_USEC(I40E_ITR_RX_DEF));
-	adapter->vsi.tx_itr_setting = (I40E_ITR_DYNAMIC |
-				       ITR_REG_TO_USEC(I40E_ITR_TX_DEF));
 	vsi->netdev = adapter->netdev;
 	vsi->qs_handle = adapter->vsi_res->qset_handle;
 	if (vfres->vf_offload_flags & I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF) {
@@ -2453,6 +2458,7 @@ static void i40evf_init_task(struct work_struct *work)
 		goto err_sw_init;
 
 	netif_carrier_off(netdev);
+	adapter->link_up = false;
 
 	if (!adapter->netdev_registered) {
 		err = register_netdev(netdev);
@@ -2831,7 +2837,8 @@ static int __init i40evf_init_module(void)
 
 	pr_info("%s\n", i40evf_copyright);
 
-	i40evf_wq = create_singlethread_workqueue(i40evf_driver_name);
+	i40evf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
+				    i40evf_driver_name);
 	if (!i40evf_wq) {
 		pr_err("%s: Failed to create workqueue\n", i40evf_driver_name);
 		return -ENOMEM;
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
index d76c221d4c8a..ddf478d6322b 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
@@ -817,6 +817,45 @@ void i40evf_set_rss_lut(struct i40evf_adapter *adapter)
 }
 
 /**
+ * i40evf_print_link_message - print link up or down
+ * @adapter: adapter structure
+ *
+ * Log a message telling the world of our wonderous link status
+ */
+static void i40evf_print_link_message(struct i40evf_adapter *adapter)
+{
+	struct net_device *netdev = adapter->netdev;
+	char *speed = "Unknown ";
+
+	if (!adapter->link_up) {
+		netdev_info(netdev, "NIC Link is Down\n");
+		return;
+	}
+
+	switch (adapter->link_speed) {
+	case I40E_LINK_SPEED_40GB:
+		speed = "40 G";
+		break;
+	case I40E_LINK_SPEED_20GB:
+		speed = "20 G";
+		break;
+	case I40E_LINK_SPEED_10GB:
+		speed = "10 G";
+		break;
+	case I40E_LINK_SPEED_1GB:
+		speed = "1000 M";
+		break;
+	case I40E_LINK_SPEED_100MB:
+		speed = "100 M";
+		break;
+	default:
+		break;
+	}
+
+	netdev_info(netdev, "NIC Link is Up %sbps Full Duplex\n", speed);
+}
+
+/**
  * i40evf_request_reset
  * @adapter: adapter structure
  *
@@ -853,16 +892,20 @@ void i40evf_virtchnl_completion(struct i40evf_adapter *adapter,
 			(struct i40e_virtchnl_pf_event *)msg;
 		switch (vpe->event) {
 		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
-			adapter->link_up =
-				vpe->event_data.link_event.link_status;
-			if (adapter->link_up && !netif_carrier_ok(netdev)) {
-				dev_info(&adapter->pdev->dev, "NIC Link is Up\n");
-				netif_carrier_on(netdev);
-				netif_tx_wake_all_queues(netdev);
-			} else if (!adapter->link_up) {
-				dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
-				netif_carrier_off(netdev);
-				netif_tx_stop_all_queues(netdev);
+			adapter->link_speed =
+				vpe->event_data.link_event.link_speed;
+			if (adapter->link_up !=
+			    vpe->event_data.link_event.link_status) {
+				adapter->link_up =
+					vpe->event_data.link_event.link_status;
+				if (adapter->link_up) {
+					netif_tx_start_all_queues(netdev);
+					netif_carrier_on(netdev);
+				} else {
+					netif_tx_stop_all_queues(netdev);
+					netif_carrier_off(netdev);
+				}
+				i40evf_print_link_message(adapter);
 			}
 			break;
 		case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
@@ -937,8 +980,6 @@ void i40evf_virtchnl_completion(struct i40evf_adapter *adapter,
 	case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
 		/* enable transmits */
 		i40evf_irq_enable(adapter, true);
-		netif_tx_start_all_queues(adapter->netdev);
-		netif_carrier_on(adapter->netdev);
 		break;
 	case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
 		i40evf_free_all_tx_resources(adapter);
diff --git a/drivers/net/ethernet/intel/igb/e1000_82575.h b/drivers/net/ethernet/intel/igb/e1000_82575.h
index 199ff98209cf..acf06051e111 100644
--- a/drivers/net/ethernet/intel/igb/e1000_82575.h
+++ b/drivers/net/ethernet/intel/igb/e1000_82575.h
@@ -188,6 +188,11 @@ struct e1000_adv_tx_context_desc {
 /* ETQF register bit definitions */
 #define E1000_ETQF_FILTER_ENABLE   BIT(26)
 #define E1000_ETQF_1588            BIT(30)
+#define E1000_ETQF_IMM_INT         BIT(29)
+#define E1000_ETQF_QUEUE_ENABLE    BIT(31)
+#define E1000_ETQF_QUEUE_SHIFT     16
+#define E1000_ETQF_QUEUE_MASK      0x00070000
+#define E1000_ETQF_ETYPE_MASK      0x0000FFFF
 
 /* FTQF register bit definitions */
 #define E1000_FTQF_VF_BP               0x00008000
diff --git a/drivers/net/ethernet/intel/igb/e1000_defines.h b/drivers/net/ethernet/intel/igb/e1000_defines.h
index 2997c443c5dc..2688180a7acd 100644
--- a/drivers/net/ethernet/intel/igb/e1000_defines.h
+++ b/drivers/net/ethernet/intel/igb/e1000_defines.h
@@ -1024,4 +1024,8 @@
 #define E1000_RTTBCNRC_RF_INT_MASK	\
 	(E1000_RTTBCNRC_RF_DEC_MASK << E1000_RTTBCNRC_RF_INT_SHIFT)
 
+#define E1000_VLAPQF_QUEUE_SEL(_n, q_idx) (q_idx << ((_n) * 4))
+#define E1000_VLAPQF_P_VALID(_n)	(0x1 << (3 + (_n) * 4))
+#define E1000_VLAPQF_QUEUE_MASK	0x03
+
 #endif
diff --git a/drivers/net/ethernet/intel/igb/e1000_regs.h b/drivers/net/ethernet/intel/igb/e1000_regs.h
index 21d9d02885cb..d84afdd83e53 100644
--- a/drivers/net/ethernet/intel/igb/e1000_regs.h
+++ b/drivers/net/ethernet/intel/igb/e1000_regs.h
@@ -309,6 +309,7 @@
 					(0x054E0 + ((_i - 16) * 8)))
 #define E1000_RAH(_i)  (((_i) <= 15) ? (0x05404 + ((_i) * 8)) : \
 					(0x054E4 + ((_i - 16) * 8)))
+#define E1000_VLAPQF	0x055B0  /* VLAN Priority Queue Filter VLAPQF */
 #define E1000_IP4AT_REG(_i)     (0x05840 + ((_i) * 8))
 #define E1000_IP6AT_REG(_i)     (0x05880 + ((_i) * 4))
 #define E1000_WUPM_REG(_i)      (0x05A00 + ((_i) * 4))
diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h
index 5387b3a96489..d11093dce1b9 100644
--- a/drivers/net/ethernet/intel/igb/igb.h
+++ b/drivers/net/ethernet/intel/igb/igb.h
@@ -350,11 +350,49 @@ struct hwmon_buff {
 	};
 #endif
 
+/* The number of L2 ether-type filter registers, Index 3 is reserved
+ * for PTP 1588 timestamp
+ */
+#define MAX_ETYPE_FILTER	(4 - 1)
+/* ETQF filter list: one static filter per filter consumer. This is
+ * to avoid filter collisions later. Add new filters here!!
+ *
+ * Current filters:		Filter 3
+ */
+#define IGB_ETQF_FILTER_1588	3
+
 #define IGB_N_EXTTS	2
 #define IGB_N_PEROUT	2
 #define IGB_N_SDP	4
 #define IGB_RETA_SIZE	128
 
+enum igb_filter_match_flags {
+	IGB_FILTER_FLAG_ETHER_TYPE = 0x1,
+	IGB_FILTER_FLAG_VLAN_TCI   = 0x2,
+};
+
+#define IGB_MAX_RXNFC_FILTERS 16
+
+/* RX network flow classification data structure */
+struct igb_nfc_input {
+	/* Byte layout in order, all values with MSB first:
+	 * match_flags - 1 byte
+	 * etype - 2 bytes
+	 * vlan_tci - 2 bytes
+	 */
+	u8 match_flags;
+	__be16 etype;
+	__be16 vlan_tci;
+};
+
+struct igb_nfc_filter {
+	struct hlist_node nfc_node;
+	struct igb_nfc_input filter;
+	u16 etype_reg_index;
+	u16 sw_idx;
+	u16 action;
+};
+
 /* board specific private data structure */
 struct igb_adapter {
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
@@ -451,6 +489,7 @@ struct igb_adapter {
 	struct timecounter tc;
 	u32 tx_hwtstamp_timeouts;
 	u32 rx_hwtstamp_cleared;
+	bool pps_sys_wrap_on;
 
 	struct ptp_pin_desc sdp_config[IGB_N_SDP];
 	struct {
@@ -473,6 +512,13 @@ struct igb_adapter {
 	int copper_tries;
 	struct e1000_info ei;
 	u16 eee_advert;
+
+	/* RX network flow classification support */
+	struct hlist_head nfc_filter_list;
+	unsigned int nfc_filter_count;
+	/* lock for RX network flow classification filter */
+	spinlock_t nfc_lock;
+	bool etype_bitmap[MAX_ETYPE_FILTER];
 };
 
 /* flags controlling PTP/1588 function */
@@ -599,4 +645,9 @@ static inline struct netdev_queue *txring_txq(const struct igb_ring *tx_ring)
 	return netdev_get_tx_queue(tx_ring->netdev, tx_ring->queue_index);
 }
 
+int igb_add_filter(struct igb_adapter *adapter,
+		   struct igb_nfc_filter *input);
+int igb_erase_filter(struct igb_adapter *adapter,
+		     struct igb_nfc_filter *input);
+
 #endif /* _IGB_H_ */
diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c
index 64e91c575a39..737b664d004c 100644
--- a/drivers/net/ethernet/intel/igb/igb_ethtool.c
+++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c
@@ -2431,6 +2431,63 @@ static int igb_get_ts_info(struct net_device *dev,
 	}
 }
 
+#define ETHER_TYPE_FULL_MASK ((__force __be16)~0)
+static int igb_get_ethtool_nfc_entry(struct igb_adapter *adapter,
+				     struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fsp = &cmd->fs;
+	struct igb_nfc_filter *rule = NULL;
+
+	/* report total rule count */
+	cmd->data = IGB_MAX_RXNFC_FILTERS;
+
+	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) {
+		if (fsp->location <= rule->sw_idx)
+			break;
+	}
+
+	if (!rule || fsp->location != rule->sw_idx)
+		return -EINVAL;
+
+	if (rule->filter.match_flags) {
+		fsp->flow_type = ETHER_FLOW;
+		fsp->ring_cookie = rule->action;
+		if (rule->filter.match_flags & IGB_FILTER_FLAG_ETHER_TYPE) {
+			fsp->h_u.ether_spec.h_proto = rule->filter.etype;
+			fsp->m_u.ether_spec.h_proto = ETHER_TYPE_FULL_MASK;
+		}
+		if (rule->filter.match_flags & IGB_FILTER_FLAG_VLAN_TCI) {
+			fsp->flow_type |= FLOW_EXT;
+			fsp->h_ext.vlan_tci = rule->filter.vlan_tci;
+			fsp->m_ext.vlan_tci = htons(VLAN_PRIO_MASK);
+		}
+		return 0;
+	}
+	return -EINVAL;
+}
+
+static int igb_get_ethtool_nfc_all(struct igb_adapter *adapter,
+				   struct ethtool_rxnfc *cmd,
+				   u32 *rule_locs)
+{
+	struct igb_nfc_filter *rule;
+	int cnt = 0;
+
+	/* report total rule count */
+	cmd->data = IGB_MAX_RXNFC_FILTERS;
+
+	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) {
+		if (cnt == cmd->rule_cnt)
+			return -EMSGSIZE;
+		rule_locs[cnt] = rule->sw_idx;
+		cnt++;
+	}
+
+	cmd->rule_cnt = cnt;
+
+	return 0;
+}
+
 static int igb_get_rss_hash_opts(struct igb_adapter *adapter,
 				 struct ethtool_rxnfc *cmd)
 {
@@ -2484,6 +2541,16 @@ static int igb_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
 		cmd->data = adapter->num_rx_queues;
 		ret = 0;
 		break;
+	case ETHTOOL_GRXCLSRLCNT:
+		cmd->rule_cnt = adapter->nfc_filter_count;
+		ret = 0;
+		break;
+	case ETHTOOL_GRXCLSRULE:
+		ret = igb_get_ethtool_nfc_entry(adapter, cmd);
+		break;
+	case ETHTOOL_GRXCLSRLALL:
+		ret = igb_get_ethtool_nfc_all(adapter, cmd, rule_locs);
+		break;
 	case ETHTOOL_GRXFH:
 		ret = igb_get_rss_hash_opts(adapter, cmd);
 		break;
@@ -2598,6 +2665,279 @@ static int igb_set_rss_hash_opt(struct igb_adapter *adapter,
 	return 0;
 }
 
+static int igb_rxnfc_write_etype_filter(struct igb_adapter *adapter,
+					struct igb_nfc_filter *input)
+{
+	struct e1000_hw *hw = &adapter->hw;
+	u8 i;
+	u32 etqf;
+	u16 etype;
+
+	/* find an empty etype filter register */
+	for (i = 0; i < MAX_ETYPE_FILTER; ++i) {
+		if (!adapter->etype_bitmap[i])
+			break;
+	}
+	if (i == MAX_ETYPE_FILTER) {
+		dev_err(&adapter->pdev->dev, "ethtool -N: etype filters are all used.\n");
+		return -EINVAL;
+	}
+
+	adapter->etype_bitmap[i] = true;
+
+	etqf = rd32(E1000_ETQF(i));
+	etype = ntohs(input->filter.etype & ETHER_TYPE_FULL_MASK);
+
+	etqf |= E1000_ETQF_FILTER_ENABLE;
+	etqf &= ~E1000_ETQF_ETYPE_MASK;
+	etqf |= (etype & E1000_ETQF_ETYPE_MASK);
+
+	etqf &= ~E1000_ETQF_QUEUE_MASK;
+	etqf |= ((input->action << E1000_ETQF_QUEUE_SHIFT)
+		& E1000_ETQF_QUEUE_MASK);
+	etqf |= E1000_ETQF_QUEUE_ENABLE;
+
+	wr32(E1000_ETQF(i), etqf);
+
+	input->etype_reg_index = i;
+
+	return 0;
+}
+
+static int igb_rxnfc_write_vlan_prio_filter(struct igb_adapter *adapter,
+					    struct igb_nfc_filter *input)
+{
+	struct e1000_hw *hw = &adapter->hw;
+	u8 vlan_priority;
+	u16 queue_index;
+	u32 vlapqf;
+
+	vlapqf = rd32(E1000_VLAPQF);
+	vlan_priority = (ntohs(input->filter.vlan_tci) & VLAN_PRIO_MASK)
+				>> VLAN_PRIO_SHIFT;
+	queue_index = (vlapqf >> (vlan_priority * 4)) & E1000_VLAPQF_QUEUE_MASK;
+
+	/* check whether this vlan prio is already set */
+	if ((vlapqf & E1000_VLAPQF_P_VALID(vlan_priority)) &&
+	    (queue_index != input->action)) {
+		dev_err(&adapter->pdev->dev, "ethtool rxnfc set vlan prio filter failed.\n");
+		return -EEXIST;
+	}
+
+	vlapqf |= E1000_VLAPQF_P_VALID(vlan_priority);
+	vlapqf |= E1000_VLAPQF_QUEUE_SEL(vlan_priority, input->action);
+
+	wr32(E1000_VLAPQF, vlapqf);
+
+	return 0;
+}
+
+int igb_add_filter(struct igb_adapter *adapter, struct igb_nfc_filter *input)
+{
+	int err = -EINVAL;
+
+	if (input->filter.match_flags & IGB_FILTER_FLAG_ETHER_TYPE) {
+		err = igb_rxnfc_write_etype_filter(adapter, input);
+		if (err)
+			return err;
+	}
+
+	if (input->filter.match_flags & IGB_FILTER_FLAG_VLAN_TCI)
+		err = igb_rxnfc_write_vlan_prio_filter(adapter, input);
+
+	return err;
+}
+
+static void igb_clear_etype_filter_regs(struct igb_adapter *adapter,
+					u16 reg_index)
+{
+	struct e1000_hw *hw = &adapter->hw;
+	u32 etqf = rd32(E1000_ETQF(reg_index));
+
+	etqf &= ~E1000_ETQF_QUEUE_ENABLE;
+	etqf &= ~E1000_ETQF_QUEUE_MASK;
+	etqf &= ~E1000_ETQF_FILTER_ENABLE;
+
+	wr32(E1000_ETQF(reg_index), etqf);
+
+	adapter->etype_bitmap[reg_index] = false;
+}
+
+static void igb_clear_vlan_prio_filter(struct igb_adapter *adapter,
+				       u16 vlan_tci)
+{
+	struct e1000_hw *hw = &adapter->hw;
+	u8 vlan_priority;
+	u32 vlapqf;
+
+	vlan_priority = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+
+	vlapqf = rd32(E1000_VLAPQF);
+	vlapqf &= ~E1000_VLAPQF_P_VALID(vlan_priority);
+	vlapqf &= ~E1000_VLAPQF_QUEUE_SEL(vlan_priority,
+						E1000_VLAPQF_QUEUE_MASK);
+
+	wr32(E1000_VLAPQF, vlapqf);
+}
+
+int igb_erase_filter(struct igb_adapter *adapter, struct igb_nfc_filter *input)
+{
+	if (input->filter.match_flags & IGB_FILTER_FLAG_ETHER_TYPE)
+		igb_clear_etype_filter_regs(adapter,
+					    input->etype_reg_index);
+
+	if (input->filter.match_flags & IGB_FILTER_FLAG_VLAN_TCI)
+		igb_clear_vlan_prio_filter(adapter,
+					   ntohs(input->filter.vlan_tci));
+
+	return 0;
+}
+
+static int igb_update_ethtool_nfc_entry(struct igb_adapter *adapter,
+					struct igb_nfc_filter *input,
+					u16 sw_idx)
+{
+	struct igb_nfc_filter *rule, *parent;
+	int err = -EINVAL;
+
+	parent = NULL;
+	rule = NULL;
+
+	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) {
+		/* hash found, or no matching entry */
+		if (rule->sw_idx >= sw_idx)
+			break;
+		parent = rule;
+	}
+
+	/* if there is an old rule occupying our place remove it */
+	if (rule && (rule->sw_idx == sw_idx)) {
+		if (!input)
+			err = igb_erase_filter(adapter, rule);
+
+		hlist_del(&rule->nfc_node);
+		kfree(rule);
+		adapter->nfc_filter_count--;
+	}
+
+	/* If no input this was a delete, err should be 0 if a rule was
+	 * successfully found and removed from the list else -EINVAL
+	 */
+	if (!input)
+		return err;
+
+	/* initialize node */
+	INIT_HLIST_NODE(&input->nfc_node);
+
+	/* add filter to the list */
+	if (parent)
+		hlist_add_behind(&parent->nfc_node, &input->nfc_node);
+	else
+		hlist_add_head(&input->nfc_node, &adapter->nfc_filter_list);
+
+	/* update counts */
+	adapter->nfc_filter_count++;
+
+	return 0;
+}
+
+static int igb_add_ethtool_nfc_entry(struct igb_adapter *adapter,
+				     struct ethtool_rxnfc *cmd)
+{
+	struct net_device *netdev = adapter->netdev;
+	struct ethtool_rx_flow_spec *fsp =
+		(struct ethtool_rx_flow_spec *)&cmd->fs;
+	struct igb_nfc_filter *input, *rule;
+	int err = 0;
+
+	if (!(netdev->hw_features & NETIF_F_NTUPLE))
+		return -EOPNOTSUPP;
+
+	/* Don't allow programming if the action is a queue greater than
+	 * the number of online Rx queues.
+	 */
+	if ((fsp->ring_cookie == RX_CLS_FLOW_DISC) ||
+	    (fsp->ring_cookie >= adapter->num_rx_queues)) {
+		dev_err(&adapter->pdev->dev, "ethtool -N: The specified action is invalid\n");
+		return -EINVAL;
+	}
+
+	/* Don't allow indexes to exist outside of available space */
+	if (fsp->location >= IGB_MAX_RXNFC_FILTERS) {
+		dev_err(&adapter->pdev->dev, "Location out of range\n");
+		return -EINVAL;
+	}
+
+	if ((fsp->flow_type & ~FLOW_EXT) != ETHER_FLOW)
+		return -EINVAL;
+
+	if (fsp->m_u.ether_spec.h_proto != ETHER_TYPE_FULL_MASK &&
+	    fsp->m_ext.vlan_tci != htons(VLAN_PRIO_MASK))
+		return -EINVAL;
+
+	input = kzalloc(sizeof(*input), GFP_KERNEL);
+	if (!input)
+		return -ENOMEM;
+
+	if (fsp->m_u.ether_spec.h_proto == ETHER_TYPE_FULL_MASK) {
+		input->filter.etype = fsp->h_u.ether_spec.h_proto;
+		input->filter.match_flags = IGB_FILTER_FLAG_ETHER_TYPE;
+	}
+
+	if ((fsp->flow_type & FLOW_EXT) && fsp->m_ext.vlan_tci) {
+		if (fsp->m_ext.vlan_tci != htons(VLAN_PRIO_MASK)) {
+			err = -EINVAL;
+			goto err_out;
+		}
+		input->filter.vlan_tci = fsp->h_ext.vlan_tci;
+		input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI;
+	}
+
+	input->action = fsp->ring_cookie;
+	input->sw_idx = fsp->location;
+
+	spin_lock(&adapter->nfc_lock);
+
+	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) {
+		if (!memcmp(&input->filter, &rule->filter,
+			    sizeof(input->filter))) {
+			err = -EEXIST;
+			dev_err(&adapter->pdev->dev,
+				"ethtool: this filter is already set\n");
+			goto err_out_w_lock;
+		}
+	}
+
+	err = igb_add_filter(adapter, input);
+	if (err)
+		goto err_out_w_lock;
+
+	igb_update_ethtool_nfc_entry(adapter, input, input->sw_idx);
+
+	spin_unlock(&adapter->nfc_lock);
+	return 0;
+
+err_out_w_lock:
+	spin_unlock(&adapter->nfc_lock);
+err_out:
+	kfree(input);
+	return err;
+}
+
+static int igb_del_ethtool_nfc_entry(struct igb_adapter *adapter,
+				     struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fsp =
+		(struct ethtool_rx_flow_spec *)&cmd->fs;
+	int err;
+
+	spin_lock(&adapter->nfc_lock);
+	err = igb_update_ethtool_nfc_entry(adapter, NULL, fsp->location);
+	spin_unlock(&adapter->nfc_lock);
+
+	return err;
+}
+
 static int igb_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
 {
 	struct igb_adapter *adapter = netdev_priv(dev);
@@ -2607,6 +2947,11 @@ static int igb_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
 	case ETHTOOL_SRXFH:
 		ret = igb_set_rss_hash_opt(adapter, cmd);
 		break;
+	case ETHTOOL_SRXCLSRLINS:
+		ret = igb_add_ethtool_nfc_entry(adapter, cmd);
+		break;
+	case ETHTOOL_SRXCLSRLDEL:
+		ret = igb_del_ethtool_nfc_entry(adapter, cmd);
 	default:
 		break;
 	}
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 942a89fb0090..edc9a6ac5169 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -58,7 +58,7 @@
 #include "igb.h"
 
 #define MAJ 5
-#define MIN 3
+#define MIN 4
 #define BUILD 0
 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
 __stringify(BUILD) "-k"
@@ -169,13 +169,15 @@ static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
-			       int vf, u16 vlan, u8 qos);
+			       int vf, u16 vlan, u8 qos, __be16 vlan_proto);
 static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
 				   bool setting);
 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 				 struct ifla_vf_info *ivi);
 static void igb_check_vf_rate_limit(struct igb_adapter *);
+static void igb_nfc_filter_exit(struct igb_adapter *adapter);
+static void igb_nfc_filter_restore(struct igb_adapter *adapter);
 
 #ifdef CONFIG_PCI_IOV
 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
@@ -1611,6 +1613,7 @@ static void igb_configure(struct igb_adapter *adapter)
 	igb_setup_mrqc(adapter);
 	igb_setup_rctl(adapter);
 
+	igb_nfc_filter_restore(adapter);
 	igb_configure_tx(adapter);
 	igb_configure_rx(adapter);
 
@@ -2059,6 +2062,21 @@ static int igb_set_features(struct net_device *netdev,
 	if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
 		return 0;
 
+	if (!(features & NETIF_F_NTUPLE)) {
+		struct hlist_node *node2;
+		struct igb_nfc_filter *rule;
+
+		spin_lock(&adapter->nfc_lock);
+		hlist_for_each_entry_safe(rule, node2,
+					  &adapter->nfc_filter_list, nfc_node) {
+			igb_erase_filter(adapter, rule);
+			hlist_del(&rule->nfc_node);
+			kfree(rule);
+		}
+		spin_unlock(&adapter->nfc_lock);
+		adapter->nfc_filter_count = 0;
+	}
+
 	netdev->features = features;
 
 	if (netif_running(netdev))
@@ -3053,6 +3071,7 @@ static int igb_sw_init(struct igb_adapter *adapter)
 				  VLAN_HLEN;
 	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
 
+	spin_lock_init(&adapter->nfc_lock);
 	spin_lock_init(&adapter->stats64_lock);
 #ifdef CONFIG_PCI_IOV
 	switch (hw->mac.type) {
@@ -3240,6 +3259,8 @@ static int __igb_close(struct net_device *netdev, bool suspending)
 	igb_down(adapter);
 	igb_free_irq(adapter);
 
+	igb_nfc_filter_exit(adapter);
+
 	igb_free_all_tx_resources(adapter);
 	igb_free_all_rx_resources(adapter);
 
@@ -6201,14 +6222,17 @@ static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf)
 	return 0;
 }
 
-static int igb_ndo_set_vf_vlan(struct net_device *netdev,
-			       int vf, u16 vlan, u8 qos)
+static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf,
+			       u16 vlan, u8 qos, __be16 vlan_proto)
 {
 	struct igb_adapter *adapter = netdev_priv(netdev);
 
 	if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
 		return -EINVAL;
 
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
 	return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) :
 			       igb_disable_port_vlan(adapter, vf);
 }
@@ -8306,4 +8330,28 @@ int igb_reinit_queues(struct igb_adapter *adapter)
 
 	return err;
 }
+
+static void igb_nfc_filter_exit(struct igb_adapter *adapter)
+{
+	struct igb_nfc_filter *rule;
+
+	spin_lock(&adapter->nfc_lock);
+
+	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
+		igb_erase_filter(adapter, rule);
+
+	spin_unlock(&adapter->nfc_lock);
+}
+
+static void igb_nfc_filter_restore(struct igb_adapter *adapter)
+{
+	struct igb_nfc_filter *rule;
+
+	spin_lock(&adapter->nfc_lock);
+
+	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
+		igb_add_filter(adapter, rule);
+
+	spin_unlock(&adapter->nfc_lock);
+}
 /* igb_main.c */
diff --git a/drivers/net/ethernet/intel/igb/igb_ptp.c b/drivers/net/ethernet/intel/igb/igb_ptp.c
index e61b647f5f2a..a7895c4cbcc3 100644
--- a/drivers/net/ethernet/intel/igb/igb_ptp.c
+++ b/drivers/net/ethernet/intel/igb/igb_ptp.c
@@ -591,6 +591,7 @@ static int igb_ptp_feature_enable_i210(struct ptp_clock_info *ptp,
 			tsim |= TSINTR_SYS_WRAP;
 		else
 			tsim &= ~TSINTR_SYS_WRAP;
+		igb->pps_sys_wrap_on = !!on;
 		wr32(E1000_TSIM, tsim);
 		spin_unlock_irqrestore(&igb->tmreg_lock, flags);
 		return 0;
@@ -744,7 +745,8 @@ static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter)
 		}
 	}
 
-	shhwtstamps.hwtstamp = ktime_sub_ns(shhwtstamps.hwtstamp, adjust);
+	shhwtstamps.hwtstamp =
+		ktime_add_ns(shhwtstamps.hwtstamp, adjust);
 
 	skb_tstamp_tx(adapter->ptp_tx_skb, &shhwtstamps);
 	dev_kfree_skb_any(adapter->ptp_tx_skb);
@@ -767,13 +769,32 @@ void igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector,
 			 struct sk_buff *skb)
 {
 	__le64 *regval = (__le64 *)va;
+	struct igb_adapter *adapter = q_vector->adapter;
+	int adjust = 0;
 
 	/* The timestamp is recorded in little endian format.
 	 * DWORD: 0        1        2        3
 	 * Field: Reserved Reserved SYSTIML  SYSTIMH
 	 */
-	igb_ptp_systim_to_hwtstamp(q_vector->adapter, skb_hwtstamps(skb),
+	igb_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb),
 				   le64_to_cpu(regval[1]));
+
+	/* adjust timestamp for the RX latency based on link speed */
+	if (adapter->hw.mac.type == e1000_i210) {
+		switch (adapter->link_speed) {
+		case SPEED_10:
+			adjust = IGB_I210_RX_LATENCY_10;
+			break;
+		case SPEED_100:
+			adjust = IGB_I210_RX_LATENCY_100;
+			break;
+		case SPEED_1000:
+			adjust = IGB_I210_RX_LATENCY_1000;
+			break;
+		}
+	}
+	skb_hwtstamps(skb)->hwtstamp =
+		ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
 }
 
 /**
@@ -825,7 +846,7 @@ void igb_ptp_rx_rgtstamp(struct igb_q_vector *q_vector,
 		}
 	}
 	skb_hwtstamps(skb)->hwtstamp =
-		ktime_add_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
+		ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
 
 	/* Update the last_rx_timestamp timer in order to enable watchdog check
 	 * for error case of latched timestamp on a dropped packet.
@@ -978,12 +999,12 @@ static int igb_ptp_set_timestamp_mode(struct igb_adapter *adapter,
 
 	/* define ethertype filter for timestamped packets */
 	if (is_l2)
-		wr32(E1000_ETQF(3),
+		wr32(E1000_ETQF(IGB_ETQF_FILTER_1588),
 		     (E1000_ETQF_FILTER_ENABLE | /* enable filter */
 		      E1000_ETQF_1588 | /* enable timestamping */
 		      ETH_P_1588));     /* 1588 eth protocol type */
 	else
-		wr32(E1000_ETQF(3), 0);
+		wr32(E1000_ETQF(IGB_ETQF_FILTER_1588), 0);
 
 	/* L4 Queue Filter[3]: filter by destination port and protocol */
 	if (is_l4) {
@@ -1139,7 +1160,7 @@ void igb_ptp_init(struct igb_adapter *adapter)
 	if (IS_ERR(adapter->ptp_clock)) {
 		adapter->ptp_clock = NULL;
 		dev_err(&adapter->pdev->dev, "ptp_clock_register failed\n");
-	} else {
+	} else if (adapter->ptp_clock) {
 		dev_info(&adapter->pdev->dev, "added PHC on %s\n",
 			 adapter->netdev->name);
 		adapter->ptp_flags |= IGB_PTP_ENABLED;
@@ -1215,7 +1236,9 @@ void igb_ptp_reset(struct igb_adapter *adapter)
 	case e1000_i211:
 		wr32(E1000_TSAUXC, 0x0);
 		wr32(E1000_TSSDP, 0x0);
-		wr32(E1000_TSIM, TSYNC_INTERRUPTS);
+		wr32(E1000_TSIM,
+		     TSYNC_INTERRUPTS |
+		     (adapter->pps_sys_wrap_on ? TSINTR_SYS_WRAP : 0));
 		wr32(E1000_IMS, E1000_IMS_TS);
 		break;
 	default:
diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c
index b0778ba65083..12bb877df860 100644
--- a/drivers/net/ethernet/intel/igbvf/netdev.c
+++ b/drivers/net/ethernet/intel/igbvf/netdev.c
@@ -47,7 +47,7 @@
 
 #include "igbvf.h"
 
-#define DRV_VERSION "2.0.2-k"
+#define DRV_VERSION "2.4.0-k"
 char igbvf_driver_name[] = "igbvf";
 const char igbvf_driver_version[] = DRV_VERSION;
 static const char igbvf_driver_string[] =
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
index 9475ff9055aa..b06e32d0d22a 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
@@ -45,10 +45,10 @@
 #include "ixgbe_type.h"
 #include "ixgbe_common.h"
 #include "ixgbe_dcb.h"
-#if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
+#if IS_ENABLED(CONFIG_FCOE)
 #define IXGBE_FCOE
 #include "ixgbe_fcoe.h"
-#endif /* CONFIG_FCOE or CONFIG_FCOE_MODULE */
+#endif /* IS_ENABLED(CONFIG_FCOE) */
 #ifdef CONFIG_IXGBE_DCA
 #include <linux/dca.h>
 #endif
@@ -645,6 +645,7 @@ struct ixgbe_adapter {
 #define IXGBE_FLAG_RX_HWTSTAMP_ENABLED		BIT(25)
 #define IXGBE_FLAG_RX_HWTSTAMP_IN_REGISTER	BIT(26)
 #define IXGBE_FLAG_DCB_CAPABLE			BIT(27)
+#define IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE	BIT(28)
 
 	u32 flags2;
 #define IXGBE_FLAG2_RSC_CAPABLE			BIT(0)
@@ -653,13 +654,12 @@ struct ixgbe_adapter {
 #define IXGBE_FLAG2_TEMP_SENSOR_EVENT		BIT(3)
 #define IXGBE_FLAG2_SEARCH_FOR_SFP		BIT(4)
 #define IXGBE_FLAG2_SFP_NEEDS_RESET		BIT(5)
-#define IXGBE_FLAG2_RESET_REQUESTED		BIT(6)
 #define IXGBE_FLAG2_FDIR_REQUIRES_REINIT	BIT(7)
 #define IXGBE_FLAG2_RSS_FIELD_IPV4_UDP		BIT(8)
 #define IXGBE_FLAG2_RSS_FIELD_IPV6_UDP		BIT(9)
 #define IXGBE_FLAG2_PTP_PPS_ENABLED		BIT(10)
 #define IXGBE_FLAG2_PHY_INTERRUPT		BIT(11)
-#define IXGBE_FLAG2_VXLAN_REREG_NEEDED		BIT(12)
+#define IXGBE_FLAG2_UDP_TUN_REREG_NEEDED	BIT(12)
 #define IXGBE_FLAG2_VLAN_PROMISC		BIT(13)
 
 	/* Tx fast path data */
@@ -673,6 +673,7 @@ struct ixgbe_adapter {
 
 	/* Port number used to identify VXLAN traffic */
 	__be16 vxlan_port;
+	__be16 geneve_port;
 
 	/* TX */
 	struct ixgbe_ring *tx_ring[MAX_TX_QUEUES] ____cacheline_aligned_in_smp;
@@ -840,6 +841,7 @@ enum ixgbe_state_t {
 	__IXGBE_IN_SFP_INIT,
 	__IXGBE_PTP_RUNNING,
 	__IXGBE_PTP_TX_IN_PROGRESS,
+	__IXGBE_RESET_REQUESTED,
 };
 
 struct ixgbe_cb {
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
index b4217f30e89c..77d3039283f6 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
@@ -99,6 +99,7 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw)
 		case IXGBE_DEV_ID_X550T:
 		case IXGBE_DEV_ID_X550T1:
 		case IXGBE_DEV_ID_X550EM_X_10G_T:
+		case IXGBE_DEV_ID_X550EM_A_10G_T:
 			supported = true;
 			break;
 		default:
@@ -2958,8 +2959,10 @@ s32 ixgbe_clear_vmdq_generic(struct ixgbe_hw *hw, u32 rar, u32 vmdq)
 	}
 
 	/* was that the last pool using this rar? */
-	if (mpsar_lo == 0 && mpsar_hi == 0 && rar != 0)
+	if (mpsar_lo == 0 && mpsar_hi == 0 &&
+	    rar != 0 && rar != hw->mac.san_mac_rar_index)
 		hw->mac.ops.clear_rar(hw, rar);
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
index 0d7209eb5abf..f49f80380aa5 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
@@ -193,7 +193,9 @@ static int ixgbe_get_settings(struct net_device *netdev,
 	if (supported_link & IXGBE_LINK_SPEED_10GB_FULL)
 		ecmd->supported |= ixgbe_get_supported_10gtypes(hw);
 	if (supported_link & IXGBE_LINK_SPEED_1GB_FULL)
-		ecmd->supported |= SUPPORTED_1000baseT_Full;
+		ecmd->supported |= (ixgbe_isbackplane(hw->phy.media_type)) ?
+				   SUPPORTED_1000baseKX_Full :
+				   SUPPORTED_1000baseT_Full;
 	if (supported_link & IXGBE_LINK_SPEED_100_FULL)
 		ecmd->supported |= ixgbe_isbackplane(hw->phy.media_type) ?
 				   SUPPORTED_1000baseKX_Full :
@@ -311,6 +313,25 @@ static int ixgbe_get_settings(struct net_device *netdev,
 		break;
 	}
 
+	/* Indicate pause support */
+	ecmd->supported |= SUPPORTED_Pause;
+
+	switch (hw->fc.requested_mode) {
+	case ixgbe_fc_full:
+		ecmd->advertising |= ADVERTISED_Pause;
+		break;
+	case ixgbe_fc_rx_pause:
+		ecmd->advertising |= ADVERTISED_Pause |
+				     ADVERTISED_Asym_Pause;
+		break;
+	case ixgbe_fc_tx_pause:
+		ecmd->advertising |= ADVERTISED_Asym_Pause;
+		break;
+	default:
+		ecmd->advertising &= ~(ADVERTISED_Pause |
+				       ADVERTISED_Asym_Pause);
+	}
+
 	if (netif_carrier_ok(netdev)) {
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
@@ -2926,9 +2947,13 @@ static u32 ixgbe_rss_indir_size(struct net_device *netdev)
 static void ixgbe_get_reta(struct ixgbe_adapter *adapter, u32 *indir)
 {
 	int i, reta_size = ixgbe_rss_indir_tbl_entries(adapter);
+	u16 rss_m = adapter->ring_feature[RING_F_RSS].mask;
+
+	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
+		rss_m = adapter->ring_feature[RING_F_RSS].indices - 1;
 
 	for (i = 0; i < reta_size; i++)
-		indir[i] = adapter->rss_indir_tbl[i];
+		indir[i] = adapter->rss_indir_tbl[i] & rss_m;
 }
 
 static int ixgbe_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
@@ -3039,8 +3064,8 @@ static unsigned int ixgbe_max_channels(struct ixgbe_adapter *adapter)
 		/* We only support one q_vector without MSI-X */
 		max_combined = 1;
 	} else if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
-		/* SR-IOV currently only allows one queue on the PF */
-		max_combined = 1;
+		/* Limit value based on the queue mask */
+		max_combined = adapter->ring_feature[RING_F_RSS].mask + 1;
 	} else if (tcs > 1) {
 		/* For DCB report channels per traffic class */
 		if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
index bcdc88444ceb..15ab337fd7ad 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
@@ -515,15 +515,16 @@ static bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter)
 	vmdq_i = min_t(u16, IXGBE_MAX_VMDQ_INDICES, vmdq_i);
 
 	/* 64 pool mode with 2 queues per pool */
-	if ((vmdq_i > 32) || (rss_i < 4) || (vmdq_i > 16 && pools)) {
+	if ((vmdq_i > 32) || (vmdq_i > 16 && pools)) {
 		vmdq_m = IXGBE_82599_VMDQ_2Q_MASK;
 		rss_m = IXGBE_RSS_2Q_MASK;
 		rss_i = min_t(u16, rss_i, 2);
-	/* 32 pool mode with 4 queues per pool */
+	/* 32 pool mode with up to 4 queues per pool */
 	} else {
 		vmdq_m = IXGBE_82599_VMDQ_4Q_MASK;
 		rss_m = IXGBE_RSS_4Q_MASK;
-		rss_i = 4;
+		/* We can support 4, 2, or 1 queues */
+		rss_i = (rss_i > 3) ? 4 : (rss_i > 1) ? 2 : 1;
 	}
 
 #ifdef IXGBE_FCOE
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 5418c69a7463..a244d9a67264 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -137,6 +137,7 @@ static const struct pci_device_id ixgbe_pci_tbl[] = {
 	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },
 	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a },
 	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },
+	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), board_x550em_a},
 	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },
 	/* required last entry */
 	{0, }
@@ -1103,7 +1104,7 @@ static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
 
 	/* Do the reset outside of interrupt context */
 	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
-		adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
+		set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
 		e_warn(drv, "initiating reset due to tx timeout\n");
 		ixgbe_service_event_schedule(adapter);
 	}
@@ -1495,7 +1496,6 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
 				     struct sk_buff *skb)
 {
 	__le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
-	__le16 hdr_info = rx_desc->wb.lower.lo_dword.hs_rss.hdr_info;
 	bool encap_pkt = false;
 
 	skb_checksum_none_assert(skb);
@@ -1504,8 +1504,8 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
 	if (!(ring->netdev->features & NETIF_F_RXCSUM))
 		return;
 
-	if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) &&
-	    (hdr_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_TUNNEL >> 16))) {
+	/* check for VXLAN and Geneve packets */
+	if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) {
 		encap_pkt = true;
 		skb->encapsulation = 1;
 	}
@@ -2777,7 +2777,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data)
 		}
 		if (eicr & IXGBE_EICR_ECC) {
 			e_info(link, "Received ECC Err, initiating reset\n");
-			adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
+			set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
 			ixgbe_service_event_schedule(adapter);
 			IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
 		}
@@ -3007,7 +3007,7 @@ static irqreturn_t ixgbe_intr(int irq, void *data)
 	case ixgbe_mac_x550em_a:
 		if (eicr & IXGBE_EICR_ECC) {
 			e_info(link, "Received ECC Err, initiating reset\n");
-			adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
+			set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
 			ixgbe_service_event_schedule(adapter);
 			IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
 		}
@@ -3224,7 +3224,7 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
 		txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
 	} while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
 	if (!wait_loop)
-		e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
+		hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
 }
 
 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
@@ -3248,7 +3248,8 @@ static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
 			mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
 		else if (tcs > 1)
 			mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
-		else if (adapter->ring_feature[RING_F_RSS].indices == 4)
+		else if (adapter->ring_feature[RING_F_VMDQ].mask ==
+			 IXGBE_82599_VMDQ_4Q_MASK)
 			mtqc |= IXGBE_MTQC_32VF;
 		else
 			mtqc |= IXGBE_MTQC_64VF;
@@ -3475,12 +3476,12 @@ static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)
 	u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
 	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
 
-	/* Program table for at least 2 queues w/ SR-IOV so that VFs can
+	/* Program table for at least 4 queues w/ SR-IOV so that VFs can
 	 * make full use of any rings they may have.  We will use the
 	 * PSRTYPE register to control how many rings we use within the PF.
 	 */
-	if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 2))
-		rss_i = 2;
+	if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 4))
+		rss_i = 4;
 
 	/* Fill out hash function seeds */
 	for (i = 0; i < 10; i++)
@@ -3544,7 +3545,8 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
 				mrqc = IXGBE_MRQC_VMDQRT8TCEN;	/* 8 TCs */
 			else if (tcs > 1)
 				mrqc = IXGBE_MRQC_VMDQRT4TCEN;	/* 4 TCs */
-			else if (adapter->ring_feature[RING_F_RSS].indices == 4)
+			else if (adapter->ring_feature[RING_F_VMDQ].mask ==
+				 IXGBE_82599_VMDQ_4Q_MASK)
 				mrqc = IXGBE_MRQC_VMDQRSS32EN;
 			else
 				mrqc = IXGBE_MRQC_VMDQRSS64EN;
@@ -3922,6 +3924,9 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
 	rfctl &= ~IXGBE_RFCTL_RSC_DIS;
 	if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))
 		rfctl |= IXGBE_RFCTL_RSC_DIS;
+
+	/* disable NFS filtering */
+	rfctl |= (IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS);
 	IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
 
 	/* Program registers for the distribution of queues */
@@ -4100,24 +4105,22 @@ static void ixgbe_vlan_promisc_enable(struct ixgbe_adapter *adapter)
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32 vlnctrl, i;
 
-	switch (hw->mac.type) {
-	case ixgbe_mac_82599EB:
-	case ixgbe_mac_X540:
-	case ixgbe_mac_X550:
-	case ixgbe_mac_X550EM_x:
-	case ixgbe_mac_x550em_a:
-	default:
-		if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED)
-			break;
-		/* fall through */
-	case ixgbe_mac_82598EB:
-		/* legacy case, we can just disable VLAN filtering */
-		vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
-		vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
+	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
+
+	if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) {
+	/* For VMDq and SR-IOV we must leave VLAN filtering enabled */
+		vlnctrl |= IXGBE_VLNCTRL_VFE;
+		IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
+	} else {
+		vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 		IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 		return;
 	}
 
+	/* Nothing to do for 82598 */
+	if (hw->mac.type == ixgbe_mac_82598EB)
+		return;
+
 	/* We are already in VLAN promisc, nothing to do */
 	if (adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)
 		return;
@@ -4191,23 +4194,14 @@ static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter)
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32 vlnctrl, i;
 
-	switch (hw->mac.type) {
-	case ixgbe_mac_82599EB:
-	case ixgbe_mac_X540:
-	case ixgbe_mac_X550:
-	case ixgbe_mac_X550EM_x:
-	case ixgbe_mac_x550em_a:
-	default:
-		if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED)
-			break;
-		/* fall through */
-	case ixgbe_mac_82598EB:
-		vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
-		vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
-		vlnctrl |= IXGBE_VLNCTRL_VFE;
-		IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
+	/* Set VLAN filtering to enabled */
+	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
+	vlnctrl |= IXGBE_VLNCTRL_VFE;
+	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
+
+	if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) ||
+	    hw->mac.type == ixgbe_mac_82598EB)
 		return;
-	}
 
 	/* We are not in VLAN promisc, nothing to do */
 	if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
@@ -4580,18 +4574,23 @@ static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
 	}
 }
 
-static void ixgbe_clear_vxlan_port(struct ixgbe_adapter *adapter)
+static void ixgbe_clear_udp_tunnel_port(struct ixgbe_adapter *adapter, u32 mask)
 {
-	switch (adapter->hw.mac.type) {
-	case ixgbe_mac_X550:
-	case ixgbe_mac_X550EM_x:
-	case ixgbe_mac_x550em_a:
-		IXGBE_WRITE_REG(&adapter->hw, IXGBE_VXLANCTRL, 0);
+	struct ixgbe_hw *hw = &adapter->hw;
+	u32 vxlanctrl;
+
+	if (!(adapter->flags & (IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE |
+				IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE)))
+		return;
+
+	vxlanctrl = IXGBE_READ_REG(hw, IXGBE_VXLANCTRL) && ~mask;
+	IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, vxlanctrl);
+
+	if (mask & IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK)
 		adapter->vxlan_port = 0;
-		break;
-	default:
-		break;
-	}
+
+	if (mask & IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK)
+		adapter->geneve_port = 0;
 }
 
 #ifdef CONFIG_IXGBE_DCB
@@ -5494,8 +5493,8 @@ void ixgbe_down(struct ixgbe_adapter *adapter)
 
 	ixgbe_napi_disable_all(adapter);
 
-	adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT |
-			     IXGBE_FLAG2_RESET_REQUESTED);
+	clear_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
+	adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
 	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
 
 	del_timer_sync(&adapter->service_timer);
@@ -5705,8 +5704,10 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter)
 		if (fwsm & IXGBE_FWSM_TS_ENABLED)
 			adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
 		break;
-	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_x550em_a:
+		adapter->flags |= IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE;
+	/* fall through */
+	case ixgbe_mac_X550EM_x:
 #ifdef CONFIG_IXGBE_DCB
 		adapter->flags &= ~IXGBE_FLAG_DCB_CAPABLE;
 #endif
@@ -6138,7 +6139,7 @@ int ixgbe_open(struct net_device *netdev)
 
 	ixgbe_up_complete(adapter);
 
-	ixgbe_clear_vxlan_port(adapter);
+	ixgbe_clear_udp_tunnel_port(adapter, IXGBE_VXLANCTRL_ALL_UDPPORT_MASK);
 	udp_tunnel_get_rx_info(netdev);
 
 	return 0;
@@ -6915,7 +6916,7 @@ static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
 			 * (Do the reset outside of interrupt context).
 			 */
 			e_warn(drv, "initiating reset to clear Tx work after link loss\n");
-			adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
+			set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
 		}
 	}
 }
@@ -7181,11 +7182,9 @@ static void ixgbe_phy_interrupt_subtask(struct ixgbe_adapter *adapter)
 
 static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
 {
-	if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED))
+	if (!test_and_clear_bit(__IXGBE_RESET_REQUESTED, &adapter->state))
 		return;
 
-	adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED;
-
 	/* If we're already down, removing or resetting, just bail */
 	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
 	    test_bit(__IXGBE_REMOVING, &adapter->state) ||
@@ -7219,9 +7218,9 @@ static void ixgbe_service_task(struct work_struct *work)
 		ixgbe_service_event_complete(adapter);
 		return;
 	}
-	if (adapter->flags2 & IXGBE_FLAG2_VXLAN_REREG_NEEDED) {
+	if (adapter->flags2 & IXGBE_FLAG2_UDP_TUN_REREG_NEEDED) {
 		rtnl_lock();
-		adapter->flags2 &= ~IXGBE_FLAG2_VXLAN_REREG_NEEDED;
+		adapter->flags2 &= ~IXGBE_FLAG2_UDP_TUN_REREG_NEEDED;
 		udp_tunnel_get_rx_info(adapter->netdev);
 		rtnl_unlock();
 	}
@@ -7661,6 +7660,10 @@ static void ixgbe_atr(struct ixgbe_ring *ring,
 		if (adapter->vxlan_port &&
 		    udp_hdr(skb)->dest == adapter->vxlan_port)
 			hdr.network = skb_inner_network_header(skb);
+
+		if (adapter->geneve_port &&
+		    udp_hdr(skb)->dest == adapter->geneve_port)
+			hdr.network = skb_inner_network_header(skb);
 	}
 
 	/* Currently only IPv4/IPv6 with TCP is supported */
@@ -8390,12 +8393,14 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter,
 			    struct tcf_exts *exts, u64 *action, u8 *queue)
 {
 	const struct tc_action *a;
+	LIST_HEAD(actions);
 	int err;
 
 	if (tc_no_actions(exts))
 		return -EINVAL;
 
-	tc_for_each_action(a, exts) {
+	tcf_exts_to_list(exts, &actions);
+	list_for_each_entry(a, &actions, list) {
 
 		/* Drop action */
 		if (is_tcf_gact_shot(a)) {
@@ -8794,10 +8799,23 @@ static int ixgbe_set_features(struct net_device *netdev,
 	netdev->features = features;
 
 	if ((adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE)) {
-		if (features & NETIF_F_RXCSUM)
-			adapter->flags2 |= IXGBE_FLAG2_VXLAN_REREG_NEEDED;
-		else
-			ixgbe_clear_vxlan_port(adapter);
+		if (features & NETIF_F_RXCSUM) {
+			adapter->flags2 |= IXGBE_FLAG2_UDP_TUN_REREG_NEEDED;
+		} else {
+			u32 port_mask = IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK;
+
+			ixgbe_clear_udp_tunnel_port(adapter, port_mask);
+		}
+	}
+
+	if ((adapter->flags & IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE)) {
+		if (features & NETIF_F_RXCSUM) {
+			adapter->flags2 |= IXGBE_FLAG2_UDP_TUN_REREG_NEEDED;
+		} else {
+			u32 port_mask = IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK;
+
+			ixgbe_clear_udp_tunnel_port(adapter, port_mask);
+		}
 	}
 
 	if (need_reset)
@@ -8810,67 +8828,115 @@ static int ixgbe_set_features(struct net_device *netdev,
 }
 
 /**
- * ixgbe_add_vxlan_port - Get notifications about VXLAN ports that come up
+ * ixgbe_add_udp_tunnel_port - Get notifications about adding UDP tunnel ports
  * @dev: The port's netdev
  * @ti: Tunnel endpoint information
  **/
-static void ixgbe_add_vxlan_port(struct net_device *dev,
-				 struct udp_tunnel_info *ti)
+static void ixgbe_add_udp_tunnel_port(struct net_device *dev,
+				      struct udp_tunnel_info *ti)
 {
 	struct ixgbe_adapter *adapter = netdev_priv(dev);
 	struct ixgbe_hw *hw = &adapter->hw;
 	__be16 port = ti->port;
-
-	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
-		return;
+	u32 port_shift = 0;
+	u32 reg;
 
 	if (ti->sa_family != AF_INET)
 		return;
 
-	if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
-		return;
+	switch (ti->type) {
+	case UDP_TUNNEL_TYPE_VXLAN:
+		if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
+			return;
 
-	if (adapter->vxlan_port == port)
-		return;
+		if (adapter->vxlan_port == port)
+			return;
+
+		if (adapter->vxlan_port) {
+			netdev_info(dev,
+				    "VXLAN port %d set, not adding port %d\n",
+				    ntohs(adapter->vxlan_port),
+				    ntohs(port));
+			return;
+		}
 
-	if (adapter->vxlan_port) {
-		netdev_info(dev,
-			    "Hit Max num of VXLAN ports, not adding port %d\n",
-			    ntohs(port));
+		adapter->vxlan_port = port;
+		break;
+	case UDP_TUNNEL_TYPE_GENEVE:
+		if (!(adapter->flags & IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE))
+			return;
+
+		if (adapter->geneve_port == port)
+			return;
+
+		if (adapter->geneve_port) {
+			netdev_info(dev,
+				    "GENEVE port %d set, not adding port %d\n",
+				    ntohs(adapter->geneve_port),
+				    ntohs(port));
+			return;
+		}
+
+		port_shift = IXGBE_VXLANCTRL_GENEVE_UDPPORT_SHIFT;
+		adapter->geneve_port = port;
+		break;
+	default:
 		return;
 	}
 
-	adapter->vxlan_port = port;
-	IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, ntohs(port));
+	reg = IXGBE_READ_REG(hw, IXGBE_VXLANCTRL) | ntohs(port) << port_shift;
+	IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, reg);
 }
 
 /**
- * ixgbe_del_vxlan_port - Get notifications about VXLAN ports that go away
+ * ixgbe_del_udp_tunnel_port - Get notifications about removing UDP tunnel ports
  * @dev: The port's netdev
  * @ti: Tunnel endpoint information
  **/
-static void ixgbe_del_vxlan_port(struct net_device *dev,
-				 struct udp_tunnel_info *ti)
+static void ixgbe_del_udp_tunnel_port(struct net_device *dev,
+				      struct udp_tunnel_info *ti)
 {
 	struct ixgbe_adapter *adapter = netdev_priv(dev);
+	u32 port_mask;
 
-	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+	if (ti->type != UDP_TUNNEL_TYPE_VXLAN &&
+	    ti->type != UDP_TUNNEL_TYPE_GENEVE)
 		return;
 
 	if (ti->sa_family != AF_INET)
 		return;
 
-	if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
-		return;
+	switch (ti->type) {
+	case UDP_TUNNEL_TYPE_VXLAN:
+		if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
+			return;
 
-	if (adapter->vxlan_port != ti->port) {
-		netdev_info(dev, "Port %d was not found, not deleting\n",
-			    ntohs(ti->port));
+		if (adapter->vxlan_port != ti->port) {
+			netdev_info(dev, "VXLAN port %d not found\n",
+				    ntohs(ti->port));
+			return;
+		}
+
+		port_mask = IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK;
+		break;
+	case UDP_TUNNEL_TYPE_GENEVE:
+		if (!(adapter->flags & IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE))
+			return;
+
+		if (adapter->geneve_port != ti->port) {
+			netdev_info(dev, "GENEVE port %d not found\n",
+				    ntohs(ti->port));
+			return;
+		}
+
+		port_mask = IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK;
+		break;
+	default:
 		return;
 	}
 
-	ixgbe_clear_vxlan_port(adapter);
-	adapter->flags2 |= IXGBE_FLAG2_VXLAN_REREG_NEEDED;
+	ixgbe_clear_udp_tunnel_port(adapter, port_mask);
+	adapter->flags2 |= IXGBE_FLAG2_UDP_TUN_REREG_NEEDED;
 }
 
 static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
@@ -9184,8 +9250,8 @@ static const struct net_device_ops ixgbe_netdev_ops = {
 	.ndo_bridge_getlink	= ixgbe_ndo_bridge_getlink,
 	.ndo_dfwd_add_station	= ixgbe_fwd_add,
 	.ndo_dfwd_del_station	= ixgbe_fwd_del,
-	.ndo_udp_tunnel_add	= ixgbe_add_vxlan_port,
-	.ndo_udp_tunnel_del	= ixgbe_del_vxlan_port,
+	.ndo_udp_tunnel_add	= ixgbe_add_udp_tunnel_port,
+	.ndo_udp_tunnel_del	= ixgbe_del_udp_tunnel_port,
 	.ndo_features_check	= ixgbe_features_check,
 };
 
@@ -9517,6 +9583,7 @@ skip_sriov:
 
 	/* copy netdev features into list of user selectable features */
 	netdev->hw_features |= netdev->features |
+			       NETIF_F_HW_VLAN_CTAG_FILTER |
 			       NETIF_F_HW_VLAN_CTAG_RX |
 			       NETIF_F_HW_VLAN_CTAG_TX |
 			       NETIF_F_RXALL |
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c
index db0731e05401..021ab9b89c71 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c
@@ -346,8 +346,8 @@ s32 ixgbe_identify_phy_generic(struct ixgbe_hw *hw)
 				return 0;
 			}
 		}
-		/* clear value if nothing found */
-		hw->phy.mdio.prtad = 0;
+		/* indicate no PHY found */
+		hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
 		return IXGBE_ERR_PHY_ADDR_INVALID;
 	}
 	return 0;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c
index e5431bfe3339..a92277683a64 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c
@@ -1254,7 +1254,7 @@ static long ixgbe_ptp_create_clock(struct ixgbe_adapter *adapter)
 		adapter->ptp_clock = NULL;
 		e_dev_err("ptp_clock_register failed\n");
 		return err;
-	} else
+	} else if (adapter->ptp_clock)
 		e_dev_info("registered PHC device on %s\n", netdev->name);
 
 	/* set default timestamp mode to disabled here. We do this in
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
index 8618599dfd6f..7e5d9850e4b2 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
@@ -329,13 +329,15 @@ static int ixgbe_pci_sriov_enable(struct pci_dev *dev, int num_vfs)
 	for (i = 0; i < adapter->num_vfs; i++)
 		ixgbe_vf_configuration(dev, (i | 0x10000000));
 
+	/* reset before enabling SRIOV to avoid mailbox issues */
+	ixgbe_sriov_reinit(adapter);
+
 	err = pci_enable_sriov(dev, num_vfs);
 	if (err) {
 		e_dev_warn("Failed to enable PCI sriov: %d\n", err);
 		return err;
 	}
 	ixgbe_get_vfs(adapter);
-	ixgbe_sriov_reinit(adapter);
 
 	return num_vfs;
 #else
@@ -1354,13 +1356,16 @@ static int ixgbe_disable_port_vlan(struct ixgbe_adapter *adapter, int vf)
 	return err;
 }
 
-int ixgbe_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
+int ixgbe_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
+			  u8 qos, __be16 vlan_proto)
 {
 	int err = 0;
 	struct ixgbe_adapter *adapter = netdev_priv(netdev);
 
 	if ((vf >= adapter->num_vfs) || (vlan > 4095) || (qos > 7))
 		return -EINVAL;
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
 	if (vlan || qos) {
 		/* Check if there is already a port VLAN set, if so
 		 * we have to delete the old one first before we
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
index 47e65e2f886a..0c7977d27b71 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
@@ -43,7 +43,7 @@ void ixgbe_disable_tx_rx(struct ixgbe_adapter *adapter);
 void ixgbe_ping_all_vfs(struct ixgbe_adapter *adapter);
 int ixgbe_ndo_set_vf_mac(struct net_device *netdev, int queue, u8 *mac);
 int ixgbe_ndo_set_vf_vlan(struct net_device *netdev, int queue, u16 vlan,
-			   u8 qos);
+			   u8 qos, __be16 vlan_proto);
 int ixgbe_link_mbps(struct ixgbe_adapter *adapter);
 int ixgbe_ndo_set_vf_bw(struct net_device *netdev, int vf, int min_tx_rate,
 			int max_tx_rate);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
index 1248a9936f7a..31d82e3abac8 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
@@ -90,6 +90,7 @@
 #define IXGBE_DEV_ID_X550EM_A_SFP_N	0x15C4
 #define IXGBE_DEV_ID_X550EM_A_SGMII	0x15C6
 #define IXGBE_DEV_ID_X550EM_A_SGMII_L	0x15C7
+#define IXGBE_DEV_ID_X550EM_A_10G_T	0x15C8
 #define IXGBE_DEV_ID_X550EM_A_SFP	0x15CE
 
 /* VF Device IDs */
@@ -487,6 +488,13 @@ struct ixgbe_thermal_sensor_data {
 #define IXGBE_FHFT_EXT(_n)	(0x09800 + ((_n) * 0x100)) /* Ext Flexible Host
 							    * Filter Table */
 
+/* masks for accessing VXLAN and GENEVE UDP ports */
+#define IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK     0x0000ffff /* VXLAN port */
+#define IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK    0xffff0000 /* GENEVE port */
+#define IXGBE_VXLANCTRL_ALL_UDPPORT_MASK       0xffffffff /* GENEVE/VXLAN */
+
+#define IXGBE_VXLANCTRL_GENEVE_UDPPORT_SHIFT   16
+
 #define IXGBE_FLEXIBLE_FILTER_COUNT_MAX         4
 #define IXGBE_EXT_FLEXIBLE_FILTER_COUNT_MAX     2
 
@@ -1823,6 +1831,9 @@ enum {
 #define IXGBE_LED_IVRT(_i)       IXGBE_LED_OFFSET(IXGBE_LED_IVRT_BASE, _i)
 #define IXGBE_LED_BLINK(_i)      IXGBE_LED_OFFSET(IXGBE_LED_BLINK_BASE, _i)
 #define IXGBE_LED_MODE_MASK(_i)  IXGBE_LED_OFFSET(IXGBE_LED_MODE_MASK_BASE, _i)
+#define IXGBE_X557_LED_MANUAL_SET_MASK	BIT(8)
+#define IXGBE_X557_MAX_LED_INDEX	3
+#define IXGBE_X557_LED_PROVISIONING	0xC430
 
 /* LED modes */
 #define IXGBE_LED_LINK_UP       0x0
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
index 4716ca499e67..7e6b9267ca9d 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
@@ -295,6 +295,12 @@ static s32 ixgbe_identify_phy_x550em(struct ixgbe_hw *hw)
 	case IXGBE_DEV_ID_X550EM_A_KR_L:
 		hw->phy.type = ixgbe_phy_x550em_kr;
 		break;
+	case IXGBE_DEV_ID_X550EM_A_10G_T:
+		if (hw->bus.lan_id)
+			hw->phy.phy_semaphore_mask = IXGBE_GSSR_PHY1_SM;
+		else
+			hw->phy.phy_semaphore_mask = IXGBE_GSSR_PHY0_SM;
+		/* Fallthrough */
 	case IXGBE_DEV_ID_X550EM_X_1G_T:
 	case IXGBE_DEV_ID_X550EM_X_10G_T:
 		return ixgbe_identify_phy_generic(hw);
@@ -1453,7 +1459,7 @@ ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw, ixgbe_link_speed speed,
 	/* Configure internal PHY for KR/KX. */
 	ixgbe_setup_kr_speed_x550em(hw, speed);
 
-	if (!hw->phy.mdio.prtad || hw->phy.mdio.prtad == 0xFFFF)
+	if (hw->phy.mdio.prtad == MDIO_PRTAD_NONE)
 		return IXGBE_ERR_PHY_ADDR_INVALID;
 
 	/* Get external PHY device id */
@@ -2114,6 +2120,50 @@ static s32 ixgbe_reset_phy_t_X550em(struct ixgbe_hw *hw)
 	return ixgbe_enable_lasi_ext_t_x550em(hw);
 }
 
+/**
+ *  ixgbe_led_on_t_x550em - Turns on the software controllable LEDs.
+ *  @hw: pointer to hardware structure
+ *  @led_idx: led number to turn on
+ **/
+static s32 ixgbe_led_on_t_x550em(struct ixgbe_hw *hw, u32 led_idx)
+{
+	u16 phy_data;
+
+	if (led_idx >= IXGBE_X557_MAX_LED_INDEX)
+		return IXGBE_ERR_PARAM;
+
+	/* To turn on the LED, set mode to ON. */
+	hw->phy.ops.read_reg(hw, IXGBE_X557_LED_PROVISIONING + led_idx,
+			     IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &phy_data);
+	phy_data |= IXGBE_X557_LED_MANUAL_SET_MASK;
+	hw->phy.ops.write_reg(hw, IXGBE_X557_LED_PROVISIONING + led_idx,
+			      IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, phy_data);
+
+	return 0;
+}
+
+/**
+ *  ixgbe_led_off_t_x550em - Turns off the software controllable LEDs.
+ *  @hw: pointer to hardware structure
+ *  @led_idx: led number to turn off
+ **/
+static s32 ixgbe_led_off_t_x550em(struct ixgbe_hw *hw, u32 led_idx)
+{
+	u16 phy_data;
+
+	if (led_idx >= IXGBE_X557_MAX_LED_INDEX)
+		return IXGBE_ERR_PARAM;
+
+	/* To turn on the LED, set mode to ON. */
+	hw->phy.ops.read_reg(hw, IXGBE_X557_LED_PROVISIONING + led_idx,
+			     IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &phy_data);
+	phy_data &= ~IXGBE_X557_LED_MANUAL_SET_MASK;
+	hw->phy.ops.write_reg(hw, IXGBE_X557_LED_PROVISIONING + led_idx,
+			      IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, phy_data);
+
+	return 0;
+}
+
 /** ixgbe_get_lcd_x550em - Determine lowest common denominator
  *  @hw: pointer to hardware structure
  *  @lcd_speed: pointer to lowest common link speed
@@ -2344,18 +2394,12 @@ static void ixgbe_read_mng_if_sel_x550em(struct ixgbe_hw *hw)
 	/* If X552 (X550EM_a) and MDIO is connected to external PHY, then set
 	 * PHY address. This register field was has only been used for X552.
 	 */
-	if (!hw->phy.nw_mng_if_sel) {
-		if (hw->mac.type == ixgbe_mac_x550em_a) {
-			struct ixgbe_adapter *adapter = hw->back;
-
-			e_warn(drv, "nw_mng_if_sel not set\n");
-		}
-		return;
+	if (hw->mac.type == ixgbe_mac_x550em_a &&
+	    hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_MDIO_ACT) {
+		hw->phy.mdio.prtad = (hw->phy.nw_mng_if_sel &
+				      IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD) >>
+				     IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT;
 	}
-
-	hw->phy.mdio.prtad = (hw->phy.nw_mng_if_sel &
-			      IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD) >>
-			     IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT;
 }
 
 /** ixgbe_init_phy_ops_X550em - PHY/SFP specific init
@@ -2456,6 +2500,7 @@ static enum ixgbe_media_type ixgbe_get_media_type_X550em(struct ixgbe_hw *hw)
 		break;
 	case IXGBE_DEV_ID_X550EM_X_1G_T:
 	case IXGBE_DEV_ID_X550EM_X_10G_T:
+	case IXGBE_DEV_ID_X550EM_A_10G_T:
 		media_type = ixgbe_media_type_copper;
 		break;
 	default:
@@ -2514,6 +2559,9 @@ static void ixgbe_set_mdio_speed(struct ixgbe_hw *hw)
 
 	switch (hw->device_id) {
 	case IXGBE_DEV_ID_X550EM_X_10G_T:
+	case IXGBE_DEV_ID_X550EM_A_SGMII:
+	case IXGBE_DEV_ID_X550EM_A_SGMII_L:
+	case IXGBE_DEV_ID_X550EM_A_10G_T:
 	case IXGBE_DEV_ID_X550EM_A_SFP:
 		/* Config MDIO clock speed before the first MDIO PHY access */
 		hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
@@ -2853,8 +2901,6 @@ static s32 ixgbe_write_phy_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr,
 	.write_analog_reg8		= NULL, \
 	.set_rxpba			= &ixgbe_set_rxpba_generic, \
 	.check_link			= &ixgbe_check_mac_link_generic, \
-	.led_on				= &ixgbe_led_on_generic, \
-	.led_off			= &ixgbe_led_off_generic, \
 	.blink_led_start		= &ixgbe_blink_led_start_X540, \
 	.blink_led_stop			= &ixgbe_blink_led_stop_X540, \
 	.set_rar			= &ixgbe_set_rar_generic, \
@@ -2886,6 +2932,8 @@ static s32 ixgbe_write_phy_reg_x550a(struct ixgbe_hw *hw, u32 reg_addr,
 
 static const struct ixgbe_mac_operations mac_ops_X550 = {
 	X550_COMMON_MAC
+	.led_on			= ixgbe_led_on_generic,
+	.led_off		= ixgbe_led_off_generic,
 	.reset_hw		= &ixgbe_reset_hw_X540,
 	.get_media_type		= &ixgbe_get_media_type_X540,
 	.get_san_mac_addr	= &ixgbe_get_san_mac_addr_generic,
@@ -2904,6 +2952,8 @@ static const struct ixgbe_mac_operations mac_ops_X550 = {
 
 static const struct ixgbe_mac_operations mac_ops_X550EM_x = {
 	X550_COMMON_MAC
+	.led_on			= ixgbe_led_on_t_x550em,
+	.led_off		= ixgbe_led_off_t_x550em,
 	.reset_hw		= &ixgbe_reset_hw_X550em,
 	.get_media_type		= &ixgbe_get_media_type_X550em,
 	.get_san_mac_addr	= NULL,
@@ -2922,6 +2972,8 @@ static const struct ixgbe_mac_operations mac_ops_X550EM_x = {
 
 static struct ixgbe_mac_operations mac_ops_x550em_a = {
 	X550_COMMON_MAC
+	.led_on			= ixgbe_led_on_t_x550em,
+	.led_off		= ixgbe_led_off_t_x550em,
 	.reset_hw		= ixgbe_reset_hw_X550em,
 	.get_media_type		= ixgbe_get_media_type_X550em,
 	.get_san_mac_addr	= NULL,
@@ -2997,6 +3049,8 @@ static const struct ixgbe_phy_operations phy_ops_x550em_a = {
 	.identify		= &ixgbe_identify_phy_x550em,
 	.read_reg		= &ixgbe_read_phy_reg_x550a,
 	.write_reg		= &ixgbe_write_phy_reg_x550a,
+	.read_reg_mdi		= &ixgbe_read_phy_reg_mdi,
+	.write_reg_mdi		= &ixgbe_write_phy_reg_mdi,
 };
 
 static const u32 ixgbe_mvals_X550[IXGBE_MVALS_IDX_LIMIT] = {
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
index be52f597688b..5639fbe294d0 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
@@ -502,12 +502,9 @@ extern void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector);
 void ixgbe_napi_add_all(struct ixgbevf_adapter *adapter);
 void ixgbe_napi_del_all(struct ixgbevf_adapter *adapter);
 
-#ifdef DEBUG
-char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw);
-#define hw_dbg(hw, format, arg...) \
-	printk(KERN_DEBUG "%s: " format, ixgbevf_get_hw_dev_name(hw), ##arg)
-#else
-#define hw_dbg(hw, format, arg...) do {} while (0)
-#endif
+#define ixgbevf_hw_to_netdev(hw) \
+	(((struct ixgbevf_adapter *)(hw)->back)->netdev)
 
+#define hw_dbg(hw, format, arg...) \
+	netdev_dbg(ixgbevf_hw_to_netdev(hw), format, ## arg)
 #endif /* _IXGBEVF_H_ */
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index d9d6616f02a4..7eaac3234049 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -1612,7 +1612,7 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
 		txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
 	}  while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
 	if (!wait_loop)
-		pr_err("Could not enable Tx Queue %d\n", reg_idx);
+		hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
 }
 
 /**
@@ -1810,8 +1810,10 @@ static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
 	if (hw->mac.type >= ixgbe_mac_X550_vf)
 		ixgbevf_setup_vfmrqc(adapter);
 
+	spin_lock_bh(&adapter->mbx_lock);
 	/* notify the PF of our intent to use this size of frame */
 	ret = hw->mac.ops.set_rlpml(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN);
+	spin_unlock_bh(&adapter->mbx_lock);
 	if (ret)
 		dev_err(&adapter->pdev->dev,
 			"Failed to set MTU at %d\n", netdev->mtu);
@@ -2993,6 +2995,7 @@ static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
  **/
 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
 {
+	struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
 	int size;
 
 	size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
@@ -3757,8 +3760,10 @@ static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
 	if ((new_mtu < 68) || (max_frame > max_possible_frame))
 		return -EINVAL;
 
+	spin_lock_bh(&adapter->mbx_lock);
 	/* notify the PF of our intent to use this size of frame */
 	ret = hw->mac.ops.set_rlpml(hw, max_frame);
+	spin_unlock_bh(&adapter->mbx_lock);
 	if (ret)
 		return -EINVAL;
 
diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.c b/drivers/net/ethernet/intel/ixgbevf/vf.c
index a52f70ec42b6..d46ba1dabcb7 100644
--- a/drivers/net/ethernet/intel/ixgbevf/vf.c
+++ b/drivers/net/ethernet/intel/ixgbevf/vf.c
@@ -284,7 +284,8 @@ static s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr)
 	if (addr)
 		ether_addr_copy(msg_addr, addr);
 
-	ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 3);
+	ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
+					     sizeof(msgbuf) / sizeof(u32));
 	if (!ret_val) {
 		msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS;
 
@@ -441,7 +442,8 @@ static s32 ixgbevf_set_rar_vf(struct ixgbe_hw *hw, u32 index, u8 *addr,
 	msgbuf[0] = IXGBE_VF_SET_MAC_ADDR;
 	ether_addr_copy(msg_addr, addr);
 
-	ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 2);
+	ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
+					     sizeof(msgbuf) / sizeof(u32));
 
 	msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS;
 
@@ -551,7 +553,8 @@ static s32 ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode)
 	msgbuf[0] = IXGBE_VF_UPDATE_XCAST_MODE;
 	msgbuf[1] = xcast_mode;
 
-	err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 2);
+	err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
+					 sizeof(msgbuf) / sizeof(u32));
 	if (err)
 		return err;
 
@@ -588,7 +591,8 @@ static s32 ixgbevf_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind,
 	/* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
 	msgbuf[0] |= vlan_on << IXGBE_VT_MSGINFO_SHIFT;
 
-	err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 2);
+	err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
+					 sizeof(msgbuf) / sizeof(u32));
 	if (err)
 		goto mbx_err;
 
@@ -791,7 +795,8 @@ static s32 ixgbevf_set_rlpml_vf(struct ixgbe_hw *hw, u16 max_size)
 	msgbuf[0] = IXGBE_VF_SET_LPE;
 	msgbuf[1] = max_size;
 
-	ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf, 2);
+	ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
+					     sizeof(msgbuf) / sizeof(u32));
 	if (ret_val)
 		return ret_val;
 	if ((msgbuf[0] & IXGBE_VF_SET_LPE) &&
@@ -837,7 +842,8 @@ static int ixgbevf_negotiate_api_version_vf(struct ixgbe_hw *hw, int api)
 	msg[1] = api;
 	msg[2] = 0;
 
-	err = ixgbevf_write_msg_read_ack(hw, msg, msg, 3);
+	err = ixgbevf_write_msg_read_ack(hw, msg, msg,
+					 sizeof(msg) / sizeof(u32));
 	if (!err) {
 		msg[0] &= ~IXGBE_VT_MSGTYPE_CTS;
 
@@ -887,7 +893,8 @@ int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs,
 	msg[0] = IXGBE_VF_GET_QUEUE;
 	msg[1] = msg[2] = msg[3] = msg[4] = 0;
 
-	err = ixgbevf_write_msg_read_ack(hw, msg, msg, 5);
+	err = ixgbevf_write_msg_read_ack(hw, msg, msg,
+					 sizeof(msg) / sizeof(u32));
 	if (!err) {
 		msg[0] &= ~IXGBE_VT_MSGTYPE_CTS;
 
diff --git a/drivers/net/ethernet/marvell/mvmdio.c b/drivers/net/ethernet/marvell/mvmdio.c
index 8982c882af1b..a0d1b084ecec 100644
--- a/drivers/net/ethernet/marvell/mvmdio.c
+++ b/drivers/net/ethernet/marvell/mvmdio.c
@@ -211,8 +211,7 @@ static int orion_mdio_probe(struct platform_device *pdev)
 	dev->regs = devm_ioremap(&pdev->dev, r->start, resource_size(r));
 	if (!dev->regs) {
 		dev_err(&pdev->dev, "Unable to remap SMI register\n");
-		ret = -ENODEV;
-		goto out_mdio;
+		return -ENODEV;
 	}
 
 	init_waitqueue_head(&dev->smi_busy_wait);
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index d41c28d00b57..5cb07c2017bf 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -382,7 +382,8 @@ struct mvneta_port {
 	struct mvneta_rx_queue *rxqs;
 	struct mvneta_tx_queue *txqs;
 	struct net_device *dev;
-	struct notifier_block cpu_notifier;
+	struct hlist_node node_online;
+	struct hlist_node node_dead;
 	int rxq_def;
 	/* Protect the access to the percpu interrupt registers,
 	 * ensuring that the configuration remains coherent.
@@ -399,7 +400,6 @@ struct mvneta_port {
 	u16 rx_ring_size;
 
 	struct mii_bus *mii_bus;
-	struct phy_device *phy_dev;
 	phy_interface_t phy_interface;
 	struct device_node *phy_node;
 	unsigned int link;
@@ -574,6 +574,7 @@ struct mvneta_rx_queue {
 	int next_desc_to_proc;
 };
 
+static enum cpuhp_state online_hpstate;
 /* The hardware supports eight (8) rx queues, but we are only allowing
  * the first one to be used. Therefore, let's just allocate one queue.
  */
@@ -635,8 +636,9 @@ static void mvneta_mib_counters_clear(struct mvneta_port *pp)
 }
 
 /* Get System Network Statistics */
-struct rtnl_link_stats64 *mvneta_get_stats64(struct net_device *dev,
-					     struct rtnl_link_stats64 *stats)
+static struct rtnl_link_stats64 *
+mvneta_get_stats64(struct net_device *dev,
+		   struct rtnl_link_stats64 *stats)
 {
 	struct mvneta_port *pp = netdev_priv(dev);
 	unsigned int start;
@@ -2651,6 +2653,7 @@ static int mvneta_poll(struct napi_struct *napi, int budget)
 	u32 cause_rx_tx;
 	int rx_queue;
 	struct mvneta_port *pp = netdev_priv(napi->dev);
+	struct net_device *ndev = pp->dev;
 	struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports);
 
 	if (!netif_running(pp->dev)) {
@@ -2668,7 +2671,7 @@ static int mvneta_poll(struct napi_struct *napi, int budget)
 				(MVNETA_CAUSE_PHY_STATUS_CHANGE |
 				 MVNETA_CAUSE_LINK_CHANGE |
 				 MVNETA_CAUSE_PSC_SYNC_CHANGE))) {
-			mvneta_fixed_link_update(pp, pp->phy_dev);
+			mvneta_fixed_link_update(pp, ndev->phydev);
 		}
 	}
 
@@ -2963,6 +2966,7 @@ static int mvneta_setup_txqs(struct mvneta_port *pp)
 static void mvneta_start_dev(struct mvneta_port *pp)
 {
 	int cpu;
+	struct net_device *ndev = pp->dev;
 
 	mvneta_max_rx_size_set(pp, pp->pkt_size);
 	mvneta_txq_max_tx_size_set(pp, pp->pkt_size);
@@ -2985,15 +2989,16 @@ static void mvneta_start_dev(struct mvneta_port *pp)
 		    MVNETA_CAUSE_LINK_CHANGE |
 		    MVNETA_CAUSE_PSC_SYNC_CHANGE);
 
-	phy_start(pp->phy_dev);
+	phy_start(ndev->phydev);
 	netif_tx_start_all_queues(pp->dev);
 }
 
 static void mvneta_stop_dev(struct mvneta_port *pp)
 {
 	unsigned int cpu;
+	struct net_device *ndev = pp->dev;
 
-	phy_stop(pp->phy_dev);
+	phy_stop(ndev->phydev);
 
 	for_each_online_cpu(cpu) {
 		struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu);
@@ -3166,7 +3171,7 @@ static int mvneta_set_mac_addr(struct net_device *dev, void *addr)
 static void mvneta_adjust_link(struct net_device *ndev)
 {
 	struct mvneta_port *pp = netdev_priv(ndev);
-	struct phy_device *phydev = pp->phy_dev;
+	struct phy_device *phydev = ndev->phydev;
 	int status_change = 0;
 
 	if (phydev->link) {
@@ -3244,7 +3249,6 @@ static int mvneta_mdio_probe(struct mvneta_port *pp)
 	phy_dev->supported &= PHY_GBIT_FEATURES;
 	phy_dev->advertising = phy_dev->supported;
 
-	pp->phy_dev = phy_dev;
 	pp->link    = 0;
 	pp->duplex  = 0;
 	pp->speed   = 0;
@@ -3254,8 +3258,9 @@ static int mvneta_mdio_probe(struct mvneta_port *pp)
 
 static void mvneta_mdio_remove(struct mvneta_port *pp)
 {
-	phy_disconnect(pp->phy_dev);
-	pp->phy_dev = NULL;
+	struct net_device *ndev = pp->dev;
+
+	phy_disconnect(ndev->phydev);
 }
 
 /* Electing a CPU must be done in an atomic way: it should be done
@@ -3311,101 +3316,104 @@ static void mvneta_percpu_elect(struct mvneta_port *pp)
 	}
 };
 
-static int mvneta_percpu_notifier(struct notifier_block *nfb,
-				  unsigned long action, void *hcpu)
+static int mvneta_cpu_online(unsigned int cpu, struct hlist_node *node)
 {
-	struct mvneta_port *pp = container_of(nfb, struct mvneta_port,
-					      cpu_notifier);
-	int cpu = (unsigned long)hcpu, other_cpu;
+	int other_cpu;
+	struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port,
+						  node_online);
 	struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu);
 
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-	case CPU_DOWN_FAILED:
-	case CPU_DOWN_FAILED_FROZEN:
-		spin_lock(&pp->lock);
-		/* Configuring the driver for a new CPU while the
-		 * driver is stopping is racy, so just avoid it.
-		 */
-		if (pp->is_stopped) {
-			spin_unlock(&pp->lock);
-			break;
-		}
-		netif_tx_stop_all_queues(pp->dev);
 
-		/* We have to synchronise on tha napi of each CPU
-		 * except the one just being waked up
-		 */
-		for_each_online_cpu(other_cpu) {
-			if (other_cpu != cpu) {
-				struct mvneta_pcpu_port *other_port =
-					per_cpu_ptr(pp->ports, other_cpu);
+	spin_lock(&pp->lock);
+	/*
+	 * Configuring the driver for a new CPU while the driver is
+	 * stopping is racy, so just avoid it.
+	 */
+	if (pp->is_stopped) {
+		spin_unlock(&pp->lock);
+		return 0;
+	}
+	netif_tx_stop_all_queues(pp->dev);
 
-				napi_synchronize(&other_port->napi);
-			}
+	/*
+	 * We have to synchronise on tha napi of each CPU except the one
+	 * just being woken up
+	 */
+	for_each_online_cpu(other_cpu) {
+		if (other_cpu != cpu) {
+			struct mvneta_pcpu_port *other_port =
+				per_cpu_ptr(pp->ports, other_cpu);
+
+			napi_synchronize(&other_port->napi);
 		}
+	}
 
-		/* Mask all ethernet port interrupts */
-		on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
-		napi_enable(&port->napi);
+	/* Mask all ethernet port interrupts */
+	on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
+	napi_enable(&port->napi);
 
+	/*
+	 * Enable per-CPU interrupts on the CPU that is
+	 * brought up.
+	 */
+	mvneta_percpu_enable(pp);
 
-		/* Enable per-CPU interrupts on the CPU that is
-		 * brought up.
-		 */
-		mvneta_percpu_enable(pp);
+	/*
+	 * Enable per-CPU interrupt on the one CPU we care
+	 * about.
+	 */
+	mvneta_percpu_elect(pp);
 
-		/* Enable per-CPU interrupt on the one CPU we care
-		 * about.
-		 */
-		mvneta_percpu_elect(pp);
-
-		/* Unmask all ethernet port interrupts */
-		on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
-		mvreg_write(pp, MVNETA_INTR_MISC_MASK,
-			MVNETA_CAUSE_PHY_STATUS_CHANGE |
-			MVNETA_CAUSE_LINK_CHANGE |
-			MVNETA_CAUSE_PSC_SYNC_CHANGE);
-		netif_tx_start_all_queues(pp->dev);
-		spin_unlock(&pp->lock);
-		break;
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		netif_tx_stop_all_queues(pp->dev);
-		/* Thanks to this lock we are sure that any pending
-		 * cpu election is done
-		 */
-		spin_lock(&pp->lock);
-		/* Mask all ethernet port interrupts */
-		on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
-		spin_unlock(&pp->lock);
+	/* Unmask all ethernet port interrupts */
+	on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
+	mvreg_write(pp, MVNETA_INTR_MISC_MASK,
+		    MVNETA_CAUSE_PHY_STATUS_CHANGE |
+		    MVNETA_CAUSE_LINK_CHANGE |
+		    MVNETA_CAUSE_PSC_SYNC_CHANGE);
+	netif_tx_start_all_queues(pp->dev);
+	spin_unlock(&pp->lock);
+	return 0;
+}
 
-		napi_synchronize(&port->napi);
-		napi_disable(&port->napi);
-		/* Disable per-CPU interrupts on the CPU that is
-		 * brought down.
-		 */
-		mvneta_percpu_disable(pp);
+static int mvneta_cpu_down_prepare(unsigned int cpu, struct hlist_node *node)
+{
+	struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port,
+						  node_online);
+	struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu);
 
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		/* Check if a new CPU must be elected now this on is down */
-		spin_lock(&pp->lock);
-		mvneta_percpu_elect(pp);
-		spin_unlock(&pp->lock);
-		/* Unmask all ethernet port interrupts */
-		on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
-		mvreg_write(pp, MVNETA_INTR_MISC_MASK,
-			MVNETA_CAUSE_PHY_STATUS_CHANGE |
-			MVNETA_CAUSE_LINK_CHANGE |
-			MVNETA_CAUSE_PSC_SYNC_CHANGE);
-		netif_tx_start_all_queues(pp->dev);
-		break;
-	}
+	/*
+	 * Thanks to this lock we are sure that any pending cpu election is
+	 * done.
+	 */
+	spin_lock(&pp->lock);
+	/* Mask all ethernet port interrupts */
+	on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
+	spin_unlock(&pp->lock);
 
-	return NOTIFY_OK;
+	napi_synchronize(&port->napi);
+	napi_disable(&port->napi);
+	/* Disable per-CPU interrupts on the CPU that is brought down. */
+	mvneta_percpu_disable(pp);
+	return 0;
+}
+
+static int mvneta_cpu_dead(unsigned int cpu, struct hlist_node *node)
+{
+	struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port,
+						  node_dead);
+
+	/* Check if a new CPU must be elected now this on is down */
+	spin_lock(&pp->lock);
+	mvneta_percpu_elect(pp);
+	spin_unlock(&pp->lock);
+	/* Unmask all ethernet port interrupts */
+	on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
+	mvreg_write(pp, MVNETA_INTR_MISC_MASK,
+		    MVNETA_CAUSE_PHY_STATUS_CHANGE |
+		    MVNETA_CAUSE_LINK_CHANGE |
+		    MVNETA_CAUSE_PSC_SYNC_CHANGE);
+	netif_tx_start_all_queues(pp->dev);
+	return 0;
 }
 
 static int mvneta_open(struct net_device *dev)
@@ -3442,7 +3450,15 @@ static int mvneta_open(struct net_device *dev)
 	/* Register a CPU notifier to handle the case where our CPU
 	 * might be taken offline.
 	 */
-	register_cpu_notifier(&pp->cpu_notifier);
+	ret = cpuhp_state_add_instance_nocalls(online_hpstate,
+					       &pp->node_online);
+	if (ret)
+		goto err_free_irq;
+
+	ret = cpuhp_state_add_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
+					       &pp->node_dead);
+	if (ret)
+		goto err_free_online_hp;
 
 	/* In default link is down */
 	netif_carrier_off(pp->dev);
@@ -3450,15 +3466,19 @@ static int mvneta_open(struct net_device *dev)
 	ret = mvneta_mdio_probe(pp);
 	if (ret < 0) {
 		netdev_err(dev, "cannot probe MDIO bus\n");
-		goto err_free_irq;
+		goto err_free_dead_hp;
 	}
 
 	mvneta_start_dev(pp);
 
 	return 0;
 
+err_free_dead_hp:
+	cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
+					    &pp->node_dead);
+err_free_online_hp:
+	cpuhp_state_remove_instance_nocalls(online_hpstate, &pp->node_online);
 err_free_irq:
-	unregister_cpu_notifier(&pp->cpu_notifier);
 	on_each_cpu(mvneta_percpu_disable, pp, true);
 	free_percpu_irq(pp->dev->irq, pp->ports);
 err_cleanup_txqs:
@@ -3484,7 +3504,10 @@ static int mvneta_stop(struct net_device *dev)
 
 	mvneta_stop_dev(pp);
 	mvneta_mdio_remove(pp);
-	unregister_cpu_notifier(&pp->cpu_notifier);
+
+	cpuhp_state_remove_instance_nocalls(online_hpstate, &pp->node_online);
+	cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
+					    &pp->node_dead);
 	on_each_cpu(mvneta_percpu_disable, pp, true);
 	free_percpu_irq(dev->irq, pp->ports);
 	mvneta_cleanup_rxqs(pp);
@@ -3495,42 +3518,31 @@ static int mvneta_stop(struct net_device *dev)
 
 static int mvneta_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 {
-	struct mvneta_port *pp = netdev_priv(dev);
-
-	if (!pp->phy_dev)
+	if (!dev->phydev)
 		return -ENOTSUPP;
 
-	return phy_mii_ioctl(pp->phy_dev, ifr, cmd);
+	return phy_mii_ioctl(dev->phydev, ifr, cmd);
 }
 
 /* Ethtool methods */
 
-/* Get settings (phy address, speed) for ethtools */
-int mvneta_ethtool_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+/* Set link ksettings (phy address, speed) for ethtools */
+static int
+mvneta_ethtool_set_link_ksettings(struct net_device *ndev,
+				  const struct ethtool_link_ksettings *cmd)
 {
-	struct mvneta_port *pp = netdev_priv(dev);
-
-	if (!pp->phy_dev)
-		return -ENODEV;
-
-	return phy_ethtool_gset(pp->phy_dev, cmd);
-}
-
-/* Set settings (phy address, speed) for ethtools */
-int mvneta_ethtool_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-	struct mvneta_port *pp = netdev_priv(dev);
-	struct phy_device *phydev = pp->phy_dev;
+	struct mvneta_port *pp = netdev_priv(ndev);
+	struct phy_device *phydev = ndev->phydev;
 
 	if (!phydev)
 		return -ENODEV;
 
-	if ((cmd->autoneg == AUTONEG_ENABLE) != pp->use_inband_status) {
+	if ((cmd->base.autoneg == AUTONEG_ENABLE) != pp->use_inband_status) {
 		u32 val;
 
-		mvneta_set_autoneg(pp, cmd->autoneg == AUTONEG_ENABLE);
+		mvneta_set_autoneg(pp, cmd->base.autoneg == AUTONEG_ENABLE);
 
-		if (cmd->autoneg == AUTONEG_DISABLE) {
+		if (cmd->base.autoneg == AUTONEG_DISABLE) {
 			val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
 			val &= ~(MVNETA_GMAC_CONFIG_MII_SPEED |
 				 MVNETA_GMAC_CONFIG_GMII_SPEED |
@@ -3547,17 +3559,17 @@ int mvneta_ethtool_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
 			mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
 		}
 
-		pp->use_inband_status = (cmd->autoneg == AUTONEG_ENABLE);
+		pp->use_inband_status = (cmd->base.autoneg == AUTONEG_ENABLE);
 		netdev_info(pp->dev, "autoneg status set to %i\n",
 			    pp->use_inband_status);
 
-		if (netif_running(dev)) {
+		if (netif_running(ndev)) {
 			mvneta_port_down(pp);
 			mvneta_port_up(pp);
 		}
 	}
 
-	return phy_ethtool_sset(pp->phy_dev, cmd);
+	return phy_ethtool_ksettings_set(ndev->phydev, cmd);
 }
 
 /* Set interrupt coalescing for ethtools */
@@ -3821,8 +3833,6 @@ static const struct net_device_ops mvneta_netdev_ops = {
 
 const struct ethtool_ops mvneta_eth_tool_ops = {
 	.get_link       = ethtool_op_get_link,
-	.get_settings   = mvneta_ethtool_get_settings,
-	.set_settings   = mvneta_ethtool_set_settings,
 	.set_coalesce   = mvneta_ethtool_set_coalesce,
 	.get_coalesce   = mvneta_ethtool_get_coalesce,
 	.get_drvinfo    = mvneta_ethtool_get_drvinfo,
@@ -3835,6 +3845,8 @@ const struct ethtool_ops mvneta_eth_tool_ops = {
 	.get_rxnfc	= mvneta_ethtool_get_rxnfc,
 	.get_rxfh	= mvneta_ethtool_get_rxfh,
 	.set_rxfh	= mvneta_ethtool_set_rxfh,
+	.get_link_ksettings = phy_ethtool_get_link_ksettings,
+	.set_link_ksettings = mvneta_ethtool_set_link_ksettings,
 };
 
 /* Initialize hw */
@@ -4024,7 +4036,6 @@ static int mvneta_probe(struct platform_device *pdev)
 	err = of_property_read_string(dn, "managed", &managed);
 	pp->use_inband_status = (err == 0 &&
 				 strcmp(managed, "in-band-status") == 0);
-	pp->cpu_notifier.notifier_call = mvneta_percpu_notifier;
 
 	pp->rxq_def = rxq_def;
 
@@ -4227,7 +4238,42 @@ static struct platform_driver mvneta_driver = {
 	},
 };
 
-module_platform_driver(mvneta_driver);
+static int __init mvneta_driver_init(void)
+{
+	int ret;
+
+	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/mvmeta:online",
+				      mvneta_cpu_online,
+				      mvneta_cpu_down_prepare);
+	if (ret < 0)
+		goto out;
+	online_hpstate = ret;
+	ret = cpuhp_setup_state_multi(CPUHP_NET_MVNETA_DEAD, "net/mvneta:dead",
+				      NULL, mvneta_cpu_dead);
+	if (ret)
+		goto err_dead;
+
+	ret = platform_driver_register(&mvneta_driver);
+	if (ret)
+		goto err;
+	return 0;
+
+err:
+	cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD);
+err_dead:
+	cpuhp_remove_multi_state(online_hpstate);
+out:
+	return ret;
+}
+module_init(mvneta_driver_init);
+
+static void __exit mvneta_driver_exit(void)
+{
+	platform_driver_unregister(&mvneta_driver);
+	cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD);
+	cpuhp_remove_multi_state(online_hpstate);
+}
+module_exit(mvneta_driver_exit);
 
 MODULE_DESCRIPTION("Marvell NETA Ethernet Driver - www.marvell.com");
 MODULE_AUTHOR("Rami Rosen <rosenr@marvell.com>, Thomas Petazzoni <thomas.petazzoni@free-electrons.com>");
diff --git a/drivers/net/ethernet/marvell/mvneta_bm.h b/drivers/net/ethernet/marvell/mvneta_bm.h
index e74fd44a92f7..a32de432800c 100644
--- a/drivers/net/ethernet/marvell/mvneta_bm.h
+++ b/drivers/net/ethernet/marvell/mvneta_bm.h
@@ -133,7 +133,7 @@ struct mvneta_bm_pool {
 void *mvneta_frag_alloc(unsigned int frag_size);
 void mvneta_frag_free(unsigned int frag_size, void *data);
 
-#if defined(CONFIG_MVNETA_BM) || defined(CONFIG_MVNETA_BM_MODULE)
+#if IS_ENABLED(CONFIG_MVNETA_BM)
 void mvneta_bm_pool_destroy(struct mvneta_bm *priv,
 			    struct mvneta_bm_pool *bm_pool, u8 port_map);
 void mvneta_bm_bufs_free(struct mvneta_bm *priv, struct mvneta_bm_pool *bm_pool,
diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c
index 467138b423d3..f05ea56dcff2 100644
--- a/drivers/net/ethernet/marvell/sky2.c
+++ b/drivers/net/ethernet/marvell/sky2.c
@@ -3070,7 +3070,7 @@ static int sky2_poll(struct napi_struct *napi, int work_limit)
 			goto done;
 	}
 
-	napi_complete(napi);
+	napi_complete_done(napi, work_done);
 	sky2_read32(hw, B0_Y2_SP_LISR);
 done:
 
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index b57ae3afb994..4a62ffd7729d 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -18,6 +18,7 @@
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 #include <linux/clk.h>
+#include <linux/pm_runtime.h>
 #include <linux/if_vlan.h>
 #include <linux/reset.h>
 #include <linux/tcp.h>
@@ -50,6 +51,10 @@ static const struct mtk_ethtool_stats {
 	MTK_ETHTOOL_STAT(rx_flow_control_packets),
 };
 
+static const char * const mtk_clks_source_name[] = {
+	"ethif", "esw", "gp1", "gp2", "trgpll"
+};
+
 void mtk_w32(struct mtk_eth *eth, u32 val, unsigned reg)
 {
 	__raw_writel(val, eth->base + reg);
@@ -130,6 +135,33 @@ static int mtk_mdio_read(struct mii_bus *bus, int phy_addr, int phy_reg)
 	return _mtk_mdio_read(eth, phy_addr, phy_reg);
 }
 
+static void mtk_gmac0_rgmii_adjust(struct mtk_eth *eth, int speed)
+{
+	u32 val;
+	int ret;
+
+	val = (speed == SPEED_1000) ?
+		INTF_MODE_RGMII_1000 : INTF_MODE_RGMII_10_100;
+	mtk_w32(eth, val, INTF_MODE);
+
+	regmap_update_bits(eth->ethsys, ETHSYS_CLKCFG0,
+			   ETHSYS_TRGMII_CLK_SEL362_5,
+			   ETHSYS_TRGMII_CLK_SEL362_5);
+
+	val = (speed == SPEED_1000) ? 250000000 : 500000000;
+	ret = clk_set_rate(eth->clks[MTK_CLK_TRGPLL], val);
+	if (ret)
+		dev_err(eth->dev, "Failed to set trgmii pll: %d\n", ret);
+
+	val = (speed == SPEED_1000) ?
+		RCK_CTRL_RGMII_1000 : RCK_CTRL_RGMII_10_100;
+	mtk_w32(eth, val, TRGMII_RCK_CTRL);
+
+	val = (speed == SPEED_1000) ?
+		TCK_CTRL_RGMII_1000 : TCK_CTRL_RGMII_10_100;
+	mtk_w32(eth, val, TRGMII_TCK_CTRL);
+}
+
 static void mtk_phy_link_adjust(struct net_device *dev)
 {
 	struct mtk_mac *mac = netdev_priv(dev);
@@ -140,7 +172,10 @@ static void mtk_phy_link_adjust(struct net_device *dev)
 		  MAC_MCR_RX_EN | MAC_MCR_BACKOFF_EN |
 		  MAC_MCR_BACKPR_EN;
 
-	switch (mac->phy_dev->speed) {
+	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
+		return;
+
+	switch (dev->phydev->speed) {
 	case SPEED_1000:
 		mcr |= MAC_MCR_SPEED_1000;
 		break;
@@ -149,20 +184,23 @@ static void mtk_phy_link_adjust(struct net_device *dev)
 		break;
 	};
 
-	if (mac->phy_dev->link)
+	if (mac->id == 0 && !mac->trgmii)
+		mtk_gmac0_rgmii_adjust(mac->hw, dev->phydev->speed);
+
+	if (dev->phydev->link)
 		mcr |= MAC_MCR_FORCE_LINK;
 
-	if (mac->phy_dev->duplex) {
+	if (dev->phydev->duplex) {
 		mcr |= MAC_MCR_FORCE_DPX;
 
-		if (mac->phy_dev->pause)
+		if (dev->phydev->pause)
 			rmt_adv = LPA_PAUSE_CAP;
-		if (mac->phy_dev->asym_pause)
+		if (dev->phydev->asym_pause)
 			rmt_adv |= LPA_PAUSE_ASYM;
 
-		if (mac->phy_dev->advertising & ADVERTISED_Pause)
+		if (dev->phydev->advertising & ADVERTISED_Pause)
 			lcl_adv |= ADVERTISE_PAUSE_CAP;
-		if (mac->phy_dev->advertising & ADVERTISED_Asym_Pause)
+		if (dev->phydev->advertising & ADVERTISED_Asym_Pause)
 			lcl_adv |= ADVERTISE_PAUSE_ASYM;
 
 		flowctrl = mii_resolve_flowctrl_fdx(lcl_adv, rmt_adv);
@@ -179,7 +217,7 @@ static void mtk_phy_link_adjust(struct net_device *dev)
 
 	mtk_w32(mac->hw, mcr, MTK_MAC_MCR(mac->id));
 
-	if (mac->phy_dev->link)
+	if (dev->phydev->link)
 		netif_carrier_on(dev);
 	else
 		netif_carrier_off(dev);
@@ -188,17 +226,9 @@ static void mtk_phy_link_adjust(struct net_device *dev)
 static int mtk_phy_connect_node(struct mtk_eth *eth, struct mtk_mac *mac,
 				struct device_node *phy_node)
 {
-	const __be32 *_addr = NULL;
 	struct phy_device *phydev;
-	int phy_mode, addr;
-
-	_addr = of_get_property(phy_node, "reg", NULL);
+	int phy_mode;
 
-	if (!_addr || (be32_to_cpu(*_addr) >= 0x20)) {
-		pr_err("%s: invalid phy address\n", phy_node->name);
-		return -EINVAL;
-	}
-	addr = be32_to_cpu(*_addr);
 	phy_mode = of_get_phy_mode(phy_node);
 	if (phy_mode < 0) {
 		dev_err(eth->dev, "incorrect phy-mode %d\n", phy_mode);
@@ -217,17 +247,17 @@ static int mtk_phy_connect_node(struct mtk_eth *eth, struct mtk_mac *mac,
 		 mac->id, phydev_name(phydev), phydev->phy_id,
 		 phydev->drv->name);
 
-	mac->phy_dev = phydev;
-
 	return 0;
 }
 
-static int mtk_phy_connect(struct mtk_mac *mac)
+static int mtk_phy_connect(struct net_device *dev)
 {
-	struct mtk_eth *eth = mac->hw;
+	struct mtk_mac *mac = netdev_priv(dev);
+	struct mtk_eth *eth;
 	struct device_node *np;
-	u32 val, ge_mode;
+	u32 val;
 
+	eth = mac->hw;
 	np = of_parse_phandle(mac->of_node, "phy-handle", 0);
 	if (!np && of_phy_is_fixed_link(mac->of_node))
 		if (!of_phy_register_fixed_link(mac->of_node))
@@ -236,46 +266,67 @@ static int mtk_phy_connect(struct mtk_mac *mac)
 		return -ENODEV;
 
 	switch (of_get_phy_mode(np)) {
+	case PHY_INTERFACE_MODE_TRGMII:
+		mac->trgmii = true;
 	case PHY_INTERFACE_MODE_RGMII_TXID:
 	case PHY_INTERFACE_MODE_RGMII_RXID:
 	case PHY_INTERFACE_MODE_RGMII_ID:
 	case PHY_INTERFACE_MODE_RGMII:
-		ge_mode = 0;
+		mac->ge_mode = 0;
 		break;
 	case PHY_INTERFACE_MODE_MII:
-		ge_mode = 1;
+		mac->ge_mode = 1;
+		break;
+	case PHY_INTERFACE_MODE_REVMII:
+		mac->ge_mode = 2;
 		break;
 	case PHY_INTERFACE_MODE_RMII:
-		ge_mode = 2;
+		if (!mac->id)
+			goto err_phy;
+		mac->ge_mode = 3;
 		break;
 	default:
-		dev_err(eth->dev, "invalid phy_mode\n");
-		return -1;
+		goto err_phy;
 	}
 
 	/* put the gmac into the right mode */
 	regmap_read(eth->ethsys, ETHSYS_SYSCFG0, &val);
 	val &= ~SYSCFG0_GE_MODE(SYSCFG0_GE_MASK, mac->id);
-	val |= SYSCFG0_GE_MODE(ge_mode, mac->id);
+	val |= SYSCFG0_GE_MODE(mac->ge_mode, mac->id);
 	regmap_write(eth->ethsys, ETHSYS_SYSCFG0, val);
 
-	mtk_phy_connect_node(eth, mac, np);
-	mac->phy_dev->autoneg = AUTONEG_ENABLE;
-	mac->phy_dev->speed = 0;
-	mac->phy_dev->duplex = 0;
-	mac->phy_dev->supported &= PHY_GBIT_FEATURES | SUPPORTED_Pause |
+	/* couple phydev to net_device */
+	if (mtk_phy_connect_node(eth, mac, np))
+		goto err_phy;
+
+	dev->phydev->autoneg = AUTONEG_ENABLE;
+	dev->phydev->speed = 0;
+	dev->phydev->duplex = 0;
+
+	if (of_phy_is_fixed_link(mac->of_node))
+		dev->phydev->supported |=
+		SUPPORTED_Pause | SUPPORTED_Asym_Pause;
+
+	dev->phydev->supported &= PHY_GBIT_FEATURES | SUPPORTED_Pause |
 				   SUPPORTED_Asym_Pause;
-	mac->phy_dev->advertising = mac->phy_dev->supported |
+	dev->phydev->advertising = dev->phydev->supported |
 				    ADVERTISED_Autoneg;
-	phy_start_aneg(mac->phy_dev);
+	phy_start_aneg(dev->phydev);
+
+	of_node_put(np);
 
 	return 0;
+
+err_phy:
+	of_node_put(np);
+	dev_err(eth->dev, "%s: invalid phy\n", __func__);
+	return -EINVAL;
 }
 
 static int mtk_mdio_init(struct mtk_eth *eth)
 {
 	struct device_node *mii_np;
-	int err;
+	int ret;
 
 	mii_np = of_get_child_by_name(eth->dev->of_node, "mdio-bus");
 	if (!mii_np) {
@@ -284,13 +335,13 @@ static int mtk_mdio_init(struct mtk_eth *eth)
 	}
 
 	if (!of_device_is_available(mii_np)) {
-		err = 0;
+		ret = -ENODEV;
 		goto err_put_node;
 	}
 
-	eth->mii_bus = mdiobus_alloc();
+	eth->mii_bus = devm_mdiobus_alloc(eth->dev);
 	if (!eth->mii_bus) {
-		err = -ENOMEM;
+		ret = -ENOMEM;
 		goto err_put_node;
 	}
 
@@ -301,19 +352,11 @@ static int mtk_mdio_init(struct mtk_eth *eth)
 	eth->mii_bus->parent = eth->dev;
 
 	snprintf(eth->mii_bus->id, MII_BUS_ID_SIZE, "%s", mii_np->name);
-	err = of_mdiobus_register(eth->mii_bus, mii_np);
-	if (err)
-		goto err_free_bus;
-
-	return 0;
-
-err_free_bus:
-	mdiobus_free(eth->mii_bus);
+	ret = of_mdiobus_register(eth->mii_bus, mii_np);
 
 err_put_node:
 	of_node_put(mii_np);
-	eth->mii_bus = NULL;
-	return err;
+	return ret;
 }
 
 static void mtk_mdio_cleanup(struct mtk_eth *eth)
@@ -322,29 +365,29 @@ static void mtk_mdio_cleanup(struct mtk_eth *eth)
 		return;
 
 	mdiobus_unregister(eth->mii_bus);
-	of_node_put(eth->mii_bus->dev.of_node);
-	mdiobus_free(eth->mii_bus);
 }
 
-static inline void mtk_irq_disable(struct mtk_eth *eth, u32 mask)
+static inline void mtk_irq_disable(struct mtk_eth *eth,
+				   unsigned reg, u32 mask)
 {
 	unsigned long flags;
 	u32 val;
 
 	spin_lock_irqsave(&eth->irq_lock, flags);
-	val = mtk_r32(eth, MTK_QDMA_INT_MASK);
-	mtk_w32(eth, val & ~mask, MTK_QDMA_INT_MASK);
+	val = mtk_r32(eth, reg);
+	mtk_w32(eth, val & ~mask, reg);
 	spin_unlock_irqrestore(&eth->irq_lock, flags);
 }
 
-static inline void mtk_irq_enable(struct mtk_eth *eth, u32 mask)
+static inline void mtk_irq_enable(struct mtk_eth *eth,
+				  unsigned reg, u32 mask)
 {
 	unsigned long flags;
 	u32 val;
 
 	spin_lock_irqsave(&eth->irq_lock, flags);
-	val = mtk_r32(eth, MTK_QDMA_INT_MASK);
-	mtk_w32(eth, val | mask, MTK_QDMA_INT_MASK);
+	val = mtk_r32(eth, reg);
+	mtk_w32(eth, val | mask, reg);
 	spin_unlock_irqrestore(&eth->irq_lock, flags);
 }
 
@@ -353,18 +396,20 @@ static int mtk_set_mac_address(struct net_device *dev, void *p)
 	int ret = eth_mac_addr(dev, p);
 	struct mtk_mac *mac = netdev_priv(dev);
 	const char *macaddr = dev->dev_addr;
-	unsigned long flags;
 
 	if (ret)
 		return ret;
 
-	spin_lock_irqsave(&mac->hw->page_lock, flags);
+	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
+		return -EBUSY;
+
+	spin_lock_bh(&mac->hw->page_lock);
 	mtk_w32(mac->hw, (macaddr[0] << 8) | macaddr[1],
 		MTK_GDMA_MAC_ADRH(mac->id));
 	mtk_w32(mac->hw, (macaddr[2] << 24) | (macaddr[3] << 16) |
 		(macaddr[4] << 8) | macaddr[5],
 		MTK_GDMA_MAC_ADRL(mac->id));
-	spin_unlock_irqrestore(&mac->hw->page_lock, flags);
+	spin_unlock_bh(&mac->hw->page_lock);
 
 	return 0;
 }
@@ -542,15 +587,15 @@ static inline struct mtk_tx_buf *mtk_desc_to_tx_buf(struct mtk_tx_ring *ring,
 	return &ring->buf[idx];
 }
 
-static void mtk_tx_unmap(struct device *dev, struct mtk_tx_buf *tx_buf)
+static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf)
 {
 	if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) {
-		dma_unmap_single(dev,
+		dma_unmap_single(eth->dev,
 				 dma_unmap_addr(tx_buf, dma_addr0),
 				 dma_unmap_len(tx_buf, dma_len0),
 				 DMA_TO_DEVICE);
 	} else if (tx_buf->flags & MTK_TX_FLAGS_PAGE0) {
-		dma_unmap_page(dev,
+		dma_unmap_page(eth->dev,
 			       dma_unmap_addr(tx_buf, dma_addr0),
 			       dma_unmap_len(tx_buf, dma_len0),
 			       DMA_TO_DEVICE);
@@ -572,14 +617,15 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev,
 	dma_addr_t mapped_addr;
 	unsigned int nr_frags;
 	int i, n_desc = 1;
-	u32 txd4 = 0;
+	u32 txd4 = 0, fport;
 
 	itxd = ring->next_free;
 	if (itxd == ring->last_free)
 		return -ENOMEM;
 
 	/* set the forward port */
-	txd4 |= (mac->id + 1) << TX_DMA_FPORT_SHIFT;
+	fport = (mac->id + 1) << TX_DMA_FPORT_SHIFT;
+	txd4 |= fport;
 
 	tx_buf = mtk_desc_to_tx_buf(ring, itxd);
 	memset(tx_buf, 0, sizeof(*tx_buf));
@@ -595,9 +641,9 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev,
 	if (skb_vlan_tag_present(skb))
 		txd4 |= TX_DMA_INS_VLAN | skb_vlan_tag_get(skb);
 
-	mapped_addr = dma_map_single(&dev->dev, skb->data,
+	mapped_addr = dma_map_single(eth->dev, skb->data,
 				     skb_headlen(skb), DMA_TO_DEVICE);
-	if (unlikely(dma_mapping_error(&dev->dev, mapped_addr)))
+	if (unlikely(dma_mapping_error(eth->dev, mapped_addr)))
 		return -ENOMEM;
 
 	WRITE_ONCE(itxd->txd1, mapped_addr);
@@ -623,10 +669,10 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev,
 
 			n_desc++;
 			frag_map_size = min(frag_size, MTK_TX_DMA_BUF_LEN);
-			mapped_addr = skb_frag_dma_map(&dev->dev, frag, offset,
+			mapped_addr = skb_frag_dma_map(eth->dev, frag, offset,
 						       frag_map_size,
 						       DMA_TO_DEVICE);
-			if (unlikely(dma_mapping_error(&dev->dev, mapped_addr)))
+			if (unlikely(dma_mapping_error(eth->dev, mapped_addr)))
 				goto err_dma;
 
 			if (i == nr_frags - 1 &&
@@ -637,7 +683,7 @@ static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev,
 			WRITE_ONCE(txd->txd3, (TX_DMA_SWC |
 					       TX_DMA_PLEN0(frag_map_size) |
 					       last_frag * TX_DMA_LS0));
-			WRITE_ONCE(txd->txd4, 0);
+			WRITE_ONCE(txd->txd4, fport);
 
 			tx_buf->skb = (struct sk_buff *)MTK_DMA_DUMMY_DESC;
 			tx_buf = mtk_desc_to_tx_buf(ring, txd);
@@ -679,7 +725,7 @@ err_dma:
 		tx_buf = mtk_desc_to_tx_buf(ring, itxd);
 
 		/* unmap dma */
-		mtk_tx_unmap(&dev->dev, tx_buf);
+		mtk_tx_unmap(eth, tx_buf);
 
 		itxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
 		itxd = mtk_qdma_phys_to_virt(ring, itxd->txd2);
@@ -748,7 +794,6 @@ static int mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	struct mtk_eth *eth = mac->hw;
 	struct mtk_tx_ring *ring = &eth->tx_ring;
 	struct net_device_stats *stats = &dev->stats;
-	unsigned long flags;
 	bool gso = false;
 	int tx_num;
 
@@ -756,14 +801,17 @@ static int mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	 * however we have 2 queues running on the same ring so we need to lock
 	 * the ring access
 	 */
-	spin_lock_irqsave(&eth->page_lock, flags);
+	spin_lock(&eth->page_lock);
+
+	if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
+		goto drop;
 
 	tx_num = mtk_cal_txd_req(skb);
 	if (unlikely(atomic_read(&ring->free_count) <= tx_num)) {
 		mtk_stop_queue(eth);
 		netif_err(eth, tx_queued, dev,
 			  "Tx Ring full when queue awake!\n");
-		spin_unlock_irqrestore(&eth->page_lock, flags);
+		spin_unlock(&eth->page_lock);
 		return NETDEV_TX_BUSY;
 	}
 
@@ -788,22 +836,62 @@ static int mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	if (unlikely(atomic_read(&ring->free_count) <= ring->thresh))
 		mtk_stop_queue(eth);
 
-	spin_unlock_irqrestore(&eth->page_lock, flags);
+	spin_unlock(&eth->page_lock);
 
 	return NETDEV_TX_OK;
 
 drop:
-	spin_unlock_irqrestore(&eth->page_lock, flags);
+	spin_unlock(&eth->page_lock);
 	stats->tx_dropped++;
 	dev_kfree_skb(skb);
 	return NETDEV_TX_OK;
 }
 
+static struct mtk_rx_ring *mtk_get_rx_ring(struct mtk_eth *eth)
+{
+	int i;
+	struct mtk_rx_ring *ring;
+	int idx;
+
+	if (!eth->hwlro)
+		return &eth->rx_ring[0];
+
+	for (i = 0; i < MTK_MAX_RX_RING_NUM; i++) {
+		ring = &eth->rx_ring[i];
+		idx = NEXT_RX_DESP_IDX(ring->calc_idx, ring->dma_size);
+		if (ring->dma[idx].rxd2 & RX_DMA_DONE) {
+			ring->calc_idx_update = true;
+			return ring;
+		}
+	}
+
+	return NULL;
+}
+
+static void mtk_update_rx_cpu_idx(struct mtk_eth *eth)
+{
+	struct mtk_rx_ring *ring;
+	int i;
+
+	if (!eth->hwlro) {
+		ring = &eth->rx_ring[0];
+		mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg);
+	} else {
+		for (i = 0; i < MTK_MAX_RX_RING_NUM; i++) {
+			ring = &eth->rx_ring[i];
+			if (ring->calc_idx_update) {
+				ring->calc_idx_update = false;
+				mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg);
+			}
+		}
+	}
+}
+
 static int mtk_poll_rx(struct napi_struct *napi, int budget,
 		       struct mtk_eth *eth)
 {
-	struct mtk_rx_ring *ring = &eth->rx_ring;
-	int idx = ring->calc_idx;
+	struct mtk_rx_ring *ring;
+	int idx;
 	struct sk_buff *skb;
 	u8 *data, *new_data;
 	struct mtk_rx_dma *rxd, trxd;
@@ -815,7 +903,11 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
 		dma_addr_t dma_addr;
 		int mac = 0;
 
-		idx = NEXT_RX_DESP_IDX(idx);
+		ring = mtk_get_rx_ring(eth);
+		if (unlikely(!ring))
+			goto rx_done;
+
+		idx = NEXT_RX_DESP_IDX(ring->calc_idx, ring->dma_size);
 		rxd = &ring->dma[idx];
 		data = ring->data[idx];
 
@@ -830,17 +922,20 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
 
 		netdev = eth->netdev[mac];
 
+		if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
+			goto release_desc;
+
 		/* alloc new buffer */
 		new_data = napi_alloc_frag(ring->frag_size);
 		if (unlikely(!new_data)) {
 			netdev->stats.rx_dropped++;
 			goto release_desc;
 		}
-		dma_addr = dma_map_single(&eth->netdev[mac]->dev,
+		dma_addr = dma_map_single(eth->dev,
 					  new_data + NET_SKB_PAD,
 					  ring->buf_size,
 					  DMA_FROM_DEVICE);
-		if (unlikely(dma_mapping_error(&netdev->dev, dma_addr))) {
+		if (unlikely(dma_mapping_error(eth->dev, dma_addr))) {
 			skb_free_frag(new_data);
 			netdev->stats.rx_dropped++;
 			goto release_desc;
@@ -849,13 +944,13 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
 		/* receive data */
 		skb = build_skb(data, ring->frag_size);
 		if (unlikely(!skb)) {
-			put_page(virt_to_head_page(new_data));
+			skb_free_frag(new_data);
 			netdev->stats.rx_dropped++;
 			goto release_desc;
 		}
 		skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
 
-		dma_unmap_single(&netdev->dev, trxd.rxd1,
+		dma_unmap_single(eth->dev, trxd.rxd1,
 				 ring->buf_size, DMA_FROM_DEVICE);
 		pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
 		skb->dev = netdev;
@@ -879,17 +974,19 @@ release_desc:
 		rxd->rxd2 = RX_DMA_PLEN0(ring->buf_size);
 
 		ring->calc_idx = idx;
+
+		done++;
+	}
+
+rx_done:
+	if (done) {
 		/* make sure that all changes to the dma ring are flushed before
 		 * we continue
 		 */
 		wmb();
-		mtk_w32(eth, ring->calc_idx, MTK_QRX_CRX_IDX0);
-		done++;
+		mtk_update_rx_cpu_idx(eth);
 	}
 
-	if (done < budget)
-		mtk_w32(eth, MTK_RX_DONE_INT, MTK_QMTK_INT_STATUS);
-
 	return done;
 }
 
@@ -937,7 +1034,7 @@ static int mtk_poll_tx(struct mtk_eth *eth, int budget)
 			done[mac]++;
 			budget--;
 		}
-		mtk_tx_unmap(eth->dev, tx_buf);
+		mtk_tx_unmap(eth, tx_buf);
 
 		ring->last_free = desc;
 		atomic_inc(&ring->free_count);
@@ -998,7 +1095,7 @@ static int mtk_napi_tx(struct napi_struct *napi, int budget)
 		return budget;
 
 	napi_complete(napi);
-	mtk_irq_enable(eth, MTK_TX_DONE_INT);
+	mtk_irq_enable(eth, MTK_QDMA_INT_MASK, MTK_TX_DONE_INT);
 
 	return tx_done;
 }
@@ -1008,30 +1105,33 @@ static int mtk_napi_rx(struct napi_struct *napi, int budget)
 	struct mtk_eth *eth = container_of(napi, struct mtk_eth, rx_napi);
 	u32 status, mask;
 	int rx_done = 0;
+	int remain_budget = budget;
 
 	mtk_handle_status_irq(eth);
-	mtk_w32(eth, MTK_RX_DONE_INT, MTK_QMTK_INT_STATUS);
-	rx_done = mtk_poll_rx(napi, budget, eth);
+
+poll_again:
+	mtk_w32(eth, MTK_RX_DONE_INT, MTK_PDMA_INT_STATUS);
+	rx_done = mtk_poll_rx(napi, remain_budget, eth);
 
 	if (unlikely(netif_msg_intr(eth))) {
-		status = mtk_r32(eth, MTK_QMTK_INT_STATUS);
-		mask = mtk_r32(eth, MTK_QDMA_INT_MASK);
+		status = mtk_r32(eth, MTK_PDMA_INT_STATUS);
+		mask = mtk_r32(eth, MTK_PDMA_INT_MASK);
 		dev_info(eth->dev,
 			 "done rx %d, intr 0x%08x/0x%x\n",
 			 rx_done, status, mask);
 	}
-
-	if (rx_done == budget)
-		return budget;
-
-	status = mtk_r32(eth, MTK_QMTK_INT_STATUS);
-	if (status & MTK_RX_DONE_INT)
+	if (rx_done == remain_budget)
 		return budget;
 
+	status = mtk_r32(eth, MTK_PDMA_INT_STATUS);
+	if (status & MTK_RX_DONE_INT) {
+		remain_budget -= rx_done;
+		goto poll_again;
+	}
 	napi_complete(napi);
-	mtk_irq_enable(eth, MTK_RX_DONE_INT);
+	mtk_irq_enable(eth, MTK_PDMA_INT_MASK, MTK_RX_DONE_INT);
 
-	return rx_done;
+	return rx_done + budget - remain_budget;
 }
 
 static int mtk_tx_alloc(struct mtk_eth *eth)
@@ -1078,6 +1178,7 @@ static int mtk_tx_alloc(struct mtk_eth *eth)
 	mtk_w32(eth,
 		ring->phys + ((MTK_DMA_SIZE - 1) * sz),
 		MTK_QTX_DRX_PTR);
+	mtk_w32(eth, (QDMA_RES_THRES << 8) | QDMA_RES_THRES, MTK_QTX_CFG(0));
 
 	return 0;
 
@@ -1092,7 +1193,7 @@ static void mtk_tx_clean(struct mtk_eth *eth)
 
 	if (ring->buf) {
 		for (i = 0; i < MTK_DMA_SIZE; i++)
-			mtk_tx_unmap(eth->dev, &ring->buf[i]);
+			mtk_tx_unmap(eth, &ring->buf[i]);
 		kfree(ring->buf);
 		ring->buf = NULL;
 	}
@@ -1106,32 +1207,41 @@ static void mtk_tx_clean(struct mtk_eth *eth)
 	}
 }
 
-static int mtk_rx_alloc(struct mtk_eth *eth)
+static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
 {
-	struct mtk_rx_ring *ring = &eth->rx_ring;
+	struct mtk_rx_ring *ring = &eth->rx_ring[ring_no];
+	int rx_data_len, rx_dma_size;
 	int i;
 
-	ring->frag_size = mtk_max_frag_size(ETH_DATA_LEN);
+	if (rx_flag == MTK_RX_FLAGS_HWLRO) {
+		rx_data_len = MTK_MAX_LRO_RX_LENGTH;
+		rx_dma_size = MTK_HW_LRO_DMA_SIZE;
+	} else {
+		rx_data_len = ETH_DATA_LEN;
+		rx_dma_size = MTK_DMA_SIZE;
+	}
+
+	ring->frag_size = mtk_max_frag_size(rx_data_len);
 	ring->buf_size = mtk_max_buf_size(ring->frag_size);
-	ring->data = kcalloc(MTK_DMA_SIZE, sizeof(*ring->data),
+	ring->data = kcalloc(rx_dma_size, sizeof(*ring->data),
 			     GFP_KERNEL);
 	if (!ring->data)
 		return -ENOMEM;
 
-	for (i = 0; i < MTK_DMA_SIZE; i++) {
+	for (i = 0; i < rx_dma_size; i++) {
 		ring->data[i] = netdev_alloc_frag(ring->frag_size);
 		if (!ring->data[i])
 			return -ENOMEM;
 	}
 
 	ring->dma = dma_alloc_coherent(eth->dev,
-				       MTK_DMA_SIZE * sizeof(*ring->dma),
+				       rx_dma_size * sizeof(*ring->dma),
 				       &ring->phys,
 				       GFP_ATOMIC | __GFP_ZERO);
 	if (!ring->dma)
 		return -ENOMEM;
 
-	for (i = 0; i < MTK_DMA_SIZE; i++) {
+	for (i = 0; i < rx_dma_size; i++) {
 		dma_addr_t dma_addr = dma_map_single(eth->dev,
 				ring->data[i] + NET_SKB_PAD,
 				ring->buf_size,
@@ -1142,28 +1252,30 @@ static int mtk_rx_alloc(struct mtk_eth *eth)
 
 		ring->dma[i].rxd2 = RX_DMA_PLEN0(ring->buf_size);
 	}
-	ring->calc_idx = MTK_DMA_SIZE - 1;
+	ring->dma_size = rx_dma_size;
+	ring->calc_idx_update = false;
+	ring->calc_idx = rx_dma_size - 1;
+	ring->crx_idx_reg = MTK_PRX_CRX_IDX_CFG(ring_no);
 	/* make sure that all changes to the dma ring are flushed before we
 	 * continue
 	 */
 	wmb();
 
-	mtk_w32(eth, eth->rx_ring.phys, MTK_QRX_BASE_PTR0);
-	mtk_w32(eth, MTK_DMA_SIZE, MTK_QRX_MAX_CNT0);
-	mtk_w32(eth, eth->rx_ring.calc_idx, MTK_QRX_CRX_IDX0);
-	mtk_w32(eth, MTK_PST_DRX_IDX0, MTK_QDMA_RST_IDX);
-	mtk_w32(eth, (QDMA_RES_THRES << 8) | QDMA_RES_THRES, MTK_QTX_CFG(0));
+	mtk_w32(eth, ring->phys, MTK_PRX_BASE_PTR_CFG(ring_no));
+	mtk_w32(eth, rx_dma_size, MTK_PRX_MAX_CNT_CFG(ring_no));
+	mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg);
+	mtk_w32(eth, MTK_PST_DRX_IDX_CFG(ring_no), MTK_PDMA_RST_IDX);
 
 	return 0;
 }
 
-static void mtk_rx_clean(struct mtk_eth *eth)
+static void mtk_rx_clean(struct mtk_eth *eth, int ring_no)
 {
-	struct mtk_rx_ring *ring = &eth->rx_ring;
+	struct mtk_rx_ring *ring = &eth->rx_ring[ring_no];
 	int i;
 
 	if (ring->data && ring->dma) {
-		for (i = 0; i < MTK_DMA_SIZE; i++) {
+		for (i = 0; i < ring->dma_size; i++) {
 			if (!ring->data[i])
 				continue;
 			if (!ring->dma[i].rxd1)
@@ -1180,13 +1292,275 @@ static void mtk_rx_clean(struct mtk_eth *eth)
 
 	if (ring->dma) {
 		dma_free_coherent(eth->dev,
-				  MTK_DMA_SIZE * sizeof(*ring->dma),
+				  ring->dma_size * sizeof(*ring->dma),
 				  ring->dma,
 				  ring->phys);
 		ring->dma = NULL;
 	}
 }
 
+static int mtk_hwlro_rx_init(struct mtk_eth *eth)
+{
+	int i;
+	u32 ring_ctrl_dw1 = 0, ring_ctrl_dw2 = 0, ring_ctrl_dw3 = 0;
+	u32 lro_ctrl_dw0 = 0, lro_ctrl_dw3 = 0;
+
+	/* set LRO rings to auto-learn modes */
+	ring_ctrl_dw2 |= MTK_RING_AUTO_LERAN_MODE;
+
+	/* validate LRO ring */
+	ring_ctrl_dw2 |= MTK_RING_VLD;
+
+	/* set AGE timer (unit: 20us) */
+	ring_ctrl_dw2 |= MTK_RING_AGE_TIME_H;
+	ring_ctrl_dw1 |= MTK_RING_AGE_TIME_L;
+
+	/* set max AGG timer (unit: 20us) */
+	ring_ctrl_dw2 |= MTK_RING_MAX_AGG_TIME;
+
+	/* set max LRO AGG count */
+	ring_ctrl_dw2 |= MTK_RING_MAX_AGG_CNT_L;
+	ring_ctrl_dw3 |= MTK_RING_MAX_AGG_CNT_H;
+
+	for (i = 1; i < MTK_MAX_RX_RING_NUM; i++) {
+		mtk_w32(eth, ring_ctrl_dw1, MTK_LRO_CTRL_DW1_CFG(i));
+		mtk_w32(eth, ring_ctrl_dw2, MTK_LRO_CTRL_DW2_CFG(i));
+		mtk_w32(eth, ring_ctrl_dw3, MTK_LRO_CTRL_DW3_CFG(i));
+	}
+
+	/* IPv4 checksum update enable */
+	lro_ctrl_dw0 |= MTK_L3_CKS_UPD_EN;
+
+	/* switch priority comparison to packet count mode */
+	lro_ctrl_dw0 |= MTK_LRO_ALT_PKT_CNT_MODE;
+
+	/* bandwidth threshold setting */
+	mtk_w32(eth, MTK_HW_LRO_BW_THRE, MTK_PDMA_LRO_CTRL_DW2);
+
+	/* auto-learn score delta setting */
+	mtk_w32(eth, MTK_HW_LRO_REPLACE_DELTA, MTK_PDMA_LRO_ALT_SCORE_DELTA);
+
+	/* set refresh timer for altering flows to 1 sec. (unit: 20us) */
+	mtk_w32(eth, (MTK_HW_LRO_TIMER_UNIT << 16) | MTK_HW_LRO_REFRESH_TIME,
+		MTK_PDMA_LRO_ALT_REFRESH_TIMER);
+
+	/* set HW LRO mode & the max aggregation count for rx packets */
+	lro_ctrl_dw3 |= MTK_ADMA_MODE | (MTK_HW_LRO_MAX_AGG_CNT & 0xff);
+
+	/* the minimal remaining room of SDL0 in RXD for lro aggregation */
+	lro_ctrl_dw3 |= MTK_LRO_MIN_RXD_SDL;
+
+	/* enable HW LRO */
+	lro_ctrl_dw0 |= MTK_LRO_EN;
+
+	mtk_w32(eth, lro_ctrl_dw3, MTK_PDMA_LRO_CTRL_DW3);
+	mtk_w32(eth, lro_ctrl_dw0, MTK_PDMA_LRO_CTRL_DW0);
+
+	return 0;
+}
+
+static void mtk_hwlro_rx_uninit(struct mtk_eth *eth)
+{
+	int i;
+	u32 val;
+
+	/* relinquish lro rings, flush aggregated packets */
+	mtk_w32(eth, MTK_LRO_RING_RELINQUISH_REQ, MTK_PDMA_LRO_CTRL_DW0);
+
+	/* wait for relinquishments done */
+	for (i = 0; i < 10; i++) {
+		val = mtk_r32(eth, MTK_PDMA_LRO_CTRL_DW0);
+		if (val & MTK_LRO_RING_RELINQUISH_DONE) {
+			msleep(20);
+			continue;
+		}
+		break;
+	}
+
+	/* invalidate lro rings */
+	for (i = 1; i < MTK_MAX_RX_RING_NUM; i++)
+		mtk_w32(eth, 0, MTK_LRO_CTRL_DW2_CFG(i));
+
+	/* disable HW LRO */
+	mtk_w32(eth, 0, MTK_PDMA_LRO_CTRL_DW0);
+}
+
+static void mtk_hwlro_val_ipaddr(struct mtk_eth *eth, int idx, __be32 ip)
+{
+	u32 reg_val;
+
+	reg_val = mtk_r32(eth, MTK_LRO_CTRL_DW2_CFG(idx));
+
+	/* invalidate the IP setting */
+	mtk_w32(eth, (reg_val & ~MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
+
+	mtk_w32(eth, ip, MTK_LRO_DIP_DW0_CFG(idx));
+
+	/* validate the IP setting */
+	mtk_w32(eth, (reg_val | MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
+}
+
+static void mtk_hwlro_inval_ipaddr(struct mtk_eth *eth, int idx)
+{
+	u32 reg_val;
+
+	reg_val = mtk_r32(eth, MTK_LRO_CTRL_DW2_CFG(idx));
+
+	/* invalidate the IP setting */
+	mtk_w32(eth, (reg_val & ~MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
+
+	mtk_w32(eth, 0, MTK_LRO_DIP_DW0_CFG(idx));
+}
+
+static int mtk_hwlro_get_ip_cnt(struct mtk_mac *mac)
+{
+	int cnt = 0;
+	int i;
+
+	for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
+		if (mac->hwlro_ip[i])
+			cnt++;
+	}
+
+	return cnt;
+}
+
+static int mtk_hwlro_add_ipaddr(struct net_device *dev,
+				struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fsp =
+		(struct ethtool_rx_flow_spec *)&cmd->fs;
+	struct mtk_mac *mac = netdev_priv(dev);
+	struct mtk_eth *eth = mac->hw;
+	int hwlro_idx;
+
+	if ((fsp->flow_type != TCP_V4_FLOW) ||
+	    (!fsp->h_u.tcp_ip4_spec.ip4dst) ||
+	    (fsp->location > 1))
+		return -EINVAL;
+
+	mac->hwlro_ip[fsp->location] = htonl(fsp->h_u.tcp_ip4_spec.ip4dst);
+	hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + fsp->location;
+
+	mac->hwlro_ip_cnt = mtk_hwlro_get_ip_cnt(mac);
+
+	mtk_hwlro_val_ipaddr(eth, hwlro_idx, mac->hwlro_ip[fsp->location]);
+
+	return 0;
+}
+
+static int mtk_hwlro_del_ipaddr(struct net_device *dev,
+				struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fsp =
+		(struct ethtool_rx_flow_spec *)&cmd->fs;
+	struct mtk_mac *mac = netdev_priv(dev);
+	struct mtk_eth *eth = mac->hw;
+	int hwlro_idx;
+
+	if (fsp->location > 1)
+		return -EINVAL;
+
+	mac->hwlro_ip[fsp->location] = 0;
+	hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + fsp->location;
+
+	mac->hwlro_ip_cnt = mtk_hwlro_get_ip_cnt(mac);
+
+	mtk_hwlro_inval_ipaddr(eth, hwlro_idx);
+
+	return 0;
+}
+
+static void mtk_hwlro_netdev_disable(struct net_device *dev)
+{
+	struct mtk_mac *mac = netdev_priv(dev);
+	struct mtk_eth *eth = mac->hw;
+	int i, hwlro_idx;
+
+	for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
+		mac->hwlro_ip[i] = 0;
+		hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + i;
+
+		mtk_hwlro_inval_ipaddr(eth, hwlro_idx);
+	}
+
+	mac->hwlro_ip_cnt = 0;
+}
+
+static int mtk_hwlro_get_fdir_entry(struct net_device *dev,
+				    struct ethtool_rxnfc *cmd)
+{
+	struct mtk_mac *mac = netdev_priv(dev);
+	struct ethtool_rx_flow_spec *fsp =
+		(struct ethtool_rx_flow_spec *)&cmd->fs;
+
+	/* only tcp dst ipv4 is meaningful, others are meaningless */
+	fsp->flow_type = TCP_V4_FLOW;
+	fsp->h_u.tcp_ip4_spec.ip4dst = ntohl(mac->hwlro_ip[fsp->location]);
+	fsp->m_u.tcp_ip4_spec.ip4dst = 0;
+
+	fsp->h_u.tcp_ip4_spec.ip4src = 0;
+	fsp->m_u.tcp_ip4_spec.ip4src = 0xffffffff;
+	fsp->h_u.tcp_ip4_spec.psrc = 0;
+	fsp->m_u.tcp_ip4_spec.psrc = 0xffff;
+	fsp->h_u.tcp_ip4_spec.pdst = 0;
+	fsp->m_u.tcp_ip4_spec.pdst = 0xffff;
+	fsp->h_u.tcp_ip4_spec.tos = 0;
+	fsp->m_u.tcp_ip4_spec.tos = 0xff;
+
+	return 0;
+}
+
+static int mtk_hwlro_get_fdir_all(struct net_device *dev,
+				  struct ethtool_rxnfc *cmd,
+				  u32 *rule_locs)
+{
+	struct mtk_mac *mac = netdev_priv(dev);
+	int cnt = 0;
+	int i;
+
+	for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
+		if (mac->hwlro_ip[i]) {
+			rule_locs[cnt] = i;
+			cnt++;
+		}
+	}
+
+	cmd->rule_cnt = cnt;
+
+	return 0;
+}
+
+static netdev_features_t mtk_fix_features(struct net_device *dev,
+					  netdev_features_t features)
+{
+	if (!(features & NETIF_F_LRO)) {
+		struct mtk_mac *mac = netdev_priv(dev);
+		int ip_cnt = mtk_hwlro_get_ip_cnt(mac);
+
+		if (ip_cnt) {
+			netdev_info(dev, "RX flow is programmed, LRO should keep on\n");
+
+			features |= NETIF_F_LRO;
+		}
+	}
+
+	return features;
+}
+
+static int mtk_set_features(struct net_device *dev, netdev_features_t features)
+{
+	int err = 0;
+
+	if (!((dev->features ^ features) & NETIF_F_LRO))
+		return 0;
+
+	if (!(features & NETIF_F_LRO))
+		mtk_hwlro_netdev_disable(dev);
+
+	return err;
+}
+
 /* wait for DMA to finish whatever it is doing before we start using it again */
 static int mtk_dma_busy_wait(struct mtk_eth *eth)
 {
@@ -1207,6 +1581,7 @@ static int mtk_dma_busy_wait(struct mtk_eth *eth)
 static int mtk_dma_init(struct mtk_eth *eth)
 {
 	int err;
+	u32 i;
 
 	if (mtk_dma_busy_wait(eth))
 		return -EBUSY;
@@ -1222,10 +1597,21 @@ static int mtk_dma_init(struct mtk_eth *eth)
 	if (err)
 		return err;
 
-	err = mtk_rx_alloc(eth);
+	err = mtk_rx_alloc(eth, 0, MTK_RX_FLAGS_NORMAL);
 	if (err)
 		return err;
 
+	if (eth->hwlro) {
+		for (i = 1; i < MTK_MAX_RX_RING_NUM; i++) {
+			err = mtk_rx_alloc(eth, i, MTK_RX_FLAGS_HWLRO);
+			if (err)
+				return err;
+		}
+		err = mtk_hwlro_rx_init(eth);
+		if (err)
+			return err;
+	}
+
 	/* Enable random early drop and set drop threshold automatically */
 	mtk_w32(eth, FC_THRES_DROP_MODE | FC_THRES_DROP_EN | FC_THRES_MIN,
 		MTK_QDMA_FC_THRES);
@@ -1250,7 +1636,14 @@ static void mtk_dma_free(struct mtk_eth *eth)
 		eth->phy_scratch_ring = 0;
 	}
 	mtk_tx_clean(eth);
-	mtk_rx_clean(eth);
+	mtk_rx_clean(eth, 0);
+
+	if (eth->hwlro) {
+		mtk_hwlro_rx_uninit(eth);
+		for (i = 1; i < MTK_MAX_RX_RING_NUM; i++)
+			mtk_rx_clean(eth, i);
+	}
+
 	kfree(eth->scratch_head);
 }
 
@@ -1271,7 +1664,7 @@ static irqreturn_t mtk_handle_irq_rx(int irq, void *_eth)
 
 	if (likely(napi_schedule_prep(&eth->rx_napi))) {
 		__napi_schedule(&eth->rx_napi);
-		mtk_irq_disable(eth, MTK_RX_DONE_INT);
+		mtk_irq_disable(eth, MTK_PDMA_INT_MASK, MTK_RX_DONE_INT);
 	}
 
 	return IRQ_HANDLED;
@@ -1283,7 +1676,7 @@ static irqreturn_t mtk_handle_irq_tx(int irq, void *_eth)
 
 	if (likely(napi_schedule_prep(&eth->tx_napi))) {
 		__napi_schedule(&eth->tx_napi);
-		mtk_irq_disable(eth, MTK_TX_DONE_INT);
+		mtk_irq_disable(eth, MTK_QDMA_INT_MASK, MTK_TX_DONE_INT);
 	}
 
 	return IRQ_HANDLED;
@@ -1294,11 +1687,12 @@ static void mtk_poll_controller(struct net_device *dev)
 {
 	struct mtk_mac *mac = netdev_priv(dev);
 	struct mtk_eth *eth = mac->hw;
-	u32 int_mask = MTK_TX_DONE_INT | MTK_RX_DONE_INT;
 
-	mtk_irq_disable(eth, int_mask);
+	mtk_irq_disable(eth, MTK_QDMA_INT_MASK, MTK_TX_DONE_INT);
+	mtk_irq_disable(eth, MTK_PDMA_INT_MASK, MTK_RX_DONE_INT);
 	mtk_handle_irq_rx(eth->irq[2], dev);
-	mtk_irq_enable(eth, int_mask);
+	mtk_irq_enable(eth, MTK_QDMA_INT_MASK, MTK_TX_DONE_INT);
+	mtk_irq_enable(eth, MTK_PDMA_INT_MASK, MTK_RX_DONE_INT);
 }
 #endif
 
@@ -1313,11 +1707,15 @@ static int mtk_start_dma(struct mtk_eth *eth)
 	}
 
 	mtk_w32(eth,
-		MTK_TX_WB_DDONE | MTK_RX_DMA_EN | MTK_TX_DMA_EN |
-		MTK_RX_2B_OFFSET | MTK_DMA_SIZE_16DWORDS |
-		MTK_RX_BT_32DWORDS | MTK_NDP_CO_PRO,
+		MTK_TX_WB_DDONE | MTK_TX_DMA_EN |
+		MTK_DMA_SIZE_16DWORDS | MTK_NDP_CO_PRO,
 		MTK_QDMA_GLO_CFG);
 
+	mtk_w32(eth,
+		MTK_RX_DMA_EN | MTK_RX_2B_OFFSET |
+		MTK_RX_BT_32DWORDS | MTK_MULTI_EN,
+		MTK_PDMA_GLO_CFG);
+
 	return 0;
 }
 
@@ -1335,11 +1733,12 @@ static int mtk_open(struct net_device *dev)
 
 		napi_enable(&eth->tx_napi);
 		napi_enable(&eth->rx_napi);
-		mtk_irq_enable(eth, MTK_TX_DONE_INT | MTK_RX_DONE_INT);
+		mtk_irq_enable(eth, MTK_QDMA_INT_MASK, MTK_TX_DONE_INT);
+		mtk_irq_enable(eth, MTK_PDMA_INT_MASK, MTK_RX_DONE_INT);
 	}
 	atomic_inc(&eth->dma_refcnt);
 
-	phy_start(mac->phy_dev);
+	phy_start(dev->phydev);
 	netif_start_queue(dev);
 
 	return 0;
@@ -1347,16 +1746,15 @@ static int mtk_open(struct net_device *dev)
 
 static void mtk_stop_dma(struct mtk_eth *eth, u32 glo_cfg)
 {
-	unsigned long flags;
 	u32 val;
 	int i;
 
 	/* stop the dma engine */
-	spin_lock_irqsave(&eth->page_lock, flags);
+	spin_lock_bh(&eth->page_lock);
 	val = mtk_r32(eth, glo_cfg);
 	mtk_w32(eth, val & ~(MTK_TX_WB_DDONE | MTK_RX_DMA_EN | MTK_TX_DMA_EN),
 		glo_cfg);
-	spin_unlock_irqrestore(&eth->page_lock, flags);
+	spin_unlock_bh(&eth->page_lock);
 
 	/* wait for dma stop */
 	for (i = 0; i < 10; i++) {
@@ -1375,32 +1773,63 @@ static int mtk_stop(struct net_device *dev)
 	struct mtk_eth *eth = mac->hw;
 
 	netif_tx_disable(dev);
-	phy_stop(mac->phy_dev);
+	phy_stop(dev->phydev);
 
 	/* only shutdown DMA if this is the last user */
 	if (!atomic_dec_and_test(&eth->dma_refcnt))
 		return 0;
 
-	mtk_irq_disable(eth, MTK_TX_DONE_INT | MTK_RX_DONE_INT);
+	mtk_irq_disable(eth, MTK_QDMA_INT_MASK, MTK_TX_DONE_INT);
+	mtk_irq_disable(eth, MTK_PDMA_INT_MASK, MTK_RX_DONE_INT);
 	napi_disable(&eth->tx_napi);
 	napi_disable(&eth->rx_napi);
 
 	mtk_stop_dma(eth, MTK_QDMA_GLO_CFG);
+	mtk_stop_dma(eth, MTK_PDMA_GLO_CFG);
 
 	mtk_dma_free(eth);
 
 	return 0;
 }
 
-static int __init mtk_hw_init(struct mtk_eth *eth)
+static void ethsys_reset(struct mtk_eth *eth, u32 reset_bits)
 {
-	int err, i;
+	regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL,
+			   reset_bits,
+			   reset_bits);
+
+	usleep_range(1000, 1100);
+	regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL,
+			   reset_bits,
+			   ~reset_bits);
+	mdelay(10);
+}
+
+static int mtk_hw_init(struct mtk_eth *eth)
+{
+	int i, val;
 
-	/* reset the frame engine */
-	reset_control_assert(eth->rstc);
-	usleep_range(10, 20);
-	reset_control_deassert(eth->rstc);
-	usleep_range(10, 20);
+	if (test_and_set_bit(MTK_HW_INIT, &eth->state))
+		return 0;
+
+	pm_runtime_enable(eth->dev);
+	pm_runtime_get_sync(eth->dev);
+
+	clk_prepare_enable(eth->clks[MTK_CLK_ETHIF]);
+	clk_prepare_enable(eth->clks[MTK_CLK_ESW]);
+	clk_prepare_enable(eth->clks[MTK_CLK_GP1]);
+	clk_prepare_enable(eth->clks[MTK_CLK_GP2]);
+	ethsys_reset(eth, RSTCTRL_FE);
+	ethsys_reset(eth, RSTCTRL_PPE);
+
+	regmap_read(eth->ethsys, ETHSYS_SYSCFG0, &val);
+	for (i = 0; i < MTK_MAC_COUNT; i++) {
+		if (!eth->mac[i])
+			continue;
+		val &= ~SYSCFG0_GE_MODE(SYSCFG0_GE_MASK, eth->mac[i]->id);
+		val |= SYSCFG0_GE_MODE(eth->mac[i]->ge_mode, eth->mac[i]->id);
+	}
+	regmap_write(eth->ethsys, ETHSYS_SYSCFG0, val);
 
 	/* Set GE2 driving and slew rate */
 	regmap_write(eth->pctl, GPIO_DRV_SEL10, 0xa00);
@@ -1420,22 +1849,11 @@ static int __init mtk_hw_init(struct mtk_eth *eth)
 	/* Enable RX VLan Offloading */
 	mtk_w32(eth, 1, MTK_CDMP_EG_CTRL);
 
-	err = devm_request_irq(eth->dev, eth->irq[1], mtk_handle_irq_tx, 0,
-			       dev_name(eth->dev), eth);
-	if (err)
-		return err;
-	err = devm_request_irq(eth->dev, eth->irq[2], mtk_handle_irq_rx, 0,
-			       dev_name(eth->dev), eth);
-	if (err)
-		return err;
-
-	err = mtk_mdio_init(eth);
-	if (err)
-		return err;
-
 	/* disable delay and normal interrupt */
 	mtk_w32(eth, 0, MTK_QDMA_DELAY_INT);
-	mtk_irq_disable(eth, ~0);
+	mtk_w32(eth, 0, MTK_PDMA_DELAY_INT);
+	mtk_irq_disable(eth, MTK_QDMA_INT_MASK, ~0);
+	mtk_irq_disable(eth, MTK_PDMA_INT_MASK, ~0);
 	mtk_w32(eth, RST_GL_PSE, MTK_RST_GL);
 	mtk_w32(eth, 0, MTK_RST_GL);
 
@@ -1449,9 +1867,8 @@ static int __init mtk_hw_init(struct mtk_eth *eth)
 	for (i = 0; i < 2; i++) {
 		u32 val = mtk_r32(eth, MTK_GDMA_FWD_CFG(i));
 
-		/* setup the forward port to send frame to QDMA */
+		/* setup the forward port to send frame to PDMA */
 		val &= ~0xffff;
-		val |= 0x5555;
 
 		/* Enable RX checksum */
 		val |= MTK_GDMA_ICS_EN | MTK_GDMA_TCS_EN | MTK_GDMA_UCS_EN;
@@ -1463,6 +1880,22 @@ static int __init mtk_hw_init(struct mtk_eth *eth)
 	return 0;
 }
 
+static int mtk_hw_deinit(struct mtk_eth *eth)
+{
+	if (!test_and_clear_bit(MTK_HW_INIT, &eth->state))
+		return 0;
+
+	clk_disable_unprepare(eth->clks[MTK_CLK_GP2]);
+	clk_disable_unprepare(eth->clks[MTK_CLK_GP1]);
+	clk_disable_unprepare(eth->clks[MTK_CLK_ESW]);
+	clk_disable_unprepare(eth->clks[MTK_CLK_ETHIF]);
+
+	pm_runtime_put_sync(eth->dev);
+	pm_runtime_disable(eth->dev);
+
+	return 0;
+}
+
 static int __init mtk_init(struct net_device *dev)
 {
 	struct mtk_mac *mac = netdev_priv(dev);
@@ -1481,7 +1914,7 @@ static int __init mtk_init(struct net_device *dev)
 		dev->addr_assign_type = NET_ADDR_RANDOM;
 	}
 
-	return mtk_phy_connect(mac);
+	return mtk_phy_connect(dev);
 }
 
 static void mtk_uninit(struct net_device *dev)
@@ -1489,22 +1922,18 @@ static void mtk_uninit(struct net_device *dev)
 	struct mtk_mac *mac = netdev_priv(dev);
 	struct mtk_eth *eth = mac->hw;
 
-	phy_disconnect(mac->phy_dev);
-	mtk_mdio_cleanup(eth);
-	mtk_irq_disable(eth, ~0);
-	free_irq(eth->irq[1], dev);
-	free_irq(eth->irq[2], dev);
+	phy_disconnect(dev->phydev);
+	mtk_irq_disable(eth, MTK_QDMA_INT_MASK, ~0);
+	mtk_irq_disable(eth, MTK_PDMA_INT_MASK, ~0);
 }
 
 static int mtk_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 {
-	struct mtk_mac *mac = netdev_priv(dev);
-
 	switch (cmd) {
 	case SIOCGMIIPHY:
 	case SIOCGMIIREG:
 	case SIOCSMIIREG:
-		return phy_mii_ioctl(mac->phy_dev, ifr, cmd);
+		return phy_mii_ioctl(dev->phydev, ifr, cmd);
 	default:
 		break;
 	}
@@ -1520,6 +1949,12 @@ static void mtk_pending_work(struct work_struct *work)
 
 	rtnl_lock();
 
+	dev_dbg(eth->dev, "[%s][%d] reset\n", __func__, __LINE__);
+
+	while (test_and_set_bit_lock(MTK_RESETTING, &eth->state))
+		cpu_relax();
+
+	dev_dbg(eth->dev, "[%s][%d] mtk_stop starts\n", __func__, __LINE__);
 	/* stop all devices to make sure that dma is properly shut down */
 	for (i = 0; i < MTK_MAC_COUNT; i++) {
 		if (!eth->netdev[i])
@@ -1527,6 +1962,27 @@ static void mtk_pending_work(struct work_struct *work)
 		mtk_stop(eth->netdev[i]);
 		__set_bit(i, &restart);
 	}
+	dev_dbg(eth->dev, "[%s][%d] mtk_stop ends\n", __func__, __LINE__);
+
+	/* restart underlying hardware such as power, clock, pin mux
+	 * and the connected phy
+	 */
+	mtk_hw_deinit(eth);
+
+	if (eth->dev->pins)
+		pinctrl_select_state(eth->dev->pins->p,
+				     eth->dev->pins->default_state);
+	mtk_hw_init(eth);
+
+	for (i = 0; i < MTK_MAC_COUNT; i++) {
+		if (!eth->mac[i] ||
+		    of_phy_is_fixed_link(eth->mac[i]->of_node))
+			continue;
+		err = phy_init_hw(eth->netdev[i]->phydev);
+		if (err)
+			dev_err(eth->dev, "%s: PHY init failed.\n",
+				eth->netdev[i]->name);
+	}
 
 	/* restart DMA and enable IRQs */
 	for (i = 0; i < MTK_MAC_COUNT; i++) {
@@ -1539,51 +1995,69 @@ static void mtk_pending_work(struct work_struct *work)
 			dev_close(eth->netdev[i]);
 		}
 	}
+
+	dev_dbg(eth->dev, "[%s][%d] reset done\n", __func__, __LINE__);
+
+	clear_bit_unlock(MTK_RESETTING, &eth->state);
+
 	rtnl_unlock();
 }
 
-static int mtk_cleanup(struct mtk_eth *eth)
+static int mtk_free_dev(struct mtk_eth *eth)
 {
 	int i;
 
 	for (i = 0; i < MTK_MAC_COUNT; i++) {
 		if (!eth->netdev[i])
 			continue;
+		free_netdev(eth->netdev[i]);
+	}
 
+	return 0;
+}
+
+static int mtk_unreg_dev(struct mtk_eth *eth)
+{
+	int i;
+
+	for (i = 0; i < MTK_MAC_COUNT; i++) {
+		if (!eth->netdev[i])
+			continue;
 		unregister_netdev(eth->netdev[i]);
-		free_netdev(eth->netdev[i]);
 	}
+
+	return 0;
+}
+
+static int mtk_cleanup(struct mtk_eth *eth)
+{
+	mtk_unreg_dev(eth);
+	mtk_free_dev(eth);
 	cancel_work_sync(&eth->pending_work);
 
 	return 0;
 }
 
-static int mtk_get_settings(struct net_device *dev,
-			    struct ethtool_cmd *cmd)
+static int mtk_get_link_ksettings(struct net_device *ndev,
+				  struct ethtool_link_ksettings *cmd)
 {
-	struct mtk_mac *mac = netdev_priv(dev);
-	int err;
+	struct mtk_mac *mac = netdev_priv(ndev);
 
-	err = phy_read_status(mac->phy_dev);
-	if (err)
-		return -ENODEV;
+	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
+		return -EBUSY;
 
-	return phy_ethtool_gset(mac->phy_dev, cmd);
+	return phy_ethtool_ksettings_get(ndev->phydev, cmd);
 }
 
-static int mtk_set_settings(struct net_device *dev,
-			    struct ethtool_cmd *cmd)
+static int mtk_set_link_ksettings(struct net_device *ndev,
+				  const struct ethtool_link_ksettings *cmd)
 {
-	struct mtk_mac *mac = netdev_priv(dev);
+	struct mtk_mac *mac = netdev_priv(ndev);
 
-	if (cmd->phy_address != mac->phy_dev->mdio.addr) {
-		mac->phy_dev = mdiobus_get_phy(mac->hw->mii_bus,
-					       cmd->phy_address);
-		if (!mac->phy_dev)
-			return -ENODEV;
-	}
+	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
+		return -EBUSY;
 
-	return phy_ethtool_sset(mac->phy_dev, cmd);
+	return phy_ethtool_ksettings_set(ndev->phydev, cmd);
 }
 
 static void mtk_get_drvinfo(struct net_device *dev,
@@ -1614,7 +2088,10 @@ static int mtk_nway_reset(struct net_device *dev)
 {
 	struct mtk_mac *mac = netdev_priv(dev);
 
-	return genphy_restart_aneg(mac->phy_dev);
+	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
+		return -EBUSY;
+
+	return genphy_restart_aneg(dev->phydev);
 }
 
 static u32 mtk_get_link(struct net_device *dev)
@@ -1622,11 +2099,14 @@ static u32 mtk_get_link(struct net_device *dev)
 	struct mtk_mac *mac = netdev_priv(dev);
 	int err;
 
-	err = genphy_update_link(mac->phy_dev);
+	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
+		return -EBUSY;
+
+	err = genphy_update_link(dev->phydev);
 	if (err)
 		return ethtool_op_get_link(dev);
 
-	return mac->phy_dev->link;
+	return dev->phydev->link;
 }
 
 static void mtk_get_strings(struct net_device *dev, u32 stringset, u8 *data)
@@ -1662,6 +2142,9 @@ static void mtk_get_ethtool_stats(struct net_device *dev,
 	unsigned int start;
 	int i;
 
+	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
+		return;
+
 	if (netif_running(dev) && netif_device_present(dev)) {
 		if (spin_trylock(&hwstats->stats_lock)) {
 			mtk_stats_update_mac(mac);
@@ -1669,8 +2152,9 @@ static void mtk_get_ethtool_stats(struct net_device *dev,
 		}
 	}
 
+	data_src = (u64 *)hwstats;
+
 	do {
-		data_src = (u64*)hwstats;
 		data_dst = data;
 		start = u64_stats_fetch_begin_irq(&hwstats->syncp);
 
@@ -1679,9 +2163,65 @@ static void mtk_get_ethtool_stats(struct net_device *dev,
 	} while (u64_stats_fetch_retry_irq(&hwstats->syncp, start));
 }
 
-static struct ethtool_ops mtk_ethtool_ops = {
-	.get_settings		= mtk_get_settings,
-	.set_settings		= mtk_set_settings,
+static int mtk_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
+			 u32 *rule_locs)
+{
+	int ret = -EOPNOTSUPP;
+
+	switch (cmd->cmd) {
+	case ETHTOOL_GRXRINGS:
+		if (dev->features & NETIF_F_LRO) {
+			cmd->data = MTK_MAX_RX_RING_NUM;
+			ret = 0;
+		}
+		break;
+	case ETHTOOL_GRXCLSRLCNT:
+		if (dev->features & NETIF_F_LRO) {
+			struct mtk_mac *mac = netdev_priv(dev);
+
+			cmd->rule_cnt = mac->hwlro_ip_cnt;
+			ret = 0;
+		}
+		break;
+	case ETHTOOL_GRXCLSRULE:
+		if (dev->features & NETIF_F_LRO)
+			ret = mtk_hwlro_get_fdir_entry(dev, cmd);
+		break;
+	case ETHTOOL_GRXCLSRLALL:
+		if (dev->features & NETIF_F_LRO)
+			ret = mtk_hwlro_get_fdir_all(dev, cmd,
+						     rule_locs);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static int mtk_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
+{
+	int ret = -EOPNOTSUPP;
+
+	switch (cmd->cmd) {
+	case ETHTOOL_SRXCLSRLINS:
+		if (dev->features & NETIF_F_LRO)
+			ret = mtk_hwlro_add_ipaddr(dev, cmd);
+		break;
+	case ETHTOOL_SRXCLSRLDEL:
+		if (dev->features & NETIF_F_LRO)
+			ret = mtk_hwlro_del_ipaddr(dev, cmd);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static const struct ethtool_ops mtk_ethtool_ops = {
+	.get_link_ksettings	= mtk_get_link_ksettings,
+	.set_link_ksettings	= mtk_set_link_ksettings,
 	.get_drvinfo		= mtk_get_drvinfo,
 	.get_msglevel		= mtk_get_msglevel,
 	.set_msglevel		= mtk_set_msglevel,
@@ -1690,6 +2230,8 @@ static struct ethtool_ops mtk_ethtool_ops = {
 	.get_strings		= mtk_get_strings,
 	.get_sset_count		= mtk_get_sset_count,
 	.get_ethtool_stats	= mtk_get_ethtool_stats,
+	.get_rxnfc		= mtk_get_rxnfc,
+	.set_rxnfc              = mtk_set_rxnfc,
 };
 
 static const struct net_device_ops mtk_netdev_ops = {
@@ -1704,6 +2246,8 @@ static const struct net_device_ops mtk_netdev_ops = {
 	.ndo_change_mtu		= eth_change_mtu,
 	.ndo_tx_timeout		= mtk_tx_timeout,
 	.ndo_get_stats64        = mtk_get_stats64,
+	.ndo_fix_features	= mtk_fix_features,
+	.ndo_set_features	= mtk_set_features,
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller	= mtk_poll_controller,
 #endif
@@ -1742,6 +2286,9 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
 	mac->hw = eth;
 	mac->of_node = np;
 
+	memset(mac->hwlro_ip, 0, sizeof(mac->hwlro_ip));
+	mac->hwlro_ip_cnt = 0;
+
 	mac->hw_stats = devm_kzalloc(eth->dev,
 				     sizeof(*mac->hw_stats),
 				     GFP_KERNEL);
@@ -1751,27 +2298,24 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
 		goto free_netdev;
 	}
 	spin_lock_init(&mac->hw_stats->stats_lock);
+	u64_stats_init(&mac->hw_stats->syncp);
 	mac->hw_stats->reg_offset = id * MTK_STAT_OFFSET;
 
 	SET_NETDEV_DEV(eth->netdev[id], eth->dev);
 	eth->netdev[id]->watchdog_timeo = 5 * HZ;
 	eth->netdev[id]->netdev_ops = &mtk_netdev_ops;
 	eth->netdev[id]->base_addr = (unsigned long)eth->base;
+
+	eth->netdev[id]->hw_features = MTK_HW_FEATURES;
+	if (eth->hwlro)
+		eth->netdev[id]->hw_features |= NETIF_F_LRO;
+
 	eth->netdev[id]->vlan_features = MTK_HW_FEATURES &
 		~(NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX);
 	eth->netdev[id]->features |= MTK_HW_FEATURES;
 	eth->netdev[id]->ethtool_ops = &mtk_ethtool_ops;
 
-	err = register_netdev(eth->netdev[id]);
-	if (err) {
-		dev_err(eth->dev, "error bringing up device\n");
-		goto free_netdev;
-	}
 	eth->netdev[id]->irq = eth->irq[0];
-	netif_info(eth, probe, eth->netdev[id],
-		   "mediatek frame engine at 0x%08lx, irq %d\n",
-		   eth->netdev[id]->base_addr, eth->irq[0]);
-
 	return 0;
 
 free_netdev:
@@ -1779,6 +2323,41 @@ free_netdev:
 	return err;
 }
 
+static int mtk_get_chip_id(struct mtk_eth *eth, u32 *chip_id)
+{
+	u32 val[2], id[4];
+
+	regmap_read(eth->ethsys, ETHSYS_CHIPID0_3, &val[0]);
+	regmap_read(eth->ethsys, ETHSYS_CHIPID4_7, &val[1]);
+
+	id[3] = ((val[0] >> 16) & 0xff) - '0';
+	id[2] = ((val[0] >> 24) & 0xff) - '0';
+	id[1] = (val[1] & 0xff) - '0';
+	id[0] = ((val[1] >> 8) & 0xff) - '0';
+
+	*chip_id = (id[3] * 1000) + (id[2] * 100) +
+		   (id[1] * 10) + id[0];
+
+	if (!(*chip_id)) {
+		dev_err(eth->dev, "failed to get chip id\n");
+		return -ENODEV;
+	}
+
+	dev_info(eth->dev, "chip id = %d\n", *chip_id);
+
+	return 0;
+}
+
+static bool mtk_is_hwlro_supported(struct mtk_eth *eth)
+{
+	switch (eth->chip_id) {
+	case MT7623_ETH:
+		return true;
+	}
+
+	return false;
+}
+
 static int mtk_probe(struct platform_device *pdev)
 {
 	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -1796,6 +2375,7 @@ static int mtk_probe(struct platform_device *pdev)
 	if (!eth)
 		return -ENOMEM;
 
+	eth->dev = &pdev->dev;
 	eth->base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(eth->base))
 		return PTR_ERR(eth->base);
@@ -1817,12 +2397,6 @@ static int mtk_probe(struct platform_device *pdev)
 		return PTR_ERR(eth->pctl);
 	}
 
-	eth->rstc = devm_reset_control_get(&pdev->dev, "eth");
-	if (IS_ERR(eth->rstc)) {
-		dev_err(&pdev->dev, "no eth reset found\n");
-		return PTR_ERR(eth->rstc);
-	}
-
 	for (i = 0; i < 3; i++) {
 		eth->irq[i] = platform_get_irq(pdev, i);
 		if (eth->irq[i] < 0) {
@@ -1830,21 +2404,16 @@ static int mtk_probe(struct platform_device *pdev)
 			return -ENXIO;
 		}
 	}
+	for (i = 0; i < ARRAY_SIZE(eth->clks); i++) {
+		eth->clks[i] = devm_clk_get(eth->dev,
+					    mtk_clks_source_name[i]);
+		if (IS_ERR(eth->clks[i])) {
+			if (PTR_ERR(eth->clks[i]) == -EPROBE_DEFER)
+				return -EPROBE_DEFER;
+			return -ENODEV;
+		}
+	}
 
-	eth->clk_ethif = devm_clk_get(&pdev->dev, "ethif");
-	eth->clk_esw = devm_clk_get(&pdev->dev, "esw");
-	eth->clk_gp1 = devm_clk_get(&pdev->dev, "gp1");
-	eth->clk_gp2 = devm_clk_get(&pdev->dev, "gp2");
-	if (IS_ERR(eth->clk_esw) || IS_ERR(eth->clk_gp1) ||
-	    IS_ERR(eth->clk_gp2) || IS_ERR(eth->clk_ethif))
-		return -ENODEV;
-
-	clk_prepare_enable(eth->clk_ethif);
-	clk_prepare_enable(eth->clk_esw);
-	clk_prepare_enable(eth->clk_gp1);
-	clk_prepare_enable(eth->clk_gp2);
-
-	eth->dev = &pdev->dev;
 	eth->msg_enable = netif_msg_init(mtk_msg_level, MTK_DEFAULT_MSG_ENABLE);
 	INIT_WORK(&eth->pending_work, mtk_pending_work);
 
@@ -1852,6 +2421,12 @@ static int mtk_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
+	err = mtk_get_chip_id(eth, &eth->chip_id);
+	if (err)
+		return err;
+
+	eth->hwlro = mtk_is_hwlro_supported(eth);
+
 	for_each_child_of_node(pdev->dev.of_node, mac_np) {
 		if (!of_device_is_compatible(mac_np,
 					     "mediatek,eth-mac"))
@@ -1862,7 +2437,35 @@ static int mtk_probe(struct platform_device *pdev)
 
 		err = mtk_add_mac(eth, mac_np);
 		if (err)
-			goto err_free_dev;
+			goto err_deinit_hw;
+	}
+
+	err = devm_request_irq(eth->dev, eth->irq[1], mtk_handle_irq_tx, 0,
+			       dev_name(eth->dev), eth);
+	if (err)
+		goto err_free_dev;
+
+	err = devm_request_irq(eth->dev, eth->irq[2], mtk_handle_irq_rx, 0,
+			       dev_name(eth->dev), eth);
+	if (err)
+		goto err_free_dev;
+
+	err = mtk_mdio_init(eth);
+	if (err)
+		goto err_free_dev;
+
+	for (i = 0; i < MTK_MAX_DEVS; i++) {
+		if (!eth->netdev[i])
+			continue;
+
+		err = register_netdev(eth->netdev[i]);
+		if (err) {
+			dev_err(eth->dev, "error bringing up device\n");
+			goto err_deinit_mdio;
+		} else
+			netif_info(eth, probe, eth->netdev[i],
+				   "mediatek frame engine at 0x%08lx, irq %d\n",
+				   eth->netdev[i]->base_addr, eth->irq[0]);
 	}
 
 	/* we run 2 devices on the same DMA ring so we need a dummy device
@@ -1878,24 +2481,34 @@ static int mtk_probe(struct platform_device *pdev)
 
 	return 0;
 
+err_deinit_mdio:
+	mtk_mdio_cleanup(eth);
 err_free_dev:
-	mtk_cleanup(eth);
+	mtk_free_dev(eth);
+err_deinit_hw:
+	mtk_hw_deinit(eth);
+
 	return err;
 }
 
 static int mtk_remove(struct platform_device *pdev)
 {
 	struct mtk_eth *eth = platform_get_drvdata(pdev);
+	int i;
+
+	/* stop all devices to make sure that dma is properly shut down */
+	for (i = 0; i < MTK_MAC_COUNT; i++) {
+		if (!eth->netdev[i])
+			continue;
+		mtk_stop(eth->netdev[i]);
+	}
 
-	clk_disable_unprepare(eth->clk_ethif);
-	clk_disable_unprepare(eth->clk_esw);
-	clk_disable_unprepare(eth->clk_gp1);
-	clk_disable_unprepare(eth->clk_gp2);
+	mtk_hw_deinit(eth);
 
 	netif_napi_del(&eth->tx_napi);
 	netif_napi_del(&eth->rx_napi);
 	mtk_cleanup(eth);
-	platform_set_drvdata(pdev, NULL);
+	mtk_mdio_cleanup(eth);
 
 	return 0;
 }
@@ -1904,6 +2517,7 @@ const struct of_device_id of_mtk_match[] = {
 	{ .compatible = "mediatek,mt7623-eth" },
 	{},
 };
+MODULE_DEVICE_TABLE(of, of_mtk_match);
 
 static struct platform_driver mtk_driver = {
 	.probe = mtk_probe,
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
index f82e3acb947b..99b1c8e9f16f 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
@@ -39,7 +39,21 @@
 				 NETIF_F_SG | NETIF_F_TSO | \
 				 NETIF_F_TSO6 | \
 				 NETIF_F_IPV6_CSUM)
-#define NEXT_RX_DESP_IDX(X)	(((X) + 1) & (MTK_DMA_SIZE - 1))
+#define NEXT_RX_DESP_IDX(X, Y)	(((X) + 1) & ((Y) - 1))
+
+#define MTK_MAX_RX_RING_NUM	4
+#define MTK_HW_LRO_DMA_SIZE	8
+
+#define	MTK_MAX_LRO_RX_LENGTH		(4096 * 3)
+#define	MTK_MAX_LRO_IP_CNT		2
+#define	MTK_HW_LRO_TIMER_UNIT		1	/* 20 us */
+#define	MTK_HW_LRO_REFRESH_TIME		50000	/* 1 sec. */
+#define	MTK_HW_LRO_AGG_TIME		10	/* 200us */
+#define	MTK_HW_LRO_AGE_TIME		50	/* 1ms */
+#define	MTK_HW_LRO_MAX_AGG_CNT		64
+#define	MTK_HW_LRO_BW_THRE		3000
+#define	MTK_HW_LRO_REPLACE_DELTA	1000
+#define	MTK_HW_LRO_SDL_REMAIN_ROOM	1522
 
 /* Frame Engine Global Reset Register */
 #define MTK_RST_GL		0x04
@@ -50,6 +64,9 @@
 #define MTK_GDM1_AF		BIT(28)
 #define MTK_GDM2_AF		BIT(29)
 
+/* PDMA HW LRO Alter Flow Timer Register */
+#define MTK_PDMA_LRO_ALT_REFRESH_TIMER	0x1c
+
 /* Frame Engine Interrupt Grouping Register */
 #define MTK_FE_INT_GRP		0x20
 
@@ -68,10 +85,77 @@
 /* Unicast Filter MAC Address Register - High */
 #define MTK_GDMA_MAC_ADRH(x)	(0x50C + (x * 0x1000))
 
+/* PDMA RX Base Pointer Register */
+#define MTK_PRX_BASE_PTR0	0x900
+#define MTK_PRX_BASE_PTR_CFG(x)	(MTK_PRX_BASE_PTR0 + (x * 0x10))
+
+/* PDMA RX Maximum Count Register */
+#define MTK_PRX_MAX_CNT0	0x904
+#define MTK_PRX_MAX_CNT_CFG(x)	(MTK_PRX_MAX_CNT0 + (x * 0x10))
+
+/* PDMA RX CPU Pointer Register */
+#define MTK_PRX_CRX_IDX0	0x908
+#define MTK_PRX_CRX_IDX_CFG(x)	(MTK_PRX_CRX_IDX0 + (x * 0x10))
+
+/* PDMA HW LRO Control Registers */
+#define MTK_PDMA_LRO_CTRL_DW0	0x980
+#define MTK_LRO_EN			BIT(0)
+#define MTK_L3_CKS_UPD_EN		BIT(7)
+#define MTK_LRO_ALT_PKT_CNT_MODE	BIT(21)
+#define MTK_LRO_RING_RELINQUISH_REQ	(0x7 << 26)
+#define MTK_LRO_RING_RELINQUISH_DONE	(0x7 << 29)
+
+#define MTK_PDMA_LRO_CTRL_DW1	0x984
+#define MTK_PDMA_LRO_CTRL_DW2	0x988
+#define MTK_PDMA_LRO_CTRL_DW3	0x98c
+#define MTK_ADMA_MODE		BIT(15)
+#define MTK_LRO_MIN_RXD_SDL	(MTK_HW_LRO_SDL_REMAIN_ROOM << 16)
+
+/* PDMA Global Configuration Register */
+#define MTK_PDMA_GLO_CFG	0xa04
+#define MTK_MULTI_EN		BIT(10)
+
+/* PDMA Reset Index Register */
+#define MTK_PDMA_RST_IDX	0xa08
+#define MTK_PST_DRX_IDX0	BIT(16)
+#define MTK_PST_DRX_IDX_CFG(x)	(MTK_PST_DRX_IDX0 << (x))
+
+/* PDMA Delay Interrupt Register */
+#define MTK_PDMA_DELAY_INT	0xa0c
+
+/* PDMA Interrupt Status Register */
+#define MTK_PDMA_INT_STATUS	0xa20
+
+/* PDMA Interrupt Mask Register */
+#define MTK_PDMA_INT_MASK	0xa28
+
+/* PDMA HW LRO Alter Flow Delta Register */
+#define MTK_PDMA_LRO_ALT_SCORE_DELTA	0xa4c
+
 /* PDMA Interrupt grouping registers */
 #define MTK_PDMA_INT_GRP1	0xa50
 #define MTK_PDMA_INT_GRP2	0xa54
 
+/* PDMA HW LRO IP Setting Registers */
+#define MTK_LRO_RX_RING0_DIP_DW0	0xb04
+#define MTK_LRO_DIP_DW0_CFG(x)		(MTK_LRO_RX_RING0_DIP_DW0 + (x * 0x40))
+#define MTK_RING_MYIP_VLD		BIT(9)
+
+/* PDMA HW LRO Ring Control Registers */
+#define MTK_LRO_RX_RING0_CTRL_DW1	0xb28
+#define MTK_LRO_RX_RING0_CTRL_DW2	0xb2c
+#define MTK_LRO_RX_RING0_CTRL_DW3	0xb30
+#define MTK_LRO_CTRL_DW1_CFG(x)		(MTK_LRO_RX_RING0_CTRL_DW1 + (x * 0x40))
+#define MTK_LRO_CTRL_DW2_CFG(x)		(MTK_LRO_RX_RING0_CTRL_DW2 + (x * 0x40))
+#define MTK_LRO_CTRL_DW3_CFG(x)		(MTK_LRO_RX_RING0_CTRL_DW3 + (x * 0x40))
+#define MTK_RING_AGE_TIME_L		((MTK_HW_LRO_AGE_TIME & 0x3ff) << 22)
+#define MTK_RING_AGE_TIME_H		((MTK_HW_LRO_AGE_TIME >> 10) & 0x3f)
+#define MTK_RING_AUTO_LERAN_MODE	(3 << 6)
+#define MTK_RING_VLD			BIT(8)
+#define MTK_RING_MAX_AGG_TIME		((MTK_HW_LRO_AGG_TIME & 0xffff) << 10)
+#define MTK_RING_MAX_AGG_CNT_L		((MTK_HW_LRO_MAX_AGG_CNT & 0x3f) << 26)
+#define MTK_RING_MAX_AGG_CNT_H		((MTK_HW_LRO_MAX_AGG_CNT >> 6) & 0x3)
+
 /* QDMA TX Queue Configuration Registers */
 #define MTK_QTX_CFG(x)		(0x1800 + (x * 0x10))
 #define QDMA_RES_THRES		4
@@ -106,7 +190,6 @@
 
 /* QDMA Reset Index Register */
 #define MTK_QDMA_RST_IDX	0x1A08
-#define MTK_PST_DRX_IDX0	BIT(16)
 
 /* QDMA Delay Interrupt Register */
 #define MTK_QDMA_DELAY_INT	0x1A0C
@@ -119,13 +202,16 @@
 
 /* QDMA Interrupt Status Register */
 #define MTK_QMTK_INT_STATUS	0x1A18
+#define MTK_RX_DONE_INT3	BIT(19)
+#define MTK_RX_DONE_INT2	BIT(18)
 #define MTK_RX_DONE_INT1	BIT(17)
 #define MTK_RX_DONE_INT0	BIT(16)
 #define MTK_TX_DONE_INT3	BIT(3)
 #define MTK_TX_DONE_INT2	BIT(2)
 #define MTK_TX_DONE_INT1	BIT(1)
 #define MTK_TX_DONE_INT0	BIT(0)
-#define MTK_RX_DONE_INT		(MTK_RX_DONE_INT0 | MTK_RX_DONE_INT1)
+#define MTK_RX_DONE_INT		(MTK_RX_DONE_INT0 | MTK_RX_DONE_INT1 | \
+				 MTK_RX_DONE_INT2 | MTK_RX_DONE_INT3)
 #define MTK_TX_DONE_INT		(MTK_TX_DONE_INT0 | MTK_TX_DONE_INT1 | \
 				 MTK_TX_DONE_INT2 | MTK_TX_DONE_INT3)
 
@@ -227,16 +313,54 @@
 				 MAC_MCR_FORCE_TX_FC | MAC_MCR_SPEED_1000 | \
 				 MAC_MCR_FORCE_DPX | MAC_MCR_FORCE_LINK)
 
+/* TRGMII RXC control register */
+#define TRGMII_RCK_CTRL		0x10300
+#define DQSI0(x)		((x << 0) & GENMASK(6, 0))
+#define DQSI1(x)		((x << 8) & GENMASK(14, 8))
+#define RXCTL_DMWTLAT(x)	((x << 16) & GENMASK(18, 16))
+#define RXC_DQSISEL		BIT(30)
+#define RCK_CTRL_RGMII_1000	(RXC_DQSISEL | RXCTL_DMWTLAT(2) | DQSI1(16))
+#define RCK_CTRL_RGMII_10_100	RXCTL_DMWTLAT(2)
+
+/* TRGMII RXC control register */
+#define TRGMII_TCK_CTRL		0x10340
+#define TXCTL_DMWTLAT(x)	((x << 16) & GENMASK(18, 16))
+#define TXC_INV			BIT(30)
+#define TCK_CTRL_RGMII_1000	TXCTL_DMWTLAT(2)
+#define TCK_CTRL_RGMII_10_100	(TXC_INV | TXCTL_DMWTLAT(2))
+
+/* TRGMII Interface mode register */
+#define INTF_MODE		0x10390
+#define TRGMII_INTF_DIS		BIT(0)
+#define TRGMII_MODE		BIT(1)
+#define TRGMII_CENTRAL_ALIGNED	BIT(2)
+#define INTF_MODE_RGMII_1000    (TRGMII_MODE | TRGMII_CENTRAL_ALIGNED)
+#define INTF_MODE_RGMII_10_100  0
+
 /* GPIO port control registers for GMAC 2*/
 #define GPIO_OD33_CTRL8		0x4c0
 #define GPIO_BIAS_CTRL		0xed0
 #define GPIO_DRV_SEL10		0xf00
 
+/* ethernet subsystem chip id register */
+#define ETHSYS_CHIPID0_3	0x0
+#define ETHSYS_CHIPID4_7	0x4
+#define MT7623_ETH		7623
+
 /* ethernet subsystem config register */
 #define ETHSYS_SYSCFG0		0x14
 #define SYSCFG0_GE_MASK		0x3
 #define SYSCFG0_GE_MODE(x, y)	(x << (12 + (y * 2)))
 
+/* ethernet subsystem clock register */
+#define ETHSYS_CLKCFG0		0x2c
+#define ETHSYS_TRGMII_CLK_SEL362_5	BIT(11)
+
+/* ethernet reset control register */
+#define ETHSYS_RSTCTRL		0x34
+#define RSTCTRL_FE		BIT(6)
+#define RSTCTRL_PPE		BIT(31)
+
 struct mtk_rx_dma {
 	unsigned int rxd1;
 	unsigned int rxd2;
@@ -290,6 +414,23 @@ enum mtk_tx_flags {
 	MTK_TX_FLAGS_PAGE0	= 0x02,
 };
 
+/* This enum allows us to identify how the clock is defined on the array of the
+ * clock in the order
+ */
+enum mtk_clks_map {
+	MTK_CLK_ETHIF,
+	MTK_CLK_ESW,
+	MTK_CLK_GP1,
+	MTK_CLK_GP2,
+	MTK_CLK_TRGPLL,
+	MTK_CLK_MAX
+};
+
+enum mtk_dev_state {
+	MTK_HW_INIT,
+	MTK_RESETTING
+};
+
 /* struct mtk_tx_buf -	This struct holds the pointers to the memory pointed at
  *			by the TX descriptor	s
  * @skb:		The SKB pointer of the packet being sent
@@ -327,6 +468,12 @@ struct mtk_tx_ring {
 	atomic_t free_count;
 };
 
+/* PDMA rx ring mode */
+enum mtk_rx_flags {
+	MTK_RX_FLAGS_NORMAL = 0,
+	MTK_RX_FLAGS_HWLRO,
+};
+
 /* struct mtk_rx_ring -	This struct holds info describing a RX ring
  * @dma:		The descriptor ring
  * @data:		The memory pointed at by the ring
@@ -341,7 +488,10 @@ struct mtk_rx_ring {
 	dma_addr_t phys;
 	u16 frag_size;
 	u16 buf_size;
+	u16 dma_size;
+	bool calc_idx_update;
 	u16 calc_idx;
+	u32 crx_idx_reg;
 };
 
 /* currently no SoC has more than 2 macs */
@@ -370,18 +520,15 @@ struct mtk_rx_ring {
  * @scratch_ring:	Newer SoCs need memory for a second HW managed TX ring
  * @phy_scratch_ring:	physical address of scratch_ring
  * @scratch_head:	The scratch memory that scratch_ring points to.
- * @clk_ethif:		The ethif clock
- * @clk_esw:		The switch clock
- * @clk_gp1:		The gmac1 clock
- * @clk_gp2:		The gmac2 clock
+ * @clks:		clock array for all clocks required
  * @mii_bus:		If there is a bus we need to create an instance for it
  * @pending_work:	The workqueue used to reset the dma ring
+ * @state               Initialization and runtime state of the device.
  */
 
 struct mtk_eth {
 	struct device			*dev;
 	void __iomem			*base;
-	struct reset_control		*rstc;
 	spinlock_t			page_lock;
 	spinlock_t			irq_lock;
 	struct net_device		dummy_dev;
@@ -392,36 +539,42 @@ struct mtk_eth {
 	unsigned long			sysclk;
 	struct regmap			*ethsys;
 	struct regmap			*pctl;
+	u32				chip_id;
+	bool				hwlro;
 	atomic_t			dma_refcnt;
 	struct mtk_tx_ring		tx_ring;
-	struct mtk_rx_ring		rx_ring;
+	struct mtk_rx_ring		rx_ring[MTK_MAX_RX_RING_NUM];
 	struct napi_struct		tx_napi;
 	struct napi_struct		rx_napi;
 	struct mtk_tx_dma		*scratch_ring;
 	dma_addr_t			phy_scratch_ring;
 	void				*scratch_head;
-	struct clk			*clk_ethif;
-	struct clk			*clk_esw;
-	struct clk			*clk_gp1;
-	struct clk			*clk_gp2;
+	struct clk			*clks[MTK_CLK_MAX];
+
 	struct mii_bus			*mii_bus;
 	struct work_struct		pending_work;
+	unsigned long			state;
 };
 
 /* struct mtk_mac -	the structure that holds the info about the MACs of the
  *			SoC
  * @id:			The number of the MAC
+ * @ge_mode:            Interface mode kept for setup restoring
  * @of_node:		Our devicetree node
  * @hw:			Backpointer to our main datastruture
  * @hw_stats:		Packet statistics counter
- * @phy_dev:		The attached PHY if available
+ * @trgmii		Indicate if the MAC uses TRGMII connected to internal
+			switch
  */
 struct mtk_mac {
 	int				id;
+	int				ge_mode;
 	struct device_node		*of_node;
 	struct mtk_eth			*hw;
 	struct mtk_hw_stats		*hw_stats;
-	struct phy_device		*phy_dev;
+	__be32				hwlro_ip[MTK_MAX_LRO_IP_CNT];
+	int				hwlro_ip_cnt;
+	bool				trgmii;
 };
 
 /* the struct describing the SoC. these are declared in the soc_xyz.c files */
diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c
index f04a423ff79d..b1cef7a0f7ca 100644
--- a/drivers/net/ethernet/mellanox/mlx4/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c
@@ -785,17 +785,23 @@ int __mlx4_cmd(struct mlx4_dev *dev, u64 in_param, u64 *out_param,
 		return mlx4_cmd_reset_flow(dev, op, op_modifier, -EIO);
 
 	if (!mlx4_is_mfunc(dev) || (native && mlx4_is_master(dev))) {
+		int ret;
+
 		if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)
 			return mlx4_internal_err_ret_value(dev, op,
 							  op_modifier);
+		down_read(&mlx4_priv(dev)->cmd.switch_sem);
 		if (mlx4_priv(dev)->cmd.use_events)
-			return mlx4_cmd_wait(dev, in_param, out_param,
-					     out_is_imm, in_modifier,
-					     op_modifier, op, timeout);
+			ret = mlx4_cmd_wait(dev, in_param, out_param,
+					    out_is_imm, in_modifier,
+					    op_modifier, op, timeout);
 		else
-			return mlx4_cmd_poll(dev, in_param, out_param,
-					     out_is_imm, in_modifier,
-					     op_modifier, op, timeout);
+			ret = mlx4_cmd_poll(dev, in_param, out_param,
+					    out_is_imm, in_modifier,
+					    op_modifier, op, timeout);
+
+		up_read(&mlx4_priv(dev)->cmd.switch_sem);
+		return ret;
 	}
 	return mlx4_slave_cmd(dev, in_param, out_param, out_is_imm,
 			      in_modifier, op_modifier, op, timeout);
@@ -1845,6 +1851,7 @@ static int mlx4_master_immediate_activate_vlan_qos(struct mlx4_priv *priv,
 
 	if (vp_oper->state.default_vlan == vp_admin->default_vlan &&
 	    vp_oper->state.default_qos == vp_admin->default_qos &&
+	    vp_oper->state.vlan_proto == vp_admin->vlan_proto &&
 	    vp_oper->state.link_state == vp_admin->link_state &&
 	    vp_oper->state.qos_vport == vp_admin->qos_vport)
 		return 0;
@@ -1903,6 +1910,7 @@ static int mlx4_master_immediate_activate_vlan_qos(struct mlx4_priv *priv,
 
 	vp_oper->state.default_vlan = vp_admin->default_vlan;
 	vp_oper->state.default_qos = vp_admin->default_qos;
+	vp_oper->state.vlan_proto = vp_admin->vlan_proto;
 	vp_oper->state.link_state = vp_admin->link_state;
 	vp_oper->state.qos_vport = vp_admin->qos_vport;
 
@@ -1916,6 +1924,7 @@ static int mlx4_master_immediate_activate_vlan_qos(struct mlx4_priv *priv,
 	work->qos_vport = vp_oper->state.qos_vport;
 	work->vlan_id = vp_oper->state.default_vlan;
 	work->vlan_ix = vp_oper->vlan_idx;
+	work->vlan_proto = vp_oper->state.vlan_proto;
 	work->priv = priv;
 	INIT_WORK(&work->work, mlx4_vf_immed_vlan_work_handler);
 	queue_work(priv->mfunc.master.comm_wq, &work->work);
@@ -1986,6 +1995,8 @@ static int mlx4_master_activate_admin_state(struct mlx4_priv *priv, int slave)
 	int port, err;
 	struct mlx4_vport_state *vp_admin;
 	struct mlx4_vport_oper_state *vp_oper;
+	struct mlx4_slave_state *slave_state =
+		&priv->mfunc.master.slave_state[slave];
 	struct mlx4_active_ports actv_ports = mlx4_get_active_ports(
 			&priv->dev, slave);
 	int min_port = find_first_bit(actv_ports.ports,
@@ -2000,12 +2011,26 @@ static int mlx4_master_activate_admin_state(struct mlx4_priv *priv, int slave)
 			priv->mfunc.master.vf_admin[slave].enable_smi[port];
 		vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port];
 		vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port];
-		vp_oper->state = *vp_admin;
+		if (vp_admin->vlan_proto != htons(ETH_P_8021AD) ||
+		    slave_state->vst_qinq_supported) {
+			vp_oper->state.vlan_proto   = vp_admin->vlan_proto;
+			vp_oper->state.default_vlan = vp_admin->default_vlan;
+			vp_oper->state.default_qos  = vp_admin->default_qos;
+		}
+		vp_oper->state.link_state = vp_admin->link_state;
+		vp_oper->state.mac        = vp_admin->mac;
+		vp_oper->state.spoofchk   = vp_admin->spoofchk;
+		vp_oper->state.tx_rate    = vp_admin->tx_rate;
+		vp_oper->state.qos_vport  = vp_admin->qos_vport;
+		vp_oper->state.guid       = vp_admin->guid;
+
 		if (MLX4_VGT != vp_admin->default_vlan) {
 			err = __mlx4_register_vlan(&priv->dev, port,
 						   vp_admin->default_vlan, &(vp_oper->vlan_idx));
 			if (err) {
 				vp_oper->vlan_idx = NO_INDX;
+				vp_oper->state.default_vlan = MLX4_VGT;
+				vp_oper->state.vlan_proto = htons(ETH_P_8021Q);
 				mlx4_warn(&priv->dev,
 					  "No vlan resources slave %d, port %d\n",
 					  slave, port);
@@ -2086,6 +2111,7 @@ static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd,
 		mlx4_warn(dev, "Received reset from slave:%d\n", slave);
 		slave_state[slave].active = false;
 		slave_state[slave].old_vlan_api = false;
+		slave_state[slave].vst_qinq_supported = false;
 		mlx4_master_deactivate_admin_state(priv, slave);
 		for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) {
 				slave_state[slave].event_eq[i].eqn = -1;
@@ -2353,6 +2379,7 @@ int mlx4_multi_func_init(struct mlx4_dev *dev)
 			vf_oper = &priv->mfunc.master.vf_oper[i];
 			s_state = &priv->mfunc.master.slave_state[i];
 			s_state->last_cmd = MLX4_COMM_CMD_RESET;
+			s_state->vst_qinq_supported = false;
 			mutex_init(&priv->mfunc.master.gen_eqe_mutex[i]);
 			for (j = 0; j < MLX4_EVENT_TYPES_NUM; ++j)
 				s_state->event_eq[j].eqn = -1;
@@ -2382,6 +2409,8 @@ int mlx4_multi_func_init(struct mlx4_dev *dev)
 				admin_vport->qos_vport =
 						MLX4_VPP_DEFAULT_VPORT;
 				oper_vport->qos_vport = MLX4_VPP_DEFAULT_VPORT;
+				admin_vport->vlan_proto = htons(ETH_P_8021Q);
+				oper_vport->vlan_proto = htons(ETH_P_8021Q);
 				vf_oper->vport[port].vlan_idx = NO_INDX;
 				vf_oper->vport[port].mac_idx = NO_INDX;
 				mlx4_set_random_admin_guid(dev, i, port);
@@ -2454,6 +2483,7 @@ int mlx4_cmd_init(struct mlx4_dev *dev)
 	int flags = 0;
 
 	if (!priv->cmd.initialized) {
+		init_rwsem(&priv->cmd.switch_sem);
 		mutex_init(&priv->cmd.slave_cmd_mutex);
 		sema_init(&priv->cmd.poll_sem, 1);
 		priv->cmd.use_events = 0;
@@ -2583,6 +2613,7 @@ int mlx4_cmd_use_events(struct mlx4_dev *dev)
 	if (!priv->cmd.context)
 		return -ENOMEM;
 
+	down_write(&priv->cmd.switch_sem);
 	for (i = 0; i < priv->cmd.max_cmds; ++i) {
 		priv->cmd.context[i].token = i;
 		priv->cmd.context[i].next  = i + 1;
@@ -2606,6 +2637,7 @@ int mlx4_cmd_use_events(struct mlx4_dev *dev)
 
 	down(&priv->cmd.poll_sem);
 	priv->cmd.use_events = 1;
+	up_write(&priv->cmd.switch_sem);
 
 	return err;
 }
@@ -2618,6 +2650,7 @@ void mlx4_cmd_use_polling(struct mlx4_dev *dev)
 	struct mlx4_priv *priv = mlx4_priv(dev);
 	int i;
 
+	down_write(&priv->cmd.switch_sem);
 	priv->cmd.use_events = 0;
 
 	for (i = 0; i < priv->cmd.max_cmds; ++i)
@@ -2626,6 +2659,7 @@ void mlx4_cmd_use_polling(struct mlx4_dev *dev)
 	kfree(priv->cmd.context);
 
 	up(&priv->cmd.poll_sem);
+	up_write(&priv->cmd.switch_sem);
 }
 
 struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev)
@@ -2937,10 +2971,13 @@ int mlx4_set_vf_mac(struct mlx4_dev *dev, int port, int vf, u64 mac)
 EXPORT_SYMBOL_GPL(mlx4_set_vf_mac);
 
 
-int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos)
+int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos,
+		     __be16 proto)
 {
 	struct mlx4_priv *priv = mlx4_priv(dev);
 	struct mlx4_vport_state *vf_admin;
+	struct mlx4_slave_state *slave_state;
+	struct mlx4_vport_oper_state *vf_oper;
 	int slave;
 
 	if ((!mlx4_is_master(dev)) ||
@@ -2950,12 +2987,31 @@ int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos)
 	if ((vlan > 4095) || (qos > 7))
 		return -EINVAL;
 
+	if (proto == htons(ETH_P_8021AD) &&
+	    !(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SVLAN_BY_QP))
+		return -EPROTONOSUPPORT;
+
+	if (proto != htons(ETH_P_8021Q) &&
+	    proto != htons(ETH_P_8021AD))
+		return -EINVAL;
+
+	if ((proto == htons(ETH_P_8021AD)) &&
+	    ((vlan == 0) || (vlan == MLX4_VGT)))
+		return -EINVAL;
+
 	slave = mlx4_get_slave_indx(dev, vf);
 	if (slave < 0)
 		return -EINVAL;
 
+	slave_state = &priv->mfunc.master.slave_state[slave];
+	if ((proto == htons(ETH_P_8021AD)) && (slave_state->active) &&
+	    (!slave_state->vst_qinq_supported)) {
+		mlx4_err(dev, "vf %d does not support VST QinQ mode\n", vf);
+		return -EPROTONOSUPPORT;
+	}
 	port = mlx4_slaves_closest_port(dev, slave, port);
 	vf_admin = &priv->mfunc.master.vf_admin[slave].vport[port];
+	vf_oper = &priv->mfunc.master.vf_oper[slave].vport[port];
 
 	if (!mlx4_valid_vf_state_change(dev, port, vf_admin, vlan, qos))
 		return -EPERM;
@@ -2965,6 +3021,7 @@ int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos)
 	else
 		vf_admin->default_vlan = vlan;
 	vf_admin->default_qos = qos;
+	vf_admin->vlan_proto = proto;
 
 	/* If rate was configured prior to VST, we saved the configured rate
 	 * in vf_admin->rate and now, if priority supported we enforce the QoS
@@ -2973,7 +3030,12 @@ int mlx4_set_vf_vlan(struct mlx4_dev *dev, int port, int vf, u16 vlan, u8 qos)
 	    vf_admin->tx_rate)
 		vf_admin->qos_vport = slave;
 
-	if (mlx4_master_immediate_activate_vlan_qos(priv, slave, port))
+	/* Try to activate new vf state without restart,
+	 * this option is not supported while moving to VST QinQ mode.
+	 */
+	if ((proto == htons(ETH_P_8021AD) &&
+	     vf_oper->state.vlan_proto != proto) ||
+	    mlx4_master_immediate_activate_vlan_qos(priv, slave, port))
 		mlx4_info(dev,
 			  "updating vf %d port %d config will take effect on next VF restart\n",
 			  vf, port);
@@ -3117,6 +3179,7 @@ int mlx4_get_vf_config(struct mlx4_dev *dev, int port, int vf, struct ifla_vf_in
 
 	ivf->vlan		= s_info->default_vlan;
 	ivf->qos		= s_info->default_qos;
+	ivf->vlan_proto		= s_info->vlan_proto;
 
 	if (mlx4_is_vf_vst_and_prio_qos(dev, port, s_info))
 		ivf->max_tx_rate = s_info->tx_rate;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_clock.c b/drivers/net/ethernet/mellanox/mlx4/en_clock.c
index 1494997c4f7e..08fc5fc56d43 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_clock.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_clock.c
@@ -298,7 +298,7 @@ void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev)
 	if (IS_ERR(mdev->ptp_clock)) {
 		mdev->ptp_clock = NULL;
 		mlx4_err(mdev, "ptp_clock_register failed\n");
-	} else {
+	} else if (mdev->ptp_clock) {
 		mlx4_info(mdev, "registered PHC clock\n");
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c b/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c
index 99c6bbdff501..b04760a5034b 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c
@@ -94,7 +94,7 @@ static u8 mlx4_en_dcbnl_getcap(struct net_device *dev, int capid, u8 *cap)
 		*cap = true;
 		break;
 	case DCB_CAP_ATTR_DCBX:
-		*cap = priv->cee_params.dcbx_cap;
+		*cap = priv->dcbx_cap;
 		break;
 	case DCB_CAP_ATTR_PFC_TCS:
 		*cap = 1 <<  mlx4_max_tc(priv->mdev->dev);
@@ -111,14 +111,14 @@ static u8 mlx4_en_dcbnl_getpfcstate(struct net_device *netdev)
 {
 	struct mlx4_en_priv *priv = netdev_priv(netdev);
 
-	return priv->cee_params.dcb_cfg.pfc_state;
+	return priv->cee_config.pfc_state;
 }
 
 static void mlx4_en_dcbnl_setpfcstate(struct net_device *netdev, u8 state)
 {
 	struct mlx4_en_priv *priv = netdev_priv(netdev);
 
-	priv->cee_params.dcb_cfg.pfc_state = state;
+	priv->cee_config.pfc_state = state;
 }
 
 static void mlx4_en_dcbnl_get_pfc_cfg(struct net_device *netdev, int priority,
@@ -126,7 +126,7 @@ static void mlx4_en_dcbnl_get_pfc_cfg(struct net_device *netdev, int priority,
 {
 	struct mlx4_en_priv *priv = netdev_priv(netdev);
 
-	*setting = priv->cee_params.dcb_cfg.tc_config[priority].dcb_pfc;
+	*setting = priv->cee_config.dcb_pfc[priority];
 }
 
 static void mlx4_en_dcbnl_set_pfc_cfg(struct net_device *netdev, int priority,
@@ -134,8 +134,8 @@ static void mlx4_en_dcbnl_set_pfc_cfg(struct net_device *netdev, int priority,
 {
 	struct mlx4_en_priv *priv = netdev_priv(netdev);
 
-	priv->cee_params.dcb_cfg.tc_config[priority].dcb_pfc = setting;
-	priv->cee_params.dcb_cfg.pfc_state = true;
+	priv->cee_config.dcb_pfc[priority] = setting;
+	priv->cee_config.pfc_state = true;
 }
 
 static int mlx4_en_dcbnl_getnumtcs(struct net_device *netdev, int tcid, u8 *num)
@@ -157,13 +157,11 @@ static u8 mlx4_en_dcbnl_set_all(struct net_device *netdev)
 {
 	struct mlx4_en_priv *priv = netdev_priv(netdev);
 	struct mlx4_en_dev *mdev = priv->mdev;
-	struct mlx4_en_cee_config *dcb_cfg = &priv->cee_params.dcb_cfg;
-	int err = 0;
 
-	if (!(priv->cee_params.dcbx_cap & DCB_CAP_DCBX_VER_CEE))
-		return -EINVAL;
+	if (!(priv->dcbx_cap & DCB_CAP_DCBX_VER_CEE))
+		return 1;
 
-	if (dcb_cfg->pfc_state) {
+	if (priv->cee_config.pfc_state) {
 		int tc;
 
 		priv->prof->rx_pause = 0;
@@ -171,7 +169,7 @@ static u8 mlx4_en_dcbnl_set_all(struct net_device *netdev)
 		for (tc = 0; tc < CEE_DCBX_MAX_PRIO; tc++) {
 			u8 tc_mask = 1 << tc;
 
-			switch (dcb_cfg->tc_config[tc].dcb_pfc) {
+			switch (priv->cee_config.dcb_pfc[tc]) {
 			case pfc_disabled:
 				priv->prof->tx_ppp &= ~tc_mask;
 				priv->prof->rx_ppp &= ~tc_mask;
@@ -199,15 +197,17 @@ static u8 mlx4_en_dcbnl_set_all(struct net_device *netdev)
 		en_dbg(DRV, priv, "Set pfc off\n");
 	}
 
-	err = mlx4_SET_PORT_general(mdev->dev, priv->port,
-				    priv->rx_skb_size + ETH_FCS_LEN,
-				    priv->prof->tx_pause,
-				    priv->prof->tx_ppp,
-				    priv->prof->rx_pause,
-				    priv->prof->rx_ppp);
-	if (err)
+	if (mlx4_SET_PORT_general(mdev->dev, priv->port,
+				  priv->rx_skb_size + ETH_FCS_LEN,
+				  priv->prof->tx_pause,
+				  priv->prof->tx_ppp,
+				  priv->prof->rx_pause,
+				  priv->prof->rx_ppp)) {
 		en_err(priv, "Failed setting pause params\n");
-	return err;
+		return 1;
+	}
+
+	return 0;
 }
 
 static u8 mlx4_en_dcbnl_get_state(struct net_device *dev)
@@ -225,7 +225,7 @@ static u8 mlx4_en_dcbnl_set_state(struct net_device *dev, u8 state)
 	struct mlx4_en_priv *priv = netdev_priv(dev);
 	int num_tcs = 0;
 
-	if (!(priv->cee_params.dcbx_cap & DCB_CAP_DCBX_VER_CEE))
+	if (!(priv->dcbx_cap & DCB_CAP_DCBX_VER_CEE))
 		return 1;
 
 	if (!!(state) == !!(priv->flags & MLX4_EN_FLAG_DCB_ENABLED))
@@ -238,7 +238,10 @@ static u8 mlx4_en_dcbnl_set_state(struct net_device *dev, u8 state)
 		priv->flags &= ~MLX4_EN_FLAG_DCB_ENABLED;
 	}
 
-	return mlx4_en_setup_tc(dev, num_tcs);
+	if (mlx4_en_setup_tc(dev, num_tcs))
+		return 1;
+
+	return 0;
 }
 
 /* On success returns a non-zero 802.1p user priority bitmap
@@ -252,7 +255,7 @@ static int mlx4_en_dcbnl_getapp(struct net_device *netdev, u8 idtype, u16 id)
 				.selector = idtype,
 				.protocol = id,
 			     };
-	if (!(priv->cee_params.dcbx_cap & DCB_CAP_DCBX_VER_CEE))
+	if (!(priv->dcbx_cap & DCB_CAP_DCBX_VER_CEE))
 		return 0;
 
 	return dcb_getapp(netdev, &app);
@@ -264,7 +267,7 @@ static int mlx4_en_dcbnl_setapp(struct net_device *netdev, u8 idtype,
 	struct mlx4_en_priv *priv = netdev_priv(netdev);
 	struct dcb_app app;
 
-	if (!(priv->cee_params.dcbx_cap & DCB_CAP_DCBX_VER_CEE))
+	if (!(priv->dcbx_cap & DCB_CAP_DCBX_VER_CEE))
 		return -EINVAL;
 
 	memset(&app, 0, sizeof(struct dcb_app));
@@ -433,7 +436,7 @@ static u8 mlx4_en_dcbnl_getdcbx(struct net_device *dev)
 {
 	struct mlx4_en_priv *priv = netdev_priv(dev);
 
-	return priv->cee_params.dcbx_cap;
+	return priv->dcbx_cap;
 }
 
 static u8 mlx4_en_dcbnl_setdcbx(struct net_device *dev, u8 mode)
@@ -442,7 +445,7 @@ static u8 mlx4_en_dcbnl_setdcbx(struct net_device *dev, u8 mode)
 	struct ieee_ets ets = {0};
 	struct ieee_pfc pfc = {0};
 
-	if (mode == priv->cee_params.dcbx_cap)
+	if (mode == priv->dcbx_cap)
 		return 0;
 
 	if ((mode & DCB_CAP_DCBX_LLD_MANAGED) ||
@@ -451,7 +454,7 @@ static u8 mlx4_en_dcbnl_setdcbx(struct net_device *dev, u8 mode)
 	    !(mode & DCB_CAP_DCBX_HOST))
 		goto err;
 
-	priv->cee_params.dcbx_cap = mode;
+	priv->dcbx_cap = mode;
 
 	ets.ets_cap = IEEE_8021QAZ_MAX_TCS;
 	pfc.pfc_cap = IEEE_8021QAZ_MAX_TCS;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index 4198e9bf89d0..7e703bed7b82 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -71,10 +71,11 @@ int mlx4_en_setup_tc(struct net_device *dev, u8 up)
 #ifdef CONFIG_MLX4_EN_DCB
 	if (!mlx4_is_slave(priv->mdev->dev)) {
 		if (up) {
-			priv->flags |= MLX4_EN_FLAG_DCB_ENABLED;
+			if (priv->dcbx_cap)
+				priv->flags |= MLX4_EN_FLAG_DCB_ENABLED;
 		} else {
 			priv->flags &= ~MLX4_EN_FLAG_DCB_ENABLED;
-			priv->cee_params.dcb_cfg.pfc_state = false;
+			priv->cee_config.pfc_state = false;
 		}
 	}
 #endif /* CONFIG_MLX4_EN_DCB */
@@ -2399,12 +2400,14 @@ static int mlx4_en_set_vf_mac(struct net_device *dev, int queue, u8 *mac)
 	return mlx4_set_vf_mac(mdev->dev, en_priv->port, queue, mac_u64);
 }
 
-static int mlx4_en_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos)
+static int mlx4_en_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
+			       __be16 vlan_proto)
 {
 	struct mlx4_en_priv *en_priv = netdev_priv(dev);
 	struct mlx4_en_dev *mdev = en_priv->mdev;
 
-	return mlx4_set_vf_vlan(mdev->dev, en_priv->port, vf, vlan, qos);
+	return mlx4_set_vf_vlan(mdev->dev, en_priv->port, vf, vlan, qos,
+				vlan_proto);
 }
 
 static int mlx4_en_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
@@ -2642,12 +2645,16 @@ static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog)
 			if (IS_ERR(prog))
 				return PTR_ERR(prog);
 		}
+		mutex_lock(&mdev->state_lock);
 		for (i = 0; i < priv->rx_ring_num; i++) {
-			/* This xchg is paired with READ_ONCE in the fastpath */
-			old_prog = xchg(&priv->rx_ring[i]->xdp_prog, prog);
+			old_prog = rcu_dereference_protected(
+					priv->rx_ring[i]->xdp_prog,
+					lockdep_is_held(&mdev->state_lock));
+			rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog);
 			if (old_prog)
 				bpf_prog_put(old_prog);
 		}
+		mutex_unlock(&mdev->state_lock);
 		return 0;
 	}
 
@@ -2680,7 +2687,10 @@ static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog)
 							priv->xdp_ring_num);
 
 	for (i = 0; i < priv->rx_ring_num; i++) {
-		old_prog = xchg(&priv->rx_ring[i]->xdp_prog, prog);
+		old_prog = rcu_dereference_protected(
+					priv->rx_ring[i]->xdp_prog,
+					lockdep_is_held(&mdev->state_lock));
+		rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog);
 		if (old_prog)
 			bpf_prog_put(old_prog);
 	}
@@ -3048,9 +3058,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
 	struct mlx4_en_priv *priv;
 	int i;
 	int err;
-#ifdef CONFIG_MLX4_EN_DCB
-	struct tc_configuration *tc;
-#endif
 
 	dev = alloc_etherdev_mqs(sizeof(struct mlx4_en_priv),
 				 MAX_TX_RINGS, MAX_RX_RINGS);
@@ -3117,16 +3124,13 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
 	priv->msg_enable = MLX4_EN_MSG_LEVEL;
 #ifdef CONFIG_MLX4_EN_DCB
 	if (!mlx4_is_slave(priv->mdev->dev)) {
-		priv->cee_params.dcbx_cap = DCB_CAP_DCBX_VER_CEE |
-					    DCB_CAP_DCBX_HOST |
-					    DCB_CAP_DCBX_VER_IEEE;
+		priv->dcbx_cap = DCB_CAP_DCBX_VER_CEE | DCB_CAP_DCBX_HOST |
+			DCB_CAP_DCBX_VER_IEEE;
 		priv->flags |= MLX4_EN_DCB_ENABLED;
-		priv->cee_params.dcb_cfg.pfc_state = false;
+		priv->cee_config.pfc_state = false;
 
-		for (i = 0; i < MLX4_EN_NUM_UP; i++) {
-			tc = &priv->cee_params.dcb_cfg.tc_config[i];
-			tc->dcb_pfc = pfc_disabled;
-		}
+		for (i = 0; i < MLX4_EN_NUM_UP; i++)
+			priv->cee_config.dcb_pfc[i] = pfc_disabled;
 
 		if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) {
 			dev->dcbnl_ops = &mlx4_en_dcbnl_ops;
@@ -3222,6 +3226,7 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
 	}
 
 	if (mlx4_is_slave(mdev->dev)) {
+		bool vlan_offload_disabled;
 		int phv;
 
 		err = get_phv_bit(mdev->dev, port, &phv);
@@ -3229,6 +3234,18 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
 			dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX;
 			priv->pflags |= MLX4_EN_PRIV_FLAGS_PHV;
 		}
+		err = mlx4_get_is_vlan_offload_disabled(mdev->dev, port,
+							&vlan_offload_disabled);
+		if (!err && vlan_offload_disabled) {
+			dev->hw_features &= ~(NETIF_F_HW_VLAN_CTAG_TX |
+					      NETIF_F_HW_VLAN_CTAG_RX |
+					      NETIF_F_HW_VLAN_STAG_TX |
+					      NETIF_F_HW_VLAN_STAG_RX);
+			dev->features &= ~(NETIF_F_HW_VLAN_CTAG_TX |
+					   NETIF_F_HW_VLAN_CTAG_RX |
+					   NETIF_F_HW_VLAN_STAG_TX |
+					   NETIF_F_HW_VLAN_STAG_RX);
+		}
 	} else {
 		if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN &&
 		    !(mdev->dev->caps.flags2 &
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index 2040dad8611d..f2e8beddcf44 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -72,7 +72,7 @@ static int mlx4_alloc_pages(struct mlx4_en_priv *priv,
 	}
 	dma = dma_map_page(priv->ddev, page, 0, PAGE_SIZE << order,
 			   frag_info->dma_dir);
-	if (dma_mapping_error(priv->ddev, dma)) {
+	if (unlikely(dma_mapping_error(priv->ddev, dma))) {
 		put_page(page);
 		return -ENOMEM;
 	}
@@ -108,7 +108,8 @@ static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv,
 		    ring_alloc[i].page_size)
 			continue;
 
-		if (mlx4_alloc_pages(priv, &page_alloc[i], frag_info, gfp))
+		if (unlikely(mlx4_alloc_pages(priv, &page_alloc[i],
+					      frag_info, gfp)))
 			goto out;
 	}
 
@@ -537,7 +538,9 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
 	struct mlx4_en_rx_ring *ring = *pring;
 	struct bpf_prog *old_prog;
 
-	old_prog = READ_ONCE(ring->xdp_prog);
+	old_prog = rcu_dereference_protected(
+					ring->xdp_prog,
+					lockdep_is_held(&mdev->state_lock));
 	if (old_prog)
 		bpf_prog_put(old_prog);
 	mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
@@ -583,7 +586,7 @@ static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
 		frag_info = &priv->frag_info[nr];
 		if (length <= frag_info->frag_prefix_size)
 			break;
-		if (!frags[nr].page)
+		if (unlikely(!frags[nr].page))
 			goto fail;
 
 		dma = be64_to_cpu(rx_desc->data[nr].addr);
@@ -623,7 +626,7 @@ static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
 	dma_addr_t dma;
 
 	skb = netdev_alloc_skb(priv->dev, SMALL_PACKET_SIZE + NET_IP_ALIGN);
-	if (!skb) {
+	if (unlikely(!skb)) {
 		en_dbg(RX_ERR, priv, "Failed allocating skb\n");
 		return NULL;
 	}
@@ -734,7 +737,8 @@ static int get_fixed_ipv6_csum(__wsum hw_checksum, struct sk_buff *skb,
 {
 	__wsum csum_pseudo_hdr = 0;
 
-	if (ipv6h->nexthdr == IPPROTO_FRAGMENT || ipv6h->nexthdr == IPPROTO_HOPOPTS)
+	if (unlikely(ipv6h->nexthdr == IPPROTO_FRAGMENT ||
+		     ipv6h->nexthdr == IPPROTO_HOPOPTS))
 		return -1;
 	hw_checksum = csum_add(hw_checksum, (__force __wsum)htons(ipv6h->nexthdr));
 
@@ -767,7 +771,7 @@ static int check_csum(struct mlx4_cqe *cqe, struct sk_buff *skb, void *va,
 		get_fixed_ipv4_csum(hw_checksum, skb, hdr);
 #if IS_ENABLED(CONFIG_IPV6)
 	else if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV6))
-		if (get_fixed_ipv6_csum(hw_checksum, skb, hdr))
+		if (unlikely(get_fixed_ipv6_csum(hw_checksum, skb, hdr)))
 			return -1;
 #endif
 	return 0;
@@ -794,13 +798,15 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
 	u64 timestamp;
 	bool l2_tunnel;
 
-	if (!priv->port_up)
+	if (unlikely(!priv->port_up))
 		return 0;
 
-	if (budget <= 0)
+	if (unlikely(budget <= 0))
 		return polled;
 
-	xdp_prog = READ_ONCE(ring->xdp_prog);
+	/* Protect accesses to: ring->xdp_prog, priv->mac_hash list */
+	rcu_read_lock();
+	xdp_prog = rcu_dereference(ring->xdp_prog);
 	doorbell_pending = 0;
 	tx_index = (priv->tx_ring_num - priv->xdp_ring_num) + cq->ring;
 
@@ -858,15 +864,11 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
 				/* Drop the packet, since HW loopback-ed it */
 				mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX];
 				bucket = &priv->mac_hash[mac_hash];
-				rcu_read_lock();
 				hlist_for_each_entry_rcu(entry, bucket, hlist) {
 					if (ether_addr_equal_64bits(entry->mac,
-								    ethh->h_source)) {
-						rcu_read_unlock();
+								    ethh->h_source))
 						goto next;
-					}
 				}
-				rcu_read_unlock();
 			}
 		}
 
@@ -902,16 +904,17 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
 			case XDP_PASS:
 				break;
 			case XDP_TX:
-				if (!mlx4_en_xmit_frame(frags, dev,
+				if (likely(!mlx4_en_xmit_frame(frags, dev,
 							length, tx_index,
-							&doorbell_pending))
+							&doorbell_pending)))
 					goto consumed;
-				break;
+				goto xdp_drop; /* Drop on xmit failure */
 			default:
 				bpf_warn_invalid_xdp_action(act);
 			case XDP_ABORTED:
 			case XDP_DROP:
-				if (mlx4_en_rx_recycle(ring, frags))
+xdp_drop:
+				if (likely(mlx4_en_rx_recycle(ring, frags)))
 					goto consumed;
 				goto next;
 			}
@@ -1015,12 +1018,12 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
 
 		/* GRO not possible, complete processing here */
 		skb = mlx4_en_rx_skb(priv, rx_desc, frags, length);
-		if (!skb) {
+		if (unlikely(!skb)) {
 			ring->dropped++;
 			goto next;
 		}
 
-                if (unlikely(priv->validate_loopback)) {
+		if (unlikely(priv->validate_loopback)) {
 			validate_loopback(priv, skb);
 			goto next;
 		}
@@ -1077,6 +1080,7 @@ consumed:
 	}
 
 out:
+	rcu_read_unlock();
 	if (doorbell_pending)
 		mlx4_en_xmit_doorbell(priv->tx_ring[tx_index]);
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
index 9df87ca0515a..e2509bba3e7c 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
@@ -818,7 +818,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
 	real_size = get_real_size(skb, shinfo, dev, &lso_header_size,
 				  &inline_ok, &fragptr);
 	if (unlikely(!real_size))
-		goto tx_drop;
+		goto tx_drop_count;
 
 	/* Align descriptor to TXBB size */
 	desc_size = ALIGN(real_size, TXBB_SIZE);
@@ -826,7 +826,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev)
 	if (unlikely(nr_txbb > MAX_DESC_TXBBS)) {
 		if (netif_msg_tx_err(priv))
 			en_warn(priv, "Oversized header or SG list\n");
-		goto tx_drop;
+		goto tx_drop_count;
 	}
 
 	bf_ok = ring->bf_enabled;
@@ -1071,9 +1071,10 @@ tx_drop_unmap:
 			       PCI_DMA_TODEVICE);
 	}
 
+tx_drop_count:
+	ring->tx_dropped++;
 tx_drop:
 	dev_kfree_skb_any(skb);
-	ring->tx_dropped++;
 	return NETDEV_TX_OK;
 }
 
@@ -1106,7 +1107,7 @@ netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_alloc *frame,
 		goto tx_drop;
 
 	if (mlx4_en_is_tx_ring_full(ring))
-		goto tx_drop;
+		goto tx_drop_count;
 
 	/* fetch ring->cons far ahead before needing it to avoid stall */
 	ring_cons = READ_ONCE(ring->cons);
@@ -1176,7 +1177,8 @@ netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_alloc *frame,
 
 	return NETDEV_TX_OK;
 
-tx_drop:
+tx_drop_count:
 	ring->tx_dropped++;
+tx_drop:
 	return NETDEV_TX_BUSY;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c
index f613977455e0..cf8f8a72a801 100644
--- a/drivers/net/ethernet/mellanox/mlx4/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/eq.c
@@ -1305,8 +1305,8 @@ int mlx4_init_eq_table(struct mlx4_dev *dev)
 	return 0;
 
 err_out_unmap:
-	while (i >= 0)
-		mlx4_free_eq(dev, &priv->eq_table.eq[i--]);
+	while (i > 0)
+		mlx4_free_eq(dev, &priv->eq_table.eq[--i]);
 #ifdef CONFIG_RFS_ACCEL
 	for (i = 1; i <= dev->caps.num_ports; i++) {
 		if (mlx4_priv(dev)->port[i].rmap) {
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c
index d728704d0c7b..c41ab31a39f8 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.c
@@ -158,7 +158,9 @@ static void dump_dev_cap_flags2(struct mlx4_dev *dev, u64 flags)
 		[31] = "Modifying loopback source checks using UPDATE_QP support",
 		[32] = "Loopback source checks support",
 		[33] = "RoCEv2 support",
-		[34] = "DMFS Sniffer support (UC & MC)"
+		[34] = "DMFS Sniffer support (UC & MC)",
+		[35] = "QinQ VST mode support",
+		[36] = "sl to vl mapping table change event support"
 	};
 	int i;
 
@@ -248,6 +250,72 @@ out:
 	return err;
 }
 
+static int mlx4_activate_vst_qinq(struct mlx4_priv *priv, int slave, int port)
+{
+	struct mlx4_vport_oper_state *vp_oper;
+	struct mlx4_vport_state *vp_admin;
+	int err;
+
+	vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port];
+	vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port];
+
+	if (vp_admin->default_vlan != vp_oper->state.default_vlan) {
+		err = __mlx4_register_vlan(&priv->dev, port,
+					   vp_admin->default_vlan,
+					   &vp_oper->vlan_idx);
+		if (err) {
+			vp_oper->vlan_idx = NO_INDX;
+			mlx4_warn(&priv->dev,
+				  "No vlan resources slave %d, port %d\n",
+				  slave, port);
+			return err;
+		}
+		mlx4_dbg(&priv->dev, "alloc vlan %d idx  %d slave %d port %d\n",
+			 (int)(vp_oper->state.default_vlan),
+			 vp_oper->vlan_idx, slave, port);
+	}
+	vp_oper->state.vlan_proto   = vp_admin->vlan_proto;
+	vp_oper->state.default_vlan = vp_admin->default_vlan;
+	vp_oper->state.default_qos  = vp_admin->default_qos;
+
+	return 0;
+}
+
+static int mlx4_handle_vst_qinq(struct mlx4_priv *priv, int slave, int port)
+{
+	struct mlx4_vport_oper_state *vp_oper;
+	struct mlx4_slave_state *slave_state;
+	struct mlx4_vport_state *vp_admin;
+	int err;
+
+	vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port];
+	vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port];
+	slave_state = &priv->mfunc.master.slave_state[slave];
+
+	if ((vp_admin->vlan_proto != htons(ETH_P_8021AD)) ||
+	    (!slave_state->active))
+		return 0;
+
+	if (vp_oper->state.vlan_proto == vp_admin->vlan_proto &&
+	    vp_oper->state.default_vlan == vp_admin->default_vlan &&
+	    vp_oper->state.default_qos == vp_admin->default_qos)
+		return 0;
+
+	if (!slave_state->vst_qinq_supported) {
+		/* Warn and revert the request to set vst QinQ mode */
+		vp_admin->vlan_proto   = vp_oper->state.vlan_proto;
+		vp_admin->default_vlan = vp_oper->state.default_vlan;
+		vp_admin->default_qos  = vp_oper->state.default_qos;
+
+		mlx4_warn(&priv->dev,
+			  "Slave %d does not support VST QinQ mode\n", slave);
+		return 0;
+	}
+
+	err = mlx4_activate_vst_qinq(priv, slave, port);
+	return err;
+}
+
 int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
 				struct mlx4_vhcr *vhcr,
 				struct mlx4_cmd_mailbox *inbox,
@@ -311,14 +379,18 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
 #define QUERY_FUNC_CAP_VF_ENABLE_QP0		0x08
 
 #define QUERY_FUNC_CAP_FLAGS0_FORCE_PHY_WQE_GID 0x80
-#define QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS (1 << 31)
 #define QUERY_FUNC_CAP_PHV_BIT			0x40
+#define QUERY_FUNC_CAP_VLAN_OFFLOAD_DISABLE	0x20
+
+#define QUERY_FUNC_CAP_SUPPORTS_VST_QINQ	BIT(30)
+#define QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS BIT(31)
 
 	if (vhcr->op_modifier == 1) {
 		struct mlx4_active_ports actv_ports =
 			mlx4_get_active_ports(dev, slave);
 		int converted_port = mlx4_slave_convert_port(
 				dev, slave, vhcr->in_modifier);
+		struct mlx4_vport_oper_state *vp_oper;
 
 		if (converted_port < 0)
 			return -EINVAL;
@@ -357,15 +429,24 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
 		MLX4_PUT(outbox->buf, dev->caps.phys_port_id[vhcr->in_modifier],
 			 QUERY_FUNC_CAP_PHYS_PORT_ID);
 
-		if (dev->caps.phv_bit[port]) {
-			field = QUERY_FUNC_CAP_PHV_BIT;
-			MLX4_PUT(outbox->buf, field,
-				 QUERY_FUNC_CAP_FLAGS0_OFFSET);
-		}
+		vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port];
+		err = mlx4_handle_vst_qinq(priv, slave, port);
+		if (err)
+			return err;
+
+		field = 0;
+		if (dev->caps.phv_bit[port])
+			field |= QUERY_FUNC_CAP_PHV_BIT;
+		if (vp_oper->state.vlan_proto == htons(ETH_P_8021AD))
+			field |= QUERY_FUNC_CAP_VLAN_OFFLOAD_DISABLE;
+		MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS0_OFFSET);
 
 	} else if (vhcr->op_modifier == 0) {
 		struct mlx4_active_ports actv_ports =
 			mlx4_get_active_ports(dev, slave);
+		struct mlx4_slave_state *slave_state =
+			&priv->mfunc.master.slave_state[slave];
+
 		/* enable rdma and ethernet interfaces, new quota locations,
 		 * and reserved lkey
 		 */
@@ -439,6 +520,10 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
 
 		size = dev->caps.reserved_lkey + ((slave << 8) & 0xFF00);
 		MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_RESD_LKEY_OFFSET);
+
+		if (vhcr->in_modifier & QUERY_FUNC_CAP_SUPPORTS_VST_QINQ)
+			slave_state->vst_qinq_supported = true;
+
 	} else
 		err = -EINVAL;
 
@@ -454,10 +539,12 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u8 gen_or_port,
 	u32			size, qkey;
 	int			err = 0, quotas = 0;
 	u32                     in_modifier;
+	u32			slave_caps;
 
 	op_modifier = !!gen_or_port; /* 0 = general, 1 = logical port */
-	in_modifier = op_modifier ? gen_or_port :
+	slave_caps = QUERY_FUNC_CAP_SUPPORTS_VST_QINQ |
 		QUERY_FUNC_CAP_SUPPORTS_NON_POWER_OF_2_NUM_EQS;
+	in_modifier = op_modifier ? gen_or_port : slave_caps;
 
 	mailbox = mlx4_alloc_cmd_mailbox(dev);
 	if (IS_ERR(mailbox))
@@ -612,8 +699,7 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u8 gen_or_port,
 		MLX4_GET(func_cap->phys_port_id, outbox,
 			 QUERY_FUNC_CAP_PHYS_PORT_ID);
 
-	MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS0_OFFSET);
-	func_cap->flags |= (field & QUERY_FUNC_CAP_PHV_BIT);
+	MLX4_GET(func_cap->flags0, outbox, QUERY_FUNC_CAP_FLAGS0_OFFSET);
 
 	/* All other resources are allocated by the master, but we still report
 	 * 'num' and 'reserved' capabilities as follows:
@@ -690,6 +776,7 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 #define QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET	0x52
 #define QUERY_DEV_CAP_MAX_SG_RQ_OFFSET		0x55
 #define QUERY_DEV_CAP_MAX_DESC_SZ_RQ_OFFSET	0x56
+#define QUERY_DEV_CAP_SVLAN_BY_QP_OFFSET	0x5D
 #define QUERY_DEV_CAP_MAX_QP_MCG_OFFSET		0x61
 #define QUERY_DEV_CAP_RSVD_MCG_OFFSET		0x62
 #define QUERY_DEV_CAP_MAX_MCG_OFFSET		0x63
@@ -703,6 +790,7 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 #define QUERY_DEV_CAP_FLOW_STEERING_IPOIB_OFFSET	0x74
 #define QUERY_DEV_CAP_FLOW_STEERING_RANGE_EN_OFFSET	0x76
 #define QUERY_DEV_CAP_FLOW_STEERING_MAX_QP_OFFSET	0x77
+#define QUERY_DEV_CAP_SL2VL_EVENT_OFFSET	0x78
 #define QUERY_DEV_CAP_CQ_EQ_CACHE_LINE_STRIDE	0x7a
 #define QUERY_DEV_CAP_ECN_QCN_VER_OFFSET	0x7b
 #define QUERY_DEV_CAP_RDMARC_ENTRY_SZ_OFFSET	0x80
@@ -767,12 +855,8 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 	dev_cap->max_eqs = 1 << (field & 0xf);
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MTT_OFFSET);
 	dev_cap->reserved_mtts = 1 << (field >> 4);
-	MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MRW_SZ_OFFSET);
-	dev_cap->max_mrw_sz = 1 << field;
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MRW_OFFSET);
 	dev_cap->reserved_mrws = 1 << (field & 0xf);
-	MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_MTT_SEG_OFFSET);
-	dev_cap->max_mtt_seg = 1 << (field & 0x3f);
 	MLX4_GET(size, outbox, QUERY_DEV_CAP_NUM_SYS_EQ_OFFSET);
 	dev_cap->num_sys_eqs = size & 0xfff;
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_REQ_QP_OFFSET);
@@ -822,6 +906,9 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_DMFS_IPOIB;
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_FLOW_STEERING_MAX_QP_OFFSET);
 	dev_cap->fs_max_num_qp_per_entry = field;
+	MLX4_GET(field, outbox, QUERY_DEV_CAP_SL2VL_EVENT_OFFSET);
+	if (field & (1 << 5))
+		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_SL_TO_VL_CHANGE_EVENT;
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_ECN_QCN_VER_OFFSET);
 	if (field & 0x1)
 		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_QCN;
@@ -857,6 +944,9 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 	MLX4_GET(size, outbox, QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET);
 	dev_cap->max_sq_desc_sz = size;
 
+	MLX4_GET(field, outbox, QUERY_DEV_CAP_SVLAN_BY_QP_OFFSET);
+	if (field & 0x1)
+		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_SVLAN_BY_QP;
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_MCG_OFFSET);
 	dev_cap->max_qp_per_mcg = 1 << field;
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_MCG_OFFSET);
@@ -2698,7 +2788,6 @@ static int mlx4_check_smp_firewall_active(struct mlx4_dev *dev,
 int mlx4_config_mad_demux(struct mlx4_dev *dev)
 {
 	struct mlx4_cmd_mailbox *mailbox;
-	int secure_host_active;
 	int err;
 
 	/* Check if mad_demux is supported */
@@ -2721,7 +2810,8 @@ int mlx4_config_mad_demux(struct mlx4_dev *dev)
 		goto out;
 	}
 
-	secure_host_active = mlx4_check_smp_firewall_active(dev, mailbox);
+	if (mlx4_check_smp_firewall_active(dev, mailbox))
+		dev->flags |= MLX4_FLAG_SECURE_HOST;
 
 	/* Config mad_demux to handle all MADs returned by the query above */
 	err = mlx4_cmd(dev, mailbox->dma, 0x01 /* subn mgmt class */,
@@ -2732,7 +2822,7 @@ int mlx4_config_mad_demux(struct mlx4_dev *dev)
 		goto out;
 	}
 
-	if (secure_host_active)
+	if (dev->flags & MLX4_FLAG_SECURE_HOST)
 		mlx4_warn(dev, "HCA operating in secure-host mode. SMP firewall activated.\n");
 out:
 	mlx4_free_cmd_mailbox(dev, mailbox);
@@ -2914,7 +3004,7 @@ int get_phv_bit(struct mlx4_dev *dev, u8 port, int *phv)
 	memset(&func_cap, 0, sizeof(func_cap));
 	err = mlx4_QUERY_FUNC_CAP(dev, port, &func_cap);
 	if (!err)
-		*phv = func_cap.flags & QUERY_FUNC_CAP_PHV_BIT;
+		*phv = func_cap.flags0 & QUERY_FUNC_CAP_PHV_BIT;
 	return err;
 }
 EXPORT_SYMBOL(get_phv_bit);
@@ -2938,6 +3028,22 @@ int set_phv_bit(struct mlx4_dev *dev, u8 port, int new_val)
 }
 EXPORT_SYMBOL(set_phv_bit);
 
+int mlx4_get_is_vlan_offload_disabled(struct mlx4_dev *dev, u8 port,
+				      bool *vlan_offload_disabled)
+{
+	struct mlx4_func_cap func_cap;
+	int err;
+
+	memset(&func_cap, 0, sizeof(func_cap));
+	err = mlx4_QUERY_FUNC_CAP(dev, port, &func_cap);
+	if (!err)
+		*vlan_offload_disabled =
+			!!(func_cap.flags0 &
+			   QUERY_FUNC_CAP_VLAN_OFFLOAD_DISABLE);
+	return err;
+}
+EXPORT_SYMBOL(mlx4_get_is_vlan_offload_disabled);
+
 void mlx4_replace_zero_macs(struct mlx4_dev *dev)
 {
 	int i;
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.h b/drivers/net/ethernet/mellanox/mlx4/fw.h
index cdbd76f10ced..5343a0599253 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.h
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.h
@@ -80,9 +80,7 @@ struct mlx4_dev_cap {
 	int max_eqs;
 	int num_sys_eqs;
 	int reserved_mtts;
-	int max_mrw_sz;
 	int reserved_mrws;
-	int max_mtt_seg;
 	int max_requester_per_qp;
 	int max_responder_per_qp;
 	int max_rdma_global;
@@ -152,7 +150,7 @@ struct mlx4_func_cap {
 	u32	qp1_proxy_qpn;
 	u32	reserved_lkey;
 	u8	physical_port;
-	u8	port_flags;
+	u8	flags0;
 	u8	flags1;
 	u64	phys_port_id;
 	u32	extra_flags;
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 75dd2e3d3059..7183ac4135d2 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -2970,6 +2970,7 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
 		mlx4_err(dev, "Failed to create mtu file for port %d\n", port);
 		device_remove_file(&info->dev->persist->pdev->dev,
 				   &info->port_attr);
+		devlink_port_unregister(&info->devlink_port);
 		info->port = -1;
 	}
 
@@ -2984,6 +2985,8 @@ static void mlx4_cleanup_port_info(struct mlx4_port_info *info)
 	device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr);
 	device_remove_file(&info->dev->persist->pdev->dev,
 			   &info->port_mtu_attr);
+	devlink_port_unregister(&info->devlink_port);
+
 #ifdef CONFIG_RFS_ACCEL
 	free_irq_cpu_rmap(info->rmap);
 	info->rmap = NULL;
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4.h b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
index c9d7fc5159f2..e4878f31e45d 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
@@ -46,6 +46,7 @@
 #include <linux/interrupt.h>
 #include <linux/spinlock.h>
 #include <net/devlink.h>
+#include <linux/rwsem.h>
 
 #include <linux/mlx4/device.h>
 #include <linux/mlx4/driver.h>
@@ -482,6 +483,7 @@ struct mlx4_slave_state {
 	u8 init_port_mask;
 	bool active;
 	bool old_vlan_api;
+	bool vst_qinq_supported;
 	u8 function;
 	dma_addr_t vhcr_dma;
 	u16 mtu[MLX4_MAX_PORTS + 1];
@@ -507,6 +509,7 @@ struct mlx4_vport_state {
 	u64 mac;
 	u16 default_vlan;
 	u8  default_qos;
+	__be16 vlan_proto;
 	u32 tx_rate;
 	bool spoofchk;
 	u32 link_state;
@@ -627,6 +630,7 @@ struct mlx4_cmd {
 	struct mutex		slave_cmd_mutex;
 	struct semaphore	poll_sem;
 	struct semaphore	event_sem;
+	struct rw_semaphore	switch_sem;
 	int			max_cmds;
 	spinlock_t		context_lock;
 	int			free_head;
@@ -655,6 +659,7 @@ struct mlx4_vf_immed_vlan_work {
 	u8                      qos_vport;
 	u16			vlan_id;
 	u16			orig_vlan_id;
+	__be16			vlan_proto;
 };
 
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index 2c2913dcae98..a3528dd1e72e 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -340,7 +340,7 @@ struct mlx4_en_rx_ring {
 	u8  fcs_del;
 	void *buf;
 	void *rx_info;
-	struct bpf_prog *xdp_prog;
+	struct bpf_prog __rcu *xdp_prog;
 	struct mlx4_en_page_cache page_cache;
 	unsigned long bytes;
 	unsigned long packets;
@@ -482,20 +482,10 @@ enum dcb_pfc_type {
 	pfc_enabled_rx
 };
 
-struct tc_configuration {
-	enum dcb_pfc_type  dcb_pfc;
-};
-
 struct mlx4_en_cee_config {
 	bool	pfc_state;
-	struct	tc_configuration tc_config[MLX4_EN_NUM_UP];
+	enum	dcb_pfc_type dcb_pfc[MLX4_EN_NUM_UP];
 };
-
-struct mlx4_en_cee_params {
-	u8 dcbx_cap;
-	struct mlx4_en_cee_config dcb_cfg;
-};
-
 #endif
 
 struct ethtool_flow_id {
@@ -624,7 +614,8 @@ struct mlx4_en_priv {
 	struct ieee_ets ets;
 	u16 maxrate[IEEE_8021QAZ_MAX_TCS];
 	enum dcbnl_cndd_states cndd_state[IEEE_8021QAZ_MAX_TCS];
-	struct mlx4_en_cee_params cee_params;
+	struct mlx4_en_cee_config cee_config;
+	u8 dcbx_cap;
 #endif
 #ifdef CONFIG_RFS_ACCEL
 	spinlock_t filters_lock;
diff --git a/drivers/net/ethernet/mellanox/mlx4/port.c b/drivers/net/ethernet/mellanox/mlx4/port.c
index 3d2095e5c61c..c5b2064297a1 100644
--- a/drivers/net/ethernet/mellanox/mlx4/port.c
+++ b/drivers/net/ethernet/mellanox/mlx4/port.c
@@ -52,7 +52,7 @@
 
 #define MLX4_FLAG_V_IGNORE_FCS_MASK		0x2
 #define MLX4_IGNORE_FCS_MASK			0x1
-#define MLNX4_TX_MAX_NUMBER			8
+#define MLX4_TC_MAX_NUMBER			8
 
 void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table)
 {
@@ -2022,7 +2022,7 @@ int mlx4_max_tc(struct mlx4_dev *dev)
 	u8 num_tc = dev->caps.max_tc_eth;
 
 	if (!num_tc)
-		num_tc = MLNX4_TX_MAX_NUMBER;
+		num_tc = MLX4_TC_MAX_NUMBER;
 
 	return num_tc;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
index 8b81114bdc72..84d7857ccc27 100644
--- a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
+++ b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
@@ -790,10 +790,22 @@ static int update_vport_qp_param(struct mlx4_dev *dev,
 				MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED |
 				MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED;
 		} else if (0 != vp_oper->state.default_vlan) {
-			qpc->pri_path.vlan_control |=
-				MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
-				MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
-				MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
+			if (vp_oper->state.vlan_proto == htons(ETH_P_8021AD)) {
+				/* vst QinQ should block untagged on TX,
+				 * but cvlan is in payload and phv is set so
+				 * hw see it as untagged. Block tagged instead.
+				 */
+				qpc->pri_path.vlan_control |=
+					MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED |
+					MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
+					MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
+					MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
+			} else { /* vst 802.1Q */
+				qpc->pri_path.vlan_control |=
+					MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
+					MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
+					MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
+			}
 		} else { /* priority tagged */
 			qpc->pri_path.vlan_control |=
 				MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
@@ -802,7 +814,11 @@ static int update_vport_qp_param(struct mlx4_dev *dev,
 
 		qpc->pri_path.fvl_rx |= MLX4_FVL_RX_FORCE_ETH_VLAN;
 		qpc->pri_path.vlan_index = vp_oper->vlan_idx;
-		qpc->pri_path.fl |= MLX4_FL_CV | MLX4_FL_ETH_HIDE_CQE_VLAN;
+		qpc->pri_path.fl |= MLX4_FL_ETH_HIDE_CQE_VLAN;
+		if (vp_oper->state.vlan_proto == htons(ETH_P_8021AD))
+			qpc->pri_path.fl |= MLX4_FL_SV;
+		else
+			qpc->pri_path.fl |= MLX4_FL_CV;
 		qpc->pri_path.feup |= MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN;
 		qpc->pri_path.sched_queue &= 0xC7;
 		qpc->pri_path.sched_queue |= (vp_oper->state.default_qos) << 3;
@@ -5238,6 +5254,7 @@ void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work)
 	u64 qp_path_mask = ((1ULL << MLX4_UPD_QP_PATH_MASK_VLAN_INDEX) |
 		       (1ULL << MLX4_UPD_QP_PATH_MASK_FVL) |
 		       (1ULL << MLX4_UPD_QP_PATH_MASK_CV) |
+		       (1ULL << MLX4_UPD_QP_PATH_MASK_SV) |
 		       (1ULL << MLX4_UPD_QP_PATH_MASK_ETH_HIDE_CQE_VLAN) |
 		       (1ULL << MLX4_UPD_QP_PATH_MASK_FEUP) |
 		       (1ULL << MLX4_UPD_QP_PATH_MASK_FVL_RX) |
@@ -5266,7 +5283,12 @@ void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work)
 	else if (!work->vlan_id)
 		vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
 			MLX4_VLAN_CTRL_ETH_RX_BLOCK_TAGGED;
-	else
+	else if (work->vlan_proto == htons(ETH_P_8021AD))
+		vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_PRIO_TAGGED |
+			MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
+			MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
+			MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
+	else  /* vst 802.1Q */
 		vlan_control = MLX4_VLAN_CTRL_ETH_TX_BLOCK_TAGGED |
 			MLX4_VLAN_CTRL_ETH_RX_BLOCK_PRIO_TAGGED |
 			MLX4_VLAN_CTRL_ETH_RX_BLOCK_UNTAGGED;
@@ -5311,7 +5333,11 @@ void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work)
 				upd_context->qp_context.pri_path.fvl_rx =
 					qp->fvl_rx | MLX4_FVL_RX_FORCE_ETH_VLAN;
 				upd_context->qp_context.pri_path.fl =
-					qp->pri_path_fl | MLX4_FL_CV | MLX4_FL_ETH_HIDE_CQE_VLAN;
+					qp->pri_path_fl | MLX4_FL_ETH_HIDE_CQE_VLAN;
+				if (work->vlan_proto == htons(ETH_P_8021AD))
+					upd_context->qp_context.pri_path.fl |= MLX4_FL_SV;
+				else
+					upd_context->qp_context.pri_path.fl |= MLX4_FL_CV;
 				upd_context->qp_context.pri_path.feup =
 					qp->feup | MLX4_FEUP_FORCE_ETH_UP | MLX4_FVL_FORCE_ETH_VLAN;
 				upd_context->qp_context.pri_path.sched_queue =
diff --git a/drivers/net/ethernet/mellanox/mlx4/srq.c b/drivers/net/ethernet/mellanox/mlx4/srq.c
index 67146624eb58..f44d089e2ca6 100644
--- a/drivers/net/ethernet/mellanox/mlx4/srq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/srq.c
@@ -45,15 +45,12 @@ void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type)
 	struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
 	struct mlx4_srq *srq;
 
-	spin_lock(&srq_table->lock);
-
+	rcu_read_lock();
 	srq = radix_tree_lookup(&srq_table->tree, srqn & (dev->caps.num_srqs - 1));
+	rcu_read_unlock();
 	if (srq)
 		atomic_inc(&srq->refcount);
-
-	spin_unlock(&srq_table->lock);
-
-	if (!srq) {
+	else {
 		mlx4_warn(dev, "Async event for bogus SRQ %08x\n", srqn);
 		return;
 	}
@@ -301,12 +298,11 @@ struct mlx4_srq *mlx4_srq_lookup(struct mlx4_dev *dev, u32 srqn)
 {
 	struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
 	struct mlx4_srq *srq;
-	unsigned long flags;
 
-	spin_lock_irqsave(&srq_table->lock, flags);
+	rcu_read_lock();
 	srq = radix_tree_lookup(&srq_table->tree,
 				srqn & (dev->caps.num_srqs - 1));
-	spin_unlock_irqrestore(&srq_table->lock, flags);
+	rcu_read_unlock();
 
 	return srq;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 05cc1effc13c..0343725d7f44 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -3,7 +3,7 @@ obj-$(CONFIG_MLX5_CORE)		+= mlx5_core.o
 mlx5_core-y :=	main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
 		health.o mcg.o cq.o srq.o alloc.o qp.o port.o mr.o pd.o \
 		mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o \
-		fs_counters.o rl.o
+		fs_counters.o rl.o lag.o dev.o
 
 mlx5_core-$(CONFIG_MLX5_CORE_EN) += wq.o eswitch.o eswitch_offloads.o \
 		en_main.o en_common.o en_fs.o en_ethtool.o en_tx.o \
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
index d6e2a1cae19a..1e639f886021 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
@@ -143,13 +143,14 @@ static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
 	return cmd->cmd_buf + (idx << cmd->log_stride);
 }
 
-static u8 xor8_buf(void *buf, int len)
+static u8 xor8_buf(void *buf, size_t offset, int len)
 {
 	u8 *ptr = buf;
 	u8 sum = 0;
 	int i;
+	int end = len + offset;
 
-	for (i = 0; i < len; i++)
+	for (i = offset; i < end; i++)
 		sum ^= ptr[i];
 
 	return sum;
@@ -157,41 +158,49 @@ static u8 xor8_buf(void *buf, int len)
 
 static int verify_block_sig(struct mlx5_cmd_prot_block *block)
 {
-	if (xor8_buf(block->rsvd0, sizeof(*block) - sizeof(block->data) - 1) != 0xff)
+	size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
+	int xor_len = sizeof(*block) - sizeof(block->data) - 1;
+
+	if (xor8_buf(block, rsvd0_off, xor_len) != 0xff)
 		return -EINVAL;
 
-	if (xor8_buf(block, sizeof(*block)) != 0xff)
+	if (xor8_buf(block, 0, sizeof(*block)) != 0xff)
 		return -EINVAL;
 
 	return 0;
 }
 
-static void calc_block_sig(struct mlx5_cmd_prot_block *block, u8 token,
-			   int csum)
+static void calc_block_sig(struct mlx5_cmd_prot_block *block)
 {
-	block->token = token;
-	if (csum) {
-		block->ctrl_sig = ~xor8_buf(block->rsvd0, sizeof(*block) -
-					    sizeof(block->data) - 2);
-		block->sig = ~xor8_buf(block, sizeof(*block) - 1);
-	}
+	int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2;
+	size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
+
+	block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len);
+	block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1);
 }
 
-static void calc_chain_sig(struct mlx5_cmd_msg *msg, u8 token, int csum)
+static void calc_chain_sig(struct mlx5_cmd_msg *msg)
 {
 	struct mlx5_cmd_mailbox *next = msg->next;
-
-	while (next) {
-		calc_block_sig(next->buf, token, csum);
+	int size = msg->len;
+	int blen = size - min_t(int, sizeof(msg->first.data), size);
+	int n = (blen + MLX5_CMD_DATA_BLOCK_SIZE - 1)
+		/ MLX5_CMD_DATA_BLOCK_SIZE;
+	int i = 0;
+
+	for (i = 0; i < n && next; i++)  {
+		calc_block_sig(next->buf);
 		next = next->next;
 	}
 }
 
 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
 {
-	ent->lay->sig = ~xor8_buf(ent->lay, sizeof(*ent->lay));
-	calc_chain_sig(ent->in, ent->token, csum);
-	calc_chain_sig(ent->out, ent->token, csum);
+	ent->lay->sig = ~xor8_buf(ent->lay, 0,  sizeof(*ent->lay));
+	if (csum) {
+		calc_chain_sig(ent->in);
+		calc_chain_sig(ent->out);
+	}
 }
 
 static void poll_timeout(struct mlx5_cmd_work_ent *ent)
@@ -222,12 +231,17 @@ static int verify_signature(struct mlx5_cmd_work_ent *ent)
 	struct mlx5_cmd_mailbox *next = ent->out->next;
 	int err;
 	u8 sig;
+	int size = ent->out->len;
+	int blen = size - min_t(int, sizeof(ent->out->first.data), size);
+	int n = (blen + MLX5_CMD_DATA_BLOCK_SIZE - 1)
+		/ MLX5_CMD_DATA_BLOCK_SIZE;
+	int i = 0;
 
-	sig = xor8_buf(ent->lay, sizeof(*ent->lay));
+	sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));
 	if (sig != 0xff)
 		return -EINVAL;
 
-	while (next) {
+	for (i = 0; i < n && next; i++) {
 		err = verify_block_sig(next->buf);
 		if (err)
 			return err;
@@ -280,11 +294,13 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
 	case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
 	case MLX5_CMD_OP_DEALLOC_PD:
 	case MLX5_CMD_OP_DEALLOC_UAR:
-	case MLX5_CMD_OP_DETTACH_FROM_MCG:
+	case MLX5_CMD_OP_DETACH_FROM_MCG:
 	case MLX5_CMD_OP_DEALLOC_XRCD:
 	case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
 	case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
 	case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
+	case MLX5_CMD_OP_DESTROY_LAG:
+	case MLX5_CMD_OP_DESTROY_VPORT_LAG:
 	case MLX5_CMD_OP_DESTROY_TIR:
 	case MLX5_CMD_OP_DESTROY_SQ:
 	case MLX5_CMD_OP_DESTROY_RQ:
@@ -301,6 +317,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
 	case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 	case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT:
+	case MLX5_CMD_OP_DEALLOC_ENCAP_HEADER:
 		return MLX5_CMD_STAT_OK;
 
 	case MLX5_CMD_OP_QUERY_HCA_CAP:
@@ -375,6 +392,10 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
 	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 	case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
+	case MLX5_CMD_OP_CREATE_LAG:
+	case MLX5_CMD_OP_MODIFY_LAG:
+	case MLX5_CMD_OP_QUERY_LAG:
+	case MLX5_CMD_OP_CREATE_VPORT_LAG:
 	case MLX5_CMD_OP_CREATE_TIR:
 	case MLX5_CMD_OP_MODIFY_TIR:
 	case MLX5_CMD_OP_QUERY_TIR:
@@ -402,6 +423,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
 	case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 	case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
+	case MLX5_CMD_OP_ALLOC_ENCAP_HEADER:
 		*status = MLX5_DRIVER_STATUS_ABORTED;
 		*synd = MLX5_DRIVER_SYND;
 		return -EIO;
@@ -490,7 +512,7 @@ const char *mlx5_command_str(int command)
 	MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION);
 	MLX5_COMMAND_STR_CASE(ACCESS_REG);
 	MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG);
-	MLX5_COMMAND_STR_CASE(DETTACH_FROM_MCG);
+	MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG);
 	MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG);
 	MLX5_COMMAND_STR_CASE(MAD_IFC);
 	MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX);
@@ -512,6 +534,12 @@ const char *mlx5_command_str(int command)
 	MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY);
 	MLX5_COMMAND_STR_CASE(SET_WOL_ROL);
 	MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL);
+	MLX5_COMMAND_STR_CASE(CREATE_LAG);
+	MLX5_COMMAND_STR_CASE(MODIFY_LAG);
+	MLX5_COMMAND_STR_CASE(QUERY_LAG);
+	MLX5_COMMAND_STR_CASE(DESTROY_LAG);
+	MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG);
+	MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG);
 	MLX5_COMMAND_STR_CASE(CREATE_TIR);
 	MLX5_COMMAND_STR_CASE(MODIFY_TIR);
 	MLX5_COMMAND_STR_CASE(DESTROY_TIR);
@@ -550,15 +578,130 @@ const char *mlx5_command_str(int command)
 	MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
 	MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
 	MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE);
+	MLX5_COMMAND_STR_CASE(ALLOC_ENCAP_HEADER);
+	MLX5_COMMAND_STR_CASE(DEALLOC_ENCAP_HEADER);
 	default: return "unknown command opcode";
 	}
 }
 
+static const char *cmd_status_str(u8 status)
+{
+	switch (status) {
+	case MLX5_CMD_STAT_OK:
+		return "OK";
+	case MLX5_CMD_STAT_INT_ERR:
+		return "internal error";
+	case MLX5_CMD_STAT_BAD_OP_ERR:
+		return "bad operation";
+	case MLX5_CMD_STAT_BAD_PARAM_ERR:
+		return "bad parameter";
+	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
+		return "bad system state";
+	case MLX5_CMD_STAT_BAD_RES_ERR:
+		return "bad resource";
+	case MLX5_CMD_STAT_RES_BUSY:
+		return "resource busy";
+	case MLX5_CMD_STAT_LIM_ERR:
+		return "limits exceeded";
+	case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
+		return "bad resource state";
+	case MLX5_CMD_STAT_IX_ERR:
+		return "bad index";
+	case MLX5_CMD_STAT_NO_RES_ERR:
+		return "no resources";
+	case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
+		return "bad input length";
+	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
+		return "bad output length";
+	case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
+		return "bad QP state";
+	case MLX5_CMD_STAT_BAD_PKT_ERR:
+		return "bad packet (discarded)";
+	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
+		return "bad size too many outstanding CQEs";
+	default:
+		return "unknown status";
+	}
+}
+
+static int cmd_status_to_err(u8 status)
+{
+	switch (status) {
+	case MLX5_CMD_STAT_OK:				return 0;
+	case MLX5_CMD_STAT_INT_ERR:			return -EIO;
+	case MLX5_CMD_STAT_BAD_OP_ERR:			return -EINVAL;
+	case MLX5_CMD_STAT_BAD_PARAM_ERR:		return -EINVAL;
+	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:		return -EIO;
+	case MLX5_CMD_STAT_BAD_RES_ERR:			return -EINVAL;
+	case MLX5_CMD_STAT_RES_BUSY:			return -EBUSY;
+	case MLX5_CMD_STAT_LIM_ERR:			return -ENOMEM;
+	case MLX5_CMD_STAT_BAD_RES_STATE_ERR:		return -EINVAL;
+	case MLX5_CMD_STAT_IX_ERR:			return -EINVAL;
+	case MLX5_CMD_STAT_NO_RES_ERR:			return -EAGAIN;
+	case MLX5_CMD_STAT_BAD_INP_LEN_ERR:		return -EIO;
+	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:		return -EIO;
+	case MLX5_CMD_STAT_BAD_QP_STATE_ERR:		return -EINVAL;
+	case MLX5_CMD_STAT_BAD_PKT_ERR:			return -EINVAL;
+	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:	return -EINVAL;
+	default:					return -EIO;
+	}
+}
+
+struct mlx5_ifc_mbox_out_bits {
+	u8         status[0x8];
+	u8         reserved_at_8[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_at_40[0x40];
+};
+
+struct mlx5_ifc_mbox_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_at_10[0x10];
+
+	u8         reserved_at_20[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_at_40[0x40];
+};
+
+void mlx5_cmd_mbox_status(void *out, u8 *status, u32 *syndrome)
+{
+	*status = MLX5_GET(mbox_out, out, status);
+	*syndrome = MLX5_GET(mbox_out, out, syndrome);
+}
+
+static int mlx5_cmd_check(struct mlx5_core_dev *dev, void *in, void *out)
+{
+	u32 syndrome;
+	u8  status;
+	u16 opcode;
+	u16 op_mod;
+
+	mlx5_cmd_mbox_status(out, &status, &syndrome);
+	if (!status)
+		return 0;
+
+	opcode = MLX5_GET(mbox_in, in, opcode);
+	op_mod = MLX5_GET(mbox_in, in, op_mod);
+
+	mlx5_core_err(dev,
+		      "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x)\n",
+		      mlx5_command_str(opcode),
+		      opcode, op_mod,
+		      cmd_status_str(status),
+		      status,
+		      syndrome);
+
+	return cmd_status_to_err(status);
+}
+
 static void dump_command(struct mlx5_core_dev *dev,
 			 struct mlx5_cmd_work_ent *ent, int input)
 {
-	u16 op = be16_to_cpu(((struct mlx5_inbox_hdr *)(ent->lay->in))->opcode);
 	struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
+	u16 op = MLX5_GET(mbox_in, ent->lay->in, opcode);
 	struct mlx5_cmd_mailbox *next = msg->next;
 	int data_only;
 	u32 offset = 0;
@@ -608,9 +751,7 @@ static void dump_command(struct mlx5_core_dev *dev,
 
 static u16 msg_to_opcode(struct mlx5_cmd_msg *in)
 {
-	struct mlx5_inbox_hdr *hdr = (struct mlx5_inbox_hdr *)(in->first.data);
-
-	return be16_to_cpu(hdr->opcode);
+	return MLX5_GET(mbox_in, in->first.data, opcode);
 }
 
 static void cb_timeout_handler(struct work_struct *work)
@@ -656,7 +797,6 @@ static void cmd_work_handler(struct work_struct *work)
 		spin_unlock_irqrestore(&cmd->alloc_lock, flags);
 	}
 
-	ent->token = alloc_token(cmd);
 	cmd->ent_arr[ent->idx] = ent;
 	lay = get_inst(cmd, ent->idx);
 	ent->lay = lay;
@@ -749,16 +889,6 @@ static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
 	return err;
 }
 
-static __be32 *get_synd_ptr(struct mlx5_outbox_hdr *out)
-{
-	return &out->syndrome;
-}
-
-static u8 *get_status_ptr(struct mlx5_outbox_hdr *out)
-{
-	return &out->status;
-}
-
 /*  Notes:
  *    1. Callback functions may not sleep
  *    2. page queue commands do not support asynchrous completion
@@ -766,7 +896,8 @@ static u8 *get_status_ptr(struct mlx5_outbox_hdr *out)
 static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
 			   struct mlx5_cmd_msg *out, void *uout, int uout_size,
 			   mlx5_cmd_cbk_t callback,
-			   void *context, int page_queue, u8 *status)
+			   void *context, int page_queue, u8 *status,
+			   u8 token)
 {
 	struct mlx5_cmd *cmd = &dev->cmd;
 	struct mlx5_cmd_work_ent *ent;
@@ -783,6 +914,8 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
 	if (IS_ERR(ent))
 		return PTR_ERR(ent);
 
+	ent->token = token;
+
 	if (!callback)
 		init_completion(&ent->done);
 
@@ -804,7 +937,7 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
 		goto out_free;
 
 	ds = ent->ts2 - ent->ts1;
-	op = be16_to_cpu(((struct mlx5_inbox_hdr *)in->first.data)->opcode);
+	op = MLX5_GET(mbox_in, in->first.data, opcode);
 	if (op < ARRAY_SIZE(cmd->stats)) {
 		stats = &cmd->stats[op];
 		spin_lock_irq(&stats->lock);
@@ -854,7 +987,8 @@ static const struct file_operations fops = {
 	.write	= dbg_write,
 };
 
-static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size)
+static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size,
+			    u8 token)
 {
 	struct mlx5_cmd_prot_block *block;
 	struct mlx5_cmd_mailbox *next;
@@ -880,6 +1014,7 @@ static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size)
 		memcpy(block->data, from, copy);
 		from += copy;
 		size -= copy;
+		block->token = token;
 		next = next->next;
 	}
 
@@ -949,7 +1084,8 @@ static void free_cmd_box(struct mlx5_core_dev *dev,
 }
 
 static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
-					       gfp_t flags, int size)
+					       gfp_t flags, int size,
+					       u8 token)
 {
 	struct mlx5_cmd_mailbox *tmp, *head = NULL;
 	struct mlx5_cmd_prot_block *block;
@@ -978,6 +1114,7 @@ static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
 		tmp->next = head;
 		block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
 		block->block_num = cpu_to_be32(n - i - 1);
+		block->token = token;
 		head = tmp;
 	}
 	msg->next = head;
@@ -1015,7 +1152,6 @@ static ssize_t data_write(struct file *filp, const char __user *buf,
 	struct mlx5_core_dev *dev = filp->private_data;
 	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
 	void *ptr;
-	int err;
 
 	if (*pos != 0)
 		return -EINVAL;
@@ -1023,25 +1159,15 @@ static ssize_t data_write(struct file *filp, const char __user *buf,
 	kfree(dbg->in_msg);
 	dbg->in_msg = NULL;
 	dbg->inlen = 0;
-
-	ptr = kzalloc(count, GFP_KERNEL);
-	if (!ptr)
-		return -ENOMEM;
-
-	if (copy_from_user(ptr, buf, count)) {
-		err = -EFAULT;
-		goto out;
-	}
+	ptr = memdup_user(buf, count);
+	if (IS_ERR(ptr))
+		return PTR_ERR(ptr);
 	dbg->in_msg = ptr;
 	dbg->inlen = count;
 
 	*pos = count;
 
 	return count;
-
-out:
-	kfree(ptr);
-	return err;
 }
 
 static ssize_t data_read(struct file *filp, char __user *buf, size_t count,
@@ -1301,11 +1427,16 @@ void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec)
 				callback = ent->callback;
 				context = ent->context;
 				err = ent->ret;
-				if (!err)
+				if (!err) {
 					err = mlx5_copy_from_msg(ent->uout,
 								 ent->out,
 								 ent->uout_size);
 
+					err = err ? err : mlx5_cmd_check(dev,
+									ent->in->first.data,
+									ent->uout);
+				}
+
 				mlx5_free_cmd_msg(dev, ent->out);
 				free_msg(dev, ent->in);
 
@@ -1352,19 +1483,14 @@ static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
 	}
 
 	if (IS_ERR(msg))
-		msg = mlx5_alloc_cmd_msg(dev, gfp, in_size);
+		msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0);
 
 	return msg;
 }
 
-static u16 opcode_from_in(struct mlx5_inbox_hdr *in)
+static int is_manage_pages(void *in)
 {
-	return be16_to_cpu(in->opcode);
-}
-
-static int is_manage_pages(struct mlx5_inbox_hdr *in)
-{
-	return be16_to_cpu(in->opcode) == MLX5_CMD_OP_MANAGE_PAGES;
+	return MLX5_GET(mbox_in, in, opcode) == MLX5_CMD_OP_MANAGE_PAGES;
 }
 
 static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
@@ -1377,12 +1503,15 @@ static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
 	int err;
 	u8 status = 0;
 	u32 drv_synd;
+	u8 token;
 
 	if (pci_channel_offline(dev->pdev) ||
 	    dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
-		err = mlx5_internal_err_ret_value(dev, opcode_from_in(in), &drv_synd, &status);
-		*get_synd_ptr(out) = cpu_to_be32(drv_synd);
-		*get_status_ptr(out) = status;
+		u16 opcode = MLX5_GET(mbox_in, in, opcode);
+
+		err = mlx5_internal_err_ret_value(dev, opcode, &drv_synd, &status);
+		MLX5_SET(mbox_out, out, status, status);
+		MLX5_SET(mbox_out, out, syndrome, drv_synd);
 		return err;
 	}
 
@@ -1395,20 +1524,22 @@ static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
 		return err;
 	}
 
-	err = mlx5_copy_to_msg(inb, in, in_size);
+	token = alloc_token(&dev->cmd);
+
+	err = mlx5_copy_to_msg(inb, in, in_size, token);
 	if (err) {
 		mlx5_core_warn(dev, "err %d\n", err);
 		goto out_in;
 	}
 
-	outb = mlx5_alloc_cmd_msg(dev, gfp, out_size);
+	outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token);
 	if (IS_ERR(outb)) {
 		err = PTR_ERR(outb);
 		goto out_in;
 	}
 
 	err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,
-			      pages_queue, &status);
+			      pages_queue, &status, token);
 	if (err)
 		goto out_out;
 
@@ -1434,7 +1565,10 @@ out_in:
 int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
 		  int out_size)
 {
-	return cmd_exec(dev, in, in_size, out, out_size, NULL, NULL);
+	int err;
+
+	err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL);
+	return err ? : mlx5_cmd_check(dev, in, out);
 }
 EXPORT_SYMBOL(mlx5_cmd_exec);
 
@@ -1476,7 +1610,7 @@ static int create_msg_cache(struct mlx5_core_dev *dev)
 	INIT_LIST_HEAD(&cmd->cache.med.head);
 
 	for (i = 0; i < NUM_LONG_LISTS; i++) {
-		msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, LONG_LIST_SIZE);
+		msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, LONG_LIST_SIZE, 0);
 		if (IS_ERR(msg)) {
 			err = PTR_ERR(msg);
 			goto ex_err;
@@ -1486,7 +1620,7 @@ static int create_msg_cache(struct mlx5_core_dev *dev)
 	}
 
 	for (i = 0; i < NUM_MED_LISTS; i++) {
-		msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, MED_LIST_SIZE);
+		msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, MED_LIST_SIZE, 0);
 		if (IS_ERR(msg)) {
 			err = PTR_ERR(msg);
 			goto ex_err;
@@ -1671,96 +1805,3 @@ void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
 	pci_pool_destroy(cmd->pool);
 }
 EXPORT_SYMBOL(mlx5_cmd_cleanup);
-
-static const char *cmd_status_str(u8 status)
-{
-	switch (status) {
-	case MLX5_CMD_STAT_OK:
-		return "OK";
-	case MLX5_CMD_STAT_INT_ERR:
-		return "internal error";
-	case MLX5_CMD_STAT_BAD_OP_ERR:
-		return "bad operation";
-	case MLX5_CMD_STAT_BAD_PARAM_ERR:
-		return "bad parameter";
-	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
-		return "bad system state";
-	case MLX5_CMD_STAT_BAD_RES_ERR:
-		return "bad resource";
-	case MLX5_CMD_STAT_RES_BUSY:
-		return "resource busy";
-	case MLX5_CMD_STAT_LIM_ERR:
-		return "limits exceeded";
-	case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
-		return "bad resource state";
-	case MLX5_CMD_STAT_IX_ERR:
-		return "bad index";
-	case MLX5_CMD_STAT_NO_RES_ERR:
-		return "no resources";
-	case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
-		return "bad input length";
-	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
-		return "bad output length";
-	case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
-		return "bad QP state";
-	case MLX5_CMD_STAT_BAD_PKT_ERR:
-		return "bad packet (discarded)";
-	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
-		return "bad size too many outstanding CQEs";
-	default:
-		return "unknown status";
-	}
-}
-
-static int cmd_status_to_err(u8 status)
-{
-	switch (status) {
-	case MLX5_CMD_STAT_OK:				return 0;
-	case MLX5_CMD_STAT_INT_ERR:			return -EIO;
-	case MLX5_CMD_STAT_BAD_OP_ERR:			return -EINVAL;
-	case MLX5_CMD_STAT_BAD_PARAM_ERR:		return -EINVAL;
-	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:		return -EIO;
-	case MLX5_CMD_STAT_BAD_RES_ERR:			return -EINVAL;
-	case MLX5_CMD_STAT_RES_BUSY:			return -EBUSY;
-	case MLX5_CMD_STAT_LIM_ERR:			return -ENOMEM;
-	case MLX5_CMD_STAT_BAD_RES_STATE_ERR:		return -EINVAL;
-	case MLX5_CMD_STAT_IX_ERR:			return -EINVAL;
-	case MLX5_CMD_STAT_NO_RES_ERR:			return -EAGAIN;
-	case MLX5_CMD_STAT_BAD_INP_LEN_ERR:		return -EIO;
-	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:		return -EIO;
-	case MLX5_CMD_STAT_BAD_QP_STATE_ERR:		return -EINVAL;
-	case MLX5_CMD_STAT_BAD_PKT_ERR:			return -EINVAL;
-	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:	return -EINVAL;
-	default:					return -EIO;
-	}
-}
-
-/* this will be available till all the commands use set/get macros */
-int mlx5_cmd_status_to_err(struct mlx5_outbox_hdr *hdr)
-{
-	if (!hdr->status)
-		return 0;
-
-	pr_warn("command failed, status %s(0x%x), syndrome 0x%x\n",
-		cmd_status_str(hdr->status), hdr->status,
-		be32_to_cpu(hdr->syndrome));
-
-	return cmd_status_to_err(hdr->status);
-}
-
-int mlx5_cmd_status_to_err_v2(void *ptr)
-{
-	u32	syndrome;
-	u8	status;
-
-	status = be32_to_cpu(*(__be32 *)ptr) >> 24;
-	if (!status)
-		return 0;
-
-	syndrome = be32_to_cpu(*(__be32 *)(ptr + 4));
-
-	pr_warn("command failed, status %s(0x%x), syndrome 0x%x\n",
-		cmd_status_str(status), status, syndrome);
-
-	return cmd_status_to_err(status);
-}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cq.c b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
index 873a631ad155..32d4af9b594d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/cq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cq.c
@@ -134,33 +134,29 @@ void mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type)
 		complete(&cq->free);
 }
 
-
 int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
-			struct mlx5_create_cq_mbox_in *in, int inlen)
+			u32 *in, int inlen)
 {
-	int err;
 	struct mlx5_cq_table *table = &dev->priv.cq_table;
-	struct mlx5_create_cq_mbox_out out;
-	struct mlx5_destroy_cq_mbox_in din;
-	struct mlx5_destroy_cq_mbox_out dout;
+	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
+	u32 din[MLX5_ST_SZ_DW(destroy_cq_in)];
+	u32 dout[MLX5_ST_SZ_DW(destroy_cq_out)];
 	int eqn = MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context),
 			   c_eqn);
 	struct mlx5_eq *eq;
+	int err;
 
 	eq = mlx5_eqn2eq(dev, eqn);
 	if (IS_ERR(eq))
 		return PTR_ERR(eq);
 
-	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_CQ);
-	memset(&out, 0, sizeof(out));
-	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
+	memset(out, 0, sizeof(out));
+	MLX5_SET(create_cq_in, in, opcode, MLX5_CMD_OP_CREATE_CQ);
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (err)
 		return err;
 
-	if (out.hdr.status)
-		return mlx5_cmd_status_to_err(&out.hdr);
-
-	cq->cqn = be32_to_cpu(out.cqn) & 0xffffff;
+	cq->cqn = MLX5_GET(create_cq_out, out, cqn);
 	cq->cons_index = 0;
 	cq->arm_sn     = 0;
 	atomic_set(&cq->refcount, 1);
@@ -186,10 +182,11 @@ int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
 	return 0;
 
 err_cmd:
-	memset(&din, 0, sizeof(din));
-	memset(&dout, 0, sizeof(dout));
-	din.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ);
-	mlx5_cmd_exec(dev, &din, sizeof(din), &dout, sizeof(dout));
+	memset(din, 0, sizeof(din));
+	memset(dout, 0, sizeof(dout));
+	MLX5_SET(destroy_cq_in, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
+	MLX5_SET(destroy_cq_in, din, cqn, cq->cqn);
+	mlx5_cmd_exec(dev, din, sizeof(din), dout, sizeof(dout));
 	return err;
 }
 EXPORT_SYMBOL(mlx5_core_create_cq);
@@ -197,8 +194,8 @@ EXPORT_SYMBOL(mlx5_core_create_cq);
 int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq)
 {
 	struct mlx5_cq_table *table = &dev->priv.cq_table;
-	struct mlx5_destroy_cq_mbox_in in;
-	struct mlx5_destroy_cq_mbox_out out;
+	u32 out[MLX5_ST_SZ_DW(destroy_cq_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_cq_in)] = {0};
 	struct mlx5_core_cq *tmp;
 	int err;
 
@@ -214,17 +211,12 @@ int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq)
 		return -EINVAL;
 	}
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ);
-	in.cqn = cpu_to_be32(cq->cqn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+	MLX5_SET(destroy_cq_in, in, opcode, MLX5_CMD_OP_DESTROY_CQ);
+	MLX5_SET(destroy_cq_in, in, cqn, cq->cqn);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 	if (err)
 		return err;
 
-	if (out.hdr.status)
-		return mlx5_cmd_status_to_err(&out.hdr);
-
 	synchronize_irq(cq->irqn);
 
 	mlx5_debug_cq_remove(dev, cq);
@@ -237,44 +229,23 @@ int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq)
 EXPORT_SYMBOL(mlx5_core_destroy_cq);
 
 int mlx5_core_query_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
-		       struct mlx5_query_cq_mbox_out *out)
+		       u32 *out, int outlen)
 {
-	struct mlx5_query_cq_mbox_in in;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(out, 0, sizeof(*out));
-
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_CQ);
-	in.cqn = cpu_to_be32(cq->cqn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out));
-	if (err)
-		return err;
-
-	if (out->hdr.status)
-		return mlx5_cmd_status_to_err(&out->hdr);
+	u32 in[MLX5_ST_SZ_DW(query_cq_in)] = {0};
 
-	return err;
+	MLX5_SET(query_cq_in, in, opcode, MLX5_CMD_OP_QUERY_CQ);
+	MLX5_SET(query_cq_in, in, cqn, cq->cqn);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 }
 EXPORT_SYMBOL(mlx5_core_query_cq);
 
-
 int mlx5_core_modify_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
-			struct mlx5_modify_cq_mbox_in *in, int in_sz)
+			u32 *in, int inlen)
 {
-	struct mlx5_modify_cq_mbox_out out;
-	int err;
-
-	memset(&out, 0, sizeof(out));
-	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MODIFY_CQ);
-	err = mlx5_cmd_exec(dev, in, in_sz, &out, sizeof(out));
-	if (err)
-		return err;
+	u32 out[MLX5_ST_SZ_DW(modify_cq_out)] = {0};
 
-	if (out.hdr.status)
-		return mlx5_cmd_status_to_err(&out.hdr);
-
-	return 0;
+	MLX5_SET(modify_cq_in, in, opcode, MLX5_CMD_OP_MODIFY_CQ);
+	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_modify_cq);
 
@@ -283,18 +254,20 @@ int mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev,
 				   u16 cq_period,
 				   u16 cq_max_count)
 {
-	struct mlx5_modify_cq_mbox_in in;
-
-	memset(&in, 0, sizeof(in));
-
-	in.cqn              = cpu_to_be32(cq->cqn);
-	in.ctx.cq_period    = cpu_to_be16(cq_period);
-	in.ctx.cq_max_count = cpu_to_be16(cq_max_count);
-	in.field_select     = cpu_to_be32(MLX5_CQ_MODIFY_PERIOD |
-					  MLX5_CQ_MODIFY_COUNT);
-
-	return mlx5_core_modify_cq(dev, cq, &in, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {0};
+	void *cqc;
+
+	MLX5_SET(modify_cq_in, in, cqn, cq->cqn);
+	cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context);
+	MLX5_SET(cqc, cqc, cq_period, cq_period);
+	MLX5_SET(cqc, cqc, cq_max_count, cq_max_count);
+	MLX5_SET(modify_cq_in, in,
+		 modify_field_select_resize_field_select.modify_field_select.modify_field_select,
+		 MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT);
+
+	return mlx5_core_modify_cq(dev, cq, in, sizeof(in));
 }
+EXPORT_SYMBOL(mlx5_core_modify_cq_moderation);
 
 int mlx5_init_cq_table(struct mlx5_core_dev *dev)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c
index 5210d92e6bc7..e94a9532e218 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c
@@ -277,24 +277,28 @@ void mlx5_cq_debugfs_cleanup(struct mlx5_core_dev *dev)
 static u64 qp_read_field(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
 			 int index, int *is_str)
 {
-	struct mlx5_query_qp_mbox_out *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
 	struct mlx5_qp_context *ctx;
 	u64 param = 0;
+	u32 *out;
 	int err;
 	int no_sq;
 
-	out = kzalloc(sizeof(*out), GFP_KERNEL);
+	out = kzalloc(outlen, GFP_KERNEL);
 	if (!out)
 		return param;
 
-	err = mlx5_core_qp_query(dev, qp, out, sizeof(*out));
+	err = mlx5_core_qp_query(dev, qp, out, outlen);
 	if (err) {
-		mlx5_core_warn(dev, "failed to query qp\n");
+		mlx5_core_warn(dev, "failed to query qp err=%d\n", err);
 		goto out;
 	}
 
 	*is_str = 0;
-	ctx = &out->ctx;
+
+	/* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */
+	ctx = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, out, qpc);
+
 	switch (index) {
 	case QP_PID:
 		param = qp->pid;
@@ -358,32 +362,32 @@ out:
 static u64 eq_read_field(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
 			 int index)
 {
-	struct mlx5_query_eq_mbox_out *out;
-	struct mlx5_eq_context *ctx;
+	int outlen = MLX5_ST_SZ_BYTES(query_eq_out);
 	u64 param = 0;
+	void *ctx;
+	u32 *out;
 	int err;
 
-	out = kzalloc(sizeof(*out), GFP_KERNEL);
+	out = kzalloc(outlen, GFP_KERNEL);
 	if (!out)
 		return param;
 
-	ctx = &out->ctx;
-
-	err = mlx5_core_eq_query(dev, eq, out, sizeof(*out));
+	err = mlx5_core_eq_query(dev, eq, out, outlen);
 	if (err) {
 		mlx5_core_warn(dev, "failed to query eq\n");
 		goto out;
 	}
+	ctx = MLX5_ADDR_OF(query_eq_out, out, eq_context_entry);
 
 	switch (index) {
 	case EQ_NUM_EQES:
-		param = 1 << ((be32_to_cpu(ctx->log_sz_usr_page) >> 24) & 0x1f);
+		param = 1 << MLX5_GET(eqc, ctx, log_eq_size);
 		break;
 	case EQ_INTR:
-		param = ctx->intr;
+		param = MLX5_GET(eqc, ctx, intr);
 		break;
 	case EQ_LOG_PG_SZ:
-		param = (ctx->log_page_size & 0x1f) + 12;
+		param = MLX5_GET(eqc, ctx, log_page_size) + 12;
 		break;
 	}
 
@@ -395,37 +399,37 @@ out:
 static u64 cq_read_field(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
 			 int index)
 {
-	struct mlx5_query_cq_mbox_out *out;
-	struct mlx5_cq_context *ctx;
+	int outlen = MLX5_ST_SZ_BYTES(query_cq_out);
 	u64 param = 0;
+	void *ctx;
+	u32 *out;
 	int err;
 
-	out = kzalloc(sizeof(*out), GFP_KERNEL);
+	out = mlx5_vzalloc(outlen);
 	if (!out)
 		return param;
 
-	ctx = &out->ctx;
-
-	err = mlx5_core_query_cq(dev, cq, out);
+	err = mlx5_core_query_cq(dev, cq, out, outlen);
 	if (err) {
 		mlx5_core_warn(dev, "failed to query cq\n");
 		goto out;
 	}
+	ctx = MLX5_ADDR_OF(query_cq_out, out, cq_context);
 
 	switch (index) {
 	case CQ_PID:
 		param = cq->pid;
 		break;
 	case CQ_NUM_CQES:
-		param = 1 << ((be32_to_cpu(ctx->log_sz_usr_page) >> 24) & 0x1f);
+		param = 1 << MLX5_GET(cqc, ctx, log_cq_size);
 		break;
 	case CQ_LOG_PG_SZ:
-		param = (ctx->log_pg_sz & 0x1f) + 12;
+		param = MLX5_GET(cqc, ctx, log_page_size);
 		break;
 	}
 
 out:
-	kfree(out);
+	kvfree(out);
 	return param;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
new file mode 100644
index 000000000000..a9dbc28f6b97
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
@@ -0,0 +1,345 @@
+/*
+ * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/mlx5/driver.h>
+#include "mlx5_core.h"
+
+static LIST_HEAD(intf_list);
+static LIST_HEAD(mlx5_dev_list);
+/* intf dev list mutex */
+static DEFINE_MUTEX(mlx5_intf_mutex);
+
+struct mlx5_device_context {
+	struct list_head	list;
+	struct mlx5_interface  *intf;
+	void		       *context;
+	unsigned long		state;
+};
+
+enum {
+	MLX5_INTERFACE_ADDED,
+	MLX5_INTERFACE_ATTACHED,
+};
+
+void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
+{
+	struct mlx5_device_context *dev_ctx;
+	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
+
+	if (!mlx5_lag_intf_add(intf, priv))
+		return;
+
+	dev_ctx = kzalloc(sizeof(*dev_ctx), GFP_KERNEL);
+	if (!dev_ctx)
+		return;
+
+	dev_ctx->intf = intf;
+	dev_ctx->context = intf->add(dev);
+	set_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state);
+	if (intf->attach)
+		set_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state);
+
+	if (dev_ctx->context) {
+		spin_lock_irq(&priv->ctx_lock);
+		list_add_tail(&dev_ctx->list, &priv->ctx_list);
+		spin_unlock_irq(&priv->ctx_lock);
+	} else {
+		kfree(dev_ctx);
+	}
+}
+
+static struct mlx5_device_context *mlx5_get_device(struct mlx5_interface *intf,
+						   struct mlx5_priv *priv)
+{
+	struct mlx5_device_context *dev_ctx;
+
+	list_for_each_entry(dev_ctx, &priv->ctx_list, list)
+		if (dev_ctx->intf == intf)
+			return dev_ctx;
+	return NULL;
+}
+
+void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
+{
+	struct mlx5_device_context *dev_ctx;
+	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
+
+	dev_ctx = mlx5_get_device(intf, priv);
+	if (!dev_ctx)
+		return;
+
+	spin_lock_irq(&priv->ctx_lock);
+	list_del(&dev_ctx->list);
+	spin_unlock_irq(&priv->ctx_lock);
+
+	if (test_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state))
+		intf->remove(dev, dev_ctx->context);
+
+	kfree(dev_ctx);
+}
+
+static void mlx5_attach_interface(struct mlx5_interface *intf, struct mlx5_priv *priv)
+{
+	struct mlx5_device_context *dev_ctx;
+	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
+
+	dev_ctx = mlx5_get_device(intf, priv);
+	if (!dev_ctx)
+		return;
+
+	if (intf->attach) {
+		if (test_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state))
+			return;
+		intf->attach(dev, dev_ctx->context);
+		set_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state);
+	} else {
+		if (test_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state))
+			return;
+		dev_ctx->context = intf->add(dev);
+		set_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state);
+	}
+}
+
+void mlx5_attach_device(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_interface *intf;
+
+	mutex_lock(&mlx5_intf_mutex);
+	list_for_each_entry(intf, &intf_list, list)
+		mlx5_attach_interface(intf, priv);
+	mutex_unlock(&mlx5_intf_mutex);
+}
+
+static void mlx5_detach_interface(struct mlx5_interface *intf, struct mlx5_priv *priv)
+{
+	struct mlx5_device_context *dev_ctx;
+	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
+
+	dev_ctx = mlx5_get_device(intf, priv);
+	if (!dev_ctx)
+		return;
+
+	if (intf->detach) {
+		if (!test_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state))
+			return;
+		intf->detach(dev, dev_ctx->context);
+		clear_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state);
+	} else {
+		if (!test_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state))
+			return;
+		intf->remove(dev, dev_ctx->context);
+		clear_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state);
+	}
+}
+
+void mlx5_detach_device(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_interface *intf;
+
+	mutex_lock(&mlx5_intf_mutex);
+	list_for_each_entry(intf, &intf_list, list)
+		mlx5_detach_interface(intf, priv);
+	mutex_unlock(&mlx5_intf_mutex);
+}
+
+bool mlx5_device_registered(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv;
+	bool found = false;
+
+	mutex_lock(&mlx5_intf_mutex);
+	list_for_each_entry(priv, &mlx5_dev_list, dev_list)
+		if (priv == &dev->priv)
+			found = true;
+	mutex_unlock(&mlx5_intf_mutex);
+
+	return found;
+}
+
+int mlx5_register_device(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_interface *intf;
+
+	mutex_lock(&mlx5_intf_mutex);
+	list_add_tail(&priv->dev_list, &mlx5_dev_list);
+	list_for_each_entry(intf, &intf_list, list)
+		mlx5_add_device(intf, priv);
+	mutex_unlock(&mlx5_intf_mutex);
+
+	return 0;
+}
+
+void mlx5_unregister_device(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_interface *intf;
+
+	mutex_lock(&mlx5_intf_mutex);
+	list_for_each_entry(intf, &intf_list, list)
+		mlx5_remove_device(intf, priv);
+	list_del(&priv->dev_list);
+	mutex_unlock(&mlx5_intf_mutex);
+}
+
+int mlx5_register_interface(struct mlx5_interface *intf)
+{
+	struct mlx5_priv *priv;
+
+	if (!intf->add || !intf->remove)
+		return -EINVAL;
+
+	mutex_lock(&mlx5_intf_mutex);
+	list_add_tail(&intf->list, &intf_list);
+	list_for_each_entry(priv, &mlx5_dev_list, dev_list)
+		mlx5_add_device(intf, priv);
+	mutex_unlock(&mlx5_intf_mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL(mlx5_register_interface);
+
+void mlx5_unregister_interface(struct mlx5_interface *intf)
+{
+	struct mlx5_priv *priv;
+
+	mutex_lock(&mlx5_intf_mutex);
+	list_for_each_entry(priv, &mlx5_dev_list, dev_list)
+		mlx5_remove_device(intf, priv);
+	list_del(&intf->list);
+	mutex_unlock(&mlx5_intf_mutex);
+}
+EXPORT_SYMBOL(mlx5_unregister_interface);
+
+void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol)
+{
+	struct mlx5_priv *priv = &mdev->priv;
+	struct mlx5_device_context *dev_ctx;
+	unsigned long flags;
+	void *result = NULL;
+
+	spin_lock_irqsave(&priv->ctx_lock, flags);
+
+	list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list)
+		if ((dev_ctx->intf->protocol == protocol) &&
+		    dev_ctx->intf->get_dev) {
+			result = dev_ctx->intf->get_dev(dev_ctx->context);
+			break;
+		}
+
+	spin_unlock_irqrestore(&priv->ctx_lock, flags);
+
+	return result;
+}
+EXPORT_SYMBOL(mlx5_get_protocol_dev);
+
+/* Must be called with intf_mutex held */
+void mlx5_add_dev_by_protocol(struct mlx5_core_dev *dev, int protocol)
+{
+	struct mlx5_interface *intf;
+
+	list_for_each_entry(intf, &intf_list, list)
+		if (intf->protocol == protocol) {
+			mlx5_add_device(intf, &dev->priv);
+			break;
+		}
+}
+
+/* Must be called with intf_mutex held */
+void mlx5_remove_dev_by_protocol(struct mlx5_core_dev *dev, int protocol)
+{
+	struct mlx5_interface *intf;
+
+	list_for_each_entry(intf, &intf_list, list)
+		if (intf->protocol == protocol) {
+			mlx5_remove_device(intf, &dev->priv);
+			break;
+		}
+}
+
+static u16 mlx5_gen_pci_id(struct mlx5_core_dev *dev)
+{
+	return (u16)((dev->pdev->bus->number << 8) |
+		     PCI_SLOT(dev->pdev->devfn));
+}
+
+/* Must be called with intf_mutex held */
+struct mlx5_core_dev *mlx5_get_next_phys_dev(struct mlx5_core_dev *dev)
+{
+	u16 pci_id = mlx5_gen_pci_id(dev);
+	struct mlx5_core_dev *res = NULL;
+	struct mlx5_core_dev *tmp_dev;
+	struct mlx5_priv *priv;
+
+	list_for_each_entry(priv, &mlx5_dev_list, dev_list) {
+		tmp_dev = container_of(priv, struct mlx5_core_dev, priv);
+		if ((dev != tmp_dev) && (mlx5_gen_pci_id(tmp_dev) == pci_id)) {
+			res = tmp_dev;
+			break;
+		}
+	}
+
+	return res;
+}
+
+void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
+		     unsigned long param)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_device_context *dev_ctx;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->ctx_lock, flags);
+
+	list_for_each_entry(dev_ctx, &priv->ctx_list, list)
+		if (dev_ctx->intf->event)
+			dev_ctx->intf->event(dev, dev_ctx->context, event, param);
+
+	spin_unlock_irqrestore(&priv->ctx_lock, flags);
+}
+
+void mlx5_dev_list_lock(void)
+{
+	mutex_lock(&mlx5_intf_mutex);
+}
+
+void mlx5_dev_list_unlock(void)
+{
+	mutex_unlock(&mlx5_intf_mutex);
+}
+
+int mlx5_dev_list_trylock(void)
+{
+	return mutex_trylock(&mlx5_intf_mutex);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 1b495efa7490..460363b66cb1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -62,19 +62,25 @@
 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE                0xd
 
 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW            0x1
-#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW            0x4
+#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW            0x3
 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW            0x6
 
+#define MLX5_RX_HEADROOM NET_SKB_PAD
+
 #define MLX5_MPWRQ_LOG_STRIDE_SIZE		6  /* >= 6, HW restriction */
 #define MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS	8  /* >= 6, HW restriction */
-#define MLX5_MPWRQ_LOG_WQE_SZ			17
+#define MLX5_MPWRQ_LOG_WQE_SZ			18
 #define MLX5_MPWRQ_WQE_PAGE_ORDER  (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \
 				    MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT : 0)
 #define MLX5_MPWRQ_PAGES_PER_WQE		BIT(MLX5_MPWRQ_WQE_PAGE_ORDER)
 #define MLX5_MPWRQ_STRIDES_PER_PAGE		(MLX5_MPWRQ_NUM_STRIDES >> \
 						 MLX5_MPWRQ_WQE_PAGE_ORDER)
-#define MLX5_CHANNEL_MAX_NUM_MTTS (ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8) * \
-				   BIT(MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW))
+
+#define MLX5_MTT_OCTW(npages) (ALIGN(npages, 8) / 2)
+#define MLX5E_REQUIRED_MTTS(rqs, wqes)\
+	(rqs * wqes * ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8))
+#define MLX5E_VALID_NUM_MTTS(num_mtts) (MLX5_MTT_OCTW(num_mtts) <= U16_MAX)
+
 #define MLX5_UMR_ALIGN				(2048)
 #define MLX5_MPWRQ_SMALL_PACKET_THRESHOLD	(128)
 
@@ -95,6 +101,18 @@
 #define MLX5E_UPDATE_STATS_INTERVAL    200 /* msecs */
 #define MLX5E_SQ_BF_BUDGET             16
 
+#define MLX5E_ICOSQ_MAX_WQEBBS \
+	(DIV_ROUND_UP(sizeof(struct mlx5e_umr_wqe), MLX5_SEND_WQE_BB))
+
+#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
+#define MLX5E_XDP_IHS_DS_COUNT \
+	DIV_ROUND_UP(MLX5E_XDP_MIN_INLINE - 2, MLX5_SEND_WQE_DS)
+#define MLX5E_XDP_TX_DS_COUNT \
+	(MLX5E_XDP_IHS_DS_COUNT + \
+	 (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) + 1 /* SG DS */)
+#define MLX5E_XDP_TX_WQEBBS \
+	DIV_ROUND_UP(MLX5E_XDP_TX_DS_COUNT, MLX5_SEND_WQEBB_NUM_DS)
+
 #define MLX5E_NUM_MAIN_GROUPS 9
 
 static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
@@ -219,9 +237,8 @@ struct mlx5e_tstamp {
 };
 
 enum {
-	MLX5E_RQ_STATE_POST_WQES_ENABLE,
+	MLX5E_RQ_STATE_FLUSH,
 	MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS,
-	MLX5E_RQ_STATE_FLUSH_TIMEOUT,
 	MLX5E_RQ_STATE_AM,
 };
 
@@ -284,20 +301,44 @@ struct mlx5e_rx_am { /* Adaptive Moderation */
 	u8					tired;
 };
 
+/* a single cache unit is capable to serve one napi call (for non-striding rq)
+ * or a MPWQE (for striding rq).
+ */
+#define MLX5E_CACHE_UNIT	(MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \
+				 MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT)
+#define MLX5E_CACHE_SIZE	(2 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
+struct mlx5e_page_cache {
+	u32 head;
+	u32 tail;
+	struct mlx5e_dma_info page_cache[MLX5E_CACHE_SIZE];
+};
+
 struct mlx5e_rq {
 	/* data path */
 	struct mlx5_wq_ll      wq;
-	u32                    wqe_sz;
-	struct sk_buff       **skb;
-	struct mlx5e_mpw_info *wqe_info;
+
+	union {
+		struct mlx5e_dma_info *dma_info;
+		struct {
+			struct mlx5e_mpw_info *info;
+			void                  *mtt_no_align;
+			u32                    mtt_offset;
+		} mpwqe;
+	};
+	struct {
+		u8             page_order;
+		u32            wqe_sz;    /* wqe data buffer size */
+		u8             map_dir;   /* dma map direction */
+	} buff;
 	__be32                 mkey_be;
-	__be32                 umr_mkey_be;
 
 	struct device         *pdev;
 	struct net_device     *netdev;
 	struct mlx5e_tstamp   *tstamp;
 	struct mlx5e_rq_stats  stats;
 	struct mlx5e_cq        cq;
+	struct mlx5e_page_cache page_cache;
+
 	mlx5e_fp_handle_rx_cqe handle_rx_cqe;
 	mlx5e_fp_alloc_wqe     alloc_wqe;
 	mlx5e_fp_dealloc_wqe   dealloc_wqe;
@@ -306,6 +347,7 @@ struct mlx5e_rq {
 	int                    ix;
 
 	struct mlx5e_rx_am     am; /* Adaptive Moderation */
+	struct bpf_prog       *xdp_prog;
 
 	/* control */
 	struct mlx5_wq_ctrl    wq_ctrl;
@@ -319,32 +361,15 @@ struct mlx5e_rq {
 
 struct mlx5e_umr_dma_info {
 	__be64                *mtt;
-	__be64                *mtt_no_align;
 	dma_addr_t             mtt_addr;
-	struct mlx5e_dma_info *dma_info;
+	struct mlx5e_dma_info  dma_info[MLX5_MPWRQ_PAGES_PER_WQE];
+	struct mlx5e_umr_wqe   wqe;
 };
 
 struct mlx5e_mpw_info {
-	union {
-		struct mlx5e_dma_info     dma_info;
-		struct mlx5e_umr_dma_info umr;
-	};
+	struct mlx5e_umr_dma_info umr;
 	u16 consumed_strides;
 	u16 skbs_frags[MLX5_MPWRQ_PAGES_PER_WQE];
-
-	void (*dma_pre_sync)(struct device *pdev,
-			     struct mlx5e_mpw_info *wi,
-			     u32 wqe_offset, u32 len);
-	void (*add_skb_frag)(struct mlx5e_rq *rq,
-			     struct sk_buff *skb,
-			     struct mlx5e_mpw_info *wi,
-			     u32 page_idx, u32 frag_offset, u32 len);
-	void (*copy_skb_header)(struct device *pdev,
-				struct sk_buff *skb,
-				struct mlx5e_mpw_info *wi,
-				u32 page_idx, u32 offset,
-				u32 headlen);
-	void (*free_wqe)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
 };
 
 struct mlx5e_tx_wqe_info {
@@ -365,16 +390,21 @@ struct mlx5e_sq_dma {
 };
 
 enum {
-	MLX5E_SQ_STATE_WAKE_TXQ_ENABLE,
+	MLX5E_SQ_STATE_FLUSH,
 	MLX5E_SQ_STATE_BF_ENABLE,
-	MLX5E_SQ_STATE_TX_TIMEOUT,
 };
 
-struct mlx5e_ico_wqe_info {
+struct mlx5e_sq_wqe_info {
 	u8  opcode;
 	u8  num_wqebbs;
 };
 
+enum mlx5e_sq_type {
+	MLX5E_SQ_TXQ,
+	MLX5E_SQ_ICO,
+	MLX5E_SQ_XDP
+};
+
 struct mlx5e_sq {
 	/* data path */
 
@@ -392,10 +422,20 @@ struct mlx5e_sq {
 
 	struct mlx5e_cq            cq;
 
-	/* pointers to per packet info: write@xmit, read@completion */
-	struct sk_buff           **skb;
-	struct mlx5e_sq_dma       *dma_fifo;
-	struct mlx5e_tx_wqe_info  *wqe_info;
+	/* pointers to per tx element info: write@xmit, read@completion */
+	union {
+		struct {
+			struct sk_buff           **skb;
+			struct mlx5e_sq_dma       *dma_fifo;
+			struct mlx5e_tx_wqe_info  *wqe_info;
+		} txq;
+		struct mlx5e_sq_wqe_info *ico_wqe;
+		struct {
+			struct mlx5e_sq_wqe_info  *wqe_info;
+			struct mlx5e_dma_info     *di;
+			bool                       doorbell;
+		} xdp;
+	} db;
 
 	/* read only */
 	struct mlx5_wq_cyc         wq;
@@ -417,8 +457,8 @@ struct mlx5e_sq {
 	struct mlx5_uar            uar;
 	struct mlx5e_channel      *channel;
 	int                        tc;
-	struct mlx5e_ico_wqe_info *ico_wqe_info;
 	u32                        rate_limit;
+	u8                         type;
 } ____cacheline_aligned_in_smp;
 
 static inline bool mlx5e_sq_has_room_for(struct mlx5e_sq *sq, u16 n)
@@ -434,8 +474,10 @@ enum channel_flags {
 struct mlx5e_channel {
 	/* data path */
 	struct mlx5e_rq            rq;
+	struct mlx5e_sq            xdp_sq;
 	struct mlx5e_sq            sq[MLX5E_MAX_NUM_TC];
 	struct mlx5e_sq            icosq;   /* internal control operations */
+	bool                       xdp;
 	struct napi_struct         napi;
 	struct device             *pdev;
 	struct net_device         *netdev;
@@ -617,6 +659,7 @@ struct mlx5e_priv {
 	/* priv data path fields - start */
 	struct mlx5e_sq            **txq_to_sq_map;
 	int channeltc_to_txq_map[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC];
+	struct bpf_prog *xdp_prog;
 	/* priv data path fields - end */
 
 	unsigned long              state;
@@ -651,40 +694,6 @@ struct mlx5e_priv {
 	void                      *ppriv;
 };
 
-enum mlx5e_link_mode {
-	MLX5E_1000BASE_CX_SGMII	 = 0,
-	MLX5E_1000BASE_KX	 = 1,
-	MLX5E_10GBASE_CX4	 = 2,
-	MLX5E_10GBASE_KX4	 = 3,
-	MLX5E_10GBASE_KR	 = 4,
-	MLX5E_20GBASE_KR2	 = 5,
-	MLX5E_40GBASE_CR4	 = 6,
-	MLX5E_40GBASE_KR4	 = 7,
-	MLX5E_56GBASE_R4	 = 8,
-	MLX5E_10GBASE_CR	 = 12,
-	MLX5E_10GBASE_SR	 = 13,
-	MLX5E_10GBASE_ER	 = 14,
-	MLX5E_40GBASE_SR4	 = 15,
-	MLX5E_40GBASE_LR4	 = 16,
-	MLX5E_50GBASE_SR2	 = 18,
-	MLX5E_100GBASE_CR4	 = 20,
-	MLX5E_100GBASE_SR4	 = 21,
-	MLX5E_100GBASE_KR4	 = 22,
-	MLX5E_100GBASE_LR4	 = 23,
-	MLX5E_100BASE_TX	 = 24,
-	MLX5E_1000BASE_T	 = 25,
-	MLX5E_10GBASE_T		 = 26,
-	MLX5E_25GBASE_CR	 = 27,
-	MLX5E_25GBASE_KR	 = 28,
-	MLX5E_25GBASE_SR	 = 29,
-	MLX5E_50GBASE_CR2	 = 30,
-	MLX5E_50GBASE_KR2	 = 31,
-	MLX5E_LINK_MODES_NUMBER,
-};
-
-#define MLX5E_PROT_MASK(link_mode) (1 << link_mode)
-
-
 void mlx5e_build_ptys2ethtool_map(void);
 
 void mlx5e_send_nop(struct mlx5e_sq *sq, bool notify_hw);
@@ -697,31 +706,19 @@ void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
 int mlx5e_napi_poll(struct napi_struct *napi, int budget);
 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
-void mlx5e_free_tx_descs(struct mlx5e_sq *sq);
-void mlx5e_free_rx_descs(struct mlx5e_rq *rq);
+void mlx5e_free_sq_descs(struct mlx5e_sq *sq);
 
+void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
+			bool recycle);
 void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
 void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
 bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq);
 int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix);
-int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix);
+int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe,	u16 ix);
 void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
-void mlx5e_post_rx_fragmented_mpwqe(struct mlx5e_rq *rq);
-void mlx5e_complete_rx_linear_mpwqe(struct mlx5e_rq *rq,
-				    struct mlx5_cqe64 *cqe,
-				    u16 byte_cnt,
-				    struct mlx5e_mpw_info *wi,
-				    struct sk_buff *skb);
-void mlx5e_complete_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
-					struct mlx5_cqe64 *cqe,
-					u16 byte_cnt,
-					struct mlx5e_mpw_info *wi,
-					struct sk_buff *skb);
-void mlx5e_free_rx_linear_mpwqe(struct mlx5e_rq *rq,
-				struct mlx5e_mpw_info *wi);
-void mlx5e_free_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
-				    struct mlx5e_mpw_info *wi);
+void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq);
+void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
 struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq);
 
 void mlx5e_rx_am(struct mlx5e_rq *rq);
@@ -808,15 +805,16 @@ static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
 	mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, NULL, cq->wq.cc);
 }
 
-static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
+static inline u32 mlx5e_get_wqe_mtt_offset(struct mlx5e_rq *rq, u16 wqe_ix)
 {
-	return min_t(int, mdev->priv.eq_table.num_comp_vectors,
-		     MLX5E_MAX_NUM_CHANNELS);
+	return rq->mpwqe.mtt_offset +
+		wqe_ix * ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8);
 }
 
-static inline int mlx5e_get_mtt_octw(int npages)
+static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
 {
-	return ALIGN(npages, 8) / 2;
+	return min_t(int, mdev->priv.eq_table.num_comp_vectors,
+		     MLX5E_MAX_NUM_CHANNELS);
 }
 
 extern const struct ethtool_ops mlx5e_ethtool_ops;
@@ -871,6 +869,7 @@ void mlx5e_nic_rep_unload(struct mlx5_eswitch *esw,
 int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv);
 void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv);
 int mlx5e_attr_get(struct net_device *dev, struct switchdev_attr *attr);
+void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
 
 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv);
 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt);
@@ -881,9 +880,12 @@ void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv);
 int mlx5e_close(struct net_device *netdev);
 int mlx5e_open(struct net_device *netdev);
 void mlx5e_update_stats_work(struct work_struct *work);
-void *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
-			  const struct mlx5e_profile *profile, void *ppriv);
+struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
+				       const struct mlx5e_profile *profile,
+				       void *ppriv);
 void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv);
+int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev);
+void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev);
 struct rtnl_link_stats64 *
 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_clock.c b/drivers/net/ethernet/mellanox/mlx5/core/en_clock.c
index 847a8f3ac2b2..13dc388667b6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_clock.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_clock.c
@@ -273,7 +273,7 @@ void mlx5e_timestamp_init(struct mlx5e_priv *priv)
 
 	tstamp->ptp = ptp_clock_register(&tstamp->ptp_info,
 					 &priv->mdev->pdev->dev);
-	if (IS_ERR_OR_NULL(tstamp->ptp)) {
+	if (IS_ERR(tstamp->ptp)) {
 		mlx5_core_warn(priv->mdev, "ptp_clock_register failed %ld\n",
 			       PTR_ERR(tstamp->ptp));
 		tstamp->ptp = NULL;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
index 673043ccd76c..029e856f72a0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
@@ -60,24 +60,27 @@ void mlx5e_destroy_tir(struct mlx5_core_dev *mdev,
 static int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
 			     struct mlx5_core_mkey *mkey)
 {
-	struct mlx5_create_mkey_mbox_in *in;
+	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
+	void *mkc;
+	u32 *in;
 	int err;
 
-	in = mlx5_vzalloc(sizeof(*in));
+	in = mlx5_vzalloc(inlen);
 	if (!in)
 		return -ENOMEM;
 
-	in->seg.flags = MLX5_PERM_LOCAL_WRITE |
-			MLX5_PERM_LOCAL_READ  |
-			MLX5_ACCESS_MODE_PA;
-	in->seg.flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64);
-	in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+	MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_PA);
+	MLX5_SET(mkc, mkc, lw, 1);
+	MLX5_SET(mkc, mkc, lr, 1);
 
-	err = mlx5_core_create_mkey(mdev, mkey, in, sizeof(*in), NULL, NULL,
-				    NULL);
+	MLX5_SET(mkc, mkc, pd, pdn);
+	MLX5_SET(mkc, mkc, length64, 1);
+	MLX5_SET(mkc, mkc, qpn, 0xffffff);
 
-	kvfree(in);
+	err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
 
+	kvfree(in);
 	return err;
 }
 
@@ -139,7 +142,7 @@ int mlx5e_refresh_tirs_self_loopback_enable(struct mlx5_core_dev *mdev)
 	struct mlx5e_tir *tir;
 	void *in;
 	int inlen;
-	int err;
+	int err = 0;
 
 	inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
 	in = mlx5_vzalloc(inlen);
@@ -151,10 +154,11 @@ int mlx5e_refresh_tirs_self_loopback_enable(struct mlx5_core_dev *mdev)
 	list_for_each_entry(tir, &mdev->mlx5e_res.td.tirs_list, list) {
 		err = mlx5_core_modify_tir(mdev, tir->tirn, in, inlen);
 		if (err)
-			return err;
+			goto out;
 	}
 
+out:
 	kvfree(in);
 
-	return 0;
+	return err;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
index caa9a3ccc3f3..762af16ed021 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
@@ -127,29 +127,40 @@ int mlx5e_dcbnl_ieee_setets_core(struct mlx5e_priv *priv, struct ieee_ets *ets)
 	return mlx5_set_port_tc_bw_alloc(mdev, tc_tx_bw);
 }
 
-static int mlx5e_dbcnl_validate_ets(struct ieee_ets *ets)
+static int mlx5e_dbcnl_validate_ets(struct net_device *netdev,
+				    struct ieee_ets *ets)
 {
 	int bw_sum = 0;
 	int i;
 
 	/* Validate Priority */
 	for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
-		if (ets->prio_tc[i] >= MLX5E_MAX_PRIORITY)
+		if (ets->prio_tc[i] >= MLX5E_MAX_PRIORITY) {
+			netdev_err(netdev,
+				   "Failed to validate ETS: priority value greater than max(%d)\n",
+				    MLX5E_MAX_PRIORITY);
 			return -EINVAL;
+		}
 	}
 
 	/* Validate Bandwidth Sum */
 	for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
 		if (ets->tc_tsa[i] == IEEE_8021QAZ_TSA_ETS) {
-			if (!ets->tc_tx_bw[i])
+			if (!ets->tc_tx_bw[i]) {
+				netdev_err(netdev,
+					   "Failed to validate ETS: BW 0 is illegal\n");
 				return -EINVAL;
+			}
 
 			bw_sum += ets->tc_tx_bw[i];
 		}
 	}
 
-	if (bw_sum != 0 && bw_sum != 100)
+	if (bw_sum != 0 && bw_sum != 100) {
+		netdev_err(netdev,
+			   "Failed to validate ETS: BW sum is illegal\n");
 		return -EINVAL;
+	}
 	return 0;
 }
 
@@ -159,7 +170,7 @@ static int mlx5e_dcbnl_ieee_setets(struct net_device *netdev,
 	struct mlx5e_priv *priv = netdev_priv(netdev);
 	int err;
 
-	err = mlx5e_dbcnl_validate_ets(ets);
+	err = mlx5e_dbcnl_validate_ets(netdev, ets);
 	if (err)
 		return err;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 4a3757e60441..27ff401cec20 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -331,7 +331,7 @@ static void mlx5e_get_ethtool_stats(struct net_device *dev,
 	if (mlx5e_query_global_pause_combined(priv)) {
 		for (i = 0; i < NUM_PPORT_PER_PRIO_PFC_COUNTERS; i++) {
 			data[idx++] = MLX5E_READ_CTR64_BE(&priv->stats.pport.per_prio_counters[0],
-							  pport_per_prio_pfc_stats_desc, 0);
+							  pport_per_prio_pfc_stats_desc, i);
 		}
 	}
 
@@ -352,15 +352,61 @@ static void mlx5e_get_ethtool_stats(struct net_device *dev,
 								   sq_stats_desc, j);
 }
 
+static u32 mlx5e_rx_wqes_to_packets(struct mlx5e_priv *priv, int rq_wq_type,
+				    int num_wqe)
+{
+	int packets_per_wqe;
+	int stride_size;
+	int num_strides;
+	int wqe_size;
+
+	if (rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
+		return num_wqe;
+
+	stride_size = 1 << priv->params.mpwqe_log_stride_sz;
+	num_strides = 1 << priv->params.mpwqe_log_num_strides;
+	wqe_size = stride_size * num_strides;
+
+	packets_per_wqe = wqe_size /
+			  ALIGN(ETH_DATA_LEN, stride_size);
+	return (1 << (order_base_2(num_wqe * packets_per_wqe) - 1));
+}
+
+static u32 mlx5e_packets_to_rx_wqes(struct mlx5e_priv *priv, int rq_wq_type,
+				    int num_packets)
+{
+	int packets_per_wqe;
+	int stride_size;
+	int num_strides;
+	int wqe_size;
+	int num_wqes;
+
+	if (rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
+		return num_packets;
+
+	stride_size = 1 << priv->params.mpwqe_log_stride_sz;
+	num_strides = 1 << priv->params.mpwqe_log_num_strides;
+	wqe_size = stride_size * num_strides;
+
+	num_packets = (1 << order_base_2(num_packets));
+
+	packets_per_wqe = wqe_size /
+			  ALIGN(ETH_DATA_LEN, stride_size);
+	num_wqes = DIV_ROUND_UP(num_packets, packets_per_wqe);
+	return 1 << (order_base_2(num_wqes));
+}
+
 static void mlx5e_get_ringparam(struct net_device *dev,
 				struct ethtool_ringparam *param)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	int rq_wq_type = priv->params.rq_wq_type;
 
-	param->rx_max_pending = 1 << mlx5_max_log_rq_size(rq_wq_type);
+	param->rx_max_pending = mlx5e_rx_wqes_to_packets(priv, rq_wq_type,
+							 1 << mlx5_max_log_rq_size(rq_wq_type));
 	param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE;
-	param->rx_pending     = 1 << priv->params.log_rq_size;
+	param->rx_pending = mlx5e_rx_wqes_to_packets(priv, rq_wq_type,
+						     1 << priv->params.log_rq_size);
 	param->tx_pending     = 1 << priv->params.log_sq_size;
 }
 
@@ -370,9 +416,13 @@ static int mlx5e_set_ringparam(struct net_device *dev,
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	bool was_opened;
 	int rq_wq_type = priv->params.rq_wq_type;
+	u32 rx_pending_wqes;
+	u32 min_rq_size;
+	u32 max_rq_size;
 	u16 min_rx_wqes;
 	u8 log_rq_size;
 	u8 log_sq_size;
+	u32 num_mtts;
 	int err = 0;
 
 	if (param->rx_jumbo_pending) {
@@ -385,18 +435,36 @@ static int mlx5e_set_ringparam(struct net_device *dev,
 			    __func__);
 		return -EINVAL;
 	}
-	if (param->rx_pending < (1 << mlx5_min_log_rq_size(rq_wq_type))) {
+
+	min_rq_size = mlx5e_rx_wqes_to_packets(priv, rq_wq_type,
+					       1 << mlx5_min_log_rq_size(rq_wq_type));
+	max_rq_size = mlx5e_rx_wqes_to_packets(priv, rq_wq_type,
+					       1 << mlx5_max_log_rq_size(rq_wq_type));
+	rx_pending_wqes = mlx5e_packets_to_rx_wqes(priv, rq_wq_type,
+						   param->rx_pending);
+
+	if (param->rx_pending < min_rq_size) {
 		netdev_info(dev, "%s: rx_pending (%d) < min (%d)\n",
 			    __func__, param->rx_pending,
-			    1 << mlx5_min_log_rq_size(rq_wq_type));
+			    min_rq_size);
 		return -EINVAL;
 	}
-	if (param->rx_pending > (1 << mlx5_max_log_rq_size(rq_wq_type))) {
+	if (param->rx_pending > max_rq_size) {
 		netdev_info(dev, "%s: rx_pending (%d) > max (%d)\n",
 			    __func__, param->rx_pending,
-			    1 << mlx5_max_log_rq_size(rq_wq_type));
+			    max_rq_size);
+		return -EINVAL;
+	}
+
+	num_mtts = MLX5E_REQUIRED_MTTS(priv->params.num_channels,
+				       rx_pending_wqes);
+	if (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
+	    !MLX5E_VALID_NUM_MTTS(num_mtts)) {
+		netdev_info(dev, "%s: rx_pending (%d) request can't be satisfied, try to reduce.\n",
+			    __func__, param->rx_pending);
 		return -EINVAL;
 	}
+
 	if (param->tx_pending < (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)) {
 		netdev_info(dev, "%s: tx_pending (%d) < min (%d)\n",
 			    __func__, param->tx_pending,
@@ -410,9 +478,9 @@ static int mlx5e_set_ringparam(struct net_device *dev,
 		return -EINVAL;
 	}
 
-	log_rq_size = order_base_2(param->rx_pending);
+	log_rq_size = order_base_2(rx_pending_wqes);
 	log_sq_size = order_base_2(param->tx_pending);
-	min_rx_wqes = mlx5_min_rx_wqes(rq_wq_type, param->rx_pending);
+	min_rx_wqes = mlx5_min_rx_wqes(rq_wq_type, rx_pending_wqes);
 
 	if (log_rq_size == priv->params.log_rq_size &&
 	    log_sq_size == priv->params.log_sq_size &&
@@ -454,6 +522,7 @@ static int mlx5e_set_channels(struct net_device *dev,
 	unsigned int count = ch->combined_count;
 	bool arfs_enabled;
 	bool was_opened;
+	u32 num_mtts;
 	int err = 0;
 
 	if (!count) {
@@ -472,6 +541,14 @@ static int mlx5e_set_channels(struct net_device *dev,
 		return -EINVAL;
 	}
 
+	num_mtts = MLX5E_REQUIRED_MTTS(count, BIT(priv->params.log_rq_size));
+	if (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
+	    !MLX5E_VALID_NUM_MTTS(num_mtts)) {
+		netdev_info(dev, "%s: rx count (%d) request can't be satisfied, try to reduce.\n",
+			    __func__, count);
+		return -EINVAL;
+	}
+
 	if (priv->params.num_channels == count)
 		return 0;
 
@@ -582,9 +659,10 @@ out:
 static void ptys2ethtool_supported_link(unsigned long *supported_modes,
 					u32 eth_proto_cap)
 {
+	unsigned long proto_cap = eth_proto_cap;
 	int proto;
 
-	for_each_set_bit(proto, (unsigned long *)&eth_proto_cap, MLX5E_LINK_MODES_NUMBER)
+	for_each_set_bit(proto, &proto_cap, MLX5E_LINK_MODES_NUMBER)
 		bitmap_or(supported_modes, supported_modes,
 			  ptys2ethtool_table[proto].supported,
 			  __ETHTOOL_LINK_MODE_MASK_NBITS);
@@ -593,9 +671,10 @@ static void ptys2ethtool_supported_link(unsigned long *supported_modes,
 static void ptys2ethtool_adver_link(unsigned long *advertising_modes,
 				    u32 eth_proto_cap)
 {
+	unsigned long proto_cap = eth_proto_cap;
 	int proto;
 
-	for_each_set_bit(proto, (unsigned long *)&eth_proto_cap, MLX5E_LINK_MODES_NUMBER)
+	for_each_set_bit(proto, &proto_cap, MLX5E_LINK_MODES_NUMBER)
 		bitmap_or(advertising_modes, advertising_modes,
 			  ptys2ethtool_table[proto].advertised,
 			  __ETHTOOL_LINK_MODE_MASK_NBITS);
@@ -726,7 +805,7 @@ static int mlx5e_get_link_ksettings(struct net_device *netdev,
 {
 	struct mlx5e_priv *priv    = netdev_priv(netdev);
 	struct mlx5_core_dev *mdev = priv->mdev;
-	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
+	u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0};
 	u32 eth_proto_cap;
 	u32 eth_proto_admin;
 	u32 eth_proto_lp;
@@ -736,7 +815,6 @@ static int mlx5e_get_link_ksettings(struct net_device *netdev,
 	int err;
 
 	err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1);
-
 	if (err) {
 		netdev_err(netdev, "%s: query port ptys failed: %d\n",
 			   __func__, err);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index 1587a9fd5724..36fbc6b21a33 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -294,6 +294,36 @@ int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __always_unused __be16 proto,
 	return 0;
 }
 
+static void mlx5e_add_vlan_rules(struct mlx5e_priv *priv)
+{
+	int i;
+
+	mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
+
+	for_each_set_bit(i, priv->fs.vlan.active_vlans, VLAN_N_VID) {
+		mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, i);
+	}
+
+	if (priv->fs.vlan.filter_disabled &&
+	    !(priv->netdev->flags & IFF_PROMISC))
+		mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID, 0);
+}
+
+static void mlx5e_del_vlan_rules(struct mlx5e_priv *priv)
+{
+	int i;
+
+	mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
+
+	for_each_set_bit(i, priv->fs.vlan.active_vlans, VLAN_N_VID) {
+		mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, i);
+	}
+
+	if (priv->fs.vlan.filter_disabled &&
+	    !(priv->netdev->flags & IFF_PROMISC))
+		mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID, 0);
+}
+
 #define mlx5e_for_each_hash_node(hn, tmp, hash, i) \
 	for (i = 0; i < MLX5E_L2_ADDR_HASH_SIZE; i++) \
 		hlist_for_each_entry_safe(hn, tmp, &hash[i], hlist)
@@ -1024,14 +1054,10 @@ static int mlx5e_create_vlan_table(struct mlx5e_priv *priv)
 	if (err)
 		goto err_free_g;
 
-	err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
-	if (err)
-		goto err_destroy_vlan_flow_groups;
+	mlx5e_add_vlan_rules(priv);
 
 	return 0;
 
-err_destroy_vlan_flow_groups:
-	mlx5e_destroy_groups(ft);
 err_free_g:
 	kfree(ft->g);
 err_destroy_vlan_table:
@@ -1043,6 +1069,7 @@ err_destroy_vlan_table:
 
 static void mlx5e_destroy_vlan_table(struct mlx5e_priv *priv)
 {
+	mlx5e_del_vlan_rules(priv);
 	mlx5e_destroy_flow_table(&priv->fs.vlan.ft);
 }
 
@@ -1100,7 +1127,6 @@ err_destroy_arfs_tables:
 
 void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv)
 {
-	mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
 	mlx5e_destroy_vlan_table(priv);
 	mlx5e_destroy_l2_table(priv);
 	mlx5e_destroy_ttc_table(priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 870bea37c57c..7eaf38020a8f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -34,18 +34,12 @@
 #include <net/pkt_cls.h>
 #include <linux/mlx5/fs.h>
 #include <net/vxlan.h>
+#include <linux/bpf.h>
 #include "en.h"
 #include "en_tc.h"
 #include "eswitch.h"
 #include "vxlan.h"
 
-enum {
-	MLX5_EN_QP_FLUSH_TIMEOUT_MS	= 5000,
-	MLX5_EN_QP_FLUSH_MSLEEP_QUANT	= 20,
-	MLX5_EN_QP_FLUSH_MAX_ITER	= MLX5_EN_QP_FLUSH_TIMEOUT_MS /
-					  MLX5_EN_QP_FLUSH_MSLEEP_QUANT,
-};
-
 struct mlx5e_rq_param {
 	u32			rqc[MLX5_ST_SZ_DW(rqc)];
 	struct mlx5_wq_param	wq;
@@ -57,7 +51,7 @@ struct mlx5e_sq_param {
 	struct mlx5_wq_param       wq;
 	u16                        max_inline;
 	u8                         min_inline_mode;
-	bool                       icosq;
+	enum mlx5e_sq_type         type;
 };
 
 struct mlx5e_cq_param {
@@ -70,12 +64,55 @@ struct mlx5e_cq_param {
 struct mlx5e_channel_param {
 	struct mlx5e_rq_param      rq;
 	struct mlx5e_sq_param      sq;
+	struct mlx5e_sq_param      xdp_sq;
 	struct mlx5e_sq_param      icosq;
 	struct mlx5e_cq_param      rx_cq;
 	struct mlx5e_cq_param      tx_cq;
 	struct mlx5e_cq_param      icosq_cq;
 };
 
+static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
+{
+	return MLX5_CAP_GEN(mdev, striding_rq) &&
+		MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
+		MLX5_CAP_ETH(mdev, reg_umr_sq);
+}
+
+static void mlx5e_set_rq_type_params(struct mlx5e_priv *priv, u8 rq_type)
+{
+	priv->params.rq_wq_type = rq_type;
+	switch (priv->params.rq_wq_type) {
+	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
+		priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
+		priv->params.mpwqe_log_stride_sz = priv->params.rx_cqe_compress ?
+			MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS :
+			MLX5_MPWRQ_LOG_STRIDE_SIZE;
+		priv->params.mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
+			priv->params.mpwqe_log_stride_sz;
+		break;
+	default: /* MLX5_WQ_TYPE_LINKED_LIST */
+		priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
+	}
+	priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
+					       BIT(priv->params.log_rq_size));
+
+	mlx5_core_info(priv->mdev,
+		       "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
+		       priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
+		       BIT(priv->params.log_rq_size),
+		       BIT(priv->params.mpwqe_log_stride_sz),
+		       priv->params.rx_cqe_compress_admin);
+}
+
+static void mlx5e_set_rq_priv_params(struct mlx5e_priv *priv)
+{
+	u8 rq_type = mlx5e_check_fragmented_striding_rq_cap(priv->mdev) &&
+		    !priv->xdp_prog ?
+		    MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
+		    MLX5_WQ_TYPE_LINKED_LIST;
+	mlx5e_set_rq_type_params(priv, rq_type);
+}
+
 static void mlx5e_update_carrier(struct mlx5e_priv *priv)
 {
 	struct mlx5_core_dev *mdev = priv->mdev;
@@ -143,12 +180,18 @@ static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
 		s->rx_csum_none	+= rq_stats->csum_none;
 		s->rx_csum_complete += rq_stats->csum_complete;
 		s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
+		s->rx_xdp_drop += rq_stats->xdp_drop;
+		s->rx_xdp_tx += rq_stats->xdp_tx;
+		s->rx_xdp_tx_full += rq_stats->xdp_tx_full;
 		s->rx_wqe_err   += rq_stats->wqe_err;
 		s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
-		s->rx_mpwqe_frag   += rq_stats->mpwqe_frag;
 		s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
 		s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
 		s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
+		s->rx_cache_reuse += rq_stats->cache_reuse;
+		s->rx_cache_full  += rq_stats->cache_full;
+		s->rx_cache_empty += rq_stats->cache_empty;
+		s->rx_cache_busy  += rq_stats->cache_busy;
 
 		for (j = 0; j < priv->params.num_tc; j++) {
 			sq_stats = &priv->channel[i]->sq[j].stats;
@@ -162,6 +205,7 @@ static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
 			s->tx_queue_stopped	+= sq_stats->stopped;
 			s->tx_queue_wake	+= sq_stats->wake;
 			s->tx_queue_dropped	+= sq_stats->dropped;
+			s->tx_xmit_more		+= sq_stats->xmit_more;
 			s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
 			tx_offload_none		+= sq_stats->csum_none;
 		}
@@ -180,18 +224,15 @@ static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
 {
 	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 	u32 *out = (u32 *)priv->stats.vport.query_vport_out;
-	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
+	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
 	struct mlx5_core_dev *mdev = priv->mdev;
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(query_vport_counter_in, in, opcode,
 		 MLX5_CMD_OP_QUERY_VPORT_COUNTER);
 	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
 	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
 
 	memset(out, 0, outlen);
-
 	mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
 }
 
@@ -304,6 +345,117 @@ static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 #define MLX5E_HW2SW_MTU(hwmtu) (hwmtu - (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 #define MLX5E_SW2HW_MTU(swmtu) (swmtu + (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
 
+static inline int mlx5e_get_wqe_mtt_sz(void)
+{
+	/* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
+	 * To avoid copying garbage after the mtt array, we allocate
+	 * a little more.
+	 */
+	return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
+		     MLX5_UMR_MTT_ALIGNMENT);
+}
+
+static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, struct mlx5e_sq *sq,
+				       struct mlx5e_umr_wqe *wqe, u16 ix)
+{
+	struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
+	struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
+	struct mlx5_wqe_data_seg      *dseg = &wqe->data;
+	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
+	u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
+	u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
+
+	cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
+				      ds_cnt);
+	cseg->fm_ce_se  = MLX5_WQE_CTRL_CQ_UPDATE;
+	cseg->imm       = rq->mkey_be;
+
+	ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
+	ucseg->klm_octowords =
+		cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
+	ucseg->bsf_octowords =
+		cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
+	ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
+
+	dseg->lkey = sq->mkey_be;
+	dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
+}
+
+static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
+				     struct mlx5e_channel *c)
+{
+	int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
+	int mtt_sz = mlx5e_get_wqe_mtt_sz();
+	int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
+	int i;
+
+	rq->mpwqe.info = kzalloc_node(wq_sz * sizeof(*rq->mpwqe.info),
+				      GFP_KERNEL, cpu_to_node(c->cpu));
+	if (!rq->mpwqe.info)
+		goto err_out;
+
+	/* We allocate more than mtt_sz as we will align the pointer */
+	rq->mpwqe.mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
+					cpu_to_node(c->cpu));
+	if (unlikely(!rq->mpwqe.mtt_no_align))
+		goto err_free_wqe_info;
+
+	for (i = 0; i < wq_sz; i++) {
+		struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
+
+		wi->umr.mtt = PTR_ALIGN(rq->mpwqe.mtt_no_align + i * mtt_alloc,
+					MLX5_UMR_ALIGN);
+		wi->umr.mtt_addr = dma_map_single(c->pdev, wi->umr.mtt, mtt_sz,
+						  PCI_DMA_TODEVICE);
+		if (unlikely(dma_mapping_error(c->pdev, wi->umr.mtt_addr)))
+			goto err_unmap_mtts;
+
+		mlx5e_build_umr_wqe(rq, &c->icosq, &wi->umr.wqe, i);
+	}
+
+	return 0;
+
+err_unmap_mtts:
+	while (--i >= 0) {
+		struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
+
+		dma_unmap_single(c->pdev, wi->umr.mtt_addr, mtt_sz,
+				 PCI_DMA_TODEVICE);
+	}
+	kfree(rq->mpwqe.mtt_no_align);
+err_free_wqe_info:
+	kfree(rq->mpwqe.info);
+
+err_out:
+	return -ENOMEM;
+}
+
+static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
+{
+	int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
+	int mtt_sz = mlx5e_get_wqe_mtt_sz();
+	int i;
+
+	for (i = 0; i < wq_sz; i++) {
+		struct mlx5e_mpw_info *wi = &rq->mpwqe.info[i];
+
+		dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz,
+				 PCI_DMA_TODEVICE);
+	}
+	kfree(rq->mpwqe.mtt_no_align);
+	kfree(rq->mpwqe.info);
+}
+
+static bool mlx5e_is_vf_vport_rep(struct mlx5e_priv *priv)
+{
+	struct mlx5_eswitch_rep *rep = (struct mlx5_eswitch_rep *)priv->ppriv;
+
+	if (rep && rep->vport != FDB_UPLINK_VPORT)
+		return true;
+
+	return false;
+}
+
 static int mlx5e_create_rq(struct mlx5e_channel *c,
 			   struct mlx5e_rq_param *param,
 			   struct mlx5e_rq *rq)
@@ -313,6 +465,8 @@ static int mlx5e_create_rq(struct mlx5e_channel *c,
 	void *rqc = param->rqc;
 	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
 	u32 byte_count;
+	u32 frag_sz;
+	int npages;
 	int wq_sz;
 	int err;
 	int i;
@@ -328,60 +482,92 @@ static int mlx5e_create_rq(struct mlx5e_channel *c,
 
 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 
+	rq->wq_type = priv->params.rq_wq_type;
+	rq->pdev    = c->pdev;
+	rq->netdev  = c->netdev;
+	rq->tstamp  = &priv->tstamp;
+	rq->channel = c;
+	rq->ix      = c->ix;
+	rq->priv    = c->priv;
+	rq->xdp_prog = priv->xdp_prog;
+
+	rq->buff.map_dir = DMA_FROM_DEVICE;
+	if (rq->xdp_prog)
+		rq->buff.map_dir = DMA_BIDIRECTIONAL;
+
 	switch (priv->params.rq_wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
-		rq->wqe_info = kzalloc_node(wq_sz * sizeof(*rq->wqe_info),
-					    GFP_KERNEL, cpu_to_node(c->cpu));
-		if (!rq->wqe_info) {
-			err = -ENOMEM;
+		if (mlx5e_is_vf_vport_rep(priv)) {
+			err = -EINVAL;
 			goto err_rq_wq_destroy;
 		}
+
 		rq->handle_rx_cqe = mlx5e_handle_rx_cqe_mpwrq;
 		rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
 		rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
 
+		rq->mpwqe.mtt_offset = c->ix *
+			MLX5E_REQUIRED_MTTS(1, BIT(priv->params.log_rq_size));
+
 		rq->mpwqe_stride_sz = BIT(priv->params.mpwqe_log_stride_sz);
 		rq->mpwqe_num_strides = BIT(priv->params.mpwqe_log_num_strides);
-		rq->wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
-		byte_count = rq->wqe_sz;
+
+		rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
+		byte_count = rq->buff.wqe_sz;
+		rq->mkey_be = cpu_to_be32(c->priv->umr_mkey.key);
+		err = mlx5e_rq_alloc_mpwqe_info(rq, c);
+		if (err)
+			goto err_rq_wq_destroy;
 		break;
 	default: /* MLX5_WQ_TYPE_LINKED_LIST */
-		rq->skb = kzalloc_node(wq_sz * sizeof(*rq->skb), GFP_KERNEL,
-				       cpu_to_node(c->cpu));
-		if (!rq->skb) {
+		rq->dma_info = kzalloc_node(wq_sz * sizeof(*rq->dma_info),
+					    GFP_KERNEL, cpu_to_node(c->cpu));
+		if (!rq->dma_info) {
 			err = -ENOMEM;
 			goto err_rq_wq_destroy;
 		}
-		rq->handle_rx_cqe = mlx5e_handle_rx_cqe;
+
+		if (mlx5e_is_vf_vport_rep(priv))
+			rq->handle_rx_cqe = mlx5e_handle_rx_cqe_rep;
+		else
+			rq->handle_rx_cqe = mlx5e_handle_rx_cqe;
+
 		rq->alloc_wqe = mlx5e_alloc_rx_wqe;
 		rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 
-		rq->wqe_sz = (priv->params.lro_en) ?
+		rq->buff.wqe_sz = (priv->params.lro_en) ?
 				priv->params.lro_wqe_sz :
 				MLX5E_SW2HW_MTU(priv->netdev->mtu);
-		rq->wqe_sz = SKB_DATA_ALIGN(rq->wqe_sz);
-		byte_count = rq->wqe_sz;
+		byte_count = rq->buff.wqe_sz;
+
+		/* calc the required page order */
+		frag_sz = MLX5_RX_HEADROOM +
+			  byte_count /* packet data */ +
+			  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+		frag_sz = SKB_DATA_ALIGN(frag_sz);
+
+		npages = DIV_ROUND_UP(frag_sz, PAGE_SIZE);
+		rq->buff.page_order = order_base_2(npages);
+
 		byte_count |= MLX5_HW_START_PADDING;
+		rq->mkey_be = c->mkey_be;
 	}
 
 	for (i = 0; i < wq_sz; i++) {
 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
 
 		wqe->data.byte_count = cpu_to_be32(byte_count);
+		wqe->data.lkey = rq->mkey_be;
 	}
 
 	INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
 	rq->am.mode = priv->params.rx_cq_period_mode;
 
-	rq->wq_type = priv->params.rq_wq_type;
-	rq->pdev    = c->pdev;
-	rq->netdev  = c->netdev;
-	rq->tstamp  = &priv->tstamp;
-	rq->channel = c;
-	rq->ix      = c->ix;
-	rq->priv    = c->priv;
-	rq->mkey_be = c->mkey_be;
-	rq->umr_mkey_be = cpu_to_be32(c->priv->umr_mkey.key);
+	rq->page_cache.head = 0;
+	rq->page_cache.tail = 0;
+
+	if (rq->xdp_prog)
+		bpf_prog_add(rq->xdp_prog, 1);
 
 	return 0;
 
@@ -393,14 +579,25 @@ err_rq_wq_destroy:
 
 static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
 {
+	int i;
+
+	if (rq->xdp_prog)
+		bpf_prog_put(rq->xdp_prog);
+
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
-		kfree(rq->wqe_info);
+		mlx5e_rq_free_mpwqe_info(rq);
 		break;
 	default: /* MLX5_WQ_TYPE_LINKED_LIST */
-		kfree(rq->skb);
+		kfree(rq->dma_info);
 	}
 
+	for (i = rq->page_cache.head; i != rq->page_cache.tail;
+	     i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
+		struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
+
+		mlx5e_page_release(rq, dma_info, false);
+	}
 	mlx5_wq_destroy(&rq->wq_ctrl);
 }
 
@@ -428,7 +625,6 @@ static int mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 
 	MLX5_SET(rqc,  rqc, cqn,		rq->cq.mcq.cqn);
 	MLX5_SET(rqc,  rqc, state,		MLX5_RQC_STATE_RST);
-	MLX5_SET(rqc,  rqc, flush_in_error_en,	1);
 	MLX5_SET(rqc,  rqc, vsd, priv->params.vlan_strip_disable);
 	MLX5_SET(wq,   wq,  log_wq_pg_sz,	rq->wq_ctrl.buf.page_shift -
 						MLX5_ADAPTER_PAGE_SHIFT);
@@ -492,7 +688,8 @@ static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 
 	MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
-	MLX5_SET64(modify_rq_in, in, modify_bitmask, MLX5_RQ_BITMASK_VSD);
+	MLX5_SET64(modify_rq_in, in, modify_bitmask,
+		   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
 	MLX5_SET(rqc, rqc, vsd, vsd);
 	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
 
@@ -525,6 +722,27 @@ static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
 	return -ETIMEDOUT;
 }
 
+static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
+{
+	struct mlx5_wq_ll *wq = &rq->wq;
+	struct mlx5e_rx_wqe *wqe;
+	__be16 wqe_ix_be;
+	u16 wqe_ix;
+
+	/* UMR WQE (if in progress) is always at wq->head */
+	if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
+		mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
+
+	while (!mlx5_wq_ll_is_empty(wq)) {
+		wqe_ix_be = *wq->tail_next;
+		wqe_ix    = be16_to_cpu(wqe_ix_be);
+		wqe       = mlx5_wq_ll_get_wqe(&rq->wq, wqe_ix);
+		rq->dealloc_wqe(rq, wqe_ix);
+		mlx5_wq_ll_pop(&rq->wq, wqe_ix_be,
+			       &wqe->next.next_wqe_index);
+	}
+}
+
 static int mlx5e_open_rq(struct mlx5e_channel *c,
 			 struct mlx5e_rq_param *param,
 			 struct mlx5e_rq *rq)
@@ -548,10 +766,8 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
 	if (param->am_enabled)
 		set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
 
-	set_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE, &rq->state);
-
-	sq->ico_wqe_info[pi].opcode     = MLX5_OPCODE_NOP;
-	sq->ico_wqe_info[pi].num_wqebbs = 1;
+	sq->db.ico_wqe[pi].opcode     = MLX5_OPCODE_NOP;
+	sq->db.ico_wqe[pi].num_wqebbs = 1;
 	mlx5e_send_nop(sq, true); /* trigger mlx5e_post_rx_wqes() */
 
 	return 0;
@@ -566,23 +782,8 @@ err_destroy_rq:
 
 static void mlx5e_close_rq(struct mlx5e_rq *rq)
 {
-	int tout = 0;
-	int err;
-
-	clear_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE, &rq->state);
+	set_bit(MLX5E_RQ_STATE_FLUSH, &rq->state);
 	napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
-
-	err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
-	while (!mlx5_wq_ll_is_empty(&rq->wq) && !err &&
-	       tout++ < MLX5_EN_QP_FLUSH_MAX_ITER)
-		msleep(MLX5_EN_QP_FLUSH_MSLEEP_QUANT);
-
-	if (err || tout == MLX5_EN_QP_FLUSH_MAX_ITER)
-		set_bit(MLX5E_RQ_STATE_FLUSH_TIMEOUT, &rq->state);
-
-	/* avoid destroying rq before mlx5e_poll_rx_cq() is done with it */
-	napi_synchronize(&rq->channel->napi);
-
 	cancel_work_sync(&rq->am.work);
 
 	mlx5e_disable_rq(rq);
@@ -590,26 +791,65 @@ static void mlx5e_close_rq(struct mlx5e_rq *rq)
 	mlx5e_destroy_rq(rq);
 }
 
-static void mlx5e_free_sq_db(struct mlx5e_sq *sq)
+static void mlx5e_free_sq_xdp_db(struct mlx5e_sq *sq)
 {
-	kfree(sq->wqe_info);
-	kfree(sq->dma_fifo);
-	kfree(sq->skb);
+	kfree(sq->db.xdp.di);
+	kfree(sq->db.xdp.wqe_info);
 }
 
-static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)
+static int mlx5e_alloc_sq_xdp_db(struct mlx5e_sq *sq, int numa)
 {
 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
-	int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
 
-	sq->skb = kzalloc_node(wq_sz * sizeof(*sq->skb), GFP_KERNEL, numa);
-	sq->dma_fifo = kzalloc_node(df_sz * sizeof(*sq->dma_fifo), GFP_KERNEL,
-				    numa);
-	sq->wqe_info = kzalloc_node(wq_sz * sizeof(*sq->wqe_info), GFP_KERNEL,
-				    numa);
+	sq->db.xdp.di = kzalloc_node(sizeof(*sq->db.xdp.di) * wq_sz,
+				     GFP_KERNEL, numa);
+	sq->db.xdp.wqe_info = kzalloc_node(sizeof(*sq->db.xdp.wqe_info) * wq_sz,
+					   GFP_KERNEL, numa);
+	if (!sq->db.xdp.di || !sq->db.xdp.wqe_info) {
+		mlx5e_free_sq_xdp_db(sq);
+		return -ENOMEM;
+	}
 
-	if (!sq->skb || !sq->dma_fifo || !sq->wqe_info) {
-		mlx5e_free_sq_db(sq);
+	return 0;
+}
+
+static void mlx5e_free_sq_ico_db(struct mlx5e_sq *sq)
+{
+	kfree(sq->db.ico_wqe);
+}
+
+static int mlx5e_alloc_sq_ico_db(struct mlx5e_sq *sq, int numa)
+{
+	u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
+
+	sq->db.ico_wqe = kzalloc_node(sizeof(*sq->db.ico_wqe) * wq_sz,
+				      GFP_KERNEL, numa);
+	if (!sq->db.ico_wqe)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void mlx5e_free_sq_txq_db(struct mlx5e_sq *sq)
+{
+	kfree(sq->db.txq.wqe_info);
+	kfree(sq->db.txq.dma_fifo);
+	kfree(sq->db.txq.skb);
+}
+
+static int mlx5e_alloc_sq_txq_db(struct mlx5e_sq *sq, int numa)
+{
+	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
+	int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
+
+	sq->db.txq.skb = kzalloc_node(wq_sz * sizeof(*sq->db.txq.skb),
+				      GFP_KERNEL, numa);
+	sq->db.txq.dma_fifo = kzalloc_node(df_sz * sizeof(*sq->db.txq.dma_fifo),
+					   GFP_KERNEL, numa);
+	sq->db.txq.wqe_info = kzalloc_node(wq_sz * sizeof(*sq->db.txq.wqe_info),
+					   GFP_KERNEL, numa);
+	if (!sq->db.txq.skb || !sq->db.txq.dma_fifo || !sq->db.txq.wqe_info) {
+		mlx5e_free_sq_txq_db(sq);
 		return -ENOMEM;
 	}
 
@@ -618,6 +858,46 @@ static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)
 	return 0;
 }
 
+static void mlx5e_free_sq_db(struct mlx5e_sq *sq)
+{
+	switch (sq->type) {
+	case MLX5E_SQ_TXQ:
+		mlx5e_free_sq_txq_db(sq);
+		break;
+	case MLX5E_SQ_ICO:
+		mlx5e_free_sq_ico_db(sq);
+		break;
+	case MLX5E_SQ_XDP:
+		mlx5e_free_sq_xdp_db(sq);
+		break;
+	}
+}
+
+static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)
+{
+	switch (sq->type) {
+	case MLX5E_SQ_TXQ:
+		return mlx5e_alloc_sq_txq_db(sq, numa);
+	case MLX5E_SQ_ICO:
+		return mlx5e_alloc_sq_ico_db(sq, numa);
+	case MLX5E_SQ_XDP:
+		return mlx5e_alloc_sq_xdp_db(sq, numa);
+	}
+
+	return 0;
+}
+
+static int mlx5e_sq_get_max_wqebbs(u8 sq_type)
+{
+	switch (sq_type) {
+	case MLX5E_SQ_ICO:
+		return MLX5E_ICOSQ_MAX_WQEBBS;
+	case MLX5E_SQ_XDP:
+		return MLX5E_XDP_TX_WQEBBS;
+	}
+	return MLX5_SEND_WQE_MAX_WQEBBS;
+}
+
 static int mlx5e_create_sq(struct mlx5e_channel *c,
 			   int tc,
 			   struct mlx5e_sq_param *param,
@@ -630,6 +910,13 @@ static int mlx5e_create_sq(struct mlx5e_channel *c,
 	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
 	int err;
 
+	sq->type      = param->type;
+	sq->pdev      = c->pdev;
+	sq->tstamp    = &priv->tstamp;
+	sq->mkey_be   = c->mkey_be;
+	sq->channel   = c;
+	sq->tc        = tc;
+
 	err = mlx5_alloc_map_uar(mdev, &sq->uar, !!MLX5_CAP_GEN(mdev, bf));
 	if (err)
 		return err;
@@ -658,18 +945,7 @@ static int mlx5e_create_sq(struct mlx5e_channel *c,
 	if (err)
 		goto err_sq_wq_destroy;
 
-	if (param->icosq) {
-		u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
-
-		sq->ico_wqe_info = kzalloc_node(sizeof(*sq->ico_wqe_info) *
-						wq_sz,
-						GFP_KERNEL,
-						cpu_to_node(c->cpu));
-		if (!sq->ico_wqe_info) {
-			err = -ENOMEM;
-			goto err_free_sq_db;
-		}
-	} else {
+	if (sq->type == MLX5E_SQ_TXQ) {
 		int txq_ix;
 
 		txq_ix = c->ix + tc * priv->params.num_channels;
@@ -677,19 +953,11 @@ static int mlx5e_create_sq(struct mlx5e_channel *c,
 		priv->txq_to_sq_map[txq_ix] = sq;
 	}
 
-	sq->pdev      = c->pdev;
-	sq->tstamp    = &priv->tstamp;
-	sq->mkey_be   = c->mkey_be;
-	sq->channel   = c;
-	sq->tc        = tc;
-	sq->edge      = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;
+	sq->edge = (sq->wq.sz_m1 + 1) - mlx5e_sq_get_max_wqebbs(sq->type);
 	sq->bf_budget = MLX5E_SQ_BF_BUDGET;
 
 	return 0;
 
-err_free_sq_db:
-	mlx5e_free_sq_db(sq);
-
 err_sq_wq_destroy:
 	mlx5_wq_destroy(&sq->wq_ctrl);
 
@@ -704,7 +972,6 @@ static void mlx5e_destroy_sq(struct mlx5e_sq *sq)
 	struct mlx5e_channel *c = sq->channel;
 	struct mlx5e_priv *priv = c->priv;
 
-	kfree(sq->ico_wqe_info);
 	mlx5e_free_sq_db(sq);
 	mlx5_wq_destroy(&sq->wq_ctrl);
 	mlx5_unmap_free_uar(priv->mdev, &sq->uar);
@@ -733,11 +1000,12 @@ static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
 
 	memcpy(sqc, param->sqc, sizeof(param->sqc));
 
-	MLX5_SET(sqc,  sqc, tis_num_0, param->icosq ? 0 : priv->tisn[sq->tc]);
+	MLX5_SET(sqc,  sqc, tis_num_0, param->type == MLX5E_SQ_ICO ?
+				       0 : priv->tisn[sq->tc]);
 	MLX5_SET(sqc,  sqc, cqn,		sq->cq.mcq.cqn);
 	MLX5_SET(sqc,  sqc, min_wqe_inline_mode, sq->min_inline_mode);
 	MLX5_SET(sqc,  sqc, state,		MLX5_SQC_STATE_RST);
-	MLX5_SET(sqc,  sqc, tis_lst_sz,		param->icosq ? 0 : 1);
+	MLX5_SET(sqc,  sqc, tis_lst_sz, param->type == MLX5E_SQ_ICO ? 0 : 1);
 	MLX5_SET(sqc,  sqc, flush_in_error_en,	1);
 
 	MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
@@ -821,7 +1089,6 @@ static int mlx5e_open_sq(struct mlx5e_channel *c,
 		goto err_disable_sq;
 
 	if (sq->txq) {
-		set_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state);
 		netdev_tx_reset_queue(sq->txq);
 		netif_tx_start_queue(sq->txq);
 	}
@@ -845,38 +1112,22 @@ static inline void netif_tx_disable_queue(struct netdev_queue *txq)
 
 static void mlx5e_close_sq(struct mlx5e_sq *sq)
 {
-	int tout = 0;
-	int err;
+	set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
+	/* prevent netif_tx_wake_queue */
+	napi_synchronize(&sq->channel->napi);
 
 	if (sq->txq) {
-		clear_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state);
-		/* prevent netif_tx_wake_queue */
-		napi_synchronize(&sq->channel->napi);
 		netif_tx_disable_queue(sq->txq);
 
-		/* ensure hw is notified of all pending wqes */
-		if (mlx5e_sq_has_room_for(sq, 1))
+		/* last doorbell out, godspeed .. */
+		if (mlx5e_sq_has_room_for(sq, 1)) {
+			sq->db.txq.skb[(sq->pc & sq->wq.sz_m1)] = NULL;
 			mlx5e_send_nop(sq, true);
-
-		err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY,
-				      MLX5_SQC_STATE_ERR, false, 0);
-		if (err)
-			set_bit(MLX5E_SQ_STATE_TX_TIMEOUT, &sq->state);
-	}
-
-	/* wait till sq is empty, unless a TX timeout occurred on this SQ */
-	while (sq->cc != sq->pc &&
-	       !test_bit(MLX5E_SQ_STATE_TX_TIMEOUT, &sq->state)) {
-		msleep(MLX5_EN_QP_FLUSH_MSLEEP_QUANT);
-		if (tout++ > MLX5_EN_QP_FLUSH_MAX_ITER)
-			set_bit(MLX5E_SQ_STATE_TX_TIMEOUT, &sq->state);
+		}
 	}
 
-	/* avoid destroying sq before mlx5e_poll_tx_cq() is done with it */
-	napi_synchronize(&sq->channel->napi);
-
-	mlx5e_free_tx_descs(sq);
 	mlx5e_disable_sq(sq);
+	mlx5e_free_sq_descs(sq);
 	mlx5e_destroy_sq(sq);
 }
 
@@ -1237,14 +1488,31 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 		}
 	}
 
+	if (priv->xdp_prog) {
+		/* XDP SQ CQ params are same as normal TXQ sq CQ params */
+		err = mlx5e_open_cq(c, &cparam->tx_cq, &c->xdp_sq.cq,
+				    priv->params.tx_cq_moderation);
+		if (err)
+			goto err_close_sqs;
+
+		err = mlx5e_open_sq(c, 0, &cparam->xdp_sq, &c->xdp_sq);
+		if (err) {
+			mlx5e_close_cq(&c->xdp_sq.cq);
+			goto err_close_sqs;
+		}
+	}
+
+	c->xdp = !!priv->xdp_prog;
 	err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
 	if (err)
-		goto err_close_sqs;
+		goto err_close_xdp_sq;
 
 	netif_set_xps_queue(netdev, get_cpu_mask(c->cpu), ix);
 	*cp = c;
 
 	return 0;
+err_close_xdp_sq:
+	mlx5e_close_sq(&c->xdp_sq);
 
 err_close_sqs:
 	mlx5e_close_sqs(c);
@@ -1273,9 +1541,13 @@ err_napi_del:
 static void mlx5e_close_channel(struct mlx5e_channel *c)
 {
 	mlx5e_close_rq(&c->rq);
+	if (c->xdp)
+		mlx5e_close_sq(&c->xdp_sq);
 	mlx5e_close_sqs(c);
 	mlx5e_close_sq(&c->icosq);
 	napi_disable(&c->napi);
+	if (c->xdp)
+		mlx5e_close_cq(&c->xdp_sq.cq);
 	mlx5e_close_cq(&c->rq.cq);
 	mlx5e_close_tx_cqs(c);
 	mlx5e_close_cq(&c->icosq.cq);
@@ -1349,6 +1621,7 @@ static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
 
 	param->max_inline = priv->params.tx_max_inline;
 	param->min_inline_mode = priv->params.tx_min_inline_mode;
+	param->type = MLX5E_SQ_TXQ;
 }
 
 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
@@ -1422,7 +1695,22 @@ static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
 	MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
 	MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
 
-	param->icosq = true;
+	param->type = MLX5E_SQ_ICO;
+}
+
+static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
+				    struct mlx5e_sq_param *param)
+{
+	void *sqc = param->sqc;
+	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
+
+	mlx5e_build_sq_param_common(priv, param);
+	MLX5_SET(wq, wq, log_wq_sz,     priv->params.log_sq_size);
+
+	param->max_inline = priv->params.tx_max_inline;
+	/* FOR XDP SQs will support only L2 inline mode */
+	param->min_inline_mode = MLX5_INLINE_MODE_NONE;
+	param->type = MLX5E_SQ_XDP;
 }
 
 static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam)
@@ -1431,6 +1719,7 @@ static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_chan
 
 	mlx5e_build_rq_param(priv, &cparam->rq);
 	mlx5e_build_sq_param(priv, &cparam->sq);
+	mlx5e_build_xdpsq_param(priv, &cparam->xdp_sq);
 	mlx5e_build_icosq_param(priv, &cparam->icosq, icosq_log_wq_sz);
 	mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
 	mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
@@ -1826,10 +2115,6 @@ int mlx5e_open_locked(struct net_device *netdev)
 	netif_set_real_num_tx_queues(netdev, num_txqs);
 	netif_set_real_num_rx_queues(netdev, priv->params.num_channels);
 
-	err = mlx5e_set_dev_port_mtu(netdev);
-	if (err)
-		goto err_clear_state_opened_flag;
-
 	err = mlx5e_open_channels(priv);
 	if (err) {
 		netdev_err(netdev, "%s: mlx5e_open_channels failed, %d\n",
@@ -1908,6 +2193,9 @@ int mlx5e_close(struct net_device *netdev)
 	struct mlx5e_priv *priv = netdev_priv(netdev);
 	int err;
 
+	if (!netif_device_present(netdev))
+		return -ENODEV;
+
 	mutex_lock(&priv->state_lock);
 	err = mlx5e_close_locked(netdev);
 	mutex_unlock(&priv->state_lock);
@@ -2022,14 +2310,15 @@ static void mlx5e_close_drop_rq(struct mlx5e_priv *priv)
 static int mlx5e_create_tis(struct mlx5e_priv *priv, int tc)
 {
 	struct mlx5_core_dev *mdev = priv->mdev;
-	u32 in[MLX5_ST_SZ_DW(create_tis_in)];
+	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
 	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(tisc, tisc, prio, tc << 1);
 	MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
 
+	if (mlx5_lag_is_lacp_owner(mdev))
+		MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
+
 	return mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]);
 }
 
@@ -2573,6 +2862,7 @@ static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
 	u16 max_mtu;
 	u16 min_mtu;
 	int err = 0;
+	bool reset;
 
 	mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
 
@@ -2588,13 +2878,18 @@ static int mlx5e_change_mtu(struct net_device *netdev, int new_mtu)
 
 	mutex_lock(&priv->state_lock);
 
+	reset = !priv->params.lro_en &&
+		(priv->params.rq_wq_type !=
+		 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ);
+
 	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
-	if (was_opened)
+	if (was_opened && reset)
 		mlx5e_close_locked(netdev);
 
 	netdev->mtu = new_mtu;
+	mlx5e_set_dev_port_mtu(netdev);
 
-	if (was_opened)
+	if (was_opened && reset)
 		err = mlx5e_open_locked(netdev);
 
 	mutex_unlock(&priv->state_lock);
@@ -2622,11 +2917,15 @@ static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
 	return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
 }
 
-static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos)
+static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
+			     __be16 vlan_proto)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5_core_dev *mdev = priv->mdev;
 
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
 	return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
 					   vlan, qos);
 }
@@ -2794,7 +3093,7 @@ static void mlx5e_tx_timeout(struct net_device *dev)
 		if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i)))
 			continue;
 		sched_work = true;
-		set_bit(MLX5E_SQ_STATE_TX_TIMEOUT, &sq->state);
+		set_bit(MLX5E_SQ_STATE_FLUSH, &sq->state);
 		netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n",
 			   i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc);
 	}
@@ -2803,6 +3102,106 @@ static void mlx5e_tx_timeout(struct net_device *dev)
 		schedule_work(&priv->tx_timeout_work);
 }
 
+static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
+{
+	struct mlx5e_priv *priv = netdev_priv(netdev);
+	struct bpf_prog *old_prog;
+	int err = 0;
+	bool reset, was_opened;
+	int i;
+
+	mutex_lock(&priv->state_lock);
+
+	if ((netdev->features & NETIF_F_LRO) && prog) {
+		netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
+		err = -EINVAL;
+		goto unlock;
+	}
+
+	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
+	/* no need for full reset when exchanging programs */
+	reset = (!priv->xdp_prog || !prog);
+
+	if (was_opened && reset)
+		mlx5e_close_locked(netdev);
+
+	/* exchange programs */
+	old_prog = xchg(&priv->xdp_prog, prog);
+	if (prog)
+		bpf_prog_add(prog, 1);
+	if (old_prog)
+		bpf_prog_put(old_prog);
+
+	if (reset) /* change RQ type according to priv->xdp_prog */
+		mlx5e_set_rq_priv_params(priv);
+
+	if (was_opened && reset)
+		mlx5e_open_locked(netdev);
+
+	if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
+		goto unlock;
+
+	/* exchanging programs w/o reset, we update ref counts on behalf
+	 * of the channels RQs here.
+	 */
+	bpf_prog_add(prog, priv->params.num_channels);
+	for (i = 0; i < priv->params.num_channels; i++) {
+		struct mlx5e_channel *c = priv->channel[i];
+
+		set_bit(MLX5E_RQ_STATE_FLUSH, &c->rq.state);
+		napi_synchronize(&c->napi);
+		/* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */
+
+		old_prog = xchg(&c->rq.xdp_prog, prog);
+
+		clear_bit(MLX5E_RQ_STATE_FLUSH, &c->rq.state);
+		/* napi_schedule in case we have missed anything */
+		set_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
+		napi_schedule(&c->napi);
+
+		if (old_prog)
+			bpf_prog_put(old_prog);
+	}
+
+unlock:
+	mutex_unlock(&priv->state_lock);
+	return err;
+}
+
+static bool mlx5e_xdp_attached(struct net_device *dev)
+{
+	struct mlx5e_priv *priv = netdev_priv(dev);
+
+	return !!priv->xdp_prog;
+}
+
+static int mlx5e_xdp(struct net_device *dev, struct netdev_xdp *xdp)
+{
+	switch (xdp->command) {
+	case XDP_SETUP_PROG:
+		return mlx5e_xdp_set(dev, xdp->prog);
+	case XDP_QUERY_PROG:
+		xdp->prog_attached = mlx5e_xdp_attached(dev);
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+#ifdef CONFIG_NET_POLL_CONTROLLER
+/* Fake "interrupt" called by netpoll (eg netconsole) to send skbs without
+ * reenabling interrupts.
+ */
+static void mlx5e_netpoll(struct net_device *dev)
+{
+	struct mlx5e_priv *priv = netdev_priv(dev);
+	int i;
+
+	for (i = 0; i < priv->params.num_channels; i++)
+		napi_schedule(&priv->channel[i]->napi);
+}
+#endif
+
 static const struct net_device_ops mlx5e_netdev_ops_basic = {
 	.ndo_open                = mlx5e_open,
 	.ndo_stop                = mlx5e_close,
@@ -2822,6 +3221,10 @@ static const struct net_device_ops mlx5e_netdev_ops_basic = {
 	.ndo_rx_flow_steer	 = mlx5e_rx_flow_steer,
 #endif
 	.ndo_tx_timeout          = mlx5e_tx_timeout,
+	.ndo_xdp		 = mlx5e_xdp,
+#ifdef CONFIG_NET_POLL_CONTROLLER
+	.ndo_poll_controller     = mlx5e_netpoll,
+#endif
 };
 
 static const struct net_device_ops mlx5e_netdev_ops_sriov = {
@@ -2853,6 +3256,10 @@ static const struct net_device_ops mlx5e_netdev_ops_sriov = {
 	.ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
 	.ndo_get_vf_stats        = mlx5e_get_vf_stats,
 	.ndo_tx_timeout          = mlx5e_tx_timeout,
+	.ndo_xdp		 = mlx5e_xdp,
+#ifdef CONFIG_NET_POLL_CONTROLLER
+	.ndo_poll_controller     = mlx5e_netpoll,
+#endif
 };
 
 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
@@ -2927,13 +3334,6 @@ void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
 		indirection_rqt[i] = i % num_channels;
 }
 
-static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
-{
-	return MLX5_CAP_GEN(mdev, striding_rq) &&
-		MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
-		MLX5_CAP_ETH(mdev, reg_umr_sq);
-}
-
 static int mlx5e_get_pci_bw(struct mlx5_core_dev *mdev, u32 *pci_bw)
 {
 	enum pcie_link_width width;
@@ -3013,11 +3413,13 @@ static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
 					 MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
 					 MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
 
-	priv->params.log_sq_size           =
-		MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
-	priv->params.rq_wq_type = mlx5e_check_fragmented_striding_rq_cap(mdev) ?
-		MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
-		MLX5_WQ_TYPE_LINKED_LIST;
+	priv->mdev                         = mdev;
+	priv->netdev                       = netdev;
+	priv->params.num_channels          = profile->max_nch(mdev);
+	priv->profile                      = profile;
+	priv->ppriv                        = ppriv;
+
+	priv->params.log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
 
 	/* set CQE compression */
 	priv->params.rx_cqe_compress_admin = false;
@@ -3030,33 +3432,11 @@ static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
 		priv->params.rx_cqe_compress_admin =
 			cqe_compress_heuristic(link_speed, pci_bw);
 	}
-
 	priv->params.rx_cqe_compress = priv->params.rx_cqe_compress_admin;
 
-	switch (priv->params.rq_wq_type) {
-	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
-		priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW;
-		priv->params.mpwqe_log_stride_sz =
-			priv->params.rx_cqe_compress ?
-			MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS :
-			MLX5_MPWRQ_LOG_STRIDE_SIZE;
-		priv->params.mpwqe_log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ -
-			priv->params.mpwqe_log_stride_sz;
+	mlx5e_set_rq_priv_params(priv);
+	if (priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
 		priv->params.lro_en = true;
-		break;
-	default: /* MLX5_WQ_TYPE_LINKED_LIST */
-		priv->params.log_rq_size = MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
-	}
-
-	mlx5_core_info(mdev,
-		       "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
-		       priv->params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
-		       BIT(priv->params.log_rq_size),
-		       BIT(priv->params.mpwqe_log_stride_sz),
-		       priv->params.rx_cqe_compress_admin);
-
-	priv->params.min_rx_wqes = mlx5_min_rx_wqes(priv->params.rq_wq_type,
-					    BIT(priv->params.log_rq_size));
 
 	priv->params.rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
 	mlx5e_set_rx_cq_mode_params(&priv->params, cq_period_mode);
@@ -3076,19 +3456,16 @@ static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
 	mlx5e_build_default_indir_rqt(mdev, priv->params.indirection_rqt,
 				      MLX5E_INDIR_RQT_SIZE, profile->max_nch(mdev));
 
-	priv->params.lro_wqe_sz            =
-		MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
+	priv->params.lro_wqe_sz =
+		MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ -
+		/* Extra room needed for build_skb */
+		MLX5_RX_HEADROOM -
+		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 
 	/* Initialize pflags */
 	MLX5E_SET_PRIV_FLAG(priv, MLX5E_PFLAG_RX_CQE_BASED_MODER,
 			    priv->params.rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
 
-	priv->mdev                         = mdev;
-	priv->netdev                       = netdev;
-	priv->params.num_channels          = profile->max_nch(mdev);
-	priv->profile                      = profile;
-	priv->ppriv                        = ppriv;
-
 #ifdef CONFIG_MLX5_CORE_EN_DCB
 	mlx5e_ets_init(priv);
 #endif
@@ -3228,35 +3605,37 @@ static void mlx5e_destroy_q_counter(struct mlx5e_priv *priv)
 static int mlx5e_create_umr_mkey(struct mlx5e_priv *priv)
 {
 	struct mlx5_core_dev *mdev = priv->mdev;
-	struct mlx5_create_mkey_mbox_in *in;
-	struct mlx5_mkey_seg *mkc;
-	int inlen = sizeof(*in);
-	u64 npages =
-		priv->profile->max_nch(mdev) * MLX5_CHANNEL_MAX_NUM_MTTS;
+	u64 npages = MLX5E_REQUIRED_MTTS(priv->profile->max_nch(mdev),
+					 BIT(MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW));
+	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
+	void *mkc;
+	u32 *in;
 	int err;
 
 	in = mlx5_vzalloc(inlen);
 	if (!in)
 		return -ENOMEM;
 
-	mkc = &in->seg;
-	mkc->status = MLX5_MKEY_STATUS_FREE;
-	mkc->flags = MLX5_PERM_UMR_EN |
-		     MLX5_PERM_LOCAL_READ |
-		     MLX5_PERM_LOCAL_WRITE |
-		     MLX5_ACCESS_MODE_MTT;
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 
-	mkc->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
-	mkc->flags_pd = cpu_to_be32(mdev->mlx5e_res.pdn);
-	mkc->len = cpu_to_be64(npages << PAGE_SHIFT);
-	mkc->xlt_oct_size = cpu_to_be32(mlx5e_get_mtt_octw(npages));
-	mkc->log2_page_size = PAGE_SHIFT;
+	npages = min_t(u32, ALIGN(U16_MAX, 4) * 2, npages);
 
-	err = mlx5_core_create_mkey(mdev, &priv->umr_mkey, in, inlen, NULL,
-				    NULL, NULL);
+	MLX5_SET(mkc, mkc, free, 1);
+	MLX5_SET(mkc, mkc, umr_en, 1);
+	MLX5_SET(mkc, mkc, lw, 1);
+	MLX5_SET(mkc, mkc, lr, 1);
+	MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
 
-	kvfree(in);
+	MLX5_SET(mkc, mkc, qpn, 0xffffff);
+	MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
+	MLX5_SET64(mkc, mkc, len, npages << PAGE_SHIFT);
+	MLX5_SET(mkc, mkc, translations_octword_size,
+		 MLX5_MTT_OCTW(npages));
+	MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
 
+	err = mlx5_core_create_mkey(mdev, &priv->umr_mkey, in, inlen);
+
+	kvfree(in);
 	return err;
 }
 
@@ -3375,6 +3754,8 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv)
 	struct mlx5_eswitch *esw = mdev->priv.eswitch;
 	struct mlx5_eswitch_rep rep;
 
+	mlx5_lag_add(mdev, netdev);
+
 	if (mlx5e_vxlan_allowed(mdev)) {
 		rtnl_lock();
 		udp_tunnel_get_rx_info(netdev);
@@ -3385,11 +3766,12 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv)
 	queue_work(priv->wq, &priv->set_rx_mode_work);
 
 	if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
+		mlx5_query_nic_vport_mac_address(mdev, 0, rep.hw_id);
 		rep.load = mlx5e_nic_rep_load;
 		rep.unload = mlx5e_nic_rep_unload;
-		rep.vport = 0;
+		rep.vport = FDB_UPLINK_VPORT;
 		rep.priv_data = priv;
-		mlx5_eswitch_register_vport_rep(esw, &rep);
+		mlx5_eswitch_register_vport_rep(esw, 0, &rep);
 	}
 }
 
@@ -3397,6 +3779,7 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv)
 {
 	queue_work(priv->wq, &priv->set_rx_mode_work);
 	mlx5e_disable_async_events(priv);
+	mlx5_lag_remove(priv->mdev);
 }
 
 static const struct mlx5e_profile mlx5e_nic_profile = {
@@ -3413,13 +3796,13 @@ static const struct mlx5e_profile mlx5e_nic_profile = {
 	.max_tc		   = MLX5E_MAX_NUM_TC,
 };
 
-void *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
-			  const struct mlx5e_profile *profile, void *ppriv)
+struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
+				       const struct mlx5e_profile *profile,
+				       void *ppriv)
 {
+	int nch = profile->max_nch(mdev);
 	struct net_device *netdev;
 	struct mlx5e_priv *priv;
-	int nch = profile->max_nch(mdev);
-	int err;
 
 	netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
 				    nch * profile->max_tc,
@@ -3437,12 +3820,31 @@ void *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
 
 	priv->wq = create_singlethread_workqueue("mlx5e");
 	if (!priv->wq)
-		goto err_free_netdev;
+		goto err_cleanup_nic;
+
+	return netdev;
+
+err_cleanup_nic:
+	profile->cleanup(priv);
+	free_netdev(netdev);
+
+	return NULL;
+}
+
+int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
+{
+	const struct mlx5e_profile *profile;
+	struct mlx5e_priv *priv;
+	int err;
+
+	priv = netdev_priv(netdev);
+	profile = priv->profile;
+	clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
 
 	err = mlx5e_create_umr_mkey(priv);
 	if (err) {
 		mlx5_core_err(mdev, "create umr mkey failed, %d\n", err);
-		goto err_destroy_wq;
+		goto out;
 	}
 
 	err = profile->init_tx(priv);
@@ -3463,20 +3865,18 @@ void *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
 
 	mlx5e_init_l2_addr(priv);
 
-	err = register_netdev(netdev);
-	if (err) {
-		mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
-		goto err_dealloc_q_counters;
-	}
+	mlx5e_set_dev_port_mtu(netdev);
 
 	if (profile->enable)
 		profile->enable(priv);
 
-	return priv;
+	rtnl_lock();
+	if (netif_running(netdev))
+		mlx5e_open(netdev);
+	netif_device_attach(netdev);
+	rtnl_unlock();
 
-err_dealloc_q_counters:
-	mlx5e_destroy_q_counter(priv);
-	profile->cleanup_rx(priv);
+	return 0;
 
 err_close_drop_rq:
 	mlx5e_close_drop_rq(priv);
@@ -3487,13 +3887,8 @@ err_cleanup_tx:
 err_destroy_umr_mkey:
 	mlx5_core_destroy_mkey(mdev, &priv->umr_mkey);
 
-err_destroy_wq:
-	destroy_workqueue(priv->wq);
-
-err_free_netdev:
-	free_netdev(netdev);
-
-	return NULL;
+out:
+	return err;
 }
 
 static void mlx5e_register_vport_rep(struct mlx5_core_dev *mdev)
@@ -3501,30 +3896,98 @@ static void mlx5e_register_vport_rep(struct mlx5_core_dev *mdev)
 	struct mlx5_eswitch *esw = mdev->priv.eswitch;
 	int total_vfs = MLX5_TOTAL_VPORTS(mdev);
 	int vport;
+	u8 mac[ETH_ALEN];
 
 	if (!MLX5_CAP_GEN(mdev, vport_group_manager))
 		return;
 
+	mlx5_query_nic_vport_mac_address(mdev, 0, mac);
+
 	for (vport = 1; vport < total_vfs; vport++) {
 		struct mlx5_eswitch_rep rep;
 
 		rep.load = mlx5e_vport_rep_load;
 		rep.unload = mlx5e_vport_rep_unload;
 		rep.vport = vport;
-		mlx5_eswitch_register_vport_rep(esw, &rep);
+		ether_addr_copy(rep.hw_id, mac);
+		mlx5_eswitch_register_vport_rep(esw, vport, &rep);
 	}
 }
 
+void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev)
+{
+	struct mlx5e_priv *priv = netdev_priv(netdev);
+	const struct mlx5e_profile *profile = priv->profile;
+
+	set_bit(MLX5E_STATE_DESTROYING, &priv->state);
+	if (profile->disable)
+		profile->disable(priv);
+
+	flush_workqueue(priv->wq);
+
+	rtnl_lock();
+	if (netif_running(netdev))
+		mlx5e_close(netdev);
+	netif_device_detach(netdev);
+	rtnl_unlock();
+
+	mlx5e_destroy_q_counter(priv);
+	profile->cleanup_rx(priv);
+	mlx5e_close_drop_rq(priv);
+	profile->cleanup_tx(priv);
+	mlx5_core_destroy_mkey(priv->mdev, &priv->umr_mkey);
+	cancel_delayed_work_sync(&priv->update_stats_work);
+}
+
+/* mlx5e_attach and mlx5e_detach scope should be only creating/destroying
+ * hardware contexts and to connect it to the current netdev.
+ */
+static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv)
+{
+	struct mlx5e_priv *priv = vpriv;
+	struct net_device *netdev = priv->netdev;
+	int err;
+
+	if (netif_device_present(netdev))
+		return 0;
+
+	err = mlx5e_create_mdev_resources(mdev);
+	if (err)
+		return err;
+
+	err = mlx5e_attach_netdev(mdev, netdev);
+	if (err) {
+		mlx5e_destroy_mdev_resources(mdev);
+		return err;
+	}
+
+	return 0;
+}
+
+static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv)
+{
+	struct mlx5e_priv *priv = vpriv;
+	struct net_device *netdev = priv->netdev;
+
+	if (!netif_device_present(netdev))
+		return;
+
+	mlx5e_detach_netdev(mdev, netdev);
+	mlx5e_destroy_mdev_resources(mdev);
+}
+
 static void *mlx5e_add(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_eswitch *esw = mdev->priv.eswitch;
+	int total_vfs = MLX5_TOTAL_VPORTS(mdev);
 	void *ppriv = NULL;
-	void *ret;
-
-	if (mlx5e_check_required_hca_cap(mdev))
-		return NULL;
+	void *priv;
+	int vport;
+	int err;
+	struct net_device *netdev;
 
-	if (mlx5e_create_mdev_resources(mdev))
+	err = mlx5e_check_required_hca_cap(mdev);
+	if (err)
 		return NULL;
 
 	mlx5e_register_vport_rep(mdev);
@@ -3532,12 +3995,39 @@ static void *mlx5e_add(struct mlx5_core_dev *mdev)
 	if (MLX5_CAP_GEN(mdev, vport_group_manager))
 		ppriv = &esw->offloads.vport_reps[0];
 
-	ret = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, ppriv);
-	if (!ret) {
-		mlx5e_destroy_mdev_resources(mdev);
-		return NULL;
+	netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, ppriv);
+	if (!netdev) {
+		mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
+		goto err_unregister_reps;
+	}
+
+	priv = netdev_priv(netdev);
+
+	err = mlx5e_attach(mdev, priv);
+	if (err) {
+		mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err);
+		goto err_destroy_netdev;
+	}
+
+	err = register_netdev(netdev);
+	if (err) {
+		mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
+		goto err_detach;
 	}
-	return ret;
+
+	return priv;
+
+err_detach:
+	mlx5e_detach(mdev, priv);
+
+err_destroy_netdev:
+	mlx5e_destroy_netdev(mdev, priv);
+
+err_unregister_reps:
+	for (vport = 1; vport < total_vfs; vport++)
+		mlx5_eswitch_unregister_vport_rep(esw, vport);
+
+	return NULL;
 }
 
 void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv)
@@ -3545,30 +4035,11 @@ void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv)
 	const struct mlx5e_profile *profile = priv->profile;
 	struct net_device *netdev = priv->netdev;
 
-	set_bit(MLX5E_STATE_DESTROYING, &priv->state);
-	if (profile->disable)
-		profile->disable(priv);
-
-	flush_workqueue(priv->wq);
-	if (test_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &mdev->intf_state)) {
-		netif_device_detach(netdev);
-		mlx5e_close(netdev);
-	} else {
-		unregister_netdev(netdev);
-	}
-
-	mlx5e_destroy_q_counter(priv);
-	profile->cleanup_rx(priv);
-	mlx5e_close_drop_rq(priv);
-	profile->cleanup_tx(priv);
-	mlx5_core_destroy_mkey(priv->mdev, &priv->umr_mkey);
-	cancel_delayed_work_sync(&priv->update_stats_work);
+	unregister_netdev(netdev);
 	destroy_workqueue(priv->wq);
 	if (profile->cleanup)
 		profile->cleanup(priv);
-
-	if (!test_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &mdev->intf_state))
-		free_netdev(netdev);
+	free_netdev(netdev);
 }
 
 static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv)
@@ -3578,12 +4049,11 @@ static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv)
 	struct mlx5e_priv *priv = vpriv;
 	int vport;
 
-	mlx5e_destroy_netdev(mdev, priv);
-
 	for (vport = 1; vport < total_vfs; vport++)
 		mlx5_eswitch_unregister_vport_rep(esw, vport);
 
-	mlx5e_destroy_mdev_resources(mdev);
+	mlx5e_detach(mdev, vpriv);
+	mlx5e_destroy_netdev(mdev, priv);
 }
 
 static void *mlx5e_get_netdev(void *vpriv)
@@ -3596,6 +4066,8 @@ static void *mlx5e_get_netdev(void *vpriv)
 static struct mlx5_interface mlx5e_interface = {
 	.add       = mlx5e_add,
 	.remove    = mlx5e_remove,
+	.attach    = mlx5e_attach,
+	.detach    = mlx5e_detach,
 	.event     = mlx5e_async_event,
 	.protocol  = MLX5_INTERFACE_PROTOCOL_ETH,
 	.get_dev   = mlx5e_get_netdev,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 1c7d8b8314bf..3c97da103d30 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -135,17 +135,16 @@ static const struct ethtool_ops mlx5e_rep_ethtool_ops = {
 int mlx5e_attr_get(struct net_device *dev, struct switchdev_attr *attr)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
+	struct mlx5_eswitch_rep *rep = priv->ppriv;
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
-	u8 mac[ETH_ALEN];
 
 	if (esw->mode == SRIOV_NONE)
 		return -EOPNOTSUPP;
 
 	switch (attr->id) {
 	case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
-		mlx5_query_nic_vport_mac_address(priv->mdev, 0, mac);
 		attr->u.ppid.id_len = ETH_ALEN;
-		memcpy(&attr->u.ppid.id, &mac, ETH_ALEN);
+		ether_addr_copy(attr->u.ppid.id, rep->hw_id);
 		break;
 	default:
 		return -EOPNOTSUPP;
@@ -414,19 +413,50 @@ static struct mlx5e_profile mlx5e_rep_profile = {
 int mlx5e_vport_rep_load(struct mlx5_eswitch *esw,
 			 struct mlx5_eswitch_rep *rep)
 {
-	rep->priv_data = mlx5e_create_netdev(esw->dev, &mlx5e_rep_profile, rep);
-	if (!rep->priv_data) {
-		pr_warn("Failed to create representor for vport %d\n",
+	struct net_device *netdev;
+	int err;
+
+	netdev = mlx5e_create_netdev(esw->dev, &mlx5e_rep_profile, rep);
+	if (!netdev) {
+		pr_warn("Failed to create representor netdev for vport %d\n",
 			rep->vport);
 		return -EINVAL;
 	}
+
+	rep->priv_data = netdev_priv(netdev);
+
+	err = mlx5e_attach_netdev(esw->dev, netdev);
+	if (err) {
+		pr_warn("Failed to attach representor netdev for vport %d\n",
+			rep->vport);
+		goto err_destroy_netdev;
+	}
+
+	err = register_netdev(netdev);
+	if (err) {
+		pr_warn("Failed to register representor netdev for vport %d\n",
+			rep->vport);
+		goto err_detach_netdev;
+	}
+
 	return 0;
+
+err_detach_netdev:
+	mlx5e_detach_netdev(esw->dev, netdev);
+
+err_destroy_netdev:
+	mlx5e_destroy_netdev(esw->dev, rep->priv_data);
+
+	return err;
+
 }
 
 void mlx5e_vport_rep_unload(struct mlx5_eswitch *esw,
 			    struct mlx5_eswitch_rep *rep)
 {
 	struct mlx5e_priv *priv = rep->priv_data;
+	struct net_device *netdev = priv->netdev;
 
+	mlx5e_detach_netdev(esw->dev, netdev);
 	mlx5e_destroy_netdev(esw->dev, priv);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 9f2a16a507e0..c6de6fba5843 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -36,6 +36,7 @@
 #include <net/busy_poll.h>
 #include "en.h"
 #include "en_tc.h"
+#include "eswitch.h"
 
 static inline bool mlx5e_rx_hw_stamp(struct mlx5e_tstamp *tstamp)
 {
@@ -179,96 +180,111 @@ unlock:
 	mutex_unlock(&priv->state_lock);
 }
 
-int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
-{
-	struct sk_buff *skb;
-	dma_addr_t dma_addr;
+#define RQ_PAGE_SIZE(rq) ((1 << rq->buff.page_order) << PAGE_SHIFT)
 
-	skb = napi_alloc_skb(rq->cq.napi, rq->wqe_sz);
-	if (unlikely(!skb))
-		return -ENOMEM;
+static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
+				      struct mlx5e_dma_info *dma_info)
+{
+	struct mlx5e_page_cache *cache = &rq->page_cache;
+	u32 tail_next = (cache->tail + 1) & (MLX5E_CACHE_SIZE - 1);
 
-	dma_addr = dma_map_single(rq->pdev,
-				  /* hw start padding */
-				  skb->data,
-				  /* hw end padding */
-				  rq->wqe_sz,
-				  DMA_FROM_DEVICE);
+	if (tail_next == cache->head) {
+		rq->stats.cache_full++;
+		return false;
+	}
 
-	if (unlikely(dma_mapping_error(rq->pdev, dma_addr)))
-		goto err_free_skb;
+	cache->page_cache[cache->tail] = *dma_info;
+	cache->tail = tail_next;
+	return true;
+}
 
-	*((dma_addr_t *)skb->cb) = dma_addr;
-	wqe->data.addr = cpu_to_be64(dma_addr);
-	wqe->data.lkey = rq->mkey_be;
+static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
+				      struct mlx5e_dma_info *dma_info)
+{
+	struct mlx5e_page_cache *cache = &rq->page_cache;
 
-	rq->skb[ix] = skb;
+	if (unlikely(cache->head == cache->tail)) {
+		rq->stats.cache_empty++;
+		return false;
+	}
 
-	return 0;
+	if (page_ref_count(cache->page_cache[cache->head].page) != 1) {
+		rq->stats.cache_busy++;
+		return false;
+	}
 
-err_free_skb:
-	dev_kfree_skb(skb);
+	*dma_info = cache->page_cache[cache->head];
+	cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
+	rq->stats.cache_reuse++;
 
-	return -ENOMEM;
+	dma_sync_single_for_device(rq->pdev, dma_info->addr,
+				   RQ_PAGE_SIZE(rq),
+				   DMA_FROM_DEVICE);
+	return true;
 }
 
-void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
+static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
+					  struct mlx5e_dma_info *dma_info)
 {
-	struct sk_buff *skb = rq->skb[ix];
+	struct page *page;
+
+	if (mlx5e_rx_cache_get(rq, dma_info))
+		return 0;
+
+	page = dev_alloc_pages(rq->buff.page_order);
+	if (unlikely(!page))
+		return -ENOMEM;
 
-	if (skb) {
-		rq->skb[ix] = NULL;
-		dma_unmap_single(rq->pdev,
-				 *((dma_addr_t *)skb->cb),
-				 rq->wqe_sz,
-				 DMA_FROM_DEVICE);
-		dev_kfree_skb(skb);
+	dma_info->page = page;
+	dma_info->addr = dma_map_page(rq->pdev, page, 0,
+				      RQ_PAGE_SIZE(rq), rq->buff.map_dir);
+	if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
+		put_page(page);
+		return -ENOMEM;
 	}
+
+	return 0;
 }
 
-static inline int mlx5e_mpwqe_strides_per_page(struct mlx5e_rq *rq)
+void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
+			bool recycle)
 {
-	return rq->mpwqe_num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER;
+	if (likely(recycle) && mlx5e_rx_cache_put(rq, dma_info))
+		return;
+
+	dma_unmap_page(rq->pdev, dma_info->addr, RQ_PAGE_SIZE(rq),
+		       rq->buff.map_dir);
+	put_page(dma_info->page);
 }
 
-static inline void
-mlx5e_dma_pre_sync_linear_mpwqe(struct device *pdev,
-				struct mlx5e_mpw_info *wi,
-				u32 wqe_offset, u32 len)
+int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
 {
-	dma_sync_single_for_cpu(pdev, wi->dma_info.addr + wqe_offset,
-				len, DMA_FROM_DEVICE);
+	struct mlx5e_dma_info *di = &rq->dma_info[ix];
+
+	if (unlikely(mlx5e_page_alloc_mapped(rq, di)))
+		return -ENOMEM;
+
+	wqe->data.addr = cpu_to_be64(di->addr + MLX5_RX_HEADROOM);
+	return 0;
 }
 
-static inline void
-mlx5e_dma_pre_sync_fragmented_mpwqe(struct device *pdev,
-				    struct mlx5e_mpw_info *wi,
-				    u32 wqe_offset, u32 len)
+void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
 {
-	/* No dma pre sync for fragmented MPWQE */
+	struct mlx5e_dma_info *di = &rq->dma_info[ix];
+
+	mlx5e_page_release(rq, di, true);
 }
 
-static inline void
-mlx5e_add_skb_frag_linear_mpwqe(struct mlx5e_rq *rq,
-				struct sk_buff *skb,
-				struct mlx5e_mpw_info *wi,
-				u32 page_idx, u32 frag_offset,
-				u32 len)
+static inline int mlx5e_mpwqe_strides_per_page(struct mlx5e_rq *rq)
 {
-	unsigned int truesize =	ALIGN(len, rq->mpwqe_stride_sz);
-
-	wi->skbs_frags[page_idx]++;
-	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
-			&wi->dma_info.page[page_idx], frag_offset,
-			len, truesize);
+	return rq->mpwqe_num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER;
 }
 
-static inline void
-mlx5e_add_skb_frag_fragmented_mpwqe(struct mlx5e_rq *rq,
-				    struct sk_buff *skb,
-				    struct mlx5e_mpw_info *wi,
-				    u32 page_idx, u32 frag_offset,
-				    u32 len)
+static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq,
+					    struct sk_buff *skb,
+					    struct mlx5e_mpw_info *wi,
+					    u32 page_idx, u32 frag_offset,
+					    u32 len)
 {
 	unsigned int truesize =	ALIGN(len, rq->mpwqe_stride_sz);
 
@@ -282,24 +298,11 @@ mlx5e_add_skb_frag_fragmented_mpwqe(struct mlx5e_rq *rq,
 }
 
 static inline void
-mlx5e_copy_skb_header_linear_mpwqe(struct device *pdev,
-				   struct sk_buff *skb,
-				   struct mlx5e_mpw_info *wi,
-				   u32 page_idx, u32 offset,
-				   u32 headlen)
-{
-	struct page *page = &wi->dma_info.page[page_idx];
-
-	skb_copy_to_linear_data(skb, page_address(page) + offset,
-				ALIGN(headlen, sizeof(long)));
-}
-
-static inline void
-mlx5e_copy_skb_header_fragmented_mpwqe(struct device *pdev,
-				       struct sk_buff *skb,
-				       struct mlx5e_mpw_info *wi,
-				       u32 page_idx, u32 offset,
-				       u32 headlen)
+mlx5e_copy_skb_header_mpwqe(struct device *pdev,
+			    struct sk_buff *skb,
+			    struct mlx5e_mpw_info *wi,
+			    u32 page_idx, u32 offset,
+			    u32 headlen)
 {
 	u16 headlen_pg = min_t(u32, headlen, PAGE_SIZE - offset);
 	struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[page_idx];
@@ -324,46 +327,9 @@ mlx5e_copy_skb_header_fragmented_mpwqe(struct device *pdev,
 	}
 }
 
-static u16 mlx5e_get_wqe_mtt_offset(u16 rq_ix, u16 wqe_ix)
-{
-	return rq_ix * MLX5_CHANNEL_MAX_NUM_MTTS +
-		wqe_ix * ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8);
-}
-
-static void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
-				struct mlx5e_sq *sq,
-				struct mlx5e_umr_wqe *wqe,
-				u16 ix)
-{
-	struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
-	struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
-	struct mlx5_wqe_data_seg      *dseg = &wqe->data;
-	struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
-	u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
-	u16 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq->ix, ix);
-
-	memset(wqe, 0, sizeof(*wqe));
-	cseg->opmod_idx_opcode =
-		cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
-			    MLX5_OPCODE_UMR);
-	cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
-				      ds_cnt);
-	cseg->fm_ce_se  = MLX5_WQE_CTRL_CQ_UPDATE;
-	cseg->imm       = rq->umr_mkey_be;
-
-	ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
-	ucseg->klm_octowords =
-		cpu_to_be16(mlx5e_get_mtt_octw(MLX5_MPWRQ_PAGES_PER_WQE));
-	ucseg->bsf_octowords =
-		cpu_to_be16(mlx5e_get_mtt_octw(umr_wqe_mtt_offset));
-	ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
-
-	dseg->lkey = sq->mkey_be;
-	dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
-}
-
-static void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
+static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
 {
+	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
 	struct mlx5e_sq *sq = &rq->channel->icosq;
 	struct mlx5_wq_cyc *wq = &sq->wq;
 	struct mlx5e_umr_wqe *wqe;
@@ -372,142 +338,86 @@ static void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
 
 	/* fill sq edge with nops to avoid wqe wrap around */
 	while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
-		sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_NOP;
-		sq->ico_wqe_info[pi].num_wqebbs = 1;
+		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
+		sq->db.ico_wqe[pi].num_wqebbs = 1;
 		mlx5e_send_nop(sq, true);
 	}
 
 	wqe = mlx5_wq_cyc_get_wqe(wq, pi);
-	mlx5e_build_umr_wqe(rq, sq, wqe, ix);
-	sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_UMR;
-	sq->ico_wqe_info[pi].num_wqebbs = num_wqebbs;
+	memcpy(wqe, &wi->umr.wqe, sizeof(*wqe));
+	wqe->ctrl.opmod_idx_opcode =
+		cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
+			    MLX5_OPCODE_UMR);
+
+	sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
+	sq->db.ico_wqe[pi].num_wqebbs = num_wqebbs;
 	sq->pc += num_wqebbs;
 	mlx5e_tx_notify_hw(sq, &wqe->ctrl, 0);
 }
 
-static inline int mlx5e_get_wqe_mtt_sz(void)
-{
-	/* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
-	 * To avoid copying garbage after the mtt array, we allocate
-	 * a little more.
-	 */
-	return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
-		     MLX5_UMR_MTT_ALIGNMENT);
-}
-
-static int mlx5e_alloc_and_map_page(struct mlx5e_rq *rq,
-				    struct mlx5e_mpw_info *wi,
-				    int i)
-{
-	struct page *page;
-
-	page = dev_alloc_page();
-	if (unlikely(!page))
-		return -ENOMEM;
-
-	wi->umr.dma_info[i].page = page;
-	wi->umr.dma_info[i].addr = dma_map_page(rq->pdev, page, 0, PAGE_SIZE,
-						PCI_DMA_FROMDEVICE);
-	if (unlikely(dma_mapping_error(rq->pdev, wi->umr.dma_info[i].addr))) {
-		put_page(page);
-		return -ENOMEM;
-	}
-	wi->umr.mtt[i] = cpu_to_be64(wi->umr.dma_info[i].addr | MLX5_EN_WR);
-
-	return 0;
-}
-
-static int mlx5e_alloc_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
-					   struct mlx5e_rx_wqe *wqe,
-					   u16 ix)
+static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq,
+				    struct mlx5e_rx_wqe *wqe,
+				    u16 ix)
 {
-	struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
-	int mtt_sz = mlx5e_get_wqe_mtt_sz();
-	u32 dma_offset = mlx5e_get_wqe_mtt_offset(rq->ix, ix) << PAGE_SHIFT;
+	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
+	u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, ix) << PAGE_SHIFT;
+	int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
+	int err;
 	int i;
 
-	wi->umr.dma_info = kmalloc(sizeof(*wi->umr.dma_info) *
-				   MLX5_MPWRQ_PAGES_PER_WQE,
-				   GFP_ATOMIC);
-	if (unlikely(!wi->umr.dma_info))
-		goto err_out;
-
-	/* We allocate more than mtt_sz as we will align the pointer */
-	wi->umr.mtt_no_align = kzalloc(mtt_sz + MLX5_UMR_ALIGN - 1,
-				       GFP_ATOMIC);
-	if (unlikely(!wi->umr.mtt_no_align))
-		goto err_free_umr;
-
-	wi->umr.mtt = PTR_ALIGN(wi->umr.mtt_no_align, MLX5_UMR_ALIGN);
-	wi->umr.mtt_addr = dma_map_single(rq->pdev, wi->umr.mtt, mtt_sz,
-					  PCI_DMA_TODEVICE);
-	if (unlikely(dma_mapping_error(rq->pdev, wi->umr.mtt_addr)))
-		goto err_free_mtt;
-
 	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
-		if (unlikely(mlx5e_alloc_and_map_page(rq, wi, i)))
+		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];
+
+		err = mlx5e_page_alloc_mapped(rq, dma_info);
+		if (unlikely(err))
 			goto err_unmap;
-		page_ref_add(wi->umr.dma_info[i].page,
-			     mlx5e_mpwqe_strides_per_page(rq));
+		wi->umr.mtt[i] = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
+		page_ref_add(dma_info->page, pg_strides);
 		wi->skbs_frags[i] = 0;
 	}
 
 	wi->consumed_strides = 0;
-	wi->dma_pre_sync = mlx5e_dma_pre_sync_fragmented_mpwqe;
-	wi->add_skb_frag = mlx5e_add_skb_frag_fragmented_mpwqe;
-	wi->copy_skb_header = mlx5e_copy_skb_header_fragmented_mpwqe;
-	wi->free_wqe     = mlx5e_free_rx_fragmented_mpwqe;
-	wqe->data.lkey = rq->umr_mkey_be;
 	wqe->data.addr = cpu_to_be64(dma_offset);
 
 	return 0;
 
 err_unmap:
 	while (--i >= 0) {
-		dma_unmap_page(rq->pdev, wi->umr.dma_info[i].addr, PAGE_SIZE,
-			       PCI_DMA_FROMDEVICE);
-		page_ref_sub(wi->umr.dma_info[i].page,
-			     mlx5e_mpwqe_strides_per_page(rq));
-		put_page(wi->umr.dma_info[i].page);
-	}
-	dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz, PCI_DMA_TODEVICE);
+		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];
 
-err_free_mtt:
-	kfree(wi->umr.mtt_no_align);
-
-err_free_umr:
-	kfree(wi->umr.dma_info);
+		page_ref_sub(dma_info->page, pg_strides);
+		mlx5e_page_release(rq, dma_info, true);
+	}
 
-err_out:
-	return -ENOMEM;
+	return err;
 }
 
-void mlx5e_free_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
-				    struct mlx5e_mpw_info *wi)
+void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
 {
-	int mtt_sz = mlx5e_get_wqe_mtt_sz();
+	int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
 	int i;
 
 	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
-		dma_unmap_page(rq->pdev, wi->umr.dma_info[i].addr, PAGE_SIZE,
-			       PCI_DMA_FROMDEVICE);
-		page_ref_sub(wi->umr.dma_info[i].page,
-			mlx5e_mpwqe_strides_per_page(rq) - wi->skbs_frags[i]);
-		put_page(wi->umr.dma_info[i].page);
+		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];
+
+		page_ref_sub(dma_info->page, pg_strides - wi->skbs_frags[i]);
+		mlx5e_page_release(rq, dma_info, true);
 	}
-	dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz, PCI_DMA_TODEVICE);
-	kfree(wi->umr.mtt_no_align);
-	kfree(wi->umr.dma_info);
 }
 
-void mlx5e_post_rx_fragmented_mpwqe(struct mlx5e_rq *rq)
+void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
 {
 	struct mlx5_wq_ll *wq = &rq->wq;
 	struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
 
 	clear_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);
+
+	if (unlikely(test_bit(MLX5E_RQ_STATE_FLUSH, &rq->state))) {
+		mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
+		return;
+	}
+
 	mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
-	rq->stats.mpwqe_frag++;
 
 	/* ensure wqes are visible to device before updating doorbell record */
 	dma_wmb();
@@ -515,106 +425,28 @@ void mlx5e_post_rx_fragmented_mpwqe(struct mlx5e_rq *rq)
 	mlx5_wq_ll_update_db_record(wq);
 }
 
-static int mlx5e_alloc_rx_linear_mpwqe(struct mlx5e_rq *rq,
-				       struct mlx5e_rx_wqe *wqe,
-				       u16 ix)
-{
-	struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
-	gfp_t gfp_mask;
-	int i;
-
-	gfp_mask = GFP_ATOMIC | __GFP_COLD | __GFP_MEMALLOC;
-	wi->dma_info.page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
-					     MLX5_MPWRQ_WQE_PAGE_ORDER);
-	if (unlikely(!wi->dma_info.page))
-		return -ENOMEM;
-
-	wi->dma_info.addr = dma_map_page(rq->pdev, wi->dma_info.page, 0,
-					 rq->wqe_sz, PCI_DMA_FROMDEVICE);
-	if (unlikely(dma_mapping_error(rq->pdev, wi->dma_info.addr))) {
-		put_page(wi->dma_info.page);
-		return -ENOMEM;
-	}
-
-	/* We split the high-order page into order-0 ones and manage their
-	 * reference counter to minimize the memory held by small skb fragments
-	 */
-	split_page(wi->dma_info.page, MLX5_MPWRQ_WQE_PAGE_ORDER);
-	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
-		page_ref_add(&wi->dma_info.page[i],
-			     mlx5e_mpwqe_strides_per_page(rq));
-		wi->skbs_frags[i] = 0;
-	}
-
-	wi->consumed_strides = 0;
-	wi->dma_pre_sync = mlx5e_dma_pre_sync_linear_mpwqe;
-	wi->add_skb_frag = mlx5e_add_skb_frag_linear_mpwqe;
-	wi->copy_skb_header = mlx5e_copy_skb_header_linear_mpwqe;
-	wi->free_wqe     = mlx5e_free_rx_linear_mpwqe;
-	wqe->data.lkey = rq->mkey_be;
-	wqe->data.addr = cpu_to_be64(wi->dma_info.addr);
-
-	return 0;
-}
-
-void mlx5e_free_rx_linear_mpwqe(struct mlx5e_rq *rq,
-				struct mlx5e_mpw_info *wi)
-{
-	int i;
-
-	dma_unmap_page(rq->pdev, wi->dma_info.addr, rq->wqe_sz,
-		       PCI_DMA_FROMDEVICE);
-	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
-		page_ref_sub(&wi->dma_info.page[i],
-			mlx5e_mpwqe_strides_per_page(rq) - wi->skbs_frags[i]);
-		put_page(&wi->dma_info.page[i]);
-	}
-}
-
 int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
 {
 	int err;
 
-	err = mlx5e_alloc_rx_linear_mpwqe(rq, wqe, ix);
-	if (unlikely(err)) {
-		err = mlx5e_alloc_rx_fragmented_mpwqe(rq, wqe, ix);
-		if (unlikely(err))
-			return err;
-		set_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);
-		mlx5e_post_umr_wqe(rq, ix);
-		return -EBUSY;
-	}
-
-	return 0;
+	err = mlx5e_alloc_rx_umr_mpwqe(rq, wqe, ix);
+	if (unlikely(err))
+		return err;
+	set_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);
+	mlx5e_post_umr_wqe(rq, ix);
+	return -EBUSY;
 }
 
 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 {
-	struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
-
-	wi->free_wqe(rq, wi);
-}
-
-void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
-{
-	struct mlx5_wq_ll *wq = &rq->wq;
-	struct mlx5e_rx_wqe *wqe;
-	__be16 wqe_ix_be;
-	u16 wqe_ix;
+	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
 
-	while (!mlx5_wq_ll_is_empty(wq)) {
-		wqe_ix_be = *wq->tail_next;
-		wqe_ix    = be16_to_cpu(wqe_ix_be);
-		wqe       = mlx5_wq_ll_get_wqe(&rq->wq, wqe_ix);
-		rq->dealloc_wqe(rq, wqe_ix);
-		mlx5_wq_ll_pop(&rq->wq, wqe_ix_be,
-			       &wqe->next.next_wqe_index);
-	}
+	mlx5e_free_rx_mpwqe(rq, wi);
 }
 
 #define RQ_CANNOT_POST(rq) \
-		(!test_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE, &rq->state) || \
-		 test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
+	(test_bit(MLX5E_RQ_STATE_FLUSH, &rq->state) || \
+	 test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
 
 bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
 {
@@ -628,9 +460,10 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
 		int err;
 
 		err = rq->alloc_wqe(rq, wqe, wq->head);
+		if (err == -EBUSY)
+			return true;
 		if (unlikely(err)) {
-			if (err != -EBUSY)
-				rq->stats.buff_alloc_err++;
+			rq->stats.buff_alloc_err++;
 			break;
 		}
 
@@ -648,24 +481,32 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
 static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe,
 				 u32 cqe_bcnt)
 {
-	struct ethhdr	*eth	= (struct ethhdr *)(skb->data);
-	struct iphdr	*ipv4	= (struct iphdr *)(skb->data + ETH_HLEN);
-	struct ipv6hdr	*ipv6	= (struct ipv6hdr *)(skb->data + ETH_HLEN);
+	struct ethhdr	*eth = (struct ethhdr *)(skb->data);
+	struct iphdr	*ipv4;
+	struct ipv6hdr	*ipv6;
 	struct tcphdr	*tcp;
+	int network_depth = 0;
+	__be16 proto;
+	u16 tot_len;
 
 	u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
 	int tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA  == l4_hdr_type) ||
 		       (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type));
 
-	u16 tot_len = cqe_bcnt - ETH_HLEN;
+	skb->mac_len = ETH_HLEN;
+	proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
+
+	ipv4 = (struct iphdr *)(skb->data + network_depth);
+	ipv6 = (struct ipv6hdr *)(skb->data + network_depth);
+	tot_len = cqe_bcnt - network_depth;
 
-	if (eth->h_proto == htons(ETH_P_IP)) {
-		tcp = (struct tcphdr *)(skb->data + ETH_HLEN +
+	if (proto == htons(ETH_P_IP)) {
+		tcp = (struct tcphdr *)(skb->data + network_depth +
 					sizeof(struct iphdr));
 		ipv6 = NULL;
 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 	} else {
-		tcp = (struct tcphdr *)(skb->data + ETH_HLEN +
+		tcp = (struct tcphdr *)(skb->data + network_depth +
 					sizeof(struct ipv6hdr));
 		ipv4 = NULL;
 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
@@ -789,40 +630,207 @@ static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq,
 	rq->stats.packets++;
 	rq->stats.bytes += cqe_bcnt;
 	mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);
-	napi_gro_receive(rq->cq.napi, skb);
+}
+
+static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_sq *sq)
+{
+	struct mlx5_wq_cyc *wq = &sq->wq;
+	struct mlx5e_tx_wqe *wqe;
+	u16 pi = (sq->pc - MLX5E_XDP_TX_WQEBBS) & wq->sz_m1; /* last pi */
+
+	wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
+
+	wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
+	mlx5e_tx_notify_hw(sq, &wqe->ctrl, 0);
+}
+
+static inline void mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
+					struct mlx5e_dma_info *di,
+					unsigned int data_offset,
+					int len)
+{
+	struct mlx5e_sq          *sq   = &rq->channel->xdp_sq;
+	struct mlx5_wq_cyc       *wq   = &sq->wq;
+	u16                      pi    = sq->pc & wq->sz_m1;
+	struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
+	struct mlx5e_sq_wqe_info *wi   = &sq->db.xdp.wqe_info[pi];
+
+	struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
+	struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
+	struct mlx5_wqe_data_seg *dseg;
+
+	dma_addr_t dma_addr  = di->addr + data_offset + MLX5E_XDP_MIN_INLINE;
+	unsigned int dma_len = len - MLX5E_XDP_MIN_INLINE;
+	void *data           = page_address(di->page) + data_offset;
+
+	if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_XDP_TX_WQEBBS))) {
+		if (sq->db.xdp.doorbell) {
+			/* SQ is full, ring doorbell */
+			mlx5e_xmit_xdp_doorbell(sq);
+			sq->db.xdp.doorbell = false;
+		}
+		rq->stats.xdp_tx_full++;
+		mlx5e_page_release(rq, di, true);
+		return;
+	}
+
+	dma_sync_single_for_device(sq->pdev, dma_addr, dma_len,
+				   PCI_DMA_TODEVICE);
+
+	memset(wqe, 0, sizeof(*wqe));
+
+	/* copy the inline part */
+	memcpy(eseg->inline_hdr_start, data, MLX5E_XDP_MIN_INLINE);
+	eseg->inline_hdr_sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
+
+	dseg = (struct mlx5_wqe_data_seg *)cseg + (MLX5E_XDP_TX_DS_COUNT - 1);
+
+	/* write the dma part */
+	dseg->addr       = cpu_to_be64(dma_addr);
+	dseg->byte_count = cpu_to_be32(dma_len);
+	dseg->lkey       = sq->mkey_be;
+
+	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND);
+	cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | MLX5E_XDP_TX_DS_COUNT);
+
+	sq->db.xdp.di[pi] = *di;
+	wi->opcode     = MLX5_OPCODE_SEND;
+	wi->num_wqebbs = MLX5E_XDP_TX_WQEBBS;
+	sq->pc += MLX5E_XDP_TX_WQEBBS;
+
+	sq->db.xdp.doorbell = true;
+	rq->stats.xdp_tx++;
+}
+
+/* returns true if packet was consumed by xdp */
+static inline bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
+				    const struct bpf_prog *prog,
+				    struct mlx5e_dma_info *di,
+				    void *data, u16 len)
+{
+	struct xdp_buff xdp;
+	u32 act;
+
+	if (!prog)
+		return false;
+
+	xdp.data = data;
+	xdp.data_end = xdp.data + len;
+	act = bpf_prog_run_xdp(prog, &xdp);
+	switch (act) {
+	case XDP_PASS:
+		return false;
+	case XDP_TX:
+		mlx5e_xmit_xdp_frame(rq, di, MLX5_RX_HEADROOM, len);
+		return true;
+	default:
+		bpf_warn_invalid_xdp_action(act);
+	case XDP_ABORTED:
+	case XDP_DROP:
+		rq->stats.xdp_drop++;
+		mlx5e_page_release(rq, di, true);
+		return true;
+	}
+}
+
+static inline
+struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
+			     u16 wqe_counter, u32 cqe_bcnt)
+{
+	struct bpf_prog *xdp_prog = READ_ONCE(rq->xdp_prog);
+	struct mlx5e_dma_info *di;
+	struct sk_buff *skb;
+	void *va, *data;
+
+	di             = &rq->dma_info[wqe_counter];
+	va             = page_address(di->page);
+	data           = va + MLX5_RX_HEADROOM;
+
+	dma_sync_single_range_for_cpu(rq->pdev,
+				      di->addr,
+				      MLX5_RX_HEADROOM,
+				      rq->buff.wqe_sz,
+				      DMA_FROM_DEVICE);
+	prefetch(data);
+
+	if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
+		rq->stats.wqe_err++;
+		mlx5e_page_release(rq, di, true);
+		return NULL;
+	}
+
+	if (mlx5e_xdp_handle(rq, xdp_prog, di, data, cqe_bcnt))
+		return NULL; /* page/packet was consumed by XDP */
+
+	skb = build_skb(va, RQ_PAGE_SIZE(rq));
+	if (unlikely(!skb)) {
+		rq->stats.buff_alloc_err++;
+		mlx5e_page_release(rq, di, true);
+		return NULL;
+	}
+
+	/* queue up for recycling ..*/
+	page_ref_inc(di->page);
+	mlx5e_page_release(rq, di, true);
+
+	skb_reserve(skb, MLX5_RX_HEADROOM);
+	skb_put(skb, cqe_bcnt);
+
+	return skb;
 }
 
 void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 {
 	struct mlx5e_rx_wqe *wqe;
-	struct sk_buff *skb;
 	__be16 wqe_counter_be;
+	struct sk_buff *skb;
 	u16 wqe_counter;
 	u32 cqe_bcnt;
 
 	wqe_counter_be = cqe->wqe_counter;
 	wqe_counter    = be16_to_cpu(wqe_counter_be);
 	wqe            = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
-	skb            = rq->skb[wqe_counter];
-	prefetch(skb->data);
-	rq->skb[wqe_counter] = NULL;
+	cqe_bcnt       = be32_to_cpu(cqe->byte_cnt);
 
-	dma_unmap_single(rq->pdev,
-			 *((dma_addr_t *)skb->cb),
-			 rq->wqe_sz,
-			 DMA_FROM_DEVICE);
-
-	if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
-		rq->stats.wqe_err++;
-		dev_kfree_skb(skb);
+	skb = skb_from_cqe(rq, cqe, wqe_counter, cqe_bcnt);
+	if (!skb)
 		goto wq_ll_pop;
-	}
 
-	cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
-	skb_put(skb, cqe_bcnt);
+	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
+	napi_gro_receive(rq->cq.napi, skb);
+
+wq_ll_pop:
+	mlx5_wq_ll_pop(&rq->wq, wqe_counter_be,
+		       &wqe->next.next_wqe_index);
+}
+
+void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
+{
+	struct net_device *netdev = rq->netdev;
+	struct mlx5e_priv *priv = netdev_priv(netdev);
+	struct mlx5_eswitch_rep *rep = priv->ppriv;
+	struct mlx5e_rx_wqe *wqe;
+	struct sk_buff *skb;
+	__be16 wqe_counter_be;
+	u16 wqe_counter;
+	u32 cqe_bcnt;
+
+	wqe_counter_be = cqe->wqe_counter;
+	wqe_counter    = be16_to_cpu(wqe_counter_be);
+	wqe            = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
+	cqe_bcnt       = be32_to_cpu(cqe->byte_cnt);
+
+	skb = skb_from_cqe(rq, cqe, wqe_counter, cqe_bcnt);
+	if (!skb)
+		goto wq_ll_pop;
 
 	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
 
+	if (rep->vlan && skb_vlan_tag_present(skb))
+		skb_vlan_pop(skb);
+
+	napi_gro_receive(rq->cq.napi, skb);
+
 wq_ll_pop:
 	mlx5_wq_ll_pop(&rq->wq, wqe_counter_be,
 		       &wqe->next.next_wqe_index);
@@ -834,7 +842,6 @@ static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq,
 					   u32 cqe_bcnt,
 					   struct sk_buff *skb)
 {
-	u32 consumed_bytes = ALIGN(cqe_bcnt, rq->mpwqe_stride_sz);
 	u16 stride_ix      = mpwrq_get_cqe_stride_index(cqe);
 	u32 wqe_offset     = stride_ix * rq->mpwqe_stride_sz;
 	u32 head_offset    = wqe_offset & (PAGE_SIZE - 1);
@@ -848,21 +855,20 @@ static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq,
 		page_idx++;
 		frag_offset -= PAGE_SIZE;
 	}
-	wi->dma_pre_sync(rq->pdev, wi, wqe_offset, consumed_bytes);
 
 	while (byte_cnt) {
 		u32 pg_consumed_bytes =
 			min_t(u32, PAGE_SIZE - frag_offset, byte_cnt);
 
-		wi->add_skb_frag(rq, skb, wi, page_idx, frag_offset,
-				 pg_consumed_bytes);
+		mlx5e_add_skb_frag_mpwqe(rq, skb, wi, page_idx, frag_offset,
+					 pg_consumed_bytes);
 		byte_cnt -= pg_consumed_bytes;
 		frag_offset = 0;
 		page_idx++;
 	}
 	/* copy header */
-	wi->copy_skb_header(rq->pdev, skb, wi, head_page_idx, head_offset,
-			    headlen);
+	mlx5e_copy_skb_header_mpwqe(rq->pdev, skb, wi, head_page_idx,
+				    head_offset, headlen);
 	/* skb linear part was allocated with headlen and aligned to long */
 	skb->tail += headlen;
 	skb->len  += headlen;
@@ -872,7 +878,7 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 {
 	u16 cstrides       = mpwrq_get_cqe_consumed_strides(cqe);
 	u16 wqe_id         = be16_to_cpu(cqe->wqe_id);
-	struct mlx5e_mpw_info *wi = &rq->wqe_info[wqe_id];
+	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
 	struct mlx5e_rx_wqe  *wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_id);
 	struct sk_buff *skb;
 	u16 cqe_bcnt;
@@ -902,21 +908,23 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 
 	mlx5e_mpwqe_fill_rx_skb(rq, cqe, wi, cqe_bcnt, skb);
 	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
+	napi_gro_receive(rq->cq.napi, skb);
 
 mpwrq_cqe_out:
 	if (likely(wi->consumed_strides < rq->mpwqe_num_strides))
 		return;
 
-	wi->free_wqe(rq, wi);
+	mlx5e_free_rx_mpwqe(rq, wi);
 	mlx5_wq_ll_pop(&rq->wq, cqe->wqe_id, &wqe->next.next_wqe_index);
 }
 
 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
 {
 	struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
+	struct mlx5e_sq *xdp_sq = &rq->channel->xdp_sq;
 	int work_done = 0;
 
-	if (unlikely(test_bit(MLX5E_RQ_STATE_FLUSH_TIMEOUT, &rq->state)))
+	if (unlikely(test_bit(MLX5E_RQ_STATE_FLUSH, &rq->state)))
 		return 0;
 
 	if (cq->decmprs_left)
@@ -940,6 +948,11 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
 		rq->handle_rx_cqe(rq, cqe);
 	}
 
+	if (xdp_sq->db.xdp.doorbell) {
+		mlx5e_xmit_xdp_doorbell(xdp_sq);
+		xdp_sq->db.xdp.doorbell = false;
+	}
+
 	mlx5_cqwq_update_db_record(&cq->wq);
 
 	/* ensure cq space is freed before enabling more cqes */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index 7b9d8a989b52..57452fdc5154 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -65,17 +65,24 @@ struct mlx5e_sw_stats {
 	u64 rx_csum_none;
 	u64 rx_csum_complete;
 	u64 rx_csum_unnecessary_inner;
+	u64 rx_xdp_drop;
+	u64 rx_xdp_tx;
+	u64 rx_xdp_tx_full;
 	u64 tx_csum_partial;
 	u64 tx_csum_partial_inner;
 	u64 tx_queue_stopped;
 	u64 tx_queue_wake;
 	u64 tx_queue_dropped;
+	u64 tx_xmit_more;
 	u64 rx_wqe_err;
 	u64 rx_mpwqe_filler;
-	u64 rx_mpwqe_frag;
 	u64 rx_buff_alloc_err;
 	u64 rx_cqe_compress_blks;
 	u64 rx_cqe_compress_pkts;
+	u64 rx_cache_reuse;
+	u64 rx_cache_full;
+	u64 rx_cache_empty;
+	u64 rx_cache_busy;
 
 	/* Special handling counters */
 	u64 link_down_events_phy;
@@ -96,17 +103,24 @@ static const struct counter_desc sw_stats_desc[] = {
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_none) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_complete) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_unnecessary_inner) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial_inner) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_stopped) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_wake) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_dropped) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xmit_more) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_wqe_err) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_filler) },
-	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_frag) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_buff_alloc_err) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_blks) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_pkts) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_reuse) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, link_down_events_phy) },
 };
 
@@ -270,12 +284,18 @@ struct mlx5e_rq_stats {
 	u64 csum_none;
 	u64 lro_packets;
 	u64 lro_bytes;
+	u64 xdp_drop;
+	u64 xdp_tx;
+	u64 xdp_tx_full;
 	u64 wqe_err;
 	u64 mpwqe_filler;
-	u64 mpwqe_frag;
 	u64 buff_alloc_err;
 	u64 cqe_compress_blks;
 	u64 cqe_compress_pkts;
+	u64 cache_reuse;
+	u64 cache_full;
+	u64 cache_empty;
+	u64 cache_busy;
 };
 
 static const struct counter_desc rq_stats_desc[] = {
@@ -284,20 +304,27 @@ static const struct counter_desc rq_stats_desc[] = {
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_complete) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_unnecessary_inner) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_none) },
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_drop) },
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx) },
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx_full) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_packets) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_bytes) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, wqe_err) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, mpwqe_filler) },
-	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, mpwqe_frag) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, buff_alloc_err) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_reuse) },
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) },
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) },
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_busy) },
 };
 
 struct mlx5e_sq_stats {
 	/* commonly accessed in data path */
 	u64 packets;
 	u64 bytes;
+	u64 xmit_more;
 	u64 tso_packets;
 	u64 tso_bytes;
 	u64 tso_inner_packets;
@@ -324,6 +351,7 @@ static const struct counter_desc sq_stats_desc[] = {
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, stopped) },
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, wake) },
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, dropped) },
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, xmit_more) },
 };
 
 #define NUM_SW_COUNTERS			ARRAY_SIZE(sw_stats_desc)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index 0f19b01e3fff..ce8c54d18906 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -39,6 +39,7 @@
 #include <linux/rhashtable.h>
 #include <net/switchdev.h>
 #include <net/tc_act/tc_mirred.h>
+#include <net/tc_act/tc_vlan.h>
 #include "en.h"
 #include "en_tc.h"
 #include "eswitch.h"
@@ -47,6 +48,7 @@ struct mlx5e_tc_flow {
 	struct rhash_head	node;
 	u64			cookie;
 	struct mlx5_flow_rule	*rule;
+	struct mlx5_esw_flow_attr *attr;
 };
 
 #define MLX5E_TC_TABLE_NUM_ENTRIES 1024
@@ -114,27 +116,30 @@ err_create_ft:
 
 static struct mlx5_flow_rule *mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv,
 						    struct mlx5_flow_spec *spec,
-						    u32 action, u32 dst_vport)
+						    struct mlx5_esw_flow_attr *attr)
 {
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
-	struct mlx5_eswitch_rep *rep = priv->ppriv;
-	u32 src_vport;
+	int err;
 
-	if (rep->vport) /* set source vport for the flow */
-		src_vport = rep->vport;
-	else
-		src_vport = FDB_UPLINK_VPORT;
+	err = mlx5_eswitch_add_vlan_action(esw, attr);
+	if (err)
+		return ERR_PTR(err);
 
-	return mlx5_eswitch_add_offloaded_rule(esw, spec, action, src_vport, dst_vport);
+	return mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
 }
 
 static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
-			      struct mlx5_flow_rule *rule)
+			      struct mlx5_flow_rule *rule,
+			      struct mlx5_esw_flow_attr *attr)
 {
+	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	struct mlx5_fc *counter = NULL;
 
 	counter = mlx5_flow_rule_counter(rule);
 
+	if (esw && esw->mode == SRIOV_OFFLOADS)
+		mlx5_eswitch_del_vlan_action(esw, attr);
+
 	mlx5_del_flow_rule(rule);
 
 	mlx5_fc_destroy(priv->mdev, counter);
@@ -159,6 +164,7 @@ static int parse_cls_flower(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec
 	    ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
 	      BIT(FLOW_DISSECTOR_KEY_BASIC) |
 	      BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
+	      BIT(FLOW_DISSECTOR_KEY_VLAN) |
 	      BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 	      BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 	      BIT(FLOW_DISSECTOR_KEY_PORTS))) {
@@ -170,7 +176,7 @@ static int parse_cls_flower(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec
 	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
 		struct flow_dissector_key_control *key =
 			skb_flow_dissector_target(f->dissector,
-						  FLOW_DISSECTOR_KEY_BASIC,
+						  FLOW_DISSECTOR_KEY_CONTROL,
 						  f->key);
 		addr_type = key->addr_type;
 	}
@@ -222,6 +228,24 @@ static int parse_cls_flower(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec
 				key->src);
 	}
 
+	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
+		struct flow_dissector_key_vlan *key =
+			skb_flow_dissector_target(f->dissector,
+						  FLOW_DISSECTOR_KEY_VLAN,
+						  f->key);
+		struct flow_dissector_key_vlan *mask =
+			skb_flow_dissector_target(f->dissector,
+						  FLOW_DISSECTOR_KEY_VLAN,
+						  f->mask);
+		if (mask->vlan_id) {
+			MLX5_SET(fte_match_set_lyr_2_4, headers_c, vlan_tag, 1);
+			MLX5_SET(fte_match_set_lyr_2_4, headers_v, vlan_tag, 1);
+
+			MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_vid, mask->vlan_id);
+			MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, key->vlan_id);
+		}
+	}
+
 	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 		struct flow_dissector_key_ipv4_addrs *key =
 			skb_flow_dissector_target(f->dissector,
@@ -318,6 +342,7 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 				u32 *action, u32 *flow_tag)
 {
 	const struct tc_action *a;
+	LIST_HEAD(actions);
 
 	if (tc_no_actions(exts))
 		return -EINVAL;
@@ -325,7 +350,8 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 	*flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
 	*action = 0;
 
-	tc_for_each_action(a, exts) {
+	tcf_exts_to_list(exts, &actions);
+	list_for_each_entry(a, &actions, list) {
 		/* Only support a single action per rule */
 		if (*action)
 			return -EINVAL;
@@ -359,23 +385,22 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 }
 
 static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
-				u32 *action, u32 *dest_vport)
+				struct mlx5_esw_flow_attr *attr)
 {
 	const struct tc_action *a;
+	LIST_HEAD(actions);
 
 	if (tc_no_actions(exts))
 		return -EINVAL;
 
-	*action = 0;
-
-	tc_for_each_action(a, exts) {
-		/* Only support a single action per rule */
-		if (*action)
-			return -EINVAL;
+	memset(attr, 0, sizeof(*attr));
+	attr->in_rep = priv->ppriv;
 
+	tcf_exts_to_list(exts, &actions);
+	list_for_each_entry(a, &actions, list) {
 		if (is_tcf_gact_shot(a)) {
-			*action = MLX5_FLOW_CONTEXT_ACTION_DROP |
-				  MLX5_FLOW_CONTEXT_ACTION_COUNT;
+			attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
+					MLX5_FLOW_CONTEXT_ACTION_COUNT;
 			continue;
 		}
 
@@ -383,7 +408,6 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 			int ifindex = tcf_mirred_ifindex(a);
 			struct net_device *out_dev;
 			struct mlx5e_priv *out_priv;
-			struct mlx5_eswitch_rep *out_rep;
 
 			out_dev = __dev_get_by_index(dev_net(priv->netdev), ifindex);
 
@@ -393,13 +417,22 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 				return -EINVAL;
 			}
 
+			attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 			out_priv = netdev_priv(out_dev);
-			out_rep  = out_priv->ppriv;
-			if (out_rep->vport == 0)
-				*dest_vport = FDB_UPLINK_VPORT;
-			else
-				*dest_vport = out_rep->vport;
-			*action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+			attr->out_rep = out_priv->ppriv;
+			continue;
+		}
+
+		if (is_tcf_vlan(a)) {
+			if (tcf_vlan_action(a) == VLAN_F_POP) {
+				attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP;
+			} else if (tcf_vlan_action(a) == VLAN_F_PUSH) {
+				if (tcf_vlan_push_proto(a) != htons(ETH_P_8021Q))
+					return -EOPNOTSUPP;
+
+				attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
+				attr->vlan = tcf_vlan_push_vid(a);
+			}
 			continue;
 		}
 
@@ -413,18 +446,29 @@ int mlx5e_configure_flower(struct mlx5e_priv *priv, __be16 protocol,
 {
 	struct mlx5e_tc_table *tc = &priv->fs.tc;
 	int err = 0;
-	u32 flow_tag, action, dest_vport = 0;
+	bool fdb_flow = false;
+	u32 flow_tag, action;
 	struct mlx5e_tc_flow *flow;
 	struct mlx5_flow_spec *spec;
 	struct mlx5_flow_rule *old = NULL;
+	struct mlx5_esw_flow_attr *old_attr = NULL;
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 
+	if (esw && esw->mode == SRIOV_OFFLOADS)
+		fdb_flow = true;
+
 	flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
 				      tc->ht_params);
-	if (flow)
+	if (flow) {
 		old = flow->rule;
-	else
-		flow = kzalloc(sizeof(*flow), GFP_KERNEL);
+		old_attr = flow->attr;
+	} else {
+		if (fdb_flow)
+			flow = kzalloc(sizeof(*flow) + sizeof(struct mlx5_esw_flow_attr),
+				       GFP_KERNEL);
+		else
+			flow = kzalloc(sizeof(*flow), GFP_KERNEL);
+	}
 
 	spec = mlx5_vzalloc(sizeof(*spec));
 	if (!spec || !flow) {
@@ -438,11 +482,12 @@ int mlx5e_configure_flower(struct mlx5e_priv *priv, __be16 protocol,
 	if (err < 0)
 		goto err_free;
 
-	if (esw && esw->mode == SRIOV_OFFLOADS) {
-		err = parse_tc_fdb_actions(priv, f->exts, &action, &dest_vport);
+	if (fdb_flow) {
+		flow->attr  = (struct mlx5_esw_flow_attr *)(flow + 1);
+		err = parse_tc_fdb_actions(priv, f->exts, flow->attr);
 		if (err < 0)
 			goto err_free;
-		flow->rule = mlx5e_tc_add_fdb_flow(priv, spec, action, dest_vport);
+		flow->rule = mlx5e_tc_add_fdb_flow(priv, spec, flow->attr);
 	} else {
 		err = parse_tc_nic_actions(priv, f->exts, &action, &flow_tag);
 		if (err < 0)
@@ -461,7 +506,7 @@ int mlx5e_configure_flower(struct mlx5e_priv *priv, __be16 protocol,
 		goto err_del_rule;
 
 	if (old)
-		mlx5e_tc_del_flow(priv, old);
+		mlx5e_tc_del_flow(priv, old, old_attr);
 
 	goto out;
 
@@ -489,7 +534,7 @@ int mlx5e_delete_flower(struct mlx5e_priv *priv,
 
 	rhashtable_remove_fast(&tc->ht, &flow->node, tc->ht_params);
 
-	mlx5e_tc_del_flow(priv, flow->rule);
+	mlx5e_tc_del_flow(priv, flow->rule, flow->attr);
 
 	kfree(flow);
 
@@ -503,6 +548,7 @@ int mlx5e_stats_flower(struct mlx5e_priv *priv,
 	struct mlx5e_tc_flow *flow;
 	struct tc_action *a;
 	struct mlx5_fc *counter;
+	LIST_HEAD(actions);
 	u64 bytes;
 	u64 packets;
 	u64 lastuse;
@@ -518,7 +564,8 @@ int mlx5e_stats_flower(struct mlx5e_priv *priv,
 
 	mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse);
 
-	tc_for_each_action(a, f->exts)
+	tcf_exts_to_list(f->exts, &actions);
+	list_for_each_entry(a, &actions, list)
 		tcf_action_stats_update(a, bytes, packets, lastuse);
 
 	return 0;
@@ -544,7 +591,7 @@ static void _mlx5e_tc_del_flow(void *ptr, void *arg)
 	struct mlx5e_tc_flow *flow = ptr;
 	struct mlx5e_priv *priv = arg;
 
-	mlx5e_tc_del_flow(priv, flow->rule);
+	mlx5e_tc_del_flow(priv, flow->rule, flow->attr);
 	kfree(flow);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index e073bf59890d..70a717382357 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -52,7 +52,6 @@ void mlx5e_send_nop(struct mlx5e_sq *sq, bool notify_hw)
 	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
 	cseg->qpn_ds           = cpu_to_be32((sq->sqn << 8) | 0x01);
 
-	sq->skb[pi] = NULL;
 	sq->pc++;
 	sq->stats.nop++;
 
@@ -82,15 +81,17 @@ static inline void mlx5e_dma_push(struct mlx5e_sq *sq,
 				  u32 size,
 				  enum mlx5e_dma_map_type map_type)
 {
-	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].addr = addr;
-	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].size = size;
-	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].type = map_type;
+	u32 i = sq->dma_fifo_pc & sq->dma_fifo_mask;
+
+	sq->db.txq.dma_fifo[i].addr = addr;
+	sq->db.txq.dma_fifo[i].size = size;
+	sq->db.txq.dma_fifo[i].type = map_type;
 	sq->dma_fifo_pc++;
 }
 
 static inline struct mlx5e_sq_dma *mlx5e_dma_get(struct mlx5e_sq *sq, u32 i)
 {
-	return &sq->dma_fifo[i & sq->dma_fifo_mask];
+	return &sq->db.txq.dma_fifo[i & sq->dma_fifo_mask];
 }
 
 static void mlx5e_dma_unmap_wqe_err(struct mlx5e_sq *sq, u8 num_dma)
@@ -221,7 +222,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
 
 	u16 pi = sq->pc & wq->sz_m1;
 	struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
-	struct mlx5e_tx_wqe_info *wi   = &sq->wqe_info[pi];
+	struct mlx5e_tx_wqe_info *wi   = &sq->db.txq.wqe_info[pi];
 
 	struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
 	struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
@@ -341,7 +342,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
 	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
 	cseg->qpn_ds           = cpu_to_be32((sq->sqn << 8) | ds_cnt);
 
-	sq->skb[pi] = skb;
+	sq->db.txq.skb[pi] = skb;
 
 	wi->num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
 	sq->pc += wi->num_wqebbs;
@@ -356,6 +357,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
 		sq->stats.stopped++;
 	}
 
+	sq->stats.xmit_more += skb->xmit_more;
 	if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) {
 		int bf_sz = 0;
 
@@ -367,8 +369,10 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
 	}
 
 	/* fill sq edge with nops to avoid wqe wrap around */
-	while ((sq->pc & wq->sz_m1) > sq->edge)
+	while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
+		sq->db.txq.skb[pi] = NULL;
 		mlx5e_send_nop(sq, false);
+	}
 
 	if (bf)
 		sq->bf_budget--;
@@ -394,35 +398,6 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
 	return mlx5e_sq_xmit(sq, skb);
 }
 
-void mlx5e_free_tx_descs(struct mlx5e_sq *sq)
-{
-	struct mlx5e_tx_wqe_info *wi;
-	struct sk_buff *skb;
-	u16 ci;
-	int i;
-
-	while (sq->cc != sq->pc) {
-		ci = sq->cc & sq->wq.sz_m1;
-		skb = sq->skb[ci];
-		wi = &sq->wqe_info[ci];
-
-		if (!skb) { /* nop */
-			sq->cc++;
-			continue;
-		}
-
-		for (i = 0; i < wi->num_dma; i++) {
-			struct mlx5e_sq_dma *dma =
-				mlx5e_dma_get(sq, sq->dma_fifo_cc++);
-
-			mlx5e_tx_dma_unmap(sq->pdev, dma);
-		}
-
-		dev_kfree_skb_any(skb);
-		sq->cc += wi->num_wqebbs;
-	}
-}
-
 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 {
 	struct mlx5e_sq *sq;
@@ -434,7 +409,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 
 	sq = container_of(cq, struct mlx5e_sq, cq);
 
-	if (unlikely(test_bit(MLX5E_SQ_STATE_TX_TIMEOUT, &sq->state)))
+	if (unlikely(test_bit(MLX5E_SQ_STATE_FLUSH, &sq->state)))
 		return false;
 
 	npkts = 0;
@@ -470,8 +445,8 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 			last_wqe = (sqcc == wqe_counter);
 
 			ci = sqcc & sq->wq.sz_m1;
-			skb = sq->skb[ci];
-			wi = &sq->wqe_info[ci];
+			skb = sq->db.txq.skb[ci];
+			wi = &sq->db.txq.wqe_info[ci];
 
 			if (unlikely(!skb)) { /* nop */
 				sqcc++;
@@ -512,11 +487,73 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 	netdev_tx_completed_queue(sq->txq, npkts, nbytes);
 
 	if (netif_tx_queue_stopped(sq->txq) &&
-	    mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) &&
-	    likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) {
-				netif_tx_wake_queue(sq->txq);
-				sq->stats.wake++;
+	    mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM)) {
+		netif_tx_wake_queue(sq->txq);
+		sq->stats.wake++;
 	}
 
 	return (i == MLX5E_TX_CQ_POLL_BUDGET);
 }
+
+static void mlx5e_free_txq_sq_descs(struct mlx5e_sq *sq)
+{
+	struct mlx5e_tx_wqe_info *wi;
+	struct sk_buff *skb;
+	u16 ci;
+	int i;
+
+	while (sq->cc != sq->pc) {
+		ci = sq->cc & sq->wq.sz_m1;
+		skb = sq->db.txq.skb[ci];
+		wi = &sq->db.txq.wqe_info[ci];
+
+		if (!skb) { /* nop */
+			sq->cc++;
+			continue;
+		}
+
+		for (i = 0; i < wi->num_dma; i++) {
+			struct mlx5e_sq_dma *dma =
+				mlx5e_dma_get(sq, sq->dma_fifo_cc++);
+
+			mlx5e_tx_dma_unmap(sq->pdev, dma);
+		}
+
+		dev_kfree_skb_any(skb);
+		sq->cc += wi->num_wqebbs;
+	}
+}
+
+static void mlx5e_free_xdp_sq_descs(struct mlx5e_sq *sq)
+{
+	struct mlx5e_sq_wqe_info *wi;
+	struct mlx5e_dma_info *di;
+	u16 ci;
+
+	while (sq->cc != sq->pc) {
+		ci = sq->cc & sq->wq.sz_m1;
+		di = &sq->db.xdp.di[ci];
+		wi = &sq->db.xdp.wqe_info[ci];
+
+		if (wi->opcode == MLX5_OPCODE_NOP) {
+			sq->cc++;
+			continue;
+		}
+
+		sq->cc += wi->num_wqebbs;
+
+		mlx5e_page_release(&sq->channel->rq, di, false);
+	}
+}
+
+void mlx5e_free_sq_descs(struct mlx5e_sq *sq)
+{
+	switch (sq->type) {
+	case MLX5E_SQ_TXQ:
+		mlx5e_free_txq_sq_descs(sq);
+		break;
+	case MLX5E_SQ_XDP:
+		mlx5e_free_xdp_sq_descs(sq);
+		break;
+	}
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index 64ae2e800daa..5703f19a6a24 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -51,16 +51,18 @@ struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq)
 
 static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
 {
+	struct mlx5e_sq *sq = container_of(cq, struct mlx5e_sq, cq);
 	struct mlx5_wq_cyc *wq;
 	struct mlx5_cqe64 *cqe;
-	struct mlx5e_sq *sq;
 	u16 sqcc;
 
+	if (unlikely(test_bit(MLX5E_SQ_STATE_FLUSH, &sq->state)))
+		return;
+
 	cqe = mlx5e_get_cqe(cq);
 	if (likely(!cqe))
 		return;
 
-	sq = container_of(cq, struct mlx5e_sq, cq);
 	wq = &sq->wq;
 
 	/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
@@ -70,7 +72,7 @@ static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
 
 	do {
 		u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1;
-		struct mlx5e_ico_wqe_info *icowi = &sq->ico_wqe_info[ci];
+		struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci];
 
 		mlx5_cqwq_pop(&cq->wq);
 		sqcc += icowi->num_wqebbs;
@@ -85,7 +87,7 @@ static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
 		case MLX5_OPCODE_NOP:
 			break;
 		case MLX5_OPCODE_UMR:
-			mlx5e_post_rx_fragmented_mpwqe(&sq->channel->rq);
+			mlx5e_post_rx_mpwqe(&sq->channel->rq);
 			break;
 		default:
 			WARN_ONCE(true,
@@ -103,6 +105,66 @@ static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
 	sq->cc = sqcc;
 }
 
+static inline bool mlx5e_poll_xdp_tx_cq(struct mlx5e_cq *cq)
+{
+	struct mlx5e_sq *sq;
+	u16 sqcc;
+	int i;
+
+	sq = container_of(cq, struct mlx5e_sq, cq);
+
+	if (unlikely(test_bit(MLX5E_SQ_STATE_FLUSH, &sq->state)))
+		return false;
+
+	/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
+	 * otherwise a cq overrun may occur
+	 */
+	sqcc = sq->cc;
+
+	for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) {
+		struct mlx5_cqe64 *cqe;
+		u16 wqe_counter;
+		bool last_wqe;
+
+		cqe = mlx5e_get_cqe(cq);
+		if (!cqe)
+			break;
+
+		mlx5_cqwq_pop(&cq->wq);
+
+		wqe_counter = be16_to_cpu(cqe->wqe_counter);
+
+		do {
+			struct mlx5e_sq_wqe_info *wi;
+			struct mlx5e_dma_info *di;
+			u16 ci;
+
+			last_wqe = (sqcc == wqe_counter);
+
+			ci = sqcc & sq->wq.sz_m1;
+			di = &sq->db.xdp.di[ci];
+			wi = &sq->db.xdp.wqe_info[ci];
+
+			if (unlikely(wi->opcode == MLX5_OPCODE_NOP)) {
+				sqcc++;
+				continue;
+			}
+
+			sqcc += wi->num_wqebbs;
+			/* Recycle RX page */
+			mlx5e_page_release(&sq->channel->rq, di, true);
+		} while (!last_wqe);
+	}
+
+	mlx5_cqwq_update_db_record(&cq->wq);
+
+	/* ensure cq space is freed before enabling more cqes */
+	wmb();
+
+	sq->cc = sqcc;
+	return (i == MLX5E_TX_CQ_POLL_BUDGET);
+}
+
 int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 {
 	struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel,
@@ -119,6 +181,9 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 	work_done = mlx5e_poll_rx_cq(&c->rq.cq, budget);
 	busy |= work_done == budget;
 
+	if (c->xdp)
+		busy |= mlx5e_poll_xdp_tx_cq(&c->xdp_sq.cq);
+
 	mlx5e_poll_ico_cq(&c->icosq.cq);
 
 	busy |= mlx5e_post_rx_wqes(&c->rq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 0e30602ef76d..aaca09002ca6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -86,23 +86,12 @@ struct cre_des_eq {
 
 static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
 {
-	struct mlx5_destroy_eq_mbox_in in;
-	struct mlx5_destroy_eq_mbox_out out;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_EQ);
-	in.eqn = eqn;
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (!err)
-		goto ex;
-
-	if (out.hdr.status)
-		err = mlx5_cmd_status_to_err(&out.hdr);
+	u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_eq_in)]   = {0};
 
-ex:
-	return err;
+	MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
+	MLX5_SET(destroy_eq_in, in, eq_number, eqn);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 static struct mlx5_eqe *get_eqe(struct mlx5_eq *eq, u32 entry)
@@ -351,11 +340,13 @@ static void init_eq_buf(struct mlx5_eq *eq)
 int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
 		       int nent, u64 mask, const char *name, struct mlx5_uar *uar)
 {
+	u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
 	struct mlx5_priv *priv = &dev->priv;
-	struct mlx5_create_eq_mbox_in *in;
-	struct mlx5_create_eq_mbox_out out;
-	int err;
+	__be64 *pas;
+	void *eqc;
 	int inlen;
+	u32 *in;
+	int err;
 
 	eq->nent = roundup_pow_of_two(nent + MLX5_NUM_SPARE_EQE);
 	eq->cons_index = 0;
@@ -365,35 +356,36 @@ int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
 
 	init_eq_buf(eq);
 
-	inlen = sizeof(*in) + sizeof(in->pas[0]) * eq->buf.npages;
+	inlen = MLX5_ST_SZ_BYTES(create_eq_in) +
+		MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->buf.npages;
+
 	in = mlx5_vzalloc(inlen);
 	if (!in) {
 		err = -ENOMEM;
 		goto err_buf;
 	}
-	memset(&out, 0, sizeof(out));
 
-	mlx5_fill_page_array(&eq->buf, in->pas);
+	pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas);
+	mlx5_fill_page_array(&eq->buf, pas);
 
-	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_EQ);
-	in->ctx.log_sz_usr_page = cpu_to_be32(ilog2(eq->nent) << 24 | uar->index);
-	in->ctx.intr = vecidx;
-	in->ctx.log_page_size = eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
-	in->events_mask = cpu_to_be64(mask);
+	MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
+	MLX5_SET64(create_eq_in, in, event_bitmask, mask);
 
-	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
-	if (err)
-		goto err_in;
+	eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
+	MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent));
+	MLX5_SET(eqc, eqc, uar_page, uar->index);
+	MLX5_SET(eqc, eqc, intr, vecidx);
+	MLX5_SET(eqc, eqc, log_page_size,
+		 eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
 
-	if (out.hdr.status) {
-		err = mlx5_cmd_status_to_err(&out.hdr);
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+	if (err)
 		goto err_in;
-	}
 
 	snprintf(priv->irq_info[vecidx].name, MLX5_MAX_IRQ_NAME, "%s@pci:%s",
 		 name, pci_name(dev->pdev));
 
-	eq->eqn = out.eq_number;
+	eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
 	eq->irqn = priv->msix_arr[vecidx].vector;
 	eq->dev = dev;
 	eq->doorbell = uar->map + MLX5_EQ_DOORBEL_OFFSET;
@@ -547,22 +539,12 @@ int mlx5_stop_eqs(struct mlx5_core_dev *dev)
 }
 
 int mlx5_core_eq_query(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
-		       struct mlx5_query_eq_mbox_out *out, int outlen)
+		       u32 *out, int outlen)
 {
-	struct mlx5_query_eq_mbox_in in;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(out, 0, outlen);
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_EQ);
-	in.eqn = eq->eqn;
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
-	if (err)
-		return err;
+	u32 in[MLX5_ST_SZ_DW(query_eq_in)] = {0};
 
-	if (out->hdr.status)
-		err = mlx5_cmd_status_to_err(&out->hdr);
-
-	return err;
+	MLX5_SET(query_eq_in, in, opcode, MLX5_CMD_OP_QUERY_EQ);
+	MLX5_SET(query_eq_in, in, eq_number, eq->eqn);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 }
 EXPORT_SYMBOL_GPL(mlx5_core_eq_query);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index f6d667797ee1..abbf2c369923 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -81,19 +81,12 @@ enum {
 			    MC_ADDR_CHANGE | \
 			    PROMISC_CHANGE)
 
-int esw_offloads_init(struct mlx5_eswitch *esw, int nvports);
-void esw_offloads_cleanup(struct mlx5_eswitch *esw, int nvports);
-
 static int arm_vport_context_events_cmd(struct mlx5_core_dev *dev, u16 vport,
 					u32 events_mask)
 {
-	int in[MLX5_ST_SZ_DW(modify_nic_vport_context_in)];
-	int out[MLX5_ST_SZ_DW(modify_nic_vport_context_out)];
+	int in[MLX5_ST_SZ_DW(modify_nic_vport_context_in)]   = {0};
+	int out[MLX5_ST_SZ_DW(modify_nic_vport_context_out)] = {0};
 	void *nic_vport_ctx;
-	int err;
-
-	memset(out, 0, sizeof(out));
-	memset(in, 0, sizeof(in));
 
 	MLX5_SET(modify_nic_vport_context_in, in,
 		 opcode, MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT);
@@ -116,113 +109,44 @@ static int arm_vport_context_events_cmd(struct mlx5_core_dev *dev, u16 vport,
 		MLX5_SET(nic_vport_context, nic_vport_ctx,
 			 event_on_promisc_change, 1);
 
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-	if (err)
-		goto ex;
-	err = mlx5_cmd_status_to_err_v2(out);
-	if (err)
-		goto ex;
-	return 0;
-ex:
-	return err;
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 /* E-Switch vport context HW commands */
-static int query_esw_vport_context_cmd(struct mlx5_core_dev *mdev, u32 vport,
-				       u32 *out, int outlen)
-{
-	u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)];
-
-	memset(in, 0, sizeof(in));
-
-	MLX5_SET(query_nic_vport_context_in, in, opcode,
-		 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
-
-	MLX5_SET(query_esw_vport_context_in, in, vport_number, vport);
-	if (vport)
-		MLX5_SET(query_esw_vport_context_in, in, other_vport, 1);
-
-	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out, outlen);
-}
-
-static int query_esw_vport_cvlan(struct mlx5_core_dev *dev, u32 vport,
-				 u16 *vlan, u8 *qos)
-{
-	u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)];
-	int err;
-	bool cvlan_strip;
-	bool cvlan_insert;
-
-	memset(out, 0, sizeof(out));
-
-	*vlan = 0;
-	*qos = 0;
-
-	if (!MLX5_CAP_ESW(dev, vport_cvlan_strip) ||
-	    !MLX5_CAP_ESW(dev, vport_cvlan_insert_if_not_exist))
-		return -ENOTSUPP;
-
-	err = query_esw_vport_context_cmd(dev, vport, out, sizeof(out));
-	if (err)
-		goto out;
-
-	cvlan_strip = MLX5_GET(query_esw_vport_context_out, out,
-			       esw_vport_context.vport_cvlan_strip);
-
-	cvlan_insert = MLX5_GET(query_esw_vport_context_out, out,
-				esw_vport_context.vport_cvlan_insert);
-
-	if (cvlan_strip || cvlan_insert) {
-		*vlan = MLX5_GET(query_esw_vport_context_out, out,
-				 esw_vport_context.cvlan_id);
-		*qos = MLX5_GET(query_esw_vport_context_out, out,
-				esw_vport_context.cvlan_pcp);
-	}
-
-	esw_debug(dev, "Query Vport[%d] cvlan: VLAN %d qos=%d\n",
-		  vport, *vlan, *qos);
-out:
-	return err;
-}
-
 static int modify_esw_vport_context_cmd(struct mlx5_core_dev *dev, u16 vport,
 					void *in, int inlen)
 {
-	u32 out[MLX5_ST_SZ_DW(modify_esw_vport_context_out)];
-
-	memset(out, 0, sizeof(out));
+	u32 out[MLX5_ST_SZ_DW(modify_esw_vport_context_out)] = {0};
 
+	MLX5_SET(modify_esw_vport_context_in, in, opcode,
+		 MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT);
 	MLX5_SET(modify_esw_vport_context_in, in, vport_number, vport);
 	if (vport)
 		MLX5_SET(modify_esw_vport_context_in, in, other_vport, 1);
-
-	MLX5_SET(modify_esw_vport_context_in, in, opcode,
-		 MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT);
-
-	return mlx5_cmd_exec_check_status(dev, in, inlen,
-					  out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 }
 
 static int modify_esw_vport_cvlan(struct mlx5_core_dev *dev, u32 vport,
-				  u16 vlan, u8 qos, bool set)
+				  u16 vlan, u8 qos, u8 set_flags)
 {
-	u32 in[MLX5_ST_SZ_DW(modify_esw_vport_context_in)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {0};
 
 	if (!MLX5_CAP_ESW(dev, vport_cvlan_strip) ||
 	    !MLX5_CAP_ESW(dev, vport_cvlan_insert_if_not_exist))
 		return -ENOTSUPP;
 
-	esw_debug(dev, "Set Vport[%d] VLAN %d qos %d set=%d\n",
-		  vport, vlan, qos, set);
+	esw_debug(dev, "Set Vport[%d] VLAN %d qos %d set=%x\n",
+		  vport, vlan, qos, set_flags);
 
-	if (set) {
+	if (set_flags & SET_VLAN_STRIP)
 		MLX5_SET(modify_esw_vport_context_in, in,
 			 esw_vport_context.vport_cvlan_strip, 1);
+
+	if (set_flags & SET_VLAN_INSERT) {
 		/* insert only if no vlan in packet */
 		MLX5_SET(modify_esw_vport_context_in, in,
 			 esw_vport_context.vport_cvlan_insert, 1);
+
 		MLX5_SET(modify_esw_vport_context_in, in,
 			 esw_vport_context.cvlan_pcp, qos);
 		MLX5_SET(modify_esw_vport_context_in, in,
@@ -241,13 +165,10 @@ static int modify_esw_vport_cvlan(struct mlx5_core_dev *dev, u32 vport,
 static int set_l2_table_entry_cmd(struct mlx5_core_dev *dev, u32 index,
 				  u8 *mac, u8 vlan_valid, u16 vlan)
 {
-	u32 in[MLX5_ST_SZ_DW(set_l2_table_entry_in)];
-	u32 out[MLX5_ST_SZ_DW(set_l2_table_entry_out)];
+	u32 in[MLX5_ST_SZ_DW(set_l2_table_entry_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(set_l2_table_entry_out)] = {0};
 	u8 *in_mac_addr;
 
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
-
 	MLX5_SET(set_l2_table_entry_in, in, opcode,
 		 MLX5_CMD_OP_SET_L2_TABLE_ENTRY);
 	MLX5_SET(set_l2_table_entry_in, in, table_index, index);
@@ -257,23 +178,18 @@ static int set_l2_table_entry_cmd(struct mlx5_core_dev *dev, u32 index,
 	in_mac_addr = MLX5_ADDR_OF(set_l2_table_entry_in, in, mac_address);
 	ether_addr_copy(&in_mac_addr[2], mac);
 
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in),
-					  out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 static int del_l2_table_entry_cmd(struct mlx5_core_dev *dev, u32 index)
 {
-	u32 in[MLX5_ST_SZ_DW(delete_l2_table_entry_in)];
-	u32 out[MLX5_ST_SZ_DW(delete_l2_table_entry_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(delete_l2_table_entry_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(delete_l2_table_entry_out)] = {0};
 
 	MLX5_SET(delete_l2_table_entry_in, in, opcode,
 		 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
 	MLX5_SET(delete_l2_table_entry_in, in, table_index, index);
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in),
-					  out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 static int alloc_l2_table_index(struct mlx5_l2_table *l2_table, u32 *ix)
@@ -340,7 +256,7 @@ __esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u32 vport, bool rx_rule,
 
 	spec = mlx5_vzalloc(sizeof(*spec));
 	if (!spec) {
-		pr_warn("FDB: Failed to alloc match parameters\n");
+		esw_warn(esw->dev, "FDB: Failed to alloc match parameters\n");
 		return NULL;
 	}
 	dmac_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
@@ -374,8 +290,8 @@ __esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u32 vport, bool rx_rule,
 				   MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 				   0, &dest);
 	if (IS_ERR(flow_rule)) {
-		pr_warn(
-			"FDB: Failed to add flow rule: dmac_v(%pM) dmac_c(%pM) -> vport(%d), err(%ld)\n",
+		esw_warn(esw->dev,
+			 "FDB: Failed to add flow rule: dmac_v(%pM) dmac_c(%pM) -> vport(%d), err(%ld)\n",
 			 dmac_v, dmac_c, vport, PTR_ERR(flow_rule));
 		flow_rule = NULL;
 	}
@@ -955,7 +871,7 @@ static void esw_update_vport_rx_mode(struct mlx5_eswitch *esw, u32 vport_num)
 	esw_debug(esw->dev, "vport[%d] context update rx mode promisc_all=%d, all_multi=%d\n",
 		  vport_num, promisc_all, promisc_mc);
 
-	if (!vport->trusted || !vport->enabled) {
+	if (!vport->info.trusted || !vport->enabled) {
 		promisc_uc = 0;
 		promisc_mc = 0;
 		promisc_all = 0;
@@ -1291,30 +1207,20 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 				    struct mlx5_vport *vport)
 {
 	struct mlx5_flow_spec *spec;
-	u8 smac[ETH_ALEN];
 	int err = 0;
 	u8 *smac_v;
 
-	if (vport->spoofchk) {
-		err = mlx5_query_nic_vport_mac_address(esw->dev, vport->vport, smac);
-		if (err) {
-			esw_warn(esw->dev,
-				 "vport[%d] configure ingress rules failed, query smac failed, err(%d)\n",
-				 vport->vport, err);
-			return err;
-		}
+	if (vport->info.spoofchk && !is_valid_ether_addr(vport->info.mac)) {
+		mlx5_core_warn(esw->dev,
+			       "vport[%d] configure ingress rules failed, illegal mac with spoofchk\n",
+			       vport->vport);
+		return -EPERM;
 
-		if (!is_valid_ether_addr(smac)) {
-			mlx5_core_warn(esw->dev,
-				       "vport[%d] configure ingress rules failed, illegal mac with spoofchk\n",
-				       vport->vport);
-			return -EPERM;
-		}
 	}
 
 	esw_vport_cleanup_ingress_rules(esw, vport);
 
-	if (!vport->vlan && !vport->qos && !vport->spoofchk) {
+	if (!vport->info.vlan && !vport->info.qos && !vport->info.spoofchk) {
 		esw_vport_disable_ingress_acl(esw, vport);
 		return 0;
 	}
@@ -1323,7 +1229,7 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 
 	esw_debug(esw->dev,
 		  "vport[%d] configure ingress rules, vlan(%d) qos(%d)\n",
-		  vport->vport, vport->vlan, vport->qos);
+		  vport->vport, vport->info.vlan, vport->info.qos);
 
 	spec = mlx5_vzalloc(sizeof(*spec));
 	if (!spec) {
@@ -1333,16 +1239,16 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 		goto out;
 	}
 
-	if (vport->vlan || vport->qos)
+	if (vport->info.vlan || vport->info.qos)
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.vlan_tag);
 
-	if (vport->spoofchk) {
+	if (vport->info.spoofchk) {
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_47_16);
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_15_0);
 		smac_v = MLX5_ADDR_OF(fte_match_param,
 				      spec->match_value,
 				      outer_headers.smac_47_16);
-		ether_addr_copy(smac_v, smac);
+		ether_addr_copy(smac_v, vport->info.mac);
 	}
 
 	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
@@ -1352,8 +1258,9 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 				   0, NULL);
 	if (IS_ERR(vport->ingress.allow_rule)) {
 		err = PTR_ERR(vport->ingress.allow_rule);
-		pr_warn("vport[%d] configure ingress allow rule, err(%d)\n",
-			vport->vport, err);
+		esw_warn(esw->dev,
+			 "vport[%d] configure ingress allow rule, err(%d)\n",
+			 vport->vport, err);
 		vport->ingress.allow_rule = NULL;
 		goto out;
 	}
@@ -1365,8 +1272,9 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 				   0, NULL);
 	if (IS_ERR(vport->ingress.drop_rule)) {
 		err = PTR_ERR(vport->ingress.drop_rule);
-		pr_warn("vport[%d] configure ingress drop rule, err(%d)\n",
-			vport->vport, err);
+		esw_warn(esw->dev,
+			 "vport[%d] configure ingress drop rule, err(%d)\n",
+			 vport->vport, err);
 		vport->ingress.drop_rule = NULL;
 		goto out;
 	}
@@ -1386,7 +1294,7 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 
 	esw_vport_cleanup_egress_rules(esw, vport);
 
-	if (!vport->vlan && !vport->qos) {
+	if (!vport->info.vlan && !vport->info.qos) {
 		esw_vport_disable_egress_acl(esw, vport);
 		return 0;
 	}
@@ -1395,7 +1303,7 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 
 	esw_debug(esw->dev,
 		  "vport[%d] configure egress rules, vlan(%d) qos(%d)\n",
-		  vport->vport, vport->vlan, vport->qos);
+		  vport->vport, vport->info.vlan, vport->info.qos);
 
 	spec = mlx5_vzalloc(sizeof(*spec));
 	if (!spec) {
@@ -1409,7 +1317,7 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.vlan_tag);
 	MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.vlan_tag);
 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid);
-	MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vport->vlan);
+	MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vport->info.vlan);
 
 	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 	vport->egress.allowed_vlan =
@@ -1418,8 +1326,9 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 				   0, NULL);
 	if (IS_ERR(vport->egress.allowed_vlan)) {
 		err = PTR_ERR(vport->egress.allowed_vlan);
-		pr_warn("vport[%d] configure egress allowed vlan rule failed, err(%d)\n",
-			vport->vport, err);
+		esw_warn(esw->dev,
+			 "vport[%d] configure egress allowed vlan rule failed, err(%d)\n",
+			 vport->vport, err);
 		vport->egress.allowed_vlan = NULL;
 		goto out;
 	}
@@ -1432,8 +1341,9 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 				   0, NULL);
 	if (IS_ERR(vport->egress.drop_rule)) {
 		err = PTR_ERR(vport->egress.drop_rule);
-		pr_warn("vport[%d] configure egress drop rule failed, err(%d)\n",
-			vport->vport, err);
+		esw_warn(esw->dev,
+			 "vport[%d] configure egress drop rule failed, err(%d)\n",
+			 vport->vport, err);
 		vport->egress.drop_rule = NULL;
 	}
 out:
@@ -1441,6 +1351,41 @@ out:
 	return err;
 }
 
+static void node_guid_gen_from_mac(u64 *node_guid, u8 mac[ETH_ALEN])
+{
+	((u8 *)node_guid)[7] = mac[0];
+	((u8 *)node_guid)[6] = mac[1];
+	((u8 *)node_guid)[5] = mac[2];
+	((u8 *)node_guid)[4] = 0xff;
+	((u8 *)node_guid)[3] = 0xfe;
+	((u8 *)node_guid)[2] = mac[3];
+	((u8 *)node_guid)[1] = mac[4];
+	((u8 *)node_guid)[0] = mac[5];
+}
+
+static void esw_apply_vport_conf(struct mlx5_eswitch *esw,
+				 struct mlx5_vport *vport)
+{
+	int vport_num = vport->vport;
+
+	if (!vport_num)
+		return;
+
+	mlx5_modify_vport_admin_state(esw->dev,
+				      MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
+				      vport_num,
+				      vport->info.link_state);
+	mlx5_modify_nic_vport_mac_address(esw->dev, vport_num, vport->info.mac);
+	mlx5_modify_nic_vport_node_guid(esw->dev, vport_num, vport->info.node_guid);
+	modify_esw_vport_cvlan(esw->dev, vport_num, vport->info.vlan, vport->info.qos,
+			       (vport->info.vlan || vport->info.qos));
+
+	/* Only legacy mode needs ACLs */
+	if (esw->mode == SRIOV_LEGACY) {
+		esw_vport_ingress_config(esw, vport);
+		esw_vport_egress_config(esw, vport);
+	}
+}
 static void esw_enable_vport(struct mlx5_eswitch *esw, int vport_num,
 			     int enable_events)
 {
@@ -1451,22 +1396,17 @@ static void esw_enable_vport(struct mlx5_eswitch *esw, int vport_num,
 
 	esw_debug(esw->dev, "Enabling VPORT(%d)\n", vport_num);
 
-	if (vport_num) { /* Only VFs need ACLs for VST and spoofchk filtering */
-		esw_vport_ingress_config(esw, vport);
-		esw_vport_egress_config(esw, vport);
-	}
-
-	mlx5_modify_vport_admin_state(esw->dev,
-				      MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
-				      vport_num,
-				      MLX5_ESW_VPORT_ADMIN_STATE_AUTO);
+	/* Restore old vport configuration */
+	esw_apply_vport_conf(esw, vport);
 
 	/* Sync with current vport context */
 	vport->enabled_events = enable_events;
 	vport->enabled = true;
 
 	/* only PF is trusted by default */
-	vport->trusted = (vport_num) ? false : true;
+	if (!vport_num)
+		vport->info.trusted = true;
+
 	esw_vport_change_handle_locked(vport);
 
 	esw->enabled_vports++;
@@ -1486,11 +1426,6 @@ static void esw_disable_vport(struct mlx5_eswitch *esw, int vport_num)
 	vport->enabled = false;
 
 	synchronize_irq(mlx5_get_msix_vec(esw->dev, MLX5_EQ_VEC_ASYNC));
-
-	mlx5_modify_vport_admin_state(esw->dev,
-				      MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
-				      vport_num,
-				      MLX5_ESW_VPORT_ADMIN_STATE_DOWN);
 	/* Wait for current already scheduled events to complete */
 	flush_workqueue(esw->work_queue);
 	/* Disable events from this vport */
@@ -1502,7 +1437,12 @@ static void esw_disable_vport(struct mlx5_eswitch *esw, int vport_num)
 	 */
 	esw_vport_change_handle_locked(vport);
 	vport->enabled_events = 0;
-	if (vport_num) {
+
+	if (vport_num && esw->mode == SRIOV_LEGACY) {
+		mlx5_modify_vport_admin_state(esw->dev,
+					      MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
+					      vport_num,
+					      MLX5_ESW_VPORT_ADMIN_STATE_DOWN);
 		esw_vport_disable_egress_acl(esw, vport);
 		esw_vport_disable_ingress_acl(esw, vport);
 	}
@@ -1553,6 +1493,7 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
 
 abort:
 	esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
+	esw->mode = SRIOV_NONE;
 	return err;
 }
 
@@ -1588,6 +1529,25 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw)
 	esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
 }
 
+void mlx5_eswitch_attach(struct mlx5_eswitch *esw)
+{
+	if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager) ||
+	    MLX5_CAP_GEN(esw->dev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
+		return;
+
+	esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
+	/* VF Vports will be enabled when SRIOV is enabled */
+}
+
+void mlx5_eswitch_detach(struct mlx5_eswitch *esw)
+{
+	if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager) ||
+	    MLX5_CAP_GEN(esw->dev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
+		return;
+
+	esw_disable_vport(esw, 0);
+}
+
 int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 {
 	int l2_table_size = 1 << MLX5_CAP_GEN(dev, log_max_l2_table);
@@ -1655,6 +1615,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 		struct mlx5_vport *vport = &esw->vports[vport_num];
 
 		vport->vport = vport_num;
+		vport->info.link_state = MLX5_ESW_VPORT_ADMIN_STATE_AUTO;
 		vport->dev = dev;
 		INIT_WORK(&vport->vport_change_handler,
 			  esw_vport_change_handler);
@@ -1665,8 +1626,6 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 	esw->mode = SRIOV_NONE;
 
 	dev->priv.eswitch = esw;
-	esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
-	/* VF Vports will be enabled when SRIOV is enabled */
 	return 0;
 abort:
 	if (esw->work_queue)
@@ -1685,7 +1644,6 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw)
 		return;
 
 	esw_info(esw->dev, "cleanup\n");
-	esw_disable_vport(esw, 0);
 
 	esw->dev->priv.eswitch = NULL;
 	destroy_workqueue(esw->work_queue);
@@ -1718,18 +1676,6 @@ void mlx5_eswitch_vport_event(struct mlx5_eswitch *esw, struct mlx5_eqe *eqe)
 	(esw && MLX5_CAP_GEN(esw->dev, vport_group_manager) && mlx5_core_is_pf(esw->dev))
 #define LEGAL_VPORT(esw, vport) (vport >= 0 && vport < esw->total_vports)
 
-static void node_guid_gen_from_mac(u64 *node_guid, u8 mac[ETH_ALEN])
-{
-	((u8 *)node_guid)[7] = mac[0];
-	((u8 *)node_guid)[6] = mac[1];
-	((u8 *)node_guid)[5] = mac[2];
-	((u8 *)node_guid)[4] = 0xff;
-	((u8 *)node_guid)[3] = 0xfe;
-	((u8 *)node_guid)[2] = mac[3];
-	((u8 *)node_guid)[1] = mac[4];
-	((u8 *)node_guid)[0] = mac[5];
-}
-
 int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw,
 			       int vport, u8 mac[ETH_ALEN])
 {
@@ -1742,13 +1688,15 @@ int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw,
 	if (!LEGAL_VPORT(esw, vport))
 		return -EINVAL;
 
+	mutex_lock(&esw->state_lock);
 	evport = &esw->vports[vport];
 
-	if (evport->spoofchk && !is_valid_ether_addr(mac)) {
+	if (evport->info.spoofchk && !is_valid_ether_addr(mac)) {
 		mlx5_core_warn(esw->dev,
 			       "MAC invalidation is not allowed when spoofchk is on, vport(%d)\n",
 			       vport);
-		return -EPERM;
+		err = -EPERM;
+		goto unlock;
 	}
 
 	err = mlx5_modify_nic_vport_mac_address(esw->dev, vport, mac);
@@ -1756,7 +1704,7 @@ int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw,
 		mlx5_core_warn(esw->dev,
 			       "Failed to mlx5_modify_nic_vport_mac vport(%d) err=(%d)\n",
 			       vport, err);
-		return err;
+		goto unlock;
 	}
 
 	node_guid_gen_from_mac(&node_guid, mac);
@@ -1766,9 +1714,12 @@ int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw,
 			       "Failed to set vport %d node guid, err = %d. RDMA_CM will not function properly for this VF.\n",
 			       vport, err);
 
-	mutex_lock(&esw->state_lock);
-	if (evport->enabled)
+	ether_addr_copy(evport->info.mac, mac);
+	evport->info.node_guid = node_guid;
+	if (evport->enabled && esw->mode == SRIOV_LEGACY)
 		err = esw_vport_ingress_config(esw, evport);
+
+unlock:
 	mutex_unlock(&esw->state_lock);
 	return err;
 }
@@ -1776,22 +1727,38 @@ int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw,
 int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw,
 				 int vport, int link_state)
 {
+	struct mlx5_vport *evport;
+	int err = 0;
+
 	if (!ESW_ALLOWED(esw))
 		return -EPERM;
 	if (!LEGAL_VPORT(esw, vport))
 		return -EINVAL;
 
-	return mlx5_modify_vport_admin_state(esw->dev,
-					     MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
-					     vport, link_state);
+	mutex_lock(&esw->state_lock);
+	evport = &esw->vports[vport];
+
+	err = mlx5_modify_vport_admin_state(esw->dev,
+					    MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
+					    vport, link_state);
+	if (err) {
+		mlx5_core_warn(esw->dev,
+			       "Failed to set vport %d link state, err = %d",
+			       vport, err);
+		goto unlock;
+	}
+
+	evport->info.link_state = link_state;
+
+unlock:
+	mutex_unlock(&esw->state_lock);
+	return 0;
 }
 
 int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw,
 				  int vport, struct ifla_vf_info *ivi)
 {
 	struct mlx5_vport *evport;
-	u16 vlan;
-	u8 qos;
 
 	if (!ESW_ALLOWED(esw))
 		return -EPERM;
@@ -1803,54 +1770,61 @@ int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw,
 	memset(ivi, 0, sizeof(*ivi));
 	ivi->vf = vport - 1;
 
-	mlx5_query_nic_vport_mac_address(esw->dev, vport, ivi->mac);
-	ivi->linkstate = mlx5_query_vport_admin_state(esw->dev,
-						      MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
-						      vport);
-	query_esw_vport_cvlan(esw->dev, vport, &vlan, &qos);
-	ivi->vlan = vlan;
-	ivi->qos = qos;
-	ivi->spoofchk = evport->spoofchk;
+	mutex_lock(&esw->state_lock);
+	ether_addr_copy(ivi->mac, evport->info.mac);
+	ivi->linkstate = evport->info.link_state;
+	ivi->vlan = evport->info.vlan;
+	ivi->qos = evport->info.qos;
+	ivi->spoofchk = evport->info.spoofchk;
+	ivi->trusted = evport->info.trusted;
+	mutex_unlock(&esw->state_lock);
 
 	return 0;
 }
 
-int mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw,
-				int vport, u16 vlan, u8 qos)
+int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw,
+				  int vport, u16 vlan, u8 qos, u8 set_flags)
 {
 	struct mlx5_vport *evport;
 	int err = 0;
-	int set = 0;
 
 	if (!ESW_ALLOWED(esw))
 		return -EPERM;
 	if (!LEGAL_VPORT(esw, vport) || (vlan > 4095) || (qos > 7))
 		return -EINVAL;
 
-	if (vlan || qos)
-		set = 1;
-
+	mutex_lock(&esw->state_lock);
 	evport = &esw->vports[vport];
 
-	err = modify_esw_vport_cvlan(esw->dev, vport, vlan, qos, set);
+	err = modify_esw_vport_cvlan(esw->dev, vport, vlan, qos, set_flags);
 	if (err)
-		return err;
+		goto unlock;
 
-	mutex_lock(&esw->state_lock);
-	evport->vlan = vlan;
-	evport->qos = qos;
-	if (evport->enabled) {
+	evport->info.vlan = vlan;
+	evport->info.qos = qos;
+	if (evport->enabled && esw->mode == SRIOV_LEGACY) {
 		err = esw_vport_ingress_config(esw, evport);
 		if (err)
-			goto out;
+			goto unlock;
 		err = esw_vport_egress_config(esw, evport);
 	}
 
-out:
+unlock:
 	mutex_unlock(&esw->state_lock);
 	return err;
 }
 
+int mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw,
+				int vport, u16 vlan, u8 qos)
+{
+	u8 set_flags = 0;
+
+	if (vlan || qos)
+		set_flags = SET_VLAN_STRIP | SET_VLAN_INSERT;
+
+	return __mlx5_eswitch_set_vport_vlan(esw, vport, vlan, qos, set_flags);
+}
+
 int mlx5_eswitch_set_vport_spoofchk(struct mlx5_eswitch *esw,
 				    int vport, bool spoofchk)
 {
@@ -1863,15 +1837,14 @@ int mlx5_eswitch_set_vport_spoofchk(struct mlx5_eswitch *esw,
 	if (!LEGAL_VPORT(esw, vport))
 		return -EINVAL;
 
-	evport = &esw->vports[vport];
-
 	mutex_lock(&esw->state_lock);
-	pschk = evport->spoofchk;
-	evport->spoofchk = spoofchk;
-	if (evport->enabled)
+	evport = &esw->vports[vport];
+	pschk = evport->info.spoofchk;
+	evport->info.spoofchk = spoofchk;
+	if (evport->enabled && esw->mode == SRIOV_LEGACY)
 		err = esw_vport_ingress_config(esw, evport);
 	if (err)
-		evport->spoofchk = pschk;
+		evport->info.spoofchk = pschk;
 	mutex_unlock(&esw->state_lock);
 
 	return err;
@@ -1887,10 +1860,9 @@ int mlx5_eswitch_set_vport_trust(struct mlx5_eswitch *esw,
 	if (!LEGAL_VPORT(esw, vport))
 		return -EINVAL;
 
-	evport = &esw->vports[vport];
-
 	mutex_lock(&esw->state_lock);
-	evport->trusted = setting;
+	evport = &esw->vports[vport];
+	evport->info.trusted = setting;
 	if (evport->enabled)
 		esw_vport_change_handle_locked(evport);
 	mutex_unlock(&esw->state_lock);
@@ -1903,7 +1875,7 @@ int mlx5_eswitch_get_vport_stats(struct mlx5_eswitch *esw,
 				 struct ifla_vf_stats *vf_stats)
 {
 	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
-	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
+	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
 	int err = 0;
 	u32 *out;
 
@@ -1916,8 +1888,6 @@ int mlx5_eswitch_get_vport_stats(struct mlx5_eswitch *esw,
 	if (!out)
 		return -ENOMEM;
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(query_vport_counter_in, in, opcode,
 		 MLX5_CMD_OP_QUERY_VPORT_COUNTER);
 	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index c0b05603fc31..2e2938e08cda 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -109,6 +109,16 @@ struct vport_egress {
 	struct mlx5_flow_rule  *drop_rule;
 };
 
+struct mlx5_vport_info {
+	u8                      mac[ETH_ALEN];
+	u16                     vlan;
+	u8                      qos;
+	u64                     node_guid;
+	int                     link_state;
+	bool                    spoofchk;
+	bool                    trusted;
+};
+
 struct mlx5_vport {
 	struct mlx5_core_dev    *dev;
 	int                     vport;
@@ -121,10 +131,8 @@ struct mlx5_vport {
 	struct vport_ingress    ingress;
 	struct vport_egress     egress;
 
-	u16                     vlan;
-	u8                      qos;
-	bool                    spoofchk;
-	bool                    trusted;
+	struct mlx5_vport_info  info;
+
 	bool                    enabled;
 	u16                     enabled_events;
 };
@@ -149,6 +157,7 @@ struct mlx5_eswitch_fdb {
 			struct mlx5_flow_group *send_to_vport_grp;
 			struct mlx5_flow_group *miss_grp;
 			struct mlx5_flow_rule  *miss_rule;
+			int vlan_push_pop_refcount;
 		} offloads;
 	};
 };
@@ -170,9 +179,13 @@ struct mlx5_eswitch_rep {
 	void		       (*unload)(struct mlx5_eswitch *esw,
 					 struct mlx5_eswitch_rep *rep);
 	u16		       vport;
-	struct mlx5_flow_rule *vport_rx_rule;
+	u8		       hw_id[ETH_ALEN];
 	void		      *priv_data;
+
+	struct mlx5_flow_rule *vport_rx_rule;
 	struct list_head       vport_sqs_list;
+	u16		       vlan;
+	u32		       vlan_refcount;
 	bool		       valid;
 };
 
@@ -200,9 +213,14 @@ struct mlx5_eswitch {
 	int                     mode;
 };
 
+void esw_offloads_cleanup(struct mlx5_eswitch *esw, int nvports);
+int esw_offloads_init(struct mlx5_eswitch *esw, int nvports);
+
 /* E-Switch API */
 int mlx5_eswitch_init(struct mlx5_core_dev *dev);
 void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw);
+void mlx5_eswitch_attach(struct mlx5_eswitch *esw);
+void mlx5_eswitch_detach(struct mlx5_eswitch *esw);
 void mlx5_eswitch_vport_event(struct mlx5_eswitch *esw, struct mlx5_eqe *eqe);
 int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode);
 void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw);
@@ -223,14 +241,32 @@ int mlx5_eswitch_get_vport_stats(struct mlx5_eswitch *esw,
 				 struct ifla_vf_stats *vf_stats);
 
 struct mlx5_flow_spec;
+struct mlx5_esw_flow_attr;
 
 struct mlx5_flow_rule *
 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 				struct mlx5_flow_spec *spec,
-				u32 action, u32 src_vport, u32 dst_vport);
+				struct mlx5_esw_flow_attr *attr);
 struct mlx5_flow_rule *
 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, int vport, u32 tirn);
 
+enum {
+	SET_VLAN_STRIP	= BIT(0),
+	SET_VLAN_INSERT	= BIT(1)
+};
+
+#define MLX5_FLOW_CONTEXT_ACTION_VLAN_POP  0x40
+#define MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH 0x80
+
+struct mlx5_esw_flow_attr {
+	struct mlx5_eswitch_rep *in_rep;
+	struct mlx5_eswitch_rep *out_rep;
+
+	int	action;
+	u16	vlan;
+	bool	vlan_handled;
+};
+
 int mlx5_eswitch_sqs2vport_start(struct mlx5_eswitch *esw,
 				 struct mlx5_eswitch_rep *rep,
 				 u16 *sqns_array, int sqns_num);
@@ -240,9 +276,17 @@ void mlx5_eswitch_sqs2vport_stop(struct mlx5_eswitch *esw,
 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode);
 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode);
 void mlx5_eswitch_register_vport_rep(struct mlx5_eswitch *esw,
+				     int vport_index,
 				     struct mlx5_eswitch_rep *rep);
 void mlx5_eswitch_unregister_vport_rep(struct mlx5_eswitch *esw,
-				       int vport);
+				       int vport_index);
+
+int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
+				 struct mlx5_esw_flow_attr *attr);
+int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw,
+				 struct mlx5_esw_flow_attr *attr);
+int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw,
+				  int vport, u16 vlan, u8 qos, u8 set_flags);
 
 #define MLX5_DEBUG_ESWITCH_MASK BIT(3)
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index a357e8eeeed8..c55ad8d00c05 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -46,19 +46,22 @@ enum {
 struct mlx5_flow_rule *
 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 				struct mlx5_flow_spec *spec,
-				u32 action, u32 src_vport, u32 dst_vport)
+				struct mlx5_esw_flow_attr *attr)
 {
 	struct mlx5_flow_destination dest = { 0 };
 	struct mlx5_fc *counter = NULL;
 	struct mlx5_flow_rule *rule;
 	void *misc;
+	int action;
 
 	if (esw->mode != SRIOV_OFFLOADS)
 		return ERR_PTR(-EOPNOTSUPP);
 
+	action = attr->action;
+
 	if (action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
 		dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
-		dest.vport_num = dst_vport;
+		dest.vport_num = attr->out_rep->vport;
 		action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 	} else if (action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
 		counter = mlx5_fc_create(esw->dev, true);
@@ -69,7 +72,7 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 	}
 
 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
-	MLX5_SET(fte_match_set_misc, misc, source_port, src_vport);
+	MLX5_SET(fte_match_set_misc, misc, source_port, attr->in_rep->vport);
 
 	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
 	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
@@ -86,6 +89,186 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
 	return rule;
 }
 
+static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val)
+{
+	struct mlx5_eswitch_rep *rep;
+	int vf_vport, err = 0;
+
+	esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none");
+	for (vf_vport = 1; vf_vport < esw->enabled_vports; vf_vport++) {
+		rep = &esw->offloads.vport_reps[vf_vport];
+		if (!rep->valid)
+			continue;
+
+		err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val);
+		if (err)
+			goto out;
+	}
+
+out:
+	return err;
+}
+
+static struct mlx5_eswitch_rep *
+esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop)
+{
+	struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL;
+
+	in_rep  = attr->in_rep;
+	out_rep = attr->out_rep;
+
+	if (push)
+		vport = in_rep;
+	else if (pop)
+		vport = out_rep;
+	else
+		vport = in_rep;
+
+	return vport;
+}
+
+static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr,
+				     bool push, bool pop, bool fwd)
+{
+	struct mlx5_eswitch_rep *in_rep, *out_rep;
+
+	if ((push || pop) && !fwd)
+		goto out_notsupp;
+
+	in_rep  = attr->in_rep;
+	out_rep = attr->out_rep;
+
+	if (push && in_rep->vport == FDB_UPLINK_VPORT)
+		goto out_notsupp;
+
+	if (pop && out_rep->vport == FDB_UPLINK_VPORT)
+		goto out_notsupp;
+
+	/* vport has vlan push configured, can't offload VF --> wire rules w.o it */
+	if (!push && !pop && fwd)
+		if (in_rep->vlan && out_rep->vport == FDB_UPLINK_VPORT)
+			goto out_notsupp;
+
+	/* protects against (1) setting rules with different vlans to push and
+	 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0)
+	 */
+	if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan))
+		goto out_notsupp;
+
+	return 0;
+
+out_notsupp:
+	return -ENOTSUPP;
+}
+
+int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
+				 struct mlx5_esw_flow_attr *attr)
+{
+	struct offloads_fdb *offloads = &esw->fdb_table.offloads;
+	struct mlx5_eswitch_rep *vport = NULL;
+	bool push, pop, fwd;
+	int err = 0;
+
+	push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
+	pop  = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
+	fwd  = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST);
+
+	err = esw_add_vlan_action_check(attr, push, pop, fwd);
+	if (err)
+		return err;
+
+	attr->vlan_handled = false;
+
+	vport = esw_vlan_action_get_vport(attr, push, pop);
+
+	if (!push && !pop && fwd) {
+		/* tracks VF --> wire rules without vlan push action */
+		if (attr->out_rep->vport == FDB_UPLINK_VPORT) {
+			vport->vlan_refcount++;
+			attr->vlan_handled = true;
+		}
+
+		return 0;
+	}
+
+	if (!push && !pop)
+		return 0;
+
+	if (!(offloads->vlan_push_pop_refcount)) {
+		/* it's the 1st vlan rule, apply global vlan pop policy */
+		err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP);
+		if (err)
+			goto out;
+	}
+	offloads->vlan_push_pop_refcount++;
+
+	if (push) {
+		if (vport->vlan_refcount)
+			goto skip_set_push;
+
+		err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, attr->vlan, 0,
+						    SET_VLAN_INSERT | SET_VLAN_STRIP);
+		if (err)
+			goto out;
+		vport->vlan = attr->vlan;
+skip_set_push:
+		vport->vlan_refcount++;
+	}
+out:
+	if (!err)
+		attr->vlan_handled = true;
+	return err;
+}
+
+int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw,
+				 struct mlx5_esw_flow_attr *attr)
+{
+	struct offloads_fdb *offloads = &esw->fdb_table.offloads;
+	struct mlx5_eswitch_rep *vport = NULL;
+	bool push, pop, fwd;
+	int err = 0;
+
+	if (!attr->vlan_handled)
+		return 0;
+
+	push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
+	pop  = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
+	fwd  = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST);
+
+	vport = esw_vlan_action_get_vport(attr, push, pop);
+
+	if (!push && !pop && fwd) {
+		/* tracks VF --> wire rules without vlan push action */
+		if (attr->out_rep->vport == FDB_UPLINK_VPORT)
+			vport->vlan_refcount--;
+
+		return 0;
+	}
+
+	if (push) {
+		vport->vlan_refcount--;
+		if (vport->vlan_refcount)
+			goto skip_unset_push;
+
+		vport->vlan = 0;
+		err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport,
+						    0, 0, SET_VLAN_STRIP);
+		if (err)
+			goto out;
+	}
+
+skip_unset_push:
+	offloads->vlan_push_pop_refcount--;
+	if (offloads->vlan_push_pop_refcount)
+		return 0;
+
+	/* no more vlan rules, stop global vlan pop policy */
+	err = esw_set_global_vlan_pop(esw, 0);
+
+out:
+	return err;
+}
+
 static struct mlx5_flow_rule *
 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, int vport, u32 sqn)
 {
@@ -113,7 +296,7 @@ mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, int vport, u32 sqn
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
 	dest.vport_num = vport;
 
-	flow_rule = mlx5_add_flow_rule(esw->fdb_table.fdb, spec,
+	flow_rule = mlx5_add_flow_rule(esw->fdb_table.offloads.fdb, spec,
 				       MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 				       0, &dest);
 	if (IS_ERR(flow_rule))
@@ -144,16 +327,12 @@ int mlx5_eswitch_sqs2vport_start(struct mlx5_eswitch *esw,
 {
 	struct mlx5_flow_rule *flow_rule;
 	struct mlx5_esw_sq *esw_sq;
-	int vport;
 	int err;
 	int i;
 
 	if (esw->mode != SRIOV_OFFLOADS)
 		return 0;
 
-	vport = rep->vport == 0 ?
-		FDB_UPLINK_VPORT : rep->vport;
-
 	for (i = 0; i < sqns_num; i++) {
 		esw_sq = kzalloc(sizeof(*esw_sq), GFP_KERNEL);
 		if (!esw_sq) {
@@ -163,7 +342,7 @@ int mlx5_eswitch_sqs2vport_start(struct mlx5_eswitch *esw,
 
 		/* Add re-inject rule to the PF/representor sqs */
 		flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw,
-								vport,
+								rep->vport,
 								sqns_array[i]);
 		if (IS_ERR(flow_rule)) {
 			err = PTR_ERR(flow_rule);
@@ -446,7 +625,7 @@ out:
 
 static int esw_offloads_start(struct mlx5_eswitch *esw)
 {
-	int err, num_vfs = esw->dev->priv.sriov.num_vfs;
+	int err, err1, num_vfs = esw->dev->priv.sriov.num_vfs;
 
 	if (esw->mode != SRIOV_LEGACY) {
 		esw_warn(esw->dev, "Can't set offloads mode, SRIOV legacy not enabled\n");
@@ -455,8 +634,12 @@ static int esw_offloads_start(struct mlx5_eswitch *esw)
 
 	mlx5_eswitch_disable_sriov(esw);
 	err = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_OFFLOADS);
-	if (err)
-		esw_warn(esw->dev, "Failed set eswitch to offloads, err %d\n", err);
+	if (err) {
+		esw_warn(esw->dev, "Failed setting eswitch to offloads, err %d\n", err);
+		err1 = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_LEGACY);
+		if (err1)
+			esw_warn(esw->dev, "Failed setting eswitch back to legacy, err %d\n", err);
+	}
 	return err;
 }
 
@@ -508,12 +691,16 @@ create_ft_err:
 
 static int esw_offloads_stop(struct mlx5_eswitch *esw)
 {
-	int err, num_vfs = esw->dev->priv.sriov.num_vfs;
+	int err, err1, num_vfs = esw->dev->priv.sriov.num_vfs;
 
 	mlx5_eswitch_disable_sriov(esw);
 	err = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_LEGACY);
-	if (err)
-		esw_warn(esw->dev, "Failed set eswitch legacy mode. err %d\n", err);
+	if (err) {
+		esw_warn(esw->dev, "Failed setting eswitch to legacy, err %d\n", err);
+		err1 = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_OFFLOADS);
+		if (err1)
+			esw_warn(esw->dev, "Failed setting eswitch back to offloads, err %d\n", err);
+	}
 
 	return err;
 }
@@ -535,7 +722,7 @@ void esw_offloads_cleanup(struct mlx5_eswitch *esw, int nvports)
 	esw_destroy_offloads_fdb_table(esw);
 }
 
-static int mlx5_esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
+static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
 {
 	switch (mode) {
 	case DEVLINK_ESWITCH_MODE_LEGACY:
@@ -551,6 +738,22 @@ static int mlx5_esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
 	return 0;
 }
 
+static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
+{
+	switch (mlx5_mode) {
+	case SRIOV_LEGACY:
+		*mode = DEVLINK_ESWITCH_MODE_LEGACY;
+		break;
+	case SRIOV_OFFLOADS:
+		*mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode)
 {
 	struct mlx5_core_dev *dev;
@@ -566,7 +769,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode)
 	if (cur_mlx5_mode == SRIOV_NONE)
 		return -EOPNOTSUPP;
 
-	if (mlx5_esw_mode_from_devlink(mode, &mlx5_mode))
+	if (esw_mode_from_devlink(mode, &mlx5_mode))
 		return -EINVAL;
 
 	if (cur_mlx5_mode == mlx5_mode)
@@ -592,33 +795,40 @@ int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
 	if (dev->priv.eswitch->mode == SRIOV_NONE)
 		return -EOPNOTSUPP;
 
-	*mode = dev->priv.eswitch->mode;
-
-	return 0;
+	return esw_mode_to_devlink(dev->priv.eswitch->mode, mode);
 }
 
 void mlx5_eswitch_register_vport_rep(struct mlx5_eswitch *esw,
-				     struct mlx5_eswitch_rep *rep)
+				     int vport_index,
+				     struct mlx5_eswitch_rep *__rep)
 {
 	struct mlx5_esw_offload *offloads = &esw->offloads;
+	struct mlx5_eswitch_rep *rep;
+
+	rep = &offloads->vport_reps[vport_index];
+
+	memset(rep, 0, sizeof(*rep));
 
-	memcpy(&offloads->vport_reps[rep->vport], rep,
-	       sizeof(struct mlx5_eswitch_rep));
+	rep->load   = __rep->load;
+	rep->unload = __rep->unload;
+	rep->vport  = __rep->vport;
+	rep->priv_data = __rep->priv_data;
+	ether_addr_copy(rep->hw_id, __rep->hw_id);
 
-	INIT_LIST_HEAD(&offloads->vport_reps[rep->vport].vport_sqs_list);
-	offloads->vport_reps[rep->vport].valid = true;
+	INIT_LIST_HEAD(&rep->vport_sqs_list);
+	rep->valid = true;
 }
 
 void mlx5_eswitch_unregister_vport_rep(struct mlx5_eswitch *esw,
-				       int vport)
+				       int vport_index)
 {
 	struct mlx5_esw_offload *offloads = &esw->offloads;
 	struct mlx5_eswitch_rep *rep;
 
-	rep = &offloads->vport_reps[vport];
+	rep = &offloads->vport_reps[vport_index];
 
-	if (esw->mode == SRIOV_OFFLOADS && esw->vports[vport].enabled)
+	if (esw->mode == SRIOV_OFFLOADS && esw->vports[vport_index].enabled)
 		rep->unload(esw, rep);
 
-	offloads->vport_reps[vport].valid = false;
+	rep->valid = false;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
index 9134010e2921..113c32326333 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
@@ -41,10 +41,8 @@
 int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev,
 			    struct mlx5_flow_table *ft)
 {
-	u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)];
-	u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {0};
 
 	MLX5_SET(set_flow_table_root_in, in, opcode,
 		 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
@@ -55,30 +53,23 @@ int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev,
 		MLX5_SET(set_flow_table_root_in, in, other_vport, 1);
 	}
 
-	memset(out, 0, sizeof(out));
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					  sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 int mlx5_cmd_create_flow_table(struct mlx5_core_dev *dev,
 			       u16 vport,
+			       enum fs_flow_table_op_mod op_mod,
 			       enum fs_flow_table_type type, unsigned int level,
 			       unsigned int log_size, struct mlx5_flow_table
 			       *next_ft, unsigned int *table_id)
 {
-	u32 out[MLX5_ST_SZ_DW(create_flow_table_out)];
-	u32 in[MLX5_ST_SZ_DW(create_flow_table_in)];
+	u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(create_flow_table_in)]   = {0};
 	int err;
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(create_flow_table_in, in, opcode,
 		 MLX5_CMD_OP_CREATE_FLOW_TABLE);
 
-	if (next_ft) {
-		MLX5_SET(create_flow_table_in, in, table_miss_mode, 1);
-		MLX5_SET(create_flow_table_in, in, table_miss_id, next_ft->id);
-	}
 	MLX5_SET(create_flow_table_in, in, table_type, type);
 	MLX5_SET(create_flow_table_in, in, level, level);
 	MLX5_SET(create_flow_table_in, in, log_size, log_size);
@@ -87,10 +78,23 @@ int mlx5_cmd_create_flow_table(struct mlx5_core_dev *dev,
 		MLX5_SET(create_flow_table_in, in, other_vport, 1);
 	}
 
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					 sizeof(out));
+	switch (op_mod) {
+	case FS_FT_OP_MOD_NORMAL:
+		if (next_ft) {
+			MLX5_SET(create_flow_table_in, in, table_miss_mode, 1);
+			MLX5_SET(create_flow_table_in, in, table_miss_id, next_ft->id);
+		}
+		break;
+
+	case FS_FT_OP_MOD_LAG_DEMUX:
+		MLX5_SET(create_flow_table_in, in, op_mod, 0x1);
+		if (next_ft)
+			MLX5_SET(create_flow_table_in, in, lag_master_next_table_id,
+				 next_ft->id);
+		break;
+	}
 
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 	if (!err)
 		*table_id = MLX5_GET(create_flow_table_out, out,
 				     table_id);
@@ -100,11 +104,8 @@ int mlx5_cmd_create_flow_table(struct mlx5_core_dev *dev,
 int mlx5_cmd_destroy_flow_table(struct mlx5_core_dev *dev,
 				struct mlx5_flow_table *ft)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)] = {0};
 
 	MLX5_SET(destroy_flow_table_in, in, opcode,
 		 MLX5_CMD_OP_DESTROY_FLOW_TABLE);
@@ -115,39 +116,49 @@ int mlx5_cmd_destroy_flow_table(struct mlx5_core_dev *dev,
 		MLX5_SET(destroy_flow_table_in, in, other_vport, 1);
 	}
 
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					  sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 int mlx5_cmd_modify_flow_table(struct mlx5_core_dev *dev,
 			       struct mlx5_flow_table *ft,
 			       struct mlx5_flow_table *next_ft)
 {
-	u32 in[MLX5_ST_SZ_DW(modify_flow_table_in)];
-	u32 out[MLX5_ST_SZ_DW(modify_flow_table_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(modify_flow_table_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(modify_flow_table_out)] = {0};
 
 	MLX5_SET(modify_flow_table_in, in, opcode,
 		 MLX5_CMD_OP_MODIFY_FLOW_TABLE);
 	MLX5_SET(modify_flow_table_in, in, table_type, ft->type);
 	MLX5_SET(modify_flow_table_in, in, table_id, ft->id);
-	if (ft->vport) {
-		MLX5_SET(modify_flow_table_in, in, vport_number, ft->vport);
-		MLX5_SET(modify_flow_table_in, in, other_vport, 1);
-	}
-	MLX5_SET(modify_flow_table_in, in, modify_field_select,
-		 MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID);
-	if (next_ft) {
-		MLX5_SET(modify_flow_table_in, in, table_miss_mode, 1);
-		MLX5_SET(modify_flow_table_in, in, table_miss_id, next_ft->id);
+
+	if (ft->op_mod == FS_FT_OP_MOD_LAG_DEMUX) {
+		MLX5_SET(modify_flow_table_in, in, modify_field_select,
+			 MLX5_MODIFY_FLOW_TABLE_LAG_NEXT_TABLE_ID);
+		if (next_ft) {
+			MLX5_SET(modify_flow_table_in, in,
+				 lag_master_next_table_id, next_ft->id);
+		} else {
+			MLX5_SET(modify_flow_table_in, in,
+				 lag_master_next_table_id, 0);
+		}
 	} else {
-		MLX5_SET(modify_flow_table_in, in, table_miss_mode, 0);
+		if (ft->vport) {
+			MLX5_SET(modify_flow_table_in, in, vport_number,
+				 ft->vport);
+			MLX5_SET(modify_flow_table_in, in, other_vport, 1);
+		}
+		MLX5_SET(modify_flow_table_in, in, modify_field_select,
+			 MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID);
+		if (next_ft) {
+			MLX5_SET(modify_flow_table_in, in, table_miss_mode, 1);
+			MLX5_SET(modify_flow_table_in, in, table_miss_id,
+				 next_ft->id);
+		} else {
+			MLX5_SET(modify_flow_table_in, in, table_miss_mode, 0);
+		}
 	}
 
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					  sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 int mlx5_cmd_create_flow_group(struct mlx5_core_dev *dev,
@@ -155,12 +166,10 @@ int mlx5_cmd_create_flow_group(struct mlx5_core_dev *dev,
 			       u32 *in,
 			       unsigned int *group_id)
 {
+	u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {0};
 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
-	u32 out[MLX5_ST_SZ_DW(create_flow_group_out)];
 	int err;
 
-	memset(out, 0, sizeof(out));
-
 	MLX5_SET(create_flow_group_in, in, opcode,
 		 MLX5_CMD_OP_CREATE_FLOW_GROUP);
 	MLX5_SET(create_flow_group_in, in, table_type, ft->type);
@@ -170,13 +179,10 @@ int mlx5_cmd_create_flow_group(struct mlx5_core_dev *dev,
 		MLX5_SET(create_flow_group_in, in, other_vport, 1);
 	}
 
-	err = mlx5_cmd_exec_check_status(dev, in,
-					 inlen, out,
-					 sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*group_id = MLX5_GET(create_flow_group_out, out,
 				     group_id);
-
 	return err;
 }
 
@@ -184,11 +190,8 @@ int mlx5_cmd_destroy_flow_group(struct mlx5_core_dev *dev,
 				struct mlx5_flow_table *ft,
 				unsigned int group_id)
 {
-	u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)];
-	u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)]   = {0};
 
 	MLX5_SET(destroy_flow_group_in, in, opcode,
 		 MLX5_CMD_OP_DESTROY_FLOW_GROUP);
@@ -200,8 +203,7 @@ int mlx5_cmd_destroy_flow_group(struct mlx5_core_dev *dev,
 		MLX5_SET(destroy_flow_group_in, in, other_vport, 1);
 	}
 
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					  sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
@@ -212,7 +214,7 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
 {
 	unsigned int inlen = MLX5_ST_SZ_BYTES(set_fte_in) +
 		fte->dests_size * MLX5_ST_SZ_BYTES(dest_format_struct);
-	u32 out[MLX5_ST_SZ_DW(set_fte_out)];
+	u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0};
 	struct mlx5_flow_rule *dst;
 	void *in_flow_context;
 	void *in_match_value;
@@ -290,11 +292,8 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
 			 list_size);
 	}
 
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(dev, in, inlen, out,
-					 sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	kvfree(in);
-
 	return err;
 }
 
@@ -303,7 +302,7 @@ int mlx5_cmd_create_fte(struct mlx5_core_dev *dev,
 			unsigned group_id,
 			struct fs_fte *fte)
 {
-	return	mlx5_cmd_set_fte(dev, 0, 0, ft, group_id, fte);
+	return mlx5_cmd_set_fte(dev, 0, 0, ft, group_id, fte);
 }
 
 int mlx5_cmd_update_fte(struct mlx5_core_dev *dev,
@@ -327,12 +326,8 @@ int mlx5_cmd_delete_fte(struct mlx5_core_dev *dev,
 			struct mlx5_flow_table *ft,
 			unsigned int index)
 {
-	u32 out[MLX5_ST_SZ_DW(delete_fte_out)];
-	u32 in[MLX5_ST_SZ_DW(delete_fte_in)];
-	int err;
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 out[MLX5_ST_SZ_DW(delete_fte_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(delete_fte_in)]   = {0};
 
 	MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
 	MLX5_SET(delete_fte_in, in, table_type, ft->type);
@@ -343,74 +338,55 @@ int mlx5_cmd_delete_fte(struct mlx5_core_dev *dev,
 		MLX5_SET(delete_fte_in, in, other_vport, 1);
 	}
 
-	err =  mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
-
-	return err;
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u16 *id)
 {
-	u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)];
-	u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)];
+	u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {0};
 	int err;
 
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
-
 	MLX5_SET(alloc_flow_counter_in, in, opcode,
 		 MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
 
-	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					 sizeof(out));
-	if (err)
-		return err;
-
-	*id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
-
-	return 0;
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (!err)
+		*id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
+	return err;
 }
 
 int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u16 id)
 {
-	u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)];
-	u32 out[MLX5_ST_SZ_DW(dealloc_flow_counter_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(dealloc_flow_counter_out)] = {0};
 
 	MLX5_SET(dealloc_flow_counter_in, in, opcode,
 		 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
 	MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id);
-
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					  sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u16 id,
 		      u64 *packets, u64 *bytes)
 {
 	u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
-		MLX5_ST_SZ_BYTES(traffic_counter)];
-	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)];
+		MLX5_ST_SZ_BYTES(traffic_counter)]   = {0};
+	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {0};
 	void *stats;
 	int err = 0;
 
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
-
 	MLX5_SET(query_flow_counter_in, in, opcode,
 		 MLX5_CMD_OP_QUERY_FLOW_COUNTER);
 	MLX5_SET(query_flow_counter_in, in, op_mod, 0);
 	MLX5_SET(query_flow_counter_in, in, flow_counter_id, id);
-
-	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 	if (err)
 		return err;
 
 	stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics);
 	*packets = MLX5_GET64(traffic_counter, stats, packets);
 	*bytes = MLX5_GET64(traffic_counter, stats, octets);
-
 	return 0;
 }
 
@@ -425,11 +401,11 @@ struct mlx5_cmd_fc_bulk *
 mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev, u16 id, int num)
 {
 	struct mlx5_cmd_fc_bulk *b;
-	int outlen = sizeof(*b) +
+	int outlen =
 		MLX5_ST_SZ_BYTES(query_flow_counter_out) +
 		MLX5_ST_SZ_BYTES(traffic_counter) * num;
 
-	b = kzalloc(outlen, GFP_KERNEL);
+	b = kzalloc(sizeof(*b) + outlen, GFP_KERNEL);
 	if (!b)
 		return NULL;
 
@@ -448,18 +424,14 @@ void mlx5_cmd_fc_bulk_free(struct mlx5_cmd_fc_bulk *b)
 int
 mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, struct mlx5_cmd_fc_bulk *b)
 {
-	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {0};
 
 	MLX5_SET(query_flow_counter_in, in, opcode,
 		 MLX5_CMD_OP_QUERY_FLOW_COUNTER);
 	MLX5_SET(query_flow_counter_in, in, op_mod, 0);
 	MLX5_SET(query_flow_counter_in, in, flow_counter_id, b->id);
 	MLX5_SET(query_flow_counter_in, in, num_of_counters, b->num);
-
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in),
-					  b->out, b->outlen);
+	return mlx5_cmd_exec(dev, in, sizeof(in), b->out, b->outlen);
 }
 
 void mlx5_cmd_fc_bulk_get(struct mlx5_core_dev *dev,
@@ -480,3 +452,51 @@ void mlx5_cmd_fc_bulk_get(struct mlx5_core_dev *dev,
 	*packets = MLX5_GET64(traffic_counter, stats, packets);
 	*bytes = MLX5_GET64(traffic_counter, stats, octets);
 }
+
+#define MAX_ENCAP_SIZE (128)
+
+int mlx5_cmd_alloc_encap(struct mlx5_core_dev *dev,
+			 int header_type,
+			 size_t size,
+			 void *encap_header,
+			 u32 *encap_id)
+{
+	u32 out[MLX5_ST_SZ_DW(alloc_encap_header_out)];
+	u32 in[MLX5_ST_SZ_DW(alloc_encap_header_in) +
+	      (MAX_ENCAP_SIZE / sizeof(u32))];
+	void *encap_header_in = MLX5_ADDR_OF(alloc_encap_header_in, in,
+					     encap_header);
+	void *header = MLX5_ADDR_OF(encap_header_in, encap_header_in,
+				    encap_header);
+	int inlen = header - (void *)in  + size;
+	int err;
+
+	if (size > MAX_ENCAP_SIZE)
+		return -EINVAL;
+
+	memset(in, 0, inlen);
+	MLX5_SET(alloc_encap_header_in, in, opcode,
+		 MLX5_CMD_OP_ALLOC_ENCAP_HEADER);
+	MLX5_SET(encap_header_in, encap_header_in, encap_header_size, size);
+	MLX5_SET(encap_header_in, encap_header_in, header_type, header_type);
+	memcpy(header, encap_header, size);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+
+	*encap_id = MLX5_GET(alloc_encap_header_out, out, encap_id);
+	return err;
+}
+
+void mlx5_cmd_dealloc_encap(struct mlx5_core_dev *dev, u32 encap_id)
+{
+	u32 in[MLX5_ST_SZ_DW(dealloc_encap_header_in)];
+	u32 out[MLX5_ST_SZ_DW(dealloc_encap_header_out)];
+
+	memset(in, 0, sizeof(in));
+	MLX5_SET(dealloc_encap_header_in, in, opcode,
+		 MLX5_CMD_OP_DEALLOC_ENCAP_HEADER);
+	MLX5_SET(dealloc_encap_header_in, in, encap_id, encap_id);
+
+	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h
index 158844cef82b..c5bc4686c832 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h
@@ -35,6 +35,7 @@
 
 int mlx5_cmd_create_flow_table(struct mlx5_core_dev *dev,
 			       u16 vport,
+			       enum fs_flow_table_op_mod op_mod,
 			       enum fs_flow_table_type type, unsigned int level,
 			       unsigned int log_size, struct mlx5_flow_table
 			       *next_ft, unsigned int *table_id);
@@ -88,4 +89,11 @@ void mlx5_cmd_fc_bulk_get(struct mlx5_core_dev *dev,
 			  struct mlx5_cmd_fc_bulk *b, u16 id,
 			  u64 *packets, u64 *bytes);
 
+int mlx5_cmd_alloc_encap(struct mlx5_core_dev *dev,
+			 int header_type,
+			 size_t size,
+			 void *encap_header,
+			 u32 *encap_id);
+void mlx5_cmd_dealloc_encap(struct mlx5_core_dev *dev, u32 encap_id);
+
 #endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index 75bb8c864557..5da2cc878582 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -80,7 +80,7 @@
 			   LEFTOVERS_NUM_PRIOS)
 
 #define ETHTOOL_PRIO_NUM_LEVELS 1
-#define ETHTOOL_NUM_PRIOS 10
+#define ETHTOOL_NUM_PRIOS 11
 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
 /* Vlan, mac, ttc, aRFS */
 #define KERNEL_NIC_PRIO_NUM_LEVELS 4
@@ -96,6 +96,10 @@
 #define OFFLOADS_NUM_PRIOS 1
 #define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + 1)
 
+#define LAG_PRIO_NUM_LEVELS 1
+#define LAG_NUM_PRIOS 1
+#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1)
+
 struct node_caps {
 	size_t	arr_sz;
 	long	*caps;
@@ -111,12 +115,16 @@ static struct init_tree_node {
 	int num_levels;
 } root_fs = {
 	.type = FS_TYPE_NAMESPACE,
-	.ar_size = 6,
+	.ar_size = 7,
 	.children = (struct init_tree_node[]) {
 		ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0,
 			 FS_CHAINING_CAPS,
 			 ADD_NS(ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
 						  BY_PASS_PRIO_NUM_LEVELS))),
+		ADD_PRIO(0, LAG_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS,
+			 ADD_NS(ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS,
+						  LAG_PRIO_NUM_LEVELS))),
 		ADD_PRIO(0, OFFLOADS_MIN_LEVEL, 0, {},
 			 ADD_NS(ADD_MULTIPLE_PRIO(OFFLOADS_NUM_PRIOS, OFFLOADS_MAX_FT))),
 		ADD_PRIO(0, ETHTOOL_MIN_LEVEL, 0,
@@ -345,7 +353,7 @@ static void del_flow_table(struct fs_node *node)
 
 	err = mlx5_cmd_destroy_flow_table(dev, ft);
 	if (err)
-		pr_warn("flow steering can't destroy ft\n");
+		mlx5_core_warn(dev, "flow steering can't destroy ft\n");
 	fs_get_obj(prio, ft->node.parent);
 	prio->num_ft--;
 }
@@ -364,7 +372,7 @@ static void del_rule(struct fs_node *node)
 
 	match_value = mlx5_vzalloc(match_len);
 	if (!match_value) {
-		pr_warn("failed to allocate inbox\n");
+		mlx5_core_warn(dev, "failed to allocate inbox\n");
 		return;
 	}
 
@@ -387,8 +395,9 @@ static void del_rule(struct fs_node *node)
 					  modify_mask,
 					  fte);
 		if (err)
-			pr_warn("%s can't del rule fg id=%d fte_index=%d\n",
-				__func__, fg->id, fte->index);
+			mlx5_core_warn(dev,
+				       "%s can't del rule fg id=%d fte_index=%d\n",
+				       __func__, fg->id, fte->index);
 	}
 	kvfree(match_value);
 }
@@ -409,8 +418,9 @@ static void del_fte(struct fs_node *node)
 	err = mlx5_cmd_delete_fte(dev, ft,
 				  fte->index);
 	if (err)
-		pr_warn("flow steering can't delete fte in index %d of flow group id %d\n",
-			fte->index, fg->id);
+		mlx5_core_warn(dev,
+			       "flow steering can't delete fte in index %d of flow group id %d\n",
+			       fte->index, fg->id);
 
 	fte->status = 0;
 	fg->num_ftes--;
@@ -427,8 +437,8 @@ static void del_flow_group(struct fs_node *node)
 	dev = get_dev(&ft->node);
 
 	if (mlx5_cmd_destroy_flow_group(dev, ft, fg->id))
-		pr_warn("flow steering can't destroy fg %d of ft %d\n",
-			fg->id, ft->id);
+		mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n",
+			       fg->id, ft->id);
 }
 
 static struct fs_fte *alloc_fte(u8 action,
@@ -475,7 +485,8 @@ static struct mlx5_flow_group *alloc_flow_group(u32 *create_fg_in)
 }
 
 static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, int max_fte,
-						enum fs_flow_table_type table_type)
+						enum fs_flow_table_type table_type,
+						enum fs_flow_table_op_mod op_mod)
 {
 	struct mlx5_flow_table *ft;
 
@@ -485,6 +496,7 @@ static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, int max_ft
 
 	ft->level = level;
 	ft->node.type = FS_TYPE_FLOW_TABLE;
+	ft->op_mod = op_mod;
 	ft->type = table_type;
 	ft->vport = vport;
 	ft->max_fte = max_fte;
@@ -722,6 +734,7 @@ static void list_add_flow_table(struct mlx5_flow_table *ft,
 }
 
 static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
+							enum fs_flow_table_op_mod op_mod,
 							u16 vport, int prio,
 							int max_fte, u32 level)
 {
@@ -754,18 +767,19 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa
 	level += fs_prio->start_level;
 	ft = alloc_flow_table(level,
 			      vport,
-			      roundup_pow_of_two(max_fte),
-			      root->table_type);
+			      max_fte ? roundup_pow_of_two(max_fte) : 0,
+			      root->table_type,
+			      op_mod);
 	if (!ft) {
 		err = -ENOMEM;
 		goto unlock_root;
 	}
 
 	tree_init_node(&ft->node, 1, del_flow_table);
-	log_table_sz = ilog2(ft->max_fte);
+	log_table_sz = ft->max_fte ? ilog2(ft->max_fte) : 0;
 	next_ft = find_next_chained_ft(fs_prio);
-	err = mlx5_cmd_create_flow_table(root->dev, ft->vport, ft->type, ft->level,
-					 log_table_sz, next_ft, &ft->id);
+	err = mlx5_cmd_create_flow_table(root->dev, ft->vport, ft->op_mod, ft->type,
+					 ft->level, log_table_sz, next_ft, &ft->id);
 	if (err)
 		goto free_ft;
 
@@ -792,15 +806,26 @@ struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
 					       int prio, int max_fte,
 					       u32 level)
 {
-	return __mlx5_create_flow_table(ns, 0, prio, max_fte, level);
+	return __mlx5_create_flow_table(ns, FS_FT_OP_MOD_NORMAL, 0, prio,
+					max_fte, level);
 }
 
 struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
 						     int prio, int max_fte,
 						     u32 level, u16 vport)
 {
-	return __mlx5_create_flow_table(ns, vport, prio, max_fte, level);
+	return __mlx5_create_flow_table(ns, FS_FT_OP_MOD_NORMAL, vport, prio,
+					max_fte, level);
+}
+
+struct mlx5_flow_table *mlx5_create_lag_demux_flow_table(
+					       struct mlx5_flow_namespace *ns,
+					       int prio, u32 level)
+{
+	return __mlx5_create_flow_table(ns, FS_FT_OP_MOD_LAG_DEMUX, 0, prio, 0,
+					level);
 }
+EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table);
 
 struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
 							    int prio,
@@ -1379,6 +1404,7 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
 
 	switch (type) {
 	case MLX5_FLOW_NAMESPACE_BYPASS:
+	case MLX5_FLOW_NAMESPACE_LAG:
 	case MLX5_FLOW_NAMESPACE_OFFLOADS:
 	case MLX5_FLOW_NAMESPACE_ETHTOOL:
 	case MLX5_FLOW_NAMESPACE_KERNEL:
@@ -1401,6 +1427,16 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
 			return &steering->esw_ingress_root_ns->ns;
 		else
 			return NULL;
+	case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
+		if (steering->sniffer_rx_root_ns)
+			return &steering->sniffer_rx_root_ns->ns;
+		else
+			return NULL;
+	case MLX5_FLOW_NAMESPACE_SNIFFER_TX:
+		if (steering->sniffer_tx_root_ns)
+			return &steering->sniffer_tx_root_ns->ns;
+		else
+			return NULL;
 	default:
 		return NULL;
 	}
@@ -1700,10 +1736,46 @@ void mlx5_cleanup_fs(struct mlx5_core_dev *dev)
 	cleanup_root_ns(steering->esw_egress_root_ns);
 	cleanup_root_ns(steering->esw_ingress_root_ns);
 	cleanup_root_ns(steering->fdb_root_ns);
+	cleanup_root_ns(steering->sniffer_rx_root_ns);
+	cleanup_root_ns(steering->sniffer_tx_root_ns);
 	mlx5_cleanup_fc_stats(dev);
 	kfree(steering);
 }
 
+static int init_sniffer_tx_root_ns(struct mlx5_flow_steering *steering)
+{
+	struct fs_prio *prio;
+
+	steering->sniffer_tx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_TX);
+	if (!steering->sniffer_tx_root_ns)
+		return -ENOMEM;
+
+	/* Create single prio */
+	prio = fs_create_prio(&steering->sniffer_tx_root_ns->ns, 0, 1);
+	if (IS_ERR(prio)) {
+		cleanup_root_ns(steering->sniffer_tx_root_ns);
+		return PTR_ERR(prio);
+	}
+	return 0;
+}
+
+static int init_sniffer_rx_root_ns(struct mlx5_flow_steering *steering)
+{
+	struct fs_prio *prio;
+
+	steering->sniffer_rx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_RX);
+	if (!steering->sniffer_rx_root_ns)
+		return -ENOMEM;
+
+	/* Create single prio */
+	prio = fs_create_prio(&steering->sniffer_rx_root_ns->ns, 0, 1);
+	if (IS_ERR(prio)) {
+		cleanup_root_ns(steering->sniffer_rx_root_ns);
+		return PTR_ERR(prio);
+	}
+	return 0;
+}
+
 static int init_fdb_root_ns(struct mlx5_flow_steering *steering)
 {
 	struct fs_prio *prio;
@@ -1800,6 +1872,18 @@ int mlx5_init_fs(struct mlx5_core_dev *dev)
 		}
 	}
 
+	if (MLX5_CAP_FLOWTABLE_SNIFFER_RX(dev, ft_support)) {
+		err = init_sniffer_rx_root_ns(steering);
+		if (err)
+			goto err;
+	}
+
+	if (MLX5_CAP_FLOWTABLE_SNIFFER_TX(dev, ft_support)) {
+		err = init_sniffer_tx_root_ns(steering);
+		if (err)
+			goto err;
+	}
+
 	return 0;
 err:
 	mlx5_cleanup_fs(dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
index 9cffb6aeb4e9..71ff03bceabb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
@@ -49,6 +49,13 @@ enum fs_flow_table_type {
 	FS_FT_ESW_EGRESS_ACL  = 0x2,
 	FS_FT_ESW_INGRESS_ACL = 0x3,
 	FS_FT_FDB             = 0X4,
+	FS_FT_SNIFFER_RX	= 0X5,
+	FS_FT_SNIFFER_TX	= 0X6,
+};
+
+enum fs_flow_table_op_mod {
+	FS_FT_OP_MOD_NORMAL,
+	FS_FT_OP_MOD_LAG_DEMUX,
 };
 
 enum fs_fte_status {
@@ -61,6 +68,8 @@ struct mlx5_flow_steering {
 	struct mlx5_flow_root_namespace *fdb_root_ns;
 	struct mlx5_flow_root_namespace *esw_egress_root_ns;
 	struct mlx5_flow_root_namespace *esw_ingress_root_ns;
+	struct mlx5_flow_root_namespace	*sniffer_tx_root_ns;
+	struct mlx5_flow_root_namespace	*sniffer_rx_root_ns;
 };
 
 struct fs_node {
@@ -93,6 +102,7 @@ struct mlx5_flow_table {
 	unsigned int			max_fte;
 	unsigned int			level;
 	enum fs_flow_table_type		type;
+	enum fs_flow_table_op_mod	op_mod;
 	struct {
 		bool			active;
 		unsigned int		required_groups;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
index c2877e9de8a1..3a9195b4169d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
@@ -126,12 +126,21 @@ static struct rb_node *mlx5_fc_stats_query(struct mlx5_core_dev *dev,
 	for (node = &first->node; node; node = rb_next(node)) {
 		struct mlx5_fc *counter = rb_entry(node, struct mlx5_fc, node);
 		struct mlx5_fc_cache *c = &counter->cache;
+		u64 packets;
+		u64 bytes;
 
 		if (counter->id > last_id)
 			break;
 
 		mlx5_cmd_fc_bulk_get(dev, b,
-				     counter->id, &c->packets, &c->bytes);
+				     counter->id, &packets, &bytes);
+
+		if (c->packets == packets)
+			continue;
+
+		c->packets = packets;
+		c->bytes = bytes;
+		c->lastuse = jiffies;
 	}
 
 out:
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
index 77fc1aa26114..5718aada6605 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
@@ -38,13 +38,10 @@
 static int mlx5_cmd_query_adapter(struct mlx5_core_dev *dev, u32 *out,
 				  int outlen)
 {
-	u32 in[MLX5_ST_SZ_DW(query_adapter_in)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(query_adapter_in)] = {0};
 
 	MLX5_SET(query_adapter_in, in, opcode, MLX5_CMD_OP_QUERY_ADAPTER);
-
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, outlen);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 }
 
 int mlx5_query_board_id(struct mlx5_core_dev *dev)
@@ -162,38 +159,18 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
 
 int mlx5_cmd_init_hca(struct mlx5_core_dev *dev)
 {
-	struct mlx5_cmd_init_hca_mbox_in in;
-	struct mlx5_cmd_init_hca_mbox_out out;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_INIT_HCA);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		err = mlx5_cmd_status_to_err(&out.hdr);
+	u32 out[MLX5_ST_SZ_DW(init_hca_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(init_hca_in)]   = {0};
 
-	return err;
+	MLX5_SET(init_hca_in, in, opcode, MLX5_CMD_OP_INIT_HCA);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev)
 {
-	struct mlx5_cmd_teardown_hca_mbox_in in;
-	struct mlx5_cmd_teardown_hca_mbox_out out;
-	int err;
+	u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(teardown_hca_in)]   = {0};
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_TEARDOWN_HCA);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		err = mlx5_cmd_status_to_err(&out.hdr);
-
-	return err;
+	MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag.c
new file mode 100644
index 000000000000..55957246c0e8
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag.c
@@ -0,0 +1,588 @@
+/*
+ * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/netdevice.h>
+#include <linux/mlx5/driver.h>
+#include <linux/mlx5/vport.h>
+#include "mlx5_core.h"
+
+enum {
+	MLX5_LAG_FLAG_BONDED = 1 << 0,
+};
+
+struct lag_func {
+	struct mlx5_core_dev *dev;
+	struct net_device    *netdev;
+};
+
+/* Used for collection of netdev event info. */
+struct lag_tracker {
+	enum   netdev_lag_tx_type           tx_type;
+	struct netdev_lag_lower_state_info  netdev_state[MLX5_MAX_PORTS];
+	bool is_bonded;
+};
+
+/* LAG data of a ConnectX card.
+ * It serves both its phys functions.
+ */
+struct mlx5_lag {
+	u8                        flags;
+	u8                        v2p_map[MLX5_MAX_PORTS];
+	struct lag_func           pf[MLX5_MAX_PORTS];
+	struct lag_tracker        tracker;
+	struct delayed_work       bond_work;
+	struct notifier_block     nb;
+};
+
+/* General purpose, use for short periods of time.
+ * Beware of lock dependencies (preferably, no locks should be acquired
+ * under it).
+ */
+static DEFINE_MUTEX(lag_mutex);
+
+static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, u8 remap_port1,
+			       u8 remap_port2)
+{
+	u32   in[MLX5_ST_SZ_DW(create_lag_in)]   = {0};
+	u32   out[MLX5_ST_SZ_DW(create_lag_out)] = {0};
+	void *lag_ctx = MLX5_ADDR_OF(create_lag_in, in, ctx);
+
+	MLX5_SET(create_lag_in, in, opcode, MLX5_CMD_OP_CREATE_LAG);
+
+	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, remap_port1);
+	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, remap_port2);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+
+static int mlx5_cmd_modify_lag(struct mlx5_core_dev *dev, u8 remap_port1,
+			       u8 remap_port2)
+{
+	u32   in[MLX5_ST_SZ_DW(modify_lag_in)]   = {0};
+	u32   out[MLX5_ST_SZ_DW(modify_lag_out)] = {0};
+	void *lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx);
+
+	MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG);
+	MLX5_SET(modify_lag_in, in, field_select, 0x1);
+
+	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, remap_port1);
+	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, remap_port2);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+
+static int mlx5_cmd_destroy_lag(struct mlx5_core_dev *dev)
+{
+	u32  in[MLX5_ST_SZ_DW(destroy_lag_in)]  = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_lag_out)] = {0};
+
+	MLX5_SET(destroy_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_LAG);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+
+int mlx5_cmd_create_vport_lag(struct mlx5_core_dev *dev)
+{
+	u32  in[MLX5_ST_SZ_DW(create_vport_lag_in)]  = {0};
+	u32 out[MLX5_ST_SZ_DW(create_vport_lag_out)] = {0};
+
+	MLX5_SET(create_vport_lag_in, in, opcode, MLX5_CMD_OP_CREATE_VPORT_LAG);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+EXPORT_SYMBOL(mlx5_cmd_create_vport_lag);
+
+int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev)
+{
+	u32  in[MLX5_ST_SZ_DW(destroy_vport_lag_in)]  = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_vport_lag_out)] = {0};
+
+	MLX5_SET(destroy_vport_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_VPORT_LAG);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+EXPORT_SYMBOL(mlx5_cmd_destroy_vport_lag);
+
+static struct mlx5_lag *mlx5_lag_dev_get(struct mlx5_core_dev *dev)
+{
+	return dev->priv.lag;
+}
+
+static int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev,
+				       struct net_device *ndev)
+{
+	int i;
+
+	for (i = 0; i < MLX5_MAX_PORTS; i++)
+		if (ldev->pf[i].netdev == ndev)
+			return i;
+
+	return -1;
+}
+
+static bool mlx5_lag_is_bonded(struct mlx5_lag *ldev)
+{
+	return !!(ldev->flags & MLX5_LAG_FLAG_BONDED);
+}
+
+static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
+					   u8 *port1, u8 *port2)
+{
+	if (tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
+		if (tracker->netdev_state[0].tx_enabled) {
+			*port1 = 1;
+			*port2 = 1;
+		} else {
+			*port1 = 2;
+			*port2 = 2;
+		}
+	} else {
+		*port1 = 1;
+		*port2 = 2;
+		if (!tracker->netdev_state[0].link_up)
+			*port1 = 2;
+		else if (!tracker->netdev_state[1].link_up)
+			*port2 = 1;
+	}
+}
+
+static void mlx5_activate_lag(struct mlx5_lag *ldev,
+			      struct lag_tracker *tracker)
+{
+	struct mlx5_core_dev *dev0 = ldev->pf[0].dev;
+	int err;
+
+	ldev->flags |= MLX5_LAG_FLAG_BONDED;
+
+	mlx5_infer_tx_affinity_mapping(tracker, &ldev->v2p_map[0],
+				       &ldev->v2p_map[1]);
+
+	err = mlx5_cmd_create_lag(dev0, ldev->v2p_map[0], ldev->v2p_map[1]);
+	if (err)
+		mlx5_core_err(dev0,
+			      "Failed to create LAG (%d)\n",
+			      err);
+}
+
+static void mlx5_deactivate_lag(struct mlx5_lag *ldev)
+{
+	struct mlx5_core_dev *dev0 = ldev->pf[0].dev;
+	int err;
+
+	ldev->flags &= ~MLX5_LAG_FLAG_BONDED;
+
+	err = mlx5_cmd_destroy_lag(dev0);
+	if (err)
+		mlx5_core_err(dev0,
+			      "Failed to destroy LAG (%d)\n",
+			      err);
+}
+
+static void mlx5_do_bond(struct mlx5_lag *ldev)
+{
+	struct mlx5_core_dev *dev0 = ldev->pf[0].dev;
+	struct mlx5_core_dev *dev1 = ldev->pf[1].dev;
+	struct lag_tracker tracker;
+	u8 v2p_port1, v2p_port2;
+	int i, err;
+
+	if (!dev0 || !dev1)
+		return;
+
+	mutex_lock(&lag_mutex);
+	tracker = ldev->tracker;
+	mutex_unlock(&lag_mutex);
+
+	if (tracker.is_bonded && !mlx5_lag_is_bonded(ldev)) {
+		if (mlx5_sriov_is_enabled(dev0) ||
+		    mlx5_sriov_is_enabled(dev1)) {
+			mlx5_core_warn(dev0, "LAG is not supported with SRIOV");
+			return;
+		}
+
+		for (i = 0; i < MLX5_MAX_PORTS; i++)
+			mlx5_remove_dev_by_protocol(ldev->pf[i].dev,
+						    MLX5_INTERFACE_PROTOCOL_IB);
+
+		mlx5_activate_lag(ldev, &tracker);
+
+		mlx5_add_dev_by_protocol(dev0, MLX5_INTERFACE_PROTOCOL_IB);
+		mlx5_nic_vport_enable_roce(dev1);
+	} else if (tracker.is_bonded && mlx5_lag_is_bonded(ldev)) {
+		mlx5_infer_tx_affinity_mapping(&tracker, &v2p_port1,
+					       &v2p_port2);
+
+		if ((v2p_port1 != ldev->v2p_map[0]) ||
+		    (v2p_port2 != ldev->v2p_map[1])) {
+			ldev->v2p_map[0] = v2p_port1;
+			ldev->v2p_map[1] = v2p_port2;
+
+			err = mlx5_cmd_modify_lag(dev0, v2p_port1, v2p_port2);
+			if (err)
+				mlx5_core_err(dev0,
+					      "Failed to modify LAG (%d)\n",
+					      err);
+		}
+	} else if (!tracker.is_bonded && mlx5_lag_is_bonded(ldev)) {
+		mlx5_remove_dev_by_protocol(dev0, MLX5_INTERFACE_PROTOCOL_IB);
+		mlx5_nic_vport_disable_roce(dev1);
+
+		mlx5_deactivate_lag(ldev);
+
+		for (i = 0; i < MLX5_MAX_PORTS; i++)
+			if (ldev->pf[i].dev)
+				mlx5_add_dev_by_protocol(ldev->pf[i].dev,
+							 MLX5_INTERFACE_PROTOCOL_IB);
+	}
+}
+
+static void mlx5_queue_bond_work(struct mlx5_lag *ldev, unsigned long delay)
+{
+	schedule_delayed_work(&ldev->bond_work, delay);
+}
+
+static void mlx5_do_bond_work(struct work_struct *work)
+{
+	struct delayed_work *delayed_work = to_delayed_work(work);
+	struct mlx5_lag *ldev = container_of(delayed_work, struct mlx5_lag,
+					     bond_work);
+	int status;
+
+	status = mlx5_dev_list_trylock();
+	if (!status) {
+		/* 1 sec delay. */
+		mlx5_queue_bond_work(ldev, HZ);
+		return;
+	}
+
+	mlx5_do_bond(ldev);
+	mlx5_dev_list_unlock();
+}
+
+static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
+					 struct lag_tracker *tracker,
+					 struct net_device *ndev,
+					 struct netdev_notifier_changeupper_info *info)
+{
+	struct net_device *upper = info->upper_dev, *ndev_tmp;
+	struct netdev_lag_upper_info *lag_upper_info;
+	bool is_bonded;
+	int bond_status = 0;
+	int num_slaves = 0;
+	int idx;
+
+	if (!netif_is_lag_master(upper))
+		return 0;
+
+	lag_upper_info = info->upper_info;
+
+	/* The event may still be of interest if the slave does not belong to
+	 * us, but is enslaved to a master which has one or more of our netdevs
+	 * as slaves (e.g., if a new slave is added to a master that bonds two
+	 * of our netdevs, we should unbond).
+	 */
+	rcu_read_lock();
+	for_each_netdev_in_bond_rcu(upper, ndev_tmp) {
+		idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev_tmp);
+		if (idx > -1)
+			bond_status |= (1 << idx);
+
+		num_slaves++;
+	}
+	rcu_read_unlock();
+
+	/* None of this lagdev's netdevs are slaves of this master. */
+	if (!(bond_status & 0x3))
+		return 0;
+
+	if (lag_upper_info)
+		tracker->tx_type = lag_upper_info->tx_type;
+
+	/* Determine bonding status:
+	 * A device is considered bonded if both its physical ports are slaves
+	 * of the same lag master, and only them.
+	 * Lag mode must be activebackup or hash.
+	 */
+	is_bonded = (num_slaves == MLX5_MAX_PORTS) &&
+		    (bond_status == 0x3) &&
+		    ((tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) ||
+		     (tracker->tx_type == NETDEV_LAG_TX_TYPE_HASH));
+
+	if (tracker->is_bonded != is_bonded) {
+		tracker->is_bonded = is_bonded;
+		return 1;
+	}
+
+	return 0;
+}
+
+static int mlx5_handle_changelowerstate_event(struct mlx5_lag *ldev,
+					      struct lag_tracker *tracker,
+					      struct net_device *ndev,
+					      struct netdev_notifier_changelowerstate_info *info)
+{
+	struct netdev_lag_lower_state_info *lag_lower_info;
+	int idx;
+
+	if (!netif_is_lag_port(ndev))
+		return 0;
+
+	idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev);
+	if (idx == -1)
+		return 0;
+
+	/* This information is used to determine virtual to physical
+	 * port mapping.
+	 */
+	lag_lower_info = info->lower_state_info;
+	if (!lag_lower_info)
+		return 0;
+
+	tracker->netdev_state[idx] = *lag_lower_info;
+
+	return 1;
+}
+
+static int mlx5_lag_netdev_event(struct notifier_block *this,
+				 unsigned long event, void *ptr)
+{
+	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
+	struct lag_tracker tracker;
+	struct mlx5_lag *ldev;
+	int changed = 0;
+
+	if (!net_eq(dev_net(ndev), &init_net))
+		return NOTIFY_DONE;
+
+	if ((event != NETDEV_CHANGEUPPER) && (event != NETDEV_CHANGELOWERSTATE))
+		return NOTIFY_DONE;
+
+	ldev    = container_of(this, struct mlx5_lag, nb);
+	tracker = ldev->tracker;
+
+	switch (event) {
+	case NETDEV_CHANGEUPPER:
+		changed = mlx5_handle_changeupper_event(ldev, &tracker, ndev,
+							ptr);
+		break;
+	case NETDEV_CHANGELOWERSTATE:
+		changed = mlx5_handle_changelowerstate_event(ldev, &tracker,
+							     ndev, ptr);
+		break;
+	}
+
+	mutex_lock(&lag_mutex);
+	ldev->tracker = tracker;
+	mutex_unlock(&lag_mutex);
+
+	if (changed)
+		mlx5_queue_bond_work(ldev, 0);
+
+	return NOTIFY_DONE;
+}
+
+static struct mlx5_lag *mlx5_lag_dev_alloc(void)
+{
+	struct mlx5_lag *ldev;
+
+	ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
+	if (!ldev)
+		return NULL;
+
+	INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work);
+
+	return ldev;
+}
+
+static void mlx5_lag_dev_free(struct mlx5_lag *ldev)
+{
+	kfree(ldev);
+}
+
+static void mlx5_lag_dev_add_pf(struct mlx5_lag *ldev,
+				struct mlx5_core_dev *dev,
+				struct net_device *netdev)
+{
+	unsigned int fn = PCI_FUNC(dev->pdev->devfn);
+
+	if (fn >= MLX5_MAX_PORTS)
+		return;
+
+	mutex_lock(&lag_mutex);
+	ldev->pf[fn].dev    = dev;
+	ldev->pf[fn].netdev = netdev;
+	ldev->tracker.netdev_state[fn].link_up = 0;
+	ldev->tracker.netdev_state[fn].tx_enabled = 0;
+
+	dev->priv.lag = ldev;
+	mutex_unlock(&lag_mutex);
+}
+
+static void mlx5_lag_dev_remove_pf(struct mlx5_lag *ldev,
+				   struct mlx5_core_dev *dev)
+{
+	int i;
+
+	for (i = 0; i < MLX5_MAX_PORTS; i++)
+		if (ldev->pf[i].dev == dev)
+			break;
+
+	if (i == MLX5_MAX_PORTS)
+		return;
+
+	mutex_lock(&lag_mutex);
+	memset(&ldev->pf[i], 0, sizeof(*ldev->pf));
+
+	dev->priv.lag = NULL;
+	mutex_unlock(&lag_mutex);
+}
+
+
+/* Must be called with intf_mutex held */
+void mlx5_lag_add(struct mlx5_core_dev *dev, struct net_device *netdev)
+{
+	struct mlx5_lag *ldev = NULL;
+	struct mlx5_core_dev *tmp_dev;
+
+	if (!MLX5_CAP_GEN(dev, vport_group_manager) ||
+	    !MLX5_CAP_GEN(dev, lag_master) ||
+	    (MLX5_CAP_GEN(dev, num_lag_ports) != MLX5_MAX_PORTS))
+		return;
+
+	tmp_dev = mlx5_get_next_phys_dev(dev);
+	if (tmp_dev)
+		ldev = tmp_dev->priv.lag;
+
+	if (!ldev) {
+		ldev = mlx5_lag_dev_alloc();
+		if (!ldev) {
+			mlx5_core_err(dev, "Failed to alloc lag dev\n");
+			return;
+		}
+	}
+
+	mlx5_lag_dev_add_pf(ldev, dev, netdev);
+
+	if (!ldev->nb.notifier_call) {
+		ldev->nb.notifier_call = mlx5_lag_netdev_event;
+		if (register_netdevice_notifier(&ldev->nb)) {
+			ldev->nb.notifier_call = NULL;
+			mlx5_core_err(dev, "Failed to register LAG netdev notifier\n");
+		}
+	}
+}
+
+/* Must be called with intf_mutex held */
+void mlx5_lag_remove(struct mlx5_core_dev *dev)
+{
+	struct mlx5_lag *ldev;
+	int i;
+
+	ldev = mlx5_lag_dev_get(dev);
+	if (!ldev)
+		return;
+
+	if (mlx5_lag_is_bonded(ldev))
+		mlx5_deactivate_lag(ldev);
+
+	mlx5_lag_dev_remove_pf(ldev, dev);
+
+	for (i = 0; i < MLX5_MAX_PORTS; i++)
+		if (ldev->pf[i].dev)
+			break;
+
+	if (i == MLX5_MAX_PORTS) {
+		if (ldev->nb.notifier_call)
+			unregister_netdevice_notifier(&ldev->nb);
+		cancel_delayed_work_sync(&ldev->bond_work);
+		mlx5_lag_dev_free(ldev);
+	}
+}
+
+bool mlx5_lag_is_active(struct mlx5_core_dev *dev)
+{
+	struct mlx5_lag *ldev;
+	bool res;
+
+	mutex_lock(&lag_mutex);
+	ldev = mlx5_lag_dev_get(dev);
+	res  = ldev && mlx5_lag_is_bonded(ldev);
+	mutex_unlock(&lag_mutex);
+
+	return res;
+}
+EXPORT_SYMBOL(mlx5_lag_is_active);
+
+struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev)
+{
+	struct net_device *ndev = NULL;
+	struct mlx5_lag *ldev;
+
+	mutex_lock(&lag_mutex);
+	ldev = mlx5_lag_dev_get(dev);
+
+	if (!(ldev && mlx5_lag_is_bonded(ldev)))
+		goto unlock;
+
+	if (ldev->tracker.tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
+		ndev = ldev->tracker.netdev_state[0].tx_enabled ?
+		       ldev->pf[0].netdev : ldev->pf[1].netdev;
+	} else {
+		ndev = ldev->pf[0].netdev;
+	}
+	if (ndev)
+		dev_hold(ndev);
+
+unlock:
+	mutex_unlock(&lag_mutex);
+
+	return ndev;
+}
+EXPORT_SYMBOL(mlx5_lag_get_roce_netdev);
+
+bool mlx5_lag_intf_add(struct mlx5_interface *intf, struct mlx5_priv *priv)
+{
+	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev,
+						 priv);
+	struct mlx5_lag *ldev;
+
+	if (intf->protocol != MLX5_INTERFACE_PROTOCOL_IB)
+		return true;
+
+	ldev = mlx5_lag_dev_get(dev);
+	if (!ldev || !mlx5_lag_is_bonded(ldev) || ldev->pf[0].dev == dev)
+		return true;
+
+	/* If bonded, we do not add an IB device for PF1. */
+	return false;
+}
+
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mad.c b/drivers/net/ethernet/mellanox/mlx5/core/mad.c
index 1368dac00da0..3a3b0005fd2b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mad.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mad.c
@@ -39,36 +39,33 @@
 int mlx5_core_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,
 		      u16 opmod, u8 port)
 {
-	struct mlx5_mad_ifc_mbox_in *in = NULL;
-	struct mlx5_mad_ifc_mbox_out *out = NULL;
-	int err;
+	int outlen = MLX5_ST_SZ_BYTES(mad_ifc_out);
+	int inlen = MLX5_ST_SZ_BYTES(mad_ifc_in);
+	int err = -ENOMEM;
+	void *data;
+	void *resp;
+	u32 *out;
+	u32 *in;
 
-	in = kzalloc(sizeof(*in), GFP_KERNEL);
-	if (!in)
-		return -ENOMEM;
-
-	out = kzalloc(sizeof(*out), GFP_KERNEL);
-	if (!out) {
-		err = -ENOMEM;
+	in = kzalloc(inlen, GFP_KERNEL);
+	out = kzalloc(outlen, GFP_KERNEL);
+	if (!in || !out)
 		goto out;
-	}
 
-	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MAD_IFC);
-	in->hdr.opmod = cpu_to_be16(opmod);
-	in->port = port;
+	MLX5_SET(mad_ifc_in, in, opcode, MLX5_CMD_OP_MAD_IFC);
+	MLX5_SET(mad_ifc_in, in, op_mod, opmod);
+	MLX5_SET(mad_ifc_in, in, port, port);
 
-	memcpy(in->data, inb, sizeof(in->data));
+	data = MLX5_ADDR_OF(mad_ifc_in, in, mad);
+	memcpy(data, inb, MLX5_FLD_SZ_BYTES(mad_ifc_in, mad));
 
-	err = mlx5_cmd_exec(dev, in, sizeof(*in), out, sizeof(*out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, outlen);
 	if (err)
 		goto out;
 
-	if (out->hdr.status) {
-		err = mlx5_cmd_status_to_err(&out->hdr);
-		goto out;
-	}
-
-	memcpy(outb, out->data, sizeof(out->data));
+	resp = MLX5_ADDR_OF(mad_ifc_out, out, response_mad_packet);
+	memcpy(outb, resp,
+	       MLX5_FLD_SZ_BYTES(mad_ifc_out, response_mad_packet));
 
 out:
 	kfree(out);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 4f491d43e77d..d9c3c70b29e4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -72,16 +72,6 @@ static int prof_sel = MLX5_DEFAULT_PROF;
 module_param_named(prof_sel, prof_sel, int, 0444);
 MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
 
-static LIST_HEAD(intf_list);
-static LIST_HEAD(dev_list);
-static DEFINE_MUTEX(intf_mutex);
-
-struct mlx5_device_context {
-	struct list_head	list;
-	struct mlx5_interface  *intf;
-	void		       *context;
-};
-
 enum {
 	MLX5_ATOMIC_REQ_MODE_BE = 0x0,
 	MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1,
@@ -324,7 +314,7 @@ enum {
 				MLX5_DEV_CAP_FLAG_DCT,
 };
 
-static u16 to_fw_pkey_sz(u32 size)
+static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size)
 {
 	switch (size) {
 	case 128:
@@ -340,7 +330,7 @@ static u16 to_fw_pkey_sz(u32 size)
 	case 4096:
 		return 5;
 	default:
-		pr_warn("invalid pkey table size %d\n", size);
+		mlx5_core_warn(dev, "invalid pkey table size %d\n", size);
 		return 0;
 	}
 }
@@ -363,10 +353,6 @@ static int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev,
 	MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 	MLX5_SET(query_hca_cap_in, in, op_mod, opmod);
 	err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz);
-	if (err)
-		goto query_ex;
-
-	err = mlx5_cmd_status_to_err_v2(out);
 	if (err) {
 		mlx5_core_warn(dev,
 			       "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n",
@@ -409,20 +395,11 @@ int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type)
 
 static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz, int opmod)
 {
-	u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)];
-	int err;
-
-	memset(out, 0, sizeof(out));
+	u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)] = {0};
 
 	MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP);
 	MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1);
-	err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
-	if (err)
-		return err;
-
-	err = mlx5_cmd_status_to_err_v2(out);
-
-	return err;
+	return mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
 }
 
 static int handle_hca_cap_atomic(struct mlx5_core_dev *dev)
@@ -490,7 +467,7 @@ static int handle_hca_cap(struct mlx5_core_dev *dev)
 		      128);
 	/* we limit the size of the pkey table to 128 entries for now */
 	MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size,
-		 to_fw_pkey_sz(128));
+		 to_fw_pkey_sz(dev, 128));
 
 	if (prof->mask & MLX5_PROF_MASK_QP_SIZE)
 		MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp,
@@ -528,37 +505,22 @@ static int set_hca_ctrl(struct mlx5_core_dev *dev)
 
 int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id)
 {
-	u32 out[MLX5_ST_SZ_DW(enable_hca_out)];
-	u32 in[MLX5_ST_SZ_DW(enable_hca_in)];
-	int err;
+	u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(enable_hca_in)]   = {0};
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA);
 	MLX5_SET(enable_hca_in, in, function_id, func_id);
-	memset(out, 0, sizeof(out));
-
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	return mlx5_cmd_status_to_err_v2(out);
+	return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
 }
 
 int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id)
 {
-	u32 out[MLX5_ST_SZ_DW(disable_hca_out)];
-	u32 in[MLX5_ST_SZ_DW(disable_hca_in)];
-	int err;
+	u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(disable_hca_in)]   = {0};
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
 	MLX5_SET(disable_hca_in, in, function_id, func_id);
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-	if (err)
-		return err;
-
-	return mlx5_cmd_status_to_err_v2(out);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 cycle_t mlx5_read_internal_timer(struct mlx5_core_dev *dev)
@@ -758,44 +720,40 @@ clean:
 
 static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
 {
-	u32 query_in[MLX5_ST_SZ_DW(query_issi_in)];
-	u32 query_out[MLX5_ST_SZ_DW(query_issi_out)];
-	u32 set_in[MLX5_ST_SZ_DW(set_issi_in)];
-	u32 set_out[MLX5_ST_SZ_DW(set_issi_out)];
-	int err;
+	u32 query_in[MLX5_ST_SZ_DW(query_issi_in)]   = {0};
+	u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {0};
 	u32 sup_issi;
-
-	memset(query_in, 0, sizeof(query_in));
-	memset(query_out, 0, sizeof(query_out));
+	int err;
 
 	MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI);
-
-	err = mlx5_cmd_exec_check_status(dev, query_in, sizeof(query_in),
-					 query_out, sizeof(query_out));
+	err = mlx5_cmd_exec(dev, query_in, sizeof(query_in),
+			    query_out, sizeof(query_out));
 	if (err) {
-		if (((struct mlx5_outbox_hdr *)query_out)->status ==
-		    MLX5_CMD_STAT_BAD_OP_ERR) {
+		u32 syndrome;
+		u8 status;
+
+		mlx5_cmd_mbox_status(query_out, &status, &syndrome);
+		if (status == MLX5_CMD_STAT_BAD_OP_ERR) {
 			pr_debug("Only ISSI 0 is supported\n");
 			return 0;
 		}
 
-		pr_err("failed to query ISSI\n");
+		pr_err("failed to query ISSI err(%d)\n", err);
 		return err;
 	}
 
 	sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0);
 
 	if (sup_issi & (1 << 1)) {
-		memset(set_in, 0, sizeof(set_in));
-		memset(set_out, 0, sizeof(set_out));
+		u32 set_in[MLX5_ST_SZ_DW(set_issi_in)]   = {0};
+		u32 set_out[MLX5_ST_SZ_DW(set_issi_out)] = {0};
 
 		MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI);
 		MLX5_SET(set_issi_in, set_in, current_issi, 1);
-
-		err = mlx5_cmd_exec_check_status(dev, set_in, sizeof(set_in),
-						 set_out, sizeof(set_out));
+		err = mlx5_cmd_exec(dev, set_in, sizeof(set_in),
+				    set_out, sizeof(set_out));
 		if (err) {
-			pr_err("failed to set ISSI=1\n");
+			pr_err("failed to set ISSI=1 err(%d)\n", err);
 			return err;
 		}
 
@@ -809,120 +767,6 @@ static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
 	return -ENOTSUPP;
 }
 
-static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
-{
-	struct mlx5_device_context *dev_ctx;
-	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
-
-	dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL);
-	if (!dev_ctx)
-		return;
-
-	dev_ctx->intf    = intf;
-	dev_ctx->context = intf->add(dev);
-
-	if (dev_ctx->context) {
-		spin_lock_irq(&priv->ctx_lock);
-		list_add_tail(&dev_ctx->list, &priv->ctx_list);
-		spin_unlock_irq(&priv->ctx_lock);
-	} else {
-		kfree(dev_ctx);
-	}
-}
-
-static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
-{
-	struct mlx5_device_context *dev_ctx;
-	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
-
-	list_for_each_entry(dev_ctx, &priv->ctx_list, list)
-		if (dev_ctx->intf == intf) {
-			spin_lock_irq(&priv->ctx_lock);
-			list_del(&dev_ctx->list);
-			spin_unlock_irq(&priv->ctx_lock);
-
-			intf->remove(dev, dev_ctx->context);
-			kfree(dev_ctx);
-			return;
-		}
-}
-
-static int mlx5_register_device(struct mlx5_core_dev *dev)
-{
-	struct mlx5_priv *priv = &dev->priv;
-	struct mlx5_interface *intf;
-
-	mutex_lock(&intf_mutex);
-	list_add_tail(&priv->dev_list, &dev_list);
-	list_for_each_entry(intf, &intf_list, list)
-		mlx5_add_device(intf, priv);
-	mutex_unlock(&intf_mutex);
-
-	return 0;
-}
-
-static void mlx5_unregister_device(struct mlx5_core_dev *dev)
-{
-	struct mlx5_priv *priv = &dev->priv;
-	struct mlx5_interface *intf;
-
-	mutex_lock(&intf_mutex);
-	list_for_each_entry(intf, &intf_list, list)
-		mlx5_remove_device(intf, priv);
-	list_del(&priv->dev_list);
-	mutex_unlock(&intf_mutex);
-}
-
-int mlx5_register_interface(struct mlx5_interface *intf)
-{
-	struct mlx5_priv *priv;
-
-	if (!intf->add || !intf->remove)
-		return -EINVAL;
-
-	mutex_lock(&intf_mutex);
-	list_add_tail(&intf->list, &intf_list);
-	list_for_each_entry(priv, &dev_list, dev_list)
-		mlx5_add_device(intf, priv);
-	mutex_unlock(&intf_mutex);
-
-	return 0;
-}
-EXPORT_SYMBOL(mlx5_register_interface);
-
-void mlx5_unregister_interface(struct mlx5_interface *intf)
-{
-	struct mlx5_priv *priv;
-
-	mutex_lock(&intf_mutex);
-	list_for_each_entry(priv, &dev_list, dev_list)
-		mlx5_remove_device(intf, priv);
-	list_del(&intf->list);
-	mutex_unlock(&intf_mutex);
-}
-EXPORT_SYMBOL(mlx5_unregister_interface);
-
-void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol)
-{
-	struct mlx5_priv *priv = &mdev->priv;
-	struct mlx5_device_context *dev_ctx;
-	unsigned long flags;
-	void *result = NULL;
-
-	spin_lock_irqsave(&priv->ctx_lock, flags);
-
-	list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list)
-		if ((dev_ctx->intf->protocol == protocol) &&
-		    dev_ctx->intf->get_dev) {
-			result = dev_ctx->intf->get_dev(dev_ctx->context);
-			break;
-		}
-
-	spin_unlock_irqrestore(&priv->ctx_lock, flags);
-
-	return result;
-}
-EXPORT_SYMBOL(mlx5_get_protocol_dev);
 
 static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 {
@@ -995,8 +839,102 @@ static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 	debugfs_remove(priv->dbg_root);
 }
 
-#define MLX5_IB_MOD "mlx5_ib"
-static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
+static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
+{
+	struct pci_dev *pdev = dev->pdev;
+	int err;
+
+	err = mlx5_query_hca_caps(dev);
+	if (err) {
+		dev_err(&pdev->dev, "query hca failed\n");
+		goto out;
+	}
+
+	err = mlx5_query_board_id(dev);
+	if (err) {
+		dev_err(&pdev->dev, "query board id failed\n");
+		goto out;
+	}
+
+	err = mlx5_eq_init(dev);
+	if (err) {
+		dev_err(&pdev->dev, "failed to initialize eq\n");
+		goto out;
+	}
+
+	MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock);
+
+	err = mlx5_init_cq_table(dev);
+	if (err) {
+		dev_err(&pdev->dev, "failed to initialize cq table\n");
+		goto err_eq_cleanup;
+	}
+
+	mlx5_init_qp_table(dev);
+
+	mlx5_init_srq_table(dev);
+
+	mlx5_init_mkey_table(dev);
+
+	err = mlx5_init_rl_table(dev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init rate limiting\n");
+		goto err_tables_cleanup;
+	}
+
+#ifdef CONFIG_MLX5_CORE_EN
+	err = mlx5_eswitch_init(dev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init eswitch %d\n", err);
+		goto err_rl_cleanup;
+	}
+#endif
+
+	err = mlx5_sriov_init(dev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init sriov %d\n", err);
+		goto err_eswitch_cleanup;
+	}
+
+	return 0;
+
+err_eswitch_cleanup:
+#ifdef CONFIG_MLX5_CORE_EN
+	mlx5_eswitch_cleanup(dev->priv.eswitch);
+
+err_rl_cleanup:
+#endif
+	mlx5_cleanup_rl_table(dev);
+
+err_tables_cleanup:
+	mlx5_cleanup_mkey_table(dev);
+	mlx5_cleanup_srq_table(dev);
+	mlx5_cleanup_qp_table(dev);
+	mlx5_cleanup_cq_table(dev);
+
+err_eq_cleanup:
+	mlx5_eq_cleanup(dev);
+
+out:
+	return err;
+}
+
+static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
+{
+	mlx5_sriov_cleanup(dev);
+#ifdef CONFIG_MLX5_CORE_EN
+	mlx5_eswitch_cleanup(dev->priv.eswitch);
+#endif
+	mlx5_cleanup_rl_table(dev);
+	mlx5_cleanup_mkey_table(dev);
+	mlx5_cleanup_srq_table(dev);
+	mlx5_cleanup_qp_table(dev);
+	mlx5_cleanup_cq_table(dev);
+	mlx5_eq_cleanup(dev);
+}
+
+static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
+			 bool boot)
 {
 	struct pci_dev *pdev = dev->pdev;
 	int err;
@@ -1029,12 +967,10 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 		goto out_err;
 	}
 
-	mlx5_pagealloc_init(dev);
-
 	err = mlx5_core_enable_hca(dev, 0);
 	if (err) {
 		dev_err(&pdev->dev, "enable hca failed\n");
-		goto err_pagealloc_cleanup;
+		goto err_cmd_cleanup;
 	}
 
 	err = mlx5_core_set_issi(dev);
@@ -1087,34 +1023,21 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 
 	mlx5_start_health_poll(dev);
 
-	err = mlx5_query_hca_caps(dev);
-	if (err) {
-		dev_err(&pdev->dev, "query hca failed\n");
-		goto err_stop_poll;
-	}
-
-	err = mlx5_query_board_id(dev);
-	if (err) {
-		dev_err(&pdev->dev, "query board id failed\n");
+	if (boot && mlx5_init_once(dev, priv)) {
+		dev_err(&pdev->dev, "sw objs init failed\n");
 		goto err_stop_poll;
 	}
 
 	err = mlx5_enable_msix(dev);
 	if (err) {
 		dev_err(&pdev->dev, "enable msix failed\n");
-		goto err_stop_poll;
-	}
-
-	err = mlx5_eq_init(dev);
-	if (err) {
-		dev_err(&pdev->dev, "failed to initialize eq\n");
-		goto disable_msix;
+		goto err_cleanup_once;
 	}
 
 	err = mlx5_alloc_uuars(dev, &priv->uuari);
 	if (err) {
 		dev_err(&pdev->dev, "Failed allocating uar, aborting\n");
-		goto err_eq_cleanup;
+		goto err_disable_msix;
 	}
 
 	err = mlx5_start_eqs(dev);
@@ -1130,15 +1053,10 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 	}
 
 	err = mlx5_irq_set_affinity_hints(dev);
-	if (err)
+	if (err) {
 		dev_err(&pdev->dev, "Failed to alloc affinity hint cpumask\n");
-
-	MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock);
-
-	mlx5_init_cq_table(dev);
-	mlx5_init_qp_table(dev);
-	mlx5_init_srq_table(dev);
-	mlx5_init_mkey_table(dev);
+		goto err_affinity_hints;
+	}
 
 	err = mlx5_init_fs(dev);
 	if (err) {
@@ -1146,36 +1064,26 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 		goto err_fs;
 	}
 
-	err = mlx5_init_rl_table(dev);
-	if (err) {
-		dev_err(&pdev->dev, "Failed to init rate limiting\n");
-		goto err_rl;
-	}
-
 #ifdef CONFIG_MLX5_CORE_EN
-	err = mlx5_eswitch_init(dev);
-	if (err) {
-		dev_err(&pdev->dev, "eswitch init failed %d\n", err);
-		goto err_reg_dev;
-	}
+	mlx5_eswitch_attach(dev->priv.eswitch);
 #endif
 
-	err = mlx5_sriov_init(dev);
+	err = mlx5_sriov_attach(dev);
 	if (err) {
 		dev_err(&pdev->dev, "sriov init failed %d\n", err);
 		goto err_sriov;
 	}
 
-	err = mlx5_register_device(dev);
-	if (err) {
-		dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err);
-		goto err_reg_dev;
+	if (mlx5_device_registered(dev)) {
+		mlx5_attach_device(dev);
+	} else {
+		err = mlx5_register_device(dev);
+		if (err) {
+			dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err);
+			goto err_reg_dev;
+		}
 	}
 
-	err = request_module_nowait(MLX5_IB_MOD);
-	if (err)
-		pr_info("failed request module on %s\n", MLX5_IB_MOD);
-
 	clear_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state);
 	set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
 out:
@@ -1183,23 +1091,19 @@ out:
 
 	return 0;
 
-err_sriov:
-	if (mlx5_sriov_cleanup(dev))
-		dev_err(&dev->pdev->dev, "sriov cleanup failed\n");
+err_reg_dev:
+	mlx5_sriov_detach(dev);
 
+err_sriov:
 #ifdef CONFIG_MLX5_CORE_EN
-	mlx5_eswitch_cleanup(dev->priv.eswitch);
+	mlx5_eswitch_detach(dev->priv.eswitch);
 #endif
-err_reg_dev:
-	mlx5_cleanup_rl_table(dev);
-err_rl:
 	mlx5_cleanup_fs(dev);
+
 err_fs:
-	mlx5_cleanup_mkey_table(dev);
-	mlx5_cleanup_srq_table(dev);
-	mlx5_cleanup_qp_table(dev);
-	mlx5_cleanup_cq_table(dev);
 	mlx5_irq_clear_affinity_hints(dev);
+
+err_affinity_hints:
 	free_comp_eqs(dev);
 
 err_stop_eqs:
@@ -1208,12 +1112,13 @@ err_stop_eqs:
 err_free_uar:
 	mlx5_free_uuars(dev, &priv->uuari);
 
-err_eq_cleanup:
-	mlx5_eq_cleanup(dev);
-
-disable_msix:
+err_disable_msix:
 	mlx5_disable_msix(dev);
 
+err_cleanup_once:
+	if (boot)
+		mlx5_cleanup_once(dev);
+
 err_stop_poll:
 	mlx5_stop_health_poll(dev);
 	if (mlx5_cmd_teardown_hca(dev)) {
@@ -1230,8 +1135,7 @@ reclaim_boot_pages:
 err_disable_hca:
 	mlx5_core_disable_hca(dev, 0);
 
-err_pagealloc_cleanup:
-	mlx5_pagealloc_cleanup(dev);
+err_cmd_cleanup:
 	mlx5_cmd_cleanup(dev);
 
 out_err:
@@ -1241,40 +1145,35 @@ out_err:
 	return err;
 }
 
-static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
+static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
+			   bool cleanup)
 {
 	int err = 0;
 
-	err = mlx5_sriov_cleanup(dev);
-	if (err) {
-		dev_warn(&dev->pdev->dev, "%s: sriov cleanup failed - abort\n",
-			 __func__);
-		return err;
-	}
-
 	mutex_lock(&dev->intf_state_mutex);
 	if (test_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state)) {
 		dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n",
 			 __func__);
+		if (cleanup)
+			mlx5_cleanup_once(dev);
 		goto out;
 	}
-	mlx5_unregister_device(dev);
+
+	if (mlx5_device_registered(dev))
+		mlx5_detach_device(dev);
+
+	mlx5_sriov_detach(dev);
 #ifdef CONFIG_MLX5_CORE_EN
-	mlx5_eswitch_cleanup(dev->priv.eswitch);
+	mlx5_eswitch_detach(dev->priv.eswitch);
 #endif
-
-	mlx5_cleanup_rl_table(dev);
 	mlx5_cleanup_fs(dev);
-	mlx5_cleanup_mkey_table(dev);
-	mlx5_cleanup_srq_table(dev);
-	mlx5_cleanup_qp_table(dev);
-	mlx5_cleanup_cq_table(dev);
 	mlx5_irq_clear_affinity_hints(dev);
 	free_comp_eqs(dev);
 	mlx5_stop_eqs(dev);
 	mlx5_free_uuars(dev, &priv->uuari);
-	mlx5_eq_cleanup(dev);
 	mlx5_disable_msix(dev);
+	if (cleanup)
+		mlx5_cleanup_once(dev);
 	mlx5_stop_health_poll(dev);
 	err = mlx5_cmd_teardown_hca(dev);
 	if (err) {
@@ -1284,7 +1183,6 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 	mlx5_pagealloc_stop(dev);
 	mlx5_reclaim_startup_pages(dev);
 	mlx5_core_disable_hca(dev, 0);
-	mlx5_pagealloc_cleanup(dev);
 	mlx5_cmd_cleanup(dev);
 
 out:
@@ -1294,22 +1192,6 @@ out:
 	return err;
 }
 
-void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
-		     unsigned long param)
-{
-	struct mlx5_priv *priv = &dev->priv;
-	struct mlx5_device_context *dev_ctx;
-	unsigned long flags;
-
-	spin_lock_irqsave(&priv->ctx_lock, flags);
-
-	list_for_each_entry(dev_ctx, &priv->ctx_list, list)
-		if (dev_ctx->intf->event)
-			dev_ctx->intf->event(dev, dev_ctx->context, event, param);
-
-	spin_unlock_irqrestore(&priv->ctx_lock, flags);
-}
-
 struct mlx5_core_event_handler {
 	void (*event)(struct mlx5_core_dev *dev,
 		      enum mlx5_dev_event event,
@@ -1323,6 +1205,7 @@ static const struct devlink_ops mlx5_devlink_ops = {
 #endif
 };
 
+#define MLX5_IB_MOD "mlx5_ib"
 static int init_one(struct pci_dev *pdev,
 		    const struct pci_device_id *id)
 {
@@ -1344,8 +1227,9 @@ static int init_one(struct pci_dev *pdev,
 	pci_set_drvdata(pdev, dev);
 
 	if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profile)) {
-		pr_warn("selected profile out of range, selecting default (%d)\n",
-			MLX5_DEFAULT_PROF);
+		mlx5_core_warn(dev,
+			       "selected profile out of range, selecting default (%d)\n",
+			       MLX5_DEFAULT_PROF);
 		prof_sel = MLX5_DEFAULT_PROF;
 	}
 	dev->profile = &profile[prof_sel];
@@ -1368,12 +1252,18 @@ static int init_one(struct pci_dev *pdev,
 		goto close_pci;
 	}
 
-	err = mlx5_load_one(dev, priv);
+	mlx5_pagealloc_init(dev);
+
+	err = mlx5_load_one(dev, priv, true);
 	if (err) {
 		dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err);
 		goto clean_health;
 	}
 
+	err = request_module_nowait(MLX5_IB_MOD);
+	if (err)
+		pr_info("failed request module on %s\n", MLX5_IB_MOD);
+
 	err = devlink_register(devlink, &pdev->dev);
 	if (err)
 		goto clean_load;
@@ -1381,8 +1271,9 @@ static int init_one(struct pci_dev *pdev,
 	return 0;
 
 clean_load:
-	mlx5_unload_one(dev, priv);
+	mlx5_unload_one(dev, priv, true);
 clean_health:
+	mlx5_pagealloc_cleanup(dev);
 	mlx5_health_cleanup(dev);
 close_pci:
 	mlx5_pci_close(dev, priv);
@@ -1400,11 +1291,15 @@ static void remove_one(struct pci_dev *pdev)
 	struct mlx5_priv *priv = &dev->priv;
 
 	devlink_unregister(devlink);
-	if (mlx5_unload_one(dev, priv)) {
+	mlx5_unregister_device(dev);
+
+	if (mlx5_unload_one(dev, priv, true)) {
 		dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n");
 		mlx5_health_cleanup(dev);
 		return;
 	}
+
+	mlx5_pagealloc_cleanup(dev);
 	mlx5_health_cleanup(dev);
 	mlx5_pci_close(dev, priv);
 	pci_set_drvdata(pdev, NULL);
@@ -1419,37 +1314,13 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
 
 	dev_info(&pdev->dev, "%s was called\n", __func__);
 	mlx5_enter_error_state(dev);
-	mlx5_unload_one(dev, priv);
+	mlx5_unload_one(dev, priv, false);
+	pci_save_state(pdev);
 	mlx5_pci_disable_device(dev);
 	return state == pci_channel_io_perm_failure ?
 		PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
 }
 
-static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev)
-{
-	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
-	int err = 0;
-
-	dev_info(&pdev->dev, "%s was called\n", __func__);
-
-	err = mlx5_pci_enable_device(dev);
-	if (err) {
-		dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n"
-			, __func__, err);
-		return PCI_ERS_RESULT_DISCONNECT;
-	}
-	pci_set_master(pdev);
-	pci_set_power_state(pdev, PCI_D0);
-	pci_restore_state(pdev);
-
-	return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
-}
-
-void mlx5_disable_device(struct mlx5_core_dev *dev)
-{
-	mlx5_pci_err_detected(dev->pdev, 0);
-}
-
 /* wait for the device to show vital signs by waiting
  * for the health counter to start counting.
  */
@@ -1477,22 +1348,45 @@ static int wait_vital(struct pci_dev *pdev)
 	return -ETIMEDOUT;
 }
 
-static void mlx5_pci_resume(struct pci_dev *pdev)
+static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev)
 {
 	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
-	struct mlx5_priv *priv = &dev->priv;
 	int err;
 
 	dev_info(&pdev->dev, "%s was called\n", __func__);
 
-	pci_save_state(pdev);
-	err = wait_vital(pdev);
+	err = mlx5_pci_enable_device(dev);
 	if (err) {
+		dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n"
+			, __func__, err);
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	pci_set_master(pdev);
+	pci_restore_state(pdev);
+
+	if (wait_vital(pdev)) {
 		dev_err(&pdev->dev, "%s: wait_vital timed out\n", __func__);
-		return;
+		return PCI_ERS_RESULT_DISCONNECT;
 	}
 
-	err = mlx5_load_one(dev, priv);
+	return PCI_ERS_RESULT_RECOVERED;
+}
+
+void mlx5_disable_device(struct mlx5_core_dev *dev)
+{
+	mlx5_pci_err_detected(dev->pdev, 0);
+}
+
+static void mlx5_pci_resume(struct pci_dev *pdev)
+{
+	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
+	struct mlx5_priv *priv = &dev->priv;
+	int err;
+
+	dev_info(&pdev->dev, "%s was called\n", __func__);
+
+	err = mlx5_load_one(dev, priv, false);
 	if (err)
 		dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n"
 			, __func__, err);
@@ -1514,7 +1408,7 @@ static void shutdown(struct pci_dev *pdev)
 	dev_info(&pdev->dev, "Shutdown was called\n");
 	/* Notify mlx5 clients that the kernel is being shut down */
 	set_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &dev->intf_state);
-	mlx5_unload_one(dev, priv);
+	mlx5_unload_one(dev, priv, false);
 	mlx5_pci_disable_device(dev);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mcg.c b/drivers/net/ethernet/mellanox/mlx5/core/mcg.c
index d5a0c2d61a18..ba2b09cc192f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mcg.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mcg.c
@@ -37,70 +37,30 @@
 #include <rdma/ib_verbs.h>
 #include "mlx5_core.h"
 
-struct mlx5_attach_mcg_mbox_in {
-	struct mlx5_inbox_hdr	hdr;
-	__be32			qpn;
-	__be32			rsvd;
-	u8			gid[16];
-};
-
-struct mlx5_attach_mcg_mbox_out {
-	struct mlx5_outbox_hdr	hdr;
-	u8			rsvf[8];
-};
-
-struct mlx5_detach_mcg_mbox_in {
-	struct mlx5_inbox_hdr	hdr;
-	__be32			qpn;
-	__be32			rsvd;
-	u8			gid[16];
-};
-
-struct mlx5_detach_mcg_mbox_out {
-	struct mlx5_outbox_hdr	hdr;
-	u8			rsvf[8];
-};
-
 int mlx5_core_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, u32 qpn)
 {
-	struct mlx5_attach_mcg_mbox_in in;
-	struct mlx5_attach_mcg_mbox_out out;
-	int err;
+	u32 out[MLX5_ST_SZ_DW(attach_to_mcg_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(attach_to_mcg_in)]   = {0};
+	void *gid;
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ATTACH_TO_MCG);
-	memcpy(in.gid, mgid, sizeof(*mgid));
-	in.qpn = cpu_to_be32(qpn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		err = mlx5_cmd_status_to_err(&out.hdr);
-
-	return err;
+	MLX5_SET(attach_to_mcg_in, in, opcode, MLX5_CMD_OP_ATTACH_TO_MCG);
+	MLX5_SET(attach_to_mcg_in, in, qpn, qpn);
+	gid = MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid);
+	memcpy(gid, mgid, sizeof(*mgid));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_attach_mcg);
 
 int mlx5_core_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, u32 qpn)
 {
-	struct mlx5_detach_mcg_mbox_in in;
-	struct mlx5_detach_mcg_mbox_out out;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DETTACH_FROM_MCG);
-	memcpy(in.gid, mgid, sizeof(*mgid));
-	in.qpn = cpu_to_be32(qpn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		err = mlx5_cmd_status_to_err(&out.hdr);
+	u32 out[MLX5_ST_SZ_DW(detach_from_mcg_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(detach_from_mcg_in)]   = {0};
+	void *gid;
 
-	return err;
+	MLX5_SET(detach_from_mcg_in, in, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
+	MLX5_SET(detach_from_mcg_in, in, qpn, qpn);
+	gid = MLX5_ADDR_OF(detach_from_mcg_in, in, multicast_gid);
+	memcpy(gid, mgid, sizeof(*mgid));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_detach_mcg);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index 2f86ec6fcf25..3d0cfb9f18f9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -58,8 +58,8 @@ do {									\
 } while (0)
 
 #define mlx5_core_err(__dev, format, ...)				\
-	dev_err(&(__dev)->pdev->dev, "%s:%s:%d:(pid %d): " format,	\
-	       (__dev)->priv.name, __func__, __LINE__, current->pid,	\
+	dev_err(&(__dev)->pdev->dev, "%s:%d:(pid %d): " format,	\
+		__func__, __LINE__, current->pid,	\
 	       ##__VA_ARGS__)
 
 #define mlx5_core_warn(__dev, format, ...)				\
@@ -75,19 +75,6 @@ enum {
 	MLX5_CMD_TIME, /* print command execution time */
 };
 
-static inline int mlx5_cmd_exec_check_status(struct mlx5_core_dev *dev, u32 *in,
-					     int in_size, u32 *out,
-					     int out_size)
-{
-	int err;
-
-	err = mlx5_cmd_exec(dev, in, in_size, out, out_size);
-	if (err)
-		return err;
-
-	return mlx5_cmd_status_to_err((struct mlx5_outbox_hdr *)out);
-}
-
 int mlx5_query_hca_caps(struct mlx5_core_dev *dev);
 int mlx5_query_board_id(struct mlx5_core_dev *dev);
 int mlx5_cmd_init_hca(struct mlx5_core_dev *dev);
@@ -96,7 +83,12 @@ void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
 		     unsigned long param);
 void mlx5_enter_error_state(struct mlx5_core_dev *dev);
 void mlx5_disable_device(struct mlx5_core_dev *dev);
+int mlx5_sriov_init(struct mlx5_core_dev *dev);
+void mlx5_sriov_cleanup(struct mlx5_core_dev *dev);
+int mlx5_sriov_attach(struct mlx5_core_dev *dev);
+void mlx5_sriov_detach(struct mlx5_core_dev *dev);
 int mlx5_core_sriov_configure(struct pci_dev *dev, int num_vfs);
+bool mlx5_sriov_is_enabled(struct mlx5_core_dev *dev);
 int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id);
 int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id);
 int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev);
@@ -105,7 +97,38 @@ u32 mlx5_get_msix_vec(struct mlx5_core_dev *dev, int vecidx);
 struct mlx5_eq *mlx5_eqn2eq(struct mlx5_core_dev *dev, int eqn);
 void mlx5_cq_tasklet_cb(unsigned long data);
 
+void mlx5_lag_add(struct mlx5_core_dev *dev, struct net_device *netdev);
+void mlx5_lag_remove(struct mlx5_core_dev *dev);
+
+void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv);
+void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv);
+void mlx5_attach_device(struct mlx5_core_dev *dev);
+void mlx5_detach_device(struct mlx5_core_dev *dev);
+bool mlx5_device_registered(struct mlx5_core_dev *dev);
+int mlx5_register_device(struct mlx5_core_dev *dev);
+void mlx5_unregister_device(struct mlx5_core_dev *dev);
+void mlx5_add_dev_by_protocol(struct mlx5_core_dev *dev, int protocol);
+void mlx5_remove_dev_by_protocol(struct mlx5_core_dev *dev, int protocol);
+struct mlx5_core_dev *mlx5_get_next_phys_dev(struct mlx5_core_dev *dev);
+void mlx5_dev_list_lock(void);
+void mlx5_dev_list_unlock(void);
+int mlx5_dev_list_trylock(void);
+
+bool mlx5_lag_intf_add(struct mlx5_interface *intf, struct mlx5_priv *priv);
+
 void mlx5e_init(void);
 void mlx5e_cleanup(void);
 
+static inline int mlx5_lag_is_lacp_owner(struct mlx5_core_dev *dev)
+{
+	/* LACP owner conditions:
+	 * 1) Function is physical.
+	 * 2) LAG is supported by FW.
+	 * 3) LAG is managed by driver (currently the only option).
+	 */
+	return  MLX5_CAP_GEN(dev, vport_group_manager) &&
+		   (MLX5_CAP_GEN(dev, num_lag_ports) > 1) &&
+		    MLX5_CAP_GEN(dev, lag_master);
+}
+
 #endif /* __MLX5_CORE_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mr.c b/drivers/net/ethernet/mellanox/mlx5/core/mr.c
index 77a7293921d5..b9736f505bdf 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mr.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mr.c
@@ -49,48 +49,43 @@ void mlx5_cleanup_mkey_table(struct mlx5_core_dev *dev)
 {
 }
 
-int mlx5_core_create_mkey(struct mlx5_core_dev *dev,
-			  struct mlx5_core_mkey *mkey,
-			  struct mlx5_create_mkey_mbox_in *in, int inlen,
-			  mlx5_cmd_cbk_t callback, void *context,
-			  struct mlx5_create_mkey_mbox_out *out)
+int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
+			     struct mlx5_core_mkey *mkey,
+			     u32 *in, int inlen,
+			     u32 *out, int outlen,
+			     mlx5_cmd_cbk_t callback, void *context)
 {
 	struct mlx5_mkey_table *table = &dev->priv.mkey_table;
-	struct mlx5_create_mkey_mbox_out lout;
+	u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {0};
+	u32 mkey_index;
+	void *mkc;
 	int err;
 	u8 key;
 
-	memset(&lout, 0, sizeof(lout));
 	spin_lock_irq(&dev->priv.mkey_lock);
 	key = dev->priv.mkey_key++;
 	spin_unlock_irq(&dev->priv.mkey_lock);
-	in->seg.qpn_mkey7_0 |= cpu_to_be32(key);
-	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_MKEY);
-	if (callback) {
-		err = mlx5_cmd_exec_cb(dev, in, inlen, out, sizeof(*out),
-				       callback, context);
-		return err;
-	} else {
-		err = mlx5_cmd_exec(dev, in, inlen, &lout, sizeof(lout));
-	}
+	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 
-	if (err) {
-		mlx5_core_dbg(dev, "cmd exec failed %d\n", err);
-		return err;
-	}
+	MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
+	MLX5_SET(mkc, mkc, mkey_7_0, key);
 
-	if (lout.hdr.status) {
-		mlx5_core_dbg(dev, "status %d\n", lout.hdr.status);
-		return mlx5_cmd_status_to_err(&lout.hdr);
-	}
+	if (callback)
+		return mlx5_cmd_exec_cb(dev, in, inlen, out, outlen,
+					callback, context);
+
+	err = mlx5_cmd_exec(dev, in, inlen, lout, sizeof(lout));
+	if (err)
+		return err;
 
-	mkey->iova = be64_to_cpu(in->seg.start_addr);
-	mkey->size = be64_to_cpu(in->seg.len);
-	mkey->key = mlx5_idx_to_mkey(be32_to_cpu(lout.mkey) & 0xffffff) | key;
-	mkey->pd = be32_to_cpu(in->seg.flags_pd) & 0xffffff;
+	mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index);
+	mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
+	mkey->size = MLX5_GET64(mkc, mkc, len);
+	mkey->key = mlx5_idx_to_mkey(mkey_index) | key;
+	mkey->pd = MLX5_GET(mkc, mkc, pd);
 
 	mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n",
-		      be32_to_cpu(lout.mkey), key, mkey->key);
+		      mkey_index, key, mkey->key);
 
 	/* connect to mkey tree */
 	write_lock_irq(&table->lock);
@@ -104,20 +99,25 @@ int mlx5_core_create_mkey(struct mlx5_core_dev *dev,
 
 	return err;
 }
+EXPORT_SYMBOL(mlx5_core_create_mkey_cb);
+
+int mlx5_core_create_mkey(struct mlx5_core_dev *dev,
+			  struct mlx5_core_mkey *mkey,
+			  u32 *in, int inlen)
+{
+	return mlx5_core_create_mkey_cb(dev, mkey, in, inlen,
+					NULL, 0, NULL, NULL);
+}
 EXPORT_SYMBOL(mlx5_core_create_mkey);
 
 int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev,
 			   struct mlx5_core_mkey *mkey)
 {
 	struct mlx5_mkey_table *table = &dev->priv.mkey_table;
-	struct mlx5_destroy_mkey_mbox_in in;
-	struct mlx5_destroy_mkey_mbox_out out;
+	u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)]   = {0};
 	struct mlx5_core_mkey *deleted_mkey;
 	unsigned long flags;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
 
 	write_lock_irqsave(&table->lock, flags);
 	deleted_mkey = radix_tree_delete(&table->tree, mlx5_base_mkey(mkey->key));
@@ -128,94 +128,71 @@ int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev,
 		return -ENOENT;
 	}
 
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_MKEY);
-	in.mkey = cpu_to_be32(mlx5_mkey_to_idx(mkey->key));
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		return mlx5_cmd_status_to_err(&out.hdr);
-
-	return err;
+	MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
+	MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_destroy_mkey);
 
 int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey,
-			 struct mlx5_query_mkey_mbox_out *out, int outlen)
+			 u32 *out, int outlen)
 {
-	struct mlx5_query_mkey_mbox_in in;
-	int err;
+	u32 in[MLX5_ST_SZ_DW(query_mkey_in)] = {0};
 
-	memset(&in, 0, sizeof(in));
 	memset(out, 0, outlen);
-
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_MKEY);
-	in.mkey = cpu_to_be32(mlx5_mkey_to_idx(mkey->key));
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
-	if (err)
-		return err;
-
-	if (out->hdr.status)
-		return mlx5_cmd_status_to_err(&out->hdr);
-
-	return err;
+	MLX5_SET(query_mkey_in, in, opcode, MLX5_CMD_OP_QUERY_MKEY);
+	MLX5_SET(query_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 }
 EXPORT_SYMBOL(mlx5_core_query_mkey);
 
 int mlx5_core_dump_fill_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *_mkey,
 			     u32 *mkey)
 {
-	struct mlx5_query_special_ctxs_mbox_in in;
-	struct mlx5_query_special_ctxs_mbox_out out;
+	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)]   = {0};
 	int err;
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		return mlx5_cmd_status_to_err(&out.hdr);
-
-	*mkey = be32_to_cpu(out.dump_fill_mkey);
-
+	MLX5_SET(query_special_contexts_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (!err)
+		*mkey = MLX5_GET(query_special_contexts_out, out,
+				 dump_fill_mkey);
 	return err;
 }
 EXPORT_SYMBOL(mlx5_core_dump_fill_mkey);
 
+static inline u32 mlx5_get_psv(u32 *out, int psv_index)
+{
+	switch (psv_index) {
+	case 1: return MLX5_GET(create_psv_out, out, psv1_index);
+	case 2: return MLX5_GET(create_psv_out, out, psv2_index);
+	case 3: return MLX5_GET(create_psv_out, out, psv3_index);
+	default: return MLX5_GET(create_psv_out, out, psv0_index);
+	}
+}
+
 int mlx5_core_create_psv(struct mlx5_core_dev *dev, u32 pdn,
 			 int npsvs, u32 *sig_index)
 {
-	struct mlx5_allocate_psv_in in;
-	struct mlx5_allocate_psv_out out;
+	u32 out[MLX5_ST_SZ_DW(create_psv_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(create_psv_in)]   = {0};
 	int i, err;
 
 	if (npsvs > MLX5_MAX_PSVS)
 		return -EINVAL;
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
+	MLX5_SET(create_psv_in, in, opcode, MLX5_CMD_OP_CREATE_PSV);
+	MLX5_SET(create_psv_in, in, pd, pdn);
+	MLX5_SET(create_psv_in, in, num_psv, npsvs);
 
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_PSV);
-	in.npsv_pd = cpu_to_be32((npsvs << 28) | pdn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err) {
-		mlx5_core_err(dev, "cmd exec failed %d\n", err);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (err)
 		return err;
-	}
-
-	if (out.hdr.status) {
-		mlx5_core_err(dev, "create_psv bad status %d\n",
-			      out.hdr.status);
-		return mlx5_cmd_status_to_err(&out.hdr);
-	}
 
 	for (i = 0; i < npsvs; i++)
-		sig_index[i] = be32_to_cpu(out.psv_idx[i]) & 0xffffff;
+		sig_index[i] = mlx5_get_psv(out, i);
 
 	return err;
 }
@@ -223,29 +200,11 @@ EXPORT_SYMBOL(mlx5_core_create_psv);
 
 int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num)
 {
-	struct mlx5_destroy_psv_in in;
-	struct mlx5_destroy_psv_out out;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
+	u32 out[MLX5_ST_SZ_DW(destroy_psv_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_psv_in)]   = {0};
 
-	in.psv_number = cpu_to_be32(psv_num);
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_PSV);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err) {
-		mlx5_core_err(dev, "destroy_psv cmd exec failed %d\n", err);
-		goto out;
-	}
-
-	if (out.hdr.status) {
-		mlx5_core_err(dev, "destroy_psv bad status %d\n",
-			      out.hdr.status);
-		err = mlx5_cmd_status_to_err(&out.hdr);
-		goto out;
-	}
-
-out:
-	return err;
+	MLX5_SET(destroy_psv_in, in, opcode, MLX5_CMD_OP_DESTROY_PSV);
+	MLX5_SET(destroy_psv_in, in, psvn, psv_num);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_destroy_psv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c
index 32dea3524cee..d4585154151d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c
@@ -44,12 +44,6 @@ enum {
 	MLX5_PAGES_TAKE		= 2
 };
 
-enum {
-	MLX5_BOOT_PAGES		= 1,
-	MLX5_INIT_PAGES		= 2,
-	MLX5_POST_INIT_PAGES	= 3
-};
-
 struct mlx5_pages_req {
 	struct mlx5_core_dev *dev;
 	u16	func_id;
@@ -67,33 +61,6 @@ struct fw_page {
 	unsigned		free_count;
 };
 
-struct mlx5_query_pages_inbox {
-	struct mlx5_inbox_hdr	hdr;
-	u8			rsvd[8];
-};
-
-struct mlx5_query_pages_outbox {
-	struct mlx5_outbox_hdr	hdr;
-	__be16			rsvd;
-	__be16			func_id;
-	__be32			num_pages;
-};
-
-struct mlx5_manage_pages_inbox {
-	struct mlx5_inbox_hdr	hdr;
-	__be16			rsvd;
-	__be16			func_id;
-	__be32			num_entries;
-	__be64			pas[0];
-};
-
-struct mlx5_manage_pages_outbox {
-	struct mlx5_outbox_hdr	hdr;
-	__be32			num_entries;
-	u8			rsvd[4];
-	__be64			pas[0];
-};
-
 enum {
 	MAX_RECLAIM_TIME_MSECS	= 5000,
 	MAX_RECLAIM_VFS_PAGES_TIME_MSECS = 2 * 1000 * 60,
@@ -167,24 +134,21 @@ static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr)
 static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id,
 				s32 *npages, int boot)
 {
-	struct mlx5_query_pages_inbox	in;
-	struct mlx5_query_pages_outbox	out;
+	u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(query_pages_in)]   = {0};
 	int err;
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_PAGES);
-	in.hdr.opmod = boot ? cpu_to_be16(MLX5_BOOT_PAGES) : cpu_to_be16(MLX5_INIT_PAGES);
+	MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES);
+	MLX5_SET(query_pages_in, in, op_mod, boot ?
+		 MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES :
+		 MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES);
 
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 	if (err)
 		return err;
 
-	if (out.hdr.status)
-		return mlx5_cmd_status_to_err(&out.hdr);
-
-	*npages = be32_to_cpu(out.num_pages);
-	*func_id = be16_to_cpu(out.func_id);
+	*npages = MLX5_GET(query_pages_out, out, num_pages);
+	*func_id = MLX5_GET(query_pages_out, out, function_id);
 
 	return err;
 }
@@ -280,46 +244,37 @@ out_alloc:
 
 static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id)
 {
-	struct mlx5_manage_pages_inbox *in;
-	struct mlx5_manage_pages_outbox out;
+	u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(manage_pages_in)]   = {0};
 	int err;
 
-	in = kzalloc(sizeof(*in), GFP_KERNEL);
-	if (!in)
-		return;
-
-	memset(&out, 0, sizeof(out));
-	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
-	in->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE);
-	in->func_id = cpu_to_be16(func_id);
-	err = mlx5_cmd_exec(dev, in, sizeof(*in), &out, sizeof(out));
-	if (!err)
-		err = mlx5_cmd_status_to_err(&out.hdr);
+	MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
+	MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_CANT_GIVE);
+	MLX5_SET(manage_pages_in, in, function_id, func_id);
 
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 	if (err)
-		mlx5_core_warn(dev, "page notify failed\n");
-
-	kfree(in);
+		mlx5_core_warn(dev, "page notify failed func_id(%d) err(%d)\n",
+			       func_id, err);
 }
 
 static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 		      int notify_fail)
 {
-	struct mlx5_manage_pages_inbox *in;
-	struct mlx5_manage_pages_outbox out;
-	int inlen;
+	u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0};
+	int inlen = MLX5_ST_SZ_BYTES(manage_pages_in);
 	u64 addr;
 	int err;
+	u32 *in;
 	int i;
 
-	inlen = sizeof(*in) + npages * sizeof(in->pas[0]);
+	inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]);
 	in = mlx5_vzalloc(inlen);
 	if (!in) {
 		err = -ENOMEM;
 		mlx5_core_warn(dev, "vzalloc failed %d\n", inlen);
 		goto out_free;
 	}
-	memset(&out, 0, sizeof(out));
 
 	for (i = 0; i < npages; i++) {
 retry:
@@ -332,27 +287,21 @@ retry:
 
 			goto retry;
 		}
-		in->pas[i] = cpu_to_be64(addr);
+		MLX5_SET64(manage_pages_in, in, pas[i], addr);
 	}
 
-	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
-	in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE);
-	in->func_id = cpu_to_be16(func_id);
-	in->num_entries = cpu_to_be32(npages);
-	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
+	MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
+	MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE);
+	MLX5_SET(manage_pages_in, in, function_id, func_id);
+	MLX5_SET(manage_pages_in, in, input_num_entries, npages);
+
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (err) {
 		mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n",
 			       func_id, npages, err);
 		goto out_4k;
 	}
 
-	err = mlx5_cmd_status_to_err(&out.hdr);
-	if (err) {
-		mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n",
-			       func_id, npages, out.hdr.status);
-		goto out_4k;
-	}
-
 	dev->priv.fw_pages += npages;
 	if (func_id)
 		dev->priv.vfs_pages += npages;
@@ -364,7 +313,7 @@ retry:
 
 out_4k:
 	for (i--; i >= 0; i--)
-		free_4k(dev, be64_to_cpu(in->pas[i]));
+		free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i]));
 out_free:
 	kvfree(in);
 	if (notify_fail)
@@ -373,64 +322,67 @@ out_free:
 }
 
 static int reclaim_pages_cmd(struct mlx5_core_dev *dev,
-			     struct mlx5_manage_pages_inbox *in, int in_size,
-			     struct mlx5_manage_pages_outbox *out, int out_size)
+			     u32 *in, int in_size, u32 *out, int out_size)
 {
 	struct fw_page *fwp;
 	struct rb_node *p;
+	u32 func_id;
 	u32 npages;
 	u32 i = 0;
 
 	if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR)
-		return mlx5_cmd_exec_check_status(dev, (u32 *)in, in_size,
-						  (u32 *)out, out_size);
+		return mlx5_cmd_exec(dev, in, in_size, out, out_size);
 
-	npages = be32_to_cpu(in->num_entries);
+	/* No hard feelings, we want our pages back! */
+	npages = MLX5_GET(manage_pages_in, in, input_num_entries);
+	func_id = MLX5_GET(manage_pages_in, in, function_id);
 
 	p = rb_first(&dev->priv.page_root);
 	while (p && i < npages) {
 		fwp = rb_entry(p, struct fw_page, rb_node);
-		out->pas[i] = cpu_to_be64(fwp->addr);
 		p = rb_next(p);
+		if (fwp->func_id != func_id)
+			continue;
+
+		MLX5_SET64(manage_pages_out, out, pas[i], fwp->addr);
 		i++;
 	}
 
-	out->num_entries = cpu_to_be32(i);
+	MLX5_SET(manage_pages_out, out, output_num_entries, i);
 	return 0;
 }
 
 static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages,
 			 int *nclaimed)
 {
-	struct mlx5_manage_pages_inbox   in;
-	struct mlx5_manage_pages_outbox *out;
+	int outlen = MLX5_ST_SZ_BYTES(manage_pages_out);
+	u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0};
 	int num_claimed;
-	int outlen;
-	u64 addr;
+	u32 *out;
 	int err;
 	int i;
 
 	if (nclaimed)
 		*nclaimed = 0;
 
-	memset(&in, 0, sizeof(in));
-	outlen = sizeof(*out) + npages * sizeof(out->pas[0]);
+	outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]);
 	out = mlx5_vzalloc(outlen);
 	if (!out)
 		return -ENOMEM;
 
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
-	in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE);
-	in.func_id = cpu_to_be16(func_id);
-	in.num_entries = cpu_to_be32(npages);
+	MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
+	MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE);
+	MLX5_SET(manage_pages_in, in, function_id, func_id);
+	MLX5_SET(manage_pages_in, in, input_num_entries, npages);
+
 	mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen);
-	err = reclaim_pages_cmd(dev, &in, sizeof(in), out, outlen);
+	err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen);
 	if (err) {
 		mlx5_core_err(dev, "failed reclaiming pages: err %d\n", err);
 		goto out_free;
 	}
 
-	num_claimed = be32_to_cpu(out->num_entries);
+	num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries);
 	if (num_claimed > npages) {
 		mlx5_core_warn(dev, "fw returned %d, driver asked %d => corruption\n",
 			       num_claimed, npages);
@@ -438,10 +390,9 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages,
 		goto out_free;
 	}
 
-	for (i = 0; i < num_claimed; i++) {
-		addr = be64_to_cpu(out->pas[i]);
-		free_4k(dev, addr);
-	}
+	for (i = 0; i < num_claimed; i++)
+		free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i]));
+
 
 	if (nclaimed)
 		*nclaimed = num_claimed;
@@ -518,8 +469,8 @@ static int optimal_reclaimed_pages(void)
 	int ret;
 
 	ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) -
-	       sizeof(struct mlx5_manage_pages_outbox)) /
-	       FIELD_SIZEOF(struct mlx5_manage_pages_outbox, pas[0]);
+	       MLX5_ST_SZ_BYTES(manage_pages_out)) /
+	       MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]);
 
 	return ret;
 }
@@ -594,6 +545,12 @@ int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev)
 	unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_VFS_PAGES_TIME_MSECS);
 	int prev_vfs_pages = dev->priv.vfs_pages;
 
+	/* In case of internal error we will free the pages manually later */
+	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
+		mlx5_core_warn(dev, "Skipping wait for vf pages stage");
+		return 0;
+	}
+
 	mlx5_core_dbg(dev, "Waiting for %d pages from %s\n", prev_vfs_pages,
 		      dev->priv.name);
 	while (dev->priv.vfs_pages) {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pd.c b/drivers/net/ethernet/mellanox/mlx5/core/pd.c
index f2d3aee909e8..bd830d8d6c5f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/pd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/pd.c
@@ -36,66 +36,27 @@
 #include <linux/mlx5/cmd.h>
 #include "mlx5_core.h"
 
-struct mlx5_alloc_pd_mbox_in {
-	struct mlx5_inbox_hdr	hdr;
-	u8			rsvd[8];
-};
-
-struct mlx5_alloc_pd_mbox_out {
-	struct mlx5_outbox_hdr	hdr;
-	__be32			pdn;
-	u8			rsvd[4];
-};
-
-struct mlx5_dealloc_pd_mbox_in {
-	struct mlx5_inbox_hdr	hdr;
-	__be32			pdn;
-	u8			rsvd[4];
-};
-
-struct mlx5_dealloc_pd_mbox_out {
-	struct mlx5_outbox_hdr	hdr;
-	u8			rsvd[8];
-};
-
 int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn)
 {
-	struct mlx5_alloc_pd_mbox_in	in;
-	struct mlx5_alloc_pd_mbox_out	out;
+	u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)]   = {0};
 	int err;
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ALLOC_PD);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		return mlx5_cmd_status_to_err(&out.hdr);
-
-	*pdn = be32_to_cpu(out.pdn) & 0xffffff;
+	MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (!err)
+		*pdn = MLX5_GET(alloc_pd_out, out, pd);
 	return err;
 }
 EXPORT_SYMBOL(mlx5_core_alloc_pd);
 
 int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn)
 {
-	struct mlx5_dealloc_pd_mbox_in	in;
-	struct mlx5_dealloc_pd_mbox_out	out;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DEALLOC_PD);
-	in.pdn = cpu_to_be32(pdn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
+	u32 out[MLX5_ST_SZ_DW(dealloc_pd_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)]   = {0};
 
-	if (out.hdr.status)
-		return mlx5_cmd_status_to_err(&out.hdr);
-
-	return err;
+	MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD);
+	MLX5_SET(dealloc_pd_in, in, pd, pdn);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_dealloc_pd);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c
index 752c08127138..34e7184e23c9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/port.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c
@@ -38,45 +38,42 @@
 
 int mlx5_core_access_reg(struct mlx5_core_dev *dev, void *data_in,
 			 int size_in, void *data_out, int size_out,
-			 u16 reg_num, int arg, int write)
+			 u16 reg_id, int arg, int write)
 {
-	struct mlx5_access_reg_mbox_in *in = NULL;
-	struct mlx5_access_reg_mbox_out *out = NULL;
+	int outlen = MLX5_ST_SZ_BYTES(access_register_out) + size_out;
+	int inlen = MLX5_ST_SZ_BYTES(access_register_in) + size_in;
 	int err = -ENOMEM;
+	u32 *out = NULL;
+	u32 *in = NULL;
+	void *data;
 
-	in = mlx5_vzalloc(sizeof(*in) + size_in);
-	if (!in)
-		return -ENOMEM;
-
-	out = mlx5_vzalloc(sizeof(*out) + size_out);
-	if (!out)
-		goto ex1;
-
-	memcpy(in->data, data_in, size_in);
-	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ACCESS_REG);
-	in->hdr.opmod = cpu_to_be16(!write);
-	in->arg = cpu_to_be32(arg);
-	in->register_id = cpu_to_be16(reg_num);
-	err = mlx5_cmd_exec(dev, in, sizeof(*in) + size_in, out,
-			    sizeof(*out) + size_out);
-	if (err)
-		goto ex2;
+	in = mlx5_vzalloc(inlen);
+	out = mlx5_vzalloc(outlen);
+	if (!in || !out)
+		goto out;
 
-	if (out->hdr.status)
-		err = mlx5_cmd_status_to_err(&out->hdr);
+	data = MLX5_ADDR_OF(access_register_in, in, register_data);
+	memcpy(data, data_in, size_in);
 
-	if (!err)
-		memcpy(data_out, out->data, size_out);
+	MLX5_SET(access_register_in, in, opcode, MLX5_CMD_OP_ACCESS_REG);
+	MLX5_SET(access_register_in, in, op_mod, !write);
+	MLX5_SET(access_register_in, in, argument, arg);
+	MLX5_SET(access_register_in, in, register_id, reg_id);
+
+	err = mlx5_cmd_exec(dev, in, inlen, out, outlen);
+	if (err)
+		goto out;
+
+	data = MLX5_ADDR_OF(access_register_out, out, register_data);
+	memcpy(data_out, data, size_out);
 
-ex2:
+out:
 	kvfree(out);
-ex1:
 	kvfree(in);
 	return err;
 }
 EXPORT_SYMBOL_GPL(mlx5_core_access_reg);
 
-
 struct mlx5_reg_pcap {
 	u8			rsvd0;
 	u8			port_num;
@@ -104,12 +101,10 @@ EXPORT_SYMBOL_GPL(mlx5_set_port_caps);
 int mlx5_query_port_ptys(struct mlx5_core_dev *dev, u32 *ptys,
 			 int ptys_size, int proto_mask, u8 local_port)
 {
-	u32 in[MLX5_ST_SZ_DW(ptys_reg)];
+	u32 in[MLX5_ST_SZ_DW(ptys_reg)] = {0};
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(ptys_reg, in, local_port, local_port);
 	MLX5_SET(ptys_reg, in, proto_mask, proto_mask);
-
 	return mlx5_core_access_reg(dev, in, sizeof(in), ptys,
 				    ptys_size, MLX5_REG_PTYS, 0, 0);
 }
@@ -117,13 +112,11 @@ EXPORT_SYMBOL_GPL(mlx5_query_port_ptys);
 
 int mlx5_set_port_beacon(struct mlx5_core_dev *dev, u16 beacon_duration)
 {
+	u32 in[MLX5_ST_SZ_DW(mlcr_reg)]  = {0};
 	u32 out[MLX5_ST_SZ_DW(mlcr_reg)];
-	u32 in[MLX5_ST_SZ_DW(mlcr_reg)];
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(mlcr_reg, in, local_port, 1);
 	MLX5_SET(mlcr_reg, in, beacon_duration, beacon_duration);
-
 	return mlx5_core_access_reg(dev, in, sizeof(in), out,
 				    sizeof(out), MLX5_REG_MLCR, 0, 1);
 }
@@ -182,25 +175,39 @@ int mlx5_query_port_link_width_oper(struct mlx5_core_dev *dev,
 }
 EXPORT_SYMBOL_GPL(mlx5_query_port_link_width_oper);
 
-int mlx5_query_port_proto_oper(struct mlx5_core_dev *dev,
-			       u8 *proto_oper, int proto_mask,
-			       u8 local_port)
+int mlx5_query_port_eth_proto_oper(struct mlx5_core_dev *dev,
+				   u32 *proto_oper, u8 local_port)
 {
 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
 	int err;
 
-	err = mlx5_query_port_ptys(dev, out, sizeof(out), proto_mask, local_port);
+	err = mlx5_query_port_ptys(dev, out, sizeof(out), MLX5_PTYS_EN,
+				   local_port);
 	if (err)
 		return err;
 
-	if (proto_mask == MLX5_PTYS_EN)
-		*proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper);
-	else
-		*proto_oper = MLX5_GET(ptys_reg, out, ib_proto_oper);
+	*proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper);
+
+	return 0;
+}
+EXPORT_SYMBOL(mlx5_query_port_eth_proto_oper);
+
+int mlx5_query_port_ib_proto_oper(struct mlx5_core_dev *dev,
+				  u8 *proto_oper, u8 local_port)
+{
+	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
+	int err;
+
+	err = mlx5_query_port_ptys(dev, out, sizeof(out), MLX5_PTYS_IB,
+				   local_port);
+	if (err)
+		return err;
+
+	*proto_oper = MLX5_GET(ptys_reg, out, ib_proto_oper);
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(mlx5_query_port_proto_oper);
+EXPORT_SYMBOL(mlx5_query_port_ib_proto_oper);
 
 int mlx5_set_port_ptys(struct mlx5_core_dev *dev, bool an_disable,
 		       u32 proto_admin, int proto_mask)
@@ -246,15 +253,12 @@ EXPORT_SYMBOL_GPL(mlx5_toggle_port_link);
 int mlx5_set_port_admin_status(struct mlx5_core_dev *dev,
 			       enum mlx5_port_status status)
 {
-	u32 in[MLX5_ST_SZ_DW(paos_reg)];
+	u32 in[MLX5_ST_SZ_DW(paos_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(paos_reg)];
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(paos_reg, in, local_port, 1);
 	MLX5_SET(paos_reg, in, admin_status, status);
 	MLX5_SET(paos_reg, in, ase, 1);
-
 	return mlx5_core_access_reg(dev, in, sizeof(in), out,
 				    sizeof(out), MLX5_REG_PAOS, 0, 1);
 }
@@ -263,19 +267,15 @@ EXPORT_SYMBOL_GPL(mlx5_set_port_admin_status);
 int mlx5_query_port_admin_status(struct mlx5_core_dev *dev,
 				 enum mlx5_port_status *status)
 {
-	u32 in[MLX5_ST_SZ_DW(paos_reg)];
+	u32 in[MLX5_ST_SZ_DW(paos_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(paos_reg)];
 	int err;
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(paos_reg, in, local_port, 1);
-
 	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
 				   sizeof(out), MLX5_REG_PAOS, 0, 0);
 	if (err)
 		return err;
-
 	*status = MLX5_GET(paos_reg, out, admin_status);
 	return 0;
 }
@@ -284,13 +284,10 @@ EXPORT_SYMBOL_GPL(mlx5_query_port_admin_status);
 static void mlx5_query_port_mtu(struct mlx5_core_dev *dev, u16 *admin_mtu,
 				u16 *max_mtu, u16 *oper_mtu, u8 port)
 {
-	u32 in[MLX5_ST_SZ_DW(pmtu_reg)];
+	u32 in[MLX5_ST_SZ_DW(pmtu_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(pmtu_reg)];
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(pmtu_reg, in, local_port, port);
-
 	mlx5_core_access_reg(dev, in, sizeof(in), out,
 			     sizeof(out), MLX5_REG_PMTU, 0, 0);
 
@@ -304,14 +301,11 @@ static void mlx5_query_port_mtu(struct mlx5_core_dev *dev, u16 *admin_mtu,
 
 int mlx5_set_port_mtu(struct mlx5_core_dev *dev, u16 mtu, u8 port)
 {
-	u32 in[MLX5_ST_SZ_DW(pmtu_reg)];
+	u32 in[MLX5_ST_SZ_DW(pmtu_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(pmtu_reg)];
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(pmtu_reg, in, admin_mtu, mtu);
 	MLX5_SET(pmtu_reg, in, local_port, port);
-
 	return mlx5_core_access_reg(dev, in, sizeof(in), out,
 				   sizeof(out), MLX5_REG_PMTU, 0, 1);
 }
@@ -333,15 +327,12 @@ EXPORT_SYMBOL_GPL(mlx5_query_port_oper_mtu);
 
 static int mlx5_query_module_num(struct mlx5_core_dev *dev, int *module_num)
 {
+	u32 in[MLX5_ST_SZ_DW(pmlp_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(pmlp_reg)];
-	u32 in[MLX5_ST_SZ_DW(pmlp_reg)];
 	int module_mapping;
 	int err;
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(pmlp_reg, in, local_port, 1);
-
 	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
 				   MLX5_REG_PMLP, 0, 0);
 	if (err)
@@ -410,11 +401,9 @@ EXPORT_SYMBOL_GPL(mlx5_query_module_eeprom);
 static int mlx5_query_port_pvlc(struct mlx5_core_dev *dev, u32 *pvlc,
 				int pvlc_size,  u8 local_port)
 {
-	u32 in[MLX5_ST_SZ_DW(pvlc_reg)];
+	u32 in[MLX5_ST_SZ_DW(pvlc_reg)] = {0};
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(pvlc_reg, in, local_port, local_port);
-
 	return mlx5_core_access_reg(dev, in, sizeof(in), pvlc,
 				    pvlc_size, MLX5_REG_PVLC, 0, 0);
 }
@@ -460,10 +449,9 @@ EXPORT_SYMBOL_GPL(mlx5_core_query_ib_ppcnt);
 
 int mlx5_set_port_pause(struct mlx5_core_dev *dev, u32 rx_pause, u32 tx_pause)
 {
-	u32 in[MLX5_ST_SZ_DW(pfcc_reg)];
+	u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(pfcc_reg)];
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(pfcc_reg, in, local_port, 1);
 	MLX5_SET(pfcc_reg, in, pptx, tx_pause);
 	MLX5_SET(pfcc_reg, in, pprx, rx_pause);
@@ -476,13 +464,11 @@ EXPORT_SYMBOL_GPL(mlx5_set_port_pause);
 int mlx5_query_port_pause(struct mlx5_core_dev *dev,
 			  u32 *rx_pause, u32 *tx_pause)
 {
-	u32 in[MLX5_ST_SZ_DW(pfcc_reg)];
+	u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(pfcc_reg)];
 	int err;
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(pfcc_reg, in, local_port, 1);
-
 	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
 				   sizeof(out), MLX5_REG_PFCC, 0, 0);
 	if (err)
@@ -500,10 +486,9 @@ EXPORT_SYMBOL_GPL(mlx5_query_port_pause);
 
 int mlx5_set_port_pfc(struct mlx5_core_dev *dev, u8 pfc_en_tx, u8 pfc_en_rx)
 {
-	u32 in[MLX5_ST_SZ_DW(pfcc_reg)];
+	u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(pfcc_reg)];
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(pfcc_reg, in, local_port, 1);
 	MLX5_SET(pfcc_reg, in, pfctx, pfc_en_tx);
 	MLX5_SET(pfcc_reg, in, pfcrx, pfc_en_rx);
@@ -517,13 +502,11 @@ EXPORT_SYMBOL_GPL(mlx5_set_port_pfc);
 
 int mlx5_query_port_pfc(struct mlx5_core_dev *dev, u8 *pfc_en_tx, u8 *pfc_en_rx)
 {
-	u32 in[MLX5_ST_SZ_DW(pfcc_reg)];
+	u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(pfcc_reg)];
 	int err;
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(pfcc_reg, in, local_port, 1);
-
 	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
 				   sizeof(out), MLX5_REG_PFCC, 0, 0);
 	if (err)
@@ -567,12 +550,11 @@ int mlx5_max_tc(struct mlx5_core_dev *mdev)
 
 int mlx5_set_port_prio_tc(struct mlx5_core_dev *mdev, u8 *prio_tc)
 {
-	u32 in[MLX5_ST_SZ_DW(qtct_reg)];
+	u32 in[MLX5_ST_SZ_DW(qtct_reg)] = {0};
 	u32 out[MLX5_ST_SZ_DW(qtct_reg)];
 	int err;
 	int i;
 
-	memset(in, 0, sizeof(in));
 	for (i = 0; i < 8; i++) {
 		if (prio_tc[i] > mlx5_max_tc(mdev))
 			return -EINVAL;
@@ -617,11 +599,9 @@ static int mlx5_query_port_qetcr_reg(struct mlx5_core_dev *mdev, u32 *out,
 
 int mlx5_set_port_tc_group(struct mlx5_core_dev *mdev, u8 *tc_group)
 {
-	u32 in[MLX5_ST_SZ_DW(qetc_reg)];
+	u32 in[MLX5_ST_SZ_DW(qetc_reg)] = {0};
 	int i;
 
-	memset(in, 0, sizeof(in));
-
 	for (i = 0; i <= mlx5_max_tc(mdev); i++) {
 		MLX5_SET(qetc_reg, in, tc_configuration[i].g, 1);
 		MLX5_SET(qetc_reg, in, tc_configuration[i].group, tc_group[i]);
@@ -633,11 +613,9 @@ EXPORT_SYMBOL_GPL(mlx5_set_port_tc_group);
 
 int mlx5_set_port_tc_bw_alloc(struct mlx5_core_dev *mdev, u8 *tc_bw)
 {
-	u32 in[MLX5_ST_SZ_DW(qetc_reg)];
+	u32 in[MLX5_ST_SZ_DW(qetc_reg)] = {0};
 	int i;
 
-	memset(in, 0, sizeof(in));
-
 	for (i = 0; i <= mlx5_max_tc(mdev); i++) {
 		MLX5_SET(qetc_reg, in, tc_configuration[i].b, 1);
 		MLX5_SET(qetc_reg, in, tc_configuration[i].bw_allocation, tc_bw[i]);
@@ -651,12 +629,10 @@ int mlx5_modify_port_ets_rate_limit(struct mlx5_core_dev *mdev,
 				    u8 *max_bw_value,
 				    u8 *max_bw_units)
 {
-	u32 in[MLX5_ST_SZ_DW(qetc_reg)];
+	u32 in[MLX5_ST_SZ_DW(qetc_reg)] = {0};
 	void *ets_tcn_conf;
 	int i;
 
-	memset(in, 0, sizeof(in));
-
 	MLX5_SET(qetc_reg, in, port_number, 1);
 
 	for (i = 0; i <= mlx5_max_tc(mdev); i++) {
@@ -701,35 +677,24 @@ EXPORT_SYMBOL_GPL(mlx5_query_port_ets_rate_limit);
 
 int mlx5_set_port_wol(struct mlx5_core_dev *mdev, u8 wol_mode)
 {
-	u32 in[MLX5_ST_SZ_DW(set_wol_rol_in)];
-	u32 out[MLX5_ST_SZ_DW(set_wol_rol_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(set_wol_rol_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(set_wol_rol_out)] = {0};
 
 	MLX5_SET(set_wol_rol_in, in, opcode, MLX5_CMD_OP_SET_WOL_ROL);
 	MLX5_SET(set_wol_rol_in, in, wol_mode_valid, 1);
 	MLX5_SET(set_wol_rol_in, in, wol_mode, wol_mode);
-
-	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in),
-					  out, sizeof(out));
+	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL_GPL(mlx5_set_port_wol);
 
 int mlx5_query_port_wol(struct mlx5_core_dev *mdev, u8 *wol_mode)
 {
-	u32 in[MLX5_ST_SZ_DW(query_wol_rol_in)];
-	u32 out[MLX5_ST_SZ_DW(query_wol_rol_out)];
+	u32 in[MLX5_ST_SZ_DW(query_wol_rol_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(query_wol_rol_out)] = {0};
 	int err;
 
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
-
 	MLX5_SET(query_wol_rol_in, in, opcode, MLX5_CMD_OP_QUERY_WOL_ROL);
-
-	err = mlx5_cmd_exec_check_status(mdev, in, sizeof(in),
-					 out, sizeof(out));
-
+	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
 	if (!err)
 		*wol_mode = MLX5_GET(query_wol_rol_out, out, wol_mode);
 
@@ -740,11 +705,9 @@ EXPORT_SYMBOL_GPL(mlx5_query_port_wol);
 static int mlx5_query_ports_check(struct mlx5_core_dev *mdev, u32 *out,
 				  int outlen)
 {
-	u32 in[MLX5_ST_SZ_DW(pcmr_reg)];
+	u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {0};
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(pcmr_reg, in, local_port, 1);
-
 	return mlx5_core_access_reg(mdev, in, sizeof(in), out,
 				    outlen, MLX5_REG_PCMR, 0, 0);
 }
@@ -759,12 +722,10 @@ static int mlx5_set_ports_check(struct mlx5_core_dev *mdev, u32 *in, int inlen)
 
 int mlx5_set_port_fcs(struct mlx5_core_dev *mdev, u8 enable)
 {
-	u32 in[MLX5_ST_SZ_DW(pcmr_reg)];
+	u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {0};
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(pcmr_reg, in, local_port, 1);
 	MLX5_SET(pcmr_reg, in, fcs_chk, enable);
-
 	return mlx5_set_ports_check(mdev, in, sizeof(in));
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/qp.c b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
index b82d65802d96..d0a4005fe63a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/qp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
@@ -271,30 +271,20 @@ static void destroy_qprqsq_common(struct mlx5_core_dev *dev,
 
 int mlx5_core_create_qp(struct mlx5_core_dev *dev,
 			struct mlx5_core_qp *qp,
-			struct mlx5_create_qp_mbox_in *in,
-			int inlen)
+			u32 *in, int inlen)
 {
-	struct mlx5_create_qp_mbox_out out;
-	struct mlx5_destroy_qp_mbox_in din;
-	struct mlx5_destroy_qp_mbox_out dout;
+	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {0};
+	u32 dout[MLX5_ST_SZ_DW(destroy_qp_out)];
+	u32 din[MLX5_ST_SZ_DW(destroy_qp_in)];
 	int err;
 
-	memset(&out, 0, sizeof(out));
-	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_QP);
+	MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
 
-	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
-	if (err) {
-		mlx5_core_warn(dev, "ret %d\n", err);
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+	if (err)
 		return err;
-	}
-
-	if (out.hdr.status) {
-		mlx5_core_warn(dev, "current num of QPs 0x%x\n",
-			       atomic_read(&dev->num_qps));
-		return mlx5_cmd_status_to_err(&out.hdr);
-	}
 
-	qp->qpn = be32_to_cpu(out.qpn) & 0xffffff;
+	qp->qpn = MLX5_GET(create_qp_out, out, qpn);
 	mlx5_core_dbg(dev, "qpn = 0x%x\n", qp->qpn);
 
 	err = create_qprqsq_common(dev, qp, MLX5_RES_QP);
@@ -311,12 +301,11 @@ int mlx5_core_create_qp(struct mlx5_core_dev *dev,
 	return 0;
 
 err_cmd:
-	memset(&din, 0, sizeof(din));
-	memset(&dout, 0, sizeof(dout));
-	din.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_QP);
-	din.qpn = cpu_to_be32(qp->qpn);
-	mlx5_cmd_exec(dev, &din, sizeof(din), &out, sizeof(dout));
-
+	memset(din, 0, sizeof(din));
+	memset(dout, 0, sizeof(dout));
+	MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
+	MLX5_SET(destroy_qp_in, in, qpn, qp->qpn);
+	mlx5_cmd_exec(dev, din, sizeof(din), dout, sizeof(dout));
 	return err;
 }
 EXPORT_SYMBOL_GPL(mlx5_core_create_qp);
@@ -324,45 +313,145 @@ EXPORT_SYMBOL_GPL(mlx5_core_create_qp);
 int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
 			 struct mlx5_core_qp *qp)
 {
-	struct mlx5_destroy_qp_mbox_in in;
-	struct mlx5_destroy_qp_mbox_out out;
+	u32 out[MLX5_ST_SZ_DW(destroy_qp_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_qp_in)]   = {0};
 	int err;
 
 	mlx5_debug_qp_remove(dev, qp);
 
 	destroy_qprqsq_common(dev, qp);
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_QP);
-	in.qpn = cpu_to_be32(qp->qpn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+	MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
+	MLX5_SET(destroy_qp_in, in, qpn, qp->qpn);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 	if (err)
 		return err;
 
-	if (out.hdr.status)
-		return mlx5_cmd_status_to_err(&out.hdr);
-
 	atomic_dec(&dev->num_qps);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mlx5_core_destroy_qp);
 
-int mlx5_core_qp_modify(struct mlx5_core_dev *dev, u16 operation,
-			struct mlx5_modify_qp_mbox_in *in, int sqd_event,
+struct mbox_info {
+	u32 *in;
+	u32 *out;
+	int inlen;
+	int outlen;
+};
+
+static int mbox_alloc(struct mbox_info *mbox, int inlen, int outlen)
+{
+	mbox->inlen  = inlen;
+	mbox->outlen = outlen;
+	mbox->in = kzalloc(mbox->inlen, GFP_KERNEL);
+	mbox->out = kzalloc(mbox->outlen, GFP_KERNEL);
+	if (!mbox->in || !mbox->out) {
+		kfree(mbox->in);
+		kfree(mbox->out);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void mbox_free(struct mbox_info *mbox)
+{
+	kfree(mbox->in);
+	kfree(mbox->out);
+}
+
+static int modify_qp_mbox_alloc(struct mlx5_core_dev *dev, u16 opcode, int qpn,
+				u32 opt_param_mask, void *qpc,
+				struct mbox_info *mbox)
+{
+	mbox->out = NULL;
+	mbox->in = NULL;
+
+#define MBOX_ALLOC(mbox, typ)  \
+	mbox_alloc(mbox, MLX5_ST_SZ_BYTES(typ##_in), MLX5_ST_SZ_BYTES(typ##_out))
+
+#define MOD_QP_IN_SET(typ, in, _opcode, _qpn) \
+	MLX5_SET(typ##_in, in, opcode, _opcode); \
+	MLX5_SET(typ##_in, in, qpn, _qpn)
+
+#define MOD_QP_IN_SET_QPC(typ, in, _opcode, _qpn, _opt_p, _qpc) \
+	MOD_QP_IN_SET(typ, in, _opcode, _qpn); \
+	MLX5_SET(typ##_in, in, opt_param_mask, _opt_p); \
+	memcpy(MLX5_ADDR_OF(typ##_in, in, qpc), _qpc, MLX5_ST_SZ_BYTES(qpc))
+
+	switch (opcode) {
+	/* 2RST & 2ERR */
+	case MLX5_CMD_OP_2RST_QP:
+		if (MBOX_ALLOC(mbox, qp_2rst))
+			return -ENOMEM;
+		MOD_QP_IN_SET(qp_2rst, mbox->in, opcode, qpn);
+		break;
+	case MLX5_CMD_OP_2ERR_QP:
+		if (MBOX_ALLOC(mbox, qp_2err))
+			return -ENOMEM;
+		MOD_QP_IN_SET(qp_2err, mbox->in, opcode, qpn);
+		break;
+
+	/* MODIFY with QPC */
+	case MLX5_CMD_OP_RST2INIT_QP:
+		if (MBOX_ALLOC(mbox, rst2init_qp))
+			return -ENOMEM;
+		 MOD_QP_IN_SET_QPC(rst2init_qp, mbox->in, opcode, qpn,
+				   opt_param_mask, qpc);
+		 break;
+	case MLX5_CMD_OP_INIT2RTR_QP:
+		if (MBOX_ALLOC(mbox, init2rtr_qp))
+			return -ENOMEM;
+		 MOD_QP_IN_SET_QPC(init2rtr_qp, mbox->in, opcode, qpn,
+				   opt_param_mask, qpc);
+		 break;
+	case MLX5_CMD_OP_RTR2RTS_QP:
+		if (MBOX_ALLOC(mbox, rtr2rts_qp))
+			return -ENOMEM;
+		 MOD_QP_IN_SET_QPC(rtr2rts_qp, mbox->in, opcode, qpn,
+				   opt_param_mask, qpc);
+		 break;
+	case MLX5_CMD_OP_RTS2RTS_QP:
+		if (MBOX_ALLOC(mbox, rts2rts_qp))
+			return -ENOMEM;
+		MOD_QP_IN_SET_QPC(rts2rts_qp, mbox->in, opcode, qpn,
+				  opt_param_mask, qpc);
+		break;
+	case MLX5_CMD_OP_SQERR2RTS_QP:
+		if (MBOX_ALLOC(mbox, sqerr2rts_qp))
+			return -ENOMEM;
+		MOD_QP_IN_SET_QPC(sqerr2rts_qp, mbox->in, opcode, qpn,
+				  opt_param_mask, qpc);
+		break;
+	case MLX5_CMD_OP_INIT2INIT_QP:
+		if (MBOX_ALLOC(mbox, init2init_qp))
+			return -ENOMEM;
+		MOD_QP_IN_SET_QPC(init2init_qp, mbox->in, opcode, qpn,
+				  opt_param_mask, qpc);
+		break;
+	default:
+		mlx5_core_err(dev, "Unknown transition for modify QP: OP(0x%x) QPN(0x%x)\n",
+			      opcode, qpn);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int mlx5_core_qp_modify(struct mlx5_core_dev *dev, u16 opcode,
+			u32 opt_param_mask, void *qpc,
 			struct mlx5_core_qp *qp)
 {
-	struct mlx5_modify_qp_mbox_out out;
-	int err = 0;
+	struct mbox_info mbox;
+	int err;
 
-	memset(&out, 0, sizeof(out));
-	in->hdr.opcode = cpu_to_be16(operation);
-	in->qpn = cpu_to_be32(qp->qpn);
-	err = mlx5_cmd_exec(dev, in, sizeof(*in), &out, sizeof(out));
+	err = modify_qp_mbox_alloc(dev, opcode, qp->qpn,
+				   opt_param_mask, qpc, &mbox);
 	if (err)
 		return err;
 
-	return mlx5_cmd_status_to_err(&out.hdr);
+	err = mlx5_cmd_exec(dev, mbox.in, mbox.inlen, mbox.out, mbox.outlen);
+	mbox_free(&mbox);
+	return err;
 }
 EXPORT_SYMBOL_GPL(mlx5_core_qp_modify);
 
@@ -382,66 +471,38 @@ void mlx5_cleanup_qp_table(struct mlx5_core_dev *dev)
 }
 
 int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
-		       struct mlx5_query_qp_mbox_out *out, int outlen)
+		       u32 *out, int outlen)
 {
-	struct mlx5_query_qp_mbox_in in;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(out, 0, outlen);
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_QP);
-	in.qpn = cpu_to_be32(qp->qpn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
-	if (err)
-		return err;
+	u32 in[MLX5_ST_SZ_DW(query_qp_in)] = {0};
 
-	if (out->hdr.status)
-		return mlx5_cmd_status_to_err(&out->hdr);
-
-	return err;
+	MLX5_SET(query_qp_in, in, opcode, MLX5_CMD_OP_QUERY_QP);
+	MLX5_SET(query_qp_in, in, qpn, qp->qpn);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 }
 EXPORT_SYMBOL_GPL(mlx5_core_qp_query);
 
 int mlx5_core_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn)
 {
-	struct mlx5_alloc_xrcd_mbox_in in;
-	struct mlx5_alloc_xrcd_mbox_out out;
+	u32 out[MLX5_ST_SZ_DW(alloc_xrcd_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)]   = {0};
 	int err;
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ALLOC_XRCD);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		err = mlx5_cmd_status_to_err(&out.hdr);
-	else
-		*xrcdn = be32_to_cpu(out.xrcdn) & 0xffffff;
-
+	MLX5_SET(alloc_xrcd_in, in, opcode, MLX5_CMD_OP_ALLOC_XRCD);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (!err)
+		*xrcdn = MLX5_GET(alloc_xrcd_out, out, xrcd);
 	return err;
 }
 EXPORT_SYMBOL_GPL(mlx5_core_xrcd_alloc);
 
 int mlx5_core_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn)
 {
-	struct mlx5_dealloc_xrcd_mbox_in in;
-	struct mlx5_dealloc_xrcd_mbox_out out;
-	int err;
+	u32 out[MLX5_ST_SZ_DW(dealloc_xrcd_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)]   = {0};
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DEALLOC_XRCD);
-	in.xrcdn = cpu_to_be32(xrcdn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		err = mlx5_cmd_status_to_err(&out.hdr);
-
-	return err;
+	MLX5_SET(dealloc_xrcd_in, in, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
+	MLX5_SET(dealloc_xrcd_in, in, xrcd, xrcdn);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL_GPL(mlx5_core_xrcd_dealloc);
 
@@ -449,28 +510,23 @@ EXPORT_SYMBOL_GPL(mlx5_core_xrcd_dealloc);
 int mlx5_core_page_fault_resume(struct mlx5_core_dev *dev, u32 qpn,
 				u8 flags, int error)
 {
-	struct mlx5_page_fault_resume_mbox_in in;
-	struct mlx5_page_fault_resume_mbox_out out;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_PAGE_FAULT_RESUME);
-	in.hdr.opmod = 0;
-	flags &= (MLX5_PAGE_FAULT_RESUME_REQUESTOR |
-		  MLX5_PAGE_FAULT_RESUME_WRITE	   |
-		  MLX5_PAGE_FAULT_RESUME_RDMA);
-	flags |= (error ? MLX5_PAGE_FAULT_RESUME_ERROR : 0);
-	in.flags_qpn = cpu_to_be32((qpn & MLX5_QPN_MASK) |
-				   (flags << MLX5_QPN_BITS));
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		return err;
-
-	if (out.hdr.status)
-		err = mlx5_cmd_status_to_err(&out.hdr);
-
-	return err;
+	u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)]   = {0};
+
+	MLX5_SET(page_fault_resume_in, in, opcode,
+		 MLX5_CMD_OP_PAGE_FAULT_RESUME);
+	MLX5_SET(page_fault_resume_in, in, qpn, qpn);
+
+	if (flags & MLX5_PAGE_FAULT_RESUME_REQUESTOR)
+		MLX5_SET(page_fault_resume_in, in, req_res, 1);
+	if (flags & MLX5_PAGE_FAULT_RESUME_WRITE)
+		MLX5_SET(page_fault_resume_in, in, read_write, 1);
+	if (flags & MLX5_PAGE_FAULT_RESUME_RDMA)
+		MLX5_SET(page_fault_resume_in, in, rdma, 1);
+	if (error)
+		MLX5_SET(page_fault_resume_in, in, error, 1);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL_GPL(mlx5_core_page_fault_resume);
 #endif
@@ -541,15 +597,12 @@ EXPORT_SYMBOL(mlx5_core_destroy_sq_tracked);
 
 int mlx5_core_alloc_q_counter(struct mlx5_core_dev *dev, u16 *counter_id)
 {
-	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)];
-	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)];
+	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {0};
 	int err;
 
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
-
 	MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
-	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 	if (!err)
 		*counter_id = MLX5_GET(alloc_q_counter_out, out,
 				       counter_set_id);
@@ -559,31 +612,25 @@ EXPORT_SYMBOL_GPL(mlx5_core_alloc_q_counter);
 
 int mlx5_core_dealloc_q_counter(struct mlx5_core_dev *dev, u16 counter_id)
 {
-	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)];
-	u32 out[MLX5_ST_SZ_DW(dealloc_q_counter_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(dealloc_q_counter_out)] = {0};
 
 	MLX5_SET(dealloc_q_counter_in, in, opcode,
 		 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
 	MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter_id);
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					  sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL_GPL(mlx5_core_dealloc_q_counter);
 
 int mlx5_core_query_q_counter(struct mlx5_core_dev *dev, u16 counter_id,
 			      int reset, void *out, int out_size)
 {
-	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {0};
 
 	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
 	MLX5_SET(query_q_counter_in, in, clear, reset);
 	MLX5_SET(query_q_counter_in, in, counter_set_id, counter_id);
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, out_size);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, out_size);
 }
 EXPORT_SYMBOL_GPL(mlx5_core_query_q_counter);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/rl.c b/drivers/net/ethernet/mellanox/mlx5/core/rl.c
index c07c28bd3d55..104902a93a0b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/rl.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/rl.c
@@ -63,19 +63,14 @@ static struct mlx5_rl_entry *find_rl_entry(struct mlx5_rl_table *table,
 static int mlx5_set_rate_limit_cmd(struct mlx5_core_dev *dev,
 				   u32 rate, u16 index)
 {
-	u32 in[MLX5_ST_SZ_DW(set_rate_limit_in)];
-	u32 out[MLX5_ST_SZ_DW(set_rate_limit_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(set_rate_limit_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(set_rate_limit_out)] = {0};
 
 	MLX5_SET(set_rate_limit_in, in, opcode,
 		 MLX5_CMD_OP_SET_RATE_LIMIT);
 	MLX5_SET(set_rate_limit_in, in, rate_limit_index, index);
 	MLX5_SET(set_rate_limit_in, in, rate_limit, rate);
-
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in),
-					  out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 bool mlx5_rl_is_in_range(struct mlx5_core_dev *dev, u32 rate)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
index b380a6bc1f85..e08627785590 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
@@ -37,198 +37,200 @@
 #include "eswitch.h"
 #endif
 
-static void enable_vfs(struct mlx5_core_dev *dev, int num_vfs)
+bool mlx5_sriov_is_enabled(struct mlx5_core_dev *dev)
+{
+	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
+
+	return !!sriov->num_vfs;
+}
+
+static int mlx5_device_enable_sriov(struct mlx5_core_dev *dev, int num_vfs)
 {
 	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
 	int err;
 	int vf;
 
-	for (vf = 1; vf <= num_vfs; vf++) {
-		err = mlx5_core_enable_hca(dev, vf);
+	if (sriov->enabled_vfs) {
+		mlx5_core_warn(dev,
+			       "failed to enable SRIOV on device, already enabled with %d vfs\n",
+			       sriov->enabled_vfs);
+		return -EBUSY;
+	}
+
+#ifdef CONFIG_MLX5_CORE_EN
+	err = mlx5_eswitch_enable_sriov(dev->priv.eswitch, num_vfs, SRIOV_LEGACY);
+	if (err) {
+		mlx5_core_warn(dev,
+			       "failed to enable eswitch SRIOV (%d)\n", err);
+		return err;
+	}
+#endif
+
+	for (vf = 0; vf < num_vfs; vf++) {
+		err = mlx5_core_enable_hca(dev, vf + 1);
 		if (err) {
-			mlx5_core_warn(dev, "failed to enable VF %d\n", vf - 1);
-		} else {
-			sriov->vfs_ctx[vf - 1].enabled = 1;
-			mlx5_core_dbg(dev, "successfully enabled VF %d\n", vf - 1);
+			mlx5_core_warn(dev, "failed to enable VF %d (%d)\n", vf, err);
+			continue;
 		}
+		sriov->vfs_ctx[vf].enabled = 1;
+		sriov->enabled_vfs++;
+		mlx5_core_dbg(dev, "successfully enabled VF* %d\n", vf);
+
 	}
+
+	return 0;
 }
 
-static void disable_vfs(struct mlx5_core_dev *dev, int num_vfs)
+static void mlx5_device_disable_sriov(struct mlx5_core_dev *dev)
 {
 	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
+	int err;
 	int vf;
 
-	for (vf = 1; vf <= num_vfs; vf++) {
-		if (sriov->vfs_ctx[vf - 1].enabled) {
-			if (mlx5_core_disable_hca(dev, vf))
-				mlx5_core_warn(dev, "failed to disable VF %d\n", vf - 1);
-			else
-				sriov->vfs_ctx[vf - 1].enabled = 0;
+	if (!sriov->enabled_vfs)
+		return;
+
+	for (vf = 0; vf < sriov->num_vfs; vf++) {
+		if (!sriov->vfs_ctx[vf].enabled)
+			continue;
+		err = mlx5_core_disable_hca(dev, vf + 1);
+		if (err) {
+			mlx5_core_warn(dev, "failed to disable VF %d\n", vf);
+			continue;
 		}
+		sriov->vfs_ctx[vf].enabled = 0;
+		sriov->enabled_vfs--;
 	}
+
+#ifdef CONFIG_MLX5_CORE_EN
+	mlx5_eswitch_disable_sriov(dev->priv.eswitch);
+#endif
+
+	if (mlx5_wait_for_vf_pages(dev))
+		mlx5_core_warn(dev, "timeout reclaiming VFs pages\n");
 }
 
-static int mlx5_core_create_vfs(struct pci_dev *pdev, int num_vfs)
+static int mlx5_pci_enable_sriov(struct pci_dev *pdev, int num_vfs)
 {
 	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
-	int err;
-
-	if (pci_num_vf(pdev))
-		pci_disable_sriov(pdev);
+	int err = 0;
 
-	enable_vfs(dev, num_vfs);
-
-	err = pci_enable_sriov(pdev, num_vfs);
-	if (err) {
-		dev_warn(&pdev->dev, "enable sriov failed %d\n", err);
-		goto ex;
+	if (pci_num_vf(pdev)) {
+		mlx5_core_warn(dev, "Unable to enable pci sriov, already enabled\n");
+		return -EBUSY;
 	}
 
-	return 0;
+	err = pci_enable_sriov(pdev, num_vfs);
+	if (err)
+		mlx5_core_warn(dev, "pci_enable_sriov failed : %d\n", err);
 
-ex:
-	disable_vfs(dev, num_vfs);
 	return err;
 }
 
-static int mlx5_core_sriov_enable(struct pci_dev *pdev, int num_vfs)
+static void mlx5_pci_disable_sriov(struct pci_dev *pdev)
+{
+	pci_disable_sriov(pdev);
+}
+
+static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs)
 {
 	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
 	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
-	int err;
+	int err = 0;
 
-	kfree(sriov->vfs_ctx);
-	sriov->vfs_ctx = kcalloc(num_vfs, sizeof(*sriov->vfs_ctx), GFP_ATOMIC);
-	if (!sriov->vfs_ctx)
-		return -ENOMEM;
+	err = mlx5_device_enable_sriov(dev, num_vfs);
+	if (err) {
+		mlx5_core_warn(dev, "mlx5_device_enable_sriov failed : %d\n", err);
+		return err;
+	}
 
-	sriov->enabled_vfs = num_vfs;
-	err = mlx5_core_create_vfs(pdev, num_vfs);
+	err = mlx5_pci_enable_sriov(pdev, num_vfs);
 	if (err) {
-		kfree(sriov->vfs_ctx);
-		sriov->vfs_ctx = NULL;
+		mlx5_core_warn(dev, "mlx5_pci_enable_sriov failed : %d\n", err);
+		mlx5_device_disable_sriov(dev);
 		return err;
 	}
 
+	sriov->num_vfs = num_vfs;
+
 	return 0;
 }
 
-static void mlx5_core_init_vfs(struct mlx5_core_dev *dev, int num_vfs)
+static void mlx5_sriov_disable(struct pci_dev *pdev)
 {
+	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
 	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
 
-	sriov->num_vfs = num_vfs;
-}
-
-static void mlx5_core_cleanup_vfs(struct mlx5_core_dev *dev)
-{
-	struct mlx5_core_sriov *sriov;
-
-	sriov = &dev->priv.sriov;
-	disable_vfs(dev, sriov->num_vfs);
-
-	if (mlx5_wait_for_vf_pages(dev))
-		mlx5_core_warn(dev, "timeout claiming VFs pages\n");
-
+	mlx5_pci_disable_sriov(pdev);
+	mlx5_device_disable_sriov(dev);
 	sriov->num_vfs = 0;
 }
 
 int mlx5_core_sriov_configure(struct pci_dev *pdev, int num_vfs)
 {
 	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
-	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
-	int err;
+	int err = 0;
 
 	mlx5_core_dbg(dev, "requested num_vfs %d\n", num_vfs);
 	if (!mlx5_core_is_pf(dev))
 		return -EPERM;
 
-	mlx5_core_cleanup_vfs(dev);
-
-	if (!num_vfs) {
-#ifdef CONFIG_MLX5_CORE_EN
-		mlx5_eswitch_disable_sriov(dev->priv.eswitch);
-#endif
-		kfree(sriov->vfs_ctx);
-		sriov->vfs_ctx = NULL;
-		if (!pci_vfs_assigned(pdev))
-			pci_disable_sriov(pdev);
-		else
-			pr_info("unloading PF driver while leaving orphan VFs\n");
-		return 0;
+	if (num_vfs && mlx5_lag_is_active(dev)) {
+		mlx5_core_warn(dev, "can't turn sriov on while LAG is active");
+		return -EINVAL;
 	}
 
-	err = mlx5_core_sriov_enable(pdev, num_vfs);
-	if (err) {
-		dev_warn(&pdev->dev, "mlx5_core_sriov_enable failed %d\n", err);
-		return err;
-	}
+	if (num_vfs)
+		err = mlx5_sriov_enable(pdev, num_vfs);
+	else
+		mlx5_sriov_disable(pdev);
 
-	mlx5_core_init_vfs(dev, num_vfs);
-#ifdef CONFIG_MLX5_CORE_EN
-	mlx5_eswitch_enable_sriov(dev->priv.eswitch, num_vfs, SRIOV_LEGACY);
-#endif
-
-	return num_vfs;
+	return err ? err : num_vfs;
 }
 
-static int sync_required(struct pci_dev *pdev)
+int mlx5_sriov_attach(struct mlx5_core_dev *dev)
 {
-	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
 	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
-	int cur_vfs = pci_num_vf(pdev);
 
-	if (cur_vfs != sriov->num_vfs) {
-		pr_info("current VFs %d, registered %d - sync needed\n", cur_vfs, sriov->num_vfs);
-		return 1;
-	}
+	if (!mlx5_core_is_pf(dev) || !sriov->num_vfs)
+		return 0;
 
-	return 0;
+	/* If sriov VFs exist in PCI level, enable them in device level */
+	return mlx5_device_enable_sriov(dev, sriov->num_vfs);
+}
+
+void mlx5_sriov_detach(struct mlx5_core_dev *dev)
+{
+	if (!mlx5_core_is_pf(dev))
+		return;
+
+	mlx5_device_disable_sriov(dev);
 }
 
 int mlx5_sriov_init(struct mlx5_core_dev *dev)
 {
 	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
 	struct pci_dev *pdev = dev->pdev;
-	int cur_vfs;
+	int total_vfs;
 
 	if (!mlx5_core_is_pf(dev))
 		return 0;
 
-	if (!sync_required(dev->pdev))
-		return 0;
-
-	cur_vfs = pci_num_vf(pdev);
-	sriov->vfs_ctx = kcalloc(cur_vfs, sizeof(*sriov->vfs_ctx), GFP_KERNEL);
+	total_vfs = pci_sriov_get_totalvfs(pdev);
+	sriov->num_vfs = pci_num_vf(pdev);
+	sriov->vfs_ctx = kcalloc(total_vfs, sizeof(*sriov->vfs_ctx), GFP_KERNEL);
 	if (!sriov->vfs_ctx)
 		return -ENOMEM;
 
-	sriov->enabled_vfs = cur_vfs;
-
-	mlx5_core_init_vfs(dev, cur_vfs);
-#ifdef CONFIG_MLX5_CORE_EN
-	if (cur_vfs)
-		mlx5_eswitch_enable_sriov(dev->priv.eswitch, cur_vfs,
-					  SRIOV_LEGACY);
-#endif
-
-	enable_vfs(dev, cur_vfs);
-
 	return 0;
 }
 
-int mlx5_sriov_cleanup(struct mlx5_core_dev *dev)
+void mlx5_sriov_cleanup(struct mlx5_core_dev *dev)
 {
-	struct pci_dev *pdev = dev->pdev;
-	int err;
+	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
 
 	if (!mlx5_core_is_pf(dev))
-		return 0;
+		return;
 
-	err = mlx5_core_sriov_configure(pdev, 0);
-	if (err)
-		return err;
-
-	return 0;
+	kfree(sriov->vfs_ctx);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/srq.c b/drivers/net/ethernet/mellanox/mlx5/core/srq.c
index c07f4d01b70e..3099630015d7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/srq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/srq.c
@@ -175,8 +175,8 @@ static int create_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
 	MLX5_SET(create_srq_in, create_in, opcode,
 		 MLX5_CMD_OP_CREATE_SRQ);
 
-	err = mlx5_cmd_exec_check_status(dev, create_in, inlen, create_out,
-					 sizeof(create_out));
+	err = mlx5_cmd_exec(dev, create_in, inlen, create_out,
+			    sizeof(create_out));
 	kvfree(create_in);
 	if (!err)
 		srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);
@@ -194,8 +194,8 @@ static int destroy_srq_cmd(struct mlx5_core_dev *dev,
 		 MLX5_CMD_OP_DESTROY_SRQ);
 	MLX5_SET(destroy_srq_in, srq_in, srqn, srq->srqn);
 
-	return mlx5_cmd_exec_check_status(dev, srq_in, sizeof(srq_in),
-					  srq_out, sizeof(srq_out));
+	return mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
+			     srq_out, sizeof(srq_out));
 }
 
 static int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
@@ -209,8 +209,8 @@ static int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
 	MLX5_SET(arm_xrc_srq_in, srq_in, xrc_srqn, srq->srqn);
 	MLX5_SET(arm_xrc_srq_in, srq_in, lwm,      lwm);
 
-	return  mlx5_cmd_exec_check_status(dev, srq_in, sizeof(srq_in),
-					   srq_out, sizeof(srq_out));
+	return  mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
+			      srq_out, sizeof(srq_out));
 }
 
 static int query_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
@@ -228,9 +228,8 @@ static int query_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
 	MLX5_SET(query_srq_in, srq_in, opcode,
 		 MLX5_CMD_OP_QUERY_SRQ);
 	MLX5_SET(query_srq_in, srq_in, srqn, srq->srqn);
-	err =  mlx5_cmd_exec_check_status(dev, srq_in, sizeof(srq_in),
-					  srq_out,
-					  MLX5_ST_SZ_BYTES(query_srq_out));
+	err =  mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
+			     srq_out, MLX5_ST_SZ_BYTES(query_srq_out));
 	if (err)
 		goto out;
 
@@ -272,8 +271,8 @@ static int create_xrc_srq_cmd(struct mlx5_core_dev *dev,
 		 MLX5_CMD_OP_CREATE_XRC_SRQ);
 
 	memset(create_out, 0, sizeof(create_out));
-	err = mlx5_cmd_exec_check_status(dev, create_in, inlen, create_out,
-					 sizeof(create_out));
+	err = mlx5_cmd_exec(dev, create_in, inlen, create_out,
+			    sizeof(create_out));
 	if (err)
 		goto out;
 
@@ -286,36 +285,30 @@ out:
 static int destroy_xrc_srq_cmd(struct mlx5_core_dev *dev,
 			       struct mlx5_core_srq *srq)
 {
-	u32 xrcsrq_in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)];
-	u32 xrcsrq_out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)];
-
-	memset(xrcsrq_in, 0, sizeof(xrcsrq_in));
-	memset(xrcsrq_out, 0, sizeof(xrcsrq_out));
+	u32 xrcsrq_in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)]   = {0};
+	u32 xrcsrq_out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)] = {0};
 
 	MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, opcode,
 		 MLX5_CMD_OP_DESTROY_XRC_SRQ);
 	MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
 
-	return mlx5_cmd_exec_check_status(dev, xrcsrq_in, sizeof(xrcsrq_in),
-					  xrcsrq_out, sizeof(xrcsrq_out));
+	return mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in),
+			     xrcsrq_out, sizeof(xrcsrq_out));
 }
 
 static int arm_xrc_srq_cmd(struct mlx5_core_dev *dev,
 			   struct mlx5_core_srq *srq, u16 lwm)
 {
-	u32 xrcsrq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)];
-	u32 xrcsrq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)];
-
-	memset(xrcsrq_in, 0, sizeof(xrcsrq_in));
-	memset(xrcsrq_out, 0, sizeof(xrcsrq_out));
+	u32 xrcsrq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)]   = {0};
+	u32 xrcsrq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0};
 
 	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, opcode,   MLX5_CMD_OP_ARM_XRC_SRQ);
 	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, op_mod,   MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
 	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
 	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, lwm,      lwm);
 
-	return  mlx5_cmd_exec_check_status(dev, xrcsrq_in, sizeof(xrcsrq_in),
-					   xrcsrq_out, sizeof(xrcsrq_out));
+	return  mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in),
+			      xrcsrq_out, sizeof(xrcsrq_out));
 }
 
 static int query_xrc_srq_cmd(struct mlx5_core_dev *dev,
@@ -335,9 +328,9 @@ static int query_xrc_srq_cmd(struct mlx5_core_dev *dev,
 	MLX5_SET(query_xrc_srq_in, xrcsrq_in, opcode,
 		 MLX5_CMD_OP_QUERY_XRC_SRQ);
 	MLX5_SET(query_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
-	err =  mlx5_cmd_exec_check_status(dev, xrcsrq_in, sizeof(xrcsrq_in),
-					  xrcsrq_out,
-					  MLX5_ST_SZ_BYTES(query_xrc_srq_out));
+
+	err =  mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in), xrcsrq_out,
+			     MLX5_ST_SZ_BYTES(query_xrc_srq_out));
 	if (err)
 		goto out;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
index 28274a6fbafe..a00ff49eec18 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
@@ -36,17 +36,14 @@
 
 int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
 {
-	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)];
-	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)];
+	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0};
 	int err;
 
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
-
 	MLX5_SET(alloc_transport_domain_in, in, opcode,
 		 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
 
-	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 	if (!err)
 		*tdn = MLX5_GET(alloc_transport_domain_out, out,
 				transport_domain);
@@ -57,29 +54,23 @@ EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);
 
 void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
 {
-	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)];
-	u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)] = {0};
 
 	MLX5_SET(dealloc_transport_domain_in, in, opcode,
 		 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
 	MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
-
-	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);
 
 int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)
 {
-	u32 out[MLX5_ST_SZ_DW(create_rq_out)];
+	u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {0};
 	int err;
 
 	MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
-
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*rqn = MLX5_GET(create_rq_out, out, rqn);
 
@@ -95,21 +86,18 @@ int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in, int inlen)
 	MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);
 
 	memset(out, 0, sizeof(out));
-	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_modify_rq);
 
 void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_rq_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_rq_out)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(destroy_rq_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_rq_out)] = {0};
 
 	MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
 	MLX5_SET(destroy_rq_in, in, rqn, rqn);
-
-	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_destroy_rq);
 
@@ -121,19 +109,17 @@ int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out)
 	MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
 	MLX5_SET(query_rq_in, in, rqn, rqn);
 
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, outlen);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 }
 EXPORT_SYMBOL(mlx5_core_query_rq);
 
 int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
 {
-	u32 out[MLX5_ST_SZ_DW(create_sq_out)];
+	u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {0};
 	int err;
 
 	MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
-
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*sqn = MLX5_GET(create_sq_out, out, sqn);
 
@@ -142,27 +128,22 @@ int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
 
 int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in, int inlen)
 {
-	u32 out[MLX5_ST_SZ_DW(modify_sq_out)];
+	u32 out[MLX5_ST_SZ_DW(modify_sq_out)] = {0};
 
 	MLX5_SET(modify_sq_in, in, sqn, sqn);
 	MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
-
-	memset(out, 0, sizeof(out));
-	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_modify_sq);
 
 void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_sq_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_sq_out)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(destroy_sq_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_sq_out)] = {0};
 
 	MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
 	MLX5_SET(destroy_sq_in, in, sqn, sqn);
-
-	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)
@@ -172,21 +153,20 @@ int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)
 
 	MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);
 	MLX5_SET(query_sq_in, in, sqn, sqn);
-
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, outlen);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 }
 EXPORT_SYMBOL(mlx5_core_query_sq);
 
 int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
 			 u32 *tirn)
 {
-	u32 out[MLX5_ST_SZ_DW(create_tir_out)];
+	u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {0};
 	int err;
 
 	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
 
 	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*tirn = MLX5_GET(create_tir_out, out, tirn);
 
@@ -197,39 +177,32 @@ EXPORT_SYMBOL(mlx5_core_create_tir);
 int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in,
 			 int inlen)
 {
-	u32 out[MLX5_ST_SZ_DW(modify_tir_out)];
+	u32 out[MLX5_ST_SZ_DW(modify_tir_out)] = {0};
 
 	MLX5_SET(modify_tir_in, in, tirn, tirn);
 	MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);
-
-	memset(out, 0, sizeof(out));
-	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 }
 
 void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_tir_out)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_tir_out)] = {0};
 
 	MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
 	MLX5_SET(destroy_tir_in, in, tirn, tirn);
-
-	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_destroy_tir);
 
 int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
 			 u32 *tisn)
 {
-	u32 out[MLX5_ST_SZ_DW(create_tis_out)];
+	u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {0};
 	int err;
 
 	MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
-
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*tisn = MLX5_GET(create_tis_out, out, tisn);
 
@@ -245,34 +218,29 @@ int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in,
 	MLX5_SET(modify_tis_in, in, tisn, tisn);
 	MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);
 
-	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_modify_tis);
 
 void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_tis_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_tis_out)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(destroy_tis_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_tis_out)] = {0};
 
 	MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
 	MLX5_SET(destroy_tis_in, in, tisn, tisn);
-
-	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_destroy_tis);
 
 int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen,
 			 u32 *rmpn)
 {
-	u32 out[MLX5_ST_SZ_DW(create_rmp_out)];
+	u32 out[MLX5_ST_SZ_DW(create_rmp_out)] = {0};
 	int err;
 
 	MLX5_SET(create_rmp_in, in, opcode, MLX5_CMD_OP_CREATE_RMP);
-
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*rmpn = MLX5_GET(create_rmp_out, out, rmpn);
 
@@ -281,38 +249,31 @@ int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen,
 
 int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen)
 {
-	u32 out[MLX5_ST_SZ_DW(modify_rmp_out)];
+	u32 out[MLX5_ST_SZ_DW(modify_rmp_out)] = {0};
 
 	MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);
-
-	memset(out, 0, sizeof(out));
-	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 }
 
 int mlx5_core_destroy_rmp(struct mlx5_core_dev *dev, u32 rmpn)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)] = {0};
 
 	MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
 	MLX5_SET(destroy_rmp_in, in, rmpn, rmpn);
-
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
+	return mlx5_cmd_exec(dev, in, sizeof(in), out,
 					  sizeof(out));
 }
 
 int mlx5_core_query_rmp(struct mlx5_core_dev *dev, u32 rmpn, u32 *out)
 {
-	u32 in[MLX5_ST_SZ_DW(query_rmp_in)];
+	u32 in[MLX5_ST_SZ_DW(query_rmp_in)] = {0};
 	int outlen = MLX5_ST_SZ_BYTES(query_rmp_out);
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(query_rmp_in, in, opcode, MLX5_CMD_OP_QUERY_RMP);
 	MLX5_SET(query_rmp_in, in, rmpn,   rmpn);
-
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, outlen);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 }
 
 int mlx5_core_arm_rmp(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm)
@@ -347,13 +308,11 @@ int mlx5_core_arm_rmp(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm)
 int mlx5_core_create_xsrq(struct mlx5_core_dev *dev, u32 *in, int inlen,
 			  u32 *xsrqn)
 {
-	u32 out[MLX5_ST_SZ_DW(create_xrc_srq_out)];
+	u32 out[MLX5_ST_SZ_DW(create_xrc_srq_out)] = {0};
 	int err;
 
 	MLX5_SET(create_xrc_srq_in, in, opcode,     MLX5_CMD_OP_CREATE_XRC_SRQ);
-
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*xsrqn = MLX5_GET(create_xrc_srq_out, out, xrc_srqn);
 
@@ -362,33 +321,25 @@ int mlx5_core_create_xsrq(struct mlx5_core_dev *dev, u32 *in, int inlen,
 
 int mlx5_core_destroy_xsrq(struct mlx5_core_dev *dev, u32 xsrqn)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)] = {0};
 
 	MLX5_SET(destroy_xrc_srq_in, in, opcode,   MLX5_CMD_OP_DESTROY_XRC_SRQ);
 	MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, xsrqn);
-
-	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					  sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 int mlx5_core_query_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u32 *out)
 {
-	u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)];
+	u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)] = {0};
 	void *srqc;
 	void *xrc_srqc;
 	int err;
 
-	memset(in, 0, sizeof(in));
 	MLX5_SET(query_xrc_srq_in, in, opcode,   MLX5_CMD_OP_QUERY_XRC_SRQ);
 	MLX5_SET(query_xrc_srq_in, in, xrc_srqn, xsrqn);
-
-	err =  mlx5_cmd_exec_check_status(dev, in, sizeof(in),
-					  out,
-					  MLX5_ST_SZ_BYTES(query_xrc_srq_out));
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out,
+			    MLX5_ST_SZ_BYTES(query_xrc_srq_out));
 	if (!err) {
 		xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, out,
 					xrc_srq_context_entry);
@@ -401,32 +352,25 @@ int mlx5_core_query_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u32 *out)
 
 int mlx5_core_arm_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u16 lwm)
 {
-	u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)];
-	u32 out[MLX5_ST_SZ_DW(arm_xrc_srq_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0};
 
 	MLX5_SET(arm_xrc_srq_in, in, opcode,   MLX5_CMD_OP_ARM_XRC_SRQ);
 	MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, xsrqn);
 	MLX5_SET(arm_xrc_srq_in, in, lwm,      lwm);
 	MLX5_SET(arm_xrc_srq_in, in, op_mod,
 		 MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
-
-	return  mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
-					   sizeof(out));
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
 int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
 			 u32 *rqtn)
 {
-	u32 out[MLX5_ST_SZ_DW(create_rqt_out)];
+	u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
 	int err;
 
 	MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
-
-	memset(out, 0, sizeof(out));
-	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*rqtn = MLX5_GET(create_rqt_out, out, rqtn);
 
@@ -437,25 +381,20 @@ EXPORT_SYMBOL(mlx5_core_create_rqt);
 int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
 			 int inlen)
 {
-	u32 out[MLX5_ST_SZ_DW(modify_rqt_out)];
+	u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {0};
 
 	MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
 	MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
-
-	memset(out, 0, sizeof(out));
-	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 }
 
 void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)];
-	u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0};
 
 	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
 	MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
-
-	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_destroy_rqt);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/uar.c b/drivers/net/ethernet/mellanox/mlx5/core/uar.c
index 5ff8af472bf5..ab0b896621a0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/uar.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/uar.c
@@ -42,73 +42,28 @@ enum {
 	NUM_LOW_LAT_UUARS	= 4,
 };
 
-
-struct mlx5_alloc_uar_mbox_in {
-	struct mlx5_inbox_hdr	hdr;
-	u8			rsvd[8];
-};
-
-struct mlx5_alloc_uar_mbox_out {
-	struct mlx5_outbox_hdr	hdr;
-	__be32			uarn;
-	u8			rsvd[4];
-};
-
-struct mlx5_free_uar_mbox_in {
-	struct mlx5_inbox_hdr	hdr;
-	__be32			uarn;
-	u8			rsvd[4];
-};
-
-struct mlx5_free_uar_mbox_out {
-	struct mlx5_outbox_hdr	hdr;
-	u8			rsvd[8];
-};
-
 int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn)
 {
-	struct mlx5_alloc_uar_mbox_in	in;
-	struct mlx5_alloc_uar_mbox_out	out;
+	u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(alloc_uar_in)]   = {0};
 	int err;
 
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ALLOC_UAR);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		goto ex;
-
-	if (out.hdr.status) {
-		err = mlx5_cmd_status_to_err(&out.hdr);
-		goto ex;
-	}
-
-	*uarn = be32_to_cpu(out.uarn) & 0xffffff;
-
-ex:
+	MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (!err)
+		*uarn = MLX5_GET(alloc_uar_out, out, uar);
 	return err;
 }
 EXPORT_SYMBOL(mlx5_cmd_alloc_uar);
 
 int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn)
 {
-	struct mlx5_free_uar_mbox_in	in;
-	struct mlx5_free_uar_mbox_out	out;
-	int err;
-
-	memset(&in, 0, sizeof(in));
-	memset(&out, 0, sizeof(out));
-	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DEALLOC_UAR);
-	in.uarn = cpu_to_be32(uarn);
-	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
-	if (err)
-		goto ex;
+	u32 out[MLX5_ST_SZ_DW(dealloc_uar_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)]   = {0};
 
-	if (out.hdr.status)
-		err = mlx5_cmd_status_to_err(&out.hdr);
-
-ex:
-	return err;
+	MLX5_SET(dealloc_uar_in, in, opcode, MLX5_CMD_OP_DEALLOC_UAR);
+	MLX5_SET(dealloc_uar_in, in, uar, uarn);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_cmd_free_uar);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
index 21365d06982b..525f17af108e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
@@ -39,10 +39,7 @@
 static int _mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod,
 				   u16 vport, u32 *out, int outlen)
 {
-	int err;
-	u32 in[MLX5_ST_SZ_DW(query_vport_state_in)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {0};
 
 	MLX5_SET(query_vport_state_in, in, opcode,
 		 MLX5_CMD_OP_QUERY_VPORT_STATE);
@@ -51,11 +48,7 @@ static int _mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod,
 	if (vport)
 		MLX5_SET(query_vport_state_in, in, other_vport, 1);
 
-	err = mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out, outlen);
-	if (err)
-		mlx5_core_warn(mdev, "MLX5_CMD_OP_QUERY_VPORT_STATE failed\n");
-
-	return err;
+	return mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
 }
 
 u8 mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport)
@@ -81,58 +74,43 @@ EXPORT_SYMBOL_GPL(mlx5_query_vport_admin_state);
 int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod,
 				  u16 vport, u8 state)
 {
-	u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)];
-	u32 out[MLX5_ST_SZ_DW(modify_vport_state_out)];
-	int err;
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(modify_vport_state_out)] = {0};
 
 	MLX5_SET(modify_vport_state_in, in, opcode,
 		 MLX5_CMD_OP_MODIFY_VPORT_STATE);
 	MLX5_SET(modify_vport_state_in, in, op_mod, opmod);
 	MLX5_SET(modify_vport_state_in, in, vport_number, vport);
-
 	if (vport)
 		MLX5_SET(modify_vport_state_in, in, other_vport, 1);
-
 	MLX5_SET(modify_vport_state_in, in, admin_state, state);
 
-	err = mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out,
-					 sizeof(out));
-	if (err)
-		mlx5_core_warn(mdev, "MLX5_CMD_OP_MODIFY_VPORT_STATE failed\n");
-
-	return err;
+	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL_GPL(mlx5_modify_vport_admin_state);
 
 static int mlx5_query_nic_vport_context(struct mlx5_core_dev *mdev, u16 vport,
 					u32 *out, int outlen)
 {
-	u32 in[MLX5_ST_SZ_DW(query_nic_vport_context_in)];
-
-	memset(in, 0, sizeof(in));
+	u32 in[MLX5_ST_SZ_DW(query_nic_vport_context_in)] = {0};
 
 	MLX5_SET(query_nic_vport_context_in, in, opcode,
 		 MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT);
-
 	MLX5_SET(query_nic_vport_context_in, in, vport_number, vport);
 	if (vport)
 		MLX5_SET(query_nic_vport_context_in, in, other_vport, 1);
 
-	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out, outlen);
+	return mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
 }
 
 static int mlx5_modify_nic_vport_context(struct mlx5_core_dev *mdev, void *in,
 					 int inlen)
 {
-	u32 out[MLX5_ST_SZ_DW(modify_nic_vport_context_out)];
+	u32 out[MLX5_ST_SZ_DW(modify_nic_vport_context_out)] = {0};
 
 	MLX5_SET(modify_nic_vport_context_in, in, opcode,
 		 MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT);
-
-	memset(out, 0, sizeof(out));
-	return mlx5_cmd_exec_check_status(mdev, in, inlen, out, sizeof(out));
+	return mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
 }
 
 void mlx5_query_nic_vport_min_inline(struct mlx5_core_dev *mdev,
@@ -147,6 +125,26 @@ void mlx5_query_nic_vport_min_inline(struct mlx5_core_dev *mdev,
 }
 EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_min_inline);
 
+int mlx5_modify_nic_vport_min_inline(struct mlx5_core_dev *mdev,
+				     u16 vport, u8 min_inline)
+{
+	u32 in[MLX5_ST_SZ_DW(modify_nic_vport_context_in)] = {0};
+	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
+	void *nic_vport_ctx;
+
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 field_select.min_inline, 1);
+	MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport);
+	MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1);
+
+	nic_vport_ctx = MLX5_ADDR_OF(modify_nic_vport_context_in,
+				     in, nic_vport_context);
+	MLX5_SET(nic_vport_context, nic_vport_ctx,
+		 min_wqe_inline_mode, min_inline);
+
+	return mlx5_modify_nic_vport_context(mdev, in, inlen);
+}
+
 int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev,
 				     u16 vport, u8 *addr)
 {
@@ -254,7 +252,7 @@ int mlx5_query_nic_vport_mac_list(struct mlx5_core_dev *dev,
 				  u8 addr_list[][ETH_ALEN],
 				  int *list_size)
 {
-	u32 in[MLX5_ST_SZ_DW(query_nic_vport_context_in)];
+	u32 in[MLX5_ST_SZ_DW(query_nic_vport_context_in)] = {0};
 	void *nic_vport_ctx;
 	int max_list_size;
 	int req_list_size;
@@ -278,7 +276,6 @@ int mlx5_query_nic_vport_mac_list(struct mlx5_core_dev *dev,
 	out_sz = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in) +
 			req_list_size * MLX5_ST_SZ_BYTES(mac_address_layout);
 
-	memset(in, 0, sizeof(in));
 	out = kzalloc(out_sz, GFP_KERNEL);
 	if (!out)
 		return -ENOMEM;
@@ -291,7 +288,7 @@ int mlx5_query_nic_vport_mac_list(struct mlx5_core_dev *dev,
 	if (vport)
 		MLX5_SET(query_nic_vport_context_in, in, other_vport, 1);
 
-	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, out_sz);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz);
 	if (err)
 		goto out;
 
@@ -361,7 +358,7 @@ int mlx5_modify_nic_vport_mac_list(struct mlx5_core_dev *dev,
 		ether_addr_copy(curr_mac, addr_list[i]);
 	}
 
-	err = mlx5_cmd_exec_check_status(dev, in, in_sz, out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
 	kfree(in);
 	return err;
 }
@@ -406,7 +403,7 @@ int mlx5_query_nic_vport_vlans(struct mlx5_core_dev *dev,
 	if (vport)
 		MLX5_SET(query_nic_vport_context_in, in, other_vport, 1);
 
-	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, out_sz);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz);
 	if (err)
 		goto out;
 
@@ -473,7 +470,7 @@ int mlx5_modify_nic_vport_vlans(struct mlx5_core_dev *dev,
 		MLX5_SET(vlan_layout, vlan_addr, vlan, vlans[i]);
 	}
 
-	err = mlx5_cmd_exec_check_status(dev, in, in_sz, out, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
 	kfree(in);
 	return err;
 }
@@ -631,10 +628,6 @@ int mlx5_query_hca_vport_gid(struct mlx5_core_dev *dev, u8 other_vport,
 	if (err)
 		goto out;
 
-	err = mlx5_cmd_status_to_err_v2(out);
-	if (err)
-		goto out;
-
 	tmp = out + MLX5_ST_SZ_BYTES(query_hca_vport_gid_out);
 	gid->global.subnet_prefix = tmp->global.subnet_prefix;
 	gid->global.interface_id = tmp->global.interface_id;
@@ -700,10 +693,6 @@ int mlx5_query_hca_vport_pkey(struct mlx5_core_dev *dev, u8 other_vport,
 	if (err)
 		goto out;
 
-	err = mlx5_cmd_status_to_err_v2(out);
-	if (err)
-		goto out;
-
 	pkarr = MLX5_ADDR_OF(query_hca_vport_pkey_out, out, pkey);
 	for (i = 0; i < nout; i++, pkey++, pkarr += MLX5_ST_SZ_BYTES(pkey))
 		*pkey = MLX5_GET_PR(pkey, pkarr, pkey);
@@ -721,7 +710,7 @@ int mlx5_query_hca_vport_context(struct mlx5_core_dev *dev,
 				 struct mlx5_hca_vport_context *rep)
 {
 	int out_sz = MLX5_ST_SZ_BYTES(query_hca_vport_context_out);
-	int in[MLX5_ST_SZ_DW(query_hca_vport_context_in)];
+	int in[MLX5_ST_SZ_DW(query_hca_vport_context_in)] = {0};
 	int is_group_manager;
 	void *out;
 	void *ctx;
@@ -729,7 +718,6 @@ int mlx5_query_hca_vport_context(struct mlx5_core_dev *dev,
 
 	is_group_manager = MLX5_CAP_GEN(dev, vport_group_manager);
 
-	memset(in, 0, sizeof(in));
 	out = kzalloc(out_sz, GFP_KERNEL);
 	if (!out)
 		return -ENOMEM;
@@ -752,9 +740,6 @@ int mlx5_query_hca_vport_context(struct mlx5_core_dev *dev,
 	err = mlx5_cmd_exec(dev, in, sizeof(in), out,  out_sz);
 	if (err)
 		goto ex;
-	err = mlx5_cmd_status_to_err_v2(out);
-	if (err)
-		goto ex;
 
 	ctx = MLX5_ADDR_OF(query_hca_vport_context_out, out, hca_vport_context);
 	rep->field_select = MLX5_GET_PR(hca_vport_context, ctx, field_select);
@@ -969,10 +954,6 @@ int mlx5_core_query_vport_counter(struct mlx5_core_dev *dev, u8 other_vport,
 		MLX5_SET(query_vport_counter_in, in, port_num, port_num);
 
 	err = mlx5_cmd_exec(dev, in, in_sz, out,  out_sz);
-	if (err)
-		goto free;
-	err = mlx5_cmd_status_to_err_v2(out);
-
 free:
 	kvfree(in);
 	return err;
@@ -1035,11 +1016,6 @@ int mlx5_core_modify_hca_vport_context(struct mlx5_core_dev *dev,
 	MLX5_SET(hca_vport_context, ctx, qkey_violation_counter, req->qkey_violation_counter);
 	MLX5_SET(hca_vport_context, ctx, pkey_violation_counter, req->pkey_violation_counter);
 	err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
-	if (err)
-		goto ex;
-
-	err = mlx5_cmd_status_to_err_v2(out);
-
 ex:
 	kfree(in);
 	return err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vxlan.c b/drivers/net/ethernet/mellanox/mlx5/core/vxlan.c
index e25a73ed2981..07a9ba6cfc70 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/vxlan.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/vxlan.c
@@ -46,41 +46,24 @@ void mlx5e_vxlan_init(struct mlx5e_priv *priv)
 
 static int mlx5e_vxlan_core_add_port_cmd(struct mlx5_core_dev *mdev, u16 port)
 {
-	struct mlx5_outbox_hdr *hdr;
-	int err;
-
-	u32 in[MLX5_ST_SZ_DW(add_vxlan_udp_dport_in)];
-	u32 out[MLX5_ST_SZ_DW(add_vxlan_udp_dport_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(add_vxlan_udp_dport_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(add_vxlan_udp_dport_out)] = {0};
 
 	MLX5_SET(add_vxlan_udp_dport_in, in, opcode,
 		 MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT);
 	MLX5_SET(add_vxlan_udp_dport_in, in, vxlan_udp_port, port);
-
-	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
-	if (err)
-		return err;
-
-	hdr = (struct mlx5_outbox_hdr *)out;
-	return hdr->status ? -ENOMEM : 0;
+	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
 }
 
 static int mlx5e_vxlan_core_del_port_cmd(struct mlx5_core_dev *mdev, u16 port)
 {
-	u32 in[MLX5_ST_SZ_DW(delete_vxlan_udp_dport_in)];
-	u32 out[MLX5_ST_SZ_DW(delete_vxlan_udp_dport_out)];
-
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	u32 in[MLX5_ST_SZ_DW(delete_vxlan_udp_dport_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(delete_vxlan_udp_dport_out)] = {0};
 
 	MLX5_SET(delete_vxlan_udp_dport_in, in, opcode,
 		 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
 	MLX5_SET(delete_vxlan_udp_dport_in, in, vxlan_udp_port, port);
-
-	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out,
-					  sizeof(out));
+	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
 }
 
 struct mlx5e_vxlan *mlx5e_vxlan_lookup_port(struct mlx5e_priv *priv, u16 port)
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c
index 068ee65a960b..aa33d58b9f81 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.c
@@ -1100,10 +1100,15 @@ int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 		goto err_alloc_stats;
 	}
 
-	if (mlxsw_driver->profile->used_max_lag &&
-	    mlxsw_driver->profile->used_max_port_per_lag) {
-		alloc_size = sizeof(u8) * mlxsw_driver->profile->max_lag *
-			     mlxsw_driver->profile->max_port_per_lag;
+	err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile,
+			      &mlxsw_core->resources);
+	if (err)
+		goto err_bus_init;
+
+	if (mlxsw_core->resources.max_lag_valid &&
+	    mlxsw_core->resources.max_ports_in_lag_valid) {
+		alloc_size = sizeof(u8) * mlxsw_core->resources.max_lag *
+			mlxsw_core->resources.max_ports_in_lag;
 		mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL);
 		if (!mlxsw_core->lag.mapping) {
 			err = -ENOMEM;
@@ -1111,11 +1116,6 @@ int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 		}
 	}
 
-	err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile,
-			      &mlxsw_core->resources);
-	if (err)
-		goto err_bus_init;
-
 	err = mlxsw_emad_init(mlxsw_core);
 	if (err)
 		goto err_emad_init;
@@ -1146,10 +1146,10 @@ err_hwmon_init:
 err_devlink_register:
 	mlxsw_emad_fini(mlxsw_core);
 err_emad_init:
-	mlxsw_bus->fini(bus_priv);
-err_bus_init:
 	kfree(mlxsw_core->lag.mapping);
 err_alloc_lag_mapping:
+	mlxsw_bus->fini(bus_priv);
+err_bus_init:
 	free_percpu(mlxsw_core->pcpu_stats);
 err_alloc_stats:
 	devlink_free(devlink);
@@ -1615,7 +1615,7 @@ EXPORT_SYMBOL(mlxsw_core_skb_receive);
 static int mlxsw_core_lag_mapping_index(struct mlxsw_core *mlxsw_core,
 					u16 lag_id, u8 port_index)
 {
-	return mlxsw_core->driver->profile->max_port_per_lag * lag_id +
+	return mlxsw_core->resources.max_ports_in_lag * lag_id +
 	       port_index;
 }
 
@@ -1644,7 +1644,7 @@ void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core,
 {
 	int i;
 
-	for (i = 0; i < mlxsw_core->driver->profile->max_port_per_lag; i++) {
+	for (i = 0; i < mlxsw_core->resources.max_ports_in_lag; i++) {
 		int index = mlxsw_core_lag_mapping_index(mlxsw_core,
 							 lag_id, i);
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h
index d3476ead9982..c4f550b6f783 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.h
@@ -87,6 +87,7 @@ struct mlxsw_rx_listener {
 	void (*func)(struct sk_buff *skb, u8 local_port, void *priv);
 	u8 local_port;
 	u16 trap_id;
+	enum mlxsw_reg_hpkt_action action;
 };
 
 struct mlxsw_event_listener {
@@ -178,8 +179,6 @@ struct mlxsw_swid_config {
 
 struct mlxsw_config_profile {
 	u16	used_max_vepa_channels:1,
-		used_max_lag:1,
-		used_max_port_per_lag:1,
 		used_max_mid:1,
 		used_max_pgt:1,
 		used_max_system_port:1,
@@ -191,10 +190,9 @@ struct mlxsw_config_profile {
 		used_max_pkey:1,
 		used_ar_sec:1,
 		used_adaptive_routing_group_cap:1,
-		used_kvd_sizes:1;
+		used_kvd_split_data:1; /* indicate for the kvd's values */
+
 	u8	max_vepa_channels;
-	u16	max_lag;
-	u16	max_port_per_lag;
 	u16	max_mid;
 	u16	max_pgt;
 	u16	max_system_port;
@@ -213,8 +211,9 @@ struct mlxsw_config_profile {
 	u16	adaptive_routing_group_cap;
 	u8	arn;
 	u32	kvd_linear_size;
-	u32	kvd_hash_single_size;
-	u32	kvd_hash_double_size;
+	u16	kvd_hash_granularity;
+	u8	kvd_hash_single_parts;
+	u8	kvd_hash_double_parts;
 	u8	resource_query_enable;
 	struct mlxsw_swid_config swid_config[MLXSW_CONFIG_PROFILE_SWID_COUNT];
 };
@@ -268,8 +267,35 @@ struct mlxsw_driver {
 };
 
 struct mlxsw_resources {
-	u8	max_span_valid:1;
+	u32	max_span_valid:1,
+		max_lag_valid:1,
+		max_ports_in_lag_valid:1,
+		kvd_size_valid:1,
+		kvd_single_min_size_valid:1,
+		kvd_double_min_size_valid:1,
+		max_virtual_routers_valid:1,
+		max_system_ports_valid:1,
+		max_vlan_groups_valid:1,
+		max_regions_valid:1,
+		max_rif_valid:1;
 	u8      max_span;
+	u8	max_lag;
+	u8	max_ports_in_lag;
+	u32	kvd_size;
+	u32	kvd_single_min_size;
+	u32	kvd_double_min_size;
+	u16     max_virtual_routers;
+	u16	max_system_ports;
+	u16	max_vlan_groups;
+	u16	max_regions;
+	u16	max_rif;
+
+	/* Internal resources.
+	 * Determined by the SW, not queried from the HW.
+	 */
+	u32	kvd_single_size;
+	u32	kvd_double_size;
+	u32	kvd_linear_size;
 };
 
 struct mlxsw_resources *mlxsw_core_resources_get(struct mlxsw_core *mlxsw_core);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c
index 1d1360c178bb..e742bd4e8894 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/pci.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c
@@ -1156,6 +1156,16 @@ mlxsw_pci_config_profile_swid_config(struct mlxsw_pci *mlxsw_pci,
 
 #define MLXSW_RESOURCES_TABLE_END_ID 0xffff
 #define MLXSW_MAX_SPAN_ID 0x2420
+#define MLXSW_MAX_LAG_ID 0x2520
+#define MLXSW_MAX_PORTS_IN_LAG_ID 0x2521
+#define MLXSW_KVD_SIZE_ID 0x1001
+#define MLXSW_KVD_SINGLE_MIN_SIZE_ID 0x1002
+#define MLXSW_KVD_DOUBLE_MIN_SIZE_ID 0x1003
+#define MLXSW_MAX_VIRTUAL_ROUTERS_ID 0x2C01
+#define MLXSW_MAX_SYSTEM_PORT_ID 0x2502
+#define MLXSW_MAX_VLAN_GROUPS_ID 0x2906
+#define MLXSW_MAX_REGIONS_ID 0x2901
+#define MLXSW_MAX_RIF_ID 0x2C02
 #define MLXSW_RESOURCES_QUERY_MAX_QUERIES 100
 #define MLXSW_RESOURCES_PER_QUERY 32
 
@@ -1167,6 +1177,46 @@ static void mlxsw_pci_resources_query_parse(int id, u64 val,
 		resources->max_span = val;
 		resources->max_span_valid = 1;
 		break;
+	case MLXSW_MAX_LAG_ID:
+		resources->max_lag = val;
+		resources->max_lag_valid = 1;
+		break;
+	case MLXSW_MAX_PORTS_IN_LAG_ID:
+		resources->max_ports_in_lag = val;
+		resources->max_ports_in_lag_valid = 1;
+		break;
+	case MLXSW_KVD_SIZE_ID:
+		resources->kvd_size = val;
+		resources->kvd_size_valid = 1;
+		break;
+	case MLXSW_KVD_SINGLE_MIN_SIZE_ID:
+		resources->kvd_single_min_size = val;
+		resources->kvd_single_min_size_valid = 1;
+		break;
+	case MLXSW_KVD_DOUBLE_MIN_SIZE_ID:
+		resources->kvd_double_min_size = val;
+		resources->kvd_double_min_size_valid = 1;
+		break;
+	case MLXSW_MAX_VIRTUAL_ROUTERS_ID:
+		resources->max_virtual_routers = val;
+		resources->max_virtual_routers_valid = 1;
+		break;
+	case MLXSW_MAX_SYSTEM_PORT_ID:
+		resources->max_system_ports = val;
+		resources->max_system_ports_valid = 1;
+		break;
+	case MLXSW_MAX_VLAN_GROUPS_ID:
+		resources->max_vlan_groups = val;
+		resources->max_vlan_groups_valid = 1;
+		break;
+	case MLXSW_MAX_REGIONS_ID:
+		resources->max_regions = val;
+		resources->max_regions_valid = 1;
+		break;
+	case MLXSW_MAX_RIF_ID:
+		resources->max_rif = val;
+		resources->max_rif_valid = 1;
+		break;
 	default:
 		break;
 	}
@@ -1209,10 +1259,52 @@ static int mlxsw_pci_resources_query(struct mlxsw_pci *mlxsw_pci, char *mbox,
 	return -EIO;
 }
 
+static int mlxsw_pci_profile_get_kvd_sizes(const struct mlxsw_config_profile *profile,
+					   struct mlxsw_resources *resources)
+{
+	u32 singles_size, doubles_size, linear_size;
+
+	if (!resources->kvd_single_min_size_valid ||
+	    !resources->kvd_double_min_size_valid ||
+	    !profile->used_kvd_split_data)
+		return -EIO;
+
+	linear_size = profile->kvd_linear_size;
+
+	/* The hash part is what left of the kvd without the
+	 * linear part. It is split to the single size and
+	 * double size by the parts ratio from the profile.
+	 * Both sizes must be a multiplications of the
+	 * granularity from the profile.
+	 */
+	doubles_size = (resources->kvd_size - linear_size);
+	doubles_size *= profile->kvd_hash_double_parts;
+	doubles_size /= (profile->kvd_hash_double_parts +
+			 profile->kvd_hash_single_parts);
+	doubles_size /= profile->kvd_hash_granularity;
+	doubles_size *= profile->kvd_hash_granularity;
+	singles_size = resources->kvd_size - doubles_size -
+		       linear_size;
+
+	/* Check results are legal. */
+	if (singles_size < resources->kvd_single_min_size ||
+	    doubles_size < resources->kvd_double_min_size ||
+	    resources->kvd_size < linear_size)
+		return -EIO;
+
+	resources->kvd_single_size = singles_size;
+	resources->kvd_double_size = doubles_size;
+	resources->kvd_linear_size = linear_size;
+
+	return 0;
+}
+
 static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
-				    const struct mlxsw_config_profile *profile)
+				    const struct mlxsw_config_profile *profile,
+				    struct mlxsw_resources *resources)
 {
 	int i;
+	int err;
 
 	mlxsw_cmd_mbox_zero(mbox);
 
@@ -1222,18 +1314,6 @@ static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
 		mlxsw_cmd_mbox_config_profile_max_vepa_channels_set(
 			mbox, profile->max_vepa_channels);
 	}
-	if (profile->used_max_lag) {
-		mlxsw_cmd_mbox_config_profile_set_max_lag_set(
-			mbox, 1);
-		mlxsw_cmd_mbox_config_profile_max_lag_set(
-			mbox, profile->max_lag);
-	}
-	if (profile->used_max_port_per_lag) {
-		mlxsw_cmd_mbox_config_profile_set_max_port_per_lag_set(
-			mbox, 1);
-		mlxsw_cmd_mbox_config_profile_max_port_per_lag_set(
-			mbox, profile->max_port_per_lag);
-	}
 	if (profile->used_max_mid) {
 		mlxsw_cmd_mbox_config_profile_set_max_mid_set(
 			mbox, 1);
@@ -1310,19 +1390,22 @@ static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
 		mlxsw_cmd_mbox_config_profile_adaptive_routing_group_cap_set(
 			mbox, profile->adaptive_routing_group_cap);
 	}
-	if (profile->used_kvd_sizes) {
-		mlxsw_cmd_mbox_config_profile_set_kvd_linear_size_set(
-			mbox, 1);
-		mlxsw_cmd_mbox_config_profile_kvd_linear_size_set(
-			mbox, profile->kvd_linear_size);
-		mlxsw_cmd_mbox_config_profile_set_kvd_hash_single_size_set(
-			mbox, 1);
-		mlxsw_cmd_mbox_config_profile_kvd_hash_single_size_set(
-			mbox, profile->kvd_hash_single_size);
+	if (resources->kvd_size_valid) {
+		err = mlxsw_pci_profile_get_kvd_sizes(profile, resources);
+		if (err)
+			return err;
+
+		mlxsw_cmd_mbox_config_profile_set_kvd_linear_size_set(mbox, 1);
+		mlxsw_cmd_mbox_config_profile_kvd_linear_size_set(mbox,
+						resources->kvd_linear_size);
+		mlxsw_cmd_mbox_config_profile_set_kvd_hash_single_size_set(mbox,
+									   1);
+		mlxsw_cmd_mbox_config_profile_kvd_hash_single_size_set(mbox,
+						resources->kvd_single_size);
 		mlxsw_cmd_mbox_config_profile_set_kvd_hash_double_size_set(
-			mbox, 1);
-		mlxsw_cmd_mbox_config_profile_kvd_hash_double_size_set(
-			mbox, profile->kvd_hash_double_size);
+								mbox, 1);
+		mlxsw_cmd_mbox_config_profile_kvd_hash_double_size_set(mbox,
+						resources->kvd_double_size);
 	}
 
 	for (i = 0; i < MLXSW_CONFIG_PROFILE_SWID_COUNT; i++)
@@ -1524,7 +1607,7 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
 	if (err)
 		goto err_query_resources;
 
-	err = mlxsw_pci_config_profile(mlxsw_pci, mbox, profile);
+	err = mlxsw_pci_config_profile(mlxsw_pci, mbox, profile, resources);
 	if (err)
 		goto err_config_profile;
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/port.h b/drivers/net/ethernet/mellanox/mlxsw/port.h
index f33b997f2b61..af371a82c35b 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/port.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/port.h
@@ -56,6 +56,7 @@
 #define MLXSW_PORT_PHY_BITS_MASK	(MLXSW_PORT_MAX_PHY_PORTS - 1)
 
 #define MLXSW_PORT_CPU_PORT		0x0
+#define MLXSW_PORT_ROUTER_PORT		(MLXSW_PORT_MAX_PHY_PORTS + 2)
 
 #define MLXSW_PORT_DONT_CARE		(MLXSW_PORT_MAX_PORTS)
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h
index 7ca9201f7dcb..6460c7256f2b 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/reg.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h
@@ -591,6 +591,12 @@ static const struct mlxsw_reg_info mlxsw_reg_sfn = {
  */
 MLXSW_ITEM32(reg, sfn, swid, 0x00, 24, 8);
 
+/* reg_sfn_end
+ * Forces the current session to end.
+ * Access: OP
+ */
+MLXSW_ITEM32(reg, sfn, end, 0x04, 20, 1);
+
 /* reg_sfn_num_rec
  * Request: Number of learned notifications and aged-out notification
  * records requested.
@@ -605,6 +611,7 @@ static inline void mlxsw_reg_sfn_pack(char *payload)
 {
 	MLXSW_REG_ZERO(sfn, payload);
 	mlxsw_reg_sfn_swid_set(payload, 0);
+	mlxsw_reg_sfn_end_set(payload, 1);
 	mlxsw_reg_sfn_num_rec_set(payload, MLXSW_REG_SFN_REC_MAX_COUNT);
 }
 
@@ -1385,7 +1392,7 @@ static inline void mlxsw_reg_slcr_pack(char *payload, u16 lag_hash)
 {
 	MLXSW_REG_ZERO(slcr, payload);
 	mlxsw_reg_slcr_pp_set(payload, MLXSW_REG_SLCR_PP_GLOBAL);
-	mlxsw_reg_slcr_type_set(payload, MLXSW_REG_SLCR_TYPE_XOR);
+	mlxsw_reg_slcr_type_set(payload, MLXSW_REG_SLCR_TYPE_CRC);
 	mlxsw_reg_slcr_lag_hash_set(payload, lag_hash);
 }
 
@@ -2131,6 +2138,18 @@ MLXSW_ITEM32(reg, ptys, local_port, 0x00, 16, 8);
  */
 MLXSW_ITEM32(reg, ptys, proto_mask, 0x00, 0, 3);
 
+enum {
+	MLXSW_REG_PTYS_AN_STATUS_NA,
+	MLXSW_REG_PTYS_AN_STATUS_OK,
+	MLXSW_REG_PTYS_AN_STATUS_FAIL,
+};
+
+/* reg_ptys_an_status
+ * Autonegotiation status.
+ * Access: RO
+ */
+MLXSW_ITEM32(reg, ptys, an_status, 0x04, 28, 4);
+
 #define MLXSW_REG_PTYS_ETH_SPEED_SGMII			BIT(0)
 #define MLXSW_REG_PTYS_ETH_SPEED_1000BASE_KX		BIT(1)
 #define MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CX4		BIT(2)
@@ -2145,6 +2164,7 @@ MLXSW_ITEM32(reg, ptys, proto_mask, 0x00, 0, 3);
 #define MLXSW_REG_PTYS_ETH_SPEED_10GBASE_ER_LR		BIT(14)
 #define MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4		BIT(15)
 #define MLXSW_REG_PTYS_ETH_SPEED_40GBASE_LR4_ER4	BIT(16)
+#define MLXSW_REG_PTYS_ETH_SPEED_50GBASE_SR2		BIT(18)
 #define MLXSW_REG_PTYS_ETH_SPEED_50GBASE_KR4		BIT(19)
 #define MLXSW_REG_PTYS_ETH_SPEED_100GBASE_CR4		BIT(20)
 #define MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4		BIT(21)
@@ -2177,6 +2197,13 @@ MLXSW_ITEM32(reg, ptys, eth_proto_admin, 0x18, 0, 32);
  */
 MLXSW_ITEM32(reg, ptys, eth_proto_oper, 0x24, 0, 32);
 
+/* reg_ptys_eth_proto_lp_advertise
+ * The protocols that were advertised by the link partner during
+ * autonegotiation.
+ * Access: RO
+ */
+MLXSW_ITEM32(reg, ptys, eth_proto_lp_advertise, 0x30, 0, 32);
+
 static inline void mlxsw_reg_ptys_pack(char *payload, u8 local_port,
 				       u32 proto_admin)
 {
@@ -3383,6 +3410,15 @@ MLXSW_ITEM32(reg, ritr, ipv4_fe, 0x04, 29, 1);
  */
 MLXSW_ITEM32(reg, ritr, ipv6_fe, 0x04, 28, 1);
 
+/* reg_ritr_lb_en
+ * Loop-back filter enable for unicast packets.
+ * If the flag is set then loop-back filter for unicast packets is
+ * implemented on the RIF. Multicast packets are always subject to
+ * loop-back filtering.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ritr, lb_en, 0x04, 24, 1);
+
 /* reg_ritr_virtual_router
  * Virtual router ID associated with the router interface.
  * Access: RW
@@ -3484,6 +3520,7 @@ static inline void mlxsw_reg_ritr_pack(char *payload, bool enable,
 	mlxsw_reg_ritr_op_set(payload, op);
 	mlxsw_reg_ritr_rif_set(payload, rif);
 	mlxsw_reg_ritr_ipv4_fe_set(payload, 1);
+	mlxsw_reg_ritr_lb_en_set(payload, 1);
 	mlxsw_reg_ritr_mtu_set(payload, mtu);
 	mlxsw_reg_ritr_if_mac_memcpy_to(payload, mac);
 }
@@ -4000,6 +4037,7 @@ static inline void mlxsw_reg_ralue_pack(char *payload,
 {
 	MLXSW_REG_ZERO(ralue, payload);
 	mlxsw_reg_ralue_protocol_set(payload, protocol);
+	mlxsw_reg_ralue_op_set(payload, op);
 	mlxsw_reg_ralue_virtual_router_set(payload, virtual_router);
 	mlxsw_reg_ralue_prefix_len_set(payload, prefix_len);
 	mlxsw_reg_ralue_entry_type_set(payload,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index c3e61500819d..1ec0a4ce3c46 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -56,6 +56,7 @@
 #include <generated/utsrelease.h>
 #include <net/pkt_cls.h>
 #include <net/tc_act/tc_mirred.h>
+#include <net/netevent.h>
 
 #include "spectrum.h"
 #include "core.h"
@@ -247,7 +248,8 @@ static void mlxsw_sp_span_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 	span_entry->used = false;
 }
 
-struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_find(struct mlxsw_sp_port *port)
+static struct mlxsw_sp_span_entry *
+mlxsw_sp_span_entry_find(struct mlxsw_sp_port *port)
 {
 	struct mlxsw_sp *mlxsw_sp = port->mlxsw_sp;
 	int i;
@@ -261,7 +263,8 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_find(struct mlxsw_sp_port *port)
 	return NULL;
 }
 
-struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port)
+static struct mlxsw_sp_span_entry
+*mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port)
 {
 	struct mlxsw_sp_span_entry *span_entry;
 
@@ -363,7 +366,8 @@ mlxsw_sp_span_inspected_port_bind(struct mlxsw_sp_port *port,
 	}
 
 	/* bind the port to the SPAN entry */
-	mlxsw_reg_mpar_pack(mpar_pl, port->local_port, type, true, pa_id);
+	mlxsw_reg_mpar_pack(mpar_pl, port->local_port,
+			    (enum mlxsw_reg_mpar_i_e) type, true, pa_id);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mpar), mpar_pl);
 	if (err)
 		goto err_mpar_reg_write;
@@ -404,7 +408,8 @@ mlxsw_sp_span_inspected_port_unbind(struct mlxsw_sp_port *port,
 		return;
 
 	/* remove the inspected port */
-	mlxsw_reg_mpar_pack(mpar_pl, port->local_port, type, false, pa_id);
+	mlxsw_reg_mpar_pack(mpar_pl, port->local_port,
+			    (enum mlxsw_reg_mpar_i_e) type, false, pa_id);
 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mpar), mpar_pl);
 
 	/* remove the SBIB buffer if it was egress SPAN */
@@ -555,8 +560,9 @@ int mlxsw_sp_port_vid_to_fid_set(struct mlxsw_sp_port *mlxsw_sp_port,
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(svfa), svfa_pl);
 }
 
-static int mlxsw_sp_port_vid_learning_set(struct mlxsw_sp_port *mlxsw_sp_port,
-					  u16 vid, bool learn_enable)
+int __mlxsw_sp_port_vid_learning_set(struct mlxsw_sp_port *mlxsw_sp_port,
+				     u16 vid_begin, u16 vid_end,
+				     bool learn_enable)
 {
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	char *spvmlr_pl;
@@ -565,13 +571,20 @@ static int mlxsw_sp_port_vid_learning_set(struct mlxsw_sp_port *mlxsw_sp_port,
 	spvmlr_pl = kmalloc(MLXSW_REG_SPVMLR_LEN, GFP_KERNEL);
 	if (!spvmlr_pl)
 		return -ENOMEM;
-	mlxsw_reg_spvmlr_pack(spvmlr_pl, mlxsw_sp_port->local_port, vid, vid,
-			      learn_enable);
+	mlxsw_reg_spvmlr_pack(spvmlr_pl, mlxsw_sp_port->local_port, vid_begin,
+			      vid_end, learn_enable);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(spvmlr), spvmlr_pl);
 	kfree(spvmlr_pl);
 	return err;
 }
 
+static int mlxsw_sp_port_vid_learning_set(struct mlxsw_sp_port *mlxsw_sp_port,
+					  u16 vid, bool learn_enable)
+{
+	return __mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, vid,
+						learn_enable);
+}
+
 static int
 mlxsw_sp_port_system_port_mapping_set(struct mlxsw_sp_port *mlxsw_sp_port)
 {
@@ -810,9 +823,9 @@ err_span_port_mtu_update:
 	return err;
 }
 
-static struct rtnl_link_stats64 *
-mlxsw_sp_port_get_stats64(struct net_device *dev,
-			  struct rtnl_link_stats64 *stats)
+static int
+mlxsw_sp_port_get_sw_stats64(const struct net_device *dev,
+			     struct rtnl_link_stats64 *stats)
 {
 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
 	struct mlxsw_sp_port_pcpu_stats *p;
@@ -839,6 +852,107 @@ mlxsw_sp_port_get_stats64(struct net_device *dev,
 		tx_dropped	+= p->tx_dropped;
 	}
 	stats->tx_dropped	= tx_dropped;
+	return 0;
+}
+
+static bool mlxsw_sp_port_has_offload_stats(int attr_id)
+{
+	switch (attr_id) {
+	case IFLA_OFFLOAD_XSTATS_CPU_HIT:
+		return true;
+	}
+
+	return false;
+}
+
+static int mlxsw_sp_port_get_offload_stats(int attr_id, const struct net_device *dev,
+					   void *sp)
+{
+	switch (attr_id) {
+	case IFLA_OFFLOAD_XSTATS_CPU_HIT:
+		return mlxsw_sp_port_get_sw_stats64(dev, sp);
+	}
+
+	return -EINVAL;
+}
+
+static int mlxsw_sp_port_get_stats_raw(struct net_device *dev, int grp,
+				       int prio, char *ppcnt_pl)
+{
+	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
+	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
+
+	mlxsw_reg_ppcnt_pack(ppcnt_pl, mlxsw_sp_port->local_port, grp, prio);
+	return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ppcnt), ppcnt_pl);
+}
+
+static int mlxsw_sp_port_get_hw_stats(struct net_device *dev,
+				      struct rtnl_link_stats64 *stats)
+{
+	char ppcnt_pl[MLXSW_REG_PPCNT_LEN];
+	int err;
+
+	err = mlxsw_sp_port_get_stats_raw(dev, MLXSW_REG_PPCNT_IEEE_8023_CNT,
+					  0, ppcnt_pl);
+	if (err)
+		goto out;
+
+	stats->tx_packets =
+		mlxsw_reg_ppcnt_a_frames_transmitted_ok_get(ppcnt_pl);
+	stats->rx_packets =
+		mlxsw_reg_ppcnt_a_frames_received_ok_get(ppcnt_pl);
+	stats->tx_bytes =
+		mlxsw_reg_ppcnt_a_octets_transmitted_ok_get(ppcnt_pl);
+	stats->rx_bytes =
+		mlxsw_reg_ppcnt_a_octets_received_ok_get(ppcnt_pl);
+	stats->multicast =
+		mlxsw_reg_ppcnt_a_multicast_frames_received_ok_get(ppcnt_pl);
+
+	stats->rx_crc_errors =
+		mlxsw_reg_ppcnt_a_frame_check_sequence_errors_get(ppcnt_pl);
+	stats->rx_frame_errors =
+		mlxsw_reg_ppcnt_a_alignment_errors_get(ppcnt_pl);
+
+	stats->rx_length_errors = (
+		mlxsw_reg_ppcnt_a_in_range_length_errors_get(ppcnt_pl) +
+		mlxsw_reg_ppcnt_a_out_of_range_length_field_get(ppcnt_pl) +
+		mlxsw_reg_ppcnt_a_frame_too_long_errors_get(ppcnt_pl));
+
+	stats->rx_errors = (stats->rx_crc_errors +
+		stats->rx_frame_errors + stats->rx_length_errors);
+
+out:
+	return err;
+}
+
+static void update_stats_cache(struct work_struct *work)
+{
+	struct mlxsw_sp_port *mlxsw_sp_port =
+		container_of(work, struct mlxsw_sp_port,
+			     hw_stats.update_dw.work);
+
+	if (!netif_carrier_ok(mlxsw_sp_port->dev))
+		goto out;
+
+	mlxsw_sp_port_get_hw_stats(mlxsw_sp_port->dev,
+				   mlxsw_sp_port->hw_stats.cache);
+
+out:
+	mlxsw_core_schedule_dw(&mlxsw_sp_port->hw_stats.update_dw,
+			       MLXSW_HW_STATS_UPDATE_TIME);
+}
+
+/* Return the stats from a cache that is updated periodically,
+ * as this function might get called in an atomic context.
+ */
+static struct rtnl_link_stats64 *
+mlxsw_sp_port_get_stats64(struct net_device *dev,
+			  struct rtnl_link_stats64 *stats)
+{
+	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
+
+	memcpy(stats, mlxsw_sp_port->hw_stats.cache, sizeof(*stats));
+
 	return stats;
 }
 
@@ -942,8 +1056,8 @@ static void mlxsw_sp_port_vport_destroy(struct mlxsw_sp_port *mlxsw_sp_vport)
 	kfree(mlxsw_sp_vport);
 }
 
-int mlxsw_sp_port_add_vid(struct net_device *dev, __be16 __always_unused proto,
-			  u16 vid)
+static int mlxsw_sp_port_add_vid(struct net_device *dev,
+				 __be16 __always_unused proto, u16 vid)
 {
 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
 	struct mlxsw_sp_port *mlxsw_sp_vport;
@@ -956,16 +1070,12 @@ int mlxsw_sp_port_add_vid(struct net_device *dev, __be16 __always_unused proto,
 	if (!vid)
 		return 0;
 
-	if (mlxsw_sp_port_vport_find(mlxsw_sp_port, vid)) {
-		netdev_warn(dev, "VID=%d already configured\n", vid);
+	if (mlxsw_sp_port_vport_find(mlxsw_sp_port, vid))
 		return 0;
-	}
 
 	mlxsw_sp_vport = mlxsw_sp_port_vport_create(mlxsw_sp_port, vid);
-	if (!mlxsw_sp_vport) {
-		netdev_err(dev, "Failed to create vPort for VID=%d\n", vid);
+	if (!mlxsw_sp_vport)
 		return -ENOMEM;
-	}
 
 	/* When adding the first VLAN interface on a bridged port we need to
 	 * transition all the active 802.1Q bridge VLANs to use explicit
@@ -973,30 +1083,17 @@ int mlxsw_sp_port_add_vid(struct net_device *dev, __be16 __always_unused proto,
 	 */
 	if (list_is_singular(&mlxsw_sp_port->vports_list)) {
 		err = mlxsw_sp_port_vp_mode_trans(mlxsw_sp_port);
-		if (err) {
-			netdev_err(dev, "Failed to set to Virtual mode\n");
+		if (err)
 			goto err_port_vp_mode_trans;
-		}
-	}
-
-	err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_vport, vid, false);
-	if (err) {
-		netdev_err(dev, "Failed to disable learning for VID=%d\n", vid);
-		goto err_port_vid_learning_set;
 	}
 
 	err = mlxsw_sp_port_vlan_set(mlxsw_sp_vport, vid, vid, true, untagged);
-	if (err) {
-		netdev_err(dev, "Failed to set VLAN membership for VID=%d\n",
-			   vid);
+	if (err)
 		goto err_port_add_vid;
-	}
 
 	return 0;
 
 err_port_add_vid:
-	mlxsw_sp_port_vid_learning_set(mlxsw_sp_vport, vid, true);
-err_port_vid_learning_set:
 	if (list_is_singular(&mlxsw_sp_port->vports_list))
 		mlxsw_sp_port_vlan_mode_trans(mlxsw_sp_port);
 err_port_vp_mode_trans:
@@ -1010,7 +1107,6 @@ static int mlxsw_sp_port_kill_vid(struct net_device *dev,
 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
 	struct mlxsw_sp_port *mlxsw_sp_vport;
 	struct mlxsw_sp_fid *f;
-	int err;
 
 	/* VLAN 0 is removed from HW filter when device goes down, but
 	 * it is reserved in our case, so simply return.
@@ -1019,23 +1115,10 @@ static int mlxsw_sp_port_kill_vid(struct net_device *dev,
 		return 0;
 
 	mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
-	if (!mlxsw_sp_vport) {
-		netdev_warn(dev, "VID=%d does not exist\n", vid);
+	if (WARN_ON(!mlxsw_sp_vport))
 		return 0;
-	}
 
-	err = mlxsw_sp_port_vlan_set(mlxsw_sp_vport, vid, vid, false, false);
-	if (err) {
-		netdev_err(dev, "Failed to set VLAN membership for VID=%d\n",
-			   vid);
-		return err;
-	}
-
-	err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_vport, vid, true);
-	if (err) {
-		netdev_err(dev, "Failed to enable learning for VID=%d\n", vid);
-		return err;
-	}
+	mlxsw_sp_port_vlan_set(mlxsw_sp_vport, vid, vid, false, false);
 
 	/* Drop FID reference. If this was the last reference the
 	 * resources will be freed.
@@ -1048,13 +1131,8 @@ static int mlxsw_sp_port_kill_vid(struct net_device *dev,
 	 * transition all active 802.1Q bridge VLANs to use VID to FID
 	 * mappings and set port's mode to VLAN mode.
 	 */
-	if (list_is_singular(&mlxsw_sp_port->vports_list)) {
-		err = mlxsw_sp_port_vlan_mode_trans(mlxsw_sp_port);
-		if (err) {
-			netdev_err(dev, "Failed to set to VLAN mode\n");
-			return err;
-		}
-	}
+	if (list_is_singular(&mlxsw_sp_port->vports_list))
+		mlxsw_sp_port_vlan_mode_trans(mlxsw_sp_port);
 
 	mlxsw_sp_port_vport_destroy(mlxsw_sp_vport);
 
@@ -1149,6 +1227,7 @@ static int mlxsw_sp_port_add_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
 					  bool ingress)
 {
 	const struct tc_action *a;
+	LIST_HEAD(actions);
 	int err;
 
 	if (!tc_single_action(cls->exts)) {
@@ -1156,7 +1235,8 @@ static int mlxsw_sp_port_add_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
 		return -ENOTSUPP;
 	}
 
-	tc_for_each_action(a, cls->exts) {
+	tcf_exts_to_list(cls->exts, &actions);
+	list_for_each_entry(a, &actions, list) {
 		if (!is_tcf_mirred_mirror(a) || protocol != htons(ETH_P_ALL))
 			return -ENOTSUPP;
 
@@ -1234,6 +1314,8 @@ static const struct net_device_ops mlxsw_sp_port_netdev_ops = {
 	.ndo_set_mac_address	= mlxsw_sp_port_set_mac_address,
 	.ndo_change_mtu		= mlxsw_sp_port_change_mtu,
 	.ndo_get_stats64	= mlxsw_sp_port_get_stats64,
+	.ndo_has_offload_stats	= mlxsw_sp_port_has_offload_stats,
+	.ndo_get_offload_stats	= mlxsw_sp_port_get_offload_stats,
 	.ndo_vlan_rx_add_vid	= mlxsw_sp_port_add_vid,
 	.ndo_vlan_rx_kill_vid	= mlxsw_sp_port_kill_vid,
 	.ndo_neigh_construct	= mlxsw_sp_router_neigh_construct,
@@ -1572,8 +1654,6 @@ static void __mlxsw_sp_port_get_stats(struct net_device *dev,
 				      enum mlxsw_reg_ppcnt_grp grp, int prio,
 				      u64 *data, int data_index)
 {
-	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	struct mlxsw_sp_port_hw_stats *hw_stats;
 	char ppcnt_pl[MLXSW_REG_PPCNT_LEN];
 	int i, len;
@@ -1582,10 +1662,9 @@ static void __mlxsw_sp_port_get_stats(struct net_device *dev,
 	err = mlxsw_sp_get_hw_stats_by_group(&hw_stats, &len, grp);
 	if (err)
 		return;
-	mlxsw_reg_ppcnt_pack(ppcnt_pl, mlxsw_sp_port->local_port, grp, prio);
-	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ppcnt), ppcnt_pl);
+	mlxsw_sp_port_get_stats_raw(dev, grp, prio, ppcnt_pl);
 	for (i = 0; i < len; i++)
-		data[data_index + i] = !err ? hw_stats[i].getter(ppcnt_pl) : 0;
+		data[data_index + i] = hw_stats[i].getter(ppcnt_pl);
 }
 
 static void mlxsw_sp_port_get_stats(struct net_device *dev,
@@ -1624,112 +1703,149 @@ static int mlxsw_sp_port_get_sset_count(struct net_device *dev, int sset)
 }
 
 struct mlxsw_sp_port_link_mode {
+	enum ethtool_link_mode_bit_indices mask_ethtool;
 	u32 mask;
-	u32 supported;
-	u32 advertised;
 	u32 speed;
 };
 
 static const struct mlxsw_sp_port_link_mode mlxsw_sp_port_link_mode[] = {
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_100BASE_T,
-		.supported	= SUPPORTED_100baseT_Full,
-		.advertised	= ADVERTISED_100baseT_Full,
-		.speed		= 100,
-	},
-	{
-		.mask		= MLXSW_REG_PTYS_ETH_SPEED_100BASE_TX,
-		.speed		= 100,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_100baseT_Full_BIT,
+		.speed		= SPEED_100,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_SGMII |
 				  MLXSW_REG_PTYS_ETH_SPEED_1000BASE_KX,
-		.supported	= SUPPORTED_1000baseKX_Full,
-		.advertised	= ADVERTISED_1000baseKX_Full,
-		.speed		= 1000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_1000baseKX_Full_BIT,
+		.speed		= SPEED_1000,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_10GBASE_T,
-		.supported	= SUPPORTED_10000baseT_Full,
-		.advertised	= ADVERTISED_10000baseT_Full,
-		.speed		= 10000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+		.speed		= SPEED_10000,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CX4 |
 				  MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KX4,
-		.supported	= SUPPORTED_10000baseKX4_Full,
-		.advertised	= ADVERTISED_10000baseKX4_Full,
-		.speed		= 10000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT,
+		.speed		= SPEED_10000,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KR |
 				  MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CR |
 				  MLXSW_REG_PTYS_ETH_SPEED_10GBASE_SR |
 				  MLXSW_REG_PTYS_ETH_SPEED_10GBASE_ER_LR,
-		.supported	= SUPPORTED_10000baseKR_Full,
-		.advertised	= ADVERTISED_10000baseKR_Full,
-		.speed		= 10000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_10000baseKR_Full_BIT,
+		.speed		= SPEED_10000,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_20GBASE_KR2,
-		.supported	= SUPPORTED_20000baseKR2_Full,
-		.advertised	= ADVERTISED_20000baseKR2_Full,
-		.speed		= 20000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT,
+		.speed		= SPEED_20000,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_40GBASE_CR4,
-		.supported	= SUPPORTED_40000baseCR4_Full,
-		.advertised	= ADVERTISED_40000baseCR4_Full,
-		.speed		= 40000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT,
+		.speed		= SPEED_40000,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_40GBASE_KR4,
-		.supported	= SUPPORTED_40000baseKR4_Full,
-		.advertised	= ADVERTISED_40000baseKR4_Full,
-		.speed		= 40000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT,
+		.speed		= SPEED_40000,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4,
-		.supported	= SUPPORTED_40000baseSR4_Full,
-		.advertised	= ADVERTISED_40000baseSR4_Full,
-		.speed		= 40000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT,
+		.speed		= SPEED_40000,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_40GBASE_LR4_ER4,
-		.supported	= SUPPORTED_40000baseLR4_Full,
-		.advertised	= ADVERTISED_40000baseLR4_Full,
-		.speed		= 40000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT,
+		.speed		= SPEED_40000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_25GBASE_CR,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_25000baseCR_Full_BIT,
+		.speed		= SPEED_25000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_25GBASE_KR,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_25000baseKR_Full_BIT,
+		.speed		= SPEED_25000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_25GBASE_SR,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_25000baseSR_Full_BIT,
+		.speed		= SPEED_25000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_25GBASE_SR,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_25000baseSR_Full_BIT,
+		.speed		= SPEED_25000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_50GBASE_CR2,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT,
+		.speed		= SPEED_50000,
 	},
 	{
-		.mask		= MLXSW_REG_PTYS_ETH_SPEED_25GBASE_CR |
-				  MLXSW_REG_PTYS_ETH_SPEED_25GBASE_KR |
-				  MLXSW_REG_PTYS_ETH_SPEED_25GBASE_SR,
-		.speed		= 25000,
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_50GBASE_KR2,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT,
+		.speed		= SPEED_50000,
 	},
 	{
-		.mask		= MLXSW_REG_PTYS_ETH_SPEED_50GBASE_KR4 |
-				  MLXSW_REG_PTYS_ETH_SPEED_50GBASE_CR2 |
-				  MLXSW_REG_PTYS_ETH_SPEED_50GBASE_KR2,
-		.speed		= 50000,
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_50GBASE_SR2,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT,
+		.speed		= SPEED_50000,
 	},
 	{
 		.mask		= MLXSW_REG_PTYS_ETH_SPEED_56GBASE_R4,
-		.supported	= SUPPORTED_56000baseKR4_Full,
-		.advertised	= ADVERTISED_56000baseKR4_Full,
-		.speed		= 56000,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT,
+		.speed		= SPEED_56000,
 	},
 	{
-		.mask		= MLXSW_REG_PTYS_ETH_SPEED_100GBASE_CR4 |
-				  MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4 |
-				  MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4 |
-				  MLXSW_REG_PTYS_ETH_SPEED_100GBASE_LR4_ER4,
-		.speed		= 100000,
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_56GBASE_R4,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT,
+		.speed		= SPEED_56000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_56GBASE_R4,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT,
+		.speed		= SPEED_56000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_56GBASE_R4,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT,
+		.speed		= SPEED_56000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_100GBASE_CR4,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT,
+		.speed		= SPEED_100000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT,
+		.speed		= SPEED_100000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT,
+		.speed		= SPEED_100000,
+	},
+	{
+		.mask		= MLXSW_REG_PTYS_ETH_SPEED_100GBASE_LR4_ER4,
+		.mask_ethtool	= ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT,
+		.speed		= SPEED_100000,
 	},
 };
 
 #define MLXSW_SP_PORT_LINK_MODE_LEN ARRAY_SIZE(mlxsw_sp_port_link_mode)
 
-static u32 mlxsw_sp_from_ptys_supported_port(u32 ptys_eth_proto)
+static void
+mlxsw_sp_from_ptys_supported_port(u32 ptys_eth_proto,
+				  struct ethtool_link_ksettings *cmd)
 {
 	if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CR |
 			      MLXSW_REG_PTYS_ETH_SPEED_10GBASE_SR |
@@ -1737,43 +1853,29 @@ static u32 mlxsw_sp_from_ptys_supported_port(u32 ptys_eth_proto)
 			      MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4 |
 			      MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4 |
 			      MLXSW_REG_PTYS_ETH_SPEED_SGMII))
-		return SUPPORTED_FIBRE;
+		ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE);
 
 	if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KR |
 			      MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KX4 |
 			      MLXSW_REG_PTYS_ETH_SPEED_40GBASE_KR4 |
 			      MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4 |
 			      MLXSW_REG_PTYS_ETH_SPEED_1000BASE_KX))
-		return SUPPORTED_Backplane;
-	return 0;
-}
-
-static u32 mlxsw_sp_from_ptys_supported_link(u32 ptys_eth_proto)
-{
-	u32 modes = 0;
-	int i;
-
-	for (i = 0; i < MLXSW_SP_PORT_LINK_MODE_LEN; i++) {
-		if (ptys_eth_proto & mlxsw_sp_port_link_mode[i].mask)
-			modes |= mlxsw_sp_port_link_mode[i].supported;
-	}
-	return modes;
+		ethtool_link_ksettings_add_link_mode(cmd, supported, Backplane);
 }
 
-static u32 mlxsw_sp_from_ptys_advert_link(u32 ptys_eth_proto)
+static void mlxsw_sp_from_ptys_link(u32 ptys_eth_proto, unsigned long *mode)
 {
-	u32 modes = 0;
 	int i;
 
 	for (i = 0; i < MLXSW_SP_PORT_LINK_MODE_LEN; i++) {
 		if (ptys_eth_proto & mlxsw_sp_port_link_mode[i].mask)
-			modes |= mlxsw_sp_port_link_mode[i].advertised;
+			__set_bit(mlxsw_sp_port_link_mode[i].mask_ethtool,
+				  mode);
 	}
-	return modes;
 }
 
 static void mlxsw_sp_from_ptys_speed_duplex(bool carrier_ok, u32 ptys_eth_proto,
-					    struct ethtool_cmd *cmd)
+					    struct ethtool_link_ksettings *cmd)
 {
 	u32 speed = SPEED_UNKNOWN;
 	u8 duplex = DUPLEX_UNKNOWN;
@@ -1790,8 +1892,8 @@ static void mlxsw_sp_from_ptys_speed_duplex(bool carrier_ok, u32 ptys_eth_proto,
 		}
 	}
 out:
-	ethtool_cmd_speed_set(cmd, speed);
-	cmd->duplex = duplex;
+	cmd->base.speed = speed;
+	cmd->base.duplex = duplex;
 }
 
 static u8 mlxsw_sp_port_connector_port(u32 ptys_eth_proto)
@@ -1816,49 +1918,15 @@ static u8 mlxsw_sp_port_connector_port(u32 ptys_eth_proto)
 	return PORT_OTHER;
 }
 
-static int mlxsw_sp_port_get_settings(struct net_device *dev,
-				      struct ethtool_cmd *cmd)
-{
-	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-	char ptys_pl[MLXSW_REG_PTYS_LEN];
-	u32 eth_proto_cap;
-	u32 eth_proto_admin;
-	u32 eth_proto_oper;
-	int err;
-
-	mlxsw_reg_ptys_pack(ptys_pl, mlxsw_sp_port->local_port, 0);
-	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl);
-	if (err) {
-		netdev_err(dev, "Failed to get proto");
-		return err;
-	}
-	mlxsw_reg_ptys_unpack(ptys_pl, &eth_proto_cap,
-			      &eth_proto_admin, &eth_proto_oper);
-
-	cmd->supported = mlxsw_sp_from_ptys_supported_port(eth_proto_cap) |
-			 mlxsw_sp_from_ptys_supported_link(eth_proto_cap) |
-			 SUPPORTED_Pause | SUPPORTED_Asym_Pause |
-			 SUPPORTED_Autoneg;
-	cmd->advertising = mlxsw_sp_from_ptys_advert_link(eth_proto_admin);
-	mlxsw_sp_from_ptys_speed_duplex(netif_carrier_ok(dev),
-					eth_proto_oper, cmd);
-
-	eth_proto_oper = eth_proto_oper ? eth_proto_oper : eth_proto_cap;
-	cmd->port = mlxsw_sp_port_connector_port(eth_proto_oper);
-	cmd->lp_advertising = mlxsw_sp_from_ptys_advert_link(eth_proto_oper);
-
-	cmd->transceiver = XCVR_INTERNAL;
-	return 0;
-}
-
-static u32 mlxsw_sp_to_ptys_advert_link(u32 advertising)
+static u32
+mlxsw_sp_to_ptys_advert_link(const struct ethtool_link_ksettings *cmd)
 {
 	u32 ptys_proto = 0;
 	int i;
 
 	for (i = 0; i < MLXSW_SP_PORT_LINK_MODE_LEN; i++) {
-		if (advertising & mlxsw_sp_port_link_mode[i].advertised)
+		if (test_bit(mlxsw_sp_port_link_mode[i].mask_ethtool,
+			     cmd->link_modes.advertising))
 			ptys_proto |= mlxsw_sp_port_link_mode[i].mask;
 	}
 	return ptys_proto;
@@ -1888,61 +1956,113 @@ static u32 mlxsw_sp_to_ptys_upper_speed(u32 upper_speed)
 	return ptys_proto;
 }
 
-static int mlxsw_sp_port_set_settings(struct net_device *dev,
-				      struct ethtool_cmd *cmd)
+static void mlxsw_sp_port_get_link_supported(u32 eth_proto_cap,
+					     struct ethtool_link_ksettings *cmd)
 {
+	ethtool_link_ksettings_add_link_mode(cmd, supported, Asym_Pause);
+	ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg);
+	ethtool_link_ksettings_add_link_mode(cmd, supported, Pause);
+
+	mlxsw_sp_from_ptys_supported_port(eth_proto_cap, cmd);
+	mlxsw_sp_from_ptys_link(eth_proto_cap, cmd->link_modes.supported);
+}
+
+static void mlxsw_sp_port_get_link_advertise(u32 eth_proto_admin, bool autoneg,
+					     struct ethtool_link_ksettings *cmd)
+{
+	if (!autoneg)
+		return;
+
+	ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg);
+	mlxsw_sp_from_ptys_link(eth_proto_admin, cmd->link_modes.advertising);
+}
+
+static void
+mlxsw_sp_port_get_link_lp_advertise(u32 eth_proto_lp, u8 autoneg_status,
+				    struct ethtool_link_ksettings *cmd)
+{
+	if (autoneg_status != MLXSW_REG_PTYS_AN_STATUS_OK || !eth_proto_lp)
+		return;
+
+	ethtool_link_ksettings_add_link_mode(cmd, lp_advertising, Autoneg);
+	mlxsw_sp_from_ptys_link(eth_proto_lp, cmd->link_modes.lp_advertising);
+}
+
+static int mlxsw_sp_port_get_link_ksettings(struct net_device *dev,
+					    struct ethtool_link_ksettings *cmd)
+{
+	u32 eth_proto_cap, eth_proto_admin, eth_proto_oper, eth_proto_lp;
 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	char ptys_pl[MLXSW_REG_PTYS_LEN];
-	u32 speed;
-	u32 eth_proto_new;
-	u32 eth_proto_cap;
-	u32 eth_proto_admin;
+	u8 autoneg_status;
+	bool autoneg;
 	int err;
 
-	speed = ethtool_cmd_speed(cmd);
+	autoneg = mlxsw_sp_port->link.autoneg;
+	mlxsw_reg_ptys_pack(ptys_pl, mlxsw_sp_port->local_port, 0);
+	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl);
+	if (err)
+		return err;
+	mlxsw_reg_ptys_unpack(ptys_pl, &eth_proto_cap, &eth_proto_admin,
+			      &eth_proto_oper);
+
+	mlxsw_sp_port_get_link_supported(eth_proto_cap, cmd);
+
+	mlxsw_sp_port_get_link_advertise(eth_proto_admin, autoneg, cmd);
 
-	eth_proto_new = cmd->autoneg == AUTONEG_ENABLE ?
-		mlxsw_sp_to_ptys_advert_link(cmd->advertising) :
-		mlxsw_sp_to_ptys_speed(speed);
+	eth_proto_lp = mlxsw_reg_ptys_eth_proto_lp_advertise_get(ptys_pl);
+	autoneg_status = mlxsw_reg_ptys_an_status_get(ptys_pl);
+	mlxsw_sp_port_get_link_lp_advertise(eth_proto_lp, autoneg_status, cmd);
+
+	cmd->base.autoneg = autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE;
+	cmd->base.port = mlxsw_sp_port_connector_port(eth_proto_oper);
+	mlxsw_sp_from_ptys_speed_duplex(netif_carrier_ok(dev), eth_proto_oper,
+					cmd);
+
+	return 0;
+}
+
+static int
+mlxsw_sp_port_set_link_ksettings(struct net_device *dev,
+				 const struct ethtool_link_ksettings *cmd)
+{
+	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
+	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
+	char ptys_pl[MLXSW_REG_PTYS_LEN];
+	u32 eth_proto_cap, eth_proto_new;
+	bool autoneg;
+	int err;
 
 	mlxsw_reg_ptys_pack(ptys_pl, mlxsw_sp_port->local_port, 0);
 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl);
-	if (err) {
-		netdev_err(dev, "Failed to get proto");
+	if (err)
 		return err;
-	}
-	mlxsw_reg_ptys_unpack(ptys_pl, &eth_proto_cap, &eth_proto_admin, NULL);
+	mlxsw_reg_ptys_unpack(ptys_pl, &eth_proto_cap, NULL, NULL);
+
+	autoneg = cmd->base.autoneg == AUTONEG_ENABLE;
+	eth_proto_new = autoneg ?
+		mlxsw_sp_to_ptys_advert_link(cmd) :
+		mlxsw_sp_to_ptys_speed(cmd->base.speed);
 
 	eth_proto_new = eth_proto_new & eth_proto_cap;
 	if (!eth_proto_new) {
-		netdev_err(dev, "Not supported proto admin requested");
+		netdev_err(dev, "No supported speed requested\n");
 		return -EINVAL;
 	}
-	if (eth_proto_new == eth_proto_admin)
-		return 0;
 
 	mlxsw_reg_ptys_pack(ptys_pl, mlxsw_sp_port->local_port, eth_proto_new);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptys), ptys_pl);
-	if (err) {
-		netdev_err(dev, "Failed to set proto admin");
+	if (err)
 		return err;
-	}
 
 	if (!netif_running(dev))
 		return 0;
 
-	err = mlxsw_sp_port_admin_status_set(mlxsw_sp_port, false);
-	if (err) {
-		netdev_err(dev, "Failed to set admin status");
-		return err;
-	}
+	mlxsw_sp_port->link.autoneg = autoneg;
 
-	err = mlxsw_sp_port_admin_status_set(mlxsw_sp_port, true);
-	if (err) {
-		netdev_err(dev, "Failed to set admin status");
-		return err;
-	}
+	mlxsw_sp_port_admin_status_set(mlxsw_sp_port, false);
+	mlxsw_sp_port_admin_status_set(mlxsw_sp_port, true);
 
 	return 0;
 }
@@ -1956,8 +2076,8 @@ static const struct ethtool_ops mlxsw_sp_port_ethtool_ops = {
 	.set_phys_id		= mlxsw_sp_port_set_phys_id,
 	.get_ethtool_stats	= mlxsw_sp_port_get_stats,
 	.get_sset_count		= mlxsw_sp_port_get_sset_count,
-	.get_settings		= mlxsw_sp_port_get_settings,
-	.set_settings		= mlxsw_sp_port_set_settings,
+	.get_link_ksettings	= mlxsw_sp_port_get_link_ksettings,
+	.set_link_ksettings	= mlxsw_sp_port_set_link_ksettings,
 };
 
 static int
@@ -2076,6 +2196,18 @@ static int mlxsw_sp_port_ets_init(struct mlxsw_sp_port *mlxsw_sp_port)
 	return 0;
 }
 
+static int mlxsw_sp_port_pvid_vport_create(struct mlxsw_sp_port *mlxsw_sp_port)
+{
+	mlxsw_sp_port->pvid = 1;
+
+	return mlxsw_sp_port_add_vid(mlxsw_sp_port->dev, 0, 1);
+}
+
+static int mlxsw_sp_port_pvid_vport_destroy(struct mlxsw_sp_port *mlxsw_sp_port)
+{
+	return mlxsw_sp_port_kill_vid(mlxsw_sp_port->dev, 0, 1);
+}
+
 static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 				bool split, u8 module, u8 width, u8 lane)
 {
@@ -2095,6 +2227,7 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 	mlxsw_sp_port->mapping.module = module;
 	mlxsw_sp_port->mapping.width = width;
 	mlxsw_sp_port->mapping.lane = lane;
+	mlxsw_sp_port->link.autoneg = 1;
 	bytes = DIV_ROUND_UP(VLAN_N_VID, BITS_PER_BYTE);
 	mlxsw_sp_port->active_vlans = kzalloc(bytes, GFP_KERNEL);
 	if (!mlxsw_sp_port->active_vlans) {
@@ -2116,9 +2249,26 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 		goto err_alloc_stats;
 	}
 
+	mlxsw_sp_port->hw_stats.cache =
+		kzalloc(sizeof(*mlxsw_sp_port->hw_stats.cache), GFP_KERNEL);
+
+	if (!mlxsw_sp_port->hw_stats.cache) {
+		err = -ENOMEM;
+		goto err_alloc_hw_stats;
+	}
+	INIT_DELAYED_WORK(&mlxsw_sp_port->hw_stats.update_dw,
+			  &update_stats_cache);
+
 	dev->netdev_ops = &mlxsw_sp_port_netdev_ops;
 	dev->ethtool_ops = &mlxsw_sp_port_ethtool_ops;
 
+	err = mlxsw_sp_port_swid_set(mlxsw_sp_port, 0);
+	if (err) {
+		dev_err(mlxsw_sp->bus_info->dev, "Port %d: Failed to set SWID\n",
+			mlxsw_sp_port->local_port);
+		goto err_port_swid_set;
+	}
+
 	err = mlxsw_sp_port_dev_addr_init(mlxsw_sp_port);
 	if (err) {
 		dev_err(mlxsw_sp->bus_info->dev, "Port %d: Unable to init port mac address\n",
@@ -2135,7 +2285,7 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 	/* Each packet needs to have a Tx header (metadata) on top all other
 	 * headers.
 	 */
-	dev->hard_header_len += MLXSW_TXHDR_LEN;
+	dev->needed_headroom = MLXSW_TXHDR_LEN;
 
 	err = mlxsw_sp_port_system_port_mapping_set(mlxsw_sp_port);
 	if (err) {
@@ -2144,13 +2294,6 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 		goto err_port_system_port_mapping_set;
 	}
 
-	err = mlxsw_sp_port_swid_set(mlxsw_sp_port, 0);
-	if (err) {
-		dev_err(mlxsw_sp->bus_info->dev, "Port %d: Failed to set SWID\n",
-			mlxsw_sp_port->local_port);
-		goto err_port_swid_set;
-	}
-
 	err = mlxsw_sp_port_speed_by_width_set(mlxsw_sp_port, width);
 	if (err) {
 		dev_err(mlxsw_sp->bus_info->dev, "Port %d: Failed to enable speeds\n",
@@ -2191,7 +2334,15 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 		goto err_port_dcb_init;
 	}
 
+	err = mlxsw_sp_port_pvid_vport_create(mlxsw_sp_port);
+	if (err) {
+		dev_err(mlxsw_sp->bus_info->dev, "Port %d: Failed to create PVID vPort\n",
+			mlxsw_sp_port->local_port);
+		goto err_port_pvid_vport_create;
+	}
+
 	mlxsw_sp_port_switchdev_init(mlxsw_sp_port);
+	mlxsw_sp->ports[local_port] = mlxsw_sp_port;
 	err = register_netdev(dev);
 	if (err) {
 		dev_err(mlxsw_sp->bus_info->dev, "Port %d: Failed to register netdev\n",
@@ -2208,27 +2359,29 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 		goto err_core_port_init;
 	}
 
-	err = mlxsw_sp_port_vlan_init(mlxsw_sp_port);
-	if (err)
-		goto err_port_vlan_init;
-
-	mlxsw_sp->ports[local_port] = mlxsw_sp_port;
+	mlxsw_core_schedule_dw(&mlxsw_sp_port->hw_stats.update_dw, 0);
 	return 0;
 
-err_port_vlan_init:
-	mlxsw_core_port_fini(&mlxsw_sp_port->core_port);
 err_core_port_init:
 	unregister_netdev(dev);
 err_register_netdev:
+	mlxsw_sp->ports[local_port] = NULL;
+	mlxsw_sp_port_switchdev_fini(mlxsw_sp_port);
+	mlxsw_sp_port_pvid_vport_destroy(mlxsw_sp_port);
+err_port_pvid_vport_create:
+	mlxsw_sp_port_dcb_fini(mlxsw_sp_port);
 err_port_dcb_init:
 err_port_ets_init:
 err_port_buffers_init:
 err_port_admin_status_set:
 err_port_mtu_set:
 err_port_speed_by_width_set:
-err_port_swid_set:
 err_port_system_port_mapping_set:
 err_dev_addr_init:
+	mlxsw_sp_port_swid_set(mlxsw_sp_port, MLXSW_PORT_SWID_DISABLED_PORT);
+err_port_swid_set:
+	kfree(mlxsw_sp_port->hw_stats.cache);
+err_alloc_hw_stats:
 	free_percpu(mlxsw_sp_port->pcpu_stats);
 err_alloc_stats:
 	kfree(mlxsw_sp_port->untagged_vlans);
@@ -2245,15 +2398,17 @@ static void mlxsw_sp_port_remove(struct mlxsw_sp *mlxsw_sp, u8 local_port)
 
 	if (!mlxsw_sp_port)
 		return;
-	mlxsw_sp->ports[local_port] = NULL;
+	cancel_delayed_work_sync(&mlxsw_sp_port->hw_stats.update_dw);
 	mlxsw_core_port_fini(&mlxsw_sp_port->core_port);
 	unregister_netdev(mlxsw_sp_port->dev); /* This calls ndo_stop */
-	mlxsw_sp_port_dcb_fini(mlxsw_sp_port);
-	mlxsw_sp_port_kill_vid(mlxsw_sp_port->dev, 0, 1);
+	mlxsw_sp->ports[local_port] = NULL;
 	mlxsw_sp_port_switchdev_fini(mlxsw_sp_port);
+	mlxsw_sp_port_pvid_vport_destroy(mlxsw_sp_port);
+	mlxsw_sp_port_dcb_fini(mlxsw_sp_port);
 	mlxsw_sp_port_swid_set(mlxsw_sp_port, MLXSW_PORT_SWID_DISABLED_PORT);
 	mlxsw_sp_port_module_unmap(mlxsw_sp, mlxsw_sp_port->local_port);
 	free_percpu(mlxsw_sp_port->pcpu_stats);
+	kfree(mlxsw_sp_port->hw_stats.cache);
 	kfree(mlxsw_sp_port->untagged_vlans);
 	kfree(mlxsw_sp_port->active_vlans);
 	WARN_ON_ONCE(!list_empty(&mlxsw_sp_port->vports_list));
@@ -2577,103 +2732,47 @@ static void mlxsw_sp_rx_listener_func(struct sk_buff *skb, u8 local_port,
 	netif_receive_skb(skb);
 }
 
+static void mlxsw_sp_rx_listener_mark_func(struct sk_buff *skb, u8 local_port,
+					   void *priv)
+{
+	skb->offload_fwd_mark = 1;
+	return mlxsw_sp_rx_listener_func(skb, local_port, priv);
+}
+
+#define MLXSW_SP_RXL(_func, _trap_id, _action)			\
+	{							\
+		.func = _func,					\
+		.local_port = MLXSW_PORT_DONT_CARE,		\
+		.trap_id = MLXSW_TRAP_ID_##_trap_id,		\
+		.action = MLXSW_REG_HPKT_ACTION_##_action,	\
+	}
+
 static const struct mlxsw_rx_listener mlxsw_sp_rx_listener[] = {
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_FDB_MC,
-	},
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, FDB_MC, TRAP_TO_CPU),
 	/* Traps for specific L2 packet types, not trapped as FDB MC */
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_STP,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_LACP,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_EAPOL,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_LLDP,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_MMRP,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_MVRP,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_RPVST,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_DHCP,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_IGMP_QUERY,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_IGMP_V1_REPORT,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_IGMP_V2_REPORT,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_IGMP_V2_LEAVE,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_IGMP_V3_REPORT,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_ARPBC,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_ARPUC,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_IP2ME,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_RTR_INGRESS0,
-	},
-	{
-		.func = mlxsw_sp_rx_listener_func,
-		.local_port = MLXSW_PORT_DONT_CARE,
-		.trap_id = MLXSW_TRAP_ID_HOST_MISS_IPV4,
-	},
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, STP, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, LACP, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, EAPOL, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, LLDP, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, MMRP, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, MVRP, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, RPVST, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_mark_func, DHCP, MIRROR_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_mark_func, IGMP_QUERY, MIRROR_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, IGMP_V1_REPORT, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, IGMP_V2_REPORT, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, IGMP_V2_LEAVE, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, IGMP_V3_REPORT, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_mark_func, ARPBC, MIRROR_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_mark_func, ARPUC, MIRROR_TO_CPU),
+	/* L3 traps */
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, MTUERROR, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, TTLERROR, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, LBERROR, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_mark_func, OSPF, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, IP2ME, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, RTR_INGRESS0, TRAP_TO_CPU),
+	MLXSW_SP_RXL(mlxsw_sp_rx_listener_func, HOST_MISS_IPV4, TRAP_TO_CPU),
 };
 
 static int mlxsw_sp_traps_init(struct mlxsw_sp *mlxsw_sp)
@@ -2700,7 +2799,7 @@ static int mlxsw_sp_traps_init(struct mlxsw_sp *mlxsw_sp)
 		if (err)
 			goto err_rx_listener_register;
 
-		mlxsw_reg_hpkt_pack(hpkt_pl, MLXSW_REG_HPKT_ACTION_TRAP_TO_CPU,
+		mlxsw_reg_hpkt_pack(hpkt_pl, mlxsw_sp_rx_listener[i].action,
 				    mlxsw_sp_rx_listener[i].trap_id);
 		err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(hpkt), hpkt_pl);
 		if (err)
@@ -2788,7 +2887,9 @@ static int mlxsw_sp_flood_init(struct mlxsw_sp *mlxsw_sp)
 
 static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp)
 {
+	struct mlxsw_resources *resources;
 	char slcr_pl[MLXSW_REG_SLCR_LEN];
+	int err;
 
 	mlxsw_reg_slcr_pack(slcr_pl, MLXSW_REG_SLCR_LAG_HASH_SMAC |
 				     MLXSW_REG_SLCR_LAG_HASH_DMAC |
@@ -2799,7 +2900,26 @@ static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp)
 				     MLXSW_REG_SLCR_LAG_HASH_SPORT |
 				     MLXSW_REG_SLCR_LAG_HASH_DPORT |
 				     MLXSW_REG_SLCR_LAG_HASH_IPPROTO);
-	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(slcr), slcr_pl);
+	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(slcr), slcr_pl);
+	if (err)
+		return err;
+
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	if (!(resources->max_lag_valid && resources->max_ports_in_lag_valid))
+		return -EIO;
+
+	mlxsw_sp->lags = kcalloc(resources->max_lag,
+				 sizeof(struct mlxsw_sp_upper),
+				 GFP_KERNEL);
+	if (!mlxsw_sp->lags)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void mlxsw_sp_lag_fini(struct mlxsw_sp *mlxsw_sp)
+{
+	kfree(mlxsw_sp->lags);
 }
 
 static int mlxsw_sp_init(struct mlxsw_core *mlxsw_core,
@@ -2883,6 +3003,7 @@ err_span_init:
 err_router_init:
 	mlxsw_sp_switchdev_fini(mlxsw_sp);
 err_switchdev_init:
+	mlxsw_sp_lag_fini(mlxsw_sp);
 err_lag_init:
 	mlxsw_sp_buffers_fini(mlxsw_sp);
 err_buffers_init:
@@ -2896,38 +3017,26 @@ err_rx_listener_register:
 static void mlxsw_sp_fini(struct mlxsw_core *mlxsw_core)
 {
 	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
-	int i;
 
 	mlxsw_sp_ports_remove(mlxsw_sp);
 	mlxsw_sp_span_fini(mlxsw_sp);
 	mlxsw_sp_router_fini(mlxsw_sp);
 	mlxsw_sp_switchdev_fini(mlxsw_sp);
+	mlxsw_sp_lag_fini(mlxsw_sp);
 	mlxsw_sp_buffers_fini(mlxsw_sp);
 	mlxsw_sp_traps_fini(mlxsw_sp);
 	mlxsw_sp_event_unregister(mlxsw_sp, MLXSW_TRAP_ID_PUDE);
 	WARN_ON(!list_empty(&mlxsw_sp->vfids.list));
 	WARN_ON(!list_empty(&mlxsw_sp->fids));
-	for (i = 0; i < MLXSW_SP_RIF_MAX; i++)
-		WARN_ON_ONCE(mlxsw_sp->rifs[i]);
 }
 
 static struct mlxsw_config_profile mlxsw_sp_config_profile = {
 	.used_max_vepa_channels		= 1,
 	.max_vepa_channels		= 0,
-	.used_max_lag			= 1,
-	.max_lag			= MLXSW_SP_LAG_MAX,
-	.used_max_port_per_lag		= 1,
-	.max_port_per_lag		= MLXSW_SP_PORT_PER_LAG_MAX,
 	.used_max_mid			= 1,
 	.max_mid			= MLXSW_SP_MID_MAX,
 	.used_max_pgt			= 1,
 	.max_pgt			= 0,
-	.used_max_system_port		= 1,
-	.max_system_port		= 64,
-	.used_max_vlan_groups		= 1,
-	.max_vlan_groups		= 127,
-	.used_max_regions		= 1,
-	.max_regions			= 400,
 	.used_flood_tables		= 1,
 	.used_flood_mode		= 1,
 	.flood_mode			= 3,
@@ -2939,10 +3048,11 @@ static struct mlxsw_config_profile mlxsw_sp_config_profile = {
 	.max_ib_mc			= 0,
 	.used_max_pkey			= 1,
 	.max_pkey			= 0,
-	.used_kvd_sizes			= 1,
+	.used_kvd_split_data		= 1,
+	.kvd_hash_granularity		= MLXSW_SP_KVD_GRANULARITY,
+	.kvd_hash_single_parts		= 2,
+	.kvd_hash_double_parts		= 1,
 	.kvd_linear_size		= MLXSW_SP_KVD_LINEAR_SIZE,
-	.kvd_hash_single_size		= MLXSW_SP_KVD_HASH_SINGLE_SIZE,
-	.kvd_hash_double_size		= MLXSW_SP_KVD_HASH_DOUBLE_SIZE,
 	.swid_config			= {
 		{
 			.used_type	= 1,
@@ -3059,13 +3169,15 @@ static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *r,
 
 static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp)
 {
+	struct mlxsw_resources *resources;
 	int i;
 
-	for (i = 0; i < MLXSW_SP_RIF_MAX; i++)
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	for (i = 0; i < resources->max_rif; i++)
 		if (!mlxsw_sp->rifs[i])
 			return i;
 
-	return MLXSW_SP_RIF_MAX;
+	return MLXSW_SP_INVALID_RIF;
 }
 
 static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport,
@@ -3145,7 +3257,7 @@ mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport,
 	int err;
 
 	rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
-	if (rif == MLXSW_SP_RIF_MAX)
+	if (rif == MLXSW_SP_INVALID_RIF)
 		return ERR_PTR(-ERANGE);
 
 	err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, l3_dev, rif, true);
@@ -3311,6 +3423,39 @@ static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp,
 	return mlxsw_sp_fid_find(mlxsw_sp, fid);
 }
 
+static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid)
+{
+	return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID :
+	       MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST;
+}
+
+static u16 mlxsw_sp_flood_table_index_get(u16 fid)
+{
+	return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid;
+}
+
+static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid,
+					  bool set)
+{
+	enum mlxsw_flood_table_type table_type;
+	char *sftr_pl;
+	u16 index;
+	int err;
+
+	sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL);
+	if (!sftr_pl)
+		return -ENOMEM;
+
+	table_type = mlxsw_sp_flood_table_type_get(fid);
+	index = mlxsw_sp_flood_table_index_get(fid);
+	mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BM, index, table_type,
+			    1, MLXSW_PORT_ROUTER_PORT, set);
+	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
+
+	kfree(sftr_pl);
+	return err;
+}
+
 static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid)
 {
 	if (mlxsw_sp_fid_is_vfid(fid))
@@ -3344,13 +3489,17 @@ static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp,
 	int err;
 
 	rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
-	if (rif == MLXSW_SP_RIF_MAX)
+	if (rif == MLXSW_SP_INVALID_RIF)
 		return -ERANGE;
 
-	err = mlxsw_sp_rif_bridge_op(mlxsw_sp, l3_dev, f->fid, rif, true);
+	err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true);
 	if (err)
 		return err;
 
+	err = mlxsw_sp_rif_bridge_op(mlxsw_sp, l3_dev, f->fid, rif, true);
+	if (err)
+		goto err_rif_bridge_op;
+
 	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true);
 	if (err)
 		goto err_rif_fdb_op;
@@ -3372,6 +3521,8 @@ err_rif_alloc:
 	mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
 err_rif_fdb_op:
 	mlxsw_sp_rif_bridge_op(mlxsw_sp, l3_dev, f->fid, rif, false);
+err_rif_bridge_op:
+	mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
 	return err;
 }
 
@@ -3391,6 +3542,8 @@ void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
 
 	mlxsw_sp_rif_bridge_op(mlxsw_sp, l3_dev, f->fid, rif, false);
 
+	mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
+
 	netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif);
 }
 
@@ -3543,12 +3696,14 @@ static bool mlxsw_sp_port_fdb_should_flush(struct mlxsw_sp_port *mlxsw_sp_port,
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	u8 local_port = mlxsw_sp_port->local_port;
 	u16 lag_id = mlxsw_sp_port->lag_id;
+	struct mlxsw_resources *resources;
 	int i, count = 0;
 
 	if (!mlxsw_sp_port->lagged)
 		return true;
 
-	for (i = 0; i < MLXSW_SP_PORT_PER_LAG_MAX; i++) {
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	for (i = 0; i < resources->max_ports_in_lag; i++) {
 		struct mlxsw_sp_port *lag_port;
 
 		lag_port = mlxsw_sp_port_lagged_get(mlxsw_sp, lag_id, i);
@@ -3754,11 +3909,13 @@ static int mlxsw_sp_lag_index_get(struct mlxsw_sp *mlxsw_sp,
 				  struct net_device *lag_dev,
 				  u16 *p_lag_id)
 {
+	struct mlxsw_resources *resources;
 	struct mlxsw_sp_upper *lag;
 	int free_lag_id = -1;
 	int i;
 
-	for (i = 0; i < MLXSW_SP_LAG_MAX; i++) {
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	for (i = 0; i < resources->max_lag; i++) {
 		lag = mlxsw_sp_lag_get(mlxsw_sp, i);
 		if (lag->ref_count) {
 			if (lag->dev == lag_dev) {
@@ -3792,9 +3949,11 @@ mlxsw_sp_master_lag_check(struct mlxsw_sp *mlxsw_sp,
 static int mlxsw_sp_port_lag_index_get(struct mlxsw_sp *mlxsw_sp,
 				       u16 lag_id, u8 *p_port_index)
 {
+	struct mlxsw_resources *resources;
 	int i;
 
-	for (i = 0; i < MLXSW_SP_PORT_PER_LAG_MAX; i++) {
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	for (i = 0; i < resources->max_ports_in_lag; i++) {
 		if (!mlxsw_sp_port_lagged_get(mlxsw_sp, lag_id, i)) {
 			*p_port_index = i;
 			return 0;
@@ -4487,18 +4646,26 @@ static struct notifier_block mlxsw_sp_inetaddr_nb __read_mostly = {
 	.priority = 10,	/* Must be called before FIB notifier block */
 };
 
+static struct notifier_block mlxsw_sp_router_netevent_nb __read_mostly = {
+	.notifier_call = mlxsw_sp_router_netevent_event,
+};
+
 static int __init mlxsw_sp_module_init(void)
 {
 	int err;
 
 	register_netdevice_notifier(&mlxsw_sp_netdevice_nb);
 	register_inetaddr_notifier(&mlxsw_sp_inetaddr_nb);
+	register_netevent_notifier(&mlxsw_sp_router_netevent_nb);
+
 	err = mlxsw_core_driver_register(&mlxsw_sp_driver);
 	if (err)
 		goto err_core_driver_register;
 	return 0;
 
 err_core_driver_register:
+	unregister_netevent_notifier(&mlxsw_sp_router_netevent_nb);
+	unregister_inetaddr_notifier(&mlxsw_sp_inetaddr_nb);
 	unregister_netdevice_notifier(&mlxsw_sp_netdevice_nb);
 	return err;
 }
@@ -4506,6 +4673,7 @@ err_core_driver_register:
 static void __exit mlxsw_sp_module_exit(void)
 {
 	mlxsw_core_driver_unregister(&mlxsw_sp_driver);
+	unregister_netevent_notifier(&mlxsw_sp_router_netevent_nb);
 	unregister_inetaddr_notifier(&mlxsw_sp_inetaddr_nb);
 	unregister_netdevice_notifier(&mlxsw_sp_netdevice_nb);
 }
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
index f69aa37d1521..9b22863a924b 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
@@ -45,7 +45,7 @@
 #include <linux/list.h>
 #include <linux/dcbnl.h>
 #include <linux/in6.h>
-#include <net/switchdev.h>
+#include <linux/notifier.h>
 
 #include "port.h"
 #include "core.h"
@@ -54,10 +54,7 @@
 #define MLXSW_SP_VFID_MAX 6656	/* Bridged VLAN interfaces */
 
 #define MLXSW_SP_RFID_BASE 15360
-#define MLXSW_SP_RIF_MAX 800
-
-#define MLXSW_SP_LAG_MAX 64
-#define MLXSW_SP_PORT_PER_LAG_MAX 16
+#define MLXSW_SP_INVALID_RIF 0xffff
 
 #define MLXSW_SP_MID_MAX 7000
 
@@ -67,8 +64,6 @@
 #define MLXSW_SP_LPM_TREE_MAX 22
 #define MLXSW_SP_LPM_TREE_COUNT (MLXSW_SP_LPM_TREE_MAX - MLXSW_SP_LPM_TREE_MIN)
 
-#define MLXSW_SP_VIRTUAL_ROUTER_MAX 256
-
 #define MLXSW_SP_PORT_BASE_SPEED 25000	/* Mb/s */
 
 #define MLXSW_SP_BYTES_PER_CELL 96
@@ -77,8 +72,7 @@
 #define MLXSW_SP_CELLS_TO_BYTES(c) (c * MLXSW_SP_BYTES_PER_CELL)
 
 #define MLXSW_SP_KVD_LINEAR_SIZE 65536 /* entries */
-#define MLXSW_SP_KVD_HASH_SINGLE_SIZE 163840 /* entries */
-#define MLXSW_SP_KVD_HASH_DOUBLE_SIZE 32768 /* entries */
+#define MLXSW_SP_KVD_GRANULARITY 128
 
 /* Maximum delay buffer needed in case of PAUSE frames, in cells.
  * Assumes 100m cable and maximum MTU.
@@ -253,7 +247,7 @@ struct mlxsw_sp_port_mall_tc_entry {
 
 struct mlxsw_sp_router {
 	struct mlxsw_sp_lpm_tree lpm_trees[MLXSW_SP_LPM_TREE_COUNT];
-	struct mlxsw_sp_vr vrs[MLXSW_SP_VIRTUAL_ROUTER_MAX];
+	struct mlxsw_sp_vr *vrs;
 	struct rhashtable neigh_ht;
 	struct {
 		struct delayed_work dw;
@@ -263,6 +257,7 @@ struct mlxsw_sp_router {
 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
 	struct list_head nexthop_group_list;
 	struct list_head nexthop_neighs_list;
+	bool aborted;
 };
 
 struct mlxsw_sp {
@@ -275,7 +270,7 @@ struct mlxsw_sp {
 		DECLARE_BITMAP(mapped, MLXSW_SP_MID_MAX);
 	} br_mids;
 	struct list_head fids;	/* VLAN-aware bridge FIDs */
-	struct mlxsw_sp_rif *rifs[MLXSW_SP_RIF_MAX];
+	struct mlxsw_sp_rif **rifs;
 	struct mlxsw_sp_port **ports;
 	struct mlxsw_core *core;
 	const struct mlxsw_bus_info *bus_info;
@@ -290,7 +285,7 @@ struct mlxsw_sp {
 #define MLXSW_SP_DEFAULT_AGEING_TIME 300
 	u32 ageing_time;
 	struct mlxsw_sp_upper master_bridge;
-	struct mlxsw_sp_upper lags[MLXSW_SP_LAG_MAX];
+	struct mlxsw_sp_upper *lags;
 	u8 port_to_module[MLXSW_PORT_MAX_PORTS];
 	struct mlxsw_sp_sb sb;
 	struct mlxsw_sp_router router;
@@ -302,6 +297,7 @@ struct mlxsw_sp {
 		struct mlxsw_sp_span_entry *entries;
 		int entries_count;
 	} span;
+	struct notifier_block fib_nb;
 };
 
 static inline struct mlxsw_sp_upper *
@@ -341,7 +337,8 @@ struct mlxsw_sp_port {
 	} vport;
 	struct {
 		u8 tx_pause:1,
-		   rx_pause:1;
+		   rx_pause:1,
+		   autoneg:1;
 	} link;
 	struct {
 		struct ieee_ets *ets;
@@ -360,6 +357,11 @@ struct mlxsw_sp_port {
 	struct list_head vports_list;
 	/* TC handles */
 	struct list_head mall_tc_list;
+	struct {
+		#define MLXSW_HW_STATS_UPDATE_TIME HZ
+		struct rtnl_link_stats64 *cache;
+		struct delayed_work update_dw;
+	} hw_stats;
 };
 
 struct mlxsw_sp_port *mlxsw_sp_port_lower_dev_hold(struct net_device *dev);
@@ -477,9 +479,12 @@ static inline struct mlxsw_sp_rif *
 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 			 const struct net_device *dev)
 {
+	struct mlxsw_resources *resources;
 	int i;
 
-	for (i = 0; i < MLXSW_SP_RIF_MAX; i++)
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+
+	for (i = 0; i < resources->max_rif; i++)
 		if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev)
 			return mlxsw_sp->rifs[i];
 
@@ -536,8 +541,6 @@ int mlxsw_sp_port_vid_to_fid_set(struct mlxsw_sp_port *mlxsw_sp_port,
 				 u16 vid);
 int mlxsw_sp_port_vlan_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid_begin,
 			   u16 vid_end, bool is_member, bool untagged);
-int mlxsw_sp_port_add_vid(struct net_device *dev, __be16 __always_unused proto,
-			  u16 vid);
 int mlxsw_sp_vport_flood_set(struct mlxsw_sp_port *mlxsw_sp_vport, u16 fid,
 			     bool set);
 void mlxsw_sp_port_active_vlans_del(struct mlxsw_sp_port *mlxsw_sp_port);
@@ -560,6 +563,9 @@ int __mlxsw_sp_port_headroom_set(struct mlxsw_sp_port *mlxsw_sp_port, int mtu,
 int mlxsw_sp_port_ets_maxrate_set(struct mlxsw_sp_port *mlxsw_sp_port,
 				  enum mlxsw_reg_qeec_hr hr, u8 index,
 				  u8 next_index, u32 maxrate);
+int __mlxsw_sp_port_vid_learning_set(struct mlxsw_sp_port *mlxsw_sp_port,
+				     u16 vid_begin, u16 vid_end,
+				     bool learn_enable);
 
 #ifdef CONFIG_MLXSW_SPECTRUM_DCB
 
@@ -580,15 +586,12 @@ static inline void mlxsw_sp_port_dcb_fini(struct mlxsw_sp_port *mlxsw_sp_port)
 
 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp);
-int mlxsw_sp_router_fib4_add(struct mlxsw_sp_port *mlxsw_sp_port,
-			     const struct switchdev_obj_ipv4_fib *fib4,
-			     struct switchdev_trans *trans);
-int mlxsw_sp_router_fib4_del(struct mlxsw_sp_port *mlxsw_sp_port,
-			     const struct switchdev_obj_ipv4_fib *fib4);
 int mlxsw_sp_router_neigh_construct(struct net_device *dev,
 				    struct neighbour *n);
 void mlxsw_sp_router_neigh_destroy(struct net_device *dev,
 				   struct neighbour *n);
+int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
+				   unsigned long event, void *ptr);
 
 int mlxsw_sp_kvdl_alloc(struct mlxsw_sp *mlxsw_sp, unsigned int entry_count);
 void mlxsw_sp_kvdl_free(struct mlxsw_sp *mlxsw_sp, int entry_index);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c
index 074cdda7b6f3..bcaed8a38037 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c
@@ -330,7 +330,7 @@ static const struct mlxsw_sp_sb_cm mlxsw_sp_cpu_port_sb_cms[] = {
 	MLXSW_SP_CPU_PORT_SB_CM,
 	MLXSW_SP_CPU_PORT_SB_CM,
 	MLXSW_SP_CPU_PORT_SB_CM,
-	MLXSW_SP_CPU_PORT_SB_CM,
+	MLXSW_SP_SB_CM(MLXSW_SP_BYTES_TO_CELLS(10000), 0, 0),
 	MLXSW_SP_CPU_PORT_SB_CM,
 	MLXSW_SP_CPU_PORT_SB_CM,
 	MLXSW_SP_CPU_PORT_SB_CM,
@@ -595,9 +595,9 @@ int mlxsw_sp_sb_pool_get(struct mlxsw_core *mlxsw_core,
 	enum mlxsw_reg_sbxx_dir dir = dir_get(pool_index);
 	struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool, dir);
 
-	pool_info->pool_type = dir;
+	pool_info->pool_type = (enum devlink_sb_pool_type) dir;
 	pool_info->size = MLXSW_SP_CELLS_TO_BYTES(pr->size);
-	pool_info->threshold_type = pr->mode;
+	pool_info->threshold_type = (enum devlink_sb_threshold_type) pr->mode;
 	return 0;
 }
 
@@ -608,9 +608,10 @@ int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core,
 	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
 	u8 pool = pool_get(pool_index);
 	enum mlxsw_reg_sbxx_dir dir = dir_get(pool_index);
-	enum mlxsw_reg_sbpr_mode mode = threshold_type;
 	u32 pool_size = MLXSW_SP_BYTES_TO_CELLS(size);
+	enum mlxsw_reg_sbpr_mode mode;
 
+	mode = (enum mlxsw_reg_sbpr_mode) threshold_type;
 	return mlxsw_sp_sb_pr_write(mlxsw_sp, pool, dir, mode, pool_size);
 }
 
@@ -696,13 +697,13 @@ int mlxsw_sp_sb_tc_pool_bind_get(struct mlxsw_core_port *mlxsw_core_port,
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	u8 local_port = mlxsw_sp_port->local_port;
 	u8 pg_buff = tc_index;
-	enum mlxsw_reg_sbxx_dir dir = pool_type;
+	enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
 	struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port,
 						       pg_buff, dir);
 
 	*p_threshold = mlxsw_sp_sb_threshold_out(mlxsw_sp, cm->pool, dir,
 						 cm->max_buff);
-	*p_pool_index = pool_index_get(cm->pool, pool_type);
+	*p_pool_index = pool_index_get(cm->pool, dir);
 	return 0;
 }
 
@@ -716,23 +717,19 @@ int mlxsw_sp_sb_tc_pool_bind_set(struct mlxsw_core_port *mlxsw_core_port,
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	u8 local_port = mlxsw_sp_port->local_port;
 	u8 pg_buff = tc_index;
-	enum mlxsw_reg_sbxx_dir dir = pool_type;
-	u8 pool = pool_index;
+	enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
+	u8 pool = pool_get(pool_index);
 	u32 max_buff;
 	int err;
 
+	if (dir != dir_get(pool_index))
+		return -EINVAL;
+
 	err = mlxsw_sp_sb_threshold_in(mlxsw_sp, pool, dir,
 				       threshold, &max_buff);
 	if (err)
 		return err;
 
-	if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS) {
-		if (pool < MLXSW_SP_SB_POOL_COUNT)
-			return -EINVAL;
-		pool -= MLXSW_SP_SB_POOL_COUNT;
-	} else if (pool >= MLXSW_SP_SB_POOL_COUNT) {
-		return -EINVAL;
-	}
 	return mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, pg_buff, dir,
 				    0, max_buff, pool);
 }
@@ -947,7 +944,7 @@ int mlxsw_sp_sb_occ_tc_port_bind_get(struct mlxsw_core_port *mlxsw_core_port,
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	u8 local_port = mlxsw_sp_port->local_port;
 	u8 pg_buff = tc_index;
-	enum mlxsw_reg_sbxx_dir dir = pool_type;
+	enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
 	struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port,
 						       pg_buff, dir);
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c
index 01cfb7512827..b6ed7f7c531e 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c
@@ -341,6 +341,8 @@ static int mlxsw_sp_port_pfc_set(struct mlxsw_sp_port *mlxsw_sp_port,
 	char pfcc_pl[MLXSW_REG_PFCC_LEN];
 
 	mlxsw_reg_pfcc_pack(pfcc_pl, mlxsw_sp_port->local_port);
+	mlxsw_reg_pfcc_pprx_set(pfcc_pl, mlxsw_sp_port->link.rx_pause);
+	mlxsw_reg_pfcc_pptx_set(pfcc_pl, mlxsw_sp_port->link.tx_pause);
 	mlxsw_reg_pfcc_prio_pack(pfcc_pl, pfc->pfc_en);
 
 	return mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(pfcc),
@@ -351,17 +353,17 @@ static int mlxsw_sp_dcbnl_ieee_setpfc(struct net_device *dev,
 				      struct ieee_pfc *pfc)
 {
 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
+	bool pause_en = mlxsw_sp_port_is_pause_en(mlxsw_sp_port);
 	int err;
 
-	if ((mlxsw_sp_port->link.tx_pause || mlxsw_sp_port->link.rx_pause) &&
-	    pfc->pfc_en) {
+	if (pause_en && pfc->pfc_en) {
 		netdev_err(dev, "PAUSE frames already enabled on port\n");
 		return -EINVAL;
 	}
 
 	err = __mlxsw_sp_port_headroom_set(mlxsw_sp_port, dev->mtu,
 					   mlxsw_sp_port->dcb.ets->prio_tc,
-					   false, pfc);
+					   pause_en, pfc);
 	if (err) {
 		netdev_err(dev, "Failed to configure port's headroom for PFC\n");
 		return err;
@@ -380,7 +382,7 @@ static int mlxsw_sp_dcbnl_ieee_setpfc(struct net_device *dev,
 
 err_port_pfc_set:
 	__mlxsw_sp_port_headroom_set(mlxsw_sp_port, dev->mtu,
-				     mlxsw_sp_port->dcb.ets->prio_tc, false,
+				     mlxsw_sp_port->dcb.ets->prio_tc, pause_en,
 				     mlxsw_sp_port->dcb.pfc);
 	return err;
 }
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
index 81418d629231..78fc557d6dd7 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
@@ -43,6 +43,7 @@
 #include <net/netevent.h>
 #include <net/neighbour.h>
 #include <net/arp.h>
+#include <net/ip_fib.h>
 
 #include "spectrum.h"
 #include "core.h"
@@ -107,6 +108,7 @@ mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 }
 
 struct mlxsw_sp_fib_key {
+	struct net_device *dev;
 	unsigned char addr[sizeof(struct in6_addr)];
 	unsigned char prefix_len;
 };
@@ -121,17 +123,20 @@ struct mlxsw_sp_nexthop_group;
 
 struct mlxsw_sp_fib_entry {
 	struct rhash_head ht_node;
+	struct list_head list;
 	struct mlxsw_sp_fib_key key;
 	enum mlxsw_sp_fib_entry_type type;
-	u8 added:1;
+	unsigned int ref_count;
 	u16 rif; /* used for action local */
 	struct mlxsw_sp_vr *vr;
+	struct fib_info *fi;
 	struct list_head nexthop_group_node;
 	struct mlxsw_sp_nexthop_group *nh_group;
 };
 
 struct mlxsw_sp_fib {
 	struct rhashtable ht;
+	struct list_head entry_list;
 	unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 	struct mlxsw_sp_prefix_usage prefix_usage;
 };
@@ -153,6 +158,7 @@ static int mlxsw_sp_fib_entry_insert(struct mlxsw_sp_fib *fib,
 				     mlxsw_sp_fib_ht_params);
 	if (err)
 		return err;
+	list_add_tail(&fib_entry->list, &fib->entry_list);
 	if (fib->prefix_ref_count[prefix_len]++ == 0)
 		mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
 	return 0;
@@ -165,19 +171,22 @@ static void mlxsw_sp_fib_entry_remove(struct mlxsw_sp_fib *fib,
 
 	if (--fib->prefix_ref_count[prefix_len] == 0)
 		mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
+	list_del(&fib_entry->list);
 	rhashtable_remove_fast(&fib->ht, &fib_entry->ht_node,
 			       mlxsw_sp_fib_ht_params);
 }
 
 static struct mlxsw_sp_fib_entry *
 mlxsw_sp_fib_entry_create(struct mlxsw_sp_fib *fib, const void *addr,
-			  size_t addr_len, unsigned char prefix_len)
+			  size_t addr_len, unsigned char prefix_len,
+			  struct net_device *dev)
 {
 	struct mlxsw_sp_fib_entry *fib_entry;
 
 	fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
 	if (!fib_entry)
 		return NULL;
+	fib_entry->key.dev = dev;
 	memcpy(fib_entry->key.addr, addr, addr_len);
 	fib_entry->key.prefix_len = prefix_len;
 	return fib_entry;
@@ -190,10 +199,13 @@ static void mlxsw_sp_fib_entry_destroy(struct mlxsw_sp_fib_entry *fib_entry)
 
 static struct mlxsw_sp_fib_entry *
 mlxsw_sp_fib_entry_lookup(struct mlxsw_sp_fib *fib, const void *addr,
-			  size_t addr_len, unsigned char prefix_len)
+			  size_t addr_len, unsigned char prefix_len,
+			  struct net_device *dev)
 {
-	struct mlxsw_sp_fib_key key = {{ 0 } };
+	struct mlxsw_sp_fib_key key;
 
+	memset(&key, 0, sizeof(key));
+	key.dev = dev;
 	memcpy(key.addr, addr, addr_len);
 	key.prefix_len = prefix_len;
 	return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
@@ -210,6 +222,7 @@ static struct mlxsw_sp_fib *mlxsw_sp_fib_create(void)
 	err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 	if (err)
 		goto err_rhashtable_init;
+	INIT_LIST_HEAD(&fib->entry_list);
 	return fib;
 
 err_rhashtable_init:
@@ -246,7 +259,9 @@ static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 {
 	char ralta_pl[MLXSW_REG_RALTA_LEN];
 
-	mlxsw_reg_ralta_pack(ralta_pl, true, lpm_tree->proto, lpm_tree->id);
+	mlxsw_reg_ralta_pack(ralta_pl, true,
+			     (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
+			     lpm_tree->id);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 }
 
@@ -255,7 +270,9 @@ static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 {
 	char ralta_pl[MLXSW_REG_RALTA_LEN];
 
-	mlxsw_reg_ralta_pack(ralta_pl, false, lpm_tree->proto, lpm_tree->id);
+	mlxsw_reg_ralta_pack(ralta_pl, false,
+			     (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
+			     lpm_tree->id);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 }
 
@@ -362,10 +379,12 @@ static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 
 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 {
+	struct mlxsw_resources *resources;
 	struct mlxsw_sp_vr *vr;
 	int i;
 
-	for (i = 0; i < MLXSW_SP_VIRTUAL_ROUTER_MAX; i++) {
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	for (i = 0; i < resources->max_virtual_routers; i++) {
 		vr = &mlxsw_sp->router.vrs[i];
 		if (!vr->used)
 			return vr;
@@ -378,7 +397,9 @@ static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 {
 	char raltb_pl[MLXSW_REG_RALTB_LEN];
 
-	mlxsw_reg_raltb_pack(raltb_pl, vr->id, vr->proto, vr->lpm_tree->id);
+	mlxsw_reg_raltb_pack(raltb_pl, vr->id,
+			     (enum mlxsw_reg_ralxx_protocol) vr->proto,
+			     vr->lpm_tree->id);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 }
 
@@ -388,7 +409,8 @@ static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 	char raltb_pl[MLXSW_REG_RALTB_LEN];
 
 	/* Bind to tree 0 which is default */
-	mlxsw_reg_raltb_pack(raltb_pl, vr->id, vr->proto, 0);
+	mlxsw_reg_raltb_pack(raltb_pl, vr->id,
+			     (enum mlxsw_reg_ralxx_protocol) vr->proto, 0);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 }
 
@@ -404,11 +426,14 @@ static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 					    u32 tb_id,
 					    enum mlxsw_sp_l3proto proto)
 {
+	struct mlxsw_resources *resources;
 	struct mlxsw_sp_vr *vr;
 	int i;
 
 	tb_id = mlxsw_sp_fix_tb_id(tb_id);
-	for (i = 0; i < MLXSW_SP_VIRTUAL_ROUTER_MAX; i++) {
+
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	for (i = 0; i < resources->max_virtual_routers; i++) {
 		vr = &mlxsw_sp->router.vrs[i];
 		if (vr->used && vr->proto == proto && vr->tb_id == tb_id)
 			return vr;
@@ -542,15 +567,33 @@ static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 					   &vr->fib->prefix_usage);
 }
 
-static void mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
+static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 {
+	struct mlxsw_resources *resources;
 	struct mlxsw_sp_vr *vr;
 	int i;
 
-	for (i = 0; i < MLXSW_SP_VIRTUAL_ROUTER_MAX; i++) {
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	if (!resources->max_virtual_routers_valid)
+		return -EIO;
+
+	mlxsw_sp->router.vrs = kcalloc(resources->max_virtual_routers,
+				       sizeof(struct mlxsw_sp_vr),
+				       GFP_KERNEL);
+	if (!mlxsw_sp->router.vrs)
+		return -ENOMEM;
+
+	for (i = 0; i < resources->max_virtual_routers; i++) {
 		vr = &mlxsw_sp->router.vrs[i];
 		vr->id = i;
 	}
+
+	return 0;
+}
+
+static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
+{
+	kfree(mlxsw_sp->router.vrs);
 }
 
 struct mlxsw_sp_neigh_key {
@@ -657,7 +700,7 @@ int mlxsw_sp_router_neigh_construct(struct net_device *dev,
 		return 0;
 	}
 
-	r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
+	r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 	if (WARN_ON(!r))
 		return -EINVAL;
 
@@ -938,8 +981,8 @@ static void mlxsw_sp_router_neigh_update_hw(struct work_struct *work)
 	mlxsw_sp_port_dev_put(mlxsw_sp_port);
 }
 
-static int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
-					  unsigned long event, void *ptr)
+int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
+				   unsigned long event, void *ptr)
 {
 	struct mlxsw_sp_neigh_entry *neigh_entry;
 	struct mlxsw_sp_port *mlxsw_sp_port;
@@ -1009,10 +1052,6 @@ static int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 	return NOTIFY_DONE;
 }
 
-static struct notifier_block mlxsw_sp_router_netevent_nb __read_mostly = {
-	.notifier_call = mlxsw_sp_router_netevent_event,
-};
-
 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
 {
 	int err;
@@ -1027,10 +1066,6 @@ static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
 	 */
 	mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
 
-	err = register_netevent_notifier(&mlxsw_sp_router_netevent_nb);
-	if (err)
-		goto err_register_netevent_notifier;
-
 	/* Create the delayed works for the activity_update */
 	INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
 			  mlxsw_sp_router_neighs_update_work);
@@ -1039,17 +1074,12 @@ static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
 	mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
 	mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
 	return 0;
-
-err_register_netevent_notifier:
-	rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
-	return err;
 }
 
 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
 {
 	cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
 	cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
-	unregister_netevent_notifier(&mlxsw_sp_router_netevent_nb);
 	rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
 }
 
@@ -1088,9 +1118,10 @@ static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
 {
 	char raleu_pl[MLXSW_REG_RALEU_LEN];
 
-	mlxsw_reg_raleu_pack(raleu_pl, vr->proto, vr->id,
-			     adj_index, ecmp_size,
-			     new_adj_index, new_ecmp_size);
+	mlxsw_reg_raleu_pack(raleu_pl,
+			     (enum mlxsw_reg_ralxx_protocol) vr->proto, vr->id,
+			     adj_index, ecmp_size, new_adj_index,
+			     new_ecmp_size);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
 }
 
@@ -1496,43 +1527,6 @@ static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
 	mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
 }
 
-static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
-{
-	char rgcr_pl[MLXSW_REG_RGCR_LEN];
-
-	mlxsw_reg_rgcr_pack(rgcr_pl, true);
-	mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, MLXSW_SP_RIF_MAX);
-	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
-}
-
-static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
-{
-	char rgcr_pl[MLXSW_REG_RGCR_LEN];
-
-	mlxsw_reg_rgcr_pack(rgcr_pl, false);
-	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
-}
-
-int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
-{
-	int err;
-
-	INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
-	INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_group_list);
-	err = __mlxsw_sp_router_init(mlxsw_sp);
-	if (err)
-		return err;
-	mlxsw_sp_lpm_init(mlxsw_sp);
-	mlxsw_sp_vrs_init(mlxsw_sp);
-	return mlxsw_sp_neigh_init(mlxsw_sp);
-}
-
-void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
-{
-	mlxsw_sp_neigh_fini(mlxsw_sp);
-	__mlxsw_sp_router_fini(mlxsw_sp);
-}
-
 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
 					 struct mlxsw_sp_fib_entry *fib_entry,
 					 enum mlxsw_reg_ralue_op op)
@@ -1558,8 +1552,9 @@ static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
 		trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
 	}
 
-	mlxsw_reg_ralue_pack4(ralue_pl, vr->proto, op, vr->id,
-			      fib_entry->key.prefix_len, *p_dip);
+	mlxsw_reg_ralue_pack4(ralue_pl,
+			      (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
+			      vr->id, fib_entry->key.prefix_len, *p_dip);
 	mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
 					adjacency_index, ecmp_size);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
@@ -1573,8 +1568,9 @@ static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
 	u32 *p_dip = (u32 *) fib_entry->key.addr;
 	struct mlxsw_sp_vr *vr = fib_entry->vr;
 
-	mlxsw_reg_ralue_pack4(ralue_pl, vr->proto, op, vr->id,
-			      fib_entry->key.prefix_len, *p_dip);
+	mlxsw_reg_ralue_pack4(ralue_pl,
+			      (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
+			      vr->id, fib_entry->key.prefix_len, *p_dip);
 	mlxsw_reg_ralue_act_local_pack(ralue_pl,
 				       MLXSW_REG_RALUE_TRAP_ACTION_NOP, 0,
 				       fib_entry->rif);
@@ -1589,8 +1585,9 @@ static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
 	u32 *p_dip = (u32 *) fib_entry->key.addr;
 	struct mlxsw_sp_vr *vr = fib_entry->vr;
 
-	mlxsw_reg_ralue_pack4(ralue_pl, vr->proto, op, vr->id,
-			      fib_entry->key.prefix_len, *p_dip);
+	mlxsw_reg_ralue_pack4(ralue_pl,
+			      (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
+			      vr->id, fib_entry->key.prefix_len, *p_dip);
 	mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
 }
@@ -1626,11 +1623,8 @@ static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
 				     struct mlxsw_sp_fib_entry *fib_entry)
 {
-	enum mlxsw_reg_ralue_op op;
-
-	op = !fib_entry->added ? MLXSW_REG_RALUE_OP_WRITE_WRITE :
-				 MLXSW_REG_RALUE_OP_WRITE_UPDATE;
-	return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
+	return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
+				     MLXSW_REG_RALUE_OP_WRITE_WRITE);
 }
 
 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
@@ -1640,128 +1634,180 @@ static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
 				     MLXSW_REG_RALUE_OP_WRITE_DELETE);
 }
 
-struct mlxsw_sp_router_fib4_add_info {
-	struct switchdev_trans_item tritem;
-	struct mlxsw_sp *mlxsw_sp;
-	struct mlxsw_sp_fib_entry *fib_entry;
-};
-
-static void mlxsw_sp_router_fib4_add_info_destroy(void const *data)
-{
-	const struct mlxsw_sp_router_fib4_add_info *info = data;
-	struct mlxsw_sp_fib_entry *fib_entry = info->fib_entry;
-	struct mlxsw_sp *mlxsw_sp = info->mlxsw_sp;
-
-	mlxsw_sp_fib_entry_destroy(fib_entry);
-	mlxsw_sp_vr_put(mlxsw_sp, fib_entry->vr);
-	kfree(info);
-}
-
 static int
 mlxsw_sp_router_fib4_entry_init(struct mlxsw_sp *mlxsw_sp,
-				const struct switchdev_obj_ipv4_fib *fib4,
+				const struct fib_entry_notifier_info *fen_info,
 				struct mlxsw_sp_fib_entry *fib_entry)
 {
-	struct fib_info *fi = fib4->fi;
+	struct fib_info *fi = fen_info->fi;
+	struct mlxsw_sp_rif *r = NULL;
+	int nhsel;
+	int err;
 
-	if (fib4->type == RTN_LOCAL || fib4->type == RTN_BROADCAST) {
+	if (fen_info->type == RTN_LOCAL || fen_info->type == RTN_BROADCAST) {
 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
 		return 0;
 	}
-	if (fib4->type != RTN_UNICAST)
+	if (fen_info->type != RTN_UNICAST)
 		return -EINVAL;
 
-	if (fi->fib_scope != RT_SCOPE_UNIVERSE) {
-		struct mlxsw_sp_rif *r;
+	for (nhsel = 0; nhsel < fi->fib_nhs; nhsel++) {
+		const struct fib_nh *nh = &fi->fib_nh[nhsel];
+
+		if (!nh->nh_dev)
+			continue;
+		r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, nh->nh_dev);
+		if (!r) {
+			/* In case router interface is not found for
+			 * at least one of the nexthops, that means
+			 * the nexthop points to some device unrelated
+			 * to us. Set trap and pass the packets for
+			 * this prefix to kernel.
+			 */
+			break;
+		}
+	}
+
+	if (!r) {
+		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
+		return 0;
+	}
 
+	if (fi->fib_scope != RT_SCOPE_UNIVERSE) {
 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
-		r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fi->fib_dev);
-		if (!r)
-			return -EINVAL;
 		fib_entry->rif = r->rif;
-		return 0;
+	} else {
+		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
+		err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fi);
+		if (err)
+			return err;
 	}
-	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
-	return mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fi);
+	fib_info_offload_inc(fen_info->fi);
+	return 0;
 }
 
 static void
 mlxsw_sp_router_fib4_entry_fini(struct mlxsw_sp *mlxsw_sp,
 				struct mlxsw_sp_fib_entry *fib_entry)
 {
-	if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_REMOTE)
-		return;
-	mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
+	if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
+		fib_info_offload_dec(fib_entry->fi);
+	if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_REMOTE)
+		mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
 }
 
-static int
-mlxsw_sp_router_fib4_add_prepare(struct mlxsw_sp_port *mlxsw_sp_port,
-				 const struct switchdev_obj_ipv4_fib *fib4,
-				 struct switchdev_trans *trans)
+static struct mlxsw_sp_fib_entry *
+mlxsw_sp_fib_entry_get(struct mlxsw_sp *mlxsw_sp,
+		       const struct fib_entry_notifier_info *fen_info)
 {
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-	struct mlxsw_sp_router_fib4_add_info *info;
 	struct mlxsw_sp_fib_entry *fib_entry;
+	struct fib_info *fi = fen_info->fi;
 	struct mlxsw_sp_vr *vr;
 	int err;
 
-	vr = mlxsw_sp_vr_get(mlxsw_sp, fib4->dst_len, fib4->tb_id,
+	vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->dst_len, fen_info->tb_id,
 			     MLXSW_SP_L3_PROTO_IPV4);
 	if (IS_ERR(vr))
-		return PTR_ERR(vr);
-
-	fib_entry = mlxsw_sp_fib_entry_create(vr->fib, &fib4->dst,
-					      sizeof(fib4->dst), fib4->dst_len);
+		return ERR_CAST(vr);
+
+	fib_entry = mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
+					      sizeof(fen_info->dst),
+					      fen_info->dst_len, fi->fib_dev);
+	if (fib_entry) {
+		/* Already exists, just take a reference */
+		fib_entry->ref_count++;
+		return fib_entry;
+	}
+	fib_entry = mlxsw_sp_fib_entry_create(vr->fib, &fen_info->dst,
+					      sizeof(fen_info->dst),
+					      fen_info->dst_len, fi->fib_dev);
 	if (!fib_entry) {
 		err = -ENOMEM;
 		goto err_fib_entry_create;
 	}
 	fib_entry->vr = vr;
+	fib_entry->fi = fi;
+	fib_entry->ref_count = 1;
 
-	err = mlxsw_sp_router_fib4_entry_init(mlxsw_sp, fib4, fib_entry);
+	err = mlxsw_sp_router_fib4_entry_init(mlxsw_sp, fen_info, fib_entry);
 	if (err)
 		goto err_fib4_entry_init;
 
-	info = kmalloc(sizeof(*info), GFP_KERNEL);
-	if (!info) {
-		err = -ENOMEM;
-		goto err_alloc_info;
-	}
-	info->mlxsw_sp = mlxsw_sp;
-	info->fib_entry = fib_entry;
-	switchdev_trans_item_enqueue(trans, info,
-				     mlxsw_sp_router_fib4_add_info_destroy,
-				     &info->tritem);
-	return 0;
+	return fib_entry;
 
-err_alloc_info:
-	mlxsw_sp_router_fib4_entry_fini(mlxsw_sp, fib_entry);
 err_fib4_entry_init:
 	mlxsw_sp_fib_entry_destroy(fib_entry);
 err_fib_entry_create:
 	mlxsw_sp_vr_put(mlxsw_sp, vr);
-	return err;
+
+	return ERR_PTR(err);
 }
 
-static int
-mlxsw_sp_router_fib4_add_commit(struct mlxsw_sp_port *mlxsw_sp_port,
-				const struct switchdev_obj_ipv4_fib *fib4,
-				struct switchdev_trans *trans)
+static struct mlxsw_sp_fib_entry *
+mlxsw_sp_fib_entry_find(struct mlxsw_sp *mlxsw_sp,
+			const struct fib_entry_notifier_info *fen_info)
+{
+	struct mlxsw_sp_vr *vr;
+
+	vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id,
+			      MLXSW_SP_L3_PROTO_IPV4);
+	if (!vr)
+		return NULL;
+
+	return mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
+					 sizeof(fen_info->dst),
+					 fen_info->dst_len,
+					 fen_info->fi->fib_dev);
+}
+
+static void mlxsw_sp_fib_entry_put(struct mlxsw_sp *mlxsw_sp,
+				   struct mlxsw_sp_fib_entry *fib_entry)
+{
+	struct mlxsw_sp_vr *vr = fib_entry->vr;
+
+	if (--fib_entry->ref_count == 0) {
+		mlxsw_sp_router_fib4_entry_fini(mlxsw_sp, fib_entry);
+		mlxsw_sp_fib_entry_destroy(fib_entry);
+	}
+	mlxsw_sp_vr_put(mlxsw_sp, vr);
+}
+
+static void mlxsw_sp_fib_entry_put_all(struct mlxsw_sp *mlxsw_sp,
+				       struct mlxsw_sp_fib_entry *fib_entry)
+{
+	unsigned int last_ref_count;
+
+	do {
+		last_ref_count = fib_entry->ref_count;
+		mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
+	} while (last_ref_count != 1);
+}
+
+static int mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
+				    struct fib_entry_notifier_info *fen_info)
 {
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-	struct mlxsw_sp_router_fib4_add_info *info;
 	struct mlxsw_sp_fib_entry *fib_entry;
 	struct mlxsw_sp_vr *vr;
 	int err;
 
-	info = switchdev_trans_item_dequeue(trans);
-	fib_entry = info->fib_entry;
-	kfree(info);
+	if (mlxsw_sp->router.aborted)
+		return 0;
+
+	fib_entry = mlxsw_sp_fib_entry_get(mlxsw_sp, fen_info);
+	if (IS_ERR(fib_entry)) {
+		dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB4 entry being added.\n");
+		return PTR_ERR(fib_entry);
+	}
+
+	if (fib_entry->ref_count != 1)
+		return 0;
 
 	vr = fib_entry->vr;
-	err = mlxsw_sp_fib_entry_insert(fib_entry->vr->fib, fib_entry);
-	if (err)
+	err = mlxsw_sp_fib_entry_insert(vr->fib, fib_entry);
+	if (err) {
+		dev_warn(mlxsw_sp->bus_info->dev, "Failed to insert FIB4 entry being added.\n");
 		goto err_fib_entry_insert;
+	}
 	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
 	if (err)
 		goto err_fib_entry_add;
@@ -1770,45 +1816,198 @@ mlxsw_sp_router_fib4_add_commit(struct mlxsw_sp_port *mlxsw_sp_port,
 err_fib_entry_add:
 	mlxsw_sp_fib_entry_remove(vr->fib, fib_entry);
 err_fib_entry_insert:
-	mlxsw_sp_router_fib4_entry_fini(mlxsw_sp, fib_entry);
-	mlxsw_sp_fib_entry_destroy(fib_entry);
-	mlxsw_sp_vr_put(mlxsw_sp, vr);
+	mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
 	return err;
 }
 
-int mlxsw_sp_router_fib4_add(struct mlxsw_sp_port *mlxsw_sp_port,
-			     const struct switchdev_obj_ipv4_fib *fib4,
-			     struct switchdev_trans *trans)
+static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
+				    struct fib_entry_notifier_info *fen_info)
 {
-	if (switchdev_trans_ph_prepare(trans))
-		return mlxsw_sp_router_fib4_add_prepare(mlxsw_sp_port,
-							fib4, trans);
-	return mlxsw_sp_router_fib4_add_commit(mlxsw_sp_port,
-					       fib4, trans);
+	struct mlxsw_sp_fib_entry *fib_entry;
+
+	if (mlxsw_sp->router.aborted)
+		return 0;
+
+	fib_entry = mlxsw_sp_fib_entry_find(mlxsw_sp, fen_info);
+	if (!fib_entry) {
+		dev_warn(mlxsw_sp->bus_info->dev, "Failed to find FIB4 entry being removed.\n");
+		return -ENOENT;
+	}
+
+	if (fib_entry->ref_count == 1) {
+		mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
+		mlxsw_sp_fib_entry_remove(fib_entry->vr->fib, fib_entry);
+	}
+
+	mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
+	return 0;
 }
 
-int mlxsw_sp_router_fib4_del(struct mlxsw_sp_port *mlxsw_sp_port,
-			     const struct switchdev_obj_ipv4_fib *fib4)
+static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
 {
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
+	char ralta_pl[MLXSW_REG_RALTA_LEN];
+	char ralst_pl[MLXSW_REG_RALST_LEN];
+	char raltb_pl[MLXSW_REG_RALTB_LEN];
+	char ralue_pl[MLXSW_REG_RALUE_LEN];
+	int err;
+
+	mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
+			     MLXSW_SP_LPM_TREE_MIN);
+	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
+	if (err)
+		return err;
+
+	mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
+	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
+	if (err)
+		return err;
+
+	mlxsw_reg_raltb_pack(raltb_pl, 0, MLXSW_REG_RALXX_PROTOCOL_IPV4, 0);
+	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
+	if (err)
+		return err;
+
+	mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
+			      MLXSW_REG_RALUE_OP_WRITE_WRITE, 0, 0, 0);
+	mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
+	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
+}
+
+static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
+{
+	struct mlxsw_resources *resources;
 	struct mlxsw_sp_fib_entry *fib_entry;
+	struct mlxsw_sp_fib_entry *tmp;
 	struct mlxsw_sp_vr *vr;
+	int i;
+	int err;
 
-	vr = mlxsw_sp_vr_find(mlxsw_sp, fib4->tb_id, MLXSW_SP_L3_PROTO_IPV4);
-	if (!vr) {
-		dev_warn(mlxsw_sp->bus_info->dev, "Failed to find virtual router for FIB4 entry being removed.\n");
-		return -ENOENT;
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	for (i = 0; i < resources->max_virtual_routers; i++) {
+		vr = &mlxsw_sp->router.vrs[i];
+		if (!vr->used)
+			continue;
+
+		list_for_each_entry_safe(fib_entry, tmp,
+					 &vr->fib->entry_list, list) {
+			bool do_break = &tmp->list == &vr->fib->entry_list;
+
+			mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
+			mlxsw_sp_fib_entry_remove(fib_entry->vr->fib,
+						  fib_entry);
+			mlxsw_sp_fib_entry_put_all(mlxsw_sp, fib_entry);
+			if (do_break)
+				break;
+		}
 	}
-	fib_entry = mlxsw_sp_fib_entry_lookup(vr->fib, &fib4->dst,
-					      sizeof(fib4->dst), fib4->dst_len);
-	if (!fib_entry) {
-		dev_warn(mlxsw_sp->bus_info->dev, "Failed to find FIB4 entry being removed.\n");
-		return -ENOENT;
+	mlxsw_sp->router.aborted = true;
+	err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
+	if (err)
+		dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
+}
+
+static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
+{
+	struct mlxsw_resources *resources;
+	char rgcr_pl[MLXSW_REG_RGCR_LEN];
+	int err;
+
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	if (!resources->max_rif_valid)
+		return -EIO;
+
+	mlxsw_sp->rifs = kcalloc(resources->max_rif,
+				 sizeof(struct mlxsw_sp_rif *), GFP_KERNEL);
+	if (!mlxsw_sp->rifs)
+		return -ENOMEM;
+
+	mlxsw_reg_rgcr_pack(rgcr_pl, true);
+	mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, resources->max_rif);
+	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
+	if (err)
+		goto err_rgcr_fail;
+
+	return 0;
+
+err_rgcr_fail:
+	kfree(mlxsw_sp->rifs);
+	return err;
+}
+
+static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
+{
+	struct mlxsw_resources *resources;
+	char rgcr_pl[MLXSW_REG_RGCR_LEN];
+	int i;
+
+	mlxsw_reg_rgcr_pack(rgcr_pl, false);
+	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
+
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	for (i = 0; i < resources->max_rif; i++)
+		WARN_ON_ONCE(mlxsw_sp->rifs[i]);
+
+	kfree(mlxsw_sp->rifs);
+}
+
+static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
+				     unsigned long event, void *ptr)
+{
+	struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
+	struct fib_entry_notifier_info *fen_info = ptr;
+	int err;
+
+	switch (event) {
+	case FIB_EVENT_ENTRY_ADD:
+		err = mlxsw_sp_router_fib4_add(mlxsw_sp, fen_info);
+		if (err)
+			mlxsw_sp_router_fib4_abort(mlxsw_sp);
+		break;
+	case FIB_EVENT_ENTRY_DEL:
+		mlxsw_sp_router_fib4_del(mlxsw_sp, fen_info);
+		break;
+	case FIB_EVENT_RULE_ADD: /* fall through */
+	case FIB_EVENT_RULE_DEL:
+		mlxsw_sp_router_fib4_abort(mlxsw_sp);
+		break;
 	}
-	mlxsw_sp_fib_entry_del(mlxsw_sp_port->mlxsw_sp, fib_entry);
-	mlxsw_sp_fib_entry_remove(vr->fib, fib_entry);
-	mlxsw_sp_router_fib4_entry_fini(mlxsw_sp, fib_entry);
-	mlxsw_sp_fib_entry_destroy(fib_entry);
-	mlxsw_sp_vr_put(mlxsw_sp, vr);
+	return NOTIFY_DONE;
+}
+
+int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
+{
+	int err;
+
+	INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
+	INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_group_list);
+	err = __mlxsw_sp_router_init(mlxsw_sp);
+	if (err)
+		return err;
+
+	mlxsw_sp_lpm_init(mlxsw_sp);
+	err = mlxsw_sp_vrs_init(mlxsw_sp);
+	if (err)
+		goto err_vrs_init;
+
+	err =  mlxsw_sp_neigh_init(mlxsw_sp);
+	if (err)
+		goto err_neigh_init;
+
+	mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
+	register_fib_notifier(&mlxsw_sp->fib_nb);
 	return 0;
+
+err_neigh_init:
+	mlxsw_sp_vrs_fini(mlxsw_sp);
+err_vrs_init:
+	__mlxsw_sp_router_fini(mlxsw_sp);
+	return err;
+}
+
+void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
+{
+	unregister_fib_notifier(&mlxsw_sp->fib_nb);
+	mlxsw_sp_neigh_fini(mlxsw_sp);
+	mlxsw_sp_vrs_fini(mlxsw_sp);
+	__mlxsw_sp_router_fini(mlxsw_sp);
 }
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
index a1ad5e6bdfa8..5e00c79e8133 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
@@ -167,8 +167,8 @@ static int mlxsw_sp_port_attr_stp_state_set(struct mlxsw_sp_port *mlxsw_sp_port,
 }
 
 static int __mlxsw_sp_port_flood_set(struct mlxsw_sp_port *mlxsw_sp_port,
-				     u16 idx_begin, u16 idx_end, bool set,
-				     bool only_uc)
+				     u16 idx_begin, u16 idx_end, bool uc_set,
+				     bool bm_set)
 {
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	u16 local_port = mlxsw_sp_port->local_port;
@@ -187,28 +187,22 @@ static int __mlxsw_sp_port_flood_set(struct mlxsw_sp_port *mlxsw_sp_port,
 		return -ENOMEM;
 
 	mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_UC, idx_begin,
-			    table_type, range, local_port, set);
+			    table_type, range, local_port, uc_set);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
 	if (err)
 		goto buffer_out;
 
-	/* Flooding control allows one to decide whether a given port will
-	 * flood unicast traffic for which there is no FDB entry.
-	 */
-	if (only_uc)
-		goto buffer_out;
-
 	mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BM, idx_begin,
-			    table_type, range, local_port, set);
+			    table_type, range, local_port, bm_set);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
 	if (err)
 		goto err_flood_bm_set;
-	else
-		goto buffer_out;
+
+	goto buffer_out;
 
 err_flood_bm_set:
 	mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_UC, idx_begin,
-			    table_type, range, local_port, !set);
+			    table_type, range, local_port, !uc_set);
 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
 buffer_out:
 	kfree(sftr_pl);
@@ -257,16 +251,43 @@ int mlxsw_sp_vport_flood_set(struct mlxsw_sp_port *mlxsw_sp_vport, u16 fid,
 	 * the start of the vFIDs range.
 	 */
 	vfid = mlxsw_sp_fid_to_vfid(fid);
-	return __mlxsw_sp_port_flood_set(mlxsw_sp_vport, vfid, vfid, set,
-					 false);
+	return __mlxsw_sp_port_flood_set(mlxsw_sp_vport, vfid, vfid, set, set);
+}
+
+static int mlxsw_sp_port_learning_set(struct mlxsw_sp_port *mlxsw_sp_port,
+				      bool set)
+{
+	u16 vid;
+	int err;
+
+	if (mlxsw_sp_port_is_vport(mlxsw_sp_port)) {
+		vid = mlxsw_sp_vport_vid_get(mlxsw_sp_port);
+
+		return __mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, vid,
+							set);
+	}
+
+	for_each_set_bit(vid, mlxsw_sp_port->active_vlans, VLAN_N_VID) {
+		err = __mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, vid,
+						       set);
+		if (err)
+			goto err_port_vid_learning_set;
+	}
+
+	return 0;
+
+err_port_vid_learning_set:
+	for_each_set_bit(vid, mlxsw_sp_port->active_vlans, VLAN_N_VID)
+		__mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, vid, !set);
+	return err;
 }
 
 static int mlxsw_sp_port_attr_br_flags_set(struct mlxsw_sp_port *mlxsw_sp_port,
 					   struct switchdev_trans *trans,
 					   unsigned long brport_flags)
 {
+	unsigned long learning = mlxsw_sp_port->learning ? BR_LEARNING : 0;
 	unsigned long uc_flood = mlxsw_sp_port->uc_flood ? BR_FLOOD : 0;
-	bool set;
 	int err;
 
 	if (!mlxsw_sp_port->bridged)
@@ -276,17 +297,30 @@ static int mlxsw_sp_port_attr_br_flags_set(struct mlxsw_sp_port *mlxsw_sp_port,
 		return 0;
 
 	if ((uc_flood ^ brport_flags) & BR_FLOOD) {
-		set = mlxsw_sp_port->uc_flood ? false : true;
-		err = mlxsw_sp_port_uc_flood_set(mlxsw_sp_port, set);
+		err = mlxsw_sp_port_uc_flood_set(mlxsw_sp_port,
+						 !mlxsw_sp_port->uc_flood);
 		if (err)
 			return err;
 	}
 
+	if ((learning ^ brport_flags) & BR_LEARNING) {
+		err = mlxsw_sp_port_learning_set(mlxsw_sp_port,
+						 !mlxsw_sp_port->learning);
+		if (err)
+			goto err_port_learning_set;
+	}
+
 	mlxsw_sp_port->uc_flood = brport_flags & BR_FLOOD ? 1 : 0;
 	mlxsw_sp_port->learning = brport_flags & BR_LEARNING ? 1 : 0;
 	mlxsw_sp_port->learning_sync = brport_flags & BR_LEARNING_SYNC ? 1 : 0;
 
 	return 0;
+
+err_port_learning_set:
+	if ((uc_flood ^ brport_flags) & BR_FLOOD)
+		mlxsw_sp_port_uc_flood_set(mlxsw_sp_port,
+					   mlxsw_sp_port->uc_flood);
+	return err;
 }
 
 static int mlxsw_sp_ageing_set(struct mlxsw_sp *mlxsw_sp, u32 ageing_time)
@@ -450,6 +484,8 @@ void mlxsw_sp_fid_destroy(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fid *f)
 
 	kfree(f);
 
+	mlxsw_sp_fid_map(mlxsw_sp, fid, false);
+
 	mlxsw_sp_fid_op(mlxsw_sp, fid, false);
 }
 
@@ -458,6 +494,9 @@ static int __mlxsw_sp_port_fid_join(struct mlxsw_sp_port *mlxsw_sp_port,
 {
 	struct mlxsw_sp_fid *f;
 
+	if (test_bit(fid, mlxsw_sp_port->active_vlans))
+		return 0;
+
 	f = mlxsw_sp_fid_find(mlxsw_sp_port->mlxsw_sp, fid);
 	if (!f) {
 		f = mlxsw_sp_fid_create(mlxsw_sp_port->mlxsw_sp, fid);
@@ -515,7 +554,7 @@ static int mlxsw_sp_port_fid_join(struct mlxsw_sp_port *mlxsw_sp_port,
 	}
 
 	err = __mlxsw_sp_port_flood_set(mlxsw_sp_port, fid_begin, fid_end,
-					true, false);
+					mlxsw_sp_port->uc_flood, true);
 	if (err)
 		goto err_port_flood_set;
 
@@ -633,6 +672,27 @@ static int __mlxsw_sp_port_vlans_set(struct mlxsw_sp_port *mlxsw_sp_port,
 	return 0;
 }
 
+static int mlxsw_sp_port_vid_learning_set(struct mlxsw_sp_port *mlxsw_sp_port,
+					  u16 vid_begin, u16 vid_end,
+					  bool learn_enable)
+{
+	u16 vid, vid_e;
+	int err;
+
+	for (vid = vid_begin; vid <= vid_end;
+	     vid += MLXSW_REG_SPVMLR_REC_MAX_COUNT) {
+		vid_e = min((u16) (vid + MLXSW_REG_SPVMLR_REC_MAX_COUNT - 1),
+			    vid_end);
+
+		err = __mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid,
+						       vid_e, learn_enable);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
 static int __mlxsw_sp_port_vlans_add(struct mlxsw_sp_port *mlxsw_sp_port,
 				     u16 vid_begin, u16 vid_end,
 				     bool flag_untagged, bool flag_pvid)
@@ -673,6 +733,14 @@ static int __mlxsw_sp_port_vlans_add(struct mlxsw_sp_port *mlxsw_sp_port,
 		}
 	}
 
+	err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid_begin, vid_end,
+					     mlxsw_sp_port->learning);
+	if (err) {
+		netdev_err(dev, "Failed to set learning for VIDs %d-%d\n",
+			   vid_begin, vid_end);
+		goto err_port_vid_learning_set;
+	}
+
 	/* Changing activity bits only if HW operation succeded */
 	for (vid = vid_begin; vid <= vid_end; vid++) {
 		set_bit(vid, mlxsw_sp_port->active_vlans);
@@ -695,6 +763,9 @@ static int __mlxsw_sp_port_vlans_add(struct mlxsw_sp_port *mlxsw_sp_port,
 err_port_stp_state_set:
 	for (vid = vid_begin; vid <= vid_end; vid++)
 		clear_bit(vid, mlxsw_sp_port->active_vlans);
+	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid_begin, vid_end,
+				       false);
+err_port_vid_learning_set:
 	if (old_pvid != mlxsw_sp_port->pvid)
 		mlxsw_sp_port_pvid_set(mlxsw_sp_port, old_pvid);
 err_port_pvid_set:
@@ -973,11 +1044,6 @@ static int mlxsw_sp_port_obj_add(struct net_device *dev,
 					      SWITCHDEV_OBJ_PORT_VLAN(obj),
 					      trans);
 		break;
-	case SWITCHDEV_OBJ_ID_IPV4_FIB:
-		err = mlxsw_sp_router_fib4_add(mlxsw_sp_port,
-					       SWITCHDEV_OBJ_IPV4_FIB(obj),
-					       trans);
-		break;
 	case SWITCHDEV_OBJ_ID_PORT_FDB:
 		err = mlxsw_sp_port_fdb_static_add(mlxsw_sp_port,
 						   SWITCHDEV_OBJ_PORT_FDB(obj),
@@ -997,38 +1063,25 @@ static int mlxsw_sp_port_obj_add(struct net_device *dev,
 }
 
 static int __mlxsw_sp_port_vlans_del(struct mlxsw_sp_port *mlxsw_sp_port,
-				     u16 vid_begin, u16 vid_end, bool init)
+				     u16 vid_begin, u16 vid_end)
 {
-	struct net_device *dev = mlxsw_sp_port->dev;
 	u16 vid, pvid;
-	int err;
 
-	if (!init && !mlxsw_sp_port->bridged)
+	if (!mlxsw_sp_port->bridged)
 		return -EINVAL;
 
-	err = __mlxsw_sp_port_vlans_set(mlxsw_sp_port, vid_begin, vid_end,
-					false, false);
-	if (err) {
-		netdev_err(dev, "Unable to del VIDs %d-%d\n", vid_begin,
-			   vid_end);
-		return err;
-	}
-
-	if (init)
-		goto out;
+	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid_begin, vid_end,
+				       false);
 
 	pvid = mlxsw_sp_port->pvid;
-	if (pvid >= vid_begin && pvid <= vid_end) {
-		err = mlxsw_sp_port_pvid_set(mlxsw_sp_port, 0);
-		if (err) {
-			netdev_err(dev, "Unable to del PVID %d\n", pvid);
-			return err;
-		}
-	}
+	if (pvid >= vid_begin && pvid <= vid_end)
+		mlxsw_sp_port_pvid_set(mlxsw_sp_port, 0);
+
+	__mlxsw_sp_port_vlans_set(mlxsw_sp_port, vid_begin, vid_end, false,
+				  false);
 
 	mlxsw_sp_port_fid_leave(mlxsw_sp_port, vid_begin, vid_end);
 
-out:
 	/* Changing activity bits only if HW operation succeded */
 	for (vid = vid_begin; vid <= vid_end; vid++)
 		clear_bit(vid, mlxsw_sp_port->active_vlans);
@@ -1039,8 +1092,8 @@ out:
 static int mlxsw_sp_port_vlans_del(struct mlxsw_sp_port *mlxsw_sp_port,
 				   const struct switchdev_obj_port_vlan *vlan)
 {
-	return __mlxsw_sp_port_vlans_del(mlxsw_sp_port,
-					 vlan->vid_begin, vlan->vid_end, false);
+	return __mlxsw_sp_port_vlans_del(mlxsw_sp_port, vlan->vid_begin,
+					 vlan->vid_end);
 }
 
 void mlxsw_sp_port_active_vlans_del(struct mlxsw_sp_port *mlxsw_sp_port)
@@ -1048,7 +1101,7 @@ void mlxsw_sp_port_active_vlans_del(struct mlxsw_sp_port *mlxsw_sp_port)
 	u16 vid;
 
 	for_each_set_bit(vid, mlxsw_sp_port->active_vlans, VLAN_N_VID)
-		__mlxsw_sp_port_vlans_del(mlxsw_sp_port, vid, vid, false);
+		__mlxsw_sp_port_vlans_del(mlxsw_sp_port, vid, vid);
 }
 
 static int
@@ -1123,10 +1176,6 @@ static int mlxsw_sp_port_obj_del(struct net_device *dev,
 		err = mlxsw_sp_port_vlans_del(mlxsw_sp_port,
 					      SWITCHDEV_OBJ_PORT_VLAN(obj));
 		break;
-	case SWITCHDEV_OBJ_ID_IPV4_FIB:
-		err = mlxsw_sp_router_fib4_del(mlxsw_sp_port,
-					       SWITCHDEV_OBJ_IPV4_FIB(obj));
-		break;
 	case SWITCHDEV_OBJ_ID_PORT_FDB:
 		err = mlxsw_sp_port_fdb_static_del(mlxsw_sp_port,
 						   SWITCHDEV_OBJ_PORT_FDB(obj));
@@ -1147,9 +1196,11 @@ static struct mlxsw_sp_port *mlxsw_sp_lag_rep_port(struct mlxsw_sp *mlxsw_sp,
 						   u16 lag_id)
 {
 	struct mlxsw_sp_port *mlxsw_sp_port;
+	struct mlxsw_resources *resources;
 	int i;
 
-	for (i = 0; i < MLXSW_SP_PORT_PER_LAG_MAX; i++) {
+	resources = mlxsw_core_resources_get(mlxsw_sp->core);
+	for (i = 0; i < resources->max_ports_in_lag; i++) {
 		mlxsw_sp_port = mlxsw_sp_port_lagged_get(mlxsw_sp, lag_id, i);
 		if (mlxsw_sp_port)
 			return mlxsw_sp_port;
@@ -1368,8 +1419,6 @@ static void mlxsw_sp_fdb_notify_mac_process(struct mlxsw_sp *mlxsw_sp,
 		vid = fid;
 	}
 
-	adding = adding && mlxsw_sp_port->learning;
-
 do_fdb_op:
 	err = mlxsw_sp_port_fdb_uc_op(mlxsw_sp, local_port, mac, fid,
 				      adding, true);
@@ -1431,8 +1480,6 @@ static void mlxsw_sp_fdb_notify_mac_lag_process(struct mlxsw_sp *mlxsw_sp,
 		vid = fid;
 	}
 
-	adding = adding && mlxsw_sp_port->learning;
-
 do_fdb_op:
 	err = mlxsw_sp_port_fdb_uc_lag_op(mlxsw_sp, lag_id, mac, fid, lag_vid,
 					  adding, true);
@@ -1498,20 +1545,18 @@ static void mlxsw_sp_fdb_notify_work(struct work_struct *work)
 	mlxsw_sp = container_of(work, struct mlxsw_sp, fdb_notify.dw.work);
 
 	rtnl_lock();
-	do {
-		mlxsw_reg_sfn_pack(sfn_pl);
-		err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(sfn), sfn_pl);
-		if (err) {
-			dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get FDB notifications\n");
-			break;
-		}
-		num_rec = mlxsw_reg_sfn_num_rec_get(sfn_pl);
-		for (i = 0; i < num_rec; i++)
-			mlxsw_sp_fdb_notify_rec_process(mlxsw_sp, sfn_pl, i);
+	mlxsw_reg_sfn_pack(sfn_pl);
+	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(sfn), sfn_pl);
+	if (err) {
+		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get FDB notifications\n");
+		goto out;
+	}
+	num_rec = mlxsw_reg_sfn_num_rec_get(sfn_pl);
+	for (i = 0; i < num_rec; i++)
+		mlxsw_sp_fdb_notify_rec_process(mlxsw_sp, sfn_pl, i);
 
-	} while (num_rec);
+out:
 	rtnl_unlock();
-
 	kfree(sfn_pl);
 	mlxsw_sp_fdb_notify_work_schedule(mlxsw_sp);
 }
@@ -1546,32 +1591,6 @@ void mlxsw_sp_switchdev_fini(struct mlxsw_sp *mlxsw_sp)
 	mlxsw_sp_fdb_fini(mlxsw_sp);
 }
 
-int mlxsw_sp_port_vlan_init(struct mlxsw_sp_port *mlxsw_sp_port)
-{
-	struct net_device *dev = mlxsw_sp_port->dev;
-	int err;
-
-	/* Allow only untagged packets to ingress and tag them internally
-	 * with VID 1.
-	 */
-	mlxsw_sp_port->pvid = 1;
-	err = __mlxsw_sp_port_vlans_del(mlxsw_sp_port, 0, VLAN_N_VID - 1,
-					true);
-	if (err) {
-		netdev_err(dev, "Unable to init VLANs\n");
-		return err;
-	}
-
-	/* Add implicit VLAN interface in the device, so that untagged
-	 * packets will be classified to the default vFID.
-	 */
-	err = mlxsw_sp_port_add_vid(dev, 0, 1);
-	if (err)
-		netdev_err(dev, "Failed to configure default vFID\n");
-
-	return err;
-}
-
 void mlxsw_sp_port_switchdev_init(struct mlxsw_sp_port *mlxsw_sp_port)
 {
 	mlxsw_sp_port->dev->switchdev_ops = &mlxsw_sp_port_switchdev_ops;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c
index 377daa4d509c..c0c23e2f3275 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c
@@ -997,7 +997,7 @@ static int mlxsw_sx_port_create(struct mlxsw_sx *mlxsw_sx, u8 local_port)
 	/* Each packet needs to have a Tx header (metadata) on top all other
 	 * headers.
 	 */
-	dev->hard_header_len += MLXSW_TXHDR_LEN;
+	dev->needed_headroom = MLXSW_TXHDR_LEN;
 
 	err = mlxsw_sx_port_module_check(mlxsw_sx_port, &usable);
 	if (err) {
@@ -1512,10 +1512,6 @@ static void mlxsw_sx_fini(struct mlxsw_core *mlxsw_core)
 static struct mlxsw_config_profile mlxsw_sx_config_profile = {
 	.used_max_vepa_channels		= 1,
 	.max_vepa_channels		= 0,
-	.used_max_lag			= 1,
-	.max_lag			= 64,
-	.used_max_port_per_lag		= 1,
-	.max_port_per_lag		= 16,
 	.used_max_mid			= 1,
 	.max_mid			= 7000,
 	.used_max_pgt			= 1,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/trap.h b/drivers/net/ethernet/mellanox/mlxsw/trap.h
index 470d7696e9fe..ed8e30186400 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/trap.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/trap.h
@@ -56,6 +56,10 @@ enum {
 	MLXSW_TRAP_ID_IGMP_V3_REPORT = 0x34,
 	MLXSW_TRAP_ID_ARPBC = 0x50,
 	MLXSW_TRAP_ID_ARPUC = 0x51,
+	MLXSW_TRAP_ID_MTUERROR = 0x52,
+	MLXSW_TRAP_ID_TTLERROR = 0x53,
+	MLXSW_TRAP_ID_LBERROR = 0x54,
+	MLXSW_TRAP_ID_OSPF = 0x55,
 	MLXSW_TRAP_ID_IP2ME = 0x5F,
 	MLXSW_TRAP_ID_RTR_INGRESS0 = 0x70,
 	MLXSW_TRAP_ID_HOST_MISS_IPV4 = 0x90,
diff --git a/drivers/net/ethernet/microchip/encx24j600.c b/drivers/net/ethernet/microchip/encx24j600.c
index 42e34076d2de..b14f0305aa31 100644
--- a/drivers/net/ethernet/microchip/encx24j600.c
+++ b/drivers/net/ethernet/microchip/encx24j600.c
@@ -821,7 +821,7 @@ static void encx24j600_set_multicast_list(struct net_device *dev)
 	}
 
 	if (oldfilter != priv->rxfilter)
-		queue_kthread_work(&priv->kworker, &priv->setrx_work);
+		kthread_queue_work(&priv->kworker, &priv->setrx_work);
 }
 
 static void encx24j600_hw_tx(struct encx24j600_priv *priv)
@@ -879,7 +879,7 @@ static netdev_tx_t encx24j600_tx(struct sk_buff *skb, struct net_device *dev)
 	/* Remember the skb for deferred processing */
 	priv->tx_skb = skb;
 
-	queue_kthread_work(&priv->kworker, &priv->tx_work);
+	kthread_queue_work(&priv->kworker, &priv->tx_work);
 
 	return NETDEV_TX_OK;
 }
@@ -1037,9 +1037,9 @@ static int encx24j600_spi_probe(struct spi_device *spi)
 		goto out_free;
 	}
 
-	init_kthread_worker(&priv->kworker);
-	init_kthread_work(&priv->tx_work, encx24j600_tx_proc);
-	init_kthread_work(&priv->setrx_work, encx24j600_setrx_proc);
+	kthread_init_worker(&priv->kworker);
+	kthread_init_work(&priv->tx_work, encx24j600_tx_proc);
+	kthread_init_work(&priv->setrx_work, encx24j600_setrx_proc);
 
 	priv->kworker_task = kthread_run(kthread_worker_fn, &priv->kworker,
 					 "encx24j600");
diff --git a/drivers/net/ethernet/natsemi/ns83820.c b/drivers/net/ethernet/natsemi/ns83820.c
index eb807b0dc72a..569ade6cf85c 100644
--- a/drivers/net/ethernet/natsemi/ns83820.c
+++ b/drivers/net/ethernet/natsemi/ns83820.c
@@ -134,7 +134,7 @@ static int lnksts = 0;		/* CFG_LNKSTS bit polarity */
 
 /* tunables */
 #define RX_BUF_SIZE	1500	/* 8192 */
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
 #define NS83820_VLAN_ACCEL_SUPPORT
 #endif
 
diff --git a/drivers/net/ethernet/netronome/nfp/Makefile b/drivers/net/ethernet/netronome/nfp/Makefile
index 68178819ff12..0efb2ba9a558 100644
--- a/drivers/net/ethernet/netronome/nfp/Makefile
+++ b/drivers/net/ethernet/netronome/nfp/Makefile
@@ -3,6 +3,13 @@ obj-$(CONFIG_NFP_NETVF)	+= nfp_netvf.o
 nfp_netvf-objs := \
 	    nfp_net_common.o \
 	    nfp_net_ethtool.o \
+	    nfp_net_offload.o \
 	    nfp_netvf_main.o
 
+ifeq ($(CONFIG_BPF_SYSCALL),y)
+nfp_netvf-objs += \
+	    nfp_bpf_verifier.o \
+	    nfp_bpf_jit.o
+endif
+
 nfp_netvf-$(CONFIG_NFP_NET_DEBUG) += nfp_net_debugfs.o
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_asm.h b/drivers/net/ethernet/netronome/nfp/nfp_asm.h
new file mode 100644
index 000000000000..22484b6fd3e8
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_asm.h
@@ -0,0 +1,233 @@
+/*
+ * Copyright (C) 2016 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __NFP_ASM_H__
+#define __NFP_ASM_H__ 1
+
+#include "nfp_bpf.h"
+
+#define REG_NONE	0
+
+#define RE_REG_NO_DST	0x020
+#define RE_REG_IMM	0x020
+#define RE_REG_IMM_encode(x)					\
+	(RE_REG_IMM | ((x) & 0x1f) | (((x) & 0x60) << 1))
+#define RE_REG_IMM_MAX	 0x07fULL
+#define RE_REG_XFR	0x080
+
+#define UR_REG_XFR	0x180
+#define UR_REG_NN	0x280
+#define UR_REG_NO_DST	0x300
+#define UR_REG_IMM	UR_REG_NO_DST
+#define UR_REG_IMM_encode(x) (UR_REG_IMM | (x))
+#define UR_REG_IMM_MAX	 0x0ffULL
+
+#define OP_BR_BASE	0x0d800000020ULL
+#define OP_BR_BASE_MASK	0x0f8000c3ce0ULL
+#define OP_BR_MASK	0x0000000001fULL
+#define OP_BR_EV_PIP	0x00000000300ULL
+#define OP_BR_CSS	0x0000003c000ULL
+#define OP_BR_DEFBR	0x00000300000ULL
+#define OP_BR_ADDR_LO	0x007ffc00000ULL
+#define OP_BR_ADDR_HI	0x10000000000ULL
+
+#define nfp_is_br(_insn)				\
+	(((_insn) & OP_BR_BASE_MASK) == OP_BR_BASE)
+
+enum br_mask {
+	BR_BEQ = 0x00,
+	BR_BNE = 0x01,
+	BR_BHS = 0x04,
+	BR_BLO = 0x05,
+	BR_BGE = 0x08,
+	BR_UNC = 0x18,
+};
+
+enum br_ev_pip {
+	BR_EV_PIP_UNCOND = 0,
+	BR_EV_PIP_COND = 1,
+};
+
+enum br_ctx_signal_state {
+	BR_CSS_NONE = 2,
+};
+
+#define OP_BBYTE_BASE	0x0c800000000ULL
+#define OP_BB_A_SRC	0x000000000ffULL
+#define OP_BB_BYTE	0x00000000300ULL
+#define OP_BB_B_SRC	0x0000003fc00ULL
+#define OP_BB_I8	0x00000040000ULL
+#define OP_BB_EQ	0x00000080000ULL
+#define OP_BB_DEFBR	0x00000300000ULL
+#define OP_BB_ADDR_LO	0x007ffc00000ULL
+#define OP_BB_ADDR_HI	0x10000000000ULL
+
+#define OP_BALU_BASE	0x0e800000000ULL
+#define OP_BA_A_SRC	0x000000003ffULL
+#define OP_BA_B_SRC	0x000000ffc00ULL
+#define OP_BA_DEFBR	0x00000300000ULL
+#define OP_BA_ADDR_HI	0x0007fc00000ULL
+
+#define OP_IMMED_A_SRC	0x000000003ffULL
+#define OP_IMMED_B_SRC	0x000000ffc00ULL
+#define OP_IMMED_IMM	0x0000ff00000ULL
+#define OP_IMMED_WIDTH	0x00060000000ULL
+#define OP_IMMED_INV	0x00080000000ULL
+#define OP_IMMED_SHIFT	0x00600000000ULL
+#define OP_IMMED_BASE	0x0f000000000ULL
+#define OP_IMMED_WR_AB	0x20000000000ULL
+
+enum immed_width {
+	IMMED_WIDTH_ALL = 0,
+	IMMED_WIDTH_BYTE = 1,
+	IMMED_WIDTH_WORD = 2,
+};
+
+enum immed_shift {
+	IMMED_SHIFT_0B = 0,
+	IMMED_SHIFT_1B = 1,
+	IMMED_SHIFT_2B = 2,
+};
+
+#define OP_SHF_BASE	0x08000000000ULL
+#define OP_SHF_A_SRC	0x000000000ffULL
+#define OP_SHF_SC	0x00000000300ULL
+#define OP_SHF_B_SRC	0x0000003fc00ULL
+#define OP_SHF_I8	0x00000040000ULL
+#define OP_SHF_SW	0x00000080000ULL
+#define OP_SHF_DST	0x0000ff00000ULL
+#define OP_SHF_SHIFT	0x001f0000000ULL
+#define OP_SHF_OP	0x00e00000000ULL
+#define OP_SHF_DST_AB	0x01000000000ULL
+#define OP_SHF_WR_AB	0x20000000000ULL
+
+enum shf_op {
+	SHF_OP_NONE = 0,
+	SHF_OP_AND = 2,
+	SHF_OP_OR = 5,
+};
+
+enum shf_sc {
+	SHF_SC_R_ROT = 0,
+	SHF_SC_R_SHF = 1,
+	SHF_SC_L_SHF = 2,
+	SHF_SC_R_DSHF = 3,
+};
+
+#define OP_ALU_A_SRC	0x000000003ffULL
+#define OP_ALU_B_SRC	0x000000ffc00ULL
+#define OP_ALU_DST	0x0003ff00000ULL
+#define OP_ALU_SW	0x00040000000ULL
+#define OP_ALU_OP	0x00f80000000ULL
+#define OP_ALU_DST_AB	0x01000000000ULL
+#define OP_ALU_BASE	0x0a000000000ULL
+#define OP_ALU_WR_AB	0x20000000000ULL
+
+enum alu_op {
+	ALU_OP_NONE	= 0x00,
+	ALU_OP_ADD	= 0x01,
+	ALU_OP_NEG	= 0x04,
+	ALU_OP_AND	= 0x08,
+	ALU_OP_SUB_C	= 0x0d,
+	ALU_OP_ADD_C	= 0x11,
+	ALU_OP_OR	= 0x14,
+	ALU_OP_SUB	= 0x15,
+	ALU_OP_XOR	= 0x18,
+};
+
+enum alu_dst_ab {
+	ALU_DST_A = 0,
+	ALU_DST_B = 1,
+};
+
+#define OP_LDF_BASE	0x0c000000000ULL
+#define OP_LDF_A_SRC	0x000000000ffULL
+#define OP_LDF_SC	0x00000000300ULL
+#define OP_LDF_B_SRC	0x0000003fc00ULL
+#define OP_LDF_I8	0x00000040000ULL
+#define OP_LDF_SW	0x00000080000ULL
+#define OP_LDF_ZF	0x00000100000ULL
+#define OP_LDF_BMASK	0x0000f000000ULL
+#define OP_LDF_SHF	0x001f0000000ULL
+#define OP_LDF_WR_AB	0x20000000000ULL
+
+#define OP_CMD_A_SRC	 0x000000000ffULL
+#define OP_CMD_CTX	 0x00000000300ULL
+#define OP_CMD_B_SRC	 0x0000003fc00ULL
+#define OP_CMD_TOKEN	 0x000000c0000ULL
+#define OP_CMD_XFER	 0x00001f00000ULL
+#define OP_CMD_CNT	 0x0000e000000ULL
+#define OP_CMD_SIG	 0x000f0000000ULL
+#define OP_CMD_TGT_CMD	 0x07f00000000ULL
+#define OP_CMD_MODE	0x1c0000000000ULL
+
+struct cmd_tgt_act {
+	u8 token;
+	u8 tgt_cmd;
+};
+
+enum cmd_tgt_map {
+	CMD_TGT_READ8,
+	CMD_TGT_WRITE8,
+	CMD_TGT_READ_LE,
+	CMD_TGT_READ_SWAP_LE,
+	__CMD_TGT_MAP_SIZE,
+};
+
+enum cmd_mode {
+	CMD_MODE_40b_AB	= 0,
+	CMD_MODE_40b_BA	= 1,
+	CMD_MODE_32b	= 4,
+};
+
+enum cmd_ctx_swap {
+	CMD_CTX_SWAP = 0,
+	CMD_CTX_NO_SWAP = 3,
+};
+
+#define OP_LCSR_BASE	0x0fc00000000ULL
+#define OP_LCSR_A_SRC	0x000000003ffULL
+#define OP_LCSR_B_SRC	0x000000ffc00ULL
+#define OP_LCSR_WRITE	0x00000200000ULL
+#define OP_LCSR_ADDR	0x001ffc00000ULL
+
+enum lcsr_wr_src {
+	LCSR_WR_AREG,
+	LCSR_WR_BREG,
+	LCSR_WR_IMM,
+};
+
+#define OP_CARB_BASE	0x0e000000000ULL
+#define OP_CARB_OR	0x00000010000ULL
+
+#endif
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_bpf.h b/drivers/net/ethernet/netronome/nfp/nfp_bpf.h
new file mode 100644
index 000000000000..87aa8a3e9112
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_bpf.h
@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) 2016 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __NFP_BPF_H__
+#define __NFP_BPF_H__ 1
+
+#include <linux/bitfield.h>
+#include <linux/bpf.h>
+#include <linux/list.h>
+#include <linux/types.h>
+
+#define FIELD_FIT(mask, val)  (!((((u64)val) << __bf_shf(mask)) & ~(mask)))
+
+/* For branch fixup logic use up-most byte of branch instruction as scratch
+ * area.  Remember to clear this before sending instructions to HW!
+ */
+#define OP_BR_SPECIAL	0xff00000000000000ULL
+
+enum br_special {
+	OP_BR_NORMAL = 0,
+	OP_BR_GO_OUT,
+	OP_BR_GO_ABORT,
+};
+
+enum static_regs {
+	STATIC_REG_PKT		= 1,
+#define REG_PKT_BANK	ALU_DST_A
+	STATIC_REG_IMM		= 2, /* Bank AB */
+};
+
+enum nfp_bpf_action_type {
+	NN_ACT_TC_DROP,
+	NN_ACT_TC_REDIR,
+	NN_ACT_DIRECT,
+};
+
+/* Software register representation, hardware encoding in asm.h */
+#define NN_REG_TYPE	GENMASK(31, 24)
+#define NN_REG_VAL	GENMASK(7, 0)
+
+enum nfp_bpf_reg_type {
+	NN_REG_GPR_A =	BIT(0),
+	NN_REG_GPR_B =	BIT(1),
+	NN_REG_NNR =	BIT(2),
+	NN_REG_XFER =	BIT(3),
+	NN_REG_IMM =	BIT(4),
+	NN_REG_NONE =	BIT(5),
+};
+
+#define NN_REG_GPR_BOTH	(NN_REG_GPR_A | NN_REG_GPR_B)
+
+#define reg_both(x)	((x) | FIELD_PREP(NN_REG_TYPE, NN_REG_GPR_BOTH))
+#define reg_a(x)	((x) | FIELD_PREP(NN_REG_TYPE, NN_REG_GPR_A))
+#define reg_b(x)	((x) | FIELD_PREP(NN_REG_TYPE, NN_REG_GPR_B))
+#define reg_nnr(x)	((x) | FIELD_PREP(NN_REG_TYPE, NN_REG_NNR))
+#define reg_xfer(x)	((x) | FIELD_PREP(NN_REG_TYPE, NN_REG_XFER))
+#define reg_imm(x)	((x) | FIELD_PREP(NN_REG_TYPE, NN_REG_IMM))
+#define reg_none()	(FIELD_PREP(NN_REG_TYPE, NN_REG_NONE))
+
+#define pkt_reg(np)	reg_a((np)->regs_per_thread - STATIC_REG_PKT)
+#define imm_a(np)	reg_a((np)->regs_per_thread - STATIC_REG_IMM)
+#define imm_b(np)	reg_b((np)->regs_per_thread - STATIC_REG_IMM)
+#define imm_both(np)	reg_both((np)->regs_per_thread - STATIC_REG_IMM)
+
+#define NFP_BPF_ABI_FLAGS	reg_nnr(0)
+#define   NFP_BPF_ABI_FLAG_MARK	1
+#define NFP_BPF_ABI_MARK	reg_nnr(1)
+#define NFP_BPF_ABI_PKT		reg_nnr(2)
+#define NFP_BPF_ABI_LEN		reg_nnr(3)
+
+struct nfp_prog;
+struct nfp_insn_meta;
+typedef int (*instr_cb_t)(struct nfp_prog *, struct nfp_insn_meta *);
+
+#define nfp_prog_first_meta(nfp_prog)					\
+	list_first_entry(&(nfp_prog)->insns, struct nfp_insn_meta, l)
+#define nfp_prog_last_meta(nfp_prog)					\
+	list_last_entry(&(nfp_prog)->insns, struct nfp_insn_meta, l)
+#define nfp_meta_next(meta)	list_next_entry(meta, l)
+#define nfp_meta_prev(meta)	list_prev_entry(meta, l)
+
+/**
+ * struct nfp_insn_meta - BPF instruction wrapper
+ * @insn: BPF instruction
+ * @off: index of first generated machine instruction (in nfp_prog.prog)
+ * @n: eBPF instruction number
+ * @skip: skip this instruction (optimized out)
+ * @double_cb: callback for second part of the instruction
+ * @l: link on nfp_prog->insns list
+ */
+struct nfp_insn_meta {
+	struct bpf_insn insn;
+	unsigned int off;
+	unsigned short n;
+	bool skip;
+	instr_cb_t double_cb;
+
+	struct list_head l;
+};
+
+#define BPF_SIZE_MASK	0x18
+
+static inline u8 mbpf_class(const struct nfp_insn_meta *meta)
+{
+	return BPF_CLASS(meta->insn.code);
+}
+
+static inline u8 mbpf_src(const struct nfp_insn_meta *meta)
+{
+	return BPF_SRC(meta->insn.code);
+}
+
+static inline u8 mbpf_op(const struct nfp_insn_meta *meta)
+{
+	return BPF_OP(meta->insn.code);
+}
+
+static inline u8 mbpf_mode(const struct nfp_insn_meta *meta)
+{
+	return BPF_MODE(meta->insn.code);
+}
+
+/**
+ * struct nfp_prog - nfp BPF program
+ * @prog: machine code
+ * @prog_len: number of valid instructions in @prog array
+ * @__prog_alloc_len: alloc size of @prog array
+ * @act: BPF program/action type (TC DA, TC with action, XDP etc.)
+ * @num_regs: number of registers used by this program
+ * @regs_per_thread: number of basic registers allocated per thread
+ * @start_off: address of the first instruction in the memory
+ * @tgt_out: jump target for normal exit
+ * @tgt_abort: jump target for abort (e.g. access outside of packet buffer)
+ * @tgt_done: jump target to get the next packet
+ * @n_translated: number of successfully translated instructions (for errors)
+ * @error: error code if something went wrong
+ * @insns: list of BPF instruction wrappers (struct nfp_insn_meta)
+ */
+struct nfp_prog {
+	u64 *prog;
+	unsigned int prog_len;
+	unsigned int __prog_alloc_len;
+
+	enum nfp_bpf_action_type act;
+
+	unsigned int num_regs;
+	unsigned int regs_per_thread;
+
+	unsigned int start_off;
+	unsigned int tgt_out;
+	unsigned int tgt_abort;
+	unsigned int tgt_done;
+
+	unsigned int n_translated;
+	int error;
+
+	struct list_head insns;
+};
+
+struct nfp_bpf_result {
+	unsigned int n_instr;
+	bool dense_mode;
+};
+
+int
+nfp_bpf_jit(struct bpf_prog *filter, void *prog, enum nfp_bpf_action_type act,
+	    unsigned int prog_start, unsigned int prog_done,
+	    unsigned int prog_sz, struct nfp_bpf_result *res);
+
+int nfp_prog_verify(struct nfp_prog *nfp_prog, struct bpf_prog *prog);
+
+#endif
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_bpf_jit.c b/drivers/net/ethernet/netronome/nfp/nfp_bpf_jit.c
new file mode 100644
index 000000000000..f8df5300f49c
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_bpf_jit.c
@@ -0,0 +1,1813 @@
+/*
+ * Copyright (C) 2016 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define pr_fmt(fmt)	"NFP net bpf: " fmt
+
+#include <linux/kernel.h>
+#include <linux/bpf.h>
+#include <linux/filter.h>
+#include <linux/pkt_cls.h>
+#include <linux/unistd.h>
+
+#include "nfp_asm.h"
+#include "nfp_bpf.h"
+
+/* --- NFP prog --- */
+/* Foreach "multiple" entries macros provide pos and next<n> pointers.
+ * It's safe to modify the next pointers (but not pos).
+ */
+#define nfp_for_each_insn_walk2(nfp_prog, pos, next)			\
+	for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
+	     next = list_next_entry(pos, l);			\
+	     &(nfp_prog)->insns != &pos->l &&			\
+	     &(nfp_prog)->insns != &next->l;			\
+	     pos = nfp_meta_next(pos),				\
+	     next = nfp_meta_next(pos))
+
+#define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2)		\
+	for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
+	     next = list_next_entry(pos, l),			\
+	     next2 = list_next_entry(next, l);			\
+	     &(nfp_prog)->insns != &pos->l &&			\
+	     &(nfp_prog)->insns != &next->l &&			\
+	     &(nfp_prog)->insns != &next2->l;			\
+	     pos = nfp_meta_next(pos),				\
+	     next = nfp_meta_next(pos),				\
+	     next2 = nfp_meta_next(next))
+
+static bool
+nfp_meta_has_next(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return meta->l.next != &nfp_prog->insns;
+}
+
+static bool
+nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return meta->l.prev != &nfp_prog->insns;
+}
+
+static void nfp_prog_free(struct nfp_prog *nfp_prog)
+{
+	struct nfp_insn_meta *meta, *tmp;
+
+	list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
+		list_del(&meta->l);
+		kfree(meta);
+	}
+	kfree(nfp_prog);
+}
+
+static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)
+{
+	if (nfp_prog->__prog_alloc_len == nfp_prog->prog_len) {
+		nfp_prog->error = -ENOSPC;
+		return;
+	}
+
+	nfp_prog->prog[nfp_prog->prog_len] = insn;
+	nfp_prog->prog_len++;
+}
+
+static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
+{
+	return nfp_prog->start_off + nfp_prog->prog_len;
+}
+
+static unsigned int
+nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset)
+{
+	return offset - nfp_prog->start_off;
+}
+
+/* --- SW reg --- */
+struct nfp_insn_ur_regs {
+	enum alu_dst_ab dst_ab;
+	u16 dst;
+	u16 areg, breg;
+	bool swap;
+	bool wr_both;
+};
+
+struct nfp_insn_re_regs {
+	enum alu_dst_ab dst_ab;
+	u8 dst;
+	u8 areg, breg;
+	bool swap;
+	bool wr_both;
+	bool i8;
+};
+
+static u16 nfp_swreg_to_unreg(u32 swreg, bool is_dst)
+{
+	u16 val = FIELD_GET(NN_REG_VAL, swreg);
+
+	switch (FIELD_GET(NN_REG_TYPE, swreg)) {
+	case NN_REG_GPR_A:
+	case NN_REG_GPR_B:
+	case NN_REG_GPR_BOTH:
+		return val;
+	case NN_REG_NNR:
+		return UR_REG_NN | val;
+	case NN_REG_XFER:
+		return UR_REG_XFR | val;
+	case NN_REG_IMM:
+		if (val & ~0xff) {
+			pr_err("immediate too large\n");
+			return 0;
+		}
+		return UR_REG_IMM_encode(val);
+	case NN_REG_NONE:
+		return is_dst ? UR_REG_NO_DST : REG_NONE;
+	default:
+		pr_err("unrecognized reg encoding %08x\n", swreg);
+		return 0;
+	}
+}
+
+static int
+swreg_to_unrestricted(u32 dst, u32 lreg, u32 rreg, struct nfp_insn_ur_regs *reg)
+{
+	memset(reg, 0, sizeof(*reg));
+
+	/* Decode destination */
+	if (FIELD_GET(NN_REG_TYPE, dst) == NN_REG_IMM)
+		return -EFAULT;
+
+	if (FIELD_GET(NN_REG_TYPE, dst) == NN_REG_GPR_B)
+		reg->dst_ab = ALU_DST_B;
+	if (FIELD_GET(NN_REG_TYPE, dst) == NN_REG_GPR_BOTH)
+		reg->wr_both = true;
+	reg->dst = nfp_swreg_to_unreg(dst, true);
+
+	/* Decode source operands */
+	if (FIELD_GET(NN_REG_TYPE, lreg) == FIELD_GET(NN_REG_TYPE, rreg))
+		return -EFAULT;
+
+	if (FIELD_GET(NN_REG_TYPE, lreg) == NN_REG_GPR_B ||
+	    FIELD_GET(NN_REG_TYPE, rreg) == NN_REG_GPR_A) {
+		reg->areg = nfp_swreg_to_unreg(rreg, false);
+		reg->breg = nfp_swreg_to_unreg(lreg, false);
+		reg->swap = true;
+	} else {
+		reg->areg = nfp_swreg_to_unreg(lreg, false);
+		reg->breg = nfp_swreg_to_unreg(rreg, false);
+	}
+
+	return 0;
+}
+
+static u16 nfp_swreg_to_rereg(u32 swreg, bool is_dst, bool has_imm8, bool *i8)
+{
+	u16 val = FIELD_GET(NN_REG_VAL, swreg);
+
+	switch (FIELD_GET(NN_REG_TYPE, swreg)) {
+	case NN_REG_GPR_A:
+	case NN_REG_GPR_B:
+	case NN_REG_GPR_BOTH:
+		return val;
+	case NN_REG_XFER:
+		return RE_REG_XFR | val;
+	case NN_REG_IMM:
+		if (val & ~(0x7f | has_imm8 << 7)) {
+			pr_err("immediate too large\n");
+			return 0;
+		}
+		*i8 = val & 0x80;
+		return RE_REG_IMM_encode(val & 0x7f);
+	case NN_REG_NONE:
+		return is_dst ? RE_REG_NO_DST : REG_NONE;
+	default:
+		pr_err("unrecognized reg encoding\n");
+		return 0;
+	}
+}
+
+static int
+swreg_to_restricted(u32 dst, u32 lreg, u32 rreg, struct nfp_insn_re_regs *reg,
+		    bool has_imm8)
+{
+	memset(reg, 0, sizeof(*reg));
+
+	/* Decode destination */
+	if (FIELD_GET(NN_REG_TYPE, dst) == NN_REG_IMM)
+		return -EFAULT;
+
+	if (FIELD_GET(NN_REG_TYPE, dst) == NN_REG_GPR_B)
+		reg->dst_ab = ALU_DST_B;
+	if (FIELD_GET(NN_REG_TYPE, dst) == NN_REG_GPR_BOTH)
+		reg->wr_both = true;
+	reg->dst = nfp_swreg_to_rereg(dst, true, false, NULL);
+
+	/* Decode source operands */
+	if (FIELD_GET(NN_REG_TYPE, lreg) == FIELD_GET(NN_REG_TYPE, rreg))
+		return -EFAULT;
+
+	if (FIELD_GET(NN_REG_TYPE, lreg) == NN_REG_GPR_B ||
+	    FIELD_GET(NN_REG_TYPE, rreg) == NN_REG_GPR_A) {
+		reg->areg = nfp_swreg_to_rereg(rreg, false, has_imm8, &reg->i8);
+		reg->breg = nfp_swreg_to_rereg(lreg, false, has_imm8, &reg->i8);
+		reg->swap = true;
+	} else {
+		reg->areg = nfp_swreg_to_rereg(lreg, false, has_imm8, &reg->i8);
+		reg->breg = nfp_swreg_to_rereg(rreg, false, has_imm8, &reg->i8);
+	}
+
+	return 0;
+}
+
+/* --- Emitters --- */
+static const struct cmd_tgt_act cmd_tgt_act[__CMD_TGT_MAP_SIZE] = {
+	[CMD_TGT_WRITE8] =		{ 0x00, 0x42 },
+	[CMD_TGT_READ8] =		{ 0x01, 0x43 },
+	[CMD_TGT_READ_LE] =		{ 0x01, 0x40 },
+	[CMD_TGT_READ_SWAP_LE] =	{ 0x03, 0x40 },
+};
+
+static void
+__emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
+	   u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, bool sync)
+{
+	enum cmd_ctx_swap ctx;
+	u64 insn;
+
+	if (sync)
+		ctx = CMD_CTX_SWAP;
+	else
+		ctx = CMD_CTX_NO_SWAP;
+
+	insn =	FIELD_PREP(OP_CMD_A_SRC, areg) |
+		FIELD_PREP(OP_CMD_CTX, ctx) |
+		FIELD_PREP(OP_CMD_B_SRC, breg) |
+		FIELD_PREP(OP_CMD_TOKEN, cmd_tgt_act[op].token) |
+		FIELD_PREP(OP_CMD_XFER, xfer) |
+		FIELD_PREP(OP_CMD_CNT, size) |
+		FIELD_PREP(OP_CMD_SIG, sync) |
+		FIELD_PREP(OP_CMD_TGT_CMD, cmd_tgt_act[op].tgt_cmd) |
+		FIELD_PREP(OP_CMD_MODE, mode);
+
+	nfp_prog_push(nfp_prog, insn);
+}
+
+static void
+emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
+	 u8 mode, u8 xfer, u32 lreg, u32 rreg, u8 size, bool sync)
+{
+	struct nfp_insn_re_regs reg;
+	int err;
+
+	err = swreg_to_restricted(reg_none(), lreg, rreg, &reg, false);
+	if (err) {
+		nfp_prog->error = err;
+		return;
+	}
+	if (reg.swap) {
+		pr_err("cmd can't swap arguments\n");
+		nfp_prog->error = -EFAULT;
+		return;
+	}
+
+	__emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, sync);
+}
+
+static void
+__emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip,
+	  enum br_ctx_signal_state css, u16 addr, u8 defer)
+{
+	u16 addr_lo, addr_hi;
+	u64 insn;
+
+	addr_lo = addr & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));
+	addr_hi = addr != addr_lo;
+
+	insn = OP_BR_BASE |
+		FIELD_PREP(OP_BR_MASK, mask) |
+		FIELD_PREP(OP_BR_EV_PIP, ev_pip) |
+		FIELD_PREP(OP_BR_CSS, css) |
+		FIELD_PREP(OP_BR_DEFBR, defer) |
+		FIELD_PREP(OP_BR_ADDR_LO, addr_lo) |
+		FIELD_PREP(OP_BR_ADDR_HI, addr_hi);
+
+	nfp_prog_push(nfp_prog, insn);
+}
+
+static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer)
+{
+	if (defer > 2) {
+		pr_err("BUG: branch defer out of bounds %d\n", defer);
+		nfp_prog->error = -EFAULT;
+		return;
+	}
+	__emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer);
+}
+
+static void
+emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
+{
+	__emit_br(nfp_prog, mask,
+		  mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
+		  BR_CSS_NONE, addr, defer);
+}
+
+static void
+__emit_br_byte(struct nfp_prog *nfp_prog, u8 areg, u8 breg, bool imm8,
+	       u8 byte, bool equal, u16 addr, u8 defer)
+{
+	u16 addr_lo, addr_hi;
+	u64 insn;
+
+	addr_lo = addr & (OP_BB_ADDR_LO >> __bf_shf(OP_BB_ADDR_LO));
+	addr_hi = addr != addr_lo;
+
+	insn = OP_BBYTE_BASE |
+		FIELD_PREP(OP_BB_A_SRC, areg) |
+		FIELD_PREP(OP_BB_BYTE, byte) |
+		FIELD_PREP(OP_BB_B_SRC, breg) |
+		FIELD_PREP(OP_BB_I8, imm8) |
+		FIELD_PREP(OP_BB_EQ, equal) |
+		FIELD_PREP(OP_BB_DEFBR, defer) |
+		FIELD_PREP(OP_BB_ADDR_LO, addr_lo) |
+		FIELD_PREP(OP_BB_ADDR_HI, addr_hi);
+
+	nfp_prog_push(nfp_prog, insn);
+}
+
+static void
+emit_br_byte_neq(struct nfp_prog *nfp_prog,
+		 u32 dst, u8 imm, u8 byte, u16 addr, u8 defer)
+{
+	struct nfp_insn_re_regs reg;
+	int err;
+
+	err = swreg_to_restricted(reg_none(), dst, reg_imm(imm), &reg, true);
+	if (err) {
+		nfp_prog->error = err;
+		return;
+	}
+
+	__emit_br_byte(nfp_prog, reg.areg, reg.breg, reg.i8, byte, false, addr,
+		       defer);
+}
+
+static void
+__emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
+	     enum immed_width width, bool invert,
+	     enum immed_shift shift, bool wr_both)
+{
+	u64 insn;
+
+	insn = OP_IMMED_BASE |
+		FIELD_PREP(OP_IMMED_A_SRC, areg) |
+		FIELD_PREP(OP_IMMED_B_SRC, breg) |
+		FIELD_PREP(OP_IMMED_IMM, imm_hi) |
+		FIELD_PREP(OP_IMMED_WIDTH, width) |
+		FIELD_PREP(OP_IMMED_INV, invert) |
+		FIELD_PREP(OP_IMMED_SHIFT, shift) |
+		FIELD_PREP(OP_IMMED_WR_AB, wr_both);
+
+	nfp_prog_push(nfp_prog, insn);
+}
+
+static void
+emit_immed(struct nfp_prog *nfp_prog, u32 dst, u16 imm,
+	   enum immed_width width, bool invert, enum immed_shift shift)
+{
+	struct nfp_insn_ur_regs reg;
+	int err;
+
+	if (FIELD_GET(NN_REG_TYPE, dst) == NN_REG_IMM) {
+		nfp_prog->error = -EFAULT;
+		return;
+	}
+
+	err = swreg_to_unrestricted(dst, dst, reg_imm(imm & 0xff), &reg);
+	if (err) {
+		nfp_prog->error = err;
+		return;
+	}
+
+	__emit_immed(nfp_prog, reg.areg, reg.breg, imm >> 8, width,
+		     invert, shift, reg.wr_both);
+}
+
+static void
+__emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
+	   enum shf_sc sc, u8 shift,
+	   u16 areg, enum shf_op op, u16 breg, bool i8, bool sw, bool wr_both)
+{
+	u64 insn;
+
+	if (!FIELD_FIT(OP_SHF_SHIFT, shift)) {
+		nfp_prog->error = -EFAULT;
+		return;
+	}
+
+	if (sc == SHF_SC_L_SHF)
+		shift = 32 - shift;
+
+	insn = OP_SHF_BASE |
+		FIELD_PREP(OP_SHF_A_SRC, areg) |
+		FIELD_PREP(OP_SHF_SC, sc) |
+		FIELD_PREP(OP_SHF_B_SRC, breg) |
+		FIELD_PREP(OP_SHF_I8, i8) |
+		FIELD_PREP(OP_SHF_SW, sw) |
+		FIELD_PREP(OP_SHF_DST, dst) |
+		FIELD_PREP(OP_SHF_SHIFT, shift) |
+		FIELD_PREP(OP_SHF_OP, op) |
+		FIELD_PREP(OP_SHF_DST_AB, dst_ab) |
+		FIELD_PREP(OP_SHF_WR_AB, wr_both);
+
+	nfp_prog_push(nfp_prog, insn);
+}
+
+static void
+emit_shf(struct nfp_prog *nfp_prog, u32 dst, u32 lreg, enum shf_op op, u32 rreg,
+	 enum shf_sc sc, u8 shift)
+{
+	struct nfp_insn_re_regs reg;
+	int err;
+
+	err = swreg_to_restricted(dst, lreg, rreg, &reg, true);
+	if (err) {
+		nfp_prog->error = err;
+		return;
+	}
+
+	__emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift,
+		   reg.areg, op, reg.breg, reg.i8, reg.swap, reg.wr_both);
+}
+
+static void
+__emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
+	   u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both)
+{
+	u64 insn;
+
+	insn = OP_ALU_BASE |
+		FIELD_PREP(OP_ALU_A_SRC, areg) |
+		FIELD_PREP(OP_ALU_B_SRC, breg) |
+		FIELD_PREP(OP_ALU_DST, dst) |
+		FIELD_PREP(OP_ALU_SW, swap) |
+		FIELD_PREP(OP_ALU_OP, op) |
+		FIELD_PREP(OP_ALU_DST_AB, dst_ab) |
+		FIELD_PREP(OP_ALU_WR_AB, wr_both);
+
+	nfp_prog_push(nfp_prog, insn);
+}
+
+static void
+emit_alu(struct nfp_prog *nfp_prog, u32 dst, u32 lreg, enum alu_op op, u32 rreg)
+{
+	struct nfp_insn_ur_regs reg;
+	int err;
+
+	err = swreg_to_unrestricted(dst, lreg, rreg, &reg);
+	if (err) {
+		nfp_prog->error = err;
+		return;
+	}
+
+	__emit_alu(nfp_prog, reg.dst, reg.dst_ab,
+		   reg.areg, op, reg.breg, reg.swap, reg.wr_both);
+}
+
+static void
+__emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc,
+		u8 areg, u8 bmask, u8 breg, u8 shift, bool imm8,
+		bool zero, bool swap, bool wr_both)
+{
+	u64 insn;
+
+	insn = OP_LDF_BASE |
+		FIELD_PREP(OP_LDF_A_SRC, areg) |
+		FIELD_PREP(OP_LDF_SC, sc) |
+		FIELD_PREP(OP_LDF_B_SRC, breg) |
+		FIELD_PREP(OP_LDF_I8, imm8) |
+		FIELD_PREP(OP_LDF_SW, swap) |
+		FIELD_PREP(OP_LDF_ZF, zero) |
+		FIELD_PREP(OP_LDF_BMASK, bmask) |
+		FIELD_PREP(OP_LDF_SHF, shift) |
+		FIELD_PREP(OP_LDF_WR_AB, wr_both);
+
+	nfp_prog_push(nfp_prog, insn);
+}
+
+static void
+emit_ld_field_any(struct nfp_prog *nfp_prog, enum shf_sc sc, u8 shift,
+		  u32 dst, u8 bmask, u32 src, bool zero)
+{
+	struct nfp_insn_re_regs reg;
+	int err;
+
+	err = swreg_to_restricted(reg_none(), dst, src, &reg, true);
+	if (err) {
+		nfp_prog->error = err;
+		return;
+	}
+
+	__emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift,
+			reg.i8, zero, reg.swap, reg.wr_both);
+}
+
+static void
+emit_ld_field(struct nfp_prog *nfp_prog, u32 dst, u8 bmask, u32 src,
+	      enum shf_sc sc, u8 shift)
+{
+	emit_ld_field_any(nfp_prog, sc, shift, dst, bmask, src, false);
+}
+
+/* --- Wrappers --- */
+static bool pack_immed(u32 imm, u16 *val, enum immed_shift *shift)
+{
+	if (!(imm & 0xffff0000)) {
+		*val = imm;
+		*shift = IMMED_SHIFT_0B;
+	} else if (!(imm & 0xff0000ff)) {
+		*val = imm >> 8;
+		*shift = IMMED_SHIFT_1B;
+	} else if (!(imm & 0x0000ffff)) {
+		*val = imm >> 16;
+		*shift = IMMED_SHIFT_2B;
+	} else {
+		return false;
+	}
+
+	return true;
+}
+
+static void wrp_immed(struct nfp_prog *nfp_prog, u32 dst, u32 imm)
+{
+	enum immed_shift shift;
+	u16 val;
+
+	if (pack_immed(imm, &val, &shift)) {
+		emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift);
+	} else if (pack_immed(~imm, &val, &shift)) {
+		emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift);
+	} else {
+		emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL,
+			   false, IMMED_SHIFT_0B);
+		emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD,
+			   false, IMMED_SHIFT_2B);
+	}
+}
+
+/* ur_load_imm_any() - encode immediate or use tmp register (unrestricted)
+ * If the @imm is small enough encode it directly in operand and return
+ * otherwise load @imm to a spare register and return its encoding.
+ */
+static u32 ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, u32 tmp_reg)
+{
+	if (FIELD_FIT(UR_REG_IMM_MAX, imm))
+		return reg_imm(imm);
+
+	wrp_immed(nfp_prog, tmp_reg, imm);
+	return tmp_reg;
+}
+
+/* re_load_imm_any() - encode immediate or use tmp register (restricted)
+ * If the @imm is small enough encode it directly in operand and return
+ * otherwise load @imm to a spare register and return its encoding.
+ */
+static u32 re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, u32 tmp_reg)
+{
+	if (FIELD_FIT(RE_REG_IMM_MAX, imm))
+		return reg_imm(imm);
+
+	wrp_immed(nfp_prog, tmp_reg, imm);
+	return tmp_reg;
+}
+
+static void
+wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask,
+	       enum br_special special)
+{
+	emit_br(nfp_prog, mask, 0, 0);
+
+	nfp_prog->prog[nfp_prog->prog_len - 1] |=
+		FIELD_PREP(OP_BR_SPECIAL, special);
+}
+
+static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src)
+{
+	emit_alu(nfp_prog, reg_both(dst), reg_none(), ALU_OP_NONE, reg_b(src));
+}
+
+static int
+construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset,
+		      u16 src, bool src_valid, u8 size)
+{
+	unsigned int i;
+	u16 shift, sz;
+	u32 tmp_reg;
+
+	/* We load the value from the address indicated in @offset and then
+	 * shift out the data we don't need.  Note: this is big endian!
+	 */
+	sz = size < 4 ? 4 : size;
+	shift = size < 4 ? 4 - size : 0;
+
+	if (src_valid) {
+		/* Calculate the true offset (src_reg + imm) */
+		tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
+		emit_alu(nfp_prog, imm_both(nfp_prog),
+			 reg_a(src), ALU_OP_ADD, tmp_reg);
+		/* Check packet length (size guaranteed to fit b/c it's u8) */
+		emit_alu(nfp_prog, imm_a(nfp_prog),
+			 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));
+		emit_alu(nfp_prog, reg_none(),
+			 NFP_BPF_ABI_LEN, ALU_OP_SUB, imm_a(nfp_prog));
+		wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
+		/* Load data */
+		emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0,
+			 pkt_reg(nfp_prog), imm_b(nfp_prog), sz - 1, true);
+	} else {
+		/* Check packet length */
+		tmp_reg = ur_load_imm_any(nfp_prog, offset + size,
+					  imm_a(nfp_prog));
+		emit_alu(nfp_prog, reg_none(),
+			 NFP_BPF_ABI_LEN, ALU_OP_SUB, tmp_reg);
+		wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
+		/* Load data */
+		tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
+		emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0,
+			 pkt_reg(nfp_prog), tmp_reg, sz - 1, true);
+	}
+
+	i = 0;
+	if (shift)
+		emit_shf(nfp_prog, reg_both(0), reg_none(), SHF_OP_NONE,
+			 reg_xfer(0), SHF_SC_R_SHF, shift * 8);
+	else
+		for (; i * 4 < size; i++)
+			emit_alu(nfp_prog, reg_both(i),
+				 reg_none(), ALU_OP_NONE, reg_xfer(i));
+
+	if (i < 2)
+		wrp_immed(nfp_prog, reg_both(1), 0);
+
+	return 0;
+}
+
+static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
+{
+	return construct_data_ind_ld(nfp_prog, offset, 0, false, size);
+}
+
+static int wrp_set_mark(struct nfp_prog *nfp_prog, u8 src)
+{
+	emit_alu(nfp_prog, NFP_BPF_ABI_MARK,
+		 reg_none(), ALU_OP_NONE, reg_b(src));
+	emit_alu(nfp_prog, NFP_BPF_ABI_FLAGS,
+		 NFP_BPF_ABI_FLAGS, ALU_OP_OR, reg_imm(NFP_BPF_ABI_FLAG_MARK));
+
+	return 0;
+}
+
+static void
+wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm)
+{
+	u32 tmp_reg;
+
+	if (alu_op == ALU_OP_AND) {
+		if (!imm)
+			wrp_immed(nfp_prog, reg_both(dst), 0);
+		if (!imm || !~imm)
+			return;
+	}
+	if (alu_op == ALU_OP_OR) {
+		if (!~imm)
+			wrp_immed(nfp_prog, reg_both(dst), ~0U);
+		if (!imm || !~imm)
+			return;
+	}
+	if (alu_op == ALU_OP_XOR) {
+		if (!~imm)
+			emit_alu(nfp_prog, reg_both(dst), reg_none(),
+				 ALU_OP_NEG, reg_b(dst));
+		if (!imm || !~imm)
+			return;
+	}
+
+	tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
+	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg);
+}
+
+static int
+wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+	      enum alu_op alu_op, bool skip)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 imm = insn->imm; /* sign extend */
+
+	if (skip) {
+		meta->skip = true;
+		return 0;
+	}
+
+	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U);
+	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32);
+
+	return 0;
+}
+
+static int
+wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+	      enum alu_op alu_op)
+{
+	u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
+
+	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
+	emit_alu(nfp_prog, reg_both(dst + 1),
+		 reg_a(dst + 1), alu_op, reg_b(src + 1));
+
+	return 0;
+}
+
+static int
+wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+	      enum alu_op alu_op, bool skip)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	if (skip) {
+		meta->skip = true;
+		return 0;
+	}
+
+	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm);
+	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
+
+	return 0;
+}
+
+static int
+wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+	      enum alu_op alu_op)
+{
+	u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
+
+	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
+	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
+
+	return 0;
+}
+
+static void
+wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src,
+		 enum br_mask br_mask, u16 off)
+{
+	emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src));
+	emit_br(nfp_prog, br_mask, off, 0);
+}
+
+static int
+wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+	     enum alu_op alu_op, enum br_mask br_mask)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	if (insn->off < 0) /* TODO */
+		return -ENOTSUPP;
+
+	wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op,
+			 insn->src_reg * 2, br_mask, insn->off);
+	wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op,
+			 insn->src_reg * 2 + 1, br_mask, insn->off);
+
+	return 0;
+}
+
+static int
+wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+	    enum br_mask br_mask, bool swap)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 imm = insn->imm; /* sign extend */
+	u8 reg = insn->dst_reg * 2;
+	u32 tmp_reg;
+
+	if (insn->off < 0) /* TODO */
+		return -ENOTSUPP;
+
+	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
+	if (!swap)
+		emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg);
+	else
+		emit_alu(nfp_prog, reg_none(), tmp_reg, ALU_OP_SUB, reg_a(reg));
+
+	tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
+	if (!swap)
+		emit_alu(nfp_prog, reg_none(),
+			 reg_a(reg + 1), ALU_OP_SUB_C, tmp_reg);
+	else
+		emit_alu(nfp_prog, reg_none(),
+			 tmp_reg, ALU_OP_SUB_C, reg_a(reg + 1));
+
+	emit_br(nfp_prog, br_mask, insn->off, 0);
+
+	return 0;
+}
+
+static int
+wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+	    enum br_mask br_mask, bool swap)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u8 areg = insn->src_reg * 2, breg = insn->dst_reg * 2;
+
+	if (insn->off < 0) /* TODO */
+		return -ENOTSUPP;
+
+	if (swap) {
+		areg ^= breg;
+		breg ^= areg;
+		areg ^= breg;
+	}
+
+	emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg));
+	emit_alu(nfp_prog, reg_none(),
+		 reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1));
+	emit_br(nfp_prog, br_mask, insn->off, 0);
+
+	return 0;
+}
+
+/* --- Callbacks --- */
+static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2);
+	wrp_reg_mov(nfp_prog, insn->dst_reg * 2 + 1, insn->src_reg * 2 + 1);
+
+	return 0;
+}
+
+static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	u64 imm = meta->insn.imm; /* sign extend */
+
+	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U);
+	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32);
+
+	return 0;
+}
+
+static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR);
+}
+
+static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm);
+}
+
+static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND);
+}
+
+static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
+}
+
+static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR);
+}
+
+static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
+}
+
+static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
+		 reg_a(insn->dst_reg * 2), ALU_OP_ADD,
+		 reg_b(insn->src_reg * 2));
+	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
+		 reg_a(insn->dst_reg * 2 + 1), ALU_OP_ADD_C,
+		 reg_b(insn->src_reg * 2 + 1));
+
+	return 0;
+}
+
+static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 imm = insn->imm; /* sign extend */
+
+	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U);
+	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32);
+
+	return 0;
+}
+
+static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
+		 reg_a(insn->dst_reg * 2), ALU_OP_SUB,
+		 reg_b(insn->src_reg * 2));
+	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
+		 reg_a(insn->dst_reg * 2 + 1), ALU_OP_SUB_C,
+		 reg_b(insn->src_reg * 2 + 1));
+
+	return 0;
+}
+
+static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 imm = insn->imm; /* sign extend */
+
+	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U);
+	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32);
+
+	return 0;
+}
+
+static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	if (insn->imm != 32)
+		return 1; /* TODO */
+
+	wrp_reg_mov(nfp_prog, insn->dst_reg * 2 + 1, insn->dst_reg * 2);
+	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), 0);
+
+	return 0;
+}
+
+static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	if (insn->imm != 32)
+		return 1; /* TODO */
+
+	wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->dst_reg * 2 + 1);
+	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
+
+	return 0;
+}
+
+static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	wrp_reg_mov(nfp_prog, insn->dst_reg * 2,  insn->src_reg * 2);
+	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
+
+	return 0;
+}
+
+static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm);
+	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
+
+	return 0;
+}
+
+static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR);
+}
+
+static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR, !~meta->insn.imm);
+}
+
+static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND);
+}
+
+static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
+}
+
+static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR);
+}
+
+static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
+}
+
+static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD);
+}
+
+static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD, !meta->insn.imm);
+}
+
+static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB);
+}
+
+static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB, !meta->insn.imm);
+}
+
+static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	if (!insn->imm)
+		return 1; /* TODO: zero shift means indirect */
+
+	emit_shf(nfp_prog, reg_both(insn->dst_reg * 2),
+		 reg_none(), SHF_OP_NONE, reg_b(insn->dst_reg * 2),
+		 SHF_SC_L_SHF, insn->imm);
+	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
+
+	return 0;
+}
+
+static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	wrp_immed(nfp_prog, reg_both(nfp_meta_prev(meta)->insn.dst_reg * 2 + 1),
+		  meta->insn.imm);
+
+	return 0;
+}
+
+static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	meta->double_cb = imm_ld8_part2;
+	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm);
+
+	return 0;
+}
+
+static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return construct_data_ld(nfp_prog, meta->insn.imm, 1);
+}
+
+static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return construct_data_ld(nfp_prog, meta->insn.imm, 2);
+}
+
+static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return construct_data_ld(nfp_prog, meta->insn.imm, 4);
+}
+
+static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return construct_data_ind_ld(nfp_prog, meta->insn.imm,
+				     meta->insn.src_reg * 2, true, 1);
+}
+
+static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return construct_data_ind_ld(nfp_prog, meta->insn.imm,
+				     meta->insn.src_reg * 2, true, 2);
+}
+
+static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return construct_data_ind_ld(nfp_prog, meta->insn.imm,
+				     meta->insn.src_reg * 2, true, 4);
+}
+
+static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	if (meta->insn.off == offsetof(struct sk_buff, len))
+		emit_alu(nfp_prog, reg_both(meta->insn.dst_reg * 2),
+			 reg_none(), ALU_OP_NONE, NFP_BPF_ABI_LEN);
+	else
+		return -ENOTSUPP;
+
+	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
+
+	return 0;
+}
+
+static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	if (meta->insn.off == offsetof(struct sk_buff, mark))
+		return wrp_set_mark(nfp_prog, meta->insn.src_reg * 2);
+
+	return -ENOTSUPP;
+}
+
+static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	if (meta->insn.off < 0) /* TODO */
+		return -ENOTSUPP;
+	emit_br(nfp_prog, BR_UNC, meta->insn.off, 0);
+
+	return 0;
+}
+
+static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 imm = insn->imm; /* sign extend */
+	u32 or1 = reg_a(insn->dst_reg * 2), or2 = reg_b(insn->dst_reg * 2 + 1);
+	u32 tmp_reg;
+
+	if (insn->off < 0) /* TODO */
+		return -ENOTSUPP;
+
+	if (imm & ~0U) {
+		tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
+		emit_alu(nfp_prog, imm_a(nfp_prog),
+			 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
+		or1 = imm_a(nfp_prog);
+	}
+
+	if (imm >> 32) {
+		tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
+		emit_alu(nfp_prog, imm_b(nfp_prog),
+			 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg);
+		or2 = imm_b(nfp_prog);
+	}
+
+	emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2);
+	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
+
+	return 0;
+}
+
+static int jgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_imm(nfp_prog, meta, BR_BLO, false);
+}
+
+static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true);
+}
+
+static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 imm = insn->imm; /* sign extend */
+	u32 tmp_reg;
+
+	if (insn->off < 0) /* TODO */
+		return -ENOTSUPP;
+
+	if (!imm) {
+		meta->skip = true;
+		return 0;
+	}
+
+	if (imm & ~0U) {
+		tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
+		emit_alu(nfp_prog, reg_none(),
+			 reg_a(insn->dst_reg * 2), ALU_OP_AND, tmp_reg);
+		emit_br(nfp_prog, BR_BNE, insn->off, 0);
+	}
+
+	if (imm >> 32) {
+		tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
+		emit_alu(nfp_prog, reg_none(),
+			 reg_a(insn->dst_reg * 2 + 1), ALU_OP_AND, tmp_reg);
+		emit_br(nfp_prog, BR_BNE, insn->off, 0);
+	}
+
+	return 0;
+}
+
+static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+	u64 imm = insn->imm; /* sign extend */
+	u32 tmp_reg;
+
+	if (insn->off < 0) /* TODO */
+		return -ENOTSUPP;
+
+	if (!imm) {
+		emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2),
+			 ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1));
+		emit_br(nfp_prog, BR_BNE, insn->off, 0);
+	}
+
+	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
+	emit_alu(nfp_prog, reg_none(),
+		 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
+	emit_br(nfp_prog, BR_BNE, insn->off, 0);
+
+	tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
+	emit_alu(nfp_prog, reg_none(),
+		 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg);
+	emit_br(nfp_prog, BR_BNE, insn->off, 0);
+
+	return 0;
+}
+
+static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	const struct bpf_insn *insn = &meta->insn;
+
+	if (insn->off < 0) /* TODO */
+		return -ENOTSUPP;
+
+	emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2),
+		 ALU_OP_XOR, reg_b(insn->src_reg * 2));
+	emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1),
+		 ALU_OP_XOR, reg_b(insn->src_reg * 2 + 1));
+	emit_alu(nfp_prog, reg_none(),
+		 imm_a(nfp_prog), ALU_OP_OR, imm_b(nfp_prog));
+	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
+
+	return 0;
+}
+
+static int jgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_reg(nfp_prog, meta, BR_BLO, false);
+}
+
+static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true);
+}
+
+static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
+}
+
+static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
+}
+
+static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT);
+
+	return 0;
+}
+
+static const instr_cb_t instr_cb[256] = {
+	[BPF_ALU64 | BPF_MOV | BPF_X] =	mov_reg64,
+	[BPF_ALU64 | BPF_MOV | BPF_K] =	mov_imm64,
+	[BPF_ALU64 | BPF_XOR | BPF_X] =	xor_reg64,
+	[BPF_ALU64 | BPF_XOR | BPF_K] =	xor_imm64,
+	[BPF_ALU64 | BPF_AND | BPF_X] =	and_reg64,
+	[BPF_ALU64 | BPF_AND | BPF_K] =	and_imm64,
+	[BPF_ALU64 | BPF_OR | BPF_X] =	or_reg64,
+	[BPF_ALU64 | BPF_OR | BPF_K] =	or_imm64,
+	[BPF_ALU64 | BPF_ADD | BPF_X] =	add_reg64,
+	[BPF_ALU64 | BPF_ADD | BPF_K] =	add_imm64,
+	[BPF_ALU64 | BPF_SUB | BPF_X] =	sub_reg64,
+	[BPF_ALU64 | BPF_SUB | BPF_K] =	sub_imm64,
+	[BPF_ALU64 | BPF_LSH | BPF_K] =	shl_imm64,
+	[BPF_ALU64 | BPF_RSH | BPF_K] =	shr_imm64,
+	[BPF_ALU | BPF_MOV | BPF_X] =	mov_reg,
+	[BPF_ALU | BPF_MOV | BPF_K] =	mov_imm,
+	[BPF_ALU | BPF_XOR | BPF_X] =	xor_reg,
+	[BPF_ALU | BPF_XOR | BPF_K] =	xor_imm,
+	[BPF_ALU | BPF_AND | BPF_X] =	and_reg,
+	[BPF_ALU | BPF_AND | BPF_K] =	and_imm,
+	[BPF_ALU | BPF_OR | BPF_X] =	or_reg,
+	[BPF_ALU | BPF_OR | BPF_K] =	or_imm,
+	[BPF_ALU | BPF_ADD | BPF_X] =	add_reg,
+	[BPF_ALU | BPF_ADD | BPF_K] =	add_imm,
+	[BPF_ALU | BPF_SUB | BPF_X] =	sub_reg,
+	[BPF_ALU | BPF_SUB | BPF_K] =	sub_imm,
+	[BPF_ALU | BPF_LSH | BPF_K] =	shl_imm,
+	[BPF_LD | BPF_IMM | BPF_DW] =	imm_ld8,
+	[BPF_LD | BPF_ABS | BPF_B] =	data_ld1,
+	[BPF_LD | BPF_ABS | BPF_H] =	data_ld2,
+	[BPF_LD | BPF_ABS | BPF_W] =	data_ld4,
+	[BPF_LD | BPF_IND | BPF_B] =	data_ind_ld1,
+	[BPF_LD | BPF_IND | BPF_H] =	data_ind_ld2,
+	[BPF_LD | BPF_IND | BPF_W] =	data_ind_ld4,
+	[BPF_LDX | BPF_MEM | BPF_W] =	mem_ldx4,
+	[BPF_STX | BPF_MEM | BPF_W] =	mem_stx4,
+	[BPF_JMP | BPF_JA | BPF_K] =	jump,
+	[BPF_JMP | BPF_JEQ | BPF_K] =	jeq_imm,
+	[BPF_JMP | BPF_JGT | BPF_K] =	jgt_imm,
+	[BPF_JMP | BPF_JGE | BPF_K] =	jge_imm,
+	[BPF_JMP | BPF_JSET | BPF_K] =	jset_imm,
+	[BPF_JMP | BPF_JNE | BPF_K] =	jne_imm,
+	[BPF_JMP | BPF_JEQ | BPF_X] =	jeq_reg,
+	[BPF_JMP | BPF_JGT | BPF_X] =	jgt_reg,
+	[BPF_JMP | BPF_JGE | BPF_X] =	jge_reg,
+	[BPF_JMP | BPF_JSET | BPF_X] =	jset_reg,
+	[BPF_JMP | BPF_JNE | BPF_X] =	jne_reg,
+	[BPF_JMP | BPF_EXIT] =		goto_out,
+};
+
+/* --- Misc code --- */
+static void br_set_offset(u64 *instr, u16 offset)
+{
+	u16 addr_lo, addr_hi;
+
+	addr_lo = offset & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));
+	addr_hi = offset != addr_lo;
+	*instr &= ~(OP_BR_ADDR_HI | OP_BR_ADDR_LO);
+	*instr |= FIELD_PREP(OP_BR_ADDR_HI, addr_hi);
+	*instr |= FIELD_PREP(OP_BR_ADDR_LO, addr_lo);
+}
+
+/* --- Assembler logic --- */
+static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
+{
+	struct nfp_insn_meta *meta, *next;
+	u32 off, br_idx;
+	u32 idx;
+
+	nfp_for_each_insn_walk2(nfp_prog, meta, next) {
+		if (meta->skip)
+			continue;
+		if (BPF_CLASS(meta->insn.code) != BPF_JMP)
+			continue;
+
+		br_idx = nfp_prog_offset_to_index(nfp_prog, next->off) - 1;
+		if (!nfp_is_br(nfp_prog->prog[br_idx])) {
+			pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n",
+			       br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
+			return -ELOOP;
+		}
+		/* Leave special branches for later */
+		if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]))
+			continue;
+
+		/* Find the target offset in assembler realm */
+		off = meta->insn.off;
+		if (!off) {
+			pr_err("Fixup found zero offset!!\n");
+			return -ELOOP;
+		}
+
+		while (off && nfp_meta_has_next(nfp_prog, next)) {
+			next = nfp_meta_next(next);
+			off--;
+		}
+		if (off) {
+			pr_err("Fixup found too large jump!! %d\n", off);
+			return -ELOOP;
+		}
+
+		if (next->skip) {
+			pr_err("Branch landing on removed instruction!!\n");
+			return -ELOOP;
+		}
+
+		for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off);
+		     idx <= br_idx; idx++) {
+			if (!nfp_is_br(nfp_prog->prog[idx]))
+				continue;
+			br_set_offset(&nfp_prog->prog[idx], next->off);
+		}
+	}
+
+	/* Fixup 'goto out's separately, they can be scattered around */
+	for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) {
+		enum br_special special;
+
+		if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE)
+			continue;
+
+		special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]);
+		switch (special) {
+		case OP_BR_NORMAL:
+			break;
+		case OP_BR_GO_OUT:
+			br_set_offset(&nfp_prog->prog[br_idx],
+				      nfp_prog->tgt_out);
+			break;
+		case OP_BR_GO_ABORT:
+			br_set_offset(&nfp_prog->prog[br_idx],
+				      nfp_prog->tgt_abort);
+			break;
+		}
+
+		nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL;
+	}
+
+	return 0;
+}
+
+static void nfp_intro(struct nfp_prog *nfp_prog)
+{
+	emit_alu(nfp_prog, pkt_reg(nfp_prog),
+		 reg_none(), ALU_OP_NONE, NFP_BPF_ABI_PKT);
+}
+
+static void nfp_outro_tc_legacy(struct nfp_prog *nfp_prog)
+{
+	const u8 act2code[] = {
+		[NN_ACT_TC_DROP]  = 0x22,
+		[NN_ACT_TC_REDIR] = 0x24
+	};
+	/* Target for aborts */
+	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
+	wrp_immed(nfp_prog, reg_both(0), 0);
+
+	/* Target for normal exits */
+	nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
+	/* Legacy TC mode:
+	 *   0        0x11 -> pass,  count as stat0
+	 *  -1  drop  0x22 -> drop,  count as stat1
+	 *     redir  0x24 -> redir, count as stat1
+	 *  ife mark  0x21 -> pass,  count as stat1
+	 *  ife + tx  0x24 -> redir, count as stat1
+	 */
+	emit_br_byte_neq(nfp_prog, reg_b(0), 0xff, 0, nfp_prog->tgt_done, 2);
+	emit_alu(nfp_prog, reg_a(0),
+		 reg_none(), ALU_OP_NONE, NFP_BPF_ABI_FLAGS);
+	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
+
+	emit_br(nfp_prog, BR_UNC, nfp_prog->tgt_done, 1);
+	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(act2code[nfp_prog->act]),
+		      SHF_SC_L_SHF, 16);
+}
+
+static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
+{
+	/* TC direct-action mode:
+	 *   0,1   ok        NOT SUPPORTED[1]
+	 *   2   drop  0x22 -> drop,  count as stat1
+	 *   4,5 nuke  0x02 -> drop
+	 *   7  redir  0x44 -> redir, count as stat2
+	 *   * unspec  0x11 -> pass,  count as stat0
+	 *
+	 * [1] We can't support OK and RECLASSIFY because we can't tell TC
+	 *     the exact decision made.  We are forced to support UNSPEC
+	 *     to handle aborts so that's the only one we handle for passing
+	 *     packets up the stack.
+	 */
+	/* Target for aborts */
+	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
+
+	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+
+	emit_alu(nfp_prog, reg_a(0),
+		 reg_none(), ALU_OP_NONE, NFP_BPF_ABI_FLAGS);
+	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
+
+	/* Target for normal exits */
+	nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
+
+	/* if R0 > 7 jump to abort */
+	emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0));
+	emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
+	emit_alu(nfp_prog, reg_a(0),
+		 reg_none(), ALU_OP_NONE, NFP_BPF_ABI_FLAGS);
+
+	wrp_immed(nfp_prog, reg_b(2), 0x41221211);
+	wrp_immed(nfp_prog, reg_b(3), 0x41001211);
+
+	emit_shf(nfp_prog, reg_a(1),
+		 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 2);
+
+	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
+	emit_shf(nfp_prog, reg_a(2),
+		 reg_imm(0xf), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
+
+	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
+	emit_shf(nfp_prog, reg_b(2),
+		 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0);
+
+	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+
+	emit_shf(nfp_prog, reg_b(2),
+		 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4);
+	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
+}
+
+static void nfp_outro(struct nfp_prog *nfp_prog)
+{
+	switch (nfp_prog->act) {
+	case NN_ACT_DIRECT:
+		nfp_outro_tc_da(nfp_prog);
+		break;
+	case NN_ACT_TC_DROP:
+	case NN_ACT_TC_REDIR:
+		nfp_outro_tc_legacy(nfp_prog);
+		break;
+	}
+}
+
+static int nfp_translate(struct nfp_prog *nfp_prog)
+{
+	struct nfp_insn_meta *meta;
+	int err;
+
+	nfp_intro(nfp_prog);
+	if (nfp_prog->error)
+		return nfp_prog->error;
+
+	list_for_each_entry(meta, &nfp_prog->insns, l) {
+		instr_cb_t cb = instr_cb[meta->insn.code];
+
+		meta->off = nfp_prog_current_offset(nfp_prog);
+
+		if (meta->skip) {
+			nfp_prog->n_translated++;
+			continue;
+		}
+
+		if (nfp_meta_has_prev(nfp_prog, meta) &&
+		    nfp_meta_prev(meta)->double_cb)
+			cb = nfp_meta_prev(meta)->double_cb;
+		if (!cb)
+			return -ENOENT;
+		err = cb(nfp_prog, meta);
+		if (err)
+			return err;
+
+		nfp_prog->n_translated++;
+	}
+
+	nfp_outro(nfp_prog);
+	if (nfp_prog->error)
+		return nfp_prog->error;
+
+	return nfp_fixup_branches(nfp_prog);
+}
+
+static int
+nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
+		 unsigned int cnt)
+{
+	unsigned int i;
+
+	for (i = 0; i < cnt; i++) {
+		struct nfp_insn_meta *meta;
+
+		meta = kzalloc(sizeof(*meta), GFP_KERNEL);
+		if (!meta)
+			return -ENOMEM;
+
+		meta->insn = prog[i];
+		meta->n = i;
+
+		list_add_tail(&meta->l, &nfp_prog->insns);
+	}
+
+	return 0;
+}
+
+/* --- Optimizations --- */
+static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog)
+{
+	struct nfp_insn_meta *meta;
+
+	list_for_each_entry(meta, &nfp_prog->insns, l) {
+		struct bpf_insn insn = meta->insn;
+
+		/* Programs converted from cBPF start with register xoring */
+		if (insn.code == (BPF_ALU64 | BPF_XOR | BPF_X) &&
+		    insn.src_reg == insn.dst_reg)
+			continue;
+
+		/* Programs start with R6 = R1 but we ignore the skb pointer */
+		if (insn.code == (BPF_ALU64 | BPF_MOV | BPF_X) &&
+		    insn.src_reg == 1 && insn.dst_reg == 6)
+			meta->skip = true;
+
+		/* Return as soon as something doesn't match */
+		if (!meta->skip)
+			return;
+	}
+}
+
+/* Try to rename registers so that program uses only low ones */
+static int nfp_bpf_opt_reg_rename(struct nfp_prog *nfp_prog)
+{
+	bool reg_used[MAX_BPF_REG] = {};
+	u8 tgt_reg[MAX_BPF_REG] = {};
+	struct nfp_insn_meta *meta;
+	unsigned int i, j;
+
+	list_for_each_entry(meta, &nfp_prog->insns, l) {
+		if (meta->skip)
+			continue;
+
+		reg_used[meta->insn.src_reg] = true;
+		reg_used[meta->insn.dst_reg] = true;
+	}
+
+	for (i = 0, j = 0; i < ARRAY_SIZE(tgt_reg); i++) {
+		if (!reg_used[i])
+			continue;
+
+		tgt_reg[i] = j++;
+	}
+	nfp_prog->num_regs = j;
+
+	list_for_each_entry(meta, &nfp_prog->insns, l) {
+		meta->insn.src_reg = tgt_reg[meta->insn.src_reg];
+		meta->insn.dst_reg = tgt_reg[meta->insn.dst_reg];
+	}
+
+	return 0;
+}
+
+/* Remove masking after load since our load guarantees this is not needed */
+static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog)
+{
+	struct nfp_insn_meta *meta1, *meta2;
+	const s32 exp_mask[] = {
+		[BPF_B] = 0x000000ffU,
+		[BPF_H] = 0x0000ffffU,
+		[BPF_W] = 0xffffffffU,
+	};
+
+	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
+		struct bpf_insn insn, next;
+
+		insn = meta1->insn;
+		next = meta2->insn;
+
+		if (BPF_CLASS(insn.code) != BPF_LD)
+			continue;
+		if (BPF_MODE(insn.code) != BPF_ABS &&
+		    BPF_MODE(insn.code) != BPF_IND)
+			continue;
+
+		if (next.code != (BPF_ALU64 | BPF_AND | BPF_K))
+			continue;
+
+		if (!exp_mask[BPF_SIZE(insn.code)])
+			continue;
+		if (exp_mask[BPF_SIZE(insn.code)] != next.imm)
+			continue;
+
+		if (next.src_reg || next.dst_reg)
+			continue;
+
+		meta2->skip = true;
+	}
+}
+
+static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog)
+{
+	struct nfp_insn_meta *meta1, *meta2, *meta3;
+
+	nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) {
+		struct bpf_insn insn, next1, next2;
+
+		insn = meta1->insn;
+		next1 = meta2->insn;
+		next2 = meta3->insn;
+
+		if (BPF_CLASS(insn.code) != BPF_LD)
+			continue;
+		if (BPF_MODE(insn.code) != BPF_ABS &&
+		    BPF_MODE(insn.code) != BPF_IND)
+			continue;
+		if (BPF_SIZE(insn.code) != BPF_W)
+			continue;
+
+		if (!(next1.code == (BPF_LSH | BPF_K | BPF_ALU64) &&
+		      next2.code == (BPF_RSH | BPF_K | BPF_ALU64)) &&
+		    !(next1.code == (BPF_RSH | BPF_K | BPF_ALU64) &&
+		      next2.code == (BPF_LSH | BPF_K | BPF_ALU64)))
+			continue;
+
+		if (next1.src_reg || next1.dst_reg ||
+		    next2.src_reg || next2.dst_reg)
+			continue;
+
+		if (next1.imm != 0x20 || next2.imm != 0x20)
+			continue;
+
+		meta2->skip = true;
+		meta3->skip = true;
+	}
+}
+
+static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
+{
+	int ret;
+
+	nfp_bpf_opt_reg_init(nfp_prog);
+
+	ret = nfp_bpf_opt_reg_rename(nfp_prog);
+	if (ret)
+		return ret;
+
+	nfp_bpf_opt_ld_mask(nfp_prog);
+	nfp_bpf_opt_ld_shift(nfp_prog);
+
+	return 0;
+}
+
+/**
+ * nfp_bpf_jit() - translate BPF code into NFP assembly
+ * @filter:	kernel BPF filter struct
+ * @prog_mem:	memory to store assembler instructions
+ * @act:	action attached to this eBPF program
+ * @prog_start:	offset of the first instruction when loaded
+ * @prog_done:	where to jump on exit
+ * @prog_sz:	size of @prog_mem in instructions
+ * @res:	achieved parameters of translation results
+ */
+int
+nfp_bpf_jit(struct bpf_prog *filter, void *prog_mem,
+	    enum nfp_bpf_action_type act,
+	    unsigned int prog_start, unsigned int prog_done,
+	    unsigned int prog_sz, struct nfp_bpf_result *res)
+{
+	struct nfp_prog *nfp_prog;
+	int ret;
+
+	nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
+	if (!nfp_prog)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&nfp_prog->insns);
+	nfp_prog->act = act;
+	nfp_prog->start_off = prog_start;
+	nfp_prog->tgt_done = prog_done;
+
+	ret = nfp_prog_prepare(nfp_prog, filter->insnsi, filter->len);
+	if (ret)
+		goto out;
+
+	ret = nfp_prog_verify(nfp_prog, filter);
+	if (ret)
+		goto out;
+
+	ret = nfp_bpf_optimize(nfp_prog);
+	if (ret)
+		goto out;
+
+	if (nfp_prog->num_regs <= 7)
+		nfp_prog->regs_per_thread = 16;
+	else
+		nfp_prog->regs_per_thread = 32;
+
+	nfp_prog->prog = prog_mem;
+	nfp_prog->__prog_alloc_len = prog_sz;
+
+	ret = nfp_translate(nfp_prog);
+	if (ret) {
+		pr_err("Translation failed with error %d (translated: %u)\n",
+		       ret, nfp_prog->n_translated);
+		ret = -EINVAL;
+	}
+
+	res->n_instr = nfp_prog->prog_len;
+	res->dense_mode = nfp_prog->num_regs <= 7;
+out:
+	nfp_prog_free(nfp_prog);
+
+	return ret;
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_bpf_verifier.c b/drivers/net/ethernet/netronome/nfp/nfp_bpf_verifier.c
new file mode 100644
index 000000000000..144cae87f63a
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_bpf_verifier.c
@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2016 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define pr_fmt(fmt)	"NFP net bpf: " fmt
+
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+#include <linux/kernel.h>
+#include <linux/pkt_cls.h>
+
+#include "nfp_bpf.h"
+
+/* Analyzer/verifier definitions */
+struct nfp_bpf_analyzer_priv {
+	struct nfp_prog *prog;
+	struct nfp_insn_meta *meta;
+};
+
+static struct nfp_insn_meta *
+nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+		  unsigned int insn_idx, unsigned int n_insns)
+{
+	unsigned int forward, backward, i;
+
+	backward = meta->n - insn_idx;
+	forward = insn_idx - meta->n;
+
+	if (min(forward, backward) > n_insns - insn_idx - 1) {
+		backward = n_insns - insn_idx - 1;
+		meta = nfp_prog_last_meta(nfp_prog);
+	}
+	if (min(forward, backward) > insn_idx && backward > insn_idx) {
+		forward = insn_idx;
+		meta = nfp_prog_first_meta(nfp_prog);
+	}
+
+	if (forward < backward)
+		for (i = 0; i < forward; i++)
+			meta = nfp_meta_next(meta);
+	else
+		for (i = 0; i < backward; i++)
+			meta = nfp_meta_prev(meta);
+
+	return meta;
+}
+
+static int
+nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
+		   const struct bpf_verifier_env *env)
+{
+	const struct bpf_reg_state *reg0 = &env->cur_state.regs[0];
+
+	if (reg0->type != CONST_IMM) {
+		pr_info("unsupported exit state: %d, imm: %llx\n",
+			reg0->type, reg0->imm);
+		return -EINVAL;
+	}
+
+	if (nfp_prog->act != NN_ACT_DIRECT &&
+	    reg0->imm != 0 && (reg0->imm & ~0U) != ~0U) {
+		pr_info("unsupported exit state: %d, imm: %llx\n",
+			reg0->type, reg0->imm);
+		return -EINVAL;
+	}
+
+	if (nfp_prog->act == NN_ACT_DIRECT && reg0->imm <= TC_ACT_REDIRECT &&
+	    reg0->imm != TC_ACT_SHOT && reg0->imm != TC_ACT_STOLEN &&
+	    reg0->imm != TC_ACT_QUEUED) {
+		pr_info("unsupported exit state: %d, imm: %llx\n",
+			reg0->type, reg0->imm);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+nfp_bpf_check_ctx_ptr(struct nfp_prog *nfp_prog,
+		      const struct bpf_verifier_env *env, u8 reg)
+{
+	if (env->cur_state.regs[reg].type != PTR_TO_CTX)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx)
+{
+	struct nfp_bpf_analyzer_priv *priv = env->analyzer_priv;
+	struct nfp_insn_meta *meta = priv->meta;
+
+	meta = nfp_bpf_goto_meta(priv->prog, meta, insn_idx, env->prog->len);
+	priv->meta = meta;
+
+	if (meta->insn.src_reg == BPF_REG_10 ||
+	    meta->insn.dst_reg == BPF_REG_10) {
+		pr_err("stack not yet supported\n");
+		return -EINVAL;
+	}
+	if (meta->insn.src_reg >= MAX_BPF_REG ||
+	    meta->insn.dst_reg >= MAX_BPF_REG) {
+		pr_err("program uses extended registers - jit hardening?\n");
+		return -EINVAL;
+	}
+
+	if (meta->insn.code == (BPF_JMP | BPF_EXIT))
+		return nfp_bpf_check_exit(priv->prog, env);
+
+	if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_LDX | BPF_MEM))
+		return nfp_bpf_check_ctx_ptr(priv->prog, env,
+					     meta->insn.src_reg);
+	if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_MEM))
+		return nfp_bpf_check_ctx_ptr(priv->prog, env,
+					     meta->insn.dst_reg);
+
+	return 0;
+}
+
+static const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops = {
+	.insn_hook = nfp_verify_insn,
+};
+
+int nfp_prog_verify(struct nfp_prog *nfp_prog, struct bpf_prog *prog)
+{
+	struct nfp_bpf_analyzer_priv *priv;
+	int ret;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->prog = nfp_prog;
+	priv->meta = nfp_prog_first_meta(nfp_prog);
+
+	ret = bpf_analyzer(prog, &nfp_bpf_analyzer_ops, priv);
+
+	kfree(priv);
+
+	return ret;
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
index 690635660195..ed824e11a1e3 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
@@ -62,6 +62,9 @@
 /* Max time to wait for NFP to respond on updates (in seconds) */
 #define NFP_NET_POLL_TIMEOUT	5
 
+/* Interval for reading offloaded filter stats */
+#define NFP_NET_STAT_POLL_IVL	msecs_to_jiffies(100)
+
 /* Bar allocation */
 #define NFP_NET_CTRL_BAR	0
 #define NFP_NET_Q0_BAR		2
@@ -220,7 +223,7 @@ struct nfp_net_tx_ring {
 #define PCIE_DESC_RX_I_TCP_CSUM_OK	cpu_to_le16(BIT(11))
 #define PCIE_DESC_RX_I_UDP_CSUM		cpu_to_le16(BIT(10))
 #define PCIE_DESC_RX_I_UDP_CSUM_OK	cpu_to_le16(BIT(9))
-#define PCIE_DESC_RX_SPARE		cpu_to_le16(BIT(8))
+#define PCIE_DESC_RX_BPF		cpu_to_le16(BIT(8))
 #define PCIE_DESC_RX_EOP		cpu_to_le16(BIT(7))
 #define PCIE_DESC_RX_IP4_CSUM		cpu_to_le16(BIT(6))
 #define PCIE_DESC_RX_IP4_CSUM_OK	cpu_to_le16(BIT(5))
@@ -266,6 +269,8 @@ struct nfp_net_rx_desc {
 	};
 };
 
+#define NFP_NET_META_FIELD_MASK GENMASK(NFP_NET_META_FIELD_SIZE - 1, 0)
+
 struct nfp_net_rx_hash {
 	__be32 hash_type;
 	__be32 hash;
@@ -405,6 +410,11 @@ static inline bool nfp_net_fw_ver_eq(struct nfp_net_fw_version *fw_ver,
 	       fw_ver->minor == minor;
 }
 
+struct nfp_stat_pair {
+	u64 pkts;
+	u64 bytes;
+};
+
 /**
  * struct nfp_net - NFP network device structure
  * @pdev:               Backpointer to PCI device
@@ -413,6 +423,7 @@ static inline bool nfp_net_fw_ver_eq(struct nfp_net_fw_version *fw_ver,
  * @is_vf:              Is the driver attached to a VF?
  * @is_nfp3200:         Is the driver for a NFP-3200 card?
  * @fw_loaded:          Is the firmware loaded?
+ * @bpf_offload_skip_sw:  Offloaded BPF program will not be rerun by cls_bpf
  * @ctrl:               Local copy of the control register/word.
  * @fl_bufsz:           Currently configured size of the freelist buffers
  * @rx_offset:		Offset in the RX buffers where packet data starts
@@ -427,6 +438,11 @@ static inline bool nfp_net_fw_ver_eq(struct nfp_net_fw_version *fw_ver,
  * @rss_cfg:            RSS configuration
  * @rss_key:            RSS secret key
  * @rss_itbl:           RSS indirection table
+ * @rx_filter:		Filter offload statistics - dropped packets/bytes
+ * @rx_filter_prev:	Filter offload statistics - values from previous update
+ * @rx_filter_change:	Jiffies when statistics last changed
+ * @rx_filter_stats_timer:  Timer for polling filter offload statistics
+ * @rx_filter_lock:	Lock protecting timer state changes (teardown)
  * @max_tx_rings:       Maximum number of TX rings supported by the Firmware
  * @max_rx_rings:       Maximum number of RX rings supported by the Firmware
  * @num_tx_rings:       Currently configured number of TX rings
@@ -473,6 +489,7 @@ struct nfp_net {
 	unsigned is_vf:1;
 	unsigned is_nfp3200:1;
 	unsigned fw_loaded:1;
+	unsigned bpf_offload_skip_sw:1;
 
 	u32 ctrl;
 	u32 fl_bufsz;
@@ -502,6 +519,11 @@ struct nfp_net {
 	u8 rss_key[NFP_NET_CFG_RSS_KEY_SZ];
 	u8 rss_itbl[NFP_NET_CFG_RSS_ITBL_SZ];
 
+	struct nfp_stat_pair rx_filter, rx_filter_prev;
+	unsigned long rx_filter_change;
+	struct timer_list rx_filter_stats_timer;
+	spinlock_t rx_filter_lock;
+
 	int max_tx_rings;
 	int max_rx_rings;
 
@@ -561,12 +583,28 @@ struct nfp_net {
 /* Functions to read/write from/to a BAR
  * Performs any endian conversion necessary.
  */
+static inline u16 nn_readb(struct nfp_net *nn, int off)
+{
+	return readb(nn->ctrl_bar + off);
+}
+
 static inline void nn_writeb(struct nfp_net *nn, int off, u8 val)
 {
 	writeb(val, nn->ctrl_bar + off);
 }
 
-/* NFP-3200 can't handle 16-bit accesses too well - hence no readw/writew */
+/* NFP-3200 can't handle 16-bit accesses too well */
+static inline u16 nn_readw(struct nfp_net *nn, int off)
+{
+	WARN_ON_ONCE(nn->is_nfp3200);
+	return readw(nn->ctrl_bar + off);
+}
+
+static inline void nn_writew(struct nfp_net *nn, int off, u16 val)
+{
+	WARN_ON_ONCE(nn->is_nfp3200);
+	writew(val, nn->ctrl_bar + off);
+}
 
 static inline u32 nn_readl(struct nfp_net *nn, int off)
 {
@@ -757,4 +795,9 @@ static inline void nfp_net_debugfs_adapter_del(struct nfp_net *nn)
 }
 #endif /* CONFIG_NFP_NET_DEBUG */
 
+void nfp_net_filter_stats_timer(unsigned long data);
+int
+nfp_net_bpf_offload(struct nfp_net *nn, u32 handle, __be16 proto,
+		    struct tc_cls_bpf_offload *cls_bpf);
+
 #endif /* _NFP_NET_H_ */
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 88678c172b19..aee3fd2b6538 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -41,7 +41,6 @@
  *          Chris Telfer <chris.telfer@netronome.com>
  */
 
-#include <linux/version.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -61,6 +60,7 @@
 
 #include <linux/ktime.h>
 
+#include <net/pkt_cls.h>
 #include <net/vxlan.h>
 
 #include "nfp_net_ctrl.h"
@@ -1293,38 +1293,72 @@ static void nfp_net_rx_csum(struct nfp_net *nn, struct nfp_net_r_vector *r_vec,
 	}
 }
 
-/**
- * nfp_net_set_hash() - Set SKB hash data
- * @netdev: adapter's net_device structure
- * @skb:   SKB to set the hash data on
- * @rxd:   RX descriptor
- *
- * The RSS hash and hash-type are pre-pended to the packet data.
- * Extract and decode it and set the skb fields.
- */
 static void nfp_net_set_hash(struct net_device *netdev, struct sk_buff *skb,
-			     struct nfp_net_rx_desc *rxd)
+			     unsigned int type, __be32 *hash)
 {
-	struct nfp_net_rx_hash *rx_hash;
-
-	if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS) ||
-	    !(netdev->features & NETIF_F_RXHASH))
+	if (!(netdev->features & NETIF_F_RXHASH))
 		return;
 
-	rx_hash = (struct nfp_net_rx_hash *)(skb->data - sizeof(*rx_hash));
-
-	switch (be32_to_cpu(rx_hash->hash_type)) {
+	switch (type) {
 	case NFP_NET_RSS_IPV4:
 	case NFP_NET_RSS_IPV6:
 	case NFP_NET_RSS_IPV6_EX:
-		skb_set_hash(skb, be32_to_cpu(rx_hash->hash), PKT_HASH_TYPE_L3);
+		skb_set_hash(skb, get_unaligned_be32(hash), PKT_HASH_TYPE_L3);
 		break;
 	default:
-		skb_set_hash(skb, be32_to_cpu(rx_hash->hash), PKT_HASH_TYPE_L4);
+		skb_set_hash(skb, get_unaligned_be32(hash), PKT_HASH_TYPE_L4);
 		break;
 	}
 }
 
+static void
+nfp_net_set_hash_desc(struct net_device *netdev, struct sk_buff *skb,
+		      struct nfp_net_rx_desc *rxd)
+{
+	struct nfp_net_rx_hash *rx_hash;
+
+	if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS))
+		return;
+
+	rx_hash = (struct nfp_net_rx_hash *)(skb->data - sizeof(*rx_hash));
+
+	nfp_net_set_hash(netdev, skb, get_unaligned_be32(&rx_hash->hash_type),
+			 &rx_hash->hash);
+}
+
+static void *
+nfp_net_parse_meta(struct net_device *netdev, struct sk_buff *skb,
+		   int meta_len)
+{
+	u8 *data = skb->data - meta_len;
+	u32 meta_info;
+
+	meta_info = get_unaligned_be32(data);
+	data += 4;
+
+	while (meta_info) {
+		switch (meta_info & NFP_NET_META_FIELD_MASK) {
+		case NFP_NET_META_HASH:
+			meta_info >>= NFP_NET_META_FIELD_SIZE;
+			nfp_net_set_hash(netdev, skb,
+					 meta_info & NFP_NET_META_FIELD_MASK,
+					 (__be32 *)data);
+			data += 4;
+			break;
+		case NFP_NET_META_MARK:
+			skb->mark = get_unaligned_be32(data);
+			data += 4;
+			break;
+		default:
+			return NULL;
+		}
+
+		meta_info >>= NFP_NET_META_FIELD_SIZE;
+	}
+
+	return data;
+}
+
 /**
  * nfp_net_rx() - receive up to @budget packets on @rx_ring
  * @rx_ring:   RX ring to receive from
@@ -1439,18 +1473,29 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
 			skb_reserve(skb, nn->rx_offset);
 		skb_put(skb, data_len - meta_len);
 
-		nfp_net_set_hash(nn->netdev, skb, rxd);
-
-		/* Pad small frames to minimum */
-		if (skb_put_padto(skb, 60))
-			break;
-
 		/* Stats update */
 		u64_stats_update_begin(&r_vec->rx_sync);
 		r_vec->rx_pkts++;
 		r_vec->rx_bytes += skb->len;
 		u64_stats_update_end(&r_vec->rx_sync);
 
+		if (nn->fw_ver.major <= 3) {
+			nfp_net_set_hash_desc(nn->netdev, skb, rxd);
+		} else if (meta_len) {
+			void *end;
+
+			end = nfp_net_parse_meta(nn->netdev, skb, meta_len);
+			if (unlikely(end != skb->data)) {
+				u64_stats_update_begin(&r_vec->rx_sync);
+				r_vec->rx_drops++;
+				u64_stats_update_end(&r_vec->rx_sync);
+
+				dev_kfree_skb_any(skb);
+				nn_warn_ratelimit(nn, "invalid RX packet metadata\n");
+				continue;
+			}
+		}
+
 		skb_record_rx_queue(skb, rx_ring->idx);
 		skb->protocol = eth_type_trans(skb, nn->netdev);
 
@@ -2049,12 +2094,16 @@ static int nfp_net_netdev_open(struct net_device *netdev)
 
 	nn->rx_rings = kcalloc(nn->num_rx_rings, sizeof(*nn->rx_rings),
 			       GFP_KERNEL);
-	if (!nn->rx_rings)
+	if (!nn->rx_rings) {
+		err = -ENOMEM;
 		goto err_free_lsc;
+	}
 	nn->tx_rings = kcalloc(nn->num_tx_rings, sizeof(*nn->tx_rings),
 			       GFP_KERNEL);
-	if (!nn->tx_rings)
+	if (!nn->tx_rings) {
+		err = -ENOMEM;
 		goto err_free_rx_rings;
+	}
 
 	for (r = 0; r < nn->num_r_vecs; r++) {
 		err = nfp_net_prepare_vector(nn, &nn->r_vecs[r], r);
@@ -2387,6 +2436,31 @@ static struct rtnl_link_stats64 *nfp_net_stat64(struct net_device *netdev,
 	return stats;
 }
 
+static bool nfp_net_ebpf_capable(struct nfp_net *nn)
+{
+	if (nn->cap & NFP_NET_CFG_CTRL_BPF &&
+	    nn_readb(nn, NFP_NET_CFG_BPF_ABI) == NFP_NET_BPF_ABI)
+		return true;
+	return false;
+}
+
+static int
+nfp_net_setup_tc(struct net_device *netdev, u32 handle, __be16 proto,
+		 struct tc_to_netdev *tc)
+{
+	struct nfp_net *nn = netdev_priv(netdev);
+
+	if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
+		return -ENOTSUPP;
+	if (proto != htons(ETH_P_ALL))
+		return -ENOTSUPP;
+
+	if (tc->type == TC_SETUP_CLSBPF && nfp_net_ebpf_capable(nn))
+		return nfp_net_bpf_offload(nn, handle, proto, tc->cls_bpf);
+
+	return -EINVAL;
+}
+
 static int nfp_net_set_features(struct net_device *netdev,
 				netdev_features_t features)
 {
@@ -2441,6 +2515,11 @@ static int nfp_net_set_features(struct net_device *netdev,
 			new_ctrl &= ~NFP_NET_CFG_CTRL_GATHER;
 	}
 
+	if (changed & NETIF_F_HW_TC && nn->ctrl & NFP_NET_CFG_CTRL_BPF) {
+		nn_err(nn, "Cannot disable HW TC offload while in use\n");
+		return -EBUSY;
+	}
+
 	nn_dbg(nn, "Feature change 0x%llx -> 0x%llx (changed=0x%llx)\n",
 	       netdev->features, features, changed);
 
@@ -2590,6 +2669,7 @@ static const struct net_device_ops nfp_net_netdev_ops = {
 	.ndo_stop		= nfp_net_netdev_close,
 	.ndo_start_xmit		= nfp_net_tx,
 	.ndo_get_stats64	= nfp_net_stat64,
+	.ndo_setup_tc		= nfp_net_setup_tc,
 	.ndo_tx_timeout		= nfp_net_tx_timeout,
 	.ndo_set_rx_mode	= nfp_net_set_rx_mode,
 	.ndo_change_mtu		= nfp_net_change_mtu,
@@ -2615,7 +2695,7 @@ void nfp_net_info(struct nfp_net *nn)
 		nn->fw_ver.resv, nn->fw_ver.class,
 		nn->fw_ver.major, nn->fw_ver.minor,
 		nn->max_mtu);
-	nn_info(nn, "CAP: %#x %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
+	nn_info(nn, "CAP: %#x %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
 		nn->cap,
 		nn->cap & NFP_NET_CFG_CTRL_PROMISC  ? "PROMISC "  : "",
 		nn->cap & NFP_NET_CFG_CTRL_L2BC     ? "L2BCFILT " : "",
@@ -2632,7 +2712,8 @@ void nfp_net_info(struct nfp_net *nn)
 		nn->cap & NFP_NET_CFG_CTRL_MSIXAUTO ? "AUTOMASK " : "",
 		nn->cap & NFP_NET_CFG_CTRL_IRQMOD   ? "IRQMOD "   : "",
 		nn->cap & NFP_NET_CFG_CTRL_VXLAN    ? "VXLAN "    : "",
-		nn->cap & NFP_NET_CFG_CTRL_NVGRE    ? "NVGRE "	  : "");
+		nn->cap & NFP_NET_CFG_CTRL_NVGRE    ? "NVGRE "	  : "",
+		nfp_net_ebpf_capable(nn)            ? "BPF "	  : "");
 }
 
 /**
@@ -2675,10 +2756,13 @@ struct nfp_net *nfp_net_netdev_alloc(struct pci_dev *pdev,
 	nn->rxd_cnt = NFP_NET_RX_DESCS_DEFAULT;
 
 	spin_lock_init(&nn->reconfig_lock);
+	spin_lock_init(&nn->rx_filter_lock);
 	spin_lock_init(&nn->link_status_lock);
 
 	setup_timer(&nn->reconfig_timer,
 		    nfp_net_reconfig_timer, (unsigned long)nn);
+	setup_timer(&nn->rx_filter_stats_timer,
+		    nfp_net_filter_stats_timer, (unsigned long)nn);
 
 	return nn;
 }
@@ -2800,6 +2884,9 @@ int nfp_net_netdev_init(struct net_device *netdev)
 
 	netdev->features = netdev->hw_features;
 
+	if (nfp_net_ebpf_capable(nn))
+		netdev->hw_features |= NETIF_F_HW_TC;
+
 	/* Advertise but disable TSO by default. */
 	netdev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6);
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
index ad6c4e31cedd..93b10b441acb 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
@@ -66,6 +66,13 @@
 #define NFP_NET_LSO_MAX_HDR_SZ		255
 
 /**
+ * Prepend field types
+ */
+#define NFP_NET_META_FIELD_SIZE		4
+#define NFP_NET_META_HASH		1 /* next field carries hash type */
+#define NFP_NET_META_MARK		2
+
+/**
  * Hash type pre-pended when a RSS hash was computed
  */
 #define NFP_NET_RSS_NONE                0
@@ -123,6 +130,7 @@
 #define   NFP_NET_CFG_CTRL_L2SWITCH_LOCAL (0x1 << 23) /* Switch to local */
 #define   NFP_NET_CFG_CTRL_VXLAN	  (0x1 << 24) /* VXLAN tunnel support */
 #define   NFP_NET_CFG_CTRL_NVGRE	  (0x1 << 25) /* NVGRE tunnel support */
+#define   NFP_NET_CFG_CTRL_BPF		  (0x1 << 27) /* BPF offload capable */
 #define NFP_NET_CFG_UPDATE              0x0004
 #define   NFP_NET_CFG_UPDATE_GEN          (0x1 <<  0) /* General update */
 #define   NFP_NET_CFG_UPDATE_RING         (0x1 <<  1) /* Ring config change */
@@ -134,6 +142,7 @@
 #define   NFP_NET_CFG_UPDATE_RESET        (0x1 <<  7) /* Update due to FLR */
 #define   NFP_NET_CFG_UPDATE_IRQMOD       (0x1 <<  8) /* IRQ mod change */
 #define   NFP_NET_CFG_UPDATE_VXLAN	  (0x1 <<  9) /* VXLAN port change */
+#define   NFP_NET_CFG_UPDATE_BPF	  (0x1 << 10) /* BPF program load */
 #define   NFP_NET_CFG_UPDATE_ERR          (0x1 << 31) /* A error occurred */
 #define NFP_NET_CFG_TXRS_ENABLE         0x0008
 #define NFP_NET_CFG_RXRS_ENABLE         0x0010
@@ -196,10 +205,37 @@
 #define NFP_NET_CFG_VXLAN_SZ		  0x0008
 
 /**
- * 64B reserved for future use (0x0080 - 0x00c0)
+ * NFP6000 - BPF section
+ * @NFP_NET_CFG_BPF_ABI:	BPF ABI version
+ * @NFP_NET_CFG_BPF_CAP:	BPF capabilities
+ * @NFP_NET_CFG_BPF_MAX_LEN:	Maximum size of JITed BPF code in bytes
+ * @NFP_NET_CFG_BPF_START:	Offset at which BPF will be loaded
+ * @NFP_NET_CFG_BPF_DONE:	Offset to jump to on exit
+ * @NFP_NET_CFG_BPF_STACK_SZ:	Total size of stack area in 64B chunks
+ * @NFP_NET_CFG_BPF_INL_MTU:	Packet data split offset in 64B chunks
+ * @NFP_NET_CFG_BPF_SIZE:	Size of the JITed BPF code in instructions
+ * @NFP_NET_CFG_BPF_ADDR:	DMA address of the buffer with JITed BPF code
  */
-#define NFP_NET_CFG_RESERVED            0x0080
-#define NFP_NET_CFG_RESERVED_SZ         0x0040
+#define NFP_NET_CFG_BPF_ABI		0x0080
+#define   NFP_NET_BPF_ABI		1
+#define NFP_NET_CFG_BPF_CAP		0x0081
+#define   NFP_NET_BPF_CAP_RELO		(1 << 0) /* seamless reload */
+#define NFP_NET_CFG_BPF_MAX_LEN		0x0082
+#define NFP_NET_CFG_BPF_START		0x0084
+#define NFP_NET_CFG_BPF_DONE		0x0086
+#define NFP_NET_CFG_BPF_STACK_SZ	0x0088
+#define NFP_NET_CFG_BPF_INL_MTU		0x0089
+#define NFP_NET_CFG_BPF_SIZE		0x008e
+#define NFP_NET_CFG_BPF_ADDR		0x0090
+#define   NFP_NET_CFG_BPF_CFG_8CTX	(1 << 0) /* 8ctx mode */
+#define   NFP_NET_CFG_BPF_CFG_MASK	7ULL
+#define   NFP_NET_CFG_BPF_ADDR_MASK	(~NFP_NET_CFG_BPF_CFG_MASK)
+
+/**
+ * 40B reserved for future use (0x0098 - 0x00c0)
+ */
+#define NFP_NET_CFG_RESERVED            0x0098
+#define NFP_NET_CFG_RESERVED_SZ         0x0028
 
 /**
  * RSS configuration (0x0100 - 0x01ac):
@@ -303,6 +339,15 @@
 #define NFP_NET_CFG_STATS_TX_MC_FRAMES  (NFP_NET_CFG_STATS_BASE + 0x80)
 #define NFP_NET_CFG_STATS_TX_BC_FRAMES  (NFP_NET_CFG_STATS_BASE + 0x88)
 
+#define NFP_NET_CFG_STATS_APP0_FRAMES	(NFP_NET_CFG_STATS_BASE + 0x90)
+#define NFP_NET_CFG_STATS_APP0_BYTES	(NFP_NET_CFG_STATS_BASE + 0x98)
+#define NFP_NET_CFG_STATS_APP1_FRAMES	(NFP_NET_CFG_STATS_BASE + 0xa0)
+#define NFP_NET_CFG_STATS_APP1_BYTES	(NFP_NET_CFG_STATS_BASE + 0xa8)
+#define NFP_NET_CFG_STATS_APP2_FRAMES	(NFP_NET_CFG_STATS_BASE + 0xb0)
+#define NFP_NET_CFG_STATS_APP2_BYTES	(NFP_NET_CFG_STATS_BASE + 0xb8)
+#define NFP_NET_CFG_STATS_APP3_FRAMES	(NFP_NET_CFG_STATS_BASE + 0xc0)
+#define NFP_NET_CFG_STATS_APP3_BYTES	(NFP_NET_CFG_STATS_BASE + 0xc8)
+
 /**
  * Per ring stats (0x1000 - 0x1800)
  * options, 64bit per entry
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
index 7d7933d00b8f..3418f2277e9d 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
@@ -40,7 +40,6 @@
  *          Brad Petrus <brad.petrus@netronome.com>
  */
 
-#include <linux/version.h>
 #include <linux/kernel.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
@@ -107,6 +106,18 @@ static const struct _nfp_net_et_stats nfp_net_et_stats[] = {
 	{"dev_tx_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_FRAMES)},
 	{"dev_tx_mc_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_MC_FRAMES)},
 	{"dev_tx_bc_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_BC_FRAMES)},
+
+	{"bpf_pass_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP0_FRAMES)},
+	{"bpf_pass_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP0_BYTES)},
+	/* see comments in outro functions in nfp_bpf_jit.c to find out
+	 * how different BPF modes use app-specific counters
+	 */
+	{"bpf_app1_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP1_FRAMES)},
+	{"bpf_app1_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP1_BYTES)},
+	{"bpf_app2_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP2_FRAMES)},
+	{"bpf_app2_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP2_BYTES)},
+	{"bpf_app3_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP3_FRAMES)},
+	{"bpf_app3_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP3_BYTES)},
 };
 
 #define NN_ET_GLOBAL_STATS_LEN ARRAY_SIZE(nfp_net_et_stats)
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_offload.c b/drivers/net/ethernet/netronome/nfp/nfp_net_offload.c
new file mode 100644
index 000000000000..8acfb631a0ea
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_offload.c
@@ -0,0 +1,294 @@
+/*
+ * Copyright (C) 2016 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * nfp_net_offload.c
+ * Netronome network device driver: TC offload functions for PF and VF
+ */
+
+#include <linux/kernel.h>
+#include <linux/netdevice.h>
+#include <linux/pci.h>
+#include <linux/jiffies.h>
+#include <linux/timer.h>
+#include <linux/list.h>
+
+#include <net/pkt_cls.h>
+#include <net/tc_act/tc_gact.h>
+#include <net/tc_act/tc_mirred.h>
+
+#include "nfp_bpf.h"
+#include "nfp_net_ctrl.h"
+#include "nfp_net.h"
+
+void nfp_net_filter_stats_timer(unsigned long data)
+{
+	struct nfp_net *nn = (void *)data;
+	struct nfp_stat_pair latest;
+
+	spin_lock_bh(&nn->rx_filter_lock);
+
+	if (nn->ctrl & NFP_NET_CFG_CTRL_BPF)
+		mod_timer(&nn->rx_filter_stats_timer,
+			  jiffies + NFP_NET_STAT_POLL_IVL);
+
+	spin_unlock_bh(&nn->rx_filter_lock);
+
+	latest.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES);
+	latest.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES);
+
+	if (latest.pkts != nn->rx_filter.pkts)
+		nn->rx_filter_change = jiffies;
+
+	nn->rx_filter = latest;
+}
+
+static void nfp_net_bpf_stats_reset(struct nfp_net *nn)
+{
+	nn->rx_filter.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES);
+	nn->rx_filter.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES);
+	nn->rx_filter_prev = nn->rx_filter;
+	nn->rx_filter_change = jiffies;
+}
+
+static int
+nfp_net_bpf_stats_update(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
+{
+	struct tc_action *a;
+	LIST_HEAD(actions);
+	u64 bytes, pkts;
+
+	pkts = nn->rx_filter.pkts - nn->rx_filter_prev.pkts;
+	bytes = nn->rx_filter.bytes - nn->rx_filter_prev.bytes;
+	bytes -= pkts * ETH_HLEN;
+
+	nn->rx_filter_prev = nn->rx_filter;
+
+	preempt_disable();
+
+	tcf_exts_to_list(cls_bpf->exts, &actions);
+	list_for_each_entry(a, &actions, list)
+		tcf_action_stats_update(a, bytes, pkts, nn->rx_filter_change);
+
+	preempt_enable();
+
+	return 0;
+}
+
+static int
+nfp_net_bpf_get_act(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
+{
+	const struct tc_action *a;
+	LIST_HEAD(actions);
+
+	/* TC direct action */
+	if (cls_bpf->exts_integrated) {
+		if (tc_no_actions(cls_bpf->exts))
+			return NN_ACT_DIRECT;
+
+		return -ENOTSUPP;
+	}
+
+	/* TC legacy mode */
+	if (!tc_single_action(cls_bpf->exts))
+		return -ENOTSUPP;
+
+	tcf_exts_to_list(cls_bpf->exts, &actions);
+	list_for_each_entry(a, &actions, list) {
+		if (is_tcf_gact_shot(a))
+			return NN_ACT_TC_DROP;
+
+		if (is_tcf_mirred_redirect(a) &&
+		    tcf_mirred_ifindex(a) == nn->netdev->ifindex)
+			return NN_ACT_TC_REDIR;
+	}
+
+	return -ENOTSUPP;
+}
+
+static int
+nfp_net_bpf_offload_prepare(struct nfp_net *nn,
+			    struct tc_cls_bpf_offload *cls_bpf,
+			    struct nfp_bpf_result *res,
+			    void **code, dma_addr_t *dma_addr, u16 max_instr)
+{
+	unsigned int code_sz = max_instr * sizeof(u64);
+	enum nfp_bpf_action_type act;
+	u16 start_off, done_off;
+	unsigned int max_mtu;
+	int ret;
+
+	if (!IS_ENABLED(CONFIG_BPF_SYSCALL))
+		return -ENOTSUPP;
+
+	ret = nfp_net_bpf_get_act(nn, cls_bpf);
+	if (ret < 0)
+		return ret;
+	act = ret;
+
+	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
+	if (max_mtu < nn->netdev->mtu) {
+		nn_info(nn, "BPF offload not supported with MTU larger than HW packet split boundary\n");
+		return -ENOTSUPP;
+	}
+
+	start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
+	done_off = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
+
+	*code = dma_zalloc_coherent(&nn->pdev->dev, code_sz, dma_addr,
+				    GFP_KERNEL);
+	if (!*code)
+		return -ENOMEM;
+
+	ret = nfp_bpf_jit(cls_bpf->prog, *code, act, start_off, done_off,
+			  max_instr, res);
+	if (ret)
+		goto out;
+
+	return 0;
+
+out:
+	dma_free_coherent(&nn->pdev->dev, code_sz, *code, *dma_addr);
+	return ret;
+}
+
+static void
+nfp_net_bpf_load_and_start(struct nfp_net *nn, u32 tc_flags,
+			   void *code, dma_addr_t dma_addr,
+			   unsigned int code_sz, unsigned int n_instr,
+			   bool dense_mode)
+{
+	u64 bpf_addr = dma_addr;
+	int err;
+
+	nn->bpf_offload_skip_sw = !!(tc_flags & TCA_CLS_FLAGS_SKIP_SW);
+
+	if (dense_mode)
+		bpf_addr |= NFP_NET_CFG_BPF_CFG_8CTX;
+
+	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, n_instr);
+	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, bpf_addr);
+
+	/* Load up the JITed code */
+	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
+	if (err)
+		nn_err(nn, "FW command error while loading BPF: %d\n", err);
+
+	/* Enable passing packets through BPF function */
+	nn->ctrl |= NFP_NET_CFG_CTRL_BPF;
+	nn_writel(nn, NFP_NET_CFG_CTRL, nn->ctrl);
+	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
+	if (err)
+		nn_err(nn, "FW command error while enabling BPF: %d\n", err);
+
+	dma_free_coherent(&nn->pdev->dev, code_sz, code, dma_addr);
+
+	nfp_net_bpf_stats_reset(nn);
+	mod_timer(&nn->rx_filter_stats_timer, jiffies + NFP_NET_STAT_POLL_IVL);
+}
+
+static int nfp_net_bpf_stop(struct nfp_net *nn)
+{
+	if (!(nn->ctrl & NFP_NET_CFG_CTRL_BPF))
+		return 0;
+
+	spin_lock_bh(&nn->rx_filter_lock);
+	nn->ctrl &= ~NFP_NET_CFG_CTRL_BPF;
+	spin_unlock_bh(&nn->rx_filter_lock);
+	nn_writel(nn, NFP_NET_CFG_CTRL, nn->ctrl);
+
+	del_timer_sync(&nn->rx_filter_stats_timer);
+	nn->bpf_offload_skip_sw = 0;
+
+	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
+}
+
+int
+nfp_net_bpf_offload(struct nfp_net *nn, u32 handle, __be16 proto,
+		    struct tc_cls_bpf_offload *cls_bpf)
+{
+	struct nfp_bpf_result res;
+	dma_addr_t dma_addr;
+	u16 max_instr;
+	void *code;
+	int err;
+
+	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
+
+	switch (cls_bpf->command) {
+	case TC_CLSBPF_REPLACE:
+		/* There is nothing stopping us from implementing seamless
+		 * replace but the simple method of loading I adopted in
+		 * the firmware does not handle atomic replace (i.e. we have to
+		 * stop the BPF offload and re-enable it).  Leaking-in a few
+		 * frames which didn't have BPF applied in the hardware should
+		 * be fine if software fallback is available, though.
+		 */
+		if (nn->bpf_offload_skip_sw)
+			return -EBUSY;
+
+		err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code,
+						  &dma_addr, max_instr);
+		if (err)
+			return err;
+
+		nfp_net_bpf_stop(nn);
+		nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code,
+					   dma_addr, max_instr * sizeof(u64),
+					   res.n_instr, res.dense_mode);
+		return 0;
+
+	case TC_CLSBPF_ADD:
+		if (nn->ctrl & NFP_NET_CFG_CTRL_BPF)
+			return -EBUSY;
+
+		err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code,
+						  &dma_addr, max_instr);
+		if (err)
+			return err;
+
+		nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code,
+					   dma_addr, max_instr * sizeof(u64),
+					   res.n_instr, res.dense_mode);
+		return 0;
+
+	case TC_CLSBPF_DESTROY:
+		return nfp_net_bpf_stop(nn);
+
+	case TC_CLSBPF_STATS:
+		return nfp_net_bpf_stats_update(nn, cls_bpf);
+
+	default:
+		return -ENOTSUPP;
+	}
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
index 37abef016a0a..2800bbf65a89 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
@@ -38,7 +38,6 @@
  *         Rolf Neugebauer <rolf.neugebauer@netronome.com>
  */
 
-#include <linux/version.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -134,7 +133,7 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev,
 	}
 
 	nfp_net_get_fw_version(&fw_ver, ctrl_bar);
-	if (fw_ver.class != NFP_NET_CFG_VERSION_CLASS_GENERIC) {
+	if (fw_ver.resv || fw_ver.class != NFP_NET_CFG_VERSION_CLASS_GENERIC) {
 		dev_err(&pdev->dev, "Unknown Firmware ABI %d.%d.%d.%d\n",
 			fw_ver.resv, fw_ver.class, fw_ver.major, fw_ver.minor);
 		err = -EINVAL;
@@ -142,16 +141,14 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev,
 	}
 
 	/* Determine stride */
-	if (nfp_net_fw_ver_eq(&fw_ver, 0, 0, 0, 0) ||
-	    nfp_net_fw_ver_eq(&fw_ver, 0, 0, 0, 1) ||
-	    nfp_net_fw_ver_eq(&fw_ver, 0, 0, 0x12, 0x48)) {
+	if (nfp_net_fw_ver_eq(&fw_ver, 0, 0, 0, 1)) {
 		stride = 2;
 		tx_bar_no = NFP_NET_Q0_BAR;
 		rx_bar_no = NFP_NET_Q1_BAR;
 		dev_warn(&pdev->dev, "OBSOLETE Firmware detected - VF isolation not available\n");
 	} else {
 		switch (fw_ver.major) {
-		case 1 ... 3:
+		case 1 ... 4:
 			if (is_nfp3200) {
 				stride = 2;
 				tx_bar_no = NFP_NET_Q0_BAR;
diff --git a/drivers/net/ethernet/nuvoton/w90p910_ether.c b/drivers/net/ethernet/nuvoton/w90p910_ether.c
index 87b7b814778b..712d8bcb7d8c 100644
--- a/drivers/net/ethernet/nuvoton/w90p910_ether.c
+++ b/drivers/net/ethernet/nuvoton/w90p910_ether.c
@@ -751,7 +751,7 @@ static void netdev_rx(struct net_device *dev)
 				dev_err(&pdev->dev, "rx crc err\n");
 				ether->stats.rx_crc_errors++;
 			} else if (status & RXDS_ALIE) {
-				dev_err(&pdev->dev, "rx aligment err\n");
+				dev_err(&pdev->dev, "rx alignment err\n");
 				ether->stats.rx_frame_errors++;
 			} else if (status & RXDS_PTLE) {
 				dev_err(&pdev->dev, "rx longer err\n");
diff --git a/drivers/net/ethernet/nxp/lpc_eth.c b/drivers/net/ethernet/nxp/lpc_eth.c
index 4d4ecba0aad9..8e13ec84c538 100644
--- a/drivers/net/ethernet/nxp/lpc_eth.c
+++ b/drivers/net/ethernet/nxp/lpc_eth.c
@@ -475,14 +475,6 @@ static void __lpc_get_mac(struct netdata_local *pldat, u8 *mac)
 	mac[5] = tmp >> 8;
 }
 
-static void __lpc_eth_clock_enable(struct netdata_local *pldat, bool enable)
-{
-	if (enable)
-		clk_prepare_enable(pldat->clk);
-	else
-		clk_disable_unprepare(pldat->clk);
-}
-
 static void __lpc_params_setup(struct netdata_local *pldat)
 {
 	u32 tmp;
@@ -1056,7 +1048,7 @@ static int lpc_eth_close(struct net_device *ndev)
 	writel(0, LPC_ENET_MAC2(pldat->net_base));
 	spin_unlock_irqrestore(&pldat->lock, flags);
 
-	__lpc_eth_clock_enable(pldat, false);
+	clk_disable_unprepare(pldat->clk);
 
 	return 0;
 }
@@ -1197,11 +1189,14 @@ static int lpc_eth_ioctl(struct net_device *ndev, struct ifreq *req, int cmd)
 static int lpc_eth_open(struct net_device *ndev)
 {
 	struct netdata_local *pldat = netdev_priv(ndev);
+	int ret;
 
 	if (netif_msg_ifup(pldat))
 		dev_dbg(&pldat->pdev->dev, "enabling %s\n", ndev->name);
 
-	__lpc_eth_clock_enable(pldat, true);
+	ret = clk_prepare_enable(pldat->clk);
+	if (ret)
+		return ret;
 
 	/* Suspended PHY makes LPC ethernet core block, so resume now */
 	phy_resume(ndev->phydev);
@@ -1320,7 +1315,9 @@ static int lpc_eth_drv_probe(struct platform_device *pdev)
 	}
 
 	/* Enable network clock */
-	__lpc_eth_clock_enable(pldat, true);
+	ret = clk_prepare_enable(pldat->clk);
+	if (ret)
+		goto err_out_clk_put;
 
 	/* Map IO space */
 	pldat->net_base = ioremap(res->start, resource_size(res));
@@ -1454,6 +1451,7 @@ err_out_iounmap:
 	iounmap(pldat->net_base);
 err_out_disable_clocks:
 	clk_disable_unprepare(pldat->clk);
+err_out_clk_put:
 	clk_put(pldat->clk);
 err_out_free_dev:
 	free_netdev(ndev);
diff --git a/drivers/net/ethernet/qlogic/Kconfig b/drivers/net/ethernet/qlogic/Kconfig
index 6ba48406899e..0df1391f9663 100644
--- a/drivers/net/ethernet/qlogic/Kconfig
+++ b/drivers/net/ethernet/qlogic/Kconfig
@@ -88,6 +88,9 @@ config QED
 	---help---
 	  This enables the support for ...
 
+config QED_LL2
+	bool
+
 config QED_SRIOV
 	bool "QLogic QED 25/40/100Gb SR-IOV support"
 	depends on QED && PCI_IOV
@@ -104,4 +107,15 @@ config QEDE
 	---help---
 	  This enables the support for ...
 
+config INFINIBAND_QEDR
+	tristate "QLogic qede RoCE sources [debug]"
+	depends on QEDE && 64BIT
+	select QED_LL2
+	default n
+	---help---
+	  This provides a temporary node that allows the compilation
+	  and logical testing of the InfiniBand over Ethernet support
+	  for QLogic QED. This would be replaced by the 'real' option
+	  once the QEDR driver is added [+relocated].
+
 endif # NET_VENDOR_QLOGIC
diff --git a/drivers/net/ethernet/qlogic/qed/Makefile b/drivers/net/ethernet/qlogic/qed/Makefile
index d1f157e439cf..cda0af7fbc20 100644
--- a/drivers/net/ethernet/qlogic/qed/Makefile
+++ b/drivers/net/ethernet/qlogic/qed/Makefile
@@ -2,5 +2,7 @@ obj-$(CONFIG_QED) := qed.o
 
 qed-y := qed_cxt.o qed_dev.o qed_hw.o qed_init_fw_funcs.o qed_init_ops.o \
 	 qed_int.o qed_main.o qed_mcp.o qed_sp_commands.o qed_spq.o qed_l2.o \
-	 qed_selftest.o qed_dcbx.o
+	 qed_selftest.o qed_dcbx.o qed_debug.o
 qed-$(CONFIG_QED_SRIOV) += qed_sriov.o qed_vf.o
+qed-$(CONFIG_QED_LL2) += qed_ll2.o
+qed-$(CONFIG_INFINIBAND_QEDR) += qed_roce.o
diff --git a/drivers/net/ethernet/qlogic/qed/qed.h b/drivers/net/ethernet/qlogic/qed/qed.h
index 35e53771533f..653bb5735f0c 100644
--- a/drivers/net/ethernet/qlogic/qed/qed.h
+++ b/drivers/net/ethernet/qlogic/qed/qed.h
@@ -23,10 +23,11 @@
 #include <linux/zlib.h>
 #include <linux/hashtable.h>
 #include <linux/qed/qed_if.h>
+#include "qed_debug.h"
 #include "qed_hsi.h"
 
 extern const struct qed_common_ops qed_common_ops_pass;
-#define DRV_MODULE_VERSION "8.7.1.20"
+#define DRV_MODULE_VERSION "8.10.9.20"
 
 #define MAX_HWFNS_PER_DEVICE    (4)
 #define NAME_SIZE 16
@@ -34,6 +35,9 @@ extern const struct qed_common_ops qed_common_ops_pass;
 
 #define QED_WFQ_UNIT	100
 
+#define QED_WID_SIZE            (1024)
+#define QED_PF_DEMS_SIZE        (4)
+
 /* cau states */
 enum qed_coalescing_mode {
 	QED_COAL_MODE_DISABLE,
@@ -42,11 +46,21 @@ enum qed_coalescing_mode {
 
 struct qed_eth_cb_ops;
 struct qed_dev_info;
+union qed_mcp_protocol_stats;
+enum qed_mcp_protocol_type;
 
 /* helpers */
 static inline u32 qed_db_addr(u32 cid, u32 DEMS)
 {
 	u32 db_addr = FIELD_VALUE(DB_LEGACY_ADDR_DEMS, DEMS) |
+		      (cid * QED_PF_DEMS_SIZE);
+
+	return db_addr;
+}
+
+static inline u32 qed_db_addr_vf(u32 cid, u32 DEMS)
+{
+	u32 db_addr = FIELD_VALUE(DB_LEGACY_ADDR_DEMS, DEMS) |
 		      FIELD_VALUE(DB_LEGACY_ADDR_ICID, cid);
 
 	return db_addr;
@@ -69,6 +83,7 @@ struct qed_sb_info;
 struct qed_sb_attn_info;
 struct qed_cxt_mngr;
 struct qed_sb_sp_info;
+struct qed_ll2_info;
 struct qed_mcp_info;
 
 struct qed_rt_data {
@@ -148,13 +163,17 @@ enum QED_RESOURCES {
 	QED_RL,
 	QED_MAC,
 	QED_VLAN,
+	QED_RDMA_CNQ_RAM,
 	QED_ILT,
+	QED_LL2_QUEUE,
+	QED_RDMA_STATS_QUEUE,
 	QED_MAX_RESC,
 };
 
 enum QED_FEATURE {
 	QED_PF_L2_QUE,
 	QED_VF,
+	QED_RDMA_CNQ,
 	QED_MAX_FEATURES,
 };
 
@@ -357,6 +376,9 @@ struct qed_hwfn {
 	struct qed_sb_attn_info		*p_sb_attn;
 
 	/* Protocol related */
+	bool				using_ll2;
+	struct qed_ll2_info		*p_ll2_info;
+	struct qed_rdma_info		*p_rdma_info;
 	struct qed_pf_params		pf_params;
 
 	bool b_rdma_enabled_in_prs;
@@ -393,6 +415,19 @@ struct qed_hwfn {
 	/* Buffer for unzipping firmware data */
 	void				*unzip_buf;
 
+	struct dbg_tools_data		dbg_info;
+
+	/* PWM region specific data */
+	u32				dpi_size;
+	u32				dpi_count;
+
+	/* This is used to calculate the doorbell address */
+	u32 dpi_start_offset;
+
+	/* If one of the following is set then EDPM shouldn't be used */
+	u8 dcbx_no_edpm;
+	u8 db_bar_no_edpm;
+
 	struct qed_simd_fp_handler	simd_proto_handler[64];
 
 #ifdef CONFIG_QED_SRIOV
@@ -402,6 +437,7 @@ struct qed_hwfn {
 #endif
 
 	struct z_stream_s		*stream;
+	struct qed_roce_ll2_info	*ll2;
 };
 
 struct pci_params {
@@ -426,6 +462,21 @@ struct qed_int_params {
 	bool			fp_initialized;
 	u8			fp_msix_base;
 	u8			fp_msix_cnt;
+	u8			rdma_msix_base;
+	u8			rdma_msix_cnt;
+};
+
+struct qed_dbg_feature {
+	struct dentry *dentry;
+	u8 *dump_buf;
+	u32 buf_size;
+	u32 dumped_dwords;
+};
+
+struct qed_dbg_params {
+	struct qed_dbg_feature features[DBG_FEATURE_NUM];
+	u8 engine_for_debug;
+	bool print_data;
 };
 
 struct qed_dev {
@@ -442,6 +493,8 @@ struct qed_dev {
 				 CHIP_REV_IS_A0(dev))
 #define QED_IS_BB_B0(dev)       (QED_IS_BB(dev) && \
 				 CHIP_REV_IS_B0(dev))
+#define QED_IS_AH(dev)  ((dev)->type == QED_DEV_TYPE_AH)
+#define QED_IS_K2(dev)  QED_IS_AH(dev)
 
 #define QED_GET_TYPE(dev)       (QED_IS_BB_A0(dev) ? CHIP_BB_A0 : \
 				 QED_IS_BB_B0(dev) ? CHIP_BB_B0 : CHIP_K2)
@@ -517,7 +570,6 @@ struct qed_dev {
 
 	bool				b_is_vf;
 	u32				drv_type;
-
 	struct qed_eth_stats		*reset_stats;
 	struct qed_fw_data		*fw_data;
 
@@ -542,7 +594,18 @@ struct qed_dev {
 	} protocol_ops;
 	void				*ops_cookie;
 
+	struct qed_dbg_params		dbg_params;
+
+#ifdef CONFIG_QED_LL2
+	struct qed_cb_ll2_info		*ll2;
+	u8				ll2_mac_address[ETH_ALEN];
+#endif
+
 	const struct firmware		*firmware;
+
+	u32 rdma_max_sge;
+	u32 rdma_max_inline;
+	u32 rdma_max_srq_sge;
 };
 
 #define NUM_OF_VFS(dev)         MAX_NUM_VFS_BB
@@ -561,9 +624,18 @@ struct qed_dev {
 static inline u8 qed_concrete_to_sw_fid(struct qed_dev *cdev,
 					u32 concrete_fid)
 {
+	u8 vfid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID);
 	u8 pfid = GET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID);
+	u8 vf_valid = GET_FIELD(concrete_fid,
+				PXP_CONCRETE_FID_VFVALID);
+	u8 sw_fid;
 
-	return pfid;
+	if (vf_valid)
+		sw_fid = vfid + MAX_NUM_PFS;
+	else
+		sw_fid = pfid;
+
+	return sw_fid;
 }
 
 #define PURE_LB_TC 8
@@ -597,7 +669,9 @@ void qed_link_update(struct qed_hwfn *hwfn);
 u32 qed_unzip_data(struct qed_hwfn *p_hwfn,
 		   u32 input_len, u8 *input_buf,
 		   u32 max_size, u8 *unzip_buf);
-
+void qed_get_protocol_stats(struct qed_dev *cdev,
+			    enum qed_mcp_protocol_type type,
+			    union qed_mcp_protocol_stats *stats);
 int qed_slowpath_irq_req(struct qed_hwfn *hwfn);
 
 #endif /* _QED_H */
diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.c b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
index 1c35f376143e..82370a1a59ad 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_cxt.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
@@ -48,7 +48,13 @@
 #define TM_ELEM_SIZE    4
 
 /* ILT constants */
+#if IS_ENABLED(CONFIG_INFINIBAND_QEDR)
+/* For RoCE we configure to 64K to cover for RoCE max tasks 256K purpose. */
+#define ILT_DEFAULT_HW_P_SIZE		4
+#else
 #define ILT_DEFAULT_HW_P_SIZE		3
+#endif
+
 #define ILT_PAGE_IN_BYTES(hw_p_size)	(1U << ((hw_p_size) + 12))
 #define ILT_CFG_REG(cli, reg)	PSWRQ2_REG_ ## cli ## _ ## reg ## _RT_OFFSET
 
@@ -377,9 +383,8 @@ static void qed_cxt_set_proto_cid_count(struct qed_hwfn *p_hwfn,
 	}
 }
 
-u32 qed_cxt_get_proto_cid_count(struct qed_hwfn		*p_hwfn,
-				enum protocol_type	type,
-				u32			*vf_cid)
+u32 qed_cxt_get_proto_cid_count(struct qed_hwfn *p_hwfn,
+				enum protocol_type type, u32 *vf_cid)
 {
 	if (vf_cid)
 		*vf_cid = p_hwfn->p_cxt_mngr->conn_cfg[type].cids_per_vf;
@@ -405,10 +410,10 @@ u32 qed_cxt_get_proto_tid_count(struct qed_hwfn *p_hwfn,
 	return cnt;
 }
 
-static void
-qed_cxt_set_proto_tid_count(struct qed_hwfn *p_hwfn,
-			    enum protocol_type proto,
-			    u8 seg, u8 seg_type, u32 count, bool has_fl)
+static void qed_cxt_set_proto_tid_count(struct qed_hwfn *p_hwfn,
+					enum protocol_type proto,
+					u8 seg,
+					u8 seg_type, u32 count, bool has_fl)
 {
 	struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
 	struct qed_tid_seg *p_seg = &p_mngr->conn_cfg[proto].tid_seg[seg];
@@ -420,8 +425,7 @@ qed_cxt_set_proto_tid_count(struct qed_hwfn *p_hwfn,
 
 static void qed_ilt_cli_blk_fill(struct qed_ilt_client_cfg *p_cli,
 				 struct qed_ilt_cli_blk *p_blk,
-				 u32 start_line, u32 total_size,
-				 u32 elem_size)
+				 u32 start_line, u32 total_size, u32 elem_size)
 {
 	u32 ilt_size = ILT_PAGE_IN_BYTES(p_cli->p_size.val);
 
@@ -448,8 +452,7 @@ static void qed_ilt_cli_adv_line(struct qed_hwfn *p_hwfn,
 		p_cli->first.val = *p_line;
 
 	p_cli->active = true;
-	*p_line += DIV_ROUND_UP(p_blk->total_size,
-				p_blk->real_size_in_page);
+	*p_line += DIV_ROUND_UP(p_blk->total_size, p_blk->real_size_in_page);
 	p_cli->last.val = *p_line - 1;
 
 	DP_VERBOSE(p_hwfn, QED_MSG_ILT,
@@ -795,10 +798,9 @@ static int qed_cxt_src_t2_alloc(struct qed_hwfn *p_hwfn)
 	p_mngr->t2_num_pages = DIV_ROUND_UP(total_size, psz);
 
 	/* allocate t2 */
-	p_mngr->t2 = kzalloc(p_mngr->t2_num_pages * sizeof(struct qed_dma_mem),
+	p_mngr->t2 = kcalloc(p_mngr->t2_num_pages, sizeof(struct qed_dma_mem),
 			     GFP_KERNEL);
 	if (!p_mngr->t2) {
-		DP_NOTICE(p_hwfn, "Failed to allocate t2 table\n");
 		rc = -ENOMEM;
 		goto t2_fail;
 	}
@@ -926,12 +928,9 @@ static int qed_ilt_blk_alloc(struct qed_hwfn *p_hwfn,
 		void *p_virt;
 		u32 size;
 
-		size = min_t(u32, sz_left,
-			     p_blk->real_size_in_page);
+		size = min_t(u32, sz_left, p_blk->real_size_in_page);
 		p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
-					    size,
-					    &p_phys,
-					    GFP_KERNEL);
+					    size, &p_phys, GFP_KERNEL);
 		if (!p_virt)
 			return -ENOMEM;
 		memset(p_virt, 0, size);
@@ -963,7 +962,6 @@ static int qed_ilt_shadow_alloc(struct qed_hwfn *p_hwfn)
 	p_mngr->ilt_shadow = kcalloc(size, sizeof(struct qed_dma_mem),
 				     GFP_KERNEL);
 	if (!p_mngr->ilt_shadow) {
-		DP_NOTICE(p_hwfn, "Failed to allocate ilt shadow table\n");
 		rc = -ENOMEM;
 		goto ilt_shadow_fail;
 	}
@@ -976,7 +974,7 @@ static int qed_ilt_shadow_alloc(struct qed_hwfn *p_hwfn)
 		for (j = 0; j < ILT_CLI_PF_BLOCKS; j++) {
 			p_blk = &clients[i].pf_blks[j];
 			rc = qed_ilt_blk_alloc(p_hwfn, p_blk, i, 0);
-			if (rc != 0)
+			if (rc)
 				goto ilt_shadow_fail;
 		}
 		for (k = 0; k < p_mngr->vf_count; k++) {
@@ -985,7 +983,7 @@ static int qed_ilt_shadow_alloc(struct qed_hwfn *p_hwfn)
 
 				p_blk = &clients[i].vf_blks[j];
 				rc = qed_ilt_blk_alloc(p_hwfn, p_blk, i, lines);
-				if (rc != 0)
+				if (rc)
 					goto ilt_shadow_fail;
 			}
 		}
@@ -1056,10 +1054,8 @@ int qed_cxt_mngr_alloc(struct qed_hwfn *p_hwfn)
 	u32 i;
 
 	p_mngr = kzalloc(sizeof(*p_mngr), GFP_KERNEL);
-	if (!p_mngr) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `struct qed_cxt_mngr'\n");
+	if (!p_mngr)
 		return -ENOMEM;
-	}
 
 	/* Initialize ILT client registers */
 	clients = p_mngr->clients;
@@ -1111,24 +1107,18 @@ int qed_cxt_tables_alloc(struct qed_hwfn *p_hwfn)
 
 	/* Allocate the ILT shadow table */
 	rc = qed_ilt_shadow_alloc(p_hwfn);
-	if (rc) {
-		DP_NOTICE(p_hwfn, "Failed to allocate ilt memory\n");
+	if (rc)
 		goto tables_alloc_fail;
-	}
 
 	/* Allocate the T2  table */
 	rc = qed_cxt_src_t2_alloc(p_hwfn);
-	if (rc) {
-		DP_NOTICE(p_hwfn, "Failed to allocate T2 memory\n");
+	if (rc)
 		goto tables_alloc_fail;
-	}
 
 	/* Allocate and initialize the acquired cids bitmaps */
 	rc = qed_cid_map_alloc(p_hwfn);
-	if (rc) {
-		DP_NOTICE(p_hwfn, "Failed to allocate cid maps\n");
+	if (rc)
 		goto tables_alloc_fail;
-	}
 
 	return 0;
 
@@ -1672,7 +1662,7 @@ static void qed_tm_init_pf(struct qed_hwfn *p_hwfn)
 		     p_hwfn->rel_pf_id * NUM_TASK_PF_SEGMENTS + i);
 
 		STORE_RT_REG_AGG(p_hwfn, rt_reg, cfg_word);
-		active_seg_mask |= (tm_iids.pf_tids[i] ? (1 << i) : 0);
+		active_seg_mask |= (tm_iids.pf_tids[i] ? BIT(i) : 0);
 
 		tm_offset += tm_iids.pf_tids[i];
 	}
@@ -1702,8 +1692,7 @@ void qed_cxt_hw_init_pf(struct qed_hwfn *p_hwfn)
 }
 
 int qed_cxt_acquire_cid(struct qed_hwfn *p_hwfn,
-			enum protocol_type type,
-			u32 *p_cid)
+			enum protocol_type type, u32 *p_cid)
 {
 	struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
 	u32 rel_cid;
@@ -1717,8 +1706,7 @@ int qed_cxt_acquire_cid(struct qed_hwfn *p_hwfn,
 				      p_mngr->acquired[type].max_count);
 
 	if (rel_cid >= p_mngr->acquired[type].max_count) {
-		DP_NOTICE(p_hwfn, "no CID available for protocol %d\n",
-			  type);
+		DP_NOTICE(p_hwfn, "no CID available for protocol %d\n", type);
 		return -EINVAL;
 	}
 
@@ -1730,8 +1718,7 @@ int qed_cxt_acquire_cid(struct qed_hwfn *p_hwfn,
 }
 
 static bool qed_cxt_test_cid_acquired(struct qed_hwfn *p_hwfn,
-				      u32 cid,
-				      enum protocol_type *p_type)
+				      u32 cid, enum protocol_type *p_type)
 {
 	struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
 	struct qed_cid_acquired_map *p_map;
@@ -1763,8 +1750,7 @@ static bool qed_cxt_test_cid_acquired(struct qed_hwfn *p_hwfn,
 	return true;
 }
 
-void qed_cxt_release_cid(struct qed_hwfn *p_hwfn,
-			 u32 cid)
+void qed_cxt_release_cid(struct qed_hwfn *p_hwfn, u32 cid)
 {
 	struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
 	enum protocol_type type;
@@ -1781,8 +1767,7 @@ void qed_cxt_release_cid(struct qed_hwfn *p_hwfn,
 	__clear_bit(rel_cid, p_mngr->acquired[type].cid_map);
 }
 
-int qed_cxt_get_cid_info(struct qed_hwfn *p_hwfn,
-			 struct qed_cxt_info *p_info)
+int qed_cxt_get_cid_info(struct qed_hwfn *p_hwfn, struct qed_cxt_info *p_info)
 {
 	struct qed_cxt_mngr *p_mngr = p_hwfn->p_cxt_mngr;
 	u32 conn_cxt_size, hw_p_size, cxts_per_p, line;
@@ -1860,6 +1845,8 @@ int qed_cxt_set_pf_params(struct qed_hwfn *p_hwfn)
 	/* Set the number of required CORE connections */
 	u32 core_cids = 1; /* SPQ */
 
+	if (p_hwfn->using_ll2)
+		core_cids += 4;
 	qed_cxt_set_proto_cid_count(p_hwfn, PROTOCOLID_CORE, core_cids, 0);
 
 	switch (p_hwfn->hw_info.personality) {
diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.h b/drivers/net/ethernet/qlogic/qed/qed_cxt.h
index c6f6f2e8192d..2b8bdaa77800 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_cxt.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.h
@@ -170,6 +170,13 @@ int qed_qm_reconf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
  */
 void qed_cxt_release_cid(struct qed_hwfn *p_hwfn,
 			 u32 cid);
+int qed_cxt_dynamic_ilt_alloc(struct qed_hwfn *p_hwfn,
+			      enum qed_cxt_elem_type elem_type, u32 iid);
+u32 qed_cxt_get_proto_tid_count(struct qed_hwfn *p_hwfn,
+				enum protocol_type type);
+u32 qed_cxt_get_proto_cid_start(struct qed_hwfn *p_hwfn,
+				enum protocol_type type);
+int qed_cxt_free_proto_ilt(struct qed_hwfn *p_hwfn, enum protocol_type proto);
 
 #define QED_CTX_WORKING_MEM 0
 #define QED_CTX_FL_MEM 1
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
index d0dc28f93c0e..130da1c0490b 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
@@ -19,6 +19,7 @@
 #include "qed_dcbx.h"
 #include "qed_hsi.h"
 #include "qed_sp.h"
+#include "qed_sriov.h"
 #ifdef CONFIG_DCB
 #include <linux/qed/qed_eth_if.h>
 #endif
@@ -52,40 +53,94 @@ static bool qed_dcbx_app_ethtype(u32 app_info_bitmap)
 		  DCBX_APP_SF_ETHTYPE);
 }
 
+static bool qed_dcbx_ieee_app_ethtype(u32 app_info_bitmap)
+{
+	u8 mfw_val = QED_MFW_GET_FIELD(app_info_bitmap, DCBX_APP_SF_IEEE);
+
+	/* Old MFW */
+	if (mfw_val == DCBX_APP_SF_IEEE_RESERVED)
+		return qed_dcbx_app_ethtype(app_info_bitmap);
+
+	return !!(mfw_val == DCBX_APP_SF_IEEE_ETHTYPE);
+}
+
 static bool qed_dcbx_app_port(u32 app_info_bitmap)
 {
 	return !!(QED_MFW_GET_FIELD(app_info_bitmap, DCBX_APP_SF) ==
 		  DCBX_APP_SF_PORT);
 }
 
-static bool qed_dcbx_default_tlv(u32 app_info_bitmap, u16 proto_id)
+static bool qed_dcbx_ieee_app_port(u32 app_info_bitmap, u8 type)
+{
+	u8 mfw_val = QED_MFW_GET_FIELD(app_info_bitmap, DCBX_APP_SF_IEEE);
+
+	/* Old MFW */
+	if (mfw_val == DCBX_APP_SF_IEEE_RESERVED)
+		return qed_dcbx_app_port(app_info_bitmap);
+
+	return !!(mfw_val == type || mfw_val == DCBX_APP_SF_IEEE_TCP_UDP_PORT);
+}
+
+static bool qed_dcbx_default_tlv(u32 app_info_bitmap, u16 proto_id, bool ieee)
 {
-	return !!(qed_dcbx_app_ethtype(app_info_bitmap) &&
-		  proto_id == QED_ETH_TYPE_DEFAULT);
+	bool ethtype;
+
+	if (ieee)
+		ethtype = qed_dcbx_ieee_app_ethtype(app_info_bitmap);
+	else
+		ethtype = qed_dcbx_app_ethtype(app_info_bitmap);
+
+	return !!(ethtype && (proto_id == QED_ETH_TYPE_DEFAULT));
 }
 
-static bool qed_dcbx_iscsi_tlv(u32 app_info_bitmap, u16 proto_id)
+static bool qed_dcbx_iscsi_tlv(u32 app_info_bitmap, u16 proto_id, bool ieee)
 {
-	return !!(qed_dcbx_app_port(app_info_bitmap) &&
-		  proto_id == QED_TCP_PORT_ISCSI);
+	bool port;
+
+	if (ieee)
+		port = qed_dcbx_ieee_app_port(app_info_bitmap,
+					      DCBX_APP_SF_IEEE_TCP_PORT);
+	else
+		port = qed_dcbx_app_port(app_info_bitmap);
+
+	return !!(port && (proto_id == QED_TCP_PORT_ISCSI));
 }
 
-static bool qed_dcbx_fcoe_tlv(u32 app_info_bitmap, u16 proto_id)
+static bool qed_dcbx_fcoe_tlv(u32 app_info_bitmap, u16 proto_id, bool ieee)
 {
-	return !!(qed_dcbx_app_ethtype(app_info_bitmap) &&
-		  proto_id == QED_ETH_TYPE_FCOE);
+	bool ethtype;
+
+	if (ieee)
+		ethtype = qed_dcbx_ieee_app_ethtype(app_info_bitmap);
+	else
+		ethtype = qed_dcbx_app_ethtype(app_info_bitmap);
+
+	return !!(ethtype && (proto_id == QED_ETH_TYPE_FCOE));
 }
 
-static bool qed_dcbx_roce_tlv(u32 app_info_bitmap, u16 proto_id)
+static bool qed_dcbx_roce_tlv(u32 app_info_bitmap, u16 proto_id, bool ieee)
 {
-	return !!(qed_dcbx_app_ethtype(app_info_bitmap) &&
-		  proto_id == QED_ETH_TYPE_ROCE);
+	bool ethtype;
+
+	if (ieee)
+		ethtype = qed_dcbx_ieee_app_ethtype(app_info_bitmap);
+	else
+		ethtype = qed_dcbx_app_ethtype(app_info_bitmap);
+
+	return !!(ethtype && (proto_id == QED_ETH_TYPE_ROCE));
 }
 
-static bool qed_dcbx_roce_v2_tlv(u32 app_info_bitmap, u16 proto_id)
+static bool qed_dcbx_roce_v2_tlv(u32 app_info_bitmap, u16 proto_id, bool ieee)
 {
-	return !!(qed_dcbx_app_port(app_info_bitmap) &&
-		  proto_id == QED_UDP_PORT_TYPE_ROCE_V2);
+	bool port;
+
+	if (ieee)
+		port = qed_dcbx_ieee_app_port(app_info_bitmap,
+					      DCBX_APP_SF_IEEE_UDP_PORT);
+	else
+		port = qed_dcbx_app_port(app_info_bitmap);
+
+	return !!(port && (proto_id == QED_UDP_PORT_TYPE_ROCE_V2));
 }
 
 static void
@@ -164,17 +219,17 @@ qed_dcbx_update_app_info(struct qed_dcbx_results *p_data,
 static bool
 qed_dcbx_get_app_protocol_type(struct qed_hwfn *p_hwfn,
 			       u32 app_prio_bitmap,
-			       u16 id, enum dcbx_protocol_type *type)
+			       u16 id, enum dcbx_protocol_type *type, bool ieee)
 {
-	if (qed_dcbx_fcoe_tlv(app_prio_bitmap, id)) {
+	if (qed_dcbx_fcoe_tlv(app_prio_bitmap, id, ieee)) {
 		*type = DCBX_PROTOCOL_FCOE;
-	} else if (qed_dcbx_roce_tlv(app_prio_bitmap, id)) {
+	} else if (qed_dcbx_roce_tlv(app_prio_bitmap, id, ieee)) {
 		*type = DCBX_PROTOCOL_ROCE;
-	} else if (qed_dcbx_iscsi_tlv(app_prio_bitmap, id)) {
+	} else if (qed_dcbx_iscsi_tlv(app_prio_bitmap, id, ieee)) {
 		*type = DCBX_PROTOCOL_ISCSI;
-	} else if (qed_dcbx_default_tlv(app_prio_bitmap, id)) {
+	} else if (qed_dcbx_default_tlv(app_prio_bitmap, id, ieee)) {
 		*type = DCBX_PROTOCOL_ETH;
-	} else if (qed_dcbx_roce_v2_tlv(app_prio_bitmap, id)) {
+	} else if (qed_dcbx_roce_v2_tlv(app_prio_bitmap, id, ieee)) {
 		*type = DCBX_PROTOCOL_ROCE_V2;
 	} else {
 		*type = DCBX_MAX_PROTOCOL_TYPE;
@@ -194,17 +249,18 @@ static int
 qed_dcbx_process_tlv(struct qed_hwfn *p_hwfn,
 		     struct qed_dcbx_results *p_data,
 		     struct dcbx_app_priority_entry *p_tbl,
-		     u32 pri_tc_tbl, int count, bool dcbx_enabled)
+		     u32 pri_tc_tbl, int count, u8 dcbx_version)
 {
 	u8 tc, priority_map;
 	enum dcbx_protocol_type type;
+	bool enable, ieee;
 	u16 protocol_id;
 	int priority;
-	bool enable;
 	int i;
 
 	DP_VERBOSE(p_hwfn, QED_MSG_DCB, "Num APP entries = %d\n", count);
 
+	ieee = (dcbx_version == DCBX_CONFIG_VERSION_IEEE);
 	/* Parse APP TLV */
 	for (i = 0; i < count; i++) {
 		protocol_id = QED_MFW_GET_FIELD(p_tbl[i].entry,
@@ -219,7 +275,7 @@ qed_dcbx_process_tlv(struct qed_hwfn *p_hwfn,
 
 		tc = QED_DCBX_PRIO2TC(pri_tc_tbl, priority);
 		if (qed_dcbx_get_app_protocol_type(p_hwfn, p_tbl[i].entry,
-						   protocol_id, &type)) {
+						   protocol_id, &type, ieee)) {
 			/* ETH always have the enable bit reset, as it gets
 			 * vlan information per packet. For other protocols,
 			 * should be set according to the dcbx_enabled
@@ -275,15 +331,12 @@ static int qed_dcbx_process_mib_info(struct qed_hwfn *p_hwfn)
 	struct dcbx_ets_feature *p_ets;
 	struct qed_hw_info *p_info;
 	u32 pri_tc_tbl, flags;
-	bool dcbx_enabled;
+	u8 dcbx_version;
 	int num_entries;
 	int rc = 0;
 
-	/* If DCBx version is non zero, then negotiation was
-	 * successfuly performed
-	 */
 	flags = p_hwfn->p_dcbx_info->operational.flags;
-	dcbx_enabled = !!QED_MFW_GET_FIELD(flags, DCBX_CONFIG_VERSION);
+	dcbx_version = QED_MFW_GET_FIELD(flags, DCBX_CONFIG_VERSION);
 
 	p_app = &p_hwfn->p_dcbx_info->operational.features.app;
 	p_tbl = p_app->app_pri_tbl;
@@ -295,13 +348,13 @@ static int qed_dcbx_process_mib_info(struct qed_hwfn *p_hwfn)
 	num_entries = QED_MFW_GET_FIELD(p_app->flags, DCBX_APP_NUM_ENTRIES);
 
 	rc = qed_dcbx_process_tlv(p_hwfn, &data, p_tbl, pri_tc_tbl,
-				  num_entries, dcbx_enabled);
+				  num_entries, dcbx_version);
 	if (rc)
 		return rc;
 
 	p_info->num_tc = QED_MFW_GET_FIELD(p_ets->flags, DCBX_ETS_MAX_TCS);
 	data.pf_id = p_hwfn->rel_pf_id;
-	data.dcbx_enabled = dcbx_enabled;
+	data.dcbx_enabled = !!dcbx_version;
 
 	qed_dcbx_dp_protocol(p_hwfn, &data);
 
@@ -400,7 +453,7 @@ static void
 qed_dcbx_get_app_data(struct qed_hwfn *p_hwfn,
 		      struct dcbx_app_priority_feature *p_app,
 		      struct dcbx_app_priority_entry *p_tbl,
-		      struct qed_dcbx_params *p_params)
+		      struct qed_dcbx_params *p_params, bool ieee)
 {
 	struct qed_app_entry *entry;
 	u8 pri_map;
@@ -414,15 +467,46 @@ qed_dcbx_get_app_data(struct qed_hwfn *p_hwfn,
 						      DCBX_APP_NUM_ENTRIES);
 	for (i = 0; i < DCBX_MAX_APP_PROTOCOL; i++) {
 		entry = &p_params->app_entry[i];
-		entry->ethtype = !(QED_MFW_GET_FIELD(p_tbl[i].entry,
-						     DCBX_APP_SF));
+		if (ieee) {
+			u8 sf_ieee;
+			u32 val;
+
+			sf_ieee = QED_MFW_GET_FIELD(p_tbl[i].entry,
+						    DCBX_APP_SF_IEEE);
+			switch (sf_ieee) {
+			case DCBX_APP_SF_IEEE_RESERVED:
+				/* Old MFW */
+				val = QED_MFW_GET_FIELD(p_tbl[i].entry,
+							DCBX_APP_SF);
+				entry->sf_ieee = val ?
+				    QED_DCBX_SF_IEEE_TCP_UDP_PORT :
+				    QED_DCBX_SF_IEEE_ETHTYPE;
+				break;
+			case DCBX_APP_SF_IEEE_ETHTYPE:
+				entry->sf_ieee = QED_DCBX_SF_IEEE_ETHTYPE;
+				break;
+			case DCBX_APP_SF_IEEE_TCP_PORT:
+				entry->sf_ieee = QED_DCBX_SF_IEEE_TCP_PORT;
+				break;
+			case DCBX_APP_SF_IEEE_UDP_PORT:
+				entry->sf_ieee = QED_DCBX_SF_IEEE_UDP_PORT;
+				break;
+			case DCBX_APP_SF_IEEE_TCP_UDP_PORT:
+				entry->sf_ieee = QED_DCBX_SF_IEEE_TCP_UDP_PORT;
+				break;
+			}
+		} else {
+			entry->ethtype = !(QED_MFW_GET_FIELD(p_tbl[i].entry,
+							     DCBX_APP_SF));
+		}
+
 		pri_map = QED_MFW_GET_FIELD(p_tbl[i].entry, DCBX_APP_PRI_MAP);
 		entry->prio = ffs(pri_map) - 1;
 		entry->proto_id = QED_MFW_GET_FIELD(p_tbl[i].entry,
 						    DCBX_APP_PROTOCOL_ID);
 		qed_dcbx_get_app_protocol_type(p_hwfn, p_tbl[i].entry,
 					       entry->proto_id,
-					       &entry->proto_type);
+					       &entry->proto_type, ieee);
 	}
 
 	DP_VERBOSE(p_hwfn, QED_MSG_DCB,
@@ -483,7 +567,7 @@ qed_dcbx_get_ets_data(struct qed_hwfn *p_hwfn,
 	bw_map[1] = be32_to_cpu(p_ets->tc_bw_tbl[1]);
 	tsa_map[0] = be32_to_cpu(p_ets->tc_tsa_tbl[0]);
 	tsa_map[1] = be32_to_cpu(p_ets->tc_tsa_tbl[1]);
-	pri_map = be32_to_cpu(p_ets->pri_tc_tbl[0]);
+	pri_map = p_ets->pri_tc_tbl[0];
 	for (i = 0; i < QED_MAX_PFC_PRIORITIES; i++) {
 		p_params->ets_tc_bw_tbl[i] = ((u8 *)bw_map)[i];
 		p_params->ets_tc_tsa_tbl[i] = ((u8 *)tsa_map)[i];
@@ -500,9 +584,9 @@ qed_dcbx_get_common_params(struct qed_hwfn *p_hwfn,
 			   struct dcbx_app_priority_feature *p_app,
 			   struct dcbx_app_priority_entry *p_tbl,
 			   struct dcbx_ets_feature *p_ets,
-			   u32 pfc, struct qed_dcbx_params *p_params)
+			   u32 pfc, struct qed_dcbx_params *p_params, bool ieee)
 {
-	qed_dcbx_get_app_data(p_hwfn, p_app, p_tbl, p_params);
+	qed_dcbx_get_app_data(p_hwfn, p_app, p_tbl, p_params, ieee);
 	qed_dcbx_get_ets_data(p_hwfn, p_ets, p_params);
 	qed_dcbx_get_pfc_data(p_hwfn, pfc, p_params);
 }
@@ -516,7 +600,7 @@ qed_dcbx_get_local_params(struct qed_hwfn *p_hwfn,
 	p_feat = &p_hwfn->p_dcbx_info->local_admin.features;
 	qed_dcbx_get_common_params(p_hwfn, &p_feat->app,
 				   p_feat->app.app_pri_tbl, &p_feat->ets,
-				   p_feat->pfc, &params->local.params);
+				   p_feat->pfc, &params->local.params, false);
 	params->local.valid = true;
 }
 
@@ -529,7 +613,7 @@ qed_dcbx_get_remote_params(struct qed_hwfn *p_hwfn,
 	p_feat = &p_hwfn->p_dcbx_info->remote.features;
 	qed_dcbx_get_common_params(p_hwfn, &p_feat->app,
 				   p_feat->app.app_pri_tbl, &p_feat->ets,
-				   p_feat->pfc, &params->remote.params);
+				   p_feat->pfc, &params->remote.params, false);
 	params->remote.valid = true;
 }
 
@@ -574,7 +658,8 @@ qed_dcbx_get_operational_params(struct qed_hwfn *p_hwfn,
 
 	qed_dcbx_get_common_params(p_hwfn, &p_feat->app,
 				   p_feat->app.app_pri_tbl, &p_feat->ets,
-				   p_feat->pfc, &params->operational.params);
+				   p_feat->pfc, &params->operational.params,
+				   p_operational->ieee);
 	qed_dcbx_get_priority_info(p_hwfn, &p_operational->app_prio, p_results);
 	err = QED_MFW_GET_FIELD(p_feat->app.flags, DCBX_APP_ERROR);
 	p_operational->err = err;
@@ -790,11 +875,8 @@ int qed_dcbx_info_alloc(struct qed_hwfn *p_hwfn)
 	int rc = 0;
 
 	p_hwfn->p_dcbx_info = kzalloc(sizeof(*p_hwfn->p_dcbx_info), GFP_KERNEL);
-	if (!p_hwfn->p_dcbx_info) {
-		DP_NOTICE(p_hwfn,
-			  "Failed to allocate 'struct qed_dcbx_info'\n");
+	if (!p_hwfn->p_dcbx_info)
 		rc = -ENOMEM;
-	}
 
 	return rc;
 }
@@ -861,6 +943,9 @@ static int qed_dcbx_query_params(struct qed_hwfn *p_hwfn,
 	struct qed_ptt *p_ptt;
 	int rc;
 
+	if (IS_VF(p_hwfn->cdev))
+		return -EINVAL;
+
 	p_ptt = qed_ptt_acquire(p_hwfn);
 	if (!p_ptt)
 		return -EBUSY;
@@ -900,6 +985,7 @@ qed_dcbx_set_pfc_data(struct qed_hwfn *p_hwfn,
 		if (p_params->pfc.prio[i])
 			pfc_map |= BIT(i);
 
+	*pfc &= ~DCBX_PFC_PRI_EN_BITMAP_MASK;
 	*pfc |= (pfc_map << DCBX_PFC_PRI_EN_BITMAP_SHIFT);
 
 	DP_VERBOSE(p_hwfn, QED_MSG_DCB, "pfc = 0x%x\n", *pfc);
@@ -944,7 +1030,6 @@ qed_dcbx_set_ets_data(struct qed_hwfn *p_hwfn,
 		val = (((u32)p_params->ets_pri_tc_tbl[i]) << ((7 - i) * 4));
 		p_ets->pri_tc_tbl[0] |= val;
 	}
-	p_ets->pri_tc_tbl[0] = cpu_to_be32(p_ets->pri_tc_tbl[0]);
 	for (i = 0; i < 2; i++) {
 		p_ets->tc_bw_tbl[i] = cpu_to_be32(p_ets->tc_bw_tbl[i]);
 		p_ets->tc_tsa_tbl[i] = cpu_to_be32(p_ets->tc_tsa_tbl[i]);
@@ -954,7 +1039,7 @@ qed_dcbx_set_ets_data(struct qed_hwfn *p_hwfn,
 static void
 qed_dcbx_set_app_data(struct qed_hwfn *p_hwfn,
 		      struct dcbx_app_priority_feature *p_app,
-		      struct qed_dcbx_params *p_params)
+		      struct qed_dcbx_params *p_params, bool ieee)
 {
 	u32 *entry;
 	int i;
@@ -975,12 +1060,45 @@ qed_dcbx_set_app_data(struct qed_hwfn *p_hwfn,
 
 	for (i = 0; i < DCBX_MAX_APP_PROTOCOL; i++) {
 		entry = &p_app->app_pri_tbl[i].entry;
-		*entry &= ~DCBX_APP_SF_MASK;
-		if (p_params->app_entry[i].ethtype)
-			*entry |= ((u32)DCBX_APP_SF_ETHTYPE <<
-				   DCBX_APP_SF_SHIFT);
-		else
-			*entry |= ((u32)DCBX_APP_SF_PORT << DCBX_APP_SF_SHIFT);
+		*entry = 0;
+		if (ieee) {
+			*entry &= ~(DCBX_APP_SF_IEEE_MASK | DCBX_APP_SF_MASK);
+			switch (p_params->app_entry[i].sf_ieee) {
+			case QED_DCBX_SF_IEEE_ETHTYPE:
+				*entry |= ((u32)DCBX_APP_SF_IEEE_ETHTYPE <<
+					   DCBX_APP_SF_IEEE_SHIFT);
+				*entry |= ((u32)DCBX_APP_SF_ETHTYPE <<
+					   DCBX_APP_SF_SHIFT);
+				break;
+			case QED_DCBX_SF_IEEE_TCP_PORT:
+				*entry |= ((u32)DCBX_APP_SF_IEEE_TCP_PORT <<
+					   DCBX_APP_SF_IEEE_SHIFT);
+				*entry |= ((u32)DCBX_APP_SF_PORT <<
+					   DCBX_APP_SF_SHIFT);
+				break;
+			case QED_DCBX_SF_IEEE_UDP_PORT:
+				*entry |= ((u32)DCBX_APP_SF_IEEE_UDP_PORT <<
+					   DCBX_APP_SF_IEEE_SHIFT);
+				*entry |= ((u32)DCBX_APP_SF_PORT <<
+					   DCBX_APP_SF_SHIFT);
+				break;
+			case QED_DCBX_SF_IEEE_TCP_UDP_PORT:
+				*entry |= ((u32)DCBX_APP_SF_IEEE_TCP_UDP_PORT <<
+					   DCBX_APP_SF_IEEE_SHIFT);
+				*entry |= ((u32)DCBX_APP_SF_PORT <<
+					   DCBX_APP_SF_SHIFT);
+				break;
+			}
+		} else {
+			*entry &= ~DCBX_APP_SF_MASK;
+			if (p_params->app_entry[i].ethtype)
+				*entry |= ((u32)DCBX_APP_SF_ETHTYPE <<
+					   DCBX_APP_SF_SHIFT);
+			else
+				*entry |= ((u32)DCBX_APP_SF_PORT <<
+					   DCBX_APP_SF_SHIFT);
+		}
+
 		*entry &= ~DCBX_APP_PROTOCOL_ID_MASK;
 		*entry |= ((u32)p_params->app_entry[i].proto_id <<
 			   DCBX_APP_PROTOCOL_ID_SHIFT);
@@ -995,15 +1113,19 @@ qed_dcbx_set_local_params(struct qed_hwfn *p_hwfn,
 			  struct dcbx_local_params *local_admin,
 			  struct qed_dcbx_set *params)
 {
+	bool ieee = false;
+
 	local_admin->flags = 0;
 	memcpy(&local_admin->features,
 	       &p_hwfn->p_dcbx_info->operational.features,
 	       sizeof(local_admin->features));
 
-	if (params->enabled)
+	if (params->enabled) {
 		local_admin->config = params->ver_num;
-	else
+		ieee = !!(params->ver_num & DCBX_CONFIG_VERSION_IEEE);
+	} else {
 		local_admin->config = DCBX_CONFIG_VERSION_DISABLED;
+	}
 
 	if (params->override_flags & QED_DCBX_OVERRIDE_PFC_CFG)
 		qed_dcbx_set_pfc_data(p_hwfn, &local_admin->features.pfc,
@@ -1015,7 +1137,7 @@ qed_dcbx_set_local_params(struct qed_hwfn *p_hwfn,
 
 	if (params->override_flags & QED_DCBX_OVERRIDE_APP_CFG)
 		qed_dcbx_set_app_data(p_hwfn, &local_admin->features.app,
-				      &params->config.params);
+				      &params->config.params, ieee);
 }
 
 int qed_dcbx_config_params(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
@@ -1064,11 +1186,9 @@ int qed_dcbx_get_config_params(struct qed_hwfn *p_hwfn,
 		return 0;
 	}
 
-	dcbx_info = kmalloc(sizeof(*dcbx_info), GFP_KERNEL);
-	if (!dcbx_info) {
-		DP_ERR(p_hwfn, "Failed to allocate struct qed_dcbx_info\n");
+	dcbx_info = kzalloc(sizeof(*dcbx_info), GFP_KERNEL);
+	if (!dcbx_info)
 		return -ENOMEM;
-	}
 
 	rc = qed_dcbx_query_params(p_hwfn, dcbx_info, QED_DCBX_OPERATIONAL_MIB);
 	if (rc) {
@@ -1101,11 +1221,9 @@ static struct qed_dcbx_get *qed_dcbnl_get_dcbx(struct qed_hwfn *hwfn,
 {
 	struct qed_dcbx_get *dcbx_info;
 
-	dcbx_info = kmalloc(sizeof(*dcbx_info), GFP_KERNEL);
-	if (!dcbx_info) {
-		DP_ERR(hwfn->cdev, "Failed to allocate memory for dcbx_info\n");
+	dcbx_info = kzalloc(sizeof(*dcbx_info), GFP_KERNEL);
+	if (!dcbx_info)
 		return NULL;
-	}
 
 	if (qed_dcbx_query_params(hwfn, dcbx_info, type)) {
 		kfree(dcbx_info);
@@ -1596,8 +1714,10 @@ static int qed_dcbnl_setapp(struct qed_dev *cdev,
 		if ((entry->ethtype == ethtype) && (entry->proto_id == idval))
 			break;
 		/* First empty slot */
-		if (!entry->proto_id)
+		if (!entry->proto_id) {
+			dcbx_set.config.params.num_app_entries++;
 			break;
+		}
 	}
 
 	if (i == QED_DCBX_MAX_APP_PROTOCOL) {
@@ -1855,6 +1975,7 @@ static int qed_dcbnl_get_ieee_pfc(struct qed_dev *cdev,
 
 	if (!dcbx_info->operational.ieee) {
 		DP_INFO(hwfn, "DCBX is not enabled/operational in IEEE mode\n");
+		kfree(dcbx_info);
 		return -EINVAL;
 	}
 
@@ -2023,17 +2144,19 @@ static int qed_dcbnl_ieee_setets(struct qed_dev *cdev, struct ieee_ets *ets)
 	return rc;
 }
 
-int qed_dcbnl_ieee_peer_getets(struct qed_dev *cdev, struct ieee_ets *ets)
+static int
+qed_dcbnl_ieee_peer_getets(struct qed_dev *cdev, struct ieee_ets *ets)
 {
 	return qed_dcbnl_get_ieee_ets(cdev, ets, true);
 }
 
-int qed_dcbnl_ieee_peer_getpfc(struct qed_dev *cdev, struct ieee_pfc *pfc)
+static int
+qed_dcbnl_ieee_peer_getpfc(struct qed_dev *cdev, struct ieee_pfc *pfc)
 {
 	return qed_dcbnl_get_ieee_pfc(cdev, pfc, true);
 }
 
-int qed_dcbnl_ieee_getapp(struct qed_dev *cdev, struct dcb_app *app)
+static int qed_dcbnl_ieee_getapp(struct qed_dev *cdev, struct dcb_app *app)
 {
 	struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
 	struct qed_dcbx_get *dcbx_info;
@@ -2077,7 +2200,7 @@ int qed_dcbnl_ieee_getapp(struct qed_dev *cdev, struct dcb_app *app)
 	return 0;
 }
 
-int qed_dcbnl_ieee_setapp(struct qed_dev *cdev, struct dcb_app *app)
+static int qed_dcbnl_ieee_setapp(struct qed_dev *cdev, struct dcb_app *app)
 {
 	struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
 	struct qed_dcbx_get *dcbx_info;
@@ -2117,8 +2240,10 @@ int qed_dcbnl_ieee_setapp(struct qed_dev *cdev, struct dcb_app *app)
 		    (entry->proto_id == app->protocol))
 			break;
 		/* First empty slot */
-		if (!entry->proto_id)
+		if (!entry->proto_id) {
+			dcbx_set.config.params.num_app_entries++;
 			break;
+		}
 	}
 
 	if (i == QED_DCBX_MAX_APP_PROTOCOL) {
diff --git a/drivers/net/ethernet/qlogic/qed/qed_debug.c b/drivers/net/ethernet/qlogic/qed/qed_debug.c
new file mode 100644
index 000000000000..88e7d5bef909
--- /dev/null
+++ b/drivers/net/ethernet/qlogic/qed/qed_debug.c
@@ -0,0 +1,6898 @@
+/* QLogic qed NIC Driver
+ * Copyright (c) 2015 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <linux/crc32.h>
+#include "qed.h"
+#include "qed_hsi.h"
+#include "qed_hw.h"
+#include "qed_mcp.h"
+#include "qed_reg_addr.h"
+
+/* Chip IDs enum */
+enum chip_ids {
+	CHIP_RESERVED,
+	CHIP_BB_B0,
+	CHIP_K2,
+	MAX_CHIP_IDS
+};
+
+/* Memory groups enum */
+enum mem_groups {
+	MEM_GROUP_PXP_MEM,
+	MEM_GROUP_DMAE_MEM,
+	MEM_GROUP_CM_MEM,
+	MEM_GROUP_QM_MEM,
+	MEM_GROUP_TM_MEM,
+	MEM_GROUP_BRB_RAM,
+	MEM_GROUP_BRB_MEM,
+	MEM_GROUP_PRS_MEM,
+	MEM_GROUP_SDM_MEM,
+	MEM_GROUP_PBUF,
+	MEM_GROUP_IOR,
+	MEM_GROUP_RAM,
+	MEM_GROUP_BTB_RAM,
+	MEM_GROUP_RDIF_CTX,
+	MEM_GROUP_TDIF_CTX,
+	MEM_GROUP_CONN_CFC_MEM,
+	MEM_GROUP_TASK_CFC_MEM,
+	MEM_GROUP_CAU_PI,
+	MEM_GROUP_CAU_MEM,
+	MEM_GROUP_PXP_ILT,
+	MEM_GROUP_MULD_MEM,
+	MEM_GROUP_BTB_MEM,
+	MEM_GROUP_IGU_MEM,
+	MEM_GROUP_IGU_MSIX,
+	MEM_GROUP_CAU_SB,
+	MEM_GROUP_BMB_RAM,
+	MEM_GROUP_BMB_MEM,
+	MEM_GROUPS_NUM
+};
+
+/* Memory groups names */
+static const char * const s_mem_group_names[] = {
+	"PXP_MEM",
+	"DMAE_MEM",
+	"CM_MEM",
+	"QM_MEM",
+	"TM_MEM",
+	"BRB_RAM",
+	"BRB_MEM",
+	"PRS_MEM",
+	"SDM_MEM",
+	"PBUF",
+	"IOR",
+	"RAM",
+	"BTB_RAM",
+	"RDIF_CTX",
+	"TDIF_CTX",
+	"CONN_CFC_MEM",
+	"TASK_CFC_MEM",
+	"CAU_PI",
+	"CAU_MEM",
+	"PXP_ILT",
+	"MULD_MEM",
+	"BTB_MEM",
+	"IGU_MEM",
+	"IGU_MSIX",
+	"CAU_SB",
+	"BMB_RAM",
+	"BMB_MEM",
+};
+
+/* Idle check conditions */
+static u32 cond4(const u32 *r, const u32 *imm)
+{
+	return ((r[0] & imm[0]) != imm[1]) && ((r[1] & imm[2]) != imm[3]);
+}
+
+static u32 cond6(const u32 *r, const u32 *imm)
+{
+	return ((r[0] >> imm[0]) & imm[1]) != imm[2];
+}
+
+static u32 cond5(const u32 *r, const u32 *imm)
+{
+	return (r[0] & imm[0]) != imm[1];
+}
+
+static u32 cond8(const u32 *r, const u32 *imm)
+{
+	return ((r[0] & imm[0]) >> imm[1]) !=
+	    (((r[0] & imm[2]) >> imm[3]) | ((r[1] & imm[4]) << imm[5]));
+}
+
+static u32 cond9(const u32 *r, const u32 *imm)
+{
+	return ((r[0] & imm[0]) >> imm[1]) != (r[0] & imm[2]);
+}
+
+static u32 cond1(const u32 *r, const u32 *imm)
+{
+	return (r[0] & ~imm[0]) != imm[1];
+}
+
+static u32 cond0(const u32 *r, const u32 *imm)
+{
+	return r[0] != imm[0];
+}
+
+static u32 cond10(const u32 *r, const u32 *imm)
+{
+	return r[0] != r[1] && r[2] == imm[0];
+}
+
+static u32 cond11(const u32 *r, const u32 *imm)
+{
+	return r[0] != r[1] && r[2] > imm[0];
+}
+
+static u32 cond3(const u32 *r, const u32 *imm)
+{
+	return r[0] != r[1];
+}
+
+static u32 cond12(const u32 *r, const u32 *imm)
+{
+	return r[0] & imm[0];
+}
+
+static u32 cond7(const u32 *r, const u32 *imm)
+{
+	return r[0] < (r[1] - imm[0]);
+}
+
+static u32 cond2(const u32 *r, const u32 *imm)
+{
+	return r[0] > imm[0];
+}
+
+/* Array of Idle Check conditions */
+static u32(*cond_arr[]) (const u32 *r, const u32 *imm) = {
+	cond0,
+	cond1,
+	cond2,
+	cond3,
+	cond4,
+	cond5,
+	cond6,
+	cond7,
+	cond8,
+	cond9,
+	cond10,
+	cond11,
+	cond12,
+};
+
+/******************************* Data Types **********************************/
+
+enum platform_ids {
+	PLATFORM_ASIC,
+	PLATFORM_RESERVED,
+	PLATFORM_RESERVED2,
+	PLATFORM_RESERVED3,
+	MAX_PLATFORM_IDS
+};
+
+struct dbg_array {
+	const u32 *ptr;
+	u32 size_in_dwords;
+};
+
+/* Chip constant definitions */
+struct chip_defs {
+	const char *name;
+	struct {
+		u8 num_ports;
+		u8 num_pfs;
+	} per_platform[MAX_PLATFORM_IDS];
+};
+
+/* Platform constant definitions */
+struct platform_defs {
+	const char *name;
+	u32 delay_factor;
+};
+
+/* Storm constant definitions */
+struct storm_defs {
+	char letter;
+	enum block_id block_id;
+	enum dbg_bus_clients dbg_client_id[MAX_CHIP_IDS];
+	bool has_vfc;
+	u32 sem_fast_mem_addr;
+	u32 sem_frame_mode_addr;
+	u32 sem_slow_enable_addr;
+	u32 sem_slow_mode_addr;
+	u32 sem_slow_mode1_conf_addr;
+	u32 sem_sync_dbg_empty_addr;
+	u32 sem_slow_dbg_empty_addr;
+	u32 cm_ctx_wr_addr;
+	u32 cm_conn_ag_ctx_lid_size; /* In quad-regs */
+	u32 cm_conn_ag_ctx_rd_addr;
+	u32 cm_conn_st_ctx_lid_size; /* In quad-regs */
+	u32 cm_conn_st_ctx_rd_addr;
+	u32 cm_task_ag_ctx_lid_size; /* In quad-regs */
+	u32 cm_task_ag_ctx_rd_addr;
+	u32 cm_task_st_ctx_lid_size; /* In quad-regs */
+	u32 cm_task_st_ctx_rd_addr;
+};
+
+/* Block constant definitions */
+struct block_defs {
+	const char *name;
+	bool has_dbg_bus[MAX_CHIP_IDS];
+	bool associated_to_storm;
+	u32 storm_id; /* Valid only if associated_to_storm is true */
+	enum dbg_bus_clients dbg_client_id[MAX_CHIP_IDS];
+	u32 dbg_select_addr;
+	u32 dbg_cycle_enable_addr;
+	u32 dbg_shift_addr;
+	u32 dbg_force_valid_addr;
+	u32 dbg_force_frame_addr;
+	bool has_reset_bit;
+	bool unreset; /* If true, the block is taken out of reset before dump */
+	enum dbg_reset_regs reset_reg;
+	u8 reset_bit_offset; /* Bit offset in reset register */
+};
+
+/* Reset register definitions */
+struct reset_reg_defs {
+	u32 addr;
+	u32 unreset_val;
+	bool exists[MAX_CHIP_IDS];
+};
+
+struct grc_param_defs {
+	u32 default_val[MAX_CHIP_IDS];
+	u32 min;
+	u32 max;
+	bool is_preset;
+	u32 exclude_all_preset_val;
+	u32 crash_preset_val;
+};
+
+struct rss_mem_defs {
+	const char *mem_name;
+	const char *type_name;
+	u32 addr; /* In 128b units */
+	u32 num_entries[MAX_CHIP_IDS];
+	u32 entry_width[MAX_CHIP_IDS]; /* In bits */
+};
+
+struct vfc_ram_defs {
+	const char *mem_name;
+	const char *type_name;
+	u32 base_row;
+	u32 num_rows;
+};
+
+struct big_ram_defs {
+	const char *instance_name;
+	enum mem_groups mem_group_id;
+	enum mem_groups ram_mem_group_id;
+	enum dbg_grc_params grc_param;
+	u32 addr_reg_addr;
+	u32 data_reg_addr;
+	u32 num_of_blocks[MAX_CHIP_IDS];
+};
+
+struct phy_defs {
+	const char *phy_name;
+	u32 base_addr;
+	u32 tbus_addr_lo_addr;
+	u32 tbus_addr_hi_addr;
+	u32 tbus_data_lo_addr;
+	u32 tbus_data_hi_addr;
+};
+
+/******************************** Constants **********************************/
+
+#define MAX_LCIDS			320
+#define MAX_LTIDS			320
+#define NUM_IOR_SETS			2
+#define IORS_PER_SET			176
+#define IOR_SET_OFFSET(set_id)		((set_id) * 256)
+#define BYTES_IN_DWORD			sizeof(u32)
+
+/* In the macros below, size and offset are specified in bits */
+#define CEIL_DWORDS(size)		DIV_ROUND_UP(size, 32)
+#define FIELD_BIT_OFFSET(type, field)	type ## _ ## field ## _ ## OFFSET
+#define FIELD_BIT_SIZE(type, field)	type ## _ ## field ## _ ## SIZE
+#define FIELD_DWORD_OFFSET(type, field) \
+	 (int)(FIELD_BIT_OFFSET(type, field) / 32)
+#define FIELD_DWORD_SHIFT(type, field)	(FIELD_BIT_OFFSET(type, field) % 32)
+#define FIELD_BIT_MASK(type, field) \
+	(((1 << FIELD_BIT_SIZE(type, field)) - 1) << \
+	 FIELD_DWORD_SHIFT(type, field))
+#define SET_VAR_FIELD(var, type, field, val) \
+	do { \
+		var[FIELD_DWORD_OFFSET(type, field)] &=	\
+		(~FIELD_BIT_MASK(type, field));	\
+		var[FIELD_DWORD_OFFSET(type, field)] |= \
+		(val) << FIELD_DWORD_SHIFT(type, field); \
+	} while (0)
+#define ARR_REG_WR(dev, ptt, addr, arr, arr_size) \
+	do { \
+		for (i = 0; i < (arr_size); i++) \
+			qed_wr(dev, ptt, addr,	(arr)[i]); \
+	} while (0)
+#define ARR_REG_RD(dev, ptt, addr, arr, arr_size) \
+	do { \
+		for (i = 0; i < (arr_size); i++) \
+			(arr)[i] = qed_rd(dev, ptt, addr); \
+	} while (0)
+
+#define DWORDS_TO_BYTES(dwords)		((dwords) * BYTES_IN_DWORD)
+#define BYTES_TO_DWORDS(bytes)		((bytes) / BYTES_IN_DWORD)
+#define RAM_LINES_TO_DWORDS(lines)	((lines) * 2)
+#define RAM_LINES_TO_BYTES(lines) \
+	DWORDS_TO_BYTES(RAM_LINES_TO_DWORDS(lines))
+#define REG_DUMP_LEN_SHIFT		24
+#define MEM_DUMP_ENTRY_SIZE_DWORDS \
+	BYTES_TO_DWORDS(sizeof(struct dbg_dump_mem))
+#define IDLE_CHK_RULE_SIZE_DWORDS \
+	BYTES_TO_DWORDS(sizeof(struct dbg_idle_chk_rule))
+#define IDLE_CHK_RESULT_HDR_DWORDS \
+	BYTES_TO_DWORDS(sizeof(struct dbg_idle_chk_result_hdr))
+#define IDLE_CHK_RESULT_REG_HDR_DWORDS \
+	BYTES_TO_DWORDS(sizeof(struct dbg_idle_chk_result_reg_hdr))
+#define IDLE_CHK_MAX_ENTRIES_SIZE	32
+
+/* The sizes and offsets below are specified in bits */
+#define VFC_CAM_CMD_STRUCT_SIZE		64
+#define VFC_CAM_CMD_ROW_OFFSET		48
+#define VFC_CAM_CMD_ROW_SIZE		9
+#define VFC_CAM_ADDR_STRUCT_SIZE	16
+#define VFC_CAM_ADDR_OP_OFFSET		0
+#define VFC_CAM_ADDR_OP_SIZE		4
+#define VFC_CAM_RESP_STRUCT_SIZE	256
+#define VFC_RAM_ADDR_STRUCT_SIZE	16
+#define VFC_RAM_ADDR_OP_OFFSET		0
+#define VFC_RAM_ADDR_OP_SIZE		2
+#define VFC_RAM_ADDR_ROW_OFFSET		2
+#define VFC_RAM_ADDR_ROW_SIZE		10
+#define VFC_RAM_RESP_STRUCT_SIZE	256
+#define VFC_CAM_CMD_DWORDS		CEIL_DWORDS(VFC_CAM_CMD_STRUCT_SIZE)
+#define VFC_CAM_ADDR_DWORDS		CEIL_DWORDS(VFC_CAM_ADDR_STRUCT_SIZE)
+#define VFC_CAM_RESP_DWORDS		CEIL_DWORDS(VFC_CAM_RESP_STRUCT_SIZE)
+#define VFC_RAM_CMD_DWORDS		VFC_CAM_CMD_DWORDS
+#define VFC_RAM_ADDR_DWORDS		CEIL_DWORDS(VFC_RAM_ADDR_STRUCT_SIZE)
+#define VFC_RAM_RESP_DWORDS		CEIL_DWORDS(VFC_RAM_RESP_STRUCT_SIZE)
+#define NUM_VFC_RAM_TYPES		4
+#define VFC_CAM_NUM_ROWS		512
+#define VFC_OPCODE_CAM_RD		14
+#define VFC_OPCODE_RAM_RD		0
+#define NUM_RSS_MEM_TYPES		5
+#define NUM_BIG_RAM_TYPES		3
+#define BIG_RAM_BLOCK_SIZE_BYTES	128
+#define BIG_RAM_BLOCK_SIZE_DWORDS \
+	BYTES_TO_DWORDS(BIG_RAM_BLOCK_SIZE_BYTES)
+#define NUM_PHY_TBUS_ADDRESSES		2048
+#define PHY_DUMP_SIZE_DWORDS		(NUM_PHY_TBUS_ADDRESSES / 2)
+#define RESET_REG_UNRESET_OFFSET	4
+#define STALL_DELAY_MS			500
+#define STATIC_DEBUG_LINE_DWORDS	9
+#define NUM_DBG_BUS_LINES		256
+#define NUM_COMMON_GLOBAL_PARAMS	8
+#define FW_IMG_MAIN			1
+#define REG_FIFO_DEPTH_ELEMENTS		32
+#define REG_FIFO_ELEMENT_DWORDS		2
+#define REG_FIFO_DEPTH_DWORDS \
+	(REG_FIFO_ELEMENT_DWORDS * REG_FIFO_DEPTH_ELEMENTS)
+#define IGU_FIFO_DEPTH_ELEMENTS		64
+#define IGU_FIFO_ELEMENT_DWORDS		4
+#define IGU_FIFO_DEPTH_DWORDS \
+	(IGU_FIFO_ELEMENT_DWORDS * IGU_FIFO_DEPTH_ELEMENTS)
+#define PROTECTION_OVERRIDE_DEPTH_ELEMENTS	20
+#define PROTECTION_OVERRIDE_ELEMENT_DWORDS	2
+#define PROTECTION_OVERRIDE_DEPTH_DWORDS \
+	(PROTECTION_OVERRIDE_DEPTH_ELEMENTS * \
+	 PROTECTION_OVERRIDE_ELEMENT_DWORDS)
+#define MCP_SPAD_TRACE_OFFSIZE_ADDR \
+	(MCP_REG_SCRATCH + \
+	 offsetof(struct static_init, sections[SPAD_SECTION_TRACE]))
+#define MCP_TRACE_META_IMAGE_SIGNATURE  0x669955aa
+#define EMPTY_FW_VERSION_STR		"???_???_???_???"
+#define EMPTY_FW_IMAGE_STR		"???????????????"
+
+/***************************** Constant Arrays *******************************/
+
+/* Debug arrays */
+static struct dbg_array s_dbg_arrays[MAX_BIN_DBG_BUFFER_TYPE] = { {0} };
+
+/* Chip constant definitions array */
+static struct chip_defs s_chip_defs[MAX_CHIP_IDS] = {
+	{ "reserved", { {0, 0}, {0, 0}, {0, 0}, {0, 0} } },
+	{ "bb_b0",
+	  { {MAX_NUM_PORTS_BB, MAX_NUM_PFS_BB}, {0, 0}, {0, 0}, {0, 0} } },
+	{ "k2", { {MAX_NUM_PORTS_K2, MAX_NUM_PFS_K2}, {0, 0}, {0, 0}, {0, 0} } }
+};
+
+/* Storm constant definitions array */
+static struct storm_defs s_storm_defs[] = {
+	/* Tstorm */
+	{'T', BLOCK_TSEM,
+	 {DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT,
+	  DBG_BUS_CLIENT_RBCT}, true,
+	 TSEM_REG_FAST_MEMORY,
+	 TSEM_REG_DBG_FRAME_MODE, TSEM_REG_SLOW_DBG_ACTIVE,
+	 TSEM_REG_SLOW_DBG_MODE, TSEM_REG_DBG_MODE1_CFG,
+	 TSEM_REG_SYNC_DBG_EMPTY, TSEM_REG_SLOW_DBG_EMPTY,
+	 TCM_REG_CTX_RBC_ACCS,
+	 4, TCM_REG_AGG_CON_CTX,
+	 16, TCM_REG_SM_CON_CTX,
+	 2, TCM_REG_AGG_TASK_CTX,
+	 4, TCM_REG_SM_TASK_CTX},
+	/* Mstorm */
+	{'M', BLOCK_MSEM,
+	 {DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT,
+	  DBG_BUS_CLIENT_RBCM}, false,
+	 MSEM_REG_FAST_MEMORY,
+	 MSEM_REG_DBG_FRAME_MODE, MSEM_REG_SLOW_DBG_ACTIVE,
+	 MSEM_REG_SLOW_DBG_MODE, MSEM_REG_DBG_MODE1_CFG,
+	 MSEM_REG_SYNC_DBG_EMPTY, MSEM_REG_SLOW_DBG_EMPTY,
+	 MCM_REG_CTX_RBC_ACCS,
+	 1, MCM_REG_AGG_CON_CTX,
+	 10, MCM_REG_SM_CON_CTX,
+	 2, MCM_REG_AGG_TASK_CTX,
+	 7, MCM_REG_SM_TASK_CTX},
+	/* Ustorm */
+	{'U', BLOCK_USEM,
+	 {DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU,
+	  DBG_BUS_CLIENT_RBCU}, false,
+	 USEM_REG_FAST_MEMORY,
+	 USEM_REG_DBG_FRAME_MODE, USEM_REG_SLOW_DBG_ACTIVE,
+	 USEM_REG_SLOW_DBG_MODE, USEM_REG_DBG_MODE1_CFG,
+	 USEM_REG_SYNC_DBG_EMPTY, USEM_REG_SLOW_DBG_EMPTY,
+	 UCM_REG_CTX_RBC_ACCS,
+	 2, UCM_REG_AGG_CON_CTX,
+	 13, UCM_REG_SM_CON_CTX,
+	 3, UCM_REG_AGG_TASK_CTX,
+	 3, UCM_REG_SM_TASK_CTX},
+	/* Xstorm */
+	{'X', BLOCK_XSEM,
+	 {DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX,
+	  DBG_BUS_CLIENT_RBCX}, false,
+	 XSEM_REG_FAST_MEMORY,
+	 XSEM_REG_DBG_FRAME_MODE, XSEM_REG_SLOW_DBG_ACTIVE,
+	 XSEM_REG_SLOW_DBG_MODE, XSEM_REG_DBG_MODE1_CFG,
+	 XSEM_REG_SYNC_DBG_EMPTY, XSEM_REG_SLOW_DBG_EMPTY,
+	 XCM_REG_CTX_RBC_ACCS,
+	 9, XCM_REG_AGG_CON_CTX,
+	 15, XCM_REG_SM_CON_CTX,
+	 0, 0,
+	 0, 0},
+	/* Ystorm */
+	{'Y', BLOCK_YSEM,
+	 {DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX,
+	  DBG_BUS_CLIENT_RBCY}, false,
+	 YSEM_REG_FAST_MEMORY,
+	 YSEM_REG_DBG_FRAME_MODE, YSEM_REG_SLOW_DBG_ACTIVE,
+	 YSEM_REG_SLOW_DBG_MODE, YSEM_REG_DBG_MODE1_CFG,
+	 YSEM_REG_SYNC_DBG_EMPTY, TSEM_REG_SLOW_DBG_EMPTY,
+	 YCM_REG_CTX_RBC_ACCS,
+	 2, YCM_REG_AGG_CON_CTX,
+	 3, YCM_REG_SM_CON_CTX,
+	 2, YCM_REG_AGG_TASK_CTX,
+	 12, YCM_REG_SM_TASK_CTX},
+	/* Pstorm */
+	{'P', BLOCK_PSEM,
+	 {DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS,
+	  DBG_BUS_CLIENT_RBCS}, true,
+	 PSEM_REG_FAST_MEMORY,
+	 PSEM_REG_DBG_FRAME_MODE, PSEM_REG_SLOW_DBG_ACTIVE,
+	 PSEM_REG_SLOW_DBG_MODE, PSEM_REG_DBG_MODE1_CFG,
+	 PSEM_REG_SYNC_DBG_EMPTY, PSEM_REG_SLOW_DBG_EMPTY,
+	 PCM_REG_CTX_RBC_ACCS,
+	 0, 0,
+	 10, PCM_REG_SM_CON_CTX,
+	 0, 0,
+	 0, 0}
+};
+
+/* Block definitions array */
+static struct block_defs block_grc_defs = {
+	"grc", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN},
+	GRC_REG_DBG_SELECT, GRC_REG_DBG_DWORD_ENABLE,
+	GRC_REG_DBG_SHIFT, GRC_REG_DBG_FORCE_VALID,
+	GRC_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISC_PL_UA, 1
+};
+
+static struct block_defs block_miscs_defs = {
+	"miscs", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_misc_defs = {
+	"misc", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_dbu_defs = {
+	"dbu", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_pglue_b_defs = {
+	"pglue_b", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCH, DBG_BUS_CLIENT_RBCH, DBG_BUS_CLIENT_RBCH},
+	PGLUE_B_REG_DBG_SELECT, PGLUE_B_REG_DBG_DWORD_ENABLE,
+	PGLUE_B_REG_DBG_SHIFT, PGLUE_B_REG_DBG_FORCE_VALID,
+	PGLUE_B_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISCS_PL_HV, 1
+};
+
+static struct block_defs block_cnig_defs = {
+	"cnig", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCW},
+	CNIG_REG_DBG_SELECT_K2, CNIG_REG_DBG_DWORD_ENABLE_K2,
+	CNIG_REG_DBG_SHIFT_K2, CNIG_REG_DBG_FORCE_VALID_K2,
+	CNIG_REG_DBG_FORCE_FRAME_K2,
+	true, false, DBG_RESET_REG_MISCS_PL_HV, 0
+};
+
+static struct block_defs block_cpmu_defs = {
+	"cpmu", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, false, DBG_RESET_REG_MISCS_PL_HV, 8
+};
+
+static struct block_defs block_ncsi_defs = {
+	"ncsi", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ},
+	NCSI_REG_DBG_SELECT, NCSI_REG_DBG_DWORD_ENABLE,
+	NCSI_REG_DBG_SHIFT, NCSI_REG_DBG_FORCE_VALID,
+	NCSI_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISCS_PL_HV, 5
+};
+
+static struct block_defs block_opte_defs = {
+	"opte", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, false, DBG_RESET_REG_MISCS_PL_HV, 4
+};
+
+static struct block_defs block_bmb_defs = {
+	"bmb", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCB},
+	BMB_REG_DBG_SELECT, BMB_REG_DBG_DWORD_ENABLE,
+	BMB_REG_DBG_SHIFT, BMB_REG_DBG_FORCE_VALID,
+	BMB_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISCS_PL_UA, 7
+};
+
+static struct block_defs block_pcie_defs = {
+	"pcie", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH},
+	PCIE_REG_DBG_COMMON_SELECT, PCIE_REG_DBG_COMMON_DWORD_ENABLE,
+	PCIE_REG_DBG_COMMON_SHIFT, PCIE_REG_DBG_COMMON_FORCE_VALID,
+	PCIE_REG_DBG_COMMON_FORCE_FRAME,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_mcp_defs = {
+	"mcp", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_mcp2_defs = {
+	"mcp2", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ},
+	MCP2_REG_DBG_SELECT, MCP2_REG_DBG_DWORD_ENABLE,
+	MCP2_REG_DBG_SHIFT, MCP2_REG_DBG_FORCE_VALID,
+	MCP2_REG_DBG_FORCE_FRAME,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_pswhst_defs = {
+	"pswhst", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	PSWHST_REG_DBG_SELECT, PSWHST_REG_DBG_DWORD_ENABLE,
+	PSWHST_REG_DBG_SHIFT, PSWHST_REG_DBG_FORCE_VALID,
+	PSWHST_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISC_PL_HV, 0
+};
+
+static struct block_defs block_pswhst2_defs = {
+	"pswhst2", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	PSWHST2_REG_DBG_SELECT, PSWHST2_REG_DBG_DWORD_ENABLE,
+	PSWHST2_REG_DBG_SHIFT, PSWHST2_REG_DBG_FORCE_VALID,
+	PSWHST2_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISC_PL_HV, 0
+};
+
+static struct block_defs block_pswrd_defs = {
+	"pswrd", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	PSWRD_REG_DBG_SELECT, PSWRD_REG_DBG_DWORD_ENABLE,
+	PSWRD_REG_DBG_SHIFT, PSWRD_REG_DBG_FORCE_VALID,
+	PSWRD_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISC_PL_HV, 2
+};
+
+static struct block_defs block_pswrd2_defs = {
+	"pswrd2", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	PSWRD2_REG_DBG_SELECT, PSWRD2_REG_DBG_DWORD_ENABLE,
+	PSWRD2_REG_DBG_SHIFT, PSWRD2_REG_DBG_FORCE_VALID,
+	PSWRD2_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISC_PL_HV, 2
+};
+
+static struct block_defs block_pswwr_defs = {
+	"pswwr", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	PSWWR_REG_DBG_SELECT, PSWWR_REG_DBG_DWORD_ENABLE,
+	PSWWR_REG_DBG_SHIFT, PSWWR_REG_DBG_FORCE_VALID,
+	PSWWR_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISC_PL_HV, 3
+};
+
+static struct block_defs block_pswwr2_defs = {
+	"pswwr2", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, false, DBG_RESET_REG_MISC_PL_HV, 3
+};
+
+static struct block_defs block_pswrq_defs = {
+	"pswrq", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	PSWRQ_REG_DBG_SELECT, PSWRQ_REG_DBG_DWORD_ENABLE,
+	PSWRQ_REG_DBG_SHIFT, PSWRQ_REG_DBG_FORCE_VALID,
+	PSWRQ_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISC_PL_HV, 1
+};
+
+static struct block_defs block_pswrq2_defs = {
+	"pswrq2", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	PSWRQ2_REG_DBG_SELECT, PSWRQ2_REG_DBG_DWORD_ENABLE,
+	PSWRQ2_REG_DBG_SHIFT, PSWRQ2_REG_DBG_FORCE_VALID,
+	PSWRQ2_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISC_PL_HV, 1
+};
+
+static struct block_defs block_pglcs_defs = {
+	"pglcs", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH},
+	PGLCS_REG_DBG_SELECT, PGLCS_REG_DBG_DWORD_ENABLE,
+	PGLCS_REG_DBG_SHIFT, PGLCS_REG_DBG_FORCE_VALID,
+	PGLCS_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISCS_PL_HV, 2
+};
+
+static struct block_defs block_ptu_defs = {
+	"ptu", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	PTU_REG_DBG_SELECT, PTU_REG_DBG_DWORD_ENABLE,
+	PTU_REG_DBG_SHIFT, PTU_REG_DBG_FORCE_VALID,
+	PTU_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 20
+};
+
+static struct block_defs block_dmae_defs = {
+	"dmae", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	DMAE_REG_DBG_SELECT, DMAE_REG_DBG_DWORD_ENABLE,
+	DMAE_REG_DBG_SHIFT, DMAE_REG_DBG_FORCE_VALID,
+	DMAE_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 28
+};
+
+static struct block_defs block_tcm_defs = {
+	"tcm", {true, true, true}, true, DBG_TSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
+	TCM_REG_DBG_SELECT, TCM_REG_DBG_DWORD_ENABLE,
+	TCM_REG_DBG_SHIFT, TCM_REG_DBG_FORCE_VALID,
+	TCM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 5
+};
+
+static struct block_defs block_mcm_defs = {
+	"mcm", {true, true, true}, true, DBG_MSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	MCM_REG_DBG_SELECT, MCM_REG_DBG_DWORD_ENABLE,
+	MCM_REG_DBG_SHIFT, MCM_REG_DBG_FORCE_VALID,
+	MCM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 3
+};
+
+static struct block_defs block_ucm_defs = {
+	"ucm", {true, true, true}, true, DBG_USTORM_ID,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	UCM_REG_DBG_SELECT, UCM_REG_DBG_DWORD_ENABLE,
+	UCM_REG_DBG_SHIFT, UCM_REG_DBG_FORCE_VALID,
+	UCM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 8
+};
+
+static struct block_defs block_xcm_defs = {
+	"xcm", {true, true, true}, true, DBG_XSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
+	XCM_REG_DBG_SELECT, XCM_REG_DBG_DWORD_ENABLE,
+	XCM_REG_DBG_SHIFT, XCM_REG_DBG_FORCE_VALID,
+	XCM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 19
+};
+
+static struct block_defs block_ycm_defs = {
+	"ycm", {true, true, true}, true, DBG_YSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY},
+	YCM_REG_DBG_SELECT, YCM_REG_DBG_DWORD_ENABLE,
+	YCM_REG_DBG_SHIFT, YCM_REG_DBG_FORCE_VALID,
+	YCM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 5
+};
+
+static struct block_defs block_pcm_defs = {
+	"pcm", {true, true, true}, true, DBG_PSTORM_ID,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	PCM_REG_DBG_SELECT, PCM_REG_DBG_DWORD_ENABLE,
+	PCM_REG_DBG_SHIFT, PCM_REG_DBG_FORCE_VALID,
+	PCM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 4
+};
+
+static struct block_defs block_qm_defs = {
+	"qm", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCQ},
+	QM_REG_DBG_SELECT, QM_REG_DBG_DWORD_ENABLE,
+	QM_REG_DBG_SHIFT, QM_REG_DBG_FORCE_VALID,
+	QM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 16
+};
+
+static struct block_defs block_tm_defs = {
+	"tm", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	TM_REG_DBG_SELECT, TM_REG_DBG_DWORD_ENABLE,
+	TM_REG_DBG_SHIFT, TM_REG_DBG_FORCE_VALID,
+	TM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 17
+};
+
+static struct block_defs block_dorq_defs = {
+	"dorq", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY},
+	DORQ_REG_DBG_SELECT, DORQ_REG_DBG_DWORD_ENABLE,
+	DORQ_REG_DBG_SHIFT, DORQ_REG_DBG_FORCE_VALID,
+	DORQ_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 18
+};
+
+static struct block_defs block_brb_defs = {
+	"brb", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR},
+	BRB_REG_DBG_SELECT, BRB_REG_DBG_DWORD_ENABLE,
+	BRB_REG_DBG_SHIFT, BRB_REG_DBG_FORCE_VALID,
+	BRB_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 0
+};
+
+static struct block_defs block_src_defs = {
+	"src", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
+	SRC_REG_DBG_SELECT, SRC_REG_DBG_DWORD_ENABLE,
+	SRC_REG_DBG_SHIFT, SRC_REG_DBG_FORCE_VALID,
+	SRC_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 2
+};
+
+static struct block_defs block_prs_defs = {
+	"prs", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR},
+	PRS_REG_DBG_SELECT, PRS_REG_DBG_DWORD_ENABLE,
+	PRS_REG_DBG_SHIFT, PRS_REG_DBG_FORCE_VALID,
+	PRS_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 1
+};
+
+static struct block_defs block_tsdm_defs = {
+	"tsdm", {true, true, true}, true, DBG_TSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
+	TSDM_REG_DBG_SELECT, TSDM_REG_DBG_DWORD_ENABLE,
+	TSDM_REG_DBG_SHIFT, TSDM_REG_DBG_FORCE_VALID,
+	TSDM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 3
+};
+
+static struct block_defs block_msdm_defs = {
+	"msdm", {true, true, true}, true, DBG_MSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	MSDM_REG_DBG_SELECT, MSDM_REG_DBG_DWORD_ENABLE,
+	MSDM_REG_DBG_SHIFT, MSDM_REG_DBG_FORCE_VALID,
+	MSDM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 6
+};
+
+static struct block_defs block_usdm_defs = {
+	"usdm", {true, true, true}, true, DBG_USTORM_ID,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	USDM_REG_DBG_SELECT, USDM_REG_DBG_DWORD_ENABLE,
+	USDM_REG_DBG_SHIFT, USDM_REG_DBG_FORCE_VALID,
+	USDM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 7
+};
+
+static struct block_defs block_xsdm_defs = {
+	"xsdm", {true, true, true}, true, DBG_XSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
+	XSDM_REG_DBG_SELECT, XSDM_REG_DBG_DWORD_ENABLE,
+	XSDM_REG_DBG_SHIFT, XSDM_REG_DBG_FORCE_VALID,
+	XSDM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 20
+};
+
+static struct block_defs block_ysdm_defs = {
+	"ysdm", {true, true, true}, true, DBG_YSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY},
+	YSDM_REG_DBG_SELECT, YSDM_REG_DBG_DWORD_ENABLE,
+	YSDM_REG_DBG_SHIFT, YSDM_REG_DBG_FORCE_VALID,
+	YSDM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 8
+};
+
+static struct block_defs block_psdm_defs = {
+	"psdm", {true, true, true}, true, DBG_PSTORM_ID,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	PSDM_REG_DBG_SELECT, PSDM_REG_DBG_DWORD_ENABLE,
+	PSDM_REG_DBG_SHIFT, PSDM_REG_DBG_FORCE_VALID,
+	PSDM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 7
+};
+
+static struct block_defs block_tsem_defs = {
+	"tsem", {true, true, true}, true, DBG_TSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
+	TSEM_REG_DBG_SELECT, TSEM_REG_DBG_DWORD_ENABLE,
+	TSEM_REG_DBG_SHIFT, TSEM_REG_DBG_FORCE_VALID,
+	TSEM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 4
+};
+
+static struct block_defs block_msem_defs = {
+	"msem", {true, true, true}, true, DBG_MSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	MSEM_REG_DBG_SELECT, MSEM_REG_DBG_DWORD_ENABLE,
+	MSEM_REG_DBG_SHIFT, MSEM_REG_DBG_FORCE_VALID,
+	MSEM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 9
+};
+
+static struct block_defs block_usem_defs = {
+	"usem", {true, true, true}, true, DBG_USTORM_ID,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	USEM_REG_DBG_SELECT, USEM_REG_DBG_DWORD_ENABLE,
+	USEM_REG_DBG_SHIFT, USEM_REG_DBG_FORCE_VALID,
+	USEM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 9
+};
+
+static struct block_defs block_xsem_defs = {
+	"xsem", {true, true, true}, true, DBG_XSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
+	XSEM_REG_DBG_SELECT, XSEM_REG_DBG_DWORD_ENABLE,
+	XSEM_REG_DBG_SHIFT, XSEM_REG_DBG_FORCE_VALID,
+	XSEM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 21
+};
+
+static struct block_defs block_ysem_defs = {
+	"ysem", {true, true, true}, true, DBG_YSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY},
+	YSEM_REG_DBG_SELECT, YSEM_REG_DBG_DWORD_ENABLE,
+	YSEM_REG_DBG_SHIFT, YSEM_REG_DBG_FORCE_VALID,
+	YSEM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 11
+};
+
+static struct block_defs block_psem_defs = {
+	"psem", {true, true, true}, true, DBG_PSTORM_ID,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	PSEM_REG_DBG_SELECT, PSEM_REG_DBG_DWORD_ENABLE,
+	PSEM_REG_DBG_SHIFT, PSEM_REG_DBG_FORCE_VALID,
+	PSEM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 10
+};
+
+static struct block_defs block_rss_defs = {
+	"rss", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
+	RSS_REG_DBG_SELECT, RSS_REG_DBG_DWORD_ENABLE,
+	RSS_REG_DBG_SHIFT, RSS_REG_DBG_FORCE_VALID,
+	RSS_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 18
+};
+
+static struct block_defs block_tmld_defs = {
+	"tmld", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	TMLD_REG_DBG_SELECT, TMLD_REG_DBG_DWORD_ENABLE,
+	TMLD_REG_DBG_SHIFT, TMLD_REG_DBG_FORCE_VALID,
+	TMLD_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 13
+};
+
+static struct block_defs block_muld_defs = {
+	"muld", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	MULD_REG_DBG_SELECT, MULD_REG_DBG_DWORD_ENABLE,
+	MULD_REG_DBG_SHIFT, MULD_REG_DBG_FORCE_VALID,
+	MULD_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 14
+};
+
+static struct block_defs block_yuld_defs = {
+	"yuld", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	YULD_REG_DBG_SELECT, YULD_REG_DBG_DWORD_ENABLE,
+	YULD_REG_DBG_SHIFT, YULD_REG_DBG_FORCE_VALID,
+	YULD_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 15
+};
+
+static struct block_defs block_xyld_defs = {
+	"xyld", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
+	XYLD_REG_DBG_SELECT, XYLD_REG_DBG_DWORD_ENABLE,
+	XYLD_REG_DBG_SHIFT, XYLD_REG_DBG_FORCE_VALID,
+	XYLD_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 12
+};
+
+static struct block_defs block_prm_defs = {
+	"prm", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	PRM_REG_DBG_SELECT, PRM_REG_DBG_DWORD_ENABLE,
+	PRM_REG_DBG_SHIFT, PRM_REG_DBG_FORCE_VALID,
+	PRM_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 21
+};
+
+static struct block_defs block_pbf_pb1_defs = {
+	"pbf_pb1", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCV},
+	PBF_PB1_REG_DBG_SELECT, PBF_PB1_REG_DBG_DWORD_ENABLE,
+	PBF_PB1_REG_DBG_SHIFT, PBF_PB1_REG_DBG_FORCE_VALID,
+	PBF_PB1_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1,
+	11
+};
+
+static struct block_defs block_pbf_pb2_defs = {
+	"pbf_pb2", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCV},
+	PBF_PB2_REG_DBG_SELECT, PBF_PB2_REG_DBG_DWORD_ENABLE,
+	PBF_PB2_REG_DBG_SHIFT, PBF_PB2_REG_DBG_FORCE_VALID,
+	PBF_PB2_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1,
+	12
+};
+
+static struct block_defs block_rpb_defs = {
+	"rpb", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	RPB_REG_DBG_SELECT, RPB_REG_DBG_DWORD_ENABLE,
+	RPB_REG_DBG_SHIFT, RPB_REG_DBG_FORCE_VALID,
+	RPB_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 13
+};
+
+static struct block_defs block_btb_defs = {
+	"btb", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCV},
+	BTB_REG_DBG_SELECT, BTB_REG_DBG_DWORD_ENABLE,
+	BTB_REG_DBG_SHIFT, BTB_REG_DBG_FORCE_VALID,
+	BTB_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 10
+};
+
+static struct block_defs block_pbf_defs = {
+	"pbf", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCV},
+	PBF_REG_DBG_SELECT, PBF_REG_DBG_DWORD_ENABLE,
+	PBF_REG_DBG_SHIFT, PBF_REG_DBG_FORCE_VALID,
+	PBF_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 15
+};
+
+static struct block_defs block_rdif_defs = {
+	"rdif", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	RDIF_REG_DBG_SELECT, RDIF_REG_DBG_DWORD_ENABLE,
+	RDIF_REG_DBG_SHIFT, RDIF_REG_DBG_FORCE_VALID,
+	RDIF_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 16
+};
+
+static struct block_defs block_tdif_defs = {
+	"tdif", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	TDIF_REG_DBG_SELECT, TDIF_REG_DBG_DWORD_ENABLE,
+	TDIF_REG_DBG_SHIFT, TDIF_REG_DBG_FORCE_VALID,
+	TDIF_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 17
+};
+
+static struct block_defs block_cdu_defs = {
+	"cdu", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
+	CDU_REG_DBG_SELECT, CDU_REG_DBG_DWORD_ENABLE,
+	CDU_REG_DBG_SHIFT, CDU_REG_DBG_FORCE_VALID,
+	CDU_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 23
+};
+
+static struct block_defs block_ccfc_defs = {
+	"ccfc", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
+	CCFC_REG_DBG_SELECT, CCFC_REG_DBG_DWORD_ENABLE,
+	CCFC_REG_DBG_SHIFT, CCFC_REG_DBG_FORCE_VALID,
+	CCFC_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 24
+};
+
+static struct block_defs block_tcfc_defs = {
+	"tcfc", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
+	TCFC_REG_DBG_SELECT, TCFC_REG_DBG_DWORD_ENABLE,
+	TCFC_REG_DBG_SHIFT, TCFC_REG_DBG_FORCE_VALID,
+	TCFC_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 25
+};
+
+static struct block_defs block_igu_defs = {
+	"igu", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	IGU_REG_DBG_SELECT, IGU_REG_DBG_DWORD_ENABLE,
+	IGU_REG_DBG_SHIFT, IGU_REG_DBG_FORCE_VALID,
+	IGU_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1, 27
+};
+
+static struct block_defs block_cau_defs = {
+	"cau", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	CAU_REG_DBG_SELECT, CAU_REG_DBG_DWORD_ENABLE,
+	CAU_REG_DBG_SHIFT, CAU_REG_DBG_FORCE_VALID,
+	CAU_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 19
+};
+
+static struct block_defs block_umac_defs = {
+	"umac", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCZ},
+	UMAC_REG_DBG_SELECT, UMAC_REG_DBG_DWORD_ENABLE,
+	UMAC_REG_DBG_SHIFT, UMAC_REG_DBG_FORCE_VALID,
+	UMAC_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISCS_PL_HV, 6
+};
+
+static struct block_defs block_xmac_defs = {
+	"xmac", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_dbg_defs = {
+	"dbg", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VAUX, 3
+};
+
+static struct block_defs block_nig_defs = {
+	"nig", {true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN},
+	NIG_REG_DBG_SELECT, NIG_REG_DBG_DWORD_ENABLE,
+	NIG_REG_DBG_SHIFT, NIG_REG_DBG_FORCE_VALID,
+	NIG_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VAUX, 0
+};
+
+static struct block_defs block_wol_defs = {
+	"wol", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCZ},
+	WOL_REG_DBG_SELECT, WOL_REG_DBG_DWORD_ENABLE,
+	WOL_REG_DBG_SHIFT, WOL_REG_DBG_FORCE_VALID,
+	WOL_REG_DBG_FORCE_FRAME,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VAUX, 7
+};
+
+static struct block_defs block_bmbn_defs = {
+	"bmbn", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCB},
+	BMBN_REG_DBG_SELECT, BMBN_REG_DBG_DWORD_ENABLE,
+	BMBN_REG_DBG_SHIFT, BMBN_REG_DBG_FORCE_VALID,
+	BMBN_REG_DBG_FORCE_FRAME,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_ipc_defs = {
+	"ipc", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, false, DBG_RESET_REG_MISCS_PL_UA, 8
+};
+
+static struct block_defs block_nwm_defs = {
+	"nwm", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCW},
+	NWM_REG_DBG_SELECT, NWM_REG_DBG_DWORD_ENABLE,
+	NWM_REG_DBG_SHIFT, NWM_REG_DBG_FORCE_VALID,
+	NWM_REG_DBG_FORCE_FRAME,
+	true, false, DBG_RESET_REG_MISCS_PL_HV_2, 0
+};
+
+static struct block_defs block_nws_defs = {
+	"nws", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, false, DBG_RESET_REG_MISCS_PL_HV, 12
+};
+
+static struct block_defs block_ms_defs = {
+	"ms", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, false, DBG_RESET_REG_MISCS_PL_HV, 13
+};
+
+static struct block_defs block_phy_pcie_defs = {
+	"phy_pcie", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH},
+	PCIE_REG_DBG_COMMON_SELECT, PCIE_REG_DBG_COMMON_DWORD_ENABLE,
+	PCIE_REG_DBG_COMMON_SHIFT, PCIE_REG_DBG_COMMON_FORCE_VALID,
+	PCIE_REG_DBG_COMMON_FORCE_FRAME,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_led_defs = {
+	"led", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, true, DBG_RESET_REG_MISCS_PL_HV, 14
+};
+
+static struct block_defs block_misc_aeu_defs = {
+	"misc_aeu", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs block_bar0_map_defs = {
+	"bar0_map", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	false, false, MAX_DBG_RESET_REGS, 0
+};
+
+static struct block_defs *s_block_defs[MAX_BLOCK_ID] = {
+	&block_grc_defs,
+	&block_miscs_defs,
+	&block_misc_defs,
+	&block_dbu_defs,
+	&block_pglue_b_defs,
+	&block_cnig_defs,
+	&block_cpmu_defs,
+	&block_ncsi_defs,
+	&block_opte_defs,
+	&block_bmb_defs,
+	&block_pcie_defs,
+	&block_mcp_defs,
+	&block_mcp2_defs,
+	&block_pswhst_defs,
+	&block_pswhst2_defs,
+	&block_pswrd_defs,
+	&block_pswrd2_defs,
+	&block_pswwr_defs,
+	&block_pswwr2_defs,
+	&block_pswrq_defs,
+	&block_pswrq2_defs,
+	&block_pglcs_defs,
+	&block_dmae_defs,
+	&block_ptu_defs,
+	&block_tcm_defs,
+	&block_mcm_defs,
+	&block_ucm_defs,
+	&block_xcm_defs,
+	&block_ycm_defs,
+	&block_pcm_defs,
+	&block_qm_defs,
+	&block_tm_defs,
+	&block_dorq_defs,
+	&block_brb_defs,
+	&block_src_defs,
+	&block_prs_defs,
+	&block_tsdm_defs,
+	&block_msdm_defs,
+	&block_usdm_defs,
+	&block_xsdm_defs,
+	&block_ysdm_defs,
+	&block_psdm_defs,
+	&block_tsem_defs,
+	&block_msem_defs,
+	&block_usem_defs,
+	&block_xsem_defs,
+	&block_ysem_defs,
+	&block_psem_defs,
+	&block_rss_defs,
+	&block_tmld_defs,
+	&block_muld_defs,
+	&block_yuld_defs,
+	&block_xyld_defs,
+	&block_prm_defs,
+	&block_pbf_pb1_defs,
+	&block_pbf_pb2_defs,
+	&block_rpb_defs,
+	&block_btb_defs,
+	&block_pbf_defs,
+	&block_rdif_defs,
+	&block_tdif_defs,
+	&block_cdu_defs,
+	&block_ccfc_defs,
+	&block_tcfc_defs,
+	&block_igu_defs,
+	&block_cau_defs,
+	&block_umac_defs,
+	&block_xmac_defs,
+	&block_dbg_defs,
+	&block_nig_defs,
+	&block_wol_defs,
+	&block_bmbn_defs,
+	&block_ipc_defs,
+	&block_nwm_defs,
+	&block_nws_defs,
+	&block_ms_defs,
+	&block_phy_pcie_defs,
+	&block_led_defs,
+	&block_misc_aeu_defs,
+	&block_bar0_map_defs,
+};
+
+static struct platform_defs s_platform_defs[] = {
+	{"asic", 1},
+	{"reserved", 0},
+	{"reserved2", 0},
+	{"reserved3", 0}
+};
+
+static struct grc_param_defs s_grc_param_defs[] = {
+	{{1, 1, 1}, 0, 1, false, 1, 1},	/* DBG_GRC_PARAM_DUMP_TSTORM */
+	{{1, 1, 1}, 0, 1, false, 1, 1},	/* DBG_GRC_PARAM_DUMP_MSTORM */
+	{{1, 1, 1}, 0, 1, false, 1, 1},	/* DBG_GRC_PARAM_DUMP_USTORM */
+	{{1, 1, 1}, 0, 1, false, 1, 1},	/* DBG_GRC_PARAM_DUMP_XSTORM */
+	{{1, 1, 1}, 0, 1, false, 1, 1},	/* DBG_GRC_PARAM_DUMP_YSTORM */
+	{{1, 1, 1}, 0, 1, false, 1, 1},	/* DBG_GRC_PARAM_DUMP_PSTORM */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_REGS */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_RAM */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_PBUF */
+	{{0, 0, 0}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_IOR */
+	{{0, 0, 0}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_VFC */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_CM_CTX */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_ILT */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_RSS */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_CAU */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_QM */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_MCP */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_RESERVED */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_CFC */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_IGU */
+	{{0, 0, 0}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_BRB */
+	{{0, 0, 0}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_BTB */
+	{{0, 0, 0}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_BMB */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_NIG */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_MULD */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_PRS */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_DMAE */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_TM */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_SDM */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_DIF */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_STATIC */
+	{{0, 0, 0}, 0, 1, false, 0, 0},	/* DBG_GRC_PARAM_UNSTALL */
+	{{MAX_LCIDS, MAX_LCIDS, MAX_LCIDS}, 1, MAX_LCIDS, false, MAX_LCIDS,
+	 MAX_LCIDS},			/* DBG_GRC_PARAM_NUM_LCIDS */
+	{{MAX_LTIDS, MAX_LTIDS, MAX_LTIDS}, 1, MAX_LTIDS, false, MAX_LTIDS,
+	 MAX_LTIDS},			/* DBG_GRC_PARAM_NUM_LTIDS */
+	{{0, 0, 0}, 0, 1, true, 0, 0},	/* DBG_GRC_PARAM_EXCLUDE_ALL */
+	{{0, 0, 0}, 0, 1, true, 0, 0},	/* DBG_GRC_PARAM_CRASH */
+	{{0, 0, 0}, 0, 1, false, 1, 0},	/* DBG_GRC_PARAM_PARITY_SAFE */
+	{{1, 1, 1}, 0, 1, false, 0, 1},	/* DBG_GRC_PARAM_DUMP_CM */
+	{{1, 1, 1}, 0, 1, false, 0, 1}	/* DBG_GRC_PARAM_DUMP_PHY */
+};
+
+static struct rss_mem_defs s_rss_mem_defs[] = {
+	{ "rss_mem_cid", "rss_cid", 0,
+	  {256, 256, 320},
+	  {32, 32, 32} },
+	{ "rss_mem_key_msb", "rss_key", 1024,
+	  {128, 128, 208},
+	  {256, 256, 256} },
+	{ "rss_mem_key_lsb", "rss_key", 2048,
+	  {128, 128, 208},
+	  {64, 64, 64} },
+	{ "rss_mem_info", "rss_info", 3072,
+	  {128, 128, 208},
+	  {16, 16, 16} },
+	{ "rss_mem_ind", "rss_ind", 4096,
+	  {(128 * 128), (128 * 128), (128 * 208)},
+	  {16, 16, 16} }
+};
+
+static struct vfc_ram_defs s_vfc_ram_defs[] = {
+	{"vfc_ram_tt1", "vfc_ram", 0, 512},
+	{"vfc_ram_mtt2", "vfc_ram", 512, 128},
+	{"vfc_ram_stt2", "vfc_ram", 640, 32},
+	{"vfc_ram_ro_vect", "vfc_ram", 672, 32}
+};
+
+static struct big_ram_defs s_big_ram_defs[] = {
+	{ "BRB", MEM_GROUP_BRB_MEM, MEM_GROUP_BRB_RAM, DBG_GRC_PARAM_DUMP_BRB,
+	  BRB_REG_BIG_RAM_ADDRESS, BRB_REG_BIG_RAM_DATA,
+	  {4800, 4800, 5632} },
+	{ "BTB", MEM_GROUP_BTB_MEM, MEM_GROUP_BTB_RAM, DBG_GRC_PARAM_DUMP_BTB,
+	  BTB_REG_BIG_RAM_ADDRESS, BTB_REG_BIG_RAM_DATA,
+	  {2880, 2880, 3680} },
+	{ "BMB", MEM_GROUP_BMB_MEM, MEM_GROUP_BMB_RAM, DBG_GRC_PARAM_DUMP_BMB,
+	  BMB_REG_BIG_RAM_ADDRESS, BMB_REG_BIG_RAM_DATA,
+	  {1152, 1152, 1152} }
+};
+
+static struct reset_reg_defs s_reset_regs_defs[] = {
+	{ MISCS_REG_RESET_PL_UA, 0x0,
+	  {true, true, true} },		/* DBG_RESET_REG_MISCS_PL_UA */
+	{ MISCS_REG_RESET_PL_HV, 0x0,
+	  {true, true, true} },		/* DBG_RESET_REG_MISCS_PL_HV */
+	{ MISCS_REG_RESET_PL_HV_2, 0x0,
+	  {false, false, true} },	/* DBG_RESET_REG_MISCS_PL_HV_2 */
+	{ MISC_REG_RESET_PL_UA, 0x0,
+	  {true, true, true} },		/* DBG_RESET_REG_MISC_PL_UA */
+	{ MISC_REG_RESET_PL_HV, 0x0,
+	  {true, true, true} },		/* DBG_RESET_REG_MISC_PL_HV */
+	{ MISC_REG_RESET_PL_PDA_VMAIN_1, 0x4404040,
+	  {true, true, true} },		/* DBG_RESET_REG_MISC_PL_PDA_VMAIN_1 */
+	{ MISC_REG_RESET_PL_PDA_VMAIN_2, 0x7c00007,
+	  {true, true, true} },		/* DBG_RESET_REG_MISC_PL_PDA_VMAIN_2 */
+	{ MISC_REG_RESET_PL_PDA_VAUX, 0x2,
+	  {true, true, true} },		/* DBG_RESET_REG_MISC_PL_PDA_VAUX */
+};
+
+static struct phy_defs s_phy_defs[] = {
+	{"nw_phy", NWS_REG_NWS_CMU, PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_7_0,
+	 PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_15_8,
+	 PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_7_0,
+	 PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_11_8},
+	{"sgmii_phy", MS_REG_MS_CMU, PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X132,
+	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X133,
+	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X130,
+	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X131},
+	{"pcie_phy0", PHY_PCIE_REG_PHY0, PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X132,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X133,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X130,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X131},
+	{"pcie_phy1", PHY_PCIE_REG_PHY1, PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X132,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X133,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X130,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X131},
+};
+
+/**************************** Private Functions ******************************/
+
+/* Reads and returns a single dword from the specified unaligned buffer */
+static u32 qed_read_unaligned_dword(u8 *buf)
+{
+	u32 dword;
+
+	memcpy((u8 *)&dword, buf, sizeof(dword));
+	return dword;
+}
+
+/* Initializes debug data for the specified device */
+static enum dbg_status qed_dbg_dev_init(struct qed_hwfn *p_hwfn,
+					struct qed_ptt *p_ptt)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+
+	if (dev_data->initialized)
+		return DBG_STATUS_OK;
+
+	if (QED_IS_K2(p_hwfn->cdev)) {
+		dev_data->chip_id = CHIP_K2;
+		dev_data->mode_enable[MODE_K2] = 1;
+	} else if (QED_IS_BB_B0(p_hwfn->cdev)) {
+		dev_data->chip_id = CHIP_BB_B0;
+		dev_data->mode_enable[MODE_BB_B0] = 1;
+	} else {
+		return DBG_STATUS_UNKNOWN_CHIP;
+	}
+
+	dev_data->platform_id = PLATFORM_ASIC;
+	dev_data->mode_enable[MODE_ASIC] = 1;
+	dev_data->initialized = true;
+	return DBG_STATUS_OK;
+}
+
+/* Reads the FW info structure for the specified Storm from the chip,
+ * and writes it to the specified fw_info pointer.
+ */
+static void qed_read_fw_info(struct qed_hwfn *p_hwfn,
+			     struct qed_ptt *p_ptt,
+			     u8 storm_id, struct fw_info *fw_info)
+{
+	/* Read first the address that points to fw_info location.
+	 * The address is located in the last line of the Storm RAM.
+	 */
+	u32 addr = s_storm_defs[storm_id].sem_fast_mem_addr +
+		   SEM_FAST_REG_INT_RAM +
+		   DWORDS_TO_BYTES(SEM_FAST_REG_INT_RAM_SIZE) -
+		   sizeof(struct fw_info_location);
+	struct fw_info_location fw_info_location;
+	u32 *dest = (u32 *)&fw_info_location;
+	u32 i;
+
+	memset(&fw_info_location, 0, sizeof(fw_info_location));
+	memset(fw_info, 0, sizeof(*fw_info));
+	for (i = 0; i < BYTES_TO_DWORDS(sizeof(fw_info_location));
+	     i++, addr += BYTES_IN_DWORD)
+		dest[i] = qed_rd(p_hwfn, p_ptt, addr);
+	if (fw_info_location.size > 0 && fw_info_location.size <=
+	    sizeof(*fw_info)) {
+		/* Read FW version info from Storm RAM */
+		addr = fw_info_location.grc_addr;
+		dest = (u32 *)fw_info;
+		for (i = 0; i < BYTES_TO_DWORDS(fw_info_location.size);
+		     i++, addr += BYTES_IN_DWORD)
+			dest[i] = qed_rd(p_hwfn, p_ptt, addr);
+	}
+}
+
+/* Dumps the specified string to the specified buffer. Returns the dumped size
+ * in bytes (actual length + 1 for the null character termination).
+ */
+static u32 qed_dump_str(char *dump_buf, bool dump, const char *str)
+{
+	if (dump)
+		strcpy(dump_buf, str);
+	return (u32)strlen(str) + 1;
+}
+
+/* Dumps zeros to align the specified buffer to dwords. Returns the dumped size
+ * in bytes.
+ */
+static u32 qed_dump_align(char *dump_buf, bool dump, u32 byte_offset)
+{
+	u8 offset_in_dword = (u8)(byte_offset & 0x3), align_size;
+
+	align_size = offset_in_dword ? BYTES_IN_DWORD - offset_in_dword : 0;
+
+	if (dump && align_size)
+		memset(dump_buf, 0, align_size);
+	return align_size;
+}
+
+/* Writes the specified string param to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_dump_str_param(u32 *dump_buf,
+			      bool dump,
+			      const char *param_name, const char *param_val)
+{
+	char *char_buf = (char *)dump_buf;
+	u32 offset = 0;
+
+	/* Dump param name */
+	offset += qed_dump_str(char_buf + offset, dump, param_name);
+
+	/* Indicate a string param value */
+	if (dump)
+		*(char_buf + offset) = 1;
+	offset++;
+
+	/* Dump param value */
+	offset += qed_dump_str(char_buf + offset, dump, param_val);
+
+	/* Align buffer to next dword */
+	offset += qed_dump_align(char_buf + offset, dump, offset);
+	return BYTES_TO_DWORDS(offset);
+}
+
+/* Writes the specified numeric param to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_dump_num_param(u32 *dump_buf,
+			      bool dump, const char *param_name, u32 param_val)
+{
+	char *char_buf = (char *)dump_buf;
+	u32 offset = 0;
+
+	/* Dump param name */
+	offset += qed_dump_str(char_buf + offset, dump, param_name);
+
+	/* Indicate a numeric param value */
+	if (dump)
+		*(char_buf + offset) = 0;
+	offset++;
+
+	/* Align buffer to next dword */
+	offset += qed_dump_align(char_buf + offset, dump, offset);
+
+	/* Dump param value (and change offset from bytes to dwords) */
+	offset = BYTES_TO_DWORDS(offset);
+	if (dump)
+		*(dump_buf + offset) = param_val;
+	offset++;
+	return offset;
+}
+
+/* Reads the FW version and writes it as a param to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_dump_fw_ver_param(struct qed_hwfn *p_hwfn,
+				 struct qed_ptt *p_ptt,
+				 u32 *dump_buf, bool dump)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	char fw_ver_str[16] = EMPTY_FW_VERSION_STR;
+	char fw_img_str[16] = EMPTY_FW_IMAGE_STR;
+	struct fw_info fw_info = { {0}, {0} };
+	int printed_chars;
+	u32 offset = 0;
+
+	if (dump) {
+		/* Read FW image/version from PRAM in a non-reset SEMI */
+		bool found = false;
+		u8 storm_id;
+
+		for (storm_id = 0; storm_id < MAX_DBG_STORMS && !found;
+		     storm_id++) {
+			/* Read FW version/image  */
+			if (!dev_data->block_in_reset
+			    [s_storm_defs[storm_id].block_id]) {
+				/* read FW info for the current Storm */
+				qed_read_fw_info(p_hwfn,
+						 p_ptt, storm_id, &fw_info);
+
+				/* Create FW version/image strings */
+				printed_chars =
+				    snprintf(fw_ver_str,
+					     sizeof(fw_ver_str),
+					     "%d_%d_%d_%d",
+					     fw_info.ver.num.major,
+					     fw_info.ver.num.minor,
+					     fw_info.ver.num.rev,
+					     fw_info.ver.num.eng);
+				if (printed_chars < 0 || printed_chars >=
+				    sizeof(fw_ver_str))
+					DP_NOTICE(p_hwfn,
+						  "Unexpected debug error: invalid FW version string\n");
+				switch (fw_info.ver.image_id) {
+				case FW_IMG_MAIN:
+					strcpy(fw_img_str, "main");
+					break;
+				default:
+					strcpy(fw_img_str, "unknown");
+					break;
+				}
+
+				found = true;
+			}
+		}
+	}
+
+	/* Dump FW version, image and timestamp */
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "fw-version", fw_ver_str);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "fw-image", fw_img_str);
+	offset += qed_dump_num_param(dump_buf + offset,
+				     dump,
+				     "fw-timestamp", fw_info.ver.timestamp);
+	return offset;
+}
+
+/* Reads the MFW version and writes it as a param to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_dump_mfw_ver_param(struct qed_hwfn *p_hwfn,
+				  struct qed_ptt *p_ptt,
+				  u32 *dump_buf, bool dump)
+{
+	char mfw_ver_str[16] = EMPTY_FW_VERSION_STR;
+
+	if (dump) {
+		u32 global_section_offsize, global_section_addr, mfw_ver;
+		u32 public_data_addr, global_section_offsize_addr;
+		int printed_chars;
+
+		/* Find MCP public data GRC address.
+		 * Needs to be ORed with MCP_REG_SCRATCH due to a HW bug.
+		 */
+		public_data_addr = qed_rd(p_hwfn, p_ptt,
+					  MISC_REG_SHARED_MEM_ADDR) |
+					  MCP_REG_SCRATCH;
+
+		/* Find MCP public global section offset */
+		global_section_offsize_addr = public_data_addr +
+					      offsetof(struct mcp_public_data,
+						       sections) +
+					      sizeof(offsize_t) * PUBLIC_GLOBAL;
+		global_section_offsize = qed_rd(p_hwfn, p_ptt,
+						global_section_offsize_addr);
+		global_section_addr = MCP_REG_SCRATCH +
+				      (global_section_offsize &
+				       OFFSIZE_OFFSET_MASK) * 4;
+
+		/* Read MFW version from MCP public global section */
+		mfw_ver = qed_rd(p_hwfn, p_ptt,
+				 global_section_addr +
+				 offsetof(struct public_global, mfw_ver));
+
+		/* Dump MFW version param */
+		printed_chars = snprintf(mfw_ver_str, sizeof(mfw_ver_str),
+					 "%d_%d_%d_%d",
+					 (u8) (mfw_ver >> 24),
+					 (u8) (mfw_ver >> 16),
+					 (u8) (mfw_ver >> 8),
+					 (u8) mfw_ver);
+		if (printed_chars < 0 || printed_chars >= sizeof(mfw_ver_str))
+			DP_NOTICE(p_hwfn,
+				  "Unexpected debug error: invalid MFW version string\n");
+	}
+
+	return qed_dump_str_param(dump_buf, dump, "mfw-version", mfw_ver_str);
+}
+
+/* Writes a section header to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_dump_section_hdr(u32 *dump_buf,
+				bool dump, const char *name, u32 num_params)
+{
+	return qed_dump_num_param(dump_buf, dump, name, num_params);
+}
+
+/* Writes the common global params to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_dump_common_global_params(struct qed_hwfn *p_hwfn,
+					 struct qed_ptt *p_ptt,
+					 u32 *dump_buf,
+					 bool dump,
+					 u8 num_specific_global_params)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 offset = 0;
+
+	/* Find platform string and dump global params section header */
+	offset += qed_dump_section_hdr(dump_buf + offset,
+				       dump,
+				       "global_params",
+				       NUM_COMMON_GLOBAL_PARAMS +
+				       num_specific_global_params);
+
+	/* Store params */
+	offset += qed_dump_fw_ver_param(p_hwfn, p_ptt, dump_buf + offset, dump);
+	offset += qed_dump_mfw_ver_param(p_hwfn,
+					 p_ptt, dump_buf + offset, dump);
+	offset += qed_dump_num_param(dump_buf + offset,
+				     dump, "tools-version", TOOLS_VERSION);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump,
+				     "chip",
+				     s_chip_defs[dev_data->chip_id].name);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump,
+				     "platform",
+				     s_platform_defs[dev_data->platform_id].
+				     name);
+	offset +=
+	    qed_dump_num_param(dump_buf + offset, dump, "pci-func",
+			       p_hwfn->abs_pf_id);
+	return offset;
+}
+
+/* Writes the last section to the specified buffer at the given offset.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_dump_last_section(u32 *dump_buf, u32 offset, bool dump)
+{
+	u32 start_offset = offset, crc = ~0;
+
+	/* Dump CRC section header */
+	offset += qed_dump_section_hdr(dump_buf + offset, dump, "last", 0);
+
+	/* Calculate CRC32 and add it to the dword following the "last" section.
+	 */
+	if (dump)
+		*(dump_buf + offset) = ~crc32(crc, (u8 *)dump_buf,
+					      DWORDS_TO_BYTES(offset));
+	offset++;
+	return offset - start_offset;
+}
+
+/* Update blocks reset state  */
+static void qed_update_blocks_reset_state(struct qed_hwfn *p_hwfn,
+					  struct qed_ptt *p_ptt)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 reg_val[MAX_DBG_RESET_REGS] = { 0 };
+	u32 i;
+
+	/* Read reset registers */
+	for (i = 0; i < MAX_DBG_RESET_REGS; i++)
+		if (s_reset_regs_defs[i].exists[dev_data->chip_id])
+			reg_val[i] = qed_rd(p_hwfn,
+					    p_ptt, s_reset_regs_defs[i].addr);
+
+	/* Check if blocks are in reset */
+	for (i = 0; i < MAX_BLOCK_ID; i++)
+		dev_data->block_in_reset[i] =
+		    s_block_defs[i]->has_reset_bit &&
+		    !(reg_val[s_block_defs[i]->reset_reg] &
+		      BIT(s_block_defs[i]->reset_bit_offset));
+}
+
+/* Enable / disable the Debug block */
+static void qed_bus_enable_dbg_block(struct qed_hwfn *p_hwfn,
+				     struct qed_ptt *p_ptt, bool enable)
+{
+	qed_wr(p_hwfn, p_ptt, DBG_REG_DBG_BLOCK_ON, enable ? 1 : 0);
+}
+
+/* Resets the Debug block */
+static void qed_bus_reset_dbg_block(struct qed_hwfn *p_hwfn,
+				    struct qed_ptt *p_ptt)
+{
+	u32 dbg_reset_reg_addr, old_reset_reg_val, new_reset_reg_val;
+
+	dbg_reset_reg_addr =
+		s_reset_regs_defs[s_block_defs[BLOCK_DBG]->reset_reg].addr;
+	old_reset_reg_val = qed_rd(p_hwfn, p_ptt, dbg_reset_reg_addr);
+	new_reset_reg_val = old_reset_reg_val &
+			    ~BIT(s_block_defs[BLOCK_DBG]->reset_bit_offset);
+
+	qed_wr(p_hwfn, p_ptt, dbg_reset_reg_addr, new_reset_reg_val);
+	qed_wr(p_hwfn, p_ptt, dbg_reset_reg_addr, old_reset_reg_val);
+}
+
+static void qed_bus_set_framing_mode(struct qed_hwfn *p_hwfn,
+				     struct qed_ptt *p_ptt,
+				     enum dbg_bus_frame_modes mode)
+{
+	qed_wr(p_hwfn, p_ptt, DBG_REG_FRAMING_MODE, (u8)mode);
+}
+
+/* Enable / disable Debug Bus clients according to the specified mask.
+ * (1 = enable, 0 = disable)
+ */
+static void qed_bus_enable_clients(struct qed_hwfn *p_hwfn,
+				   struct qed_ptt *p_ptt, u32 client_mask)
+{
+	qed_wr(p_hwfn, p_ptt, DBG_REG_CLIENT_ENABLE, client_mask);
+}
+
+static bool qed_is_mode_match(struct qed_hwfn *p_hwfn, u16 *modes_buf_offset)
+{
+	const u32 *ptr = s_dbg_arrays[BIN_BUF_DBG_MODE_TREE].ptr;
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u8 tree_val = ((u8 *)ptr)[(*modes_buf_offset)++];
+	bool arg1, arg2;
+
+	switch (tree_val) {
+	case INIT_MODE_OP_NOT:
+		return !qed_is_mode_match(p_hwfn, modes_buf_offset);
+	case INIT_MODE_OP_OR:
+	case INIT_MODE_OP_AND:
+		arg1 = qed_is_mode_match(p_hwfn, modes_buf_offset);
+		arg2 = qed_is_mode_match(p_hwfn, modes_buf_offset);
+		return (tree_val == INIT_MODE_OP_OR) ? (arg1 ||
+							arg2) : (arg1 && arg2);
+	default:
+		return dev_data->mode_enable[tree_val - MAX_INIT_MODE_OPS] > 0;
+	}
+}
+
+/* Returns the value of the specified GRC param */
+static u32 qed_grc_get_param(struct qed_hwfn *p_hwfn,
+			     enum dbg_grc_params grc_param)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+
+	return dev_data->grc.param_val[grc_param];
+}
+
+/* Clear all GRC params */
+static void qed_dbg_grc_clear_params(struct qed_hwfn *p_hwfn)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 i;
+
+	for (i = 0; i < MAX_DBG_GRC_PARAMS; i++)
+		dev_data->grc.param_set_by_user[i] = 0;
+}
+
+/* Assign default GRC param values */
+static void qed_dbg_grc_set_params_default(struct qed_hwfn *p_hwfn)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 i;
+
+	for (i = 0; i < MAX_DBG_GRC_PARAMS; i++)
+		if (!dev_data->grc.param_set_by_user[i])
+			dev_data->grc.param_val[i] =
+			    s_grc_param_defs[i].default_val[dev_data->chip_id];
+}
+
+/* Returns true if the specified entity (indicated by GRC param) should be
+ * included in the dump, false otherwise.
+ */
+static bool qed_grc_is_included(struct qed_hwfn *p_hwfn,
+				enum dbg_grc_params grc_param)
+{
+	return qed_grc_get_param(p_hwfn, grc_param) > 0;
+}
+
+/* Returns true of the specified Storm should be included in the dump, false
+ * otherwise.
+ */
+static bool qed_grc_is_storm_included(struct qed_hwfn *p_hwfn,
+				      enum dbg_storms storm)
+{
+	return qed_grc_get_param(p_hwfn, (enum dbg_grc_params)storm) > 0;
+}
+
+/* Returns true if the specified memory should be included in the dump, false
+ * otherwise.
+ */
+static bool qed_grc_is_mem_included(struct qed_hwfn *p_hwfn,
+				    enum block_id block_id, u8 mem_group_id)
+{
+	u8 i;
+
+	/* Check Storm match */
+	if (s_block_defs[block_id]->associated_to_storm &&
+	    !qed_grc_is_storm_included(p_hwfn,
+			(enum dbg_storms)s_block_defs[block_id]->storm_id))
+		return false;
+
+	for (i = 0; i < NUM_BIG_RAM_TYPES; i++)
+		if (mem_group_id == s_big_ram_defs[i].mem_group_id ||
+		    mem_group_id == s_big_ram_defs[i].ram_mem_group_id)
+			return qed_grc_is_included(p_hwfn,
+						   s_big_ram_defs[i].grc_param);
+	if (mem_group_id == MEM_GROUP_PXP_ILT || mem_group_id ==
+	    MEM_GROUP_PXP_MEM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_PXP);
+	if (mem_group_id == MEM_GROUP_RAM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_RAM);
+	if (mem_group_id == MEM_GROUP_PBUF)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_PBUF);
+	if (mem_group_id == MEM_GROUP_CAU_MEM ||
+	    mem_group_id == MEM_GROUP_CAU_SB ||
+	    mem_group_id == MEM_GROUP_CAU_PI)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_CAU);
+	if (mem_group_id == MEM_GROUP_QM_MEM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_QM);
+	if (mem_group_id == MEM_GROUP_CONN_CFC_MEM ||
+	    mem_group_id == MEM_GROUP_TASK_CFC_MEM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_CFC);
+	if (mem_group_id == MEM_GROUP_IGU_MEM || mem_group_id ==
+	    MEM_GROUP_IGU_MSIX)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_IGU);
+	if (mem_group_id == MEM_GROUP_MULD_MEM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_MULD);
+	if (mem_group_id == MEM_GROUP_PRS_MEM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_PRS);
+	if (mem_group_id == MEM_GROUP_DMAE_MEM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_DMAE);
+	if (mem_group_id == MEM_GROUP_TM_MEM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_TM);
+	if (mem_group_id == MEM_GROUP_SDM_MEM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_SDM);
+	if (mem_group_id == MEM_GROUP_TDIF_CTX || mem_group_id ==
+	    MEM_GROUP_RDIF_CTX)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_DIF);
+	if (mem_group_id == MEM_GROUP_CM_MEM)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_CM);
+	if (mem_group_id == MEM_GROUP_IOR)
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_IOR);
+
+	return true;
+}
+
+/* Stalls all Storms */
+static void qed_grc_stall_storms(struct qed_hwfn *p_hwfn,
+				 struct qed_ptt *p_ptt, bool stall)
+{
+	u8 reg_val = stall ? 1 : 0;
+	u8 storm_id;
+
+	for (storm_id = 0; storm_id < MAX_DBG_STORMS; storm_id++) {
+		if (qed_grc_is_storm_included(p_hwfn,
+					      (enum dbg_storms)storm_id)) {
+			u32 reg_addr =
+			    s_storm_defs[storm_id].sem_fast_mem_addr +
+			    SEM_FAST_REG_STALL_0;
+
+			qed_wr(p_hwfn, p_ptt, reg_addr, reg_val);
+		}
+	}
+
+	msleep(STALL_DELAY_MS);
+}
+
+/* Takes all blocks out of reset */
+static void qed_grc_unreset_blocks(struct qed_hwfn *p_hwfn,
+				   struct qed_ptt *p_ptt)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 reg_val[MAX_DBG_RESET_REGS] = { 0 };
+	u32 i;
+
+	/* Fill reset regs values */
+	for (i = 0; i < MAX_BLOCK_ID; i++)
+		if (s_block_defs[i]->has_reset_bit && s_block_defs[i]->unreset)
+			reg_val[s_block_defs[i]->reset_reg] |=
+			    BIT(s_block_defs[i]->reset_bit_offset);
+
+	/* Write reset registers */
+	for (i = 0; i < MAX_DBG_RESET_REGS; i++) {
+		if (s_reset_regs_defs[i].exists[dev_data->chip_id]) {
+			reg_val[i] |= s_reset_regs_defs[i].unreset_val;
+			if (reg_val[i])
+				qed_wr(p_hwfn,
+				       p_ptt,
+				       s_reset_regs_defs[i].addr +
+				       RESET_REG_UNRESET_OFFSET, reg_val[i]);
+		}
+	}
+}
+
+/* Returns the attention name offsets of the specified block */
+static const struct dbg_attn_block_type_data *
+qed_get_block_attn_data(enum block_id block_id, enum dbg_attn_type attn_type)
+{
+	const struct dbg_attn_block *base_attn_block_arr =
+		(const struct dbg_attn_block *)
+		s_dbg_arrays[BIN_BUF_DBG_ATTN_BLOCKS].ptr;
+
+	return &base_attn_block_arr[block_id].per_type_data[attn_type];
+}
+
+/* Returns the attention registers of the specified block */
+static const struct dbg_attn_reg *
+qed_get_block_attn_regs(enum block_id block_id, enum dbg_attn_type attn_type,
+			u8 *num_attn_regs)
+{
+	const struct dbg_attn_block_type_data *block_type_data =
+		qed_get_block_attn_data(block_id, attn_type);
+
+	*num_attn_regs = block_type_data->num_regs;
+	return &((const struct dbg_attn_reg *)
+		 s_dbg_arrays[BIN_BUF_DBG_ATTN_REGS].ptr)[block_type_data->
+							  regs_offset];
+}
+
+/* For each block, clear the status of all parities */
+static void qed_grc_clear_all_prty(struct qed_hwfn *p_hwfn,
+				   struct qed_ptt *p_ptt)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u8 reg_idx, num_attn_regs;
+	u32 block_id;
+
+	for (block_id = 0; block_id < MAX_BLOCK_ID; block_id++) {
+		const struct dbg_attn_reg *attn_reg_arr;
+
+		if (dev_data->block_in_reset[block_id])
+			continue;
+
+		attn_reg_arr = qed_get_block_attn_regs((enum block_id)block_id,
+						       ATTN_TYPE_PARITY,
+						       &num_attn_regs);
+		for (reg_idx = 0; reg_idx < num_attn_regs; reg_idx++) {
+			const struct dbg_attn_reg *reg_data =
+				&attn_reg_arr[reg_idx];
+
+			/* Check mode */
+			bool eval_mode = GET_FIELD(reg_data->mode.data,
+						   DBG_MODE_HDR_EVAL_MODE) > 0;
+			u16 modes_buf_offset =
+				GET_FIELD(reg_data->mode.data,
+					  DBG_MODE_HDR_MODES_BUF_OFFSET);
+
+			if (!eval_mode ||
+			    qed_is_mode_match(p_hwfn, &modes_buf_offset))
+				/* Mode match - read parity status read-clear
+				 * register.
+				 */
+				qed_rd(p_hwfn, p_ptt,
+				       DWORDS_TO_BYTES(reg_data->
+						       sts_clr_address));
+		}
+	}
+}
+
+/* Dumps GRC registers section header. Returns the dumped size in dwords.
+ * The following parameters are dumped:
+ * - 'count' = num_dumped_entries
+ * - 'split' = split_type
+ * - 'id'i = split_id (dumped only if split_id >= 0)
+ * - 'param_name' = param_val (user param, dumped only if param_name != NULL and
+ *	param_val != NULL)
+ */
+static u32 qed_grc_dump_regs_hdr(u32 *dump_buf,
+				 bool dump,
+				 u32 num_reg_entries,
+				 const char *split_type,
+				 int split_id,
+				 const char *param_name, const char *param_val)
+{
+	u8 num_params = 2 + (split_id >= 0 ? 1 : 0) + (param_name ? 1 : 0);
+	u32 offset = 0;
+
+	offset += qed_dump_section_hdr(dump_buf + offset,
+				       dump, "grc_regs", num_params);
+	offset += qed_dump_num_param(dump_buf + offset,
+				     dump, "count", num_reg_entries);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "split", split_type);
+	if (split_id >= 0)
+		offset += qed_dump_num_param(dump_buf + offset,
+					     dump, "id", split_id);
+	if (param_name && param_val)
+		offset += qed_dump_str_param(dump_buf + offset,
+					     dump, param_name, param_val);
+	return offset;
+}
+
+/* Dumps GRC register/memory. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_reg_entry(struct qed_hwfn *p_hwfn,
+				  struct qed_ptt *p_ptt, u32 *dump_buf,
+				  bool dump, u32 addr, u32 len)
+{
+	u32 offset = 0, i;
+
+	if (dump) {
+		*(dump_buf + offset++) = addr | (len << REG_DUMP_LEN_SHIFT);
+		for (i = 0; i < len; i++, addr++, offset++)
+			*(dump_buf + offset) = qed_rd(p_hwfn,
+						      p_ptt,
+						      DWORDS_TO_BYTES(addr));
+	} else {
+		offset += len + 1;
+	}
+
+	return offset;
+}
+
+/* Dumps GRC registers entries. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_regs_entries(struct qed_hwfn *p_hwfn,
+				     struct qed_ptt *p_ptt,
+				     struct dbg_array input_regs_arr,
+				     u32 *dump_buf,
+				     bool dump,
+				     bool block_enable[MAX_BLOCK_ID],
+				     u32 *num_dumped_reg_entries)
+{
+	u32 i, offset = 0, input_offset = 0;
+	bool mode_match = true;
+
+	*num_dumped_reg_entries = 0;
+	while (input_offset < input_regs_arr.size_in_dwords) {
+		const struct dbg_dump_cond_hdr *cond_hdr =
+		    (const struct dbg_dump_cond_hdr *)
+		    &input_regs_arr.ptr[input_offset++];
+		bool eval_mode = GET_FIELD(cond_hdr->mode.data,
+					   DBG_MODE_HDR_EVAL_MODE) > 0;
+
+		/* Check mode/block */
+		if (eval_mode) {
+			u16 modes_buf_offset =
+				GET_FIELD(cond_hdr->mode.data,
+					  DBG_MODE_HDR_MODES_BUF_OFFSET);
+			mode_match = qed_is_mode_match(p_hwfn,
+						       &modes_buf_offset);
+		}
+
+		if (mode_match && block_enable[cond_hdr->block_id]) {
+			for (i = 0; i < cond_hdr->data_size;
+			     i++, input_offset++) {
+				const struct dbg_dump_reg *reg =
+				    (const struct dbg_dump_reg *)
+				    &input_regs_arr.ptr[input_offset];
+
+				offset +=
+					qed_grc_dump_reg_entry(p_hwfn, p_ptt,
+						    dump_buf + offset, dump,
+						    GET_FIELD(reg->data,
+							DBG_DUMP_REG_ADDRESS),
+						    GET_FIELD(reg->data,
+							DBG_DUMP_REG_LENGTH));
+				(*num_dumped_reg_entries)++;
+			}
+		} else {
+			input_offset += cond_hdr->data_size;
+		}
+	}
+
+	return offset;
+}
+
+/* Dumps GRC registers entries. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_split_data(struct qed_hwfn *p_hwfn,
+				   struct qed_ptt *p_ptt,
+				   struct dbg_array input_regs_arr,
+				   u32 *dump_buf,
+				   bool dump,
+				   bool block_enable[MAX_BLOCK_ID],
+				   const char *split_type_name,
+				   u32 split_id,
+				   const char *param_name,
+				   const char *param_val)
+{
+	u32 num_dumped_reg_entries, offset;
+
+	/* Calculate register dump header size (and skip it for now) */
+	offset = qed_grc_dump_regs_hdr(dump_buf,
+				       false,
+				       0,
+				       split_type_name,
+				       split_id, param_name, param_val);
+
+	/* Dump registers */
+	offset += qed_grc_dump_regs_entries(p_hwfn,
+					    p_ptt,
+					    input_regs_arr,
+					    dump_buf + offset,
+					    dump,
+					    block_enable,
+					    &num_dumped_reg_entries);
+
+	/* Write register dump header */
+	if (dump && num_dumped_reg_entries > 0)
+		qed_grc_dump_regs_hdr(dump_buf,
+				      dump,
+				      num_dumped_reg_entries,
+				      split_type_name,
+				      split_id, param_name, param_val);
+
+	return num_dumped_reg_entries > 0 ? offset : 0;
+}
+
+/* Dumps registers according to the input registers array.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_grc_dump_registers(struct qed_hwfn *p_hwfn,
+				  struct qed_ptt *p_ptt,
+				  u32 *dump_buf,
+				  bool dump,
+				  bool block_enable[MAX_BLOCK_ID],
+				  const char *param_name, const char *param_val)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 offset = 0, input_offset = 0;
+	u8 port_id, pf_id;
+
+	if (dump)
+		DP_VERBOSE(p_hwfn, QED_MSG_DEBUG, "Dumping registers...\n");
+	while (input_offset <
+	       s_dbg_arrays[BIN_BUF_DBG_DUMP_REG].size_in_dwords) {
+		const struct dbg_dump_split_hdr *split_hdr =
+			(const struct dbg_dump_split_hdr *)
+			&s_dbg_arrays[BIN_BUF_DBG_DUMP_REG].ptr[input_offset++];
+		u8 split_type_id = GET_FIELD(split_hdr->hdr,
+					     DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID);
+		u32 split_data_size = GET_FIELD(split_hdr->hdr,
+						DBG_DUMP_SPLIT_HDR_DATA_SIZE);
+		struct dbg_array curr_input_regs_arr = {
+			&s_dbg_arrays[BIN_BUF_DBG_DUMP_REG].ptr[input_offset],
+			split_data_size};
+
+		switch (split_type_id) {
+		case SPLIT_TYPE_NONE:
+		case SPLIT_TYPE_VF:
+			offset += qed_grc_dump_split_data(p_hwfn,
+							  p_ptt,
+							  curr_input_regs_arr,
+							  dump_buf + offset,
+							  dump,
+							  block_enable,
+							  "eng",
+							  (u32)(-1),
+							  param_name,
+							  param_val);
+			break;
+		case SPLIT_TYPE_PORT:
+			for (port_id = 0;
+			     port_id <
+			     s_chip_defs[dev_data->chip_id].
+			     per_platform[dev_data->platform_id].num_ports;
+			     port_id++) {
+				if (dump)
+					qed_port_pretend(p_hwfn, p_ptt,
+							 port_id);
+				offset +=
+				    qed_grc_dump_split_data(p_hwfn, p_ptt,
+							    curr_input_regs_arr,
+							    dump_buf + offset,
+							    dump, block_enable,
+							    "port", port_id,
+							    param_name,
+							    param_val);
+			}
+			break;
+		case SPLIT_TYPE_PF:
+		case SPLIT_TYPE_PORT_PF:
+			for (pf_id = 0;
+			     pf_id <
+			     s_chip_defs[dev_data->chip_id].
+			     per_platform[dev_data->platform_id].num_pfs;
+			     pf_id++) {
+				if (dump)
+					qed_fid_pretend(p_hwfn, p_ptt, pf_id);
+				offset += qed_grc_dump_split_data(p_hwfn,
+							p_ptt,
+							curr_input_regs_arr,
+							dump_buf + offset,
+							dump, block_enable,
+							"pf", pf_id, param_name,
+							param_val);
+			}
+			break;
+		default:
+			break;
+		}
+
+		input_offset += split_data_size;
+	}
+
+	/* Pretend to original PF */
+	if (dump)
+		qed_fid_pretend(p_hwfn, p_ptt, p_hwfn->rel_pf_id);
+	return offset;
+}
+
+/* Dump reset registers. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_reset_regs(struct qed_hwfn *p_hwfn,
+				   struct qed_ptt *p_ptt,
+				   u32 *dump_buf, bool dump)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 i, offset = 0, num_regs = 0;
+
+	/* Calculate header size */
+	offset += qed_grc_dump_regs_hdr(dump_buf,
+					false, 0, "eng", -1, NULL, NULL);
+
+	/* Write reset registers */
+	for (i = 0; i < MAX_DBG_RESET_REGS; i++) {
+		if (s_reset_regs_defs[i].exists[dev_data->chip_id]) {
+			offset += qed_grc_dump_reg_entry(p_hwfn,
+							 p_ptt,
+							 dump_buf + offset,
+							 dump,
+							 BYTES_TO_DWORDS
+							 (s_reset_regs_defs
+							  [i].addr), 1);
+			num_regs++;
+		}
+	}
+
+	/* Write header */
+	if (dump)
+		qed_grc_dump_regs_hdr(dump_buf,
+				      true, num_regs, "eng", -1, NULL, NULL);
+	return offset;
+}
+
+/* Dump registers that are modified during GRC Dump and therefore must be dumped
+ * first. Returns the dumped size in dwords.
+ */
+static u32 qed_grc_dump_modified_regs(struct qed_hwfn *p_hwfn,
+				      struct qed_ptt *p_ptt,
+				      u32 *dump_buf, bool dump)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 offset = 0, num_reg_entries = 0, block_id;
+	u8 storm_id, reg_idx, num_attn_regs;
+
+	/* Calculate header size */
+	offset += qed_grc_dump_regs_hdr(dump_buf,
+					false, 0, "eng", -1, NULL, NULL);
+
+	/* Write parity registers */
+	for (block_id = 0; block_id < MAX_BLOCK_ID; block_id++) {
+		const struct dbg_attn_reg *attn_reg_arr;
+
+		if (dev_data->block_in_reset[block_id] && dump)
+			continue;
+
+		attn_reg_arr = qed_get_block_attn_regs((enum block_id)block_id,
+						       ATTN_TYPE_PARITY,
+						       &num_attn_regs);
+		for (reg_idx = 0; reg_idx < num_attn_regs; reg_idx++) {
+			const struct dbg_attn_reg *reg_data =
+				&attn_reg_arr[reg_idx];
+			u16 modes_buf_offset;
+			bool eval_mode;
+
+			/* Check mode */
+			eval_mode = GET_FIELD(reg_data->mode.data,
+					      DBG_MODE_HDR_EVAL_MODE) > 0;
+			modes_buf_offset =
+				GET_FIELD(reg_data->mode.data,
+					  DBG_MODE_HDR_MODES_BUF_OFFSET);
+			if (!eval_mode ||
+			    qed_is_mode_match(p_hwfn, &modes_buf_offset)) {
+				/* Mode match - read and dump registers */
+				offset += qed_grc_dump_reg_entry(p_hwfn,
+							p_ptt,
+							dump_buf + offset,
+							dump,
+							reg_data->mask_address,
+							1);
+				offset += qed_grc_dump_reg_entry(p_hwfn,
+						p_ptt,
+						dump_buf + offset,
+						dump,
+						GET_FIELD(reg_data->data,
+						    DBG_ATTN_REG_STS_ADDRESS),
+						1);
+				num_reg_entries += 2;
+			}
+		}
+	}
+
+	/* Write storm stall status registers */
+	for (storm_id = 0; storm_id < MAX_DBG_STORMS; storm_id++) {
+		if (dev_data->block_in_reset[s_storm_defs[storm_id].block_id] &&
+		    dump)
+			continue;
+
+		offset += qed_grc_dump_reg_entry(p_hwfn,
+					p_ptt,
+					dump_buf + offset,
+					dump,
+					BYTES_TO_DWORDS(s_storm_defs[storm_id].
+							sem_fast_mem_addr +
+							SEM_FAST_REG_STALLED),
+					1);
+		num_reg_entries++;
+	}
+
+	/* Write header */
+	if (dump)
+		qed_grc_dump_regs_hdr(dump_buf,
+				      true,
+				      num_reg_entries, "eng", -1, NULL, NULL);
+	return offset;
+}
+
+/* Dumps a GRC memory header (section and params).
+ * The following parameters are dumped:
+ * name - name is dumped only if it's not NULL.
+ * addr - byte_addr is dumped only if name is NULL.
+ * len - dword_len is always dumped.
+ * width - bit_width is dumped if it's not zero.
+ * packed - packed=1 is dumped if it's not false.
+ * mem_group - mem_group is always dumped.
+ * is_storm - true only if the memory is related to a Storm.
+ * storm_letter - storm letter (valid only if is_storm is true).
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_grc_dump_mem_hdr(struct qed_hwfn *p_hwfn,
+				u32 *dump_buf,
+				bool dump,
+				const char *name,
+				u32 byte_addr,
+				u32 dword_len,
+				u32 bit_width,
+				bool packed,
+				const char *mem_group,
+				bool is_storm, char storm_letter)
+{
+	u8 num_params = 3;
+	u32 offset = 0;
+	char buf[64];
+
+	if (!dword_len)
+		DP_NOTICE(p_hwfn,
+			  "Unexpected GRC Dump error: dumped memory size must be non-zero\n");
+	if (bit_width)
+		num_params++;
+	if (packed)
+		num_params++;
+
+	/* Dump section header */
+	offset += qed_dump_section_hdr(dump_buf + offset,
+				       dump, "grc_mem", num_params);
+	if (name) {
+		/* Dump name */
+		if (is_storm) {
+			strcpy(buf, "?STORM_");
+			buf[0] = storm_letter;
+			strcpy(buf + strlen(buf), name);
+		} else {
+			strcpy(buf, name);
+		}
+
+		offset += qed_dump_str_param(dump_buf + offset,
+					     dump, "name", buf);
+		if (dump)
+			DP_VERBOSE(p_hwfn,
+				   QED_MSG_DEBUG,
+				   "Dumping %d registers from %s...\n",
+				   dword_len, buf);
+	} else {
+		/* Dump address */
+		offset += qed_dump_num_param(dump_buf + offset,
+					     dump, "addr", byte_addr);
+		if (dump && dword_len > 64)
+			DP_VERBOSE(p_hwfn,
+				   QED_MSG_DEBUG,
+				   "Dumping %d registers from address 0x%x...\n",
+				   dword_len, byte_addr);
+	}
+
+	/* Dump len */
+	offset += qed_dump_num_param(dump_buf + offset, dump, "len", dword_len);
+
+	/* Dump bit width */
+	if (bit_width)
+		offset += qed_dump_num_param(dump_buf + offset,
+					     dump, "width", bit_width);
+
+	/* Dump packed */
+	if (packed)
+		offset += qed_dump_num_param(dump_buf + offset,
+					     dump, "packed", 1);
+
+	/* Dump reg type */
+	if (is_storm) {
+		strcpy(buf, "?STORM_");
+		buf[0] = storm_letter;
+		strcpy(buf + strlen(buf), mem_group);
+	} else {
+		strcpy(buf, mem_group);
+	}
+
+	offset += qed_dump_str_param(dump_buf + offset, dump, "type", buf);
+	return offset;
+}
+
+/* Dumps a single GRC memory. If name is NULL, the memory is stored by address.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_grc_dump_mem(struct qed_hwfn *p_hwfn,
+			    struct qed_ptt *p_ptt,
+			    u32 *dump_buf,
+			    bool dump,
+			    const char *name,
+			    u32 byte_addr,
+			    u32 dword_len,
+			    u32 bit_width,
+			    bool packed,
+			    const char *mem_group,
+			    bool is_storm, char storm_letter)
+{
+	u32 offset = 0;
+
+	offset += qed_grc_dump_mem_hdr(p_hwfn,
+				       dump_buf + offset,
+				       dump,
+				       name,
+				       byte_addr,
+				       dword_len,
+				       bit_width,
+				       packed,
+				       mem_group, is_storm, storm_letter);
+	if (dump) {
+		u32 i;
+
+		for (i = 0; i < dword_len;
+		     i++, byte_addr += BYTES_IN_DWORD, offset++)
+			*(dump_buf + offset) = qed_rd(p_hwfn, p_ptt, byte_addr);
+	} else {
+		offset += dword_len;
+	}
+
+	return offset;
+}
+
+/* Dumps GRC memories entries. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_mem_entries(struct qed_hwfn *p_hwfn,
+				    struct qed_ptt *p_ptt,
+				    struct dbg_array input_mems_arr,
+				    u32 *dump_buf, bool dump)
+{
+	u32 i, offset = 0, input_offset = 0;
+	bool mode_match = true;
+
+	while (input_offset < input_mems_arr.size_in_dwords) {
+		const struct dbg_dump_cond_hdr *cond_hdr;
+		u32 num_entries;
+		bool eval_mode;
+
+		cond_hdr = (const struct dbg_dump_cond_hdr *)
+			   &input_mems_arr.ptr[input_offset++];
+		eval_mode = GET_FIELD(cond_hdr->mode.data,
+				      DBG_MODE_HDR_EVAL_MODE) > 0;
+
+		/* Check required mode */
+		if (eval_mode) {
+			u16 modes_buf_offset =
+				GET_FIELD(cond_hdr->mode.data,
+					  DBG_MODE_HDR_MODES_BUF_OFFSET);
+
+			mode_match = qed_is_mode_match(p_hwfn,
+						       &modes_buf_offset);
+		}
+
+		if (!mode_match) {
+			input_offset += cond_hdr->data_size;
+			continue;
+		}
+
+		num_entries = cond_hdr->data_size / MEM_DUMP_ENTRY_SIZE_DWORDS;
+		for (i = 0; i < num_entries;
+		     i++, input_offset += MEM_DUMP_ENTRY_SIZE_DWORDS) {
+			const struct dbg_dump_mem *mem =
+				(const struct dbg_dump_mem *)
+				&input_mems_arr.ptr[input_offset];
+			u8 mem_group_id;
+
+			mem_group_id = GET_FIELD(mem->dword0,
+						 DBG_DUMP_MEM_MEM_GROUP_ID);
+			if (mem_group_id >= MEM_GROUPS_NUM) {
+				DP_NOTICE(p_hwfn, "Invalid mem_group_id\n");
+				return 0;
+			}
+
+			if (qed_grc_is_mem_included(p_hwfn,
+					(enum block_id)cond_hdr->block_id,
+					mem_group_id)) {
+				u32 mem_byte_addr =
+					DWORDS_TO_BYTES(GET_FIELD(mem->dword0,
+							DBG_DUMP_MEM_ADDRESS));
+				u32 mem_len = GET_FIELD(mem->dword1,
+							DBG_DUMP_MEM_LENGTH);
+				char storm_letter = 'a';
+				bool is_storm = false;
+
+				/* Update memory length for CCFC/TCFC memories
+				 * according to number of LCIDs/LTIDs.
+				 */
+				if (mem_group_id == MEM_GROUP_CONN_CFC_MEM)
+					mem_len = qed_grc_get_param(p_hwfn,
+							DBG_GRC_PARAM_NUM_LCIDS)
+							* (mem_len / MAX_LCIDS);
+				else if (mem_group_id == MEM_GROUP_TASK_CFC_MEM)
+					mem_len = qed_grc_get_param(p_hwfn,
+							DBG_GRC_PARAM_NUM_LTIDS)
+							* (mem_len / MAX_LTIDS);
+
+				/* If memory is associated with Storm, update
+				 * Storm details.
+				 */
+				if (s_block_defs[cond_hdr->block_id]->
+							associated_to_storm) {
+					is_storm = true;
+					storm_letter =
+						s_storm_defs[s_block_defs[
+						cond_hdr->block_id]->
+						storm_id].letter;
+				}
+
+				/* Dump memory */
+				offset += qed_grc_dump_mem(p_hwfn, p_ptt,
+						dump_buf + offset, dump, NULL,
+						mem_byte_addr, mem_len, 0,
+						false,
+						s_mem_group_names[mem_group_id],
+						is_storm, storm_letter);
+				}
+			}
+	}
+
+	return offset;
+}
+
+/* Dumps GRC memories according to the input array dump_mem.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_grc_dump_memories(struct qed_hwfn *p_hwfn,
+				 struct qed_ptt *p_ptt,
+				 u32 *dump_buf, bool dump)
+{
+	u32 offset = 0, input_offset = 0;
+
+	while (input_offset <
+	       s_dbg_arrays[BIN_BUF_DBG_DUMP_MEM].size_in_dwords) {
+		const struct dbg_dump_split_hdr *split_hdr =
+			(const struct dbg_dump_split_hdr *)
+			&s_dbg_arrays[BIN_BUF_DBG_DUMP_MEM].ptr[input_offset++];
+		u8 split_type_id = GET_FIELD(split_hdr->hdr,
+					     DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID);
+		u32 split_data_size = GET_FIELD(split_hdr->hdr,
+						DBG_DUMP_SPLIT_HDR_DATA_SIZE);
+		struct dbg_array curr_input_mems_arr = {
+			&s_dbg_arrays[BIN_BUF_DBG_DUMP_MEM].ptr[input_offset],
+			split_data_size};
+
+		switch (split_type_id) {
+		case SPLIT_TYPE_NONE:
+			offset += qed_grc_dump_mem_entries(p_hwfn,
+							   p_ptt,
+							   curr_input_mems_arr,
+							   dump_buf + offset,
+							   dump);
+			break;
+		default:
+			DP_NOTICE(p_hwfn,
+				  "Dumping split memories is currently not supported\n");
+			break;
+		}
+
+		input_offset += split_data_size;
+	}
+
+	return offset;
+}
+
+/* Dumps GRC context data for the specified Storm.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_grc_dump_ctx_data(struct qed_hwfn *p_hwfn,
+				 struct qed_ptt *p_ptt,
+				 u32 *dump_buf,
+				 bool dump,
+				 const char *name,
+				 u32 num_lids,
+				 u32 lid_size,
+				 u32 rd_reg_addr,
+				 u8 storm_id)
+{
+	u32 i, lid, total_size;
+	u32 offset = 0;
+
+	if (!lid_size)
+		return 0;
+	lid_size *= BYTES_IN_DWORD;
+	total_size = num_lids * lid_size;
+	offset += qed_grc_dump_mem_hdr(p_hwfn,
+				       dump_buf + offset,
+				       dump,
+				       name,
+				       0,
+				       total_size,
+				       lid_size * 32,
+				       false,
+				       name,
+				       true, s_storm_defs[storm_id].letter);
+
+	/* Dump context data */
+	if (dump) {
+		for (lid = 0; lid < num_lids; lid++) {
+			for (i = 0; i < lid_size; i++, offset++) {
+				qed_wr(p_hwfn,
+				       p_ptt,
+				       s_storm_defs[storm_id].cm_ctx_wr_addr,
+				       BIT(9) | lid);
+				*(dump_buf + offset) = qed_rd(p_hwfn,
+							      p_ptt,
+							      rd_reg_addr);
+			}
+		}
+	} else {
+		offset += total_size;
+	}
+
+	return offset;
+}
+
+/* Dumps GRC contexts. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_ctx(struct qed_hwfn *p_hwfn,
+			    struct qed_ptt *p_ptt, u32 *dump_buf, bool dump)
+{
+	u32 offset = 0;
+	u8 storm_id;
+
+	for (storm_id = 0; storm_id < MAX_DBG_STORMS; storm_id++) {
+		if (!qed_grc_is_storm_included(p_hwfn,
+					       (enum dbg_storms)storm_id))
+			continue;
+
+		/* Dump Conn AG context size */
+		offset +=
+			qed_grc_dump_ctx_data(p_hwfn,
+					      p_ptt,
+					      dump_buf + offset,
+					      dump,
+					      "CONN_AG_CTX",
+					      qed_grc_get_param(p_hwfn,
+						    DBG_GRC_PARAM_NUM_LCIDS),
+					      s_storm_defs[storm_id].
+						    cm_conn_ag_ctx_lid_size,
+					      s_storm_defs[storm_id].
+						    cm_conn_ag_ctx_rd_addr,
+					      storm_id);
+
+		/* Dump Conn ST context size */
+		offset +=
+			qed_grc_dump_ctx_data(p_hwfn,
+					      p_ptt,
+					      dump_buf + offset,
+					      dump,
+					      "CONN_ST_CTX",
+					      qed_grc_get_param(p_hwfn,
+						    DBG_GRC_PARAM_NUM_LCIDS),
+					      s_storm_defs[storm_id].
+						    cm_conn_st_ctx_lid_size,
+					      s_storm_defs[storm_id].
+						    cm_conn_st_ctx_rd_addr,
+					      storm_id);
+
+		/* Dump Task AG context size */
+		offset +=
+			qed_grc_dump_ctx_data(p_hwfn,
+					      p_ptt,
+					      dump_buf + offset,
+					      dump,
+					      "TASK_AG_CTX",
+					      qed_grc_get_param(p_hwfn,
+						    DBG_GRC_PARAM_NUM_LTIDS),
+					      s_storm_defs[storm_id].
+						    cm_task_ag_ctx_lid_size,
+					      s_storm_defs[storm_id].
+						    cm_task_ag_ctx_rd_addr,
+					      storm_id);
+
+		/* Dump Task ST context size */
+		offset +=
+			qed_grc_dump_ctx_data(p_hwfn,
+					      p_ptt,
+					      dump_buf + offset,
+					      dump,
+					      "TASK_ST_CTX",
+					      qed_grc_get_param(p_hwfn,
+						    DBG_GRC_PARAM_NUM_LTIDS),
+					      s_storm_defs[storm_id].
+						    cm_task_st_ctx_lid_size,
+					      s_storm_defs[storm_id].
+						    cm_task_st_ctx_rd_addr,
+					      storm_id);
+	}
+
+	return offset;
+}
+
+/* Dumps GRC IORs data. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_iors(struct qed_hwfn *p_hwfn,
+			     struct qed_ptt *p_ptt, u32 *dump_buf, bool dump)
+{
+	char buf[10] = "IOR_SET_?";
+	u8 storm_id, set_id;
+	u32 offset = 0;
+
+	for (storm_id = 0; storm_id < MAX_DBG_STORMS; storm_id++) {
+		if (qed_grc_is_storm_included(p_hwfn,
+					      (enum dbg_storms)storm_id)) {
+			for (set_id = 0; set_id < NUM_IOR_SETS; set_id++) {
+				u32 addr =
+				    s_storm_defs[storm_id].sem_fast_mem_addr +
+				    SEM_FAST_REG_STORM_REG_FILE +
+				    DWORDS_TO_BYTES(IOR_SET_OFFSET(set_id));
+
+				buf[strlen(buf) - 1] = '0' + set_id;
+				offset += qed_grc_dump_mem(p_hwfn,
+							   p_ptt,
+							   dump_buf + offset,
+							   dump,
+							   buf,
+							   addr,
+							   IORS_PER_SET,
+							   32,
+							   false,
+							   "ior",
+							   true,
+							   s_storm_defs
+							   [storm_id].letter);
+			}
+		}
+	}
+
+	return offset;
+}
+
+/* Dump VFC CAM. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_vfc_cam(struct qed_hwfn *p_hwfn,
+				struct qed_ptt *p_ptt,
+				u32 *dump_buf, bool dump, u8 storm_id)
+{
+	u32 total_size = VFC_CAM_NUM_ROWS * VFC_CAM_RESP_DWORDS;
+	u32 cam_addr[VFC_CAM_ADDR_DWORDS] = { 0 };
+	u32 cam_cmd[VFC_CAM_CMD_DWORDS] = { 0 };
+	u32 offset = 0;
+	u32 row, i;
+
+	offset += qed_grc_dump_mem_hdr(p_hwfn,
+				       dump_buf + offset,
+				       dump,
+				       "vfc_cam",
+				       0,
+				       total_size,
+				       256,
+				       false,
+				       "vfc_cam",
+				       true, s_storm_defs[storm_id].letter);
+	if (dump) {
+		/* Prepare CAM address */
+		SET_VAR_FIELD(cam_addr, VFC_CAM_ADDR, OP, VFC_OPCODE_CAM_RD);
+		for (row = 0; row < VFC_CAM_NUM_ROWS;
+		     row++, offset += VFC_CAM_RESP_DWORDS) {
+			/* Write VFC CAM command */
+			SET_VAR_FIELD(cam_cmd, VFC_CAM_CMD, ROW, row);
+			ARR_REG_WR(p_hwfn,
+				   p_ptt,
+				   s_storm_defs[storm_id].sem_fast_mem_addr +
+				   SEM_FAST_REG_VFC_DATA_WR,
+				   cam_cmd, VFC_CAM_CMD_DWORDS);
+
+			/* Write VFC CAM address */
+			ARR_REG_WR(p_hwfn,
+				   p_ptt,
+				   s_storm_defs[storm_id].sem_fast_mem_addr +
+				   SEM_FAST_REG_VFC_ADDR,
+				   cam_addr, VFC_CAM_ADDR_DWORDS);
+
+			/* Read VFC CAM read response */
+			ARR_REG_RD(p_hwfn,
+				   p_ptt,
+				   s_storm_defs[storm_id].sem_fast_mem_addr +
+				   SEM_FAST_REG_VFC_DATA_RD,
+				   dump_buf + offset, VFC_CAM_RESP_DWORDS);
+		}
+	} else {
+		offset += total_size;
+	}
+
+	return offset;
+}
+
+/* Dump VFC RAM. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_vfc_ram(struct qed_hwfn *p_hwfn,
+				struct qed_ptt *p_ptt,
+				u32 *dump_buf,
+				bool dump,
+				u8 storm_id, struct vfc_ram_defs *ram_defs)
+{
+	u32 total_size = ram_defs->num_rows * VFC_RAM_RESP_DWORDS;
+	u32 ram_addr[VFC_RAM_ADDR_DWORDS] = { 0 };
+	u32 ram_cmd[VFC_RAM_CMD_DWORDS] = { 0 };
+	u32 offset = 0;
+	u32 row, i;
+
+	offset += qed_grc_dump_mem_hdr(p_hwfn,
+				       dump_buf + offset,
+				       dump,
+				       ram_defs->mem_name,
+				       0,
+				       total_size,
+				       256,
+				       false,
+				       ram_defs->type_name,
+				       true, s_storm_defs[storm_id].letter);
+
+	/* Prepare RAM address */
+	SET_VAR_FIELD(ram_addr, VFC_RAM_ADDR, OP, VFC_OPCODE_RAM_RD);
+
+	if (!dump)
+		return offset + total_size;
+
+	for (row = ram_defs->base_row;
+	     row < ram_defs->base_row + ram_defs->num_rows;
+	     row++, offset += VFC_RAM_RESP_DWORDS) {
+		/* Write VFC RAM command */
+		ARR_REG_WR(p_hwfn,
+			   p_ptt,
+			   s_storm_defs[storm_id].sem_fast_mem_addr +
+			   SEM_FAST_REG_VFC_DATA_WR,
+			   ram_cmd, VFC_RAM_CMD_DWORDS);
+
+		/* Write VFC RAM address */
+		SET_VAR_FIELD(ram_addr, VFC_RAM_ADDR, ROW, row);
+		ARR_REG_WR(p_hwfn,
+			   p_ptt,
+			   s_storm_defs[storm_id].sem_fast_mem_addr +
+			   SEM_FAST_REG_VFC_ADDR,
+			   ram_addr, VFC_RAM_ADDR_DWORDS);
+
+		/* Read VFC RAM read response */
+		ARR_REG_RD(p_hwfn,
+			   p_ptt,
+			   s_storm_defs[storm_id].sem_fast_mem_addr +
+			   SEM_FAST_REG_VFC_DATA_RD,
+			   dump_buf + offset, VFC_RAM_RESP_DWORDS);
+	}
+
+	return offset;
+}
+
+/* Dumps GRC VFC data. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_vfc(struct qed_hwfn *p_hwfn,
+			    struct qed_ptt *p_ptt, u32 *dump_buf, bool dump)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u8 storm_id, i;
+	u32 offset = 0;
+
+	for (storm_id = 0; storm_id < MAX_DBG_STORMS; storm_id++) {
+		if (qed_grc_is_storm_included(p_hwfn,
+					      (enum dbg_storms)storm_id) &&
+		    s_storm_defs[storm_id].has_vfc &&
+		    (storm_id != DBG_PSTORM_ID ||
+		     dev_data->platform_id == PLATFORM_ASIC)) {
+			/* Read CAM */
+			offset += qed_grc_dump_vfc_cam(p_hwfn,
+						       p_ptt,
+						       dump_buf + offset,
+						       dump, storm_id);
+
+			/* Read RAM */
+			for (i = 0; i < NUM_VFC_RAM_TYPES; i++)
+				offset += qed_grc_dump_vfc_ram(p_hwfn,
+							       p_ptt,
+							       dump_buf +
+							       offset,
+							       dump,
+							       storm_id,
+							       &s_vfc_ram_defs
+							       [i]);
+		}
+	}
+
+	return offset;
+}
+
+/* Dumps GRC RSS data. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_rss(struct qed_hwfn *p_hwfn,
+			    struct qed_ptt *p_ptt, u32 *dump_buf, bool dump)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 offset = 0;
+	u8 rss_mem_id;
+
+	for (rss_mem_id = 0; rss_mem_id < NUM_RSS_MEM_TYPES; rss_mem_id++) {
+		struct rss_mem_defs *rss_defs = &s_rss_mem_defs[rss_mem_id];
+		u32 num_entries = rss_defs->num_entries[dev_data->chip_id];
+		u32 entry_width = rss_defs->entry_width[dev_data->chip_id];
+		u32 total_size = (num_entries * entry_width) / 32;
+		bool packed = (entry_width == 16);
+		u32 addr = rss_defs->addr;
+		u32 i, j;
+
+		offset += qed_grc_dump_mem_hdr(p_hwfn,
+					       dump_buf + offset,
+					       dump,
+					       rss_defs->mem_name,
+					       addr,
+					       total_size,
+					       entry_width,
+					       packed,
+					       rss_defs->type_name, false, 0);
+
+		if (!dump) {
+			offset += total_size;
+			continue;
+		}
+
+		/* Dump RSS data */
+		for (i = 0; i < BYTES_TO_DWORDS(total_size); i++, addr++) {
+			qed_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_ADDR, addr);
+			for (j = 0; j < BYTES_IN_DWORD; j++, offset++)
+				*(dump_buf + offset) =
+					qed_rd(p_hwfn, p_ptt,
+					       RSS_REG_RSS_RAM_DATA +
+					       DWORDS_TO_BYTES(j));
+		}
+	}
+
+	return offset;
+}
+
+/* Dumps GRC Big RAM. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_big_ram(struct qed_hwfn *p_hwfn,
+				struct qed_ptt *p_ptt,
+				u32 *dump_buf, bool dump, u8 big_ram_id)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	char mem_name[12] = "???_BIG_RAM";
+	char type_name[8] = "???_RAM";
+	u32 ram_size, total_blocks;
+	u32 offset = 0, i, j;
+
+	total_blocks =
+		s_big_ram_defs[big_ram_id].num_of_blocks[dev_data->chip_id];
+	ram_size = total_blocks * BIG_RAM_BLOCK_SIZE_DWORDS;
+
+	strncpy(type_name, s_big_ram_defs[big_ram_id].instance_name,
+		strlen(s_big_ram_defs[big_ram_id].instance_name));
+	strncpy(mem_name, s_big_ram_defs[big_ram_id].instance_name,
+		strlen(s_big_ram_defs[big_ram_id].instance_name));
+
+	/* Dump memory header */
+	offset += qed_grc_dump_mem_hdr(p_hwfn,
+				       dump_buf + offset,
+				       dump,
+				       mem_name,
+				       0,
+				       ram_size,
+				       BIG_RAM_BLOCK_SIZE_BYTES * 8,
+				       false, type_name, false, 0);
+
+	if (!dump)
+		return offset + ram_size;
+
+	/* Read and dump Big RAM data */
+	for (i = 0; i < total_blocks / 2; i++) {
+		qed_wr(p_hwfn, p_ptt, s_big_ram_defs[big_ram_id].addr_reg_addr,
+		       i);
+		for (j = 0; j < 2 * BIG_RAM_BLOCK_SIZE_DWORDS; j++, offset++)
+			*(dump_buf + offset) = qed_rd(p_hwfn, p_ptt,
+						s_big_ram_defs[big_ram_id].
+							data_reg_addr +
+						DWORDS_TO_BYTES(j));
+	}
+
+	return offset;
+}
+
+static u32 qed_grc_dump_mcp(struct qed_hwfn *p_hwfn,
+			    struct qed_ptt *p_ptt, u32 *dump_buf, bool dump)
+{
+	bool block_enable[MAX_BLOCK_ID] = { 0 };
+	bool halted = false;
+	u32 offset = 0;
+
+	/* Halt MCP */
+	if (dump) {
+		halted = !qed_mcp_halt(p_hwfn, p_ptt);
+		if (!halted)
+			DP_NOTICE(p_hwfn, "MCP halt failed!\n");
+	}
+
+	/* Dump MCP scratchpad */
+	offset += qed_grc_dump_mem(p_hwfn,
+				   p_ptt,
+				   dump_buf + offset,
+				   dump,
+				   NULL,
+				   MCP_REG_SCRATCH,
+				   MCP_REG_SCRATCH_SIZE,
+				   0, false, "MCP", false, 0);
+
+	/* Dump MCP cpu_reg_file */
+	offset += qed_grc_dump_mem(p_hwfn,
+				   p_ptt,
+				   dump_buf + offset,
+				   dump,
+				   NULL,
+				   MCP_REG_CPU_REG_FILE,
+				   MCP_REG_CPU_REG_FILE_SIZE,
+				   0, false, "MCP", false, 0);
+
+	/* Dump MCP registers */
+	block_enable[BLOCK_MCP] = true;
+	offset += qed_grc_dump_registers(p_hwfn,
+					 p_ptt,
+					 dump_buf + offset,
+					 dump, block_enable, "block", "MCP");
+
+	/* Dump required non-MCP registers */
+	offset += qed_grc_dump_regs_hdr(dump_buf + offset,
+					dump, 1, "eng", -1, "block", "MCP");
+	offset += qed_grc_dump_reg_entry(p_hwfn,
+					 p_ptt,
+					 dump_buf + offset,
+					 dump,
+					 BYTES_TO_DWORDS
+					 (MISC_REG_SHARED_MEM_ADDR), 1);
+
+	/* Release MCP */
+	if (halted && qed_mcp_resume(p_hwfn, p_ptt))
+		DP_NOTICE(p_hwfn, "Failed to resume MCP after halt!\n");
+	return offset;
+}
+
+/* Dumps the tbus indirect memory for all PHYs. */
+static u32 qed_grc_dump_phy(struct qed_hwfn *p_hwfn,
+			    struct qed_ptt *p_ptt, u32 *dump_buf, bool dump)
+{
+	u32 offset = 0, tbus_lo_offset, tbus_hi_offset;
+	char mem_name[32];
+	u8 phy_id;
+
+	for (phy_id = 0; phy_id < ARRAY_SIZE(s_phy_defs); phy_id++) {
+		struct phy_defs *phy_defs = &s_phy_defs[phy_id];
+		int printed_chars;
+
+		printed_chars = snprintf(mem_name, sizeof(mem_name), "tbus_%s",
+					 phy_defs->phy_name);
+		if (printed_chars < 0 || printed_chars >= sizeof(mem_name))
+			DP_NOTICE(p_hwfn,
+				  "Unexpected debug error: invalid PHY memory name\n");
+		offset += qed_grc_dump_mem_hdr(p_hwfn,
+					       dump_buf + offset,
+					       dump,
+					       mem_name,
+					       0,
+					       PHY_DUMP_SIZE_DWORDS,
+					       16, true, mem_name, false, 0);
+		if (dump) {
+			u32 addr_lo_addr = phy_defs->base_addr +
+					   phy_defs->tbus_addr_lo_addr;
+			u32 addr_hi_addr = phy_defs->base_addr +
+					   phy_defs->tbus_addr_hi_addr;
+			u32 data_lo_addr = phy_defs->base_addr +
+					   phy_defs->tbus_data_lo_addr;
+			u32 data_hi_addr = phy_defs->base_addr +
+					   phy_defs->tbus_data_hi_addr;
+			u8 *bytes_buf = (u8 *)(dump_buf + offset);
+
+			for (tbus_hi_offset = 0;
+			     tbus_hi_offset < (NUM_PHY_TBUS_ADDRESSES >> 8);
+			     tbus_hi_offset++) {
+				qed_wr(p_hwfn,
+				       p_ptt, addr_hi_addr, tbus_hi_offset);
+				for (tbus_lo_offset = 0; tbus_lo_offset < 256;
+				     tbus_lo_offset++) {
+					qed_wr(p_hwfn,
+					       p_ptt,
+					       addr_lo_addr, tbus_lo_offset);
+					*(bytes_buf++) =
+						(u8)qed_rd(p_hwfn, p_ptt,
+							   data_lo_addr);
+					*(bytes_buf++) =
+						(u8)qed_rd(p_hwfn, p_ptt,
+							   data_hi_addr);
+				}
+			}
+		}
+
+		offset += PHY_DUMP_SIZE_DWORDS;
+	}
+
+	return offset;
+}
+
+static void qed_config_dbg_line(struct qed_hwfn *p_hwfn,
+				struct qed_ptt *p_ptt,
+				enum block_id block_id,
+				u8 line_id,
+				u8 cycle_en,
+				u8 right_shift, u8 force_valid, u8 force_frame)
+{
+	struct block_defs *p_block_defs = s_block_defs[block_id];
+
+	qed_wr(p_hwfn, p_ptt, p_block_defs->dbg_select_addr, line_id);
+	qed_wr(p_hwfn, p_ptt, p_block_defs->dbg_cycle_enable_addr, cycle_en);
+	qed_wr(p_hwfn, p_ptt, p_block_defs->dbg_shift_addr, right_shift);
+	qed_wr(p_hwfn, p_ptt, p_block_defs->dbg_force_valid_addr, force_valid);
+	qed_wr(p_hwfn, p_ptt, p_block_defs->dbg_force_frame_addr, force_frame);
+}
+
+/* Dumps Static Debug data. Returns the dumped size in dwords. */
+static u32 qed_grc_dump_static_debug(struct qed_hwfn *p_hwfn,
+				     struct qed_ptt *p_ptt,
+				     u32 *dump_buf, bool dump)
+{
+	u32 block_dwords = NUM_DBG_BUS_LINES * STATIC_DEBUG_LINE_DWORDS;
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 offset = 0, block_id, line_id, addr, i;
+	struct block_defs *p_block_defs;
+
+	if (dump) {
+		DP_VERBOSE(p_hwfn,
+			   QED_MSG_DEBUG, "Dumping static debug data...\n");
+
+		/* Disable all blocks debug output */
+		for (block_id = 0; block_id < MAX_BLOCK_ID; block_id++) {
+			p_block_defs = s_block_defs[block_id];
+
+			if (p_block_defs->has_dbg_bus[dev_data->chip_id])
+				qed_wr(p_hwfn, p_ptt,
+				       p_block_defs->dbg_cycle_enable_addr, 0);
+		}
+
+		qed_bus_reset_dbg_block(p_hwfn, p_ptt);
+		qed_bus_set_framing_mode(p_hwfn,
+					 p_ptt, DBG_BUS_FRAME_MODE_8HW_0ST);
+		qed_wr(p_hwfn,
+		       p_ptt, DBG_REG_DEBUG_TARGET, DBG_BUS_TARGET_ID_INT_BUF);
+		qed_wr(p_hwfn, p_ptt, DBG_REG_FULL_MODE, 1);
+		qed_bus_enable_dbg_block(p_hwfn, p_ptt, true);
+	}
+
+	/* Dump all static debug lines for each relevant block */
+	for (block_id = 0; block_id < MAX_BLOCK_ID; block_id++) {
+		p_block_defs = s_block_defs[block_id];
+
+		if (!p_block_defs->has_dbg_bus[dev_data->chip_id])
+			continue;
+
+		/* Dump static section params */
+		offset += qed_grc_dump_mem_hdr(p_hwfn,
+					       dump_buf + offset,
+					       dump,
+					       p_block_defs->name, 0,
+					       block_dwords, 32, false,
+					       "STATIC", false, 0);
+
+		if (dump && !dev_data->block_in_reset[block_id]) {
+			u8 dbg_client_id =
+				p_block_defs->dbg_client_id[dev_data->chip_id];
+
+			/* Enable block's client */
+			qed_bus_enable_clients(p_hwfn, p_ptt,
+					       BIT(dbg_client_id));
+
+			for (line_id = 0; line_id < NUM_DBG_BUS_LINES;
+			     line_id++) {
+				/* Configure debug line ID */
+				qed_config_dbg_line(p_hwfn,
+						    p_ptt,
+						    (enum block_id)block_id,
+						    (u8)line_id,
+						    0xf, 0, 0, 0);
+
+				/* Read debug line info */
+				for (i = 0, addr = DBG_REG_CALENDAR_OUT_DATA;
+				     i < STATIC_DEBUG_LINE_DWORDS;
+				     i++, offset++, addr += BYTES_IN_DWORD)
+					dump_buf[offset] = qed_rd(p_hwfn, p_ptt,
+								  addr);
+			}
+
+			/* Disable block's client and debug output */
+			qed_bus_enable_clients(p_hwfn, p_ptt, 0);
+			qed_wr(p_hwfn, p_ptt,
+			       p_block_defs->dbg_cycle_enable_addr, 0);
+		} else {
+			/* All lines are invalid - dump zeros */
+			if (dump)
+				memset(dump_buf + offset, 0,
+				       DWORDS_TO_BYTES(block_dwords));
+			offset += block_dwords;
+		}
+	}
+
+	if (dump) {
+		qed_bus_enable_dbg_block(p_hwfn, p_ptt, false);
+		qed_bus_enable_clients(p_hwfn, p_ptt, 0);
+	}
+
+	return offset;
+}
+
+/* Performs GRC Dump to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static enum dbg_status qed_grc_dump(struct qed_hwfn *p_hwfn,
+				    struct qed_ptt *p_ptt,
+				    u32 *dump_buf,
+				    bool dump, u32 *num_dumped_dwords)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	bool parities_masked = false;
+	u8 i, port_mode = 0;
+	u32 offset = 0;
+
+	/* Check if emulation platform */
+	*num_dumped_dwords = 0;
+
+	/* Fill GRC parameters that were not set by the user with their default
+	 * value.
+	 */
+	qed_dbg_grc_set_params_default(p_hwfn);
+
+	/* Find port mode */
+	if (dump) {
+		switch (qed_rd(p_hwfn, p_ptt, MISC_REG_PORT_MODE)) {
+		case 0:
+			port_mode = 1;
+			break;
+		case 1:
+			port_mode = 2;
+			break;
+		case 2:
+			port_mode = 4;
+			break;
+		}
+	}
+
+	/* Update reset state */
+	if (dump)
+		qed_update_blocks_reset_state(p_hwfn, p_ptt);
+
+	/* Dump global params */
+	offset += qed_dump_common_global_params(p_hwfn,
+						p_ptt,
+						dump_buf + offset, dump, 4);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "dump-type", "grc-dump");
+	offset += qed_dump_num_param(dump_buf + offset,
+				     dump,
+				     "num-lcids",
+				     qed_grc_get_param(p_hwfn,
+						DBG_GRC_PARAM_NUM_LCIDS));
+	offset += qed_dump_num_param(dump_buf + offset,
+				     dump,
+				     "num-ltids",
+				     qed_grc_get_param(p_hwfn,
+						DBG_GRC_PARAM_NUM_LTIDS));
+	offset += qed_dump_num_param(dump_buf + offset,
+				     dump, "num-ports", port_mode);
+
+	/* Dump reset registers (dumped before taking blocks out of reset ) */
+	if (qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_REGS))
+		offset += qed_grc_dump_reset_regs(p_hwfn,
+						  p_ptt,
+						  dump_buf + offset, dump);
+
+	/* Take all blocks out of reset (using reset registers) */
+	if (dump) {
+		qed_grc_unreset_blocks(p_hwfn, p_ptt);
+		qed_update_blocks_reset_state(p_hwfn, p_ptt);
+	}
+
+	/* Disable all parities using MFW command */
+	if (dump) {
+		parities_masked = !qed_mcp_mask_parities(p_hwfn, p_ptt, 1);
+		if (!parities_masked) {
+			if (qed_grc_get_param
+			    (p_hwfn, DBG_GRC_PARAM_PARITY_SAFE))
+				return DBG_STATUS_MCP_COULD_NOT_MASK_PRTY;
+			else
+				DP_NOTICE(p_hwfn,
+					  "Failed to mask parities using MFW\n");
+		}
+	}
+
+	/* Dump modified registers (dumped before modifying them) */
+	if (qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_REGS))
+		offset += qed_grc_dump_modified_regs(p_hwfn,
+						     p_ptt,
+						     dump_buf + offset, dump);
+
+	/* Stall storms */
+	if (dump &&
+	    (qed_grc_is_included(p_hwfn,
+				 DBG_GRC_PARAM_DUMP_IOR) ||
+	     qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_VFC)))
+		qed_grc_stall_storms(p_hwfn, p_ptt, true);
+
+	/* Dump all regs  */
+	if (qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_REGS)) {
+		/* Dump all blocks except MCP */
+		bool block_enable[MAX_BLOCK_ID];
+
+		for (i = 0; i < MAX_BLOCK_ID; i++)
+			block_enable[i] = true;
+		block_enable[BLOCK_MCP] = false;
+		offset += qed_grc_dump_registers(p_hwfn,
+						 p_ptt,
+						 dump_buf +
+						 offset,
+						 dump,
+						 block_enable, NULL, NULL);
+	}
+
+	/* Dump memories */
+	offset += qed_grc_dump_memories(p_hwfn, p_ptt, dump_buf + offset, dump);
+
+	/* Dump MCP */
+	if (qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_MCP))
+		offset += qed_grc_dump_mcp(p_hwfn,
+					   p_ptt, dump_buf + offset, dump);
+
+	/* Dump context */
+	if (qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_CM_CTX))
+		offset += qed_grc_dump_ctx(p_hwfn,
+					   p_ptt, dump_buf + offset, dump);
+
+	/* Dump RSS memories */
+	if (qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_RSS))
+		offset += qed_grc_dump_rss(p_hwfn,
+					   p_ptt, dump_buf + offset, dump);
+
+	/* Dump Big RAM */
+	for (i = 0; i < NUM_BIG_RAM_TYPES; i++)
+		if (qed_grc_is_included(p_hwfn, s_big_ram_defs[i].grc_param))
+			offset += qed_grc_dump_big_ram(p_hwfn,
+						       p_ptt,
+						       dump_buf + offset,
+						       dump, i);
+
+	/* Dump IORs */
+	if (qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_IOR))
+		offset += qed_grc_dump_iors(p_hwfn,
+					    p_ptt, dump_buf + offset, dump);
+
+	/* Dump VFC */
+	if (qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_VFC))
+		offset += qed_grc_dump_vfc(p_hwfn,
+					   p_ptt, dump_buf + offset, dump);
+
+	/* Dump PHY tbus */
+	if (qed_grc_is_included(p_hwfn,
+				DBG_GRC_PARAM_DUMP_PHY) && dev_data->chip_id ==
+	    CHIP_K2 && dev_data->platform_id == PLATFORM_ASIC)
+		offset += qed_grc_dump_phy(p_hwfn,
+					   p_ptt, dump_buf + offset, dump);
+
+	/* Dump static debug data  */
+	if (qed_grc_is_included(p_hwfn,
+				DBG_GRC_PARAM_DUMP_STATIC) &&
+	    dev_data->bus.state == DBG_BUS_STATE_IDLE)
+		offset += qed_grc_dump_static_debug(p_hwfn,
+						    p_ptt,
+						    dump_buf + offset, dump);
+
+	/* Dump last section */
+	offset += qed_dump_last_section(dump_buf, offset, dump);
+	if (dump) {
+		/* Unstall storms */
+		if (qed_grc_get_param(p_hwfn, DBG_GRC_PARAM_UNSTALL))
+			qed_grc_stall_storms(p_hwfn, p_ptt, false);
+
+		/* Clear parity status */
+		qed_grc_clear_all_prty(p_hwfn, p_ptt);
+
+		/* Enable all parities using MFW command */
+		if (parities_masked)
+			qed_mcp_mask_parities(p_hwfn, p_ptt, 0);
+	}
+
+	*num_dumped_dwords = offset;
+
+	return DBG_STATUS_OK;
+}
+
+/* Writes the specified failing Idle Check rule to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_idle_chk_dump_failure(struct qed_hwfn *p_hwfn,
+				     struct qed_ptt *p_ptt,
+				     u32 *
+				     dump_buf,
+				     bool dump,
+				     u16 rule_id,
+				     const struct dbg_idle_chk_rule *rule,
+				     u16 fail_entry_id, u32 *cond_reg_values)
+{
+	const union dbg_idle_chk_reg *regs = &((const union dbg_idle_chk_reg *)
+					       s_dbg_arrays
+					       [BIN_BUF_DBG_IDLE_CHK_REGS].
+					       ptr)[rule->reg_offset];
+	const struct dbg_idle_chk_cond_reg *cond_regs = &regs[0].cond_reg;
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	struct dbg_idle_chk_result_hdr *hdr =
+		(struct dbg_idle_chk_result_hdr *)dump_buf;
+	const struct dbg_idle_chk_info_reg *info_regs =
+		&regs[rule->num_cond_regs].info_reg;
+	u32 next_reg_offset = 0, i, offset = 0;
+	u8 reg_id;
+
+	/* Dump rule data */
+	if (dump) {
+		memset(hdr, 0, sizeof(*hdr));
+		hdr->rule_id = rule_id;
+		hdr->mem_entry_id = fail_entry_id;
+		hdr->severity = rule->severity;
+		hdr->num_dumped_cond_regs = rule->num_cond_regs;
+	}
+
+	offset += IDLE_CHK_RESULT_HDR_DWORDS;
+
+	/* Dump condition register values */
+	for (reg_id = 0; reg_id < rule->num_cond_regs; reg_id++) {
+		const struct dbg_idle_chk_cond_reg *reg = &cond_regs[reg_id];
+
+		/* Write register header */
+		if (dump) {
+			struct dbg_idle_chk_result_reg_hdr *reg_hdr =
+			    (struct dbg_idle_chk_result_reg_hdr *)(dump_buf
+								   + offset);
+			offset += IDLE_CHK_RESULT_REG_HDR_DWORDS;
+			memset(reg_hdr, 0,
+			       sizeof(struct dbg_idle_chk_result_reg_hdr));
+			reg_hdr->start_entry = reg->start_entry;
+			reg_hdr->size = reg->entry_size;
+			SET_FIELD(reg_hdr->data,
+				  DBG_IDLE_CHK_RESULT_REG_HDR_IS_MEM,
+				  reg->num_entries > 1 || reg->start_entry > 0
+				  ? 1 : 0);
+			SET_FIELD(reg_hdr->data,
+				  DBG_IDLE_CHK_RESULT_REG_HDR_REG_ID, reg_id);
+
+			/* Write register values */
+			for (i = 0; i < reg_hdr->size;
+			     i++, next_reg_offset++, offset++)
+				dump_buf[offset] =
+				    cond_reg_values[next_reg_offset];
+		} else {
+			offset += IDLE_CHK_RESULT_REG_HDR_DWORDS +
+			    reg->entry_size;
+		}
+	}
+
+	/* Dump info register values */
+	for (reg_id = 0; reg_id < rule->num_info_regs; reg_id++) {
+		const struct dbg_idle_chk_info_reg *reg = &info_regs[reg_id];
+		u32 block_id;
+
+		if (!dump) {
+			offset += IDLE_CHK_RESULT_REG_HDR_DWORDS + reg->size;
+			continue;
+		}
+
+		/* Check if register's block is in reset */
+		block_id = GET_FIELD(reg->data, DBG_IDLE_CHK_INFO_REG_BLOCK_ID);
+		if (block_id >= MAX_BLOCK_ID) {
+			DP_NOTICE(p_hwfn, "Invalid block_id\n");
+			return 0;
+		}
+
+		if (!dev_data->block_in_reset[block_id]) {
+			bool eval_mode = GET_FIELD(reg->mode.data,
+						   DBG_MODE_HDR_EVAL_MODE) > 0;
+			bool mode_match = true;
+
+			/* Check mode */
+			if (eval_mode) {
+				u16 modes_buf_offset =
+					GET_FIELD(reg->mode.data,
+						DBG_MODE_HDR_MODES_BUF_OFFSET);
+				mode_match =
+					qed_is_mode_match(p_hwfn,
+							  &modes_buf_offset);
+			}
+
+			if (mode_match) {
+				u32 grc_addr =
+					DWORDS_TO_BYTES(GET_FIELD(reg->data,
+						DBG_IDLE_CHK_INFO_REG_ADDRESS));
+
+				/* Write register header */
+				struct dbg_idle_chk_result_reg_hdr *reg_hdr =
+					(struct dbg_idle_chk_result_reg_hdr *)
+					(dump_buf + offset);
+
+				offset += IDLE_CHK_RESULT_REG_HDR_DWORDS;
+				hdr->num_dumped_info_regs++;
+				memset(reg_hdr, 0, sizeof(*reg_hdr));
+				reg_hdr->size = reg->size;
+				SET_FIELD(reg_hdr->data,
+					DBG_IDLE_CHK_RESULT_REG_HDR_REG_ID,
+					rule->num_cond_regs + reg_id);
+
+				/* Write register values */
+				for (i = 0; i < reg->size;
+				     i++, offset++, grc_addr += 4)
+					dump_buf[offset] =
+						qed_rd(p_hwfn, p_ptt, grc_addr);
+				}
+			}
+	}
+
+	return offset;
+}
+
+/* Dumps idle check rule entries. Returns the dumped size in dwords. */
+static u32
+qed_idle_chk_dump_rule_entries(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+			       u32 *dump_buf, bool dump,
+			       const struct dbg_idle_chk_rule *input_rules,
+			       u32 num_input_rules, u32 *num_failing_rules)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	u32 cond_reg_values[IDLE_CHK_MAX_ENTRIES_SIZE];
+	u32 i, j, offset = 0;
+	u16 entry_id;
+	u8 reg_id;
+
+	*num_failing_rules = 0;
+	for (i = 0; i < num_input_rules; i++) {
+		const struct dbg_idle_chk_cond_reg *cond_regs;
+		const struct dbg_idle_chk_rule *rule;
+		const union dbg_idle_chk_reg *regs;
+		u16 num_reg_entries = 1;
+		bool check_rule = true;
+		const u32 *imm_values;
+
+		rule = &input_rules[i];
+		regs = &((const union dbg_idle_chk_reg *)
+			 s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_REGS].ptr)
+			[rule->reg_offset];
+		cond_regs = &regs[0].cond_reg;
+		imm_values = &s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_IMMS].ptr
+			     [rule->imm_offset];
+
+		/* Check if all condition register blocks are out of reset, and
+		 * find maximal number of entries (all condition registers that
+		 * are memories must have the same size, which is > 1).
+		 */
+		for (reg_id = 0; reg_id < rule->num_cond_regs && check_rule;
+		     reg_id++) {
+			u32 block_id = GET_FIELD(cond_regs[reg_id].data,
+						DBG_IDLE_CHK_COND_REG_BLOCK_ID);
+
+			if (block_id >= MAX_BLOCK_ID) {
+				DP_NOTICE(p_hwfn, "Invalid block_id\n");
+				return 0;
+			}
+
+			check_rule = !dev_data->block_in_reset[block_id];
+			if (cond_regs[reg_id].num_entries > num_reg_entries)
+				num_reg_entries = cond_regs[reg_id].num_entries;
+		}
+
+		if (!check_rule && dump)
+			continue;
+
+		/* Go over all register entries (number of entries is the same
+		 * for all condition registers).
+		 */
+		for (entry_id = 0; entry_id < num_reg_entries; entry_id++) {
+			/* Read current entry of all condition registers */
+			if (dump) {
+				u32 next_reg_offset = 0;
+
+				for (reg_id = 0;
+				     reg_id < rule->num_cond_regs;
+				     reg_id++) {
+					const struct dbg_idle_chk_cond_reg
+						*reg = &cond_regs[reg_id];
+
+					/* Find GRC address (if it's a memory,
+					 * the address of the specific entry is
+					 * calculated).
+					 */
+					u32 grc_addr =
+					   DWORDS_TO_BYTES(
+						GET_FIELD(reg->data,
+						    DBG_IDLE_CHK_COND_REG_ADDRESS));
+
+					if (reg->num_entries > 1 ||
+					    reg->start_entry > 0) {
+						u32 padded_entry_size =
+							reg->entry_size > 1 ?
+							roundup_pow_of_two
+							(reg->entry_size) : 1;
+
+						grc_addr +=
+							DWORDS_TO_BYTES(
+								(reg->start_entry +
+								entry_id)
+								* padded_entry_size);
+					}
+
+					/* Read registers */
+					if (next_reg_offset + reg->entry_size >=
+					    IDLE_CHK_MAX_ENTRIES_SIZE) {
+						DP_NOTICE(p_hwfn,
+							  "idle check registers entry is too large\n");
+						return 0;
+					}
+
+					for (j = 0; j < reg->entry_size;
+					     j++, next_reg_offset++,
+					     grc_addr += 4)
+					     cond_reg_values[next_reg_offset] =
+						qed_rd(p_hwfn, p_ptt, grc_addr);
+				}
+			}
+
+			/* Call rule's condition function - a return value of
+			 * true indicates failure.
+			 */
+			if ((*cond_arr[rule->cond_id])(cond_reg_values,
+						       imm_values) || !dump) {
+				offset +=
+					qed_idle_chk_dump_failure(p_hwfn,
+							p_ptt,
+							dump_buf + offset,
+							dump,
+							rule->rule_id,
+							rule,
+							entry_id,
+							cond_reg_values);
+				(*num_failing_rules)++;
+				break;
+			}
+		}
+	}
+
+	return offset;
+}
+
+/* Performs Idle Check Dump to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_idle_chk_dump(struct qed_hwfn *p_hwfn,
+			     struct qed_ptt *p_ptt, u32 *dump_buf, bool dump)
+{
+	u32 offset = 0, input_offset = 0, num_failing_rules = 0;
+	u32 num_failing_rules_offset;
+
+	/* Dump global params */
+	offset += qed_dump_common_global_params(p_hwfn,
+						p_ptt,
+						dump_buf + offset, dump, 1);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "dump-type", "idle-chk");
+
+	/* Dump idle check section header with a single parameter */
+	offset += qed_dump_section_hdr(dump_buf + offset, dump, "idle_chk", 1);
+	num_failing_rules_offset = offset;
+	offset += qed_dump_num_param(dump_buf + offset, dump, "num_rules", 0);
+	while (input_offset <
+	       s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_RULES].size_in_dwords) {
+		const struct dbg_idle_chk_cond_hdr *cond_hdr =
+			(const struct dbg_idle_chk_cond_hdr *)
+			&s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_RULES].ptr
+			[input_offset++];
+		bool eval_mode = GET_FIELD(cond_hdr->mode.data,
+					   DBG_MODE_HDR_EVAL_MODE) > 0;
+		bool mode_match = true;
+
+		/* Check mode */
+		if (eval_mode) {
+			u16 modes_buf_offset =
+				GET_FIELD(cond_hdr->mode.data,
+					  DBG_MODE_HDR_MODES_BUF_OFFSET);
+
+			mode_match = qed_is_mode_match(p_hwfn,
+						       &modes_buf_offset);
+		}
+
+		if (mode_match) {
+			u32 curr_failing_rules;
+
+			offset +=
+			    qed_idle_chk_dump_rule_entries(p_hwfn,
+				p_ptt,
+				dump_buf + offset,
+				dump,
+				(const struct dbg_idle_chk_rule *)
+				&s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_RULES].
+				ptr[input_offset],
+				cond_hdr->data_size / IDLE_CHK_RULE_SIZE_DWORDS,
+				&curr_failing_rules);
+			num_failing_rules += curr_failing_rules;
+		}
+
+		input_offset += cond_hdr->data_size;
+	}
+
+	/* Overwrite num_rules parameter */
+	if (dump)
+		qed_dump_num_param(dump_buf + num_failing_rules_offset,
+				   dump, "num_rules", num_failing_rules);
+
+	return offset;
+}
+
+/* Finds the meta data image in NVRAM. */
+static enum dbg_status qed_find_nvram_image(struct qed_hwfn *p_hwfn,
+					    struct qed_ptt *p_ptt,
+					    u32 image_type,
+					    u32 *nvram_offset_bytes,
+					    u32 *nvram_size_bytes)
+{
+	u32 ret_mcp_resp, ret_mcp_param, ret_txn_size;
+	struct mcp_file_att file_att;
+
+	/* Call NVRAM get file command */
+	if (qed_mcp_nvm_rd_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_NVM_GET_FILE_ATT,
+			       image_type, &ret_mcp_resp, &ret_mcp_param,
+			       &ret_txn_size, (u32 *)&file_att) != 0)
+		return DBG_STATUS_NVRAM_GET_IMAGE_FAILED;
+
+	/* Check response */
+	if ((ret_mcp_resp & FW_MSG_CODE_MASK) != FW_MSG_CODE_NVM_OK)
+		return DBG_STATUS_NVRAM_GET_IMAGE_FAILED;
+
+	/* Update return values */
+	*nvram_offset_bytes = file_att.nvm_start_addr;
+	*nvram_size_bytes = file_att.len;
+	DP_VERBOSE(p_hwfn,
+		   QED_MSG_DEBUG,
+		   "find_nvram_image: found NVRAM image of type %d in NVRAM offset %d bytes with size %d bytes\n",
+		   image_type, *nvram_offset_bytes, *nvram_size_bytes);
+
+	/* Check alignment */
+	if (*nvram_size_bytes & 0x3)
+		return DBG_STATUS_NON_ALIGNED_NVRAM_IMAGE;
+	return DBG_STATUS_OK;
+}
+
+static enum dbg_status qed_nvram_read(struct qed_hwfn *p_hwfn,
+				      struct qed_ptt *p_ptt,
+				      u32 nvram_offset_bytes,
+				      u32 nvram_size_bytes, u32 *ret_buf)
+{
+	u32 ret_mcp_resp, ret_mcp_param, ret_read_size;
+	u32 bytes_to_copy, read_offset = 0;
+	s32 bytes_left = nvram_size_bytes;
+
+	DP_VERBOSE(p_hwfn,
+		   QED_MSG_DEBUG,
+		   "nvram_read: reading image of size %d bytes from NVRAM\n",
+		   nvram_size_bytes);
+	do {
+		bytes_to_copy =
+		    (bytes_left >
+		     MCP_DRV_NVM_BUF_LEN) ? MCP_DRV_NVM_BUF_LEN : bytes_left;
+
+		/* Call NVRAM read command */
+		if (qed_mcp_nvm_rd_cmd(p_hwfn, p_ptt,
+				       DRV_MSG_CODE_NVM_READ_NVRAM,
+				       (nvram_offset_bytes +
+					read_offset) |
+				       (bytes_to_copy <<
+					DRV_MB_PARAM_NVM_LEN_SHIFT),
+				       &ret_mcp_resp, &ret_mcp_param,
+				       &ret_read_size,
+				       (u32 *)((u8 *)ret_buf +
+					       read_offset)) != 0)
+			return DBG_STATUS_NVRAM_READ_FAILED;
+
+		/* Check response */
+		if ((ret_mcp_resp & FW_MSG_CODE_MASK) != FW_MSG_CODE_NVM_OK)
+			return DBG_STATUS_NVRAM_READ_FAILED;
+
+		/* Update read offset */
+		read_offset += ret_read_size;
+		bytes_left -= ret_read_size;
+	} while (bytes_left > 0);
+
+	return DBG_STATUS_OK;
+}
+
+/* Get info on the MCP Trace data in the scratchpad:
+ * - trace_data_grc_addr - the GRC address of the trace data
+ * - trace_data_size_bytes - the size in bytes of the MCP Trace data (without
+ *	the header)
+ */
+static enum dbg_status qed_mcp_trace_get_data_info(struct qed_hwfn *p_hwfn,
+						   struct qed_ptt *p_ptt,
+						   u32 *trace_data_grc_addr,
+						   u32 *trace_data_size_bytes)
+{
+	/* Read MCP trace section offsize structure from MCP scratchpad */
+	u32 spad_trace_offsize = qed_rd(p_hwfn,
+					p_ptt,
+					MCP_SPAD_TRACE_OFFSIZE_ADDR);
+	u32 signature;
+
+	/* Extract MCP trace section GRC address from offsize structure (within
+	 * scratchpad).
+	 */
+	*trace_data_grc_addr =
+		MCP_REG_SCRATCH + SECTION_OFFSET(spad_trace_offsize);
+
+	/* Read signature from MCP trace section */
+	signature = qed_rd(p_hwfn, p_ptt,
+			   *trace_data_grc_addr +
+			   offsetof(struct mcp_trace, signature));
+	if (signature != MFW_TRACE_SIGNATURE)
+		return DBG_STATUS_INVALID_TRACE_SIGNATURE;
+
+	/* Read trace size from MCP trace section */
+	*trace_data_size_bytes = qed_rd(p_hwfn,
+					p_ptt,
+					*trace_data_grc_addr +
+					offsetof(struct mcp_trace, size));
+	return DBG_STATUS_OK;
+}
+
+/* Reads MCP trace meta data image from NVRAM.
+ * - running_bundle_id (OUT) - the running bundle ID (invalid when loaded from
+ *	file)
+ * - trace_meta_offset_bytes (OUT) - the NVRAM offset in bytes in which the MCP
+ *	Trace meta data starts (invalid when loaded from file)
+ * - trace_meta_size_bytes (OUT) - the size in bytes of the MCP Trace meta data
+ */
+static enum dbg_status qed_mcp_trace_get_meta_info(struct qed_hwfn *p_hwfn,
+						   struct qed_ptt *p_ptt,
+						   u32 trace_data_size_bytes,
+						   u32 *running_bundle_id,
+						   u32 *trace_meta_offset_bytes,
+						   u32 *trace_meta_size_bytes)
+{
+	/* Read MCP trace section offsize structure from MCP scratchpad */
+	u32 spad_trace_offsize = qed_rd(p_hwfn,
+					p_ptt,
+					MCP_SPAD_TRACE_OFFSIZE_ADDR);
+
+	/* Find running bundle ID */
+	u32 running_mfw_addr =
+		MCP_REG_SCRATCH + SECTION_OFFSET(spad_trace_offsize) +
+		QED_SECTION_SIZE(spad_trace_offsize) + trace_data_size_bytes;
+	enum dbg_status status;
+	u32 nvram_image_type;
+
+	*running_bundle_id = qed_rd(p_hwfn, p_ptt, running_mfw_addr);
+	if (*running_bundle_id > 1)
+		return DBG_STATUS_INVALID_NVRAM_BUNDLE;
+
+	/* Find image in NVRAM */
+	nvram_image_type =
+	    (*running_bundle_id ==
+	     DIR_ID_1) ? NVM_TYPE_MFW_TRACE1 : NVM_TYPE_MFW_TRACE2;
+	status = qed_find_nvram_image(p_hwfn,
+				      p_ptt,
+				      nvram_image_type,
+				      trace_meta_offset_bytes,
+				      trace_meta_size_bytes);
+
+	return status;
+}
+
+/* Reads the MCP Trace data from the specified GRC address into the specified
+ * buffer.
+ */
+static void qed_mcp_trace_read_data(struct qed_hwfn *p_hwfn,
+				    struct qed_ptt *p_ptt,
+				    u32 grc_addr, u32 size_in_dwords, u32 *buf)
+{
+	u32 i;
+
+	DP_VERBOSE(p_hwfn,
+		   QED_MSG_DEBUG,
+		   "mcp_trace_read_data: reading trace data of size %d dwords from GRC address 0x%x\n",
+		   size_in_dwords, grc_addr);
+	for (i = 0; i < size_in_dwords; i++, grc_addr += BYTES_IN_DWORD)
+		buf[i] = qed_rd(p_hwfn, p_ptt, grc_addr);
+}
+
+/* Reads the MCP Trace meta data (from NVRAM or buffer) into the specified
+ * buffer.
+ */
+static enum dbg_status qed_mcp_trace_read_meta(struct qed_hwfn *p_hwfn,
+					       struct qed_ptt *p_ptt,
+					       u32 nvram_offset_in_bytes,
+					       u32 size_in_bytes, u32 *buf)
+{
+	u8 *byte_buf = (u8 *)buf;
+	u8 modules_num, i;
+	u32 signature;
+
+	/* Read meta data from NVRAM */
+	enum dbg_status status = qed_nvram_read(p_hwfn,
+						p_ptt,
+						nvram_offset_in_bytes,
+						size_in_bytes,
+						buf);
+
+	if (status != DBG_STATUS_OK)
+		return status;
+
+	/* Extract and check first signature */
+	signature = qed_read_unaligned_dword(byte_buf);
+	byte_buf += sizeof(u32);
+	if (signature != MCP_TRACE_META_IMAGE_SIGNATURE)
+		return DBG_STATUS_INVALID_TRACE_SIGNATURE;
+
+	/* Extract number of modules */
+	modules_num = *(byte_buf++);
+
+	/* Skip all modules */
+	for (i = 0; i < modules_num; i++) {
+		u8 module_len = *(byte_buf++);
+
+		byte_buf += module_len;
+	}
+
+	/* Extract and check second signature */
+	signature = qed_read_unaligned_dword(byte_buf);
+	byte_buf += sizeof(u32);
+	if (signature != MCP_TRACE_META_IMAGE_SIGNATURE)
+		return DBG_STATUS_INVALID_TRACE_SIGNATURE;
+	return DBG_STATUS_OK;
+}
+
+/* Dump MCP Trace */
+enum dbg_status qed_mcp_trace_dump(struct qed_hwfn *p_hwfn,
+				   struct qed_ptt *p_ptt,
+				   u32 *dump_buf,
+				   bool dump, u32 *num_dumped_dwords)
+{
+	u32 trace_data_grc_addr, trace_data_size_bytes, trace_data_size_dwords;
+	u32 trace_meta_size_dwords, running_bundle_id, offset = 0;
+	u32 trace_meta_offset_bytes, trace_meta_size_bytes;
+	enum dbg_status status;
+	int halted = 0;
+
+	*num_dumped_dwords = 0;
+
+	/* Get trace data info */
+	status = qed_mcp_trace_get_data_info(p_hwfn,
+					     p_ptt,
+					     &trace_data_grc_addr,
+					     &trace_data_size_bytes);
+	if (status != DBG_STATUS_OK)
+		return status;
+
+	/* Dump global params */
+	offset += qed_dump_common_global_params(p_hwfn,
+						p_ptt,
+						dump_buf + offset, dump, 1);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "dump-type", "mcp-trace");
+
+	/* Halt MCP while reading from scratchpad so the read data will be
+	 * consistent if halt fails, MCP trace is taken anyway, with a small
+	 * risk that it may be corrupt.
+	 */
+	if (dump) {
+		halted = !qed_mcp_halt(p_hwfn, p_ptt);
+		if (!halted)
+			DP_NOTICE(p_hwfn, "MCP halt failed!\n");
+	}
+
+	/* Find trace data size */
+	trace_data_size_dwords =
+		DIV_ROUND_UP(trace_data_size_bytes + sizeof(struct mcp_trace),
+			     BYTES_IN_DWORD);
+
+	/* Dump trace data section header and param */
+	offset += qed_dump_section_hdr(dump_buf + offset,
+				       dump, "mcp_trace_data", 1);
+	offset += qed_dump_num_param(dump_buf + offset,
+				     dump, "size", trace_data_size_dwords);
+
+	/* Read trace data from scratchpad into dump buffer */
+	if (dump)
+		qed_mcp_trace_read_data(p_hwfn,
+					p_ptt,
+					trace_data_grc_addr,
+					trace_data_size_dwords,
+					dump_buf + offset);
+	offset += trace_data_size_dwords;
+
+	/* Resume MCP (only if halt succeeded) */
+	if (halted && qed_mcp_resume(p_hwfn, p_ptt) != 0)
+		DP_NOTICE(p_hwfn, "Failed to resume MCP after halt!\n");
+
+	/* Dump trace meta section header */
+	offset += qed_dump_section_hdr(dump_buf + offset,
+				       dump, "mcp_trace_meta", 1);
+
+	/* Read trace meta info */
+	status = qed_mcp_trace_get_meta_info(p_hwfn,
+					     p_ptt,
+					     trace_data_size_bytes,
+					     &running_bundle_id,
+					     &trace_meta_offset_bytes,
+					     &trace_meta_size_bytes);
+	if (status != DBG_STATUS_OK)
+		return status;
+
+	/* Dump trace meta size param (trace_meta_size_bytes is always
+	 * dword-aligned).
+	 */
+	trace_meta_size_dwords = BYTES_TO_DWORDS(trace_meta_size_bytes);
+	offset += qed_dump_num_param(dump_buf + offset,	dump, "size",
+				     trace_meta_size_dwords);
+
+	/* Read trace meta image into dump buffer */
+	if (dump) {
+		status = qed_mcp_trace_read_meta(p_hwfn,
+						p_ptt,
+						trace_meta_offset_bytes,
+						trace_meta_size_bytes,
+						dump_buf + offset);
+		if (status != DBG_STATUS_OK)
+			return status;
+	}
+
+	offset += trace_meta_size_dwords;
+
+	*num_dumped_dwords = offset;
+
+	return DBG_STATUS_OK;
+}
+
+/* Dump GRC FIFO */
+enum dbg_status qed_reg_fifo_dump(struct qed_hwfn *p_hwfn,
+				  struct qed_ptt *p_ptt,
+				  u32 *dump_buf,
+				  bool dump, u32 *num_dumped_dwords)
+{
+	u32 offset = 0, dwords_read, size_param_offset;
+	bool fifo_has_data;
+
+	*num_dumped_dwords = 0;
+
+	/* Dump global params */
+	offset += qed_dump_common_global_params(p_hwfn,
+						p_ptt,
+						dump_buf + offset, dump, 1);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "dump-type", "reg-fifo");
+
+	/* Dump fifo data section header and param. The size param is 0 for now,
+	 * and is overwritten after reading the FIFO.
+	 */
+	offset += qed_dump_section_hdr(dump_buf + offset,
+				       dump, "reg_fifo_data", 1);
+	size_param_offset = offset;
+	offset += qed_dump_num_param(dump_buf + offset, dump, "size", 0);
+
+	if (!dump) {
+		/* FIFO max size is REG_FIFO_DEPTH_DWORDS. There is no way to
+		 * test how much data is available, except for reading it.
+		 */
+		offset += REG_FIFO_DEPTH_DWORDS;
+		*num_dumped_dwords = offset;
+		return DBG_STATUS_OK;
+	}
+
+	fifo_has_data = qed_rd(p_hwfn, p_ptt,
+			       GRC_REG_TRACE_FIFO_VALID_DATA) > 0;
+
+	/* Pull available data from fifo. Use DMAE since this is widebus memory
+	 * and must be accessed atomically. Test for dwords_read not passing
+	 * buffer size since more entries could be added to the buffer as we are
+	 * emptying it.
+	 */
+	for (dwords_read = 0;
+	     fifo_has_data && dwords_read < REG_FIFO_DEPTH_DWORDS;
+	     dwords_read += REG_FIFO_ELEMENT_DWORDS, offset +=
+	     REG_FIFO_ELEMENT_DWORDS) {
+		if (qed_dmae_grc2host(p_hwfn, p_ptt, GRC_REG_TRACE_FIFO,
+				      (u64)(uintptr_t)(&dump_buf[offset]),
+				      REG_FIFO_ELEMENT_DWORDS, 0))
+			return DBG_STATUS_DMAE_FAILED;
+		fifo_has_data = qed_rd(p_hwfn, p_ptt,
+				       GRC_REG_TRACE_FIFO_VALID_DATA) > 0;
+	}
+
+	qed_dump_num_param(dump_buf + size_param_offset, dump, "size",
+			   dwords_read);
+
+	*num_dumped_dwords = offset;
+	return DBG_STATUS_OK;
+}
+
+/* Dump IGU FIFO */
+enum dbg_status qed_igu_fifo_dump(struct qed_hwfn *p_hwfn,
+				  struct qed_ptt *p_ptt,
+				  u32 *dump_buf,
+				  bool dump, u32 *num_dumped_dwords)
+{
+	u32 offset = 0, dwords_read, size_param_offset;
+	bool fifo_has_data;
+
+	*num_dumped_dwords = 0;
+
+	/* Dump global params */
+	offset += qed_dump_common_global_params(p_hwfn,
+						p_ptt,
+						dump_buf + offset, dump, 1);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "dump-type", "igu-fifo");
+
+	/* Dump fifo data section header and param. The size param is 0 for now,
+	 * and is overwritten after reading the FIFO.
+	 */
+	offset += qed_dump_section_hdr(dump_buf + offset,
+				       dump, "igu_fifo_data", 1);
+	size_param_offset = offset;
+	offset += qed_dump_num_param(dump_buf + offset, dump, "size", 0);
+
+	if (!dump) {
+		/* FIFO max size is IGU_FIFO_DEPTH_DWORDS. There is no way to
+		 * test how much data is available, except for reading it.
+		 */
+		offset += IGU_FIFO_DEPTH_DWORDS;
+		*num_dumped_dwords = offset;
+		return DBG_STATUS_OK;
+	}
+
+	fifo_has_data = qed_rd(p_hwfn, p_ptt,
+			       IGU_REG_ERROR_HANDLING_DATA_VALID) > 0;
+
+	/* Pull available data from fifo. Use DMAE since this is widebus memory
+	 * and must be accessed atomically. Test for dwords_read not passing
+	 * buffer size since more entries could be added to the buffer as we are
+	 * emptying it.
+	 */
+	for (dwords_read = 0;
+	     fifo_has_data && dwords_read < IGU_FIFO_DEPTH_DWORDS;
+	     dwords_read += IGU_FIFO_ELEMENT_DWORDS, offset +=
+	     IGU_FIFO_ELEMENT_DWORDS) {
+		if (qed_dmae_grc2host(p_hwfn, p_ptt,
+				      IGU_REG_ERROR_HANDLING_MEMORY,
+				      (u64)(uintptr_t)(&dump_buf[offset]),
+				      IGU_FIFO_ELEMENT_DWORDS, 0))
+			return DBG_STATUS_DMAE_FAILED;
+		fifo_has_data =	qed_rd(p_hwfn, p_ptt,
+				       IGU_REG_ERROR_HANDLING_DATA_VALID) > 0;
+	}
+
+	qed_dump_num_param(dump_buf + size_param_offset, dump, "size",
+			   dwords_read);
+
+	*num_dumped_dwords = offset;
+	return DBG_STATUS_OK;
+}
+
+/* Protection Override dump */
+enum dbg_status qed_protection_override_dump(struct qed_hwfn *p_hwfn,
+					     struct qed_ptt *p_ptt,
+					     u32 *dump_buf,
+					     bool dump, u32 *num_dumped_dwords)
+{
+	u32 offset = 0, size_param_offset, override_window_dwords;
+
+	*num_dumped_dwords = 0;
+
+	/* Dump global params */
+	offset += qed_dump_common_global_params(p_hwfn,
+						p_ptt,
+						dump_buf + offset, dump, 1);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "dump-type", "protection-override");
+
+	/* Dump data section header and param. The size param is 0 for now, and
+	 * is overwritten after reading the data.
+	 */
+	offset += qed_dump_section_hdr(dump_buf + offset,
+				       dump, "protection_override_data", 1);
+	size_param_offset = offset;
+	offset += qed_dump_num_param(dump_buf + offset, dump, "size", 0);
+
+	if (!dump) {
+		offset += PROTECTION_OVERRIDE_DEPTH_DWORDS;
+		*num_dumped_dwords = offset;
+		return DBG_STATUS_OK;
+	}
+
+	/* Add override window info to buffer */
+	override_window_dwords =
+		qed_rd(p_hwfn, p_ptt,
+		       GRC_REG_NUMBER_VALID_OVERRIDE_WINDOW) *
+		       PROTECTION_OVERRIDE_ELEMENT_DWORDS;
+	if (qed_dmae_grc2host(p_hwfn, p_ptt,
+			      GRC_REG_PROTECTION_OVERRIDE_WINDOW,
+			      (u64)(uintptr_t)(dump_buf + offset),
+			      override_window_dwords, 0))
+		return DBG_STATUS_DMAE_FAILED;
+	offset += override_window_dwords;
+	qed_dump_num_param(dump_buf + size_param_offset, dump, "size",
+			   override_window_dwords);
+
+	*num_dumped_dwords = offset;
+	return DBG_STATUS_OK;
+}
+
+/* Performs FW Asserts Dump to the specified buffer.
+ * Returns the dumped size in dwords.
+ */
+static u32 qed_fw_asserts_dump(struct qed_hwfn *p_hwfn,
+			       struct qed_ptt *p_ptt, u32 *dump_buf, bool dump)
+{
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+	char storm_letter_str[2] = "?";
+	struct fw_info fw_info;
+	u32 offset = 0, i;
+	u8 storm_id;
+
+	/* Dump global params */
+	offset += qed_dump_common_global_params(p_hwfn,
+						p_ptt,
+						dump_buf + offset, dump, 1);
+	offset += qed_dump_str_param(dump_buf + offset,
+				     dump, "dump-type", "fw-asserts");
+	for (storm_id = 0; storm_id < MAX_DBG_STORMS; storm_id++) {
+		u32 fw_asserts_section_addr, next_list_idx_addr, next_list_idx,
+			last_list_idx, element_addr;
+
+		if (dev_data->block_in_reset[s_storm_defs[storm_id].block_id])
+			continue;
+
+		/* Read FW info for the current Storm */
+		qed_read_fw_info(p_hwfn, p_ptt, storm_id, &fw_info);
+
+		/* Dump FW Asserts section header and params */
+		storm_letter_str[0] = s_storm_defs[storm_id].letter;
+		offset += qed_dump_section_hdr(dump_buf + offset, dump,
+					       "fw_asserts", 2);
+		offset += qed_dump_str_param(dump_buf + offset, dump, "storm",
+					     storm_letter_str);
+		offset += qed_dump_num_param(dump_buf + offset, dump, "size",
+					     fw_info.fw_asserts_section.
+					     list_element_dword_size);
+
+		if (!dump) {
+			offset += fw_info.fw_asserts_section.
+				  list_element_dword_size;
+			continue;
+		}
+
+		/* Read and dump FW Asserts data */
+		fw_asserts_section_addr =
+			s_storm_defs[storm_id].sem_fast_mem_addr +
+			SEM_FAST_REG_INT_RAM +
+			RAM_LINES_TO_BYTES(fw_info.fw_asserts_section.
+					   section_ram_line_offset);
+		next_list_idx_addr =
+			fw_asserts_section_addr +
+			DWORDS_TO_BYTES(fw_info.fw_asserts_section.
+					list_next_index_dword_offset);
+		next_list_idx = qed_rd(p_hwfn, p_ptt, next_list_idx_addr);
+		last_list_idx = (next_list_idx > 0
+				 ? next_list_idx
+				 : fw_info.fw_asserts_section.list_num_elements)
+				- 1;
+		element_addr =
+			fw_asserts_section_addr +
+			DWORDS_TO_BYTES(fw_info.fw_asserts_section.
+					list_dword_offset) +
+			last_list_idx *
+			DWORDS_TO_BYTES(fw_info.fw_asserts_section.
+					list_element_dword_size);
+		for (i = 0;
+		     i < fw_info.fw_asserts_section.list_element_dword_size;
+		     i++, offset++, element_addr += BYTES_IN_DWORD)
+			dump_buf[offset] = qed_rd(p_hwfn, p_ptt, element_addr);
+	}
+
+	/* Dump last section */
+	offset += qed_dump_section_hdr(dump_buf + offset, dump, "last", 0);
+	return offset;
+}
+
+/***************************** Public Functions *******************************/
+
+enum dbg_status qed_dbg_set_bin_ptr(const u8 * const bin_ptr)
+{
+	/* Convert binary data to debug arrays */
+	u32 num_of_buffers = *(u32 *)bin_ptr;
+	struct bin_buffer_hdr *buf_array;
+	u8 buf_id;
+
+	buf_array = (struct bin_buffer_hdr *)((u32 *)bin_ptr + 1);
+
+	for (buf_id = 0; buf_id < num_of_buffers; buf_id++) {
+		s_dbg_arrays[buf_id].ptr =
+		    (u32 *)(bin_ptr + buf_array[buf_id].offset);
+		s_dbg_arrays[buf_id].size_in_dwords =
+		    BYTES_TO_DWORDS(buf_array[buf_id].length);
+	}
+
+	return DBG_STATUS_OK;
+}
+
+enum dbg_status qed_dbg_grc_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+					      struct qed_ptt *p_ptt,
+					      u32 *buf_size)
+{
+	enum dbg_status status = qed_dbg_dev_init(p_hwfn, p_ptt);
+
+	*buf_size = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	if (!s_dbg_arrays[BIN_BUF_DBG_MODE_TREE].ptr ||
+	    !s_dbg_arrays[BIN_BUF_DBG_DUMP_REG].ptr ||
+	    !s_dbg_arrays[BIN_BUF_DBG_DUMP_MEM].ptr ||
+	    !s_dbg_arrays[BIN_BUF_DBG_ATTN_BLOCKS].ptr ||
+	    !s_dbg_arrays[BIN_BUF_DBG_ATTN_REGS].ptr)
+		return DBG_STATUS_DBG_ARRAY_NOT_SET;
+	return qed_grc_dump(p_hwfn, p_ptt, NULL, false, buf_size);
+}
+
+enum dbg_status qed_dbg_grc_dump(struct qed_hwfn *p_hwfn,
+				 struct qed_ptt *p_ptt,
+				 u32 *dump_buf,
+				 u32 buf_size_in_dwords,
+				 u32 *num_dumped_dwords)
+{
+	u32 needed_buf_size_in_dwords;
+	enum dbg_status status;
+
+	status = qed_dbg_grc_get_dump_buf_size(p_hwfn, p_ptt,
+					       &needed_buf_size_in_dwords);
+
+	*num_dumped_dwords = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	if (buf_size_in_dwords < needed_buf_size_in_dwords)
+		return DBG_STATUS_DUMP_BUF_TOO_SMALL;
+
+	/* GRC Dump */
+	status = qed_grc_dump(p_hwfn, p_ptt, dump_buf, true, num_dumped_dwords);
+
+	/* Clear all GRC params */
+	qed_dbg_grc_clear_params(p_hwfn);
+	return status;
+}
+
+enum dbg_status qed_dbg_idle_chk_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						   struct qed_ptt *p_ptt,
+						   u32 *buf_size)
+{
+	enum dbg_status status = qed_dbg_dev_init(p_hwfn, p_ptt);
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
+
+	*buf_size = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	if (!s_dbg_arrays[BIN_BUF_DBG_MODE_TREE].ptr ||
+	    !s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_REGS].ptr ||
+	    !s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_IMMS].ptr ||
+	    !s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_RULES].ptr)
+		return DBG_STATUS_DBG_ARRAY_NOT_SET;
+	if (!dev_data->idle_chk.buf_size_set) {
+		dev_data->idle_chk.buf_size = qed_idle_chk_dump(p_hwfn,
+								p_ptt,
+								NULL, false);
+		dev_data->idle_chk.buf_size_set = true;
+	}
+
+	*buf_size = dev_data->idle_chk.buf_size;
+	return DBG_STATUS_OK;
+}
+
+enum dbg_status qed_dbg_idle_chk_dump(struct qed_hwfn *p_hwfn,
+				      struct qed_ptt *p_ptt,
+				      u32 *dump_buf,
+				      u32 buf_size_in_dwords,
+				      u32 *num_dumped_dwords)
+{
+	u32 needed_buf_size_in_dwords;
+	enum dbg_status status;
+
+	status = qed_dbg_idle_chk_get_dump_buf_size(p_hwfn, p_ptt,
+						    &needed_buf_size_in_dwords);
+
+	*num_dumped_dwords = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	if (buf_size_in_dwords < needed_buf_size_in_dwords)
+		return DBG_STATUS_DUMP_BUF_TOO_SMALL;
+
+	/* Update reset state */
+	qed_update_blocks_reset_state(p_hwfn, p_ptt);
+
+	/* Idle Check Dump */
+	*num_dumped_dwords = qed_idle_chk_dump(p_hwfn, p_ptt, dump_buf, true);
+	return DBG_STATUS_OK;
+}
+
+enum dbg_status qed_dbg_mcp_trace_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						    struct qed_ptt *p_ptt,
+						    u32 *buf_size)
+{
+	enum dbg_status status = qed_dbg_dev_init(p_hwfn, p_ptt);
+
+	*buf_size = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	return qed_mcp_trace_dump(p_hwfn, p_ptt, NULL, false, buf_size);
+}
+
+enum dbg_status qed_dbg_mcp_trace_dump(struct qed_hwfn *p_hwfn,
+				       struct qed_ptt *p_ptt,
+				       u32 *dump_buf,
+				       u32 buf_size_in_dwords,
+				       u32 *num_dumped_dwords)
+{
+	u32 needed_buf_size_in_dwords;
+	enum dbg_status status;
+
+	status = qed_dbg_mcp_trace_get_dump_buf_size(p_hwfn, p_ptt,
+						&needed_buf_size_in_dwords);
+
+	if (status != DBG_STATUS_OK)
+		return status;
+	if (buf_size_in_dwords < needed_buf_size_in_dwords)
+		return DBG_STATUS_DUMP_BUF_TOO_SMALL;
+
+	/* Update reset state */
+	qed_update_blocks_reset_state(p_hwfn, p_ptt);
+
+	/* Perform dump */
+	return qed_mcp_trace_dump(p_hwfn,
+				  p_ptt, dump_buf, true, num_dumped_dwords);
+}
+
+enum dbg_status qed_dbg_reg_fifo_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						   struct qed_ptt *p_ptt,
+						   u32 *buf_size)
+{
+	enum dbg_status status = qed_dbg_dev_init(p_hwfn, p_ptt);
+
+	*buf_size = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	return qed_reg_fifo_dump(p_hwfn, p_ptt, NULL, false, buf_size);
+}
+
+enum dbg_status qed_dbg_reg_fifo_dump(struct qed_hwfn *p_hwfn,
+				      struct qed_ptt *p_ptt,
+				      u32 *dump_buf,
+				      u32 buf_size_in_dwords,
+				      u32 *num_dumped_dwords)
+{
+	u32 needed_buf_size_in_dwords;
+	enum dbg_status status;
+
+	status = qed_dbg_reg_fifo_get_dump_buf_size(p_hwfn, p_ptt,
+						    &needed_buf_size_in_dwords);
+
+	*num_dumped_dwords = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	if (buf_size_in_dwords < needed_buf_size_in_dwords)
+		return DBG_STATUS_DUMP_BUF_TOO_SMALL;
+
+	/* Update reset state */
+	qed_update_blocks_reset_state(p_hwfn, p_ptt);
+	return qed_reg_fifo_dump(p_hwfn,
+				 p_ptt, dump_buf, true, num_dumped_dwords);
+}
+
+enum dbg_status qed_dbg_igu_fifo_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						   struct qed_ptt *p_ptt,
+						   u32 *buf_size)
+{
+	enum dbg_status status = qed_dbg_dev_init(p_hwfn, p_ptt);
+
+	*buf_size = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	return qed_igu_fifo_dump(p_hwfn, p_ptt, NULL, false, buf_size);
+}
+
+enum dbg_status qed_dbg_igu_fifo_dump(struct qed_hwfn *p_hwfn,
+				      struct qed_ptt *p_ptt,
+				      u32 *dump_buf,
+				      u32 buf_size_in_dwords,
+				      u32 *num_dumped_dwords)
+{
+	u32 needed_buf_size_in_dwords;
+	enum dbg_status status;
+
+	status = qed_dbg_igu_fifo_get_dump_buf_size(p_hwfn, p_ptt,
+						    &needed_buf_size_in_dwords);
+
+	*num_dumped_dwords = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	if (buf_size_in_dwords < needed_buf_size_in_dwords)
+		return DBG_STATUS_DUMP_BUF_TOO_SMALL;
+
+	/* Update reset state */
+	qed_update_blocks_reset_state(p_hwfn, p_ptt);
+	return qed_igu_fifo_dump(p_hwfn,
+				 p_ptt, dump_buf, true, num_dumped_dwords);
+}
+
+enum dbg_status
+qed_dbg_protection_override_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+					      struct qed_ptt *p_ptt,
+					      u32 *buf_size)
+{
+	enum dbg_status status = qed_dbg_dev_init(p_hwfn, p_ptt);
+
+	*buf_size = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	return qed_protection_override_dump(p_hwfn,
+					    p_ptt, NULL, false, buf_size);
+}
+
+enum dbg_status qed_dbg_protection_override_dump(struct qed_hwfn *p_hwfn,
+						 struct qed_ptt *p_ptt,
+						 u32 *dump_buf,
+						 u32 buf_size_in_dwords,
+						 u32 *num_dumped_dwords)
+{
+	u32 needed_buf_size_in_dwords;
+	enum dbg_status status;
+
+	status = qed_dbg_protection_override_get_dump_buf_size(p_hwfn, p_ptt,
+						&needed_buf_size_in_dwords);
+
+	*num_dumped_dwords = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	if (buf_size_in_dwords < needed_buf_size_in_dwords)
+		return DBG_STATUS_DUMP_BUF_TOO_SMALL;
+
+	/* Update reset state */
+	qed_update_blocks_reset_state(p_hwfn, p_ptt);
+	return qed_protection_override_dump(p_hwfn,
+					    p_ptt,
+					    dump_buf, true, num_dumped_dwords);
+}
+
+enum dbg_status qed_dbg_fw_asserts_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						     struct qed_ptt *p_ptt,
+						     u32 *buf_size)
+{
+	enum dbg_status status = qed_dbg_dev_init(p_hwfn, p_ptt);
+
+	*buf_size = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+
+	/* Update reset state */
+	qed_update_blocks_reset_state(p_hwfn, p_ptt);
+	*buf_size = qed_fw_asserts_dump(p_hwfn, p_ptt, NULL, false);
+	return DBG_STATUS_OK;
+}
+
+enum dbg_status qed_dbg_fw_asserts_dump(struct qed_hwfn *p_hwfn,
+					struct qed_ptt *p_ptt,
+					u32 *dump_buf,
+					u32 buf_size_in_dwords,
+					u32 *num_dumped_dwords)
+{
+	u32 needed_buf_size_in_dwords;
+	enum dbg_status status;
+
+	status = qed_dbg_fw_asserts_get_dump_buf_size(p_hwfn, p_ptt,
+						&needed_buf_size_in_dwords);
+
+	*num_dumped_dwords = 0;
+	if (status != DBG_STATUS_OK)
+		return status;
+	if (buf_size_in_dwords < needed_buf_size_in_dwords)
+		return DBG_STATUS_DUMP_BUF_TOO_SMALL;
+
+	*num_dumped_dwords = qed_fw_asserts_dump(p_hwfn, p_ptt, dump_buf, true);
+	return DBG_STATUS_OK;
+}
+
+/******************************* Data Types **********************************/
+
+struct mcp_trace_format {
+	u32 data;
+#define MCP_TRACE_FORMAT_MODULE_MASK	0x0000ffff
+#define MCP_TRACE_FORMAT_MODULE_SHIFT	0
+#define MCP_TRACE_FORMAT_LEVEL_MASK	0x00030000
+#define MCP_TRACE_FORMAT_LEVEL_SHIFT	16
+#define MCP_TRACE_FORMAT_P1_SIZE_MASK	0x000c0000
+#define MCP_TRACE_FORMAT_P1_SIZE_SHIFT	18
+#define MCP_TRACE_FORMAT_P2_SIZE_MASK	0x00300000
+#define MCP_TRACE_FORMAT_P2_SIZE_SHIFT	20
+#define MCP_TRACE_FORMAT_P3_SIZE_MASK	0x00c00000
+#define MCP_TRACE_FORMAT_P3_SIZE_SHIFT	22
+#define MCP_TRACE_FORMAT_LEN_MASK	0xff000000
+#define MCP_TRACE_FORMAT_LEN_SHIFT	24
+	char *format_str;
+};
+
+struct mcp_trace_meta {
+	u32 modules_num;
+	char **modules;
+	u32 formats_num;
+	struct mcp_trace_format *formats;
+};
+
+/* Reg fifo element */
+struct reg_fifo_element {
+	u64 data;
+#define REG_FIFO_ELEMENT_ADDRESS_SHIFT		0
+#define REG_FIFO_ELEMENT_ADDRESS_MASK		0x7fffff
+#define REG_FIFO_ELEMENT_ACCESS_SHIFT		23
+#define REG_FIFO_ELEMENT_ACCESS_MASK		0x1
+#define REG_FIFO_ELEMENT_PF_SHIFT		24
+#define REG_FIFO_ELEMENT_PF_MASK		0xf
+#define REG_FIFO_ELEMENT_VF_SHIFT		28
+#define REG_FIFO_ELEMENT_VF_MASK		0xff
+#define REG_FIFO_ELEMENT_PORT_SHIFT		36
+#define REG_FIFO_ELEMENT_PORT_MASK		0x3
+#define REG_FIFO_ELEMENT_PRIVILEGE_SHIFT	38
+#define REG_FIFO_ELEMENT_PRIVILEGE_MASK		0x3
+#define REG_FIFO_ELEMENT_PROTECTION_SHIFT	40
+#define REG_FIFO_ELEMENT_PROTECTION_MASK	0x7
+#define REG_FIFO_ELEMENT_MASTER_SHIFT		43
+#define REG_FIFO_ELEMENT_MASTER_MASK		0xf
+#define REG_FIFO_ELEMENT_ERROR_SHIFT		47
+#define REG_FIFO_ELEMENT_ERROR_MASK		0x1f
+};
+
+/* IGU fifo element */
+struct igu_fifo_element {
+	u32 dword0;
+#define IGU_FIFO_ELEMENT_DWORD0_FID_SHIFT		0
+#define IGU_FIFO_ELEMENT_DWORD0_FID_MASK		0xff
+#define IGU_FIFO_ELEMENT_DWORD0_IS_PF_SHIFT		8
+#define IGU_FIFO_ELEMENT_DWORD0_IS_PF_MASK		0x1
+#define IGU_FIFO_ELEMENT_DWORD0_SOURCE_SHIFT		9
+#define IGU_FIFO_ELEMENT_DWORD0_SOURCE_MASK		0xf
+#define IGU_FIFO_ELEMENT_DWORD0_ERR_TYPE_SHIFT		13
+#define IGU_FIFO_ELEMENT_DWORD0_ERR_TYPE_MASK		0xf
+#define IGU_FIFO_ELEMENT_DWORD0_CMD_ADDR_SHIFT		17
+#define IGU_FIFO_ELEMENT_DWORD0_CMD_ADDR_MASK		0x7fff
+	u32 dword1;
+	u32 dword2;
+#define IGU_FIFO_ELEMENT_DWORD12_IS_WR_CMD_SHIFT	0
+#define IGU_FIFO_ELEMENT_DWORD12_IS_WR_CMD_MASK		0x1
+#define IGU_FIFO_ELEMENT_DWORD12_WR_DATA_SHIFT		1
+#define IGU_FIFO_ELEMENT_DWORD12_WR_DATA_MASK		0xffffffff
+	u32 reserved;
+};
+
+struct igu_fifo_wr_data {
+	u32 data;
+#define IGU_FIFO_WR_DATA_PROD_CONS_SHIFT		0
+#define IGU_FIFO_WR_DATA_PROD_CONS_MASK			0xffffff
+#define IGU_FIFO_WR_DATA_UPDATE_FLAG_SHIFT		24
+#define IGU_FIFO_WR_DATA_UPDATE_FLAG_MASK		0x1
+#define IGU_FIFO_WR_DATA_EN_DIS_INT_FOR_SB_SHIFT	25
+#define IGU_FIFO_WR_DATA_EN_DIS_INT_FOR_SB_MASK		0x3
+#define IGU_FIFO_WR_DATA_SEGMENT_SHIFT			27
+#define IGU_FIFO_WR_DATA_SEGMENT_MASK			0x1
+#define IGU_FIFO_WR_DATA_TIMER_MASK_SHIFT		28
+#define IGU_FIFO_WR_DATA_TIMER_MASK_MASK		0x1
+#define IGU_FIFO_WR_DATA_CMD_TYPE_SHIFT			31
+#define IGU_FIFO_WR_DATA_CMD_TYPE_MASK			0x1
+};
+
+struct igu_fifo_cleanup_wr_data {
+	u32 data;
+#define IGU_FIFO_CLEANUP_WR_DATA_RESERVED_SHIFT		0
+#define IGU_FIFO_CLEANUP_WR_DATA_RESERVED_MASK		0x7ffffff
+#define IGU_FIFO_CLEANUP_WR_DATA_CLEANUP_VAL_SHIFT	27
+#define IGU_FIFO_CLEANUP_WR_DATA_CLEANUP_VAL_MASK	0x1
+#define IGU_FIFO_CLEANUP_WR_DATA_CLEANUP_TYPE_SHIFT	28
+#define IGU_FIFO_CLEANUP_WR_DATA_CLEANUP_TYPE_MASK	0x7
+#define IGU_FIFO_CLEANUP_WR_DATA_CMD_TYPE_SHIFT		31
+#define IGU_FIFO_CLEANUP_WR_DATA_CMD_TYPE_MASK		0x1
+};
+
+/* Protection override element */
+struct protection_override_element {
+	u64 data;
+#define PROTECTION_OVERRIDE_ELEMENT_ADDRESS_SHIFT		0
+#define PROTECTION_OVERRIDE_ELEMENT_ADDRESS_MASK		0x7fffff
+#define PROTECTION_OVERRIDE_ELEMENT_WINDOW_SIZE_SHIFT		23
+#define PROTECTION_OVERRIDE_ELEMENT_WINDOW_SIZE_MASK		0xffffff
+#define PROTECTION_OVERRIDE_ELEMENT_READ_SHIFT			47
+#define PROTECTION_OVERRIDE_ELEMENT_READ_MASK			0x1
+#define PROTECTION_OVERRIDE_ELEMENT_WRITE_SHIFT			48
+#define PROTECTION_OVERRIDE_ELEMENT_WRITE_MASK			0x1
+#define PROTECTION_OVERRIDE_ELEMENT_READ_PROTECTION_SHIFT	49
+#define PROTECTION_OVERRIDE_ELEMENT_READ_PROTECTION_MASK	0x7
+#define PROTECTION_OVERRIDE_ELEMENT_WRITE_PROTECTION_SHIFT	52
+#define PROTECTION_OVERRIDE_ELEMENT_WRITE_PROTECTION_MASK	0x7
+};
+
+enum igu_fifo_sources {
+	IGU_SRC_PXP0,
+	IGU_SRC_PXP1,
+	IGU_SRC_PXP2,
+	IGU_SRC_PXP3,
+	IGU_SRC_PXP4,
+	IGU_SRC_PXP5,
+	IGU_SRC_PXP6,
+	IGU_SRC_PXP7,
+	IGU_SRC_CAU,
+	IGU_SRC_ATTN,
+	IGU_SRC_GRC
+};
+
+enum igu_fifo_addr_types {
+	IGU_ADDR_TYPE_MSIX_MEM,
+	IGU_ADDR_TYPE_WRITE_PBA,
+	IGU_ADDR_TYPE_WRITE_INT_ACK,
+	IGU_ADDR_TYPE_WRITE_ATTN_BITS,
+	IGU_ADDR_TYPE_READ_INT,
+	IGU_ADDR_TYPE_WRITE_PROD_UPDATE,
+	IGU_ADDR_TYPE_RESERVED
+};
+
+struct igu_fifo_addr_data {
+	u16 start_addr;
+	u16 end_addr;
+	char *desc;
+	char *vf_desc;
+	enum igu_fifo_addr_types type;
+};
+
+/******************************** Constants **********************************/
+
+#define MAX_MSG_LEN				1024
+#define MCP_TRACE_MAX_MODULE_LEN		8
+#define MCP_TRACE_FORMAT_MAX_PARAMS		3
+#define MCP_TRACE_FORMAT_PARAM_WIDTH \
+	(MCP_TRACE_FORMAT_P2_SIZE_SHIFT - MCP_TRACE_FORMAT_P1_SIZE_SHIFT)
+#define REG_FIFO_ELEMENT_ADDR_FACTOR		4
+#define REG_FIFO_ELEMENT_IS_PF_VF_VAL		127
+#define PROTECTION_OVERRIDE_ELEMENT_ADDR_FACTOR	4
+
+/********************************* Macros ************************************/
+
+#define BYTES_TO_DWORDS(bytes)			((bytes) / BYTES_IN_DWORD)
+
+/***************************** Constant Arrays *******************************/
+
+/* Status string array */
+static const char * const s_status_str[] = {
+	"Operation completed successfully",
+	"Debug application version wasn't set",
+	"Unsupported debug application version",
+	"The debug block wasn't reset since the last recording",
+	"Invalid arguments",
+	"The debug output was already set",
+	"Invalid PCI buffer size",
+	"PCI buffer allocation failed",
+	"A PCI buffer wasn't allocated",
+	"Too many inputs were enabled. Enabled less inputs, or set 'unifyInputs' to true",
+	"GRC/Timestamp input overlap in cycle dword 0",
+	"Cannot record Storm data since the entire recording cycle is used by HW",
+	"The Storm was already enabled",
+	"The specified Storm wasn't enabled",
+	"The block was already enabled",
+	"The specified block wasn't enabled",
+	"No input was enabled for recording",
+	"Filters and triggers are not allowed when recording in 64b units",
+	"The filter was already enabled",
+	"The trigger was already enabled",
+	"The trigger wasn't enabled",
+	"A constraint can be added only after a filter was enabled or a trigger state was added",
+	"Cannot add more than 3 trigger states",
+	"Cannot add more than 4 constraints per filter or trigger state",
+	"The recording wasn't started",
+	"A trigger was configured, but it didn't trigger",
+	"No data was recorded",
+	"Dump buffer is too small",
+	"Dumped data is not aligned to chunks",
+	"Unknown chip",
+	"Failed allocating virtual memory",
+	"The input block is in reset",
+	"Invalid MCP trace signature found in NVRAM",
+	"Invalid bundle ID found in NVRAM",
+	"Failed getting NVRAM image",
+	"NVRAM image is not dword-aligned",
+	"Failed reading from NVRAM",
+	"Idle check parsing failed",
+	"MCP Trace data is corrupt",
+	"Dump doesn't contain meta data - it must be provided in an image file",
+	"Failed to halt MCP",
+	"Failed to resume MCP after halt",
+	"DMAE transaction failed",
+	"Failed to empty SEMI sync FIFO",
+	"IGU FIFO data is corrupt",
+	"MCP failed to mask parities",
+	"FW Asserts parsing failed",
+	"GRC FIFO data is corrupt",
+	"Protection Override data is corrupt",
+	"Debug arrays were not set (when using binary files, dbg_set_bin_ptr must be called)",
+	"When a block is filtered, no other blocks can be recorded unless inputs are unified (due to a HW bug)"
+};
+
+/* Idle check severity names array */
+static const char * const s_idle_chk_severity_str[] = {
+	"Error",
+	"Error if no traffic",
+	"Warning"
+};
+
+/* MCP Trace level names array */
+static const char * const s_mcp_trace_level_str[] = {
+	"ERROR",
+	"TRACE",
+	"DEBUG"
+};
+
+/* Parsing strings */
+static const char * const s_access_strs[] = {
+	"read",
+	"write"
+};
+
+static const char * const s_privilege_strs[] = {
+	"VF",
+	"PDA",
+	"HV",
+	"UA"
+};
+
+static const char * const s_protection_strs[] = {
+	"(default)",
+	"(default)",
+	"(default)",
+	"(default)",
+	"override VF",
+	"override PDA",
+	"override HV",
+	"override UA"
+};
+
+static const char * const s_master_strs[] = {
+	"???",
+	"pxp",
+	"mcp",
+	"msdm",
+	"psdm",
+	"ysdm",
+	"usdm",
+	"tsdm",
+	"xsdm",
+	"dbu",
+	"dmae",
+	"???",
+	"???",
+	"???",
+	"???",
+	"???"
+};
+
+static const char * const s_reg_fifo_error_strs[] = {
+	"grc timeout",
+	"address doesn't belong to any block",
+	"reserved address in block or write to read-only address",
+	"privilege/protection mismatch",
+	"path isolation error"
+};
+
+static const char * const s_igu_fifo_source_strs[] = {
+	"TSTORM",
+	"MSTORM",
+	"USTORM",
+	"XSTORM",
+	"YSTORM",
+	"PSTORM",
+	"PCIE",
+	"NIG_QM_PBF",
+	"CAU",
+	"ATTN",
+	"GRC",
+};
+
+static const char * const s_igu_fifo_error_strs[] = {
+	"no error",
+	"length error",
+	"function disabled",
+	"VF sent command to attnetion address",
+	"host sent prod update command",
+	"read of during interrupt register while in MIMD mode",
+	"access to PXP BAR reserved address",
+	"producer update command to attention index",
+	"unknown error",
+	"SB index not valid",
+	"SB relative index and FID not found",
+	"FID not match",
+	"command with error flag asserted (PCI error or CAU discard)",
+	"VF sent cleanup and RF cleanup is disabled",
+	"cleanup command on type bigger than 4"
+};
+
+/* IGU FIFO address data */
+static const struct igu_fifo_addr_data s_igu_fifo_addr_data[] = {
+	{0x0, 0x101, "MSI-X Memory", NULL, IGU_ADDR_TYPE_MSIX_MEM},
+	{0x102, 0x1ff, "reserved", NULL, IGU_ADDR_TYPE_RESERVED},
+	{0x200, 0x200, "Write PBA[0:63]", NULL, IGU_ADDR_TYPE_WRITE_PBA},
+	{0x201, 0x201, "Write PBA[64:127]", "reserved",
+	 IGU_ADDR_TYPE_WRITE_PBA},
+	{0x202, 0x202, "Write PBA[128]", "reserved", IGU_ADDR_TYPE_WRITE_PBA},
+	{0x203, 0x3ff, "reserved", NULL, IGU_ADDR_TYPE_RESERVED},
+	{0x400, 0x5ef, "Write interrupt acknowledgment", NULL,
+	 IGU_ADDR_TYPE_WRITE_INT_ACK},
+	{0x5f0, 0x5f0, "Attention bits update", NULL,
+	 IGU_ADDR_TYPE_WRITE_ATTN_BITS},
+	{0x5f1, 0x5f1, "Attention bits set", NULL,
+	 IGU_ADDR_TYPE_WRITE_ATTN_BITS},
+	{0x5f2, 0x5f2, "Attention bits clear", NULL,
+	 IGU_ADDR_TYPE_WRITE_ATTN_BITS},
+	{0x5f3, 0x5f3, "Read interrupt 0:63 with mask", NULL,
+	 IGU_ADDR_TYPE_READ_INT},
+	{0x5f4, 0x5f4, "Read interrupt 0:31 with mask", NULL,
+	 IGU_ADDR_TYPE_READ_INT},
+	{0x5f5, 0x5f5, "Read interrupt 32:63 with mask", NULL,
+	 IGU_ADDR_TYPE_READ_INT},
+	{0x5f6, 0x5f6, "Read interrupt 0:63 without mask", NULL,
+	 IGU_ADDR_TYPE_READ_INT},
+	{0x5f7, 0x5ff, "reserved", NULL, IGU_ADDR_TYPE_RESERVED},
+	{0x600, 0x7ff, "Producer update", NULL, IGU_ADDR_TYPE_WRITE_PROD_UPDATE}
+};
+
+/******************************** Variables **********************************/
+
+/* MCP Trace meta data - used in case the dump doesn't contain the meta data
+ * (e.g. due to no NVRAM access).
+ */
+static struct dbg_array s_mcp_trace_meta = { NULL, 0 };
+
+/* Temporary buffer, used for print size calculations */
+static char s_temp_buf[MAX_MSG_LEN];
+
+/***************************** Public Functions *******************************/
+
+enum dbg_status qed_dbg_user_set_bin_ptr(const u8 * const bin_ptr)
+{
+	/* Convert binary data to debug arrays */
+	u32 num_of_buffers = *(u32 *)bin_ptr;
+	struct bin_buffer_hdr *buf_array;
+	u8 buf_id;
+
+	buf_array = (struct bin_buffer_hdr *)((u32 *)bin_ptr + 1);
+
+	for (buf_id = 0; buf_id < num_of_buffers; buf_id++) {
+		s_dbg_arrays[buf_id].ptr =
+		    (u32 *)(bin_ptr + buf_array[buf_id].offset);
+		s_dbg_arrays[buf_id].size_in_dwords =
+		    BYTES_TO_DWORDS(buf_array[buf_id].length);
+	}
+
+	return DBG_STATUS_OK;
+}
+
+static u32 qed_cyclic_add(u32 a, u32 b, u32 size)
+{
+	return (a + b) % size;
+}
+
+static u32 qed_cyclic_sub(u32 a, u32 b, u32 size)
+{
+	return (size + a - b) % size;
+}
+
+/* Reads the specified number of bytes from the specified cyclic buffer (up to 4
+ * bytes) and returns them as a dword value. the specified buffer offset is
+ * updated.
+ */
+static u32 qed_read_from_cyclic_buf(void *buf,
+				    u32 *offset,
+				    u32 buf_size, u8 num_bytes_to_read)
+{
+	u8 *bytes_buf = (u8 *)buf;
+	u8 *val_ptr;
+	u32 val = 0;
+	u8 i;
+
+	val_ptr = (u8 *)&val;
+
+	for (i = 0; i < num_bytes_to_read; i++) {
+		val_ptr[i] = bytes_buf[*offset];
+		*offset = qed_cyclic_add(*offset, 1, buf_size);
+	}
+
+	return val;
+}
+
+/* Reads and returns the next byte from the specified buffer.
+ * The specified buffer offset is updated.
+ */
+static u8 qed_read_byte_from_buf(void *buf, u32 *offset)
+{
+	return ((u8 *)buf)[(*offset)++];
+}
+
+/* Reads and returns the next dword from the specified buffer.
+ * The specified buffer offset is updated.
+ */
+static u32 qed_read_dword_from_buf(void *buf, u32 *offset)
+{
+	u32 dword_val = *(u32 *)&((u8 *)buf)[*offset];
+
+	*offset += 4;
+	return dword_val;
+}
+
+/* Reads the next string from the specified buffer, and copies it to the
+ * specified pointer. The specified buffer offset is updated.
+ */
+static void qed_read_str_from_buf(void *buf, u32 *offset, u32 size, char *dest)
+{
+	const char *source_str = &((const char *)buf)[*offset];
+
+	strncpy(dest, source_str, size);
+	dest[size - 1] = '\0';
+	*offset += size;
+}
+
+/* Returns a pointer to the specified offset (in bytes) of the specified buffer.
+ * If the specified buffer in NULL, a temporary buffer pointer is returned.
+ */
+static char *qed_get_buf_ptr(void *buf, u32 offset)
+{
+	return buf ? (char *)buf + offset : s_temp_buf;
+}
+
+/* Reads a param from the specified buffer. Returns the number of dwords read.
+ * If the returned str_param is NULL, the param is numeric and its value is
+ * returned in num_param.
+ * Otheriwise, the param is a string and its pointer is returned in str_param.
+ */
+static u32 qed_read_param(u32 *dump_buf,
+			  const char **param_name,
+			  const char **param_str_val, u32 *param_num_val)
+{
+	char *char_buf = (char *)dump_buf;
+	u32 offset = 0; /* In bytes */
+
+	/* Extract param name */
+	*param_name = char_buf;
+	offset += strlen(*param_name) + 1;
+
+	/* Check param type */
+	if (*(char_buf + offset++)) {
+		/* String param */
+		*param_str_val = char_buf + offset;
+		offset += strlen(*param_str_val) + 1;
+		if (offset & 0x3)
+			offset += (4 - (offset & 0x3));
+	} else {
+		/* Numeric param */
+		*param_str_val = NULL;
+		if (offset & 0x3)
+			offset += (4 - (offset & 0x3));
+		*param_num_val = *(u32 *)(char_buf + offset);
+		offset += 4;
+	}
+
+	return offset / 4;
+}
+
+/* Reads a section header from the specified buffer.
+ * Returns the number of dwords read.
+ */
+static u32 qed_read_section_hdr(u32 *dump_buf,
+				const char **section_name,
+				u32 *num_section_params)
+{
+	const char *param_str_val;
+
+	return qed_read_param(dump_buf,
+			      section_name, &param_str_val, num_section_params);
+}
+
+/* Reads section params from the specified buffer and prints them to the results
+ * buffer. Returns the number of dwords read.
+ */
+static u32 qed_print_section_params(u32 *dump_buf,
+				    u32 num_section_params,
+				    char *results_buf, u32 *num_chars_printed)
+{
+	u32 i, dump_offset = 0, results_offset = 0;
+
+	for (i = 0; i < num_section_params; i++) {
+		const char *param_name;
+		const char *param_str_val;
+		u32 param_num_val = 0;
+
+		dump_offset += qed_read_param(dump_buf + dump_offset,
+					      &param_name,
+					      &param_str_val, &param_num_val);
+		if (param_str_val)
+			/* String param */
+			results_offset +=
+				sprintf(qed_get_buf_ptr(results_buf,
+							results_offset),
+					"%s: %s\n", param_name, param_str_val);
+		else if (strcmp(param_name, "fw-timestamp"))
+			/* Numeric param */
+			results_offset +=
+				sprintf(qed_get_buf_ptr(results_buf,
+							results_offset),
+					"%s: %d\n", param_name, param_num_val);
+	}
+
+	results_offset +=
+	    sprintf(qed_get_buf_ptr(results_buf, results_offset), "\n");
+	*num_chars_printed = results_offset;
+	return dump_offset;
+}
+
+const char *qed_dbg_get_status_str(enum dbg_status status)
+{
+	return (status <
+		MAX_DBG_STATUS) ? s_status_str[status] : "Invalid debug status";
+}
+
+/* Parses the idle check rules and returns the number of characters printed.
+ * In case of parsing error, returns 0.
+ */
+static u32 qed_parse_idle_chk_dump_rules(struct qed_hwfn *p_hwfn,
+					 u32 *dump_buf,
+					 u32 *dump_buf_end,
+					 u32 num_rules,
+					 bool print_fw_idle_chk,
+					 char *results_buf,
+					 u32 *num_errors, u32 *num_warnings)
+{
+	u32 rule_idx, results_offset = 0; /* Offset in results_buf in bytes */
+	u16 i, j;
+
+	*num_errors = 0;
+	*num_warnings = 0;
+
+	/* Go over dumped results */
+	for (rule_idx = 0; rule_idx < num_rules && dump_buf < dump_buf_end;
+	     rule_idx++) {
+		const struct dbg_idle_chk_rule_parsing_data *rule_parsing_data;
+		struct dbg_idle_chk_result_hdr *hdr;
+		const char *parsing_str;
+		u32 parsing_str_offset;
+		const char *lsi_msg;
+		u8 curr_reg_id = 0;
+		bool has_fw_msg;
+
+		hdr = (struct dbg_idle_chk_result_hdr *)dump_buf;
+		rule_parsing_data =
+			(const struct dbg_idle_chk_rule_parsing_data *)
+			&s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_PARSING_DATA].
+			ptr[hdr->rule_id];
+		parsing_str_offset =
+			GET_FIELD(rule_parsing_data->data,
+				  DBG_IDLE_CHK_RULE_PARSING_DATA_STR_OFFSET);
+		has_fw_msg =
+			GET_FIELD(rule_parsing_data->data,
+				DBG_IDLE_CHK_RULE_PARSING_DATA_HAS_FW_MSG) > 0;
+		parsing_str = &((const char *)
+				s_dbg_arrays[BIN_BUF_DBG_PARSING_STRINGS].ptr)
+				[parsing_str_offset];
+		lsi_msg = parsing_str;
+
+		if (hdr->severity >= MAX_DBG_IDLE_CHK_SEVERITY_TYPES)
+			return 0;
+
+		/* Skip rule header */
+		dump_buf += (sizeof(struct dbg_idle_chk_result_hdr) / 4);
+
+		/* Update errors/warnings count */
+		if (hdr->severity == IDLE_CHK_SEVERITY_ERROR ||
+		    hdr->severity == IDLE_CHK_SEVERITY_ERROR_NO_TRAFFIC)
+			(*num_errors)++;
+		else
+			(*num_warnings)++;
+
+		/* Print rule severity */
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset), "%s: ",
+			    s_idle_chk_severity_str[hdr->severity]);
+
+		/* Print rule message */
+		if (has_fw_msg)
+			parsing_str += strlen(parsing_str) + 1;
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset), "%s.",
+			    has_fw_msg &&
+			    print_fw_idle_chk ? parsing_str : lsi_msg);
+		parsing_str += strlen(parsing_str) + 1;
+
+		/* Print register values */
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset), " Registers:");
+		for (i = 0;
+		     i < hdr->num_dumped_cond_regs + hdr->num_dumped_info_regs;
+		     i++) {
+			struct dbg_idle_chk_result_reg_hdr *reg_hdr
+			    = (struct dbg_idle_chk_result_reg_hdr *)
+			    dump_buf;
+			bool is_mem =
+				GET_FIELD(reg_hdr->data,
+					  DBG_IDLE_CHK_RESULT_REG_HDR_IS_MEM);
+			u8 reg_id =
+				GET_FIELD(reg_hdr->data,
+					  DBG_IDLE_CHK_RESULT_REG_HDR_REG_ID);
+
+			/* Skip reg header */
+			dump_buf +=
+			    (sizeof(struct dbg_idle_chk_result_reg_hdr) / 4);
+
+			/* Skip register names until the required reg_id is
+			 * reached.
+			 */
+			for (; reg_id > curr_reg_id;
+			     curr_reg_id++,
+			     parsing_str += strlen(parsing_str) + 1);
+
+			results_offset +=
+			    sprintf(qed_get_buf_ptr(results_buf,
+						    results_offset), " %s",
+				    parsing_str);
+			if (i < hdr->num_dumped_cond_regs && is_mem)
+				results_offset +=
+				    sprintf(qed_get_buf_ptr(results_buf,
+							    results_offset),
+					    "[%d]", hdr->mem_entry_id +
+					    reg_hdr->start_entry);
+			results_offset +=
+			    sprintf(qed_get_buf_ptr(results_buf,
+						    results_offset), "=");
+			for (j = 0; j < reg_hdr->size; j++, dump_buf++) {
+				results_offset +=
+				    sprintf(qed_get_buf_ptr(results_buf,
+							    results_offset),
+					    "0x%x", *dump_buf);
+				if (j < reg_hdr->size - 1)
+					results_offset +=
+					    sprintf(qed_get_buf_ptr
+						    (results_buf,
+						     results_offset), ",");
+			}
+		}
+
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf, results_offset), "\n");
+	}
+
+	/* Check if end of dump buffer was exceeded */
+	if (dump_buf > dump_buf_end)
+		return 0;
+	return results_offset;
+}
+
+/* Parses an idle check dump buffer.
+ * If result_buf is not NULL, the idle check results are printed to it.
+ * In any case, the required results buffer size is assigned to
+ * parsed_results_bytes.
+ * The parsing status is returned.
+ */
+static enum dbg_status qed_parse_idle_chk_dump(struct qed_hwfn *p_hwfn,
+					       u32 *dump_buf,
+					       u32 num_dumped_dwords,
+					       char *results_buf,
+					       u32 *parsed_results_bytes,
+					       u32 *num_errors,
+					       u32 *num_warnings)
+{
+	const char *section_name, *param_name, *param_str_val;
+	u32 *dump_buf_end = dump_buf + num_dumped_dwords;
+	u32 num_section_params = 0, num_rules;
+	u32 results_offset = 0;	/* Offset in results_buf in bytes */
+
+	*parsed_results_bytes = 0;
+	*num_errors = 0;
+	*num_warnings = 0;
+	if (!s_dbg_arrays[BIN_BUF_DBG_PARSING_STRINGS].ptr ||
+	    !s_dbg_arrays[BIN_BUF_DBG_IDLE_CHK_PARSING_DATA].ptr)
+		return DBG_STATUS_DBG_ARRAY_NOT_SET;
+
+	/* Read global_params section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "global_params"))
+		return DBG_STATUS_IDLE_CHK_PARSE_FAILED;
+
+	/* Print global params */
+	dump_buf += qed_print_section_params(dump_buf,
+					     num_section_params,
+					     results_buf, &results_offset);
+
+	/* Read idle_chk section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "idle_chk") || num_section_params != 1)
+		return DBG_STATUS_IDLE_CHK_PARSE_FAILED;
+
+	dump_buf += qed_read_param(dump_buf,
+				   &param_name, &param_str_val, &num_rules);
+	if (strcmp(param_name, "num_rules") != 0)
+		return DBG_STATUS_IDLE_CHK_PARSE_FAILED;
+
+	if (num_rules) {
+		u32 rules_print_size;
+
+		/* Print FW output */
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset),
+			    "FW_IDLE_CHECK:\n");
+		rules_print_size =
+			qed_parse_idle_chk_dump_rules(p_hwfn, dump_buf,
+						      dump_buf_end, num_rules,
+						      true,
+						      results_buf ?
+						      results_buf +
+						      results_offset : NULL,
+						      num_errors, num_warnings);
+		results_offset += rules_print_size;
+		if (rules_print_size == 0)
+			return DBG_STATUS_IDLE_CHK_PARSE_FAILED;
+
+		/* Print LSI output */
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset),
+			    "\nLSI_IDLE_CHECK:\n");
+		rules_print_size =
+			qed_parse_idle_chk_dump_rules(p_hwfn, dump_buf,
+						      dump_buf_end, num_rules,
+						      false,
+						      results_buf ?
+						      results_buf +
+						      results_offset : NULL,
+						      num_errors, num_warnings);
+		results_offset += rules_print_size;
+		if (rules_print_size == 0)
+			return DBG_STATUS_IDLE_CHK_PARSE_FAILED;
+	}
+
+	/* Print errors/warnings count */
+	if (*num_errors) {
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset),
+			    "\nIdle Check failed!!! (with %d errors and %d warnings)\n",
+			    *num_errors, *num_warnings);
+	} else if (*num_warnings) {
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset),
+			    "\nIdle Check completed successfuly (with %d warnings)\n",
+			    *num_warnings);
+	} else {
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset),
+			    "\nIdle Check completed successfuly\n");
+	}
+
+	/* Add 1 for string NULL termination */
+	*parsed_results_bytes = results_offset + 1;
+	return DBG_STATUS_OK;
+}
+
+enum dbg_status qed_get_idle_chk_results_buf_size(struct qed_hwfn *p_hwfn,
+						  u32 *dump_buf,
+						  u32 num_dumped_dwords,
+						  u32 *results_buf_size)
+{
+	u32 num_errors, num_warnings;
+
+	return qed_parse_idle_chk_dump(p_hwfn,
+				       dump_buf,
+				       num_dumped_dwords,
+				       NULL,
+				       results_buf_size,
+				       &num_errors, &num_warnings);
+}
+
+enum dbg_status qed_print_idle_chk_results(struct qed_hwfn *p_hwfn,
+					   u32 *dump_buf,
+					   u32 num_dumped_dwords,
+					   char *results_buf,
+					   u32 *num_errors, u32 *num_warnings)
+{
+	u32 parsed_buf_size;
+
+	return qed_parse_idle_chk_dump(p_hwfn,
+				       dump_buf,
+				       num_dumped_dwords,
+				       results_buf,
+				       &parsed_buf_size,
+				       num_errors, num_warnings);
+}
+
+/* Frees the specified MCP Trace meta data */
+static void qed_mcp_trace_free_meta(struct qed_hwfn *p_hwfn,
+				    struct mcp_trace_meta *meta)
+{
+	u32 i;
+
+	/* Release modules */
+	if (meta->modules) {
+		for (i = 0; i < meta->modules_num; i++)
+			kfree(meta->modules[i]);
+		kfree(meta->modules);
+	}
+
+	/* Release formats */
+	if (meta->formats) {
+		for (i = 0; i < meta->formats_num; i++)
+			kfree(meta->formats[i].format_str);
+		kfree(meta->formats);
+	}
+}
+
+/* Allocates and fills MCP Trace meta data based on the specified meta data
+ * dump buffer.
+ * Returns debug status code.
+ */
+static enum dbg_status qed_mcp_trace_alloc_meta(struct qed_hwfn *p_hwfn,
+						const u32 *meta_buf,
+						struct mcp_trace_meta *meta)
+{
+	u8 *meta_buf_bytes = (u8 *)meta_buf;
+	u32 offset = 0, signature, i;
+
+	memset(meta, 0, sizeof(*meta));
+
+	/* Read first signature */
+	signature = qed_read_dword_from_buf(meta_buf_bytes, &offset);
+	if (signature != MCP_TRACE_META_IMAGE_SIGNATURE)
+		return DBG_STATUS_INVALID_TRACE_SIGNATURE;
+
+	/* Read number of modules and allocate memory for all the modules
+	 * pointers.
+	 */
+	meta->modules_num = qed_read_byte_from_buf(meta_buf_bytes, &offset);
+	meta->modules = kzalloc(meta->modules_num * sizeof(char *), GFP_KERNEL);
+	if (!meta->modules)
+		return DBG_STATUS_VIRT_MEM_ALLOC_FAILED;
+
+	/* Allocate and read all module strings */
+	for (i = 0; i < meta->modules_num; i++) {
+		u8 module_len = qed_read_byte_from_buf(meta_buf_bytes, &offset);
+
+		*(meta->modules + i) = kzalloc(module_len, GFP_KERNEL);
+		if (!(*(meta->modules + i))) {
+			/* Update number of modules to be released */
+			meta->modules_num = i ? i - 1 : 0;
+			return DBG_STATUS_VIRT_MEM_ALLOC_FAILED;
+		}
+
+		qed_read_str_from_buf(meta_buf_bytes, &offset, module_len,
+				      *(meta->modules + i));
+		if (module_len > MCP_TRACE_MAX_MODULE_LEN)
+			(*(meta->modules + i))[MCP_TRACE_MAX_MODULE_LEN] = '\0';
+	}
+
+	/* Read second signature */
+	signature = qed_read_dword_from_buf(meta_buf_bytes, &offset);
+	if (signature != MCP_TRACE_META_IMAGE_SIGNATURE)
+		return DBG_STATUS_INVALID_TRACE_SIGNATURE;
+
+	/* Read number of formats and allocate memory for all formats */
+	meta->formats_num = qed_read_dword_from_buf(meta_buf_bytes, &offset);
+	meta->formats = kzalloc(meta->formats_num *
+				sizeof(struct mcp_trace_format),
+				GFP_KERNEL);
+	if (!meta->formats)
+		return DBG_STATUS_VIRT_MEM_ALLOC_FAILED;
+
+	/* Allocate and read all strings */
+	for (i = 0; i < meta->formats_num; i++) {
+		struct mcp_trace_format *format_ptr = &meta->formats[i];
+		u8 format_len;
+
+		format_ptr->data = qed_read_dword_from_buf(meta_buf_bytes,
+							   &offset);
+		format_len =
+		    (format_ptr->data &
+		     MCP_TRACE_FORMAT_LEN_MASK) >> MCP_TRACE_FORMAT_LEN_SHIFT;
+		format_ptr->format_str = kzalloc(format_len, GFP_KERNEL);
+		if (!format_ptr->format_str) {
+			/* Update number of modules to be released */
+			meta->formats_num = i ? i - 1 : 0;
+			return DBG_STATUS_VIRT_MEM_ALLOC_FAILED;
+		}
+
+		qed_read_str_from_buf(meta_buf_bytes,
+				      &offset,
+				      format_len, format_ptr->format_str);
+	}
+
+	return DBG_STATUS_OK;
+}
+
+/* Parses an MCP Trace dump buffer.
+ * If result_buf is not NULL, the MCP Trace results are printed to it.
+ * In any case, the required results buffer size is assigned to
+ * parsed_results_bytes.
+ * The parsing status is returned.
+ */
+static enum dbg_status qed_parse_mcp_trace_dump(struct qed_hwfn *p_hwfn,
+						u32 *dump_buf,
+						u32 num_dumped_dwords,
+						char *results_buf,
+						u32 *parsed_results_bytes)
+{
+	u32 results_offset = 0, param_mask, param_shift, param_num_val;
+	u32 num_section_params, offset, end_offset, bytes_left;
+	const char *section_name, *param_name, *param_str_val;
+	u32 trace_data_dwords, trace_meta_dwords;
+	struct mcp_trace_meta meta;
+	struct mcp_trace *trace;
+	enum dbg_status status;
+	const u32 *meta_buf;
+	u8 *trace_buf;
+
+	*parsed_results_bytes = 0;
+
+	/* Read global_params section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "global_params"))
+		return DBG_STATUS_MCP_TRACE_BAD_DATA;
+
+	/* Print global params */
+	dump_buf += qed_print_section_params(dump_buf,
+					     num_section_params,
+					     results_buf, &results_offset);
+
+	/* Read trace_data section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "mcp_trace_data") || num_section_params != 1)
+		return DBG_STATUS_MCP_TRACE_BAD_DATA;
+	dump_buf += qed_read_param(dump_buf,
+				   &param_name, &param_str_val, &param_num_val);
+	if (strcmp(param_name, "size"))
+		return DBG_STATUS_MCP_TRACE_BAD_DATA;
+	trace_data_dwords = param_num_val;
+
+	/* Prepare trace info */
+	trace = (struct mcp_trace *)dump_buf;
+	trace_buf = (u8 *)dump_buf + sizeof(struct mcp_trace);
+	offset = trace->trace_oldest;
+	end_offset = trace->trace_prod;
+	bytes_left = qed_cyclic_sub(end_offset, offset, trace->size);
+	dump_buf += trace_data_dwords;
+
+	/* Read meta_data section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "mcp_trace_meta"))
+		return DBG_STATUS_MCP_TRACE_BAD_DATA;
+	dump_buf += qed_read_param(dump_buf,
+				   &param_name, &param_str_val, &param_num_val);
+	if (strcmp(param_name, "size") != 0)
+		return DBG_STATUS_MCP_TRACE_BAD_DATA;
+	trace_meta_dwords = param_num_val;
+
+	/* Choose meta data buffer */
+	if (!trace_meta_dwords) {
+		/* Dump doesn't include meta data */
+		if (!s_mcp_trace_meta.ptr)
+			return DBG_STATUS_MCP_TRACE_NO_META;
+		meta_buf = s_mcp_trace_meta.ptr;
+	} else {
+		/* Dump includes meta data */
+		meta_buf = dump_buf;
+	}
+
+	/* Allocate meta data memory */
+	status = qed_mcp_trace_alloc_meta(p_hwfn, meta_buf, &meta);
+	if (status != DBG_STATUS_OK)
+		goto free_mem;
+
+	/* Ignore the level and modules masks - just print everything that is
+	 * already in the buffer.
+	 */
+	while (bytes_left) {
+		struct mcp_trace_format *format_ptr;
+		u8 format_level, format_module;
+		u32 params[3] = { 0, 0, 0 };
+		u32 header, format_idx, i;
+
+		if (bytes_left < MFW_TRACE_ENTRY_SIZE) {
+			status = DBG_STATUS_MCP_TRACE_BAD_DATA;
+			goto free_mem;
+		}
+
+		header = qed_read_from_cyclic_buf(trace_buf,
+						  &offset,
+						  trace->size,
+						  MFW_TRACE_ENTRY_SIZE);
+		bytes_left -= MFW_TRACE_ENTRY_SIZE;
+		format_idx = header & MFW_TRACE_EVENTID_MASK;
+
+		/* Skip message if its  index doesn't exist in the meta data */
+		if (format_idx > meta.formats_num) {
+			u8 format_size =
+			    (u8)((header &
+				  MFW_TRACE_PRM_SIZE_MASK) >>
+				 MFW_TRACE_PRM_SIZE_SHIFT);
+
+			if (bytes_left < format_size) {
+				status = DBG_STATUS_MCP_TRACE_BAD_DATA;
+				goto free_mem;
+			}
+
+			offset = qed_cyclic_add(offset,
+						format_size, trace->size);
+			bytes_left -= format_size;
+			continue;
+		}
+
+		format_ptr = &meta.formats[format_idx];
+		for (i = 0,
+		     param_mask = MCP_TRACE_FORMAT_P1_SIZE_MASK, param_shift =
+		     MCP_TRACE_FORMAT_P1_SIZE_SHIFT;
+		     i < MCP_TRACE_FORMAT_MAX_PARAMS;
+		     i++, param_mask <<= MCP_TRACE_FORMAT_PARAM_WIDTH,
+		     param_shift += MCP_TRACE_FORMAT_PARAM_WIDTH) {
+			/* Extract param size (0..3) */
+			u8 param_size =
+			    (u8)((format_ptr->data &
+				  param_mask) >> param_shift);
+
+			/* If the param size is zero, there are no other
+			 * parameters.
+			 */
+			if (!param_size)
+				break;
+
+			/* Size is encoded using 2 bits, where 3 is used to
+			 * encode 4.
+			 */
+			if (param_size == 3)
+				param_size = 4;
+			if (bytes_left < param_size) {
+				status = DBG_STATUS_MCP_TRACE_BAD_DATA;
+				goto free_mem;
+			}
+
+			params[i] = qed_read_from_cyclic_buf(trace_buf,
+							     &offset,
+							     trace->size,
+							     param_size);
+			bytes_left -= param_size;
+		}
+
+		format_level =
+		    (u8)((format_ptr->data &
+			  MCP_TRACE_FORMAT_LEVEL_MASK) >>
+			  MCP_TRACE_FORMAT_LEVEL_SHIFT);
+		format_module =
+		    (u8)((format_ptr->data &
+			  MCP_TRACE_FORMAT_MODULE_MASK) >>
+			 MCP_TRACE_FORMAT_MODULE_SHIFT);
+		if (format_level >= ARRAY_SIZE(s_mcp_trace_level_str)) {
+			status = DBG_STATUS_MCP_TRACE_BAD_DATA;
+			goto free_mem;
+		}
+
+		/* Print current message to results buffer */
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset), "%s %-8s: ",
+			    s_mcp_trace_level_str[format_level],
+			    meta.modules[format_module]);
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset),
+			    format_ptr->format_str, params[0], params[1],
+			    params[2]);
+	}
+
+free_mem:
+	*parsed_results_bytes = results_offset + 1;
+	qed_mcp_trace_free_meta(p_hwfn, &meta);
+	return status;
+}
+
+enum dbg_status qed_get_mcp_trace_results_buf_size(struct qed_hwfn *p_hwfn,
+						   u32 *dump_buf,
+						   u32 num_dumped_dwords,
+						   u32 *results_buf_size)
+{
+	return qed_parse_mcp_trace_dump(p_hwfn,
+					dump_buf,
+					num_dumped_dwords,
+					NULL, results_buf_size);
+}
+
+enum dbg_status qed_print_mcp_trace_results(struct qed_hwfn *p_hwfn,
+					    u32 *dump_buf,
+					    u32 num_dumped_dwords,
+					    char *results_buf)
+{
+	u32 parsed_buf_size;
+
+	return qed_parse_mcp_trace_dump(p_hwfn,
+					dump_buf,
+					num_dumped_dwords,
+					results_buf, &parsed_buf_size);
+}
+
+/* Parses a Reg FIFO dump buffer.
+ * If result_buf is not NULL, the Reg FIFO results are printed to it.
+ * In any case, the required results buffer size is assigned to
+ * parsed_results_bytes.
+ * The parsing status is returned.
+ */
+static enum dbg_status qed_parse_reg_fifo_dump(struct qed_hwfn *p_hwfn,
+					       u32 *dump_buf,
+					       u32 num_dumped_dwords,
+					       char *results_buf,
+					       u32 *parsed_results_bytes)
+{
+	u32 results_offset = 0, param_num_val, num_section_params, num_elements;
+	const char *section_name, *param_name, *param_str_val;
+	struct reg_fifo_element *elements;
+	u8 i, j, err_val, vf_val;
+	char vf_str[4];
+
+	/* Read global_params section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "global_params"))
+		return DBG_STATUS_REG_FIFO_BAD_DATA;
+
+	/* Print global params */
+	dump_buf += qed_print_section_params(dump_buf,
+					     num_section_params,
+					     results_buf, &results_offset);
+
+	/* Read reg_fifo_data section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "reg_fifo_data"))
+		return DBG_STATUS_REG_FIFO_BAD_DATA;
+	dump_buf += qed_read_param(dump_buf,
+				   &param_name, &param_str_val, &param_num_val);
+	if (strcmp(param_name, "size"))
+		return DBG_STATUS_REG_FIFO_BAD_DATA;
+	if (param_num_val % REG_FIFO_ELEMENT_DWORDS)
+		return DBG_STATUS_REG_FIFO_BAD_DATA;
+	num_elements = param_num_val / REG_FIFO_ELEMENT_DWORDS;
+	elements = (struct reg_fifo_element *)dump_buf;
+
+	/* Decode elements */
+	for (i = 0; i < num_elements; i++) {
+		bool err_printed = false;
+
+		/* Discover if element belongs to a VF or a PF */
+		vf_val = GET_FIELD(elements[i].data, REG_FIFO_ELEMENT_VF);
+		if (vf_val == REG_FIFO_ELEMENT_IS_PF_VF_VAL)
+			sprintf(vf_str, "%s", "N/A");
+		else
+			sprintf(vf_str, "%d", vf_val);
+
+		/* Add parsed element to parsed buffer */
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset),
+			    "raw: 0x%016llx, address: 0x%07llx, access: %-5s, pf: %2lld, vf: %s, port: %lld, privilege: %-3s, protection: %-12s, master: %-4s, errors: ",
+			    elements[i].data,
+			    GET_FIELD(elements[i].data,
+				      REG_FIFO_ELEMENT_ADDRESS) *
+				      REG_FIFO_ELEMENT_ADDR_FACTOR,
+				      s_access_strs[GET_FIELD(elements[i].data,
+						    REG_FIFO_ELEMENT_ACCESS)],
+			    GET_FIELD(elements[i].data,
+				      REG_FIFO_ELEMENT_PF), vf_str,
+			    GET_FIELD(elements[i].data,
+				      REG_FIFO_ELEMENT_PORT),
+				      s_privilege_strs[GET_FIELD(elements[i].
+				      data,
+				      REG_FIFO_ELEMENT_PRIVILEGE)],
+			    s_protection_strs[GET_FIELD(elements[i].data,
+						REG_FIFO_ELEMENT_PROTECTION)],
+			    s_master_strs[GET_FIELD(elements[i].data,
+						REG_FIFO_ELEMENT_MASTER)]);
+
+		/* Print errors */
+		for (j = 0,
+		     err_val = GET_FIELD(elements[i].data,
+					 REG_FIFO_ELEMENT_ERROR);
+		     j < ARRAY_SIZE(s_reg_fifo_error_strs);
+		     j++, err_val >>= 1) {
+			if (!(err_val & 0x1))
+				continue;
+			if (err_printed)
+				results_offset +=
+					sprintf(qed_get_buf_ptr(results_buf,
+								results_offset),
+						", ");
+			results_offset +=
+				sprintf(qed_get_buf_ptr(results_buf,
+							results_offset), "%s",
+					s_reg_fifo_error_strs[j]);
+			err_printed = true;
+		}
+
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf, results_offset), "\n");
+	}
+
+	results_offset += sprintf(qed_get_buf_ptr(results_buf,
+						  results_offset),
+				  "fifo contained %d elements", num_elements);
+
+	/* Add 1 for string NULL termination */
+	*parsed_results_bytes = results_offset + 1;
+	return DBG_STATUS_OK;
+}
+
+enum dbg_status qed_get_reg_fifo_results_buf_size(struct qed_hwfn *p_hwfn,
+						  u32 *dump_buf,
+						  u32 num_dumped_dwords,
+						  u32 *results_buf_size)
+{
+	return qed_parse_reg_fifo_dump(p_hwfn,
+				       dump_buf,
+				       num_dumped_dwords,
+				       NULL, results_buf_size);
+}
+
+enum dbg_status qed_print_reg_fifo_results(struct qed_hwfn *p_hwfn,
+					   u32 *dump_buf,
+					   u32 num_dumped_dwords,
+					   char *results_buf)
+{
+	u32 parsed_buf_size;
+
+	return qed_parse_reg_fifo_dump(p_hwfn,
+				       dump_buf,
+				       num_dumped_dwords,
+				       results_buf, &parsed_buf_size);
+}
+
+/* Parses an IGU FIFO dump buffer.
+ * If result_buf is not NULL, the IGU FIFO results are printed to it.
+ * In any case, the required results buffer size is assigned to
+ * parsed_results_bytes.
+ * The parsing status is returned.
+ */
+static enum dbg_status qed_parse_igu_fifo_dump(struct qed_hwfn *p_hwfn,
+					       u32 *dump_buf,
+					       u32 num_dumped_dwords,
+					       char *results_buf,
+					       u32 *parsed_results_bytes)
+{
+	u32 results_offset = 0, param_num_val, num_section_params, num_elements;
+	const char *section_name, *param_name, *param_str_val;
+	struct igu_fifo_element *elements;
+	char parsed_addr_data[32];
+	char parsed_wr_data[256];
+	u8 i, j;
+
+	/* Read global_params section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "global_params"))
+		return DBG_STATUS_IGU_FIFO_BAD_DATA;
+
+	/* Print global params */
+	dump_buf += qed_print_section_params(dump_buf,
+					     num_section_params,
+					     results_buf, &results_offset);
+
+	/* Read igu_fifo_data section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "igu_fifo_data"))
+		return DBG_STATUS_IGU_FIFO_BAD_DATA;
+	dump_buf += qed_read_param(dump_buf,
+				   &param_name, &param_str_val, &param_num_val);
+	if (strcmp(param_name, "size"))
+		return DBG_STATUS_IGU_FIFO_BAD_DATA;
+	if (param_num_val % IGU_FIFO_ELEMENT_DWORDS)
+		return DBG_STATUS_IGU_FIFO_BAD_DATA;
+	num_elements = param_num_val / IGU_FIFO_ELEMENT_DWORDS;
+	elements = (struct igu_fifo_element *)dump_buf;
+
+	/* Decode elements */
+	for (i = 0; i < num_elements; i++) {
+		/* dword12 (dword index 1 and 2) contains bits 32..95 of the
+		 * FIFO element.
+		 */
+		u64 dword12 =
+		    ((u64)elements[i].dword2 << 32) | elements[i].dword1;
+		bool is_wr_cmd = GET_FIELD(dword12,
+					   IGU_FIFO_ELEMENT_DWORD12_IS_WR_CMD);
+		bool is_pf = GET_FIELD(elements[i].dword0,
+				       IGU_FIFO_ELEMENT_DWORD0_IS_PF);
+		u16 cmd_addr = GET_FIELD(elements[i].dword0,
+					 IGU_FIFO_ELEMENT_DWORD0_CMD_ADDR);
+		u8 source = GET_FIELD(elements[i].dword0,
+				      IGU_FIFO_ELEMENT_DWORD0_SOURCE);
+		u8 err_type = GET_FIELD(elements[i].dword0,
+					IGU_FIFO_ELEMENT_DWORD0_ERR_TYPE);
+		const struct igu_fifo_addr_data *addr_data = NULL;
+
+		if (source >= ARRAY_SIZE(s_igu_fifo_source_strs))
+			return DBG_STATUS_IGU_FIFO_BAD_DATA;
+		if (err_type >= ARRAY_SIZE(s_igu_fifo_error_strs))
+			return DBG_STATUS_IGU_FIFO_BAD_DATA;
+
+		/* Find address data */
+		for (j = 0; j < ARRAY_SIZE(s_igu_fifo_addr_data) && !addr_data;
+		     j++)
+			if (cmd_addr >= s_igu_fifo_addr_data[j].start_addr &&
+			    cmd_addr <= s_igu_fifo_addr_data[j].end_addr)
+				addr_data = &s_igu_fifo_addr_data[j];
+		if (!addr_data)
+			return DBG_STATUS_IGU_FIFO_BAD_DATA;
+
+		/* Prepare parsed address data */
+		switch (addr_data->type) {
+		case IGU_ADDR_TYPE_MSIX_MEM:
+			sprintf(parsed_addr_data,
+				" vector_num=0x%x", cmd_addr / 2);
+			break;
+		case IGU_ADDR_TYPE_WRITE_INT_ACK:
+		case IGU_ADDR_TYPE_WRITE_PROD_UPDATE:
+			sprintf(parsed_addr_data,
+				" SB=0x%x", cmd_addr - addr_data->start_addr);
+			break;
+		default:
+			parsed_addr_data[0] = '\0';
+		}
+
+		/* Prepare parsed write data */
+		if (is_wr_cmd) {
+			u32 wr_data = GET_FIELD(dword12,
+					IGU_FIFO_ELEMENT_DWORD12_WR_DATA);
+			u32 prod_cons = GET_FIELD(wr_data,
+						  IGU_FIFO_WR_DATA_PROD_CONS);
+			u8 is_cleanup = GET_FIELD(wr_data,
+						  IGU_FIFO_WR_DATA_CMD_TYPE);
+
+			if (source == IGU_SRC_ATTN) {
+				sprintf(parsed_wr_data,
+					"prod: 0x%x, ", prod_cons);
+			} else {
+				if (is_cleanup) {
+					u8 cleanup_val = GET_FIELD(wr_data,
+								   IGU_FIFO_CLEANUP_WR_DATA_CLEANUP_VAL);
+					u8 cleanup_type = GET_FIELD(wr_data,
+								    IGU_FIFO_CLEANUP_WR_DATA_CLEANUP_TYPE);
+
+					sprintf(parsed_wr_data,
+						"cmd_type: cleanup, cleanup_val: %s, cleanup_type: %d, ",
+						cleanup_val ? "set" : "clear",
+						cleanup_type);
+				} else {
+					u8 update_flag = GET_FIELD(wr_data,
+								   IGU_FIFO_WR_DATA_UPDATE_FLAG);
+					u8 en_dis_int_for_sb =
+					    GET_FIELD(wr_data,
+						      IGU_FIFO_WR_DATA_EN_DIS_INT_FOR_SB);
+					u8 segment = GET_FIELD(wr_data,
+							       IGU_FIFO_WR_DATA_SEGMENT);
+					u8 timer_mask = GET_FIELD(wr_data,
+								  IGU_FIFO_WR_DATA_TIMER_MASK);
+
+					sprintf(parsed_wr_data,
+						"cmd_type: prod/cons update, prod/cons: 0x%x, update_flag: %s, en_dis_int_for_sb: %s, segment: %s, timer_mask=%d, ",
+						prod_cons,
+						update_flag ? "update" : "nop",
+						en_dis_int_for_sb
+						? (en_dis_int_for_sb ==
+						   1 ? "disable" : "nop") :
+						"enable",
+						segment ? "attn" : "regular",
+						timer_mask);
+				}
+			}
+		} else {
+			parsed_wr_data[0] = '\0';
+		}
+
+		/* Add parsed element to parsed buffer */
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset),
+			    "raw: 0x%01x%08x%08x, %s: %d, source: %s, type: %s, cmd_addr: 0x%x (%s%s), %serror: %s\n",
+			    elements[i].dword2, elements[i].dword1,
+			    elements[i].dword0,
+			    is_pf ? "pf" : "vf",
+			    GET_FIELD(elements[i].dword0,
+				      IGU_FIFO_ELEMENT_DWORD0_FID),
+			    s_igu_fifo_source_strs[source],
+			    is_wr_cmd ? "wr" : "rd", cmd_addr,
+			    (!is_pf && addr_data->vf_desc)
+			    ? addr_data->vf_desc : addr_data->desc,
+			    parsed_addr_data, parsed_wr_data,
+			    s_igu_fifo_error_strs[err_type]);
+	}
+
+	results_offset += sprintf(qed_get_buf_ptr(results_buf,
+						  results_offset),
+				  "fifo contained %d elements", num_elements);
+
+	/* Add 1 for string NULL termination */
+	*parsed_results_bytes = results_offset + 1;
+	return DBG_STATUS_OK;
+}
+
+enum dbg_status qed_get_igu_fifo_results_buf_size(struct qed_hwfn *p_hwfn,
+						  u32 *dump_buf,
+						  u32 num_dumped_dwords,
+						  u32 *results_buf_size)
+{
+	return qed_parse_igu_fifo_dump(p_hwfn,
+				       dump_buf,
+				       num_dumped_dwords,
+				       NULL, results_buf_size);
+}
+
+enum dbg_status qed_print_igu_fifo_results(struct qed_hwfn *p_hwfn,
+					   u32 *dump_buf,
+					   u32 num_dumped_dwords,
+					   char *results_buf)
+{
+	u32 parsed_buf_size;
+
+	return qed_parse_igu_fifo_dump(p_hwfn,
+				       dump_buf,
+				       num_dumped_dwords,
+				       results_buf, &parsed_buf_size);
+}
+
+static enum dbg_status
+qed_parse_protection_override_dump(struct qed_hwfn *p_hwfn,
+				   u32 *dump_buf,
+				   u32 num_dumped_dwords,
+				   char *results_buf,
+				   u32 *parsed_results_bytes)
+{
+	u32 results_offset = 0, param_num_val, num_section_params, num_elements;
+	const char *section_name, *param_name, *param_str_val;
+	struct protection_override_element *elements;
+	u8 i;
+
+	/* Read global_params section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "global_params"))
+		return DBG_STATUS_PROTECTION_OVERRIDE_BAD_DATA;
+
+	/* Print global params */
+	dump_buf += qed_print_section_params(dump_buf,
+					     num_section_params,
+					     results_buf, &results_offset);
+
+	/* Read protection_override_data section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "protection_override_data"))
+		return DBG_STATUS_PROTECTION_OVERRIDE_BAD_DATA;
+	dump_buf += qed_read_param(dump_buf,
+				   &param_name, &param_str_val, &param_num_val);
+	if (strcmp(param_name, "size"))
+		return DBG_STATUS_PROTECTION_OVERRIDE_BAD_DATA;
+	if (param_num_val % PROTECTION_OVERRIDE_ELEMENT_DWORDS != 0)
+		return DBG_STATUS_PROTECTION_OVERRIDE_BAD_DATA;
+	num_elements = param_num_val / PROTECTION_OVERRIDE_ELEMENT_DWORDS;
+	elements = (struct protection_override_element *)dump_buf;
+
+	/* Decode elements */
+	for (i = 0; i < num_elements; i++) {
+		u32 address = GET_FIELD(elements[i].data,
+					PROTECTION_OVERRIDE_ELEMENT_ADDRESS) *
+					PROTECTION_OVERRIDE_ELEMENT_ADDR_FACTOR;
+
+		results_offset +=
+		    sprintf(qed_get_buf_ptr(results_buf,
+					    results_offset),
+			    "window %2d, address: 0x%07x, size: %7lld regs, read: %lld, write: %lld, read protection: %-12s, write protection: %-12s\n",
+			    i, address,
+			    GET_FIELD(elements[i].data,
+				      PROTECTION_OVERRIDE_ELEMENT_WINDOW_SIZE),
+			    GET_FIELD(elements[i].data,
+				      PROTECTION_OVERRIDE_ELEMENT_READ),
+			    GET_FIELD(elements[i].data,
+				      PROTECTION_OVERRIDE_ELEMENT_WRITE),
+			    s_protection_strs[GET_FIELD(elements[i].data,
+				PROTECTION_OVERRIDE_ELEMENT_READ_PROTECTION)],
+			    s_protection_strs[GET_FIELD(elements[i].data,
+				PROTECTION_OVERRIDE_ELEMENT_WRITE_PROTECTION)]);
+	}
+
+	results_offset += sprintf(qed_get_buf_ptr(results_buf,
+						  results_offset),
+				  "protection override contained %d elements",
+				  num_elements);
+
+	/* Add 1 for string NULL termination */
+	*parsed_results_bytes = results_offset + 1;
+	return DBG_STATUS_OK;
+}
+
+enum dbg_status
+qed_get_protection_override_results_buf_size(struct qed_hwfn *p_hwfn,
+					     u32 *dump_buf,
+					     u32 num_dumped_dwords,
+					     u32 *results_buf_size)
+{
+	return qed_parse_protection_override_dump(p_hwfn,
+						  dump_buf,
+						  num_dumped_dwords,
+						  NULL, results_buf_size);
+}
+
+enum dbg_status qed_print_protection_override_results(struct qed_hwfn *p_hwfn,
+						      u32 *dump_buf,
+						      u32 num_dumped_dwords,
+						      char *results_buf)
+{
+	u32 parsed_buf_size;
+
+	return qed_parse_protection_override_dump(p_hwfn,
+						  dump_buf,
+						  num_dumped_dwords,
+						  results_buf,
+						  &parsed_buf_size);
+}
+
+/* Parses a FW Asserts dump buffer.
+ * If result_buf is not NULL, the FW Asserts results are printed to it.
+ * In any case, the required results buffer size is assigned to
+ * parsed_results_bytes.
+ * The parsing status is returned.
+ */
+static enum dbg_status qed_parse_fw_asserts_dump(struct qed_hwfn *p_hwfn,
+						 u32 *dump_buf,
+						 u32 num_dumped_dwords,
+						 char *results_buf,
+						 u32 *parsed_results_bytes)
+{
+	u32 results_offset = 0, num_section_params, param_num_val, i;
+	const char *param_name, *param_str_val, *section_name;
+	bool last_section_found = false;
+
+	*parsed_results_bytes = 0;
+
+	/* Read global_params section */
+	dump_buf += qed_read_section_hdr(dump_buf,
+					 &section_name, &num_section_params);
+	if (strcmp(section_name, "global_params"))
+		return DBG_STATUS_FW_ASSERTS_PARSE_FAILED;
+
+	/* Print global params */
+	dump_buf += qed_print_section_params(dump_buf,
+					     num_section_params,
+					     results_buf, &results_offset);
+	while (!last_section_found) {
+		const char *storm_letter = NULL;
+		u32 storm_dump_size = 0;
+
+		dump_buf += qed_read_section_hdr(dump_buf,
+						 &section_name,
+						 &num_section_params);
+		if (!strcmp(section_name, "last")) {
+			last_section_found = true;
+			continue;
+		} else if (strcmp(section_name, "fw_asserts")) {
+			return DBG_STATUS_FW_ASSERTS_PARSE_FAILED;
+		}
+
+		/* Extract params */
+		for (i = 0; i < num_section_params; i++) {
+			dump_buf += qed_read_param(dump_buf,
+						   &param_name,
+						   &param_str_val,
+						   &param_num_val);
+			if (!strcmp(param_name, "storm"))
+				storm_letter = param_str_val;
+			else if (!strcmp(param_name, "size"))
+				storm_dump_size = param_num_val;
+			else
+				return DBG_STATUS_FW_ASSERTS_PARSE_FAILED;
+		}
+
+		if (!storm_letter || !storm_dump_size)
+			return DBG_STATUS_FW_ASSERTS_PARSE_FAILED;
+
+		/* Print data */
+		results_offset += sprintf(qed_get_buf_ptr(results_buf,
+							  results_offset),
+					  "\n%sSTORM_ASSERT: size=%d\n",
+					  storm_letter, storm_dump_size);
+		for (i = 0; i < storm_dump_size; i++, dump_buf++)
+			results_offset +=
+			    sprintf(qed_get_buf_ptr(results_buf,
+						    results_offset),
+				    "%08x\n", *dump_buf);
+	}
+
+	/* Add 1 for string NULL termination */
+	*parsed_results_bytes = results_offset + 1;
+	return DBG_STATUS_OK;
+}
+
+enum dbg_status qed_get_fw_asserts_results_buf_size(struct qed_hwfn *p_hwfn,
+						    u32 *dump_buf,
+						    u32 num_dumped_dwords,
+						    u32 *results_buf_size)
+{
+	return qed_parse_fw_asserts_dump(p_hwfn,
+					 dump_buf,
+					 num_dumped_dwords,
+					 NULL, results_buf_size);
+}
+
+enum dbg_status qed_print_fw_asserts_results(struct qed_hwfn *p_hwfn,
+					     u32 *dump_buf,
+					     u32 num_dumped_dwords,
+					     char *results_buf)
+{
+	u32 parsed_buf_size;
+
+	return qed_parse_fw_asserts_dump(p_hwfn,
+					 dump_buf,
+					 num_dumped_dwords,
+					 results_buf, &parsed_buf_size);
+}
+
+/* Wrapper for unifying the idle_chk and mcp_trace api */
+enum dbg_status qed_print_idle_chk_results_wrapper(struct qed_hwfn *p_hwfn,
+						   u32 *dump_buf,
+						   u32 num_dumped_dwords,
+						   char *results_buf)
+{
+	u32 num_errors, num_warnnings;
+
+	return qed_print_idle_chk_results(p_hwfn, dump_buf, num_dumped_dwords,
+					  results_buf, &num_errors,
+					  &num_warnnings);
+}
+
+/* Feature meta data lookup table */
+static struct {
+	char *name;
+	enum dbg_status (*get_size)(struct qed_hwfn *p_hwfn,
+				    struct qed_ptt *p_ptt, u32 *size);
+	enum dbg_status (*perform_dump)(struct qed_hwfn *p_hwfn,
+					struct qed_ptt *p_ptt, u32 *dump_buf,
+					u32 buf_size, u32 *dumped_dwords);
+	enum dbg_status (*print_results)(struct qed_hwfn *p_hwfn,
+					 u32 *dump_buf, u32 num_dumped_dwords,
+					 char *results_buf);
+	enum dbg_status (*results_buf_size)(struct qed_hwfn *p_hwfn,
+					    u32 *dump_buf,
+					    u32 num_dumped_dwords,
+					    u32 *results_buf_size);
+} qed_features_lookup[] = {
+	{
+	"grc", qed_dbg_grc_get_dump_buf_size,
+		    qed_dbg_grc_dump, NULL, NULL}, {
+	"idle_chk",
+		    qed_dbg_idle_chk_get_dump_buf_size,
+		    qed_dbg_idle_chk_dump,
+		    qed_print_idle_chk_results_wrapper,
+		    qed_get_idle_chk_results_buf_size}, {
+	"mcp_trace",
+		    qed_dbg_mcp_trace_get_dump_buf_size,
+		    qed_dbg_mcp_trace_dump, qed_print_mcp_trace_results,
+		    qed_get_mcp_trace_results_buf_size}, {
+	"reg_fifo",
+		    qed_dbg_reg_fifo_get_dump_buf_size,
+		    qed_dbg_reg_fifo_dump, qed_print_reg_fifo_results,
+		    qed_get_reg_fifo_results_buf_size}, {
+	"igu_fifo",
+		    qed_dbg_igu_fifo_get_dump_buf_size,
+		    qed_dbg_igu_fifo_dump, qed_print_igu_fifo_results,
+		    qed_get_igu_fifo_results_buf_size}, {
+	"protection_override",
+		    qed_dbg_protection_override_get_dump_buf_size,
+		    qed_dbg_protection_override_dump,
+		    qed_print_protection_override_results,
+		    qed_get_protection_override_results_buf_size}, {
+	"fw_asserts",
+		    qed_dbg_fw_asserts_get_dump_buf_size,
+		    qed_dbg_fw_asserts_dump,
+		    qed_print_fw_asserts_results,
+		    qed_get_fw_asserts_results_buf_size},};
+
+static void qed_dbg_print_feature(u8 *p_text_buf, u32 text_size)
+{
+	u32 i, precision = 80;
+
+	if (!p_text_buf)
+		return;
+
+	pr_notice("\n%.*s", precision, p_text_buf);
+	for (i = precision; i < text_size; i += precision)
+		pr_cont("%.*s", precision, p_text_buf + i);
+	pr_cont("\n");
+}
+
+#define QED_RESULTS_BUF_MIN_SIZE 16
+/* Generic function for decoding debug feature info */
+enum dbg_status format_feature(struct qed_hwfn *p_hwfn,
+			       enum qed_dbg_features feature_idx)
+{
+	struct qed_dbg_feature *feature =
+	    &p_hwfn->cdev->dbg_params.features[feature_idx];
+	u32 text_size_bytes, null_char_pos, i;
+	enum dbg_status rc;
+	char *text_buf;
+
+	/* Check if feature supports formatting capability */
+	if (!qed_features_lookup[feature_idx].results_buf_size)
+		return DBG_STATUS_OK;
+
+	/* Obtain size of formatted output */
+	rc = qed_features_lookup[feature_idx].
+		results_buf_size(p_hwfn, (u32 *)feature->dump_buf,
+				 feature->dumped_dwords, &text_size_bytes);
+	if (rc != DBG_STATUS_OK)
+		return rc;
+
+	/* Make sure that the allocated size is a multiple of dword (4 bytes) */
+	null_char_pos = text_size_bytes - 1;
+	text_size_bytes = (text_size_bytes + 3) & ~0x3;
+
+	if (text_size_bytes < QED_RESULTS_BUF_MIN_SIZE) {
+		DP_NOTICE(p_hwfn->cdev,
+			  "formatted size of feature was too small %d. Aborting\n",
+			  text_size_bytes);
+		return DBG_STATUS_INVALID_ARGS;
+	}
+
+	/* Allocate temp text buf */
+	text_buf = vzalloc(text_size_bytes);
+	if (!text_buf)
+		return DBG_STATUS_VIRT_MEM_ALLOC_FAILED;
+
+	/* Decode feature opcodes to string on temp buf */
+	rc = qed_features_lookup[feature_idx].
+		print_results(p_hwfn, (u32 *)feature->dump_buf,
+			      feature->dumped_dwords, text_buf);
+	if (rc != DBG_STATUS_OK) {
+		vfree(text_buf);
+		return rc;
+	}
+
+	/* Replace the original null character with a '\n' character.
+	 * The bytes that were added as a result of the dword alignment are also
+	 * padded with '\n' characters.
+	 */
+	for (i = null_char_pos; i < text_size_bytes; i++)
+		text_buf[i] = '\n';
+
+	/* Dump printable feature to log */
+	if (p_hwfn->cdev->dbg_params.print_data)
+		qed_dbg_print_feature(text_buf, text_size_bytes);
+
+	/* Free the old dump_buf and point the dump_buf to the newly allocagted
+	 * and formatted text buffer.
+	 */
+	vfree(feature->dump_buf);
+	feature->dump_buf = text_buf;
+	feature->buf_size = text_size_bytes;
+	feature->dumped_dwords = text_size_bytes / 4;
+	return rc;
+}
+
+/* Generic function for performing the dump of a debug feature. */
+enum dbg_status qed_dbg_dump(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+			     enum qed_dbg_features feature_idx)
+{
+	struct qed_dbg_feature *feature =
+	    &p_hwfn->cdev->dbg_params.features[feature_idx];
+	u32 buf_size_dwords;
+	enum dbg_status rc;
+
+	DP_NOTICE(p_hwfn->cdev, "Collecting a debug feature [\"%s\"]\n",
+		  qed_features_lookup[feature_idx].name);
+
+	/* Dump_buf was already allocated need to free (this can happen if dump
+	 * was called but file was never read).
+	 * We can't use the buffer as is since size may have changed.
+	 */
+	if (feature->dump_buf) {
+		vfree(feature->dump_buf);
+		feature->dump_buf = NULL;
+	}
+
+	/* Get buffer size from hsi, allocate accordingly, and perform the
+	 * dump.
+	 */
+	rc = qed_features_lookup[feature_idx].get_size(p_hwfn, p_ptt,
+						       &buf_size_dwords);
+	if (rc != DBG_STATUS_OK)
+		return rc;
+	feature->buf_size = buf_size_dwords * sizeof(u32);
+	feature->dump_buf = vmalloc(feature->buf_size);
+	if (!feature->dump_buf)
+		return DBG_STATUS_VIRT_MEM_ALLOC_FAILED;
+
+	rc = qed_features_lookup[feature_idx].
+		perform_dump(p_hwfn, p_ptt, (u32 *)feature->dump_buf,
+			     feature->buf_size / sizeof(u32),
+			     &feature->dumped_dwords);
+
+	/* If mcp is stuck we get DBG_STATUS_NVRAM_GET_IMAGE_FAILED error.
+	 * In this case the buffer holds valid binary data, but we wont able
+	 * to parse it (since parsing relies on data in NVRAM which is only
+	 * accessible when MFW is responsive). skip the formatting but return
+	 * success so that binary data is provided.
+	 */
+	if (rc == DBG_STATUS_NVRAM_GET_IMAGE_FAILED)
+		return DBG_STATUS_OK;
+
+	if (rc != DBG_STATUS_OK)
+		return rc;
+
+	/* Format output */
+	rc = format_feature(p_hwfn, feature_idx);
+	return rc;
+}
+
+int qed_dbg_grc(struct qed_dev *cdev, void *buffer, u32 *num_dumped_bytes)
+{
+	return qed_dbg_feature(cdev, buffer, DBG_FEATURE_GRC, num_dumped_bytes);
+}
+
+int qed_dbg_grc_size(struct qed_dev *cdev)
+{
+	return qed_dbg_feature_size(cdev, DBG_FEATURE_GRC);
+}
+
+int qed_dbg_idle_chk(struct qed_dev *cdev, void *buffer, u32 *num_dumped_bytes)
+{
+	return qed_dbg_feature(cdev, buffer, DBG_FEATURE_IDLE_CHK,
+			       num_dumped_bytes);
+}
+
+int qed_dbg_idle_chk_size(struct qed_dev *cdev)
+{
+	return qed_dbg_feature_size(cdev, DBG_FEATURE_IDLE_CHK);
+}
+
+int qed_dbg_reg_fifo(struct qed_dev *cdev, void *buffer, u32 *num_dumped_bytes)
+{
+	return qed_dbg_feature(cdev, buffer, DBG_FEATURE_REG_FIFO,
+			       num_dumped_bytes);
+}
+
+int qed_dbg_reg_fifo_size(struct qed_dev *cdev)
+{
+	return qed_dbg_feature_size(cdev, DBG_FEATURE_REG_FIFO);
+}
+
+int qed_dbg_igu_fifo(struct qed_dev *cdev, void *buffer, u32 *num_dumped_bytes)
+{
+	return qed_dbg_feature(cdev, buffer, DBG_FEATURE_IGU_FIFO,
+			       num_dumped_bytes);
+}
+
+int qed_dbg_igu_fifo_size(struct qed_dev *cdev)
+{
+	return qed_dbg_feature_size(cdev, DBG_FEATURE_IGU_FIFO);
+}
+
+int qed_dbg_protection_override(struct qed_dev *cdev, void *buffer,
+				u32 *num_dumped_bytes)
+{
+	return qed_dbg_feature(cdev, buffer, DBG_FEATURE_PROTECTION_OVERRIDE,
+			       num_dumped_bytes);
+}
+
+int qed_dbg_protection_override_size(struct qed_dev *cdev)
+{
+	return qed_dbg_feature_size(cdev, DBG_FEATURE_PROTECTION_OVERRIDE);
+}
+
+int qed_dbg_fw_asserts(struct qed_dev *cdev, void *buffer,
+		       u32 *num_dumped_bytes)
+{
+	return qed_dbg_feature(cdev, buffer, DBG_FEATURE_FW_ASSERTS,
+			       num_dumped_bytes);
+}
+
+int qed_dbg_fw_asserts_size(struct qed_dev *cdev)
+{
+	return qed_dbg_feature_size(cdev, DBG_FEATURE_FW_ASSERTS);
+}
+
+int qed_dbg_mcp_trace(struct qed_dev *cdev, void *buffer,
+		      u32 *num_dumped_bytes)
+{
+	return qed_dbg_feature(cdev, buffer, DBG_FEATURE_MCP_TRACE,
+			       num_dumped_bytes);
+}
+
+int qed_dbg_mcp_trace_size(struct qed_dev *cdev)
+{
+	return qed_dbg_feature_size(cdev, DBG_FEATURE_MCP_TRACE);
+}
+
+/* Defines the amount of bytes allocated for recording the length of debugfs
+ * feature buffer.
+ */
+#define REGDUMP_HEADER_SIZE			sizeof(u32)
+#define REGDUMP_HEADER_FEATURE_SHIFT		24
+#define REGDUMP_HEADER_ENGINE_SHIFT		31
+#define REGDUMP_HEADER_OMIT_ENGINE_SHIFT	30
+enum debug_print_features {
+	OLD_MODE = 0,
+	IDLE_CHK = 1,
+	GRC_DUMP = 2,
+	MCP_TRACE = 3,
+	REG_FIFO = 4,
+	PROTECTION_OVERRIDE = 5,
+	IGU_FIFO = 6,
+	PHY = 7,
+	FW_ASSERTS = 8,
+};
+
+static u32 qed_calc_regdump_header(enum debug_print_features feature,
+				   int engine, u32 feature_size, u8 omit_engine)
+{
+	/* Insert the engine, feature and mode inside the header and combine it
+	 * with feature size.
+	 */
+	return feature_size | (feature << REGDUMP_HEADER_FEATURE_SHIFT) |
+	       (omit_engine << REGDUMP_HEADER_OMIT_ENGINE_SHIFT) |
+	       (engine << REGDUMP_HEADER_ENGINE_SHIFT);
+}
+
+int qed_dbg_all_data(struct qed_dev *cdev, void *buffer)
+{
+	u8 cur_engine, omit_engine = 0, org_engine;
+	u32 offset = 0, feature_size;
+	int rc;
+
+	if (cdev->num_hwfns == 1)
+		omit_engine = 1;
+
+	org_engine = qed_get_debug_engine(cdev);
+	for (cur_engine = 0; cur_engine < cdev->num_hwfns; cur_engine++) {
+		/* Collect idle_chks and grcDump for each hw function */
+		DP_VERBOSE(cdev, QED_MSG_DEBUG,
+			   "obtaining idle_chk and grcdump for current engine\n");
+		qed_set_debug_engine(cdev, cur_engine);
+
+		/* First idle_chk */
+		rc = qed_dbg_idle_chk(cdev, (u8 *)buffer + offset +
+				      REGDUMP_HEADER_SIZE, &feature_size);
+		if (!rc) {
+			*(u32 *)((u8 *)buffer + offset) =
+			    qed_calc_regdump_header(IDLE_CHK, cur_engine,
+						    feature_size, omit_engine);
+			offset += (feature_size + REGDUMP_HEADER_SIZE);
+		} else {
+			DP_ERR(cdev, "qed_dbg_idle_chk failed. rc = %d\n", rc);
+		}
+
+		/* Second idle_chk */
+		rc = qed_dbg_idle_chk(cdev, (u8 *)buffer + offset +
+				      REGDUMP_HEADER_SIZE, &feature_size);
+		if (!rc) {
+			*(u32 *)((u8 *)buffer + offset) =
+			    qed_calc_regdump_header(IDLE_CHK, cur_engine,
+						    feature_size, omit_engine);
+			offset += (feature_size + REGDUMP_HEADER_SIZE);
+		} else {
+			DP_ERR(cdev, "qed_dbg_idle_chk failed. rc = %d\n", rc);
+		}
+
+		/* reg_fifo dump */
+		rc = qed_dbg_reg_fifo(cdev, (u8 *)buffer + offset +
+				      REGDUMP_HEADER_SIZE, &feature_size);
+		if (!rc) {
+			*(u32 *)((u8 *)buffer + offset) =
+			    qed_calc_regdump_header(REG_FIFO, cur_engine,
+						    feature_size, omit_engine);
+			offset += (feature_size + REGDUMP_HEADER_SIZE);
+		} else {
+			DP_ERR(cdev, "qed_dbg_reg_fifo failed. rc = %d\n", rc);
+		}
+
+		/* igu_fifo dump */
+		rc = qed_dbg_igu_fifo(cdev, (u8 *)buffer + offset +
+				      REGDUMP_HEADER_SIZE, &feature_size);
+		if (!rc) {
+			*(u32 *)((u8 *)buffer + offset) =
+			    qed_calc_regdump_header(IGU_FIFO, cur_engine,
+						    feature_size, omit_engine);
+			offset += (feature_size + REGDUMP_HEADER_SIZE);
+		} else {
+			DP_ERR(cdev, "qed_dbg_igu_fifo failed. rc = %d", rc);
+		}
+
+		/* protection_override dump */
+		rc = qed_dbg_protection_override(cdev, (u8 *)buffer + offset +
+						 REGDUMP_HEADER_SIZE,
+						 &feature_size);
+		if (!rc) {
+			*(u32 *)((u8 *)buffer + offset) =
+			    qed_calc_regdump_header(PROTECTION_OVERRIDE,
+						    cur_engine,
+						    feature_size, omit_engine);
+			offset += (feature_size + REGDUMP_HEADER_SIZE);
+		} else {
+			DP_ERR(cdev,
+			       "qed_dbg_protection_override failed. rc = %d\n",
+			       rc);
+		}
+
+		/* fw_asserts dump */
+		rc = qed_dbg_fw_asserts(cdev, (u8 *)buffer + offset +
+					REGDUMP_HEADER_SIZE, &feature_size);
+		if (!rc) {
+			*(u32 *)((u8 *)buffer + offset) =
+			    qed_calc_regdump_header(FW_ASSERTS, cur_engine,
+						    feature_size, omit_engine);
+			offset += (feature_size + REGDUMP_HEADER_SIZE);
+		} else {
+			DP_ERR(cdev, "qed_dbg_fw_asserts failed. rc = %d\n",
+			       rc);
+		}
+
+		/* GRC dump - must be last because when mcp stuck it will
+		 * clutter idle_chk, reg_fifo, ...
+		 */
+		rc = qed_dbg_grc(cdev, (u8 *)buffer + offset +
+				 REGDUMP_HEADER_SIZE, &feature_size);
+		if (!rc) {
+			*(u32 *)((u8 *)buffer + offset) =
+			    qed_calc_regdump_header(GRC_DUMP, cur_engine,
+						    feature_size, omit_engine);
+			offset += (feature_size + REGDUMP_HEADER_SIZE);
+		} else {
+			DP_ERR(cdev, "qed_dbg_grc failed. rc = %d", rc);
+		}
+	}
+
+	/* mcp_trace */
+	rc = qed_dbg_mcp_trace(cdev, (u8 *)buffer + offset +
+			       REGDUMP_HEADER_SIZE, &feature_size);
+	if (!rc) {
+		*(u32 *)((u8 *)buffer + offset) =
+		    qed_calc_regdump_header(MCP_TRACE, cur_engine,
+					    feature_size, omit_engine);
+		offset += (feature_size + REGDUMP_HEADER_SIZE);
+	} else {
+		DP_ERR(cdev, "qed_dbg_mcp_trace failed. rc = %d\n", rc);
+	}
+
+	qed_set_debug_engine(cdev, org_engine);
+
+	return 0;
+}
+
+int qed_dbg_all_data_size(struct qed_dev *cdev)
+{
+	u8 cur_engine, org_engine;
+	u32 regs_len = 0;
+
+	org_engine = qed_get_debug_engine(cdev);
+	for (cur_engine = 0; cur_engine < cdev->num_hwfns; cur_engine++) {
+		/* Engine specific */
+		DP_VERBOSE(cdev, QED_MSG_DEBUG,
+			   "calculating idle_chk and grcdump register length for current engine\n");
+		qed_set_debug_engine(cdev, cur_engine);
+		regs_len += REGDUMP_HEADER_SIZE + qed_dbg_idle_chk_size(cdev) +
+			    REGDUMP_HEADER_SIZE + qed_dbg_idle_chk_size(cdev) +
+			    REGDUMP_HEADER_SIZE + qed_dbg_grc_size(cdev) +
+			    REGDUMP_HEADER_SIZE + qed_dbg_reg_fifo_size(cdev) +
+			    REGDUMP_HEADER_SIZE + qed_dbg_igu_fifo_size(cdev) +
+			    REGDUMP_HEADER_SIZE +
+			    qed_dbg_protection_override_size(cdev) +
+			    REGDUMP_HEADER_SIZE + qed_dbg_fw_asserts_size(cdev);
+	}
+
+	/* Engine common */
+	regs_len += REGDUMP_HEADER_SIZE + qed_dbg_mcp_trace_size(cdev);
+	qed_set_debug_engine(cdev, org_engine);
+
+	return regs_len;
+}
+
+int qed_dbg_feature(struct qed_dev *cdev, void *buffer,
+		    enum qed_dbg_features feature, u32 *num_dumped_bytes)
+{
+	struct qed_hwfn *p_hwfn =
+		&cdev->hwfns[cdev->dbg_params.engine_for_debug];
+	struct qed_dbg_feature *qed_feature =
+		&cdev->dbg_params.features[feature];
+	enum dbg_status dbg_rc;
+	struct qed_ptt *p_ptt;
+	int rc = 0;
+
+	/* Acquire ptt */
+	p_ptt = qed_ptt_acquire(p_hwfn);
+	if (!p_ptt)
+		return -EINVAL;
+
+	/* Get dump */
+	dbg_rc = qed_dbg_dump(p_hwfn, p_ptt, feature);
+	if (dbg_rc != DBG_STATUS_OK) {
+		DP_VERBOSE(cdev, QED_MSG_DEBUG, "%s\n",
+			   qed_dbg_get_status_str(dbg_rc));
+		*num_dumped_bytes = 0;
+		rc = -EINVAL;
+		goto out;
+	}
+
+	DP_VERBOSE(cdev, QED_MSG_DEBUG,
+		   "copying debugfs feature to external buffer\n");
+	memcpy(buffer, qed_feature->dump_buf, qed_feature->buf_size);
+	*num_dumped_bytes = cdev->dbg_params.features[feature].dumped_dwords *
+			    4;
+
+out:
+	qed_ptt_release(p_hwfn, p_ptt);
+	return rc;
+}
+
+int qed_dbg_feature_size(struct qed_dev *cdev, enum qed_dbg_features feature)
+{
+	struct qed_hwfn *p_hwfn =
+		&cdev->hwfns[cdev->dbg_params.engine_for_debug];
+	struct qed_ptt *p_ptt = qed_ptt_acquire(p_hwfn);
+	struct qed_dbg_feature *qed_feature =
+		&cdev->dbg_params.features[feature];
+	u32 buf_size_dwords;
+	enum dbg_status rc;
+
+	if (!p_ptt)
+		return -EINVAL;
+
+	rc = qed_features_lookup[feature].get_size(p_hwfn, p_ptt,
+						   &buf_size_dwords);
+	if (rc != DBG_STATUS_OK)
+		buf_size_dwords = 0;
+
+	qed_ptt_release(p_hwfn, p_ptt);
+	qed_feature->buf_size = buf_size_dwords * sizeof(u32);
+	return qed_feature->buf_size;
+}
+
+u8 qed_get_debug_engine(struct qed_dev *cdev)
+{
+	return cdev->dbg_params.engine_for_debug;
+}
+
+void qed_set_debug_engine(struct qed_dev *cdev, int engine_number)
+{
+	DP_VERBOSE(cdev, QED_MSG_DEBUG, "set debug engine to %d\n",
+		   engine_number);
+	cdev->dbg_params.engine_for_debug = engine_number;
+}
+
+void qed_dbg_pf_init(struct qed_dev *cdev)
+{
+	const u8 *dbg_values;
+
+	/* Debug values are after init values.
+	 * The offset is the first dword of the file.
+	 */
+	dbg_values = cdev->firmware->data + *(u32 *)cdev->firmware->data;
+	qed_dbg_set_bin_ptr((u8 *)dbg_values);
+	qed_dbg_user_set_bin_ptr((u8 *)dbg_values);
+}
+
+void qed_dbg_pf_exit(struct qed_dev *cdev)
+{
+	struct qed_dbg_feature *feature = NULL;
+	enum qed_dbg_features feature_idx;
+
+	/* Debug features' buffers may be allocated if debug feature was used
+	 * but dump wasn't called.
+	 */
+	for (feature_idx = 0; feature_idx < DBG_FEATURE_NUM; feature_idx++) {
+		feature = &cdev->dbg_params.features[feature_idx];
+		if (feature->dump_buf) {
+			vfree(feature->dump_buf);
+			feature->dump_buf = NULL;
+		}
+	}
+}
diff --git a/drivers/net/ethernet/qlogic/qed/qed_debug.h b/drivers/net/ethernet/qlogic/qed/qed_debug.h
new file mode 100644
index 000000000000..f872d7324814
--- /dev/null
+++ b/drivers/net/ethernet/qlogic/qed/qed_debug.h
@@ -0,0 +1,54 @@
+/* QLogic qed NIC Driver
+ * Copyright (c) 2015 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#ifndef _QED_DEBUGFS_H
+#define _QED_DEBUGFS_H
+
+enum qed_dbg_features {
+	DBG_FEATURE_GRC,
+	DBG_FEATURE_IDLE_CHK,
+	DBG_FEATURE_MCP_TRACE,
+	DBG_FEATURE_REG_FIFO,
+	DBG_FEATURE_IGU_FIFO,
+	DBG_FEATURE_PROTECTION_OVERRIDE,
+	DBG_FEATURE_FW_ASSERTS,
+	DBG_FEATURE_NUM
+};
+
+int qed_dbg_grc(struct qed_dev *cdev, void *buffer, u32 *num_dumped_bytes);
+int qed_dbg_grc_size(struct qed_dev *cdev);
+int qed_dbg_idle_chk(struct qed_dev *cdev, void *buffer,
+		     u32 *num_dumped_bytes);
+int qed_dbg_idle_chk_size(struct qed_dev *cdev);
+int qed_dbg_reg_fifo(struct qed_dev *cdev, void *buffer,
+		     u32 *num_dumped_bytes);
+int qed_dbg_reg_fifo_size(struct qed_dev *cdev);
+int qed_dbg_igu_fifo(struct qed_dev *cdev, void *buffer,
+		     u32 *num_dumped_bytes);
+int qed_dbg_igu_fifo_size(struct qed_dev *cdev);
+int qed_dbg_protection_override(struct qed_dev *cdev, void *buffer,
+				u32 *num_dumped_bytes);
+int qed_dbg_protection_override_size(struct qed_dev *cdev);
+int qed_dbg_fw_asserts(struct qed_dev *cdev, void *buffer,
+		       u32 *num_dumped_bytes);
+int qed_dbg_fw_asserts_size(struct qed_dev *cdev);
+int qed_dbg_mcp_trace(struct qed_dev *cdev, void *buffer,
+		      u32 *num_dumped_bytes);
+int qed_dbg_mcp_trace_size(struct qed_dev *cdev);
+int qed_dbg_all_data(struct qed_dev *cdev, void *buffer);
+int qed_dbg_all_data_size(struct qed_dev *cdev);
+u8 qed_get_debug_engine(struct qed_dev *cdev);
+void qed_set_debug_engine(struct qed_dev *cdev, int engine_number);
+int qed_dbg_feature(struct qed_dev *cdev, void *buffer,
+		    enum qed_dbg_features feature, u32 *num_dumped_bytes);
+int qed_dbg_feature_size(struct qed_dev *cdev, enum qed_dbg_features feature);
+
+void qed_dbg_pf_init(struct qed_dev *cdev);
+void qed_dbg_pf_exit(struct qed_dev *cdev);
+
+#endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev.c b/drivers/net/ethernet/qlogic/qed/qed_dev.c
index 0e4f4a9306b5..754f6a908858 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dev.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dev.c
@@ -29,14 +29,18 @@
 #include "qed_hw.h"
 #include "qed_init_ops.h"
 #include "qed_int.h"
+#include "qed_ll2.h"
 #include "qed_mcp.h"
 #include "qed_reg_addr.h"
 #include "qed_sp.h"
 #include "qed_sriov.h"
 #include "qed_vf.h"
+#include "qed_roce.h"
 
-static spinlock_t qm_lock;
-static bool qm_lock_init = false;
+static DEFINE_SPINLOCK(qm_lock);
+
+#define QED_MIN_DPIS            (4)
+#define QED_MIN_PWM_REGION      (QED_WID_SIZE * QED_MIN_DPIS)
 
 /* API common to all protocols */
 enum BAR_ID {
@@ -44,8 +48,7 @@ enum BAR_ID {
 	BAR_ID_1        /* Used for doorbells */
 };
 
-static u32 qed_hw_bar_size(struct qed_hwfn	*p_hwfn,
-			   enum BAR_ID		bar_id)
+static u32 qed_hw_bar_size(struct qed_hwfn *p_hwfn, enum BAR_ID bar_id)
 {
 	u32 bar_reg = (bar_id == BAR_ID_0 ?
 		       PGLUE_B_REG_PF_BAR0_SIZE : PGLUE_B_REG_PF_BAR1_SIZE);
@@ -70,8 +73,7 @@ static u32 qed_hw_bar_size(struct qed_hwfn	*p_hwfn,
 	}
 }
 
-void qed_init_dp(struct qed_dev *cdev,
-		 u32 dp_module, u8 dp_level)
+void qed_init_dp(struct qed_dev *cdev, u32 dp_module, u8 dp_level)
 {
 	u32 i;
 
@@ -150,6 +152,9 @@ void qed_resc_free(struct qed_dev *cdev)
 		qed_eq_free(p_hwfn, p_hwfn->p_eq);
 		qed_consq_free(p_hwfn, p_hwfn->p_consq);
 		qed_int_free(p_hwfn);
+#ifdef CONFIG_QED_LL2
+		qed_ll2_free(p_hwfn, p_hwfn->p_ll2_info);
+#endif
 		qed_iov_free(p_hwfn);
 		qed_dmae_info_free(p_hwfn);
 		qed_dcbx_info_free(p_hwfn, p_hwfn->p_dcbx_info);
@@ -343,7 +348,6 @@ static int qed_init_qm_info(struct qed_hwfn *p_hwfn, bool b_sleepable)
 	return 0;
 
 alloc_err:
-	DP_NOTICE(p_hwfn, "Failed to allocate memory for QM params\n");
 	qed_qm_info_free(p_hwfn);
 	return -ENOMEM;
 }
@@ -407,6 +411,9 @@ int qed_qm_reconf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 
 int qed_resc_alloc(struct qed_dev *cdev)
 {
+#ifdef CONFIG_QED_LL2
+	struct qed_ll2_info *p_ll2_info;
+#endif
 	struct qed_consq *p_consq;
 	struct qed_eq *p_eq;
 	int i, rc = 0;
@@ -427,18 +434,12 @@ int qed_resc_alloc(struct qed_dev *cdev)
 				     RESC_NUM(p_hwfn, QED_L2_QUEUE);
 
 		p_hwfn->p_tx_cids = kzalloc(tx_size, GFP_KERNEL);
-		if (!p_hwfn->p_tx_cids) {
-			DP_NOTICE(p_hwfn,
-				  "Failed to allocate memory for Tx Cids\n");
+		if (!p_hwfn->p_tx_cids)
 			goto alloc_no_mem;
-		}
 
 		p_hwfn->p_rx_cids = kzalloc(rx_size, GFP_KERNEL);
-		if (!p_hwfn->p_rx_cids) {
-			DP_NOTICE(p_hwfn,
-				  "Failed to allocate memory for Rx Cids\n");
+		if (!p_hwfn->p_rx_cids)
 			goto alloc_no_mem;
-		}
 	}
 
 	for_each_hwfn(cdev, i) {
@@ -523,29 +524,29 @@ int qed_resc_alloc(struct qed_dev *cdev)
 			goto alloc_no_mem;
 		p_hwfn->p_consq = p_consq;
 
+#ifdef CONFIG_QED_LL2
+		if (p_hwfn->using_ll2) {
+			p_ll2_info = qed_ll2_alloc(p_hwfn);
+			if (!p_ll2_info)
+				goto alloc_no_mem;
+			p_hwfn->p_ll2_info = p_ll2_info;
+		}
+#endif
+
 		/* DMA info initialization */
 		rc = qed_dmae_info_alloc(p_hwfn);
-		if (rc) {
-			DP_NOTICE(p_hwfn,
-				  "Failed to allocate memory for dmae_info structure\n");
+		if (rc)
 			goto alloc_err;
-		}
 
 		/* DCBX initialization */
 		rc = qed_dcbx_info_alloc(p_hwfn);
-		if (rc) {
-			DP_NOTICE(p_hwfn,
-				  "Failed to allocate memory for dcbx structure\n");
+		if (rc)
 			goto alloc_err;
-		}
 	}
 
 	cdev->reset_stats = kzalloc(sizeof(*cdev->reset_stats), GFP_KERNEL);
-	if (!cdev->reset_stats) {
-		DP_NOTICE(cdev, "Failed to allocate reset statistics\n");
-		rc = -ENOMEM;
-		goto alloc_err;
-	}
+	if (!cdev->reset_stats)
+		goto alloc_no_mem;
 
 	return 0;
 
@@ -580,6 +581,10 @@ void qed_resc_setup(struct qed_dev *cdev)
 		qed_int_setup(p_hwfn, p_hwfn->p_main_ptt);
 
 		qed_iov_setup(p_hwfn, p_hwfn->p_main_ptt);
+#ifdef CONFIG_QED_LL2
+		if (p_hwfn->using_ll2)
+			qed_ll2_setup(p_hwfn, p_hwfn->p_ll2_info);
+#endif
 	}
 }
 
@@ -605,9 +610,8 @@ int qed_final_cleanup(struct qed_hwfn *p_hwfn,
 
 	/* Make sure notification is not set before initiating final cleanup */
 	if (REG_RD(p_hwfn, addr)) {
-		DP_NOTICE(
-			p_hwfn,
-			"Unexpected; Found final cleanup notification before initiating final cleanup\n");
+		DP_NOTICE(p_hwfn,
+			  "Unexpected; Found final cleanup notification before initiating final cleanup\n");
 		REG_WR(p_hwfn, addr, 0);
 	}
 
@@ -701,17 +705,14 @@ static void qed_init_cau_rt_data(struct qed_dev *cdev)
 				continue;
 
 			qed_init_cau_sb_entry(p_hwfn, &sb_entry,
-					      p_block->function_id,
-					      0, 0);
-			STORE_RT_REG_AGG(p_hwfn, offset + sb_id * 2,
-					 sb_entry);
+					      p_block->function_id, 0, 0);
+			STORE_RT_REG_AGG(p_hwfn, offset + sb_id * 2, sb_entry);
 		}
 	}
 }
 
 static int qed_hw_init_common(struct qed_hwfn *p_hwfn,
-			      struct qed_ptt *p_ptt,
-			      int hw_mode)
+			      struct qed_ptt *p_ptt, int hw_mode)
 {
 	struct qed_qm_info *qm_info = &p_hwfn->qm_info;
 	struct qed_qm_common_rt_init_params params;
@@ -759,7 +760,7 @@ static int qed_hw_init_common(struct qed_hwfn *p_hwfn,
 	qed_port_unpretend(p_hwfn, p_ptt);
 
 	rc = qed_init_run(p_hwfn, p_ptt, PHASE_ENGINE, ANY_PHASE_ID, hw_mode);
-	if (rc != 0)
+	if (rc)
 		return rc;
 
 	qed_wr(p_hwfn, p_ptt, PSWRQ2_REG_L2P_VALIDATE_VFID, 0);
@@ -780,6 +781,9 @@ static int qed_hw_init_common(struct qed_hwfn *p_hwfn,
 		concrete_fid = qed_vfid_to_concrete(p_hwfn, vf_id);
 		qed_fid_pretend(p_hwfn, p_ptt, (u16) concrete_fid);
 		qed_wr(p_hwfn, p_ptt, CCFC_REG_STRONG_ENABLE_VF, 0x1);
+		qed_wr(p_hwfn, p_ptt, CCFC_REG_WEAK_ENABLE_VF, 0x0);
+		qed_wr(p_hwfn, p_ptt, TCFC_REG_STRONG_ENABLE_VF, 0x1);
+		qed_wr(p_hwfn, p_ptt, TCFC_REG_WEAK_ENABLE_VF, 0x0);
 	}
 	/* pretend to original PF */
 	qed_fid_pretend(p_hwfn, p_ptt, p_hwfn->rel_pf_id);
@@ -787,38 +791,141 @@ static int qed_hw_init_common(struct qed_hwfn *p_hwfn,
 	return rc;
 }
 
-static int qed_hw_init_port(struct qed_hwfn *p_hwfn,
-			    struct qed_ptt *p_ptt,
-			    int hw_mode)
+static int
+qed_hw_init_dpi_size(struct qed_hwfn *p_hwfn,
+		     struct qed_ptt *p_ptt, u32 pwm_region_size, u32 n_cpus)
 {
-	int rc = 0;
+	u32 dpi_page_size_1, dpi_page_size_2, dpi_page_size;
+	u32 dpi_bit_shift, dpi_count;
+	u32 min_dpis;
 
-	rc = qed_init_run(p_hwfn, p_ptt, PHASE_PORT, p_hwfn->port_id, hw_mode);
-	if (rc != 0)
-		return rc;
+	/* Calculate DPI size */
+	dpi_page_size_1 = QED_WID_SIZE * n_cpus;
+	dpi_page_size_2 = max_t(u32, QED_WID_SIZE, PAGE_SIZE);
+	dpi_page_size = max_t(u32, dpi_page_size_1, dpi_page_size_2);
+	dpi_page_size = roundup_pow_of_two(dpi_page_size);
+	dpi_bit_shift = ilog2(dpi_page_size / 4096);
 
-	if (hw_mode & (1 << MODE_MF_SI)) {
-		u8 pf_id = 0;
+	dpi_count = pwm_region_size / dpi_page_size;
 
-		if (!qed_hw_init_first_eth(p_hwfn, p_ptt, &pf_id)) {
-			DP_VERBOSE(p_hwfn, NETIF_MSG_IFUP,
-				   "PF[%08x] is first eth on engine\n", pf_id);
+	min_dpis = p_hwfn->pf_params.rdma_pf_params.min_dpis;
+	min_dpis = max_t(u32, QED_MIN_DPIS, min_dpis);
 
-			/* We should have configured BIT for ppfid, i.e., the
-			 * relative function number in the port. But there's a
-			 * bug in LLH in BB where the ppfid is actually engine
-			 * based, so we need to take this into account.
-			 */
-			qed_wr(p_hwfn, p_ptt,
-			       NIG_REG_LLH_TAGMAC_DEF_PF_VECTOR, 1 << pf_id);
-		}
+	p_hwfn->dpi_size = dpi_page_size;
+	p_hwfn->dpi_count = dpi_count;
+
+	qed_wr(p_hwfn, p_ptt, DORQ_REG_PF_DPI_BIT_SHIFT, dpi_bit_shift);
+
+	if (dpi_count < min_dpis)
+		return -EINVAL;
+
+	return 0;
+}
+
+enum QED_ROCE_EDPM_MODE {
+	QED_ROCE_EDPM_MODE_ENABLE = 0,
+	QED_ROCE_EDPM_MODE_FORCE_ON = 1,
+	QED_ROCE_EDPM_MODE_DISABLE = 2,
+};
 
-		/* Take the protocol-based hit vector if there is a hit,
-		 * otherwise take the other vector.
+static int
+qed_hw_init_pf_doorbell_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	u32 pwm_regsize, norm_regsize;
+	u32 non_pwm_conn, min_addr_reg1;
+	u32 db_bar_size, n_cpus;
+	u32 roce_edpm_mode;
+	u32 pf_dems_shift;
+	int rc = 0;
+	u8 cond;
+
+	db_bar_size = qed_hw_bar_size(p_hwfn, BAR_ID_1);
+	if (p_hwfn->cdev->num_hwfns > 1)
+		db_bar_size /= 2;
+
+	/* Calculate doorbell regions */
+	non_pwm_conn = qed_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_CORE) +
+		       qed_cxt_get_proto_cid_count(p_hwfn, PROTOCOLID_CORE,
+						   NULL) +
+		       qed_cxt_get_proto_cid_count(p_hwfn, PROTOCOLID_ETH,
+						   NULL);
+	norm_regsize = roundup(QED_PF_DEMS_SIZE * non_pwm_conn, 4096);
+	min_addr_reg1 = norm_regsize / 4096;
+	pwm_regsize = db_bar_size - norm_regsize;
+
+	/* Check that the normal and PWM sizes are valid */
+	if (db_bar_size < norm_regsize) {
+		DP_ERR(p_hwfn->cdev,
+		       "Doorbell BAR size 0x%x is too small (normal region is 0x%0x )\n",
+		       db_bar_size, norm_regsize);
+		return -EINVAL;
+	}
+
+	if (pwm_regsize < QED_MIN_PWM_REGION) {
+		DP_ERR(p_hwfn->cdev,
+		       "PWM region size 0x%0x is too small. Should be at least 0x%0x (Doorbell BAR size is 0x%x and normal region size is 0x%0x)\n",
+		       pwm_regsize,
+		       QED_MIN_PWM_REGION, db_bar_size, norm_regsize);
+		return -EINVAL;
+	}
+
+	/* Calculate number of DPIs */
+	roce_edpm_mode = p_hwfn->pf_params.rdma_pf_params.roce_edpm_mode;
+	if ((roce_edpm_mode == QED_ROCE_EDPM_MODE_ENABLE) ||
+	    ((roce_edpm_mode == QED_ROCE_EDPM_MODE_FORCE_ON))) {
+		/* Either EDPM is mandatory, or we are attempting to allocate a
+		 * WID per CPU.
 		 */
-		qed_wr(p_hwfn, p_ptt, NIG_REG_LLH_CLS_TYPE_DUALMODE, 0x2);
+		n_cpus = num_active_cpus();
+		rc = qed_hw_init_dpi_size(p_hwfn, p_ptt, pwm_regsize, n_cpus);
 	}
-	return rc;
+
+	cond = (rc && (roce_edpm_mode == QED_ROCE_EDPM_MODE_ENABLE)) ||
+	       (roce_edpm_mode == QED_ROCE_EDPM_MODE_DISABLE);
+	if (cond || p_hwfn->dcbx_no_edpm) {
+		/* Either EDPM is disabled from user configuration, or it is
+		 * disabled via DCBx, or it is not mandatory and we failed to
+		 * allocated a WID per CPU.
+		 */
+		n_cpus = 1;
+		rc = qed_hw_init_dpi_size(p_hwfn, p_ptt, pwm_regsize, n_cpus);
+
+		if (cond)
+			qed_rdma_dpm_bar(p_hwfn, p_ptt);
+	}
+
+	DP_INFO(p_hwfn,
+		"doorbell bar: normal_region_size=%d, pwm_region_size=%d, dpi_size=%d, dpi_count=%d, roce_edpm=%s\n",
+		norm_regsize,
+		pwm_regsize,
+		p_hwfn->dpi_size,
+		p_hwfn->dpi_count,
+		((p_hwfn->dcbx_no_edpm) || (p_hwfn->db_bar_no_edpm)) ?
+		"disabled" : "enabled");
+
+	if (rc) {
+		DP_ERR(p_hwfn,
+		       "Failed to allocate enough DPIs. Allocated %d but the current minimum is %d.\n",
+		       p_hwfn->dpi_count,
+		       p_hwfn->pf_params.rdma_pf_params.min_dpis);
+		return -EINVAL;
+	}
+
+	p_hwfn->dpi_start_offset = norm_regsize;
+
+	/* DEMS size is configured log2 of DWORDs, hence the division by 4 */
+	pf_dems_shift = ilog2(QED_PF_DEMS_SIZE / 4);
+	qed_wr(p_hwfn, p_ptt, DORQ_REG_PF_ICID_BIT_SHIFT_NORM, pf_dems_shift);
+	qed_wr(p_hwfn, p_ptt, DORQ_REG_PF_MIN_ADDR_REG1, min_addr_reg1);
+
+	return 0;
+}
+
+static int qed_hw_init_port(struct qed_hwfn *p_hwfn,
+			    struct qed_ptt *p_ptt, int hw_mode)
+{
+	return qed_init_run(p_hwfn, p_ptt, PHASE_PORT,
+			    p_hwfn->port_id, hw_mode);
 }
 
 static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
@@ -848,7 +955,7 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
 	qed_int_igu_init_rt(p_hwfn);
 
 	/* Set VLAN in NIG if needed */
-	if (hw_mode & (1 << MODE_MF_SD)) {
+	if (hw_mode & BIT(MODE_MF_SD)) {
 		DP_VERBOSE(p_hwfn, NETIF_MSG_HW, "Configuring LLH_FUNC_TAG\n");
 		STORE_RT_REG(p_hwfn, NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET, 1);
 		STORE_RT_REG(p_hwfn, NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET,
@@ -856,7 +963,7 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
 	}
 
 	/* Enable classification by MAC if needed */
-	if (hw_mode & (1 << MODE_MF_SI)) {
+	if (hw_mode & BIT(MODE_MF_SI)) {
 		DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
 			   "Configuring TAGMAC_CLS_TYPE\n");
 		STORE_RT_REG(p_hwfn,
@@ -871,7 +978,7 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
 
 	/* Cleanup chip from previous driver if such remains exist */
 	rc = qed_final_cleanup(p_hwfn, p_ptt, rel_pf_id, false);
-	if (rc != 0)
+	if (rc)
 		return rc;
 
 	/* PF Init sequence */
@@ -887,20 +994,9 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
 	/* Pure runtime initializations - directly to the HW  */
 	qed_int_igu_init_pure_rt(p_hwfn, p_ptt, true, true);
 
-	if (hw_mode & (1 << MODE_MF_SI)) {
-		u8 pf_id = 0;
-		u32 val = 0;
-
-		if (!qed_hw_init_first_eth(p_hwfn, p_ptt, &pf_id)) {
-			if (p_hwfn->rel_pf_id == pf_id) {
-				DP_VERBOSE(p_hwfn, NETIF_MSG_IFUP,
-					   "PF[%d] is first ETH on engine\n",
-					   pf_id);
-				val = 1;
-			}
-			qed_wr(p_hwfn, p_ptt, PRS_REG_MSG_INFO, val);
-		}
-	}
+	rc = qed_hw_init_pf_doorbell_bar(p_hwfn, p_ptt);
+	if (rc)
+		return rc;
 
 	if (b_hw_start) {
 		/* enable interrupts */
@@ -950,8 +1046,7 @@ static void qed_reset_mb_shadow(struct qed_hwfn *p_hwfn,
 	/* Read shadow of current MFW mailbox */
 	qed_mcp_read_mb(p_hwfn, p_main_ptt);
 	memcpy(p_hwfn->mcp_info->mfw_mb_shadow,
-	       p_hwfn->mcp_info->mfw_mb_cur,
-	       p_hwfn->mcp_info->mfw_mb_length);
+	       p_hwfn->mcp_info->mfw_mb_cur, p_hwfn->mcp_info->mfw_mb_length);
 }
 
 int qed_hw_init(struct qed_dev *cdev,
@@ -971,7 +1066,7 @@ int qed_hw_init(struct qed_dev *cdev,
 
 	if (IS_PF(cdev)) {
 		rc = qed_init_fw_data(cdev, bin_fw_data);
-		if (rc != 0)
+		if (rc)
 			return rc;
 	}
 
@@ -988,8 +1083,7 @@ int qed_hw_init(struct qed_dev *cdev,
 
 		qed_calc_hw_mode(p_hwfn);
 
-		rc = qed_mcp_load_req(p_hwfn, p_hwfn->p_main_ptt,
-				      &load_code);
+		rc = qed_mcp_load_req(p_hwfn, p_hwfn->p_main_ptt, &load_code);
 		if (rc) {
 			DP_NOTICE(p_hwfn, "Failed sending LOAD_REQ command\n");
 			return rc;
@@ -1004,11 +1098,6 @@ int qed_hw_init(struct qed_dev *cdev,
 		p_hwfn->first_on_engine = (load_code ==
 					   FW_MSG_CODE_DRV_LOAD_ENGINE);
 
-		if (!qm_lock_init) {
-			spin_lock_init(&qm_lock);
-			qm_lock_init = true;
-		}
-
 		switch (load_code) {
 		case FW_MSG_CODE_DRV_LOAD_ENGINE:
 			rc = qed_hw_init_common(p_hwfn, p_hwfn->p_main_ptt,
@@ -1071,9 +1160,8 @@ int qed_hw_init(struct qed_dev *cdev,
 }
 
 #define QED_HW_STOP_RETRY_LIMIT (10)
-static inline void qed_hw_timers_stop(struct qed_dev *cdev,
-				      struct qed_hwfn *p_hwfn,
-				      struct qed_ptt *p_ptt)
+static void qed_hw_timers_stop(struct qed_dev *cdev,
+			       struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	int i;
 
@@ -1084,8 +1172,7 @@ static inline void qed_hw_timers_stop(struct qed_dev *cdev,
 	for (i = 0; i < QED_HW_STOP_RETRY_LIMIT; i++) {
 		if ((!qed_rd(p_hwfn, p_ptt,
 			     TM_REG_PF_SCAN_ACTIVE_CONN)) &&
-		    (!qed_rd(p_hwfn, p_ptt,
-			     TM_REG_PF_SCAN_ACTIVE_TASK)))
+		    (!qed_rd(p_hwfn, p_ptt, TM_REG_PF_SCAN_ACTIVE_TASK)))
 			break;
 
 		/* Dependent on number of connection/tasks, possibly
@@ -1190,8 +1277,7 @@ void qed_hw_stop_fastpath(struct qed_dev *cdev)
 		}
 
 		DP_VERBOSE(p_hwfn,
-			   NETIF_MSG_IFDOWN,
-			   "Shutting down the fastpath\n");
+			   NETIF_MSG_IFDOWN, "Shutting down the fastpath\n");
 
 		qed_wr(p_hwfn, p_ptt,
 		       NIG_REG_RX_LLH_BRB_GATE_DNTFWD_PERPF, 0x1);
@@ -1219,14 +1305,13 @@ void qed_hw_start_fastpath(struct qed_hwfn *p_hwfn)
 	       NIG_REG_RX_LLH_BRB_GATE_DNTFWD_PERPF, 0x0);
 }
 
-static int qed_reg_assert(struct qed_hwfn *hwfn,
-			  struct qed_ptt *ptt, u32 reg,
-			  bool expected)
+static int qed_reg_assert(struct qed_hwfn *p_hwfn,
+			  struct qed_ptt *p_ptt, u32 reg, bool expected)
 {
-	u32 assert_val = qed_rd(hwfn, ptt, reg);
+	u32 assert_val = qed_rd(p_hwfn, p_ptt, reg);
 
 	if (assert_val != expected) {
-		DP_NOTICE(hwfn, "Value at address 0x%x != 0x%08x\n",
+		DP_NOTICE(p_hwfn, "Value at address 0x%08x != 0x%08x\n",
 			  reg, expected);
 		return -EINVAL;
 	}
@@ -1306,8 +1391,7 @@ static void qed_hw_hwfn_prepare(struct qed_hwfn *p_hwfn)
 
 	/* Clean Previous errors if such exist */
 	qed_wr(p_hwfn, p_hwfn->p_main_ptt,
-	       PGLUE_B_REG_WAS_ERROR_PF_31_0_CLR,
-	       1 << p_hwfn->abs_pf_id);
+	       PGLUE_B_REG_WAS_ERROR_PF_31_0_CLR, 1 << p_hwfn->abs_pf_id);
 
 	/* enable internal target-read */
 	qed_wr(p_hwfn, p_hwfn->p_main_ptt,
@@ -1317,7 +1401,8 @@ static void qed_hw_hwfn_prepare(struct qed_hwfn *p_hwfn)
 static void get_function_id(struct qed_hwfn *p_hwfn)
 {
 	/* ME Register */
-	p_hwfn->hw_info.opaque_fid = (u16)REG_RD(p_hwfn, PXP_PF_ME_OPAQUE_ADDR);
+	p_hwfn->hw_info.opaque_fid = (u16) REG_RD(p_hwfn,
+						  PXP_PF_ME_OPAQUE_ADDR);
 
 	p_hwfn->hw_info.concrete_fid = REG_RD(p_hwfn, PXP_PF_ME_CONCRETE_ADDR);
 
@@ -1326,6 +1411,10 @@ static void get_function_id(struct qed_hwfn *p_hwfn)
 				      PXP_CONCRETE_FID_PFID);
 	p_hwfn->port_id = GET_FIELD(p_hwfn->hw_info.concrete_fid,
 				    PXP_CONCRETE_FID_PORT);
+
+	DP_VERBOSE(p_hwfn, NETIF_MSG_PROBE,
+		   "Read ME register: Concrete 0x%08x Opaque 0x%04x\n",
+		   p_hwfn->hw_info.concrete_fid, p_hwfn->hw_info.opaque_fid);
 }
 
 static void qed_hw_set_feat(struct qed_hwfn *p_hwfn)
@@ -1333,6 +1422,19 @@ static void qed_hw_set_feat(struct qed_hwfn *p_hwfn)
 	u32 *feat_num = p_hwfn->hw_info.feat_num;
 	int num_features = 1;
 
+#if IS_ENABLED(CONFIG_INFINIBAND_QEDR)
+	/* Roce CNQ each requires: 1 status block + 1 CNQ. We divide the
+	 * status blocks equally between L2 / RoCE but with consideration as
+	 * to how many l2 queues / cnqs we have
+	 */
+	if (p_hwfn->hw_info.personality == QED_PCI_ETH_ROCE) {
+		num_features++;
+
+		feat_num[QED_RDMA_CNQ] =
+			min_t(u32, RESC_NUM(p_hwfn, QED_SB) / num_features,
+			      RESC_NUM(p_hwfn, QED_RDMA_CNQ_RAM));
+	}
+#endif
 	feat_num[QED_PF_L2_QUE] = min_t(u32, RESC_NUM(p_hwfn, QED_SB) /
 						num_features,
 					RESC_NUM(p_hwfn, QED_L2_QUEUE));
@@ -1373,6 +1475,10 @@ static int qed_hw_get_resc(struct qed_hwfn *p_hwfn)
 	resc_num[QED_VLAN] = (ETH_NUM_VLAN_FILTERS - 1 /*For vlan0*/) /
 			     num_funcs;
 	resc_num[QED_ILT] = PXP_NUM_ILT_RECORDS_BB / num_funcs;
+	resc_num[QED_LL2_QUEUE] = MAX_NUM_LL2_RX_QUEUES / num_funcs;
+	resc_num[QED_RDMA_CNQ_RAM] = NUM_OF_CMDQS_CQS / num_funcs;
+	resc_num[QED_RDMA_STATS_QUEUE] = RDMA_NUM_STATISTIC_COUNTERS_BB /
+					 num_funcs;
 
 	for (i = 0; i < QED_MAX_RESC; i++)
 		resc_start[i] = resc_num[i] * enabled_func_idx;
@@ -1396,7 +1502,8 @@ static int qed_hw_get_resc(struct qed_hwfn *p_hwfn)
 		   "RL = %d start = %d\n"
 		   "MAC = %d start = %d\n"
 		   "VLAN = %d start = %d\n"
-		   "ILT = %d start = %d\n",
+		   "ILT = %d start = %d\n"
+		   "LL2_QUEUE = %d start = %d\n",
 		   p_hwfn->hw_info.resc_num[QED_SB],
 		   p_hwfn->hw_info.resc_start[QED_SB],
 		   p_hwfn->hw_info.resc_num[QED_L2_QUEUE],
@@ -1412,13 +1519,14 @@ static int qed_hw_get_resc(struct qed_hwfn *p_hwfn)
 		   p_hwfn->hw_info.resc_num[QED_VLAN],
 		   p_hwfn->hw_info.resc_start[QED_VLAN],
 		   p_hwfn->hw_info.resc_num[QED_ILT],
-		   p_hwfn->hw_info.resc_start[QED_ILT]);
+		   p_hwfn->hw_info.resc_start[QED_ILT],
+		   RESC_NUM(p_hwfn, QED_LL2_QUEUE),
+		   RESC_START(p_hwfn, QED_LL2_QUEUE));
 
 	return 0;
 }
 
-static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn,
-			       struct qed_ptt *p_ptt)
+static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	u32 nvm_cfg1_offset, mf_mode, addr, generic_cont0, core_cfg;
 	u32 port_cfg_addr, link_temp, nvm_cfg_addr, device_capabilities;
@@ -1472,8 +1580,7 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn,
 		p_hwfn->hw_info.port_mode = QED_PORT_MODE_DE_1X25G;
 		break;
 	default:
-		DP_NOTICE(p_hwfn, "Unknown port mode in 0x%08x\n",
-			  core_cfg);
+		DP_NOTICE(p_hwfn, "Unknown port mode in 0x%08x\n", core_cfg);
 		break;
 	}
 
@@ -1484,11 +1591,11 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn,
 	link_temp = qed_rd(p_hwfn, p_ptt,
 			   port_cfg_addr +
 			   offsetof(struct nvm_cfg1_port, speed_cap_mask));
-	link->speed.advertised_speeds =
-		link_temp & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_MASK;
+	link_temp &= NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_MASK;
+	link->speed.advertised_speeds = link_temp;
 
-	p_hwfn->mcp_info->link_capabilities.speed_capabilities =
-						link->speed.advertised_speeds;
+	link_temp = link->speed.advertised_speeds;
+	p_hwfn->mcp_info->link_capabilities.speed_capabilities = link_temp;
 
 	link_temp = qed_rd(p_hwfn, p_ptt,
 			   port_cfg_addr +
@@ -1517,8 +1624,7 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn,
 		link->speed.forced_speed = 100000;
 		break;
 	default:
-		DP_NOTICE(p_hwfn, "Unknown Speed in 0x%08x\n",
-			  link_temp);
+		DP_NOTICE(p_hwfn, "Unknown Speed in 0x%08x\n", link_temp);
 	}
 
 	link_temp &= NVM_CFG1_PORT_DRV_FLOW_CONTROL_MASK;
@@ -1628,10 +1734,10 @@ static void qed_get_num_funcs(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 
 	DP_VERBOSE(p_hwfn,
 		   NETIF_MSG_PROBE,
-		   "PF [rel_id %d, abs_id %d] within the %d enabled functions on the engine\n",
+		   "PF [rel_id %d, abs_id %d] occupies index %d within the %d enabled functions on the engine\n",
 		   p_hwfn->rel_pf_id,
 		   p_hwfn->abs_pf_id,
-		   p_hwfn->num_funcs_on_engine);
+		   p_hwfn->enabled_func_idx, p_hwfn->num_funcs_on_engine);
 }
 
 static int
@@ -1703,10 +1809,9 @@ static int qed_get_dev_info(struct qed_dev *cdev)
 	u32 tmp;
 
 	/* Read Vendor Id / Device Id */
-	pci_read_config_word(cdev->pdev, PCI_VENDOR_ID,
-			     &cdev->vendor_id);
-	pci_read_config_word(cdev->pdev, PCI_DEVICE_ID,
-			     &cdev->device_id);
+	pci_read_config_word(cdev->pdev, PCI_VENDOR_ID, &cdev->vendor_id);
+	pci_read_config_word(cdev->pdev, PCI_DEVICE_ID, &cdev->device_id);
+
 	cdev->chip_num = (u16)qed_rd(p_hwfn, p_hwfn->p_main_ptt,
 				     MISCS_REG_CHIP_NUM);
 	cdev->chip_rev = (u16)qed_rd(p_hwfn, p_hwfn->p_main_ptt,
@@ -1771,10 +1876,8 @@ static int qed_hw_prepare_single(struct qed_hwfn *p_hwfn,
 
 	/* Allocate PTT pool */
 	rc = qed_ptt_pool_alloc(p_hwfn);
-	if (rc) {
-		DP_NOTICE(p_hwfn, "Failed to prepare hwfn's hw\n");
+	if (rc)
 		goto err0;
-	}
 
 	/* Allocate the main PTT */
 	p_hwfn->p_main_ptt = qed_get_reserved_ptt(p_hwfn, RESERVED_PTT_MAIN);
@@ -1782,7 +1885,7 @@ static int qed_hw_prepare_single(struct qed_hwfn *p_hwfn,
 	/* First hwfn learns basic information, e.g., number of hwfns */
 	if (!p_hwfn->my_id) {
 		rc = qed_get_dev_info(p_hwfn->cdev);
-		if (rc != 0)
+		if (rc)
 			goto err1;
 	}
 
@@ -1804,10 +1907,8 @@ static int qed_hw_prepare_single(struct qed_hwfn *p_hwfn,
 
 	/* Allocate the init RT array and initialize the init-ops engine */
 	rc = qed_init_alloc(p_hwfn);
-	if (rc) {
-		DP_NOTICE(p_hwfn, "Failed to allocate the init array\n");
+	if (rc)
 		goto err2;
-	}
 
 	return rc;
 err2:
@@ -2015,10 +2116,8 @@ qed_chain_alloc_next_ptr(struct qed_dev *cdev, struct qed_chain *p_chain)
 		p_virt = dma_alloc_coherent(&cdev->pdev->dev,
 					    QED_CHAIN_PAGE_SIZE,
 					    &p_phys, GFP_KERNEL);
-		if (!p_virt) {
-			DP_NOTICE(cdev, "Failed to allocate chain memory\n");
+		if (!p_virt)
 			return -ENOMEM;
-		}
 
 		if (i == 0) {
 			qed_chain_init_mem(p_chain, p_virt, p_phys);
@@ -2048,10 +2147,8 @@ qed_chain_alloc_single(struct qed_dev *cdev, struct qed_chain *p_chain)
 
 	p_virt = dma_alloc_coherent(&cdev->pdev->dev,
 				    QED_CHAIN_PAGE_SIZE, &p_phys, GFP_KERNEL);
-	if (!p_virt) {
-		DP_NOTICE(cdev, "Failed to allocate chain memory\n");
+	if (!p_virt)
 		return -ENOMEM;
-	}
 
 	qed_chain_init_mem(p_chain, p_virt, p_phys);
 	qed_chain_reset(p_chain);
@@ -2068,13 +2165,9 @@ static int qed_chain_alloc_pbl(struct qed_dev *cdev, struct qed_chain *p_chain)
 	void *p_virt = NULL;
 
 	size = page_cnt * sizeof(*pp_virt_addr_tbl);
-	pp_virt_addr_tbl = vmalloc(size);
-	if (!pp_virt_addr_tbl) {
-		DP_NOTICE(cdev,
-			  "Failed to allocate memory for the chain virtual addresses table\n");
+	pp_virt_addr_tbl = vzalloc(size);
+	if (!pp_virt_addr_tbl)
 		return -ENOMEM;
-	}
-	memset(pp_virt_addr_tbl, 0, size);
 
 	/* The allocation of the PBL table is done with its full size, since it
 	 * is expected to be successive.
@@ -2087,19 +2180,15 @@ static int qed_chain_alloc_pbl(struct qed_dev *cdev, struct qed_chain *p_chain)
 					size, &p_pbl_phys, GFP_KERNEL);
 	qed_chain_init_pbl_mem(p_chain, p_pbl_virt, p_pbl_phys,
 			       pp_virt_addr_tbl);
-	if (!p_pbl_virt) {
-		DP_NOTICE(cdev, "Failed to allocate chain pbl memory\n");
+	if (!p_pbl_virt)
 		return -ENOMEM;
-	}
 
 	for (i = 0; i < page_cnt; i++) {
 		p_virt = dma_alloc_coherent(&cdev->pdev->dev,
 					    QED_CHAIN_PAGE_SIZE,
 					    &p_phys, GFP_KERNEL);
-		if (!p_virt) {
-			DP_NOTICE(cdev, "Failed to allocate chain memory\n");
+		if (!p_virt)
 			return -ENOMEM;
-		}
 
 		if (i == 0) {
 			qed_chain_init_mem(p_chain, p_virt, p_phys);
@@ -2134,7 +2223,8 @@ int qed_chain_alloc(struct qed_dev *cdev,
 	rc = qed_chain_alloc_sanity_check(cdev, cnt_type, elem_size, page_cnt);
 	if (rc) {
 		DP_NOTICE(cdev,
-			  "Cannot allocate a chain with the given arguments:\n"
+			  "Cannot allocate a chain with the given arguments:\n");
+		DP_NOTICE(cdev,
 			  "[use_mode %d, mode %d, cnt_type %d, num_elems %d, elem_size %zu]\n",
 			  intended_use, mode, cnt_type, num_elems, elem_size);
 		return rc;
@@ -2183,8 +2273,7 @@ int qed_fw_l2_queue(struct qed_hwfn *p_hwfn, u16 src_id, u16 *dst_id)
 	return 0;
 }
 
-int qed_fw_vport(struct qed_hwfn *p_hwfn,
-		 u8 src_id, u8 *dst_id)
+int qed_fw_vport(struct qed_hwfn *p_hwfn, u8 src_id, u8 *dst_id)
 {
 	if (src_id >= RESC_NUM(p_hwfn, QED_VPORT)) {
 		u8 min, max;
@@ -2203,8 +2292,7 @@ int qed_fw_vport(struct qed_hwfn *p_hwfn,
 	return 0;
 }
 
-int qed_fw_rss_eng(struct qed_hwfn *p_hwfn,
-		   u8 src_id, u8 *dst_id)
+int qed_fw_rss_eng(struct qed_hwfn *p_hwfn, u8 src_id, u8 *dst_id)
 {
 	if (src_id >= RESC_NUM(p_hwfn, QED_RSS_ENG)) {
 		u8 min, max;
@@ -2223,6 +2311,98 @@ int qed_fw_rss_eng(struct qed_hwfn *p_hwfn,
 	return 0;
 }
 
+static void qed_llh_mac_to_filter(u32 *p_high, u32 *p_low,
+				  u8 *p_filter)
+{
+	*p_high = p_filter[1] | (p_filter[0] << 8);
+	*p_low = p_filter[5] | (p_filter[4] << 8) |
+		 (p_filter[3] << 16) | (p_filter[2] << 24);
+}
+
+int qed_llh_add_mac_filter(struct qed_hwfn *p_hwfn,
+			   struct qed_ptt *p_ptt, u8 *p_filter)
+{
+	u32 high = 0, low = 0, en;
+	int i;
+
+	if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn)))
+		return 0;
+
+	qed_llh_mac_to_filter(&high, &low, p_filter);
+
+	/* Find a free entry and utilize it */
+	for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) {
+		en = qed_rd(p_hwfn, p_ptt,
+			    NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32));
+		if (en)
+			continue;
+		qed_wr(p_hwfn, p_ptt,
+		       NIG_REG_LLH_FUNC_FILTER_VALUE +
+		       2 * i * sizeof(u32), low);
+		qed_wr(p_hwfn, p_ptt,
+		       NIG_REG_LLH_FUNC_FILTER_VALUE +
+		       (2 * i + 1) * sizeof(u32), high);
+		qed_wr(p_hwfn, p_ptt,
+		       NIG_REG_LLH_FUNC_FILTER_MODE + i * sizeof(u32), 0);
+		qed_wr(p_hwfn, p_ptt,
+		       NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE +
+		       i * sizeof(u32), 0);
+		qed_wr(p_hwfn, p_ptt,
+		       NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 1);
+		break;
+	}
+	if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE) {
+		DP_NOTICE(p_hwfn,
+			  "Failed to find an empty LLH filter to utilize\n");
+		return -EINVAL;
+	}
+
+	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
+		   "mac: %pM is added at %d\n",
+		   p_filter, i);
+
+	return 0;
+}
+
+void qed_llh_remove_mac_filter(struct qed_hwfn *p_hwfn,
+			       struct qed_ptt *p_ptt, u8 *p_filter)
+{
+	u32 high = 0, low = 0;
+	int i;
+
+	if (!(IS_MF_SI(p_hwfn) || IS_MF_DEFAULT(p_hwfn)))
+		return;
+
+	qed_llh_mac_to_filter(&high, &low, p_filter);
+
+	/* Find the entry and clean it */
+	for (i = 0; i < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; i++) {
+		if (qed_rd(p_hwfn, p_ptt,
+			   NIG_REG_LLH_FUNC_FILTER_VALUE +
+			   2 * i * sizeof(u32)) != low)
+			continue;
+		if (qed_rd(p_hwfn, p_ptt,
+			   NIG_REG_LLH_FUNC_FILTER_VALUE +
+			   (2 * i + 1) * sizeof(u32)) != high)
+			continue;
+
+		qed_wr(p_hwfn, p_ptt,
+		       NIG_REG_LLH_FUNC_FILTER_EN + i * sizeof(u32), 0);
+		qed_wr(p_hwfn, p_ptt,
+		       NIG_REG_LLH_FUNC_FILTER_VALUE + 2 * i * sizeof(u32), 0);
+		qed_wr(p_hwfn, p_ptt,
+		       NIG_REG_LLH_FUNC_FILTER_VALUE +
+		       (2 * i + 1) * sizeof(u32), 0);
+
+		DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
+			   "mac: %pM is removed from %d\n",
+			   p_filter, i);
+		break;
+	}
+	if (i >= NIG_REG_LLH_FUNC_FILTER_EN_SIZE)
+		DP_NOTICE(p_hwfn, "Tried to remove a non-configured filter\n");
+}
+
 static int qed_set_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 			    u32 hw_addr, void *p_eth_qzone,
 			    size_t eth_qzone_size, u8 timeset)
@@ -2386,8 +2566,7 @@ static void qed_disable_wfq_for_all_vports(struct qed_hwfn *p_hwfn,
  * 3. total_req_min_rate [all vports min rate sum] shouldn't exceed min_pf_rate.
  */
 static int qed_init_wfq_param(struct qed_hwfn *p_hwfn,
-			      u16 vport_id, u32 req_rate,
-			      u32 min_pf_rate)
+			      u16 vport_id, u32 req_rate, u32 min_pf_rate)
 {
 	u32 total_req_min_rate = 0, total_left_rate = 0, left_rate_per_vp = 0;
 	int non_requested_count = 0, req_count = 0, i, num_vports;
@@ -2471,7 +2650,7 @@ static int __qed_configure_vport_wfq(struct qed_hwfn *p_hwfn,
 
 	rc = qed_init_wfq_param(p_hwfn, vp_id, rate, p_link->min_pf_rate);
 
-	if (rc == 0)
+	if (!rc)
 		qed_configure_wfq_for_all_vports(p_hwfn, p_ptt,
 						 p_link->min_pf_rate);
 	else
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev_api.h b/drivers/net/ethernet/qlogic/qed/qed_dev_api.h
index 343bb0344f62..b6711c106597 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dev_api.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_dev_api.h
@@ -310,6 +310,26 @@ int qed_fw_rss_eng(struct qed_hwfn *p_hwfn,
 		   u8 *dst_id);
 
 /**
+ * @brief qed_llh_add_mac_filter - configures a MAC filter in llh
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ * @param p_filter - MAC to add
+ */
+int qed_llh_add_mac_filter(struct qed_hwfn *p_hwfn,
+			   struct qed_ptt *p_ptt, u8 *p_filter);
+
+/**
+ * @brief qed_llh_remove_mac_filter - removes a MAC filter from llh
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ * @param p_filter - MAC to remove
+ */
+void qed_llh_remove_mac_filter(struct qed_hwfn *p_hwfn,
+			       struct qed_ptt *p_ptt, u8 *p_filter);
+
+/**
  * *@brief Cleanup of previous driver remains prior to load
  *
  * @param p_hwfn
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hsi.h b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
index 592784019994..72eee29c677f 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hsi.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
@@ -536,6 +536,247 @@ struct core_conn_context {
 	struct regpair ustorm_st_padding[2];
 };
 
+enum core_error_handle {
+	LL2_DROP_PACKET,
+	LL2_DO_NOTHING,
+	LL2_ASSERT,
+	MAX_CORE_ERROR_HANDLE
+};
+
+enum core_event_opcode {
+	CORE_EVENT_TX_QUEUE_START,
+	CORE_EVENT_TX_QUEUE_STOP,
+	CORE_EVENT_RX_QUEUE_START,
+	CORE_EVENT_RX_QUEUE_STOP,
+	MAX_CORE_EVENT_OPCODE
+};
+
+enum core_l4_pseudo_checksum_mode {
+	CORE_L4_PSEUDO_CSUM_CORRECT_LENGTH,
+	CORE_L4_PSEUDO_CSUM_ZERO_LENGTH,
+	MAX_CORE_L4_PSEUDO_CHECKSUM_MODE
+};
+
+struct core_ll2_port_stats {
+	struct regpair gsi_invalid_hdr;
+	struct regpair gsi_invalid_pkt_length;
+	struct regpair gsi_unsupported_pkt_typ;
+	struct regpair gsi_crcchksm_error;
+};
+
+struct core_ll2_pstorm_per_queue_stat {
+	struct regpair sent_ucast_bytes;
+	struct regpair sent_mcast_bytes;
+	struct regpair sent_bcast_bytes;
+	struct regpair sent_ucast_pkts;
+	struct regpair sent_mcast_pkts;
+	struct regpair sent_bcast_pkts;
+};
+
+struct core_ll2_rx_prod {
+	__le16 bd_prod;
+	__le16 cqe_prod;
+	__le32 reserved;
+};
+
+struct core_ll2_tstorm_per_queue_stat {
+	struct regpair packet_too_big_discard;
+	struct regpair no_buff_discard;
+};
+
+struct core_ll2_ustorm_per_queue_stat {
+	struct regpair rcv_ucast_bytes;
+	struct regpair rcv_mcast_bytes;
+	struct regpair rcv_bcast_bytes;
+	struct regpair rcv_ucast_pkts;
+	struct regpair rcv_mcast_pkts;
+	struct regpair rcv_bcast_pkts;
+};
+
+enum core_ramrod_cmd_id {
+	CORE_RAMROD_UNUSED,
+	CORE_RAMROD_RX_QUEUE_START,
+	CORE_RAMROD_TX_QUEUE_START,
+	CORE_RAMROD_RX_QUEUE_STOP,
+	CORE_RAMROD_TX_QUEUE_STOP,
+	MAX_CORE_RAMROD_CMD_ID
+};
+
+enum core_roce_flavor_type {
+	CORE_ROCE,
+	CORE_RROCE,
+	MAX_CORE_ROCE_FLAVOR_TYPE
+};
+
+struct core_rx_action_on_error {
+	u8 error_type;
+#define CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG_MASK	0x3
+#define CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG_SHIFT 0
+#define CORE_RX_ACTION_ON_ERROR_NO_BUFF_MASK	0x3
+#define CORE_RX_ACTION_ON_ERROR_NO_BUFF_SHIFT	2
+#define CORE_RX_ACTION_ON_ERROR_RESERVED_MASK	0xF
+#define CORE_RX_ACTION_ON_ERROR_RESERVED_SHIFT	4
+};
+
+struct core_rx_bd {
+	struct regpair addr;
+	__le16 reserved[4];
+};
+
+struct core_rx_bd_with_buff_len {
+	struct regpair addr;
+	__le16 buff_length;
+	__le16 reserved[3];
+};
+
+union core_rx_bd_union {
+	struct core_rx_bd rx_bd;
+	struct core_rx_bd_with_buff_len rx_bd_with_len;
+};
+
+struct core_rx_cqe_opaque_data {
+	__le32 data[2];
+};
+
+enum core_rx_cqe_type {
+	CORE_RX_CQE_ILLIGAL_TYPE,
+	CORE_RX_CQE_TYPE_REGULAR,
+	CORE_RX_CQE_TYPE_GSI_OFFLOAD,
+	CORE_RX_CQE_TYPE_SLOW_PATH,
+	MAX_CORE_RX_CQE_TYPE
+};
+
+struct core_rx_fast_path_cqe {
+	u8 type;
+	u8 placement_offset;
+	struct parsing_and_err_flags parse_flags;
+	__le16 packet_length;
+	__le16 vlan;
+	struct core_rx_cqe_opaque_data opaque_data;
+	__le32 reserved[4];
+};
+
+struct core_rx_gsi_offload_cqe {
+	u8 type;
+	u8 data_length_error;
+	struct parsing_and_err_flags parse_flags;
+	__le16 data_length;
+	__le16 vlan;
+	__le32 src_mac_addrhi;
+	__le16 src_mac_addrlo;
+	u8 reserved1[2];
+	__le32 gid_dst[4];
+};
+
+struct core_rx_slow_path_cqe {
+	u8 type;
+	u8 ramrod_cmd_id;
+	__le16 echo;
+	__le32 reserved1[7];
+};
+
+union core_rx_cqe_union {
+	struct core_rx_fast_path_cqe rx_cqe_fp;
+	struct core_rx_gsi_offload_cqe rx_cqe_gsi;
+	struct core_rx_slow_path_cqe rx_cqe_sp;
+};
+
+struct core_rx_start_ramrod_data {
+	struct regpair bd_base;
+	struct regpair cqe_pbl_addr;
+	__le16 mtu;
+	__le16 sb_id;
+	u8 sb_index;
+	u8 complete_cqe_flg;
+	u8 complete_event_flg;
+	u8 drop_ttl0_flg;
+	__le16 num_of_pbl_pages;
+	u8 inner_vlan_removal_en;
+	u8 queue_id;
+	u8 main_func_queue;
+	u8 mf_si_bcast_accept_all;
+	u8 mf_si_mcast_accept_all;
+	struct core_rx_action_on_error action_on_error;
+	u8 gsi_offload_flag;
+	u8 reserved[7];
+};
+
+struct core_rx_stop_ramrod_data {
+	u8 complete_cqe_flg;
+	u8 complete_event_flg;
+	u8 queue_id;
+	u8 reserved1;
+	__le16 reserved2[2];
+};
+
+struct core_tx_bd_flags {
+	u8 as_bitfield;
+#define CORE_TX_BD_FLAGS_FORCE_VLAN_MODE_MASK	0x1
+#define CORE_TX_BD_FLAGS_FORCE_VLAN_MODE_SHIFT	0
+#define CORE_TX_BD_FLAGS_VLAN_INSERTION_MASK	0x1
+#define CORE_TX_BD_FLAGS_VLAN_INSERTION_SHIFT	1
+#define CORE_TX_BD_FLAGS_START_BD_MASK	0x1
+#define CORE_TX_BD_FLAGS_START_BD_SHIFT	2
+#define CORE_TX_BD_FLAGS_IP_CSUM_MASK	0x1
+#define CORE_TX_BD_FLAGS_IP_CSUM_SHIFT	3
+#define CORE_TX_BD_FLAGS_L4_CSUM_MASK	0x1
+#define CORE_TX_BD_FLAGS_L4_CSUM_SHIFT	4
+#define CORE_TX_BD_FLAGS_IPV6_EXT_MASK	0x1
+#define CORE_TX_BD_FLAGS_IPV6_EXT_SHIFT	5
+#define CORE_TX_BD_FLAGS_L4_PROTOCOL_MASK	0x1
+#define CORE_TX_BD_FLAGS_L4_PROTOCOL_SHIFT	6
+#define CORE_TX_BD_FLAGS_L4_PSEUDO_CSUM_MODE_MASK	0x1
+#define CORE_TX_BD_FLAGS_L4_PSEUDO_CSUM_MODE_SHIFT 7
+#define CORE_TX_BD_FLAGS_ROCE_FLAV_MASK		0x1
+#define CORE_TX_BD_FLAGS_ROCE_FLAV_SHIFT	12
+
+};
+
+struct core_tx_bd {
+	struct regpair addr;
+	__le16 nbytes;
+	__le16 nw_vlan_or_lb_echo;
+	u8 bitfield0;
+#define CORE_TX_BD_NBDS_MASK	0xF
+#define CORE_TX_BD_NBDS_SHIFT	0
+#define CORE_TX_BD_ROCE_FLAV_MASK	0x1
+#define CORE_TX_BD_ROCE_FLAV_SHIFT	4
+#define CORE_TX_BD_RESERVED0_MASK	0x7
+#define CORE_TX_BD_RESERVED0_SHIFT	5
+	struct core_tx_bd_flags bd_flags;
+	__le16 bitfield1;
+#define CORE_TX_BD_L4_HDR_OFFSET_W_MASK	0x3FFF
+#define CORE_TX_BD_L4_HDR_OFFSET_W_SHIFT 0
+#define CORE_TX_BD_TX_DST_MASK	0x1
+#define CORE_TX_BD_TX_DST_SHIFT	14
+#define CORE_TX_BD_RESERVED1_MASK	0x1
+#define CORE_TX_BD_RESERVED1_SHIFT	15
+};
+
+enum core_tx_dest {
+	CORE_TX_DEST_NW,
+	CORE_TX_DEST_LB,
+	MAX_CORE_TX_DEST
+};
+
+struct core_tx_start_ramrod_data {
+	struct regpair pbl_base_addr;
+	__le16 mtu;
+	__le16 sb_id;
+	u8 sb_index;
+	u8 stats_en;
+	u8 stats_id;
+	u8 conn_type;
+	__le16 pbl_size;
+	__le16 qm_pq_id;
+	u8 gsi_offload_flag;
+	u8 resrved[3];
+};
+
+struct core_tx_stop_ramrod_data {
+	__le32 reserved0[2];
+};
+
 struct eth_mstorm_per_pf_stat {
 	struct regpair gre_discard_pkts;
 	struct regpair vxlan_discard_pkts;
@@ -636,9 +877,33 @@ struct hsi_fp_ver_struct {
 };
 
 /* Mstorm non-triggering VF zone */
+enum malicious_vf_error_id {
+	MALICIOUS_VF_NO_ERROR,
+	VF_PF_CHANNEL_NOT_READY,
+	VF_ZONE_MSG_NOT_VALID,
+	VF_ZONE_FUNC_NOT_ENABLED,
+	ETH_PACKET_TOO_SMALL,
+	ETH_ILLEGAL_VLAN_MODE,
+	ETH_MTU_VIOLATION,
+	ETH_ILLEGAL_INBAND_TAGS,
+	ETH_VLAN_INSERT_AND_INBAND_VLAN,
+	ETH_ILLEGAL_NBDS,
+	ETH_FIRST_BD_WO_SOP,
+	ETH_INSUFFICIENT_BDS,
+	ETH_ILLEGAL_LSO_HDR_NBDS,
+	ETH_ILLEGAL_LSO_MSS,
+	ETH_ZERO_SIZE_BD,
+	ETH_ILLEGAL_LSO_HDR_LEN,
+	ETH_INSUFFICIENT_PAYLOAD,
+	ETH_EDPM_OUT_OF_SYNC,
+	ETH_TUNN_IPV6_EXT_NBD_ERR,
+	ETH_CONTROL_PACKET_VIOLATION,
+	MAX_MALICIOUS_VF_ERROR_ID
+};
+
 struct mstorm_non_trigger_vf_zone {
 	struct eth_mstorm_per_queue_stat eth_queue_stat;
-	struct eth_rx_prod_data eth_rx_queue_producers[ETH_MAX_NUM_RX_QUEUES_PER_VF];
+	struct eth_rx_prod_data eth_rx_queue_producers[ETH_MAX_NUM_RX_QUEUES_PER_VF_QUAD];
 };
 
 /* Mstorm VF zone */
@@ -705,13 +970,17 @@ struct pf_start_ramrod_data {
 
 struct protocol_dcb_data {
 	u8 dcb_enable_flag;
+	u8 reserved_a;
 	u8 dcb_priority;
 	u8 dcb_tc;
-	u8 reserved;
+	u8 reserved_b;
+	u8 reserved0;
 };
 
 struct pf_update_tunnel_config {
 	u8 update_rx_pf_clss;
+	u8 update_rx_def_ucast_clss;
+	u8 update_rx_def_non_ucast_clss;
 	u8 update_tx_pf_clss;
 	u8 set_vxlan_udp_port_flg;
 	u8 set_geneve_udp_port_flg;
@@ -727,7 +996,7 @@ struct pf_update_tunnel_config {
 	u8 tunnel_clss_ipgre;
 	__le16 vxlan_udp_port;
 	__le16 geneve_udp_port;
-	__le16 reserved[3];
+	__le16 reserved[2];
 };
 
 struct pf_update_ramrod_data {
@@ -736,16 +1005,17 @@ struct pf_update_ramrod_data {
 	u8 update_fcoe_dcb_data_flag;
 	u8 update_iscsi_dcb_data_flag;
 	u8 update_roce_dcb_data_flag;
+	u8 update_rroce_dcb_data_flag;
 	u8 update_iwarp_dcb_data_flag;
 	u8 update_mf_vlan_flag;
-	u8 reserved;
 	struct protocol_dcb_data eth_dcb_data;
 	struct protocol_dcb_data fcoe_dcb_data;
 	struct protocol_dcb_data iscsi_dcb_data;
 	struct protocol_dcb_data roce_dcb_data;
+	struct protocol_dcb_data rroce_dcb_data;
 	struct protocol_dcb_data iwarp_dcb_data;
 	__le16 mf_vlan;
-	__le16 reserved2;
+	__le16 reserved;
 	struct pf_update_tunnel_config tunnel_config;
 };
 
@@ -766,10 +1036,14 @@ enum protocol_version_array_key {
 	MAX_PROTOCOL_VERSION_ARRAY_KEY
 };
 
-/* Pstorm non-triggering VF zone */
+struct rdma_sent_stats {
+	struct regpair sent_bytes;
+	struct regpair sent_pkts;
+};
+
 struct pstorm_non_trigger_vf_zone {
 	struct eth_pstorm_per_queue_stat eth_queue_stat;
-	struct regpair reserved[2];
+	struct rdma_sent_stats rdma_stats;
 };
 
 /* Pstorm VF zone */
@@ -786,7 +1060,11 @@ struct ramrod_header {
 	__le16 echo;
 };
 
-/* Slowpath Element (SPQE) */
+struct rdma_rcv_stats {
+	struct regpair rcv_bytes;
+	struct regpair rcv_pkts;
+};
+
 struct slow_path_element {
 	struct ramrod_header hdr;
 	struct regpair data_ptr;
@@ -794,7 +1072,7 @@ struct slow_path_element {
 
 /* Tstorm non-triggering VF zone */
 struct tstorm_non_trigger_vf_zone {
-	struct regpair reserved[2];
+	struct rdma_rcv_stats rdma_stats;
 };
 
 struct tstorm_per_port_stat {
@@ -802,9 +1080,14 @@ struct tstorm_per_port_stat {
 	struct regpair mac_error_discard;
 	struct regpair mftag_filter_discard;
 	struct regpair eth_mac_filter_discard;
-	struct regpair reserved[5];
+	struct regpair ll2_mac_filter_discard;
+	struct regpair ll2_conn_disabled_discard;
+	struct regpair iscsi_irregular_pkt;
+	struct regpair reserved;
+	struct regpair roce_irregular_pkt;
 	struct regpair eth_irregular_pkt;
-	struct regpair reserved1[2];
+	struct regpair reserved1;
+	struct regpair preroce_irregular_pkt;
 	struct regpair eth_gre_tunn_filter_discard;
 	struct regpair eth_vxlan_tunn_filter_discard;
 	struct regpair eth_geneve_tunn_filter_discard;
@@ -870,7 +1153,13 @@ struct vf_stop_ramrod_data {
 	__le32 reserved2;
 };
 
-/* Attentions status block */
+enum vf_zone_size_mode {
+	VF_ZONE_SIZE_MODE_DEFAULT,
+	VF_ZONE_SIZE_MODE_DOUBLE,
+	VF_ZONE_SIZE_MODE_QUAD,
+	MAX_VF_ZONE_SIZE_MODE
+};
+
 struct atten_status_block {
 	__le32 atten_bits;
 	__le32 atten_ack;
@@ -1442,13 +1731,6 @@ enum bin_dbg_buffer_type {
 	MAX_BIN_DBG_BUFFER_TYPE
 };
 
-/* Chip IDs */
-enum chip_ids {
-	CHIP_RESERVED,
-	CHIP_BB_B0,
-	CHIP_RESERVED2,
-	MAX_CHIP_IDS
-};
 
 /* Attention bit mapping */
 struct dbg_attn_bit_mapping {
@@ -1527,6 +1809,371 @@ enum dbg_attn_type {
 	MAX_DBG_ATTN_TYPE
 };
 
+/* condition header for registers dump */
+struct dbg_dump_cond_hdr {
+	struct dbg_mode_hdr mode; /* Mode header */
+	u8 block_id; /* block ID */
+	u8 data_size; /* size in dwords of the data following this header */
+};
+
+/* memory data for registers dump */
+struct dbg_dump_mem {
+	__le32 dword0;
+#define DBG_DUMP_MEM_ADDRESS_MASK       0xFFFFFF
+#define DBG_DUMP_MEM_ADDRESS_SHIFT      0
+#define DBG_DUMP_MEM_MEM_GROUP_ID_MASK  0xFF
+#define DBG_DUMP_MEM_MEM_GROUP_ID_SHIFT 24
+	__le32 dword1;
+#define DBG_DUMP_MEM_LENGTH_MASK        0xFFFFFF
+#define DBG_DUMP_MEM_LENGTH_SHIFT       0
+#define DBG_DUMP_MEM_RESERVED_MASK      0xFF
+#define DBG_DUMP_MEM_RESERVED_SHIFT     24
+};
+
+/* register data for registers dump */
+struct dbg_dump_reg {
+	__le32 data;
+#define DBG_DUMP_REG_ADDRESS_MASK  0xFFFFFF /* register address (in dwords) */
+#define DBG_DUMP_REG_ADDRESS_SHIFT 0
+#define DBG_DUMP_REG_LENGTH_MASK   0xFF /* register size (in dwords) */
+#define DBG_DUMP_REG_LENGTH_SHIFT  24
+};
+
+/* split header for registers dump */
+struct dbg_dump_split_hdr {
+	__le32 hdr;
+#define DBG_DUMP_SPLIT_HDR_DATA_SIZE_MASK      0xFFFFFF
+#define DBG_DUMP_SPLIT_HDR_DATA_SIZE_SHIFT     0
+#define DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID_MASK  0xFF
+#define DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID_SHIFT 24
+};
+
+/* condition header for idle check */
+struct dbg_idle_chk_cond_hdr {
+	struct dbg_mode_hdr mode; /* Mode header */
+	__le16 data_size; /* size in dwords of the data following this header */
+};
+
+/* Idle Check condition register */
+struct dbg_idle_chk_cond_reg {
+	__le32 data;
+#define DBG_IDLE_CHK_COND_REG_ADDRESS_MASK   0xFFFFFF
+#define DBG_IDLE_CHK_COND_REG_ADDRESS_SHIFT  0
+#define DBG_IDLE_CHK_COND_REG_BLOCK_ID_MASK  0xFF
+#define DBG_IDLE_CHK_COND_REG_BLOCK_ID_SHIFT 24
+	__le16 num_entries; /* number of registers entries to check */
+	u8 entry_size; /* size of registers entry (in dwords) */
+	u8 start_entry; /* index of the first entry to check */
+};
+
+/* Idle Check info register */
+struct dbg_idle_chk_info_reg {
+	__le32 data;
+#define DBG_IDLE_CHK_INFO_REG_ADDRESS_MASK   0xFFFFFF
+#define DBG_IDLE_CHK_INFO_REG_ADDRESS_SHIFT  0
+#define DBG_IDLE_CHK_INFO_REG_BLOCK_ID_MASK  0xFF
+#define DBG_IDLE_CHK_INFO_REG_BLOCK_ID_SHIFT 24
+	__le16 size; /* register size in dwords */
+	struct dbg_mode_hdr mode; /* Mode header */
+};
+
+/* Idle Check register */
+union dbg_idle_chk_reg {
+	struct dbg_idle_chk_cond_reg cond_reg; /* condition register */
+	struct dbg_idle_chk_info_reg info_reg; /* info register */
+};
+
+/* Idle Check result header */
+struct dbg_idle_chk_result_hdr {
+	__le16 rule_id; /* Failing rule index */
+	__le16 mem_entry_id; /* Failing memory entry index */
+	u8 num_dumped_cond_regs; /* number of dumped condition registers */
+	u8 num_dumped_info_regs; /* number of dumped condition registers */
+	u8 severity; /* from dbg_idle_chk_severity_types enum */
+	u8 reserved;
+};
+
+/* Idle Check result register header */
+struct dbg_idle_chk_result_reg_hdr {
+	u8 data;
+#define DBG_IDLE_CHK_RESULT_REG_HDR_IS_MEM_MASK  0x1
+#define DBG_IDLE_CHK_RESULT_REG_HDR_IS_MEM_SHIFT 0
+#define DBG_IDLE_CHK_RESULT_REG_HDR_REG_ID_MASK  0x7F
+#define DBG_IDLE_CHK_RESULT_REG_HDR_REG_ID_SHIFT 1
+	u8 start_entry; /* index of the first checked entry */
+	__le16 size; /* register size in dwords */
+};
+
+/* Idle Check rule */
+struct dbg_idle_chk_rule {
+	__le16 rule_id; /* Idle Check rule ID */
+	u8 severity; /* value from dbg_idle_chk_severity_types enum */
+	u8 cond_id; /* Condition ID */
+	u8 num_cond_regs; /* number of condition registers */
+	u8 num_info_regs; /* number of info registers */
+	u8 num_imms; /* number of immediates in the condition */
+	u8 reserved1;
+	__le16 reg_offset; /* offset of this rules registers in the idle check
+			    * register array (in dbg_idle_chk_reg units).
+			    */
+	__le16 imm_offset; /* offset of this rules immediate values in the
+			    * immediate values array (in dwords).
+			    */
+};
+
+/* Idle Check rule parsing data */
+struct dbg_idle_chk_rule_parsing_data {
+	__le32 data;
+#define DBG_IDLE_CHK_RULE_PARSING_DATA_HAS_FW_MSG_MASK  0x1
+#define DBG_IDLE_CHK_RULE_PARSING_DATA_HAS_FW_MSG_SHIFT 0
+#define DBG_IDLE_CHK_RULE_PARSING_DATA_STR_OFFSET_MASK  0x7FFFFFFF
+#define DBG_IDLE_CHK_RULE_PARSING_DATA_STR_OFFSET_SHIFT 1
+};
+
+/* idle check severity types */
+enum dbg_idle_chk_severity_types {
+	/* idle check failure should cause an error */
+	IDLE_CHK_SEVERITY_ERROR,
+	/* idle check failure should cause an error only if theres no traffic */
+	IDLE_CHK_SEVERITY_ERROR_NO_TRAFFIC,
+	/* idle check failure should cause a warning */
+	IDLE_CHK_SEVERITY_WARNING,
+	MAX_DBG_IDLE_CHK_SEVERITY_TYPES
+};
+
+/* Debug Bus block data */
+struct dbg_bus_block_data {
+	u8 enabled; /* Indicates if the block is enabled for recording (0/1) */
+	u8 hw_id; /* HW ID associated with the block */
+	u8 line_num; /* Debug line number to select */
+	u8 right_shift; /* Number of units to  right the debug data (0-3) */
+	u8 cycle_en; /* 4-bit value: bit i set -> unit i is enabled. */
+	u8 force_valid; /* 4-bit value: bit i set -> unit i is forced valid. */
+	u8 force_frame; /* 4-bit value: bit i set -> unit i frame bit is forced.
+			 */
+	u8 reserved;
+};
+
+/* Debug Bus Clients */
+enum dbg_bus_clients {
+	DBG_BUS_CLIENT_RBCN,
+	DBG_BUS_CLIENT_RBCP,
+	DBG_BUS_CLIENT_RBCR,
+	DBG_BUS_CLIENT_RBCT,
+	DBG_BUS_CLIENT_RBCU,
+	DBG_BUS_CLIENT_RBCF,
+	DBG_BUS_CLIENT_RBCX,
+	DBG_BUS_CLIENT_RBCS,
+	DBG_BUS_CLIENT_RBCH,
+	DBG_BUS_CLIENT_RBCZ,
+	DBG_BUS_CLIENT_OTHER_ENGINE,
+	DBG_BUS_CLIENT_TIMESTAMP,
+	DBG_BUS_CLIENT_CPU,
+	DBG_BUS_CLIENT_RBCY,
+	DBG_BUS_CLIENT_RBCQ,
+	DBG_BUS_CLIENT_RBCM,
+	DBG_BUS_CLIENT_RBCB,
+	DBG_BUS_CLIENT_RBCW,
+	DBG_BUS_CLIENT_RBCV,
+	MAX_DBG_BUS_CLIENTS
+};
+
+/* Debug Bus memory address */
+struct dbg_bus_mem_addr {
+	__le32 lo;
+	__le32 hi;
+};
+
+/* Debug Bus PCI buffer data */
+struct dbg_bus_pci_buf_data {
+	struct dbg_bus_mem_addr phys_addr; /* PCI buffer physical address */
+	struct dbg_bus_mem_addr virt_addr; /* PCI buffer virtual address */
+	__le32 size; /* PCI buffer size in bytes */
+};
+
+/* Debug Bus Storm EID range filter params */
+struct dbg_bus_storm_eid_range_params {
+	u8 min; /* Minimal event ID to filter on */
+	u8 max; /* Maximal event ID to filter on */
+};
+
+/* Debug Bus Storm EID mask filter params */
+struct dbg_bus_storm_eid_mask_params {
+	u8 val; /* Event ID value */
+	u8 mask; /* Event ID mask. 1s in the mask = dont care bits. */
+};
+
+/* Debug Bus Storm EID filter params */
+union dbg_bus_storm_eid_params {
+	struct dbg_bus_storm_eid_range_params range;
+	struct dbg_bus_storm_eid_mask_params mask;
+};
+
+/* Debug Bus Storm data */
+struct dbg_bus_storm_data {
+	u8 fast_enabled;
+	u8 fast_mode;
+	u8 slow_enabled;
+	u8 slow_mode;
+	u8 hw_id;
+	u8 eid_filter_en;
+	u8 eid_range_not_mask;
+	u8 cid_filter_en;
+	union dbg_bus_storm_eid_params eid_filter_params;
+	__le16 reserved;
+	__le32 cid;
+};
+
+/* Debug Bus data */
+struct dbg_bus_data {
+	__le32 app_version; /* The tools version number of the application */
+	u8 state; /* The current debug bus state */
+	u8 hw_dwords; /* HW dwords per cycle */
+	u8 next_hw_id; /* Next HW ID to be associated with an input */
+	u8 num_enabled_blocks; /* Number of blocks enabled for recording */
+	u8 num_enabled_storms; /* Number of Storms enabled for recording */
+	u8 target; /* Output target */
+	u8 next_trigger_state; /* ID of next trigger state to be added */
+	u8 next_constraint_id; /* ID of next filter/trigger constraint to be
+				* added.
+				*/
+	u8 one_shot_en; /* Indicates if one-shot mode is enabled (0/1) */
+	u8 grc_input_en; /* Indicates if GRC recording is enabled (0/1) */
+	u8 timestamp_input_en; /* Indicates if timestamp recording is enabled
+				* (0/1).
+				*/
+	u8 filter_en; /* Indicates if the recording filter is enabled (0/1) */
+	u8 trigger_en; /* Indicates if the recording trigger is enabled (0/1) */
+	u8 adding_filter; /* If true, the next added constraint belong to the
+			   * filter. Otherwise, it belongs to the last added
+			   * trigger state. Valid only if either filter or
+			   * triggers are enabled.
+			   */
+	u8 filter_pre_trigger; /* Indicates if the recording filter should be
+				* applied before the trigger. Valid only if both
+				* filter and trigger are enabled (0/1).
+				*/
+	u8 filter_post_trigger; /* Indicates if the recording filter should be
+				 * applied after the trigger. Valid only if both
+				 * filter and trigger are enabled (0/1).
+				 */
+	u8 unify_inputs; /* If true, all inputs are associated with HW ID 0.
+			  * Otherwise, each input is assigned a different HW ID
+			  * (0/1).
+			  */
+	u8 rcv_from_other_engine; /* Indicates if the other engine sends it NW
+				   * recording to this engine (0/1).
+				   */
+	struct dbg_bus_pci_buf_data pci_buf; /* Debug Bus PCI buffer data. Valid
+					      * only when the target is
+					      * DBG_BUS_TARGET_ID_PCI.
+					      */
+	__le16 reserved;
+	struct dbg_bus_block_data blocks[80];/* Debug Bus data for each block */
+	struct dbg_bus_storm_data storms[6]; /* Debug Bus data for each block */
+};
+
+/* Debug bus frame modes */
+enum dbg_bus_frame_modes {
+	DBG_BUS_FRAME_MODE_0HW_4ST = 0, /* 0 HW dwords, 4 Storm dwords */
+	DBG_BUS_FRAME_MODE_4HW_0ST = 3, /* 4 HW dwords, 0 Storm dwords */
+	DBG_BUS_FRAME_MODE_8HW_0ST = 4, /* 8 HW dwords, 0 Storm dwords */
+	MAX_DBG_BUS_FRAME_MODES
+};
+
+/* Debug bus states */
+enum dbg_bus_states {
+	DBG_BUS_STATE_IDLE, /* debug bus idle state (not recording) */
+	DBG_BUS_STATE_READY, /* debug bus is ready for configuration and
+			      * recording.
+			      */
+	DBG_BUS_STATE_RECORDING, /* debug bus is currently recording */
+	DBG_BUS_STATE_STOPPED, /* debug bus recording has stopped */
+	MAX_DBG_BUS_STATES
+};
+
+/* Debug bus target IDs */
+enum dbg_bus_targets {
+	/* records debug bus to DBG block internal buffer */
+	DBG_BUS_TARGET_ID_INT_BUF,
+	/* records debug bus to the NW */
+	DBG_BUS_TARGET_ID_NIG,
+	/* records debug bus to a PCI buffer */
+	DBG_BUS_TARGET_ID_PCI,
+	MAX_DBG_BUS_TARGETS
+};
+
+/* GRC Dump data */
+struct dbg_grc_data {
+	__le32 param_val[40]; /* Value of each GRC parameter. Array size must
+			       * match the enum dbg_grc_params.
+			       */
+	u8 param_set_by_user[40]; /* Indicates for each GRC parameter if it was
+				   * set by the user (0/1). Array size must
+				   * match the enum dbg_grc_params.
+				   */
+};
+
+/* Debug GRC params */
+enum dbg_grc_params {
+	DBG_GRC_PARAM_DUMP_TSTORM, /* dump Tstorm memories (0/1) */
+	DBG_GRC_PARAM_DUMP_MSTORM, /* dump Mstorm memories (0/1) */
+	DBG_GRC_PARAM_DUMP_USTORM, /* dump Ustorm memories (0/1) */
+	DBG_GRC_PARAM_DUMP_XSTORM, /* dump Xstorm memories (0/1) */
+	DBG_GRC_PARAM_DUMP_YSTORM, /* dump Ystorm memories (0/1) */
+	DBG_GRC_PARAM_DUMP_PSTORM, /* dump Pstorm memories (0/1) */
+	DBG_GRC_PARAM_DUMP_REGS, /* dump non-memory registers (0/1) */
+	DBG_GRC_PARAM_DUMP_RAM, /* dump Storm internal RAMs (0/1) */
+	DBG_GRC_PARAM_DUMP_PBUF, /* dump Storm passive buffer (0/1) */
+	DBG_GRC_PARAM_DUMP_IOR, /* dump Storm IORs (0/1) */
+	DBG_GRC_PARAM_DUMP_VFC, /* dump VFC memories (0/1) */
+	DBG_GRC_PARAM_DUMP_CM_CTX, /* dump CM contexts (0/1) */
+	DBG_GRC_PARAM_DUMP_PXP, /* dump PXP memories (0/1) */
+	DBG_GRC_PARAM_DUMP_RSS, /* dump RSS memories (0/1) */
+	DBG_GRC_PARAM_DUMP_CAU, /* dump CAU memories (0/1) */
+	DBG_GRC_PARAM_DUMP_QM, /* dump QM memories (0/1) */
+	DBG_GRC_PARAM_DUMP_MCP, /* dump MCP memories (0/1) */
+	DBG_GRC_PARAM_RESERVED, /* reserved */
+	DBG_GRC_PARAM_DUMP_CFC, /* dump CFC memories (0/1) */
+	DBG_GRC_PARAM_DUMP_IGU, /* dump IGU memories (0/1) */
+	DBG_GRC_PARAM_DUMP_BRB, /* dump BRB memories (0/1) */
+	DBG_GRC_PARAM_DUMP_BTB, /* dump BTB memories (0/1) */
+	DBG_GRC_PARAM_DUMP_BMB, /* dump BMB memories (0/1) */
+	DBG_GRC_PARAM_DUMP_NIG, /* dump NIG memories (0/1) */
+	DBG_GRC_PARAM_DUMP_MULD, /* dump MULD memories (0/1) */
+	DBG_GRC_PARAM_DUMP_PRS, /* dump PRS memories (0/1) */
+	DBG_GRC_PARAM_DUMP_DMAE, /* dump PRS memories (0/1) */
+	DBG_GRC_PARAM_DUMP_TM, /* dump TM (timers) memories (0/1) */
+	DBG_GRC_PARAM_DUMP_SDM, /* dump SDM memories (0/1) */
+	DBG_GRC_PARAM_DUMP_DIF, /* dump DIF memories (0/1) */
+	DBG_GRC_PARAM_DUMP_STATIC, /* dump static debug data (0/1) */
+	DBG_GRC_PARAM_UNSTALL, /* un-stall Storms after dump (0/1) */
+	DBG_GRC_PARAM_NUM_LCIDS, /* number of LCIDs (0..320) */
+	DBG_GRC_PARAM_NUM_LTIDS, /* number of LTIDs (0..320) */
+	/* preset: exclude all memories from dump (1 only) */
+	DBG_GRC_PARAM_EXCLUDE_ALL,
+	/* preset: include memories for crash dump (1 only) */
+	DBG_GRC_PARAM_CRASH,
+	/* perform dump only if MFW is responding (0/1) */
+	DBG_GRC_PARAM_PARITY_SAFE,
+	DBG_GRC_PARAM_DUMP_CM, /* dump CM memories (0/1) */
+	DBG_GRC_PARAM_DUMP_PHY, /* dump PHY memories (0/1) */
+	MAX_DBG_GRC_PARAMS
+};
+
+/* Debug reset registers */
+enum dbg_reset_regs {
+	DBG_RESET_REG_MISCS_PL_UA,
+	DBG_RESET_REG_MISCS_PL_HV,
+	DBG_RESET_REG_MISCS_PL_HV_2,
+	DBG_RESET_REG_MISC_PL_UA,
+	DBG_RESET_REG_MISC_PL_HV,
+	DBG_RESET_REG_MISC_PL_PDA_VMAIN_1,
+	DBG_RESET_REG_MISC_PL_PDA_VMAIN_2,
+	DBG_RESET_REG_MISC_PL_PDA_VAUX,
+	MAX_DBG_RESET_REGS
+};
+
 /* Debug status codes */
 enum dbg_status {
 	DBG_STATUS_OK,
@@ -1579,9 +2226,45 @@ enum dbg_status {
 	DBG_STATUS_REG_FIFO_BAD_DATA,
 	DBG_STATUS_PROTECTION_OVERRIDE_BAD_DATA,
 	DBG_STATUS_DBG_ARRAY_NOT_SET,
+	DBG_STATUS_MULTI_BLOCKS_WITH_FILTER,
 	MAX_DBG_STATUS
 };
 
+/* Debug Storms IDs */
+enum dbg_storms {
+	DBG_TSTORM_ID,
+	DBG_MSTORM_ID,
+	DBG_USTORM_ID,
+	DBG_XSTORM_ID,
+	DBG_YSTORM_ID,
+	DBG_PSTORM_ID,
+	MAX_DBG_STORMS
+};
+
+/* Idle Check data */
+struct idle_chk_data {
+	__le32 buf_size; /* Idle check buffer size in dwords */
+	u8 buf_size_set; /* Indicates if the idle check buffer size was set
+			  * (0/1).
+			  */
+	u8 reserved1;
+	__le16 reserved2;
+};
+
+/* Debug Tools data (per HW function) */
+struct dbg_tools_data {
+	struct dbg_grc_data grc; /* GRC Dump data */
+	struct dbg_bus_data bus; /* Debug Bus data */
+	struct idle_chk_data idle_chk; /* Idle Check data */
+	u8 mode_enable[40]; /* Indicates if a mode is enabled (0/1) */
+	u8 block_in_reset[80]; /* Indicates if a block is in reset state (0/1).
+				*/
+	u8 chip_id; /* Chip ID (from enum chip_ids) */
+	u8 platform_id; /* Platform ID (from enum platform_ids) */
+	u8 initialized; /* Indicates if the data was initialized */
+	u8 reserved;
+};
+
 /********************************/
 /* HSI Init Functions constants */
 /********************************/
@@ -1589,7 +2272,41 @@ enum dbg_status {
 /* Number of VLAN priorities */
 #define NUM_OF_VLAN_PRIORITIES	8
 
-/* QM per-port init parameters */
+struct init_brb_ram_req {
+	__le32 guranteed_per_tc;
+	__le32 headroom_per_tc;
+	__le32 min_pkt_size;
+	__le32 max_ports_per_engine;
+	u8 num_active_tcs[MAX_NUM_PORTS];
+};
+
+struct init_ets_tc_req {
+	u8 use_sp;
+	u8 use_wfq;
+	__le16 weight;
+};
+
+struct init_ets_req {
+	__le32 mtu;
+	struct init_ets_tc_req tc_req[NUM_OF_TCS];
+};
+
+struct init_nig_lb_rl_req {
+	__le16 lb_mac_rate;
+	__le16 lb_rate;
+	__le32 mtu;
+	__le16 tc_rate[NUM_OF_PHYS_TCS];
+};
+
+struct init_nig_pri_tc_map_entry {
+	u8 tc_id;
+	u8 valid;
+};
+
+struct init_nig_pri_tc_map_req {
+	struct init_nig_pri_tc_map_entry pri[NUM_OF_VLAN_PRIORITIES];
+};
+
 struct init_qm_port_params {
 	u8 active;
 	u8 active_phys_tcs;
@@ -1619,7 +2336,7 @@ struct init_qm_vport_params {
 
 /* Width of GRC address in bits (addresses are specified in dwords) */
 #define GRC_ADDR_BITS	23
-#define MAX_GRC_ADDR	((1 << GRC_ADDR_BITS) - 1)
+#define MAX_GRC_ADDR	(BIT(GRC_ADDR_BITS) - 1)
 
 /* indicates an init that should be applied to any phase ID */
 #define ANY_PHASE_ID	0xffff
@@ -1627,15 +2344,50 @@ struct init_qm_vport_params {
 /* Max size in dwords of a zipped array */
 #define MAX_ZIPPED_SIZE	8192
 
+struct fw_asserts_ram_section {
+	__le16 section_ram_line_offset;
+	__le16 section_ram_line_size;
+	u8 list_dword_offset;
+	u8 list_element_dword_size;
+	u8 list_num_elements;
+	u8 list_next_index_dword_offset;
+};
+
+struct fw_ver_num {
+	u8 major; /* Firmware major version number */
+	u8 minor; /* Firmware minor version number */
+	u8 rev; /* Firmware revision version number */
+	u8 eng; /* Firmware engineering version number (for bootleg versions) */
+};
+
+struct fw_ver_info {
+	__le16 tools_ver; /* Tools version number */
+	u8 image_id; /* FW image ID (e.g. main) */
+	u8 reserved1;
+	struct fw_ver_num num; /* FW version number */
+	__le32 timestamp; /* FW Timestamp in unix time  (sec. since 1970) */
+	__le32 reserved2;
+};
+
+struct fw_info {
+	struct fw_ver_info ver;
+	struct fw_asserts_ram_section fw_asserts_section;
+};
+
+struct fw_info_location {
+	__le32 grc_addr;
+	__le32 size;
+};
+
 enum init_modes {
 	MODE_RESERVED,
 	MODE_BB_B0,
-	MODE_RESERVED2,
+	MODE_K2,
 	MODE_ASIC,
+	MODE_RESERVED2,
 	MODE_RESERVED3,
 	MODE_RESERVED4,
 	MODE_RESERVED5,
-	MODE_RESERVED6,
 	MODE_SF,
 	MODE_MF_SD,
 	MODE_MF_SI,
@@ -1644,7 +2396,7 @@ enum init_modes {
 	MODE_PORTS_PER_ENG_4,
 	MODE_100G,
 	MODE_40G,
-	MODE_RESERVED7,
+	MODE_RESERVED6,
 	MAX_INIT_MODES
 };
 
@@ -1674,11 +2426,11 @@ struct bin_buffer_hdr {
 
 /* binary init buffer types */
 enum bin_init_buffer_type {
-	BIN_BUF_FW_VER_INFO,
+	BIN_BUF_INIT_FW_VER_INFO,
 	BIN_BUF_INIT_CMD,
 	BIN_BUF_INIT_VAL,
 	BIN_BUF_INIT_MODE_TREE,
-	BIN_BUF_IRO,
+	BIN_BUF_INIT_IRO,
 	MAX_BIN_INIT_BUFFER_TYPE
 };
 
@@ -1902,8 +2654,276 @@ struct iro {
 	__le16 size;
 };
 
+/***************************** Public Functions *******************************/
+/**
+ * @brief qed_dbg_set_bin_ptr - Sets a pointer to the binary data with debug
+ *	arrays.
+ *
+ * @param bin_ptr - a pointer to the binary data with debug arrays.
+ */
+enum dbg_status qed_dbg_set_bin_ptr(const u8 * const bin_ptr);
+/**
+ * @brief qed_dbg_grc_get_dump_buf_size - Returns the required buffer size for
+ *	GRC Dump.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param buf_size - OUT: required buffer size (in dwords) for the GRC Dump
+ *	data.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_grc_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+					      struct qed_ptt *p_ptt,
+					      u32 *buf_size);
+/**
+ * @brief qed_dbg_grc_dump - Dumps GRC data into the specified buffer.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param dump_buf - Pointer to write the collected GRC data into.
+ * @param buf_size_in_dwords - Size of the specified buffer in dwords.
+ * @param num_dumped_dwords - OUT: number of dumped dwords.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ *	- the specified dump buffer is too small
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_grc_dump(struct qed_hwfn *p_hwfn,
+				 struct qed_ptt *p_ptt,
+				 u32 *dump_buf,
+				 u32 buf_size_in_dwords,
+				 u32 *num_dumped_dwords);
+/**
+ * @brief qed_dbg_idle_chk_get_dump_buf_size - Returns the required buffer size
+ *	for idle check results.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param buf_size - OUT: required buffer size (in dwords) for the idle check
+ *	data.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_idle_chk_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						   struct qed_ptt *p_ptt,
+						   u32 *buf_size);
+/**
+ * @brief qed_dbg_idle_chk_dump - Performs idle check and writes the results
+ *	into the specified buffer.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param dump_buf - Pointer to write the idle check data into.
+ * @param buf_size_in_dwords - Size of the specified buffer in dwords.
+ * @param num_dumped_dwords - OUT: number of dumped dwords.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ *	- the specified buffer is too small
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_idle_chk_dump(struct qed_hwfn *p_hwfn,
+				      struct qed_ptt *p_ptt,
+				      u32 *dump_buf,
+				      u32 buf_size_in_dwords,
+				      u32 *num_dumped_dwords);
+/**
+ * @brief qed_dbg_mcp_trace_get_dump_buf_size - Returns the required buffer size
+ *	for mcp trace results.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param buf_size - OUT: required buffer size (in dwords) for mcp trace data.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ *	- the trace data in MCP scratchpad contain an invalid signature
+ *	- the bundle ID in NVRAM is invalid
+ *	- the trace meta data cannot be found (in NVRAM or image file)
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_mcp_trace_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						    struct qed_ptt *p_ptt,
+						    u32 *buf_size);
+/**
+ * @brief qed_dbg_mcp_trace_dump - Performs mcp trace and writes the results
+ *	into the specified buffer.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param dump_buf - Pointer to write the mcp trace data into.
+ * @param buf_size_in_dwords - Size of the specified buffer in dwords.
+ * @param num_dumped_dwords - OUT: number of dumped dwords.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ *	- the specified buffer is too small
+ *	- the trace data in MCP scratchpad contain an invalid signature
+ *	- the bundle ID in NVRAM is invalid
+ *	- the trace meta data cannot be found (in NVRAM or image file)
+ *	- the trace meta data cannot be read (from NVRAM or image file)
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_mcp_trace_dump(struct qed_hwfn *p_hwfn,
+				       struct qed_ptt *p_ptt,
+				       u32 *dump_buf,
+				       u32 buf_size_in_dwords,
+				       u32 *num_dumped_dwords);
+/**
+ * @brief qed_dbg_reg_fifo_get_dump_buf_size - Returns the required buffer size
+ *	for grc trace fifo results.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param buf_size - OUT: required buffer size (in dwords) for reg fifo data.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_reg_fifo_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						   struct qed_ptt *p_ptt,
+						   u32 *buf_size);
 /**
- * @brief qed_dbg_print_attn - Prints attention registers values in the specified results struct.
+ * @brief qed_dbg_reg_fifo_dump - Reads the reg fifo and writes the results into
+ *	the specified buffer.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param dump_buf - Pointer to write the reg fifo data into.
+ * @param buf_size_in_dwords - Size of the specified buffer in dwords.
+ * @param num_dumped_dwords - OUT: number of dumped dwords.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ *	- the specified buffer is too small
+ *	- DMAE transaction failed
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_reg_fifo_dump(struct qed_hwfn *p_hwfn,
+				      struct qed_ptt *p_ptt,
+				      u32 *dump_buf,
+				      u32 buf_size_in_dwords,
+				      u32 *num_dumped_dwords);
+/**
+ * @brief qed_dbg_igu_fifo_get_dump_buf_size - Returns the required buffer size
+ *	for the IGU fifo results.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param buf_size - OUT: required buffer size (in dwords) for the IGU fifo
+ *	data.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_igu_fifo_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						   struct qed_ptt *p_ptt,
+						   u32 *buf_size);
+/**
+ * @brief qed_dbg_igu_fifo_dump - Reads the IGU fifo and writes the results into
+ *	the specified buffer.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param dump_buf - Pointer to write the IGU fifo data into.
+ * @param buf_size_in_dwords - Size of the specified buffer in dwords.
+ * @param num_dumped_dwords - OUT: number of dumped dwords.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ *	- the specified buffer is too small
+ *	- DMAE transaction failed
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_igu_fifo_dump(struct qed_hwfn *p_hwfn,
+				      struct qed_ptt *p_ptt,
+				      u32 *dump_buf,
+				      u32 buf_size_in_dwords,
+				      u32 *num_dumped_dwords);
+/**
+ * @brief qed_dbg_protection_override_get_dump_buf_size - Returns the required
+ *	buffer size for protection override window results.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param buf_size - OUT: required buffer size (in dwords) for protection
+ *	override data.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ * Otherwise, returns ok.
+ */
+enum dbg_status
+qed_dbg_protection_override_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+					      struct qed_ptt *p_ptt,
+					      u32 *buf_size);
+/**
+ * @brief qed_dbg_protection_override_dump - Reads protection override window
+ *	entries and writes the results into the specified buffer.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param dump_buf - Pointer to write the protection override data into.
+ * @param buf_size_in_dwords - Size of the specified buffer in dwords.
+ * @param num_dumped_dwords - OUT: number of dumped dwords.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ *	- the specified buffer is too small
+ *	- DMAE transaction failed
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_protection_override_dump(struct qed_hwfn *p_hwfn,
+						 struct qed_ptt *p_ptt,
+						 u32 *dump_buf,
+						 u32 buf_size_in_dwords,
+						 u32 *num_dumped_dwords);
+/**
+ * @brief qed_dbg_fw_asserts_get_dump_buf_size - Returns the required buffer
+ *	size for FW Asserts results.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param buf_size - OUT: required buffer size (in dwords) for FW Asserts data.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_fw_asserts_get_dump_buf_size(struct qed_hwfn *p_hwfn,
+						     struct qed_ptt *p_ptt,
+						     u32 *buf_size);
+/**
+ * @brief qed_dbg_fw_asserts_dump - Reads the FW Asserts and writes the results
+ *	into the specified buffer.
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param dump_buf - Pointer to write the FW Asserts data into.
+ * @param buf_size_in_dwords - Size of the specified buffer in dwords.
+ * @param num_dumped_dwords - OUT: number of dumped dwords.
+ *
+ * @return error if one of the following holds:
+ *	- the version wasn't set
+ *	- the specified buffer is too small
+ * Otherwise, returns ok.
+ */
+enum dbg_status qed_dbg_fw_asserts_dump(struct qed_hwfn *p_hwfn,
+					struct qed_ptt *p_ptt,
+					u32 *dump_buf,
+					u32 buf_size_in_dwords,
+					u32 *num_dumped_dwords);
+/**
+ * @brief qed_dbg_print_attn - Prints attention registers values in the
+ *	specified results struct.
  *
  * @param p_hwfn
  * @param results - Pointer to the attention read results
@@ -1915,47 +2935,241 @@ struct iro {
 enum dbg_status qed_dbg_print_attn(struct qed_hwfn *p_hwfn,
 				   struct dbg_attn_block_result *results);
 
+/******************************** Constants **********************************/
+
 #define MAX_NAME_LEN	16
 
+/***************************** Public Functions *******************************/
+/**
+ * @brief qed_dbg_user_set_bin_ptr - Sets a pointer to the binary data with
+ *	debug arrays.
+ *
+ * @param bin_ptr - a pointer to the binary data with debug arrays.
+ */
+enum dbg_status qed_dbg_user_set_bin_ptr(const u8 * const bin_ptr);
+/**
+ * @brief qed_dbg_get_status_str - Returns a string for the specified status.
+ *
+ * @param status - a debug status code.
+ *
+ * @return a string for the specified status
+ */
+const char *qed_dbg_get_status_str(enum dbg_status status);
+/**
+ * @brief qed_get_idle_chk_results_buf_size - Returns the required buffer size
+ *	for idle check results (in bytes).
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - idle check dump buffer.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf_size - OUT: required buffer size (in bytes) for the parsed
+ *	results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_get_idle_chk_results_buf_size(struct qed_hwfn *p_hwfn,
+						  u32 *dump_buf,
+						  u32  num_dumped_dwords,
+						  u32 *results_buf_size);
+/**
+ * @brief qed_print_idle_chk_results - Prints idle check results
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - idle check dump buffer.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf - buffer for printing the idle check results.
+ * @param num_errors - OUT: number of errors found in idle check.
+ * @param num_warnings - OUT: number of warnings found in idle check.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_print_idle_chk_results(struct qed_hwfn *p_hwfn,
+					   u32 *dump_buf,
+					   u32 num_dumped_dwords,
+					   char *results_buf,
+					   u32 *num_errors,
+					   u32 *num_warnings);
+/**
+ * @brief qed_get_mcp_trace_results_buf_size - Returns the required buffer size
+ *	for MCP Trace results (in bytes).
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - MCP Trace dump buffer.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf_size - OUT: required buffer size (in bytes) for the parsed
+ *	results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_get_mcp_trace_results_buf_size(struct qed_hwfn *p_hwfn,
+						   u32 *dump_buf,
+						   u32 num_dumped_dwords,
+						   u32 *results_buf_size);
+/**
+ * @brief qed_print_mcp_trace_results - Prints MCP Trace results
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - mcp trace dump buffer, starting from the header.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf - buffer for printing the mcp trace results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_print_mcp_trace_results(struct qed_hwfn *p_hwfn,
+					    u32 *dump_buf,
+					    u32 num_dumped_dwords,
+					    char *results_buf);
+/**
+ * @brief qed_get_reg_fifo_results_buf_size - Returns the required buffer size
+ *	for reg_fifo results (in bytes).
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - reg fifo dump buffer.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf_size - OUT: required buffer size (in bytes) for the parsed
+ *	results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_get_reg_fifo_results_buf_size(struct qed_hwfn *p_hwfn,
+						  u32 *dump_buf,
+						  u32 num_dumped_dwords,
+						  u32 *results_buf_size);
+/**
+ * @brief qed_print_reg_fifo_results - Prints reg fifo results
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - reg fifo dump buffer, starting from the header.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf - buffer for printing the reg fifo results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_print_reg_fifo_results(struct qed_hwfn *p_hwfn,
+					   u32 *dump_buf,
+					   u32 num_dumped_dwords,
+					   char *results_buf);
+/**
+ * @brief qed_get_igu_fifo_results_buf_size - Returns the required buffer size
+ *	for igu_fifo results (in bytes).
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - IGU fifo dump buffer.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf_size - OUT: required buffer size (in bytes) for the parsed
+ *	results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_get_igu_fifo_results_buf_size(struct qed_hwfn *p_hwfn,
+						  u32 *dump_buf,
+						  u32 num_dumped_dwords,
+						  u32 *results_buf_size);
+/**
+ * @brief qed_print_igu_fifo_results - Prints IGU fifo results
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - IGU fifo dump buffer, starting from the header.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf - buffer for printing the IGU fifo results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_print_igu_fifo_results(struct qed_hwfn *p_hwfn,
+					   u32 *dump_buf,
+					   u32 num_dumped_dwords,
+					   char *results_buf);
+/**
+ * @brief qed_get_protection_override_results_buf_size - Returns the required
+ *	buffer size for protection override results (in bytes).
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - protection override dump buffer.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf_size - OUT: required buffer size (in bytes) for the parsed
+ *	results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status
+qed_get_protection_override_results_buf_size(struct qed_hwfn *p_hwfn,
+					     u32 *dump_buf,
+					     u32 num_dumped_dwords,
+					     u32 *results_buf_size);
+/**
+ * @brief qed_print_protection_override_results - Prints protection override
+ *	results.
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - protection override dump buffer, starting from the header.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf - buffer for printing the reg fifo results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_print_protection_override_results(struct qed_hwfn *p_hwfn,
+						      u32 *dump_buf,
+						      u32 num_dumped_dwords,
+						      char *results_buf);
+/**
+ * @brief qed_get_fw_asserts_results_buf_size - Returns the required buffer size
+ *	for FW Asserts results (in bytes).
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - FW Asserts dump buffer.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf_size - OUT: required buffer size (in bytes) for the parsed
+ *	results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_get_fw_asserts_results_buf_size(struct qed_hwfn *p_hwfn,
+						    u32 *dump_buf,
+						    u32 num_dumped_dwords,
+						    u32 *results_buf_size);
+/**
+ * @brief qed_print_fw_asserts_results - Prints FW Asserts results
+ *
+ * @param p_hwfn - HW device data
+ * @param dump_buf - FW Asserts dump buffer, starting from the header.
+ * @param num_dumped_dwords - number of dwords that were dumped.
+ * @param results_buf - buffer for printing the FW Asserts results.
+ *
+ * @return error if the parsing fails, ok otherwise.
+ */
+enum dbg_status qed_print_fw_asserts_results(struct qed_hwfn *p_hwfn,
+					     u32 *dump_buf,
+					     u32 num_dumped_dwords,
+					     char *results_buf);
 /* Win 2 */
-#define GTT_BAR0_MAP_REG_IGU_CMD \
-	0x00f000UL
+#define GTT_BAR0_MAP_REG_IGU_CMD	0x00f000UL
 
 /* Win 3 */
-#define GTT_BAR0_MAP_REG_TSDM_RAM \
-	0x010000UL
+#define GTT_BAR0_MAP_REG_TSDM_RAM	0x010000UL
 
 /* Win 4 */
-#define GTT_BAR0_MAP_REG_MSDM_RAM \
-	0x011000UL
+#define GTT_BAR0_MAP_REG_MSDM_RAM	0x011000UL
 
 /* Win 5 */
-#define GTT_BAR0_MAP_REG_MSDM_RAM_1024 \
-	0x012000UL
+#define GTT_BAR0_MAP_REG_MSDM_RAM_1024	0x012000UL
 
 /* Win 6 */
-#define GTT_BAR0_MAP_REG_USDM_RAM \
-	0x013000UL
+#define GTT_BAR0_MAP_REG_USDM_RAM	0x013000UL
 
 /* Win 7 */
-#define GTT_BAR0_MAP_REG_USDM_RAM_1024 \
-	0x014000UL
+#define GTT_BAR0_MAP_REG_USDM_RAM_1024	0x014000UL
 
 /* Win 8 */
-#define GTT_BAR0_MAP_REG_USDM_RAM_2048 \
-	0x015000UL
+#define GTT_BAR0_MAP_REG_USDM_RAM_2048	0x015000UL
 
 /* Win 9 */
-#define GTT_BAR0_MAP_REG_XSDM_RAM \
-	0x016000UL
+#define GTT_BAR0_MAP_REG_XSDM_RAM	0x016000UL
 
 /* Win 10 */
-#define GTT_BAR0_MAP_REG_YSDM_RAM \
-	0x017000UL
+#define GTT_BAR0_MAP_REG_YSDM_RAM	0x017000UL
 
 /* Win 11 */
-#define GTT_BAR0_MAP_REG_PSDM_RAM \
-	0x018000UL
+#define GTT_BAR0_MAP_REG_PSDM_RAM	0x018000UL
 
 /**
  * @brief qed_qm_pf_mem_size - prepare QM ILT sizes
@@ -2003,7 +3217,7 @@ struct qed_qm_pf_rt_init_params {
 	u16 num_vf_pqs;
 	u8 start_vport;
 	u8 num_vports;
-	u8 pf_wfq;
+	u16 pf_wfq;
 	u32 pf_rl;
 	struct init_qm_pq_params *pq_params;
 	struct init_qm_vport_params *vport_params;
@@ -2138,6 +3352,9 @@ void qed_set_geneve_enable(struct qed_hwfn *p_hwfn,
 #define	TSTORM_PORT_STAT_OFFSET(port_id) \
 	(IRO[1].base + ((port_id) * IRO[1].m1))
 #define	TSTORM_PORT_STAT_SIZE				(IRO[1].size)
+#define TSTORM_LL2_PORT_STAT_OFFSET(port_id) \
+	(IRO[2].base + ((port_id) * IRO[2].m1))
+#define TSTORM_LL2_PORT_STAT_SIZE			(IRO[2].size)
 #define	USTORM_VF_PF_CHANNEL_READY_OFFSET(vf_id) \
 	(IRO[3].base + ((vf_id) * IRO[3].m1))
 #define	USTORM_VF_PF_CHANNEL_READY_SIZE			(IRO[3].size)
@@ -2153,42 +3370,90 @@ void qed_set_geneve_enable(struct qed_hwfn *p_hwfn,
 #define	USTORM_COMMON_QUEUE_CONS_OFFSET(queue_zone_id) \
 	(IRO[7].base + ((queue_zone_id) * IRO[7].m1))
 #define	USTORM_COMMON_QUEUE_CONS_SIZE			(IRO[7].size)
+#define TSTORM_LL2_RX_PRODS_OFFSET(core_rx_queue_id) \
+	(IRO[14].base +	((core_rx_queue_id) * IRO[14].m1))
+#define TSTORM_LL2_RX_PRODS_SIZE			(IRO[14].size)
+#define CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) \
+	(IRO[15].base + ((core_rx_queue_id) * IRO[15].m1))
+#define CORE_LL2_TSTORM_PER_QUEUE_STAT_SIZE		(IRO[15].size)
+#define CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) \
+	(IRO[16].base +	((core_rx_queue_id) * IRO[16].m1))
+#define CORE_LL2_USTORM_PER_QUEUE_STAT_SIZE		(IRO[16].size)
+#define CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(core_tx_stats_id) \
+	(IRO[17].base +	((core_tx_stats_id) * IRO[17].m1))
+#define CORE_LL2_PSTORM_PER_QUEUE_STAT_SIZE	(IRO[17].	size)
 #define	MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
 	(IRO[18].base + ((stat_counter_id) * IRO[18].m1))
 #define	MSTORM_QUEUE_STAT_SIZE				(IRO[18].size)
 #define	MSTORM_ETH_PF_PRODS_OFFSET(queue_id) \
 	(IRO[19].base + ((queue_id) * IRO[19].m1))
 #define	MSTORM_ETH_PF_PRODS_SIZE			(IRO[19].size)
-#define	MSTORM_TPA_TIMEOUT_US_OFFSET			(IRO[20].base)
-#define	MSTORM_TPA_TIMEOUT_US_SIZE			(IRO[20].size)
+#define MSTORM_ETH_VF_PRODS_OFFSET(vf_id, vf_queue_id) \
+	(IRO[20].base +	((vf_id) * IRO[20].m1) + ((vf_queue_id) * IRO[20].m2))
+#define MSTORM_ETH_VF_PRODS_SIZE			(IRO[20].size)
+#define	MSTORM_TPA_TIMEOUT_US_OFFSET			(IRO[21].base)
+#define	MSTORM_TPA_TIMEOUT_US_SIZE			(IRO[21].size)
 #define	MSTORM_ETH_PF_STAT_OFFSET(pf_id) \
-	(IRO[21].base + ((pf_id) * IRO[21].m1))
+	(IRO[22].base + ((pf_id) * IRO[22].m1))
 #define	MSTORM_ETH_PF_STAT_SIZE				(IRO[21].size)
 #define	USTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
-	(IRO[22].base + ((stat_counter_id) * IRO[22].m1))
-#define	USTORM_QUEUE_STAT_SIZE				(IRO[22].size)
+	(IRO[23].base + ((stat_counter_id) * IRO[23].m1))
+#define	USTORM_QUEUE_STAT_SIZE				(IRO[23].size)
 #define	USTORM_ETH_PF_STAT_OFFSET(pf_id) \
-	(IRO[23].base + ((pf_id) * IRO[23].m1))
-#define	USTORM_ETH_PF_STAT_SIZE				(IRO[23].size)
+	(IRO[24].base + ((pf_id) * IRO[24].m1))
+#define	USTORM_ETH_PF_STAT_SIZE				(IRO[24].size)
 #define	PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
-	(IRO[24].base + ((stat_counter_id) * IRO[24].m1))
-#define	PSTORM_QUEUE_STAT_SIZE				(IRO[24].size)
+	(IRO[25].base + ((stat_counter_id) * IRO[25].m1))
+#define	PSTORM_QUEUE_STAT_SIZE				(IRO[25].size)
 #define	PSTORM_ETH_PF_STAT_OFFSET(pf_id) \
-	(IRO[25].base + ((pf_id) * IRO[25].m1))
-#define	PSTORM_ETH_PF_STAT_SIZE				(IRO[25].size)
+	(IRO[26].base + ((pf_id) * IRO[26].m1))
+#define	PSTORM_ETH_PF_STAT_SIZE				(IRO[26].size)
 #define	PSTORM_CTL_FRAME_ETHTYPE_OFFSET(ethtype) \
-	(IRO[26].base + ((ethtype) * IRO[26].m1))
-#define	PSTORM_CTL_FRAME_ETHTYPE_SIZE			(IRO[26].size)
-#define	TSTORM_ETH_PRS_INPUT_OFFSET			(IRO[27].base)
-#define	TSTORM_ETH_PRS_INPUT_SIZE			(IRO[27].size)
+	(IRO[27].base + ((ethtype) * IRO[27].m1))
+#define	PSTORM_CTL_FRAME_ETHTYPE_SIZE			(IRO[27].size)
+#define	TSTORM_ETH_PRS_INPUT_OFFSET			(IRO[28].base)
+#define	TSTORM_ETH_PRS_INPUT_SIZE			(IRO[28].size)
 #define	ETH_RX_RATE_LIMIT_OFFSET(pf_id) \
-	(IRO[28].base + ((pf_id) * IRO[28].m1))
-#define	ETH_RX_RATE_LIMIT_SIZE				(IRO[28].size)
+	(IRO[29].base + ((pf_id) * IRO[29].m1))
+#define	ETH_RX_RATE_LIMIT_SIZE				(IRO[29].size)
 #define	XSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) \
-	(IRO[29].base + ((queue_id) * IRO[29].m1))
-#define	XSTORM_ETH_QUEUE_ZONE_SIZE			(IRO[29].size)
-
-static const struct iro iro_arr[46] = {
+	(IRO[30].base + ((queue_id) * IRO[30].m1))
+#define	XSTORM_ETH_QUEUE_ZONE_SIZE			(IRO[30].size)
+#define TSTORM_SCSI_CMDQ_CONS_OFFSET(cmdq_queue_id) \
+	(IRO[34].base +	((cmdq_queue_id) * IRO[34].m1))
+#define TSTORM_SCSI_CMDQ_CONS_SIZE				(IRO[34].size)
+#define TSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) \
+	(IRO[35].base +	((func_id) * IRO[35].m1) + ((bdq_id) * IRO[35].m2))
+#define TSTORM_SCSI_BDQ_EXT_PROD_SIZE				(IRO[35].size)
+#define MSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) \
+	(IRO[36].base +	((func_id) * IRO[36].m1) + ((bdq_id) * IRO[36].m2))
+#define MSTORM_SCSI_BDQ_EXT_PROD_SIZE				(IRO[36].size)
+#define TSTORM_ISCSI_RX_STATS_OFFSET(pf_id) \
+	(IRO[37].base +	((pf_id) * IRO[37].m1))
+#define TSTORM_ISCSI_RX_STATS_SIZE				(IRO[37].size)
+#define MSTORM_ISCSI_RX_STATS_OFFSET(pf_id) \
+	(IRO[38].base +	((pf_id) * IRO[38].m1))
+#define MSTORM_ISCSI_RX_STATS_SIZE				(IRO[38].size)
+#define USTORM_ISCSI_RX_STATS_OFFSET(pf_id) \
+	(IRO[39].base +	((pf_id) * IRO[39].m1))
+#define USTORM_ISCSI_RX_STATS_SIZE				(IRO[39].size)
+#define XSTORM_ISCSI_TX_STATS_OFFSET(pf_id) \
+	(IRO[40].base +	((pf_id) * IRO[40].m1))
+#define XSTORM_ISCSI_TX_STATS_SIZE				(IRO[40].size)
+#define YSTORM_ISCSI_TX_STATS_OFFSET(pf_id) \
+	(IRO[41].base +	((pf_id) * IRO[41].m1))
+#define YSTORM_ISCSI_TX_STATS_SIZE				(IRO[41].size)
+#define PSTORM_ISCSI_TX_STATS_OFFSET(pf_id) \
+	(IRO[42].base +	((pf_id) * IRO[42].m1))
+#define PSTORM_ISCSI_TX_STATS_SIZE				(IRO[42].size)
+#define PSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) \
+	(IRO[45].base +	((rdma_stat_counter_id) * IRO[45].m1))
+#define PSTORM_RDMA_QUEUE_STAT_SIZE				(IRO[45].size)
+#define TSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) \
+	(IRO[46].base +	((rdma_stat_counter_id) * IRO[46].m1))
+#define TSTORM_RDMA_QUEUE_STAT_SIZE				(IRO[46].size)
+
+static const struct iro iro_arr[47] = {
 	{0x0, 0x0, 0x0, 0x0, 0x8},
 	{0x4cb0, 0x78, 0x0, 0x0, 0x78},
 	{0x6318, 0x20, 0x0, 0x0, 0x20},
@@ -2201,20 +3466,21 @@ static const struct iro iro_arr[46] = {
 	{0x3df0, 0x0, 0x0, 0x0, 0x78},
 	{0x29b0, 0x0, 0x0, 0x0, 0x78},
 	{0x4c38, 0x0, 0x0, 0x0, 0x78},
-	{0x4a48, 0x0, 0x0, 0x0, 0x78},
+	{0x4990, 0x0, 0x0, 0x0, 0x78},
 	{0x7e48, 0x0, 0x0, 0x0, 0x78},
 	{0xa28, 0x8, 0x0, 0x0, 0x8},
 	{0x60f8, 0x10, 0x0, 0x0, 0x10},
 	{0xb820, 0x30, 0x0, 0x0, 0x30},
 	{0x95b8, 0x30, 0x0, 0x0, 0x30},
-	{0x4c18, 0x80, 0x0, 0x0, 0x40},
+	{0x4b60, 0x80, 0x0, 0x0, 0x40},
 	{0x1f8, 0x4, 0x0, 0x0, 0x4},
-	{0xc9a8, 0x0, 0x0, 0x0, 0x4},
-	{0x4c58, 0x80, 0x0, 0x0, 0x20},
+	{0x53a0, 0x80, 0x4, 0x0, 0x4},
+	{0xc8f0, 0x0, 0x0, 0x0, 0x4},
+	{0x4ba0, 0x80, 0x0, 0x0, 0x20},
 	{0x8050, 0x40, 0x0, 0x0, 0x30},
 	{0xe770, 0x60, 0x0, 0x0, 0x60},
 	{0x2b48, 0x80, 0x0, 0x0, 0x38},
-	{0xdf88, 0x78, 0x0, 0x0, 0x78},
+	{0xf188, 0x78, 0x0, 0x0, 0x78},
 	{0x1f8, 0x4, 0x0, 0x0, 0x4},
 	{0xacf0, 0x0, 0x0, 0x0, 0xf0},
 	{0xade0, 0x8, 0x0, 0x0, 0x8},
@@ -2226,455 +3492,457 @@ static const struct iro iro_arr[46] = {
 	{0x200, 0x10, 0x8, 0x0, 0x8},
 	{0xb78, 0x10, 0x8, 0x0, 0x2},
 	{0xd888, 0x38, 0x0, 0x0, 0x24},
-	{0x12120, 0x10, 0x0, 0x0, 0x8},
-	{0x11b20, 0x38, 0x0, 0x0, 0x18},
+	{0x12c38, 0x10, 0x0, 0x0, 0x8},
+	{0x11aa0, 0x38, 0x0, 0x0, 0x18},
 	{0xa8c0, 0x30, 0x0, 0x0, 0x10},
 	{0x86f8, 0x28, 0x0, 0x0, 0x18},
-	{0xeff8, 0x10, 0x0, 0x0, 0x10},
+	{0x101f8, 0x10, 0x0, 0x0, 0x10},
 	{0xdd08, 0x48, 0x0, 0x0, 0x38},
-	{0xf460, 0x20, 0x0, 0x0, 0x20},
+	{0x10660, 0x20, 0x0, 0x0, 0x20},
 	{0x2b80, 0x80, 0x0, 0x0, 0x10},
 	{0x5000, 0x10, 0x0, 0x0, 0x10},
 };
 
 /* Runtime array offsets */
-#define DORQ_REG_PF_MAX_ICID_0_RT_OFFSET 0
-#define DORQ_REG_PF_MAX_ICID_1_RT_OFFSET 1
-#define DORQ_REG_PF_MAX_ICID_2_RT_OFFSET 2
-#define DORQ_REG_PF_MAX_ICID_3_RT_OFFSET 3
-#define DORQ_REG_PF_MAX_ICID_4_RT_OFFSET 4
-#define DORQ_REG_PF_MAX_ICID_5_RT_OFFSET 5
-#define DORQ_REG_PF_MAX_ICID_6_RT_OFFSET 6
-#define DORQ_REG_PF_MAX_ICID_7_RT_OFFSET 7
-#define DORQ_REG_VF_MAX_ICID_0_RT_OFFSET 8
-#define DORQ_REG_VF_MAX_ICID_1_RT_OFFSET 9
-#define DORQ_REG_VF_MAX_ICID_2_RT_OFFSET 10
-#define DORQ_REG_VF_MAX_ICID_3_RT_OFFSET 11
-#define DORQ_REG_VF_MAX_ICID_4_RT_OFFSET 12
-#define DORQ_REG_VF_MAX_ICID_5_RT_OFFSET 13
-#define DORQ_REG_VF_MAX_ICID_6_RT_OFFSET 14
-#define DORQ_REG_VF_MAX_ICID_7_RT_OFFSET 15
-#define DORQ_REG_PF_WAKE_ALL_RT_OFFSET 16
-#define DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET 17
-#define IGU_REG_PF_CONFIGURATION_RT_OFFSET 18
-#define IGU_REG_VF_CONFIGURATION_RT_OFFSET 19
-#define IGU_REG_ATTN_MSG_ADDR_L_RT_OFFSET 20
-#define IGU_REG_ATTN_MSG_ADDR_H_RT_OFFSET 21
-#define IGU_REG_LEADING_EDGE_LATCH_RT_OFFSET 22
-#define IGU_REG_TRAILING_EDGE_LATCH_RT_OFFSET 23
-#define CAU_REG_CQE_AGG_UNIT_SIZE_RT_OFFSET 24
-#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET 761
-#define CAU_REG_SB_VAR_MEMORY_RT_SIZE 736
-#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET 761
-#define CAU_REG_SB_VAR_MEMORY_RT_SIZE 736
-#define CAU_REG_SB_ADDR_MEMORY_RT_OFFSET 1497
-#define CAU_REG_SB_ADDR_MEMORY_RT_SIZE 736
-#define CAU_REG_PI_MEMORY_RT_OFFSET 2233
-#define CAU_REG_PI_MEMORY_RT_SIZE 4416
-#define PRS_REG_SEARCH_RESP_INITIATOR_TYPE_RT_OFFSET 6649
-#define PRS_REG_TASK_ID_MAX_INITIATOR_PF_RT_OFFSET 6650
-#define PRS_REG_TASK_ID_MAX_INITIATOR_VF_RT_OFFSET 6651
-#define PRS_REG_TASK_ID_MAX_TARGET_PF_RT_OFFSET 6652
-#define PRS_REG_TASK_ID_MAX_TARGET_VF_RT_OFFSET 6653
-#define PRS_REG_SEARCH_TCP_RT_OFFSET 6654
-#define PRS_REG_SEARCH_FCOE_RT_OFFSET 6655
-#define PRS_REG_SEARCH_ROCE_RT_OFFSET 6656
-#define PRS_REG_ROCE_DEST_QP_MAX_VF_RT_OFFSET 6657
-#define PRS_REG_ROCE_DEST_QP_MAX_PF_RT_OFFSET 6658
-#define PRS_REG_SEARCH_OPENFLOW_RT_OFFSET 6659
-#define PRS_REG_SEARCH_NON_IP_AS_OPENFLOW_RT_OFFSET 6660
-#define PRS_REG_OPENFLOW_SUPPORT_ONLY_KNOWN_OVER_IP_RT_OFFSET 6661
-#define PRS_REG_OPENFLOW_SEARCH_KEY_MASK_RT_OFFSET 6662
-#define PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET 6663
-#define PRS_REG_LIGHT_L2_ETHERTYPE_EN_RT_OFFSET 6664
-#define SRC_REG_FIRSTFREE_RT_OFFSET 6665
-#define SRC_REG_FIRSTFREE_RT_SIZE 2
-#define SRC_REG_LASTFREE_RT_OFFSET 6667
-#define SRC_REG_LASTFREE_RT_SIZE 2
-#define SRC_REG_COUNTFREE_RT_OFFSET 6669
-#define SRC_REG_NUMBER_HASH_BITS_RT_OFFSET 6670
-#define PSWRQ2_REG_CDUT_P_SIZE_RT_OFFSET 6671
-#define PSWRQ2_REG_CDUC_P_SIZE_RT_OFFSET 6672
-#define PSWRQ2_REG_TM_P_SIZE_RT_OFFSET 6673
-#define PSWRQ2_REG_QM_P_SIZE_RT_OFFSET 6674
-#define PSWRQ2_REG_SRC_P_SIZE_RT_OFFSET 6675
-#define PSWRQ2_REG_TSDM_P_SIZE_RT_OFFSET 6676
-#define PSWRQ2_REG_TM_FIRST_ILT_RT_OFFSET 6677
-#define PSWRQ2_REG_TM_LAST_ILT_RT_OFFSET 6678
-#define PSWRQ2_REG_QM_FIRST_ILT_RT_OFFSET 6679
-#define PSWRQ2_REG_QM_LAST_ILT_RT_OFFSET 6680
-#define PSWRQ2_REG_SRC_FIRST_ILT_RT_OFFSET 6681
-#define PSWRQ2_REG_SRC_LAST_ILT_RT_OFFSET 6682
-#define PSWRQ2_REG_CDUC_FIRST_ILT_RT_OFFSET 6683
-#define PSWRQ2_REG_CDUC_LAST_ILT_RT_OFFSET 6684
-#define PSWRQ2_REG_CDUT_FIRST_ILT_RT_OFFSET 6685
-#define PSWRQ2_REG_CDUT_LAST_ILT_RT_OFFSET 6686
-#define PSWRQ2_REG_TSDM_FIRST_ILT_RT_OFFSET 6687
-#define PSWRQ2_REG_TSDM_LAST_ILT_RT_OFFSET 6688
-#define PSWRQ2_REG_TM_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6689
-#define PSWRQ2_REG_CDUT_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6690
-#define PSWRQ2_REG_CDUC_NUMBER_OF_PF_BLOCKS_RT_OFFSET 6691
-#define PSWRQ2_REG_TM_VF_BLOCKS_RT_OFFSET 6692
-#define PSWRQ2_REG_CDUT_VF_BLOCKS_RT_OFFSET 6693
-#define PSWRQ2_REG_CDUC_VF_BLOCKS_RT_OFFSET 6694
-#define PSWRQ2_REG_TM_BLOCKS_FACTOR_RT_OFFSET 6695
-#define PSWRQ2_REG_CDUT_BLOCKS_FACTOR_RT_OFFSET 6696
-#define PSWRQ2_REG_CDUC_BLOCKS_FACTOR_RT_OFFSET 6697
-#define PSWRQ2_REG_VF_BASE_RT_OFFSET 6698
-#define PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET 6699
-#define PSWRQ2_REG_WR_MBS0_RT_OFFSET 6700
-#define PSWRQ2_REG_RD_MBS0_RT_OFFSET 6701
-#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET 6702
-#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET 6703
-#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET 6704
-#define PSWRQ2_REG_ILT_MEMORY_RT_SIZE 22000
-#define PGLUE_REG_B_VF_BASE_RT_OFFSET 28704
-#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET 28705
-#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET 28706
-#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET 28707
-#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET 28708
-#define TM_REG_VF_ENABLE_CONN_RT_OFFSET 28709
-#define TM_REG_PF_ENABLE_CONN_RT_OFFSET 28710
-#define TM_REG_PF_ENABLE_TASK_RT_OFFSET 28711
-#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET 28712
-#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET 28713
-#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET 28714
-#define TM_REG_CONFIG_CONN_MEM_RT_SIZE 416
-#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET 29130
-#define TM_REG_CONFIG_TASK_MEM_RT_SIZE 512
-#define QM_REG_MAXPQSIZE_0_RT_OFFSET 29642
-#define QM_REG_MAXPQSIZE_1_RT_OFFSET 29643
-#define QM_REG_MAXPQSIZE_2_RT_OFFSET 29644
-#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET 29645
-#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET 29646
-#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET 29647
-#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET 29648
-#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET 29649
-#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET 29650
-#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET 29651
-#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET 29652
-#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET 29653
-#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET 29654
-#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET 29655
-#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET 29656
-#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET 29657
-#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET 29658
-#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET 29659
-#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET 29660
-#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET 29661
-#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET 29662
-#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET 29663
-#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET 29664
-#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET 29665
-#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET 29666
-#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET 29667
-#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET 29668
-#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET 29669
-#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET 29670
-#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET 29671
-#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET 29672
-#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET 29673
-#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET 29674
-#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET 29675
-#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET 29676
-#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET 29677
-#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET 29678
-#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET 29679
-#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET 29680
-#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET 29681
-#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET 29682
-#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET 29683
-#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET 29684
-#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET 29685
-#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET 29686
-#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET 29687
-#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET 29688
-#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET 29689
-#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET 29690
-#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET 29691
-#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET 29692
-#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET 29693
-#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET 29694
-#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET 29695
-#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET 29696
-#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET 29697
-#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET 29698
-#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET 29699
-#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET 29700
-#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET 29701
-#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET 29702
-#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET 29703
-#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET 29704
-#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET 29705
-#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET 29706
-#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET 29707
-#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET 29708
-#define QM_REG_BASEADDROTHERPQ_RT_OFFSET 29709
-#define QM_REG_BASEADDROTHERPQ_RT_SIZE 128
-#define QM_REG_VOQCRDLINE_RT_OFFSET 29837
-#define QM_REG_VOQCRDLINE_RT_SIZE 20
-#define QM_REG_VOQINITCRDLINE_RT_OFFSET 29857
-#define QM_REG_VOQINITCRDLINE_RT_SIZE 20
-#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET 29877
-#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET 29878
-#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET 29879
-#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET 29880
-#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET 29881
-#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET 29882
-#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET 29883
-#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET 29884
-#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET 29885
-#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET 29886
-#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET 29887
-#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET 29888
-#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET 29889
-#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET 29890
-#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET 29891
-#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET 29892
-#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET 29893
-#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET 29894
-#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET 29895
-#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET 29896
-#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET 29897
-#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET 29898
-#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET 29899
-#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET 29900
-#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET 29901
-#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET 29902
-#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET 29903
-#define QM_REG_PQTX2PF_0_RT_OFFSET 29904
-#define QM_REG_PQTX2PF_1_RT_OFFSET 29905
-#define QM_REG_PQTX2PF_2_RT_OFFSET 29906
-#define QM_REG_PQTX2PF_3_RT_OFFSET 29907
-#define QM_REG_PQTX2PF_4_RT_OFFSET 29908
-#define QM_REG_PQTX2PF_5_RT_OFFSET 29909
-#define QM_REG_PQTX2PF_6_RT_OFFSET 29910
-#define QM_REG_PQTX2PF_7_RT_OFFSET 29911
-#define QM_REG_PQTX2PF_8_RT_OFFSET 29912
-#define QM_REG_PQTX2PF_9_RT_OFFSET 29913
-#define QM_REG_PQTX2PF_10_RT_OFFSET 29914
-#define QM_REG_PQTX2PF_11_RT_OFFSET 29915
-#define QM_REG_PQTX2PF_12_RT_OFFSET 29916
-#define QM_REG_PQTX2PF_13_RT_OFFSET 29917
-#define QM_REG_PQTX2PF_14_RT_OFFSET 29918
-#define QM_REG_PQTX2PF_15_RT_OFFSET 29919
-#define QM_REG_PQTX2PF_16_RT_OFFSET 29920
-#define QM_REG_PQTX2PF_17_RT_OFFSET 29921
-#define QM_REG_PQTX2PF_18_RT_OFFSET 29922
-#define QM_REG_PQTX2PF_19_RT_OFFSET 29923
-#define QM_REG_PQTX2PF_20_RT_OFFSET 29924
-#define QM_REG_PQTX2PF_21_RT_OFFSET 29925
-#define QM_REG_PQTX2PF_22_RT_OFFSET 29926
-#define QM_REG_PQTX2PF_23_RT_OFFSET 29927
-#define QM_REG_PQTX2PF_24_RT_OFFSET 29928
-#define QM_REG_PQTX2PF_25_RT_OFFSET 29929
-#define QM_REG_PQTX2PF_26_RT_OFFSET 29930
-#define QM_REG_PQTX2PF_27_RT_OFFSET 29931
-#define QM_REG_PQTX2PF_28_RT_OFFSET 29932
-#define QM_REG_PQTX2PF_29_RT_OFFSET 29933
-#define QM_REG_PQTX2PF_30_RT_OFFSET 29934
-#define QM_REG_PQTX2PF_31_RT_OFFSET 29935
-#define QM_REG_PQTX2PF_32_RT_OFFSET 29936
-#define QM_REG_PQTX2PF_33_RT_OFFSET 29937
-#define QM_REG_PQTX2PF_34_RT_OFFSET 29938
-#define QM_REG_PQTX2PF_35_RT_OFFSET 29939
-#define QM_REG_PQTX2PF_36_RT_OFFSET 29940
-#define QM_REG_PQTX2PF_37_RT_OFFSET 29941
-#define QM_REG_PQTX2PF_38_RT_OFFSET 29942
-#define QM_REG_PQTX2PF_39_RT_OFFSET 29943
-#define QM_REG_PQTX2PF_40_RT_OFFSET 29944
-#define QM_REG_PQTX2PF_41_RT_OFFSET 29945
-#define QM_REG_PQTX2PF_42_RT_OFFSET 29946
-#define QM_REG_PQTX2PF_43_RT_OFFSET 29947
-#define QM_REG_PQTX2PF_44_RT_OFFSET 29948
-#define QM_REG_PQTX2PF_45_RT_OFFSET 29949
-#define QM_REG_PQTX2PF_46_RT_OFFSET 29950
-#define QM_REG_PQTX2PF_47_RT_OFFSET 29951
-#define QM_REG_PQTX2PF_48_RT_OFFSET 29952
-#define QM_REG_PQTX2PF_49_RT_OFFSET 29953
-#define QM_REG_PQTX2PF_50_RT_OFFSET 29954
-#define QM_REG_PQTX2PF_51_RT_OFFSET 29955
-#define QM_REG_PQTX2PF_52_RT_OFFSET 29956
-#define QM_REG_PQTX2PF_53_RT_OFFSET 29957
-#define QM_REG_PQTX2PF_54_RT_OFFSET 29958
-#define QM_REG_PQTX2PF_55_RT_OFFSET 29959
-#define QM_REG_PQTX2PF_56_RT_OFFSET 29960
-#define QM_REG_PQTX2PF_57_RT_OFFSET 29961
-#define QM_REG_PQTX2PF_58_RT_OFFSET 29962
-#define QM_REG_PQTX2PF_59_RT_OFFSET 29963
-#define QM_REG_PQTX2PF_60_RT_OFFSET 29964
-#define QM_REG_PQTX2PF_61_RT_OFFSET 29965
-#define QM_REG_PQTX2PF_62_RT_OFFSET 29966
-#define QM_REG_PQTX2PF_63_RT_OFFSET 29967
-#define QM_REG_PQOTHER2PF_0_RT_OFFSET 29968
-#define QM_REG_PQOTHER2PF_1_RT_OFFSET 29969
-#define QM_REG_PQOTHER2PF_2_RT_OFFSET 29970
-#define QM_REG_PQOTHER2PF_3_RT_OFFSET 29971
-#define QM_REG_PQOTHER2PF_4_RT_OFFSET 29972
-#define QM_REG_PQOTHER2PF_5_RT_OFFSET 29973
-#define QM_REG_PQOTHER2PF_6_RT_OFFSET 29974
-#define QM_REG_PQOTHER2PF_7_RT_OFFSET 29975
-#define QM_REG_PQOTHER2PF_8_RT_OFFSET 29976
-#define QM_REG_PQOTHER2PF_9_RT_OFFSET 29977
-#define QM_REG_PQOTHER2PF_10_RT_OFFSET 29978
-#define QM_REG_PQOTHER2PF_11_RT_OFFSET 29979
-#define QM_REG_PQOTHER2PF_12_RT_OFFSET 29980
-#define QM_REG_PQOTHER2PF_13_RT_OFFSET 29981
-#define QM_REG_PQOTHER2PF_14_RT_OFFSET 29982
-#define QM_REG_PQOTHER2PF_15_RT_OFFSET 29983
-#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET 29984
-#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET 29985
-#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET 29986
-#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET 29987
-#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET 29988
-#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET 29989
-#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET 29990
-#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET 29991
-#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET 29992
-#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET 29993
-#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET 29994
-#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET 29995
-#define QM_REG_RLGLBLINCVAL_RT_OFFSET 29996
-#define QM_REG_RLGLBLINCVAL_RT_SIZE 256
-#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET 30252
-#define QM_REG_RLGLBLUPPERBOUND_RT_SIZE 256
-#define QM_REG_RLGLBLCRD_RT_OFFSET 30508
-#define QM_REG_RLGLBLCRD_RT_SIZE 256
-#define QM_REG_RLGLBLENABLE_RT_OFFSET 30764
-#define QM_REG_RLPFPERIOD_RT_OFFSET 30765
-#define QM_REG_RLPFPERIODTIMER_RT_OFFSET 30766
-#define QM_REG_RLPFINCVAL_RT_OFFSET 30767
-#define QM_REG_RLPFINCVAL_RT_SIZE 16
-#define QM_REG_RLPFUPPERBOUND_RT_OFFSET 30783
-#define QM_REG_RLPFUPPERBOUND_RT_SIZE 16
-#define QM_REG_RLPFCRD_RT_OFFSET 30799
-#define QM_REG_RLPFCRD_RT_SIZE 16
-#define QM_REG_RLPFENABLE_RT_OFFSET 30815
-#define QM_REG_RLPFVOQENABLE_RT_OFFSET 30816
-#define QM_REG_WFQPFWEIGHT_RT_OFFSET 30817
-#define QM_REG_WFQPFWEIGHT_RT_SIZE 16
-#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET 30833
-#define QM_REG_WFQPFUPPERBOUND_RT_SIZE 16
-#define QM_REG_WFQPFCRD_RT_OFFSET 30849
-#define QM_REG_WFQPFCRD_RT_SIZE 160
-#define QM_REG_WFQPFENABLE_RT_OFFSET 31009
-#define QM_REG_WFQVPENABLE_RT_OFFSET 31010
-#define QM_REG_BASEADDRTXPQ_RT_OFFSET 31011
-#define QM_REG_BASEADDRTXPQ_RT_SIZE 512
-#define QM_REG_TXPQMAP_RT_OFFSET 31523
-#define QM_REG_TXPQMAP_RT_SIZE 512
-#define QM_REG_WFQVPWEIGHT_RT_OFFSET 32035
-#define QM_REG_WFQVPWEIGHT_RT_SIZE 512
-#define QM_REG_WFQVPCRD_RT_OFFSET 32547
-#define QM_REG_WFQVPCRD_RT_SIZE 512
-#define QM_REG_WFQVPMAP_RT_OFFSET 33059
-#define QM_REG_WFQVPMAP_RT_SIZE 512
-#define QM_REG_WFQPFCRD_MSB_RT_OFFSET 33571
-#define QM_REG_WFQPFCRD_MSB_RT_SIZE 160
-#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET 33731
-#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET 33732
-#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET 33733
-#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET 33734
-#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET 33735
-#define NIG_REG_OUTER_TAG_VALUE_MASK_RT_OFFSET 33736
-#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET 33737
-#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET 33738
-#define NIG_REG_LLH_FUNC_TAG_EN_RT_SIZE 4
-#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_OFFSET 33742
-#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_SIZE 4
-#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET 33746
-#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_SIZE 4
-#define NIG_REG_LLH_FUNC_NO_TAG_RT_OFFSET 33750
-#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET 33751
-#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_SIZE 32
-#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET 33783
-#define NIG_REG_LLH_FUNC_FILTER_EN_RT_SIZE 16
-#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET 33799
-#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_SIZE 16
-#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET 33815
-#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE 16
-#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET 33831
-#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE 16
-#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET 33847
-#define NIG_REG_ROCE_DUPLICATE_TO_HOST_RT_OFFSET 33848
-#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET 33849
-#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET 33850
-#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET 33851
-#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET 33852
-#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET 33853
-#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET 33854
-#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET 33855
-#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET 33856
-#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET 33857
-#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET 33858
-#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET 33859
-#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET 33860
-#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET 33861
-#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET 33862
-#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET 33863
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET 33864
-#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET 33865
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET 33866
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET 33867
-#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET 33868
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET 33869
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET 33870
-#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET 33871
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET 33872
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET 33873
-#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET 33874
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET 33875
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET 33876
-#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET 33877
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET 33878
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET 33879
-#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET 33880
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET 33881
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET 33882
-#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET 33883
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET 33884
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET 33885
-#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET 33886
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET 33887
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET 33888
-#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET 33889
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET 33890
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET 33891
-#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET 33892
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET 33893
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET 33894
-#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET 33895
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET 33896
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET 33897
-#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET 33898
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET 33899
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET 33900
-#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET 33901
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET 33902
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET 33903
-#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET 33904
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET 33905
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET 33906
-#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET 33907
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET 33908
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET 33909
-#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET 33910
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET 33911
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET 33912
-#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET 33913
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET 33914
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET 33915
-#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET 33916
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET 33917
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET 33918
-#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET 33919
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET 33920
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET 33921
-#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET 33922
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET 33923
-#define XCM_REG_CON_PHY_Q3_RT_OFFSET 33924
-
-#define RUNTIME_ARRAY_SIZE 33925
+#define DORQ_REG_PF_MAX_ICID_0_RT_OFFSET	0
+#define DORQ_REG_PF_MAX_ICID_1_RT_OFFSET	1
+#define DORQ_REG_PF_MAX_ICID_2_RT_OFFSET	2
+#define DORQ_REG_PF_MAX_ICID_3_RT_OFFSET	3
+#define DORQ_REG_PF_MAX_ICID_4_RT_OFFSET	4
+#define DORQ_REG_PF_MAX_ICID_5_RT_OFFSET	5
+#define DORQ_REG_PF_MAX_ICID_6_RT_OFFSET	6
+#define DORQ_REG_PF_MAX_ICID_7_RT_OFFSET	7
+#define DORQ_REG_VF_MAX_ICID_0_RT_OFFSET	8
+#define DORQ_REG_VF_MAX_ICID_1_RT_OFFSET	9
+#define DORQ_REG_VF_MAX_ICID_2_RT_OFFSET	10
+#define DORQ_REG_VF_MAX_ICID_3_RT_OFFSET	11
+#define DORQ_REG_VF_MAX_ICID_4_RT_OFFSET	12
+#define DORQ_REG_VF_MAX_ICID_5_RT_OFFSET	13
+#define DORQ_REG_VF_MAX_ICID_6_RT_OFFSET	14
+#define DORQ_REG_VF_MAX_ICID_7_RT_OFFSET	15
+#define DORQ_REG_PF_WAKE_ALL_RT_OFFSET	16
+#define DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET	17
+#define IGU_REG_PF_CONFIGURATION_RT_OFFSET	18
+#define IGU_REG_VF_CONFIGURATION_RT_OFFSET	19
+#define IGU_REG_ATTN_MSG_ADDR_L_RT_OFFSET	20
+#define IGU_REG_ATTN_MSG_ADDR_H_RT_OFFSET	21
+#define IGU_REG_LEADING_EDGE_LATCH_RT_OFFSET	22
+#define IGU_REG_TRAILING_EDGE_LATCH_RT_OFFSET	23
+#define CAU_REG_CQE_AGG_UNIT_SIZE_RT_OFFSET	24
+#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET	761
+#define CAU_REG_SB_VAR_MEMORY_RT_SIZE	736
+#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET	761
+#define CAU_REG_SB_VAR_MEMORY_RT_SIZE	736
+#define CAU_REG_SB_ADDR_MEMORY_RT_OFFSET	1497
+#define CAU_REG_SB_ADDR_MEMORY_RT_SIZE	736
+#define CAU_REG_PI_MEMORY_RT_OFFSET	2233
+#define CAU_REG_PI_MEMORY_RT_SIZE	4416
+#define PRS_REG_SEARCH_RESP_INITIATOR_TYPE_RT_OFFSET	6649
+#define PRS_REG_TASK_ID_MAX_INITIATOR_PF_RT_OFFSET	6650
+#define PRS_REG_TASK_ID_MAX_INITIATOR_VF_RT_OFFSET	6651
+#define PRS_REG_TASK_ID_MAX_TARGET_PF_RT_OFFSET	6652
+#define PRS_REG_TASK_ID_MAX_TARGET_VF_RT_OFFSET	6653
+#define PRS_REG_SEARCH_TCP_RT_OFFSET	6654
+#define PRS_REG_SEARCH_FCOE_RT_OFFSET	6655
+#define PRS_REG_SEARCH_ROCE_RT_OFFSET	6656
+#define PRS_REG_ROCE_DEST_QP_MAX_VF_RT_OFFSET	6657
+#define PRS_REG_ROCE_DEST_QP_MAX_PF_RT_OFFSET	6658
+#define PRS_REG_SEARCH_OPENFLOW_RT_OFFSET	6659
+#define PRS_REG_SEARCH_NON_IP_AS_OPENFLOW_RT_OFFSET	6660
+#define PRS_REG_OPENFLOW_SUPPORT_ONLY_KNOWN_OVER_IP_RT_OFFSET	6661
+#define PRS_REG_OPENFLOW_SEARCH_KEY_MASK_RT_OFFSET	6662
+#define PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET	6663
+#define PRS_REG_LIGHT_L2_ETHERTYPE_EN_RT_OFFSET	6664
+#define SRC_REG_FIRSTFREE_RT_OFFSET	6665
+#define SRC_REG_FIRSTFREE_RT_SIZE	2
+#define SRC_REG_LASTFREE_RT_OFFSET	6667
+#define SRC_REG_LASTFREE_RT_SIZE	2
+#define SRC_REG_COUNTFREE_RT_OFFSET	6669
+#define SRC_REG_NUMBER_HASH_BITS_RT_OFFSET	6670
+#define PSWRQ2_REG_CDUT_P_SIZE_RT_OFFSET	6671
+#define PSWRQ2_REG_CDUC_P_SIZE_RT_OFFSET	6672
+#define PSWRQ2_REG_TM_P_SIZE_RT_OFFSET	6673
+#define PSWRQ2_REG_QM_P_SIZE_RT_OFFSET	6674
+#define PSWRQ2_REG_SRC_P_SIZE_RT_OFFSET	6675
+#define PSWRQ2_REG_TSDM_P_SIZE_RT_OFFSET	6676
+#define PSWRQ2_REG_TM_FIRST_ILT_RT_OFFSET	6677
+#define PSWRQ2_REG_TM_LAST_ILT_RT_OFFSET	6678
+#define PSWRQ2_REG_QM_FIRST_ILT_RT_OFFSET	6679
+#define PSWRQ2_REG_QM_LAST_ILT_RT_OFFSET	6680
+#define PSWRQ2_REG_SRC_FIRST_ILT_RT_OFFSET	6681
+#define PSWRQ2_REG_SRC_LAST_ILT_RT_OFFSET	6682
+#define PSWRQ2_REG_CDUC_FIRST_ILT_RT_OFFSET	6683
+#define PSWRQ2_REG_CDUC_LAST_ILT_RT_OFFSET	6684
+#define PSWRQ2_REG_CDUT_FIRST_ILT_RT_OFFSET	6685
+#define PSWRQ2_REG_CDUT_LAST_ILT_RT_OFFSET	6686
+#define PSWRQ2_REG_TSDM_FIRST_ILT_RT_OFFSET	6687
+#define PSWRQ2_REG_TSDM_LAST_ILT_RT_OFFSET	6688
+#define PSWRQ2_REG_TM_NUMBER_OF_PF_BLOCKS_RT_OFFSET	6689
+#define PSWRQ2_REG_CDUT_NUMBER_OF_PF_BLOCKS_RT_OFFSET	6690
+#define PSWRQ2_REG_CDUC_NUMBER_OF_PF_BLOCKS_RT_OFFSET	6691
+#define PSWRQ2_REG_TM_VF_BLOCKS_RT_OFFSET	6692
+#define PSWRQ2_REG_CDUT_VF_BLOCKS_RT_OFFSET	6693
+#define PSWRQ2_REG_CDUC_VF_BLOCKS_RT_OFFSET	6694
+#define PSWRQ2_REG_TM_BLOCKS_FACTOR_RT_OFFSET	6695
+#define PSWRQ2_REG_CDUT_BLOCKS_FACTOR_RT_OFFSET	6696
+#define PSWRQ2_REG_CDUC_BLOCKS_FACTOR_RT_OFFSET	6697
+#define PSWRQ2_REG_VF_BASE_RT_OFFSET	6698
+#define PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET	6699
+#define PSWRQ2_REG_WR_MBS0_RT_OFFSET	6700
+#define PSWRQ2_REG_RD_MBS0_RT_OFFSET	6701
+#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET	6702
+#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET	6703
+#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET	6704
+#define PSWRQ2_REG_ILT_MEMORY_RT_SIZE	22000
+#define PGLUE_REG_B_VF_BASE_RT_OFFSET	28704
+#define PGLUE_REG_B_MSDM_OFFSET_MASK_B_RT_OFFSET	28705
+#define PGLUE_REG_B_MSDM_VF_SHIFT_B_RT_OFFSET	28706
+#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET	28707
+#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET	28708
+#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET	28709
+#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET	28710
+#define TM_REG_VF_ENABLE_CONN_RT_OFFSET	28711
+#define TM_REG_PF_ENABLE_CONN_RT_OFFSET	28712
+#define TM_REG_PF_ENABLE_TASK_RT_OFFSET	28713
+#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET	28714
+#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET	28715
+#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET	28716
+#define TM_REG_CONFIG_CONN_MEM_RT_SIZE	416
+#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET	29132
+#define TM_REG_CONFIG_TASK_MEM_RT_SIZE	512
+#define QM_REG_MAXPQSIZE_0_RT_OFFSET	29644
+#define QM_REG_MAXPQSIZE_1_RT_OFFSET	29645
+#define QM_REG_MAXPQSIZE_2_RT_OFFSET	29646
+#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET	29647
+#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET	29648
+#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET	29649
+#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET	29650
+#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET	29651
+#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET	29652
+#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET	29653
+#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET	29654
+#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET	29655
+#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET	29656
+#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET	29657
+#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET	29658
+#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET	29659
+#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET	29660
+#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET	29661
+#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET	29662
+#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET	29663
+#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET	29664
+#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET	29665
+#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET	29666
+#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET	29667
+#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET	29668
+#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET	29669
+#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET	29670
+#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET	29671
+#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET	29672
+#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET	29673
+#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET	29674
+#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET	29675
+#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET	29676
+#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET	29677
+#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET	29678
+#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET	29679
+#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET	29680
+#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET	29681
+#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET	29682
+#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET	29683
+#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET	29684
+#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET	29685
+#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET	29686
+#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET	29687
+#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET	29688
+#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET	29689
+#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET	29690
+#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET	29691
+#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET	29692
+#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET	29693
+#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET	29694
+#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET	29695
+#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET	29696
+#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET	29697
+#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET	29698
+#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET	29699
+#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET	29700
+#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET	29701
+#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET	29702
+#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET	29703
+#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET	29704
+#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET	29705
+#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET	29706
+#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET	29707
+#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET	29708
+#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET	29709
+#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET	29710
+#define QM_REG_BASEADDROTHERPQ_RT_OFFSET	29711
+#define QM_REG_BASEADDROTHERPQ_RT_SIZE	128
+#define QM_REG_VOQCRDLINE_RT_OFFSET	29839
+#define QM_REG_VOQCRDLINE_RT_SIZE	20
+#define QM_REG_VOQINITCRDLINE_RT_OFFSET	29859
+#define QM_REG_VOQINITCRDLINE_RT_SIZE	20
+#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET	29879
+#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET	29880
+#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET	29881
+#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET	29882
+#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET	29883
+#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET	29884
+#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET	29885
+#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET	29886
+#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET	29887
+#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET	29888
+#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET	29889
+#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET	29890
+#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET	29891
+#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET	29892
+#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET	29893
+#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET	29894
+#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET	29895
+#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET	29896
+#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET	29897
+#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET	29898
+#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET	29899
+#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET	29900
+#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET	29901
+#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET	29902
+#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET	29903
+#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET	29904
+#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET	29905
+#define QM_REG_PQTX2PF_0_RT_OFFSET	29906
+#define QM_REG_PQTX2PF_1_RT_OFFSET	29907
+#define QM_REG_PQTX2PF_2_RT_OFFSET	29908
+#define QM_REG_PQTX2PF_3_RT_OFFSET	29909
+#define QM_REG_PQTX2PF_4_RT_OFFSET	29910
+#define QM_REG_PQTX2PF_5_RT_OFFSET	29911
+#define QM_REG_PQTX2PF_6_RT_OFFSET	29912
+#define QM_REG_PQTX2PF_7_RT_OFFSET	29913
+#define QM_REG_PQTX2PF_8_RT_OFFSET	29914
+#define QM_REG_PQTX2PF_9_RT_OFFSET	29915
+#define QM_REG_PQTX2PF_10_RT_OFFSET	29916
+#define QM_REG_PQTX2PF_11_RT_OFFSET	29917
+#define QM_REG_PQTX2PF_12_RT_OFFSET	29918
+#define QM_REG_PQTX2PF_13_RT_OFFSET	29919
+#define QM_REG_PQTX2PF_14_RT_OFFSET	29920
+#define QM_REG_PQTX2PF_15_RT_OFFSET	29921
+#define QM_REG_PQTX2PF_16_RT_OFFSET	29922
+#define QM_REG_PQTX2PF_17_RT_OFFSET	29923
+#define QM_REG_PQTX2PF_18_RT_OFFSET	29924
+#define QM_REG_PQTX2PF_19_RT_OFFSET	29925
+#define QM_REG_PQTX2PF_20_RT_OFFSET	29926
+#define QM_REG_PQTX2PF_21_RT_OFFSET	29927
+#define QM_REG_PQTX2PF_22_RT_OFFSET	29928
+#define QM_REG_PQTX2PF_23_RT_OFFSET	29929
+#define QM_REG_PQTX2PF_24_RT_OFFSET	29930
+#define QM_REG_PQTX2PF_25_RT_OFFSET	29931
+#define QM_REG_PQTX2PF_26_RT_OFFSET	29932
+#define QM_REG_PQTX2PF_27_RT_OFFSET	29933
+#define QM_REG_PQTX2PF_28_RT_OFFSET	29934
+#define QM_REG_PQTX2PF_29_RT_OFFSET	29935
+#define QM_REG_PQTX2PF_30_RT_OFFSET	29936
+#define QM_REG_PQTX2PF_31_RT_OFFSET	29937
+#define QM_REG_PQTX2PF_32_RT_OFFSET	29938
+#define QM_REG_PQTX2PF_33_RT_OFFSET	29939
+#define QM_REG_PQTX2PF_34_RT_OFFSET	29940
+#define QM_REG_PQTX2PF_35_RT_OFFSET	29941
+#define QM_REG_PQTX2PF_36_RT_OFFSET	29942
+#define QM_REG_PQTX2PF_37_RT_OFFSET	29943
+#define QM_REG_PQTX2PF_38_RT_OFFSET	29944
+#define QM_REG_PQTX2PF_39_RT_OFFSET	29945
+#define QM_REG_PQTX2PF_40_RT_OFFSET	29946
+#define QM_REG_PQTX2PF_41_RT_OFFSET	29947
+#define QM_REG_PQTX2PF_42_RT_OFFSET	29948
+#define QM_REG_PQTX2PF_43_RT_OFFSET	29949
+#define QM_REG_PQTX2PF_44_RT_OFFSET	29950
+#define QM_REG_PQTX2PF_45_RT_OFFSET	29951
+#define QM_REG_PQTX2PF_46_RT_OFFSET	29952
+#define QM_REG_PQTX2PF_47_RT_OFFSET	29953
+#define QM_REG_PQTX2PF_48_RT_OFFSET	29954
+#define QM_REG_PQTX2PF_49_RT_OFFSET	29955
+#define QM_REG_PQTX2PF_50_RT_OFFSET	29956
+#define QM_REG_PQTX2PF_51_RT_OFFSET	29957
+#define QM_REG_PQTX2PF_52_RT_OFFSET	29958
+#define QM_REG_PQTX2PF_53_RT_OFFSET	29959
+#define QM_REG_PQTX2PF_54_RT_OFFSET	29960
+#define QM_REG_PQTX2PF_55_RT_OFFSET	29961
+#define QM_REG_PQTX2PF_56_RT_OFFSET	29962
+#define QM_REG_PQTX2PF_57_RT_OFFSET	29963
+#define QM_REG_PQTX2PF_58_RT_OFFSET	29964
+#define QM_REG_PQTX2PF_59_RT_OFFSET	29965
+#define QM_REG_PQTX2PF_60_RT_OFFSET	29966
+#define QM_REG_PQTX2PF_61_RT_OFFSET	29967
+#define QM_REG_PQTX2PF_62_RT_OFFSET	29968
+#define QM_REG_PQTX2PF_63_RT_OFFSET	29969
+#define QM_REG_PQOTHER2PF_0_RT_OFFSET	29970
+#define QM_REG_PQOTHER2PF_1_RT_OFFSET	29971
+#define QM_REG_PQOTHER2PF_2_RT_OFFSET	29972
+#define QM_REG_PQOTHER2PF_3_RT_OFFSET	29973
+#define QM_REG_PQOTHER2PF_4_RT_OFFSET	29974
+#define QM_REG_PQOTHER2PF_5_RT_OFFSET	29975
+#define QM_REG_PQOTHER2PF_6_RT_OFFSET	29976
+#define QM_REG_PQOTHER2PF_7_RT_OFFSET	29977
+#define QM_REG_PQOTHER2PF_8_RT_OFFSET	29978
+#define QM_REG_PQOTHER2PF_9_RT_OFFSET	29979
+#define QM_REG_PQOTHER2PF_10_RT_OFFSET	29980
+#define QM_REG_PQOTHER2PF_11_RT_OFFSET	29981
+#define QM_REG_PQOTHER2PF_12_RT_OFFSET	29982
+#define QM_REG_PQOTHER2PF_13_RT_OFFSET	29983
+#define QM_REG_PQOTHER2PF_14_RT_OFFSET	29984
+#define QM_REG_PQOTHER2PF_15_RT_OFFSET	29985
+#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET	29986
+#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET	29987
+#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET	29988
+#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET	29989
+#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET	29990
+#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET	29991
+#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET	29992
+#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET	29993
+#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET	29994
+#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET	29995
+#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET	29996
+#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET	29997
+#define QM_REG_RLGLBLINCVAL_RT_OFFSET	29998
+#define QM_REG_RLGLBLINCVAL_RT_SIZE	256
+#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET	30254
+#define QM_REG_RLGLBLUPPERBOUND_RT_SIZE	256
+#define QM_REG_RLGLBLCRD_RT_OFFSET	30510
+#define QM_REG_RLGLBLCRD_RT_SIZE	256
+#define QM_REG_RLGLBLENABLE_RT_OFFSET	30766
+#define QM_REG_RLPFPERIOD_RT_OFFSET	30767
+#define QM_REG_RLPFPERIODTIMER_RT_OFFSET	30768
+#define QM_REG_RLPFINCVAL_RT_OFFSET	30769
+#define QM_REG_RLPFINCVAL_RT_SIZE	16
+#define QM_REG_RLPFUPPERBOUND_RT_OFFSET	30785
+#define QM_REG_RLPFUPPERBOUND_RT_SIZE	16
+#define QM_REG_RLPFCRD_RT_OFFSET	30801
+#define QM_REG_RLPFCRD_RT_SIZE	16
+#define QM_REG_RLPFENABLE_RT_OFFSET	30817
+#define QM_REG_RLPFVOQENABLE_RT_OFFSET	30818
+#define QM_REG_WFQPFWEIGHT_RT_OFFSET	30819
+#define QM_REG_WFQPFWEIGHT_RT_SIZE	16
+#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET	30835
+#define QM_REG_WFQPFUPPERBOUND_RT_SIZE	16
+#define QM_REG_WFQPFCRD_RT_OFFSET	30851
+#define QM_REG_WFQPFCRD_RT_SIZE	160
+#define QM_REG_WFQPFENABLE_RT_OFFSET	31011
+#define QM_REG_WFQVPENABLE_RT_OFFSET	31012
+#define QM_REG_BASEADDRTXPQ_RT_OFFSET	31013
+#define QM_REG_BASEADDRTXPQ_RT_SIZE	512
+#define QM_REG_TXPQMAP_RT_OFFSET	31525
+#define QM_REG_TXPQMAP_RT_SIZE	512
+#define QM_REG_WFQVPWEIGHT_RT_OFFSET	32037
+#define QM_REG_WFQVPWEIGHT_RT_SIZE	512
+#define QM_REG_WFQVPCRD_RT_OFFSET	32549
+#define QM_REG_WFQVPCRD_RT_SIZE	512
+#define QM_REG_WFQVPMAP_RT_OFFSET	33061
+#define QM_REG_WFQVPMAP_RT_SIZE	512
+#define QM_REG_WFQPFCRD_MSB_RT_OFFSET	33573
+#define QM_REG_WFQPFCRD_MSB_RT_SIZE	160
+#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET	33733
+#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET	33734
+#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET	33735
+#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET	33736
+#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET	33737
+#define NIG_REG_OUTER_TAG_VALUE_MASK_RT_OFFSET	33738
+#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET	33739
+#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET	33740
+#define NIG_REG_LLH_FUNC_TAG_EN_RT_SIZE	4
+#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_OFFSET	33744
+#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_SIZE	4
+#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET	33748
+#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_SIZE	4
+#define NIG_REG_LLH_FUNC_NO_TAG_RT_OFFSET	33752
+#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET	33753
+#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_SIZE	32
+#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET	33785
+#define NIG_REG_LLH_FUNC_FILTER_EN_RT_SIZE	16
+#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET	33801
+#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_SIZE	16
+#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET	33817
+#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE	16
+#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET	33833
+#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE	16
+#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET	33849
+#define NIG_REG_ROCE_DUPLICATE_TO_HOST_RT_OFFSET	33850
+#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET	33851
+#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET	33852
+#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET	33853
+#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET	33854
+#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET	33855
+#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET	33856
+#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET	33857
+#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET	33858
+#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET	33859
+#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET	33860
+#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET	33861
+#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET	33862
+#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET	33863
+#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET	33864
+#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET	33865
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET	33866
+#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET	33867
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET	33868
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET	33869
+#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET	33870
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET	33871
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET	33872
+#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET	33873
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET	33874
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET	33875
+#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET	33876
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET	33877
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET	33878
+#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET	33879
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET	33880
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET	33881
+#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET	33882
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET	33883
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET	33884
+#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET	33885
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET	33886
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET	33887
+#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET	33888
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET	33889
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET	33890
+#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET	33891
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET	33892
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET	33893
+#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET	33894
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET	33895
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET	33896
+#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET	33897
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET	33898
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET	33899
+#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET	33900
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET	33901
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET	33902
+#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET	33903
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET	33904
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET	33905
+#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET	33906
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET	33907
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET	33908
+#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET	33909
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET	33910
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET	33911
+#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET	33912
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET	33913
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET	33914
+#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET	33915
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET	33916
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET	33917
+#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET	33918
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET	33919
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET	33920
+#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET	33921
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET	33922
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET	33923
+#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET	33924
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET	33925
+#define XCM_REG_CON_PHY_Q3_RT_OFFSET	33926
+
+#define RUNTIME_ARRAY_SIZE 33927
 
 /* The eth storm context for the Tstorm */
 struct tstorm_eth_conn_st_ctx {
@@ -3201,7 +4469,31 @@ struct eth_conn_context {
 	struct mstorm_eth_conn_st_ctx mstorm_st_context;
 };
 
-/* opcodes for the event ring */
+enum eth_error_code {
+	ETH_OK = 0x00,
+	ETH_FILTERS_MAC_ADD_FAIL_FULL,
+	ETH_FILTERS_MAC_ADD_FAIL_FULL_MTT2,
+	ETH_FILTERS_MAC_ADD_FAIL_DUP_MTT2,
+	ETH_FILTERS_MAC_ADD_FAIL_DUP_STT2,
+	ETH_FILTERS_MAC_DEL_FAIL_NOF,
+	ETH_FILTERS_MAC_DEL_FAIL_NOF_MTT2,
+	ETH_FILTERS_MAC_DEL_FAIL_NOF_STT2,
+	ETH_FILTERS_MAC_ADD_FAIL_ZERO_MAC,
+	ETH_FILTERS_VLAN_ADD_FAIL_FULL,
+	ETH_FILTERS_VLAN_ADD_FAIL_DUP,
+	ETH_FILTERS_VLAN_DEL_FAIL_NOF,
+	ETH_FILTERS_VLAN_DEL_FAIL_NOF_TT1,
+	ETH_FILTERS_PAIR_ADD_FAIL_DUP,
+	ETH_FILTERS_PAIR_ADD_FAIL_FULL,
+	ETH_FILTERS_PAIR_ADD_FAIL_FULL_MAC,
+	ETH_FILTERS_PAIR_DEL_FAIL_NOF,
+	ETH_FILTERS_PAIR_DEL_FAIL_NOF_TT1,
+	ETH_FILTERS_PAIR_ADD_FAIL_ZERO_MAC,
+	ETH_FILTERS_VNI_ADD_FAIL_FULL,
+	ETH_FILTERS_VNI_ADD_FAIL_DUP,
+	MAX_ETH_ERROR_CODE
+};
+
 enum eth_event_opcode {
 	ETH_EVENT_UNUSED,
 	ETH_EVENT_VPORT_START,
@@ -3269,7 +4561,13 @@ enum eth_filter_type {
 	MAX_ETH_FILTER_TYPE
 };
 
-/* Ethernet Ramrod Command IDs */
+enum eth_ipv4_frag_type {
+	ETH_IPV4_NOT_FRAG,
+	ETH_IPV4_FIRST_FRAG,
+	ETH_IPV4_NON_FIRST_FRAG,
+	MAX_ETH_IPV4_FRAG_TYPE
+};
+
 enum eth_ramrod_cmd_id {
 	ETH_RAMROD_UNUSED,
 	ETH_RAMROD_VPORT_START,
@@ -3451,8 +4749,8 @@ struct rx_queue_start_ramrod_data {
 	u8 toggle_val;
 
 	u8 vf_rx_prod_index;
-
-	u8 reserved[6];
+	u8 vf_rx_prod_use_zone_a;
+	u8 reserved[5];
 	__le16 reserved1;
 	struct regpair cqe_pbl_addr;
 	struct regpair bd_base;
@@ -3526,10 +4824,11 @@ struct tx_queue_start_ramrod_data {
 	__le16 pxp_st_index;
 	__le16 comp_agg_size;
 	__le16 queue_zone_id;
-	__le16 test_dup_count;
+	__le16 reserved2;
 	__le16 pbl_size;
 	__le16 tx_queue_id;
-
+	__le16 same_as_last_id;
+	__le16 reserved[3];
 	struct regpair pbl_base_addr;
 	struct regpair bd_cons_address;
 };
@@ -4926,8 +6225,8 @@ struct roce_create_qp_resp_ramrod_data {
 #define ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG_SHIFT             5
 #define ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN_MASK  0x1
 #define ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN_SHIFT 6
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED0_MASK            0x1
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED0_SHIFT           7
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN_MASK	0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN_SHIFT	7
 #define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_MASK                  0x7
 #define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_SHIFT                 8
 #define ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_MASK    0x1F
@@ -4988,6 +6287,10 @@ enum roce_event_opcode {
 	MAX_ROCE_EVENT_OPCODE
 };
 
+struct roce_init_func_ramrod_data {
+	struct rdma_init_func_ramrod_data rdma;
+};
+
 struct roce_modify_qp_req_ramrod_data {
 	__le16 flags;
 #define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG_MASK      0x1
@@ -6639,6 +7942,35 @@ struct ystorm_iscsi_conn_ag_ctx {
 	__le32 reg2;
 	__le32 reg3;
 };
+
+#define MFW_TRACE_SIGNATURE     0x25071946
+
+/* The trace in the buffer */
+#define MFW_TRACE_EVENTID_MASK          0x00ffff
+#define MFW_TRACE_PRM_SIZE_MASK         0x0f0000
+#define MFW_TRACE_PRM_SIZE_SHIFT        16
+#define MFW_TRACE_ENTRY_SIZE            3
+
+struct mcp_trace {
+	u32 signature;		/* Help to identify that the trace is valid */
+	u32 size;		/* the size of the trace buffer in bytes */
+	u32 curr_level;		/* 2 - all will be written to the buffer
+				 * 1 - debug trace will not be written
+				 * 0 - just errors will be written to the buffer
+				 */
+	u32 modules_mask[2];	/* a bit per module, 1 means write it, 0 means
+				 * mask it.
+				 */
+
+	/* Warning: the following pointers are assumed to be 32bits as they are
+	 * used only in the MFW.
+	 */
+	u32 trace_prod; /* The next trace will be written to this offset */
+	u32 trace_oldest; /* The oldest valid trace starts at this offset
+			   * (usually very close after the current producer).
+			   */
+};
+
 #define VF_MAX_STATIC 192
 
 #define MCP_GLOB_PATH_MAX	2
@@ -6646,6 +7978,7 @@ struct ystorm_iscsi_conn_ag_ctx {
 #define MCP_GLOB_PORT_MAX	4
 #define MCP_GLOB_FUNC_MAX	16
 
+typedef u32 offsize_t;		/* In DWORDS !!! */
 /* Offset from the beginning of the MCP scratchpad */
 #define OFFSIZE_OFFSET_SHIFT	0
 #define OFFSIZE_OFFSET_MASK	0x0000ffff
@@ -6850,6 +8183,14 @@ struct dcbx_app_priority_entry {
 #define DCBX_APP_SF_SHIFT		8
 #define DCBX_APP_SF_ETHTYPE		0
 #define DCBX_APP_SF_PORT		1
+#define DCBX_APP_SF_IEEE_MASK		0x0000f000
+#define DCBX_APP_SF_IEEE_SHIFT		12
+#define DCBX_APP_SF_IEEE_RESERVED	0
+#define DCBX_APP_SF_IEEE_ETHTYPE	1
+#define DCBX_APP_SF_IEEE_TCP_PORT	2
+#define DCBX_APP_SF_IEEE_UDP_PORT	3
+#define DCBX_APP_SF_IEEE_TCP_UDP_PORT	4
+
 #define DCBX_APP_PROTOCOL_ID_MASK	0xffff0000
 #define DCBX_APP_PROTOCOL_ID_SHIFT	16
 };
@@ -7228,8 +8569,19 @@ struct public_drv_mb {
 #define DRV_MSG_CODE_NIG_DRAIN			0x30000000
 #define DRV_MSG_CODE_VF_DISABLED_DONE		0xc0000000
 #define DRV_MSG_CODE_CFG_VF_MSIX		0xc0010000
+#define DRV_MSG_CODE_NVM_GET_FILE_ATT		0x00030000
+#define DRV_MSG_CODE_NVM_READ_NVRAM		0x00050000
 #define DRV_MSG_CODE_MCP_RESET			0x00090000
 #define DRV_MSG_CODE_SET_VERSION		0x000f0000
+#define DRV_MSG_CODE_MCP_HALT                   0x00100000
+
+#define DRV_MSG_CODE_GET_STATS                  0x00130000
+#define DRV_MSG_CODE_STATS_TYPE_LAN             1
+#define DRV_MSG_CODE_STATS_TYPE_FCOE            2
+#define DRV_MSG_CODE_STATS_TYPE_ISCSI           3
+#define DRV_MSG_CODE_STATS_TYPE_RDMA            4
+
+#define DRV_MSG_CODE_MASK_PARITIES              0x001a0000
 
 #define DRV_MSG_CODE_BIST_TEST			0x001e0000
 #define DRV_MSG_CODE_SET_LED_MODE		0x00200000
@@ -7240,6 +8592,9 @@ struct public_drv_mb {
 #define DRV_MB_PARAM_UNLOAD_WOL_MCP		0x00000001
 #define DRV_MB_PARAM_DCBX_NOTIFY_MASK		0x000000FF
 #define DRV_MB_PARAM_DCBX_NOTIFY_SHIFT		3
+
+#define DRV_MB_PARAM_NVM_LEN_SHIFT		24
+
 #define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_SHIFT	0
 #define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_MASK	0x000000FF
 #define DRV_MB_PARAM_CFG_VF_MSIX_SB_NUM_SHIFT	8
@@ -7277,6 +8632,8 @@ struct public_drv_mb {
 #define FW_MSG_CODE_DRV_UNLOAD_FUNCTION		0x20130000
 #define FW_MSG_CODE_DRV_UNLOAD_DONE		0x21100000
 #define FW_MSG_CODE_DRV_CFG_VF_MSIX_DONE	0xb0010000
+
+#define FW_MSG_CODE_NVM_OK			0x00010000
 #define FW_MSG_CODE_OK				0x00160000
 
 #define FW_MSG_SEQ_NUMBER_MASK			0x0000ffff
@@ -7307,10 +8664,10 @@ enum MFW_DRV_MSG_TYPE {
 	MFW_DRV_MSG_RESERVED4,
 	MFW_DRV_MSG_BW_UPDATE,
 	MFW_DRV_MSG_BW_UPDATE5,
-	MFW_DRV_MSG_BW_UPDATE6,
-	MFW_DRV_MSG_BW_UPDATE7,
-	MFW_DRV_MSG_BW_UPDATE8,
-	MFW_DRV_MSG_BW_UPDATE9,
+	MFW_DRV_MSG_GET_LAN_STATS,
+	MFW_DRV_MSG_GET_FCOE_STATS,
+	MFW_DRV_MSG_GET_ISCSI_STATS,
+	MFW_DRV_MSG_GET_RDMA_STATS,
 	MFW_DRV_MSG_BW_UPDATE10,
 	MFW_DRV_MSG_TRANSCEIVER_STATE_CHANGE,
 	MFW_DRV_MSG_BW_UPDATE11,
@@ -7513,4 +8870,101 @@ struct nvm_cfg1 {
 	struct nvm_cfg1_port port[MCP_GLOB_PORT_MAX];
 	struct nvm_cfg1_func func[MCP_GLOB_FUNC_MAX];
 };
+
+enum spad_sections {
+	SPAD_SECTION_TRACE,
+	SPAD_SECTION_NVM_CFG,
+	SPAD_SECTION_PUBLIC,
+	SPAD_SECTION_PRIVATE,
+	SPAD_SECTION_MAX
+};
+
+#define MCP_TRACE_SIZE          2048	/* 2kb */
+
+/* This section is located at a fixed location in the beginning of the
+ * scratchpad, to ensure that the MCP trace is not run over during MFW upgrade.
+ * All the rest of data has a floating location which differs from version to
+ * version, and is pointed by the mcp_meta_data below.
+ * Moreover, the spad_layout section is part of the MFW firmware, and is loaded
+ * with it from nvram in order to clear this portion.
+ */
+struct static_init {
+	u32 num_sections;
+	offsize_t sections[SPAD_SECTION_MAX];
+#define SECTION(_sec_) (*((offsize_t *)(STRUCT_OFFSET(sections[_sec_]))))
+
+	struct mcp_trace trace;
+#define MCP_TRACE_P ((struct mcp_trace *)(STRUCT_OFFSET(trace)))
+	u8 trace_buffer[MCP_TRACE_SIZE];
+#define MCP_TRACE_BUF ((u8 *)(STRUCT_OFFSET(trace_buffer)))
+	/* running_mfw has the same definition as in nvm_map.h.
+	 * This bit indicate both the running dir, and the running bundle.
+	 * It is set once when the LIM is loaded.
+	 */
+	u32 running_mfw;
+#define RUNNING_MFW (*((u32 *)(STRUCT_OFFSET(running_mfw))))
+	u32 build_time;
+#define MFW_BUILD_TIME (*((u32 *)(STRUCT_OFFSET(build_time))))
+	u32 reset_type;
+#define RESET_TYPE (*((u32 *)(STRUCT_OFFSET(reset_type))))
+	u32 mfw_secure_mode;
+#define MFW_SECURE_MODE (*((u32 *)(STRUCT_OFFSET(mfw_secure_mode))))
+	u16 pme_status_pf_bitmap;
+#define PME_STATUS_PF_BITMAP (*((u16 *)(STRUCT_OFFSET(pme_status_pf_bitmap))))
+	u16 pme_enable_pf_bitmap;
+#define PME_ENABLE_PF_BITMAP (*((u16 *)(STRUCT_OFFSET(pme_enable_pf_bitmap))))
+	u32 mim_nvm_addr;
+	u32 mim_start_addr;
+	u32 ah_pcie_link_params;
+#define AH_PCIE_LINK_PARAMS_LINK_SPEED_MASK     (0x000000ff)
+#define AH_PCIE_LINK_PARAMS_LINK_SPEED_SHIFT    (0)
+#define AH_PCIE_LINK_PARAMS_LINK_WIDTH_MASK     (0x0000ff00)
+#define AH_PCIE_LINK_PARAMS_LINK_WIDTH_SHIFT    (8)
+#define AH_PCIE_LINK_PARAMS_ASPM_MODE_MASK      (0x00ff0000)
+#define AH_PCIE_LINK_PARAMS_ASPM_MODE_SHIFT     (16)
+#define AH_PCIE_LINK_PARAMS_ASPM_CAP_MASK       (0xff000000)
+#define AH_PCIE_LINK_PARAMS_ASPM_CAP_SHIFT      (24)
+#define AH_PCIE_LINK_PARAMS (*((u32 *)(STRUCT_OFFSET(ah_pcie_link_params))))
+
+	u32 rsrv_persist[5];	/* Persist reserved for MFW upgrades */
+};
+
+enum nvm_image_type {
+	NVM_TYPE_TIM1 = 0x01,
+	NVM_TYPE_TIM2 = 0x02,
+	NVM_TYPE_MIM1 = 0x03,
+	NVM_TYPE_MIM2 = 0x04,
+	NVM_TYPE_MBA = 0x05,
+	NVM_TYPE_MODULES_PN = 0x06,
+	NVM_TYPE_VPD = 0x07,
+	NVM_TYPE_MFW_TRACE1 = 0x08,
+	NVM_TYPE_MFW_TRACE2 = 0x09,
+	NVM_TYPE_NVM_CFG1 = 0x0a,
+	NVM_TYPE_L2B = 0x0b,
+	NVM_TYPE_DIR1 = 0x0c,
+	NVM_TYPE_EAGLE_FW1 = 0x0d,
+	NVM_TYPE_FALCON_FW1 = 0x0e,
+	NVM_TYPE_PCIE_FW1 = 0x0f,
+	NVM_TYPE_HW_SET = 0x10,
+	NVM_TYPE_LIM = 0x11,
+	NVM_TYPE_AVS_FW1 = 0x12,
+	NVM_TYPE_DIR2 = 0x13,
+	NVM_TYPE_CCM = 0x14,
+	NVM_TYPE_EAGLE_FW2 = 0x15,
+	NVM_TYPE_FALCON_FW2 = 0x16,
+	NVM_TYPE_PCIE_FW2 = 0x17,
+	NVM_TYPE_AVS_FW2 = 0x18,
+	NVM_TYPE_INIT_HW = 0x19,
+	NVM_TYPE_DEFAULT_CFG = 0x1a,
+	NVM_TYPE_MDUMP = 0x1b,
+	NVM_TYPE_META = 0x1c,
+	NVM_TYPE_ISCSI_CFG = 0x1d,
+	NVM_TYPE_FCOE_CFG = 0x1f,
+	NVM_TYPE_ETH_PHY_FW1 = 0x20,
+	NVM_TYPE_ETH_PHY_FW2 = 0x21,
+	NVM_TYPE_MAX,
+};
+
+#define DIR_ID_1    (0)
+
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hw.c b/drivers/net/ethernet/qlogic/qed/qed_hw.c
index e17885321faf..6e4fae9b1430 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hw.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_hw.c
@@ -44,8 +44,7 @@ struct qed_ptt_pool {
 
 int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn)
 {
-	struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool),
-					      GFP_KERNEL);
+	struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL);
 	int i;
 
 	if (!p_pool)
@@ -113,16 +112,14 @@ struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn)
 	return NULL;
 }
 
-void qed_ptt_release(struct qed_hwfn *p_hwfn,
-		     struct qed_ptt *p_ptt)
+void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	spin_lock_bh(&p_hwfn->p_ptt_pool->lock);
 	list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list);
 	spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
 }
 
-u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn,
-			struct qed_ptt *p_ptt)
+u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	/* The HW is using DWORDS and we need to translate it to Bytes */
 	return le32_to_cpu(p_ptt->pxp.offset) << 2;
@@ -141,8 +138,7 @@ u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt)
 }
 
 void qed_ptt_set_win(struct qed_hwfn *p_hwfn,
-		     struct qed_ptt *p_ptt,
-		     u32 new_hw_addr)
+		     struct qed_ptt *p_ptt, u32 new_hw_addr)
 {
 	u32 prev_hw_addr;
 
@@ -166,8 +162,7 @@ void qed_ptt_set_win(struct qed_hwfn *p_hwfn,
 }
 
 static u32 qed_set_ptt(struct qed_hwfn *p_hwfn,
-		       struct qed_ptt *p_ptt,
-		       u32 hw_addr)
+		       struct qed_ptt *p_ptt, u32 hw_addr)
 {
 	u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt);
 	u32 offset;
@@ -224,10 +219,7 @@ u32 qed_rd(struct qed_hwfn *p_hwfn,
 
 static void qed_memcpy_hw(struct qed_hwfn *p_hwfn,
 			  struct qed_ptt *p_ptt,
-			  void *addr,
-			  u32 hw_addr,
-			  size_t n,
-			  bool to_device)
+			  void *addr, u32 hw_addr, size_t n, bool to_device)
 {
 	u32 dw_count, *host_addr, hw_offset;
 	size_t quota, done = 0;
@@ -259,8 +251,7 @@ static void qed_memcpy_hw(struct qed_hwfn *p_hwfn,
 }
 
 void qed_memcpy_from(struct qed_hwfn *p_hwfn,
-		     struct qed_ptt *p_ptt,
-		     void *dest, u32 hw_addr, size_t n)
+		     struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n)
 {
 	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
 		   "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n",
@@ -270,8 +261,7 @@ void qed_memcpy_from(struct qed_hwfn *p_hwfn,
 }
 
 void qed_memcpy_to(struct qed_hwfn *p_hwfn,
-		   struct qed_ptt *p_ptt,
-		   u32 hw_addr, void *src, size_t n)
+		   struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n)
 {
 	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
 		   "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n",
@@ -280,9 +270,7 @@ void qed_memcpy_to(struct qed_hwfn *p_hwfn,
 	qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true);
 }
 
-void qed_fid_pretend(struct qed_hwfn *p_hwfn,
-		     struct qed_ptt *p_ptt,
-		     u16 fid)
+void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid)
 {
 	u16 control = 0;
 
@@ -309,8 +297,7 @@ void qed_fid_pretend(struct qed_hwfn *p_hwfn,
 }
 
 void qed_port_pretend(struct qed_hwfn *p_hwfn,
-		      struct qed_ptt *p_ptt,
-		      u8 port_id)
+		      struct qed_ptt *p_ptt, u8 port_id)
 {
 	u16 control = 0;
 
@@ -326,8 +313,7 @@ void qed_port_pretend(struct qed_hwfn *p_hwfn,
 	       *(u32 *)&p_ptt->pxp.pretend);
 }
 
-void qed_port_unpretend(struct qed_hwfn *p_hwfn,
-			struct qed_ptt *p_ptt)
+void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	u16 control = 0;
 
@@ -429,28 +415,27 @@ u32 qed_dmae_idx_to_go_cmd(u8 idx)
 	return DMAE_REG_GO_C0 + (idx << 2);
 }
 
-static int
-qed_dmae_post_command(struct qed_hwfn *p_hwfn,
-		      struct qed_ptt *p_ptt)
+static int qed_dmae_post_command(struct qed_hwfn *p_hwfn,
+				 struct qed_ptt *p_ptt)
 {
-	struct dmae_cmd *command = p_hwfn->dmae_info.p_dmae_cmd;
+	struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd;
 	u8 idx_cmd = p_hwfn->dmae_info.channel, i;
 	int qed_status = 0;
 
 	/* verify address is not NULL */
-	if ((((command->dst_addr_lo == 0) && (command->dst_addr_hi == 0)) ||
-	     ((command->src_addr_lo == 0) && (command->src_addr_hi == 0)))) {
+	if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) ||
+	     ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) {
 		DP_NOTICE(p_hwfn,
 			  "source or destination address 0 idx_cmd=%d\n"
 			  "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
-			   idx_cmd,
-			   le32_to_cpu(command->opcode),
-			   le16_to_cpu(command->opcode_b),
-			   le16_to_cpu(command->length_dw),
-			   le32_to_cpu(command->src_addr_hi),
-			   le32_to_cpu(command->src_addr_lo),
-			   le32_to_cpu(command->dst_addr_hi),
-			   le32_to_cpu(command->dst_addr_lo));
+			  idx_cmd,
+			  le32_to_cpu(p_command->opcode),
+			  le16_to_cpu(p_command->opcode_b),
+			  le16_to_cpu(p_command->length_dw),
+			  le32_to_cpu(p_command->src_addr_hi),
+			  le32_to_cpu(p_command->src_addr_lo),
+			  le32_to_cpu(p_command->dst_addr_hi),
+			  le32_to_cpu(p_command->dst_addr_lo));
 
 		return -EINVAL;
 	}
@@ -459,13 +444,13 @@ qed_dmae_post_command(struct qed_hwfn *p_hwfn,
 		   NETIF_MSG_HW,
 		   "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
 		   idx_cmd,
-		   le32_to_cpu(command->opcode),
-		   le16_to_cpu(command->opcode_b),
-		   le16_to_cpu(command->length_dw),
-		   le32_to_cpu(command->src_addr_hi),
-		   le32_to_cpu(command->src_addr_lo),
-		   le32_to_cpu(command->dst_addr_hi),
-		   le32_to_cpu(command->dst_addr_lo));
+		   le32_to_cpu(p_command->opcode),
+		   le16_to_cpu(p_command->opcode_b),
+		   le16_to_cpu(p_command->length_dw),
+		   le32_to_cpu(p_command->src_addr_hi),
+		   le32_to_cpu(p_command->src_addr_lo),
+		   le32_to_cpu(p_command->dst_addr_hi),
+		   le32_to_cpu(p_command->dst_addr_lo));
 
 	/* Copy the command to DMAE - need to do it before every call
 	 * for source/dest address no reset.
@@ -475,7 +460,7 @@ qed_dmae_post_command(struct qed_hwfn *p_hwfn,
 	 */
 	for (i = 0; i < DMAE_CMD_SIZE; i++) {
 		u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ?
-			   *(((u32 *)command) + i) : 0;
+			   *(((u32 *)p_command) + i) : 0;
 
 		qed_wr(p_hwfn, p_ptt,
 		       DMAE_REG_CMD_MEM +
@@ -483,9 +468,7 @@ qed_dmae_post_command(struct qed_hwfn *p_hwfn,
 		       (i * sizeof(u32)), data);
 	}
 
-	qed_wr(p_hwfn, p_ptt,
-	       qed_dmae_idx_to_go_cmd(idx_cmd),
-	       DMAE_GO_VALUE);
+	qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE);
 
 	return qed_status;
 }
@@ -498,31 +481,23 @@ int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn)
 	u32 **p_comp = &p_hwfn->dmae_info.p_completion_word;
 
 	*p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
-				     sizeof(u32),
-				     p_addr,
-				     GFP_KERNEL);
-	if (!*p_comp) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `p_completion_word'\n");
+				     sizeof(u32), p_addr, GFP_KERNEL);
+	if (!*p_comp)
 		goto err;
-	}
 
 	p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr;
 	*p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 				    sizeof(struct dmae_cmd),
 				    p_addr, GFP_KERNEL);
-	if (!*p_cmd) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `struct dmae_cmd'\n");
+	if (!*p_cmd)
 		goto err;
-	}
 
 	p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr;
 	*p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 				     sizeof(u32) * DMAE_MAX_RW_SIZE,
 				     p_addr, GFP_KERNEL);
-	if (!*p_buff) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `intermediate_buffer'\n");
+	if (!*p_buff)
 		goto err;
-	}
 
 	p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id;
 
@@ -543,8 +518,7 @@ void qed_dmae_info_free(struct qed_hwfn *p_hwfn)
 		p_phys = p_hwfn->dmae_info.completion_word_phys_addr;
 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 				  sizeof(u32),
-				  p_hwfn->dmae_info.p_completion_word,
-				  p_phys);
+				  p_hwfn->dmae_info.p_completion_word, p_phys);
 		p_hwfn->dmae_info.p_completion_word = NULL;
 	}
 
@@ -552,8 +526,7 @@ void qed_dmae_info_free(struct qed_hwfn *p_hwfn)
 		p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr;
 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 				  sizeof(struct dmae_cmd),
-				  p_hwfn->dmae_info.p_dmae_cmd,
-				  p_phys);
+				  p_hwfn->dmae_info.p_dmae_cmd, p_phys);
 		p_hwfn->dmae_info.p_dmae_cmd = NULL;
 	}
 
@@ -571,9 +544,7 @@ void qed_dmae_info_free(struct qed_hwfn *p_hwfn)
 
 static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn)
 {
-	u32 wait_cnt = 0;
-	u32 wait_cnt_limit = 10000;
-
+	u32 wait_cnt_limit = 10000, wait_cnt = 0;
 	int qed_status = 0;
 
 	barrier();
@@ -606,7 +577,7 @@ static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn,
 					  u64 dst_addr,
 					  u8 src_type,
 					  u8 dst_type,
-					  u32 length)
+					  u32 length_dw)
 {
 	dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
 	struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
@@ -624,7 +595,7 @@ static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn,
 		cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys));
 		memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0],
 		       (void *)(uintptr_t)src_addr,
-		       length * sizeof(u32));
+		       length_dw * sizeof(u32));
 		break;
 	default:
 		return -EINVAL;
@@ -645,7 +616,7 @@ static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn,
 		return -EINVAL;
 	}
 
-	cmd->length_dw = cpu_to_le16((u16)length);
+	cmd->length_dw = cpu_to_le16((u16)length_dw);
 
 	qed_dmae_post_command(p_hwfn, p_ptt);
 
@@ -654,16 +625,14 @@ static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn,
 	if (qed_status) {
 		DP_NOTICE(p_hwfn,
 			  "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n",
-			  src_addr,
-			  dst_addr,
-			  length);
+			  src_addr, dst_addr, length_dw);
 		return qed_status;
 	}
 
 	if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT)
 		memcpy((void *)(uintptr_t)(dst_addr),
 		       &p_hwfn->dmae_info.p_intermediate_buffer[0],
-		       length * sizeof(u32));
+		       length_dw * sizeof(u32));
 
 	return 0;
 }
@@ -730,10 +699,7 @@ static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn,
 		if (qed_status) {
 			DP_NOTICE(p_hwfn,
 				  "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n",
-				  qed_status,
-				  src_addr,
-				  dst_addr,
-				  length_cur);
+				  qed_status, src_addr, dst_addr, length_cur);
 			break;
 		}
 	}
@@ -743,10 +709,7 @@ static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn,
 
 int qed_dmae_host2grc(struct qed_hwfn *p_hwfn,
 		      struct qed_ptt *p_ptt,
-		      u64 source_addr,
-		      u32 grc_addr,
-		      u32 size_in_dwords,
-		      u32 flags)
+		  u64 source_addr, u32 grc_addr, u32 size_in_dwords, u32 flags)
 {
 	u32 grc_addr_in_dw = grc_addr / sizeof(u32);
 	struct qed_dmae_params params;
@@ -768,9 +731,10 @@ int qed_dmae_host2grc(struct qed_hwfn *p_hwfn,
 	return rc;
 }
 
-int
-qed_dmae_grc2host(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u32 grc_addr,
-		  dma_addr_t dest_addr, u32 size_in_dwords, u32 flags)
+int qed_dmae_grc2host(struct qed_hwfn *p_hwfn,
+		      struct qed_ptt *p_ptt,
+		      u32 grc_addr,
+		      dma_addr_t dest_addr, u32 size_in_dwords, u32 flags)
 {
 	u32 grc_addr_in_dw = grc_addr / sizeof(u32);
 	struct qed_dmae_params params;
@@ -791,12 +755,11 @@ qed_dmae_grc2host(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u32 grc_addr,
 	return rc;
 }
 
-int
-qed_dmae_host2host(struct qed_hwfn *p_hwfn,
-		   struct qed_ptt *p_ptt,
-		   dma_addr_t source_addr,
-		   dma_addr_t dest_addr,
-		   u32 size_in_dwords, struct qed_dmae_params *p_params)
+int qed_dmae_host2host(struct qed_hwfn *p_hwfn,
+		       struct qed_ptt *p_ptt,
+		       dma_addr_t source_addr,
+		       dma_addr_t dest_addr,
+		       u32 size_in_dwords, struct qed_dmae_params *p_params)
 {
 	int rc;
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_init_ops.c b/drivers/net/ethernet/qlogic/qed/qed_init_ops.c
index 9866a20d2128..d567ba94c8d1 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_init_ops.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_init_ops.c
@@ -59,17 +59,14 @@ void qed_init_clear_rt_data(struct qed_hwfn *p_hwfn)
 		p_hwfn->rt_data.b_valid[i] = false;
 }
 
-void qed_init_store_rt_reg(struct qed_hwfn *p_hwfn,
-			   u32 rt_offset,
-			   u32 val)
+void qed_init_store_rt_reg(struct qed_hwfn *p_hwfn, u32 rt_offset, u32 val)
 {
 	p_hwfn->rt_data.init_val[rt_offset] = val;
 	p_hwfn->rt_data.b_valid[rt_offset] = true;
 }
 
 void qed_init_store_rt_agg(struct qed_hwfn *p_hwfn,
-			   u32 rt_offset, u32 *p_val,
-			   size_t size)
+			   u32 rt_offset, u32 *p_val, size_t size)
 {
 	size_t i;
 
@@ -81,10 +78,7 @@ void qed_init_store_rt_agg(struct qed_hwfn *p_hwfn,
 
 static int qed_init_rt(struct qed_hwfn	*p_hwfn,
 		       struct qed_ptt *p_ptt,
-		       u32 addr,
-		       u16 rt_offset,
-		       u16 size,
-		       bool b_must_dmae)
+		       u32 addr, u16 rt_offset, u16 size, bool b_must_dmae)
 {
 	u32 *p_init_val = &p_hwfn->rt_data.init_val[rt_offset];
 	bool *p_valid = &p_hwfn->rt_data.b_valid[rt_offset];
@@ -102,8 +96,7 @@ static int qed_init_rt(struct qed_hwfn	*p_hwfn,
 		 * simply write the data instead of using dmae.
 		 */
 		if (!b_must_dmae) {
-			qed_wr(p_hwfn, p_ptt, addr + (i << 2),
-			       p_init_val[i]);
+			qed_wr(p_hwfn, p_ptt, addr + (i << 2), p_init_val[i]);
 			continue;
 		}
 
@@ -115,7 +108,7 @@ static int qed_init_rt(struct qed_hwfn	*p_hwfn,
 		rc = qed_dmae_host2grc(p_hwfn, p_ptt,
 				       (uintptr_t)(p_init_val + i),
 				       addr + (i << 2), segment, 0);
-		if (rc != 0)
+		if (rc)
 			return rc;
 
 		/* Jump over the entire segment, including invalid entry */
@@ -182,9 +175,7 @@ static int qed_init_array_dmae(struct qed_hwfn *p_hwfn,
 
 static int qed_init_fill_dmae(struct qed_hwfn *p_hwfn,
 			      struct qed_ptt *p_ptt,
-			      u32 addr,
-			      u32 fill,
-			      u32 fill_count)
+			      u32 addr, u32 fill, u32 fill_count)
 {
 	static u32 zero_buffer[DMAE_MAX_RW_SIZE];
 
@@ -199,15 +190,12 @@ static int qed_init_fill_dmae(struct qed_hwfn *p_hwfn,
 
 	return qed_dmae_host2grc(p_hwfn, p_ptt,
 				 (uintptr_t)(&zero_buffer[0]),
-				 addr, fill_count,
-				 QED_DMAE_FLAG_RW_REPL_SRC);
+				 addr, fill_count, QED_DMAE_FLAG_RW_REPL_SRC);
 }
 
 static void qed_init_fill(struct qed_hwfn *p_hwfn,
 			  struct qed_ptt *p_ptt,
-			  u32 addr,
-			  u32 fill,
-			  u32 fill_count)
+			  u32 addr, u32 fill, u32 fill_count)
 {
 	u32 i;
 
@@ -218,12 +206,12 @@ static void qed_init_fill(struct qed_hwfn *p_hwfn,
 static int qed_init_cmd_array(struct qed_hwfn *p_hwfn,
 			      struct qed_ptt *p_ptt,
 			      struct init_write_op *cmd,
-			      bool b_must_dmae,
-			      bool b_can_dmae)
+			      bool b_must_dmae, bool b_can_dmae)
 {
+	u32 dmae_array_offset = le32_to_cpu(cmd->args.array_offset);
 	u32 data = le32_to_cpu(cmd->data);
 	u32 addr = GET_FIELD(data, INIT_WRITE_OP_ADDRESS) << 2;
-	u32 dmae_array_offset = le32_to_cpu(cmd->args.array_offset);
+
 	u32 offset, output_len, input_len, max_size;
 	struct qed_dev *cdev = p_hwfn->cdev;
 	union init_array_hdr *hdr;
@@ -233,8 +221,7 @@ static int qed_init_cmd_array(struct qed_hwfn *p_hwfn,
 
 	array_data = cdev->fw_data->arr_data;
 
-	hdr = (union init_array_hdr *)(array_data +
-				       dmae_array_offset);
+	hdr = (union init_array_hdr *)(array_data + dmae_array_offset);
 	data = le32_to_cpu(hdr->raw.data);
 	switch (GET_FIELD(data, INIT_ARRAY_RAW_HDR_TYPE)) {
 	case INIT_ARR_ZIPPED:
@@ -290,13 +277,12 @@ static int qed_init_cmd_array(struct qed_hwfn *p_hwfn,
 /* init_ops write command */
 static int qed_init_cmd_wr(struct qed_hwfn *p_hwfn,
 			   struct qed_ptt *p_ptt,
-			   struct init_write_op *cmd,
-			   bool b_can_dmae)
+			   struct init_write_op *p_cmd, bool b_can_dmae)
 {
-	u32 data = le32_to_cpu(cmd->data);
-	u32 addr = GET_FIELD(data, INIT_WRITE_OP_ADDRESS) << 2;
+	u32 data = le32_to_cpu(p_cmd->data);
 	bool b_must_dmae = GET_FIELD(data, INIT_WRITE_OP_WIDE_BUS);
-	union init_write_args *arg = &cmd->args;
+	u32 addr = GET_FIELD(data, INIT_WRITE_OP_ADDRESS) << 2;
+	union init_write_args *arg = &p_cmd->args;
 	int rc = 0;
 
 	/* Sanitize */
@@ -309,20 +295,18 @@ static int qed_init_cmd_wr(struct qed_hwfn *p_hwfn,
 
 	switch (GET_FIELD(data, INIT_WRITE_OP_SOURCE)) {
 	case INIT_SRC_INLINE:
-		qed_wr(p_hwfn, p_ptt, addr,
-		       le32_to_cpu(arg->inline_val));
+		data = le32_to_cpu(p_cmd->args.inline_val);
+		qed_wr(p_hwfn, p_ptt, addr, data);
 		break;
 	case INIT_SRC_ZEROS:
-		if (b_must_dmae ||
-		    (b_can_dmae && (le32_to_cpu(arg->zeros_count) >= 64)))
-			rc = qed_init_fill_dmae(p_hwfn, p_ptt, addr, 0,
-						le32_to_cpu(arg->zeros_count));
+		data = le32_to_cpu(p_cmd->args.zeros_count);
+		if (b_must_dmae || (b_can_dmae && (data >= 64)))
+			rc = qed_init_fill_dmae(p_hwfn, p_ptt, addr, 0, data);
 		else
-			qed_init_fill(p_hwfn, p_ptt, addr, 0,
-				      le32_to_cpu(arg->zeros_count));
+			qed_init_fill(p_hwfn, p_ptt, addr, 0, data);
 		break;
 	case INIT_SRC_ARRAY:
-		rc = qed_init_cmd_array(p_hwfn, p_ptt, cmd,
+		rc = qed_init_cmd_array(p_hwfn, p_ptt, p_cmd,
 					b_must_dmae, b_can_dmae);
 		break;
 	case INIT_SRC_RUNTIME:
@@ -353,8 +337,7 @@ static inline bool comp_or(u32 val, u32 expected_val)
 
 /* init_ops read/poll commands */
 static void qed_init_cmd_rd(struct qed_hwfn *p_hwfn,
-			    struct qed_ptt *p_ptt,
-			    struct init_read_op *cmd)
+			    struct qed_ptt *p_ptt, struct init_read_op *cmd)
 {
 	bool (*comp_check)(u32 val, u32 expected_val);
 	u32 delay = QED_INIT_POLL_PERIOD_US, val;
@@ -412,35 +395,33 @@ static void qed_init_cmd_cb(struct qed_hwfn *p_hwfn,
 }
 
 static u8 qed_init_cmd_mode_match(struct qed_hwfn *p_hwfn,
-				  u16 *offset,
-				  int modes)
+				  u16 *p_offset, int modes)
 {
 	struct qed_dev *cdev = p_hwfn->cdev;
 	const u8 *modes_tree_buf;
 	u8 arg1, arg2, tree_val;
 
 	modes_tree_buf = cdev->fw_data->modes_tree_buf;
-	tree_val = modes_tree_buf[(*offset)++];
+	tree_val = modes_tree_buf[(*p_offset)++];
 	switch (tree_val) {
 	case INIT_MODE_OP_NOT:
-		return qed_init_cmd_mode_match(p_hwfn, offset, modes) ^ 1;
+		return qed_init_cmd_mode_match(p_hwfn, p_offset, modes) ^ 1;
 	case INIT_MODE_OP_OR:
-		arg1	= qed_init_cmd_mode_match(p_hwfn, offset, modes);
-		arg2	= qed_init_cmd_mode_match(p_hwfn, offset, modes);
+		arg1 = qed_init_cmd_mode_match(p_hwfn, p_offset, modes);
+		arg2 = qed_init_cmd_mode_match(p_hwfn, p_offset, modes);
 		return arg1 | arg2;
 	case INIT_MODE_OP_AND:
-		arg1	= qed_init_cmd_mode_match(p_hwfn, offset, modes);
-		arg2	= qed_init_cmd_mode_match(p_hwfn, offset, modes);
+		arg1 = qed_init_cmd_mode_match(p_hwfn, p_offset, modes);
+		arg2 = qed_init_cmd_mode_match(p_hwfn, p_offset, modes);
 		return arg1 & arg2;
 	default:
 		tree_val -= MAX_INIT_MODE_OPS;
-		return (modes & (1 << tree_val)) ? 1 : 0;
+		return (modes & BIT(tree_val)) ? 1 : 0;
 	}
 }
 
 static u32 qed_init_cmd_mode(struct qed_hwfn *p_hwfn,
-			     struct init_if_mode_op *p_cmd,
-			     int modes)
+			     struct init_if_mode_op *p_cmd, int modes)
 {
 	u16 offset = le16_to_cpu(p_cmd->modes_buf_offset);
 
@@ -453,8 +434,7 @@ static u32 qed_init_cmd_mode(struct qed_hwfn *p_hwfn,
 
 static u32 qed_init_cmd_phase(struct qed_hwfn *p_hwfn,
 			      struct init_if_phase_op *p_cmd,
-			      u32 phase,
-			      u32 phase_id)
+			      u32 phase, u32 phase_id)
 {
 	u32 data = le32_to_cpu(p_cmd->phase_data);
 	u32 op_data = le32_to_cpu(p_cmd->op_data);
@@ -468,10 +448,7 @@ static u32 qed_init_cmd_phase(struct qed_hwfn *p_hwfn,
 }
 
 int qed_init_run(struct qed_hwfn *p_hwfn,
-		 struct qed_ptt *p_ptt,
-		 int phase,
-		 int phase_id,
-		 int modes)
+		 struct qed_ptt *p_ptt, int phase, int phase_id, int modes)
 {
 	struct qed_dev *cdev = p_hwfn->cdev;
 	u32 cmd_num, num_init_ops;
@@ -483,10 +460,8 @@ int qed_init_run(struct qed_hwfn *p_hwfn,
 	init_ops = cdev->fw_data->init_ops;
 
 	p_hwfn->unzip_buf = kzalloc(MAX_ZIPPED_SIZE * 4, GFP_ATOMIC);
-	if (!p_hwfn->unzip_buf) {
-		DP_NOTICE(p_hwfn, "Failed to allocate unzip buffer\n");
+	if (!p_hwfn->unzip_buf)
 		return -ENOMEM;
-	}
 
 	for (cmd_num = 0; cmd_num < num_init_ops; cmd_num++) {
 		union init_op *cmd = &init_ops[cmd_num];
@@ -557,7 +532,7 @@ int qed_init_fw_data(struct qed_dev *cdev, const u8 *data)
 	/* First Dword contains metadata and should be skipped */
 	buf_hdr = (struct bin_buffer_hdr *)(data + sizeof(u32));
 
-	offset = buf_hdr[BIN_BUF_FW_VER_INFO].offset;
+	offset = buf_hdr[BIN_BUF_INIT_FW_VER_INFO].offset;
 	fw->fw_ver_info = (struct fw_ver_info *)(data + offset);
 
 	offset = buf_hdr[BIN_BUF_INIT_CMD].offset;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.c b/drivers/net/ethernet/qlogic/qed/qed_int.c
index 8fa50fa23c8d..2adedc6fb6cf 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_int.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_int.c
@@ -1775,10 +1775,9 @@ struct qed_sb_attn_info {
 };
 
 static inline u16 qed_attn_update_idx(struct qed_hwfn *p_hwfn,
-				      struct qed_sb_attn_info   *p_sb_desc)
+				      struct qed_sb_attn_info *p_sb_desc)
 {
-	u16     rc = 0;
-	u16     index;
+	u16 rc = 0, index;
 
 	/* Make certain HW write took affect */
 	mmiowb();
@@ -1802,15 +1801,13 @@ static inline u16 qed_attn_update_idx(struct qed_hwfn *p_hwfn,
  *  @param asserted_bits newly asserted bits
  *  @return int
  */
-static int qed_int_assertion(struct qed_hwfn *p_hwfn,
-			     u16 asserted_bits)
+static int qed_int_assertion(struct qed_hwfn *p_hwfn, u16 asserted_bits)
 {
 	struct qed_sb_attn_info *sb_attn_sw = p_hwfn->p_sb_attn;
 	u32 igu_mask;
 
 	/* Mask the source of the attention in the IGU */
-	igu_mask = qed_rd(p_hwfn, p_hwfn->p_dpc_ptt,
-			  IGU_REG_ATTENTION_ENABLE);
+	igu_mask = qed_rd(p_hwfn, p_hwfn->p_dpc_ptt, IGU_REG_ATTENTION_ENABLE);
 	DP_VERBOSE(p_hwfn, NETIF_MSG_INTR, "IGU mask: 0x%08x --> 0x%08x\n",
 		   igu_mask, igu_mask & ~(asserted_bits & ATTN_BITS_MASKABLE));
 	igu_mask &= ~(asserted_bits & ATTN_BITS_MASKABLE);
@@ -2041,7 +2038,7 @@ static int qed_int_deassertion(struct qed_hwfn  *p_hwfn,
 			struct aeu_invert_reg_bit *p_bit = &p_aeu->bits[j];
 
 			if ((p_bit->flags & ATTENTION_PARITY) &&
-			    !!(parities & (1 << bit_idx)))
+			    !!(parities & BIT(bit_idx)))
 				qed_int_deassertion_parity(p_hwfn, p_bit,
 							   bit_idx);
 
@@ -2114,8 +2111,7 @@ static int qed_int_deassertion(struct qed_hwfn  *p_hwfn,
 				    ~((u32)deasserted_bits));
 
 	/* Unmask deasserted attentions in IGU */
-	aeu_mask = qed_rd(p_hwfn, p_hwfn->p_dpc_ptt,
-			  IGU_REG_ATTENTION_ENABLE);
+	aeu_mask = qed_rd(p_hwfn, p_hwfn->p_dpc_ptt, IGU_REG_ATTENTION_ENABLE);
 	aeu_mask |= (deasserted_bits & ATTN_BITS_MASKABLE);
 	qed_wr(p_hwfn, p_hwfn->p_dpc_ptt, IGU_REG_ATTENTION_ENABLE, aeu_mask);
 
@@ -2160,8 +2156,7 @@ static int qed_int_attentions(struct qed_hwfn *p_hwfn)
 			index, attn_bits, attn_acks, asserted_bits,
 			deasserted_bits, p_sb_attn_sw->known_attn);
 	} else if (asserted_bits == 0x100) {
-		DP_INFO(p_hwfn,
-			"MFW indication via attention\n");
+		DP_INFO(p_hwfn, "MFW indication via attention\n");
 	} else {
 		DP_VERBOSE(p_hwfn, NETIF_MSG_INTR,
 			   "MFW indication [deassertion]\n");
@@ -2173,18 +2168,14 @@ static int qed_int_attentions(struct qed_hwfn *p_hwfn)
 			return rc;
 	}
 
-	if (deasserted_bits) {
+	if (deasserted_bits)
 		rc = qed_int_deassertion(p_hwfn, deasserted_bits);
-		if (rc)
-			return rc;
-	}
 
 	return rc;
 }
 
 static void qed_sb_ack_attn(struct qed_hwfn *p_hwfn,
-			    void __iomem *igu_addr,
-			    u32 ack_cons)
+			    void __iomem *igu_addr, u32 ack_cons)
 {
 	struct igu_prod_cons_update igu_ack = { 0 };
 
@@ -2242,9 +2233,8 @@ void qed_int_sp_dpc(unsigned long hwfn_cookie)
 
 	/* Gather Interrupts/Attentions information */
 	if (!sb_info->sb_virt) {
-		DP_ERR(
-			p_hwfn->cdev,
-			"Interrupt Status block is NULL - cannot check for new interrupts!\n");
+		DP_ERR(p_hwfn->cdev,
+		       "Interrupt Status block is NULL - cannot check for new interrupts!\n");
 	} else {
 		u32 tmp_index = sb_info->sb_ack;
 
@@ -2255,9 +2245,8 @@ void qed_int_sp_dpc(unsigned long hwfn_cookie)
 	}
 
 	if (!sb_attn || !sb_attn->sb_attn) {
-		DP_ERR(
-			p_hwfn->cdev,
-			"Attentions Status block is NULL - cannot check for new attentions!\n");
+		DP_ERR(p_hwfn->cdev,
+		       "Attentions Status block is NULL - cannot check for new attentions!\n");
 	} else {
 		u16 tmp_index = sb_attn->index;
 
@@ -2313,8 +2302,7 @@ static void qed_int_sb_attn_free(struct qed_hwfn *p_hwfn)
 	if (p_sb->sb_attn)
 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 				  SB_ATTN_ALIGNED_SIZE(p_hwfn),
-				  p_sb->sb_attn,
-				  p_sb->sb_phys);
+				  p_sb->sb_attn, p_sb->sb_phys);
 	kfree(p_sb);
 }
 
@@ -2337,8 +2325,7 @@ static void qed_int_sb_attn_setup(struct qed_hwfn *p_hwfn,
 
 static void qed_int_sb_attn_init(struct qed_hwfn *p_hwfn,
 				 struct qed_ptt *p_ptt,
-				 void *sb_virt_addr,
-				 dma_addr_t sb_phy_addr)
+				 void *sb_virt_addr, dma_addr_t sb_phy_addr)
 {
 	struct qed_sb_attn_info *sb_info = p_hwfn->p_sb_attn;
 	int i, j, k;
@@ -2378,15 +2365,13 @@ static int qed_int_sb_attn_alloc(struct qed_hwfn *p_hwfn,
 {
 	struct qed_dev *cdev = p_hwfn->cdev;
 	struct qed_sb_attn_info *p_sb;
-	void *p_virt;
 	dma_addr_t p_phys = 0;
+	void *p_virt;
 
 	/* SB struct */
 	p_sb = kmalloc(sizeof(*p_sb), GFP_KERNEL);
-	if (!p_sb) {
-		DP_NOTICE(cdev, "Failed to allocate `struct qed_sb_attn_info'\n");
+	if (!p_sb)
 		return -ENOMEM;
-	}
 
 	/* SB ring  */
 	p_virt = dma_alloc_coherent(&cdev->pdev->dev,
@@ -2394,7 +2379,6 @@ static int qed_int_sb_attn_alloc(struct qed_hwfn *p_hwfn,
 				    &p_phys, GFP_KERNEL);
 
 	if (!p_virt) {
-		DP_NOTICE(cdev, "Failed to allocate status block (attentions)\n");
 		kfree(p_sb);
 		return -ENOMEM;
 	}
@@ -2412,9 +2396,7 @@ static int qed_int_sb_attn_alloc(struct qed_hwfn *p_hwfn,
 
 void qed_init_cau_sb_entry(struct qed_hwfn *p_hwfn,
 			   struct cau_sb_entry *p_sb_entry,
-			   u8 pf_id,
-			   u16 vf_number,
-			   u8 vf_valid)
+			   u8 pf_id, u16 vf_number, u8 vf_valid)
 {
 	struct qed_dev *cdev = p_hwfn->cdev;
 	u32 cau_state;
@@ -2428,12 +2410,6 @@ void qed_init_cau_sb_entry(struct qed_hwfn *p_hwfn,
 	SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_SB_TIMESET0, 0x7F);
 	SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_SB_TIMESET1, 0x7F);
 
-	/* setting the time resultion to a fixed value ( = 1) */
-	SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_TIMER_RES0,
-		  QED_CAU_DEF_RX_TIMER_RES);
-	SET_FIELD(p_sb_entry->params, CAU_SB_ENTRY_TIMER_RES1,
-		  QED_CAU_DEF_TX_TIMER_RES);
-
 	cau_state = CAU_HC_DISABLE_STATE;
 
 	if (cdev->int_coalescing_mode == QED_COAL_MODE_ENABLE) {
@@ -2468,9 +2444,7 @@ void qed_init_cau_sb_entry(struct qed_hwfn *p_hwfn,
 void qed_int_cau_conf_sb(struct qed_hwfn *p_hwfn,
 			 struct qed_ptt *p_ptt,
 			 dma_addr_t sb_phys,
-			 u16 igu_sb_id,
-			 u16 vf_number,
-			 u8 vf_valid)
+			 u16 igu_sb_id, u16 vf_number, u8 vf_valid)
 {
 	struct cau_sb_entry sb_entry;
 
@@ -2514,8 +2488,7 @@ void qed_int_cau_conf_sb(struct qed_hwfn *p_hwfn,
 			timer_res = 2;
 		timeset = (u8)(p_hwfn->cdev->rx_coalesce_usecs >> timer_res);
 		qed_int_cau_conf_pi(p_hwfn, p_ptt, igu_sb_id, RX_PI,
-				    QED_COAL_RX_STATE_MACHINE,
-				    timeset);
+				    QED_COAL_RX_STATE_MACHINE, timeset);
 
 		if (p_hwfn->cdev->tx_coalesce_usecs <= 0x7F)
 			timer_res = 0;
@@ -2541,8 +2514,7 @@ void qed_int_cau_conf_pi(struct qed_hwfn *p_hwfn,
 			 u8 timeset)
 {
 	struct cau_pi_entry pi_entry;
-	u32 sb_offset;
-	u32 pi_offset;
+	u32 sb_offset, pi_offset;
 
 	if (IS_VF(p_hwfn->cdev))
 		return;
@@ -2569,8 +2541,7 @@ void qed_int_cau_conf_pi(struct qed_hwfn *p_hwfn,
 }
 
 void qed_int_sb_setup(struct qed_hwfn *p_hwfn,
-		      struct qed_ptt *p_ptt,
-		      struct qed_sb_info *sb_info)
+		      struct qed_ptt *p_ptt, struct qed_sb_info *sb_info)
 {
 	/* zero status block and ack counter */
 	sb_info->sb_ack = 0;
@@ -2590,8 +2561,7 @@ void qed_int_sb_setup(struct qed_hwfn *p_hwfn,
  *
  * @return u16
  */
-static u16 qed_get_igu_sb_id(struct qed_hwfn *p_hwfn,
-			     u16 sb_id)
+static u16 qed_get_igu_sb_id(struct qed_hwfn *p_hwfn, u16 sb_id)
 {
 	u16 igu_sb_id;
 
@@ -2603,8 +2573,12 @@ static u16 qed_get_igu_sb_id(struct qed_hwfn *p_hwfn,
 	else
 		igu_sb_id = qed_vf_get_igu_sb_id(p_hwfn, sb_id);
 
-	DP_VERBOSE(p_hwfn, NETIF_MSG_INTR, "SB [%s] index is 0x%04x\n",
-		   (sb_id == QED_SP_SB_ID) ? "DSB" : "non-DSB", igu_sb_id);
+	if (sb_id == QED_SP_SB_ID)
+		DP_VERBOSE(p_hwfn, NETIF_MSG_INTR,
+			   "Slowpath SB index in IGU is 0x%04x\n", igu_sb_id);
+	else
+		DP_VERBOSE(p_hwfn, NETIF_MSG_INTR,
+			   "SB [%04x] <--> IGU SB [%04x]\n", sb_id, igu_sb_id);
 
 	return igu_sb_id;
 }
@@ -2612,9 +2586,7 @@ static u16 qed_get_igu_sb_id(struct qed_hwfn *p_hwfn,
 int qed_int_sb_init(struct qed_hwfn *p_hwfn,
 		    struct qed_ptt *p_ptt,
 		    struct qed_sb_info *sb_info,
-		    void *sb_virt_addr,
-		    dma_addr_t sb_phy_addr,
-		    u16 sb_id)
+		    void *sb_virt_addr, dma_addr_t sb_phy_addr, u16 sb_id)
 {
 	sb_info->sb_virt = sb_virt_addr;
 	sb_info->sb_phys = sb_phy_addr;
@@ -2650,8 +2622,7 @@ int qed_int_sb_init(struct qed_hwfn *p_hwfn,
 }
 
 int qed_int_sb_release(struct qed_hwfn *p_hwfn,
-		       struct qed_sb_info *sb_info,
-		       u16 sb_id)
+		       struct qed_sb_info *sb_info, u16 sb_id)
 {
 	if (sb_id == QED_SP_SB_ID) {
 		DP_ERR(p_hwfn, "Do Not free sp sb using this function");
@@ -2685,8 +2656,7 @@ static void qed_int_sp_sb_free(struct qed_hwfn *p_hwfn)
 	kfree(p_sb);
 }
 
-static int qed_int_sp_sb_alloc(struct qed_hwfn *p_hwfn,
-			       struct qed_ptt *p_ptt)
+static int qed_int_sp_sb_alloc(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	struct qed_sb_sp_info *p_sb;
 	dma_addr_t p_phys = 0;
@@ -2694,17 +2664,14 @@ static int qed_int_sp_sb_alloc(struct qed_hwfn *p_hwfn,
 
 	/* SB struct */
 	p_sb = kmalloc(sizeof(*p_sb), GFP_KERNEL);
-	if (!p_sb) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `struct qed_sb_info'\n");
+	if (!p_sb)
 		return -ENOMEM;
-	}
 
 	/* SB ring  */
 	p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 				    SB_ALIGNED_SIZE(p_hwfn),
 				    &p_phys, GFP_KERNEL);
 	if (!p_virt) {
-		DP_NOTICE(p_hwfn, "Failed to allocate status block\n");
 		kfree(p_sb);
 		return -ENOMEM;
 	}
@@ -2721,9 +2688,7 @@ static int qed_int_sp_sb_alloc(struct qed_hwfn *p_hwfn,
 
 int qed_int_register_cb(struct qed_hwfn *p_hwfn,
 			qed_int_comp_cb_t comp_cb,
-			void *cookie,
-			u8 *sb_idx,
-			__le16 **p_fw_cons)
+			void *cookie, u8 *sb_idx, __le16 **p_fw_cons)
 {
 	struct qed_sb_sp_info *p_sp_sb = p_hwfn->p_sp_sb;
 	int rc = -ENOMEM;
@@ -2764,8 +2729,7 @@ u16 qed_int_get_sp_sb_id(struct qed_hwfn *p_hwfn)
 }
 
 void qed_int_igu_enable_int(struct qed_hwfn *p_hwfn,
-			    struct qed_ptt *p_ptt,
-			    enum qed_int_mode int_mode)
+			    struct qed_ptt *p_ptt, enum qed_int_mode int_mode)
 {
 	u32 igu_pf_conf = IGU_PF_CONF_FUNC_EN | IGU_PF_CONF_ATTN_BIT_EN;
 
@@ -2809,7 +2773,7 @@ int qed_int_igu_enable(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	qed_wr(p_hwfn, p_ptt, MISC_REG_AEU_MASK_ATTN_IGU, 0xff);
 	if ((int_mode != QED_INT_MODE_INTA) || IS_LEAD_HWFN(p_hwfn)) {
 		rc = qed_slowpath_irq_req(p_hwfn);
-		if (rc != 0) {
+		if (rc) {
 			DP_NOTICE(p_hwfn, "Slowpath IRQ request failed\n");
 			return -EINVAL;
 		}
@@ -2822,8 +2786,7 @@ int qed_int_igu_enable(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	return rc;
 }
 
-void qed_int_igu_disable_int(struct qed_hwfn *p_hwfn,
-			     struct qed_ptt *p_ptt)
+void qed_int_igu_disable_int(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	p_hwfn->b_int_enabled = 0;
 
@@ -2950,13 +2913,11 @@ void qed_int_igu_init_pure_rt(struct qed_hwfn *p_hwfn,
 					p_hwfn->hw_info.opaque_fid, b_set);
 }
 
-static u32 qed_int_igu_read_cam_block(struct qed_hwfn	*p_hwfn,
-				      struct qed_ptt	*p_ptt,
-				      u16		sb_id)
+static u32 qed_int_igu_read_cam_block(struct qed_hwfn *p_hwfn,
+				      struct qed_ptt *p_ptt, u16 sb_id)
 {
 	u32 val = qed_rd(p_hwfn, p_ptt,
-			 IGU_REG_MAPPING_MEMORY +
-			 sizeof(u32) * sb_id);
+			 IGU_REG_MAPPING_MEMORY + sizeof(u32) * sb_id);
 	struct qed_igu_block *p_block;
 
 	p_block = &p_hwfn->hw_info.p_igu_info->igu_map.igu_blocks[sb_id];
@@ -2983,8 +2944,7 @@ out:
 	return val;
 }
 
-int qed_int_igu_read_cam(struct qed_hwfn *p_hwfn,
-			 struct qed_ptt *p_ptt)
+int qed_int_igu_read_cam(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	struct qed_igu_info *p_igu_info;
 	u32 val, min_vf = 0, max_vf = 0;
@@ -2993,7 +2953,6 @@ int qed_int_igu_read_cam(struct qed_hwfn *p_hwfn,
 	u16 prev_sb_id = 0xFF;
 
 	p_hwfn->hw_info.p_igu_info = kzalloc(sizeof(*p_igu_info), GFP_KERNEL);
-
 	if (!p_hwfn->hw_info.p_igu_info)
 		return -ENOMEM;
 
@@ -3104,22 +3063,19 @@ int qed_int_igu_read_cam(struct qed_hwfn *p_hwfn,
  */
 void qed_int_igu_init_rt(struct qed_hwfn *p_hwfn)
 {
-	u32 igu_pf_conf = 0;
-
-	igu_pf_conf |= IGU_PF_CONF_FUNC_EN;
+	u32 igu_pf_conf = IGU_PF_CONF_FUNC_EN;
 
 	STORE_RT_REG(p_hwfn, IGU_REG_PF_CONFIGURATION_RT_OFFSET, igu_pf_conf);
 }
 
 u64 qed_int_igu_read_sisr_reg(struct qed_hwfn *p_hwfn)
 {
-	u64 intr_status = 0;
-	u32 intr_status_lo = 0;
-	u32 intr_status_hi = 0;
 	u32 lsb_igu_cmd_addr = IGU_REG_SISR_MDPC_WMASK_LSB_UPPER -
 			       IGU_CMD_INT_ACK_BASE;
 	u32 msb_igu_cmd_addr = IGU_REG_SISR_MDPC_WMASK_MSB_UPPER -
 			       IGU_CMD_INT_ACK_BASE;
+	u32 intr_status_hi = 0, intr_status_lo = 0;
+	u64 intr_status = 0;
 
 	intr_status_lo = REG_RD(p_hwfn,
 				GTT_BAR0_MAP_REG_IGU_CMD +
@@ -3153,26 +3109,20 @@ static void qed_int_sp_dpc_free(struct qed_hwfn *p_hwfn)
 	kfree(p_hwfn->sp_dpc);
 }
 
-int qed_int_alloc(struct qed_hwfn *p_hwfn,
-		  struct qed_ptt *p_ptt)
+int qed_int_alloc(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	int rc = 0;
 
 	rc = qed_int_sp_dpc_alloc(p_hwfn);
-	if (rc) {
-		DP_ERR(p_hwfn->cdev, "Failed to allocate sp dpc mem\n");
+	if (rc)
 		return rc;
-	}
+
 	rc = qed_int_sp_sb_alloc(p_hwfn, p_ptt);
-	if (rc) {
-		DP_ERR(p_hwfn->cdev, "Failed to allocate sp sb mem\n");
+	if (rc)
 		return rc;
-	}
+
 	rc = qed_int_sb_attn_alloc(p_hwfn, p_ptt);
-	if (rc) {
-		DP_ERR(p_hwfn->cdev, "Failed to allocate sb attn mem\n");
-		return rc;
-	}
+
 	return rc;
 }
 
@@ -3183,8 +3133,7 @@ void qed_int_free(struct qed_hwfn *p_hwfn)
 	qed_int_sp_dpc_free(p_hwfn);
 }
 
-void qed_int_setup(struct qed_hwfn *p_hwfn,
-		   struct qed_ptt *p_ptt)
+void qed_int_setup(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	qed_int_sb_setup(p_hwfn, p_ptt, &p_hwfn->p_sp_sb->sb_info);
 	qed_int_sb_attn_setup(p_hwfn, p_ptt);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.c b/drivers/net/ethernet/qlogic/qed/qed_l2.c
index 401e738543b5..ddd410a91e13 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_l2.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_l2.c
@@ -52,7 +52,7 @@ int qed_sp_eth_vport_start(struct qed_hwfn *p_hwfn,
 	u16 rx_mode = 0;
 
 	rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_vport_id);
-	if (rc != 0)
+	if (rc)
 		return rc;
 
 	memset(&init_data, 0, sizeof(init_data));
@@ -80,8 +80,7 @@ int qed_sp_eth_vport_start(struct qed_hwfn *p_hwfn,
 	p_ramrod->rx_mode.state = cpu_to_le16(rx_mode);
 
 	/* TPA related fields */
-	memset(&p_ramrod->tpa_param, 0,
-	       sizeof(struct eth_vport_tpa_param));
+	memset(&p_ramrod->tpa_param, 0, sizeof(struct eth_vport_tpa_param));
 
 	p_ramrod->tpa_param.max_buff_num = p_params->max_buffers_per_cqe;
 
@@ -102,6 +101,9 @@ int qed_sp_eth_vport_start(struct qed_hwfn *p_hwfn,
 
 	p_ramrod->tx_switching_en = p_params->tx_switching;
 
+	p_ramrod->ctl_frame_mac_check_en = !!p_params->check_mac;
+	p_ramrod->ctl_frame_ethtype_check_en = !!p_params->check_ethtype;
+
 	/* Software Function ID in hwfn (PFs are 0 - 15, VFs are 16 - 135) */
 	p_ramrod->sw_fid = qed_concrete_to_sw_fid(p_hwfn->cdev,
 						  p_params->concrete_fid);
@@ -109,8 +111,8 @@ int qed_sp_eth_vport_start(struct qed_hwfn *p_hwfn,
 	return qed_spq_post(p_hwfn, p_ent, NULL);
 }
 
-int qed_sp_vport_start(struct qed_hwfn *p_hwfn,
-		       struct qed_sp_vport_start_params *p_params)
+static int qed_sp_vport_start(struct qed_hwfn *p_hwfn,
+			      struct qed_sp_vport_start_params *p_params)
 {
 	if (IS_VF(p_hwfn->cdev)) {
 		return qed_vf_pf_vport_start(p_hwfn, p_params->vport_id,
@@ -306,14 +308,14 @@ qed_sp_update_mcast_bin(struct qed_hwfn *p_hwfn,
 	memset(&p_ramrod->approx_mcast.bins, 0,
 	       sizeof(p_ramrod->approx_mcast.bins));
 
-	if (p_params->update_approx_mcast_flg) {
-		p_ramrod->common.update_approx_mcast_flg = 1;
-		for (i = 0; i < ETH_MULTICAST_MAC_BINS_IN_REGS; i++) {
-			u32 *p_bins = (u32 *)p_params->bins;
-			__le32 val = cpu_to_le32(p_bins[i]);
+	if (!p_params->update_approx_mcast_flg)
+		return;
 
-			p_ramrod->approx_mcast.bins[i] = val;
-		}
+	p_ramrod->common.update_approx_mcast_flg = 1;
+	for (i = 0; i < ETH_MULTICAST_MAC_BINS_IN_REGS; i++) {
+		u32 *p_bins = (u32 *)p_params->bins;
+
+		p_ramrod->approx_mcast.bins[i] = cpu_to_le32(p_bins[i]);
 	}
 }
 
@@ -336,7 +338,7 @@ int qed_sp_vport_update(struct qed_hwfn *p_hwfn,
 	}
 
 	rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_vport_id);
-	if (rc != 0)
+	if (rc)
 		return rc;
 
 	memset(&init_data, 0, sizeof(init_data));
@@ -361,8 +363,8 @@ int qed_sp_vport_update(struct qed_hwfn *p_hwfn,
 	p_cmn->tx_active_flg = p_params->vport_active_tx_flg;
 	p_cmn->update_tx_active_flg = p_params->update_vport_active_tx_flg;
 	p_cmn->accept_any_vlan = p_params->accept_any_vlan;
-	p_cmn->update_accept_any_vlan_flg =
-			p_params->update_accept_any_vlan_flg;
+	val = p_params->update_accept_any_vlan_flg;
+	p_cmn->update_accept_any_vlan_flg = val;
 
 	p_cmn->inner_vlan_removal_en = p_params->inner_vlan_removal_flg;
 	val = p_params->update_inner_vlan_removal_flg;
@@ -411,7 +413,7 @@ int qed_sp_vport_stop(struct qed_hwfn *p_hwfn, u16 opaque_fid, u8 vport_id)
 		return qed_vf_pf_vport_stop(p_hwfn);
 
 	rc = qed_fw_vport(p_hwfn, vport_id, &abs_vport_id);
-	if (rc != 0)
+	if (rc)
 		return rc;
 
 	memset(&init_data, 0, sizeof(init_data));
@@ -476,7 +478,7 @@ static int qed_filter_accept_cmd(struct qed_dev *cdev,
 
 		rc = qed_sp_vport_update(p_hwfn, &vport_update_params,
 					 comp_mode, p_comp_data);
-		if (rc != 0) {
+		if (rc) {
 			DP_ERR(cdev, "Update rx_mode failed %d\n", rc);
 			return rc;
 		}
@@ -511,11 +513,12 @@ static int qed_sp_release_queue_cid(
 int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
 				u16 opaque_fid,
 				u32 cid,
-				struct qed_queue_start_common_params *params,
+				struct qed_queue_start_common_params *p_params,
 				u8 stats_id,
 				u16 bd_max_bytes,
 				dma_addr_t bd_chain_phys_addr,
-				dma_addr_t cqe_pbl_addr, u16 cqe_pbl_size)
+				dma_addr_t cqe_pbl_addr,
+				u16 cqe_pbl_size, bool b_use_zone_a_prod)
 {
 	struct rx_queue_start_ramrod_data *p_ramrod = NULL;
 	struct qed_spq_entry *p_ent = NULL;
@@ -526,23 +529,23 @@ int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
 	int rc = -EINVAL;
 
 	/* Store information for the stop */
-	p_rx_cid		= &p_hwfn->p_rx_cids[params->queue_id];
-	p_rx_cid->cid		= cid;
-	p_rx_cid->opaque_fid	= opaque_fid;
-	p_rx_cid->vport_id	= params->vport_id;
+	p_rx_cid = &p_hwfn->p_rx_cids[p_params->queue_id];
+	p_rx_cid->cid = cid;
+	p_rx_cid->opaque_fid = opaque_fid;
+	p_rx_cid->vport_id = p_params->vport_id;
 
-	rc = qed_fw_vport(p_hwfn, params->vport_id, &abs_vport_id);
-	if (rc != 0)
+	rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_vport_id);
+	if (rc)
 		return rc;
 
-	rc = qed_fw_l2_queue(p_hwfn, params->queue_id, &abs_rx_q_id);
-	if (rc != 0)
+	rc = qed_fw_l2_queue(p_hwfn, p_params->queue_id, &abs_rx_q_id);
+	if (rc)
 		return rc;
 
 	DP_VERBOSE(p_hwfn, QED_MSG_SP,
 		   "opaque_fid=0x%x, cid=0x%x, rx_qid=0x%x, vport_id=0x%x, sb_id=0x%x\n",
-		   opaque_fid, cid, params->queue_id, params->vport_id,
-		   params->sb);
+		   opaque_fid,
+		   cid, p_params->queue_id, p_params->vport_id, p_params->sb);
 
 	/* Get SPQ entry */
 	memset(&init_data, 0, sizeof(init_data));
@@ -558,24 +561,28 @@ int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
 
 	p_ramrod = &p_ent->ramrod.rx_queue_start;
 
-	p_ramrod->sb_id			= cpu_to_le16(params->sb);
-	p_ramrod->sb_index		= params->sb_idx;
-	p_ramrod->vport_id		= abs_vport_id;
-	p_ramrod->stats_counter_id	= stats_id;
-	p_ramrod->rx_queue_id		= cpu_to_le16(abs_rx_q_id);
-	p_ramrod->complete_cqe_flg	= 0;
-	p_ramrod->complete_event_flg	= 1;
+	p_ramrod->sb_id = cpu_to_le16(p_params->sb);
+	p_ramrod->sb_index = p_params->sb_idx;
+	p_ramrod->vport_id = abs_vport_id;
+	p_ramrod->stats_counter_id = stats_id;
+	p_ramrod->rx_queue_id = cpu_to_le16(abs_rx_q_id);
+	p_ramrod->complete_cqe_flg = 0;
+	p_ramrod->complete_event_flg = 1;
 
-	p_ramrod->bd_max_bytes	= cpu_to_le16(bd_max_bytes);
+	p_ramrod->bd_max_bytes = cpu_to_le16(bd_max_bytes);
 	DMA_REGPAIR_LE(p_ramrod->bd_base, bd_chain_phys_addr);
 
-	p_ramrod->num_of_pbl_pages	= cpu_to_le16(cqe_pbl_size);
+	p_ramrod->num_of_pbl_pages = cpu_to_le16(cqe_pbl_size);
 	DMA_REGPAIR_LE(p_ramrod->cqe_pbl_addr, cqe_pbl_addr);
 
-	p_ramrod->vf_rx_prod_index = params->vf_qid;
-	if (params->vf_qid)
+	if (p_params->vf_qid || b_use_zone_a_prod) {
+		p_ramrod->vf_rx_prod_index = p_params->vf_qid;
 		DP_VERBOSE(p_hwfn, QED_MSG_SP,
-			   "Queue is meant for VF rxq[%04x]\n", params->vf_qid);
+			   "Queue%s is meant for VF rxq[%02x]\n",
+			   b_use_zone_a_prod ? " [legacy]" : "",
+			   p_params->vf_qid);
+		p_ramrod->vf_rx_prod_use_zone_a = b_use_zone_a_prod;
+	}
 
 	return qed_spq_post(p_hwfn, p_ent, NULL);
 }
@@ -583,7 +590,7 @@ int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
 static int
 qed_sp_eth_rx_queue_start(struct qed_hwfn *p_hwfn,
 			  u16 opaque_fid,
-			  struct qed_queue_start_common_params *params,
+			  struct qed_queue_start_common_params *p_params,
 			  u16 bd_max_bytes,
 			  dma_addr_t bd_chain_phys_addr,
 			  dma_addr_t cqe_pbl_addr,
@@ -597,20 +604,20 @@ qed_sp_eth_rx_queue_start(struct qed_hwfn *p_hwfn,
 
 	if (IS_VF(p_hwfn->cdev)) {
 		return qed_vf_pf_rxq_start(p_hwfn,
-					   params->queue_id,
-					   params->sb,
-					   params->sb_idx,
+					   p_params->queue_id,
+					   p_params->sb,
+					   (u8)p_params->sb_idx,
 					   bd_max_bytes,
 					   bd_chain_phys_addr,
 					   cqe_pbl_addr, cqe_pbl_size, pp_prod);
 	}
 
-	rc = qed_fw_l2_queue(p_hwfn, params->queue_id, &abs_l2_queue);
-	if (rc != 0)
+	rc = qed_fw_l2_queue(p_hwfn, p_params->queue_id, &abs_l2_queue);
+	if (rc)
 		return rc;
 
-	rc = qed_fw_vport(p_hwfn, params->vport_id, &abs_stats_id);
-	if (rc != 0)
+	rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_stats_id);
+	if (rc)
 		return rc;
 
 	*pp_prod = (u8 __iomem *)p_hwfn->regview +
@@ -622,9 +629,8 @@ qed_sp_eth_rx_queue_start(struct qed_hwfn *p_hwfn,
 			  (u32 *)(&init_prod_val));
 
 	/* Allocate a CID for the queue */
-	p_rx_cid = &p_hwfn->p_rx_cids[params->queue_id];
-	rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH,
-				 &p_rx_cid->cid);
+	p_rx_cid = &p_hwfn->p_rx_cids[p_params->queue_id];
+	rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH, &p_rx_cid->cid);
 	if (rc) {
 		DP_NOTICE(p_hwfn, "Failed to acquire cid\n");
 		return rc;
@@ -634,14 +640,13 @@ qed_sp_eth_rx_queue_start(struct qed_hwfn *p_hwfn,
 	rc = qed_sp_eth_rxq_start_ramrod(p_hwfn,
 					 opaque_fid,
 					 p_rx_cid->cid,
-					 params,
+					 p_params,
 					 abs_stats_id,
 					 bd_max_bytes,
 					 bd_chain_phys_addr,
-					 cqe_pbl_addr,
-					 cqe_pbl_size);
+					 cqe_pbl_addr, cqe_pbl_size, false);
 
-	if (rc != 0)
+	if (rc)
 		qed_sp_release_queue_cid(p_hwfn, p_rx_cid);
 
 	return rc;
@@ -788,21 +793,20 @@ int qed_sp_eth_txq_start_ramrod(struct qed_hwfn  *p_hwfn,
 	if (rc)
 		return rc;
 
-	p_ramrod		= &p_ent->ramrod.tx_queue_start;
-	p_ramrod->vport_id	= abs_vport_id;
+	p_ramrod = &p_ent->ramrod.tx_queue_start;
+	p_ramrod->vport_id = abs_vport_id;
 
-	p_ramrod->sb_id			= cpu_to_le16(p_params->sb);
-	p_ramrod->sb_index		= p_params->sb_idx;
-	p_ramrod->stats_counter_id	= stats_id;
+	p_ramrod->sb_id = cpu_to_le16(p_params->sb);
+	p_ramrod->sb_index = p_params->sb_idx;
+	p_ramrod->stats_counter_id = stats_id;
 
-	p_ramrod->queue_zone_id		= cpu_to_le16(abs_tx_q_id);
-	p_ramrod->pbl_size		= cpu_to_le16(pbl_size);
+	p_ramrod->queue_zone_id = cpu_to_le16(abs_tx_q_id);
+
+	p_ramrod->pbl_size = cpu_to_le16(pbl_size);
 	DMA_REGPAIR_LE(p_ramrod->pbl_base_addr, pbl_addr);
 
-	pq_id			= qed_get_qm_pq(p_hwfn,
-						PROTOCOLID_ETH,
-						p_pq_params);
-	p_ramrod->qm_pq_id	= cpu_to_le16(pq_id);
+	pq_id = qed_get_qm_pq(p_hwfn, PROTOCOLID_ETH, p_pq_params);
+	p_ramrod->qm_pq_id = cpu_to_le16(pq_id);
 
 	return qed_spq_post(p_hwfn, p_ent, NULL);
 }
@@ -836,8 +840,7 @@ qed_sp_eth_tx_queue_start(struct qed_hwfn *p_hwfn,
 	memset(&pq_params, 0, sizeof(pq_params));
 
 	/* Allocate a CID for the queue */
-	rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH,
-				 &p_tx_cid->cid);
+	rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_ETH, &p_tx_cid->cid);
 	if (rc) {
 		DP_NOTICE(p_hwfn, "Failed to acquire cid\n");
 		return rc;
@@ -896,8 +899,7 @@ int qed_sp_eth_tx_queue_stop(struct qed_hwfn *p_hwfn, u16 tx_queue_id)
 	return qed_sp_release_queue_cid(p_hwfn, p_tx_cid);
 }
 
-static enum eth_filter_action
-qed_filter_action(enum qed_filter_opcode opcode)
+static enum eth_filter_action qed_filter_action(enum qed_filter_opcode opcode)
 {
 	enum eth_filter_action action = MAX_ETH_FILTER_ACTION;
 
@@ -1033,19 +1035,19 @@ qed_filter_ucast_common(struct qed_hwfn *p_hwfn,
 		p_first_filter->vni = cpu_to_le32(p_filter_cmd->vni);
 
 	if (p_filter_cmd->opcode == QED_FILTER_MOVE) {
-		p_second_filter->type		= p_first_filter->type;
-		p_second_filter->mac_msb	= p_first_filter->mac_msb;
-		p_second_filter->mac_mid	= p_first_filter->mac_mid;
-		p_second_filter->mac_lsb	= p_first_filter->mac_lsb;
-		p_second_filter->vlan_id	= p_first_filter->vlan_id;
-		p_second_filter->vni		= p_first_filter->vni;
+		p_second_filter->type = p_first_filter->type;
+		p_second_filter->mac_msb = p_first_filter->mac_msb;
+		p_second_filter->mac_mid = p_first_filter->mac_mid;
+		p_second_filter->mac_lsb = p_first_filter->mac_lsb;
+		p_second_filter->vlan_id = p_first_filter->vlan_id;
+		p_second_filter->vni = p_first_filter->vni;
 
 		p_first_filter->action = ETH_FILTER_ACTION_REMOVE;
 
 		p_first_filter->vport_id = vport_to_remove_from;
 
-		p_second_filter->action		= ETH_FILTER_ACTION_ADD;
-		p_second_filter->vport_id	= vport_to_add_to;
+		p_second_filter->action = ETH_FILTER_ACTION_ADD;
+		p_second_filter->vport_id = vport_to_add_to;
 	} else if (p_filter_cmd->opcode == QED_FILTER_REPLACE) {
 		p_first_filter->vport_id = vport_to_add_to;
 		memcpy(p_second_filter, p_first_filter,
@@ -1086,7 +1088,7 @@ int qed_sp_eth_filter_ucast(struct qed_hwfn *p_hwfn,
 	rc = qed_filter_ucast_common(p_hwfn, opaque_fid, p_filter_cmd,
 				     &p_ramrod, &p_ent,
 				     comp_mode, p_comp_data);
-	if (rc != 0) {
+	if (rc) {
 		DP_ERR(p_hwfn, "Uni. filter command failed %d\n", rc);
 		return rc;
 	}
@@ -1094,10 +1096,8 @@ int qed_sp_eth_filter_ucast(struct qed_hwfn *p_hwfn,
 	p_header->assert_on_error = p_filter_cmd->assert_on_error;
 
 	rc = qed_spq_post(p_hwfn, p_ent, NULL);
-	if (rc != 0) {
-		DP_ERR(p_hwfn,
-		       "Unicast filter ADD command failed %d\n",
-		       rc);
+	if (rc) {
+		DP_ERR(p_hwfn, "Unicast filter ADD command failed %d\n", rc);
 		return rc;
 	}
 
@@ -1136,15 +1136,10 @@ int qed_sp_eth_filter_ucast(struct qed_hwfn *p_hwfn,
  * Return:
  ******************************************************************************/
 static u32 qed_calc_crc32c(u8 *crc32_packet,
-			   u32 crc32_length,
-			   u32 crc32_seed,
-			   u8 complement)
+			   u32 crc32_length, u32 crc32_seed, u8 complement)
 {
-	u32 byte = 0;
-	u32 bit = 0;
-	u8 msb = 0;
-	u8 current_byte = 0;
-	u32 crc32_result = crc32_seed;
+	u32 byte = 0, bit = 0, crc32_result = crc32_seed;
+	u8 msb = 0, current_byte = 0;
 
 	if ((!crc32_packet) ||
 	    (crc32_length == 0) ||
@@ -1164,9 +1159,7 @@ static u32 qed_calc_crc32c(u8 *crc32_packet,
 	return crc32_result;
 }
 
-static inline u32 qed_crc32c_le(u32 seed,
-				u8 *mac,
-				u32 len)
+static u32 qed_crc32c_le(u32 seed, u8 *mac, u32 len)
 {
 	u32 packet_buf[2] = { 0 };
 
@@ -1196,17 +1189,14 @@ qed_sp_eth_filter_mcast(struct qed_hwfn *p_hwfn,
 	u8 abs_vport_id = 0;
 	int rc, i;
 
-	if (p_filter_cmd->opcode == QED_FILTER_ADD) {
+	if (p_filter_cmd->opcode == QED_FILTER_ADD)
 		rc = qed_fw_vport(p_hwfn, p_filter_cmd->vport_to_add_to,
 				  &abs_vport_id);
-		if (rc)
-			return rc;
-	} else {
+	else
 		rc = qed_fw_vport(p_hwfn, p_filter_cmd->vport_to_remove_from,
 				  &abs_vport_id);
-		if (rc)
-			return rc;
-	}
+	if (rc)
+		return rc;
 
 	/* Get SPQ entry */
 	memset(&init_data, 0, sizeof(init_data));
@@ -1244,11 +1234,11 @@ qed_sp_eth_filter_mcast(struct qed_hwfn *p_hwfn,
 
 		/* Convert to correct endianity */
 		for (i = 0; i < ETH_MULTICAST_MAC_BINS_IN_REGS; i++) {
+			struct vport_update_ramrod_mcast *p_ramrod_bins;
 			u32 *p_bins = (u32 *)bins;
-			struct vport_update_ramrod_mcast *approx_mcast;
 
-			approx_mcast = &p_ramrod->approx_mcast;
-			approx_mcast->bins[i] = cpu_to_le32(p_bins[i]);
+			p_ramrod_bins = &p_ramrod->approx_mcast;
+			p_ramrod_bins->bins[i] = cpu_to_le32(p_bins[i]);
 		}
 	}
 
@@ -1286,8 +1276,7 @@ static int qed_filter_mcast_cmd(struct qed_dev *cdev,
 		rc = qed_sp_eth_filter_mcast(p_hwfn,
 					     opaque_fid,
 					     p_filter_cmd,
-					     comp_mode,
-					     p_comp_data);
+					     comp_mode, p_comp_data);
 	}
 	return rc;
 }
@@ -1314,9 +1303,8 @@ static int qed_filter_ucast_cmd(struct qed_dev *cdev,
 		rc = qed_sp_eth_filter_ucast(p_hwfn,
 					     opaque_fid,
 					     p_filter_cmd,
-					     comp_mode,
-					     p_comp_data);
-		if (rc != 0)
+					     comp_mode, p_comp_data);
+		if (rc)
 			break;
 	}
 
@@ -1590,8 +1578,7 @@ out:
 	}
 }
 
-void qed_get_vport_stats(struct qed_dev *cdev,
-			 struct qed_eth_stats *stats)
+void qed_get_vport_stats(struct qed_dev *cdev, struct qed_eth_stats *stats)
 {
 	u32 i;
 
@@ -1698,6 +1685,8 @@ static int qed_fill_eth_dev_info(struct qed_dev *cdev,
 		qed_vf_get_num_vlan_filters(&cdev->hwfns[0],
 					    &info->num_vlan_filters);
 		qed_vf_get_port_mac(&cdev->hwfns[0], info->port_mac);
+
+		info->is_legacy = !!cdev->hwfns[0].vf_iov_info->b_pre_fp_hsi;
 	}
 
 	qed_fill_dev_info(cdev, &info->common);
@@ -1766,8 +1755,7 @@ static int qed_start_vport(struct qed_dev *cdev,
 	return 0;
 }
 
-static int qed_stop_vport(struct qed_dev *cdev,
-			  u8 vport_id)
+static int qed_stop_vport(struct qed_dev *cdev, u8 vport_id)
 {
 	int rc, i;
 
@@ -1775,8 +1763,7 @@ static int qed_stop_vport(struct qed_dev *cdev,
 		struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
 
 		rc = qed_sp_vport_stop(p_hwfn,
-				       p_hwfn->hw_info.opaque_fid,
-				       vport_id);
+				       p_hwfn->hw_info.opaque_fid, vport_id);
 
 		if (rc) {
 			DP_ERR(cdev, "Failed to stop VPORT\n");
@@ -1801,10 +1788,8 @@ static int qed_update_vport(struct qed_dev *cdev,
 
 	/* Translate protocol params into sp params */
 	sp_params.vport_id = params->vport_id;
-	sp_params.update_vport_active_rx_flg =
-		params->update_vport_active_flg;
-	sp_params.update_vport_active_tx_flg =
-		params->update_vport_active_flg;
+	sp_params.update_vport_active_rx_flg = params->update_vport_active_flg;
+	sp_params.update_vport_active_tx_flg = params->update_vport_active_flg;
 	sp_params.vport_active_rx_flg = params->vport_active_flg;
 	sp_params.vport_active_tx_flg = params->vport_active_flg;
 	sp_params.update_tx_switching_flg = params->update_tx_switching_flg;
@@ -1817,8 +1802,7 @@ static int qed_update_vport(struct qed_dev *cdev,
 	 * We need to re-fix the rss values per engine for CMT.
 	 */
 	if (cdev->num_hwfns > 1 && params->update_rss_flg) {
-		struct qed_update_vport_rss_params *rss =
-			&params->rss_params;
+		struct qed_update_vport_rss_params *rss = &params->rss_params;
 		int k, max = 0;
 
 		/* Find largest entry, since it's possible RSS needs to
@@ -1861,8 +1845,8 @@ static int qed_update_vport(struct qed_dev *cdev,
 		       QED_RSS_IND_TABLE_SIZE * sizeof(u16));
 		memcpy(sp_rss_params.rss_key, params->rss_params.rss_key,
 		       QED_RSS_KEY_SIZE * sizeof(u32));
+		sp_params.rss_params = &sp_rss_params;
 	}
-	sp_params.rss_params = &sp_rss_params;
 
 	for_each_hwfn(cdev, i) {
 		struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
@@ -1893,8 +1877,8 @@ static int qed_start_rxq(struct qed_dev *cdev,
 			 u16 cqe_pbl_size,
 			 void __iomem **pp_prod)
 {
-	int rc, hwfn_index;
 	struct qed_hwfn *p_hwfn;
+	int rc, hwfn_index;
 
 	hwfn_index = params->rss_id % cdev->num_hwfns;
 	p_hwfn = &cdev->hwfns[hwfn_index];
@@ -1935,8 +1919,7 @@ static int qed_stop_rxq(struct qed_dev *cdev,
 
 	rc = qed_sp_eth_rx_queue_stop(p_hwfn,
 				      params->rx_queue_id / cdev->num_hwfns,
-				      params->eq_completion_only,
-				      false);
+				      params->eq_completion_only, false);
 	if (rc) {
 		DP_ERR(cdev, "Failed to stop RXQ#%d\n", params->rx_queue_id);
 		return rc;
@@ -2047,11 +2030,11 @@ static int qed_configure_filter_rx_mode(struct qed_dev *cdev,
 
 	memset(&accept_flags, 0, sizeof(accept_flags));
 
-	accept_flags.update_rx_mode_config	= 1;
-	accept_flags.update_tx_mode_config	= 1;
-	accept_flags.rx_accept_filter		= QED_ACCEPT_UCAST_MATCHED |
-						  QED_ACCEPT_MCAST_MATCHED |
-						  QED_ACCEPT_BCAST;
+	accept_flags.update_rx_mode_config = 1;
+	accept_flags.update_tx_mode_config = 1;
+	accept_flags.rx_accept_filter = QED_ACCEPT_UCAST_MATCHED |
+					QED_ACCEPT_MCAST_MATCHED |
+					QED_ACCEPT_BCAST;
 	accept_flags.tx_accept_filter = QED_ACCEPT_UCAST_MATCHED |
 					QED_ACCEPT_MCAST_MATCHED |
 					QED_ACCEPT_BCAST;
@@ -2072,9 +2055,8 @@ static int qed_configure_filter_ucast(struct qed_dev *cdev,
 	struct qed_filter_ucast ucast;
 
 	if (!params->vlan_valid && !params->mac_valid) {
-		DP_NOTICE(
-			cdev,
-			"Tried configuring a unicast filter, but both MAC and VLAN are not set\n");
+		DP_NOTICE(cdev,
+			  "Tried configuring a unicast filter, but both MAC and VLAN are not set\n");
 		return -EINVAL;
 	}
 
@@ -2135,8 +2117,7 @@ static int qed_configure_filter_mcast(struct qed_dev *cdev,
 	for (i = 0; i < mcast.num_mc_addrs; i++)
 		ether_addr_copy(mcast.mac[i], params->mac[i]);
 
-	return qed_filter_mcast_cmd(cdev, &mcast,
-				    QED_SPQ_MODE_CB, NULL);
+	return qed_filter_mcast_cmd(cdev, &mcast, QED_SPQ_MODE_CB, NULL);
 }
 
 static int qed_configure_filter(struct qed_dev *cdev,
@@ -2153,15 +2134,13 @@ static int qed_configure_filter(struct qed_dev *cdev,
 		accept_flags = params->filter.accept_flags;
 		return qed_configure_filter_rx_mode(cdev, accept_flags);
 	default:
-		DP_NOTICE(cdev, "Unknown filter type %d\n",
-			  (int)params->type);
+		DP_NOTICE(cdev, "Unknown filter type %d\n", (int)params->type);
 		return -EINVAL;
 	}
 }
 
 static int qed_fp_cqe_completion(struct qed_dev *dev,
-				 u8 rss_id,
-				 struct eth_slow_path_rx_cqe *cqe)
+				 u8 rss_id, struct eth_slow_path_rx_cqe *cqe)
 {
 	return qed_eth_cqe_completion(&dev->hwfns[rss_id % dev->num_hwfns],
 				      cqe);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.h b/drivers/net/ethernet/qlogic/qed/qed_l2.h
index 002114543451..e495d62fcc03 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_l2.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_l2.h
@@ -102,6 +102,8 @@ struct qed_sp_vport_start_params {
 	u16 opaque_fid;
 	u8 vport_id;
 	u16 mtu;
+	bool check_mac;
+	bool check_ethtype;
 };
 
 int qed_sp_eth_vport_start(struct qed_hwfn *p_hwfn,
@@ -213,6 +215,8 @@ qed_sp_eth_rx_queues_update(struct qed_hwfn *p_hwfn,
 			    enum spq_mode comp_mode,
 			    struct qed_spq_comp_cb *p_comp_data);
 
+void qed_get_vport_stats(struct qed_dev *cdev, struct qed_eth_stats *stats);
+
 int qed_sp_eth_vport_start(struct qed_hwfn *p_hwfn,
 			   struct qed_sp_vport_start_params *p_params);
 
@@ -223,7 +227,8 @@ int qed_sp_eth_rxq_start_ramrod(struct qed_hwfn *p_hwfn,
 				u8 stats_id,
 				u16 bd_max_bytes,
 				dma_addr_t bd_chain_phys_addr,
-				dma_addr_t cqe_pbl_addr, u16 cqe_pbl_size);
+				dma_addr_t cqe_pbl_addr,
+				u16 cqe_pbl_size, bool b_use_zone_a_prod);
 
 int qed_sp_eth_txq_start_ramrod(struct qed_hwfn  *p_hwfn,
 				u16  opaque_fid,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.c b/drivers/net/ethernet/qlogic/qed/qed_ll2.c
new file mode 100644
index 000000000000..a6db10717d5c
--- /dev/null
+++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.c
@@ -0,0 +1,1792 @@
+/* QLogic qed NIC Driver
+ *
+ * Copyright (c) 2015 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#include <linux/types.h>
+#include <asm/byteorder.h>
+#include <linux/dma-mapping.h>
+#include <linux/if_vlan.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/stddef.h>
+#include <linux/version.h>
+#include <linux/workqueue.h>
+#include <net/ipv6.h>
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/errno.h>
+#include <linux/etherdevice.h>
+#include <linux/io.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/qed/qed_ll2_if.h>
+#include "qed.h"
+#include "qed_cxt.h"
+#include "qed_dev_api.h"
+#include "qed_hsi.h"
+#include "qed_hw.h"
+#include "qed_int.h"
+#include "qed_ll2.h"
+#include "qed_mcp.h"
+#include "qed_reg_addr.h"
+#include "qed_sp.h"
+
+#define QED_LL2_RX_REGISTERED(ll2)	((ll2)->rx_queue.b_cb_registred)
+#define QED_LL2_TX_REGISTERED(ll2)	((ll2)->tx_queue.b_cb_registred)
+
+#define QED_LL2_TX_SIZE (256)
+#define QED_LL2_RX_SIZE (4096)
+
+struct qed_cb_ll2_info {
+	int rx_cnt;
+	u32 rx_size;
+	u8 handle;
+	bool frags_mapped;
+
+	/* Lock protecting LL2 buffer lists in sleepless context */
+	spinlock_t lock;
+	struct list_head list;
+
+	const struct qed_ll2_cb_ops *cbs;
+	void *cb_cookie;
+};
+
+struct qed_ll2_buffer {
+	struct list_head list;
+	void *data;
+	dma_addr_t phys_addr;
+};
+
+static void qed_ll2b_complete_tx_packet(struct qed_hwfn *p_hwfn,
+					u8 connection_handle,
+					void *cookie,
+					dma_addr_t first_frag_addr,
+					bool b_last_fragment,
+					bool b_last_packet)
+{
+	struct qed_dev *cdev = p_hwfn->cdev;
+	struct sk_buff *skb = cookie;
+
+	/* All we need to do is release the mapping */
+	dma_unmap_single(&p_hwfn->cdev->pdev->dev, first_frag_addr,
+			 skb_headlen(skb), DMA_TO_DEVICE);
+
+	if (cdev->ll2->cbs && cdev->ll2->cbs->tx_cb)
+		cdev->ll2->cbs->tx_cb(cdev->ll2->cb_cookie, skb,
+				      b_last_fragment);
+
+	if (cdev->ll2->frags_mapped)
+		/* Case where mapped frags were received, need to
+		 * free skb with nr_frags marked as 0
+		 */
+		skb_shinfo(skb)->nr_frags = 0;
+
+	dev_kfree_skb_any(skb);
+}
+
+static int qed_ll2_alloc_buffer(struct qed_dev *cdev,
+				u8 **data, dma_addr_t *phys_addr)
+{
+	*data = kmalloc(cdev->ll2->rx_size, GFP_ATOMIC);
+	if (!(*data)) {
+		DP_INFO(cdev, "Failed to allocate LL2 buffer data\n");
+		return -ENOMEM;
+	}
+
+	*phys_addr = dma_map_single(&cdev->pdev->dev,
+				    ((*data) + NET_SKB_PAD),
+				    cdev->ll2->rx_size, DMA_FROM_DEVICE);
+	if (dma_mapping_error(&cdev->pdev->dev, *phys_addr)) {
+		DP_INFO(cdev, "Failed to map LL2 buffer data\n");
+		kfree((*data));
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int qed_ll2_dealloc_buffer(struct qed_dev *cdev,
+				 struct qed_ll2_buffer *buffer)
+{
+	spin_lock_bh(&cdev->ll2->lock);
+
+	dma_unmap_single(&cdev->pdev->dev, buffer->phys_addr,
+			 cdev->ll2->rx_size, DMA_FROM_DEVICE);
+	kfree(buffer->data);
+	list_del(&buffer->list);
+
+	cdev->ll2->rx_cnt--;
+	if (!cdev->ll2->rx_cnt)
+		DP_INFO(cdev, "All LL2 entries were removed\n");
+
+	spin_unlock_bh(&cdev->ll2->lock);
+
+	return 0;
+}
+
+static void qed_ll2_kill_buffers(struct qed_dev *cdev)
+{
+	struct qed_ll2_buffer *buffer, *tmp_buffer;
+
+	list_for_each_entry_safe(buffer, tmp_buffer, &cdev->ll2->list, list)
+		qed_ll2_dealloc_buffer(cdev, buffer);
+}
+
+void qed_ll2b_complete_rx_packet(struct qed_hwfn *p_hwfn,
+				 u8 connection_handle,
+				 struct qed_ll2_rx_packet *p_pkt,
+				 struct core_rx_fast_path_cqe *p_cqe,
+				 bool b_last_packet)
+{
+	u16 packet_length = le16_to_cpu(p_cqe->packet_length);
+	struct qed_ll2_buffer *buffer = p_pkt->cookie;
+	struct qed_dev *cdev = p_hwfn->cdev;
+	u16 vlan = le16_to_cpu(p_cqe->vlan);
+	u32 opaque_data_0, opaque_data_1;
+	u8 pad = p_cqe->placement_offset;
+	dma_addr_t new_phys_addr;
+	struct sk_buff *skb;
+	bool reuse = false;
+	int rc = -EINVAL;
+	u8 *new_data;
+
+	opaque_data_0 = le32_to_cpu(p_cqe->opaque_data.data[0]);
+	opaque_data_1 = le32_to_cpu(p_cqe->opaque_data.data[1]);
+
+	DP_VERBOSE(p_hwfn,
+		   (NETIF_MSG_RX_STATUS | QED_MSG_STORAGE | NETIF_MSG_PKTDATA),
+		   "Got an LL2 Rx completion: [Buffer at phys 0x%llx, offset 0x%02x] Length 0x%04x Parse_flags 0x%04x vlan 0x%04x Opaque data [0x%08x:0x%08x]\n",
+		   (u64)p_pkt->rx_buf_addr, pad, packet_length,
+		   le16_to_cpu(p_cqe->parse_flags.flags), vlan,
+		   opaque_data_0, opaque_data_1);
+
+	if ((cdev->dp_module & NETIF_MSG_PKTDATA) && buffer->data) {
+		print_hex_dump(KERN_INFO, "",
+			       DUMP_PREFIX_OFFSET, 16, 1,
+			       buffer->data, packet_length, false);
+	}
+
+	/* Determine if data is valid */
+	if (packet_length < ETH_HLEN)
+		reuse = true;
+
+	/* Allocate a replacement for buffer; Reuse upon failure */
+	if (!reuse)
+		rc = qed_ll2_alloc_buffer(p_hwfn->cdev, &new_data,
+					  &new_phys_addr);
+
+	/* If need to reuse or there's no replacement buffer, repost this */
+	if (rc)
+		goto out_post;
+
+	skb = build_skb(buffer->data, 0);
+	if (!skb) {
+		rc = -ENOMEM;
+		goto out_post;
+	}
+
+	pad += NET_SKB_PAD;
+	skb_reserve(skb, pad);
+	skb_put(skb, packet_length);
+	skb_checksum_none_assert(skb);
+
+	/* Get parital ethernet information instead of eth_type_trans(),
+	 * Since we don't have an associated net_device.
+	 */
+	skb_reset_mac_header(skb);
+	skb->protocol = eth_hdr(skb)->h_proto;
+
+	/* Pass SKB onward */
+	if (cdev->ll2->cbs && cdev->ll2->cbs->rx_cb) {
+		if (vlan)
+			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
+		cdev->ll2->cbs->rx_cb(cdev->ll2->cb_cookie, skb,
+				      opaque_data_0, opaque_data_1);
+	}
+
+	/* Update Buffer information and update FW producer */
+	buffer->data = new_data;
+	buffer->phys_addr = new_phys_addr;
+
+out_post:
+	rc = qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev), cdev->ll2->handle,
+				    buffer->phys_addr, 0,  buffer, 1);
+
+	if (rc)
+		qed_ll2_dealloc_buffer(cdev, buffer);
+}
+
+static struct qed_ll2_info *__qed_ll2_handle_sanity(struct qed_hwfn *p_hwfn,
+						    u8 connection_handle,
+						    bool b_lock,
+						    bool b_only_active)
+{
+	struct qed_ll2_info *p_ll2_conn, *p_ret = NULL;
+
+	if (connection_handle >= QED_MAX_NUM_OF_LL2_CONNECTIONS)
+		return NULL;
+
+	if (!p_hwfn->p_ll2_info)
+		return NULL;
+
+	p_ll2_conn = &p_hwfn->p_ll2_info[connection_handle];
+
+	if (b_only_active) {
+		if (b_lock)
+			mutex_lock(&p_ll2_conn->mutex);
+		if (p_ll2_conn->b_active)
+			p_ret = p_ll2_conn;
+		if (b_lock)
+			mutex_unlock(&p_ll2_conn->mutex);
+	} else {
+		p_ret = p_ll2_conn;
+	}
+
+	return p_ret;
+}
+
+static struct qed_ll2_info *qed_ll2_handle_sanity(struct qed_hwfn *p_hwfn,
+						  u8 connection_handle)
+{
+	return __qed_ll2_handle_sanity(p_hwfn, connection_handle, false, true);
+}
+
+static struct qed_ll2_info *qed_ll2_handle_sanity_lock(struct qed_hwfn *p_hwfn,
+						       u8 connection_handle)
+{
+	return __qed_ll2_handle_sanity(p_hwfn, connection_handle, true, true);
+}
+
+static struct qed_ll2_info *qed_ll2_handle_sanity_inactive(struct qed_hwfn
+							   *p_hwfn,
+							   u8 connection_handle)
+{
+	return __qed_ll2_handle_sanity(p_hwfn, connection_handle, false, false);
+}
+
+static void qed_ll2_txq_flush(struct qed_hwfn *p_hwfn, u8 connection_handle)
+{
+	bool b_last_packet = false, b_last_frag = false;
+	struct qed_ll2_tx_packet *p_pkt = NULL;
+	struct qed_ll2_info *p_ll2_conn;
+	struct qed_ll2_tx_queue *p_tx;
+	dma_addr_t tx_frag;
+
+	p_ll2_conn = qed_ll2_handle_sanity_inactive(p_hwfn, connection_handle);
+	if (!p_ll2_conn)
+		return;
+
+	p_tx = &p_ll2_conn->tx_queue;
+
+	while (!list_empty(&p_tx->active_descq)) {
+		p_pkt = list_first_entry(&p_tx->active_descq,
+					 struct qed_ll2_tx_packet, list_entry);
+		if (!p_pkt)
+			break;
+
+		list_del(&p_pkt->list_entry);
+		b_last_packet = list_empty(&p_tx->active_descq);
+		list_add_tail(&p_pkt->list_entry, &p_tx->free_descq);
+		p_tx->cur_completing_packet = *p_pkt;
+		p_tx->cur_completing_bd_idx = 1;
+		b_last_frag = p_tx->cur_completing_bd_idx == p_pkt->bd_used;
+		tx_frag = p_pkt->bds_set[0].tx_frag;
+		if (p_ll2_conn->gsi_enable)
+			qed_ll2b_release_tx_gsi_packet(p_hwfn,
+						       p_ll2_conn->my_id,
+						       p_pkt->cookie,
+						       tx_frag,
+						       b_last_frag,
+						       b_last_packet);
+		else
+			qed_ll2b_complete_tx_packet(p_hwfn,
+						    p_ll2_conn->my_id,
+						    p_pkt->cookie,
+						    tx_frag,
+						    b_last_frag,
+						    b_last_packet);
+
+	}
+}
+
+static int qed_ll2_txq_completion(struct qed_hwfn *p_hwfn, void *p_cookie)
+{
+	struct qed_ll2_info *p_ll2_conn = p_cookie;
+	struct qed_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
+	u16 new_idx = 0, num_bds = 0, num_bds_in_packet = 0;
+	struct qed_ll2_tx_packet *p_pkt;
+	bool b_last_frag = false;
+	unsigned long flags;
+	dma_addr_t tx_frag;
+	int rc = -EINVAL;
+
+	spin_lock_irqsave(&p_tx->lock, flags);
+	if (p_tx->b_completing_packet) {
+		rc = -EBUSY;
+		goto out;
+	}
+
+	new_idx = le16_to_cpu(*p_tx->p_fw_cons);
+	num_bds = ((s16)new_idx - (s16)p_tx->bds_idx);
+	while (num_bds) {
+		if (list_empty(&p_tx->active_descq))
+			goto out;
+
+		p_pkt = list_first_entry(&p_tx->active_descq,
+					 struct qed_ll2_tx_packet, list_entry);
+		if (!p_pkt)
+			goto out;
+
+		p_tx->b_completing_packet = true;
+		p_tx->cur_completing_packet = *p_pkt;
+		num_bds_in_packet = p_pkt->bd_used;
+		list_del(&p_pkt->list_entry);
+
+		if (num_bds < num_bds_in_packet) {
+			DP_NOTICE(p_hwfn,
+				  "Rest of BDs does not cover whole packet\n");
+			goto out;
+		}
+
+		num_bds -= num_bds_in_packet;
+		p_tx->bds_idx += num_bds_in_packet;
+		while (num_bds_in_packet--)
+			qed_chain_consume(&p_tx->txq_chain);
+
+		p_tx->cur_completing_bd_idx = 1;
+		b_last_frag = p_tx->cur_completing_bd_idx == p_pkt->bd_used;
+		list_add_tail(&p_pkt->list_entry, &p_tx->free_descq);
+
+		spin_unlock_irqrestore(&p_tx->lock, flags);
+		tx_frag = p_pkt->bds_set[0].tx_frag;
+		if (p_ll2_conn->gsi_enable)
+			qed_ll2b_complete_tx_gsi_packet(p_hwfn,
+							p_ll2_conn->my_id,
+							p_pkt->cookie,
+							tx_frag,
+							b_last_frag, !num_bds);
+		else
+			qed_ll2b_complete_tx_packet(p_hwfn,
+						    p_ll2_conn->my_id,
+						    p_pkt->cookie,
+						    tx_frag,
+						    b_last_frag, !num_bds);
+		spin_lock_irqsave(&p_tx->lock, flags);
+	}
+
+	p_tx->b_completing_packet = false;
+	rc = 0;
+out:
+	spin_unlock_irqrestore(&p_tx->lock, flags);
+	return rc;
+}
+
+static int
+qed_ll2_rxq_completion_gsi(struct qed_hwfn *p_hwfn,
+			   struct qed_ll2_info *p_ll2_info,
+			   union core_rx_cqe_union *p_cqe,
+			   unsigned long lock_flags, bool b_last_cqe)
+{
+	struct qed_ll2_rx_queue *p_rx = &p_ll2_info->rx_queue;
+	struct qed_ll2_rx_packet *p_pkt = NULL;
+	u16 packet_length, parse_flags, vlan;
+	u32 src_mac_addrhi;
+	u16 src_mac_addrlo;
+
+	if (!list_empty(&p_rx->active_descq))
+		p_pkt = list_first_entry(&p_rx->active_descq,
+					 struct qed_ll2_rx_packet, list_entry);
+	if (!p_pkt) {
+		DP_NOTICE(p_hwfn,
+			  "GSI Rx completion but active_descq is empty\n");
+		return -EIO;
+	}
+
+	list_del(&p_pkt->list_entry);
+	parse_flags = le16_to_cpu(p_cqe->rx_cqe_gsi.parse_flags.flags);
+	packet_length = le16_to_cpu(p_cqe->rx_cqe_gsi.data_length);
+	vlan = le16_to_cpu(p_cqe->rx_cqe_gsi.vlan);
+	src_mac_addrhi = le32_to_cpu(p_cqe->rx_cqe_gsi.src_mac_addrhi);
+	src_mac_addrlo = le16_to_cpu(p_cqe->rx_cqe_gsi.src_mac_addrlo);
+	if (qed_chain_consume(&p_rx->rxq_chain) != p_pkt->rxq_bd)
+		DP_NOTICE(p_hwfn,
+			  "Mismatch between active_descq and the LL2 Rx chain\n");
+	list_add_tail(&p_pkt->list_entry, &p_rx->free_descq);
+
+	spin_unlock_irqrestore(&p_rx->lock, lock_flags);
+	qed_ll2b_complete_rx_gsi_packet(p_hwfn,
+					p_ll2_info->my_id,
+					p_pkt->cookie,
+					p_pkt->rx_buf_addr,
+					packet_length,
+					p_cqe->rx_cqe_gsi.data_length_error,
+					parse_flags,
+					vlan,
+					src_mac_addrhi,
+					src_mac_addrlo, b_last_cqe);
+	spin_lock_irqsave(&p_rx->lock, lock_flags);
+
+	return 0;
+}
+
+static int qed_ll2_rxq_completion_reg(struct qed_hwfn *p_hwfn,
+				      struct qed_ll2_info *p_ll2_conn,
+				      union core_rx_cqe_union *p_cqe,
+				      unsigned long lock_flags,
+				      bool b_last_cqe)
+{
+	struct qed_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
+	struct qed_ll2_rx_packet *p_pkt = NULL;
+
+	if (!list_empty(&p_rx->active_descq))
+		p_pkt = list_first_entry(&p_rx->active_descq,
+					 struct qed_ll2_rx_packet, list_entry);
+	if (!p_pkt) {
+		DP_NOTICE(p_hwfn,
+			  "LL2 Rx completion but active_descq is empty\n");
+		return -EIO;
+	}
+	list_del(&p_pkt->list_entry);
+
+	if (qed_chain_consume(&p_rx->rxq_chain) != p_pkt->rxq_bd)
+		DP_NOTICE(p_hwfn,
+			  "Mismatch between active_descq and the LL2 Rx chain\n");
+	list_add_tail(&p_pkt->list_entry, &p_rx->free_descq);
+
+	spin_unlock_irqrestore(&p_rx->lock, lock_flags);
+	qed_ll2b_complete_rx_packet(p_hwfn, p_ll2_conn->my_id,
+				    p_pkt, &p_cqe->rx_cqe_fp, b_last_cqe);
+	spin_lock_irqsave(&p_rx->lock, lock_flags);
+
+	return 0;
+}
+
+static int qed_ll2_rxq_completion(struct qed_hwfn *p_hwfn, void *cookie)
+{
+	struct qed_ll2_info *p_ll2_conn = cookie;
+	struct qed_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
+	union core_rx_cqe_union *cqe = NULL;
+	u16 cq_new_idx = 0, cq_old_idx = 0;
+	unsigned long flags = 0;
+	int rc = 0;
+
+	spin_lock_irqsave(&p_rx->lock, flags);
+	cq_new_idx = le16_to_cpu(*p_rx->p_fw_cons);
+	cq_old_idx = qed_chain_get_cons_idx(&p_rx->rcq_chain);
+
+	while (cq_new_idx != cq_old_idx) {
+		bool b_last_cqe = (cq_new_idx == cq_old_idx);
+
+		cqe = qed_chain_consume(&p_rx->rcq_chain);
+		cq_old_idx = qed_chain_get_cons_idx(&p_rx->rcq_chain);
+
+		DP_VERBOSE(p_hwfn,
+			   QED_MSG_LL2,
+			   "LL2 [sw. cons %04x, fw. at %04x] - Got Packet of type %02x\n",
+			   cq_old_idx, cq_new_idx, cqe->rx_cqe_sp.type);
+
+		switch (cqe->rx_cqe_sp.type) {
+		case CORE_RX_CQE_TYPE_SLOW_PATH:
+			DP_NOTICE(p_hwfn, "LL2 - unexpected Rx CQE slowpath\n");
+			rc = -EINVAL;
+			break;
+		case CORE_RX_CQE_TYPE_GSI_OFFLOAD:
+			rc = qed_ll2_rxq_completion_gsi(p_hwfn, p_ll2_conn,
+							cqe, flags, b_last_cqe);
+			break;
+		case CORE_RX_CQE_TYPE_REGULAR:
+			rc = qed_ll2_rxq_completion_reg(p_hwfn, p_ll2_conn,
+							cqe, flags, b_last_cqe);
+			break;
+		default:
+			rc = -EIO;
+		}
+	}
+
+	spin_unlock_irqrestore(&p_rx->lock, flags);
+	return rc;
+}
+
+void qed_ll2_rxq_flush(struct qed_hwfn *p_hwfn, u8 connection_handle)
+{
+	struct qed_ll2_info *p_ll2_conn = NULL;
+	struct qed_ll2_rx_packet *p_pkt = NULL;
+	struct qed_ll2_rx_queue *p_rx;
+
+	p_ll2_conn = qed_ll2_handle_sanity_inactive(p_hwfn, connection_handle);
+	if (!p_ll2_conn)
+		return;
+
+	p_rx = &p_ll2_conn->rx_queue;
+
+	while (!list_empty(&p_rx->active_descq)) {
+		dma_addr_t rx_buf_addr;
+		void *cookie;
+		bool b_last;
+
+		p_pkt = list_first_entry(&p_rx->active_descq,
+					 struct qed_ll2_rx_packet, list_entry);
+		if (!p_pkt)
+			break;
+
+		list_del(&p_pkt->list_entry);
+		list_add_tail(&p_pkt->list_entry, &p_rx->free_descq);
+
+		rx_buf_addr = p_pkt->rx_buf_addr;
+		cookie = p_pkt->cookie;
+
+		b_last = list_empty(&p_rx->active_descq);
+	}
+}
+
+static int qed_sp_ll2_rx_queue_start(struct qed_hwfn *p_hwfn,
+				     struct qed_ll2_info *p_ll2_conn,
+				     u8 action_on_error)
+{
+	enum qed_ll2_conn_type conn_type = p_ll2_conn->conn_type;
+	struct qed_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue;
+	struct core_rx_start_ramrod_data *p_ramrod = NULL;
+	struct qed_spq_entry *p_ent = NULL;
+	struct qed_sp_init_data init_data;
+	u16 cqe_pbl_size;
+	int rc = 0;
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = p_ll2_conn->cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 CORE_RAMROD_RX_QUEUE_START,
+				 PROTOCOLID_CORE, &init_data);
+	if (rc)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.core_rx_queue_start;
+
+	p_ramrod->sb_id = cpu_to_le16(qed_int_get_sp_sb_id(p_hwfn));
+	p_ramrod->sb_index = p_rx->rx_sb_index;
+	p_ramrod->complete_event_flg = 1;
+
+	p_ramrod->mtu = cpu_to_le16(p_ll2_conn->mtu);
+	DMA_REGPAIR_LE(p_ramrod->bd_base,
+		       p_rx->rxq_chain.p_phys_addr);
+	cqe_pbl_size = (u16)qed_chain_get_page_cnt(&p_rx->rcq_chain);
+	p_ramrod->num_of_pbl_pages = cpu_to_le16(cqe_pbl_size);
+	DMA_REGPAIR_LE(p_ramrod->cqe_pbl_addr,
+		       qed_chain_get_pbl_phys(&p_rx->rcq_chain));
+
+	p_ramrod->drop_ttl0_flg = p_ll2_conn->rx_drop_ttl0_flg;
+	p_ramrod->inner_vlan_removal_en = p_ll2_conn->rx_vlan_removal_en;
+	p_ramrod->queue_id = p_ll2_conn->queue_id;
+	p_ramrod->main_func_queue = 1;
+
+	if ((IS_MF_DEFAULT(p_hwfn) || IS_MF_SI(p_hwfn)) &&
+	    p_ramrod->main_func_queue && (conn_type != QED_LL2_TYPE_ROCE)) {
+		p_ramrod->mf_si_bcast_accept_all = 1;
+		p_ramrod->mf_si_mcast_accept_all = 1;
+	} else {
+		p_ramrod->mf_si_bcast_accept_all = 0;
+		p_ramrod->mf_si_mcast_accept_all = 0;
+	}
+
+	p_ramrod->action_on_error.error_type = action_on_error;
+	p_ramrod->gsi_offload_flag = p_ll2_conn->gsi_enable;
+	return qed_spq_post(p_hwfn, p_ent, NULL);
+}
+
+static int qed_sp_ll2_tx_queue_start(struct qed_hwfn *p_hwfn,
+				     struct qed_ll2_info *p_ll2_conn)
+{
+	enum qed_ll2_conn_type conn_type = p_ll2_conn->conn_type;
+	struct qed_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
+	struct core_tx_start_ramrod_data *p_ramrod = NULL;
+	struct qed_spq_entry *p_ent = NULL;
+	struct qed_sp_init_data init_data;
+	union qed_qm_pq_params pq_params;
+	u16 pq_id = 0, pbl_size;
+	int rc = -EINVAL;
+
+	if (!QED_LL2_TX_REGISTERED(p_ll2_conn))
+		return 0;
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = p_ll2_conn->cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 CORE_RAMROD_TX_QUEUE_START,
+				 PROTOCOLID_CORE, &init_data);
+	if (rc)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.core_tx_queue_start;
+
+	p_ramrod->sb_id = cpu_to_le16(qed_int_get_sp_sb_id(p_hwfn));
+	p_ramrod->sb_index = p_tx->tx_sb_index;
+	p_ramrod->mtu = cpu_to_le16(p_ll2_conn->mtu);
+	p_ll2_conn->tx_stats_en = 1;
+	p_ramrod->stats_en = p_ll2_conn->tx_stats_en;
+	p_ramrod->stats_id = p_ll2_conn->tx_stats_id;
+
+	DMA_REGPAIR_LE(p_ramrod->pbl_base_addr,
+		       qed_chain_get_pbl_phys(&p_tx->txq_chain));
+	pbl_size = qed_chain_get_page_cnt(&p_tx->txq_chain);
+	p_ramrod->pbl_size = cpu_to_le16(pbl_size);
+
+	memset(&pq_params, 0, sizeof(pq_params));
+	pq_params.core.tc = p_ll2_conn->tx_tc;
+	pq_id = qed_get_qm_pq(p_hwfn, PROTOCOLID_CORE, &pq_params);
+	p_ramrod->qm_pq_id = cpu_to_le16(pq_id);
+
+	switch (conn_type) {
+	case QED_LL2_TYPE_ISCSI:
+	case QED_LL2_TYPE_ISCSI_OOO:
+		p_ramrod->conn_type = PROTOCOLID_ISCSI;
+		break;
+	case QED_LL2_TYPE_ROCE:
+		p_ramrod->conn_type = PROTOCOLID_ROCE;
+		break;
+	default:
+		p_ramrod->conn_type = PROTOCOLID_ETH;
+		DP_NOTICE(p_hwfn, "Unknown connection type: %d\n", conn_type);
+	}
+
+	p_ramrod->gsi_offload_flag = p_ll2_conn->gsi_enable;
+	return qed_spq_post(p_hwfn, p_ent, NULL);
+}
+
+static int qed_sp_ll2_rx_queue_stop(struct qed_hwfn *p_hwfn,
+				    struct qed_ll2_info *p_ll2_conn)
+{
+	struct core_rx_stop_ramrod_data *p_ramrod = NULL;
+	struct qed_spq_entry *p_ent = NULL;
+	struct qed_sp_init_data init_data;
+	int rc = -EINVAL;
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = p_ll2_conn->cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 CORE_RAMROD_RX_QUEUE_STOP,
+				 PROTOCOLID_CORE, &init_data);
+	if (rc)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.core_rx_queue_stop;
+
+	p_ramrod->complete_event_flg = 1;
+	p_ramrod->queue_id = p_ll2_conn->queue_id;
+
+	return qed_spq_post(p_hwfn, p_ent, NULL);
+}
+
+static int qed_sp_ll2_tx_queue_stop(struct qed_hwfn *p_hwfn,
+				    struct qed_ll2_info *p_ll2_conn)
+{
+	struct qed_spq_entry *p_ent = NULL;
+	struct qed_sp_init_data init_data;
+	int rc = -EINVAL;
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = p_ll2_conn->cid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 CORE_RAMROD_TX_QUEUE_STOP,
+				 PROTOCOLID_CORE, &init_data);
+	if (rc)
+		return rc;
+
+	return qed_spq_post(p_hwfn, p_ent, NULL);
+}
+
+static int
+qed_ll2_acquire_connection_rx(struct qed_hwfn *p_hwfn,
+			      struct qed_ll2_info *p_ll2_info, u16 rx_num_desc)
+{
+	struct qed_ll2_rx_packet *p_descq;
+	u32 capacity;
+	int rc = 0;
+
+	if (!rx_num_desc)
+		goto out;
+
+	rc = qed_chain_alloc(p_hwfn->cdev,
+			     QED_CHAIN_USE_TO_CONSUME_PRODUCE,
+			     QED_CHAIN_MODE_NEXT_PTR,
+			     QED_CHAIN_CNT_TYPE_U16,
+			     rx_num_desc,
+			     sizeof(struct core_rx_bd),
+			     &p_ll2_info->rx_queue.rxq_chain);
+	if (rc) {
+		DP_NOTICE(p_hwfn, "Failed to allocate ll2 rxq chain\n");
+		goto out;
+	}
+
+	capacity = qed_chain_get_capacity(&p_ll2_info->rx_queue.rxq_chain);
+	p_descq = kcalloc(capacity, sizeof(struct qed_ll2_rx_packet),
+			  GFP_KERNEL);
+	if (!p_descq) {
+		rc = -ENOMEM;
+		DP_NOTICE(p_hwfn, "Failed to allocate ll2 Rx desc\n");
+		goto out;
+	}
+	p_ll2_info->rx_queue.descq_array = p_descq;
+
+	rc = qed_chain_alloc(p_hwfn->cdev,
+			     QED_CHAIN_USE_TO_CONSUME_PRODUCE,
+			     QED_CHAIN_MODE_PBL,
+			     QED_CHAIN_CNT_TYPE_U16,
+			     rx_num_desc,
+			     sizeof(struct core_rx_fast_path_cqe),
+			     &p_ll2_info->rx_queue.rcq_chain);
+	if (rc) {
+		DP_NOTICE(p_hwfn, "Failed to allocate ll2 rcq chain\n");
+		goto out;
+	}
+
+	DP_VERBOSE(p_hwfn, QED_MSG_LL2,
+		   "Allocated LL2 Rxq [Type %08x] with 0x%08x buffers\n",
+		   p_ll2_info->conn_type, rx_num_desc);
+
+out:
+	return rc;
+}
+
+static int qed_ll2_acquire_connection_tx(struct qed_hwfn *p_hwfn,
+					 struct qed_ll2_info *p_ll2_info,
+					 u16 tx_num_desc)
+{
+	struct qed_ll2_tx_packet *p_descq;
+	u32 capacity;
+	int rc = 0;
+
+	if (!tx_num_desc)
+		goto out;
+
+	rc = qed_chain_alloc(p_hwfn->cdev,
+			     QED_CHAIN_USE_TO_CONSUME_PRODUCE,
+			     QED_CHAIN_MODE_PBL,
+			     QED_CHAIN_CNT_TYPE_U16,
+			     tx_num_desc,
+			     sizeof(struct core_tx_bd),
+			     &p_ll2_info->tx_queue.txq_chain);
+	if (rc)
+		goto out;
+
+	capacity = qed_chain_get_capacity(&p_ll2_info->tx_queue.txq_chain);
+	p_descq = kcalloc(capacity, sizeof(struct qed_ll2_tx_packet),
+			  GFP_KERNEL);
+	if (!p_descq) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	p_ll2_info->tx_queue.descq_array = p_descq;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_LL2,
+		   "Allocated LL2 Txq [Type %08x] with 0x%08x buffers\n",
+		   p_ll2_info->conn_type, tx_num_desc);
+
+out:
+	if (rc)
+		DP_NOTICE(p_hwfn,
+			  "Can't allocate memory for Tx LL2 with 0x%08x buffers\n",
+			  tx_num_desc);
+	return rc;
+}
+
+int qed_ll2_acquire_connection(struct qed_hwfn *p_hwfn,
+			       struct qed_ll2_info *p_params,
+			       u16 rx_num_desc,
+			       u16 tx_num_desc,
+			       u8 *p_connection_handle)
+{
+	qed_int_comp_cb_t comp_rx_cb, comp_tx_cb;
+	struct qed_ll2_info *p_ll2_info = NULL;
+	int rc;
+	u8 i;
+
+	if (!p_connection_handle || !p_hwfn->p_ll2_info)
+		return -EINVAL;
+
+	/* Find a free connection to be used */
+	for (i = 0; (i < QED_MAX_NUM_OF_LL2_CONNECTIONS); i++) {
+		mutex_lock(&p_hwfn->p_ll2_info[i].mutex);
+		if (p_hwfn->p_ll2_info[i].b_active) {
+			mutex_unlock(&p_hwfn->p_ll2_info[i].mutex);
+			continue;
+		}
+
+		p_hwfn->p_ll2_info[i].b_active = true;
+		p_ll2_info = &p_hwfn->p_ll2_info[i];
+		mutex_unlock(&p_hwfn->p_ll2_info[i].mutex);
+		break;
+	}
+	if (!p_ll2_info)
+		return -EBUSY;
+
+	p_ll2_info->conn_type = p_params->conn_type;
+	p_ll2_info->mtu = p_params->mtu;
+	p_ll2_info->rx_drop_ttl0_flg = p_params->rx_drop_ttl0_flg;
+	p_ll2_info->rx_vlan_removal_en = p_params->rx_vlan_removal_en;
+	p_ll2_info->tx_tc = p_params->tx_tc;
+	p_ll2_info->tx_dest = p_params->tx_dest;
+	p_ll2_info->ai_err_packet_too_big = p_params->ai_err_packet_too_big;
+	p_ll2_info->ai_err_no_buf = p_params->ai_err_no_buf;
+	p_ll2_info->gsi_enable = p_params->gsi_enable;
+
+	rc = qed_ll2_acquire_connection_rx(p_hwfn, p_ll2_info, rx_num_desc);
+	if (rc)
+		goto q_allocate_fail;
+
+	rc = qed_ll2_acquire_connection_tx(p_hwfn, p_ll2_info, tx_num_desc);
+	if (rc)
+		goto q_allocate_fail;
+
+	/* Register callbacks for the Rx/Tx queues */
+	comp_rx_cb = qed_ll2_rxq_completion;
+	comp_tx_cb = qed_ll2_txq_completion;
+
+	if (rx_num_desc) {
+		qed_int_register_cb(p_hwfn, comp_rx_cb,
+				    &p_hwfn->p_ll2_info[i],
+				    &p_ll2_info->rx_queue.rx_sb_index,
+				    &p_ll2_info->rx_queue.p_fw_cons);
+		p_ll2_info->rx_queue.b_cb_registred = true;
+	}
+
+	if (tx_num_desc) {
+		qed_int_register_cb(p_hwfn,
+				    comp_tx_cb,
+				    &p_hwfn->p_ll2_info[i],
+				    &p_ll2_info->tx_queue.tx_sb_index,
+				    &p_ll2_info->tx_queue.p_fw_cons);
+		p_ll2_info->tx_queue.b_cb_registred = true;
+	}
+
+	*p_connection_handle = i;
+	return rc;
+
+q_allocate_fail:
+	qed_ll2_release_connection(p_hwfn, i);
+	return -ENOMEM;
+}
+
+static int qed_ll2_establish_connection_rx(struct qed_hwfn *p_hwfn,
+					   struct qed_ll2_info *p_ll2_conn)
+{
+	u8 action_on_error = 0;
+
+	if (!QED_LL2_RX_REGISTERED(p_ll2_conn))
+		return 0;
+
+	DIRECT_REG_WR(p_ll2_conn->rx_queue.set_prod_addr, 0x0);
+
+	SET_FIELD(action_on_error,
+		  CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG,
+		  p_ll2_conn->ai_err_packet_too_big);
+	SET_FIELD(action_on_error,
+		  CORE_RX_ACTION_ON_ERROR_NO_BUFF, p_ll2_conn->ai_err_no_buf);
+
+	return qed_sp_ll2_rx_queue_start(p_hwfn, p_ll2_conn, action_on_error);
+}
+
+int qed_ll2_establish_connection(struct qed_hwfn *p_hwfn, u8 connection_handle)
+{
+	struct qed_ll2_info *p_ll2_conn;
+	struct qed_ll2_rx_queue *p_rx;
+	struct qed_ll2_tx_queue *p_tx;
+	int rc = -EINVAL;
+	u32 i, capacity;
+	u8 qid;
+
+	p_ll2_conn = qed_ll2_handle_sanity_lock(p_hwfn, connection_handle);
+	if (!p_ll2_conn)
+		return -EINVAL;
+	p_rx = &p_ll2_conn->rx_queue;
+	p_tx = &p_ll2_conn->tx_queue;
+
+	qed_chain_reset(&p_rx->rxq_chain);
+	qed_chain_reset(&p_rx->rcq_chain);
+	INIT_LIST_HEAD(&p_rx->active_descq);
+	INIT_LIST_HEAD(&p_rx->free_descq);
+	INIT_LIST_HEAD(&p_rx->posting_descq);
+	spin_lock_init(&p_rx->lock);
+	capacity = qed_chain_get_capacity(&p_rx->rxq_chain);
+	for (i = 0; i < capacity; i++)
+		list_add_tail(&p_rx->descq_array[i].list_entry,
+			      &p_rx->free_descq);
+	*p_rx->p_fw_cons = 0;
+
+	qed_chain_reset(&p_tx->txq_chain);
+	INIT_LIST_HEAD(&p_tx->active_descq);
+	INIT_LIST_HEAD(&p_tx->free_descq);
+	INIT_LIST_HEAD(&p_tx->sending_descq);
+	spin_lock_init(&p_tx->lock);
+	capacity = qed_chain_get_capacity(&p_tx->txq_chain);
+	for (i = 0; i < capacity; i++)
+		list_add_tail(&p_tx->descq_array[i].list_entry,
+			      &p_tx->free_descq);
+	p_tx->cur_completing_bd_idx = 0;
+	p_tx->bds_idx = 0;
+	p_tx->b_completing_packet = false;
+	p_tx->cur_send_packet = NULL;
+	p_tx->cur_send_frag_num = 0;
+	p_tx->cur_completing_frag_num = 0;
+	*p_tx->p_fw_cons = 0;
+
+	qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_ll2_conn->cid);
+
+	qid = p_hwfn->hw_info.resc_start[QED_LL2_QUEUE] + connection_handle;
+	p_ll2_conn->queue_id = qid;
+	p_ll2_conn->tx_stats_id = qid;
+	p_rx->set_prod_addr = (u8 __iomem *)p_hwfn->regview +
+					    GTT_BAR0_MAP_REG_TSDM_RAM +
+					    TSTORM_LL2_RX_PRODS_OFFSET(qid);
+	p_tx->doorbell_addr = (u8 __iomem *)p_hwfn->doorbells +
+					    qed_db_addr(p_ll2_conn->cid,
+							DQ_DEMS_LEGACY);
+
+	rc = qed_ll2_establish_connection_rx(p_hwfn, p_ll2_conn);
+	if (rc)
+		return rc;
+
+	rc = qed_sp_ll2_tx_queue_start(p_hwfn, p_ll2_conn);
+	if (rc)
+		return rc;
+
+	if (p_hwfn->hw_info.personality != QED_PCI_ETH_ROCE)
+		qed_wr(p_hwfn, p_hwfn->p_main_ptt, PRS_REG_USE_LIGHT_L2, 1);
+
+	return rc;
+}
+
+static void qed_ll2_post_rx_buffer_notify_fw(struct qed_hwfn *p_hwfn,
+					     struct qed_ll2_rx_queue *p_rx,
+					     struct qed_ll2_rx_packet *p_curp)
+{
+	struct qed_ll2_rx_packet *p_posting_packet = NULL;
+	struct core_ll2_rx_prod rx_prod = { 0, 0, 0 };
+	bool b_notify_fw = false;
+	u16 bd_prod, cq_prod;
+
+	/* This handles the flushing of already posted buffers */
+	while (!list_empty(&p_rx->posting_descq)) {
+		p_posting_packet = list_first_entry(&p_rx->posting_descq,
+						    struct qed_ll2_rx_packet,
+						    list_entry);
+		list_del(&p_posting_packet->list_entry);
+		list_add_tail(&p_posting_packet->list_entry,
+			      &p_rx->active_descq);
+		b_notify_fw = true;
+	}
+
+	/* This handles the supplied packet [if there is one] */
+	if (p_curp) {
+		list_add_tail(&p_curp->list_entry, &p_rx->active_descq);
+		b_notify_fw = true;
+	}
+
+	if (!b_notify_fw)
+		return;
+
+	bd_prod = qed_chain_get_prod_idx(&p_rx->rxq_chain);
+	cq_prod = qed_chain_get_prod_idx(&p_rx->rcq_chain);
+	rx_prod.bd_prod = cpu_to_le16(bd_prod);
+	rx_prod.cqe_prod = cpu_to_le16(cq_prod);
+	DIRECT_REG_WR(p_rx->set_prod_addr, *((u32 *)&rx_prod));
+}
+
+int qed_ll2_post_rx_buffer(struct qed_hwfn *p_hwfn,
+			   u8 connection_handle,
+			   dma_addr_t addr,
+			   u16 buf_len, void *cookie, u8 notify_fw)
+{
+	struct core_rx_bd_with_buff_len *p_curb = NULL;
+	struct qed_ll2_rx_packet *p_curp = NULL;
+	struct qed_ll2_info *p_ll2_conn;
+	struct qed_ll2_rx_queue *p_rx;
+	unsigned long flags;
+	void *p_data;
+	int rc = 0;
+
+	p_ll2_conn = qed_ll2_handle_sanity(p_hwfn, connection_handle);
+	if (!p_ll2_conn)
+		return -EINVAL;
+	p_rx = &p_ll2_conn->rx_queue;
+
+	spin_lock_irqsave(&p_rx->lock, flags);
+	if (!list_empty(&p_rx->free_descq))
+		p_curp = list_first_entry(&p_rx->free_descq,
+					  struct qed_ll2_rx_packet, list_entry);
+	if (p_curp) {
+		if (qed_chain_get_elem_left(&p_rx->rxq_chain) &&
+		    qed_chain_get_elem_left(&p_rx->rcq_chain)) {
+			p_data = qed_chain_produce(&p_rx->rxq_chain);
+			p_curb = (struct core_rx_bd_with_buff_len *)p_data;
+			qed_chain_produce(&p_rx->rcq_chain);
+		}
+	}
+
+	/* If we're lacking entires, let's try to flush buffers to FW */
+	if (!p_curp || !p_curb) {
+		rc = -EBUSY;
+		p_curp = NULL;
+		goto out_notify;
+	}
+
+	/* We have an Rx packet we can fill */
+	DMA_REGPAIR_LE(p_curb->addr, addr);
+	p_curb->buff_length = cpu_to_le16(buf_len);
+	p_curp->rx_buf_addr = addr;
+	p_curp->cookie = cookie;
+	p_curp->rxq_bd = p_curb;
+	p_curp->buf_length = buf_len;
+	list_del(&p_curp->list_entry);
+
+	/* Check if we only want to enqueue this packet without informing FW */
+	if (!notify_fw) {
+		list_add_tail(&p_curp->list_entry, &p_rx->posting_descq);
+		goto out;
+	}
+
+out_notify:
+	qed_ll2_post_rx_buffer_notify_fw(p_hwfn, p_rx, p_curp);
+out:
+	spin_unlock_irqrestore(&p_rx->lock, flags);
+	return rc;
+}
+
+static void qed_ll2_prepare_tx_packet_set(struct qed_hwfn *p_hwfn,
+					  struct qed_ll2_tx_queue *p_tx,
+					  struct qed_ll2_tx_packet *p_curp,
+					  u8 num_of_bds,
+					  dma_addr_t first_frag,
+					  u16 first_frag_len, void *p_cookie,
+					  u8 notify_fw)
+{
+	list_del(&p_curp->list_entry);
+	p_curp->cookie = p_cookie;
+	p_curp->bd_used = num_of_bds;
+	p_curp->notify_fw = notify_fw;
+	p_tx->cur_send_packet = p_curp;
+	p_tx->cur_send_frag_num = 0;
+
+	p_curp->bds_set[p_tx->cur_send_frag_num].tx_frag = first_frag;
+	p_curp->bds_set[p_tx->cur_send_frag_num].frag_len = first_frag_len;
+	p_tx->cur_send_frag_num++;
+}
+
+static void qed_ll2_prepare_tx_packet_set_bd(struct qed_hwfn *p_hwfn,
+					     struct qed_ll2_info *p_ll2,
+					     struct qed_ll2_tx_packet *p_curp,
+					     u8 num_of_bds,
+					     enum core_tx_dest tx_dest,
+					     u16 vlan,
+					     u8 bd_flags,
+					     u16 l4_hdr_offset_w,
+					     enum core_roce_flavor_type type,
+					     dma_addr_t first_frag,
+					     u16 first_frag_len)
+{
+	struct qed_chain *p_tx_chain = &p_ll2->tx_queue.txq_chain;
+	u16 prod_idx = qed_chain_get_prod_idx(p_tx_chain);
+	struct core_tx_bd *start_bd = NULL;
+	u16 frag_idx;
+
+	start_bd = (struct core_tx_bd *)qed_chain_produce(p_tx_chain);
+	start_bd->nw_vlan_or_lb_echo = cpu_to_le16(vlan);
+	SET_FIELD(start_bd->bitfield1, CORE_TX_BD_L4_HDR_OFFSET_W,
+		  cpu_to_le16(l4_hdr_offset_w));
+	SET_FIELD(start_bd->bitfield1, CORE_TX_BD_TX_DST, tx_dest);
+	start_bd->bd_flags.as_bitfield = bd_flags;
+	start_bd->bd_flags.as_bitfield |= CORE_TX_BD_FLAGS_START_BD_MASK <<
+	    CORE_TX_BD_FLAGS_START_BD_SHIFT;
+	SET_FIELD(start_bd->bitfield0, CORE_TX_BD_NBDS, num_of_bds);
+	DMA_REGPAIR_LE(start_bd->addr, first_frag);
+	start_bd->nbytes = cpu_to_le16(first_frag_len);
+
+	SET_FIELD(start_bd->bd_flags.as_bitfield, CORE_TX_BD_FLAGS_ROCE_FLAV,
+		  type);
+
+	DP_VERBOSE(p_hwfn,
+		   (NETIF_MSG_TX_QUEUED | QED_MSG_LL2),
+		   "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Tx Producer at [0x%04x] - set with a %04x bytes %02x BDs buffer at %08x:%08x\n",
+		   p_ll2->queue_id,
+		   p_ll2->cid,
+		   p_ll2->conn_type,
+		   prod_idx,
+		   first_frag_len,
+		   num_of_bds,
+		   le32_to_cpu(start_bd->addr.hi),
+		   le32_to_cpu(start_bd->addr.lo));
+
+	if (p_ll2->tx_queue.cur_send_frag_num == num_of_bds)
+		return;
+
+	/* Need to provide the packet with additional BDs for frags */
+	for (frag_idx = p_ll2->tx_queue.cur_send_frag_num;
+	     frag_idx < num_of_bds; frag_idx++) {
+		struct core_tx_bd **p_bd = &p_curp->bds_set[frag_idx].txq_bd;
+
+		*p_bd = (struct core_tx_bd *)qed_chain_produce(p_tx_chain);
+		(*p_bd)->bd_flags.as_bitfield = 0;
+		(*p_bd)->bitfield1 = 0;
+		(*p_bd)->bitfield0 = 0;
+		p_curp->bds_set[frag_idx].tx_frag = 0;
+		p_curp->bds_set[frag_idx].frag_len = 0;
+	}
+}
+
+/* This should be called while the Txq spinlock is being held */
+static void qed_ll2_tx_packet_notify(struct qed_hwfn *p_hwfn,
+				     struct qed_ll2_info *p_ll2_conn)
+{
+	bool b_notify = p_ll2_conn->tx_queue.cur_send_packet->notify_fw;
+	struct qed_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue;
+	struct qed_ll2_tx_packet *p_pkt = NULL;
+	struct core_db_data db_msg = { 0, 0, 0 };
+	u16 bd_prod;
+
+	/* If there are missing BDs, don't do anything now */
+	if (p_ll2_conn->tx_queue.cur_send_frag_num !=
+	    p_ll2_conn->tx_queue.cur_send_packet->bd_used)
+		return;
+
+	/* Push the current packet to the list and clean after it */
+	list_add_tail(&p_ll2_conn->tx_queue.cur_send_packet->list_entry,
+		      &p_ll2_conn->tx_queue.sending_descq);
+	p_ll2_conn->tx_queue.cur_send_packet = NULL;
+	p_ll2_conn->tx_queue.cur_send_frag_num = 0;
+
+	/* Notify FW of packet only if requested to */
+	if (!b_notify)
+		return;
+
+	bd_prod = qed_chain_get_prod_idx(&p_ll2_conn->tx_queue.txq_chain);
+
+	while (!list_empty(&p_tx->sending_descq)) {
+		p_pkt = list_first_entry(&p_tx->sending_descq,
+					 struct qed_ll2_tx_packet, list_entry);
+		if (!p_pkt)
+			break;
+
+		list_del(&p_pkt->list_entry);
+		list_add_tail(&p_pkt->list_entry, &p_tx->active_descq);
+	}
+
+	SET_FIELD(db_msg.params, CORE_DB_DATA_DEST, DB_DEST_XCM);
+	SET_FIELD(db_msg.params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_SET);
+	SET_FIELD(db_msg.params, CORE_DB_DATA_AGG_VAL_SEL,
+		  DQ_XCM_CORE_TX_BD_PROD_CMD);
+	db_msg.agg_flags = DQ_XCM_CORE_DQ_CF_CMD;
+	db_msg.spq_prod = cpu_to_le16(bd_prod);
+
+	/* Make sure the BDs data is updated before ringing the doorbell */
+	wmb();
+
+	DIRECT_REG_WR(p_tx->doorbell_addr, *((u32 *)&db_msg));
+
+	DP_VERBOSE(p_hwfn,
+		   (NETIF_MSG_TX_QUEUED | QED_MSG_LL2),
+		   "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Doorbelled [producer 0x%04x]\n",
+		   p_ll2_conn->queue_id,
+		   p_ll2_conn->cid, p_ll2_conn->conn_type, db_msg.spq_prod);
+}
+
+int qed_ll2_prepare_tx_packet(struct qed_hwfn *p_hwfn,
+			      u8 connection_handle,
+			      u8 num_of_bds,
+			      u16 vlan,
+			      u8 bd_flags,
+			      u16 l4_hdr_offset_w,
+			      enum qed_ll2_roce_flavor_type qed_roce_flavor,
+			      dma_addr_t first_frag,
+			      u16 first_frag_len, void *cookie, u8 notify_fw)
+{
+	struct qed_ll2_tx_packet *p_curp = NULL;
+	struct qed_ll2_info *p_ll2_conn = NULL;
+	enum core_roce_flavor_type roce_flavor;
+	struct qed_ll2_tx_queue *p_tx;
+	struct qed_chain *p_tx_chain;
+	unsigned long flags;
+	int rc = 0;
+
+	p_ll2_conn = qed_ll2_handle_sanity(p_hwfn, connection_handle);
+	if (!p_ll2_conn)
+		return -EINVAL;
+	p_tx = &p_ll2_conn->tx_queue;
+	p_tx_chain = &p_tx->txq_chain;
+
+	if (num_of_bds > CORE_LL2_TX_MAX_BDS_PER_PACKET)
+		return -EIO;
+
+	spin_lock_irqsave(&p_tx->lock, flags);
+	if (p_tx->cur_send_packet) {
+		rc = -EEXIST;
+		goto out;
+	}
+
+	/* Get entry, but only if we have tx elements for it */
+	if (!list_empty(&p_tx->free_descq))
+		p_curp = list_first_entry(&p_tx->free_descq,
+					  struct qed_ll2_tx_packet, list_entry);
+	if (p_curp && qed_chain_get_elem_left(p_tx_chain) < num_of_bds)
+		p_curp = NULL;
+
+	if (!p_curp) {
+		rc = -EBUSY;
+		goto out;
+	}
+
+	if (qed_roce_flavor == QED_LL2_ROCE) {
+		roce_flavor = CORE_ROCE;
+	} else if (qed_roce_flavor == QED_LL2_RROCE) {
+		roce_flavor = CORE_RROCE;
+	} else {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	/* Prepare packet and BD, and perhaps send a doorbell to FW */
+	qed_ll2_prepare_tx_packet_set(p_hwfn, p_tx, p_curp,
+				      num_of_bds, first_frag,
+				      first_frag_len, cookie, notify_fw);
+	qed_ll2_prepare_tx_packet_set_bd(p_hwfn, p_ll2_conn, p_curp,
+					 num_of_bds, CORE_TX_DEST_NW,
+					 vlan, bd_flags, l4_hdr_offset_w,
+					 roce_flavor,
+					 first_frag, first_frag_len);
+
+	qed_ll2_tx_packet_notify(p_hwfn, p_ll2_conn);
+
+out:
+	spin_unlock_irqrestore(&p_tx->lock, flags);
+	return rc;
+}
+
+int qed_ll2_set_fragment_of_tx_packet(struct qed_hwfn *p_hwfn,
+				      u8 connection_handle,
+				      dma_addr_t addr, u16 nbytes)
+{
+	struct qed_ll2_tx_packet *p_cur_send_packet = NULL;
+	struct qed_ll2_info *p_ll2_conn = NULL;
+	u16 cur_send_frag_num = 0;
+	struct core_tx_bd *p_bd;
+	unsigned long flags;
+
+	p_ll2_conn = qed_ll2_handle_sanity(p_hwfn, connection_handle);
+	if (!p_ll2_conn)
+		return -EINVAL;
+
+	if (!p_ll2_conn->tx_queue.cur_send_packet)
+		return -EINVAL;
+
+	p_cur_send_packet = p_ll2_conn->tx_queue.cur_send_packet;
+	cur_send_frag_num = p_ll2_conn->tx_queue.cur_send_frag_num;
+
+	if (cur_send_frag_num >= p_cur_send_packet->bd_used)
+		return -EINVAL;
+
+	/* Fill the BD information, and possibly notify FW */
+	p_bd = p_cur_send_packet->bds_set[cur_send_frag_num].txq_bd;
+	DMA_REGPAIR_LE(p_bd->addr, addr);
+	p_bd->nbytes = cpu_to_le16(nbytes);
+	p_cur_send_packet->bds_set[cur_send_frag_num].tx_frag = addr;
+	p_cur_send_packet->bds_set[cur_send_frag_num].frag_len = nbytes;
+
+	p_ll2_conn->tx_queue.cur_send_frag_num++;
+
+	spin_lock_irqsave(&p_ll2_conn->tx_queue.lock, flags);
+	qed_ll2_tx_packet_notify(p_hwfn, p_ll2_conn);
+	spin_unlock_irqrestore(&p_ll2_conn->tx_queue.lock, flags);
+
+	return 0;
+}
+
+int qed_ll2_terminate_connection(struct qed_hwfn *p_hwfn, u8 connection_handle)
+{
+	struct qed_ll2_info *p_ll2_conn = NULL;
+	int rc = -EINVAL;
+
+	p_ll2_conn = qed_ll2_handle_sanity_lock(p_hwfn, connection_handle);
+	if (!p_ll2_conn)
+		return -EINVAL;
+
+	/* Stop Tx & Rx of connection, if needed */
+	if (QED_LL2_TX_REGISTERED(p_ll2_conn)) {
+		rc = qed_sp_ll2_tx_queue_stop(p_hwfn, p_ll2_conn);
+		if (rc)
+			return rc;
+		qed_ll2_txq_flush(p_hwfn, connection_handle);
+	}
+
+	if (QED_LL2_RX_REGISTERED(p_ll2_conn)) {
+		rc = qed_sp_ll2_rx_queue_stop(p_hwfn, p_ll2_conn);
+		if (rc)
+			return rc;
+		qed_ll2_rxq_flush(p_hwfn, connection_handle);
+	}
+
+	return rc;
+}
+
+void qed_ll2_release_connection(struct qed_hwfn *p_hwfn, u8 connection_handle)
+{
+	struct qed_ll2_info *p_ll2_conn = NULL;
+
+	p_ll2_conn = qed_ll2_handle_sanity(p_hwfn, connection_handle);
+	if (!p_ll2_conn)
+		return;
+
+	if (QED_LL2_RX_REGISTERED(p_ll2_conn)) {
+		p_ll2_conn->rx_queue.b_cb_registred = false;
+		qed_int_unregister_cb(p_hwfn, p_ll2_conn->rx_queue.rx_sb_index);
+	}
+
+	if (QED_LL2_TX_REGISTERED(p_ll2_conn)) {
+		p_ll2_conn->tx_queue.b_cb_registred = false;
+		qed_int_unregister_cb(p_hwfn, p_ll2_conn->tx_queue.tx_sb_index);
+	}
+
+	kfree(p_ll2_conn->tx_queue.descq_array);
+	qed_chain_free(p_hwfn->cdev, &p_ll2_conn->tx_queue.txq_chain);
+
+	kfree(p_ll2_conn->rx_queue.descq_array);
+	qed_chain_free(p_hwfn->cdev, &p_ll2_conn->rx_queue.rxq_chain);
+	qed_chain_free(p_hwfn->cdev, &p_ll2_conn->rx_queue.rcq_chain);
+
+	qed_cxt_release_cid(p_hwfn, p_ll2_conn->cid);
+
+	mutex_lock(&p_ll2_conn->mutex);
+	p_ll2_conn->b_active = false;
+	mutex_unlock(&p_ll2_conn->mutex);
+}
+
+struct qed_ll2_info *qed_ll2_alloc(struct qed_hwfn *p_hwfn)
+{
+	struct qed_ll2_info *p_ll2_connections;
+	u8 i;
+
+	/* Allocate LL2's set struct */
+	p_ll2_connections = kcalloc(QED_MAX_NUM_OF_LL2_CONNECTIONS,
+				    sizeof(struct qed_ll2_info), GFP_KERNEL);
+	if (!p_ll2_connections) {
+		DP_NOTICE(p_hwfn, "Failed to allocate `struct qed_ll2'\n");
+		return NULL;
+	}
+
+	for (i = 0; i < QED_MAX_NUM_OF_LL2_CONNECTIONS; i++)
+		p_ll2_connections[i].my_id = i;
+
+	return p_ll2_connections;
+}
+
+void qed_ll2_setup(struct qed_hwfn *p_hwfn,
+		   struct qed_ll2_info *p_ll2_connections)
+{
+	int i;
+
+	for (i = 0; i < QED_MAX_NUM_OF_LL2_CONNECTIONS; i++)
+		mutex_init(&p_ll2_connections[i].mutex);
+}
+
+void qed_ll2_free(struct qed_hwfn *p_hwfn,
+		  struct qed_ll2_info *p_ll2_connections)
+{
+	kfree(p_ll2_connections);
+}
+
+static void _qed_ll2_get_tstats(struct qed_hwfn *p_hwfn,
+				struct qed_ptt *p_ptt,
+				struct qed_ll2_info *p_ll2_conn,
+				struct qed_ll2_stats *p_stats)
+{
+	struct core_ll2_tstorm_per_queue_stat tstats;
+	u8 qid = p_ll2_conn->queue_id;
+	u32 tstats_addr;
+
+	memset(&tstats, 0, sizeof(tstats));
+	tstats_addr = BAR0_MAP_REG_TSDM_RAM +
+		      CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(qid);
+	qed_memcpy_from(p_hwfn, p_ptt, &tstats, tstats_addr, sizeof(tstats));
+
+	p_stats->packet_too_big_discard =
+			HILO_64_REGPAIR(tstats.packet_too_big_discard);
+	p_stats->no_buff_discard = HILO_64_REGPAIR(tstats.no_buff_discard);
+}
+
+static void _qed_ll2_get_ustats(struct qed_hwfn *p_hwfn,
+				struct qed_ptt *p_ptt,
+				struct qed_ll2_info *p_ll2_conn,
+				struct qed_ll2_stats *p_stats)
+{
+	struct core_ll2_ustorm_per_queue_stat ustats;
+	u8 qid = p_ll2_conn->queue_id;
+	u32 ustats_addr;
+
+	memset(&ustats, 0, sizeof(ustats));
+	ustats_addr = BAR0_MAP_REG_USDM_RAM +
+		      CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(qid);
+	qed_memcpy_from(p_hwfn, p_ptt, &ustats, ustats_addr, sizeof(ustats));
+
+	p_stats->rcv_ucast_bytes = HILO_64_REGPAIR(ustats.rcv_ucast_bytes);
+	p_stats->rcv_mcast_bytes = HILO_64_REGPAIR(ustats.rcv_mcast_bytes);
+	p_stats->rcv_bcast_bytes = HILO_64_REGPAIR(ustats.rcv_bcast_bytes);
+	p_stats->rcv_ucast_pkts = HILO_64_REGPAIR(ustats.rcv_ucast_pkts);
+	p_stats->rcv_mcast_pkts = HILO_64_REGPAIR(ustats.rcv_mcast_pkts);
+	p_stats->rcv_bcast_pkts = HILO_64_REGPAIR(ustats.rcv_bcast_pkts);
+}
+
+static void _qed_ll2_get_pstats(struct qed_hwfn *p_hwfn,
+				struct qed_ptt *p_ptt,
+				struct qed_ll2_info *p_ll2_conn,
+				struct qed_ll2_stats *p_stats)
+{
+	struct core_ll2_pstorm_per_queue_stat pstats;
+	u8 stats_id = p_ll2_conn->tx_stats_id;
+	u32 pstats_addr;
+
+	memset(&pstats, 0, sizeof(pstats));
+	pstats_addr = BAR0_MAP_REG_PSDM_RAM +
+		      CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(stats_id);
+	qed_memcpy_from(p_hwfn, p_ptt, &pstats, pstats_addr, sizeof(pstats));
+
+	p_stats->sent_ucast_bytes = HILO_64_REGPAIR(pstats.sent_ucast_bytes);
+	p_stats->sent_mcast_bytes = HILO_64_REGPAIR(pstats.sent_mcast_bytes);
+	p_stats->sent_bcast_bytes = HILO_64_REGPAIR(pstats.sent_bcast_bytes);
+	p_stats->sent_ucast_pkts = HILO_64_REGPAIR(pstats.sent_ucast_pkts);
+	p_stats->sent_mcast_pkts = HILO_64_REGPAIR(pstats.sent_mcast_pkts);
+	p_stats->sent_bcast_pkts = HILO_64_REGPAIR(pstats.sent_bcast_pkts);
+}
+
+int qed_ll2_get_stats(struct qed_hwfn *p_hwfn,
+		      u8 connection_handle, struct qed_ll2_stats *p_stats)
+{
+	struct qed_ll2_info *p_ll2_conn = NULL;
+	struct qed_ptt *p_ptt;
+
+	memset(p_stats, 0, sizeof(*p_stats));
+
+	if ((connection_handle >= QED_MAX_NUM_OF_LL2_CONNECTIONS) ||
+	    !p_hwfn->p_ll2_info)
+		return -EINVAL;
+
+	p_ll2_conn = &p_hwfn->p_ll2_info[connection_handle];
+
+	p_ptt = qed_ptt_acquire(p_hwfn);
+	if (!p_ptt) {
+		DP_ERR(p_hwfn, "Failed to acquire ptt\n");
+		return -EINVAL;
+	}
+
+	_qed_ll2_get_tstats(p_hwfn, p_ptt, p_ll2_conn, p_stats);
+	_qed_ll2_get_ustats(p_hwfn, p_ptt, p_ll2_conn, p_stats);
+	if (p_ll2_conn->tx_stats_en)
+		_qed_ll2_get_pstats(p_hwfn, p_ptt, p_ll2_conn, p_stats);
+
+	qed_ptt_release(p_hwfn, p_ptt);
+	return 0;
+}
+
+static void qed_ll2_register_cb_ops(struct qed_dev *cdev,
+				    const struct qed_ll2_cb_ops *ops,
+				    void *cookie)
+{
+	cdev->ll2->cbs = ops;
+	cdev->ll2->cb_cookie = cookie;
+}
+
+static int qed_ll2_start(struct qed_dev *cdev, struct qed_ll2_params *params)
+{
+	struct qed_ll2_info ll2_info;
+	struct qed_ll2_buffer *buffer;
+	enum qed_ll2_conn_type conn_type;
+	struct qed_ptt *p_ptt;
+	int rc, i;
+
+	/* Initialize LL2 locks & lists */
+	INIT_LIST_HEAD(&cdev->ll2->list);
+	spin_lock_init(&cdev->ll2->lock);
+	cdev->ll2->rx_size = NET_SKB_PAD + ETH_HLEN +
+			     L1_CACHE_BYTES + params->mtu;
+	cdev->ll2->frags_mapped = params->frags_mapped;
+
+	/*Allocate memory for LL2 */
+	DP_INFO(cdev, "Allocating LL2 buffers of size %08x bytes\n",
+		cdev->ll2->rx_size);
+	for (i = 0; i < QED_LL2_RX_SIZE; i++) {
+		buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+		if (!buffer) {
+			DP_INFO(cdev, "Failed to allocate LL2 buffers\n");
+			goto fail;
+		}
+
+		rc = qed_ll2_alloc_buffer(cdev, (u8 **)&buffer->data,
+					  &buffer->phys_addr);
+		if (rc) {
+			kfree(buffer);
+			goto fail;
+		}
+
+		list_add_tail(&buffer->list, &cdev->ll2->list);
+	}
+
+	switch (QED_LEADING_HWFN(cdev)->hw_info.personality) {
+	case QED_PCI_ISCSI:
+		conn_type = QED_LL2_TYPE_ISCSI;
+		break;
+	case QED_PCI_ETH_ROCE:
+		conn_type = QED_LL2_TYPE_ROCE;
+		break;
+	default:
+		conn_type = QED_LL2_TYPE_TEST;
+	}
+
+	/* Prepare the temporary ll2 information */
+	memset(&ll2_info, 0, sizeof(ll2_info));
+	ll2_info.conn_type = conn_type;
+	ll2_info.mtu = params->mtu;
+	ll2_info.rx_drop_ttl0_flg = params->drop_ttl0_packets;
+	ll2_info.rx_vlan_removal_en = params->rx_vlan_stripping;
+	ll2_info.tx_tc = 0;
+	ll2_info.tx_dest = CORE_TX_DEST_NW;
+	ll2_info.gsi_enable = 1;
+
+	rc = qed_ll2_acquire_connection(QED_LEADING_HWFN(cdev), &ll2_info,
+					QED_LL2_RX_SIZE, QED_LL2_TX_SIZE,
+					&cdev->ll2->handle);
+	if (rc) {
+		DP_INFO(cdev, "Failed to acquire LL2 connection\n");
+		goto fail;
+	}
+
+	rc = qed_ll2_establish_connection(QED_LEADING_HWFN(cdev),
+					  cdev->ll2->handle);
+	if (rc) {
+		DP_INFO(cdev, "Failed to establish LL2 connection\n");
+		goto release_fail;
+	}
+
+	/* Post all Rx buffers to FW */
+	spin_lock_bh(&cdev->ll2->lock);
+	list_for_each_entry(buffer, &cdev->ll2->list, list) {
+		rc = qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev),
+					    cdev->ll2->handle,
+					    buffer->phys_addr, 0, buffer, 1);
+		if (rc) {
+			DP_INFO(cdev,
+				"Failed to post an Rx buffer; Deleting it\n");
+			dma_unmap_single(&cdev->pdev->dev, buffer->phys_addr,
+					 cdev->ll2->rx_size, DMA_FROM_DEVICE);
+			kfree(buffer->data);
+			list_del(&buffer->list);
+			kfree(buffer);
+		} else {
+			cdev->ll2->rx_cnt++;
+		}
+	}
+	spin_unlock_bh(&cdev->ll2->lock);
+
+	if (!cdev->ll2->rx_cnt) {
+		DP_INFO(cdev, "Failed passing even a single Rx buffer\n");
+		goto release_terminate;
+	}
+
+	if (!is_valid_ether_addr(params->ll2_mac_address)) {
+		DP_INFO(cdev, "Invalid Ethernet address\n");
+		goto release_terminate;
+	}
+
+	p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev));
+	if (!p_ptt) {
+		DP_INFO(cdev, "Failed to acquire PTT\n");
+		goto release_terminate;
+	}
+
+	rc = qed_llh_add_mac_filter(QED_LEADING_HWFN(cdev), p_ptt,
+				    params->ll2_mac_address);
+	qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt);
+	if (rc) {
+		DP_ERR(cdev, "Failed to allocate LLH filter\n");
+		goto release_terminate_all;
+	}
+
+	ether_addr_copy(cdev->ll2_mac_address, params->ll2_mac_address);
+
+	return 0;
+
+release_terminate_all:
+
+release_terminate:
+	qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), cdev->ll2->handle);
+release_fail:
+	qed_ll2_release_connection(QED_LEADING_HWFN(cdev), cdev->ll2->handle);
+fail:
+	qed_ll2_kill_buffers(cdev);
+	cdev->ll2->handle = QED_LL2_UNUSED_HANDLE;
+	return -EINVAL;
+}
+
+static int qed_ll2_stop(struct qed_dev *cdev)
+{
+	struct qed_ptt *p_ptt;
+	int rc;
+
+	if (cdev->ll2->handle == QED_LL2_UNUSED_HANDLE)
+		return 0;
+
+	p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev));
+	if (!p_ptt) {
+		DP_INFO(cdev, "Failed to acquire PTT\n");
+		goto fail;
+	}
+
+	qed_llh_remove_mac_filter(QED_LEADING_HWFN(cdev), p_ptt,
+				  cdev->ll2_mac_address);
+	qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt);
+	eth_zero_addr(cdev->ll2_mac_address);
+
+	rc = qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev),
+					  cdev->ll2->handle);
+	if (rc)
+		DP_INFO(cdev, "Failed to terminate LL2 connection\n");
+
+	qed_ll2_kill_buffers(cdev);
+
+	qed_ll2_release_connection(QED_LEADING_HWFN(cdev), cdev->ll2->handle);
+	cdev->ll2->handle = QED_LL2_UNUSED_HANDLE;
+
+	return rc;
+fail:
+	return -EINVAL;
+}
+
+static int qed_ll2_start_xmit(struct qed_dev *cdev, struct sk_buff *skb)
+{
+	const skb_frag_t *frag;
+	int rc = -EINVAL, i;
+	dma_addr_t mapping;
+	u16 vlan = 0;
+	u8 flags = 0;
+
+	if (unlikely(skb->ip_summed != CHECKSUM_NONE)) {
+		DP_INFO(cdev, "Cannot transmit a checksumed packet\n");
+		return -EINVAL;
+	}
+
+	if (1 + skb_shinfo(skb)->nr_frags > CORE_LL2_TX_MAX_BDS_PER_PACKET) {
+		DP_ERR(cdev, "Cannot transmit a packet with %d fragments\n",
+		       1 + skb_shinfo(skb)->nr_frags);
+		return -EINVAL;
+	}
+
+	mapping = dma_map_single(&cdev->pdev->dev, skb->data,
+				 skb->len, DMA_TO_DEVICE);
+	if (unlikely(dma_mapping_error(&cdev->pdev->dev, mapping))) {
+		DP_NOTICE(cdev, "SKB mapping failed\n");
+		return -EINVAL;
+	}
+
+	/* Request HW to calculate IP csum */
+	if (!((vlan_get_protocol(skb) == htons(ETH_P_IPV6)) &&
+	      ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
+		flags |= BIT(CORE_TX_BD_FLAGS_IP_CSUM_SHIFT);
+
+	if (skb_vlan_tag_present(skb)) {
+		vlan = skb_vlan_tag_get(skb);
+		flags |= BIT(CORE_TX_BD_FLAGS_VLAN_INSERTION_SHIFT);
+	}
+
+	rc = qed_ll2_prepare_tx_packet(QED_LEADING_HWFN(cdev),
+				       cdev->ll2->handle,
+				       1 + skb_shinfo(skb)->nr_frags,
+				       vlan, flags, 0, 0 /* RoCE FLAVOR */,
+				       mapping, skb->len, skb, 1);
+	if (rc)
+		goto err;
+
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		frag = &skb_shinfo(skb)->frags[i];
+		if (!cdev->ll2->frags_mapped) {
+			mapping = skb_frag_dma_map(&cdev->pdev->dev, frag, 0,
+						   skb_frag_size(frag),
+						   DMA_TO_DEVICE);
+
+			if (unlikely(dma_mapping_error(&cdev->pdev->dev,
+						       mapping))) {
+				DP_NOTICE(cdev,
+					  "Unable to map frag - dropping packet\n");
+				goto err;
+			}
+		} else {
+			mapping = page_to_phys(skb_frag_page(frag)) |
+			    frag->page_offset;
+		}
+
+		rc = qed_ll2_set_fragment_of_tx_packet(QED_LEADING_HWFN(cdev),
+						       cdev->ll2->handle,
+						       mapping,
+						       skb_frag_size(frag));
+
+		/* if failed not much to do here, partial packet has been posted
+		 * we can't free memory, will need to wait for completion.
+		 */
+		if (rc)
+			goto err2;
+	}
+
+	return 0;
+
+err:
+	dma_unmap_single(&cdev->pdev->dev, mapping, skb->len, DMA_TO_DEVICE);
+
+err2:
+	return rc;
+}
+
+static int qed_ll2_stats(struct qed_dev *cdev, struct qed_ll2_stats *stats)
+{
+	if (!cdev->ll2)
+		return -EINVAL;
+
+	return qed_ll2_get_stats(QED_LEADING_HWFN(cdev),
+				 cdev->ll2->handle, stats);
+}
+
+const struct qed_ll2_ops qed_ll2_ops_pass = {
+	.start = &qed_ll2_start,
+	.stop = &qed_ll2_stop,
+	.start_xmit = &qed_ll2_start_xmit,
+	.register_cb_ops = &qed_ll2_register_cb_ops,
+	.get_stats = &qed_ll2_stats,
+};
+
+int qed_ll2_alloc_if(struct qed_dev *cdev)
+{
+	cdev->ll2 = kzalloc(sizeof(*cdev->ll2), GFP_KERNEL);
+	return cdev->ll2 ? 0 : -ENOMEM;
+}
+
+void qed_ll2_dealloc_if(struct qed_dev *cdev)
+{
+	kfree(cdev->ll2);
+	cdev->ll2 = NULL;
+}
diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.h b/drivers/net/ethernet/qlogic/qed/qed_ll2.h
new file mode 100644
index 000000000000..80a5dc2d652d
--- /dev/null
+++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.h
@@ -0,0 +1,316 @@
+/* QLogic qed NIC Driver
+ *
+ * Copyright (c) 2015 QLogic Corporation
+ *
+ * This software is available under the terms of the GNU General Public License
+ * (GPL) Version 2, available from the file COPYING in the main directory of
+ * this source tree.
+ */
+
+#ifndef _QED_LL2_H
+#define _QED_LL2_H
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/qed/qed_chain.h>
+#include <linux/qed/qed_ll2_if.h>
+#include "qed.h"
+#include "qed_hsi.h"
+#include "qed_sp.h"
+
+#define QED_MAX_NUM_OF_LL2_CONNECTIONS                    (4)
+
+enum qed_ll2_roce_flavor_type {
+	QED_LL2_ROCE,
+	QED_LL2_RROCE,
+	MAX_QED_LL2_ROCE_FLAVOR_TYPE
+};
+
+enum qed_ll2_conn_type {
+	QED_LL2_TYPE_RESERVED,
+	QED_LL2_TYPE_ISCSI,
+	QED_LL2_TYPE_TEST,
+	QED_LL2_TYPE_ISCSI_OOO,
+	QED_LL2_TYPE_RESERVED2,
+	QED_LL2_TYPE_ROCE,
+	QED_LL2_TYPE_RESERVED3,
+	MAX_QED_LL2_RX_CONN_TYPE
+};
+
+struct qed_ll2_rx_packet {
+	struct list_head list_entry;
+	struct core_rx_bd_with_buff_len *rxq_bd;
+	dma_addr_t rx_buf_addr;
+	u16 buf_length;
+	void *cookie;
+	u8 placement_offset;
+	u16 parse_flags;
+	u16 packet_length;
+	u16 vlan;
+	u32 opaque_data[2];
+};
+
+struct qed_ll2_tx_packet {
+	struct list_head list_entry;
+	u16 bd_used;
+	u16 vlan;
+	u16 l4_hdr_offset_w;
+	u8 bd_flags;
+	bool notify_fw;
+	void *cookie;
+
+	struct {
+		struct core_tx_bd *txq_bd;
+		dma_addr_t tx_frag;
+		u16 frag_len;
+	} bds_set[ETH_TX_MAX_BDS_PER_NON_LSO_PACKET];
+};
+
+struct qed_ll2_rx_queue {
+	/* Lock protecting the Rx queue manipulation */
+	spinlock_t lock;
+	struct qed_chain rxq_chain;
+	struct qed_chain rcq_chain;
+	u8 rx_sb_index;
+	bool b_cb_registred;
+	__le16 *p_fw_cons;
+	struct list_head active_descq;
+	struct list_head free_descq;
+	struct list_head posting_descq;
+	struct qed_ll2_rx_packet *descq_array;
+	void __iomem *set_prod_addr;
+};
+
+struct qed_ll2_tx_queue {
+	/* Lock protecting the Tx queue manipulation */
+	spinlock_t lock;
+	struct qed_chain txq_chain;
+	u8 tx_sb_index;
+	bool b_cb_registred;
+	__le16 *p_fw_cons;
+	struct list_head active_descq;
+	struct list_head free_descq;
+	struct list_head sending_descq;
+	struct qed_ll2_tx_packet *descq_array;
+	struct qed_ll2_tx_packet *cur_send_packet;
+	struct qed_ll2_tx_packet cur_completing_packet;
+	u16 cur_completing_bd_idx;
+	void __iomem *doorbell_addr;
+	u16 bds_idx;
+	u16 cur_send_frag_num;
+	u16 cur_completing_frag_num;
+	bool b_completing_packet;
+};
+
+struct qed_ll2_info {
+	/* Lock protecting the state of LL2 */
+	struct mutex mutex;
+	enum qed_ll2_conn_type conn_type;
+	u32 cid;
+	u8 my_id;
+	u8 queue_id;
+	u8 tx_stats_id;
+	bool b_active;
+	u16 mtu;
+	u8 rx_drop_ttl0_flg;
+	u8 rx_vlan_removal_en;
+	u8 tx_tc;
+	enum core_tx_dest tx_dest;
+	enum core_error_handle ai_err_packet_too_big;
+	enum core_error_handle ai_err_no_buf;
+	u8 tx_stats_en;
+	struct qed_ll2_rx_queue rx_queue;
+	struct qed_ll2_tx_queue tx_queue;
+	u8 gsi_enable;
+};
+
+/**
+ * @brief qed_ll2_acquire_connection - allocate resources,
+ *        starts rx & tx (if relevant) queues pair. Provides
+ *        connecion handler as output parameter.
+ *
+ * @param p_hwfn
+ * @param p_params		Contain various configuration properties
+ * @param rx_num_desc
+ * @param tx_num_desc
+ *
+ * @param p_connection_handle  Output container for LL2 connection's handle
+ *
+ * @return 0 on success, failure otherwise
+ */
+int qed_ll2_acquire_connection(struct qed_hwfn *p_hwfn,
+			       struct qed_ll2_info *p_params,
+			       u16 rx_num_desc,
+			       u16 tx_num_desc,
+			       u8 *p_connection_handle);
+
+/**
+ * @brief qed_ll2_establish_connection - start previously
+ *        allocated LL2 queues pair
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ * @param connection_handle	LL2 connection's handle obtained from
+ *                              qed_ll2_require_connection
+ *
+ * @return 0 on success, failure otherwise
+ */
+int qed_ll2_establish_connection(struct qed_hwfn *p_hwfn, u8 connection_handle);
+
+/**
+ * @brief qed_ll2_post_rx_buffers - submit buffers to LL2 Rx queue.
+ *
+ * @param p_hwfn
+ * @param connection_handle	LL2 connection's handle obtained from
+ *				qed_ll2_require_connection
+ * @param addr			rx (physical address) buffers to submit
+ * @param cookie
+ * @param notify_fw		produce corresponding Rx BD immediately
+ *
+ * @return 0 on success, failure otherwise
+ */
+int qed_ll2_post_rx_buffer(struct qed_hwfn *p_hwfn,
+			   u8 connection_handle,
+			   dma_addr_t addr,
+			   u16 buf_len, void *cookie, u8 notify_fw);
+
+/**
+ * @brief qed_ll2_prepare_tx_packet - request for start Tx BD
+ *				      to prepare Tx packet submission to FW.
+ *
+ * @param p_hwfn
+ * @param connection_handle	LL2 connection's handle obtained from
+ *				qed_ll2_require_connection
+ * @param num_of_bds		a number of requested BD equals a number of
+ *				fragments in Tx packet
+ * @param vlan			VLAN to insert to packet (if insertion set)
+ * @param bd_flags
+ * @param l4_hdr_offset_w	L4 Header Offset from start of packet
+ *				(in words). This is needed if both l4_csum
+ *				and ipv6_ext are set
+ * @param first_frag
+ * @param first_frag_len
+ * @param cookie
+ *
+ * @param notify_fw
+ *
+ * @return 0 on success, failure otherwise
+ */
+int qed_ll2_prepare_tx_packet(struct qed_hwfn *p_hwfn,
+			      u8 connection_handle,
+			      u8 num_of_bds,
+			      u16 vlan,
+			      u8 bd_flags,
+			      u16 l4_hdr_offset_w,
+			      enum qed_ll2_roce_flavor_type qed_roce_flavor,
+			      dma_addr_t first_frag,
+			      u16 first_frag_len, void *cookie, u8 notify_fw);
+
+/**
+ * @brief qed_ll2_release_connection -	releases resources
+ *					allocated for LL2 connection
+ *
+ * @param p_hwfn
+ * @param connection_handle		LL2 connection's handle obtained from
+ *					qed_ll2_require_connection
+ */
+void qed_ll2_release_connection(struct qed_hwfn *p_hwfn, u8 connection_handle);
+
+/**
+ * @brief qed_ll2_set_fragment_of_tx_packet -	provides fragments to fill
+ *						Tx BD of BDs requested by
+ *						qed_ll2_prepare_tx_packet
+ *
+ * @param p_hwfn
+ * @param connection_handle			LL2 connection's handle
+ *						obtained from
+ *						qed_ll2_require_connection
+ * @param addr
+ * @param nbytes
+ *
+ * @return 0 on success, failure otherwise
+ */
+int qed_ll2_set_fragment_of_tx_packet(struct qed_hwfn *p_hwfn,
+				      u8 connection_handle,
+				      dma_addr_t addr, u16 nbytes);
+
+/**
+ * @brief qed_ll2_terminate_connection -	stops Tx/Rx queues
+ *
+ *
+ * @param p_hwfn
+ * @param connection_handle			LL2 connection's handle
+ *						obtained from
+ *						qed_ll2_require_connection
+ *
+ * @return 0 on success, failure otherwise
+ */
+int qed_ll2_terminate_connection(struct qed_hwfn *p_hwfn, u8 connection_handle);
+
+/**
+ * @brief qed_ll2_get_stats -	get LL2 queue's statistics
+ *
+ *
+ * @param p_hwfn
+ * @param connection_handle	LL2 connection's handle obtained from
+ *				qed_ll2_require_connection
+ * @param p_stats
+ *
+ * @return 0 on success, failure otherwise
+ */
+int qed_ll2_get_stats(struct qed_hwfn *p_hwfn,
+		      u8 connection_handle, struct qed_ll2_stats *p_stats);
+
+/**
+ * @brief qed_ll2_alloc - Allocates LL2 connections set
+ *
+ * @param p_hwfn
+ *
+ * @return pointer to alocated qed_ll2_info or NULL
+ */
+struct qed_ll2_info *qed_ll2_alloc(struct qed_hwfn *p_hwfn);
+
+/**
+ * @brief qed_ll2_setup - Inits LL2 connections set
+ *
+ * @param p_hwfn
+ * @param p_ll2_connections
+ *
+ */
+void qed_ll2_setup(struct qed_hwfn *p_hwfn,
+		   struct qed_ll2_info *p_ll2_connections);
+
+/**
+ * @brief qed_ll2_free - Releases LL2 connections set
+ *
+ * @param p_hwfn
+ * @param p_ll2_connections
+ *
+ */
+void qed_ll2_free(struct qed_hwfn *p_hwfn,
+		  struct qed_ll2_info *p_ll2_connections);
+void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn *p_hwfn,
+				     u8 connection_handle,
+				     void *cookie,
+				     dma_addr_t rx_buf_addr,
+				     u16 data_length,
+				     u8 data_length_error,
+				     u16 parse_flags,
+				     u16 vlan,
+				     u32 src_mac_addr_hi,
+				     u16 src_mac_addr_lo, bool b_last_packet);
+void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn *p_hwfn,
+				     u8 connection_handle,
+				     void *cookie,
+				     dma_addr_t first_frag_addr,
+				     bool b_last_fragment, bool b_last_packet);
+void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn *p_hwfn,
+				    u8 connection_handle,
+				    void *cookie,
+				    dma_addr_t first_frag_addr,
+				    bool b_last_fragment, bool b_last_packet);
+#endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c
index c7dc34bfdd0a..4ee3151e80c2 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_main.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_main.c
@@ -22,15 +22,22 @@
 #include <linux/etherdevice.h>
 #include <linux/vmalloc.h>
 #include <linux/qed/qed_if.h>
+#include <linux/qed/qed_ll2_if.h>
 
 #include "qed.h"
 #include "qed_sriov.h"
 #include "qed_sp.h"
 #include "qed_dev_api.h"
+#include "qed_ll2.h"
 #include "qed_mcp.h"
 #include "qed_hw.h"
 #include "qed_selftest.h"
 
+#if IS_ENABLED(CONFIG_INFINIBAND_QEDR)
+#define QED_ROCE_QPS			(8192)
+#define QED_ROCE_DPIS			(8)
+#endif
+
 static char version[] =
 	"QLogic FastLinQ 4xxxx Core Module qed " DRV_MODULE_VERSION "\n";
 
@@ -51,8 +58,6 @@ MODULE_FIRMWARE(QED_FW_FILE_NAME);
 
 static int __init qed_init(void)
 {
-	pr_notice("qed_init called\n");
-
 	pr_info("%s", version);
 
 	return 0;
@@ -106,8 +111,7 @@ static void qed_free_pci(struct qed_dev *cdev)
 /* Performs PCI initializations as well as initializing PCI-related parameters
  * in the device structrue. Returns 0 in case of success.
  */
-static int qed_init_pci(struct qed_dev *cdev,
-			struct pci_dev *pdev)
+static int qed_init_pci(struct qed_dev *cdev, struct pci_dev *pdev)
 {
 	u8 rev_id;
 	int rc;
@@ -207,8 +211,8 @@ int qed_fill_dev_info(struct qed_dev *cdev,
 	dev_info->pci_mem_start = cdev->pci_params.mem_start;
 	dev_info->pci_mem_end = cdev->pci_params.mem_end;
 	dev_info->pci_irq = cdev->pci_params.irq;
-	dev_info->rdma_supported =
-	    (cdev->hwfns[0].hw_info.personality == QED_PCI_ETH_ROCE);
+	dev_info->rdma_supported = (cdev->hwfns[0].hw_info.personality ==
+				    QED_PCI_ETH_ROCE);
 	dev_info->is_mf_default = IS_MF_DEFAULT(&cdev->hwfns[0]);
 	ether_addr_copy(dev_info->hw_mac, cdev->hwfns[0].hw_info.hw_mac_addr);
 
@@ -263,8 +267,7 @@ static struct qed_dev *qed_alloc_cdev(struct pci_dev *pdev)
 }
 
 /* Sets the requested power state */
-static int qed_set_power_state(struct qed_dev *cdev,
-			       pci_power_t state)
+static int qed_set_power_state(struct qed_dev *cdev, pci_power_t state)
 {
 	if (!cdev)
 		return -ENODEV;
@@ -366,8 +369,8 @@ static int qed_enable_msix(struct qed_dev *cdev,
 		DP_NOTICE(cdev,
 			  "Trying to enable MSI-X with less vectors (%d out of %d)\n",
 			  cnt, int_params->in.num_vectors);
-		rc = pci_enable_msix_exact(cdev->pdev,
-					   int_params->msix_table, cnt);
+		rc = pci_enable_msix_exact(cdev->pdev, int_params->msix_table,
+					   cnt);
 		if (!rc)
 			rc = cnt;
 	}
@@ -439,6 +442,11 @@ static int qed_set_int_mode(struct qed_dev *cdev, bool force_mode)
 	}
 
 out:
+	if (!rc)
+		DP_INFO(cdev, "Using %s interrupts\n",
+			int_params->out.int_mode == QED_INT_MODE_INTA ?
+			"INTa" : int_params->out.int_mode == QED_INT_MODE_MSI ?
+			"MSI" : "MSIX");
 	cdev->int_coalescing_mode = QED_COAL_MODE_ENABLE;
 
 	return rc;
@@ -514,19 +522,18 @@ static irqreturn_t qed_single_int(int irq, void *dev_instance)
 int qed_slowpath_irq_req(struct qed_hwfn *hwfn)
 {
 	struct qed_dev *cdev = hwfn->cdev;
+	u32 int_mode;
 	int rc = 0;
 	u8 id;
 
-	if (cdev->int_params.out.int_mode == QED_INT_MODE_MSIX) {
+	int_mode = cdev->int_params.out.int_mode;
+	if (int_mode == QED_INT_MODE_MSIX) {
 		id = hwfn->my_id;
 		snprintf(hwfn->name, NAME_SIZE, "sp-%d-%02x:%02x.%02x",
 			 id, cdev->pdev->bus->number,
 			 PCI_SLOT(cdev->pdev->devfn), hwfn->abs_pf_id);
 		rc = request_irq(cdev->int_params.msix_table[id].vector,
 				 qed_msix_sp_int, 0, hwfn->name, hwfn->sp_dpc);
-		if (!rc)
-			DP_VERBOSE(hwfn, (NETIF_MSG_INTR | QED_MSG_SP),
-				   "Requested slowpath MSI-X\n");
 	} else {
 		unsigned long flags = 0;
 
@@ -541,6 +548,13 @@ int qed_slowpath_irq_req(struct qed_hwfn *hwfn)
 				 flags, cdev->name, cdev);
 	}
 
+	if (rc)
+		DP_NOTICE(cdev, "request_irq failed, rc = %d\n", rc);
+	else
+		DP_VERBOSE(hwfn, (NETIF_MSG_INTR | QED_MSG_SP),
+			   "Requested slowpath %s\n",
+			   (int_mode == QED_INT_MODE_MSIX) ? "MSI-X" : "IRQ");
+
 	return rc;
 }
 
@@ -581,6 +595,8 @@ static int qed_nic_stop(struct qed_dev *cdev)
 		}
 	}
 
+	qed_dbg_pf_exit(cdev);
+
 	return rc;
 }
 
@@ -599,7 +615,16 @@ static int qed_nic_reset(struct qed_dev *cdev)
 
 static int qed_nic_setup(struct qed_dev *cdev)
 {
-	int rc;
+	int rc, i;
+
+	/* Determine if interface is going to require LL2 */
+	if (QED_LEADING_HWFN(cdev)->hw_info.personality != QED_PCI_ETH) {
+		for (i = 0; i < cdev->num_hwfns; i++) {
+			struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
+
+			p_hwfn->using_ll2 = true;
+		}
+	}
 
 	rc = qed_resc_alloc(cdev);
 	if (rc)
@@ -657,6 +682,9 @@ static int qed_slowpath_setup_int(struct qed_dev *cdev,
 				  enum qed_int_mode int_mode)
 {
 	struct qed_sb_cnt_info sb_cnt_info;
+#if IS_ENABLED(CONFIG_INFINIBAND_QEDR)
+	int num_l2_queues;
+#endif
 	int rc;
 	int i;
 
@@ -687,6 +715,31 @@ static int qed_slowpath_setup_int(struct qed_dev *cdev,
 	cdev->int_params.fp_msix_cnt = cdev->int_params.out.num_vectors -
 				       cdev->num_hwfns;
 
+#if IS_ENABLED(CONFIG_INFINIBAND_QEDR)
+	num_l2_queues = 0;
+	for_each_hwfn(cdev, i)
+		num_l2_queues += FEAT_NUM(&cdev->hwfns[i], QED_PF_L2_QUE);
+
+	DP_VERBOSE(cdev, QED_MSG_RDMA,
+		   "cdev->int_params.fp_msix_cnt=%d num_l2_queues=%d\n",
+		   cdev->int_params.fp_msix_cnt, num_l2_queues);
+
+	if (cdev->int_params.fp_msix_cnt > num_l2_queues) {
+		cdev->int_params.rdma_msix_cnt =
+			(cdev->int_params.fp_msix_cnt - num_l2_queues)
+			/ cdev->num_hwfns;
+		cdev->int_params.rdma_msix_base =
+			cdev->int_params.fp_msix_base + num_l2_queues;
+		cdev->int_params.fp_msix_cnt = num_l2_queues;
+	} else {
+		cdev->int_params.rdma_msix_cnt = 0;
+	}
+
+	DP_VERBOSE(cdev, QED_MSG_RDMA, "roce_msix_cnt=%d roce_msix_base=%d\n",
+		   cdev->int_params.rdma_msix_cnt,
+		   cdev->int_params.rdma_msix_base);
+#endif
+
 	return 0;
 }
 
@@ -790,6 +843,13 @@ static void qed_update_pf_params(struct qed_dev *cdev,
 {
 	int i;
 
+#if IS_ENABLED(CONFIG_INFINIBAND_QEDR)
+	params->rdma_pf_params.num_qps = QED_ROCE_QPS;
+	params->rdma_pf_params.min_dpis = QED_ROCE_DPIS;
+	/* divide by 3 the MRs to avoid MF ILT overflow */
+	params->rdma_pf_params.num_mrs = RDMA_MAX_TIDS;
+	params->rdma_pf_params.gl_pi = QED_ROCE_PROTOCOL_INDEX;
+#endif
 	for (i = 0; i < cdev->num_hwfns; i++) {
 		struct qed_hwfn *p_hwfn = &cdev->hwfns[i];
 
@@ -834,13 +894,13 @@ static int qed_slowpath_start(struct qed_dev *cdev,
 	if (IS_PF(cdev)) {
 		/* Allocate stream for unzipping */
 		rc = qed_alloc_stream_mem(cdev);
-		if (rc) {
-			DP_NOTICE(cdev, "Failed to allocate stream memory\n");
+		if (rc)
 			goto err2;
-		}
 
 		/* First Dword used to diffrentiate between various sources */
 		data = cdev->firmware->data + sizeof(u32);
+
+		qed_dbg_pf_init(cdev);
 	}
 
 	memset(&tunn_info, 0, sizeof(tunn_info));
@@ -864,6 +924,12 @@ static int qed_slowpath_start(struct qed_dev *cdev,
 	DP_INFO(cdev,
 		"HW initialization and function start completed successfully\n");
 
+	/* Allocate LL2 interface if needed */
+	if (QED_LEADING_HWFN(cdev)->using_ll2) {
+		rc = qed_ll2_alloc_if(cdev);
+		if (rc)
+			goto err3;
+	}
 	if (IS_PF(cdev)) {
 		hwfn = QED_LEADING_HWFN(cdev);
 		drv_version.version = (params->drv_major << 24) |
@@ -884,6 +950,8 @@ static int qed_slowpath_start(struct qed_dev *cdev,
 
 	return 0;
 
+err3:
+	qed_hw_stop(cdev);
 err2:
 	qed_hw_timers_stop_all(cdev);
 	if (IS_PF(cdev))
@@ -906,6 +974,8 @@ static int qed_slowpath_stop(struct qed_dev *cdev)
 	if (!cdev)
 		return -ENODEV;
 
+	qed_ll2_dealloc_if(cdev);
+
 	if (IS_PF(cdev)) {
 		qed_free_stream_mem(cdev);
 		if (IS_QED_ETH_IF(cdev))
@@ -974,8 +1044,7 @@ static u32 qed_sb_init(struct qed_dev *cdev,
 }
 
 static u32 qed_sb_release(struct qed_dev *cdev,
-			  struct qed_sb_info *sb_info,
-			  u16 sb_id)
+			  struct qed_sb_info *sb_info, u16 sb_id)
 {
 	struct qed_hwfn *p_hwfn;
 	int hwfn_index;
@@ -1025,20 +1094,23 @@ static int qed_set_link(struct qed_dev *cdev, struct qed_link_params *params)
 		link_params->speed.autoneg = params->autoneg;
 	if (params->override_flags & QED_LINK_OVERRIDE_SPEED_ADV_SPEEDS) {
 		link_params->speed.advertised_speeds = 0;
-		if ((params->adv_speeds & SUPPORTED_1000baseT_Half) ||
-		    (params->adv_speeds & SUPPORTED_1000baseT_Full))
+		if ((params->adv_speeds & QED_LM_1000baseT_Half_BIT) ||
+		    (params->adv_speeds & QED_LM_1000baseT_Full_BIT))
+			link_params->speed.advertised_speeds |=
+			    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G;
+		if (params->adv_speeds & QED_LM_10000baseKR_Full_BIT)
 			link_params->speed.advertised_speeds |=
-				NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G;
-		if (params->adv_speeds & SUPPORTED_10000baseKR_Full)
+			    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G;
+		if (params->adv_speeds & QED_LM_25000baseKR_Full_BIT)
 			link_params->speed.advertised_speeds |=
-				NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G;
-		if (params->adv_speeds & SUPPORTED_40000baseLR4_Full)
+			    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G;
+		if (params->adv_speeds & QED_LM_40000baseLR4_Full_BIT)
 			link_params->speed.advertised_speeds |=
-				NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G;
-		if (params->adv_speeds & 0)
+			    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G;
+		if (params->adv_speeds & QED_LM_50000baseKR2_Full_BIT)
 			link_params->speed.advertised_speeds |=
-				NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G;
-		if (params->adv_speeds & 0)
+			    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G;
+		if (params->adv_speeds & QED_LM_100000baseKR4_Full_BIT)
 			link_params->speed.advertised_speeds |=
 			    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G;
 	}
@@ -1168,50 +1240,56 @@ static void qed_fill_link(struct qed_hwfn *hwfn,
 		if_link->link_up = true;
 
 	/* TODO - at the moment assume supported and advertised speed equal */
-	if_link->supported_caps = SUPPORTED_FIBRE;
+	if_link->supported_caps = QED_LM_FIBRE_BIT;
 	if (params.speed.autoneg)
-		if_link->supported_caps |= SUPPORTED_Autoneg;
+		if_link->supported_caps |= QED_LM_Autoneg_BIT;
 	if (params.pause.autoneg ||
 	    (params.pause.forced_rx && params.pause.forced_tx))
-		if_link->supported_caps |= SUPPORTED_Asym_Pause;
+		if_link->supported_caps |= QED_LM_Asym_Pause_BIT;
 	if (params.pause.autoneg || params.pause.forced_rx ||
 	    params.pause.forced_tx)
-		if_link->supported_caps |= SUPPORTED_Pause;
+		if_link->supported_caps |= QED_LM_Pause_BIT;
 
 	if_link->advertised_caps = if_link->supported_caps;
 	if (params.speed.advertised_speeds &
 	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G)
-		if_link->advertised_caps |= SUPPORTED_1000baseT_Half |
-					   SUPPORTED_1000baseT_Full;
+		if_link->advertised_caps |= QED_LM_1000baseT_Half_BIT |
+		    QED_LM_1000baseT_Full_BIT;
 	if (params.speed.advertised_speeds &
 	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G)
-		if_link->advertised_caps |= SUPPORTED_10000baseKR_Full;
+		if_link->advertised_caps |= QED_LM_10000baseKR_Full_BIT;
 	if (params.speed.advertised_speeds &
-		NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G)
-		if_link->advertised_caps |= SUPPORTED_40000baseLR4_Full;
+	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G)
+		if_link->advertised_caps |= QED_LM_25000baseKR_Full_BIT;
 	if (params.speed.advertised_speeds &
-		NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G)
-		if_link->advertised_caps |= 0;
+	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G)
+		if_link->advertised_caps |= QED_LM_40000baseLR4_Full_BIT;
+	if (params.speed.advertised_speeds &
+	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G)
+		if_link->advertised_caps |= QED_LM_50000baseKR2_Full_BIT;
 	if (params.speed.advertised_speeds &
 	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G)
-		if_link->advertised_caps |= 0;
+		if_link->advertised_caps |= QED_LM_100000baseKR4_Full_BIT;
 
 	if (link_caps.speed_capabilities &
 	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G)
-		if_link->supported_caps |= SUPPORTED_1000baseT_Half |
-					   SUPPORTED_1000baseT_Full;
+		if_link->supported_caps |= QED_LM_1000baseT_Half_BIT |
+		    QED_LM_1000baseT_Full_BIT;
 	if (link_caps.speed_capabilities &
 	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G)
-		if_link->supported_caps |= SUPPORTED_10000baseKR_Full;
+		if_link->supported_caps |= QED_LM_10000baseKR_Full_BIT;
+	if (link_caps.speed_capabilities &
+	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G)
+		if_link->supported_caps |= QED_LM_25000baseKR_Full_BIT;
 	if (link_caps.speed_capabilities &
-		NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G)
-		if_link->supported_caps |= SUPPORTED_40000baseLR4_Full;
+	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_40G)
+		if_link->supported_caps |= QED_LM_40000baseLR4_Full_BIT;
 	if (link_caps.speed_capabilities &
-		NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G)
-		if_link->supported_caps |= 0;
+	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G)
+		if_link->supported_caps |= QED_LM_50000baseKR2_Full_BIT;
 	if (link_caps.speed_capabilities &
 	    NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G)
-		if_link->supported_caps |= 0;
+		if_link->supported_caps |= QED_LM_100000baseKR4_Full_BIT;
 
 	if (link.link_up)
 		if_link->speed = link.speed;
@@ -1231,33 +1309,29 @@ static void qed_fill_link(struct qed_hwfn *hwfn,
 		if_link->pause_config |= QED_LINK_PAUSE_TX_ENABLE;
 
 	/* Link partner capabilities */
-	if (link.partner_adv_speed &
-	    QED_LINK_PARTNER_SPEED_1G_HD)
-		if_link->lp_caps |= SUPPORTED_1000baseT_Half;
-	if (link.partner_adv_speed &
-	    QED_LINK_PARTNER_SPEED_1G_FD)
-		if_link->lp_caps |= SUPPORTED_1000baseT_Full;
-	if (link.partner_adv_speed &
-	    QED_LINK_PARTNER_SPEED_10G)
-		if_link->lp_caps |= SUPPORTED_10000baseKR_Full;
-	if (link.partner_adv_speed &
-	    QED_LINK_PARTNER_SPEED_40G)
-		if_link->lp_caps |= SUPPORTED_40000baseLR4_Full;
-	if (link.partner_adv_speed &
-	    QED_LINK_PARTNER_SPEED_50G)
-		if_link->lp_caps |= 0;
-	if (link.partner_adv_speed &
-	    QED_LINK_PARTNER_SPEED_100G)
-		if_link->lp_caps |= 0;
+	if (link.partner_adv_speed & QED_LINK_PARTNER_SPEED_1G_HD)
+		if_link->lp_caps |= QED_LM_1000baseT_Half_BIT;
+	if (link.partner_adv_speed & QED_LINK_PARTNER_SPEED_1G_FD)
+		if_link->lp_caps |= QED_LM_1000baseT_Full_BIT;
+	if (link.partner_adv_speed & QED_LINK_PARTNER_SPEED_10G)
+		if_link->lp_caps |= QED_LM_10000baseKR_Full_BIT;
+	if (link.partner_adv_speed & QED_LINK_PARTNER_SPEED_25G)
+		if_link->lp_caps |= QED_LM_25000baseKR_Full_BIT;
+	if (link.partner_adv_speed & QED_LINK_PARTNER_SPEED_40G)
+		if_link->lp_caps |= QED_LM_40000baseLR4_Full_BIT;
+	if (link.partner_adv_speed & QED_LINK_PARTNER_SPEED_50G)
+		if_link->lp_caps |= QED_LM_50000baseKR2_Full_BIT;
+	if (link.partner_adv_speed & QED_LINK_PARTNER_SPEED_100G)
+		if_link->lp_caps |= QED_LM_100000baseKR4_Full_BIT;
 
 	if (link.an_complete)
-		if_link->lp_caps |= SUPPORTED_Autoneg;
+		if_link->lp_caps |= QED_LM_Autoneg_BIT;
 
 	if (link.partner_adv_pause)
-		if_link->lp_caps |= SUPPORTED_Pause;
+		if_link->lp_caps |= QED_LM_Pause_BIT;
 	if (link.partner_adv_pause == QED_LINK_PARTNER_ASYMMETRIC_PAUSE ||
 	    link.partner_adv_pause == QED_LINK_PARTNER_BOTH_PAUSE)
-		if_link->lp_caps |= SUPPORTED_Asym_Pause;
+		if_link->lp_caps |= QED_LM_Asym_Pause_BIT;
 }
 
 static void qed_get_current_link(struct qed_dev *cdev,
@@ -1385,9 +1459,32 @@ const struct qed_common_ops qed_common_ops_pass = {
 	.get_link = &qed_get_current_link,
 	.drain = &qed_drain,
 	.update_msglvl = &qed_init_dp,
+	.dbg_all_data = &qed_dbg_all_data,
+	.dbg_all_data_size = &qed_dbg_all_data_size,
 	.chain_alloc = &qed_chain_alloc,
 	.chain_free = &qed_chain_free,
 	.get_coalesce = &qed_get_coalesce,
 	.set_coalesce = &qed_set_coalesce,
 	.set_led = &qed_set_led,
 };
+
+void qed_get_protocol_stats(struct qed_dev *cdev,
+			    enum qed_mcp_protocol_type type,
+			    union qed_mcp_protocol_stats *stats)
+{
+	struct qed_eth_stats eth_stats;
+
+	memset(stats, 0, sizeof(*stats));
+
+	switch (type) {
+	case QED_MCP_LAN_STATS:
+		qed_get_vport_stats(cdev, &eth_stats);
+		stats->lan_stats.ucast_rx_pkts = eth_stats.rx_ucast_pkts;
+		stats->lan_stats.ucast_tx_pkts = eth_stats.tx_ucast_pkts;
+		stats->lan_stats.fcs_err = -1;
+		break;
+	default:
+		DP_ERR(cdev, "Invalid protocol type = %d\n", type);
+		return;
+	}
+}
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.c b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
index a240f26344a4..bdc9ba92f6d4 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
@@ -54,8 +54,7 @@ bool qed_mcp_is_init(struct qed_hwfn *p_hwfn)
 	return true;
 }
 
-void qed_mcp_cmd_port_init(struct qed_hwfn *p_hwfn,
-			   struct qed_ptt *p_ptt)
+void qed_mcp_cmd_port_init(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	u32 addr = SECTION_OFFSIZE_ADDR(p_hwfn->mcp_info->public_base,
 					PUBLIC_PORT);
@@ -68,8 +67,7 @@ void qed_mcp_cmd_port_init(struct qed_hwfn *p_hwfn,
 		   p_hwfn->mcp_info->port_addr, MFW_PORT(p_hwfn));
 }
 
-void qed_mcp_read_mb(struct qed_hwfn *p_hwfn,
-		     struct qed_ptt *p_ptt)
+void qed_mcp_read_mb(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	u32 length = MFW_DRV_MSG_MAX_DWORDS(p_hwfn->mcp_info->mfw_mb_length);
 	u32 tmp, i;
@@ -99,8 +97,7 @@ int qed_mcp_free(struct qed_hwfn *p_hwfn)
 	return 0;
 }
 
-static int qed_load_mcp_offsets(struct qed_hwfn *p_hwfn,
-				struct qed_ptt *p_ptt)
+static int qed_load_mcp_offsets(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	struct qed_mcp_info *p_info = p_hwfn->mcp_info;
 	u32 drv_mb_offsize, mfw_mb_offsize;
@@ -143,8 +140,7 @@ static int qed_load_mcp_offsets(struct qed_hwfn *p_hwfn,
 	return 0;
 }
 
-int qed_mcp_cmd_init(struct qed_hwfn *p_hwfn,
-		     struct qed_ptt *p_ptt)
+int qed_mcp_cmd_init(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	struct qed_mcp_info *p_info;
 	u32 size;
@@ -165,9 +161,7 @@ int qed_mcp_cmd_init(struct qed_hwfn *p_hwfn,
 
 	size = MFW_DRV_MSG_MAX_DWORDS(p_info->mfw_mb_length) * sizeof(u32);
 	p_info->mfw_mb_cur = kzalloc(size, GFP_KERNEL);
-	p_info->mfw_mb_shadow =
-		kzalloc(sizeof(u32) * MFW_DRV_MSG_MAX_DWORDS(
-				p_info->mfw_mb_length), GFP_KERNEL);
+	p_info->mfw_mb_shadow = kzalloc(size, GFP_KERNEL);
 	if (!p_info->mfw_mb_shadow || !p_info->mfw_mb_addr)
 		goto err;
 
@@ -177,7 +171,6 @@ int qed_mcp_cmd_init(struct qed_hwfn *p_hwfn,
 	return 0;
 
 err:
-	DP_NOTICE(p_hwfn, "Failed to allocate mcp memory\n");
 	qed_mcp_free(p_hwfn);
 	return -ENOMEM;
 }
@@ -189,8 +182,7 @@ err:
  * access is achieved by setting a blocking flag, which will fail other
  * competing contexts to send their mailboxes.
  */
-static int qed_mcp_mb_lock(struct qed_hwfn *p_hwfn,
-			   u32 cmd)
+static int qed_mcp_mb_lock(struct qed_hwfn *p_hwfn, u32 cmd)
 {
 	spin_lock_bh(&p_hwfn->mcp_info->lock);
 
@@ -221,15 +213,13 @@ static int qed_mcp_mb_lock(struct qed_hwfn *p_hwfn,
 	return 0;
 }
 
-static void qed_mcp_mb_unlock(struct qed_hwfn	*p_hwfn,
-			      u32		cmd)
+static void qed_mcp_mb_unlock(struct qed_hwfn *p_hwfn, u32 cmd)
 {
 	if (cmd != DRV_MSG_CODE_LOAD_REQ && cmd != DRV_MSG_CODE_UNLOAD_REQ)
 		spin_unlock_bh(&p_hwfn->mcp_info->lock);
 }
 
-int qed_mcp_reset(struct qed_hwfn *p_hwfn,
-		  struct qed_ptt *p_ptt)
+int qed_mcp_reset(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	u32 seq = ++p_hwfn->mcp_info->drv_mb_seq;
 	u8 delay = CHIP_MCP_RESP_ITER_US;
@@ -326,7 +316,8 @@ static int qed_do_mcp_cmd(struct qed_hwfn *p_hwfn,
 		*o_mcp_param = DRV_MB_RD(p_hwfn, p_ptt, fw_mb_param);
 	} else {
 		/* FW BUG! */
-		DP_ERR(p_hwfn, "MFW failed to respond!\n");
+		DP_ERR(p_hwfn, "MFW failed to respond [cmd 0x%x param 0x%x]\n",
+		       cmd, param);
 		*o_mcp_resp = 0;
 		rc = -EAGAIN;
 	}
@@ -342,7 +333,7 @@ static int qed_mcp_cmd_and_union(struct qed_hwfn *p_hwfn,
 
 	/* MCP not initialized */
 	if (!qed_mcp_is_init(p_hwfn)) {
-		DP_NOTICE(p_hwfn, "MFW is not initialized !\n");
+		DP_NOTICE(p_hwfn, "MFW is not initialized!\n");
 		return -EBUSY;
 	}
 
@@ -398,9 +389,36 @@ int qed_mcp_cmd(struct qed_hwfn *p_hwfn,
 	return 0;
 }
 
+int qed_mcp_nvm_rd_cmd(struct qed_hwfn *p_hwfn,
+		       struct qed_ptt *p_ptt,
+		       u32 cmd,
+		       u32 param,
+		       u32 *o_mcp_resp,
+		       u32 *o_mcp_param, u32 *o_txn_size, u32 *o_buf)
+{
+	struct qed_mcp_mb_params mb_params;
+	union drv_union_data union_data;
+	int rc;
+
+	memset(&mb_params, 0, sizeof(mb_params));
+	mb_params.cmd = cmd;
+	mb_params.param = param;
+	mb_params.p_data_dst = &union_data;
+	rc = qed_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params);
+	if (rc)
+		return rc;
+
+	*o_mcp_resp = mb_params.mcp_resp;
+	*o_mcp_param = mb_params.mcp_param;
+
+	*o_txn_size = *o_mcp_param;
+	memcpy(o_buf, &union_data.raw_data, *o_txn_size);
+
+	return 0;
+}
+
 int qed_mcp_load_req(struct qed_hwfn *p_hwfn,
-		     struct qed_ptt *p_ptt,
-		     u32 *p_load_code)
+		     struct qed_ptt *p_ptt, u32 *p_load_code)
 {
 	struct qed_dev *cdev = p_hwfn->cdev;
 	struct qed_mcp_mb_params mb_params;
@@ -527,8 +545,7 @@ static void qed_mcp_handle_transceiver_change(struct qed_hwfn *p_hwfn,
 		   "Received transceiver state update [0x%08x] from mfw [Addr 0x%x]\n",
 		   transceiver_state,
 		   (u32)(p_hwfn->mcp_info->port_addr +
-			 offsetof(struct public_port,
-				  transceiver_data)));
+			  offsetof(struct public_port, transceiver_data)));
 
 	transceiver_state = GET_FIELD(transceiver_state,
 				      ETH_TRANSCEIVER_STATE);
@@ -540,8 +557,7 @@ static void qed_mcp_handle_transceiver_change(struct qed_hwfn *p_hwfn,
 }
 
 static void qed_mcp_handle_link_change(struct qed_hwfn *p_hwfn,
-				       struct qed_ptt *p_ptt,
-				       bool b_reset)
+				       struct qed_ptt *p_ptt, bool b_reset)
 {
 	struct qed_mcp_link_state *p_link;
 	u8 max_bw, min_bw;
@@ -557,8 +573,7 @@ static void qed_mcp_handle_link_change(struct qed_hwfn *p_hwfn,
 			   "Received link update [0x%08x] from mfw [Addr 0x%x]\n",
 			   status,
 			   (u32)(p_hwfn->mcp_info->port_addr +
-				 offsetof(struct public_port,
-					  link_status)));
+				 offsetof(struct public_port, link_status)));
 	} else {
 		DP_VERBOSE(p_hwfn, NETIF_MSG_LINK,
 			   "Resetting link indications\n");
@@ -635,6 +650,9 @@ static void qed_mcp_handle_link_change(struct qed_hwfn *p_hwfn,
 		(status & LINK_STATUS_LINK_PARTNER_20G_CAPABLE) ?
 		QED_LINK_PARTNER_SPEED_20G : 0;
 	p_link->partner_adv_speed |=
+		(status & LINK_STATUS_LINK_PARTNER_25G_CAPABLE) ?
+		QED_LINK_PARTNER_SPEED_25G : 0;
+	p_link->partner_adv_speed |=
 		(status & LINK_STATUS_LINK_PARTNER_40G_CAPABLE) ?
 		QED_LINK_PARTNER_SPEED_40G : 0;
 	p_link->partner_adv_speed |=
@@ -722,6 +740,48 @@ int qed_mcp_set_link(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, bool b_up)
 	return 0;
 }
 
+static void qed_mcp_send_protocol_stats(struct qed_hwfn *p_hwfn,
+					struct qed_ptt *p_ptt,
+					enum MFW_DRV_MSG_TYPE type)
+{
+	enum qed_mcp_protocol_type stats_type;
+	union qed_mcp_protocol_stats stats;
+	struct qed_mcp_mb_params mb_params;
+	union drv_union_data union_data;
+	u32 hsi_param;
+
+	switch (type) {
+	case MFW_DRV_MSG_GET_LAN_STATS:
+		stats_type = QED_MCP_LAN_STATS;
+		hsi_param = DRV_MSG_CODE_STATS_TYPE_LAN;
+		break;
+	case MFW_DRV_MSG_GET_FCOE_STATS:
+		stats_type = QED_MCP_FCOE_STATS;
+		hsi_param = DRV_MSG_CODE_STATS_TYPE_FCOE;
+		break;
+	case MFW_DRV_MSG_GET_ISCSI_STATS:
+		stats_type = QED_MCP_ISCSI_STATS;
+		hsi_param = DRV_MSG_CODE_STATS_TYPE_ISCSI;
+		break;
+	case MFW_DRV_MSG_GET_RDMA_STATS:
+		stats_type = QED_MCP_RDMA_STATS;
+		hsi_param = DRV_MSG_CODE_STATS_TYPE_RDMA;
+		break;
+	default:
+		DP_NOTICE(p_hwfn, "Invalid protocol type %d\n", type);
+		return;
+	}
+
+	qed_get_protocol_stats(p_hwfn->cdev, stats_type, &stats);
+
+	memset(&mb_params, 0, sizeof(mb_params));
+	mb_params.cmd = DRV_MSG_CODE_GET_STATS;
+	mb_params.param = hsi_param;
+	memcpy(&union_data, &stats, sizeof(stats));
+	mb_params.p_data_src = &union_data;
+	qed_mcp_cmd_and_union(p_hwfn, p_ptt, &mb_params);
+}
+
 static void qed_read_pf_bandwidth(struct qed_hwfn *p_hwfn,
 				  struct public_func *p_shmem_info)
 {
@@ -752,8 +812,7 @@ static void qed_read_pf_bandwidth(struct qed_hwfn *p_hwfn,
 
 static u32 qed_mcp_get_shmem_func(struct qed_hwfn *p_hwfn,
 				  struct qed_ptt *p_ptt,
-				  struct public_func *p_data,
-				  int pfid)
+				  struct public_func *p_data, int pfid)
 {
 	u32 addr = SECTION_OFFSIZE_ADDR(p_hwfn->mcp_info->public_base,
 					PUBLIC_FUNC);
@@ -763,51 +822,20 @@ static u32 qed_mcp_get_shmem_func(struct qed_hwfn *p_hwfn,
 
 	memset(p_data, 0, sizeof(*p_data));
 
-	size = min_t(u32, sizeof(*p_data),
-		     QED_SECTION_SIZE(mfw_path_offsize));
+	size = min_t(u32, sizeof(*p_data), QED_SECTION_SIZE(mfw_path_offsize));
 	for (i = 0; i < size / sizeof(u32); i++)
 		((u32 *)p_data)[i] = qed_rd(p_hwfn, p_ptt,
 					    func_addr + (i << 2));
 	return size;
 }
 
-int qed_hw_init_first_eth(struct qed_hwfn *p_hwfn,
-			  struct qed_ptt *p_ptt, u8 *p_pf)
-{
-	struct public_func shmem_info;
-	int i;
-
-	/* Find first Ethernet interface in port */
-	for (i = 0; i < NUM_OF_ENG_PFS(p_hwfn->cdev);
-	     i += p_hwfn->cdev->num_ports_in_engines) {
-		qed_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info,
-				       MCP_PF_ID_BY_REL(p_hwfn, i));
-
-		if (shmem_info.config & FUNC_MF_CFG_FUNC_HIDE)
-			continue;
-
-		if ((shmem_info.config & FUNC_MF_CFG_PROTOCOL_MASK) ==
-		    FUNC_MF_CFG_PROTOCOL_ETHERNET) {
-			*p_pf = (u8)i;
-			return 0;
-		}
-	}
-
-	DP_NOTICE(p_hwfn,
-		  "Failed to find on port an ethernet interface in MF_SI mode\n");
-
-	return -EINVAL;
-}
-
-static void qed_mcp_update_bw(struct qed_hwfn *p_hwfn,
-			      struct qed_ptt *p_ptt)
+static void qed_mcp_update_bw(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	struct qed_mcp_function_info *p_info;
 	struct public_func shmem_info;
 	u32 resp = 0, param = 0;
 
-	qed_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info,
-			       MCP_PF_ID(p_hwfn));
+	qed_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info, MCP_PF_ID(p_hwfn));
 
 	qed_read_pf_bandwidth(p_hwfn, &shmem_info);
 
@@ -867,6 +895,12 @@ int qed_mcp_handle_events(struct qed_hwfn *p_hwfn,
 		case MFW_DRV_MSG_TRANSCEIVER_STATE_CHANGE:
 			qed_mcp_handle_transceiver_change(p_hwfn, p_ptt);
 			break;
+		case MFW_DRV_MSG_GET_LAN_STATS:
+		case MFW_DRV_MSG_GET_FCOE_STATS:
+		case MFW_DRV_MSG_GET_ISCSI_STATS:
+		case MFW_DRV_MSG_GET_RDMA_STATS:
+			qed_mcp_send_protocol_stats(p_hwfn, p_ptt, i);
+			break;
 		case MFW_DRV_MSG_BW_UPDATE:
 			qed_mcp_update_bw(p_hwfn, p_ptt);
 			break;
@@ -940,8 +974,7 @@ int qed_mcp_get_mfw_ver(struct qed_hwfn *p_hwfn,
 	return 0;
 }
 
-int qed_mcp_get_media_type(struct qed_dev *cdev,
-			   u32 *p_media_type)
+int qed_mcp_get_media_type(struct qed_dev *cdev, u32 *p_media_type)
 {
 	struct qed_hwfn *p_hwfn = &cdev->hwfns[0];
 	struct qed_ptt  *p_ptt;
@@ -950,7 +983,7 @@ int qed_mcp_get_media_type(struct qed_dev *cdev,
 		return -EINVAL;
 
 	if (!qed_mcp_is_init(p_hwfn)) {
-		DP_NOTICE(p_hwfn, "MFW is not initialized !\n");
+		DP_NOTICE(p_hwfn, "MFW is not initialized!\n");
 		return -EBUSY;
 	}
 
@@ -1003,15 +1036,13 @@ int qed_mcp_fill_shmem_func_info(struct qed_hwfn *p_hwfn,
 	struct qed_mcp_function_info *info;
 	struct public_func shmem_info;
 
-	qed_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info,
-			       MCP_PF_ID(p_hwfn));
+	qed_mcp_get_shmem_func(p_hwfn, p_ptt, &shmem_info, MCP_PF_ID(p_hwfn));
 	info = &p_hwfn->mcp_info->func_info;
 
 	info->pause_on_host = (shmem_info.config &
 			       FUNC_MF_CFG_PAUSE_ON_HOST_RING) ? 1 : 0;
 
-	if (qed_mcp_get_shmem_proto(p_hwfn, &shmem_info,
-				    &info->protocol)) {
+	if (qed_mcp_get_shmem_proto(p_hwfn, &shmem_info, &info->protocol)) {
 		DP_ERR(p_hwfn, "Unknown personality %08x\n",
 		       (u32)(shmem_info.config & FUNC_MF_CFG_PROTOCOL_MASK));
 		return -EINVAL;
@@ -1072,15 +1103,13 @@ struct qed_mcp_link_capabilities
 	return &p_hwfn->mcp_info->link_capabilities;
 }
 
-int qed_mcp_drain(struct qed_hwfn *p_hwfn,
-		  struct qed_ptt *p_ptt)
+int qed_mcp_drain(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	u32 resp = 0, param = 0;
 	int rc;
 
 	rc = qed_mcp_cmd(p_hwfn, p_ptt,
-			 DRV_MSG_CODE_NIG_DRAIN, 1000,
-			 &resp, &param);
+			 DRV_MSG_CODE_NIG_DRAIN, 1000, &resp, &param);
 
 	/* Wait for the drain to complete before returning */
 	msleep(1020);
@@ -1089,8 +1118,7 @@ int qed_mcp_drain(struct qed_hwfn *p_hwfn,
 }
 
 int qed_mcp_get_flash_size(struct qed_hwfn *p_hwfn,
-			   struct qed_ptt *p_ptt,
-			   u32 *p_flash_size)
+			   struct qed_ptt *p_ptt, u32 *p_flash_size)
 {
 	u32 flash_size;
 
@@ -1153,8 +1181,8 @@ qed_mcp_send_drv_version(struct qed_hwfn *p_hwfn,
 	p_drv_version = &union_data.drv_version;
 	p_drv_version->version = p_ver->version;
 
-	for (i = 0; i < MCP_DRV_VER_STR_SIZE - 1; i += 4) {
-		val = cpu_to_be32(p_ver->name[i]);
+	for (i = 0; i < (MCP_DRV_VER_STR_SIZE - 4) / sizeof(u32); i++) {
+		val = cpu_to_be32(*((u32 *)&p_ver->name[i * sizeof(u32)]));
 		*(__be32 *)&p_drv_version->name[i * sizeof(u32)] = val;
 	}
 
@@ -1168,8 +1196,35 @@ qed_mcp_send_drv_version(struct qed_hwfn *p_hwfn,
 	return rc;
 }
 
-int qed_mcp_set_led(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-		    enum qed_led_mode mode)
+int qed_mcp_halt(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	u32 resp = 0, param = 0;
+	int rc;
+
+	rc = qed_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_MCP_HALT, 0, &resp,
+			 &param);
+	if (rc)
+		DP_ERR(p_hwfn, "MCP response failure, aborting\n");
+
+	return rc;
+}
+
+int qed_mcp_resume(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	u32 value, cpu_mode;
+
+	qed_wr(p_hwfn, p_ptt, MCP_REG_CPU_STATE, 0xffffffff);
+
+	value = qed_rd(p_hwfn, p_ptt, MCP_REG_CPU_MODE);
+	value &= ~MCP_REG_CPU_MODE_SOFT_HALT;
+	qed_wr(p_hwfn, p_ptt, MCP_REG_CPU_MODE, value);
+	cpu_mode = qed_rd(p_hwfn, p_ptt, MCP_REG_CPU_MODE);
+
+	return (cpu_mode & MCP_REG_CPU_MODE_SOFT_HALT) ? -EAGAIN : 0;
+}
+
+int qed_mcp_set_led(struct qed_hwfn *p_hwfn,
+		    struct qed_ptt *p_ptt, enum qed_led_mode mode)
 {
 	u32 resp = 0, param = 0, drv_mb_param;
 	int rc;
@@ -1195,6 +1250,27 @@ int qed_mcp_set_led(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	return rc;
 }
 
+int qed_mcp_mask_parities(struct qed_hwfn *p_hwfn,
+			  struct qed_ptt *p_ptt, u32 mask_parities)
+{
+	u32 resp = 0, param = 0;
+	int rc;
+
+	rc = qed_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_MASK_PARITIES,
+			 mask_parities, &resp, &param);
+
+	if (rc) {
+		DP_ERR(p_hwfn,
+		       "MCP response failure for mask parities, aborting\n");
+	} else if (resp != FW_MSG_CODE_OK) {
+		DP_ERR(p_hwfn,
+		       "MCP did not acknowledge mask parity request. Old MFW?\n");
+		rc = -EINVAL;
+	}
+
+	return rc;
+}
+
 int qed_mcp_bist_register_test(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	u32 drv_mb_param = 0, rsp, param;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.h b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
index 7f319aa1b229..dff520ed069b 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
@@ -60,9 +60,10 @@ struct qed_mcp_link_state {
 #define QED_LINK_PARTNER_SPEED_1G_FD    BIT(1)
 #define QED_LINK_PARTNER_SPEED_10G      BIT(2)
 #define QED_LINK_PARTNER_SPEED_20G      BIT(3)
-#define QED_LINK_PARTNER_SPEED_40G      BIT(4)
-#define QED_LINK_PARTNER_SPEED_50G      BIT(5)
-#define QED_LINK_PARTNER_SPEED_100G     BIT(6)
+#define QED_LINK_PARTNER_SPEED_25G      BIT(4)
+#define QED_LINK_PARTNER_SPEED_40G      BIT(5)
+#define QED_LINK_PARTNER_SPEED_50G      BIT(6)
+#define QED_LINK_PARTNER_SPEED_100G     BIT(7)
 	u32     partner_adv_speed;
 
 	bool    partner_tx_flow_ctrl_en;
@@ -105,6 +106,47 @@ struct qed_mcp_drv_version {
 	u8	name[MCP_DRV_VER_STR_SIZE - 4];
 };
 
+struct qed_mcp_lan_stats {
+	u64 ucast_rx_pkts;
+	u64 ucast_tx_pkts;
+	u32 fcs_err;
+};
+
+struct qed_mcp_fcoe_stats {
+	u64 rx_pkts;
+	u64 tx_pkts;
+	u32 fcs_err;
+	u32 login_failure;
+};
+
+struct qed_mcp_iscsi_stats {
+	u64 rx_pdus;
+	u64 tx_pdus;
+	u64 rx_bytes;
+	u64 tx_bytes;
+};
+
+struct qed_mcp_rdma_stats {
+	u64 rx_pkts;
+	u64 tx_pkts;
+	u64 rx_bytes;
+	u64 tx_byts;
+};
+
+enum qed_mcp_protocol_type {
+	QED_MCP_LAN_STATS,
+	QED_MCP_FCOE_STATS,
+	QED_MCP_ISCSI_STATS,
+	QED_MCP_RDMA_STATS
+};
+
+union qed_mcp_protocol_stats {
+	struct qed_mcp_lan_stats lan_stats;
+	struct qed_mcp_fcoe_stats fcoe_stats;
+	struct qed_mcp_iscsi_stats iscsi_stats;
+	struct qed_mcp_rdma_stats rdma_stats;
+};
+
 /**
  * @brief - returns the link params of the hw function
  *
@@ -426,6 +468,29 @@ int qed_mcp_reset(struct qed_hwfn *p_hwfn,
 		  struct qed_ptt *p_ptt);
 
 /**
+ * @brief - Sends an NVM read command request to the MFW to get
+ *        a buffer.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ * @param cmd - Command: DRV_MSG_CODE_NVM_GET_FILE_DATA or
+ *            DRV_MSG_CODE_NVM_READ_NVRAM commands
+ * @param param - [0:23] - Offset [24:31] - Size
+ * @param o_mcp_resp - MCP response
+ * @param o_mcp_param - MCP response param
+ * @param o_txn_size -  Buffer size output
+ * @param o_buf - Pointer to the buffer returned by the MFW.
+ *
+ * @param return 0 upon success.
+ */
+int qed_mcp_nvm_rd_cmd(struct qed_hwfn *p_hwfn,
+		       struct qed_ptt *p_ptt,
+		       u32 cmd,
+		       u32 param,
+		       u32 *o_mcp_resp,
+		       u32 *o_mcp_param, u32 *o_txn_size, u32 *o_buf);
+
+/**
  * @brief indicates whether the MFW objects [under mcp_info] are accessible
  *
  * @param p_hwfn
@@ -447,6 +512,26 @@ bool qed_mcp_is_init(struct qed_hwfn *p_hwfn);
 int qed_mcp_config_vf_msix(struct qed_hwfn *p_hwfn,
 			   struct qed_ptt *p_ptt, u8 vf_id, u8 num);
 
+/**
+ * @brief - Halt the MCP.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ *
+ * @param return 0 upon success.
+ */
+int qed_mcp_halt(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
+
+/**
+ * @brief - Wake up the MCP.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ *
+ * @param return 0 upon success.
+ */
+int qed_mcp_resume(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
+
 int qed_configure_pf_min_bandwidth(struct qed_dev *cdev, u8 min_bw);
 int qed_configure_pf_max_bandwidth(struct qed_dev *cdev, u8 max_bw);
 int __qed_configure_pf_max_bandwidth(struct qed_hwfn *p_hwfn,
@@ -458,6 +543,7 @@ int __qed_configure_pf_min_bandwidth(struct qed_hwfn *p_hwfn,
 				     struct qed_mcp_link_state *p_link,
 				     u8 min_bw);
 
-int qed_hw_init_first_eth(struct qed_hwfn *p_hwfn,
-			  struct qed_ptt *p_ptt, u8 *p_pf);
+int qed_mcp_mask_parities(struct qed_hwfn *p_hwfn,
+			  struct qed_ptt *p_ptt, u32 mask_parities);
+
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
index f6b86ca1ff79..b414a0542177 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
@@ -116,8 +116,14 @@
 	0x1009c4UL
 #define  QM_REG_PF_EN \
 	0x2f2ea4UL
+#define TCFC_REG_WEAK_ENABLE_VF \
+	0x2d0704UL
 #define  TCFC_REG_STRONG_ENABLE_PF \
 	0x2d0708UL
+#define  TCFC_REG_STRONG_ENABLE_VF \
+	0x2d070cUL
+#define CCFC_REG_WEAK_ENABLE_VF \
+	0x2e0704UL
 #define  CCFC_REG_STRONG_ENABLE_PF \
 	0x2e0708UL
 #define  PGLUE_B_REG_PGL_ADDR_88_F0 \
@@ -202,6 +208,26 @@
 	0x50196cUL
 #define NIG_REG_LLH_CLS_TYPE_DUALMODE \
 	0x501964UL
+#define NIG_REG_LLH_FUNC_FILTER_VALUE \
+	0x501a00UL
+#define NIG_REG_LLH_FUNC_FILTER_VALUE_SIZE \
+	32
+#define NIG_REG_LLH_FUNC_FILTER_EN \
+	0x501a80UL
+#define NIG_REG_LLH_FUNC_FILTER_EN_SIZE	\
+	16
+#define NIG_REG_LLH_FUNC_FILTER_MODE \
+	0x501ac0UL
+#define NIG_REG_LLH_FUNC_FILTER_MODE_SIZE \
+	16
+#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE \
+	0x501b00UL
+#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_SIZE \
+	16
+#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL	\
+	0x501b40UL
+#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_SIZE \
+	16
 #define  NCSI_REG_CONFIG	\
 	0x040200UL
 #define  PBF_REG_INIT \
@@ -258,6 +284,8 @@
 	0x1f0a1cUL
 #define PRS_REG_ROCE_DEST_QP_MAX_PF \
 	0x1f0430UL
+#define PRS_REG_USE_LIGHT_L2 \
+	0x1f096cUL
 #define  PSDM_REG_ENABLE_IN1 \
 	0xfa0004UL
 #define  PSEM_REG_ENABLE_IN \
@@ -521,4 +549,910 @@
 
 #define QM_REG_WFQPFWEIGHT	0x2f4e80UL
 #define QM_REG_WFQVPWEIGHT	0x2fa000UL
+
+#define PGLCS_REG_DBG_SELECT \
+	0x001d14UL
+#define PGLCS_REG_DBG_DWORD_ENABLE \
+	0x001d18UL
+#define PGLCS_REG_DBG_SHIFT \
+	0x001d1cUL
+#define PGLCS_REG_DBG_FORCE_VALID \
+	0x001d20UL
+#define PGLCS_REG_DBG_FORCE_FRAME \
+	0x001d24UL
+#define MISC_REG_RESET_PL_PDA_VMAIN_1 \
+	0x008070UL
+#define MISC_REG_RESET_PL_PDA_VMAIN_2 \
+	0x008080UL
+#define MISC_REG_RESET_PL_PDA_VAUX \
+	0x008090UL
+#define MISCS_REG_RESET_PL_UA \
+	0x009050UL
+#define MISCS_REG_RESET_PL_HV \
+	0x009060UL
+#define MISCS_REG_RESET_PL_HV_2	\
+	0x009150UL
+#define DMAE_REG_DBG_SELECT \
+	0x00c510UL
+#define DMAE_REG_DBG_DWORD_ENABLE \
+	0x00c514UL
+#define DMAE_REG_DBG_SHIFT \
+	0x00c518UL
+#define DMAE_REG_DBG_FORCE_VALID \
+	0x00c51cUL
+#define DMAE_REG_DBG_FORCE_FRAME \
+	0x00c520UL
+#define NCSI_REG_DBG_SELECT \
+	0x040474UL
+#define NCSI_REG_DBG_DWORD_ENABLE \
+	0x040478UL
+#define NCSI_REG_DBG_SHIFT \
+	0x04047cUL
+#define NCSI_REG_DBG_FORCE_VALID \
+	0x040480UL
+#define NCSI_REG_DBG_FORCE_FRAME \
+	0x040484UL
+#define GRC_REG_DBG_SELECT \
+	0x0500a4UL
+#define GRC_REG_DBG_DWORD_ENABLE \
+	0x0500a8UL
+#define GRC_REG_DBG_SHIFT \
+	0x0500acUL
+#define GRC_REG_DBG_FORCE_VALID	\
+	0x0500b0UL
+#define GRC_REG_DBG_FORCE_FRAME	\
+	0x0500b4UL
+#define UMAC_REG_DBG_SELECT \
+	0x051094UL
+#define UMAC_REG_DBG_DWORD_ENABLE \
+	0x051098UL
+#define UMAC_REG_DBG_SHIFT \
+	0x05109cUL
+#define UMAC_REG_DBG_FORCE_VALID \
+	0x0510a0UL
+#define UMAC_REG_DBG_FORCE_FRAME \
+	0x0510a4UL
+#define MCP2_REG_DBG_SELECT \
+	0x052400UL
+#define MCP2_REG_DBG_DWORD_ENABLE \
+	0x052404UL
+#define MCP2_REG_DBG_SHIFT \
+	0x052408UL
+#define MCP2_REG_DBG_FORCE_VALID \
+	0x052440UL
+#define MCP2_REG_DBG_FORCE_FRAME \
+	0x052444UL
+#define PCIE_REG_DBG_SELECT \
+	0x0547e8UL
+#define PCIE_REG_DBG_DWORD_ENABLE \
+	0x0547ecUL
+#define PCIE_REG_DBG_SHIFT \
+	0x0547f0UL
+#define PCIE_REG_DBG_FORCE_VALID \
+	0x0547f4UL
+#define PCIE_REG_DBG_FORCE_FRAME \
+	0x0547f8UL
+#define DORQ_REG_DBG_SELECT \
+	0x100ad0UL
+#define DORQ_REG_DBG_DWORD_ENABLE \
+	0x100ad4UL
+#define DORQ_REG_DBG_SHIFT \
+	0x100ad8UL
+#define DORQ_REG_DBG_FORCE_VALID \
+	0x100adcUL
+#define DORQ_REG_DBG_FORCE_FRAME \
+	0x100ae0UL
+#define IGU_REG_DBG_SELECT \
+	0x181578UL
+#define IGU_REG_DBG_DWORD_ENABLE \
+	0x18157cUL
+#define IGU_REG_DBG_SHIFT \
+	0x181580UL
+#define IGU_REG_DBG_FORCE_VALID	\
+	0x181584UL
+#define IGU_REG_DBG_FORCE_FRAME	\
+	0x181588UL
+#define CAU_REG_DBG_SELECT \
+	0x1c0ea8UL
+#define CAU_REG_DBG_DWORD_ENABLE \
+	0x1c0eacUL
+#define CAU_REG_DBG_SHIFT \
+	0x1c0eb0UL
+#define CAU_REG_DBG_FORCE_VALID	\
+	0x1c0eb4UL
+#define CAU_REG_DBG_FORCE_FRAME	\
+	0x1c0eb8UL
+#define PRS_REG_DBG_SELECT \
+	0x1f0b6cUL
+#define PRS_REG_DBG_DWORD_ENABLE \
+	0x1f0b70UL
+#define PRS_REG_DBG_SHIFT \
+	0x1f0b74UL
+#define PRS_REG_DBG_FORCE_VALID	\
+	0x1f0ba0UL
+#define PRS_REG_DBG_FORCE_FRAME	\
+	0x1f0ba4UL
+#define CNIG_REG_DBG_SELECT_K2 \
+	0x218254UL
+#define CNIG_REG_DBG_DWORD_ENABLE_K2 \
+	0x218258UL
+#define CNIG_REG_DBG_SHIFT_K2 \
+	0x21825cUL
+#define CNIG_REG_DBG_FORCE_VALID_K2 \
+	0x218260UL
+#define CNIG_REG_DBG_FORCE_FRAME_K2 \
+	0x218264UL
+#define PRM_REG_DBG_SELECT \
+	0x2306a8UL
+#define PRM_REG_DBG_DWORD_ENABLE \
+	0x2306acUL
+#define PRM_REG_DBG_SHIFT \
+	0x2306b0UL
+#define PRM_REG_DBG_FORCE_VALID	\
+	0x2306b4UL
+#define PRM_REG_DBG_FORCE_FRAME	\
+	0x2306b8UL
+#define SRC_REG_DBG_SELECT \
+	0x238700UL
+#define SRC_REG_DBG_DWORD_ENABLE \
+	0x238704UL
+#define SRC_REG_DBG_SHIFT \
+	0x238708UL
+#define SRC_REG_DBG_FORCE_VALID	\
+	0x23870cUL
+#define SRC_REG_DBG_FORCE_FRAME	\
+	0x238710UL
+#define RSS_REG_DBG_SELECT \
+	0x238c4cUL
+#define RSS_REG_DBG_DWORD_ENABLE \
+	0x238c50UL
+#define RSS_REG_DBG_SHIFT \
+	0x238c54UL
+#define RSS_REG_DBG_FORCE_VALID	\
+	0x238c58UL
+#define RSS_REG_DBG_FORCE_FRAME	\
+	0x238c5cUL
+#define RPB_REG_DBG_SELECT \
+	0x23c728UL
+#define RPB_REG_DBG_DWORD_ENABLE \
+	0x23c72cUL
+#define RPB_REG_DBG_SHIFT \
+	0x23c730UL
+#define RPB_REG_DBG_FORCE_VALID	\
+	0x23c734UL
+#define RPB_REG_DBG_FORCE_FRAME	\
+	0x23c738UL
+#define PSWRQ2_REG_DBG_SELECT \
+	0x240100UL
+#define PSWRQ2_REG_DBG_DWORD_ENABLE \
+	0x240104UL
+#define PSWRQ2_REG_DBG_SHIFT \
+	0x240108UL
+#define PSWRQ2_REG_DBG_FORCE_VALID \
+	0x24010cUL
+#define PSWRQ2_REG_DBG_FORCE_FRAME \
+	0x240110UL
+#define PSWRQ_REG_DBG_SELECT \
+	0x280020UL
+#define PSWRQ_REG_DBG_DWORD_ENABLE \
+	0x280024UL
+#define PSWRQ_REG_DBG_SHIFT \
+	0x280028UL
+#define PSWRQ_REG_DBG_FORCE_VALID \
+	0x28002cUL
+#define PSWRQ_REG_DBG_FORCE_FRAME \
+	0x280030UL
+#define PSWWR_REG_DBG_SELECT \
+	0x29a084UL
+#define PSWWR_REG_DBG_DWORD_ENABLE \
+	0x29a088UL
+#define PSWWR_REG_DBG_SHIFT \
+	0x29a08cUL
+#define PSWWR_REG_DBG_FORCE_VALID \
+	0x29a090UL
+#define PSWWR_REG_DBG_FORCE_FRAME \
+	0x29a094UL
+#define PSWRD_REG_DBG_SELECT \
+	0x29c040UL
+#define PSWRD_REG_DBG_DWORD_ENABLE \
+	0x29c044UL
+#define PSWRD_REG_DBG_SHIFT \
+	0x29c048UL
+#define PSWRD_REG_DBG_FORCE_VALID \
+	0x29c04cUL
+#define PSWRD_REG_DBG_FORCE_FRAME \
+	0x29c050UL
+#define PSWRD2_REG_DBG_SELECT \
+	0x29d400UL
+#define PSWRD2_REG_DBG_DWORD_ENABLE \
+	0x29d404UL
+#define PSWRD2_REG_DBG_SHIFT \
+	0x29d408UL
+#define PSWRD2_REG_DBG_FORCE_VALID \
+	0x29d40cUL
+#define PSWRD2_REG_DBG_FORCE_FRAME \
+	0x29d410UL
+#define PSWHST2_REG_DBG_SELECT \
+	0x29e058UL
+#define PSWHST2_REG_DBG_DWORD_ENABLE \
+	0x29e05cUL
+#define PSWHST2_REG_DBG_SHIFT \
+	0x29e060UL
+#define PSWHST2_REG_DBG_FORCE_VALID \
+	0x29e064UL
+#define PSWHST2_REG_DBG_FORCE_FRAME \
+	0x29e068UL
+#define PSWHST_REG_DBG_SELECT \
+	0x2a0100UL
+#define PSWHST_REG_DBG_DWORD_ENABLE \
+	0x2a0104UL
+#define PSWHST_REG_DBG_SHIFT \
+	0x2a0108UL
+#define PSWHST_REG_DBG_FORCE_VALID \
+	0x2a010cUL
+#define PSWHST_REG_DBG_FORCE_FRAME \
+	0x2a0110UL
+#define PGLUE_B_REG_DBG_SELECT \
+	0x2a8400UL
+#define PGLUE_B_REG_DBG_DWORD_ENABLE \
+	0x2a8404UL
+#define PGLUE_B_REG_DBG_SHIFT \
+	0x2a8408UL
+#define PGLUE_B_REG_DBG_FORCE_VALID \
+	0x2a840cUL
+#define PGLUE_B_REG_DBG_FORCE_FRAME \
+	0x2a8410UL
+#define TM_REG_DBG_SELECT \
+	0x2c07a8UL
+#define TM_REG_DBG_DWORD_ENABLE	\
+	0x2c07acUL
+#define TM_REG_DBG_SHIFT \
+	0x2c07b0UL
+#define TM_REG_DBG_FORCE_VALID \
+	0x2c07b4UL
+#define TM_REG_DBG_FORCE_FRAME \
+	0x2c07b8UL
+#define TCFC_REG_DBG_SELECT \
+	0x2d0500UL
+#define TCFC_REG_DBG_DWORD_ENABLE \
+	0x2d0504UL
+#define TCFC_REG_DBG_SHIFT \
+	0x2d0508UL
+#define TCFC_REG_DBG_FORCE_VALID \
+	0x2d050cUL
+#define TCFC_REG_DBG_FORCE_FRAME \
+	0x2d0510UL
+#define CCFC_REG_DBG_SELECT \
+	0x2e0500UL
+#define CCFC_REG_DBG_DWORD_ENABLE \
+	0x2e0504UL
+#define CCFC_REG_DBG_SHIFT \
+	0x2e0508UL
+#define CCFC_REG_DBG_FORCE_VALID \
+	0x2e050cUL
+#define CCFC_REG_DBG_FORCE_FRAME \
+	0x2e0510UL
+#define QM_REG_DBG_SELECT \
+	0x2f2e74UL
+#define QM_REG_DBG_DWORD_ENABLE	\
+	0x2f2e78UL
+#define QM_REG_DBG_SHIFT \
+	0x2f2e7cUL
+#define QM_REG_DBG_FORCE_VALID \
+	0x2f2e80UL
+#define QM_REG_DBG_FORCE_FRAME \
+	0x2f2e84UL
+#define RDIF_REG_DBG_SELECT \
+	0x300500UL
+#define RDIF_REG_DBG_DWORD_ENABLE \
+	0x300504UL
+#define RDIF_REG_DBG_SHIFT \
+	0x300508UL
+#define RDIF_REG_DBG_FORCE_VALID \
+	0x30050cUL
+#define RDIF_REG_DBG_FORCE_FRAME \
+	0x300510UL
+#define TDIF_REG_DBG_SELECT \
+	0x310500UL
+#define TDIF_REG_DBG_DWORD_ENABLE \
+	0x310504UL
+#define TDIF_REG_DBG_SHIFT \
+	0x310508UL
+#define TDIF_REG_DBG_FORCE_VALID \
+	0x31050cUL
+#define TDIF_REG_DBG_FORCE_FRAME \
+	0x310510UL
+#define BRB_REG_DBG_SELECT \
+	0x340ed0UL
+#define BRB_REG_DBG_DWORD_ENABLE \
+	0x340ed4UL
+#define BRB_REG_DBG_SHIFT \
+	0x340ed8UL
+#define BRB_REG_DBG_FORCE_VALID	\
+	0x340edcUL
+#define BRB_REG_DBG_FORCE_FRAME	\
+	0x340ee0UL
+#define XYLD_REG_DBG_SELECT \
+	0x4c1600UL
+#define XYLD_REG_DBG_DWORD_ENABLE \
+	0x4c1604UL
+#define XYLD_REG_DBG_SHIFT \
+	0x4c1608UL
+#define XYLD_REG_DBG_FORCE_VALID \
+	0x4c160cUL
+#define XYLD_REG_DBG_FORCE_FRAME \
+	0x4c1610UL
+#define YULD_REG_DBG_SELECT \
+	0x4c9600UL
+#define YULD_REG_DBG_DWORD_ENABLE \
+	0x4c9604UL
+#define YULD_REG_DBG_SHIFT \
+	0x4c9608UL
+#define YULD_REG_DBG_FORCE_VALID \
+	0x4c960cUL
+#define YULD_REG_DBG_FORCE_FRAME \
+	0x4c9610UL
+#define TMLD_REG_DBG_SELECT \
+	0x4d1600UL
+#define TMLD_REG_DBG_DWORD_ENABLE \
+	0x4d1604UL
+#define TMLD_REG_DBG_SHIFT \
+	0x4d1608UL
+#define TMLD_REG_DBG_FORCE_VALID \
+	0x4d160cUL
+#define TMLD_REG_DBG_FORCE_FRAME \
+	0x4d1610UL
+#define MULD_REG_DBG_SELECT \
+	0x4e1600UL
+#define MULD_REG_DBG_DWORD_ENABLE \
+	0x4e1604UL
+#define MULD_REG_DBG_SHIFT \
+	0x4e1608UL
+#define MULD_REG_DBG_FORCE_VALID \
+	0x4e160cUL
+#define MULD_REG_DBG_FORCE_FRAME \
+	0x4e1610UL
+#define NIG_REG_DBG_SELECT \
+	0x502140UL
+#define NIG_REG_DBG_DWORD_ENABLE \
+	0x502144UL
+#define NIG_REG_DBG_SHIFT \
+	0x502148UL
+#define NIG_REG_DBG_FORCE_VALID	\
+	0x50214cUL
+#define NIG_REG_DBG_FORCE_FRAME	\
+	0x502150UL
+#define BMB_REG_DBG_SELECT \
+	0x540a7cUL
+#define BMB_REG_DBG_DWORD_ENABLE \
+	0x540a80UL
+#define BMB_REG_DBG_SHIFT \
+	0x540a84UL
+#define BMB_REG_DBG_FORCE_VALID	\
+	0x540a88UL
+#define BMB_REG_DBG_FORCE_FRAME	\
+	0x540a8cUL
+#define PTU_REG_DBG_SELECT \
+	0x560100UL
+#define PTU_REG_DBG_DWORD_ENABLE \
+	0x560104UL
+#define PTU_REG_DBG_SHIFT \
+	0x560108UL
+#define PTU_REG_DBG_FORCE_VALID	\
+	0x56010cUL
+#define PTU_REG_DBG_FORCE_FRAME	\
+	0x560110UL
+#define CDU_REG_DBG_SELECT \
+	0x580704UL
+#define CDU_REG_DBG_DWORD_ENABLE \
+	0x580708UL
+#define CDU_REG_DBG_SHIFT \
+	0x58070cUL
+#define CDU_REG_DBG_FORCE_VALID	\
+	0x580710UL
+#define CDU_REG_DBG_FORCE_FRAME	\
+	0x580714UL
+#define WOL_REG_DBG_SELECT \
+	0x600140UL
+#define WOL_REG_DBG_DWORD_ENABLE \
+	0x600144UL
+#define WOL_REG_DBG_SHIFT \
+	0x600148UL
+#define WOL_REG_DBG_FORCE_VALID	\
+	0x60014cUL
+#define WOL_REG_DBG_FORCE_FRAME	\
+	0x600150UL
+#define BMBN_REG_DBG_SELECT \
+	0x610140UL
+#define BMBN_REG_DBG_DWORD_ENABLE \
+	0x610144UL
+#define BMBN_REG_DBG_SHIFT \
+	0x610148UL
+#define BMBN_REG_DBG_FORCE_VALID \
+	0x61014cUL
+#define BMBN_REG_DBG_FORCE_FRAME \
+	0x610150UL
+#define NWM_REG_DBG_SELECT \
+	0x8000ecUL
+#define NWM_REG_DBG_DWORD_ENABLE \
+	0x8000f0UL
+#define NWM_REG_DBG_SHIFT \
+	0x8000f4UL
+#define NWM_REG_DBG_FORCE_VALID	\
+	0x8000f8UL
+#define NWM_REG_DBG_FORCE_FRAME	\
+	0x8000fcUL
+#define PBF_REG_DBG_SELECT \
+	0xd80060UL
+#define PBF_REG_DBG_DWORD_ENABLE \
+	0xd80064UL
+#define PBF_REG_DBG_SHIFT \
+	0xd80068UL
+#define PBF_REG_DBG_FORCE_VALID	\
+	0xd8006cUL
+#define PBF_REG_DBG_FORCE_FRAME	\
+	0xd80070UL
+#define PBF_PB1_REG_DBG_SELECT \
+	0xda0728UL
+#define PBF_PB1_REG_DBG_DWORD_ENABLE \
+	0xda072cUL
+#define PBF_PB1_REG_DBG_SHIFT \
+	0xda0730UL
+#define PBF_PB1_REG_DBG_FORCE_VALID \
+	0xda0734UL
+#define PBF_PB1_REG_DBG_FORCE_FRAME \
+	0xda0738UL
+#define PBF_PB2_REG_DBG_SELECT \
+	0xda4728UL
+#define PBF_PB2_REG_DBG_DWORD_ENABLE \
+	0xda472cUL
+#define PBF_PB2_REG_DBG_SHIFT \
+	0xda4730UL
+#define PBF_PB2_REG_DBG_FORCE_VALID \
+	0xda4734UL
+#define PBF_PB2_REG_DBG_FORCE_FRAME \
+	0xda4738UL
+#define BTB_REG_DBG_SELECT \
+	0xdb08c8UL
+#define BTB_REG_DBG_DWORD_ENABLE \
+	0xdb08ccUL
+#define BTB_REG_DBG_SHIFT \
+	0xdb08d0UL
+#define BTB_REG_DBG_FORCE_VALID	\
+	0xdb08d4UL
+#define BTB_REG_DBG_FORCE_FRAME	\
+	0xdb08d8UL
+#define XSDM_REG_DBG_SELECT \
+	0xf80e28UL
+#define XSDM_REG_DBG_DWORD_ENABLE \
+	0xf80e2cUL
+#define XSDM_REG_DBG_SHIFT \
+	0xf80e30UL
+#define XSDM_REG_DBG_FORCE_VALID \
+	0xf80e34UL
+#define XSDM_REG_DBG_FORCE_FRAME \
+	0xf80e38UL
+#define YSDM_REG_DBG_SELECT \
+	0xf90e28UL
+#define YSDM_REG_DBG_DWORD_ENABLE \
+	0xf90e2cUL
+#define YSDM_REG_DBG_SHIFT \
+	0xf90e30UL
+#define YSDM_REG_DBG_FORCE_VALID \
+	0xf90e34UL
+#define YSDM_REG_DBG_FORCE_FRAME \
+	0xf90e38UL
+#define PSDM_REG_DBG_SELECT \
+	0xfa0e28UL
+#define PSDM_REG_DBG_DWORD_ENABLE \
+	0xfa0e2cUL
+#define PSDM_REG_DBG_SHIFT \
+	0xfa0e30UL
+#define PSDM_REG_DBG_FORCE_VALID \
+	0xfa0e34UL
+#define PSDM_REG_DBG_FORCE_FRAME \
+	0xfa0e38UL
+#define TSDM_REG_DBG_SELECT \
+	0xfb0e28UL
+#define TSDM_REG_DBG_DWORD_ENABLE \
+	0xfb0e2cUL
+#define TSDM_REG_DBG_SHIFT \
+	0xfb0e30UL
+#define TSDM_REG_DBG_FORCE_VALID \
+	0xfb0e34UL
+#define TSDM_REG_DBG_FORCE_FRAME \
+	0xfb0e38UL
+#define MSDM_REG_DBG_SELECT \
+	0xfc0e28UL
+#define MSDM_REG_DBG_DWORD_ENABLE \
+	0xfc0e2cUL
+#define MSDM_REG_DBG_SHIFT \
+	0xfc0e30UL
+#define MSDM_REG_DBG_FORCE_VALID \
+	0xfc0e34UL
+#define MSDM_REG_DBG_FORCE_FRAME \
+	0xfc0e38UL
+#define USDM_REG_DBG_SELECT \
+	0xfd0e28UL
+#define USDM_REG_DBG_DWORD_ENABLE \
+	0xfd0e2cUL
+#define USDM_REG_DBG_SHIFT \
+	0xfd0e30UL
+#define USDM_REG_DBG_FORCE_VALID \
+	0xfd0e34UL
+#define USDM_REG_DBG_FORCE_FRAME \
+	0xfd0e38UL
+#define XCM_REG_DBG_SELECT \
+	0x1000040UL
+#define XCM_REG_DBG_DWORD_ENABLE \
+	0x1000044UL
+#define XCM_REG_DBG_SHIFT \
+	0x1000048UL
+#define XCM_REG_DBG_FORCE_VALID	\
+	0x100004cUL
+#define XCM_REG_DBG_FORCE_FRAME	\
+	0x1000050UL
+#define YCM_REG_DBG_SELECT \
+	0x1080040UL
+#define YCM_REG_DBG_DWORD_ENABLE \
+	0x1080044UL
+#define YCM_REG_DBG_SHIFT \
+	0x1080048UL
+#define YCM_REG_DBG_FORCE_VALID	\
+	0x108004cUL
+#define YCM_REG_DBG_FORCE_FRAME	\
+	0x1080050UL
+#define PCM_REG_DBG_SELECT \
+	0x1100040UL
+#define PCM_REG_DBG_DWORD_ENABLE \
+	0x1100044UL
+#define PCM_REG_DBG_SHIFT \
+	0x1100048UL
+#define PCM_REG_DBG_FORCE_VALID	\
+	0x110004cUL
+#define PCM_REG_DBG_FORCE_FRAME	\
+	0x1100050UL
+#define TCM_REG_DBG_SELECT \
+	0x1180040UL
+#define TCM_REG_DBG_DWORD_ENABLE \
+	0x1180044UL
+#define TCM_REG_DBG_SHIFT \
+	0x1180048UL
+#define TCM_REG_DBG_FORCE_VALID	\
+	0x118004cUL
+#define TCM_REG_DBG_FORCE_FRAME	\
+	0x1180050UL
+#define MCM_REG_DBG_SELECT \
+	0x1200040UL
+#define MCM_REG_DBG_DWORD_ENABLE \
+	0x1200044UL
+#define MCM_REG_DBG_SHIFT \
+	0x1200048UL
+#define MCM_REG_DBG_FORCE_VALID	\
+	0x120004cUL
+#define MCM_REG_DBG_FORCE_FRAME	\
+	0x1200050UL
+#define UCM_REG_DBG_SELECT \
+	0x1280050UL
+#define UCM_REG_DBG_DWORD_ENABLE \
+	0x1280054UL
+#define UCM_REG_DBG_SHIFT \
+	0x1280058UL
+#define UCM_REG_DBG_FORCE_VALID	\
+	0x128005cUL
+#define UCM_REG_DBG_FORCE_FRAME	\
+	0x1280060UL
+#define XSEM_REG_DBG_SELECT \
+	0x1401528UL
+#define XSEM_REG_DBG_DWORD_ENABLE \
+	0x140152cUL
+#define XSEM_REG_DBG_SHIFT \
+	0x1401530UL
+#define XSEM_REG_DBG_FORCE_VALID \
+	0x1401534UL
+#define XSEM_REG_DBG_FORCE_FRAME \
+	0x1401538UL
+#define YSEM_REG_DBG_SELECT \
+	0x1501528UL
+#define YSEM_REG_DBG_DWORD_ENABLE \
+	0x150152cUL
+#define YSEM_REG_DBG_SHIFT \
+	0x1501530UL
+#define YSEM_REG_DBG_FORCE_VALID \
+	0x1501534UL
+#define YSEM_REG_DBG_FORCE_FRAME \
+	0x1501538UL
+#define PSEM_REG_DBG_SELECT \
+	0x1601528UL
+#define PSEM_REG_DBG_DWORD_ENABLE \
+	0x160152cUL
+#define PSEM_REG_DBG_SHIFT \
+	0x1601530UL
+#define PSEM_REG_DBG_FORCE_VALID \
+	0x1601534UL
+#define PSEM_REG_DBG_FORCE_FRAME \
+	0x1601538UL
+#define TSEM_REG_DBG_SELECT \
+	0x1701528UL
+#define TSEM_REG_DBG_DWORD_ENABLE \
+	0x170152cUL
+#define TSEM_REG_DBG_SHIFT \
+	0x1701530UL
+#define TSEM_REG_DBG_FORCE_VALID \
+	0x1701534UL
+#define TSEM_REG_DBG_FORCE_FRAME \
+	0x1701538UL
+#define MSEM_REG_DBG_SELECT \
+	0x1801528UL
+#define MSEM_REG_DBG_DWORD_ENABLE \
+	0x180152cUL
+#define MSEM_REG_DBG_SHIFT \
+	0x1801530UL
+#define MSEM_REG_DBG_FORCE_VALID \
+	0x1801534UL
+#define MSEM_REG_DBG_FORCE_FRAME \
+	0x1801538UL
+#define USEM_REG_DBG_SELECT \
+	0x1901528UL
+#define USEM_REG_DBG_DWORD_ENABLE \
+	0x190152cUL
+#define USEM_REG_DBG_SHIFT \
+	0x1901530UL
+#define USEM_REG_DBG_FORCE_VALID \
+	0x1901534UL
+#define USEM_REG_DBG_FORCE_FRAME \
+	0x1901538UL
+#define PCIE_REG_DBG_COMMON_SELECT \
+	0x054398UL
+#define PCIE_REG_DBG_COMMON_DWORD_ENABLE \
+	0x05439cUL
+#define PCIE_REG_DBG_COMMON_SHIFT \
+	0x0543a0UL
+#define PCIE_REG_DBG_COMMON_FORCE_VALID	\
+	0x0543a4UL
+#define PCIE_REG_DBG_COMMON_FORCE_FRAME	\
+	0x0543a8UL
+#define MISC_REG_RESET_PL_UA \
+	0x008050UL
+#define MISC_REG_RESET_PL_HV \
+	0x008060UL
+#define XCM_REG_CTX_RBC_ACCS \
+	0x1001800UL
+#define XCM_REG_AGG_CON_CTX \
+	0x1001804UL
+#define XCM_REG_SM_CON_CTX \
+	0x1001808UL
+#define YCM_REG_CTX_RBC_ACCS \
+	0x1081800UL
+#define YCM_REG_AGG_CON_CTX \
+	0x1081804UL
+#define YCM_REG_AGG_TASK_CTX \
+	0x1081808UL
+#define YCM_REG_SM_CON_CTX \
+	0x108180cUL
+#define YCM_REG_SM_TASK_CTX \
+	0x1081810UL
+#define PCM_REG_CTX_RBC_ACCS \
+	0x1101440UL
+#define PCM_REG_SM_CON_CTX \
+	0x1101444UL
+#define TCM_REG_CTX_RBC_ACCS \
+	0x11814c0UL
+#define TCM_REG_AGG_CON_CTX \
+	0x11814c4UL
+#define TCM_REG_AGG_TASK_CTX \
+	0x11814c8UL
+#define TCM_REG_SM_CON_CTX \
+	0x11814ccUL
+#define TCM_REG_SM_TASK_CTX \
+	0x11814d0UL
+#define MCM_REG_CTX_RBC_ACCS \
+	0x1201800UL
+#define MCM_REG_AGG_CON_CTX \
+	0x1201804UL
+#define MCM_REG_AGG_TASK_CTX \
+	0x1201808UL
+#define MCM_REG_SM_CON_CTX \
+	0x120180cUL
+#define MCM_REG_SM_TASK_CTX \
+	0x1201810UL
+#define UCM_REG_CTX_RBC_ACCS \
+	0x1281700UL
+#define UCM_REG_AGG_CON_CTX \
+	0x1281704UL
+#define UCM_REG_AGG_TASK_CTX \
+	0x1281708UL
+#define UCM_REG_SM_CON_CTX \
+	0x128170cUL
+#define UCM_REG_SM_TASK_CTX \
+	0x1281710UL
+#define XSEM_REG_SLOW_DBG_EMPTY	\
+	0x1401140UL
+#define XSEM_REG_SYNC_DBG_EMPTY	\
+	0x1401160UL
+#define XSEM_REG_SLOW_DBG_ACTIVE \
+	0x1401400UL
+#define XSEM_REG_SLOW_DBG_MODE \
+	0x1401404UL
+#define XSEM_REG_DBG_FRAME_MODE	\
+	0x1401408UL
+#define XSEM_REG_DBG_MODE1_CFG \
+	0x1401420UL
+#define XSEM_REG_FAST_MEMORY \
+	0x1440000UL
+#define YSEM_REG_SYNC_DBG_EMPTY	\
+	0x1501160UL
+#define YSEM_REG_SLOW_DBG_ACTIVE \
+	0x1501400UL
+#define YSEM_REG_SLOW_DBG_MODE \
+	0x1501404UL
+#define YSEM_REG_DBG_FRAME_MODE	\
+	0x1501408UL
+#define YSEM_REG_DBG_MODE1_CFG \
+	0x1501420UL
+#define YSEM_REG_FAST_MEMORY \
+	0x1540000UL
+#define PSEM_REG_SLOW_DBG_EMPTY	\
+	0x1601140UL
+#define PSEM_REG_SYNC_DBG_EMPTY	\
+	0x1601160UL
+#define PSEM_REG_SLOW_DBG_ACTIVE \
+	0x1601400UL
+#define PSEM_REG_SLOW_DBG_MODE \
+	0x1601404UL
+#define PSEM_REG_DBG_FRAME_MODE	\
+	0x1601408UL
+#define PSEM_REG_DBG_MODE1_CFG \
+	0x1601420UL
+#define PSEM_REG_FAST_MEMORY \
+	0x1640000UL
+#define TSEM_REG_SLOW_DBG_EMPTY	\
+	0x1701140UL
+#define TSEM_REG_SYNC_DBG_EMPTY	\
+	0x1701160UL
+#define TSEM_REG_SLOW_DBG_ACTIVE \
+	0x1701400UL
+#define TSEM_REG_SLOW_DBG_MODE \
+	0x1701404UL
+#define TSEM_REG_DBG_FRAME_MODE	\
+	0x1701408UL
+#define TSEM_REG_DBG_MODE1_CFG \
+	0x1701420UL
+#define TSEM_REG_FAST_MEMORY \
+	0x1740000UL
+#define MSEM_REG_SLOW_DBG_EMPTY	\
+	0x1801140UL
+#define MSEM_REG_SYNC_DBG_EMPTY	\
+	0x1801160UL
+#define MSEM_REG_SLOW_DBG_ACTIVE \
+	0x1801400UL
+#define MSEM_REG_SLOW_DBG_MODE \
+	0x1801404UL
+#define MSEM_REG_DBG_FRAME_MODE	\
+	0x1801408UL
+#define MSEM_REG_DBG_MODE1_CFG \
+	0x1801420UL
+#define MSEM_REG_FAST_MEMORY \
+	0x1840000UL
+#define USEM_REG_SLOW_DBG_EMPTY	\
+	0x1901140UL
+#define USEM_REG_SYNC_DBG_EMPTY	\
+	0x1901160UL
+#define USEM_REG_SLOW_DBG_ACTIVE \
+	0x1901400UL
+#define USEM_REG_SLOW_DBG_MODE \
+	0x1901404UL
+#define USEM_REG_DBG_FRAME_MODE	\
+	0x1901408UL
+#define USEM_REG_DBG_MODE1_CFG \
+	0x1901420UL
+#define USEM_REG_FAST_MEMORY \
+	0x1940000UL
+#define SEM_FAST_REG_INT_RAM \
+	0x020000UL
+#define SEM_FAST_REG_INT_RAM_SIZE \
+	20480
+#define GRC_REG_TRACE_FIFO_VALID_DATA \
+	0x050064UL
+#define GRC_REG_NUMBER_VALID_OVERRIDE_WINDOW \
+	0x05040cUL
+#define GRC_REG_PROTECTION_OVERRIDE_WINDOW \
+	0x050500UL
+#define IGU_REG_ERROR_HANDLING_MEMORY \
+	0x181520UL
+#define MCP_REG_CPU_MODE \
+	0xe05000UL
+#define MCP_REG_CPU_MODE_SOFT_HALT \
+		(0x1 << 10)
+#define BRB_REG_BIG_RAM_ADDRESS \
+	0x340800UL
+#define BRB_REG_BIG_RAM_DATA \
+	0x341500UL
+#define SEM_FAST_REG_STALL_0 \
+	0x000488UL
+#define SEM_FAST_REG_STALLED \
+	0x000494UL
+#define BTB_REG_BIG_RAM_ADDRESS \
+	0xdb0800UL
+#define BTB_REG_BIG_RAM_DATA \
+	0xdb0c00UL
+#define BMB_REG_BIG_RAM_ADDRESS \
+	0x540800UL
+#define BMB_REG_BIG_RAM_DATA \
+	0x540f00UL
+#define SEM_FAST_REG_STORM_REG_FILE \
+	0x008000UL
+#define RSS_REG_RSS_RAM_ADDR \
+	0x238c30UL
+#define MISCS_REG_BLOCK_256B_EN \
+	0x009074UL
+#define MCP_REG_SCRATCH_SIZE \
+	57344
+#define MCP_REG_CPU_REG_FILE \
+	0xe05200UL
+#define MCP_REG_CPU_REG_FILE_SIZE \
+	32
+#define DBG_REG_DEBUG_TARGET \
+	0x01005cUL
+#define DBG_REG_FULL_MODE \
+	0x010060UL
+#define DBG_REG_CALENDAR_OUT_DATA \
+	0x010480UL
+#define GRC_REG_TRACE_FIFO \
+	0x050068UL
+#define IGU_REG_ERROR_HANDLING_DATA_VALID \
+	0x181530UL
+#define DBG_REG_DBG_BLOCK_ON \
+	0x010454UL
+#define DBG_REG_FRAMING_MODE \
+	0x010058UL
+#define SEM_FAST_REG_VFC_DATA_WR \
+	0x000b40UL
+#define SEM_FAST_REG_VFC_ADDR \
+	0x000b44UL
+#define SEM_FAST_REG_VFC_DATA_RD \
+	0x000b48UL
+#define RSS_REG_RSS_RAM_DATA \
+	0x238c20UL
+#define MISC_REG_BLOCK_256B_EN \
+	0x008c14UL
+#define NWS_REG_NWS_CMU	\
+	0x720000UL
+#define PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_7_0 \
+	0x000680UL
+#define PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_15_8 \
+	0x000684UL
+#define PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_7_0 \
+	0x0006c0UL
+#define PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_11_8 \
+	0x0006c4UL
+#define MS_REG_MS_CMU \
+	0x6a4000UL
+#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X130 \
+	0x000208UL
+#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X132 \
+	0x000210UL
+#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X131 \
+	0x00020cUL
+#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X133 \
+	0x000214UL
+#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X130 \
+	0x000208UL
+#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X131 \
+	0x00020cUL
+#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X132 \
+	0x000210UL
+#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X133 \
+	0x000214UL
+#define PHY_PCIE_REG_PHY0 \
+	0x620000UL
+#define PHY_PCIE_REG_PHY1 \
+	0x624000UL
+#define NIG_REG_ROCE_DUPLICATE_TO_HOST 0x5088f0UL
+#define PRS_REG_LIGHT_L2_ETHERTYPE_EN 0x1f0968UL
+#define NIG_REG_LLH_ENG_CLS_ENG_ID_TBL 0x501b90UL
+#define DORQ_REG_PF_DPM_ENABLE 0x100510UL
+#define DORQ_REG_PF_ICID_BIT_SHIFT_NORM	0x100448UL
+#define DORQ_REG_PF_MIN_ADDR_REG1 0x100400UL
+#define DORQ_REG_PF_DPI_BIT_SHIFT 0x100450UL
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_roce.c b/drivers/net/ethernet/qlogic/qed/qed_roce.c
new file mode 100644
index 000000000000..23430059471c
--- /dev/null
+++ b/drivers/net/ethernet/qlogic/qed/qed_roce.c
@@ -0,0 +1,2954 @@
+/* QLogic qed NIC Driver
+ * Copyright (c) 2015-2016  QLogic Corporation
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and /or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <linux/types.h>
+#include <asm/byteorder.h>
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/etherdevice.h>
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include <linux/io.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/tcp.h>
+#include <linux/bitops.h>
+#include <linux/qed/qed_roce_if.h>
+#include <linux/qed/qed_roce_if.h>
+#include "qed.h"
+#include "qed_cxt.h"
+#include "qed_hsi.h"
+#include "qed_hw.h"
+#include "qed_init_ops.h"
+#include "qed_int.h"
+#include "qed_ll2.h"
+#include "qed_mcp.h"
+#include "qed_reg_addr.h"
+#include "qed_sp.h"
+#include "qed_roce.h"
+#include "qed_ll2.h"
+
+void qed_async_roce_event(struct qed_hwfn *p_hwfn,
+			  struct event_ring_entry *p_eqe)
+{
+	struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
+
+	p_rdma_info->events.affiliated_event(p_rdma_info->events.context,
+					     p_eqe->opcode, &p_eqe->data);
+}
+
+static int qed_rdma_bmap_alloc(struct qed_hwfn *p_hwfn,
+			       struct qed_bmap *bmap, u32 max_count)
+{
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "max_count = %08x\n", max_count);
+
+	bmap->max_count = max_count;
+
+	bmap->bitmap = kzalloc(BITS_TO_LONGS(max_count) * sizeof(long),
+			       GFP_KERNEL);
+	if (!bmap->bitmap) {
+		DP_NOTICE(p_hwfn,
+			  "qed bmap alloc failed: cannot allocate memory (bitmap)\n");
+		return -ENOMEM;
+	}
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocated bitmap %p\n",
+		   bmap->bitmap);
+	return 0;
+}
+
+static int qed_rdma_bmap_alloc_id(struct qed_hwfn *p_hwfn,
+				  struct qed_bmap *bmap, u32 *id_num)
+{
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "bmap = %p\n", bmap);
+
+	*id_num = find_first_zero_bit(bmap->bitmap, bmap->max_count);
+
+	if (*id_num >= bmap->max_count) {
+		DP_NOTICE(p_hwfn, "no id available max_count=%d\n",
+			  bmap->max_count);
+		return -EINVAL;
+	}
+
+	__set_bit(*id_num, bmap->bitmap);
+
+	return 0;
+}
+
+static void qed_bmap_release_id(struct qed_hwfn *p_hwfn,
+				struct qed_bmap *bmap, u32 id_num)
+{
+	bool b_acquired;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "id_num = %08x", id_num);
+	if (id_num >= bmap->max_count)
+		return;
+
+	b_acquired = test_and_clear_bit(id_num, bmap->bitmap);
+	if (!b_acquired) {
+		DP_NOTICE(p_hwfn, "ID %d already released\n", id_num);
+		return;
+	}
+}
+
+u32 qed_rdma_get_sb_id(void *p_hwfn, u32 rel_sb_id)
+{
+	/* First sb id for RoCE is after all the l2 sb */
+	return FEAT_NUM((struct qed_hwfn *)p_hwfn, QED_PF_L2_QUE) + rel_sb_id;
+}
+
+u32 qed_rdma_query_cau_timer_res(void *rdma_cxt)
+{
+	return QED_CAU_DEF_RX_TIMER_RES;
+}
+
+static int qed_rdma_alloc(struct qed_hwfn *p_hwfn,
+			  struct qed_ptt *p_ptt,
+			  struct qed_rdma_start_in_params *params)
+{
+	struct qed_rdma_info *p_rdma_info;
+	u32 num_cons, num_tasks;
+	int rc = -ENOMEM;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocating RDMA\n");
+
+	/* Allocate a struct with current pf rdma info */
+	p_rdma_info = kzalloc(sizeof(*p_rdma_info), GFP_KERNEL);
+	if (!p_rdma_info) {
+		DP_NOTICE(p_hwfn,
+			  "qed rdma alloc failed: cannot allocate memory (rdma info). rc = %d\n",
+			  rc);
+		return rc;
+	}
+
+	p_hwfn->p_rdma_info = p_rdma_info;
+	p_rdma_info->proto = PROTOCOLID_ROCE;
+
+	num_cons = qed_cxt_get_proto_cid_count(p_hwfn, p_rdma_info->proto, 0);
+
+	p_rdma_info->num_qps = num_cons / 2;
+
+	num_tasks = qed_cxt_get_proto_tid_count(p_hwfn, PROTOCOLID_ROCE);
+
+	/* Each MR uses a single task */
+	p_rdma_info->num_mrs = num_tasks;
+
+	/* Queue zone lines are shared between RoCE and L2 in such a way that
+	 * they can be used by each without obstructing the other.
+	 */
+	p_rdma_info->queue_zone_base = (u16)FEAT_NUM(p_hwfn, QED_L2_QUEUE);
+
+	/* Allocate a struct with device params and fill it */
+	p_rdma_info->dev = kzalloc(sizeof(*p_rdma_info->dev), GFP_KERNEL);
+	if (!p_rdma_info->dev) {
+		DP_NOTICE(p_hwfn,
+			  "qed rdma alloc failed: cannot allocate memory (rdma info dev). rc = %d\n",
+			  rc);
+		goto free_rdma_info;
+	}
+
+	/* Allocate a struct with port params and fill it */
+	p_rdma_info->port = kzalloc(sizeof(*p_rdma_info->port), GFP_KERNEL);
+	if (!p_rdma_info->port) {
+		DP_NOTICE(p_hwfn,
+			  "qed rdma alloc failed: cannot allocate memory (rdma info port). rc = %d\n",
+			  rc);
+		goto free_rdma_dev;
+	}
+
+	/* Allocate bit map for pd's */
+	rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->pd_map, RDMA_MAX_PDS);
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+			   "Failed to allocate pd_map, rc = %d\n",
+			   rc);
+		goto free_rdma_port;
+	}
+
+	/* Allocate DPI bitmap */
+	rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->dpi_map,
+				 p_hwfn->dpi_count);
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+			   "Failed to allocate DPI bitmap, rc = %d\n", rc);
+		goto free_pd_map;
+	}
+
+	/* Allocate bitmap for cq's. The maximum number of CQs is bounded to
+	 * twice the number of QPs.
+	 */
+	rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cq_map,
+				 p_rdma_info->num_qps * 2);
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+			   "Failed to allocate cq bitmap, rc = %d\n", rc);
+		goto free_dpi_map;
+	}
+
+	/* Allocate bitmap for toggle bit for cq icids
+	 * We toggle the bit every time we create or resize cq for a given icid.
+	 * The maximum number of CQs is bounded to  twice the number of QPs.
+	 */
+	rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->toggle_bits,
+				 p_rdma_info->num_qps * 2);
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+			   "Failed to allocate toogle bits, rc = %d\n", rc);
+		goto free_cq_map;
+	}
+
+	/* Allocate bitmap for itids */
+	rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->tid_map,
+				 p_rdma_info->num_mrs);
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+			   "Failed to allocate itids bitmaps, rc = %d\n", rc);
+		goto free_toggle_map;
+	}
+
+	/* Allocate bitmap for cids used for qps. */
+	rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cid_map, num_cons);
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+			   "Failed to allocate cid bitmap, rc = %d\n", rc);
+		goto free_tid_map;
+	}
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocation successful\n");
+	return 0;
+
+free_tid_map:
+	kfree(p_rdma_info->tid_map.bitmap);
+free_toggle_map:
+	kfree(p_rdma_info->toggle_bits.bitmap);
+free_cq_map:
+	kfree(p_rdma_info->cq_map.bitmap);
+free_dpi_map:
+	kfree(p_rdma_info->dpi_map.bitmap);
+free_pd_map:
+	kfree(p_rdma_info->pd_map.bitmap);
+free_rdma_port:
+	kfree(p_rdma_info->port);
+free_rdma_dev:
+	kfree(p_rdma_info->dev);
+free_rdma_info:
+	kfree(p_rdma_info);
+
+	return rc;
+}
+
+void qed_rdma_resc_free(struct qed_hwfn *p_hwfn)
+{
+	struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
+
+	kfree(p_rdma_info->cid_map.bitmap);
+	kfree(p_rdma_info->tid_map.bitmap);
+	kfree(p_rdma_info->toggle_bits.bitmap);
+	kfree(p_rdma_info->cq_map.bitmap);
+	kfree(p_rdma_info->dpi_map.bitmap);
+	kfree(p_rdma_info->pd_map.bitmap);
+
+	kfree(p_rdma_info->port);
+	kfree(p_rdma_info->dev);
+
+	kfree(p_rdma_info);
+}
+
+static void qed_rdma_free(struct qed_hwfn *p_hwfn)
+{
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Freeing RDMA\n");
+
+	qed_rdma_resc_free(p_hwfn);
+}
+
+static void qed_rdma_get_guid(struct qed_hwfn *p_hwfn, u8 *guid)
+{
+	guid[0] = p_hwfn->hw_info.hw_mac_addr[0] ^ 2;
+	guid[1] = p_hwfn->hw_info.hw_mac_addr[1];
+	guid[2] = p_hwfn->hw_info.hw_mac_addr[2];
+	guid[3] = 0xff;
+	guid[4] = 0xfe;
+	guid[5] = p_hwfn->hw_info.hw_mac_addr[3];
+	guid[6] = p_hwfn->hw_info.hw_mac_addr[4];
+	guid[7] = p_hwfn->hw_info.hw_mac_addr[5];
+}
+
+static void qed_rdma_init_events(struct qed_hwfn *p_hwfn,
+				 struct qed_rdma_start_in_params *params)
+{
+	struct qed_rdma_events *events;
+
+	events = &p_hwfn->p_rdma_info->events;
+
+	events->unaffiliated_event = params->events->unaffiliated_event;
+	events->affiliated_event = params->events->affiliated_event;
+	events->context = params->events->context;
+}
+
+static void qed_rdma_init_devinfo(struct qed_hwfn *p_hwfn,
+				  struct qed_rdma_start_in_params *params)
+{
+	struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev;
+	struct qed_dev *cdev = p_hwfn->cdev;
+	u32 pci_status_control;
+	u32 num_qps;
+
+	/* Vendor specific information */
+	dev->vendor_id = cdev->vendor_id;
+	dev->vendor_part_id = cdev->device_id;
+	dev->hw_ver = 0;
+	dev->fw_ver = (FW_MAJOR_VERSION << 24) | (FW_MINOR_VERSION << 16) |
+		      (FW_REVISION_VERSION << 8) | (FW_ENGINEERING_VERSION);
+
+	qed_rdma_get_guid(p_hwfn, (u8 *)&dev->sys_image_guid);
+	dev->node_guid = dev->sys_image_guid;
+
+	dev->max_sge = min_t(u32, RDMA_MAX_SGE_PER_SQ_WQE,
+			     RDMA_MAX_SGE_PER_RQ_WQE);
+
+	if (cdev->rdma_max_sge)
+		dev->max_sge = min_t(u32, cdev->rdma_max_sge, dev->max_sge);
+
+	dev->max_inline = ROCE_REQ_MAX_INLINE_DATA_SIZE;
+
+	dev->max_inline = (cdev->rdma_max_inline) ?
+			  min_t(u32, cdev->rdma_max_inline, dev->max_inline) :
+			  dev->max_inline;
+
+	dev->max_wqe = QED_RDMA_MAX_WQE;
+	dev->max_cnq = (u8)FEAT_NUM(p_hwfn, QED_RDMA_CNQ);
+
+	/* The number of QPs may be higher than QED_ROCE_MAX_QPS, because
+	 * it is up-aligned to 16 and then to ILT page size within qed cxt.
+	 * This is OK in terms of ILT but we don't want to configure the FW
+	 * above its abilities
+	 */
+	num_qps = ROCE_MAX_QPS;
+	num_qps = min_t(u64, num_qps, p_hwfn->p_rdma_info->num_qps);
+	dev->max_qp = num_qps;
+
+	/* CQs uses the same icids that QPs use hence they are limited by the
+	 * number of icids. There are two icids per QP.
+	 */
+	dev->max_cq = num_qps * 2;
+
+	/* The number of mrs is smaller by 1 since the first is reserved */
+	dev->max_mr = p_hwfn->p_rdma_info->num_mrs - 1;
+	dev->max_mr_size = QED_RDMA_MAX_MR_SIZE;
+
+	/* The maximum CQE capacity per CQ supported.
+	 * max number of cqes will be in two layer pbl,
+	 * 8 is the pointer size in bytes
+	 * 32 is the size of cq element in bytes
+	 */
+	if (params->cq_mode == QED_RDMA_CQ_MODE_32_BITS)
+		dev->max_cqe = QED_RDMA_MAX_CQE_32_BIT;
+	else
+		dev->max_cqe = QED_RDMA_MAX_CQE_16_BIT;
+
+	dev->max_mw = 0;
+	dev->max_fmr = QED_RDMA_MAX_FMR;
+	dev->max_mr_mw_fmr_pbl = (PAGE_SIZE / 8) * (PAGE_SIZE / 8);
+	dev->max_mr_mw_fmr_size = dev->max_mr_mw_fmr_pbl * PAGE_SIZE;
+	dev->max_pkey = QED_RDMA_MAX_P_KEY;
+
+	dev->max_qp_resp_rd_atomic_resc = RDMA_RING_PAGE_SIZE /
+					  (RDMA_RESP_RD_ATOMIC_ELM_SIZE * 2);
+	dev->max_qp_req_rd_atomic_resc = RDMA_RING_PAGE_SIZE /
+					 RDMA_REQ_RD_ATOMIC_ELM_SIZE;
+	dev->max_dev_resp_rd_atomic_resc = dev->max_qp_resp_rd_atomic_resc *
+					   p_hwfn->p_rdma_info->num_qps;
+	dev->page_size_caps = QED_RDMA_PAGE_SIZE_CAPS;
+	dev->dev_ack_delay = QED_RDMA_ACK_DELAY;
+	dev->max_pd = RDMA_MAX_PDS;
+	dev->max_ah = p_hwfn->p_rdma_info->num_qps;
+	dev->max_stats_queues = (u8)RESC_NUM(p_hwfn, QED_RDMA_STATS_QUEUE);
+
+	/* Set capablities */
+	dev->dev_caps = 0;
+	SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RNR_NAK, 1);
+	SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_ACTIVE_EVENT, 1);
+	SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_CHANGE_EVENT, 1);
+	SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RESIZE_CQ, 1);
+	SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_MEMORY_EXT, 1);
+	SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_QUEUE_EXT, 1);
+	SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ZBVA, 1);
+	SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_LOCAL_INV_FENCE, 1);
+
+	/* Check atomic operations support in PCI configuration space. */
+	pci_read_config_dword(cdev->pdev,
+			      cdev->pdev->pcie_cap + PCI_EXP_DEVCTL2,
+			      &pci_status_control);
+
+	if (pci_status_control & PCI_EXP_DEVCTL2_LTR_EN)
+		SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ATOMIC_OP, 1);
+}
+
+static void qed_rdma_init_port(struct qed_hwfn *p_hwfn)
+{
+	struct qed_rdma_port *port = p_hwfn->p_rdma_info->port;
+	struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev;
+
+	port->port_state = p_hwfn->mcp_info->link_output.link_up ?
+			   QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN;
+
+	port->max_msg_size = min_t(u64,
+				   (dev->max_mr_mw_fmr_size *
+				    p_hwfn->cdev->rdma_max_sge),
+				   BIT(31));
+
+	port->pkey_bad_counter = 0;
+}
+
+static int qed_rdma_init_hw(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	u32 ll2_ethertype_en;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW\n");
+	p_hwfn->b_rdma_enabled_in_prs = false;
+
+	qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0);
+
+	p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_ROCE;
+
+	/* We delay writing to this reg until first cid is allocated. See
+	 * qed_cxt_dynamic_ilt_alloc function for more details
+	 */
+	ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN);
+	qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN,
+	       (ll2_ethertype_en | 0x01));
+
+	if (qed_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_ROCE) % 2) {
+		DP_NOTICE(p_hwfn, "The first RoCE's cid should be even\n");
+		return -EINVAL;
+	}
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW - Done\n");
+	return 0;
+}
+
+static int qed_rdma_start_fw(struct qed_hwfn *p_hwfn,
+			     struct qed_rdma_start_in_params *params,
+			     struct qed_ptt *p_ptt)
+{
+	struct rdma_init_func_ramrod_data *p_ramrod;
+	struct qed_rdma_cnq_params *p_cnq_pbl_list;
+	struct rdma_init_func_hdr *p_params_header;
+	struct rdma_cnq_params *p_cnq_params;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	u32 cnq_id, sb_id;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Starting FW\n");
+
+	/* Save the number of cnqs for the function close ramrod */
+	p_hwfn->p_rdma_info->num_cnqs = params->desired_cnq;
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_INIT,
+				 p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.roce_init_func.rdma;
+
+	p_params_header = &p_ramrod->params_header;
+	p_params_header->cnq_start_offset = (u8)RESC_START(p_hwfn,
+							   QED_RDMA_CNQ_RAM);
+	p_params_header->num_cnqs = params->desired_cnq;
+
+	if (params->cq_mode == QED_RDMA_CQ_MODE_16_BITS)
+		p_params_header->cq_ring_mode = 1;
+	else
+		p_params_header->cq_ring_mode = 0;
+
+	for (cnq_id = 0; cnq_id < params->desired_cnq; cnq_id++) {
+		sb_id = qed_rdma_get_sb_id(p_hwfn, cnq_id);
+		p_cnq_params = &p_ramrod->cnq_params[cnq_id];
+		p_cnq_pbl_list = &params->cnq_pbl_list[cnq_id];
+		p_cnq_params->sb_num =
+			cpu_to_le16(p_hwfn->sbs_info[sb_id]->igu_sb_id);
+
+		p_cnq_params->sb_index = p_hwfn->pf_params.rdma_pf_params.gl_pi;
+		p_cnq_params->num_pbl_pages = p_cnq_pbl_list->num_pbl_pages;
+
+		DMA_REGPAIR_LE(p_cnq_params->pbl_base_addr,
+			       p_cnq_pbl_list->pbl_ptr);
+
+		/* we assume here that cnq_id and qz_offset are the same */
+		p_cnq_params->queue_zone_num =
+			cpu_to_le16(p_hwfn->p_rdma_info->queue_zone_base +
+				    cnq_id);
+	}
+
+	return qed_spq_post(p_hwfn, p_ent, NULL);
+}
+
+static int qed_rdma_reserve_lkey(struct qed_hwfn *p_hwfn)
+{
+	struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev;
+
+	/* The first DPI is reserved for the Kernel */
+	__set_bit(0, p_hwfn->p_rdma_info->dpi_map.bitmap);
+
+	/* Tid 0 will be used as the key for "reserved MR".
+	 * The driver should allocate memory for it so it can be loaded but no
+	 * ramrod should be passed on it.
+	 */
+	qed_rdma_alloc_tid(p_hwfn, &dev->reserved_lkey);
+	if (dev->reserved_lkey != RDMA_RESERVED_LKEY) {
+		DP_NOTICE(p_hwfn,
+			  "Reserved lkey should be equal to RDMA_RESERVED_LKEY\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int qed_rdma_setup(struct qed_hwfn *p_hwfn,
+			  struct qed_ptt *p_ptt,
+			  struct qed_rdma_start_in_params *params)
+{
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA setup\n");
+
+	spin_lock_init(&p_hwfn->p_rdma_info->lock);
+
+	qed_rdma_init_devinfo(p_hwfn, params);
+	qed_rdma_init_port(p_hwfn);
+	qed_rdma_init_events(p_hwfn, params);
+
+	rc = qed_rdma_reserve_lkey(p_hwfn);
+	if (rc)
+		return rc;
+
+	rc = qed_rdma_init_hw(p_hwfn, p_ptt);
+	if (rc)
+		return rc;
+
+	return qed_rdma_start_fw(p_hwfn, params, p_ptt);
+}
+
+int qed_rdma_stop(void *rdma_cxt)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct rdma_close_func_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	struct qed_ptt *p_ptt;
+	u32 ll2_ethertype_en;
+	int rc = -EBUSY;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop\n");
+
+	p_ptt = qed_ptt_acquire(p_hwfn);
+	if (!p_ptt) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Failed to acquire PTT\n");
+		return rc;
+	}
+
+	/* Disable RoCE search */
+	qed_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 0);
+	p_hwfn->b_rdma_enabled_in_prs = false;
+
+	qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0);
+
+	ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN);
+
+	qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN,
+	       (ll2_ethertype_en & 0xFFFE));
+
+	qed_ptt_release(p_hwfn, p_ptt);
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	/* Stop RoCE */
+	rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_CLOSE,
+				 p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc)
+		goto out;
+
+	p_ramrod = &p_ent->ramrod.rdma_close_func;
+
+	p_ramrod->num_cnqs = p_hwfn->p_rdma_info->num_cnqs;
+	p_ramrod->cnq_start_offset = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM);
+
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+
+out:
+	qed_rdma_free(p_hwfn);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop done, rc = %d\n", rc);
+	return rc;
+}
+
+int qed_rdma_add_user(void *rdma_cxt,
+		      struct qed_rdma_add_user_out_params *out_params)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	u32 dpi_start_offset;
+	u32 returned_id = 0;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding User\n");
+
+	/* Allocate DPI */
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+	rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map,
+				    &returned_id);
+	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
+
+	out_params->dpi = (u16)returned_id;
+
+	/* Calculate the corresponding DPI address */
+	dpi_start_offset = p_hwfn->dpi_start_offset;
+
+	out_params->dpi_addr = (u64)((u8 __iomem *)p_hwfn->doorbells +
+				     dpi_start_offset +
+				     ((out_params->dpi) * p_hwfn->dpi_size));
+
+	out_params->dpi_phys_addr = p_hwfn->cdev->db_phys_addr +
+				    dpi_start_offset +
+				    ((out_params->dpi) * p_hwfn->dpi_size);
+
+	out_params->dpi_size = p_hwfn->dpi_size;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding user - done, rc = %d\n", rc);
+	return rc;
+}
+
+struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct qed_rdma_port *p_port = p_hwfn->p_rdma_info->port;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA Query port\n");
+
+	/* Link may have changed */
+	p_port->port_state = p_hwfn->mcp_info->link_output.link_up ?
+			     QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN;
+
+	p_port->link_speed = p_hwfn->mcp_info->link_output.speed;
+
+	return p_port;
+}
+
+struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query device\n");
+
+	/* Return struct with device parameters */
+	return p_hwfn->p_rdma_info->dev;
+}
+
+void qed_rdma_free_tid(void *rdma_cxt, u32 itid)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid);
+
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+	qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tid_map, itid);
+	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
+}
+
+int qed_rdma_alloc_tid(void *rdma_cxt, u32 *itid)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID\n");
+
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+	rc = qed_rdma_bmap_alloc_id(p_hwfn,
+				    &p_hwfn->p_rdma_info->tid_map, itid);
+	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
+	if (rc)
+		goto out;
+
+	rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_TASK, *itid);
+out:
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID - done, rc = %d\n", rc);
+	return rc;
+}
+
+void qed_rdma_cnq_prod_update(void *rdma_cxt, u8 qz_offset, u16 prod)
+{
+	struct qed_hwfn *p_hwfn;
+	u16 qz_num;
+	u32 addr;
+
+	p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	qz_num = p_hwfn->p_rdma_info->queue_zone_base + qz_offset;
+	addr = GTT_BAR0_MAP_REG_USDM_RAM +
+	       USTORM_COMMON_QUEUE_CONS_OFFSET(qz_num);
+
+	REG_WR16(p_hwfn, addr, prod);
+
+	/* keep prod updates ordered */
+	wmb();
+}
+
+static int qed_fill_rdma_dev_info(struct qed_dev *cdev,
+				  struct qed_dev_rdma_info *info)
+{
+	memset(info, 0, sizeof(*info));
+
+	info->rdma_type = QED_RDMA_TYPE_ROCE;
+
+	qed_fill_dev_info(cdev, &info->common);
+
+	return 0;
+}
+
+static int qed_rdma_get_sb_start(struct qed_dev *cdev)
+{
+	int feat_num;
+
+	if (cdev->num_hwfns > 1)
+		feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE);
+	else
+		feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE) *
+			   cdev->num_hwfns;
+
+	return feat_num;
+}
+
+static int qed_rdma_get_min_cnq_msix(struct qed_dev *cdev)
+{
+	int n_cnq = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_RDMA_CNQ);
+	int n_msix = cdev->int_params.rdma_msix_cnt;
+
+	return min_t(int, n_cnq, n_msix);
+}
+
+static int qed_rdma_set_int(struct qed_dev *cdev, u16 cnt)
+{
+	int limit = 0;
+
+	/* Mark the fastpath as free/used */
+	cdev->int_params.fp_initialized = cnt ? true : false;
+
+	if (cdev->int_params.out.int_mode != QED_INT_MODE_MSIX) {
+		DP_ERR(cdev,
+		       "qed roce supports only MSI-X interrupts (detected %d).\n",
+		       cdev->int_params.out.int_mode);
+		return -EINVAL;
+	} else if (cdev->int_params.fp_msix_cnt) {
+		limit = cdev->int_params.rdma_msix_cnt;
+	}
+
+	if (!limit)
+		return -ENOMEM;
+
+	return min_t(int, cnt, limit);
+}
+
+static int qed_rdma_get_int(struct qed_dev *cdev, struct qed_int_info *info)
+{
+	memset(info, 0, sizeof(*info));
+
+	if (!cdev->int_params.fp_initialized) {
+		DP_INFO(cdev,
+			"Protocol driver requested interrupt information, but its support is not yet configured\n");
+		return -EINVAL;
+	}
+
+	if (cdev->int_params.out.int_mode == QED_INT_MODE_MSIX) {
+		int msix_base = cdev->int_params.rdma_msix_base;
+
+		info->msix_cnt = cdev->int_params.rdma_msix_cnt;
+		info->msix = &cdev->int_params.msix_table[msix_base];
+
+		DP_VERBOSE(cdev, QED_MSG_RDMA, "msix_cnt = %d msix_base=%d\n",
+			   info->msix_cnt, msix_base);
+	}
+
+	return 0;
+}
+
+int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	u32 returned_id;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD\n");
+
+	/* Allocates an unused protection domain */
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+	rc = qed_rdma_bmap_alloc_id(p_hwfn,
+				    &p_hwfn->p_rdma_info->pd_map, &returned_id);
+	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
+
+	*pd = (u16)returned_id;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD - done, rc = %d\n", rc);
+	return rc;
+}
+
+void qed_rdma_free_pd(void *rdma_cxt, u16 pd)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "pd = %08x\n", pd);
+
+	/* Returns a previously allocated protection domain for reuse */
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+	qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->pd_map, pd);
+	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
+}
+
+static enum qed_rdma_toggle_bit
+qed_rdma_toggle_bit_create_resize_cq(struct qed_hwfn *p_hwfn, u16 icid)
+{
+	struct qed_rdma_info *p_info = p_hwfn->p_rdma_info;
+	enum qed_rdma_toggle_bit toggle_bit;
+	u32 bmap_id;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", icid);
+
+	/* the function toggle the bit that is related to a given icid
+	 * and returns the new toggle bit's value
+	 */
+	bmap_id = icid - qed_cxt_get_proto_cid_start(p_hwfn, p_info->proto);
+
+	spin_lock_bh(&p_info->lock);
+	toggle_bit = !test_and_change_bit(bmap_id,
+					  p_info->toggle_bits.bitmap);
+	spin_unlock_bh(&p_info->lock);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QED_RDMA_TOGGLE_BIT_= %d\n",
+		   toggle_bit);
+
+	return toggle_bit;
+}
+
+int qed_rdma_create_cq(void *rdma_cxt,
+		       struct qed_rdma_create_cq_in_params *params, u16 *icid)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct qed_rdma_info *p_info = p_hwfn->p_rdma_info;
+	struct rdma_create_cq_ramrod_data *p_ramrod;
+	enum qed_rdma_toggle_bit toggle_bit;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	u32 returned_id, start_cid;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "cq_handle = %08x%08x\n",
+		   params->cq_handle_hi, params->cq_handle_lo);
+
+	/* Allocate icid */
+	spin_lock_bh(&p_info->lock);
+	rc = qed_rdma_bmap_alloc_id(p_hwfn,
+				    &p_info->cq_map, &returned_id);
+	spin_unlock_bh(&p_info->lock);
+
+	if (rc) {
+		DP_NOTICE(p_hwfn, "Can't create CQ, rc = %d\n", rc);
+		return rc;
+	}
+
+	start_cid = qed_cxt_get_proto_cid_start(p_hwfn,
+						p_info->proto);
+	*icid = returned_id + start_cid;
+
+	/* Check if icid requires a page allocation */
+	rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, *icid);
+	if (rc)
+		goto err;
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = *icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	/* Send create CQ ramrod */
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 RDMA_RAMROD_CREATE_CQ,
+				 p_info->proto, &init_data);
+	if (rc)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.rdma_create_cq;
+
+	p_ramrod->cq_handle.hi = cpu_to_le32(params->cq_handle_hi);
+	p_ramrod->cq_handle.lo = cpu_to_le32(params->cq_handle_lo);
+	p_ramrod->dpi = cpu_to_le16(params->dpi);
+	p_ramrod->is_two_level_pbl = params->pbl_two_level;
+	p_ramrod->max_cqes = cpu_to_le32(params->cq_size);
+	DMA_REGPAIR_LE(p_ramrod->pbl_addr, params->pbl_ptr);
+	p_ramrod->pbl_num_pages = cpu_to_le16(params->pbl_num_pages);
+	p_ramrod->cnq_id = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM) +
+			   params->cnq_id;
+	p_ramrod->int_timeout = params->int_timeout;
+
+	/* toggle the bit for every resize or create cq for a given icid */
+	toggle_bit = qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid);
+
+	p_ramrod->toggle_bit = toggle_bit;
+
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+	if (rc) {
+		/* restore toggle bit */
+		qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid);
+		goto err;
+	}
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Created CQ, rc = %d\n", rc);
+	return rc;
+
+err:
+	/* release allocated icid */
+	qed_bmap_release_id(p_hwfn, &p_info->cq_map, returned_id);
+	DP_NOTICE(p_hwfn, "Create CQ failed, rc = %d\n", rc);
+
+	return rc;
+}
+
+int qed_rdma_resize_cq(void *rdma_cxt,
+		       struct qed_rdma_resize_cq_in_params *in_params,
+		       struct qed_rdma_resize_cq_out_params *out_params)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct rdma_resize_cq_output_params *p_ramrod_res;
+	struct rdma_resize_cq_ramrod_data *p_ramrod;
+	enum qed_rdma_toggle_bit toggle_bit;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	dma_addr_t ramrod_res_phys;
+	u8 fw_return_code;
+	int rc = -ENOMEM;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", in_params->icid);
+
+	p_ramrod_res =
+	    (struct rdma_resize_cq_output_params *)
+	    dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+			       sizeof(struct rdma_resize_cq_output_params),
+			       &ramrod_res_phys, GFP_KERNEL);
+	if (!p_ramrod_res) {
+		DP_NOTICE(p_hwfn,
+			  "qed resize cq failed: cannot allocate memory (ramrod)\n");
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = in_params->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 RDMA_RAMROD_RESIZE_CQ,
+				 p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.rdma_resize_cq;
+
+	p_ramrod->flags = 0;
+
+	/* toggle the bit for every resize or create cq for a given icid */
+	toggle_bit = qed_rdma_toggle_bit_create_resize_cq(p_hwfn,
+							  in_params->icid);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT, toggle_bit);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL,
+		  in_params->pbl_two_level);
+
+	p_ramrod->pbl_log_page_size = in_params->pbl_page_size_log - 12;
+	p_ramrod->pbl_num_pages = cpu_to_le16(in_params->pbl_num_pages);
+	p_ramrod->max_cqes = cpu_to_le32(in_params->cq_size);
+	DMA_REGPAIR_LE(p_ramrod->pbl_addr, in_params->pbl_ptr);
+	DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
+
+	rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code);
+	if (rc)
+		goto err;
+
+	if (fw_return_code != RDMA_RETURN_OK) {
+		DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code);
+		rc = -EINVAL;
+		goto err;
+	}
+
+	out_params->prod = le32_to_cpu(p_ramrod_res->old_cq_prod);
+	out_params->cons = le32_to_cpu(p_ramrod_res->old_cq_cons);
+
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+			  sizeof(struct rdma_resize_cq_output_params),
+			  p_ramrod_res, ramrod_res_phys);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Resized CQ, rc = %d\n", rc);
+
+	return rc;
+
+err:	dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+			  sizeof(struct rdma_resize_cq_output_params),
+			  p_ramrod_res, ramrod_res_phys);
+	DP_NOTICE(p_hwfn, "Resized CQ, Failed - rc = %d\n", rc);
+
+	return rc;
+}
+
+int qed_rdma_destroy_cq(void *rdma_cxt,
+			struct qed_rdma_destroy_cq_in_params *in_params,
+			struct qed_rdma_destroy_cq_out_params *out_params)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct rdma_destroy_cq_output_params *p_ramrod_res;
+	struct rdma_destroy_cq_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	dma_addr_t ramrod_res_phys;
+	int rc = -ENOMEM;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", in_params->icid);
+
+	p_ramrod_res =
+	    (struct rdma_destroy_cq_output_params *)
+	    dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+			       sizeof(struct rdma_destroy_cq_output_params),
+			       &ramrod_res_phys, GFP_KERNEL);
+	if (!p_ramrod_res) {
+		DP_NOTICE(p_hwfn,
+			  "qed destroy cq failed: cannot allocate memory (ramrod)\n");
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = in_params->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	/* Send destroy CQ ramrod */
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 RDMA_RAMROD_DESTROY_CQ,
+				 p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.rdma_destroy_cq;
+	DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
+
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+	if (rc)
+		goto err;
+
+	out_params->num_cq_notif = le16_to_cpu(p_ramrod_res->cnq_num);
+
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+			  sizeof(struct rdma_destroy_cq_output_params),
+			  p_ramrod_res, ramrod_res_phys);
+
+	/* Free icid */
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+
+	qed_bmap_release_id(p_hwfn,
+			    &p_hwfn->p_rdma_info->cq_map,
+			    (in_params->icid -
+			     qed_cxt_get_proto_cid_start(p_hwfn,
+							 p_hwfn->
+							 p_rdma_info->proto)));
+
+	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroyed CQ, rc = %d\n", rc);
+	return rc;
+
+err:	dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+			  sizeof(struct rdma_destroy_cq_output_params),
+			  p_ramrod_res, ramrod_res_phys);
+
+	return rc;
+}
+
+static void qed_rdma_set_fw_mac(u16 *p_fw_mac, u8 *p_qed_mac)
+{
+	p_fw_mac[0] = cpu_to_le16((p_qed_mac[0] << 8) + p_qed_mac[1]);
+	p_fw_mac[1] = cpu_to_le16((p_qed_mac[2] << 8) + p_qed_mac[3]);
+	p_fw_mac[2] = cpu_to_le16((p_qed_mac[4] << 8) + p_qed_mac[5]);
+}
+
+static void qed_rdma_copy_gids(struct qed_rdma_qp *qp, __le32 *src_gid,
+			       __le32 *dst_gid)
+{
+	u32 i;
+
+	if (qp->roce_mode == ROCE_V2_IPV4) {
+		/* The IPv4 addresses shall be aligned to the highest word.
+		 * The lower words must be zero.
+		 */
+		memset(src_gid, 0, sizeof(union qed_gid));
+		memset(dst_gid, 0, sizeof(union qed_gid));
+		src_gid[3] = cpu_to_le32(qp->sgid.ipv4_addr);
+		dst_gid[3] = cpu_to_le32(qp->dgid.ipv4_addr);
+	} else {
+		/* GIDs and IPv6 addresses coincide in location and size */
+		for (i = 0; i < ARRAY_SIZE(qp->sgid.dwords); i++) {
+			src_gid[i] = cpu_to_le32(qp->sgid.dwords[i]);
+			dst_gid[i] = cpu_to_le32(qp->dgid.dwords[i]);
+		}
+	}
+}
+
+static enum roce_flavor qed_roce_mode_to_flavor(enum roce_mode roce_mode)
+{
+	enum roce_flavor flavor;
+
+	switch (roce_mode) {
+	case ROCE_V1:
+		flavor = PLAIN_ROCE;
+		break;
+	case ROCE_V2_IPV4:
+		flavor = RROCE_IPV4;
+		break;
+	case ROCE_V2_IPV6:
+		flavor = ROCE_V2_IPV6;
+		break;
+	default:
+		flavor = MAX_ROCE_MODE;
+		break;
+	}
+	return flavor;
+}
+
+int qed_roce_alloc_cid(struct qed_hwfn *p_hwfn, u16 *cid)
+{
+	struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
+	u32 responder_icid;
+	u32 requester_icid;
+	int rc;
+
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+	rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map,
+				    &responder_icid);
+	if (rc) {
+		spin_unlock_bh(&p_rdma_info->lock);
+		return rc;
+	}
+
+	rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map,
+				    &requester_icid);
+
+	spin_unlock_bh(&p_rdma_info->lock);
+	if (rc)
+		goto err;
+
+	/* the two icid's should be adjacent */
+	if ((requester_icid - responder_icid) != 1) {
+		DP_NOTICE(p_hwfn, "Failed to allocate two adjacent qp's'\n");
+		rc = -EINVAL;
+		goto err;
+	}
+
+	responder_icid += qed_cxt_get_proto_cid_start(p_hwfn,
+						      p_rdma_info->proto);
+	requester_icid += qed_cxt_get_proto_cid_start(p_hwfn,
+						      p_rdma_info->proto);
+
+	/* If these icids require a new ILT line allocate DMA-able context for
+	 * an ILT page
+	 */
+	rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, responder_icid);
+	if (rc)
+		goto err;
+
+	rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, requester_icid);
+	if (rc)
+		goto err;
+
+	*cid = (u16)responder_icid;
+	return rc;
+
+err:
+	spin_lock_bh(&p_rdma_info->lock);
+	qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, responder_icid);
+	qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, requester_icid);
+
+	spin_unlock_bh(&p_rdma_info->lock);
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+		   "Allocate CID - failed, rc = %d\n", rc);
+	return rc;
+}
+
+static int qed_roce_sp_create_responder(struct qed_hwfn *p_hwfn,
+					struct qed_rdma_qp *qp)
+{
+	struct roce_create_qp_resp_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	union qed_qm_pq_params qm_params;
+	enum roce_flavor roce_flavor;
+	struct qed_spq_entry *p_ent;
+	u16 physical_queue0 = 0;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	/* Allocate DMA-able memory for IRQ */
+	qp->irq_num_pages = 1;
+	qp->irq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+				     RDMA_RING_PAGE_SIZE,
+				     &qp->irq_phys_addr, GFP_KERNEL);
+	if (!qp->irq) {
+		rc = -ENOMEM;
+		DP_NOTICE(p_hwfn,
+			  "qed create responder failed: cannot allocate memory (irq). rc = %d\n",
+			  rc);
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_CREATE_QP,
+				 PROTOCOLID_ROCE, &init_data);
+	if (rc)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.roce_create_qp_resp;
+
+	p_ramrod->flags = 0;
+
+	roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode);
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR, roce_flavor);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN,
+		  qp->incoming_rdma_read_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN,
+		  qp->incoming_rdma_write_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN,
+		  qp->incoming_atomic_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN,
+		  qp->e2e_flow_control_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG, qp->use_srq);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN,
+		  qp->fmr_and_reserved_lkey);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER,
+		  qp->min_rnr_nak_timer);
+
+	p_ramrod->max_ird = qp->max_rd_atomic_resp;
+	p_ramrod->traffic_class = qp->traffic_class_tos;
+	p_ramrod->hop_limit = qp->hop_limit_ttl;
+	p_ramrod->irq_num_pages = qp->irq_num_pages;
+	p_ramrod->p_key = cpu_to_le16(qp->pkey);
+	p_ramrod->flow_label = cpu_to_le32(qp->flow_label);
+	p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp);
+	p_ramrod->mtu = cpu_to_le16(qp->mtu);
+	p_ramrod->initial_psn = cpu_to_le32(qp->rq_psn);
+	p_ramrod->pd = cpu_to_le16(qp->pd);
+	p_ramrod->rq_num_pages = cpu_to_le16(qp->rq_num_pages);
+	DMA_REGPAIR_LE(p_ramrod->rq_pbl_addr, qp->rq_pbl_ptr);
+	DMA_REGPAIR_LE(p_ramrod->irq_pbl_addr, qp->irq_phys_addr);
+	qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
+	p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi);
+	p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo);
+	p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi);
+	p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo);
+	p_ramrod->stats_counter_id = p_hwfn->rel_pf_id;
+	p_ramrod->cq_cid = cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) |
+				       qp->rq_cq_id);
+
+	memset(&qm_params, 0, sizeof(qm_params));
+	qm_params.roce.qpid = qp->icid >> 1;
+	physical_queue0 = qed_get_qm_pq(p_hwfn, PROTOCOLID_ROCE, &qm_params);
+
+	p_ramrod->physical_queue0 = cpu_to_le16(physical_queue0);
+	p_ramrod->dpi = cpu_to_le16(qp->dpi);
+
+	qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr);
+	qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr);
+
+	p_ramrod->udp_src_port = qp->udp_src_port;
+	p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id);
+	p_ramrod->srq_id.srq_idx = cpu_to_le16(qp->srq_id);
+	p_ramrod->srq_id.opaque_fid = cpu_to_le16(p_hwfn->hw_info.opaque_fid);
+
+	p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) +
+				     qp->stats_queue;
+
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d physical_queue0 = 0x%x\n",
+		   rc, physical_queue0);
+
+	if (rc)
+		goto err;
+
+	qp->resp_offloaded = true;
+
+	return rc;
+
+err:
+	DP_NOTICE(p_hwfn, "create responder - failed, rc = %d\n", rc);
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+			  qp->irq_num_pages * RDMA_RING_PAGE_SIZE,
+			  qp->irq, qp->irq_phys_addr);
+
+	return rc;
+}
+
+static int qed_roce_sp_create_requester(struct qed_hwfn *p_hwfn,
+					struct qed_rdma_qp *qp)
+{
+	struct roce_create_qp_req_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	union qed_qm_pq_params qm_params;
+	enum roce_flavor roce_flavor;
+	struct qed_spq_entry *p_ent;
+	u16 physical_queue0 = 0;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	/* Allocate DMA-able memory for ORQ */
+	qp->orq_num_pages = 1;
+	qp->orq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+				     RDMA_RING_PAGE_SIZE,
+				     &qp->orq_phys_addr, GFP_KERNEL);
+	if (!qp->orq) {
+		rc = -ENOMEM;
+		DP_NOTICE(p_hwfn,
+			  "qed create requester failed: cannot allocate memory (orq). rc = %d\n",
+			  rc);
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid + 1;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 ROCE_RAMROD_CREATE_QP,
+				 PROTOCOLID_ROCE, &init_data);
+	if (rc)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.roce_create_qp_req;
+
+	p_ramrod->flags = 0;
+
+	roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode);
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR, roce_flavor);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN,
+		  qp->fmr_and_reserved_lkey);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP, qp->signal_all);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT,
+		  qp->rnr_retry_cnt);
+
+	p_ramrod->max_ord = qp->max_rd_atomic_req;
+	p_ramrod->traffic_class = qp->traffic_class_tos;
+	p_ramrod->hop_limit = qp->hop_limit_ttl;
+	p_ramrod->orq_num_pages = qp->orq_num_pages;
+	p_ramrod->p_key = cpu_to_le16(qp->pkey);
+	p_ramrod->flow_label = cpu_to_le32(qp->flow_label);
+	p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp);
+	p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout);
+	p_ramrod->mtu = cpu_to_le16(qp->mtu);
+	p_ramrod->initial_psn = cpu_to_le32(qp->sq_psn);
+	p_ramrod->pd = cpu_to_le16(qp->pd);
+	p_ramrod->sq_num_pages = cpu_to_le16(qp->sq_num_pages);
+	DMA_REGPAIR_LE(p_ramrod->sq_pbl_addr, qp->sq_pbl_ptr);
+	DMA_REGPAIR_LE(p_ramrod->orq_pbl_addr, qp->orq_phys_addr);
+	qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
+	p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi);
+	p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo);
+	p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi);
+	p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo);
+	p_ramrod->stats_counter_id = p_hwfn->rel_pf_id;
+	p_ramrod->cq_cid = cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) |
+				       qp->sq_cq_id);
+
+	memset(&qm_params, 0, sizeof(qm_params));
+	qm_params.roce.qpid = qp->icid >> 1;
+	physical_queue0 = qed_get_qm_pq(p_hwfn, PROTOCOLID_ROCE, &qm_params);
+
+	p_ramrod->physical_queue0 = cpu_to_le16(physical_queue0);
+	p_ramrod->dpi = cpu_to_le16(qp->dpi);
+
+	qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr);
+	qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr);
+
+	p_ramrod->udp_src_port = qp->udp_src_port;
+	p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id);
+	p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) +
+				     qp->stats_queue;
+
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
+
+	if (rc)
+		goto err;
+
+	qp->req_offloaded = true;
+
+	return rc;
+
+err:
+	DP_NOTICE(p_hwfn, "Create requested - failed, rc = %d\n", rc);
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+			  qp->orq_num_pages * RDMA_RING_PAGE_SIZE,
+			  qp->orq, qp->orq_phys_addr);
+	return rc;
+}
+
+static int qed_roce_sp_modify_responder(struct qed_hwfn *p_hwfn,
+					struct qed_rdma_qp *qp,
+					bool move_to_err, u32 modify_flags)
+{
+	struct roce_modify_qp_resp_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	if (move_to_err && !qp->resp_offloaded)
+		return 0;
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 ROCE_EVENT_MODIFY_QP,
+				 PROTOCOLID_ROCE, &init_data);
+	if (rc) {
+		DP_NOTICE(p_hwfn, "rc = %d\n", rc);
+		return rc;
+	}
+
+	p_ramrod = &p_ent->ramrod.roce_modify_qp_resp;
+
+	p_ramrod->flags = 0;
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN,
+		  qp->incoming_rdma_read_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN,
+		  qp->incoming_rdma_write_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN,
+		  qp->incoming_atomic_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN,
+		  qp->e2e_flow_control_en);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG,
+		  GET_FIELD(modify_flags,
+			    QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG,
+		  GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG,
+		  GET_FIELD(modify_flags,
+			    QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG,
+		  GET_FIELD(modify_flags,
+			    QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG,
+		  GET_FIELD(modify_flags,
+			    QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER));
+
+	p_ramrod->fields = 0;
+	SET_FIELD(p_ramrod->fields,
+		  ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER,
+		  qp->min_rnr_nak_timer);
+
+	p_ramrod->max_ird = qp->max_rd_atomic_resp;
+	p_ramrod->traffic_class = qp->traffic_class_tos;
+	p_ramrod->hop_limit = qp->hop_limit_ttl;
+	p_ramrod->p_key = cpu_to_le16(qp->pkey);
+	p_ramrod->flow_label = cpu_to_le32(qp->flow_label);
+	p_ramrod->mtu = cpu_to_le16(qp->mtu);
+	qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify responder, rc = %d\n", rc);
+	return rc;
+}
+
+static int qed_roce_sp_modify_requester(struct qed_hwfn *p_hwfn,
+					struct qed_rdma_qp *qp,
+					bool move_to_sqd,
+					bool move_to_err, u32 modify_flags)
+{
+	struct roce_modify_qp_req_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	if (move_to_err && !(qp->req_offloaded))
+		return 0;
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid + 1;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 ROCE_EVENT_MODIFY_QP,
+				 PROTOCOLID_ROCE, &init_data);
+	if (rc) {
+		DP_NOTICE(p_hwfn, "rc = %d\n", rc);
+		return rc;
+	}
+
+	p_ramrod = &p_ent->ramrod.roce_modify_qp_req;
+
+	p_ramrod->flags = 0;
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG, move_to_sqd);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY,
+		  qp->sqd_async);
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG,
+		  GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG,
+		  GET_FIELD(modify_flags,
+			    QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG,
+		  GET_FIELD(modify_flags,
+			    QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG,
+		  GET_FIELD(modify_flags,
+			    QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG,
+		  GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT));
+
+	SET_FIELD(p_ramrod->flags,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG,
+		  GET_FIELD(modify_flags,
+			    QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT));
+
+	p_ramrod->fields = 0;
+	SET_FIELD(p_ramrod->fields,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt);
+
+	SET_FIELD(p_ramrod->fields,
+		  ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT,
+		  qp->rnr_retry_cnt);
+
+	p_ramrod->max_ord = qp->max_rd_atomic_req;
+	p_ramrod->traffic_class = qp->traffic_class_tos;
+	p_ramrod->hop_limit = qp->hop_limit_ttl;
+	p_ramrod->p_key = cpu_to_le16(qp->pkey);
+	p_ramrod->flow_label = cpu_to_le32(qp->flow_label);
+	p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout);
+	p_ramrod->mtu = cpu_to_le16(qp->mtu);
+	qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify requester, rc = %d\n", rc);
+	return rc;
+}
+
+static int qed_roce_sp_destroy_qp_responder(struct qed_hwfn *p_hwfn,
+					    struct qed_rdma_qp *qp,
+					    u32 *num_invalidated_mw)
+{
+	struct roce_destroy_qp_resp_output_params *p_ramrod_res;
+	struct roce_destroy_qp_resp_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	dma_addr_t ramrod_res_phys;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	if (!qp->resp_offloaded)
+		return 0;
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent,
+				 ROCE_RAMROD_DESTROY_QP,
+				 PROTOCOLID_ROCE, &init_data);
+	if (rc)
+		return rc;
+
+	p_ramrod = &p_ent->ramrod.roce_destroy_qp_resp;
+
+	p_ramrod_res = (struct roce_destroy_qp_resp_output_params *)
+	    dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res),
+			       &ramrod_res_phys, GFP_KERNEL);
+
+	if (!p_ramrod_res) {
+		rc = -ENOMEM;
+		DP_NOTICE(p_hwfn,
+			  "qed destroy responder failed: cannot allocate memory (ramrod). rc = %d\n",
+			  rc);
+		return rc;
+	}
+
+	DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
+
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+	if (rc)
+		goto err;
+
+	*num_invalidated_mw = le32_to_cpu(p_ramrod_res->num_invalidated_mw);
+
+	/* Free IRQ - only if ramrod succeeded, in case FW is still using it */
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+			  qp->irq_num_pages * RDMA_RING_PAGE_SIZE,
+			  qp->irq, qp->irq_phys_addr);
+
+	qp->resp_offloaded = false;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy responder, rc = %d\n", rc);
+
+err:
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+			  sizeof(struct roce_destroy_qp_resp_output_params),
+			  p_ramrod_res, ramrod_res_phys);
+
+	return rc;
+}
+
+static int qed_roce_sp_destroy_qp_requester(struct qed_hwfn *p_hwfn,
+					    struct qed_rdma_qp *qp,
+					    u32 *num_bound_mw)
+{
+	struct roce_destroy_qp_req_output_params *p_ramrod_res;
+	struct roce_destroy_qp_req_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	dma_addr_t ramrod_res_phys;
+	int rc = -ENOMEM;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	if (!qp->req_offloaded)
+		return 0;
+
+	p_ramrod_res = (struct roce_destroy_qp_req_output_params *)
+		       dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+					  sizeof(*p_ramrod_res),
+					  &ramrod_res_phys, GFP_KERNEL);
+	if (!p_ramrod_res) {
+		DP_NOTICE(p_hwfn,
+			  "qed destroy requester failed: cannot allocate memory (ramrod)\n");
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid + 1;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_DESTROY_QP,
+				 PROTOCOLID_ROCE, &init_data);
+	if (rc)
+		goto err;
+
+	p_ramrod = &p_ent->ramrod.roce_destroy_qp_req;
+	DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
+
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+	if (rc)
+		goto err;
+
+	*num_bound_mw = le32_to_cpu(p_ramrod_res->num_bound_mw);
+
+	/* Free ORQ - only if ramrod succeeded, in case FW is still using it */
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+			  qp->orq_num_pages * RDMA_RING_PAGE_SIZE,
+			  qp->orq, qp->orq_phys_addr);
+
+	qp->req_offloaded = false;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy requester, rc = %d\n", rc);
+
+err:
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res),
+			  p_ramrod_res, ramrod_res_phys);
+
+	return rc;
+}
+
+int qed_roce_query_qp(struct qed_hwfn *p_hwfn,
+		      struct qed_rdma_qp *qp,
+		      struct qed_rdma_query_qp_out_params *out_params)
+{
+	struct roce_query_qp_resp_output_params *p_resp_ramrod_res;
+	struct roce_query_qp_req_output_params *p_req_ramrod_res;
+	struct roce_query_qp_resp_ramrod_data *p_resp_ramrod;
+	struct roce_query_qp_req_ramrod_data *p_req_ramrod;
+	struct qed_sp_init_data init_data;
+	dma_addr_t resp_ramrod_res_phys;
+	dma_addr_t req_ramrod_res_phys;
+	struct qed_spq_entry *p_ent;
+	bool rq_err_state;
+	bool sq_err_state;
+	bool sq_draining;
+	int rc = -ENOMEM;
+
+	if ((!(qp->resp_offloaded)) && (!(qp->req_offloaded))) {
+		/* We can't send ramrod to the fw since this qp wasn't offloaded
+		 * to the fw yet
+		 */
+		out_params->draining = false;
+		out_params->rq_psn = qp->rq_psn;
+		out_params->sq_psn = qp->sq_psn;
+		out_params->state = qp->cur_state;
+
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "No QPs as no offload\n");
+		return 0;
+	}
+
+	if (!(qp->resp_offloaded)) {
+		DP_NOTICE(p_hwfn,
+			  "The responder's qp should be offloded before requester's\n");
+		return -EINVAL;
+	}
+
+	/* Send a query responder ramrod to FW to get RQ-PSN and state */
+	p_resp_ramrod_res = (struct roce_query_qp_resp_output_params *)
+	    dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+			       sizeof(*p_resp_ramrod_res),
+			       &resp_ramrod_res_phys, GFP_KERNEL);
+	if (!p_resp_ramrod_res) {
+		DP_NOTICE(p_hwfn,
+			  "qed query qp failed: cannot allocate memory (ramrod)\n");
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.cid = qp->icid;
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+	rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP,
+				 PROTOCOLID_ROCE, &init_data);
+	if (rc)
+		goto err_resp;
+
+	p_resp_ramrod = &p_ent->ramrod.roce_query_qp_resp;
+	DMA_REGPAIR_LE(p_resp_ramrod->output_params_addr, resp_ramrod_res_phys);
+
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+	if (rc)
+		goto err_resp;
+
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res),
+			  p_resp_ramrod_res, resp_ramrod_res_phys);
+
+	out_params->rq_psn = le32_to_cpu(p_resp_ramrod_res->psn);
+	rq_err_state = GET_FIELD(le32_to_cpu(p_resp_ramrod_res->err_flag),
+				 ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG);
+
+	if (!(qp->req_offloaded)) {
+		/* Don't send query qp for the requester */
+		out_params->sq_psn = qp->sq_psn;
+		out_params->draining = false;
+
+		if (rq_err_state)
+			qp->cur_state = QED_ROCE_QP_STATE_ERR;
+
+		out_params->state = qp->cur_state;
+
+		return 0;
+	}
+
+	/* Send a query requester ramrod to FW to get SQ-PSN and state */
+	p_req_ramrod_res = (struct roce_query_qp_req_output_params *)
+			   dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+					      sizeof(*p_req_ramrod_res),
+					      &req_ramrod_res_phys,
+					      GFP_KERNEL);
+	if (!p_req_ramrod_res) {
+		rc = -ENOMEM;
+		DP_NOTICE(p_hwfn,
+			  "qed query qp failed: cannot allocate memory (ramrod)\n");
+		return rc;
+	}
+
+	/* Get SPQ entry */
+	init_data.cid = qp->icid + 1;
+	rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP,
+				 PROTOCOLID_ROCE, &init_data);
+	if (rc)
+		goto err_req;
+
+	p_req_ramrod = &p_ent->ramrod.roce_query_qp_req;
+	DMA_REGPAIR_LE(p_req_ramrod->output_params_addr, req_ramrod_res_phys);
+
+	rc = qed_spq_post(p_hwfn, p_ent, NULL);
+	if (rc)
+		goto err_req;
+
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res),
+			  p_req_ramrod_res, req_ramrod_res_phys);
+
+	out_params->sq_psn = le32_to_cpu(p_req_ramrod_res->psn);
+	sq_err_state = GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags),
+				 ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG);
+	sq_draining =
+		GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags),
+			  ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG);
+
+	out_params->draining = false;
+
+	if (rq_err_state)
+		qp->cur_state = QED_ROCE_QP_STATE_ERR;
+	else if (sq_err_state)
+		qp->cur_state = QED_ROCE_QP_STATE_SQE;
+	else if (sq_draining)
+		out_params->draining = true;
+	out_params->state = qp->cur_state;
+
+	return 0;
+
+err_req:
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res),
+			  p_req_ramrod_res, req_ramrod_res_phys);
+	return rc;
+err_resp:
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res),
+			  p_resp_ramrod_res, resp_ramrod_res_phys);
+	return rc;
+}
+
+int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp)
+{
+	u32 num_invalidated_mw = 0;
+	u32 num_bound_mw = 0;
+	u32 start_cid;
+	int rc;
+
+	/* Destroys the specified QP */
+	if ((qp->cur_state != QED_ROCE_QP_STATE_RESET) &&
+	    (qp->cur_state != QED_ROCE_QP_STATE_ERR) &&
+	    (qp->cur_state != QED_ROCE_QP_STATE_INIT)) {
+		DP_NOTICE(p_hwfn,
+			  "QP must be in error, reset or init state before destroying it\n");
+		return -EINVAL;
+	}
+
+	rc = qed_roce_sp_destroy_qp_responder(p_hwfn, qp, &num_invalidated_mw);
+	if (rc)
+		return rc;
+
+	/* Send destroy requester ramrod */
+	rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp, &num_bound_mw);
+	if (rc)
+		return rc;
+
+	if (num_invalidated_mw != num_bound_mw) {
+		DP_NOTICE(p_hwfn,
+			  "number of invalidate memory windows is different from bounded ones\n");
+		return -EINVAL;
+	}
+
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+
+	start_cid = qed_cxt_get_proto_cid_start(p_hwfn,
+						p_hwfn->p_rdma_info->proto);
+
+	/* Release responder's icid */
+	qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map,
+			    qp->icid - start_cid);
+
+	/* Release requester's icid */
+	qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map,
+			    qp->icid + 1 - start_cid);
+
+	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
+
+	return 0;
+}
+
+int qed_rdma_query_qp(void *rdma_cxt,
+		      struct qed_rdma_qp *qp,
+		      struct qed_rdma_query_qp_out_params *out_params)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	/* The following fields are filled in from qp and not FW as they can't
+	 * be modified by FW
+	 */
+	out_params->mtu = qp->mtu;
+	out_params->dest_qp = qp->dest_qp;
+	out_params->incoming_atomic_en = qp->incoming_atomic_en;
+	out_params->e2e_flow_control_en = qp->e2e_flow_control_en;
+	out_params->incoming_rdma_read_en = qp->incoming_rdma_read_en;
+	out_params->incoming_rdma_write_en = qp->incoming_rdma_write_en;
+	out_params->dgid = qp->dgid;
+	out_params->flow_label = qp->flow_label;
+	out_params->hop_limit_ttl = qp->hop_limit_ttl;
+	out_params->traffic_class_tos = qp->traffic_class_tos;
+	out_params->timeout = qp->ack_timeout;
+	out_params->rnr_retry = qp->rnr_retry_cnt;
+	out_params->retry_cnt = qp->retry_cnt;
+	out_params->min_rnr_nak_timer = qp->min_rnr_nak_timer;
+	out_params->pkey_index = 0;
+	out_params->max_rd_atomic = qp->max_rd_atomic_req;
+	out_params->max_dest_rd_atomic = qp->max_rd_atomic_resp;
+	out_params->sqd_async = qp->sqd_async;
+
+	rc = qed_roce_query_qp(p_hwfn, qp, out_params);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query QP, rc = %d\n", rc);
+	return rc;
+}
+
+int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	int rc = 0;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
+
+	rc = qed_roce_destroy_qp(p_hwfn, qp);
+
+	/* free qp params struct */
+	kfree(qp);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP destroyed\n");
+	return rc;
+}
+
+struct qed_rdma_qp *
+qed_rdma_create_qp(void *rdma_cxt,
+		   struct qed_rdma_create_qp_in_params *in_params,
+		   struct qed_rdma_create_qp_out_params *out_params)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct qed_rdma_qp *qp;
+	u8 max_stats_queues;
+	int rc;
+
+	if (!rdma_cxt || !in_params || !out_params || !p_hwfn->p_rdma_info) {
+		DP_ERR(p_hwfn->cdev,
+		       "qed roce create qp failed due to NULL entry (rdma_cxt=%p, in=%p, out=%p, roce_info=?\n",
+		       rdma_cxt, in_params, out_params);
+		return NULL;
+	}
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+		   "qed rdma create qp called with qp_handle = %08x%08x\n",
+		   in_params->qp_handle_hi, in_params->qp_handle_lo);
+
+	/* Some sanity checks... */
+	max_stats_queues = p_hwfn->p_rdma_info->dev->max_stats_queues;
+	if (in_params->stats_queue >= max_stats_queues) {
+		DP_ERR(p_hwfn->cdev,
+		       "qed rdma create qp failed due to invalid statistics queue %d. maximum is %d\n",
+		       in_params->stats_queue, max_stats_queues);
+		return NULL;
+	}
+
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp) {
+		DP_NOTICE(p_hwfn, "Failed to allocate qed_rdma_qp\n");
+		return NULL;
+	}
+
+	rc = qed_roce_alloc_cid(p_hwfn, &qp->icid);
+	qp->qpid = ((0xFF << 16) | qp->icid);
+
+	DP_INFO(p_hwfn, "ROCE qpid=%x\n", qp->qpid);
+
+	if (rc) {
+		kfree(qp);
+		return NULL;
+	}
+
+	qp->cur_state = QED_ROCE_QP_STATE_RESET;
+	qp->qp_handle.hi = cpu_to_le32(in_params->qp_handle_hi);
+	qp->qp_handle.lo = cpu_to_le32(in_params->qp_handle_lo);
+	qp->qp_handle_async.hi = cpu_to_le32(in_params->qp_handle_async_hi);
+	qp->qp_handle_async.lo = cpu_to_le32(in_params->qp_handle_async_lo);
+	qp->use_srq = in_params->use_srq;
+	qp->signal_all = in_params->signal_all;
+	qp->fmr_and_reserved_lkey = in_params->fmr_and_reserved_lkey;
+	qp->pd = in_params->pd;
+	qp->dpi = in_params->dpi;
+	qp->sq_cq_id = in_params->sq_cq_id;
+	qp->sq_num_pages = in_params->sq_num_pages;
+	qp->sq_pbl_ptr = in_params->sq_pbl_ptr;
+	qp->rq_cq_id = in_params->rq_cq_id;
+	qp->rq_num_pages = in_params->rq_num_pages;
+	qp->rq_pbl_ptr = in_params->rq_pbl_ptr;
+	qp->srq_id = in_params->srq_id;
+	qp->req_offloaded = false;
+	qp->resp_offloaded = false;
+	qp->e2e_flow_control_en = qp->use_srq ? false : true;
+	qp->stats_queue = in_params->stats_queue;
+
+	out_params->icid = qp->icid;
+	out_params->qp_id = qp->qpid;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Create QP, rc = %d\n", rc);
+	return qp;
+}
+
+static int qed_roce_modify_qp(struct qed_hwfn *p_hwfn,
+			      struct qed_rdma_qp *qp,
+			      enum qed_roce_qp_state prev_state,
+			      struct qed_rdma_modify_qp_in_params *params)
+{
+	u32 num_invalidated_mw = 0, num_bound_mw = 0;
+	int rc = 0;
+
+	/* Perform additional operations according to the current state and the
+	 * next state
+	 */
+	if (((prev_state == QED_ROCE_QP_STATE_INIT) ||
+	     (prev_state == QED_ROCE_QP_STATE_RESET)) &&
+	    (qp->cur_state == QED_ROCE_QP_STATE_RTR)) {
+		/* Init->RTR or Reset->RTR */
+		rc = qed_roce_sp_create_responder(p_hwfn, qp);
+		return rc;
+	} else if ((prev_state == QED_ROCE_QP_STATE_RTR) &&
+		   (qp->cur_state == QED_ROCE_QP_STATE_RTS)) {
+		/* RTR-> RTS */
+		rc = qed_roce_sp_create_requester(p_hwfn, qp);
+		if (rc)
+			return rc;
+
+		/* Send modify responder ramrod */
+		rc = qed_roce_sp_modify_responder(p_hwfn, qp, false,
+						  params->modify_flags);
+		return rc;
+	} else if ((prev_state == QED_ROCE_QP_STATE_RTS) &&
+		   (qp->cur_state == QED_ROCE_QP_STATE_RTS)) {
+		/* RTS->RTS */
+		rc = qed_roce_sp_modify_responder(p_hwfn, qp, false,
+						  params->modify_flags);
+		if (rc)
+			return rc;
+
+		rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false,
+						  params->modify_flags);
+		return rc;
+	} else if ((prev_state == QED_ROCE_QP_STATE_RTS) &&
+		   (qp->cur_state == QED_ROCE_QP_STATE_SQD)) {
+		/* RTS->SQD */
+		rc = qed_roce_sp_modify_requester(p_hwfn, qp, true, false,
+						  params->modify_flags);
+		return rc;
+	} else if ((prev_state == QED_ROCE_QP_STATE_SQD) &&
+		   (qp->cur_state == QED_ROCE_QP_STATE_SQD)) {
+		/* SQD->SQD */
+		rc = qed_roce_sp_modify_responder(p_hwfn, qp, false,
+						  params->modify_flags);
+		if (rc)
+			return rc;
+
+		rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false,
+						  params->modify_flags);
+		return rc;
+	} else if ((prev_state == QED_ROCE_QP_STATE_SQD) &&
+		   (qp->cur_state == QED_ROCE_QP_STATE_RTS)) {
+		/* SQD->RTS */
+		rc = qed_roce_sp_modify_responder(p_hwfn, qp, false,
+						  params->modify_flags);
+		if (rc)
+			return rc;
+
+		rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false,
+						  params->modify_flags);
+
+		return rc;
+	} else if (qp->cur_state == QED_ROCE_QP_STATE_ERR ||
+		   qp->cur_state == QED_ROCE_QP_STATE_SQE) {
+		/* ->ERR */
+		rc = qed_roce_sp_modify_responder(p_hwfn, qp, true,
+						  params->modify_flags);
+		if (rc)
+			return rc;
+
+		rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, true,
+						  params->modify_flags);
+		return rc;
+	} else if (qp->cur_state == QED_ROCE_QP_STATE_RESET) {
+		/* Any state -> RESET */
+
+		rc = qed_roce_sp_destroy_qp_responder(p_hwfn, qp,
+						      &num_invalidated_mw);
+		if (rc)
+			return rc;
+
+		rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp,
+						      &num_bound_mw);
+
+		if (num_invalidated_mw != num_bound_mw) {
+			DP_NOTICE(p_hwfn,
+				  "number of invalidate memory windows is different from bounded ones\n");
+			return -EINVAL;
+		}
+	} else {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "0\n");
+	}
+
+	return rc;
+}
+
+int qed_rdma_modify_qp(void *rdma_cxt,
+		       struct qed_rdma_qp *qp,
+		       struct qed_rdma_modify_qp_in_params *params)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	enum qed_roce_qp_state prev_state;
+	int rc = 0;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x params->new_state=%d\n",
+		   qp->icid, params->new_state);
+
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
+		return rc;
+	}
+
+	if (GET_FIELD(params->modify_flags,
+		      QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)) {
+		qp->incoming_rdma_read_en = params->incoming_rdma_read_en;
+		qp->incoming_rdma_write_en = params->incoming_rdma_write_en;
+		qp->incoming_atomic_en = params->incoming_atomic_en;
+	}
+
+	/* Update QP structure with the updated values */
+	if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_ROCE_MODE))
+		qp->roce_mode = params->roce_mode;
+	if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY))
+		qp->pkey = params->pkey;
+	if (GET_FIELD(params->modify_flags,
+		      QED_ROCE_MODIFY_QP_VALID_E2E_FLOW_CONTROL_EN))
+		qp->e2e_flow_control_en = params->e2e_flow_control_en;
+	if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_DEST_QP))
+		qp->dest_qp = params->dest_qp;
+	if (GET_FIELD(params->modify_flags,
+		      QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)) {
+		/* Indicates that the following parameters have changed:
+		 * Traffic class, flow label, hop limit, source GID,
+		 * destination GID, loopback indicator
+		 */
+		qp->traffic_class_tos = params->traffic_class_tos;
+		qp->flow_label = params->flow_label;
+		qp->hop_limit_ttl = params->hop_limit_ttl;
+
+		qp->sgid = params->sgid;
+		qp->dgid = params->dgid;
+		qp->udp_src_port = 0;
+		qp->vlan_id = params->vlan_id;
+		qp->mtu = params->mtu;
+		qp->lb_indication = params->lb_indication;
+		memcpy((u8 *)&qp->remote_mac_addr[0],
+		       (u8 *)&params->remote_mac_addr[0], ETH_ALEN);
+		if (params->use_local_mac) {
+			memcpy((u8 *)&qp->local_mac_addr[0],
+			       (u8 *)&params->local_mac_addr[0], ETH_ALEN);
+		} else {
+			memcpy((u8 *)&qp->local_mac_addr[0],
+			       (u8 *)&p_hwfn->hw_info.hw_mac_addr, ETH_ALEN);
+		}
+	}
+	if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RQ_PSN))
+		qp->rq_psn = params->rq_psn;
+	if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_SQ_PSN))
+		qp->sq_psn = params->sq_psn;
+	if (GET_FIELD(params->modify_flags,
+		      QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ))
+		qp->max_rd_atomic_req = params->max_rd_atomic_req;
+	if (GET_FIELD(params->modify_flags,
+		      QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP))
+		qp->max_rd_atomic_resp = params->max_rd_atomic_resp;
+	if (GET_FIELD(params->modify_flags,
+		      QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT))
+		qp->ack_timeout = params->ack_timeout;
+	if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT))
+		qp->retry_cnt = params->retry_cnt;
+	if (GET_FIELD(params->modify_flags,
+		      QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT))
+		qp->rnr_retry_cnt = params->rnr_retry_cnt;
+	if (GET_FIELD(params->modify_flags,
+		      QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER))
+		qp->min_rnr_nak_timer = params->min_rnr_nak_timer;
+
+	qp->sqd_async = params->sqd_async;
+
+	prev_state = qp->cur_state;
+	if (GET_FIELD(params->modify_flags,
+		      QED_RDMA_MODIFY_QP_VALID_NEW_STATE)) {
+		qp->cur_state = params->new_state;
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "qp->cur_state=%d\n",
+			   qp->cur_state);
+	}
+
+	rc = qed_roce_modify_qp(p_hwfn, qp, prev_state, params);
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify QP, rc = %d\n", rc);
+	return rc;
+}
+
+int qed_rdma_register_tid(void *rdma_cxt,
+			  struct qed_rdma_register_tid_in_params *params)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct rdma_register_tid_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	enum rdma_tid_type tid_type;
+	u8 fw_return_code;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", params->itid);
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_REGISTER_MR,
+				 p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
+		return rc;
+	}
+
+	if (p_hwfn->p_rdma_info->last_tid < params->itid)
+		p_hwfn->p_rdma_info->last_tid = params->itid;
+
+	p_ramrod = &p_ent->ramrod.rdma_register_tid;
+
+	p_ramrod->flags = 0;
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL,
+		  params->pbl_two_level);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED, params->zbva);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR, params->phy_mr);
+
+	/* Don't initialize D/C field, as it may override other bits. */
+	if (!(params->tid_type == QED_RDMA_TID_FMR) && !(params->dma_mr))
+		SET_FIELD(p_ramrod->flags,
+			  RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG,
+			  params->page_size_log - 12);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_MAX_ID,
+		  p_hwfn->p_rdma_info->last_tid);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ,
+		  params->remote_read);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE,
+		  params->remote_write);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC,
+		  params->remote_atomic);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE,
+		  params->local_write);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ, params->local_read);
+
+	SET_FIELD(p_ramrod->flags,
+		  RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND,
+		  params->mw_bind);
+
+	SET_FIELD(p_ramrod->flags1,
+		  RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG,
+		  params->pbl_page_size_log - 12);
+
+	SET_FIELD(p_ramrod->flags2,
+		  RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR, params->dma_mr);
+
+	switch (params->tid_type) {
+	case QED_RDMA_TID_REGISTERED_MR:
+		tid_type = RDMA_TID_REGISTERED_MR;
+		break;
+	case QED_RDMA_TID_FMR:
+		tid_type = RDMA_TID_FMR;
+		break;
+	case QED_RDMA_TID_MW_TYPE1:
+		tid_type = RDMA_TID_MW_TYPE1;
+		break;
+	case QED_RDMA_TID_MW_TYPE2A:
+		tid_type = RDMA_TID_MW_TYPE2A;
+		break;
+	default:
+		rc = -EINVAL;
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
+		return rc;
+	}
+	SET_FIELD(p_ramrod->flags1,
+		  RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE, tid_type);
+
+	p_ramrod->itid = cpu_to_le32(params->itid);
+	p_ramrod->key = params->key;
+	p_ramrod->pd = cpu_to_le16(params->pd);
+	p_ramrod->length_hi = (u8)(params->length >> 32);
+	p_ramrod->length_lo = DMA_LO_LE(params->length);
+	if (params->zbva) {
+		/* Lower 32 bits of the registered MR address.
+		 * In case of zero based MR, will hold FBO
+		 */
+		p_ramrod->va.hi = 0;
+		p_ramrod->va.lo = cpu_to_le32(params->fbo);
+	} else {
+		DMA_REGPAIR_LE(p_ramrod->va, params->vaddr);
+	}
+	DMA_REGPAIR_LE(p_ramrod->pbl_base, params->pbl_ptr);
+
+	/* DIF */
+	if (params->dif_enabled) {
+		SET_FIELD(p_ramrod->flags2,
+			  RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG, 1);
+		DMA_REGPAIR_LE(p_ramrod->dif_error_addr,
+			       params->dif_error_addr);
+		DMA_REGPAIR_LE(p_ramrod->dif_runt_addr, params->dif_runt_addr);
+	}
+
+	rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code);
+
+	if (fw_return_code != RDMA_RETURN_OK) {
+		DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code);
+		return -EINVAL;
+	}
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Register TID, rc = %d\n", rc);
+	return rc;
+}
+
+int qed_rdma_deregister_tid(void *rdma_cxt, u32 itid)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct rdma_deregister_tid_ramrod_data *p_ramrod;
+	struct qed_sp_init_data init_data;
+	struct qed_spq_entry *p_ent;
+	struct qed_ptt *p_ptt;
+	u8 fw_return_code;
+	int rc;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid);
+
+	/* Get SPQ entry */
+	memset(&init_data, 0, sizeof(init_data));
+	init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
+	init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
+
+	rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_DEREGISTER_MR,
+				 p_hwfn->p_rdma_info->proto, &init_data);
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
+		return rc;
+	}
+
+	p_ramrod = &p_ent->ramrod.rdma_deregister_tid;
+	p_ramrod->itid = cpu_to_le32(itid);
+
+	rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code);
+	if (rc) {
+		DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
+		return rc;
+	}
+
+	if (fw_return_code == RDMA_RETURN_DEREGISTER_MR_BAD_STATE_ERR) {
+		DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code);
+		return -EINVAL;
+	} else if (fw_return_code == RDMA_RETURN_NIG_DRAIN_REQ) {
+		/* Bit indicating that the TID is in use and a nig drain is
+		 * required before sending the ramrod again
+		 */
+		p_ptt = qed_ptt_acquire(p_hwfn);
+		if (!p_ptt) {
+			rc = -EBUSY;
+			DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+				   "Failed to acquire PTT\n");
+			return rc;
+		}
+
+		rc = qed_mcp_drain(p_hwfn, p_ptt);
+		if (rc) {
+			qed_ptt_release(p_hwfn, p_ptt);
+			DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+				   "Drain failed\n");
+			return rc;
+		}
+
+		qed_ptt_release(p_hwfn, p_ptt);
+
+		/* Resend the ramrod */
+		rc = qed_sp_init_request(p_hwfn, &p_ent,
+					 RDMA_RAMROD_DEREGISTER_MR,
+					 p_hwfn->p_rdma_info->proto,
+					 &init_data);
+		if (rc) {
+			DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+				   "Failed to init sp-element\n");
+			return rc;
+		}
+
+		rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code);
+		if (rc) {
+			DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+				   "Ramrod failed\n");
+			return rc;
+		}
+
+		if (fw_return_code != RDMA_RETURN_OK) {
+			DP_NOTICE(p_hwfn, "fw_return_code = %d\n",
+				  fw_return_code);
+			return rc;
+		}
+	}
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "De-registered TID, rc = %d\n", rc);
+	return rc;
+}
+
+static void *qed_rdma_get_rdma_ctx(struct qed_dev *cdev)
+{
+	return QED_LEADING_HWFN(cdev);
+}
+
+static void qed_rdma_dpm_conf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	u32 val;
+
+	val = (p_hwfn->dcbx_no_edpm || p_hwfn->db_bar_no_edpm) ? 0 : 1;
+
+	qed_wr(p_hwfn, p_ptt, DORQ_REG_PF_DPM_ENABLE, val);
+	DP_VERBOSE(p_hwfn, (QED_MSG_DCB | QED_MSG_RDMA),
+		   "Changing DPM_EN state to %d (DCBX=%d, DB_BAR=%d)\n",
+		   val, p_hwfn->dcbx_no_edpm, p_hwfn->db_bar_no_edpm);
+}
+
+void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	p_hwfn->db_bar_no_edpm = true;
+
+	qed_rdma_dpm_conf(p_hwfn, p_ptt);
+}
+
+int qed_rdma_start(void *rdma_cxt, struct qed_rdma_start_in_params *params)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct qed_ptt *p_ptt;
+	int rc = -EBUSY;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
+		   "desired_cnq = %08x\n", params->desired_cnq);
+
+	p_ptt = qed_ptt_acquire(p_hwfn);
+	if (!p_ptt)
+		goto err;
+
+	rc = qed_rdma_alloc(p_hwfn, p_ptt, params);
+	if (rc)
+		goto err1;
+
+	rc = qed_rdma_setup(p_hwfn, p_ptt, params);
+	if (rc)
+		goto err2;
+
+	qed_ptt_release(p_hwfn, p_ptt);
+
+	return rc;
+
+err2:
+	qed_rdma_free(p_hwfn);
+err1:
+	qed_ptt_release(p_hwfn, p_ptt);
+err:
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA start - error, rc = %d\n", rc);
+	return rc;
+}
+
+static int qed_rdma_init(struct qed_dev *cdev,
+			 struct qed_rdma_start_in_params *params)
+{
+	return qed_rdma_start(QED_LEADING_HWFN(cdev), params);
+}
+
+void qed_rdma_remove_user(void *rdma_cxt, u16 dpi)
+{
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "dpi = %08x\n", dpi);
+
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+	qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, dpi);
+	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
+}
+
+void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn *p_hwfn,
+				     u8 connection_handle,
+				     void *cookie,
+				     dma_addr_t first_frag_addr,
+				     bool b_last_fragment, bool b_last_packet)
+{
+	struct qed_roce_ll2_packet *packet = cookie;
+	struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2;
+
+	roce_ll2->cbs.tx_cb(roce_ll2->cb_cookie, packet);
+}
+
+void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn *p_hwfn,
+				    u8 connection_handle,
+				    void *cookie,
+				    dma_addr_t first_frag_addr,
+				    bool b_last_fragment, bool b_last_packet)
+{
+	qed_ll2b_complete_tx_gsi_packet(p_hwfn, connection_handle,
+					cookie, first_frag_addr,
+					b_last_fragment, b_last_packet);
+}
+
+void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn *p_hwfn,
+				     u8 connection_handle,
+				     void *cookie,
+				     dma_addr_t rx_buf_addr,
+				     u16 data_length,
+				     u8 data_length_error,
+				     u16 parse_flags,
+				     u16 vlan,
+				     u32 src_mac_addr_hi,
+				     u16 src_mac_addr_lo, bool b_last_packet)
+{
+	struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2;
+	struct qed_roce_ll2_rx_params params;
+	struct qed_dev *cdev = p_hwfn->cdev;
+	struct qed_roce_ll2_packet pkt;
+
+	DP_VERBOSE(cdev,
+		   QED_MSG_LL2,
+		   "roce ll2 rx complete: bus_addr=%p, len=%d, data_len_err=%d\n",
+		   (void *)(uintptr_t)rx_buf_addr,
+		   data_length, data_length_error);
+
+	memset(&pkt, 0, sizeof(pkt));
+	pkt.n_seg = 1;
+	pkt.payload[0].baddr = rx_buf_addr;
+	pkt.payload[0].len = data_length;
+
+	memset(&params, 0, sizeof(params));
+	params.vlan_id = vlan;
+	*((u32 *)&params.smac[0]) = ntohl(src_mac_addr_hi);
+	*((u16 *)&params.smac[4]) = ntohs(src_mac_addr_lo);
+
+	if (data_length_error) {
+		DP_ERR(cdev,
+		       "roce ll2 rx complete: data length error %d, length=%d\n",
+		       data_length_error, data_length);
+		params.rc = -EINVAL;
+	}
+
+	roce_ll2->cbs.rx_cb(roce_ll2->cb_cookie, &pkt, &params);
+}
+
+static int qed_roce_ll2_set_mac_filter(struct qed_dev *cdev,
+				       u8 *old_mac_address,
+				       u8 *new_mac_address)
+{
+	struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+	struct qed_ptt *p_ptt;
+	int rc = 0;
+
+	if (!hwfn->ll2 || hwfn->ll2->handle == QED_LL2_UNUSED_HANDLE) {
+		DP_ERR(cdev,
+		       "qed roce mac filter failed - roce_info/ll2 NULL\n");
+		return -EINVAL;
+	}
+
+	p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev));
+	if (!p_ptt) {
+		DP_ERR(cdev,
+		       "qed roce ll2 mac filter set: failed to acquire PTT\n");
+		return -EINVAL;
+	}
+
+	mutex_lock(&hwfn->ll2->lock);
+	if (old_mac_address)
+		qed_llh_remove_mac_filter(QED_LEADING_HWFN(cdev), p_ptt,
+					  old_mac_address);
+	if (new_mac_address)
+		rc = qed_llh_add_mac_filter(QED_LEADING_HWFN(cdev), p_ptt,
+					    new_mac_address);
+	mutex_unlock(&hwfn->ll2->lock);
+
+	qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt);
+
+	if (rc)
+		DP_ERR(cdev,
+		       "qed roce ll2 mac filter set: failed to add mac filter\n");
+
+	return rc;
+}
+
+static int qed_roce_ll2_start(struct qed_dev *cdev,
+			      struct qed_roce_ll2_params *params)
+{
+	struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+	struct qed_roce_ll2_info *roce_ll2;
+	struct qed_ll2_info ll2_params;
+	int rc;
+
+	if (!params) {
+		DP_ERR(cdev, "qed roce ll2 start: failed due to NULL params\n");
+		return -EINVAL;
+	}
+	if (!params->cbs.tx_cb || !params->cbs.rx_cb) {
+		DP_ERR(cdev,
+		       "qed roce ll2 start: failed due to NULL tx/rx. tx_cb=%p, rx_cb=%p\n",
+		       params->cbs.tx_cb, params->cbs.rx_cb);
+		return -EINVAL;
+	}
+	if (!is_valid_ether_addr(params->mac_address)) {
+		DP_ERR(cdev,
+		       "qed roce ll2 start: failed due to invalid Ethernet address %pM\n",
+		       params->mac_address);
+		return -EINVAL;
+	}
+
+	/* Initialize */
+	roce_ll2 = kzalloc(sizeof(*roce_ll2), GFP_ATOMIC);
+	if (!roce_ll2) {
+		DP_ERR(cdev, "qed roce ll2 start: failed memory allocation\n");
+		return -ENOMEM;
+	}
+	memset(roce_ll2, 0, sizeof(*roce_ll2));
+	roce_ll2->handle = QED_LL2_UNUSED_HANDLE;
+	roce_ll2->cbs = params->cbs;
+	roce_ll2->cb_cookie = params->cb_cookie;
+	mutex_init(&roce_ll2->lock);
+
+	memset(&ll2_params, 0, sizeof(ll2_params));
+	ll2_params.conn_type = QED_LL2_TYPE_ROCE;
+	ll2_params.mtu = params->mtu;
+	ll2_params.rx_drop_ttl0_flg = true;
+	ll2_params.rx_vlan_removal_en = false;
+	ll2_params.tx_dest = CORE_TX_DEST_NW;
+	ll2_params.ai_err_packet_too_big = LL2_DROP_PACKET;
+	ll2_params.ai_err_no_buf = LL2_DROP_PACKET;
+	ll2_params.gsi_enable = true;
+
+	rc = qed_ll2_acquire_connection(QED_LEADING_HWFN(cdev), &ll2_params,
+					params->max_rx_buffers,
+					params->max_tx_buffers,
+					&roce_ll2->handle);
+	if (rc) {
+		DP_ERR(cdev,
+		       "qed roce ll2 start: failed to acquire LL2 connection (rc=%d)\n",
+		       rc);
+		goto err;
+	}
+
+	rc = qed_ll2_establish_connection(QED_LEADING_HWFN(cdev),
+					  roce_ll2->handle);
+	if (rc) {
+		DP_ERR(cdev,
+		       "qed roce ll2 start: failed to establish LL2 connection (rc=%d)\n",
+		       rc);
+		goto err1;
+	}
+
+	hwfn->ll2 = roce_ll2;
+
+	rc = qed_roce_ll2_set_mac_filter(cdev, NULL, params->mac_address);
+	if (rc) {
+		hwfn->ll2 = NULL;
+		goto err2;
+	}
+	ether_addr_copy(roce_ll2->mac_address, params->mac_address);
+
+	return 0;
+
+err2:
+	qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle);
+err1:
+	qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle);
+err:
+	kfree(roce_ll2);
+	return rc;
+}
+
+static int qed_roce_ll2_stop(struct qed_dev *cdev)
+{
+	struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+	struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2;
+	int rc;
+
+	if (!cdev) {
+		DP_ERR(cdev, "qed roce ll2 stop: invalid cdev\n");
+		return -EINVAL;
+	}
+
+	if (roce_ll2->handle == QED_LL2_UNUSED_HANDLE) {
+		DP_ERR(cdev, "qed roce ll2 stop: cannot stop an unused LL2\n");
+		return -EINVAL;
+	}
+
+	/* remove LL2 MAC address filter */
+	rc = qed_roce_ll2_set_mac_filter(cdev, roce_ll2->mac_address, NULL);
+	eth_zero_addr(roce_ll2->mac_address);
+
+	rc = qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev),
+					  roce_ll2->handle);
+	if (rc)
+		DP_ERR(cdev,
+		       "qed roce ll2 stop: failed to terminate LL2 connection (rc=%d)\n",
+		       rc);
+
+	qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle);
+
+	roce_ll2->handle = QED_LL2_UNUSED_HANDLE;
+
+	kfree(roce_ll2);
+
+	return rc;
+}
+
+static int qed_roce_ll2_tx(struct qed_dev *cdev,
+			   struct qed_roce_ll2_packet *pkt,
+			   struct qed_roce_ll2_tx_params *params)
+{
+	struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+	struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2;
+	enum qed_ll2_roce_flavor_type qed_roce_flavor;
+	u8 flags = 0;
+	int rc;
+	int i;
+
+	if (!cdev || !pkt || !params) {
+		DP_ERR(cdev,
+		       "roce ll2 tx: failed tx because one of the following is NULL - drv=%p, pkt=%p, params=%p\n",
+		       cdev, pkt, params);
+		return -EINVAL;
+	}
+
+	qed_roce_flavor = (pkt->roce_mode == ROCE_V1) ? QED_LL2_ROCE
+						      : QED_LL2_RROCE;
+
+	if (pkt->roce_mode == ROCE_V2_IPV4)
+		flags |= BIT(CORE_TX_BD_FLAGS_IP_CSUM_SHIFT);
+
+	/* Tx header */
+	rc = qed_ll2_prepare_tx_packet(QED_LEADING_HWFN(cdev), roce_ll2->handle,
+				       1 + pkt->n_seg, 0, flags, 0,
+				       qed_roce_flavor, pkt->header.baddr,
+				       pkt->header.len, pkt, 1);
+	if (rc) {
+		DP_ERR(cdev, "roce ll2 tx: header failed (rc=%d)\n", rc);
+		return QED_ROCE_TX_HEAD_FAILURE;
+	}
+
+	/* Tx payload */
+	for (i = 0; i < pkt->n_seg; i++) {
+		rc = qed_ll2_set_fragment_of_tx_packet(QED_LEADING_HWFN(cdev),
+						       roce_ll2->handle,
+						       pkt->payload[i].baddr,
+						       pkt->payload[i].len);
+		if (rc) {
+			/* If failed not much to do here, partial packet has
+			 * been posted * we can't free memory, will need to wait
+			 * for completion
+			 */
+			DP_ERR(cdev,
+			       "roce ll2 tx: payload failed (rc=%d)\n", rc);
+			return QED_ROCE_TX_FRAG_FAILURE;
+		}
+	}
+
+	return 0;
+}
+
+static int qed_roce_ll2_post_rx_buffer(struct qed_dev *cdev,
+				       struct qed_roce_ll2_buffer *buf,
+				       u64 cookie, u8 notify_fw)
+{
+	return qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev),
+				      QED_LEADING_HWFN(cdev)->ll2->handle,
+				      buf->baddr, buf->len,
+				      (void *)(uintptr_t)cookie, notify_fw);
+}
+
+static int qed_roce_ll2_stats(struct qed_dev *cdev, struct qed_ll2_stats *stats)
+{
+	struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
+	struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2;
+
+	return qed_ll2_get_stats(QED_LEADING_HWFN(cdev),
+				 roce_ll2->handle, stats);
+}
+
+static const struct qed_rdma_ops qed_rdma_ops_pass = {
+	.common = &qed_common_ops_pass,
+	.fill_dev_info = &qed_fill_rdma_dev_info,
+	.rdma_get_rdma_ctx = &qed_rdma_get_rdma_ctx,
+	.rdma_init = &qed_rdma_init,
+	.rdma_add_user = &qed_rdma_add_user,
+	.rdma_remove_user = &qed_rdma_remove_user,
+	.rdma_stop = &qed_rdma_stop,
+	.rdma_query_port = &qed_rdma_query_port,
+	.rdma_query_device = &qed_rdma_query_device,
+	.rdma_get_start_sb = &qed_rdma_get_sb_start,
+	.rdma_get_rdma_int = &qed_rdma_get_int,
+	.rdma_set_rdma_int = &qed_rdma_set_int,
+	.rdma_get_min_cnq_msix = &qed_rdma_get_min_cnq_msix,
+	.rdma_cnq_prod_update = &qed_rdma_cnq_prod_update,
+	.rdma_alloc_pd = &qed_rdma_alloc_pd,
+	.rdma_dealloc_pd = &qed_rdma_free_pd,
+	.rdma_create_cq = &qed_rdma_create_cq,
+	.rdma_destroy_cq = &qed_rdma_destroy_cq,
+	.rdma_create_qp = &qed_rdma_create_qp,
+	.rdma_modify_qp = &qed_rdma_modify_qp,
+	.rdma_query_qp = &qed_rdma_query_qp,
+	.rdma_destroy_qp = &qed_rdma_destroy_qp,
+	.rdma_alloc_tid = &qed_rdma_alloc_tid,
+	.rdma_free_tid = &qed_rdma_free_tid,
+	.rdma_register_tid = &qed_rdma_register_tid,
+	.rdma_deregister_tid = &qed_rdma_deregister_tid,
+	.roce_ll2_start = &qed_roce_ll2_start,
+	.roce_ll2_stop = &qed_roce_ll2_stop,
+	.roce_ll2_tx = &qed_roce_ll2_tx,
+	.roce_ll2_post_rx_buffer = &qed_roce_ll2_post_rx_buffer,
+	.roce_ll2_set_mac_filter = &qed_roce_ll2_set_mac_filter,
+	.roce_ll2_stats = &qed_roce_ll2_stats,
+};
+
+const struct qed_rdma_ops *qed_get_rdma_ops()
+{
+	return &qed_rdma_ops_pass;
+}
+EXPORT_SYMBOL(qed_get_rdma_ops);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_roce.h b/drivers/net/ethernet/qlogic/qed/qed_roce.h
new file mode 100644
index 000000000000..2f091e8a0f40
--- /dev/null
+++ b/drivers/net/ethernet/qlogic/qed/qed_roce.h
@@ -0,0 +1,216 @@
+/* QLogic qed NIC Driver
+ * Copyright (c) 2015-2016  QLogic Corporation
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and /or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef _QED_ROCE_H
+#define _QED_ROCE_H
+#include <linux/types.h>
+#include <linux/bitops.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/qed/qed_if.h>
+#include <linux/qed/qed_roce_if.h>
+#include "qed.h"
+#include "qed_dev_api.h"
+#include "qed_hsi.h"
+#include "qed_ll2.h"
+
+#define QED_RDMA_MAX_FMR                    (RDMA_MAX_TIDS)
+#define QED_RDMA_MAX_P_KEY                  (1)
+#define QED_RDMA_MAX_WQE                    (0x7FFF)
+#define QED_RDMA_MAX_SRQ_WQE_ELEM           (0x7FFF)
+#define QED_RDMA_PAGE_SIZE_CAPS             (0xFFFFF000)
+#define QED_RDMA_ACK_DELAY                  (15)
+#define QED_RDMA_MAX_MR_SIZE                (0x10000000000ULL)
+#define QED_RDMA_MAX_CQS                    (RDMA_MAX_CQS)
+#define QED_RDMA_MAX_MRS                    (RDMA_MAX_TIDS)
+/* Add 1 for header element */
+#define QED_RDMA_MAX_SRQ_ELEM_PER_WQE	    (RDMA_MAX_SGE_PER_RQ_WQE + 1)
+#define QED_RDMA_MAX_SGE_PER_SRQ_WQE        (RDMA_MAX_SGE_PER_RQ_WQE)
+#define QED_RDMA_SRQ_WQE_ELEM_SIZE          (16)
+#define QED_RDMA_MAX_SRQS                   (32 * 1024)
+
+#define QED_RDMA_MAX_CQE_32_BIT             (0x7FFFFFFF - 1)
+#define QED_RDMA_MAX_CQE_16_BIT             (0x7FFF - 1)
+
+enum qed_rdma_toggle_bit {
+	QED_RDMA_TOGGLE_BIT_CLEAR = 0,
+	QED_RDMA_TOGGLE_BIT_SET = 1
+};
+
+struct qed_bmap {
+	unsigned long *bitmap;
+	u32 max_count;
+};
+
+struct qed_rdma_info {
+	/* spin lock to protect bitmaps */
+	spinlock_t lock;
+
+	struct qed_bmap cq_map;
+	struct qed_bmap pd_map;
+	struct qed_bmap tid_map;
+	struct qed_bmap qp_map;
+	struct qed_bmap srq_map;
+	struct qed_bmap cid_map;
+	struct qed_bmap dpi_map;
+	struct qed_bmap toggle_bits;
+	struct qed_rdma_events events;
+	struct qed_rdma_device *dev;
+	struct qed_rdma_port *port;
+	u32 last_tid;
+	u8 num_cnqs;
+	u32 num_qps;
+	u32 num_mrs;
+	u16 queue_zone_base;
+	enum protocol_type proto;
+};
+
+struct qed_rdma_resize_cq_in_params {
+	u16 icid;
+	u32 cq_size;
+	bool pbl_two_level;
+	u64 pbl_ptr;
+	u16 pbl_num_pages;
+	u8 pbl_page_size_log;
+};
+
+struct qed_rdma_resize_cq_out_params {
+	u32 prod;
+	u32 cons;
+};
+
+struct qed_rdma_resize_cnq_in_params {
+	u32 cnq_id;
+	u32 pbl_page_size_log;
+	u64 pbl_ptr;
+};
+
+struct qed_rdma_qp {
+	struct regpair qp_handle;
+	struct regpair qp_handle_async;
+	u32 qpid;
+	u16 icid;
+	enum qed_roce_qp_state cur_state;
+	bool use_srq;
+	bool signal_all;
+	bool fmr_and_reserved_lkey;
+
+	bool incoming_rdma_read_en;
+	bool incoming_rdma_write_en;
+	bool incoming_atomic_en;
+	bool e2e_flow_control_en;
+
+	u16 pd;
+	u16 pkey;
+	u32 dest_qp;
+	u16 mtu;
+	u16 srq_id;
+	u8 traffic_class_tos;
+	u8 hop_limit_ttl;
+	u16 dpi;
+	u32 flow_label;
+	bool lb_indication;
+	u16 vlan_id;
+	u32 ack_timeout;
+	u8 retry_cnt;
+	u8 rnr_retry_cnt;
+	u8 min_rnr_nak_timer;
+	bool sqd_async;
+	union qed_gid sgid;
+	union qed_gid dgid;
+	enum roce_mode roce_mode;
+	u16 udp_src_port;
+	u8 stats_queue;
+
+	/* requeseter */
+	u8 max_rd_atomic_req;
+	u32 sq_psn;
+	u16 sq_cq_id;
+	u16 sq_num_pages;
+	dma_addr_t sq_pbl_ptr;
+	void *orq;
+	dma_addr_t orq_phys_addr;
+	u8 orq_num_pages;
+	bool req_offloaded;
+
+	/* responder */
+	u8 max_rd_atomic_resp;
+	u32 rq_psn;
+	u16 rq_cq_id;
+	u16 rq_num_pages;
+	dma_addr_t rq_pbl_ptr;
+	void *irq;
+	dma_addr_t irq_phys_addr;
+	u8 irq_num_pages;
+	bool resp_offloaded;
+
+	u8 remote_mac_addr[6];
+	u8 local_mac_addr[6];
+
+	void *shared_queue;
+	dma_addr_t shared_queue_phys_addr;
+};
+
+int
+qed_rdma_add_user(void *rdma_cxt,
+		  struct qed_rdma_add_user_out_params *out_params);
+int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd);
+int qed_rdma_alloc_tid(void *rdma_cxt, u32 *tid);
+int qed_rdma_deregister_tid(void *rdma_cxt, u32 tid);
+void qed_rdma_free_tid(void *rdma_cxt, u32 tid);
+struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt);
+struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt);
+int
+qed_rdma_register_tid(void *rdma_cxt,
+		      struct qed_rdma_register_tid_in_params *params);
+void qed_rdma_remove_user(void *rdma_cxt, u16 dpi);
+int qed_rdma_start(void *p_hwfn, struct qed_rdma_start_in_params *params);
+int qed_rdma_stop(void *rdma_cxt);
+u32 qed_rdma_get_sb_id(void *p_hwfn, u32 rel_sb_id);
+u32 qed_rdma_query_cau_timer_res(void *p_hwfn);
+void qed_rdma_cnq_prod_update(void *rdma_cxt, u8 cnq_index, u16 prod);
+void qed_rdma_resc_free(struct qed_hwfn *p_hwfn);
+void qed_async_roce_event(struct qed_hwfn *p_hwfn,
+			  struct event_ring_entry *p_eqe);
+int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp);
+int qed_rdma_modify_qp(void *rdma_cxt, struct qed_rdma_qp *qp,
+		       struct qed_rdma_modify_qp_in_params *params);
+int qed_rdma_query_qp(void *rdma_cxt, struct qed_rdma_qp *qp,
+		      struct qed_rdma_query_qp_out_params *out_params);
+
+#if IS_ENABLED(CONFIG_INFINIBAND_QEDR)
+void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
+#else
+void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) {}
+#endif
+#endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_selftest.c b/drivers/net/ethernet/qlogic/qed/qed_selftest.c
index a342bfe4280d..9b7678f26909 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_selftest.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_selftest.c
@@ -2,6 +2,7 @@
 #include "qed_dev_api.h"
 #include "qed_mcp.h"
 #include "qed_sp.h"
+#include "qed_selftest.h"
 
 int qed_selftest_memory(struct qed_dev *cdev)
 {
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp.h b/drivers/net/ethernet/qlogic/qed/qed_sp.h
index a548504c3420..652c90819758 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sp.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_sp.h
@@ -61,6 +61,10 @@ union ramrod_data {
 	struct vport_start_ramrod_data vport_start;
 	struct vport_stop_ramrod_data vport_stop;
 	struct vport_update_ramrod_data vport_update;
+	struct core_rx_start_ramrod_data core_rx_queue_start;
+	struct core_rx_stop_ramrod_data core_rx_queue_stop;
+	struct core_tx_start_ramrod_data core_tx_queue_start;
+	struct core_tx_stop_ramrod_data core_tx_queue_stop;
 	struct vport_filter_update_ramrod_data vport_filter_update;
 
 	struct rdma_init_func_ramrod_data rdma_init_func;
@@ -81,6 +85,7 @@ union ramrod_data {
 	struct rdma_srq_create_ramrod_data rdma_create_srq;
 	struct rdma_srq_destroy_ramrod_data rdma_destroy_srq;
 	struct rdma_srq_modify_ramrod_data rdma_modify_srq;
+	struct roce_init_func_ramrod_data roce_init_func;
 
 	struct iscsi_slow_path_hdr iscsi_empty;
 	struct iscsi_init_ramrod_params iscsi_init;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
index a52f3fc051f5..2888eb0628f8 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
@@ -25,9 +25,7 @@
 
 int qed_sp_init_request(struct qed_hwfn *p_hwfn,
 			struct qed_spq_entry **pp_ent,
-			u8 cmd,
-			u8 protocol,
-			struct qed_sp_init_data *p_data)
+			u8 cmd, u8 protocol, struct qed_sp_init_data *p_data)
 {
 	u32 opaque_cid = p_data->opaque_fid << 16 | p_data->cid;
 	struct qed_spq_entry *p_ent = NULL;
@@ -38,7 +36,7 @@ int qed_sp_init_request(struct qed_hwfn *p_hwfn,
 
 	rc = qed_spq_get_entry(p_hwfn, pp_ent);
 
-	if (rc != 0)
+	if (rc)
 		return rc;
 
 	p_ent = *pp_ent;
@@ -321,8 +319,7 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 
 	rc = qed_sp_init_request(p_hwfn, &p_ent,
 				 COMMON_RAMROD_PF_START,
-				 PROTOCOLID_COMMON,
-				 &init_data);
+				 PROTOCOLID_COMMON, &init_data);
 	if (rc)
 		return rc;
 
@@ -356,8 +353,7 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 	DMA_REGPAIR_LE(p_ramrod->consolid_q_pbl_addr,
 		       p_hwfn->p_consq->chain.pbl.p_phys_table);
 
-	qed_tunn_set_pf_start_params(p_hwfn, p_tunn,
-				     &p_ramrod->tunnel_config);
+	qed_tunn_set_pf_start_params(p_hwfn, p_tunn, &p_ramrod->tunnel_config);
 
 	if (IS_MF_SI(p_hwfn))
 		p_ramrod->allow_npar_tx_switching = allow_npar_tx_switch;
@@ -389,8 +385,7 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 
 	DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 		   "Setting event_ring_sb [id %04x index %02x], outer_tag [%d]\n",
-		   sb, sb_index,
-		   p_ramrod->outer_tag);
+		   sb, sb_index, p_ramrod->outer_tag);
 
 	rc = qed_spq_post(p_hwfn, p_ent, NULL);
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_spq.c b/drivers/net/ethernet/qlogic/qed/qed_spq.c
index d73456eab1d7..caff41544898 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_spq.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_spq.c
@@ -28,6 +28,9 @@
 #include "qed_reg_addr.h"
 #include "qed_sp.h"
 #include "qed_sriov.h"
+#if IS_ENABLED(CONFIG_INFINIBAND_QEDR)
+#include "qed_roce.h"
+#endif
 
 /***************************************************************************
 * Structures & Definitions
@@ -41,8 +44,7 @@
 ***************************************************************************/
 static void qed_spq_blocking_cb(struct qed_hwfn *p_hwfn,
 				void *cookie,
-				union event_ring_data *data,
-				u8 fw_return_code)
+				union event_ring_data *data, u8 fw_return_code)
 {
 	struct qed_spq_comp_done *comp_done;
 
@@ -109,9 +111,8 @@ static int qed_spq_block(struct qed_hwfn *p_hwfn,
 /***************************************************************************
 * SPQ entries inner API
 ***************************************************************************/
-static int
-qed_spq_fill_entry(struct qed_hwfn *p_hwfn,
-		   struct qed_spq_entry *p_ent)
+static int qed_spq_fill_entry(struct qed_hwfn *p_hwfn,
+			      struct qed_spq_entry *p_ent)
 {
 	p_ent->flags = 0;
 
@@ -189,8 +190,7 @@ static void qed_spq_hw_initialize(struct qed_hwfn *p_hwfn,
 }
 
 static int qed_spq_hw_post(struct qed_hwfn *p_hwfn,
-			   struct qed_spq *p_spq,
-			   struct qed_spq_entry *p_ent)
+			   struct qed_spq *p_spq, struct qed_spq_entry *p_ent)
 {
 	struct qed_chain *p_chain = &p_hwfn->p_spq->chain;
 	u16 echo = qed_chain_get_prod_idx(p_chain);
@@ -240,6 +240,11 @@ qed_async_event_completion(struct qed_hwfn *p_hwfn,
 			   struct event_ring_entry *p_eqe)
 {
 	switch (p_eqe->protocol_id) {
+#if IS_ENABLED(CONFIG_INFINIBAND_QEDR)
+	case PROTOCOLID_ROCE:
+		qed_async_roce_event(p_hwfn, p_eqe);
+		return 0;
+#endif
 	case PROTOCOLID_COMMON:
 		return qed_sriov_eqe_event(p_hwfn,
 					   p_eqe->opcode,
@@ -255,8 +260,7 @@ qed_async_event_completion(struct qed_hwfn *p_hwfn,
 /***************************************************************************
 * EQ API
 ***************************************************************************/
-void qed_eq_prod_update(struct qed_hwfn *p_hwfn,
-			u16 prod)
+void qed_eq_prod_update(struct qed_hwfn *p_hwfn, u16 prod)
 {
 	u32 addr = GTT_BAR0_MAP_REG_USDM_RAM +
 		   USTORM_EQE_CONS_OFFSET(p_hwfn->rel_pf_id);
@@ -267,9 +271,7 @@ void qed_eq_prod_update(struct qed_hwfn *p_hwfn,
 	mmiowb();
 }
 
-int qed_eq_completion(struct qed_hwfn *p_hwfn,
-		      void *cookie)
-
+int qed_eq_completion(struct qed_hwfn *p_hwfn, void *cookie)
 {
 	struct qed_eq *p_eq = cookie;
 	struct qed_chain *p_chain = &p_eq->chain;
@@ -323,17 +325,14 @@ int qed_eq_completion(struct qed_hwfn *p_hwfn,
 	return rc;
 }
 
-struct qed_eq *qed_eq_alloc(struct qed_hwfn *p_hwfn,
-			    u16 num_elem)
+struct qed_eq *qed_eq_alloc(struct qed_hwfn *p_hwfn, u16 num_elem)
 {
 	struct qed_eq *p_eq;
 
 	/* Allocate EQ struct */
 	p_eq = kzalloc(sizeof(*p_eq), GFP_KERNEL);
-	if (!p_eq) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `struct qed_eq'\n");
+	if (!p_eq)
 		return NULL;
-	}
 
 	/* Allocate and initialize EQ chain*/
 	if (qed_chain_alloc(p_hwfn->cdev,
@@ -342,17 +341,12 @@ struct qed_eq *qed_eq_alloc(struct qed_hwfn *p_hwfn,
 			    QED_CHAIN_CNT_TYPE_U16,
 			    num_elem,
 			    sizeof(union event_ring_element),
-			    &p_eq->chain)) {
-		DP_NOTICE(p_hwfn, "Failed to allocate eq chain\n");
+			    &p_eq->chain))
 		goto eq_allocate_fail;
-	}
 
 	/* register EQ completion on the SP SB */
-	qed_int_register_cb(p_hwfn,
-			    qed_eq_completion,
-			    p_eq,
-			    &p_eq->eq_sb_index,
-			    &p_eq->p_fw_cons);
+	qed_int_register_cb(p_hwfn, qed_eq_completion,
+			    p_eq, &p_eq->eq_sb_index, &p_eq->p_fw_cons);
 
 	return p_eq;
 
@@ -361,14 +355,12 @@ eq_allocate_fail:
 	return NULL;
 }
 
-void qed_eq_setup(struct qed_hwfn *p_hwfn,
-		  struct qed_eq *p_eq)
+void qed_eq_setup(struct qed_hwfn *p_hwfn, struct qed_eq *p_eq)
 {
 	qed_chain_reset(&p_eq->chain);
 }
 
-void qed_eq_free(struct qed_hwfn *p_hwfn,
-		 struct qed_eq *p_eq)
+void qed_eq_free(struct qed_hwfn *p_hwfn, struct qed_eq *p_eq)
 {
 	if (!p_eq)
 		return;
@@ -379,10 +371,9 @@ void qed_eq_free(struct qed_hwfn *p_hwfn,
 /***************************************************************************
 * CQE API - manipulate EQ functionality
 ***************************************************************************/
-static int qed_cqe_completion(
-	struct qed_hwfn *p_hwfn,
-	struct eth_slow_path_rx_cqe *cqe,
-	enum protocol_type protocol)
+static int qed_cqe_completion(struct qed_hwfn *p_hwfn,
+			      struct eth_slow_path_rx_cqe *cqe,
+			      enum protocol_type protocol)
 {
 	if (IS_VF(p_hwfn->cdev))
 		return 0;
@@ -463,12 +454,9 @@ int qed_spq_alloc(struct qed_hwfn *p_hwfn)
 	u32 capacity;
 
 	/* SPQ struct */
-	p_spq =
-		kzalloc(sizeof(struct qed_spq), GFP_KERNEL);
-	if (!p_spq) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `struct qed_spq'\n");
+	p_spq = kzalloc(sizeof(struct qed_spq), GFP_KERNEL);
+	if (!p_spq)
 		return -ENOMEM;
-	}
 
 	/* SPQ ring  */
 	if (qed_chain_alloc(p_hwfn->cdev,
@@ -477,18 +465,14 @@ int qed_spq_alloc(struct qed_hwfn *p_hwfn)
 			    QED_CHAIN_CNT_TYPE_U16,
 			    0,   /* N/A when the mode is SINGLE */
 			    sizeof(struct slow_path_element),
-			    &p_spq->chain)) {
-		DP_NOTICE(p_hwfn, "Failed to allocate spq chain\n");
+			    &p_spq->chain))
 		goto spq_allocate_fail;
-	}
 
 	/* allocate and fill the SPQ elements (incl. ramrod data list) */
 	capacity = qed_chain_get_capacity(&p_spq->chain);
 	p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
-				    capacity *
-				    sizeof(struct qed_spq_entry),
+				    capacity * sizeof(struct qed_spq_entry),
 				    &p_phys, GFP_KERNEL);
-
 	if (!p_virt)
 		goto spq_allocate_fail;
 
@@ -525,9 +509,7 @@ void qed_spq_free(struct qed_hwfn *p_hwfn)
 	kfree(p_spq);
 }
 
-int
-qed_spq_get_entry(struct qed_hwfn *p_hwfn,
-		  struct qed_spq_entry **pp_ent)
+int qed_spq_get_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry **pp_ent)
 {
 	struct qed_spq *p_spq = p_hwfn->p_spq;
 	struct qed_spq_entry *p_ent = NULL;
@@ -538,14 +520,15 @@ qed_spq_get_entry(struct qed_hwfn *p_hwfn,
 	if (list_empty(&p_spq->free_pool)) {
 		p_ent = kzalloc(sizeof(*p_ent), GFP_ATOMIC);
 		if (!p_ent) {
+			DP_NOTICE(p_hwfn,
+				  "Failed to allocate an SPQ entry for a pending ramrod\n");
 			rc = -ENOMEM;
 			goto out_unlock;
 		}
 		p_ent->queue = &p_spq->unlimited_pending;
 	} else {
 		p_ent = list_first_entry(&p_spq->free_pool,
-					 struct qed_spq_entry,
-					 list);
+					 struct qed_spq_entry, list);
 		list_del(&p_ent->list);
 		p_ent->queue = &p_spq->pending;
 	}
@@ -564,8 +547,7 @@ static void __qed_spq_return_entry(struct qed_hwfn *p_hwfn,
 	list_add_tail(&p_ent->list, &p_hwfn->p_spq->free_pool);
 }
 
-void qed_spq_return_entry(struct qed_hwfn *p_hwfn,
-			  struct qed_spq_entry *p_ent)
+void qed_spq_return_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry *p_ent)
 {
 	spin_lock_bh(&p_hwfn->p_spq->lock);
 	__qed_spq_return_entry(p_hwfn, p_ent);
@@ -586,10 +568,9 @@ void qed_spq_return_entry(struct qed_hwfn *p_hwfn,
  *
  * @return int
  */
-static int
-qed_spq_add_entry(struct qed_hwfn *p_hwfn,
-		  struct qed_spq_entry *p_ent,
-		  enum spq_priority priority)
+static int qed_spq_add_entry(struct qed_hwfn *p_hwfn,
+			     struct qed_spq_entry *p_ent,
+			     enum spq_priority priority)
 {
 	struct qed_spq *p_spq = p_hwfn->p_spq;
 
@@ -604,8 +585,7 @@ qed_spq_add_entry(struct qed_hwfn *p_hwfn,
 			struct qed_spq_entry *p_en2;
 
 			p_en2 = list_first_entry(&p_spq->free_pool,
-						 struct qed_spq_entry,
-						 list);
+						 struct qed_spq_entry, list);
 			list_del(&p_en2->list);
 
 			/* Copy the ring element physical pointer to the new
@@ -655,8 +635,7 @@ u32 qed_spq_get_cid(struct qed_hwfn *p_hwfn)
 * Posting new Ramrods
 ***************************************************************************/
 static int qed_spq_post_list(struct qed_hwfn *p_hwfn,
-			     struct list_head *head,
-			     u32 keep_reserve)
+			     struct list_head *head, u32 keep_reserve)
 {
 	struct qed_spq *p_spq = p_hwfn->p_spq;
 	int rc;
@@ -690,8 +669,7 @@ static int qed_spq_pend_post(struct qed_hwfn *p_hwfn)
 			break;
 
 		p_ent = list_first_entry(&p_spq->unlimited_pending,
-					 struct qed_spq_entry,
-					 list);
+					 struct qed_spq_entry, list);
 		if (!p_ent)
 			return -EINVAL;
 
@@ -705,8 +683,7 @@ static int qed_spq_pend_post(struct qed_hwfn *p_hwfn)
 }
 
 int qed_spq_post(struct qed_hwfn *p_hwfn,
-		 struct qed_spq_entry *p_ent,
-		 u8 *fw_return_code)
+		 struct qed_spq_entry *p_ent, u8 *fw_return_code)
 {
 	int rc = 0;
 	struct qed_spq *p_spq = p_hwfn ? p_hwfn->p_spq : NULL;
@@ -803,8 +780,7 @@ int qed_spq_completion(struct qed_hwfn *p_hwfn,
 		return -EINVAL;
 
 	spin_lock_bh(&p_spq->lock);
-	list_for_each_entry_safe(p_ent, tmp, &p_spq->completion_pending,
-				 list) {
+	list_for_each_entry_safe(p_ent, tmp, &p_spq->completion_pending, list) {
 		if (p_ent->elem.hdr.echo == echo) {
 			u16 pos = le16_to_cpu(echo) % SPQ_RING_SIZE;
 
@@ -846,15 +822,22 @@ int qed_spq_completion(struct qed_hwfn *p_hwfn,
 
 	if (!found) {
 		DP_NOTICE(p_hwfn,
-			  "Failed to find an entry this EQE completes\n");
+			  "Failed to find an entry this EQE [echo %04x] completes\n",
+			  le16_to_cpu(echo));
 		return -EEXIST;
 	}
 
-	DP_VERBOSE(p_hwfn, QED_MSG_SPQ, "Complete: func %p cookie %p)\n",
+	DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
+		   "Complete EQE [echo %04x]: func %p cookie %p)\n",
+		   le16_to_cpu(echo),
 		   p_ent->comp_cb.function, p_ent->comp_cb.cookie);
 	if (found->comp_cb.function)
 		found->comp_cb.function(p_hwfn, found->comp_cb.cookie, p_data,
 					fw_return_code);
+	else
+		DP_VERBOSE(p_hwfn,
+			   QED_MSG_SPQ,
+			   "Got a completion without a callback function\n");
 
 	if ((found->comp_mode != QED_SPQ_MODE_EBLOCK) ||
 	    (found->queue == &p_spq->unlimited_pending))
@@ -878,10 +861,8 @@ struct qed_consq *qed_consq_alloc(struct qed_hwfn *p_hwfn)
 
 	/* Allocate ConsQ struct */
 	p_consq = kzalloc(sizeof(*p_consq), GFP_KERNEL);
-	if (!p_consq) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `struct qed_consq'\n");
+	if (!p_consq)
 		return NULL;
-	}
 
 	/* Allocate and initialize EQ chain*/
 	if (qed_chain_alloc(p_hwfn->cdev,
@@ -889,10 +870,8 @@ struct qed_consq *qed_consq_alloc(struct qed_hwfn *p_hwfn)
 			    QED_CHAIN_MODE_PBL,
 			    QED_CHAIN_CNT_TYPE_U16,
 			    QED_CHAIN_PAGE_SIZE / 0x80,
-			    0x80, &p_consq->chain)) {
-		DP_NOTICE(p_hwfn, "Failed to allocate consq chain");
+			    0x80, &p_consq->chain))
 		goto consq_allocate_fail;
-	}
 
 	return p_consq;
 
@@ -901,14 +880,12 @@ consq_allocate_fail:
 	return NULL;
 }
 
-void qed_consq_setup(struct qed_hwfn *p_hwfn,
-		     struct qed_consq *p_consq)
+void qed_consq_setup(struct qed_hwfn *p_hwfn, struct qed_consq *p_consq)
 {
 	qed_chain_reset(&p_consq->chain);
 }
 
-void qed_consq_free(struct qed_hwfn *p_hwfn,
-		    struct qed_consq *p_consq)
+void qed_consq_free(struct qed_hwfn *p_hwfn, struct qed_consq *p_consq)
 {
 	if (!p_consq)
 		return;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.c b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
index 15399da268d9..d2d6621fe0e5 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
@@ -60,7 +60,8 @@ static int qed_sp_vf_start(struct qed_hwfn *p_hwfn, struct qed_vf_info *p_vf)
 	}
 
 	fp_minor = p_vf->acquire.vfdev_info.eth_fp_hsi_minor;
-	if (fp_minor > ETH_HSI_VER_MINOR) {
+	if (fp_minor > ETH_HSI_VER_MINOR &&
+	    fp_minor != ETH_HSI_VER_NO_PKT_LEN_TUNN) {
 		DP_VERBOSE(p_hwfn,
 			   QED_MSG_IOV,
 			   "VF [%d] - Requested fp hsi %02x.%02x which is slightly newer than PF's %02x.%02x; Configuring PFs version\n",
@@ -107,8 +108,8 @@ static int qed_sp_vf_stop(struct qed_hwfn *p_hwfn,
 	return qed_spq_post(p_hwfn, p_ent, NULL);
 }
 
-bool qed_iov_is_valid_vfid(struct qed_hwfn *p_hwfn,
-			   int rel_vf_id, bool b_enabled_only)
+static bool qed_iov_is_valid_vfid(struct qed_hwfn *p_hwfn,
+				  int rel_vf_id, bool b_enabled_only)
 {
 	if (!p_hwfn->pf_iov_info) {
 		DP_NOTICE(p_hwfn->cdev, "No iov info\n");
@@ -185,8 +186,8 @@ static bool qed_iov_validate_sb(struct qed_hwfn *p_hwfn,
 	return false;
 }
 
-int qed_iov_post_vf_bulletin(struct qed_hwfn *p_hwfn,
-			     int vfid, struct qed_ptt *p_ptt)
+static int qed_iov_post_vf_bulletin(struct qed_hwfn *p_hwfn,
+				    int vfid, struct qed_ptt *p_ptt)
 {
 	struct qed_bulletin_content *p_bulletin;
 	int crc_size = sizeof(p_bulletin->crc);
@@ -454,10 +455,8 @@ int qed_iov_alloc(struct qed_hwfn *p_hwfn)
 	}
 
 	p_sriov = kzalloc(sizeof(*p_sriov), GFP_KERNEL);
-	if (!p_sriov) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `struct qed_sriov'\n");
+	if (!p_sriov)
 		return -ENOMEM;
-	}
 
 	p_hwfn->pf_iov_info = p_sriov;
 
@@ -506,10 +505,9 @@ int qed_iov_hw_info(struct qed_hwfn *p_hwfn)
 
 	/* Allocate a new struct for IOV information */
 	cdev->p_iov_info = kzalloc(sizeof(*cdev->p_iov_info), GFP_KERNEL);
-	if (!cdev->p_iov_info) {
-		DP_NOTICE(p_hwfn, "Can't support IOV due to lack of memory\n");
+	if (!cdev->p_iov_info)
 		return -ENOMEM;
-	}
+
 	cdev->p_iov_info->pos = pos;
 
 	rc = qed_iov_pci_cfg_info(cdev);
@@ -575,7 +573,7 @@ static void qed_iov_set_vf_to_disable(struct qed_dev *cdev,
 	}
 }
 
-void qed_iov_set_vfs_to_disable(struct qed_dev *cdev, u8 to_disable)
+static void qed_iov_set_vfs_to_disable(struct qed_dev *cdev, u8 to_disable)
 {
 	u16 i;
 
@@ -699,7 +697,7 @@ static void qed_iov_config_perm_table(struct qed_hwfn *p_hwfn,
 				&qzone_id);
 
 		reg_addr = PSWHST_REG_ZONE_PERMISSION_TABLE + qzone_id * 4;
-		val = enable ? (vf->abs_vf_id | (1 << 8)) : 0;
+		val = enable ? (vf->abs_vf_id | BIT(8)) : 0;
 		qed_wr(p_hwfn, p_ptt, reg_addr, val);
 	}
 }
@@ -1090,13 +1088,13 @@ static u16 qed_iov_prep_vp_update_resp_tlvs(struct qed_hwfn *p_hwfn,
 
 	/* Prepare response for all extended tlvs if they are found by PF */
 	for (i = 0; i < QED_IOV_VP_UPDATE_MAX; i++) {
-		if (!(tlvs_mask & (1 << i)))
+		if (!(tlvs_mask & BIT(i)))
 			continue;
 
 		resp = qed_add_tlv(p_hwfn, &p_mbx->offset,
 				   qed_iov_vport_to_tlv(p_hwfn, i), size);
 
-		if (tlvs_accepted & (1 << i))
+		if (tlvs_accepted & BIT(i))
 			resp->hdr.status = status;
 		else
 			resp->hdr.status = PFVF_STATUS_NOT_SUPPORTED;
@@ -1132,9 +1130,10 @@ static void qed_iov_prepare_resp(struct qed_hwfn *p_hwfn,
 	qed_iov_send_response(p_hwfn, p_ptt, vf_info, length, status);
 }
 
-struct qed_public_vf_info *qed_iov_get_public_vf_info(struct qed_hwfn *p_hwfn,
-						      u16 relative_vf_id,
-						      bool b_enabled_only)
+static struct
+qed_public_vf_info *qed_iov_get_public_vf_info(struct qed_hwfn *p_hwfn,
+					       u16 relative_vf_id,
+					       bool b_enabled_only)
 {
 	struct qed_vf_info *vf = NULL;
 
@@ -1145,7 +1144,7 @@ struct qed_public_vf_info *qed_iov_get_public_vf_info(struct qed_hwfn *p_hwfn,
 	return &vf->p_vf_info;
 }
 
-void qed_iov_clean_vf(struct qed_hwfn *p_hwfn, u8 vfid)
+static void qed_iov_clean_vf(struct qed_hwfn *p_hwfn, u8 vfid)
 {
 	struct qed_public_vf_info *vf_info;
 
@@ -1241,6 +1240,16 @@ static u8 qed_iov_vf_mbx_acquire_resc(struct qed_hwfn *p_hwfn,
 			   p_req->num_vlan_filters,
 			   p_resp->num_vlan_filters,
 			   p_req->num_mc_filters, p_resp->num_mc_filters);
+
+		/* Some legacy OSes are incapable of correctly handling this
+		 * failure.
+		 */
+		if ((p_vf->acquire.vfdev_info.eth_fp_hsi_minor ==
+		     ETH_HSI_VER_NO_PKT_LEN_TUNN) &&
+		    (p_vf->acquire.vfdev_info.os_type ==
+		     VFPF_ACQUIRE_OS_WINDOWS))
+			return PFVF_STATUS_SUCCESS;
+
 		return PFVF_STATUS_NO_RESOURCE;
 	}
 
@@ -1280,22 +1289,42 @@ static void qed_iov_vf_mbx_acquire(struct qed_hwfn *p_hwfn,
 
 	memset(resp, 0, sizeof(*resp));
 
+	/* Write the PF version so that VF would know which version
+	 * is supported - might be later overriden. This guarantees that
+	 * VF could recognize legacy PF based on lack of versions in reply.
+	 */
+	pfdev_info->major_fp_hsi = ETH_HSI_VER_MAJOR;
+	pfdev_info->minor_fp_hsi = ETH_HSI_VER_MINOR;
+
+	if (vf->state != VF_FREE && vf->state != VF_STOPPED) {
+		DP_VERBOSE(p_hwfn,
+			   QED_MSG_IOV,
+			   "VF[%d] sent ACQUIRE but is already in state %d - fail request\n",
+			   vf->abs_vf_id, vf->state);
+		goto out;
+	}
+
 	/* Validate FW compatibility */
 	if (req->vfdev_info.eth_fp_hsi_major != ETH_HSI_VER_MAJOR) {
-		DP_INFO(p_hwfn,
-			"VF[%d] needs fastpath HSI %02x.%02x, which is incompatible with loaded FW's faspath HSI %02x.%02x\n",
-			vf->abs_vf_id,
-			req->vfdev_info.eth_fp_hsi_major,
-			req->vfdev_info.eth_fp_hsi_minor,
-			ETH_HSI_VER_MAJOR, ETH_HSI_VER_MINOR);
-
-		/* Write the PF version so that VF would know which version
-		 * is supported.
-		 */
-		pfdev_info->major_fp_hsi = ETH_HSI_VER_MAJOR;
-		pfdev_info->minor_fp_hsi = ETH_HSI_VER_MINOR;
+		if (req->vfdev_info.capabilities &
+		    VFPF_ACQUIRE_CAP_PRE_FP_HSI) {
+			struct vf_pf_vfdev_info *p_vfdev = &req->vfdev_info;
 
-		goto out;
+			DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+				   "VF[%d] is pre-fastpath HSI\n",
+				   vf->abs_vf_id);
+			p_vfdev->eth_fp_hsi_major = ETH_HSI_VER_MAJOR;
+			p_vfdev->eth_fp_hsi_minor = ETH_HSI_VER_NO_PKT_LEN_TUNN;
+		} else {
+			DP_INFO(p_hwfn,
+				"VF[%d] needs fastpath HSI %02x.%02x, which is incompatible with loaded FW's faspath HSI %02x.%02x\n",
+				vf->abs_vf_id,
+				req->vfdev_info.eth_fp_hsi_major,
+				req->vfdev_info.eth_fp_hsi_minor,
+				ETH_HSI_VER_MAJOR, ETH_HSI_VER_MINOR);
+
+			goto out;
+		}
 	}
 
 	/* On 100g PFs, prevent old VFs from loading */
@@ -1334,8 +1363,11 @@ static void qed_iov_vf_mbx_acquire(struct qed_hwfn *p_hwfn,
 	pfdev_info->fw_minor = FW_MINOR_VERSION;
 	pfdev_info->fw_rev = FW_REVISION_VERSION;
 	pfdev_info->fw_eng = FW_ENGINEERING_VERSION;
-	pfdev_info->minor_fp_hsi = min_t(u8,
-					 ETH_HSI_VER_MINOR,
+
+	/* Incorrect when legacy, but doesn't matter as legacy isn't reading
+	 * this field.
+	 */
+	pfdev_info->minor_fp_hsi = min_t(u8, ETH_HSI_VER_MINOR,
 					 req->vfdev_info.eth_fp_hsi_minor);
 	pfdev_info->os_type = VFPF_ACQUIRE_OS_LINUX;
 	qed_mcp_get_mfw_ver(p_hwfn, p_ptt, &pfdev_info->mfw_ver, NULL);
@@ -1438,14 +1470,11 @@ static int qed_iov_reconfigure_unicast_vlan(struct qed_hwfn *p_hwfn,
 
 		filter.type = QED_FILTER_VLAN;
 		filter.vlan = p_vf->shadow_config.vlans[i].vid;
-		DP_VERBOSE(p_hwfn,
-			   QED_MSG_IOV,
+		DP_VERBOSE(p_hwfn, QED_MSG_IOV,
 			   "Reconfiguring VLAN [0x%04x] for VF [%04x]\n",
 			   filter.vlan, p_vf->relative_vf_id);
-		rc = qed_sp_eth_filter_ucast(p_hwfn,
-					     p_vf->opaque_fid,
-					     &filter,
-					     QED_SPQ_MODE_CB, NULL);
+		rc = qed_sp_eth_filter_ucast(p_hwfn, p_vf->opaque_fid,
+					     &filter, QED_SPQ_MODE_CB, NULL);
 		if (rc) {
 			DP_NOTICE(p_hwfn,
 				  "Failed to configure VLAN [%04x] to VF [%04x]\n",
@@ -1463,7 +1492,7 @@ qed_iov_reconfigure_unicast_shadow(struct qed_hwfn *p_hwfn,
 {
 	int rc = 0;
 
-	if ((events & (1 << VLAN_ADDR_FORCED)) &&
+	if ((events & BIT(VLAN_ADDR_FORCED)) &&
 	    !(p_vf->configured_features & (1 << VLAN_ADDR_FORCED)))
 		rc = qed_iov_reconfigure_unicast_vlan(p_hwfn, p_vf);
 
@@ -1479,7 +1508,7 @@ static int qed_iov_configure_vport_forced(struct qed_hwfn *p_hwfn,
 	if (!p_vf->vport_instance)
 		return -EINVAL;
 
-	if (events & (1 << MAC_ADDR_FORCED)) {
+	if (events & BIT(MAC_ADDR_FORCED)) {
 		/* Since there's no way [currently] of removing the MAC,
 		 * we can always assume this means we need to force it.
 		 */
@@ -1502,7 +1531,7 @@ static int qed_iov_configure_vport_forced(struct qed_hwfn *p_hwfn,
 		p_vf->configured_features |= 1 << MAC_ADDR_FORCED;
 	}
 
-	if (events & (1 << VLAN_ADDR_FORCED)) {
+	if (events & BIT(VLAN_ADDR_FORCED)) {
 		struct qed_sp_vport_update_params vport_update;
 		u8 removal;
 		int i;
@@ -1572,7 +1601,7 @@ static int qed_iov_configure_vport_forced(struct qed_hwfn *p_hwfn,
 		if (filter.vlan)
 			p_vf->configured_features |= 1 << VLAN_ADDR_FORCED;
 		else
-			p_vf->configured_features &= ~(1 << VLAN_ADDR_FORCED);
+			p_vf->configured_features &= ~BIT(VLAN_ADDR_FORCED);
 	}
 
 	/* If forced features are terminated, we need to configure the shadow
@@ -1619,8 +1648,7 @@ static void qed_iov_vf_mbx_start_vport(struct qed_hwfn *p_hwfn,
 
 		qed_int_cau_conf_sb(p_hwfn, p_ptt,
 				    start->sb_addr[sb_id],
-				    vf->igu_sbs[sb_id],
-				    vf->abs_vf_id, 1);
+				    vf->igu_sbs[sb_id], vf->abs_vf_id, 1);
 	}
 	qed_iov_enable_vf_traffic(p_hwfn, p_ptt, vf);
 
@@ -1632,7 +1660,7 @@ static void qed_iov_vf_mbx_start_vport(struct qed_hwfn *p_hwfn,
 	 * vfs that would still be fine, since they passed '0' as padding].
 	 */
 	p_bitmap = &vf_info->bulletin.p_virt->valid_bitmap;
-	if (!(*p_bitmap & (1 << VFPF_BULLETIN_UNTAGGED_DEFAULT_FORCED))) {
+	if (!(*p_bitmap & BIT(VFPF_BULLETIN_UNTAGGED_DEFAULT_FORCED))) {
 		u8 vf_req = start->only_untagged;
 
 		vf_info->bulletin.p_virt->default_only_untagged = vf_req;
@@ -1650,9 +1678,10 @@ static void qed_iov_vf_mbx_start_vport(struct qed_hwfn *p_hwfn,
 	params.vport_id = vf->vport_id;
 	params.max_buffers_per_cqe = start->max_buffers_per_cqe;
 	params.mtu = vf->mtu;
+	params.check_mac = true;
 
 	rc = qed_sp_eth_vport_start(p_hwfn, &params);
-	if (rc != 0) {
+	if (rc) {
 		DP_ERR(p_hwfn,
 		       "qed_iov_vf_mbx_start_vport returned error %d\n", rc);
 		status = PFVF_STATUS_FAILURE;
@@ -1679,7 +1708,7 @@ static void qed_iov_vf_mbx_stop_vport(struct qed_hwfn *p_hwfn,
 	vf->spoof_chk = false;
 
 	rc = qed_sp_vport_stop(p_hwfn, vf->opaque_fid, vf->vport_id);
-	if (rc != 0) {
+	if (rc) {
 		DP_ERR(p_hwfn, "qed_iov_vf_mbx_stop_vport returned error %d\n",
 		       rc);
 		status = PFVF_STATUS_FAILURE;
@@ -1695,21 +1724,32 @@ static void qed_iov_vf_mbx_stop_vport(struct qed_hwfn *p_hwfn,
 
 static void qed_iov_vf_mbx_start_rxq_resp(struct qed_hwfn *p_hwfn,
 					  struct qed_ptt *p_ptt,
-					  struct qed_vf_info *vf, u8 status)
+					  struct qed_vf_info *vf,
+					  u8 status, bool b_legacy)
 {
 	struct qed_iov_vf_mbx *mbx = &vf->vf_mbx;
 	struct pfvf_start_queue_resp_tlv *p_tlv;
 	struct vfpf_start_rxq_tlv *req;
+	u16 length;
 
 	mbx->offset = (u8 *)mbx->reply_virt;
 
+	/* Taking a bigger struct instead of adding a TLV to list was a
+	 * mistake, but one which we're now stuck with, as some older
+	 * clients assume the size of the previous response.
+	 */
+	if (!b_legacy)
+		length = sizeof(*p_tlv);
+	else
+		length = sizeof(struct pfvf_def_resp_tlv);
+
 	p_tlv = qed_add_tlv(p_hwfn, &mbx->offset, CHANNEL_TLV_START_RXQ,
-			    sizeof(*p_tlv));
+			    length);
 	qed_add_tlv(p_hwfn, &mbx->offset, CHANNEL_TLV_LIST_END,
 		    sizeof(struct channel_list_end_tlv));
 
 	/* Update the TLV with the response */
-	if (status == PFVF_STATUS_SUCCESS) {
+	if ((status == PFVF_STATUS_SUCCESS) && !b_legacy) {
 		req = &mbx->req_virt->start_rxq;
 		p_tlv->offset = PXP_VF_BAR0_START_MSDM_ZONE_B +
 				offsetof(struct mstorm_vf_zone,
@@ -1717,7 +1757,7 @@ static void qed_iov_vf_mbx_start_rxq_resp(struct qed_hwfn *p_hwfn,
 				sizeof(struct eth_rx_prod_data) * req->rx_qid;
 	}
 
-	qed_iov_send_response(p_hwfn, p_ptt, vf, sizeof(*p_tlv), status);
+	qed_iov_send_response(p_hwfn, p_ptt, vf, length, status);
 }
 
 static void qed_iov_vf_mbx_start_rxq(struct qed_hwfn *p_hwfn,
@@ -1728,6 +1768,7 @@ static void qed_iov_vf_mbx_start_rxq(struct qed_hwfn *p_hwfn,
 	struct qed_iov_vf_mbx *mbx = &vf->vf_mbx;
 	u8 status = PFVF_STATUS_NO_RESOURCE;
 	struct vfpf_start_rxq_tlv *req;
+	bool b_legacy_vf = false;
 	int rc;
 
 	memset(&params, 0, sizeof(params));
@@ -1743,13 +1784,27 @@ static void qed_iov_vf_mbx_start_rxq(struct qed_hwfn *p_hwfn,
 	params.sb = req->hw_sb;
 	params.sb_idx = req->sb_index;
 
+	/* Legacy VFs have their Producers in a different location, which they
+	 * calculate on their own and clean the producer prior to this.
+	 */
+	if (vf->acquire.vfdev_info.eth_fp_hsi_minor ==
+	    ETH_HSI_VER_NO_PKT_LEN_TUNN) {
+		b_legacy_vf = true;
+	} else {
+		REG_WR(p_hwfn,
+		       GTT_BAR0_MAP_REG_MSDM_RAM +
+		       MSTORM_ETH_VF_PRODS_OFFSET(vf->abs_vf_id, req->rx_qid),
+		       0);
+	}
+
 	rc = qed_sp_eth_rxq_start_ramrod(p_hwfn, vf->opaque_fid,
 					 vf->vf_queues[req->rx_qid].fw_cid,
 					 &params,
 					 vf->abs_vf_id + 0x10,
 					 req->bd_max_bytes,
 					 req->rxq_addr,
-					 req->cqe_pbl_addr, req->cqe_pbl_size);
+					 req->cqe_pbl_addr, req->cqe_pbl_size,
+					 b_legacy_vf);
 
 	if (rc) {
 		status = PFVF_STATUS_FAILURE;
@@ -1760,7 +1815,7 @@ static void qed_iov_vf_mbx_start_rxq(struct qed_hwfn *p_hwfn,
 	}
 
 out:
-	qed_iov_vf_mbx_start_rxq_resp(p_hwfn, p_ptt, vf, status);
+	qed_iov_vf_mbx_start_rxq_resp(p_hwfn, p_ptt, vf, status, b_legacy_vf);
 }
 
 static void qed_iov_vf_mbx_start_txq_resp(struct qed_hwfn *p_hwfn,
@@ -1769,23 +1824,38 @@ static void qed_iov_vf_mbx_start_txq_resp(struct qed_hwfn *p_hwfn,
 {
 	struct qed_iov_vf_mbx *mbx = &p_vf->vf_mbx;
 	struct pfvf_start_queue_resp_tlv *p_tlv;
+	bool b_legacy = false;
+	u16 length;
 
 	mbx->offset = (u8 *)mbx->reply_virt;
 
+	/* Taking a bigger struct instead of adding a TLV to list was a
+	 * mistake, but one which we're now stuck with, as some older
+	 * clients assume the size of the previous response.
+	 */
+	if (p_vf->acquire.vfdev_info.eth_fp_hsi_minor ==
+	    ETH_HSI_VER_NO_PKT_LEN_TUNN)
+		b_legacy = true;
+
+	if (!b_legacy)
+		length = sizeof(*p_tlv);
+	else
+		length = sizeof(struct pfvf_def_resp_tlv);
+
 	p_tlv = qed_add_tlv(p_hwfn, &mbx->offset, CHANNEL_TLV_START_TXQ,
-			    sizeof(*p_tlv));
+			    length);
 	qed_add_tlv(p_hwfn, &mbx->offset, CHANNEL_TLV_LIST_END,
 		    sizeof(struct channel_list_end_tlv));
 
 	/* Update the TLV with the response */
-	if (status == PFVF_STATUS_SUCCESS) {
+	if ((status == PFVF_STATUS_SUCCESS) && !b_legacy) {
 		u16 qid = mbx->req_virt->start_txq.tx_qid;
 
-		p_tlv->offset = qed_db_addr(p_vf->vf_queues[qid].fw_cid,
-					    DQ_DEMS_LEGACY);
+		p_tlv->offset = qed_db_addr_vf(p_vf->vf_queues[qid].fw_cid,
+					       DQ_DEMS_LEGACY);
 	}
 
-	qed_iov_send_response(p_hwfn, p_ptt, p_vf, sizeof(*p_tlv), status);
+	qed_iov_send_response(p_hwfn, p_ptt, p_vf, length, status);
 }
 
 static void qed_iov_vf_mbx_start_txq(struct qed_hwfn *p_hwfn,
@@ -2045,7 +2115,7 @@ qed_iov_vp_update_vlan_param(struct qed_hwfn *p_hwfn,
 	p_vf->shadow_config.inner_vlan_removal = p_vlan_tlv->remove_vlan;
 
 	/* Ignore the VF request if we're forcing a vlan */
-	if (!(p_vf->configured_features & (1 << VLAN_ADDR_FORCED))) {
+	if (!(p_vf->configured_features & BIT(VLAN_ADDR_FORCED))) {
 		p_data->update_inner_vlan_removal_flg = 1;
 		p_data->inner_vlan_removal_flg = p_vlan_tlv->remove_vlan;
 	}
@@ -2340,7 +2410,7 @@ static int qed_iov_vf_update_vlan_shadow(struct qed_hwfn *p_hwfn,
 	/* In forced mode, we're willing to remove entries - but we don't add
 	 * new ones.
 	 */
-	if (p_vf->bulletin.p_virt->valid_bitmap & (1 << VLAN_ADDR_FORCED))
+	if (p_vf->bulletin.p_virt->valid_bitmap & BIT(VLAN_ADDR_FORCED))
 		return 0;
 
 	if (p_params->opcode == QED_FILTER_ADD ||
@@ -2374,7 +2444,7 @@ static int qed_iov_vf_update_mac_shadow(struct qed_hwfn *p_hwfn,
 	int i;
 
 	/* If we're in forced-mode, we don't allow any change */
-	if (p_vf->bulletin.p_virt->valid_bitmap & (1 << MAC_ADDR_FORCED))
+	if (p_vf->bulletin.p_virt->valid_bitmap & BIT(MAC_ADDR_FORCED))
 		return 0;
 
 	/* First remove entries and then add new ones */
@@ -2441,8 +2511,8 @@ qed_iov_vf_update_unicast_shadow(struct qed_hwfn *p_hwfn,
 	return rc;
 }
 
-int qed_iov_chk_ucast(struct qed_hwfn *hwfn,
-		      int vfid, struct qed_filter_ucast *params)
+static int qed_iov_chk_ucast(struct qed_hwfn *hwfn,
+			     int vfid, struct qed_filter_ucast *params)
 {
 	struct qed_public_vf_info *vf;
 
@@ -2509,7 +2579,7 @@ static void qed_iov_vf_mbx_ucast_filter(struct qed_hwfn *p_hwfn,
 	}
 
 	/* Determine if the unicast filtering is acceptible by PF */
-	if ((p_bulletin->valid_bitmap & (1 << VLAN_ADDR_FORCED)) &&
+	if ((p_bulletin->valid_bitmap & BIT(VLAN_ADDR_FORCED)) &&
 	    (params.type == QED_FILTER_VLAN ||
 	     params.type == QED_FILTER_MAC_VLAN)) {
 		/* Once VLAN is forced or PVID is set, do not allow
@@ -2521,7 +2591,7 @@ static void qed_iov_vf_mbx_ucast_filter(struct qed_hwfn *p_hwfn,
 		goto out;
 	}
 
-	if ((p_bulletin->valid_bitmap & (1 << MAC_ADDR_FORCED)) &&
+	if ((p_bulletin->valid_bitmap & BIT(MAC_ADDR_FORCED)) &&
 	    (params.type == QED_FILTER_MAC ||
 	     params.type == QED_FILTER_MAC_VLAN)) {
 		if (!ether_addr_equal(p_bulletin->mac, params.mac) ||
@@ -2749,7 +2819,7 @@ cleanup:
 		/* Mark VF for ack and clean pending state */
 		if (p_vf->state == VF_RESET)
 			p_vf->state = VF_STOPPED;
-		ack_vfs[vfid / 32] |= (1 << (vfid % 32));
+		ack_vfs[vfid / 32] |= BIT((vfid % 32));
 		p_hwfn->pf_iov_info->pending_flr[rel_vf_id / 64] &=
 		    ~(1ULL << (rel_vf_id % 64));
 		p_hwfn->pf_iov_info->pending_events[rel_vf_id / 64] &=
@@ -2759,7 +2829,8 @@ cleanup:
 	return rc;
 }
 
-int qed_iov_vf_flr_cleanup(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+static int
+qed_iov_vf_flr_cleanup(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	u32 ack_vfs[VF_MAX_STATIC / 32];
 	int rc = 0;
@@ -2805,7 +2876,7 @@ int qed_iov_mark_vf_flr(struct qed_hwfn *p_hwfn, u32 *p_disabled_vfs)
 			continue;
 
 		vfid = p_vf->abs_vf_id;
-		if ((1 << (vfid % 32)) & p_disabled_vfs[vfid / 32]) {
+		if (BIT((vfid % 32)) & p_disabled_vfs[vfid / 32]) {
 			u64 *p_flr = p_hwfn->pf_iov_info->pending_flr;
 			u16 rel_vf_id = p_vf->relative_vf_id;
 
@@ -2946,7 +3017,7 @@ static void qed_iov_process_mbx_req(struct qed_hwfn *p_hwfn,
 	}
 }
 
-void qed_iov_pf_add_pending_events(struct qed_hwfn *p_hwfn, u8 vfid)
+static void qed_iov_pf_add_pending_events(struct qed_hwfn *p_hwfn, u8 vfid)
 {
 	u64 add_bit = 1ULL << (vfid % 64);
 
@@ -3064,14 +3135,13 @@ static void qed_iov_bulletin_set_forced_mac(struct qed_hwfn *p_hwfn,
 
 	vf_info->bulletin.p_virt->valid_bitmap |= feature;
 	/* Forced MAC will disable MAC_ADDR */
-	vf_info->bulletin.p_virt->valid_bitmap &=
-				~(1 << VFPF_BULLETIN_MAC_ADDR);
+	vf_info->bulletin.p_virt->valid_bitmap &= ~BIT(VFPF_BULLETIN_MAC_ADDR);
 
 	qed_iov_configure_vport_forced(p_hwfn, vf_info, feature);
 }
 
-void qed_iov_bulletin_set_forced_vlan(struct qed_hwfn *p_hwfn,
-				      u16 pvid, int vfid)
+static void qed_iov_bulletin_set_forced_vlan(struct qed_hwfn *p_hwfn,
+					     u16 pvid, int vfid)
 {
 	struct qed_vf_info *vf_info;
 	u64 feature;
@@ -3104,7 +3174,7 @@ static bool qed_iov_vf_has_vport_instance(struct qed_hwfn *p_hwfn, int vfid)
 	return !!p_vf_info->vport_instance;
 }
 
-bool qed_iov_is_vf_stopped(struct qed_hwfn *p_hwfn, int vfid)
+static bool qed_iov_is_vf_stopped(struct qed_hwfn *p_hwfn, int vfid)
 {
 	struct qed_vf_info *p_vf_info;
 
@@ -3126,7 +3196,7 @@ static bool qed_iov_spoofchk_get(struct qed_hwfn *p_hwfn, int vfid)
 	return vf_info->spoof_chk;
 }
 
-int qed_iov_spoofchk_set(struct qed_hwfn *p_hwfn, int vfid, bool val)
+static int qed_iov_spoofchk_set(struct qed_hwfn *p_hwfn, int vfid, bool val)
 {
 	struct qed_vf_info *vf;
 	int rc = -EINVAL;
@@ -3163,13 +3233,14 @@ static u8 *qed_iov_bulletin_get_forced_mac(struct qed_hwfn *p_hwfn,
 	if (!p_vf || !p_vf->bulletin.p_virt)
 		return NULL;
 
-	if (!(p_vf->bulletin.p_virt->valid_bitmap & (1 << MAC_ADDR_FORCED)))
+	if (!(p_vf->bulletin.p_virt->valid_bitmap & BIT(MAC_ADDR_FORCED)))
 		return NULL;
 
 	return p_vf->bulletin.p_virt->mac;
 }
 
-u16 qed_iov_bulletin_get_forced_vlan(struct qed_hwfn *p_hwfn, u16 rel_vf_id)
+static u16
+qed_iov_bulletin_get_forced_vlan(struct qed_hwfn *p_hwfn, u16 rel_vf_id)
 {
 	struct qed_vf_info *p_vf;
 
@@ -3177,7 +3248,7 @@ u16 qed_iov_bulletin_get_forced_vlan(struct qed_hwfn *p_hwfn, u16 rel_vf_id)
 	if (!p_vf || !p_vf->bulletin.p_virt)
 		return 0;
 
-	if (!(p_vf->bulletin.p_virt->valid_bitmap & (1 << VLAN_ADDR_FORCED)))
+	if (!(p_vf->bulletin.p_virt->valid_bitmap & BIT(VLAN_ADDR_FORCED)))
 		return 0;
 
 	return p_vf->bulletin.p_virt->pvid;
@@ -3201,7 +3272,8 @@ static int qed_iov_configure_tx_rate(struct qed_hwfn *p_hwfn,
 	return qed_init_vport_rl(p_hwfn, p_ptt, abs_vp_id, (u32)val);
 }
 
-int qed_iov_configure_min_tx_rate(struct qed_dev *cdev, int vfid, u32 rate)
+static int
+qed_iov_configure_min_tx_rate(struct qed_dev *cdev, int vfid, u32 rate)
 {
 	struct qed_vf_info *vf;
 	u8 vport_id;
@@ -3760,7 +3832,8 @@ static void qed_handle_bulletin_post(struct qed_hwfn *hwfn)
 	qed_ptt_release(hwfn, ptt);
 }
 
-void qed_iov_pf_task(struct work_struct *work)
+static void qed_iov_pf_task(struct work_struct *work)
+
 {
 	struct qed_hwfn *hwfn = container_of(work, struct qed_hwfn,
 					     iov_task.work);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_vf.c b/drivers/net/ethernet/qlogic/qed/qed_vf.c
index 9b780b31b15c..abf5bf11f865 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_vf.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_vf.c
@@ -46,6 +46,17 @@ static void *qed_vf_pf_prep(struct qed_hwfn *p_hwfn, u16 type, u16 length)
 	return p_tlv;
 }
 
+static void qed_vf_pf_req_end(struct qed_hwfn *p_hwfn, int req_status)
+{
+	union pfvf_tlvs *resp = p_hwfn->vf_iov_info->pf2vf_reply;
+
+	DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+		   "VF request status = 0x%x, PF reply status = 0x%x\n",
+		   req_status, resp->default_resp.hdr.status);
+
+	mutex_unlock(&(p_hwfn->vf_iov_info->mutex));
+}
+
 static int qed_send_msg2pf(struct qed_hwfn *p_hwfn, u8 *done, u32 resp_size)
 {
 	union vfpf_tlvs *p_req = p_hwfn->vf_iov_info->vf2pf_request;
@@ -103,16 +114,12 @@ static int qed_send_msg2pf(struct qed_hwfn *p_hwfn, u8 *done, u32 resp_size)
 			   "VF <-- PF Timeout [Type %d]\n",
 			   p_req->first_tlv.tl.type);
 		rc = -EBUSY;
-		goto exit;
 	} else {
 		DP_VERBOSE(p_hwfn, QED_MSG_IOV,
 			   "PF response: %d [Type %d]\n",
 			   *done, p_req->first_tlv.tl.type);
 	}
 
-exit:
-	mutex_unlock(&(p_hwfn->vf_iov_info->mutex));
-
 	return rc;
 }
 
@@ -191,6 +198,9 @@ static int qed_vf_pf_acquire(struct qed_hwfn *p_hwfn)
 		DP_VERBOSE(p_hwfn,
 			   QED_MSG_IOV, "attempting to acquire resources\n");
 
+		/* Clear response buffer, as this might be a re-send */
+		memset(p_iov->pf2vf_reply, 0, sizeof(union pfvf_tlvs));
+
 		/* send acquire request */
 		rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
 		if (rc)
@@ -205,9 +215,12 @@ static int qed_vf_pf_acquire(struct qed_hwfn *p_hwfn)
 			/* PF agrees to allocate our resources */
 			if (!(resp->pfdev_info.capabilities &
 			      PFVF_ACQUIRE_CAP_POST_FW_OVERRIDE)) {
-				DP_INFO(p_hwfn,
-					"PF is using old incompatible driver; Either downgrade driver or request provider to update hypervisor version\n");
-				return -EINVAL;
+				/* It's possible legacy PF mistakenly accepted;
+				 * but we don't care - simply mark it as
+				 * legacy and continue.
+				 */
+				req->vfdev_info.capabilities |=
+				    VFPF_ACQUIRE_CAP_PRE_FP_HSI;
 			}
 			DP_VERBOSE(p_hwfn, QED_MSG_IOV, "resources acquired\n");
 			resources_acquired = true;
@@ -215,27 +228,55 @@ static int qed_vf_pf_acquire(struct qed_hwfn *p_hwfn)
 			   attempts < VF_ACQUIRE_THRESH) {
 			qed_vf_pf_acquire_reduce_resc(p_hwfn, p_resc,
 						      &resp->resc);
+		} else if (resp->hdr.status == PFVF_STATUS_NOT_SUPPORTED) {
+			if (pfdev_info->major_fp_hsi &&
+			    (pfdev_info->major_fp_hsi != ETH_HSI_VER_MAJOR)) {
+				DP_NOTICE(p_hwfn,
+					  "PF uses an incompatible fastpath HSI %02x.%02x [VF requires %02x.%02x]. Please change to a VF driver using %02x.xx.\n",
+					  pfdev_info->major_fp_hsi,
+					  pfdev_info->minor_fp_hsi,
+					  ETH_HSI_VER_MAJOR,
+					  ETH_HSI_VER_MINOR,
+					  pfdev_info->major_fp_hsi);
+				rc = -EINVAL;
+				goto exit;
+			}
 
-			/* Clear response buffer */
-			memset(p_iov->pf2vf_reply, 0, sizeof(union pfvf_tlvs));
-		} else if ((resp->hdr.status == PFVF_STATUS_NOT_SUPPORTED) &&
-			   pfdev_info->major_fp_hsi &&
-			   (pfdev_info->major_fp_hsi != ETH_HSI_VER_MAJOR)) {
-			DP_NOTICE(p_hwfn,
-				  "PF uses an incompatible fastpath HSI %02x.%02x [VF requires %02x.%02x]. Please change to a VF driver using %02x.xx.\n",
-				  pfdev_info->major_fp_hsi,
-				  pfdev_info->minor_fp_hsi,
-				  ETH_HSI_VER_MAJOR,
-				  ETH_HSI_VER_MINOR, pfdev_info->major_fp_hsi);
-			return -EINVAL;
+			if (!pfdev_info->major_fp_hsi) {
+				if (req->vfdev_info.capabilities &
+				    VFPF_ACQUIRE_CAP_PRE_FP_HSI) {
+					DP_NOTICE(p_hwfn,
+						  "PF uses very old drivers. Please change to a VF driver using no later than 8.8.x.x.\n");
+					rc = -EINVAL;
+					goto exit;
+				} else {
+					DP_INFO(p_hwfn,
+						"PF is old - try re-acquire to see if it supports FW-version override\n");
+					req->vfdev_info.capabilities |=
+					    VFPF_ACQUIRE_CAP_PRE_FP_HSI;
+					continue;
+				}
+			}
+
+			/* If PF/VF are using same Major, PF must have had
+			 * it's reasons. Simply fail.
+			 */
+			DP_NOTICE(p_hwfn, "PF rejected acquisition by VF\n");
+			rc = -EINVAL;
+			goto exit;
 		} else {
 			DP_ERR(p_hwfn,
 			       "PF returned error %d to VF acquisition request\n",
 			       resp->hdr.status);
-			return -EAGAIN;
+			rc = -EAGAIN;
+			goto exit;
 		}
 	}
 
+	/* Mark the PF as legacy, if needed */
+	if (req->vfdev_info.capabilities & VFPF_ACQUIRE_CAP_PRE_FP_HSI)
+		p_iov->b_pre_fp_hsi = true;
+
 	/* Update bulletin board size with response from PF */
 	p_iov->bulletin.size = resp->bulletin_size;
 
@@ -253,14 +294,18 @@ static int qed_vf_pf_acquire(struct qed_hwfn *p_hwfn)
 		}
 	}
 
-	if (ETH_HSI_VER_MINOR &&
+	if (!p_iov->b_pre_fp_hsi &&
+	    ETH_HSI_VER_MINOR &&
 	    (resp->pfdev_info.minor_fp_hsi < ETH_HSI_VER_MINOR)) {
 		DP_INFO(p_hwfn,
 			"PF is using older fastpath HSI; %02x.%02x is configured\n",
 			ETH_HSI_VER_MAJOR, resp->pfdev_info.minor_fp_hsi);
 	}
 
-	return 0;
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
+
+	return rc;
 }
 
 int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
@@ -286,31 +331,23 @@ int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
 
 	/* Allocate vf sriov info */
 	p_iov = kzalloc(sizeof(*p_iov), GFP_KERNEL);
-	if (!p_iov) {
-		DP_NOTICE(p_hwfn, "Failed to allocate `struct qed_sriov'\n");
+	if (!p_iov)
 		return -ENOMEM;
-	}
 
 	/* Allocate vf2pf msg */
 	p_iov->vf2pf_request = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 						  sizeof(union vfpf_tlvs),
 						  &p_iov->vf2pf_request_phys,
 						  GFP_KERNEL);
-	if (!p_iov->vf2pf_request) {
-		DP_NOTICE(p_hwfn,
-			  "Failed to allocate `vf2pf_request' DMA memory\n");
+	if (!p_iov->vf2pf_request)
 		goto free_p_iov;
-	}
 
 	p_iov->pf2vf_reply = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 						sizeof(union pfvf_tlvs),
 						&p_iov->pf2vf_reply_phys,
 						GFP_KERNEL);
-	if (!p_iov->pf2vf_reply) {
-		DP_NOTICE(p_hwfn,
-			  "Failed to allocate `pf2vf_reply' DMA memory\n");
+	if (!p_iov->pf2vf_reply)
 		goto free_vf2pf_request;
-	}
 
 	DP_VERBOSE(p_hwfn,
 		   QED_MSG_IOV,
@@ -347,6 +384,9 @@ free_p_iov:
 
 	return -ENOMEM;
 }
+#define TSTORM_QZONE_START   PXP_VF_BAR0_START_SDM_ZONE_A
+#define MSTORM_QZONE_START(dev)   (TSTORM_QZONE_START +	\
+				   (TSTORM_QZONE_SIZE * NUM_OF_L2_QUEUES(dev)))
 
 int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
 			u8 rx_qid,
@@ -374,6 +414,21 @@ int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
 	req->bd_max_bytes = bd_max_bytes;
 	req->stat_id = -1;
 
+	/* If PF is legacy, we'll need to calculate producers ourselves
+	 * as well as clean them.
+	 */
+	if (pp_prod && p_iov->b_pre_fp_hsi) {
+		u8 hw_qid = p_iov->acquire_resp.resc.hw_qid[rx_qid];
+		u32 init_prod_val = 0;
+
+		*pp_prod = (u8 __iomem *)p_hwfn->regview +
+					 MSTORM_QZONE_START(p_hwfn->cdev) +
+					 hw_qid * MSTORM_QZONE_SIZE;
+
+		/* Init the rcq, rx bd and rx sge (if valid) producers to 0 */
+		__internal_ram_wr(p_hwfn, *pp_prod, sizeof(u32),
+				  (u32 *)(&init_prod_val));
+	}
 	/* add list termination tlv */
 	qed_add_tlv(p_hwfn, &p_iov->offset,
 		    CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv));
@@ -381,13 +436,15 @@ int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
 	resp = &p_iov->pf2vf_reply->queue_start;
 	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
 	if (rc)
-		return rc;
+		goto exit;
 
-	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-		return -EINVAL;
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+		rc = -EINVAL;
+		goto exit;
+	}
 
 	/* Learn the address of the producer from the response */
-	if (pp_prod) {
+	if (pp_prod && !p_iov->b_pre_fp_hsi) {
 		u32 init_prod_val = 0;
 
 		*pp_prod = (u8 __iomem *)p_hwfn->regview + resp->offset;
@@ -399,6 +456,8 @@ int qed_vf_pf_rxq_start(struct qed_hwfn *p_hwfn,
 		__internal_ram_wr(p_hwfn, *pp_prod, sizeof(u32),
 				  (u32 *)&init_prod_val);
 	}
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
 
 	return rc;
 }
@@ -424,10 +483,15 @@ int qed_vf_pf_rxq_stop(struct qed_hwfn *p_hwfn, u16 rx_qid, bool cqe_completion)
 	resp = &p_iov->pf2vf_reply->default_resp;
 	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
 	if (rc)
-		return rc;
+		goto exit;
+
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+		rc = -EINVAL;
+		goto exit;
+	}
 
-	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-		return -EINVAL;
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
 
 	return rc;
 }
@@ -470,13 +534,27 @@ int qed_vf_pf_txq_start(struct qed_hwfn *p_hwfn,
 	}
 
 	if (pp_doorbell) {
-		*pp_doorbell = (u8 __iomem *)p_hwfn->doorbells + resp->offset;
+		/* Modern PFs provide the actual offsets, while legacy
+		 * provided only the queue id.
+		 */
+		if (!p_iov->b_pre_fp_hsi) {
+			*pp_doorbell = (u8 __iomem *)p_hwfn->doorbells +
+						     resp->offset;
+		} else {
+			u8 cid = p_iov->acquire_resp.resc.cid[tx_queue_id];
+			u32 db_addr;
+
+			db_addr = qed_db_addr_vf(cid, DQ_DEMS_LEGACY);
+			*pp_doorbell = (u8 __iomem *)p_hwfn->doorbells +
+						     db_addr;
+		}
 
 		DP_VERBOSE(p_hwfn, QED_MSG_IOV,
 			   "Txq[0x%02x]: doorbell at %p [offset 0x%08x]\n",
 			   tx_queue_id, *pp_doorbell, resp->offset);
 	}
 exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
 
 	return rc;
 }
@@ -501,10 +579,15 @@ int qed_vf_pf_txq_stop(struct qed_hwfn *p_hwfn, u16 tx_qid)
 	resp = &p_iov->pf2vf_reply->default_resp;
 	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
 	if (rc)
-		return rc;
+		goto exit;
 
-	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-		return -EINVAL;
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+		rc = -EINVAL;
+		goto exit;
+	}
+
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
 
 	return rc;
 }
@@ -543,10 +626,15 @@ int qed_vf_pf_vport_start(struct qed_hwfn *p_hwfn,
 	resp = &p_iov->pf2vf_reply->default_resp;
 	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
 	if (rc)
-		return rc;
+		goto exit;
 
-	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-		return -EINVAL;
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+		rc = -EINVAL;
+		goto exit;
+	}
+
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
 
 	return rc;
 }
@@ -567,10 +655,15 @@ int qed_vf_pf_vport_stop(struct qed_hwfn *p_hwfn)
 
 	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
 	if (rc)
-		return rc;
+		goto exit;
+
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+		rc = -EINVAL;
+		goto exit;
+	}
 
-	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-		return -EINVAL;
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
 
 	return rc;
 }
@@ -770,13 +863,18 @@ int qed_vf_pf_vport_update(struct qed_hwfn *p_hwfn,
 
 	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, resp_size);
 	if (rc)
-		return rc;
+		goto exit;
 
-	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-		return -EINVAL;
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+		rc = -EINVAL;
+		goto exit;
+	}
 
 	qed_vf_handle_vp_update_tlvs_resp(p_hwfn, p_params);
 
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
+
 	return rc;
 }
 
@@ -797,14 +895,19 @@ int qed_vf_pf_reset(struct qed_hwfn *p_hwfn)
 	resp = &p_iov->pf2vf_reply->default_resp;
 	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
 	if (rc)
-		return rc;
+		goto exit;
 
-	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-		return -EAGAIN;
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+		rc = -EAGAIN;
+		goto exit;
+	}
 
 	p_hwfn->b_int_enabled = 0;
 
-	return 0;
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
+
+	return rc;
 }
 
 int qed_vf_pf_release(struct qed_hwfn *p_hwfn)
@@ -828,6 +931,8 @@ int qed_vf_pf_release(struct qed_hwfn *p_hwfn)
 	if (!rc && resp->hdr.status != PFVF_STATUS_SUCCESS)
 		rc = -EAGAIN;
 
+	qed_vf_pf_req_end(p_hwfn, rc);
+
 	p_hwfn->b_int_enabled = 0;
 
 	if (p_iov->vf2pf_request)
@@ -896,12 +1001,17 @@ int qed_vf_pf_filter_ucast(struct qed_hwfn *p_hwfn,
 	resp = &p_iov->pf2vf_reply->default_resp;
 	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
 	if (rc)
-		return rc;
+		goto exit;
 
-	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-		return -EAGAIN;
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+		rc = -EAGAIN;
+		goto exit;
+	}
 
-	return 0;
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
+
+	return rc;
 }
 
 int qed_vf_pf_int_cleanup(struct qed_hwfn *p_hwfn)
@@ -920,12 +1030,17 @@ int qed_vf_pf_int_cleanup(struct qed_hwfn *p_hwfn)
 
 	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
 	if (rc)
-		return rc;
+		goto exit;
 
-	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
-		return -EINVAL;
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS) {
+		rc = -EINVAL;
+		goto exit;
+	}
 
-	return 0;
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
+
+	return rc;
 }
 
 u16 qed_vf_get_igu_sb_id(struct qed_hwfn *p_hwfn, u16 sb_id)
@@ -1071,8 +1186,8 @@ bool qed_vf_check_mac(struct qed_hwfn *p_hwfn, u8 *mac)
 	return false;
 }
 
-bool qed_vf_bulletin_get_forced_mac(struct qed_hwfn *hwfn,
-				    u8 *dst_mac, u8 *p_is_forced)
+static bool qed_vf_bulletin_get_forced_mac(struct qed_hwfn *hwfn,
+					   u8 *dst_mac, u8 *p_is_forced)
 {
 	struct qed_bulletin_content *bulletin;
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_vf.h b/drivers/net/ethernet/qlogic/qed/qed_vf.h
index b23ce58e932f..35db7a28aa13 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_vf.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_vf.h
@@ -86,7 +86,7 @@ struct vfpf_acquire_tlv {
 	struct vfpf_first_tlv first_tlv;
 
 	struct vf_pf_vfdev_info {
-#define VFPF_ACQUIRE_CAP_OBSOLETE	(1 << 0)
+#define VFPF_ACQUIRE_CAP_PRE_FP_HSI     (1 << 0) /* VF pre-FP hsi version */
 #define VFPF_ACQUIRE_CAP_100G		(1 << 1) /* VF can support 100g */
 		u64 capabilities;
 		u8 fw_major;
@@ -551,6 +551,11 @@ struct qed_vf_iov {
 
 	/* we set aside a copy of the acquire response */
 	struct pfvf_acquire_resp_tlv acquire_resp;
+
+	/* In case PF originates prior to the fp-hsi version comparison,
+	 * this has to be propagated as it affects the fastpath.
+	 */
+	bool b_pre_fp_hsi;
 };
 
 #ifdef CONFIG_QED_SRIOV
diff --git a/drivers/net/ethernet/qlogic/qede/Makefile b/drivers/net/ethernet/qlogic/qede/Makefile
index 74a49850d74d..28dc58919c85 100644
--- a/drivers/net/ethernet/qlogic/qede/Makefile
+++ b/drivers/net/ethernet/qlogic/qede/Makefile
@@ -2,3 +2,4 @@ obj-$(CONFIG_QEDE) := qede.o
 
 qede-y := qede_main.o qede_ethtool.o
 qede-$(CONFIG_DCB) += qede_dcbnl.o
+qede-$(CONFIG_INFINIBAND_QEDR) += qede_roce.o
diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h
index 02b06d4e40ae..28c0e9f42c9e 100644
--- a/drivers/net/ethernet/qlogic/qede/qede.h
+++ b/drivers/net/ethernet/qlogic/qede/qede.h
@@ -25,7 +25,7 @@
 
 #define QEDE_MAJOR_VERSION		8
 #define QEDE_MINOR_VERSION		10
-#define QEDE_REVISION_VERSION		1
+#define QEDE_REVISION_VERSION		9
 #define QEDE_ENGINEERING_VERSION	20
 #define DRV_MODULE_VERSION __stringify(QEDE_MAJOR_VERSION) "."	\
 		__stringify(QEDE_MINOR_VERSION) "."		\
@@ -36,6 +36,8 @@
 
 struct qede_stats {
 	u64 no_buff_discards;
+	u64 packet_too_big_discard;
+	u64 ttl0_discard;
 	u64 rx_ucast_bytes;
 	u64 rx_mcast_bytes;
 	u64 rx_bcast_bytes;
@@ -104,6 +106,13 @@ struct qede_vlan {
 	bool configured;
 };
 
+struct qede_rdma_dev {
+	struct qedr_dev *qedr_dev;
+	struct list_head entry;
+	struct list_head roce_event_list;
+	struct workqueue_struct *roce_wq;
+};
+
 struct qede_dev {
 	struct qed_dev			*cdev;
 	struct net_device		*ndev;
@@ -124,16 +133,22 @@ struct qede_dev {
 				 (edev)->dev_info.num_tc)
 
 	struct qede_fastpath		*fp_array;
-	u16				req_rss;
-	u16				num_rss;
+	u8				req_num_tx;
+	u8				fp_num_tx;
+	u8				req_num_rx;
+	u8				fp_num_rx;
+	u16				req_queues;
+	u16				num_queues;
 	u8				num_tc;
-#define QEDE_RSS_CNT(edev)		((edev)->num_rss)
-#define QEDE_TSS_CNT(edev)		((edev)->num_rss *	\
-					 (edev)->num_tc)
-#define QEDE_TSS_IDX(edev, txqidx)	((txqidx) % (edev)->num_rss)
-#define QEDE_TC_IDX(edev, txqidx)	((txqidx) / (edev)->num_rss)
+#define QEDE_QUEUE_CNT(edev)	((edev)->num_queues)
+#define QEDE_RSS_COUNT(edev)	((edev)->num_queues - (edev)->fp_num_tx)
+#define QEDE_TSS_COUNT(edev)	(((edev)->num_queues - (edev)->fp_num_rx) * \
+				 (edev)->num_tc)
+#define QEDE_TX_IDX(edev, txqidx)	((edev)->fp_num_rx + (txqidx) % \
+					 QEDE_TSS_COUNT(edev))
+#define QEDE_TC_IDX(edev, txqidx)	((txqidx) / QEDE_TSS_COUNT(edev))
 #define QEDE_TX_QUEUE(edev, txqidx)	\
-	(&(edev)->fp_array[QEDE_TSS_IDX((edev), (txqidx))].txqs[QEDE_TC_IDX( \
+	(&(edev)->fp_array[QEDE_TX_IDX((edev), (txqidx))].txqs[QEDE_TC_IDX(\
 							(edev), (txqidx))])
 
 	struct qed_int_info		int_info;
@@ -177,6 +192,8 @@ struct qede_dev {
 	unsigned long			sp_flags;
 	u16				vxlan_dst_port;
 	u16				geneve_dst_port;
+
+	struct qede_rdma_dev		rdma_info;
 };
 
 enum QEDE_STATE {
@@ -235,6 +252,7 @@ struct qede_rx_queue {
 	u16			num_rx_buffers;
 	u16			rxq_id;
 
+	u64			rcv_pkts;
 	u64			rx_hw_errors;
 	u64			rx_alloc_errors;
 	u64			rx_ip_frags;
@@ -263,6 +281,10 @@ struct qede_tx_queue {
 	union db_prod		tx_db;
 
 	u16			num_tx_buffers;
+	u64			xmit_pkts;
+	u64			stopped_cnt;
+
+	bool			is_legacy;
 };
 
 #define BD_UNMAP_ADDR(bd)		HILO_U64(le32_to_cpu((bd)->addr.hi), \
@@ -277,7 +299,11 @@ struct qede_tx_queue {
 
 struct qede_fastpath {
 	struct qede_dev	*edev;
-	u8			rss_id;
+#define QEDE_FASTPATH_TX	BIT(0)
+#define QEDE_FASTPATH_RX	BIT(1)
+#define QEDE_FASTPATH_COMBINED	(QEDE_FASTPATH_TX | QEDE_FASTPATH_RX)
+	u8			type;
+	u8			id;
 	struct napi_struct	napi;
 	struct qed_sb_info	*sb_info;
 	struct qede_rx_queue	*rxq;
@@ -337,6 +363,6 @@ void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, struct qede_dev *edev,
 
 #define QEDE_MIN_PKT_LEN	64
 #define QEDE_RX_HDR_SIZE	256
-#define	for_each_rss(i) for (i = 0; i < edev->num_rss; i++)
+#define	for_each_queue(i) for (i = 0; i < edev->num_queues; i++)
 
 #endif /* _QEDE_H_ */
diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
index f8492cac9290..25a9b293ee8f 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
@@ -35,6 +35,7 @@ static const struct {
 	u64 offset;
 	char string[ETH_GSTRING_LEN];
 } qede_rqstats_arr[] = {
+	QEDE_RQSTAT(rcv_pkts),
 	QEDE_RQSTAT(rx_hw_errors),
 	QEDE_RQSTAT(rx_alloc_errors),
 	QEDE_RQSTAT(rx_ip_frags),
@@ -44,6 +45,24 @@ static const struct {
 #define QEDE_RQSTATS_DATA(dev, sindex, rqindex) \
 	(*((u64 *)(((char *)(dev->fp_array[(rqindex)].rxq)) +\
 		    qede_rqstats_arr[(sindex)].offset)))
+#define QEDE_TQSTAT_OFFSET(stat_name) \
+	(offsetof(struct qede_tx_queue, stat_name))
+#define QEDE_TQSTAT_STRING(stat_name) (#stat_name)
+#define QEDE_TQSTAT(stat_name) \
+	{QEDE_TQSTAT_OFFSET(stat_name), QEDE_TQSTAT_STRING(stat_name)}
+#define QEDE_NUM_TQSTATS ARRAY_SIZE(qede_tqstats_arr)
+static const struct {
+	u64 offset;
+	char string[ETH_GSTRING_LEN];
+} qede_tqstats_arr[] = {
+	QEDE_TQSTAT(xmit_pkts),
+	QEDE_TQSTAT(stopped_cnt),
+};
+
+#define QEDE_TQSTATS_DATA(dev, sindex, tssid, tcid) \
+	(*((u64 *)(((void *)(&dev->fp_array[tssid].txqs[tcid])) +\
+		   qede_tqstats_arr[(sindex)].offset)))
+
 static const struct {
 	u64 offset;
 	char string[ETH_GSTRING_LEN];
@@ -107,6 +126,8 @@ static const struct {
 	QEDE_PF_STAT(mftag_filter_discards),
 	QEDE_PF_STAT(mac_filter_discards),
 	QEDE_STAT(tx_err_drop_pkts),
+	QEDE_STAT(ttl0_discard),
+	QEDE_STAT(packet_too_big_discard),
 
 	QEDE_STAT(coalesced_pkts),
 	QEDE_STAT(coalesced_events),
@@ -151,17 +172,29 @@ static void qede_get_strings_stats(struct qede_dev *edev, u8 *buf)
 {
 	int i, j, k;
 
+	for (i = 0, k = 0; i < QEDE_QUEUE_CNT(edev); i++) {
+		int tc;
+
+		for (j = 0; j < QEDE_NUM_RQSTATS; j++)
+			sprintf(buf + (k + j) * ETH_GSTRING_LEN,
+				"%d:   %s", i, qede_rqstats_arr[j].string);
+		k += QEDE_NUM_RQSTATS;
+		for (tc = 0; tc < edev->num_tc; tc++) {
+			for (j = 0; j < QEDE_NUM_TQSTATS; j++)
+				sprintf(buf + (k + j) * ETH_GSTRING_LEN,
+					"%d.%d: %s", i, tc,
+					qede_tqstats_arr[j].string);
+			k += QEDE_NUM_TQSTATS;
+		}
+	}
+
 	for (i = 0, j = 0; i < QEDE_NUM_STATS; i++) {
 		if (IS_VF(edev) && qede_stats_arr[i].pf_only)
 			continue;
-		strcpy(buf + j * ETH_GSTRING_LEN,
+		strcpy(buf + (k + j) * ETH_GSTRING_LEN,
 		       qede_stats_arr[i].string);
 		j++;
 	}
-
-	for (k = 0; k < QEDE_NUM_RQSTATS; k++, j++)
-		strcpy(buf + j * ETH_GSTRING_LEN,
-		       qede_rqstats_arr[k].string);
 }
 
 static void qede_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
@@ -197,19 +230,30 @@ static void qede_get_ethtool_stats(struct net_device *dev,
 
 	mutex_lock(&edev->qede_lock);
 
+	for (qid = 0; qid < QEDE_QUEUE_CNT(edev); qid++) {
+		int tc;
+
+		if (edev->fp_array[qid].type & QEDE_FASTPATH_RX) {
+			for (sidx = 0; sidx < QEDE_NUM_RQSTATS; sidx++)
+				buf[cnt++] = QEDE_RQSTATS_DATA(edev, sidx, qid);
+		}
+
+		if (edev->fp_array[qid].type & QEDE_FASTPATH_TX) {
+			for (tc = 0; tc < edev->num_tc; tc++) {
+				for (sidx = 0; sidx < QEDE_NUM_TQSTATS; sidx++)
+					buf[cnt++] = QEDE_TQSTATS_DATA(edev,
+								       sidx,
+								       qid, tc);
+			}
+		}
+	}
+
 	for (sidx = 0; sidx < QEDE_NUM_STATS; sidx++) {
 		if (IS_VF(edev) && qede_stats_arr[sidx].pf_only)
 			continue;
 		buf[cnt++] = QEDE_STATS_DATA(edev, sidx);
 	}
 
-	for (sidx = 0; sidx < QEDE_NUM_RQSTATS; sidx++) {
-		buf[cnt] = 0;
-		for (qid = 0; qid < edev->num_rss; qid++)
-			buf[cnt] += QEDE_RQSTATS_DATA(edev, sidx, qid);
-		cnt++;
-	}
-
 	mutex_unlock(&edev->qede_lock);
 }
 
@@ -227,7 +271,8 @@ static int qede_get_sset_count(struct net_device *dev, int stringset)
 				if (qede_stats_arr[i].pf_only)
 					num_stats--;
 		}
-		return num_stats + QEDE_NUM_RQSTATS;
+		return num_stats + QEDE_RSS_COUNT(edev) * QEDE_NUM_RQSTATS +
+		       QEDE_TSS_COUNT(edev) * QEDE_NUM_TQSTATS * edev->num_tc;
 	case ETH_SS_PRIV_FLAGS:
 		return QEDE_PRI_FLAG_LEN;
 	case ETH_SS_TEST:
@@ -249,78 +294,150 @@ static u32 qede_get_priv_flags(struct net_device *dev)
 	return (!!(edev->dev_info.common.num_hwfns > 1)) << QEDE_PRI_FLAG_CMT;
 }
 
-static int qede_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+struct qede_link_mode_mapping {
+	u32 qed_link_mode;
+	u32 ethtool_link_mode;
+};
+
+static const struct qede_link_mode_mapping qed_lm_map[] = {
+	{QED_LM_FIBRE_BIT, ETHTOOL_LINK_MODE_FIBRE_BIT},
+	{QED_LM_Autoneg_BIT, ETHTOOL_LINK_MODE_Autoneg_BIT},
+	{QED_LM_Asym_Pause_BIT, ETHTOOL_LINK_MODE_Asym_Pause_BIT},
+	{QED_LM_Pause_BIT, ETHTOOL_LINK_MODE_Pause_BIT},
+	{QED_LM_1000baseT_Half_BIT, ETHTOOL_LINK_MODE_1000baseT_Half_BIT},
+	{QED_LM_1000baseT_Full_BIT, ETHTOOL_LINK_MODE_1000baseT_Full_BIT},
+	{QED_LM_10000baseKR_Full_BIT, ETHTOOL_LINK_MODE_10000baseKR_Full_BIT},
+	{QED_LM_25000baseKR_Full_BIT, ETHTOOL_LINK_MODE_25000baseKR_Full_BIT},
+	{QED_LM_40000baseLR4_Full_BIT, ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT},
+	{QED_LM_50000baseKR2_Full_BIT, ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT},
+	{QED_LM_100000baseKR4_Full_BIT,
+	 ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT},
+};
+
+#define QEDE_DRV_TO_ETHTOOL_CAPS(caps, lk_ksettings, name)	\
+{								\
+	int i;							\
+								\
+	for (i = 0; i < QED_LM_COUNT; i++) {			\
+		if ((caps) & (qed_lm_map[i].qed_link_mode))	\
+			__set_bit(qed_lm_map[i].ethtool_link_mode,\
+				  lk_ksettings->link_modes.name); \
+	}							\
+}
+
+#define QEDE_ETHTOOL_TO_DRV_CAPS(caps, lk_ksettings, name)	\
+{								\
+	int i;							\
+								\
+	for (i = 0; i < QED_LM_COUNT; i++) {			\
+		if (test_bit(qed_lm_map[i].ethtool_link_mode,	\
+			     lk_ksettings->link_modes.name))	\
+			caps |= qed_lm_map[i].qed_link_mode;	\
+	}							\
+}
+
+static int qede_get_link_ksettings(struct net_device *dev,
+				   struct ethtool_link_ksettings *cmd)
 {
+	struct ethtool_link_settings *base = &cmd->base;
 	struct qede_dev *edev = netdev_priv(dev);
 	struct qed_link_output current_link;
 
 	memset(&current_link, 0, sizeof(current_link));
 	edev->ops->common->get_link(edev->cdev, &current_link);
 
-	cmd->supported = current_link.supported_caps;
-	cmd->advertising = current_link.advertised_caps;
+	ethtool_link_ksettings_zero_link_mode(cmd, supported);
+	QEDE_DRV_TO_ETHTOOL_CAPS(current_link.supported_caps, cmd, supported)
+
+	ethtool_link_ksettings_zero_link_mode(cmd, advertising);
+	QEDE_DRV_TO_ETHTOOL_CAPS(current_link.advertised_caps, cmd, advertising)
+
+	ethtool_link_ksettings_zero_link_mode(cmd, lp_advertising);
+	QEDE_DRV_TO_ETHTOOL_CAPS(current_link.lp_caps, cmd, lp_advertising)
+
 	if ((edev->state == QEDE_STATE_OPEN) && (current_link.link_up)) {
-		ethtool_cmd_speed_set(cmd, current_link.speed);
-		cmd->duplex = current_link.duplex;
+		base->speed = current_link.speed;
+		base->duplex = current_link.duplex;
 	} else {
-		cmd->duplex = DUPLEX_UNKNOWN;
-		ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN);
+		base->speed = SPEED_UNKNOWN;
+		base->duplex = DUPLEX_UNKNOWN;
 	}
-	cmd->port = current_link.port;
-	cmd->autoneg = (current_link.autoneg) ? AUTONEG_ENABLE :
-						AUTONEG_DISABLE;
-	cmd->lp_advertising = current_link.lp_caps;
+	base->port = current_link.port;
+	base->autoneg = (current_link.autoneg) ? AUTONEG_ENABLE :
+			AUTONEG_DISABLE;
 
 	return 0;
 }
 
-static int qede_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+static int qede_set_link_ksettings(struct net_device *dev,
+				   const struct ethtool_link_ksettings *cmd)
 {
+	const struct ethtool_link_settings *base = &cmd->base;
 	struct qede_dev *edev = netdev_priv(dev);
 	struct qed_link_output current_link;
 	struct qed_link_params params;
-	u32 speed;
 
 	if (!edev->ops || !edev->ops->common->can_link_change(edev->cdev)) {
-		DP_INFO(edev,
-			"Link settings are not allowed to be changed\n");
+		DP_INFO(edev, "Link settings are not allowed to be changed\n");
 		return -EOPNOTSUPP;
 	}
-
 	memset(&current_link, 0, sizeof(current_link));
 	memset(&params, 0, sizeof(params));
 	edev->ops->common->get_link(edev->cdev, &current_link);
 
-	speed = ethtool_cmd_speed(cmd);
 	params.override_flags |= QED_LINK_OVERRIDE_SPEED_ADV_SPEEDS;
 	params.override_flags |= QED_LINK_OVERRIDE_SPEED_AUTONEG;
-	if (cmd->autoneg == AUTONEG_ENABLE) {
+	if (base->autoneg == AUTONEG_ENABLE) {
 		params.autoneg = true;
 		params.forced_speed = 0;
-		params.adv_speeds = cmd->advertising;
-	} else { /* forced speed */
+		QEDE_ETHTOOL_TO_DRV_CAPS(params.adv_speeds, cmd, advertising)
+	} else {		/* forced speed */
 		params.override_flags |= QED_LINK_OVERRIDE_SPEED_FORCED_SPEED;
 		params.autoneg = false;
-		params.forced_speed = speed;
-		switch (speed) {
+		params.forced_speed = base->speed;
+		switch (base->speed) {
 		case SPEED_10000:
 			if (!(current_link.supported_caps &
-			    SUPPORTED_10000baseKR_Full)) {
+			      QED_LM_10000baseKR_Full_BIT)) {
 				DP_INFO(edev, "10G speed not supported\n");
 				return -EINVAL;
 			}
-			params.adv_speeds = SUPPORTED_10000baseKR_Full;
+			params.adv_speeds = QED_LM_10000baseKR_Full_BIT;
+			break;
+		case SPEED_25000:
+			if (!(current_link.supported_caps &
+			      QED_LM_25000baseKR_Full_BIT)) {
+				DP_INFO(edev, "25G speed not supported\n");
+				return -EINVAL;
+			}
+			params.adv_speeds = QED_LM_25000baseKR_Full_BIT;
 			break;
 		case SPEED_40000:
 			if (!(current_link.supported_caps &
-			    SUPPORTED_40000baseLR4_Full)) {
+			      QED_LM_40000baseLR4_Full_BIT)) {
 				DP_INFO(edev, "40G speed not supported\n");
 				return -EINVAL;
 			}
-			params.adv_speeds = SUPPORTED_40000baseLR4_Full;
+			params.adv_speeds = QED_LM_40000baseLR4_Full_BIT;
+			break;
+		case SPEED_50000:
+			if (!(current_link.supported_caps &
+			      QED_LM_50000baseKR2_Full_BIT)) {
+				DP_INFO(edev, "50G speed not supported\n");
+				return -EINVAL;
+			}
+			params.adv_speeds = QED_LM_50000baseKR2_Full_BIT;
+			break;
+		case SPEED_100000:
+			if (!(current_link.supported_caps &
+			      QED_LM_100000baseKR4_Full_BIT)) {
+				DP_INFO(edev, "100G speed not supported\n");
+				return -EINVAL;
+			}
+			params.adv_speeds = QED_LM_100000baseKR4_Full_BIT;
 			break;
 		default:
-			DP_INFO(edev, "Unsupported speed %u\n", speed);
+			DP_INFO(edev, "Unsupported speed %u\n", base->speed);
 			return -EINVAL;
 		}
 	}
@@ -368,8 +485,7 @@ static u32 qede_get_msglevel(struct net_device *ndev)
 {
 	struct qede_dev *edev = netdev_priv(ndev);
 
-	return ((u32)edev->dp_level << QED_LOG_LEVEL_SHIFT) |
-	       edev->dp_module;
+	return ((u32)edev->dp_level << QED_LOG_LEVEL_SHIFT) | edev->dp_module;
 }
 
 static void qede_set_msglevel(struct net_device *ndev, u32 level)
@@ -393,8 +509,7 @@ static int qede_nway_reset(struct net_device *dev)
 	struct qed_link_params link_params;
 
 	if (!edev->ops || !edev->ops->common->can_link_change(edev->cdev)) {
-		DP_INFO(edev,
-			"Link settings are not allowed to be changed\n");
+		DP_INFO(edev, "Link settings are not allowed to be changed\n");
 		return -EOPNOTSUPP;
 	}
 
@@ -467,7 +582,7 @@ static int qede_set_coalesce(struct net_device *dev,
 
 	rxc = (u16)coal->rx_coalesce_usecs;
 	txc = (u16)coal->tx_coalesce_usecs;
-	for_each_rss(i) {
+	for_each_queue(i) {
 		sb_id = edev->fp_array[i].sb_info->igu_sb_id;
 		rc = edev->ops->common->set_coalesce(edev->cdev, rxc, txc,
 						     (u8)i, sb_id);
@@ -563,7 +678,7 @@ static int qede_set_pauseparam(struct net_device *dev,
 	memset(&params, 0, sizeof(params));
 	params.override_flags |= QED_LINK_OVERRIDE_PAUSE_CONFIG;
 	if (epause->autoneg) {
-		if (!(current_link.supported_caps & SUPPORTED_Autoneg)) {
+		if (!(current_link.supported_caps & QED_LM_Autoneg_BIT)) {
 			DP_INFO(edev, "autoneg not supported\n");
 			return -EINVAL;
 		}
@@ -580,6 +695,28 @@ static int qede_set_pauseparam(struct net_device *dev,
 	return 0;
 }
 
+static void qede_get_regs(struct net_device *ndev,
+			  struct ethtool_regs *regs, void *buffer)
+{
+	struct qede_dev *edev = netdev_priv(ndev);
+
+	regs->version = 0;
+	memset(buffer, 0, regs->len);
+
+	if (edev->ops && edev->ops->common)
+		edev->ops->common->dbg_all_data(edev->cdev, buffer);
+}
+
+static int qede_get_regs_len(struct net_device *ndev)
+{
+	struct qede_dev *edev = netdev_priv(ndev);
+
+	if (edev->ops && edev->ops->common)
+		return edev->ops->common->dbg_all_data_size(edev->cdev);
+	else
+		return -EINVAL;
+}
+
 static void qede_update_mtu(struct qede_dev *edev, union qede_reload_args *args)
 {
 	edev->ndev->mtu = args->mtu;
@@ -619,45 +756,70 @@ static void qede_get_channels(struct net_device *dev,
 	struct qede_dev *edev = netdev_priv(dev);
 
 	channels->max_combined = QEDE_MAX_RSS_CNT(edev);
-	channels->combined_count = QEDE_RSS_CNT(edev);
+	channels->combined_count = QEDE_QUEUE_CNT(edev) - edev->fp_num_tx -
+					edev->fp_num_rx;
+	channels->tx_count = edev->fp_num_tx;
+	channels->rx_count = edev->fp_num_rx;
 }
 
 static int qede_set_channels(struct net_device *dev,
 			     struct ethtool_channels *channels)
 {
 	struct qede_dev *edev = netdev_priv(dev);
+	u32 count;
 
 	DP_VERBOSE(edev, (NETIF_MSG_IFUP | NETIF_MSG_IFDOWN),
 		   "set-channels command parameters: rx = %d, tx = %d, other = %d, combined = %d\n",
 		   channels->rx_count, channels->tx_count,
 		   channels->other_count, channels->combined_count);
 
-	/* We don't support separate rx / tx, nor `other' channels. */
-	if (channels->rx_count || channels->tx_count ||
-	    channels->other_count || (channels->combined_count == 0) ||
-	    (channels->combined_count > QEDE_MAX_RSS_CNT(edev))) {
+	count = channels->rx_count + channels->tx_count +
+			channels->combined_count;
+
+	/* We don't support `other' channels */
+	if (channels->other_count) {
 		DP_VERBOSE(edev, (NETIF_MSG_IFUP | NETIF_MSG_IFDOWN),
 			   "command parameters not supported\n");
 		return -EINVAL;
 	}
 
+	if (!(channels->combined_count || (channels->rx_count &&
+					   channels->tx_count))) {
+		DP_VERBOSE(edev, (NETIF_MSG_IFUP | NETIF_MSG_IFDOWN),
+			   "need to request at least one transmit and one receive channel\n");
+		return -EINVAL;
+	}
+
+	if (count > QEDE_MAX_RSS_CNT(edev)) {
+		DP_VERBOSE(edev, (NETIF_MSG_IFUP | NETIF_MSG_IFDOWN),
+			   "requested channels = %d max supported channels = %d\n",
+			   count, QEDE_MAX_RSS_CNT(edev));
+		return -EINVAL;
+	}
+
 	/* Check if there was a change in the active parameters */
-	if (channels->combined_count == QEDE_RSS_CNT(edev)) {
+	if ((count == QEDE_QUEUE_CNT(edev)) &&
+	    (channels->tx_count == edev->fp_num_tx) &&
+	    (channels->rx_count == edev->fp_num_rx)) {
 		DP_VERBOSE(edev, (NETIF_MSG_IFUP | NETIF_MSG_IFDOWN),
 			   "No change in active parameters\n");
 		return 0;
 	}
 
 	/* We need the number of queues to be divisible between the hwfns */
-	if (channels->combined_count % edev->dev_info.common.num_hwfns) {
+	if ((count % edev->dev_info.common.num_hwfns) ||
+	    (channels->tx_count % edev->dev_info.common.num_hwfns) ||
+	    (channels->rx_count % edev->dev_info.common.num_hwfns)) {
 		DP_VERBOSE(edev, (NETIF_MSG_IFUP | NETIF_MSG_IFDOWN),
-			   "Number of channels must be divisable by %04x\n",
+			   "Number of channels must be divisible by %04x\n",
 			   edev->dev_info.common.num_hwfns);
 		return -EINVAL;
 	}
 
 	/* Set number of queues and reload if necessary */
-	edev->req_rss = channels->combined_count;
+	edev->req_queues = count;
+	edev->req_num_tx = channels->tx_count;
+	edev->req_num_rx = channels->rx_count;
 	if (netif_running(dev))
 		qede_reload(edev, NULL, NULL);
 
@@ -727,7 +889,7 @@ static int qede_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
 
 	switch (info->cmd) {
 	case ETHTOOL_GRXRINGS:
-		info->data = edev->num_rss;
+		info->data = QEDE_RSS_COUNT(edev);
 		return 0;
 	case ETHTOOL_GRXFH:
 		return qede_get_rss_flags(edev, info);
@@ -930,7 +1092,7 @@ static void qede_netif_start(struct qede_dev *edev)
 	if (!netif_running(edev->ndev))
 		return;
 
-	for_each_rss(i) {
+	for_each_queue(i) {
 		/* Update and reenable interrupts */
 		qed_sb_ack(edev->fp_array[i].sb_info, IGU_INT_ENABLE, 1);
 		napi_enable(&edev->fp_array[i].napi);
@@ -942,7 +1104,7 @@ static void qede_netif_stop(struct qede_dev *edev)
 {
 	int i;
 
-	for_each_rss(i) {
+	for_each_queue(i) {
 		napi_disable(&edev->fp_array[i].napi);
 		/* Disable interrupts */
 		qed_sb_ack(edev->fp_array[i].sb_info, IGU_INT_DISABLE, 0);
@@ -952,11 +1114,23 @@ static void qede_netif_stop(struct qede_dev *edev)
 static int qede_selftest_transmit_traffic(struct qede_dev *edev,
 					  struct sk_buff *skb)
 {
-	struct qede_tx_queue *txq = &edev->fp_array[0].txqs[0];
+	struct qede_tx_queue *txq = NULL;
 	struct eth_tx_1st_bd *first_bd;
 	dma_addr_t mapping;
 	int i, idx, val;
 
+	for_each_queue(i) {
+		if (edev->fp_array[i].type & QEDE_FASTPATH_TX) {
+			txq = edev->fp_array[i].txqs;
+			break;
+		}
+	}
+
+	if (!txq) {
+		DP_NOTICE(edev, "Tx path is not available\n");
+		return -1;
+	}
+
 	/* Fill the entry in the SW ring and the BDs in the FW ring */
 	idx = txq->sw_tx_prod & NUM_TX_BDS_MAX;
 	txq->sw_tx_ring[idx].skb = skb;
@@ -1020,14 +1194,26 @@ static int qede_selftest_transmit_traffic(struct qede_dev *edev,
 
 static int qede_selftest_receive_traffic(struct qede_dev *edev)
 {
-	struct qede_rx_queue *rxq = edev->fp_array[0].rxq;
 	u16 hw_comp_cons, sw_comp_cons, sw_rx_index, len;
 	struct eth_fast_path_rx_reg_cqe *fp_cqe;
+	struct qede_rx_queue *rxq = NULL;
 	struct sw_rx_data *sw_rx_data;
 	union eth_rx_cqe *cqe;
 	u8 *data_ptr;
 	int i;
 
+	for_each_queue(i) {
+		if (edev->fp_array[i].type & QEDE_FASTPATH_RX) {
+			rxq = edev->fp_array[i].rxq;
+			break;
+		}
+	}
+
+	if (!rxq) {
+		DP_NOTICE(edev, "Rx path is not available\n");
+		return -1;
+	}
+
 	/* The packet is expected to receive on rx-queue 0 even though RSS is
 	 * enabled. This is because the queue 0 is configured as the default
 	 * queue and that the loopback traffic is not IP.
@@ -1228,9 +1414,11 @@ static int qede_get_tunable(struct net_device *dev,
 }
 
 static const struct ethtool_ops qede_ethtool_ops = {
-	.get_settings = qede_get_settings,
-	.set_settings = qede_set_settings,
+	.get_link_ksettings = qede_get_link_ksettings,
+	.set_link_ksettings = qede_set_link_ksettings,
 	.get_drvinfo = qede_get_drvinfo,
+	.get_regs_len = qede_get_regs_len,
+	.get_regs = qede_get_regs,
 	.get_msglevel = qede_get_msglevel,
 	.set_msglevel = qede_set_msglevel,
 	.nway_reset = qede_nway_reset,
@@ -1260,7 +1448,7 @@ static const struct ethtool_ops qede_ethtool_ops = {
 };
 
 static const struct ethtool_ops qede_vf_ethtool_ops = {
-	.get_settings = qede_get_settings,
+	.get_link_ksettings = qede_get_link_ksettings,
 	.get_drvinfo = qede_get_drvinfo,
 	.get_msglevel = qede_get_msglevel,
 	.set_msglevel = qede_set_msglevel,
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index e4bd02e46e57..343038ca047d 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -36,7 +36,7 @@
 #include <linux/random.h>
 #include <net/ip6_checksum.h>
 #include <linux/bitops.h>
-
+#include <linux/qed/qede_roce.h>
 #include "qede.h"
 
 static char version[] =
@@ -100,7 +100,8 @@ static int qede_alloc_rx_buffer(struct qede_dev *edev,
 static void qede_link_update(void *dev, struct qed_link_output *link);
 
 #ifdef CONFIG_QED_SRIOV
-static int qede_set_vf_vlan(struct net_device *ndev, int vf, u16 vlan, u8 qos)
+static int qede_set_vf_vlan(struct net_device *ndev, int vf, u16 vlan, u8 qos,
+			    __be16 vlan_proto)
 {
 	struct qede_dev *edev = netdev_priv(ndev);
 
@@ -109,6 +110,9 @@ static int qede_set_vf_vlan(struct net_device *ndev, int vf, u16 vlan, u8 qos)
 		return -EINVAL;
 	}
 
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
 	DP_VERBOSE(edev, QED_MSG_IOV, "Setting Vlan 0x%04x to VF [%d]\n",
 		   vlan, vf);
 
@@ -189,8 +193,7 @@ static int qede_netdev_event(struct notifier_block *this, unsigned long event,
 	struct ethtool_drvinfo drvinfo;
 	struct qede_dev *edev;
 
-	/* Currently only support name change */
-	if (event != NETDEV_CHANGENAME)
+	if (event != NETDEV_CHANGENAME && event != NETDEV_CHANGEADDR)
 		goto done;
 
 	/* Check whether this is a qede device */
@@ -203,11 +206,18 @@ static int qede_netdev_event(struct notifier_block *this, unsigned long event,
 		goto done;
 	edev = netdev_priv(ndev);
 
-	/* Notify qed of the name change */
-	if (!edev->ops || !edev->ops->common)
-		goto done;
-	edev->ops->common->set_id(edev->cdev, edev->ndev->name,
-				  "qede");
+	switch (event) {
+	case NETDEV_CHANGENAME:
+		/* Notify qed of the name change */
+		if (!edev->ops || !edev->ops->common)
+			goto done;
+		edev->ops->common->set_id(edev->cdev, edev->ndev->name, "qede");
+		break;
+	case NETDEV_CHANGEADDR:
+		edev = netdev_priv(ndev);
+		qede_roce_event_changeaddr(edev);
+		break;
+	}
 
 done:
 	return NOTIFY_DONE;
@@ -222,7 +232,7 @@ int __init qede_init(void)
 {
 	int ret;
 
-	pr_notice("qede_init: %s\n", version);
+	pr_info("qede_init: %s\n", version);
 
 	qed_ops = qed_get_eth_ops();
 	if (!qed_ops) {
@@ -253,7 +263,8 @@ int __init qede_init(void)
 
 static void __exit qede_cleanup(void)
 {
-	pr_notice("qede_cleanup called\n");
+	if (debug & QED_LOG_INFO_MASK)
+		pr_info("qede_cleanup called\n");
 
 	unregister_netdevice_notifier(&qede_netdev_notifier);
 	pci_unregister_driver(&qede_pci_driver);
@@ -270,8 +281,7 @@ module_exit(qede_cleanup);
 
 /* Unmap the data and free skb */
 static int qede_free_tx_pkt(struct qede_dev *edev,
-			    struct qede_tx_queue *txq,
-			    int *len)
+			    struct qede_tx_queue *txq, int *len)
 {
 	u16 idx = txq->sw_tx_cons & NUM_TX_BDS_MAX;
 	struct sk_buff *skb = txq->sw_tx_ring[idx].skb;
@@ -329,8 +339,7 @@ static int qede_free_tx_pkt(struct qede_dev *edev,
 static void qede_free_failed_tx_pkt(struct qede_dev *edev,
 				    struct qede_tx_queue *txq,
 				    struct eth_tx_1st_bd *first_bd,
-				    int nbd,
-				    bool data_split)
+				    int nbd, bool data_split)
 {
 	u16 idx = txq->sw_tx_prod & NUM_TX_BDS_MAX;
 	struct sk_buff *skb = txq->sw_tx_ring[idx].skb;
@@ -339,8 +348,7 @@ static void qede_free_failed_tx_pkt(struct qede_dev *edev,
 
 	/* Return prod to its position before this skb was handled */
 	qed_chain_set_prod(&txq->tx_pbl,
-			   le16_to_cpu(txq->tx_db.data.bd_prod),
-			   first_bd);
+			   le16_to_cpu(txq->tx_db.data.bd_prod), first_bd);
 
 	first_bd = (struct eth_tx_1st_bd *)qed_chain_produce(&txq->tx_pbl);
 
@@ -366,8 +374,7 @@ static void qede_free_failed_tx_pkt(struct qede_dev *edev,
 
 	/* Return again prod to its position before this skb was handled */
 	qed_chain_set_prod(&txq->tx_pbl,
-			   le16_to_cpu(txq->tx_db.data.bd_prod),
-			   first_bd);
+			   le16_to_cpu(txq->tx_db.data.bd_prod), first_bd);
 
 	/* Free skb */
 	dev_kfree_skb_any(skb);
@@ -376,8 +383,7 @@ static void qede_free_failed_tx_pkt(struct qede_dev *edev,
 }
 
 static u32 qede_xmit_type(struct qede_dev *edev,
-			  struct sk_buff *skb,
-			  int *ipv6_ext)
+			  struct sk_buff *skb, int *ipv6_ext)
 {
 	u32 rc = XMIT_L4_CSUM;
 	__be16 l3_proto;
@@ -434,15 +440,13 @@ static void qede_set_params_for_ipv6_ext(struct sk_buff *skb,
 }
 
 static int map_frag_to_bd(struct qede_dev *edev,
-			  skb_frag_t *frag,
-			  struct eth_tx_bd *bd)
+			  skb_frag_t *frag, struct eth_tx_bd *bd)
 {
 	dma_addr_t mapping;
 
 	/* Map skb non-linear frag data for DMA */
 	mapping = skb_frag_dma_map(&edev->pdev->dev, frag, 0,
-				   skb_frag_size(frag),
-				   DMA_TO_DEVICE);
+				   skb_frag_size(frag), DMA_TO_DEVICE);
 	if (unlikely(dma_mapping_error(&edev->pdev->dev, mapping))) {
 		DP_NOTICE(edev, "Unable to map frag - dropping packet\n");
 		return -ENOMEM;
@@ -504,9 +508,8 @@ static inline void qede_update_tx_producer(struct qede_tx_queue *txq)
 }
 
 /* Main transmit function */
-static
-netdev_tx_t qede_start_xmit(struct sk_buff *skb,
-			    struct net_device *ndev)
+static netdev_tx_t qede_start_xmit(struct sk_buff *skb,
+				   struct net_device *ndev)
 {
 	struct qede_dev *edev = netdev_priv(ndev);
 	struct netdev_queue *netdev_txq;
@@ -526,12 +529,11 @@ netdev_tx_t qede_start_xmit(struct sk_buff *skb,
 
 	/* Get tx-queue context and netdev index */
 	txq_index = skb_get_queue_mapping(skb);
-	WARN_ON(txq_index >= QEDE_TSS_CNT(edev));
+	WARN_ON(txq_index >= QEDE_TSS_COUNT(edev));
 	txq = QEDE_TX_QUEUE(edev, txq_index);
 	netdev_txq = netdev_get_tx_queue(ndev, txq_index);
 
-	WARN_ON(qed_chain_get_elem_left(&txq->tx_pbl) <
-			       (MAX_SKB_FRAGS + 1));
+	WARN_ON(qed_chain_get_elem_left(&txq->tx_pbl) < (MAX_SKB_FRAGS + 1));
 
 	xmit_type = qede_xmit_type(edev, skb, &ipv6_ext);
 
@@ -606,6 +608,14 @@ netdev_tx_t qede_start_xmit(struct sk_buff *skb,
 			    1 << ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT;
 		}
 
+		/* Legacy FW had flipped behavior in regard to this bit -
+		 * I.e., needed to set to prevent FW from touching encapsulated
+		 * packets when it didn't need to.
+		 */
+		if (unlikely(txq->is_legacy))
+			first_bd->data.bitfields ^=
+			    1 << ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT;
+
 		/* If the packet is IPv6 with extension header, indicate that
 		 * to FW and pass few params, since the device cracker doesn't
 		 * support parsing IPv6 with extension header/s.
@@ -722,12 +732,16 @@ netdev_tx_t qede_start_xmit(struct sk_buff *skb,
 	txq->tx_db.data.bd_prod =
 		cpu_to_le16(qed_chain_get_prod_idx(&txq->tx_pbl));
 
-	if (!skb->xmit_more || netif_tx_queue_stopped(netdev_txq))
+	if (!skb->xmit_more || netif_xmit_stopped(netdev_txq))
 		qede_update_tx_producer(txq);
 
 	if (unlikely(qed_chain_get_elem_left(&txq->tx_pbl)
 		      < (MAX_SKB_FRAGS + 1))) {
+		if (skb->xmit_more)
+			qede_update_tx_producer(txq);
+
 		netif_tx_stop_queue(netdev_txq);
+		txq->stopped_cnt++;
 		DP_VERBOSE(edev, NETIF_MSG_TX_QUEUED,
 			   "Stop queue was called\n");
 		/* paired memory barrier is in qede_tx_int(), we have to keep
@@ -761,8 +775,7 @@ int qede_txq_has_work(struct qede_tx_queue *txq)
 	return hw_bd_cons != qed_chain_get_cons_idx(&txq->tx_pbl);
 }
 
-static int qede_tx_int(struct qede_dev *edev,
-		       struct qede_tx_queue *txq)
+static int qede_tx_int(struct qede_dev *edev, struct qede_tx_queue *txq)
 {
 	struct netdev_queue *netdev_txq;
 	u16 hw_bd_cons;
@@ -788,6 +801,7 @@ static int qede_tx_int(struct qede_dev *edev,
 		bytes_compl += len;
 		pkts_compl++;
 		txq->sw_tx_cons++;
+		txq->xmit_pkts++;
 	}
 
 	netdev_tx_completed_queue(netdev_txq, pkts_compl, bytes_compl);
@@ -960,8 +974,7 @@ static inline void qede_update_rx_prod(struct qede_dev *edev,
 
 static u32 qede_get_rxhash(struct qede_dev *edev,
 			   u8 bitfields,
-			   __le32 rss_hash,
-			   enum pkt_hash_types *rxhash_type)
+			   __le32 rss_hash, enum pkt_hash_types *rxhash_type)
 {
 	enum rss_hash_type htype;
 
@@ -990,12 +1003,10 @@ static void qede_set_skb_csum(struct sk_buff *skb, u8 csum_flag)
 
 static inline void qede_skb_receive(struct qede_dev *edev,
 				    struct qede_fastpath *fp,
-				    struct sk_buff *skb,
-				    u16 vlan_tag)
+				    struct sk_buff *skb, u16 vlan_tag)
 {
 	if (vlan_tag)
-		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
-				       vlan_tag);
+		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 
 	napi_gro_receive(&fp->napi, skb);
 }
@@ -1018,8 +1029,7 @@ static void qede_set_gro_params(struct qede_dev *edev,
 
 static int qede_fill_frag_skb(struct qede_dev *edev,
 			      struct qede_rx_queue *rxq,
-			      u8 tpa_agg_index,
-			      u16 len_on_bd)
+			      u8 tpa_agg_index, u16 len_on_bd)
 {
 	struct sw_rx_data *current_bd = &rxq->sw_rx_ring[rxq->sw_rx_cons &
 							 NUM_RX_BDS_MAX];
@@ -1206,7 +1216,7 @@ static void qede_gro_receive(struct qede_dev *edev,
 #endif
 
 send_skb:
-	skb_record_rx_queue(skb, fp->rss_id);
+	skb_record_rx_queue(skb, fp->rxq->rxq_id);
 	qede_skb_receive(edev, fp, skb, vlan_tag);
 }
 
@@ -1410,7 +1420,7 @@ static int qede_rx_int(struct qede_fastpath *fp, int budget)
 
 		if (unlikely(cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH)) {
 			edev->ops->eth_cqe_completion(
-					edev->cdev, fp->rss_id,
+					edev->cdev, fp->id,
 					(struct eth_slow_path_rx_cqe *)cqe);
 			goto next_cqe;
 		}
@@ -1467,7 +1477,7 @@ alloc_skb:
 		skb = netdev_alloc_skb(edev->ndev, QEDE_RX_HDR_SIZE);
 		if (unlikely(!skb)) {
 			DP_NOTICE(edev,
-				  "Build_skb failed, dropping incoming packet\n");
+				  "skb allocation failed, dropping incoming packet\n");
 			qede_recycle_rx_bd_ring(rxq, edev, fp_cqe->bd_num);
 			rxq->rx_alloc_errors++;
 			goto next_cqe;
@@ -1575,14 +1585,13 @@ alloc_skb:
 		skb->protocol = eth_type_trans(skb, edev->ndev);
 
 		rx_hash = qede_get_rxhash(edev, fp_cqe->bitfields,
-					  fp_cqe->rss_hash,
-					  &rxhash_type);
+					  fp_cqe->rss_hash, &rxhash_type);
 
 		skb_set_hash(skb, rx_hash, rxhash_type);
 
 		qede_set_skb_csum(skb, csum_flag);
 
-		skb_record_rx_queue(skb, fp->rss_id);
+		skb_record_rx_queue(skb, fp->rxq->rxq_id);
 
 		qede_skb_receive(edev, fp, skb, le16_to_cpu(fp_cqe->vlan_tag));
 next_rx_only:
@@ -1601,6 +1610,8 @@ next_cqe: /* don't consume bd rx buffer */
 	/* Update producers */
 	qede_update_rx_prod(edev, rxq);
 
+	rxq->rcv_pkts += rx_pkt;
+
 	return rx_pkt;
 }
 
@@ -1613,10 +1624,12 @@ static int qede_poll(struct napi_struct *napi, int budget)
 	u8 tc;
 
 	for (tc = 0; tc < edev->num_tc; tc++)
-		if (qede_txq_has_work(&fp->txqs[tc]))
+		if (likely(fp->type & QEDE_FASTPATH_TX) &&
+		    qede_txq_has_work(&fp->txqs[tc]))
 			qede_tx_int(edev, &fp->txqs[tc]);
 
-	rx_work_done = qede_has_rx_work(fp->rxq) ?
+	rx_work_done = (likely(fp->type & QEDE_FASTPATH_RX) &&
+			qede_has_rx_work(fp->rxq)) ?
 			qede_rx_int(fp, budget) : 0;
 	if (rx_work_done < budget) {
 		qed_sb_update_sb_idx(fp->sb_info);
@@ -1636,8 +1649,10 @@ static int qede_poll(struct napi_struct *napi, int budget)
 		rmb();
 
 		/* Fall out from the NAPI loop if needed */
-		if (!(qede_has_rx_work(fp->rxq) ||
-		      qede_has_tx_work(fp))) {
+		if (!((likely(fp->type & QEDE_FASTPATH_RX) &&
+		       qede_has_rx_work(fp->rxq)) ||
+		      (likely(fp->type & QEDE_FASTPATH_TX) &&
+		       qede_has_tx_work(fp)))) {
 			napi_complete(napi);
 
 			/* Update and reenable interrupts */
@@ -1708,6 +1723,8 @@ void qede_fill_by_demand_stats(struct qede_dev *edev)
 
 	edev->ops->get_vport_stats(edev->cdev, &stats);
 	edev->stats.no_buff_discards = stats.no_buff_discards;
+	edev->stats.packet_too_big_discard = stats.packet_too_big_discard;
+	edev->stats.ttl0_discard = stats.ttl0_discard;
 	edev->stats.rx_ucast_bytes = stats.rx_ucast_bytes;
 	edev->stats.rx_mcast_bytes = stats.rx_mcast_bytes;
 	edev->stats.rx_bcast_bytes = stats.rx_bcast_bytes;
@@ -1787,9 +1804,9 @@ void qede_fill_by_demand_stats(struct qede_dev *edev)
 	edev->stats.tx_mac_ctrl_frames = stats.tx_mac_ctrl_frames;
 }
 
-static struct rtnl_link_stats64 *qede_get_stats64(
-			    struct net_device *dev,
-			    struct rtnl_link_stats64 *stats)
+static
+struct rtnl_link_stats64 *qede_get_stats64(struct net_device *dev,
+					   struct rtnl_link_stats64 *stats)
 {
 	struct qede_dev *edev = netdev_priv(dev);
 
@@ -2103,14 +2120,13 @@ static void qede_vlan_mark_nonconfigured(struct qede_dev *edev)
 		}
 
 		DP_VERBOSE(edev, NETIF_MSG_IFDOWN,
-			   "marked vlan %d as non-configured\n",
-			   vlan->vid);
+			   "marked vlan %d as non-configured\n", vlan->vid);
 	}
 
 	edev->accept_any_vlan = false;
 }
 
-int qede_set_features(struct net_device *dev, netdev_features_t features)
+static int qede_set_features(struct net_device *dev, netdev_features_t features)
 {
 	struct qede_dev *edev = netdev_priv(dev);
 	netdev_features_t changes = features ^ dev->features;
@@ -2146,7 +2162,7 @@ static void qede_udp_tunnel_add(struct net_device *dev,
 
 		edev->vxlan_dst_port = t_port;
 
-		DP_VERBOSE(edev, QED_MSG_DEBUG, "Added vxlan port=%d",
+		DP_VERBOSE(edev, QED_MSG_DEBUG, "Added vxlan port=%d\n",
 			   t_port);
 
 		set_bit(QEDE_SP_VXLAN_PORT_CONFIG, &edev->sp_flags);
@@ -2157,7 +2173,7 @@ static void qede_udp_tunnel_add(struct net_device *dev,
 
 		edev->geneve_dst_port = t_port;
 
-		DP_VERBOSE(edev, QED_MSG_DEBUG, "Added geneve port=%d",
+		DP_VERBOSE(edev, QED_MSG_DEBUG, "Added geneve port=%d\n",
 			   t_port);
 		set_bit(QEDE_SP_GENEVE_PORT_CONFIG, &edev->sp_flags);
 		break;
@@ -2181,7 +2197,7 @@ static void qede_udp_tunnel_del(struct net_device *dev,
 
 		edev->vxlan_dst_port = 0;
 
-		DP_VERBOSE(edev, QED_MSG_DEBUG, "Deleted vxlan port=%d",
+		DP_VERBOSE(edev, QED_MSG_DEBUG, "Deleted vxlan port=%d\n",
 			   t_port);
 
 		set_bit(QEDE_SP_VXLAN_PORT_CONFIG, &edev->sp_flags);
@@ -2192,7 +2208,7 @@ static void qede_udp_tunnel_del(struct net_device *dev,
 
 		edev->geneve_dst_port = 0;
 
-		DP_VERBOSE(edev, QED_MSG_DEBUG, "Deleted geneve port=%d",
+		DP_VERBOSE(edev, QED_MSG_DEBUG, "Deleted geneve port=%d\n",
 			   t_port);
 		set_bit(QEDE_SP_GENEVE_PORT_CONFIG, &edev->sp_flags);
 		break;
@@ -2237,15 +2253,13 @@ static const struct net_device_ops qede_netdev_ops = {
 static struct qede_dev *qede_alloc_etherdev(struct qed_dev *cdev,
 					    struct pci_dev *pdev,
 					    struct qed_dev_eth_info *info,
-					    u32 dp_module,
-					    u8 dp_level)
+					    u32 dp_module, u8 dp_level)
 {
 	struct net_device *ndev;
 	struct qede_dev *edev;
 
 	ndev = alloc_etherdev_mqs(sizeof(*edev),
-				  info->num_queues,
-				  info->num_queues);
+				  info->num_queues, info->num_queues);
 	if (!ndev) {
 		pr_err("etherdev allocation failed\n");
 		return NULL;
@@ -2261,6 +2275,9 @@ static struct qede_dev *qede_alloc_etherdev(struct qed_dev *cdev,
 	edev->q_num_rx_buffers = NUM_RX_BDS_DEF;
 	edev->q_num_tx_buffers = NUM_TX_BDS_DEF;
 
+	DP_INFO(edev, "Allocated netdev with %d tx queues and %d rx queues\n",
+		info->num_queues, info->num_queues);
+
 	SET_NETDEV_DEV(ndev, &pdev->dev);
 
 	memset(&edev->stats, 0, sizeof(edev->stats));
@@ -2349,7 +2366,7 @@ static void qede_free_fp_array(struct qede_dev *edev)
 		struct qede_fastpath *fp;
 		int i;
 
-		for_each_rss(i) {
+		for_each_queue(i) {
 			fp = &edev->fp_array[i];
 
 			kfree(fp->sb_info);
@@ -2358,22 +2375,33 @@ static void qede_free_fp_array(struct qede_dev *edev)
 		}
 		kfree(edev->fp_array);
 	}
-	edev->num_rss = 0;
+
+	edev->num_queues = 0;
+	edev->fp_num_tx = 0;
+	edev->fp_num_rx = 0;
 }
 
 static int qede_alloc_fp_array(struct qede_dev *edev)
 {
+	u8 fp_combined, fp_rx = edev->fp_num_rx;
 	struct qede_fastpath *fp;
 	int i;
 
-	edev->fp_array = kcalloc(QEDE_RSS_CNT(edev),
+	edev->fp_array = kcalloc(QEDE_QUEUE_CNT(edev),
 				 sizeof(*edev->fp_array), GFP_KERNEL);
 	if (!edev->fp_array) {
 		DP_NOTICE(edev, "fp array allocation failed\n");
 		goto err;
 	}
 
-	for_each_rss(i) {
+	fp_combined = QEDE_QUEUE_CNT(edev) - fp_rx - edev->fp_num_tx;
+
+	/* Allocate the FP elements for Rx queues followed by combined and then
+	 * the Tx. This ordering should be maintained so that the respective
+	 * queues (Rx or Tx) will be together in the fastpath array and the
+	 * associated ids will be sequential.
+	 */
+	for_each_queue(i) {
 		fp = &edev->fp_array[i];
 
 		fp->sb_info = kcalloc(1, sizeof(*fp->sb_info), GFP_KERNEL);
@@ -2382,16 +2410,33 @@ static int qede_alloc_fp_array(struct qede_dev *edev)
 			goto err;
 		}
 
-		fp->rxq = kcalloc(1, sizeof(*fp->rxq), GFP_KERNEL);
-		if (!fp->rxq) {
-			DP_NOTICE(edev, "RXQ struct allocation failed\n");
-			goto err;
+		if (fp_rx) {
+			fp->type = QEDE_FASTPATH_RX;
+			fp_rx--;
+		} else if (fp_combined) {
+			fp->type = QEDE_FASTPATH_COMBINED;
+			fp_combined--;
+		} else {
+			fp->type = QEDE_FASTPATH_TX;
 		}
 
-		fp->txqs = kcalloc(edev->num_tc, sizeof(*fp->txqs), GFP_KERNEL);
-		if (!fp->txqs) {
-			DP_NOTICE(edev, "TXQ array allocation failed\n");
-			goto err;
+		if (fp->type & QEDE_FASTPATH_TX) {
+			fp->txqs = kcalloc(edev->num_tc, sizeof(*fp->txqs),
+					   GFP_KERNEL);
+			if (!fp->txqs) {
+				DP_NOTICE(edev,
+					  "TXQ array allocation failed\n");
+				goto err;
+			}
+		}
+
+		if (fp->type & QEDE_FASTPATH_RX) {
+			fp->rxq = kcalloc(1, sizeof(*fp->rxq), GFP_KERNEL);
+			if (!fp->rxq) {
+				DP_NOTICE(edev,
+					  "RXQ struct allocation failed\n");
+				goto err;
+			}
 		}
 	}
 
@@ -2453,7 +2498,7 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
 			bool is_vf, enum qede_probe_mode mode)
 {
 	struct qed_probe_params probe_params;
-	struct qed_slowpath_params params;
+	struct qed_slowpath_params sp_params;
 	struct qed_dev_eth_info dev_info;
 	struct qede_dev *edev;
 	struct qed_dev *cdev;
@@ -2476,14 +2521,14 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
 	qede_update_pf_params(cdev);
 
 	/* Start the Slowpath-process */
-	memset(&params, 0, sizeof(struct qed_slowpath_params));
-	params.int_mode = QED_INT_MODE_MSIX;
-	params.drv_major = QEDE_MAJOR_VERSION;
-	params.drv_minor = QEDE_MINOR_VERSION;
-	params.drv_rev = QEDE_REVISION_VERSION;
-	params.drv_eng = QEDE_ENGINEERING_VERSION;
-	strlcpy(params.name, "qede LAN", QED_DRV_VER_STR_SIZE);
-	rc = qed_ops->common->slowpath_start(cdev, &params);
+	memset(&sp_params, 0, sizeof(sp_params));
+	sp_params.int_mode = QED_INT_MODE_MSIX;
+	sp_params.drv_major = QEDE_MAJOR_VERSION;
+	sp_params.drv_minor = QEDE_MINOR_VERSION;
+	sp_params.drv_rev = QEDE_REVISION_VERSION;
+	sp_params.drv_eng = QEDE_ENGINEERING_VERSION;
+	strlcpy(sp_params.name, "qede LAN", QED_DRV_VER_STR_SIZE);
+	rc = qed_ops->common->slowpath_start(cdev, &sp_params);
 	if (rc) {
 		pr_notice("Cannot start slowpath\n");
 		goto err1;
@@ -2506,10 +2551,14 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
 
 	qede_init_ndev(edev);
 
+	rc = qede_roce_dev_add(edev);
+	if (rc)
+		goto err3;
+
 	rc = register_netdev(edev->ndev);
 	if (rc) {
 		DP_NOTICE(edev, "Cannot register net-device\n");
-		goto err3;
+		goto err4;
 	}
 
 	edev->ops->common->set_id(cdev, edev->ndev->name, DRV_MODULE_VERSION);
@@ -2517,7 +2566,8 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
 	edev->ops->register_ops(cdev, &qede_ll_ops, edev);
 
 #ifdef CONFIG_DCB
-	qede_set_dcbnl_ops(edev->ndev);
+	if (!IS_VF(edev))
+		qede_set_dcbnl_ops(edev->ndev);
 #endif
 
 	INIT_DELAYED_WORK(&edev->sp_task, qede_sp_task);
@@ -2528,6 +2578,8 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
 
 	return 0;
 
+err4:
+	qede_roce_dev_remove(edev);
 err3:
 	free_netdev(edev->ndev);
 err2:
@@ -2574,8 +2626,11 @@ static void __qede_remove(struct pci_dev *pdev, enum qede_remove_mode mode)
 	DP_INFO(edev, "Starting qede_remove\n");
 
 	cancel_delayed_work_sync(&edev->sp_task);
+
 	unregister_netdev(ndev);
 
+	qede_roce_dev_remove(edev);
+
 	edev->ops->common->set_power_state(cdev, PCI_D0);
 
 	pci_set_drvdata(pdev, NULL);
@@ -2586,7 +2641,7 @@ static void __qede_remove(struct pci_dev *pdev, enum qede_remove_mode mode)
 	qed_ops->common->slowpath_stop(cdev);
 	qed_ops->common->remove(cdev);
 
-	pr_notice("Ending successfully qede_remove\n");
+	dev_info(&pdev->dev, "Ending qede_remove successfully\n");
 }
 
 static void qede_remove(struct pci_dev *pdev)
@@ -2605,8 +2660,8 @@ static int qede_set_num_queues(struct qede_dev *edev)
 	u16 rss_num;
 
 	/* Setup queues according to possible resources*/
-	if (edev->req_rss)
-		rss_num = edev->req_rss;
+	if (edev->req_queues)
+		rss_num = edev->req_queues;
 	else
 		rss_num = netif_get_num_default_rss_queues() *
 			  edev->dev_info.common.num_hwfns;
@@ -2616,11 +2671,15 @@ static int qede_set_num_queues(struct qede_dev *edev)
 	rc = edev->ops->common->set_fp_int(edev->cdev, rss_num);
 	if (rc > 0) {
 		/* Managed to request interrupts for our queues */
-		edev->num_rss = rc;
+		edev->num_queues = rc;
 		DP_INFO(edev, "Managed %d [of %d] RSS queues\n",
-			QEDE_RSS_CNT(edev), rss_num);
+			QEDE_QUEUE_CNT(edev), rss_num);
 		rc = 0;
 	}
+
+	edev->fp_num_tx = edev->req_num_tx;
+	edev->fp_num_rx = edev->req_num_rx;
+
 	return rc;
 }
 
@@ -2634,16 +2693,14 @@ static void qede_free_mem_sb(struct qede_dev *edev,
 
 /* This function allocates fast-path status block memory */
 static int qede_alloc_mem_sb(struct qede_dev *edev,
-			     struct qed_sb_info *sb_info,
-			     u16 sb_id)
+			     struct qed_sb_info *sb_info, u16 sb_id)
 {
 	struct status_block *sb_virt;
 	dma_addr_t sb_phys;
 	int rc;
 
 	sb_virt = dma_alloc_coherent(&edev->pdev->dev,
-				     sizeof(*sb_virt),
-				     &sb_phys, GFP_KERNEL);
+				     sizeof(*sb_virt), &sb_phys, GFP_KERNEL);
 	if (!sb_virt) {
 		DP_ERR(edev, "Status block allocation failed\n");
 		return -ENOMEM;
@@ -2675,16 +2732,15 @@ static void qede_free_rx_buffers(struct qede_dev *edev,
 		data = rx_buf->data;
 
 		dma_unmap_page(&edev->pdev->dev,
-			       rx_buf->mapping,
-			       PAGE_SIZE, DMA_FROM_DEVICE);
+			       rx_buf->mapping, PAGE_SIZE, DMA_FROM_DEVICE);
 
 		rx_buf->data = NULL;
 		__free_page(data);
 	}
 }
 
-static void qede_free_sge_mem(struct qede_dev *edev,
-			      struct qede_rx_queue *rxq) {
+static void qede_free_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
+{
 	int i;
 
 	if (edev->gro_disable)
@@ -2703,8 +2759,7 @@ static void qede_free_sge_mem(struct qede_dev *edev,
 	}
 }
 
-static void qede_free_mem_rxq(struct qede_dev *edev,
-			      struct qede_rx_queue *rxq)
+static void qede_free_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
 {
 	qede_free_sge_mem(edev, rxq);
 
@@ -2726,9 +2781,6 @@ static int qede_alloc_rx_buffer(struct qede_dev *edev,
 	struct eth_rx_bd *rx_bd;
 	dma_addr_t mapping;
 	struct page *data;
-	u16 rx_buf_size;
-
-	rx_buf_size = rxq->rx_buf_size;
 
 	data = alloc_pages(GFP_ATOMIC, 0);
 	if (unlikely(!data)) {
@@ -2763,8 +2815,7 @@ static int qede_alloc_rx_buffer(struct qede_dev *edev,
 	return 0;
 }
 
-static int qede_alloc_sge_mem(struct qede_dev *edev,
-			      struct qede_rx_queue *rxq)
+static int qede_alloc_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
 {
 	dma_addr_t mapping;
 	int i;
@@ -2811,15 +2862,14 @@ err:
 }
 
 /* This function allocates all memory needed per Rx queue */
-static int qede_alloc_mem_rxq(struct qede_dev *edev,
-			      struct qede_rx_queue *rxq)
+static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
 {
 	int i, rc, size;
 
 	rxq->num_rx_buffers = edev->q_num_rx_buffers;
 
-	rxq->rx_buf_size = NET_IP_ALIGN + ETH_OVERHEAD +
-			   edev->ndev->mtu;
+	rxq->rx_buf_size = NET_IP_ALIGN + ETH_OVERHEAD + edev->ndev->mtu;
+
 	if (rxq->rx_buf_size > PAGE_SIZE)
 		rxq->rx_buf_size = PAGE_SIZE;
 
@@ -2873,8 +2923,7 @@ err:
 	return rc;
 }
 
-static void qede_free_mem_txq(struct qede_dev *edev,
-			      struct qede_tx_queue *txq)
+static void qede_free_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
 {
 	/* Free the parallel SW ring */
 	kfree(txq->sw_tx_ring);
@@ -2884,8 +2933,7 @@ static void qede_free_mem_txq(struct qede_dev *edev,
 }
 
 /* This function allocates all memory needed per Tx queue */
-static int qede_alloc_mem_txq(struct qede_dev *edev,
-			      struct qede_tx_queue *txq)
+static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
 {
 	int size, rc;
 	union eth_tx_bd_types *p_virt;
@@ -2917,41 +2965,45 @@ err:
 }
 
 /* This function frees all memory of a single fp */
-static void qede_free_mem_fp(struct qede_dev *edev,
-			     struct qede_fastpath *fp)
+static void qede_free_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
 {
 	int tc;
 
 	qede_free_mem_sb(edev, fp->sb_info);
 
-	qede_free_mem_rxq(edev, fp->rxq);
+	if (fp->type & QEDE_FASTPATH_RX)
+		qede_free_mem_rxq(edev, fp->rxq);
 
-	for (tc = 0; tc < edev->num_tc; tc++)
-		qede_free_mem_txq(edev, &fp->txqs[tc]);
+	if (fp->type & QEDE_FASTPATH_TX)
+		for (tc = 0; tc < edev->num_tc; tc++)
+			qede_free_mem_txq(edev, &fp->txqs[tc]);
 }
 
 /* This function allocates all memory needed for a single fp (i.e. an entity
- * which contains status block, one rx queue and multiple per-TC tx queues.
+ * which contains status block, one rx queue and/or multiple per-TC tx queues.
  */
-static int qede_alloc_mem_fp(struct qede_dev *edev,
-			     struct qede_fastpath *fp)
+static int qede_alloc_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
 {
 	int rc, tc;
 
-	rc = qede_alloc_mem_sb(edev, fp->sb_info, fp->rss_id);
+	rc = qede_alloc_mem_sb(edev, fp->sb_info, fp->id);
 	if (rc)
 		goto err;
 
-	rc = qede_alloc_mem_rxq(edev, fp->rxq);
-	if (rc)
-		goto err;
-
-	for (tc = 0; tc < edev->num_tc; tc++) {
-		rc = qede_alloc_mem_txq(edev, &fp->txqs[tc]);
+	if (fp->type & QEDE_FASTPATH_RX) {
+		rc = qede_alloc_mem_rxq(edev, fp->rxq);
 		if (rc)
 			goto err;
 	}
 
+	if (fp->type & QEDE_FASTPATH_TX) {
+		for (tc = 0; tc < edev->num_tc; tc++) {
+			rc = qede_alloc_mem_txq(edev, &fp->txqs[tc]);
+			if (rc)
+				goto err;
+		}
+	}
+
 	return 0;
 err:
 	return rc;
@@ -2961,7 +3013,7 @@ static void qede_free_mem_load(struct qede_dev *edev)
 {
 	int i;
 
-	for_each_rss(i) {
+	for_each_queue(i) {
 		struct qede_fastpath *fp = &edev->fp_array[i];
 
 		qede_free_mem_fp(edev, fp);
@@ -2971,16 +3023,16 @@ static void qede_free_mem_load(struct qede_dev *edev)
 /* This function allocates all qede memory at NIC load. */
 static int qede_alloc_mem_load(struct qede_dev *edev)
 {
-	int rc = 0, rss_id;
+	int rc = 0, queue_id;
 
-	for (rss_id = 0; rss_id < QEDE_RSS_CNT(edev); rss_id++) {
-		struct qede_fastpath *fp = &edev->fp_array[rss_id];
+	for (queue_id = 0; queue_id < QEDE_QUEUE_CNT(edev); queue_id++) {
+		struct qede_fastpath *fp = &edev->fp_array[queue_id];
 
 		rc = qede_alloc_mem_fp(edev, fp);
 		if (rc) {
 			DP_ERR(edev,
 			       "Failed to allocate memory for fastpath - rss id = %d\n",
-			       rss_id);
+			       queue_id);
 			qede_free_mem_load(edev);
 			return rc;
 		}
@@ -2992,30 +3044,38 @@ static int qede_alloc_mem_load(struct qede_dev *edev)
 /* This function inits fp content and resets the SB, RXQ and TXQ structures */
 static void qede_init_fp(struct qede_dev *edev)
 {
-	int rss_id, txq_index, tc;
+	int queue_id, rxq_index = 0, txq_index = 0, tc;
 	struct qede_fastpath *fp;
 
-	for_each_rss(rss_id) {
-		fp = &edev->fp_array[rss_id];
+	for_each_queue(queue_id) {
+		fp = &edev->fp_array[queue_id];
 
 		fp->edev = edev;
-		fp->rss_id = rss_id;
+		fp->id = queue_id;
 
 		memset((void *)&fp->napi, 0, sizeof(fp->napi));
 
 		memset((void *)fp->sb_info, 0, sizeof(*fp->sb_info));
 
-		memset((void *)fp->rxq, 0, sizeof(*fp->rxq));
-		fp->rxq->rxq_id = rss_id;
+		if (fp->type & QEDE_FASTPATH_RX) {
+			memset((void *)fp->rxq, 0, sizeof(*fp->rxq));
+			fp->rxq->rxq_id = rxq_index++;
+		}
 
-		memset((void *)fp->txqs, 0, (edev->num_tc * sizeof(*fp->txqs)));
-		for (tc = 0; tc < edev->num_tc; tc++) {
-			txq_index = tc * QEDE_RSS_CNT(edev) + rss_id;
-			fp->txqs[tc].index = txq_index;
+		if (fp->type & QEDE_FASTPATH_TX) {
+			memset((void *)fp->txqs, 0,
+			       (edev->num_tc * sizeof(*fp->txqs)));
+			for (tc = 0; tc < edev->num_tc; tc++) {
+				fp->txqs[tc].index = txq_index +
+				    tc * QEDE_TSS_COUNT(edev);
+				if (edev->dev_info.is_legacy)
+					fp->txqs[tc].is_legacy = true;
+			}
+			txq_index++;
 		}
 
 		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
-			 edev->ndev->name, rss_id);
+			 edev->ndev->name, queue_id);
 	}
 
 	edev->gro_disable = !(edev->ndev->features & NETIF_F_GRO);
@@ -3025,12 +3085,13 @@ static int qede_set_real_num_queues(struct qede_dev *edev)
 {
 	int rc = 0;
 
-	rc = netif_set_real_num_tx_queues(edev->ndev, QEDE_TSS_CNT(edev));
+	rc = netif_set_real_num_tx_queues(edev->ndev, QEDE_TSS_COUNT(edev));
 	if (rc) {
 		DP_NOTICE(edev, "Failed to set real number of Tx queues\n");
 		return rc;
 	}
-	rc = netif_set_real_num_rx_queues(edev->ndev, QEDE_RSS_CNT(edev));
+
+	rc = netif_set_real_num_rx_queues(edev->ndev, QEDE_RSS_COUNT(edev));
 	if (rc) {
 		DP_NOTICE(edev, "Failed to set real number of Rx queues\n");
 		return rc;
@@ -3043,7 +3104,7 @@ static void qede_napi_disable_remove(struct qede_dev *edev)
 {
 	int i;
 
-	for_each_rss(i) {
+	for_each_queue(i) {
 		napi_disable(&edev->fp_array[i].napi);
 
 		netif_napi_del(&edev->fp_array[i].napi);
@@ -3055,7 +3116,7 @@ static void qede_napi_add_enable(struct qede_dev *edev)
 	int i;
 
 	/* Add NAPI objects */
-	for_each_rss(i) {
+	for_each_queue(i) {
 		netif_napi_add(edev->ndev, &edev->fp_array[i].napi,
 			       qede_poll, NAPI_POLL_WEIGHT);
 		napi_enable(&edev->fp_array[i].napi);
@@ -3084,14 +3145,14 @@ static int qede_req_msix_irqs(struct qede_dev *edev)
 	int i, rc;
 
 	/* Sanitize number of interrupts == number of prepared RSS queues */
-	if (QEDE_RSS_CNT(edev) > edev->int_info.msix_cnt) {
+	if (QEDE_QUEUE_CNT(edev) > edev->int_info.msix_cnt) {
 		DP_ERR(edev,
 		       "Interrupt mismatch: %d RSS queues > %d MSI-x vectors\n",
-		       QEDE_RSS_CNT(edev), edev->int_info.msix_cnt);
+		       QEDE_QUEUE_CNT(edev), edev->int_info.msix_cnt);
 		return -EINVAL;
 	}
 
-	for (i = 0; i < QEDE_RSS_CNT(edev); i++) {
+	for (i = 0; i < QEDE_QUEUE_CNT(edev); i++) {
 		rc = request_irq(edev->int_info.msix[i].vector,
 				 qede_msix_fp_int, 0, edev->fp_array[i].name,
 				 &edev->fp_array[i]);
@@ -3136,18 +3197,17 @@ static int qede_setup_irqs(struct qede_dev *edev)
 
 		/* qed should learn receive the RSS ids and callbacks */
 		ops = edev->ops->common;
-		for (i = 0; i < QEDE_RSS_CNT(edev); i++)
+		for (i = 0; i < QEDE_QUEUE_CNT(edev); i++)
 			ops->simd_handler_config(edev->cdev,
 						 &edev->fp_array[i], i,
 						 qede_simd_fp_handler);
-		edev->int_info.used_cnt = QEDE_RSS_CNT(edev);
+		edev->int_info.used_cnt = QEDE_QUEUE_CNT(edev);
 	}
 	return 0;
 }
 
 static int qede_drain_txq(struct qede_dev *edev,
-			  struct qede_tx_queue *txq,
-			  bool allow_drain)
+			  struct qede_tx_queue *txq, bool allow_drain)
 {
 	int rc, cnt = 1000;
 
@@ -3199,45 +3259,53 @@ static int qede_stop_queues(struct qede_dev *edev)
 	}
 
 	/* Flush Tx queues. If needed, request drain from MCP */
-	for_each_rss(i) {
+	for_each_queue(i) {
 		struct qede_fastpath *fp = &edev->fp_array[i];
 
-		for (tc = 0; tc < edev->num_tc; tc++) {
-			struct qede_tx_queue *txq = &fp->txqs[tc];
+		if (fp->type & QEDE_FASTPATH_TX) {
+			for (tc = 0; tc < edev->num_tc; tc++) {
+				struct qede_tx_queue *txq = &fp->txqs[tc];
 
-			rc = qede_drain_txq(edev, txq, true);
-			if (rc)
-				return rc;
+				rc = qede_drain_txq(edev, txq, true);
+				if (rc)
+					return rc;
+			}
 		}
 	}
 
-	/* Stop all Queues in reverse order*/
-	for (i = QEDE_RSS_CNT(edev) - 1; i >= 0; i--) {
+	/* Stop all Queues in reverse order */
+	for (i = QEDE_QUEUE_CNT(edev) - 1; i >= 0; i--) {
 		struct qed_stop_rxq_params rx_params;
 
-		/* Stop the Tx Queue(s)*/
-		for (tc = 0; tc < edev->num_tc; tc++) {
-			struct qed_stop_txq_params tx_params;
-
-			tx_params.rss_id = i;
-			tx_params.tx_queue_id = tc * QEDE_RSS_CNT(edev) + i;
-			rc = edev->ops->q_tx_stop(cdev, &tx_params);
-			if (rc) {
-				DP_ERR(edev, "Failed to stop TXQ #%d\n",
-				       tx_params.tx_queue_id);
-				return rc;
+		/* Stop the Tx Queue(s) */
+		if (edev->fp_array[i].type & QEDE_FASTPATH_TX) {
+			for (tc = 0; tc < edev->num_tc; tc++) {
+				struct qed_stop_txq_params tx_params;
+				u8 val;
+
+				tx_params.rss_id = i;
+				val = edev->fp_array[i].txqs[tc].index;
+				tx_params.tx_queue_id = val;
+				rc = edev->ops->q_tx_stop(cdev, &tx_params);
+				if (rc) {
+					DP_ERR(edev, "Failed to stop TXQ #%d\n",
+					       tx_params.tx_queue_id);
+					return rc;
+				}
 			}
 		}
 
-		/* Stop the Rx Queue*/
-		memset(&rx_params, 0, sizeof(rx_params));
-		rx_params.rss_id = i;
-		rx_params.rx_queue_id = i;
+		/* Stop the Rx Queue */
+		if (edev->fp_array[i].type & QEDE_FASTPATH_RX) {
+			memset(&rx_params, 0, sizeof(rx_params));
+			rx_params.rss_id = i;
+			rx_params.rx_queue_id = edev->fp_array[i].rxq->rxq_id;
 
-		rc = edev->ops->q_rx_stop(cdev, &rx_params);
-		if (rc) {
-			DP_ERR(edev, "Failed to stop RXQ #%d\n", i);
-			return rc;
+			rc = edev->ops->q_rx_stop(cdev, &rx_params);
+			if (rc) {
+				DP_ERR(edev, "Failed to stop RXQ #%d\n", i);
+				return rc;
+			}
 		}
 	}
 
@@ -3260,7 +3328,7 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
 	struct qed_start_vport_params start = {0};
 	bool reset_rss_indir = false;
 
-	if (!edev->num_rss) {
+	if (!edev->num_queues) {
 		DP_ERR(edev,
 		       "Cannot update V-VPORT as active as there are no Rx queues\n");
 		return -EINVAL;
@@ -3284,50 +3352,66 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
 		   "Start vport ramrod passed, vport_id = %d, MTU = %d, vlan_removal_en = %d\n",
 		   start.vport_id, edev->ndev->mtu + 0xe, vlan_removal_en);
 
-	for_each_rss(i) {
+	for_each_queue(i) {
 		struct qede_fastpath *fp = &edev->fp_array[i];
-		dma_addr_t phys_table = fp->rxq->rx_comp_ring.pbl.p_phys_table;
-
-		memset(&q_params, 0, sizeof(q_params));
-		q_params.rss_id = i;
-		q_params.queue_id = i;
-		q_params.vport_id = 0;
-		q_params.sb = fp->sb_info->igu_sb_id;
-		q_params.sb_idx = RX_PI;
-
-		rc = edev->ops->q_rx_start(cdev, &q_params,
-					   fp->rxq->rx_buf_size,
-					   fp->rxq->rx_bd_ring.p_phys_addr,
-					   phys_table,
-					   fp->rxq->rx_comp_ring.page_cnt,
-					   &fp->rxq->hw_rxq_prod_addr);
-		if (rc) {
-			DP_ERR(edev, "Start RXQ #%d failed %d\n", i, rc);
-			return rc;
-		}
+		dma_addr_t p_phys_table;
+		u32 page_cnt;
+
+		if (fp->type & QEDE_FASTPATH_RX) {
+			struct qede_rx_queue *rxq = fp->rxq;
+			__le16 *val;
+
+			memset(&q_params, 0, sizeof(q_params));
+			q_params.rss_id = i;
+			q_params.queue_id = rxq->rxq_id;
+			q_params.vport_id = 0;
+			q_params.sb = fp->sb_info->igu_sb_id;
+			q_params.sb_idx = RX_PI;
+
+			p_phys_table =
+			    qed_chain_get_pbl_phys(&rxq->rx_comp_ring);
+			page_cnt = qed_chain_get_page_cnt(&rxq->rx_comp_ring);
+
+			rc = edev->ops->q_rx_start(cdev, &q_params,
+						   rxq->rx_buf_size,
+						   rxq->rx_bd_ring.p_phys_addr,
+						   p_phys_table,
+						   page_cnt,
+						   &rxq->hw_rxq_prod_addr);
+			if (rc) {
+				DP_ERR(edev, "Start RXQ #%d failed %d\n", i,
+				       rc);
+				return rc;
+			}
 
-		fp->rxq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[RX_PI];
+			val = &fp->sb_info->sb_virt->pi_array[RX_PI];
+			rxq->hw_cons_ptr = val;
 
-		qede_update_rx_prod(edev, fp->rxq);
+			qede_update_rx_prod(edev, rxq);
+		}
+
+		if (!(fp->type & QEDE_FASTPATH_TX))
+			continue;
 
 		for (tc = 0; tc < edev->num_tc; tc++) {
 			struct qede_tx_queue *txq = &fp->txqs[tc];
-			int txq_index = tc * QEDE_RSS_CNT(edev) + i;
+
+			p_phys_table = qed_chain_get_pbl_phys(&txq->tx_pbl);
+			page_cnt = qed_chain_get_page_cnt(&txq->tx_pbl);
 
 			memset(&q_params, 0, sizeof(q_params));
 			q_params.rss_id = i;
-			q_params.queue_id = txq_index;
+			q_params.queue_id = txq->index;
 			q_params.vport_id = 0;
 			q_params.sb = fp->sb_info->igu_sb_id;
 			q_params.sb_idx = TX_PI(tc);
 
 			rc = edev->ops->q_tx_start(cdev, &q_params,
-						   txq->tx_pbl.pbl.p_phys_table,
-						   txq->tx_pbl.page_cnt,
+						   p_phys_table, page_cnt,
 						   &txq->doorbell_addr);
 			if (rc) {
 				DP_ERR(edev, "Start TXQ #%d failed %d\n",
-				       txq_index, rc);
+				       txq->index, rc);
 				return rc;
 			}
 
@@ -3358,13 +3442,13 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
 	}
 
 	/* Fill struct with RSS params */
-	if (QEDE_RSS_CNT(edev) > 1) {
+	if (QEDE_RSS_COUNT(edev) > 1) {
 		vport_update_params.update_rss_flg = 1;
 
 		/* Need to validate current RSS config uses valid entries */
 		for (i = 0; i < QED_RSS_IND_TABLE_SIZE; i++) {
 			if (edev->rss_params.rss_ind_table[i] >=
-			    edev->num_rss) {
+			    QEDE_RSS_COUNT(edev)) {
 				reset_rss_indir = true;
 				break;
 			}
@@ -3377,7 +3461,7 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
 			for (i = 0; i < QED_RSS_IND_TABLE_SIZE; i++) {
 				u16 indir_val;
 
-				val = QEDE_RSS_CNT(edev);
+				val = QEDE_RSS_COUNT(edev);
 				indir_val = ethtool_rxfh_indir_default(i, val);
 				edev->rss_params.rss_ind_table[i] = indir_val;
 			}
@@ -3443,6 +3527,7 @@ static void qede_unload(struct qede_dev *edev, enum qede_unload_mode mode)
 
 	DP_INFO(edev, "Starting qede unload\n");
 
+	qede_roce_dev_event_close(edev);
 	mutex_lock(&edev->qede_lock);
 	edev->state = QEDE_STATE_CLOSED;
 
@@ -3506,7 +3591,7 @@ static int qede_load(struct qede_dev *edev, enum qede_load_mode mode)
 	if (rc)
 		goto err1;
 	DP_INFO(edev, "Allocated %d RSS queues on %d TC/s\n",
-		QEDE_RSS_CNT(edev), edev->num_tc);
+		QEDE_QUEUE_CNT(edev), edev->num_tc);
 
 	rc = qede_set_real_num_queues(edev);
 	if (rc)
@@ -3543,6 +3628,7 @@ static int qede_load(struct qede_dev *edev, enum qede_load_mode mode)
 	/* Query whether link is already-up */
 	memset(&link_output, 0, sizeof(link_output));
 	edev->ops->common->get_link(edev->cdev, &link_output);
+	qede_roce_dev_event_open(edev);
 	qede_link_update(edev, &link_output);
 
 	DP_INFO(edev, "Ending successfully qede load\n");
@@ -3559,7 +3645,9 @@ err2:
 err1:
 	edev->ops->common->set_fp_int(edev->cdev, 0);
 	qede_free_fp_array(edev);
-	edev->num_rss = 0;
+	edev->num_queues = 0;
+	edev->fp_num_tx = 0;
+	edev->fp_num_rx = 0;
 err0:
 	return rc;
 }
diff --git a/drivers/net/ethernet/qlogic/qede/qede_roce.c b/drivers/net/ethernet/qlogic/qede/qede_roce.c
new file mode 100644
index 000000000000..9867f960b063
--- /dev/null
+++ b/drivers/net/ethernet/qlogic/qede/qede_roce.c
@@ -0,0 +1,314 @@
+/* QLogic qedr NIC Driver
+ * Copyright (c) 2015-2016  QLogic Corporation
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and /or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/qed/qede_roce.h>
+#include "qede.h"
+
+static struct qedr_driver *qedr_drv;
+static LIST_HEAD(qedr_dev_list);
+static DEFINE_MUTEX(qedr_dev_list_lock);
+
+bool qede_roce_supported(struct qede_dev *dev)
+{
+	return dev->dev_info.common.rdma_supported;
+}
+
+static void _qede_roce_dev_add(struct qede_dev *edev)
+{
+	if (!qedr_drv)
+		return;
+
+	edev->rdma_info.qedr_dev = qedr_drv->add(edev->cdev, edev->pdev,
+						 edev->ndev);
+}
+
+static int qede_roce_create_wq(struct qede_dev *edev)
+{
+	INIT_LIST_HEAD(&edev->rdma_info.roce_event_list);
+	edev->rdma_info.roce_wq = create_singlethread_workqueue("roce_wq");
+	if (!edev->rdma_info.roce_wq) {
+		DP_NOTICE(edev, "qedr: Could not create workqueue\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void qede_roce_cleanup_event(struct qede_dev *edev)
+{
+	struct list_head *head = &edev->rdma_info.roce_event_list;
+	struct qede_roce_event_work *event_node;
+
+	flush_workqueue(edev->rdma_info.roce_wq);
+	while (!list_empty(head)) {
+		event_node = list_entry(head->next, struct qede_roce_event_work,
+					list);
+		cancel_work_sync(&event_node->work);
+		list_del(&event_node->list);
+		kfree(event_node);
+	}
+}
+
+static void qede_roce_destroy_wq(struct qede_dev *edev)
+{
+	qede_roce_cleanup_event(edev);
+	destroy_workqueue(edev->rdma_info.roce_wq);
+}
+
+int qede_roce_dev_add(struct qede_dev *edev)
+{
+	int rc = 0;
+
+	if (qede_roce_supported(edev)) {
+		rc = qede_roce_create_wq(edev);
+		if (rc)
+			return rc;
+
+		INIT_LIST_HEAD(&edev->rdma_info.entry);
+		mutex_lock(&qedr_dev_list_lock);
+		list_add_tail(&edev->rdma_info.entry, &qedr_dev_list);
+		_qede_roce_dev_add(edev);
+		mutex_unlock(&qedr_dev_list_lock);
+	}
+
+	return rc;
+}
+
+static void _qede_roce_dev_remove(struct qede_dev *edev)
+{
+	if (qedr_drv && qedr_drv->remove && edev->rdma_info.qedr_dev)
+		qedr_drv->remove(edev->rdma_info.qedr_dev);
+	edev->rdma_info.qedr_dev = NULL;
+}
+
+void qede_roce_dev_remove(struct qede_dev *edev)
+{
+	if (!qede_roce_supported(edev))
+		return;
+
+	qede_roce_destroy_wq(edev);
+	mutex_lock(&qedr_dev_list_lock);
+	_qede_roce_dev_remove(edev);
+	list_del(&edev->rdma_info.entry);
+	mutex_unlock(&qedr_dev_list_lock);
+}
+
+static void _qede_roce_dev_open(struct qede_dev *edev)
+{
+	if (qedr_drv && edev->rdma_info.qedr_dev && qedr_drv->notify)
+		qedr_drv->notify(edev->rdma_info.qedr_dev, QEDE_UP);
+}
+
+static void qede_roce_dev_open(struct qede_dev *edev)
+{
+	if (!qede_roce_supported(edev))
+		return;
+
+	mutex_lock(&qedr_dev_list_lock);
+	_qede_roce_dev_open(edev);
+	mutex_unlock(&qedr_dev_list_lock);
+}
+
+static void _qede_roce_dev_close(struct qede_dev *edev)
+{
+	if (qedr_drv && edev->rdma_info.qedr_dev && qedr_drv->notify)
+		qedr_drv->notify(edev->rdma_info.qedr_dev, QEDE_DOWN);
+}
+
+static void qede_roce_dev_close(struct qede_dev *edev)
+{
+	if (!qede_roce_supported(edev))
+		return;
+
+	mutex_lock(&qedr_dev_list_lock);
+	_qede_roce_dev_close(edev);
+	mutex_unlock(&qedr_dev_list_lock);
+}
+
+static void qede_roce_dev_shutdown(struct qede_dev *edev)
+{
+	if (!qede_roce_supported(edev))
+		return;
+
+	mutex_lock(&qedr_dev_list_lock);
+	if (qedr_drv && edev->rdma_info.qedr_dev && qedr_drv->notify)
+		qedr_drv->notify(edev->rdma_info.qedr_dev, QEDE_CLOSE);
+	mutex_unlock(&qedr_dev_list_lock);
+}
+
+int qede_roce_register_driver(struct qedr_driver *drv)
+{
+	struct qede_dev *edev;
+	u8 qedr_counter = 0;
+
+	mutex_lock(&qedr_dev_list_lock);
+	if (qedr_drv) {
+		mutex_unlock(&qedr_dev_list_lock);
+		return -EINVAL;
+	}
+	qedr_drv = drv;
+
+	list_for_each_entry(edev, &qedr_dev_list, rdma_info.entry) {
+		struct net_device *ndev;
+
+		qedr_counter++;
+		_qede_roce_dev_add(edev);
+		ndev = edev->ndev;
+		if (netif_running(ndev) && netif_oper_up(ndev))
+			_qede_roce_dev_open(edev);
+	}
+	mutex_unlock(&qedr_dev_list_lock);
+
+	DP_INFO(edev, "qedr: discovered and registered %d RoCE funcs\n",
+		qedr_counter);
+
+	return 0;
+}
+EXPORT_SYMBOL(qede_roce_register_driver);
+
+void qede_roce_unregister_driver(struct qedr_driver *drv)
+{
+	struct qede_dev *edev;
+
+	mutex_lock(&qedr_dev_list_lock);
+	list_for_each_entry(edev, &qedr_dev_list, rdma_info.entry) {
+		if (edev->rdma_info.qedr_dev)
+			_qede_roce_dev_remove(edev);
+	}
+	qedr_drv = NULL;
+	mutex_unlock(&qedr_dev_list_lock);
+}
+EXPORT_SYMBOL(qede_roce_unregister_driver);
+
+static void qede_roce_changeaddr(struct qede_dev *edev)
+{
+	if (!qede_roce_supported(edev))
+		return;
+
+	if (qedr_drv && edev->rdma_info.qedr_dev && qedr_drv->notify)
+		qedr_drv->notify(edev->rdma_info.qedr_dev, QEDE_CHANGE_ADDR);
+}
+
+struct qede_roce_event_work *qede_roce_get_free_event_node(struct qede_dev
+							   *edev)
+{
+	struct qede_roce_event_work *event_node = NULL;
+	struct list_head *list_node = NULL;
+	bool found = false;
+
+	list_for_each(list_node, &edev->rdma_info.roce_event_list) {
+		event_node = list_entry(list_node, struct qede_roce_event_work,
+					list);
+		if (!work_pending(&event_node->work)) {
+			found = true;
+			break;
+		}
+	}
+
+	if (!found) {
+		event_node = kzalloc(sizeof(*event_node), GFP_KERNEL);
+		if (!event_node) {
+			DP_NOTICE(edev,
+				  "qedr: Could not allocate memory for roce work\n");
+			return NULL;
+		}
+		list_add_tail(&event_node->list,
+			      &edev->rdma_info.roce_event_list);
+	}
+
+	return event_node;
+}
+
+static void qede_roce_handle_event(struct work_struct *work)
+{
+	struct qede_roce_event_work *event_node;
+	enum qede_roce_event event;
+	struct qede_dev *edev;
+
+	event_node = container_of(work, struct qede_roce_event_work, work);
+	event = event_node->event;
+	edev = event_node->ptr;
+
+	switch (event) {
+	case QEDE_UP:
+		qede_roce_dev_open(edev);
+		break;
+	case QEDE_DOWN:
+		qede_roce_dev_close(edev);
+		break;
+	case QEDE_CLOSE:
+		qede_roce_dev_shutdown(edev);
+		break;
+	case QEDE_CHANGE_ADDR:
+		qede_roce_changeaddr(edev);
+		break;
+	default:
+		DP_NOTICE(edev, "Invalid roce event %d", event);
+	}
+}
+
+static void qede_roce_add_event(struct qede_dev *edev,
+				enum qede_roce_event event)
+{
+	struct qede_roce_event_work *event_node;
+
+	if (!edev->rdma_info.qedr_dev)
+		return;
+
+	event_node = qede_roce_get_free_event_node(edev);
+	if (!event_node)
+		return;
+
+	event_node->event = event;
+	event_node->ptr = edev;
+
+	INIT_WORK(&event_node->work, qede_roce_handle_event);
+	queue_work(edev->rdma_info.roce_wq, &event_node->work);
+}
+
+void qede_roce_dev_event_open(struct qede_dev *edev)
+{
+	qede_roce_add_event(edev, QEDE_UP);
+}
+
+void qede_roce_dev_event_close(struct qede_dev *edev)
+{
+	qede_roce_add_event(edev, QEDE_DOWN);
+}
+
+void qede_roce_event_changeaddr(struct qede_dev *edev)
+{
+	qede_roce_add_event(edev, QEDE_CHANGE_ADDR);
+}
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
index fd973f4f16c7..49bad00a0f8f 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
@@ -37,8 +37,8 @@
 
 #define _QLCNIC_LINUX_MAJOR 5
 #define _QLCNIC_LINUX_MINOR 3
-#define _QLCNIC_LINUX_SUBVERSION 64
-#define QLCNIC_LINUX_VERSIONID  "5.3.64"
+#define _QLCNIC_LINUX_SUBVERSION 65
+#define QLCNIC_LINUX_VERSIONID  "5.3.65"
 #define QLCNIC_DRV_IDC_VER  0x01
 #define QLCNIC_DRIVER_VERSION  ((_QLCNIC_LINUX_MAJOR << 16) |\
 		 (_QLCNIC_LINUX_MINOR << 8) | (_QLCNIC_LINUX_SUBVERSION))
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c
index 87c642d3b075..fedd7366713c 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c
@@ -102,7 +102,6 @@
 #define QLCNIC_RESPONSE_DESC	0x05
 #define QLCNIC_LRO_DESC  	0x12
 
-#define QLCNIC_TX_POLL_BUDGET		128
 #define QLCNIC_TCP_HDR_SIZE		20
 #define QLCNIC_TCP_TS_OPTION_SIZE	12
 #define QLCNIC_FETCH_RING_ID(handle)	((handle) >> 63)
@@ -2008,7 +2007,6 @@ static int qlcnic_83xx_msix_tx_poll(struct napi_struct *napi, int budget)
 	struct qlcnic_host_tx_ring *tx_ring;
 	struct qlcnic_adapter *adapter;
 
-	budget = QLCNIC_TX_POLL_BUDGET;
 	tx_ring = container_of(napi, struct qlcnic_host_tx_ring, napi);
 	adapter = tx_ring->adapter;
 	work_done = qlcnic_process_cmd_ring(adapter, tx_ring, budget);
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
index 3ebef27e0964..3ae3968b0edf 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
@@ -432,18 +432,19 @@ static int qlcnic_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
 
 static int qlcnic_fdb_dump(struct sk_buff *skb, struct netlink_callback *ncb,
 			struct net_device *netdev,
-			struct net_device *filter_dev, int idx)
+			struct net_device *filter_dev, int *idx)
 {
 	struct qlcnic_adapter *adapter = netdev_priv(netdev);
+	int err = 0;
 
 	if (!adapter->fdb_mac_learn)
 		return ndo_dflt_fdb_dump(skb, ncb, netdev, filter_dev, idx);
 
 	if ((adapter->flags & QLCNIC_ESWITCH_ENABLED) ||
 	    qlcnic_sriov_check(adapter))
-		idx = ndo_dflt_fdb_dump(skb, ncb, netdev, filter_dev, idx);
+		err = ndo_dflt_fdb_dump(skb, ncb, netdev, filter_dev, idx);
 
-	return idx;
+	return err;
 }
 
 static void qlcnic_82xx_cancel_idc_work(struct qlcnic_adapter *adapter)
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov.h b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov.h
index 017d8c2c8285..5f327659efa7 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov.h
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov.h
@@ -156,10 +156,8 @@ struct qlcnic_vf_info {
 	spinlock_t			vlan_list_lock;	/* Lock for VLAN list */
 };
 
-struct qlcnic_async_work_list {
+struct qlcnic_async_cmd {
 	struct list_head	list;
-	struct work_struct	work;
-	void			*ptr;
 	struct qlcnic_cmd_args	*cmd;
 };
 
@@ -168,7 +166,10 @@ struct qlcnic_back_channel {
 	struct workqueue_struct *bc_trans_wq;
 	struct workqueue_struct *bc_async_wq;
 	struct workqueue_struct *bc_flr_wq;
-	struct list_head	async_list;
+	struct qlcnic_adapter	*adapter;
+	struct list_head	async_cmd_list;
+	struct work_struct	vf_async_work;
+	spinlock_t		queue_lock; /* async_cmd_list queue lock */
 };
 
 struct qlcnic_sriov {
@@ -237,7 +238,7 @@ int qlcnic_sriov_set_vf_mac(struct net_device *, int, u8 *);
 int qlcnic_sriov_set_vf_tx_rate(struct net_device *, int, int, int);
 int qlcnic_sriov_get_vf_config(struct net_device *, int ,
 			       struct ifla_vf_info *);
-int qlcnic_sriov_set_vf_vlan(struct net_device *, int, u16, u8);
+int qlcnic_sriov_set_vf_vlan(struct net_device *, int, u16, u8, __be16);
 int qlcnic_sriov_set_vf_spoofchk(struct net_device *, int, bool);
 #else
 static inline void qlcnic_sriov_pf_disable(struct qlcnic_adapter *adapter) {}
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c
index 7327b729ba2e..d7107055ec60 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c
@@ -29,6 +29,7 @@
 #define QLC_83XX_VF_RESET_FAIL_THRESH	8
 #define QLC_BC_CMD_MAX_RETRY_CNT	5
 
+static void qlcnic_sriov_handle_async_issue_cmd(struct work_struct *work);
 static void qlcnic_sriov_vf_free_mac_list(struct qlcnic_adapter *);
 static int qlcnic_sriov_alloc_bc_mbx_args(struct qlcnic_cmd_args *, u32);
 static void qlcnic_sriov_vf_poll_dev_state(struct work_struct *);
@@ -177,7 +178,10 @@ int qlcnic_sriov_init(struct qlcnic_adapter *adapter, int num_vfs)
 	}
 
 	bc->bc_async_wq =  wq;
-	INIT_LIST_HEAD(&bc->async_list);
+	INIT_LIST_HEAD(&bc->async_cmd_list);
+	INIT_WORK(&bc->vf_async_work, qlcnic_sriov_handle_async_issue_cmd);
+	spin_lock_init(&bc->queue_lock);
+	bc->adapter = adapter;
 
 	for (i = 0; i < num_vfs; i++) {
 		vf = &sriov->vf_info[i];
@@ -1517,17 +1521,21 @@ static void qlcnic_vf_add_mc_list(struct net_device *netdev, const u8 *mac,
 
 void qlcnic_sriov_cleanup_async_list(struct qlcnic_back_channel *bc)
 {
-	struct list_head *head = &bc->async_list;
-	struct qlcnic_async_work_list *entry;
+	struct list_head *head = &bc->async_cmd_list;
+	struct qlcnic_async_cmd *entry;
 
 	flush_workqueue(bc->bc_async_wq);
+	cancel_work_sync(&bc->vf_async_work);
+
+	spin_lock(&bc->queue_lock);
 	while (!list_empty(head)) {
-		entry = list_entry(head->next, struct qlcnic_async_work_list,
+		entry = list_entry(head->next, struct qlcnic_async_cmd,
 				   list);
-		cancel_work_sync(&entry->work);
 		list_del(&entry->list);
+		kfree(entry->cmd);
 		kfree(entry);
 	}
+	spin_unlock(&bc->queue_lock);
 }
 
 void qlcnic_sriov_vf_set_multi(struct net_device *netdev)
@@ -1587,57 +1595,64 @@ void qlcnic_sriov_vf_set_multi(struct net_device *netdev)
 
 static void qlcnic_sriov_handle_async_issue_cmd(struct work_struct *work)
 {
-	struct qlcnic_async_work_list *entry;
-	struct qlcnic_adapter *adapter;
+	struct qlcnic_async_cmd *entry, *tmp;
+	struct qlcnic_back_channel *bc;
 	struct qlcnic_cmd_args *cmd;
+	struct list_head *head;
+	LIST_HEAD(del_list);
+
+	bc = container_of(work, struct qlcnic_back_channel, vf_async_work);
+	head = &bc->async_cmd_list;
+
+	spin_lock(&bc->queue_lock);
+	list_splice_init(head, &del_list);
+	spin_unlock(&bc->queue_lock);
+
+	list_for_each_entry_safe(entry, tmp, &del_list, list) {
+		list_del(&entry->list);
+		cmd = entry->cmd;
+		__qlcnic_sriov_issue_cmd(bc->adapter, cmd);
+		kfree(entry);
+	}
+
+	if (!list_empty(head))
+		queue_work(bc->bc_async_wq, &bc->vf_async_work);
 
-	entry = container_of(work, struct qlcnic_async_work_list, work);
-	adapter = entry->ptr;
-	cmd = entry->cmd;
-	__qlcnic_sriov_issue_cmd(adapter, cmd);
 	return;
 }
 
-static struct qlcnic_async_work_list *
-qlcnic_sriov_get_free_node_async_work(struct qlcnic_back_channel *bc)
+static struct qlcnic_async_cmd *
+qlcnic_sriov_alloc_async_cmd(struct qlcnic_back_channel *bc,
+			     struct qlcnic_cmd_args *cmd)
 {
-	struct list_head *node;
-	struct qlcnic_async_work_list *entry = NULL;
-	u8 empty = 0;
+	struct qlcnic_async_cmd *entry = NULL;
 
-	list_for_each(node, &bc->async_list) {
-		entry = list_entry(node, struct qlcnic_async_work_list, list);
-		if (!work_pending(&entry->work)) {
-			empty = 1;
-			break;
-		}
-	}
+	entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
+	if (!entry)
+		return NULL;
 
-	if (!empty) {
-		entry = kzalloc(sizeof(struct qlcnic_async_work_list),
-				GFP_ATOMIC);
-		if (entry == NULL)
-			return NULL;
-		list_add_tail(&entry->list, &bc->async_list);
-	}
+	entry->cmd = cmd;
+
+	spin_lock(&bc->queue_lock);
+	list_add_tail(&entry->list, &bc->async_cmd_list);
+	spin_unlock(&bc->queue_lock);
 
 	return entry;
 }
 
 static void qlcnic_sriov_schedule_async_cmd(struct qlcnic_back_channel *bc,
-					    work_func_t func, void *data,
 					    struct qlcnic_cmd_args *cmd)
 {
-	struct qlcnic_async_work_list *entry = NULL;
+	struct qlcnic_async_cmd *entry = NULL;
 
-	entry = qlcnic_sriov_get_free_node_async_work(bc);
-	if (!entry)
+	entry = qlcnic_sriov_alloc_async_cmd(bc, cmd);
+	if (!entry) {
+		qlcnic_free_mbx_args(cmd);
+		kfree(cmd);
 		return;
+	}
 
-	entry->ptr = data;
-	entry->cmd = cmd;
-	INIT_WORK(&entry->work, func);
-	queue_work(bc->bc_async_wq, &entry->work);
+	queue_work(bc->bc_async_wq, &bc->vf_async_work);
 }
 
 static int qlcnic_sriov_async_issue_cmd(struct qlcnic_adapter *adapter,
@@ -1649,8 +1664,8 @@ static int qlcnic_sriov_async_issue_cmd(struct qlcnic_adapter *adapter,
 	if (adapter->need_fw_reset)
 		return -EIO;
 
-	qlcnic_sriov_schedule_async_cmd(bc, qlcnic_sriov_handle_async_issue_cmd,
-					adapter, cmd);
+	qlcnic_sriov_schedule_async_cmd(bc, cmd);
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c
index afd687e5e779..50eaafa3eaba 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c
@@ -1915,7 +1915,7 @@ int qlcnic_sriov_set_vf_tx_rate(struct net_device *netdev, int vf,
 }
 
 int qlcnic_sriov_set_vf_vlan(struct net_device *netdev, int vf,
-			     u16 vlan, u8 qos)
+			     u16 vlan, u8 qos, __be16 vlan_proto)
 {
 	struct qlcnic_adapter *adapter = netdev_priv(netdev);
 	struct qlcnic_sriov *sriov = adapter->ahw->sriov;
@@ -1928,6 +1928,9 @@ int qlcnic_sriov_set_vf_vlan(struct net_device *netdev, int vf,
 	if (vf >= sriov->num_vfs || qos > 7)
 		return -EINVAL;
 
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
 	if (vlan > MAX_VLAN_ID) {
 		netdev_err(netdev,
 			   "Invalid VLAN ID, allowed range is [0 - %d]\n",
diff --git a/drivers/net/ethernet/qualcomm/Kconfig b/drivers/net/ethernet/qualcomm/Kconfig
index a76e380cf89a..d7720bf92d49 100644
--- a/drivers/net/ethernet/qualcomm/Kconfig
+++ b/drivers/net/ethernet/qualcomm/Kconfig
@@ -24,4 +24,17 @@ config QCA7000
 	  To compile this driver as a module, choose M here. The module
 	  will be called qcaspi.
 
+config QCOM_EMAC
+	tristate "Qualcomm Technologies, Inc. EMAC Gigabit Ethernet support"
+	depends on HAS_DMA && HAS_IOMEM
+	select CRC32
+	select PHYLIB
+	---help---
+	  This driver supports the Qualcomm Technologies, Inc. Gigabit
+	  Ethernet Media Access Controller (EMAC). The controller
+	  supports IEEE 802.3-2002, half-duplex mode at 10/100 Mb/s,
+	  full-duplex mode at 10/100/1000Mb/s, Wake On LAN (WOL) for
+	  low power, Receive-Side Scaling (RSS), and IEEE 1588-2008
+	  Precision Clock Synchronization Protocol.
+
 endif # NET_VENDOR_QUALCOMM
diff --git a/drivers/net/ethernet/qualcomm/Makefile b/drivers/net/ethernet/qualcomm/Makefile
index 9da2d75db700..aacb0a585c68 100644
--- a/drivers/net/ethernet/qualcomm/Makefile
+++ b/drivers/net/ethernet/qualcomm/Makefile
@@ -4,3 +4,5 @@
 
 obj-$(CONFIG_QCA7000) += qcaspi.o
 qcaspi-objs := qca_spi.o qca_framing.o qca_7k.o qca_debug.o
+
+obj-y += emac/
diff --git a/drivers/net/ethernet/qualcomm/emac/Makefile b/drivers/net/ethernet/qualcomm/emac/Makefile
new file mode 100644
index 000000000000..01ee144c6386
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/emac/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for the Qualcomm Technologies, Inc. EMAC Gigabit Ethernet driver
+#
+
+obj-$(CONFIG_QCOM_EMAC) += qcom-emac.o
+
+qcom-emac-objs := emac.o emac-mac.o emac-phy.o emac-sgmii.o
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-mac.c b/drivers/net/ethernet/qualcomm/emac/emac-mac.c
new file mode 100644
index 000000000000..e97968ed4b8f
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/emac/emac-mac.c
@@ -0,0 +1,1528 @@
+/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support
+ */
+
+#include <linux/tcp.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/crc32.h>
+#include <linux/if_vlan.h>
+#include <linux/jiffies.h>
+#include <linux/phy.h>
+#include <linux/of.h>
+#include <net/ip6_checksum.h>
+#include "emac.h"
+#include "emac-sgmii.h"
+
+/* EMAC base register offsets */
+#define EMAC_MAC_CTRL			0x001480
+#define EMAC_WOL_CTRL0			0x0014a0
+#define EMAC_RSS_KEY0			0x0014b0
+#define EMAC_H1TPD_BASE_ADDR_LO		0x0014e0
+#define EMAC_H2TPD_BASE_ADDR_LO		0x0014e4
+#define EMAC_H3TPD_BASE_ADDR_LO		0x0014e8
+#define EMAC_INTER_SRAM_PART9		0x001534
+#define EMAC_DESC_CTRL_0		0x001540
+#define EMAC_DESC_CTRL_1		0x001544
+#define EMAC_DESC_CTRL_2		0x001550
+#define EMAC_DESC_CTRL_10		0x001554
+#define EMAC_DESC_CTRL_12		0x001558
+#define EMAC_DESC_CTRL_13		0x00155c
+#define EMAC_DESC_CTRL_3		0x001560
+#define EMAC_DESC_CTRL_4		0x001564
+#define EMAC_DESC_CTRL_5		0x001568
+#define EMAC_DESC_CTRL_14		0x00156c
+#define EMAC_DESC_CTRL_15		0x001570
+#define EMAC_DESC_CTRL_16		0x001574
+#define EMAC_DESC_CTRL_6		0x001578
+#define EMAC_DESC_CTRL_8		0x001580
+#define EMAC_DESC_CTRL_9		0x001584
+#define EMAC_DESC_CTRL_11		0x001588
+#define EMAC_TXQ_CTRL_0			0x001590
+#define EMAC_TXQ_CTRL_1			0x001594
+#define EMAC_TXQ_CTRL_2			0x001598
+#define EMAC_RXQ_CTRL_0			0x0015a0
+#define EMAC_RXQ_CTRL_1			0x0015a4
+#define EMAC_RXQ_CTRL_2			0x0015a8
+#define EMAC_RXQ_CTRL_3			0x0015ac
+#define EMAC_BASE_CPU_NUMBER		0x0015b8
+#define EMAC_DMA_CTRL			0x0015c0
+#define EMAC_MAILBOX_0			0x0015e0
+#define EMAC_MAILBOX_5			0x0015e4
+#define EMAC_MAILBOX_6			0x0015e8
+#define EMAC_MAILBOX_13			0x0015ec
+#define EMAC_MAILBOX_2			0x0015f4
+#define EMAC_MAILBOX_3			0x0015f8
+#define EMAC_MAILBOX_11			0x00160c
+#define EMAC_AXI_MAST_CTRL		0x001610
+#define EMAC_MAILBOX_12			0x001614
+#define EMAC_MAILBOX_9			0x001618
+#define EMAC_MAILBOX_10			0x00161c
+#define EMAC_ATHR_HEADER_CTRL		0x001620
+#define EMAC_CLK_GATE_CTRL		0x001814
+#define EMAC_MISC_CTRL			0x001990
+#define EMAC_MAILBOX_7			0x0019e0
+#define EMAC_MAILBOX_8			0x0019e4
+#define EMAC_MAILBOX_15			0x001bd4
+#define EMAC_MAILBOX_16			0x001bd8
+
+/* EMAC_MAC_CTRL */
+#define SINGLE_PAUSE_MODE       	0x10000000
+#define DEBUG_MODE                      0x08000000
+#define BROAD_EN                        0x04000000
+#define MULTI_ALL                       0x02000000
+#define RX_CHKSUM_EN                    0x01000000
+#define HUGE                            0x00800000
+#define SPEED(x)			(((x) & 0x3) << 20)
+#define SPEED_MASK			SPEED(0x3)
+#define SIMR                            0x00080000
+#define TPAUSE                          0x00010000
+#define PROM_MODE                       0x00008000
+#define VLAN_STRIP                      0x00004000
+#define PRLEN_BMSK                      0x00003c00
+#define PRLEN_SHFT                      10
+#define HUGEN                           0x00000200
+#define FLCHK                           0x00000100
+#define PCRCE                           0x00000080
+#define CRCE                            0x00000040
+#define FULLD                           0x00000020
+#define MAC_LP_EN                       0x00000010
+#define RXFC                            0x00000008
+#define TXFC                            0x00000004
+#define RXEN                            0x00000002
+#define TXEN                            0x00000001
+
+
+/* EMAC_WOL_CTRL0 */
+#define LK_CHG_PME			0x20
+#define LK_CHG_EN			0x10
+#define MG_FRAME_PME			0x8
+#define MG_FRAME_EN			0x4
+#define WK_FRAME_EN			0x1
+
+/* EMAC_DESC_CTRL_3 */
+#define RFD_RING_SIZE_BMSK                                       0xfff
+
+/* EMAC_DESC_CTRL_4 */
+#define RX_BUFFER_SIZE_BMSK                                     0xffff
+
+/* EMAC_DESC_CTRL_6 */
+#define RRD_RING_SIZE_BMSK                                       0xfff
+
+/* EMAC_DESC_CTRL_9 */
+#define TPD_RING_SIZE_BMSK                                      0xffff
+
+/* EMAC_TXQ_CTRL_0 */
+#define NUM_TXF_BURST_PREF_BMSK                             0xffff0000
+#define NUM_TXF_BURST_PREF_SHFT                                     16
+#define LS_8023_SP                                                0x80
+#define TXQ_MODE                                                  0x40
+#define TXQ_EN                                                    0x20
+#define IP_OP_SP                                                  0x10
+#define NUM_TPD_BURST_PREF_BMSK                                    0xf
+#define NUM_TPD_BURST_PREF_SHFT                                      0
+
+/* EMAC_TXQ_CTRL_1 */
+#define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK                        0x7ff
+
+/* EMAC_TXQ_CTRL_2 */
+#define TXF_HWM_BMSK                                         0xfff0000
+#define TXF_LWM_BMSK                                             0xfff
+
+/* EMAC_RXQ_CTRL_0 */
+#define RXQ_EN                                                 BIT(31)
+#define CUT_THRU_EN                                            BIT(30)
+#define RSS_HASH_EN                                            BIT(29)
+#define NUM_RFD_BURST_PREF_BMSK                              0x3f00000
+#define NUM_RFD_BURST_PREF_SHFT                                     20
+#define IDT_TABLE_SIZE_BMSK                                    0x1ff00
+#define IDT_TABLE_SIZE_SHFT                                          8
+#define SP_IPV6                                                   0x80
+
+/* EMAC_RXQ_CTRL_1 */
+#define JUMBO_1KAH_BMSK                                         0xf000
+#define JUMBO_1KAH_SHFT                                             12
+#define RFD_PREF_LOW_TH                                           0x10
+#define RFD_PREF_LOW_THRESHOLD_BMSK                              0xfc0
+#define RFD_PREF_LOW_THRESHOLD_SHFT                                  6
+#define RFD_PREF_UP_TH                                            0x10
+#define RFD_PREF_UP_THRESHOLD_BMSK                                0x3f
+#define RFD_PREF_UP_THRESHOLD_SHFT                                   0
+
+/* EMAC_RXQ_CTRL_2 */
+#define RXF_DOF_THRESFHOLD                                       0x1a0
+#define RXF_DOF_THRESHOLD_BMSK                               0xfff0000
+#define RXF_DOF_THRESHOLD_SHFT                                      16
+#define RXF_UOF_THRESFHOLD                                        0xbe
+#define RXF_UOF_THRESHOLD_BMSK                                   0xfff
+#define RXF_UOF_THRESHOLD_SHFT                                       0
+
+/* EMAC_RXQ_CTRL_3 */
+#define RXD_TIMER_BMSK                                      0xffff0000
+#define RXD_THRESHOLD_BMSK                                       0xfff
+#define RXD_THRESHOLD_SHFT                                           0
+
+/* EMAC_DMA_CTRL */
+#define DMAW_DLY_CNT_BMSK                                      0xf0000
+#define DMAW_DLY_CNT_SHFT                                           16
+#define DMAR_DLY_CNT_BMSK                                       0xf800
+#define DMAR_DLY_CNT_SHFT                                           11
+#define DMAR_REQ_PRI                                             0x400
+#define REGWRBLEN_BMSK                                           0x380
+#define REGWRBLEN_SHFT                                               7
+#define REGRDBLEN_BMSK                                            0x70
+#define REGRDBLEN_SHFT                                               4
+#define OUT_ORDER_MODE                                             0x4
+#define ENH_ORDER_MODE                                             0x2
+#define IN_ORDER_MODE                                              0x1
+
+/* EMAC_MAILBOX_13 */
+#define RFD3_PROC_IDX_BMSK                                   0xfff0000
+#define RFD3_PROC_IDX_SHFT                                          16
+#define RFD3_PROD_IDX_BMSK                                       0xfff
+#define RFD3_PROD_IDX_SHFT                                           0
+
+/* EMAC_MAILBOX_2 */
+#define NTPD_CONS_IDX_BMSK                                  0xffff0000
+#define NTPD_CONS_IDX_SHFT                                          16
+
+/* EMAC_MAILBOX_3 */
+#define RFD0_CONS_IDX_BMSK                                       0xfff
+#define RFD0_CONS_IDX_SHFT                                           0
+
+/* EMAC_MAILBOX_11 */
+#define H3TPD_PROD_IDX_BMSK                                 0xffff0000
+#define H3TPD_PROD_IDX_SHFT                                         16
+
+/* EMAC_AXI_MAST_CTRL */
+#define DATA_BYTE_SWAP                                             0x8
+#define MAX_BOUND                                                  0x2
+#define MAX_BTYPE                                                  0x1
+
+/* EMAC_MAILBOX_12 */
+#define H3TPD_CONS_IDX_BMSK                                 0xffff0000
+#define H3TPD_CONS_IDX_SHFT                                         16
+
+/* EMAC_MAILBOX_9 */
+#define H2TPD_PROD_IDX_BMSK                                     0xffff
+#define H2TPD_PROD_IDX_SHFT                                          0
+
+/* EMAC_MAILBOX_10 */
+#define H1TPD_CONS_IDX_BMSK                                 0xffff0000
+#define H1TPD_CONS_IDX_SHFT                                         16
+#define H2TPD_CONS_IDX_BMSK                                     0xffff
+#define H2TPD_CONS_IDX_SHFT                                          0
+
+/* EMAC_ATHR_HEADER_CTRL */
+#define HEADER_CNT_EN                                              0x2
+#define HEADER_ENABLE                                              0x1
+
+/* EMAC_MAILBOX_0 */
+#define RFD0_PROC_IDX_BMSK                                   0xfff0000
+#define RFD0_PROC_IDX_SHFT                                          16
+#define RFD0_PROD_IDX_BMSK                                       0xfff
+#define RFD0_PROD_IDX_SHFT                                           0
+
+/* EMAC_MAILBOX_5 */
+#define RFD1_PROC_IDX_BMSK                                   0xfff0000
+#define RFD1_PROC_IDX_SHFT                                          16
+#define RFD1_PROD_IDX_BMSK                                       0xfff
+#define RFD1_PROD_IDX_SHFT                                           0
+
+/* EMAC_MISC_CTRL */
+#define RX_UNCPL_INT_EN                                            0x1
+
+/* EMAC_MAILBOX_7 */
+#define RFD2_CONS_IDX_BMSK                                   0xfff0000
+#define RFD2_CONS_IDX_SHFT                                          16
+#define RFD1_CONS_IDX_BMSK                                       0xfff
+#define RFD1_CONS_IDX_SHFT                                           0
+
+/* EMAC_MAILBOX_8 */
+#define RFD3_CONS_IDX_BMSK                                       0xfff
+#define RFD3_CONS_IDX_SHFT                                           0
+
+/* EMAC_MAILBOX_15 */
+#define NTPD_PROD_IDX_BMSK                                      0xffff
+#define NTPD_PROD_IDX_SHFT                                           0
+
+/* EMAC_MAILBOX_16 */
+#define H1TPD_PROD_IDX_BMSK                                     0xffff
+#define H1TPD_PROD_IDX_SHFT                                          0
+
+#define RXQ0_RSS_HSTYP_IPV6_TCP_EN                                0x20
+#define RXQ0_RSS_HSTYP_IPV6_EN                                    0x10
+#define RXQ0_RSS_HSTYP_IPV4_TCP_EN                                 0x8
+#define RXQ0_RSS_HSTYP_IPV4_EN                                     0x4
+
+/* EMAC_EMAC_WRAPPER_TX_TS_INX */
+#define EMAC_WRAPPER_TX_TS_EMPTY                               BIT(31)
+#define EMAC_WRAPPER_TX_TS_INX_BMSK                             0xffff
+
+struct emac_skb_cb {
+	u32           tpd_idx;
+	unsigned long jiffies;
+};
+
+#define EMAC_SKB_CB(skb)	((struct emac_skb_cb *)(skb)->cb)
+#define EMAC_RSS_IDT_SIZE	256
+#define JUMBO_1KAH		0x4
+#define RXD_TH			0x100
+#define EMAC_TPD_LAST_FRAGMENT	0x80000000
+#define EMAC_TPD_TSTAMP_SAVE	0x80000000
+
+/* EMAC Errors in emac_rrd.word[3] */
+#define EMAC_RRD_L4F		BIT(14)
+#define EMAC_RRD_IPF		BIT(15)
+#define EMAC_RRD_CRC		BIT(21)
+#define EMAC_RRD_FAE		BIT(22)
+#define EMAC_RRD_TRN		BIT(23)
+#define EMAC_RRD_RNT		BIT(24)
+#define EMAC_RRD_INC		BIT(25)
+#define EMAC_RRD_FOV		BIT(29)
+#define EMAC_RRD_LEN		BIT(30)
+
+/* Error bits that will result in a received frame being discarded */
+#define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \
+			EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \
+			EMAC_RRD_FOV | EMAC_RRD_LEN)
+#define EMAC_RRD_STATS_DW_IDX 3
+
+#define EMAC_RRD(RXQ, SIZE, IDX)	((RXQ)->rrd.v_addr + (SIZE * (IDX)))
+#define EMAC_RFD(RXQ, SIZE, IDX)	((RXQ)->rfd.v_addr + (SIZE * (IDX)))
+#define EMAC_TPD(TXQ, SIZE, IDX)	((TXQ)->tpd.v_addr + (SIZE * (IDX)))
+
+#define GET_RFD_BUFFER(RXQ, IDX)	(&((RXQ)->rfd.rfbuff[(IDX)]))
+#define GET_TPD_BUFFER(RTQ, IDX)	(&((RTQ)->tpd.tpbuff[(IDX)]))
+
+#define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD	8
+
+#define ISR_RX_PKT      (\
+	RX_PKT_INT0     |\
+	RX_PKT_INT1     |\
+	RX_PKT_INT2     |\
+	RX_PKT_INT3)
+
+#define EMAC_MAC_IRQ_RES                                    	"core0"
+
+void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr)
+{
+	u32 crc32, bit, reg, mta;
+
+	/* Calculate the CRC of the MAC address */
+	crc32 = ether_crc(ETH_ALEN, addr);
+
+	/* The HASH Table is an array of 2 32-bit registers. It is
+	 * treated like an array of 64 bits (BitArray[hash_value]).
+	 * Use the upper 6 bits of the above CRC as the hash value.
+	 */
+	reg = (crc32 >> 31) & 0x1;
+	bit = (crc32 >> 26) & 0x1F;
+
+	mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
+	mta |= BIT(bit);
+	writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
+}
+
+void emac_mac_multicast_addr_clear(struct emac_adapter *adpt)
+{
+	writel(0, adpt->base + EMAC_HASH_TAB_REG0);
+	writel(0, adpt->base + EMAC_HASH_TAB_REG1);
+}
+
+/* definitions for RSS */
+#define EMAC_RSS_KEY(_i, _type) \
+		(EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))
+#define EMAC_RSS_TBL(_i, _type) \
+		(EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))
+
+/* Config MAC modes */
+void emac_mac_mode_config(struct emac_adapter *adpt)
+{
+	struct net_device *netdev = adpt->netdev;
+	u32 mac;
+
+	mac = readl(adpt->base + EMAC_MAC_CTRL);
+	mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN);
+
+	if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
+		mac |= VLAN_STRIP;
+
+	if (netdev->flags & IFF_PROMISC)
+		mac |= PROM_MODE;
+
+	if (netdev->flags & IFF_ALLMULTI)
+		mac |= MULTI_ALL;
+
+	writel(mac, adpt->base + EMAC_MAC_CTRL);
+}
+
+/* Config descriptor rings */
+static void emac_mac_dma_rings_config(struct emac_adapter *adpt)
+{
+	static const unsigned short tpd_q_offset[] = {
+		EMAC_DESC_CTRL_8,        EMAC_H1TPD_BASE_ADDR_LO,
+		EMAC_H2TPD_BASE_ADDR_LO, EMAC_H3TPD_BASE_ADDR_LO};
+	static const unsigned short rfd_q_offset[] = {
+		EMAC_DESC_CTRL_2,        EMAC_DESC_CTRL_10,
+		EMAC_DESC_CTRL_12,       EMAC_DESC_CTRL_13};
+	static const unsigned short rrd_q_offset[] = {
+		EMAC_DESC_CTRL_5,        EMAC_DESC_CTRL_14,
+		EMAC_DESC_CTRL_15,       EMAC_DESC_CTRL_16};
+
+	/* TPD (Transmit Packet Descriptor) */
+	writel(upper_32_bits(adpt->tx_q.tpd.dma_addr),
+	       adpt->base + EMAC_DESC_CTRL_1);
+
+	writel(lower_32_bits(adpt->tx_q.tpd.dma_addr),
+	       adpt->base + tpd_q_offset[0]);
+
+	writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK,
+	       adpt->base + EMAC_DESC_CTRL_9);
+
+	/* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */
+	writel(upper_32_bits(adpt->rx_q.rfd.dma_addr),
+	       adpt->base + EMAC_DESC_CTRL_0);
+
+	writel(lower_32_bits(adpt->rx_q.rfd.dma_addr),
+	       adpt->base + rfd_q_offset[0]);
+	writel(lower_32_bits(adpt->rx_q.rrd.dma_addr),
+	       adpt->base + rrd_q_offset[0]);
+
+	writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK,
+	       adpt->base + EMAC_DESC_CTRL_3);
+	writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK,
+	       adpt->base + EMAC_DESC_CTRL_6);
+
+	writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK,
+	       adpt->base + EMAC_DESC_CTRL_4);
+
+	writel(0, adpt->base + EMAC_DESC_CTRL_11);
+
+	/* Load all of the base addresses above and ensure that triggering HW to
+	 * read ring pointers is flushed
+	 */
+	writel(1, adpt->base + EMAC_INTER_SRAM_PART9);
+}
+
+/* Config transmit parameters */
+static void emac_mac_tx_config(struct emac_adapter *adpt)
+{
+	u32 val;
+
+	writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) &
+	       JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1);
+
+	val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) &
+	       NUM_TPD_BURST_PREF_BMSK;
+
+	val |= TXQ_MODE | LS_8023_SP;
+	val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) &
+		NUM_TXF_BURST_PREF_BMSK;
+
+	writel(val, adpt->base + EMAC_TXQ_CTRL_0);
+	emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2,
+			  (TXF_HWM_BMSK | TXF_LWM_BMSK), 0);
+}
+
+/* Config receive parameters */
+static void emac_mac_rx_config(struct emac_adapter *adpt)
+{
+	u32 val;
+
+	val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) &
+	       NUM_RFD_BURST_PREF_BMSK;
+	val |= (SP_IPV6 | CUT_THRU_EN);
+
+	writel(val, adpt->base + EMAC_RXQ_CTRL_0);
+
+	val = readl(adpt->base + EMAC_RXQ_CTRL_1);
+	val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK |
+		 RFD_PREF_UP_THRESHOLD_BMSK);
+	val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) |
+		(RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) |
+		(RFD_PREF_UP_TH  << RFD_PREF_UP_THRESHOLD_SHFT);
+	writel(val, adpt->base + EMAC_RXQ_CTRL_1);
+
+	val = readl(adpt->base + EMAC_RXQ_CTRL_2);
+	val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK);
+	val |= (RXF_DOF_THRESFHOLD  << RXF_DOF_THRESHOLD_SHFT) |
+		(RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT);
+	writel(val, adpt->base + EMAC_RXQ_CTRL_2);
+
+	val = readl(adpt->base + EMAC_RXQ_CTRL_3);
+	val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK);
+	val |= RXD_TH << RXD_THRESHOLD_SHFT;
+	writel(val, adpt->base + EMAC_RXQ_CTRL_3);
+}
+
+/* Config dma */
+static void emac_mac_dma_config(struct emac_adapter *adpt)
+{
+	u32 dma_ctrl = DMAR_REQ_PRI;
+
+	switch (adpt->dma_order) {
+	case emac_dma_ord_in:
+		dma_ctrl |= IN_ORDER_MODE;
+		break;
+	case emac_dma_ord_enh:
+		dma_ctrl |= ENH_ORDER_MODE;
+		break;
+	case emac_dma_ord_out:
+		dma_ctrl |= OUT_ORDER_MODE;
+		break;
+	default:
+		break;
+	}
+
+	dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) &
+						REGRDBLEN_BMSK;
+	dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) &
+						REGWRBLEN_BMSK;
+	dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) &
+						DMAR_DLY_CNT_BMSK;
+	dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) &
+						DMAW_DLY_CNT_BMSK;
+
+	/* config DMA and ensure that configuration is flushed to HW */
+	writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL);
+}
+
+/* set MAC address */
+static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr)
+{
+	u32 sta;
+
+	/* for example: 00-A0-C6-11-22-33
+	 * 0<-->C6112233, 1<-->00A0.
+	 */
+
+	/* low 32bit word */
+	sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) |
+	      (((u32)addr[4]) << 8)  | (((u32)addr[5]));
+	writel(sta, adpt->base + EMAC_MAC_STA_ADDR0);
+
+	/* hight 32bit word */
+	sta = (((u32)addr[0]) << 8) | (u32)addr[1];
+	writel(sta, adpt->base + EMAC_MAC_STA_ADDR1);
+}
+
+static void emac_mac_config(struct emac_adapter *adpt)
+{
+	struct net_device *netdev = adpt->netdev;
+	unsigned int max_frame;
+	u32 val;
+
+	emac_set_mac_address(adpt, netdev->dev_addr);
+
+	max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
+	adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?
+		ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;
+
+	emac_mac_dma_rings_config(adpt);
+
+	writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN,
+	       adpt->base + EMAC_MAX_FRAM_LEN_CTRL);
+
+	emac_mac_tx_config(adpt);
+	emac_mac_rx_config(adpt);
+	emac_mac_dma_config(adpt);
+
+	val = readl(adpt->base + EMAC_AXI_MAST_CTRL);
+	val &= ~(DATA_BYTE_SWAP | MAX_BOUND);
+	val |= MAX_BTYPE;
+	writel(val, adpt->base + EMAC_AXI_MAST_CTRL);
+	writel(0, adpt->base + EMAC_CLK_GATE_CTRL);
+	writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL);
+}
+
+void emac_mac_reset(struct emac_adapter *adpt)
+{
+	emac_mac_stop(adpt);
+
+	emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST);
+	usleep_range(100, 150); /* reset may take up to 100usec */
+
+	/* interrupt clear-on-read */
+	emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN);
+}
+
+void emac_mac_start(struct emac_adapter *adpt)
+{
+	struct phy_device *phydev = adpt->phydev;
+	u32 mac, csr1;
+
+	/* enable tx queue */
+	emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN);
+
+	/* enable rx queue */
+	emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN);
+
+	/* enable mac control */
+	mac = readl(adpt->base + EMAC_MAC_CTRL);
+	csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
+
+	mac |= TXEN | RXEN;     /* enable RX/TX */
+
+	/* We don't have ethtool support yet, so force flow-control mode
+	 * to 'full' always.
+	 */
+	mac |= TXFC | RXFC;
+
+	/* setup link speed */
+	mac &= ~SPEED_MASK;
+	if (phydev->speed == SPEED_1000) {
+		mac |= SPEED(2);
+		csr1 |= FREQ_MODE;
+	} else {
+		mac |= SPEED(1);
+		csr1 &= ~FREQ_MODE;
+	}
+
+	if (phydev->duplex == DUPLEX_FULL)
+		mac |= FULLD;
+	else
+		mac &= ~FULLD;
+
+	/* other parameters */
+	mac |= (CRCE | PCRCE);
+	mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK);
+	mac |= BROAD_EN;
+	mac |= FLCHK;
+	mac &= ~RX_CHKSUM_EN;
+	mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL |
+		 DEBUG_MODE | SINGLE_PAUSE_MODE);
+
+	writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
+
+	writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL);
+
+	/* enable interrupt read clear, low power sleep mode and
+	 * the irq moderators
+	 */
+
+	writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT);
+	writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN |
+			IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL);
+
+	emac_mac_mode_config(adpt);
+
+	emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL,
+			  (HEADER_ENABLE | HEADER_CNT_EN), 0);
+
+	emac_reg_update32(adpt->csr + EMAC_EMAC_WRAPPER_CSR2, 0, WOL_EN);
+}
+
+void emac_mac_stop(struct emac_adapter *adpt)
+{
+	emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0);
+	emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0);
+	emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0);
+	usleep_range(1000, 1050); /* stopping mac may take upto 1msec */
+}
+
+/* Free all descriptors of given transmit queue */
+static void emac_tx_q_descs_free(struct emac_adapter *adpt)
+{
+	struct emac_tx_queue *tx_q = &adpt->tx_q;
+	unsigned int i;
+	size_t size;
+
+	/* ring already cleared, nothing to do */
+	if (!tx_q->tpd.tpbuff)
+		return;
+
+	for (i = 0; i < tx_q->tpd.count; i++) {
+		struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i);
+
+		if (tpbuf->dma_addr) {
+			dma_unmap_single(adpt->netdev->dev.parent,
+					 tpbuf->dma_addr, tpbuf->length,
+					 DMA_TO_DEVICE);
+			tpbuf->dma_addr = 0;
+		}
+		if (tpbuf->skb) {
+			dev_kfree_skb_any(tpbuf->skb);
+			tpbuf->skb = NULL;
+		}
+	}
+
+	size = sizeof(struct emac_buffer) * tx_q->tpd.count;
+	memset(tx_q->tpd.tpbuff, 0, size);
+
+	/* clear the descriptor ring */
+	memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size);
+
+	tx_q->tpd.consume_idx = 0;
+	tx_q->tpd.produce_idx = 0;
+}
+
+/* Free all descriptors of given receive queue */
+static void emac_rx_q_free_descs(struct emac_adapter *adpt)
+{
+	struct device *dev = adpt->netdev->dev.parent;
+	struct emac_rx_queue *rx_q = &adpt->rx_q;
+	unsigned int i;
+	size_t size;
+
+	/* ring already cleared, nothing to do */
+	if (!rx_q->rfd.rfbuff)
+		return;
+
+	for (i = 0; i < rx_q->rfd.count; i++) {
+		struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i);
+
+		if (rfbuf->dma_addr) {
+			dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length,
+					 DMA_FROM_DEVICE);
+			rfbuf->dma_addr = 0;
+		}
+		if (rfbuf->skb) {
+			dev_kfree_skb(rfbuf->skb);
+			rfbuf->skb = NULL;
+		}
+	}
+
+	size =  sizeof(struct emac_buffer) * rx_q->rfd.count;
+	memset(rx_q->rfd.rfbuff, 0, size);
+
+	/* clear the descriptor rings */
+	memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size);
+	rx_q->rrd.produce_idx = 0;
+	rx_q->rrd.consume_idx = 0;
+
+	memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size);
+	rx_q->rfd.produce_idx = 0;
+	rx_q->rfd.consume_idx = 0;
+}
+
+/* Free all buffers associated with given transmit queue */
+static void emac_tx_q_bufs_free(struct emac_adapter *adpt)
+{
+	struct emac_tx_queue *tx_q = &adpt->tx_q;
+
+	emac_tx_q_descs_free(adpt);
+
+	kfree(tx_q->tpd.tpbuff);
+	tx_q->tpd.tpbuff = NULL;
+	tx_q->tpd.v_addr = NULL;
+	tx_q->tpd.dma_addr = 0;
+	tx_q->tpd.size = 0;
+}
+
+/* Allocate TX descriptor ring for the given transmit queue */
+static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,
+				struct emac_tx_queue *tx_q)
+{
+	struct emac_ring_header *ring_header = &adpt->ring_header;
+	size_t size;
+
+	size = sizeof(struct emac_buffer) * tx_q->tpd.count;
+	tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL);
+	if (!tx_q->tpd.tpbuff)
+		return -ENOMEM;
+
+	tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4);
+	tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used;
+	tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used;
+	ring_header->used += ALIGN(tx_q->tpd.size, 8);
+	tx_q->tpd.produce_idx = 0;
+	tx_q->tpd.consume_idx = 0;
+
+	return 0;
+}
+
+/* Free all buffers associated with given transmit queue */
+static void emac_rx_q_bufs_free(struct emac_adapter *adpt)
+{
+	struct emac_rx_queue *rx_q = &adpt->rx_q;
+
+	emac_rx_q_free_descs(adpt);
+
+	kfree(rx_q->rfd.rfbuff);
+	rx_q->rfd.rfbuff   = NULL;
+
+	rx_q->rfd.v_addr   = NULL;
+	rx_q->rfd.dma_addr = 0;
+	rx_q->rfd.size     = 0;
+
+	rx_q->rrd.v_addr   = NULL;
+	rx_q->rrd.dma_addr = 0;
+	rx_q->rrd.size     = 0;
+}
+
+/* Allocate RX descriptor rings for the given receive queue */
+static int emac_rx_descs_alloc(struct emac_adapter *adpt)
+{
+	struct emac_ring_header *ring_header = &adpt->ring_header;
+	struct emac_rx_queue *rx_q = &adpt->rx_q;
+	size_t size;
+
+	size = sizeof(struct emac_buffer) * rx_q->rfd.count;
+	rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL);
+	if (!rx_q->rfd.rfbuff)
+		return -ENOMEM;
+
+	rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4);
+	rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4);
+
+	rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used;
+	rx_q->rrd.v_addr   = ring_header->v_addr + ring_header->used;
+	ring_header->used += ALIGN(rx_q->rrd.size, 8);
+
+	rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used;
+	rx_q->rfd.v_addr   = ring_header->v_addr + ring_header->used;
+	ring_header->used += ALIGN(rx_q->rfd.size, 8);
+
+	rx_q->rrd.produce_idx = 0;
+	rx_q->rrd.consume_idx = 0;
+
+	rx_q->rfd.produce_idx = 0;
+	rx_q->rfd.consume_idx = 0;
+
+	return 0;
+}
+
+/* Allocate all TX and RX descriptor rings */
+int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt)
+{
+	struct emac_ring_header *ring_header = &adpt->ring_header;
+	struct device *dev = adpt->netdev->dev.parent;
+	unsigned int num_tx_descs = adpt->tx_desc_cnt;
+	unsigned int num_rx_descs = adpt->rx_desc_cnt;
+	int ret;
+
+	adpt->tx_q.tpd.count = adpt->tx_desc_cnt;
+
+	adpt->rx_q.rrd.count = adpt->rx_desc_cnt;
+	adpt->rx_q.rfd.count = adpt->rx_desc_cnt;
+
+	/* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,
+	 * hence the additional padding bytes are allocated.
+	 */
+	ring_header->size = num_tx_descs * (adpt->tpd_size * 4) +
+			    num_rx_descs * (adpt->rfd_size * 4) +
+			    num_rx_descs * (adpt->rrd_size * 4) +
+			    8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */
+
+	ring_header->used = 0;
+	ring_header->v_addr = dma_zalloc_coherent(dev, ring_header->size,
+						 &ring_header->dma_addr,
+						 GFP_KERNEL);
+	if (!ring_header->v_addr)
+		return -ENOMEM;
+
+	ring_header->used = ALIGN(ring_header->dma_addr, 8) -
+							ring_header->dma_addr;
+
+	ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q);
+	if (ret) {
+		netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n");
+		goto err_alloc_tx;
+	}
+
+	ret = emac_rx_descs_alloc(adpt);
+	if (ret) {
+		netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n");
+		goto err_alloc_rx;
+	}
+
+	return 0;
+
+err_alloc_rx:
+	emac_tx_q_bufs_free(adpt);
+err_alloc_tx:
+	dma_free_coherent(dev, ring_header->size,
+			  ring_header->v_addr, ring_header->dma_addr);
+
+	ring_header->v_addr   = NULL;
+	ring_header->dma_addr = 0;
+	ring_header->size     = 0;
+	ring_header->used     = 0;
+
+	return ret;
+}
+
+/* Free all TX and RX descriptor rings */
+void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt)
+{
+	struct emac_ring_header *ring_header = &adpt->ring_header;
+	struct device *dev = adpt->netdev->dev.parent;
+
+	emac_tx_q_bufs_free(adpt);
+	emac_rx_q_bufs_free(adpt);
+
+	dma_free_coherent(dev, ring_header->size,
+			  ring_header->v_addr, ring_header->dma_addr);
+
+	ring_header->v_addr   = NULL;
+	ring_header->dma_addr = 0;
+	ring_header->size     = 0;
+	ring_header->used     = 0;
+}
+
+/* Initialize descriptor rings */
+static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt)
+{
+	unsigned int i;
+
+	adpt->tx_q.tpd.produce_idx = 0;
+	adpt->tx_q.tpd.consume_idx = 0;
+	for (i = 0; i < adpt->tx_q.tpd.count; i++)
+		adpt->tx_q.tpd.tpbuff[i].dma_addr = 0;
+
+	adpt->rx_q.rrd.produce_idx = 0;
+	adpt->rx_q.rrd.consume_idx = 0;
+	adpt->rx_q.rfd.produce_idx = 0;
+	adpt->rx_q.rfd.consume_idx = 0;
+	for (i = 0; i < adpt->rx_q.rfd.count; i++)
+		adpt->rx_q.rfd.rfbuff[i].dma_addr = 0;
+}
+
+/* Produce new receive free descriptor */
+static void emac_mac_rx_rfd_create(struct emac_adapter *adpt,
+				   struct emac_rx_queue *rx_q,
+				   dma_addr_t addr)
+{
+	u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx);
+
+	*(hw_rfd++) = lower_32_bits(addr);
+	*hw_rfd = upper_32_bits(addr);
+
+	if (++rx_q->rfd.produce_idx == rx_q->rfd.count)
+		rx_q->rfd.produce_idx = 0;
+}
+
+/* Fill up receive queue's RFD with preallocated receive buffers */
+static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,
+				    struct emac_rx_queue *rx_q)
+{
+	struct emac_buffer *curr_rxbuf;
+	struct emac_buffer *next_rxbuf;
+	unsigned int count = 0;
+	u32 next_produce_idx;
+
+	next_produce_idx = rx_q->rfd.produce_idx + 1;
+	if (next_produce_idx == rx_q->rfd.count)
+		next_produce_idx = 0;
+
+	curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
+	next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
+
+	/* this always has a blank rx_buffer*/
+	while (!next_rxbuf->dma_addr) {
+		struct sk_buff *skb;
+		int ret;
+
+		skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size);
+		if (!skb)
+			break;
+
+		curr_rxbuf->dma_addr =
+			dma_map_single(adpt->netdev->dev.parent, skb->data,
+				       curr_rxbuf->length, DMA_FROM_DEVICE);
+		ret = dma_mapping_error(adpt->netdev->dev.parent,
+					curr_rxbuf->dma_addr);
+		if (ret) {
+			dev_kfree_skb(skb);
+			break;
+		}
+		curr_rxbuf->skb = skb;
+		curr_rxbuf->length = adpt->rxbuf_size;
+
+		emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr);
+		next_produce_idx = rx_q->rfd.produce_idx + 1;
+		if (next_produce_idx == rx_q->rfd.count)
+			next_produce_idx = 0;
+
+		curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
+		next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
+		count++;
+	}
+
+	if (count) {
+		u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) &
+				rx_q->produce_mask;
+		emac_reg_update32(adpt->base + rx_q->produce_reg,
+				  rx_q->produce_mask, prod_idx);
+	}
+}
+
+static void emac_adjust_link(struct net_device *netdev)
+{
+	struct emac_adapter *adpt = netdev_priv(netdev);
+	struct phy_device *phydev = netdev->phydev;
+
+	if (phydev->link)
+		emac_mac_start(adpt);
+	else
+		emac_mac_stop(adpt);
+
+	phy_print_status(phydev);
+}
+
+/* Bringup the interface/HW */
+int emac_mac_up(struct emac_adapter *adpt)
+{
+	struct net_device *netdev = adpt->netdev;
+	struct emac_irq	*irq = &adpt->irq;
+	int ret;
+
+	emac_mac_rx_tx_ring_reset_all(adpt);
+	emac_mac_config(adpt);
+
+	ret = request_irq(irq->irq, emac_isr, 0, EMAC_MAC_IRQ_RES, irq);
+	if (ret) {
+		netdev_err(adpt->netdev, "could not request %s irq\n",
+			   EMAC_MAC_IRQ_RES);
+		return ret;
+	}
+
+	emac_mac_rx_descs_refill(adpt, &adpt->rx_q);
+
+	ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link,
+				 PHY_INTERFACE_MODE_SGMII);
+	if (ret) {
+		netdev_err(adpt->netdev, "could not connect phy\n");
+		free_irq(irq->irq, irq);
+		return ret;
+	}
+
+	/* enable mac irq */
+	writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS);
+	writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK);
+
+	adpt->phydev->irq = PHY_IGNORE_INTERRUPT;
+	phy_start(adpt->phydev);
+
+	napi_enable(&adpt->rx_q.napi);
+	netif_start_queue(netdev);
+
+	return 0;
+}
+
+/* Bring down the interface/HW */
+void emac_mac_down(struct emac_adapter *adpt)
+{
+	struct net_device *netdev = adpt->netdev;
+
+	netif_stop_queue(netdev);
+	napi_disable(&adpt->rx_q.napi);
+
+	phy_stop(adpt->phydev);
+	phy_disconnect(adpt->phydev);
+
+	/* disable mac irq */
+	writel(DIS_INT, adpt->base + EMAC_INT_STATUS);
+	writel(0, adpt->base + EMAC_INT_MASK);
+	synchronize_irq(adpt->irq.irq);
+	free_irq(adpt->irq.irq, &adpt->irq);
+
+	emac_mac_reset(adpt);
+
+	emac_tx_q_descs_free(adpt);
+	netdev_reset_queue(adpt->netdev);
+	emac_rx_q_free_descs(adpt);
+}
+
+/* Consume next received packet descriptor */
+static bool emac_rx_process_rrd(struct emac_adapter *adpt,
+				struct emac_rx_queue *rx_q,
+				struct emac_rrd *rrd)
+{
+	u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx);
+
+	rrd->word[3] = *(hw_rrd + 3);
+
+	if (!RRD_UPDT(rrd))
+		return false;
+
+	rrd->word[4] = 0;
+	rrd->word[5] = 0;
+
+	rrd->word[0] = *(hw_rrd++);
+	rrd->word[1] = *(hw_rrd++);
+	rrd->word[2] = *(hw_rrd++);
+
+	if (unlikely(RRD_NOR(rrd) != 1)) {
+		netdev_err(adpt->netdev,
+			   "error: multi-RFD not support yet! nor:%lu\n",
+			   RRD_NOR(rrd));
+	}
+
+	/* mark rrd as processed */
+	RRD_UPDT_SET(rrd, 0);
+	*hw_rrd = rrd->word[3];
+
+	if (++rx_q->rrd.consume_idx == rx_q->rrd.count)
+		rx_q->rrd.consume_idx = 0;
+
+	return true;
+}
+
+/* Produce new transmit descriptor */
+static void emac_tx_tpd_create(struct emac_adapter *adpt,
+			       struct emac_tx_queue *tx_q, struct emac_tpd *tpd)
+{
+	u32 *hw_tpd;
+
+	tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx;
+	hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx);
+
+	if (++tx_q->tpd.produce_idx == tx_q->tpd.count)
+		tx_q->tpd.produce_idx = 0;
+
+	*(hw_tpd++) = tpd->word[0];
+	*(hw_tpd++) = tpd->word[1];
+	*(hw_tpd++) = tpd->word[2];
+	*hw_tpd = tpd->word[3];
+}
+
+/* Mark the last transmit descriptor as such (for the transmit packet) */
+static void emac_tx_tpd_mark_last(struct emac_adapter *adpt,
+				  struct emac_tx_queue *tx_q)
+{
+	u32 *hw_tpd =
+		EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx);
+	u32 tmp_tpd;
+
+	tmp_tpd = *(hw_tpd + 1);
+	tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;
+	*(hw_tpd + 1) = tmp_tpd;
+}
+
+static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd)
+{
+	struct emac_buffer *rfbuf = rx_q->rfd.rfbuff;
+	u32 consume_idx = RRD_SI(rrd);
+	unsigned int i;
+
+	for (i = 0; i < RRD_NOR(rrd); i++) {
+		rfbuf[consume_idx].skb = NULL;
+		if (++consume_idx == rx_q->rfd.count)
+			consume_idx = 0;
+	}
+
+	rx_q->rfd.consume_idx = consume_idx;
+	rx_q->rfd.process_idx = consume_idx;
+}
+
+/* Push the received skb to upper layers */
+static void emac_receive_skb(struct emac_rx_queue *rx_q,
+			     struct sk_buff *skb,
+			     u16 vlan_tag, bool vlan_flag)
+{
+	if (vlan_flag) {
+		u16 vlan;
+
+		EMAC_TAG_TO_VLAN(vlan_tag, vlan);
+		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
+	}
+
+	napi_gro_receive(&rx_q->napi, skb);
+}
+
+/* Process receive event */
+void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,
+			 int *num_pkts, int max_pkts)
+{
+	u32 proc_idx, hw_consume_idx, num_consume_pkts;
+	struct net_device *netdev  = adpt->netdev;
+	struct emac_buffer *rfbuf;
+	unsigned int count = 0;
+	struct emac_rrd rrd;
+	struct sk_buff *skb;
+	u32 reg;
+
+	reg = readl_relaxed(adpt->base + rx_q->consume_reg);
+
+	hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift;
+	num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ?
+		(hw_consume_idx -  rx_q->rrd.consume_idx) :
+		(hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx);
+
+	do {
+		if (!num_consume_pkts)
+			break;
+
+		if (!emac_rx_process_rrd(adpt, rx_q, &rrd))
+			break;
+
+		if (likely(RRD_NOR(&rrd) == 1)) {
+			/* good receive */
+			rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd));
+			dma_unmap_single(adpt->netdev->dev.parent,
+					 rfbuf->dma_addr, rfbuf->length,
+					 DMA_FROM_DEVICE);
+			rfbuf->dma_addr = 0;
+			skb = rfbuf->skb;
+		} else {
+			netdev_err(adpt->netdev,
+				   "error: multi-RFD not support yet!\n");
+			break;
+		}
+		emac_rx_rfd_clean(rx_q, &rrd);
+		num_consume_pkts--;
+		count++;
+
+		/* Due to a HW issue in L4 check sum detection (UDP/TCP frags
+		 * with DF set are marked as error), drop packets based on the
+		 * error mask rather than the summary bit (ignoring L4F errors)
+		 */
+		if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) {
+			netif_dbg(adpt, rx_status, adpt->netdev,
+				  "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
+				  rrd.word[0], rrd.word[1],
+				  rrd.word[2], rrd.word[3]);
+
+			dev_kfree_skb(skb);
+			continue;
+		}
+
+		skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN);
+		skb->dev = netdev;
+		skb->protocol = eth_type_trans(skb, skb->dev);
+		if (netdev->features & NETIF_F_RXCSUM)
+			skb->ip_summed = RRD_L4F(&rrd) ?
+					  CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
+		else
+			skb_checksum_none_assert(skb);
+
+		emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd),
+				 (bool)RRD_CVTAG(&rrd));
+
+		netdev->last_rx = jiffies;
+		(*num_pkts)++;
+	} while (*num_pkts < max_pkts);
+
+	if (count) {
+		proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) &
+				rx_q->process_mask;
+		emac_reg_update32(adpt->base + rx_q->process_reg,
+				  rx_q->process_mask, proc_idx);
+		emac_mac_rx_descs_refill(adpt, rx_q);
+	}
+}
+
+/* get the number of free transmit descriptors */
+static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q)
+{
+	u32 produce_idx = tx_q->tpd.produce_idx;
+	u32 consume_idx = tx_q->tpd.consume_idx;
+
+	return (consume_idx > produce_idx) ?
+		(consume_idx - produce_idx - 1) :
+		(tx_q->tpd.count + consume_idx - produce_idx - 1);
+}
+
+/* Process transmit event */
+void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)
+{
+	u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg);
+	u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0;
+	struct emac_buffer *tpbuf;
+
+	hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift;
+
+	while (tx_q->tpd.consume_idx != hw_consume_idx) {
+		tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);
+		if (tpbuf->dma_addr) {
+			dma_unmap_single(adpt->netdev->dev.parent,
+					 tpbuf->dma_addr, tpbuf->length,
+					 DMA_TO_DEVICE);
+			tpbuf->dma_addr = 0;
+		}
+
+		if (tpbuf->skb) {
+			pkts_compl++;
+			bytes_compl += tpbuf->skb->len;
+			dev_kfree_skb_irq(tpbuf->skb);
+			tpbuf->skb = NULL;
+		}
+
+		if (++tx_q->tpd.consume_idx == tx_q->tpd.count)
+			tx_q->tpd.consume_idx = 0;
+	}
+
+	netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);
+
+	if (netif_queue_stopped(adpt->netdev))
+		if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1))
+			netif_wake_queue(adpt->netdev);
+}
+
+/* Initialize all queue data structures */
+void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,
+				  struct emac_adapter *adpt)
+{
+	adpt->rx_q.netdev = adpt->netdev;
+
+	adpt->rx_q.produce_reg  = EMAC_MAILBOX_0;
+	adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK;
+	adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT;
+
+	adpt->rx_q.process_reg  = EMAC_MAILBOX_0;
+	adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK;
+	adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT;
+
+	adpt->rx_q.consume_reg  = EMAC_MAILBOX_3;
+	adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK;
+	adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT;
+
+	adpt->rx_q.irq          = &adpt->irq;
+	adpt->rx_q.intr         = adpt->irq.mask & ISR_RX_PKT;
+
+	adpt->tx_q.produce_reg  = EMAC_MAILBOX_15;
+	adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK;
+	adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT;
+
+	adpt->tx_q.consume_reg  = EMAC_MAILBOX_2;
+	adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK;
+	adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT;
+}
+
+/* Fill up transmit descriptors with TSO and Checksum offload information */
+static int emac_tso_csum(struct emac_adapter *adpt,
+			 struct emac_tx_queue *tx_q,
+			 struct sk_buff *skb,
+			 struct emac_tpd *tpd)
+{
+	unsigned int hdr_len;
+	int ret;
+
+	if (skb_is_gso(skb)) {
+		if (skb_header_cloned(skb)) {
+			ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
+			if (unlikely(ret))
+				return ret;
+		}
+
+		if (skb->protocol == htons(ETH_P_IP)) {
+			u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data)
+				       + ntohs(ip_hdr(skb)->tot_len);
+			if (skb->len > pkt_len)
+				pskb_trim(skb, pkt_len);
+		}
+
+		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
+		if (unlikely(skb->len == hdr_len)) {
+			/* we only need to do csum */
+			netif_warn(adpt, tx_err, adpt->netdev,
+				   "tso not needed for packet with 0 data\n");
+			goto do_csum;
+		}
+
+		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
+			ip_hdr(skb)->check = 0;
+			tcp_hdr(skb)->check =
+				~csum_tcpudp_magic(ip_hdr(skb)->saddr,
+						   ip_hdr(skb)->daddr,
+						   0, IPPROTO_TCP, 0);
+			TPD_IPV4_SET(tpd, 1);
+		}
+
+		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
+			/* ipv6 tso need an extra tpd */
+			struct emac_tpd extra_tpd;
+
+			memset(tpd, 0, sizeof(*tpd));
+			memset(&extra_tpd, 0, sizeof(extra_tpd));
+
+			ipv6_hdr(skb)->payload_len = 0;
+			tcp_hdr(skb)->check =
+				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
+						 &ipv6_hdr(skb)->daddr,
+						 0, IPPROTO_TCP, 0);
+			TPD_PKT_LEN_SET(&extra_tpd, skb->len);
+			TPD_LSO_SET(&extra_tpd, 1);
+			TPD_LSOV_SET(&extra_tpd, 1);
+			emac_tx_tpd_create(adpt, tx_q, &extra_tpd);
+			TPD_LSOV_SET(tpd, 1);
+		}
+
+		TPD_LSO_SET(tpd, 1);
+		TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb));
+		TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size);
+		return 0;
+	}
+
+do_csum:
+	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
+		unsigned int css, cso;
+
+		cso = skb_transport_offset(skb);
+		if (unlikely(cso & 0x1)) {
+			netdev_err(adpt->netdev,
+				   "error: payload offset should be even\n");
+			return -EINVAL;
+		}
+		css = cso + skb->csum_offset;
+
+		TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1);
+		TPD_CXSUM_OFFSET_SET(tpd, css >> 1);
+		TPD_CSX_SET(tpd, 1);
+	}
+
+	return 0;
+}
+
+/* Fill up transmit descriptors */
+static void emac_tx_fill_tpd(struct emac_adapter *adpt,
+			     struct emac_tx_queue *tx_q, struct sk_buff *skb,
+			     struct emac_tpd *tpd)
+{
+	unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
+	unsigned int first = tx_q->tpd.produce_idx;
+	unsigned int len = skb_headlen(skb);
+	struct emac_buffer *tpbuf = NULL;
+	unsigned int mapped_len = 0;
+	unsigned int i;
+	int count = 0;
+	int ret;
+
+	/* if Large Segment Offload is (in TCP Segmentation Offload struct) */
+	if (TPD_LSO(tpd)) {
+		mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
+
+		tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
+		tpbuf->length = mapped_len;
+		tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
+						 skb->data, tpbuf->length,
+						 DMA_TO_DEVICE);
+		ret = dma_mapping_error(adpt->netdev->dev.parent,
+					tpbuf->dma_addr);
+		if (ret)
+			goto error;
+
+		TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
+		TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
+		TPD_BUF_LEN_SET(tpd, tpbuf->length);
+		emac_tx_tpd_create(adpt, tx_q, tpd);
+		count++;
+	}
+
+	if (mapped_len < len) {
+		tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
+		tpbuf->length = len - mapped_len;
+		tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
+						 skb->data + mapped_len,
+						 tpbuf->length, DMA_TO_DEVICE);
+		ret = dma_mapping_error(adpt->netdev->dev.parent,
+					tpbuf->dma_addr);
+		if (ret)
+			goto error;
+
+		TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
+		TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
+		TPD_BUF_LEN_SET(tpd, tpbuf->length);
+		emac_tx_tpd_create(adpt, tx_q, tpd);
+		count++;
+	}
+
+	for (i = 0; i < nr_frags; i++) {
+		struct skb_frag_struct *frag;
+
+		frag = &skb_shinfo(skb)->frags[i];
+
+		tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
+		tpbuf->length = frag->size;
+		tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
+					       frag->page.p, frag->page_offset,
+					       tpbuf->length, DMA_TO_DEVICE);
+		ret = dma_mapping_error(adpt->netdev->dev.parent,
+					tpbuf->dma_addr);
+		if (ret)
+			goto error;
+
+		TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
+		TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
+		TPD_BUF_LEN_SET(tpd, tpbuf->length);
+		emac_tx_tpd_create(adpt, tx_q, tpd);
+		count++;
+	}
+
+	/* The last tpd */
+	wmb();
+	emac_tx_tpd_mark_last(adpt, tx_q);
+
+	/* The last buffer info contain the skb address,
+	 * so it will be freed after unmap
+	 */
+	tpbuf->skb = skb;
+
+	return;
+
+error:
+	/* One of the memory mappings failed, so undo everything */
+	tx_q->tpd.produce_idx = first;
+
+	while (count--) {
+		tpbuf = GET_TPD_BUFFER(tx_q, first);
+		dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr,
+			       tpbuf->length, DMA_TO_DEVICE);
+		tpbuf->dma_addr = 0;
+		tpbuf->length = 0;
+
+		if (++first == tx_q->tpd.count)
+			first = 0;
+	}
+
+	dev_kfree_skb(skb);
+}
+
+/* Transmit the packet using specified transmit queue */
+int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q,
+			 struct sk_buff *skb)
+{
+	struct emac_tpd tpd;
+	u32 prod_idx;
+
+	memset(&tpd, 0, sizeof(tpd));
+
+	if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) {
+		dev_kfree_skb_any(skb);
+		return NETDEV_TX_OK;
+	}
+
+	if (skb_vlan_tag_present(skb)) {
+		u16 tag;
+
+		EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag);
+		TPD_CVLAN_TAG_SET(&tpd, tag);
+		TPD_INSTC_SET(&tpd, 1);
+	}
+
+	if (skb_network_offset(skb) != ETH_HLEN)
+		TPD_TYP_SET(&tpd, 1);
+
+	emac_tx_fill_tpd(adpt, tx_q, skb, &tpd);
+
+	netdev_sent_queue(adpt->netdev, skb->len);
+
+	/* Make sure the are enough free descriptors to hold one
+	 * maximum-sized SKB.  We need one desc for each fragment,
+	 * one for the checksum (emac_tso_csum), one for TSO, and
+	 * and one for the SKB header.
+	 */
+	if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3))
+		netif_stop_queue(adpt->netdev);
+
+	/* update produce idx */
+	prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) &
+		    tx_q->produce_mask;
+	emac_reg_update32(adpt->base + tx_q->produce_reg,
+			  tx_q->produce_mask, prod_idx);
+
+	return NETDEV_TX_OK;
+}
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-mac.h b/drivers/net/ethernet/qualcomm/emac/emac-mac.h
new file mode 100644
index 000000000000..f3aa24dc4a29
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/emac/emac-mac.h
@@ -0,0 +1,248 @@
+/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/* EMAC DMA HW engine uses three rings:
+ * Tx:
+ *   TPD: Transmit Packet Descriptor ring.
+ * Rx:
+ *   RFD: Receive Free Descriptor ring.
+ *     Ring of descriptors with empty buffers to be filled by Rx HW.
+ *   RRD: Receive Return Descriptor ring.
+ *     Ring of descriptors with buffers filled with received data.
+ */
+
+#ifndef _EMAC_HW_H_
+#define _EMAC_HW_H_
+
+/* EMAC_CSR register offsets */
+#define EMAC_EMAC_WRAPPER_CSR1                                0x000000
+#define EMAC_EMAC_WRAPPER_CSR2                                0x000004
+#define EMAC_EMAC_WRAPPER_TX_TS_LO                            0x000104
+#define EMAC_EMAC_WRAPPER_TX_TS_HI                            0x000108
+#define EMAC_EMAC_WRAPPER_TX_TS_INX                           0x00010c
+
+/* DMA Order Settings */
+enum emac_dma_order {
+	emac_dma_ord_in = 1,
+	emac_dma_ord_enh = 2,
+	emac_dma_ord_out = 4
+};
+
+enum emac_dma_req_block {
+	emac_dma_req_128 = 0,
+	emac_dma_req_256 = 1,
+	emac_dma_req_512 = 2,
+	emac_dma_req_1024 = 3,
+	emac_dma_req_2048 = 4,
+	emac_dma_req_4096 = 5
+};
+
+/* Returns the value of bits idx...idx+n_bits */
+#define BITS_GET(val, lo, hi) ((le32_to_cpu(val) & GENMASK((hi), (lo))) >> lo)
+#define BITS_SET(val, lo, hi, new_val) \
+	val = cpu_to_le32((le32_to_cpu(val) & (~GENMASK((hi), (lo)))) |	\
+		(((new_val) << (lo)) & GENMASK((hi), (lo))))
+
+/* RRD (Receive Return Descriptor) */
+struct emac_rrd {
+	u32	word[6];
+
+/* number of RFD */
+#define RRD_NOR(rrd)			BITS_GET((rrd)->word[0], 16, 19)
+/* start consumer index of rfd-ring */
+#define RRD_SI(rrd)			BITS_GET((rrd)->word[0], 20, 31)
+/* vlan-tag (CVID, CFI and PRI) */
+#define RRD_CVALN_TAG(rrd)		BITS_GET((rrd)->word[2], 0, 15)
+/* length of the packet */
+#define RRD_PKT_SIZE(rrd)		BITS_GET((rrd)->word[3], 0, 13)
+/* L4(TCP/UDP) checksum failed */
+#define RRD_L4F(rrd)			BITS_GET((rrd)->word[3], 14, 14)
+/* vlan tagged */
+#define RRD_CVTAG(rrd)			BITS_GET((rrd)->word[3], 16, 16)
+/* When set, indicates that the descriptor is updated by the IP core.
+ * When cleared, indicates that the descriptor is invalid.
+ */
+#define RRD_UPDT(rrd)			BITS_GET((rrd)->word[3], 31, 31)
+#define RRD_UPDT_SET(rrd, val)		BITS_SET((rrd)->word[3], 31, 31, val)
+/* timestamp low */
+#define RRD_TS_LOW(rrd)			BITS_GET((rrd)->word[4], 0, 29)
+/* timestamp high */
+#define RRD_TS_HI(rrd)			le32_to_cpu((rrd)->word[5])
+};
+
+/* TPD (Transmit Packet Descriptor) */
+struct emac_tpd {
+	u32				word[4];
+
+/* Number of bytes of the transmit packet. (include 4-byte CRC) */
+#define TPD_BUF_LEN_SET(tpd, val)	BITS_SET((tpd)->word[0], 0, 15, val)
+/* Custom Checksum Offload: When set, ask IP core to offload custom checksum */
+#define TPD_CSX_SET(tpd, val)		BITS_SET((tpd)->word[1], 8, 8, val)
+/* TCP Large Send Offload: When set, ask IP core to do offload TCP Large Send */
+#define TPD_LSO(tpd)			BITS_GET((tpd)->word[1], 12, 12)
+#define TPD_LSO_SET(tpd, val)		BITS_SET((tpd)->word[1], 12, 12, val)
+/*  Large Send Offload Version: When set, indicates this is an LSOv2
+ * (for both IPv4 and IPv6). When cleared, indicates this is an LSOv1
+ * (only for IPv4).
+ */
+#define TPD_LSOV_SET(tpd, val)		BITS_SET((tpd)->word[1], 13, 13, val)
+/* IPv4 packet: When set, indicates this is an  IPv4 packet, this bit is only
+ * for LSOV2 format.
+ */
+#define TPD_IPV4_SET(tpd, val)		BITS_SET((tpd)->word[1], 16, 16, val)
+/* 0: Ethernet   frame (DA+SA+TYPE+DATA+CRC)
+ * 1: IEEE 802.3 frame (DA+SA+LEN+DSAP+SSAP+CTL+ORG+TYPE+DATA+CRC)
+ */
+#define TPD_TYP_SET(tpd, val)		BITS_SET((tpd)->word[1], 17, 17, val)
+/* Low-32bit Buffer Address */
+#define TPD_BUFFER_ADDR_L_SET(tpd, val)	((tpd)->word[2] = cpu_to_le32(val))
+/* CVLAN Tag to be inserted if INS_VLAN_TAG is set, CVLAN TPID based on global
+ * register configuration.
+ */
+#define TPD_CVLAN_TAG_SET(tpd, val)	BITS_SET((tpd)->word[3], 0, 15, val)
+/*  Insert CVlan Tag: When set, ask MAC to insert CVLAN TAG to outgoing packet
+ */
+#define TPD_INSTC_SET(tpd, val)		BITS_SET((tpd)->word[3], 17, 17, val)
+/* High-14bit Buffer Address, So, the 64b-bit address is
+ * {DESC_CTRL_11_TX_DATA_HIADDR[17:0],(register) BUFFER_ADDR_H, BUFFER_ADDR_L}
+ */
+#define TPD_BUFFER_ADDR_H_SET(tpd, val)	BITS_SET((tpd)->word[3], 18, 30, val)
+/* Format D. Word offset from the 1st byte of this packet to start to calculate
+ * the custom checksum.
+ */
+#define TPD_PAYLOAD_OFFSET_SET(tpd, val) BITS_SET((tpd)->word[1], 0, 7, val)
+/*  Format D. Word offset from the 1st byte of this packet to fill the custom
+ * checksum to
+ */
+#define TPD_CXSUM_OFFSET_SET(tpd, val)	BITS_SET((tpd)->word[1], 18, 25, val)
+
+/* Format C. TCP Header offset from the 1st byte of this packet. (byte unit) */
+#define TPD_TCPHDR_OFFSET_SET(tpd, val)	BITS_SET((tpd)->word[1], 0, 7, val)
+/* Format C. MSS (Maximum Segment Size) got from the protocol layer. (byte unit)
+ */
+#define TPD_MSS_SET(tpd, val)		BITS_SET((tpd)->word[1], 18, 30, val)
+/* packet length in ext tpd */
+#define TPD_PKT_LEN_SET(tpd, val)	((tpd)->word[2] = cpu_to_le32(val))
+};
+
+/* emac_ring_header represents a single, contiguous block of DMA space
+ * mapped for the three descriptor rings (tpd, rfd, rrd)
+ */
+struct emac_ring_header {
+	void			*v_addr;	/* virtual address */
+	dma_addr_t		dma_addr;	/* dma address */
+	size_t			size;		/* length in bytes */
+	size_t			used;
+};
+
+/* emac_buffer is wrapper around a pointer to a socket buffer
+ * so a DMA handle can be stored along with the skb
+ */
+struct emac_buffer {
+	struct sk_buff		*skb;		/* socket buffer */
+	u16			length;		/* rx buffer length */
+	dma_addr_t		dma_addr;	/* dma address */
+};
+
+/* receive free descriptor (rfd) ring */
+struct emac_rfd_ring {
+	struct emac_buffer	*rfbuff;
+	u32			*v_addr;	/* virtual address */
+	dma_addr_t		dma_addr;	/* dma address */
+	size_t			size;		/* length in bytes */
+	unsigned int		count;		/* number of desc in the ring */
+	unsigned int		produce_idx;
+	unsigned int		process_idx;
+	unsigned int		consume_idx;	/* unused */
+};
+
+/* Receive Return Desciptor (RRD) ring */
+struct emac_rrd_ring {
+	u32			*v_addr;	/* virtual address */
+	dma_addr_t		dma_addr;	/* physical address */
+	size_t			size;		/* length in bytes */
+	unsigned int		count;		/* number of desc in the ring */
+	unsigned int		produce_idx;	/* unused */
+	unsigned int		consume_idx;
+};
+
+/* Rx queue */
+struct emac_rx_queue {
+	struct net_device	*netdev;	/* netdev ring belongs to */
+	struct emac_rrd_ring	rrd;
+	struct emac_rfd_ring	rfd;
+	struct napi_struct	napi;
+	struct emac_irq		*irq;
+
+	u32			intr;
+	u32			produce_mask;
+	u32			process_mask;
+	u32			consume_mask;
+
+	u16			produce_reg;
+	u16			process_reg;
+	u16			consume_reg;
+
+	u8			produce_shift;
+	u8			process_shft;
+	u8			consume_shift;
+};
+
+/* Transimit Packet Descriptor (tpd) ring */
+struct emac_tpd_ring {
+	struct emac_buffer	*tpbuff;
+	u32			*v_addr;	/* virtual address */
+	dma_addr_t		dma_addr;	/* dma address */
+
+	size_t			size;		/* length in bytes */
+	unsigned int		count;		/* number of desc in the ring */
+	unsigned int		produce_idx;
+	unsigned int		consume_idx;
+	unsigned int		last_produce_idx;
+};
+
+/* Tx queue */
+struct emac_tx_queue {
+	struct emac_tpd_ring	tpd;
+
+	u32			produce_mask;
+	u32			consume_mask;
+
+	u16			max_packets;	/* max packets per interrupt */
+	u16			produce_reg;
+	u16			consume_reg;
+
+	u8			produce_shift;
+	u8			consume_shift;
+};
+
+struct emac_adapter;
+
+int  emac_mac_up(struct emac_adapter *adpt);
+void emac_mac_down(struct emac_adapter *adpt);
+void emac_mac_reset(struct emac_adapter *adpt);
+void emac_mac_start(struct emac_adapter *adpt);
+void emac_mac_stop(struct emac_adapter *adpt);
+void emac_mac_mode_config(struct emac_adapter *adpt);
+void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,
+			 int *num_pkts, int max_pkts);
+int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q,
+			 struct sk_buff *skb);
+void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q);
+void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,
+				  struct emac_adapter *adpt);
+int  emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt);
+void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt);
+void emac_mac_multicast_addr_clear(struct emac_adapter *adpt);
+void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr);
+
+#endif /*_EMAC_HW_H_*/
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-phy.c b/drivers/net/ethernet/qualcomm/emac/emac-phy.c
new file mode 100644
index 000000000000..da4e90db4d98
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/emac/emac-phy.c
@@ -0,0 +1,227 @@
+/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/* Qualcomm Technologies, Inc. EMAC PHY Controller driver.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include <linux/of_mdio.h>
+#include <linux/phy.h>
+#include <linux/iopoll.h>
+#include <linux/acpi.h>
+#include "emac.h"
+#include "emac-mac.h"
+#include "emac-phy.h"
+#include "emac-sgmii.h"
+
+/* EMAC base register offsets */
+#define EMAC_MDIO_CTRL                                        0x001414
+#define EMAC_PHY_STS                                          0x001418
+#define EMAC_MDIO_EX_CTRL                                     0x001440
+
+/* EMAC_MDIO_CTRL */
+#define MDIO_MODE                                              BIT(30)
+#define MDIO_PR                                                BIT(29)
+#define MDIO_AP_EN                                             BIT(28)
+#define MDIO_BUSY                                              BIT(27)
+#define MDIO_CLK_SEL_BMSK                                    0x7000000
+#define MDIO_CLK_SEL_SHFT                                           24
+#define MDIO_START                                             BIT(23)
+#define SUP_PREAMBLE                                           BIT(22)
+#define MDIO_RD_NWR                                            BIT(21)
+#define MDIO_REG_ADDR_BMSK                                    0x1f0000
+#define MDIO_REG_ADDR_SHFT                                          16
+#define MDIO_DATA_BMSK                                          0xffff
+#define MDIO_DATA_SHFT                                               0
+
+/* EMAC_PHY_STS */
+#define PHY_ADDR_BMSK                                         0x1f0000
+#define PHY_ADDR_SHFT                                               16
+
+#define MDIO_CLK_25_4                                                0
+#define MDIO_CLK_25_28                                               7
+
+#define MDIO_WAIT_TIMES                                           1000
+
+#define EMAC_LINK_SPEED_DEFAULT (\
+		EMAC_LINK_SPEED_10_HALF  |\
+		EMAC_LINK_SPEED_10_FULL  |\
+		EMAC_LINK_SPEED_100_HALF |\
+		EMAC_LINK_SPEED_100_FULL |\
+		EMAC_LINK_SPEED_1GB_FULL)
+
+/**
+ * emac_phy_mdio_autopoll_disable() - disable mdio autopoll
+ * @adpt: the emac adapter
+ *
+ * The autopoll feature takes over the MDIO bus.  In order for
+ * the PHY driver to be able to talk to the PHY over the MDIO
+ * bus, we need to temporarily disable the autopoll feature.
+ */
+static int emac_phy_mdio_autopoll_disable(struct emac_adapter *adpt)
+{
+	u32 val;
+
+	/* disable autopoll */
+	emac_reg_update32(adpt->base + EMAC_MDIO_CTRL, MDIO_AP_EN, 0);
+
+	/* wait for any mdio polling to complete */
+	if (!readl_poll_timeout(adpt->base + EMAC_MDIO_CTRL, val,
+				!(val & MDIO_BUSY), 100, MDIO_WAIT_TIMES * 100))
+		return 0;
+
+	/* failed to disable; ensure it is enabled before returning */
+	emac_reg_update32(adpt->base + EMAC_MDIO_CTRL, 0, MDIO_AP_EN);
+
+	return -EBUSY;
+}
+
+/**
+ * emac_phy_mdio_autopoll_disable() - disable mdio autopoll
+ * @adpt: the emac adapter
+ *
+ * The EMAC has the ability to poll the external PHY on the MDIO
+ * bus for link state changes.  This eliminates the need for the
+ * driver to poll the phy.  If if the link state does change,
+ * the EMAC issues an interrupt on behalf of the PHY.
+ */
+static void emac_phy_mdio_autopoll_enable(struct emac_adapter *adpt)
+{
+	emac_reg_update32(adpt->base + EMAC_MDIO_CTRL, 0, MDIO_AP_EN);
+}
+
+static int emac_mdio_read(struct mii_bus *bus, int addr, int regnum)
+{
+	struct emac_adapter *adpt = bus->priv;
+	u32 reg;
+	int ret;
+
+	ret = emac_phy_mdio_autopoll_disable(adpt);
+	if (ret)
+		return ret;
+
+	emac_reg_update32(adpt->base + EMAC_PHY_STS, PHY_ADDR_BMSK,
+			  (addr << PHY_ADDR_SHFT));
+
+	reg = SUP_PREAMBLE |
+	      ((MDIO_CLK_25_4 << MDIO_CLK_SEL_SHFT) & MDIO_CLK_SEL_BMSK) |
+	      ((regnum << MDIO_REG_ADDR_SHFT) & MDIO_REG_ADDR_BMSK) |
+	      MDIO_START | MDIO_RD_NWR;
+
+	writel(reg, adpt->base + EMAC_MDIO_CTRL);
+
+	if (readl_poll_timeout(adpt->base + EMAC_MDIO_CTRL, reg,
+			       !(reg & (MDIO_START | MDIO_BUSY)),
+			       100, MDIO_WAIT_TIMES * 100))
+		ret = -EIO;
+	else
+		ret = (reg >> MDIO_DATA_SHFT) & MDIO_DATA_BMSK;
+
+	emac_phy_mdio_autopoll_enable(adpt);
+
+	return ret;
+}
+
+static int emac_mdio_write(struct mii_bus *bus, int addr, int regnum, u16 val)
+{
+	struct emac_adapter *adpt = bus->priv;
+	u32 reg;
+	int ret;
+
+	ret = emac_phy_mdio_autopoll_disable(adpt);
+	if (ret)
+		return ret;
+
+	emac_reg_update32(adpt->base + EMAC_PHY_STS, PHY_ADDR_BMSK,
+			  (addr << PHY_ADDR_SHFT));
+
+	reg = SUP_PREAMBLE |
+		((MDIO_CLK_25_4 << MDIO_CLK_SEL_SHFT) & MDIO_CLK_SEL_BMSK) |
+		((regnum << MDIO_REG_ADDR_SHFT) & MDIO_REG_ADDR_BMSK) |
+		((val << MDIO_DATA_SHFT) & MDIO_DATA_BMSK) |
+		MDIO_START;
+
+	writel(reg, adpt->base + EMAC_MDIO_CTRL);
+
+	if (readl_poll_timeout(adpt->base + EMAC_MDIO_CTRL, reg,
+			       !(reg & (MDIO_START | MDIO_BUSY)), 100,
+			       MDIO_WAIT_TIMES * 100))
+		ret = -EIO;
+
+	emac_phy_mdio_autopoll_enable(adpt);
+
+	return ret;
+}
+
+/* Configure the MDIO bus and connect the external PHY */
+int emac_phy_config(struct platform_device *pdev, struct emac_adapter *adpt)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct mii_bus *mii_bus;
+	int ret;
+
+	/* Create the mii_bus object for talking to the MDIO bus */
+	adpt->mii_bus = mii_bus = devm_mdiobus_alloc(&pdev->dev);
+	if (!mii_bus)
+		return -ENOMEM;
+
+	mii_bus->name = "emac-mdio";
+	snprintf(mii_bus->id, MII_BUS_ID_SIZE, "%s", pdev->name);
+	mii_bus->read = emac_mdio_read;
+	mii_bus->write = emac_mdio_write;
+	mii_bus->parent = &pdev->dev;
+	mii_bus->priv = adpt;
+
+	if (has_acpi_companion(&pdev->dev)) {
+		u32 phy_addr;
+
+		ret = mdiobus_register(mii_bus);
+		if (ret) {
+			dev_err(&pdev->dev, "could not register mdio bus\n");
+			return ret;
+		}
+		ret = device_property_read_u32(&pdev->dev, "phy-channel",
+					       &phy_addr);
+		if (ret)
+			/* If we can't read a valid phy address, then assume
+			 * that there is only one phy on this mdio bus.
+			 */
+			adpt->phydev = phy_find_first(mii_bus);
+		else
+			adpt->phydev = mdiobus_get_phy(mii_bus, phy_addr);
+
+	} else {
+		struct device_node *phy_np;
+
+		ret = of_mdiobus_register(mii_bus, np);
+		if (ret) {
+			dev_err(&pdev->dev, "could not register mdio bus\n");
+			return ret;
+		}
+
+		phy_np = of_parse_phandle(np, "phy-handle", 0);
+		adpt->phydev = of_phy_find_device(phy_np);
+	}
+
+	if (!adpt->phydev) {
+		dev_err(&pdev->dev, "could not find external phy\n");
+		mdiobus_unregister(mii_bus);
+		return -ENODEV;
+	}
+
+	if (adpt->phydev->drv)
+		phy_attached_print(adpt->phydev, NULL);
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-phy.h b/drivers/net/ethernet/qualcomm/emac/emac-phy.h
new file mode 100644
index 000000000000..49f3701a6dd7
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/emac/emac-phy.h
@@ -0,0 +1,33 @@
+/* Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License version 2 and
+* only version 2 as published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*/
+
+#ifndef _EMAC_PHY_H_
+#define _EMAC_PHY_H_
+
+typedef int (*emac_sgmii_initialize)(struct emac_adapter *adpt);
+
+/** emac_phy - internal emac phy
+ * @base base address
+ * @digital per-lane digital block
+ * @initialize initialization function
+ */
+struct emac_phy {
+	void __iomem		*base;
+	void __iomem		*digital;
+	emac_sgmii_initialize	initialize;
+};
+
+struct emac_adapter;
+
+int emac_phy_config(struct platform_device *pdev, struct emac_adapter *adpt);
+
+#endif /* _EMAC_PHY_H_ */
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c
new file mode 100644
index 000000000000..75c1b530e39e
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.c
@@ -0,0 +1,784 @@
+/* Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/* Qualcomm Technologies, Inc. EMAC SGMII Controller driver.
+ */
+
+#include <linux/iopoll.h>
+#include <linux/acpi.h>
+#include <linux/of_device.h>
+#include "emac.h"
+#include "emac-mac.h"
+#include "emac-sgmii.h"
+
+/* EMAC_QSERDES register offsets */
+#define EMAC_QSERDES_COM_SYS_CLK_CTRL		0x000000
+#define EMAC_QSERDES_COM_PLL_CNTRL		0x000014
+#define EMAC_QSERDES_COM_PLL_IP_SETI		0x000018
+#define EMAC_QSERDES_COM_PLL_CP_SETI		0x000024
+#define EMAC_QSERDES_COM_PLL_IP_SETP		0x000028
+#define EMAC_QSERDES_COM_PLL_CP_SETP		0x00002c
+#define EMAC_QSERDES_COM_SYSCLK_EN_SEL		0x000038
+#define EMAC_QSERDES_COM_RESETSM_CNTRL		0x000040
+#define EMAC_QSERDES_COM_PLLLOCK_CMP1		0x000044
+#define EMAC_QSERDES_COM_PLLLOCK_CMP2		0x000048
+#define EMAC_QSERDES_COM_PLLLOCK_CMP3		0x00004c
+#define EMAC_QSERDES_COM_PLLLOCK_CMP_EN		0x000050
+#define EMAC_QSERDES_COM_DEC_START1		0x000064
+#define EMAC_QSERDES_COM_DIV_FRAC_START1	0x000098
+#define EMAC_QSERDES_COM_DIV_FRAC_START2	0x00009c
+#define EMAC_QSERDES_COM_DIV_FRAC_START3	0x0000a0
+#define EMAC_QSERDES_COM_DEC_START2		0x0000a4
+#define EMAC_QSERDES_COM_PLL_CRCTRL		0x0000ac
+#define EMAC_QSERDES_COM_RESET_SM		0x0000bc
+#define EMAC_QSERDES_TX_BIST_MODE_LANENO	0x000100
+#define EMAC_QSERDES_TX_TX_EMP_POST1_LVL	0x000108
+#define EMAC_QSERDES_TX_TX_DRV_LVL		0x00010c
+#define EMAC_QSERDES_TX_LANE_MODE		0x000150
+#define EMAC_QSERDES_TX_TRAN_DRVR_EMP_EN	0x000170
+#define EMAC_QSERDES_RX_CDR_CONTROL		0x000200
+#define EMAC_QSERDES_RX_CDR_CONTROL2		0x000210
+#define EMAC_QSERDES_RX_RX_EQ_GAIN12		0x000230
+
+/* EMAC_SGMII register offsets */
+#define EMAC_SGMII_PHY_SERDES_START		0x000000
+#define EMAC_SGMII_PHY_CMN_PWR_CTRL		0x000004
+#define EMAC_SGMII_PHY_RX_PWR_CTRL		0x000008
+#define EMAC_SGMII_PHY_TX_PWR_CTRL		0x00000C
+#define EMAC_SGMII_PHY_LANE_CTRL1		0x000018
+#define EMAC_SGMII_PHY_AUTONEG_CFG2		0x000048
+#define EMAC_SGMII_PHY_CDR_CTRL0		0x000058
+#define EMAC_SGMII_PHY_SPEED_CFG1		0x000074
+#define EMAC_SGMII_PHY_POW_DWN_CTRL0		0x000080
+#define EMAC_SGMII_PHY_RESET_CTRL		0x0000a8
+#define EMAC_SGMII_PHY_IRQ_CMD			0x0000ac
+#define EMAC_SGMII_PHY_INTERRUPT_CLEAR		0x0000b0
+#define EMAC_SGMII_PHY_INTERRUPT_MASK		0x0000b4
+#define EMAC_SGMII_PHY_INTERRUPT_STATUS		0x0000b8
+#define EMAC_SGMII_PHY_RX_CHK_STATUS		0x0000d4
+#define EMAC_SGMII_PHY_AUTONEG0_STATUS		0x0000e0
+#define EMAC_SGMII_PHY_AUTONEG1_STATUS		0x0000e4
+
+/* EMAC_QSERDES_COM_PLL_IP_SETI */
+#define PLL_IPSETI(x)				((x) & 0x3f)
+
+/* EMAC_QSERDES_COM_PLL_CP_SETI */
+#define PLL_CPSETI(x)				((x) & 0xff)
+
+/* EMAC_QSERDES_COM_PLL_IP_SETP */
+#define PLL_IPSETP(x)				((x) & 0x3f)
+
+/* EMAC_QSERDES_COM_PLL_CP_SETP */
+#define PLL_CPSETP(x)				((x) & 0x1f)
+
+/* EMAC_QSERDES_COM_PLL_CRCTRL */
+#define PLL_RCTRL(x)				(((x) & 0xf) << 4)
+#define PLL_CCTRL(x)				((x) & 0xf)
+
+/* SGMII v2 PHY registers per lane */
+#define EMAC_SGMII_PHY_LN_OFFSET		0x0400
+
+/* SGMII v2 digital lane registers */
+#define EMAC_SGMII_LN_DRVR_CTRL0		0x00C
+#define EMAC_SGMII_LN_DRVR_TAP_EN		0x018
+#define EMAC_SGMII_LN_TX_MARGINING		0x01C
+#define EMAC_SGMII_LN_TX_PRE			0x020
+#define EMAC_SGMII_LN_TX_POST			0x024
+#define EMAC_SGMII_LN_TX_BAND_MODE		0x060
+#define EMAC_SGMII_LN_LANE_MODE			0x064
+#define EMAC_SGMII_LN_PARALLEL_RATE		0x078
+#define EMAC_SGMII_LN_CML_CTRL_MODE0		0x0B8
+#define EMAC_SGMII_LN_MIXER_CTRL_MODE0		0x0D0
+#define EMAC_SGMII_LN_VGA_INITVAL		0x134
+#define EMAC_SGMII_LN_UCDR_FO_GAIN_MODE0	0x17C
+#define EMAC_SGMII_LN_UCDR_SO_GAIN_MODE0	0x188
+#define EMAC_SGMII_LN_UCDR_SO_CONFIG		0x194
+#define EMAC_SGMII_LN_RX_BAND			0x19C
+#define EMAC_SGMII_LN_RX_RCVR_PATH1_MODE0	0x1B8
+#define EMAC_SGMII_LN_RSM_CONFIG		0x1F0
+#define EMAC_SGMII_LN_SIGDET_ENABLES		0x224
+#define EMAC_SGMII_LN_SIGDET_CNTRL		0x228
+#define EMAC_SGMII_LN_SIGDET_DEGLITCH_CNTRL	0x22C
+#define EMAC_SGMII_LN_RX_EN_SIGNAL		0x2A0
+#define EMAC_SGMII_LN_RX_MISC_CNTRL0		0x2AC
+#define EMAC_SGMII_LN_DRVR_LOGIC_CLKDIV		0x2BC
+
+/* SGMII v2 digital lane register values */
+#define UCDR_STEP_BY_TWO_MODE0			BIT(7)
+#define UCDR_xO_GAIN_MODE(x)			((x) & 0x7f)
+#define UCDR_ENABLE				BIT(6)
+#define UCDR_SO_SATURATION(x)			((x) & 0x3f)
+#define SIGDET_LP_BYP_PS4			BIT(7)
+#define SIGDET_EN_PS0_TO_PS2			BIT(6)
+#define EN_ACCOUPLEVCM_SW_MUX			BIT(5)
+#define EN_ACCOUPLEVCM_SW			BIT(4)
+#define RX_SYNC_EN				BIT(3)
+#define RXTERM_HIGHZ_PS5			BIT(2)
+#define SIGDET_EN_PS3				BIT(1)
+#define EN_ACCOUPLE_VCM_PS3			BIT(0)
+#define UFS_MODE				BIT(5)
+#define TXVAL_VALID_INIT			BIT(4)
+#define TXVAL_VALID_MUX				BIT(3)
+#define TXVAL_VALID				BIT(2)
+#define USB3P1_MODE				BIT(1)
+#define KR_PCIGEN3_MODE				BIT(0)
+#define PRE_EN					BIT(3)
+#define POST_EN					BIT(2)
+#define MAIN_EN_MUX				BIT(1)
+#define MAIN_EN					BIT(0)
+#define TX_MARGINING_MUX			BIT(6)
+#define TX_MARGINING(x)				((x) & 0x3f)
+#define TX_PRE_MUX				BIT(6)
+#define TX_PRE(x)				((x) & 0x3f)
+#define TX_POST_MUX				BIT(6)
+#define TX_POST(x)				((x) & 0x3f)
+#define CML_GEAR_MODE(x)			(((x) & 7) << 3)
+#define CML2CMOS_IBOOST_MODE(x)			((x) & 7)
+#define MIXER_LOADB_MODE(x)			(((x) & 0xf) << 2)
+#define MIXER_DATARATE_MODE(x)			((x) & 3)
+#define VGA_THRESH_DFE(x)			((x) & 0x3f)
+#define SIGDET_LP_BYP_PS0_TO_PS2		BIT(5)
+#define SIGDET_LP_BYP_MUX			BIT(4)
+#define SIGDET_LP_BYP				BIT(3)
+#define SIGDET_EN_MUX				BIT(2)
+#define SIGDET_EN				BIT(1)
+#define SIGDET_FLT_BYP				BIT(0)
+#define SIGDET_LVL(x)				(((x) & 0xf) << 4)
+#define SIGDET_BW_CTRL(x)			((x) & 0xf)
+#define SIGDET_DEGLITCH_CTRL(x)			(((x) & 0xf) << 1)
+#define SIGDET_DEGLITCH_BYP			BIT(0)
+#define INVERT_PCS_RX_CLK			BIT(7)
+#define PWM_EN					BIT(6)
+#define RXBIAS_SEL(x)				(((x) & 0x3) << 4)
+#define EBDAC_SIGN				BIT(3)
+#define EDAC_SIGN				BIT(2)
+#define EN_AUXTAP1SIGN_INVERT			BIT(1)
+#define EN_DAC_CHOPPING				BIT(0)
+#define DRVR_LOGIC_CLK_EN			BIT(4)
+#define DRVR_LOGIC_CLK_DIV(x)			((x) & 0xf)
+#define PARALLEL_RATE_MODE2(x)			(((x) & 0x3) << 4)
+#define PARALLEL_RATE_MODE1(x)			(((x) & 0x3) << 2)
+#define PARALLEL_RATE_MODE0(x)			((x) & 0x3)
+#define BAND_MODE2(x)				(((x) & 0x3) << 4)
+#define BAND_MODE1(x)				(((x) & 0x3) << 2)
+#define BAND_MODE0(x)				((x) & 0x3)
+#define LANE_SYNC_MODE				BIT(5)
+#define LANE_MODE(x)				((x) & 0x1f)
+#define CDR_PD_SEL_MODE0(x)			(((x) & 0x3) << 5)
+#define EN_DLL_MODE0				BIT(4)
+#define EN_IQ_DCC_MODE0				BIT(3)
+#define EN_IQCAL_MODE0				BIT(2)
+#define EN_QPATH_MODE0				BIT(1)
+#define EN_EPATH_MODE0				BIT(0)
+#define FORCE_TSYNC_ACK				BIT(7)
+#define FORCE_CMN_ACK				BIT(6)
+#define FORCE_CMN_READY				BIT(5)
+#define EN_RCLK_DEGLITCH			BIT(4)
+#define BYPASS_RSM_CDR_RESET			BIT(3)
+#define BYPASS_RSM_TSYNC			BIT(2)
+#define BYPASS_RSM_SAMP_CAL			BIT(1)
+#define BYPASS_RSM_DLL_CAL			BIT(0)
+
+/* EMAC_QSERDES_COM_SYS_CLK_CTRL */
+#define SYSCLK_CM				BIT(4)
+#define SYSCLK_AC_COUPLE			BIT(3)
+
+/* EMAC_QSERDES_COM_PLL_CNTRL */
+#define OCP_EN					BIT(5)
+#define PLL_DIV_FFEN				BIT(2)
+#define PLL_DIV_ORD				BIT(1)
+
+/* EMAC_QSERDES_COM_SYSCLK_EN_SEL */
+#define SYSCLK_SEL_CMOS				BIT(3)
+
+/* EMAC_QSERDES_COM_RESETSM_CNTRL */
+#define FRQ_TUNE_MODE				BIT(4)
+
+/* EMAC_QSERDES_COM_PLLLOCK_CMP_EN */
+#define PLLLOCK_CMP_EN				BIT(0)
+
+/* EMAC_QSERDES_COM_DEC_START1 */
+#define DEC_START1_MUX				BIT(7)
+#define DEC_START1(x)				((x) & 0x7f)
+
+/* EMAC_QSERDES_COM_DIV_FRAC_START1 * EMAC_QSERDES_COM_DIV_FRAC_START2 */
+#define DIV_FRAC_START_MUX			BIT(7)
+#define DIV_FRAC_START(x)			((x) & 0x7f)
+
+/* EMAC_QSERDES_COM_DIV_FRAC_START3 */
+#define DIV_FRAC_START3_MUX			BIT(4)
+#define DIV_FRAC_START3(x)			((x) & 0xf)
+
+/* EMAC_QSERDES_COM_DEC_START2 */
+#define DEC_START2_MUX				BIT(1)
+#define DEC_START2				BIT(0)
+
+/* EMAC_QSERDES_COM_RESET_SM */
+#define READY					BIT(5)
+
+/* EMAC_QSERDES_TX_TX_EMP_POST1_LVL */
+#define TX_EMP_POST1_LVL_MUX			BIT(5)
+#define TX_EMP_POST1_LVL(x)			((x) & 0x1f)
+#define TX_EMP_POST1_LVL_BMSK			0x1f
+#define TX_EMP_POST1_LVL_SHFT			0
+
+/* EMAC_QSERDES_TX_TX_DRV_LVL */
+#define TX_DRV_LVL_MUX				BIT(4)
+#define TX_DRV_LVL(x)				((x) & 0xf)
+
+/* EMAC_QSERDES_TX_TRAN_DRVR_EMP_EN */
+#define EMP_EN_MUX				BIT(1)
+#define EMP_EN					BIT(0)
+
+/* EMAC_QSERDES_RX_CDR_CONTROL & EMAC_QSERDES_RX_CDR_CONTROL2 */
+#define HBW_PD_EN				BIT(7)
+#define SECONDORDERENABLE			BIT(6)
+#define FIRSTORDER_THRESH(x)			(((x) & 0x7) << 3)
+#define SECONDORDERGAIN(x)			((x) & 0x7)
+
+/* EMAC_QSERDES_RX_RX_EQ_GAIN12 */
+#define RX_EQ_GAIN2(x)				(((x) & 0xf) << 4)
+#define RX_EQ_GAIN1(x)				((x) & 0xf)
+
+/* EMAC_SGMII_PHY_SERDES_START */
+#define SERDES_START				BIT(0)
+
+/* EMAC_SGMII_PHY_CMN_PWR_CTRL */
+#define BIAS_EN					BIT(6)
+#define PLL_EN					BIT(5)
+#define SYSCLK_EN				BIT(4)
+#define CLKBUF_L_EN				BIT(3)
+#define PLL_TXCLK_EN				BIT(1)
+#define PLL_RXCLK_EN				BIT(0)
+
+/* EMAC_SGMII_PHY_RX_PWR_CTRL */
+#define L0_RX_SIGDET_EN				BIT(7)
+#define L0_RX_TERM_MODE(x)			(((x) & 3) << 4)
+#define L0_RX_I_EN				BIT(1)
+
+/* EMAC_SGMII_PHY_TX_PWR_CTRL */
+#define L0_TX_EN				BIT(5)
+#define L0_CLKBUF_EN				BIT(4)
+#define L0_TRAN_BIAS_EN				BIT(1)
+
+/* EMAC_SGMII_PHY_LANE_CTRL1 */
+#define L0_RX_EQUALIZE_ENABLE			BIT(6)
+#define L0_RESET_TSYNC_EN			BIT(4)
+#define L0_DRV_LVL(x)				((x) & 0xf)
+
+/* EMAC_SGMII_PHY_AUTONEG_CFG2 */
+#define FORCE_AN_TX_CFG				BIT(5)
+#define FORCE_AN_RX_CFG				BIT(4)
+#define AN_ENABLE				BIT(0)
+
+/* EMAC_SGMII_PHY_SPEED_CFG1 */
+#define DUPLEX_MODE				BIT(4)
+#define SPDMODE_1000				BIT(1)
+#define SPDMODE_100				BIT(0)
+#define SPDMODE_10				0
+#define SPDMODE_BMSK				3
+#define SPDMODE_SHFT				0
+
+/* EMAC_SGMII_PHY_POW_DWN_CTRL0 */
+#define PWRDN_B					BIT(0)
+#define CDR_MAX_CNT(x)				((x) & 0xff)
+
+/* EMAC_QSERDES_TX_BIST_MODE_LANENO */
+#define BIST_LANE_NUMBER(x)			(((x) & 3) << 5)
+#define BISTMODE(x)				((x) & 0x1f)
+
+/* EMAC_QSERDES_COM_PLLLOCK_CMPx */
+#define PLLLOCK_CMP(x)				((x) & 0xff)
+
+/* EMAC_SGMII_PHY_RESET_CTRL */
+#define PHY_SW_RESET				BIT(0)
+
+/* EMAC_SGMII_PHY_IRQ_CMD */
+#define IRQ_GLOBAL_CLEAR			BIT(0)
+
+/* EMAC_SGMII_PHY_INTERRUPT_MASK */
+#define DECODE_CODE_ERR				BIT(7)
+#define DECODE_DISP_ERR				BIT(6)
+#define PLL_UNLOCK				BIT(5)
+#define AN_ILLEGAL_TERM				BIT(4)
+#define SYNC_FAIL				BIT(3)
+#define AN_START				BIT(2)
+#define AN_END					BIT(1)
+#define AN_REQUEST				BIT(0)
+
+#define SGMII_PHY_IRQ_CLR_WAIT_TIME		10
+
+#define SGMII_PHY_INTERRUPT_ERR (\
+	DECODE_CODE_ERR         |\
+	DECODE_DISP_ERR)
+
+#define SGMII_ISR_AN_MASK       (\
+	AN_REQUEST              |\
+	AN_START                |\
+	AN_END                  |\
+	AN_ILLEGAL_TERM         |\
+	PLL_UNLOCK              |\
+	SYNC_FAIL)
+
+#define SGMII_ISR_MASK          (\
+	SGMII_PHY_INTERRUPT_ERR |\
+	SGMII_ISR_AN_MASK)
+
+/* SGMII TX_CONFIG */
+#define TXCFG_LINK				0x8000
+#define TXCFG_MODE_BMSK				0x1c00
+#define TXCFG_1000_FULL				0x1800
+#define TXCFG_100_FULL				0x1400
+#define TXCFG_100_HALF				0x0400
+#define TXCFG_10_FULL				0x1000
+#define TXCFG_10_HALF				0x0000
+
+#define SERDES_START_WAIT_TIMES			100
+
+struct emac_reg_write {
+	unsigned int offset;
+	u32 val;
+};
+
+static void emac_reg_write_all(void __iomem *base,
+			       const struct emac_reg_write *itr, size_t size)
+{
+	size_t i;
+
+	for (i = 0; i < size; ++itr, ++i)
+		writel(itr->val, base + itr->offset);
+}
+
+static const struct emac_reg_write physical_coding_sublayer_programming_v1[] = {
+	{EMAC_SGMII_PHY_CDR_CTRL0, CDR_MAX_CNT(15)},
+	{EMAC_SGMII_PHY_POW_DWN_CTRL0, PWRDN_B},
+	{EMAC_SGMII_PHY_CMN_PWR_CTRL,
+		BIAS_EN | SYSCLK_EN | CLKBUF_L_EN | PLL_TXCLK_EN | PLL_RXCLK_EN},
+	{EMAC_SGMII_PHY_TX_PWR_CTRL, L0_TX_EN | L0_CLKBUF_EN | L0_TRAN_BIAS_EN},
+	{EMAC_SGMII_PHY_RX_PWR_CTRL,
+		L0_RX_SIGDET_EN | L0_RX_TERM_MODE(1) | L0_RX_I_EN},
+	{EMAC_SGMII_PHY_CMN_PWR_CTRL,
+		BIAS_EN | PLL_EN | SYSCLK_EN | CLKBUF_L_EN | PLL_TXCLK_EN |
+		PLL_RXCLK_EN},
+	{EMAC_SGMII_PHY_LANE_CTRL1,
+		L0_RX_EQUALIZE_ENABLE | L0_RESET_TSYNC_EN | L0_DRV_LVL(15)},
+};
+
+static const struct emac_reg_write sysclk_refclk_setting[] = {
+	{EMAC_QSERDES_COM_SYSCLK_EN_SEL, SYSCLK_SEL_CMOS},
+	{EMAC_QSERDES_COM_SYS_CLK_CTRL,	SYSCLK_CM | SYSCLK_AC_COUPLE},
+};
+
+static const struct emac_reg_write pll_setting[] = {
+	{EMAC_QSERDES_COM_PLL_IP_SETI, PLL_IPSETI(1)},
+	{EMAC_QSERDES_COM_PLL_CP_SETI, PLL_CPSETI(59)},
+	{EMAC_QSERDES_COM_PLL_IP_SETP, PLL_IPSETP(10)},
+	{EMAC_QSERDES_COM_PLL_CP_SETP, PLL_CPSETP(9)},
+	{EMAC_QSERDES_COM_PLL_CRCTRL, PLL_RCTRL(15) | PLL_CCTRL(11)},
+	{EMAC_QSERDES_COM_PLL_CNTRL, OCP_EN | PLL_DIV_FFEN | PLL_DIV_ORD},
+	{EMAC_QSERDES_COM_DEC_START1, DEC_START1_MUX | DEC_START1(2)},
+	{EMAC_QSERDES_COM_DEC_START2, DEC_START2_MUX | DEC_START2},
+	{EMAC_QSERDES_COM_DIV_FRAC_START1,
+		DIV_FRAC_START_MUX | DIV_FRAC_START(85)},
+	{EMAC_QSERDES_COM_DIV_FRAC_START2,
+		DIV_FRAC_START_MUX | DIV_FRAC_START(42)},
+	{EMAC_QSERDES_COM_DIV_FRAC_START3,
+		DIV_FRAC_START3_MUX | DIV_FRAC_START3(3)},
+	{EMAC_QSERDES_COM_PLLLOCK_CMP1, PLLLOCK_CMP(43)},
+	{EMAC_QSERDES_COM_PLLLOCK_CMP2, PLLLOCK_CMP(104)},
+	{EMAC_QSERDES_COM_PLLLOCK_CMP3, PLLLOCK_CMP(0)},
+	{EMAC_QSERDES_COM_PLLLOCK_CMP_EN, PLLLOCK_CMP_EN},
+	{EMAC_QSERDES_COM_RESETSM_CNTRL, FRQ_TUNE_MODE},
+};
+
+static const struct emac_reg_write cdr_setting[] = {
+	{EMAC_QSERDES_RX_CDR_CONTROL,
+		SECONDORDERENABLE | FIRSTORDER_THRESH(3) | SECONDORDERGAIN(2)},
+	{EMAC_QSERDES_RX_CDR_CONTROL2,
+		SECONDORDERENABLE | FIRSTORDER_THRESH(3) | SECONDORDERGAIN(4)},
+};
+
+static const struct emac_reg_write tx_rx_setting[] = {
+	{EMAC_QSERDES_TX_BIST_MODE_LANENO, 0},
+	{EMAC_QSERDES_TX_TX_DRV_LVL, TX_DRV_LVL_MUX | TX_DRV_LVL(15)},
+	{EMAC_QSERDES_TX_TRAN_DRVR_EMP_EN, EMP_EN_MUX | EMP_EN},
+	{EMAC_QSERDES_TX_TX_EMP_POST1_LVL,
+		TX_EMP_POST1_LVL_MUX | TX_EMP_POST1_LVL(1)},
+	{EMAC_QSERDES_RX_RX_EQ_GAIN12, RX_EQ_GAIN2(15) | RX_EQ_GAIN1(15)},
+	{EMAC_QSERDES_TX_LANE_MODE, LANE_MODE(8)},
+};
+
+static const struct emac_reg_write sgmii_v2_laned[] = {
+	/* CDR Settings */
+	{EMAC_SGMII_LN_UCDR_FO_GAIN_MODE0,
+		UCDR_STEP_BY_TWO_MODE0 | UCDR_xO_GAIN_MODE(10)},
+	{EMAC_SGMII_LN_UCDR_SO_GAIN_MODE0, UCDR_xO_GAIN_MODE(6)},
+	{EMAC_SGMII_LN_UCDR_SO_CONFIG, UCDR_ENABLE | UCDR_SO_SATURATION(12)},
+
+	/* TX/RX Settings */
+	{EMAC_SGMII_LN_RX_EN_SIGNAL, SIGDET_LP_BYP_PS4 | SIGDET_EN_PS0_TO_PS2},
+
+	{EMAC_SGMII_LN_DRVR_CTRL0, TXVAL_VALID_INIT | KR_PCIGEN3_MODE},
+	{EMAC_SGMII_LN_DRVR_TAP_EN, MAIN_EN},
+	{EMAC_SGMII_LN_TX_MARGINING, TX_MARGINING_MUX | TX_MARGINING(25)},
+	{EMAC_SGMII_LN_TX_PRE, TX_PRE_MUX},
+	{EMAC_SGMII_LN_TX_POST, TX_POST_MUX},
+
+	{EMAC_SGMII_LN_CML_CTRL_MODE0,
+		CML_GEAR_MODE(1) | CML2CMOS_IBOOST_MODE(1)},
+	{EMAC_SGMII_LN_MIXER_CTRL_MODE0,
+		MIXER_LOADB_MODE(12) | MIXER_DATARATE_MODE(1)},
+	{EMAC_SGMII_LN_VGA_INITVAL, VGA_THRESH_DFE(31)},
+	{EMAC_SGMII_LN_SIGDET_ENABLES,
+		SIGDET_LP_BYP_PS0_TO_PS2 | SIGDET_FLT_BYP},
+	{EMAC_SGMII_LN_SIGDET_CNTRL, SIGDET_LVL(8)},
+
+	{EMAC_SGMII_LN_SIGDET_DEGLITCH_CNTRL, SIGDET_DEGLITCH_CTRL(4)},
+	{EMAC_SGMII_LN_RX_MISC_CNTRL0, 0},
+	{EMAC_SGMII_LN_DRVR_LOGIC_CLKDIV,
+		DRVR_LOGIC_CLK_EN | DRVR_LOGIC_CLK_DIV(4)},
+
+	{EMAC_SGMII_LN_PARALLEL_RATE, PARALLEL_RATE_MODE0(1)},
+	{EMAC_SGMII_LN_TX_BAND_MODE, BAND_MODE0(2)},
+	{EMAC_SGMII_LN_RX_BAND, BAND_MODE0(3)},
+	{EMAC_SGMII_LN_LANE_MODE, LANE_MODE(26)},
+	{EMAC_SGMII_LN_RX_RCVR_PATH1_MODE0, CDR_PD_SEL_MODE0(3)},
+	{EMAC_SGMII_LN_RSM_CONFIG, BYPASS_RSM_SAMP_CAL | BYPASS_RSM_DLL_CAL},
+};
+
+static const struct emac_reg_write physical_coding_sublayer_programming_v2[] = {
+	{EMAC_SGMII_PHY_POW_DWN_CTRL0, PWRDN_B},
+	{EMAC_SGMII_PHY_CDR_CTRL0, CDR_MAX_CNT(15)},
+	{EMAC_SGMII_PHY_TX_PWR_CTRL, 0},
+	{EMAC_SGMII_PHY_LANE_CTRL1, L0_RX_EQUALIZE_ENABLE},
+};
+
+static int emac_sgmii_link_init(struct emac_adapter *adpt)
+{
+	struct phy_device *phydev = adpt->phydev;
+	struct emac_phy *phy = &adpt->phy;
+	u32 val;
+
+	val = readl(phy->base + EMAC_SGMII_PHY_AUTONEG_CFG2);
+
+	if (phydev->autoneg == AUTONEG_ENABLE) {
+		val &= ~(FORCE_AN_RX_CFG | FORCE_AN_TX_CFG);
+		val |= AN_ENABLE;
+		writel(val, phy->base + EMAC_SGMII_PHY_AUTONEG_CFG2);
+	} else {
+		u32 speed_cfg;
+
+		switch (phydev->speed) {
+		case SPEED_10:
+			speed_cfg = SPDMODE_10;
+			break;
+		case SPEED_100:
+			speed_cfg = SPDMODE_100;
+			break;
+		case SPEED_1000:
+			speed_cfg = SPDMODE_1000;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		if (phydev->duplex == DUPLEX_FULL)
+			speed_cfg |= DUPLEX_MODE;
+
+		val &= ~AN_ENABLE;
+		writel(speed_cfg, phy->base + EMAC_SGMII_PHY_SPEED_CFG1);
+		writel(val, phy->base + EMAC_SGMII_PHY_AUTONEG_CFG2);
+	}
+
+	return 0;
+}
+
+static int emac_sgmii_irq_clear(struct emac_adapter *adpt, u32 irq_bits)
+{
+	struct emac_phy *phy = &adpt->phy;
+	u32 status;
+
+	writel_relaxed(irq_bits, phy->base + EMAC_SGMII_PHY_INTERRUPT_CLEAR);
+	writel_relaxed(IRQ_GLOBAL_CLEAR, phy->base + EMAC_SGMII_PHY_IRQ_CMD);
+	/* Ensure interrupt clear command is written to HW */
+	wmb();
+
+	/* After set the IRQ_GLOBAL_CLEAR bit, the status clearing must
+	 * be confirmed before clearing the bits in other registers.
+	 * It takes a few cycles for hw to clear the interrupt status.
+	 */
+	if (readl_poll_timeout_atomic(phy->base +
+				      EMAC_SGMII_PHY_INTERRUPT_STATUS,
+				      status, !(status & irq_bits), 1,
+				      SGMII_PHY_IRQ_CLR_WAIT_TIME)) {
+		netdev_err(adpt->netdev,
+			   "error: failed clear SGMII irq: status:0x%x bits:0x%x\n",
+			   status, irq_bits);
+		return -EIO;
+	}
+
+	/* Finalize clearing procedure */
+	writel_relaxed(0, phy->base + EMAC_SGMII_PHY_IRQ_CMD);
+	writel_relaxed(0, phy->base + EMAC_SGMII_PHY_INTERRUPT_CLEAR);
+
+	/* Ensure that clearing procedure finalization is written to HW */
+	wmb();
+
+	return 0;
+}
+
+int emac_sgmii_init_v1(struct emac_adapter *adpt)
+{
+	struct emac_phy *phy = &adpt->phy;
+	unsigned int i;
+	int ret;
+
+	ret = emac_sgmii_link_init(adpt);
+	if (ret)
+		return ret;
+
+	emac_reg_write_all(phy->base, physical_coding_sublayer_programming_v1,
+			   ARRAY_SIZE(physical_coding_sublayer_programming_v1));
+	emac_reg_write_all(phy->base, sysclk_refclk_setting,
+			   ARRAY_SIZE(sysclk_refclk_setting));
+	emac_reg_write_all(phy->base, pll_setting, ARRAY_SIZE(pll_setting));
+	emac_reg_write_all(phy->base, cdr_setting, ARRAY_SIZE(cdr_setting));
+	emac_reg_write_all(phy->base, tx_rx_setting,
+			   ARRAY_SIZE(tx_rx_setting));
+
+	/* Power up the Ser/Des engine */
+	writel(SERDES_START, phy->base + EMAC_SGMII_PHY_SERDES_START);
+
+	for (i = 0; i < SERDES_START_WAIT_TIMES; i++) {
+		if (readl(phy->base + EMAC_QSERDES_COM_RESET_SM) & READY)
+			break;
+		usleep_range(100, 200);
+	}
+
+	if (i == SERDES_START_WAIT_TIMES) {
+		netdev_err(adpt->netdev, "error: ser/des failed to start\n");
+		return -EIO;
+	}
+	/* Mask out all the SGMII Interrupt */
+	writel(0, phy->base + EMAC_SGMII_PHY_INTERRUPT_MASK);
+
+	emac_sgmii_irq_clear(adpt, SGMII_PHY_INTERRUPT_ERR);
+
+	return 0;
+}
+
+int emac_sgmii_init_v2(struct emac_adapter *adpt)
+{
+	struct emac_phy *phy = &adpt->phy;
+	void __iomem *phy_regs = phy->base;
+	void __iomem *laned = phy->digital;
+	unsigned int i;
+	u32 lnstatus;
+	int ret;
+
+	ret = emac_sgmii_link_init(adpt);
+	if (ret)
+		return ret;
+
+	/* PCS lane-x init */
+	emac_reg_write_all(phy->base, physical_coding_sublayer_programming_v2,
+			   ARRAY_SIZE(physical_coding_sublayer_programming_v2));
+
+	/* SGMII lane-x init */
+	emac_reg_write_all(phy->digital,
+			   sgmii_v2_laned, ARRAY_SIZE(sgmii_v2_laned));
+
+	/* Power up PCS and start reset lane state machine */
+
+	writel(0, phy_regs + EMAC_SGMII_PHY_RESET_CTRL);
+	writel(1, laned + SGMII_LN_RSM_START);
+
+	/* Wait for c_ready assertion */
+	for (i = 0; i < SERDES_START_WAIT_TIMES; i++) {
+		lnstatus = readl(phy_regs + SGMII_PHY_LN_LANE_STATUS);
+		if (lnstatus & BIT(1))
+			break;
+		usleep_range(100, 200);
+	}
+
+	if (i == SERDES_START_WAIT_TIMES) {
+		netdev_err(adpt->netdev, "SGMII failed to start\n");
+		return -EIO;
+	}
+
+	/* Disable digital and SERDES loopback */
+	writel(0, phy_regs + SGMII_PHY_LN_BIST_GEN0);
+	writel(0, phy_regs + SGMII_PHY_LN_BIST_GEN2);
+	writel(0, phy_regs + SGMII_PHY_LN_CDR_CTRL1);
+
+	/* Mask out all the SGMII Interrupt */
+	writel(0, phy_regs + EMAC_SGMII_PHY_INTERRUPT_MASK);
+
+	emac_sgmii_irq_clear(adpt, SGMII_PHY_INTERRUPT_ERR);
+
+	return 0;
+}
+
+static void emac_sgmii_reset_prepare(struct emac_adapter *adpt)
+{
+	struct emac_phy *phy = &adpt->phy;
+	u32 val;
+
+	/* Reset PHY */
+	val = readl(phy->base + EMAC_EMAC_WRAPPER_CSR2);
+	writel(((val & ~PHY_RESET) | PHY_RESET), phy->base +
+	       EMAC_EMAC_WRAPPER_CSR2);
+	/* Ensure phy-reset command is written to HW before the release cmd */
+	msleep(50);
+	val = readl(phy->base + EMAC_EMAC_WRAPPER_CSR2);
+	writel((val & ~PHY_RESET), phy->base + EMAC_EMAC_WRAPPER_CSR2);
+	/* Ensure phy-reset release command is written to HW before initializing
+	 * SGMII
+	 */
+	msleep(50);
+}
+
+void emac_sgmii_reset(struct emac_adapter *adpt)
+{
+	int ret;
+
+	clk_set_rate(adpt->clk[EMAC_CLK_HIGH_SPEED], 19200000);
+	emac_sgmii_reset_prepare(adpt);
+
+	ret = adpt->phy.initialize(adpt);
+	if (ret)
+		netdev_err(adpt->netdev,
+			   "could not reinitialize internal PHY (error=%i)\n",
+			   ret);
+
+	clk_set_rate(adpt->clk[EMAC_CLK_HIGH_SPEED], 125000000);
+}
+
+static int emac_sgmii_acpi_match(struct device *dev, void *data)
+{
+	static const struct acpi_device_id match_table[] = {
+		{
+			.id = "QCOM8071",
+			.driver_data = (kernel_ulong_t)emac_sgmii_init_v2,
+		},
+		{}
+	};
+	const struct acpi_device_id *id = acpi_match_device(match_table, dev);
+	emac_sgmii_initialize *initialize = data;
+
+	if (id)
+		*initialize = (emac_sgmii_initialize)id->driver_data;
+
+	return !!id;
+}
+
+static const struct of_device_id emac_sgmii_dt_match[] = {
+	{
+		.compatible = "qcom,fsm9900-emac-sgmii",
+		.data = emac_sgmii_init_v1,
+	},
+	{
+		.compatible = "qcom,qdf2432-emac-sgmii",
+		.data = emac_sgmii_init_v2,
+	},
+	{}
+};
+
+int emac_sgmii_config(struct platform_device *pdev, struct emac_adapter *adpt)
+{
+	struct platform_device *sgmii_pdev = NULL;
+	struct emac_phy *phy = &adpt->phy;
+	struct resource *res;
+	int ret;
+
+	if (has_acpi_companion(&pdev->dev)) {
+		struct device *dev;
+
+		dev = device_find_child(&pdev->dev, &phy->initialize,
+					emac_sgmii_acpi_match);
+
+		if (!dev) {
+			dev_err(&pdev->dev, "cannot find internal phy node\n");
+			return -ENODEV;
+		}
+
+		sgmii_pdev = to_platform_device(dev);
+	} else {
+		const struct of_device_id *match;
+		struct device_node *np;
+
+		np = of_parse_phandle(pdev->dev.of_node, "internal-phy", 0);
+		if (!np) {
+			dev_err(&pdev->dev, "missing internal-phy property\n");
+			return -ENODEV;
+		}
+
+		sgmii_pdev = of_find_device_by_node(np);
+		if (!sgmii_pdev) {
+			dev_err(&pdev->dev, "invalid internal-phy property\n");
+			return -ENODEV;
+		}
+
+		match = of_match_device(emac_sgmii_dt_match, &sgmii_pdev->dev);
+		if (!match) {
+			dev_err(&pdev->dev, "unrecognized internal phy node\n");
+			ret = -ENODEV;
+			goto error_put_device;
+		}
+
+		phy->initialize = (emac_sgmii_initialize)match->data;
+	}
+
+	/* Base address is the first address */
+	res = platform_get_resource(sgmii_pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		ret = -EINVAL;
+		goto error_put_device;
+	}
+
+	phy->base = ioremap(res->start, resource_size(res));
+	if (!phy->base) {
+		ret = -ENOMEM;
+		goto error_put_device;
+	}
+
+	/* v2 SGMII has a per-lane digital digital, so parse it if it exists */
+	res = platform_get_resource(sgmii_pdev, IORESOURCE_MEM, 1);
+	if (res) {
+		phy->digital = ioremap(res->start, resource_size(res));
+		if (!phy->digital) {
+			ret = -ENOMEM;
+			goto error_unmap_base;
+		}
+	}
+
+	ret = phy->initialize(adpt);
+	if (ret)
+		goto error;
+
+	/* We've remapped the addresses, so we don't need the device any
+	 * more.  of_find_device_by_node() says we should release it.
+	 */
+	put_device(&sgmii_pdev->dev);
+
+	return 0;
+
+error:
+	if (phy->digital)
+		iounmap(phy->digital);
+error_unmap_base:
+	iounmap(phy->base);
+error_put_device:
+	put_device(&sgmii_pdev->dev);
+
+	return ret;
+}
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-sgmii.h b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.h
new file mode 100644
index 000000000000..ce79212ff403
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/emac/emac-sgmii.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _EMAC_SGMII_H_
+#define _EMAC_SGMII_H_
+
+struct emac_adapter;
+struct platform_device;
+
+int emac_sgmii_init_v1(struct emac_adapter *adpt);
+int emac_sgmii_init_v2(struct emac_adapter *adpt);
+int emac_sgmii_config(struct platform_device *pdev, struct emac_adapter *adpt);
+void emac_sgmii_reset(struct emac_adapter *adpt);
+
+#endif
diff --git a/drivers/net/ethernet/qualcomm/emac/emac.c b/drivers/net/ethernet/qualcomm/emac/emac.c
new file mode 100644
index 000000000000..9bf3b2b82e95
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/emac/emac.c
@@ -0,0 +1,755 @@
+/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/* Qualcomm Technologies, Inc. EMAC Gigabit Ethernet Driver */
+
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include <linux/of_device.h>
+#include <linux/phy.h>
+#include <linux/platform_device.h>
+#include <linux/acpi.h>
+#include "emac.h"
+#include "emac-mac.h"
+#include "emac-phy.h"
+#include "emac-sgmii.h"
+
+#define EMAC_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK |  \
+		NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP)
+
+#define EMAC_RRD_SIZE					     4
+/* The RRD size if timestamping is enabled: */
+#define EMAC_TS_RRD_SIZE				     6
+#define EMAC_TPD_SIZE					     4
+#define EMAC_RFD_SIZE					     2
+
+#define REG_MAC_RX_STATUS_BIN		 EMAC_RXMAC_STATC_REG0
+#define REG_MAC_RX_STATUS_END		EMAC_RXMAC_STATC_REG22
+#define REG_MAC_TX_STATUS_BIN		 EMAC_TXMAC_STATC_REG0
+#define REG_MAC_TX_STATUS_END		EMAC_TXMAC_STATC_REG24
+
+#define RXQ0_NUM_RFD_PREF_DEF				     8
+#define TXQ0_NUM_TPD_PREF_DEF				     5
+
+#define EMAC_PREAMBLE_DEF				     7
+
+#define DMAR_DLY_CNT_DEF				    15
+#define DMAW_DLY_CNT_DEF				     4
+
+#define IMR_NORMAL_MASK         (\
+		ISR_ERROR       |\
+		ISR_GPHY_LINK   |\
+		ISR_TX_PKT      |\
+		GPHY_WAKEUP_INT)
+
+#define IMR_EXTENDED_MASK       (\
+		SW_MAN_INT      |\
+		ISR_OVER        |\
+		ISR_ERROR       |\
+		ISR_GPHY_LINK   |\
+		ISR_TX_PKT      |\
+		GPHY_WAKEUP_INT)
+
+#define ISR_TX_PKT      (\
+	TX_PKT_INT      |\
+	TX_PKT_INT1     |\
+	TX_PKT_INT2     |\
+	TX_PKT_INT3)
+
+#define ISR_GPHY_LINK        (\
+	GPHY_LINK_UP_INT     |\
+	GPHY_LINK_DOWN_INT)
+
+#define ISR_OVER        (\
+	RFD0_UR_INT     |\
+	RFD1_UR_INT     |\
+	RFD2_UR_INT     |\
+	RFD3_UR_INT     |\
+	RFD4_UR_INT     |\
+	RXF_OF_INT      |\
+	TXF_UR_INT)
+
+#define ISR_ERROR       (\
+	DMAR_TO_INT     |\
+	DMAW_TO_INT     |\
+	TXQ_TO_INT)
+
+/* in sync with enum emac_clk_id */
+static const char * const emac_clk_name[] = {
+	"axi_clk", "cfg_ahb_clk", "high_speed_clk", "mdio_clk", "tx_clk",
+	"rx_clk", "sys_clk"
+};
+
+void emac_reg_update32(void __iomem *addr, u32 mask, u32 val)
+{
+	u32 data = readl(addr);
+
+	writel(((data & ~mask) | val), addr);
+}
+
+/* reinitialize */
+int emac_reinit_locked(struct emac_adapter *adpt)
+{
+	int ret;
+
+	mutex_lock(&adpt->reset_lock);
+
+	emac_mac_down(adpt);
+	emac_sgmii_reset(adpt);
+	ret = emac_mac_up(adpt);
+
+	mutex_unlock(&adpt->reset_lock);
+
+	return ret;
+}
+
+/* NAPI */
+static int emac_napi_rtx(struct napi_struct *napi, int budget)
+{
+	struct emac_rx_queue *rx_q =
+		container_of(napi, struct emac_rx_queue, napi);
+	struct emac_adapter *adpt = netdev_priv(rx_q->netdev);
+	struct emac_irq *irq = rx_q->irq;
+	int work_done = 0;
+
+	emac_mac_rx_process(adpt, rx_q, &work_done, budget);
+
+	if (work_done < budget) {
+		napi_complete(napi);
+
+		irq->mask |= rx_q->intr;
+		writel(irq->mask, adpt->base + EMAC_INT_MASK);
+	}
+
+	return work_done;
+}
+
+/* Transmit the packet */
+static int emac_start_xmit(struct sk_buff *skb, struct net_device *netdev)
+{
+	struct emac_adapter *adpt = netdev_priv(netdev);
+
+	return emac_mac_tx_buf_send(adpt, &adpt->tx_q, skb);
+}
+
+irqreturn_t emac_isr(int _irq, void *data)
+{
+	struct emac_irq *irq = data;
+	struct emac_adapter *adpt =
+		container_of(irq, struct emac_adapter, irq);
+	struct emac_rx_queue *rx_q = &adpt->rx_q;
+	u32 isr, status;
+
+	/* disable the interrupt */
+	writel(0, adpt->base + EMAC_INT_MASK);
+
+	isr = readl_relaxed(adpt->base + EMAC_INT_STATUS);
+
+	status = isr & irq->mask;
+	if (status == 0)
+		goto exit;
+
+	if (status & ISR_ERROR) {
+		netif_warn(adpt,  intr, adpt->netdev,
+			   "warning: error irq status 0x%lx\n",
+			   status & ISR_ERROR);
+		/* reset MAC */
+		schedule_work(&adpt->work_thread);
+	}
+
+	/* Schedule the napi for receive queue with interrupt
+	 * status bit set
+	 */
+	if (status & rx_q->intr) {
+		if (napi_schedule_prep(&rx_q->napi)) {
+			irq->mask &= ~rx_q->intr;
+			__napi_schedule(&rx_q->napi);
+		}
+	}
+
+	if (status & TX_PKT_INT)
+		emac_mac_tx_process(adpt, &adpt->tx_q);
+
+	if (status & ISR_OVER)
+		net_warn_ratelimited("warning: TX/RX overflow\n");
+
+	/* link event */
+	if (status & ISR_GPHY_LINK)
+		phy_mac_interrupt(adpt->phydev, !!(status & GPHY_LINK_UP_INT));
+
+exit:
+	/* enable the interrupt */
+	writel(irq->mask, adpt->base + EMAC_INT_MASK);
+
+	return IRQ_HANDLED;
+}
+
+/* Configure VLAN tag strip/insert feature */
+static int emac_set_features(struct net_device *netdev,
+			     netdev_features_t features)
+{
+	netdev_features_t changed = features ^ netdev->features;
+	struct emac_adapter *adpt = netdev_priv(netdev);
+
+	/* We only need to reprogram the hardware if the VLAN tag features
+	 * have changed, and if it's already running.
+	 */
+	if (!(changed & (NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX)))
+		return 0;
+
+	if (!netif_running(netdev))
+		return 0;
+
+	/* emac_mac_mode_config() uses netdev->features to configure the EMAC,
+	 * so make sure it's set first.
+	 */
+	netdev->features = features;
+
+	return emac_reinit_locked(adpt);
+}
+
+/* Configure Multicast and Promiscuous modes */
+static void emac_rx_mode_set(struct net_device *netdev)
+{
+	struct emac_adapter *adpt = netdev_priv(netdev);
+	struct netdev_hw_addr *ha;
+
+	emac_mac_mode_config(adpt);
+
+	/* update multicast address filtering */
+	emac_mac_multicast_addr_clear(adpt);
+	netdev_for_each_mc_addr(ha, netdev)
+		emac_mac_multicast_addr_set(adpt, ha->addr);
+}
+
+/* Change the Maximum Transfer Unit (MTU) */
+static int emac_change_mtu(struct net_device *netdev, int new_mtu)
+{
+	unsigned int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
+	struct emac_adapter *adpt = netdev_priv(netdev);
+
+	if ((max_frame < EMAC_MIN_ETH_FRAME_SIZE) ||
+	    (max_frame > EMAC_MAX_ETH_FRAME_SIZE)) {
+		netdev_err(adpt->netdev, "error: invalid MTU setting\n");
+		return -EINVAL;
+	}
+
+	netif_info(adpt, hw, adpt->netdev,
+		   "changing MTU from %d to %d\n", netdev->mtu,
+		   new_mtu);
+	netdev->mtu = new_mtu;
+
+	if (netif_running(netdev))
+		return emac_reinit_locked(adpt);
+
+	return 0;
+}
+
+/* Called when the network interface is made active */
+static int emac_open(struct net_device *netdev)
+{
+	struct emac_adapter *adpt = netdev_priv(netdev);
+	int ret;
+
+	/* allocate rx/tx dma buffer & descriptors */
+	ret = emac_mac_rx_tx_rings_alloc_all(adpt);
+	if (ret) {
+		netdev_err(adpt->netdev, "error allocating rx/tx rings\n");
+		return ret;
+	}
+
+	ret = emac_mac_up(adpt);
+	if (ret) {
+		emac_mac_rx_tx_rings_free_all(adpt);
+		return ret;
+	}
+
+	emac_mac_start(adpt);
+
+	return 0;
+}
+
+/* Called when the network interface is disabled */
+static int emac_close(struct net_device *netdev)
+{
+	struct emac_adapter *adpt = netdev_priv(netdev);
+
+	mutex_lock(&adpt->reset_lock);
+
+	emac_mac_down(adpt);
+	emac_mac_rx_tx_rings_free_all(adpt);
+
+	mutex_unlock(&adpt->reset_lock);
+
+	return 0;
+}
+
+/* Respond to a TX hang */
+static void emac_tx_timeout(struct net_device *netdev)
+{
+	struct emac_adapter *adpt = netdev_priv(netdev);
+
+	schedule_work(&adpt->work_thread);
+}
+
+/* IOCTL support for the interface */
+static int emac_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
+{
+	if (!netif_running(netdev))
+		return -EINVAL;
+
+	if (!netdev->phydev)
+		return -ENODEV;
+
+	return phy_mii_ioctl(netdev->phydev, ifr, cmd);
+}
+
+/* Provide network statistics info for the interface */
+static struct rtnl_link_stats64 *emac_get_stats64(struct net_device *netdev,
+						  struct rtnl_link_stats64 *net_stats)
+{
+	struct emac_adapter *adpt = netdev_priv(netdev);
+	unsigned int addr = REG_MAC_RX_STATUS_BIN;
+	struct emac_stats *stats = &adpt->stats;
+	u64 *stats_itr = &adpt->stats.rx_ok;
+	u32 val;
+
+	spin_lock(&stats->lock);
+
+	while (addr <= REG_MAC_RX_STATUS_END) {
+		val = readl_relaxed(adpt->base + addr);
+		*stats_itr += val;
+		stats_itr++;
+		addr += sizeof(u32);
+	}
+
+	/* additional rx status */
+	val = readl_relaxed(adpt->base + EMAC_RXMAC_STATC_REG23);
+	adpt->stats.rx_crc_align += val;
+	val = readl_relaxed(adpt->base + EMAC_RXMAC_STATC_REG24);
+	adpt->stats.rx_jabbers += val;
+
+	/* update tx status */
+	addr = REG_MAC_TX_STATUS_BIN;
+	stats_itr = &adpt->stats.tx_ok;
+
+	while (addr <= REG_MAC_TX_STATUS_END) {
+		val = readl_relaxed(adpt->base + addr);
+		*stats_itr += val;
+		++stats_itr;
+		addr += sizeof(u32);
+	}
+
+	/* additional tx status */
+	val = readl_relaxed(adpt->base + EMAC_TXMAC_STATC_REG25);
+	adpt->stats.tx_col += val;
+
+	/* return parsed statistics */
+	net_stats->rx_packets = stats->rx_ok;
+	net_stats->tx_packets = stats->tx_ok;
+	net_stats->rx_bytes = stats->rx_byte_cnt;
+	net_stats->tx_bytes = stats->tx_byte_cnt;
+	net_stats->multicast = stats->rx_mcast;
+	net_stats->collisions = stats->tx_1_col + stats->tx_2_col * 2 +
+				stats->tx_late_col + stats->tx_abort_col;
+
+	net_stats->rx_errors = stats->rx_frag + stats->rx_fcs_err +
+			       stats->rx_len_err + stats->rx_sz_ov +
+			       stats->rx_align_err;
+	net_stats->rx_fifo_errors = stats->rx_rxf_ov;
+	net_stats->rx_length_errors = stats->rx_len_err;
+	net_stats->rx_crc_errors = stats->rx_fcs_err;
+	net_stats->rx_frame_errors = stats->rx_align_err;
+	net_stats->rx_over_errors = stats->rx_rxf_ov;
+	net_stats->rx_missed_errors = stats->rx_rxf_ov;
+
+	net_stats->tx_errors = stats->tx_late_col + stats->tx_abort_col +
+			       stats->tx_underrun + stats->tx_trunc;
+	net_stats->tx_fifo_errors = stats->tx_underrun;
+	net_stats->tx_aborted_errors = stats->tx_abort_col;
+	net_stats->tx_window_errors = stats->tx_late_col;
+
+	spin_unlock(&stats->lock);
+
+	return net_stats;
+}
+
+static const struct net_device_ops emac_netdev_ops = {
+	.ndo_open		= emac_open,
+	.ndo_stop		= emac_close,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_start_xmit		= emac_start_xmit,
+	.ndo_set_mac_address	= eth_mac_addr,
+	.ndo_change_mtu		= emac_change_mtu,
+	.ndo_do_ioctl		= emac_ioctl,
+	.ndo_tx_timeout		= emac_tx_timeout,
+	.ndo_get_stats64	= emac_get_stats64,
+	.ndo_set_features       = emac_set_features,
+	.ndo_set_rx_mode        = emac_rx_mode_set,
+};
+
+/* Watchdog task routine, called to reinitialize the EMAC */
+static void emac_work_thread(struct work_struct *work)
+{
+	struct emac_adapter *adpt =
+		container_of(work, struct emac_adapter, work_thread);
+
+	emac_reinit_locked(adpt);
+}
+
+/* Initialize various data structures  */
+static void emac_init_adapter(struct emac_adapter *adpt)
+{
+	u32 reg;
+
+	/* descriptors */
+	adpt->tx_desc_cnt = EMAC_DEF_TX_DESCS;
+	adpt->rx_desc_cnt = EMAC_DEF_RX_DESCS;
+
+	/* dma */
+	adpt->dma_order = emac_dma_ord_out;
+	adpt->dmar_block = emac_dma_req_4096;
+	adpt->dmaw_block = emac_dma_req_128;
+	adpt->dmar_dly_cnt = DMAR_DLY_CNT_DEF;
+	adpt->dmaw_dly_cnt = DMAW_DLY_CNT_DEF;
+	adpt->tpd_burst = TXQ0_NUM_TPD_PREF_DEF;
+	adpt->rfd_burst = RXQ0_NUM_RFD_PREF_DEF;
+
+	/* irq moderator */
+	reg = ((EMAC_DEF_RX_IRQ_MOD >> 1) << IRQ_MODERATOR2_INIT_SHFT) |
+	      ((EMAC_DEF_TX_IRQ_MOD >> 1) << IRQ_MODERATOR_INIT_SHFT);
+	adpt->irq_mod = reg;
+
+	/* others */
+	adpt->preamble = EMAC_PREAMBLE_DEF;
+}
+
+/* Get the clock */
+static int emac_clks_get(struct platform_device *pdev,
+			 struct emac_adapter *adpt)
+{
+	unsigned int i;
+
+	for (i = 0; i < EMAC_CLK_CNT; i++) {
+		struct clk *clk = devm_clk_get(&pdev->dev, emac_clk_name[i]);
+
+		if (IS_ERR(clk)) {
+			dev_err(&pdev->dev,
+				"could not claim clock %s (error=%li)\n",
+				emac_clk_name[i], PTR_ERR(clk));
+
+			return PTR_ERR(clk);
+		}
+
+		adpt->clk[i] = clk;
+	}
+
+	return 0;
+}
+
+/* Initialize clocks */
+static int emac_clks_phase1_init(struct platform_device *pdev,
+				 struct emac_adapter *adpt)
+{
+	int ret;
+
+	ret = emac_clks_get(pdev, adpt);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(adpt->clk[EMAC_CLK_AXI]);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(adpt->clk[EMAC_CLK_CFG_AHB]);
+	if (ret)
+		return ret;
+
+	ret = clk_set_rate(adpt->clk[EMAC_CLK_HIGH_SPEED], 19200000);
+	if (ret)
+		return ret;
+
+	return clk_prepare_enable(adpt->clk[EMAC_CLK_HIGH_SPEED]);
+}
+
+/* Enable clocks; needs emac_clks_phase1_init to be called before */
+static int emac_clks_phase2_init(struct platform_device *pdev,
+				 struct emac_adapter *adpt)
+{
+	int ret;
+
+	ret = clk_set_rate(adpt->clk[EMAC_CLK_TX], 125000000);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(adpt->clk[EMAC_CLK_TX]);
+	if (ret)
+		return ret;
+
+	ret = clk_set_rate(adpt->clk[EMAC_CLK_HIGH_SPEED], 125000000);
+	if (ret)
+		return ret;
+
+	ret = clk_set_rate(adpt->clk[EMAC_CLK_MDIO], 25000000);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(adpt->clk[EMAC_CLK_MDIO]);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(adpt->clk[EMAC_CLK_RX]);
+	if (ret)
+		return ret;
+
+	return clk_prepare_enable(adpt->clk[EMAC_CLK_SYS]);
+}
+
+static void emac_clks_teardown(struct emac_adapter *adpt)
+{
+
+	unsigned int i;
+
+	for (i = 0; i < EMAC_CLK_CNT; i++)
+		clk_disable_unprepare(adpt->clk[i]);
+}
+
+/* Get the resources */
+static int emac_probe_resources(struct platform_device *pdev,
+				struct emac_adapter *adpt)
+{
+	struct net_device *netdev = adpt->netdev;
+	struct resource *res;
+	char maddr[ETH_ALEN];
+	int ret = 0;
+
+	/* get mac address */
+	if (device_get_mac_address(&pdev->dev, maddr, ETH_ALEN))
+		ether_addr_copy(netdev->dev_addr, maddr);
+	else
+		eth_hw_addr_random(netdev);
+
+	/* Core 0 interrupt */
+	ret = platform_get_irq(pdev, 0);
+	if (ret < 0) {
+		dev_err(&pdev->dev,
+			"error: missing core0 irq resource (error=%i)\n", ret);
+		return ret;
+	}
+	adpt->irq.irq = ret;
+
+	/* base register address */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	adpt->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(adpt->base))
+		return PTR_ERR(adpt->base);
+
+	/* CSR register address */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	adpt->csr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(adpt->csr))
+		return PTR_ERR(adpt->csr);
+
+	netdev->base_addr = (unsigned long)adpt->base;
+
+	return 0;
+}
+
+static const struct of_device_id emac_dt_match[] = {
+	{
+		.compatible = "qcom,fsm9900-emac",
+	},
+	{}
+};
+
+#if IS_ENABLED(CONFIG_ACPI)
+static const struct acpi_device_id emac_acpi_match[] = {
+	{
+		.id = "QCOM8070",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(acpi, emac_acpi_match);
+#endif
+
+static int emac_probe(struct platform_device *pdev)
+{
+	struct net_device *netdev;
+	struct emac_adapter *adpt;
+	struct emac_phy *phy;
+	u16 devid, revid;
+	u32 reg;
+	int ret;
+
+	/* The EMAC itself is capable of 64-bit DMA, so try that first. */
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		/* Some platforms may restrict the EMAC's address bus to less
+		 * then the size of DDR. In this case, we need to try a
+		 * smaller mask.  We could try every possible smaller mask,
+		 * but that's overkill.  Instead, just fall to 32-bit, which
+		 * should always work.
+		 */
+		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+		if (ret) {
+			dev_err(&pdev->dev, "could not set DMA mask\n");
+			return ret;
+		}
+	}
+
+	netdev = alloc_etherdev(sizeof(struct emac_adapter));
+	if (!netdev)
+		return -ENOMEM;
+
+	dev_set_drvdata(&pdev->dev, netdev);
+	SET_NETDEV_DEV(netdev, &pdev->dev);
+
+	adpt = netdev_priv(netdev);
+	adpt->netdev = netdev;
+	adpt->msg_enable = EMAC_MSG_DEFAULT;
+
+	phy = &adpt->phy;
+
+	mutex_init(&adpt->reset_lock);
+	spin_lock_init(&adpt->stats.lock);
+
+	adpt->irq.mask = RX_PKT_INT0 | IMR_NORMAL_MASK;
+
+	ret = emac_probe_resources(pdev, adpt);
+	if (ret)
+		goto err_undo_netdev;
+
+	/* initialize clocks */
+	ret = emac_clks_phase1_init(pdev, adpt);
+	if (ret) {
+		dev_err(&pdev->dev, "could not initialize clocks\n");
+		goto err_undo_netdev;
+	}
+
+	netdev->watchdog_timeo = EMAC_WATCHDOG_TIME;
+	netdev->irq = adpt->irq.irq;
+
+	adpt->rrd_size = EMAC_RRD_SIZE;
+	adpt->tpd_size = EMAC_TPD_SIZE;
+	adpt->rfd_size = EMAC_RFD_SIZE;
+
+	netdev->netdev_ops = &emac_netdev_ops;
+
+	emac_init_adapter(adpt);
+
+	/* init external phy */
+	ret = emac_phy_config(pdev, adpt);
+	if (ret)
+		goto err_undo_clocks;
+
+	/* init internal sgmii phy */
+	ret = emac_sgmii_config(pdev, adpt);
+	if (ret)
+		goto err_undo_mdiobus;
+
+	/* enable clocks */
+	ret = emac_clks_phase2_init(pdev, adpt);
+	if (ret) {
+		dev_err(&pdev->dev, "could not initialize clocks\n");
+		goto err_undo_mdiobus;
+	}
+
+	emac_mac_reset(adpt);
+
+	/* set hw features */
+	netdev->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
+			NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_VLAN_CTAG_RX |
+			NETIF_F_HW_VLAN_CTAG_TX;
+	netdev->hw_features = netdev->features;
+
+	netdev->vlan_features |= NETIF_F_SG | NETIF_F_HW_CSUM |
+				 NETIF_F_TSO | NETIF_F_TSO6;
+
+	INIT_WORK(&adpt->work_thread, emac_work_thread);
+
+	/* Initialize queues */
+	emac_mac_rx_tx_ring_init_all(pdev, adpt);
+
+	netif_napi_add(netdev, &adpt->rx_q.napi, emac_napi_rtx,
+		       NAPI_POLL_WEIGHT);
+
+	ret = register_netdev(netdev);
+	if (ret) {
+		dev_err(&pdev->dev, "could not register net device\n");
+		goto err_undo_napi;
+	}
+
+	reg =  readl_relaxed(adpt->base + EMAC_DMA_MAS_CTRL);
+	devid = (reg & DEV_ID_NUM_BMSK)  >> DEV_ID_NUM_SHFT;
+	revid = (reg & DEV_REV_NUM_BMSK) >> DEV_REV_NUM_SHFT;
+	reg = readl_relaxed(adpt->base + EMAC_CORE_HW_VERSION);
+
+	netif_info(adpt, probe, netdev,
+		   "hardware id %d.%d, hardware version %d.%d.%d\n",
+		   devid, revid,
+		   (reg & MAJOR_BMSK) >> MAJOR_SHFT,
+		   (reg & MINOR_BMSK) >> MINOR_SHFT,
+		   (reg & STEP_BMSK)  >> STEP_SHFT);
+
+	return 0;
+
+err_undo_napi:
+	netif_napi_del(&adpt->rx_q.napi);
+err_undo_mdiobus:
+	mdiobus_unregister(adpt->mii_bus);
+err_undo_clocks:
+	emac_clks_teardown(adpt);
+err_undo_netdev:
+	free_netdev(netdev);
+
+	return ret;
+}
+
+static int emac_remove(struct platform_device *pdev)
+{
+	struct net_device *netdev = dev_get_drvdata(&pdev->dev);
+	struct emac_adapter *adpt = netdev_priv(netdev);
+
+	unregister_netdev(netdev);
+	netif_napi_del(&adpt->rx_q.napi);
+
+	emac_clks_teardown(adpt);
+
+	mdiobus_unregister(adpt->mii_bus);
+	free_netdev(netdev);
+
+	if (adpt->phy.digital)
+		iounmap(adpt->phy.digital);
+	iounmap(adpt->phy.base);
+
+	return 0;
+}
+
+static struct platform_driver emac_platform_driver = {
+	.probe	= emac_probe,
+	.remove	= emac_remove,
+	.driver = {
+		.name		= "qcom-emac",
+		.of_match_table = emac_dt_match,
+		.acpi_match_table = ACPI_PTR(emac_acpi_match),
+	},
+};
+
+module_platform_driver(emac_platform_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:qcom-emac");
diff --git a/drivers/net/ethernet/qualcomm/emac/emac.h b/drivers/net/ethernet/qualcomm/emac/emac.h
new file mode 100644
index 000000000000..0c76e6cb8c9e
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/emac/emac.h
@@ -0,0 +1,335 @@
+/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _EMAC_H_
+#define _EMAC_H_
+
+#include <linux/irqreturn.h>
+#include <linux/netdevice.h>
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include "emac-mac.h"
+#include "emac-phy.h"
+
+/* EMAC base register offsets */
+#define EMAC_DMA_MAS_CTRL                                     0x001400
+#define EMAC_IRQ_MOD_TIM_INIT                                 0x001408
+#define EMAC_BLK_IDLE_STS                                     0x00140c
+#define EMAC_PHY_LINK_DELAY                                   0x00141c
+#define EMAC_SYS_ALIV_CTRL                                    0x001434
+#define EMAC_MAC_IPGIFG_CTRL                                  0x001484
+#define EMAC_MAC_STA_ADDR0                                    0x001488
+#define EMAC_MAC_STA_ADDR1                                    0x00148c
+#define EMAC_HASH_TAB_REG0                                    0x001490
+#define EMAC_HASH_TAB_REG1                                    0x001494
+#define EMAC_MAC_HALF_DPLX_CTRL                               0x001498
+#define EMAC_MAX_FRAM_LEN_CTRL                                0x00149c
+#define EMAC_INT_STATUS                                       0x001600
+#define EMAC_INT_MASK                                         0x001604
+#define EMAC_RXMAC_STATC_REG0                                 0x001700
+#define EMAC_RXMAC_STATC_REG22                                0x001758
+#define EMAC_TXMAC_STATC_REG0                                 0x001760
+#define EMAC_TXMAC_STATC_REG24                                0x0017c0
+#define EMAC_CORE_HW_VERSION                                  0x001974
+#define EMAC_IDT_TABLE0                                       0x001b00
+#define EMAC_RXMAC_STATC_REG23                                0x001bc8
+#define EMAC_RXMAC_STATC_REG24                                0x001bcc
+#define EMAC_TXMAC_STATC_REG25                                0x001bd0
+#define EMAC_INT1_MASK                                        0x001bf0
+#define EMAC_INT1_STATUS                                      0x001bf4
+#define EMAC_INT2_MASK                                        0x001bf8
+#define EMAC_INT2_STATUS                                      0x001bfc
+#define EMAC_INT3_MASK                                        0x001c00
+#define EMAC_INT3_STATUS                                      0x001c04
+
+/* EMAC_DMA_MAS_CTRL */
+#define DEV_ID_NUM_BMSK                                     0x7f000000
+#define DEV_ID_NUM_SHFT                                             24
+#define DEV_REV_NUM_BMSK                                      0xff0000
+#define DEV_REV_NUM_SHFT                                            16
+#define INT_RD_CLR_EN                                           0x4000
+#define IRQ_MODERATOR2_EN                                        0x800
+#define IRQ_MODERATOR_EN                                         0x400
+#define LPW_CLK_SEL                                               0x80
+#define LPW_STATE                                                 0x20
+#define LPW_MODE                                                  0x10
+#define SOFT_RST                                                   0x1
+
+/* EMAC_IRQ_MOD_TIM_INIT */
+#define IRQ_MODERATOR2_INIT_BMSK                            0xffff0000
+#define IRQ_MODERATOR2_INIT_SHFT                                    16
+#define IRQ_MODERATOR_INIT_BMSK                                 0xffff
+#define IRQ_MODERATOR_INIT_SHFT                                      0
+
+/* EMAC_INT_STATUS */
+#define DIS_INT                                                BIT(31)
+#define PTP_INT                                                BIT(30)
+#define RFD4_UR_INT                                            BIT(29)
+#define TX_PKT_INT3                                            BIT(26)
+#define TX_PKT_INT2                                            BIT(25)
+#define TX_PKT_INT1                                            BIT(24)
+#define RX_PKT_INT3                                            BIT(19)
+#define RX_PKT_INT2                                            BIT(18)
+#define RX_PKT_INT1                                            BIT(17)
+#define RX_PKT_INT0                                            BIT(16)
+#define TX_PKT_INT                                             BIT(15)
+#define TXQ_TO_INT                                             BIT(14)
+#define GPHY_WAKEUP_INT                                        BIT(13)
+#define GPHY_LINK_DOWN_INT                                     BIT(12)
+#define GPHY_LINK_UP_INT                                       BIT(11)
+#define DMAW_TO_INT                                            BIT(10)
+#define DMAR_TO_INT                                             BIT(9)
+#define TXF_UR_INT                                              BIT(8)
+#define RFD3_UR_INT                                             BIT(7)
+#define RFD2_UR_INT                                             BIT(6)
+#define RFD1_UR_INT                                             BIT(5)
+#define RFD0_UR_INT                                             BIT(4)
+#define RXF_OF_INT                                              BIT(3)
+#define SW_MAN_INT                                              BIT(2)
+
+/* EMAC_MAILBOX_6 */
+#define RFD2_PROC_IDX_BMSK                                   0xfff0000
+#define RFD2_PROC_IDX_SHFT                                          16
+#define RFD2_PROD_IDX_BMSK                                       0xfff
+#define RFD2_PROD_IDX_SHFT                                           0
+
+/* EMAC_CORE_HW_VERSION */
+#define MAJOR_BMSK                                          0xf0000000
+#define MAJOR_SHFT                                                  28
+#define MINOR_BMSK                                           0xfff0000
+#define MINOR_SHFT                                                  16
+#define STEP_BMSK                                               0xffff
+#define STEP_SHFT                                                    0
+
+/* EMAC_EMAC_WRAPPER_CSR1 */
+#define TX_INDX_FIFO_SYNC_RST                                  BIT(23)
+#define TX_TS_FIFO_SYNC_RST                                    BIT(22)
+#define RX_TS_FIFO2_SYNC_RST                                   BIT(21)
+#define RX_TS_FIFO1_SYNC_RST                                   BIT(20)
+#define TX_TS_ENABLE                                           BIT(16)
+#define DIS_1588_CLKS                                          BIT(11)
+#define FREQ_MODE                                               BIT(9)
+#define ENABLE_RRD_TIMESTAMP                                    BIT(3)
+
+/* EMAC_EMAC_WRAPPER_CSR2 */
+#define HDRIVE_BMSK                                             0x3000
+#define HDRIVE_SHFT                                                 12
+#define SLB_EN                                                  BIT(9)
+#define PLB_EN                                                  BIT(8)
+#define WOL_EN                                                  BIT(3)
+#define PHY_RESET                                               BIT(0)
+
+#define EMAC_DEV_ID                                             0x0040
+
+/* SGMII v2 per lane registers */
+#define SGMII_LN_RSM_START             0x029C
+
+/* SGMII v2 PHY common registers */
+#define SGMII_PHY_CMN_CTRL            0x0408
+#define SGMII_PHY_CMN_RESET_CTRL      0x0410
+
+/* SGMII v2 PHY registers per lane */
+#define SGMII_PHY_LN_OFFSET          0x0400
+#define SGMII_PHY_LN_LANE_STATUS     0x00DC
+#define SGMII_PHY_LN_BIST_GEN0       0x008C
+#define SGMII_PHY_LN_BIST_GEN1       0x0090
+#define SGMII_PHY_LN_BIST_GEN2       0x0094
+#define SGMII_PHY_LN_BIST_GEN3       0x0098
+#define SGMII_PHY_LN_CDR_CTRL1       0x005C
+
+enum emac_clk_id {
+	EMAC_CLK_AXI,
+	EMAC_CLK_CFG_AHB,
+	EMAC_CLK_HIGH_SPEED,
+	EMAC_CLK_MDIO,
+	EMAC_CLK_TX,
+	EMAC_CLK_RX,
+	EMAC_CLK_SYS,
+	EMAC_CLK_CNT
+};
+
+#define EMAC_LINK_SPEED_UNKNOWN                                    0x0
+#define EMAC_LINK_SPEED_10_HALF                                 BIT(0)
+#define EMAC_LINK_SPEED_10_FULL                                 BIT(1)
+#define EMAC_LINK_SPEED_100_HALF                                BIT(2)
+#define EMAC_LINK_SPEED_100_FULL                                BIT(3)
+#define EMAC_LINK_SPEED_1GB_FULL                                BIT(5)
+
+#define EMAC_MAX_SETUP_LNK_CYCLE                                   100
+
+/* Wake On Lan */
+#define EMAC_WOL_PHY                     0x00000001 /* PHY Status Change */
+#define EMAC_WOL_MAGIC                   0x00000002 /* Magic Packet */
+
+struct emac_stats {
+	/* rx */
+	u64 rx_ok;              /* good packets */
+	u64 rx_bcast;           /* good broadcast packets */
+	u64 rx_mcast;           /* good multicast packets */
+	u64 rx_pause;           /* pause packet */
+	u64 rx_ctrl;            /* control packets other than pause frame. */
+	u64 rx_fcs_err;         /* packets with bad FCS. */
+	u64 rx_len_err;         /* packets with length mismatch */
+	u64 rx_byte_cnt;        /* good bytes count (without FCS) */
+	u64 rx_runt;            /* runt packets */
+	u64 rx_frag;            /* fragment count */
+	u64 rx_sz_64;	        /* packets that are 64 bytes */
+	u64 rx_sz_65_127;       /* packets that are 65-127 bytes */
+	u64 rx_sz_128_255;      /* packets that are 128-255 bytes */
+	u64 rx_sz_256_511;      /* packets that are 256-511 bytes */
+	u64 rx_sz_512_1023;     /* packets that are 512-1023 bytes */
+	u64 rx_sz_1024_1518;    /* packets that are 1024-1518 bytes */
+	u64 rx_sz_1519_max;     /* packets that are 1519-MTU bytes*/
+	u64 rx_sz_ov;           /* packets that are >MTU bytes (truncated) */
+	u64 rx_rxf_ov;          /* packets dropped due to RX FIFO overflow */
+	u64 rx_align_err;       /* alignment errors */
+	u64 rx_bcast_byte_cnt;  /* broadcast packets byte count (without FCS) */
+	u64 rx_mcast_byte_cnt;  /* multicast packets byte count (without FCS) */
+	u64 rx_err_addr;        /* packets dropped due to address filtering */
+	u64 rx_crc_align;       /* CRC align errors */
+	u64 rx_jabbers;         /* jabbers */
+
+	/* tx */
+	u64 tx_ok;              /* good packets */
+	u64 tx_bcast;           /* good broadcast packets */
+	u64 tx_mcast;           /* good multicast packets */
+	u64 tx_pause;           /* pause packets */
+	u64 tx_exc_defer;       /* packets with excessive deferral */
+	u64 tx_ctrl;            /* control packets other than pause frame */
+	u64 tx_defer;           /* packets that are deferred. */
+	u64 tx_byte_cnt;        /* good bytes count (without FCS) */
+	u64 tx_sz_64;           /* packets that are 64 bytes */
+	u64 tx_sz_65_127;       /* packets that are 65-127 bytes */
+	u64 tx_sz_128_255;      /* packets that are 128-255 bytes */
+	u64 tx_sz_256_511;      /* packets that are 256-511 bytes */
+	u64 tx_sz_512_1023;     /* packets that are 512-1023 bytes */
+	u64 tx_sz_1024_1518;    /* packets that are 1024-1518 bytes */
+	u64 tx_sz_1519_max;     /* packets that are 1519-MTU bytes */
+	u64 tx_1_col;           /* packets single prior collision */
+	u64 tx_2_col;           /* packets with multiple prior collisions */
+	u64 tx_late_col;        /* packets with late collisions */
+	u64 tx_abort_col;       /* packets aborted due to excess collisions */
+	u64 tx_underrun;        /* packets aborted due to FIFO underrun */
+	u64 tx_rd_eop;          /* count of reads beyond EOP */
+	u64 tx_len_err;         /* packets with length mismatch */
+	u64 tx_trunc;           /* packets truncated due to size >MTU */
+	u64 tx_bcast_byte;      /* broadcast packets byte count (without FCS) */
+	u64 tx_mcast_byte;      /* multicast packets byte count (without FCS) */
+	u64 tx_col;             /* collisions */
+
+	spinlock_t lock;	/* prevent multiple simultaneous readers */
+};
+
+/* RSS hstype Definitions */
+#define EMAC_RSS_HSTYP_IPV4_EN				    0x00000001
+#define EMAC_RSS_HSTYP_TCP4_EN				    0x00000002
+#define EMAC_RSS_HSTYP_IPV6_EN				    0x00000004
+#define EMAC_RSS_HSTYP_TCP6_EN				    0x00000008
+#define EMAC_RSS_HSTYP_ALL_EN (\
+		EMAC_RSS_HSTYP_IPV4_EN   |\
+		EMAC_RSS_HSTYP_TCP4_EN   |\
+		EMAC_RSS_HSTYP_IPV6_EN   |\
+		EMAC_RSS_HSTYP_TCP6_EN)
+
+#define EMAC_VLAN_TO_TAG(_vlan, _tag) \
+		(_tag =  ((((_vlan) >> 8) & 0xFF) | (((_vlan) & 0xFF) << 8)))
+
+#define EMAC_TAG_TO_VLAN(_tag, _vlan) \
+		(_vlan = ((((_tag) >> 8) & 0xFF) | (((_tag) & 0xFF) << 8)))
+
+#define EMAC_DEF_RX_BUF_SIZE					  1536
+#define EMAC_MAX_JUMBO_PKT_SIZE				    (9 * 1024)
+#define EMAC_MAX_TX_OFFLOAD_THRESH			    (9 * 1024)
+
+#define EMAC_MAX_ETH_FRAME_SIZE		       EMAC_MAX_JUMBO_PKT_SIZE
+#define EMAC_MIN_ETH_FRAME_SIZE					    68
+
+#define EMAC_DEF_TX_QUEUES					     1
+#define EMAC_DEF_RX_QUEUES					     1
+
+#define EMAC_MIN_TX_DESCS					   128
+#define EMAC_MIN_RX_DESCS					   128
+
+#define EMAC_MAX_TX_DESCS					 16383
+#define EMAC_MAX_RX_DESCS					  2047
+
+#define EMAC_DEF_TX_DESCS					   512
+#define EMAC_DEF_RX_DESCS					   256
+
+#define EMAC_DEF_RX_IRQ_MOD					   250
+#define EMAC_DEF_TX_IRQ_MOD					   250
+
+#define EMAC_WATCHDOG_TIME				      (5 * HZ)
+
+/* by default check link every 4 seconds */
+#define EMAC_TRY_LINK_TIMEOUT				      (4 * HZ)
+
+/* emac_irq per-device (per-adapter) irq properties.
+ * @irq:	irq number.
+ * @mask	mask to use over status register.
+ */
+struct emac_irq {
+	unsigned int	irq;
+	u32		mask;
+};
+
+/* The device's main data structure */
+struct emac_adapter {
+	struct net_device		*netdev;
+	struct mii_bus			*mii_bus;
+	struct phy_device		*phydev;
+
+	void __iomem			*base;
+	void __iomem			*csr;
+
+	struct emac_phy			phy;
+	struct emac_stats		stats;
+
+	struct emac_irq			irq;
+	struct clk			*clk[EMAC_CLK_CNT];
+
+	/* All Descriptor memory */
+	struct emac_ring_header		ring_header;
+	struct emac_tx_queue		tx_q;
+	struct emac_rx_queue		rx_q;
+	unsigned int			tx_desc_cnt;
+	unsigned int			rx_desc_cnt;
+	unsigned int			rrd_size; /* in quad words */
+	unsigned int			rfd_size; /* in quad words */
+	unsigned int			tpd_size; /* in quad words */
+
+	unsigned int			rxbuf_size;
+
+	/* Ring parameter */
+	u8				tpd_burst;
+	u8				rfd_burst;
+	unsigned int			dmaw_dly_cnt;
+	unsigned int			dmar_dly_cnt;
+	enum emac_dma_req_block		dmar_block;
+	enum emac_dma_req_block		dmaw_block;
+	enum emac_dma_order		dma_order;
+
+	u32				irq_mod;
+	u32				preamble;
+
+	struct work_struct		work_thread;
+
+	u16				msg_enable;
+
+	struct mutex			reset_lock;
+};
+
+int emac_reinit_locked(struct emac_adapter *adpt);
+void emac_reg_update32(void __iomem *addr, u32 mask, u32 val);
+irqreturn_t emac_isr(int irq, void *data);
+
+#endif /* _EMAC_H_ */
diff --git a/drivers/net/ethernet/rdc/r6040.c b/drivers/net/ethernet/rdc/r6040.c
index cb29ee24cf1b..5ef5d728c250 100644
--- a/drivers/net/ethernet/rdc/r6040.c
+++ b/drivers/net/ethernet/rdc/r6040.c
@@ -1062,14 +1062,12 @@ static int r6040_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* this should always be supported */
 	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 	if (err) {
-		dev_err(&pdev->dev, "32-bit PCI DMA addresses"
-				"not supported by the card\n");
+		dev_err(&pdev->dev, "32-bit PCI DMA addresses not supported by the card\n");
 		goto err_out_disable_dev;
 	}
 	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
 	if (err) {
-		dev_err(&pdev->dev, "32-bit PCI DMA addresses"
-				"not supported by the card\n");
+		dev_err(&pdev->dev, "32-bit PCI DMA addresses not supported by the card\n");
 		goto err_out_disable_dev;
 	}
 
diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c
index deae10d7426d..5297bf77211c 100644
--- a/drivers/net/ethernet/realtek/8139cp.c
+++ b/drivers/net/ethernet/realtek/8139cp.c
@@ -467,8 +467,8 @@ static int cp_rx_poll(struct napi_struct *napi, int budget)
 	unsigned int rx_tail = cp->rx_tail;
 	int rx;
 
-rx_status_loop:
 	rx = 0;
+rx_status_loop:
 	cpw16(IntrStatus, cp_rx_intr_mask);
 
 	while (rx < budget) {
diff --git a/drivers/net/ethernet/renesas/Kconfig b/drivers/net/ethernet/renesas/Kconfig
index 4f132cf177cd..85ec447c2d18 100644
--- a/drivers/net/ethernet/renesas/Kconfig
+++ b/drivers/net/ethernet/renesas/Kconfig
@@ -27,7 +27,7 @@ config SH_ETH
 	  Renesas SuperH Ethernet device driver.
 	  This driver supporting CPUs are:
 		- SH7619, SH7710, SH7712, SH7724, SH7734, SH7763, SH7757,
-		  R8A7740, R8A777x and R8A779x.
+		  R8A7740, R8A774x, R8A777x and R8A779x.
 
 config RAVB
 	tristate "Renesas Ethernet AVB support"
diff --git a/drivers/net/ethernet/renesas/ravb.h b/drivers/net/ethernet/renesas/ravb.h
index 4e5d5e953e15..f1109661a533 100644
--- a/drivers/net/ethernet/renesas/ravb.h
+++ b/drivers/net/ethernet/renesas/ravb.h
@@ -1011,7 +1011,6 @@ struct ravb_private {
 	struct work_struct work;
 	/* MII transceiver section. */
 	struct mii_bus *mii_bus;	/* MDIO bus control */
-	struct phy_device *phydev;	/* PHY device control */
 	int link;
 	phy_interface_t phy_interface;
 	int msg_enable;
diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
index 1e1cc0fad17f..630536bc72f9 100644
--- a/drivers/net/ethernet/renesas/ravb_main.c
+++ b/drivers/net/ethernet/renesas/ravb_main.c
@@ -942,7 +942,7 @@ out:
 static void ravb_adjust_link(struct net_device *ndev)
 {
 	struct ravb_private *priv = netdev_priv(ndev);
-	struct phy_device *phydev = priv->phydev;
+	struct phy_device *phydev = ndev->phydev;
 	bool new_state = false;
 
 	if (phydev->link) {
@@ -1032,48 +1032,47 @@ static int ravb_phy_init(struct net_device *ndev)
 
 	phy_attached_info(phydev);
 
-	priv->phydev = phydev;
-
 	return 0;
 }
 
 /* PHY control start function */
 static int ravb_phy_start(struct net_device *ndev)
 {
-	struct ravb_private *priv = netdev_priv(ndev);
 	int error;
 
 	error = ravb_phy_init(ndev);
 	if (error)
 		return error;
 
-	phy_start(priv->phydev);
+	phy_start(ndev->phydev);
 
 	return 0;
 }
 
-static int ravb_get_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
+static int ravb_get_link_ksettings(struct net_device *ndev,
+				   struct ethtool_link_ksettings *cmd)
 {
 	struct ravb_private *priv = netdev_priv(ndev);
 	int error = -ENODEV;
 	unsigned long flags;
 
-	if (priv->phydev) {
+	if (ndev->phydev) {
 		spin_lock_irqsave(&priv->lock, flags);
-		error = phy_ethtool_gset(priv->phydev, ecmd);
+		error = phy_ethtool_ksettings_get(ndev->phydev, cmd);
 		spin_unlock_irqrestore(&priv->lock, flags);
 	}
 
 	return error;
 }
 
-static int ravb_set_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
+static int ravb_set_link_ksettings(struct net_device *ndev,
+				   const struct ethtool_link_ksettings *cmd)
 {
 	struct ravb_private *priv = netdev_priv(ndev);
 	unsigned long flags;
 	int error;
 
-	if (!priv->phydev)
+	if (!ndev->phydev)
 		return -ENODEV;
 
 	spin_lock_irqsave(&priv->lock, flags);
@@ -1081,11 +1080,11 @@ static int ravb_set_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
 	/* Disable TX and RX */
 	ravb_rcv_snd_disable(ndev);
 
-	error = phy_ethtool_sset(priv->phydev, ecmd);
+	error = phy_ethtool_ksettings_set(ndev->phydev, cmd);
 	if (error)
 		goto error_exit;
 
-	if (ecmd->duplex == DUPLEX_FULL)
+	if (cmd->base.duplex == DUPLEX_FULL)
 		priv->duplex = 1;
 	else
 		priv->duplex = 0;
@@ -1110,9 +1109,9 @@ static int ravb_nway_reset(struct net_device *ndev)
 	int error = -ENODEV;
 	unsigned long flags;
 
-	if (priv->phydev) {
+	if (ndev->phydev) {
 		spin_lock_irqsave(&priv->lock, flags);
-		error = phy_start_aneg(priv->phydev);
+		error = phy_start_aneg(ndev->phydev);
 		spin_unlock_irqrestore(&priv->lock, flags);
 	}
 
@@ -1309,8 +1308,6 @@ static int ravb_get_ts_info(struct net_device *ndev,
 }
 
 static const struct ethtool_ops ravb_ethtool_ops = {
-	.get_settings		= ravb_get_settings,
-	.set_settings		= ravb_set_settings,
 	.nway_reset		= ravb_nway_reset,
 	.get_msglevel		= ravb_get_msglevel,
 	.set_msglevel		= ravb_set_msglevel,
@@ -1321,6 +1318,8 @@ static const struct ethtool_ops ravb_ethtool_ops = {
 	.get_ringparam		= ravb_get_ringparam,
 	.set_ringparam		= ravb_set_ringparam,
 	.get_ts_info		= ravb_get_ts_info,
+	.get_link_ksettings	= ravb_get_link_ksettings,
+	.set_link_ksettings	= ravb_set_link_ksettings,
 };
 
 static inline int ravb_hook_irq(unsigned int irq, irq_handler_t handler,
@@ -1661,10 +1660,9 @@ static int ravb_close(struct net_device *ndev)
 	}
 
 	/* PHY disconnect */
-	if (priv->phydev) {
-		phy_stop(priv->phydev);
-		phy_disconnect(priv->phydev);
-		priv->phydev = NULL;
+	if (ndev->phydev) {
+		phy_stop(ndev->phydev);
+		phy_disconnect(ndev->phydev);
 	}
 
 	if (priv->chip_id != RCAR_GEN2) {
@@ -1753,8 +1751,7 @@ static int ravb_hwtstamp_set(struct net_device *ndev, struct ifreq *req)
 /* ioctl to device function */
 static int ravb_do_ioctl(struct net_device *ndev, struct ifreq *req, int cmd)
 {
-	struct ravb_private *priv = netdev_priv(ndev);
-	struct phy_device *phydev = priv->phydev;
+	struct phy_device *phydev = ndev->phydev;
 
 	if (!netif_running(ndev))
 		return -EINVAL;
@@ -1876,6 +1873,20 @@ static int ravb_set_gti(struct net_device *ndev)
 	return 0;
 }
 
+static void ravb_set_config_mode(struct net_device *ndev)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+
+	if (priv->chip_id == RCAR_GEN2) {
+		ravb_modify(ndev, CCC, CCC_OPC, CCC_OPC_CONFIG);
+		/* Set CSEL value */
+		ravb_modify(ndev, CCC, CCC_CSEL, CCC_CSEL_HPB);
+	} else {
+		ravb_modify(ndev, CCC, CCC_OPC, CCC_OPC_CONFIG |
+			    CCC_GAC | CCC_CSEL_HPB);
+	}
+}
+
 static int ravb_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
@@ -1978,14 +1989,7 @@ static int ravb_probe(struct platform_device *pdev)
 	ndev->ethtool_ops = &ravb_ethtool_ops;
 
 	/* Set AVB config mode */
-	if (chip_id == RCAR_GEN2) {
-		ravb_modify(ndev, CCC, CCC_OPC, CCC_OPC_CONFIG);
-		/* Set CSEL value */
-		ravb_modify(ndev, CCC, CCC_CSEL, CCC_CSEL_HPB);
-	} else {
-		ravb_modify(ndev, CCC, CCC_OPC, CCC_OPC_CONFIG |
-			    CCC_GAC | CCC_CSEL_HPB);
-	}
+	ravb_set_config_mode(ndev);
 
 	/* Set GTI value */
 	error = ravb_set_gti(ndev);
@@ -2096,8 +2100,55 @@ static int ravb_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM
-static int ravb_runtime_nop(struct device *dev)
+static int __maybe_unused ravb_suspend(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	int ret = 0;
+
+	if (netif_running(ndev)) {
+		netif_device_detach(ndev);
+		ret = ravb_close(ndev);
+	}
+
+	return ret;
+}
+
+static int __maybe_unused ravb_resume(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct ravb_private *priv = netdev_priv(ndev);
+	int ret = 0;
+
+	/* All register have been reset to default values.
+	 * Restore all registers which where setup at probe time and
+	 * reopen device if it was running before system suspended.
+	 */
+
+	/* Set AVB config mode */
+	ravb_set_config_mode(ndev);
+
+	/* Set GTI value */
+	ret = ravb_set_gti(ndev);
+	if (ret)
+		return ret;
+
+	/* Request GTI loading */
+	ravb_modify(ndev, GCCR, GCCR_LTI, GCCR_LTI);
+
+	/* Restore descriptor base address table */
+	ravb_write(ndev, priv->desc_bat_dma, DBAT);
+
+	if (netif_running(ndev)) {
+		ret = ravb_open(ndev);
+		if (ret < 0)
+			return ret;
+		netif_device_attach(ndev);
+	}
+
+	return ret;
+}
+
+static int __maybe_unused ravb_runtime_nop(struct device *dev)
 {
 	/* Runtime PM callback shared between ->runtime_suspend()
 	 * and ->runtime_resume(). Simply returns success.
@@ -2110,20 +2161,16 @@ static int ravb_runtime_nop(struct device *dev)
 }
 
 static const struct dev_pm_ops ravb_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(ravb_suspend, ravb_resume)
 	SET_RUNTIME_PM_OPS(ravb_runtime_nop, ravb_runtime_nop, NULL)
 };
 
-#define RAVB_PM_OPS (&ravb_dev_pm_ops)
-#else
-#define RAVB_PM_OPS NULL
-#endif
-
 static struct platform_driver ravb_driver = {
 	.probe		= ravb_probe,
 	.remove		= ravb_remove,
 	.driver = {
 		.name	= "ravb",
-		.pm	= RAVB_PM_OPS,
+		.pm	= &ravb_dev_pm_ops,
 		.of_match_table = ravb_match_table,
 	},
 };
diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c
index 799d58d86e6d..05b0dc55de77 100644
--- a/drivers/net/ethernet/renesas/sh_eth.c
+++ b/drivers/net/ethernet/renesas/sh_eth.c
@@ -201,9 +201,14 @@ static const u16 sh_eth_offset_fast_rz[SH_ETH_MAX_REGISTER_OFFSET] = {
 
 	[ARSTR]		= 0x0000,
 	[TSU_CTRST]	= 0x0004,
+	[TSU_FWSLC]	= 0x0038,
 	[TSU_VTAG0]	= 0x0058,
 	[TSU_ADSBSY]	= 0x0060,
 	[TSU_TEN]	= 0x0064,
+	[TSU_POST1]	= 0x0070,
+	[TSU_POST2]	= 0x0074,
+	[TSU_POST3]	= 0x0078,
+	[TSU_POST4]	= 0x007c,
 	[TSU_ADRH0]	= 0x0100,
 
 	[TXNLCR0]	= 0x0080,
@@ -1723,7 +1728,7 @@ out:
 static void sh_eth_adjust_link(struct net_device *ndev)
 {
 	struct sh_eth_private *mdp = netdev_priv(ndev);
-	struct phy_device *phydev = mdp->phydev;
+	struct phy_device *phydev = ndev->phydev;
 	int new_state = 0;
 
 	if (phydev->link) {
@@ -1800,51 +1805,48 @@ static int sh_eth_phy_init(struct net_device *ndev)
 
 	phy_attached_info(phydev);
 
-	mdp->phydev = phydev;
-
 	return 0;
 }
 
 /* PHY control start function */
 static int sh_eth_phy_start(struct net_device *ndev)
 {
-	struct sh_eth_private *mdp = netdev_priv(ndev);
 	int ret;
 
 	ret = sh_eth_phy_init(ndev);
 	if (ret)
 		return ret;
 
-	phy_start(mdp->phydev);
+	phy_start(ndev->phydev);
 
 	return 0;
 }
 
-static int sh_eth_get_settings(struct net_device *ndev,
-			       struct ethtool_cmd *ecmd)
+static int sh_eth_get_link_ksettings(struct net_device *ndev,
+				     struct ethtool_link_ksettings *cmd)
 {
 	struct sh_eth_private *mdp = netdev_priv(ndev);
 	unsigned long flags;
 	int ret;
 
-	if (!mdp->phydev)
+	if (!ndev->phydev)
 		return -ENODEV;
 
 	spin_lock_irqsave(&mdp->lock, flags);
-	ret = phy_ethtool_gset(mdp->phydev, ecmd);
+	ret = phy_ethtool_ksettings_get(ndev->phydev, cmd);
 	spin_unlock_irqrestore(&mdp->lock, flags);
 
 	return ret;
 }
 
-static int sh_eth_set_settings(struct net_device *ndev,
-			       struct ethtool_cmd *ecmd)
+static int sh_eth_set_link_ksettings(struct net_device *ndev,
+				     const struct ethtool_link_ksettings *cmd)
 {
 	struct sh_eth_private *mdp = netdev_priv(ndev);
 	unsigned long flags;
 	int ret;
 
-	if (!mdp->phydev)
+	if (!ndev->phydev)
 		return -ENODEV;
 
 	spin_lock_irqsave(&mdp->lock, flags);
@@ -1852,11 +1854,11 @@ static int sh_eth_set_settings(struct net_device *ndev,
 	/* disable tx and rx */
 	sh_eth_rcv_snd_disable(ndev);
 
-	ret = phy_ethtool_sset(mdp->phydev, ecmd);
+	ret = phy_ethtool_ksettings_set(ndev->phydev, cmd);
 	if (ret)
 		goto error_exit;
 
-	if (ecmd->duplex == DUPLEX_FULL)
+	if (cmd->base.duplex == DUPLEX_FULL)
 		mdp->duplex = 1;
 	else
 		mdp->duplex = 0;
@@ -2067,11 +2069,11 @@ static int sh_eth_nway_reset(struct net_device *ndev)
 	unsigned long flags;
 	int ret;
 
-	if (!mdp->phydev)
+	if (!ndev->phydev)
 		return -ENODEV;
 
 	spin_lock_irqsave(&mdp->lock, flags);
-	ret = phy_start_aneg(mdp->phydev);
+	ret = phy_start_aneg(ndev->phydev);
 	spin_unlock_irqrestore(&mdp->lock, flags);
 
 	return ret;
@@ -2198,8 +2200,6 @@ static int sh_eth_set_ringparam(struct net_device *ndev,
 }
 
 static const struct ethtool_ops sh_eth_ethtool_ops = {
-	.get_settings	= sh_eth_get_settings,
-	.set_settings	= sh_eth_set_settings,
 	.get_regs_len	= sh_eth_get_regs_len,
 	.get_regs	= sh_eth_get_regs,
 	.nway_reset	= sh_eth_nway_reset,
@@ -2211,6 +2211,8 @@ static const struct ethtool_ops sh_eth_ethtool_ops = {
 	.get_sset_count     = sh_eth_get_sset_count,
 	.get_ringparam	= sh_eth_get_ringparam,
 	.set_ringparam	= sh_eth_set_ringparam,
+	.get_link_ksettings = sh_eth_get_link_ksettings,
+	.set_link_ksettings = sh_eth_set_link_ksettings,
 };
 
 /* network device open function */
@@ -2408,10 +2410,9 @@ static int sh_eth_close(struct net_device *ndev)
 	sh_eth_dev_exit(ndev);
 
 	/* PHY Disconnect */
-	if (mdp->phydev) {
-		phy_stop(mdp->phydev);
-		phy_disconnect(mdp->phydev);
-		mdp->phydev = NULL;
+	if (ndev->phydev) {
+		phy_stop(ndev->phydev);
+		phy_disconnect(ndev->phydev);
 	}
 
 	free_irq(ndev->irq, ndev);
@@ -2429,8 +2430,7 @@ static int sh_eth_close(struct net_device *ndev)
 /* ioctl to device function */
 static int sh_eth_do_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
 {
-	struct sh_eth_private *mdp = netdev_priv(ndev);
-	struct phy_device *phydev = mdp->phydev;
+	struct phy_device *phydev = ndev->phydev;
 
 	if (!netif_running(ndev))
 		return -EINVAL;
@@ -2786,6 +2786,8 @@ static void sh_eth_tsu_init(struct sh_eth_private *mdp)
 {
 	if (sh_eth_is_rz_fast_ether(mdp)) {
 		sh_eth_tsu_write(mdp, 0, TSU_TEN); /* Disable all CAM entry */
+		sh_eth_tsu_write(mdp, TSU_FWSLC_POSTENU | TSU_FWSLC_POSTENL,
+				 TSU_FWSLC);	/* Enable POST registers */
 		return;
 	}
 
@@ -2957,6 +2959,8 @@ static struct sh_eth_plat_data *sh_eth_parse_dt(struct device *dev)
 
 static const struct of_device_id sh_eth_match_table[] = {
 	{ .compatible = "renesas,gether-r8a7740", .data = &r8a7740_data },
+	{ .compatible = "renesas,ether-r8a7743", .data = &r8a779x_data },
+	{ .compatible = "renesas,ether-r8a7745", .data = &r8a779x_data },
 	{ .compatible = "renesas,ether-r8a7778", .data = &r8a777x_data },
 	{ .compatible = "renesas,ether-r8a7779", .data = &r8a777x_data },
 	{ .compatible = "renesas,ether-r8a7790", .data = &r8a779x_data },
diff --git a/drivers/net/ethernet/renesas/sh_eth.h b/drivers/net/ethernet/renesas/sh_eth.h
index c62380e34a1d..d050f37f3e0f 100644
--- a/drivers/net/ethernet/renesas/sh_eth.h
+++ b/drivers/net/ethernet/renesas/sh_eth.h
@@ -518,7 +518,6 @@ struct sh_eth_private {
 	/* MII transceiver section. */
 	u32 phy_id;			/* PHY ID */
 	struct mii_bus *mii_bus;	/* MDIO bus control */
-	struct phy_device *phydev;	/* PHY device control */
 	int link;
 	phy_interface_t phy_interface;
 	int msg_enable;
diff --git a/drivers/net/ethernet/rocker/rocker.h b/drivers/net/ethernet/rocker/rocker.h
index 1ab995f7146b..2eb9b49569d5 100644
--- a/drivers/net/ethernet/rocker/rocker.h
+++ b/drivers/net/ethernet/rocker/rocker.h
@@ -15,6 +15,7 @@
 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/netdevice.h>
+#include <linux/notifier.h>
 #include <net/neighbour.h>
 #include <net/switchdev.h>
 
@@ -52,6 +53,9 @@ struct rocker_port {
 	struct rocker_dma_ring_info rx_ring;
 };
 
+struct rocker_port *rocker_port_dev_lower_find(struct net_device *dev,
+					       struct rocker *rocker);
+
 struct rocker_world_ops;
 
 struct rocker {
@@ -66,6 +70,7 @@ struct rocker {
 	spinlock_t cmd_ring_lock;		/* for cmd ring accesses */
 	struct rocker_dma_ring_info cmd_ring;
 	struct rocker_dma_ring_info event_ring;
+	struct notifier_block fib_nb;
 	struct rocker_world_ops *wops;
 	void *wpriv;
 };
@@ -117,11 +122,6 @@ struct rocker_world_ops {
 	int (*port_obj_vlan_dump)(const struct rocker_port *rocker_port,
 				  struct switchdev_obj_port_vlan *vlan,
 				  switchdev_obj_dump_cb_t *cb);
-	int (*port_obj_fib4_add)(struct rocker_port *rocker_port,
-				 const struct switchdev_obj_ipv4_fib *fib4,
-				 struct switchdev_trans *trans);
-	int (*port_obj_fib4_del)(struct rocker_port *rocker_port,
-				 const struct switchdev_obj_ipv4_fib *fib4);
 	int (*port_obj_fdb_add)(struct rocker_port *rocker_port,
 				const struct switchdev_obj_port_fdb *fdb,
 				struct switchdev_trans *trans);
@@ -141,6 +141,11 @@ struct rocker_world_ops {
 	int (*port_ev_mac_vlan_seen)(struct rocker_port *rocker_port,
 				     const unsigned char *addr,
 				     __be16 vlan_id);
+	int (*fib4_add)(struct rocker *rocker,
+			const struct fib_entry_notifier_info *fen_info);
+	int (*fib4_del)(struct rocker *rocker,
+			const struct fib_entry_notifier_info *fen_info);
+	void (*fib4_abort)(struct rocker *rocker);
 };
 
 extern struct rocker_world_ops rocker_ofdpa_ops;
diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c
index f0b09b05ed3f..5424fb341613 100644
--- a/drivers/net/ethernet/rocker/rocker_main.c
+++ b/drivers/net/ethernet/rocker/rocker_main.c
@@ -1625,29 +1625,6 @@ rocker_world_port_obj_vlan_dump(const struct rocker_port *rocker_port,
 }
 
 static int
-rocker_world_port_obj_fib4_add(struct rocker_port *rocker_port,
-			       const struct switchdev_obj_ipv4_fib *fib4,
-			       struct switchdev_trans *trans)
-{
-	struct rocker_world_ops *wops = rocker_port->rocker->wops;
-
-	if (!wops->port_obj_fib4_add)
-		return -EOPNOTSUPP;
-	return wops->port_obj_fib4_add(rocker_port, fib4, trans);
-}
-
-static int
-rocker_world_port_obj_fib4_del(struct rocker_port *rocker_port,
-			       const struct switchdev_obj_ipv4_fib *fib4)
-{
-	struct rocker_world_ops *wops = rocker_port->rocker->wops;
-
-	if (!wops->port_obj_fib4_del)
-		return -EOPNOTSUPP;
-	return wops->port_obj_fib4_del(rocker_port, fib4);
-}
-
-static int
 rocker_world_port_obj_fdb_add(struct rocker_port *rocker_port,
 			      const struct switchdev_obj_port_fdb *fdb,
 			      struct switchdev_trans *trans)
@@ -1733,6 +1710,34 @@ static int rocker_world_port_ev_mac_vlan_seen(struct rocker_port *rocker_port,
 	return wops->port_ev_mac_vlan_seen(rocker_port, addr, vlan_id);
 }
 
+static int rocker_world_fib4_add(struct rocker *rocker,
+				 const struct fib_entry_notifier_info *fen_info)
+{
+	struct rocker_world_ops *wops = rocker->wops;
+
+	if (!wops->fib4_add)
+		return 0;
+	return wops->fib4_add(rocker, fen_info);
+}
+
+static int rocker_world_fib4_del(struct rocker *rocker,
+				 const struct fib_entry_notifier_info *fen_info)
+{
+	struct rocker_world_ops *wops = rocker->wops;
+
+	if (!wops->fib4_del)
+		return 0;
+	return wops->fib4_del(rocker, fen_info);
+}
+
+static void rocker_world_fib4_abort(struct rocker *rocker)
+{
+	struct rocker_world_ops *wops = rocker->wops;
+
+	if (wops->fib4_abort)
+		wops->fib4_abort(rocker);
+}
+
 /*****************
  * Net device ops
  *****************/
@@ -2096,11 +2101,6 @@ static int rocker_port_obj_add(struct net_device *dev,
 						     SWITCHDEV_OBJ_PORT_VLAN(obj),
 						     trans);
 		break;
-	case SWITCHDEV_OBJ_ID_IPV4_FIB:
-		err = rocker_world_port_obj_fib4_add(rocker_port,
-						     SWITCHDEV_OBJ_IPV4_FIB(obj),
-						     trans);
-		break;
 	case SWITCHDEV_OBJ_ID_PORT_FDB:
 		err = rocker_world_port_obj_fdb_add(rocker_port,
 						    SWITCHDEV_OBJ_PORT_FDB(obj),
@@ -2125,10 +2125,6 @@ static int rocker_port_obj_del(struct net_device *dev,
 		err = rocker_world_port_obj_vlan_del(rocker_port,
 						     SWITCHDEV_OBJ_PORT_VLAN(obj));
 		break;
-	case SWITCHDEV_OBJ_ID_IPV4_FIB:
-		err = rocker_world_port_obj_fib4_del(rocker_port,
-						     SWITCHDEV_OBJ_IPV4_FIB(obj));
-		break;
 	case SWITCHDEV_OBJ_ID_PORT_FDB:
 		err = rocker_world_port_obj_fdb_del(rocker_port,
 						    SWITCHDEV_OBJ_PORT_FDB(obj));
@@ -2175,6 +2171,31 @@ static const struct switchdev_ops rocker_port_switchdev_ops = {
 	.switchdev_port_obj_dump	= rocker_port_obj_dump,
 };
 
+static int rocker_router_fib_event(struct notifier_block *nb,
+				   unsigned long event, void *ptr)
+{
+	struct rocker *rocker = container_of(nb, struct rocker, fib_nb);
+	struct fib_entry_notifier_info *fen_info = ptr;
+	int err;
+
+	switch (event) {
+	case FIB_EVENT_ENTRY_ADD:
+		err = rocker_world_fib4_add(rocker, fen_info);
+		if (err)
+			rocker_world_fib4_abort(rocker);
+		else
+		break;
+	case FIB_EVENT_ENTRY_DEL:
+		rocker_world_fib4_del(rocker, fen_info);
+		break;
+	case FIB_EVENT_RULE_ADD: /* fall through */
+	case FIB_EVENT_RULE_DEL:
+		rocker_world_fib4_abort(rocker);
+		break;
+	}
+	return NOTIFY_DONE;
+}
+
 /********************
  * ethtool interface
  ********************/
@@ -2412,7 +2433,7 @@ static int rocker_port_rx_proc(const struct rocker *rocker,
 	skb->protocol = eth_type_trans(skb, rocker_port->dev);
 
 	if (rx_flags & ROCKER_RX_FLAGS_FWD_OFFLOAD)
-		skb->offload_fwd_mark = rocker_port->dev->offload_fwd_mark;
+		skb->offload_fwd_mark = 1;
 
 	rocker_port->dev->stats.rx_packets++;
 	rocker_port->dev->stats.rx_bytes += skb->len;
@@ -2740,6 +2761,9 @@ static int rocker_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto err_probe_ports;
 	}
 
+	rocker->fib_nb.notifier_call = rocker_router_fib_event;
+	register_fib_notifier(&rocker->fib_nb);
+
 	dev_info(&pdev->dev, "Rocker switch with id %*phN\n",
 		 (int)sizeof(rocker->hw.id), &rocker->hw.id);
 
@@ -2771,6 +2795,7 @@ static void rocker_remove(struct pci_dev *pdev)
 {
 	struct rocker *rocker = pci_get_drvdata(pdev);
 
+	unregister_fib_notifier(&rocker->fib_nb);
 	rocker_write32(rocker, CONTROL, ROCKER_CONTROL_RESET);
 	rocker_remove_ports(rocker);
 	free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT), rocker);
@@ -2799,6 +2824,37 @@ static bool rocker_port_dev_check(const struct net_device *dev)
 	return dev->netdev_ops == &rocker_port_netdev_ops;
 }
 
+static bool rocker_port_dev_check_under(const struct net_device *dev,
+					struct rocker *rocker)
+{
+	struct rocker_port *rocker_port;
+
+	if (!rocker_port_dev_check(dev))
+		return false;
+
+	rocker_port = netdev_priv(dev);
+	if (rocker_port->rocker != rocker)
+		return false;
+
+	return true;
+}
+
+struct rocker_port *rocker_port_dev_lower_find(struct net_device *dev,
+					       struct rocker *rocker)
+{
+	struct net_device *lower_dev;
+	struct list_head *iter;
+
+	if (rocker_port_dev_check_under(dev, rocker))
+		return netdev_priv(dev);
+
+	netdev_for_each_all_lower_dev(dev, lower_dev, iter) {
+		if (rocker_port_dev_check_under(lower_dev, rocker))
+			return netdev_priv(lower_dev);
+	}
+	return NULL;
+}
+
 static int rocker_netdevice_event(struct notifier_block *unused,
 				  unsigned long event, void *ptr)
 {
diff --git a/drivers/net/ethernet/rocker/rocker_ofdpa.c b/drivers/net/ethernet/rocker/rocker_ofdpa.c
index 1ca796316173..431a60804272 100644
--- a/drivers/net/ethernet/rocker/rocker_ofdpa.c
+++ b/drivers/net/ethernet/rocker/rocker_ofdpa.c
@@ -99,6 +99,7 @@ struct ofdpa_flow_tbl_entry {
 	struct ofdpa_flow_tbl_key key;
 	size_t key_len;
 	u32 key_crc32; /* key */
+	struct fib_info *fi;
 };
 
 struct ofdpa_group_tbl_entry {
@@ -189,6 +190,7 @@ struct ofdpa {
 	spinlock_t neigh_tbl_lock;		/* for neigh tbl accesses */
 	u32 neigh_tbl_next_index;
 	unsigned long ageing_time;
+	bool fib_aborted;
 };
 
 struct ofdpa_port {
@@ -1043,7 +1045,8 @@ static int ofdpa_flow_tbl_ucast4_routing(struct ofdpa_port *ofdpa_port,
 					 __be16 eth_type, __be32 dst,
 					 __be32 dst_mask, u32 priority,
 					 enum rocker_of_dpa_table_id goto_tbl,
-					 u32 group_id, int flags)
+					 u32 group_id, struct fib_info *fi,
+					 int flags)
 {
 	struct ofdpa_flow_tbl_entry *entry;
 
@@ -1060,6 +1063,7 @@ static int ofdpa_flow_tbl_ucast4_routing(struct ofdpa_port *ofdpa_port,
 	entry->key.ucast_routing.group_id = group_id;
 	entry->key_len = offsetof(struct ofdpa_flow_tbl_key,
 				  ucast_routing.group_id);
+	entry->fi = fi;
 
 	return ofdpa_flow_tbl_do(ofdpa_port, trans, flags, entry);
 }
@@ -1425,7 +1429,7 @@ static int ofdpa_port_ipv4_neigh(struct ofdpa_port *ofdpa_port,
 						    eth_type, ip_addr,
 						    inet_make_mask(32),
 						    priority, goto_tbl,
-						    group_id, flags);
+						    group_id, NULL, flags);
 
 		if (err)
 			netdev_err(ofdpa_port->dev, "Error (%d) /32 unicast route %pI4 group 0x%08x\n",
@@ -2390,7 +2394,7 @@ found:
 
 static int ofdpa_port_fib_ipv4(struct ofdpa_port *ofdpa_port,
 			       struct switchdev_trans *trans, __be32 dst,
-			       int dst_len, const struct fib_info *fi,
+			       int dst_len, struct fib_info *fi,
 			       u32 tb_id, int flags)
 {
 	const struct fib_nh *nh;
@@ -2426,7 +2430,7 @@ static int ofdpa_port_fib_ipv4(struct ofdpa_port *ofdpa_port,
 
 	err = ofdpa_flow_tbl_ucast4_routing(ofdpa_port, trans, eth_type, dst,
 					    dst_mask, priority, goto_tbl,
-					    group_id, flags);
+					    group_id, fi, flags);
 	if (err)
 		netdev_err(ofdpa_port->dev, "Error (%d) IPv4 route %pI4\n",
 			   err, &dst);
@@ -2558,7 +2562,6 @@ static int ofdpa_port_init(struct rocker_port *rocker_port)
 	struct ofdpa_port *ofdpa_port = rocker_port->wpriv;
 	int err;
 
-	switchdev_port_fwd_mark_set(ofdpa_port->dev, NULL, false);
 	rocker_port_set_learning(rocker_port,
 				 !!(ofdpa_port->brport_flags & BR_LEARNING));
 
@@ -2719,28 +2722,6 @@ static int ofdpa_port_obj_vlan_dump(const struct rocker_port *rocker_port,
 	return err;
 }
 
-static int ofdpa_port_obj_fib4_add(struct rocker_port *rocker_port,
-				   const struct switchdev_obj_ipv4_fib *fib4,
-				   struct switchdev_trans *trans)
-{
-	struct ofdpa_port *ofdpa_port = rocker_port->wpriv;
-
-	return ofdpa_port_fib_ipv4(ofdpa_port, trans,
-				   htonl(fib4->dst), fib4->dst_len,
-				   fib4->fi, fib4->tb_id, 0);
-}
-
-static int ofdpa_port_obj_fib4_del(struct rocker_port *rocker_port,
-				   const struct switchdev_obj_ipv4_fib *fib4)
-{
-	struct ofdpa_port *ofdpa_port = rocker_port->wpriv;
-
-	return ofdpa_port_fib_ipv4(ofdpa_port, NULL,
-				   htonl(fib4->dst), fib4->dst_len,
-				   fib4->fi, fib4->tb_id,
-				   OFDPA_OP_FLAG_REMOVE);
-}
-
 static int ofdpa_port_obj_fdb_add(struct rocker_port *rocker_port,
 				  const struct switchdev_obj_port_fdb *fdb,
 				  struct switchdev_trans *trans)
@@ -2817,7 +2798,6 @@ static int ofdpa_port_bridge_join(struct ofdpa_port *ofdpa_port,
 		ofdpa_port_internal_vlan_id_get(ofdpa_port, bridge->ifindex);
 
 	ofdpa_port->bridge_dev = bridge;
-	switchdev_port_fwd_mark_set(ofdpa_port->dev, bridge, true);
 
 	return ofdpa_port_vlan_add(ofdpa_port, NULL, OFDPA_UNTAGGED_VID, 0);
 }
@@ -2836,8 +2816,6 @@ static int ofdpa_port_bridge_leave(struct ofdpa_port *ofdpa_port)
 		ofdpa_port_internal_vlan_id_get(ofdpa_port,
 						ofdpa_port->dev->ifindex);
 
-	switchdev_port_fwd_mark_set(ofdpa_port->dev, ofdpa_port->bridge_dev,
-				    false);
 	ofdpa_port->bridge_dev = NULL;
 
 	err = ofdpa_port_vlan_add(ofdpa_port, NULL, OFDPA_UNTAGGED_VID, 0);
@@ -2926,6 +2904,82 @@ static int ofdpa_port_ev_mac_vlan_seen(struct rocker_port *rocker_port,
 	return ofdpa_port_fdb(ofdpa_port, NULL, addr, vlan_id, flags);
 }
 
+static struct ofdpa_port *ofdpa_port_dev_lower_find(struct net_device *dev,
+						    struct rocker *rocker)
+{
+	struct rocker_port *rocker_port;
+
+	rocker_port = rocker_port_dev_lower_find(dev, rocker);
+	return rocker_port ? rocker_port->wpriv : NULL;
+}
+
+static int ofdpa_fib4_add(struct rocker *rocker,
+			  const struct fib_entry_notifier_info *fen_info)
+{
+	struct ofdpa *ofdpa = rocker->wpriv;
+	struct ofdpa_port *ofdpa_port;
+	int err;
+
+	if (ofdpa->fib_aborted)
+		return 0;
+	ofdpa_port = ofdpa_port_dev_lower_find(fen_info->fi->fib_dev, rocker);
+	if (!ofdpa_port)
+		return 0;
+	err = ofdpa_port_fib_ipv4(ofdpa_port, NULL, htonl(fen_info->dst),
+				  fen_info->dst_len, fen_info->fi,
+				  fen_info->tb_id, 0);
+	if (err)
+		return err;
+	fib_info_offload_inc(fen_info->fi);
+	return 0;
+}
+
+static int ofdpa_fib4_del(struct rocker *rocker,
+			  const struct fib_entry_notifier_info *fen_info)
+{
+	struct ofdpa *ofdpa = rocker->wpriv;
+	struct ofdpa_port *ofdpa_port;
+
+	if (ofdpa->fib_aborted)
+		return 0;
+	ofdpa_port = ofdpa_port_dev_lower_find(fen_info->fi->fib_dev, rocker);
+	if (!ofdpa_port)
+		return 0;
+	fib_info_offload_dec(fen_info->fi);
+	return ofdpa_port_fib_ipv4(ofdpa_port, NULL, htonl(fen_info->dst),
+				   fen_info->dst_len, fen_info->fi,
+				   fen_info->tb_id, OFDPA_OP_FLAG_REMOVE);
+}
+
+static void ofdpa_fib4_abort(struct rocker *rocker)
+{
+	struct ofdpa *ofdpa = rocker->wpriv;
+	struct ofdpa_port *ofdpa_port;
+	struct ofdpa_flow_tbl_entry *flow_entry;
+	struct hlist_node *tmp;
+	unsigned long flags;
+	int bkt;
+
+	if (ofdpa->fib_aborted)
+		return;
+
+	spin_lock_irqsave(&ofdpa->flow_tbl_lock, flags);
+	hash_for_each_safe(ofdpa->flow_tbl, bkt, tmp, flow_entry, entry) {
+		if (flow_entry->key.tbl_id !=
+		    ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING)
+			continue;
+		ofdpa_port = ofdpa_port_dev_lower_find(flow_entry->fi->fib_dev,
+						       rocker);
+		if (!ofdpa_port)
+			continue;
+		fib_info_offload_dec(flow_entry->fi);
+		ofdpa_flow_tbl_del(ofdpa_port, NULL, OFDPA_OP_FLAG_REMOVE,
+				   flow_entry);
+	}
+	spin_unlock_irqrestore(&ofdpa->flow_tbl_lock, flags);
+	ofdpa->fib_aborted = true;
+}
+
 struct rocker_world_ops rocker_ofdpa_ops = {
 	.kind = "ofdpa",
 	.priv_size = sizeof(struct ofdpa),
@@ -2945,8 +2999,6 @@ struct rocker_world_ops rocker_ofdpa_ops = {
 	.port_obj_vlan_add = ofdpa_port_obj_vlan_add,
 	.port_obj_vlan_del = ofdpa_port_obj_vlan_del,
 	.port_obj_vlan_dump = ofdpa_port_obj_vlan_dump,
-	.port_obj_fib4_add = ofdpa_port_obj_fib4_add,
-	.port_obj_fib4_del = ofdpa_port_obj_fib4_del,
 	.port_obj_fdb_add = ofdpa_port_obj_fdb_add,
 	.port_obj_fdb_del = ofdpa_port_obj_fdb_del,
 	.port_obj_fdb_dump = ofdpa_port_obj_fdb_dump,
@@ -2955,4 +3007,7 @@ struct rocker_world_ops rocker_ofdpa_ops = {
 	.port_neigh_update = ofdpa_port_neigh_update,
 	.port_neigh_destroy = ofdpa_port_neigh_destroy,
 	.port_ev_mac_vlan_seen = ofdpa_port_ev_mac_vlan_seen,
+	.fib4_add = ofdpa_fib4_add,
+	.fib4_del = ofdpa_fib4_del,
+	.fib4_abort = ofdpa_fib4_abort,
 };
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index f658fee74f18..00279da6a1e8 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -177,7 +177,7 @@ static int efx_ef10_get_vf_index(struct efx_nic *efx)
 
 static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
 {
-	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_OUT_LEN);
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_V2_OUT_LEN);
 	struct efx_ef10_nic_data *nic_data = efx->nic_data;
 	size_t outlen;
 	int rc;
@@ -188,7 +188,7 @@ static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
 			  outbuf, sizeof(outbuf), &outlen);
 	if (rc)
 		return rc;
-	if (outlen < sizeof(outbuf)) {
+	if (outlen < MC_CMD_GET_CAPABILITIES_OUT_LEN) {
 		netif_err(efx, drv, efx->net_dev,
 			  "unable to read datapath firmware capabilities\n");
 		return -EIO;
@@ -197,6 +197,12 @@ static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
 	nic_data->datapath_caps =
 		MCDI_DWORD(outbuf, GET_CAPABILITIES_OUT_FLAGS1);
 
+	if (outlen >= MC_CMD_GET_CAPABILITIES_V2_OUT_LEN)
+		nic_data->datapath_caps2 = MCDI_DWORD(outbuf,
+				GET_CAPABILITIES_V2_OUT_FLAGS2);
+	else
+		nic_data->datapath_caps2 = 0;
+
 	/* record the DPCPU firmware IDs to determine VEB vswitching support.
 	 */
 	nic_data->rx_dpcpu_fw_id =
@@ -227,6 +233,116 @@ static int efx_ef10_get_sysclk_freq(struct efx_nic *efx)
 	return rc > 0 ? rc : -ERANGE;
 }
 
+static int efx_ef10_get_timer_workarounds(struct efx_nic *efx)
+{
+	struct efx_ef10_nic_data *nic_data = efx->nic_data;
+	unsigned int implemented;
+	unsigned int enabled;
+	int rc;
+
+	nic_data->workaround_35388 = false;
+	nic_data->workaround_61265 = false;
+
+	rc = efx_mcdi_get_workarounds(efx, &implemented, &enabled);
+
+	if (rc == -ENOSYS) {
+		/* Firmware without GET_WORKAROUNDS - not a problem. */
+		rc = 0;
+	} else if (rc == 0) {
+		/* Bug61265 workaround is always enabled if implemented. */
+		if (enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG61265)
+			nic_data->workaround_61265 = true;
+
+		if (enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG35388) {
+			nic_data->workaround_35388 = true;
+		} else if (implemented & MC_CMD_GET_WORKAROUNDS_OUT_BUG35388) {
+			/* Workaround is implemented but not enabled.
+			 * Try to enable it.
+			 */
+			rc = efx_mcdi_set_workaround(efx,
+						     MC_CMD_WORKAROUND_BUG35388,
+						     true, NULL);
+			if (rc == 0)
+				nic_data->workaround_35388 = true;
+			/* If we failed to set the workaround just carry on. */
+			rc = 0;
+		}
+	}
+
+	netif_dbg(efx, probe, efx->net_dev,
+		  "workaround for bug 35388 is %sabled\n",
+		  nic_data->workaround_35388 ? "en" : "dis");
+	netif_dbg(efx, probe, efx->net_dev,
+		  "workaround for bug 61265 is %sabled\n",
+		  nic_data->workaround_61265 ? "en" : "dis");
+
+	return rc;
+}
+
+static void efx_ef10_process_timer_config(struct efx_nic *efx,
+					  const efx_dword_t *data)
+{
+	unsigned int max_count;
+
+	if (EFX_EF10_WORKAROUND_61265(efx)) {
+		efx->timer_quantum_ns = MCDI_DWORD(data,
+			GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_STEP_NS);
+		efx->timer_max_ns = MCDI_DWORD(data,
+			GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_MAX_NS);
+	} else if (EFX_EF10_WORKAROUND_35388(efx)) {
+		efx->timer_quantum_ns = MCDI_DWORD(data,
+			GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_NS_PER_COUNT);
+		max_count = MCDI_DWORD(data,
+			GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_MAX_COUNT);
+		efx->timer_max_ns = max_count * efx->timer_quantum_ns;
+	} else {
+		efx->timer_quantum_ns = MCDI_DWORD(data,
+			GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_NS_PER_COUNT);
+		max_count = MCDI_DWORD(data,
+			GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_MAX_COUNT);
+		efx->timer_max_ns = max_count * efx->timer_quantum_ns;
+	}
+
+	netif_dbg(efx, probe, efx->net_dev,
+		  "got timer properties from MC: quantum %u ns; max %u ns\n",
+		  efx->timer_quantum_ns, efx->timer_max_ns);
+}
+
+static int efx_ef10_get_timer_config(struct efx_nic *efx)
+{
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN);
+	int rc;
+
+	rc = efx_ef10_get_timer_workarounds(efx);
+	if (rc)
+		return rc;
+
+	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_GET_EVQ_TMR_PROPERTIES, NULL, 0,
+				outbuf, sizeof(outbuf), NULL);
+
+	if (rc == 0) {
+		efx_ef10_process_timer_config(efx, outbuf);
+	} else if (rc == -ENOSYS || rc == -EPERM) {
+		/* Not available - fall back to Huntington defaults. */
+		unsigned int quantum;
+
+		rc = efx_ef10_get_sysclk_freq(efx);
+		if (rc < 0)
+			return rc;
+
+		quantum = 1536000 / rc; /* 1536 cycles */
+		efx->timer_quantum_ns = quantum;
+		efx->timer_max_ns = efx->type->timer_period_max * quantum;
+		rc = 0;
+	} else {
+		efx_mcdi_display_error(efx, MC_CMD_GET_EVQ_TMR_PROPERTIES,
+				       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN,
+				       NULL, 0, rc);
+	}
+
+	return rc;
+}
+
 static int efx_ef10_get_mac_address_pf(struct efx_nic *efx, u8 *mac_address)
 {
 	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_MAC_ADDRESSES_OUT_LEN);
@@ -527,32 +643,9 @@ static int efx_ef10_probe(struct efx_nic *efx)
 	if (rc)
 		goto fail5;
 
-	rc = efx_ef10_get_sysclk_freq(efx);
+	rc = efx_ef10_get_timer_config(efx);
 	if (rc < 0)
 		goto fail5;
-	efx->timer_quantum_ns = 1536000 / rc; /* 1536 cycles */
-
-	/* Check whether firmware supports bug 35388 workaround.
-	 * First try to enable it, then if we get EPERM, just
-	 * ask if it's already enabled
-	 */
-	rc = efx_mcdi_set_workaround(efx, MC_CMD_WORKAROUND_BUG35388, true, NULL);
-	if (rc == 0) {
-		nic_data->workaround_35388 = true;
-	} else if (rc == -EPERM) {
-		unsigned int enabled;
-
-		rc = efx_mcdi_get_workarounds(efx, NULL, &enabled);
-		if (rc)
-			goto fail3;
-		nic_data->workaround_35388 = enabled &
-			MC_CMD_GET_WORKAROUNDS_OUT_BUG35388;
-	} else if (rc != -ENOSYS && rc != -ENOENT) {
-		goto fail5;
-	}
-	netif_dbg(efx, probe, efx->net_dev,
-		  "workaround for bug 35388 is %sabled\n",
-		  nic_data->workaround_35388 ? "en" : "dis");
 
 	rc = efx_mcdi_mon_probe(efx);
 	if (rc && rc != -EPERM)
@@ -1440,9 +1533,10 @@ static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = {
 			       (1ULL << GENERIC_STAT_rx_nodesc_trunc) |	\
 			       (1ULL << GENERIC_STAT_rx_noskb_drops))
 
-/* These statistics are only provided by the 10G MAC.  For a 10G/40G
- * switchable port we do not expose these because they might not
- * include all the packets they should.
+/* On 7000 series NICs, these statistics are only provided by the 10G MAC.
+ * For a 10G/40G switchable port we do not expose these because they might
+ * not include all the packets they should.
+ * On 8000 series NICs these statistics are always provided.
  */
 #define HUNT_10G_ONLY_STAT_MASK ((1ULL << EF10_STAT_port_tx_control) |	\
 				 (1ULL << EF10_STAT_port_tx_lt64) |	\
@@ -1488,10 +1582,15 @@ static u64 efx_ef10_raw_stat_mask(struct efx_nic *efx)
 	      1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL))
 		return 0;
 
-	if (port_caps & (1 << MC_CMD_PHY_CAP_40000FDX_LBN))
+	if (port_caps & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) {
 		raw_mask |= HUNT_40G_EXTRA_STAT_MASK;
-	else
+		/* 8000 series have everything even at 40G */
+		if (nic_data->datapath_caps2 &
+		    (1 << MC_CMD_GET_CAPABILITIES_V2_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN))
+			raw_mask |= HUNT_10G_ONLY_STAT_MASK;
+	} else {
 		raw_mask |= HUNT_10G_ONLY_STAT_MASK;
+	}
 
 	if (nic_data->datapath_caps &
 	    (1 << MC_CMD_GET_CAPABILITIES_OUT_PM_AND_RXDP_COUNTERS_LBN))
@@ -1517,13 +1616,14 @@ static void efx_ef10_get_stat_mask(struct efx_nic *efx, unsigned long *mask)
 	}
 
 #if BITS_PER_LONG == 64
+	BUILD_BUG_ON(BITS_TO_LONGS(EF10_STAT_COUNT) != 2);
 	mask[0] = raw_mask[0];
 	mask[1] = raw_mask[1];
 #else
+	BUILD_BUG_ON(BITS_TO_LONGS(EF10_STAT_COUNT) != 3);
 	mask[0] = raw_mask[0] & 0xffffffff;
 	mask[1] = raw_mask[0] >> 32;
 	mask[2] = raw_mask[1] & 0xffffffff;
-	mask[3] = raw_mask[1] >> 32;
 #endif
 }
 
@@ -1616,7 +1716,6 @@ static int efx_ef10_try_update_nic_stats_pf(struct efx_nic *efx)
 	efx_ef10_get_stat_mask(efx, mask);
 
 	dma_stats = efx->stats_buffer.addr;
-	nic_data = efx->nic_data;
 
 	generation_end = dma_stats[MC_CMD_MAC_GENERATION_END];
 	if (generation_end == EFX_MC_STATS_GENERATION_INVALID)
@@ -1743,27 +1842,43 @@ static size_t efx_ef10_update_stats_vf(struct efx_nic *efx, u64 *full_stats,
 static void efx_ef10_push_irq_moderation(struct efx_channel *channel)
 {
 	struct efx_nic *efx = channel->efx;
-	unsigned int mode, value;
+	unsigned int mode, usecs;
 	efx_dword_t timer_cmd;
 
-	if (channel->irq_moderation) {
+	if (channel->irq_moderation_us) {
 		mode = 3;
-		value = channel->irq_moderation - 1;
+		usecs = channel->irq_moderation_us;
 	} else {
 		mode = 0;
-		value = 0;
+		usecs = 0;
 	}
 
-	if (EFX_EF10_WORKAROUND_35388(efx)) {
+	if (EFX_EF10_WORKAROUND_61265(efx)) {
+		MCDI_DECLARE_BUF(inbuf, MC_CMD_SET_EVQ_TMR_IN_LEN);
+		unsigned int ns = usecs * 1000;
+
+		MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_INSTANCE,
+			       channel->channel);
+		MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_LOAD_REQ_NS, ns);
+		MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_RELOAD_REQ_NS, ns);
+		MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_MODE, mode);
+
+		efx_mcdi_rpc_async(efx, MC_CMD_SET_EVQ_TMR,
+				   inbuf, sizeof(inbuf), 0, NULL, 0);
+	} else if (EFX_EF10_WORKAROUND_35388(efx)) {
+		unsigned int ticks = efx_usecs_to_ticks(efx, usecs);
+
 		EFX_POPULATE_DWORD_3(timer_cmd, ERF_DD_EVQ_IND_TIMER_FLAGS,
 				     EFE_DD_EVQ_IND_TIMER_FLAGS,
 				     ERF_DD_EVQ_IND_TIMER_MODE, mode,
-				     ERF_DD_EVQ_IND_TIMER_VAL, value);
+				     ERF_DD_EVQ_IND_TIMER_VAL, ticks);
 		efx_writed_page(efx, &timer_cmd, ER_DD_EVQ_INDIRECT,
 				channel->channel);
 	} else {
+		unsigned int ticks = efx_usecs_to_ticks(efx, usecs);
+
 		EFX_POPULATE_DWORD_2(timer_cmd, ERF_DZ_TC_TIMER_MODE, mode,
-				     ERF_DZ_TC_TIMER_VAL, value);
+				     ERF_DZ_TC_TIMER_VAL, ticks);
 		efx_writed_page(efx, &timer_cmd, ER_DZ_EVQ_TMR,
 				channel->channel);
 	}
@@ -1934,14 +2049,18 @@ static irqreturn_t efx_ef10_legacy_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static void efx_ef10_irq_test_generate(struct efx_nic *efx)
+static int efx_ef10_irq_test_generate(struct efx_nic *efx)
 {
 	MCDI_DECLARE_BUF(inbuf, MC_CMD_TRIGGER_INTERRUPT_IN_LEN);
 
+	if (efx_mcdi_set_workaround(efx, MC_CMD_WORKAROUND_BUG41750, true,
+				    NULL) == 0)
+		return -ENOTSUPP;
+
 	BUILD_BUG_ON(MC_CMD_TRIGGER_INTERRUPT_OUT_LEN != 0);
 
 	MCDI_SET_DWORD(inbuf, TRIGGER_INTERRUPT_IN_INTR_LEVEL, efx->irq_level);
-	(void) efx_mcdi_rpc(efx, MC_CMD_TRIGGER_INTERRUPT,
+	return efx_mcdi_rpc(efx, MC_CMD_TRIGGER_INTERRUPT,
 			    inbuf, sizeof(inbuf), NULL, 0, NULL);
 }
 
@@ -2535,13 +2654,12 @@ fail:
 static int efx_ef10_ev_init(struct efx_channel *channel)
 {
 	MCDI_DECLARE_BUF(inbuf,
-			 MC_CMD_INIT_EVQ_IN_LEN(EFX_MAX_EVQ_SIZE * 8 /
-						EFX_BUF_SIZE));
-	MCDI_DECLARE_BUF(outbuf, MC_CMD_INIT_EVQ_OUT_LEN);
+			 MC_CMD_INIT_EVQ_V2_IN_LEN(EFX_MAX_EVQ_SIZE * 8 /
+						   EFX_BUF_SIZE));
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_INIT_EVQ_V2_OUT_LEN);
 	size_t entries = channel->eventq.buf.len / EFX_BUF_SIZE;
 	struct efx_nic *efx = channel->efx;
 	struct efx_ef10_nic_data *nic_data;
-	bool supports_rx_merge;
 	size_t inlen, outlen;
 	unsigned int enabled, implemented;
 	dma_addr_t dma_addr;
@@ -2549,9 +2667,6 @@ static int efx_ef10_ev_init(struct efx_channel *channel)
 	int i;
 
 	nic_data = efx->nic_data;
-	supports_rx_merge =
-		!!(nic_data->datapath_caps &
-		   1 << MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN);
 
 	/* Fill event queue with all ones (i.e. empty events) */
 	memset(channel->eventq.buf.addr, 0xff, channel->eventq.buf.len);
@@ -2560,11 +2675,6 @@ static int efx_ef10_ev_init(struct efx_channel *channel)
 	MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_INSTANCE, channel->channel);
 	/* INIT_EVQ expects index in vector table, not absolute */
 	MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_IRQ_NUM, channel->channel);
-	MCDI_POPULATE_DWORD_4(inbuf, INIT_EVQ_IN_FLAGS,
-			      INIT_EVQ_IN_FLAG_INTERRUPTING, 1,
-			      INIT_EVQ_IN_FLAG_RX_MERGE, 1,
-			      INIT_EVQ_IN_FLAG_TX_MERGE, 1,
-			      INIT_EVQ_IN_FLAG_CUT_THRU, !supports_rx_merge);
 	MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_TMR_MODE,
 		       MC_CMD_INIT_EVQ_IN_TMR_MODE_DIS);
 	MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_TMR_LOAD, 0);
@@ -2573,6 +2683,27 @@ static int efx_ef10_ev_init(struct efx_channel *channel)
 		       MC_CMD_INIT_EVQ_IN_COUNT_MODE_DIS);
 	MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_COUNT_THRSHLD, 0);
 
+	if (nic_data->datapath_caps2 &
+	    1 << MC_CMD_GET_CAPABILITIES_V2_OUT_INIT_EVQ_V2_LBN) {
+		/* Use the new generic approach to specifying event queue
+		 * configuration, requesting lower latency or higher throughput.
+		 * The options that actually get used appear in the output.
+		 */
+		MCDI_POPULATE_DWORD_2(inbuf, INIT_EVQ_V2_IN_FLAGS,
+				      INIT_EVQ_V2_IN_FLAG_INTERRUPTING, 1,
+				      INIT_EVQ_V2_IN_FLAG_TYPE,
+				      MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_AUTO);
+	} else {
+		bool cut_thru = !(nic_data->datapath_caps &
+			1 << MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN);
+
+		MCDI_POPULATE_DWORD_4(inbuf, INIT_EVQ_IN_FLAGS,
+				      INIT_EVQ_IN_FLAG_INTERRUPTING, 1,
+				      INIT_EVQ_IN_FLAG_RX_MERGE, 1,
+				      INIT_EVQ_IN_FLAG_TX_MERGE, 1,
+				      INIT_EVQ_IN_FLAG_CUT_THRU, cut_thru);
+	}
+
 	dma_addr = channel->eventq.buf.dma_addr;
 	for (i = 0; i < entries; ++i) {
 		MCDI_SET_ARRAY_QWORD(inbuf, INIT_EVQ_IN_DMA_ADDR, i, dma_addr);
@@ -2583,6 +2714,13 @@ static int efx_ef10_ev_init(struct efx_channel *channel)
 
 	rc = efx_mcdi_rpc(efx, MC_CMD_INIT_EVQ, inbuf, inlen,
 			  outbuf, sizeof(outbuf), &outlen);
+
+	if (outlen >= MC_CMD_INIT_EVQ_V2_OUT_LEN)
+		netif_dbg(efx, drv, efx->net_dev,
+			  "Channel %d using event queue flags %08x\n",
+			  channel->channel,
+			  MCDI_DWORD(outbuf, INIT_EVQ_V2_OUT_FLAGS));
+
 	/* IRQ return is ignored */
 	if (channel->channel || rc)
 		return rc;
@@ -2590,8 +2728,8 @@ static int efx_ef10_ev_init(struct efx_channel *channel)
 	/* Successfully created event queue on channel 0 */
 	rc = efx_mcdi_get_workarounds(efx, &implemented, &enabled);
 	if (rc == -ENOSYS) {
-		/* GET_WORKAROUNDS was implemented before the bug26807
-		 * workaround, thus the latter must be unavailable in this fw
+		/* GET_WORKAROUNDS was implemented before this workaround,
+		 * thus it must be unavailable in this firmware.
 		 */
 		nic_data->workaround_26807 = false;
 		rc = 0;
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 14b821b1c880..3cf3557106c2 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -281,6 +281,27 @@ static int efx_process_channel(struct efx_channel *channel, int budget)
  * NAPI guarantees serialisation of polls of the same device, which
  * provides the guarantee required by efx_process_channel().
  */
+static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
+{
+	int step = efx->irq_mod_step_us;
+
+	if (channel->irq_mod_score < irq_adapt_low_thresh) {
+		if (channel->irq_moderation_us > step) {
+			channel->irq_moderation_us -= step;
+			efx->type->push_irq_moderation(channel);
+		}
+	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {
+		if (channel->irq_moderation_us <
+		    efx->irq_rx_moderation_us) {
+			channel->irq_moderation_us += step;
+			efx->type->push_irq_moderation(channel);
+		}
+	}
+
+	channel->irq_count = 0;
+	channel->irq_mod_score = 0;
+}
+
 static int efx_poll(struct napi_struct *napi, int budget)
 {
 	struct efx_channel *channel =
@@ -301,22 +322,7 @@ static int efx_poll(struct napi_struct *napi, int budget)
 		if (efx_channel_has_rx_queue(channel) &&
 		    efx->irq_rx_adaptive &&
 		    unlikely(++channel->irq_count == 1000)) {
-			if (unlikely(channel->irq_mod_score <
-				     irq_adapt_low_thresh)) {
-				if (channel->irq_moderation > 1) {
-					channel->irq_moderation -= 1;
-					efx->type->push_irq_moderation(channel);
-				}
-			} else if (unlikely(channel->irq_mod_score >
-					    irq_adapt_high_thresh)) {
-				if (channel->irq_moderation <
-				    efx->irq_rx_moderation) {
-					channel->irq_moderation += 1;
-					efx->type->push_irq_moderation(channel);
-				}
-			}
-			channel->irq_count = 0;
-			channel->irq_mod_score = 0;
+			efx_update_irq_mod(efx, channel);
 		}
 
 		efx_filter_rfs_expire(channel);
@@ -1703,6 +1709,7 @@ static int efx_probe_nic(struct efx_nic *efx)
 	netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
 
 	/* Initialise the interrupt moderation settings */
+	efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000);
 	efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true,
 				true);
 
@@ -1949,14 +1956,21 @@ static void efx_remove_all(struct efx_nic *efx)
  * Interrupt moderation
  *
  **************************************************************************/
-
-static unsigned int irq_mod_ticks(unsigned int usecs, unsigned int quantum_ns)
+unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs)
 {
 	if (usecs == 0)
 		return 0;
-	if (usecs * 1000 < quantum_ns)
+	if (usecs * 1000 < efx->timer_quantum_ns)
 		return 1; /* never round down to 0 */
-	return usecs * 1000 / quantum_ns;
+	return usecs * 1000 / efx->timer_quantum_ns;
+}
+
+unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks)
+{
+	/* We must round up when converting ticks to microseconds
+	 * because we round down when converting the other way.
+	 */
+	return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000);
 }
 
 /* Set interrupt moderation parameters */
@@ -1965,21 +1979,16 @@ int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
 			    bool rx_may_override_tx)
 {
 	struct efx_channel *channel;
-	unsigned int irq_mod_max = DIV_ROUND_UP(efx->type->timer_period_max *
-						efx->timer_quantum_ns,
-						1000);
-	unsigned int tx_ticks;
-	unsigned int rx_ticks;
+	unsigned int timer_max_us;
 
 	EFX_ASSERT_RESET_SERIALISED(efx);
 
-	if (tx_usecs > irq_mod_max || rx_usecs > irq_mod_max)
-		return -EINVAL;
+	timer_max_us = efx->timer_max_ns / 1000;
 
-	tx_ticks = irq_mod_ticks(tx_usecs, efx->timer_quantum_ns);
-	rx_ticks = irq_mod_ticks(rx_usecs, efx->timer_quantum_ns);
+	if (tx_usecs > timer_max_us || rx_usecs > timer_max_us)
+		return -EINVAL;
 
-	if (tx_ticks != rx_ticks && efx->tx_channel_offset == 0 &&
+	if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 &&
 	    !rx_may_override_tx) {
 		netif_err(efx, drv, efx->net_dev, "Channels are shared. "
 			  "RX and TX IRQ moderation must be equal\n");
@@ -1987,12 +1996,12 @@ int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
 	}
 
 	efx->irq_rx_adaptive = rx_adaptive;
-	efx->irq_rx_moderation = rx_ticks;
+	efx->irq_rx_moderation_us = rx_usecs;
 	efx_for_each_channel(channel, efx) {
 		if (efx_channel_has_rx_queue(channel))
-			channel->irq_moderation = rx_ticks;
+			channel->irq_moderation_us = rx_usecs;
 		else if (efx_channel_has_tx_queues(channel))
-			channel->irq_moderation = tx_ticks;
+			channel->irq_moderation_us = tx_usecs;
 	}
 
 	return 0;
@@ -2001,26 +2010,21 @@ int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
 			    unsigned int *rx_usecs, bool *rx_adaptive)
 {
-	/* We must round up when converting ticks to microseconds
-	 * because we round down when converting the other way.
-	 */
-
 	*rx_adaptive = efx->irq_rx_adaptive;
-	*rx_usecs = DIV_ROUND_UP(efx->irq_rx_moderation *
-				 efx->timer_quantum_ns,
-				 1000);
+	*rx_usecs = efx->irq_rx_moderation_us;
 
 	/* If channels are shared between RX and TX, so is IRQ
 	 * moderation.  Otherwise, IRQ moderation is the same for all
 	 * TX channels and is not adaptive.
 	 */
-	if (efx->tx_channel_offset == 0)
+	if (efx->tx_channel_offset == 0) {
 		*tx_usecs = *rx_usecs;
-	else
-		*tx_usecs = DIV_ROUND_UP(
-			efx->channel[efx->tx_channel_offset]->irq_moderation *
-			efx->timer_quantum_ns,
-			1000);
+	} else {
+		struct efx_channel *tx_channel;
+
+		tx_channel = efx->channel[efx->tx_channel_offset];
+		*tx_usecs = tx_channel->irq_moderation_us;
+	}
 }
 
 /**************************************************************************
@@ -2259,8 +2263,18 @@ static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
 	rc = efx_check_disabled(efx);
 	if (rc)
 		return rc;
-	if (new_mtu > EFX_MAX_MTU)
+	if (new_mtu > EFX_MAX_MTU) {
+		netif_err(efx, drv, efx->net_dev,
+			  "Requested MTU of %d too big (max: %d)\n",
+			  new_mtu, EFX_MAX_MTU);
 		return -EINVAL;
+	}
+	if (new_mtu < EFX_MIN_MTU) {
+		netif_err(efx, drv, efx->net_dev,
+			  "Requested MTU of %d too small (min: %d)\n",
+			  new_mtu, EFX_MIN_MTU);
+		return -EINVAL;
+	}
 
 	netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
 
diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h
index c3ae739e9c7a..342ae16e1f2d 100644
--- a/drivers/net/ethernet/sfc/efx.h
+++ b/drivers/net/ethernet/sfc/efx.h
@@ -204,6 +204,8 @@ int efx_try_recovery(struct efx_nic *efx);
 
 /* Global */
 void efx_schedule_reset(struct efx_nic *efx, enum reset_type type);
+unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs);
+unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks);
 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
 			    unsigned int rx_usecs, bool rx_adaptive,
 			    bool rx_may_override_tx);
diff --git a/drivers/net/ethernet/sfc/falcon.c b/drivers/net/ethernet/sfc/falcon.c
index d790cb8d9db3..1a7092602aec 100644
--- a/drivers/net/ethernet/sfc/falcon.c
+++ b/drivers/net/ethernet/sfc/falcon.c
@@ -378,12 +378,15 @@ static void falcon_push_irq_moderation(struct efx_channel *channel)
 	struct efx_nic *efx = channel->efx;
 
 	/* Set timer register */
-	if (channel->irq_moderation) {
+	if (channel->irq_moderation_us) {
+		unsigned int ticks;
+
+		ticks = efx_usecs_to_ticks(efx, channel->irq_moderation_us);
 		EFX_POPULATE_DWORD_2(timer_cmd,
 				     FRF_AB_TC_TIMER_MODE,
 				     FFE_BB_TIMER_MODE_INT_HLDOFF,
 				     FRF_AB_TC_TIMER_VAL,
-				     channel->irq_moderation - 1);
+				     ticks - 1);
 	} else {
 		EFX_POPULATE_DWORD_2(timer_cmd,
 				     FRF_AB_TC_TIMER_MODE,
@@ -2373,6 +2376,8 @@ static int falcon_probe_nic(struct efx_nic *efx)
 			     EFX_MAX_CHANNELS);
 	efx->max_tx_channels = efx->max_channels;
 	efx->timer_quantum_ns = 4968; /* 621 cycles */
+	efx->timer_max_ns = efx->type->timer_period_max *
+			    efx->timer_quantum_ns;
 
 	/* Initialise I2C adapter */
 	board = falcon_board(efx);
diff --git a/drivers/net/ethernet/sfc/falcon_boards.c b/drivers/net/ethernet/sfc/falcon_boards.c
index 1736f4b806af..f6883b2b5da3 100644
--- a/drivers/net/ethernet/sfc/falcon_boards.c
+++ b/drivers/net/ethernet/sfc/falcon_boards.c
@@ -64,7 +64,7 @@
 #define LM87_ALARM_TEMP_INT		0x10
 #define LM87_ALARM_TEMP_EXT1		0x20
 
-#if defined(CONFIG_SENSORS_LM87) || defined(CONFIG_SENSORS_LM87_MODULE)
+#if IS_ENABLED(CONFIG_SENSORS_LM87)
 
 static int efx_poke_lm87(struct i2c_client *client, const u8 *reg_values)
 {
@@ -455,7 +455,7 @@ static int sfe4001_init(struct efx_nic *efx)
 	struct falcon_board *board = falcon_board(efx);
 	int rc;
 
-#if defined(CONFIG_SENSORS_LM90) || defined(CONFIG_SENSORS_LM90_MODULE)
+#if IS_ENABLED(CONFIG_SENSORS_LM90)
 	board->hwmon_client =
 		i2c_new_device(&board->i2c_adap, &sfe4001_hwmon_info);
 #else
diff --git a/drivers/net/ethernet/sfc/farch.c b/drivers/net/ethernet/sfc/farch.c
index 4c83739d158f..4762ec444cb8 100644
--- a/drivers/net/ethernet/sfc/farch.c
+++ b/drivers/net/ethernet/sfc/farch.c
@@ -1477,9 +1477,10 @@ void efx_farch_irq_disable_master(struct efx_nic *efx)
  * Interrupt must already have been enabled, otherwise nasty things
  * may happen.
  */
-void efx_farch_irq_test_generate(struct efx_nic *efx)
+int efx_farch_irq_test_generate(struct efx_nic *efx)
 {
 	efx_farch_interrupts(efx, true, true);
+	return 0;
 }
 
 /* Process a fatal interrupt
diff --git a/drivers/net/ethernet/sfc/mcdi.c b/drivers/net/ethernet/sfc/mcdi.c
index d28e7dd8fa3c..241520943ada 100644
--- a/drivers/net/ethernet/sfc/mcdi.c
+++ b/drivers/net/ethernet/sfc/mcdi.c
@@ -548,7 +548,10 @@ static bool efx_mcdi_complete_async(struct efx_mcdi_iface *mcdi, bool timeout)
 		efx_mcdi_display_error(efx, async->cmd, async->inlen, errbuf,
 				       err_len, rc);
 	}
-	async->complete(efx, async->cookie, rc, outbuf, data_len);
+
+	if (async->complete)
+		async->complete(efx, async->cookie, rc, outbuf,
+				min(async->outlen, data_len));
 	kfree(async);
 
 	efx_mcdi_release(mcdi);
@@ -1153,7 +1156,8 @@ void efx_mcdi_flush_async(struct efx_nic *efx)
 	 * acquired locks in the wrong order.
 	 */
 	list_for_each_entry_safe(async, next, &mcdi->async_list, list) {
-		async->complete(efx, async->cookie, -ENETDOWN, NULL, 0);
+		if (async->complete)
+			async->complete(efx, async->cookie, -ENETDOWN, NULL, 0);
 		list_del(&async->list);
 		kfree(async);
 	}
diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
index c9a5b003caaf..ccceafc15896 100644
--- a/drivers/net/ethernet/sfc/mcdi_pcol.h
+++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
@@ -2645,16 +2645,20 @@
 #define          MC_CMD_POLL_BIST_MEM_BUS_MC 0x0
 /* enum: CSR IREG bus. */
 #define          MC_CMD_POLL_BIST_MEM_BUS_CSR 0x1
-/* enum: RX DPCPU bus. */
+/* enum: RX0 DPCPU bus. */
 #define          MC_CMD_POLL_BIST_MEM_BUS_DPCPU_RX 0x2
 /* enum: TX0 DPCPU bus. */
 #define          MC_CMD_POLL_BIST_MEM_BUS_DPCPU_TX0 0x3
 /* enum: TX1 DPCPU bus. */
 #define          MC_CMD_POLL_BIST_MEM_BUS_DPCPU_TX1 0x4
-/* enum: RX DICPU bus. */
+/* enum: RX0 DICPU bus. */
 #define          MC_CMD_POLL_BIST_MEM_BUS_DICPU_RX 0x5
 /* enum: TX DICPU bus. */
 #define          MC_CMD_POLL_BIST_MEM_BUS_DICPU_TX 0x6
+/* enum: RX1 DPCPU bus. */
+#define          MC_CMD_POLL_BIST_MEM_BUS_DPCPU_RX1 0x7
+/* enum: RX1 DICPU bus. */
+#define          MC_CMD_POLL_BIST_MEM_BUS_DICPU_RX1 0x8
 /* Pattern written to RAM / register */
 #define       MC_CMD_POLL_BIST_OUT_MEM_EXPECT_OFST 16
 /* Actual value read from RAM / register */
@@ -3612,6 +3616,8 @@
 #define        MC_CMD_NVRAM_INFO_OUT_PROTECTED_WIDTH 1
 #define        MC_CMD_NVRAM_INFO_OUT_TLV_LBN 1
 #define        MC_CMD_NVRAM_INFO_OUT_TLV_WIDTH 1
+#define        MC_CMD_NVRAM_INFO_OUT_CMAC_LBN 6
+#define        MC_CMD_NVRAM_INFO_OUT_CMAC_WIDTH 1
 #define        MC_CMD_NVRAM_INFO_OUT_A_B_LBN 7
 #define        MC_CMD_NVRAM_INFO_OUT_A_B_WIDTH 1
 #define       MC_CMD_NVRAM_INFO_OUT_PHYSDEV_OFST 16
@@ -4389,6 +4395,8 @@
  * the command will fail with MC_CMD_ERR_FILTERS_PRESENT.
  */
 #define          MC_CMD_WORKAROUND_BUG26807 0x6
+/* enum: Bug 61265 work around (broken EVQ TMR writes). */
+#define          MC_CMD_WORKAROUND_BUG61265 0x7
 /* 0 = disable the workaround indicated by TYPE; any non-zero value = enable
  * the workaround
  */
@@ -4413,7 +4421,6 @@
  * (GET_PHY_CFG_OUT_MEDIA_TYPE); the valid 'page number' input values, and the
  * output data, are interpreted on a per-type basis. For SFP+: PAGE=0 or 1
  * returns a 128-byte block read from module I2C address 0xA0 offset 0 or 0x80.
- * Anything else: currently undefined. Locks required: None. Return code: 0.
  */
 #define MC_CMD_GET_PHY_MEDIA_INFO 0x4b
 
@@ -5479,6 +5486,8 @@
 #define        LICENSED_V3_FEATURES_TX_SNIFF_WIDTH 1
 #define        LICENSED_V3_FEATURES_PROXY_FILTER_OPS_LBN 8
 #define        LICENSED_V3_FEATURES_PROXY_FILTER_OPS_WIDTH 1
+#define        LICENSED_V3_FEATURES_EVENT_CUT_THROUGH_LBN 9
+#define        LICENSED_V3_FEATURES_EVENT_CUT_THROUGH_WIDTH 1
 #define       LICENSED_V3_FEATURES_MASK_LBN 0
 #define       LICENSED_V3_FEATURES_MASK_WIDTH 64
 
@@ -5634,6 +5643,109 @@
 /* Only valid if INTRFLAG was true */
 #define       MC_CMD_INIT_EVQ_OUT_IRQ_OFST 0
 
+/* MC_CMD_INIT_EVQ_V2_IN msgrequest */
+#define    MC_CMD_INIT_EVQ_V2_IN_LENMIN 44
+#define    MC_CMD_INIT_EVQ_V2_IN_LENMAX 548
+#define    MC_CMD_INIT_EVQ_V2_IN_LEN(num) (36+8*(num))
+/* Size, in entries */
+#define       MC_CMD_INIT_EVQ_V2_IN_SIZE_OFST 0
+/* Desired instance. Must be set to a specific instance, which is a function
+ * local queue index.
+ */
+#define       MC_CMD_INIT_EVQ_V2_IN_INSTANCE_OFST 4
+/* The initial timer value. The load value is ignored if the timer mode is DIS.
+ */
+#define       MC_CMD_INIT_EVQ_V2_IN_TMR_LOAD_OFST 8
+/* The reload value is ignored in one-shot modes */
+#define       MC_CMD_INIT_EVQ_V2_IN_TMR_RELOAD_OFST 12
+/* tbd */
+#define       MC_CMD_INIT_EVQ_V2_IN_FLAGS_OFST 16
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_INTERRUPTING_LBN 0
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_INTERRUPTING_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_RPTR_DOS_LBN 1
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_RPTR_DOS_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_INT_ARMD_LBN 2
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_INT_ARMD_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_CUT_THRU_LBN 3
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_CUT_THRU_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_RX_MERGE_LBN 4
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_RX_MERGE_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_TX_MERGE_LBN 5
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_TX_MERGE_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_USE_TIMER_LBN 6
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_USE_TIMER_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_LBN 7
+#define        MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_WIDTH 4
+/* enum: All initialisation flags specified by host. */
+#define          MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_MANUAL 0x0
+/* enum: MEDFORD only. Certain initialisation flags specified by host may be
+ * over-ridden by firmware based on licenses and firmware variant in order to
+ * provide the lowest latency achievable. See
+ * MC_CMD_INIT_EVQ_V2/MC_CMD_INIT_EVQ_V2_OUT/FLAGS for list of affected flags.
+ */
+#define          MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_LOW_LATENCY 0x1
+/* enum: MEDFORD only. Certain initialisation flags specified by host may be
+ * over-ridden by firmware based on licenses and firmware variant in order to
+ * provide the best throughput achievable. See
+ * MC_CMD_INIT_EVQ_V2/MC_CMD_INIT_EVQ_V2_OUT/FLAGS for list of affected flags.
+ */
+#define          MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_THROUGHPUT 0x2
+/* enum: MEDFORD only. Certain initialisation flags may be over-ridden by
+ * firmware based on licenses and firmware variant. See
+ * MC_CMD_INIT_EVQ_V2/MC_CMD_INIT_EVQ_V2_OUT/FLAGS for list of affected flags.
+ */
+#define          MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_AUTO 0x3
+#define       MC_CMD_INIT_EVQ_V2_IN_TMR_MODE_OFST 20
+/* enum: Disabled */
+#define          MC_CMD_INIT_EVQ_V2_IN_TMR_MODE_DIS 0x0
+/* enum: Immediate */
+#define          MC_CMD_INIT_EVQ_V2_IN_TMR_IMMED_START 0x1
+/* enum: Triggered */
+#define          MC_CMD_INIT_EVQ_V2_IN_TMR_TRIG_START 0x2
+/* enum: Hold-off */
+#define          MC_CMD_INIT_EVQ_V2_IN_TMR_INT_HLDOFF 0x3
+/* Target EVQ for wakeups if in wakeup mode. */
+#define       MC_CMD_INIT_EVQ_V2_IN_TARGET_EVQ_OFST 24
+/* Target interrupt if in interrupting mode (note union with target EVQ). Use
+ * MC_CMD_RESOURCE_INSTANCE_ANY unless a specific one required for test
+ * purposes.
+ */
+#define       MC_CMD_INIT_EVQ_V2_IN_IRQ_NUM_OFST 24
+/* Event Counter Mode. */
+#define       MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_OFST 28
+/* enum: Disabled */
+#define          MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_DIS 0x0
+/* enum: Disabled */
+#define          MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_RX 0x1
+/* enum: Disabled */
+#define          MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_TX 0x2
+/* enum: Disabled */
+#define          MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_RXTX 0x3
+/* Event queue packet count threshold. */
+#define       MC_CMD_INIT_EVQ_V2_IN_COUNT_THRSHLD_OFST 32
+/* 64-bit address of 4k of 4k-aligned host memory buffer */
+#define       MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_OFST 36
+#define       MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LEN 8
+#define       MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LO_OFST 36
+#define       MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_HI_OFST 40
+#define       MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_MINNUM 1
+#define       MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_MAXNUM 64
+
+/* MC_CMD_INIT_EVQ_V2_OUT msgresponse */
+#define    MC_CMD_INIT_EVQ_V2_OUT_LEN 8
+/* Only valid if INTRFLAG was true */
+#define       MC_CMD_INIT_EVQ_V2_OUT_IRQ_OFST 0
+/* Actual configuration applied on the card */
+#define       MC_CMD_INIT_EVQ_V2_OUT_FLAGS_OFST 4
+#define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_CUT_THRU_LBN 0
+#define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_CUT_THRU_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_RX_MERGE_LBN 1
+#define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_RX_MERGE_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_TX_MERGE_LBN 2
+#define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_TX_MERGE_WIDTH 1
+#define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_RXQ_FORCE_EV_MERGING_LBN 3
+#define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_RXQ_FORCE_EV_MERGING_WIDTH 1
+
 /* QUEUE_CRC_MODE structuredef */
 #define    QUEUE_CRC_MODE_LEN 1
 #define       QUEUE_CRC_MODE_MODE_LBN 0
@@ -5697,8 +5809,8 @@
 #define        MC_CMD_INIT_RXQ_IN_FLAG_PREFIX_WIDTH 1
 #define        MC_CMD_INIT_RXQ_IN_FLAG_DISABLE_SCATTER_LBN 9
 #define        MC_CMD_INIT_RXQ_IN_FLAG_DISABLE_SCATTER_WIDTH 1
-#define        MC_CMD_INIT_RXQ_IN_FLAG_FORCE_EV_MERGING_LBN 10
-#define        MC_CMD_INIT_RXQ_IN_FLAG_FORCE_EV_MERGING_WIDTH 1
+#define        MC_CMD_INIT_RXQ_IN_UNUSED_LBN 10
+#define        MC_CMD_INIT_RXQ_IN_UNUSED_WIDTH 1
 /* Owner ID to use if in buffer mode (zero if physical) */
 #define       MC_CMD_INIT_RXQ_IN_OWNER_ID_OFST 20
 /* The port ID associated with the v-adaptor which should contain this DMAQ. */
@@ -7854,6 +7966,20 @@
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_EVENT_CUT_THROUGH_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_CUT_THROUGH_LBN 4
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_CUT_THROUGH_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_VFIFO_ULL_MODE_LBN 5
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_VFIFO_ULL_MODE_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN 6
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_MAC_STATS_40G_TX_SIZE_BINS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_INIT_EVQ_V2_LBN 7
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_INIT_EVQ_V2_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MAC_TIMESTAMPING_LBN 8
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MAC_TIMESTAMPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TIMESTAMP_LBN 9
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TIMESTAMP_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_SNIFF_LBN 10
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_RX_SNIFF_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_SNIFF_LBN 11
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_SNIFF_WIDTH 1
 /* Number of FATSOv2 contexts per datapath supported by this NIC. Not present
  * on older firmware (check the length).
  */
@@ -7910,6 +8036,288 @@
 #define       MC_CMD_GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF_OFST 70
 #define       MC_CMD_GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF_LEN 2
 
+/* MC_CMD_GET_CAPABILITIES_V3_OUT msgresponse */
+#define    MC_CMD_GET_CAPABILITIES_V3_OUT_LEN 73
+/* First word of flags. */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_FLAGS1_OFST 0
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_VPORT_RECONFIGURE_LBN 3
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_VPORT_RECONFIGURE_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_STRIPING_LBN 4
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_STRIPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_VADAPTOR_QUERY_LBN 5
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_VADAPTOR_QUERY_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_EVB_PORT_VLAN_RESTRICT_LBN 6
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_EVB_PORT_VLAN_RESTRICT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_DRV_ATTACH_PREBOOT_LBN 7
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_DRV_ATTACH_PREBOOT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_FORCE_EVENT_MERGING_LBN 8
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_FORCE_EVENT_MERGING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_SET_MAC_ENHANCED_LBN 9
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_SET_MAC_ENHANCED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_LBN 10
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 11
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MAC_SECURITY_FILTERING_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MAC_SECURITY_FILTERING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_ADDITIONAL_RSS_MODES_LBN 13
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_ADDITIONAL_RSS_MODES_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_QBB_LBN 14
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_QBB_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PACKED_STREAM_VAR_BUFFERS_LBN 15
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PACKED_STREAM_VAR_BUFFERS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_RSS_LIMITED_LBN 16
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_RSS_LIMITED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PACKED_STREAM_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PACKED_STREAM_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_INCLUDE_FCS_LBN 18
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_INCLUDE_FCS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_VLAN_INSERTION_LBN 19
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_VLAN_INSERTION_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_VLAN_STRIPPING_LBN 20
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_VLAN_STRIPPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_LBN 21
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PREFIX_LEN_0_LBN 22
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PREFIX_LEN_0_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PREFIX_LEN_14_LBN 23
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_PREFIX_LEN_14_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_TIMESTAMP_LBN 24
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_TIMESTAMP_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_BATCHING_LBN 25
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_BATCHING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_MCAST_FILTER_CHAINING_LBN 26
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_MCAST_FILTER_CHAINING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_PM_AND_RXDP_COUNTERS_LBN 27
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_PM_AND_RXDP_COUNTERS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DISABLE_SCATTER_LBN 28
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DISABLE_SCATTER_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MCAST_UDP_LOOPBACK_LBN 29
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MCAST_UDP_LOOPBACK_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_EVB_LBN 30
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_EVB_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_VXLAN_NVGRE_LBN 31
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_VXLAN_NVGRE_WIDTH 1
+/* RxDPCPU firmware id. */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DPCPU_FW_ID_OFST 4
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DPCPU_FW_ID_LEN 2
+/* enum: Standard RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP  0x0
+/* enum: Low latency RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_LOW_LATENCY  0x1
+/* enum: Packed stream RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_PACKED_STREAM  0x2
+/* enum: BIST RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_BIST  0x10a
+/* enum: RXDP Test firmware image 1 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_TO_MC_CUT_THROUGH  0x101
+/* enum: RXDP Test firmware image 2 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD  0x102
+/* enum: RXDP Test firmware image 3 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD_FIRST  0x103
+/* enum: RXDP Test firmware image 4 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_EVERY_EVENT_BATCHABLE  0x104
+/* enum: RXDP Test firmware image 5 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_BACKPRESSURE  0x105
+/* enum: RXDP Test firmware image 6 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_PACKET_EDITS  0x106
+/* enum: RXDP Test firmware image 7 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_RX_HDR_SPLIT  0x107
+/* enum: RXDP Test firmware image 8 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_DISABLE_DL  0x108
+/* enum: RXDP Test firmware image 9 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_DOORBELL_DELAY  0x10b
+/* TxDPCPU firmware id. */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DPCPU_FW_ID_OFST 6
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DPCPU_FW_ID_LEN 2
+/* enum: Standard TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP  0x0
+/* enum: Low latency TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_LOW_LATENCY  0x1
+/* enum: High packet rate TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_HIGH_PACKET_RATE  0x3
+/* enum: BIST TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_BIST  0x12d
+/* enum: TXDP Test firmware image 1 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_TEST_FW_TSO_EDIT  0x101
+/* enum: TXDP Test firmware image 2 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_TEST_FW_PACKET_EDITS  0x102
+/* enum: TXDP CSR bus test firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_TEST_FW_CSR  0x103
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_OFST 8
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_LEN 2
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_REV_LBN 0
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_REV_WIDTH 12
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_TYPE_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_VERSION_TYPE_WIDTH 4
+/* enum: reserved value - do not use (may indicate alternative interpretation
+ * of REV field in future)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_RESERVED  0x0
+/* enum: Trivial RX PD firmware for early Huntington development (Huntington
+ * development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_FIRST_PKT  0x1
+/* enum: RX PD firmware with approximately Siena-compatible behaviour
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_SIENA_COMPAT  0x2
+/* enum: Virtual switching (full feature) RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_VSWITCH  0x3
+/* enum: siena_compat variant RX PD firmware using PM rather than MAC
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_SIENA_COMPAT_PM  0x4
+/* enum: Low latency RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_LOW_LATENCY  0x5
+/* enum: Packed stream RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_PACKED_STREAM  0x6
+/* enum: RX PD firmware handling layer 2 only for high packet rate performance
+ * tests (Medford development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_LAYER2_PERF  0x7
+/* enum: Rules engine RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
+/* enum: RX PD firmware parsing but not filtering network overlay tunnel
+ * encapsulations (Medford development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_TESTFW_ENCAP_PARSING_ONLY  0xf
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_OFST 10
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_LEN 2
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_REV_LBN 0
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_REV_WIDTH 12
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_TYPE_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_VERSION_TYPE_WIDTH 4
+/* enum: reserved value - do not use (may indicate alternative interpretation
+ * of REV field in future)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_RESERVED  0x0
+/* enum: Trivial TX PD firmware for early Huntington development (Huntington
+ * development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_FIRST_PKT  0x1
+/* enum: TX PD firmware with approximately Siena-compatible behaviour
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_SIENA_COMPAT  0x2
+/* enum: Virtual switching (full feature) TX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_VSWITCH  0x3
+/* enum: siena_compat variant TX PD firmware using PM rather than MAC
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_SIENA_COMPAT_PM  0x4
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_LOW_LATENCY  0x5 /* enum */
+/* enum: TX PD firmware handling layer 2 only for high packet rate performance
+ * tests (Medford development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_LAYER2_PERF  0x7
+/* enum: Rules engine TX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
+/* Hardware capabilities of NIC */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_HW_CAPABILITIES_OFST 12
+/* Licensed capabilities */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_LICENSE_CAPABILITIES_OFST 16
+/* Second word of flags. Not present on older firmware (check the length). */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_FLAGS2_OFST 20
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_LBN 0
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_ENCAP_LBN 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_ENCAP_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_EVQ_TIMER_CTRL_LBN 2
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_EVQ_TIMER_CTRL_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_EVENT_CUT_THROUGH_LBN 3
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_EVENT_CUT_THROUGH_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_CUT_THROUGH_LBN 4
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_CUT_THROUGH_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_VFIFO_ULL_MODE_LBN 5
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_VFIFO_ULL_MODE_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN 6
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_MAC_STATS_40G_TX_SIZE_BINS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_INIT_EVQ_V2_LBN 7
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_INIT_EVQ_V2_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MAC_TIMESTAMPING_LBN 8
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_MAC_TIMESTAMPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TIMESTAMP_LBN 9
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TIMESTAMP_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_SNIFF_LBN 10
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_RX_SNIFF_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_SNIFF_LBN 11
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_SNIFF_WIDTH 1
+/* Number of FATSOv2 contexts per datapath supported by this NIC. Not present
+ * on older firmware (check the length).
+ */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_N_CONTEXTS_OFST 24
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_N_CONTEXTS_LEN 2
+/* One byte per PF containing the number of the external port assigned to this
+ * PF, indexed by PF number. Special values indicate that a PF is either not
+ * present or not assigned.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_PFS_TO_PORTS_ASSIGNMENT_OFST 26
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_PFS_TO_PORTS_ASSIGNMENT_LEN 1
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_PFS_TO_PORTS_ASSIGNMENT_NUM 16
+/* enum: The caller is not permitted to access information on this PF. */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_ACCESS_NOT_PERMITTED  0xff
+/* enum: PF does not exist. */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_PF_NOT_PRESENT  0xfe
+/* enum: PF does exist but is not assigned to any external port. */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_PF_NOT_ASSIGNED  0xfd
+/* enum: This value indicates that PF is assigned, but it cannot be expressed
+ * in this field. It is intended for a possible future situation where a more
+ * complex scheme of PFs to ports mapping is being used. The future driver
+ * should look for a new field supporting the new scheme. The current/old
+ * driver should treat this value as PF_NOT_ASSIGNED.
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_INCOMPATIBLE_ASSIGNMENT  0xfc
+/* One byte per PF containing the number of its VFs, indexed by PF number. A
+ * special value indicates that a PF is not present.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VFS_PER_PF_OFST 42
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VFS_PER_PF_LEN 1
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VFS_PER_PF_NUM 16
+/* enum: The caller is not permitted to access information on this PF. */
+/*               MC_CMD_GET_CAPABILITIES_V3_OUT_ACCESS_NOT_PERMITTED  0xff */
+/* enum: PF does not exist. */
+/*               MC_CMD_GET_CAPABILITIES_V3_OUT_PF_NOT_PRESENT  0xfe */
+/* Number of VIs available for each external port */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VIS_PER_PORT_OFST 58
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VIS_PER_PORT_LEN 2
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_VIS_PER_PORT_NUM 4
+/* Size of RX descriptor cache expressed as binary logarithm The actual size
+ * equals (2 ^ RX_DESC_CACHE_SIZE)
+ */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DESC_CACHE_SIZE_OFST 66
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_RX_DESC_CACHE_SIZE_LEN 1
+/* Size of TX descriptor cache expressed as binary logarithm The actual size
+ * equals (2 ^ TX_DESC_CACHE_SIZE)
+ */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DESC_CACHE_SIZE_OFST 67
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DESC_CACHE_SIZE_LEN 1
+/* Total number of available PIO buffers */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_PIO_BUFFS_OFST 68
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_NUM_PIO_BUFFS_LEN 2
+/* Size of a single PIO buffer */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_SIZE_PIO_BUFF_OFST 70
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_SIZE_PIO_BUFF_LEN 2
+/* On chips later than Medford the amount of address space assigned to each VI
+ * is configurable. This is a global setting that the driver must query to
+ * discover the VI to address mapping. Cut-through PIO (CTPIO) in not available
+ * with 8k VI windows.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_OFST 72
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_LEN 1
+/* enum: Each VI occupies 8k as on Huntington and Medford. PIO is at offset 4k.
+ * CTPIO is not mapped.
+ */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_8K   0x0
+/* enum: Each VI occupies 16k. PIO is at offset 4k. CTPIO is at offset 12k. */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_16K  0x1
+/* enum: Each VI occupies 64k. PIO is at offset 4k. CTPIO is at offset 12k. */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_64K  0x2
+
 
 /***********************************/
 /* MC_CMD_V2_EXTN
@@ -9026,7 +9434,7 @@
  */
 #define MC_CMD_GET_RXDP_CONFIG 0xc2
 
-#define MC_CMD_0xc2_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xc2_PRIVILEGE_CTG SRIOV_CTG_GENERAL
 
 /* MC_CMD_GET_RXDP_CONFIG_IN msgrequest */
 #define    MC_CMD_GET_RXDP_CONFIG_IN_LEN 0
@@ -10125,7 +10533,9 @@
  * that this operation returns a zero-length response
  */
 #define          MC_CMD_LICENSING_V3_IN_OP_UPDATE_LICENSE  0x0
-/* enum: report counts of installed licenses */
+/* enum: report counts of installed licenses Returns EAGAIN if license
+ * processing (updating) has been started but not yet completed.
+ */
 #define          MC_CMD_LICENSING_V3_IN_OP_REPORT_LICENSE  0x1
 
 /* MC_CMD_LICENSING_V3_OUT msgresponse */
@@ -10763,6 +11173,8 @@
 #define          MC_CMD_GET_WORKAROUNDS_OUT_BUG42008 0x20
 /* enum: Bug 26807 features present in firmware (multicast filter chaining) */
 #define          MC_CMD_GET_WORKAROUNDS_OUT_BUG26807 0x40
+/* enum: Bug 61265 work around (broken EVQ TMR writes). */
+#define          MC_CMD_GET_WORKAROUNDS_OUT_BUG61265 0x80
 
 
 /***********************************/
@@ -11280,22 +11692,110 @@
 #define MC_CMD_0x118_PRIVILEGE_CTG SRIOV_CTG_ADMIN
 
 /* MC_CMD_RX_BALANCING_IN msgrequest */
-#define    MC_CMD_RX_BALANCING_IN_LEN 4
+#define    MC_CMD_RX_BALANCING_IN_LEN 16
 /* The RX port whose upconverter table will be modified */
 #define       MC_CMD_RX_BALANCING_IN_PORT_OFST 0
-#define       MC_CMD_RX_BALANCING_IN_PORT_LEN 1
 /* The VLAN priority associated to the table index and vFIFO */
-#define       MC_CMD_RX_BALANCING_IN_PRIORITY_OFST 1
-#define       MC_CMD_RX_BALANCING_IN_PRIORITY_LEN 1
+#define       MC_CMD_RX_BALANCING_IN_PRIORITY_OFST 4
 /* The resulting bit of SRC^DST for indexing the table */
-#define       MC_CMD_RX_BALANCING_IN_SRC_DST_OFST 2
-#define       MC_CMD_RX_BALANCING_IN_SRC_DST_LEN 1
+#define       MC_CMD_RX_BALANCING_IN_SRC_DST_OFST 8
 /* The RX engine to which the vFIFO in the table entry will point to */
-#define       MC_CMD_RX_BALANCING_IN_ENG_OFST 3
-#define       MC_CMD_RX_BALANCING_IN_ENG_LEN 1
+#define       MC_CMD_RX_BALANCING_IN_ENG_OFST 12
 
 /* MC_CMD_RX_BALANCING_OUT msgresponse */
 #define    MC_CMD_RX_BALANCING_OUT_LEN 0
 
+/***********************************/
+/* MC_CMD_SET_EVQ_TMR
+ * Update the timer load, timer reload and timer mode values for a given EVQ.
+ * The requested timer values (in TMR_LOAD_REQ_NS and TMR_RELOAD_REQ_NS) will
+ * be rounded up to the granularity supported by the hardware, then truncated
+ * to the range supported by the hardware. The resulting value after the
+ * rounding and truncation will be returned to the caller (in TMR_LOAD_ACT_NS
+ * and TMR_RELOAD_ACT_NS).
+ */
+#define MC_CMD_SET_EVQ_TMR 0x120
+
+#define MC_CMD_0x120_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_SET_EVQ_TMR_IN msgrequest */
+#define    MC_CMD_SET_EVQ_TMR_IN_LEN 16
+/* Function-relative queue instance */
+#define       MC_CMD_SET_EVQ_TMR_IN_INSTANCE_OFST 0
+/* Requested value for timer load (in nanoseconds) */
+#define       MC_CMD_SET_EVQ_TMR_IN_TMR_LOAD_REQ_NS_OFST 4
+/* Requested value for timer reload (in nanoseconds) */
+#define       MC_CMD_SET_EVQ_TMR_IN_TMR_RELOAD_REQ_NS_OFST 8
+/* Timer mode. Meanings as per EVQ_TMR_REG.TC_TIMER_VAL */
+#define       MC_CMD_SET_EVQ_TMR_IN_TMR_MODE_OFST 12
+#define          MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_DIS  0x0 /* enum */
+#define          MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_IMMED_START  0x1 /* enum */
+#define          MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_TRIG_START  0x2 /* enum */
+#define          MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_INT_HLDOFF  0x3 /* enum */
+
+/* MC_CMD_SET_EVQ_TMR_OUT msgresponse */
+#define    MC_CMD_SET_EVQ_TMR_OUT_LEN 8
+/* Actual value for timer load (in nanoseconds) */
+#define       MC_CMD_SET_EVQ_TMR_OUT_TMR_LOAD_ACT_NS_OFST 0
+/* Actual value for timer reload (in nanoseconds) */
+#define       MC_CMD_SET_EVQ_TMR_OUT_TMR_RELOAD_ACT_NS_OFST 4
+
+
+/***********************************/
+/* MC_CMD_GET_EVQ_TMR_PROPERTIES
+ * Query properties about the event queue timers.
+ */
+#define MC_CMD_GET_EVQ_TMR_PROPERTIES 0x122
+
+#define MC_CMD_0x122_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
+/* MC_CMD_GET_EVQ_TMR_PROPERTIES_IN msgrequest */
+#define    MC_CMD_GET_EVQ_TMR_PROPERTIES_IN_LEN 0
+
+/* MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT msgresponse */
+#define    MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN 36
+/* Reserved for future use. */
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_FLAGS_OFST 0
+/* For timers updated via writes to EVQ_TMR_REG, this is the time interval (in
+ * nanoseconds) for each increment of the timer load/reload count. The
+ * requested duration of a timer is this value multiplied by the timer
+ * load/reload count.
+ */
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_NS_PER_COUNT_OFST 4
+/* For timers updated via writes to EVQ_TMR_REG, this is the maximum value
+ * allowed for timer load/reload counts.
+ */
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_MAX_COUNT_OFST 8
+/* For timers updated via writes to EVQ_TMR_REG, timer load/reload counts not a
+ * multiple of this step size will be rounded in an implementation defined
+ * manner.
+ */
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_STEP_OFST 12
+/* Maximum timer duration (in nanoseconds) for timers updated via MCDI. Only
+ * meaningful if MC_CMD_SET_EVQ_TMR is implemented.
+ */
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_MAX_NS_OFST 16
+/* Timer durations requested via MCDI that are not a multiple of this step size
+ * will be rounded up. Only meaningful if MC_CMD_SET_EVQ_TMR is implemented.
+ */
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_STEP_NS_OFST 20
+/* For timers updated using the bug35388 workaround, this is the time interval
+ * (in nanoseconds) for each increment of the timer load/reload count. The
+ * requested duration of a timer is this value multiplied by the timer
+ * load/reload count. This field is only meaningful if the bug35388 workaround
+ * is enabled.
+ */
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_NS_PER_COUNT_OFST 24
+/* For timers updated using the bug35388 workaround, this is the maximum value
+ * allowed for timer load/reload counts. This field is only meaningful if the
+ * bug35388 workaround is enabled.
+ */
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_MAX_COUNT_OFST 28
+/* For timers updated using the bug35388 workaround, timer load/reload counts
+ * not a multiple of this step size will be rounded in an implementation
+ * defined manner. This field is only meaningful if the bug35388 workaround is
+ * enabled.
+ */
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_STEP_OFST 32
 
 #endif /* MCDI_PCOL_H */
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index 9ff062a36ea8..99d8c82124bb 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -76,6 +76,9 @@
 /* Maximum possible MTU the driver supports */
 #define EFX_MAX_MTU (9 * 1024)
 
+/* Minimum MTU, from RFC791 (IP) */
+#define EFX_MIN_MTU 68
+
 /* Size of an RX scatter buffer.  Small enough to pack 2 into a 4K page,
  * and should be a multiple of the cache line size.
  */
@@ -392,7 +395,7 @@ enum efx_sync_events_state {
  * @eventq_init: Event queue initialised flag
  * @enabled: Channel enabled indicator
  * @irq: IRQ number (MSI and MSI-X only)
- * @irq_moderation: IRQ moderation value (in hardware ticks)
+ * @irq_moderation_us: IRQ moderation value (in microseconds)
  * @napi_dev: Net device used with NAPI
  * @napi_str: NAPI control structure
  * @state: state for NAPI vs busy polling
@@ -433,7 +436,7 @@ struct efx_channel {
 	bool eventq_init;
 	bool enabled;
 	int irq;
-	unsigned int irq_moderation;
+	unsigned int irq_moderation_us;
 	struct net_device *napi_dev;
 	struct napi_struct napi_str;
 #ifdef CONFIG_NET_RX_BUSY_POLL
@@ -810,8 +813,10 @@ struct vfdi_status;
  * @membase: Memory BAR value
  * @interrupt_mode: Interrupt mode
  * @timer_quantum_ns: Interrupt timer quantum, in nanoseconds
+ * @timer_max_ns: Interrupt timer maximum value, in nanoseconds
  * @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues
- * @irq_rx_moderation: IRQ moderation time for RX event queues
+ * @irq_rx_mod_step_us: Step size for IRQ moderation for RX event queues
+ * @irq_rx_moderation_us: IRQ moderation time for RX event queues
  * @msg_enable: Log message enable flags
  * @state: Device state number (%STATE_*). Serialised by the rtnl_lock.
  * @reset_pending: Bitmask for pending resets
@@ -940,8 +945,10 @@ struct efx_nic {
 
 	enum efx_int_mode interrupt_mode;
 	unsigned int timer_quantum_ns;
+	unsigned int timer_max_ns;
 	bool irq_rx_adaptive;
-	unsigned int irq_rx_moderation;
+	unsigned int irq_mod_step_us;
+	unsigned int irq_rx_moderation_us;
 	u32 msg_enable;
 
 	enum nic_state state;
@@ -1271,7 +1278,7 @@ struct efx_nic_type {
 	int (*mcdi_poll_reboot)(struct efx_nic *efx);
 	void (*mcdi_reboot_detected)(struct efx_nic *efx);
 	void (*irq_enable_master)(struct efx_nic *efx);
-	void (*irq_test_generate)(struct efx_nic *efx);
+	int (*irq_test_generate)(struct efx_nic *efx);
 	void (*irq_disable_non_ev)(struct efx_nic *efx);
 	irqreturn_t (*irq_handle_msi)(int irq, void *dev_id);
 	irqreturn_t (*irq_handle_legacy)(int irq, void *dev_id);
diff --git a/drivers/net/ethernet/sfc/nic.c b/drivers/net/ethernet/sfc/nic.c
index 89b83e59e1dc..aa1945a858d5 100644
--- a/drivers/net/ethernet/sfc/nic.c
+++ b/drivers/net/ethernet/sfc/nic.c
@@ -66,11 +66,11 @@ void efx_nic_event_test_start(struct efx_channel *channel)
 	channel->efx->type->ev_test_generate(channel);
 }
 
-void efx_nic_irq_test_start(struct efx_nic *efx)
+int efx_nic_irq_test_start(struct efx_nic *efx)
 {
 	efx->last_irq_cpu = -1;
 	smp_wmb();
-	efx->type->irq_test_generate(efx);
+	return efx->type->irq_test_generate(efx);
 }
 
 /* Hook interrupt handler(s)
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
index 96944c3c9d14..73bee7ea332a 100644
--- a/drivers/net/ethernet/sfc/nic.h
+++ b/drivers/net/ethernet/sfc/nic.h
@@ -507,10 +507,13 @@ enum {
  * @stats: Hardware statistics
  * @workaround_35388: Flag: firmware supports workaround for bug 35388
  * @workaround_26807: Flag: firmware supports workaround for bug 26807
+ * @workaround_61265: Flag: firmware supports workaround for bug 61265
  * @must_check_datapath_caps: Flag: @datapath_caps needs to be revalidated
  *	after MC reboot
  * @datapath_caps: Capabilities of datapath firmware (FLAGS1 field of
  *	%MC_CMD_GET_CAPABILITIES response)
+ * @datapath_caps2: Further Capabilities of datapath firmware (FLAGS2 field of
+ * %MC_CMD_GET_CAPABILITIES response)
  * @rx_dpcpu_fw_id: Firmware ID of the RxDPCPU
  * @tx_dpcpu_fw_id: Firmware ID of the TxDPCPU
  * @vport_id: The function's vport ID, only relevant for PFs
@@ -540,8 +543,10 @@ struct efx_ef10_nic_data {
 	u64 stats[EF10_STAT_COUNT];
 	bool workaround_35388;
 	bool workaround_26807;
+	bool workaround_61265;
 	bool must_check_datapath_caps;
 	u32 datapath_caps;
+	u32 datapath_caps2;
 	unsigned int rx_dpcpu_fw_id;
 	unsigned int tx_dpcpu_fw_id;
 	unsigned int vport_id;
@@ -741,12 +746,12 @@ static inline void efx_update_diff_stat(u64 *stat, u64 diff)
 
 /* Interrupts */
 int efx_nic_init_interrupt(struct efx_nic *efx);
-void efx_nic_irq_test_start(struct efx_nic *efx);
+int efx_nic_irq_test_start(struct efx_nic *efx);
 void efx_nic_fini_interrupt(struct efx_nic *efx);
 
 /* Falcon/Siena interrupts */
 void efx_farch_irq_enable_master(struct efx_nic *efx);
-void efx_farch_irq_test_generate(struct efx_nic *efx);
+int efx_farch_irq_test_generate(struct efx_nic *efx);
 void efx_farch_irq_disable_master(struct efx_nic *efx);
 irqreturn_t efx_farch_msi_interrupt(int irq, void *dev_id);
 irqreturn_t efx_farch_legacy_interrupt(int irq, void *dev_id);
diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c
index c771e0af4e06..77a5364f7a10 100644
--- a/drivers/net/ethernet/sfc/ptp.c
+++ b/drivers/net/ethernet/sfc/ptp.c
@@ -1269,13 +1269,13 @@ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel)
 		if (IS_ERR(ptp->phc_clock)) {
 			rc = PTR_ERR(ptp->phc_clock);
 			goto fail3;
-		}
-
-		INIT_WORK(&ptp->pps_work, efx_ptp_pps_worker);
-		ptp->pps_workwq = create_singlethread_workqueue("sfc_pps");
-		if (!ptp->pps_workwq) {
-			rc = -ENOMEM;
-			goto fail4;
+		} else if (ptp->phc_clock) {
+			INIT_WORK(&ptp->pps_work, efx_ptp_pps_worker);
+			ptp->pps_workwq = create_singlethread_workqueue("sfc_pps");
+			if (!ptp->pps_workwq) {
+				rc = -ENOMEM;
+				goto fail4;
+			}
 		}
 	}
 	ptp->nic_ts_enabled = false;
@@ -1306,7 +1306,7 @@ static int efx_ptp_probe_channel(struct efx_channel *channel)
 {
 	struct efx_nic *efx = channel->efx;
 
-	channel->irq_moderation = 0;
+	channel->irq_moderation_us = 0;
 	channel->rx_queue.core_index = 0;
 
 	return efx_ptp_probe(efx, channel);
diff --git a/drivers/net/ethernet/sfc/selftest.c b/drivers/net/ethernet/sfc/selftest.c
index 9d78830da609..cd38b44ae23a 100644
--- a/drivers/net/ethernet/sfc/selftest.c
+++ b/drivers/net/ethernet/sfc/selftest.c
@@ -135,11 +135,19 @@ static int efx_test_interrupts(struct efx_nic *efx,
 {
 	unsigned long timeout, wait;
 	int cpu;
+	int rc;
 
 	netif_dbg(efx, drv, efx->net_dev, "testing interrupts\n");
 	tests->interrupt = -1;
 
-	efx_nic_irq_test_start(efx);
+	rc = efx_nic_irq_test_start(efx);
+	if (rc == -ENOTSUPP) {
+		netif_dbg(efx, drv, efx->net_dev,
+			  "direct interrupt testing not supported\n");
+		tests->interrupt = 0;
+		return 0;
+	}
+
 	timeout = jiffies + IRQ_TIMEOUT;
 	wait = 1;
 
diff --git a/drivers/net/ethernet/sfc/selftest.h b/drivers/net/ethernet/sfc/selftest.h
index 009dbe88f3be..32a427253a03 100644
--- a/drivers/net/ethernet/sfc/selftest.h
+++ b/drivers/net/ethernet/sfc/selftest.h
@@ -28,7 +28,7 @@ struct efx_loopback_self_tests {
 
 /* Efx self test results
  * For fields which are not counters, 1 indicates success and -1
- * indicates failure.
+ * indicates failure; 0 indicates test could not be run.
  */
 struct efx_self_tests {
 	/* online tests */
diff --git a/drivers/net/ethernet/sfc/siena.c b/drivers/net/ethernet/sfc/siena.c
index 2219b5424d2b..04ed1b4c7cd9 100644
--- a/drivers/net/ethernet/sfc/siena.c
+++ b/drivers/net/ethernet/sfc/siena.c
@@ -34,19 +34,24 @@ static void siena_init_wol(struct efx_nic *efx);
 
 static void siena_push_irq_moderation(struct efx_channel *channel)
 {
+	struct efx_nic *efx = channel->efx;
 	efx_dword_t timer_cmd;
 
-	if (channel->irq_moderation)
+	if (channel->irq_moderation_us) {
+		unsigned int ticks;
+
+		ticks = efx_usecs_to_ticks(efx, channel->irq_moderation_us);
 		EFX_POPULATE_DWORD_2(timer_cmd,
 				     FRF_CZ_TC_TIMER_MODE,
 				     FFE_CZ_TIMER_MODE_INT_HLDOFF,
 				     FRF_CZ_TC_TIMER_VAL,
-				     channel->irq_moderation - 1);
-	else
+				     ticks - 1);
+	} else {
 		EFX_POPULATE_DWORD_2(timer_cmd,
 				     FRF_CZ_TC_TIMER_MODE,
 				     FFE_CZ_TIMER_MODE_DIS,
 				     FRF_CZ_TC_TIMER_VAL, 0);
+	}
 	efx_writed_page_locked(channel->efx, &timer_cmd, FR_BZ_TIMER_COMMAND_P0,
 			       channel->channel);
 }
@@ -222,6 +227,9 @@ static int siena_probe_nvconfig(struct efx_nic *efx)
 	efx->timer_quantum_ns =
 		(caps & (1 << MC_CMD_CAPABILITIES_TURBO_ACTIVE_LBN)) ?
 		3072 : 6144; /* 768 cycles */
+	efx->timer_max_ns = efx->type->timer_period_max *
+			    efx->timer_quantum_ns;
+
 	return rc;
 }
 
diff --git a/drivers/net/ethernet/sfc/sriov.c b/drivers/net/ethernet/sfc/sriov.c
index 816c44689e67..9abcf4aded30 100644
--- a/drivers/net/ethernet/sfc/sriov.c
+++ b/drivers/net/ethernet/sfc/sriov.c
@@ -22,7 +22,7 @@ int efx_sriov_set_vf_mac(struct net_device *net_dev, int vf_i, u8 *mac)
 }
 
 int efx_sriov_set_vf_vlan(struct net_device *net_dev, int vf_i, u16 vlan,
-			  u8 qos)
+			  u8 qos, __be16 vlan_proto)
 {
 	struct efx_nic *efx = netdev_priv(net_dev);
 
@@ -31,6 +31,9 @@ int efx_sriov_set_vf_vlan(struct net_device *net_dev, int vf_i, u16 vlan,
 		    (qos & ~(VLAN_PRIO_MASK >> VLAN_PRIO_SHIFT)))
 			return -EINVAL;
 
+		if (vlan_proto != htons(ETH_P_8021Q))
+			return -EPROTONOSUPPORT;
+
 		return efx->type->sriov_set_vf_vlan(efx, vf_i, vlan, qos);
 	} else {
 		return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/sfc/sriov.h b/drivers/net/ethernet/sfc/sriov.h
index 400df526586d..ba1762e7f216 100644
--- a/drivers/net/ethernet/sfc/sriov.h
+++ b/drivers/net/ethernet/sfc/sriov.h
@@ -16,7 +16,7 @@
 
 int efx_sriov_set_vf_mac(struct net_device *net_dev, int vf_i, u8 *mac);
 int efx_sriov_set_vf_vlan(struct net_device *net_dev, int vf_i, u16 vlan,
-			  u8 qos);
+			  u8 qos, __be16 vlan_proto);
 int efx_sriov_set_vf_spoofchk(struct net_device *net_dev, int vf_i,
 			      bool spoofchk);
 int efx_sriov_get_vf_config(struct net_device *net_dev, int vf_i,
diff --git a/drivers/net/ethernet/sfc/workarounds.h b/drivers/net/ethernet/sfc/workarounds.h
index 2310b75d4ec2..351cd14cb9f9 100644
--- a/drivers/net/ethernet/sfc/workarounds.h
+++ b/drivers/net/ethernet/sfc/workarounds.h
@@ -50,4 +50,8 @@
 #define EFX_WORKAROUND_35388(efx)					\
 	(efx_nic_rev(efx) == EFX_REV_HUNT_A0 && EFX_EF10_WORKAROUND_35388(efx))
 
+/* Moderation timer access must go through MCDI */
+#define EFX_EF10_WORKAROUND_61265(efx)					\
+	(((struct efx_ef10_nic_data *)efx->nic_data)->workaround_61265)
+
 #endif /* EFX_WORKAROUNDS_H */
diff --git a/drivers/net/ethernet/sis/sis900.c b/drivers/net/ethernet/sis/sis900.c
index 95001ee408ab..6f85276376e8 100644
--- a/drivers/net/ethernet/sis/sis900.c
+++ b/drivers/net/ethernet/sis/sis900.c
@@ -1426,7 +1426,7 @@ static void sis900_set_mode(struct sis900_private *sp, int speed, int duplex)
 		rx_flags |= RxATX;
 	}
 
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
 	/* Can accept Jumbo packet */
 	rx_flags |= RxAJAB;
 #endif
@@ -1750,7 +1750,7 @@ static int sis900_rx(struct net_device *net_dev)
 		data_size = rx_status & DSIZE;
 		rx_size = data_size - CRC_SIZE;
 
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
 		/* ``TOOLONG'' flag means jumbo packet received. */
 		if ((rx_status & TOOLONG) && data_size <= MAX_FRAME_SIZE)
 			rx_status &= (~ ((unsigned int)TOOLONG));
diff --git a/drivers/net/ethernet/sis/sis900.h b/drivers/net/ethernet/sis/sis900.h
index 7d430d322931..f0da3dc52c01 100644
--- a/drivers/net/ethernet/sis/sis900.h
+++ b/drivers/net/ethernet/sis/sis900.h
@@ -310,7 +310,7 @@ enum sis630_revision_id {
 #define CRC_SIZE                4
 #define MAC_HEADER_SIZE         14
 
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
 #define MAX_FRAME_SIZE  (1518 + 4)
 #else
 #define MAX_FRAME_SIZE  1518
diff --git a/drivers/net/ethernet/smsc/smc91x.c b/drivers/net/ethernet/smsc/smc91x.c
index 726b80f45906..73212590d04a 100644
--- a/drivers/net/ethernet/smsc/smc91x.c
+++ b/drivers/net/ethernet/smsc/smc91x.c
@@ -2275,6 +2275,13 @@ static int smc_drv_probe(struct platform_device *pdev)
 	if (pd) {
 		memcpy(&lp->cfg, pd, sizeof(lp->cfg));
 		lp->io_shift = SMC91X_IO_SHIFT(lp->cfg.flags);
+
+		if (!SMC_8BIT(lp) && !SMC_16BIT(lp)) {
+			dev_err(&pdev->dev,
+				"at least one of 8-bit or 16-bit access support is required.\n");
+			ret = -ENXIO;
+			goto out_free_netdev;
+		}
 	}
 
 #if IS_BUILTIN(CONFIG_OF)
@@ -2316,6 +2323,9 @@ static int smc_drv_probe(struct platform_device *pdev)
 		} else {
 			lp->cfg.flags |= SMC91X_USE_16BIT;
 		}
+		if (!device_property_read_u32(&pdev->dev, "reg-shift",
+					      &val))
+			lp->io_shift = val;
 	}
 #endif
 
diff --git a/drivers/net/ethernet/smsc/smc91x.h b/drivers/net/ethernet/smsc/smc91x.h
index 1a55c7976df0..ea8465467469 100644
--- a/drivers/net/ethernet/smsc/smc91x.h
+++ b/drivers/net/ethernet/smsc/smc91x.h
@@ -37,6 +37,27 @@
 #include <linux/smc91x.h>
 
 /*
+ * Any 16-bit access is performed with two 8-bit accesses if the hardware
+ * can't do it directly. Most registers are 16-bit so those are mandatory.
+ */
+#define SMC_outw_b(x, a, r)						\
+	do {								\
+		unsigned int __val16 = (x);				\
+		unsigned int __reg = (r);				\
+		SMC_outb(__val16, a, __reg);				\
+		SMC_outb(__val16 >> 8, a, __reg + (1 << SMC_IO_SHIFT));	\
+	} while (0)
+
+#define SMC_inw_b(a, r)							\
+	({								\
+		unsigned int __val16;					\
+		unsigned int __reg = r;					\
+		__val16  = SMC_inb(a, __reg);				\
+		__val16 |= SMC_inb(a, __reg + (1 << SMC_IO_SHIFT)) << 8; \
+		__val16;						\
+	})
+
+/*
  * Define your architecture specific bus configuration parameters here.
  */
 
@@ -55,10 +76,30 @@
 #define SMC_IO_SHIFT		(lp->io_shift)
 
 #define SMC_inb(a, r)		readb((a) + (r))
-#define SMC_inw(a, r)		readw((a) + (r))
+#define SMC_inw(a, r)							\
+	({								\
+		unsigned int __smc_r = r;				\
+		SMC_16BIT(lp) ? readw((a) + __smc_r) :			\
+		SMC_8BIT(lp) ? SMC_inw_b(a, __smc_r) :			\
+		({ BUG(); 0; });					\
+	})
+
 #define SMC_inl(a, r)		readl((a) + (r))
 #define SMC_outb(v, a, r)	writeb(v, (a) + (r))
+#define SMC_outw(v, a, r)						\
+	do {								\
+		unsigned int __v = v, __smc_r = r;			\
+		if (SMC_16BIT(lp))					\
+			__SMC_outw(__v, a, __smc_r);			\
+		else if (SMC_8BIT(lp))					\
+			SMC_outw_b(__v, a, __smc_r);			\
+		else							\
+			BUG();						\
+	} while (0)
+
 #define SMC_outl(v, a, r)	writel(v, (a) + (r))
+#define SMC_insb(a, r, p, l)	readsb((a) + (r), p, l)
+#define SMC_outsb(a, r, p, l)	writesb((a) + (r), p, l)
 #define SMC_insw(a, r, p, l)	readsw((a) + (r), p, l)
 #define SMC_outsw(a, r, p, l)	writesw((a) + (r), p, l)
 #define SMC_insl(a, r, p, l)	readsl((a) + (r), p, l)
@@ -66,7 +107,7 @@
 #define SMC_IRQ_FLAGS		(-1)	/* from resource */
 
 /* We actually can't write halfwords properly if not word aligned */
-static inline void SMC_outw(u16 val, void __iomem *ioaddr, int reg)
+static inline void __SMC_outw(u16 val, void __iomem *ioaddr, int reg)
 {
 	if ((machine_is_mainstone() || machine_is_stargate2() ||
 	     machine_is_pxa_idp()) && reg & 2) {
@@ -416,24 +457,8 @@ smc_pxa_dma_insw(void __iomem *ioaddr, struct smc_local *lp, int reg, int dma,
 
 #if ! SMC_CAN_USE_16BIT
 
-/*
- * Any 16-bit access is performed with two 8-bit accesses if the hardware
- * can't do it directly. Most registers are 16-bit so those are mandatory.
- */
-#define SMC_outw(x, ioaddr, reg)					\
-	do {								\
-		unsigned int __val16 = (x);				\
-		SMC_outb( __val16, ioaddr, reg );			\
-		SMC_outb( __val16 >> 8, ioaddr, reg + (1 << SMC_IO_SHIFT));\
-	} while (0)
-#define SMC_inw(ioaddr, reg)						\
-	({								\
-		unsigned int __val16;					\
-		__val16 =  SMC_inb( ioaddr, reg );			\
-		__val16 |= SMC_inb( ioaddr, reg + (1 << SMC_IO_SHIFT)) << 8; \
-		__val16;						\
-	})
-
+#define SMC_outw(x, ioaddr, reg)	SMC_outw_b(x, ioaddr, reg)
+#define SMC_inw(ioaddr, reg)		SMC_inw_b(ioaddr, reg)
 #define SMC_insw(a, r, p, l)		BUG()
 #define SMC_outsw(a, r, p, l)		BUG()
 
@@ -445,7 +470,9 @@ smc_pxa_dma_insw(void __iomem *ioaddr, struct smc_local *lp, int reg, int dma,
 #endif
 
 #if ! SMC_CAN_USE_8BIT
+#undef SMC_inb
 #define SMC_inb(ioaddr, reg)		({ BUG(); 0; })
+#undef SMC_outb
 #define SMC_outb(x, ioaddr, reg)	BUG()
 #define SMC_insb(a, r, p, l)		BUG()
 #define SMC_outsb(a, r, p, l)		BUG()
diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c
index ca3134540d2d..e9b8579e6241 100644
--- a/drivers/net/ethernet/smsc/smsc911x.c
+++ b/drivers/net/ethernet/smsc/smsc911x.c
@@ -62,6 +62,7 @@
 #include <linux/acpi.h>
 #include <linux/pm_runtime.h>
 #include <linux/property.h>
+#include <linux/gpio/consumer.h>
 
 #include "smsc911x.h"
 
@@ -147,6 +148,9 @@ struct smsc911x_data {
 	/* regulators */
 	struct regulator_bulk_data supplies[SMSC911X_NUM_SUPPLIES];
 
+	/* Reset GPIO */
+	struct gpio_desc *reset_gpiod;
+
 	/* clock */
 	struct clk *clk;
 };
@@ -438,6 +442,11 @@ static int smsc911x_request_resources(struct platform_device *pdev)
 		netdev_err(ndev, "couldn't get regulators %d\n",
 				ret);
 
+	/* Request optional RESET GPIO */
+	pdata->reset_gpiod = devm_gpiod_get_optional(&pdev->dev,
+						     "reset",
+						     GPIOD_OUT_LOW);
+
 	/* Request clock */
 	pdata->clk = clk_get(&pdev->dev, NULL);
 	if (IS_ERR(pdata->clk))
@@ -1099,15 +1108,8 @@ static int smsc911x_mii_init(struct platform_device *pdev,
 		goto err_out_free_bus_2;
 	}
 
-	if (smsc911x_mii_probe(dev) < 0) {
-		SMSC_WARN(pdata, probe, "Error registering mii bus");
-		goto err_out_unregister_bus_3;
-	}
-
 	return 0;
 
-err_out_unregister_bus_3:
-	mdiobus_unregister(pdata->mii_bus);
 err_out_free_bus_2:
 	mdiobus_free(pdata->mii_bus);
 err_out_1:
@@ -1514,23 +1516,90 @@ static void smsc911x_disable_irq_chip(struct net_device *dev)
 	smsc911x_reg_write(pdata, INT_STS, 0xFFFFFFFF);
 }
 
+static irqreturn_t smsc911x_irqhandler(int irq, void *dev_id)
+{
+	struct net_device *dev = dev_id;
+	struct smsc911x_data *pdata = netdev_priv(dev);
+	u32 intsts = smsc911x_reg_read(pdata, INT_STS);
+	u32 inten = smsc911x_reg_read(pdata, INT_EN);
+	int serviced = IRQ_NONE;
+	u32 temp;
+
+	if (unlikely(intsts & inten & INT_STS_SW_INT_)) {
+		temp = smsc911x_reg_read(pdata, INT_EN);
+		temp &= (~INT_EN_SW_INT_EN_);
+		smsc911x_reg_write(pdata, INT_EN, temp);
+		smsc911x_reg_write(pdata, INT_STS, INT_STS_SW_INT_);
+		pdata->software_irq_signal = 1;
+		smp_wmb();
+		serviced = IRQ_HANDLED;
+	}
+
+	if (unlikely(intsts & inten & INT_STS_RXSTOP_INT_)) {
+		/* Called when there is a multicast update scheduled and
+		 * it is now safe to complete the update */
+		SMSC_TRACE(pdata, intr, "RX Stop interrupt");
+		smsc911x_reg_write(pdata, INT_STS, INT_STS_RXSTOP_INT_);
+		if (pdata->multicast_update_pending)
+			smsc911x_rx_multicast_update_workaround(pdata);
+		serviced = IRQ_HANDLED;
+	}
+
+	if (intsts & inten & INT_STS_TDFA_) {
+		temp = smsc911x_reg_read(pdata, FIFO_INT);
+		temp |= FIFO_INT_TX_AVAIL_LEVEL_;
+		smsc911x_reg_write(pdata, FIFO_INT, temp);
+		smsc911x_reg_write(pdata, INT_STS, INT_STS_TDFA_);
+		netif_wake_queue(dev);
+		serviced = IRQ_HANDLED;
+	}
+
+	if (unlikely(intsts & inten & INT_STS_RXE_)) {
+		SMSC_TRACE(pdata, intr, "RX Error interrupt");
+		smsc911x_reg_write(pdata, INT_STS, INT_STS_RXE_);
+		serviced = IRQ_HANDLED;
+	}
+
+	if (likely(intsts & inten & INT_STS_RSFL_)) {
+		if (likely(napi_schedule_prep(&pdata->napi))) {
+			/* Disable Rx interrupts */
+			temp = smsc911x_reg_read(pdata, INT_EN);
+			temp &= (~INT_EN_RSFL_EN_);
+			smsc911x_reg_write(pdata, INT_EN, temp);
+			/* Schedule a NAPI poll */
+			__napi_schedule(&pdata->napi);
+		} else {
+			SMSC_WARN(pdata, rx_err, "napi_schedule_prep failed");
+		}
+		serviced = IRQ_HANDLED;
+	}
+
+	return serviced;
+}
+
 static int smsc911x_open(struct net_device *dev)
 {
 	struct smsc911x_data *pdata = netdev_priv(dev);
 	unsigned int timeout;
 	unsigned int temp;
 	unsigned int intcfg;
+	int retval;
+	int irq_flags;
 
-	/* if the phy is not yet registered, retry later*/
+	/* find and start the given phy */
 	if (!dev->phydev) {
-		SMSC_WARN(pdata, hw, "phy_dev is NULL");
-		return -EAGAIN;
+		retval = smsc911x_mii_probe(dev);
+		if (retval < 0) {
+			SMSC_WARN(pdata, probe, "Error starting phy");
+			goto out;
+		}
 	}
 
 	/* Reset the LAN911x */
-	if (smsc911x_soft_reset(pdata)) {
+	retval = smsc911x_soft_reset(pdata);
+	if (retval) {
 		SMSC_WARN(pdata, hw, "soft reset failed");
-		return -EIO;
+		goto mii_free_out;
 	}
 
 	smsc911x_reg_write(pdata, HW_CFG, 0x00050000);
@@ -1586,6 +1655,15 @@ static int smsc911x_open(struct net_device *dev)
 	pdata->software_irq_signal = 0;
 	smp_wmb();
 
+	irq_flags = irq_get_trigger_type(dev->irq);
+	retval = request_irq(dev->irq, smsc911x_irqhandler,
+			     irq_flags | IRQF_SHARED, dev->name, dev);
+	if (retval) {
+		SMSC_WARN(pdata, probe,
+			  "Unable to claim requested irq: %d", dev->irq);
+		goto mii_free_out;
+	}
+
 	temp = smsc911x_reg_read(pdata, INT_EN);
 	temp |= INT_EN_SW_INT_EN_;
 	smsc911x_reg_write(pdata, INT_EN, temp);
@@ -1600,7 +1678,8 @@ static int smsc911x_open(struct net_device *dev)
 	if (!pdata->software_irq_signal) {
 		netdev_warn(dev, "ISR failed signaling test (IRQ %d)\n",
 			    dev->irq);
-		return -ENODEV;
+		retval = -ENODEV;
+		goto irq_stop_out;
 	}
 	SMSC_TRACE(pdata, ifup, "IRQ handler passed test using IRQ %d",
 		   dev->irq);
@@ -1646,6 +1725,14 @@ static int smsc911x_open(struct net_device *dev)
 
 	netif_start_queue(dev);
 	return 0;
+
+irq_stop_out:
+	free_irq(dev->irq, dev);
+mii_free_out:
+	phy_disconnect(dev->phydev);
+	dev->phydev = NULL;
+out:
+	return retval;
 }
 
 /* Entry point for stopping the interface */
@@ -1667,9 +1754,15 @@ static int smsc911x_stop(struct net_device *dev)
 	dev->stats.rx_dropped += smsc911x_reg_read(pdata, RX_DROP);
 	smsc911x_tx_update_txcounters(dev);
 
+	free_irq(dev->irq, dev);
+
 	/* Bring the PHY down */
-	if (dev->phydev)
+	if (dev->phydev) {
 		phy_stop(dev->phydev);
+		phy_disconnect(dev->phydev);
+		dev->phydev = NULL;
+	}
+	netif_carrier_off(dev);
 
 	SMSC_TRACE(pdata, ifdown, "Interface stopped");
 	return 0;
@@ -1811,67 +1904,6 @@ static void smsc911x_set_multicast_list(struct net_device *dev)
 	spin_unlock_irqrestore(&pdata->mac_lock, flags);
 }
 
-static irqreturn_t smsc911x_irqhandler(int irq, void *dev_id)
-{
-	struct net_device *dev = dev_id;
-	struct smsc911x_data *pdata = netdev_priv(dev);
-	u32 intsts = smsc911x_reg_read(pdata, INT_STS);
-	u32 inten = smsc911x_reg_read(pdata, INT_EN);
-	int serviced = IRQ_NONE;
-	u32 temp;
-
-	if (unlikely(intsts & inten & INT_STS_SW_INT_)) {
-		temp = smsc911x_reg_read(pdata, INT_EN);
-		temp &= (~INT_EN_SW_INT_EN_);
-		smsc911x_reg_write(pdata, INT_EN, temp);
-		smsc911x_reg_write(pdata, INT_STS, INT_STS_SW_INT_);
-		pdata->software_irq_signal = 1;
-		smp_wmb();
-		serviced = IRQ_HANDLED;
-	}
-
-	if (unlikely(intsts & inten & INT_STS_RXSTOP_INT_)) {
-		/* Called when there is a multicast update scheduled and
-		 * it is now safe to complete the update */
-		SMSC_TRACE(pdata, intr, "RX Stop interrupt");
-		smsc911x_reg_write(pdata, INT_STS, INT_STS_RXSTOP_INT_);
-		if (pdata->multicast_update_pending)
-			smsc911x_rx_multicast_update_workaround(pdata);
-		serviced = IRQ_HANDLED;
-	}
-
-	if (intsts & inten & INT_STS_TDFA_) {
-		temp = smsc911x_reg_read(pdata, FIFO_INT);
-		temp |= FIFO_INT_TX_AVAIL_LEVEL_;
-		smsc911x_reg_write(pdata, FIFO_INT, temp);
-		smsc911x_reg_write(pdata, INT_STS, INT_STS_TDFA_);
-		netif_wake_queue(dev);
-		serviced = IRQ_HANDLED;
-	}
-
-	if (unlikely(intsts & inten & INT_STS_RXE_)) {
-		SMSC_TRACE(pdata, intr, "RX Error interrupt");
-		smsc911x_reg_write(pdata, INT_STS, INT_STS_RXE_);
-		serviced = IRQ_HANDLED;
-	}
-
-	if (likely(intsts & inten & INT_STS_RSFL_)) {
-		if (likely(napi_schedule_prep(&pdata->napi))) {
-			/* Disable Rx interrupts */
-			temp = smsc911x_reg_read(pdata, INT_EN);
-			temp &= (~INT_EN_RSFL_EN_);
-			smsc911x_reg_write(pdata, INT_EN, temp);
-			/* Schedule a NAPI poll */
-			__napi_schedule(&pdata->napi);
-		} else {
-			SMSC_WARN(pdata, rx_err, "napi_schedule_prep failed");
-		}
-		serviced = IRQ_HANDLED;
-	}
-
-	return serviced;
-}
-
 #ifdef CONFIG_NET_POLL_CONTROLLER
 static void smsc911x_poll_controller(struct net_device *dev)
 {
@@ -2291,16 +2323,14 @@ static int smsc911x_drv_remove(struct platform_device *pdev)
 	pdata = netdev_priv(dev);
 	BUG_ON(!pdata);
 	BUG_ON(!pdata->ioaddr);
-	BUG_ON(!dev->phydev);
+	WARN_ON(dev->phydev);
 
 	SMSC_TRACE(pdata, ifdown, "Stopping driver");
 
-	phy_disconnect(dev->phydev);
 	mdiobus_unregister(pdata->mii_bus);
 	mdiobus_free(pdata->mii_bus);
 
 	unregister_netdev(dev);
-	free_irq(dev->irq, dev);
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 					   "smsc911x-memory");
 	if (!res)
@@ -2385,8 +2415,7 @@ static int smsc911x_drv_probe(struct platform_device *pdev)
 	struct smsc911x_data *pdata;
 	struct smsc911x_platform_config *config = dev_get_platdata(&pdev->dev);
 	struct resource *res;
-	unsigned int intcfg = 0;
-	int res_size, irq, irq_flags;
+	int res_size, irq;
 	int retval;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
@@ -2425,7 +2454,6 @@ static int smsc911x_drv_probe(struct platform_device *pdev)
 
 	pdata = netdev_priv(dev);
 	dev->irq = irq;
-	irq_flags = irq_get_trigger_type(irq);
 	pdata->ioaddr = ioremap_nocache(res->start, res_size);
 
 	pdata->dev = dev;
@@ -2472,43 +2500,23 @@ static int smsc911x_drv_probe(struct platform_device *pdev)
 	if (retval < 0)
 		goto out_disable_resources;
 
-	/* configure irq polarity and type before connecting isr */
-	if (pdata->config.irq_polarity == SMSC911X_IRQ_POLARITY_ACTIVE_HIGH)
-		intcfg |= INT_CFG_IRQ_POL_;
-
-	if (pdata->config.irq_type == SMSC911X_IRQ_TYPE_PUSH_PULL)
-		intcfg |= INT_CFG_IRQ_TYPE_;
-
-	smsc911x_reg_write(pdata, INT_CFG, intcfg);
-
-	/* Ensure interrupts are globally disabled before connecting ISR */
-	smsc911x_disable_irq_chip(dev);
+	netif_carrier_off(dev);
 
-	retval = request_irq(dev->irq, smsc911x_irqhandler,
-			     irq_flags | IRQF_SHARED, dev->name, dev);
+	retval = smsc911x_mii_init(pdev, dev);
 	if (retval) {
-		SMSC_WARN(pdata, probe,
-			  "Unable to claim requested irq: %d", dev->irq);
+		SMSC_WARN(pdata, probe, "Error %i initialising mii", retval);
 		goto out_disable_resources;
 	}
 
-	netif_carrier_off(dev);
-
 	retval = register_netdev(dev);
 	if (retval) {
 		SMSC_WARN(pdata, probe, "Error %i registering device", retval);
-		goto out_free_irq;
+		goto out_disable_resources;
 	} else {
 		SMSC_TRACE(pdata, probe,
 			   "Network interface: \"%s\"", dev->name);
 	}
 
-	retval = smsc911x_mii_init(pdev, dev);
-	if (retval) {
-		SMSC_WARN(pdata, probe, "Error %i initialising mii", retval);
-		goto out_unregister_netdev_5;
-	}
-
 	spin_lock_irq(&pdata->mac_lock);
 
 	/* Check if mac address has been specified when bringing interface up */
@@ -2544,10 +2552,6 @@ static int smsc911x_drv_probe(struct platform_device *pdev)
 
 	return 0;
 
-out_unregister_netdev_5:
-	unregister_netdev(dev);
-out_free_irq:
-	free_irq(dev->irq, dev);
 out_disable_resources:
 	pm_runtime_put(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index 8f06a6621ab1..3818c5e06eba 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -61,13 +61,13 @@ config DWMAC_LPC18XX
 config DWMAC_MESON
 	tristate "Amlogic Meson dwmac support"
 	default ARCH_MESON
-	depends on OF && (ARCH_MESON || COMPILE_TEST)
+	depends on OF && COMMON_CLK && (ARCH_MESON || COMPILE_TEST)
 	help
 	  Support for Ethernet controller on Amlogic Meson SoCs.
 
 	  This selects the Amlogic Meson SoC glue layer support for
-	  the stmmac device driver. This driver is used for Meson6 and
-	  Meson8 SoCs.
+	  the stmmac device driver. This driver is used for Meson6,
+	  Meson8, Meson8b and GXBB SoCs.
 
 config DWMAC_ROCKCHIP
 	tristate "Rockchip dwmac support"
@@ -104,6 +104,18 @@ config DWMAC_STI
 	  device driver. This driver is used on for the STi series
 	  SOCs GMAC ethernet controller.
 
+config DWMAC_STM32
+	tristate "STM32 DWMAC support"
+	default ARCH_STM32
+	depends on OF && HAS_IOMEM
+	select MFD_SYSCON
+	---help---
+	  Support for ethernet controller on STM32 SOCs.
+
+	  This selects STM32 SoC glue layer support for the stmmac
+	  device driver. This driver is used on for the STM32 series
+	  SOCs GMAC ethernet controller.
+
 config DWMAC_SUNXI
 	tristate "Allwinner GMAC support"
 	default ARCH_SUNXI
diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile
index 44b630cd1755..5d6ece5919b3 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Makefile
+++ b/drivers/net/ethernet/stmicro/stmmac/Makefile
@@ -9,10 +9,11 @@ stmmac-objs:= stmmac_main.o stmmac_ethtool.o stmmac_mdio.o ring_mode.o	\
 obj-$(CONFIG_STMMAC_PLATFORM)	+= stmmac-platform.o
 obj-$(CONFIG_DWMAC_IPQ806X)	+= dwmac-ipq806x.o
 obj-$(CONFIG_DWMAC_LPC18XX)	+= dwmac-lpc18xx.o
-obj-$(CONFIG_DWMAC_MESON)	+= dwmac-meson.o
+obj-$(CONFIG_DWMAC_MESON)	+= dwmac-meson.o dwmac-meson8b.o
 obj-$(CONFIG_DWMAC_ROCKCHIP)	+= dwmac-rk.o
 obj-$(CONFIG_DWMAC_SOCFPGA)	+= dwmac-altr-socfpga.o
 obj-$(CONFIG_DWMAC_STI)		+= dwmac-sti.o
+obj-$(CONFIG_DWMAC_STM32)	+= dwmac-stm32.o
 obj-$(CONFIG_DWMAC_SUNXI)	+= dwmac-sunxi.o
 obj-$(CONFIG_DWMAC_GENERIC)	+= dwmac-generic.o
 stmmac-platform-objs:= stmmac_platform.o
diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h
index 2533b91f1421..d3292c4a6eda 100644
--- a/drivers/net/ethernet/stmicro/stmmac/common.h
+++ b/drivers/net/ethernet/stmicro/stmmac/common.h
@@ -30,7 +30,7 @@
 #include <linux/stmmac.h>
 #include <linux/phy.h>
 #include <linux/module.h>
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
 #define STMMAC_VLAN_TAG_USED
 #include <linux/if_vlan.h>
 #endif
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson.c
index c1bac1912b37..309d99536a2c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson.c
@@ -1,5 +1,5 @@
 /*
- * Amlogic Meson DWMAC glue layer
+ * Amlogic Meson6 and Meson8 DWMAC glue layer
  *
  * Copyright (C) 2014 Beniamino Galvani <b.galvani@gmail.com>
  *
@@ -96,5 +96,5 @@ static struct platform_driver meson6_dwmac_driver = {
 module_platform_driver(meson6_dwmac_driver);
 
 MODULE_AUTHOR("Beniamino Galvani <b.galvani@gmail.com>");
-MODULE_DESCRIPTION("Amlogic Meson DWMAC glue layer");
+MODULE_DESCRIPTION("Amlogic Meson6 and Meson8 DWMAC glue layer");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
new file mode 100644
index 000000000000..250e4ceafc8d
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
@@ -0,0 +1,324 @@
+/*
+ * Amlogic Meson8b and GXBB DWMAC glue layer
+ *
+ * Copyright (C) 2016 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/ethtool.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/module.h>
+#include <linux/of_net.h>
+#include <linux/mfd/syscon.h>
+#include <linux/platform_device.h>
+#include <linux/stmmac.h>
+
+#include "stmmac_platform.h"
+
+#define PRG_ETH0			0x0
+
+#define PRG_ETH0_RGMII_MODE		BIT(0)
+
+/* mux to choose between fclk_div2 (bit unset) and mpll2 (bit set) */
+#define PRG_ETH0_CLK_M250_SEL_SHIFT	4
+#define PRG_ETH0_CLK_M250_SEL_MASK	GENMASK(4, 4)
+
+#define PRG_ETH0_TXDLY_SHIFT		5
+#define PRG_ETH0_TXDLY_MASK		GENMASK(6, 5)
+#define PRG_ETH0_TXDLY_OFF		(0x0 << PRG_ETH0_TXDLY_SHIFT)
+#define PRG_ETH0_TXDLY_QUARTER		(0x1 << PRG_ETH0_TXDLY_SHIFT)
+#define PRG_ETH0_TXDLY_HALF		(0x2 << PRG_ETH0_TXDLY_SHIFT)
+#define PRG_ETH0_TXDLY_THREE_QUARTERS	(0x3 << PRG_ETH0_TXDLY_SHIFT)
+
+/* divider for the result of m250_sel */
+#define PRG_ETH0_CLK_M250_DIV_SHIFT	7
+#define PRG_ETH0_CLK_M250_DIV_WIDTH	3
+
+/* divides the result of m25_sel by either 5 (bit unset) or 10 (bit set) */
+#define PRG_ETH0_CLK_M25_DIV_SHIFT	10
+#define PRG_ETH0_CLK_M25_DIV_WIDTH	1
+
+#define PRG_ETH0_INVERTED_RMII_CLK	BIT(11)
+#define PRG_ETH0_TX_AND_PHY_REF_CLK	BIT(12)
+
+#define MUX_CLK_NUM_PARENTS		2
+
+struct meson8b_dwmac {
+	struct platform_device	*pdev;
+
+	void __iomem		*regs;
+
+	phy_interface_t		phy_mode;
+
+	struct clk_mux		m250_mux;
+	struct clk		*m250_mux_clk;
+	struct clk		*m250_mux_parent[MUX_CLK_NUM_PARENTS];
+
+	struct clk_divider	m250_div;
+	struct clk		*m250_div_clk;
+
+	struct clk_divider	m25_div;
+	struct clk		*m25_div_clk;
+};
+
+static void meson8b_dwmac_mask_bits(struct meson8b_dwmac *dwmac, u32 reg,
+				    u32 mask, u32 value)
+{
+	u32 data;
+
+	data = readl(dwmac->regs + reg);
+	data &= ~mask;
+	data |= (value & mask);
+
+	writel(data, dwmac->regs + reg);
+}
+
+static int meson8b_init_clk(struct meson8b_dwmac *dwmac)
+{
+	struct clk_init_data init;
+	int i, ret;
+	struct device *dev = &dwmac->pdev->dev;
+	char clk_name[32];
+	const char *clk_div_parents[1];
+	const char *mux_parent_names[MUX_CLK_NUM_PARENTS];
+	static struct clk_div_table clk_25m_div_table[] = {
+		{ .val = 0, .div = 5 },
+		{ .val = 1, .div = 10 },
+		{ /* sentinel */ },
+	};
+
+	/* get the mux parents from DT */
+	for (i = 0; i < MUX_CLK_NUM_PARENTS; i++) {
+		char name[16];
+
+		snprintf(name, sizeof(name), "clkin%d", i);
+		dwmac->m250_mux_parent[i] = devm_clk_get(dev, name);
+		if (IS_ERR(dwmac->m250_mux_parent[i])) {
+			ret = PTR_ERR(dwmac->m250_mux_parent[i]);
+			if (ret != -EPROBE_DEFER)
+				dev_err(dev, "Missing clock %s\n", name);
+			return ret;
+		}
+
+		mux_parent_names[i] =
+			__clk_get_name(dwmac->m250_mux_parent[i]);
+	}
+
+	/* create the m250_mux */
+	snprintf(clk_name, sizeof(clk_name), "%s#m250_sel", dev_name(dev));
+	init.name = clk_name;
+	init.ops = &clk_mux_ops;
+	init.flags = 0;
+	init.parent_names = mux_parent_names;
+	init.num_parents = MUX_CLK_NUM_PARENTS;
+
+	dwmac->m250_mux.reg = dwmac->regs + PRG_ETH0;
+	dwmac->m250_mux.shift = PRG_ETH0_CLK_M250_SEL_SHIFT;
+	dwmac->m250_mux.mask = PRG_ETH0_CLK_M250_SEL_MASK;
+	dwmac->m250_mux.flags = 0;
+	dwmac->m250_mux.table = NULL;
+	dwmac->m250_mux.hw.init = &init;
+
+	dwmac->m250_mux_clk = devm_clk_register(dev, &dwmac->m250_mux.hw);
+	if (WARN_ON(IS_ERR(dwmac->m250_mux_clk)))
+		return PTR_ERR(dwmac->m250_mux_clk);
+
+	/* create the m250_div */
+	snprintf(clk_name, sizeof(clk_name), "%s#m250_div", dev_name(dev));
+	init.name = devm_kstrdup(dev, clk_name, GFP_KERNEL);
+	init.ops = &clk_divider_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	clk_div_parents[0] = __clk_get_name(dwmac->m250_mux_clk);
+	init.parent_names = clk_div_parents;
+	init.num_parents = ARRAY_SIZE(clk_div_parents);
+
+	dwmac->m250_div.reg = dwmac->regs + PRG_ETH0;
+	dwmac->m250_div.shift = PRG_ETH0_CLK_M250_DIV_SHIFT;
+	dwmac->m250_div.width = PRG_ETH0_CLK_M250_DIV_WIDTH;
+	dwmac->m250_div.hw.init = &init;
+	dwmac->m250_div.flags = CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO;
+
+	dwmac->m250_div_clk = devm_clk_register(dev, &dwmac->m250_div.hw);
+	if (WARN_ON(IS_ERR(dwmac->m250_div_clk)))
+		return PTR_ERR(dwmac->m250_div_clk);
+
+	/* create the m25_div */
+	snprintf(clk_name, sizeof(clk_name), "%s#m25_div", dev_name(dev));
+	init.name = devm_kstrdup(dev, clk_name, GFP_KERNEL);
+	init.ops = &clk_divider_ops;
+	init.flags = CLK_IS_BASIC | CLK_SET_RATE_PARENT;
+	clk_div_parents[0] = __clk_get_name(dwmac->m250_div_clk);
+	init.parent_names = clk_div_parents;
+	init.num_parents = ARRAY_SIZE(clk_div_parents);
+
+	dwmac->m25_div.reg = dwmac->regs + PRG_ETH0;
+	dwmac->m25_div.shift = PRG_ETH0_CLK_M25_DIV_SHIFT;
+	dwmac->m25_div.width = PRG_ETH0_CLK_M25_DIV_WIDTH;
+	dwmac->m25_div.table = clk_25m_div_table;
+	dwmac->m25_div.hw.init = &init;
+	dwmac->m25_div.flags = CLK_DIVIDER_ALLOW_ZERO;
+
+	dwmac->m25_div_clk = devm_clk_register(dev, &dwmac->m25_div.hw);
+	if (WARN_ON(IS_ERR(dwmac->m25_div_clk)))
+		return PTR_ERR(dwmac->m25_div_clk);
+
+	return 0;
+}
+
+static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac)
+{
+	int ret;
+	unsigned long clk_rate;
+
+	switch (dwmac->phy_mode) {
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		/* Generate a 25MHz clock for the PHY */
+		clk_rate = 25 * 1000 * 1000;
+
+		/* enable RGMII mode */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_RGMII_MODE,
+					PRG_ETH0_RGMII_MODE);
+
+		/* only relevant for RMII mode -> disable in RGMII mode */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0,
+					PRG_ETH0_INVERTED_RMII_CLK, 0);
+
+		/* TX clock delay - all known boards use a 1/4 cycle delay */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_TXDLY_MASK,
+					PRG_ETH0_TXDLY_QUARTER);
+		break;
+
+	case PHY_INTERFACE_MODE_RMII:
+		/* Use the rate of the mux clock for the internal RMII PHY */
+		clk_rate = clk_get_rate(dwmac->m250_mux_clk);
+
+		/* disable RGMII mode -> enables RMII mode */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_RGMII_MODE,
+					0);
+
+		/* invert internal clk_rmii_i to generate 25/2.5 tx_rx_clk */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0,
+					PRG_ETH0_INVERTED_RMII_CLK,
+					PRG_ETH0_INVERTED_RMII_CLK);
+
+		/* TX clock delay cannot be configured in RMII mode */
+		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_TXDLY_MASK,
+					0);
+
+		break;
+
+	default:
+		dev_err(&dwmac->pdev->dev, "unsupported phy-mode %s\n",
+			phy_modes(dwmac->phy_mode));
+		return -EINVAL;
+	}
+
+	ret = clk_prepare_enable(dwmac->m25_div_clk);
+	if (ret) {
+		dev_err(&dwmac->pdev->dev, "failed to enable the PHY clock\n");
+		return ret;
+	}
+
+	ret = clk_set_rate(dwmac->m25_div_clk, clk_rate);
+	if (ret) {
+		clk_disable_unprepare(dwmac->m25_div_clk);
+
+		dev_err(&dwmac->pdev->dev, "failed to set PHY clock\n");
+		return ret;
+	}
+
+	/* enable TX_CLK and PHY_REF_CLK generator */
+	meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_TX_AND_PHY_REF_CLK,
+				PRG_ETH0_TX_AND_PHY_REF_CLK);
+
+	return 0;
+}
+
+static int meson8b_dwmac_probe(struct platform_device *pdev)
+{
+	struct plat_stmmacenet_data *plat_dat;
+	struct stmmac_resources stmmac_res;
+	struct resource *res;
+	struct meson8b_dwmac *dwmac;
+	int ret;
+
+	ret = stmmac_get_platform_resources(pdev, &stmmac_res);
+	if (ret)
+		return ret;
+
+	plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return PTR_ERR(plat_dat);
+
+	dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL);
+	if (!dwmac)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	dwmac->regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(dwmac->regs))
+		return PTR_ERR(dwmac->regs);
+
+	dwmac->pdev = pdev;
+	dwmac->phy_mode = of_get_phy_mode(pdev->dev.of_node);
+	if (dwmac->phy_mode < 0) {
+		dev_err(&pdev->dev, "missing phy-mode property\n");
+		return -EINVAL;
+	}
+
+	ret = meson8b_init_clk(dwmac);
+	if (ret)
+		return ret;
+
+	ret = meson8b_init_prg_eth(dwmac);
+	if (ret)
+		return ret;
+
+	plat_dat->bsp_priv = dwmac;
+
+	return stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res);
+}
+
+static int meson8b_dwmac_remove(struct platform_device *pdev)
+{
+	struct meson8b_dwmac *dwmac = get_stmmac_bsp_priv(&pdev->dev);
+
+	clk_disable_unprepare(dwmac->m25_div_clk);
+
+	return stmmac_pltfr_remove(pdev);
+}
+
+static const struct of_device_id meson8b_dwmac_match[] = {
+	{ .compatible = "amlogic,meson8b-dwmac" },
+	{ .compatible = "amlogic,meson-gxbb-dwmac" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, meson8b_dwmac_match);
+
+static struct platform_driver meson8b_dwmac_driver = {
+	.probe  = meson8b_dwmac_probe,
+	.remove = meson8b_dwmac_remove,
+	.driver = {
+		.name           = "meson8b-dwmac",
+		.pm		= &stmmac_pltfr_pm_ops,
+		.of_match_table = meson8b_dwmac_match,
+	},
+};
+module_platform_driver(meson8b_dwmac_driver);
+
+MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
+MODULE_DESCRIPTION("Amlogic Meson8b and GXBB DWMAC glue layer");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
index 92105916ef40..3740a4417fa0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
@@ -30,6 +30,7 @@
 #include <linux/delay.h>
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
+#include <linux/pm_runtime.h>
 
 #include "stmmac_platform.h"
 
@@ -301,6 +302,118 @@ static const struct rk_gmac_ops rk3288_ops = {
 	.set_rmii_speed = rk3288_set_rmii_speed,
 };
 
+#define RK3366_GRF_SOC_CON6	0x0418
+#define RK3366_GRF_SOC_CON7	0x041c
+
+/* RK3366_GRF_SOC_CON6 */
+#define RK3366_GMAC_PHY_INTF_SEL_RGMII	(GRF_BIT(9) | GRF_CLR_BIT(10) | \
+					 GRF_CLR_BIT(11))
+#define RK3366_GMAC_PHY_INTF_SEL_RMII	(GRF_CLR_BIT(9) | GRF_CLR_BIT(10) | \
+					 GRF_BIT(11))
+#define RK3366_GMAC_FLOW_CTRL		GRF_BIT(8)
+#define RK3366_GMAC_FLOW_CTRL_CLR	GRF_CLR_BIT(8)
+#define RK3366_GMAC_SPEED_10M		GRF_CLR_BIT(7)
+#define RK3366_GMAC_SPEED_100M		GRF_BIT(7)
+#define RK3366_GMAC_RMII_CLK_25M	GRF_BIT(3)
+#define RK3366_GMAC_RMII_CLK_2_5M	GRF_CLR_BIT(3)
+#define RK3366_GMAC_CLK_125M		(GRF_CLR_BIT(4) | GRF_CLR_BIT(5))
+#define RK3366_GMAC_CLK_25M		(GRF_BIT(4) | GRF_BIT(5))
+#define RK3366_GMAC_CLK_2_5M		(GRF_CLR_BIT(4) | GRF_BIT(5))
+#define RK3366_GMAC_RMII_MODE		GRF_BIT(6)
+#define RK3366_GMAC_RMII_MODE_CLR	GRF_CLR_BIT(6)
+
+/* RK3366_GRF_SOC_CON7 */
+#define RK3366_GMAC_TXCLK_DLY_ENABLE	GRF_BIT(7)
+#define RK3366_GMAC_TXCLK_DLY_DISABLE	GRF_CLR_BIT(7)
+#define RK3366_GMAC_RXCLK_DLY_ENABLE	GRF_BIT(15)
+#define RK3366_GMAC_RXCLK_DLY_DISABLE	GRF_CLR_BIT(15)
+#define RK3366_GMAC_CLK_RX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 8)
+#define RK3366_GMAC_CLK_TX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 0)
+
+static void rk3366_set_to_rgmii(struct rk_priv_data *bsp_priv,
+				int tx_delay, int rx_delay)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	regmap_write(bsp_priv->grf, RK3366_GRF_SOC_CON6,
+		     RK3366_GMAC_PHY_INTF_SEL_RGMII |
+		     RK3366_GMAC_RMII_MODE_CLR);
+	regmap_write(bsp_priv->grf, RK3366_GRF_SOC_CON7,
+		     RK3366_GMAC_RXCLK_DLY_ENABLE |
+		     RK3366_GMAC_TXCLK_DLY_ENABLE |
+		     RK3366_GMAC_CLK_RX_DL_CFG(rx_delay) |
+		     RK3366_GMAC_CLK_TX_DL_CFG(tx_delay));
+}
+
+static void rk3366_set_to_rmii(struct rk_priv_data *bsp_priv)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	regmap_write(bsp_priv->grf, RK3366_GRF_SOC_CON6,
+		     RK3366_GMAC_PHY_INTF_SEL_RMII | RK3366_GMAC_RMII_MODE);
+}
+
+static void rk3366_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	if (speed == 10)
+		regmap_write(bsp_priv->grf, RK3366_GRF_SOC_CON6,
+			     RK3366_GMAC_CLK_2_5M);
+	else if (speed == 100)
+		regmap_write(bsp_priv->grf, RK3366_GRF_SOC_CON6,
+			     RK3366_GMAC_CLK_25M);
+	else if (speed == 1000)
+		regmap_write(bsp_priv->grf, RK3366_GRF_SOC_CON6,
+			     RK3366_GMAC_CLK_125M);
+	else
+		dev_err(dev, "unknown speed value for RGMII! speed=%d", speed);
+}
+
+static void rk3366_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	if (speed == 10) {
+		regmap_write(bsp_priv->grf, RK3366_GRF_SOC_CON6,
+			     RK3366_GMAC_RMII_CLK_2_5M |
+			     RK3366_GMAC_SPEED_10M);
+	} else if (speed == 100) {
+		regmap_write(bsp_priv->grf, RK3366_GRF_SOC_CON6,
+			     RK3366_GMAC_RMII_CLK_25M |
+			     RK3366_GMAC_SPEED_100M);
+	} else {
+		dev_err(dev, "unknown speed value for RMII! speed=%d", speed);
+	}
+}
+
+static const struct rk_gmac_ops rk3366_ops = {
+	.set_to_rgmii = rk3366_set_to_rgmii,
+	.set_to_rmii = rk3366_set_to_rmii,
+	.set_rgmii_speed = rk3366_set_rgmii_speed,
+	.set_rmii_speed = rk3366_set_rmii_speed,
+};
+
 #define RK3368_GRF_SOC_CON15	0x043c
 #define RK3368_GRF_SOC_CON16	0x0440
 
@@ -413,6 +526,118 @@ static const struct rk_gmac_ops rk3368_ops = {
 	.set_rmii_speed = rk3368_set_rmii_speed,
 };
 
+#define RK3399_GRF_SOC_CON5	0xc214
+#define RK3399_GRF_SOC_CON6	0xc218
+
+/* RK3399_GRF_SOC_CON5 */
+#define RK3399_GMAC_PHY_INTF_SEL_RGMII	(GRF_BIT(9) | GRF_CLR_BIT(10) | \
+					 GRF_CLR_BIT(11))
+#define RK3399_GMAC_PHY_INTF_SEL_RMII	(GRF_CLR_BIT(9) | GRF_CLR_BIT(10) | \
+					 GRF_BIT(11))
+#define RK3399_GMAC_FLOW_CTRL		GRF_BIT(8)
+#define RK3399_GMAC_FLOW_CTRL_CLR	GRF_CLR_BIT(8)
+#define RK3399_GMAC_SPEED_10M		GRF_CLR_BIT(7)
+#define RK3399_GMAC_SPEED_100M		GRF_BIT(7)
+#define RK3399_GMAC_RMII_CLK_25M	GRF_BIT(3)
+#define RK3399_GMAC_RMII_CLK_2_5M	GRF_CLR_BIT(3)
+#define RK3399_GMAC_CLK_125M		(GRF_CLR_BIT(4) | GRF_CLR_BIT(5))
+#define RK3399_GMAC_CLK_25M		(GRF_BIT(4) | GRF_BIT(5))
+#define RK3399_GMAC_CLK_2_5M		(GRF_CLR_BIT(4) | GRF_BIT(5))
+#define RK3399_GMAC_RMII_MODE		GRF_BIT(6)
+#define RK3399_GMAC_RMII_MODE_CLR	GRF_CLR_BIT(6)
+
+/* RK3399_GRF_SOC_CON6 */
+#define RK3399_GMAC_TXCLK_DLY_ENABLE	GRF_BIT(7)
+#define RK3399_GMAC_TXCLK_DLY_DISABLE	GRF_CLR_BIT(7)
+#define RK3399_GMAC_RXCLK_DLY_ENABLE	GRF_BIT(15)
+#define RK3399_GMAC_RXCLK_DLY_DISABLE	GRF_CLR_BIT(15)
+#define RK3399_GMAC_CLK_RX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 8)
+#define RK3399_GMAC_CLK_TX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 0)
+
+static void rk3399_set_to_rgmii(struct rk_priv_data *bsp_priv,
+				int tx_delay, int rx_delay)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	regmap_write(bsp_priv->grf, RK3399_GRF_SOC_CON5,
+		     RK3399_GMAC_PHY_INTF_SEL_RGMII |
+		     RK3399_GMAC_RMII_MODE_CLR);
+	regmap_write(bsp_priv->grf, RK3399_GRF_SOC_CON6,
+		     RK3399_GMAC_RXCLK_DLY_ENABLE |
+		     RK3399_GMAC_TXCLK_DLY_ENABLE |
+		     RK3399_GMAC_CLK_RX_DL_CFG(rx_delay) |
+		     RK3399_GMAC_CLK_TX_DL_CFG(tx_delay));
+}
+
+static void rk3399_set_to_rmii(struct rk_priv_data *bsp_priv)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	regmap_write(bsp_priv->grf, RK3399_GRF_SOC_CON5,
+		     RK3399_GMAC_PHY_INTF_SEL_RMII | RK3399_GMAC_RMII_MODE);
+}
+
+static void rk3399_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	if (speed == 10)
+		regmap_write(bsp_priv->grf, RK3399_GRF_SOC_CON5,
+			     RK3399_GMAC_CLK_2_5M);
+	else if (speed == 100)
+		regmap_write(bsp_priv->grf, RK3399_GRF_SOC_CON5,
+			     RK3399_GMAC_CLK_25M);
+	else if (speed == 1000)
+		regmap_write(bsp_priv->grf, RK3399_GRF_SOC_CON5,
+			     RK3399_GMAC_CLK_125M);
+	else
+		dev_err(dev, "unknown speed value for RGMII! speed=%d", speed);
+}
+
+static void rk3399_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	if (speed == 10) {
+		regmap_write(bsp_priv->grf, RK3399_GRF_SOC_CON5,
+			     RK3399_GMAC_RMII_CLK_2_5M |
+			     RK3399_GMAC_SPEED_10M);
+	} else if (speed == 100) {
+		regmap_write(bsp_priv->grf, RK3399_GRF_SOC_CON5,
+			     RK3399_GMAC_RMII_CLK_25M |
+			     RK3399_GMAC_SPEED_100M);
+	} else {
+		dev_err(dev, "unknown speed value for RMII! speed=%d", speed);
+	}
+}
+
+static const struct rk_gmac_ops rk3399_ops = {
+	.set_to_rgmii = rk3399_set_to_rgmii,
+	.set_to_rmii = rk3399_set_to_rmii,
+	.set_rgmii_speed = rk3399_set_rgmii_speed,
+	.set_rmii_speed = rk3399_set_rmii_speed,
+};
+
 static int gmac_clk_init(struct rk_priv_data *bsp_priv)
 {
 	struct device *dev = &bsp_priv->pdev->dev;
@@ -629,6 +854,16 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev,
 							"rockchip,grf");
 	bsp_priv->pdev = pdev;
 
+	gmac_clk_init(bsp_priv);
+
+	return bsp_priv;
+}
+
+static int rk_gmac_powerup(struct rk_priv_data *bsp_priv)
+{
+	int ret;
+	struct device *dev = &bsp_priv->pdev->dev;
+
 	/*rmii or rgmii*/
 	if (bsp_priv->phy_iface == PHY_INTERFACE_MODE_RGMII) {
 		dev_info(dev, "init for RGMII\n");
@@ -641,15 +876,6 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev,
 		dev_err(dev, "NO interface defined!\n");
 	}
 
-	gmac_clk_init(bsp_priv);
-
-	return bsp_priv;
-}
-
-static int rk_gmac_powerup(struct rk_priv_data *bsp_priv)
-{
-	int ret;
-
 	ret = phy_power_on(bsp_priv, true);
 	if (ret)
 		return ret;
@@ -658,11 +884,19 @@ static int rk_gmac_powerup(struct rk_priv_data *bsp_priv)
 	if (ret)
 		return ret;
 
+	pm_runtime_enable(dev);
+	pm_runtime_get_sync(dev);
+
 	return 0;
 }
 
 static void rk_gmac_powerdown(struct rk_priv_data *gmac)
 {
+	struct device *dev = &gmac->pdev->dev;
+
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+
 	phy_power_on(gmac, false);
 	gmac_clk_enable(gmac, false);
 }
@@ -760,7 +994,9 @@ static int rk_gmac_probe(struct platform_device *pdev)
 static const struct of_device_id rk_gmac_dwmac_match[] = {
 	{ .compatible = "rockchip,rk3228-gmac", .data = &rk3228_ops },
 	{ .compatible = "rockchip,rk3288-gmac", .data = &rk3288_ops },
+	{ .compatible = "rockchip,rk3366-gmac", .data = &rk3366_ops },
 	{ .compatible = "rockchip,rk3368-gmac", .data = &rk3368_ops },
+	{ .compatible = "rockchip,rk3399-gmac", .data = &rk3399_ops },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, rk_gmac_dwmac_match);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c
new file mode 100644
index 000000000000..e5a926b8bee7
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-stm32.c
@@ -0,0 +1,194 @@
+/*
+ * dwmac-stm32.c - DWMAC Specific Glue layer for STM32 MCU
+ *
+ * Copyright (C) Alexandre Torgue 2015
+ * Author:  Alexandre Torgue <alexandre.torgue@gmail.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_net.h>
+#include <linux/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/stmmac.h>
+
+#include "stmmac_platform.h"
+
+#define MII_PHY_SEL_MASK	BIT(23)
+
+struct stm32_dwmac {
+	struct clk *clk_tx;
+	struct clk *clk_rx;
+	u32 mode_reg;		/* MAC glue-logic mode register */
+	struct regmap *regmap;
+	u32 speed;
+};
+
+static int stm32_dwmac_init(struct plat_stmmacenet_data *plat_dat)
+{
+	struct stm32_dwmac *dwmac = plat_dat->bsp_priv;
+	u32 reg = dwmac->mode_reg;
+	u32 val;
+	int ret;
+
+	val = (plat_dat->interface == PHY_INTERFACE_MODE_MII) ? 0 : 1;
+	ret = regmap_update_bits(dwmac->regmap, reg, MII_PHY_SEL_MASK, val);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(dwmac->clk_tx);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(dwmac->clk_rx);
+	if (ret)
+		clk_disable_unprepare(dwmac->clk_tx);
+
+	return ret;
+}
+
+static void stm32_dwmac_clk_disable(struct stm32_dwmac *dwmac)
+{
+	clk_disable_unprepare(dwmac->clk_tx);
+	clk_disable_unprepare(dwmac->clk_rx);
+}
+
+static int stm32_dwmac_parse_data(struct stm32_dwmac *dwmac,
+				  struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+	int err;
+
+	/*  Get TX/RX clocks */
+	dwmac->clk_tx = devm_clk_get(dev, "mac-clk-tx");
+	if (IS_ERR(dwmac->clk_tx)) {
+		dev_err(dev, "No tx clock provided...\n");
+		return PTR_ERR(dwmac->clk_tx);
+	}
+	dwmac->clk_rx = devm_clk_get(dev, "mac-clk-rx");
+	if (IS_ERR(dwmac->clk_rx)) {
+		dev_err(dev, "No rx clock provided...\n");
+		return PTR_ERR(dwmac->clk_rx);
+	}
+
+	/* Get mode register */
+	dwmac->regmap = syscon_regmap_lookup_by_phandle(np, "st,syscon");
+	if (IS_ERR(dwmac->regmap))
+		return PTR_ERR(dwmac->regmap);
+
+	err = of_property_read_u32_index(np, "st,syscon", 1, &dwmac->mode_reg);
+	if (err)
+		dev_err(dev, "Can't get sysconfig mode offset (%d)\n", err);
+
+	return err;
+}
+
+static int stm32_dwmac_probe(struct platform_device *pdev)
+{
+	struct plat_stmmacenet_data *plat_dat;
+	struct stmmac_resources stmmac_res;
+	struct stm32_dwmac *dwmac;
+	int ret;
+
+	ret = stmmac_get_platform_resources(pdev, &stmmac_res);
+	if (ret)
+		return ret;
+
+	plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return PTR_ERR(plat_dat);
+
+	dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL);
+	if (!dwmac)
+		return -ENOMEM;
+
+	ret = stm32_dwmac_parse_data(dwmac, &pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "Unable to parse OF data\n");
+		return ret;
+	}
+
+	plat_dat->bsp_priv = dwmac;
+
+	ret = stm32_dwmac_init(plat_dat);
+	if (ret)
+		return ret;
+
+	ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res);
+	if (ret)
+		stm32_dwmac_clk_disable(dwmac);
+
+	return ret;
+}
+
+static int stm32_dwmac_remove(struct platform_device *pdev)
+{
+	struct net_device *ndev = platform_get_drvdata(pdev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	int ret = stmmac_dvr_remove(&pdev->dev);
+
+	stm32_dwmac_clk_disable(priv->plat->bsp_priv);
+
+	return ret;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int stm32_dwmac_suspend(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	int ret;
+
+	ret = stmmac_suspend(dev);
+	stm32_dwmac_clk_disable(priv->plat->bsp_priv);
+
+	return ret;
+}
+
+static int stm32_dwmac_resume(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	int ret;
+
+	ret = stm32_dwmac_init(priv->plat);
+	if (ret)
+		return ret;
+
+	ret = stmmac_resume(dev);
+
+	return ret;
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static SIMPLE_DEV_PM_OPS(stm32_dwmac_pm_ops,
+	stm32_dwmac_suspend, stm32_dwmac_resume);
+
+static const struct of_device_id stm32_dwmac_match[] = {
+	{ .compatible = "st,stm32-dwmac"},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, stm32_dwmac_match);
+
+static struct platform_driver stm32_dwmac_driver = {
+	.probe  = stm32_dwmac_probe,
+	.remove = stm32_dwmac_remove,
+	.driver = {
+		.name           = "stm32-dwmac",
+		.pm		= &stm32_dwmac_pm_ops,
+		.of_match_table = stm32_dwmac_match,
+	},
+};
+module_platform_driver(stm32_dwmac_driver);
+
+MODULE_AUTHOR("Alexandre Torgue <alexandre.torgue@gmail.com>");
+MODULE_DESCRIPTION("STMicroelectronics MCU DWMAC Specific Glue layer");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
index cbefe9e2207c..7df4ff158f3d 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
@@ -145,7 +145,7 @@ static void dwmac1000_set_mchash(void __iomem *ioaddr, u32 *mcfilterbits,
 		numhashregs = 8;
 		break;
 	default:
-		pr_debug("STMMAC: err in setting mulitcast filter\n");
+		pr_debug("STMMAC: err in setting multicast filter\n");
 		return;
 		break;
 	}
@@ -261,7 +261,7 @@ static void dwmac1000_pmt(struct mac_device_info *hw, unsigned long mode)
 	}
 	if (mode & WAKE_UCAST) {
 		pr_debug("GMAC: WOL on global unicast\n");
-		pmt |= global_unicast;
+		pmt |= power_down | global_unicast | wake_up_frame_en;
 	}
 
 	writel(pmt, ioaddr + GMAC_PMT);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
index df5580dcdfed..51019b794be5 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
@@ -102,7 +102,7 @@ static void dwmac4_pmt(struct mac_device_info *hw, unsigned long mode)
 	}
 	if (mode & WAKE_UCAST) {
 		pr_debug("GMAC: WOL on global unicast\n");
-		pmt |= global_unicast;
+		pmt |= power_down | global_unicast | wake_up_frame_en;
 	}
 
 	writel(pmt, ioaddr + GMAC_PMT);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
index 756bb548e81a..0a0d6a86f397 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
@@ -265,6 +265,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, const char **mac)
 	 * once needed on other platforms.
 	 */
 	if (of_device_is_compatible(np, "st,spear600-gmac") ||
+		of_device_is_compatible(np, "snps,dwmac-3.50a") ||
 		of_device_is_compatible(np, "snps,dwmac-3.70a") ||
 		of_device_is_compatible(np, "snps,dwmac")) {
 		/* Note that the max-frame-size parameter as defined in the
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.h b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.h
index ffeb8d9e2b2e..64e147f53a9c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.h
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.h
@@ -30,4 +30,12 @@ int stmmac_get_platform_resources(struct platform_device *pdev,
 int stmmac_pltfr_remove(struct platform_device *pdev);
 extern const struct dev_pm_ops stmmac_pltfr_pm_ops;
 
+static inline void *get_stmmac_bsp_priv(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	return priv->plat->bsp_priv;
+}
+
 #endif /* __STMMAC_PLATFORM_H__ */
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
index 170a18b61281..6e3b82972ce8 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
@@ -187,7 +187,7 @@ int stmmac_ptp_register(struct stmmac_priv *priv)
 	if (IS_ERR(priv->ptp_clock)) {
 		priv->ptp_clock = NULL;
 		pr_err("ptp_clock_register() failed on %s\n", priv->dev->name);
-	} else
+	} else if (priv->ptp_clock)
 		pr_debug("Added PTP HW clock successfully on %s\n",
 			 priv->dev->name);
 
diff --git a/drivers/net/ethernet/sun/ldmvsw.c b/drivers/net/ethernet/sun/ldmvsw.c
index e15bf84fc6b2..0ac449acaf5b 100644
--- a/drivers/net/ethernet/sun/ldmvsw.c
+++ b/drivers/net/ethernet/sun/ldmvsw.c
@@ -11,7 +11,6 @@
 #include <linux/highmem.h>
 #include <linux/if_vlan.h>
 #include <linux/init.h>
-#include <linux/kconfig.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
diff --git a/drivers/net/ethernet/synopsys/dwc_eth_qos.c b/drivers/net/ethernet/synopsys/dwc_eth_qos.c
index 9f159a775af3..0d0053128542 100644
--- a/drivers/net/ethernet/synopsys/dwc_eth_qos.c
+++ b/drivers/net/ethernet/synopsys/dwc_eth_qos.c
@@ -1246,7 +1246,7 @@ static int dwceqos_mii_init(struct net_local *lp)
 	lp->mii_bus->read  = &dwceqos_mdio_read;
 	lp->mii_bus->write = &dwceqos_mdio_write;
 	lp->mii_bus->priv = lp;
-	lp->mii_bus->parent = &lp->ndev->dev;
+	lp->mii_bus->parent = &lp->pdev->dev;
 
 	of_address_to_resource(lp->pdev->dev.of_node, 0, &res);
 	snprintf(lp->mii_bus->id, MII_BUS_ID_SIZE, "%.8llx",
@@ -1622,13 +1622,7 @@ static void dwceqos_init_hw(struct net_local *lp)
 		DWCEQOS_MMC_CTRL_RSTONRD);
 	dwceqos_enable_mmc_interrupt(lp);
 
-	/* Enable Interrupts */
-	dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_IE,
-		      DWCEQOS_DMA_CH0_IE_NIE |
-		      DWCEQOS_DMA_CH0_IE_RIE | DWCEQOS_DMA_CH0_IE_TIE |
-		      DWCEQOS_DMA_CH0_IE_AIE |
-		      DWCEQOS_DMA_CH0_IE_FBEE);
-
+	dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_IE, 0);
 	dwceqos_write(lp, REG_DWCEQOS_MAC_IE, 0);
 
 	dwceqos_write(lp, REG_DWCEQOS_MAC_CFG, DWCEQOS_MAC_CFG_IPC |
@@ -1905,6 +1899,15 @@ static int dwceqos_open(struct net_device *ndev)
 	netif_start_queue(ndev);
 	tasklet_enable(&lp->tx_bdreclaim_tasklet);
 
+	/* Enable Interrupts -- do this only after we enable NAPI and the
+	 * tasklet.
+	 */
+	dwceqos_write(lp, REG_DWCEQOS_DMA_CH0_IE,
+		      DWCEQOS_DMA_CH0_IE_NIE |
+		      DWCEQOS_DMA_CH0_IE_RIE | DWCEQOS_DMA_CH0_IE_TIE |
+		      DWCEQOS_DMA_CH0_IE_AIE |
+		      DWCEQOS_DMA_CH0_IE_FBEE);
+
 	return 0;
 }
 
@@ -2740,7 +2743,7 @@ static void dwceqos_set_msglevel(struct net_device *ndev, u32 msglevel)
 	lp->msg_enable = msglevel;
 }
 
-static struct ethtool_ops dwceqos_ethtool_ops = {
+static const struct ethtool_ops dwceqos_ethtool_ops = {
 	.get_drvinfo    = dwceqos_get_drvinfo,
 	.get_link       = ethtool_op_get_link,
 	.get_pauseparam = dwceqos_get_pauseparam,
@@ -2758,7 +2761,7 @@ static struct ethtool_ops dwceqos_ethtool_ops = {
 	.set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
-static struct net_device_ops netdev_ops = {
+static const struct net_device_ops netdev_ops = {
 	.ndo_open		= dwceqos_open,
 	.ndo_stop		= dwceqos_stop,
 	.ndo_start_xmit		= dwceqos_start_xmit,
@@ -2850,25 +2853,17 @@ static int dwceqos_probe(struct platform_device *pdev)
 
 	ndev->features = ndev->hw_features;
 
-	netif_napi_add(ndev, &lp->napi, dwceqos_rx_poll, NAPI_POLL_WEIGHT);
-
-	ret = register_netdev(ndev);
-	if (ret) {
-		dev_err(&pdev->dev, "Cannot register net device, aborting.\n");
-		goto err_out_clk_dis_aper;
-	}
-
 	lp->phy_ref_clk = devm_clk_get(&pdev->dev, "phy_ref_clk");
 	if (IS_ERR(lp->phy_ref_clk)) {
 		dev_err(&pdev->dev, "phy_ref_clk clock not found.\n");
 		ret = PTR_ERR(lp->phy_ref_clk);
-		goto err_out_unregister_netdev;
+		goto err_out_clk_dis_aper;
 	}
 
 	ret = clk_prepare_enable(lp->phy_ref_clk);
 	if (ret) {
 		dev_err(&pdev->dev, "Unable to enable device clock.\n");
-		goto err_out_unregister_netdev;
+		goto err_out_clk_dis_aper;
 	}
 
 	lp->phy_node = of_parse_phandle(lp->pdev->dev.of_node,
@@ -2877,7 +2872,7 @@ static int dwceqos_probe(struct platform_device *pdev)
 		ret = of_phy_register_fixed_link(lp->pdev->dev.of_node);
 		if (ret < 0) {
 			dev_err(&pdev->dev, "invalid fixed-link");
-			goto err_out_unregister_clk_notifier;
+			goto err_out_clk_dis_phy;
 		}
 
 		lp->phy_node = of_node_get(lp->pdev->dev.of_node);
@@ -2886,7 +2881,7 @@ static int dwceqos_probe(struct platform_device *pdev)
 	ret = of_get_phy_mode(lp->pdev->dev.of_node);
 	if (ret < 0) {
 		dev_err(&lp->pdev->dev, "error in getting phy i/f\n");
-		goto err_out_unregister_clk_notifier;
+		goto err_out_clk_dis_phy;
 	}
 
 	lp->phy_interface = ret;
@@ -2894,14 +2889,14 @@ static int dwceqos_probe(struct platform_device *pdev)
 	ret = dwceqos_mii_init(lp);
 	if (ret) {
 		dev_err(&lp->pdev->dev, "error in dwceqos_mii_init\n");
-		goto err_out_unregister_clk_notifier;
+		goto err_out_clk_dis_phy;
 	}
 
 	ret = dwceqos_mii_probe(ndev);
 	if (ret != 0) {
 		netdev_err(ndev, "mii_probe fail.\n");
 		ret = -ENXIO;
-		goto err_out_unregister_clk_notifier;
+		goto err_out_clk_dis_phy;
 	}
 
 	dwceqos_set_umac_addr(lp, lp->ndev->dev_addr, 0);
@@ -2919,7 +2914,7 @@ static int dwceqos_probe(struct platform_device *pdev)
 	if (ret) {
 		dev_err(&lp->pdev->dev, "Unable to retrieve DT, error %d\n",
 			ret);
-		goto err_out_unregister_clk_notifier;
+		goto err_out_clk_dis_phy;
 	}
 	dev_info(&lp->pdev->dev, "pdev->id %d, baseaddr 0x%08lx, irq %d\n",
 		 pdev->id, ndev->base_addr, ndev->irq);
@@ -2929,18 +2924,24 @@ static int dwceqos_probe(struct platform_device *pdev)
 	if (ret) {
 		dev_err(&lp->pdev->dev, "Unable to request IRQ %d, error %d\n",
 			ndev->irq, ret);
-		goto err_out_unregister_clk_notifier;
+		goto err_out_clk_dis_phy;
 	}
 
 	if (netif_msg_probe(lp))
 		netdev_dbg(ndev, "net_local@%p\n", lp);
 
+	netif_napi_add(ndev, &lp->napi, dwceqos_rx_poll, NAPI_POLL_WEIGHT);
+
+	ret = register_netdev(ndev);
+	if (ret) {
+		dev_err(&pdev->dev, "Cannot register net device, aborting.\n");
+			goto err_out_clk_dis_phy;
+	}
+
 	return 0;
 
-err_out_unregister_clk_notifier:
+err_out_clk_dis_phy:
 	clk_disable_unprepare(lp->phy_ref_clk);
-err_out_unregister_netdev:
-	unregister_netdev(ndev);
 err_out_clk_dis_aper:
 	clk_disable_unprepare(lp->apb_pclk);
 err_out_free_netdev:
diff --git a/drivers/net/ethernet/tehuti/tehuti.c b/drivers/net/ethernet/tehuti/tehuti.c
index 7452b5f9d024..7108c68f16d3 100644
--- a/drivers/net/ethernet/tehuti/tehuti.c
+++ b/drivers/net/ethernet/tehuti/tehuti.c
@@ -1987,7 +1987,7 @@ bdx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if ((readl(nic->regs + FPGA_VER) & 0xFFF) >= 378) {
 		err = pci_enable_msi(pdev);
 		if (err)
-			pr_err("Can't eneble msi. error is %d\n", err);
+			pr_err("Can't enable msi. error is %d\n", err);
 		else
 			nic->irq_type = IRQ_MSI;
 	} else
diff --git a/drivers/net/ethernet/ti/cpmac.c b/drivers/net/ethernet/ti/cpmac.c
index d300d536d06f..fa0cfda24fd9 100644
--- a/drivers/net/ethernet/ti/cpmac.c
+++ b/drivers/net/ethernet/ti/cpmac.c
@@ -546,7 +546,8 @@ fatal_error:
 
 static int cpmac_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
-	int queue, len;
+	int queue;
+	unsigned int len;
 	struct cpmac_desc *desc;
 	struct cpmac_priv *priv = netdev_priv(dev);
 
@@ -556,7 +557,7 @@ static int cpmac_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	if (unlikely(skb_padto(skb, ETH_ZLEN)))
 		return NETDEV_TX_OK;
 
-	len = max(skb->len, ETH_ZLEN);
+	len = max_t(unsigned int, skb->len, ETH_ZLEN);
 	queue = skb_get_queue_mapping(skb);
 	netif_stop_subqueue(dev, queue);
 
diff --git a/drivers/net/ethernet/ti/cpsw-phy-sel.c b/drivers/net/ethernet/ti/cpsw-phy-sel.c
index c3e85acfdc70..054a8dd23dae 100644
--- a/drivers/net/ethernet/ti/cpsw-phy-sel.c
+++ b/drivers/net/ethernet/ti/cpsw-phy-sel.c
@@ -30,6 +30,8 @@
 
 #define AM33XX_GMII_SEL_RMII2_IO_CLK_EN	BIT(7)
 #define AM33XX_GMII_SEL_RMII1_IO_CLK_EN	BIT(6)
+#define AM33XX_GMII_SEL_RGMII2_IDMODE	BIT(5)
+#define AM33XX_GMII_SEL_RGMII1_IDMODE	BIT(4)
 
 #define GMII_SEL_MODE_MASK		0x3
 
@@ -48,6 +50,7 @@ static void cpsw_gmii_sel_am3352(struct cpsw_phy_sel_priv *priv,
 	u32 reg;
 	u32 mask;
 	u32 mode = 0;
+	bool rgmii_id = false;
 
 	reg = readl(priv->gmii_sel);
 
@@ -57,10 +60,14 @@ static void cpsw_gmii_sel_am3352(struct cpsw_phy_sel_priv *priv,
 		break;
 
 	case PHY_INTERFACE_MODE_RGMII:
+		mode = AM33XX_GMII_SEL_MODE_RGMII;
+		break;
+
 	case PHY_INTERFACE_MODE_RGMII_ID:
 	case PHY_INTERFACE_MODE_RGMII_RXID:
 	case PHY_INTERFACE_MODE_RGMII_TXID:
 		mode = AM33XX_GMII_SEL_MODE_RGMII;
+		rgmii_id = true;
 		break;
 
 	default:
@@ -83,6 +90,13 @@ static void cpsw_gmii_sel_am3352(struct cpsw_phy_sel_priv *priv,
 			mode |= AM33XX_GMII_SEL_RMII2_IO_CLK_EN;
 	}
 
+	if (rgmii_id) {
+		if (slave == 0)
+			mode |= AM33XX_GMII_SEL_RGMII1_IDMODE;
+		else
+			mode |= AM33XX_GMII_SEL_RGMII2_IDMODE;
+	}
+
 	reg &= ~mask;
 	reg |= mode;
 
diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
index c51f34693eae..c6cff3d2ff05 100644
--- a/drivers/net/ethernet/ti/cpsw.c
+++ b/drivers/net/ethernet/ti/cpsw.c
@@ -124,7 +124,7 @@ do {								\
 
 #define RX_PRIORITY_MAPPING	0x76543210
 #define TX_PRIORITY_MAPPING	0x33221100
-#define CPDMA_TX_PRIORITY_MAP	0x76543210
+#define CPDMA_TX_PRIORITY_MAP	0x01234567
 
 #define CPSW_VLAN_AWARE		BIT(1)
 #define CPSW_ALE_VLAN_AWARE	1
@@ -140,9 +140,11 @@ do {								\
 #define CPSW_CMINTMAX_INTVL	(1000 / CPSW_CMINTMIN_CNT)
 #define CPSW_CMINTMIN_INTVL	((1000 / CPSW_CMINTMAX_CNT) + 1)
 
-#define cpsw_slave_index(priv)				\
-		((priv->data.dual_emac) ? priv->emac_port :	\
-		priv->data.active_slave)
+#define cpsw_slave_index(cpsw, priv)				\
+		((cpsw->data.dual_emac) ? priv->emac_port :	\
+		cpsw->data.active_slave)
+#define IRQ_NUM			2
+#define CPSW_MAX_QUEUES		8
 
 static int debug_level;
 module_param(debug_level, int, 0);
@@ -363,38 +365,41 @@ static inline void slave_write(struct cpsw_slave *slave, u32 val, u32 offset)
 	__raw_writel(val, slave->regs + offset);
 }
 
-struct cpsw_priv {
-	struct platform_device		*pdev;
-	struct net_device		*ndev;
-	struct napi_struct		napi_rx;
-	struct napi_struct		napi_tx;
+struct cpsw_common {
 	struct device			*dev;
 	struct cpsw_platform_data	data;
+	struct napi_struct		napi_rx;
+	struct napi_struct		napi_tx;
 	struct cpsw_ss_regs __iomem	*regs;
 	struct cpsw_wr_regs __iomem	*wr_regs;
 	u8 __iomem			*hw_stats;
 	struct cpsw_host_regs __iomem	*host_port_regs;
-	u32				msg_enable;
 	u32				version;
 	u32				coal_intvl;
 	u32				bus_freq_mhz;
 	int				rx_packet_max;
-	struct clk			*clk;
-	u8				mac_addr[ETH_ALEN];
 	struct cpsw_slave		*slaves;
 	struct cpdma_ctlr		*dma;
-	struct cpdma_chan		*txch, *rxch;
+	struct cpdma_chan		*txch[CPSW_MAX_QUEUES];
+	struct cpdma_chan		*rxch[CPSW_MAX_QUEUES];
 	struct cpsw_ale			*ale;
-	bool				rx_pause;
-	bool				tx_pause;
 	bool				quirk_irq;
 	bool				rx_irq_disabled;
 	bool				tx_irq_disabled;
-	/* snapshot of IRQ numbers */
-	u32 irqs_table[4];
-	u32 num_irqs;
-	struct cpts *cpts;
+	u32 irqs_table[IRQ_NUM];
+	struct cpts			*cpts;
+	int				rx_ch_num, tx_ch_num;
+};
+
+struct cpsw_priv {
+	struct net_device		*ndev;
+	struct device			*dev;
+	u32				msg_enable;
+	u8				mac_addr[ETH_ALEN];
+	bool				rx_pause;
+	bool				tx_pause;
 	u32 emac_port;
+	struct cpsw_common *cpsw;
 };
 
 struct cpsw_stats {
@@ -455,108 +460,92 @@ static const struct cpsw_stats cpsw_gstrings_stats[] = {
 	{ "Rx Start of Frame Overruns", CPSW_STAT(rxsofoverruns) },
 	{ "Rx Middle of Frame Overruns", CPSW_STAT(rxmofoverruns) },
 	{ "Rx DMA Overruns", CPSW_STAT(rxdmaoverruns) },
-	{ "Rx DMA chan: head_enqueue", CPDMA_RX_STAT(head_enqueue) },
-	{ "Rx DMA chan: tail_enqueue", CPDMA_RX_STAT(tail_enqueue) },
-	{ "Rx DMA chan: pad_enqueue", CPDMA_RX_STAT(pad_enqueue) },
-	{ "Rx DMA chan: misqueued", CPDMA_RX_STAT(misqueued) },
-	{ "Rx DMA chan: desc_alloc_fail", CPDMA_RX_STAT(desc_alloc_fail) },
-	{ "Rx DMA chan: pad_alloc_fail", CPDMA_RX_STAT(pad_alloc_fail) },
-	{ "Rx DMA chan: runt_receive_buf", CPDMA_RX_STAT(runt_receive_buff) },
-	{ "Rx DMA chan: runt_transmit_buf", CPDMA_RX_STAT(runt_transmit_buff) },
-	{ "Rx DMA chan: empty_dequeue", CPDMA_RX_STAT(empty_dequeue) },
-	{ "Rx DMA chan: busy_dequeue", CPDMA_RX_STAT(busy_dequeue) },
-	{ "Rx DMA chan: good_dequeue", CPDMA_RX_STAT(good_dequeue) },
-	{ "Rx DMA chan: requeue", CPDMA_RX_STAT(requeue) },
-	{ "Rx DMA chan: teardown_dequeue", CPDMA_RX_STAT(teardown_dequeue) },
-	{ "Tx DMA chan: head_enqueue", CPDMA_TX_STAT(head_enqueue) },
-	{ "Tx DMA chan: tail_enqueue", CPDMA_TX_STAT(tail_enqueue) },
-	{ "Tx DMA chan: pad_enqueue", CPDMA_TX_STAT(pad_enqueue) },
-	{ "Tx DMA chan: misqueued", CPDMA_TX_STAT(misqueued) },
-	{ "Tx DMA chan: desc_alloc_fail", CPDMA_TX_STAT(desc_alloc_fail) },
-	{ "Tx DMA chan: pad_alloc_fail", CPDMA_TX_STAT(pad_alloc_fail) },
-	{ "Tx DMA chan: runt_receive_buf", CPDMA_TX_STAT(runt_receive_buff) },
-	{ "Tx DMA chan: runt_transmit_buf", CPDMA_TX_STAT(runt_transmit_buff) },
-	{ "Tx DMA chan: empty_dequeue", CPDMA_TX_STAT(empty_dequeue) },
-	{ "Tx DMA chan: busy_dequeue", CPDMA_TX_STAT(busy_dequeue) },
-	{ "Tx DMA chan: good_dequeue", CPDMA_TX_STAT(good_dequeue) },
-	{ "Tx DMA chan: requeue", CPDMA_TX_STAT(requeue) },
-	{ "Tx DMA chan: teardown_dequeue", CPDMA_TX_STAT(teardown_dequeue) },
 };
 
-#define CPSW_STATS_LEN	ARRAY_SIZE(cpsw_gstrings_stats)
+static const struct cpsw_stats cpsw_gstrings_ch_stats[] = {
+	{ "head_enqueue", CPDMA_RX_STAT(head_enqueue) },
+	{ "tail_enqueue", CPDMA_RX_STAT(tail_enqueue) },
+	{ "pad_enqueue", CPDMA_RX_STAT(pad_enqueue) },
+	{ "misqueued", CPDMA_RX_STAT(misqueued) },
+	{ "desc_alloc_fail", CPDMA_RX_STAT(desc_alloc_fail) },
+	{ "pad_alloc_fail", CPDMA_RX_STAT(pad_alloc_fail) },
+	{ "runt_receive_buf", CPDMA_RX_STAT(runt_receive_buff) },
+	{ "runt_transmit_buf", CPDMA_RX_STAT(runt_transmit_buff) },
+	{ "empty_dequeue", CPDMA_RX_STAT(empty_dequeue) },
+	{ "busy_dequeue", CPDMA_RX_STAT(busy_dequeue) },
+	{ "good_dequeue", CPDMA_RX_STAT(good_dequeue) },
+	{ "requeue", CPDMA_RX_STAT(requeue) },
+	{ "teardown_dequeue", CPDMA_RX_STAT(teardown_dequeue) },
+};
 
-#define napi_to_priv(napi)	container_of(napi, struct cpsw_priv, napi)
+#define CPSW_STATS_COMMON_LEN	ARRAY_SIZE(cpsw_gstrings_stats)
+#define CPSW_STATS_CH_LEN	ARRAY_SIZE(cpsw_gstrings_ch_stats)
+
+#define ndev_to_cpsw(ndev) (((struct cpsw_priv *)netdev_priv(ndev))->cpsw)
+#define napi_to_cpsw(napi)	container_of(napi, struct cpsw_common, napi)
 #define for_each_slave(priv, func, arg...)				\
 	do {								\
 		struct cpsw_slave *slave;				\
+		struct cpsw_common *cpsw = (priv)->cpsw;		\
 		int n;							\
-		if (priv->data.dual_emac)				\
-			(func)((priv)->slaves + priv->emac_port, ##arg);\
+		if (cpsw->data.dual_emac)				\
+			(func)((cpsw)->slaves + priv->emac_port, ##arg);\
 		else							\
-			for (n = (priv)->data.slaves,			\
-					slave = (priv)->slaves;		\
+			for (n = cpsw->data.slaves,			\
+					slave = cpsw->slaves;		\
 					n; n--)				\
 				(func)(slave++, ##arg);			\
 	} while (0)
-#define cpsw_get_slave_ndev(priv, __slave_no__)				\
-	((__slave_no__ < priv->data.slaves) ?				\
-		priv->slaves[__slave_no__].ndev : NULL)
-#define cpsw_get_slave_priv(priv, __slave_no__)				\
-	(((__slave_no__ < priv->data.slaves) &&				\
-		(priv->slaves[__slave_no__].ndev)) ?			\
-		netdev_priv(priv->slaves[__slave_no__].ndev) : NULL)	\
-
-#define cpsw_dual_emac_src_port_detect(status, priv, ndev, skb)		\
+
+#define cpsw_dual_emac_src_port_detect(cpsw, status, ndev, skb)		\
 	do {								\
-		if (!priv->data.dual_emac)				\
+		if (!cpsw->data.dual_emac)				\
 			break;						\
 		if (CPDMA_RX_SOURCE_PORT(status) == 1) {		\
-			ndev = cpsw_get_slave_ndev(priv, 0);		\
-			priv = netdev_priv(ndev);			\
+			ndev = cpsw->slaves[0].ndev;			\
 			skb->dev = ndev;				\
 		} else if (CPDMA_RX_SOURCE_PORT(status) == 2) {		\
-			ndev = cpsw_get_slave_ndev(priv, 1);		\
-			priv = netdev_priv(ndev);			\
+			ndev = cpsw->slaves[1].ndev;			\
 			skb->dev = ndev;				\
 		}							\
 	} while (0)
-#define cpsw_add_mcast(priv, addr)					\
+#define cpsw_add_mcast(cpsw, priv, addr)				\
 	do {								\
-		if (priv->data.dual_emac) {				\
-			struct cpsw_slave *slave = priv->slaves +	\
+		if (cpsw->data.dual_emac) {				\
+			struct cpsw_slave *slave = cpsw->slaves +	\
 						priv->emac_port;	\
-			int slave_port = cpsw_get_slave_port(priv,	\
+			int slave_port = cpsw_get_slave_port(		\
 						slave->slave_num);	\
-			cpsw_ale_add_mcast(priv->ale, addr,		\
+			cpsw_ale_add_mcast(cpsw->ale, addr,		\
 				1 << slave_port | ALE_PORT_HOST,	\
 				ALE_VLAN, slave->port_vlan, 0);		\
 		} else {						\
-			cpsw_ale_add_mcast(priv->ale, addr,		\
+			cpsw_ale_add_mcast(cpsw->ale, addr,		\
 				ALE_ALL_PORTS,				\
 				0, 0, 0);				\
 		}							\
 	} while (0)
 
-static inline int cpsw_get_slave_port(struct cpsw_priv *priv, u32 slave_num)
+static inline int cpsw_get_slave_port(u32 slave_num)
 {
 	return slave_num + 1;
 }
 
 static void cpsw_set_promiscious(struct net_device *ndev, bool enable)
 {
-	struct cpsw_priv *priv = netdev_priv(ndev);
-	struct cpsw_ale *ale = priv->ale;
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
+	struct cpsw_ale *ale = cpsw->ale;
 	int i;
 
-	if (priv->data.dual_emac) {
+	if (cpsw->data.dual_emac) {
 		bool flag = false;
 
 		/* Enabling promiscuous mode for one interface will be
 		 * common for both the interface as the interface shares
 		 * the same hardware resource.
 		 */
-		for (i = 0; i < priv->data.slaves; i++)
-			if (priv->slaves[i].ndev->flags & IFF_PROMISC)
+		for (i = 0; i < cpsw->data.slaves; i++)
+			if (cpsw->slaves[i].ndev->flags & IFF_PROMISC)
 				flag = true;
 
 		if (!enable && flag) {
@@ -579,7 +568,7 @@ static void cpsw_set_promiscious(struct net_device *ndev, bool enable)
 			unsigned long timeout = jiffies + HZ;
 
 			/* Disable Learn for all ports (host is port 0 and slaves are port 1 and up */
-			for (i = 0; i <= priv->data.slaves; i++) {
+			for (i = 0; i <= cpsw->data.slaves; i++) {
 				cpsw_ale_control_set(ale, i,
 						     ALE_PORT_NOLEARN, 1);
 				cpsw_ale_control_set(ale, i,
@@ -606,7 +595,7 @@ static void cpsw_set_promiscious(struct net_device *ndev, bool enable)
 			cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 0);
 
 			/* Enable Learn for all ports (host is port 0 and slaves are port 1 and up */
-			for (i = 0; i <= priv->data.slaves; i++) {
+			for (i = 0; i <= cpsw->data.slaves; i++) {
 				cpsw_ale_control_set(ale, i,
 						     ALE_PORT_NOLEARN, 0);
 				cpsw_ale_control_set(ale, i,
@@ -620,17 +609,18 @@ static void cpsw_set_promiscious(struct net_device *ndev, bool enable)
 static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = priv->cpsw;
 	int vid;
 
-	if (priv->data.dual_emac)
-		vid = priv->slaves[priv->emac_port].port_vlan;
+	if (cpsw->data.dual_emac)
+		vid = cpsw->slaves[priv->emac_port].port_vlan;
 	else
-		vid = priv->data.default_vlan;
+		vid = cpsw->data.default_vlan;
 
 	if (ndev->flags & IFF_PROMISC) {
 		/* Enable promiscuous mode */
 		cpsw_set_promiscious(ndev, true);
-		cpsw_ale_set_allmulti(priv->ale, IFF_ALLMULTI);
+		cpsw_ale_set_allmulti(cpsw->ale, IFF_ALLMULTI);
 		return;
 	} else {
 		/* Disable promiscuous mode */
@@ -638,51 +628,54 @@ static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
 	}
 
 	/* Restore allmulti on vlans if necessary */
-	cpsw_ale_set_allmulti(priv->ale, priv->ndev->flags & IFF_ALLMULTI);
+	cpsw_ale_set_allmulti(cpsw->ale, priv->ndev->flags & IFF_ALLMULTI);
 
 	/* Clear all mcast from ALE */
-	cpsw_ale_flush_multicast(priv->ale, ALE_ALL_PORTS, vid);
+	cpsw_ale_flush_multicast(cpsw->ale, ALE_ALL_PORTS, vid);
 
 	if (!netdev_mc_empty(ndev)) {
 		struct netdev_hw_addr *ha;
 
 		/* program multicast address list into ALE register */
 		netdev_for_each_mc_addr(ha, ndev) {
-			cpsw_add_mcast(priv, (u8 *)ha->addr);
+			cpsw_add_mcast(cpsw, priv, (u8 *)ha->addr);
 		}
 	}
 }
 
-static void cpsw_intr_enable(struct cpsw_priv *priv)
+static void cpsw_intr_enable(struct cpsw_common *cpsw)
 {
-	__raw_writel(0xFF, &priv->wr_regs->tx_en);
-	__raw_writel(0xFF, &priv->wr_regs->rx_en);
+	__raw_writel(0xFF, &cpsw->wr_regs->tx_en);
+	__raw_writel(0xFF, &cpsw->wr_regs->rx_en);
 
-	cpdma_ctlr_int_ctrl(priv->dma, true);
+	cpdma_ctlr_int_ctrl(cpsw->dma, true);
 	return;
 }
 
-static void cpsw_intr_disable(struct cpsw_priv *priv)
+static void cpsw_intr_disable(struct cpsw_common *cpsw)
 {
-	__raw_writel(0, &priv->wr_regs->tx_en);
-	__raw_writel(0, &priv->wr_regs->rx_en);
+	__raw_writel(0, &cpsw->wr_regs->tx_en);
+	__raw_writel(0, &cpsw->wr_regs->rx_en);
 
-	cpdma_ctlr_int_ctrl(priv->dma, false);
+	cpdma_ctlr_int_ctrl(cpsw->dma, false);
 	return;
 }
 
 static void cpsw_tx_handler(void *token, int len, int status)
 {
+	struct netdev_queue	*txq;
 	struct sk_buff		*skb = token;
 	struct net_device	*ndev = skb->dev;
-	struct cpsw_priv	*priv = netdev_priv(ndev);
+	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
 
 	/* Check whether the queue is stopped due to stalled tx dma, if the
 	 * queue is stopped then start the queue as we have free desc for tx
 	 */
-	if (unlikely(netif_queue_stopped(ndev)))
-		netif_wake_queue(ndev);
-	cpts_tx_timestamp(priv->cpts, skb);
+	txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
+	if (unlikely(netif_tx_queue_stopped(txq)))
+		netif_tx_wake_queue(txq);
+
+	cpts_tx_timestamp(cpsw->cpts, skb);
 	ndev->stats.tx_packets++;
 	ndev->stats.tx_bytes += len;
 	dev_kfree_skb_any(skb);
@@ -690,22 +683,23 @@ static void cpsw_tx_handler(void *token, int len, int status)
 
 static void cpsw_rx_handler(void *token, int len, int status)
 {
+	struct cpdma_chan	*ch;
 	struct sk_buff		*skb = token;
 	struct sk_buff		*new_skb;
 	struct net_device	*ndev = skb->dev;
-	struct cpsw_priv	*priv = netdev_priv(ndev);
 	int			ret = 0;
+	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
 
-	cpsw_dual_emac_src_port_detect(status, priv, ndev, skb);
+	cpsw_dual_emac_src_port_detect(cpsw, status, ndev, skb);
 
 	if (unlikely(status < 0) || unlikely(!netif_running(ndev))) {
 		bool ndev_status = false;
-		struct cpsw_slave *slave = priv->slaves;
+		struct cpsw_slave *slave = cpsw->slaves;
 		int n;
 
-		if (priv->data.dual_emac) {
+		if (cpsw->data.dual_emac) {
 			/* In dual emac mode check for all interfaces */
-			for (n = priv->data.slaves; n; n--, slave++)
+			for (n = cpsw->data.slaves; n; n--, slave++)
 				if (netif_running(slave->ndev))
 					ndev_status = true;
 		}
@@ -726,97 +720,133 @@ static void cpsw_rx_handler(void *token, int len, int status)
 		return;
 	}
 
-	new_skb = netdev_alloc_skb_ip_align(ndev, priv->rx_packet_max);
+	new_skb = netdev_alloc_skb_ip_align(ndev, cpsw->rx_packet_max);
 	if (new_skb) {
+		skb_copy_queue_mapping(new_skb, skb);
 		skb_put(skb, len);
-		cpts_rx_timestamp(priv->cpts, skb);
+		cpts_rx_timestamp(cpsw->cpts, skb);
 		skb->protocol = eth_type_trans(skb, ndev);
 		netif_receive_skb(skb);
 		ndev->stats.rx_bytes += len;
 		ndev->stats.rx_packets++;
+		kmemleak_not_leak(new_skb);
 	} else {
 		ndev->stats.rx_dropped++;
 		new_skb = skb;
 	}
 
 requeue:
-	ret = cpdma_chan_submit(priv->rxch, new_skb, new_skb->data,
-			skb_tailroom(new_skb), 0);
+	if (netif_dormant(ndev)) {
+		dev_kfree_skb_any(new_skb);
+		return;
+	}
+
+	ch = cpsw->rxch[skb_get_queue_mapping(new_skb)];
+	ret = cpdma_chan_submit(ch, new_skb, new_skb->data,
+				skb_tailroom(new_skb), 0);
 	if (WARN_ON(ret < 0))
 		dev_kfree_skb_any(new_skb);
 }
 
 static irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id)
 {
-	struct cpsw_priv *priv = dev_id;
+	struct cpsw_common *cpsw = dev_id;
 
-	writel(0, &priv->wr_regs->tx_en);
-	cpdma_ctlr_eoi(priv->dma, CPDMA_EOI_TX);
+	writel(0, &cpsw->wr_regs->tx_en);
+	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_TX);
 
-	if (priv->quirk_irq) {
-		disable_irq_nosync(priv->irqs_table[1]);
-		priv->tx_irq_disabled = true;
+	if (cpsw->quirk_irq) {
+		disable_irq_nosync(cpsw->irqs_table[1]);
+		cpsw->tx_irq_disabled = true;
 	}
 
-	napi_schedule(&priv->napi_tx);
+	napi_schedule(&cpsw->napi_tx);
 	return IRQ_HANDLED;
 }
 
 static irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id)
 {
-	struct cpsw_priv *priv = dev_id;
+	struct cpsw_common *cpsw = dev_id;
 
-	cpdma_ctlr_eoi(priv->dma, CPDMA_EOI_RX);
-	writel(0, &priv->wr_regs->rx_en);
+	cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_RX);
+	writel(0, &cpsw->wr_regs->rx_en);
 
-	if (priv->quirk_irq) {
-		disable_irq_nosync(priv->irqs_table[0]);
-		priv->rx_irq_disabled = true;
+	if (cpsw->quirk_irq) {
+		disable_irq_nosync(cpsw->irqs_table[0]);
+		cpsw->rx_irq_disabled = true;
 	}
 
-	napi_schedule(&priv->napi_rx);
+	napi_schedule(&cpsw->napi_rx);
 	return IRQ_HANDLED;
 }
 
 static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
 {
-	struct cpsw_priv	*priv = napi_to_priv(napi_tx);
-	int			num_tx;
+	u32			ch_map;
+	int			num_tx, ch;
+	struct cpsw_common	*cpsw = napi_to_cpsw(napi_tx);
+
+	/* process every unprocessed channel */
+	ch_map = cpdma_ctrl_txchs_state(cpsw->dma);
+	for (ch = 0, num_tx = 0; num_tx < budget; ch_map >>= 1, ch++) {
+		if (!ch_map) {
+			ch_map = cpdma_ctrl_txchs_state(cpsw->dma);
+			if (!ch_map)
+				break;
+
+			ch = 0;
+		}
+
+		if (!(ch_map & 0x01))
+			continue;
+
+		num_tx += cpdma_chan_process(cpsw->txch[ch], budget - num_tx);
+	}
 
-	num_tx = cpdma_chan_process(priv->txch, budget);
 	if (num_tx < budget) {
 		napi_complete(napi_tx);
-		writel(0xff, &priv->wr_regs->tx_en);
-		if (priv->quirk_irq && priv->tx_irq_disabled) {
-			priv->tx_irq_disabled = false;
-			enable_irq(priv->irqs_table[1]);
+		writel(0xff, &cpsw->wr_regs->tx_en);
+		if (cpsw->quirk_irq && cpsw->tx_irq_disabled) {
+			cpsw->tx_irq_disabled = false;
+			enable_irq(cpsw->irqs_table[1]);
 		}
 	}
 
-	if (num_tx)
-		cpsw_dbg(priv, intr, "poll %d tx pkts\n", num_tx);
-
 	return num_tx;
 }
 
 static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
 {
-	struct cpsw_priv	*priv = napi_to_priv(napi_rx);
-	int			num_rx;
+	u32			ch_map;
+	int			num_rx, ch;
+	struct cpsw_common	*cpsw = napi_to_cpsw(napi_rx);
+
+	/* process every unprocessed channel */
+	ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);
+	for (ch = 0, num_rx = 0; num_rx < budget; ch_map >>= 1, ch++) {
+		if (!ch_map) {
+			ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);
+			if (!ch_map)
+				break;
+
+			ch = 0;
+		}
+
+		if (!(ch_map & 0x01))
+			continue;
+
+		num_rx += cpdma_chan_process(cpsw->rxch[ch], budget - num_rx);
+	}
 
-	num_rx = cpdma_chan_process(priv->rxch, budget);
 	if (num_rx < budget) {
 		napi_complete(napi_rx);
-		writel(0xff, &priv->wr_regs->rx_en);
-		if (priv->quirk_irq && priv->rx_irq_disabled) {
-			priv->rx_irq_disabled = false;
-			enable_irq(priv->irqs_table[0]);
+		writel(0xff, &cpsw->wr_regs->rx_en);
+		if (cpsw->quirk_irq && cpsw->rx_irq_disabled) {
+			cpsw->rx_irq_disabled = false;
+			enable_irq(cpsw->irqs_table[0]);
 		}
 	}
 
-	if (num_rx)
-		cpsw_dbg(priv, intr, "poll %d rx pkts\n", num_rx);
-
 	return num_rx;
 }
 
@@ -849,17 +879,18 @@ static void _cpsw_adjust_link(struct cpsw_slave *slave,
 	struct phy_device	*phy = slave->phy;
 	u32			mac_control = 0;
 	u32			slave_port;
+	struct cpsw_common *cpsw = priv->cpsw;
 
 	if (!phy)
 		return;
 
-	slave_port = cpsw_get_slave_port(priv, slave->slave_num);
+	slave_port = cpsw_get_slave_port(slave->slave_num);
 
 	if (phy->link) {
-		mac_control = priv->data.mac_control;
+		mac_control = cpsw->data.mac_control;
 
 		/* enable forwarding */
-		cpsw_ale_control_set(priv->ale, slave_port,
+		cpsw_ale_control_set(cpsw->ale, slave_port,
 				     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
 
 		if (phy->speed == 1000)
@@ -883,7 +914,7 @@ static void _cpsw_adjust_link(struct cpsw_slave *slave,
 	} else {
 		mac_control = 0;
 		/* disable forwarding */
-		cpsw_ale_control_set(priv->ale, slave_port,
+		cpsw_ale_control_set(cpsw->ale, slave_port,
 				     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
 	}
 
@@ -905,19 +936,19 @@ static void cpsw_adjust_link(struct net_device *ndev)
 	if (link) {
 		netif_carrier_on(ndev);
 		if (netif_running(ndev))
-			netif_wake_queue(ndev);
+			netif_tx_wake_all_queues(ndev);
 	} else {
 		netif_carrier_off(ndev);
-		netif_stop_queue(ndev);
+		netif_tx_stop_all_queues(ndev);
 	}
 }
 
 static int cpsw_get_coalesce(struct net_device *ndev,
 				struct ethtool_coalesce *coal)
 {
-	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 
-	coal->rx_coalesce_usecs = priv->coal_intvl;
+	coal->rx_coalesce_usecs = cpsw->coal_intvl;
 	return 0;
 }
 
@@ -930,11 +961,12 @@ static int cpsw_set_coalesce(struct net_device *ndev,
 	u32 prescale = 0;
 	u32 addnl_dvdr = 1;
 	u32 coal_intvl = 0;
+	struct cpsw_common *cpsw = priv->cpsw;
 
 	coal_intvl = coal->rx_coalesce_usecs;
 
-	int_ctrl =  readl(&priv->wr_regs->int_control);
-	prescale = priv->bus_freq_mhz * 4;
+	int_ctrl =  readl(&cpsw->wr_regs->int_control);
+	prescale = cpsw->bus_freq_mhz * 4;
 
 	if (!coal->rx_coalesce_usecs) {
 		int_ctrl &= ~(CPSW_INTPRESCALE_MASK | CPSW_INTPACEEN);
@@ -962,53 +994,69 @@ static int cpsw_set_coalesce(struct net_device *ndev,
 	}
 
 	num_interrupts = (1000 * addnl_dvdr) / coal_intvl;
-	writel(num_interrupts, &priv->wr_regs->rx_imax);
-	writel(num_interrupts, &priv->wr_regs->tx_imax);
+	writel(num_interrupts, &cpsw->wr_regs->rx_imax);
+	writel(num_interrupts, &cpsw->wr_regs->tx_imax);
 
 	int_ctrl |= CPSW_INTPACEEN;
 	int_ctrl &= (~CPSW_INTPRESCALE_MASK);
 	int_ctrl |= (prescale & CPSW_INTPRESCALE_MASK);
 
 update_return:
-	writel(int_ctrl, &priv->wr_regs->int_control);
+	writel(int_ctrl, &cpsw->wr_regs->int_control);
 
 	cpsw_notice(priv, timer, "Set coalesce to %d usecs.\n", coal_intvl);
-	if (priv->data.dual_emac) {
-		int i;
-
-		for (i = 0; i < priv->data.slaves; i++) {
-			priv = netdev_priv(priv->slaves[i].ndev);
-			priv->coal_intvl = coal_intvl;
-		}
-	} else {
-		priv->coal_intvl = coal_intvl;
-	}
+	cpsw->coal_intvl = coal_intvl;
 
 	return 0;
 }
 
 static int cpsw_get_sset_count(struct net_device *ndev, int sset)
 {
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
+
 	switch (sset) {
 	case ETH_SS_STATS:
-		return CPSW_STATS_LEN;
+		return (CPSW_STATS_COMMON_LEN +
+		       (cpsw->rx_ch_num + cpsw->tx_ch_num) *
+		       CPSW_STATS_CH_LEN);
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
+static void cpsw_add_ch_strings(u8 **p, int ch_num, int rx_dir)
+{
+	int ch_stats_len;
+	int line;
+	int i;
+
+	ch_stats_len = CPSW_STATS_CH_LEN * ch_num;
+	for (i = 0; i < ch_stats_len; i++) {
+		line = i % CPSW_STATS_CH_LEN;
+		snprintf(*p, ETH_GSTRING_LEN,
+			 "%s DMA chan %d: %s", rx_dir ? "Rx" : "Tx",
+			 i / CPSW_STATS_CH_LEN,
+			 cpsw_gstrings_ch_stats[line].stat_string);
+		*p += ETH_GSTRING_LEN;
+	}
+}
+
 static void cpsw_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 {
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 	u8 *p = data;
 	int i;
 
 	switch (stringset) {
 	case ETH_SS_STATS:
-		for (i = 0; i < CPSW_STATS_LEN; i++) {
+		for (i = 0; i < CPSW_STATS_COMMON_LEN; i++) {
 			memcpy(p, cpsw_gstrings_stats[i].stat_string,
 			       ETH_GSTRING_LEN);
 			p += ETH_GSTRING_LEN;
 		}
+
+		cpsw_add_ch_strings(&p, cpsw->rx_ch_num, 1);
+		cpsw_add_ch_strings(&p, cpsw->tx_ch_num, 0);
 		break;
 	}
 }
@@ -1016,86 +1064,78 @@ static void cpsw_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 static void cpsw_get_ethtool_stats(struct net_device *ndev,
 				    struct ethtool_stats *stats, u64 *data)
 {
-	struct cpsw_priv *priv = netdev_priv(ndev);
-	struct cpdma_chan_stats rx_stats;
-	struct cpdma_chan_stats tx_stats;
-	u32 val;
 	u8 *p;
-	int i;
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
+	struct cpdma_chan_stats ch_stats;
+	int i, l, ch;
 
 	/* Collect Davinci CPDMA stats for Rx and Tx Channel */
-	cpdma_chan_get_stats(priv->rxch, &rx_stats);
-	cpdma_chan_get_stats(priv->txch, &tx_stats);
-
-	for (i = 0; i < CPSW_STATS_LEN; i++) {
-		switch (cpsw_gstrings_stats[i].type) {
-		case CPSW_STATS:
-			val = readl(priv->hw_stats +
-				    cpsw_gstrings_stats[i].stat_offset);
-			data[i] = val;
-			break;
-
-		case CPDMA_RX_STATS:
-			p = (u8 *)&rx_stats +
-				cpsw_gstrings_stats[i].stat_offset;
-			data[i] = *(u32 *)p;
-			break;
+	for (l = 0; l < CPSW_STATS_COMMON_LEN; l++)
+		data[l] = readl(cpsw->hw_stats +
+				cpsw_gstrings_stats[l].stat_offset);
+
+	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
+		cpdma_chan_get_stats(cpsw->rxch[ch], &ch_stats);
+		for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
+			p = (u8 *)&ch_stats +
+				cpsw_gstrings_ch_stats[i].stat_offset;
+			data[l] = *(u32 *)p;
+		}
+	}
 
-		case CPDMA_TX_STATS:
-			p = (u8 *)&tx_stats +
-				cpsw_gstrings_stats[i].stat_offset;
-			data[i] = *(u32 *)p;
-			break;
+	for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
+		cpdma_chan_get_stats(cpsw->txch[ch], &ch_stats);
+		for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
+			p = (u8 *)&ch_stats +
+				cpsw_gstrings_ch_stats[i].stat_offset;
+			data[l] = *(u32 *)p;
 		}
 	}
 }
 
-static int cpsw_common_res_usage_state(struct cpsw_priv *priv)
+static int cpsw_common_res_usage_state(struct cpsw_common *cpsw)
 {
 	u32 i;
 	u32 usage_count = 0;
 
-	if (!priv->data.dual_emac)
+	if (!cpsw->data.dual_emac)
 		return 0;
 
-	for (i = 0; i < priv->data.slaves; i++)
-		if (priv->slaves[i].open_stat)
+	for (i = 0; i < cpsw->data.slaves; i++)
+		if (cpsw->slaves[i].open_stat)
 			usage_count++;
 
 	return usage_count;
 }
 
-static inline int cpsw_tx_packet_submit(struct net_device *ndev,
-			struct cpsw_priv *priv, struct sk_buff *skb)
+static inline int cpsw_tx_packet_submit(struct cpsw_priv *priv,
+					struct sk_buff *skb,
+					struct cpdma_chan *txch)
 {
-	if (!priv->data.dual_emac)
-		return cpdma_chan_submit(priv->txch, skb, skb->data,
-				  skb->len, 0);
+	struct cpsw_common *cpsw = priv->cpsw;
 
-	if (ndev == cpsw_get_slave_ndev(priv, 0))
-		return cpdma_chan_submit(priv->txch, skb, skb->data,
-				  skb->len, 1);
-	else
-		return cpdma_chan_submit(priv->txch, skb, skb->data,
-				  skb->len, 2);
+	return cpdma_chan_submit(txch, skb, skb->data, skb->len,
+				 priv->emac_port + cpsw->data.dual_emac);
 }
 
 static inline void cpsw_add_dual_emac_def_ale_entries(
 		struct cpsw_priv *priv, struct cpsw_slave *slave,
 		u32 slave_port)
 {
+	struct cpsw_common *cpsw = priv->cpsw;
 	u32 port_mask = 1 << slave_port | ALE_PORT_HOST;
 
-	if (priv->version == CPSW_VERSION_1)
+	if (cpsw->version == CPSW_VERSION_1)
 		slave_write(slave, slave->port_vlan, CPSW1_PORT_VLAN);
 	else
 		slave_write(slave, slave->port_vlan, CPSW2_PORT_VLAN);
-	cpsw_ale_add_vlan(priv->ale, slave->port_vlan, port_mask,
+	cpsw_ale_add_vlan(cpsw->ale, slave->port_vlan, port_mask,
 			  port_mask, port_mask, 0);
-	cpsw_ale_add_mcast(priv->ale, priv->ndev->broadcast,
+	cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
 			   port_mask, ALE_VLAN, slave->port_vlan, 0);
-	cpsw_ale_add_ucast(priv->ale, priv->mac_addr,
-		HOST_PORT_NUM, ALE_VLAN | ALE_SECURE, slave->port_vlan);
+	cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
+			   HOST_PORT_NUM, ALE_VLAN |
+			   ALE_SECURE, slave->port_vlan);
 }
 
 static void soft_reset_slave(struct cpsw_slave *slave)
@@ -1109,13 +1149,14 @@ static void soft_reset_slave(struct cpsw_slave *slave)
 static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
 {
 	u32 slave_port;
+	struct cpsw_common *cpsw = priv->cpsw;
 
 	soft_reset_slave(slave);
 
 	/* setup priority mapping */
 	__raw_writel(RX_PRIORITY_MAPPING, &slave->sliver->rx_pri_map);
 
-	switch (priv->version) {
+	switch (cpsw->version) {
 	case CPSW_VERSION_1:
 		slave_write(slave, TX_PRIORITY_MAPPING, CPSW1_TX_PRI_MAP);
 		break;
@@ -1127,17 +1168,17 @@ static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
 	}
 
 	/* setup max packet size, and mac address */
-	__raw_writel(priv->rx_packet_max, &slave->sliver->rx_maxlen);
+	__raw_writel(cpsw->rx_packet_max, &slave->sliver->rx_maxlen);
 	cpsw_set_slave_mac(slave, priv);
 
 	slave->mac_control = 0;	/* no link yet */
 
-	slave_port = cpsw_get_slave_port(priv, slave->slave_num);
+	slave_port = cpsw_get_slave_port(slave->slave_num);
 
-	if (priv->data.dual_emac)
+	if (cpsw->data.dual_emac)
 		cpsw_add_dual_emac_def_ale_entries(priv, slave, slave_port);
 	else
-		cpsw_ale_add_mcast(priv->ale, priv->ndev->broadcast,
+		cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
 				   1 << slave_port, 0, 0, ALE_MCAST_FWD_2);
 
 	if (slave->data->phy_node) {
@@ -1167,81 +1208,121 @@ static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
 	phy_start(slave->phy);
 
 	/* Configure GMII_SEL register */
-	cpsw_phy_sel(&priv->pdev->dev, slave->phy->interface, slave->slave_num);
+	cpsw_phy_sel(cpsw->dev, slave->phy->interface, slave->slave_num);
 }
 
 static inline void cpsw_add_default_vlan(struct cpsw_priv *priv)
 {
-	const int vlan = priv->data.default_vlan;
+	struct cpsw_common *cpsw = priv->cpsw;
+	const int vlan = cpsw->data.default_vlan;
 	u32 reg;
 	int i;
 	int unreg_mcast_mask;
 
-	reg = (priv->version == CPSW_VERSION_1) ? CPSW1_PORT_VLAN :
+	reg = (cpsw->version == CPSW_VERSION_1) ? CPSW1_PORT_VLAN :
 	       CPSW2_PORT_VLAN;
 
-	writel(vlan, &priv->host_port_regs->port_vlan);
+	writel(vlan, &cpsw->host_port_regs->port_vlan);
 
-	for (i = 0; i < priv->data.slaves; i++)
-		slave_write(priv->slaves + i, vlan, reg);
+	for (i = 0; i < cpsw->data.slaves; i++)
+		slave_write(cpsw->slaves + i, vlan, reg);
 
 	if (priv->ndev->flags & IFF_ALLMULTI)
 		unreg_mcast_mask = ALE_ALL_PORTS;
 	else
 		unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
 
-	cpsw_ale_add_vlan(priv->ale, vlan, ALE_ALL_PORTS,
+	cpsw_ale_add_vlan(cpsw->ale, vlan, ALE_ALL_PORTS,
 			  ALE_ALL_PORTS, ALE_ALL_PORTS,
 			  unreg_mcast_mask);
 }
 
 static void cpsw_init_host_port(struct cpsw_priv *priv)
 {
-	u32 control_reg;
 	u32 fifo_mode;
+	u32 control_reg;
+	struct cpsw_common *cpsw = priv->cpsw;
 
 	/* soft reset the controller and initialize ale */
-	soft_reset("cpsw", &priv->regs->soft_reset);
-	cpsw_ale_start(priv->ale);
+	soft_reset("cpsw", &cpsw->regs->soft_reset);
+	cpsw_ale_start(cpsw->ale);
 
 	/* switch to vlan unaware mode */
-	cpsw_ale_control_set(priv->ale, HOST_PORT_NUM, ALE_VLAN_AWARE,
+	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_VLAN_AWARE,
 			     CPSW_ALE_VLAN_AWARE);
-	control_reg = readl(&priv->regs->control);
+	control_reg = readl(&cpsw->regs->control);
 	control_reg |= CPSW_VLAN_AWARE;
-	writel(control_reg, &priv->regs->control);
-	fifo_mode = (priv->data.dual_emac) ? CPSW_FIFO_DUAL_MAC_MODE :
+	writel(control_reg, &cpsw->regs->control);
+	fifo_mode = (cpsw->data.dual_emac) ? CPSW_FIFO_DUAL_MAC_MODE :
 		     CPSW_FIFO_NORMAL_MODE;
-	writel(fifo_mode, &priv->host_port_regs->tx_in_ctl);
+	writel(fifo_mode, &cpsw->host_port_regs->tx_in_ctl);
 
 	/* setup host port priority mapping */
 	__raw_writel(CPDMA_TX_PRIORITY_MAP,
-		     &priv->host_port_regs->cpdma_tx_pri_map);
-	__raw_writel(0, &priv->host_port_regs->cpdma_rx_chan_map);
+		     &cpsw->host_port_regs->cpdma_tx_pri_map);
+	__raw_writel(0, &cpsw->host_port_regs->cpdma_rx_chan_map);
 
-	cpsw_ale_control_set(priv->ale, HOST_PORT_NUM,
+	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM,
 			     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
 
-	if (!priv->data.dual_emac) {
-		cpsw_ale_add_ucast(priv->ale, priv->mac_addr, HOST_PORT_NUM,
+	if (!cpsw->data.dual_emac) {
+		cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
 				   0, 0);
-		cpsw_ale_add_mcast(priv->ale, priv->ndev->broadcast,
+		cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
 				   ALE_PORT_HOST, 0, 0, ALE_MCAST_FWD_2);
 	}
 }
 
-static void cpsw_slave_stop(struct cpsw_slave *slave, struct cpsw_priv *priv)
+static int cpsw_fill_rx_channels(struct cpsw_priv *priv)
+{
+	struct cpsw_common *cpsw = priv->cpsw;
+	struct sk_buff *skb;
+	int ch_buf_num;
+	int ch, i, ret;
+
+	for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
+		ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxch[ch]);
+		for (i = 0; i < ch_buf_num; i++) {
+			skb = __netdev_alloc_skb_ip_align(priv->ndev,
+							  cpsw->rx_packet_max,
+							  GFP_KERNEL);
+			if (!skb) {
+				cpsw_err(priv, ifup, "cannot allocate skb\n");
+				return -ENOMEM;
+			}
+
+			skb_set_queue_mapping(skb, ch);
+			ret = cpdma_chan_submit(cpsw->rxch[ch], skb, skb->data,
+						skb_tailroom(skb), 0);
+			if (ret < 0) {
+				cpsw_err(priv, ifup,
+					 "cannot submit skb to channel %d rx, error %d\n",
+					 ch, ret);
+				kfree_skb(skb);
+				return ret;
+			}
+			kmemleak_not_leak(skb);
+		}
+
+		cpsw_info(priv, ifup, "ch %d rx, submitted %d descriptors\n",
+			  ch, ch_buf_num);
+	}
+
+	return 0;
+}
+
+static void cpsw_slave_stop(struct cpsw_slave *slave, struct cpsw_common *cpsw)
 {
 	u32 slave_port;
 
-	slave_port = cpsw_get_slave_port(priv, slave->slave_num);
+	slave_port = cpsw_get_slave_port(slave->slave_num);
 
 	if (!slave->phy)
 		return;
 	phy_stop(slave->phy);
 	phy_disconnect(slave->phy);
 	slave->phy = NULL;
-	cpsw_ale_control_set(priv->ale, slave_port,
+	cpsw_ale_control_set(cpsw->ale, slave_port,
 			     ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
 	soft_reset_slave(slave);
 }
@@ -1249,114 +1330,111 @@ static void cpsw_slave_stop(struct cpsw_slave *slave, struct cpsw_priv *priv)
 static int cpsw_ndo_open(struct net_device *ndev)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
-	int i, ret;
+	struct cpsw_common *cpsw = priv->cpsw;
+	int ret;
 	u32 reg;
 
-	ret = pm_runtime_get_sync(&priv->pdev->dev);
+	ret = pm_runtime_get_sync(cpsw->dev);
 	if (ret < 0) {
-		pm_runtime_put_noidle(&priv->pdev->dev);
+		pm_runtime_put_noidle(cpsw->dev);
 		return ret;
 	}
 
-	if (!cpsw_common_res_usage_state(priv))
-		cpsw_intr_disable(priv);
+	if (!cpsw_common_res_usage_state(cpsw))
+		cpsw_intr_disable(cpsw);
 	netif_carrier_off(ndev);
 
-	reg = priv->version;
+	/* Notify the stack of the actual queue counts. */
+	ret = netif_set_real_num_tx_queues(ndev, cpsw->tx_ch_num);
+	if (ret) {
+		dev_err(priv->dev, "cannot set real number of tx queues\n");
+		goto err_cleanup;
+	}
+
+	ret = netif_set_real_num_rx_queues(ndev, cpsw->rx_ch_num);
+	if (ret) {
+		dev_err(priv->dev, "cannot set real number of rx queues\n");
+		goto err_cleanup;
+	}
+
+	reg = cpsw->version;
 
 	dev_info(priv->dev, "initializing cpsw version %d.%d (%d)\n",
 		 CPSW_MAJOR_VERSION(reg), CPSW_MINOR_VERSION(reg),
 		 CPSW_RTL_VERSION(reg));
 
 	/* initialize host and slave ports */
-	if (!cpsw_common_res_usage_state(priv))
+	if (!cpsw_common_res_usage_state(cpsw))
 		cpsw_init_host_port(priv);
 	for_each_slave(priv, cpsw_slave_open, priv);
 
 	/* Add default VLAN */
-	if (!priv->data.dual_emac)
+	if (!cpsw->data.dual_emac)
 		cpsw_add_default_vlan(priv);
 	else
-		cpsw_ale_add_vlan(priv->ale, priv->data.default_vlan,
+		cpsw_ale_add_vlan(cpsw->ale, cpsw->data.default_vlan,
 				  ALE_ALL_PORTS, ALE_ALL_PORTS, 0, 0);
 
-	if (!cpsw_common_res_usage_state(priv)) {
-		struct cpsw_priv *priv_sl0 = cpsw_get_slave_priv(priv, 0);
-		int buf_num;
-
+	if (!cpsw_common_res_usage_state(cpsw)) {
 		/* setup tx dma to fixed prio and zero offset */
-		cpdma_control_set(priv->dma, CPDMA_TX_PRIO_FIXED, 1);
-		cpdma_control_set(priv->dma, CPDMA_RX_BUFFER_OFFSET, 0);
+		cpdma_control_set(cpsw->dma, CPDMA_TX_PRIO_FIXED, 1);
+		cpdma_control_set(cpsw->dma, CPDMA_RX_BUFFER_OFFSET, 0);
 
 		/* disable priority elevation */
-		__raw_writel(0, &priv->regs->ptype);
+		__raw_writel(0, &cpsw->regs->ptype);
 
 		/* enable statistics collection only on all ports */
-		__raw_writel(0x7, &priv->regs->stat_port_en);
+		__raw_writel(0x7, &cpsw->regs->stat_port_en);
 
 		/* Enable internal fifo flow control */
-		writel(0x7, &priv->regs->flow_control);
+		writel(0x7, &cpsw->regs->flow_control);
 
-		napi_enable(&priv_sl0->napi_rx);
-		napi_enable(&priv_sl0->napi_tx);
+		napi_enable(&cpsw->napi_rx);
+		napi_enable(&cpsw->napi_tx);
 
-		if (priv_sl0->tx_irq_disabled) {
-			priv_sl0->tx_irq_disabled = false;
-			enable_irq(priv->irqs_table[1]);
+		if (cpsw->tx_irq_disabled) {
+			cpsw->tx_irq_disabled = false;
+			enable_irq(cpsw->irqs_table[1]);
 		}
 
-		if (priv_sl0->rx_irq_disabled) {
-			priv_sl0->rx_irq_disabled = false;
-			enable_irq(priv->irqs_table[0]);
+		if (cpsw->rx_irq_disabled) {
+			cpsw->rx_irq_disabled = false;
+			enable_irq(cpsw->irqs_table[0]);
 		}
 
-		buf_num = cpdma_chan_get_rx_buf_num(priv->dma);
-		for (i = 0; i < buf_num; i++) {
-			struct sk_buff *skb;
+		ret = cpsw_fill_rx_channels(priv);
+		if (ret < 0)
+			goto err_cleanup;
 
-			ret = -ENOMEM;
-			skb = __netdev_alloc_skb_ip_align(priv->ndev,
-					priv->rx_packet_max, GFP_KERNEL);
-			if (!skb)
-				goto err_cleanup;
-			ret = cpdma_chan_submit(priv->rxch, skb, skb->data,
-					skb_tailroom(skb), 0);
-			if (ret < 0) {
-				kfree_skb(skb);
-				goto err_cleanup;
-			}
-		}
-		/* continue even if we didn't manage to submit all
-		 * receive descs
-		 */
-		cpsw_info(priv, ifup, "submitted %d rx descriptors\n", i);
-
-		if (cpts_register(&priv->pdev->dev, priv->cpts,
-				  priv->data.cpts_clock_mult,
-				  priv->data.cpts_clock_shift))
+		if (cpts_register(cpsw->dev, cpsw->cpts,
+				  cpsw->data.cpts_clock_mult,
+				  cpsw->data.cpts_clock_shift))
 			dev_err(priv->dev, "error registering cpts device\n");
 
 	}
 
 	/* Enable Interrupt pacing if configured */
-	if (priv->coal_intvl != 0) {
+	if (cpsw->coal_intvl != 0) {
 		struct ethtool_coalesce coal;
 
-		coal.rx_coalesce_usecs = priv->coal_intvl;
+		coal.rx_coalesce_usecs = cpsw->coal_intvl;
 		cpsw_set_coalesce(ndev, &coal);
 	}
 
-	cpdma_ctlr_start(priv->dma);
-	cpsw_intr_enable(priv);
+	cpdma_ctlr_start(cpsw->dma);
+	cpsw_intr_enable(cpsw);
+
+	if (cpsw->data.dual_emac)
+		cpsw->slaves[priv->emac_port].open_stat = true;
+
+	netif_tx_start_all_queues(ndev);
 
-	if (priv->data.dual_emac)
-		priv->slaves[priv->emac_port].open_stat = true;
 	return 0;
 
 err_cleanup:
-	cpdma_ctlr_stop(priv->dma);
-	for_each_slave(priv, cpsw_slave_stop, priv);
-	pm_runtime_put_sync(&priv->pdev->dev);
+	cpdma_ctlr_stop(cpsw->dma);
+	for_each_slave(priv, cpsw_slave_stop, cpsw);
+	pm_runtime_put_sync(cpsw->dev);
 	netif_carrier_off(priv->ndev);
 	return ret;
 }
@@ -1364,25 +1442,24 @@ err_cleanup:
 static int cpsw_ndo_stop(struct net_device *ndev)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = priv->cpsw;
 
 	cpsw_info(priv, ifdown, "shutting down cpsw device\n");
-	netif_stop_queue(priv->ndev);
+	netif_tx_stop_all_queues(priv->ndev);
 	netif_carrier_off(priv->ndev);
 
-	if (cpsw_common_res_usage_state(priv) <= 1) {
-		struct cpsw_priv *priv_sl0 = cpsw_get_slave_priv(priv, 0);
-
-		napi_disable(&priv_sl0->napi_rx);
-		napi_disable(&priv_sl0->napi_tx);
-		cpts_unregister(priv->cpts);
-		cpsw_intr_disable(priv);
-		cpdma_ctlr_stop(priv->dma);
-		cpsw_ale_stop(priv->ale);
-	}
-	for_each_slave(priv, cpsw_slave_stop, priv);
-	pm_runtime_put_sync(&priv->pdev->dev);
-	if (priv->data.dual_emac)
-		priv->slaves[priv->emac_port].open_stat = false;
+	if (cpsw_common_res_usage_state(cpsw) <= 1) {
+		napi_disable(&cpsw->napi_rx);
+		napi_disable(&cpsw->napi_tx);
+		cpts_unregister(cpsw->cpts);
+		cpsw_intr_disable(cpsw);
+		cpdma_ctlr_stop(cpsw->dma);
+		cpsw_ale_stop(cpsw->ale);
+	}
+	for_each_slave(priv, cpsw_slave_stop, cpsw);
+	pm_runtime_put_sync(cpsw->dev);
+	if (cpsw->data.dual_emac)
+		cpsw->slaves[priv->emac_port].open_stat = false;
 	return 0;
 }
 
@@ -1390,7 +1467,10 @@ static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
 				       struct net_device *ndev)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
-	int ret;
+	struct cpsw_common *cpsw = priv->cpsw;
+	struct netdev_queue *txq;
+	struct cpdma_chan *txch;
+	int ret, q_idx;
 
 	netif_trans_update(ndev);
 
@@ -1401,12 +1481,17 @@ static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
 	}
 
 	if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
-				priv->cpts->tx_enable)
+				cpsw->cpts->tx_enable)
 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 
 	skb_tx_timestamp(skb);
 
-	ret = cpsw_tx_packet_submit(ndev, priv, skb);
+	q_idx = skb_get_queue_mapping(skb);
+	if (q_idx >= cpsw->tx_ch_num)
+		q_idx = q_idx % cpsw->tx_ch_num;
+
+	txch = cpsw->txch[q_idx];
+	ret = cpsw_tx_packet_submit(priv, skb, txch);
 	if (unlikely(ret != 0)) {
 		cpsw_err(priv, tx_err, "desc submit failed\n");
 		goto fail;
@@ -1415,24 +1500,27 @@ static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
 	/* If there is no more tx desc left free then we need to
 	 * tell the kernel to stop sending us tx frames.
 	 */
-	if (unlikely(!cpdma_check_free_tx_desc(priv->txch)))
-		netif_stop_queue(ndev);
+	if (unlikely(!cpdma_check_free_tx_desc(txch))) {
+		txq = netdev_get_tx_queue(ndev, q_idx);
+		netif_tx_stop_queue(txq);
+	}
 
 	return NETDEV_TX_OK;
 fail:
 	ndev->stats.tx_dropped++;
-	netif_stop_queue(ndev);
+	txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
+	netif_tx_stop_queue(txq);
 	return NETDEV_TX_BUSY;
 }
 
 #ifdef CONFIG_TI_CPTS
 
-static void cpsw_hwtstamp_v1(struct cpsw_priv *priv)
+static void cpsw_hwtstamp_v1(struct cpsw_common *cpsw)
 {
-	struct cpsw_slave *slave = &priv->slaves[priv->data.active_slave];
+	struct cpsw_slave *slave = &cpsw->slaves[cpsw->data.active_slave];
 	u32 ts_en, seq_id;
 
-	if (!priv->cpts->tx_enable && !priv->cpts->rx_enable) {
+	if (!cpsw->cpts->tx_enable && !cpsw->cpts->rx_enable) {
 		slave_write(slave, 0, CPSW1_TS_CTL);
 		return;
 	}
@@ -1440,10 +1528,10 @@ static void cpsw_hwtstamp_v1(struct cpsw_priv *priv)
 	seq_id = (30 << CPSW_V1_SEQ_ID_OFS_SHIFT) | ETH_P_1588;
 	ts_en = EVENT_MSG_BITS << CPSW_V1_MSG_TYPE_OFS;
 
-	if (priv->cpts->tx_enable)
+	if (cpsw->cpts->tx_enable)
 		ts_en |= CPSW_V1_TS_TX_EN;
 
-	if (priv->cpts->rx_enable)
+	if (cpsw->cpts->rx_enable)
 		ts_en |= CPSW_V1_TS_RX_EN;
 
 	slave_write(slave, ts_en, CPSW1_TS_CTL);
@@ -1453,32 +1541,33 @@ static void cpsw_hwtstamp_v1(struct cpsw_priv *priv)
 static void cpsw_hwtstamp_v2(struct cpsw_priv *priv)
 {
 	struct cpsw_slave *slave;
+	struct cpsw_common *cpsw = priv->cpsw;
 	u32 ctrl, mtype;
 
-	if (priv->data.dual_emac)
-		slave = &priv->slaves[priv->emac_port];
+	if (cpsw->data.dual_emac)
+		slave = &cpsw->slaves[priv->emac_port];
 	else
-		slave = &priv->slaves[priv->data.active_slave];
+		slave = &cpsw->slaves[cpsw->data.active_slave];
 
 	ctrl = slave_read(slave, CPSW2_CONTROL);
-	switch (priv->version) {
+	switch (cpsw->version) {
 	case CPSW_VERSION_2:
 		ctrl &= ~CTRL_V2_ALL_TS_MASK;
 
-		if (priv->cpts->tx_enable)
+		if (cpsw->cpts->tx_enable)
 			ctrl |= CTRL_V2_TX_TS_BITS;
 
-		if (priv->cpts->rx_enable)
+		if (cpsw->cpts->rx_enable)
 			ctrl |= CTRL_V2_RX_TS_BITS;
 		break;
 	case CPSW_VERSION_3:
 	default:
 		ctrl &= ~CTRL_V3_ALL_TS_MASK;
 
-		if (priv->cpts->tx_enable)
+		if (cpsw->cpts->tx_enable)
 			ctrl |= CTRL_V3_TX_TS_BITS;
 
-		if (priv->cpts->rx_enable)
+		if (cpsw->cpts->rx_enable)
 			ctrl |= CTRL_V3_RX_TS_BITS;
 		break;
 	}
@@ -1487,18 +1576,19 @@ static void cpsw_hwtstamp_v2(struct cpsw_priv *priv)
 
 	slave_write(slave, mtype, CPSW2_TS_SEQ_MTYPE);
 	slave_write(slave, ctrl, CPSW2_CONTROL);
-	__raw_writel(ETH_P_1588, &priv->regs->ts_ltype);
+	__raw_writel(ETH_P_1588, &cpsw->regs->ts_ltype);
 }
 
 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
 {
 	struct cpsw_priv *priv = netdev_priv(dev);
-	struct cpts *cpts = priv->cpts;
 	struct hwtstamp_config cfg;
+	struct cpsw_common *cpsw = priv->cpsw;
+	struct cpts *cpts = cpsw->cpts;
 
-	if (priv->version != CPSW_VERSION_1 &&
-	    priv->version != CPSW_VERSION_2 &&
-	    priv->version != CPSW_VERSION_3)
+	if (cpsw->version != CPSW_VERSION_1 &&
+	    cpsw->version != CPSW_VERSION_2 &&
+	    cpsw->version != CPSW_VERSION_3)
 		return -EOPNOTSUPP;
 
 	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
@@ -1538,9 +1628,9 @@ static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
 
 	cpts->tx_enable = cfg.tx_type == HWTSTAMP_TX_ON;
 
-	switch (priv->version) {
+	switch (cpsw->version) {
 	case CPSW_VERSION_1:
-		cpsw_hwtstamp_v1(priv);
+		cpsw_hwtstamp_v1(cpsw);
 		break;
 	case CPSW_VERSION_2:
 	case CPSW_VERSION_3:
@@ -1555,13 +1645,13 @@ static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
 
 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
 {
-	struct cpsw_priv *priv = netdev_priv(dev);
-	struct cpts *cpts = priv->cpts;
+	struct cpsw_common *cpsw = ndev_to_cpsw(dev);
+	struct cpts *cpts = cpsw->cpts;
 	struct hwtstamp_config cfg;
 
-	if (priv->version != CPSW_VERSION_1 &&
-	    priv->version != CPSW_VERSION_2 &&
-	    priv->version != CPSW_VERSION_3)
+	if (cpsw->version != CPSW_VERSION_1 &&
+	    cpsw->version != CPSW_VERSION_2 &&
+	    cpsw->version != CPSW_VERSION_3)
 		return -EOPNOTSUPP;
 
 	cfg.flags = 0;
@@ -1577,7 +1667,8 @@ static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
 static int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
 {
 	struct cpsw_priv *priv = netdev_priv(dev);
-	int slave_no = cpsw_slave_index(priv);
+	struct cpsw_common *cpsw = priv->cpsw;
+	int slave_no = cpsw_slave_index(cpsw, priv);
 
 	if (!netif_running(dev))
 		return -EINVAL;
@@ -1591,27 +1682,33 @@ static int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
 #endif
 	}
 
-	if (!priv->slaves[slave_no].phy)
+	if (!cpsw->slaves[slave_no].phy)
 		return -EOPNOTSUPP;
-	return phy_mii_ioctl(priv->slaves[slave_no].phy, req, cmd);
+	return phy_mii_ioctl(cpsw->slaves[slave_no].phy, req, cmd);
 }
 
 static void cpsw_ndo_tx_timeout(struct net_device *ndev)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = priv->cpsw;
+	int ch;
 
 	cpsw_err(priv, tx_err, "transmit timeout, restarting dma\n");
 	ndev->stats.tx_errors++;
-	cpsw_intr_disable(priv);
-	cpdma_chan_stop(priv->txch);
-	cpdma_chan_start(priv->txch);
-	cpsw_intr_enable(priv);
+	cpsw_intr_disable(cpsw);
+	for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
+		cpdma_chan_stop(cpsw->txch[ch]);
+		cpdma_chan_start(cpsw->txch[ch]);
+	}
+
+	cpsw_intr_enable(cpsw);
 }
 
 static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
 	struct sockaddr *addr = (struct sockaddr *)p;
+	struct cpsw_common *cpsw = priv->cpsw;
 	int flags = 0;
 	u16 vid = 0;
 	int ret;
@@ -1619,27 +1716,27 @@ static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
 	if (!is_valid_ether_addr(addr->sa_data))
 		return -EADDRNOTAVAIL;
 
-	ret = pm_runtime_get_sync(&priv->pdev->dev);
+	ret = pm_runtime_get_sync(cpsw->dev);
 	if (ret < 0) {
-		pm_runtime_put_noidle(&priv->pdev->dev);
+		pm_runtime_put_noidle(cpsw->dev);
 		return ret;
 	}
 
-	if (priv->data.dual_emac) {
-		vid = priv->slaves[priv->emac_port].port_vlan;
+	if (cpsw->data.dual_emac) {
+		vid = cpsw->slaves[priv->emac_port].port_vlan;
 		flags = ALE_VLAN;
 	}
 
-	cpsw_ale_del_ucast(priv->ale, priv->mac_addr, HOST_PORT_NUM,
+	cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
 			   flags, vid);
-	cpsw_ale_add_ucast(priv->ale, addr->sa_data, HOST_PORT_NUM,
+	cpsw_ale_add_ucast(cpsw->ale, addr->sa_data, HOST_PORT_NUM,
 			   flags, vid);
 
 	memcpy(priv->mac_addr, addr->sa_data, ETH_ALEN);
 	memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
 	for_each_slave(priv, cpsw_set_slave_mac, priv);
 
-	pm_runtime_put(&priv->pdev->dev);
+	pm_runtime_put(cpsw->dev);
 
 	return 0;
 }
@@ -1647,12 +1744,12 @@ static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
 #ifdef CONFIG_NET_POLL_CONTROLLER
 static void cpsw_ndo_poll_controller(struct net_device *ndev)
 {
-	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 
-	cpsw_intr_disable(priv);
-	cpsw_rx_interrupt(priv->irqs_table[0], priv);
-	cpsw_tx_interrupt(priv->irqs_table[1], priv);
-	cpsw_intr_enable(priv);
+	cpsw_intr_disable(cpsw);
+	cpsw_rx_interrupt(cpsw->irqs_table[0], cpsw);
+	cpsw_tx_interrupt(cpsw->irqs_table[1], cpsw);
+	cpsw_intr_enable(cpsw);
 }
 #endif
 
@@ -1662,8 +1759,9 @@ static inline int cpsw_add_vlan_ale_entry(struct cpsw_priv *priv,
 	int ret;
 	int unreg_mcast_mask = 0;
 	u32 port_mask;
+	struct cpsw_common *cpsw = priv->cpsw;
 
-	if (priv->data.dual_emac) {
+	if (cpsw->data.dual_emac) {
 		port_mask = (1 << (priv->emac_port + 1)) | ALE_PORT_HOST;
 
 		if (priv->ndev->flags & IFF_ALLMULTI)
@@ -1677,27 +1775,27 @@ static inline int cpsw_add_vlan_ale_entry(struct cpsw_priv *priv,
 			unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
 	}
 
-	ret = cpsw_ale_add_vlan(priv->ale, vid, port_mask, 0, port_mask,
+	ret = cpsw_ale_add_vlan(cpsw->ale, vid, port_mask, 0, port_mask,
 				unreg_mcast_mask);
 	if (ret != 0)
 		return ret;
 
-	ret = cpsw_ale_add_ucast(priv->ale, priv->mac_addr,
+	ret = cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
 				 HOST_PORT_NUM, ALE_VLAN, vid);
 	if (ret != 0)
 		goto clean_vid;
 
-	ret = cpsw_ale_add_mcast(priv->ale, priv->ndev->broadcast,
+	ret = cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
 				 port_mask, ALE_VLAN, vid, 0);
 	if (ret != 0)
 		goto clean_vlan_ucast;
 	return 0;
 
 clean_vlan_ucast:
-	cpsw_ale_del_ucast(priv->ale, priv->mac_addr,
+	cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
 			   HOST_PORT_NUM, ALE_VLAN, vid);
 clean_vid:
-	cpsw_ale_del_vlan(priv->ale, vid, 0);
+	cpsw_ale_del_vlan(cpsw->ale, vid, 0);
 	return ret;
 }
 
@@ -1705,26 +1803,27 @@ static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
 				    __be16 proto, u16 vid)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = priv->cpsw;
 	int ret;
 
-	if (vid == priv->data.default_vlan)
+	if (vid == cpsw->data.default_vlan)
 		return 0;
 
-	ret = pm_runtime_get_sync(&priv->pdev->dev);
+	ret = pm_runtime_get_sync(cpsw->dev);
 	if (ret < 0) {
-		pm_runtime_put_noidle(&priv->pdev->dev);
+		pm_runtime_put_noidle(cpsw->dev);
 		return ret;
 	}
 
-	if (priv->data.dual_emac) {
+	if (cpsw->data.dual_emac) {
 		/* In dual EMAC, reserved VLAN id should not be used for
 		 * creating VLAN interfaces as this can break the dual
 		 * EMAC port separation
 		 */
 		int i;
 
-		for (i = 0; i < priv->data.slaves; i++) {
-			if (vid == priv->slaves[i].port_vlan)
+		for (i = 0; i < cpsw->data.slaves; i++) {
+			if (vid == cpsw->slaves[i].port_vlan)
 				return -EINVAL;
 		}
 	}
@@ -1732,7 +1831,7 @@ static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
 	dev_info(priv->dev, "Adding vlanid %d to vlan filter\n", vid);
 	ret = cpsw_add_vlan_ale_entry(priv, vid);
 
-	pm_runtime_put(&priv->pdev->dev);
+	pm_runtime_put(cpsw->dev);
 	return ret;
 }
 
@@ -1740,39 +1839,40 @@ static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev,
 				     __be16 proto, u16 vid)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = priv->cpsw;
 	int ret;
 
-	if (vid == priv->data.default_vlan)
+	if (vid == cpsw->data.default_vlan)
 		return 0;
 
-	ret = pm_runtime_get_sync(&priv->pdev->dev);
+	ret = pm_runtime_get_sync(cpsw->dev);
 	if (ret < 0) {
-		pm_runtime_put_noidle(&priv->pdev->dev);
+		pm_runtime_put_noidle(cpsw->dev);
 		return ret;
 	}
 
-	if (priv->data.dual_emac) {
+	if (cpsw->data.dual_emac) {
 		int i;
 
-		for (i = 0; i < priv->data.slaves; i++) {
-			if (vid == priv->slaves[i].port_vlan)
+		for (i = 0; i < cpsw->data.slaves; i++) {
+			if (vid == cpsw->slaves[i].port_vlan)
 				return -EINVAL;
 		}
 	}
 
 	dev_info(priv->dev, "removing vlanid %d from vlan filter\n", vid);
-	ret = cpsw_ale_del_vlan(priv->ale, vid, 0);
+	ret = cpsw_ale_del_vlan(cpsw->ale, vid, 0);
 	if (ret != 0)
 		return ret;
 
-	ret = cpsw_ale_del_ucast(priv->ale, priv->mac_addr,
+	ret = cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
 				 HOST_PORT_NUM, ALE_VLAN, vid);
 	if (ret != 0)
 		return ret;
 
-	ret = cpsw_ale_del_mcast(priv->ale, priv->ndev->broadcast,
+	ret = cpsw_ale_del_mcast(cpsw->ale, priv->ndev->broadcast,
 				 0, ALE_VLAN, vid);
-	pm_runtime_put(&priv->pdev->dev);
+	pm_runtime_put(cpsw->dev);
 	return ret;
 }
 
@@ -1795,31 +1895,32 @@ static const struct net_device_ops cpsw_netdev_ops = {
 
 static int cpsw_get_regs_len(struct net_device *ndev)
 {
-	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 
-	return priv->data.ale_entries * ALE_ENTRY_WORDS * sizeof(u32);
+	return cpsw->data.ale_entries * ALE_ENTRY_WORDS * sizeof(u32);
 }
 
 static void cpsw_get_regs(struct net_device *ndev,
 			  struct ethtool_regs *regs, void *p)
 {
-	struct cpsw_priv *priv = netdev_priv(ndev);
 	u32 *reg = p;
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 
 	/* update CPSW IP version */
-	regs->version = priv->version;
+	regs->version = cpsw->version;
 
-	cpsw_ale_dump(priv->ale, reg);
+	cpsw_ale_dump(cpsw->ale, reg);
 }
 
 static void cpsw_get_drvinfo(struct net_device *ndev,
 			     struct ethtool_drvinfo *info)
 {
-	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
+	struct platform_device	*pdev = to_platform_device(cpsw->dev);
 
 	strlcpy(info->driver, "cpsw", sizeof(info->driver));
 	strlcpy(info->version, "1.0", sizeof(info->version));
-	strlcpy(info->bus_info, priv->pdev->name, sizeof(info->bus_info));
+	strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
 }
 
 static u32 cpsw_get_msglevel(struct net_device *ndev)
@@ -1838,7 +1939,7 @@ static int cpsw_get_ts_info(struct net_device *ndev,
 			    struct ethtool_ts_info *info)
 {
 #ifdef CONFIG_TI_CPTS
-	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 
 	info->so_timestamping =
 		SOF_TIMESTAMPING_TX_HARDWARE |
@@ -1847,7 +1948,7 @@ static int cpsw_get_ts_info(struct net_device *ndev,
 		SOF_TIMESTAMPING_RX_SOFTWARE |
 		SOF_TIMESTAMPING_SOFTWARE |
 		SOF_TIMESTAMPING_RAW_HARDWARE;
-	info->phc_index = priv->cpts->phc_index;
+	info->phc_index = cpsw->cpts->phc_index;
 	info->tx_types =
 		(1 << HWTSTAMP_TX_OFF) |
 		(1 << HWTSTAMP_TX_ON);
@@ -1870,10 +1971,11 @@ static int cpsw_get_settings(struct net_device *ndev,
 			     struct ethtool_cmd *ecmd)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
-	int slave_no = cpsw_slave_index(priv);
+	struct cpsw_common *cpsw = priv->cpsw;
+	int slave_no = cpsw_slave_index(cpsw, priv);
 
-	if (priv->slaves[slave_no].phy)
-		return phy_ethtool_gset(priv->slaves[slave_no].phy, ecmd);
+	if (cpsw->slaves[slave_no].phy)
+		return phy_ethtool_gset(cpsw->slaves[slave_no].phy, ecmd);
 	else
 		return -EOPNOTSUPP;
 }
@@ -1881,10 +1983,11 @@ static int cpsw_get_settings(struct net_device *ndev,
 static int cpsw_set_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
-	int slave_no = cpsw_slave_index(priv);
+	struct cpsw_common *cpsw = priv->cpsw;
+	int slave_no = cpsw_slave_index(cpsw, priv);
 
-	if (priv->slaves[slave_no].phy)
-		return phy_ethtool_sset(priv->slaves[slave_no].phy, ecmd);
+	if (cpsw->slaves[slave_no].phy)
+		return phy_ethtool_sset(cpsw->slaves[slave_no].phy, ecmd);
 	else
 		return -EOPNOTSUPP;
 }
@@ -1892,22 +1995,24 @@ static int cpsw_set_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
 static void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
-	int slave_no = cpsw_slave_index(priv);
+	struct cpsw_common *cpsw = priv->cpsw;
+	int slave_no = cpsw_slave_index(cpsw, priv);
 
 	wol->supported = 0;
 	wol->wolopts = 0;
 
-	if (priv->slaves[slave_no].phy)
-		phy_ethtool_get_wol(priv->slaves[slave_no].phy, wol);
+	if (cpsw->slaves[slave_no].phy)
+		phy_ethtool_get_wol(cpsw->slaves[slave_no].phy, wol);
 }
 
 static int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
-	int slave_no = cpsw_slave_index(priv);
+	struct cpsw_common *cpsw = priv->cpsw;
+	int slave_no = cpsw_slave_index(cpsw, priv);
 
-	if (priv->slaves[slave_no].phy)
-		return phy_ethtool_set_wol(priv->slaves[slave_no].phy, wol);
+	if (cpsw->slaves[slave_no].phy)
+		return phy_ethtool_set_wol(cpsw->slaves[slave_no].phy, wol);
 	else
 		return -EOPNOTSUPP;
 }
@@ -1938,12 +2043,13 @@ static int cpsw_set_pauseparam(struct net_device *ndev,
 static int cpsw_ethtool_op_begin(struct net_device *ndev)
 {
 	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = priv->cpsw;
 	int ret;
 
-	ret = pm_runtime_get_sync(&priv->pdev->dev);
+	ret = pm_runtime_get_sync(cpsw->dev);
 	if (ret < 0) {
 		cpsw_err(priv, drv, "ethtool begin failed %d\n", ret);
-		pm_runtime_put_noidle(&priv->pdev->dev);
+		pm_runtime_put_noidle(cpsw->dev);
 	}
 
 	return ret;
@@ -1954,11 +2060,185 @@ static void cpsw_ethtool_op_complete(struct net_device *ndev)
 	struct cpsw_priv *priv = netdev_priv(ndev);
 	int ret;
 
-	ret = pm_runtime_put(&priv->pdev->dev);
+	ret = pm_runtime_put(priv->cpsw->dev);
 	if (ret < 0)
 		cpsw_err(priv, drv, "ethtool complete failed %d\n", ret);
 }
 
+static void cpsw_get_channels(struct net_device *ndev,
+			      struct ethtool_channels *ch)
+{
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
+
+	ch->max_combined = 0;
+	ch->max_rx = CPSW_MAX_QUEUES;
+	ch->max_tx = CPSW_MAX_QUEUES;
+	ch->max_other = 0;
+	ch->other_count = 0;
+	ch->rx_count = cpsw->rx_ch_num;
+	ch->tx_count = cpsw->tx_ch_num;
+	ch->combined_count = 0;
+}
+
+static int cpsw_check_ch_settings(struct cpsw_common *cpsw,
+				  struct ethtool_channels *ch)
+{
+	if (ch->combined_count)
+		return -EINVAL;
+
+	/* verify we have at least one channel in each direction */
+	if (!ch->rx_count || !ch->tx_count)
+		return -EINVAL;
+
+	if (ch->rx_count > cpsw->data.channels ||
+	    ch->tx_count > cpsw->data.channels)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx)
+{
+	int (*poll)(struct napi_struct *, int);
+	struct cpsw_common *cpsw = priv->cpsw;
+	void (*handler)(void *, int, int);
+	struct cpdma_chan **chan;
+	int ret, *ch;
+
+	if (rx) {
+		ch = &cpsw->rx_ch_num;
+		chan = cpsw->rxch;
+		handler = cpsw_rx_handler;
+		poll = cpsw_rx_poll;
+	} else {
+		ch = &cpsw->tx_ch_num;
+		chan = cpsw->txch;
+		handler = cpsw_tx_handler;
+		poll = cpsw_tx_poll;
+	}
+
+	while (*ch < ch_num) {
+		chan[*ch] = cpdma_chan_create(cpsw->dma, *ch, handler, rx);
+
+		if (IS_ERR(chan[*ch]))
+			return PTR_ERR(chan[*ch]);
+
+		if (!chan[*ch])
+			return -EINVAL;
+
+		cpsw_info(priv, ifup, "created new %d %s channel\n", *ch,
+			  (rx ? "rx" : "tx"));
+		(*ch)++;
+	}
+
+	while (*ch > ch_num) {
+		(*ch)--;
+
+		ret = cpdma_chan_destroy(chan[*ch]);
+		if (ret)
+			return ret;
+
+		cpsw_info(priv, ifup, "destroyed %d %s channel\n", *ch,
+			  (rx ? "rx" : "tx"));
+	}
+
+	return 0;
+}
+
+static int cpsw_update_channels(struct cpsw_priv *priv,
+				struct ethtool_channels *ch)
+{
+	int ret;
+
+	ret = cpsw_update_channels_res(priv, ch->rx_count, 1);
+	if (ret)
+		return ret;
+
+	ret = cpsw_update_channels_res(priv, ch->tx_count, 0);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int cpsw_set_channels(struct net_device *ndev,
+			     struct ethtool_channels *chs)
+{
+	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = priv->cpsw;
+	struct cpsw_slave *slave;
+	int i, ret;
+
+	ret = cpsw_check_ch_settings(cpsw, chs);
+	if (ret < 0)
+		return ret;
+
+	/* Disable NAPI scheduling */
+	cpsw_intr_disable(cpsw);
+
+	/* Stop all transmit queues for every network device.
+	 * Disable re-using rx descriptors with dormant_on.
+	 */
+	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
+		if (!(slave->ndev && netif_running(slave->ndev)))
+			continue;
+
+		netif_tx_stop_all_queues(slave->ndev);
+		netif_dormant_on(slave->ndev);
+	}
+
+	/* Handle rest of tx packets and stop cpdma channels */
+	cpdma_ctlr_stop(cpsw->dma);
+	ret = cpsw_update_channels(priv, chs);
+	if (ret)
+		goto err;
+
+	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
+		if (!(slave->ndev && netif_running(slave->ndev)))
+			continue;
+
+		/* Inform stack about new count of queues */
+		ret = netif_set_real_num_tx_queues(slave->ndev,
+						   cpsw->tx_ch_num);
+		if (ret) {
+			dev_err(priv->dev, "cannot set real number of tx queues\n");
+			goto err;
+		}
+
+		ret = netif_set_real_num_rx_queues(slave->ndev,
+						   cpsw->rx_ch_num);
+		if (ret) {
+			dev_err(priv->dev, "cannot set real number of rx queues\n");
+			goto err;
+		}
+
+		/* Enable rx packets handling */
+		netif_dormant_off(slave->ndev);
+	}
+
+	if (cpsw_common_res_usage_state(cpsw)) {
+		ret = cpsw_fill_rx_channels(priv);
+		if (ret)
+			goto err;
+
+		/* After this receive is started */
+		cpdma_ctlr_start(cpsw->dma);
+		cpsw_intr_enable(cpsw);
+	}
+
+	/* Resume transmit for every affected interface */
+	for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
+		if (!(slave->ndev && netif_running(slave->ndev)))
+			continue;
+		netif_tx_start_all_queues(slave->ndev);
+	}
+	return 0;
+err:
+	dev_err(priv->dev, "cannot update channels number, closing device\n");
+	dev_close(ndev);
+	return ret;
+}
+
 static const struct ethtool_ops cpsw_ethtool_ops = {
 	.get_drvinfo	= cpsw_get_drvinfo,
 	.get_msglevel	= cpsw_get_msglevel,
@@ -1980,14 +2260,16 @@ static const struct ethtool_ops cpsw_ethtool_ops = {
 	.get_regs	= cpsw_get_regs,
 	.begin		= cpsw_ethtool_op_begin,
 	.complete	= cpsw_ethtool_op_complete,
+	.get_channels	= cpsw_get_channels,
+	.set_channels	= cpsw_set_channels,
 };
 
-static void cpsw_slave_init(struct cpsw_slave *slave, struct cpsw_priv *priv,
+static void cpsw_slave_init(struct cpsw_slave *slave, struct cpsw_common *cpsw,
 			    u32 slave_reg_ofs, u32 sliver_reg_ofs)
 {
-	void __iomem		*regs = priv->regs;
+	void __iomem		*regs = cpsw->regs;
 	int			slave_num = slave->slave_num;
-	struct cpsw_slave_data	*data = priv->data.slave_data + slave_num;
+	struct cpsw_slave_data	*data = cpsw->data.slave_data + slave_num;
 
 	slave->data	= data;
 	slave->regs	= regs + slave_reg_ofs;
@@ -2158,71 +2440,50 @@ no_phy_slave:
 	return 0;
 }
 
-static int cpsw_probe_dual_emac(struct platform_device *pdev,
-				struct cpsw_priv *priv)
+static int cpsw_probe_dual_emac(struct cpsw_priv *priv)
 {
-	struct cpsw_platform_data	*data = &priv->data;
+	struct cpsw_common		*cpsw = priv->cpsw;
+	struct cpsw_platform_data	*data = &cpsw->data;
 	struct net_device		*ndev;
 	struct cpsw_priv		*priv_sl2;
-	int ret = 0, i;
+	int ret = 0;
 
-	ndev = alloc_etherdev(sizeof(struct cpsw_priv));
+	ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
 	if (!ndev) {
-		dev_err(&pdev->dev, "cpsw: error allocating net_device\n");
+		dev_err(cpsw->dev, "cpsw: error allocating net_device\n");
 		return -ENOMEM;
 	}
 
 	priv_sl2 = netdev_priv(ndev);
-	priv_sl2->data = *data;
-	priv_sl2->pdev = pdev;
+	priv_sl2->cpsw = cpsw;
 	priv_sl2->ndev = ndev;
 	priv_sl2->dev  = &ndev->dev;
 	priv_sl2->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
-	priv_sl2->rx_packet_max = max(rx_packet_max, 128);
 
 	if (is_valid_ether_addr(data->slave_data[1].mac_addr)) {
 		memcpy(priv_sl2->mac_addr, data->slave_data[1].mac_addr,
 			ETH_ALEN);
-		dev_info(&pdev->dev, "cpsw: Detected MACID = %pM\n", priv_sl2->mac_addr);
+		dev_info(cpsw->dev, "cpsw: Detected MACID = %pM\n",
+			 priv_sl2->mac_addr);
 	} else {
 		random_ether_addr(priv_sl2->mac_addr);
-		dev_info(&pdev->dev, "cpsw: Random MACID = %pM\n", priv_sl2->mac_addr);
+		dev_info(cpsw->dev, "cpsw: Random MACID = %pM\n",
+			 priv_sl2->mac_addr);
 	}
 	memcpy(ndev->dev_addr, priv_sl2->mac_addr, ETH_ALEN);
 
-	priv_sl2->slaves = priv->slaves;
-	priv_sl2->clk = priv->clk;
-
-	priv_sl2->coal_intvl = 0;
-	priv_sl2->bus_freq_mhz = priv->bus_freq_mhz;
-
-	priv_sl2->regs = priv->regs;
-	priv_sl2->host_port_regs = priv->host_port_regs;
-	priv_sl2->wr_regs = priv->wr_regs;
-	priv_sl2->hw_stats = priv->hw_stats;
-	priv_sl2->dma = priv->dma;
-	priv_sl2->txch = priv->txch;
-	priv_sl2->rxch = priv->rxch;
-	priv_sl2->ale = priv->ale;
 	priv_sl2->emac_port = 1;
-	priv->slaves[1].ndev = ndev;
-	priv_sl2->cpts = priv->cpts;
-	priv_sl2->version = priv->version;
-
-	for (i = 0; i < priv->num_irqs; i++) {
-		priv_sl2->irqs_table[i] = priv->irqs_table[i];
-		priv_sl2->num_irqs = priv->num_irqs;
-	}
+	cpsw->slaves[1].ndev = ndev;
 	ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
 
 	ndev->netdev_ops = &cpsw_netdev_ops;
 	ndev->ethtool_ops = &cpsw_ethtool_ops;
 
 	/* register the network device */
-	SET_NETDEV_DEV(ndev, &pdev->dev);
+	SET_NETDEV_DEV(ndev, cpsw->dev);
 	ret = register_netdev(ndev);
 	if (ret) {
-		dev_err(&pdev->dev, "cpsw: error registering net device\n");
+		dev_err(cpsw->dev, "cpsw: error registering net device\n");
 		free_netdev(ndev);
 		ret = -ENODEV;
 	}
@@ -2270,6 +2531,7 @@ MODULE_DEVICE_TABLE(of, cpsw_of_mtable);
 
 static int cpsw_probe(struct platform_device *pdev)
 {
+	struct clk			*clk;
 	struct cpsw_platform_data	*data;
 	struct net_device		*ndev;
 	struct cpsw_priv		*priv;
@@ -2280,10 +2542,14 @@ static int cpsw_probe(struct platform_device *pdev)
 	const struct of_device_id	*of_id;
 	struct gpio_descs		*mode;
 	u32 slave_offset, sliver_offset, slave_size;
+	struct cpsw_common		*cpsw;
 	int ret = 0, i;
 	int irq;
 
-	ndev = alloc_etherdev(sizeof(struct cpsw_priv));
+	cpsw = devm_kzalloc(&pdev->dev, sizeof(struct cpsw_common), GFP_KERNEL);
+	cpsw->dev = &pdev->dev;
+
+	ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
 	if (!ndev) {
 		dev_err(&pdev->dev, "error allocating net_device\n");
 		return -ENOMEM;
@@ -2291,13 +2557,13 @@ static int cpsw_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, ndev);
 	priv = netdev_priv(ndev);
-	priv->pdev = pdev;
+	priv->cpsw = cpsw;
 	priv->ndev = ndev;
 	priv->dev  = &ndev->dev;
 	priv->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
-	priv->rx_packet_max = max(rx_packet_max, 128);
-	priv->cpts = devm_kzalloc(&pdev->dev, sizeof(struct cpts), GFP_KERNEL);
-	if (!priv->cpts) {
+	cpsw->rx_packet_max = max(rx_packet_max, 128);
+	cpsw->cpts = devm_kzalloc(&pdev->dev, sizeof(struct cpts), GFP_KERNEL);
+	if (!cpsw->cpts) {
 		dev_err(&pdev->dev, "error allocating cpts\n");
 		ret = -ENOMEM;
 		goto clean_ndev_ret;
@@ -2318,12 +2584,14 @@ static int cpsw_probe(struct platform_device *pdev)
 	/* Select default pin state */
 	pinctrl_pm_select_default_state(&pdev->dev);
 
-	if (cpsw_probe_dt(&priv->data, pdev)) {
+	if (cpsw_probe_dt(&cpsw->data, pdev)) {
 		dev_err(&pdev->dev, "cpsw: platform data missing\n");
 		ret = -ENODEV;
 		goto clean_runtime_disable_ret;
 	}
-	data = &priv->data;
+	data = &cpsw->data;
+	cpsw->rx_ch_num = 1;
+	cpsw->tx_ch_num = 1;
 
 	if (is_valid_ether_addr(data->slave_data[0].mac_addr)) {
 		memcpy(priv->mac_addr, data->slave_data[0].mac_addr, ETH_ALEN);
@@ -2335,27 +2603,26 @@ static int cpsw_probe(struct platform_device *pdev)
 
 	memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
 
-	priv->slaves = devm_kzalloc(&pdev->dev,
+	cpsw->slaves = devm_kzalloc(&pdev->dev,
 				    sizeof(struct cpsw_slave) * data->slaves,
 				    GFP_KERNEL);
-	if (!priv->slaves) {
+	if (!cpsw->slaves) {
 		ret = -ENOMEM;
 		goto clean_runtime_disable_ret;
 	}
 	for (i = 0; i < data->slaves; i++)
-		priv->slaves[i].slave_num = i;
+		cpsw->slaves[i].slave_num = i;
 
-	priv->slaves[0].ndev = ndev;
+	cpsw->slaves[0].ndev = ndev;
 	priv->emac_port = 0;
 
-	priv->clk = devm_clk_get(&pdev->dev, "fck");
-	if (IS_ERR(priv->clk)) {
+	clk = devm_clk_get(&pdev->dev, "fck");
+	if (IS_ERR(clk)) {
 		dev_err(priv->dev, "fck is not found\n");
 		ret = -ENODEV;
 		goto clean_runtime_disable_ret;
 	}
-	priv->coal_intvl = 0;
-	priv->bus_freq_mhz = clk_get_rate(priv->clk) / 1000000;
+	cpsw->bus_freq_mhz = clk_get_rate(clk) / 1000000;
 
 	ss_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	ss_regs = devm_ioremap_resource(&pdev->dev, ss_res);
@@ -2363,7 +2630,7 @@ static int cpsw_probe(struct platform_device *pdev)
 		ret = PTR_ERR(ss_regs);
 		goto clean_runtime_disable_ret;
 	}
-	priv->regs = ss_regs;
+	cpsw->regs = ss_regs;
 
 	/* Need to enable clocks with runtime PM api to access module
 	 * registers
@@ -2373,24 +2640,24 @@ static int cpsw_probe(struct platform_device *pdev)
 		pm_runtime_put_noidle(&pdev->dev);
 		goto clean_runtime_disable_ret;
 	}
-	priv->version = readl(&priv->regs->id_ver);
+	cpsw->version = readl(&cpsw->regs->id_ver);
 	pm_runtime_put_sync(&pdev->dev);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-	priv->wr_regs = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(priv->wr_regs)) {
-		ret = PTR_ERR(priv->wr_regs);
+	cpsw->wr_regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(cpsw->wr_regs)) {
+		ret = PTR_ERR(cpsw->wr_regs);
 		goto clean_runtime_disable_ret;
 	}
 
 	memset(&dma_params, 0, sizeof(dma_params));
 	memset(&ale_params, 0, sizeof(ale_params));
 
-	switch (priv->version) {
+	switch (cpsw->version) {
 	case CPSW_VERSION_1:
-		priv->host_port_regs = ss_regs + CPSW1_HOST_PORT_OFFSET;
-		priv->cpts->reg      = ss_regs + CPSW1_CPTS_OFFSET;
-		priv->hw_stats	     = ss_regs + CPSW1_HW_STATS;
+		cpsw->host_port_regs = ss_regs + CPSW1_HOST_PORT_OFFSET;
+		cpsw->cpts->reg      = ss_regs + CPSW1_CPTS_OFFSET;
+		cpsw->hw_stats	     = ss_regs + CPSW1_HW_STATS;
 		dma_params.dmaregs   = ss_regs + CPSW1_CPDMA_OFFSET;
 		dma_params.txhdp     = ss_regs + CPSW1_STATERAM_OFFSET;
 		ale_params.ale_regs  = ss_regs + CPSW1_ALE_OFFSET;
@@ -2402,9 +2669,9 @@ static int cpsw_probe(struct platform_device *pdev)
 	case CPSW_VERSION_2:
 	case CPSW_VERSION_3:
 	case CPSW_VERSION_4:
-		priv->host_port_regs = ss_regs + CPSW2_HOST_PORT_OFFSET;
-		priv->cpts->reg      = ss_regs + CPSW2_CPTS_OFFSET;
-		priv->hw_stats	     = ss_regs + CPSW2_HW_STATS;
+		cpsw->host_port_regs = ss_regs + CPSW2_HOST_PORT_OFFSET;
+		cpsw->cpts->reg      = ss_regs + CPSW2_CPTS_OFFSET;
+		cpsw->hw_stats	     = ss_regs + CPSW2_HW_STATS;
 		dma_params.dmaregs   = ss_regs + CPSW2_CPDMA_OFFSET;
 		dma_params.txhdp     = ss_regs + CPSW2_STATERAM_OFFSET;
 		ale_params.ale_regs  = ss_regs + CPSW2_ALE_OFFSET;
@@ -2415,13 +2682,14 @@ static int cpsw_probe(struct platform_device *pdev)
 			(u32 __force) ss_res->start + CPSW2_BD_OFFSET;
 		break;
 	default:
-		dev_err(priv->dev, "unknown version 0x%08x\n", priv->version);
+		dev_err(priv->dev, "unknown version 0x%08x\n", cpsw->version);
 		ret = -ENODEV;
 		goto clean_runtime_disable_ret;
 	}
-	for (i = 0; i < priv->data.slaves; i++) {
-		struct cpsw_slave *slave = &priv->slaves[i];
-		cpsw_slave_init(slave, priv, slave_offset, sliver_offset);
+	for (i = 0; i < cpsw->data.slaves; i++) {
+		struct cpsw_slave *slave = &cpsw->slaves[i];
+
+		cpsw_slave_init(slave, cpsw, slave_offset, sliver_offset);
 		slave_offset  += slave_size;
 		sliver_offset += SLIVER_SIZE;
 	}
@@ -2441,19 +2709,16 @@ static int cpsw_probe(struct platform_device *pdev)
 	dma_params.has_ext_regs		= true;
 	dma_params.desc_hw_addr         = dma_params.desc_mem_phys;
 
-	priv->dma = cpdma_ctlr_create(&dma_params);
-	if (!priv->dma) {
+	cpsw->dma = cpdma_ctlr_create(&dma_params);
+	if (!cpsw->dma) {
 		dev_err(priv->dev, "error initializing dma\n");
 		ret = -ENOMEM;
 		goto clean_runtime_disable_ret;
 	}
 
-	priv->txch = cpdma_chan_create(priv->dma, tx_chan_num(0),
-				       cpsw_tx_handler);
-	priv->rxch = cpdma_chan_create(priv->dma, rx_chan_num(0),
-				       cpsw_rx_handler);
-
-	if (WARN_ON(!priv->txch || !priv->rxch)) {
+	cpsw->txch[0] = cpdma_chan_create(cpsw->dma, 0, cpsw_tx_handler, 0);
+	cpsw->rxch[0] = cpdma_chan_create(cpsw->dma, 0, cpsw_rx_handler, 1);
+	if (WARN_ON(!cpsw->rxch[0] || !cpsw->txch[0])) {
 		dev_err(priv->dev, "error initializing dma channels\n");
 		ret = -ENOMEM;
 		goto clean_dma_ret;
@@ -2464,8 +2729,8 @@ static int cpsw_probe(struct platform_device *pdev)
 	ale_params.ale_entries		= data->ale_entries;
 	ale_params.ale_ports		= data->slaves;
 
-	priv->ale = cpsw_ale_create(&ale_params);
-	if (!priv->ale) {
+	cpsw->ale = cpsw_ale_create(&ale_params);
+	if (!cpsw->ale) {
 		dev_err(priv->dev, "error initializing ale engine\n");
 		ret = -ENODEV;
 		goto clean_dma_ret;
@@ -2482,7 +2747,7 @@ static int cpsw_probe(struct platform_device *pdev)
 	if (of_id) {
 		pdev->id_entry = of_id->data;
 		if (pdev->id_entry->driver_data)
-			priv->quirk_irq = true;
+			cpsw->quirk_irq = true;
 	}
 
 	/* Grab RX and TX IRQs. Note that we also have RX_THRESHOLD and
@@ -2500,9 +2765,9 @@ static int cpsw_probe(struct platform_device *pdev)
 		goto clean_ale_ret;
 	}
 
-	priv->irqs_table[0] = irq;
+	cpsw->irqs_table[0] = irq;
 	ret = devm_request_irq(&pdev->dev, irq, cpsw_rx_interrupt,
-			       0, dev_name(&pdev->dev), priv);
+			       0, dev_name(&pdev->dev), cpsw);
 	if (ret < 0) {
 		dev_err(priv->dev, "error attaching irq (%d)\n", ret);
 		goto clean_ale_ret;
@@ -2515,21 +2780,20 @@ static int cpsw_probe(struct platform_device *pdev)
 		goto clean_ale_ret;
 	}
 
-	priv->irqs_table[1] = irq;
+	cpsw->irqs_table[1] = irq;
 	ret = devm_request_irq(&pdev->dev, irq, cpsw_tx_interrupt,
-			       0, dev_name(&pdev->dev), priv);
+			       0, dev_name(&pdev->dev), cpsw);
 	if (ret < 0) {
 		dev_err(priv->dev, "error attaching irq (%d)\n", ret);
 		goto clean_ale_ret;
 	}
-	priv->num_irqs = 2;
 
 	ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
 
 	ndev->netdev_ops = &cpsw_netdev_ops;
 	ndev->ethtool_ops = &cpsw_ethtool_ops;
-	netif_napi_add(ndev, &priv->napi_rx, cpsw_rx_poll, CPSW_POLL_WEIGHT);
-	netif_tx_napi_add(ndev, &priv->napi_tx, cpsw_tx_poll, CPSW_POLL_WEIGHT);
+	netif_napi_add(ndev, &cpsw->napi_rx, cpsw_rx_poll, CPSW_POLL_WEIGHT);
+	netif_tx_napi_add(ndev, &cpsw->napi_tx, cpsw_tx_poll, CPSW_POLL_WEIGHT);
 
 	/* register the network device */
 	SET_NETDEV_DEV(ndev, &pdev->dev);
@@ -2543,8 +2807,8 @@ static int cpsw_probe(struct platform_device *pdev)
 	cpsw_notice(priv, probe, "initialized device (regs %pa, irq %d)\n",
 		    &ss_res->start, ndev->irq);
 
-	if (priv->data.dual_emac) {
-		ret = cpsw_probe_dual_emac(pdev, priv);
+	if (cpsw->data.dual_emac) {
+		ret = cpsw_probe_dual_emac(priv);
 		if (ret) {
 			cpsw_err(priv, probe, "error probe slave 2 emac interface\n");
 			goto clean_ale_ret;
@@ -2554,9 +2818,9 @@ static int cpsw_probe(struct platform_device *pdev)
 	return 0;
 
 clean_ale_ret:
-	cpsw_ale_destroy(priv->ale);
+	cpsw_ale_destroy(cpsw->ale);
 clean_dma_ret:
-	cpdma_ctlr_destroy(priv->dma);
+	cpdma_ctlr_destroy(cpsw->dma);
 clean_runtime_disable_ret:
 	pm_runtime_disable(&pdev->dev);
 clean_ndev_ret:
@@ -2567,7 +2831,7 @@ clean_ndev_ret:
 static int cpsw_remove(struct platform_device *pdev)
 {
 	struct net_device *ndev = platform_get_drvdata(pdev);
-	struct cpsw_priv *priv = netdev_priv(ndev);
+	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 	int ret;
 
 	ret = pm_runtime_get_sync(&pdev->dev);
@@ -2576,17 +2840,17 @@ static int cpsw_remove(struct platform_device *pdev)
 		return ret;
 	}
 
-	if (priv->data.dual_emac)
-		unregister_netdev(cpsw_get_slave_ndev(priv, 1));
+	if (cpsw->data.dual_emac)
+		unregister_netdev(cpsw->slaves[1].ndev);
 	unregister_netdev(ndev);
 
-	cpsw_ale_destroy(priv->ale);
-	cpdma_ctlr_destroy(priv->dma);
+	cpsw_ale_destroy(cpsw->ale);
+	cpdma_ctlr_destroy(cpsw->dma);
 	of_platform_depopulate(&pdev->dev);
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-	if (priv->data.dual_emac)
-		free_netdev(cpsw_get_slave_ndev(priv, 1));
+	if (cpsw->data.dual_emac)
+		free_netdev(cpsw->slaves[1].ndev);
 	free_netdev(ndev);
 	return 0;
 }
@@ -2596,14 +2860,14 @@ static int cpsw_suspend(struct device *dev)
 {
 	struct platform_device	*pdev = to_platform_device(dev);
 	struct net_device	*ndev = platform_get_drvdata(pdev);
-	struct cpsw_priv	*priv = netdev_priv(ndev);
+	struct cpsw_common	*cpsw = ndev_to_cpsw(ndev);
 
-	if (priv->data.dual_emac) {
+	if (cpsw->data.dual_emac) {
 		int i;
 
-		for (i = 0; i < priv->data.slaves; i++) {
-			if (netif_running(priv->slaves[i].ndev))
-				cpsw_ndo_stop(priv->slaves[i].ndev);
+		for (i = 0; i < cpsw->data.slaves; i++) {
+			if (netif_running(cpsw->slaves[i].ndev))
+				cpsw_ndo_stop(cpsw->slaves[i].ndev);
 		}
 	} else {
 		if (netif_running(ndev))
@@ -2611,7 +2875,7 @@ static int cpsw_suspend(struct device *dev)
 	}
 
 	/* Select sleep pin state */
-	pinctrl_pm_select_sleep_state(&pdev->dev);
+	pinctrl_pm_select_sleep_state(dev);
 
 	return 0;
 }
@@ -2620,17 +2884,17 @@ static int cpsw_resume(struct device *dev)
 {
 	struct platform_device	*pdev = to_platform_device(dev);
 	struct net_device	*ndev = platform_get_drvdata(pdev);
-	struct cpsw_priv	*priv = netdev_priv(ndev);
+	struct cpsw_common	*cpsw = netdev_priv(ndev);
 
 	/* Select default pin state */
-	pinctrl_pm_select_default_state(&pdev->dev);
+	pinctrl_pm_select_default_state(dev);
 
-	if (priv->data.dual_emac) {
+	if (cpsw->data.dual_emac) {
 		int i;
 
-		for (i = 0; i < priv->data.slaves; i++) {
-			if (netif_running(priv->slaves[i].ndev))
-				cpsw_ndo_open(priv->slaves[i].ndev);
+		for (i = 0; i < cpsw->data.slaves; i++) {
+			if (netif_running(cpsw->slaves[i].ndev))
+				cpsw_ndo_open(cpsw->slaves[i].ndev);
 		}
 	} else {
 		if (netif_running(ndev))
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.c b/drivers/net/ethernet/ti/davinci_cpdma.c
index 19e5f32a8a64..c3f35f11a8fd 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.c
+++ b/drivers/net/ethernet/ti/davinci_cpdma.c
@@ -86,7 +86,7 @@ struct cpdma_desc_pool {
 	void __iomem		*iomap;		/* ioremap map */
 	void			*cpumap;	/* dma_alloc map */
 	int			desc_size, mem_size;
-	int			num_desc, used_desc;
+	int			num_desc;
 	struct device		*dev;
 	struct gen_pool		*gen_pool;
 };
@@ -104,6 +104,7 @@ struct cpdma_ctlr {
 	struct cpdma_desc_pool	*pool;
 	spinlock_t		lock;
 	struct cpdma_chan	*channels[2 * CPDMA_MAX_CHANNELS];
+	int chan_num;
 };
 
 struct cpdma_chan {
@@ -123,6 +124,13 @@ struct cpdma_chan {
 	int	int_set, int_clear, td;
 };
 
+#define tx_chan_num(chan)	(chan)
+#define rx_chan_num(chan)	((chan) + CPDMA_MAX_CHANNELS)
+#define is_rx_chan(chan)	((chan)->chan_num >= CPDMA_MAX_CHANNELS)
+#define is_tx_chan(chan)	(!is_rx_chan(chan))
+#define __chan_linear(chan_num)	((chan_num) & (CPDMA_MAX_CHANNELS - 1))
+#define chan_linear(chan)	__chan_linear((chan)->chan_num)
+
 /* The following make access to common cpdma_ctlr params more readable */
 #define dmaregs		params.dmaregs
 #define num_chan	params.num_chan
@@ -148,7 +156,10 @@ static void cpdma_desc_pool_destroy(struct cpdma_desc_pool *pool)
 	if (!pool)
 		return;
 
-	WARN_ON(pool->used_desc);
+	WARN(gen_pool_size(pool->gen_pool) != gen_pool_avail(pool->gen_pool),
+	     "cpdma_desc_pool size %d != avail %d",
+	     gen_pool_size(pool->gen_pool),
+	     gen_pool_avail(pool->gen_pool));
 	if (pool->cpumap)
 		dma_free_coherent(pool->dev, pool->mem_size, pool->cpumap,
 				  pool->phys);
@@ -232,21 +243,14 @@ desc_from_phys(struct cpdma_desc_pool *pool, dma_addr_t dma)
 static struct cpdma_desc __iomem *
 cpdma_desc_alloc(struct cpdma_desc_pool *pool)
 {
-	struct cpdma_desc __iomem *desc = NULL;
-
-	desc = (struct cpdma_desc __iomem *)gen_pool_alloc(pool->gen_pool,
-							   pool->desc_size);
-	if (desc)
-		pool->used_desc++;
-
-	return desc;
+	return (struct cpdma_desc __iomem *)
+		gen_pool_alloc(pool->gen_pool, pool->desc_size);
 }
 
 static void cpdma_desc_free(struct cpdma_desc_pool *pool,
 			    struct cpdma_desc __iomem *desc, int num_desc)
 {
 	gen_pool_free(pool->gen_pool, (unsigned long)desc, pool->desc_size);
-	pool->used_desc--;
 }
 
 struct cpdma_ctlr *cpdma_ctlr_create(struct cpdma_params *params)
@@ -260,6 +264,7 @@ struct cpdma_ctlr *cpdma_ctlr_create(struct cpdma_params *params)
 	ctlr->state = CPDMA_STATE_IDLE;
 	ctlr->params = *params;
 	ctlr->dev = params->dev;
+	ctlr->chan_num = 0;
 	spin_lock_init(&ctlr->lock);
 
 	ctlr->pool = cpdma_desc_pool_create(ctlr->dev,
@@ -336,12 +341,14 @@ int cpdma_ctlr_stop(struct cpdma_ctlr *ctlr)
 	}
 
 	ctlr->state = CPDMA_STATE_TEARDOWN;
+	spin_unlock_irqrestore(&ctlr->lock, flags);
 
 	for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) {
 		if (ctlr->channels[i])
 			cpdma_chan_stop(ctlr->channels[i]);
 	}
 
+	spin_lock_irqsave(&ctlr->lock, flags);
 	dma_reg_write(ctlr, CPDMA_RXINTMASKCLEAR, 0xffffffff);
 	dma_reg_write(ctlr, CPDMA_TXINTMASKCLEAR, 0xffffffff);
 
@@ -403,13 +410,52 @@ void cpdma_ctlr_eoi(struct cpdma_ctlr *ctlr, u32 value)
 }
 EXPORT_SYMBOL_GPL(cpdma_ctlr_eoi);
 
+u32 cpdma_ctrl_rxchs_state(struct cpdma_ctlr *ctlr)
+{
+	return dma_reg_read(ctlr, CPDMA_RXINTSTATMASKED);
+}
+EXPORT_SYMBOL_GPL(cpdma_ctrl_rxchs_state);
+
+u32 cpdma_ctrl_txchs_state(struct cpdma_ctlr *ctlr)
+{
+	return dma_reg_read(ctlr, CPDMA_TXINTSTATMASKED);
+}
+EXPORT_SYMBOL_GPL(cpdma_ctrl_txchs_state);
+
+/**
+ * cpdma_chan_split_pool - Splits ctrl pool between all channels.
+ * Has to be called under ctlr lock
+ */
+static void cpdma_chan_split_pool(struct cpdma_ctlr *ctlr)
+{
+	struct cpdma_desc_pool *pool = ctlr->pool;
+	struct cpdma_chan *chan;
+	int ch_desc_num;
+	int i;
+
+	if (!ctlr->chan_num)
+		return;
+
+	/* calculate average size of pool slice */
+	ch_desc_num = pool->num_desc / ctlr->chan_num;
+
+	/* split ctlr pool */
+	for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) {
+		chan = ctlr->channels[i];
+		if (chan)
+			chan->desc_num = ch_desc_num;
+	}
+}
+
 struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num,
-				     cpdma_handler_fn handler)
+				     cpdma_handler_fn handler, int rx_type)
 {
+	int offset = chan_num * 4;
 	struct cpdma_chan *chan;
-	int offset = (chan_num % CPDMA_MAX_CHANNELS) * 4;
 	unsigned long flags;
 
+	chan_num = rx_type ? rx_chan_num(chan_num) : tx_chan_num(chan_num);
+
 	if (__chan_linear(chan_num) >= ctlr->num_chan)
 		return NULL;
 
@@ -451,14 +497,25 @@ struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num,
 	spin_lock_init(&chan->lock);
 
 	ctlr->channels[chan_num] = chan;
+	ctlr->chan_num++;
+
+	cpdma_chan_split_pool(ctlr);
+
 	spin_unlock_irqrestore(&ctlr->lock, flags);
 	return chan;
 }
 EXPORT_SYMBOL_GPL(cpdma_chan_create);
 
-int cpdma_chan_get_rx_buf_num(struct cpdma_ctlr *ctlr)
+int cpdma_chan_get_rx_buf_num(struct cpdma_chan *chan)
 {
-	return ctlr->pool->num_desc / 2;
+	unsigned long flags;
+	int desc_num;
+
+	spin_lock_irqsave(&chan->lock, flags);
+	desc_num = chan->desc_num;
+	spin_unlock_irqrestore(&chan->lock, flags);
+
+	return desc_num;
 }
 EXPORT_SYMBOL_GPL(cpdma_chan_get_rx_buf_num);
 
@@ -475,6 +532,10 @@ int cpdma_chan_destroy(struct cpdma_chan *chan)
 	if (chan->state != CPDMA_STATE_IDLE)
 		cpdma_chan_stop(chan);
 	ctlr->channels[chan->chan_num] = NULL;
+	ctlr->chan_num--;
+
+	cpdma_chan_split_pool(ctlr);
+
 	spin_unlock_irqrestore(&ctlr->lock, flags);
 	return 0;
 }
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.h b/drivers/net/ethernet/ti/davinci_cpdma.h
index 4b46cd6e9a3f..a07b22b12bc1 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.h
+++ b/drivers/net/ethernet/ti/davinci_cpdma.h
@@ -17,13 +17,6 @@
 
 #define CPDMA_MAX_CHANNELS	BITS_PER_LONG
 
-#define tx_chan_num(chan)	(chan)
-#define rx_chan_num(chan)	((chan) + CPDMA_MAX_CHANNELS)
-#define is_rx_chan(chan)	((chan)->chan_num >= CPDMA_MAX_CHANNELS)
-#define is_tx_chan(chan)	(!is_rx_chan(chan))
-#define __chan_linear(chan_num)	((chan_num) & (CPDMA_MAX_CHANNELS - 1))
-#define chan_linear(chan)	__chan_linear((chan)->chan_num)
-
 #define CPDMA_RX_SOURCE_PORT(__status__)	((__status__ >> 16) & 0x7)
 
 #define CPDMA_EOI_RX_THRESH	0x0
@@ -79,8 +72,8 @@ int cpdma_ctlr_start(struct cpdma_ctlr *ctlr);
 int cpdma_ctlr_stop(struct cpdma_ctlr *ctlr);
 
 struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num,
-				     cpdma_handler_fn handler);
-int cpdma_chan_get_rx_buf_num(struct cpdma_ctlr *ctlr);
+				     cpdma_handler_fn handler, int rx_type);
+int cpdma_chan_get_rx_buf_num(struct cpdma_chan *chan);
 int cpdma_chan_destroy(struct cpdma_chan *chan);
 int cpdma_chan_start(struct cpdma_chan *chan);
 int cpdma_chan_stop(struct cpdma_chan *chan);
@@ -94,6 +87,8 @@ int cpdma_chan_process(struct cpdma_chan *chan, int quota);
 int cpdma_ctlr_int_ctrl(struct cpdma_ctlr *ctlr, bool enable);
 void cpdma_ctlr_eoi(struct cpdma_ctlr *ctlr, u32 value);
 int cpdma_chan_int_ctrl(struct cpdma_chan *chan, bool enable);
+u32 cpdma_ctrl_rxchs_state(struct cpdma_ctlr *ctlr);
+u32 cpdma_ctrl_txchs_state(struct cpdma_ctlr *ctlr);
 bool cpdma_check_free_tx_desc(struct cpdma_chan *chan);
 
 enum cpdma_control {
diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c
index 727a79f3c7dd..2fd94a5bc1f3 100644
--- a/drivers/net/ethernet/ti/davinci_emac.c
+++ b/drivers/net/ethernet/ti/davinci_emac.c
@@ -597,14 +597,14 @@ static u32 hash_get(u8 *addr)
 }
 
 /**
- * hash_add - Hash function to add mac addr from hash table
+ * emac_hash_add - Hash function to add mac addr from hash table
  * @priv: The DaVinci EMAC private adapter structure
  * @mac_addr: mac address to delete from hash table
  *
  * Adds mac address to the internal hash table
  *
  */
-static int hash_add(struct emac_priv *priv, u8 *mac_addr)
+static int emac_hash_add(struct emac_priv *priv, u8 *mac_addr)
 {
 	struct device *emac_dev = &priv->ndev->dev;
 	u32 rc = 0;
@@ -613,7 +613,7 @@ static int hash_add(struct emac_priv *priv, u8 *mac_addr)
 
 	if (hash_value >= EMAC_NUM_MULTICAST_BITS) {
 		if (netif_msg_drv(priv)) {
-			dev_err(emac_dev, "DaVinci EMAC: hash_add(): Invalid "\
+			dev_err(emac_dev, "DaVinci EMAC: emac_hash_add(): Invalid "\
 				"Hash %08x, should not be greater than %08x",
 				hash_value, (EMAC_NUM_MULTICAST_BITS - 1));
 		}
@@ -639,14 +639,14 @@ static int hash_add(struct emac_priv *priv, u8 *mac_addr)
 }
 
 /**
- * hash_del - Hash function to delete mac addr from hash table
+ * emac_hash_del - Hash function to delete mac addr from hash table
  * @priv: The DaVinci EMAC private adapter structure
  * @mac_addr: mac address to delete from hash table
  *
  * Removes mac address from the internal hash table
  *
  */
-static int hash_del(struct emac_priv *priv, u8 *mac_addr)
+static int emac_hash_del(struct emac_priv *priv, u8 *mac_addr)
 {
 	u32 hash_value;
 	u32 hash_bit;
@@ -696,10 +696,10 @@ static void emac_add_mcast(struct emac_priv *priv, u32 action, u8 *mac_addr)
 
 	switch (action) {
 	case EMAC_MULTICAST_ADD:
-		update = hash_add(priv, mac_addr);
+		update = emac_hash_add(priv, mac_addr);
 		break;
 	case EMAC_MULTICAST_DEL:
-		update = hash_del(priv, mac_addr);
+		update = emac_hash_del(priv, mac_addr);
 		break;
 	case EMAC_ALL_MULTI_SET:
 		update = 1;
@@ -1870,10 +1870,10 @@ static int davinci_emac_probe(struct platform_device *pdev)
 		goto no_pdata;
 	}
 
-	priv->txchan = cpdma_chan_create(priv->dma, tx_chan_num(EMAC_DEF_TX_CH),
-				       emac_tx_handler);
-	priv->rxchan = cpdma_chan_create(priv->dma, rx_chan_num(EMAC_DEF_RX_CH),
-				       emac_rx_handler);
+	priv->txchan = cpdma_chan_create(priv->dma, EMAC_DEF_TX_CH,
+					 emac_tx_handler, 0);
+	priv->rxchan = cpdma_chan_create(priv->dma, EMAC_DEF_RX_CH,
+					 emac_rx_handler, 1);
 	if (WARN_ON(!priv->txchan || !priv->rxchan)) {
 		rc = -ENOMEM;
 		goto no_cpdma_chan;
diff --git a/drivers/net/ethernet/toshiba/ps3_gelic_net.c b/drivers/net/ethernet/toshiba/ps3_gelic_net.c
index 79f0ec4e51ac..272f2b1cb7ad 100644
--- a/drivers/net/ethernet/toshiba/ps3_gelic_net.c
+++ b/drivers/net/ethernet/toshiba/ps3_gelic_net.c
@@ -1769,7 +1769,7 @@ static int ps3_gelic_driver_probe(struct ps3_system_bus_device *dev)
 	gelic_ether_setup_netdev_ops(netdev, &card->napi);
 	result = gelic_net_setup_netdev(netdev, card);
 	if (result) {
-		dev_dbg(&dev->core, "%s: setup_netdev failed %d",
+		dev_dbg(&dev->core, "%s: setup_netdev failed %d\n",
 			__func__, result);
 		goto fail_setup_netdev;
 	}
@@ -1791,7 +1791,7 @@ fail_alloc_rx:
 	gelic_card_free_chain(card, card->tx_chain.head);
 fail_alloc_tx:
 	free_irq(card->irq, card);
-	netdev->irq = NO_IRQ;
+	netdev->irq = 0;
 fail_request_irq:
 	ps3_sb_event_receive_port_destroy(dev, card->irq);
 fail_alloc_irq:
@@ -1843,7 +1843,7 @@ static int ps3_gelic_driver_remove(struct ps3_system_bus_device *dev)
 	netdev0 = card->netdev[GELIC_PORT_ETHERNET_0];
 	/* disconnect event port */
 	free_irq(card->irq, card);
-	netdev0->irq = NO_IRQ;
+	netdev0->irq = 0;
 	ps3_sb_event_receive_port_destroy(card->dev, card->irq);
 
 	wait_event(card->waitq,
diff --git a/drivers/net/ethernet/tundra/tsi108_eth.c b/drivers/net/ethernet/tundra/tsi108_eth.c
index 01a77145a0fa..8fd131207ee1 100644
--- a/drivers/net/ethernet/tundra/tsi108_eth.c
+++ b/drivers/net/ethernet/tundra/tsi108_eth.c
@@ -166,6 +166,7 @@ static struct platform_driver tsi_eth_driver = {
 
 static void tsi108_timed_checker(unsigned long dev_ptr);
 
+#ifdef DEBUG
 static void dump_eth_one(struct net_device *dev)
 {
 	struct tsi108_prv_data *data = netdev_priv(dev);
@@ -190,6 +191,7 @@ static void dump_eth_one(struct net_device *dev)
 	       TSI_READ(TSI108_EC_RXESTAT),
 	       TSI_READ(TSI108_EC_RXERR), data->rxpending);
 }
+#endif
 
 /* Synchronization is needed between the thread and up/down events.
  * Note that the PHY is accessed through the same registers for both
diff --git a/drivers/net/ethernet/via/via-velocity.c b/drivers/net/ethernet/via/via-velocity.c
index f38696ceee74..908e72e18ef7 100644
--- a/drivers/net/ethernet/via/via-velocity.c
+++ b/drivers/net/ethernet/via/via-velocity.c
@@ -1724,24 +1724,21 @@ static void velocity_free_tx_buf(struct velocity_info *vptr,
 		struct velocity_td_info *tdinfo, struct tx_desc *td)
 {
 	struct sk_buff *skb = tdinfo->skb;
+	int i;
 
 	/*
 	 *	Don't unmap the pre-allocated tx_bufs
 	 */
-	if (tdinfo->skb_dma) {
-		int i;
+	for (i = 0; i < tdinfo->nskb_dma; i++) {
+		size_t pktlen = max_t(size_t, skb->len, ETH_ZLEN);
 
-		for (i = 0; i < tdinfo->nskb_dma; i++) {
-			size_t pktlen = max_t(size_t, skb->len, ETH_ZLEN);
+		/* For scatter-gather */
+		if (skb_shinfo(skb)->nr_frags > 0)
+			pktlen = max_t(size_t, pktlen,
+				       td->td_buf[i].size & ~TD_QUEUE);
 
-			/* For scatter-gather */
-			if (skb_shinfo(skb)->nr_frags > 0)
-				pktlen = max_t(size_t, pktlen,
-						td->td_buf[i].size & ~TD_QUEUE);
-
-			dma_unmap_single(vptr->dev, tdinfo->skb_dma[i],
-					le16_to_cpu(pktlen), DMA_TO_DEVICE);
-		}
+		dma_unmap_single(vptr->dev, tdinfo->skb_dma[i],
+				 le16_to_cpu(pktlen), DMA_TO_DEVICE);
 	}
 	dev_kfree_skb_irq(skb);
 	tdinfo->skb = NULL;
diff --git a/drivers/net/ethernet/wiznet/w5100.c b/drivers/net/ethernet/wiznet/w5100.c
index 37ab46cdbec4..d2349a1bc6ba 100644
--- a/drivers/net/ethernet/wiznet/w5100.c
+++ b/drivers/net/ethernet/wiznet/w5100.c
@@ -9,7 +9,6 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/kconfig.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
 #include <linux/platform_device.h>
diff --git a/drivers/net/ethernet/wiznet/w5300.c b/drivers/net/ethernet/wiznet/w5300.c
index 0b37ce9f28f1..ca31a57dbc86 100644
--- a/drivers/net/ethernet/wiznet/w5300.c
+++ b/drivers/net/ethernet/wiznet/w5300.c
@@ -10,7 +10,6 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/kconfig.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
 #include <linux/platform_device.h>
diff --git a/drivers/net/ethernet/xilinx/Kconfig b/drivers/net/ethernet/xilinx/Kconfig
index 4f5c024c6192..6d68c8a8f4f2 100644
--- a/drivers/net/ethernet/xilinx/Kconfig
+++ b/drivers/net/ethernet/xilinx/Kconfig
@@ -5,7 +5,7 @@
 config NET_VENDOR_XILINX
 	bool "Xilinx devices"
 	default y
-	depends on PPC || PPC32 || MICROBLAZE || ARCH_ZYNQ
+	depends on PPC || PPC32 || MICROBLAZE || ARCH_ZYNQ || MIPS
 	---help---
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
@@ -18,7 +18,7 @@ if NET_VENDOR_XILINX
 
 config XILINX_EMACLITE
 	tristate "Xilinx 10/100 Ethernet Lite support"
-	depends on (PPC32 || MICROBLAZE || ARCH_ZYNQ)
+	depends on PPC32 || MICROBLAZE || ARCH_ZYNQ || MIPS
 	select PHYLIB
 	---help---
 	  This driver supports the 10/100 Ethernet Lite from Xilinx.
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
index 36ee7ab300ae..35f9f9742a48 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
@@ -818,7 +818,7 @@ static irqreturn_t axienet_tx_irq(int irq, void *_ndev)
 		goto out;
 	}
 	if (!(status & XAXIDMA_IRQ_ALL_MASK))
-		dev_err(&ndev->dev, "No interrupts asserted in Tx path");
+		dev_err(&ndev->dev, "No interrupts asserted in Tx path\n");
 	if (status & XAXIDMA_IRQ_ERROR_MASK) {
 		dev_err(&ndev->dev, "DMA Tx error 0x%x\n", status);
 		dev_err(&ndev->dev, "Current BD is at: 0x%x\n",
@@ -867,7 +867,7 @@ static irqreturn_t axienet_rx_irq(int irq, void *_ndev)
 		goto out;
 	}
 	if (!(status & XAXIDMA_IRQ_ALL_MASK))
-		dev_err(&ndev->dev, "No interrupts asserted in Rx path");
+		dev_err(&ndev->dev, "No interrupts asserted in Rx path\n");
 	if (status & XAXIDMA_IRQ_ERROR_MASK) {
 		dev_err(&ndev->dev, "DMA Rx error 0x%x\n", status);
 		dev_err(&ndev->dev, "Current BD is at: 0x%x\n",
@@ -1297,7 +1297,7 @@ static int axienet_ethtools_set_coalesce(struct net_device *ndev,
 	return 0;
 }
 
-static struct ethtool_ops axienet_ethtool_ops = {
+static const struct ethtool_ops axienet_ethtool_ops = {
 	.get_drvinfo    = axienet_ethtools_get_drvinfo,
 	.get_regs_len   = axienet_ethtools_get_regs_len,
 	.get_regs       = axienet_ethtools_get_regs,
diff --git a/drivers/net/ethernet/xilinx/xilinx_emaclite.c b/drivers/net/ethernet/xilinx/xilinx_emaclite.c
index 3cee84a24815..93dc10b10c09 100644
--- a/drivers/net/ethernet/xilinx/xilinx_emaclite.c
+++ b/drivers/net/ethernet/xilinx/xilinx_emaclite.c
@@ -1131,11 +1131,13 @@ static int xemaclite_of_probe(struct platform_device *ofdev)
 	lp->rx_ping_pong = get_bool(ofdev, "xlnx,rx-ping-pong");
 	mac_address = of_get_mac_address(ofdev->dev.of_node);
 
-	if (mac_address)
+	if (mac_address) {
 		/* Set the MAC address. */
 		memcpy(ndev->dev_addr, mac_address, ETH_ALEN);
-	else
-		dev_warn(dev, "No MAC address found\n");
+	} else {
+		dev_warn(dev, "No MAC address found, using random\n");
+		eth_hw_addr_random(ndev);
+	}
 
 	/* Clear the Tx CSR's in case this is a restart */
 	__raw_writel(0, lp->base_addr + XEL_TSR_OFFSET);
diff --git a/drivers/net/fjes/fjes_main.c b/drivers/net/fjes/fjes_main.c
index 9006877c53f2..e46b1ebbbff4 100644
--- a/drivers/net/fjes/fjes_main.c
+++ b/drivers/net/fjes/fjes_main.c
@@ -97,7 +97,6 @@ static struct acpi_driver fjes_acpi_driver = {
 static struct platform_driver fjes_driver = {
 	.driver = {
 		.name = DRV_NAME,
-		.owner = THIS_MODULE,
 	},
 	.probe = fjes_probe,
 	.remove = fjes_remove,
diff --git a/drivers/net/hamradio/6pack.c b/drivers/net/hamradio/6pack.c
index 5a1e98547031..470b3dcd54e5 100644
--- a/drivers/net/hamradio/6pack.c
+++ b/drivers/net/hamradio/6pack.c
@@ -127,7 +127,7 @@ struct sixpack {
 
 #define AX25_6PACK_HEADER_LEN 0
 
-static void sixpack_decode(struct sixpack *, unsigned char[], int);
+static void sixpack_decode(struct sixpack *, const unsigned char[], int);
 static int encode_sixpack(unsigned char *, unsigned char *, int, unsigned char);
 
 /*
@@ -428,7 +428,7 @@ out:
 
 /*
  * Handle the 'receiver data ready' interrupt.
- * This function is called by the 'tty_io' module in the kernel when
+ * This function is called by the tty module in the kernel when
  * a block of 6pack data has been received, which can now be decapsulated
  * and sent on to some IP layer for further processing.
  */
@@ -436,7 +436,6 @@ static void sixpack_receive_buf(struct tty_struct *tty,
 	const unsigned char *cp, char *fp, int count)
 {
 	struct sixpack *sp;
-	unsigned char buf[512];
 	int count1;
 
 	if (!count)
@@ -446,10 +445,7 @@ static void sixpack_receive_buf(struct tty_struct *tty,
 	if (!sp)
 		return;
 
-	memcpy(buf, cp, count < sizeof(buf) ? count : sizeof(buf));
-
 	/* Read the characters out of the buffer */
-
 	count1 = count;
 	while (count) {
 		count--;
@@ -459,7 +455,7 @@ static void sixpack_receive_buf(struct tty_struct *tty,
 			continue;
 		}
 	}
-	sixpack_decode(sp, buf, count1);
+	sixpack_decode(sp, cp, count1);
 
 	sp_put(sp);
 	tty_unthrottle(tty);
@@ -992,7 +988,7 @@ static void decode_std_command(struct sixpack *sp, unsigned char cmd)
 /* decode a 6pack packet */
 
 static void
-sixpack_decode(struct sixpack *sp, unsigned char *pre_rbuff, int count)
+sixpack_decode(struct sixpack *sp, const unsigned char *pre_rbuff, int count)
 {
 	unsigned char inbyte;
 	int count1;
diff --git a/drivers/net/hamradio/bpqether.c b/drivers/net/hamradio/bpqether.c
index d95a50ae996d..622ab3ab9e93 100644
--- a/drivers/net/hamradio/bpqether.c
+++ b/drivers/net/hamradio/bpqether.c
@@ -484,7 +484,7 @@ static void bpq_setup(struct net_device *dev)
 	dev->flags      = 0;
 	dev->features	= NETIF_F_LLTX;	/* Allow recursion */
 
-#if defined(CONFIG_AX25) || defined(CONFIG_AX25_MODULE)
+#if IS_ENABLED(CONFIG_AX25)
 	dev->header_ops      = &ax25_header_ops;
 #endif
 
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index 467fb8b4d083..f4fbcb5aa24a 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -84,8 +84,6 @@ struct ndis_recv_scale_cap { /* NDIS_RECEIVE_SCALE_CAPABILITIES */
 #define NDIS_RSS_HASH_SECRET_KEY_MAX_SIZE_REVISION_2   40
 
 #define ITAB_NUM 128
-#define HASH_KEYLEN NDIS_RSS_HASH_SECRET_KEY_MAX_SIZE_REVISION_2
-extern u8 netvsc_hash_key[];
 
 struct ndis_recv_scale_param { /* NDIS_RECEIVE_SCALE_PARAMETERS */
 	struct ndis_obj_header hdr;
@@ -175,7 +173,7 @@ struct rndis_device {
 struct rndis_message;
 struct netvsc_device;
 int netvsc_device_add(struct hv_device *device, void *additional_info);
-int netvsc_device_remove(struct hv_device *device);
+void netvsc_device_remove(struct hv_device *device);
 int netvsc_send(struct hv_device *device,
 		struct hv_netvsc_packet *packet,
 		struct rndis_message *rndis_msg,
@@ -435,7 +433,7 @@ struct nvsp_1_message_revoke_send_buffer {
  */
 struct nvsp_1_message_send_rndis_packet {
 	/*
-	 * This field is specified by RNIDS. They assume there's two different
+	 * This field is specified by RNDIS. They assume there's two different
 	 * channels of communication. However, the Network VSP only has one.
 	 * Therefore, the channel travels with the RNDIS packet.
 	 */
@@ -490,6 +488,7 @@ struct nvsp_2_vsc_capability {
 			u64 sriov:1;
 			u64 ieee8021q:1;
 			u64 correlation_id:1;
+			u64 teaming:1;
 		};
 	};
 } __packed;
@@ -579,7 +578,7 @@ struct nvsp_5_send_indirect_table {
 	/* The number of entries in the send indirection table */
 	u32 count;
 
-	/* The offset of the send indireciton table from top of this struct.
+	/* The offset of the send indirection table from top of this struct.
 	 * The send indirection table tells which channel to put the send
 	 * traffic on. Each entry is a channel number.
 	 */
@@ -633,23 +632,41 @@ struct multi_send_data {
 	u32 count; /* counter of batched packets */
 };
 
+struct recv_comp_data {
+	u64 tid; /* transaction id */
+	u32 status;
+};
+
+/* Netvsc Receive Slots Max */
+#define NETVSC_RECVSLOT_MAX (NETVSC_RECEIVE_BUFFER_SIZE / ETH_DATA_LEN + 1)
+
+struct multi_recv_comp {
+	void *buf; /* queued receive completions */
+	u32 first; /* first data entry */
+	u32 next; /* next entry for writing */
+};
+
 struct netvsc_stats {
 	u64 packets;
 	u64 bytes;
+	u64 broadcast;
+	u64 multicast;
 	struct u64_stats_sync syncp;
 };
 
+struct netvsc_ethtool_stats {
+	unsigned long tx_scattered;
+	unsigned long tx_no_memory;
+	unsigned long tx_no_space;
+	unsigned long tx_too_big;
+	unsigned long tx_busy;
+};
+
 struct netvsc_reconfig {
 	struct list_head list;
 	u32 event;
 };
 
-struct garp_wrk {
-	struct work_struct dwrk;
-	struct net_device *netdev;
-	struct netvsc_device *netvsc_dev;
-};
-
 /* The context of the netvsc device  */
 struct net_device_context {
 	/* point back to our device context */
@@ -667,7 +684,6 @@ struct net_device_context {
 
 	struct work_struct work;
 	u32 msg_enable; /* debug level */
-	struct garp_wrk gwrk;
 
 	struct netvsc_stats __percpu *tx_stats;
 	struct netvsc_stats __percpu *rx_stats;
@@ -675,9 +691,18 @@ struct net_device_context {
 	/* Ethtool settings */
 	u8 duplex;
 	u32 speed;
+	struct netvsc_ethtool_stats eth_stats;
 
 	/* the device is going away */
 	bool start_remove;
+
+	/* State to manage the associated VF interface. */
+	struct net_device __rcu *vf_netdev;
+
+	/* 1: allocated, serial number is valid. 0: not allocated */
+	u32 vf_alloc;
+	/* Serial number of the VF to team with */
+	u32 vf_serial;
 };
 
 /* Per netvsc device */
@@ -709,7 +734,6 @@ struct netvsc_device {
 	struct nvsp_message channel_init_pkt;
 
 	struct nvsp_message revoke_packet;
-	/* unsigned char HwMacAddr[HW_MACADDR_LEN]; */
 
 	struct vmbus_channel *chn_table[VRSS_CHANNEL_MAX];
 	u32 send_table[VRSS_SEND_TAB_SIZE];
@@ -733,15 +757,10 @@ struct netvsc_device {
 	u32 max_pkt; /* max number of pkt in one send, e.g. 8 */
 	u32 pkt_align; /* alignment bytes, e.g. 8 */
 
-	/* 1: allocated, serial number is valid. 0: not allocated */
-	u32 vf_alloc;
-	/* Serial number of the VF to team with */
-	u32 vf_serial;
+	struct multi_recv_comp mrc[VRSS_CHANNEL_MAX];
+	atomic_t num_outstanding_recvs;
+
 	atomic_t open_cnt;
-	/* State to manage the associated VF interface. */
-	bool vf_inject;
-	struct net_device *vf_netdev;
-	atomic_t vf_use_cnt;
 };
 
 static inline struct netvsc_device *
@@ -1219,7 +1238,7 @@ struct rndis_message {
 	u32 ndis_msg_type;
 
 	/* Total length of this message, from the beginning */
-	/* of the sruct rndis_message, in bytes. */
+	/* of the struct rndis_message, in bytes. */
 	u32 msg_len;
 
 	/* Actual message */
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index 20e09174ff62..720b5fa9e625 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -59,7 +59,6 @@ void netvsc_switch_datapath(struct net_device *ndev, bool vf)
 			       VM_PKT_DATA_INBAND, 0);
 }
 
-
 static struct netvsc_device *alloc_net_device(void)
 {
 	struct netvsc_device *net_device;
@@ -74,21 +73,26 @@ static struct netvsc_device *alloc_net_device(void)
 		return NULL;
 	}
 
+	net_device->mrc[0].buf = vzalloc(NETVSC_RECVSLOT_MAX *
+					 sizeof(struct recv_comp_data));
+
 	init_waitqueue_head(&net_device->wait_drain);
 	net_device->destroy = false;
 	atomic_set(&net_device->open_cnt, 0);
-	atomic_set(&net_device->vf_use_cnt, 0);
 	net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT;
 	net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT;
-
-	net_device->vf_netdev = NULL;
-	net_device->vf_inject = false;
+	init_completion(&net_device->channel_init_wait);
 
 	return net_device;
 }
 
 static void free_netvsc_device(struct netvsc_device *nvdev)
 {
+	int i;
+
+	for (i = 0; i < VRSS_CHANNEL_MAX; i++)
+		vfree(nvdev->mrc[i].buf);
+
 	kfree(nvdev->cb_buffer);
 	kfree(nvdev);
 }
@@ -111,20 +115,20 @@ static struct netvsc_device *get_inbound_net_device(struct hv_device *device)
 		goto get_in_err;
 
 	if (net_device->destroy &&
-		atomic_read(&net_device->num_outstanding_sends) == 0)
+	    atomic_read(&net_device->num_outstanding_sends) == 0 &&
+	    atomic_read(&net_device->num_outstanding_recvs) == 0)
 		net_device = NULL;
 
 get_in_err:
 	return net_device;
 }
 
-
-static int netvsc_destroy_buf(struct hv_device *device)
+static void netvsc_destroy_buf(struct hv_device *device)
 {
 	struct nvsp_message *revoke_packet;
-	int ret = 0;
 	struct net_device *ndev = hv_get_drvdata(device);
 	struct netvsc_device *net_device = net_device_to_netvsc_device(ndev);
+	int ret;
 
 	/*
 	 * If we got a section count, it means we received a
@@ -154,7 +158,7 @@ static int netvsc_destroy_buf(struct hv_device *device)
 		if (ret != 0) {
 			netdev_err(ndev, "unable to send "
 				"revoke receive buffer to netvsp\n");
-			return ret;
+			return;
 		}
 	}
 
@@ -169,7 +173,7 @@ static int netvsc_destroy_buf(struct hv_device *device)
 		if (ret != 0) {
 			netdev_err(ndev,
 				   "unable to teardown receive buffer's gpadl\n");
-			return ret;
+			return;
 		}
 		net_device->recv_buf_gpadl_handle = 0;
 	}
@@ -213,7 +217,7 @@ static int netvsc_destroy_buf(struct hv_device *device)
 		if (ret != 0) {
 			netdev_err(ndev, "unable to send "
 				   "revoke send buffer to netvsp\n");
-			return ret;
+			return;
 		}
 	}
 	/* Teardown the gpadl on the vsp end */
@@ -227,7 +231,7 @@ static int netvsc_destroy_buf(struct hv_device *device)
 		if (ret != 0) {
 			netdev_err(ndev,
 				   "unable to teardown send buffer's gpadl\n");
-			return ret;
+			return;
 		}
 		net_device->send_buf_gpadl_handle = 0;
 	}
@@ -237,8 +241,6 @@ static int netvsc_destroy_buf(struct hv_device *device)
 		net_device->send_buf = NULL;
 	}
 	kfree(net_device->send_section_map);
-
-	return ret;
 }
 
 static int netvsc_init_buf(struct hv_device *device)
@@ -280,7 +282,6 @@ static int netvsc_init_buf(struct hv_device *device)
 		goto cleanup;
 	}
 
-
 	/* Notify the NetVsp of the gpadl handle */
 	init_packet = &net_device->channel_init_pkt;
 
@@ -407,7 +408,7 @@ static int netvsc_init_buf(struct hv_device *device)
 	/* Section count is simply the size divided by the section size.
 	 */
 	net_device->send_section_cnt =
-		net_device->send_buf_size/net_device->send_section_size;
+		net_device->send_buf_size / net_device->send_section_size;
 
 	dev_info(&device->device, "Send section size: %d, Section count:%d\n",
 		 net_device->send_section_size, net_device->send_section_cnt);
@@ -416,8 +417,8 @@ static int netvsc_init_buf(struct hv_device *device)
 	net_device->map_words = DIV_ROUND_UP(net_device->send_section_cnt,
 					     BITS_PER_LONG);
 
-	net_device->send_section_map =
-		kzalloc(net_device->map_words * sizeof(ulong), GFP_KERNEL);
+	net_device->send_section_map = kcalloc(net_device->map_words,
+					       sizeof(ulong), GFP_KERNEL);
 	if (net_device->send_section_map == NULL) {
 		ret = -ENOMEM;
 		goto cleanup;
@@ -432,7 +433,6 @@ exit:
 	return ret;
 }
 
-
 /* Negotiate NVSP protocol version */
 static int negotiate_nvsp_ver(struct hv_device *device,
 			      struct netvsc_device *net_device,
@@ -472,9 +472,13 @@ static int negotiate_nvsp_ver(struct hv_device *device,
 	init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN;
 	init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1;
 
-	if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5)
+	if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) {
 		init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1;
 
+		/* Teaming bit is needed to receive link speed updates */
+		init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1;
+	}
+
 	ret = vmbus_sendpacket(device->channel, init_packet,
 				sizeof(struct nvsp_message),
 				(unsigned long)init_packet,
@@ -489,9 +493,10 @@ static int netvsc_connect_vsp(struct hv_device *device)
 	struct netvsc_device *net_device;
 	struct nvsp_message *init_packet;
 	int ndis_version;
-	u32 ver_list[] = { NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2,
+	const u32 ver_list[] = {
+		NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2,
 		NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5 };
-	int i, num_ver = 4; /* number of different NVSP versions */
+	int i;
 
 	net_device = get_outbound_net_device(device);
 	if (!net_device)
@@ -500,7 +505,7 @@ static int netvsc_connect_vsp(struct hv_device *device)
 	init_packet = &net_device->channel_init_pkt;
 
 	/* Negotiate the latest NVSP protocol supported */
-	for (i = num_ver - 1; i >= 0; i--)
+	for (i = ARRAY_SIZE(ver_list) - 1; i >= 0; i--)
 		if (negotiate_nvsp_ver(device, net_device, init_packet,
 				       ver_list[i])  == 0) {
 			net_device->nvsp_version = ver_list[i];
@@ -559,7 +564,7 @@ static void netvsc_disconnect_vsp(struct hv_device *device)
 /*
  * netvsc_device_remove - Callback when the root bus device is removed
  */
-int netvsc_device_remove(struct hv_device *device)
+void netvsc_device_remove(struct hv_device *device)
 {
 	struct net_device *ndev = hv_get_drvdata(device);
 	struct net_device_context *net_device_ctx = netdev_priv(ndev);
@@ -581,10 +586,8 @@ int netvsc_device_remove(struct hv_device *device)
 	/* Release all resources */
 	vfree(net_device->sub_cb_buf);
 	free_netvsc_device(net_device);
-	return 0;
 }
 
-
 #define RING_AVAIL_PERCENT_HIWATER 20
 #define RING_AVAIL_PERCENT_LOWATER 10
 
@@ -608,72 +611,79 @@ static inline void netvsc_free_send_slot(struct netvsc_device *net_device,
 	sync_change_bit(index, net_device->send_section_map);
 }
 
+static void netvsc_send_tx_complete(struct netvsc_device *net_device,
+				    struct vmbus_channel *incoming_channel,
+				    struct hv_device *device,
+				    struct vmpacket_descriptor *packet)
+{
+	struct sk_buff *skb = (struct sk_buff *)(unsigned long)packet->trans_id;
+	struct net_device *ndev = hv_get_drvdata(device);
+	struct net_device_context *net_device_ctx = netdev_priv(ndev);
+	struct vmbus_channel *channel = device->channel;
+	int num_outstanding_sends;
+	u16 q_idx = 0;
+	int queue_sends;
+
+	/* Notify the layer above us */
+	if (likely(skb)) {
+		struct hv_netvsc_packet *nvsc_packet
+			= (struct hv_netvsc_packet *)skb->cb;
+		u32 send_index = nvsc_packet->send_buf_index;
+
+		if (send_index != NETVSC_INVALID_INDEX)
+			netvsc_free_send_slot(net_device, send_index);
+		q_idx = nvsc_packet->q_idx;
+		channel = incoming_channel;
+
+		dev_consume_skb_any(skb);
+	}
+
+	num_outstanding_sends =
+		atomic_dec_return(&net_device->num_outstanding_sends);
+	queue_sends = atomic_dec_return(&net_device->queue_sends[q_idx]);
+
+	if (net_device->destroy && num_outstanding_sends == 0)
+		wake_up(&net_device->wait_drain);
+
+	if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) &&
+	    !net_device_ctx->start_remove &&
+	    (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER ||
+	     queue_sends < 1))
+		netif_tx_wake_queue(netdev_get_tx_queue(ndev, q_idx));
+}
+
 static void netvsc_send_completion(struct netvsc_device *net_device,
 				   struct vmbus_channel *incoming_channel,
 				   struct hv_device *device,
 				   struct vmpacket_descriptor *packet)
 {
 	struct nvsp_message *nvsp_packet;
-	struct hv_netvsc_packet *nvsc_packet;
 	struct net_device *ndev = hv_get_drvdata(device);
-	struct net_device_context *net_device_ctx = netdev_priv(ndev);
-	u32 send_index;
-	struct sk_buff *skb;
 
 	nvsp_packet = (struct nvsp_message *)((unsigned long)packet +
-			(packet->offset8 << 3));
+					      (packet->offset8 << 3));
 
-	if ((nvsp_packet->hdr.msg_type == NVSP_MSG_TYPE_INIT_COMPLETE) ||
-	    (nvsp_packet->hdr.msg_type ==
-	     NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE) ||
-	    (nvsp_packet->hdr.msg_type ==
-	     NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE) ||
-	    (nvsp_packet->hdr.msg_type ==
-	     NVSP_MSG5_TYPE_SUBCHANNEL)) {
+	switch (nvsp_packet->hdr.msg_type) {
+	case NVSP_MSG_TYPE_INIT_COMPLETE:
+	case NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE:
+	case NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE:
+	case NVSP_MSG5_TYPE_SUBCHANNEL:
 		/* Copy the response back */
 		memcpy(&net_device->channel_init_pkt, nvsp_packet,
 		       sizeof(struct nvsp_message));
 		complete(&net_device->channel_init_wait);
-	} else if (nvsp_packet->hdr.msg_type ==
-		   NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE) {
-		int num_outstanding_sends;
-		u16 q_idx = 0;
-		struct vmbus_channel *channel = device->channel;
-		int queue_sends;
-
-		/* Get the send context */
-		skb = (struct sk_buff *)(unsigned long)packet->trans_id;
-
-		/* Notify the layer above us */
-		if (skb) {
-			nvsc_packet = (struct hv_netvsc_packet *) skb->cb;
-			send_index = nvsc_packet->send_buf_index;
-			if (send_index != NETVSC_INVALID_INDEX)
-				netvsc_free_send_slot(net_device, send_index);
-			q_idx = nvsc_packet->q_idx;
-			channel = incoming_channel;
-			dev_kfree_skb_any(skb);
-		}
-
-		num_outstanding_sends =
-			atomic_dec_return(&net_device->num_outstanding_sends);
-		queue_sends = atomic_dec_return(&net_device->
-						queue_sends[q_idx]);
+		break;
 
-		if (net_device->destroy && num_outstanding_sends == 0)
-			wake_up(&net_device->wait_drain);
+	case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE:
+		netvsc_send_tx_complete(net_device, incoming_channel,
+					device, packet);
+		break;
 
-		if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) &&
-		    !net_device_ctx->start_remove &&
-		    (hv_ringbuf_avail_percent(&channel->outbound) >
-		     RING_AVAIL_PERCENT_HIWATER || queue_sends < 1))
-				netif_tx_wake_queue(netdev_get_tx_queue(
-						    ndev, q_idx));
-	} else {
-		netdev_err(ndev, "Unknown send completion packet type- "
-			   "%d received!!\n", nvsp_packet->hdr.msg_type);
+	default:
+		netdev_err(ndev,
+			   "Unknown send completion type %d received!!\n",
+			   nvsp_packet->hdr.msg_type);
 	}
-
 }
 
 static u32 netvsc_get_next_send_section(struct netvsc_device *net_device)
@@ -863,7 +873,7 @@ int netvsc_send(struct hv_device *device,
 		struct sk_buff *skb)
 {
 	struct netvsc_device *net_device;
-	int ret = 0, m_ret = 0;
+	int ret = 0;
 	struct vmbus_channel *out_channel;
 	u16 q_idx = packet->q_idx;
 	u32 pktlen = packet->total_data_buflen, msd_len = 0;
@@ -934,7 +944,7 @@ int netvsc_send(struct hv_device *device,
 		}
 
 		if (msdp->skb)
-			dev_kfree_skb_any(msdp->skb);
+			dev_consume_skb_any(msdp->skb);
 
 		if (xmit_more && !packet->cp_partial) {
 			msdp->skb = skb;
@@ -952,8 +962,8 @@ int netvsc_send(struct hv_device *device,
 	}
 
 	if (msd_send) {
-		m_ret = netvsc_send_pkt(device, msd_send, net_device,
-					NULL, msd_skb);
+		int m_ret = netvsc_send_pkt(device, msd_send, net_device,
+					    NULL, msd_skb);
 
 		if (m_ret != 0) {
 			netvsc_free_send_slot(net_device,
@@ -972,49 +982,121 @@ send_now:
 	return ret;
 }
 
-static void netvsc_send_recv_completion(struct hv_device *device,
-					struct vmbus_channel *channel,
-					struct netvsc_device *net_device,
-					u64 transaction_id, u32 status)
+static int netvsc_send_recv_completion(struct vmbus_channel *channel,
+				       u64 transaction_id, u32 status)
 {
 	struct nvsp_message recvcompMessage;
-	int retries = 0;
 	int ret;
-	struct net_device *ndev = hv_get_drvdata(device);
 
 	recvcompMessage.hdr.msg_type =
 				NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE;
 
 	recvcompMessage.msg.v1_msg.send_rndis_pkt_complete.status = status;
 
-retry_send_cmplt:
 	/* Send the completion */
 	ret = vmbus_sendpacket(channel, &recvcompMessage,
-			       sizeof(struct nvsp_message), transaction_id,
-			       VM_PKT_COMP, 0);
-	if (ret == 0) {
-		/* success */
-		/* no-op */
-	} else if (ret == -EAGAIN) {
-		/* no more room...wait a bit and attempt to retry 3 times */
-		retries++;
-		netdev_err(ndev, "unable to send receive completion pkt"
-			" (tid %llx)...retrying %d\n", transaction_id, retries);
-
-		if (retries < 4) {
-			udelay(100);
-			goto retry_send_cmplt;
-		} else {
-			netdev_err(ndev, "unable to send receive "
-				"completion pkt (tid %llx)...give up retrying\n",
-				transaction_id);
-		}
-	} else {
-		netdev_err(ndev, "unable to send receive "
-			"completion pkt - %llx\n", transaction_id);
+			       sizeof(struct nvsp_message_header) + sizeof(u32),
+			       transaction_id, VM_PKT_COMP, 0);
+
+	return ret;
+}
+
+static inline void count_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx,
+					u32 *filled, u32 *avail)
+{
+	u32 first = nvdev->mrc[q_idx].first;
+	u32 next = nvdev->mrc[q_idx].next;
+
+	*filled = (first > next) ? NETVSC_RECVSLOT_MAX - first + next :
+		  next - first;
+
+	*avail = NETVSC_RECVSLOT_MAX - *filled - 1;
+}
+
+/* Read the first filled slot, no change to index */
+static inline struct recv_comp_data *read_recv_comp_slot(struct netvsc_device
+							 *nvdev, u16 q_idx)
+{
+	u32 filled, avail;
+
+	if (!nvdev->mrc[q_idx].buf)
+		return NULL;
+
+	count_recv_comp_slot(nvdev, q_idx, &filled, &avail);
+	if (!filled)
+		return NULL;
+
+	return nvdev->mrc[q_idx].buf + nvdev->mrc[q_idx].first *
+	       sizeof(struct recv_comp_data);
+}
+
+/* Put the first filled slot back to available pool */
+static inline void put_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx)
+{
+	int num_recv;
+
+	nvdev->mrc[q_idx].first = (nvdev->mrc[q_idx].first + 1) %
+				  NETVSC_RECVSLOT_MAX;
+
+	num_recv = atomic_dec_return(&nvdev->num_outstanding_recvs);
+
+	if (nvdev->destroy && num_recv == 0)
+		wake_up(&nvdev->wait_drain);
+}
+
+/* Check and send pending recv completions */
+static void netvsc_chk_recv_comp(struct netvsc_device *nvdev,
+				 struct vmbus_channel *channel, u16 q_idx)
+{
+	struct recv_comp_data *rcd;
+	int ret;
+
+	while (true) {
+		rcd = read_recv_comp_slot(nvdev, q_idx);
+		if (!rcd)
+			break;
+
+		ret = netvsc_send_recv_completion(channel, rcd->tid,
+						  rcd->status);
+		if (ret)
+			break;
+
+		put_recv_comp_slot(nvdev, q_idx);
 	}
 }
 
+#define NETVSC_RCD_WATERMARK 80
+
+/* Get next available slot */
+static inline struct recv_comp_data *get_recv_comp_slot(
+	struct netvsc_device *nvdev, struct vmbus_channel *channel, u16 q_idx)
+{
+	u32 filled, avail, next;
+	struct recv_comp_data *rcd;
+
+	if (!nvdev->recv_section)
+		return NULL;
+
+	if (!nvdev->mrc[q_idx].buf)
+		return NULL;
+
+	if (atomic_read(&nvdev->num_outstanding_recvs) >
+	    nvdev->recv_section->num_sub_allocs * NETVSC_RCD_WATERMARK / 100)
+		netvsc_chk_recv_comp(nvdev, channel, q_idx);
+
+	count_recv_comp_slot(nvdev, q_idx, &filled, &avail);
+	if (!avail)
+		return NULL;
+
+	next = nvdev->mrc[q_idx].next;
+	rcd = nvdev->mrc[q_idx].buf + next * sizeof(struct recv_comp_data);
+	nvdev->mrc[q_idx].next = (next + 1) % NETVSC_RECVSLOT_MAX;
+
+	atomic_inc(&nvdev->num_outstanding_recvs);
+
+	return rcd;
+}
+
 static void netvsc_receive(struct netvsc_device *net_device,
 			struct vmbus_channel *channel,
 			struct hv_device *device,
@@ -1029,6 +1111,9 @@ static void netvsc_receive(struct netvsc_device *net_device,
 	int count = 0;
 	struct net_device *ndev = hv_get_drvdata(device);
 	void *data;
+	int ret;
+	struct recv_comp_data *rcd;
+	u16 q_idx = channel->offermsg.offer.sub_channel_index;
 
 	/*
 	 * All inbound packets other than send completion should be xfer page
@@ -1073,13 +1158,29 @@ static void netvsc_receive(struct netvsc_device *net_device,
 		/* Pass it to the upper layer */
 		status = rndis_filter_receive(device, netvsc_packet, &data,
 					      channel);
+	}
 
+	if (!net_device->mrc[q_idx].buf) {
+		ret = netvsc_send_recv_completion(channel,
+						  vmxferpage_packet->d.trans_id,
+						  status);
+		if (ret)
+			netdev_err(ndev, "Recv_comp q:%hd, tid:%llx, err:%d\n",
+				   q_idx, vmxferpage_packet->d.trans_id, ret);
+		return;
 	}
 
-	netvsc_send_recv_completion(device, channel, net_device,
-				    vmxferpage_packet->d.trans_id, status);
-}
+	rcd = get_recv_comp_slot(net_device, channel, q_idx);
 
+	if (!rcd) {
+		netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n",
+			   q_idx, vmxferpage_packet->d.trans_id);
+		return;
+	}
+
+	rcd->tid = vmxferpage_packet->d.trans_id;
+	rcd->status = status;
+}
 
 static void netvsc_send_table(struct hv_device *hdev,
 			      struct nvsp_message *nvmsg)
@@ -1106,16 +1207,16 @@ static void netvsc_send_table(struct hv_device *hdev,
 		nvscdev->send_table[i] = tab[i];
 }
 
-static void netvsc_send_vf(struct netvsc_device *nvdev,
+static void netvsc_send_vf(struct net_device_context *net_device_ctx,
 			   struct nvsp_message *nvmsg)
 {
-	nvdev->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated;
-	nvdev->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial;
+	net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated;
+	net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial;
 }
 
 static inline void netvsc_receive_inband(struct hv_device *hdev,
-					 struct netvsc_device *nvdev,
-					 struct nvsp_message *nvmsg)
+				 struct net_device_context *net_device_ctx,
+				 struct nvsp_message *nvmsg)
 {
 	switch (nvmsg->hdr.msg_type) {
 	case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE:
@@ -1123,7 +1224,7 @@ static inline void netvsc_receive_inband(struct hv_device *hdev,
 		break;
 
 	case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION:
-		netvsc_send_vf(nvdev, nvmsg);
+		netvsc_send_vf(net_device_ctx, nvmsg);
 		break;
 	}
 }
@@ -1136,6 +1237,7 @@ static void netvsc_process_raw_pkt(struct hv_device *device,
 				   struct vmpacket_descriptor *desc)
 {
 	struct nvsp_message *nvmsg;
+	struct net_device_context *net_device_ctx = netdev_priv(ndev);
 
 	nvmsg = (struct nvsp_message *)((unsigned long)
 		desc + (desc->offset8 << 3));
@@ -1150,7 +1252,7 @@ static void netvsc_process_raw_pkt(struct hv_device *device,
 		break;
 
 	case VM_PKT_DATA_INBAND:
-		netvsc_receive_inband(device, net_device, nvmsg);
+		netvsc_receive_inband(device, net_device_ctx, nvmsg);
 		break;
 
 	default:
@@ -1160,11 +1262,11 @@ static void netvsc_process_raw_pkt(struct hv_device *device,
 	}
 }
 
-
 void netvsc_channel_cb(void *context)
 {
 	int ret;
 	struct vmbus_channel *channel = (struct vmbus_channel *)context;
+	u16 q_idx = channel->offermsg.offer.sub_channel_index;
 	struct hv_device *device;
 	struct netvsc_device *net_device;
 	u32 bytes_recvd;
@@ -1216,8 +1318,6 @@ void netvsc_channel_cb(void *context)
 						       ndev,
 						       request_id,
 						       desc);
-
-
 			} else {
 				/*
 				 * We are done for this pass.
@@ -1244,7 +1344,8 @@ void netvsc_channel_cb(void *context)
 
 	if (bufferlen > NETVSC_PACKET_SIZE)
 		kfree(buffer);
-	return;
+
+	netvsc_chk_recv_comp(net_device, channel, q_idx);
 }
 
 /*
@@ -1266,9 +1367,6 @@ int netvsc_device_add(struct hv_device *device, void *additional_info)
 
 	net_device->ring_size = ring_size;
 
-	/* Initialize the NetVSC channel extension */
-	init_completion(&net_device->channel_init_wait);
-
 	set_per_channel_state(device->channel, net_device->cb_buffer);
 
 	/* Open the channel */
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index 41bd952cc28d..52eeb2f67276 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -40,7 +40,6 @@
 
 #include "hyperv_net.h"
 
-
 #define RING_SIZE_MIN 64
 #define LINKCHANGE_INT (2 * HZ)
 #define NETVSC_HW_FEATURES	(NETIF_F_RXCSUM | \
@@ -358,18 +357,14 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 	struct rndis_message *rndis_msg;
 	struct rndis_packet *rndis_pkt;
 	u32 rndis_msg_size;
-	bool isvlan;
-	bool linear = false;
 	struct rndis_per_packet_info *ppi;
 	struct ndis_tcp_ip_checksum_info *csum_info;
-	struct ndis_tcp_lso_info *lso_info;
 	int  hdr_offset;
 	u32 net_trans_info;
 	u32 hash;
 	u32 skb_length;
 	struct hv_page_buffer page_buf[MAX_PAGE_BUFFER_COUNT];
 	struct hv_page_buffer *pb = page_buf;
-	struct netvsc_stats *tx_stats = this_cpu_ptr(net_device_ctx->tx_stats);
 
 	/* We will atmost need two pages to describe the rndis
 	 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
@@ -377,22 +372,20 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 	 * more pages we try linearizing it.
 	 */
 
-check_size:
 	skb_length = skb->len;
 	num_data_pgs = netvsc_get_slots(skb) + 2;
-	if (num_data_pgs > MAX_PAGE_BUFFER_COUNT && linear) {
-		net_alert_ratelimited("packet too big: %u pages (%u bytes)\n",
-				      num_data_pgs, skb->len);
-		ret = -EFAULT;
-		goto drop;
-	} else if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
-		if (skb_linearize(skb)) {
-			net_alert_ratelimited("failed to linearize skb\n");
-			ret = -ENOMEM;
+
+	if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
+		++net_device_ctx->eth_stats.tx_scattered;
+
+		if (skb_linearize(skb))
+			goto no_memory;
+
+		num_data_pgs = netvsc_get_slots(skb) + 2;
+		if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
+			++net_device_ctx->eth_stats.tx_too_big;
 			goto drop;
 		}
-		linear = true;
-		goto check_size;
 	}
 
 	/*
@@ -401,17 +394,14 @@ check_size:
 	 * structure.
 	 */
 	ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
-	if (ret) {
-		netdev_err(net, "unable to alloc hv_netvsc_packet\n");
-		ret = -ENOMEM;
-		goto drop;
-	}
+	if (ret)
+		goto no_memory;
+
 	/* Use the skb control buffer for building up the packet */
 	BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
 			FIELD_SIZEOF(struct sk_buff, cb));
 	packet = (struct hv_netvsc_packet *)skb->cb;
 
-
 	packet->q_idx = skb_get_queue_mapping(skb);
 
 	packet->total_data_buflen = skb->len;
@@ -420,8 +410,6 @@ check_size:
 
 	memset(rndis_msg, 0, RNDIS_AND_PPI_SIZE);
 
-	isvlan = skb->vlan_tci & VLAN_TAG_PRESENT;
-
 	/* Add the rndis header */
 	rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
 	rndis_msg->msg_len = packet->total_data_buflen;
@@ -440,7 +428,7 @@ check_size:
 		*(u32 *)((void *)ppi + ppi->ppi_offset) = hash;
 	}
 
-	if (isvlan) {
+	if (skb_vlan_tag_present(skb)) {
 		struct ndis_pkt_8021q_info *vlan;
 
 		rndis_msg_size += NDIS_VLAN_PPI_SIZE;
@@ -461,8 +449,37 @@ check_size:
 	 * Setup the sendside checksum offload only if this is not a
 	 * GSO packet.
 	 */
-	if (skb_is_gso(skb))
-		goto do_lso;
+	if (skb_is_gso(skb)) {
+		struct ndis_tcp_lso_info *lso_info;
+
+		rndis_msg_size += NDIS_LSO_PPI_SIZE;
+		ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
+				    TCP_LARGESEND_PKTINFO);
+
+		lso_info = (struct ndis_tcp_lso_info *)((void *)ppi +
+							ppi->ppi_offset);
+
+		lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
+		if (net_trans_info & (INFO_IPV4 << 16)) {
+			lso_info->lso_v2_transmit.ip_version =
+				NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
+			ip_hdr(skb)->tot_len = 0;
+			ip_hdr(skb)->check = 0;
+			tcp_hdr(skb)->check =
+				~csum_tcpudp_magic(ip_hdr(skb)->saddr,
+						   ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
+		} else {
+			lso_info->lso_v2_transmit.ip_version =
+				NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
+			ipv6_hdr(skb)->payload_len = 0;
+			tcp_hdr(skb)->check =
+				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
+						 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
+		}
+		lso_info->lso_v2_transmit.tcp_header_offset = hdr_offset;
+		lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
+		goto do_send;
+	}
 
 	if ((skb->ip_summed == CHECKSUM_NONE) ||
 	    (skb->ip_summed == CHECKSUM_UNNECESSARY))
@@ -495,7 +512,7 @@ check_size:
 
 		ret = skb_cow_head(skb, 0);
 		if (ret)
-			goto drop;
+			goto no_memory;
 
 		uh = udp_hdr(skb);
 		udp_len = ntohs(uh->len);
@@ -509,35 +526,6 @@ check_size:
 
 		csum_info->transmit.udp_checksum = 0;
 	}
-	goto do_send;
-
-do_lso:
-	rndis_msg_size += NDIS_LSO_PPI_SIZE;
-	ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
-			    TCP_LARGESEND_PKTINFO);
-
-	lso_info = (struct ndis_tcp_lso_info *)((void *)ppi +
-			ppi->ppi_offset);
-
-	lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
-	if (net_trans_info & (INFO_IPV4 << 16)) {
-		lso_info->lso_v2_transmit.ip_version =
-			NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
-		ip_hdr(skb)->tot_len = 0;
-		ip_hdr(skb)->check = 0;
-		tcp_hdr(skb)->check =
-		~csum_tcpudp_magic(ip_hdr(skb)->saddr,
-				   ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
-	} else {
-		lso_info->lso_v2_transmit.ip_version =
-			NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
-		ipv6_hdr(skb)->payload_len = 0;
-		tcp_hdr(skb)->check =
-		~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
-				&ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
-	}
-	lso_info->lso_v2_transmit.tcp_header_offset = hdr_offset;
-	lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
 
 do_send:
 	/* Start filling in the page buffers with the rndis hdr */
@@ -550,21 +538,33 @@ do_send:
 	skb_tx_timestamp(skb);
 	ret = netvsc_send(net_device_ctx->device_ctx, packet,
 			  rndis_msg, &pb, skb);
+	if (likely(ret == 0)) {
+		struct netvsc_stats *tx_stats = this_cpu_ptr(net_device_ctx->tx_stats);
 
-drop:
-	if (ret == 0) {
 		u64_stats_update_begin(&tx_stats->syncp);
 		tx_stats->packets++;
 		tx_stats->bytes += skb_length;
 		u64_stats_update_end(&tx_stats->syncp);
-	} else {
-		if (ret != -EAGAIN) {
-			dev_kfree_skb_any(skb);
-			net->stats.tx_dropped++;
-		}
+		return NETDEV_TX_OK;
+	}
+
+	if (ret == -EAGAIN) {
+		++net_device_ctx->eth_stats.tx_busy;
+		return NETDEV_TX_BUSY;
 	}
 
-	return (ret == -EAGAIN) ? NETDEV_TX_BUSY : NETDEV_TX_OK;
+	if (ret == -ENOSPC)
+		++net_device_ctx->eth_stats.tx_no_space;
+
+drop:
+	dev_kfree_skb_any(skb);
+	net->stats.tx_dropped++;
+
+	return NETDEV_TX_OK;
+
+no_memory:
+	++net_device_ctx->eth_stats.tx_no_memory;
+	goto drop;
 }
 
 /*
@@ -579,19 +579,32 @@ void netvsc_linkstatus_callback(struct hv_device *device_obj,
 	struct netvsc_reconfig *event;
 	unsigned long flags;
 
-	/* Handle link change statuses only */
+	net = hv_get_drvdata(device_obj);
+
+	if (!net)
+		return;
+
+	ndev_ctx = netdev_priv(net);
+
+	/* Update the physical link speed when changing to another vSwitch */
+	if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
+		u32 speed;
+
+		speed = *(u32 *)((void *)indicate + indicate->
+				 status_buf_offset) / 10000;
+		ndev_ctx->speed = speed;
+		return;
+	}
+
+	/* Handle these link change statuses below */
 	if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE &&
 	    indicate->status != RNDIS_STATUS_MEDIA_CONNECT &&
 	    indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT)
 		return;
 
-	net = hv_get_drvdata(device_obj);
-
-	if (!net || net->reg_state != NETREG_REGISTERED)
+	if (net->reg_state != NETREG_REGISTERED)
 		return;
 
-	ndev_ctx = netdev_priv(net);
-
 	event = kzalloc(sizeof(*event), GFP_ATOMIC);
 	if (!event)
 		return;
@@ -604,7 +617,6 @@ void netvsc_linkstatus_callback(struct hv_device *device_obj,
 	schedule_delayed_work(&ndev_ctx->dwork, 0);
 }
 
-
 static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
 				struct hv_netvsc_packet *packet,
 				struct ndis_tcp_ip_checksum_info *csum_info,
@@ -655,50 +667,23 @@ int netvsc_recv_callback(struct hv_device *device_obj,
 {
 	struct net_device *net = hv_get_drvdata(device_obj);
 	struct net_device_context *net_device_ctx = netdev_priv(net);
+	struct net_device *vf_netdev;
 	struct sk_buff *skb;
-	struct sk_buff *vf_skb;
 	struct netvsc_stats *rx_stats;
-	struct netvsc_device *netvsc_dev = net_device_ctx->nvdev;
-	u32 bytes_recvd = packet->total_data_buflen;
-	int ret = 0;
 
-	if (!net || net->reg_state != NETREG_REGISTERED)
+	if (net->reg_state != NETREG_REGISTERED)
 		return NVSP_STAT_FAIL;
 
-	if (READ_ONCE(netvsc_dev->vf_inject)) {
-		atomic_inc(&netvsc_dev->vf_use_cnt);
-		if (!READ_ONCE(netvsc_dev->vf_inject)) {
-			/*
-			 * We raced; just move on.
-			 */
-			atomic_dec(&netvsc_dev->vf_use_cnt);
-			goto vf_injection_done;
-		}
-
-		/*
-		 * Inject this packet into the VF inerface.
-		 * On Hyper-V, multicast and brodcast packets
-		 * are only delivered on the synthetic interface
-		 * (after subjecting these to policy filters on
-		 * the host). Deliver these via the VF interface
-		 * in the guest.
-		 */
-		vf_skb = netvsc_alloc_recv_skb(netvsc_dev->vf_netdev, packet,
-					       csum_info, *data, vlan_tci);
-		if (vf_skb != NULL) {
-			++netvsc_dev->vf_netdev->stats.rx_packets;
-			netvsc_dev->vf_netdev->stats.rx_bytes += bytes_recvd;
-			netif_receive_skb(vf_skb);
-		} else {
-			++net->stats.rx_dropped;
-			ret = NVSP_STAT_FAIL;
-		}
-		atomic_dec(&netvsc_dev->vf_use_cnt);
-		return ret;
-	}
-
-vf_injection_done:
-	rx_stats = this_cpu_ptr(net_device_ctx->rx_stats);
+	/*
+	 * If necessary, inject this packet into the VF interface.
+	 * On Hyper-V, multicast and brodcast packets are only delivered
+	 * to the synthetic interface (after subjecting these to
+	 * policy filters on the host). Deliver these via the VF
+	 * interface in the guest.
+	 */
+	vf_netdev = rcu_dereference(net_device_ctx->vf_netdev);
+	if (vf_netdev && (vf_netdev->flags & IFF_UP))
+		net = vf_netdev;
 
 	/* Allocate a skb - TODO direct I/O to pages? */
 	skb = netvsc_alloc_recv_skb(net, packet, csum_info, *data, vlan_tci);
@@ -706,12 +691,25 @@ vf_injection_done:
 		++net->stats.rx_dropped;
 		return NVSP_STAT_FAIL;
 	}
-	skb_record_rx_queue(skb, channel->
-			    offermsg.offer.sub_channel_index);
 
+	if (net != vf_netdev)
+		skb_record_rx_queue(skb,
+				    channel->offermsg.offer.sub_channel_index);
+
+	/*
+	 * Even if injecting the packet, record the statistics
+	 * on the synthetic device because modifying the VF device
+	 * statistics will not work correctly.
+	 */
+	rx_stats = this_cpu_ptr(net_device_ctx->rx_stats);
 	u64_stats_update_begin(&rx_stats->syncp);
 	rx_stats->packets++;
 	rx_stats->bytes += packet->total_data_buflen;
+
+	if (skb->pkt_type == PACKET_BROADCAST)
+		++rx_stats->broadcast;
+	else if (skb->pkt_type == PACKET_MULTICAST)
+		++rx_stats->multicast;
 	u64_stats_update_end(&rx_stats->syncp);
 
 	/*
@@ -727,8 +725,12 @@ vf_injection_done:
 static void netvsc_get_drvinfo(struct net_device *net,
 			       struct ethtool_drvinfo *info)
 {
+	struct net_device_context *net_device_ctx = netdev_priv(net);
+	struct hv_device *dev = net_device_ctx->device_ctx;
+
 	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
 	strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
+	strlcpy(info->bus_info, vmbus_dev_name(dev), sizeof(info->bus_info));
 }
 
 static void netvsc_get_channels(struct net_device *net,
@@ -950,7 +952,7 @@ static struct rtnl_link_stats64 *netvsc_get_stats64(struct net_device *net,
 							    cpu);
 		struct netvsc_stats *rx_stats = per_cpu_ptr(ndev_ctx->rx_stats,
 							    cpu);
-		u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
+		u64 tx_packets, tx_bytes, rx_packets, rx_bytes, rx_multicast;
 		unsigned int start;
 
 		do {
@@ -963,12 +965,14 @@ static struct rtnl_link_stats64 *netvsc_get_stats64(struct net_device *net,
 			start = u64_stats_fetch_begin_irq(&rx_stats->syncp);
 			rx_packets = rx_stats->packets;
 			rx_bytes = rx_stats->bytes;
+			rx_multicast = rx_stats->multicast + rx_stats->broadcast;
 		} while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start));
 
 		t->tx_bytes	+= tx_bytes;
 		t->tx_packets	+= tx_packets;
 		t->rx_bytes	+= rx_bytes;
 		t->rx_packets	+= rx_packets;
+		t->multicast	+= rx_multicast;
 	}
 
 	t->tx_dropped	= net->stats.tx_dropped;
@@ -1004,6 +1008,51 @@ static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
 	return err;
 }
 
+static const struct {
+	char name[ETH_GSTRING_LEN];
+	u16 offset;
+} netvsc_stats[] = {
+	{ "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) },
+	{ "tx_no_memory",  offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
+	{ "tx_no_space",  offsetof(struct netvsc_ethtool_stats, tx_no_space) },
+	{ "tx_too_big",	  offsetof(struct netvsc_ethtool_stats, tx_too_big) },
+	{ "tx_busy",	  offsetof(struct netvsc_ethtool_stats, tx_busy) },
+};
+
+static int netvsc_get_sset_count(struct net_device *dev, int string_set)
+{
+	switch (string_set) {
+	case ETH_SS_STATS:
+		return ARRAY_SIZE(netvsc_stats);
+	default:
+		return -EINVAL;
+	}
+}
+
+static void netvsc_get_ethtool_stats(struct net_device *dev,
+				     struct ethtool_stats *stats, u64 *data)
+{
+	struct net_device_context *ndc = netdev_priv(dev);
+	const void *nds = &ndc->eth_stats;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++)
+		data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
+}
+
+static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
+{
+	int i;
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++)
+			memcpy(data + i * ETH_GSTRING_LEN,
+			       netvsc_stats[i].name, ETH_GSTRING_LEN);
+		break;
+	}
+}
+
 #ifdef CONFIG_NET_POLL_CONTROLLER
 static void netvsc_poll_controller(struct net_device *net)
 {
@@ -1016,6 +1065,9 @@ static void netvsc_poll_controller(struct net_device *net)
 static const struct ethtool_ops ethtool_ops = {
 	.get_drvinfo	= netvsc_get_drvinfo,
 	.get_link	= ethtool_op_get_link,
+	.get_ethtool_stats = netvsc_get_ethtool_stats,
+	.get_sset_count = netvsc_get_sset_count,
+	.get_strings	= netvsc_get_strings,
 	.get_channels   = netvsc_get_channels,
 	.set_channels   = netvsc_set_channels,
 	.get_ts_info	= ethtool_op_get_ts_info,
@@ -1150,36 +1202,44 @@ static void netvsc_free_netdev(struct net_device *netdev)
 	free_netdev(netdev);
 }
 
-static void netvsc_notify_peers(struct work_struct *wrk)
+static struct net_device *get_netvsc_bymac(const u8 *mac)
 {
-	struct garp_wrk *gwrk;
+	struct net_device *dev;
 
-	gwrk = container_of(wrk, struct garp_wrk, dwrk);
+	ASSERT_RTNL();
 
-	netdev_notify_peers(gwrk->netdev);
+	for_each_netdev(&init_net, dev) {
+		if (dev->netdev_ops != &device_ops)
+			continue;	/* not a netvsc device */
+
+		if (ether_addr_equal(mac, dev->perm_addr))
+			return dev;
+	}
 
-	atomic_dec(&gwrk->netvsc_dev->vf_use_cnt);
+	return NULL;
 }
 
-static struct net_device *get_netvsc_net_device(char *mac)
+static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
 {
-	struct net_device *dev, *found = NULL;
-	int rtnl_locked;
+	struct net_device *dev;
 
-	rtnl_locked = rtnl_trylock();
+	ASSERT_RTNL();
 
 	for_each_netdev(&init_net, dev) {
-		if (memcmp(dev->dev_addr, mac, ETH_ALEN) == 0) {
-			if (dev->netdev_ops != &device_ops)
-				continue;
-			found = dev;
-			break;
-		}
+		struct net_device_context *net_device_ctx;
+
+		if (dev->netdev_ops != &device_ops)
+			continue;	/* not a netvsc device */
+
+		net_device_ctx = netdev_priv(dev);
+		if (net_device_ctx->nvdev == NULL)
+			continue;	/* device is removed */
+
+		if (rtnl_dereference(net_device_ctx->vf_netdev) == vf_netdev)
+			return dev;	/* a match */
 	}
-	if (rtnl_locked)
-		rtnl_unlock();
 
-	return found;
+	return NULL;
 }
 
 static int netvsc_register_vf(struct net_device *vf_netdev)
@@ -1187,9 +1247,8 @@ static int netvsc_register_vf(struct net_device *vf_netdev)
 	struct net_device *ndev;
 	struct net_device_context *net_device_ctx;
 	struct netvsc_device *netvsc_dev;
-	const struct ethtool_ops *eth_ops = vf_netdev->ethtool_ops;
 
-	if (eth_ops == NULL || eth_ops == &ethtool_ops)
+	if (vf_netdev->addr_len != ETH_ALEN)
 		return NOTIFY_DONE;
 
 	/*
@@ -1197,13 +1256,13 @@ static int netvsc_register_vf(struct net_device *vf_netdev)
 	 * associate with the VF interface. If we don't find a matching
 	 * synthetic interface, move on.
 	 */
-	ndev = get_netvsc_net_device(vf_netdev->dev_addr);
+	ndev = get_netvsc_bymac(vf_netdev->perm_addr);
 	if (!ndev)
 		return NOTIFY_DONE;
 
 	net_device_ctx = netdev_priv(ndev);
 	netvsc_dev = net_device_ctx->nvdev;
-	if (netvsc_dev == NULL)
+	if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
 		return NOTIFY_DONE;
 
 	netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
@@ -1211,33 +1270,26 @@ static int netvsc_register_vf(struct net_device *vf_netdev)
 	 * Take a reference on the module.
 	 */
 	try_module_get(THIS_MODULE);
-	netvsc_dev->vf_netdev = vf_netdev;
+
+	dev_hold(vf_netdev);
+	rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
 	return NOTIFY_OK;
 }
 
-
 static int netvsc_vf_up(struct net_device *vf_netdev)
 {
 	struct net_device *ndev;
 	struct netvsc_device *netvsc_dev;
-	const struct ethtool_ops *eth_ops = vf_netdev->ethtool_ops;
 	struct net_device_context *net_device_ctx;
 
-	if (eth_ops == &ethtool_ops)
-		return NOTIFY_DONE;
-
-	ndev = get_netvsc_net_device(vf_netdev->dev_addr);
+	ndev = get_netvsc_byref(vf_netdev);
 	if (!ndev)
 		return NOTIFY_DONE;
 
 	net_device_ctx = netdev_priv(ndev);
 	netvsc_dev = net_device_ctx->nvdev;
 
-	if ((netvsc_dev == NULL) || (netvsc_dev->vf_netdev == NULL))
-		return NOTIFY_DONE;
-
 	netdev_info(ndev, "VF up: %s\n", vf_netdev->name);
-	netvsc_dev->vf_inject = true;
 
 	/*
 	 * Open the device before switching data path.
@@ -1252,86 +1304,54 @@ static int netvsc_vf_up(struct net_device *vf_netdev)
 
 	netif_carrier_off(ndev);
 
-	/*
-	 * Now notify peers. We are scheduling work to
-	 * notify peers; take a reference to prevent
-	 * the VF interface from vanishing.
-	 */
-	atomic_inc(&netvsc_dev->vf_use_cnt);
-	net_device_ctx->gwrk.netdev = vf_netdev;
-	net_device_ctx->gwrk.netvsc_dev = netvsc_dev;
-	schedule_work(&net_device_ctx->gwrk.dwrk);
+	/* Now notify peers through VF device. */
+	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, vf_netdev);
 
 	return NOTIFY_OK;
 }
 
-
 static int netvsc_vf_down(struct net_device *vf_netdev)
 {
 	struct net_device *ndev;
 	struct netvsc_device *netvsc_dev;
 	struct net_device_context *net_device_ctx;
-	const struct ethtool_ops *eth_ops = vf_netdev->ethtool_ops;
-
-	if (eth_ops == &ethtool_ops)
-		return NOTIFY_DONE;
 
-	ndev = get_netvsc_net_device(vf_netdev->dev_addr);
+	ndev = get_netvsc_byref(vf_netdev);
 	if (!ndev)
 		return NOTIFY_DONE;
 
 	net_device_ctx = netdev_priv(ndev);
 	netvsc_dev = net_device_ctx->nvdev;
 
-	if ((netvsc_dev == NULL) || (netvsc_dev->vf_netdev == NULL))
-		return NOTIFY_DONE;
-
 	netdev_info(ndev, "VF down: %s\n", vf_netdev->name);
-	netvsc_dev->vf_inject = false;
-	/*
-	 * Wait for currently active users to
-	 * drain out.
-	 */
-
-	while (atomic_read(&netvsc_dev->vf_use_cnt) != 0)
-		udelay(50);
 	netvsc_switch_datapath(ndev, false);
 	netdev_info(ndev, "Data path switched from VF: %s\n", vf_netdev->name);
 	rndis_filter_close(netvsc_dev);
 	netif_carrier_on(ndev);
-	/*
-	 * Notify peers.
-	 */
-	atomic_inc(&netvsc_dev->vf_use_cnt);
-	net_device_ctx->gwrk.netdev = ndev;
-	net_device_ctx->gwrk.netvsc_dev = netvsc_dev;
-	schedule_work(&net_device_ctx->gwrk.dwrk);
+
+	/* Now notify peers through netvsc device. */
+	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, ndev);
 
 	return NOTIFY_OK;
 }
 
-
 static int netvsc_unregister_vf(struct net_device *vf_netdev)
 {
 	struct net_device *ndev;
 	struct netvsc_device *netvsc_dev;
-	const struct ethtool_ops *eth_ops = vf_netdev->ethtool_ops;
 	struct net_device_context *net_device_ctx;
 
-	if (eth_ops == &ethtool_ops)
-		return NOTIFY_DONE;
-
-	ndev = get_netvsc_net_device(vf_netdev->dev_addr);
+	ndev = get_netvsc_byref(vf_netdev);
 	if (!ndev)
 		return NOTIFY_DONE;
 
 	net_device_ctx = netdev_priv(ndev);
 	netvsc_dev = net_device_ctx->nvdev;
-	if (netvsc_dev == NULL)
-		return NOTIFY_DONE;
+
 	netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
 
-	netvsc_dev->vf_netdev = NULL;
+	RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
+	dev_put(vf_netdev);
 	module_put(THIS_MODULE);
 	return NOTIFY_OK;
 }
@@ -1352,6 +1372,8 @@ static int netvsc_probe(struct hv_device *dev,
 
 	netif_carrier_off(net);
 
+	netvsc_init_settings(net);
+
 	net_device_ctx = netdev_priv(net);
 	net_device_ctx->device_ctx = dev;
 	net_device_ctx->msg_enable = netif_msg_init(debug, default_msg);
@@ -1377,7 +1399,6 @@ static int netvsc_probe(struct hv_device *dev,
 
 	INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change);
 	INIT_WORK(&net_device_ctx->work, do_set_multicast);
-	INIT_WORK(&net_device_ctx->gwrk.dwrk, netvsc_notify_peers);
 
 	spin_lock_init(&net_device_ctx->lock);
 	INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
@@ -1410,8 +1431,6 @@ static int netvsc_probe(struct hv_device *dev,
 	netif_set_real_num_tx_queues(net, nvdev->num_chn);
 	netif_set_real_num_rx_queues(net, nvdev->num_chn);
 
-	netvsc_init_settings(net);
-
 	ret = register_netdev(net);
 	if (ret != 0) {
 		pr_err("Unable to register netdev.\n");
@@ -1435,7 +1454,6 @@ static int netvsc_remove(struct hv_device *dev)
 		return 0;
 	}
 
-
 	ndev_ctx = netdev_priv(net);
 	net_device = ndev_ctx->nvdev;
 
@@ -1482,7 +1500,6 @@ static struct  hv_driver netvsc_drv = {
 	.remove = netvsc_remove,
 };
 
-
 /*
  * On Hyper-V, every VF interface is matched with a corresponding
  * synthetic interface. The synthetic interface is presented first
@@ -1494,8 +1511,21 @@ static int netvsc_netdev_event(struct notifier_block *this,
 {
 	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
 
-	/* Avoid Vlan, Bonding dev with same MAC registering as VF */
-	if (event_dev->priv_flags & (IFF_802_1Q_VLAN | IFF_BONDING))
+	/* Skip our own events */
+	if (event_dev->netdev_ops == &device_ops)
+		return NOTIFY_DONE;
+
+	/* Avoid non-Ethernet type devices */
+	if (event_dev->type != ARPHRD_ETHER)
+		return NOTIFY_DONE;
+
+	/* Avoid Vlan dev with same MAC registering as VF */
+	if (event_dev->priv_flags & IFF_802_1Q_VLAN)
+		return NOTIFY_DONE;
+
+	/* Avoid Bonding master dev with same MAC registering as VF */
+	if ((event_dev->priv_flags & IFF_BONDING) &&
+	    (event_dev->flags & IFF_MASTER))
 		return NOTIFY_DONE;
 
 	switch (event) {
diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c
index 8e830f741d47..9195d5da8485 100644
--- a/drivers/net/hyperv/rndis_filter.c
+++ b/drivers/net/hyperv/rndis_filter.c
@@ -663,13 +663,14 @@ cleanup:
 	return ret;
 }
 
-u8 netvsc_hash_key[HASH_KEYLEN] = {
+static const u8 netvsc_hash_key[] = {
 	0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
 	0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
 	0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
 	0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
 	0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa
 };
+#define HASH_KEYLEN ARRAY_SIZE(netvsc_hash_key)
 
 static int rndis_filter_set_rss_param(struct rndis_device *rdev, int num_queue)
 {
@@ -720,7 +721,6 @@ static int rndis_filter_set_rss_param(struct rndis_device *rdev, int num_queue)
 	for (i = 0; i < HASH_KEYLEN; i++)
 		keyp[i] = netvsc_hash_key[i];
 
-
 	ret = rndis_filter_send_request(rdev, request);
 	if (ret != 0)
 		goto cleanup;
@@ -738,7 +738,6 @@ cleanup:
 	return ret;
 }
 
-
 static int rndis_filter_query_device_link_status(struct rndis_device *dev)
 {
 	u32 size = sizeof(u32);
@@ -752,6 +751,28 @@ static int rndis_filter_query_device_link_status(struct rndis_device *dev)
 	return ret;
 }
 
+static int rndis_filter_query_link_speed(struct rndis_device *dev)
+{
+	u32 size = sizeof(u32);
+	u32 link_speed;
+	struct net_device_context *ndc;
+	int ret;
+
+	ret = rndis_filter_query_device(dev, RNDIS_OID_GEN_LINK_SPEED,
+					&link_speed, &size);
+
+	if (!ret) {
+		ndc = netdev_priv(dev->ndev);
+
+		/* The link speed reported from host is in 100bps unit, so
+		 * we convert it to Mbps here.
+		 */
+		ndc->speed = link_speed / 10000;
+	}
+
+	return ret;
+}
+
 int rndis_filter_set_packet_filter(struct rndis_device *dev, u32 new_filter)
 {
 	struct rndis_request *request;
@@ -792,7 +813,6 @@ cleanup:
 	return ret;
 }
 
-
 static int rndis_filter_init_device(struct rndis_device *dev)
 {
 	struct rndis_request *request;
@@ -875,11 +895,11 @@ cleanup:
 
 	/* Wait for all send completions */
 	wait_event(nvdev->wait_drain,
-		atomic_read(&nvdev->num_outstanding_sends) == 0);
+		   atomic_read(&nvdev->num_outstanding_sends) == 0 &&
+		   atomic_read(&nvdev->num_outstanding_recvs) == 0);
 
 	if (request)
 		put_rndis_request(dev, request);
-	return;
 }
 
 static int rndis_filter_open_device(struct rndis_device *dev)
@@ -931,6 +951,9 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc)
 	set_per_channel_state(new_sc, nvscdev->sub_cb_buf + (chn_index - 1) *
 			      NETVSC_PACKET_SIZE);
 
+	nvscdev->mrc[chn_index].buf = vzalloc(NETVSC_RECVSLOT_MAX *
+					      sizeof(struct recv_comp_data));
+
 	ret = vmbus_open(new_sc, nvscdev->ring_size * PAGE_SIZE,
 			 nvscdev->ring_size * PAGE_SIZE, NULL, 0,
 			 netvsc_channel_cb, new_sc);
@@ -946,7 +969,7 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc)
 }
 
 int rndis_filter_device_add(struct hv_device *dev,
-				  void *additional_info)
+			    void *additional_info)
 {
 	int ret;
 	struct net_device *net = hv_get_drvdata(dev);
@@ -1028,7 +1051,6 @@ int rndis_filter_device_add(struct hv_device *dev,
 	offloads.udp_ip_v6_csum = NDIS_OFFLOAD_PARAMETERS_TX_RX_ENABLED;
 	offloads.lso_v2_ipv4 = NDIS_OFFLOAD_PARAMETERS_LSOV2_ENABLED;
 
-
 	ret = rndis_filter_set_offload_params(net, &offloads);
 	if (ret)
 		goto err_dev_remv;
@@ -1044,6 +1066,8 @@ int rndis_filter_device_add(struct hv_device *dev,
 	if (net_device->nvsp_version < NVSP_PROTOCOL_VERSION_5)
 		return 0;
 
+	rndis_filter_query_link_speed(rndis_device);
+
 	/* vRSS setup */
 	memset(&rsscap, 0, rsscap_size);
 	ret = rndis_filter_query_device(rndis_device,
@@ -1152,7 +1176,6 @@ void rndis_filter_device_remove(struct hv_device *dev)
 	netvsc_device_remove(dev);
 }
 
-
 int rndis_filter_open(struct netvsc_device *nvdev)
 {
 	if (!nvdev)
diff --git a/drivers/net/ieee802154/fakelb.c b/drivers/net/ieee802154/fakelb.c
index 0becf0ac3926..ec387efb61d0 100644
--- a/drivers/net/ieee802154/fakelb.c
+++ b/drivers/net/ieee802154/fakelb.c
@@ -30,7 +30,7 @@
 static int numlbs = 2;
 
 static LIST_HEAD(fakelb_phys);
-static DEFINE_SPINLOCK(fakelb_phys_lock);
+static DEFINE_MUTEX(fakelb_phys_lock);
 
 static LIST_HEAD(fakelb_ifup_phys);
 static DEFINE_RWLOCK(fakelb_ifup_phys_lock);
@@ -188,9 +188,9 @@ static int fakelb_add_one(struct device *dev)
 	if (err)
 		goto err_reg;
 
-	spin_lock(&fakelb_phys_lock);
+	mutex_lock(&fakelb_phys_lock);
 	list_add_tail(&phy->list, &fakelb_phys);
-	spin_unlock(&fakelb_phys_lock);
+	mutex_unlock(&fakelb_phys_lock);
 
 	return 0;
 
@@ -222,10 +222,10 @@ static int fakelb_probe(struct platform_device *pdev)
 	return 0;
 
 err_slave:
-	spin_lock(&fakelb_phys_lock);
+	mutex_lock(&fakelb_phys_lock);
 	list_for_each_entry_safe(phy, tmp, &fakelb_phys, list)
 		fakelb_del(phy);
-	spin_unlock(&fakelb_phys_lock);
+	mutex_unlock(&fakelb_phys_lock);
 	return err;
 }
 
@@ -233,10 +233,10 @@ static int fakelb_remove(struct platform_device *pdev)
 {
 	struct fakelb_phy *phy, *tmp;
 
-	spin_lock(&fakelb_phys_lock);
+	mutex_lock(&fakelb_phys_lock);
 	list_for_each_entry_safe(phy, tmp, &fakelb_phys, list)
 		fakelb_del(phy);
-	spin_unlock(&fakelb_phys_lock);
+	mutex_unlock(&fakelb_phys_lock);
 	return 0;
 }
 
diff --git a/drivers/net/ipvlan/ipvlan.h b/drivers/net/ipvlan/ipvlan.h
index 695a5dc9ace3..7e0732f5ea07 100644
--- a/drivers/net/ipvlan/ipvlan.h
+++ b/drivers/net/ipvlan/ipvlan.h
@@ -23,11 +23,13 @@
 #include <linux/if_vlan.h>
 #include <linux/ip.h>
 #include <linux/inetdevice.h>
+#include <linux/netfilter.h>
 #include <net/ip.h>
 #include <net/ip6_route.h>
 #include <net/rtnetlink.h>
 #include <net/route.h>
 #include <net/addrconf.h>
+#include <net/l3mdev.h>
 
 #define IPVLAN_DRV	"ipvlan"
 #define IPV_DRV_VER	"0.1"
@@ -124,4 +126,8 @@ struct ipvl_addr *ipvlan_find_addr(const struct ipvl_dev *ipvlan,
 				   const void *iaddr, bool is_v6);
 bool ipvlan_addr_busy(struct ipvl_port *port, void *iaddr, bool is_v6);
 void ipvlan_ht_addr_del(struct ipvl_addr *addr);
+struct sk_buff *ipvlan_l3_rcv(struct net_device *dev, struct sk_buff *skb,
+			      u16 proto);
+unsigned int ipvlan_nf_input(void *priv, struct sk_buff *skb,
+			     const struct nf_hook_state *state);
 #endif /* __IPVLAN_H */
diff --git a/drivers/net/ipvlan/ipvlan_core.c b/drivers/net/ipvlan/ipvlan_core.c
index b5f9511d819e..b4e990743e1d 100644
--- a/drivers/net/ipvlan/ipvlan_core.c
+++ b/drivers/net/ipvlan/ipvlan_core.c
@@ -560,6 +560,7 @@ int ipvlan_queue_xmit(struct sk_buff *skb, struct net_device *dev)
 	case IPVLAN_MODE_L2:
 		return ipvlan_xmit_mode_l2(skb, dev);
 	case IPVLAN_MODE_L3:
+	case IPVLAN_MODE_L3S:
 		return ipvlan_xmit_mode_l3(skb, dev);
 	}
 
@@ -664,6 +665,8 @@ rx_handler_result_t ipvlan_handle_frame(struct sk_buff **pskb)
 		return ipvlan_handle_mode_l2(pskb, port);
 	case IPVLAN_MODE_L3:
 		return ipvlan_handle_mode_l3(pskb, port);
+	case IPVLAN_MODE_L3S:
+		return RX_HANDLER_PASS;
 	}
 
 	/* Should not reach here */
@@ -672,3 +675,94 @@ rx_handler_result_t ipvlan_handle_frame(struct sk_buff **pskb)
 	kfree_skb(skb);
 	return RX_HANDLER_CONSUMED;
 }
+
+static struct ipvl_addr *ipvlan_skb_to_addr(struct sk_buff *skb,
+					    struct net_device *dev)
+{
+	struct ipvl_addr *addr = NULL;
+	struct ipvl_port *port;
+	void *lyr3h;
+	int addr_type;
+
+	if (!dev || !netif_is_ipvlan_port(dev))
+		goto out;
+
+	port = ipvlan_port_get_rcu(dev);
+	if (!port || port->mode != IPVLAN_MODE_L3S)
+		goto out;
+
+	lyr3h = ipvlan_get_L3_hdr(skb, &addr_type);
+	if (!lyr3h)
+		goto out;
+
+	addr = ipvlan_addr_lookup(port, lyr3h, addr_type, true);
+out:
+	return addr;
+}
+
+struct sk_buff *ipvlan_l3_rcv(struct net_device *dev, struct sk_buff *skb,
+			      u16 proto)
+{
+	struct ipvl_addr *addr;
+	struct net_device *sdev;
+
+	addr = ipvlan_skb_to_addr(skb, dev);
+	if (!addr)
+		goto out;
+
+	sdev = addr->master->dev;
+	switch (proto) {
+	case AF_INET:
+	{
+		int err;
+		struct iphdr *ip4h = ip_hdr(skb);
+
+		err = ip_route_input_noref(skb, ip4h->daddr, ip4h->saddr,
+					   ip4h->tos, sdev);
+		if (unlikely(err))
+			goto out;
+		break;
+	}
+	case AF_INET6:
+	{
+		struct dst_entry *dst;
+		struct ipv6hdr *ip6h = ipv6_hdr(skb);
+		int flags = RT6_LOOKUP_F_HAS_SADDR;
+		struct flowi6 fl6 = {
+			.flowi6_iif   = sdev->ifindex,
+			.daddr        = ip6h->daddr,
+			.saddr        = ip6h->saddr,
+			.flowlabel    = ip6_flowinfo(ip6h),
+			.flowi6_mark  = skb->mark,
+			.flowi6_proto = ip6h->nexthdr,
+		};
+
+		skb_dst_drop(skb);
+		dst = ip6_route_input_lookup(dev_net(sdev), sdev, &fl6, flags);
+		skb_dst_set(skb, dst);
+		break;
+	}
+	default:
+		break;
+	}
+
+out:
+	return skb;
+}
+
+unsigned int ipvlan_nf_input(void *priv, struct sk_buff *skb,
+			     const struct nf_hook_state *state)
+{
+	struct ipvl_addr *addr;
+	unsigned int len;
+
+	addr = ipvlan_skb_to_addr(skb, skb->dev);
+	if (!addr)
+		goto out;
+
+	skb->dev = addr->master->dev;
+	len = skb->len + ETH_HLEN;
+	ipvlan_count_rx(addr->master, len, true, false);
+out:
+	return NF_ACCEPT;
+}
diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c
index 18b4e8c7f68a..f442eb366863 100644
--- a/drivers/net/ipvlan/ipvlan_main.c
+++ b/drivers/net/ipvlan/ipvlan_main.c
@@ -9,24 +9,87 @@
 
 #include "ipvlan.h"
 
+static u32 ipvl_nf_hook_refcnt = 0;
+
+static struct nf_hook_ops ipvl_nfops[] __read_mostly = {
+	{
+		.hook     = ipvlan_nf_input,
+		.pf       = NFPROTO_IPV4,
+		.hooknum  = NF_INET_LOCAL_IN,
+		.priority = INT_MAX,
+	},
+	{
+		.hook     = ipvlan_nf_input,
+		.pf       = NFPROTO_IPV6,
+		.hooknum  = NF_INET_LOCAL_IN,
+		.priority = INT_MAX,
+	},
+};
+
+static struct l3mdev_ops ipvl_l3mdev_ops __read_mostly = {
+	.l3mdev_l3_rcv = ipvlan_l3_rcv,
+};
+
 static void ipvlan_adjust_mtu(struct ipvl_dev *ipvlan, struct net_device *dev)
 {
 	ipvlan->dev->mtu = dev->mtu - ipvlan->mtu_adj;
 }
 
-static void ipvlan_set_port_mode(struct ipvl_port *port, u16 nval)
+static int ipvlan_register_nf_hook(void)
+{
+	int err = 0;
+
+	if (!ipvl_nf_hook_refcnt) {
+		err = _nf_register_hooks(ipvl_nfops, ARRAY_SIZE(ipvl_nfops));
+		if (!err)
+			ipvl_nf_hook_refcnt = 1;
+	} else {
+		ipvl_nf_hook_refcnt++;
+	}
+
+	return err;
+}
+
+static void ipvlan_unregister_nf_hook(void)
+{
+	WARN_ON(!ipvl_nf_hook_refcnt);
+
+	ipvl_nf_hook_refcnt--;
+	if (!ipvl_nf_hook_refcnt)
+		_nf_unregister_hooks(ipvl_nfops, ARRAY_SIZE(ipvl_nfops));
+}
+
+static int ipvlan_set_port_mode(struct ipvl_port *port, u16 nval)
 {
 	struct ipvl_dev *ipvlan;
+	struct net_device *mdev = port->dev;
+	int err = 0;
 
+	ASSERT_RTNL();
 	if (port->mode != nval) {
+		if (nval == IPVLAN_MODE_L3S) {
+			/* New mode is L3S */
+			err = ipvlan_register_nf_hook();
+			if (!err) {
+				mdev->l3mdev_ops = &ipvl_l3mdev_ops;
+				mdev->priv_flags |= IFF_L3MDEV_MASTER;
+			} else
+				return err;
+		} else if (port->mode == IPVLAN_MODE_L3S) {
+			/* Old mode was L3S */
+			mdev->priv_flags &= ~IFF_L3MDEV_MASTER;
+			ipvlan_unregister_nf_hook();
+			mdev->l3mdev_ops = NULL;
+		}
 		list_for_each_entry(ipvlan, &port->ipvlans, pnode) {
-			if (nval == IPVLAN_MODE_L3)
+			if (nval == IPVLAN_MODE_L3 || nval == IPVLAN_MODE_L3S)
 				ipvlan->dev->flags |= IFF_NOARP;
 			else
 				ipvlan->dev->flags &= ~IFF_NOARP;
 		}
 		port->mode = nval;
 	}
+	return err;
 }
 
 static int ipvlan_port_create(struct net_device *dev)
@@ -74,6 +137,11 @@ static void ipvlan_port_destroy(struct net_device *dev)
 	struct ipvl_port *port = ipvlan_port_get_rtnl(dev);
 
 	dev->priv_flags &= ~IFF_IPVLAN_MASTER;
+	if (port->mode == IPVLAN_MODE_L3S) {
+		dev->priv_flags &= ~IFF_L3MDEV_MASTER;
+		ipvlan_unregister_nf_hook();
+		dev->l3mdev_ops = NULL;
+	}
 	netdev_rx_handler_unregister(dev);
 	cancel_work_sync(&port->wq);
 	__skb_queue_purge(&port->backlog);
@@ -132,7 +200,8 @@ static int ipvlan_open(struct net_device *dev)
 	struct net_device *phy_dev = ipvlan->phy_dev;
 	struct ipvl_addr *addr;
 
-	if (ipvlan->port->mode == IPVLAN_MODE_L3)
+	if (ipvlan->port->mode == IPVLAN_MODE_L3 ||
+	    ipvlan->port->mode == IPVLAN_MODE_L3S)
 		dev->flags |= IFF_NOARP;
 	else
 		dev->flags &= ~IFF_NOARP;
@@ -372,13 +441,14 @@ static int ipvlan_nl_changelink(struct net_device *dev,
 {
 	struct ipvl_dev *ipvlan = netdev_priv(dev);
 	struct ipvl_port *port = ipvlan_port_get_rtnl(ipvlan->phy_dev);
+	int err = 0;
 
 	if (data && data[IFLA_IPVLAN_MODE]) {
 		u16 nmode = nla_get_u16(data[IFLA_IPVLAN_MODE]);
 
-		ipvlan_set_port_mode(port, nmode);
+		err = ipvlan_set_port_mode(port, nmode);
 	}
-	return 0;
+	return err;
 }
 
 static size_t ipvlan_nl_getsize(const struct net_device *dev)
@@ -473,10 +543,13 @@ static int ipvlan_link_new(struct net *src_net, struct net_device *dev,
 		unregister_netdevice(dev);
 		return err;
 	}
+	err = ipvlan_set_port_mode(port, mode);
+	if (err) {
+		unregister_netdevice(dev);
+		return err;
+	}
 
 	list_add_tail_rcu(&ipvlan->pnode, &port->ipvlans);
-	ipvlan_set_port_mode(port, mode);
-
 	netif_stacked_transfer_operstate(phy_dev, dev);
 	return 0;
 }
diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
index d13e6e15d7b5..3ea47f28e143 100644
--- a/drivers/net/macsec.c
+++ b/drivers/net/macsec.c
@@ -270,6 +270,7 @@ struct macsec_dev {
 	struct pcpu_secy_stats __percpu *stats;
 	struct list_head secys;
 	struct gro_cells gro_cells;
+	unsigned int nest_level;
 };
 
 /**
@@ -2699,6 +2700,8 @@ static netdev_tx_t macsec_start_xmit(struct sk_buff *skb,
 
 #define MACSEC_FEATURES \
 	(NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST)
+static struct lock_class_key macsec_netdev_addr_lock_key;
+
 static int macsec_dev_init(struct net_device *dev)
 {
 	struct macsec_dev *macsec = macsec_priv(dev);
@@ -2910,6 +2913,13 @@ static int macsec_get_iflink(const struct net_device *dev)
 	return macsec_priv(dev)->real_dev->ifindex;
 }
 
+
+static int macsec_get_nest_level(struct net_device *dev)
+{
+	return macsec_priv(dev)->nest_level;
+}
+
+
 static const struct net_device_ops macsec_netdev_ops = {
 	.ndo_init		= macsec_dev_init,
 	.ndo_uninit		= macsec_dev_uninit,
@@ -2923,6 +2933,7 @@ static const struct net_device_ops macsec_netdev_ops = {
 	.ndo_start_xmit		= macsec_start_xmit,
 	.ndo_get_stats64	= macsec_get_stats64,
 	.ndo_get_iflink		= macsec_get_iflink,
+	.ndo_get_lock_subclass  = macsec_get_nest_level,
 };
 
 static const struct device_type macsec_type = {
@@ -2962,6 +2973,7 @@ static void macsec_setup(struct net_device *dev)
 	dev->priv_flags |= IFF_NO_QUEUE;
 	dev->netdev_ops = &macsec_netdev_ops;
 	dev->destructor = macsec_free_netdev;
+	SET_NETDEV_DEVTYPE(dev, &macsec_type);
 
 	eth_zero_addr(dev->broadcast);
 }
@@ -3047,22 +3059,31 @@ static void macsec_del_dev(struct macsec_dev *macsec)
 	}
 }
 
+static void macsec_common_dellink(struct net_device *dev, struct list_head *head)
+{
+	struct macsec_dev *macsec = macsec_priv(dev);
+	struct net_device *real_dev = macsec->real_dev;
+
+	unregister_netdevice_queue(dev, head);
+	list_del_rcu(&macsec->secys);
+	macsec_del_dev(macsec);
+	netdev_upper_dev_unlink(real_dev, dev);
+
+	macsec_generation++;
+}
+
 static void macsec_dellink(struct net_device *dev, struct list_head *head)
 {
 	struct macsec_dev *macsec = macsec_priv(dev);
 	struct net_device *real_dev = macsec->real_dev;
 	struct macsec_rxh_data *rxd = macsec_data_rtnl(real_dev);
 
-	macsec_generation++;
+	macsec_common_dellink(dev, head);
 
-	unregister_netdevice_queue(dev, head);
-	list_del_rcu(&macsec->secys);
 	if (list_empty(&rxd->secys)) {
 		netdev_rx_handler_unregister(real_dev);
 		kfree(rxd);
 	}
-
-	macsec_del_dev(macsec);
 }
 
 static int register_macsec_dev(struct net_device *real_dev,
@@ -3181,6 +3202,16 @@ static int macsec_newlink(struct net *net, struct net_device *dev,
 
 	dev_hold(real_dev);
 
+	macsec->nest_level = dev_get_nest_level(real_dev) + 1;
+	netdev_lockdep_set_classes(dev);
+	lockdep_set_class_and_subclass(&dev->addr_list_lock,
+				       &macsec_netdev_addr_lock_key,
+				       macsec_get_nest_level(dev));
+
+	err = netdev_upper_dev_link(real_dev, dev);
+	if (err < 0)
+		goto unregister;
+
 	/* need to be already registered so that ->init has run and
 	 * the MAC addr is set
 	 */
@@ -3193,12 +3224,12 @@ static int macsec_newlink(struct net *net, struct net_device *dev,
 
 	if (rx_handler && sci_exists(real_dev, sci)) {
 		err = -EBUSY;
-		goto unregister;
+		goto unlink;
 	}
 
 	err = macsec_add_dev(dev, sci, icv_len);
 	if (err)
-		goto unregister;
+		goto unlink;
 
 	if (data)
 		macsec_changelink_common(dev, data);
@@ -3213,6 +3244,8 @@ static int macsec_newlink(struct net *net, struct net_device *dev,
 
 del_dev:
 	macsec_del_dev(macsec);
+unlink:
+	netdev_upper_dev_unlink(real_dev, dev);
 unregister:
 	unregister_netdevice(dev);
 	return err;
@@ -3382,8 +3415,12 @@ static int macsec_notify(struct notifier_block *this, unsigned long event,
 
 		rxd = macsec_data_rtnl(real_dev);
 		list_for_each_entry_safe(m, n, &rxd->secys, secys) {
-			macsec_dellink(m->secy.netdev, &head);
+			macsec_common_dellink(m->secy.netdev, &head);
 		}
+
+		netdev_rx_handler_unregister(real_dev);
+		kfree(rxd);
+
 		unregister_netdevice_many(&head);
 		break;
 	}
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index cd9b53834bf6..3234fcdea317 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -1315,7 +1315,7 @@ int macvlan_common_newlink(struct net *src_net, struct net_device *dev,
 	vlan->dev      = dev;
 	vlan->port     = port;
 	vlan->set_features = MACVLAN_FEATURES;
-	vlan->nest_level = dev_get_nest_level(lowerdev, netif_is_macvlan) + 1;
+	vlan->nest_level = dev_get_nest_level(lowerdev) + 1;
 
 	vlan->mode     = MACVLAN_MODE_VEPA;
 	if (data && data[IFLA_MACVLAN_MODE])
diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c
index a38c0dac514b..070e3290aa6e 100644
--- a/drivers/net/macvtap.c
+++ b/drivers/net/macvtap.c
@@ -275,7 +275,6 @@ static void macvtap_put_queue(struct macvtap_queue *q)
 	rtnl_unlock();
 
 	synchronize_rcu();
-	skb_array_cleanup(&q->skb_array);
 	sock_put(&q->sk);
 }
 
@@ -533,10 +532,8 @@ static void macvtap_sock_write_space(struct sock *sk)
 static void macvtap_sock_destruct(struct sock *sk)
 {
 	struct macvtap_queue *q = container_of(sk, struct macvtap_queue, sk);
-	struct sk_buff *skb;
 
-	while ((skb = skb_array_consume(&q->skb_array)) != NULL)
-		kfree_skb(skb);
+	skb_array_cleanup(&q->skb_array);
 }
 
 static int macvtap_open(struct inode *inode, struct file *file)
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index 47a64342cc16..2651c8d8de2f 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -15,152 +15,219 @@ if PHYLIB
 config SWPHY
 	bool
 
-comment "MII PHY device drivers"
-
-config AQUANTIA_PHY
-        tristate "Drivers for the Aquantia PHYs"
-        ---help---
-          Currently supports the Aquantia AQ1202, AQ2104, AQR105, AQR405
+comment "MDIO bus device drivers"
 
-config AT803X_PHY
-	tristate "Drivers for Atheros AT803X PHYs"
-	---help---
-	  Currently supports the AT8030 and AT8035 model
+config MDIO_BCM_IPROC
+	tristate "Broadcom iProc MDIO bus controller"
+	depends on ARCH_BCM_IPROC || COMPILE_TEST
+	depends on HAS_IOMEM && OF_MDIO
+	help
+	  This module provides a driver for the MDIO busses found in the
+	  Broadcom iProc SoC's.
 
-config AMD_PHY
-	tristate "Drivers for the AMD PHYs"
-	---help---
-	  Currently supports the am79c874
+config MDIO_BCM_UNIMAC
+	tristate "Broadcom UniMAC MDIO bus controller"
+	depends on HAS_IOMEM
+	help
+	  This module provides a driver for the Broadcom UniMAC MDIO busses.
+	  This hardware can be found in the Broadcom GENET Ethernet MAC
+	  controllers as well as some Broadcom Ethernet switches such as the
+	  Starfighter 2 switches.
 
-config MARVELL_PHY
-	tristate "Drivers for Marvell PHYs"
-	---help---
-	  Currently has a driver for the 88E1011S
-	
-config DAVICOM_PHY
-	tristate "Drivers for Davicom PHYs"
-	---help---
-	  Currently supports dm9161e and dm9131
+config MDIO_BITBANG
+	tristate "Bitbanged MDIO buses"
+	help
+	  This module implements the MDIO bus protocol in software,
+	  for use by low level drivers that export the ability to
+	  drive the relevant pins.
 
-config QSEMI_PHY
-	tristate "Drivers for Quality Semiconductor PHYs"
-	---help---
-	  Currently supports the qs6612
+	  If in doubt, say N.
 
-config LXT_PHY
-	tristate "Drivers for the Intel LXT PHYs"
-	---help---
-	  Currently supports the lxt970, lxt971
+config MDIO_BUS_MUX
+	tristate
+	depends on OF_MDIO
+	help
+	  This module provides a driver framework for MDIO bus
+	  multiplexers which connect one of several child MDIO busses
+	  to a parent bus.  Switching between child busses is done by
+	  device specific drivers.
 
-config CICADA_PHY
-	tristate "Drivers for the Cicada PHYs"
-	---help---
-	  Currently supports the cis8204
+config MDIO_BUS_MUX_BCM_IPROC
+	tristate "Broadcom iProc based MDIO bus multiplexers"
+	depends on OF && OF_MDIO && (ARCH_BCM_IPROC || COMPILE_TEST)
+	select MDIO_BUS_MUX
+	default ARCH_BCM_IPROC
+	help
+	  This module provides a driver for MDIO bus multiplexers found in
+	  iProc based Broadcom SoCs. This multiplexer connects one of several
+	  child MDIO bus to a parent bus. Buses could be internal as well as
+	  external and selection logic lies inside the same multiplexer.
 
-config VITESSE_PHY
-        tristate "Drivers for the Vitesse PHYs"
-        ---help---
-          Currently supports the vsc8244
+config MDIO_BUS_MUX_GPIO
+	tristate "GPIO controlled MDIO bus multiplexers"
+	depends on OF_GPIO && OF_MDIO
+	select MDIO_BUS_MUX
+	help
+	  This module provides a driver for MDIO bus multiplexers that
+	  are controlled via GPIO lines.  The multiplexer connects one of
+	  several child MDIO busses to a parent bus.  Child bus
+	  selection is under the control of GPIO lines.
 
-config TERANETICS_PHY
-        tristate "Drivers for the Teranetics PHYs"
-        ---help---
-          Currently supports the Teranetics TN2020
+config MDIO_BUS_MUX_MMIOREG
+	tristate "MMIO device-controlled MDIO bus multiplexers"
+	depends on OF_MDIO && HAS_IOMEM
+	select MDIO_BUS_MUX
+	help
+	  This module provides a driver for MDIO bus multiplexers that
+	  are controlled via a simple memory-mapped device, like an FPGA.
+	  The multiplexer connects one of several child MDIO busses to a
+	  parent bus.  Child bus selection is under the control of one of
+	  the FPGA's registers.
 
-config SMSC_PHY
-	tristate "Drivers for SMSC PHYs"
-	---help---
-	  Currently supports the LAN83C185, LAN8187 and LAN8700 PHYs
+	  Currently, only 8-bit registers are supported.
 
-config BCM_NET_PHYLIB
+config MDIO_CAVIUM
 	tristate
 
-config BROADCOM_PHY
-	tristate "Drivers for Broadcom PHYs"
-	select BCM_NET_PHYLIB
+config MDIO_GPIO
+	tristate "GPIO lib-based bitbanged MDIO buses"
+	depends on MDIO_BITBANG && GPIOLIB
 	---help---
-	  Currently supports the BCM5411, BCM5421, BCM5461, BCM54616S, BCM5464,
-	  BCM5481 and BCM5482 PHYs.
+	  Supports GPIO lib-based MDIO busses.
 
-config BCM_CYGNUS_PHY
-	tristate "Drivers for Broadcom Cygnus SoC internal PHY"
-	depends on ARCH_BCM_CYGNUS || COMPILE_TEST
-	depends on MDIO_BCM_IPROC
-	select BCM_NET_PHYLIB
+	  To compile this driver as a module, choose M here: the module
+	  will be called mdio-gpio.
+
+config MDIO_HISI_FEMAC
+	tristate "Hisilicon FEMAC MDIO bus controller"
+	depends on HAS_IOMEM && OF_MDIO
+	help
+	  This module provides a driver for the MDIO busses found in the
+	  Hisilicon SoC that have an Fast Ethernet MAC.
+
+config MDIO_MOXART
+        tristate "MOXA ART MDIO interface support"
+        depends on ARCH_MOXART
+        help
+          This driver supports the MDIO interface found in the network
+          interface units of the MOXA ART SoC
+
+config MDIO_OCTEON
+	tristate "Octeon and some ThunderX SOCs MDIO buses"
+	depends on 64BIT
+	depends on HAS_IOMEM
+	select MDIO_CAVIUM
+	help
+	  This module provides a driver for the Octeon and ThunderX MDIO
+	  buses. It is required by the Octeon and ThunderX ethernet device
+	  drivers on some systems.
+
+config MDIO_SUN4I
+	tristate "Allwinner sun4i MDIO interface support"
+	depends on ARCH_SUNXI
+	help
+	  This driver supports the MDIO interface found in the network
+	  interface units of the Allwinner SoC that have an EMAC (A10,
+	  A12, A10s, etc.)
+
+config MDIO_THUNDER
+	tristate "ThunderX SOCs MDIO buses"
+	depends on 64BIT
+	depends on PCI
+	select MDIO_CAVIUM
+	help
+	  This driver supports the MDIO interfaces found on Cavium
+	  ThunderX SoCs when the MDIO bus device appears as a PCI
+	  device.
+
+config MDIO_XGENE
+	tristate "APM X-Gene SoC MDIO bus controller"
+	depends on ARCH_XGENE || COMPILE_TEST
+	help
+	  This module provides a driver for the MDIO busses found in the
+	  APM X-Gene SoC's.
+
+comment "MII PHY device drivers"
+
+config AMD_PHY
+	tristate "AMD PHYs"
 	---help---
-	  This PHY driver is for the 1G internal PHYs of the Broadcom
-	  Cygnus Family SoC.
+	  Currently supports the am79c874
 
-	  Currently supports internal PHY's used in the BCM11300,
-	  BCM11320, BCM11350, BCM11360, BCM58300, BCM58302,
-	  BCM58303 & BCM58305 Broadcom Cygnus SoCs.
+config AQUANTIA_PHY
+        tristate "Aquantia PHYs"
+        ---help---
+          Currently supports the Aquantia AQ1202, AQ2104, AQR105, AQR405
+
+config AT803X_PHY
+	tristate "AT803X PHYs"
+	---help---
+	  Currently supports the AT8030 and AT8035 model
 
 config BCM63XX_PHY
-	tristate "Drivers for Broadcom 63xx SOCs internal PHY"
+	tristate "Broadcom 63xx SOCs internal PHY"
 	depends on BCM63XX
 	select BCM_NET_PHYLIB
 	---help---
 	  Currently supports the 6348 and 6358 PHYs.
 
 config BCM7XXX_PHY
-	tristate "Drivers for Broadcom 7xxx SOCs internal PHYs"
+	tristate "Broadcom 7xxx SOCs internal PHYs"
 	select BCM_NET_PHYLIB
 	---help---
 	  Currently supports the BCM7366, BCM7439, BCM7445, and
 	  40nm and 65nm generation of BCM7xxx Set Top Box SoCs.
 
 config BCM87XX_PHY
-	tristate "Driver for Broadcom BCM8706 and BCM8727 PHYs"
+	tristate "Broadcom BCM8706 and BCM8727 PHYs"
 	help
 	  Currently supports the BCM8706 and BCM8727 10G Ethernet PHYs.
 
-config ICPLUS_PHY
-	tristate "Drivers for ICPlus PHYs"
+config BCM_CYGNUS_PHY
+	tristate "Broadcom Cygnus SoC internal PHY"
+	depends on ARCH_BCM_CYGNUS || COMPILE_TEST
+	depends on MDIO_BCM_IPROC
+	select BCM_NET_PHYLIB
 	---help---
-	  Currently supports the IP175C and IP1001 PHYs.
+	  This PHY driver is for the 1G internal PHYs of the Broadcom
+	  Cygnus Family SoC.
 
-config REALTEK_PHY
-	tristate "Drivers for Realtek PHYs"
-	---help---
-	  Supports the Realtek 821x PHY.
+	  Currently supports internal PHY's used in the BCM11300,
+	  BCM11320, BCM11350, BCM11360, BCM58300, BCM58302,
+	  BCM58303 & BCM58305 Broadcom Cygnus SoCs.
 
-config NATIONAL_PHY
-	tristate "Drivers for National Semiconductor PHYs"
-	---help---
-	  Currently supports the DP83865 PHY.
+config BCM_NET_PHYLIB
+	tristate
 
-config STE10XP
-	tristate "Driver for STMicroelectronics STe10Xp PHYs"
+config BROADCOM_PHY
+	tristate "Broadcom PHYs"
+	select BCM_NET_PHYLIB
 	---help---
-	  This is the driver for the STe100p and STe101p PHYs.
+	  Currently supports the BCM5411, BCM5421, BCM5461, BCM54616S, BCM5464,
+	  BCM5481 and BCM5482 PHYs.
 
-config LSI_ET1011C_PHY
-	tristate "Driver for LSI ET1011C PHY"
+config CICADA_PHY
+	tristate "Cicada PHYs"
 	---help---
-	  Supports the LSI ET1011C PHY.
+	  Currently supports the cis8204
 
-config MICREL_PHY
-	tristate "Driver for Micrel PHYs"
+config DAVICOM_PHY
+	tristate "Davicom PHYs"
 	---help---
-	  Supports the KSZ9021, VSC8201, KS8001 PHYs.
+	  Currently supports dm9161e and dm9131
 
 config DP83848_PHY
-	tristate "Driver for Texas Instruments DP83848 PHY"
+	tristate "Texas Instruments DP83848 PHY"
 	---help---
 	  Supports the DP83848 PHY.
 
 config DP83867_PHY
-	tristate "Drivers for Texas Instruments DP83867 Gigabit PHY"
+	tristate "Texas Instruments DP83867 Gigabit PHY"
 	---help---
 	  Currently supports the DP83867 PHY.
 
-config MICROCHIP_PHY
-	tristate "Drivers for Microchip PHYs"
-	help
-	  Supports the LAN88XX PHYs.
-
 config FIXED_PHY
-	tristate "Driver for MDIO Bus/PHY emulation with fixed speed/link PHYs"
+	tristate "MDIO Bus/PHY emulation with fixed speed/link PHYs"
 	depends on PHYLIB
 	select SWPHY
 	---help---
@@ -169,143 +236,90 @@ config FIXED_PHY
 
 	  Currently tested with mpc866ads and mpc8349e-mitx.
 
-config MDIO_BITBANG
-	tristate "Support for bitbanged MDIO buses"
-	help
-	  This module implements the MDIO bus protocol in software,
-	  for use by low level drivers that export the ability to
-	  drive the relevant pins.
-
-	  If in doubt, say N.
-
-config MDIO_GPIO
-	tristate "Support for GPIO lib-based bitbanged MDIO buses"
-	depends on MDIO_BITBANG && GPIOLIB
+config ICPLUS_PHY
+	tristate "ICPlus PHYs"
 	---help---
-	  Supports GPIO lib-based MDIO busses.
-
-	  To compile this driver as a module, choose M here: the module
-	  will be called mdio-gpio.
-
-config MDIO_CAVIUM
-	tristate
-
-config MDIO_OCTEON
-	tristate "Support for MDIO buses on Octeon and some ThunderX SOCs"
-	depends on 64BIT
-	depends on HAS_IOMEM
-	select MDIO_CAVIUM
-	help
-	  This module provides a driver for the Octeon and ThunderX MDIO
-	  buses. It is required by the Octeon and ThunderX ethernet device
-	  drivers on some systems.
+	  Currently supports the IP175C and IP1001 PHYs.
 
-config MDIO_THUNDER
-	tristate "Support for MDIO buses on ThunderX SOCs"
-	depends on 64BIT
-	depends on PCI
-	select MDIO_CAVIUM
-	help
-	  This driver supports the MDIO interfaces found on Cavium
-	  ThunderX SoCs when the MDIO bus device appears as a PCI
-	  device.
+config INTEL_XWAY_PHY
+	tristate "Intel XWAY PHYs"
+	---help---
+	  Supports the Intel XWAY (former Lantiq) 11G and 22E PHYs.
+	  These PHYs are marked as standalone chips under the names
+	  PEF 7061, PEF 7071 and PEF 7072 or integrated into the Intel
+	  SoCs xRX200, xRX300, xRX330, xRX350 and xRX550.
 
+config LSI_ET1011C_PHY
+	tristate "LSI ET1011C PHY"
+	---help---
+	  Supports the LSI ET1011C PHY.
 
-config MDIO_SUN4I
-	tristate "Allwinner sun4i MDIO interface support"
-	depends on ARCH_SUNXI
-	help
-	  This driver supports the MDIO interface found in the network
-	  interface units of the Allwinner SoC that have an EMAC (A10,
-	  A12, A10s, etc.)
+config LXT_PHY
+	tristate "Intel LXT PHYs"
+	---help---
+	  Currently supports the lxt970, lxt971
 
-config MDIO_MOXART
-        tristate "MOXA ART MDIO interface support"
-        depends on ARCH_MOXART
-        help
-          This driver supports the MDIO interface found in the network
-          interface units of the MOXA ART SoC
+config MARVELL_PHY
+	tristate "Marvell PHYs"
+	---help---
+	  Currently has a driver for the 88E1011S
 
-config MDIO_BUS_MUX
-	tristate
-	depends on OF_MDIO
-	help
-	  This module provides a driver framework for MDIO bus
-	  multiplexers which connect one of several child MDIO busses
-	  to a parent bus.  Switching between child busses is done by
-	  device specific drivers.
+config MICREL_PHY
+	tristate "Micrel PHYs"
+	---help---
+	  Supports the KSZ9021, VSC8201, KS8001 PHYs.
 
-config MDIO_BUS_MUX_GPIO
-	tristate "Support for GPIO controlled MDIO bus multiplexers"
-	depends on OF_GPIO && OF_MDIO
-	select MDIO_BUS_MUX
+config MICROCHIP_PHY
+	tristate "Microchip PHYs"
 	help
-	  This module provides a driver for MDIO bus multiplexers that
-	  are controlled via GPIO lines.  The multiplexer connects one of
-	  several child MDIO busses to a parent bus.  Child bus
-	  selection is under the control of GPIO lines.
+	  Supports the LAN88XX PHYs.
 
-config MDIO_BUS_MUX_MMIOREG
-	tristate "Support for MMIO device-controlled MDIO bus multiplexers"
-	depends on OF_MDIO && HAS_IOMEM
-	select MDIO_BUS_MUX
-	help
-	  This module provides a driver for MDIO bus multiplexers that
-	  are controlled via a simple memory-mapped device, like an FPGA.
-	  The multiplexer connects one of several child MDIO busses to a
-	  parent bus.  Child bus selection is under the control of one of
-	  the FPGA's registers.
+config MICROSEMI_PHY
+	tristate "Microsemi PHYs"
+	---help---
+	  Currently supports the VSC8531 and VSC8541 PHYs
 
-	  Currently, only 8-bit registers are supported.
+config NATIONAL_PHY
+	tristate "National Semiconductor PHYs"
+	---help---
+	  Currently supports the DP83865 PHY.
 
-config MDIO_BUS_MUX_BCM_IPROC
-	tristate "Support for iProc based MDIO bus multiplexers"
-	depends on OF && OF_MDIO && (ARCH_BCM_IPROC || COMPILE_TEST)
-	select MDIO_BUS_MUX
-	default ARCH_BCM_IPROC
-	help
-	  This module provides a driver for MDIO bus multiplexers found in
-	  iProc based Broadcom SoCs. This multiplexer connects one of several
-	  child MDIO bus to a parent bus. Buses could be internal as well as
-	  external and selection logic lies inside the same multiplexer.
+config QSEMI_PHY
+	tristate "Quality Semiconductor PHYs"
+	---help---
+	  Currently supports the qs6612
 
-config MDIO_BCM_UNIMAC
-	tristate "Broadcom UniMAC MDIO bus controller"
-	depends on HAS_IOMEM
-	help
-	  This module provides a driver for the Broadcom UniMAC MDIO busses.
-	  This hardware can be found in the Broadcom GENET Ethernet MAC
-	  controllers as well as some Broadcom Ethernet switches such as the
-	  Starfighter 2 switches.
+config REALTEK_PHY
+	tristate "Realtek PHYs"
+	---help---
+	  Supports the Realtek 821x PHY.
 
-config MDIO_BCM_IPROC
-	tristate "Broadcom iProc MDIO bus controller"
-	depends on ARCH_BCM_IPROC || COMPILE_TEST
-	depends on HAS_IOMEM && OF_MDIO
-	help
-	  This module provides a driver for the MDIO busses found in the
-	  Broadcom iProc SoC's.
+config SMSC_PHY
+	tristate "SMSC PHYs"
+	---help---
+	  Currently supports the LAN83C185, LAN8187 and LAN8700 PHYs
 
-config INTEL_XWAY_PHY
-	tristate "Driver for Intel XWAY PHYs"
+config STE10XP
+	tristate "STMicroelectronics STe10Xp PHYs"
 	---help---
-	  Supports the Intel XWAY (former Lantiq) 11G and 22E PHYs.
-	  These PHYs are marked as standalone chips under the names
-	  PEF 7061, PEF 7071 and PEF 7072 or integrated into the Intel
-	  SoCs xRX200, xRX300, xRX330, xRX350 and xRX550.
+	  This is the driver for the STe100p and STe101p PHYs.
 
-config MDIO_HISI_FEMAC
-	tristate "Hisilicon FEMAC MDIO bus controller"
-	depends on HAS_IOMEM && OF_MDIO
-	help
-	  This module provides a driver for the MDIO busses found in the
-	  Hisilicon SoC that have an Fast Ethernet MAC.
+config TERANETICS_PHY
+        tristate "Teranetics PHYs"
+        ---help---
+          Currently supports the Teranetics TN2020
 
-config MDIO_XGENE
-	tristate "APM X-Gene SoC MDIO bus controller"
-	help
-	  This module provides a driver for the MDIO busses found in the
-	  APM X-Gene SoC's.
+config VITESSE_PHY
+        tristate "Vitesse PHYs"
+        ---help---
+          Currently supports the vsc8244
+
+config XILINX_GMII2RGMII
+       tristate "Xilinx GMII2RGMII converter driver"
+       ---help---
+         This driver support xilinx GMII to RGMII IP core it provides
+         the Reduced Gigabit Media Independent Interface(RGMII) between
+         Ethernet physical media devices and the Gigabit Ethernet controller.
 
 endif # PHYLIB
 
diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile
index 534dfa74d5a2..e58667d111e7 100644
--- a/drivers/net/phy/Makefile
+++ b/drivers/net/phy/Makefile
@@ -1,51 +1,55 @@
-# Makefile for Linux PHY drivers
+# Makefile for Linux PHY drivers and MDIO bus drivers
 
 libphy-y			:= phy.o phy_device.o mdio_bus.o mdio_device.o
 libphy-$(CONFIG_SWPHY)		+= swphy.o
 
 obj-$(CONFIG_PHYLIB)		+= libphy.o
+
+obj-$(CONFIG_MDIO_BCM_IPROC)	+= mdio-bcm-iproc.o
+obj-$(CONFIG_MDIO_BCM_UNIMAC)	+= mdio-bcm-unimac.o
+obj-$(CONFIG_MDIO_BITBANG)	+= mdio-bitbang.o
+obj-$(CONFIG_MDIO_BUS_MUX)	+= mdio-mux.o
+obj-$(CONFIG_MDIO_BUS_MUX_BCM_IPROC)	+= mdio-mux-bcm-iproc.o
+obj-$(CONFIG_MDIO_BUS_MUX_GPIO)	+= mdio-mux-gpio.o
+obj-$(CONFIG_MDIO_BUS_MUX_MMIOREG) += mdio-mux-mmioreg.o
+obj-$(CONFIG_MDIO_CAVIUM)	+= mdio-cavium.o
+obj-$(CONFIG_MDIO_GPIO)		+= mdio-gpio.o
+obj-$(CONFIG_MDIO_HISI_FEMAC)	+= mdio-hisi-femac.o
+obj-$(CONFIG_MDIO_MOXART)	+= mdio-moxart.o
+obj-$(CONFIG_MDIO_OCTEON)	+= mdio-octeon.o
+obj-$(CONFIG_MDIO_SUN4I)	+= mdio-sun4i.o
+obj-$(CONFIG_MDIO_THUNDER)	+= mdio-thunder.o
+obj-$(CONFIG_MDIO_XGENE)	+= mdio-xgene.o
+
+obj-$(CONFIG_AMD_PHY)		+= amd.o
 obj-$(CONFIG_AQUANTIA_PHY)	+= aquantia.o
-obj-$(CONFIG_MARVELL_PHY)	+= marvell.o
-obj-$(CONFIG_DAVICOM_PHY)	+= davicom.o
-obj-$(CONFIG_CICADA_PHY)	+= cicada.o
-obj-$(CONFIG_LXT_PHY)		+= lxt.o
-obj-$(CONFIG_QSEMI_PHY)		+= qsemi.o
-obj-$(CONFIG_SMSC_PHY)		+= smsc.o
-obj-$(CONFIG_TERANETICS_PHY)	+= teranetics.o
-obj-$(CONFIG_VITESSE_PHY)	+= vitesse.o
-obj-$(CONFIG_BCM_NET_PHYLIB)	+= bcm-phy-lib.o
-obj-$(CONFIG_BROADCOM_PHY)	+= broadcom.o
+obj-$(CONFIG_AT803X_PHY)	+= at803x.o
 obj-$(CONFIG_BCM63XX_PHY)	+= bcm63xx.o
 obj-$(CONFIG_BCM7XXX_PHY)	+= bcm7xxx.o
 obj-$(CONFIG_BCM87XX_PHY)	+= bcm87xx.o
 obj-$(CONFIG_BCM_CYGNUS_PHY)	+= bcm-cygnus.o
-obj-$(CONFIG_ICPLUS_PHY)	+= icplus.o
-obj-$(CONFIG_REALTEK_PHY)	+= realtek.o
-obj-$(CONFIG_LSI_ET1011C_PHY)	+= et1011c.o
-obj-$(CONFIG_FIXED_PHY)		+= fixed_phy.o
-obj-$(CONFIG_MDIO_BITBANG)	+= mdio-bitbang.o
-obj-$(CONFIG_MDIO_GPIO)		+= mdio-gpio.o
-obj-$(CONFIG_NATIONAL_PHY)	+= national.o
+obj-$(CONFIG_BCM_NET_PHYLIB)	+= bcm-phy-lib.o
+obj-$(CONFIG_BROADCOM_PHY)	+= broadcom.o
+obj-$(CONFIG_CICADA_PHY)	+= cicada.o
+obj-$(CONFIG_DAVICOM_PHY)	+= davicom.o
 obj-$(CONFIG_DP83640_PHY)	+= dp83640.o
 obj-$(CONFIG_DP83848_PHY)	+= dp83848.o
 obj-$(CONFIG_DP83867_PHY)	+= dp83867.o
-obj-$(CONFIG_STE10XP)		+= ste10Xp.o
-obj-$(CONFIG_MICREL_PHY)	+= micrel.o
-obj-$(CONFIG_MDIO_OCTEON)	+= mdio-octeon.o
-obj-$(CONFIG_MDIO_THUNDER)	+= mdio-thunder.o
-obj-$(CONFIG_MDIO_CAVIUM)	+= mdio-cavium.o
+obj-$(CONFIG_FIXED_PHY)		+= fixed_phy.o
+obj-$(CONFIG_ICPLUS_PHY)	+= icplus.o
+obj-$(CONFIG_INTEL_XWAY_PHY)	+= intel-xway.o
+obj-$(CONFIG_LSI_ET1011C_PHY)	+= et1011c.o
+obj-$(CONFIG_LXT_PHY)		+= lxt.o
+obj-$(CONFIG_MARVELL_PHY)	+= marvell.o
 obj-$(CONFIG_MICREL_KS8995MA)	+= spi_ks8995.o
-obj-$(CONFIG_AT803X_PHY)	+= at803x.o
-obj-$(CONFIG_AMD_PHY)		+= amd.o
-obj-$(CONFIG_MDIO_BUS_MUX)	+= mdio-mux.o
-obj-$(CONFIG_MDIO_BUS_MUX_GPIO)	+= mdio-mux-gpio.o
-obj-$(CONFIG_MDIO_BUS_MUX_MMIOREG) += mdio-mux-mmioreg.o
-obj-$(CONFIG_MDIO_BUS_MUX_BCM_IPROC)	+= mdio-mux-bcm-iproc.o
-obj-$(CONFIG_MDIO_SUN4I)	+= mdio-sun4i.o
-obj-$(CONFIG_MDIO_MOXART)	+= mdio-moxart.o
-obj-$(CONFIG_MDIO_BCM_UNIMAC)	+= mdio-bcm-unimac.o
+obj-$(CONFIG_MICREL_PHY)	+= micrel.o
 obj-$(CONFIG_MICROCHIP_PHY)	+= microchip.o
-obj-$(CONFIG_MDIO_BCM_IPROC)	+= mdio-bcm-iproc.o
-obj-$(CONFIG_INTEL_XWAY_PHY)	+= intel-xway.o
-obj-$(CONFIG_MDIO_HISI_FEMAC)	+= mdio-hisi-femac.o
-obj-$(CONFIG_MDIO_XGENE)	+= mdio-xgene.o
+obj-$(CONFIG_MICROSEMI_PHY)	+= mscc.o
+obj-$(CONFIG_NATIONAL_PHY)	+= national.o
+obj-$(CONFIG_QSEMI_PHY)		+= qsemi.o
+obj-$(CONFIG_REALTEK_PHY)	+= realtek.o
+obj-$(CONFIG_SMSC_PHY)		+= smsc.o
+obj-$(CONFIG_STE10XP)		+= ste10Xp.o
+obj-$(CONFIG_TERANETICS_PHY)	+= teranetics.o
+obj-$(CONFIG_VITESSE_PHY)	+= vitesse.o
+obj-$(CONFIG_XILINX_GMII2RGMII) += xilinx_gmii2rgmii.o
diff --git a/drivers/net/phy/mdio-xgene.c b/drivers/net/phy/mdio-xgene.c
index 775674808249..92af182951be 100644
--- a/drivers/net/phy/mdio-xgene.c
+++ b/drivers/net/phy/mdio-xgene.c
@@ -424,10 +424,8 @@ static int xgene_mdio_remove(struct platform_device *pdev)
 	mdiobus_unregister(mdio_bus);
 	mdiobus_free(mdio_bus);
 
-	if (dev->of_node) {
-		if (IS_ERR(pdata->clk))
-			clk_disable_unprepare(pdata->clk);
-	}
+	if (dev->of_node)
+		clk_disable_unprepare(pdata->clk);
 
 	return 0;
 }
diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
index 1882d9828c99..081df68d2ce1 100644
--- a/drivers/net/phy/micrel.c
+++ b/drivers/net/phy/micrel.c
@@ -439,6 +439,10 @@ static int ksz9021_config_init(struct phy_device *phydev)
 #define MII_KSZ9031RN_TX_DATA_PAD_SKEW	6
 #define MII_KSZ9031RN_CLK_PAD_SKEW	8
 
+/* MMD Address 0x1C */
+#define MII_KSZ9031RN_EDPD		0x23
+#define MII_KSZ9031RN_EDPD_ENABLE	BIT(0)
+
 static int ksz9031_extended_write(struct phy_device *phydev,
 				  u8 mode, u32 dev_addr, u32 regnum, u16 val)
 {
@@ -510,6 +514,18 @@ static int ksz9031_center_flp_timing(struct phy_device *phydev)
 	return genphy_restart_aneg(phydev);
 }
 
+/* Enable energy-detect power-down mode */
+static int ksz9031_enable_edpd(struct phy_device *phydev)
+{
+	int reg;
+
+	reg = ksz9031_extended_read(phydev, OP_DATA, 0x1C, MII_KSZ9031RN_EDPD);
+	if (reg < 0)
+		return reg;
+	return ksz9031_extended_write(phydev, OP_DATA, 0x1C, MII_KSZ9031RN_EDPD,
+				      reg | MII_KSZ9031RN_EDPD_ENABLE);
+}
+
 static int ksz9031_config_init(struct phy_device *phydev)
 {
 	const struct device *dev = &phydev->mdio.dev;
@@ -525,6 +541,11 @@ static int ksz9031_config_init(struct phy_device *phydev)
 	};
 	static const char *control_skews[2] = {"txen-skew-ps", "rxdv-skew-ps"};
 	const struct device *dev_walker;
+	int result;
+
+	result = ksz9031_enable_edpd(phydev);
+	if (result < 0)
+		return result;
 
 	/* The Micrel driver has a deprecated option to place phy OF
 	 * properties in the MAC node. Walk up the tree of devices to
@@ -677,17 +698,28 @@ static void kszphy_get_stats(struct phy_device *phydev,
 		data[i] = kszphy_get_stat(phydev, i);
 }
 
-static int kszphy_resume(struct phy_device *phydev)
+static int kszphy_suspend(struct phy_device *phydev)
 {
-	int value;
+	/* Disable PHY Interrupts */
+	if (phy_interrupt_is_valid(phydev)) {
+		phydev->interrupts = PHY_INTERRUPT_DISABLED;
+		if (phydev->drv->config_intr)
+			phydev->drv->config_intr(phydev);
+	}
 
-	mutex_lock(&phydev->lock);
+	return genphy_suspend(phydev);
+}
 
-	value = phy_read(phydev, MII_BMCR);
-	phy_write(phydev, MII_BMCR, value & ~BMCR_PDOWN);
+static int kszphy_resume(struct phy_device *phydev)
+{
+	genphy_resume(phydev);
 
-	kszphy_config_intr(phydev);
-	mutex_unlock(&phydev->lock);
+	/* Enable PHY Interrupts */
+	if (phy_interrupt_is_valid(phydev)) {
+		phydev->interrupts = PHY_INTERRUPT_ENABLED;
+		if (phydev->drv->config_intr)
+			phydev->drv->config_intr(phydev);
+	}
 
 	return 0;
 }
@@ -900,7 +932,7 @@ static struct phy_driver ksphy_driver[] = {
 	.get_sset_count = kszphy_get_sset_count,
 	.get_strings	= kszphy_get_strings,
 	.get_stats	= kszphy_get_stats,
-	.suspend	= genphy_suspend,
+	.suspend	= kszphy_suspend,
 	.resume		= kszphy_resume,
 }, {
 	.phy_id		= PHY_ID_KSZ8061,
@@ -953,7 +985,7 @@ static struct phy_driver ksphy_driver[] = {
 	.get_strings	= kszphy_get_strings,
 	.get_stats	= kszphy_get_stats,
 	.suspend	= genphy_suspend,
-	.resume		= genphy_resume,
+	.resume		= kszphy_resume,
 }, {
 	.phy_id		= PHY_ID_KSZ8873MLL,
 	.phy_id_mask	= MICREL_PHY_ID_MASK,
diff --git a/drivers/net/phy/microchip.c b/drivers/net/phy/microchip.c
index 15f820648f82..7c00e508a101 100644
--- a/drivers/net/phy/microchip.c
+++ b/drivers/net/phy/microchip.c
@@ -55,7 +55,7 @@ static int lan88xx_phy_ack_interrupt(struct phy_device *phydev)
 	return rc < 0 ? rc : 0;
 }
 
-int lan88xx_suspend(struct phy_device *phydev)
+static int lan88xx_suspend(struct phy_device *phydev)
 {
 	struct lan88xx_priv *priv = phydev->priv;
 
diff --git a/drivers/net/phy/mscc.c b/drivers/net/phy/mscc.c
new file mode 100644
index 000000000000..77a6671d572e
--- /dev/null
+++ b/drivers/net/phy/mscc.c
@@ -0,0 +1,465 @@
+/*
+ * Driver for Microsemi VSC85xx PHYs
+ *
+ * Author: Nagaraju Lakkaraju
+ * License: Dual MIT/GPL
+ * Copyright (c) 2016 Microsemi Corporation
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mdio.h>
+#include <linux/mii.h>
+#include <linux/phy.h>
+#include <linux/of.h>
+#include <dt-bindings/net/mscc-phy-vsc8531.h>
+#include <linux/netdevice.h>
+
+enum rgmii_rx_clock_delay {
+	RGMII_RX_CLK_DELAY_0_2_NS = 0,
+	RGMII_RX_CLK_DELAY_0_8_NS = 1,
+	RGMII_RX_CLK_DELAY_1_1_NS = 2,
+	RGMII_RX_CLK_DELAY_1_7_NS = 3,
+	RGMII_RX_CLK_DELAY_2_0_NS = 4,
+	RGMII_RX_CLK_DELAY_2_3_NS = 5,
+	RGMII_RX_CLK_DELAY_2_6_NS = 6,
+	RGMII_RX_CLK_DELAY_3_4_NS = 7
+};
+
+/* Microsemi VSC85xx PHY registers */
+/* IEEE 802. Std Registers */
+#define MSCC_PHY_EXT_PHY_CNTL_1           23
+#define MAC_IF_SELECTION_MASK             0x1800
+#define MAC_IF_SELECTION_GMII             0
+#define MAC_IF_SELECTION_RMII             1
+#define MAC_IF_SELECTION_RGMII            2
+#define MAC_IF_SELECTION_POS              11
+#define FAR_END_LOOPBACK_MODE_MASK        0x0008
+
+#define MII_VSC85XX_INT_MASK		  25
+#define MII_VSC85XX_INT_MASK_MASK	  0xa000
+#define MII_VSC85XX_INT_MASK_WOL	  0x0040
+#define MII_VSC85XX_INT_STATUS		  26
+
+#define MSCC_PHY_WOL_MAC_CONTROL          27
+#define EDGE_RATE_CNTL_POS                5
+#define EDGE_RATE_CNTL_MASK               0x00E0
+
+#define MSCC_EXT_PAGE_ACCESS		  31
+#define MSCC_PHY_PAGE_STANDARD		  0x0000 /* Standard registers */
+#define MSCC_PHY_PAGE_EXTENDED_2	  0x0002 /* Extended reg - page 2 */
+
+/* Extended Page 2 Registers */
+#define MSCC_PHY_RGMII_CNTL		  20
+#define RGMII_RX_CLK_DELAY_MASK		  0x0070
+#define RGMII_RX_CLK_DELAY_POS		  4
+
+#define MSCC_PHY_WOL_LOWER_MAC_ADDR	  21
+#define MSCC_PHY_WOL_MID_MAC_ADDR	  22
+#define MSCC_PHY_WOL_UPPER_MAC_ADDR	  23
+#define MSCC_PHY_WOL_LOWER_PASSWD	  24
+#define MSCC_PHY_WOL_MID_PASSWD		  25
+#define MSCC_PHY_WOL_UPPER_PASSWD	  26
+
+#define MSCC_PHY_WOL_MAC_CONTROL	  27
+#define SECURE_ON_ENABLE		  0x8000
+#define SECURE_ON_PASSWD_LEN_4		  0x4000
+
+/* Microsemi PHY ID's */
+#define PHY_ID_VSC8531			  0x00070570
+#define PHY_ID_VSC8541			  0x00070770
+
+struct edge_rate_table {
+	u16 vddmac;
+	int slowdown[MSCC_SLOWDOWN_MAX];
+};
+
+struct edge_rate_table edge_table[MSCC_VDDMAC_MAX] = {
+	{3300, { 0, -2, -4,  -7,  -10, -17, -29, -53} },
+	{2500, { 0, -3, -6,  -10, -14, -23, -37, -63} },
+	{1800, { 0, -5, -9,  -16, -23, -35, -52, -76} },
+	{1500, { 0, -6, -14, -21, -29, -42, -58, -77} },
+};
+
+struct vsc8531_private {
+	u8 edge_slowdown;
+	u16 vddmac;
+};
+
+static int vsc85xx_phy_page_set(struct phy_device *phydev, u8 page)
+{
+	int rc;
+
+	rc = phy_write(phydev, MSCC_EXT_PAGE_ACCESS, page);
+	return rc;
+}
+
+static int vsc85xx_wol_set(struct phy_device *phydev,
+			   struct ethtool_wolinfo *wol)
+{
+	int rc;
+	u16 reg_val;
+	u8  i;
+	u16 pwd[3] = {0, 0, 0};
+	struct ethtool_wolinfo *wol_conf = wol;
+	u8 *mac_addr = phydev->attached_dev->dev_addr;
+
+	mutex_lock(&phydev->lock);
+	rc = vsc85xx_phy_page_set(phydev, MSCC_PHY_PAGE_EXTENDED_2);
+	if (rc != 0)
+		goto out_unlock;
+
+	if (wol->wolopts & WAKE_MAGIC) {
+		/* Store the device address for the magic packet */
+		for (i = 0; i < ARRAY_SIZE(pwd); i++)
+			pwd[i] = mac_addr[5 - (i * 2 + 1)] << 8 |
+				 mac_addr[5 - i * 2];
+		phy_write(phydev, MSCC_PHY_WOL_LOWER_MAC_ADDR, pwd[0]);
+		phy_write(phydev, MSCC_PHY_WOL_MID_MAC_ADDR, pwd[1]);
+		phy_write(phydev, MSCC_PHY_WOL_UPPER_MAC_ADDR, pwd[2]);
+	} else {
+		phy_write(phydev, MSCC_PHY_WOL_LOWER_MAC_ADDR, 0);
+		phy_write(phydev, MSCC_PHY_WOL_MID_MAC_ADDR, 0);
+		phy_write(phydev, MSCC_PHY_WOL_UPPER_MAC_ADDR, 0);
+	}
+
+	if (wol_conf->wolopts & WAKE_MAGICSECURE) {
+		for (i = 0; i < ARRAY_SIZE(pwd); i++)
+			pwd[i] = wol_conf->sopass[5 - (i * 2 + 1)] << 8 |
+				 wol_conf->sopass[5 - i * 2];
+		phy_write(phydev, MSCC_PHY_WOL_LOWER_PASSWD, pwd[0]);
+		phy_write(phydev, MSCC_PHY_WOL_MID_PASSWD, pwd[1]);
+		phy_write(phydev, MSCC_PHY_WOL_UPPER_PASSWD, pwd[2]);
+	} else {
+		phy_write(phydev, MSCC_PHY_WOL_LOWER_PASSWD, 0);
+		phy_write(phydev, MSCC_PHY_WOL_MID_PASSWD, 0);
+		phy_write(phydev, MSCC_PHY_WOL_UPPER_PASSWD, 0);
+	}
+
+	reg_val = phy_read(phydev, MSCC_PHY_WOL_MAC_CONTROL);
+	if (wol_conf->wolopts & WAKE_MAGICSECURE)
+		reg_val |= SECURE_ON_ENABLE;
+	else
+		reg_val &= ~SECURE_ON_ENABLE;
+	phy_write(phydev, MSCC_PHY_WOL_MAC_CONTROL, reg_val);
+
+	rc = vsc85xx_phy_page_set(phydev, MSCC_PHY_PAGE_STANDARD);
+	if (rc != 0)
+		goto out_unlock;
+
+	if (wol->wolopts & WAKE_MAGIC) {
+		/* Enable the WOL interrupt */
+		reg_val = phy_read(phydev, MII_VSC85XX_INT_MASK);
+		reg_val |= MII_VSC85XX_INT_MASK_WOL;
+		rc = phy_write(phydev, MII_VSC85XX_INT_MASK, reg_val);
+		if (rc != 0)
+			goto out_unlock;
+	} else {
+		/* Disable the WOL interrupt */
+		reg_val = phy_read(phydev, MII_VSC85XX_INT_MASK);
+		reg_val &= (~MII_VSC85XX_INT_MASK_WOL);
+		rc = phy_write(phydev, MII_VSC85XX_INT_MASK, reg_val);
+		if (rc != 0)
+			goto out_unlock;
+	}
+	/* Clear WOL iterrupt status */
+	reg_val = phy_read(phydev, MII_VSC85XX_INT_STATUS);
+
+out_unlock:
+	mutex_unlock(&phydev->lock);
+
+	return rc;
+}
+
+static void vsc85xx_wol_get(struct phy_device *phydev,
+			    struct ethtool_wolinfo *wol)
+{
+	int rc;
+	u16 reg_val;
+	u8  i;
+	u16 pwd[3] = {0, 0, 0};
+	struct ethtool_wolinfo *wol_conf = wol;
+
+	mutex_lock(&phydev->lock);
+	rc = vsc85xx_phy_page_set(phydev, MSCC_PHY_PAGE_EXTENDED_2);
+	if (rc != 0)
+		goto out_unlock;
+
+	reg_val = phy_read(phydev, MSCC_PHY_WOL_MAC_CONTROL);
+	if (reg_val & SECURE_ON_ENABLE)
+		wol_conf->wolopts |= WAKE_MAGICSECURE;
+	if (wol_conf->wolopts & WAKE_MAGICSECURE) {
+		pwd[0] = phy_read(phydev, MSCC_PHY_WOL_LOWER_PASSWD);
+		pwd[1] = phy_read(phydev, MSCC_PHY_WOL_MID_PASSWD);
+		pwd[2] = phy_read(phydev, MSCC_PHY_WOL_UPPER_PASSWD);
+		for (i = 0; i < ARRAY_SIZE(pwd); i++) {
+			wol_conf->sopass[5 - i * 2] = pwd[i] & 0x00ff;
+			wol_conf->sopass[5 - (i * 2 + 1)] = (pwd[i] & 0xff00)
+							    >> 8;
+		}
+	}
+
+	rc = vsc85xx_phy_page_set(phydev, MSCC_PHY_PAGE_STANDARD);
+
+out_unlock:
+	mutex_unlock(&phydev->lock);
+}
+
+static u8 edge_rate_magic_get(u16 vddmac,
+			      int slowdown)
+{
+	int rc = (MSCC_SLOWDOWN_MAX - 1);
+	u8 vdd;
+	u8 sd;
+
+	for (vdd = 0; vdd < MSCC_VDDMAC_MAX; vdd++) {
+		if (edge_table[vdd].vddmac == vddmac) {
+			for (sd = 0; sd < MSCC_SLOWDOWN_MAX; sd++) {
+				if (edge_table[vdd].slowdown[sd] <= slowdown) {
+					rc = (MSCC_SLOWDOWN_MAX - sd - 1);
+					break;
+				}
+			}
+		}
+	}
+
+	return rc;
+}
+
+static int vsc85xx_edge_rate_cntl_set(struct phy_device *phydev,
+				      u8 edge_rate)
+{
+	int rc;
+	u16 reg_val;
+
+	mutex_lock(&phydev->lock);
+	rc = vsc85xx_phy_page_set(phydev, MSCC_PHY_PAGE_EXTENDED_2);
+	if (rc != 0)
+		goto out_unlock;
+	reg_val = phy_read(phydev, MSCC_PHY_WOL_MAC_CONTROL);
+	reg_val &= ~(EDGE_RATE_CNTL_MASK);
+	reg_val |= (edge_rate << EDGE_RATE_CNTL_POS);
+	rc = phy_write(phydev, MSCC_PHY_WOL_MAC_CONTROL, reg_val);
+	if (rc != 0)
+		goto out_unlock;
+	rc = vsc85xx_phy_page_set(phydev, MSCC_PHY_PAGE_STANDARD);
+
+out_unlock:
+	mutex_unlock(&phydev->lock);
+
+	return rc;
+}
+
+static int vsc85xx_mac_if_set(struct phy_device *phydev,
+			      phy_interface_t interface)
+{
+	int rc;
+	u16 reg_val;
+
+	mutex_lock(&phydev->lock);
+	reg_val = phy_read(phydev, MSCC_PHY_EXT_PHY_CNTL_1);
+	reg_val &= ~(MAC_IF_SELECTION_MASK);
+	switch (interface) {
+	case PHY_INTERFACE_MODE_RGMII:
+		reg_val |= (MAC_IF_SELECTION_RGMII << MAC_IF_SELECTION_POS);
+		break;
+	case PHY_INTERFACE_MODE_RMII:
+		reg_val |= (MAC_IF_SELECTION_RMII << MAC_IF_SELECTION_POS);
+		break;
+	case PHY_INTERFACE_MODE_MII:
+	case PHY_INTERFACE_MODE_GMII:
+		reg_val |= (MAC_IF_SELECTION_GMII << MAC_IF_SELECTION_POS);
+		break;
+	default:
+		rc = -EINVAL;
+		goto out_unlock;
+	}
+	rc = phy_write(phydev, MSCC_PHY_EXT_PHY_CNTL_1, reg_val);
+	if (rc != 0)
+		goto out_unlock;
+
+	rc = genphy_soft_reset(phydev);
+
+out_unlock:
+	mutex_unlock(&phydev->lock);
+
+	return rc;
+}
+
+static int vsc85xx_default_config(struct phy_device *phydev)
+{
+	int rc;
+	u16 reg_val;
+
+	mutex_lock(&phydev->lock);
+	rc = vsc85xx_phy_page_set(phydev, MSCC_PHY_PAGE_EXTENDED_2);
+	if (rc != 0)
+		goto out_unlock;
+
+	reg_val = phy_read(phydev, MSCC_PHY_RGMII_CNTL);
+	reg_val &= ~(RGMII_RX_CLK_DELAY_MASK);
+	reg_val |= (RGMII_RX_CLK_DELAY_1_1_NS << RGMII_RX_CLK_DELAY_POS);
+	phy_write(phydev, MSCC_PHY_RGMII_CNTL, reg_val);
+	rc = vsc85xx_phy_page_set(phydev, MSCC_PHY_PAGE_STANDARD);
+
+out_unlock:
+	mutex_unlock(&phydev->lock);
+
+	return rc;
+}
+
+#ifdef CONFIG_OF_MDIO
+static int vsc8531_of_init(struct phy_device *phydev)
+{
+	int rc;
+	struct vsc8531_private *vsc8531 = phydev->priv;
+	struct device *dev = &phydev->mdio.dev;
+	struct device_node *of_node = dev->of_node;
+
+	if (!of_node)
+		return -ENODEV;
+
+	rc = of_property_read_u16(of_node, "vsc8531,vddmac",
+				  &vsc8531->vddmac);
+	if (rc == -EINVAL)
+		vsc8531->vddmac = MSCC_VDDMAC_3300;
+	rc = of_property_read_u8(of_node, "vsc8531,edge-slowdown",
+				 &vsc8531->edge_slowdown);
+	if (rc == -EINVAL)
+		vsc8531->edge_slowdown = 0;
+
+	rc = 0;
+	return rc;
+}
+#else
+static int vsc8531_of_init(struct phy_device *phydev)
+{
+	return 0;
+}
+#endif /* CONFIG_OF_MDIO */
+
+static int vsc85xx_config_init(struct phy_device *phydev)
+{
+	int rc;
+	struct vsc8531_private *vsc8531 = phydev->priv;
+	u8 edge_rate;
+
+	rc = vsc8531_of_init(phydev);
+	if (rc)
+		return rc;
+
+	rc = vsc85xx_default_config(phydev);
+	if (rc)
+		return rc;
+
+	rc = vsc85xx_mac_if_set(phydev, phydev->interface);
+	if (rc)
+		return rc;
+
+	edge_rate = edge_rate_magic_get(vsc8531->vddmac,
+					-(int)vsc8531->edge_slowdown);
+	rc = vsc85xx_edge_rate_cntl_set(phydev, edge_rate);
+	if (rc)
+		return rc;
+
+	rc = genphy_config_init(phydev);
+
+	return rc;
+}
+
+static int vsc85xx_ack_interrupt(struct phy_device *phydev)
+{
+	int rc = 0;
+
+	if (phydev->interrupts == PHY_INTERRUPT_ENABLED)
+		rc = phy_read(phydev, MII_VSC85XX_INT_STATUS);
+
+	return (rc < 0) ? rc : 0;
+}
+
+static int vsc85xx_config_intr(struct phy_device *phydev)
+{
+	int rc;
+
+	if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
+		rc = phy_write(phydev, MII_VSC85XX_INT_MASK,
+			       MII_VSC85XX_INT_MASK_MASK);
+	} else {
+		rc = phy_write(phydev, MII_VSC85XX_INT_MASK, 0);
+		if (rc < 0)
+			return rc;
+		rc = phy_read(phydev, MII_VSC85XX_INT_STATUS);
+	}
+
+	return rc;
+}
+
+static int vsc85xx_probe(struct phy_device *phydev)
+{
+	struct vsc8531_private *vsc8531;
+
+	vsc8531 = devm_kzalloc(&phydev->mdio.dev, sizeof(*vsc8531), GFP_KERNEL);
+	if (!vsc8531)
+		return -ENOMEM;
+
+	phydev->priv = vsc8531;
+
+	return 0;
+}
+
+/* Microsemi VSC85xx PHYs */
+static struct phy_driver vsc85xx_driver[] = {
+{
+	.phy_id		= PHY_ID_VSC8531,
+	.name		= "Microsemi VSC8531",
+	.phy_id_mask    = 0xfffffff0,
+	.features	= PHY_GBIT_FEATURES,
+	.flags		= PHY_HAS_INTERRUPT,
+	.soft_reset	= &genphy_soft_reset,
+	.config_init    = &vsc85xx_config_init,
+	.config_aneg    = &genphy_config_aneg,
+	.aneg_done	= &genphy_aneg_done,
+	.read_status    = &genphy_read_status,
+	.ack_interrupt  = &vsc85xx_ack_interrupt,
+	.config_intr    = &vsc85xx_config_intr,
+	.suspend	= &genphy_suspend,
+	.resume		= &genphy_resume,
+	.probe          = &vsc85xx_probe,
+	.set_wol        = &vsc85xx_wol_set,
+	.get_wol        = &vsc85xx_wol_get,
+},
+{
+	.phy_id		= PHY_ID_VSC8541,
+	.name		= "Microsemi VSC8541 SyncE",
+	.phy_id_mask    = 0xfffffff0,
+	.features	= PHY_GBIT_FEATURES,
+	.flags		= PHY_HAS_INTERRUPT,
+	.soft_reset	= &genphy_soft_reset,
+	.config_init    = &vsc85xx_config_init,
+	.config_aneg    = &genphy_config_aneg,
+	.aneg_done	= &genphy_aneg_done,
+	.read_status    = &genphy_read_status,
+	.ack_interrupt  = &vsc85xx_ack_interrupt,
+	.config_intr    = &vsc85xx_config_intr,
+	.suspend	= &genphy_suspend,
+	.resume		= &genphy_resume,
+	.probe          = &vsc85xx_probe,
+	.set_wol        = &vsc85xx_wol_set,
+	.get_wol        = &vsc85xx_wol_get,
+}
+
+};
+
+module_phy_driver(vsc85xx_driver);
+
+static struct mdio_device_id __maybe_unused vsc85xx_tbl[] = {
+	{ PHY_ID_VSC8531, 0xfffffff0, },
+	{ PHY_ID_VSC8541, 0xfffffff0, },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(mdio, vsc85xx_tbl);
+
+MODULE_DESCRIPTION("Microsemi VSC85xx PHY driver");
+MODULE_AUTHOR("Nagaraju Lakkaraju");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
index c5dc2c363f96..c6f66832a1a6 100644
--- a/drivers/net/phy/phy.c
+++ b/drivers/net/phy/phy.c
@@ -722,8 +722,10 @@ phy_err:
 int phy_start_interrupts(struct phy_device *phydev)
 {
 	atomic_set(&phydev->irq_disable, 0);
-	if (request_irq(phydev->irq, phy_interrupt, 0, "phy_interrupt",
-			phydev) < 0) {
+	if (request_irq(phydev->irq, phy_interrupt,
+				IRQF_SHARED,
+				"phy_interrupt",
+				phydev) < 0) {
 		pr_warn("%s: Can't get IRQ %d (PHY)\n",
 			phydev->mdio.bus->name, phydev->irq);
 		phydev->irq = PHY_POLL;
diff --git a/drivers/net/phy/xilinx_gmii2rgmii.c b/drivers/net/phy/xilinx_gmii2rgmii.c
new file mode 100644
index 000000000000..d15dd3938ba8
--- /dev/null
+++ b/drivers/net/phy/xilinx_gmii2rgmii.c
@@ -0,0 +1,112 @@
+/* Xilinx GMII2RGMII Converter driver
+ *
+ * Copyright (C) 2016 Xilinx, Inc.
+ * Copyright (C) 2016 Andrew Lunn <andrew@lunn.ch>
+ *
+ * Author: Andrew Lunn <andrew@lunn.ch>
+ * Author: Kedareswara rao Appana <appanad@xilinx.com>
+ *
+ * Description:
+ * This driver is developed for Xilinx GMII2RGMII Converter
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/mii.h>
+#include <linux/mdio.h>
+#include <linux/phy.h>
+#include <linux/of_mdio.h>
+
+#define XILINX_GMII2RGMII_REG		0x10
+#define XILINX_GMII2RGMII_SPEED_MASK	(BMCR_SPEED1000 | BMCR_SPEED100)
+
+struct gmii2rgmii {
+	struct phy_device *phy_dev;
+	struct phy_driver *phy_drv;
+	struct phy_driver conv_phy_drv;
+	int addr;
+};
+
+static int xgmiitorgmii_read_status(struct phy_device *phydev)
+{
+	struct gmii2rgmii *priv = phydev->priv;
+	u16 val = 0;
+
+	priv->phy_drv->read_status(phydev);
+
+	val = mdiobus_read(phydev->mdio.bus, priv->addr, XILINX_GMII2RGMII_REG);
+	val &= XILINX_GMII2RGMII_SPEED_MASK;
+
+	if (phydev->speed == SPEED_1000)
+		val |= BMCR_SPEED1000;
+	else if (phydev->speed == SPEED_100)
+		val |= BMCR_SPEED100;
+	else
+		val |= BMCR_SPEED10;
+
+	mdiobus_write(phydev->mdio.bus, priv->addr, XILINX_GMII2RGMII_REG, val);
+
+	return 0;
+}
+
+static int xgmiitorgmii_probe(struct mdio_device *mdiodev)
+{
+	struct device *dev = &mdiodev->dev;
+	struct device_node *np = dev->of_node, *phy_node;
+	struct gmii2rgmii *priv;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	phy_node = of_parse_phandle(np, "phy-handle", 0);
+	if (!phy_node) {
+		dev_err(dev, "Couldn't parse phy-handle\n");
+		return -ENODEV;
+	}
+
+	priv->phy_dev = of_phy_find_device(phy_node);
+	of_node_put(phy_node);
+	if (!priv->phy_dev) {
+		dev_info(dev, "Couldn't find phydev\n");
+		return -EPROBE_DEFER;
+	}
+
+	priv->addr = mdiodev->addr;
+	priv->phy_drv = priv->phy_dev->drv;
+	memcpy(&priv->conv_phy_drv, priv->phy_dev->drv,
+	       sizeof(struct phy_driver));
+	priv->conv_phy_drv.read_status = xgmiitorgmii_read_status;
+	priv->phy_dev->priv = priv;
+	priv->phy_dev->drv = &priv->conv_phy_drv;
+
+	return 0;
+}
+
+static const struct of_device_id xgmiitorgmii_of_match[] = {
+	{ .compatible = "xlnx,gmii-to-rgmii-1.0" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, xgmiitorgmii_of_match);
+
+static struct mdio_driver xgmiitorgmii_driver = {
+	.probe	= xgmiitorgmii_probe,
+	.mdiodrv.driver = {
+		.name = "xgmiitorgmii",
+		.of_match_table = xgmiitorgmii_of_match,
+	},
+};
+
+mdio_module_driver(xgmiitorgmii_driver);
+
+MODULE_DESCRIPTION("Xilinx GMII2RGMII converter driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
index f226db4616b7..5489c0ec1d9a 100644
--- a/drivers/net/ppp/ppp_generic.c
+++ b/drivers/net/ppp/ppp_generic.c
@@ -1103,6 +1103,15 @@ static int ppp_nl_newlink(struct net *src_net, struct net_device *dev,
 	}
 
 	conf.file = file;
+
+	/* Don't use device name generated by the rtnetlink layer when ifname
+	 * isn't specified. Let ppp_dev_configure() set the device name using
+	 * the PPP unit identifer as suffix (i.e. ppp<unit_id>). This allows
+	 * userspace to infer the device name using to the PPPIOCGUNIT ioctl.
+	 */
+	if (!tb[IFLA_IFNAME])
+		conf.ifname_is_set = false;
+
 	err = ppp_dev_configure(src_net, dev, &conf);
 
 out_unlock:
@@ -1354,6 +1363,8 @@ static void ppp_setup(struct net_device *dev)
 	dev->netdev_ops = &ppp_netdev_ops;
 	SET_NETDEV_DEVTYPE(dev, &ppp_type);
 
+	dev->features |= NETIF_F_LLTX;
+
 	dev->hard_header_len = PPP_HDRLEN;
 	dev->mtu = PPP_MRU;
 	dev->addr_len = 0;
@@ -1367,12 +1378,8 @@ static void ppp_setup(struct net_device *dev)
  * Transmit-side routines.
  */
 
-/*
- * Called to do any work queued up on the transmit side
- * that can now be done.
- */
-static void
-ppp_xmit_process(struct ppp *ppp)
+/* Called to do any work queued up on the transmit side that can now be done */
+static void __ppp_xmit_process(struct ppp *ppp)
 {
 	struct sk_buff *skb;
 
@@ -1392,6 +1399,30 @@ ppp_xmit_process(struct ppp *ppp)
 	ppp_xmit_unlock(ppp);
 }
 
+static DEFINE_PER_CPU(int, ppp_xmit_recursion);
+
+static void ppp_xmit_process(struct ppp *ppp)
+{
+	local_bh_disable();
+
+	if (unlikely(__this_cpu_read(ppp_xmit_recursion)))
+		goto err;
+
+	__this_cpu_inc(ppp_xmit_recursion);
+	__ppp_xmit_process(ppp);
+	__this_cpu_dec(ppp_xmit_recursion);
+
+	local_bh_enable();
+
+	return;
+
+err:
+	local_bh_enable();
+
+	if (net_ratelimit())
+		netdev_err(ppp->dev, "recursion detected\n");
+}
+
 static inline struct sk_buff *
 pad_compress_skb(struct ppp *ppp, struct sk_buff *skb)
 {
@@ -1847,11 +1878,8 @@ static int ppp_mp_explode(struct ppp *ppp, struct sk_buff *skb)
 }
 #endif /* CONFIG_PPP_MULTILINK */
 
-/*
- * Try to send data out on a channel.
- */
-static void
-ppp_channel_push(struct channel *pch)
+/* Try to send data out on a channel */
+static void __ppp_channel_push(struct channel *pch)
 {
 	struct sk_buff *skb;
 	struct ppp *ppp;
@@ -1876,11 +1904,22 @@ ppp_channel_push(struct channel *pch)
 		read_lock_bh(&pch->upl);
 		ppp = pch->ppp;
 		if (ppp)
-			ppp_xmit_process(ppp);
+			__ppp_xmit_process(ppp);
 		read_unlock_bh(&pch->upl);
 	}
 }
 
+static void ppp_channel_push(struct channel *pch)
+{
+	local_bh_disable();
+
+	__this_cpu_inc(ppp_xmit_recursion);
+	__ppp_channel_push(pch);
+	__this_cpu_dec(ppp_xmit_recursion);
+
+	local_bh_enable();
+}
+
 /*
  * Receive-side routines.
  */
diff --git a/drivers/net/ppp/pptp.c b/drivers/net/ppp/pptp.c
index ae0905ed4a32..1951b1085cb8 100644
--- a/drivers/net/ppp/pptp.c
+++ b/drivers/net/ppp/pptp.c
@@ -37,6 +37,7 @@
 #include <net/icmp.h>
 #include <net/route.h>
 #include <net/gre.h>
+#include <net/pptp.h>
 
 #include <linux/uaccess.h>
 
@@ -53,41 +54,6 @@ static struct proto pptp_sk_proto __read_mostly;
 static const struct ppp_channel_ops pptp_chan_ops;
 static const struct proto_ops pptp_ops;
 
-#define PPP_LCP_ECHOREQ 0x09
-#define PPP_LCP_ECHOREP 0x0A
-#define SC_RCV_BITS	(SC_RCV_B7_1|SC_RCV_B7_0|SC_RCV_ODDP|SC_RCV_EVNP)
-
-#define MISSING_WINDOW 20
-#define WRAPPED(curseq, lastseq)\
-	((((curseq) & 0xffffff00) == 0) &&\
-	(((lastseq) & 0xffffff00) == 0xffffff00))
-
-#define PPTP_GRE_PROTO  0x880B
-#define PPTP_GRE_VER    0x1
-
-#define PPTP_GRE_FLAG_C	0x80
-#define PPTP_GRE_FLAG_R	0x40
-#define PPTP_GRE_FLAG_K	0x20
-#define PPTP_GRE_FLAG_S	0x10
-#define PPTP_GRE_FLAG_A	0x80
-
-#define PPTP_GRE_IS_C(f) ((f)&PPTP_GRE_FLAG_C)
-#define PPTP_GRE_IS_R(f) ((f)&PPTP_GRE_FLAG_R)
-#define PPTP_GRE_IS_K(f) ((f)&PPTP_GRE_FLAG_K)
-#define PPTP_GRE_IS_S(f) ((f)&PPTP_GRE_FLAG_S)
-#define PPTP_GRE_IS_A(f) ((f)&PPTP_GRE_FLAG_A)
-
-#define PPTP_HEADER_OVERHEAD (2+sizeof(struct pptp_gre_header))
-struct pptp_gre_header {
-	u8  flags;
-	u8  ver;
-	__be16 protocol;
-	__be16 payload_len;
-	__be16 call_id;
-	__be32 seq;
-	__be32 ack;
-} __packed;
-
 static struct pppox_sock *lookup_chan(u16 call_id, __be32 s_addr)
 {
 	struct pppox_sock *sock;
@@ -240,16 +206,14 @@ static int pptp_xmit(struct ppp_channel *chan, struct sk_buff *skb)
 	skb_push(skb, header_len);
 	hdr = (struct pptp_gre_header *)(skb->data);
 
-	hdr->flags       = PPTP_GRE_FLAG_K;
-	hdr->ver         = PPTP_GRE_VER;
-	hdr->protocol    = htons(PPTP_GRE_PROTO);
-	hdr->call_id     = htons(opt->dst_addr.call_id);
+	hdr->gre_hd.flags = GRE_KEY | GRE_VERSION_1 | GRE_SEQ;
+	hdr->gre_hd.protocol = GRE_PROTO_PPP;
+	hdr->call_id = htons(opt->dst_addr.call_id);
 
-	hdr->flags      |= PPTP_GRE_FLAG_S;
-	hdr->seq         = htonl(++opt->seq_sent);
+	hdr->seq = htonl(++opt->seq_sent);
 	if (opt->ack_sent != seq_recv)	{
 		/* send ack with this message */
-		hdr->ver |= PPTP_GRE_FLAG_A;
+		hdr->gre_hd.flags |= GRE_ACK;
 		hdr->ack  = htonl(seq_recv);
 		opt->ack_sent = seq_recv;
 	}
@@ -312,7 +276,7 @@ static int pptp_rcv_core(struct sock *sk, struct sk_buff *skb)
 	headersize  = sizeof(*header);
 
 	/* test if acknowledgement present */
-	if (PPTP_GRE_IS_A(header->ver)) {
+	if (GRE_IS_ACK(header->gre_hd.flags)) {
 		__u32 ack;
 
 		if (!pskb_may_pull(skb, headersize))
@@ -320,7 +284,7 @@ static int pptp_rcv_core(struct sock *sk, struct sk_buff *skb)
 		header = (struct pptp_gre_header *)(skb->data);
 
 		/* ack in different place if S = 0 */
-		ack = PPTP_GRE_IS_S(header->flags) ? header->ack : header->seq;
+		ack = GRE_IS_SEQ(header->gre_hd.flags) ? header->ack : header->seq;
 
 		ack = ntohl(ack);
 
@@ -333,7 +297,7 @@ static int pptp_rcv_core(struct sock *sk, struct sk_buff *skb)
 		headersize -= sizeof(header->ack);
 	}
 	/* test if payload present */
-	if (!PPTP_GRE_IS_S(header->flags))
+	if (!GRE_IS_SEQ(header->gre_hd.flags))
 		goto drop;
 
 	payload_len = ntohs(header->payload_len);
@@ -394,11 +358,11 @@ static int pptp_rcv(struct sk_buff *skb)
 
 	header = (struct pptp_gre_header *)skb->data;
 
-	if (ntohs(header->protocol) != PPTP_GRE_PROTO || /* PPTP-GRE protocol for PPTP */
-		PPTP_GRE_IS_C(header->flags) ||                /* flag C should be clear */
-		PPTP_GRE_IS_R(header->flags) ||                /* flag R should be clear */
-		!PPTP_GRE_IS_K(header->flags) ||               /* flag K should be set */
-		(header->flags&0xF) != 0)                      /* routing and recursion ctrl = 0 */
+	if (header->gre_hd.protocol != GRE_PROTO_PPP || /* PPTP-GRE protocol for PPTP */
+		GRE_IS_CSUM(header->gre_hd.flags) ||    /* flag CSUM should be clear */
+		GRE_IS_ROUTING(header->gre_hd.flags) || /* flag ROUTING should be clear */
+		!GRE_IS_KEY(header->gre_hd.flags) ||    /* flag KEY should be set */
+		(header->gre_hd.flags & GRE_FLAGS))     /* flag Recursion Ctrl should be clear */
 		/* if invalid, discard this packet */
 		goto drop;
 
diff --git a/drivers/net/team/team_mode_loadbalance.c b/drivers/net/team/team_mode_loadbalance.c
index cdb19b385d42..b228bea7931f 100644
--- a/drivers/net/team/team_mode_loadbalance.c
+++ b/drivers/net/team/team_mode_loadbalance.c
@@ -14,9 +14,23 @@
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/netdevice.h>
+#include <linux/etherdevice.h>
 #include <linux/filter.h>
 #include <linux/if_team.h>
 
+static rx_handler_result_t lb_receive(struct team *team, struct team_port *port,
+				      struct sk_buff *skb)
+{
+	if (unlikely(skb->protocol == htons(ETH_P_SLOW))) {
+		/* LACPDU packets should go to exact delivery */
+		const unsigned char *dest = eth_hdr(skb)->h_dest;
+
+		if (is_link_local_ether_addr(dest) && dest[5] == 0x02)
+			return RX_HANDLER_EXACT;
+	}
+	return RX_HANDLER_ANOTHER;
+}
+
 struct lb_priv;
 
 typedef struct team_port *lb_select_tx_port_func_t(struct team *,
@@ -652,6 +666,7 @@ static const struct team_mode_ops lb_mode_ops = {
 	.port_enter		= lb_port_enter,
 	.port_leave		= lb_port_leave,
 	.port_disabled		= lb_port_disabled,
+	.receive		= lb_receive,
 	.transmit		= lb_transmit,
 };
 
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index 9c8b5bc2b9d8..8093e39ae263 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -731,14 +731,9 @@ static int update_filter(struct tap_filter *filter, void __user *arg)
 	}
 
 	alen = ETH_ALEN * uf.count;
-	addr = kmalloc(alen, GFP_KERNEL);
-	if (!addr)
-		return -ENOMEM;
-
-	if (copy_from_user(addr, arg + sizeof(uf), alen)) {
-		err = -EFAULT;
-		goto done;
-	}
+	addr = memdup_user(arg + sizeof(uf), alen);
+	if (IS_ERR(addr))
+		return PTR_ERR(addr);
 
 	/* The filter is updated without holding any locks. Which is
 	 * perfectly safe. We disable it first and in the worst
@@ -758,7 +753,7 @@ static int update_filter(struct tap_filter *filter, void __user *arg)
 	for (; n < uf.count; n++) {
 		if (!is_multicast_ether_addr(addr[n].u)) {
 			err = 0; /* no filter */
-			goto done;
+			goto free_addr;
 		}
 		addr_hash_set(filter->mask, addr[n].u);
 	}
@@ -774,8 +769,7 @@ static int update_filter(struct tap_filter *filter, void __user *arg)
 
 	/* Return the number of exact filters */
 	err = nexact;
-
-done:
+free_addr:
 	kfree(addr);
 	return err;
 }
@@ -894,11 +888,7 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
 	if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
 		goto drop;
 
-	if (skb->sk && sk_fullsock(skb->sk)) {
-		sock_tx_timestamp(skb->sk, skb->sk->sk_tsflags,
-				  &skb_shinfo(skb)->tx_flags);
-		sw_tx_timestamp(skb);
-	}
+	skb_tx_timestamp(skb);
 
 	/* Orphan the skb - required as we might hang on to it
 	 * for indefinite time.
diff --git a/drivers/net/usb/asix.h b/drivers/net/usb/asix.h
index a2d3ea6efb20..d1092421aaa7 100644
--- a/drivers/net/usb/asix.h
+++ b/drivers/net/usb/asix.h
@@ -46,6 +46,7 @@
 #define AX_CMD_SET_SW_MII		0x06
 #define AX_CMD_READ_MII_REG		0x07
 #define AX_CMD_WRITE_MII_REG		0x08
+#define AX_CMD_STATMNGSTS_REG		0x09
 #define AX_CMD_SET_HW_MII		0x0a
 #define AX_CMD_READ_EEPROM		0x0b
 #define AX_CMD_WRITE_EEPROM		0x0c
@@ -71,6 +72,17 @@
 #define AX_CMD_SW_RESET			0x20
 #define AX_CMD_SW_PHY_STATUS		0x21
 #define AX_CMD_SW_PHY_SELECT		0x22
+#define AX_QCTCTRL			0x2A
+
+#define AX_CHIPCODE_MASK		0x70
+#define AX_AX88772_CHIPCODE		0x00
+#define AX_AX88772A_CHIPCODE		0x10
+#define AX_AX88772B_CHIPCODE		0x20
+#define AX_HOST_EN			0x01
+
+#define AX_PHYSEL_PSEL			0x01
+#define AX_PHYSEL_SSMII			0
+#define AX_PHYSEL_SSEN			0x10
 
 #define AX_PHY_SELECT_MASK		(BIT(3) | BIT(2))
 #define AX_PHY_SELECT_INTERNAL		0
@@ -173,6 +185,10 @@ struct asix_rx_fixup_info {
 };
 
 struct asix_common_private {
+	void (*resume)(struct usbnet *dev);
+	void (*suspend)(struct usbnet *dev);
+	u16 presvd_phy_advertise;
+	u16 presvd_phy_bmcr;
 	struct asix_rx_fixup_info rx_fixup_info;
 };
 
@@ -182,10 +198,10 @@ extern const struct driver_info ax88172a_info;
 #define FLAG_EEPROM_MAC		(1UL << 0)  /* init device MAC from eeprom */
 
 int asix_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
-		  u16 size, void *data);
+		  u16 size, void *data, int in_pm);
 
 int asix_write_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
-		   u16 size, void *data);
+		   u16 size, void *data, int in_pm);
 
 void asix_write_cmd_async(struct usbnet *dev, u8 cmd, u16 value,
 			  u16 index, u16 size, void *data);
@@ -197,27 +213,31 @@ int asix_rx_fixup_common(struct usbnet *dev, struct sk_buff *skb);
 struct sk_buff *asix_tx_fixup(struct usbnet *dev, struct sk_buff *skb,
 			      gfp_t flags);
 
-int asix_set_sw_mii(struct usbnet *dev);
-int asix_set_hw_mii(struct usbnet *dev);
+int asix_set_sw_mii(struct usbnet *dev, int in_pm);
+int asix_set_hw_mii(struct usbnet *dev, int in_pm);
 
 int asix_read_phy_addr(struct usbnet *dev, int internal);
 int asix_get_phy_addr(struct usbnet *dev);
 
-int asix_sw_reset(struct usbnet *dev, u8 flags);
+int asix_sw_reset(struct usbnet *dev, u8 flags, int in_pm);
 
-u16 asix_read_rx_ctl(struct usbnet *dev);
-int asix_write_rx_ctl(struct usbnet *dev, u16 mode);
+u16 asix_read_rx_ctl(struct usbnet *dev, int in_pm);
+int asix_write_rx_ctl(struct usbnet *dev, u16 mode, int in_pm);
 
-u16 asix_read_medium_status(struct usbnet *dev);
-int asix_write_medium_mode(struct usbnet *dev, u16 mode);
+u16 asix_read_medium_status(struct usbnet *dev, int in_pm);
+int asix_write_medium_mode(struct usbnet *dev, u16 mode, int in_pm);
 
-int asix_write_gpio(struct usbnet *dev, u16 value, int sleep);
+int asix_write_gpio(struct usbnet *dev, u16 value, int sleep, int in_pm);
 
 void asix_set_multicast(struct net_device *net);
 
 int asix_mdio_read(struct net_device *netdev, int phy_id, int loc);
 void asix_mdio_write(struct net_device *netdev, int phy_id, int loc, int val);
 
+int asix_mdio_read_nopm(struct net_device *netdev, int phy_id, int loc);
+void asix_mdio_write_nopm(struct net_device *netdev, int phy_id, int loc,
+			  int val);
+
 void asix_get_wol(struct net_device *net, struct ethtool_wolinfo *wolinfo);
 int asix_set_wol(struct net_device *net, struct ethtool_wolinfo *wolinfo);
 
diff --git a/drivers/net/usb/asix_common.c b/drivers/net/usb/asix_common.c
index 7de5ab589e4e..f79eb12c326a 100644
--- a/drivers/net/usb/asix_common.c
+++ b/drivers/net/usb/asix_common.c
@@ -22,24 +22,49 @@
 #include "asix.h"
 
 int asix_read_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
-		  u16 size, void *data)
+		  u16 size, void *data, int in_pm)
 {
 	int ret;
-	ret = usbnet_read_cmd(dev, cmd,
-			       USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
-			       value, index, data, size);
+	int (*fn)(struct usbnet *, u8, u8, u16, u16, void *, u16);
+
+	BUG_ON(!dev);
+
+	if (!in_pm)
+		fn = usbnet_read_cmd;
+	else
+		fn = usbnet_read_cmd_nopm;
+
+	ret = fn(dev, cmd, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+		 value, index, data, size);
+
+	if (unlikely(ret < 0))
+		netdev_warn(dev->net, "Failed to read reg index 0x%04x: %d\n",
+			    index, ret);
 
-	if (ret != size && ret >= 0)
-		return -EINVAL;
 	return ret;
 }
 
 int asix_write_cmd(struct usbnet *dev, u8 cmd, u16 value, u16 index,
-		   u16 size, void *data)
+		   u16 size, void *data, int in_pm)
 {
-	return usbnet_write_cmd(dev, cmd,
-				USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
-				value, index, data, size);
+	int ret;
+	int (*fn)(struct usbnet *, u8, u8, u16, u16, const void *, u16);
+
+	BUG_ON(!dev);
+
+	if (!in_pm)
+		fn = usbnet_write_cmd;
+	else
+		fn = usbnet_write_cmd_nopm;
+
+	ret = fn(dev, cmd, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
+		 value, index, data, size);
+
+	if (unlikely(ret < 0))
+		netdev_warn(dev->net, "Failed to write reg index 0x%04x: %d\n",
+			    index, ret);
+
+	return ret;
 }
 
 void asix_write_cmd_async(struct usbnet *dev, u8 cmd, u16 value, u16 index,
@@ -225,19 +250,20 @@ struct sk_buff *asix_tx_fixup(struct usbnet *dev, struct sk_buff *skb,
 	return skb;
 }
 
-int asix_set_sw_mii(struct usbnet *dev)
+int asix_set_sw_mii(struct usbnet *dev, int in_pm)
 {
 	int ret;
-	ret = asix_write_cmd(dev, AX_CMD_SET_SW_MII, 0x0000, 0, 0, NULL);
+	ret = asix_write_cmd(dev, AX_CMD_SET_SW_MII, 0x0000, 0, 0, NULL, in_pm);
+
 	if (ret < 0)
 		netdev_err(dev->net, "Failed to enable software MII access\n");
 	return ret;
 }
 
-int asix_set_hw_mii(struct usbnet *dev)
+int asix_set_hw_mii(struct usbnet *dev, int in_pm)
 {
 	int ret;
-	ret = asix_write_cmd(dev, AX_CMD_SET_HW_MII, 0x0000, 0, 0, NULL);
+	ret = asix_write_cmd(dev, AX_CMD_SET_HW_MII, 0x0000, 0, 0, NULL, in_pm);
 	if (ret < 0)
 		netdev_err(dev->net, "Failed to enable hardware MII access\n");
 	return ret;
@@ -247,7 +273,7 @@ int asix_read_phy_addr(struct usbnet *dev, int internal)
 {
 	int offset = (internal ? 1 : 0);
 	u8 buf[2];
-	int ret = asix_read_cmd(dev, AX_CMD_READ_PHY_ID, 0, 0, 2, buf);
+	int ret = asix_read_cmd(dev, AX_CMD_READ_PHY_ID, 0, 0, 2, buf, 0);
 
 	netdev_dbg(dev->net, "asix_get_phy_addr()\n");
 
@@ -270,21 +296,21 @@ int asix_get_phy_addr(struct usbnet *dev)
 }
 
 
-int asix_sw_reset(struct usbnet *dev, u8 flags)
+int asix_sw_reset(struct usbnet *dev, u8 flags, int in_pm)
 {
 	int ret;
 
-        ret = asix_write_cmd(dev, AX_CMD_SW_RESET, flags, 0, 0, NULL);
+	ret = asix_write_cmd(dev, AX_CMD_SW_RESET, flags, 0, 0, NULL, in_pm);
 	if (ret < 0)
 		netdev_err(dev->net, "Failed to send software reset: %02x\n", ret);
 
 	return ret;
 }
 
-u16 asix_read_rx_ctl(struct usbnet *dev)
+u16 asix_read_rx_ctl(struct usbnet *dev, int in_pm)
 {
 	__le16 v;
-	int ret = asix_read_cmd(dev, AX_CMD_READ_RX_CTL, 0, 0, 2, &v);
+	int ret = asix_read_cmd(dev, AX_CMD_READ_RX_CTL, 0, 0, 2, &v, in_pm);
 
 	if (ret < 0) {
 		netdev_err(dev->net, "Error reading RX_CTL register: %02x\n", ret);
@@ -295,12 +321,12 @@ out:
 	return ret;
 }
 
-int asix_write_rx_ctl(struct usbnet *dev, u16 mode)
+int asix_write_rx_ctl(struct usbnet *dev, u16 mode, int in_pm)
 {
 	int ret;
 
 	netdev_dbg(dev->net, "asix_write_rx_ctl() - mode = 0x%04x\n", mode);
-	ret = asix_write_cmd(dev, AX_CMD_WRITE_RX_CTL, mode, 0, 0, NULL);
+	ret = asix_write_cmd(dev, AX_CMD_WRITE_RX_CTL, mode, 0, 0, NULL, in_pm);
 	if (ret < 0)
 		netdev_err(dev->net, "Failed to write RX_CTL mode to 0x%04x: %02x\n",
 			   mode, ret);
@@ -308,10 +334,11 @@ int asix_write_rx_ctl(struct usbnet *dev, u16 mode)
 	return ret;
 }
 
-u16 asix_read_medium_status(struct usbnet *dev)
+u16 asix_read_medium_status(struct usbnet *dev, int in_pm)
 {
 	__le16 v;
-	int ret = asix_read_cmd(dev, AX_CMD_READ_MEDIUM_STATUS, 0, 0, 2, &v);
+	int ret = asix_read_cmd(dev, AX_CMD_READ_MEDIUM_STATUS,
+				0, 0, 2, &v, in_pm);
 
 	if (ret < 0) {
 		netdev_err(dev->net, "Error reading Medium Status register: %02x\n",
@@ -323,12 +350,13 @@ u16 asix_read_medium_status(struct usbnet *dev)
 
 }
 
-int asix_write_medium_mode(struct usbnet *dev, u16 mode)
+int asix_write_medium_mode(struct usbnet *dev, u16 mode, int in_pm)
 {
 	int ret;
 
 	netdev_dbg(dev->net, "asix_write_medium_mode() - mode = 0x%04x\n", mode);
-	ret = asix_write_cmd(dev, AX_CMD_WRITE_MEDIUM_MODE, mode, 0, 0, NULL);
+	ret = asix_write_cmd(dev, AX_CMD_WRITE_MEDIUM_MODE,
+			     mode, 0, 0, NULL, in_pm);
 	if (ret < 0)
 		netdev_err(dev->net, "Failed to write Medium Mode mode to 0x%04x: %02x\n",
 			   mode, ret);
@@ -336,12 +364,12 @@ int asix_write_medium_mode(struct usbnet *dev, u16 mode)
 	return ret;
 }
 
-int asix_write_gpio(struct usbnet *dev, u16 value, int sleep)
+int asix_write_gpio(struct usbnet *dev, u16 value, int sleep, int in_pm)
 {
 	int ret;
 
 	netdev_dbg(dev->net, "asix_write_gpio() - value = 0x%04x\n", value);
-	ret = asix_write_cmd(dev, AX_CMD_WRITE_GPIOS, value, 0, 0, NULL);
+	ret = asix_write_cmd(dev, AX_CMD_WRITE_GPIOS, value, 0, 0, NULL, in_pm);
 	if (ret < 0)
 		netdev_err(dev->net, "Failed to write GPIO value 0x%04x: %02x\n",
 			   value, ret);
@@ -398,16 +426,31 @@ int asix_mdio_read(struct net_device *netdev, int phy_id, int loc)
 {
 	struct usbnet *dev = netdev_priv(netdev);
 	__le16 res;
+	u8 smsr;
+	int i = 0;
+	int ret;
 
 	mutex_lock(&dev->phy_mutex);
-	asix_set_sw_mii(dev);
+	do {
+		ret = asix_set_sw_mii(dev, 0);
+		if (ret == -ENODEV)
+			break;
+		usleep_range(1000, 1100);
+		ret = asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG,
+				    0, 0, 1, &smsr, 0);
+	} while (!(smsr & AX_HOST_EN) && (i++ < 30) && (ret != -ENODEV));
+	if (ret == -ENODEV) {
+		mutex_unlock(&dev->phy_mutex);
+		return ret;
+	}
+
 	asix_read_cmd(dev, AX_CMD_READ_MII_REG, phy_id,
-				(__u16)loc, 2, &res);
-	asix_set_hw_mii(dev);
+				(__u16)loc, 2, &res, 0);
+	asix_set_hw_mii(dev, 0);
 	mutex_unlock(&dev->phy_mutex);
 
 	netdev_dbg(dev->net, "asix_mdio_read() phy_id=0x%02x, loc=0x%02x, returns=0x%04x\n",
-		   phy_id, loc, le16_to_cpu(res));
+			phy_id, loc, le16_to_cpu(res));
 
 	return le16_to_cpu(res);
 }
@@ -416,13 +459,95 @@ void asix_mdio_write(struct net_device *netdev, int phy_id, int loc, int val)
 {
 	struct usbnet *dev = netdev_priv(netdev);
 	__le16 res = cpu_to_le16(val);
+	u8 smsr;
+	int i = 0;
+	int ret;
 
 	netdev_dbg(dev->net, "asix_mdio_write() phy_id=0x%02x, loc=0x%02x, val=0x%04x\n",
-		   phy_id, loc, val);
+			phy_id, loc, val);
+
 	mutex_lock(&dev->phy_mutex);
-	asix_set_sw_mii(dev);
-	asix_write_cmd(dev, AX_CMD_WRITE_MII_REG, phy_id, (__u16)loc, 2, &res);
-	asix_set_hw_mii(dev);
+	do {
+		ret = asix_set_sw_mii(dev, 0);
+		if (ret == -ENODEV)
+			break;
+		usleep_range(1000, 1100);
+		ret = asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG,
+				    0, 0, 1, &smsr, 0);
+	} while (!(smsr & AX_HOST_EN) && (i++ < 30) && (ret != -ENODEV));
+	if (ret == -ENODEV) {
+		mutex_unlock(&dev->phy_mutex);
+		return;
+	}
+
+	asix_write_cmd(dev, AX_CMD_WRITE_MII_REG, phy_id,
+		       (__u16)loc, 2, &res, 0);
+	asix_set_hw_mii(dev, 0);
+	mutex_unlock(&dev->phy_mutex);
+}
+
+int asix_mdio_read_nopm(struct net_device *netdev, int phy_id, int loc)
+{
+	struct usbnet *dev = netdev_priv(netdev);
+	__le16 res;
+	u8 smsr;
+	int i = 0;
+	int ret;
+
+	mutex_lock(&dev->phy_mutex);
+	do {
+		ret = asix_set_sw_mii(dev, 1);
+		if (ret == -ENODEV)
+			break;
+		usleep_range(1000, 1100);
+		ret = asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG,
+				    0, 0, 1, &smsr, 1);
+	} while (!(smsr & AX_HOST_EN) && (i++ < 30) && (ret != -ENODEV));
+	if (ret == -ENODEV) {
+		mutex_unlock(&dev->phy_mutex);
+		return ret;
+	}
+
+	asix_read_cmd(dev, AX_CMD_READ_MII_REG, phy_id,
+		      (__u16)loc, 2, &res, 1);
+	asix_set_hw_mii(dev, 1);
+	mutex_unlock(&dev->phy_mutex);
+
+	netdev_dbg(dev->net, "asix_mdio_read_nopm() phy_id=0x%02x, loc=0x%02x, returns=0x%04x\n",
+			phy_id, loc, le16_to_cpu(res));
+
+	return le16_to_cpu(res);
+}
+
+void
+asix_mdio_write_nopm(struct net_device *netdev, int phy_id, int loc, int val)
+{
+	struct usbnet *dev = netdev_priv(netdev);
+	__le16 res = cpu_to_le16(val);
+	u8 smsr;
+	int i = 0;
+	int ret;
+
+	netdev_dbg(dev->net, "asix_mdio_write() phy_id=0x%02x, loc=0x%02x, val=0x%04x\n",
+			phy_id, loc, val);
+
+	mutex_lock(&dev->phy_mutex);
+	do {
+		ret = asix_set_sw_mii(dev, 1);
+		if (ret == -ENODEV)
+			break;
+		usleep_range(1000, 1100);
+		ret = asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG,
+				    0, 0, 1, &smsr, 1);
+	} while (!(smsr & AX_HOST_EN) && (i++ < 30) && (ret != -ENODEV));
+	if (ret == -ENODEV) {
+		mutex_unlock(&dev->phy_mutex);
+		return;
+	}
+
+	asix_write_cmd(dev, AX_CMD_WRITE_MII_REG, phy_id,
+		       (__u16)loc, 2, &res, 1);
+	asix_set_hw_mii(dev, 1);
 	mutex_unlock(&dev->phy_mutex);
 }
 
@@ -431,7 +556,8 @@ void asix_get_wol(struct net_device *net, struct ethtool_wolinfo *wolinfo)
 	struct usbnet *dev = netdev_priv(net);
 	u8 opt;
 
-	if (asix_read_cmd(dev, AX_CMD_READ_MONITOR_MODE, 0, 0, 1, &opt) < 0) {
+	if (asix_read_cmd(dev, AX_CMD_READ_MONITOR_MODE,
+			  0, 0, 1, &opt, 0) < 0) {
 		wolinfo->supported = 0;
 		wolinfo->wolopts = 0;
 		return;
@@ -455,7 +581,7 @@ int asix_set_wol(struct net_device *net, struct ethtool_wolinfo *wolinfo)
 		opt |= AX_MONITOR_MAGIC;
 
 	if (asix_write_cmd(dev, AX_CMD_WRITE_MONITOR_MODE,
-			      opt, 0, 0, NULL) < 0)
+			      opt, 0, 0, NULL, 0) < 0)
 		return -EINVAL;
 
 	return 0;
@@ -490,7 +616,7 @@ int asix_get_eeprom(struct net_device *net, struct ethtool_eeprom *eeprom,
 	/* ax8817x returns 2 bytes from eeprom on read */
 	for (i = first_word; i <= last_word; i++) {
 		if (asix_read_cmd(dev, AX_CMD_READ_EEPROM, i, 0, 2,
-				  &(eeprom_buff[i - first_word])) < 0) {
+				  &eeprom_buff[i - first_word], 0) < 0) {
 			kfree(eeprom_buff);
 			return -EIO;
 		}
@@ -531,7 +657,7 @@ int asix_set_eeprom(struct net_device *net, struct ethtool_eeprom *eeprom,
 	   the EEPROM */
 	if (eeprom->offset & 1) {
 		ret = asix_read_cmd(dev, AX_CMD_READ_EEPROM, first_word, 0, 2,
-				    &(eeprom_buff[0]));
+				    &eeprom_buff[0], 0);
 		if (ret < 0) {
 			netdev_err(net, "Failed to read EEPROM at offset 0x%02x.\n", first_word);
 			goto free;
@@ -540,7 +666,7 @@ int asix_set_eeprom(struct net_device *net, struct ethtool_eeprom *eeprom,
 
 	if ((eeprom->offset + eeprom->len) & 1) {
 		ret = asix_read_cmd(dev, AX_CMD_READ_EEPROM, last_word, 0, 2,
-				    &(eeprom_buff[last_word - first_word]));
+				    &eeprom_buff[last_word - first_word], 0);
 		if (ret < 0) {
 			netdev_err(net, "Failed to read EEPROM at offset 0x%02x.\n", last_word);
 			goto free;
@@ -550,7 +676,7 @@ int asix_set_eeprom(struct net_device *net, struct ethtool_eeprom *eeprom,
 	memcpy((u8 *)eeprom_buff + (eeprom->offset & 1), data, eeprom->len);
 
 	/* write data to EEPROM */
-	ret = asix_write_cmd(dev, AX_CMD_WRITE_ENABLE, 0x0000, 0, 0, NULL);
+	ret = asix_write_cmd(dev, AX_CMD_WRITE_ENABLE, 0x0000, 0, 0, NULL, 0);
 	if (ret < 0) {
 		netdev_err(net, "Failed to enable EEPROM write\n");
 		goto free;
@@ -561,7 +687,7 @@ int asix_set_eeprom(struct net_device *net, struct ethtool_eeprom *eeprom,
 		netdev_dbg(net, "write to EEPROM at offset 0x%02x, data 0x%04x\n",
 			   i, eeprom_buff[i - first_word]);
 		ret = asix_write_cmd(dev, AX_CMD_WRITE_EEPROM, i,
-				     eeprom_buff[i - first_word], 0, NULL);
+				     eeprom_buff[i - first_word], 0, NULL, 0);
 		if (ret < 0) {
 			netdev_err(net, "Failed to write EEPROM at offset 0x%02x.\n",
 				   i);
@@ -570,7 +696,7 @@ int asix_set_eeprom(struct net_device *net, struct ethtool_eeprom *eeprom,
 		msleep(20);
 	}
 
-	ret = asix_write_cmd(dev, AX_CMD_WRITE_DISABLE, 0x0000, 0, 0, NULL);
+	ret = asix_write_cmd(dev, AX_CMD_WRITE_DISABLE, 0x0000, 0, 0, NULL, 0);
 	if (ret < 0) {
 		netdev_err(net, "Failed to disable EEPROM write\n");
 		goto free;
diff --git a/drivers/net/usb/asix_devices.c b/drivers/net/usb/asix_devices.c
index 5cabefc23494..cce24950a0ab 100644
--- a/drivers/net/usb/asix_devices.c
+++ b/drivers/net/usb/asix_devices.c
@@ -35,6 +35,15 @@
 
 #define	PHY_MODE_RTL8211CL	0x000C
 
+#define AX88772A_PHY14H		0x14
+#define AX88772A_PHY14H_DEFAULT 0x442C
+
+#define AX88772A_PHY15H		0x15
+#define AX88772A_PHY15H_DEFAULT 0x03C8
+
+#define AX88772A_PHY16H		0x16
+#define AX88772A_PHY16H_DEFAULT 0x4044
+
 struct ax88172_int_data {
 	__le16 res1;
 	u8 link;
@@ -79,6 +88,8 @@ static u32 asix_get_phyid(struct usbnet *dev)
 	/* Poll for the rare case the FW or phy isn't ready yet.  */
 	for (i = 0; i < 100; i++) {
 		phy_reg = asix_mdio_read(dev->net, dev->mii.phy_id, MII_PHYSID1);
+		if (phy_reg < 0)
+			return 0;
 		if (phy_reg != 0 && phy_reg != 0xFFFF)
 			break;
 		mdelay(1);
@@ -184,7 +195,7 @@ static int ax88172_link_reset(struct usbnet *dev)
 	netdev_dbg(dev->net, "ax88172_link_reset() speed: %u duplex: %d setting mode to 0x%04x\n",
 		   ethtool_cmd_speed(&ecmd), ecmd.duplex, mode);
 
-	asix_write_medium_mode(dev, mode);
+	asix_write_medium_mode(dev, mode, 0);
 
 	return 0;
 }
@@ -201,6 +212,28 @@ static const struct net_device_ops ax88172_netdev_ops = {
 	.ndo_set_rx_mode	= ax88172_set_multicast,
 };
 
+static void asix_phy_reset(struct usbnet *dev, unsigned int reset_bits)
+{
+	unsigned int timeout = 5000;
+
+	asix_mdio_write(dev->net, dev->mii.phy_id, MII_BMCR, reset_bits);
+
+	/* give phy_id a chance to process reset */
+	udelay(500);
+
+	/* See IEEE 802.3 "22.2.4.1.1 Reset": 500ms max */
+	while (timeout--) {
+		if (asix_mdio_read(dev->net, dev->mii.phy_id, MII_BMCR)
+							& BMCR_RESET)
+			udelay(100);
+		else
+			return;
+	}
+
+	netdev_err(dev->net, "BMCR_RESET timeout on phy_id %d\n",
+		   dev->mii.phy_id);
+}
+
 static int ax88172_bind(struct usbnet *dev, struct usb_interface *intf)
 {
 	int ret = 0;
@@ -213,18 +246,19 @@ static int ax88172_bind(struct usbnet *dev, struct usb_interface *intf)
 	/* Toggle the GPIOs in a manufacturer/model specific way */
 	for (i = 2; i >= 0; i--) {
 		ret = asix_write_cmd(dev, AX_CMD_WRITE_GPIOS,
-				(gpio_bits >> (i * 8)) & 0xff, 0, 0, NULL);
+				(gpio_bits >> (i * 8)) & 0xff, 0, 0, NULL, 0);
 		if (ret < 0)
 			goto out;
 		msleep(5);
 	}
 
-	ret = asix_write_rx_ctl(dev, 0x80);
+	ret = asix_write_rx_ctl(dev, 0x80, 0);
 	if (ret < 0)
 		goto out;
 
 	/* Get the MAC address */
-	ret = asix_read_cmd(dev, AX88172_CMD_READ_NODE_ID, 0, 0, ETH_ALEN, buf);
+	ret = asix_read_cmd(dev, AX88172_CMD_READ_NODE_ID,
+			    0, 0, ETH_ALEN, buf, 0);
 	if (ret < 0) {
 		netdev_dbg(dev->net, "read AX_CMD_READ_NODE_ID failed: %d\n",
 			   ret);
@@ -246,7 +280,7 @@ static int ax88172_bind(struct usbnet *dev, struct usb_interface *intf)
 	dev->net->needed_headroom = 4; /* cf asix_tx_fixup() */
 	dev->net->needed_tailroom = 4; /* cf asix_tx_fixup() */
 
-	asix_mdio_write(dev->net, dev->mii.phy_id, MII_BMCR, BMCR_RESET);
+	asix_phy_reset(dev, BMCR_RESET);
 	asix_mdio_write(dev->net, dev->mii.phy_id, MII_ADVERTISE,
 		ADVERTISE_ALL | ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
 	mii_nway_restart(&dev->mii);
@@ -290,7 +324,7 @@ static int ax88772_link_reset(struct usbnet *dev)
 	netdev_dbg(dev->net, "ax88772_link_reset() speed: %u duplex: %d setting mode to 0x%04x\n",
 		   ethtool_cmd_speed(&ecmd), ecmd.duplex, mode);
 
-	asix_write_medium_mode(dev, mode);
+	asix_write_medium_mode(dev, mode, 0);
 
 	return 0;
 }
@@ -298,78 +332,218 @@ static int ax88772_link_reset(struct usbnet *dev)
 static int ax88772_reset(struct usbnet *dev)
 {
 	struct asix_data *data = (struct asix_data *)&dev->data;
+	int ret;
+
+	/* Rewrite MAC address */
+	ether_addr_copy(data->mac_addr, dev->net->dev_addr);
+	ret = asix_write_cmd(dev, AX_CMD_WRITE_NODE_ID, 0, 0,
+			     ETH_ALEN, data->mac_addr, 0);
+	if (ret < 0)
+		goto out;
+
+	/* Set RX_CTL to default values with 2k buffer, and enable cactus */
+	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL, 0);
+	if (ret < 0)
+		goto out;
+
+	asix_write_medium_mode(dev, AX88772_MEDIUM_DEFAULT, 0);
+	if (ret < 0)
+		goto out;
+
+	return 0;
+
+out:
+	return ret;
+}
+
+static int ax88772_hw_reset(struct usbnet *dev, int in_pm)
+{
+	struct asix_data *data = (struct asix_data *)&dev->data;
 	int ret, embd_phy;
 	u16 rx_ctl;
 
-	ret = asix_write_gpio(dev,
-			AX_GPIO_RSE | AX_GPIO_GPO_2 | AX_GPIO_GPO2EN, 5);
+	ret = asix_write_gpio(dev, AX_GPIO_RSE | AX_GPIO_GPO_2 |
+			      AX_GPIO_GPO2EN, 5, in_pm);
 	if (ret < 0)
 		goto out;
 
-	embd_phy = ((asix_get_phy_addr(dev) & 0x1f) == 0x10 ? 1 : 0);
+	embd_phy = ((dev->mii.phy_id & 0x1f) == 0x10 ? 1 : 0);
 
-	ret = asix_write_cmd(dev, AX_CMD_SW_PHY_SELECT, embd_phy, 0, 0, NULL);
+	ret = asix_write_cmd(dev, AX_CMD_SW_PHY_SELECT, embd_phy,
+			     0, 0, NULL, in_pm);
 	if (ret < 0) {
 		netdev_dbg(dev->net, "Select PHY #1 failed: %d\n", ret);
 		goto out;
 	}
 
-	ret = asix_sw_reset(dev, AX_SWRESET_IPPD | AX_SWRESET_PRL);
-	if (ret < 0)
-		goto out;
+	if (embd_phy) {
+		ret = asix_sw_reset(dev, AX_SWRESET_IPPD, in_pm);
+		if (ret < 0)
+			goto out;
 
-	msleep(150);
+		usleep_range(10000, 11000);
 
-	ret = asix_sw_reset(dev, AX_SWRESET_CLEAR);
-	if (ret < 0)
-		goto out;
+		ret = asix_sw_reset(dev, AX_SWRESET_CLEAR, in_pm);
+		if (ret < 0)
+			goto out;
 
-	msleep(150);
+		msleep(60);
 
-	if (embd_phy) {
-		ret = asix_sw_reset(dev, AX_SWRESET_IPRL);
+		ret = asix_sw_reset(dev, AX_SWRESET_IPRL | AX_SWRESET_PRL,
+				    in_pm);
 		if (ret < 0)
 			goto out;
 	} else {
-		ret = asix_sw_reset(dev, AX_SWRESET_PRTE);
+		ret = asix_sw_reset(dev, AX_SWRESET_IPPD | AX_SWRESET_PRL,
+				    in_pm);
 		if (ret < 0)
 			goto out;
 	}
 
 	msleep(150);
-	rx_ctl = asix_read_rx_ctl(dev);
-	netdev_dbg(dev->net, "RX_CTL is 0x%04x after software reset\n", rx_ctl);
-	ret = asix_write_rx_ctl(dev, 0x0000);
+
+	if (in_pm && (!asix_mdio_read_nopm(dev->net, dev->mii.phy_id,
+					   MII_PHYSID1))){
+		ret = -EIO;
+		goto out;
+	}
+
+	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL, in_pm);
 	if (ret < 0)
 		goto out;
 
-	rx_ctl = asix_read_rx_ctl(dev);
-	netdev_dbg(dev->net, "RX_CTL is 0x%04x setting to 0x0000\n", rx_ctl);
+	ret = asix_write_medium_mode(dev, AX88772_MEDIUM_DEFAULT, in_pm);
+	if (ret < 0)
+		goto out;
+
+	ret = asix_write_cmd(dev, AX_CMD_WRITE_IPG0,
+			     AX88772_IPG0_DEFAULT | AX88772_IPG1_DEFAULT,
+			     AX88772_IPG2_DEFAULT, 0, NULL, in_pm);
+	if (ret < 0) {
+		netdev_dbg(dev->net, "Write IPG,IPG1,IPG2 failed: %d\n", ret);
+		goto out;
+	}
+
+	/* Rewrite MAC address */
+	ether_addr_copy(data->mac_addr, dev->net->dev_addr);
+	ret = asix_write_cmd(dev, AX_CMD_WRITE_NODE_ID, 0, 0,
+			     ETH_ALEN, data->mac_addr, in_pm);
+	if (ret < 0)
+		goto out;
 
-	ret = asix_sw_reset(dev, AX_SWRESET_PRL);
+	/* Set RX_CTL to default values with 2k buffer, and enable cactus */
+	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL, in_pm);
 	if (ret < 0)
 		goto out;
 
-	msleep(150);
+	rx_ctl = asix_read_rx_ctl(dev, in_pm);
+	netdev_dbg(dev->net, "RX_CTL is 0x%04x after all initializations\n",
+		   rx_ctl);
 
-	ret = asix_sw_reset(dev, AX_SWRESET_IPRL | AX_SWRESET_PRL);
+	rx_ctl = asix_read_medium_status(dev, in_pm);
+	netdev_dbg(dev->net,
+		   "Medium Status is 0x%04x after all initializations\n",
+		   rx_ctl);
+
+	return 0;
+
+out:
+	return ret;
+}
+
+static int ax88772a_hw_reset(struct usbnet *dev, int in_pm)
+{
+	struct asix_data *data = (struct asix_data *)&dev->data;
+	int ret, embd_phy;
+	u16 rx_ctl, phy14h, phy15h, phy16h;
+	u8 chipcode = 0;
+
+	ret = asix_write_gpio(dev, AX_GPIO_RSE, 5, in_pm);
 	if (ret < 0)
 		goto out;
 
-	msleep(150);
+	embd_phy = ((dev->mii.phy_id & 0x1f) == 0x10 ? 1 : 0);
 
-	asix_mdio_write(dev->net, dev->mii.phy_id, MII_BMCR, BMCR_RESET);
-	asix_mdio_write(dev->net, dev->mii.phy_id, MII_ADVERTISE,
-			ADVERTISE_ALL | ADVERTISE_CSMA);
-	mii_nway_restart(&dev->mii);
+	ret = asix_write_cmd(dev, AX_CMD_SW_PHY_SELECT, embd_phy |
+			     AX_PHYSEL_SSEN, 0, 0, NULL, in_pm);
+	if (ret < 0) {
+		netdev_dbg(dev->net, "Select PHY #1 failed: %d\n", ret);
+		goto out;
+	}
+	usleep_range(10000, 11000);
+
+	ret = asix_sw_reset(dev, AX_SWRESET_IPPD | AX_SWRESET_IPRL, in_pm);
+	if (ret < 0)
+		goto out;
+
+	usleep_range(10000, 11000);
+
+	ret = asix_sw_reset(dev, AX_SWRESET_IPRL, in_pm);
+	if (ret < 0)
+		goto out;
+
+	msleep(160);
+
+	ret = asix_sw_reset(dev, AX_SWRESET_CLEAR, in_pm);
+	if (ret < 0)
+		goto out;
 
-	ret = asix_write_medium_mode(dev, AX88772_MEDIUM_DEFAULT);
+	ret = asix_sw_reset(dev, AX_SWRESET_IPRL, in_pm);
 	if (ret < 0)
 		goto out;
 
+	msleep(200);
+
+	if (in_pm && (!asix_mdio_read_nopm(dev->net, dev->mii.phy_id,
+					   MII_PHYSID1))) {
+		ret = -1;
+		goto out;
+	}
+
+	ret = asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG, 0,
+			    0, 1, &chipcode, in_pm);
+	if (ret < 0)
+		goto out;
+
+	if ((chipcode & AX_CHIPCODE_MASK) == AX_AX88772B_CHIPCODE) {
+		ret = asix_write_cmd(dev, AX_QCTCTRL, 0x8000, 0x8001,
+				     0, NULL, in_pm);
+		if (ret < 0) {
+			netdev_dbg(dev->net, "Write BQ setting failed: %d\n",
+				   ret);
+			goto out;
+		}
+	} else if ((chipcode & AX_CHIPCODE_MASK) == AX_AX88772A_CHIPCODE) {
+		/* Check if the PHY registers have default settings */
+		phy14h = asix_mdio_read_nopm(dev->net, dev->mii.phy_id,
+					     AX88772A_PHY14H);
+		phy15h = asix_mdio_read_nopm(dev->net, dev->mii.phy_id,
+					     AX88772A_PHY15H);
+		phy16h = asix_mdio_read_nopm(dev->net, dev->mii.phy_id,
+					     AX88772A_PHY16H);
+
+		netdev_dbg(dev->net,
+			   "772a_hw_reset: MR20=0x%x MR21=0x%x MR22=0x%x\n",
+			   phy14h, phy15h, phy16h);
+
+		/* Restore PHY registers default setting if not */
+		if (phy14h != AX88772A_PHY14H_DEFAULT)
+			asix_mdio_write_nopm(dev->net, dev->mii.phy_id,
+					     AX88772A_PHY14H,
+					     AX88772A_PHY14H_DEFAULT);
+		if (phy15h != AX88772A_PHY15H_DEFAULT)
+			asix_mdio_write_nopm(dev->net, dev->mii.phy_id,
+					     AX88772A_PHY15H,
+					     AX88772A_PHY15H_DEFAULT);
+		if (phy16h != AX88772A_PHY16H_DEFAULT)
+			asix_mdio_write_nopm(dev->net, dev->mii.phy_id,
+					     AX88772A_PHY16H,
+					     AX88772A_PHY16H_DEFAULT);
+	}
+
 	ret = asix_write_cmd(dev, AX_CMD_WRITE_IPG0,
 				AX88772_IPG0_DEFAULT | AX88772_IPG1_DEFAULT,
-				AX88772_IPG2_DEFAULT, 0, NULL);
+				AX88772_IPG2_DEFAULT, 0, NULL, in_pm);
 	if (ret < 0) {
 		netdev_dbg(dev->net, "Write IPG,IPG1,IPG2 failed: %d\n", ret);
 		goto out;
@@ -378,20 +552,29 @@ static int ax88772_reset(struct usbnet *dev)
 	/* Rewrite MAC address */
 	memcpy(data->mac_addr, dev->net->dev_addr, ETH_ALEN);
 	ret = asix_write_cmd(dev, AX_CMD_WRITE_NODE_ID, 0, 0, ETH_ALEN,
-							data->mac_addr);
+							data->mac_addr, in_pm);
 	if (ret < 0)
 		goto out;
 
 	/* Set RX_CTL to default values with 2k buffer, and enable cactus */
-	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL);
+	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL, in_pm);
 	if (ret < 0)
 		goto out;
 
-	rx_ctl = asix_read_rx_ctl(dev);
+	ret = asix_write_medium_mode(dev, AX88772_MEDIUM_DEFAULT, in_pm);
+	if (ret < 0)
+		return ret;
+
+	/* Set RX_CTL to default values with 2k buffer, and enable cactus */
+	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL, in_pm);
+	if (ret < 0)
+		goto out;
+
+	rx_ctl = asix_read_rx_ctl(dev, in_pm);
 	netdev_dbg(dev->net, "RX_CTL is 0x%04x after all initializations\n",
 		   rx_ctl);
 
-	rx_ctl = asix_read_medium_status(dev);
+	rx_ctl = asix_read_medium_status(dev, in_pm);
 	netdev_dbg(dev->net,
 		   "Medium Status is 0x%04x after all initializations\n",
 		   rx_ctl);
@@ -400,7 +583,6 @@ static int ax88772_reset(struct usbnet *dev)
 
 out:
 	return ret;
-
 }
 
 static const struct net_device_ops ax88772_netdev_ops = {
@@ -415,11 +597,97 @@ static const struct net_device_ops ax88772_netdev_ops = {
 	.ndo_set_rx_mode        = asix_set_multicast,
 };
 
+static void ax88772_suspend(struct usbnet *dev)
+{
+	struct asix_common_private *priv = dev->driver_priv;
+	u16 medium;
+
+	/* Stop MAC operation */
+	medium = asix_read_medium_status(dev, 0);
+	medium &= ~AX_MEDIUM_RE;
+	asix_write_medium_mode(dev, medium, 0);
+
+	netdev_dbg(dev->net, "ax88772_suspend: medium=0x%04x\n",
+		   asix_read_medium_status(dev, 0));
+
+	/* Preserve BMCR for restoring */
+	priv->presvd_phy_bmcr =
+		asix_mdio_read_nopm(dev->net, dev->mii.phy_id, MII_BMCR);
+
+	/* Preserve ANAR for restoring */
+	priv->presvd_phy_advertise =
+		asix_mdio_read_nopm(dev->net, dev->mii.phy_id, MII_ADVERTISE);
+}
+
+static int asix_suspend(struct usb_interface *intf, pm_message_t message)
+{
+	struct usbnet *dev = usb_get_intfdata(intf);
+	struct asix_common_private *priv = dev->driver_priv;
+
+	if (priv->suspend)
+		priv->suspend(dev);
+
+	return usbnet_suspend(intf, message);
+}
+
+static void ax88772_restore_phy(struct usbnet *dev)
+{
+	struct asix_common_private *priv = dev->driver_priv;
+
+	if (priv->presvd_phy_advertise) {
+		/* Restore Advertisement control reg */
+		asix_mdio_write_nopm(dev->net, dev->mii.phy_id, MII_ADVERTISE,
+				     priv->presvd_phy_advertise);
+
+		/* Restore BMCR */
+		asix_mdio_write_nopm(dev->net, dev->mii.phy_id, MII_BMCR,
+				     priv->presvd_phy_bmcr);
+
+		mii_nway_restart(&dev->mii);
+		priv->presvd_phy_advertise = 0;
+		priv->presvd_phy_bmcr = 0;
+	}
+}
+
+static void ax88772_resume(struct usbnet *dev)
+{
+	int i;
+
+	for (i = 0; i < 3; i++)
+		if (!ax88772_hw_reset(dev, 1))
+			break;
+	ax88772_restore_phy(dev);
+}
+
+static void ax88772a_resume(struct usbnet *dev)
+{
+	int i;
+
+	for (i = 0; i < 3; i++) {
+		if (!ax88772a_hw_reset(dev, 1))
+			break;
+	}
+
+	ax88772_restore_phy(dev);
+}
+
+static int asix_resume(struct usb_interface *intf)
+{
+	struct usbnet *dev = usb_get_intfdata(intf);
+	struct asix_common_private *priv = dev->driver_priv;
+
+	if (priv->resume)
+		priv->resume(dev);
+
+	return usbnet_resume(intf);
+}
+
 static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf)
 {
-	int ret, embd_phy, i;
-	u8 buf[ETH_ALEN];
+	int ret, i;
+	u8 buf[ETH_ALEN], chipcode = 0;
 	u32 phyid;
+	struct asix_common_private *priv;
 
 	usbnet_get_endpoints(dev,intf);
 
@@ -427,13 +695,13 @@ static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf)
 	if (dev->driver_info->data & FLAG_EEPROM_MAC) {
 		for (i = 0; i < (ETH_ALEN >> 1); i++) {
 			ret = asix_read_cmd(dev, AX_CMD_READ_EEPROM, 0x04 + i,
-					0, 2, buf + i * 2);
+					    0, 2, buf + i * 2, 0);
 			if (ret < 0)
 				break;
 		}
 	} else {
 		ret = asix_read_cmd(dev, AX_CMD_READ_NODE_ID,
-				0, 0, ETH_ALEN, buf);
+				0, 0, ETH_ALEN, buf, 0);
 	}
 
 	if (ret < 0) {
@@ -456,16 +724,11 @@ static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf)
 	dev->net->needed_headroom = 4; /* cf asix_tx_fixup() */
 	dev->net->needed_tailroom = 4; /* cf asix_tx_fixup() */
 
-	embd_phy = ((dev->mii.phy_id & 0x1f) == 0x10 ? 1 : 0);
-
-	/* Reset the PHY to normal operation mode */
-	ret = asix_write_cmd(dev, AX_CMD_SW_PHY_SELECT, embd_phy, 0, 0, NULL);
-	if (ret < 0) {
-		netdev_dbg(dev->net, "Select PHY #1 failed: %d\n", ret);
-		return ret;
-	}
+	asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG, 0, 0, 1, &chipcode, 0);
+	chipcode &= AX_CHIPCODE_MASK;
 
-	ax88772_reset(dev);
+	(chipcode == AX_AX88772_CHIPCODE) ? ax88772_hw_reset(dev, 0) :
+					    ax88772a_hw_reset(dev, 0);
 
 	/* Read PHYID register *AFTER* the PHY was reset properly */
 	phyid = asix_get_phyid(dev);
@@ -482,6 +745,18 @@ static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf)
 	if (!dev->driver_priv)
 		return -ENOMEM;
 
+	priv = dev->driver_priv;
+
+	priv->presvd_phy_bmcr = 0;
+	priv->presvd_phy_advertise = 0;
+	if (chipcode == AX_AX88772_CHIPCODE) {
+		priv->resume = ax88772_resume;
+		priv->suspend = ax88772_suspend;
+	} else {
+		priv->resume = ax88772a_resume;
+		priv->suspend = ax88772_suspend;
+	}
+
 	return 0;
 }
 
@@ -593,12 +868,12 @@ static int ax88178_reset(struct usbnet *dev)
 	int gpio0 = 0;
 	u32 phyid;
 
-	asix_read_cmd(dev, AX_CMD_READ_GPIOS, 0, 0, 1, &status);
+	asix_read_cmd(dev, AX_CMD_READ_GPIOS, 0, 0, 1, &status, 0);
 	netdev_dbg(dev->net, "GPIO Status: 0x%04x\n", status);
 
-	asix_write_cmd(dev, AX_CMD_WRITE_ENABLE, 0, 0, 0, NULL);
-	asix_read_cmd(dev, AX_CMD_READ_EEPROM, 0x0017, 0, 2, &eeprom);
-	asix_write_cmd(dev, AX_CMD_WRITE_DISABLE, 0, 0, 0, NULL);
+	asix_write_cmd(dev, AX_CMD_WRITE_ENABLE, 0, 0, 0, NULL, 0);
+	asix_read_cmd(dev, AX_CMD_READ_EEPROM, 0x0017, 0, 2, &eeprom, 0);
+	asix_write_cmd(dev, AX_CMD_WRITE_DISABLE, 0, 0, 0, NULL, 0);
 
 	netdev_dbg(dev->net, "EEPROM index 0x17 is 0x%04x\n", eeprom);
 
@@ -614,15 +889,16 @@ static int ax88178_reset(struct usbnet *dev)
 	netdev_dbg(dev->net, "GPIO0: %d, PhyMode: %d\n", gpio0, data->phymode);
 
 	/* Power up external GigaPHY through AX88178 GPIO pin */
-	asix_write_gpio(dev, AX_GPIO_RSE | AX_GPIO_GPO_1 | AX_GPIO_GPO1EN, 40);
+	asix_write_gpio(dev, AX_GPIO_RSE | AX_GPIO_GPO_1 |
+			AX_GPIO_GPO1EN, 40, 0);
 	if ((le16_to_cpu(eeprom) >> 8) != 1) {
-		asix_write_gpio(dev, 0x003c, 30);
-		asix_write_gpio(dev, 0x001c, 300);
-		asix_write_gpio(dev, 0x003c, 30);
+		asix_write_gpio(dev, 0x003c, 30, 0);
+		asix_write_gpio(dev, 0x001c, 300, 0);
+		asix_write_gpio(dev, 0x003c, 30, 0);
 	} else {
 		netdev_dbg(dev->net, "gpio phymode == 1 path\n");
-		asix_write_gpio(dev, AX_GPIO_GPO1EN, 30);
-		asix_write_gpio(dev, AX_GPIO_GPO1EN | AX_GPIO_GPO_1, 30);
+		asix_write_gpio(dev, AX_GPIO_GPO1EN, 30, 0);
+		asix_write_gpio(dev, AX_GPIO_GPO1EN | AX_GPIO_GPO_1, 30, 0);
 	}
 
 	/* Read PHYID register *AFTER* powering up PHY */
@@ -630,15 +906,15 @@ static int ax88178_reset(struct usbnet *dev)
 	netdev_dbg(dev->net, "PHYID=0x%08x\n", phyid);
 
 	/* Set AX88178 to enable MII/GMII/RGMII interface for external PHY */
-	asix_write_cmd(dev, AX_CMD_SW_PHY_SELECT, 0, 0, 0, NULL);
+	asix_write_cmd(dev, AX_CMD_SW_PHY_SELECT, 0, 0, 0, NULL, 0);
 
-	asix_sw_reset(dev, 0);
+	asix_sw_reset(dev, 0, 0);
 	msleep(150);
 
-	asix_sw_reset(dev, AX_SWRESET_PRL | AX_SWRESET_IPPD);
+	asix_sw_reset(dev, AX_SWRESET_PRL | AX_SWRESET_IPPD, 0);
 	msleep(150);
 
-	asix_write_rx_ctl(dev, 0);
+	asix_write_rx_ctl(dev, 0, 0);
 
 	if (data->phymode == PHY_MODE_MARVELL) {
 		marvell_phy_init(dev);
@@ -646,27 +922,23 @@ static int ax88178_reset(struct usbnet *dev)
 	} else if (data->phymode == PHY_MODE_RTL8211CL)
 		rtl8211cl_phy_init(dev);
 
-	asix_mdio_write(dev->net, dev->mii.phy_id, MII_BMCR,
-			BMCR_RESET | BMCR_ANENABLE);
+	asix_phy_reset(dev, BMCR_RESET | BMCR_ANENABLE);
 	asix_mdio_write(dev->net, dev->mii.phy_id, MII_ADVERTISE,
 			ADVERTISE_ALL | ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
 	asix_mdio_write(dev->net, dev->mii.phy_id, MII_CTRL1000,
 			ADVERTISE_1000FULL);
 
+	asix_write_medium_mode(dev, AX88178_MEDIUM_DEFAULT, 0);
 	mii_nway_restart(&dev->mii);
 
-	ret = asix_write_medium_mode(dev, AX88178_MEDIUM_DEFAULT);
-	if (ret < 0)
-		return ret;
-
 	/* Rewrite MAC address */
 	memcpy(data->mac_addr, dev->net->dev_addr, ETH_ALEN);
 	ret = asix_write_cmd(dev, AX_CMD_WRITE_NODE_ID, 0, 0, ETH_ALEN,
-							data->mac_addr);
+							data->mac_addr, 0);
 	if (ret < 0)
 		return ret;
 
-	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL);
+	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL, 0);
 	if (ret < 0)
 		return ret;
 
@@ -704,7 +976,7 @@ static int ax88178_link_reset(struct usbnet *dev)
 	netdev_dbg(dev->net, "ax88178_link_reset() speed: %u duplex: %d setting mode to 0x%04x\n",
 		   speed, ecmd.duplex, mode);
 
-	asix_write_medium_mode(dev, mode);
+	asix_write_medium_mode(dev, mode, 0);
 
 	if (data->phymode == PHY_MODE_MARVELL && data->ledmode)
 		marvell_led_status(dev, speed);
@@ -733,15 +1005,15 @@ static void ax88178_set_mfb(struct usbnet *dev)
 		mfb = AX_RX_CTL_MFB_16384;
 	}
 
-	rxctl = asix_read_rx_ctl(dev);
-	asix_write_rx_ctl(dev, (rxctl & ~AX_RX_CTL_MFB_16384) | mfb);
+	rxctl = asix_read_rx_ctl(dev, 0);
+	asix_write_rx_ctl(dev, (rxctl & ~AX_RX_CTL_MFB_16384) | mfb, 0);
 
-	medium = asix_read_medium_status(dev);
+	medium = asix_read_medium_status(dev, 0);
 	if (dev->net->mtu > 1500)
 		medium |= AX_MEDIUM_JFE;
 	else
 		medium &= ~AX_MEDIUM_JFE;
-	asix_write_medium_mode(dev, medium);
+	asix_write_medium_mode(dev, medium, 0);
 
 	if (dev->rx_urb_size > old_rx_urb_size)
 		usbnet_unlink_rx_urbs(dev);
@@ -790,7 +1062,7 @@ static int ax88178_bind(struct usbnet *dev, struct usb_interface *intf)
 	usbnet_get_endpoints(dev,intf);
 
 	/* Get the MAC address */
-	ret = asix_read_cmd(dev, AX_CMD_READ_NODE_ID, 0, 0, ETH_ALEN, buf);
+	ret = asix_read_cmd(dev, AX_CMD_READ_NODE_ID, 0, 0, ETH_ALEN, buf, 0);
 	if (ret < 0) {
 		netdev_dbg(dev->net, "Failed to read MAC address: %d\n", ret);
 		return ret;
@@ -811,10 +1083,10 @@ static int ax88178_bind(struct usbnet *dev, struct usb_interface *intf)
 	dev->net->ethtool_ops = &ax88178_ethtool_ops;
 
 	/* Blink LEDS so users know driver saw dongle */
-	asix_sw_reset(dev, 0);
+	asix_sw_reset(dev, 0, 0);
 	msleep(150);
 
-	asix_sw_reset(dev, AX_SWRESET_PRL | AX_SWRESET_IPPD);
+	asix_sw_reset(dev, AX_SWRESET_PRL | AX_SWRESET_IPPD, 0);
 	msleep(150);
 
 	/* Asix framing packs multiple eth frames into a 2K usb bulk transfer */
@@ -877,7 +1149,7 @@ static const struct driver_info ax88772_info = {
 	.unbind = ax88772_unbind,
 	.status = asix_status,
 	.link_reset = ax88772_link_reset,
-	.reset = ax88772_link_reset,
+	.reset = ax88772_reset,
 	.flags = FLAG_ETHER | FLAG_FRAMING_AX | FLAG_LINK_INTR | FLAG_MULTI_PACKET,
 	.rx_fixup = asix_rx_fixup_common,
 	.tx_fixup = asix_tx_fixup,
@@ -1005,7 +1277,7 @@ static const struct usb_device_id	products [] = {
 }, {
 	// Lenovo U2L100P 10/100
 	USB_DEVICE (0x17ef, 0x7203),
-	.driver_info = (unsigned long) &ax88772_info,
+	.driver_info = (unsigned long)&ax88772b_info,
 }, {
 	// ASIX AX88772B 10/100
 	USB_DEVICE (0x0b95, 0x772b),
@@ -1073,7 +1345,7 @@ static const struct usb_device_id	products [] = {
 }, {
 	// Asus USB Ethernet Adapter
 	USB_DEVICE (0x0b95, 0x7e2b),
-	.driver_info = (unsigned long) &ax88772_info,
+	.driver_info = (unsigned long)&ax88772b_info,
 }, {
 	/* ASIX 88172a demo board */
 	USB_DEVICE(0x0b95, 0x172a),
@@ -1095,8 +1367,8 @@ static struct usb_driver asix_driver = {
 	.name =		DRIVER_NAME,
 	.id_table =	products,
 	.probe =	usbnet_probe,
-	.suspend =	usbnet_suspend,
-	.resume =	usbnet_resume,
+	.suspend =	asix_suspend,
+	.resume =	asix_resume,
 	.disconnect =	usbnet_disconnect,
 	.supports_autosuspend = 1,
 	.disable_hub_initiated_lpm = 1,
diff --git a/drivers/net/usb/ax88172a.c b/drivers/net/usb/ax88172a.c
index 163a2c576e69..49a3bc107d05 100644
--- a/drivers/net/usb/ax88172a.c
+++ b/drivers/net/usb/ax88172a.c
@@ -81,7 +81,7 @@ static void ax88172a_adjust_link(struct net_device *netdev)
 	}
 
 	if (mode != priv->oldmode) {
-		asix_write_medium_mode(dev, mode);
+		asix_write_medium_mode(dev, mode, 0);
 		priv->oldmode = mode;
 		netdev_dbg(netdev, "speed %u duplex %d, setting mode to 0x%04x\n",
 			   phydev->speed, phydev->duplex, mode);
@@ -176,18 +176,19 @@ static int ax88172a_reset_phy(struct usbnet *dev, int embd_phy)
 {
 	int ret;
 
-	ret = asix_sw_reset(dev, AX_SWRESET_IPPD);
+	ret = asix_sw_reset(dev, AX_SWRESET_IPPD, 0);
 	if (ret < 0)
 		goto err;
 
 	msleep(150);
-	ret = asix_sw_reset(dev, AX_SWRESET_CLEAR);
+	ret = asix_sw_reset(dev, AX_SWRESET_CLEAR, 0);
 	if (ret < 0)
 		goto err;
 
 	msleep(150);
 
-	ret = asix_sw_reset(dev, embd_phy ? AX_SWRESET_IPRL : AX_SWRESET_IPPD);
+	ret = asix_sw_reset(dev, embd_phy ? AX_SWRESET_IPRL : AX_SWRESET_IPPD,
+			    0);
 	if (ret < 0)
 		goto err;
 
@@ -213,7 +214,7 @@ static int ax88172a_bind(struct usbnet *dev, struct usb_interface *intf)
 	dev->driver_priv = priv;
 
 	/* Get the MAC address */
-	ret = asix_read_cmd(dev, AX_CMD_READ_NODE_ID, 0, 0, ETH_ALEN, buf);
+	ret = asix_read_cmd(dev, AX_CMD_READ_NODE_ID, 0, 0, ETH_ALEN, buf, 0);
 	if (ret < 0) {
 		netdev_err(dev->net, "Failed to read MAC address: %d\n", ret);
 		goto free;
@@ -224,7 +225,7 @@ static int ax88172a_bind(struct usbnet *dev, struct usb_interface *intf)
 	dev->net->ethtool_ops = &ax88172a_ethtool_ops;
 
 	/* are we using the internal or the external phy? */
-	ret = asix_read_cmd(dev, AX_CMD_SW_PHY_STATUS, 0, 0, 1, buf);
+	ret = asix_read_cmd(dev, AX_CMD_SW_PHY_STATUS, 0, 0, 1, buf, 0);
 	if (ret < 0) {
 		netdev_err(dev->net, "Failed to read software interface selection register: %d\n",
 			   ret);
@@ -303,20 +304,20 @@ static int ax88172a_reset(struct usbnet *dev)
 	ax88172a_reset_phy(dev, priv->use_embdphy);
 
 	msleep(150);
-	rx_ctl = asix_read_rx_ctl(dev);
+	rx_ctl = asix_read_rx_ctl(dev, 0);
 	netdev_dbg(dev->net, "RX_CTL is 0x%04x after software reset\n", rx_ctl);
-	ret = asix_write_rx_ctl(dev, 0x0000);
+	ret = asix_write_rx_ctl(dev, 0x0000, 0);
 	if (ret < 0)
 		goto out;
 
-	rx_ctl = asix_read_rx_ctl(dev);
+	rx_ctl = asix_read_rx_ctl(dev, 0);
 	netdev_dbg(dev->net, "RX_CTL is 0x%04x setting to 0x0000\n", rx_ctl);
 
 	msleep(150);
 
 	ret = asix_write_cmd(dev, AX_CMD_WRITE_IPG0,
 			     AX88772_IPG0_DEFAULT | AX88772_IPG1_DEFAULT,
-			     AX88772_IPG2_DEFAULT, 0, NULL);
+			     AX88772_IPG2_DEFAULT, 0, NULL, 0);
 	if (ret < 0) {
 		netdev_err(dev->net, "Write IPG,IPG1,IPG2 failed: %d\n", ret);
 		goto out;
@@ -325,20 +326,20 @@ static int ax88172a_reset(struct usbnet *dev)
 	/* Rewrite MAC address */
 	memcpy(data->mac_addr, dev->net->dev_addr, ETH_ALEN);
 	ret = asix_write_cmd(dev, AX_CMD_WRITE_NODE_ID, 0, 0, ETH_ALEN,
-			     data->mac_addr);
+			     data->mac_addr, 0);
 	if (ret < 0)
 		goto out;
 
 	/* Set RX_CTL to default values with 2k buffer, and enable cactus */
-	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL);
+	ret = asix_write_rx_ctl(dev, AX_DEFAULT_RX_CTL, 0);
 	if (ret < 0)
 		goto out;
 
-	rx_ctl = asix_read_rx_ctl(dev);
+	rx_ctl = asix_read_rx_ctl(dev, 0);
 	netdev_dbg(dev->net, "RX_CTL is 0x%04x after all initializations\n",
 		   rx_ctl);
 
-	rx_ctl = asix_read_medium_status(dev);
+	rx_ctl = asix_read_medium_status(dev, 0);
 	netdev_dbg(dev->net, "Medium Status is 0x%04x after all initializations\n",
 		   rx_ctl);
 
diff --git a/drivers/net/usb/hso.c b/drivers/net/usb/hso.c
index 4b4458616693..e7b516342678 100644
--- a/drivers/net/usb/hso.c
+++ b/drivers/net/usb/hso.c
@@ -50,6 +50,8 @@
  *
  *****************************************************************************/
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/init.h>
@@ -108,23 +110,12 @@
 /*****************************************************************************/
 /* Debugging functions                                                       */
 /*****************************************************************************/
-#define D__(lvl_, fmt, arg...)				\
-	do {						\
-		printk(lvl_ "[%d:%s]: " fmt "\n",	\
-		       __LINE__, __func__, ## arg);	\
-	} while (0)
-
-#define D_(lvl, args...)				\
-	do {						\
-		if (lvl & debug)			\
-			D__(KERN_INFO, args);		\
-	} while (0)
-
-#define D1(args...)	D_(0x01, ##args)
-#define D2(args...)	D_(0x02, ##args)
-#define D3(args...)	D_(0x04, ##args)
-#define D4(args...)	D_(0x08, ##args)
-#define D5(args...)	D_(0x10, ##args)
+#define hso_dbg(lvl, fmt, ...)						\
+do {									\
+	if ((lvl) & debug)						\
+		pr_info("[%d:%s] " fmt,					\
+			__LINE__, __func__, ##__VA_ARGS__);		\
+} while (0)
 
 /*****************************************************************************/
 /* Enumerators                                                               */
@@ -649,7 +640,7 @@ static int get_free_serial_index(void)
 	}
 	spin_unlock_irqrestore(&serial_table_lock, flags);
 
-	printk(KERN_ERR "%s: no free serial devices in table\n", __func__);
+	pr_err("%s: no free serial devices in table\n", __func__);
 	return -1;
 }
 
@@ -709,7 +700,8 @@ static void handle_usb_error(int status, const char *function,
 	}
 
 	/* log a meaningful explanation of an USB status */
-	D1("%s: received USB status - %s (%d)", function, explanation, status);
+	hso_dbg(0x1, "%s: received USB status - %s (%d)\n",
+		function, explanation, status);
 }
 
 /* Network interface functions */
@@ -808,7 +800,7 @@ static netdev_tx_t hso_net_start_xmit(struct sk_buff *skb,
 	DUMP1(skb->data, skb->len);
 	/* Copy it from kernel memory to OUR memory */
 	memcpy(odev->mux_bulk_tx_buf, skb->data, skb->len);
-	D1("len: %d/%d", skb->len, MUX_BULK_TX_BUF_SIZE);
+	hso_dbg(0x1, "len: %d/%d\n", skb->len, MUX_BULK_TX_BUF_SIZE);
 
 	/* Fill in the URB for shipping it out. */
 	usb_fill_bulk_urb(odev->mux_bulk_tx_urb,
@@ -872,7 +864,7 @@ static void packetizeRx(struct hso_net *odev, unsigned char *ip_pkt,
 	unsigned char *tmp_rx_buf;
 
 	/* log if needed */
-	D1("Rx %d bytes", count);
+	hso_dbg(0x1, "Rx %d bytes\n", count);
 	DUMP(ip_pkt, min(128, (int)count));
 
 	while (count) {
@@ -912,7 +904,7 @@ static void packetizeRx(struct hso_net *odev, unsigned char *ip_pkt,
 								    frame_len);
 				if (!odev->skb_rx_buf) {
 					/* We got no receive buffer. */
-					D1("could not allocate memory");
+					hso_dbg(0x1, "could not allocate memory\n");
 					odev->rx_parse_state = WAIT_SYNC;
 					continue;
 				}
@@ -972,11 +964,11 @@ static void packetizeRx(struct hso_net *odev, unsigned char *ip_pkt,
 			break;
 
 		case WAIT_SYNC:
-			D1(" W_S");
+			hso_dbg(0x1, " W_S\n");
 			count = 0;
 			break;
 		default:
-			D1(" ");
+			hso_dbg(0x1, "\n");
 			count--;
 			break;
 		}
@@ -1020,7 +1012,7 @@ static void read_bulk_callback(struct urb *urb)
 
 	/* Sanity check */
 	if (!odev || !test_bit(HSO_NET_RUNNING, &odev->flags)) {
-		D1("BULK IN callback but driver is not active!");
+		hso_dbg(0x1, "BULK IN callback but driver is not active!\n");
 		return;
 	}
 	usb_mark_last_busy(urb->dev);
@@ -1112,11 +1104,11 @@ static void _hso_serial_set_termios(struct tty_struct *tty,
 	struct hso_serial *serial = tty->driver_data;
 
 	if (!serial) {
-		printk(KERN_ERR "%s: no tty structures", __func__);
+		pr_err("%s: no tty structures", __func__);
 		return;
 	}
 
-	D4("port %d", serial->minor);
+	hso_dbg(0x8, "port %d\n", serial->minor);
 
 	/*
 	 *	Fix up unsupported bits
@@ -1205,11 +1197,11 @@ static void hso_std_serial_read_bulk_callback(struct urb *urb)
 	struct hso_serial *serial = urb->context;
 	int status = urb->status;
 
-	D4("\n--- Got serial_read_bulk callback %02x ---", status);
+	hso_dbg(0x8, "--- Got serial_read_bulk callback %02x ---\n", status);
 
 	/* sanity check */
 	if (!serial) {
-		D1("serial == NULL");
+		hso_dbg(0x1, "serial == NULL\n");
 		return;
 	}
 	if (status) {
@@ -1217,7 +1209,7 @@ static void hso_std_serial_read_bulk_callback(struct urb *urb)
 		return;
 	}
 
-	D1("Actual length = %d\n", urb->actual_length);
+	hso_dbg(0x1, "Actual length = %d\n", urb->actual_length);
 	DUMP1(urb->transfer_buffer, urb->actual_length);
 
 	/* Anyone listening? */
@@ -1266,7 +1258,7 @@ static int hso_serial_open(struct tty_struct *tty, struct file *filp)
 	if (serial == NULL || serial->magic != HSO_SERIAL_MAGIC) {
 		WARN_ON(1);
 		tty->driver_data = NULL;
-		D1("Failed to open port");
+		hso_dbg(0x1, "Failed to open port\n");
 		return -ENODEV;
 	}
 
@@ -1275,7 +1267,7 @@ static int hso_serial_open(struct tty_struct *tty, struct file *filp)
 	if (result < 0)
 		goto err_out;
 
-	D1("Opening %d", serial->minor);
+	hso_dbg(0x1, "Opening %d\n", serial->minor);
 
 	/* setup */
 	tty->driver_data = serial;
@@ -1298,7 +1290,7 @@ static int hso_serial_open(struct tty_struct *tty, struct file *filp)
 			kref_get(&serial->parent->ref);
 		}
 	} else {
-		D1("Port was already open");
+		hso_dbg(0x1, "Port was already open\n");
 	}
 
 	usb_autopm_put_interface(serial->parent->interface);
@@ -1317,7 +1309,7 @@ static void hso_serial_close(struct tty_struct *tty, struct file *filp)
 	struct hso_serial *serial = tty->driver_data;
 	u8 usb_gone;
 
-	D1("Closing serial port");
+	hso_dbg(0x1, "Closing serial port\n");
 
 	/* Open failed, no close cleanup required */
 	if (serial == NULL)
@@ -1357,7 +1349,7 @@ static int hso_serial_write(struct tty_struct *tty, const unsigned char *buf,
 
 	/* sanity check */
 	if (serial == NULL) {
-		printk(KERN_ERR "%s: serial is NULL\n", __func__);
+		pr_err("%s: serial is NULL\n", __func__);
 		return -ENODEV;
 	}
 
@@ -1412,8 +1404,8 @@ static void hso_serial_set_termios(struct tty_struct *tty, struct ktermios *old)
 	unsigned long flags;
 
 	if (old)
-		D5("Termios called with: cflags new[%d] - old[%d]",
-		   tty->termios.c_cflag, old->c_cflag);
+		hso_dbg(0x16, "Termios called with: cflags new[%d] - old[%d]\n",
+			tty->termios.c_cflag, old->c_cflag);
 
 	/* the actual setup */
 	spin_lock_irqsave(&serial->serial_lock, flags);
@@ -1649,7 +1641,7 @@ static int hso_serial_tiocmget(struct tty_struct *tty)
 
 	/* sanity check */
 	if (!serial) {
-		D1("no tty structures");
+		hso_dbg(0x1, "no tty structures\n");
 		return -EINVAL;
 	}
 	spin_lock_irq(&serial->serial_lock);
@@ -1682,7 +1674,7 @@ static int hso_serial_tiocmset(struct tty_struct *tty,
 
 	/* sanity check */
 	if (!serial) {
-		D1("no tty structures");
+		hso_dbg(0x1, "no tty structures\n");
 		return -EINVAL;
 	}
 
@@ -1721,7 +1713,7 @@ static int hso_serial_ioctl(struct tty_struct *tty,
 {
 	struct hso_serial *serial = tty->driver_data;
 	int ret = 0;
-	D4("IOCTL cmd: %d, arg: %ld", cmd, arg);
+	hso_dbg(0x8, "IOCTL cmd: %d, arg: %ld\n", cmd, arg);
 
 	if (!serial)
 		return -ENODEV;
@@ -1783,7 +1775,7 @@ static int mux_device_request(struct hso_serial *serial, u8 type, u16 port,
 
 	/* Sanity check */
 	if (!serial || !ctrl_urb || !ctrl_req) {
-		printk(KERN_ERR "%s: Wrong arguments\n", __func__);
+		pr_err("%s: Wrong arguments\n", __func__);
 		return -EINVAL;
 	}
 
@@ -1808,9 +1800,9 @@ static int mux_device_request(struct hso_serial *serial, u8 type, u16 port,
 		pipe = usb_sndctrlpipe(serial->parent->usb, 0);
 	}
 	/* syslog */
-	D2("%s command (%02x) len: %d, port: %d",
-	   type == USB_CDC_GET_ENCAPSULATED_RESPONSE ? "Read" : "Write",
-	   ctrl_req->bRequestType, ctrl_req->wLength, port);
+	hso_dbg(0x2, "%s command (%02x) len: %d, port: %d\n",
+		type == USB_CDC_GET_ENCAPSULATED_RESPONSE ? "Read" : "Write",
+		ctrl_req->bRequestType, ctrl_req->wLength, port);
 
 	/* Load ctrl urb */
 	ctrl_urb->transfer_flags = 0;
@@ -1876,11 +1868,11 @@ static void intr_callback(struct urb *urb)
 		handle_usb_error(status, __func__, NULL);
 		return;
 	}
-	D4("\n--- Got intr callback 0x%02X ---", status);
+	hso_dbg(0x8, "--- Got intr callback 0x%02X ---\n", status);
 
 	/* what request? */
 	port_req = urb->transfer_buffer;
-	D4(" port_req = 0x%.2X\n", *port_req);
+	hso_dbg(0x8, "port_req = 0x%.2X\n", *port_req);
 	/* loop over all muxed ports to find the one sending this */
 	for (i = 0; i < 8; i++) {
 		/* max 8 channels on MUX */
@@ -1888,7 +1880,8 @@ static void intr_callback(struct urb *urb)
 			serial = get_serial_by_shared_int_and_type(shared_int,
 								   (1 << i));
 			if (serial != NULL) {
-				D1("Pending read interrupt on port %d\n", i);
+				hso_dbg(0x1, "Pending read interrupt on port %d\n",
+					i);
 				spin_lock(&serial->serial_lock);
 				if (serial->rx_state == RX_IDLE &&
 					serial->port.count > 0) {
@@ -1900,8 +1893,8 @@ static void intr_callback(struct urb *urb)
 					} else
 						serial->rx_state = RX_PENDING;
 				} else {
-					D1("Already a read pending on "
-					   "port %d or port not open\n", i);
+					hso_dbg(0x1, "Already a read pending on port %d or port not open\n",
+						i);
 				}
 				spin_unlock(&serial->serial_lock);
 			}
@@ -1933,7 +1926,7 @@ static void hso_std_serial_write_bulk_callback(struct urb *urb)
 
 	/* sanity check */
 	if (!serial) {
-		D1("serial == NULL");
+		hso_dbg(0x1, "serial == NULL\n");
 		return;
 	}
 
@@ -1948,7 +1941,7 @@ static void hso_std_serial_write_bulk_callback(struct urb *urb)
 	tty_port_tty_wakeup(&serial->port);
 	hso_kick_transmit(serial);
 
-	D1(" ");
+	hso_dbg(0x1, "\n");
 }
 
 /* called for writing diag or CS serial port */
@@ -1996,8 +1989,8 @@ static void ctrl_callback(struct urb *urb)
 
 	/* what request? */
 	req = (struct usb_ctrlrequest *)(urb->setup_packet);
-	D4("\n--- Got muxed ctrl callback 0x%02X ---", status);
-	D4("Actual length of urb = %d\n", urb->actual_length);
+	hso_dbg(0x8, "--- Got muxed ctrl callback 0x%02X ---\n", status);
+	hso_dbg(0x8, "Actual length of urb = %d\n", urb->actual_length);
 	DUMP1(urb->transfer_buffer, urb->actual_length);
 
 	if (req->bRequestType ==
@@ -2023,7 +2016,7 @@ static int put_rxbuf_data(struct urb *urb, struct hso_serial *serial)
 
 	/* Sanity check */
 	if (urb == NULL || serial == NULL) {
-		D1("serial = NULL");
+		hso_dbg(0x1, "serial = NULL\n");
 		return -2;
 	}
 
@@ -2035,7 +2028,7 @@ static int put_rxbuf_data(struct urb *urb, struct hso_serial *serial)
 	}
 
 	/* Push data to tty */
-	D1("data to push to tty");
+	hso_dbg(0x1, "data to push to tty\n");
 	count = tty_buffer_request_room(&serial->port, urb->actual_length);
 	if (count >= urb->actual_length) {
 		tty_insert_flip_string(&serial->port, urb->transfer_buffer,
@@ -2300,10 +2293,8 @@ static int hso_serial_common_create(struct hso_serial *serial, int num_urbs,
 	serial->rx_data_length = rx_size;
 	for (i = 0; i < serial->num_rx_urbs; i++) {
 		serial->rx_urb[i] = usb_alloc_urb(0, GFP_KERNEL);
-		if (!serial->rx_urb[i]) {
-			dev_err(dev, "Could not allocate urb?\n");
+		if (!serial->rx_urb[i])
 			goto exit;
-		}
 		serial->rx_urb[i]->transfer_buffer = NULL;
 		serial->rx_urb[i]->transfer_buffer_length = 0;
 		serial->rx_data[i] = kzalloc(serial->rx_data_length,
@@ -2314,10 +2305,8 @@ static int hso_serial_common_create(struct hso_serial *serial, int num_urbs,
 
 	/* TX, allocate urb and initialize */
 	serial->tx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!serial->tx_urb) {
-		dev_err(dev, "Could not allocate urb?\n");
+	if (!serial->tx_urb)
 		goto exit;
-	}
 	serial->tx_urb->transfer_buffer = NULL;
 	serial->tx_urb->transfer_buffer_length = 0;
 	/* prepare our TX buffer */
@@ -2419,7 +2408,7 @@ static void hso_net_init(struct net_device *net)
 {
 	struct hso_net *hso_net = netdev_priv(net);
 
-	D1("sizeof hso_net is %d", (int)sizeof(*hso_net));
+	hso_dbg(0x1, "sizeof hso_net is %zu\n", sizeof(*hso_net));
 
 	/* fill in the other fields */
 	net->netdev_ops = &hso_netdev_ops;
@@ -2555,20 +2544,16 @@ static struct hso_device *hso_create_net_device(struct usb_interface *interface,
 	/* start allocating */
 	for (i = 0; i < MUX_BULK_RX_BUF_COUNT; i++) {
 		hso_net->mux_bulk_rx_urb_pool[i] = usb_alloc_urb(0, GFP_KERNEL);
-		if (!hso_net->mux_bulk_rx_urb_pool[i]) {
-			dev_err(&interface->dev, "Could not allocate rx urb\n");
+		if (!hso_net->mux_bulk_rx_urb_pool[i])
 			goto exit;
-		}
 		hso_net->mux_bulk_rx_buf_pool[i] = kzalloc(MUX_BULK_RX_BUF_SIZE,
 							   GFP_KERNEL);
 		if (!hso_net->mux_bulk_rx_buf_pool[i])
 			goto exit;
 	}
 	hso_net->mux_bulk_tx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!hso_net->mux_bulk_tx_urb) {
-		dev_err(&interface->dev, "Could not allocate tx urb\n");
+	if (!hso_net->mux_bulk_tx_urb)
 		goto exit;
-	}
 	hso_net->mux_bulk_tx_buf = kzalloc(MUX_BULK_TX_BUF_SIZE, GFP_KERNEL);
 	if (!hso_net->mux_bulk_tx_buf)
 		goto exit;
@@ -2787,10 +2772,8 @@ struct hso_shared_int *hso_create_shared_int(struct usb_interface *interface)
 	}
 
 	mux->shared_intr_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!mux->shared_intr_urb) {
-		dev_err(&interface->dev, "Could not allocate intr urb?\n");
+	if (!mux->shared_intr_urb)
 		goto exit;
-	}
 	mux->shared_intr_buf =
 		kzalloc(le16_to_cpu(mux->intr_endp->wMaxPacketSize),
 			GFP_KERNEL);
@@ -3239,7 +3222,7 @@ static int __init hso_init(void)
 	int result;
 
 	/* put it in the log */
-	printk(KERN_INFO "hso: %s\n", version);
+	pr_info("%s\n", version);
 
 	/* Initialise the serial table semaphore and table */
 	spin_lock_init(&serial_table_lock);
@@ -3270,16 +3253,15 @@ static int __init hso_init(void)
 	/* register the tty driver */
 	result = tty_register_driver(tty_drv);
 	if (result) {
-		printk(KERN_ERR "%s - tty_register_driver failed(%d)\n",
-			__func__, result);
+		pr_err("%s - tty_register_driver failed(%d)\n",
+		       __func__, result);
 		goto err_free_tty;
 	}
 
 	/* register this module as an usb driver */
 	result = usb_register(&hso_driver);
 	if (result) {
-		printk(KERN_ERR "Could not register hso driver? error: %d\n",
-			result);
+		pr_err("Could not register hso driver - error: %d\n", result);
 		goto err_unreg_tty;
 	}
 
@@ -3294,7 +3276,7 @@ err_free_tty:
 
 static void __exit hso_exit(void)
 {
-	printk(KERN_INFO "hso: unloaded\n");
+	pr_info("unloaded\n");
 
 	tty_unregister_driver(tty_drv);
 	put_tty_driver(tty_drv);
@@ -3311,7 +3293,7 @@ MODULE_DESCRIPTION(MOD_DESCRIPTION);
 MODULE_LICENSE(MOD_LICENSE);
 
 /* change the debug level (eg: insmod hso.ko debug=0x04) */
-MODULE_PARM_DESC(debug, "Level of debug [0x01 | 0x02 | 0x04 | 0x08 | 0x10]");
+MODULE_PARM_DESC(debug, "debug level mask [0x01 | 0x02 | 0x04 | 0x08 | 0x10]");
 module_param(debug, int, S_IRUGO | S_IWUSR);
 
 /* set the major tty number (eg: insmod hso.ko tty_major=245) */
diff --git a/drivers/net/usb/kaweth.c b/drivers/net/usb/kaweth.c
index 770212baaf05..66b34ddbe216 100644
--- a/drivers/net/usb/kaweth.c
+++ b/drivers/net/usb/kaweth.c
@@ -265,8 +265,6 @@ static int kaweth_control(struct kaweth_device *kaweth,
 	struct usb_ctrlrequest *dr;
 	int retval;
 
-	netdev_dbg(kaweth->net, "kaweth_control()\n");
-
 	if(in_interrupt()) {
 		netdev_dbg(kaweth->net, "in_interrupt()\n");
 		return -EBUSY;
@@ -300,8 +298,6 @@ static int kaweth_read_configuration(struct kaweth_device *kaweth)
 {
 	int retval;
 
-	netdev_dbg(kaweth->net, "Reading kaweth configuration\n");
-
 	retval = kaweth_control(kaweth,
 				usb_rcvctrlpipe(kaweth->dev, 0),
 				KAWETH_COMMAND_GET_ETHERNET_DESC,
@@ -451,8 +447,6 @@ static int kaweth_trigger_firmware(struct kaweth_device *kaweth,
 	kaweth->firmware_buf[6] = 0x00;
 	kaweth->firmware_buf[7] = 0x00;
 
-	netdev_dbg(kaweth->net, "Triggering firmware\n");
-
 	return kaweth_control(kaweth,
 			      usb_sndctrlpipe(kaweth->dev, 0),
 			      KAWETH_COMMAND_SCAN,
@@ -471,7 +465,6 @@ static int kaweth_reset(struct kaweth_device *kaweth)
 {
 	int result;
 
-	netdev_dbg(kaweth->net, "kaweth_reset(%p)\n", kaweth);
 	result = usb_reset_configuration(kaweth->dev);
 	mdelay(10);
 
@@ -685,8 +678,6 @@ static int kaweth_open(struct net_device *net)
 	struct kaweth_device *kaweth = netdev_priv(net);
 	int res;
 
-	netdev_dbg(kaweth->net, "Opening network device.\n");
-
 	res = usb_autopm_get_interface(kaweth->intf);
 	if (res) {
 		dev_err(&kaweth->intf->dev, "Interface cannot be resumed.\n");
@@ -951,7 +942,6 @@ static int kaweth_suspend(struct usb_interface *intf, pm_message_t message)
 	struct kaweth_device *kaweth = usb_get_intfdata(intf);
 	unsigned long flags;
 
-	dev_dbg(&intf->dev, "Suspending device\n");
 	spin_lock_irqsave(&kaweth->device_lock, flags);
 	kaweth->status |= KAWETH_STATUS_SUSPENDING;
 	spin_unlock_irqrestore(&kaweth->device_lock, flags);
@@ -968,7 +958,6 @@ static int kaweth_resume(struct usb_interface *intf)
 	struct kaweth_device *kaweth = usb_get_intfdata(intf);
 	unsigned long flags;
 
-	dev_dbg(&intf->dev, "Resuming device\n");
 	spin_lock_irqsave(&kaweth->device_lock, flags);
 	kaweth->status &= ~KAWETH_STATUS_SUSPENDING;
 	spin_unlock_irqrestore(&kaweth->device_lock, flags);
@@ -1009,6 +998,7 @@ static int kaweth_probe(
 	struct net_device *netdev;
 	const eth_addr_t bcast_addr = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
 	int result = 0;
+	int rv = -EIO;
 
 	dev_dbg(dev,
 		"Kawasaki Device Probe (Device number:%d): 0x%4.4x:0x%4.4x:0x%4.4x\n",
@@ -1029,6 +1019,7 @@ static int kaweth_probe(
 	kaweth = netdev_priv(netdev);
 	kaweth->dev = udev;
 	kaweth->net = netdev;
+	kaweth->intf = intf;
 
 	spin_lock_init(&kaweth->device_lock);
 	init_waitqueue_head(&kaweth->term_wait);
@@ -1048,6 +1039,10 @@ static int kaweth_probe(
 		/* Download the firmware */
 		dev_info(dev, "Downloading firmware...\n");
 		kaweth->firmware_buf = (__u8 *)__get_free_page(GFP_KERNEL);
+		if (!kaweth->firmware_buf) {
+			rv = -ENOMEM;
+			goto err_free_netdev;
+		}
 		if ((result = kaweth_download_firmware(kaweth,
 						      "kaweth/new_code.bin",
 						      100,
@@ -1139,8 +1134,6 @@ err_fw:
 
 	dev_dbg(dev, "Initializing net device.\n");
 
-	kaweth->intf = intf;
-
 	kaweth->tx_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!kaweth->tx_urb)
 		goto err_free_netdev;
@@ -1186,8 +1179,6 @@ err_fw:
 	dev_info(dev, "kaweth interface created at %s\n",
 		 kaweth->net->name);
 
-	dev_dbg(dev, "Kaweth probe returning.\n");
-
 	return 0;
 
 err_intfdata:
@@ -1204,7 +1195,7 @@ err_only_tx:
 err_free_netdev:
 	free_netdev(netdev);
 
-	return -EIO;
+	return rv;
 }
 
 /****************************************************************
@@ -1215,8 +1206,6 @@ static void kaweth_disconnect(struct usb_interface *intf)
 	struct kaweth_device *kaweth = usb_get_intfdata(intf);
 	struct net_device *netdev;
 
-	dev_info(&intf->dev, "Unregistering\n");
-
 	usb_set_intfdata(intf, NULL);
 	if (!kaweth) {
 		dev_warn(&intf->dev, "unregistering non-existent device\n");
diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c
index 6a9d474b08b2..db558b8b32fe 100644
--- a/drivers/net/usb/lan78xx.c
+++ b/drivers/net/usb/lan78xx.c
@@ -1179,7 +1179,7 @@ static int lan78xx_link_reset(struct lan78xx_net *dev)
  * NOTE:  annoying asymmetry:  if it's active, schedule_work() fails,
  * but tasklet_schedule() doesn't.	hope the failure is rare.
  */
-void lan78xx_defer_kevent(struct lan78xx_net *dev, int work)
+static void lan78xx_defer_kevent(struct lan78xx_net *dev, int work)
 {
 	set_bit(work, &dev->flags);
 	if (!schedule_delayed_work(&dev->wq, 0))
@@ -1406,7 +1406,7 @@ static u32 lan78xx_get_link(struct net_device *net)
 	return net->phydev->link;
 }
 
-int lan78xx_nway_reset(struct net_device *net)
+static int lan78xx_nway_reset(struct net_device *net)
 {
 	return phy_start_aneg(net->phydev);
 }
@@ -1997,7 +1997,7 @@ static int lan78xx_change_mtu(struct net_device *netdev, int new_mtu)
 	return 0;
 }
 
-int lan78xx_set_mac_addr(struct net_device *netdev, void *p)
+static int lan78xx_set_mac_addr(struct net_device *netdev, void *p)
 {
 	struct lan78xx_net *dev = netdev_priv(netdev);
 	struct sockaddr *addr = p;
@@ -2371,7 +2371,7 @@ static void lan78xx_terminate_urbs(struct lan78xx_net *dev)
 	remove_wait_queue(&unlink_wakeup, &wait);
 }
 
-int lan78xx_stop(struct net_device *net)
+static int lan78xx_stop(struct net_device *net)
 {
 	struct lan78xx_net		*dev = netdev_priv(net);
 
@@ -2533,7 +2533,8 @@ static void lan78xx_queue_skb(struct sk_buff_head *list,
 	entry->state = state;
 }
 
-netdev_tx_t lan78xx_start_xmit(struct sk_buff *skb, struct net_device *net)
+static netdev_tx_t
+lan78xx_start_xmit(struct sk_buff *skb, struct net_device *net)
 {
 	struct lan78xx_net *dev = netdev_priv(net);
 	struct sk_buff *skb2 = NULL;
@@ -2562,7 +2563,8 @@ netdev_tx_t lan78xx_start_xmit(struct sk_buff *skb, struct net_device *net)
 	return NETDEV_TX_OK;
 }
 
-int lan78xx_get_endpoints(struct lan78xx_net *dev, struct usb_interface *intf)
+static int
+lan78xx_get_endpoints(struct lan78xx_net *dev, struct usb_interface *intf)
 {
 	int tmp;
 	struct usb_host_interface *alt = NULL;
@@ -2700,7 +2702,7 @@ static void lan78xx_rx_csum_offload(struct lan78xx_net *dev,
 	}
 }
 
-void lan78xx_skb_return(struct lan78xx_net *dev, struct sk_buff *skb)
+static void lan78xx_skb_return(struct lan78xx_net *dev, struct sk_buff *skb)
 {
 	int		status;
 
@@ -3002,10 +3004,8 @@ static void lan78xx_tx_bh(struct lan78xx_net *dev)
 
 gso_skb:
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!urb) {
-		netif_dbg(dev, tx_err, dev->net, "no urb\n");
+	if (!urb)
 		goto drop;
-	}
 
 	entry = (struct skb_data *)skb->cb;
 	entry->urb = urb;
@@ -3285,7 +3285,7 @@ static void lan78xx_disconnect(struct usb_interface *intf)
 	usb_put_dev(udev);
 }
 
-void lan78xx_tx_timeout(struct net_device *net)
+static void lan78xx_tx_timeout(struct net_device *net)
 {
 	struct lan78xx_net *dev = netdev_priv(net);
 
@@ -3605,7 +3605,7 @@ static int lan78xx_set_suspend(struct lan78xx_net *dev, u32 wol)
 	return 0;
 }
 
-int lan78xx_suspend(struct usb_interface *intf, pm_message_t message)
+static int lan78xx_suspend(struct usb_interface *intf, pm_message_t message)
 {
 	struct lan78xx_net *dev = usb_get_intfdata(intf);
 	struct lan78xx_priv *pdata = (struct lan78xx_priv *)(dev->data[0]);
@@ -3701,7 +3701,7 @@ out:
 	return ret;
 }
 
-int lan78xx_resume(struct usb_interface *intf)
+static int lan78xx_resume(struct usb_interface *intf)
 {
 	struct lan78xx_net *dev = usb_get_intfdata(intf);
 	struct sk_buff *skb;
@@ -3768,7 +3768,7 @@ int lan78xx_resume(struct usb_interface *intf)
 	return 0;
 }
 
-int lan78xx_reset_resume(struct usb_interface *intf)
+static int lan78xx_reset_resume(struct usb_interface *intf)
 {
 	struct lan78xx_net *dev = usb_get_intfdata(intf);
 
diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c
index 9bbe0161a2f4..1434e5dd5f9c 100644
--- a/drivers/net/usb/pegasus.c
+++ b/drivers/net/usb/pegasus.c
@@ -1129,7 +1129,8 @@ static int pegasus_probe(struct usb_interface *intf,
 		return -ENODEV;
 
 	if (pegasus_count == 0) {
-		pegasus_workqueue = create_singlethread_workqueue("pegasus");
+		pegasus_workqueue = alloc_workqueue("pegasus", WQ_MEM_RECLAIM,
+						    0);
 		if (!pegasus_workqueue)
 			return -ENOMEM;
 	}
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index f41a8ad4740e..44d439f50961 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -32,7 +32,7 @@
 #define NETNEXT_VERSION		"08"
 
 /* Information for net */
-#define NET_VERSION		"5"
+#define NET_VERSION		"6"
 
 #define DRIVER_VERSION		"v1." NETNEXT_VERSION "." NET_VERSION
 #define DRIVER_AUTHOR "Realtek linux nic maintainers <nic_swsd@realtek.com>"
@@ -1076,8 +1076,7 @@ static int vendor_mac_passthru_addr_read(struct r8152 *tp, struct sockaddr *sa)
 		return -ENODEV;
 	if (obj->type != ACPI_TYPE_BUFFER || obj->string.length != 0x17) {
 		netif_warn(tp, probe, tp->netdev,
-			   "Invalid buffer when reading pass-thru MAC addr: "
-			   "(%d, %d)\n",
+			   "Invalid buffer for pass-thru MAC addr: (%d, %d)\n",
 			   obj->type, obj->string.length);
 		goto amacout;
 	}
@@ -1090,8 +1089,8 @@ static int vendor_mac_passthru_addr_read(struct r8152 *tp, struct sockaddr *sa)
 	ret = hex2bin(buf, obj->string.pointer + 9, 6);
 	if (!(ret == 0 && is_valid_ether_addr(buf))) {
 		netif_warn(tp, probe, tp->netdev,
-			   "Invalid MAC when reading pass-thru MAC addr: "
-			   "%d, %pM\n", ret, buf);
+			   "Invalid MAC for pass-thru MAC addr: %d, %pM\n",
+			   ret, buf);
 		ret = -EINVAL;
 		goto amacout;
 	}
@@ -1111,9 +1110,9 @@ static int set_ethernet_addr(struct r8152 *tp)
 	struct sockaddr sa;
 	int ret;
 
-	if (tp->version == RTL_VER_01)
+	if (tp->version == RTL_VER_01) {
 		ret = pla_ocp_read(tp, PLA_IDR, 8, sa.sa_data);
-	else {
+	} else {
 		/* if this is not an RTL8153-AD, no eFuse mac pass thru set,
 		 * or system doesn't provide valid _SB.AMAC this will be
 		 * be expected to non-zero
@@ -2552,6 +2551,77 @@ static void r8152_aldps_en(struct r8152 *tp, bool enable)
 	}
 }
 
+static inline void r8152_mmd_indirect(struct r8152 *tp, u16 dev, u16 reg)
+{
+	ocp_reg_write(tp, OCP_EEE_AR, FUN_ADDR | dev);
+	ocp_reg_write(tp, OCP_EEE_DATA, reg);
+	ocp_reg_write(tp, OCP_EEE_AR, FUN_DATA | dev);
+}
+
+static u16 r8152_mmd_read(struct r8152 *tp, u16 dev, u16 reg)
+{
+	u16 data;
+
+	r8152_mmd_indirect(tp, dev, reg);
+	data = ocp_reg_read(tp, OCP_EEE_DATA);
+	ocp_reg_write(tp, OCP_EEE_AR, 0x0000);
+
+	return data;
+}
+
+static void r8152_mmd_write(struct r8152 *tp, u16 dev, u16 reg, u16 data)
+{
+	r8152_mmd_indirect(tp, dev, reg);
+	ocp_reg_write(tp, OCP_EEE_DATA, data);
+	ocp_reg_write(tp, OCP_EEE_AR, 0x0000);
+}
+
+static void r8152_eee_en(struct r8152 *tp, bool enable)
+{
+	u16 config1, config2, config3;
+	u32 ocp_data;
+
+	ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_EEE_CR);
+	config1 = ocp_reg_read(tp, OCP_EEE_CONFIG1) & ~sd_rise_time_mask;
+	config2 = ocp_reg_read(tp, OCP_EEE_CONFIG2);
+	config3 = ocp_reg_read(tp, OCP_EEE_CONFIG3) & ~fast_snr_mask;
+
+	if (enable) {
+		ocp_data |= EEE_RX_EN | EEE_TX_EN;
+		config1 |= EEE_10_CAP | EEE_NWAY_EN | TX_QUIET_EN | RX_QUIET_EN;
+		config1 |= sd_rise_time(1);
+		config2 |= RG_DACQUIET_EN | RG_LDVQUIET_EN;
+		config3 |= fast_snr(42);
+	} else {
+		ocp_data &= ~(EEE_RX_EN | EEE_TX_EN);
+		config1 &= ~(EEE_10_CAP | EEE_NWAY_EN | TX_QUIET_EN |
+			     RX_QUIET_EN);
+		config1 |= sd_rise_time(7);
+		config2 &= ~(RG_DACQUIET_EN | RG_LDVQUIET_EN);
+		config3 |= fast_snr(511);
+	}
+
+	ocp_write_word(tp, MCU_TYPE_PLA, PLA_EEE_CR, ocp_data);
+	ocp_reg_write(tp, OCP_EEE_CONFIG1, config1);
+	ocp_reg_write(tp, OCP_EEE_CONFIG2, config2);
+	ocp_reg_write(tp, OCP_EEE_CONFIG3, config3);
+}
+
+static void r8152b_enable_eee(struct r8152 *tp)
+{
+	r8152_eee_en(tp, true);
+	r8152_mmd_write(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, MDIO_EEE_100TX);
+}
+
+static void r8152b_enable_fc(struct r8152 *tp)
+{
+	u16 anar;
+
+	anar = r8152_mdio_read(tp, MII_ADVERTISE);
+	anar |= ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
+	r8152_mdio_write(tp, MII_ADVERTISE, anar);
+}
+
 static void rtl8152_disable(struct r8152 *tp)
 {
 	r8152_aldps_en(tp, false);
@@ -2561,13 +2631,9 @@ static void rtl8152_disable(struct r8152 *tp)
 
 static void r8152b_hw_phy_cfg(struct r8152 *tp)
 {
-	u16 data;
-
-	data = r8152_mdio_read(tp, MII_BMCR);
-	if (data & BMCR_PDOWN) {
-		data &= ~BMCR_PDOWN;
-		r8152_mdio_write(tp, MII_BMCR, data);
-	}
+	r8152b_enable_eee(tp);
+	r8152_aldps_en(tp, true);
+	r8152b_enable_fc(tp);
 
 	set_bit(PHY_RESET, &tp->flags);
 }
@@ -2701,20 +2767,52 @@ static void r8152b_enter_oob(struct r8152 *tp)
 	ocp_write_dword(tp, MCU_TYPE_PLA, PLA_RCR, ocp_data);
 }
 
+static void r8153_aldps_en(struct r8152 *tp, bool enable)
+{
+	u16 data;
+
+	data = ocp_reg_read(tp, OCP_POWER_CFG);
+	if (enable) {
+		data |= EN_ALDPS;
+		ocp_reg_write(tp, OCP_POWER_CFG, data);
+	} else {
+		data &= ~EN_ALDPS;
+		ocp_reg_write(tp, OCP_POWER_CFG, data);
+		msleep(20);
+	}
+}
+
+static void r8153_eee_en(struct r8152 *tp, bool enable)
+{
+	u32 ocp_data;
+	u16 config;
+
+	ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_EEE_CR);
+	config = ocp_reg_read(tp, OCP_EEE_CFG);
+
+	if (enable) {
+		ocp_data |= EEE_RX_EN | EEE_TX_EN;
+		config |= EEE10_EN;
+	} else {
+		ocp_data &= ~(EEE_RX_EN | EEE_TX_EN);
+		config &= ~EEE10_EN;
+	}
+
+	ocp_write_word(tp, MCU_TYPE_PLA, PLA_EEE_CR, ocp_data);
+	ocp_reg_write(tp, OCP_EEE_CFG, config);
+}
+
 static void r8153_hw_phy_cfg(struct r8152 *tp)
 {
 	u32 ocp_data;
 	u16 data;
 
-	if (tp->version == RTL_VER_03 || tp->version == RTL_VER_04 ||
-	    tp->version == RTL_VER_05)
-		ocp_reg_write(tp, OCP_ADC_CFG, CKADSEL_L | ADC_EN | EN_EMI_L);
+	/* disable ALDPS before updating the PHY parameters */
+	r8153_aldps_en(tp, false);
 
-	data = r8152_mdio_read(tp, MII_BMCR);
-	if (data & BMCR_PDOWN) {
-		data &= ~BMCR_PDOWN;
-		r8152_mdio_write(tp, MII_BMCR, data);
-	}
+	/* disable EEE before updating the PHY parameters */
+	r8153_eee_en(tp, false);
+	ocp_reg_write(tp, OCP_EEE_ADV, 0);
 
 	if (tp->version == RTL_VER_03) {
 		data = ocp_reg_read(tp, OCP_EEE_CFG);
@@ -2745,6 +2843,12 @@ static void r8153_hw_phy_cfg(struct r8152 *tp)
 	sram_write(tp, SRAM_10M_AMP1, 0x00af);
 	sram_write(tp, SRAM_10M_AMP2, 0x0208);
 
+	r8153_eee_en(tp, true);
+	ocp_reg_write(tp, OCP_EEE_ADV, MDIO_EEE_1000T | MDIO_EEE_100TX);
+
+	r8153_aldps_en(tp, true);
+	r8152b_enable_fc(tp);
+
 	set_bit(PHY_RESET, &tp->flags);
 }
 
@@ -2866,21 +2970,6 @@ static void r8153_enter_oob(struct r8152 *tp)
 	ocp_write_dword(tp, MCU_TYPE_PLA, PLA_RCR, ocp_data);
 }
 
-static void r8153_aldps_en(struct r8152 *tp, bool enable)
-{
-	u16 data;
-
-	data = ocp_reg_read(tp, OCP_POWER_CFG);
-	if (enable) {
-		data |= EN_ALDPS;
-		ocp_reg_write(tp, OCP_POWER_CFG, data);
-	} else {
-		data &= ~EN_ALDPS;
-		ocp_reg_write(tp, OCP_POWER_CFG, data);
-		msleep(20);
-	}
-}
-
 static void rtl8153_disable(struct r8152 *tp)
 {
 	r8153_aldps_en(tp, false);
@@ -3246,103 +3335,6 @@ static int rtl8152_close(struct net_device *netdev)
 	return res;
 }
 
-static inline void r8152_mmd_indirect(struct r8152 *tp, u16 dev, u16 reg)
-{
-	ocp_reg_write(tp, OCP_EEE_AR, FUN_ADDR | dev);
-	ocp_reg_write(tp, OCP_EEE_DATA, reg);
-	ocp_reg_write(tp, OCP_EEE_AR, FUN_DATA | dev);
-}
-
-static u16 r8152_mmd_read(struct r8152 *tp, u16 dev, u16 reg)
-{
-	u16 data;
-
-	r8152_mmd_indirect(tp, dev, reg);
-	data = ocp_reg_read(tp, OCP_EEE_DATA);
-	ocp_reg_write(tp, OCP_EEE_AR, 0x0000);
-
-	return data;
-}
-
-static void r8152_mmd_write(struct r8152 *tp, u16 dev, u16 reg, u16 data)
-{
-	r8152_mmd_indirect(tp, dev, reg);
-	ocp_reg_write(tp, OCP_EEE_DATA, data);
-	ocp_reg_write(tp, OCP_EEE_AR, 0x0000);
-}
-
-static void r8152_eee_en(struct r8152 *tp, bool enable)
-{
-	u16 config1, config2, config3;
-	u32 ocp_data;
-
-	ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_EEE_CR);
-	config1 = ocp_reg_read(tp, OCP_EEE_CONFIG1) & ~sd_rise_time_mask;
-	config2 = ocp_reg_read(tp, OCP_EEE_CONFIG2);
-	config3 = ocp_reg_read(tp, OCP_EEE_CONFIG3) & ~fast_snr_mask;
-
-	if (enable) {
-		ocp_data |= EEE_RX_EN | EEE_TX_EN;
-		config1 |= EEE_10_CAP | EEE_NWAY_EN | TX_QUIET_EN | RX_QUIET_EN;
-		config1 |= sd_rise_time(1);
-		config2 |= RG_DACQUIET_EN | RG_LDVQUIET_EN;
-		config3 |= fast_snr(42);
-	} else {
-		ocp_data &= ~(EEE_RX_EN | EEE_TX_EN);
-		config1 &= ~(EEE_10_CAP | EEE_NWAY_EN | TX_QUIET_EN |
-			     RX_QUIET_EN);
-		config1 |= sd_rise_time(7);
-		config2 &= ~(RG_DACQUIET_EN | RG_LDVQUIET_EN);
-		config3 |= fast_snr(511);
-	}
-
-	ocp_write_word(tp, MCU_TYPE_PLA, PLA_EEE_CR, ocp_data);
-	ocp_reg_write(tp, OCP_EEE_CONFIG1, config1);
-	ocp_reg_write(tp, OCP_EEE_CONFIG2, config2);
-	ocp_reg_write(tp, OCP_EEE_CONFIG3, config3);
-}
-
-static void r8152b_enable_eee(struct r8152 *tp)
-{
-	r8152_eee_en(tp, true);
-	r8152_mmd_write(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, MDIO_EEE_100TX);
-}
-
-static void r8153_eee_en(struct r8152 *tp, bool enable)
-{
-	u32 ocp_data;
-	u16 config;
-
-	ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_EEE_CR);
-	config = ocp_reg_read(tp, OCP_EEE_CFG);
-
-	if (enable) {
-		ocp_data |= EEE_RX_EN | EEE_TX_EN;
-		config |= EEE10_EN;
-	} else {
-		ocp_data &= ~(EEE_RX_EN | EEE_TX_EN);
-		config &= ~EEE10_EN;
-	}
-
-	ocp_write_word(tp, MCU_TYPE_PLA, PLA_EEE_CR, ocp_data);
-	ocp_reg_write(tp, OCP_EEE_CFG, config);
-}
-
-static void r8153_enable_eee(struct r8152 *tp)
-{
-	r8153_eee_en(tp, true);
-	ocp_reg_write(tp, OCP_EEE_ADV, MDIO_EEE_1000T | MDIO_EEE_100TX);
-}
-
-static void r8152b_enable_fc(struct r8152 *tp)
-{
-	u16 anar;
-
-	anar = r8152_mdio_read(tp, MII_ADVERTISE);
-	anar |= ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
-	r8152_mdio_write(tp, MII_ADVERTISE, anar);
-}
-
 static void rtl_tally_reset(struct r8152 *tp)
 {
 	u32 ocp_data;
@@ -3355,10 +3347,17 @@ static void rtl_tally_reset(struct r8152 *tp)
 static void r8152b_init(struct r8152 *tp)
 {
 	u32 ocp_data;
+	u16 data;
 
 	if (test_bit(RTL8152_UNPLUG, &tp->flags))
 		return;
 
+	data = r8152_mdio_read(tp, MII_BMCR);
+	if (data & BMCR_PDOWN) {
+		data &= ~BMCR_PDOWN;
+		r8152_mdio_write(tp, MII_BMCR, data);
+	}
+
 	r8152_aldps_en(tp, false);
 
 	if (tp->version == RTL_VER_01) {
@@ -3380,9 +3379,6 @@ static void r8152b_init(struct r8152 *tp)
 		   SPDWN_RXDV_MSK | SPDWN_LINKCHG_MSK;
 	ocp_write_word(tp, MCU_TYPE_PLA, PLA_GPHY_INTR_IMR, ocp_data);
 
-	r8152b_enable_eee(tp);
-	r8152_aldps_en(tp, true);
-	r8152b_enable_fc(tp);
 	rtl_tally_reset(tp);
 
 	/* enable rx aggregation */
@@ -3394,12 +3390,12 @@ static void r8152b_init(struct r8152 *tp)
 static void r8153_init(struct r8152 *tp)
 {
 	u32 ocp_data;
+	u16 data;
 	int i;
 
 	if (test_bit(RTL8152_UNPLUG, &tp->flags))
 		return;
 
-	r8153_aldps_en(tp, false);
 	r8153_u1u2en(tp, false);
 
 	for (i = 0; i < 500; i++) {
@@ -3416,6 +3412,23 @@ static void r8153_init(struct r8152 *tp)
 		msleep(20);
 	}
 
+	if (tp->version == RTL_VER_03 || tp->version == RTL_VER_04 ||
+	    tp->version == RTL_VER_05)
+		ocp_reg_write(tp, OCP_ADC_CFG, CKADSEL_L | ADC_EN | EN_EMI_L);
+
+	data = r8152_mdio_read(tp, MII_BMCR);
+	if (data & BMCR_PDOWN) {
+		data &= ~BMCR_PDOWN;
+		r8152_mdio_write(tp, MII_BMCR, data);
+	}
+
+	for (i = 0; i < 500; i++) {
+		ocp_data = ocp_reg_read(tp, OCP_PHY_STATUS) & PHY_STAT_MASK;
+		if (ocp_data == PHY_STAT_LAN_ON)
+			break;
+		msleep(20);
+	}
+
 	usb_disable_lpm(tp->udev);
 	r8153_u2p3en(tp, false);
 
@@ -3483,9 +3496,6 @@ static void r8153_init(struct r8152 *tp)
 	ocp_write_word(tp, MCU_TYPE_PLA, PLA_MAC_PWR_CTRL3, 0);
 	ocp_write_word(tp, MCU_TYPE_PLA, PLA_MAC_PWR_CTRL4, 0);
 
-	r8153_enable_eee(tp);
-	r8153_aldps_en(tp, true);
-	r8152b_enable_fc(tp);
 	rtl_tally_reset(tp);
 	r8153_u2p3en(tp, true);
 }
@@ -4032,7 +4042,7 @@ static int rtl8152_set_coalesce(struct net_device *netdev,
 	return ret;
 }
 
-static struct ethtool_ops ops = {
+static const struct ethtool_ops ops = {
 	.get_drvinfo = rtl8152_get_drvinfo,
 	.get_settings = rtl8152_get_settings,
 	.set_settings = rtl8152_set_settings,
diff --git a/drivers/net/usb/smsc95xx.c b/drivers/net/usb/smsc95xx.c
index dc989a8b5afb..831aa33d078a 100644
--- a/drivers/net/usb/smsc95xx.c
+++ b/drivers/net/usb/smsc95xx.c
@@ -33,7 +33,7 @@
 #include "smsc95xx.h"
 
 #define SMSC_CHIPNAME			"smsc95xx"
-#define SMSC_DRIVER_VERSION		"1.0.4"
+#define SMSC_DRIVER_VERSION		"1.0.5"
 #define HS_USB_PKT_SIZE			(512)
 #define FS_USB_PKT_SIZE			(64)
 #define DEFAULT_HS_BURST_CAP_SIZE	(16 * 1024 + 5 * HS_USB_PKT_SIZE)
@@ -64,6 +64,7 @@
 #define CARRIER_CHECK_DELAY (2 * HZ)
 
 struct smsc95xx_priv {
+	u32 chip_id;
 	u32 mac_cr;
 	u32 hash_hi;
 	u32 hash_lo;
@@ -71,6 +72,7 @@ struct smsc95xx_priv {
 	spinlock_t mac_cr_lock;
 	u8 features;
 	u8 suspend_flags;
+	u8 mdix_ctrl;
 	bool link_ok;
 	struct delayed_work carrier_check;
 	struct usbnet *dev;
@@ -782,14 +784,113 @@ static int smsc95xx_ethtool_set_wol(struct net_device *net,
 	return ret;
 }
 
+static int get_mdix_status(struct net_device *net)
+{
+	struct usbnet *dev = netdev_priv(net);
+	u32 val;
+	int buf;
+
+	buf = smsc95xx_mdio_read(dev->net, dev->mii.phy_id, SPECIAL_CTRL_STS);
+	if (buf & SPECIAL_CTRL_STS_OVRRD_AMDIX_) {
+		if (buf & SPECIAL_CTRL_STS_AMDIX_ENABLE_)
+			return ETH_TP_MDI_AUTO;
+		else if (buf & SPECIAL_CTRL_STS_AMDIX_STATE_)
+			return ETH_TP_MDI_X;
+	} else {
+		buf = smsc95xx_read_reg(dev, STRAP_STATUS, &val);
+		if (val & STRAP_STATUS_AMDIX_EN_)
+			return ETH_TP_MDI_AUTO;
+	}
+
+	return ETH_TP_MDI;
+}
+
+static void set_mdix_status(struct net_device *net, __u8 mdix_ctrl)
+{
+	struct usbnet *dev = netdev_priv(net);
+	struct smsc95xx_priv *pdata = (struct smsc95xx_priv *)(dev->data[0]);
+	int buf;
+
+	if ((pdata->chip_id == ID_REV_CHIP_ID_9500A_) ||
+	    (pdata->chip_id == ID_REV_CHIP_ID_9530_) ||
+	    (pdata->chip_id == ID_REV_CHIP_ID_89530_) ||
+	    (pdata->chip_id == ID_REV_CHIP_ID_9730_)) {
+		/* Extend Manual AutoMDIX timer for 9500A/9500Ai */
+		buf = smsc95xx_mdio_read(dev->net, dev->mii.phy_id,
+					 PHY_EDPD_CONFIG);
+		buf |= PHY_EDPD_CONFIG_EXT_CROSSOVER_;
+		smsc95xx_mdio_write(dev->net, dev->mii.phy_id,
+				    PHY_EDPD_CONFIG, buf);
+	}
+
+	if (mdix_ctrl == ETH_TP_MDI) {
+		buf = smsc95xx_mdio_read(dev->net, dev->mii.phy_id,
+					 SPECIAL_CTRL_STS);
+		buf |= SPECIAL_CTRL_STS_OVRRD_AMDIX_;
+		buf &= ~(SPECIAL_CTRL_STS_AMDIX_ENABLE_ |
+			 SPECIAL_CTRL_STS_AMDIX_STATE_);
+		smsc95xx_mdio_write(dev->net, dev->mii.phy_id,
+				    SPECIAL_CTRL_STS, buf);
+	} else if (mdix_ctrl == ETH_TP_MDI_X) {
+		buf = smsc95xx_mdio_read(dev->net, dev->mii.phy_id,
+					 SPECIAL_CTRL_STS);
+		buf |= SPECIAL_CTRL_STS_OVRRD_AMDIX_;
+		buf &= ~(SPECIAL_CTRL_STS_AMDIX_ENABLE_ |
+			 SPECIAL_CTRL_STS_AMDIX_STATE_);
+		buf |= SPECIAL_CTRL_STS_AMDIX_STATE_;
+		smsc95xx_mdio_write(dev->net, dev->mii.phy_id,
+				    SPECIAL_CTRL_STS, buf);
+	} else if (mdix_ctrl == ETH_TP_MDI_AUTO) {
+		buf = smsc95xx_mdio_read(dev->net, dev->mii.phy_id,
+					 SPECIAL_CTRL_STS);
+		buf &= ~SPECIAL_CTRL_STS_OVRRD_AMDIX_;
+		buf &= ~(SPECIAL_CTRL_STS_AMDIX_ENABLE_ |
+			 SPECIAL_CTRL_STS_AMDIX_STATE_);
+		buf |= SPECIAL_CTRL_STS_AMDIX_ENABLE_;
+		smsc95xx_mdio_write(dev->net, dev->mii.phy_id,
+				    SPECIAL_CTRL_STS, buf);
+	}
+	pdata->mdix_ctrl = mdix_ctrl;
+}
+
+static int smsc95xx_get_settings(struct net_device *net,
+				 struct ethtool_cmd *cmd)
+{
+	struct usbnet *dev = netdev_priv(net);
+	struct smsc95xx_priv *pdata = (struct smsc95xx_priv *)(dev->data[0]);
+	int retval;
+
+	retval = usbnet_get_settings(net, cmd);
+
+	cmd->eth_tp_mdix = pdata->mdix_ctrl;
+	cmd->eth_tp_mdix_ctrl = pdata->mdix_ctrl;
+
+	return retval;
+}
+
+static int smsc95xx_set_settings(struct net_device *net,
+				 struct ethtool_cmd *cmd)
+{
+	struct usbnet *dev = netdev_priv(net);
+	struct smsc95xx_priv *pdata = (struct smsc95xx_priv *)(dev->data[0]);
+	int retval;
+
+	if (pdata->mdix_ctrl != cmd->eth_tp_mdix_ctrl)
+		set_mdix_status(net, cmd->eth_tp_mdix_ctrl);
+
+	retval = usbnet_set_settings(net, cmd);
+
+	return retval;
+}
+
 static const struct ethtool_ops smsc95xx_ethtool_ops = {
 	.get_link	= usbnet_get_link,
 	.nway_reset	= usbnet_nway_reset,
 	.get_drvinfo	= usbnet_get_drvinfo,
 	.get_msglevel	= usbnet_get_msglevel,
 	.set_msglevel	= usbnet_set_msglevel,
-	.get_settings	= usbnet_get_settings,
-	.set_settings	= usbnet_set_settings,
+	.get_settings	= smsc95xx_get_settings,
+	.set_settings	= smsc95xx_set_settings,
 	.get_eeprom_len	= smsc95xx_ethtool_get_eeprom_len,
 	.get_eeprom	= smsc95xx_ethtool_get_eeprom,
 	.set_eeprom	= smsc95xx_ethtool_set_eeprom,
@@ -1194,6 +1295,8 @@ static int smsc95xx_bind(struct usbnet *dev, struct usb_interface *intf)
 	if (ret < 0)
 		return ret;
 	val >>= 16;
+	pdata->chip_id = val;
+	pdata->mdix_ctrl = get_mdix_status(dev->net);
 
 	if ((val == ID_REV_CHIP_ID_9500A_) || (val == ID_REV_CHIP_ID_9530_) ||
 	    (val == ID_REV_CHIP_ID_89530_) || (val == ID_REV_CHIP_ID_9730_))
diff --git a/drivers/net/usb/smsc95xx.h b/drivers/net/usb/smsc95xx.h
index 526faa0c44e6..29a4d9efce7c 100644
--- a/drivers/net/usb/smsc95xx.h
+++ b/drivers/net/usb/smsc95xx.h
@@ -144,6 +144,14 @@
 
 #define BURST_CAP			(0x38)
 
+#define	STRAP_STATUS			(0x3C)
+#define	STRAP_STATUS_PWR_SEL_		(0x00000020)
+#define	STRAP_STATUS_AMDIX_EN_		(0x00000010)
+#define	STRAP_STATUS_PORT_SWAP_		(0x00000008)
+#define	STRAP_STATUS_EEP_SIZE_		(0x00000004)
+#define	STRAP_STATUS_RMT_WKP_		(0x00000002)
+#define	STRAP_STATUS_EEP_DISABLE_	(0x00000001)
+
 #define GPIO_WAKE			(0x64)
 
 #define INT_EP_CTL			(0x68)
diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c
index 3bfb59209326..d5071e364d40 100644
--- a/drivers/net/usb/usbnet.c
+++ b/drivers/net/usb/usbnet.c
@@ -2062,11 +2062,8 @@ int usbnet_write_cmd_async(struct usbnet *dev, u8 cmd, u8 reqtype,
 		   cmd, reqtype, value, index, size);
 
 	urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!urb) {
-		netdev_err(dev->net, "Error allocating URB in"
-			   " %s!\n", __func__);
+	if (!urb)
 		goto fail;
-	}
 
 	if (data) {
 		buf = kmemdup(data, size, GFP_ATOMIC);
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index f37a6e61d4ad..fbc853e64531 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -313,7 +313,7 @@ static const struct net_device_ops veth_netdev_ops = {
 };
 
 #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \
-		       NETIF_F_RXCSUM | NETIF_F_HIGHDMA | \
+		       NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \
 		       NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \
 		       NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \
 		       NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX )
@@ -340,6 +340,7 @@ static void veth_setup(struct net_device *dev)
 
 	dev->hw_features = VETH_FEATURES;
 	dev->hw_enc_features = VETH_FEATURES;
+	dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
 }
 
 /*
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 1b5f531eeb25..fad84f3f4109 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -138,8 +138,9 @@ struct virtnet_info {
 	/* Does the affinity hint is set for virtqueues? */
 	bool affinity_hint_set;
 
-	/* CPU hot plug notifier */
-	struct notifier_block nb;
+	/* CPU hotplug instances for online & dead */
+	struct hlist_node node;
+	struct hlist_node node_dead;
 
 	/* Control VQ buffers: protected by the rtnl lock */
 	struct virtio_net_ctrl_hdr ctrl_hdr;
@@ -1237,25 +1238,53 @@ static void virtnet_set_affinity(struct virtnet_info *vi)
 	vi->affinity_hint_set = true;
 }
 
-static int virtnet_cpu_callback(struct notifier_block *nfb,
-			        unsigned long action, void *hcpu)
+static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node)
 {
-	struct virtnet_info *vi = container_of(nfb, struct virtnet_info, nb);
+	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
+						   node);
+	virtnet_set_affinity(vi);
+	return 0;
+}
 
-	switch(action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-	case CPU_DEAD:
-		virtnet_set_affinity(vi);
-		break;
-	case CPU_DOWN_PREPARE:
-		virtnet_clean_affinity(vi, (long)hcpu);
-		break;
-	default:
-		break;
-	}
+static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node)
+{
+	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
+						   node_dead);
+	virtnet_set_affinity(vi);
+	return 0;
+}
 
-	return NOTIFY_OK;
+static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
+{
+	struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info,
+						   node);
+
+	virtnet_clean_affinity(vi, cpu);
+	return 0;
+}
+
+static enum cpuhp_state virtionet_online;
+
+static int virtnet_cpu_notif_add(struct virtnet_info *vi)
+{
+	int ret;
+
+	ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node);
+	if (ret)
+		return ret;
+	ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD,
+					       &vi->node_dead);
+	if (!ret)
+		return ret;
+	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
+	return ret;
+}
+
+static void virtnet_cpu_notif_remove(struct virtnet_info *vi)
+{
+	cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node);
+	cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD,
+					    &vi->node_dead);
 }
 
 static void virtnet_get_ringparam(struct net_device *dev,
@@ -1879,8 +1908,7 @@ static int virtnet_probe(struct virtio_device *vdev)
 
 	virtio_device_ready(vdev);
 
-	vi->nb.notifier_call = &virtnet_cpu_callback;
-	err = register_hotcpu_notifier(&vi->nb);
+	err = virtnet_cpu_notif_add(vi);
 	if (err) {
 		pr_debug("virtio_net: registering cpu notifier failed\n");
 		goto free_unregister_netdev;
@@ -1934,7 +1962,7 @@ static void virtnet_remove(struct virtio_device *vdev)
 {
 	struct virtnet_info *vi = vdev->priv;
 
-	unregister_hotcpu_notifier(&vi->nb);
+	virtnet_cpu_notif_remove(vi);
 
 	/* Make sure no work handler is accessing the device. */
 	flush_work(&vi->config_work);
@@ -1953,7 +1981,7 @@ static int virtnet_freeze(struct virtio_device *vdev)
 	struct virtnet_info *vi = vdev->priv;
 	int i;
 
-	unregister_hotcpu_notifier(&vi->nb);
+	virtnet_cpu_notif_remove(vi);
 
 	/* Make sure no work handler is accessing the device */
 	flush_work(&vi->config_work);
@@ -1997,7 +2025,7 @@ static int virtnet_restore(struct virtio_device *vdev)
 	virtnet_set_queues(vi, vi->curr_queue_pairs);
 	rtnl_unlock();
 
-	err = register_hotcpu_notifier(&vi->nb);
+	err = virtnet_cpu_notif_add(vi);
 	if (err)
 		return err;
 
@@ -2039,7 +2067,41 @@ static struct virtio_driver virtio_net_driver = {
 #endif
 };
 
-module_virtio_driver(virtio_net_driver);
+static __init int virtio_net_driver_init(void)
+{
+	int ret;
+
+	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "AP_VIRT_NET_ONLINE",
+				      virtnet_cpu_online,
+				      virtnet_cpu_down_prep);
+	if (ret < 0)
+		goto out;
+	virtionet_online = ret;
+	ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "VIRT_NET_DEAD",
+				      NULL, virtnet_cpu_dead);
+	if (ret)
+		goto err_dead;
+
+        ret = register_virtio_driver(&virtio_net_driver);
+	if (ret)
+		goto err_virtio;
+	return 0;
+err_virtio:
+	cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
+err_dead:
+	cpuhp_remove_multi_state(virtionet_online);
+out:
+	return ret;
+}
+module_init(virtio_net_driver_init);
+
+static __exit void virtio_net_driver_exit(void)
+{
+	cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD);
+	cpuhp_remove_multi_state(virtionet_online);
+	unregister_virtio_driver(&virtio_net_driver);
+}
+module_exit(virtio_net_driver_exit);
 
 MODULE_DEVICE_TABLE(virtio, id_table);
 MODULE_DESCRIPTION("Virtio network driver");
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index c68fe495d3f9..b5554f2ebee4 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -914,7 +914,9 @@ vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 {
 	struct Vmxnet3_TxDataDesc *tdd;
 
-	tdd = tq->data_ring.base + tq->tx_ring.next2fill;
+	tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base +
+					    tq->tx_ring.next2fill *
+					    tq->txdata_desc_size);
 
 	memcpy(tdd->data, skb->data, ctx->copy_size);
 	netdev_dbg(adapter->netdev,
@@ -1639,7 +1641,7 @@ static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
 	}
 }
 
-void
+static void
 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter)
 {
 	int i;
@@ -3184,7 +3186,6 @@ vmxnet3_tx_timeout(struct net_device *netdev)
 
 	netdev_err(adapter->netdev, "tx hang\n");
 	schedule_work(&adapter->work);
-	netif_wake_queue(adapter->netdev);
 }
 
 
@@ -3211,6 +3212,7 @@ vmxnet3_reset_work(struct work_struct *data)
 	}
 	rtnl_unlock();
 
+	netif_wake_queue(adapter->netdev);
 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
 }
 
diff --git a/drivers/net/vmxnet3/vmxnet3_int.h b/drivers/net/vmxnet3/vmxnet3_int.h
index 74fc03072b87..7dc37a090549 100644
--- a/drivers/net/vmxnet3/vmxnet3_int.h
+++ b/drivers/net/vmxnet3/vmxnet3_int.h
@@ -69,10 +69,10 @@
 /*
  * Version numbers
  */
-#define VMXNET3_DRIVER_VERSION_STRING   "1.4.9.0-k"
+#define VMXNET3_DRIVER_VERSION_STRING   "1.4.a.0-k"
 
 /* a 32-bit int, each byte encode a verion number in VMXNET3_DRIVER_VERSION */
-#define VMXNET3_DRIVER_VERSION_NUM      0x01040900
+#define VMXNET3_DRIVER_VERSION_NUM      0x01040a00
 
 #if defined(CONFIG_PCI_MSI)
 	/* RSS only makes sense if MSI-X is supported. */
diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
index 1ce7420322ee..85c271c70d42 100644
--- a/drivers/net/vrf.c
+++ b/drivers/net/vrf.c
@@ -37,9 +37,6 @@
 #include <net/l3mdev.h>
 #include <net/fib_rules.h>
 
-#define RT_FL_TOS(oldflp4) \
-	((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
-
 #define DRV_NAME	"vrf"
 #define DRV_VERSION	"1.0"
 
@@ -137,6 +134,20 @@ static int vrf_local_xmit(struct sk_buff *skb, struct net_device *dev,
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
+static int vrf_ip6_local_out(struct net *net, struct sock *sk,
+			     struct sk_buff *skb)
+{
+	int err;
+
+	err = nf_hook(NFPROTO_IPV6, NF_INET_LOCAL_OUT, net,
+		      sk, skb, NULL, skb_dst(skb)->dev, dst_output);
+
+	if (likely(err == 1))
+		err = dst_output(net, sk, skb);
+
+	return err;
+}
+
 static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
 					   struct net_device *dev)
 {
@@ -151,7 +162,7 @@ static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
 		.flowlabel = ip6_flowinfo(iph),
 		.flowi6_mark = skb->mark,
 		.flowi6_proto = iph->nexthdr,
-		.flowi6_flags = FLOWI_FLAG_L3MDEV_SRC | FLOWI_FLAG_SKIP_NH_OIF,
+		.flowi6_flags = FLOWI_FLAG_SKIP_NH_OIF,
 	};
 	int ret = NET_XMIT_DROP;
 	struct dst_entry *dst;
@@ -207,7 +218,7 @@ static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
 	/* strip the ethernet header added for pass through VRF device */
 	__skb_pull(skb, skb_network_offset(skb));
 
-	ret = ip6_local_out(net, skb->sk, skb);
+	ret = vrf_ip6_local_out(net, skb->sk, skb);
 	if (unlikely(net_xmit_eval(ret)))
 		dev->stats.tx_errors++;
 	else
@@ -227,6 +238,20 @@ static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
 }
 #endif
 
+/* based on ip_local_out; can't use it b/c the dst is switched pointing to us */
+static int vrf_ip_local_out(struct net *net, struct sock *sk,
+			    struct sk_buff *skb)
+{
+	int err;
+
+	err = nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, net, sk,
+		      skb, NULL, skb_dst(skb)->dev, dst_output);
+	if (likely(err == 1))
+		err = dst_output(net, sk, skb);
+
+	return err;
+}
+
 static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb,
 					   struct net_device *vrf_dev)
 {
@@ -237,8 +262,7 @@ static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb,
 		.flowi4_oif = vrf_dev->ifindex,
 		.flowi4_iif = LOOPBACK_IFINDEX,
 		.flowi4_tos = RT_TOS(ip4h->tos),
-		.flowi4_flags = FLOWI_FLAG_ANYSRC | FLOWI_FLAG_L3MDEV_SRC |
-				FLOWI_FLAG_SKIP_NH_OIF,
+		.flowi4_flags = FLOWI_FLAG_ANYSRC | FLOWI_FLAG_SKIP_NH_OIF,
 		.daddr = ip4h->daddr,
 	};
 	struct net *net = dev_net(vrf_dev);
@@ -292,7 +316,7 @@ static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb,
 					       RT_SCOPE_LINK);
 	}
 
-	ret = ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
+	ret = vrf_ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
 	if (unlikely(net_xmit_eval(ret)))
 		vrf_dev->stats.tx_errors++;
 	else
@@ -377,6 +401,43 @@ static int vrf_output6(struct net *net, struct sock *sk, struct sk_buff *skb)
 			    !(IP6CB(skb)->flags & IP6SKB_REROUTED));
 }
 
+/* set dst on skb to send packet to us via dev_xmit path. Allows
+ * packet to go through device based features such as qdisc, netfilter
+ * hooks and packet sockets with skb->dev set to vrf device.
+ */
+static struct sk_buff *vrf_ip6_out(struct net_device *vrf_dev,
+				   struct sock *sk,
+				   struct sk_buff *skb)
+{
+	struct net_vrf *vrf = netdev_priv(vrf_dev);
+	struct dst_entry *dst = NULL;
+	struct rt6_info *rt6;
+
+	/* don't divert link scope packets */
+	if (rt6_need_strict(&ipv6_hdr(skb)->daddr))
+		return skb;
+
+	rcu_read_lock();
+
+	rt6 = rcu_dereference(vrf->rt6);
+	if (likely(rt6)) {
+		dst = &rt6->dst;
+		dst_hold(dst);
+	}
+
+	rcu_read_unlock();
+
+	if (unlikely(!dst)) {
+		vrf_tx_error(vrf_dev, skb);
+		return NULL;
+	}
+
+	skb_dst_drop(skb);
+	skb_dst_set(skb, dst);
+
+	return skb;
+}
+
 /* holding rtnl */
 static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf)
 {
@@ -463,6 +524,13 @@ out:
 	return rc;
 }
 #else
+static struct sk_buff *vrf_ip6_out(struct net_device *vrf_dev,
+				   struct sock *sk,
+				   struct sk_buff *skb)
+{
+	return skb;
+}
+
 static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf)
 {
 }
@@ -531,6 +599,55 @@ static int vrf_output(struct net *net, struct sock *sk, struct sk_buff *skb)
 			    !(IPCB(skb)->flags & IPSKB_REROUTED));
 }
 
+/* set dst on skb to send packet to us via dev_xmit path. Allows
+ * packet to go through device based features such as qdisc, netfilter
+ * hooks and packet sockets with skb->dev set to vrf device.
+ */
+static struct sk_buff *vrf_ip_out(struct net_device *vrf_dev,
+				  struct sock *sk,
+				  struct sk_buff *skb)
+{
+	struct net_vrf *vrf = netdev_priv(vrf_dev);
+	struct dst_entry *dst = NULL;
+	struct rtable *rth;
+
+	rcu_read_lock();
+
+	rth = rcu_dereference(vrf->rth);
+	if (likely(rth)) {
+		dst = &rth->dst;
+		dst_hold(dst);
+	}
+
+	rcu_read_unlock();
+
+	if (unlikely(!dst)) {
+		vrf_tx_error(vrf_dev, skb);
+		return NULL;
+	}
+
+	skb_dst_drop(skb);
+	skb_dst_set(skb, dst);
+
+	return skb;
+}
+
+/* called with rcu lock held */
+static struct sk_buff *vrf_l3_out(struct net_device *vrf_dev,
+				  struct sock *sk,
+				  struct sk_buff *skb,
+				  u16 proto)
+{
+	switch (proto) {
+	case AF_INET:
+		return vrf_ip_out(vrf_dev, sk, skb);
+	case AF_INET6:
+		return vrf_ip6_out(vrf_dev, sk, skb);
+	}
+
+	return skb;
+}
+
 /* holding rtnl */
 static void vrf_rtable_release(struct net_device *dev, struct net_vrf *vrf)
 {
@@ -722,63 +839,6 @@ static u32 vrf_fib_table(const struct net_device *dev)
 	return vrf->tb_id;
 }
 
-static struct rtable *vrf_get_rtable(const struct net_device *dev,
-				     const struct flowi4 *fl4)
-{
-	struct rtable *rth = NULL;
-
-	if (!(fl4->flowi4_flags & FLOWI_FLAG_L3MDEV_SRC)) {
-		struct net_vrf *vrf = netdev_priv(dev);
-
-		rcu_read_lock();
-
-		rth = rcu_dereference(vrf->rth);
-		if (likely(rth))
-			dst_hold(&rth->dst);
-
-		rcu_read_unlock();
-	}
-
-	return rth;
-}
-
-/* called under rcu_read_lock */
-static int vrf_get_saddr(struct net_device *dev, struct flowi4 *fl4)
-{
-	struct fib_result res = { .tclassid = 0 };
-	struct net *net = dev_net(dev);
-	u32 orig_tos = fl4->flowi4_tos;
-	u8 flags = fl4->flowi4_flags;
-	u8 scope = fl4->flowi4_scope;
-	u8 tos = RT_FL_TOS(fl4);
-	int rc;
-
-	if (unlikely(!fl4->daddr))
-		return 0;
-
-	fl4->flowi4_flags |= FLOWI_FLAG_SKIP_NH_OIF;
-	fl4->flowi4_iif = LOOPBACK_IFINDEX;
-	/* make sure oif is set to VRF device for lookup */
-	fl4->flowi4_oif = dev->ifindex;
-	fl4->flowi4_tos = tos & IPTOS_RT_MASK;
-	fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
-			     RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
-
-	rc = fib_lookup(net, fl4, &res, 0);
-	if (!rc) {
-		if (res.type == RTN_LOCAL)
-			fl4->saddr = res.fi->fib_prefsrc ? : fl4->daddr;
-		else
-			fib_select_path(net, &res, fl4, -1);
-	}
-
-	fl4->flowi4_flags = flags;
-	fl4->flowi4_tos = orig_tos;
-	fl4->flowi4_scope = scope;
-
-	return rc;
-}
-
 static int vrf_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
 {
 	return 0;
@@ -970,106 +1030,44 @@ static struct sk_buff *vrf_l3_rcv(struct net_device *vrf_dev,
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
-static struct dst_entry *vrf_get_rt6_dst(const struct net_device *dev,
-					 struct flowi6 *fl6)
+/* send to link-local or multicast address via interface enslaved to
+ * VRF device. Force lookup to VRF table without changing flow struct
+ */
+static struct dst_entry *vrf_link_scope_lookup(const struct net_device *dev,
+					      struct flowi6 *fl6)
 {
-	bool need_strict = rt6_need_strict(&fl6->daddr);
-	struct net_vrf *vrf = netdev_priv(dev);
 	struct net *net = dev_net(dev);
+	int flags = RT6_LOOKUP_F_IFACE;
 	struct dst_entry *dst = NULL;
 	struct rt6_info *rt;
 
-	/* send to link-local or multicast address */
-	if (need_strict) {
-		int flags = RT6_LOOKUP_F_IFACE;
-
-		/* VRF device does not have a link-local address and
-		 * sending packets to link-local or mcast addresses over
-		 * a VRF device does not make sense
-		 */
-		if (fl6->flowi6_oif == dev->ifindex) {
-			struct dst_entry *dst = &net->ipv6.ip6_null_entry->dst;
-
-			dst_hold(dst);
-			return dst;
-		}
-
-		if (!ipv6_addr_any(&fl6->saddr))
-			flags |= RT6_LOOKUP_F_HAS_SADDR;
-
-		rt = vrf_ip6_route_lookup(net, dev, fl6, fl6->flowi6_oif, flags);
-		if (rt)
-			dst = &rt->dst;
-
-	} else if (!(fl6->flowi6_flags & FLOWI_FLAG_L3MDEV_SRC)) {
-
-		rcu_read_lock();
-
-		rt = rcu_dereference(vrf->rt6);
-		if (likely(rt)) {
-			dst = &rt->dst;
-			dst_hold(dst);
-		}
-
-		rcu_read_unlock();
+	/* VRF device does not have a link-local address and
+	 * sending packets to link-local or mcast addresses over
+	 * a VRF device does not make sense
+	 */
+	if (fl6->flowi6_oif == dev->ifindex) {
+		dst = &net->ipv6.ip6_null_entry->dst;
+		dst_hold(dst);
+		return dst;
 	}
 
-	/* make sure oif is set to VRF device for lookup */
-	if (!need_strict)
-		fl6->flowi6_oif = dev->ifindex;
-
-	return dst;
-}
-
-/* called under rcu_read_lock */
-static int vrf_get_saddr6(struct net_device *dev, const struct sock *sk,
-			  struct flowi6 *fl6)
-{
-	struct net *net = dev_net(dev);
-	struct dst_entry *dst;
-	struct rt6_info *rt;
-	int err;
-
-	if (rt6_need_strict(&fl6->daddr)) {
-		rt = vrf_ip6_route_lookup(net, dev, fl6, fl6->flowi6_oif,
-					  RT6_LOOKUP_F_IFACE);
-		if (unlikely(!rt))
-			return 0;
+	if (!ipv6_addr_any(&fl6->saddr))
+		flags |= RT6_LOOKUP_F_HAS_SADDR;
 
+	rt = vrf_ip6_route_lookup(net, dev, fl6, fl6->flowi6_oif, flags);
+	if (rt)
 		dst = &rt->dst;
-	} else {
-		__u8 flags = fl6->flowi6_flags;
 
-		fl6->flowi6_flags |= FLOWI_FLAG_L3MDEV_SRC;
-		fl6->flowi6_flags |= FLOWI_FLAG_SKIP_NH_OIF;
-
-		dst = ip6_route_output(net, sk, fl6);
-		rt = (struct rt6_info *)dst;
-
-		fl6->flowi6_flags = flags;
-	}
-
-	err = dst->error;
-	if (!err) {
-		err = ip6_route_get_saddr(net, rt, &fl6->daddr,
-					  sk ? inet6_sk(sk)->srcprefs : 0,
-					  &fl6->saddr);
-	}
-
-	dst_release(dst);
-
-	return err;
+	return dst;
 }
 #endif
 
 static const struct l3mdev_ops vrf_l3mdev_ops = {
 	.l3mdev_fib_table	= vrf_fib_table,
-	.l3mdev_get_rtable	= vrf_get_rtable,
-	.l3mdev_get_saddr	= vrf_get_saddr,
 	.l3mdev_l3_rcv		= vrf_l3_rcv,
+	.l3mdev_l3_out		= vrf_l3_out,
 #if IS_ENABLED(CONFIG_IPV6)
-	.l3mdev_get_rt6_dst	= vrf_get_rt6_dst,
-	.l3mdev_get_saddr6	= vrf_get_saddr6,
+	.l3mdev_link_scope_lookup = vrf_link_scope_lookup,
 #endif
 };
 
diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index da4e3d6632f6..e7d16687538b 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -27,7 +27,6 @@
 #include <net/net_namespace.h>
 #include <net/netns/generic.h>
 #include <net/vxlan.h>
-#include <net/protocol.h>
 
 #if IS_ENABLED(CONFIG_IPV6)
 #include <net/ip6_tunnel.h>
@@ -288,7 +287,7 @@ static int vxlan_fdb_info(struct sk_buff *skb, struct vxlan_dev *vxlan,
 
 	if (!net_eq(dev_net(vxlan->dev), vxlan->net) &&
 	    nla_put_s32(skb, NDA_LINK_NETNSID,
-			peernet2id_alloc(dev_net(vxlan->dev), vxlan->net)))
+			peernet2id(dev_net(vxlan->dev), vxlan->net)))
 		goto nla_put_failure;
 
 	if (send_eth && nla_put(skb, NDA_LLADDR, ETH_ALEN, &fdb->eth_addr))
@@ -861,20 +860,20 @@ out:
 /* Dump forwarding table */
 static int vxlan_fdb_dump(struct sk_buff *skb, struct netlink_callback *cb,
 			  struct net_device *dev,
-			  struct net_device *filter_dev, int idx)
+			  struct net_device *filter_dev, int *idx)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	unsigned int h;
+	int err = 0;
 
 	for (h = 0; h < FDB_HASH_SIZE; ++h) {
 		struct vxlan_fdb *f;
-		int err;
 
 		hlist_for_each_entry_rcu(f, &vxlan->fdb_head[h], hlist) {
 			struct vxlan_rdst *rd;
 
 			list_for_each_entry_rcu(rd, &f->remotes, list) {
-				if (idx < cb->args[0])
+				if (*idx < cb->args[2])
 					goto skip;
 
 				err = vxlan_fdb_info(skb, vxlan, f,
@@ -882,17 +881,15 @@ static int vxlan_fdb_dump(struct sk_buff *skb, struct netlink_callback *cb,
 						     cb->nlh->nlmsg_seq,
 						     RTM_NEWNEIGH,
 						     NLM_F_MULTI, rd);
-				if (err < 0) {
-					cb->args[1] = err;
+				if (err < 0)
 					goto out;
-				}
 skip:
-				++idx;
+				*idx += 1;
 			}
 		}
 	}
 out:
-	return idx;
+	return err;
 }
 
 /* Watch incoming packets to learn mapping between Ethernet address
@@ -1294,7 +1291,7 @@ static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
 		struct metadata_dst *tun_dst;
 
 		tun_dst = udp_tun_rx_dst(skb, vxlan_get_sk_family(vs), TUNNEL_KEY,
-					 vxlan_vni_to_tun_id(vni), sizeof(*md));
+					 key32_to_tunnel_id(vni), sizeof(*md));
 
 		if (!tun_dst)
 			goto drop;
@@ -1811,7 +1808,7 @@ static struct rtable *vxlan_get_route(struct vxlan_dev *vxlan,
 	fl4.flowi4_mark = skb->mark;
 	fl4.flowi4_proto = IPPROTO_UDP;
 	fl4.daddr = daddr;
-	fl4.saddr = vxlan->cfg.saddr.sin.sin_addr.s_addr;
+	fl4.saddr = *saddr;
 
 	rt = ip_route_output_key(vxlan->net, &fl4);
 	if (!IS_ERR(rt)) {
@@ -1847,7 +1844,7 @@ static struct dst_entry *vxlan6_get_route(struct vxlan_dev *vxlan,
 	memset(&fl6, 0, sizeof(fl6));
 	fl6.flowi6_oif = oif;
 	fl6.daddr = *daddr;
-	fl6.saddr = vxlan->cfg.saddr.sin6.sin6_addr;
+	fl6.saddr = *saddr;
 	fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tos), label);
 	fl6.flowi6_mark = skb->mark;
 	fl6.flowi6_proto = IPPROTO_UDP;
@@ -1920,7 +1917,8 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 	struct rtable *rt = NULL;
 	const struct iphdr *old_iph;
 	union vxlan_addr *dst;
-	union vxlan_addr remote_ip;
+	union vxlan_addr remote_ip, local_ip;
+	union vxlan_addr *src;
 	struct vxlan_metadata _md;
 	struct vxlan_metadata *md = &_md;
 	__be16 src_port = 0, dst_port;
@@ -1938,6 +1936,7 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		dst_port = rdst->remote_port ? rdst->remote_port : vxlan->cfg.dst_port;
 		vni = rdst->remote_vni;
 		dst = &rdst->remote_ip;
+		src = &vxlan->cfg.saddr;
 		dst_cache = &rdst->dst_cache;
 	} else {
 		if (!info) {
@@ -1946,13 +1945,17 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 			goto drop;
 		}
 		dst_port = info->key.tp_dst ? : vxlan->cfg.dst_port;
-		vni = vxlan_tun_id_to_vni(info->key.tun_id);
+		vni = tunnel_id_to_key32(info->key.tun_id);
 		remote_ip.sa.sa_family = ip_tunnel_info_af(info);
-		if (remote_ip.sa.sa_family == AF_INET)
+		if (remote_ip.sa.sa_family == AF_INET) {
 			remote_ip.sin.sin_addr.s_addr = info->key.u.ipv4.dst;
-		else
+			local_ip.sin.sin_addr.s_addr = info->key.u.ipv4.src;
+		} else {
 			remote_ip.sin6.sin6_addr = info->key.u.ipv6.dst;
+			local_ip.sin6.sin6_addr = info->key.u.ipv6.src;
+		}
 		dst = &remote_ip;
+		src = &local_ip;
 		dst_cache = &info->dst_cache;
 	}
 
@@ -1992,15 +1995,14 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 	}
 
 	if (dst->sa.sa_family == AF_INET) {
-		__be32 saddr;
-
 		if (!vxlan->vn4_sock)
 			goto drop;
 		sk = vxlan->vn4_sock->sock->sk;
 
 		rt = vxlan_get_route(vxlan, skb,
 				     rdst ? rdst->remote_ifindex : 0, tos,
-				     dst->sin.sin_addr.s_addr, &saddr,
+				     dst->sin.sin_addr.s_addr,
+				     &src->sin.sin_addr.s_addr,
 				     dst_cache, info);
 		if (IS_ERR(rt)) {
 			netdev_dbg(dev, "no route to %pI4\n",
@@ -2017,7 +2019,7 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		}
 
 		/* Bypass encapsulation if the destination is local */
-		if (rt->rt_flags & RTCF_LOCAL &&
+		if (!info && rt->rt_flags & RTCF_LOCAL &&
 		    !(rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))) {
 			struct vxlan_dev *dst_vxlan;
 
@@ -2043,13 +2045,12 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		if (err < 0)
 			goto xmit_tx_error;
 
-		udp_tunnel_xmit_skb(rt, sk, skb, saddr,
+		udp_tunnel_xmit_skb(rt, sk, skb, src->sin.sin_addr.s_addr,
 				    dst->sin.sin_addr.s_addr, tos, ttl, df,
 				    src_port, dst_port, xnet, !udp_sum);
 #if IS_ENABLED(CONFIG_IPV6)
 	} else {
 		struct dst_entry *ndst;
-		struct in6_addr saddr;
 		u32 rt6i_flags;
 
 		if (!vxlan->vn6_sock)
@@ -2058,7 +2059,8 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 
 		ndst = vxlan6_get_route(vxlan, skb,
 					rdst ? rdst->remote_ifindex : 0, tos,
-					label, &dst->sin6.sin6_addr, &saddr,
+					label, &dst->sin6.sin6_addr,
+					&src->sin6.sin6_addr,
 					dst_cache, info);
 		if (IS_ERR(ndst)) {
 			netdev_dbg(dev, "no route to %pI6\n",
@@ -2077,7 +2079,7 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 
 		/* Bypass encapsulation if the destination is local */
 		rt6i_flags = ((struct rt6_info *)ndst)->rt6i_flags;
-		if (rt6i_flags & RTF_LOCAL &&
+		if (!info && rt6i_flags & RTF_LOCAL &&
 		    !(rt6i_flags & (RTCF_BROADCAST | RTCF_MULTICAST))) {
 			struct vxlan_dev *dst_vxlan;
 
@@ -2101,10 +2103,12 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 				      vni, md, flags, udp_sum);
 		if (err < 0) {
 			dst_release(ndst);
+			dev->stats.tx_errors++;
 			return;
 		}
 		udp_tunnel6_xmit_skb(ndst, sk, skb, dev,
-				     &saddr, &dst->sin6.sin6_addr, tos, ttl,
+				     &src->sin6.sin6_addr,
+				     &dst->sin6.sin6_addr, tos, ttl,
 				     label, src_port, dst_port, !udp_sum);
 #endif
 	}
@@ -2776,14 +2780,15 @@ static int vxlan_dev_configure(struct net *src_net, struct net_device *dev,
 	struct net_device *lowerdev = NULL;
 
 	if (conf->flags & VXLAN_F_GPE) {
-		if (conf->flags & ~VXLAN_F_ALLOWED_GPE)
-			return -EINVAL;
 		/* For now, allow GPE only together with COLLECT_METADATA.
 		 * This can be relaxed later; in such case, the other side
 		 * of the PtP link will have to be provided.
 		 */
-		if (!(conf->flags & VXLAN_F_COLLECT_METADATA))
+		if ((conf->flags & ~VXLAN_F_ALLOWED_GPE) ||
+		    !(conf->flags & VXLAN_F_COLLECT_METADATA)) {
+			pr_info("unsupported combination of extensions\n");
 			return -EINVAL;
+		}
 
 		vxlan_raw_setup(dev);
 	} else {
@@ -2836,6 +2841,9 @@ static int vxlan_dev_configure(struct net *src_net, struct net_device *dev,
 			dev->mtu = lowerdev->mtu - (use_ipv6 ? VXLAN6_HEADROOM : VXLAN_HEADROOM);
 
 		needed_headroom = lowerdev->hard_header_len;
+	} else if (vxlan_addr_multicast(&dst->remote_ip)) {
+		pr_info("multicast destination requires interface to be specified\n");
+		return -EINVAL;
 	}
 
 	if (conf->mtu) {
@@ -2868,8 +2876,10 @@ static int vxlan_dev_configure(struct net *src_net, struct net_device *dev,
 		     tmp->cfg.saddr.sa.sa_family == AF_INET6) == use_ipv6 &&
 		    tmp->cfg.dst_port == vxlan->cfg.dst_port &&
 		    (tmp->flags & VXLAN_F_RCV_FLAGS) ==
-		    (vxlan->flags & VXLAN_F_RCV_FLAGS))
-		return -EEXIST;
+		    (vxlan->flags & VXLAN_F_RCV_FLAGS)) {
+			pr_info("duplicate VNI %u\n", be32_to_cpu(conf->vni));
+			return -EEXIST;
+		}
 	}
 
 	dev->ethtool_ops = &vxlan_ethtool_ops;
@@ -2903,7 +2913,6 @@ static int vxlan_newlink(struct net *src_net, struct net_device *dev,
 			 struct nlattr *tb[], struct nlattr *data[])
 {
 	struct vxlan_config conf;
-	int err;
 
 	memset(&conf, 0, sizeof(conf));
 
@@ -3012,26 +3021,7 @@ static int vxlan_newlink(struct net *src_net, struct net_device *dev,
 	if (tb[IFLA_MTU])
 		conf.mtu = nla_get_u32(tb[IFLA_MTU]);
 
-	err = vxlan_dev_configure(src_net, dev, &conf);
-	switch (err) {
-	case -ENODEV:
-		pr_info("ifindex %d does not exist\n", conf.remote_ifindex);
-		break;
-
-	case -EPERM:
-		pr_info("IPv6 is disabled via sysctl\n");
-		break;
-
-	case -EEXIST:
-		pr_info("duplicate VNI %u\n", be32_to_cpu(conf.vni));
-		break;
-
-	case -EINVAL:
-		pr_info("unsupported combination of extensions\n");
-		break;
-	}
-
-	return err;
+	return vxlan_dev_configure(src_net, dev, &conf);
 }
 
 static void vxlan_dellink(struct net_device *dev, struct list_head *head)
diff --git a/drivers/net/wan/fsl_ucc_hdlc.c b/drivers/net/wan/fsl_ucc_hdlc.c
index 6f044450b702..65647533b401 100644
--- a/drivers/net/wan/fsl_ucc_hdlc.c
+++ b/drivers/net/wan/fsl_ucc_hdlc.c
@@ -162,7 +162,7 @@ static int uhdlc_init(struct ucc_hdlc_private *priv)
 				ALIGNMENT_OF_UCC_HDLC_PRAM);
 
 	if (priv->ucc_pram_offset < 0) {
-		dev_err(priv->dev, "Can not allocate MURAM for hdlc prameter.\n");
+		dev_err(priv->dev, "Can not allocate MURAM for hdlc parameter.\n");
 		ret = -ENOMEM;
 		goto free_tx_bd;
 	}
@@ -295,11 +295,11 @@ free_ucc_pram:
 	qe_muram_free(priv->ucc_pram_offset);
 free_tx_bd:
 	dma_free_coherent(priv->dev,
-			  TX_BD_RING_LEN * sizeof(struct qe_bd),
+			  TX_BD_RING_LEN * sizeof(struct qe_bd *),
 			  priv->tx_bd_base, priv->dma_tx_bd);
 free_rx_bd:
 	dma_free_coherent(priv->dev,
-			  RX_BD_RING_LEN * sizeof(struct qe_bd),
+			  RX_BD_RING_LEN * sizeof(struct qe_bd *),
 			  priv->rx_bd_base, priv->dma_rx_bd);
 free_uccf:
 	ucc_fast_free(priv->uccf);
@@ -688,7 +688,7 @@ static void uhdlc_memclean(struct ucc_hdlc_private *priv)
 
 	if (priv->rx_bd_base) {
 		dma_free_coherent(priv->dev,
-				  RX_BD_RING_LEN * sizeof(struct qe_bd),
+				  RX_BD_RING_LEN * sizeof(struct qe_bd *),
 				  priv->rx_bd_base, priv->dma_rx_bd);
 
 		priv->rx_bd_base = NULL;
@@ -697,7 +697,7 @@ static void uhdlc_memclean(struct ucc_hdlc_private *priv)
 
 	if (priv->tx_bd_base) {
 		dma_free_coherent(priv->dev,
-				  TX_BD_RING_LEN * sizeof(struct qe_bd),
+				  TX_BD_RING_LEN * sizeof(struct qe_bd *),
 				  priv->tx_bd_base, priv->dma_tx_bd);
 
 		priv->tx_bd_base = NULL;
diff --git a/drivers/net/wan/sbni.c b/drivers/net/wan/sbni.c
index d98c7e57137d..3a421ca8a4d0 100644
--- a/drivers/net/wan/sbni.c
+++ b/drivers/net/wan/sbni.c
@@ -582,8 +582,8 @@ handle_channel( struct net_device  *dev )
 
 
 /*
- * Routine returns 1 if it need to acknoweledge received frame.
- * Empty frame received without errors won't be acknoweledged.
+ * Routine returns 1 if it needs to acknowledge received frame.
+ * Empty frame received without errors won't be acknowledged.
  */
 
 static int
diff --git a/drivers/net/wimax/i2400m/usb-notif.c b/drivers/net/wimax/i2400m/usb-notif.c
index fc1355d98bc6..5d429f816125 100644
--- a/drivers/net/wimax/i2400m/usb-notif.c
+++ b/drivers/net/wimax/i2400m/usb-notif.c
@@ -206,7 +206,6 @@ int i2400mu_notification_setup(struct i2400mu *i2400mu)
 	i2400mu->notif_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!i2400mu->notif_urb) {
 		ret = -ENOMEM;
-		dev_err(dev, "notification: cannot allocate URB\n");
 		goto error_alloc_urb;
 	}
 	epd = usb_get_epd(i2400mu->usb_iface,
diff --git a/drivers/net/wireless/ath/ar5523/ar5523.c b/drivers/net/wireless/ath/ar5523/ar5523.c
index 8aded24bcdf4..7a60d2e652da 100644
--- a/drivers/net/wireless/ath/ar5523/ar5523.c
+++ b/drivers/net/wireless/ath/ar5523/ar5523.c
@@ -706,10 +706,8 @@ static int ar5523_alloc_rx_bufs(struct ar5523 *ar)
 
 		data->ar = ar;
 		data->urb = usb_alloc_urb(0, GFP_KERNEL);
-		if (!data->urb) {
-			ar5523_err(ar, "could not allocate rx data urb\n");
+		if (!data->urb)
 			goto err;
-		}
 		list_add_tail(&data->list, &ar->rx_data_free);
 		atomic_inc(&ar->rx_data_free_cnt);
 	}
@@ -824,7 +822,6 @@ static void ar5523_tx_work_locked(struct ar5523 *ar)
 
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
-			ar5523_err(ar, "Failed to allocate TX urb\n");
 			ieee80211_free_txskb(ar->hw, skb);
 			continue;
 		}
@@ -949,10 +946,8 @@ static int ar5523_alloc_tx_cmd(struct ar5523 *ar)
 	init_completion(&cmd->done);
 
 	cmd->urb_tx = usb_alloc_urb(0, GFP_KERNEL);
-	if (!cmd->urb_tx) {
-		ar5523_err(ar, "could not allocate urb\n");
+	if (!cmd->urb_tx)
 		return -ENOMEM;
-	}
 	cmd->buf_tx = usb_alloc_coherent(ar->dev, AR5523_MAX_TXCMDSZ,
 					 GFP_KERNEL,
 					 &cmd->urb_tx->transfer_dma);
diff --git a/drivers/net/wireless/ath/ath10k/ahb.c b/drivers/net/wireless/ath/ath10k/ahb.c
index acec16b9cf49..766c63bf05c4 100644
--- a/drivers/net/wireless/ath/ath10k/ahb.c
+++ b/drivers/net/wireless/ath/ath10k/ahb.c
@@ -91,59 +91,37 @@ static int ath10k_ahb_clock_init(struct ath10k *ar)
 {
 	struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar);
 	struct device *dev;
-	int ret;
 
 	dev = &ar_ahb->pdev->dev;
 
-	ar_ahb->cmd_clk = clk_get(dev, "wifi_wcss_cmd");
+	ar_ahb->cmd_clk = devm_clk_get(dev, "wifi_wcss_cmd");
 	if (IS_ERR_OR_NULL(ar_ahb->cmd_clk)) {
 		ath10k_err(ar, "failed to get cmd clk: %ld\n",
 			   PTR_ERR(ar_ahb->cmd_clk));
-		ret = ar_ahb->cmd_clk ? PTR_ERR(ar_ahb->cmd_clk) : -ENODEV;
-		goto out;
+		return ar_ahb->cmd_clk ? PTR_ERR(ar_ahb->cmd_clk) : -ENODEV;
 	}
 
-	ar_ahb->ref_clk = clk_get(dev, "wifi_wcss_ref");
+	ar_ahb->ref_clk = devm_clk_get(dev, "wifi_wcss_ref");
 	if (IS_ERR_OR_NULL(ar_ahb->ref_clk)) {
 		ath10k_err(ar, "failed to get ref clk: %ld\n",
 			   PTR_ERR(ar_ahb->ref_clk));
-		ret = ar_ahb->ref_clk ? PTR_ERR(ar_ahb->ref_clk) : -ENODEV;
-		goto err_cmd_clk_put;
+		return ar_ahb->ref_clk ? PTR_ERR(ar_ahb->ref_clk) : -ENODEV;
 	}
 
-	ar_ahb->rtc_clk = clk_get(dev, "wifi_wcss_rtc");
+	ar_ahb->rtc_clk = devm_clk_get(dev, "wifi_wcss_rtc");
 	if (IS_ERR_OR_NULL(ar_ahb->rtc_clk)) {
 		ath10k_err(ar, "failed to get rtc clk: %ld\n",
 			   PTR_ERR(ar_ahb->rtc_clk));
-		ret = ar_ahb->rtc_clk ? PTR_ERR(ar_ahb->rtc_clk) : -ENODEV;
-		goto err_ref_clk_put;
+		return ar_ahb->rtc_clk ? PTR_ERR(ar_ahb->rtc_clk) : -ENODEV;
 	}
 
 	return 0;
-
-err_ref_clk_put:
-	clk_put(ar_ahb->ref_clk);
-
-err_cmd_clk_put:
-	clk_put(ar_ahb->cmd_clk);
-
-out:
-	return ret;
 }
 
 static void ath10k_ahb_clock_deinit(struct ath10k *ar)
 {
 	struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar);
 
-	if (!IS_ERR_OR_NULL(ar_ahb->cmd_clk))
-		clk_put(ar_ahb->cmd_clk);
-
-	if (!IS_ERR_OR_NULL(ar_ahb->ref_clk))
-		clk_put(ar_ahb->ref_clk);
-
-	if (!IS_ERR_OR_NULL(ar_ahb->rtc_clk))
-		clk_put(ar_ahb->rtc_clk);
-
 	ar_ahb->cmd_clk = NULL;
 	ar_ahb->ref_clk = NULL;
 	ar_ahb->rtc_clk = NULL;
@@ -213,92 +191,51 @@ static int ath10k_ahb_rst_ctrl_init(struct ath10k *ar)
 {
 	struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar);
 	struct device *dev;
-	int ret;
 
 	dev = &ar_ahb->pdev->dev;
 
-	ar_ahb->core_cold_rst = reset_control_get(dev, "wifi_core_cold");
-	if (IS_ERR_OR_NULL(ar_ahb->core_cold_rst)) {
+	ar_ahb->core_cold_rst = devm_reset_control_get(dev, "wifi_core_cold");
+	if (IS_ERR(ar_ahb->core_cold_rst)) {
 		ath10k_err(ar, "failed to get core cold rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->core_cold_rst));
-		ret = ar_ahb->core_cold_rst ?
-			PTR_ERR(ar_ahb->core_cold_rst) : -ENODEV;
-		goto out;
+		return PTR_ERR(ar_ahb->core_cold_rst);
 	}
 
-	ar_ahb->radio_cold_rst = reset_control_get(dev, "wifi_radio_cold");
-	if (IS_ERR_OR_NULL(ar_ahb->radio_cold_rst)) {
+	ar_ahb->radio_cold_rst = devm_reset_control_get(dev, "wifi_radio_cold");
+	if (IS_ERR(ar_ahb->radio_cold_rst)) {
 		ath10k_err(ar, "failed to get radio cold rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->radio_cold_rst));
-		ret = ar_ahb->radio_cold_rst ?
-			PTR_ERR(ar_ahb->radio_cold_rst) : -ENODEV;
-		goto err_core_cold_rst_put;
+		return PTR_ERR(ar_ahb->radio_cold_rst);
 	}
 
-	ar_ahb->radio_warm_rst = reset_control_get(dev, "wifi_radio_warm");
-	if (IS_ERR_OR_NULL(ar_ahb->radio_warm_rst)) {
+	ar_ahb->radio_warm_rst = devm_reset_control_get(dev, "wifi_radio_warm");
+	if (IS_ERR(ar_ahb->radio_warm_rst)) {
 		ath10k_err(ar, "failed to get radio warm rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->radio_warm_rst));
-		ret = ar_ahb->radio_warm_rst ?
-			PTR_ERR(ar_ahb->radio_warm_rst) : -ENODEV;
-		goto err_radio_cold_rst_put;
+		return PTR_ERR(ar_ahb->radio_warm_rst);
 	}
 
-	ar_ahb->radio_srif_rst = reset_control_get(dev, "wifi_radio_srif");
-	if (IS_ERR_OR_NULL(ar_ahb->radio_srif_rst)) {
+	ar_ahb->radio_srif_rst = devm_reset_control_get(dev, "wifi_radio_srif");
+	if (IS_ERR(ar_ahb->radio_srif_rst)) {
 		ath10k_err(ar, "failed to get radio srif rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->radio_srif_rst));
-		ret = ar_ahb->radio_srif_rst ?
-			PTR_ERR(ar_ahb->radio_srif_rst) : -ENODEV;
-		goto err_radio_warm_rst_put;
+		return PTR_ERR(ar_ahb->radio_srif_rst);
 	}
 
-	ar_ahb->cpu_init_rst = reset_control_get(dev, "wifi_cpu_init");
-	if (IS_ERR_OR_NULL(ar_ahb->cpu_init_rst)) {
+	ar_ahb->cpu_init_rst = devm_reset_control_get(dev, "wifi_cpu_init");
+	if (IS_ERR(ar_ahb->cpu_init_rst)) {
 		ath10k_err(ar, "failed to get cpu init rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->cpu_init_rst));
-		ret = ar_ahb->cpu_init_rst ?
-			PTR_ERR(ar_ahb->cpu_init_rst) : -ENODEV;
-		goto err_radio_srif_rst_put;
+		return PTR_ERR(ar_ahb->cpu_init_rst);
 	}
 
 	return 0;
-
-err_radio_srif_rst_put:
-	reset_control_put(ar_ahb->radio_srif_rst);
-
-err_radio_warm_rst_put:
-	reset_control_put(ar_ahb->radio_warm_rst);
-
-err_radio_cold_rst_put:
-	reset_control_put(ar_ahb->radio_cold_rst);
-
-err_core_cold_rst_put:
-	reset_control_put(ar_ahb->core_cold_rst);
-
-out:
-	return ret;
 }
 
 static void ath10k_ahb_rst_ctrl_deinit(struct ath10k *ar)
 {
 	struct ath10k_ahb *ar_ahb = ath10k_ahb_priv(ar);
 
-	if (!IS_ERR_OR_NULL(ar_ahb->core_cold_rst))
-		reset_control_put(ar_ahb->core_cold_rst);
-
-	if (!IS_ERR_OR_NULL(ar_ahb->radio_cold_rst))
-		reset_control_put(ar_ahb->radio_cold_rst);
-
-	if (!IS_ERR_OR_NULL(ar_ahb->radio_warm_rst))
-		reset_control_put(ar_ahb->radio_warm_rst);
-
-	if (!IS_ERR_OR_NULL(ar_ahb->radio_srif_rst))
-		reset_control_put(ar_ahb->radio_srif_rst);
-
-	if (!IS_ERR_OR_NULL(ar_ahb->cpu_init_rst))
-		reset_control_put(ar_ahb->cpu_init_rst);
-
 	ar_ahb->core_cold_rst = NULL;
 	ar_ahb->radio_cold_rst = NULL;
 	ar_ahb->radio_warm_rst = NULL;
@@ -462,13 +399,13 @@ static void ath10k_ahb_halt_chip(struct ath10k *ar)
 static irqreturn_t ath10k_ahb_interrupt_handler(int irq, void *arg)
 {
 	struct ath10k *ar = arg;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
 
 	if (!ath10k_pci_irq_pending(ar))
 		return IRQ_NONE;
 
 	ath10k_pci_disable_and_clear_legacy_irq(ar);
-	tasklet_schedule(&ar_pci->intr_tq);
+	ath10k_pci_irq_msi_fw_mask(ar);
+	napi_schedule(&ar->napi);
 
 	return IRQ_HANDLED;
 }
@@ -572,12 +509,13 @@ static int ath10k_ahb_resource_init(struct ath10k *ar)
 	ar_ahb->irq = platform_get_irq_byname(pdev, "legacy");
 	if (ar_ahb->irq < 0) {
 		ath10k_err(ar, "failed to get irq number: %d\n", ar_ahb->irq);
+		ret = ar_ahb->irq;
 		goto err_clock_deinit;
 	}
 
 	ath10k_dbg(ar, ATH10K_DBG_BOOT, "irq: %d\n", ar_ahb->irq);
 
-	ath10k_dbg(ar, ATH10K_DBG_BOOT, "mem: 0x%p mem_len: %lu gcc mem: 0x%p tcsr_mem: 0x%p\n",
+	ath10k_dbg(ar, ATH10K_DBG_BOOT, "mem: 0x%pK mem_len: %lu gcc mem: 0x%pK tcsr_mem: 0x%pK\n",
 		   ar_ahb->mem, ar_ahb->mem_len,
 		   ar_ahb->gcc_mem, ar_ahb->tcsr_mem);
 	return 0;
@@ -717,6 +655,9 @@ static void ath10k_ahb_hif_stop(struct ath10k *ar)
 	synchronize_irq(ar_ahb->irq);
 
 	ath10k_pci_flush(ar);
+
+	napi_synchronize(&ar->napi);
+	napi_disable(&ar->napi);
 }
 
 static int ath10k_ahb_hif_power_up(struct ath10k *ar)
@@ -748,6 +689,7 @@ static int ath10k_ahb_hif_power_up(struct ath10k *ar)
 		ath10k_err(ar, "could not wake up target CPU: %d\n", ret);
 		goto err_ce_deinit;
 	}
+	napi_enable(&ar->napi);
 
 	return 0;
 
@@ -831,7 +773,7 @@ static int ath10k_ahb_probe(struct platform_device *pdev)
 		goto err_resource_deinit;
 	}
 
-	ath10k_pci_init_irq_tasklets(ar);
+	ath10k_pci_init_napi(ar);
 
 	ret = ath10k_ahb_request_irq_legacy(ar);
 	if (ret)
@@ -846,6 +788,7 @@ static int ath10k_ahb_probe(struct platform_device *pdev)
 	chip_id = ath10k_ahb_soc_read32(ar, SOC_CHIP_ID_ADDRESS);
 	if (chip_id == 0xffffffff) {
 		ath10k_err(ar, "failed to get chip id\n");
+		ret = -ENODEV;
 		goto err_halt_device;
 	}
 
diff --git a/drivers/net/wireless/ath/ath10k/bmi.c b/drivers/net/wireless/ath/ath10k/bmi.c
index 3d29b0875b3e..2872d347ea78 100644
--- a/drivers/net/wireless/ath/ath10k/bmi.c
+++ b/drivers/net/wireless/ath/ath10k/bmi.c
@@ -221,7 +221,7 @@ int ath10k_bmi_lz_data(struct ath10k *ar, const void *buffer, u32 length)
 	u32 txlen;
 	int ret;
 
-	ath10k_dbg(ar, ATH10K_DBG_BMI, "bmi lz data buffer 0x%p length %d\n",
+	ath10k_dbg(ar, ATH10K_DBG_BMI, "bmi lz data buffer 0x%pK length %d\n",
 		   buffer, length);
 
 	if (ar->bmi.done_sent) {
@@ -287,7 +287,7 @@ int ath10k_bmi_fast_download(struct ath10k *ar,
 	int ret;
 
 	ath10k_dbg(ar, ATH10K_DBG_BMI,
-		   "bmi fast download address 0x%x buffer 0x%p length %d\n",
+		   "bmi fast download address 0x%x buffer 0x%pK length %d\n",
 		   address, buffer, length);
 
 	ret = ath10k_bmi_lz_stream_start(ar, address);
diff --git a/drivers/net/wireless/ath/ath10k/ce.c b/drivers/net/wireless/ath/ath10k/ce.c
index 9fb8d7472d18..0b4d79659884 100644
--- a/drivers/net/wireless/ath/ath10k/ce.c
+++ b/drivers/net/wireless/ath/ath10k/ce.c
@@ -39,7 +39,7 @@
  * chooses what to send (buffer address, length). The destination
  * side keeps a supply of "anonymous receive buffers" available and
  * it handles incoming data as it arrives (when the destination
- * recieves an interrupt).
+ * receives an interrupt).
  *
  * The sender may send a simple buffer (address/length) or it may
  * send a small list of buffers.  When a small list is sent, hardware
@@ -433,6 +433,13 @@ void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 	unsigned int nentries_mask = dest_ring->nentries_mask;
 	unsigned int write_index = dest_ring->write_index;
 	u32 ctrl_addr = pipe->ctrl_addr;
+	u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
+
+	/* Prevent CE ring stuck issue that will occur when ring is full.
+	 * Make sure that write index is 1 less than read index.
+	 */
+	if ((cur_write_idx + nentries)  == dest_ring->sw_index)
+		nentries -= 1;
 
 	write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
@@ -840,7 +847,7 @@ static int ath10k_ce_init_src_ring(struct ath10k *ar,
 	ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
 
 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
-		   "boot init ce src ring id %d entries %d base_addr %p\n",
+		   "boot init ce src ring id %d entries %d base_addr %pK\n",
 		   ce_id, nentries, src_ring->base_addr_owner_space);
 
 	return 0;
@@ -874,7 +881,7 @@ static int ath10k_ce_init_dest_ring(struct ath10k *ar,
 	ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
 
 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
-		   "boot ce dest ring id %d entries %d base_addr %p\n",
+		   "boot ce dest ring id %d entries %d base_addr %pK\n",
 		   ce_id, nentries, dest_ring->base_addr_owner_space);
 
 	return 0;
diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c
index e88982921aa3..21ae8d663e67 100644
--- a/drivers/net/wireless/ath/ath10k/core.c
+++ b/drivers/net/wireless/ath/ath10k/core.c
@@ -60,7 +60,6 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.otp_exe_param = 0,
 		.channel_counters_freq_hz = 88000,
 		.max_probe_resp_desc_thres = 0,
-		.hw_4addr_pad = ATH10K_HW_4ADDR_PAD_AFTER,
 		.cal_data_len = 2116,
 		.fw = {
 			.dir = QCA988X_HW_2_0_FW_DIR,
@@ -68,6 +67,8 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA988X_BOARD_DATA_SZ,
 			.board_ext_size = QCA988X_BOARD_EXT_DATA_SZ,
 		},
+		.hw_ops = &qca988x_ops,
+		.decap_align_bytes = 4,
 	},
 	{
 		.id = QCA9887_HW_1_0_VERSION,
@@ -79,7 +80,6 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.otp_exe_param = 0,
 		.channel_counters_freq_hz = 88000,
 		.max_probe_resp_desc_thres = 0,
-		.hw_4addr_pad = ATH10K_HW_4ADDR_PAD_AFTER,
 		.cal_data_len = 2116,
 		.fw = {
 			.dir = QCA9887_HW_1_0_FW_DIR,
@@ -87,6 +87,8 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA9887_BOARD_DATA_SZ,
 			.board_ext_size = QCA9887_BOARD_EXT_DATA_SZ,
 		},
+		.hw_ops = &qca988x_ops,
+		.decap_align_bytes = 4,
 	},
 	{
 		.id = QCA6174_HW_2_1_VERSION,
@@ -104,6 +106,8 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA6174_BOARD_DATA_SZ,
 			.board_ext_size = QCA6174_BOARD_EXT_DATA_SZ,
 		},
+		.hw_ops = &qca988x_ops,
+		.decap_align_bytes = 4,
 	},
 	{
 		.id = QCA6174_HW_2_1_VERSION,
@@ -114,7 +118,6 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.otp_exe_param = 0,
 		.channel_counters_freq_hz = 88000,
 		.max_probe_resp_desc_thres = 0,
-		.hw_4addr_pad = ATH10K_HW_4ADDR_PAD_AFTER,
 		.cal_data_len = 8124,
 		.fw = {
 			.dir = QCA6174_HW_2_1_FW_DIR,
@@ -122,6 +125,8 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA6174_BOARD_DATA_SZ,
 			.board_ext_size = QCA6174_BOARD_EXT_DATA_SZ,
 		},
+		.hw_ops = &qca988x_ops,
+		.decap_align_bytes = 4,
 	},
 	{
 		.id = QCA6174_HW_3_0_VERSION,
@@ -132,7 +137,6 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.otp_exe_param = 0,
 		.channel_counters_freq_hz = 88000,
 		.max_probe_resp_desc_thres = 0,
-		.hw_4addr_pad = ATH10K_HW_4ADDR_PAD_AFTER,
 		.cal_data_len = 8124,
 		.fw = {
 			.dir = QCA6174_HW_3_0_FW_DIR,
@@ -140,6 +144,8 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA6174_BOARD_DATA_SZ,
 			.board_ext_size = QCA6174_BOARD_EXT_DATA_SZ,
 		},
+		.hw_ops = &qca988x_ops,
+		.decap_align_bytes = 4,
 	},
 	{
 		.id = QCA6174_HW_3_2_VERSION,
@@ -150,7 +156,6 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.otp_exe_param = 0,
 		.channel_counters_freq_hz = 88000,
 		.max_probe_resp_desc_thres = 0,
-		.hw_4addr_pad = ATH10K_HW_4ADDR_PAD_AFTER,
 		.cal_data_len = 8124,
 		.fw = {
 			/* uses same binaries as hw3.0 */
@@ -159,6 +164,8 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA6174_BOARD_DATA_SZ,
 			.board_ext_size = QCA6174_BOARD_EXT_DATA_SZ,
 		},
+		.hw_ops = &qca988x_ops,
+		.decap_align_bytes = 4,
 	},
 	{
 		.id = QCA99X0_HW_2_0_DEV_VERSION,
@@ -171,7 +178,6 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.cck_rate_map_rev2 = true,
 		.channel_counters_freq_hz = 150000,
 		.max_probe_resp_desc_thres = 24,
-		.hw_4addr_pad = ATH10K_HW_4ADDR_PAD_BEFORE,
 		.tx_chain_mask = 0xf,
 		.rx_chain_mask = 0xf,
 		.max_spatial_stream = 4,
@@ -182,6 +188,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA99X0_BOARD_DATA_SZ,
 			.board_ext_size = QCA99X0_BOARD_EXT_DATA_SZ,
 		},
+		.sw_decrypt_mcast_mgmt = true,
+		.hw_ops = &qca99x0_ops,
+		.decap_align_bytes = 1,
 	},
 	{
 		.id = QCA9984_HW_1_0_DEV_VERSION,
@@ -194,7 +203,6 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.cck_rate_map_rev2 = true,
 		.channel_counters_freq_hz = 150000,
 		.max_probe_resp_desc_thres = 24,
-		.hw_4addr_pad = ATH10K_HW_4ADDR_PAD_BEFORE,
 		.tx_chain_mask = 0xf,
 		.rx_chain_mask = 0xf,
 		.max_spatial_stream = 4,
@@ -205,6 +213,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA99X0_BOARD_DATA_SZ,
 			.board_ext_size = QCA99X0_BOARD_EXT_DATA_SZ,
 		},
+		.sw_decrypt_mcast_mgmt = true,
+		.hw_ops = &qca99x0_ops,
+		.decap_align_bytes = 1,
 	},
 	{
 		.id = QCA9888_HW_2_0_DEV_VERSION,
@@ -216,7 +227,6 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.continuous_frag_desc = true,
 		.channel_counters_freq_hz = 150000,
 		.max_probe_resp_desc_thres = 24,
-		.hw_4addr_pad = ATH10K_HW_4ADDR_PAD_BEFORE,
 		.tx_chain_mask = 3,
 		.rx_chain_mask = 3,
 		.max_spatial_stream = 2,
@@ -227,6 +237,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA99X0_BOARD_DATA_SZ,
 			.board_ext_size = QCA99X0_BOARD_EXT_DATA_SZ,
 		},
+		.sw_decrypt_mcast_mgmt = true,
+		.hw_ops = &qca99x0_ops,
+		.decap_align_bytes = 1,
 	},
 	{
 		.id = QCA9377_HW_1_0_DEV_VERSION,
@@ -244,6 +257,8 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA9377_BOARD_DATA_SZ,
 			.board_ext_size = QCA9377_BOARD_EXT_DATA_SZ,
 		},
+		.hw_ops = &qca988x_ops,
+		.decap_align_bytes = 4,
 	},
 	{
 		.id = QCA9377_HW_1_1_DEV_VERSION,
@@ -261,6 +276,8 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA9377_BOARD_DATA_SZ,
 			.board_ext_size = QCA9377_BOARD_EXT_DATA_SZ,
 		},
+		.hw_ops = &qca988x_ops,
+		.decap_align_bytes = 4,
 	},
 	{
 		.id = QCA4019_HW_1_0_DEV_VERSION,
@@ -274,7 +291,6 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.cck_rate_map_rev2 = true,
 		.channel_counters_freq_hz = 125000,
 		.max_probe_resp_desc_thres = 24,
-		.hw_4addr_pad = ATH10K_HW_4ADDR_PAD_BEFORE,
 		.tx_chain_mask = 0x3,
 		.rx_chain_mask = 0x3,
 		.max_spatial_stream = 2,
@@ -285,6 +301,9 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 			.board_size = QCA4019_BOARD_DATA_SZ,
 			.board_ext_size = QCA4019_BOARD_EXT_DATA_SZ,
 		},
+		.sw_decrypt_mcast_mgmt = true,
+		.hw_ops = &qca99x0_ops,
+		.decap_align_bytes = 1,
 	},
 };
 
@@ -304,6 +323,7 @@ static const char *const ath10k_core_fw_feature_str[] = {
 	[ATH10K_FW_FEATURE_MFP_SUPPORT] = "mfp",
 	[ATH10K_FW_FEATURE_PEER_FLOW_CONTROL] = "peer-flow-ctrl",
 	[ATH10K_FW_FEATURE_BTCOEX_PARAM] = "btcoex-param",
+	[ATH10K_FW_FEATURE_SKIP_NULL_FUNC_WAR] = "skip-null-func-war",
 };
 
 static unsigned int ath10k_core_get_fw_feature_str(char *buf,
@@ -699,7 +719,7 @@ static int ath10k_download_and_run_otp(struct ath10k *ar)
 
 	if (!ar->running_fw->fw_file.otp_data ||
 	    !ar->running_fw->fw_file.otp_len) {
-		ath10k_warn(ar, "Not running otp, calibration will be incorrect (otp-data %p otp_len %zd)!\n",
+		ath10k_warn(ar, "Not running otp, calibration will be incorrect (otp-data %pK otp_len %zd)!\n",
 			    ar->running_fw->fw_file.otp_data,
 			    ar->running_fw->fw_file.otp_len);
 		return 0;
@@ -745,7 +765,7 @@ static int ath10k_download_fw(struct ath10k *ar)
 	data = ar->running_fw->fw_file.firmware_data;
 	data_len = ar->running_fw->fw_file.firmware_len;
 
-	ret = ath10k_swap_code_seg_configure(ar);
+	ret = ath10k_swap_code_seg_configure(ar, &ar->running_fw->fw_file);
 	if (ret) {
 		ath10k_err(ar, "failed to configure fw code swap: %d\n",
 			   ret);
@@ -753,7 +773,7 @@ static int ath10k_download_fw(struct ath10k *ar)
 	}
 
 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
-		   "boot uploading firmware image %p len %d\n",
+		   "boot uploading firmware image %pK len %d\n",
 		   data, data_len);
 
 	ret = ath10k_bmi_fast_download(ar, address, data, data_len);
@@ -787,7 +807,7 @@ static void ath10k_core_free_firmware_files(struct ath10k *ar)
 	if (!IS_ERR(ar->pre_cal_file))
 		release_firmware(ar->pre_cal_file);
 
-	ath10k_swap_code_seg_release(ar);
+	ath10k_swap_code_seg_release(ar, &ar->normal_mode_fw.fw_file);
 
 	ar->normal_mode_fw.fw_file.otp_data = NULL;
 	ar->normal_mode_fw.fw_file.otp_len = 0;
@@ -1497,14 +1517,14 @@ static void ath10k_core_restart(struct work_struct *work)
 
 	ieee80211_stop_queues(ar->hw);
 	ath10k_drain_tx(ar);
-	complete_all(&ar->scan.started);
-	complete_all(&ar->scan.completed);
-	complete_all(&ar->scan.on_channel);
-	complete_all(&ar->offchan_tx_completed);
-	complete_all(&ar->install_key_done);
-	complete_all(&ar->vdev_setup_done);
-	complete_all(&ar->thermal.wmi_sync);
-	complete_all(&ar->bss_survey_done);
+	complete(&ar->scan.started);
+	complete(&ar->scan.completed);
+	complete(&ar->scan.on_channel);
+	complete(&ar->offchan_tx_completed);
+	complete(&ar->install_key_done);
+	complete(&ar->vdev_setup_done);
+	complete(&ar->thermal.wmi_sync);
+	complete(&ar->bss_survey_done);
 	wake_up(&ar->htt.empty_tx_wq);
 	wake_up(&ar->wmi.tx_credits_wq);
 	wake_up(&ar->peer_mapping_wq);
@@ -1705,6 +1725,55 @@ static int ath10k_core_init_firmware_features(struct ath10k *ar)
 	return 0;
 }
 
+static int ath10k_core_reset_rx_filter(struct ath10k *ar)
+{
+	int ret;
+	int vdev_id;
+	int vdev_type;
+	int vdev_subtype;
+	const u8 *vdev_addr;
+
+	vdev_id = 0;
+	vdev_type = WMI_VDEV_TYPE_STA;
+	vdev_subtype = ath10k_wmi_get_vdev_subtype(ar, WMI_VDEV_SUBTYPE_NONE);
+	vdev_addr = ar->mac_addr;
+
+	ret = ath10k_wmi_vdev_create(ar, vdev_id, vdev_type, vdev_subtype,
+				     vdev_addr);
+	if (ret) {
+		ath10k_err(ar, "failed to create dummy vdev: %d\n", ret);
+		return ret;
+	}
+
+	ret = ath10k_wmi_vdev_delete(ar, vdev_id);
+	if (ret) {
+		ath10k_err(ar, "failed to delete dummy vdev: %d\n", ret);
+		return ret;
+	}
+
+	/* WMI and HTT may use separate HIF pipes and are not guaranteed to be
+	 * serialized properly implicitly.
+	 *
+	 * Moreover (most) WMI commands have no explicit acknowledges. It is
+	 * possible to infer it implicitly by poking firmware with echo
+	 * command - getting a reply means all preceding comments have been
+	 * (mostly) processed.
+	 *
+	 * In case of vdev create/delete this is sufficient.
+	 *
+	 * Without this it's possible to end up with a race when HTT Rx ring is
+	 * started before vdev create/delete hack is complete allowing a short
+	 * window of opportunity to receive (and Tx ACK) a bunch of frames.
+	 */
+	ret = ath10k_wmi_barrier(ar);
+	if (ret) {
+		ath10k_err(ar, "failed to ping firmware: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
 		      const struct ath10k_fw_components *fw)
 {
@@ -1872,6 +1941,25 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
 		goto err_hif_stop;
 	}
 
+	/* Some firmware revisions do not properly set up hardware rx filter
+	 * registers.
+	 *
+	 * A known example from QCA9880 and 10.2.4 is that MAC_PCU_ADDR1_MASK
+	 * is filled with 0s instead of 1s allowing HW to respond with ACKs to
+	 * any frames that matches MAC_PCU_RX_FILTER which is also
+	 * misconfigured to accept anything.
+	 *
+	 * The ADDR1 is programmed using internal firmware structure field and
+	 * can't be (easily/sanely) reached from the driver explicitly. It is
+	 * possible to implicitly make it correct by creating a dummy vdev and
+	 * then deleting it.
+	 */
+	status = ath10k_core_reset_rx_filter(ar);
+	if (status) {
+		ath10k_err(ar, "failed to reset rx filter: %d\n", status);
+		goto err_hif_stop;
+	}
+
 	/* If firmware indicates Full Rx Reorder support it must be used in a
 	 * slightly different manner. Let HTT code know.
 	 */
@@ -1884,7 +1972,10 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
 		goto err_hif_stop;
 	}
 
-	ar->free_vdev_map = (1LL << ar->max_num_vdevs) - 1;
+	if (ar->max_num_vdevs >= 64)
+		ar->free_vdev_map = 0xFFFFFFFFFFFFFFFFLL;
+	else
+		ar->free_vdev_map = (1LL << ar->max_num_vdevs) - 1;
 
 	INIT_LIST_HEAD(&ar->arvifs);
 
@@ -2031,7 +2122,7 @@ static int ath10k_core_probe_fw(struct ath10k *ar)
 		goto err_free_firmware_files;
 	}
 
-	ret = ath10k_swap_code_seg_init(ar);
+	ret = ath10k_swap_code_seg_init(ar, &ar->normal_mode_fw.fw_file);
 	if (ret) {
 		ath10k_err(ar, "failed to initialize code swap segment: %d\n",
 			   ret);
@@ -2072,6 +2163,9 @@ static void ath10k_core_register_work(struct work_struct *work)
 	struct ath10k *ar = container_of(work, struct ath10k, register_work);
 	int status;
 
+	/* peer stats are enabled by default */
+	set_bit(ATH10K_FLAG_PEER_STATS, &ar->dev_flags);
+
 	status = ath10k_core_probe_fw(ar);
 	if (status) {
 		ath10k_err(ar, "could not probe fw (%d)\n", status);
@@ -2249,6 +2343,8 @@ struct ath10k *ath10k_core_create(size_t priv_size, struct device *dev,
 	INIT_WORK(&ar->register_work, ath10k_core_register_work);
 	INIT_WORK(&ar->restart_work, ath10k_core_restart);
 
+	init_dummy_netdev(&ar->napi_dev);
+
 	ret = ath10k_debug_create(ar);
 	if (ret)
 		goto err_free_aux_wq;
diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h
index 30ae5bf81611..dda49af1eb74 100644
--- a/drivers/net/wireless/ath/ath10k/core.h
+++ b/drivers/net/wireless/ath/ath10k/core.h
@@ -65,6 +65,10 @@
 #define ATH10K_KEEPALIVE_MAX_IDLE 3895
 #define ATH10K_KEEPALIVE_MAX_UNRESPONSIVE 3900
 
+/* NAPI poll budget */
+#define ATH10K_NAPI_BUDGET      64
+#define ATH10K_NAPI_QUOTA_LIMIT 60
+
 struct ath10k;
 
 enum ath10k_bus {
@@ -142,6 +146,7 @@ struct ath10k_wmi {
 	enum ath10k_htc_ep_id eid;
 	struct completion service_ready;
 	struct completion unified_ready;
+	struct completion barrier;
 	wait_queue_head_t tx_credits_wq;
 	DECLARE_BITMAP(svc_map, WMI_SERVICE_MAX);
 	struct wmi_cmd_map *cmd;
@@ -196,10 +201,10 @@ struct ath10k_fw_stats_pdev {
 
 	/* PDEV stats */
 	s32 ch_noise_floor;
-	u32 tx_frame_count;
-	u32 rx_frame_count;
-	u32 rx_clear_count;
-	u32 cycle_count;
+	u32 tx_frame_count; /* Cycles spent transmitting frames */
+	u32 rx_frame_count; /* Cycles spent receiving frames */
+	u32 rx_clear_count; /* Total channel busy time, evidently */
+	u32 cycle_count; /* Total on-channel time */
 	u32 phy_err_count;
 	u32 chan_tx_power;
 	u32 ack_rx_bad;
@@ -440,7 +445,7 @@ struct ath10k_debug {
 	struct completion tpc_complete;
 
 	/* protected by conf_mutex */
-	u32 fw_dbglog_mask;
+	u64 fw_dbglog_mask;
 	u32 fw_dbglog_level;
 	u32 pktlog_filter;
 	u32 reg_addr;
@@ -551,6 +556,13 @@ enum ath10k_fw_features {
 	 */
 	ATH10K_FW_FEATURE_BTCOEX_PARAM = 14,
 
+	/* Older firmware with HTT delivers incorrect tx status for null func
+	 * frames to driver, but this fixed in 10.2 and 10.4 firmware versions.
+	 * Also this workaround results in reporting of incorrect null func
+	 * status for 10.4. This flag is used to skip the workaround.
+	 */
+	ATH10K_FW_FEATURE_SKIP_NULL_FUNC_WAR = 15,
+
 	/* keep last */
 	ATH10K_FW_FEATURE_COUNT,
 };
@@ -663,6 +675,15 @@ struct ath10k_fw_file {
 
 	const void *codeswap_data;
 	size_t codeswap_len;
+
+	/* The original idea of struct ath10k_fw_file was that it only
+	 * contains struct firmware and pointers to various parts (actual
+	 * firmware binary, otp, metadata etc) of the file. This seg_info
+	 * is actually created separate but as this is used similarly as
+	 * the other firmware components it's more convenient to have it
+	 * here.
+	 */
+	struct ath10k_swap_code_seg_info *firmware_swap_code_seg_info;
 };
 
 struct ath10k_fw_components {
@@ -715,53 +736,7 @@ struct ath10k {
 	struct ath10k_htc htc;
 	struct ath10k_htt htt;
 
-	struct ath10k_hw_params {
-		u32 id;
-		u16 dev_id;
-		const char *name;
-		u32 patch_load_addr;
-		int uart_pin;
-		u32 otp_exe_param;
-
-		/* Type of hw cycle counter wraparound logic, for more info
-		 * refer enum ath10k_hw_cc_wraparound_type.
-		 */
-		enum ath10k_hw_cc_wraparound_type cc_wraparound_type;
-
-		/* Some of chip expects fragment descriptor to be continuous
-		 * memory for any TX operation. Set continuous_frag_desc flag
-		 * for the hardware which have such requirement.
-		 */
-		bool continuous_frag_desc;
-
-		/* CCK hardware rate table mapping for the newer chipsets
-		 * like QCA99X0, QCA4019 got revised. The CCK h/w rate values
-		 * are in a proper order with respect to the rate/preamble
-		 */
-		bool cck_rate_map_rev2;
-
-		u32 channel_counters_freq_hz;
-
-		/* Mgmt tx descriptors threshold for limiting probe response
-		 * frames.
-		 */
-		u32 max_probe_resp_desc_thres;
-
-		/* The padding bytes's location is different on various chips */
-		enum ath10k_hw_4addr_pad hw_4addr_pad;
-
-		u32 tx_chain_mask;
-		u32 rx_chain_mask;
-		u32 max_spatial_stream;
-		u32 cal_data_len;
-
-		struct ath10k_hw_params_fw {
-			const char *dir;
-			const char *board;
-			size_t board_size;
-			size_t board_ext_size;
-		} fw;
-	} hw_params;
+	struct ath10k_hw_params hw_params;
 
 	/* contains the firmware images used with ATH10K_FIRMWARE_MODE_NORMAL */
 	struct ath10k_fw_components normal_mode_fw;
@@ -775,10 +750,6 @@ struct ath10k {
 	const struct firmware *cal_file;
 
 	struct {
-		struct ath10k_swap_code_seg_info *firmware_swap_code_seg_info;
-	} swap;
-
-	struct {
 		u32 vendor;
 		u32 device;
 		u32 subsystem_vendor;
@@ -936,6 +907,10 @@ struct ath10k {
 	struct ath10k_thermal thermal;
 	struct ath10k_wow wow;
 
+	/* NAPI */
+	struct net_device napi_dev;
+	struct napi_struct napi;
+
 	/* must be last */
 	u8 drv_priv[0] __aligned(sizeof(void *));
 };
diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c
index 8f0fd41dfd4b..832da6ed9f13 100644
--- a/drivers/net/wireless/ath/ath10k/debug.c
+++ b/drivers/net/wireless/ath/ath10k/debug.c
@@ -1228,9 +1228,9 @@ static ssize_t ath10k_read_fw_dbglog(struct file *file,
 {
 	struct ath10k *ar = file->private_data;
 	unsigned int len;
-	char buf[64];
+	char buf[96];
 
-	len = scnprintf(buf, sizeof(buf), "0x%08x %u\n",
+	len = scnprintf(buf, sizeof(buf), "0x%16llx %u\n",
 			ar->debug.fw_dbglog_mask, ar->debug.fw_dbglog_level);
 
 	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
@@ -1242,15 +1242,16 @@ static ssize_t ath10k_write_fw_dbglog(struct file *file,
 {
 	struct ath10k *ar = file->private_data;
 	int ret;
-	char buf[64];
-	unsigned int log_level, mask;
+	char buf[96];
+	unsigned int log_level;
+	u64 mask;
 
 	simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count);
 
 	/* make sure that buf is null terminated */
 	buf[sizeof(buf) - 1] = 0;
 
-	ret = sscanf(buf, "%x %u", &mask, &log_level);
+	ret = sscanf(buf, "%llx %u", &mask, &log_level);
 
 	if (!ret)
 		return -EINVAL;
diff --git a/drivers/net/wireless/ath/ath10k/htc.c b/drivers/net/wireless/ath/ath10k/htc.c
index 5b3c6bcf9598..175aae38c375 100644
--- a/drivers/net/wireless/ath/ath10k/htc.c
+++ b/drivers/net/wireless/ath/ath10k/htc.c
@@ -44,7 +44,7 @@ static struct sk_buff *ath10k_htc_build_tx_ctrl_skb(void *ar)
 	skb_cb = ATH10K_SKB_CB(skb);
 	memset(skb_cb, 0, sizeof(*skb_cb));
 
-	ath10k_dbg(ar, ATH10K_DBG_HTC, "%s: skb %p\n", __func__, skb);
+	ath10k_dbg(ar, ATH10K_DBG_HTC, "%s: skb %pK\n", __func__, skb);
 	return skb;
 }
 
@@ -62,7 +62,7 @@ static void ath10k_htc_notify_tx_completion(struct ath10k_htc_ep *ep,
 {
 	struct ath10k *ar = ep->htc->ar;
 
-	ath10k_dbg(ar, ATH10K_DBG_HTC, "%s: ep %d skb %p\n", __func__,
+	ath10k_dbg(ar, ATH10K_DBG_HTC, "%s: ep %d skb %pK\n", __func__,
 		   ep->eid, skb);
 
 	ath10k_htc_restore_tx_skb(ep->htc, skb);
@@ -404,7 +404,7 @@ void ath10k_htc_rx_completion_handler(struct ath10k *ar, struct sk_buff *skb)
 		goto out;
 	}
 
-	ath10k_dbg(ar, ATH10K_DBG_HTC, "htc rx completion ep %d skb %p\n",
+	ath10k_dbg(ar, ATH10K_DBG_HTC, "htc rx completion ep %d skb %pK\n",
 		   eid, skb);
 	ep->ep_ops.ep_rx_complete(ar, skb);
 
diff --git a/drivers/net/wireless/ath/ath10k/htt.h b/drivers/net/wireless/ath/ath10k/htt.h
index 430a83e142aa..0d2ed09f202b 100644
--- a/drivers/net/wireless/ath/ath10k/htt.h
+++ b/drivers/net/wireless/ath/ath10k/htt.h
@@ -595,7 +595,7 @@ enum htt_rx_mpdu_status {
 	/* only accept EAPOL frames */
 	HTT_RX_IND_MPDU_STATUS_UNAUTH_PEER,
 	HTT_RX_IND_MPDU_STATUS_OUT_OF_SYNC,
-	/* Non-data in promiscous mode */
+	/* Non-data in promiscuous mode */
 	HTT_RX_IND_MPDU_STATUS_MGMT_CTRL,
 	HTT_RX_IND_MPDU_STATUS_TKIP_MIC_ERR,
 	HTT_RX_IND_MPDU_STATUS_DECRYPT_ERR,
@@ -900,7 +900,7 @@ struct htt_rx_in_ord_ind {
  *     Purpose: indicate how many 32-bit integers follow the message header
  *   - NUM_CHARS
  *     Bits 31:16
- *     Purpose: indicate how many 8-bit charaters follow the series of integers
+ *     Purpose: indicate how many 8-bit characters follow the series of integers
  */
 struct htt_rx_test {
 	u8 num_ints;
@@ -1042,10 +1042,10 @@ struct htt_dbg_stats_wal_tx_stats {
 	/* illegal rate phy errors  */
 	__le32 illgl_rate_phy_err;
 
-	/* wal pdev continous xretry */
+	/* wal pdev continuous xretry */
 	__le32 pdev_cont_xretry;
 
-	/* wal pdev continous xretry */
+	/* wal pdev continuous xretry */
 	__le32 pdev_tx_timeout;
 
 	/* wal pdev resets  */
@@ -1665,7 +1665,6 @@ struct ath10k_htt {
 
 	/* This is used to group tx/rx completions separately and process them
 	 * in batches to reduce cache stalls */
-	struct tasklet_struct txrx_compl_task;
 	struct sk_buff_head rx_compl_q;
 	struct sk_buff_head rx_in_ord_compl_q;
 	struct sk_buff_head tx_fetch_ind_q;
@@ -1798,5 +1797,6 @@ int ath10k_htt_tx(struct ath10k_htt *htt,
 		  struct sk_buff *msdu);
 void ath10k_htt_rx_pktlog_completion_handler(struct ath10k *ar,
 					     struct sk_buff *skb);
+int ath10k_htt_txrx_compl_task(struct ath10k *ar, int budget);
 
 #endif
diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c
index 78db5d679f19..0b4c1562420f 100644
--- a/drivers/net/wireless/ath/ath10k/htt_rx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_rx.c
@@ -34,7 +34,6 @@
 #define HTT_RX_RING_REFILL_RESCHED_MS 5
 
 static int ath10k_htt_rx_get_csum_state(struct sk_buff *skb);
-static void ath10k_htt_txrx_compl_task(unsigned long ptr);
 
 static struct sk_buff *
 ath10k_htt_rx_find_skb_paddr(struct ath10k *ar, u32 paddr)
@@ -226,7 +225,6 @@ int ath10k_htt_rx_ring_refill(struct ath10k *ar)
 void ath10k_htt_rx_free(struct ath10k_htt *htt)
 {
 	del_timer_sync(&htt->rx_ring.refill_retry_timer);
-	tasklet_kill(&htt->txrx_compl_task);
 
 	skb_queue_purge(&htt->rx_compl_q);
 	skb_queue_purge(&htt->rx_in_ord_compl_q);
@@ -520,9 +518,6 @@ int ath10k_htt_rx_alloc(struct ath10k_htt *htt)
 	skb_queue_head_init(&htt->tx_fetch_ind_q);
 	atomic_set(&htt->num_mpdus_ready, 0);
 
-	tasklet_init(&htt->txrx_compl_task, ath10k_htt_txrx_compl_task,
-		     (unsigned long)htt);
-
 	ath10k_dbg(ar, ATH10K_DBG_BOOT, "htt rx ring size %d fill_level %d\n",
 		   htt->rx_ring.size, htt->rx_ring.fill_level);
 	return 0;
@@ -931,7 +926,7 @@ static void ath10k_process_rx(struct ath10k *ar,
 	*status = *rx_status;
 
 	ath10k_dbg(ar, ATH10K_DBG_DATA,
-		   "rx skb %p len %u peer %pM %s %s sn %u %s%s%s%s%s %srate_idx %u vht_nss %u freq %u band %u flag 0x%llx fcs-err %i mic-err %i amsdu-more %i\n",
+		   "rx skb %pK len %u peer %pM %s %s sn %u %s%s%s%s%s %srate_idx %u vht_nss %u freq %u band %u flag 0x%llx fcs-err %i mic-err %i amsdu-more %i\n",
 		   skb,
 		   skb->len,
 		   ieee80211_get_SA(hdr),
@@ -958,7 +953,7 @@ static void ath10k_process_rx(struct ath10k *ar,
 	trace_ath10k_rx_hdr(ar, skb->data, skb->len);
 	trace_ath10k_rx_payload(ar, skb->data, skb->len);
 
-	ieee80211_rx(ar->hw, skb);
+	ieee80211_rx_napi(ar->hw, NULL, skb, &ar->napi);
 }
 
 static int ath10k_htt_rx_nwifi_hdrlen(struct ath10k *ar,
@@ -1056,9 +1051,11 @@ static void ath10k_htt_rx_h_undecap_nwifi(struct ath10k *ar,
 					  const u8 first_hdr[64])
 {
 	struct ieee80211_hdr *hdr;
+	struct htt_rx_desc *rxd;
 	size_t hdr_len;
 	u8 da[ETH_ALEN];
 	u8 sa[ETH_ALEN];
+	int l3_pad_bytes;
 
 	/* Delivered decapped frame:
 	 * [nwifi 802.11 header] <-- replaced with 802.11 hdr
@@ -1072,19 +1069,12 @@ static void ath10k_htt_rx_h_undecap_nwifi(struct ath10k *ar,
 	 */
 
 	/* pull decapped header and copy SA & DA */
-	if ((ar->hw_params.hw_4addr_pad == ATH10K_HW_4ADDR_PAD_BEFORE) &&
-	    ieee80211_has_a4(((struct ieee80211_hdr *)first_hdr)->frame_control)) {
-		/* The QCA99X0 4 address mode pad 2 bytes at the
-		 * beginning of MSDU
-		 */
-		hdr = (struct ieee80211_hdr *)(msdu->data + 2);
-		/* The skb length need be extended 2 as the 2 bytes at the tail
-		 * be excluded due to the padding
-		 */
-		skb_put(msdu, 2);
-	} else {
-		hdr = (struct ieee80211_hdr *)(msdu->data);
-	}
+	rxd = (void *)msdu->data - sizeof(*rxd);
+
+	l3_pad_bytes = ath10k_rx_desc_get_l3_pad_bytes(&ar->hw_params, rxd);
+	skb_put(msdu, l3_pad_bytes);
+
+	hdr = (struct ieee80211_hdr *)(msdu->data + l3_pad_bytes);
 
 	hdr_len = ath10k_htt_rx_nwifi_hdrlen(ar, hdr);
 	ether_addr_copy(da, ieee80211_get_DA(hdr));
@@ -1113,6 +1103,7 @@ static void *ath10k_htt_rx_h_find_rfc1042(struct ath10k *ar,
 	size_t hdr_len, crypto_len;
 	void *rfc1042;
 	bool is_first, is_last, is_amsdu;
+	int bytes_aligned = ar->hw_params.decap_align_bytes;
 
 	rxd = (void *)msdu->data - sizeof(*rxd);
 	hdr = (void *)rxd->rx_hdr_status;
@@ -1129,8 +1120,8 @@ static void *ath10k_htt_rx_h_find_rfc1042(struct ath10k *ar,
 		hdr_len = ieee80211_hdrlen(hdr->frame_control);
 		crypto_len = ath10k_htt_rx_crypto_param_len(ar, enctype);
 
-		rfc1042 += round_up(hdr_len, 4) +
-			   round_up(crypto_len, 4);
+		rfc1042 += round_up(hdr_len, bytes_aligned) +
+			   round_up(crypto_len, bytes_aligned);
 	}
 
 	if (is_amsdu)
@@ -1151,6 +1142,8 @@ static void ath10k_htt_rx_h_undecap_eth(struct ath10k *ar,
 	void *rfc1042;
 	u8 da[ETH_ALEN];
 	u8 sa[ETH_ALEN];
+	int l3_pad_bytes;
+	struct htt_rx_desc *rxd;
 
 	/* Delivered decapped frame:
 	 * [eth header] <-- replaced with 802.11 hdr & rfc1042/llc
@@ -1161,6 +1154,11 @@ static void ath10k_htt_rx_h_undecap_eth(struct ath10k *ar,
 	if (WARN_ON_ONCE(!rfc1042))
 		return;
 
+	rxd = (void *)msdu->data - sizeof(*rxd);
+	l3_pad_bytes = ath10k_rx_desc_get_l3_pad_bytes(&ar->hw_params, rxd);
+	skb_put(msdu, l3_pad_bytes);
+	skb_pull(msdu, l3_pad_bytes);
+
 	/* pull decapped header and copy SA & DA */
 	eth = (struct ethhdr *)msdu->data;
 	ether_addr_copy(da, eth->h_dest);
@@ -1191,6 +1189,8 @@ static void ath10k_htt_rx_h_undecap_snap(struct ath10k *ar,
 {
 	struct ieee80211_hdr *hdr;
 	size_t hdr_len;
+	int l3_pad_bytes;
+	struct htt_rx_desc *rxd;
 
 	/* Delivered decapped frame:
 	 * [amsdu header] <-- replaced with 802.11 hdr
@@ -1198,7 +1198,11 @@ static void ath10k_htt_rx_h_undecap_snap(struct ath10k *ar,
 	 * [payload]
 	 */
 
-	skb_pull(msdu, sizeof(struct amsdu_subframe_hdr));
+	rxd = (void *)msdu->data - sizeof(*rxd);
+	l3_pad_bytes = ath10k_rx_desc_get_l3_pad_bytes(&ar->hw_params, rxd);
+
+	skb_put(msdu, l3_pad_bytes);
+	skb_pull(msdu, sizeof(struct amsdu_subframe_hdr) + l3_pad_bytes);
 
 	hdr = (struct ieee80211_hdr *)first_hdr;
 	hdr_len = ieee80211_hdrlen(hdr->frame_control);
@@ -1525,9 +1529,9 @@ static void ath10k_htt_rx_h_filter(struct ath10k *ar,
 static int ath10k_htt_rx_handle_amsdu(struct ath10k_htt *htt)
 {
 	struct ath10k *ar = htt->ar;
-	static struct ieee80211_rx_status rx_status;
+	struct ieee80211_rx_status *rx_status = &htt->rx_status;
 	struct sk_buff_head amsdu;
-	int ret;
+	int ret, num_msdus;
 
 	__skb_queue_head_init(&amsdu);
 
@@ -1549,13 +1553,14 @@ static int ath10k_htt_rx_handle_amsdu(struct ath10k_htt *htt)
 		return ret;
 	}
 
-	ath10k_htt_rx_h_ppdu(ar, &amsdu, &rx_status, 0xffff);
+	num_msdus = skb_queue_len(&amsdu);
+	ath10k_htt_rx_h_ppdu(ar, &amsdu, rx_status, 0xffff);
 	ath10k_htt_rx_h_unchain(ar, &amsdu, ret > 0);
-	ath10k_htt_rx_h_filter(ar, &amsdu, &rx_status);
-	ath10k_htt_rx_h_mpdu(ar, &amsdu, &rx_status);
-	ath10k_htt_rx_h_deliver(ar, &amsdu, &rx_status);
+	ath10k_htt_rx_h_filter(ar, &amsdu, rx_status);
+	ath10k_htt_rx_h_mpdu(ar, &amsdu, rx_status);
+	ath10k_htt_rx_h_deliver(ar, &amsdu, rx_status);
 
-	return 0;
+	return num_msdus;
 }
 
 static void ath10k_htt_rx_proc_rx_ind(struct ath10k_htt *htt,
@@ -1579,15 +1584,6 @@ static void ath10k_htt_rx_proc_rx_ind(struct ath10k_htt *htt,
 		mpdu_count += mpdu_ranges[i].mpdu_count;
 
 	atomic_add(mpdu_count, &htt->num_mpdus_ready);
-
-	tasklet_schedule(&htt->txrx_compl_task);
-}
-
-static void ath10k_htt_rx_frag_handler(struct ath10k_htt *htt)
-{
-	atomic_inc(&htt->num_mpdus_ready);
-
-	tasklet_schedule(&htt->txrx_compl_task);
 }
 
 static void ath10k_htt_rx_tx_compl_ind(struct ath10k *ar,
@@ -1772,14 +1768,15 @@ static void ath10k_htt_rx_h_rx_offload_prot(struct ieee80211_rx_status *status,
 			RX_FLAG_MMIC_STRIPPED;
 }
 
-static void ath10k_htt_rx_h_rx_offload(struct ath10k *ar,
-				       struct sk_buff_head *list)
+static int ath10k_htt_rx_h_rx_offload(struct ath10k *ar,
+				      struct sk_buff_head *list)
 {
 	struct ath10k_htt *htt = &ar->htt;
 	struct ieee80211_rx_status *status = &htt->rx_status;
 	struct htt_rx_offload_msdu *rx;
 	struct sk_buff *msdu;
 	size_t offset;
+	int num_msdu = 0;
 
 	while ((msdu = __skb_dequeue(list))) {
 		/* Offloaded frames don't have Rx descriptor. Instead they have
@@ -1819,10 +1816,12 @@ static void ath10k_htt_rx_h_rx_offload(struct ath10k *ar,
 		ath10k_htt_rx_h_rx_offload_prot(status, msdu);
 		ath10k_htt_rx_h_channel(ar, status, NULL, rx->vdev_id);
 		ath10k_process_rx(ar, status, msdu);
+		num_msdu++;
 	}
+	return num_msdu;
 }
 
-static void ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
+static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 {
 	struct ath10k_htt *htt = &ar->htt;
 	struct htt_resp *resp = (void *)skb->data;
@@ -1835,12 +1834,12 @@ static void ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 	u8 tid;
 	bool offload;
 	bool frag;
-	int ret;
+	int ret, num_msdus = 0;
 
 	lockdep_assert_held(&htt->rx_ring.lock);
 
 	if (htt->rx_confused)
-		return;
+		return -EIO;
 
 	skb_pull(skb, sizeof(resp->hdr));
 	skb_pull(skb, sizeof(resp->rx_in_ord_ind));
@@ -1859,7 +1858,7 @@ static void ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 
 	if (skb->len < msdu_count * sizeof(*resp->rx_in_ord_ind.msdu_descs)) {
 		ath10k_warn(ar, "dropping invalid in order rx indication\n");
-		return;
+		return -EINVAL;
 	}
 
 	/* The event can deliver more than 1 A-MSDU. Each A-MSDU is later
@@ -1870,14 +1869,14 @@ static void ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 	if (ret < 0) {
 		ath10k_warn(ar, "failed to pop paddr list: %d\n", ret);
 		htt->rx_confused = true;
-		return;
+		return -EIO;
 	}
 
 	/* Offloaded frames are very different and need to be handled
 	 * separately.
 	 */
 	if (offload)
-		ath10k_htt_rx_h_rx_offload(ar, &list);
+		num_msdus = ath10k_htt_rx_h_rx_offload(ar, &list);
 
 	while (!skb_queue_empty(&list)) {
 		__skb_queue_head_init(&amsdu);
@@ -1890,6 +1889,7 @@ static void ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 			 * better to report something than nothing though. This
 			 * should still give an idea about rx rate to the user.
 			 */
+			num_msdus += skb_queue_len(&amsdu);
 			ath10k_htt_rx_h_ppdu(ar, &amsdu, status, vdev_id);
 			ath10k_htt_rx_h_filter(ar, &amsdu, status);
 			ath10k_htt_rx_h_mpdu(ar, &amsdu, status);
@@ -1902,9 +1902,10 @@ static void ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 			ath10k_warn(ar, "failed to extract amsdu: %d\n", ret);
 			htt->rx_confused = true;
 			__skb_queue_purge(&list);
-			return;
+			return -EIO;
 		}
 	}
+	return num_msdus;
 }
 
 static void ath10k_htt_rx_tx_fetch_resp_id_confirm(struct ath10k *ar,
@@ -2267,7 +2268,6 @@ bool ath10k_htt_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb)
 	}
 	case HTT_T2H_MSG_TYPE_TX_COMPL_IND:
 		ath10k_htt_rx_tx_compl_ind(htt->ar, skb);
-		tasklet_schedule(&htt->txrx_compl_task);
 		break;
 	case HTT_T2H_MSG_TYPE_SEC_IND: {
 		struct ath10k *ar = htt->ar;
@@ -2284,7 +2284,7 @@ bool ath10k_htt_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb)
 	case HTT_T2H_MSG_TYPE_RX_FRAG_IND: {
 		ath10k_dbg_dump(ar, ATH10K_DBG_HTT_DUMP, NULL, "htt event: ",
 				skb->data, skb->len);
-		ath10k_htt_rx_frag_handler(htt);
+		atomic_inc(&htt->num_mpdus_ready);
 		break;
 	}
 	case HTT_T2H_MSG_TYPE_TEST:
@@ -2320,8 +2320,7 @@ bool ath10k_htt_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb)
 		break;
 	}
 	case HTT_T2H_MSG_TYPE_RX_IN_ORD_PADDR_IND: {
-		skb_queue_tail(&htt->rx_in_ord_compl_q, skb);
-		tasklet_schedule(&htt->txrx_compl_task);
+		__skb_queue_tail(&htt->rx_in_ord_compl_q, skb);
 		return false;
 	}
 	case HTT_T2H_MSG_TYPE_TX_CREDIT_UPDATE_IND:
@@ -2347,7 +2346,6 @@ bool ath10k_htt_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb)
 			break;
 		}
 		skb_queue_tail(&htt->tx_fetch_ind_q, tx_fetch_ind);
-		tasklet_schedule(&htt->txrx_compl_task);
 		break;
 	}
 	case HTT_T2H_MSG_TYPE_TX_FETCH_CONFIRM:
@@ -2376,27 +2374,77 @@ void ath10k_htt_rx_pktlog_completion_handler(struct ath10k *ar,
 }
 EXPORT_SYMBOL(ath10k_htt_rx_pktlog_completion_handler);
 
-static void ath10k_htt_txrx_compl_task(unsigned long ptr)
+int ath10k_htt_txrx_compl_task(struct ath10k *ar, int budget)
 {
-	struct ath10k_htt *htt = (struct ath10k_htt *)ptr;
-	struct ath10k *ar = htt->ar;
+	struct ath10k_htt *htt = &ar->htt;
 	struct htt_tx_done tx_done = {};
-	struct sk_buff_head rx_ind_q;
 	struct sk_buff_head tx_ind_q;
 	struct sk_buff *skb;
 	unsigned long flags;
-	int num_mpdus;
+	int quota = 0, done, num_rx_msdus;
+	bool resched_napi = false;
 
-	__skb_queue_head_init(&rx_ind_q);
 	__skb_queue_head_init(&tx_ind_q);
 
-	spin_lock_irqsave(&htt->rx_in_ord_compl_q.lock, flags);
-	skb_queue_splice_init(&htt->rx_in_ord_compl_q, &rx_ind_q);
-	spin_unlock_irqrestore(&htt->rx_in_ord_compl_q.lock, flags);
+	/* Since in-ord-ind can deliver more than 1 A-MSDU in single event,
+	 * process it first to utilize full available quota.
+	 */
+	while (quota < budget) {
+		if (skb_queue_empty(&htt->rx_in_ord_compl_q))
+			break;
 
-	spin_lock_irqsave(&htt->tx_fetch_ind_q.lock, flags);
-	skb_queue_splice_init(&htt->tx_fetch_ind_q, &tx_ind_q);
-	spin_unlock_irqrestore(&htt->tx_fetch_ind_q.lock, flags);
+		skb = __skb_dequeue(&htt->rx_in_ord_compl_q);
+		if (!skb) {
+			resched_napi = true;
+			goto exit;
+		}
+
+		spin_lock_bh(&htt->rx_ring.lock);
+		num_rx_msdus = ath10k_htt_rx_in_ord_ind(ar, skb);
+		spin_unlock_bh(&htt->rx_ring.lock);
+		if (num_rx_msdus < 0) {
+			resched_napi = true;
+			goto exit;
+		}
+
+		dev_kfree_skb_any(skb);
+		if (num_rx_msdus > 0)
+			quota += num_rx_msdus;
+
+		if ((quota > ATH10K_NAPI_QUOTA_LIMIT) &&
+		    !skb_queue_empty(&htt->rx_in_ord_compl_q)) {
+			resched_napi = true;
+			goto exit;
+		}
+	}
+
+	while (quota < budget) {
+		/* no more data to receive */
+		if (!atomic_read(&htt->num_mpdus_ready))
+			break;
+
+		num_rx_msdus = ath10k_htt_rx_handle_amsdu(htt);
+		if (num_rx_msdus < 0) {
+			resched_napi = true;
+			goto exit;
+		}
+
+		quota += num_rx_msdus;
+		atomic_dec(&htt->num_mpdus_ready);
+		if ((quota > ATH10K_NAPI_QUOTA_LIMIT) &&
+		    atomic_read(&htt->num_mpdus_ready)) {
+			resched_napi = true;
+			goto exit;
+		}
+	}
+
+	/* From NAPI documentation:
+	 *  The napi poll() function may also process TX completions, in which
+	 *  case if it processes the entire TX ring then it should count that
+	 *  work as the rest of the budget.
+	 */
+	if ((quota < budget) && !kfifo_is_empty(&htt->txdone_fifo))
+		quota = budget;
 
 	/* kfifo_get: called only within txrx_tasklet so it's neatly serialized.
 	 * From kfifo_get() documentation:
@@ -2406,27 +2454,24 @@ static void ath10k_htt_txrx_compl_task(unsigned long ptr)
 	while (kfifo_get(&htt->txdone_fifo, &tx_done))
 		ath10k_txrx_tx_unref(htt, &tx_done);
 
-	while ((skb = __skb_dequeue(&tx_ind_q))) {
-		ath10k_htt_rx_tx_fetch_ind(ar, skb);
-		dev_kfree_skb_any(skb);
-	}
-
-	num_mpdus = atomic_read(&htt->num_mpdus_ready);
-
-	while (num_mpdus) {
-		if (ath10k_htt_rx_handle_amsdu(htt))
-			break;
+	ath10k_mac_tx_push_pending(ar);
 
-		num_mpdus--;
-		atomic_dec(&htt->num_mpdus_ready);
-	}
+	spin_lock_irqsave(&htt->tx_fetch_ind_q.lock, flags);
+	skb_queue_splice_init(&htt->tx_fetch_ind_q, &tx_ind_q);
+	spin_unlock_irqrestore(&htt->tx_fetch_ind_q.lock, flags);
 
-	while ((skb = __skb_dequeue(&rx_ind_q))) {
-		spin_lock_bh(&htt->rx_ring.lock);
-		ath10k_htt_rx_in_ord_ind(ar, skb);
-		spin_unlock_bh(&htt->rx_ring.lock);
+	while ((skb = __skb_dequeue(&tx_ind_q))) {
+		ath10k_htt_rx_tx_fetch_ind(ar, skb);
 		dev_kfree_skb_any(skb);
 	}
 
+exit:
 	ath10k_htt_rx_msdu_buff_replenish(htt);
+	/* In case of rx failure or more data to read, report budget
+	 * to reschedule NAPI poll
+	 */
+	done = resched_napi ? budget : quota;
+
+	return done;
 }
+EXPORT_SYMBOL(ath10k_htt_txrx_compl_task);
diff --git a/drivers/net/wireless/ath/ath10k/htt_tx.c b/drivers/net/wireless/ath/ath10k/htt_tx.c
index 7c072b605bc7..ae5b33fe5ba8 100644
--- a/drivers/net/wireless/ath/ath10k/htt_tx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_tx.c
@@ -390,8 +390,6 @@ void ath10k_htt_tx_free(struct ath10k_htt *htt)
 {
 	int size;
 
-	tasklet_kill(&htt->txrx_compl_task);
-
 	idr_for_each(&htt->pending_tx, ath10k_htt_tx_clean_up_pending, htt->ar);
 	idr_destroy(&htt->pending_tx);
 
diff --git a/drivers/net/wireless/ath/ath10k/hw.c b/drivers/net/wireless/ath/ath10k/hw.c
index f903d468dbe6..675e75d66db2 100644
--- a/drivers/net/wireless/ath/ath10k/hw.c
+++ b/drivers/net/wireless/ath/ath10k/hw.c
@@ -85,7 +85,7 @@ const struct ath10k_hw_regs qca99x0_regs = {
 	.ce7_base_address			= 0x0004bc00,
 	/* Note: qca99x0 supports upto 12 Copy Engines. Other than address of
 	 * CE0 and CE1 no other copy engine is directly referred in the code.
-	 * It is not really neccessary to assign address for newly supported
+	 * It is not really necessary to assign address for newly supported
 	 * CEs in this address table.
 	 *	Copy Engine		Address
 	 *	CE8			0x0004c000
@@ -219,3 +219,16 @@ void ath10k_hw_fill_survey_time(struct ath10k *ar, struct survey_info *survey,
 	survey->time = CCNT_TO_MSEC(ar, cc);
 	survey->time_busy = CCNT_TO_MSEC(ar, rcc);
 }
+
+const struct ath10k_hw_ops qca988x_ops = {
+};
+
+static int ath10k_qca99x0_rx_desc_get_l3_pad_bytes(struct htt_rx_desc *rxd)
+{
+	return MS(__le32_to_cpu(rxd->msdu_end.qca99x0.info1),
+		  RX_MSDU_END_INFO1_L3_HDR_PAD);
+}
+
+const struct ath10k_hw_ops qca99x0_ops = {
+	.rx_desc_get_l3_pad_bytes = ath10k_qca99x0_rx_desc_get_l3_pad_bytes,
+};
diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h
index e014cd732a0d..6038b7486f1d 100644
--- a/drivers/net/wireless/ath/ath10k/hw.h
+++ b/drivers/net/wireless/ath/ath10k/hw.h
@@ -284,7 +284,7 @@ void ath10k_hw_fill_survey_time(struct ath10k *ar, struct survey_info *survey,
 #define QCA_REV_9377(ar) ((ar)->hw_rev == ATH10K_HW_QCA9377)
 #define QCA_REV_40XX(ar) ((ar)->hw_rev == ATH10K_HW_QCA4019)
 
-/* Known pecularities:
+/* Known peculiarities:
  *  - raw appears in nwifi decap, raw and nwifi appear in ethernet decap
  *  - raw have FCS, nwifi doesn't
  *  - ethernet frames have 802.11 header decapped and parts (base hdr, cipher
@@ -338,11 +338,6 @@ enum ath10k_hw_rate_rev2_cck {
 	ATH10K_HW_RATE_REV2_CCK_SP_11M,
 };
 
-enum ath10k_hw_4addr_pad {
-	ATH10K_HW_4ADDR_PAD_AFTER,
-	ATH10K_HW_4ADDR_PAD_BEFORE,
-};
-
 enum ath10k_hw_cc_wraparound_type {
 	ATH10K_HW_CC_WRAP_DISABLED = 0,
 
@@ -363,6 +358,80 @@ enum ath10k_hw_cc_wraparound_type {
 	ATH10K_HW_CC_WRAP_SHIFTED_EACH = 2,
 };
 
+struct ath10k_hw_params {
+	u32 id;
+	u16 dev_id;
+	const char *name;
+	u32 patch_load_addr;
+	int uart_pin;
+	u32 otp_exe_param;
+
+	/* Type of hw cycle counter wraparound logic, for more info
+	 * refer enum ath10k_hw_cc_wraparound_type.
+	 */
+	enum ath10k_hw_cc_wraparound_type cc_wraparound_type;
+
+	/* Some of chip expects fragment descriptor to be continuous
+	 * memory for any TX operation. Set continuous_frag_desc flag
+	 * for the hardware which have such requirement.
+	 */
+	bool continuous_frag_desc;
+
+	/* CCK hardware rate table mapping for the newer chipsets
+	 * like QCA99X0, QCA4019 got revised. The CCK h/w rate values
+	 * are in a proper order with respect to the rate/preamble
+	 */
+	bool cck_rate_map_rev2;
+
+	u32 channel_counters_freq_hz;
+
+	/* Mgmt tx descriptors threshold for limiting probe response
+	 * frames.
+	 */
+	u32 max_probe_resp_desc_thres;
+
+	u32 tx_chain_mask;
+	u32 rx_chain_mask;
+	u32 max_spatial_stream;
+	u32 cal_data_len;
+
+	struct ath10k_hw_params_fw {
+		const char *dir;
+		const char *board;
+		size_t board_size;
+		size_t board_ext_size;
+	} fw;
+
+	/* qca99x0 family chips deliver broadcast/multicast management
+	 * frames encrypted and expect software do decryption.
+	 */
+	bool sw_decrypt_mcast_mgmt;
+
+	const struct ath10k_hw_ops *hw_ops;
+
+	/* Number of bytes used for alignment in rx_hdr_status of rx desc. */
+	int decap_align_bytes;
+};
+
+struct htt_rx_desc;
+
+/* Defines needed for Rx descriptor abstraction */
+struct ath10k_hw_ops {
+	int (*rx_desc_get_l3_pad_bytes)(struct htt_rx_desc *rxd);
+};
+
+extern const struct ath10k_hw_ops qca988x_ops;
+extern const struct ath10k_hw_ops qca99x0_ops;
+
+static inline int
+ath10k_rx_desc_get_l3_pad_bytes(struct ath10k_hw_params *hw,
+				struct htt_rx_desc *rxd)
+{
+	if (hw->hw_ops->rx_desc_get_l3_pad_bytes)
+		return hw->hw_ops->rx_desc_get_l3_pad_bytes(rxd);
+	return 0;
+}
+
 /* Target specific defines for MAIN firmware */
 #define TARGET_NUM_VDEVS			8
 #define TARGET_NUM_PEER_AST			2
diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c
index 0bbd0a00edcc..76297d69f1ed 100644
--- a/drivers/net/wireless/ath/ath10k/mac.c
+++ b/drivers/net/wireless/ath/ath10k/mac.c
@@ -824,7 +824,7 @@ static void ath10k_peer_cleanup(struct ath10k *ar, u32 vdev_id)
 		 */
 		for (i = 0; i < ARRAY_SIZE(ar->peer_map); i++) {
 			if (ar->peer_map[i] == peer) {
-				ath10k_warn(ar, "removing stale peer_map entry for %pM (ptr %p idx %d)\n",
+				ath10k_warn(ar, "removing stale peer_map entry for %pM (ptr %pK idx %d)\n",
 					    peer->addr, peer, i);
 				ar->peer_map[i] = NULL;
 			}
@@ -2793,7 +2793,7 @@ static void ath10k_bss_disassoc(struct ieee80211_hw *hw,
 
 	ret = ath10k_wmi_vdev_down(ar, arvif->vdev_id);
 	if (ret)
-		ath10k_warn(ar, "faield to down vdev %i: %d\n",
+		ath10k_warn(ar, "failed to down vdev %i: %d\n",
 			    arvif->vdev_id, ret);
 
 	arvif->def_wep_key_idx = -1;
@@ -3255,6 +3255,8 @@ ath10k_mac_tx_h_get_txmode(struct ath10k *ar,
 	if (ar->htt.target_version_major < 3 &&
 	    (ieee80211_is_nullfunc(fc) || ieee80211_is_qos_nullfunc(fc)) &&
 	    !test_bit(ATH10K_FW_FEATURE_HAS_WMI_MGMT_TX,
+		      ar->running_fw->fw_file.fw_features) &&
+	    !test_bit(ATH10K_FW_FEATURE_SKIP_NULL_FUNC_WAR,
 		      ar->running_fw->fw_file.fw_features))
 		return ATH10K_HW_TXRX_MGMT;
 
@@ -3524,7 +3526,7 @@ static int ath10k_mac_tx(struct ath10k *ar,
 
 	if (info->flags & IEEE80211_TX_CTL_TX_OFFCHAN) {
 		if (!ath10k_mac_tx_frm_has_freq(ar)) {
-			ath10k_dbg(ar, ATH10K_DBG_MAC, "queued offchannel skb %p\n",
+			ath10k_dbg(ar, ATH10K_DBG_MAC, "queued offchannel skb %pK\n",
 				   skb);
 
 			skb_queue_tail(&ar->offchan_tx_queue, skb);
@@ -3586,7 +3588,7 @@ void ath10k_offchan_tx_work(struct work_struct *work)
 
 		mutex_lock(&ar->conf_mutex);
 
-		ath10k_dbg(ar, ATH10K_DBG_MAC, "mac offchannel skb %p\n",
+		ath10k_dbg(ar, ATH10K_DBG_MAC, "mac offchannel skb %pK\n",
 			   skb);
 
 		hdr = (struct ieee80211_hdr *)skb->data;
@@ -3643,7 +3645,7 @@ void ath10k_offchan_tx_work(struct work_struct *work)
 		time_left =
 		wait_for_completion_timeout(&ar->offchan_tx_completed, 3 * HZ);
 		if (time_left == 0)
-			ath10k_warn(ar, "timed out waiting for offchannel skb %p\n",
+			ath10k_warn(ar, "timed out waiting for offchannel skb %pK\n",
 				    skb);
 
 		if (!peer && tmp_peer_created) {
@@ -3777,7 +3779,9 @@ int ath10k_mac_tx_push_txq(struct ieee80211_hw *hw,
 	enum ath10k_hw_txrx_mode txmode;
 	enum ath10k_mac_tx_path txpath;
 	struct sk_buff *skb;
+	struct ieee80211_hdr *hdr;
 	size_t skb_len;
+	bool is_mgmt, is_presp;
 	int ret;
 
 	spin_lock_bh(&ar->htt.tx_lock);
@@ -3801,6 +3805,22 @@ int ath10k_mac_tx_push_txq(struct ieee80211_hw *hw,
 	skb_len = skb->len;
 	txmode = ath10k_mac_tx_h_get_txmode(ar, vif, sta, skb);
 	txpath = ath10k_mac_tx_h_get_txpath(ar, skb, txmode);
+	is_mgmt = (txpath == ATH10K_MAC_TX_HTT_MGMT);
+
+	if (is_mgmt) {
+		hdr = (struct ieee80211_hdr *)skb->data;
+		is_presp = ieee80211_is_probe_resp(hdr->frame_control);
+
+		spin_lock_bh(&ar->htt.tx_lock);
+		ret = ath10k_htt_tx_mgmt_inc_pending(htt, is_mgmt, is_presp);
+
+		if (ret) {
+			ath10k_htt_tx_dec_pending(htt);
+			spin_unlock_bh(&ar->htt.tx_lock);
+			return ret;
+		}
+		spin_unlock_bh(&ar->htt.tx_lock);
+	}
 
 	ret = ath10k_mac_tx(ar, vif, sta, txmode, txpath, skb);
 	if (unlikely(ret)) {
@@ -3808,6 +3828,8 @@ int ath10k_mac_tx_push_txq(struct ieee80211_hw *hw,
 
 		spin_lock_bh(&ar->htt.tx_lock);
 		ath10k_htt_tx_dec_pending(htt);
+		if (is_mgmt)
+			ath10k_htt_tx_mgmt_dec_pending(htt);
 		spin_unlock_bh(&ar->htt.tx_lock);
 
 		return ret;
@@ -3894,7 +3916,7 @@ void __ath10k_scan_finish(struct ath10k *ar)
 		ar->scan.roc_freq = 0;
 		ath10k_offchan_tx_purge(ar);
 		cancel_delayed_work(&ar->scan.timeout);
-		complete_all(&ar->scan.completed);
+		complete(&ar->scan.completed);
 		break;
 	}
 }
@@ -4100,13 +4122,29 @@ static void ath10k_mac_op_wake_tx_queue(struct ieee80211_hw *hw,
 {
 	struct ath10k *ar = hw->priv;
 	struct ath10k_txq *artxq = (void *)txq->drv_priv;
+	struct ieee80211_txq *f_txq;
+	struct ath10k_txq *f_artxq;
+	int ret = 0;
+	int max = 16;
 
 	spin_lock_bh(&ar->txqs_lock);
 	if (list_empty(&artxq->list))
 		list_add_tail(&artxq->list, &ar->txqs);
+
+	f_artxq = list_first_entry(&ar->txqs, struct ath10k_txq, list);
+	f_txq = container_of((void *)f_artxq, struct ieee80211_txq, drv_priv);
+	list_del_init(&f_artxq->list);
+
+	while (ath10k_mac_tx_can_push(hw, f_txq) && max--) {
+		ret = ath10k_mac_tx_push_txq(hw, f_txq);
+		if (ret)
+			break;
+	}
+	if (ret != -ENOENT)
+		list_add_tail(&f_artxq->list, &ar->txqs);
 	spin_unlock_bh(&ar->txqs_lock);
 
-	ath10k_mac_tx_push_pending(ar);
+	ath10k_htt_tx_txq_update(hw, f_txq);
 	ath10k_htt_tx_txq_update(hw, txq);
 }
 
@@ -5186,7 +5224,7 @@ static void ath10k_configure_filter(struct ieee80211_hw *hw,
 
 	ret = ath10k_monitor_recalc(ar);
 	if (ret)
-		ath10k_warn(ar, "failed to recalc montior: %d\n", ret);
+		ath10k_warn(ar, "failed to recalc monitor: %d\n", ret);
 
 	mutex_unlock(&ar->conf_mutex);
 }
@@ -5984,8 +6022,8 @@ static int ath10k_sta_state(struct ieee80211_hw *hw,
 		 * Existing station deletion.
 		 */
 		ath10k_dbg(ar, ATH10K_DBG_MAC,
-			   "mac vdev %d peer delete %pM (sta gone)\n",
-			   arvif->vdev_id, sta->addr);
+			   "mac vdev %d peer delete %pM sta %pK (sta gone)\n",
+			   arvif->vdev_id, sta->addr, sta);
 
 		ret = ath10k_peer_delete(ar, arvif->vdev_id, sta->addr);
 		if (ret)
@@ -6001,7 +6039,7 @@ static int ath10k_sta_state(struct ieee80211_hw *hw,
 				continue;
 
 			if (peer->sta == sta) {
-				ath10k_warn(ar, "found sta peer %pM (ptr %p id %d) entry on vdev %i after it was supposedly removed\n",
+				ath10k_warn(ar, "found sta peer %pM (ptr %pK id %d) entry on vdev %i after it was supposedly removed\n",
 					    sta->addr, peer, i, arvif->vdev_id);
 				peer->sta = NULL;
 
@@ -6538,7 +6576,7 @@ static int ath10k_get_survey(struct ieee80211_hw *hw, int idx,
 		goto exit;
 	}
 
-	ath10k_mac_update_bss_chan_survey(ar, survey->channel);
+	ath10k_mac_update_bss_chan_survey(ar, &sband->channels[idx]);
 
 	spin_lock_bh(&ar->data_lock);
 	memcpy(survey, ar_survey, sizeof(*survey));
@@ -7134,7 +7172,7 @@ ath10k_mac_op_add_chanctx(struct ieee80211_hw *hw,
 	struct ath10k *ar = hw->priv;
 
 	ath10k_dbg(ar, ATH10K_DBG_MAC,
-		   "mac chanctx add freq %hu width %d ptr %p\n",
+		   "mac chanctx add freq %hu width %d ptr %pK\n",
 		   ctx->def.chan->center_freq, ctx->def.width, ctx);
 
 	mutex_lock(&ar->conf_mutex);
@@ -7158,7 +7196,7 @@ ath10k_mac_op_remove_chanctx(struct ieee80211_hw *hw,
 	struct ath10k *ar = hw->priv;
 
 	ath10k_dbg(ar, ATH10K_DBG_MAC,
-		   "mac chanctx remove freq %hu width %d ptr %p\n",
+		   "mac chanctx remove freq %hu width %d ptr %pK\n",
 		   ctx->def.chan->center_freq, ctx->def.width, ctx);
 
 	mutex_lock(&ar->conf_mutex);
@@ -7223,7 +7261,7 @@ ath10k_mac_op_change_chanctx(struct ieee80211_hw *hw,
 	mutex_lock(&ar->conf_mutex);
 
 	ath10k_dbg(ar, ATH10K_DBG_MAC,
-		   "mac chanctx change freq %hu width %d ptr %p changed %x\n",
+		   "mac chanctx change freq %hu width %d ptr %pK changed %x\n",
 		   ctx->def.chan->center_freq, ctx->def.width, ctx, changed);
 
 	/* This shouldn't really happen because channel switching should use
@@ -7281,7 +7319,7 @@ ath10k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw,
 	mutex_lock(&ar->conf_mutex);
 
 	ath10k_dbg(ar, ATH10K_DBG_MAC,
-		   "mac chanctx assign ptr %p vdev_id %i\n",
+		   "mac chanctx assign ptr %pK vdev_id %i\n",
 		   ctx, arvif->vdev_id);
 
 	if (WARN_ON(arvif->is_started)) {
@@ -7342,7 +7380,7 @@ ath10k_mac_op_unassign_vif_chanctx(struct ieee80211_hw *hw,
 	mutex_lock(&ar->conf_mutex);
 
 	ath10k_dbg(ar, ATH10K_DBG_MAC,
-		   "mac chanctx unassign ptr %p vdev_id %i\n",
+		   "mac chanctx unassign ptr %pK vdev_id %i\n",
 		   ctx, arvif->vdev_id);
 
 	WARN_ON(!arvif->is_started);
diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c
index 9a22c478dd1b..0457e315d336 100644
--- a/drivers/net/wireless/ath/ath10k/pci.c
+++ b/drivers/net/wireless/ath/ath10k/pci.c
@@ -1506,12 +1506,10 @@ void ath10k_pci_hif_send_complete_check(struct ath10k *ar, u8 pipe,
 	ath10k_ce_per_engine_service(ar, pipe);
 }
 
-void ath10k_pci_kill_tasklet(struct ath10k *ar)
+static void ath10k_pci_rx_retry_sync(struct ath10k *ar)
 {
 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
 
-	tasklet_kill(&ar_pci->intr_tq);
-
 	del_timer_sync(&ar_pci->rx_post_retry);
 }
 
@@ -1570,7 +1568,7 @@ void ath10k_pci_hif_get_default_pipe(struct ath10k *ar,
 						 ul_pipe, dl_pipe);
 }
 
-static void ath10k_pci_irq_msi_fw_mask(struct ath10k *ar)
+void ath10k_pci_irq_msi_fw_mask(struct ath10k *ar)
 {
 	u32 val;
 
@@ -1693,14 +1691,12 @@ static void ath10k_pci_rx_pipe_cleanup(struct ath10k_pci_pipe *pci_pipe)
 static void ath10k_pci_tx_pipe_cleanup(struct ath10k_pci_pipe *pci_pipe)
 {
 	struct ath10k *ar;
-	struct ath10k_pci *ar_pci;
 	struct ath10k_ce_pipe *ce_pipe;
 	struct ath10k_ce_ring *ce_ring;
 	struct sk_buff *skb;
 	int i;
 
 	ar = pci_pipe->hif_ce_state;
-	ar_pci = ath10k_pci_priv(ar);
 	ce_pipe = pci_pipe->ce_hdl;
 	ce_ring = ce_pipe->src_ring;
 
@@ -1753,7 +1749,7 @@ void ath10k_pci_ce_deinit(struct ath10k *ar)
 
 void ath10k_pci_flush(struct ath10k *ar)
 {
-	ath10k_pci_kill_tasklet(ar);
+	ath10k_pci_rx_retry_sync(ar);
 	ath10k_pci_buffer_cleanup(ar);
 }
 
@@ -1780,6 +1776,8 @@ static void ath10k_pci_hif_stop(struct ath10k *ar)
 	ath10k_pci_irq_disable(ar);
 	ath10k_pci_irq_sync(ar);
 	ath10k_pci_flush(ar);
+	napi_synchronize(&ar->napi);
+	napi_disable(&ar->napi);
 
 	spin_lock_irqsave(&ar_pci->ps_lock, flags);
 	WARN_ON(ar_pci->ps_wake_refcount > 0);
@@ -2533,6 +2531,7 @@ static int ath10k_pci_hif_power_up(struct ath10k *ar)
 		ath10k_err(ar, "could not wake up target CPU: %d\n", ret);
 		goto err_ce;
 	}
+	napi_enable(&ar->napi);
 
 	return 0;
 
@@ -2725,7 +2724,7 @@ static int ath10k_pci_hif_fetch_cal_eeprom(struct ath10k *ar, void **data,
 	return 0;
 
 err_free:
-	kfree(data);
+	kfree(caldata);
 
 	return -EINVAL;
 }
@@ -2772,35 +2771,53 @@ static irqreturn_t ath10k_pci_interrupt_handler(int irq, void *arg)
 		return IRQ_NONE;
 	}
 
-	if (ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_LEGACY) {
-		if (!ath10k_pci_irq_pending(ar))
-			return IRQ_NONE;
-
-		ath10k_pci_disable_and_clear_legacy_irq(ar);
-	}
+	if ((ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_LEGACY) &&
+	    !ath10k_pci_irq_pending(ar))
+		return IRQ_NONE;
 
-	tasklet_schedule(&ar_pci->intr_tq);
+	ath10k_pci_disable_and_clear_legacy_irq(ar);
+	ath10k_pci_irq_msi_fw_mask(ar);
+	napi_schedule(&ar->napi);
 
 	return IRQ_HANDLED;
 }
 
-static void ath10k_pci_tasklet(unsigned long data)
+static int ath10k_pci_napi_poll(struct napi_struct *ctx, int budget)
 {
-	struct ath10k *ar = (struct ath10k *)data;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k *ar = container_of(ctx, struct ath10k, napi);
+	int done = 0;
 
 	if (ath10k_pci_has_fw_crashed(ar)) {
-		ath10k_pci_irq_disable(ar);
 		ath10k_pci_fw_crashed_clear(ar);
 		ath10k_pci_fw_crashed_dump(ar);
-		return;
+		napi_complete(ctx);
+		return done;
 	}
 
 	ath10k_ce_per_engine_service_any(ar);
 
-	/* Re-enable legacy irq that was disabled in the irq handler */
-	if (ar_pci->oper_irq_mode == ATH10K_PCI_IRQ_LEGACY)
+	done = ath10k_htt_txrx_compl_task(ar, budget);
+
+	if (done < budget) {
+		napi_complete(ctx);
+		/* In case of MSI, it is possible that interrupts are received
+		 * while NAPI poll is inprogress. So pending interrupts that are
+		 * received after processing all copy engine pipes by NAPI poll
+		 * will not be handled again. This is causing failure to
+		 * complete boot sequence in x86 platform. So before enabling
+		 * interrupts safer to check for pending interrupts for
+		 * immediate servicing.
+		 */
+		if (CE_INTERRUPT_SUMMARY(ar)) {
+			napi_reschedule(ctx);
+			goto out;
+		}
 		ath10k_pci_enable_legacy_irq(ar);
+		ath10k_pci_irq_msi_fw_unmask(ar);
+	}
+
+out:
+	return done;
 }
 
 static int ath10k_pci_request_irq_msi(struct ath10k *ar)
@@ -2858,11 +2875,10 @@ static void ath10k_pci_free_irq(struct ath10k *ar)
 	free_irq(ar_pci->pdev->irq, ar);
 }
 
-void ath10k_pci_init_irq_tasklets(struct ath10k *ar)
+void ath10k_pci_init_napi(struct ath10k *ar)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
-
-	tasklet_init(&ar_pci->intr_tq, ath10k_pci_tasklet, (unsigned long)ar);
+	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_pci_napi_poll,
+		       ATH10K_NAPI_BUDGET);
 }
 
 static int ath10k_pci_init_irq(struct ath10k *ar)
@@ -2870,7 +2886,7 @@ static int ath10k_pci_init_irq(struct ath10k *ar)
 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
 	int ret;
 
-	ath10k_pci_init_irq_tasklets(ar);
+	ath10k_pci_init_napi(ar);
 
 	if (ath10k_pci_irq_mode != ATH10K_PCI_IRQ_AUTO)
 		ath10k_info(ar, "limiting irq mode to: %d\n",
@@ -3062,7 +3078,7 @@ static int ath10k_pci_claim(struct ath10k *ar)
 		goto err_master;
 	}
 
-	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot pci_mem 0x%p\n", ar_pci->mem);
+	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot pci_mem 0x%pK\n", ar_pci->mem);
 	return 0;
 
 err_master:
@@ -3131,7 +3147,8 @@ int ath10k_pci_setup_resource(struct ath10k *ar)
 
 void ath10k_pci_release_resource(struct ath10k *ar)
 {
-	ath10k_pci_kill_tasklet(ar);
+	ath10k_pci_rx_retry_sync(ar);
+	netif_napi_del(&ar->napi);
 	ath10k_pci_ce_deinit(ar);
 	ath10k_pci_free_pipes(ar);
 }
@@ -3162,7 +3179,6 @@ static int ath10k_pci_probe(struct pci_dev *pdev,
 		pci_hard_reset = ath10k_pci_qca988x_chip_reset;
 		break;
 	case QCA9887_1_0_DEVICE_ID:
-		dev_warn(&pdev->dev, "QCA9887 support is still experimental, there are likely bugs. You have been warned.\n");
 		hw_rev = ATH10K_HW_QCA9887;
 		pci_ps = false;
 		pci_soft_reset = ath10k_pci_warm_reset;
@@ -3298,7 +3314,7 @@ static int ath10k_pci_probe(struct pci_dev *pdev,
 
 err_free_irq:
 	ath10k_pci_free_irq(ar);
-	ath10k_pci_kill_tasklet(ar);
+	ath10k_pci_rx_retry_sync(ar);
 
 err_deinit_irq:
 	ath10k_pci_deinit_irq(ar);
diff --git a/drivers/net/wireless/ath/ath10k/pci.h b/drivers/net/wireless/ath/ath10k/pci.h
index 6eca1df2ce60..9854ad56b2de 100644
--- a/drivers/net/wireless/ath/ath10k/pci.h
+++ b/drivers/net/wireless/ath/ath10k/pci.h
@@ -177,8 +177,6 @@ struct ath10k_pci {
 	/* Operating interrupt mode */
 	enum ath10k_pci_irq_mode oper_irq_mode;
 
-	struct tasklet_struct intr_tq;
-
 	struct ath10k_pci_pipe pipe_info[CE_COUNT_MAX];
 
 	/* Copy Engine used for Diagnostic Accesses */
@@ -294,8 +292,7 @@ void ath10k_pci_free_pipes(struct ath10k *ar);
 void ath10k_pci_free_pipes(struct ath10k *ar);
 void ath10k_pci_rx_replenish_retry(unsigned long ptr);
 void ath10k_pci_ce_deinit(struct ath10k *ar);
-void ath10k_pci_init_irq_tasklets(struct ath10k *ar);
-void ath10k_pci_kill_tasklet(struct ath10k *ar);
+void ath10k_pci_init_napi(struct ath10k *ar);
 int ath10k_pci_init_pipes(struct ath10k *ar);
 int ath10k_pci_init_config(struct ath10k *ar);
 void ath10k_pci_rx_post(struct ath10k *ar);
@@ -303,6 +300,7 @@ void ath10k_pci_flush(struct ath10k *ar);
 void ath10k_pci_enable_legacy_irq(struct ath10k *ar);
 bool ath10k_pci_irq_pending(struct ath10k *ar);
 void ath10k_pci_disable_and_clear_legacy_irq(struct ath10k *ar);
+void ath10k_pci_irq_msi_fw_mask(struct ath10k *ar);
 int ath10k_pci_wait_for_target_init(struct ath10k *ar);
 int ath10k_pci_setup_resource(struct ath10k *ar);
 void ath10k_pci_release_resource(struct ath10k *ar);
diff --git a/drivers/net/wireless/ath/ath10k/swap.c b/drivers/net/wireless/ath/ath10k/swap.c
index 0c5f5863dac8..adf4592374b4 100644
--- a/drivers/net/wireless/ath/ath10k/swap.c
+++ b/drivers/net/wireless/ath/ath10k/swap.c
@@ -134,17 +134,18 @@ ath10k_swap_code_seg_alloc(struct ath10k *ar, size_t swap_bin_len)
 	return seg_info;
 }
 
-int ath10k_swap_code_seg_configure(struct ath10k *ar)
+int ath10k_swap_code_seg_configure(struct ath10k *ar,
+				   const struct ath10k_fw_file *fw_file)
 {
 	int ret;
 	struct ath10k_swap_code_seg_info *seg_info = NULL;
 
-	if (!ar->swap.firmware_swap_code_seg_info)
+	if (!fw_file->firmware_swap_code_seg_info)
 		return 0;
 
 	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot found firmware code swap binary\n");
 
-	seg_info = ar->swap.firmware_swap_code_seg_info;
+	seg_info = fw_file->firmware_swap_code_seg_info;
 
 	ret = ath10k_bmi_write_memory(ar, seg_info->target_addr,
 				      &seg_info->seg_hw_info,
@@ -158,28 +159,29 @@ int ath10k_swap_code_seg_configure(struct ath10k *ar)
 	return 0;
 }
 
-void ath10k_swap_code_seg_release(struct ath10k *ar)
+void ath10k_swap_code_seg_release(struct ath10k *ar,
+				  struct ath10k_fw_file *fw_file)
 {
-	ath10k_swap_code_seg_free(ar, ar->swap.firmware_swap_code_seg_info);
+	ath10k_swap_code_seg_free(ar, fw_file->firmware_swap_code_seg_info);
 
 	/* FIXME: these two assignments look to bein wrong place! Shouldn't
 	 * they be in ath10k_core_free_firmware_files() like the rest?
 	 */
-	ar->normal_mode_fw.fw_file.codeswap_data = NULL;
-	ar->normal_mode_fw.fw_file.codeswap_len = 0;
+	fw_file->codeswap_data = NULL;
+	fw_file->codeswap_len = 0;
 
-	ar->swap.firmware_swap_code_seg_info = NULL;
+	fw_file->firmware_swap_code_seg_info = NULL;
 }
 
-int ath10k_swap_code_seg_init(struct ath10k *ar)
+int ath10k_swap_code_seg_init(struct ath10k *ar, struct ath10k_fw_file *fw_file)
 {
 	int ret;
 	struct ath10k_swap_code_seg_info *seg_info;
 	const void *codeswap_data;
 	size_t codeswap_len;
 
-	codeswap_data = ar->normal_mode_fw.fw_file.codeswap_data;
-	codeswap_len = ar->normal_mode_fw.fw_file.codeswap_len;
+	codeswap_data = fw_file->codeswap_data;
+	codeswap_len = fw_file->codeswap_len;
 
 	if (!codeswap_len || !codeswap_data)
 		return 0;
@@ -200,7 +202,7 @@ int ath10k_swap_code_seg_init(struct ath10k *ar)
 		return ret;
 	}
 
-	ar->swap.firmware_swap_code_seg_info = seg_info;
+	fw_file->firmware_swap_code_seg_info = seg_info;
 
 	return 0;
 }
diff --git a/drivers/net/wireless/ath/ath10k/swap.h b/drivers/net/wireless/ath/ath10k/swap.h
index 36991c7b07a0..f5dc0476493e 100644
--- a/drivers/net/wireless/ath/ath10k/swap.h
+++ b/drivers/net/wireless/ath/ath10k/swap.h
@@ -23,6 +23,8 @@
 /* Currently only one swap segment is supported */
 #define ATH10K_SWAP_CODE_SEG_NUM_SUPPORTED	1
 
+struct ath10k_fw_file;
+
 struct ath10k_swap_code_seg_tlv {
 	__le32 address;
 	__le32 length;
@@ -58,8 +60,11 @@ struct ath10k_swap_code_seg_info {
 	dma_addr_t paddr[ATH10K_SWAP_CODE_SEG_NUM_SUPPORTED];
 };
 
-int ath10k_swap_code_seg_configure(struct ath10k *ar);
-void ath10k_swap_code_seg_release(struct ath10k *ar);
-int ath10k_swap_code_seg_init(struct ath10k *ar);
+int ath10k_swap_code_seg_configure(struct ath10k *ar,
+				   const struct ath10k_fw_file *fw_file);
+void ath10k_swap_code_seg_release(struct ath10k *ar,
+				  struct ath10k_fw_file *fw_file);
+int ath10k_swap_code_seg_init(struct ath10k *ar,
+			      struct ath10k_fw_file *fw_file);
 
 #endif
diff --git a/drivers/net/wireless/ath/ath10k/targaddrs.h b/drivers/net/wireless/ath/ath10k/targaddrs.h
index aaf53a81e78b..a47cab44d9c8 100644
--- a/drivers/net/wireless/ath/ath10k/targaddrs.h
+++ b/drivers/net/wireless/ath/ath10k/targaddrs.h
@@ -405,7 +405,7 @@ Fw Mode/SubMode Mask
  * 1. target firmware would check magic number and if it's a match, firmware
  *    would consider the bits[0:15] are valid and base on that to calculate
  *    the end of DRAM. Early allocation would be located at that area and
- *    may be reclaimed when necesary
+ *    may be reclaimed when necessary
  * 2. if no magic number is found, early allocation would happen at "_end"
  *    symbol of ROM which is located before the app-data and might NOT be
  *    re-claimable. If this is adopted, link script should keep this in
diff --git a/drivers/net/wireless/ath/ath10k/testmode.c b/drivers/net/wireless/ath/ath10k/testmode.c
index 120f4234d3b0..ed85f938e3c0 100644
--- a/drivers/net/wireless/ath/ath10k/testmode.c
+++ b/drivers/net/wireless/ath/ath10k/testmode.c
@@ -23,6 +23,7 @@
 #include "wmi.h"
 #include "hif.h"
 #include "hw.h"
+#include "core.h"
 
 #include "testmode_i.h"
 
@@ -45,7 +46,7 @@ bool ath10k_tm_event_wmi(struct ath10k *ar, u32 cmd_id, struct sk_buff *skb)
 	int ret;
 
 	ath10k_dbg(ar, ATH10K_DBG_TESTMODE,
-		   "testmode event wmi cmd_id %d skb %p skb->len %d\n",
+		   "testmode event wmi cmd_id %d skb %pK skb->len %d\n",
 		   cmd_id, skb, skb->len);
 
 	ath10k_dbg_dump(ar, ATH10K_DBG_TESTMODE, NULL, "", skb->data, skb->len);
@@ -240,6 +241,18 @@ static int ath10k_tm_cmd_utf_start(struct ath10k *ar, struct nlattr *tb[])
 		goto err;
 	}
 
+	if (ar->testmode.utf_mode_fw.fw_file.codeswap_data &&
+	    ar->testmode.utf_mode_fw.fw_file.codeswap_len) {
+		ret = ath10k_swap_code_seg_init(ar,
+						&ar->testmode.utf_mode_fw.fw_file);
+		if (ret) {
+			ath10k_warn(ar,
+				    "failed to init utf code swap segment: %d\n",
+				    ret);
+			goto err_release_utf_mode_fw;
+		}
+	}
+
 	spin_lock_bh(&ar->data_lock);
 	ar->testmode.utf_monitor = true;
 	spin_unlock_bh(&ar->data_lock);
@@ -279,6 +292,11 @@ err_power_down:
 	ath10k_hif_power_down(ar);
 
 err_release_utf_mode_fw:
+	if (ar->testmode.utf_mode_fw.fw_file.codeswap_data &&
+	    ar->testmode.utf_mode_fw.fw_file.codeswap_len)
+		ath10k_swap_code_seg_release(ar,
+					     &ar->testmode.utf_mode_fw.fw_file);
+
 	release_firmware(ar->testmode.utf_mode_fw.fw_file.firmware);
 	ar->testmode.utf_mode_fw.fw_file.firmware = NULL;
 
@@ -301,6 +319,11 @@ static void __ath10k_tm_cmd_utf_stop(struct ath10k *ar)
 
 	spin_unlock_bh(&ar->data_lock);
 
+	if (ar->testmode.utf_mode_fw.fw_file.codeswap_data &&
+	    ar->testmode.utf_mode_fw.fw_file.codeswap_len)
+		ath10k_swap_code_seg_release(ar,
+					     &ar->testmode.utf_mode_fw.fw_file);
+
 	release_firmware(ar->testmode.utf_mode_fw.fw_file.firmware);
 	ar->testmode.utf_mode_fw.fw_file.firmware = NULL;
 
@@ -360,7 +383,7 @@ static int ath10k_tm_cmd_wmi(struct ath10k *ar, struct nlattr *tb[])
 	cmd_id = nla_get_u32(tb[ATH10K_TM_ATTR_WMI_CMDID]);
 
 	ath10k_dbg(ar, ATH10K_DBG_TESTMODE,
-		   "testmode cmd wmi cmd_id %d buf %p buf_len %d\n",
+		   "testmode cmd wmi cmd_id %d buf %pK buf_len %d\n",
 		   cmd_id, buf, buf_len);
 
 	ath10k_dbg_dump(ar, ATH10K_DBG_TESTMODE, NULL, "", buf, buf_len);
diff --git a/drivers/net/wireless/ath/ath10k/thermal.c b/drivers/net/wireless/ath/ath10k/thermal.c
index 444b52c7e4f3..0a47269be289 100644
--- a/drivers/net/wireless/ath/ath10k/thermal.c
+++ b/drivers/net/wireless/ath/ath10k/thermal.c
@@ -192,7 +192,7 @@ int ath10k_thermal_register(struct ath10k *ar)
 
 	/* Avoid linking error on devm_hwmon_device_register_with_groups, I
 	 * guess linux/hwmon.h is missing proper stubs. */
-	if (!config_enabled(CONFIG_HWMON))
+	if (!IS_REACHABLE(CONFIG_HWMON))
 		return 0;
 
 	hwmon_dev = devm_hwmon_device_register_with_groups(ar->dev,
diff --git a/drivers/net/wireless/ath/ath10k/txrx.c b/drivers/net/wireless/ath/ath10k/txrx.c
index b29a86a26c13..9852c5d51139 100644
--- a/drivers/net/wireless/ath/ath10k/txrx.c
+++ b/drivers/net/wireless/ath/ath10k/txrx.c
@@ -44,7 +44,7 @@ static void ath10k_report_offchan_tx(struct ath10k *ar, struct sk_buff *skb)
 	complete(&ar->offchan_tx_completed);
 	ar->offchan_tx_skb = NULL; /* just for sanity */
 
-	ath10k_dbg(ar, ATH10K_DBG_HTT, "completed offchannel skb %p\n", skb);
+	ath10k_dbg(ar, ATH10K_DBG_HTT, "completed offchannel skb %pK\n", skb);
 out:
 	spin_unlock_bh(&ar->data_lock);
 }
@@ -119,8 +119,6 @@ int ath10k_txrx_tx_unref(struct ath10k_htt *htt,
 	ieee80211_tx_status(htt->ar->hw, msdu);
 	/* we do not own the msdu anymore */
 
-	ath10k_mac_tx_push_pending(ar);
-
 	return 0;
 }
 
diff --git a/drivers/net/wireless/ath/ath10k/wmi-ops.h b/drivers/net/wireless/ath/ath10k/wmi-ops.h
index 64ebd304f907..c9a8bb1186f2 100644
--- a/drivers/net/wireless/ath/ath10k/wmi-ops.h
+++ b/drivers/net/wireless/ath/ath10k/wmi-ops.h
@@ -51,6 +51,8 @@ struct wmi_ops {
 			    struct wmi_roam_ev_arg *arg);
 	int (*pull_wow_event)(struct ath10k *ar, struct sk_buff *skb,
 			      struct wmi_wow_ev_arg *arg);
+	int (*pull_echo_ev)(struct ath10k *ar, struct sk_buff *skb,
+			    struct wmi_echo_ev_arg *arg);
 	enum wmi_txbf_conf (*get_txbf_conf_scheme)(struct ath10k *ar);
 
 	struct sk_buff *(*gen_pdev_suspend)(struct ath10k *ar, u32 suspend_opt);
@@ -123,7 +125,7 @@ struct wmi_ops {
 					     enum wmi_force_fw_hang_type type,
 					     u32 delay_ms);
 	struct sk_buff *(*gen_mgmt_tx)(struct ath10k *ar, struct sk_buff *skb);
-	struct sk_buff *(*gen_dbglog_cfg)(struct ath10k *ar, u32 module_enable,
+	struct sk_buff *(*gen_dbglog_cfg)(struct ath10k *ar, u64 module_enable,
 					  u32 log_level);
 	struct sk_buff *(*gen_pktlog_enable)(struct ath10k *ar, u32 filter);
 	struct sk_buff *(*gen_pktlog_disable)(struct ath10k *ar);
@@ -194,6 +196,7 @@ struct wmi_ops {
 	struct sk_buff *(*gen_pdev_bss_chan_info_req)
 					(struct ath10k *ar,
 					 enum wmi_bss_survey_req_type type);
+	struct sk_buff *(*gen_echo)(struct ath10k *ar, u32 value);
 };
 
 int ath10k_wmi_cmd_send(struct ath10k *ar, struct sk_buff *skb, u32 cmd_id);
@@ -349,6 +352,16 @@ ath10k_wmi_pull_wow_event(struct ath10k *ar, struct sk_buff *skb,
 	return ar->wmi.ops->pull_wow_event(ar, skb, arg);
 }
 
+static inline int
+ath10k_wmi_pull_echo_ev(struct ath10k *ar, struct sk_buff *skb,
+			struct wmi_echo_ev_arg *arg)
+{
+	if (!ar->wmi.ops->pull_echo_ev)
+		return -EOPNOTSUPP;
+
+	return ar->wmi.ops->pull_echo_ev(ar, skb, arg);
+}
+
 static inline enum wmi_txbf_conf
 ath10k_wmi_get_txbf_conf_scheme(struct ath10k *ar)
 {
@@ -932,7 +945,7 @@ ath10k_wmi_force_fw_hang(struct ath10k *ar,
 }
 
 static inline int
-ath10k_wmi_dbglog_cfg(struct ath10k *ar, u32 module_enable, u32 log_level)
+ath10k_wmi_dbglog_cfg(struct ath10k *ar, u64 module_enable, u32 log_level)
 {
 	struct sk_buff *skb;
 
@@ -1382,4 +1395,20 @@ ath10k_wmi_pdev_bss_chan_info_request(struct ath10k *ar,
 				   wmi->cmd->pdev_bss_chan_info_request_cmdid);
 }
 
+static inline int
+ath10k_wmi_echo(struct ath10k *ar, u32 value)
+{
+	struct ath10k_wmi *wmi = &ar->wmi;
+	struct sk_buff *skb;
+
+	if (!wmi->ops->gen_echo)
+		return -EOPNOTSUPP;
+
+	skb = wmi->ops->gen_echo(ar, value);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	return ath10k_wmi_cmd_send(ar, skb, wmi->cmd->echo_cmdid);
+}
+
 #endif
diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.c b/drivers/net/wireless/ath/ath10k/wmi-tlv.c
index e09337ee7c96..e64f59300a7c 100644
--- a/drivers/net/wireless/ath/ath10k/wmi-tlv.c
+++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.c
@@ -1223,6 +1223,33 @@ ath10k_wmi_tlv_op_pull_wow_ev(struct ath10k *ar, struct sk_buff *skb,
 	return 0;
 }
 
+static int ath10k_wmi_tlv_op_pull_echo_ev(struct ath10k *ar,
+					  struct sk_buff *skb,
+					  struct wmi_echo_ev_arg *arg)
+{
+	const void **tb;
+	const struct wmi_echo_event *ev;
+	int ret;
+
+	tb = ath10k_wmi_tlv_parse_alloc(ar, skb->data, skb->len, GFP_ATOMIC);
+	if (IS_ERR(tb)) {
+		ret = PTR_ERR(tb);
+		ath10k_warn(ar, "failed to parse tlv: %d\n", ret);
+		return ret;
+	}
+
+	ev = tb[WMI_TLV_TAG_STRUCT_ECHO_EVENT];
+	if (!ev) {
+		kfree(tb);
+		return -EPROTO;
+	}
+
+	arg->value = ev->value;
+
+	kfree(tb);
+	return 0;
+}
+
 static struct sk_buff *
 ath10k_wmi_tlv_op_gen_pdev_suspend(struct ath10k *ar, u32 opt)
 {
@@ -2441,7 +2468,7 @@ ath10k_wmi_tlv_op_gen_force_fw_hang(struct ath10k *ar,
 }
 
 static struct sk_buff *
-ath10k_wmi_tlv_op_gen_dbglog_cfg(struct ath10k *ar, u32 module_enable,
+ath10k_wmi_tlv_op_gen_dbglog_cfg(struct ath10k *ar, u64 module_enable,
 				 u32 log_level) {
 	struct wmi_tlv_dbglog_cmd *cmd;
 	struct wmi_tlv *tlv;
@@ -3081,6 +3108,34 @@ ath10k_wmi_tlv_op_gen_adaptive_qcs(struct ath10k *ar, bool enable)
 	return skb;
 }
 
+static struct sk_buff *
+ath10k_wmi_tlv_op_gen_echo(struct ath10k *ar, u32 value)
+{
+	struct wmi_echo_cmd *cmd;
+	struct wmi_tlv *tlv;
+	struct sk_buff *skb;
+	void *ptr;
+	size_t len;
+
+	len = sizeof(*tlv) + sizeof(*cmd);
+	skb = ath10k_wmi_alloc_skb(ar, len);
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	ptr = (void *)skb->data;
+	tlv = ptr;
+	tlv->tag = __cpu_to_le16(WMI_TLV_TAG_STRUCT_ECHO_CMD);
+	tlv->len = __cpu_to_le16(sizeof(*cmd));
+	cmd = (void *)tlv->value;
+	cmd->value = cpu_to_le32(value);
+
+	ptr += sizeof(*tlv);
+	ptr += sizeof(*cmd);
+
+	ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tlv echo value 0x%08x\n", value);
+	return skb;
+}
+
 /****************/
 /* TLV mappings */
 /****************/
@@ -3429,6 +3484,7 @@ static const struct wmi_ops wmi_tlv_ops = {
 	.pull_fw_stats = ath10k_wmi_tlv_op_pull_fw_stats,
 	.pull_roam_ev = ath10k_wmi_tlv_op_pull_roam_ev,
 	.pull_wow_event = ath10k_wmi_tlv_op_pull_wow_ev,
+	.pull_echo_ev = ath10k_wmi_tlv_op_pull_echo_ev,
 	.get_txbf_conf_scheme = ath10k_wmi_tlv_txbf_conf_scheme,
 
 	.gen_pdev_suspend = ath10k_wmi_tlv_op_gen_pdev_suspend,
@@ -3485,6 +3541,7 @@ static const struct wmi_ops wmi_tlv_ops = {
 	.gen_adaptive_qcs = ath10k_wmi_tlv_op_gen_adaptive_qcs,
 	.fw_stats_fill = ath10k_wmi_main_op_fw_stats_fill,
 	.get_vdev_subtype = ath10k_wmi_op_get_vdev_subtype,
+	.gen_echo = ath10k_wmi_tlv_op_gen_echo,
 };
 
 static const struct wmi_peer_flags_map wmi_tlv_peer_flags_map = {
diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c
index d2462886b75c..54df425bb0fc 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.c
+++ b/drivers/net/wireless/ath/ath10k/wmi.c
@@ -29,6 +29,9 @@
 #include "p2p.h"
 #include "hw.h"
 
+#define ATH10K_WMI_BARRIER_ECHO_ID 0xBA991E9
+#define ATH10K_WMI_BARRIER_TIMEOUT_HZ (3 * HZ)
+
 /* MAIN WMI cmd track */
 static struct wmi_cmd_map wmi_cmd_map = {
 	.init_cmdid = WMI_INIT_CMDID,
@@ -1874,7 +1877,7 @@ ath10k_wmi_op_gen_mgmt_tx(struct ath10k *ar, struct sk_buff *msdu)
 	ether_addr_copy(cmd->hdr.peer_macaddr.addr, ieee80211_get_DA(hdr));
 	memcpy(cmd->buf, msdu->data, msdu->len);
 
-	ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mgmt tx skb %p len %d ftype %02x stype %02x\n",
+	ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mgmt tx skb %pK len %d ftype %02x stype %02x\n",
 		   msdu, skb->len, fc & IEEE80211_FCTL_FTYPE,
 		   fc & IEEE80211_FCTL_STYPE);
 	trace_ath10k_tx_hdr(ar, skb->data, skb->len);
@@ -2240,6 +2243,29 @@ static int ath10k_wmi_10_4_op_pull_mgmt_rx_ev(struct ath10k *ar,
 	return 0;
 }
 
+static bool ath10k_wmi_rx_is_decrypted(struct ath10k *ar,
+				       struct ieee80211_hdr *hdr)
+{
+	if (!ieee80211_has_protected(hdr->frame_control))
+		return false;
+
+	/* FW delivers WEP Shared Auth frame with Protected Bit set and
+	 * encrypted payload. However in case of PMF it delivers decrypted
+	 * frames with Protected Bit set.
+	 */
+	if (ieee80211_is_auth(hdr->frame_control))
+		return false;
+
+	/* qca99x0 based FW delivers broadcast or multicast management frames
+	 * (ex: group privacy action frames in mesh) as encrypted payload.
+	 */
+	if (is_multicast_ether_addr(ieee80211_get_DA(hdr)) &&
+	    ar->hw_params.sw_decrypt_mcast_mgmt)
+		return false;
+
+	return true;
+}
+
 int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb)
 {
 	struct wmi_mgmt_rx_ev_arg arg = {};
@@ -2326,11 +2352,7 @@ int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb)
 
 	ath10k_wmi_handle_wep_reauth(ar, skb, status);
 
-	/* FW delivers WEP Shared Auth frame with Protected Bit set and
-	 * encrypted payload. However in case of PMF it delivers decrypted
-	 * frames with Protected Bit set. */
-	if (ieee80211_has_protected(hdr->frame_control) &&
-	    !ieee80211_is_auth(hdr->frame_control)) {
+	if (ath10k_wmi_rx_is_decrypted(ar, hdr)) {
 		status->flag |= RX_FLAG_DECRYPTED;
 
 		if (!ieee80211_is_action(hdr->frame_control) &&
@@ -2347,7 +2369,7 @@ int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb)
 		ath10k_mac_handle_beacon(ar, skb);
 
 	ath10k_dbg(ar, ATH10K_DBG_MGMT,
-		   "event mgmt rx skb %p len %d ftype %02x stype %02x\n",
+		   "event mgmt rx skb %pK len %d ftype %02x stype %02x\n",
 		   skb, skb->len,
 		   fc & IEEE80211_FCTL_FTYPE, fc & IEEE80211_FCTL_STYPE);
 
@@ -2495,7 +2517,21 @@ exit:
 
 void ath10k_wmi_event_echo(struct ath10k *ar, struct sk_buff *skb)
 {
-	ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_ECHO_EVENTID\n");
+	struct wmi_echo_ev_arg arg = {};
+	int ret;
+
+	ret = ath10k_wmi_pull_echo_ev(ar, skb, &arg);
+	if (ret) {
+		ath10k_warn(ar, "failed to parse echo: %d\n", ret);
+		return;
+	}
+
+	ath10k_dbg(ar, ATH10K_DBG_WMI,
+		   "wmi event echo value 0x%08x\n",
+		   le32_to_cpu(arg.value));
+
+	if (le32_to_cpu(arg.value) == ATH10K_WMI_BARRIER_ECHO_ID)
+		complete(&ar->wmi.barrier);
 }
 
 int ath10k_wmi_event_debug_mesg(struct ath10k *ar, struct sk_buff *skb)
@@ -3478,6 +3514,12 @@ void ath10k_wmi_event_host_swba(struct ath10k *ar, struct sk_buff *skb)
 			continue;
 		}
 
+		/* mac80211 would have already asked us to stop beaconing and
+		 * bring the vdev down, so continue in that case
+		 */
+		if (!arvif->is_up)
+			continue;
+
 		/* There are no completions for beacons so wait for next SWBA
 		 * before telling mac80211 to decrement CSA counter
 		 *
@@ -3527,7 +3569,6 @@ void ath10k_wmi_event_host_swba(struct ath10k *ar, struct sk_buff *skb)
 				ath10k_warn(ar, "failed to map beacon: %d\n",
 					    ret);
 				dev_kfree_skb_any(bcn);
-				ret = -EIO;
 				goto skip;
 			}
 
@@ -4792,6 +4833,17 @@ static int ath10k_wmi_op_pull_roam_ev(struct ath10k *ar, struct sk_buff *skb,
 	return 0;
 }
 
+static int ath10k_wmi_op_pull_echo_ev(struct ath10k *ar,
+				      struct sk_buff *skb,
+				      struct wmi_echo_ev_arg *arg)
+{
+	struct wmi_echo_event *ev = (void *)skb->data;
+
+	arg->value = ev->value;
+
+	return 0;
+}
+
 int ath10k_wmi_event_ready(struct ath10k *ar, struct sk_buff *skb)
 {
 	struct wmi_rdy_ev_arg arg = {};
@@ -5124,6 +5176,7 @@ static void ath10k_wmi_10_2_op_rx(struct ath10k *ar, struct sk_buff *skb)
 {
 	struct wmi_cmd_hdr *cmd_hdr;
 	enum wmi_10_2_event_id id;
+	bool consumed;
 
 	cmd_hdr = (struct wmi_cmd_hdr *)skb->data;
 	id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID);
@@ -5133,6 +5186,18 @@ static void ath10k_wmi_10_2_op_rx(struct ath10k *ar, struct sk_buff *skb)
 
 	trace_ath10k_wmi_event(ar, id, skb->data, skb->len);
 
+	consumed = ath10k_tm_event_wmi(ar, id, skb);
+
+	/* Ready event must be handled normally also in UTF mode so that we
+	 * know the UTF firmware has booted, others we are just bypass WMI
+	 * events to testmode.
+	 */
+	if (consumed && id != WMI_10_2_READY_EVENTID) {
+		ath10k_dbg(ar, ATH10K_DBG_WMI,
+			   "wmi testmode consumed 0x%x\n", id);
+		goto out;
+	}
+
 	switch (id) {
 	case WMI_10_2_MGMT_RX_EVENTID:
 		ath10k_wmi_event_mgmt_rx(ar, skb);
@@ -5248,6 +5313,7 @@ static void ath10k_wmi_10_4_op_rx(struct ath10k *ar, struct sk_buff *skb)
 {
 	struct wmi_cmd_hdr *cmd_hdr;
 	enum wmi_10_4_event_id id;
+	bool consumed;
 
 	cmd_hdr = (struct wmi_cmd_hdr *)skb->data;
 	id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID);
@@ -5257,6 +5323,18 @@ static void ath10k_wmi_10_4_op_rx(struct ath10k *ar, struct sk_buff *skb)
 
 	trace_ath10k_wmi_event(ar, id, skb->data, skb->len);
 
+	consumed = ath10k_tm_event_wmi(ar, id, skb);
+
+	/* Ready event must be handled normally also in UTF mode so that we
+	 * know the UTF firmware has booted, others we are just bypass WMI
+	 * events to testmode.
+	 */
+	if (consumed && id != WMI_10_4_READY_EVENTID) {
+		ath10k_dbg(ar, ATH10K_DBG_WMI,
+			   "wmi testmode consumed 0x%x\n", id);
+		goto out;
+	}
+
 	switch (id) {
 	case WMI_10_4_MGMT_RX_EVENTID:
 		ath10k_wmi_event_mgmt_rx(ar, skb);
@@ -5306,6 +5384,7 @@ static void ath10k_wmi_10_4_op_rx(struct ath10k *ar, struct sk_buff *skb)
 		break;
 	case WMI_10_4_WOW_WAKEUP_HOST_EVENTID:
 	case WMI_10_4_PEER_RATECODE_LIST_EVENTID:
+	case WMI_10_4_WDS_PEER_EVENTID:
 		ath10k_dbg(ar, ATH10K_DBG_WMI,
 			   "received event id %d not implemented\n", id);
 		break;
@@ -6863,7 +6942,7 @@ ath10k_wmi_op_gen_force_fw_hang(struct ath10k *ar,
 }
 
 static struct sk_buff *
-ath10k_wmi_op_gen_dbglog_cfg(struct ath10k *ar, u32 module_enable,
+ath10k_wmi_op_gen_dbglog_cfg(struct ath10k *ar, u64 module_enable,
 			     u32 log_level)
 {
 	struct wmi_dbglog_cfg_cmd *cmd;
@@ -6901,6 +6980,44 @@ ath10k_wmi_op_gen_dbglog_cfg(struct ath10k *ar, u32 module_enable,
 }
 
 static struct sk_buff *
+ath10k_wmi_10_4_op_gen_dbglog_cfg(struct ath10k *ar, u64 module_enable,
+				  u32 log_level)
+{
+	struct wmi_10_4_dbglog_cfg_cmd *cmd;
+	struct sk_buff *skb;
+	u32 cfg;
+
+	skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd));
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	cmd = (struct wmi_10_4_dbglog_cfg_cmd *)skb->data;
+
+	if (module_enable) {
+		cfg = SM(log_level,
+			 ATH10K_DBGLOG_CFG_LOG_LVL);
+	} else {
+		/* set back defaults, all modules with WARN level */
+		cfg = SM(ATH10K_DBGLOG_LEVEL_WARN,
+			 ATH10K_DBGLOG_CFG_LOG_LVL);
+		module_enable = ~0;
+	}
+
+	cmd->module_enable = __cpu_to_le64(module_enable);
+	cmd->module_valid = __cpu_to_le64(~0);
+	cmd->config_enable = __cpu_to_le32(cfg);
+	cmd->config_valid = __cpu_to_le32(ATH10K_DBGLOG_CFG_LOG_LVL_MASK);
+
+	ath10k_dbg(ar, ATH10K_DBG_WMI,
+		   "wmi dbglog cfg modules 0x%016llx 0x%016llx config %08x %08x\n",
+		   __le64_to_cpu(cmd->module_enable),
+		   __le64_to_cpu(cmd->module_valid),
+		   __le32_to_cpu(cmd->config_enable),
+		   __le32_to_cpu(cmd->config_valid));
+	return skb;
+}
+
+static struct sk_buff *
 ath10k_wmi_op_gen_pktlog_enable(struct ath10k *ar, u32 ev_bitmap)
 {
 	struct wmi_pdev_pktlog_enable_cmd *cmd;
@@ -7649,6 +7766,48 @@ ath10k_wmi_10_4_ext_resource_config(struct ath10k *ar,
 	return skb;
 }
 
+static struct sk_buff *
+ath10k_wmi_op_gen_echo(struct ath10k *ar, u32 value)
+{
+	struct wmi_echo_cmd *cmd;
+	struct sk_buff *skb;
+
+	skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd));
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	cmd = (struct wmi_echo_cmd *)skb->data;
+	cmd->value = cpu_to_le32(value);
+
+	ath10k_dbg(ar, ATH10K_DBG_WMI,
+		   "wmi echo value 0x%08x\n", value);
+	return skb;
+}
+
+int
+ath10k_wmi_barrier(struct ath10k *ar)
+{
+	int ret;
+	int time_left;
+
+	spin_lock_bh(&ar->data_lock);
+	reinit_completion(&ar->wmi.barrier);
+	spin_unlock_bh(&ar->data_lock);
+
+	ret = ath10k_wmi_echo(ar, ATH10K_WMI_BARRIER_ECHO_ID);
+	if (ret) {
+		ath10k_warn(ar, "failed to submit wmi echo: %d\n", ret);
+		return ret;
+	}
+
+	time_left = wait_for_completion_timeout(&ar->wmi.barrier,
+						ATH10K_WMI_BARRIER_TIMEOUT_HZ);
+	if (!time_left)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
 static const struct wmi_ops wmi_ops = {
 	.rx = ath10k_wmi_op_rx,
 	.map_svc = wmi_main_svc_map,
@@ -7665,6 +7824,7 @@ static const struct wmi_ops wmi_ops = {
 	.pull_rdy = ath10k_wmi_op_pull_rdy_ev,
 	.pull_fw_stats = ath10k_wmi_main_op_pull_fw_stats,
 	.pull_roam_ev = ath10k_wmi_op_pull_roam_ev,
+	.pull_echo_ev = ath10k_wmi_op_pull_echo_ev,
 
 	.gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend,
 	.gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume,
@@ -7709,6 +7869,7 @@ static const struct wmi_ops wmi_ops = {
 	.gen_delba_send = ath10k_wmi_op_gen_delba_send,
 	.fw_stats_fill = ath10k_wmi_main_op_fw_stats_fill,
 	.get_vdev_subtype = ath10k_wmi_op_get_vdev_subtype,
+	.gen_echo = ath10k_wmi_op_gen_echo,
 	/* .gen_bcn_tmpl not implemented */
 	/* .gen_prb_tmpl not implemented */
 	/* .gen_p2p_go_bcn_ie not implemented */
@@ -7738,6 +7899,7 @@ static const struct wmi_ops wmi_10_1_ops = {
 	.pull_phyerr = ath10k_wmi_op_pull_phyerr_ev,
 	.pull_rdy = ath10k_wmi_op_pull_rdy_ev,
 	.pull_roam_ev = ath10k_wmi_op_pull_roam_ev,
+	.pull_echo_ev = ath10k_wmi_op_pull_echo_ev,
 
 	.gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend,
 	.gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume,
@@ -7777,6 +7939,7 @@ static const struct wmi_ops wmi_10_1_ops = {
 	.gen_delba_send = ath10k_wmi_op_gen_delba_send,
 	.fw_stats_fill = ath10k_wmi_10x_op_fw_stats_fill,
 	.get_vdev_subtype = ath10k_wmi_op_get_vdev_subtype,
+	.gen_echo = ath10k_wmi_op_gen_echo,
 	/* .gen_bcn_tmpl not implemented */
 	/* .gen_prb_tmpl not implemented */
 	/* .gen_p2p_go_bcn_ie not implemented */
@@ -7796,6 +7959,7 @@ static const struct wmi_ops wmi_10_2_ops = {
 	.pull_svc_rdy = ath10k_wmi_10x_op_pull_svc_rdy_ev,
 	.gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd,
 	.gen_start_scan = ath10k_wmi_10x_op_gen_start_scan,
+	.gen_echo = ath10k_wmi_op_gen_echo,
 
 	.pull_scan = ath10k_wmi_op_pull_scan_ev,
 	.pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev,
@@ -7807,6 +7971,7 @@ static const struct wmi_ops wmi_10_2_ops = {
 	.pull_phyerr = ath10k_wmi_op_pull_phyerr_ev,
 	.pull_rdy = ath10k_wmi_op_pull_rdy_ev,
 	.pull_roam_ev = ath10k_wmi_op_pull_roam_ev,
+	.pull_echo_ev = ath10k_wmi_op_pull_echo_ev,
 
 	.gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend,
 	.gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume,
@@ -7862,6 +8027,7 @@ static const struct wmi_ops wmi_10_2_4_ops = {
 	.pull_svc_rdy = ath10k_wmi_10x_op_pull_svc_rdy_ev,
 	.gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd,
 	.gen_start_scan = ath10k_wmi_10x_op_gen_start_scan,
+	.gen_echo = ath10k_wmi_op_gen_echo,
 
 	.pull_scan = ath10k_wmi_op_pull_scan_ev,
 	.pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev,
@@ -7873,6 +8039,7 @@ static const struct wmi_ops wmi_10_2_4_ops = {
 	.pull_phyerr = ath10k_wmi_op_pull_phyerr_ev,
 	.pull_rdy = ath10k_wmi_op_pull_rdy_ev,
 	.pull_roam_ev = ath10k_wmi_op_pull_roam_ev,
+	.pull_echo_ev = ath10k_wmi_op_pull_echo_ev,
 
 	.gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend,
 	.gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume,
@@ -7968,7 +8135,7 @@ static const struct wmi_ops wmi_10_4_ops = {
 	.gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm,
 	.gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang,
 	.gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx,
-	.gen_dbglog_cfg = ath10k_wmi_op_gen_dbglog_cfg,
+	.gen_dbglog_cfg = ath10k_wmi_10_4_op_gen_dbglog_cfg,
 	.gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable,
 	.gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable,
 	.gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode,
@@ -7980,10 +8147,12 @@ static const struct wmi_ops wmi_10_4_ops = {
 	.ext_resource_config = ath10k_wmi_10_4_ext_resource_config,
 
 	/* shared with 10.2 */
+	.pull_echo_ev = ath10k_wmi_op_pull_echo_ev,
 	.gen_request_stats = ath10k_wmi_op_gen_request_stats,
 	.gen_pdev_get_temperature = ath10k_wmi_10_2_op_gen_pdev_get_temperature,
 	.get_vdev_subtype = ath10k_wmi_10_4_op_get_vdev_subtype,
 	.gen_pdev_bss_chan_info_req = ath10k_wmi_10_2_op_gen_pdev_bss_chan_info,
+	.gen_echo = ath10k_wmi_op_gen_echo,
 };
 
 int ath10k_wmi_attach(struct ath10k *ar)
@@ -8036,6 +8205,7 @@ int ath10k_wmi_attach(struct ath10k *ar)
 
 	init_completion(&ar->wmi.service_ready);
 	init_completion(&ar->wmi.unified_ready);
+	init_completion(&ar->wmi.barrier);
 
 	INIT_WORK(&ar->svc_rdy_work, ath10k_wmi_event_service_ready_work);
 
diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h
index 3ef468893b3f..1b243c899bef 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.h
+++ b/drivers/net/wireless/ath/ath10k/wmi.h
@@ -55,7 +55,7 @@
  *    type.
  *
  * 6. Comment each parameter part of the WMI command/event structure by
- *    using the 2 stars at the begining of C comment instead of one star to
+ *    using the 2 stars at the beginning of C comment instead of one star to
  *    enable HTML document generation using Doxygen.
  *
  */
@@ -180,6 +180,7 @@ enum wmi_service {
 	WMI_SERVICE_MESH_NON_11S,
 	WMI_SERVICE_PEER_STATS,
 	WMI_SERVICE_RESTRT_CHNL_SUPPORT,
+	WMI_SERVICE_PERIODIC_CHAN_STAT_SUPPORT,
 	WMI_SERVICE_TX_MODE_PUSH_ONLY,
 	WMI_SERVICE_TX_MODE_PUSH_PULL,
 	WMI_SERVICE_TX_MODE_DYNAMIC,
@@ -305,6 +306,7 @@ enum wmi_10_4_service {
 	WMI_10_4_SERVICE_RESTRT_CHNL_SUPPORT,
 	WMI_10_4_SERVICE_PEER_STATS,
 	WMI_10_4_SERVICE_MESH_11S,
+	WMI_10_4_SERVICE_PERIODIC_CHAN_STAT_SUPPORT,
 	WMI_10_4_SERVICE_TX_MODE_PUSH_ONLY,
 	WMI_10_4_SERVICE_TX_MODE_PUSH_PULL,
 	WMI_10_4_SERVICE_TX_MODE_DYNAMIC,
@@ -402,6 +404,7 @@ static inline char *wmi_service_name(int service_id)
 	SVCSTR(WMI_SERVICE_MESH_NON_11S);
 	SVCSTR(WMI_SERVICE_PEER_STATS);
 	SVCSTR(WMI_SERVICE_RESTRT_CHNL_SUPPORT);
+	SVCSTR(WMI_SERVICE_PERIODIC_CHAN_STAT_SUPPORT);
 	SVCSTR(WMI_SERVICE_TX_MODE_PUSH_ONLY);
 	SVCSTR(WMI_SERVICE_TX_MODE_PUSH_PULL);
 	SVCSTR(WMI_SERVICE_TX_MODE_DYNAMIC);
@@ -652,6 +655,8 @@ static inline void wmi_10_4_svc_map(const __le32 *in, unsigned long *out,
 	       WMI_SERVICE_PEER_STATS, len);
 	SVCMAP(WMI_10_4_SERVICE_MESH_11S,
 	       WMI_SERVICE_MESH_11S, len);
+	SVCMAP(WMI_10_4_SERVICE_PERIODIC_CHAN_STAT_SUPPORT,
+	       WMI_SERVICE_PERIODIC_CHAN_STAT_SUPPORT, len);
 	SVCMAP(WMI_10_4_SERVICE_TX_MODE_PUSH_ONLY,
 	       WMI_SERVICE_TX_MODE_PUSH_ONLY, len);
 	SVCMAP(WMI_10_4_SERVICE_TX_MODE_PUSH_PULL,
@@ -2082,7 +2087,7 @@ struct wmi_resource_config {
 	 * In offload mode target supports features like WOW, chatter and
 	 * other protocol offloads. In order to support them some
 	 * functionalities like reorder buffering, PN checking need to be
-	 * done in target. This determines maximum number of peers suported
+	 * done in target. This determines maximum number of peers supported
 	 * by target in offload mode
 	 */
 	__le32 num_offload_peers;
@@ -2263,7 +2268,7 @@ struct wmi_resource_config {
 	 * Max. number of Tx fragments per MSDU
 	 *  This parameter controls the max number of Tx fragments per MSDU.
 	 *  This is sent by the target as part of the WMI_SERVICE_READY event
-	 *  and is overriden by the OS shim as required.
+	 *  and is overridden by the OS shim as required.
 	 */
 	__le32 max_frag_entries;
 } __packed;
@@ -2445,7 +2450,7 @@ struct wmi_resource_config_10x {
 	 * Max. number of Tx fragments per MSDU
 	 *  This parameter controls the max number of Tx fragments per MSDU.
 	 *  This is sent by the target as part of the WMI_SERVICE_READY event
-	 *  and is overriden by the OS shim as required.
+	 *  and is overridden by the OS shim as required.
 	 */
 	__le32 max_frag_entries;
 } __packed;
@@ -2739,7 +2744,7 @@ struct wmi_init_cmd {
 	struct wmi_host_mem_chunks mem_chunks;
 } __packed;
 
-/* _10x stucture is from 10.X FW API */
+/* _10x structure is from 10.X FW API */
 struct wmi_init_cmd_10x {
 	struct wmi_resource_config_10x resource_config;
 	struct wmi_host_mem_chunks mem_chunks;
@@ -3962,7 +3967,7 @@ struct wmi_pdev_stats_tx {
 	/* illegal rate phy errors  */
 	__le32 illgl_rate_phy_err;
 
-	/* wal pdev continous xretry */
+	/* wal pdev continuous xretry */
 	__le32 pdev_cont_xretry;
 
 	/* wal pdev continous xretry */
@@ -4217,10 +4222,10 @@ struct wmi_10_2_stats_event {
  */
 struct wmi_pdev_stats_base {
 	__le32 chan_nf;
-	__le32 tx_frame_count;
-	__le32 rx_frame_count;
-	__le32 rx_clear_count;
-	__le32 cycle_count;
+	__le32 tx_frame_count; /* Cycles spent transmitting frames */
+	__le32 rx_frame_count; /* Cycles spent receiving frames */
+	__le32 rx_clear_count; /* Total channel busy time, evidently */
+	__le32 cycle_count; /* Total on-channel time */
 	__le32 phy_err_count;
 	__le32 chan_tx_pwr;
 } __packed;
@@ -4456,9 +4461,9 @@ struct wmi_vdev_start_request_cmd {
 	__le32 flags;
 	/* ssid field. Only valid for AP/GO/IBSS/BTAmp VDEV type. */
 	struct wmi_ssid ssid;
-	/* beacon/probe reponse xmit rate. Applicable for SoftAP. */
+	/* beacon/probe response xmit rate. Applicable for SoftAP. */
 	__le32 bcn_tx_rate;
-	/* beacon/probe reponse xmit power. Applicable for SoftAP. */
+	/* beacon/probe response xmit power. Applicable for SoftAP. */
 	__le32 bcn_tx_power;
 	/* number of p2p NOA descriptor(s) from scan entry */
 	__le32 num_noa_descriptors;
@@ -4686,7 +4691,7 @@ enum wmi_vdev_param {
 	WMI_VDEV_PARAM_BEACON_INTERVAL,
 	/* Listen interval in TUs */
 	WMI_VDEV_PARAM_LISTEN_INTERVAL,
-	/* muticast rate in Mbps */
+	/* multicast rate in Mbps */
 	WMI_VDEV_PARAM_MULTICAST_RATE,
 	/* management frame rate in Mbps */
 	WMI_VDEV_PARAM_MGMT_TX_RATE,
@@ -4817,7 +4822,7 @@ enum wmi_10x_vdev_param {
 	WMI_10X_VDEV_PARAM_BEACON_INTERVAL,
 	/* Listen interval in TUs */
 	WMI_10X_VDEV_PARAM_LISTEN_INTERVAL,
-	/* muticast rate in Mbps */
+	/* multicast rate in Mbps */
 	WMI_10X_VDEV_PARAM_MULTICAST_RATE,
 	/* management frame rate in Mbps */
 	WMI_10X_VDEV_PARAM_MGMT_TX_RATE,
@@ -5062,7 +5067,7 @@ struct wmi_vdev_simple_event {
 } __packed;
 
 /* VDEV start response status codes */
-/* VDEV succesfully started */
+/* VDEV successfully started */
 #define WMI_INIFIED_VDEV_START_RESPONSE_STATUS_SUCCESS	0x0
 
 /* requested VDEV not found */
@@ -5378,7 +5383,7 @@ enum wmi_sta_ps_param_pspoll_count {
 #define WMI_UAPSD_AC_TYPE_TRIG 1
 
 #define WMI_UAPSD_AC_BIT_MASK(ac, type) \
-	((type ==  WMI_UAPSD_AC_TYPE_DELI) ? (1 << (ac << 1)) : (1 << ((ac << 1) + 1)))
+	(type == WMI_UAPSD_AC_TYPE_DELI ? 1 << (ac << 1) : 1 << ((ac << 1) + 1))
 
 enum wmi_sta_ps_param_uapsd {
 	WMI_STA_PS_UAPSD_AC0_DELIVERY_EN = (1 << 0),
@@ -6169,6 +6174,20 @@ struct wmi_dbglog_cfg_cmd {
 	__le32 config_valid;
 } __packed;
 
+struct wmi_10_4_dbglog_cfg_cmd {
+	/* bitmask to hold mod id config*/
+	__le64 module_enable;
+
+	/* see ATH10K_DBGLOG_CFG_ */
+	__le32 config_enable;
+
+	/* mask of module id bits to be changed */
+	__le64 module_valid;
+
+	/* mask of config bits to be changed, see ATH10K_DBGLOG_CFG_ */
+	__le32 config_valid;
+} __packed;
+
 enum wmi_roam_reason {
 	WMI_ROAM_REASON_BETTER_AP = 1,
 	WMI_ROAM_REASON_BEACON_MISS = 2,
@@ -6296,6 +6315,10 @@ struct wmi_roam_ev_arg {
 	__le32 rssi;
 };
 
+struct wmi_echo_ev_arg {
+	__le32 value;
+};
+
 struct wmi_pdev_temperature_event {
 	/* temperature value in Celcius degree */
 	__le32 temperature;
@@ -6624,5 +6647,6 @@ void ath10k_wmi_10_4_op_fw_stats_fill(struct ath10k *ar,
 				      char *buf);
 int ath10k_wmi_op_get_vdev_subtype(struct ath10k *ar,
 				   enum wmi_vdev_subtype subtype);
+int ath10k_wmi_barrier(struct ath10k *ar);
 
 #endif /* _WMI_H_ */
diff --git a/drivers/net/wireless/ath/ath5k/debug.c b/drivers/net/wireless/ath/ath5k/debug.c
index 929d7ccc031c..4f8d9ed04f5e 100644
--- a/drivers/net/wireless/ath/ath5k/debug.c
+++ b/drivers/net/wireless/ath/ath5k/debug.c
@@ -909,7 +909,7 @@ static int open_file_eeprom(struct inode *inode, struct file *file)
 	struct ath5k_hw *ah = inode->i_private;
 	bool res;
 	int i, ret;
-	u32 eesize;
+	u32 eesize;	/* NB: in 16-bit words */
 	u16 val, *buf;
 
 	/* Get eeprom size */
@@ -932,7 +932,7 @@ static int open_file_eeprom(struct inode *inode, struct file *file)
 
 	/* Create buffer and read in eeprom */
 
-	buf = vmalloc(eesize);
+	buf = vmalloc(eesize * 2);
 	if (!buf) {
 		ret = -ENOMEM;
 		goto err;
@@ -952,7 +952,7 @@ static int open_file_eeprom(struct inode *inode, struct file *file)
 	}
 
 	ep->buf = buf;
-	ep->len = i;
+	ep->len = eesize * 2;
 
 	file->private_data = (void *)ep;
 
diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c
index 72e2ec67768d..b7fe0af4cb24 100644
--- a/drivers/net/wireless/ath/ath6kl/cfg80211.c
+++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c
@@ -1449,14 +1449,14 @@ static int ath6kl_cfg80211_get_txpower(struct wiphy *wiphy,
 		return -EIO;
 
 	if (test_bit(CONNECTED, &vif->flags)) {
-		ar->tx_pwr = 0;
+		ar->tx_pwr = 255;
 
 		if (ath6kl_wmi_get_tx_pwr_cmd(ar->wmi, vif->fw_vif_idx) != 0) {
 			ath6kl_err("ath6kl_wmi_get_tx_pwr_cmd failed\n");
 			return -EIO;
 		}
 
-		wait_event_interruptible_timeout(ar->event_wq, ar->tx_pwr != 0,
+		wait_event_interruptible_timeout(ar->event_wq, ar->tx_pwr != 255,
 						 5 * HZ);
 
 		if (signal_pending(current)) {
diff --git a/drivers/net/wireless/ath/ath6kl/hif.c b/drivers/net/wireless/ath/ath6kl/hif.c
index 18c070850a09..d1942537ea10 100644
--- a/drivers/net/wireless/ath/ath6kl/hif.c
+++ b/drivers/net/wireless/ath/ath6kl/hif.c
@@ -64,7 +64,7 @@ int ath6kl_hif_rw_comp_handler(void *context, int status)
 }
 EXPORT_SYMBOL(ath6kl_hif_rw_comp_handler);
 
-#define REG_DUMP_COUNT_AR6003   60
+#define REGISTER_DUMP_COUNT     60
 #define REGISTER_DUMP_LEN_MAX   60
 
 static void ath6kl_hif_dump_fw_crash(struct ath6kl *ar)
@@ -73,9 +73,6 @@ static void ath6kl_hif_dump_fw_crash(struct ath6kl *ar)
 	u32 i, address, regdump_addr = 0;
 	int ret;
 
-	if (ar->target_type != TARGET_TYPE_AR6003)
-		return;
-
 	/* the reg dump pointer is copied to the host interest area */
 	address = ath6kl_get_hi_item_addr(ar, HI_ITEM(hi_failure_state));
 	address = TARG_VTOP(ar->target_type, address);
@@ -95,7 +92,7 @@ static void ath6kl_hif_dump_fw_crash(struct ath6kl *ar)
 
 	/* fetch register dump data */
 	ret = ath6kl_diag_read(ar, regdump_addr, (u8 *)&regdump_val[0],
-				  REG_DUMP_COUNT_AR6003 * (sizeof(u32)));
+				  REGISTER_DUMP_COUNT * (sizeof(u32)));
 	if (ret) {
 		ath6kl_warn("failed to get register dump: %d\n", ret);
 		return;
@@ -105,9 +102,9 @@ static void ath6kl_hif_dump_fw_crash(struct ath6kl *ar)
 	ath6kl_info("hw 0x%x fw %s\n", ar->wiphy->hw_version,
 		    ar->wiphy->fw_version);
 
-	BUILD_BUG_ON(REG_DUMP_COUNT_AR6003 % 4);
+	BUILD_BUG_ON(REGISTER_DUMP_COUNT % 4);
 
-	for (i = 0; i < REG_DUMP_COUNT_AR6003; i += 4) {
+	for (i = 0; i < REGISTER_DUMP_COUNT; i += 4) {
 		ath6kl_info("%d: 0x%8.8x 0x%8.8x 0x%8.8x 0x%8.8x\n",
 			    i,
 			    le32_to_cpu(regdump_val[i]),
diff --git a/drivers/net/wireless/ath/ath6kl/wmi.c b/drivers/net/wireless/ath/ath6kl/wmi.c
index b8cf04d11975..3fd1cc98fd2f 100644
--- a/drivers/net/wireless/ath/ath6kl/wmi.c
+++ b/drivers/net/wireless/ath/ath6kl/wmi.c
@@ -3520,7 +3520,7 @@ int ath6kl_wmi_set_pvb_cmd(struct wmi *wmi, u8 if_idx, u16 aid,
 	ret = ath6kl_wmi_cmd_send(wmi, if_idx, skb, WMI_AP_SET_PVB_CMDID,
 				  NO_SYNC_WMIFLAG);
 
-	return 0;
+	return ret;
 }
 
 int ath6kl_wmi_set_rx_frame_format_cmd(struct wmi *wmi, u8 if_idx,
diff --git a/drivers/net/wireless/ath/ath9k/Kconfig b/drivers/net/wireless/ath/ath9k/Kconfig
index f68cb00450e0..8f231c67dd51 100644
--- a/drivers/net/wireless/ath/ath9k/Kconfig
+++ b/drivers/net/wireless/ath/ath9k/Kconfig
@@ -180,7 +180,7 @@ config ATH9K_HTC_DEBUGFS
 config ATH9K_HWRNG
 	bool "Random number generator support"
 	depends on ATH9K && (HW_RANDOM = y || HW_RANDOM = ATH9K)
-	default y
+	default n
 	---help---
 	  This option incorporates the ADC register output as a source of
 	  randomness into Linux entropy pool (/dev/urandom and /dev/random)
diff --git a/drivers/net/wireless/ath/ath9k/ar5008_phy.c b/drivers/net/wireless/ath/ath9k/ar5008_phy.c
index 1b271b99c49e..8eea8d22e72e 100644
--- a/drivers/net/wireless/ath/ath9k/ar5008_phy.c
+++ b/drivers/net/wireless/ath/ath9k/ar5008_phy.c
@@ -260,8 +260,8 @@ void ar5008_hw_cmn_spur_mitigate(struct ath_hw *ah,
 	int cur_bin;
 	int upper, lower, cur_vit_mask;
 	int i;
-	int8_t mask_m[123];
-	int8_t mask_p[123];
+	int8_t mask_m[123] = {0};
+	int8_t mask_p[123] = {0};
 	int8_t mask_amt;
 	int tmp_mask;
 	static const int pilot_mask_reg[4] = {
@@ -274,9 +274,6 @@ void ar5008_hw_cmn_spur_mitigate(struct ath_hw *ah,
 	};
 	static const int inc[4] = { 0, 100, 0, 0 };
 
-	memset(&mask_m, 0, sizeof(int8_t) * 123);
-	memset(&mask_p, 0, sizeof(int8_t) * 123);
-
 	cur_bin = -6000;
 	upper = bin + 100;
 	lower = bin - 100;
@@ -302,7 +299,7 @@ void ar5008_hw_cmn_spur_mitigate(struct ath_hw *ah,
 	upper = bin + 120;
 	lower = bin - 120;
 
-	for (i = 0; i < 123; i++) {
+	for (i = 0; i < ARRAY_SIZE(mask_m); i++) {
 		if ((cur_vit_mask > lower) && (cur_vit_mask < upper)) {
 			/* workaround for gcc bug #37014 */
 			volatile int tmp_v = abs(cur_vit_mask - bin);
diff --git a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c
index 5bd2cbaf582d..08607d7fdb56 100644
--- a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c
+++ b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c
@@ -3252,7 +3252,8 @@ static int ar9300_eeprom_restore_flash(struct ath_hw *ah, u8 *mptr,
 	int i;
 
 	for (i = 0; i < mdata_size / 2; i++, data++)
-		ath9k_hw_nvram_read(ah, i, data);
+		if (!ath9k_hw_nvram_read(ah, i, data))
+			return -EIO;
 
 	return 0;
 }
@@ -3282,7 +3283,8 @@ static int ar9300_eeprom_restore_internal(struct ath_hw *ah,
 	if (ath9k_hw_use_flash(ah)) {
 		u8 txrx;
 
-		ar9300_eeprom_restore_flash(ah, mptr, mdata_size);
+		if (ar9300_eeprom_restore_flash(ah, mptr, mdata_size))
+			return -EIO;
 
 		/* check if eeprom contains valid data */
 		eep = (struct ar9300_eeprom *) mptr;
diff --git a/drivers/net/wireless/ath/ath9k/gpio.c b/drivers/net/wireless/ath/ath9k/gpio.c
index 490f74d9ddf0..ddb28861e7fe 100644
--- a/drivers/net/wireless/ath/ath9k/gpio.c
+++ b/drivers/net/wireless/ath/ath9k/gpio.c
@@ -22,7 +22,7 @@
 
 #ifdef CONFIG_MAC80211_LEDS
 
-void ath_fill_led_pin(struct ath_softc *sc)
+static void ath_fill_led_pin(struct ath_softc *sc)
 {
 	struct ath_hw *ah = sc->sc_ah;
 
diff --git a/drivers/net/wireless/ath/ath9k/hw.c b/drivers/net/wireless/ath/ath9k/hw.c
index d1d0c06d627c..14b13f07cd1f 100644
--- a/drivers/net/wireless/ath/ath9k/hw.c
+++ b/drivers/net/wireless/ath/ath9k/hw.c
@@ -2482,6 +2482,8 @@ int ath9k_hw_fill_cap_info(struct ath_hw *ah)
 		return -EINVAL;
 	}
 
+	ath9k_gpio_cap_init(ah);
+
 	if (AR_SREV_9485(ah) ||
 	    AR_SREV_9285(ah) ||
 	    AR_SREV_9330(ah) ||
@@ -2531,8 +2533,6 @@ int ath9k_hw_fill_cap_info(struct ath_hw *ah)
 	else
 		pCap->hw_caps &= ~ATH9K_HW_CAP_HT;
 
-	ath9k_gpio_cap_init(ah);
-
 	if (AR_SREV_9160_10_OR_LATER(ah) || AR_SREV_9100(ah))
 		pCap->rts_aggr_limit = ATH_AMPDU_LIMIT_MAX;
 	else
diff --git a/drivers/net/wireless/ath/ath9k/main.c b/drivers/net/wireless/ath/ath9k/main.c
index a394622c9022..e9f32b52fc8c 100644
--- a/drivers/net/wireless/ath/ath9k/main.c
+++ b/drivers/net/wireless/ath/ath9k/main.c
@@ -718,9 +718,12 @@ static int ath9k_start(struct ieee80211_hw *hw)
 	if (!ath_complete_reset(sc, false))
 		ah->reset_power_on = false;
 
-	if (ah->led_pin >= 0)
+	if (ah->led_pin >= 0) {
 		ath9k_hw_set_gpio(ah, ah->led_pin,
 				  (ah->config.led_active_high) ? 1 : 0);
+		ath9k_hw_gpio_request_out(ah, ah->led_pin, NULL,
+					  AR_GPIO_OUTPUT_MUX_AS_OUTPUT);
+	}
 
 	/*
 	 * Reset key cache to sane defaults (all entries cleared) instead of
@@ -864,9 +867,11 @@ static void ath9k_stop(struct ieee80211_hw *hw)
 
 	spin_lock_bh(&sc->sc_pcu_lock);
 
-	if (ah->led_pin >= 0)
+	if (ah->led_pin >= 0) {
 		ath9k_hw_set_gpio(ah, ah->led_pin,
 				  (ah->config.led_active_high) ? 0 : 1);
+		ath9k_hw_gpio_request_in(ah, ah->led_pin, NULL);
+	}
 
 	ath_prepare_reset(sc);
 
@@ -919,7 +924,7 @@ static void ath9k_vif_iter_set_beacon(struct ath9k_vif_iter_data *iter_data,
 	} else {
 		if (iter_data->primary_beacon_vif->type != NL80211_IFTYPE_AP &&
 		    vif->type == NL80211_IFTYPE_AP)
-		iter_data->primary_beacon_vif = vif;
+			iter_data->primary_beacon_vif = vif;
 	}
 
 	iter_data->beacons = true;
@@ -1154,6 +1159,7 @@ void ath9k_calculate_summary_state(struct ath_softc *sc,
 		bool changed = (iter_data.primary_sta != ctx->primary_sta);
 
 		if (iter_data.primary_sta) {
+			iter_data.primary_beacon_vif = iter_data.primary_sta;
 			iter_data.beacons = true;
 			ath9k_set_assoc_state(sc, iter_data.primary_sta,
 					      changed);
@@ -1563,13 +1569,13 @@ static int ath9k_sta_state(struct ieee80211_hw *hw,
 	struct ath_common *common = ath9k_hw_common(sc->sc_ah);
 	int ret = 0;
 
-	if (old_state == IEEE80211_STA_AUTH &&
-	    new_state == IEEE80211_STA_ASSOC) {
+	if (old_state == IEEE80211_STA_NOTEXIST &&
+	    new_state == IEEE80211_STA_NONE) {
 		ret = ath9k_sta_add(hw, vif, sta);
 		ath_dbg(common, CONFIG,
 			"Add station: %pM\n", sta->addr);
-	} else if (old_state == IEEE80211_STA_ASSOC &&
-		   new_state == IEEE80211_STA_AUTH) {
+	} else if (old_state == IEEE80211_STA_NONE &&
+		   new_state == IEEE80211_STA_NOTEXIST) {
 		ret = ath9k_sta_remove(hw, vif, sta);
 		ath_dbg(common, CONFIG,
 			"Remove station: %pM\n", sta->addr);
diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c
index 8ddd604bd00c..52bfbb988611 100644
--- a/drivers/net/wireless/ath/ath9k/xmit.c
+++ b/drivers/net/wireless/ath/ath9k/xmit.c
@@ -50,9 +50,11 @@ static u16 bits_per_symbol[][2] = {
 static void ath_tx_send_normal(struct ath_softc *sc, struct ath_txq *txq,
 			       struct ath_atx_tid *tid, struct sk_buff *skb);
 static void ath_tx_complete(struct ath_softc *sc, struct sk_buff *skb,
-			    int tx_flags, struct ath_txq *txq);
+			    int tx_flags, struct ath_txq *txq,
+			    struct ieee80211_sta *sta);
 static void ath_tx_complete_buf(struct ath_softc *sc, struct ath_buf *bf,
 				struct ath_txq *txq, struct list_head *bf_q,
+				struct ieee80211_sta *sta,
 				struct ath_tx_status *ts, int txok);
 static void ath_tx_txqaddbuf(struct ath_softc *sc, struct ath_txq *txq,
 			     struct list_head *head, bool internal);
@@ -77,6 +79,22 @@ enum {
 /* Aggregation logic */
 /*********************/
 
+static void ath_tx_status(struct ieee80211_hw *hw, struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_sta *sta = info->status.status_driver_data[0];
+
+	if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) {
+		ieee80211_tx_status(hw, skb);
+		return;
+	}
+
+	if (sta)
+		ieee80211_tx_status_noskb(hw, sta, info);
+
+	dev_kfree_skb(skb);
+}
+
 void ath_txq_lock(struct ath_softc *sc, struct ath_txq *txq)
 	__acquires(&txq->axq_lock)
 {
@@ -92,6 +110,7 @@ void ath_txq_unlock(struct ath_softc *sc, struct ath_txq *txq)
 void ath_txq_unlock_complete(struct ath_softc *sc, struct ath_txq *txq)
 	__releases(&txq->axq_lock)
 {
+	struct ieee80211_hw *hw = sc->hw;
 	struct sk_buff_head q;
 	struct sk_buff *skb;
 
@@ -100,7 +119,7 @@ void ath_txq_unlock_complete(struct ath_softc *sc, struct ath_txq *txq)
 	spin_unlock_bh(&txq->axq_lock);
 
 	while ((skb = __skb_dequeue(&q)))
-		ieee80211_tx_status(sc->hw, skb);
+		ath_tx_status(hw, skb);
 }
 
 static void ath_tx_queue_tid(struct ath_softc *sc, struct ath_txq *txq,
@@ -253,7 +272,7 @@ static void ath_tx_flush_tid(struct ath_softc *sc, struct ath_atx_tid *tid)
 		}
 
 		list_add_tail(&bf->list, &bf_head);
-		ath_tx_complete_buf(sc, bf, txq, &bf_head, &ts, 0);
+		ath_tx_complete_buf(sc, bf, txq, &bf_head, NULL, &ts, 0);
 	}
 
 	if (sendbar) {
@@ -318,12 +337,12 @@ static void ath_tid_drain(struct ath_softc *sc, struct ath_txq *txq,
 		bf = fi->bf;
 
 		if (!bf) {
-			ath_tx_complete(sc, skb, ATH_TX_ERROR, txq);
+			ath_tx_complete(sc, skb, ATH_TX_ERROR, txq, NULL);
 			continue;
 		}
 
 		list_add_tail(&bf->list, &bf_head);
-		ath_tx_complete_buf(sc, bf, txq, &bf_head, &ts, 0);
+		ath_tx_complete_buf(sc, bf, txq, &bf_head, NULL, &ts, 0);
 	}
 }
 
@@ -426,15 +445,14 @@ static void ath_tx_count_frames(struct ath_softc *sc, struct ath_buf *bf,
 
 static void ath_tx_complete_aggr(struct ath_softc *sc, struct ath_txq *txq,
 				 struct ath_buf *bf, struct list_head *bf_q,
+				 struct ieee80211_sta *sta,
+				 struct ath_atx_tid *tid,
 				 struct ath_tx_status *ts, int txok)
 {
 	struct ath_node *an = NULL;
 	struct sk_buff *skb;
-	struct ieee80211_sta *sta;
-	struct ieee80211_hw *hw = sc->hw;
 	struct ieee80211_hdr *hdr;
 	struct ieee80211_tx_info *tx_info;
-	struct ath_atx_tid *tid = NULL;
 	struct ath_buf *bf_next, *bf_last = bf->bf_lastbf;
 	struct list_head bf_head;
 	struct sk_buff_head bf_pending;
@@ -460,12 +478,7 @@ static void ath_tx_complete_aggr(struct ath_softc *sc, struct ath_txq *txq,
 	for (i = 0; i < ts->ts_rateindex; i++)
 		retries += rates[i].count;
 
-	rcu_read_lock();
-
-	sta = ieee80211_find_sta_by_ifaddr(hw, hdr->addr1, hdr->addr2);
 	if (!sta) {
-		rcu_read_unlock();
-
 		INIT_LIST_HEAD(&bf_head);
 		while (bf) {
 			bf_next = bf->bf_next;
@@ -473,7 +486,7 @@ static void ath_tx_complete_aggr(struct ath_softc *sc, struct ath_txq *txq,
 			if (!bf->bf_state.stale || bf_next != NULL)
 				list_move_tail(&bf->list, &bf_head);
 
-			ath_tx_complete_buf(sc, bf, txq, &bf_head, ts, 0);
+			ath_tx_complete_buf(sc, bf, txq, &bf_head, NULL, ts, 0);
 
 			bf = bf_next;
 		}
@@ -481,7 +494,6 @@ static void ath_tx_complete_aggr(struct ath_softc *sc, struct ath_txq *txq,
 	}
 
 	an = (struct ath_node *)sta->drv_priv;
-	tid = ath_get_skb_tid(sc, an, skb);
 	seq_first = tid->seq_start;
 	isba = ts->ts_flags & ATH9K_TX_BA;
 
@@ -583,7 +595,7 @@ static void ath_tx_complete_aggr(struct ath_softc *sc, struct ath_txq *txq,
 								ts);
 			}
 
-			ath_tx_complete_buf(sc, bf, txq, &bf_head, ts,
+			ath_tx_complete_buf(sc, bf, txq, &bf_head, sta, ts,
 				!txfail);
 		} else {
 			if (tx_info->flags & IEEE80211_TX_STATUS_EOSP) {
@@ -604,7 +616,8 @@ static void ath_tx_complete_aggr(struct ath_softc *sc, struct ath_txq *txq,
 					ath_tx_update_baw(sc, tid, seqno);
 
 					ath_tx_complete_buf(sc, bf, txq,
-							    &bf_head, ts, 0);
+							    &bf_head, NULL, ts,
+							    0);
 					bar_index = max_t(int, bar_index,
 						ATH_BA_INDEX(seq_first, seqno));
 					break;
@@ -648,8 +661,6 @@ static void ath_tx_complete_aggr(struct ath_softc *sc, struct ath_txq *txq,
 		ath_txq_lock(sc, txq);
 	}
 
-	rcu_read_unlock();
-
 	if (needreset)
 		ath9k_queue_reset(sc, RESET_TYPE_TX_ERROR);
 }
@@ -664,7 +675,11 @@ static void ath_tx_process_buffer(struct ath_softc *sc, struct ath_txq *txq,
 				  struct ath_tx_status *ts, struct ath_buf *bf,
 				  struct list_head *bf_head)
 {
+	struct ieee80211_hw *hw = sc->hw;
 	struct ieee80211_tx_info *info;
+	struct ieee80211_sta *sta;
+	struct ieee80211_hdr *hdr;
+	struct ath_atx_tid *tid = NULL;
 	bool txok, flush;
 
 	txok = !(ts->ts_status & ATH9K_TXERR_MASK);
@@ -677,6 +692,16 @@ static void ath_tx_process_buffer(struct ath_softc *sc, struct ath_txq *txq,
 
 	ts->duration = ath9k_hw_get_duration(sc->sc_ah, bf->bf_desc,
 					     ts->ts_rateindex);
+
+	hdr = (struct ieee80211_hdr *) bf->bf_mpdu->data;
+	sta = ieee80211_find_sta_by_ifaddr(hw, hdr->addr1, hdr->addr2);
+	if (sta) {
+		struct ath_node *an = (struct ath_node *)sta->drv_priv;
+		tid = ath_get_skb_tid(sc, an, bf->bf_mpdu);
+		if (ts->ts_status & (ATH9K_TXERR_FILT | ATH9K_TXERR_XRETRY))
+			tid->clear_ps_filter = true;
+	}
+
 	if (!bf_isampdu(bf)) {
 		if (!flush) {
 			info = IEEE80211_SKB_CB(bf->bf_mpdu);
@@ -685,9 +710,9 @@ static void ath_tx_process_buffer(struct ath_softc *sc, struct ath_txq *txq,
 			ath_tx_rc_status(sc, bf, ts, 1, txok ? 0 : 1, txok);
 			ath_dynack_sample_tx_ts(sc->sc_ah, bf->bf_mpdu, ts);
 		}
-		ath_tx_complete_buf(sc, bf, txq, bf_head, ts, txok);
+		ath_tx_complete_buf(sc, bf, txq, bf_head, sta, ts, txok);
 	} else
-		ath_tx_complete_aggr(sc, txq, bf, bf_head, ts, txok);
+		ath_tx_complete_aggr(sc, txq, bf, bf_head, sta, tid, ts, txok);
 
 	if (!flush)
 		ath_txq_schedule(sc, txq);
@@ -923,7 +948,7 @@ ath_tx_get_tid_subframe(struct ath_softc *sc, struct ath_txq *txq,
 			list_add(&bf->list, &bf_head);
 			__skb_unlink(skb, *q);
 			ath_tx_update_baw(sc, tid, seqno);
-			ath_tx_complete_buf(sc, bf, txq, &bf_head, &ts, 0);
+			ath_tx_complete_buf(sc, bf, txq, &bf_head, NULL, &ts, 0);
 			continue;
 		}
 
@@ -1832,6 +1857,7 @@ static void ath_drain_txq_list(struct ath_softc *sc, struct ath_txq *txq,
  */
 void ath_draintxq(struct ath_softc *sc, struct ath_txq *txq)
 {
+	rcu_read_lock();
 	ath_txq_lock(sc, txq);
 
 	if (sc->sc_ah->caps.hw_caps & ATH9K_HW_CAP_EDMA) {
@@ -1850,6 +1876,7 @@ void ath_draintxq(struct ath_softc *sc, struct ath_txq *txq)
 	ath_drain_txq_list(sc, txq, &txq->axq_q);
 
 	ath_txq_unlock_complete(sc, txq);
+	rcu_read_unlock();
 }
 
 bool ath_drain_all_txq(struct ath_softc *sc)
@@ -2472,7 +2499,8 @@ void ath_tx_cabq(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 /*****************/
 
 static void ath_tx_complete(struct ath_softc *sc, struct sk_buff *skb,
-			    int tx_flags, struct ath_txq *txq)
+			    int tx_flags, struct ath_txq *txq,
+			    struct ieee80211_sta *sta)
 {
 	struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb);
 	struct ath_common *common = ath9k_hw_common(sc->sc_ah);
@@ -2492,15 +2520,17 @@ static void ath_tx_complete(struct ath_softc *sc, struct sk_buff *skb,
 			tx_info->flags |= IEEE80211_TX_STAT_ACK;
 	}
 
-	padpos = ieee80211_hdrlen(hdr->frame_control);
-	padsize = padpos & 3;
-	if (padsize && skb->len>padpos+padsize) {
-		/*
-		 * Remove MAC header padding before giving the frame back to
-		 * mac80211.
-		 */
-		memmove(skb->data + padsize, skb->data, padpos);
-		skb_pull(skb, padsize);
+	if (tx_info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) {
+		padpos = ieee80211_hdrlen(hdr->frame_control);
+		padsize = padpos & 3;
+		if (padsize && skb->len>padpos+padsize) {
+			/*
+			 * Remove MAC header padding before giving the frame back to
+			 * mac80211.
+			 */
+			memmove(skb->data + padsize, skb->data, padpos);
+			skb_pull(skb, padsize);
+		}
 	}
 
 	spin_lock_irqsave(&sc->sc_pm_lock, flags);
@@ -2515,12 +2545,14 @@ static void ath_tx_complete(struct ath_softc *sc, struct sk_buff *skb,
 	}
 	spin_unlock_irqrestore(&sc->sc_pm_lock, flags);
 
-	__skb_queue_tail(&txq->complete_q, skb);
 	ath_txq_skb_done(sc, txq, skb);
+	tx_info->status.status_driver_data[0] = sta;
+	__skb_queue_tail(&txq->complete_q, skb);
 }
 
 static void ath_tx_complete_buf(struct ath_softc *sc, struct ath_buf *bf,
 				struct ath_txq *txq, struct list_head *bf_q,
+				struct ieee80211_sta *sta,
 				struct ath_tx_status *ts, int txok)
 {
 	struct sk_buff *skb = bf->bf_mpdu;
@@ -2548,7 +2580,7 @@ static void ath_tx_complete_buf(struct ath_softc *sc, struct ath_buf *bf,
 			complete(&sc->paprd_complete);
 	} else {
 		ath_debug_stat_tx(sc, bf, ts, txq, tx_flags);
-		ath_tx_complete(sc, skb, tx_flags, txq);
+		ath_tx_complete(sc, skb, tx_flags, txq, sta);
 	}
 skip_tx_complete:
 	/* At this point, skb (bf->bf_mpdu) is consumed...make sure we don't
@@ -2700,10 +2732,12 @@ void ath_tx_tasklet(struct ath_softc *sc)
 	u32 qcumask = ((1 << ATH9K_NUM_TX_QUEUES) - 1) & ah->intr_txqs;
 	int i;
 
+	rcu_read_lock();
 	for (i = 0; i < ATH9K_NUM_TX_QUEUES; i++) {
 		if (ATH_TXQ_SETUP(sc, i) && (qcumask & (1 << i)))
 			ath_tx_processq(sc, &sc->tx.txq[i]);
 	}
+	rcu_read_unlock();
 }
 
 void ath_tx_edma_tasklet(struct ath_softc *sc)
@@ -2717,6 +2751,7 @@ void ath_tx_edma_tasklet(struct ath_softc *sc)
 	struct list_head *fifo_list;
 	int status;
 
+	rcu_read_lock();
 	for (;;) {
 		if (test_bit(ATH_OP_HW_RESET, &common->op_flags))
 			break;
@@ -2787,6 +2822,7 @@ void ath_tx_edma_tasklet(struct ath_softc *sc)
 		ath_tx_process_buffer(sc, txq, &ts, bf, &bf_head);
 		ath_txq_unlock_complete(sc, txq);
 	}
+	rcu_read_unlock();
 }
 
 /*****************/
diff --git a/drivers/net/wireless/ath/carl9170/debug.c b/drivers/net/wireless/ath/carl9170/debug.c
index 6808db433283..ec3a64e5d2bb 100644
--- a/drivers/net/wireless/ath/carl9170/debug.c
+++ b/drivers/net/wireless/ath/carl9170/debug.c
@@ -75,7 +75,8 @@ static ssize_t carl9170_debugfs_read(struct file *file, char __user *userbuf,
 
 	if (!ar)
 		return -ENODEV;
-	dfops = container_of(file->f_op, struct carl9170_debugfs_fops, fops);
+	dfops = container_of(debugfs_real_fops(file),
+			     struct carl9170_debugfs_fops, fops);
 
 	if (!dfops->read)
 		return -ENOSYS;
@@ -127,7 +128,8 @@ static ssize_t carl9170_debugfs_write(struct file *file,
 
 	if (!ar)
 		return -ENODEV;
-	dfops = container_of(file->f_op, struct carl9170_debugfs_fops, fops);
+	dfops = container_of(debugfs_real_fops(file),
+			     struct carl9170_debugfs_fops, fops);
 
 	if (!dfops->write)
 		return -ENOSYS;
diff --git a/drivers/net/wireless/ath/carl9170/usb.c b/drivers/net/wireless/ath/carl9170/usb.c
index 76842e6ca38e..99ab20334d21 100644
--- a/drivers/net/wireless/ath/carl9170/usb.c
+++ b/drivers/net/wireless/ath/carl9170/usb.c
@@ -670,6 +670,7 @@ int carl9170_exec_cmd(struct ar9170 *ar, const enum carl9170_cmd_oids cmd,
 	ar->readlen = outlen;
 	spin_unlock_bh(&ar->cmd_lock);
 
+	reinit_completion(&ar->cmd_wait);
 	err = __carl9170_exec_cmd(ar, &ar->cmd, false);
 
 	if (!(cmd & CARL9170_CMD_ASYNC_FLAG)) {
@@ -778,10 +779,7 @@ void carl9170_usb_stop(struct ar9170 *ar)
 	spin_lock_bh(&ar->cmd_lock);
 	ar->readlen = 0;
 	spin_unlock_bh(&ar->cmd_lock);
-	complete_all(&ar->cmd_wait);
-
-	/* This is required to prevent an early completion on _start */
-	reinit_completion(&ar->cmd_wait);
+	complete(&ar->cmd_wait);
 
 	/*
 	 * Note:
diff --git a/drivers/net/wireless/ath/dfs_pattern_detector.c b/drivers/net/wireless/ath/dfs_pattern_detector.c
index 2f8136d50f78..4100ffd42a43 100644
--- a/drivers/net/wireless/ath/dfs_pattern_detector.c
+++ b/drivers/net/wireless/ath/dfs_pattern_detector.c
@@ -338,7 +338,7 @@ static bool dpd_set_domain(struct dfs_pattern_detector *dpd,
 	return true;
 }
 
-static struct dfs_pattern_detector default_dpd = {
+static const struct dfs_pattern_detector default_dpd = {
 	.exit		= dpd_exit,
 	.set_dfs_domain	= dpd_set_domain,
 	.add_pulse	= dpd_add_pulse,
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index f0e1175fb76a..d117240d9a73 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -354,10 +354,13 @@ static int wil_cfg80211_scan(struct wiphy *wiphy,
 	wil_dbg_misc(wil, "%s(), wdev=0x%p iftype=%d\n",
 		     __func__, wdev, wdev->iftype);
 
+	mutex_lock(&wil->p2p_wdev_mutex);
 	if (wil->scan_request) {
 		wil_err(wil, "Already scanning\n");
+		mutex_unlock(&wil->p2p_wdev_mutex);
 		return -EAGAIN;
 	}
+	mutex_unlock(&wil->p2p_wdev_mutex);
 
 	/* check we are client side */
 	switch (wdev->iftype) {
@@ -760,14 +763,11 @@ static enum wmi_key_usage wil_detect_key_usage(struct wil6210_priv *wil,
 	return rc;
 }
 
-static struct wil_tid_crypto_rx_single *
-wil_find_crypto_ctx(struct wil6210_priv *wil, u8 key_index,
-		    enum wmi_key_usage key_usage, const u8 *mac_addr)
+static struct wil_sta_info *
+wil_find_sta_by_key_usage(struct wil6210_priv *wil,
+			  enum wmi_key_usage key_usage, const u8 *mac_addr)
 {
 	int cid = -EINVAL;
-	int tid = 0;
-	struct wil_sta_info *s;
-	struct wil_tid_crypto_rx *c;
 
 	if (key_usage == WMI_KEY_USE_TX_GROUP)
 		return NULL; /* not needed */
@@ -778,18 +778,72 @@ wil_find_crypto_ctx(struct wil6210_priv *wil, u8 key_index,
 	else if (key_usage == WMI_KEY_USE_RX_GROUP)
 		cid = wil_find_cid_by_idx(wil, 0);
 	if (cid < 0) {
-		wil_err(wil, "No CID for %pM %s[%d]\n", mac_addr,
-			key_usage_str[key_usage], key_index);
+		wil_err(wil, "No CID for %pM %s\n", mac_addr,
+			key_usage_str[key_usage]);
 		return ERR_PTR(cid);
 	}
 
-	s = &wil->sta[cid];
-	if (key_usage == WMI_KEY_USE_PAIRWISE)
-		c = &s->tid_crypto_rx[tid];
-	else
-		c = &s->group_crypto_rx;
+	return &wil->sta[cid];
+}
+
+static void wil_set_crypto_rx(u8 key_index, enum wmi_key_usage key_usage,
+			      struct wil_sta_info *cs,
+			      struct key_params *params)
+{
+	struct wil_tid_crypto_rx_single *cc;
+	int tid;
+
+	if (!cs)
+		return;
 
-	return &c->key_id[key_index];
+	switch (key_usage) {
+	case WMI_KEY_USE_PAIRWISE:
+		for (tid = 0; tid < WIL_STA_TID_NUM; tid++) {
+			cc = &cs->tid_crypto_rx[tid].key_id[key_index];
+			if (params->seq)
+				memcpy(cc->pn, params->seq,
+				       IEEE80211_GCMP_PN_LEN);
+			else
+				memset(cc->pn, 0, IEEE80211_GCMP_PN_LEN);
+			cc->key_set = true;
+		}
+		break;
+	case WMI_KEY_USE_RX_GROUP:
+		cc = &cs->group_crypto_rx.key_id[key_index];
+		if (params->seq)
+			memcpy(cc->pn, params->seq, IEEE80211_GCMP_PN_LEN);
+		else
+			memset(cc->pn, 0, IEEE80211_GCMP_PN_LEN);
+		cc->key_set = true;
+		break;
+	default:
+		break;
+	}
+}
+
+static void wil_del_rx_key(u8 key_index, enum wmi_key_usage key_usage,
+			   struct wil_sta_info *cs)
+{
+	struct wil_tid_crypto_rx_single *cc;
+	int tid;
+
+	if (!cs)
+		return;
+
+	switch (key_usage) {
+	case WMI_KEY_USE_PAIRWISE:
+		for (tid = 0; tid < WIL_STA_TID_NUM; tid++) {
+			cc = &cs->tid_crypto_rx[tid].key_id[key_index];
+			cc->key_set = false;
+		}
+		break;
+	case WMI_KEY_USE_RX_GROUP:
+		cc = &cs->group_crypto_rx.key_id[key_index];
+		cc->key_set = false;
+		break;
+	default:
+		break;
+	}
 }
 
 static int wil_cfg80211_add_key(struct wiphy *wiphy,
@@ -801,24 +855,26 @@ static int wil_cfg80211_add_key(struct wiphy *wiphy,
 	int rc;
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
 	enum wmi_key_usage key_usage = wil_detect_key_usage(wil, pairwise);
-	struct wil_tid_crypto_rx_single *cc = wil_find_crypto_ctx(wil,
-								  key_index,
-								  key_usage,
-								  mac_addr);
+	struct wil_sta_info *cs = wil_find_sta_by_key_usage(wil, key_usage,
+							    mac_addr);
+
+	if (!params) {
+		wil_err(wil, "NULL params\n");
+		return -EINVAL;
+	}
 
 	wil_dbg_misc(wil, "%s(%pM %s[%d] PN %*phN)\n", __func__,
 		     mac_addr, key_usage_str[key_usage], key_index,
 		     params->seq_len, params->seq);
 
-	if (IS_ERR(cc)) {
+	if (IS_ERR(cs)) {
 		wil_err(wil, "Not connected, %s(%pM %s[%d] PN %*phN)\n",
 			__func__, mac_addr, key_usage_str[key_usage], key_index,
 			params->seq_len, params->seq);
 		return -EINVAL;
 	}
 
-	if (cc)
-		cc->key_set = false;
+	wil_del_rx_key(key_index, key_usage, cs);
 
 	if (params->seq && params->seq_len != IEEE80211_GCMP_PN_LEN) {
 		wil_err(wil,
@@ -831,13 +887,8 @@ static int wil_cfg80211_add_key(struct wiphy *wiphy,
 
 	rc = wmi_add_cipher_key(wil, key_index, mac_addr, params->key_len,
 				params->key, key_usage);
-	if ((rc == 0) && cc) {
-		if (params->seq)
-			memcpy(cc->pn, params->seq, IEEE80211_GCMP_PN_LEN);
-		else
-			memset(cc->pn, 0, IEEE80211_GCMP_PN_LEN);
-		cc->key_set = true;
-	}
+	if (!rc)
+		wil_set_crypto_rx(key_index, key_usage, cs, params);
 
 	return rc;
 }
@@ -849,20 +900,18 @@ static int wil_cfg80211_del_key(struct wiphy *wiphy,
 {
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
 	enum wmi_key_usage key_usage = wil_detect_key_usage(wil, pairwise);
-	struct wil_tid_crypto_rx_single *cc = wil_find_crypto_ctx(wil,
-								  key_index,
-								  key_usage,
-								  mac_addr);
+	struct wil_sta_info *cs = wil_find_sta_by_key_usage(wil, key_usage,
+							    mac_addr);
 
 	wil_dbg_misc(wil, "%s(%pM %s[%d])\n", __func__, mac_addr,
 		     key_usage_str[key_usage], key_index);
 
-	if (IS_ERR(cc))
+	if (IS_ERR(cs))
 		wil_info(wil, "Not connected, %s(%pM %s[%d])\n", __func__,
 			 mac_addr, key_usage_str[key_usage], key_index);
 
-	if (!IS_ERR_OR_NULL(cc))
-		cc->key_set = false;
+	if (!IS_ERR_OR_NULL(cs))
+		wil_del_rx_key(key_index, key_usage, cs);
 
 	return wmi_del_cipher_key(wil, key_index, mac_addr, key_usage);
 }
@@ -1363,23 +1412,16 @@ static void wil_cfg80211_stop_p2p_device(struct wiphy *wiphy,
 					 struct wireless_dev *wdev)
 {
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
-	u8 started;
+	struct wil_p2p_info *p2p = &wil->p2p;
+
+	if (!p2p->p2p_dev_started)
+		return;
 
 	wil_dbg_misc(wil, "%s: entered\n", __func__);
 	mutex_lock(&wil->mutex);
-	started = wil_p2p_stop_discovery(wil);
-	if (started && wil->scan_request) {
-		struct cfg80211_scan_info info = {
-			.aborted = true,
-		};
-
-		cfg80211_scan_done(wil->scan_request, &info);
-		wil->scan_request = NULL;
-		wil->radio_wdev = wil->wdev;
-	}
+	wil_p2p_stop_radio_operations(wil);
+	p2p->p2p_dev_started = 0;
 	mutex_unlock(&wil->mutex);
-
-	wil->p2p.p2p_dev_started = 0;
 }
 
 static struct cfg80211_ops wil_cfg80211_ops = {
@@ -1464,14 +1506,8 @@ struct wireless_dev *wil_cfg80211_init(struct device *dev)
 	set_wiphy_dev(wdev->wiphy, dev);
 	wil_wiphy_init(wdev->wiphy);
 
-	rc = wiphy_register(wdev->wiphy);
-	if (rc < 0)
-		goto out_failed_reg;
-
 	return wdev;
 
-out_failed_reg:
-	wiphy_free(wdev->wiphy);
 out:
 	kfree(wdev);
 
@@ -1487,7 +1523,6 @@ void wil_wdev_free(struct wil6210_priv *wil)
 	if (!wdev)
 		return;
 
-	wiphy_unregister(wdev->wiphy);
 	wiphy_free(wdev->wiphy);
 	kfree(wdev);
 }
@@ -1498,11 +1533,11 @@ void wil_p2p_wdev_free(struct wil6210_priv *wil)
 
 	mutex_lock(&wil->p2p_wdev_mutex);
 	p2p_wdev = wil->p2p_wdev;
+	wil->p2p_wdev = NULL;
+	wil->radio_wdev = wil_to_wdev(wil);
+	mutex_unlock(&wil->p2p_wdev_mutex);
 	if (p2p_wdev) {
-		wil->p2p_wdev = NULL;
-		wil->radio_wdev = wil_to_wdev(wil);
 		cfg80211_unregister_wdev(p2p_wdev);
 		kfree(p2p_wdev);
 	}
-	mutex_unlock(&wil->p2p_wdev_mutex);
 }
diff --git a/drivers/net/wireless/ath/wil6210/debugfs.c b/drivers/net/wireless/ath/wil6210/debugfs.c
index a8098b406cc0..5e4058a4037b 100644
--- a/drivers/net/wireless/ath/wil6210/debugfs.c
+++ b/drivers/net/wireless/ath/wil6210/debugfs.c
@@ -1553,6 +1553,56 @@ static const struct file_operations fops_led_blink_time = {
 	.open  = simple_open,
 };
 
+/*---------FW capabilities------------*/
+static int wil_fw_capabilities_debugfs_show(struct seq_file *s, void *data)
+{
+	struct wil6210_priv *wil = s->private;
+
+	seq_printf(s, "fw_capabilities : %*pb\n", WMI_FW_CAPABILITY_MAX,
+		   wil->fw_capabilities);
+
+	return 0;
+}
+
+static int wil_fw_capabilities_seq_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, wil_fw_capabilities_debugfs_show,
+			   inode->i_private);
+}
+
+static const struct file_operations fops_fw_capabilities = {
+	.open		= wil_fw_capabilities_seq_open,
+	.release	= single_release,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+};
+
+/*---------FW version------------*/
+static int wil_fw_version_debugfs_show(struct seq_file *s, void *data)
+{
+	struct wil6210_priv *wil = s->private;
+
+	if (wil->fw_version[0])
+		seq_printf(s, "%s\n", wil->fw_version);
+	else
+		seq_puts(s, "N/A\n");
+
+	return 0;
+}
+
+static int wil_fw_version_seq_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, wil_fw_version_debugfs_show,
+			   inode->i_private);
+}
+
+static const struct file_operations fops_fw_version = {
+	.open		= wil_fw_version_seq_open,
+	.release	= single_release,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+};
+
 /*----------------*/
 static void wil6210_debugfs_init_blobs(struct wil6210_priv *wil,
 				       struct dentry *dbg)
@@ -1603,6 +1653,8 @@ static const struct {
 	{"recovery",	S_IRUGO | S_IWUSR,	&fops_recovery},
 	{"led_cfg",	S_IRUGO | S_IWUSR,	&fops_led_cfg},
 	{"led_blink_time",	S_IRUGO | S_IWUSR,	&fops_led_blink_time},
+	{"fw_capabilities",	S_IRUGO,	&fops_fw_capabilities},
+	{"fw_version",	S_IRUGO,		&fops_fw_version},
 };
 
 static void wil6210_debugfs_init_files(struct wil6210_priv *wil,
@@ -1643,7 +1695,6 @@ static void wil6210_debugfs_init_isr(struct wil6210_priv *wil,
 static const struct dbg_off dbg_wil_off[] = {
 	WIL_FIELD(privacy,	S_IRUGO,		doff_u32),
 	WIL_FIELD(status[0],	S_IRUGO | S_IWUSR,	doff_ulong),
-	WIL_FIELD(fw_version,	S_IRUGO,		doff_u32),
 	WIL_FIELD(hw_version,	S_IRUGO,		doff_x32),
 	WIL_FIELD(recovery_count, S_IRUGO,		doff_u32),
 	WIL_FIELD(ap_isolate,	S_IRUGO,		doff_u32),
diff --git a/drivers/net/wireless/ath/wil6210/fw.h b/drivers/net/wireless/ath/wil6210/fw.h
index 7a2c6c129ad5..2f2b910501ba 100644
--- a/drivers/net/wireless/ath/wil6210/fw.h
+++ b/drivers/net/wireless/ath/wil6210/fw.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014,2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -58,6 +58,15 @@ struct wil_fw_record_comment { /* type == wil_fw_type_comment */
 	u8 data[0]; /* free-form data [data_size], see above */
 } __packed;
 
+/* FW capabilities encoded inside a comment record */
+#define WIL_FW_CAPABILITIES_MAGIC (0xabcddcba)
+struct wil_fw_record_capabilities { /* type == wil_fw_type_comment */
+	/* identifies capabilities record */
+	__le32 magic;
+	/* capabilities (variable size), see enum wmi_fw_capability */
+	u8 capabilities[0];
+};
+
 /* perform action
  * data_size = @head.size - offsetof(struct wil_fw_record_action, data)
  */
@@ -93,6 +102,9 @@ struct wil_fw_record_verify { /* type == wil_fw_verify */
 /* file header
  * First record of every file
  */
+/* the FW version prefix in the comment */
+#define WIL_FW_VERSION_PREFIX "FW version: "
+#define WIL_FW_VERSION_PREFIX_LEN (sizeof(WIL_FW_VERSION_PREFIX) - 1)
 struct wil_fw_record_file_header {
 	__le32 signature ; /* Wilocity signature */
 	__le32 reserved;
diff --git a/drivers/net/wireless/ath/wil6210/fw_inc.c b/drivers/net/wireless/ath/wil6210/fw_inc.c
index d30657ee7e83..8f40eb301924 100644
--- a/drivers/net/wireless/ath/wil6210/fw_inc.c
+++ b/drivers/net/wireless/ath/wil6210/fw_inc.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -118,6 +118,12 @@ static int wil_fw_verify(struct wil6210_priv *wil, const u8 *data, size_t size)
 	return (int)dlen;
 }
 
+static int fw_ignore_section(struct wil6210_priv *wil, const void *data,
+			     size_t size)
+{
+	return 0;
+}
+
 static int fw_handle_comment(struct wil6210_priv *wil, const void *data,
 			     size_t size)
 {
@@ -126,6 +132,27 @@ static int fw_handle_comment(struct wil6210_priv *wil, const void *data,
 	return 0;
 }
 
+static int
+fw_handle_capabilities(struct wil6210_priv *wil, const void *data,
+		       size_t size)
+{
+	const struct wil_fw_record_capabilities *rec = data;
+	size_t capa_size;
+
+	if (size < sizeof(*rec) ||
+	    le32_to_cpu(rec->magic) != WIL_FW_CAPABILITIES_MAGIC)
+		return 0;
+
+	capa_size = size - offsetof(struct wil_fw_record_capabilities,
+				    capabilities);
+	bitmap_zero(wil->fw_capabilities, WMI_FW_CAPABILITY_MAX);
+	memcpy(wil->fw_capabilities, rec->capabilities,
+	       min(sizeof(wil->fw_capabilities), capa_size));
+	wil_hex_dump_fw("CAPA", DUMP_PREFIX_OFFSET, 16, 1,
+			rec->capabilities, capa_size, false);
+	return 0;
+}
+
 static int fw_handle_data(struct wil6210_priv *wil, const void *data,
 			  size_t size)
 {
@@ -196,6 +223,13 @@ static int fw_handle_file_header(struct wil6210_priv *wil, const void *data,
 	wil_hex_dump_fw("", DUMP_PREFIX_OFFSET, 16, 1, d->comment,
 			sizeof(d->comment), true);
 
+	if (!memcmp(d->comment, WIL_FW_VERSION_PREFIX,
+		    WIL_FW_VERSION_PREFIX_LEN))
+		memcpy(wil->fw_version,
+		       d->comment + WIL_FW_VERSION_PREFIX_LEN,
+		       min(sizeof(d->comment) - WIL_FW_VERSION_PREFIX_LEN,
+			   sizeof(wil->fw_version) - 1));
+
 	return 0;
 }
 
@@ -383,42 +417,51 @@ static int fw_handle_gateway_data4(struct wil6210_priv *wil, const void *data,
 
 static const struct {
 	int type;
-	int (*handler)(struct wil6210_priv *wil, const void *data, size_t size);
+	int (*load_handler)(struct wil6210_priv *wil, const void *data,
+			    size_t size);
+	int (*parse_handler)(struct wil6210_priv *wil, const void *data,
+			     size_t size);
 } wil_fw_handlers[] = {
-	{wil_fw_type_comment, fw_handle_comment},
-	{wil_fw_type_data, fw_handle_data},
-	{wil_fw_type_fill, fw_handle_fill},
+	{wil_fw_type_comment, fw_handle_comment, fw_handle_capabilities},
+	{wil_fw_type_data, fw_handle_data, fw_ignore_section},
+	{wil_fw_type_fill, fw_handle_fill, fw_ignore_section},
 	/* wil_fw_type_action */
 	/* wil_fw_type_verify */
-	{wil_fw_type_file_header, fw_handle_file_header},
-	{wil_fw_type_direct_write, fw_handle_direct_write},
-	{wil_fw_type_gateway_data, fw_handle_gateway_data},
-	{wil_fw_type_gateway_data4, fw_handle_gateway_data4},
+	{wil_fw_type_file_header, fw_handle_file_header,
+		fw_handle_file_header},
+	{wil_fw_type_direct_write, fw_handle_direct_write, fw_ignore_section},
+	{wil_fw_type_gateway_data, fw_handle_gateway_data, fw_ignore_section},
+	{wil_fw_type_gateway_data4, fw_handle_gateway_data4,
+		fw_ignore_section},
 };
 
 static int wil_fw_handle_record(struct wil6210_priv *wil, int type,
-				const void *data, size_t size)
+				const void *data, size_t size, bool load)
 {
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(wil_fw_handlers); i++) {
+	for (i = 0; i < ARRAY_SIZE(wil_fw_handlers); i++)
 		if (wil_fw_handlers[i].type == type)
-			return wil_fw_handlers[i].handler(wil, data, size);
-	}
+			return load ?
+				wil_fw_handlers[i].load_handler(
+					wil, data, size) :
+				wil_fw_handlers[i].parse_handler(
+					wil, data, size);
 
 	wil_err_fw(wil, "unknown record type: %d\n", type);
 	return -EINVAL;
 }
 
 /**
- * wil_fw_load - load FW into device
- *
- * Load the FW and uCode code and data to the corresponding device
- * memory regions
+ * wil_fw_process - process section from FW file
+ * if load is true: Load the FW and uCode code and data to the
+ * corresponding device memory regions,
+ * otherwise only parse and look for capabilities
  *
  * Return error code
  */
-static int wil_fw_load(struct wil6210_priv *wil, const void *data, size_t size)
+static int wil_fw_process(struct wil6210_priv *wil, const void *data,
+			  size_t size, bool load)
 {
 	int rc = 0;
 	const struct wil_fw_record_head *hdr;
@@ -437,7 +480,7 @@ static int wil_fw_load(struct wil6210_priv *wil, const void *data, size_t size)
 			return -EINVAL;
 		}
 		rc = wil_fw_handle_record(wil, le16_to_cpu(hdr->type),
-					  &hdr[1], hdr_sz);
+					  &hdr[1], hdr_sz, load);
 		if (rc)
 			return rc;
 	}
@@ -456,13 +499,16 @@ static int wil_fw_load(struct wil6210_priv *wil, const void *data, size_t size)
 }
 
 /**
- * wil_request_firmware - Request firmware and load to device
+ * wil_request_firmware - Request firmware
  *
- * Request firmware image from the file and load it to device
+ * Request firmware image from the file
+ * If load is true, load firmware to device, otherwise
+ * only parse and extract capabilities
  *
  * Return error code
  */
-int wil_request_firmware(struct wil6210_priv *wil, const char *name)
+int wil_request_firmware(struct wil6210_priv *wil, const char *name,
+			 bool load)
 {
 	int rc, rc1;
 	const struct firmware *fw;
@@ -482,7 +528,7 @@ int wil_request_firmware(struct wil6210_priv *wil, const char *name)
 			rc = rc1;
 			goto out;
 		}
-		rc = wil_fw_load(wil, d, rc1);
+		rc = wil_fw_process(wil, d, rc1, load);
 		if (rc < 0)
 			goto out;
 	}
diff --git a/drivers/net/wireless/ath/wil6210/interrupt.c b/drivers/net/wireless/ath/wil6210/interrupt.c
index 011e7412dcc0..64046e0bd0a2 100644
--- a/drivers/net/wireless/ath/wil6210/interrupt.c
+++ b/drivers/net/wireless/ath/wil6210/interrupt.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012-2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2012-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -101,7 +101,7 @@ static void wil6210_mask_irq_misc(struct wil6210_priv *wil, bool mask_halp)
 	      mask_halp ? WIL6210_IRQ_DISABLE : WIL6210_IRQ_DISABLE_NO_HALP);
 }
 
-static void wil6210_mask_halp(struct wil6210_priv *wil)
+void wil6210_mask_halp(struct wil6210_priv *wil)
 {
 	wil_dbg_irq(wil, "%s()\n", __func__);
 
@@ -503,6 +503,13 @@ static int wil6210_debug_irq_mask(struct wil6210_priv *wil, u32 pseudo_cause)
 				offsetof(struct RGF_ICR, ICR));
 		u32 imv_misc = wil_r(wil, RGF_DMA_EP_MISC_ICR +
 				     offsetof(struct RGF_ICR, IMV));
+
+		/* HALP interrupt can be unmasked when misc interrupts are
+		 * masked
+		 */
+		if (icr_misc & BIT_DMA_EP_MISC_ICR_HALP)
+			return 0;
+
 		wil_err(wil, "IRQ when it should be masked: pseudo 0x%08x\n"
 				"Rx   icm:icr:imv 0x%08x 0x%08x 0x%08x\n"
 				"Tx   icm:icr:imv 0x%08x 0x%08x 0x%08x\n"
@@ -592,7 +599,7 @@ void wil6210_clear_irq(struct wil6210_priv *wil)
 
 void wil6210_set_halp(struct wil6210_priv *wil)
 {
-	wil_dbg_misc(wil, "%s()\n", __func__);
+	wil_dbg_irq(wil, "%s()\n", __func__);
 
 	wil_w(wil, RGF_DMA_EP_MISC_ICR + offsetof(struct RGF_ICR, ICS),
 	      BIT_DMA_EP_MISC_ICR_HALP);
@@ -600,7 +607,7 @@ void wil6210_set_halp(struct wil6210_priv *wil)
 
 void wil6210_clear_halp(struct wil6210_priv *wil)
 {
-	wil_dbg_misc(wil, "%s()\n", __func__);
+	wil_dbg_irq(wil, "%s()\n", __func__);
 
 	wil_w(wil, RGF_DMA_EP_MISC_ICR + offsetof(struct RGF_ICR, ICR),
 	      BIT_DMA_EP_MISC_ICR_HALP);
diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c
index 4bc92e54984a..e7130b54d1d8 100644
--- a/drivers/net/wireless/ath/wil6210/main.c
+++ b/drivers/net/wireless/ath/wil6210/main.c
@@ -232,6 +232,9 @@ static void _wil6210_disconnect(struct wil6210_priv *wil, const u8 *bssid,
 	struct net_device *ndev = wil_to_ndev(wil);
 	struct wireless_dev *wdev = wil->wdev;
 
+	if (unlikely(!ndev))
+		return;
+
 	might_sleep();
 	wil_info(wil, "%s(bssid=%pM, reason=%d, ev%s)\n", __func__, bssid,
 		 reason_code, from_event ? "+" : "-");
@@ -849,6 +852,7 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 	bitmap_zero(wil->status, wil_status_last);
 	mutex_unlock(&wil->wmi_mutex);
 
+	mutex_lock(&wil->p2p_wdev_mutex);
 	if (wil->scan_request) {
 		struct cfg80211_scan_info info = {
 			.aborted = true,
@@ -860,6 +864,7 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 		cfg80211_scan_done(wil->scan_request, &info);
 		wil->scan_request = NULL;
 	}
+	mutex_unlock(&wil->p2p_wdev_mutex);
 
 	wil_mask_irq(wil);
 
@@ -888,11 +893,12 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 			 WIL_FW2_NAME);
 
 		wil_halt_cpu(wil);
+		memset(wil->fw_version, 0, sizeof(wil->fw_version));
 		/* Loading f/w from the file */
-		rc = wil_request_firmware(wil, WIL_FW_NAME);
+		rc = wil_request_firmware(wil, WIL_FW_NAME, true);
 		if (rc)
 			return rc;
-		rc = wil_request_firmware(wil, WIL_FW2_NAME);
+		rc = wil_request_firmware(wil, WIL_FW2_NAME, true);
 		if (rc)
 			return rc;
 
@@ -1035,10 +1041,10 @@ int wil_up(struct wil6210_priv *wil)
 
 int __wil_down(struct wil6210_priv *wil)
 {
-	int rc;
-
 	WARN_ON(!mutex_is_locked(&wil->mutex));
 
+	set_bit(wil_status_resetting, wil->status);
+
 	if (wil->platform_ops.bus_request)
 		wil->platform_ops.bus_request(wil->platform_handle, 0);
 
@@ -1050,8 +1056,9 @@ int __wil_down(struct wil6210_priv *wil)
 	}
 	wil_enable_irq(wil);
 
-	(void)wil_p2p_stop_discovery(wil);
+	wil_p2p_stop_radio_operations(wil);
 
+	mutex_lock(&wil->p2p_wdev_mutex);
 	if (wil->scan_request) {
 		struct cfg80211_scan_info info = {
 			.aborted = true,
@@ -1063,18 +1070,7 @@ int __wil_down(struct wil6210_priv *wil)
 		cfg80211_scan_done(wil->scan_request, &info);
 		wil->scan_request = NULL;
 	}
-
-	if (test_bit(wil_status_fwconnected, wil->status) ||
-	    test_bit(wil_status_fwconnecting, wil->status)) {
-
-		mutex_unlock(&wil->mutex);
-		rc = wmi_call(wil, WMI_DISCONNECT_CMDID, NULL, 0,
-			      WMI_DISCONNECT_EVENTID, NULL, 0,
-			      WIL6210_DISCONNECT_TO_MS);
-		mutex_lock(&wil->mutex);
-		if (rc)
-			wil_err(wil, "timeout waiting for disconnect\n");
-	}
+	mutex_unlock(&wil->p2p_wdev_mutex);
 
 	wil_reset(wil, false);
 
@@ -1118,23 +1114,26 @@ void wil_halp_vote(struct wil6210_priv *wil)
 
 	mutex_lock(&wil->halp.lock);
 
-	wil_dbg_misc(wil, "%s: start, HALP ref_cnt (%d)\n", __func__,
-		     wil->halp.ref_cnt);
+	wil_dbg_irq(wil, "%s: start, HALP ref_cnt (%d)\n", __func__,
+		    wil->halp.ref_cnt);
 
 	if (++wil->halp.ref_cnt == 1) {
 		wil6210_set_halp(wil);
 		rc = wait_for_completion_timeout(&wil->halp.comp, to_jiffies);
-		if (!rc)
+		if (!rc) {
 			wil_err(wil, "%s: HALP vote timed out\n", __func__);
-		else
-			wil_dbg_misc(wil,
-				     "%s: HALP vote completed after %d ms\n",
-				     __func__,
-				     jiffies_to_msecs(to_jiffies - rc));
+			/* Mask HALP as done in case the interrupt is raised */
+			wil6210_mask_halp(wil);
+		} else {
+			wil_dbg_irq(wil,
+				    "%s: HALP vote completed after %d ms\n",
+				    __func__,
+				    jiffies_to_msecs(to_jiffies - rc));
+		}
 	}
 
-	wil_dbg_misc(wil, "%s: end, HALP ref_cnt (%d)\n", __func__,
-		     wil->halp.ref_cnt);
+	wil_dbg_irq(wil, "%s: end, HALP ref_cnt (%d)\n", __func__,
+		    wil->halp.ref_cnt);
 
 	mutex_unlock(&wil->halp.lock);
 }
@@ -1145,16 +1144,16 @@ void wil_halp_unvote(struct wil6210_priv *wil)
 
 	mutex_lock(&wil->halp.lock);
 
-	wil_dbg_misc(wil, "%s: start, HALP ref_cnt (%d)\n", __func__,
-		     wil->halp.ref_cnt);
+	wil_dbg_irq(wil, "%s: start, HALP ref_cnt (%d)\n", __func__,
+		    wil->halp.ref_cnt);
 
 	if (--wil->halp.ref_cnt == 0) {
 		wil6210_clear_halp(wil);
-		wil_dbg_misc(wil, "%s: HALP unvote\n", __func__);
+		wil_dbg_irq(wil, "%s: HALP unvote\n", __func__);
 	}
 
-	wil_dbg_misc(wil, "%s: end, HALP ref_cnt (%d)\n", __func__,
-		     wil->halp.ref_cnt);
+	wil_dbg_irq(wil, "%s: end, HALP ref_cnt (%d)\n", __func__,
+		    wil->halp.ref_cnt);
 
 	mutex_unlock(&wil->halp.lock);
 }
diff --git a/drivers/net/wireless/ath/wil6210/netdev.c b/drivers/net/wireless/ath/wil6210/netdev.c
index 098409753d5b..61de5e9f8ef0 100644
--- a/drivers/net/wireless/ath/wil6210/netdev.c
+++ b/drivers/net/wireless/ath/wil6210/netdev.c
@@ -179,13 +179,6 @@ void *wil_if_alloc(struct device *dev)
 	SET_NETDEV_DEV(ndev, wiphy_dev(wdev->wiphy));
 	wdev->netdev = ndev;
 
-	netif_napi_add(ndev, &wil->napi_rx, wil6210_netdev_poll_rx,
-		       WIL6210_NAPI_BUDGET);
-	netif_tx_napi_add(ndev, &wil->napi_tx, wil6210_netdev_poll_tx,
-		       WIL6210_NAPI_BUDGET);
-
-	netif_tx_stop_all_queues(ndev);
-
 	return wil;
 
  out_priv:
@@ -216,25 +209,48 @@ void wil_if_free(struct wil6210_priv *wil)
 
 int wil_if_add(struct wil6210_priv *wil)
 {
+	struct wireless_dev *wdev = wil_to_wdev(wil);
+	struct wiphy *wiphy = wdev->wiphy;
 	struct net_device *ndev = wil_to_ndev(wil);
 	int rc;
 
-	wil_dbg_misc(wil, "%s()\n", __func__);
+	wil_dbg_misc(wil, "entered");
+
+	strlcpy(wiphy->fw_version, wil->fw_version, sizeof(wiphy->fw_version));
+
+	rc = wiphy_register(wiphy);
+	if (rc < 0) {
+		wil_err(wil, "failed to register wiphy, err %d\n", rc);
+		return rc;
+	}
+
+	netif_napi_add(ndev, &wil->napi_rx, wil6210_netdev_poll_rx,
+		       WIL6210_NAPI_BUDGET);
+	netif_tx_napi_add(ndev, &wil->napi_tx, wil6210_netdev_poll_tx,
+			  WIL6210_NAPI_BUDGET);
+
+	netif_tx_stop_all_queues(ndev);
 
 	rc = register_netdev(ndev);
 	if (rc < 0) {
 		dev_err(&ndev->dev, "Failed to register netdev: %d\n", rc);
-		return rc;
+		goto out_wiphy;
 	}
 
 	return 0;
+
+out_wiphy:
+	wiphy_unregister(wdev->wiphy);
+	return rc;
 }
 
 void wil_if_remove(struct wil6210_priv *wil)
 {
 	struct net_device *ndev = wil_to_ndev(wil);
+	struct wireless_dev *wdev = wil_to_wdev(wil);
 
 	wil_dbg_misc(wil, "%s()\n", __func__);
 
 	unregister_netdev(ndev);
+	wiphy_unregister(wdev->wiphy);
 }
diff --git a/drivers/net/wireless/ath/wil6210/p2p.c b/drivers/net/wireless/ath/wil6210/p2p.c
index e0f8aa0ebfac..4087785d3090 100644
--- a/drivers/net/wireless/ath/wil6210/p2p.c
+++ b/drivers/net/wireless/ath/wil6210/p2p.c
@@ -263,3 +263,49 @@ void wil_p2p_search_expired(struct work_struct *work)
 		mutex_unlock(&wil->p2p_wdev_mutex);
 	}
 }
+
+void wil_p2p_stop_radio_operations(struct wil6210_priv *wil)
+{
+	struct wil_p2p_info *p2p = &wil->p2p;
+	struct cfg80211_scan_info info = {
+		.aborted = true,
+	};
+
+	lockdep_assert_held(&wil->mutex);
+
+	mutex_lock(&wil->p2p_wdev_mutex);
+
+	if (wil->radio_wdev != wil->p2p_wdev)
+		goto out;
+
+	if (!p2p->discovery_started) {
+		/* Regular scan on the p2p device */
+		if (wil->scan_request &&
+		    wil->scan_request->wdev == wil->p2p_wdev) {
+			cfg80211_scan_done(wil->scan_request, &info);
+			wil->scan_request = NULL;
+		}
+		goto out;
+	}
+
+	/* Search or listen on p2p device */
+	mutex_unlock(&wil->p2p_wdev_mutex);
+	wil_p2p_stop_discovery(wil);
+	mutex_lock(&wil->p2p_wdev_mutex);
+
+	if (wil->scan_request) {
+		/* search */
+		cfg80211_scan_done(wil->scan_request, &info);
+		wil->scan_request = NULL;
+	} else {
+		/* listen */
+		cfg80211_remain_on_channel_expired(wil->radio_wdev,
+						   p2p->cookie,
+						   &p2p->listen_chan,
+						   GFP_KERNEL);
+	}
+
+out:
+	wil->radio_wdev = wil->wdev;
+	mutex_unlock(&wil->p2p_wdev_mutex);
+}
diff --git a/drivers/net/wireless/ath/wil6210/pcie_bus.c b/drivers/net/wireless/ath/wil6210/pcie_bus.c
index 7b5c4222bc33..44746ca0d2e6 100644
--- a/drivers/net/wireless/ath/wil6210/pcie_bus.c
+++ b/drivers/net/wireless/ath/wil6210/pcie_bus.c
@@ -20,6 +20,7 @@
 #include <linux/interrupt.h>
 #include <linux/suspend.h>
 #include "wil6210.h"
+#include <linux/rtnetlink.h>
 
 static bool use_msi = true;
 module_param(use_msi, bool, S_IRUGO);
@@ -38,6 +39,7 @@ void wil_set_capabilities(struct wil6210_priv *wil)
 	u32 rev_id = wil_r(wil, RGF_USER_JTAG_DEV_ID);
 
 	bitmap_zero(wil->hw_capabilities, hw_capability_last);
+	bitmap_zero(wil->fw_capabilities, WMI_FW_CAPABILITY_MAX);
 
 	switch (rev_id) {
 	case JTAG_DEV_ID_SPARROW_B0:
@@ -51,6 +53,9 @@ void wil_set_capabilities(struct wil6210_priv *wil)
 	}
 
 	wil_info(wil, "Board hardware is %s\n", wil->hw_name);
+
+	/* extract FW capabilities from file without loading the FW */
+	wil_request_firmware(wil, WIL_FW_NAME, false);
 }
 
 void wil_disable_irq(struct wil6210_priv *wil)
@@ -293,6 +298,9 @@ static void wil_pcie_remove(struct pci_dev *pdev)
 #endif /* CONFIG_PM */
 
 	wil6210_debugfs_remove(wil);
+	rtnl_lock();
+	wil_p2p_wdev_free(wil);
+	rtnl_unlock();
 	wil_if_remove(wil);
 	wil_if_pcie_disable(wil);
 	pci_iounmap(pdev, csr);
@@ -300,7 +308,6 @@ static void wil_pcie_remove(struct pci_dev *pdev)
 	pci_disable_device(pdev);
 	if (wil->platform_ops.uninit)
 		wil->platform_ops.uninit(wil->platform_handle);
-	wil_p2p_wdev_free(wil);
 	wil_if_free(wil);
 }
 
diff --git a/drivers/net/wireless/ath/wil6210/txrx.c b/drivers/net/wireless/ath/wil6210/txrx.c
index f2f6a404d3d1..4c38520d4dd2 100644
--- a/drivers/net/wireless/ath/wil6210/txrx.c
+++ b/drivers/net/wireless/ath/wil6210/txrx.c
@@ -873,9 +873,12 @@ int wil_vring_init_tx(struct wil6210_priv *wil, int id, int size,
 		rc = -EINVAL;
 		goto out_free;
 	}
-	vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
 
+	spin_lock_bh(&txdata->lock);
+	vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
 	txdata->enabled = 1;
+	spin_unlock_bh(&txdata->lock);
+
 	if (txdata->dot1x_open && (agg_wsize >= 0))
 		wil_addba_tx_request(wil, id, agg_wsize);
 
@@ -950,9 +953,11 @@ int wil_vring_init_bcast(struct wil6210_priv *wil, int id, int size)
 		rc = -EINVAL;
 		goto out_free;
 	}
-	vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
 
+	spin_lock_bh(&txdata->lock);
+	vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
 	txdata->enabled = 1;
+	spin_unlock_bh(&txdata->lock);
 
 	return 0;
  out_free:
diff --git a/drivers/net/wireless/ath/wil6210/wil6210.h b/drivers/net/wireless/ath/wil6210/wil6210.h
index ecab4af90602..a949cd62bc4e 100644
--- a/drivers/net/wireless/ath/wil6210/wil6210.h
+++ b/drivers/net/wireless/ath/wil6210/wil6210.h
@@ -17,6 +17,7 @@
 #ifndef __WIL6210_H__
 #define __WIL6210_H__
 
+#include <linux/etherdevice.h>
 #include <linux/netdevice.h>
 #include <linux/wireless.h>
 #include <net/cfg80211.h>
@@ -576,10 +577,11 @@ struct wil6210_priv {
 	struct wireless_dev *wdev;
 	void __iomem *csr;
 	DECLARE_BITMAP(status, wil_status_last);
-	u32 fw_version;
+	u8 fw_version[ETHTOOL_FWVERS_LEN];
 	u32 hw_version;
 	const char *hw_name;
 	DECLARE_BITMAP(hw_capabilities, hw_capability_last);
+	DECLARE_BITMAP(fw_capabilities, WMI_FW_CAPABILITY_MAX);
 	u8 n_mids; /* number of additional MIDs as reported by FW */
 	u32 recovery_count; /* num of FW recovery attempts in a short time */
 	u32 recovery_state; /* FW recovery state machine */
@@ -657,7 +659,7 @@ struct wil6210_priv {
 
 	/* P2P_DEVICE vif */
 	struct wireless_dev *p2p_wdev;
-	struct mutex p2p_wdev_mutex; /* protect @p2p_wdev */
+	struct mutex p2p_wdev_mutex; /* protect @p2p_wdev and @scan_request */
 	struct wireless_dev *radio_wdev;
 
 	/* High Access Latency Policy voting */
@@ -828,6 +830,7 @@ void wil_unmask_irq(struct wil6210_priv *wil);
 void wil_configure_interrupt_moderation(struct wil6210_priv *wil);
 void wil_disable_irq(struct wil6210_priv *wil);
 void wil_enable_irq(struct wil6210_priv *wil);
+void wil6210_mask_halp(struct wil6210_priv *wil);
 
 /* P2P */
 bool wil_p2p_is_social_scan(struct cfg80211_scan_request *request);
@@ -840,6 +843,7 @@ u8 wil_p2p_stop_discovery(struct wil6210_priv *wil);
 int wil_p2p_cancel_listen(struct wil6210_priv *wil, u64 cookie);
 void wil_p2p_listen_expired(struct work_struct *work);
 void wil_p2p_search_expired(struct work_struct *work);
+void wil_p2p_stop_radio_operations(struct wil6210_priv *wil);
 
 /* WMI for P2P */
 int wmi_p2p_cfg(struct wil6210_priv *wil, int channel, int bi);
@@ -893,7 +897,8 @@ void wil6210_unmask_irq_rx(struct wil6210_priv *wil);
 int wil_iftype_nl2wmi(enum nl80211_iftype type);
 
 int wil_ioctl(struct wil6210_priv *wil, void __user *data, int cmd);
-int wil_request_firmware(struct wil6210_priv *wil, const char *name);
+int wil_request_firmware(struct wil6210_priv *wil, const char *name,
+			 bool load);
 
 int wil_can_suspend(struct wil6210_priv *wil, bool is_runtime);
 int wil_suspend(struct wil6210_priv *wil, bool is_runtime);
diff --git a/drivers/net/wireless/ath/wil6210/wmi.c b/drivers/net/wireless/ath/wil6210/wmi.c
index 4d92541913c0..fae4f1285d08 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.c
+++ b/drivers/net/wireless/ath/wil6210/wmi.c
@@ -312,14 +312,14 @@ static void wmi_evt_ready(struct wil6210_priv *wil, int id, void *d, int len)
 	struct wireless_dev *wdev = wil->wdev;
 	struct wmi_ready_event *evt = d;
 
-	wil->fw_version = le32_to_cpu(evt->sw_version);
 	wil->n_mids = evt->numof_additional_mids;
 
-	wil_info(wil, "FW ver. %d; MAC %pM; %d MID's\n", wil->fw_version,
+	wil_info(wil, "FW ver. %s(SW %d); MAC %pM; %d MID's\n",
+		 wil->fw_version, le32_to_cpu(evt->sw_version),
 		 evt->mac, wil->n_mids);
 	/* ignore MAC address, we already have it from the boot loader */
-	snprintf(wdev->wiphy->fw_version, sizeof(wdev->wiphy->fw_version),
-		 "%d", wil->fw_version);
+	strlcpy(wdev->wiphy->fw_version, wil->fw_version,
+		sizeof(wdev->wiphy->fw_version));
 
 	wil_set_recovery_state(wil, fw_recovery_idle);
 	set_bit(wil_status_fwready, wil->status);
@@ -424,6 +424,7 @@ static void wmi_evt_tx_mgmt(struct wil6210_priv *wil, int id, void *d, int len)
 static void wmi_evt_scan_complete(struct wil6210_priv *wil, int id,
 				  void *d, int len)
 {
+	mutex_lock(&wil->p2p_wdev_mutex);
 	if (wil->scan_request) {
 		struct wmi_scan_complete_event *data = d;
 		struct cfg80211_scan_info info = {
@@ -435,14 +436,13 @@ static void wmi_evt_scan_complete(struct wil6210_priv *wil, int id,
 			     wil->scan_request, info.aborted);
 
 		del_timer_sync(&wil->scan_timer);
-		mutex_lock(&wil->p2p_wdev_mutex);
 		cfg80211_scan_done(wil->scan_request, &info);
 		wil->radio_wdev = wil->wdev;
-		mutex_unlock(&wil->p2p_wdev_mutex);
 		wil->scan_request = NULL;
 	} else {
 		wil_err(wil, "SCAN_COMPLETE while not scanning\n");
 	}
+	mutex_unlock(&wil->p2p_wdev_mutex);
 }
 
 static void wmi_evt_connect(struct wil6210_priv *wil, int id, void *d, int len)
diff --git a/drivers/net/wireless/ath/wil6210/wmi.h b/drivers/net/wireless/ath/wil6210/wmi.h
index 685fe0ddea26..f430e8a80603 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.h
+++ b/drivers/net/wireless/ath/wil6210/wmi.h
@@ -46,6 +46,16 @@ enum wmi_mid {
 	MID_BROADCAST		= 0xFF,
 };
 
+/* FW capability IDs
+ * Each ID maps to a bit in a 32-bit bitmask value provided by the FW to
+ * the host
+ */
+enum wmi_fw_capability {
+	WMI_FW_CAPABILITY_FTM		= 0,
+	WMI_FW_CAPABILITY_PS_CONFIG	= 1,
+	WMI_FW_CAPABILITY_MAX,
+};
+
 /* WMI_CMD_HDR */
 struct wmi_cmd_hdr {
 	u8 mid;
@@ -120,6 +130,8 @@ enum wmi_command_id {
 	WMI_BF_SM_MGMT_CMDID			= 0x838,
 	WMI_BF_RXSS_MGMT_CMDID			= 0x839,
 	WMI_BF_TRIG_CMDID			= 0x83A,
+	WMI_LINK_MAINTAIN_CFG_WRITE_CMDID	= 0x842,
+	WMI_LINK_MAINTAIN_CFG_READ_CMDID	= 0x843,
 	WMI_SET_SECTORS_CMDID			= 0x849,
 	WMI_MAINTAIN_PAUSE_CMDID		= 0x850,
 	WMI_MAINTAIN_RESUME_CMDID		= 0x851,
@@ -134,10 +146,15 @@ enum wmi_command_id {
 	WMI_BF_CTRL_CMDID			= 0x862,
 	WMI_NOTIFY_REQ_CMDID			= 0x863,
 	WMI_GET_STATUS_CMDID			= 0x864,
+	WMI_GET_RF_STATUS_CMDID			= 0x866,
+	WMI_GET_BASEBAND_TYPE_CMDID		= 0x867,
 	WMI_UNIT_TEST_CMDID			= 0x900,
 	WMI_HICCUP_CMDID			= 0x901,
 	WMI_FLASH_READ_CMDID			= 0x902,
 	WMI_FLASH_WRITE_CMDID			= 0x903,
+	/* Power management */
+	WMI_TRAFFIC_DEFERRAL_CMDID		= 0x904,
+	WMI_TRAFFIC_RESUME_CMDID		= 0x905,
 	/* P2P */
 	WMI_P2P_CFG_CMDID			= 0x910,
 	WMI_PORT_ALLOCATE_CMDID			= 0x911,
@@ -150,6 +167,26 @@ enum wmi_command_id {
 	WMI_PCP_START_CMDID			= 0x918,
 	WMI_PCP_STOP_CMDID			= 0x919,
 	WMI_GET_PCP_FACTOR_CMDID		= 0x91B,
+	/* Power Save Configuration Commands */
+	WMI_PS_DEV_PROFILE_CFG_CMDID		= 0x91C,
+	/* Not supported yet */
+	WMI_PS_DEV_CFG_CMDID			= 0x91D,
+	/* Not supported yet */
+	WMI_PS_DEV_CFG_READ_CMDID		= 0x91E,
+	/* Per MAC Power Save Configuration commands
+	 * Not supported yet
+	 */
+	WMI_PS_MID_CFG_CMDID			= 0x91F,
+	/* Not supported yet */
+	WMI_PS_MID_CFG_READ_CMDID		= 0x920,
+	WMI_RS_CFG_CMDID			= 0x921,
+	WMI_GET_DETAILED_RS_RES_CMDID		= 0x922,
+	WMI_AOA_MEAS_CMDID			= 0x923,
+	WMI_TOF_SESSION_START_CMDID		= 0x991,
+	WMI_TOF_GET_CAPABILITIES_CMDID		= 0x992,
+	WMI_TOF_SET_LCR_CMDID			= 0x993,
+	WMI_TOF_SET_LCI_CMDID			= 0x994,
+	WMI_TOF_CHANNEL_INFO_CMDID		= 0x995,
 	WMI_SET_MAC_ADDRESS_CMDID		= 0xF003,
 	WMI_ABORT_SCAN_CMDID			= 0xF007,
 	WMI_SET_PROMISCUOUS_MODE_CMDID		= 0xF041,
@@ -291,9 +328,8 @@ enum wmi_scan_type {
 /* WMI_START_SCAN_CMDID */
 struct wmi_start_scan_cmd {
 	u8 direct_scan_mac_addr[WMI_MAC_LEN];
-	/* DMG Beacon frame is transmitted during active scanning */
+	/* run scan with discovery beacon. Relevant for ACTIVE scan only. */
 	u8 discovery_mode;
-	/* reserved */
 	u8 reserved;
 	/* Max duration in the home channel(ms) */
 	__le32 dwell_time;
@@ -453,6 +489,12 @@ struct wmi_port_delete_cmd {
 	u8 reserved[3];
 } __packed;
 
+/* WMI_TRAFFIC_DEFERRAL_CMDID */
+struct wmi_traffic_deferral_cmd {
+	/* Bit vector: bit[0] - wake on Unicast, bit[1] - wake on Broadcast */
+	u8 wakeup_trigger;
+} __packed;
+
 /* WMI_P2P_CFG_CMDID */
 enum wmi_discovery_mode {
 	WMI_DISCOVERY_MODE_NON_OFFLOAD	= 0x00,
@@ -818,85 +860,193 @@ struct wmi_pmc_cmd {
 	__le64 mem_base;
 } __packed;
 
+enum wmi_aoa_meas_type {
+	WMI_AOA_PHASE_MEAS	= 0x00,
+	WMI_AOA_PHASE_AMP_MEAS	= 0x01,
+};
+
+/* WMI_AOA_MEAS_CMDID */
+struct wmi_aoa_meas_cmd {
+	u8 mac_addr[WMI_MAC_LEN];
+	/* channels IDs:
+	 * 0 - 58320 MHz
+	 * 1 - 60480 MHz
+	 * 2 - 62640 MHz
+	 */
+	u8 channel;
+	/* enum wmi_aoa_meas_type */
+	u8 aoa_meas_type;
+	__le32 meas_rf_mask;
+} __packed;
+
+enum wmi_tof_burst_duration {
+	WMI_TOF_BURST_DURATION_250_USEC		= 2,
+	WMI_TOF_BURST_DURATION_500_USEC		= 3,
+	WMI_TOF_BURST_DURATION_1_MSEC		= 4,
+	WMI_TOF_BURST_DURATION_2_MSEC		= 5,
+	WMI_TOF_BURST_DURATION_4_MSEC		= 6,
+	WMI_TOF_BURST_DURATION_8_MSEC		= 7,
+	WMI_TOF_BURST_DURATION_16_MSEC		= 8,
+	WMI_TOF_BURST_DURATION_32_MSEC		= 9,
+	WMI_TOF_BURST_DURATION_64_MSEC		= 10,
+	WMI_TOF_BURST_DURATION_128_MSEC		= 11,
+	WMI_TOF_BURST_DURATION_NO_PREFERENCES	= 15,
+};
+
+enum wmi_tof_session_start_flags {
+	WMI_TOF_SESSION_START_FLAG_SECURED	= 0x1,
+	WMI_TOF_SESSION_START_FLAG_ASAP		= 0x2,
+	WMI_TOF_SESSION_START_FLAG_LCI_REQ	= 0x4,
+	WMI_TOF_SESSION_START_FLAG_LCR_REQ	= 0x8,
+};
+
+/* WMI_TOF_SESSION_START_CMDID */
+struct wmi_ftm_dest_info {
+	u8 channel;
+	/* wmi_tof_session_start_flags_e */
+	u8 flags;
+	u8 initial_token;
+	u8 num_of_ftm_per_burst;
+	u8 num_of_bursts_exp;
+	/* wmi_tof_burst_duration_e */
+	u8 burst_duration;
+	/* Burst Period indicate interval between two consecutive burst
+	 * instances, in units of 100 ms
+	 */
+	__le16 burst_period;
+	u8 dst_mac[WMI_MAC_LEN];
+	__le16 reserved;
+} __packed;
+
+/* WMI_TOF_SESSION_START_CMDID */
+struct wmi_tof_session_start_cmd {
+	__le32 session_id;
+	u8 num_of_aoa_measures;
+	u8 aoa_type;
+	__le16 num_of_dest;
+	u8 reserved[4];
+	struct wmi_ftm_dest_info ftm_dest_info[0];
+} __packed;
+
+enum wmi_tof_channel_info_report_type {
+	WMI_TOF_CHANNEL_INFO_TYPE_CIR			= 0x1,
+	WMI_TOF_CHANNEL_INFO_TYPE_RSSI			= 0x2,
+	WMI_TOF_CHANNEL_INFO_TYPE_SNR			= 0x4,
+	WMI_TOF_CHANNEL_INFO_TYPE_DEBUG_DATA		= 0x8,
+	WMI_TOF_CHANNEL_INFO_TYPE_VENDOR_SPECIFIC	= 0x10,
+};
+
+/* WMI_TOF_CHANNEL_INFO_CMDID */
+struct wmi_tof_channel_info_cmd {
+	/* wmi_tof_channel_info_report_type_e */
+	__le32 channel_info_report_request;
+} __packed;
+
 /* WMI Events
  * List of Events (target to host)
  */
 enum wmi_event_id {
-	WMI_READY_EVENTID			= 0x1001,
-	WMI_CONNECT_EVENTID			= 0x1002,
-	WMI_DISCONNECT_EVENTID			= 0x1003,
-	WMI_SCAN_COMPLETE_EVENTID		= 0x100A,
-	WMI_REPORT_STATISTICS_EVENTID		= 0x100B,
-	WMI_RD_MEM_RSP_EVENTID			= 0x1800,
-	WMI_FW_READY_EVENTID			= 0x1801,
-	WMI_EXIT_FAST_MEM_ACC_MODE_EVENTID	= 0x200,
-	WMI_ECHO_RSP_EVENTID			= 0x1803,
-	WMI_FS_TUNE_DONE_EVENTID		= 0x180A,
-	WMI_CORR_MEASURE_EVENTID		= 0x180B,
-	WMI_READ_RSSI_EVENTID			= 0x180C,
-	WMI_TEMP_SENSE_DONE_EVENTID		= 0x180E,
-	WMI_DC_CALIB_DONE_EVENTID		= 0x180F,
-	WMI_IQ_TX_CALIB_DONE_EVENTID		= 0x1811,
-	WMI_IQ_RX_CALIB_DONE_EVENTID		= 0x1812,
-	WMI_SET_WORK_MODE_DONE_EVENTID		= 0x1815,
-	WMI_LO_LEAKAGE_CALIB_DONE_EVENTID	= 0x1816,
-	WMI_MARLON_R_READ_DONE_EVENTID		= 0x1818,
-	WMI_MARLON_R_WRITE_DONE_EVENTID		= 0x1819,
-	WMI_MARLON_R_TXRX_SEL_DONE_EVENTID	= 0x181A,
-	WMI_SILENT_RSSI_CALIB_DONE_EVENTID	= 0x181D,
-	WMI_RF_RX_TEST_DONE_EVENTID		= 0x181E,
-	WMI_CFG_RX_CHAIN_DONE_EVENTID		= 0x1820,
-	WMI_VRING_CFG_DONE_EVENTID		= 0x1821,
-	WMI_BA_STATUS_EVENTID			= 0x1823,
-	WMI_RCP_ADDBA_REQ_EVENTID		= 0x1824,
-	WMI_RCP_ADDBA_RESP_SENT_EVENTID		= 0x1825,
-	WMI_DELBA_EVENTID			= 0x1826,
-	WMI_GET_SSID_EVENTID			= 0x1828,
-	WMI_GET_PCP_CHANNEL_EVENTID		= 0x182A,
-	WMI_SW_TX_COMPLETE_EVENTID		= 0x182B,
-	WMI_READ_MAC_RXQ_EVENTID		= 0x1830,
-	WMI_READ_MAC_TXQ_EVENTID		= 0x1831,
-	WMI_WRITE_MAC_RXQ_EVENTID		= 0x1832,
-	WMI_WRITE_MAC_TXQ_EVENTID		= 0x1833,
-	WMI_WRITE_MAC_XQ_FIELD_EVENTID		= 0x1834,
-	WMI_BEAMFORMING_MGMT_DONE_EVENTID	= 0x1836,
-	WMI_BF_TXSS_MGMT_DONE_EVENTID		= 0x1837,
-	WMI_BF_RXSS_MGMT_DONE_EVENTID		= 0x1839,
-	WMI_RS_MGMT_DONE_EVENTID		= 0x1852,
-	WMI_RF_MGMT_STATUS_EVENTID		= 0x1853,
-	WMI_THERMAL_THROTTLING_STATUS_EVENTID	= 0x1855,
-	WMI_BF_SM_MGMT_DONE_EVENTID		= 0x1838,
-	WMI_RX_MGMT_PACKET_EVENTID		= 0x1840,
-	WMI_TX_MGMT_PACKET_EVENTID		= 0x1841,
-	WMI_OTP_READ_RESULT_EVENTID		= 0x1856,
-	WMI_LED_CFG_DONE_EVENTID		= 0x1858,
+	WMI_READY_EVENTID				= 0x1001,
+	WMI_CONNECT_EVENTID				= 0x1002,
+	WMI_DISCONNECT_EVENTID				= 0x1003,
+	WMI_SCAN_COMPLETE_EVENTID			= 0x100A,
+	WMI_REPORT_STATISTICS_EVENTID			= 0x100B,
+	WMI_RD_MEM_RSP_EVENTID				= 0x1800,
+	WMI_FW_READY_EVENTID				= 0x1801,
+	WMI_EXIT_FAST_MEM_ACC_MODE_EVENTID		= 0x200,
+	WMI_ECHO_RSP_EVENTID				= 0x1803,
+	WMI_FS_TUNE_DONE_EVENTID			= 0x180A,
+	WMI_CORR_MEASURE_EVENTID			= 0x180B,
+	WMI_READ_RSSI_EVENTID				= 0x180C,
+	WMI_TEMP_SENSE_DONE_EVENTID			= 0x180E,
+	WMI_DC_CALIB_DONE_EVENTID			= 0x180F,
+	WMI_IQ_TX_CALIB_DONE_EVENTID			= 0x1811,
+	WMI_IQ_RX_CALIB_DONE_EVENTID			= 0x1812,
+	WMI_SET_WORK_MODE_DONE_EVENTID			= 0x1815,
+	WMI_LO_LEAKAGE_CALIB_DONE_EVENTID		= 0x1816,
+	WMI_MARLON_R_READ_DONE_EVENTID			= 0x1818,
+	WMI_MARLON_R_WRITE_DONE_EVENTID			= 0x1819,
+	WMI_MARLON_R_TXRX_SEL_DONE_EVENTID		= 0x181A,
+	WMI_SILENT_RSSI_CALIB_DONE_EVENTID		= 0x181D,
+	WMI_RF_RX_TEST_DONE_EVENTID			= 0x181E,
+	WMI_CFG_RX_CHAIN_DONE_EVENTID			= 0x1820,
+	WMI_VRING_CFG_DONE_EVENTID			= 0x1821,
+	WMI_BA_STATUS_EVENTID				= 0x1823,
+	WMI_RCP_ADDBA_REQ_EVENTID			= 0x1824,
+	WMI_RCP_ADDBA_RESP_SENT_EVENTID			= 0x1825,
+	WMI_DELBA_EVENTID				= 0x1826,
+	WMI_GET_SSID_EVENTID				= 0x1828,
+	WMI_GET_PCP_CHANNEL_EVENTID			= 0x182A,
+	WMI_SW_TX_COMPLETE_EVENTID			= 0x182B,
+	WMI_READ_MAC_RXQ_EVENTID			= 0x1830,
+	WMI_READ_MAC_TXQ_EVENTID			= 0x1831,
+	WMI_WRITE_MAC_RXQ_EVENTID			= 0x1832,
+	WMI_WRITE_MAC_TXQ_EVENTID			= 0x1833,
+	WMI_WRITE_MAC_XQ_FIELD_EVENTID			= 0x1834,
+	WMI_BEAMFORMING_MGMT_DONE_EVENTID		= 0x1836,
+	WMI_BF_TXSS_MGMT_DONE_EVENTID			= 0x1837,
+	WMI_BF_RXSS_MGMT_DONE_EVENTID			= 0x1839,
+	WMI_RS_MGMT_DONE_EVENTID			= 0x1852,
+	WMI_RF_MGMT_STATUS_EVENTID			= 0x1853,
+	WMI_THERMAL_THROTTLING_STATUS_EVENTID		= 0x1855,
+	WMI_BF_SM_MGMT_DONE_EVENTID			= 0x1838,
+	WMI_RX_MGMT_PACKET_EVENTID			= 0x1840,
+	WMI_TX_MGMT_PACKET_EVENTID			= 0x1841,
+	WMI_LINK_MAINTAIN_CFG_WRITE_DONE_EVENTID	= 0x1842,
+	WMI_LINK_MAINTAIN_CFG_READ_DONE_EVENTID		= 0x1843,
+	WMI_OTP_READ_RESULT_EVENTID			= 0x1856,
+	WMI_LED_CFG_DONE_EVENTID			= 0x1858,
 	/* Performance monitoring events */
-	WMI_DATA_PORT_OPEN_EVENTID		= 0x1860,
-	WMI_WBE_LINK_DOWN_EVENTID		= 0x1861,
-	WMI_BF_CTRL_DONE_EVENTID		= 0x1862,
-	WMI_NOTIFY_REQ_DONE_EVENTID		= 0x1863,
-	WMI_GET_STATUS_DONE_EVENTID		= 0x1864,
-	WMI_VRING_EN_EVENTID			= 0x1865,
-	WMI_UNIT_TEST_EVENTID			= 0x1900,
-	WMI_FLASH_READ_DONE_EVENTID		= 0x1902,
-	WMI_FLASH_WRITE_DONE_EVENTID		= 0x1903,
+	WMI_DATA_PORT_OPEN_EVENTID			= 0x1860,
+	WMI_WBE_LINK_DOWN_EVENTID			= 0x1861,
+	WMI_BF_CTRL_DONE_EVENTID			= 0x1862,
+	WMI_NOTIFY_REQ_DONE_EVENTID			= 0x1863,
+	WMI_GET_STATUS_DONE_EVENTID			= 0x1864,
+	WMI_VRING_EN_EVENTID				= 0x1865,
+	WMI_GET_RF_STATUS_EVENTID			= 0x1866,
+	WMI_GET_BASEBAND_TYPE_EVENTID			= 0x1867,
+	WMI_UNIT_TEST_EVENTID				= 0x1900,
+	WMI_FLASH_READ_DONE_EVENTID			= 0x1902,
+	WMI_FLASH_WRITE_DONE_EVENTID			= 0x1903,
+	/* Power management */
+	WMI_TRAFFIC_DEFERRAL_EVENTID			= 0x1904,
+	WMI_TRAFFIC_RESUME_EVENTID			= 0x1905,
 	/* P2P */
-	WMI_P2P_CFG_DONE_EVENTID		= 0x1910,
-	WMI_PORT_ALLOCATED_EVENTID		= 0x1911,
-	WMI_PORT_DELETED_EVENTID		= 0x1912,
-	WMI_LISTEN_STARTED_EVENTID		= 0x1914,
-	WMI_SEARCH_STARTED_EVENTID		= 0x1915,
-	WMI_DISCOVERY_STARTED_EVENTID		= 0x1916,
-	WMI_DISCOVERY_STOPPED_EVENTID		= 0x1917,
-	WMI_PCP_STARTED_EVENTID			= 0x1918,
-	WMI_PCP_STOPPED_EVENTID			= 0x1919,
-	WMI_PCP_FACTOR_EVENTID			= 0x191A,
-	WMI_SET_CHANNEL_EVENTID			= 0x9000,
-	WMI_ASSOC_REQ_EVENTID			= 0x9001,
-	WMI_EAPOL_RX_EVENTID			= 0x9002,
-	WMI_MAC_ADDR_RESP_EVENTID		= 0x9003,
-	WMI_FW_VER_EVENTID			= 0x9004,
-	WMI_ACS_PASSIVE_SCAN_COMPLETE_EVENTID	= 0x9005,
+	WMI_P2P_CFG_DONE_EVENTID			= 0x1910,
+	WMI_PORT_ALLOCATED_EVENTID			= 0x1911,
+	WMI_PORT_DELETED_EVENTID			= 0x1912,
+	WMI_LISTEN_STARTED_EVENTID			= 0x1914,
+	WMI_SEARCH_STARTED_EVENTID			= 0x1915,
+	WMI_DISCOVERY_STARTED_EVENTID			= 0x1916,
+	WMI_DISCOVERY_STOPPED_EVENTID			= 0x1917,
+	WMI_PCP_STARTED_EVENTID				= 0x1918,
+	WMI_PCP_STOPPED_EVENTID				= 0x1919,
+	WMI_PCP_FACTOR_EVENTID				= 0x191A,
+	/* Power Save Configuration Events */
+	WMI_PS_DEV_PROFILE_CFG_EVENTID			= 0x191C,
+	/* Not supported yet */
+	WMI_PS_DEV_CFG_EVENTID				= 0x191D,
+	/* Not supported yet */
+	WMI_PS_DEV_CFG_READ_EVENTID			= 0x191E,
+	/* Not supported yet */
+	WMI_PS_MID_CFG_EVENTID				= 0x191F,
+	/* Not supported yet */
+	WMI_PS_MID_CFG_READ_EVENTID			= 0x1920,
+	WMI_RS_CFG_DONE_EVENTID				= 0x1921,
+	WMI_GET_DETAILED_RS_RES_EVENTID			= 0x1922,
+	WMI_AOA_MEAS_EVENTID				= 0x1923,
+	WMI_TOF_SESSION_END_EVENTID			= 0x1991,
+	WMI_TOF_GET_CAPABILITIES_EVENTID		= 0x1992,
+	WMI_TOF_SET_LCR_EVENTID				= 0x1993,
+	WMI_TOF_SET_LCI_EVENTID				= 0x1994,
+	WMI_TOF_FTM_PER_DEST_RES_EVENTID		= 0x1995,
+	WMI_TOF_CHANNEL_INFO_EVENTID			= 0x1996,
+	WMI_SET_CHANNEL_EVENTID				= 0x9000,
+	WMI_ASSOC_REQ_EVENTID				= 0x9001,
+	WMI_EAPOL_RX_EVENTID				= 0x9002,
+	WMI_MAC_ADDR_RESP_EVENTID			= 0x9003,
+	WMI_FW_VER_EVENTID				= 0x9004,
+	WMI_ACS_PASSIVE_SCAN_COMPLETE_EVENTID		= 0x9005,
 };
 
 /* Events data structures */
@@ -943,10 +1093,85 @@ struct wmi_get_status_done_event {
 
 /* WMI_FW_VER_EVENTID */
 struct wmi_fw_ver_event {
-	u8 major;
-	u8 minor;
-	__le16 subminor;
-	__le16 build;
+	/* FW image version */
+	__le32 fw_major;
+	__le32 fw_minor;
+	__le32 fw_subminor;
+	__le32 fw_build;
+	/* FW image build time stamp */
+	__le32 hour;
+	__le32 minute;
+	__le32 second;
+	__le32 day;
+	__le32 month;
+	__le32 year;
+	/* Boot Loader image version */
+	__le32 bl_major;
+	__le32 bl_minor;
+	__le32 bl_subminor;
+	__le32 bl_build;
+	/* The number of entries in the FW capabilies array */
+	u8 fw_capabilities_len;
+	u8 reserved[3];
+	/* FW capabilities info
+	 * Must be the last member of the struct
+	 */
+	__le32 fw_capabilities[0];
+} __packed;
+
+/* WMI_GET_RF_STATUS_EVENTID */
+enum rf_type {
+	RF_UNKNOWN	= 0x00,
+	RF_MARLON	= 0x01,
+	RF_SPARROW	= 0x02,
+};
+
+/* WMI_GET_RF_STATUS_EVENTID */
+enum board_file_rf_type {
+	BF_RF_MARLON	= 0x00,
+	BF_RF_SPARROW	= 0x01,
+};
+
+/* WMI_GET_RF_STATUS_EVENTID */
+enum rf_status {
+	RF_OK			= 0x00,
+	RF_NO_COMM		= 0x01,
+	RF_WRONG_BOARD_FILE	= 0x02,
+};
+
+/* WMI_GET_RF_STATUS_EVENTID */
+struct wmi_get_rf_status_event {
+	/* enum rf_type */
+	__le32 rf_type;
+	/* attached RFs bit vector */
+	__le32 attached_rf_vector;
+	/* enabled RFs bit vector */
+	__le32 enabled_rf_vector;
+	/* enum rf_status, refers to enabled RFs */
+	u8 rf_status[32];
+	/* enum board file RF type */
+	__le32 board_file_rf_type;
+	/* board file platform type */
+	__le32 board_file_platform_type;
+	/* board file version */
+	__le32 board_file_version;
+	__le32 reserved[2];
+} __packed;
+
+/* WMI_GET_BASEBAND_TYPE_EVENTID */
+enum baseband_type {
+	BASEBAND_UNKNOWN	= 0x00,
+	BASEBAND_SPARROW_M_A0	= 0x03,
+	BASEBAND_SPARROW_M_A1	= 0x04,
+	BASEBAND_SPARROW_M_B0	= 0x05,
+	BASEBAND_SPARROW_M_C0	= 0x06,
+	BASEBAND_SPARROW_M_D0	= 0x07,
+};
+
+/* WMI_GET_BASEBAND_TYPE_EVENTID */
+struct wmi_get_baseband_type_event {
+	/* enum baseband_type */
+	__le32 baseband_type;
 } __packed;
 
 /* WMI_MAC_ADDR_RESP_EVENTID */
@@ -1410,4 +1635,553 @@ struct wmi_led_cfg_done_event {
 	__le32 status;
 } __packed;
 
+#define WMI_NUM_MCS	(13)
+
+/* Rate search parameters configuration per connection */
+struct wmi_rs_cfg {
+	/* The maximal allowed PER for each MCS
+	 * MCS will be considered as failed if PER during RS is higher
+	 */
+	u8 per_threshold[WMI_NUM_MCS];
+	/* Number of MPDUs for each MCS
+	 * this is the minimal statistic required to make an educated
+	 * decision
+	 */
+	u8 min_frame_cnt[WMI_NUM_MCS];
+	/* stop threshold [0-100] */
+	u8 stop_th;
+	/* MCS1 stop threshold [0-100] */
+	u8 mcs1_fail_th;
+	u8 max_back_failure_th;
+	/* Debug feature for disabling internal RS trigger (which is
+	 * currently triggered by BF Done)
+	 */
+	u8 dbg_disable_internal_trigger;
+	__le32 back_failure_mask;
+	__le32 mcs_en_vec;
+} __packed;
+
+/* WMI_RS_CFG_CMDID */
+struct wmi_rs_cfg_cmd {
+	/* connection id */
+	u8 cid;
+	/* enable or disable rate search */
+	u8 rs_enable;
+	/* rate search configuration */
+	struct wmi_rs_cfg rs_cfg;
+} __packed;
+
+/* WMI_RS_CFG_DONE_EVENTID */
+struct wmi_rs_cfg_done_event {
+	u8 cid;
+	/* enum wmi_fw_status */
+	u8 status;
+	u8 reserved[2];
+} __packed;
+
+/* WMI_GET_DETAILED_RS_RES_CMDID */
+struct wmi_get_detailed_rs_res_cmd {
+	/* connection id */
+	u8 cid;
+	u8 reserved[3];
+} __packed;
+
+/* RS results status */
+enum wmi_rs_results_status {
+	WMI_RS_RES_VALID	= 0x00,
+	WMI_RS_RES_INVALID	= 0x01,
+};
+
+/* Rate search results */
+struct wmi_rs_results {
+	/* number of sent MPDUs */
+	u8 num_of_tx_pkt[WMI_NUM_MCS];
+	/* number of non-acked MPDUs */
+	u8 num_of_non_acked_pkt[WMI_NUM_MCS];
+	/* RS timestamp */
+	__le32 tsf;
+	/* RS selected MCS */
+	u8 mcs;
+} __packed;
+
+/* WMI_GET_DETAILED_RS_RES_EVENTID */
+struct wmi_get_detailed_rs_res_event {
+	u8 cid;
+	/* enum wmi_rs_results_status */
+	u8 status;
+	/* detailed rs results */
+	struct wmi_rs_results rs_results;
+	u8 reserved[3];
+} __packed;
+
+/* broadcast connection ID */
+#define WMI_LINK_MAINTAIN_CFG_CID_BROADCAST	(0xFFFFFFFF)
+
+/* Types wmi_link_maintain_cfg presets for WMI_LINK_MAINTAIN_CFG_WRITE_CMD */
+enum wmi_link_maintain_cfg_type {
+	/* AP/PCP default normal (non-FST) configuration settings */
+	WMI_LINK_MAINTAIN_CFG_TYPE_DEFAULT_NORMAL_AP	= 0x00,
+	/* AP/PCP  default FST configuration settings */
+	WMI_LINK_MAINTAIN_CFG_TYPE_DEFAULT_FST_AP	= 0x01,
+	/* STA default normal (non-FST) configuration settings */
+	WMI_LINK_MAINTAIN_CFG_TYPE_DEFAULT_NORMAL_STA	= 0x02,
+	/* STA default FST configuration settings */
+	WMI_LINK_MAINTAIN_CFG_TYPE_DEFAULT_FST_STA	= 0x03,
+	/* custom configuration settings */
+	WMI_LINK_MAINTAIN_CFG_TYPE_CUSTOM		= 0x04,
+	/* number of defined configuration types */
+	WMI_LINK_MAINTAIN_CFG_TYPES_NUM			= 0x05,
+};
+
+/* Response status codes for WMI_LINK_MAINTAIN_CFG_WRITE/READ commands */
+enum wmi_link_maintain_cfg_response_status {
+	/* WMI_LINK_MAINTAIN_CFG_WRITE/READ command successfully accomplished
+	 */
+	WMI_LINK_MAINTAIN_CFG_RESPONSE_STATUS_OK		= 0x00,
+	/* ERROR due to bad argument in WMI_LINK_MAINTAIN_CFG_WRITE/READ
+	 * command request
+	 */
+	WMI_LINK_MAINTAIN_CFG_RESPONSE_STATUS_BAD_ARGUMENT	= 0x01,
+};
+
+/* Link Loss and Keep Alive configuration */
+struct wmi_link_maintain_cfg {
+	/* link_loss_enable_detectors_vec */
+	__le32 link_loss_enable_detectors_vec;
+	/* detectors check period usec */
+	__le32 check_link_loss_period_usec;
+	/* max allowed tx ageing */
+	__le32 tx_ageing_threshold_usec;
+	/* keep alive period for high SNR */
+	__le32 keep_alive_period_usec_high_snr;
+	/* keep alive period for low SNR */
+	__le32 keep_alive_period_usec_low_snr;
+	/* lower snr limit for keep alive period update */
+	__le32 keep_alive_snr_threshold_low_db;
+	/* upper snr limit for keep alive period update */
+	__le32 keep_alive_snr_threshold_high_db;
+	/* num of successive bad bcons causing link-loss */
+	__le32 bad_beacons_num_threshold;
+	/* SNR limit for bad_beacons_detector */
+	__le32 bad_beacons_snr_threshold_db;
+} __packed;
+
+/* WMI_LINK_MAINTAIN_CFG_WRITE_CMDID */
+struct wmi_link_maintain_cfg_write_cmd {
+	/* enum wmi_link_maintain_cfg_type_e - type of requested default
+	 * configuration to be applied
+	 */
+	__le32 cfg_type;
+	/* requested connection ID or WMI_LINK_MAINTAIN_CFG_CID_BROADCAST */
+	__le32 cid;
+	/* custom configuration settings to be applied (relevant only if
+	 * cfg_type==WMI_LINK_MAINTAIN_CFG_TYPE_CUSTOM)
+	 */
+	struct wmi_link_maintain_cfg lm_cfg;
+} __packed;
+
+/* WMI_LINK_MAINTAIN_CFG_READ_CMDID */
+struct wmi_link_maintain_cfg_read_cmd {
+	/* connection ID which configuration settings are requested */
+	__le32 cid;
+} __packed;
+
+/* WMI_LINK_MAINTAIN_CFG_WRITE_DONE_EVENTID */
+struct wmi_link_maintain_cfg_write_done_event {
+	/* requested connection ID */
+	__le32 cid;
+	/* wmi_link_maintain_cfg_response_status_e - write status */
+	__le32 status;
+} __packed;
+
+/* \WMI_LINK_MAINTAIN_CFG_READ_DONE_EVENT */
+struct wmi_link_maintain_cfg_read_done_event {
+	/* requested connection ID */
+	__le32 cid;
+	/* wmi_link_maintain_cfg_response_status_e - read status */
+	__le32 status;
+	/* Retrieved configuration settings */
+	struct wmi_link_maintain_cfg lm_cfg;
+} __packed;
+
+enum wmi_traffic_deferral_status {
+	WMI_TRAFFIC_DEFERRAL_APPROVED	= 0x0,
+	WMI_TRAFFIC_DEFERRAL_REJECTED	= 0x1,
+};
+
+/* WMI_TRAFFIC_DEFERRAL_EVENTID */
+struct wmi_traffic_deferral_event {
+	/* enum wmi_traffic_deferral_status_e */
+	u8 status;
+} __packed;
+
+enum wmi_traffic_resume_status {
+	WMI_TRAFFIC_RESUME_SUCCESS	= 0x0,
+	WMI_TRAFFIC_RESUME_FAILED	= 0x1,
+};
+
+/* WMI_TRAFFIC_RESUME_EVENTID */
+struct wmi_traffic_resume_event {
+	/* enum wmi_traffic_resume_status_e */
+	u8 status;
+} __packed;
+
+/* Power Save command completion status codes */
+enum wmi_ps_cfg_cmd_status {
+	WMI_PS_CFG_CMD_STATUS_SUCCESS	= 0x00,
+	WMI_PS_CFG_CMD_STATUS_BAD_PARAM	= 0x01,
+	/* other error */
+	WMI_PS_CFG_CMD_STATUS_ERROR	= 0x02,
+};
+
+/* Device Power Save Profiles */
+enum wmi_ps_profile_type {
+	WMI_PS_PROFILE_TYPE_DEFAULT		= 0x00,
+	WMI_PS_PROFILE_TYPE_PS_DISABLED		= 0x01,
+	WMI_PS_PROFILE_TYPE_MAX_PS		= 0x02,
+	WMI_PS_PROFILE_TYPE_LOW_LATENCY_PS	= 0x03,
+};
+
+/* WMI_PS_DEV_PROFILE_CFG_CMDID
+ *
+ * Power save profile to be used by the device
+ *
+ * Returned event:
+ * - WMI_PS_DEV_PROFILE_CFG_EVENTID
+ */
+struct wmi_ps_dev_profile_cfg_cmd {
+	/* wmi_ps_profile_type_e */
+	u8 ps_profile;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_PS_DEV_PROFILE_CFG_EVENTID */
+struct wmi_ps_dev_profile_cfg_event {
+	/* wmi_ps_cfg_cmd_status_e */
+	__le32 status;
+} __packed;
+
+enum wmi_ps_level {
+	WMI_PS_LEVEL_DEEP_SLEEP		= 0x00,
+	WMI_PS_LEVEL_SHALLOW_SLEEP	= 0x01,
+	/* awake = all PS mechanisms are disabled */
+	WMI_PS_LEVEL_AWAKE		= 0x02,
+};
+
+enum wmi_ps_deep_sleep_clk_level {
+	/* 33k */
+	WMI_PS_DEEP_SLEEP_CLK_LEVEL_RTC		= 0x00,
+	/* 10k */
+	WMI_PS_DEEP_SLEEP_CLK_LEVEL_OSC		= 0x01,
+	/* @RTC Low latency */
+	WMI_PS_DEEP_SLEEP_CLK_LEVEL_RTC_LT	= 0x02,
+	WMI_PS_DEEP_SLEEP_CLK_LEVEL_XTAL	= 0x03,
+	WMI_PS_DEEP_SLEEP_CLK_LEVEL_SYSCLK	= 0x04,
+	/* Not Applicable */
+	WMI_PS_DEEP_SLEEP_CLK_LEVEL_N_A		= 0xFF,
+};
+
+/* Response by the FW to a D3 entry request */
+enum wmi_ps_d3_resp_policy {
+	WMI_PS_D3_RESP_POLICY_DEFAULT	= 0x00,
+	/* debug -D3 req is always denied */
+	WMI_PS_D3_RESP_POLICY_DENIED	= 0x01,
+	/* debug -D3 req is always approved */
+	WMI_PS_D3_RESP_POLICY_APPROVED	= 0x02,
+};
+
+/* Device common power save configurations */
+struct wmi_ps_dev_cfg {
+	/* lowest level of PS allowed while unassociated, enum wmi_ps_level_e
+	 */
+	u8 ps_unassoc_min_level;
+	/* lowest deep sleep clock level while nonassoc, enum
+	 * wmi_ps_deep_sleep_clk_level_e
+	 */
+	u8 ps_unassoc_deep_sleep_min_level;
+	/* lowest level of PS allowed while associated, enum wmi_ps_level_e */
+	u8 ps_assoc_min_level;
+	/* lowest deep sleep clock level while assoc, enum
+	 * wmi_ps_deep_sleep_clk_level_e
+	 */
+	u8 ps_assoc_deep_sleep_min_level;
+	/* enum wmi_ps_deep_sleep_clk_level_e */
+	u8 ps_assoc_low_latency_ds_min_level;
+	/* enum wmi_ps_d3_resp_policy_e */
+	u8 ps_D3_response_policy;
+	/* BOOL */
+	u8 ps_D3_pm_pme_enabled;
+	/* BOOL */
+	u8 ps_halp_enable;
+	u8 ps_deep_sleep_enter_thresh_msec;
+	/* BOOL */
+	u8 ps_voltage_scaling_en;
+} __packed;
+
+/* WMI_PS_DEV_CFG_CMDID
+ *
+ * Configure common Power Save parameters of the device and all MIDs.
+ *
+ * Returned event:
+ * - WMI_PS_DEV_CFG_EVENTID
+ */
+struct wmi_ps_dev_cfg_cmd {
+	/* Device Power Save configuration to be applied */
+	struct wmi_ps_dev_cfg ps_dev_cfg;
+	/* alignment to 32b */
+	u8 reserved[2];
+} __packed;
+
+/* WMI_PS_DEV_CFG_EVENTID */
+struct wmi_ps_dev_cfg_event {
+	/* wmi_ps_cfg_cmd_status_e */
+	__le32 status;
+} __packed;
+
+/* WMI_PS_DEV_CFG_READ_CMDID
+ *
+ * request to retrieve  device Power Save configuration
+ * (WMI_PS_DEV_CFG_CMD params)
+ *
+ * Returned event:
+ * - WMI_PS_DEV_CFG_READ_EVENTID
+ */
+struct wmi_ps_dev_cfg_read_cmd {
+	__le32 reserved;
+} __packed;
+
+/* WMI_PS_DEV_CFG_READ_EVENTID */
+struct wmi_ps_dev_cfg_read_event {
+	/* wmi_ps_cfg_cmd_status_e */
+	__le32 status;
+	/* Retrieved device Power Save configuration (WMI_PS_DEV_CFG_CMD
+	 * params)
+	 */
+	struct wmi_ps_dev_cfg dev_ps_cfg;
+	/* alignment to 32b */
+	u8 reserved[2];
+} __packed;
+
+/* Per Mac Power Save configurations */
+struct wmi_ps_mid_cfg {
+	/* Low power RX in BTI is enabled, BOOL */
+	u8 beacon_lprx_enable;
+	/* Sync to sector ID enabled, BOOL */
+	u8 beacon_sync_to_sectorId_enable;
+	/* Low power RX in DTI is enabled, BOOL */
+	u8 frame_exchange_lprx_enable;
+	/* Sleep Cycle while in scheduled PS, 1-31 */
+	u8 scheduled_sleep_cycle_pow2;
+	/* Stay Awake for k BIs every (sleep_cycle - k) BIs, 1-31 */
+	u8 scheduled_num_of_awake_bis;
+	u8 am_to_traffic_load_thresh_mbp;
+	u8 traffic_to_am_load_thresh_mbps;
+	u8 traffic_to_am_num_of_no_traffic_bis;
+	/* BOOL */
+	u8 continuous_traffic_psm;
+	__le16 no_traffic_to_min_usec;
+	__le16 no_traffic_to_max_usec;
+	__le16 snoozing_sleep_interval_milisec;
+	u8 max_no_data_awake_events;
+	/* Trigger WEB after k failed beacons */
+	u8 num_of_failed_beacons_rx_to_trigger_web;
+	/* Trigger BF after k failed beacons */
+	u8 num_of_failed_beacons_rx_to_trigger_bf;
+	/* Trigger SOB after k successful beacons */
+	u8 num_of_successful_beacons_rx_to_trigger_sob;
+} __packed;
+
+/* WMI_PS_MID_CFG_CMDID
+ *
+ * Configure Power Save parameters of a specific MID.
+ * These parameters are relevant for the specific BSS this MID belongs to.
+ *
+ * Returned event:
+ * - WMI_PS_MID_CFG_EVENTID
+ */
+struct wmi_ps_mid_cfg_cmd {
+	/* MAC ID */
+	u8 mid;
+	/* mid PS configuration to be applied */
+	struct wmi_ps_mid_cfg ps_mid_cfg;
+} __packed;
+
+/* WMI_PS_MID_CFG_EVENTID */
+struct wmi_ps_mid_cfg_event {
+	/* MAC ID */
+	u8 mid;
+	/* alignment to 32b */
+	u8 reserved[3];
+	/* wmi_ps_cfg_cmd_status_e */
+	__le32 status;
+} __packed;
+
+/* WMI_PS_MID_CFG_READ_CMDID
+ *
+ * request to retrieve Power Save configuration of mid
+ * (WMI_PS_MID_CFG_CMD params)
+ *
+ * Returned event:
+ * - WMI_PS_MID_CFG_READ_EVENTID
+ */
+struct wmi_ps_mid_cfg_read_cmd {
+	/* MAC ID */
+	u8 mid;
+	/* alignment to 32b */
+	u8 reserved[3];
+} __packed;
+
+/* WMI_PS_MID_CFG_READ_EVENTID */
+struct wmi_ps_mid_cfg_read_event {
+	/* MAC ID */
+	u8 mid;
+	/* Retrieved MID Power Save configuration(WMI_PS_MID_CFG_CMD params) */
+	struct wmi_ps_mid_cfg mid_ps_cfg;
+	/* wmi_ps_cfg_cmd_status_e */
+	__le32 status;
+} __packed;
+
+#define WMI_AOA_MAX_DATA_SIZE	(128)
+
+enum wmi_aoa_meas_status {
+	WMI_AOA_MEAS_SUCCESS		= 0x00,
+	WMI_AOA_MEAS_PEER_INCAPABLE	= 0x01,
+	WMI_AOA_MEAS_FAILURE		= 0x02,
+};
+
+/* WMI_AOA_MEAS_EVENTID */
+struct wmi_aoa_meas_event {
+	u8 mac_addr[WMI_MAC_LEN];
+	/* channels IDs:
+	 * 0 - 58320 MHz
+	 * 1 - 60480 MHz
+	 * 2 - 62640 MHz
+	 */
+	u8 channel;
+	/* enum wmi_aoa_meas_type */
+	u8 aoa_meas_type;
+	/* Measurments are from RFs, defined by the mask */
+	__le32 meas_rf_mask;
+	/* enum wmi_aoa_meas_status */
+	u8 meas_status;
+	u8 reserved;
+	/* Length of meas_data in bytes */
+	__le16 length;
+	u8 meas_data[WMI_AOA_MAX_DATA_SIZE];
+} __packed;
+
+/* WMI_TOF_GET_CAPABILITIES_EVENTID */
+struct wmi_tof_get_capabilities_event {
+	u8 ftm_capability;
+	/* maximum supported number of destination to start TOF */
+	u8 max_num_of_dest;
+	/* maximum supported number of measurements per burst */
+	u8 max_num_of_meas_per_burst;
+	u8 reserved;
+	/* maximum supported multi bursts */
+	__le16 max_multi_bursts_sessions;
+	/* maximum supported FTM burst duration , wmi_tof_burst_duration_e */
+	__le16 max_ftm_burst_duration;
+	/* AOA supported types */
+	__le32 aoa_supported_types;
+} __packed;
+
+enum wmi_tof_session_end_status {
+	WMI_TOF_SESSION_END_NO_ERROR		= 0x00,
+	WMI_TOF_SESSION_END_FAIL		= 0x01,
+	WMI_TOF_SESSION_END_PARAMS_ERROR	= 0x02,
+	WMI_TOF_SESSION_END_ABORTED		= 0x03,
+};
+
+/* WMI_TOF_SESSION_END_EVENTID */
+struct wmi_tof_session_end_event {
+	/* FTM session ID */
+	__le32 session_id;
+	/* wmi_tof_session_end_status_e */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+/* Responder FTM Results */
+struct wmi_responder_ftm_res {
+	u8 t1[6];
+	u8 t2[6];
+	u8 t3[6];
+	u8 t4[6];
+	__le16 tod_err;
+	__le16 toa_err;
+	__le16 tod_err_initiator;
+	__le16 toa_err_initiator;
+} __packed;
+
+enum wmi_tof_ftm_per_dest_res_status {
+	WMI_PER_DEST_RES_NO_ERROR		= 0x00,
+	WMI_PER_DEST_RES_TX_RX_FAIL		= 0x01,
+	WMI_PER_DEST_RES_PARAM_DONT_MATCH	= 0x02,
+};
+
+enum wmi_tof_ftm_per_dest_res_flags {
+	WMI_PER_DEST_RES_REQ_START		= 0x01,
+	WMI_PER_DEST_RES_BURST_REPORT_END	= 0x02,
+	WMI_PER_DEST_RES_REQ_END		= 0x04,
+	WMI_PER_DEST_RES_PARAM_UPDATE		= 0x08,
+};
+
+/* WMI_TOF_FTM_PER_DEST_RES_EVENTID */
+struct wmi_tof_ftm_per_dest_res_event {
+	/* FTM session ID */
+	__le32 session_id;
+	/* destination MAC address */
+	u8 dst_mac[WMI_MAC_LEN];
+	/* wmi_tof_ftm_per_dest_res_flags_e */
+	u8 flags;
+	/* wmi_tof_ftm_per_dest_res_status_e */
+	u8 status;
+	/* responder ASAP */
+	u8 responder_asap;
+	/* responder number of FTM per burst */
+	u8 responder_num_ftm_per_burst;
+	/* responder number of FTM burst exponent */
+	u8 responder_num_ftm_bursts_exp;
+	/* responder burst duration ,wmi_tof_burst_duration_e */
+	u8 responder_burst_duration;
+	/* responder burst period, indicate interval between two consecutive
+	 * burst instances, in units of 100 ms
+	 */
+	__le16 responder_burst_period;
+	/* receive burst counter */
+	__le16 bursts_cnt;
+	/* tsf of responder start burst */
+	__le32 tsf_sync;
+	/* actual received ftm per burst */
+	u8 actual_ftm_per_burst;
+	u8 reserved0[7];
+	struct wmi_responder_ftm_res responder_ftm_res[0];
+} __packed;
+
+enum wmi_tof_channel_info_type {
+	WMI_TOF_CHANNEL_INFO_AOA		= 0x00,
+	WMI_TOF_CHANNEL_INFO_LCI		= 0x01,
+	WMI_TOF_CHANNEL_INFO_LCR		= 0x02,
+	WMI_TOF_CHANNEL_INFO_VENDOR_SPECIFIC	= 0x03,
+	WMI_TOF_CHANNEL_INFO_CIR		= 0x04,
+	WMI_TOF_CHANNEL_INFO_RSSI		= 0x05,
+	WMI_TOF_CHANNEL_INFO_SNR		= 0x06,
+	WMI_TOF_CHANNEL_INFO_DEBUG		= 0x07,
+};
+
+/* WMI_TOF_CHANNEL_INFO_EVENTID */
+struct wmi_tof_channel_info_event {
+	/* FTM session ID */
+	__le32 session_id;
+	/* destination MAC address */
+	u8 dst_mac[WMI_MAC_LEN];
+	/* wmi_tof_channel_info_type_e */
+	u8 type;
+	/* data report length */
+	u8 len;
+	/* data report payload */
+	u8 report[0];
+} __packed;
+
 #endif /* __WILOCITY_WMI_H__ */
diff --git a/drivers/net/wireless/broadcom/b43/debugfs.c b/drivers/net/wireless/broadcom/b43/debugfs.c
index b4bcd94aff6c..77046384dd80 100644
--- a/drivers/net/wireless/broadcom/b43/debugfs.c
+++ b/drivers/net/wireless/broadcom/b43/debugfs.c
@@ -524,7 +524,8 @@ static ssize_t b43_debugfs_read(struct file *file, char __user *userbuf,
 		goto out_unlock;
 	}
 
-	dfops = container_of(file->f_op, struct b43_debugfs_fops, fops);
+	dfops = container_of(debugfs_real_fops(file),
+			     struct b43_debugfs_fops, fops);
 	if (!dfops->read) {
 		err = -ENOSYS;
 		goto out_unlock;
@@ -585,7 +586,8 @@ static ssize_t b43_debugfs_write(struct file *file,
 		goto out_unlock;
 	}
 
-	dfops = container_of(file->f_op, struct b43_debugfs_fops, fops);
+	dfops = container_of(debugfs_real_fops(file),
+			     struct b43_debugfs_fops, fops);
 	if (!dfops->write) {
 		err = -ENOSYS;
 		goto out_unlock;
diff --git a/drivers/net/wireless/broadcom/b43legacy/debugfs.c b/drivers/net/wireless/broadcom/b43legacy/debugfs.c
index 090910ea259e..82ef56ed7ca1 100644
--- a/drivers/net/wireless/broadcom/b43legacy/debugfs.c
+++ b/drivers/net/wireless/broadcom/b43legacy/debugfs.c
@@ -221,7 +221,8 @@ static ssize_t b43legacy_debugfs_read(struct file *file, char __user *userbuf,
 		goto out_unlock;
 	}
 
-	dfops = container_of(file->f_op, struct b43legacy_debugfs_fops, fops);
+	dfops = container_of(debugfs_real_fops(file),
+			     struct b43legacy_debugfs_fops, fops);
 	if (!dfops->read) {
 		err = -ENOSYS;
 		goto out_unlock;
@@ -287,7 +288,8 @@ static ssize_t b43legacy_debugfs_write(struct file *file,
 		goto out_unlock;
 	}
 
-	dfops = container_of(file->f_op, struct b43legacy_debugfs_fops, fops);
+	dfops = container_of(debugfs_real_fops(file),
+			     struct b43legacy_debugfs_fops, fops);
 	if (!dfops->write) {
 		err = -ENOSYS;
 		goto out_unlock;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
index d1bc51f92686..038a960c5104 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
@@ -194,7 +194,7 @@ retry:
 	}
 
 	/* Check info buffer */
-	info = (void *)&msg[1];
+	info = (void *)&bcdc->buf[0];
 
 	/* Copy info buffer */
 	if (buf) {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
index f549c25608d6..72139b579b18 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
@@ -420,7 +420,7 @@ u8 brcmf_sdiod_regrb(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret)
 
 u32 brcmf_sdiod_regrl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret)
 {
-	u32 data;
+	u32 data = 0;
 	int retval;
 
 	brcmf_dbg(SDIO, "addr:0x%08x\n", addr);
@@ -1101,6 +1101,7 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = {
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43341),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43362),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4335_4339),
+	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4339),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43430),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4345),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4354),
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index 2628d5e12c64..b777e1b2f87a 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -1595,15 +1595,9 @@ static s32 brcmf_set_auth_type(struct net_device *ndev,
 		val = 1;
 		brcmf_dbg(CONN, "shared key\n");
 		break;
-	case NL80211_AUTHTYPE_AUTOMATIC:
-		val = 2;
-		brcmf_dbg(CONN, "automatic\n");
-		break;
-	case NL80211_AUTHTYPE_NETWORK_EAP:
-		brcmf_dbg(CONN, "network eap\n");
 	default:
 		val = 2;
-		brcmf_err("invalid auth type (%d)\n", sme->auth_type);
+		brcmf_dbg(CONN, "automatic, auth type (%d)\n", sme->auth_type);
 		break;
 	}
 
@@ -2533,7 +2527,7 @@ static void brcmf_fill_bss_param(struct brcmf_if *ifp, struct station_info *si)
 				     WL_BSS_INFO_MAX);
 	if (err) {
 		brcmf_err("Failed to get bss info (%d)\n", err);
-		return;
+		goto out_kfree;
 	}
 	si->filled |= BIT(NL80211_STA_INFO_BSS_PARAM);
 	si->bss_param.beacon_interval = le16_to_cpu(buf->bss_le.beacon_period);
@@ -2545,6 +2539,9 @@ static void brcmf_fill_bss_param(struct brcmf_if *ifp, struct station_info *si)
 		si->bss_param.flags |= BSS_PARAM_FLAGS_SHORT_PREAMBLE;
 	if (capability & WLAN_CAPABILITY_SHORT_SLOT_TIME)
 		si->bss_param.flags |= BSS_PARAM_FLAGS_SHORT_SLOT_TIME;
+
+out_kfree:
+	kfree(buf);
 }
 
 static s32
@@ -3703,6 +3700,7 @@ static void brcmf_configure_wowl(struct brcmf_cfg80211_info *cfg,
 				 struct cfg80211_wowlan *wowl)
 {
 	u32 wowl_config;
+	struct brcmf_wowl_wakeind_le wowl_wakeind;
 	u32 i;
 
 	brcmf_dbg(TRACE, "Suspend, wowl config.\n");
@@ -3744,8 +3742,9 @@ static void brcmf_configure_wowl(struct brcmf_cfg80211_info *cfg,
 	if (!test_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state))
 		wowl_config |= BRCMF_WOWL_UNASSOC;
 
-	brcmf_fil_iovar_data_set(ifp, "wowl_wakeind", "clear",
-				 sizeof(struct brcmf_wowl_wakeind_le));
+	memcpy(&wowl_wakeind, "clear", 6);
+	brcmf_fil_iovar_data_set(ifp, "wowl_wakeind", &wowl_wakeind,
+				 sizeof(wowl_wakeind));
 	brcmf_fil_iovar_int_set(ifp, "wowl", wowl_config);
 	brcmf_fil_iovar_int_set(ifp, "wowl_activate", 1);
 	brcmf_bus_wowl_config(cfg->pub->bus_if, true);
@@ -3884,11 +3883,11 @@ brcmf_cfg80211_del_pmksa(struct wiphy *wiphy, struct net_device *ndev,
 	if (!check_vif_up(ifp->vif))
 		return -EIO;
 
-	brcmf_dbg(CONN, "del_pmksa - PMK bssid = %pM\n", &pmksa->bssid);
+	brcmf_dbg(CONN, "del_pmksa - PMK bssid = %pM\n", pmksa->bssid);
 
 	npmk = le32_to_cpu(cfg->pmk_list.npmk);
 	for (i = 0; i < npmk; i++)
-		if (!memcmp(&pmksa->bssid, &pmk[i].bssid, ETH_ALEN))
+		if (!memcmp(pmksa->bssid, pmk[i].bssid, ETH_ALEN))
 			break;
 
 	if ((npmk > 0) && (i < npmk)) {
@@ -4502,6 +4501,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
 	u16 chanspec = chandef_to_chanspec(&cfg->d11inf, &settings->chandef);
 	bool mbss;
 	int is_11d;
+	bool supports_11d;
 
 	brcmf_dbg(TRACE, "ctrlchn=%d, center=%d, bw=%d, beacon_interval=%d, dtim_period=%d,\n",
 		  settings->chandef.chan->hw_value,
@@ -4514,11 +4514,16 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
 	mbss = ifp->vif->mbss;
 
 	/* store current 11d setting */
-	brcmf_fil_cmd_int_get(ifp, BRCMF_C_GET_REGULATORY, &ifp->vif->is_11d);
-	country_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail,
-				      settings->beacon.tail_len,
-				      WLAN_EID_COUNTRY);
-	is_11d = country_ie ? 1 : 0;
+	if (brcmf_fil_cmd_int_get(ifp, BRCMF_C_GET_REGULATORY,
+				  &ifp->vif->is_11d)) {
+		supports_11d = false;
+	} else {
+		country_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail,
+					      settings->beacon.tail_len,
+					      WLAN_EID_COUNTRY);
+		is_11d = country_ie ? 1 : 0;
+		supports_11d = true;
+	}
 
 	memset(&ssid_le, 0, sizeof(ssid_le));
 	if (settings->ssid == NULL || settings->ssid_len == 0) {
@@ -4527,7 +4532,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
 				(u8 *)&settings->beacon.head[ie_offset],
 				settings->beacon.head_len - ie_offset,
 				WLAN_EID_SSID);
-		if (!ssid_ie)
+		if (!ssid_ie || ssid_ie->len > IEEE80211_MAX_SSID_LEN)
 			return -EINVAL;
 
 		memcpy(ssid_le.SSID, ssid_ie->data, ssid_ie->len);
@@ -4577,7 +4582,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
 
 	/* Parameters shared by all radio interfaces */
 	if (!mbss) {
-		if (is_11d != ifp->vif->is_11d) {
+		if ((supports_11d) && (is_11d != ifp->vif->is_11d)) {
 			err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_REGULATORY,
 						    is_11d);
 			if (err < 0) {
@@ -4619,7 +4624,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev,
 			brcmf_err("SET INFRA error %d\n", err);
 			goto exit;
 		}
-	} else if (WARN_ON(is_11d != ifp->vif->is_11d)) {
+	} else if (WARN_ON(supports_11d && (is_11d != ifp->vif->is_11d))) {
 		/* Multiple-BSS should use same 11d configuration */
 		err = -EINVAL;
 		goto exit;
@@ -4753,11 +4758,8 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev)
 			brcmf_err("setting INFRA mode failed %d\n", err);
 		if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS))
 			brcmf_fil_iovar_int_set(ifp, "mbss", 0);
-		err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_REGULATORY,
-					    ifp->vif->is_11d);
-		if (err < 0)
-			brcmf_err("restoring REGULATORY setting failed %d\n",
-				  err);
+		brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_REGULATORY,
+				      ifp->vif->is_11d);
 		/* Bring device back up so it can be used again */
 		err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_UP, 1);
 		if (err < 0)
@@ -5635,7 +5637,7 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp,
 		  ifevent->action, ifevent->flags, ifevent->ifidx,
 		  ifevent->bsscfgidx);
 
-	mutex_lock(&event->vif_event_lock);
+	spin_lock(&event->vif_event_lock);
 	event->action = ifevent->action;
 	vif = event->vif;
 
@@ -5643,7 +5645,7 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp,
 	case BRCMF_E_IF_ADD:
 		/* waiting process may have timed out */
 		if (!cfg->vif_event.vif) {
-			mutex_unlock(&event->vif_event_lock);
+			spin_unlock(&event->vif_event_lock);
 			return -EBADF;
 		}
 
@@ -5654,24 +5656,24 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp,
 			ifp->ndev->ieee80211_ptr = &vif->wdev;
 			SET_NETDEV_DEV(ifp->ndev, wiphy_dev(cfg->wiphy));
 		}
-		mutex_unlock(&event->vif_event_lock);
+		spin_unlock(&event->vif_event_lock);
 		wake_up(&event->vif_wq);
 		return 0;
 
 	case BRCMF_E_IF_DEL:
-		mutex_unlock(&event->vif_event_lock);
+		spin_unlock(&event->vif_event_lock);
 		/* event may not be upon user request */
 		if (brcmf_cfg80211_vif_event_armed(cfg))
 			wake_up(&event->vif_wq);
 		return 0;
 
 	case BRCMF_E_IF_CHANGE:
-		mutex_unlock(&event->vif_event_lock);
+		spin_unlock(&event->vif_event_lock);
 		wake_up(&event->vif_wq);
 		return 0;
 
 	default:
-		mutex_unlock(&event->vif_event_lock);
+		spin_unlock(&event->vif_event_lock);
 		break;
 	}
 	return -EINVAL;
@@ -5792,7 +5794,7 @@ static void wl_deinit_priv(struct brcmf_cfg80211_info *cfg)
 static void init_vif_event(struct brcmf_cfg80211_vif_event *event)
 {
 	init_waitqueue_head(&event->vif_wq);
-	mutex_init(&event->vif_event_lock);
+	spin_lock_init(&event->vif_event_lock);
 }
 
 static s32 brcmf_dongle_roam(struct brcmf_if *ifp)
@@ -6691,9 +6693,9 @@ static inline bool vif_event_equals(struct brcmf_cfg80211_vif_event *event,
 {
 	u8 evt_action;
 
-	mutex_lock(&event->vif_event_lock);
+	spin_lock(&event->vif_event_lock);
 	evt_action = event->action;
-	mutex_unlock(&event->vif_event_lock);
+	spin_unlock(&event->vif_event_lock);
 	return evt_action == action;
 }
 
@@ -6702,10 +6704,10 @@ void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg,
 {
 	struct brcmf_cfg80211_vif_event *event = &cfg->vif_event;
 
-	mutex_lock(&event->vif_event_lock);
+	spin_lock(&event->vif_event_lock);
 	event->vif = vif;
 	event->action = 0;
-	mutex_unlock(&event->vif_event_lock);
+	spin_unlock(&event->vif_event_lock);
 }
 
 bool brcmf_cfg80211_vif_event_armed(struct brcmf_cfg80211_info *cfg)
@@ -6713,9 +6715,9 @@ bool brcmf_cfg80211_vif_event_armed(struct brcmf_cfg80211_info *cfg)
 	struct brcmf_cfg80211_vif_event *event = &cfg->vif_event;
 	bool armed;
 
-	mutex_lock(&event->vif_event_lock);
+	spin_lock(&event->vif_event_lock);
 	armed = event->vif != NULL;
-	mutex_unlock(&event->vif_event_lock);
+	spin_unlock(&event->vif_event_lock);
 
 	return armed;
 }
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
index 7d77f869b7f1..8889832c17e0 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
@@ -227,7 +227,7 @@ struct escan_info {
  */
 struct brcmf_cfg80211_vif_event {
 	wait_queue_head_t vif_wq;
-	struct mutex vif_event_lock;
+	spinlock_t vif_event_lock;
 	u8 action;
 	struct brcmf_cfg80211_vif *vif;
 };
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
index 8d16f0204985..5eaac13e2317 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
@@ -136,27 +136,6 @@ static void _brcmf_set_multicast_list(struct work_struct *work)
 			  err);
 }
 
-static void
-_brcmf_set_mac_address(struct work_struct *work)
-{
-	struct brcmf_if *ifp;
-	s32 err;
-
-	ifp = container_of(work, struct brcmf_if, setmacaddr_work);
-
-	brcmf_dbg(TRACE, "Enter, bsscfgidx=%d\n", ifp->bsscfgidx);
-
-	err = brcmf_fil_iovar_data_set(ifp, "cur_etheraddr", ifp->mac_addr,
-				       ETH_ALEN);
-	if (err < 0) {
-		brcmf_err("Setting cur_etheraddr failed, %d\n", err);
-	} else {
-		brcmf_dbg(TRACE, "MAC address updated to %pM\n",
-			  ifp->mac_addr);
-		memcpy(ifp->ndev->dev_addr, ifp->mac_addr, ETH_ALEN);
-	}
-}
-
 #if IS_ENABLED(CONFIG_IPV6)
 static void _brcmf_update_ndtable(struct work_struct *work)
 {
@@ -190,10 +169,20 @@ static int brcmf_netdev_set_mac_address(struct net_device *ndev, void *addr)
 {
 	struct brcmf_if *ifp = netdev_priv(ndev);
 	struct sockaddr *sa = (struct sockaddr *)addr;
+	int err;
 
-	memcpy(&ifp->mac_addr, sa->sa_data, ETH_ALEN);
-	schedule_work(&ifp->setmacaddr_work);
-	return 0;
+	brcmf_dbg(TRACE, "Enter, bsscfgidx=%d\n", ifp->bsscfgidx);
+
+	err = brcmf_fil_iovar_data_set(ifp, "cur_etheraddr", sa->sa_data,
+				       ETH_ALEN);
+	if (err < 0) {
+		brcmf_err("Setting cur_etheraddr failed, %d\n", err);
+	} else {
+		brcmf_dbg(TRACE, "updated to %pM\n", sa->sa_data);
+		memcpy(ifp->mac_addr, sa->sa_data, ETH_ALEN);
+		memcpy(ifp->ndev->dev_addr, ifp->mac_addr, ETH_ALEN);
+	}
+	return err;
 }
 
 static void brcmf_netdev_set_multicast_list(struct net_device *ndev)
@@ -519,13 +508,9 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool rtnl_locked)
 	ndev->needed_headroom += drvr->hdrlen;
 	ndev->ethtool_ops = &brcmf_ethtool_ops;
 
-	drvr->rxsz = ndev->mtu + ndev->hard_header_len +
-			      drvr->hdrlen;
-
 	/* set the mac address */
 	memcpy(ndev->dev_addr, ifp->mac_addr, ETH_ALEN);
 
-	INIT_WORK(&ifp->setmacaddr_work, _brcmf_set_mac_address);
 	INIT_WORK(&ifp->multicast_work, _brcmf_set_multicast_list);
 	INIT_WORK(&ifp->ndoffload_work, _brcmf_update_ndtable);
 
@@ -730,7 +715,6 @@ static void brcmf_del_if(struct brcmf_pub *drvr, s32 bsscfgidx,
 		}
 
 		if (ifp->ndev->netdev_ops == &brcmf_netdev_ops_pri) {
-			cancel_work_sync(&ifp->setmacaddr_work);
 			cancel_work_sync(&ifp->multicast_work);
 			cancel_work_sync(&ifp->ndoffload_work);
 		}
@@ -743,7 +727,7 @@ static void brcmf_del_if(struct brcmf_pub *drvr, s32 bsscfgidx,
 		 * serious troublesome side effects. The p2p module will clean
 		 * up the ifp if needed.
 		 */
-		brcmf_p2p_ifp_removed(ifp);
+		brcmf_p2p_ifp_removed(ifp, rtnl_locked);
 		kfree(ifp);
 	}
 }
@@ -886,9 +870,12 @@ static int brcmf_inet6addr_changed(struct notifier_block *nb,
 		}
 		break;
 	case NETDEV_DOWN:
-		if (i < NDOL_MAX_ENTRIES)
-			for (; i < ifp->ipv6addr_idx; i++)
+		if (i < NDOL_MAX_ENTRIES) {
+			for (; i < ifp->ipv6addr_idx - 1; i++)
 				table[i] = table[i + 1];
+			memset(&table[i], 0, sizeof(table[i]));
+			ifp->ipv6addr_idx--;
+		}
 		break;
 	default:
 		break;
@@ -1061,8 +1048,7 @@ fail:
 		brcmf_fws_del_interface(ifp);
 		brcmf_fws_deinit(drvr);
 	}
-	if (ifp)
-		brcmf_net_detach(ifp->ndev, false);
+	brcmf_net_detach(ifp->ndev, false);
 	if (p2p_ifp)
 		brcmf_net_detach(p2p_ifp->ndev, false);
 	drvr->iflist[0] = NULL;
@@ -1169,7 +1155,8 @@ int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp)
 				 !brcmf_get_pend_8021x_cnt(ifp),
 				 MAX_WAIT_FOR_8021X_TX);
 
-	WARN_ON(!err);
+	if (!err)
+		brcmf_err("Timed out waiting for no pending 802.1x packets\n");
 
 	return !err;
 }
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
index 8fa34cad5a96..c94dcab260d0 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
@@ -112,15 +112,11 @@ struct brcmf_pub {
 
 	/* Internal brcmf items */
 	uint hdrlen;		/* Total BRCMF header length (proto + bus) */
-	uint rxsz;		/* Rx buffer size bus module should use */
 
 	/* Dongle media info */
 	char fwver[BRCMF_DRIVER_FIRMWARE_VERSION_LEN];
 	u8 mac[ETH_ALEN];		/* MAC address obtained from dongle */
 
-	/* Multicast data packets sent to dongle */
-	unsigned long tx_multicast;
-
 	struct mac_address addresses[BRCMF_MAX_IFS];
 
 	struct brcmf_if *iflist[BRCMF_MAX_IFS];
@@ -176,7 +172,6 @@ enum brcmf_netif_stop_reason {
  * @vif: points to cfg80211 specific interface information.
  * @ndev: associated network device.
  * @stats: interface specific network statistics.
- * @setmacaddr_work: worker object for setting mac address.
  * @multicast_work: worker object for multicast provisioning.
  * @ndoffload_work: worker object for neighbor discovery offload configuration.
  * @fws_desc: interface specific firmware-signalling descriptor.
@@ -193,7 +188,6 @@ struct brcmf_if {
 	struct brcmf_cfg80211_vif *vif;
 	struct net_device *ndev;
 	struct net_device_stats stats;
-	struct work_struct setmacaddr_work;
 	struct work_struct multicast_work;
 	struct work_struct ndoffload_work;
 	struct brcmf_fws_mac_descriptor *fws_desc;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
index 7e269f9aa607..d0b738da2458 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
@@ -234,13 +234,20 @@ static void brcmf_flowring_block(struct brcmf_flowring *flow, u16 flowid,
 
 void brcmf_flowring_delete(struct brcmf_flowring *flow, u16 flowid)
 {
+	struct brcmf_bus *bus_if = dev_get_drvdata(flow->dev);
 	struct brcmf_flowring_ring *ring;
+	struct brcmf_if *ifp;
 	u16 hash_idx;
+	u8 ifidx;
 	struct sk_buff *skb;
 
 	ring = flow->rings[flowid];
 	if (!ring)
 		return;
+
+	ifidx = brcmf_flowring_ifidx_get(flow, flowid);
+	ifp = brcmf_get_ifp(bus_if->drvr, ifidx);
+
 	brcmf_flowring_block(flow, flowid, false);
 	hash_idx = ring->hash_id;
 	flow->hash[hash_idx].ifidx = BRCMF_FLOWRING_INVALID_IFIDX;
@@ -249,7 +256,7 @@ void brcmf_flowring_delete(struct brcmf_flowring *flow, u16 flowid)
 
 	skb = skb_dequeue(&ring->skblist);
 	while (skb) {
-		brcmu_pkt_buf_free_skb(skb);
+		brcmf_txfinalize(ifp, skb, false);
 		skb = skb_dequeue(&ring->skblist);
 	}
 
@@ -495,14 +502,18 @@ void brcmf_flowring_add_tdls_peer(struct brcmf_flowring *flow, int ifidx,
 	} else {
 		search = flow->tdls_entry;
 		if (memcmp(search->mac, peer, ETH_ALEN) == 0)
-			return;
+			goto free_entry;
 		while (search->next) {
 			search = search->next;
 			if (memcmp(search->mac, peer, ETH_ALEN) == 0)
-				return;
+				goto free_entry;
 		}
 		search->next = tdls_entry;
 	}
 
 	flow->tdls_active = true;
+	return;
+
+free_entry:
+	kfree(tdls_entry);
 }
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
index 9f9024a7bd64..a190f535efc9 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
@@ -2104,8 +2104,6 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb)
 	if ((skb->priority == 0) || (skb->priority > 7))
 		skb->priority = cfg80211_classify8021d(skb, NULL);
 
-	drvr->tx_multicast += !!multicast;
-
 	if (fws->avoid_queueing) {
 		rc = brcmf_proto_txdata(drvr, ifp->ifidx, 0, skb);
 		if (rc < 0)
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
index 66f942f7448e..de19c7c92bc6 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
@@ -2297,7 +2297,7 @@ int brcmf_p2p_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev)
 	return err;
 }
 
-void brcmf_p2p_ifp_removed(struct brcmf_if *ifp)
+void brcmf_p2p_ifp_removed(struct brcmf_if *ifp, bool rtnl_locked)
 {
 	struct brcmf_cfg80211_info *cfg;
 	struct brcmf_cfg80211_vif *vif;
@@ -2306,9 +2306,11 @@ void brcmf_p2p_ifp_removed(struct brcmf_if *ifp)
 	vif = ifp->vif;
 	cfg = wdev_to_cfg(&vif->wdev);
 	cfg->p2p.bss_idx[P2PAPI_BSSCFG_DEVICE].vif = NULL;
-	rtnl_lock();
+	if (!rtnl_locked)
+		rtnl_lock();
 	cfg80211_unregister_wdev(&vif->wdev);
-	rtnl_unlock();
+	if (!rtnl_locked)
+		rtnl_unlock();
 	brcmf_free_vif(vif);
 }
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h
index a3bd18c2360b..8ce9447533ef 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h
@@ -155,7 +155,7 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name,
 int brcmf_p2p_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev);
 int brcmf_p2p_ifchange(struct brcmf_cfg80211_info *cfg,
 		       enum brcmf_fil_p2p_if_types if_type);
-void brcmf_p2p_ifp_removed(struct brcmf_if *ifp);
+void brcmf_p2p_ifp_removed(struct brcmf_if *ifp, bool rtnl_locked);
 int brcmf_p2p_start_device(struct wiphy *wiphy, struct wireless_dev *wdev);
 void brcmf_p2p_stop_device(struct wiphy *wiphy, struct wireless_dev *wdev);
 int brcmf_p2p_scan_prep(struct wiphy *wiphy,
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
index 68ab3ac15650..b892dac70f4b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
@@ -313,6 +313,7 @@ struct rte_console {
 
 #define KSO_WAIT_US 50
 #define MAX_KSO_ATTEMPTS (PMU_MAX_TRANSITION_DLY/KSO_WAIT_US)
+#define BRCMF_SDIO_MAX_ACCESS_ERRORS	5
 
 /*
  * Conversion of 802.1D priority to precedence level
@@ -677,6 +678,7 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on)
 {
 	u8 wr_val = 0, rd_val, cmp_val, bmask;
 	int err = 0;
+	int err_cnt = 0;
 	int try_cnt = 0;
 
 	brcmf_dbg(TRACE, "Enter: on=%d\n", on);
@@ -712,9 +714,14 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on)
 		 */
 		rd_val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
 					   &err);
-		if (((rd_val & bmask) == cmp_val) && !err)
+		if (!err) {
+			if ((rd_val & bmask) == cmp_val)
+				break;
+			err_cnt = 0;
+		}
+		/* bail out upon subsequent access errors */
+		if (err && (err_cnt++ > BRCMF_SDIO_MAX_ACCESS_ERRORS))
 			break;
-
 		udelay(KSO_WAIT_US);
 		brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
 				  wr_val, &err);
@@ -3757,7 +3764,8 @@ static u32 brcmf_sdio_buscore_read32(void *ctx, u32 addr)
 	u32 val, rev;
 
 	val = brcmf_sdiod_regrl(sdiodev, addr, NULL);
-	if (sdiodev->func[0]->device == SDIO_DEVICE_ID_BROADCOM_4335_4339 &&
+	if ((sdiodev->func[0]->device == SDIO_DEVICE_ID_BROADCOM_4335_4339 ||
+	     sdiodev->func[0]->device == SDIO_DEVICE_ID_BROADCOM_4339) &&
 	    addr == CORE_CC_REG(SI_ENUM_BASE, chipid)) {
 		rev = (val & CID_REV_MASK) >> CID_REV_SHIFT;
 		if (rev >= 2) {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/tracepoint.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/tracepoint.c
index a10f35c5eb3d..fe6755944b7b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/tracepoint.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/tracepoint.c
@@ -19,6 +19,7 @@
 #ifndef __CHECKER__
 #define CREATE_TRACE_POINTS
 #include "tracepoint.h"
+#include "debug.h"
 
 void __brcmf_err(const char *func, const char *fmt, ...)
 {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
index 98b15a9a2779..2f978a39b58a 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
@@ -1099,15 +1099,11 @@ struct brcmf_usbdev *brcmf_usb_attach(struct brcmf_usbdev_info *devinfo,
 	devinfo->tx_freecount = ntxq;
 
 	devinfo->ctl_urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!devinfo->ctl_urb) {
-		brcmf_err("usb_alloc_urb (ctl) failed\n");
+	if (!devinfo->ctl_urb)
 		goto error;
-	}
 	devinfo->bulk_urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!devinfo->bulk_urb) {
-		brcmf_err("usb_alloc_urb (bulk) failed\n");
+	if (!devinfo->bulk_urb)
 		goto error;
-	}
 
 	return &devinfo->bus_pub;
 
@@ -1462,11 +1458,15 @@ static int brcmf_usb_reset_resume(struct usb_interface *intf)
 #define BRCMF_USB_DEVICE(dev_id)	\
 	{ USB_DEVICE(BRCM_USB_VENDOR_ID_BROADCOM, dev_id) }
 
+#define LINKSYS_USB_DEVICE(dev_id)	\
+	{ USB_DEVICE(BRCM_USB_VENDOR_ID_LINKSYS, dev_id) }
+
 static struct usb_device_id brcmf_usb_devid_table[] = {
 	BRCMF_USB_DEVICE(BRCM_USB_43143_DEVICE_ID),
 	BRCMF_USB_DEVICE(BRCM_USB_43236_DEVICE_ID),
 	BRCMF_USB_DEVICE(BRCM_USB_43242_DEVICE_ID),
 	BRCMF_USB_DEVICE(BRCM_USB_43569_DEVICE_ID),
+	LINKSYS_USB_DEVICE(BRCM_USB_43235_LINKSYS_DEVICE_ID),
 	{ USB_DEVICE(BRCM_USB_VENDOR_ID_LG, BRCM_USB_43242_LG_DEVICE_ID) },
 	/* special entry for device with firmware loaded and running */
 	BRCMF_USB_DEVICE(BRCM_USB_BCMFW_DEVICE_ID),
diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
index 3cc42bef6245..d0407d9ad782 100644
--- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
+++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
@@ -22,6 +22,7 @@
 
 #define BRCM_USB_VENDOR_ID_BROADCOM	0x0a5c
 #define BRCM_USB_VENDOR_ID_LG		0x043e
+#define BRCM_USB_VENDOR_ID_LINKSYS	0x13b1
 #define BRCM_PCIE_VENDOR_ID_BROADCOM	PCI_VENDOR_ID_BROADCOM
 
 /* Chipcommon Core Chip IDs */
@@ -58,6 +59,7 @@
 
 /* USB Device IDs */
 #define BRCM_USB_43143_DEVICE_ID	0xbd1e
+#define BRCM_USB_43235_LINKSYS_DEVICE_ID	0x0039
 #define BRCM_USB_43236_DEVICE_ID	0xbd17
 #define BRCM_USB_43242_DEVICE_ID	0xbd1f
 #define BRCM_USB_43242_LG_DEVICE_ID	0x3101
diff --git a/drivers/net/wireless/intel/iwlegacy/3945.c b/drivers/net/wireless/intel/iwlegacy/3945.c
index 209dc9988455..4db327a95414 100644
--- a/drivers/net/wireless/intel/iwlegacy/3945.c
+++ b/drivers/net/wireless/intel/iwlegacy/3945.c
@@ -2671,7 +2671,7 @@ const struct il_ops il3945_ops = {
 	.send_led_cmd = il3945_send_led_cmd,
 };
 
-static struct il_cfg il3945_bg_cfg = {
+static const struct il_cfg il3945_bg_cfg = {
 	.name = "3945BG",
 	.fw_name_pre = IL3945_FW_PRE,
 	.ucode_api_max = IL3945_UCODE_API_MAX,
@@ -2700,7 +2700,7 @@ static struct il_cfg il3945_bg_cfg = {
 	},
 };
 
-static struct il_cfg il3945_abg_cfg = {
+static const struct il_cfg il3945_abg_cfg = {
 	.name = "3945ABG",
 	.fw_name_pre = IL3945_FW_PRE,
 	.ucode_api_max = IL3945_UCODE_API_MAX,
diff --git a/drivers/net/wireless/intel/iwlegacy/common.h b/drivers/net/wireless/intel/iwlegacy/common.h
index 726ede391cb9..3bba521d2cd9 100644
--- a/drivers/net/wireless/intel/iwlegacy/common.h
+++ b/drivers/net/wireless/intel/iwlegacy/common.h
@@ -1320,7 +1320,7 @@ struct il_priv {
 	u64 timestamp;
 
 	union {
-#if defined(CONFIG_IWL3945) || defined(CONFIG_IWL3945_MODULE)
+#if IS_ENABLED(CONFIG_IWL3945)
 		struct {
 			void *shared_virt;
 			dma_addr_t shared_phys;
@@ -1351,7 +1351,7 @@ struct il_priv {
 
 		} _3945;
 #endif
-#if defined(CONFIG_IWL4965) || defined(CONFIG_IWL4965_MODULE)
+#if IS_ENABLED(CONFIG_IWL4965)
 		struct {
 			struct il_rx_phy_res last_phy_res;
 			bool last_phy_res_valid;
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/dvm/debugfs.c
index f6591c83d636..affe760c8c22 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/debugfs.c
@@ -2422,14 +2422,12 @@ int iwl_dbgfs_register(struct iwl_priv *priv, struct dentry *dbgfs_dir)
 	 */
 	if (priv->mac80211_registered) {
 		char buf[100];
-		struct dentry *mac80211_dir, *dev_dir, *root_dir;
+		struct dentry *mac80211_dir, *dev_dir;
 
 		dev_dir = dbgfs_dir->d_parent;
-		root_dir = dev_dir->d_parent;
 		mac80211_dir = priv->hw->wiphy->debugfsdir;
 
-		snprintf(buf, 100, "../../%s/%s", root_dir->d_name.name,
-			 dev_dir->d_name.name);
+		snprintf(buf, 100, "../../%pd2", dev_dir);
 
 		if (!debugfs_create_symlink("iwlwifi", mac80211_dir, buf))
 			goto err;
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c b/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c
index b662cf35b033..c7509c51e9d9 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c
@@ -46,15 +46,6 @@
  *
  ******************************************************************************/
 
-static inline const struct fw_img *
-iwl_get_ucode_image(struct iwl_priv *priv, enum iwl_ucode_type ucode_type)
-{
-	if (ucode_type >= IWL_UCODE_TYPE_MAX)
-		return NULL;
-
-	return &priv->fw->img[ucode_type];
-}
-
 /*
  *  Calibration
  */
@@ -330,7 +321,7 @@ int iwl_load_ucode_wait_alive(struct iwl_priv *priv,
 	enum iwl_ucode_type old_type;
 	static const u16 alive_cmd[] = { REPLY_ALIVE };
 
-	fw = iwl_get_ucode_image(priv, ucode_type);
+	fw = iwl_get_ucode_image(priv->fw, ucode_type);
 	if (WARN_ON(!fw))
 		return -EINVAL;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-7000.c b/drivers/net/wireless/intel/iwlwifi/iwl-7000.c
index 64690c14ff4d..d4b73dedf89b 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-7000.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-7000.c
@@ -73,13 +73,13 @@
 /* Highest firmware API version supported */
 #define IWL7260_UCODE_API_MAX	17
 #define IWL7265_UCODE_API_MAX	17
-#define IWL7265D_UCODE_API_MAX	24
-#define IWL3168_UCODE_API_MAX	24
+#define IWL7265D_UCODE_API_MAX	26
+#define IWL3168_UCODE_API_MAX	26
 
 /* Lowest firmware API version supported */
-#define IWL7260_UCODE_API_MIN	16
-#define IWL7265_UCODE_API_MIN	16
-#define IWL7265D_UCODE_API_MIN	16
+#define IWL7260_UCODE_API_MIN	17
+#define IWL7265_UCODE_API_MIN	17
+#define IWL7265D_UCODE_API_MIN	17
 #define IWL3168_UCODE_API_MIN	20
 
 /* NVM versions */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-8000.c b/drivers/net/wireless/intel/iwlwifi/iwl-8000.c
index 6c6725e808d4..d02ca1491d16 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-8000.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-8000.c
@@ -70,11 +70,11 @@
 #include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
-#define IWL8000_UCODE_API_MAX	24
-#define IWL8265_UCODE_API_MAX	24
+#define IWL8000_UCODE_API_MAX	26
+#define IWL8265_UCODE_API_MAX	26
 
 /* Lowest firmware API version supported */
-#define IWL8000_UCODE_API_MIN	16
+#define IWL8000_UCODE_API_MIN	17
 #define IWL8265_UCODE_API_MIN	20
 
 /* NVM versions */
@@ -212,6 +212,17 @@ const struct iwl_cfg iwl8265_2ac_cfg = {
 	.vht_mu_mimo_supported = true,
 };
 
+const struct iwl_cfg iwl8275_2ac_cfg = {
+	.name = "Intel(R) Dual Band Wireless AC 8275",
+	.fw_name_pre = IWL8265_FW_PRE,
+	IWL_DEVICE_8265,
+	.ht_params = &iwl8000_ht_params,
+	.nvm_ver = IWL8000_NVM_VERSION,
+	.nvm_calib_ver = IWL8000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.vht_mu_mimo_supported = true,
+};
+
 const struct iwl_cfg iwl4165_2ac_cfg = {
 	.name = "Intel(R) Dual Band Wireless AC 4165",
 	.fw_name_pre = IWL8000_FW_PRE,
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-9000.c b/drivers/net/wireless/intel/iwlwifi/iwl-9000.c
index fbaf705f3fa7..ff850410d897 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-9000.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-9000.c
@@ -55,10 +55,10 @@
 #include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
-#define IWL9000_UCODE_API_MAX	24
+#define IWL9000_UCODE_API_MAX	26
 
 /* Lowest firmware API version supported */
-#define IWL9000_UCODE_API_MIN	16
+#define IWL9000_UCODE_API_MIN	17
 
 /* NVM versions */
 #define IWL9000_NVM_VERSION		0x0a1d
@@ -72,15 +72,15 @@
 #define IWL9000_SMEM_OFFSET		0x400000
 #define IWL9000_SMEM_LEN		0x68000
 
-#define  IWL9000_FW_PRE "iwlwifi-9000-pu-a0-lc-a0-"
+#define  IWL9000_FW_PRE "iwlwifi-9000-pu-a0-jf-a0-"
 #define  IWL9260_FW_PRE "iwlwifi-9260-th-a0-jf-a0-"
-#define  IWL9260LC_FW_PRE "iwlwifi-9260-th-a0-lc-a0-"
+#define  IWL9000LC_FW_PRE "iwlwifi-9000-pu-a0-lc-a0-"
 #define IWL9000_MODULE_FIRMWARE(api) \
 	IWL9000_FW_PRE "-" __stringify(api) ".ucode"
 #define IWL9260_MODULE_FIRMWARE(api) \
 	IWL9260_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL9260LC_MODULE_FIRMWARE(api) \
-	IWL9260LC_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL9000LC_MODULE_FIRMWARE(api) \
+	IWL9000LC_FW_PRE "-" __stringify(api) ".ucode"
 
 #define NVM_HW_SECTION_NUM_FAMILY_9000		10
 
@@ -146,41 +146,73 @@ static const struct iwl_tt_params iwl9000_tt_params = {
 	.mac_addr_from_csr = true,					\
 	.rf_id = true
 
+const struct iwl_cfg iwl9160_2ac_cfg = {
+	.name = "Intel(R) Dual Band Wireless AC 9160",
+	.fw_name_pre = IWL9260_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
 const struct iwl_cfg iwl9260_2ac_cfg = {
-		.name = "Intel(R) Dual Band Wireless AC 9260",
-		.fw_name_pre = IWL9260_FW_PRE,
-		IWL_DEVICE_9000,
-		.ht_params = &iwl9000_ht_params,
-		.nvm_ver = IWL9000_NVM_VERSION,
-		.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
-		.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.name = "Intel(R) Dual Band Wireless AC 9260",
+	.fw_name_pre = IWL9260_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+const struct iwl_cfg iwl9270_2ac_cfg = {
+	.name = "Intel(R) Dual Band Wireless AC 9270",
+	.fw_name_pre = IWL9260_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+const struct iwl_cfg iwl9460_2ac_cfg = {
+	.name = "Intel(R) Dual Band Wireless AC 9460",
+	.fw_name_pre = IWL9000_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.integrated = true,
+};
+
+const struct iwl_cfg iwl9560_2ac_cfg = {
+	.name = "Intel(R) Dual Band Wireless AC 9560",
+	.fw_name_pre = IWL9000_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.integrated = true,
 };
 
 /*
  * TODO the struct below is for internal testing only this should be
  * removed by EO 2016~
  */
-const struct iwl_cfg iwl9260lc_2ac_cfg = {
-		.name = "Intel(R) Dual Band Wireless AC 9260",
-		.fw_name_pre = IWL9260LC_FW_PRE,
-		IWL_DEVICE_9000,
-		.ht_params = &iwl9000_ht_params,
-		.nvm_ver = IWL9000_NVM_VERSION,
-		.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
-		.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
-};
-
-const struct iwl_cfg iwl5165_2ac_cfg = {
-		.name = "Intel(R) Dual Band Wireless AC 5165",
-		.fw_name_pre = IWL9000_FW_PRE,
-		IWL_DEVICE_9000,
-		.ht_params = &iwl9000_ht_params,
-		.nvm_ver = IWL9000_NVM_VERSION,
-		.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
-		.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
-		.integrated = true,
+const struct iwl_cfg iwl9000lc_2ac_cfg = {
+	.name = "Intel(R) Dual Band Wireless AC 9000",
+	.fw_name_pre = IWL9000LC_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.integrated = true,
 };
 
 MODULE_FIRMWARE(IWL9000_MODULE_FIRMWARE(IWL9000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL9260_MODULE_FIRMWARE(IWL9000_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL9260LC_MODULE_FIRMWARE(IWL9000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL9000LC_MODULE_FIRMWARE(IWL9000_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-a000.c b/drivers/net/wireless/intel/iwlwifi/iwl-a000.c
index 4d78232c8afe..ea1618525878 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-a000.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-a000.c
@@ -55,7 +55,7 @@
 #include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
-#define IWL_A000_UCODE_API_MAX	24
+#define IWL_A000_UCODE_API_MAX	26
 
 /* Lowest firmware API version supported */
 #define IWL_A000_UCODE_API_MIN	24
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
index 423b23320d4f..2660cc4b9f8c 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
@@ -359,7 +359,6 @@ struct iwl_cfg {
 	    high_temp:1,
 	    mac_addr_from_csr:1,
 	    lp_xtal_workaround:1,
-	    no_power_up_nic_in_init:1,
 	    disable_dummy_notification:1,
 	    apmg_not_supported:1,
 	    mq_rx_supported:1,
@@ -445,13 +444,17 @@ extern const struct iwl_cfg iwl7265d_n_cfg;
 extern const struct iwl_cfg iwl8260_2n_cfg;
 extern const struct iwl_cfg iwl8260_2ac_cfg;
 extern const struct iwl_cfg iwl8265_2ac_cfg;
+extern const struct iwl_cfg iwl8275_2ac_cfg;
 extern const struct iwl_cfg iwl4165_2ac_cfg;
 extern const struct iwl_cfg iwl8260_2ac_sdio_cfg;
 extern const struct iwl_cfg iwl8265_2ac_sdio_cfg;
 extern const struct iwl_cfg iwl4165_2ac_sdio_cfg;
+extern const struct iwl_cfg iwl9000lc_2ac_cfg;
+extern const struct iwl_cfg iwl9160_2ac_cfg;
 extern const struct iwl_cfg iwl9260_2ac_cfg;
-extern const struct iwl_cfg iwl9260lc_2ac_cfg;
-extern const struct iwl_cfg iwl5165_2ac_cfg;
+extern const struct iwl_cfg iwl9270_2ac_cfg;
+extern const struct iwl_cfg iwl9460_2ac_cfg;
+extern const struct iwl_cfg iwl9560_2ac_cfg;
 extern const struct iwl_cfg iwla000_2ac_cfg;
 #endif /* CONFIG_IWLMVM */
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
index 871ad02fdb17..d73e9d436027 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
@@ -589,6 +589,8 @@ enum dtd_diode_reg {
  * Causes for the FH register interrupts
  */
 enum msix_fh_int_causes {
+	MSIX_FH_INT_CAUSES_Q0			= BIT(0),
+	MSIX_FH_INT_CAUSES_Q1			= BIT(1),
 	MSIX_FH_INT_CAUSES_D2S_CH0_NUM		= BIT(16),
 	MSIX_FH_INT_CAUSES_D2S_CH1_NUM		= BIT(17),
 	MSIX_FH_INT_CAUSES_S2D			= BIT(19),
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c
index 1d9dd153ef1c..50510fb6ab8c 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c
@@ -33,9 +33,6 @@
 #define CREATE_TRACE_POINTS
 #include "iwl-devtrace.h"
 
-EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_iowrite8);
-EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_ioread32);
-EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_iowrite32);
 EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_ucode_event);
 EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_ucode_error);
 EXPORT_TRACEPOINT_SYMBOL(iwlwifi_dev_ucode_cont_event);
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-fh.h b/drivers/net/wireless/intel/iwlwifi/iwl-fh.h
index 1d6f5d21a663..33ef5372d195 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-fh.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-fh.h
@@ -77,7 +77,6 @@
  */
 #define FH_MEM_LOWER_BOUND                   (0x1000)
 #define FH_MEM_UPPER_BOUND                   (0x2000)
-#define TFH_MEM_LOWER_BOUND                  (0xA06000)
 
 /**
  * Keep-Warm (KW) buffer base address.
@@ -120,7 +119,7 @@
 #define FH_MEM_CBBC_20_31_LOWER_BOUND		(FH_MEM_LOWER_BOUND + 0xB20)
 #define FH_MEM_CBBC_20_31_UPPER_BOUND		(FH_MEM_LOWER_BOUND + 0xB80)
 /* a000 TFD table address, 64 bit */
-#define TFH_TFDQ_CBB_TABLE			(TFH_MEM_LOWER_BOUND + 0x1C00)
+#define TFH_TFDQ_CBB_TABLE			(0x1C00)
 
 /* Find TFD CB base pointer for given queue */
 static inline unsigned int FH_MEM_CBBC_QUEUE(struct iwl_trans *trans,
@@ -156,7 +155,7 @@ static inline unsigned int FH_MEM_CBBC_QUEUE(struct iwl_trans *trans,
  * In case of DRAM read address which is not aligned to 128B, the TFH will
  * enable transfer size which doesn't cross 64B DRAM address boundary.
 */
-#define TFH_TRANSFER_MODE		(TFH_MEM_LOWER_BOUND + 0x1F40)
+#define TFH_TRANSFER_MODE		(0x1F40)
 #define TFH_TRANSFER_MAX_PENDING_REQ	0xc
 #define TFH_CHUNK_SIZE_128			BIT(8)
 #define TFH_CHUNK_SPLIT_MODE		BIT(10)
@@ -167,7 +166,7 @@ static inline unsigned int FH_MEM_CBBC_QUEUE(struct iwl_trans *trans,
  * the start of the TFD first TB.
  * In case of a DRAM Tx CMD update the TFH will update PN and Key ID
  */
-#define TFH_TXCMD_UPDATE_CFG		(TFH_MEM_LOWER_BOUND + 0x1F48)
+#define TFH_TXCMD_UPDATE_CFG		(0x1F48)
 /*
  * Controls TX DMA operation
  *
@@ -181,22 +180,22 @@ static inline unsigned int FH_MEM_CBBC_QUEUE(struct iwl_trans *trans,
  * set to 1 - interrupt is sent to the driver
  * Bit 0: Indicates the snoop configuration
 */
-#define TFH_SRV_DMA_CHNL0_CTRL	(TFH_MEM_LOWER_BOUND + 0x1F60)
+#define TFH_SRV_DMA_CHNL0_CTRL	(0x1F60)
 #define TFH_SRV_DMA_SNOOP	BIT(0)
 #define TFH_SRV_DMA_TO_DRIVER	BIT(24)
 #define TFH_SRV_DMA_START	BIT(31)
 
 /* Defines the DMA SRAM write start address to transfer a data block */
-#define TFH_SRV_DMA_CHNL0_SRAM_ADDR	(TFH_MEM_LOWER_BOUND + 0x1F64)
+#define TFH_SRV_DMA_CHNL0_SRAM_ADDR	(0x1F64)
 
 /* Defines the 64bits DRAM start address to read the DMA data block from */
-#define TFH_SRV_DMA_CHNL0_DRAM_ADDR	(TFH_MEM_LOWER_BOUND + 0x1F68)
+#define TFH_SRV_DMA_CHNL0_DRAM_ADDR	(0x1F68)
 
 /*
  * Defines the number of bytes to transfer from DRAM to SRAM.
  * Note that this register may be configured with non-dword aligned size.
  */
-#define TFH_SRV_DMA_CHNL0_BC	(TFH_MEM_LOWER_BOUND + 0x1F70)
+#define TFH_SRV_DMA_CHNL0_BC	(0x1F70)
 
 /**
  * Rx SRAM Control and Status Registers (RSCSR)
@@ -644,6 +643,7 @@ struct iwl_rb_status {
 #define TFD_QUEUE_BC_SIZE	(TFD_QUEUE_SIZE_MAX + TFD_QUEUE_SIZE_BC_DUP)
 #define IWL_TX_DMA_MASK        DMA_BIT_MASK(36)
 #define IWL_NUM_OF_TBS		20
+#define IWL_TFH_NUM_TBS		25
 
 static inline u8 iwl_get_dma_hi_addr(dma_addr_t addr)
 {
@@ -665,25 +665,29 @@ struct iwl_tfd_tb {
 } __packed;
 
 /**
- * struct iwl_tfd
+ * struct iwl_tfh_tb transmit buffer descriptor within transmit frame descriptor
  *
- * Transmit Frame Descriptor (TFD)
- *
- * @ __reserved1[3] reserved
- * @ num_tbs 0-4 number of active tbs
- *	     5   reserved
- * 	     6-7 padding (not used)
- * @ tbs[20]	transmit frame buffer descriptors
- * @ __pad 	padding
+ * This structure contains dma address and length of transmission address
  *
+ * @tb_len length of the tx buffer
+ * @addr 64 bits dma address
+ */
+struct iwl_tfh_tb {
+	__le16 tb_len;
+	__le64 addr;
+} __packed;
+
+/**
  * Each Tx queue uses a circular buffer of 256 TFDs stored in host DRAM.
  * Both driver and device share these circular buffers, each of which must be
- * contiguous 256 TFDs x 128 bytes-per-TFD = 32 KBytes
+ * contiguous 256 TFDs.
+ * For pre a000 HW it is 256 x 128 bytes-per-TFD = 32 KBytes
+ * For a000 HW and on it is 256 x 256 bytes-per-TFD = 65 KBytes
  *
  * Driver must indicate the physical address of the base of each
  * circular buffer via the FH_MEM_CBBC_QUEUE registers.
  *
- * Each TFD contains pointer/size information for up to 20 data buffers
+ * Each TFD contains pointer/size information for up to 20 / 25 data buffers
  * in host DRAM.  These buffers collectively contain the (one) frame described
  * by the TFD.  Each buffer must be a single contiguous block of memory within
  * itself, but buffers may be scattered in host DRAM.  Each buffer has max size
@@ -692,6 +696,16 @@ struct iwl_tfd_tb {
  *
  * A maximum of 255 (not 256!) TFDs may be on a queue waiting for Tx.
  */
+
+/**
+ * struct iwl_tfd - Transmit Frame Descriptor (TFD)
+ * @ __reserved1[3] reserved
+ * @ num_tbs 0-4 number of active tbs
+ *	     5   reserved
+ *	     6-7 padding (not used)
+ * @ tbs[20]	transmit frame buffer descriptors
+ * @ __pad	padding
+ */
 struct iwl_tfd {
 	u8 __reserved1[3];
 	u8 num_tbs;
@@ -699,6 +713,19 @@ struct iwl_tfd {
 	__le32 __pad;
 } __packed;
 
+/**
+ * struct iwl_tfh_tfd - Transmit Frame Descriptor (TFD)
+ * @ num_tbs 0-4 number of active tbs
+ *	     5 -15   reserved
+ * @ tbs[25]	transmit frame buffer descriptors
+ * @ __pad	padding
+ */
+struct iwl_tfh_tfd {
+	__le16 num_tbs;
+	struct iwl_tfh_tb tbs[IWL_TFH_NUM_TBS];
+	__le32 __pad;
+} __packed;
+
 /* Keep Warm Size */
 #define IWL_KW_SIZE 0x1000	/* 4k */
 
@@ -707,8 +734,13 @@ struct iwl_tfd {
 /**
  * struct iwlagn_schedq_bc_tbl scheduler byte count table
  *	base physical address provided by SCD_DRAM_BASE_ADDR
+ * For devices up to a000:
+ * @tfd_offset  0-12 - tx command byte count
+ *		12-16 - station index
+ * For a000 and on:
  * @tfd_offset  0-12 - tx command byte count
- *	       12-16 - station index
+ *		12-13 - number of 64 byte chunks
+ *		14-16 - reserved
  */
 struct iwlagn_scd_bc_tbl {
 	__le16 tfd_offset[TFD_QUEUE_BC_SIZE];
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-fw-file.h b/drivers/net/wireless/intel/iwlwifi/iwl-fw-file.h
index 1b1e045f8907..ceec5ca2b1ab 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-fw-file.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-fw-file.h
@@ -199,8 +199,6 @@ struct iwl_ucode_capa {
  * @IWL_UCODE_TLV_FLAGS_NEWSCAN: new uCode scan behavior on hidden SSID,
  *	treats good CRC threshold as a boolean
  * @IWL_UCODE_TLV_FLAGS_MFP: This uCode image supports MFP (802.11w).
- * @IWL_UCODE_TLV_FLAGS_P2P: This uCode image supports P2P.
- * @IWL_UCODE_TLV_FLAGS_DW_BC_TABLE: The SCD byte count table is in DWORDS
  * @IWL_UCODE_TLV_FLAGS_UAPSD_SUPPORT: This uCode image supports uAPSD
  * @IWL_UCODE_TLV_FLAGS_SHORT_BL: 16 entries of black list instead of 64 in scan
  *	offload profile config command.
@@ -210,36 +208,24 @@ struct iwl_ucode_capa {
  *	from the probe request template.
  * @IWL_UCODE_TLV_FLAGS_NEW_NSOFFL_SMALL: new NS offload (small version)
  * @IWL_UCODE_TLV_FLAGS_NEW_NSOFFL_LARGE: new NS offload (large version)
- * @IWL_UCODE_TLV_FLAGS_P2P_PM: P2P client supports PM as a stand alone MAC
- * @IWL_UCODE_TLV_FLAGS_P2P_BSS_PS_DCM: support power save on BSS station and
- *	P2P client interfaces simultaneously if they are in different bindings.
- * @IWL_UCODE_TLV_FLAGS_P2P_BSS_PS_SCM: support power save on BSS station and
- *	P2P client interfaces simultaneously if they are in same bindings.
  * @IWL_UCODE_TLV_FLAGS_UAPSD_SUPPORT: General support for uAPSD
  * @IWL_UCODE_TLV_FLAGS_P2P_PS_UAPSD: P2P client supports uAPSD power save
  * @IWL_UCODE_TLV_FLAGS_BCAST_FILTERING: uCode supports broadcast filtering.
- * @IWL_UCODE_TLV_FLAGS_GO_UAPSD: AP/GO interfaces support uAPSD clients
  * @IWL_UCODE_TLV_FLAGS_EBS_SUPPORT: this uCode image supports EBS.
  */
 enum iwl_ucode_tlv_flag {
 	IWL_UCODE_TLV_FLAGS_PAN			= BIT(0),
 	IWL_UCODE_TLV_FLAGS_NEWSCAN		= BIT(1),
 	IWL_UCODE_TLV_FLAGS_MFP			= BIT(2),
-	IWL_UCODE_TLV_FLAGS_P2P			= BIT(3),
-	IWL_UCODE_TLV_FLAGS_DW_BC_TABLE		= BIT(4),
 	IWL_UCODE_TLV_FLAGS_SHORT_BL		= BIT(7),
 	IWL_UCODE_TLV_FLAGS_D3_6_IPV6_ADDRS	= BIT(10),
 	IWL_UCODE_TLV_FLAGS_NO_BASIC_SSID	= BIT(12),
 	IWL_UCODE_TLV_FLAGS_NEW_NSOFFL_SMALL	= BIT(15),
 	IWL_UCODE_TLV_FLAGS_NEW_NSOFFL_LARGE	= BIT(16),
-	IWL_UCODE_TLV_FLAGS_P2P_PM		= BIT(21),
-	IWL_UCODE_TLV_FLAGS_BSS_P2P_PS_DCM	= BIT(22),
-	IWL_UCODE_TLV_FLAGS_BSS_P2P_PS_SCM	= BIT(23),
 	IWL_UCODE_TLV_FLAGS_UAPSD_SUPPORT	= BIT(24),
 	IWL_UCODE_TLV_FLAGS_EBS_SUPPORT		= BIT(25),
 	IWL_UCODE_TLV_FLAGS_P2P_PS_UAPSD	= BIT(26),
 	IWL_UCODE_TLV_FLAGS_BCAST_FILTERING	= BIT(29),
-	IWL_UCODE_TLV_FLAGS_GO_UAPSD		= BIT(30),
 };
 
 typedef unsigned int __bitwise__ iwl_ucode_tlv_api_t;
@@ -249,24 +235,21 @@ typedef unsigned int __bitwise__ iwl_ucode_tlv_api_t;
  * @IWL_UCODE_TLV_API_FRAGMENTED_SCAN: This ucode supports active dwell time
  *	longer than the passive one, which is essential for fragmented scan.
  * @IWL_UCODE_TLV_API_WIFI_MCC_UPDATE: ucode supports MCC updates with source.
- * @IWL_UCODE_TLV_API_WIDE_CMD_HDR: ucode supports wide command header
  * @IWL_UCODE_TLV_API_LQ_SS_PARAMS: Configure STBC/BFER via LQ CMD ss_params
  * @IWL_UCODE_TLV_API_NEW_VERSION: new versioning format
- * @IWL_UCODE_TLV_API_EXT_SCAN_PRIORITY: scan APIs use 8-level priority
- *	instead of 3.
- * @IWL_UCODE_TLV_API_TX_POWER_CHAIN: TX power API has larger command size
- *	(command version 3) that supports per-chain limits
+ * @IWL_UCODE_TLV_API_SCAN_TSF_REPORT: Scan start time reported in scan
+ *	iteration complete notification, and the timestamp reported for RX
+ *	received during scan, are reported in TSF of the mac specified in the
+ *	scan request.
  *
  * @NUM_IWL_UCODE_TLV_API: number of bits used
  */
 enum iwl_ucode_tlv_api {
 	IWL_UCODE_TLV_API_FRAGMENTED_SCAN	= (__force iwl_ucode_tlv_api_t)8,
 	IWL_UCODE_TLV_API_WIFI_MCC_UPDATE	= (__force iwl_ucode_tlv_api_t)9,
-	IWL_UCODE_TLV_API_WIDE_CMD_HDR		= (__force iwl_ucode_tlv_api_t)14,
 	IWL_UCODE_TLV_API_LQ_SS_PARAMS		= (__force iwl_ucode_tlv_api_t)18,
-	IWL_UCODE_TLV_API_NEW_VERSION           = (__force iwl_ucode_tlv_api_t)20,
-	IWL_UCODE_TLV_API_EXT_SCAN_PRIORITY	= (__force iwl_ucode_tlv_api_t)24,
-	IWL_UCODE_TLV_API_TX_POWER_CHAIN	= (__force iwl_ucode_tlv_api_t)27,
+	IWL_UCODE_TLV_API_NEW_VERSION		= (__force iwl_ucode_tlv_api_t)20,
+	IWL_UCODE_TLV_API_SCAN_TSF_REPORT	= (__force iwl_ucode_tlv_api_t)28,
 
 	NUM_IWL_UCODE_TLV_API
 #ifdef __CHECKER__
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-fw.h b/drivers/net/wireless/intel/iwlwifi/iwl-fw.h
index 74ea68d1063c..5f229556339a 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-fw.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-fw.h
@@ -329,4 +329,13 @@ iwl_fw_dbg_conf_usniffer(const struct iwl_fw *fw, u8 id)
 	return conf_tlv->usniffer;
 }
 
+static inline const struct fw_img *
+iwl_get_ucode_image(const struct iwl_fw *fw, enum iwl_ucode_type ucode_type)
+{
+	if (ucode_type >= IWL_UCODE_TYPE_MAX)
+		return NULL;
+
+	return &fw->img[ucode_type];
+}
+
 #endif  /* __iwl_fw_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-io.c b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
index 92c8b5f9a9cb..a9f69fdd170b 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-io.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
@@ -267,7 +267,7 @@ static const char *get_rfh_string(int cmd)
 		IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_WIDX, i);
 		IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_RIDX, i);
 		IWL_CMD_MQ(cmd, RFH_Q_URBD_STTS_WPTR_LSB, i);
-	};
+	}
 
 	switch (cmd) {
 	IWL_CMD(RFH_RXF_DMA_CFG);
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-notif-wait.c b/drivers/net/wireless/intel/iwlwifi/iwl-notif-wait.c
index 8aa1f2b7fdfc..88f260db3744 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-notif-wait.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-notif-wait.c
@@ -99,8 +99,12 @@ void iwl_notification_wait_notify(struct iwl_notif_wait_data *notif_wait,
 				continue;
 
 			for (i = 0; i < w->n_cmds; i++) {
-				if (w->cmds[i] ==
-				    WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)) {
+				u16 rec_id = WIDE_ID(pkt->hdr.group_id,
+						     pkt->hdr.cmd);
+
+				if (w->cmds[i] == rec_id ||
+				    (!iwl_cmd_groupid(w->cmds[i]) &&
+				     DEF_ID(w->cmds[i]) == rec_id)) {
 					found = true;
 					break;
 				}
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
index 43f8f7d45ddb..3bd6fc1b76d4 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
@@ -67,6 +67,7 @@
 #include <linux/export.h>
 #include <linux/etherdevice.h>
 #include <linux/pci.h>
+#include <linux/acpi.h>
 #include "iwl-drv.h"
 #include "iwl-modparams.h"
 #include "iwl-nvm-parse.h"
@@ -564,11 +565,16 @@ static void iwl_set_hw_address_from_csr(struct iwl_trans *trans,
 	__le32 mac_addr0 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR0_STRAP));
 	__le32 mac_addr1 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR1_STRAP));
 
-	/* If OEM did not fuse address - get it from OTP */
-	if (!mac_addr0 && !mac_addr1) {
-		mac_addr0 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR0_OTP));
-		mac_addr1 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR1_OTP));
-	}
+	iwl_flip_hw_address(mac_addr0, mac_addr1, data->hw_addr);
+	/*
+	 * If the OEM fused a valid address, use it instead of the one in the
+	 * OTP
+	 */
+	if (is_valid_ether_addr(data->hw_addr))
+		return;
+
+	mac_addr0 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR0_OTP));
+	mac_addr1 = cpu_to_le32(iwl_read32(trans, CSR_MAC_ADDR1_OTP));
 
 	iwl_flip_hw_address(mac_addr0, mac_addr1, data->hw_addr);
 }
@@ -899,3 +905,91 @@ iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
 	return regd;
 }
 IWL_EXPORT_SYMBOL(iwl_parse_nvm_mcc_info);
+
+#ifdef CONFIG_ACPI
+#define WRDD_METHOD		"WRDD"
+#define WRDD_WIFI		(0x07)
+#define WRDD_WIGIG		(0x10)
+
+static u32 iwl_wrdd_get_mcc(struct device *dev, union acpi_object *wrdd)
+{
+	union acpi_object *mcc_pkg, *domain_type, *mcc_value;
+	u32 i;
+
+	if (wrdd->type != ACPI_TYPE_PACKAGE ||
+	    wrdd->package.count < 2 ||
+	    wrdd->package.elements[0].type != ACPI_TYPE_INTEGER ||
+	    wrdd->package.elements[0].integer.value != 0) {
+		IWL_DEBUG_EEPROM(dev, "Unsupported wrdd structure\n");
+		return 0;
+	}
+
+	for (i = 1 ; i < wrdd->package.count ; ++i) {
+		mcc_pkg = &wrdd->package.elements[i];
+
+		if (mcc_pkg->type != ACPI_TYPE_PACKAGE ||
+		    mcc_pkg->package.count < 2 ||
+		    mcc_pkg->package.elements[0].type != ACPI_TYPE_INTEGER ||
+		    mcc_pkg->package.elements[1].type != ACPI_TYPE_INTEGER) {
+			mcc_pkg = NULL;
+			continue;
+		}
+
+		domain_type = &mcc_pkg->package.elements[0];
+		if (domain_type->integer.value == WRDD_WIFI)
+			break;
+
+		mcc_pkg = NULL;
+	}
+
+	if (mcc_pkg) {
+		mcc_value = &mcc_pkg->package.elements[1];
+		return mcc_value->integer.value;
+	}
+
+	return 0;
+}
+
+int iwl_get_bios_mcc(struct device *dev, char *mcc)
+{
+	acpi_handle root_handle;
+	acpi_handle handle;
+	struct acpi_buffer wrdd = {ACPI_ALLOCATE_BUFFER, NULL};
+	acpi_status status;
+	u32 mcc_val;
+
+	root_handle = ACPI_HANDLE(dev);
+	if (!root_handle) {
+		IWL_DEBUG_EEPROM(dev,
+				 "Could not retrieve root port ACPI handle\n");
+		return -ENOENT;
+	}
+
+	/* Get the method's handle */
+	status = acpi_get_handle(root_handle, (acpi_string)WRDD_METHOD,
+				 &handle);
+	if (ACPI_FAILURE(status)) {
+		IWL_DEBUG_EEPROM(dev, "WRD method not found\n");
+		return -ENOENT;
+	}
+
+	/* Call WRDD with no arguments */
+	status = acpi_evaluate_object(handle, NULL, NULL, &wrdd);
+	if (ACPI_FAILURE(status)) {
+		IWL_DEBUG_EEPROM(dev, "WRDC invocation failed (0x%x)\n",
+				 status);
+		return -ENOENT;
+	}
+
+	mcc_val = iwl_wrdd_get_mcc(dev, wrdd.pointer);
+	kfree(wrdd.pointer);
+	if (!mcc_val)
+		return -ENOENT;
+
+	mcc[0] = (mcc_val >> 8) & 0xff;
+	mcc[1] = mcc_val & 0xff;
+	mcc[2] = '\0';
+	return 0;
+}
+IWL_EXPORT_SYMBOL(iwl_get_bios_mcc);
+#endif
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
index d704d52aa7ec..7249e5b403f4 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
@@ -5,7 +5,8 @@
  *
  * GPL LICENSE SUMMARY
  *
- * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2008 - 2015 Intel Corporation. All rights reserved.
+ * Copyright(c) 2016        Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -93,4 +94,21 @@ struct ieee80211_regdomain *
 iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
 		       int num_of_ch, __le32 *channels, u16 fw_mcc);
 
+#ifdef CONFIG_ACPI
+/**
+ * iwl_get_bios_mcc - read MCC from BIOS, if available
+ *
+ * @dev: the struct device
+ * @mcc: output buffer (3 bytes) that will get the MCC
+ *
+ * This function tries to read the current MCC from ACPI if available.
+ */
+int iwl_get_bios_mcc(struct device *dev, char *mcc);
+#else
+static inline int iwl_get_bios_mcc(struct device *dev, char *mcc)
+{
+	return -ENOENT;
+}
+#endif
+
 #endif /* __iwl_nvm_parse_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-phy-db.c b/drivers/net/wireless/intel/iwlwifi/iwl-phy-db.c
index 7beba9ae5617..2893826d7d2b 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-phy-db.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-phy-db.c
@@ -110,7 +110,7 @@ enum iwl_phy_db_section_type {
 	IWL_PHY_DB_MAX
 };
 
-#define PHY_DB_CMD 0x6c /* TEMP API - The actual is 0x8c */
+#define PHY_DB_CMD 0x6c
 
 /*
  * phy db - configure operational ucode
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
index 459bf736fd5b..406ef301b8ab 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
@@ -302,22 +302,17 @@
 #define OSC_CLK_FORCE_CONTROL		(0x8)
 
 #define FH_UCODE_LOAD_STATUS		(0x1AF0)
-#define CSR_UCODE_LOAD_STATUS_ADDR	(0x1E70)
-enum secure_load_status_reg {
-	LMPM_CPU_UCODE_LOADING_STARTED			= 0x00000001,
-	LMPM_CPU_HDRS_LOADING_COMPLETED			= 0x00000003,
-	LMPM_CPU_UCODE_LOADING_COMPLETED		= 0x00000007,
-	LMPM_CPU_STATUS_NUM_OF_LAST_COMPLETED		= 0x000000F8,
-	LMPM_CPU_STATUS_NUM_OF_LAST_LOADED_BLOCK	= 0x0000FF00,
-};
 
-#define LMPM_SECURE_INSPECTOR_CODE_ADDR	(0x1E38)
-#define LMPM_SECURE_INSPECTOR_DATA_ADDR	(0x1E3C)
+/*
+ * Replacing FH_UCODE_LOAD_STATUS
+ * This register is writen by driver and is read by uCode during boot flow.
+ * Note this address is cleared after MAC reset.
+ */
+#define UREG_UCODE_LOAD_STATUS		(0xa05c40)
+
 #define LMPM_SECURE_UCODE_LOAD_CPU1_HDR_ADDR	(0x1E78)
 #define LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR	(0x1E7C)
 
-#define LMPM_SECURE_INSPECTOR_CODE_MEM_SPACE	(0x400000)
-#define LMPM_SECURE_INSPECTOR_DATA_MEM_SPACE	(0x402000)
 #define LMPM_SECURE_CPU1_HDR_MEM_SPACE		(0x420000)
 #define LMPM_SECURE_CPU2_HDR_MEM_SPACE		(0x420400)
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
index 6069a9ff53fa..d42cab291025 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
@@ -65,6 +65,7 @@
 
 #include "iwl-trans.h"
 #include "iwl-drv.h"
+#include "iwl-fh.h"
 
 struct iwl_trans *iwl_trans_alloc(unsigned int priv_size,
 				  struct device *dev,
@@ -77,7 +78,7 @@ struct iwl_trans *iwl_trans_alloc(unsigned int priv_size,
 	static struct lock_class_key __key;
 #endif
 
-	trans = kzalloc(sizeof(*trans) + priv_size, GFP_KERNEL);
+	trans = devm_kzalloc(dev, sizeof(*trans) + priv_size, GFP_KERNEL);
 	if (!trans)
 		return NULL;
 
@@ -102,18 +103,14 @@ struct iwl_trans *iwl_trans_alloc(unsigned int priv_size,
 				  SLAB_HWCACHE_ALIGN,
 				  NULL);
 	if (!trans->dev_cmd_pool)
-		goto free;
+		return NULL;
 
 	return trans;
- free:
-	kfree(trans);
-	return NULL;
 }
 
 void iwl_trans_free(struct iwl_trans *trans)
 {
 	kmem_cache_destroy(trans->dev_cmd_pool);
-	kfree(trans);
 }
 
 int iwl_trans_send_cmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd)
@@ -139,6 +136,9 @@ int iwl_trans_send_cmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd)
 	if (!(cmd->flags & CMD_ASYNC))
 		lock_map_acquire_read(&trans->sync_cmd_lockdep_map);
 
+	if (trans->wide_cmd_header && !iwl_cmd_groupid(cmd->id))
+		cmd->id = DEF_ID(cmd->id);
+
 	ret = trans->ops->send_cmd(trans, cmd);
 
 	if (!(cmd->flags & CMD_ASYNC))
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
index 5535e2238da3..0296124a7f9c 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
@@ -153,6 +153,7 @@ static inline u32 iwl_cmd_id(u8 opcode, u8 groupid, u8 version)
 
 /* make u16 wide id out of u8 group and opcode */
 #define WIDE_ID(grp, opcode) ((grp << 8) | opcode)
+#define DEF_ID(opcode) ((1 << 8) | (opcode))
 
 /* due to the conversion, this group is special; new groups
  * should be defined in the appropriate fw-api header files
@@ -262,8 +263,6 @@ static inline u32 iwl_rx_packet_payload_len(const struct iwl_rx_packet *pkt)
  *	(i.e. mark it as non-idle).
  * @CMD_WANT_ASYNC_CALLBACK: the op_mode's async callback function must be
  *	called after this command completes. Valid only with CMD_ASYNC.
- * @CMD_TB_BITMAP_POS: Position of the first bit for the TB bitmap. We need to
- *	check that we leave enough room for the TBs bitmap which needs 20 bits.
  */
 enum CMD_MODE {
 	CMD_ASYNC		= BIT(0),
@@ -274,8 +273,6 @@ enum CMD_MODE {
 	CMD_MAKE_TRANS_IDLE	= BIT(5),
 	CMD_WAKE_UP_TRANS	= BIT(6),
 	CMD_WANT_ASYNC_CALLBACK	= BIT(7),
-
-	CMD_TB_BITMAP_POS	= 11,
 };
 
 #define DEF_CMD_PAYLOAD_SIZE 320
@@ -488,7 +485,6 @@ struct iwl_hcmd_arr {
  * @bc_table_dword: set to true if the BC table expects the byte count to be
  *	in DWORD (as opposed to bytes)
  * @scd_set_active: should the transport configure the SCD for HCMD queue
- * @wide_cmd_header: firmware supports wide host command header
  * @sw_csum_tx: transport should compute the TCP checksum
  * @command_groups: array of command groups, each member is an array of the
  *	commands in the group; for debugging only
@@ -510,7 +506,6 @@ struct iwl_trans_config {
 	enum iwl_amsdu_size rx_buf_size;
 	bool bc_table_dword;
 	bool scd_set_active;
-	bool wide_cmd_header;
 	bool sw_csum_tx;
 	const struct iwl_hcmd_arr *command_groups;
 	int command_groups_size;
@@ -649,6 +644,8 @@ struct iwl_trans_ops {
 	void (*txq_set_shared_mode)(struct iwl_trans *trans, u32 txq_id,
 				    bool shared);
 
+	dma_addr_t (*get_txq_byte_table)(struct iwl_trans *trans, int txq_id);
+
 	int (*wait_tx_queue_empty)(struct iwl_trans *trans, u32 txq_bm);
 	void (*freeze_txq_timer)(struct iwl_trans *trans, unsigned long txqs,
 				 bool freeze);
@@ -772,6 +769,7 @@ enum iwl_plat_pm_mode {
  * @hw_id_str: a string with info about HW ID. Set during transport allocation.
  * @pm_support: set to true in start_hw if link pm is supported
  * @ltr_enabled: set to true if the LTR is enabled
+ * @wide_cmd_header: true when ucode supports wide command header format
  * @num_rx_queues: number of RX queues allocated by the transport;
  *	the transport must set this before calling iwl_drv_start()
  * @dev_cmd_pool: pool for Tx cmd allocation - for internal use only.
@@ -823,6 +821,7 @@ struct iwl_trans {
 
 	const struct iwl_hcmd_arr *command_groups;
 	int command_groups_size;
+	bool wide_cmd_header;
 
 	u8 num_rx_queues;
 
@@ -1073,6 +1072,15 @@ static inline void iwl_trans_txq_set_shared_mode(struct iwl_trans *trans,
 		trans->ops->txq_set_shared_mode(trans, queue, shared_mode);
 }
 
+static inline dma_addr_t iwl_trans_get_txq_byte_table(struct iwl_trans *trans,
+						      int queue)
+{
+	/* we should never be called if the trans doesn't support it */
+	BUG_ON(!trans->ops->get_txq_byte_table);
+
+	return trans->ops->get_txq_byte_table(trans, queue);
+}
+
 static inline void iwl_trans_txq_enable(struct iwl_trans *trans, int queue,
 					int fifo, int sta_id, int tid,
 					int frame_limit, u16 ssn,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
index b23271755daf..f4d75ffe3d8a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
@@ -504,6 +504,28 @@ static inline char *iwl_dbgfs_is_match(char *name, char *buf)
 	return !strncmp(name, buf, len) ? buf + len : NULL;
 }
 
+static ssize_t iwl_dbgfs_os_device_timediff_read(struct file *file,
+						 char __user *user_buf,
+						 size_t count, loff_t *ppos)
+{
+	struct ieee80211_vif *vif = file->private_data;
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mvm *mvm = mvmvif->mvm;
+	u32 curr_gp2;
+	u64 curr_os;
+	s64 diff;
+	char buf[64];
+	const size_t bufsz = sizeof(buf);
+	int pos = 0;
+
+	iwl_mvm_get_sync_time(mvm, &curr_gp2, &curr_os);
+	do_div(curr_os, NSEC_PER_USEC);
+	diff = curr_os - curr_gp2;
+	pos += scnprintf(buf + pos, bufsz - pos, "diff=%lld\n", diff);
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
+}
+
 static ssize_t iwl_dbgfs_tof_enable_write(struct ieee80211_vif *vif,
 					  char *buf,
 					  size_t count, loff_t *ppos)
@@ -1530,6 +1552,8 @@ MVM_DEBUGFS_READ_FILE_OPS(tof_range_response);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(tof_responder_params, 32);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(quota_min, 32);
 MVM_DEBUGFS_WRITE_FILE_OPS(lqm_send_cmd, 64);
+MVM_DEBUGFS_READ_FILE_OPS(os_device_timediff);
+
 
 void iwl_mvm_vif_dbgfs_register(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
@@ -1547,15 +1571,14 @@ void iwl_mvm_vif_dbgfs_register(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	mvmvif->dbgfs_dir = debugfs_create_dir("iwlmvm", dbgfs_dir);
 
 	if (!mvmvif->dbgfs_dir) {
-		IWL_ERR(mvm, "Failed to create debugfs directory under %s\n",
-			dbgfs_dir->d_name.name);
+		IWL_ERR(mvm, "Failed to create debugfs directory under %pd\n",
+			dbgfs_dir);
 		return;
 	}
 
 	if (iwlmvm_mod_params.power_scheme != IWL_POWER_SCHEME_CAM &&
 	    ((vif->type == NL80211_IFTYPE_STATION && !vif->p2p) ||
-	     (vif->type == NL80211_IFTYPE_STATION && vif->p2p &&
-	      mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_BSS_P2P_PS_DCM)))
+	     (vif->type == NL80211_IFTYPE_STATION && vif->p2p)))
 		MVM_DEBUGFS_ADD_FILE_VIF(pm_params, mvmvif->dbgfs_dir, S_IWUSR |
 					 S_IRUSR);
 
@@ -1570,6 +1593,8 @@ void iwl_mvm_vif_dbgfs_register(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	MVM_DEBUGFS_ADD_FILE_VIF(quota_min, mvmvif->dbgfs_dir,
 				 S_IRUSR | S_IWUSR);
 	MVM_DEBUGFS_ADD_FILE_VIF(lqm_send_cmd, mvmvif->dbgfs_dir, S_IWUSR);
+	MVM_DEBUGFS_ADD_FILE_VIF(os_device_timediff,
+				 mvmvif->dbgfs_dir, S_IRUSR);
 
 	if (vif->type == NL80211_IFTYPE_STATION && !vif->p2p &&
 	    mvmvif == mvm->bf_allowed_vif)
@@ -1602,17 +1627,15 @@ void iwl_mvm_vif_dbgfs_register(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	 * find
 	 * netdev:wlan0 -> ../../../ieee80211/phy0/netdev:wlan0/iwlmvm/
 	 */
-	snprintf(buf, 100, "../../../%s/%s/%s/%s",
-		 dbgfs_dir->d_parent->d_parent->d_name.name,
-		 dbgfs_dir->d_parent->d_name.name,
-		 dbgfs_dir->d_name.name,
-		 mvmvif->dbgfs_dir->d_name.name);
+	snprintf(buf, 100, "../../../%pd3/%pd",
+		 dbgfs_dir,
+		 mvmvif->dbgfs_dir);
 
 	mvmvif->dbgfs_slink = debugfs_create_symlink(dbgfs_dir->d_name.name,
 						     mvm->debugfs_dir, buf);
 	if (!mvmvif->dbgfs_slink)
-		IWL_ERR(mvm, "Can't create debugfs symbolic link under %s\n",
-			dbgfs_dir->d_name.name);
+		IWL_ERR(mvm, "Can't create debugfs symbolic link under %pd\n",
+			dbgfs_dir);
 	return;
 err:
 	IWL_ERR(mvm, "Can't create debugfs entity\n");
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
index b34489817c70..07da4efe8458 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
@@ -917,6 +917,59 @@ static ssize_t iwl_dbgfs_indirection_tbl_write(struct iwl_mvm *mvm,
 	return ret ?: count;
 }
 
+static ssize_t iwl_dbgfs_inject_packet_write(struct iwl_mvm *mvm,
+					     char *buf, size_t count,
+					     loff_t *ppos)
+{
+	struct iwl_rx_cmd_buffer rxb = {
+		._rx_page_order = 0,
+		.truesize = 0, /* not used */
+		._offset = 0,
+	};
+	struct iwl_rx_packet *pkt;
+	struct iwl_rx_mpdu_desc *desc;
+	int bin_len = count / 2;
+	int ret = -EINVAL;
+
+	/* supporting only 9000 descriptor */
+	if (!mvm->trans->cfg->mq_rx_supported)
+		return -ENOTSUPP;
+
+	rxb._page = alloc_pages(GFP_ATOMIC, 0);
+	if (!rxb._page)
+		return -ENOMEM;
+	pkt = rxb_addr(&rxb);
+
+	ret = hex2bin(page_address(rxb._page), buf, bin_len);
+	if (ret)
+		goto out;
+
+	/* avoid invalid memory access */
+	if (bin_len < sizeof(*pkt) + sizeof(*desc))
+		goto out;
+
+	/* check this is RX packet */
+	if (WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd) !=
+	    WIDE_ID(LEGACY_GROUP, REPLY_RX_MPDU_CMD))
+		goto out;
+
+	/* check the length in metadata matches actual received length */
+	desc = (void *)pkt->data;
+	if (le16_to_cpu(desc->mpdu_len) !=
+	    (bin_len - sizeof(*desc) - sizeof(*pkt)))
+		goto out;
+
+	local_bh_disable();
+	iwl_mvm_rx_mpdu_mq(mvm, NULL, &rxb, 0);
+	local_bh_enable();
+	ret = 0;
+
+out:
+	iwl_free_rxb(&rxb);
+
+	return ret ?: count;
+}
+
 static ssize_t iwl_dbgfs_fw_dbg_conf_read(struct file *file,
 					  char __user *user_buf,
 					  size_t count, loff_t *ppos)
@@ -1454,6 +1507,7 @@ MVM_DEBUGFS_WRITE_FILE_OPS(cont_recording, 8);
 MVM_DEBUGFS_WRITE_FILE_OPS(max_amsdu_len, 8);
 MVM_DEBUGFS_WRITE_FILE_OPS(indirection_tbl,
 			   (IWL_RSS_INDIRECTION_TABLE_SIZE * 2));
+MVM_DEBUGFS_WRITE_FILE_OPS(inject_packet, 512);
 
 #ifdef CONFIG_IWLWIFI_BCAST_FILTERING
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(bcast_filters, 256);
@@ -1464,6 +1518,132 @@ MVM_DEBUGFS_READ_WRITE_FILE_OPS(bcast_filters_macs, 256);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(d3_sram, 8);
 #endif
 
+static ssize_t iwl_dbgfs_mem_read(struct file *file, char __user *user_buf,
+				  size_t count, loff_t *ppos)
+{
+	struct iwl_mvm *mvm = file->private_data;
+	struct iwl_dbg_mem_access_cmd cmd = {};
+	struct iwl_dbg_mem_access_rsp *rsp;
+	struct iwl_host_cmd hcmd = {
+		.flags = CMD_WANT_SKB | CMD_SEND_IN_RFKILL,
+		.data = { &cmd, },
+		.len = { sizeof(cmd) },
+	};
+	size_t delta, len;
+	ssize_t ret;
+
+	hcmd.id = iwl_cmd_id(*ppos >> 24 ? UMAC_RD_WR : LMAC_RD_WR,
+			     DEBUG_GROUP, 0);
+	cmd.op = cpu_to_le32(DEBUG_MEM_OP_READ);
+
+	/* Take care of alignment of both the position and the length */
+	delta = *ppos & 0x3;
+	cmd.addr = cpu_to_le32(*ppos - delta);
+	cmd.len = cpu_to_le32(min(ALIGN(count + delta, 4) / 4,
+				  (size_t)DEBUG_MEM_MAX_SIZE_DWORDS));
+
+	mutex_lock(&mvm->mutex);
+	ret = iwl_mvm_send_cmd(mvm, &hcmd);
+	mutex_unlock(&mvm->mutex);
+
+	if (ret < 0)
+		return ret;
+
+	rsp = (void *)hcmd.resp_pkt->data;
+	if (le32_to_cpu(rsp->status) != DEBUG_MEM_STATUS_SUCCESS) {
+		ret = -ENXIO;
+		goto out;
+	}
+
+	len = min((size_t)le32_to_cpu(rsp->len) << 2,
+		  iwl_rx_packet_payload_len(hcmd.resp_pkt) - sizeof(*rsp));
+	len = min(len - delta, count);
+	if (len < 0) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	ret = len - copy_to_user(user_buf, (void *)rsp->data + delta, len);
+	*ppos += ret;
+
+out:
+	iwl_free_resp(&hcmd);
+	return ret;
+}
+
+static ssize_t iwl_dbgfs_mem_write(struct file *file,
+				   const char __user *user_buf, size_t count,
+				   loff_t *ppos)
+{
+	struct iwl_mvm *mvm = file->private_data;
+	struct iwl_dbg_mem_access_cmd *cmd;
+	struct iwl_dbg_mem_access_rsp *rsp;
+	struct iwl_host_cmd hcmd = {};
+	size_t cmd_size;
+	size_t data_size;
+	u32 op, len;
+	ssize_t ret;
+
+	hcmd.id = iwl_cmd_id(*ppos >> 24 ? UMAC_RD_WR : LMAC_RD_WR,
+			     DEBUG_GROUP, 0);
+
+	if (*ppos & 0x3 || count < 4) {
+		op = DEBUG_MEM_OP_WRITE_BYTES;
+		len = min(count, (size_t)(4 - (*ppos & 0x3)));
+		data_size = len;
+	} else {
+		op = DEBUG_MEM_OP_WRITE;
+		len = min(count >> 2, (size_t)DEBUG_MEM_MAX_SIZE_DWORDS);
+		data_size = len << 2;
+	}
+
+	cmd_size = sizeof(*cmd) + ALIGN(data_size, 4);
+	cmd = kzalloc(cmd_size, GFP_KERNEL);
+	if (!cmd)
+		return -ENOMEM;
+
+	cmd->op = cpu_to_le32(op);
+	cmd->len = cpu_to_le32(len);
+	cmd->addr = cpu_to_le32(*ppos);
+	if (copy_from_user((void *)cmd->data, user_buf, data_size)) {
+		kfree(cmd);
+		return -EFAULT;
+	}
+
+	hcmd.flags = CMD_WANT_SKB | CMD_SEND_IN_RFKILL,
+	hcmd.data[0] = (void *)cmd;
+	hcmd.len[0] = cmd_size;
+
+	mutex_lock(&mvm->mutex);
+	ret = iwl_mvm_send_cmd(mvm, &hcmd);
+	mutex_unlock(&mvm->mutex);
+
+	kfree(cmd);
+
+	if (ret < 0)
+		return ret;
+
+	rsp = (void *)hcmd.resp_pkt->data;
+	if (rsp->status != DEBUG_MEM_STATUS_SUCCESS) {
+		ret = -ENXIO;
+		goto out;
+	}
+
+	ret = data_size;
+	*ppos += ret;
+
+out:
+	iwl_free_resp(&hcmd);
+	return ret;
+}
+
+static const struct file_operations iwl_dbgfs_mem_ops = {
+	.read = iwl_dbgfs_mem_read,
+	.write = iwl_dbgfs_mem_write,
+	.open = simple_open,
+	.llseek = default_llseek,
+};
+
 int iwl_mvm_dbgfs_register(struct iwl_mvm *mvm, struct dentry *dbgfs_dir)
 {
 	struct dentry *bcast_dir __maybe_unused;
@@ -1502,6 +1682,7 @@ int iwl_mvm_dbgfs_register(struct iwl_mvm *mvm, struct dentry *dbgfs_dir)
 	MVM_DEBUGFS_ADD_FILE(send_echo_cmd, mvm->debugfs_dir, S_IWUSR);
 	MVM_DEBUGFS_ADD_FILE(cont_recording, mvm->debugfs_dir, S_IWUSR);
 	MVM_DEBUGFS_ADD_FILE(indirection_tbl, mvm->debugfs_dir, S_IWUSR);
+	MVM_DEBUGFS_ADD_FILE(inject_packet, mvm->debugfs_dir, S_IWUSR);
 	if (!debugfs_create_bool("enable_scan_iteration_notif",
 				 S_IRUSR | S_IWUSR,
 				 mvm->debugfs_dir,
@@ -1560,13 +1741,14 @@ int iwl_mvm_dbgfs_register(struct iwl_mvm *mvm, struct dentry *dbgfs_dir)
 				 mvm->debugfs_dir, &mvm->nvm_phy_sku_blob))
 		goto err;
 
+	debugfs_create_file("mem", S_IRUSR | S_IWUSR, dbgfs_dir, mvm,
+			    &iwl_dbgfs_mem_ops);
+
 	/*
 	 * Create a symlink with mac80211. It will be removed when mac80211
 	 * exists (before the opmode exists which removes the target.)
 	 */
-	snprintf(buf, 100, "../../%s/%s",
-		 dbgfs_dir->d_parent->d_parent->d_name.name,
-		 dbgfs_dir->d_parent->d_name.name);
+	snprintf(buf, 100, "../../%pd2", dbgfs_dir->d_parent);
 	if (!debugfs_create_symlink("iwlwifi", mvm->hw->wiphy->debugfsdir, buf))
 		goto err;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-power.h b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-power.h
index 404b0de9e2dc..3fa43d1348a2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-power.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-power.h
@@ -313,35 +313,26 @@ enum iwl_dev_tx_power_cmd_mode {
 	IWL_TX_POWER_MODE_SET_ACK = 3,
 }; /* TX_POWER_REDUCED_FLAGS_TYPE_API_E_VER_4 */;
 
+#define IWL_NUM_CHAIN_LIMITS	2
+#define IWL_NUM_SUB_BANDS	5
+
 /**
- * struct iwl_dev_tx_power_cmd_v2 - TX power reduction command
+ * struct iwl_dev_tx_power_cmd - TX power reduction command
  * @set_mode: see &enum iwl_dev_tx_power_cmd_mode
  * @mac_context_id: id of the mac ctx for which we are reducing TX power.
  * @pwr_restriction: TX power restriction in 1/8 dBms.
  * @dev_24: device TX power restriction in 1/8 dBms
  * @dev_52_low: device TX power restriction upper band - low
  * @dev_52_high: device TX power restriction upper band - high
+ * @per_chain_restriction: per chain restrictions
  */
-struct iwl_dev_tx_power_cmd_v2 {
+struct iwl_dev_tx_power_cmd_v3 {
 	__le32 set_mode;
 	__le32 mac_context_id;
 	__le16 pwr_restriction;
 	__le16 dev_24;
 	__le16 dev_52_low;
 	__le16 dev_52_high;
-} __packed; /* TX_REDUCED_POWER_API_S_VER_2 */
-
-#define IWL_NUM_CHAIN_LIMITS	2
-#define IWL_NUM_SUB_BANDS	5
-
-/**
- * struct iwl_dev_tx_power_cmd - TX power reduction command
- * @v2: version 2 of the command, embedded here for easier software handling
- * @per_chain_restriction: per chain restrictions
- */
-struct iwl_dev_tx_power_cmd_v3 {
-	/* v3 is just an extension of v2 - keep this here */
-	struct iwl_dev_tx_power_cmd_v2 v2;
 	__le16 per_chain_restriction[IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS];
 } __packed; /* TX_REDUCED_POWER_API_S_VER_3 */
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-scan.h b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-scan.h
index f01dab0d0dac..0c294c9f98e9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-scan.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-scan.h
@@ -7,6 +7,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -603,6 +604,8 @@ struct iwl_scan_req_umac_tail {
  * @uid: scan id, &enum iwl_umac_scan_uid_offsets
  * @ooc_priority: out of channel priority - &enum iwl_scan_priority
  * @general_flags: &enum iwl_umac_scan_general_flags
+ * @reserved2: for future use and alignment
+ * @scan_start_mac_id: report the scan start TSF time according to this mac TSF
  * @extended_dwell: dwell time for channels 1, 6 and 11
  * @active_dwell: dwell time for active scan
  * @passive_dwell: dwell time for passive scan
@@ -620,8 +623,10 @@ struct iwl_scan_req_umac {
 	__le32 flags;
 	__le32 uid;
 	__le32 ooc_priority;
-	/* SCAN_GENERAL_PARAMS_API_S_VER_1 */
-	__le32 general_flags;
+	/* SCAN_GENERAL_PARAMS_API_S_VER_4 */
+	__le16 general_flags;
+	u8 reserved2;
+	u8 scan_start_mac_id;
 	u8 extended_dwell;
 	u8 active_dwell;
 	u8 passive_dwell;
@@ -629,7 +634,7 @@ struct iwl_scan_req_umac {
 	__le32 max_out_time;
 	__le32 suspend_time;
 	__le32 scan_priority;
-	/* SCAN_CHANNEL_PARAMS_API_S_VER_1 */
+	/* SCAN_CHANNEL_PARAMS_API_S_VER_4 */
 	u8 channel_flags;
 	u8 n_channels;
 	__le16 reserved;
@@ -718,8 +723,8 @@ struct iwl_scan_offload_profiles_query {
  * @status: one of SCAN_COMP_STATUS_*
  * @bt_status: BT on/off status
  * @last_channel: last channel that was scanned
- * @tsf_low: TSF timer (lower half) in usecs
- * @tsf_high: TSF timer (higher half) in usecs
+ * @start_tsf: TSF timer in usecs of the scan start time for the mac specified
+ *	in &struct iwl_scan_req_umac.
  * @results: array of scan results, only "scanned_channels" of them are valid
  */
 struct iwl_umac_scan_iter_complete_notif {
@@ -728,9 +733,8 @@ struct iwl_umac_scan_iter_complete_notif {
 	u8 status;
 	u8 bt_status;
 	u8 last_channel;
-	__le32 tsf_low;
-	__le32 tsf_high;
+	__le64 start_tsf;
 	struct iwl_scan_results_notif results[];
-} __packed; /* SCAN_ITER_COMPLETE_NTF_UMAC_API_S_VER_1 */
+} __packed; /* SCAN_ITER_COMPLETE_NTF_UMAC_API_S_VER_2 */
 
 #endif
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-sta.h b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-sta.h
index d1c4fb849111..6c8e3ca79323 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-sta.h
@@ -433,25 +433,42 @@ struct iwl_mvm_rm_sta_cmd {
 } __packed; /* REMOVE_STA_CMD_API_S_VER_2 */
 
 /**
+ * struct iwl_mvm_mgmt_mcast_key_cmd_v1
+ * ( MGMT_MCAST_KEY = 0x1f )
+ * @ctrl_flags: %iwl_sta_key_flag
+ * @igtk:
+ * @k1: unused
+ * @k2: unused
+ * @sta_id: station ID that support IGTK
+ * @key_id:
+ * @receive_seq_cnt: initial RSC/PN needed for replay check
+ */
+struct iwl_mvm_mgmt_mcast_key_cmd_v1 {
+	__le32 ctrl_flags;
+	u8 igtk[16];
+	u8 k1[16];
+	u8 k2[16];
+	__le32 key_id;
+	__le32 sta_id;
+	__le64 receive_seq_cnt;
+} __packed; /* SEC_MGMT_MULTICAST_KEY_CMD_API_S_VER_1 */
+
+/**
  * struct iwl_mvm_mgmt_mcast_key_cmd
  * ( MGMT_MCAST_KEY = 0x1f )
  * @ctrl_flags: %iwl_sta_key_flag
- * @IGTK:
- * @K1: unused
- * @K2: unused
+ * @igtk: IGTK master key
  * @sta_id: station ID that support IGTK
  * @key_id:
  * @receive_seq_cnt: initial RSC/PN needed for replay check
  */
 struct iwl_mvm_mgmt_mcast_key_cmd {
 	__le32 ctrl_flags;
-	u8 IGTK[16];
-	u8 K1[16];
-	u8 K2[16];
+	u8 igtk[32];
 	__le32 key_id;
 	__le32 sta_id;
 	__le64 receive_seq_cnt;
-} __packed; /* SEC_MGMT_MULTICAST_KEY_CMD_API_S_VER_1 */
+} __packed; /* SEC_MGMT_MULTICAST_KEY_CMD_API_S_VER_2 */
 
 struct iwl_mvm_wep_key {
 	u8 key_index;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-tx.h b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-tx.h
index 4144623e1616..59ca97a11b2b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-tx.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-tx.h
@@ -89,7 +89,6 @@
  * @TX_CMD_FLG_MH_PAD: driver inserted 2 byte padding after MAC header.
  *	Should be set for 26/30 length MAC headers
  * @TX_CMD_FLG_RESP_TO_DRV: zero this if the response should go only to FW
- * @TX_CMD_FLG_CCMP_AGG: this frame uses CCMP for aggregation acceleration
  * @TX_CMD_FLG_TKIP_MIC_DONE: FW already performed TKIP MIC calculation
  * @TX_CMD_FLG_DUR: disable duration overwriting used in PS-Poll Assoc-id
  * @TX_CMD_FLG_FW_DROP: FW should mark frame to be dropped
@@ -116,7 +115,6 @@ enum iwl_tx_flags {
 	TX_CMD_FLG_KEEP_SEQ_CTL		= BIT(18),
 	TX_CMD_FLG_MH_PAD		= BIT(20),
 	TX_CMD_FLG_RESP_TO_DRV		= BIT(21),
-	TX_CMD_FLG_CCMP_AGG		= BIT(22),
 	TX_CMD_FLG_TKIP_MIC_DONE	= BIT(23),
 	TX_CMD_FLG_DUR			= BIT(25),
 	TX_CMD_FLG_FW_DROP		= BIT(26),
@@ -149,7 +147,7 @@ enum iwl_tx_pm_timeouts {
  * @TX_CMD_SEC_EXT: extended cipher algorithm.
  * @TX_CMD_SEC_GCMP: GCMP encryption algorithm.
  * @TX_CMD_SEC_KEY128: set for 104 bits WEP key.
- * @TC_CMD_SEC_KEY_FROM_TABLE: for a non-WEP key, set if the key should be taken
+ * @TX_CMD_SEC_KEY_FROM_TABLE: for a non-WEP key, set if the key should be taken
  *	from the table instead of from the TX command.
  *	If the key is taken from the key table its index should be given by the
  *	first byte of the TX command key field.
@@ -161,7 +159,7 @@ enum iwl_tx_cmd_sec_ctrl {
 	TX_CMD_SEC_EXT			= 0x04,
 	TX_CMD_SEC_GCMP			= 0x05,
 	TX_CMD_SEC_KEY128		= 0x08,
-	TC_CMD_SEC_KEY_FROM_TABLE	= 0x08,
+	TX_CMD_SEC_KEY_FROM_TABLE	= 0x08,
 };
 
 /* TODO: how does these values are OK with only 16 bit variable??? */
@@ -578,6 +576,85 @@ struct iwl_mvm_ba_notif {
 } __packed;
 
 /**
+ * struct iwl_mvm_compressed_ba_tfd - progress of a TFD queue
+ * @q_num: TFD queue number
+ * @tfd_index: Index of first un-acked frame in the  TFD queue
+ */
+struct iwl_mvm_compressed_ba_tfd {
+	u8 q_num;
+	u8 reserved;
+	__le16 tfd_index;
+} __packed; /* COMPRESSED_BA_TFD_API_S_VER_1 */
+
+/**
+ * struct iwl_mvm_compressed_ba_ratid - progress of a RA TID queue
+ * @q_num: RA TID queue number
+ * @tid: TID of the queue
+ * @ssn: BA window current SSN
+ */
+struct iwl_mvm_compressed_ba_ratid {
+	u8 q_num;
+	u8 tid;
+	__le16 ssn;
+} __packed; /* COMPRESSED_BA_RATID_API_S_VER_1 */
+
+/*
+ * enum iwl_mvm_ba_resp_flags - TX aggregation status
+ * @IWL_MVM_BA_RESP_TX_AGG: generated due to BA
+ * @IWL_MVM_BA_RESP_TX_BAR: generated due to BA after BAR
+ * @IWL_MVM_BA_RESP_TX_AGG_FAIL: aggregation didn't receive BA
+ * @IWL_MVM_BA_RESP_TX_UNDERRUN: aggregation got underrun
+ * @IWL_MVM_BA_RESP_TX_BT_KILL: aggregation got BT-kill
+ * @IWL_MVM_BA_RESP_TX_DSP_TIMEOUT: aggregation didn't finish within the
+ *	expected time
+ */
+enum iwl_mvm_ba_resp_flags {
+	IWL_MVM_BA_RESP_TX_AGG,
+	IWL_MVM_BA_RESP_TX_BAR,
+	IWL_MVM_BA_RESP_TX_AGG_FAIL,
+	IWL_MVM_BA_RESP_TX_UNDERRUN,
+	IWL_MVM_BA_RESP_TX_BT_KILL,
+	IWL_MVM_BA_RESP_TX_DSP_TIMEOUT
+};
+
+/**
+ * struct iwl_mvm_compressed_ba_notif - notifies about reception of BA
+ * ( BA_NOTIF = 0xc5 )
+ * @flags: status flag, see the &iwl_mvm_ba_resp_flags
+ * @sta_id: Index of recipient (BA-sending) station in fw's station table
+ * @reduced_txp: power reduced according to TPC. This is the actual value and
+ *	not a copy from the LQ command. Thus, if not the first rate was used
+ *	for Tx-ing then this value will be set to 0 by FW.
+ * @initial_rate: TLC rate info, initial rate index, TLC table color
+ * @retry_cnt: retry count
+ * @query_byte_cnt: SCD query byte count
+ * @query_frame_cnt: SCD query frame count
+ * @txed: number of frames sent in the aggregation (all-TIDs)
+ * @done: number of frames that were Acked by the BA (all-TIDs)
+ * @wireless_time: Wireless-media time
+ * @tx_rate: the rate the aggregation was sent at
+ * @tfd_cnt: number of TFD-Q elements
+ * @ra_tid_cnt: number of RATID-Q elements
+ */
+struct iwl_mvm_compressed_ba_notif {
+	__le32 flags;
+	u8 sta_id;
+	u8 reduced_txp;
+	u8 initial_rate;
+	u8 retry_cnt;
+	__le32 query_byte_cnt;
+	__le16 query_frame_cnt;
+	__le16 txed;
+	__le16 done;
+	__le32 wireless_time;
+	__le32 tx_rate;
+	__le16 tfd_cnt;
+	__le16 ra_tid_cnt;
+	struct iwl_mvm_compressed_ba_tfd tfd[1];
+	struct iwl_mvm_compressed_ba_ratid ra_tid[0];
+} __packed; /* COMPRESSED_BA_RES_API_S_VER_4 */
+
+/**
  * struct iwl_mac_beacon_cmd_v6 - beacon template command
  * @tx: the tx commands associated with the beacon frame
  * @template_id: currently equal to the mac context id of the coresponding
@@ -675,13 +752,21 @@ static inline u32 iwl_mvm_get_scd_ssn(struct iwl_mvm_tx_resp *tx_resp)
 			    tx_resp->frame_count) & 0xfff;
 }
 
+/* Available options for the SCD_QUEUE_CFG HCMD */
+enum iwl_scd_cfg_actions {
+	SCD_CFG_DISABLE_QUEUE		= 0x0,
+	SCD_CFG_ENABLE_QUEUE		= 0x1,
+	SCD_CFG_UPDATE_QUEUE_TID	= 0x2,
+};
+
 /**
  * struct iwl_scd_txq_cfg_cmd - New txq hw scheduler config command
  * @token:
  * @sta_id: station id
  * @tid:
  * @scd_queue: scheduler queue to confiug
- * @enable: 1 queue enable, 0 queue disable
+ * @action: 1 queue enable, 0 queue disable, 2 change txq's tid owner
+ *	Value is one of %iwl_scd_cfg_actions options
  * @aggregate: 1 aggregated queue, 0 otherwise
  * @tx_fifo: %enum iwl_mvm_tx_fifo
  * @window: BA window size
@@ -692,7 +777,7 @@ struct iwl_scd_txq_cfg_cmd {
 	u8 sta_id;
 	u8 tid;
 	u8 scd_queue;
-	u8 enable;
+	u8 action;
 	u8 aggregate;
 	u8 tx_fifo;
 	u8 window;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api.h b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api.h
index 71076f02796e..97633690f3d5 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api.h
@@ -205,7 +205,7 @@ enum {
 	/* Phy */
 	PHY_CONFIGURATION_CMD = 0x6a,
 	CALIB_RES_NOTIF_PHY_DB = 0x6b,
-	/* PHY_DB_CMD = 0x6c, */
+	PHY_DB_CMD = 0x6c,
 
 	/* ToF - 802.11mc FTM */
 	TOF_CMD = 0x10,
@@ -340,6 +340,11 @@ enum iwl_prot_offload_subcmd_ids {
 	STORED_BEACON_NTF = 0xFF,
 };
 
+enum iwl_fmac_debug_cmds {
+	LMAC_RD_WR = 0x0,
+	UMAC_RD_WR = 0x1,
+};
+
 /* command groups */
 enum {
 	LEGACY_GROUP = 0x0,
@@ -349,6 +354,7 @@ enum {
 	PHY_OPS_GROUP = 0x4,
 	DATA_PATH_GROUP = 0x5,
 	PROT_OFFLOAD_GROUP = 0xb,
+	DEBUG_GROUP = 0xf,
 };
 
 /**
@@ -482,13 +488,17 @@ struct iwl_nvm_access_cmd {
  * @block_size: the block size in powers of 2
  * @block_num: number of blocks specified in the command.
  * @device_phy_addr: virtual addresses from device side
+ *	32 bit address for API version 1, 64 bit address for API version 2.
 */
 struct iwl_fw_paging_cmd {
 	__le32 flags;
 	__le32 block_size;
 	__le32 block_num;
-	__le32 device_phy_addr[NUM_OF_FW_PAGING_BLOCKS];
-} __packed; /* FW_PAGING_BLOCK_CMD_API_S_VER_1 */
+	union {
+		__le32 addr32[NUM_OF_FW_PAGING_BLOCKS];
+		__le64 addr64[NUM_OF_FW_PAGING_BLOCKS];
+	} device_phy_addr;
+} __packed; /* FW_PAGING_BLOCK_CMD_API_S_VER_2 */
 
 /*
  * Fw items ID's
@@ -1973,8 +1983,9 @@ struct iwl_tdls_config_res {
 	struct iwl_tdls_config_sta_info_res sta_info[IWL_MVM_TDLS_STA_COUNT];
 } __packed; /* TDLS_CONFIG_RSP_API_S_VER_1 */
 
-#define TX_FIFO_MAX_NUM		8
-#define RX_FIFO_MAX_NUM		2
+#define TX_FIFO_MAX_NUM_9000		8
+#define TX_FIFO_MAX_NUM			15
+#define RX_FIFO_MAX_NUM			2
 #define TX_FIFO_INTERNAL_MAX_NUM	6
 
 /**
@@ -2000,6 +2011,21 @@ struct iwl_tdls_config_res {
  * NOTE: on firmware that don't have IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG
  *	 set, the last 3 members don't exist.
  */
+struct iwl_shared_mem_cfg_v1 {
+	__le32 shared_mem_addr;
+	__le32 shared_mem_size;
+	__le32 sample_buff_addr;
+	__le32 sample_buff_size;
+	__le32 txfifo_addr;
+	__le32 txfifo_size[TX_FIFO_MAX_NUM_9000];
+	__le32 rxfifo_size[RX_FIFO_MAX_NUM];
+	__le32 page_buff_addr;
+	__le32 page_buff_size;
+	__le32 rxfifo_addr;
+	__le32 internal_txfifo_addr;
+	__le32 internal_txfifo_size[TX_FIFO_INTERNAL_MAX_NUM];
+} __packed; /* SHARED_MEM_ALLOC_API_S_VER_2 */
+
 struct iwl_shared_mem_cfg {
 	__le32 shared_mem_addr;
 	__le32 shared_mem_size;
@@ -2013,7 +2039,7 @@ struct iwl_shared_mem_cfg {
 	__le32 rxfifo_addr;
 	__le32 internal_txfifo_addr;
 	__le32 internal_txfifo_size[TX_FIFO_INTERNAL_MAX_NUM];
-} __packed; /* SHARED_MEM_ALLOC_API_S_VER_2 */
+} __packed; /* SHARED_MEM_ALLOC_API_S_VER_3 */
 
 /**
  * VHT MU-MIMO group configuration
@@ -2129,4 +2155,48 @@ struct iwl_channel_switch_noa_notif {
 	__le32 id_and_color;
 } __packed; /* CHANNEL_SWITCH_START_NTFY_API_S_VER_1 */
 
+/* Operation types for the debug mem access */
+enum {
+	DEBUG_MEM_OP_READ = 0,
+	DEBUG_MEM_OP_WRITE = 1,
+	DEBUG_MEM_OP_WRITE_BYTES = 2,
+};
+
+#define DEBUG_MEM_MAX_SIZE_DWORDS 32
+
+/**
+ * struct iwl_dbg_mem_access_cmd - Request the device to read/write memory
+ * @op: DEBUG_MEM_OP_*
+ * @addr: address to read/write from/to
+ * @len: in dwords, to read/write
+ * @data: for write opeations, contains the source buffer
+ */
+struct iwl_dbg_mem_access_cmd {
+	__le32 op;
+	__le32 addr;
+	__le32 len;
+	__le32 data[];
+} __packed; /* DEBUG_(U|L)MAC_RD_WR_CMD_API_S_VER_1 */
+
+/* Status responses for the debug mem access */
+enum {
+	DEBUG_MEM_STATUS_SUCCESS = 0x0,
+	DEBUG_MEM_STATUS_FAILED = 0x1,
+	DEBUG_MEM_STATUS_LOCKED = 0x2,
+	DEBUG_MEM_STATUS_HIDDEN = 0x3,
+	DEBUG_MEM_STATUS_LENGTH = 0x4,
+};
+
+/**
+ * struct iwl_dbg_mem_access_rsp - Response to debug mem commands
+ * @status: DEBUG_MEM_STATUS_*
+ * @len: read dwords (0 for write operations)
+ * @data: contains the read DWs
+ */
+struct iwl_dbg_mem_access_rsp {
+	__le32 status;
+	__le32 len;
+	__le32 data[];
+} __packed; /* DEBUG_(U|L)MAC_RD_WR_RSP_API_S_VER_1 */
+
 #endif /* __fw_api_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.c
index 1abcabb9b6cd..d89d0a1fd34e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.c
@@ -440,14 +440,12 @@ static const struct iwl_prph_range iwl_prph_dump_addr_comm[] = {
 	{ .start = 0x00a04560, .end = 0x00a0457c },
 	{ .start = 0x00a04590, .end = 0x00a04598 },
 	{ .start = 0x00a045c0, .end = 0x00a045f4 },
-	{ .start = 0x00a44000, .end = 0x00a7bf80 },
 };
 
 static const struct iwl_prph_range iwl_prph_dump_addr_9000[] = {
 	{ .start = 0x00a05c00, .end = 0x00a05c18 },
 	{ .start = 0x00a05400, .end = 0x00a056e8 },
 	{ .start = 0x00a08000, .end = 0x00a098bc },
-	{ .start = 0x00adfc00, .end = 0x00adfd1c },
 	{ .start = 0x00a02400, .end = 0x00a02758 },
 };
 
@@ -559,7 +557,7 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 					 sizeof(struct iwl_fw_error_dump_fifo);
 		}
 
-		for (i = 0; i < ARRAY_SIZE(mem_cfg->txfifo_size); i++) {
+		for (i = 0; i < mem_cfg->num_txfifo_entries; i++) {
 			if (!mem_cfg->txfifo_size[i])
 				continue;
 
@@ -960,5 +958,6 @@ int iwl_mvm_start_fw_dbg_conf(struct iwl_mvm *mvm, u8 conf_id)
 	}
 
 	mvm->fw_dbg_conf = conf_id;
-	return ret;
+
+	return 0;
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.h b/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.h
index f7dff7612c9c..e9f1be9da7d4 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.h
@@ -105,7 +105,8 @@ iwl_fw_dbg_trigger_vif_match(struct iwl_fw_dbg_trigger_tlv *trig,
 {
 	u32 trig_vif = le32_to_cpu(trig->vif_type);
 
-	return trig_vif == IWL_FW_DBG_CONF_VIF_ANY || vif->type == trig_vif;
+	return trig_vif == IWL_FW_DBG_CONF_VIF_ANY ||
+	       ieee80211_vif_type_p2p(vif) == trig_vif;
 }
 
 static inline bool
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
index 7e0cdbf8bf74..872066317fa5 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
@@ -90,15 +90,6 @@ struct iwl_mvm_alive_data {
 	u32 scd_base_addr;
 };
 
-static inline const struct fw_img *
-iwl_get_ucode_image(struct iwl_mvm *mvm, enum iwl_ucode_type ucode_type)
-{
-	if (ucode_type >= IWL_UCODE_TYPE_MAX)
-		return NULL;
-
-	return &mvm->fw->img[ucode_type];
-}
-
 static int iwl_send_tx_ant_cfg(struct iwl_mvm *mvm, u8 valid_tx_ant)
 {
 	struct iwl_tx_ant_cfg_cmd tx_ant_cmd = {
@@ -385,9 +376,7 @@ static int iwl_save_fw_paging(struct iwl_mvm *mvm,
 /* send paging cmd to FW in case CPU2 has paging image */
 static int iwl_send_paging_cmd(struct iwl_mvm *mvm, const struct fw_img *fw)
 {
-	int blk_idx;
-	__le32 dev_phy_addr;
-	struct iwl_fw_paging_cmd fw_paging_cmd = {
+	struct iwl_fw_paging_cmd paging_cmd = {
 		.flags =
 			cpu_to_le32(PAGING_CMD_IS_SECURED |
 				    PAGING_CMD_IS_ENABLED |
@@ -396,18 +385,32 @@ static int iwl_send_paging_cmd(struct iwl_mvm *mvm, const struct fw_img *fw)
 		.block_size = cpu_to_le32(BLOCK_2_EXP_SIZE),
 		.block_num = cpu_to_le32(mvm->num_of_paging_blk),
 	};
+	int blk_idx, size = sizeof(paging_cmd);
+
+	/* A bit hard coded - but this is the old API and will be deprecated */
+	if (!iwl_mvm_has_new_tx_api(mvm))
+		size -= NUM_OF_FW_PAGING_BLOCKS * 4;
 
 	/* loop for for all paging blocks + CSS block */
 	for (blk_idx = 0; blk_idx < mvm->num_of_paging_blk + 1; blk_idx++) {
-		dev_phy_addr =
-			cpu_to_le32(mvm->fw_paging_db[blk_idx].fw_paging_phys >>
-				    PAGE_2_EXP_SIZE);
-		fw_paging_cmd.device_phy_addr[blk_idx] = dev_phy_addr;
+		dma_addr_t addr = mvm->fw_paging_db[blk_idx].fw_paging_phys;
+
+		addr = addr >> PAGE_2_EXP_SIZE;
+
+		if (iwl_mvm_has_new_tx_api(mvm)) {
+			__le64 phy_addr = cpu_to_le64(addr);
+
+			paging_cmd.device_phy_addr.addr64[blk_idx] = phy_addr;
+		} else {
+			__le32 phy_addr = cpu_to_le32(addr);
+
+			paging_cmd.device_phy_addr.addr32[blk_idx] = phy_addr;
+		}
 	}
 
 	return iwl_mvm_send_cmd_pdu(mvm, iwl_cmd_id(FW_PAGING_BLOCK_CMD,
 						    IWL_ALWAYS_LONG_GROUP, 0),
-				    0, sizeof(fw_paging_cmd), &fw_paging_cmd);
+				    0, size, &paging_cmd);
 }
 
 /*
@@ -580,9 +583,9 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	    iwl_fw_dbg_conf_usniffer(mvm->fw, FW_DBG_START_FROM_ALIVE) &&
 	    !(fw_has_capa(&mvm->fw->ucode_capa,
 			  IWL_UCODE_TLV_CAPA_USNIFFER_UNIFIED)))
-		fw = iwl_get_ucode_image(mvm, IWL_UCODE_REGULAR_USNIFFER);
+		fw = iwl_get_ucode_image(mvm->fw, IWL_UCODE_REGULAR_USNIFFER);
 	else
-		fw = iwl_get_ucode_image(mvm, ucode_type);
+		fw = iwl_get_ucode_image(mvm->fw, ucode_type);
 	if (WARN_ON(!fw))
 		return -EINVAL;
 	mvm->cur_ucode = ucode_type;
@@ -826,59 +829,48 @@ out:
 	return ret;
 }
 
-static void iwl_mvm_get_shared_mem_conf(struct iwl_mvm *mvm)
+static void iwl_mvm_parse_shared_mem_a000(struct iwl_mvm *mvm,
+					  struct iwl_rx_packet *pkt)
 {
-	struct iwl_host_cmd cmd = {
-		.flags = CMD_WANT_SKB,
-		.data = { NULL, },
-		.len = { 0, },
-	};
-	struct iwl_shared_mem_cfg *mem_cfg;
-	struct iwl_rx_packet *pkt;
-	u32 i;
+	struct iwl_shared_mem_cfg *mem_cfg = (void *)pkt->data;
+	int i;
 
-	lockdep_assert_held(&mvm->mutex);
+	mvm->shared_mem_cfg.num_txfifo_entries =
+		ARRAY_SIZE(mvm->shared_mem_cfg.txfifo_size);
+	for (i = 0; i < ARRAY_SIZE(mem_cfg->txfifo_size); i++)
+		mvm->shared_mem_cfg.txfifo_size[i] =
+			le32_to_cpu(mem_cfg->txfifo_size[i]);
+	for (i = 0; i < ARRAY_SIZE(mvm->shared_mem_cfg.rxfifo_size); i++)
+		mvm->shared_mem_cfg.rxfifo_size[i] =
+			le32_to_cpu(mem_cfg->rxfifo_size[i]);
 
-	if (fw_has_capa(&mvm->fw->ucode_capa,
-			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG))
-		cmd.id = iwl_cmd_id(SHARED_MEM_CFG_CMD, SYSTEM_GROUP, 0);
-	else
-		cmd.id = SHARED_MEM_CFG;
+	BUILD_BUG_ON(sizeof(mvm->shared_mem_cfg.internal_txfifo_size) !=
+		     sizeof(mem_cfg->internal_txfifo_size));
 
-	if (WARN_ON(iwl_mvm_send_cmd(mvm, &cmd)))
-		return;
+	for (i = 0; i < ARRAY_SIZE(mvm->shared_mem_cfg.internal_txfifo_size);
+	     i++)
+		mvm->shared_mem_cfg.internal_txfifo_size[i] =
+			le32_to_cpu(mem_cfg->internal_txfifo_size[i]);
+}
 
-	pkt = cmd.resp_pkt;
-	mem_cfg = (void *)pkt->data;
-
-	mvm->shared_mem_cfg.shared_mem_addr =
-		le32_to_cpu(mem_cfg->shared_mem_addr);
-	mvm->shared_mem_cfg.shared_mem_size =
-		le32_to_cpu(mem_cfg->shared_mem_size);
-	mvm->shared_mem_cfg.sample_buff_addr =
-		le32_to_cpu(mem_cfg->sample_buff_addr);
-	mvm->shared_mem_cfg.sample_buff_size =
-		le32_to_cpu(mem_cfg->sample_buff_size);
-	mvm->shared_mem_cfg.txfifo_addr = le32_to_cpu(mem_cfg->txfifo_addr);
-	for (i = 0; i < ARRAY_SIZE(mvm->shared_mem_cfg.txfifo_size); i++)
+static void iwl_mvm_parse_shared_mem(struct iwl_mvm *mvm,
+				     struct iwl_rx_packet *pkt)
+{
+	struct iwl_shared_mem_cfg_v1 *mem_cfg = (void *)pkt->data;
+	int i;
+
+	mvm->shared_mem_cfg.num_txfifo_entries =
+		ARRAY_SIZE(mvm->shared_mem_cfg.txfifo_size);
+	for (i = 0; i < ARRAY_SIZE(mem_cfg->txfifo_size); i++)
 		mvm->shared_mem_cfg.txfifo_size[i] =
 			le32_to_cpu(mem_cfg->txfifo_size[i]);
 	for (i = 0; i < ARRAY_SIZE(mvm->shared_mem_cfg.rxfifo_size); i++)
 		mvm->shared_mem_cfg.rxfifo_size[i] =
 			le32_to_cpu(mem_cfg->rxfifo_size[i]);
-	mvm->shared_mem_cfg.page_buff_addr =
-		le32_to_cpu(mem_cfg->page_buff_addr);
-	mvm->shared_mem_cfg.page_buff_size =
-		le32_to_cpu(mem_cfg->page_buff_size);
 
-	/* new API has more data */
+	/* new API has more data, from rxfifo_addr field and on */
 	if (fw_has_capa(&mvm->fw->ucode_capa,
 			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
-		mvm->shared_mem_cfg.rxfifo_addr =
-			le32_to_cpu(mem_cfg->rxfifo_addr);
-		mvm->shared_mem_cfg.internal_txfifo_addr =
-			le32_to_cpu(mem_cfg->internal_txfifo_addr);
-
 		BUILD_BUG_ON(sizeof(mvm->shared_mem_cfg.internal_txfifo_size) !=
 			     sizeof(mem_cfg->internal_txfifo_size));
 
@@ -888,6 +880,33 @@ static void iwl_mvm_get_shared_mem_conf(struct iwl_mvm *mvm)
 			mvm->shared_mem_cfg.internal_txfifo_size[i] =
 				le32_to_cpu(mem_cfg->internal_txfifo_size[i]);
 	}
+}
+
+static void iwl_mvm_get_shared_mem_conf(struct iwl_mvm *mvm)
+{
+	struct iwl_host_cmd cmd = {
+		.flags = CMD_WANT_SKB,
+		.data = { NULL, },
+		.len = { 0, },
+	};
+	struct iwl_rx_packet *pkt;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG))
+		cmd.id = iwl_cmd_id(SHARED_MEM_CFG_CMD, SYSTEM_GROUP, 0);
+	else
+		cmd.id = SHARED_MEM_CFG;
+
+	if (WARN_ON(iwl_mvm_send_cmd(mvm, &cmd)))
+		return;
+
+	pkt = cmd.resp_pkt;
+	if (iwl_mvm_has_new_tx_api(mvm))
+		iwl_mvm_parse_shared_mem_a000(mvm, pkt);
+	else
+		iwl_mvm_parse_shared_mem(mvm, pkt);
 
 	IWL_DEBUG_INFO(mvm, "SHARED MEM CFG: got memory offsets/sizes\n");
 
@@ -1027,19 +1046,11 @@ static int iwl_mvm_sar_init(struct iwl_mvm *mvm)
 {
 	struct iwl_mvm_sar_table sar_table;
 	struct iwl_dev_tx_power_cmd cmd = {
-		.v3.v2.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_CHAINS),
+		.v3.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_CHAINS),
 	};
 	int ret, i, j, idx;
 	int len = sizeof(cmd);
 
-	/* we can't do anything with the table if the FW doesn't support it */
-	if (!fw_has_api(&mvm->fw->ucode_capa,
-			IWL_UCODE_TLV_API_TX_POWER_CHAIN)) {
-		IWL_DEBUG_RADIO(mvm,
-				"FW doesn't support per-chain TX power settings.\n");
-		return 0;
-	}
-
 	if (!fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_TX_POWER_ACK))
 		len = sizeof(cmd.v3);
 
@@ -1096,27 +1107,27 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	 * (for example, if we were in RFKILL)
 	 */
 	ret = iwl_run_init_mvm_ucode(mvm, false);
-	if (ret && !iwlmvm_mod_params.init_dbg) {
+
+	if (iwlmvm_mod_params.init_dbg)
+		return 0;
+
+	if (ret) {
 		IWL_ERR(mvm, "Failed to run INIT ucode: %d\n", ret);
 		/* this can't happen */
 		if (WARN_ON(ret > 0))
 			ret = -ERFKILL;
 		goto error;
 	}
-	if (!iwlmvm_mod_params.init_dbg) {
-		/*
-		 * Stop and start the transport without entering low power
-		 * mode. This will save the state of other components on the
-		 * device that are triggered by the INIT firwmare (MFUART).
-		 */
-		_iwl_trans_stop_device(mvm->trans, false);
-		ret = _iwl_trans_start_hw(mvm->trans, false);
-		if (ret)
-			goto error;
-	}
 
-	if (iwlmvm_mod_params.init_dbg)
-		return 0;
+	/*
+	 * Stop and start the transport without entering low power
+	 * mode. This will save the state of other components on the
+	 * device that are triggered by the INIT firwmare (MFUART).
+	 */
+	_iwl_trans_stop_device(mvm->trans, false);
+	ret = _iwl_trans_start_hw(mvm->trans, false);
+	if (ret)
+		goto error;
 
 	ret = iwl_mvm_load_ucode_wait_alive(mvm, IWL_UCODE_REGULAR);
 	if (ret) {
@@ -1214,9 +1225,12 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	}
 
 	/* TODO: read the budget from BIOS / Platform NVM */
-	if (iwl_mvm_is_ctdp_supported(mvm) && mvm->cooling_dev.cur_state > 0)
+	if (iwl_mvm_is_ctdp_supported(mvm) && mvm->cooling_dev.cur_state > 0) {
 		ret = iwl_mvm_ctdp_command(mvm, CTDP_CMD_OPERATION_START,
 					   mvm->cooling_dev.cur_state);
+		if (ret)
+			goto error;
+	}
 #else
 	/* Initialize tx backoffs to the minimal possible */
 	iwl_mvm_tt_tx_backoff(mvm, 0);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
index 69c42ce45b8a..6b962d6b067a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
@@ -539,6 +539,11 @@ void iwl_mvm_mac_ctxt_release(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 			iwl_mvm_disable_txq(mvm, IWL_MVM_OFFCHANNEL_QUEUE,
 					    IWL_MVM_OFFCHANNEL_QUEUE,
 					    IWL_MAX_TID_COUNT, 0);
+		else
+			iwl_mvm_disable_txq(mvm,
+					    IWL_MVM_DQA_P2P_DEVICE_QUEUE,
+					    vif->hw_queue[0], IWL_MAX_TID_COUNT,
+					    0);
 
 		break;
 	case NL80211_IFTYPE_AP:
@@ -769,26 +774,6 @@ static void iwl_mvm_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
 		cmd->ac[txf].fifos_mask = BIT(txf);
 	}
 
-	if (vif->type == NL80211_IFTYPE_AP) {
-		/* in AP mode, the MCAST FIFO takes the EDCA params from VO */
-		cmd->ac[IWL_MVM_TX_FIFO_VO].fifos_mask |=
-			BIT(IWL_MVM_TX_FIFO_MCAST);
-
-		/*
-		 * in AP mode, pass probe requests and beacons from other APs
-		 * (needed for ht protection); when there're no any associated
-		 * station don't ask FW to pass beacons to prevent unnecessary
-		 * wake-ups.
-		 */
-		cmd->filter_flags |= cpu_to_le32(MAC_FILTER_IN_PROBE_REQUEST);
-		if (mvmvif->ap_assoc_sta_count || !mvm->drop_bcn_ap_mode) {
-			cmd->filter_flags |= cpu_to_le32(MAC_FILTER_IN_BEACON);
-			IWL_DEBUG_HC(mvm, "Asking FW to pass beacons\n");
-		} else {
-			IWL_DEBUG_HC(mvm, "No need to receive beacons\n");
-		}
-	}
-
 	if (vif->bss_conf.qos)
 		cmd->qos_flags |= cpu_to_le32(MAC_QOS_FLG_UPDATE_EDCA);
 
@@ -1186,6 +1171,7 @@ static void iwl_mvm_mac_ap_iterator(void *_data, u8 *mac,
  */
 static void iwl_mvm_mac_ctxt_cmd_fill_ap(struct iwl_mvm *mvm,
 					 struct ieee80211_vif *vif,
+					 struct iwl_mac_ctx_cmd *cmd,
 					 struct iwl_mac_data_ap *ctxt_ap,
 					 bool add)
 {
@@ -1196,6 +1182,23 @@ static void iwl_mvm_mac_ctxt_cmd_fill_ap(struct iwl_mvm *mvm,
 		.beacon_device_ts = 0
 	};
 
+	/* in AP mode, the MCAST FIFO takes the EDCA params from VO */
+	cmd->ac[IWL_MVM_TX_FIFO_VO].fifos_mask |= BIT(IWL_MVM_TX_FIFO_MCAST);
+
+	/*
+	 * in AP mode, pass probe requests and beacons from other APs
+	 * (needed for ht protection); when there're no any associated
+	 * station don't ask FW to pass beacons to prevent unnecessary
+	 * wake-ups.
+	 */
+	cmd->filter_flags |= cpu_to_le32(MAC_FILTER_IN_PROBE_REQUEST);
+	if (mvmvif->ap_assoc_sta_count || !mvm->drop_bcn_ap_mode) {
+		cmd->filter_flags |= cpu_to_le32(MAC_FILTER_IN_BEACON);
+		IWL_DEBUG_HC(mvm, "Asking FW to pass beacons\n");
+	} else {
+		IWL_DEBUG_HC(mvm, "No need to receive beacons\n");
+	}
+
 	ctxt_ap->bi = cpu_to_le32(vif->bss_conf.beacon_int);
 	ctxt_ap->bi_reciprocal =
 		cpu_to_le32(iwl_mvm_reciprocal(vif->bss_conf.beacon_int));
@@ -1253,7 +1256,7 @@ static int iwl_mvm_mac_ctxt_cmd_ap(struct iwl_mvm *mvm,
 	iwl_mvm_mac_ctxt_cmd_common(mvm, vif, &cmd, NULL, action);
 
 	/* Fill the data specific for ap mode */
-	iwl_mvm_mac_ctxt_cmd_fill_ap(mvm, vif, &cmd.ap,
+	iwl_mvm_mac_ctxt_cmd_fill_ap(mvm, vif, &cmd, &cmd.ap,
 				     action == FW_CTXT_ACTION_ADD);
 
 	return iwl_mvm_mac_ctxt_send_cmd(mvm, &cmd);
@@ -1272,7 +1275,7 @@ static int iwl_mvm_mac_ctxt_cmd_go(struct iwl_mvm *mvm,
 	iwl_mvm_mac_ctxt_cmd_common(mvm, vif, &cmd, NULL, action);
 
 	/* Fill the data specific for GO mode */
-	iwl_mvm_mac_ctxt_cmd_fill_ap(mvm, vif, &cmd.go.ap,
+	iwl_mvm_mac_ctxt_cmd_fill_ap(mvm, vif, &cmd, &cmd.go.ap,
 				     action == FW_CTXT_ACTION_ADD);
 
 	cmd.go.ctwin = cpu_to_le32(noa->oppps_ctwindow &
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
index 6d6064534d59..318efd814037 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
@@ -465,7 +465,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	hw->uapsd_queues = IWL_MVM_UAPSD_QUEUES;
 	hw->uapsd_max_sp_len = IWL_UAPSD_MAX_SP;
 
-	BUILD_BUG_ON(ARRAY_SIZE(mvm->ciphers) < ARRAY_SIZE(mvm_ciphers) + 4);
+	BUILD_BUG_ON(ARRAY_SIZE(mvm->ciphers) < ARRAY_SIZE(mvm_ciphers) + 6);
 	memcpy(mvm->ciphers, mvm_ciphers, sizeof(mvm_ciphers));
 	hw->wiphy->n_cipher_suites = ARRAY_SIZE(mvm_ciphers);
 	hw->wiphy->cipher_suites = mvm->ciphers;
@@ -479,17 +479,23 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		hw->wiphy->n_cipher_suites++;
 	}
 
-	/*
-	 * Enable 11w if advertised by firmware and software crypto
-	 * is not enabled (as the firmware will interpret some mgmt
-	 * packets, so enabling it with software crypto isn't safe)
+	/* Enable 11w if software crypto is not enabled (as the
+	 * firmware will interpret some mgmt packets, so enabling it
+	 * with software crypto isn't safe).
 	 */
-	if (mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_MFP &&
-	    !iwlwifi_mod_params.sw_crypto) {
+	if (!iwlwifi_mod_params.sw_crypto) {
 		ieee80211_hw_set(hw, MFP_CAPABLE);
 		mvm->ciphers[hw->wiphy->n_cipher_suites] =
 			WLAN_CIPHER_SUITE_AES_CMAC;
 		hw->wiphy->n_cipher_suites++;
+		if (iwl_mvm_has_new_rx_api(mvm)) {
+			mvm->ciphers[hw->wiphy->n_cipher_suites] =
+				WLAN_CIPHER_SUITE_BIP_GMAC_128;
+			hw->wiphy->n_cipher_suites++;
+			mvm->ciphers[hw->wiphy->n_cipher_suites] =
+				WLAN_CIPHER_SUITE_BIP_GMAC_256;
+			hw->wiphy->n_cipher_suites++;
+		}
 	}
 
 	/* currently FW API supports only one optional cipher scheme */
@@ -539,9 +545,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		hw->wiphy->regulatory_flags |= REGULATORY_CUSTOM_REG |
 					       REGULATORY_DISABLE_BEACON_HINTS;
 
-	if (mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_GO_UAPSD)
-		hw->wiphy->flags |= WIPHY_FLAG_AP_UAPSD;
-
+	hw->wiphy->flags |= WIPHY_FLAG_AP_UAPSD;
 	hw->wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH;
 
 	hw->wiphy->iface_combinations = iwl_mvm_iface_combinations;
@@ -624,6 +628,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	hw->wiphy->features |= NL80211_FEATURE_P2P_GO_CTWIN |
 			       NL80211_FEATURE_LOW_PRIORITY_SCAN |
 			       NL80211_FEATURE_P2P_GO_OPPPS |
+			       NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE |
 			       NL80211_FEATURE_DYNAMIC_SMPS |
 			       NL80211_FEATURE_STATIC_SMPS |
 			       NL80211_FEATURE_SUPPORTS_WMM_ADMISSION;
@@ -644,6 +649,16 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 			IWL_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT))
 		hw->wiphy->features |= NL80211_FEATURE_WFA_TPC_IE_IN_PROBES;
 
+	if (fw_has_api(&mvm->fw->ucode_capa,
+		       IWL_UCODE_TLV_API_SCAN_TSF_REPORT)) {
+		wiphy_ext_feature_set(hw->wiphy,
+				      NL80211_EXT_FEATURE_SCAN_START_TIME);
+		wiphy_ext_feature_set(hw->wiphy,
+				      NL80211_EXT_FEATURE_BSS_PARENT_TSF);
+		wiphy_ext_feature_set(hw->wiphy,
+				      NL80211_EXT_FEATURE_SET_SCAN_DWELL);
+	}
+
 	mvm->rts_threshold = IEEE80211_MAX_RTS_THRESHOLD;
 
 #ifdef CONFIG_PM_SLEEP
@@ -711,6 +726,10 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	if (ret)
 		iwl_mvm_leds_exit(mvm);
 
+	if (mvm->cfg->vht_mu_mimo_supported)
+		wiphy_ext_feature_set(hw->wiphy,
+				      NL80211_EXT_FEATURE_MU_MIMO_AIR_SNIFFER);
+
 	return ret;
 }
 
@@ -1250,20 +1269,18 @@ static int iwl_mvm_set_tx_power(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 				s16 tx_power)
 {
 	struct iwl_dev_tx_power_cmd cmd = {
-		.v3.v2.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_MAC),
-		.v3.v2.mac_context_id =
+		.v3.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_MAC),
+		.v3.mac_context_id =
 			cpu_to_le32(iwl_mvm_vif_from_mac80211(vif)->id),
-		.v3.v2.pwr_restriction = cpu_to_le16(8 * tx_power),
+		.v3.pwr_restriction = cpu_to_le16(8 * tx_power),
 	};
 	int len = sizeof(cmd);
 
 	if (tx_power == IWL_DEFAULT_MAX_TX_POWER)
-		cmd.v3.v2.pwr_restriction = cpu_to_le16(IWL_DEV_MAX_TX_POWER);
+		cmd.v3.pwr_restriction = cpu_to_le16(IWL_DEV_MAX_TX_POWER);
 
 	if (!fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_TX_POWER_ACK))
 		len = sizeof(cmd.v3);
-	if (!fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_TX_POWER_CHAIN))
-		len = sizeof(cmd.v3.v2);
 
 	return iwl_mvm_send_cmd_pdu(mvm, REDUCE_TX_POWER_CMD, 0, len, &cmd);
 }
@@ -2220,6 +2237,10 @@ static void iwl_mvm_bss_info_changed(struct ieee80211_hw *hw,
 	case NL80211_IFTYPE_ADHOC:
 		iwl_mvm_bss_info_changed_ap_ibss(mvm, vif, bss_conf, changes);
 		break;
+	case NL80211_IFTYPE_MONITOR:
+		if (changes & BSS_CHANGED_MU_GROUPS)
+			iwl_mvm_update_mu_groups(mvm, vif);
+		break;
 	default:
 		/* shouldn't happen */
 		WARN_ON_ONCE(1);
@@ -2746,6 +2767,8 @@ static int iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 		key->flags |= IEEE80211_KEY_FLAG_PUT_IV_SPACE;
 		break;
 	case WLAN_CIPHER_SUITE_AES_CMAC:
+	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
+	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
 		WARN_ON_ONCE(!ieee80211_hw_check(hw, MFP_CAPABLE));
 		break;
 	case WLAN_CIPHER_SUITE_WEP40:
@@ -2779,9 +2802,11 @@ static int iwl_mvm_mac_set_key(struct ieee80211_hw *hw,
 			 * GTK on AP interface is a TX-only key, return 0;
 			 * on IBSS they're per-station and because we're lazy
 			 * we don't support them for RX, so do the same.
-			 * CMAC in AP/IBSS modes must be done in software.
+			 * CMAC/GMAC in AP/IBSS modes must be done in software.
 			 */
-			if (key->cipher == WLAN_CIPHER_SUITE_AES_CMAC)
+			if (key->cipher == WLAN_CIPHER_SUITE_AES_CMAC ||
+			    key->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_128 ||
+			    key->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_256)
 				ret = -EOPNOTSUPP;
 			else
 				ret = 0;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index b4fc86d5d7ef..d17cbf603f7c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -467,6 +467,8 @@ struct iwl_mvm_vif {
 static inline struct iwl_mvm_vif *
 iwl_mvm_vif_from_mac80211(struct ieee80211_vif *vif)
 {
+	if (!vif)
+		return NULL;
 	return (void *)vif->drv_priv;
 }
 
@@ -602,16 +604,9 @@ enum iwl_mvm_tdls_cs_state {
 };
 
 struct iwl_mvm_shared_mem_cfg {
-	u32 shared_mem_addr;
-	u32 shared_mem_size;
-	u32 sample_buff_addr;
-	u32 sample_buff_size;
-	u32 txfifo_addr;
+	int num_txfifo_entries;
 	u32 txfifo_size[TX_FIFO_MAX_NUM];
 	u32 rxfifo_size[RX_FIFO_MAX_NUM];
-	u32 page_buff_addr;
-	u32 page_buff_size;
-	u32 rxfifo_addr;
 	u32 internal_txfifo_addr;
 	u32 internal_txfifo_size[TX_FIFO_INTERNAL_MAX_NUM];
 };
@@ -697,6 +692,10 @@ struct iwl_mvm_baid_data {
  *	it. In this state, when a new queue is needed to be allocated but no
  *	such free queue exists, an inactive queue might be freed and given to
  *	the new RA/TID.
+ * @IWL_MVM_QUEUE_RECONFIGURING: queue is being reconfigured
+ *	This is the state of a queue that has had traffic pass through it, but
+ *	needs to be reconfigured for some reason, e.g. the queue needs to
+ *	become unshared and aggregations re-enabled on.
  */
 enum iwl_mvm_queue_status {
 	IWL_MVM_QUEUE_FREE,
@@ -704,10 +703,11 @@ enum iwl_mvm_queue_status {
 	IWL_MVM_QUEUE_READY,
 	IWL_MVM_QUEUE_SHARED,
 	IWL_MVM_QUEUE_INACTIVE,
+	IWL_MVM_QUEUE_RECONFIGURING,
 };
 
 #define IWL_MVM_DQA_QUEUE_TIMEOUT	(5 * HZ)
-#define IWL_MVM_NUM_CIPHERS             8
+#define IWL_MVM_NUM_CIPHERS             10
 
 struct iwl_mvm {
 	/* for logger access */
@@ -767,6 +767,7 @@ struct iwl_mvm {
 		u8 ra_sta_id; /* The RA this queue is mapped to, if exists */
 		bool reserved; /* Is this the TXQ reserved for a STA */
 		u8 mac80211_ac; /* The mac80211 AC this queue is mapped to */
+		u8 txq_tid; /* The TID "owner" of this queue*/
 		u16 tid_bitmap; /* Bitmap of the TIDs mapped to this queue */
 		/* Timestamp for inactivation per TID of this queue */
 		unsigned long last_frame_time[IWL_MAX_TID_COUNT + 1];
@@ -820,6 +821,12 @@ struct iwl_mvm {
 	/* UMAC scan tracking */
 	u32 scan_uid_status[IWL_MVM_MAX_UMAC_SCANS];
 
+	/* start time of last scan in TSF of the mac that requested the scan */
+	u64 scan_start;
+
+	/* the vif that requested the current scan */
+	struct iwl_mvm_vif *scan_vif;
+
 	/* rx chain antennas set through debugfs for the scan command */
 	u8 scan_rx_ant;
 
@@ -1122,6 +1129,18 @@ static inline bool iwl_mvm_enter_d0i3_on_suspend(struct iwl_mvm *mvm)
 		(mvm->trans->runtime_pm_mode != IWL_PLAT_PM_MODE_D0I3);
 }
 
+static inline bool iwl_mvm_is_dqa_data_queue(struct iwl_mvm *mvm, u8 queue)
+{
+	return (queue >= IWL_MVM_DQA_MIN_DATA_QUEUE) &&
+	       (queue <= IWL_MVM_DQA_MAX_DATA_QUEUE);
+}
+
+static inline bool iwl_mvm_is_dqa_mgmt_queue(struct iwl_mvm *mvm, u8 queue)
+{
+	return (queue >= IWL_MVM_DQA_MIN_MGMT_QUEUE) &&
+	       (queue <= IWL_MVM_DQA_MAX_MGMT_QUEUE);
+}
+
 static inline bool iwl_mvm_is_lar_supported(struct iwl_mvm *mvm)
 {
 	bool nvm_lar = mvm->nvm_data->lar_enabled;
@@ -1192,6 +1211,12 @@ static inline bool iwl_mvm_has_new_rx_api(struct iwl_mvm *mvm)
 			   IWL_UCODE_TLV_CAPA_MULTI_QUEUE_RX_SUPPORT);
 }
 
+static inline bool iwl_mvm_has_new_tx_api(struct iwl_mvm *mvm)
+{
+	/* TODO - replace with TLV once defined */
+	return mvm->trans->cfg->use_tfh;
+}
+
 static inline bool iwl_mvm_is_tt_in_fw(struct iwl_mvm *mvm)
 {
 #ifdef CONFIG_THERMAL
@@ -1243,6 +1268,7 @@ u8 iwl_mvm_mac80211_idx_to_hwrate(int rate_idx);
 void iwl_mvm_dump_nic_error_log(struct iwl_mvm *mvm);
 u8 first_antenna(u8 mask);
 u8 iwl_mvm_next_antenna(struct iwl_mvm *mvm, u8 valid, u8 last_idx);
+void iwl_mvm_get_sync_time(struct iwl_mvm *mvm, u32 *gp2, u64 *boottime);
 
 /* Tx / Host Commands */
 int __must_check iwl_mvm_send_cmd(struct iwl_mvm *mvm,
@@ -1279,8 +1305,6 @@ static inline void iwl_mvm_set_tx_cmd_ccmp(struct ieee80211_tx_info *info,
 
 	tx_cmd->sec_ctl = TX_CMD_SEC_CCM;
 	memcpy(tx_cmd->key, keyconf->key, keyconf->keylen);
-	if (info->flags & IEEE80211_TX_CTL_AMPDU)
-		tx_cmd->tx_flags |= cpu_to_le32(TX_CMD_FLG_CCMP_AGG);
 }
 
 static inline void iwl_mvm_wait_for_async_handlers(struct iwl_mvm *mvm)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
index 7a686f67f007..eade099b6dbf 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
@@ -66,7 +66,6 @@
  *****************************************************************************/
 #include <linux/firmware.h>
 #include <linux/rtnetlink.h>
-#include <linux/acpi.h>
 #include "iwl-trans.h"
 #include "iwl-csr.h"
 #include "mvm.h"
@@ -751,96 +750,6 @@ exit:
 	return resp_cp;
 }
 
-#ifdef CONFIG_ACPI
-#define WRD_METHOD		"WRDD"
-#define WRDD_WIFI		(0x07)
-#define WRDD_WIGIG		(0x10)
-
-static u32 iwl_mvm_wrdd_get_mcc(struct iwl_mvm *mvm, union acpi_object *wrdd)
-{
-	union acpi_object *mcc_pkg, *domain_type, *mcc_value;
-	u32 i;
-
-	if (wrdd->type != ACPI_TYPE_PACKAGE ||
-	    wrdd->package.count < 2 ||
-	    wrdd->package.elements[0].type != ACPI_TYPE_INTEGER ||
-	    wrdd->package.elements[0].integer.value != 0) {
-		IWL_DEBUG_LAR(mvm, "Unsupported wrdd structure\n");
-		return 0;
-	}
-
-	for (i = 1 ; i < wrdd->package.count ; ++i) {
-		mcc_pkg = &wrdd->package.elements[i];
-
-		if (mcc_pkg->type != ACPI_TYPE_PACKAGE ||
-		    mcc_pkg->package.count < 2 ||
-		    mcc_pkg->package.elements[0].type != ACPI_TYPE_INTEGER ||
-		    mcc_pkg->package.elements[1].type != ACPI_TYPE_INTEGER) {
-			mcc_pkg = NULL;
-			continue;
-		}
-
-		domain_type = &mcc_pkg->package.elements[0];
-		if (domain_type->integer.value == WRDD_WIFI)
-			break;
-
-		mcc_pkg = NULL;
-	}
-
-	if (mcc_pkg) {
-		mcc_value = &mcc_pkg->package.elements[1];
-		return mcc_value->integer.value;
-	}
-
-	return 0;
-}
-
-static int iwl_mvm_get_bios_mcc(struct iwl_mvm *mvm, char *mcc)
-{
-	acpi_handle root_handle;
-	acpi_handle handle;
-	struct acpi_buffer wrdd = {ACPI_ALLOCATE_BUFFER, NULL};
-	acpi_status status;
-	u32 mcc_val;
-
-	root_handle = ACPI_HANDLE(mvm->dev);
-	if (!root_handle) {
-		IWL_DEBUG_LAR(mvm,
-			      "Could not retrieve root port ACPI handle\n");
-		return -ENOENT;
-	}
-
-	/* Get the method's handle */
-	status = acpi_get_handle(root_handle, (acpi_string)WRD_METHOD, &handle);
-	if (ACPI_FAILURE(status)) {
-		IWL_DEBUG_LAR(mvm, "WRD method not found\n");
-		return -ENOENT;
-	}
-
-	/* Call WRDD with no arguments */
-	status = acpi_evaluate_object(handle, NULL, NULL, &wrdd);
-	if (ACPI_FAILURE(status)) {
-		IWL_DEBUG_LAR(mvm, "WRDC invocation failed (0x%x)\n", status);
-		return -ENOENT;
-	}
-
-	mcc_val = iwl_mvm_wrdd_get_mcc(mvm, wrdd.pointer);
-	kfree(wrdd.pointer);
-	if (!mcc_val)
-		return -ENOENT;
-
-	mcc[0] = (mcc_val >> 8) & 0xff;
-	mcc[1] = mcc_val & 0xff;
-	mcc[2] = '\0';
-	return 0;
-}
-#else /* CONFIG_ACPI */
-static int iwl_mvm_get_bios_mcc(struct iwl_mvm *mvm, char *mcc)
-{
-	return -ENOENT;
-}
-#endif
-
 int iwl_mvm_init_mcc(struct iwl_mvm *mvm)
 {
 	bool tlv_lar;
@@ -884,7 +793,7 @@ int iwl_mvm_init_mcc(struct iwl_mvm *mvm)
 		return -EIO;
 
 	if (iwl_mvm_is_wifi_mcc_supported(mvm) &&
-	    !iwl_mvm_get_bios_mcc(mvm, mcc)) {
+	    !iwl_get_bios_mcc(mvm->dev, mcc)) {
 		kfree(regd);
 		regd = iwl_mvm_get_regdomain(mvm->hw->wiphy, mcc,
 					     MCC_SOURCE_BIOS, NULL);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index 55d9096da68c..05fe6dd1a2c8 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -359,6 +359,7 @@ static const struct iwl_hcmd_names iwl_mvm_legacy_names[] = {
 	HCMD_NAME(BT_COEX_CI),
 	HCMD_NAME(PHY_CONFIGURATION_CMD),
 	HCMD_NAME(CALIB_RES_NOTIF_PHY_DB),
+	HCMD_NAME(PHY_DB_CMD),
 	HCMD_NAME(SCAN_OFFLOAD_COMPLETE),
 	HCMD_NAME(SCAN_OFFLOAD_UPDATE_PROFILES_CMD),
 	HCMD_NAME(SCAN_OFFLOAD_CONFIG_CMD),
@@ -652,11 +653,9 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	/* the hardware splits the A-MSDU */
 	if (mvm->cfg->mq_rx_supported)
 		trans_cfg.rx_buf_size = IWL_AMSDU_4K;
-	trans_cfg.wide_cmd_header = fw_has_api(&mvm->fw->ucode_capa,
-					       IWL_UCODE_TLV_API_WIDE_CMD_HDR);
 
-	if (mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_DW_BC_TABLE)
-		trans_cfg.bc_table_dword = true;
+	trans->wide_cmd_header = true;
+	trans_cfg.bc_table_dword = true;
 
 	trans_cfg.command_groups = iwl_mvm_groups;
 	trans_cfg.command_groups_size = ARRAY_SIZE(iwl_mvm_groups);
@@ -711,37 +710,21 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 		IWL_DEBUG_EEPROM(mvm->trans->dev,
 				 "working without external nvm file\n");
 
-	if (WARN(cfg->no_power_up_nic_in_init && !mvm->nvm_file_name,
-		 "not allowing power-up and not having nvm_file\n"))
+	err = iwl_trans_start_hw(mvm->trans);
+	if (err)
 		goto out_free;
 
-	/*
-	 * Even if nvm exists in the nvm_file driver should read again the nvm
-	 * from the nic because there might be entries that exist in the OTP
-	 * and not in the file.
-	 * for nics with no_power_up_nic_in_init: rely completley on nvm_file
-	 */
-	if (cfg->no_power_up_nic_in_init && mvm->nvm_file_name) {
-		err = iwl_nvm_init(mvm, false);
-		if (err)
-			goto out_free;
-	} else {
-		err = iwl_trans_start_hw(mvm->trans);
-		if (err)
-			goto out_free;
-
-		mutex_lock(&mvm->mutex);
-		iwl_mvm_ref(mvm, IWL_MVM_REF_INIT_UCODE);
-		err = iwl_run_init_mvm_ucode(mvm, true);
-		if (!err || !iwlmvm_mod_params.init_dbg)
-			iwl_mvm_stop_device(mvm);
-		iwl_mvm_unref(mvm, IWL_MVM_REF_INIT_UCODE);
-		mutex_unlock(&mvm->mutex);
-		/* returns 0 if successful, 1 if success but in rfkill */
-		if (err < 0 && !iwlmvm_mod_params.init_dbg) {
-			IWL_ERR(mvm, "Failed to run INIT ucode: %d\n", err);
-			goto out_free;
-		}
+	mutex_lock(&mvm->mutex);
+	iwl_mvm_ref(mvm, IWL_MVM_REF_INIT_UCODE);
+	err = iwl_run_init_mvm_ucode(mvm, true);
+	if (!err || !iwlmvm_mod_params.init_dbg)
+		iwl_mvm_stop_device(mvm);
+	iwl_mvm_unref(mvm, IWL_MVM_REF_INIT_UCODE);
+	mutex_unlock(&mvm->mutex);
+	/* returns 0 if successful, 1 if success but in rfkill */
+	if (err < 0 && !iwlmvm_mod_params.init_dbg) {
+		IWL_ERR(mvm, "Failed to run INIT ucode: %d\n", err);
+		goto out_free;
 	}
 
 	scan_size = iwl_mvm_scan_size(mvm);
@@ -783,8 +766,8 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	flush_delayed_work(&mvm->fw_dump_wk);
 	iwl_phy_db_free(mvm->phy_db);
 	kfree(mvm->scan_cmd);
-	if (!cfg->no_power_up_nic_in_init || !mvm->nvm_file_name)
-		iwl_trans_op_mode_leave(trans);
+	iwl_trans_op_mode_leave(trans);
+
 	ieee80211_free_hw(mvm->hw);
 	return NULL;
 }
@@ -857,9 +840,7 @@ static void iwl_mvm_async_handlers_wk(struct work_struct *wk)
 	struct iwl_mvm *mvm =
 		container_of(wk, struct iwl_mvm, async_handlers_wk);
 	struct iwl_async_handler_entry *entry, *tmp;
-	struct list_head local_list;
-
-	INIT_LIST_HEAD(&local_list);
+	LIST_HEAD(local_list);
 
 	/* Ensure that we are not in stop flow (check iwl_mvm_mac_stop) */
 
@@ -966,10 +947,11 @@ static void iwl_mvm_rx(struct iwl_op_mode *op_mode,
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
+	u16 cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
 
-	if (likely(pkt->hdr.cmd == REPLY_RX_MPDU_CMD))
+	if (likely(cmd == WIDE_ID(LEGACY_GROUP, REPLY_RX_MPDU_CMD)))
 		iwl_mvm_rx_rx_mpdu(mvm, napi, rxb);
-	else if (pkt->hdr.cmd == REPLY_RX_PHY_CMD)
+	else if (cmd == WIDE_ID(LEGACY_GROUP, REPLY_RX_PHY_CMD))
 		iwl_mvm_rx_rx_phy_cmd(mvm, rxb);
 	else
 		iwl_mvm_rx_common(mvm, rxb, pkt);
@@ -981,13 +963,14 @@ static void iwl_mvm_rx_mq(struct iwl_op_mode *op_mode,
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
+	u16 cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
 
-	if (likely(pkt->hdr.cmd == REPLY_RX_MPDU_CMD))
+	if (likely(cmd == WIDE_ID(LEGACY_GROUP, REPLY_RX_MPDU_CMD)))
 		iwl_mvm_rx_mpdu_mq(mvm, napi, rxb, 0);
-	else if (unlikely(pkt->hdr.group_id == DATA_PATH_GROUP &&
-			  pkt->hdr.cmd == RX_QUEUES_NOTIFICATION))
+	else if (unlikely(cmd == WIDE_ID(DATA_PATH_GROUP,
+					 RX_QUEUES_NOTIFICATION)))
 		iwl_mvm_rx_queue_notif(mvm, rxb, 0);
-	else if (pkt->hdr.cmd == FRAME_RELEASE)
+	else if (cmd == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE))
 		iwl_mvm_rx_frame_release(mvm, napi, rxb, 0);
 	else
 		iwl_mvm_rx_common(mvm, rxb, pkt);
@@ -1666,13 +1649,14 @@ static void iwl_mvm_rx_mq_rss(struct iwl_op_mode *op_mode,
 {
 	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	u16 cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
 
-	if (unlikely(pkt->hdr.cmd == FRAME_RELEASE))
+	if (unlikely(cmd == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE)))
 		iwl_mvm_rx_frame_release(mvm, napi, rxb, queue);
-	else if (unlikely(pkt->hdr.cmd == RX_QUEUES_NOTIFICATION &&
-			  pkt->hdr.group_id == DATA_PATH_GROUP))
+	else if (unlikely(cmd == WIDE_ID(DATA_PATH_GROUP,
+					 RX_QUEUES_NOTIFICATION)))
 		iwl_mvm_rx_queue_notif(mvm, rxb, queue);
-	else
+	else if (likely(cmd == WIDE_ID(LEGACY_GROUP, REPLY_RX_MPDU_CMD)))
 		iwl_mvm_rx_mpdu_mq(mvm, napi, rxb, queue);
 }
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/power.c b/drivers/net/wireless/intel/iwlwifi/mvm/power.c
index ff85865b1dda..af6d10c23e5a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/power.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/power.c
@@ -694,8 +694,7 @@ static void iwl_mvm_power_set_pm(struct iwl_mvm *mvm,
 
 	/* enable PM on p2p if p2p stand alone */
 	if (vifs->p2p_active && !vifs->bss_active && !vifs->ap_active) {
-		if (mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_P2P_PM)
-			p2p_mvmvif->pm_enabled = true;
+		p2p_mvmvif->pm_enabled = true;
 		return;
 	}
 
@@ -707,12 +706,10 @@ static void iwl_mvm_power_set_pm(struct iwl_mvm *mvm,
 				   ap_mvmvif->phy_ctxt->id);
 
 	/* clients are not stand alone: enable PM if DCM */
-	if (!(client_same_channel || ap_same_channel) &&
-	    (mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_BSS_P2P_PS_DCM)) {
+	if (!(client_same_channel || ap_same_channel)) {
 		if (vifs->bss_active)
 			bss_mvmvif->pm_enabled = true;
-		if (vifs->p2p_active &&
-		    (mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_P2P_PM))
+		if (vifs->p2p_active)
 			p2p_mvmvif->pm_enabled = true;
 		return;
 	}
@@ -721,12 +718,10 @@ static void iwl_mvm_power_set_pm(struct iwl_mvm *mvm,
 	 * There is only one channel in the system and there are only
 	 * bss and p2p clients that share it
 	 */
-	if (client_same_channel && !vifs->ap_active &&
-	    (mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_BSS_P2P_PS_SCM)) {
+	if (client_same_channel && !vifs->ap_active) {
 		/* share same channel*/
 		bss_mvmvif->pm_enabled = true;
-		if (mvm->fw->ucode_capa.flags & IWL_UCODE_TLV_FLAGS_P2P_PM)
-			p2p_mvmvif->pm_enabled = true;
+		p2p_mvmvif->pm_enabled = true;
 	}
 }
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
index df6c32caa5f0..a57c6ef5bc14 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
@@ -132,7 +132,8 @@ static inline int iwl_mvm_check_pn(struct iwl_mvm *mvm, struct sk_buff *skb,
 		   IEEE80211_CCMP_PN_LEN) <= 0)
 		return -1;
 
-	memcpy(ptk_pn->q[queue].pn[tid], pn, IEEE80211_CCMP_PN_LEN);
+	if (!(stats->flag & RX_FLAG_AMSDU_MORE))
+		memcpy(ptk_pn->q[queue].pn[tid], pn, IEEE80211_CCMP_PN_LEN);
 	stats->flag |= RX_FLAG_PN_VALIDATED;
 
 	return 0;
@@ -417,10 +418,11 @@ static void iwl_mvm_release_frames(struct iwl_mvm *mvm,
 
 		ssn = ieee80211_sn_inc(ssn);
 
-		/* holes are valid since nssn indicates frames were received. */
-		if (skb_queue_empty(skb_list) || !skb_peek_tail(skb_list))
-			continue;
-		/* Empty the list. Will have more than one frame for A-MSDU */
+		/*
+		 * Empty the list. Will have more than one frame for A-MSDU.
+		 * Empty list is valid as well since nssn indicates frames were
+		 * received.
+		 */
 		while ((skb = __skb_dequeue(skb_list))) {
 			iwl_mvm_pass_packet_to_mac80211(mvm, napi, skb,
 							reorder_buf->queue,
@@ -433,7 +435,7 @@ static void iwl_mvm_release_frames(struct iwl_mvm *mvm,
 	if (reorder_buf->num_stored && !reorder_buf->removed) {
 		u16 index = reorder_buf->head_sn % reorder_buf->buf_size;
 
-		while (!skb_peek_tail(&reorder_buf->entries[index]))
+		while (skb_queue_empty(&reorder_buf->entries[index]))
 			index = (index + 1) % reorder_buf->buf_size;
 		/* modify timer to match next frame's expiration time */
 		mod_timer(&reorder_buf->reorder_timer,
@@ -451,17 +453,17 @@ void iwl_mvm_reorder_timer_expired(unsigned long data)
 	u16 sn = 0, index = 0;
 	bool expired = false;
 
-	spin_lock_bh(&buf->lock);
+	spin_lock(&buf->lock);
 
 	if (!buf->num_stored || buf->removed) {
-		spin_unlock_bh(&buf->lock);
+		spin_unlock(&buf->lock);
 		return;
 	}
 
 	for (i = 0; i < buf->buf_size ; i++) {
 		index = (buf->head_sn + i) % buf->buf_size;
 
-		if (!skb_peek_tail(&buf->entries[index]))
+		if (skb_queue_empty(&buf->entries[index]))
 			continue;
 		if (!time_after(jiffies, buf->reorder_time[index] +
 				RX_REORDER_BUF_TIMEOUT_MQ))
@@ -491,7 +493,7 @@ void iwl_mvm_reorder_timer_expired(unsigned long data)
 			  buf->reorder_time[index] +
 			  1 + RX_REORDER_BUF_TIMEOUT_MQ);
 	}
-	spin_unlock_bh(&buf->lock);
+	spin_unlock(&buf->lock);
 }
 
 static void iwl_mvm_del_ba(struct iwl_mvm *mvm, int queue,
@@ -502,7 +504,7 @@ static void iwl_mvm_del_ba(struct iwl_mvm *mvm, int queue,
 	struct iwl_mvm_reorder_buffer *reorder_buf;
 	u8 baid = data->baid;
 
-	if (WARN_ON_ONCE(baid >= IWL_RX_REORDER_DATA_INVALID_BAID))
+	if (WARN_ONCE(baid >= IWL_MAX_BAID, "invalid BAID: %x\n", baid))
 		return;
 
 	rcu_read_lock();
@@ -589,6 +591,11 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 	baid = (reorder & IWL_RX_MPDU_REORDER_BAID_MASK) >>
 		IWL_RX_MPDU_REORDER_BAID_SHIFT;
 
+	/*
+	 * This also covers the case of receiving a Block Ack Request
+	 * outside a BA session; we'll pass it to mac80211 and that
+	 * then sends a delBA action frame.
+	 */
 	if (baid == IWL_RX_REORDER_DATA_INVALID_BAID)
 		return false;
 
@@ -598,9 +605,10 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 
 	mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 
-	/* not a data packet */
-	if (!ieee80211_is_data_qos(hdr->frame_control) ||
-	    is_multicast_ether_addr(hdr->addr1))
+	/* not a data packet or a bar */
+	if (!ieee80211_is_back_req(hdr->frame_control) &&
+	    (!ieee80211_is_data_qos(hdr->frame_control) ||
+	     is_multicast_ether_addr(hdr->addr1)))
 		return false;
 
 	if (unlikely(!ieee80211_is_data_present(hdr->frame_control)))
@@ -624,6 +632,11 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 
 	spin_lock_bh(&buffer->lock);
 
+	if (ieee80211_is_back_req(hdr->frame_control)) {
+		iwl_mvm_release_frames(mvm, sta, napi, buffer, nssn);
+		goto drop;
+	}
+
 	/*
 	 * If there was a significant jump in the nssn - adjust.
 	 * If the SN is smaller than the NSSN it might need to first go into
@@ -883,6 +896,9 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 			u8 *qc = ieee80211_get_qos_ctl(hdr);
 
 			*qc &= ~IEEE80211_QOS_CTL_A_MSDU_PRESENT;
+			if (!(desc->amsdu_info &
+			      IWL_RX_MPDU_AMSDU_LAST_SUBFRAME))
+				rx_status->flag |= RX_FLAG_AMSDU_MORE;
 		}
 		if (baid != IWL_RX_REORDER_DATA_INVALID_BAID)
 			iwl_mvm_agg_rx_received(mvm, baid);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
index dac120f8861b..f279fdd6eb44 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
@@ -141,6 +141,7 @@ struct iwl_mvm_scan_params {
 	struct cfg80211_match_set *match_sets;
 	int n_scan_plans;
 	struct cfg80211_sched_scan_plan *scan_plans;
+	u32 measurement_dwell;
 };
 
 static u8 iwl_mvm_scan_rx_ant(struct iwl_mvm *mvm)
@@ -232,6 +233,27 @@ iwl_mvm_scan_type iwl_mvm_get_scan_type(struct iwl_mvm *mvm, bool p2p_device)
 	return IWL_SCAN_TYPE_WILD;
 }
 
+static int
+iwl_mvm_get_measurement_dwell(struct iwl_mvm *mvm,
+			      struct cfg80211_scan_request *req,
+			      struct iwl_mvm_scan_params *params)
+{
+	if (!req->duration)
+		return 0;
+
+	if (req->duration_mandatory &&
+	    req->duration > scan_timing[params->type].max_out_time) {
+		IWL_DEBUG_SCAN(mvm,
+			       "Measurement scan - too long dwell %hu (max out time %u)\n",
+			       req->duration,
+			       scan_timing[params->type].max_out_time);
+		return -EOPNOTSUPP;
+	}
+
+	return min_t(u32, (u32)req->duration,
+		     scan_timing[params->type].max_out_time);
+}
+
 static inline bool iwl_mvm_rrm_scan_needed(struct iwl_mvm *mvm)
 {
 	/* require rrm scan whenever the fw supports it */
@@ -717,22 +739,6 @@ iwl_mvm_build_scan_probe(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	params->preq.common_data.len = cpu_to_le16(ies->common_ie_len);
 }
 
-static __le32 iwl_mvm_scan_priority(struct iwl_mvm *mvm,
-				    enum iwl_scan_priority_ext prio)
-{
-	if (fw_has_api(&mvm->fw->ucode_capa,
-		       IWL_UCODE_TLV_API_EXT_SCAN_PRIORITY))
-		return cpu_to_le32(prio);
-
-	if (prio <= IWL_SCAN_PRIORITY_EXT_2)
-		return cpu_to_le32(IWL_SCAN_PRIORITY_LOW);
-
-	if (prio <= IWL_SCAN_PRIORITY_EXT_4)
-		return cpu_to_le32(IWL_SCAN_PRIORITY_MEDIUM);
-
-	return cpu_to_le32(IWL_SCAN_PRIORITY_HIGH);
-}
-
 static void iwl_mvm_scan_lmac_dwell(struct iwl_mvm *mvm,
 				    struct iwl_scan_req_lmac *cmd,
 				    struct iwl_mvm_scan_params *params)
@@ -743,7 +749,7 @@ static void iwl_mvm_scan_lmac_dwell(struct iwl_mvm *mvm,
 	cmd->extended_dwell = scan_timing[params->type].dwell_extended;
 	cmd->max_out_time = cpu_to_le32(scan_timing[params->type].max_out_time);
 	cmd->suspend_time = cpu_to_le32(scan_timing[params->type].suspend_time);
-	cmd->scan_prio = iwl_mvm_scan_priority(mvm, IWL_SCAN_PRIORITY_EXT_6);
+	cmd->scan_prio = cpu_to_le32(IWL_SCAN_PRIORITY_EXT_6);
 }
 
 static inline bool iwl_mvm_scan_fits(struct iwl_mvm *mvm, int n_ssids,
@@ -1033,21 +1039,24 @@ static void iwl_mvm_scan_umac_dwell(struct iwl_mvm *mvm,
 				    struct iwl_scan_req_umac *cmd,
 				    struct iwl_mvm_scan_params *params)
 {
-	cmd->extended_dwell = scan_timing[params->type].dwell_extended;
-	cmd->active_dwell = scan_timing[params->type].dwell_active;
-	cmd->passive_dwell = scan_timing[params->type].dwell_passive;
+	if (params->measurement_dwell) {
+		cmd->active_dwell = params->measurement_dwell;
+		cmd->passive_dwell = params->measurement_dwell;
+		cmd->extended_dwell = params->measurement_dwell;
+	} else {
+		cmd->active_dwell = scan_timing[params->type].dwell_active;
+		cmd->passive_dwell = scan_timing[params->type].dwell_passive;
+		cmd->extended_dwell = scan_timing[params->type].dwell_extended;
+	}
 	cmd->fragmented_dwell = scan_timing[params->type].dwell_fragmented;
 	cmd->max_out_time = cpu_to_le32(scan_timing[params->type].max_out_time);
 	cmd->suspend_time = cpu_to_le32(scan_timing[params->type].suspend_time);
-	cmd->scan_priority =
-		iwl_mvm_scan_priority(mvm, IWL_SCAN_PRIORITY_EXT_6);
+	cmd->scan_priority = cpu_to_le32(IWL_SCAN_PRIORITY_EXT_6);
 
 	if (iwl_mvm_is_regular_scan(params))
-		cmd->ooc_priority =
-			iwl_mvm_scan_priority(mvm, IWL_SCAN_PRIORITY_EXT_6);
+		cmd->ooc_priority = cpu_to_le32(IWL_SCAN_PRIORITY_EXT_6);
 	else
-		cmd->ooc_priority =
-			iwl_mvm_scan_priority(mvm, IWL_SCAN_PRIORITY_EXT_2);
+		cmd->ooc_priority = cpu_to_le32(IWL_SCAN_PRIORITY_EXT_2);
 }
 
 static void
@@ -1067,11 +1076,11 @@ iwl_mvm_umac_scan_cfg_channels(struct iwl_mvm *mvm,
 	}
 }
 
-static u32 iwl_mvm_scan_umac_flags(struct iwl_mvm *mvm,
+static u16 iwl_mvm_scan_umac_flags(struct iwl_mvm *mvm,
 				   struct iwl_mvm_scan_params *params,
 				   struct ieee80211_vif *vif)
 {
-	int flags = 0;
+	u16 flags = 0;
 
 	if (params->n_ssids == 0)
 		flags = IWL_UMAC_SCAN_GEN_FLAGS_PASSIVE;
@@ -1093,6 +1102,9 @@ static u32 iwl_mvm_scan_umac_flags(struct iwl_mvm *mvm,
 	if (!iwl_mvm_is_regular_scan(params))
 		flags |= IWL_UMAC_SCAN_GEN_FLAGS_PERIODIC;
 
+	if (params->measurement_dwell)
+		flags |= IWL_UMAC_SCAN_GEN_FLAGS_ITER_COMPLETE;
+
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	if (mvm->scan_iter_notif_enabled)
 		flags |= IWL_UMAC_SCAN_GEN_FLAGS_ITER_COMPLETE;
@@ -1119,6 +1131,7 @@ static int iwl_mvm_scan_umac(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			mvm->fw->ucode_capa.n_scan_channels;
 	int uid, i;
 	u32 ssid_bitmap = 0;
+	struct iwl_mvm_vif *scan_vif = iwl_mvm_vif_from_mac80211(vif);
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -1136,8 +1149,9 @@ static int iwl_mvm_scan_umac(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	mvm->scan_uid_status[uid] = type;
 
 	cmd->uid = cpu_to_le32(uid);
-	cmd->general_flags = cpu_to_le32(iwl_mvm_scan_umac_flags(mvm, params,
+	cmd->general_flags = cpu_to_le16(iwl_mvm_scan_umac_flags(mvm, params,
 								 vif));
+	cmd->scan_start_mac_id = scan_vif->id;
 
 	if (type == IWL_MVM_SCAN_SCHED || type == IWL_MVM_SCAN_NETDETECT)
 		cmd->flags = cpu_to_le32(IWL_UMAC_SCAN_FLAG_PREEMPTIVE);
@@ -1289,6 +1303,12 @@ int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		iwl_mvm_get_scan_type(mvm,
 				      vif->type == NL80211_IFTYPE_P2P_DEVICE);
 
+	ret = iwl_mvm_get_measurement_dwell(mvm, req, &params);
+	if (ret < 0)
+		return ret;
+
+	params.measurement_dwell = ret;
+
 	iwl_mvm_build_scan_probe(mvm, vif, ies, &params);
 
 	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_UMAC_SCAN)) {
@@ -1315,6 +1335,7 @@ int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	IWL_DEBUG_SCAN(mvm, "Scan request was sent successfully\n");
 	mvm->scan_status |= IWL_MVM_SCAN_REGULAR;
+	mvm->scan_vif = iwl_mvm_vif_from_mac80211(vif);
 	iwl_mvm_ref(mvm, IWL_MVM_REF_SCAN);
 
 	queue_delayed_work(system_wq, &mvm->scan_timeout_dwork,
@@ -1437,9 +1458,12 @@ void iwl_mvm_rx_umac_scan_complete_notif(struct iwl_mvm *mvm,
 	if (mvm->scan_uid_status[uid] == IWL_MVM_SCAN_REGULAR) {
 		struct cfg80211_scan_info info = {
 			.aborted = aborted,
+			.scan_start_tsf = mvm->scan_start,
 		};
 
+		memcpy(info.tsf_bssid, mvm->scan_vif->bssid, ETH_ALEN);
 		ieee80211_scan_completed(mvm->hw, &info);
+		mvm->scan_vif = NULL;
 		iwl_mvm_unref(mvm, IWL_MVM_REF_SCAN);
 		cancel_delayed_work(&mvm->scan_timeout_dwork);
 	} else if (mvm->scan_uid_status[uid] == IWL_MVM_SCAN_SCHED) {
@@ -1473,6 +1497,8 @@ void iwl_mvm_rx_umac_scan_iter_complete_notif(struct iwl_mvm *mvm,
 	struct iwl_umac_scan_iter_complete_notif *notif = (void *)pkt->data;
 	u8 buf[256];
 
+	mvm->scan_start = le64_to_cpu(notif->start_tsf);
+
 	IWL_DEBUG_SCAN(mvm,
 		       "UMAC Scan iteration complete: status=0x%x scanned_channels=%d channels list: %s\n",
 		       notif->status, notif->scanned_channels,
@@ -1485,6 +1511,10 @@ void iwl_mvm_rx_umac_scan_iter_complete_notif(struct iwl_mvm *mvm,
 		ieee80211_sched_scan_results(mvm->hw);
 		mvm->sched_scan_pass_all = SCHED_SCAN_PASS_ALL_ENABLED;
 	}
+
+	IWL_DEBUG_SCAN(mvm,
+		       "UMAC Scan iteration complete: scan started at %llu (TSF)\n",
+		       mvm->scan_start);
 }
 
 static int iwl_mvm_umac_scan_abort(struct iwl_mvm *mvm, int type)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
index 3130b9c68a74..fc771885e383 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
@@ -468,6 +468,11 @@ static int iwl_mvm_get_shared_queue(struct iwl_mvm *mvm,
 		    i != IWL_MVM_DQA_BSS_CLIENT_QUEUE)
 			continue;
 
+		/* Don't try and take queues being reconfigured */
+		if (mvm->queue_info[queue].status ==
+		    IWL_MVM_QUEUE_RECONFIGURING)
+			continue;
+
 		ac_to_queue[mvm->queue_info[i].mac80211_ac] = i;
 	}
 
@@ -501,31 +506,37 @@ static int iwl_mvm_get_shared_queue(struct iwl_mvm *mvm,
 		queue = ac_to_queue[IEEE80211_AC_VO];
 
 	/* Make sure queue found (or not) is legal */
-	if (!((queue >= IWL_MVM_DQA_MIN_MGMT_QUEUE &&
-	       queue <= IWL_MVM_DQA_MAX_MGMT_QUEUE) ||
-	      (queue >= IWL_MVM_DQA_MIN_DATA_QUEUE &&
-	       queue <= IWL_MVM_DQA_MAX_DATA_QUEUE) ||
-	      (queue == IWL_MVM_DQA_BSS_CLIENT_QUEUE))) {
+	if (!iwl_mvm_is_dqa_data_queue(mvm, queue) &&
+	    !iwl_mvm_is_dqa_mgmt_queue(mvm, queue) &&
+	    (queue != IWL_MVM_DQA_BSS_CLIENT_QUEUE)) {
 		IWL_ERR(mvm, "No DATA queues available to share\n");
-		queue = -ENOSPC;
+		return -ENOSPC;
+	}
+
+	/* Make sure the queue isn't in the middle of being reconfigured */
+	if (mvm->queue_info[queue].status == IWL_MVM_QUEUE_RECONFIGURING) {
+		IWL_ERR(mvm,
+			"TXQ %d is in the middle of re-config - try again\n",
+			queue);
+		return -EBUSY;
 	}
 
 	return queue;
 }
 
 /*
- * If a given queue has a higher AC than the TID stream that is being added to
- * it, the queue needs to be redirected to the lower AC. This function does that
+ * If a given queue has a higher AC than the TID stream that is being compared
+ * to, the queue needs to be redirected to the lower AC. This function does that
  * in such a case, otherwise - if no redirection required - it does nothing,
  * unless the %force param is true.
  */
-static int iwl_mvm_scd_queue_redirect(struct iwl_mvm *mvm, int queue, int tid,
-				      int ac, int ssn, unsigned int wdg_timeout,
-				      bool force)
+int iwl_mvm_scd_queue_redirect(struct iwl_mvm *mvm, int queue, int tid,
+			       int ac, int ssn, unsigned int wdg_timeout,
+			       bool force)
 {
 	struct iwl_scd_txq_cfg_cmd cmd = {
 		.scd_queue = queue,
-		.enable = 0,
+		.action = SCD_CFG_DISABLE_QUEUE,
 	};
 	bool shared_queue;
 	unsigned long mq;
@@ -551,11 +562,12 @@ static int iwl_mvm_scd_queue_redirect(struct iwl_mvm *mvm, int queue, int tid,
 
 	cmd.sta_id = mvm->queue_info[queue].ra_sta_id;
 	cmd.tx_fifo = iwl_mvm_ac_to_tx_fifo[mvm->queue_info[queue].mac80211_ac];
+	cmd.tid = mvm->queue_info[queue].txq_tid;
 	mq = mvm->queue_info[queue].hw_queue_to_mac80211;
 	shared_queue = (mvm->queue_info[queue].hw_queue_refcount > 1);
 	spin_unlock_bh(&mvm->queue_info_lock);
 
-	IWL_DEBUG_TX_QUEUES(mvm, "Redirecting shared TXQ #%d to FIFO #%d\n",
+	IWL_DEBUG_TX_QUEUES(mvm, "Redirecting TXQ #%d to FIFO #%d\n",
 			    queue, iwl_mvm_ac_to_tx_fifo[ac]);
 
 	/* Stop MAC queues and wait for this queue to empty */
@@ -576,9 +588,12 @@ static int iwl_mvm_scd_queue_redirect(struct iwl_mvm *mvm, int queue, int tid,
 			ret);
 
 	/* Make sure the SCD wrptr is correctly set before reconfiguring */
-	iwl_trans_txq_enable(mvm->trans, queue, iwl_mvm_ac_to_tx_fifo[ac],
-			     cmd.sta_id, tid, LINK_QUAL_AGG_FRAME_LIMIT_DEF,
-			     ssn, wdg_timeout);
+	iwl_trans_txq_enable_cfg(mvm->trans, queue, ssn, NULL, wdg_timeout);
+
+	/* Update the TID "owner" of the queue */
+	spin_lock_bh(&mvm->queue_info_lock);
+	mvm->queue_info[queue].txq_tid = tid;
+	spin_unlock_bh(&mvm->queue_info_lock);
 
 	/* TODO: Work-around SCD bug when moving back by multiples of 0x40 */
 
@@ -709,7 +724,7 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 	if (WARN_ON(queue <= 0)) {
 		IWL_ERR(mvm, "No available queues for tid %d on sta_id %d\n",
 			tid, cfg.sta_id);
-		return -ENOSPC;
+		return queue;
 	}
 
 	/*
@@ -728,21 +743,23 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 	if (using_inactive_queue) {
 		struct iwl_scd_txq_cfg_cmd cmd = {
 			.scd_queue = queue,
-			.enable = 0,
+			.action = SCD_CFG_DISABLE_QUEUE,
 		};
-		u8 ac;
+		u8 txq_curr_ac;
 
 		disable_agg_tids = iwl_mvm_remove_sta_queue_marking(mvm, queue);
 
 		spin_lock_bh(&mvm->queue_info_lock);
-		ac = mvm->queue_info[queue].mac80211_ac;
+		txq_curr_ac = mvm->queue_info[queue].mac80211_ac;
 		cmd.sta_id = mvm->queue_info[queue].ra_sta_id;
-		cmd.tx_fifo = iwl_mvm_ac_to_tx_fifo[ac];
+		cmd.tx_fifo = iwl_mvm_ac_to_tx_fifo[txq_curr_ac];
+		cmd.tid = mvm->queue_info[queue].txq_tid;
 		spin_unlock_bh(&mvm->queue_info_lock);
 
 		/* Disable the queue */
-		iwl_mvm_invalidate_sta_queue(mvm, queue, disable_agg_tids,
-					     true);
+		if (disable_agg_tids)
+			iwl_mvm_invalidate_sta_queue(mvm, queue,
+						     disable_agg_tids, false);
 		iwl_trans_txq_disable(mvm->trans, queue, false);
 		ret = iwl_mvm_send_cmd_pdu(mvm, SCD_QUEUE_CFG, 0, sizeof(cmd),
 					   &cmd);
@@ -758,6 +775,10 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 
 			return ret;
 		}
+
+		/* If TXQ is allocated to another STA, update removal in FW */
+		if (cmd.sta_id != mvmsta->sta_id)
+			iwl_mvm_invalidate_sta_queue(mvm, queue, 0, true);
 	}
 
 	IWL_DEBUG_TX_QUEUES(mvm,
@@ -827,6 +848,119 @@ out_err:
 	return ret;
 }
 
+static void iwl_mvm_change_queue_owner(struct iwl_mvm *mvm, int queue)
+{
+	struct iwl_scd_txq_cfg_cmd cmd = {
+		.scd_queue = queue,
+		.action = SCD_CFG_UPDATE_QUEUE_TID,
+	};
+	s8 sta_id;
+	int tid;
+	unsigned long tid_bitmap;
+	int ret;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	spin_lock_bh(&mvm->queue_info_lock);
+	sta_id = mvm->queue_info[queue].ra_sta_id;
+	tid_bitmap = mvm->queue_info[queue].tid_bitmap;
+	spin_unlock_bh(&mvm->queue_info_lock);
+
+	if (WARN(!tid_bitmap, "TXQ %d has no tids assigned to it\n", queue))
+		return;
+
+	/* Find any TID for queue */
+	tid = find_first_bit(&tid_bitmap, IWL_MAX_TID_COUNT + 1);
+	cmd.tid = tid;
+	cmd.tx_fifo = iwl_mvm_ac_to_tx_fifo[tid_to_mac80211_ac[tid]];
+
+	ret = iwl_mvm_send_cmd_pdu(mvm, SCD_QUEUE_CFG, 0, sizeof(cmd), &cmd);
+	if (ret)
+		IWL_ERR(mvm, "Failed to update owner of TXQ %d (ret=%d)\n",
+			queue, ret);
+	else
+		IWL_DEBUG_TX_QUEUES(mvm, "Changed TXQ %d ownership to tid %d\n",
+				    queue, tid);
+}
+
+static void iwl_mvm_unshare_queue(struct iwl_mvm *mvm, int queue)
+{
+	struct ieee80211_sta *sta;
+	struct iwl_mvm_sta *mvmsta;
+	s8 sta_id;
+	int tid = -1;
+	unsigned long tid_bitmap;
+	unsigned int wdg_timeout;
+	int ssn;
+	int ret = true;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	spin_lock_bh(&mvm->queue_info_lock);
+	sta_id = mvm->queue_info[queue].ra_sta_id;
+	tid_bitmap = mvm->queue_info[queue].tid_bitmap;
+	spin_unlock_bh(&mvm->queue_info_lock);
+
+	/* Find TID for queue, and make sure it is the only one on the queue */
+	tid = find_first_bit(&tid_bitmap, IWL_MAX_TID_COUNT + 1);
+	if (tid_bitmap != BIT(tid)) {
+		IWL_ERR(mvm, "Failed to unshare q %d, active tids=0x%lx\n",
+			queue, tid_bitmap);
+		return;
+	}
+
+	IWL_DEBUG_TX_QUEUES(mvm, "Unsharing TXQ %d, keeping tid %d\n", queue,
+			    tid);
+
+	sta = rcu_dereference_protected(mvm->fw_id_to_mac_id[sta_id],
+					lockdep_is_held(&mvm->mutex));
+
+	if (WARN_ON_ONCE(IS_ERR_OR_NULL(sta)))
+		return;
+
+	mvmsta = iwl_mvm_sta_from_mac80211(sta);
+	wdg_timeout = iwl_mvm_get_wd_timeout(mvm, mvmsta->vif, false, false);
+
+	ssn = IEEE80211_SEQ_TO_SN(mvmsta->tid_data[tid].seq_number);
+
+	ret = iwl_mvm_scd_queue_redirect(mvm, queue, tid,
+					 tid_to_mac80211_ac[tid], ssn,
+					 wdg_timeout, true);
+	if (ret) {
+		IWL_ERR(mvm, "Failed to redirect TXQ %d\n", queue);
+		return;
+	}
+
+	/* If aggs should be turned back on - do it */
+	if (mvmsta->tid_data[tid].state == IWL_AGG_ON) {
+		struct iwl_mvm_add_sta_cmd cmd = {0};
+
+		mvmsta->tid_disable_agg &= ~BIT(tid);
+
+		cmd.mac_id_n_color = cpu_to_le32(mvmsta->mac_id_n_color);
+		cmd.sta_id = mvmsta->sta_id;
+		cmd.add_modify = STA_MODE_MODIFY;
+		cmd.modify_mask = STA_MODIFY_TID_DISABLE_TX;
+		cmd.tfd_queue_msk = cpu_to_le32(mvmsta->tfd_queue_msk);
+		cmd.tid_disable_tx = cpu_to_le16(mvmsta->tid_disable_agg);
+
+		ret = iwl_mvm_send_cmd_pdu(mvm, ADD_STA, CMD_ASYNC,
+					   iwl_mvm_add_sta_cmd_size(mvm), &cmd);
+		if (!ret) {
+			IWL_DEBUG_TX_QUEUES(mvm,
+					    "TXQ #%d is now aggregated again\n",
+					    queue);
+
+			/* Mark queue intenally as aggregating again */
+			iwl_trans_txq_set_shared_mode(mvm->trans, queue, false);
+		}
+	}
+
+	spin_lock_bh(&mvm->queue_info_lock);
+	mvm->queue_info[queue].status = IWL_MVM_QUEUE_READY;
+	spin_unlock_bh(&mvm->queue_info_lock);
+}
+
 static inline u8 iwl_mvm_tid_to_ac_queue(int tid)
 {
 	if (tid == IWL_MAX_TID_COUNT)
@@ -894,13 +1028,42 @@ void iwl_mvm_add_new_dqa_stream_wk(struct work_struct *wk)
 	struct ieee80211_sta *sta;
 	struct iwl_mvm_sta *mvmsta;
 	unsigned long deferred_tid_traffic;
-	int sta_id, tid;
+	int queue, sta_id, tid;
 
 	/* Check inactivity of queues */
 	iwl_mvm_inactivity_check(mvm);
 
 	mutex_lock(&mvm->mutex);
 
+	/* Reconfigure queues requiring reconfiguation */
+	for (queue = 0; queue < IWL_MAX_HW_QUEUES; queue++) {
+		bool reconfig;
+		bool change_owner;
+
+		spin_lock_bh(&mvm->queue_info_lock);
+		reconfig = (mvm->queue_info[queue].status ==
+			    IWL_MVM_QUEUE_RECONFIGURING);
+
+		/*
+		 * We need to take into account a situation in which a TXQ was
+		 * allocated to TID x, and then turned shared by adding TIDs y
+		 * and z. If TID x becomes inactive and is removed from the TXQ,
+		 * ownership must be given to one of the remaining TIDs.
+		 * This is mainly because if TID x continues - a new queue can't
+		 * be allocated for it as long as it is an owner of another TXQ.
+		 */
+		change_owner = !(mvm->queue_info[queue].tid_bitmap &
+				 BIT(mvm->queue_info[queue].txq_tid)) &&
+			       (mvm->queue_info[queue].status ==
+				IWL_MVM_QUEUE_SHARED);
+		spin_unlock_bh(&mvm->queue_info_lock);
+
+		if (reconfig)
+			iwl_mvm_unshare_queue(mvm, queue);
+		else if (change_owner)
+			iwl_mvm_change_queue_owner(mvm, queue);
+	}
+
 	/* Go over all stations with deferred traffic */
 	for_each_set_bit(sta_id, mvm->sta_deferred_frames,
 			 IWL_MVM_STATION_COUNT) {
@@ -963,6 +1126,61 @@ static int iwl_mvm_reserve_sta_stream(struct iwl_mvm *mvm,
 	return 0;
 }
 
+/*
+ * In DQA mode, after a HW restart the queues should be allocated as before, in
+ * order to avoid race conditions when there are shared queues. This function
+ * does the re-mapping and queue allocation.
+ *
+ * Note that re-enabling aggregations isn't done in this function.
+ */
+static void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
+						 struct iwl_mvm_sta *mvm_sta)
+{
+	unsigned int wdg_timeout =
+			iwl_mvm_get_wd_timeout(mvm, mvm_sta->vif, false, false);
+	int i;
+	struct iwl_trans_txq_scd_cfg cfg = {
+		.sta_id = mvm_sta->sta_id,
+		.frame_limit = IWL_FRAME_LIMIT,
+	};
+
+	/* Make sure reserved queue is still marked as such (or allocated) */
+	mvm->queue_info[mvm_sta->reserved_queue].status =
+		IWL_MVM_QUEUE_RESERVED;
+
+	for (i = 0; i <= IWL_MAX_TID_COUNT; i++) {
+		struct iwl_mvm_tid_data *tid_data = &mvm_sta->tid_data[i];
+		int txq_id = tid_data->txq_id;
+		int ac;
+		u8 mac_queue;
+
+		if (txq_id == IEEE80211_INVAL_HW_QUEUE)
+			continue;
+
+		skb_queue_head_init(&tid_data->deferred_tx_frames);
+
+		ac = tid_to_mac80211_ac[i];
+		mac_queue = mvm_sta->vif->hw_queue[ac];
+
+		cfg.tid = i;
+		cfg.fifo = iwl_mvm_ac_to_tx_fifo[ac];
+		cfg.aggregate = (txq_id >= IWL_MVM_DQA_MIN_DATA_QUEUE ||
+				 txq_id == IWL_MVM_DQA_BSS_CLIENT_QUEUE);
+
+		IWL_DEBUG_TX_QUEUES(mvm,
+				    "Re-mapping sta %d tid %d to queue %d\n",
+				    mvm_sta->sta_id, i, txq_id);
+
+		iwl_mvm_enable_txq(mvm, txq_id, mac_queue,
+				   IEEE80211_SEQ_TO_SN(tid_data->seq_number),
+				   &cfg, wdg_timeout);
+
+		mvm->queue_info[txq_id].status = IWL_MVM_QUEUE_READY;
+	}
+
+	atomic_set(&mvm->pending_frames[mvm_sta->sta_id], 0);
+}
+
 int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 		    struct ieee80211_vif *vif,
 		    struct ieee80211_sta *sta)
@@ -985,6 +1203,13 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 
 	spin_lock_init(&mvm_sta->lock);
 
+	/* In DQA mode, if this is a HW restart, re-alloc existing queues */
+	if (iwl_mvm_is_dqa_supported(mvm) &&
+	    test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+		iwl_mvm_realloc_queues_after_restart(mvm, mvm_sta);
+		goto update_fw;
+	}
+
 	mvm_sta->sta_id = sta_id;
 	mvm_sta->mac_id_n_color = FW_CMD_ID_AND_COLOR(mvmvif->id,
 						      mvmvif->color);
@@ -1048,6 +1273,7 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 			goto err;
 	}
 
+update_fw:
 	ret = iwl_mvm_sta_send_to_fw(mvm, sta, false, 0);
 	if (ret)
 		goto err;
@@ -1071,13 +1297,6 @@ err:
 	return ret;
 }
 
-int iwl_mvm_update_sta(struct iwl_mvm *mvm,
-		       struct ieee80211_vif *vif,
-		       struct ieee80211_sta *sta)
-{
-	return iwl_mvm_sta_send_to_fw(mvm, sta, true, 0);
-}
-
 int iwl_mvm_drain_sta(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 		      bool drain)
 {
@@ -1270,9 +1489,31 @@ int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
 		ret = iwl_mvm_drain_sta(mvm, mvm_sta, false);
 
 		/* If DQA is supported - the queues can be disabled now */
-		if (iwl_mvm_is_dqa_supported(mvm))
+		if (iwl_mvm_is_dqa_supported(mvm)) {
+			u8 reserved_txq = mvm_sta->reserved_queue;
+			enum iwl_mvm_queue_status *status;
+
 			iwl_mvm_disable_sta_queues(mvm, vif, mvm_sta);
 
+			/*
+			 * If no traffic has gone through the reserved TXQ - it
+			 * is still marked as IWL_MVM_QUEUE_RESERVED, and
+			 * should be manually marked as free again
+			 */
+			spin_lock_bh(&mvm->queue_info_lock);
+			status = &mvm->queue_info[reserved_txq].status;
+			if (WARN((*status != IWL_MVM_QUEUE_RESERVED) &&
+				 (*status != IWL_MVM_QUEUE_FREE),
+				 "sta_id %d reserved txq %d status %d",
+				 mvm_sta->sta_id, reserved_txq, *status)) {
+				spin_unlock_bh(&mvm->queue_info_lock);
+				return -EINVAL;
+			}
+
+			*status = IWL_MVM_QUEUE_FREE;
+			spin_unlock_bh(&mvm->queue_info_lock);
+		}
+
 		if (vif->type == NL80211_IFTYPE_STATION &&
 		    mvmvif->ap_sta_id == mvm_sta->sta_id) {
 			/* if associated - we can't remove the AP STA now */
@@ -1802,11 +2043,9 @@ int iwl_mvm_sta_rx_agg(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		baid_data->baid = baid;
 		baid_data->timeout = timeout;
 		baid_data->last_rx = jiffies;
-		init_timer(&baid_data->session_timer);
-		baid_data->session_timer.function =
-			iwl_mvm_rx_agg_session_expired;
-		baid_data->session_timer.data =
-			(unsigned long)&mvm->baid_map[baid];
+		setup_timer(&baid_data->session_timer,
+			    iwl_mvm_rx_agg_session_expired,
+			    (unsigned long)&mvm->baid_map[baid]);
 		baid_data->mvm = mvm;
 		baid_data->tid = tid;
 		baid_data->sta_id = mvm_sta->sta_id;
@@ -1956,7 +2195,7 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		return -EIO;
 	}
 
-	spin_lock_bh(&mvm->queue_info_lock);
+	spin_lock(&mvm->queue_info_lock);
 
 	/*
 	 * Note the possible cases:
@@ -1967,14 +2206,20 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 *	non-DQA mode, since the TXQ hasn't yet been allocated
 	 */
 	txq_id = mvmsta->tid_data[tid].txq_id;
-	if (!iwl_mvm_is_dqa_supported(mvm) ||
+	if (iwl_mvm_is_dqa_supported(mvm) &&
+	    unlikely(mvm->queue_info[txq_id].status == IWL_MVM_QUEUE_SHARED)) {
+		ret = -ENXIO;
+		IWL_DEBUG_TX_QUEUES(mvm,
+				    "Can't start tid %d agg on shared queue!\n",
+				    tid);
+		goto release_locks;
+	} else if (!iwl_mvm_is_dqa_supported(mvm) ||
 	    mvm->queue_info[txq_id].status != IWL_MVM_QUEUE_READY) {
 		txq_id = iwl_mvm_find_free_queue(mvm, mvmsta->sta_id,
 						 mvm->first_agg_queue,
 						 mvm->last_agg_queue);
 		if (txq_id < 0) {
 			ret = txq_id;
-			spin_unlock_bh(&mvm->queue_info_lock);
 			IWL_ERR(mvm, "Failed to allocate agg queue\n");
 			goto release_locks;
 		}
@@ -1982,7 +2227,8 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		/* TXQ hasn't yet been enabled, so mark it only as reserved */
 		mvm->queue_info[txq_id].status = IWL_MVM_QUEUE_RESERVED;
 	}
-	spin_unlock_bh(&mvm->queue_info_lock);
+
+	spin_unlock(&mvm->queue_info_lock);
 
 	IWL_DEBUG_TX_QUEUES(mvm,
 			    "AGG for tid %d will be on queue #%d\n",
@@ -2006,8 +2252,11 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	}
 
 	ret = 0;
+	goto out;
 
 release_locks:
+	spin_unlock(&mvm->queue_info_lock);
+out:
 	spin_unlock_bh(&mvmsta->lock);
 
 	return ret;
@@ -2023,6 +2272,7 @@ int iwl_mvm_sta_tx_agg_oper(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		iwl_mvm_get_wd_timeout(mvm, vif, sta->tdls, false);
 	int queue, ret;
 	bool alloc_queue = true;
+	enum iwl_mvm_queue_status queue_status;
 	u16 ssn;
 
 	struct iwl_trans_txq_scd_cfg cfg = {
@@ -2048,13 +2298,15 @@ int iwl_mvm_sta_tx_agg_oper(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	cfg.fifo = iwl_mvm_ac_to_tx_fifo[tid_to_mac80211_ac[tid]];
 
+	spin_lock_bh(&mvm->queue_info_lock);
+	queue_status = mvm->queue_info[queue].status;
+	spin_unlock_bh(&mvm->queue_info_lock);
+
 	/* In DQA mode, the existing queue might need to be reconfigured */
 	if (iwl_mvm_is_dqa_supported(mvm)) {
-		spin_lock_bh(&mvm->queue_info_lock);
 		/* Maybe there is no need to even alloc a queue... */
 		if (mvm->queue_info[queue].status == IWL_MVM_QUEUE_READY)
 			alloc_queue = false;
-		spin_unlock_bh(&mvm->queue_info_lock);
 
 		/*
 		 * Only reconfig the SCD for the queue if the window size has
@@ -2089,9 +2341,12 @@ int iwl_mvm_sta_tx_agg_oper(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 				   vif->hw_queue[tid_to_mac80211_ac[tid]], ssn,
 				   &cfg, wdg_timeout);
 
-	ret = iwl_mvm_sta_tx_agg(mvm, sta, tid, queue, true);
-	if (ret)
-		return -EIO;
+	/* Send ADD_STA command to enable aggs only if the queue isn't shared */
+	if (queue_status != IWL_MVM_QUEUE_SHARED) {
+		ret = iwl_mvm_sta_tx_agg(mvm, sta, tid, queue, true);
+		if (ret)
+			return -EIO;
+	}
 
 	/* No need to mark as reserved */
 	spin_lock_bh(&mvm->queue_info_lock);
@@ -2123,7 +2378,6 @@ int iwl_mvm_sta_tx_agg_stop(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	u16 txq_id;
 	int err;
 
-
 	/*
 	 * If mac80211 is cleaning its state, then say that we finished since
 	 * our state has been cleared anyway.
@@ -2152,6 +2406,7 @@ int iwl_mvm_sta_tx_agg_stop(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 */
 	if (mvm->queue_info[txq_id].status == IWL_MVM_QUEUE_RESERVED)
 		mvm->queue_info[txq_id].status = IWL_MVM_QUEUE_FREE;
+
 	spin_unlock_bh(&mvm->queue_info_lock);
 
 	switch (tid_data->state) {
@@ -2412,9 +2667,15 @@ static int iwl_mvm_send_sta_igtk(struct iwl_mvm *mvm,
 	struct iwl_mvm_mgmt_mcast_key_cmd igtk_cmd = {};
 
 	/* verify the key details match the required command's expectations */
-	if (WARN_ON((keyconf->cipher != WLAN_CIPHER_SUITE_AES_CMAC) ||
-		    (keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE) ||
-		    (keyconf->keyidx != 4 && keyconf->keyidx != 5)))
+	if (WARN_ON((keyconf->flags & IEEE80211_KEY_FLAG_PAIRWISE) ||
+		    (keyconf->keyidx != 4 && keyconf->keyidx != 5) ||
+		    (keyconf->cipher != WLAN_CIPHER_SUITE_AES_CMAC &&
+		     keyconf->cipher != WLAN_CIPHER_SUITE_BIP_GMAC_128 &&
+		     keyconf->cipher != WLAN_CIPHER_SUITE_BIP_GMAC_256)))
+		return -EINVAL;
+
+	if (WARN_ON(!iwl_mvm_has_new_rx_api(mvm) &&
+		    keyconf->cipher != WLAN_CIPHER_SUITE_AES_CMAC))
 		return -EINVAL;
 
 	igtk_cmd.key_id = cpu_to_le32(keyconf->keyidx);
@@ -2430,11 +2691,18 @@ static int iwl_mvm_send_sta_igtk(struct iwl_mvm *mvm,
 		case WLAN_CIPHER_SUITE_AES_CMAC:
 			igtk_cmd.ctrl_flags |= cpu_to_le32(STA_KEY_FLG_CCM);
 			break;
+		case WLAN_CIPHER_SUITE_BIP_GMAC_128:
+		case WLAN_CIPHER_SUITE_BIP_GMAC_256:
+			igtk_cmd.ctrl_flags |= cpu_to_le32(STA_KEY_FLG_GCMP);
+			break;
 		default:
 			return -EINVAL;
 		}
 
-		memcpy(igtk_cmd.IGTK, keyconf->key, keyconf->keylen);
+		memcpy(igtk_cmd.igtk, keyconf->key, keyconf->keylen);
+		if (keyconf->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_256)
+			igtk_cmd.ctrl_flags |=
+				cpu_to_le32(STA_KEY_FLG_KEY_32BYTES);
 		ieee80211_get_key_rx_seq(keyconf, 0, &seq);
 		pn = seq.aes_cmac.pn;
 		igtk_cmd.receive_seq_cnt = cpu_to_le64(((u64) pn[5] << 0) |
@@ -2449,6 +2717,19 @@ static int iwl_mvm_send_sta_igtk(struct iwl_mvm *mvm,
 		       remove_key ? "removing" : "installing",
 		       igtk_cmd.sta_id);
 
+	if (!iwl_mvm_has_new_rx_api(mvm)) {
+		struct iwl_mvm_mgmt_mcast_key_cmd_v1 igtk_cmd_v1 = {
+			.ctrl_flags = igtk_cmd.ctrl_flags,
+			.key_id = igtk_cmd.key_id,
+			.sta_id = igtk_cmd.sta_id,
+			.receive_seq_cnt = igtk_cmd.receive_seq_cnt
+		};
+
+		memcpy(igtk_cmd_v1.igtk, igtk_cmd.igtk,
+		       ARRAY_SIZE(igtk_cmd_v1.igtk));
+		return iwl_mvm_send_cmd_pdu(mvm, MGMT_MCAST_KEY, 0,
+					    sizeof(igtk_cmd_v1), &igtk_cmd_v1);
+	}
 	return iwl_mvm_send_cmd_pdu(mvm, MGMT_MCAST_KEY, 0,
 				    sizeof(igtk_cmd), &igtk_cmd);
 }
@@ -2573,7 +2854,9 @@ int iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 	}
 	sta_id = mvm_sta->sta_id;
 
-	if (keyconf->cipher == WLAN_CIPHER_SUITE_AES_CMAC) {
+	if (keyconf->cipher == WLAN_CIPHER_SUITE_AES_CMAC ||
+	    keyconf->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_128 ||
+	    keyconf->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_256) {
 		ret = iwl_mvm_send_sta_igtk(mvm, keyconf, sta_id, false);
 		goto end;
 	}
@@ -2659,7 +2942,9 @@ int iwl_mvm_remove_sta_key(struct iwl_mvm *mvm,
 	IWL_DEBUG_WEP(mvm, "mvm remove dynamic key: idx=%d sta=%d\n",
 		      keyconf->keyidx, sta_id);
 
-	if (keyconf->cipher == WLAN_CIPHER_SUITE_AES_CMAC)
+	if (keyconf->cipher == WLAN_CIPHER_SUITE_AES_CMAC ||
+	    keyconf->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_128 ||
+	    keyconf->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_256)
 		return iwl_mvm_send_sta_igtk(mvm, keyconf, sta_id, true);
 
 	if (!__test_and_clear_bit(keyconf->hw_key_idx, mvm->fw_key_table)) {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
index bbc1cab2c3bf..e068d5355865 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
@@ -473,9 +473,14 @@ int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 		    struct ieee80211_vif *vif,
 		    struct ieee80211_sta *sta);
-int iwl_mvm_update_sta(struct iwl_mvm *mvm,
-		       struct ieee80211_vif *vif,
-		       struct ieee80211_sta *sta);
+
+static inline int iwl_mvm_update_sta(struct iwl_mvm *mvm,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_sta *sta)
+{
+	return iwl_mvm_sta_send_to_fw(mvm, sta, true, 0);
+}
+
 int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
 		   struct ieee80211_vif *vif,
 		   struct ieee80211_sta *sta);
@@ -554,4 +559,8 @@ void iwl_mvm_modify_all_sta_disable_tx(struct iwl_mvm *mvm,
 void iwl_mvm_csa_client_absent(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 void iwl_mvm_add_new_dqa_stream_wk(struct work_struct *wk);
 
+int iwl_mvm_scd_queue_redirect(struct iwl_mvm *mvm, int queue, int tid,
+			       int ac, int ssn, unsigned int wdg_timeout,
+			       bool force);
+
 #endif /* __sta_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
index 58fc7b3c711c..63a051be832e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
@@ -241,11 +241,8 @@ static int iwl_mvm_get_temp_cmd(struct iwl_mvm *mvm)
 	};
 	u32 cmdid;
 
-	if (fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_WIDE_CMD_HDR))
-		cmdid = iwl_cmd_id(CMD_DTS_MEASUREMENT_TRIGGER_WIDE,
-				   PHY_OPS_GROUP, 0);
-	else
-		cmdid = CMD_DTS_MEASUREMENT_TRIGGER;
+	cmdid = iwl_cmd_id(CMD_DTS_MEASUREMENT_TRIGGER_WIDE,
+			   PHY_OPS_GROUP, 0);
 
 	if (!fw_has_capa(&mvm->fw->ucode_capa,
 			 IWL_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE))
@@ -261,9 +258,6 @@ int iwl_mvm_get_temp(struct iwl_mvm *mvm, s32 *temp)
 					    DTS_MEASUREMENT_NOTIF_WIDE) };
 	int ret;
 
-	if (!fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_WIDE_CMD_HDR))
-		temp_notif[0] = DTS_MEASUREMENT_NOTIFICATION;
-
 	lockdep_assert_held(&mvm->mutex);
 
 	iwl_init_notification_wait(&mvm->notif_wait, &wait_temp_notif,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
index c6585ab48df3..66957ac12ca4 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
@@ -346,7 +346,7 @@ void iwl_mvm_set_tx_cmd_rate(struct iwl_mvm *mvm, struct iwl_tx_cmd *tx_cmd,
 
 	rate_idx = info->control.rates[0].idx;
 	/* if the rate isn't a well known legacy rate, take the lowest one */
-	if (rate_idx < 0 || rate_idx > IWL_RATE_COUNT_LEGACY)
+	if (rate_idx < 0 || rate_idx >= IWL_RATE_COUNT_LEGACY)
 		rate_idx = rate_lowest_index(
 				&mvm->nvm_data->bands[info->band], sta);
 
@@ -441,7 +441,7 @@ static void iwl_mvm_set_tx_cmd_crypto(struct iwl_mvm *mvm,
 		 * one.
 		 * Need to handle this.
 		 */
-		tx_cmd->sec_ctl |= TX_CMD_SEC_GCMP | TC_CMD_SEC_KEY_FROM_TABLE;
+		tx_cmd->sec_ctl |= TX_CMD_SEC_GCMP | TX_CMD_SEC_KEY_FROM_TABLE;
 		tx_cmd->key[0] = keyconf->hw_key_idx;
 		iwl_mvm_set_tx_cmd_pn(info, crypto_hdr);
 		break;
@@ -490,16 +490,34 @@ iwl_mvm_set_tx_params(struct iwl_mvm *mvm, struct sk_buff *skb,
 static int iwl_mvm_get_ctrl_vif_queue(struct iwl_mvm *mvm,
 				      struct ieee80211_tx_info *info, __le16 fc)
 {
-	if (iwl_mvm_is_dqa_supported(mvm)) {
-		if (info->control.vif->type == NL80211_IFTYPE_AP &&
-		    ieee80211_is_probe_resp(fc))
+	if (!iwl_mvm_is_dqa_supported(mvm))
+		return info->hw_queue;
+
+	switch (info->control.vif->type) {
+	case NL80211_IFTYPE_AP:
+		/*
+		 * handle legacy hostapd as well, where station may be added
+		 * only after assoc.
+		 */
+		if (ieee80211_is_probe_resp(fc) || ieee80211_is_auth(fc))
 			return IWL_MVM_DQA_AP_PROBE_RESP_QUEUE;
-		else if (ieee80211_is_mgmt(fc) &&
-			 info->control.vif->type == NL80211_IFTYPE_P2P_DEVICE)
+		if (info->hw_queue == info->control.vif->cab_queue)
+			return info->hw_queue;
+
+		WARN_ON_ONCE(1);
+		return IWL_MVM_DQA_AP_PROBE_RESP_QUEUE;
+	case NL80211_IFTYPE_P2P_DEVICE:
+		if (ieee80211_is_mgmt(fc))
 			return IWL_MVM_DQA_P2P_DEVICE_QUEUE;
-	}
+		if (info->hw_queue == info->control.vif->cab_queue)
+			return info->hw_queue;
 
-	return info->hw_queue;
+		WARN_ON_ONCE(1);
+		return IWL_MVM_DQA_P2P_DEVICE_QUEUE;
+	default:
+		WARN_ONCE(1, "Not a ctrl vif, no available queue\n");
+		return -1;
+	}
 }
 
 int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb)
@@ -513,6 +531,15 @@ int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb)
 	int hdrlen = ieee80211_hdrlen(hdr->frame_control);
 	int queue;
 
+	/* IWL_MVM_OFFCHANNEL_QUEUE is used for ROC packets that can be used
+	 * in 2 different types of vifs, P2P & STATION. P2P uses the offchannel
+	 * queue. STATION (HS2.0) uses the auxiliary context of the FW,
+	 * and hence needs to be sent on the aux queue
+	 */
+	if (IEEE80211_SKB_CB(skb)->hw_queue == IWL_MVM_OFFCHANNEL_QUEUE &&
+	    skb_info->control.vif->type == NL80211_IFTYPE_STATION)
+		IEEE80211_SKB_CB(skb)->hw_queue = mvm->aux_queue;
+
 	memcpy(&info, skb->cb, sizeof(info));
 
 	if (WARN_ON_ONCE(info.flags & IEEE80211_TX_CTL_AMPDU))
@@ -526,16 +553,6 @@ int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb)
 	/* This holds the amsdu headers length */
 	skb_info->driver_data[0] = (void *)(uintptr_t)0;
 
-	/*
-	 * IWL_MVM_OFFCHANNEL_QUEUE is used for ROC packets that can be used
-	 * in 2 different types of vifs, P2P & STATION. P2P uses the offchannel
-	 * queue. STATION (HS2.0) uses the auxiliary context of the FW,
-	 * and hence needs to be sent on the aux queue
-	 */
-	if (IEEE80211_SKB_CB(skb)->hw_queue == IWL_MVM_OFFCHANNEL_QUEUE &&
-	    info.control.vif->type == NL80211_IFTYPE_STATION)
-		IEEE80211_SKB_CB(skb)->hw_queue = mvm->aux_queue;
-
 	queue = info.hw_queue;
 
 	/*
@@ -560,6 +577,9 @@ int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb)
 			sta_id = mvmvif->bcast_sta.sta_id;
 			queue = iwl_mvm_get_ctrl_vif_queue(mvm, &info,
 							   hdr->frame_control);
+			if (queue < 0)
+				return -1;
+
 		} else if (info.control.vif->type == NL80211_IFTYPE_STATION &&
 			   is_multicast_ether_addr(hdr->addr1)) {
 			u8 ap_sta_id = ACCESS_ONCE(mvmvif->ap_sta_id);
@@ -838,6 +858,22 @@ static void iwl_mvm_tx_add_stream(struct iwl_mvm *mvm,
 	}
 }
 
+/* Check if there are any timed-out TIDs on a given shared TXQ */
+static bool iwl_mvm_txq_should_update(struct iwl_mvm *mvm, int txq_id)
+{
+	unsigned long queue_tid_bitmap = mvm->queue_info[txq_id].tid_bitmap;
+	unsigned long now = jiffies;
+	int tid;
+
+	for_each_set_bit(tid, &queue_tid_bitmap, IWL_MAX_TID_COUNT + 1) {
+		if (time_before(mvm->queue_info[txq_id].last_frame_time[tid] +
+				IWL_MVM_DQA_QUEUE_TIMEOUT, now))
+			return true;
+	}
+
+	return false;
+}
+
 /*
  * Sets the fields in the Tx cmd that are crypto related
  */
@@ -904,9 +940,13 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 		tid = IWL_MAX_TID_COUNT;
 	}
 
-	if (iwl_mvm_is_dqa_supported(mvm))
+	if (iwl_mvm_is_dqa_supported(mvm)) {
 		txq_id = mvmsta->tid_data[tid].txq_id;
 
+		if (ieee80211_is_mgmt(fc))
+			tx_cmd->tid_tspec = IWL_TID_NON_QOS;
+	}
+
 	/* Copy MAC header from skb into command buffer */
 	memcpy(tx_cmd->hdr, hdr, hdrlen);
 
@@ -940,7 +980,6 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 			iwl_trans_free_tx_cmd(mvm->trans, dev_cmd);
 			spin_unlock(&mvmsta->lock);
 			return 0;
-
 		}
 
 		/* If we are here - TXQ exists and needs to be re-activated */
@@ -953,8 +992,25 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 				    txq_id);
 	}
 
-	/* Keep track of the time of the last frame for this RA/TID */
-	mvm->queue_info[txq_id].last_frame_time[tid] = jiffies;
+	if (iwl_mvm_is_dqa_supported(mvm)) {
+		/* Keep track of the time of the last frame for this RA/TID */
+		mvm->queue_info[txq_id].last_frame_time[tid] = jiffies;
+
+		/*
+		 * If we have timed-out TIDs - schedule the worker that will
+		 * reconfig the queues and update them
+		 *
+		 * Note that the mvm->queue_info_lock isn't being taken here in
+		 * order to not serialize the TX flow. This isn't dangerous
+		 * because scheduling mvm->add_stream_wk can't ruin the state,
+		 * and if we DON'T schedule it due to some race condition then
+		 * next TX we get here we will.
+		 */
+		if (unlikely(mvm->queue_info[txq_id].status ==
+			     IWL_MVM_QUEUE_SHARED &&
+			     iwl_mvm_txq_should_update(mvm, txq_id)))
+			schedule_work(&mvm->add_stream_wk);
+	}
 
 	IWL_DEBUG_TX(mvm, "TX to [%d|%d] Q:%d - seq: 0x%x\n", mvmsta->sta_id,
 		     tid, txq_id, IEEE80211_SEQ_TO_SN(seq_number));
@@ -1068,9 +1124,13 @@ static void iwl_mvm_check_ratid_empty(struct iwl_mvm *mvm,
 		IWL_DEBUG_TX_QUEUES(mvm,
 				    "Can continue DELBA flow ssn = next_recl = %d\n",
 				    tid_data->next_reclaimed);
-		iwl_mvm_disable_txq(mvm, tid_data->txq_id,
-				    vif->hw_queue[tid_to_mac80211_ac[tid]], tid,
-				    CMD_ASYNC);
+		if (!iwl_mvm_is_dqa_supported(mvm)) {
+			u8 mac80211_ac = tid_to_mac80211_ac[tid];
+
+			iwl_mvm_disable_txq(mvm, tid_data->txq_id,
+					    vif->hw_queue[mac80211_ac], tid,
+					    CMD_ASYNC);
+		}
 		tid_data->state = IWL_AGG_OFF;
 		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
 		break;
@@ -1548,41 +1608,16 @@ void iwl_mvm_rx_tx_cmd(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 		iwl_mvm_rx_tx_cmd_agg(mvm, pkt);
 }
 
-static void iwl_mvm_tx_info_from_ba_notif(struct ieee80211_tx_info *info,
-					  struct iwl_mvm_ba_notif *ba_notif,
-					  struct iwl_mvm_tid_data *tid_data)
-{
-	info->flags |= IEEE80211_TX_STAT_AMPDU;
-	info->status.ampdu_ack_len = ba_notif->txed_2_done;
-	info->status.ampdu_len = ba_notif->txed;
-	iwl_mvm_hwrate_to_tx_status(tid_data->rate_n_flags,
-				    info);
-	/* TODO: not accounted if the whole A-MPDU failed */
-	info->status.tx_time = tid_data->tx_time;
-	info->status.status_driver_data[0] =
-		(void *)(uintptr_t)ba_notif->reduced_txp;
-	info->status.status_driver_data[1] =
-		(void *)(uintptr_t)tid_data->rate_n_flags;
-}
-
-void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
+static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
+			       int txq, int index,
+			       struct ieee80211_tx_info *ba_info, u32 rate)
 {
-	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_mvm_ba_notif *ba_notif = (void *)pkt->data;
 	struct sk_buff_head reclaimed_skbs;
 	struct iwl_mvm_tid_data *tid_data;
 	struct ieee80211_sta *sta;
 	struct iwl_mvm_sta *mvmsta;
 	struct sk_buff *skb;
-	int sta_id, tid, freed;
-	/* "flow" corresponds to Tx queue */
-	u16 scd_flow = le16_to_cpu(ba_notif->scd_flow);
-	/* "ssn" is start of block-ack Tx window, corresponds to index
-	 * (in Tx queue's circular buffer) of first TFD/frame in window */
-	u16 ba_resp_scd_ssn = le16_to_cpu(ba_notif->scd_ssn);
-
-	sta_id = ba_notif->sta_id;
-	tid = ba_notif->tid;
+	int freed;
 
 	if (WARN_ONCE(sta_id >= IWL_MVM_STATION_COUNT ||
 		      tid >= IWL_MAX_TID_COUNT,
@@ -1602,10 +1637,10 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	tid_data = &mvmsta->tid_data[tid];
 
-	if (tid_data->txq_id != scd_flow) {
+	if (tid_data->txq_id != txq) {
 		IWL_ERR(mvm,
-			"invalid BA notification: Q %d, tid %d, flow %d\n",
-			tid_data->txq_id, tid, scd_flow);
+			"invalid BA notification: Q %d, tid %d\n",
+			tid_data->txq_id, tid);
 		rcu_read_unlock();
 		return;
 	}
@@ -1619,27 +1654,14 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	 * block-ack window (we assume that they've been successfully
 	 * transmitted ... if not, it's too late anyway).
 	 */
-	iwl_trans_reclaim(mvm->trans, scd_flow, ba_resp_scd_ssn,
-			  &reclaimed_skbs);
-
-	IWL_DEBUG_TX_REPLY(mvm,
-			   "BA_NOTIFICATION Received from %pM, sta_id = %d\n",
-			   (u8 *)&ba_notif->sta_addr_lo32,
-			   ba_notif->sta_id);
-	IWL_DEBUG_TX_REPLY(mvm,
-			   "TID = %d, SeqCtl = %d, bitmap = 0x%llx, scd_flow = %d, scd_ssn = %d sent:%d, acked:%d\n",
-			   ba_notif->tid, le16_to_cpu(ba_notif->seq_ctl),
-			   (unsigned long long)le64_to_cpu(ba_notif->bitmap),
-			   scd_flow, ba_resp_scd_ssn, ba_notif->txed,
-			   ba_notif->txed_2_done);
+	iwl_trans_reclaim(mvm->trans, txq, index, &reclaimed_skbs);
 
-	IWL_DEBUG_TX_REPLY(mvm, "reduced txp from ba notif %d\n",
-			   ba_notif->reduced_txp);
-	tid_data->next_reclaimed = ba_resp_scd_ssn;
+	tid_data->next_reclaimed = index;
 
 	iwl_mvm_check_ratid_empty(mvm, sta, tid);
 
 	freed = 0;
+	ba_info->status.status_driver_data[1] = (void *)(uintptr_t)rate;
 
 	skb_queue_walk(&reclaimed_skbs, skb) {
 		struct ieee80211_hdr *hdr = (void *)skb->data;
@@ -1661,8 +1683,12 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 
 		/* this is the first skb we deliver in this batch */
 		/* put the rate scaling data there */
-		if (freed == 1)
-			iwl_mvm_tx_info_from_ba_notif(info, ba_notif, tid_data);
+		if (freed == 1) {
+			info->flags |= IEEE80211_TX_STAT_AMPDU;
+			memcpy(&info->status, &ba_info->status,
+			       sizeof(ba_info->status));
+			iwl_mvm_hwrate_to_tx_status(rate, info);
+		}
 	}
 
 	spin_unlock_bh(&mvmsta->lock);
@@ -1672,7 +1698,6 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	 * Still it's important to update RS about sent vs. acked.
 	 */
 	if (skb_queue_empty(&reclaimed_skbs)) {
-		struct ieee80211_tx_info ba_info = {};
 		struct ieee80211_chanctx_conf *chanctx_conf = NULL;
 
 		if (mvmsta->vif)
@@ -1682,11 +1707,11 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 		if (WARN_ON_ONCE(!chanctx_conf))
 			goto out;
 
-		ba_info.band = chanctx_conf->def.chan->band;
-		iwl_mvm_tx_info_from_ba_notif(&ba_info, ba_notif, tid_data);
+		ba_info->band = chanctx_conf->def.chan->band;
+		iwl_mvm_hwrate_to_tx_status(rate, ba_info);
 
 		IWL_DEBUG_TX_REPLY(mvm, "No reclaim. Update rs directly\n");
-		iwl_mvm_rs_tx_status(mvm, sta, tid, &ba_info, false);
+		iwl_mvm_rs_tx_status(mvm, sta, tid, ba_info, false);
 	}
 
 out:
@@ -1698,6 +1723,92 @@ out:
 	}
 }
 
+void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	int sta_id, tid, txq, index;
+	struct ieee80211_tx_info ba_info = {};
+	struct iwl_mvm_ba_notif *ba_notif;
+	struct iwl_mvm_tid_data *tid_data;
+	struct iwl_mvm_sta *mvmsta;
+
+	if (iwl_mvm_has_new_tx_api(mvm)) {
+		struct iwl_mvm_compressed_ba_notif *ba_res =
+			(void *)pkt->data;
+
+		sta_id = ba_res->sta_id;
+		ba_info.status.ampdu_ack_len = (u8)le16_to_cpu(ba_res->done);
+		ba_info.status.ampdu_len = (u8)le16_to_cpu(ba_res->txed);
+		ba_info.status.tx_time =
+			(u16)le32_to_cpu(ba_res->wireless_time);
+		ba_info.status.status_driver_data[0] =
+			(void *)(uintptr_t)ba_res->reduced_txp;
+
+		/*
+		 * TODO:
+		 * When supporting multi TID aggregations - we need to move
+		 * next_reclaimed to be per TXQ and not per TID or handle it
+		 * in a different way.
+		 * This will go together with SN and AddBA offload and cannot
+		 * be handled properly for now.
+		 */
+		WARN_ON(le16_to_cpu(ba_res->tfd_cnt) != 1);
+		iwl_mvm_tx_reclaim(mvm, sta_id, ba_res->ra_tid[0].tid,
+				   (int)ba_res->tfd[0].q_num,
+				   le16_to_cpu(ba_res->tfd[0].tfd_index),
+				   &ba_info, le32_to_cpu(ba_res->tx_rate));
+
+		IWL_DEBUG_TX_REPLY(mvm,
+				   "BA_NOTIFICATION Received from sta_id = %d, flags %x, sent:%d, acked:%d\n",
+				   sta_id, le32_to_cpu(ba_res->flags),
+				   le16_to_cpu(ba_res->txed),
+				   le16_to_cpu(ba_res->done));
+		return;
+	}
+
+	ba_notif = (void *)pkt->data;
+	sta_id = ba_notif->sta_id;
+	tid = ba_notif->tid;
+	/* "flow" corresponds to Tx queue */
+	txq = le16_to_cpu(ba_notif->scd_flow);
+	/* "ssn" is start of block-ack Tx window, corresponds to index
+	 * (in Tx queue's circular buffer) of first TFD/frame in window */
+	index = le16_to_cpu(ba_notif->scd_ssn);
+
+	rcu_read_lock();
+	mvmsta = iwl_mvm_sta_from_staid_rcu(mvm, sta_id);
+	if (WARN_ON_ONCE(!mvmsta)) {
+		rcu_read_unlock();
+		return;
+	}
+
+	tid_data = &mvmsta->tid_data[tid];
+
+	ba_info.status.ampdu_ack_len = ba_notif->txed_2_done;
+	ba_info.status.ampdu_len = ba_notif->txed;
+	ba_info.status.tx_time = tid_data->tx_time;
+	ba_info.status.status_driver_data[0] =
+		(void *)(uintptr_t)ba_notif->reduced_txp;
+
+	rcu_read_unlock();
+
+	iwl_mvm_tx_reclaim(mvm, sta_id, tid, txq, index, &ba_info,
+			   tid_data->rate_n_flags);
+
+	IWL_DEBUG_TX_REPLY(mvm,
+			   "BA_NOTIFICATION Received from %pM, sta_id = %d\n",
+			   (u8 *)&ba_notif->sta_addr_lo32, ba_notif->sta_id);
+
+	IWL_DEBUG_TX_REPLY(mvm,
+			   "TID = %d, SeqCtl = %d, bitmap = 0x%llx, scd_flow = %d, scd_ssn = %d sent:%d, acked:%d\n",
+			   ba_notif->tid, le16_to_cpu(ba_notif->seq_ctl),
+			   le64_to_cpu(ba_notif->bitmap), txq, index,
+			   ba_notif->txed, ba_notif->txed_2_done);
+
+	IWL_DEBUG_TX_REPLY(mvm, "reduced txp from ba notif %d\n",
+			   ba_notif->reduced_txp);
+}
+
 /*
  * Note that there are transports that buffer frames before they reach
  * the firmware. This means that after flush_tx_path is called, the
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
index 68f4e7fdfc11..d04babd99b53 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
@@ -512,7 +512,7 @@ void iwl_mvm_dump_nic_error_log(struct iwl_mvm *mvm)
 			base = mvm->fw->inst_errlog_ptr;
 	}
 
-	if (base < 0x800000) {
+	if (base < 0x400000) {
 		IWL_ERR(mvm,
 			"Not valid error log pointer 0x%08X for %s uCode\n",
 			base,
@@ -610,7 +610,7 @@ int iwl_mvm_reconfig_scd(struct iwl_mvm *mvm, int queue, int fifo, int sta_id,
 {
 	struct iwl_scd_txq_cfg_cmd cmd = {
 		.scd_queue = queue,
-		.enable = 1,
+		.action = SCD_CFG_ENABLE_QUEUE,
 		.window = frame_limit,
 		.sta_id = sta_id,
 		.ssn = cpu_to_le16(ssn),
@@ -669,6 +669,8 @@ void iwl_mvm_enable_txq(struct iwl_mvm *mvm, int queue, int mac80211_queue,
 				tid_to_mac80211_ac[cfg->tid];
 		else
 			mvm->queue_info[queue].mac80211_ac = IEEE80211_AC_VO;
+
+		mvm->queue_info[queue].txq_tid = cfg->tid;
 	}
 
 	IWL_DEBUG_TX_QUEUES(mvm,
@@ -682,7 +684,7 @@ void iwl_mvm_enable_txq(struct iwl_mvm *mvm, int queue, int mac80211_queue,
 	if (enable_queue) {
 		struct iwl_scd_txq_cfg_cmd cmd = {
 			.scd_queue = queue,
-			.enable = 1,
+			.action = SCD_CFG_ENABLE_QUEUE,
 			.window = cfg->frame_limit,
 			.sta_id = cfg->sta_id,
 			.ssn = cpu_to_le16(ssn),
@@ -709,7 +711,7 @@ void iwl_mvm_disable_txq(struct iwl_mvm *mvm, int queue, int mac80211_queue,
 {
 	struct iwl_scd_txq_cfg_cmd cmd = {
 		.scd_queue = queue,
-		.enable = 0,
+		.action = SCD_CFG_DISABLE_QUEUE,
 	};
 	bool remove_mac_queue = true;
 	int ret;
@@ -744,8 +746,9 @@ void iwl_mvm_disable_txq(struct iwl_mvm *mvm, int queue, int mac80211_queue,
 			~BIT(mac80211_queue);
 	mvm->queue_info[queue].hw_queue_refcount--;
 
-	cmd.enable = mvm->queue_info[queue].hw_queue_refcount ? 1 : 0;
-	if (!cmd.enable)
+	cmd.action = mvm->queue_info[queue].hw_queue_refcount ?
+		SCD_CFG_ENABLE_QUEUE : SCD_CFG_DISABLE_QUEUE;
+	if (cmd.action == SCD_CFG_DISABLE_QUEUE)
 		mvm->queue_info[queue].status = IWL_MVM_QUEUE_FREE;
 
 	IWL_DEBUG_TX_QUEUES(mvm,
@@ -755,12 +758,13 @@ void iwl_mvm_disable_txq(struct iwl_mvm *mvm, int queue, int mac80211_queue,
 			    mvm->queue_info[queue].hw_queue_to_mac80211);
 
 	/* If the queue is still enabled - nothing left to do in this func */
-	if (cmd.enable) {
+	if (cmd.action == SCD_CFG_ENABLE_QUEUE) {
 		spin_unlock_bh(&mvm->queue_info_lock);
 		return;
 	}
 
 	cmd.sta_id = mvm->queue_info[queue].ra_sta_id;
+	cmd.tid = mvm->queue_info[queue].txq_tid;
 
 	/* Make sure queue info is correct even though we overwrite it */
 	WARN(mvm->queue_info[queue].hw_queue_refcount ||
@@ -1131,7 +1135,13 @@ static void iwl_mvm_remove_inactive_tids(struct iwl_mvm *mvm,
 			BIT(mvmsta->vif->hw_queue[tid_to_mac80211_ac[tid]]);
 	}
 
-	/* TODO: if queue was shared - need to re-enable AGGs */
+	/* If the queue is marked as shared - "unshare" it */
+	if (mvm->queue_info[queue].hw_queue_refcount == 1 &&
+	    mvm->queue_info[queue].status == IWL_MVM_QUEUE_SHARED) {
+		mvm->queue_info[queue].status = IWL_MVM_QUEUE_RECONFIGURING;
+		IWL_DEBUG_TX_QUEUES(mvm, "Marking Q:%d for reconfig\n",
+				    queue);
+	}
 }
 
 void iwl_mvm_inactivity_check(struct iwl_mvm *mvm)
@@ -1215,6 +1225,28 @@ void iwl_mvm_inactivity_check(struct iwl_mvm *mvm)
 	rcu_read_unlock();
 }
 
+void iwl_mvm_get_sync_time(struct iwl_mvm *mvm, u32 *gp2, u64 *boottime)
+{
+	bool ps_disabled;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	/* Disable power save when reading GP2 */
+	ps_disabled = mvm->ps_disabled;
+	if (!ps_disabled) {
+		mvm->ps_disabled = true;
+		iwl_mvm_power_update_device(mvm);
+	}
+
+	*gp2 = iwl_read_prph(mvm->trans, DEVICE_SYSTEM_TIME_REG);
+	*boottime = ktime_get_boot_ns();
+
+	if (!ps_disabled) {
+		mvm->ps_disabled = ps_disabled;
+		iwl_mvm_power_update_device(mvm);
+	}
+}
+
 int iwl_mvm_send_lqm_cmd(struct ieee80211_vif *vif,
 			 enum iwl_lqm_cmd_operatrions operation,
 			 u32 duration, u32 timeout)
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index 78cf9a7f3eac..001be406a3d3 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -487,6 +487,7 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
 	{IWL_PCI_DEVICE(0x24FD, 0x1130, iwl8265_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x24FD, 0x0130, iwl8265_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x24FD, 0x1010, iwl8265_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x10D0, iwl8265_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x24FD, 0x0050, iwl8265_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x24FD, 0x0150, iwl8265_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x24FD, 0x9010, iwl8265_2ac_cfg)},
@@ -500,22 +501,36 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
 	{IWL_PCI_DEVICE(0x24FD, 0x0930, iwl8265_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x24FD, 0x0950, iwl8265_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x24FD, 0x0850, iwl8265_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x24FD, 0x0012, iwl8275_2ac_cfg)},
 
 /* 9000 Series */
+	{IWL_PCI_DEVICE(0x271B, 0x0010, iwl9160_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x2526, 0x0000, iwl9260_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x2526, 0x0010, iwl9260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x0A10, iwl9260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x0010, iwl9260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x0210, iwl9260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x0410, iwl9260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x0610, iwl9260_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x0310, iwl5165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x0000, iwl5165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x0510, iwl5165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x2010, iwl5165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x2526, 0x1420, iwl5165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x0710, iwl5165_2ac_cfg)},
-	{IWL_PCI_DEVICE(0x9DF0, 0x2A10, iwl5165_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x1410, iwl9270_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0A10, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0010, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0210, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0410, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0610, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0310, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0000, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0510, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x2010, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x1420, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0710, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x2A10, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x30DC, 0x0060, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0060, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0060, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0xA370, 0x0060, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0060, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0030, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x9DF0, 0x0030, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0xA370, 0x0030, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0030, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x1030, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0xA370, 0x1030, iwl9560_2ac_cfg)},
 
 /* a000 Series */
 	{IWL_PCI_DEVICE(0x2720, 0x0A10, iwla000_2ac_cfg)},
@@ -608,7 +623,6 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	const struct iwl_cfg *cfg = (struct iwl_cfg *)(ent->driver_data);
 	const struct iwl_cfg *cfg_7265d __maybe_unused = NULL;
-	const struct iwl_cfg *cfg_9260lc __maybe_unused = NULL;
 	struct iwl_trans *iwl_trans;
 	int ret;
 
@@ -637,11 +651,10 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	if (iwl_trans->cfg->rf_id) {
-		if (cfg == &iwl9260_2ac_cfg)
-			cfg_9260lc = &iwl9260lc_2ac_cfg;
-		if (cfg_9260lc && iwl_trans->hw_rf_id == CSR_HW_RF_ID_TYPE_LC) {
-			cfg = cfg_9260lc;
-			iwl_trans->cfg = cfg_9260lc;
+		if (cfg == &iwl9460_2ac_cfg &&
+		    iwl_trans->hw_rf_id == CSR_HW_RF_ID_TYPE_LC) {
+			cfg = &iwl9000lc_2ac_cfg;
+			iwl_trans->cfg = cfg;
 		}
 	}
 #endif
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
index 11e347dd44c7..cac6d99012b3 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
@@ -37,6 +37,7 @@
 #include <linux/wait.h>
 #include <linux/pci.h>
 #include <linux/timer.h>
+#include <linux/cpu.h>
 
 #include "iwl-fh.h"
 #include "iwl-csr.h"
@@ -49,7 +50,7 @@
  * be needed for potential data in the SKB's head. The remaining ones can
  * be used for frags.
  */
-#define IWL_PCIE_MAX_FRAGS (IWL_NUM_OF_TBS - 3)
+#define IWL_PCIE_MAX_FRAGS(x) (x->max_tbs - 3)
 
 /*
  * RX related structures and functions
@@ -192,41 +193,9 @@ struct iwl_cmd_meta {
 	/* only for SYNC commands, iff the reply skb is wanted */
 	struct iwl_host_cmd *source;
 	u32 flags;
+	u32 tbs;
 };
 
-/*
- * Generic queue structure
- *
- * Contains common data for Rx and Tx queues.
- *
- * Note the difference between TFD_QUEUE_SIZE_MAX and n_window: the hardware
- * always assumes 256 descriptors, so TFD_QUEUE_SIZE_MAX is always 256 (unless
- * there might be HW changes in the future). For the normal TX
- * queues, n_window, which is the size of the software queue data
- * is also 256; however, for the command queue, n_window is only
- * 32 since we don't need so many commands pending. Since the HW
- * still uses 256 BDs for DMA though, TFD_QUEUE_SIZE_MAX stays 256. As a result,
- * the software buffers (in the variables @meta, @txb in struct
- * iwl_txq) only have 32 entries, while the HW buffers (@tfds in
- * the same struct) have 256.
- * This means that we end up with the following:
- *  HW entries: | 0 | ... | N * 32 | ... | N * 32 + 31 | ... | 255 |
- *  SW entries:           | 0      | ... | 31          |
- * where N is a number between 0 and 7. This means that the SW
- * data is a window overlayed over the HW queue.
- */
-struct iwl_queue {
-	int write_ptr;       /* 1-st empty entry (index) host_w*/
-	int read_ptr;         /* last used entry (index) host_r*/
-	/* use for monitoring and recovering the stuck queue */
-	dma_addr_t dma_addr;   /* physical addr for BD's */
-	int n_window;	       /* safe queue window */
-	u32 id;
-	int low_mark;	       /* low watermark, resume queue if free
-				* space more than this */
-	int high_mark;         /* high watermark, stop queue if free
-				* space less than this */
-};
 
 #define TFD_TX_CMD_SLOTS 256
 #define TFD_CMD_SLOTS 32
@@ -273,13 +242,32 @@ struct iwl_pcie_first_tb_buf {
  * @wd_timeout: queue watchdog timeout (jiffies) - per queue
  * @frozen: tx stuck queue timer is frozen
  * @frozen_expiry_remainder: remember how long until the timer fires
+ * @write_ptr: 1-st empty entry (index) host_w
+ * @read_ptr: last used entry (index) host_r
+ * @dma_addr:  physical addr for BD's
+ * @n_window: safe queue window
+ * @id: queue id
+ * @low_mark: low watermark, resume queue if free space more than this
+ * @high_mark: high watermark, stop queue if free space less than this
  *
  * A Tx queue consists of circular buffer of BDs (a.k.a. TFDs, transmit frame
  * descriptors) and required locking structures.
+ *
+ * Note the difference between TFD_QUEUE_SIZE_MAX and n_window: the hardware
+ * always assumes 256 descriptors, so TFD_QUEUE_SIZE_MAX is always 256 (unless
+ * there might be HW changes in the future). For the normal TX
+ * queues, n_window, which is the size of the software queue data
+ * is also 256; however, for the command queue, n_window is only
+ * 32 since we don't need so many commands pending. Since the HW
+ * still uses 256 BDs for DMA though, TFD_QUEUE_SIZE_MAX stays 256.
+ * This means that we end up with the following:
+ *  HW entries: | 0 | ... | N * 32 | ... | N * 32 + 31 | ... | 255 |
+ *  SW entries:           | 0      | ... | 31          |
+ * where N is a number between 0 and 7. This means that the SW
+ * data is a window overlayed over the HW queue.
  */
 struct iwl_txq {
-	struct iwl_queue q;
-	struct iwl_tfd *tfds;
+	void *tfds;
 	struct iwl_pcie_first_tb_buf *first_tb_bufs;
 	dma_addr_t first_tb_dma;
 	struct iwl_pcie_txq_entry *entries;
@@ -294,6 +282,14 @@ struct iwl_txq {
 	bool block;
 	unsigned long wd_timeout;
 	struct sk_buff_head overflow_q;
+
+	int write_ptr;
+	int read_ptr;
+	dma_addr_t dma_addr;
+	int n_window;
+	u32 id;
+	int low_mark;
+	int high_mark;
 };
 
 static inline dma_addr_t
@@ -309,6 +305,16 @@ struct iwl_tso_hdr_page {
 };
 
 /**
+ * enum iwl_shared_irq_flags - level of sharing for irq
+ * @IWL_SHARED_IRQ_NON_RX: interrupt vector serves non rx causes.
+ * @IWL_SHARED_IRQ_FIRST_RSS: interrupt vector serves first RSS queue.
+ */
+enum iwl_shared_irq_flags {
+	IWL_SHARED_IRQ_NON_RX		= BIT(0),
+	IWL_SHARED_IRQ_FIRST_RSS	= BIT(1),
+};
+
+/**
  * struct iwl_trans_pcie - PCIe transport specific data
  * @rxq: all the RX queue data
  * @rx_pool: initial pool of iwl_rx_mem_buffer for all the queues
@@ -326,7 +332,6 @@ struct iwl_tso_hdr_page {
  * @rx_buf_size: Rx buffer size
  * @bc_table_dword: true if the BC table expects DWORD (as opposed to bytes)
  * @scd_set_active: should the transport configure the SCD for HCMD queue
- * @wide_cmd_header: true when ucode supports wide command header format
  * @sw_csum_tx: if true, then the transport will compute the csum of the TXed
  *	frame.
  * @rx_page_order: page order for receive buffer size
@@ -338,8 +343,10 @@ struct iwl_tso_hdr_page {
  * @fw_mon_size: size of the buffer for the firmware monitor
  * @msix_entries: array of MSI-X entries
  * @msix_enabled: true if managed to enable MSI-X
- * @allocated_vector: the number of interrupt vector allocated by the OS
- * @default_irq_num: default irq for non rx interrupt
+ * @shared_vec_mask: the type of causes the shared vector handles
+ *	(see iwl_shared_irq_flags).
+ * @alloc_vecs: the number of interrupt vectors allocated by the OS
+ * @def_irq: default irq for non rx causes
  * @fh_init_mask: initial unmasked fh causes
  * @hw_init_mask: initial unmasked hw causes
  * @fh_mask: current unmasked fh causes
@@ -391,11 +398,12 @@ struct iwl_trans_pcie {
 	unsigned int cmd_q_wdg_timeout;
 	u8 n_no_reclaim_cmds;
 	u8 no_reclaim_cmds[MAX_NO_RECLAIM_CMDS];
+	u8 max_tbs;
+	u16 tfd_size;
 
 	enum iwl_amsdu_size rx_buf_size;
 	bool bc_table_dword;
 	bool scd_set_active;
-	bool wide_cmd_header;
 	bool sw_csum_tx;
 	u32 rx_page_order;
 
@@ -410,12 +418,14 @@ struct iwl_trans_pcie {
 
 	struct msix_entry msix_entries[IWL_MAX_RX_HW_QUEUES];
 	bool msix_enabled;
-	u32 allocated_vector;
-	u32 default_irq_num;
+	u8 shared_vec_mask;
+	u32 alloc_vecs;
+	u32 def_irq;
 	u32 fh_init_mask;
 	u32 hw_init_mask;
 	u32 fh_mask;
 	u32 hw_mask;
+	cpumask_t affinity_mask[IWL_MAX_RX_HW_QUEUES];
 };
 
 static inline struct iwl_trans_pcie *
@@ -474,6 +484,7 @@ void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int queue,
 				bool configure_scd);
 void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id,
 					bool shared_mode);
+dma_addr_t iwl_trans_pcie_get_txq_byte_table(struct iwl_trans *trans, int txq);
 void iwl_trans_pcie_log_scd_error(struct iwl_trans *trans,
 				  struct iwl_txq *txq);
 int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
@@ -486,11 +497,20 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 			    struct sk_buff_head *skbs);
 void iwl_trans_pcie_tx_reset(struct iwl_trans *trans);
 
-static inline u16 iwl_pcie_tfd_tb_get_len(struct iwl_tfd *tfd, u8 idx)
+static inline u16 iwl_pcie_tfd_tb_get_len(struct iwl_trans *trans, void *_tfd,
+					  u8 idx)
 {
-	struct iwl_tfd_tb *tb = &tfd->tbs[idx];
+	if (trans->cfg->use_tfh) {
+		struct iwl_tfh_tfd *tfd = _tfd;
+		struct iwl_tfh_tb *tb = &tfd->tbs[idx];
+
+		return le16_to_cpu(tb->tb_len);
+	} else {
+		struct iwl_tfd *tfd = _tfd;
+		struct iwl_tfd_tb *tb = &tfd->tbs[idx];
 
-	return le16_to_cpu(tb->hi_n_len) >> 4;
+		return le16_to_cpu(tb->hi_n_len) >> 4;
+	}
 }
 
 /*****************************************************
@@ -617,9 +637,9 @@ static inline void iwl_wake_queue(struct iwl_trans *trans,
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
-	if (test_and_clear_bit(txq->q.id, trans_pcie->queue_stopped)) {
-		IWL_DEBUG_TX_QUEUES(trans, "Wake hwq %d\n", txq->q.id);
-		iwl_op_mode_queue_not_full(trans->op_mode, txq->q.id);
+	if (test_and_clear_bit(txq->id, trans_pcie->queue_stopped)) {
+		IWL_DEBUG_TX_QUEUES(trans, "Wake hwq %d\n", txq->id);
+		iwl_op_mode_queue_not_full(trans->op_mode, txq->id);
 	}
 }
 
@@ -628,22 +648,22 @@ static inline void iwl_stop_queue(struct iwl_trans *trans,
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
-	if (!test_and_set_bit(txq->q.id, trans_pcie->queue_stopped)) {
-		iwl_op_mode_queue_full(trans->op_mode, txq->q.id);
-		IWL_DEBUG_TX_QUEUES(trans, "Stop hwq %d\n", txq->q.id);
+	if (!test_and_set_bit(txq->id, trans_pcie->queue_stopped)) {
+		iwl_op_mode_queue_full(trans->op_mode, txq->id);
+		IWL_DEBUG_TX_QUEUES(trans, "Stop hwq %d\n", txq->id);
 	} else
 		IWL_DEBUG_TX_QUEUES(trans, "hwq %d already stopped\n",
-				    txq->q.id);
+				    txq->id);
 }
 
-static inline bool iwl_queue_used(const struct iwl_queue *q, int i)
+static inline bool iwl_queue_used(const struct iwl_txq *q, int i)
 {
 	return q->write_ptr >= q->read_ptr ?
 		(i >= q->read_ptr && i < q->write_ptr) :
 		!(i < q->read_ptr && i >= q->write_ptr);
 }
 
-static inline u8 get_cmd_index(struct iwl_queue *q, u32 index)
+static inline u8 get_cmd_index(struct iwl_txq *q, u32 index)
 {
 	return index & (q->n_window - 1);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
index 5c36e6d00622..6fe5546dc773 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
@@ -487,15 +487,13 @@ static void iwl_pcie_rx_allocator(struct iwl_trans *trans)
 
 	while (pending) {
 		int i;
-		struct list_head local_allocated;
+		LIST_HEAD(local_allocated);
 		gfp_t gfp_mask = GFP_KERNEL;
 
 		/* Do not post a warning if there are only a few requests */
 		if (pending < RX_PENDING_WATERMARK)
 			gfp_mask |= __GFP_NOWARN;
 
-		INIT_LIST_HEAD(&local_allocated);
-
 		for (i = 0; i < RX_CLAIM_REQ_ALLOC;) {
 			struct iwl_rx_mem_buffer *rxb;
 			struct page *page;
@@ -1108,13 +1106,14 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
 			FH_RSCSR_RXQ_POS != rxq->id);
 
 		IWL_DEBUG_RX(trans,
-			     "cmd at offset %d: %s (0x%.2x, seq 0x%x)\n",
+			     "cmd at offset %d: %s (%.2x.%2x, seq 0x%x)\n",
 			     rxcb._offset,
 			     iwl_get_cmd_string(trans,
 						iwl_cmd_id(pkt->hdr.cmd,
 							   pkt->hdr.group_id,
 							   0)),
-			     pkt->hdr.cmd, le16_to_cpu(pkt->hdr.sequence));
+			     pkt->hdr.group_id, pkt->hdr.cmd,
+			     le16_to_cpu(pkt->hdr.sequence));
 
 		len = iwl_rx_packet_len(pkt);
 		len += sizeof(u32); /* account for status word */
@@ -1142,7 +1141,7 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
 
 		sequence = le16_to_cpu(pkt->hdr.sequence);
 		index = SEQ_TO_INDEX(sequence);
-		cmd_index = get_cmd_index(&txq->q, index);
+		cmd_index = get_cmd_index(txq, index);
 
 		if (rxq->id == 0)
 			iwl_op_mode_rx(trans->op_mode, &rxq->napi,
@@ -1885,6 +1884,20 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
 			      inta_fh,
 			      iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD));
 
+	if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX) &&
+	    inta_fh & MSIX_FH_INT_CAUSES_Q0) {
+		local_bh_disable();
+		iwl_pcie_rx_handle(trans, 0);
+		local_bh_enable();
+	}
+
+	if ((trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS) &&
+	    inta_fh & MSIX_FH_INT_CAUSES_Q1) {
+		local_bh_disable();
+		iwl_pcie_rx_handle(trans, 1);
+		local_bh_enable();
+	}
+
 	/* This "Tx" DMA channel is used only for loading uCode */
 	if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM) {
 		IWL_DEBUG_ISR(trans, "uCode load interrupt\n");
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
index 74f2f035bd28..ae95533e587d 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
@@ -827,10 +827,16 @@ static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans,
 		if (ret)
 			return ret;
 
-		/* Notify the ucode of the loaded section number and status */
-		val = iwl_read_direct32(trans, FH_UCODE_LOAD_STATUS);
-		val = val | (sec_num << shift_param);
-		iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, val);
+		/* Notify ucode of loaded section number and status */
+		if (trans->cfg->use_tfh) {
+			val = iwl_read_prph(trans, UREG_UCODE_LOAD_STATUS);
+			val = val | (sec_num << shift_param);
+			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS, val);
+		} else {
+			val = iwl_read_direct32(trans, FH_UCODE_LOAD_STATUS);
+			val = val | (sec_num << shift_param);
+			iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, val);
+		}
 		sec_num = (sec_num << 1) | 0x1;
 	}
 
@@ -838,10 +844,21 @@ static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans,
 
 	iwl_enable_interrupts(trans);
 
-	if (cpu == 1)
-		iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, 0xFFFF);
-	else
-		iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, 0xFFFFFFFF);
+	if (trans->cfg->use_tfh) {
+		if (cpu == 1)
+			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
+				       0xFFFF);
+		else
+			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
+				       0xFFFFFFFF);
+	} else {
+		if (cpu == 1)
+			iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
+					   0xFFFF);
+		else
+			iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
+					   0xFFFFFFFF);
+	}
 
 	return 0;
 }
@@ -886,14 +903,6 @@ static int iwl_pcie_load_cpu_sections(struct iwl_trans *trans,
 			return ret;
 	}
 
-	if (trans->cfg->device_family == IWL_DEVICE_FAMILY_8000)
-		iwl_set_bits_prph(trans,
-				  CSR_UCODE_LOAD_STATUS_ADDR,
-				  (LMPM_CPU_UCODE_LOADING_COMPLETED |
-				   LMPM_CPU_HDRS_LOADING_COMPLETED |
-				   LMPM_CPU_UCODE_LOADING_STARTED) <<
-					shift_param);
-
 	*first_ucode_section = last_read_idx;
 
 	return 0;
@@ -1161,7 +1170,7 @@ static void iwl_pcie_synchronize_irqs(struct iwl_trans *trans)
 	if (trans_pcie->msix_enabled) {
 		int i;
 
-		for (i = 0; i < trans_pcie->allocated_vector; i++)
+		for (i = 0; i < trans_pcie->alloc_vecs; i++)
 			synchronize_irq(trans_pcie->msix_entries[i].vector);
 	} else {
 		synchronize_irq(trans_pcie->pci_dev->irq);
@@ -1420,13 +1429,58 @@ static struct iwl_causes_list causes_list[] = {
 	{MSIX_HW_INT_CAUSES_REG_HAP,		CSR_MSIX_HW_INT_MASK_AD, 0x2E},
 };
 
+static void iwl_pcie_map_non_rx_causes(struct iwl_trans *trans)
+{
+	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
+	int val = trans_pcie->def_irq | MSIX_NON_AUTO_CLEAR_CAUSE;
+	int i;
+
+	/*
+	 * Access all non RX causes and map them to the default irq.
+	 * In case we are missing at least one interrupt vector,
+	 * the first interrupt vector will serve non-RX and FBQ causes.
+	 */
+	for (i = 0; i < ARRAY_SIZE(causes_list); i++) {
+		iwl_write8(trans, CSR_MSIX_IVAR(causes_list[i].addr), val);
+		iwl_clear_bit(trans, causes_list[i].mask_reg,
+			      causes_list[i].cause_num);
+	}
+}
+
+static void iwl_pcie_map_rx_causes(struct iwl_trans *trans)
+{
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	u32 offset =
+		trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 1 : 0;
+	u32 val, idx;
+
+	/*
+	 * The first RX queue - fallback queue, which is designated for
+	 * management frame, command responses etc, is always mapped to the
+	 * first interrupt vector. The other RX queues are mapped to
+	 * the other (N - 2) interrupt vectors.
+	 */
+	val = BIT(MSIX_FH_INT_CAUSES_Q(0));
+	for (idx = 1; idx < trans->num_rx_queues; idx++) {
+		iwl_write8(trans, CSR_MSIX_RX_IVAR(idx),
+			   MSIX_FH_INT_CAUSES_Q(idx - offset));
+		val |= BIT(MSIX_FH_INT_CAUSES_Q(idx));
+	}
+	iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~val);
+
+	val = MSIX_FH_INT_CAUSES_Q(0);
+	if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX)
+		val |= MSIX_NON_AUTO_CLEAR_CAUSE;
+	iwl_write8(trans, CSR_MSIX_RX_IVAR(0), val);
+
+	if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS)
+		iwl_write8(trans, CSR_MSIX_RX_IVAR(1), val);
+}
+
 static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie)
 {
-	u32 val, max_rx_vector, i;
 	struct iwl_trans *trans = trans_pcie->trans;
 
-	max_rx_vector = trans_pcie->allocated_vector - 1;
-
 	if (!trans_pcie->msix_enabled) {
 		if (trans->cfg->mq_rx_supported)
 			iwl_write_prph(trans, UREG_CHICK,
@@ -1437,25 +1491,16 @@ static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie)
 	iwl_write_prph(trans, UREG_CHICK, UREG_CHICK_MSIX_ENABLE);
 
 	/*
-	 * Each cause from the list above and the RX causes is represented as
-	 * a byte in the IVAR table. We access the first (N - 1) bytes and map
-	 * them to the (N - 1) vectors so these vectors will be used as rx
-	 * vectors. Then access all non rx causes and map them to the
-	 * default queue (N'th queue).
+	 * Each cause from the causes list above and the RX causes is
+	 * represented as a byte in the IVAR table. The first nibble
+	 * represents the bound interrupt vector of the cause, the second
+	 * represents no auto clear for this cause. This will be set if its
+	 * interrupt vector is bound to serve other causes.
 	 */
-	for (i = 0; i < max_rx_vector; i++) {
-		iwl_write8(trans, CSR_MSIX_RX_IVAR(i), MSIX_FH_INT_CAUSES_Q(i));
-		iwl_clear_bit(trans, CSR_MSIX_FH_INT_MASK_AD,
-			      BIT(MSIX_FH_INT_CAUSES_Q(i)));
-	}
+	iwl_pcie_map_rx_causes(trans);
+
+	iwl_pcie_map_non_rx_causes(trans);
 
-	for (i = 0; i < ARRAY_SIZE(causes_list); i++) {
-		val = trans_pcie->default_irq_num |
-			MSIX_NON_AUTO_CLEAR_CAUSE;
-		iwl_write8(trans, CSR_MSIX_IVAR(causes_list[i].addr), val);
-		iwl_clear_bit(trans, causes_list[i].mask_reg,
-			      causes_list[i].cause_num);
-	}
 	trans_pcie->fh_init_mask =
 		~iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD);
 	trans_pcie->fh_mask = trans_pcie->fh_init_mask;
@@ -1468,40 +1513,55 @@ static void iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
 					struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	int max_irqs, num_irqs, i, ret, nr_online_cpus;
 	u16 pci_cmd;
-	int max_vector;
-	int ret, i;
-
-	if (trans->cfg->mq_rx_supported) {
-		max_vector = min_t(u32, (num_possible_cpus() + 2),
-				   IWL_MAX_RX_HW_QUEUES);
-		for (i = 0; i < max_vector; i++)
-			trans_pcie->msix_entries[i].entry = i;
-
-		ret = pci_enable_msix_range(pdev, trans_pcie->msix_entries,
-					    MSIX_MIN_INTERRUPT_VECTORS,
-					    max_vector);
-		if (ret > 1) {
-			IWL_DEBUG_INFO(trans,
-				       "Enable MSI-X allocate %d interrupt vector\n",
-				       ret);
-			trans_pcie->allocated_vector = ret;
-			trans_pcie->default_irq_num =
-				trans_pcie->allocated_vector - 1;
-			trans_pcie->trans->num_rx_queues =
-				trans_pcie->allocated_vector - 1;
-			trans_pcie->msix_enabled = true;
-
-			return;
-		}
+
+	if (!trans->cfg->mq_rx_supported)
+		goto enable_msi;
+
+	nr_online_cpus = num_online_cpus();
+	max_irqs = min_t(u32, nr_online_cpus + 2, IWL_MAX_RX_HW_QUEUES);
+	for (i = 0; i < max_irqs; i++)
+		trans_pcie->msix_entries[i].entry = i;
+
+	num_irqs = pci_enable_msix_range(pdev, trans_pcie->msix_entries,
+					 MSIX_MIN_INTERRUPT_VECTORS,
+					 max_irqs);
+	if (num_irqs < 0) {
 		IWL_DEBUG_INFO(trans,
-			       "ret = %d %s move to msi mode\n", ret,
-			       (ret == 1) ?
-			       "can't allocate more than 1 interrupt vector" :
-			       "failed to enable msi-x mode");
-		pci_disable_msix(pdev);
+			       "Failed to enable msi-x mode (ret %d). Moving to msi mode.\n",
+			       num_irqs);
+		goto enable_msi;
 	}
+	trans_pcie->def_irq = (num_irqs == max_irqs) ? num_irqs - 1 : 0;
+
+	IWL_DEBUG_INFO(trans,
+		       "MSI-X enabled. %d interrupt vectors were allocated\n",
+		       num_irqs);
 
+	/*
+	 * In case the OS provides fewer interrupts than requested, different
+	 * causes will share the same interrupt vector as follows:
+	 * One interrupt less: non rx causes shared with FBQ.
+	 * Two interrupts less: non rx causes shared with FBQ and RSS.
+	 * More than two interrupts: we will use fewer RSS queues.
+	 */
+	if (num_irqs <= nr_online_cpus) {
+		trans_pcie->trans->num_rx_queues = num_irqs + 1;
+		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX |
+			IWL_SHARED_IRQ_FIRST_RSS;
+	} else if (num_irqs == nr_online_cpus + 1) {
+		trans_pcie->trans->num_rx_queues = num_irqs;
+		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX;
+	} else {
+		trans_pcie->trans->num_rx_queues = num_irqs - 1;
+	}
+
+	trans_pcie->alloc_vecs = num_irqs;
+	trans_pcie->msix_enabled = true;
+	return;
+
+enable_msi:
 	ret = pci_enable_msi(pdev);
 	if (ret) {
 		dev_err(&pdev->dev, "pci_enable_msi failed - %d\n", ret);
@@ -1514,36 +1574,57 @@ static void iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
 	}
 }
 
+static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans)
+{
+	int iter_rx_q, i, ret, cpu, offset;
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
+	i = trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 0 : 1;
+	iter_rx_q = trans_pcie->trans->num_rx_queues - 1 + i;
+	offset = 1 + i;
+	for (; i < iter_rx_q ; i++) {
+		/*
+		 * Get the cpu prior to the place to search
+		 * (i.e. return will be > i - 1).
+		 */
+		cpu = cpumask_next(i - offset, cpu_online_mask);
+		cpumask_set_cpu(cpu, &trans_pcie->affinity_mask[i]);
+		ret = irq_set_affinity_hint(trans_pcie->msix_entries[i].vector,
+					    &trans_pcie->affinity_mask[i]);
+		if (ret)
+			IWL_ERR(trans_pcie->trans,
+				"Failed to set affinity mask for IRQ %d\n",
+				i);
+	}
+}
+
 static int iwl_pcie_init_msix_handler(struct pci_dev *pdev,
 				      struct iwl_trans_pcie *trans_pcie)
 {
-	int i, last_vector;
-
-	last_vector = trans_pcie->trans->num_rx_queues;
+	int i;
 
-	for (i = 0; i < trans_pcie->allocated_vector; i++) {
+	for (i = 0; i < trans_pcie->alloc_vecs; i++) {
 		int ret;
-
-		ret = request_threaded_irq(trans_pcie->msix_entries[i].vector,
-					   iwl_pcie_msix_isr,
-					   (i == last_vector) ?
-					   iwl_pcie_irq_msix_handler :
-					   iwl_pcie_irq_rx_msix_handler,
-					   IRQF_SHARED,
-					   DRV_NAME,
-					   &trans_pcie->msix_entries[i]);
+		struct msix_entry *msix_entry;
+
+		msix_entry = &trans_pcie->msix_entries[i];
+		ret = devm_request_threaded_irq(&pdev->dev,
+						msix_entry->vector,
+						iwl_pcie_msix_isr,
+						(i == trans_pcie->def_irq) ?
+						iwl_pcie_irq_msix_handler :
+						iwl_pcie_irq_rx_msix_handler,
+						IRQF_SHARED,
+						DRV_NAME,
+						msix_entry);
 		if (ret) {
-			int j;
-
 			IWL_ERR(trans_pcie->trans,
 				"Error allocating IRQ %d\n", i);
-			for (j = 0; j < i; j++)
-				free_irq(trans_pcie->msix_entries[j].vector,
-					 &trans_pcie->msix_entries[j]);
-			pci_disable_msix(pdev);
+
 			return ret;
 		}
 	}
+	iwl_pcie_irq_set_affinity(trans_pcie->trans);
 
 	return 0;
 }
@@ -1672,7 +1753,6 @@ static void iwl_trans_pcie_configure(struct iwl_trans *trans,
 	trans_pcie->rx_page_order =
 		iwl_trans_get_rb_size_order(trans_pcie->rx_buf_size);
 
-	trans_pcie->wide_cmd_header = trans_cfg->wide_cmd_header;
 	trans_pcie->bc_table_dword = trans_cfg->bc_table_dword;
 	trans_pcie->scd_set_active = trans_cfg->scd_set_active;
 	trans_pcie->sw_csum_tx = trans_cfg->sw_csum_tx;
@@ -1703,22 +1783,16 @@ void iwl_trans_pcie_free(struct iwl_trans *trans)
 	iwl_pcie_rx_free(trans);
 
 	if (trans_pcie->msix_enabled) {
-		for (i = 0; i < trans_pcie->allocated_vector; i++)
-			free_irq(trans_pcie->msix_entries[i].vector,
-				 &trans_pcie->msix_entries[i]);
+		for (i = 0; i < trans_pcie->alloc_vecs; i++) {
+			irq_set_affinity_hint(
+				trans_pcie->msix_entries[i].vector,
+				NULL);
+		}
 
-		pci_disable_msix(trans_pcie->pci_dev);
 		trans_pcie->msix_enabled = false;
 	} else {
-		free_irq(trans_pcie->pci_dev->irq, trans);
-
 		iwl_pcie_free_ict(trans);
-
-		pci_disable_msi(trans_pcie->pci_dev);
 	}
-	iounmap(trans_pcie->hw_base);
-	pci_release_regions(trans_pcie->pci_dev);
-	pci_disable_device(trans_pcie->pci_dev);
 
 	iwl_pcie_free_fw_monitor(trans);
 
@@ -1890,7 +1964,7 @@ static void iwl_trans_pcie_freeze_txq_timer(struct iwl_trans *trans,
 
 		txq->frozen = freeze;
 
-		if (txq->q.read_ptr == txq->q.write_ptr)
+		if (txq->read_ptr == txq->write_ptr)
 			goto next_queue;
 
 		if (freeze) {
@@ -1938,7 +2012,7 @@ static void iwl_trans_pcie_block_txq_ptrs(struct iwl_trans *trans, bool block)
 			txq->block--;
 			if (!txq->block) {
 				iwl_write32(trans, HBUS_TARG_WRPTR,
-					    txq->q.write_ptr | (i << 8));
+					    txq->write_ptr | (i << 8));
 			}
 		} else if (block) {
 			txq->block++;
@@ -1958,10 +2032,14 @@ void iwl_trans_pcie_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq)
 	int cnt;
 
 	IWL_ERR(trans, "Current SW read_ptr %d write_ptr %d\n",
-		txq->q.read_ptr, txq->q.write_ptr);
+		txq->read_ptr, txq->write_ptr);
+
+	if (trans->cfg->use_tfh)
+		/* TODO: access new SCD registers and dump them */
+		return;
 
 	scd_sram_addr = trans_pcie->scd_base_addr +
-			SCD_TX_STTS_QUEUE_OFFSET(txq->q.id);
+			SCD_TX_STTS_QUEUE_OFFSET(txq->id);
 	iwl_trans_read_mem_bytes(trans, scd_sram_addr, buf, sizeof(buf));
 
 	iwl_print_hex_error(trans, buf, sizeof(buf));
@@ -1996,7 +2074,6 @@ static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, u32 txq_bm)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_txq *txq;
-	struct iwl_queue *q;
 	int cnt;
 	unsigned long now = jiffies;
 	int ret = 0;
@@ -2014,13 +2091,12 @@ static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, u32 txq_bm)
 
 		IWL_DEBUG_TX_QUEUES(trans, "Emptying queue %d...\n", cnt);
 		txq = &trans_pcie->txq[cnt];
-		q = &txq->q;
-		wr_ptr = ACCESS_ONCE(q->write_ptr);
+		wr_ptr = ACCESS_ONCE(txq->write_ptr);
 
-		while (q->read_ptr != ACCESS_ONCE(q->write_ptr) &&
+		while (txq->read_ptr != ACCESS_ONCE(txq->write_ptr) &&
 		       !time_after(jiffies,
 				   now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS))) {
-			u8 write_ptr = ACCESS_ONCE(q->write_ptr);
+			u8 write_ptr = ACCESS_ONCE(txq->write_ptr);
 
 			if (WARN_ONCE(wr_ptr != write_ptr,
 				      "WR pointer moved while flushing %d -> %d\n",
@@ -2029,7 +2105,7 @@ static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, u32 txq_bm)
 			usleep_range(1000, 2000);
 		}
 
-		if (q->read_ptr != q->write_ptr) {
+		if (txq->read_ptr != txq->write_ptr) {
 			IWL_ERR(trans,
 				"fail to flush all tx fifo queues Q %d\n", cnt);
 			ret = -ETIMEDOUT;
@@ -2197,7 +2273,6 @@ static ssize_t iwl_dbgfs_tx_queue_read(struct file *file,
 	struct iwl_trans *trans = file->private_data;
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_txq *txq;
-	struct iwl_queue *q;
 	char *buf;
 	int pos = 0;
 	int cnt;
@@ -2215,10 +2290,9 @@ static ssize_t iwl_dbgfs_tx_queue_read(struct file *file,
 
 	for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) {
 		txq = &trans_pcie->txq[cnt];
-		q = &txq->q;
 		pos += scnprintf(buf + pos, bufsz - pos,
 				"hwq %.2d: read=%u write=%u use=%d stop=%d need_update=%d frozen=%d%s\n",
-				cnt, q->read_ptr, q->write_ptr,
+				cnt, txq->read_ptr, txq->write_ptr,
 				!!test_bit(cnt, trans_pcie->queue_used),
 				 !!test_bit(cnt, trans_pcie->queue_stopped),
 				 txq->need_update, txq->frozen,
@@ -2424,13 +2498,14 @@ err:
 }
 #endif /*CONFIG_IWLWIFI_DEBUGFS */
 
-static u32 iwl_trans_pcie_get_cmdlen(struct iwl_tfd *tfd)
+static u32 iwl_trans_pcie_get_cmdlen(struct iwl_trans *trans, void *tfd)
 {
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	u32 cmdlen = 0;
 	int i;
 
-	for (i = 0; i < IWL_NUM_OF_TBS; i++)
-		cmdlen += iwl_pcie_tfd_tb_get_len(tfd, i);
+	for (i = 0; i < trans_pcie->max_tbs; i++)
+		cmdlen += iwl_pcie_tfd_tb_get_len(trans, tfd, i);
 
 	return cmdlen;
 }
@@ -2645,7 +2720,7 @@ static struct iwl_trans_dump_data
 
 	/* host commands */
 	len += sizeof(*data) +
-		cmdq->q.n_window * (sizeof(*txcmd) + TFD_MAX_PAYLOAD_SIZE);
+		cmdq->n_window * (sizeof(*txcmd) + TFD_MAX_PAYLOAD_SIZE);
 
 	/* FW monitor */
 	if (trans_pcie->fw_mon_page) {
@@ -2713,12 +2788,13 @@ static struct iwl_trans_dump_data
 	data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXCMD);
 	txcmd = (void *)data->data;
 	spin_lock_bh(&cmdq->lock);
-	ptr = cmdq->q.write_ptr;
-	for (i = 0; i < cmdq->q.n_window; i++) {
-		u8 idx = get_cmd_index(&cmdq->q, ptr);
+	ptr = cmdq->write_ptr;
+	for (i = 0; i < cmdq->n_window; i++) {
+		u8 idx = get_cmd_index(cmdq, ptr);
 		u32 caplen, cmdlen;
 
-		cmdlen = iwl_trans_pcie_get_cmdlen(&cmdq->tfds[ptr]);
+		cmdlen = iwl_trans_pcie_get_cmdlen(trans, cmdq->tfds +
+						   trans_pcie->tfd_size * ptr);
 		caplen = min_t(u32, TFD_MAX_PAYLOAD_SIZE, cmdlen);
 
 		if (cmdlen) {
@@ -2788,6 +2864,8 @@ static const struct iwl_trans_ops trans_ops_pcie = {
 	.txq_disable = iwl_trans_pcie_txq_disable,
 	.txq_enable = iwl_trans_pcie_txq_enable,
 
+	.get_txq_byte_table = iwl_trans_pcie_get_txq_byte_table,
+
 	.txq_set_shared_mode = iwl_trans_pcie_txq_set_shared_mode,
 
 	.wait_tx_queue_empty = iwl_trans_pcie_wait_txq_empty,
@@ -2821,13 +2899,15 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 	struct iwl_trans *trans;
 	int ret, addr_size;
 
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return ERR_PTR(ret);
+
 	trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie),
 				&pdev->dev, cfg, &trans_ops_pcie, 0);
 	if (!trans)
 		return ERR_PTR(-ENOMEM);
 
-	trans->max_skb_frags = IWL_PCIE_MAX_FRAGS;
-
 	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
 	trans_pcie->trans = trans;
@@ -2841,9 +2921,6 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		goto out_no_pci;
 	}
 
-	ret = pci_enable_device(pdev);
-	if (ret)
-		goto out_no_pci;
 
 	if (!cfg->base_params->pcie_l1_allowed) {
 		/*
@@ -2861,6 +2938,16 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 	else
 		addr_size = 36;
 
+	if (cfg->use_tfh) {
+		trans_pcie->max_tbs = IWL_TFH_NUM_TBS;
+		trans_pcie->tfd_size = sizeof(struct iwl_tfh_tfd);
+
+	} else {
+		trans_pcie->max_tbs = IWL_NUM_OF_TBS;
+		trans_pcie->tfd_size = sizeof(struct iwl_tfd);
+	}
+	trans->max_skb_frags = IWL_PCIE_MAX_FRAGS(trans_pcie);
+
 	pci_set_master(pdev);
 
 	ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(addr_size));
@@ -2875,21 +2962,21 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		/* both attempts failed: */
 		if (ret) {
 			dev_err(&pdev->dev, "No suitable DMA available\n");
-			goto out_pci_disable_device;
+			goto out_no_pci;
 		}
 	}
 
-	ret = pci_request_regions(pdev, DRV_NAME);
+	ret = pcim_iomap_regions_request_all(pdev, BIT(0), DRV_NAME);
 	if (ret) {
-		dev_err(&pdev->dev, "pci_request_regions failed\n");
-		goto out_pci_disable_device;
+		dev_err(&pdev->dev, "pcim_iomap_regions_request_all failed\n");
+		goto out_no_pci;
 	}
 
-	trans_pcie->hw_base = pci_ioremap_bar(pdev, 0);
+	trans_pcie->hw_base = pcim_iomap_table(pdev)[0];
 	if (!trans_pcie->hw_base) {
-		dev_err(&pdev->dev, "pci_ioremap_bar failed\n");
+		dev_err(&pdev->dev, "pcim_iomap_table failed\n");
 		ret = -ENODEV;
-		goto out_pci_release_regions;
+		goto out_no_pci;
 	}
 
 	/* We disable the RETRY_TIMEOUT register (0x41) to keep
@@ -2916,7 +3003,7 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		ret = iwl_pcie_prepare_card_hw(trans);
 		if (ret) {
 			IWL_WARN(trans, "Exit HW not ready\n");
-			goto out_pci_disable_msi;
+			goto out_no_pci;
 		}
 
 		/*
@@ -2933,7 +3020,7 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 				   25000);
 		if (ret < 0) {
 			IWL_DEBUG_INFO(trans, "Failed to wake up the nic\n");
-			goto out_pci_disable_msi;
+			goto out_no_pci;
 		}
 
 		if (iwl_trans_grab_nic_access(trans, &flags)) {
@@ -2965,15 +3052,16 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 
 	if (trans_pcie->msix_enabled) {
 		if (iwl_pcie_init_msix_handler(pdev, trans_pcie))
-			goto out_pci_release_regions;
+			goto out_no_pci;
 	 } else {
 		ret = iwl_pcie_alloc_ict(trans);
 		if (ret)
-			goto out_pci_disable_msi;
+			goto out_no_pci;
 
-		ret = request_threaded_irq(pdev->irq, iwl_pcie_isr,
-					   iwl_pcie_irq_handler,
-					   IRQF_SHARED, DRV_NAME, trans);
+		ret = devm_request_threaded_irq(&pdev->dev, pdev->irq,
+						iwl_pcie_isr,
+						iwl_pcie_irq_handler,
+						IRQF_SHARED, DRV_NAME, trans);
 		if (ret) {
 			IWL_ERR(trans, "Error allocating IRQ %d\n", pdev->irq);
 			goto out_free_ict;
@@ -2991,12 +3079,6 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 
 out_free_ict:
 	iwl_pcie_free_ict(trans);
-out_pci_disable_msi:
-	pci_disable_msi(pdev);
-out_pci_release_regions:
-	pci_release_regions(pdev);
-out_pci_disable_device:
-	pci_disable_device(pdev);
 out_no_pci:
 	free_percpu(trans_pcie->tso_hdr_page);
 	iwl_trans_free(trans);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
index 18650dccdb58..e9a278b60dfd 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
@@ -71,7 +71,7 @@
  *
  ***************************************************/
 
-static int iwl_queue_space(const struct iwl_queue *q)
+static int iwl_queue_space(const struct iwl_txq *q)
 {
 	unsigned int max;
 	unsigned int used;
@@ -102,7 +102,7 @@ static int iwl_queue_space(const struct iwl_queue *q)
 /*
  * iwl_queue_init - Initialize queue's high/low-water and read/write indexes
  */
-static int iwl_queue_init(struct iwl_queue *q, int slots_num, u32 id)
+static int iwl_queue_init(struct iwl_txq *q, int slots_num, u32 id)
 {
 	q->n_window = slots_num;
 	q->id = id;
@@ -158,13 +158,13 @@ static void iwl_pcie_txq_stuck_timer(unsigned long data)
 
 	spin_lock(&txq->lock);
 	/* check if triggered erroneously */
-	if (txq->q.read_ptr == txq->q.write_ptr) {
+	if (txq->read_ptr == txq->write_ptr) {
 		spin_unlock(&txq->lock);
 		return;
 	}
 	spin_unlock(&txq->lock);
 
-	IWL_ERR(trans, "Queue %d stuck for %u ms.\n", txq->q.id,
+	IWL_ERR(trans, "Queue %d stuck for %u ms.\n", txq->id,
 		jiffies_to_msecs(txq->wd_timeout));
 
 	iwl_trans_pcie_log_scd_error(trans, txq);
@@ -176,22 +176,21 @@ static void iwl_pcie_txq_stuck_timer(unsigned long data)
  * iwl_pcie_txq_update_byte_cnt_tbl - Set up entry in Tx byte-count array
  */
 static void iwl_pcie_txq_update_byte_cnt_tbl(struct iwl_trans *trans,
-					     struct iwl_txq *txq, u16 byte_cnt)
+					     struct iwl_txq *txq, u16 byte_cnt,
+					     int num_tbs)
 {
 	struct iwlagn_scd_bc_tbl *scd_bc_tbl;
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	int write_ptr = txq->q.write_ptr;
-	int txq_id = txq->q.id;
+	int write_ptr = txq->write_ptr;
+	int txq_id = txq->id;
 	u8 sec_ctl = 0;
-	u8 sta_id = 0;
 	u16 len = byte_cnt + IWL_TX_CRC_SIZE + IWL_TX_DELIMITER_SIZE;
 	__le16 bc_ent;
 	struct iwl_tx_cmd *tx_cmd =
-		(void *) txq->entries[txq->q.write_ptr].cmd->payload;
+		(void *)txq->entries[txq->write_ptr].cmd->payload;
 
 	scd_bc_tbl = trans_pcie->scd_bc_tbls.addr;
 
-	sta_id = tx_cmd->sta_id;
 	sec_ctl = tx_cmd->sec_ctl;
 
 	switch (sec_ctl & TX_CMD_SEC_MSK) {
@@ -205,14 +204,32 @@ static void iwl_pcie_txq_update_byte_cnt_tbl(struct iwl_trans *trans,
 		len += IEEE80211_WEP_IV_LEN + IEEE80211_WEP_ICV_LEN;
 		break;
 	}
-
 	if (trans_pcie->bc_table_dword)
 		len = DIV_ROUND_UP(len, 4);
 
 	if (WARN_ON(len > 0xFFF || write_ptr >= TFD_QUEUE_SIZE_MAX))
 		return;
 
-	bc_ent = cpu_to_le16(len | (sta_id << 12));
+	if (trans->cfg->use_tfh) {
+		u8 filled_tfd_size = offsetof(struct iwl_tfh_tfd, tbs) +
+				     num_tbs * sizeof(struct iwl_tfh_tb);
+		/*
+		 * filled_tfd_size contains the number of filled bytes in the
+		 * TFD.
+		 * Dividing it by 64 will give the number of chunks to fetch
+		 * to SRAM- 0 for one chunk, 1 for 2 and so on.
+		 * If, for example, TFD contains only 3 TBs then 32 bytes
+		 * of the TFD are used, and only one chunk of 64 bytes should
+		 * be fetched
+		 */
+		u8 num_fetch_chunks = DIV_ROUND_UP(filled_tfd_size, 64) - 1;
+
+		bc_ent = cpu_to_le16(len | (num_fetch_chunks << 12));
+	} else {
+		u8 sta_id = tx_cmd->sta_id;
+
+		bc_ent = cpu_to_le16(len | (sta_id << 12));
+	}
 
 	scd_bc_tbl[txq_id].tfd_offset[write_ptr] = bc_ent;
 
@@ -227,12 +244,12 @@ static void iwl_pcie_txq_inval_byte_cnt_tbl(struct iwl_trans *trans,
 	struct iwl_trans_pcie *trans_pcie =
 		IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwlagn_scd_bc_tbl *scd_bc_tbl = trans_pcie->scd_bc_tbls.addr;
-	int txq_id = txq->q.id;
-	int read_ptr = txq->q.read_ptr;
+	int txq_id = txq->id;
+	int read_ptr = txq->read_ptr;
 	u8 sta_id = 0;
 	__le16 bc_ent;
 	struct iwl_tx_cmd *tx_cmd =
-		(void *)txq->entries[txq->q.read_ptr].cmd->payload;
+		(void *)txq->entries[read_ptr].cmd->payload;
 
 	WARN_ON(read_ptr >= TFD_QUEUE_SIZE_MAX);
 
@@ -240,6 +257,7 @@ static void iwl_pcie_txq_inval_byte_cnt_tbl(struct iwl_trans *trans,
 		sta_id = tx_cmd->sta_id;
 
 	bc_ent = cpu_to_le16(1 | (sta_id << 12));
+
 	scd_bc_tbl[txq_id].tfd_offset[read_ptr] = bc_ent;
 
 	if (read_ptr < TFD_QUEUE_SIZE_BC_DUP)
@@ -255,7 +273,7 @@ static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans,
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	u32 reg = 0;
-	int txq_id = txq->q.id;
+	int txq_id = txq->id;
 
 	lockdep_assert_held(&txq->lock);
 
@@ -289,10 +307,10 @@ static void iwl_pcie_txq_inc_wr_ptr(struct iwl_trans *trans,
 	 * if not in power-save mode, uCode will never sleep when we're
 	 * trying to tx (during RFKILL, we're not trying to tx).
 	 */
-	IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq_id, txq->q.write_ptr);
+	IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq_id, txq->write_ptr);
 	if (!txq->block)
 		iwl_write32(trans, HBUS_TARG_WRPTR,
-			    txq->q.write_ptr | (txq_id << 8));
+			    txq->write_ptr | (txq_id << 8));
 }
 
 void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans)
@@ -312,49 +330,93 @@ void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans)
 	}
 }
 
-static inline dma_addr_t iwl_pcie_tfd_tb_get_addr(struct iwl_tfd *tfd, u8 idx)
+static inline void *iwl_pcie_get_tfd(struct iwl_trans_pcie *trans_pcie,
+				     struct iwl_txq *txq, int idx)
+{
+	return txq->tfds + trans_pcie->tfd_size * idx;
+}
+
+static inline dma_addr_t iwl_pcie_tfd_tb_get_addr(struct iwl_trans *trans,
+						  void *_tfd, u8 idx)
 {
-	struct iwl_tfd_tb *tb = &tfd->tbs[idx];
 
-	dma_addr_t addr = get_unaligned_le32(&tb->lo);
-	if (sizeof(dma_addr_t) > sizeof(u32))
-		addr |=
-		((dma_addr_t)(le16_to_cpu(tb->hi_n_len) & 0xF) << 16) << 16;
+	if (trans->cfg->use_tfh) {
+		struct iwl_tfh_tfd *tfd = _tfd;
+		struct iwl_tfh_tb *tb = &tfd->tbs[idx];
+
+		return (dma_addr_t)(le64_to_cpu(tb->addr));
+	} else {
+		struct iwl_tfd *tfd = _tfd;
+		struct iwl_tfd_tb *tb = &tfd->tbs[idx];
+		dma_addr_t addr = get_unaligned_le32(&tb->lo);
+		dma_addr_t hi_len;
+
+		if (sizeof(dma_addr_t) <= sizeof(u32))
+			return addr;
 
-	return addr;
+		hi_len = le16_to_cpu(tb->hi_n_len) & 0xF;
+
+		/*
+		 * shift by 16 twice to avoid warnings on 32-bit
+		 * (where this code never runs anyway due to the
+		 * if statement above)
+		 */
+		return addr | ((hi_len << 16) << 16);
+	}
 }
 
-static inline void iwl_pcie_tfd_set_tb(struct iwl_tfd *tfd, u8 idx,
-				       dma_addr_t addr, u16 len)
+static inline void iwl_pcie_tfd_set_tb(struct iwl_trans *trans, void *tfd,
+				       u8 idx, dma_addr_t addr, u16 len)
 {
-	struct iwl_tfd_tb *tb = &tfd->tbs[idx];
-	u16 hi_n_len = len << 4;
+	if (trans->cfg->use_tfh) {
+		struct iwl_tfh_tfd *tfd_fh = (void *)tfd;
+		struct iwl_tfh_tb *tb = &tfd_fh->tbs[idx];
 
-	put_unaligned_le32(addr, &tb->lo);
-	if (sizeof(dma_addr_t) > sizeof(u32))
-		hi_n_len |= ((addr >> 16) >> 16) & 0xF;
+		put_unaligned_le64(addr, &tb->addr);
+		tb->tb_len = cpu_to_le16(len);
 
-	tb->hi_n_len = cpu_to_le16(hi_n_len);
+		tfd_fh->num_tbs = cpu_to_le16(idx + 1);
+	} else {
+		struct iwl_tfd *tfd_fh = (void *)tfd;
+		struct iwl_tfd_tb *tb = &tfd_fh->tbs[idx];
 
-	tfd->num_tbs = idx + 1;
+		u16 hi_n_len = len << 4;
+
+		put_unaligned_le32(addr, &tb->lo);
+		if (sizeof(dma_addr_t) > sizeof(u32))
+			hi_n_len |= ((addr >> 16) >> 16) & 0xF;
+
+		tb->hi_n_len = cpu_to_le16(hi_n_len);
+
+		tfd_fh->num_tbs = idx + 1;
+	}
 }
 
-static inline u8 iwl_pcie_tfd_get_num_tbs(struct iwl_tfd *tfd)
+static inline u8 iwl_pcie_tfd_get_num_tbs(struct iwl_trans *trans, void *_tfd)
 {
-	return tfd->num_tbs & 0x1f;
+	if (trans->cfg->use_tfh) {
+		struct iwl_tfh_tfd *tfd = _tfd;
+
+		return le16_to_cpu(tfd->num_tbs) & 0x1f;
+	} else {
+		struct iwl_tfd *tfd = _tfd;
+
+		return tfd->num_tbs & 0x1f;
+	}
 }
 
 static void iwl_pcie_tfd_unmap(struct iwl_trans *trans,
 			       struct iwl_cmd_meta *meta,
-			       struct iwl_tfd *tfd)
+			       struct iwl_txq *txq, int index)
 {
-	int i;
-	int num_tbs;
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	int i, num_tbs;
+	void *tfd = iwl_pcie_get_tfd(trans_pcie, txq, index);
 
 	/* Sanity check on number of chunks */
-	num_tbs = iwl_pcie_tfd_get_num_tbs(tfd);
+	num_tbs = iwl_pcie_tfd_get_num_tbs(trans, tfd);
 
-	if (num_tbs >= IWL_NUM_OF_TBS) {
+	if (num_tbs >= trans_pcie->max_tbs) {
 		IWL_ERR(trans, "Too many chunks: %i\n", num_tbs);
 		/* @todo issue fatal error, it is quite serious situation */
 		return;
@@ -363,18 +425,30 @@ static void iwl_pcie_tfd_unmap(struct iwl_trans *trans,
 	/* first TB is never freed - it's the bidirectional DMA data */
 
 	for (i = 1; i < num_tbs; i++) {
-		if (meta->flags & BIT(i + CMD_TB_BITMAP_POS))
+		if (meta->tbs & BIT(i))
 			dma_unmap_page(trans->dev,
-				       iwl_pcie_tfd_tb_get_addr(tfd, i),
-				       iwl_pcie_tfd_tb_get_len(tfd, i),
+				       iwl_pcie_tfd_tb_get_addr(trans, tfd, i),
+				       iwl_pcie_tfd_tb_get_len(trans, tfd, i),
 				       DMA_TO_DEVICE);
 		else
 			dma_unmap_single(trans->dev,
-					 iwl_pcie_tfd_tb_get_addr(tfd, i),
-					 iwl_pcie_tfd_tb_get_len(tfd, i),
+					 iwl_pcie_tfd_tb_get_addr(trans, tfd,
+								  i),
+					 iwl_pcie_tfd_tb_get_len(trans, tfd,
+								 i),
 					 DMA_TO_DEVICE);
 	}
-	tfd->num_tbs = 0;
+
+	if (trans->cfg->use_tfh) {
+		struct iwl_tfh_tfd *tfd_fh = (void *)tfd;
+
+		tfd_fh->num_tbs = 0;
+	} else {
+		struct iwl_tfd *tfd_fh = (void *)tfd;
+
+		tfd_fh->num_tbs = 0;
+	}
+
 }
 
 /*
@@ -388,20 +462,18 @@ static void iwl_pcie_tfd_unmap(struct iwl_trans *trans,
  */
 static void iwl_pcie_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq)
 {
-	struct iwl_tfd *tfd_tmp = txq->tfds;
-
 	/* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and
 	 * idx is bounded by n_window
 	 */
-	int rd_ptr = txq->q.read_ptr;
-	int idx = get_cmd_index(&txq->q, rd_ptr);
+	int rd_ptr = txq->read_ptr;
+	int idx = get_cmd_index(txq, rd_ptr);
 
 	lockdep_assert_held(&txq->lock);
 
 	/* We have only q->n_window txq->entries, but we use
 	 * TFD_QUEUE_SIZE_MAX tfds
 	 */
-	iwl_pcie_tfd_unmap(trans, &txq->entries[idx].meta, &tfd_tmp[rd_ptr]);
+	iwl_pcie_tfd_unmap(trans, &txq->entries[idx].meta, txq, rd_ptr);
 
 	/* free SKB */
 	if (txq->entries) {
@@ -423,23 +495,21 @@ static void iwl_pcie_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq)
 static int iwl_pcie_txq_build_tfd(struct iwl_trans *trans, struct iwl_txq *txq,
 				  dma_addr_t addr, u16 len, bool reset)
 {
-	struct iwl_queue *q;
-	struct iwl_tfd *tfd, *tfd_tmp;
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	void *tfd;
 	u32 num_tbs;
 
-	q = &txq->q;
-	tfd_tmp = txq->tfds;
-	tfd = &tfd_tmp[q->write_ptr];
+	tfd = txq->tfds + trans_pcie->tfd_size * txq->write_ptr;
 
 	if (reset)
-		memset(tfd, 0, sizeof(*tfd));
+		memset(tfd, 0, trans_pcie->tfd_size);
 
-	num_tbs = iwl_pcie_tfd_get_num_tbs(tfd);
+	num_tbs = iwl_pcie_tfd_get_num_tbs(trans, tfd);
 
-	/* Each TFD can point to a maximum 20 Tx buffers */
-	if (num_tbs >= IWL_NUM_OF_TBS) {
+	/* Each TFD can point to a maximum max_tbs Tx buffers */
+	if (num_tbs >= trans_pcie->max_tbs) {
 		IWL_ERR(trans, "Error can not send more than %d chunks\n",
-			IWL_NUM_OF_TBS);
+			trans_pcie->max_tbs);
 		return -EINVAL;
 	}
 
@@ -447,7 +517,7 @@ static int iwl_pcie_txq_build_tfd(struct iwl_trans *trans, struct iwl_txq *txq,
 		 "Unaligned address = %llx\n", (unsigned long long)addr))
 		return -EINVAL;
 
-	iwl_pcie_tfd_set_tb(tfd, num_tbs, addr, len);
+	iwl_pcie_tfd_set_tb(trans, tfd, num_tbs, addr, len);
 
 	return num_tbs;
 }
@@ -457,7 +527,7 @@ static int iwl_pcie_txq_alloc(struct iwl_trans *trans,
 			       u32 txq_id)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	size_t tfd_sz = sizeof(struct iwl_tfd) * TFD_QUEUE_SIZE_MAX;
+	size_t tfd_sz = trans_pcie->tfd_size * TFD_QUEUE_SIZE_MAX;
 	size_t tb0_buf_sz;
 	int i;
 
@@ -468,7 +538,7 @@ static int iwl_pcie_txq_alloc(struct iwl_trans *trans,
 		    (unsigned long)txq);
 	txq->trans_pcie = trans_pcie;
 
-	txq->q.n_window = slots_num;
+	txq->n_window = slots_num;
 
 	txq->entries = kcalloc(slots_num,
 			       sizeof(struct iwl_pcie_txq_entry),
@@ -489,7 +559,7 @@ static int iwl_pcie_txq_alloc(struct iwl_trans *trans,
 	/* Circular buffer of transmit frame descriptors (TFDs),
 	 * shared with device */
 	txq->tfds = dma_alloc_coherent(trans->dev, tfd_sz,
-				       &txq->q.dma_addr, GFP_KERNEL);
+				       &txq->dma_addr, GFP_KERNEL);
 	if (!txq->tfds)
 		goto error;
 
@@ -503,11 +573,11 @@ static int iwl_pcie_txq_alloc(struct iwl_trans *trans,
 	if (!txq->first_tb_bufs)
 		goto err_free_tfds;
 
-	txq->q.id = txq_id;
+	txq->id = txq_id;
 
 	return 0;
 err_free_tfds:
-	dma_free_coherent(trans->dev, tfd_sz, txq->tfds, txq->q.dma_addr);
+	dma_free_coherent(trans->dev, tfd_sz, txq->tfds, txq->dma_addr);
 error:
 	if (txq->entries && txq_id == trans_pcie->cmd_queue)
 		for (i = 0; i < slots_num; i++)
@@ -531,7 +601,7 @@ static int iwl_pcie_txq_init(struct iwl_trans *trans, struct iwl_txq *txq,
 	BUILD_BUG_ON(TFD_QUEUE_SIZE_MAX & (TFD_QUEUE_SIZE_MAX - 1));
 
 	/* Initialize queue's high/low-water marks, and head/tail indexes */
-	ret = iwl_queue_init(&txq->q, slots_num, txq_id);
+	ret = iwl_queue_init(txq, slots_num, txq_id);
 	if (ret)
 		return ret;
 
@@ -545,10 +615,10 @@ static int iwl_pcie_txq_init(struct iwl_trans *trans, struct iwl_txq *txq,
 	if (trans->cfg->use_tfh)
 		iwl_write_direct64(trans,
 				   FH_MEM_CBBC_QUEUE(trans, txq_id),
-				   txq->q.dma_addr);
+				   txq->dma_addr);
 	else
 		iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(trans, txq_id),
-				   txq->q.dma_addr >> 8);
+				   txq->dma_addr >> 8);
 
 	return 0;
 }
@@ -595,15 +665,14 @@ static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_txq *txq = &trans_pcie->txq[txq_id];
-	struct iwl_queue *q = &txq->q;
 
 	spin_lock_bh(&txq->lock);
-	while (q->write_ptr != q->read_ptr) {
+	while (txq->write_ptr != txq->read_ptr) {
 		IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n",
-				   txq_id, q->read_ptr);
+				   txq_id, txq->read_ptr);
 
 		if (txq_id != trans_pcie->cmd_queue) {
-			struct sk_buff *skb = txq->entries[q->read_ptr].skb;
+			struct sk_buff *skb = txq->entries[txq->read_ptr].skb;
 
 			if (WARN_ON_ONCE(!skb))
 				continue;
@@ -611,15 +680,15 @@ static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id)
 			iwl_pcie_free_tso_page(trans_pcie, skb);
 		}
 		iwl_pcie_txq_free_tfd(trans, txq);
-		q->read_ptr = iwl_queue_inc_wrap(q->read_ptr);
+		txq->read_ptr = iwl_queue_inc_wrap(txq->read_ptr);
 
-		if (q->read_ptr == q->write_ptr) {
+		if (txq->read_ptr == txq->write_ptr) {
 			unsigned long flags;
 
 			spin_lock_irqsave(&trans_pcie->reg_lock, flags);
 			if (txq_id != trans_pcie->cmd_queue) {
 				IWL_DEBUG_RPM(trans, "Q %d - last tx freed\n",
-					      q->id);
+					      txq->id);
 				iwl_trans_unref(trans);
 			} else {
 				iwl_pcie_clear_cmd_in_flight(trans);
@@ -663,7 +732,7 @@ static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id)
 
 	/* De-alloc array of command/tx buffers */
 	if (txq_id == trans_pcie->cmd_queue)
-		for (i = 0; i < txq->q.n_window; i++) {
+		for (i = 0; i < txq->n_window; i++) {
 			kzfree(txq->entries[i].cmd);
 			kzfree(txq->entries[i].free_buf);
 		}
@@ -671,13 +740,13 @@ static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id)
 	/* De-alloc circular buffer of TFDs */
 	if (txq->tfds) {
 		dma_free_coherent(dev,
-				  sizeof(struct iwl_tfd) * TFD_QUEUE_SIZE_MAX,
-				  txq->tfds, txq->q.dma_addr);
-		txq->q.dma_addr = 0;
+				  trans_pcie->tfd_size * TFD_QUEUE_SIZE_MAX,
+				  txq->tfds, txq->dma_addr);
+		txq->dma_addr = 0;
 		txq->tfds = NULL;
 
 		dma_free_coherent(dev,
-				  sizeof(*txq->first_tb_bufs) * txq->q.n_window,
+				  sizeof(*txq->first_tb_bufs) * txq->n_window,
 				  txq->first_tb_bufs, txq->first_tb_dma);
 	}
 
@@ -703,6 +772,9 @@ void iwl_pcie_tx_start(struct iwl_trans *trans, u32 scd_base_addr)
 	memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped));
 	memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used));
 
+	if (trans->cfg->use_tfh)
+		return;
+
 	trans_pcie->scd_base_addr =
 		iwl_read_prph(trans, SCD_SRAM_BASE_ADDR);
 
@@ -758,14 +830,14 @@ void iwl_trans_pcie_tx_reset(struct iwl_trans *trans)
 		if (trans->cfg->use_tfh)
 			iwl_write_direct64(trans,
 					   FH_MEM_CBBC_QUEUE(trans, txq_id),
-					   txq->q.dma_addr);
+					   txq->dma_addr);
 		else
 			iwl_write_direct32(trans,
 					   FH_MEM_CBBC_QUEUE(trans, txq_id),
-					   txq->q.dma_addr >> 8);
+					   txq->dma_addr >> 8);
 		iwl_pcie_txq_unmap(trans, txq_id);
-		txq->q.read_ptr = 0;
-		txq->q.write_ptr = 0;
+		txq->read_ptr = 0;
+		txq->write_ptr = 0;
 	}
 
 	/* Tell NIC where to find the "keep warm" buffer */
@@ -970,11 +1042,13 @@ int iwl_pcie_tx_init(struct iwl_trans *trans)
 		}
 	}
 
-	if (trans->cfg->use_tfh)
+	if (trans->cfg->use_tfh) {
 		iwl_write_direct32(trans, TFH_TRANSFER_MODE,
 				   TFH_TRANSFER_MAX_PENDING_REQ |
 				   TFH_CHUNK_SIZE_128 |
 				   TFH_CHUNK_SPLIT_MODE);
+		return 0;
+	}
 
 	iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE);
 	if (trans->cfg->base_params->num_of_queues > 20)
@@ -1007,7 +1081,7 @@ static inline void iwl_pcie_txq_progress(struct iwl_txq *txq)
 	 * if empty delete timer, otherwise move timer forward
 	 * since we're making progress on this queue
 	 */
-	if (txq->q.read_ptr == txq->q.write_ptr)
+	if (txq->read_ptr == txq->write_ptr)
 		del_timer(&txq->stuck_timer);
 	else
 		mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);
@@ -1020,7 +1094,6 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_txq *txq = &trans_pcie->txq[txq_id];
 	int tfd_num = ssn & (TFD_QUEUE_SIZE_MAX - 1);
-	struct iwl_queue *q = &txq->q;
 	int last_to_free;
 
 	/* This function is not meant to release cmd queue*/
@@ -1035,21 +1108,21 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 		goto out;
 	}
 
-	if (txq->q.read_ptr == tfd_num)
+	if (txq->read_ptr == tfd_num)
 		goto out;
 
 	IWL_DEBUG_TX_REPLY(trans, "[Q %d] %d -> %d (%d)\n",
-			   txq_id, txq->q.read_ptr, tfd_num, ssn);
+			   txq_id, txq->read_ptr, tfd_num, ssn);
 
 	/*Since we free until index _not_ inclusive, the one before index is
 	 * the last we will free. This one must be used */
 	last_to_free = iwl_queue_dec_wrap(tfd_num);
 
-	if (!iwl_queue_used(q, last_to_free)) {
+	if (!iwl_queue_used(txq, last_to_free)) {
 		IWL_ERR(trans,
 			"%s: Read index for DMA queue txq id (%d), last_to_free %d is out of range [0-%d] %d %d.\n",
 			__func__, txq_id, last_to_free, TFD_QUEUE_SIZE_MAX,
-			q->write_ptr, q->read_ptr);
+			txq->write_ptr, txq->read_ptr);
 		goto out;
 	}
 
@@ -1057,9 +1130,9 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 		goto out;
 
 	for (;
-	     q->read_ptr != tfd_num;
-	     q->read_ptr = iwl_queue_inc_wrap(q->read_ptr)) {
-		struct sk_buff *skb = txq->entries[txq->q.read_ptr].skb;
+	     txq->read_ptr != tfd_num;
+	     txq->read_ptr = iwl_queue_inc_wrap(txq->read_ptr)) {
+		struct sk_buff *skb = txq->entries[txq->read_ptr].skb;
 
 		if (WARN_ON_ONCE(!skb))
 			continue;
@@ -1068,16 +1141,17 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 
 		__skb_queue_tail(skbs, skb);
 
-		txq->entries[txq->q.read_ptr].skb = NULL;
+		txq->entries[txq->read_ptr].skb = NULL;
 
-		iwl_pcie_txq_inval_byte_cnt_tbl(trans, txq);
+		if (!trans->cfg->use_tfh)
+			iwl_pcie_txq_inval_byte_cnt_tbl(trans, txq);
 
 		iwl_pcie_txq_free_tfd(trans, txq);
 	}
 
 	iwl_pcie_txq_progress(txq);
 
-	if (iwl_queue_space(&txq->q) > txq->q.low_mark &&
+	if (iwl_queue_space(txq) > txq->low_mark &&
 	    test_bit(txq_id, trans_pcie->queue_stopped)) {
 		struct sk_buff_head overflow_skbs;
 
@@ -1109,12 +1183,12 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 		}
 		spin_lock_bh(&txq->lock);
 
-		if (iwl_queue_space(&txq->q) > txq->q.low_mark)
+		if (iwl_queue_space(txq) > txq->low_mark)
 			iwl_wake_queue(trans, txq);
 	}
 
-	if (q->read_ptr == q->write_ptr) {
-		IWL_DEBUG_RPM(trans, "Q %d - last tx reclaimed\n", q->id);
+	if (txq->read_ptr == txq->write_ptr) {
+		IWL_DEBUG_RPM(trans, "Q %d - last tx reclaimed\n", txq->id);
 		iwl_trans_unref(trans);
 	}
 
@@ -1176,31 +1250,30 @@ static void iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_txq *txq = &trans_pcie->txq[txq_id];
-	struct iwl_queue *q = &txq->q;
 	unsigned long flags;
 	int nfreed = 0;
 
 	lockdep_assert_held(&txq->lock);
 
-	if ((idx >= TFD_QUEUE_SIZE_MAX) || (!iwl_queue_used(q, idx))) {
+	if ((idx >= TFD_QUEUE_SIZE_MAX) || (!iwl_queue_used(txq, idx))) {
 		IWL_ERR(trans,
 			"%s: Read index for DMA queue txq id (%d), index %d is out of range [0-%d] %d %d.\n",
 			__func__, txq_id, idx, TFD_QUEUE_SIZE_MAX,
-			q->write_ptr, q->read_ptr);
+			txq->write_ptr, txq->read_ptr);
 		return;
 	}
 
-	for (idx = iwl_queue_inc_wrap(idx); q->read_ptr != idx;
-	     q->read_ptr = iwl_queue_inc_wrap(q->read_ptr)) {
+	for (idx = iwl_queue_inc_wrap(idx); txq->read_ptr != idx;
+	     txq->read_ptr = iwl_queue_inc_wrap(txq->read_ptr)) {
 
 		if (nfreed++ > 0) {
 			IWL_ERR(trans, "HCMD skipped: index (%d) %d %d\n",
-				idx, q->write_ptr, q->read_ptr);
+				idx, txq->write_ptr, txq->read_ptr);
 			iwl_force_nmi(trans);
 		}
 	}
 
-	if (q->read_ptr == q->write_ptr) {
+	if (txq->read_ptr == txq->write_ptr) {
 		spin_lock_irqsave(&trans_pcie->reg_lock, flags);
 		iwl_pcie_clear_cmd_in_flight(trans);
 		spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
@@ -1249,6 +1322,9 @@ void iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn,
 	if (test_and_set_bit(txq_id, trans_pcie->queue_used))
 		WARN_ONCE(1, "queue %d already used - expect issues", txq_id);
 
+	if (cfg && trans->cfg->use_tfh)
+		WARN_ONCE(1, "Expected no calls to SCD configuration");
+
 	txq->wd_timeout = msecs_to_jiffies(wdg_timeout);
 
 	if (cfg) {
@@ -1283,14 +1359,14 @@ void iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn,
 			 */
 			iwl_scd_txq_disable_agg(trans, txq_id);
 
-			ssn = txq->q.read_ptr;
+			ssn = txq->read_ptr;
 		}
 	}
 
 	/* Place first TFD at index corresponding to start sequence number.
 	 * Assumes that ssn_idx is valid (!= 0xFFF) */
-	txq->q.read_ptr = (ssn & 0xff);
-	txq->q.write_ptr = (ssn & 0xff);
+	txq->read_ptr = (ssn & 0xff);
+	txq->write_ptr = (ssn & 0xff);
 	iwl_write_direct32(trans, HBUS_TARG_WRPTR,
 			   (ssn & 0xff) | (txq_id << 8));
 
@@ -1343,6 +1419,14 @@ void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id,
 	txq->ampdu = !shared_mode;
 }
 
+dma_addr_t iwl_trans_pcie_get_txq_byte_table(struct iwl_trans *trans, int txq)
+{
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
+	return trans_pcie->scd_bc_tbls.dma +
+	       txq * sizeof(struct iwlagn_scd_bc_tbl);
+}
+
 void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id,
 				bool configure_scd)
 {
@@ -1366,6 +1450,9 @@ void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id,
 		return;
 	}
 
+	if (configure_scd && trans->cfg->use_tfh)
+		WARN_ONCE(1, "Expected no calls to SCD configuration");
+
 	if (configure_scd) {
 		iwl_scd_txq_set_inactive(trans, txq_id);
 
@@ -1395,7 +1482,6 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue];
-	struct iwl_queue *q = &txq->q;
 	struct iwl_device_cmd *out_cmd;
 	struct iwl_cmd_meta *out_meta;
 	unsigned long flags;
@@ -1410,7 +1496,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 	const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD];
 	u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD];
 
-	if (WARN(!trans_pcie->wide_cmd_header &&
+	if (WARN(!trans->wide_cmd_header &&
 		 group_id > IWL_ALWAYS_LONG_GROUP,
 		 "unsupported wide command %#x\n", cmd->id))
 		return -EINVAL;
@@ -1494,7 +1580,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 
 	spin_lock_bh(&txq->lock);
 
-	if (iwl_queue_space(q) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) {
+	if (iwl_queue_space(txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) {
 		spin_unlock_bh(&txq->lock);
 
 		IWL_ERR(trans, "No space in command queue\n");
@@ -1503,7 +1589,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 		goto free_dup_buf;
 	}
 
-	idx = get_cmd_index(q, q->write_ptr);
+	idx = get_cmd_index(txq, txq->write_ptr);
 	out_cmd = txq->entries[idx].cmd;
 	out_meta = &txq->entries[idx].meta;
 
@@ -1522,7 +1608,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 		out_cmd->hdr_wide.reserved = 0;
 		out_cmd->hdr_wide.sequence =
 			cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) |
-						 INDEX_TO_SEQ(q->write_ptr));
+						 INDEX_TO_SEQ(txq->write_ptr));
 
 		cmd_pos = sizeof(struct iwl_cmd_header_wide);
 		copy_size = sizeof(struct iwl_cmd_header_wide);
@@ -1530,7 +1616,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 		out_cmd->hdr.cmd = iwl_cmd_opcode(cmd->id);
 		out_cmd->hdr.sequence =
 			cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) |
-						 INDEX_TO_SEQ(q->write_ptr));
+						 INDEX_TO_SEQ(txq->write_ptr));
 		out_cmd->hdr.group_id = 0;
 
 		cmd_pos = sizeof(struct iwl_cmd_header);
@@ -1580,7 +1666,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 		     iwl_get_cmd_string(trans, cmd->id),
 		     group_id, out_cmd->hdr.cmd,
 		     le16_to_cpu(out_cmd->hdr.sequence),
-		     cmd_size, q->write_ptr, idx, trans_pcie->cmd_queue);
+		     cmd_size, txq->write_ptr, idx, trans_pcie->cmd_queue);
 
 	/* start the TFD with the minimum copy bytes */
 	tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE);
@@ -1596,8 +1682,8 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 					   copy_size - tb0_size,
 					   DMA_TO_DEVICE);
 		if (dma_mapping_error(trans->dev, phys_addr)) {
-			iwl_pcie_tfd_unmap(trans, out_meta,
-					   &txq->tfds[q->write_ptr]);
+			iwl_pcie_tfd_unmap(trans, out_meta, txq,
+					   txq->write_ptr);
 			idx = -ENOMEM;
 			goto out;
 		}
@@ -1620,8 +1706,8 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 		phys_addr = dma_map_single(trans->dev, (void *)data,
 					   cmdlen[i], DMA_TO_DEVICE);
 		if (dma_mapping_error(trans->dev, phys_addr)) {
-			iwl_pcie_tfd_unmap(trans, out_meta,
-					   &txq->tfds[q->write_ptr]);
+			iwl_pcie_tfd_unmap(trans, out_meta, txq,
+					   txq->write_ptr);
 			idx = -ENOMEM;
 			goto out;
 		}
@@ -1629,8 +1715,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 		iwl_pcie_txq_build_tfd(trans, txq, phys_addr, cmdlen[i], false);
 	}
 
-	BUILD_BUG_ON(IWL_NUM_OF_TBS + CMD_TB_BITMAP_POS >
-		     sizeof(out_meta->flags) * BITS_PER_BYTE);
+	BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE);
 	out_meta->flags = cmd->flags;
 	if (WARN_ON_ONCE(txq->entries[idx].free_buf))
 		kzfree(txq->entries[idx].free_buf);
@@ -1639,7 +1724,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 	trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide);
 
 	/* start timer if queue currently empty */
-	if (q->read_ptr == q->write_ptr && txq->wd_timeout)
+	if (txq->read_ptr == txq->write_ptr && txq->wd_timeout)
 		mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);
 
 	spin_lock_irqsave(&trans_pcie->reg_lock, flags);
@@ -1651,7 +1736,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 	}
 
 	/* Increment and update queue's write index */
-	q->write_ptr = iwl_queue_inc_wrap(q->write_ptr);
+	txq->write_ptr = iwl_queue_inc_wrap(txq->write_ptr);
 	iwl_pcie_txq_inc_wr_ptr(trans, txq);
 
 	spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
@@ -1689,20 +1774,20 @@ void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
 	if (WARN(txq_id != trans_pcie->cmd_queue,
 		 "wrong command queue %d (should be %d), sequence 0x%X readp=%d writep=%d\n",
 		 txq_id, trans_pcie->cmd_queue, sequence,
-		 trans_pcie->txq[trans_pcie->cmd_queue].q.read_ptr,
-		 trans_pcie->txq[trans_pcie->cmd_queue].q.write_ptr)) {
+		 trans_pcie->txq[trans_pcie->cmd_queue].read_ptr,
+		 trans_pcie->txq[trans_pcie->cmd_queue].write_ptr)) {
 		iwl_print_hex_error(trans, pkt, 32);
 		return;
 	}
 
 	spin_lock_bh(&txq->lock);
 
-	cmd_index = get_cmd_index(&txq->q, index);
+	cmd_index = get_cmd_index(txq, index);
 	cmd = txq->entries[cmd_index].cmd;
 	meta = &txq->entries[cmd_index].meta;
 	cmd_id = iwl_cmd_id(cmd->hdr.cmd, group_id, 0);
 
-	iwl_pcie_tfd_unmap(trans, meta, &txq->tfds[index]);
+	iwl_pcie_tfd_unmap(trans, meta, txq, index);
 
 	/* Input error checking is done when commands are added to queue. */
 	if (meta->flags & CMD_WANT_SKB) {
@@ -1815,14 +1900,13 @@ static int iwl_pcie_send_hcmd_sync(struct iwl_trans *trans,
 				 HOST_COMPLETE_TIMEOUT);
 	if (!ret) {
 		struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue];
-		struct iwl_queue *q = &txq->q;
 
 		IWL_ERR(trans, "Error sending %s: time out after %dms.\n",
 			iwl_get_cmd_string(trans, cmd->id),
 			jiffies_to_msecs(HOST_COMPLETE_TIMEOUT));
 
 		IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n",
-			q->read_ptr, q->write_ptr);
+			txq->read_ptr, txq->write_ptr);
 
 		clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
 		IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n",
@@ -1900,7 +1984,7 @@ static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb,
 			     struct iwl_cmd_meta *out_meta,
 			     struct iwl_device_cmd *dev_cmd, u16 tb1_len)
 {
-	struct iwl_queue *q = &txq->q;
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	u16 tb2_len;
 	int i;
 
@@ -1915,8 +1999,8 @@ static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb,
 						     skb->data + hdr_len,
 						     tb2_len, DMA_TO_DEVICE);
 		if (unlikely(dma_mapping_error(trans->dev, tb2_phys))) {
-			iwl_pcie_tfd_unmap(trans, out_meta,
-					   &txq->tfds[q->write_ptr]);
+			iwl_pcie_tfd_unmap(trans, out_meta, txq,
+					   txq->write_ptr);
 			return -EINVAL;
 		}
 		iwl_pcie_txq_build_tfd(trans, txq, tb2_phys, tb2_len, false);
@@ -1935,19 +2019,19 @@ static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb,
 					   skb_frag_size(frag), DMA_TO_DEVICE);
 
 		if (unlikely(dma_mapping_error(trans->dev, tb_phys))) {
-			iwl_pcie_tfd_unmap(trans, out_meta,
-					   &txq->tfds[q->write_ptr]);
+			iwl_pcie_tfd_unmap(trans, out_meta, txq,
+					   txq->write_ptr);
 			return -EINVAL;
 		}
 		tb_idx = iwl_pcie_txq_build_tfd(trans, txq, tb_phys,
 						skb_frag_size(frag), false);
 
-		out_meta->flags |= BIT(tb_idx + CMD_TB_BITMAP_POS);
+		out_meta->tbs |= BIT(tb_idx);
 	}
 
 	trace_iwlwifi_dev_tx(trans->dev, skb,
-			     &txq->tfds[txq->q.write_ptr],
-			     sizeof(struct iwl_tfd),
+			     iwl_pcie_get_tfd(trans_pcie, txq, txq->write_ptr),
+			     trans_pcie->tfd_size,
 			     &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len,
 			     skb->data + hdr_len, tb2_len);
 	trace_iwlwifi_dev_tx_data(trans->dev, skb,
@@ -2008,7 +2092,6 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room;
 	unsigned int mss = skb_shinfo(skb)->gso_size;
-	struct iwl_queue *q = &txq->q;
 	u16 length, iv_len, amsdu_pad;
 	u8 *start_hdr;
 	struct iwl_tso_hdr_page *hdr_page;
@@ -2022,8 +2105,8 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
 		IEEE80211_CCMP_HDR_LEN : 0;
 
 	trace_iwlwifi_dev_tx(trans->dev, skb,
-			     &txq->tfds[txq->q.write_ptr],
-			     sizeof(struct iwl_tfd),
+			     iwl_pcie_get_tfd(trans_pcie, txq, txq->write_ptr),
+			     trans_pcie->tfd_size,
 			     &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len,
 			     NULL, 0);
 
@@ -2179,7 +2262,7 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
 	return 0;
 
 out_unmap:
-	iwl_pcie_tfd_unmap(trans, out_meta, &txq->tfds[q->write_ptr]);
+	iwl_pcie_tfd_unmap(trans, out_meta, txq, txq->write_ptr);
 	return ret;
 }
 #else /* CONFIG_INET */
@@ -2203,9 +2286,9 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 	struct iwl_tx_cmd *tx_cmd = (struct iwl_tx_cmd *)dev_cmd->payload;
 	struct iwl_cmd_meta *out_meta;
 	struct iwl_txq *txq;
-	struct iwl_queue *q;
 	dma_addr_t tb0_phys, tb1_phys, scratch_phys;
 	void *tb1_addr;
+	void *tfd;
 	u16 len, tb1_len;
 	bool wait_write_ptr;
 	__le16 fc;
@@ -2214,7 +2297,6 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 	bool amsdu;
 
 	txq = &trans_pcie->txq[txq_id];
-	q = &txq->q;
 
 	if (WARN_ONCE(!test_bit(txq_id, trans_pcie->queue_used),
 		      "TX on unused queue %d\n", txq_id))
@@ -2236,7 +2318,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 	}
 
 	if (skb_is_nonlinear(skb) &&
-	    skb_shinfo(skb)->nr_frags > IWL_PCIE_MAX_FRAGS &&
+	    skb_shinfo(skb)->nr_frags > IWL_PCIE_MAX_FRAGS(trans_pcie) &&
 	    __skb_linearize(skb))
 		return -ENOMEM;
 
@@ -2249,11 +2331,11 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 
 	spin_lock(&txq->lock);
 
-	if (iwl_queue_space(q) < q->high_mark) {
+	if (iwl_queue_space(txq) < txq->high_mark) {
 		iwl_stop_queue(trans, txq);
 
 		/* don't put the packet on the ring, if there is no room */
-		if (unlikely(iwl_queue_space(q) < 3)) {
+		if (unlikely(iwl_queue_space(txq) < 3)) {
 			struct iwl_device_cmd **dev_cmd_ptr;
 
 			dev_cmd_ptr = (void *)((u8 *)skb->cb +
@@ -2274,19 +2356,19 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 	 */
 	wifi_seq = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
 	WARN_ONCE(txq->ampdu &&
-		  (wifi_seq & 0xff) != q->write_ptr,
+		  (wifi_seq & 0xff) != txq->write_ptr,
 		  "Q: %d WiFi Seq %d tfdNum %d",
-		  txq_id, wifi_seq, q->write_ptr);
+		  txq_id, wifi_seq, txq->write_ptr);
 
 	/* Set up driver data for this TFD */
-	txq->entries[q->write_ptr].skb = skb;
-	txq->entries[q->write_ptr].cmd = dev_cmd;
+	txq->entries[txq->write_ptr].skb = skb;
+	txq->entries[txq->write_ptr].cmd = dev_cmd;
 
 	dev_cmd->hdr.sequence =
 		cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) |
-			    INDEX_TO_SEQ(q->write_ptr)));
+			    INDEX_TO_SEQ(txq->write_ptr)));
 
-	tb0_phys = iwl_pcie_get_first_tb_dma(txq, q->write_ptr);
+	tb0_phys = iwl_pcie_get_first_tb_dma(txq, txq->write_ptr);
 	scratch_phys = tb0_phys + sizeof(struct iwl_cmd_header) +
 		       offsetof(struct iwl_tx_cmd, scratch);
 
@@ -2294,7 +2376,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 	tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys);
 
 	/* Set up first empty entry in queue's array of Tx/cmd buffers */
-	out_meta = &txq->entries[q->write_ptr].meta;
+	out_meta = &txq->entries[txq->write_ptr].meta;
 	out_meta->flags = 0;
 
 	/*
@@ -2319,7 +2401,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 	}
 
 	/* The first TB points to bi-directional DMA data */
-	memcpy(&txq->first_tb_bufs[q->write_ptr], &dev_cmd->hdr,
+	memcpy(&txq->first_tb_bufs[txq->write_ptr], &dev_cmd->hdr,
 	       IWL_FIRST_TB_SIZE);
 	iwl_pcie_txq_build_tfd(trans, txq, tb0_phys,
 			       IWL_FIRST_TB_SIZE, true);
@@ -2344,13 +2426,15 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 		goto out_err;
 	}
 
+	tfd = iwl_pcie_get_tfd(trans_pcie, txq, txq->write_ptr);
 	/* Set up entry for this TFD in Tx byte-count array */
-	iwl_pcie_txq_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len));
+	iwl_pcie_txq_update_byte_cnt_tbl(trans, txq, le16_to_cpu(tx_cmd->len),
+					 iwl_pcie_tfd_get_num_tbs(trans, tfd));
 
 	wait_write_ptr = ieee80211_has_morefrags(fc);
 
 	/* start timer if queue currently empty */
-	if (q->read_ptr == q->write_ptr) {
+	if (txq->read_ptr == txq->write_ptr) {
 		if (txq->wd_timeout) {
 			/*
 			 * If the TXQ is active, then set the timer, if not,
@@ -2364,12 +2448,12 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 			else
 				txq->frozen_expiry_remainder = txq->wd_timeout;
 		}
-		IWL_DEBUG_RPM(trans, "Q: %d first tx - take ref\n", q->id);
+		IWL_DEBUG_RPM(trans, "Q: %d first tx - take ref\n", txq->id);
 		iwl_trans_ref(trans);
 	}
 
 	/* Tell device the write index *just past* this latest filled TFD */
-	q->write_ptr = iwl_queue_inc_wrap(q->write_ptr);
+	txq->write_ptr = iwl_queue_inc_wrap(txq->write_ptr);
 	if (!wait_write_ptr)
 		iwl_pcie_txq_inc_wr_ptr(trans, txq);
 
diff --git a/drivers/net/wireless/intersil/hostap/hostap_ioctl.c b/drivers/net/wireless/intersil/hostap/hostap_ioctl.c
index 3e5fa7872b64..a5656bc0e6aa 100644
--- a/drivers/net/wireless/intersil/hostap/hostap_ioctl.c
+++ b/drivers/net/wireless/intersil/hostap/hostap_ioctl.c
@@ -3041,13 +3041,9 @@ static int prism2_ioctl_priv_download(local_info_t *local, struct iw_point *p)
 	    p->length > 1024 || !p->pointer)
 		return -EINVAL;
 
-	param = kmalloc(p->length, GFP_KERNEL);
-	if (param == NULL)
-		return -ENOMEM;
-
-	if (copy_from_user(param, p->pointer, p->length)) {
-		ret = -EFAULT;
-		goto out;
+	param = memdup_user(p->pointer, p->length);
+	if (IS_ERR(param)) {
+		return PTR_ERR(param);
 	}
 
 	if (p->length < sizeof(struct prism2_download_param) +
@@ -3803,13 +3799,9 @@ static int prism2_ioctl_priv_hostapd(local_info_t *local, struct iw_point *p)
 	    p->length > PRISM2_HOSTAPD_MAX_BUF_SIZE || !p->pointer)
 		return -EINVAL;
 
-	param = kmalloc(p->length, GFP_KERNEL);
-	if (param == NULL)
-		return -ENOMEM;
-
-	if (copy_from_user(param, p->pointer, p->length)) {
-		ret = -EFAULT;
-		goto out;
+	param = memdup_user(p->pointer, p->length);
+	if (IS_ERR(param)) {
+		return PTR_ERR(param);
 	}
 
 	switch (param->cmd) {
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco_usb.c b/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
index 56f109bc8394..bca6935a94db 100644
--- a/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
+++ b/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
@@ -1613,10 +1613,8 @@ static int ezusb_probe(struct usb_interface *interface,
 			}
 
 			upriv->read_urb = usb_alloc_urb(0, GFP_KERNEL);
-			if (!upriv->read_urb) {
-				err("No free urbs available");
+			if (!upriv->read_urb)
 				goto error;
-			}
 			if (le16_to_cpu(ep->wMaxPacketSize) != 64)
 				pr_warn("bulk in: wMaxPacketSize!= 64\n");
 			if (ep->bEndpointAddress != (2 | USB_DIR_IN))
diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c
index 8c35ac838fce..431f13b4faf6 100644
--- a/drivers/net/wireless/mac80211_hwsim.c
+++ b/drivers/net/wireless/mac80211_hwsim.c
@@ -487,7 +487,7 @@ static const struct ieee80211_iface_combination hwsim_if_comb_p2p_dev[] = {
 };
 
 static spinlock_t hwsim_radio_lock;
-static struct list_head hwsim_radios;
+static LIST_HEAD(hwsim_radios);
 static int hwsim_radio_idx;
 
 static struct platform_driver mac80211_hwsim_driver = {
@@ -3376,7 +3376,6 @@ static int __init init_mac80211_hwsim(void)
 		mac80211_hwsim_unassign_vif_chanctx;
 
 	spin_lock_init(&hwsim_radio_lock);
-	INIT_LIST_HEAD(&hwsim_radios);
 
 	err = register_pernet_device(&hwsim_net_ops);
 	if (err)
diff --git a/drivers/net/wireless/marvell/libertas_tf/if_usb.c b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
index 799a2efe5793..e0ade40d9497 100644
--- a/drivers/net/wireless/marvell/libertas_tf/if_usb.c
+++ b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
@@ -198,22 +198,16 @@ static int if_usb_probe(struct usb_interface *intf,
 	}
 
 	cardp->rx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!cardp->rx_urb) {
-		lbtf_deb_usbd(&udev->dev, "Rx URB allocation failed\n");
+	if (!cardp->rx_urb)
 		goto dealloc;
-	}
 
 	cardp->tx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!cardp->tx_urb) {
-		lbtf_deb_usbd(&udev->dev, "Tx URB allocation failed\n");
+	if (!cardp->tx_urb)
 		goto dealloc;
-	}
 
 	cardp->cmd_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!cardp->cmd_urb) {
-		lbtf_deb_usbd(&udev->dev, "Cmd URB allocation failed\n");
+	if (!cardp->cmd_urb)
 		goto dealloc;
-	}
 
 	cardp->ep_out_buf = kmalloc(MRVDRV_ETH_TX_PACKET_BUFFER_SIZE,
 				    GFP_KERNEL);
diff --git a/drivers/net/wireless/marvell/mwifiex/11h.c b/drivers/net/wireless/marvell/mwifiex/11h.c
index 81c60d0a1bda..43dccd5b0291 100644
--- a/drivers/net/wireless/marvell/mwifiex/11h.c
+++ b/drivers/net/wireless/marvell/mwifiex/11h.c
@@ -260,22 +260,17 @@ int mwifiex_11h_handle_radar_detected(struct mwifiex_private *priv,
 
 	rdr_event = (void *)(skb->data + sizeof(u32));
 
-	if (le32_to_cpu(rdr_event->passed)) {
-		mwifiex_dbg(priv->adapter, MSG,
-			    "radar detected; indicating kernel\n");
-		if (mwifiex_stop_radar_detection(priv, &priv->dfs_chandef))
-			mwifiex_dbg(priv->adapter, ERROR,
-				    "Failed to stop CAC in FW\n");
-		cfg80211_radar_event(priv->adapter->wiphy, &priv->dfs_chandef,
-				     GFP_KERNEL);
-		mwifiex_dbg(priv->adapter, MSG, "regdomain: %d\n",
-			    rdr_event->reg_domain);
-		mwifiex_dbg(priv->adapter, MSG, "radar detection type: %d\n",
-			    rdr_event->det_type);
-	} else {
-		mwifiex_dbg(priv->adapter, MSG,
-			    "false radar detection event!\n");
-	}
+	mwifiex_dbg(priv->adapter, MSG,
+		    "radar detected; indicating kernel\n");
+	if (mwifiex_stop_radar_detection(priv, &priv->dfs_chandef))
+		mwifiex_dbg(priv->adapter, ERROR,
+			    "Failed to stop CAC in FW\n");
+	cfg80211_radar_event(priv->adapter->wiphy, &priv->dfs_chandef,
+			     GFP_KERNEL);
+	mwifiex_dbg(priv->adapter, MSG, "regdomain: %d\n",
+		    rdr_event->reg_domain);
+	mwifiex_dbg(priv->adapter, MSG, "radar detection type: %d\n",
+		    rdr_event->det_type);
 
 	return 0;
 }
diff --git a/drivers/net/wireless/marvell/mwifiex/11n.h b/drivers/net/wireless/marvell/mwifiex/11n.h
index afdd58aa90de..ea0fa68b9913 100644
--- a/drivers/net/wireless/marvell/mwifiex/11n.h
+++ b/drivers/net/wireless/marvell/mwifiex/11n.h
@@ -171,9 +171,10 @@ mwifiex_find_stream_to_delete(struct mwifiex_private *priv, int ptr_tid,
 static inline int mwifiex_is_sta_11n_enabled(struct mwifiex_private *priv,
 					     struct mwifiex_sta_node *node)
 {
-
-	if (!node || (priv->bss_role != MWIFIEX_BSS_ROLE_UAP) ||
-	    !priv->ap_11n_enabled)
+	if (!node || ((priv->bss_role == MWIFIEX_BSS_ROLE_UAP) &&
+		      !priv->ap_11n_enabled) ||
+	    ((priv->bss_mode == NL80211_IFTYPE_ADHOC) &&
+	     !priv->adapter->adhoc_11n_enabled))
 		return 0;
 
 	return node->is_11n_enabled;
diff --git a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c
index dc49c3de1f25..c47d6366875d 100644
--- a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c
+++ b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c
@@ -205,7 +205,8 @@ mwifiex_11n_aggregate_pkt(struct mwifiex_private *priv,
 
 	do {
 		/* Check if AMSDU can accommodate this MSDU */
-		if (skb_tailroom(skb_aggr) < (skb_src->len + LLC_SNAP_LEN))
+		if ((skb_aggr->len + skb_src->len + LLC_SNAP_LEN) >
+		    adapter->tx_buf_size)
 			break;
 
 		skb_src = skb_dequeue(&pra_list->skb_head);
diff --git a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c
index a74cc43b1953..94480123efa3 100644
--- a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c
+++ b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c
@@ -78,8 +78,15 @@ static int mwifiex_11n_dispatch_amsdu_pkt(struct mwifiex_private *priv,
  */
 static int mwifiex_11n_dispatch_pkt(struct mwifiex_private *priv, void *payload)
 {
-	int ret = mwifiex_11n_dispatch_amsdu_pkt(priv, payload);
 
+	int ret;
+
+	if (!payload) {
+		mwifiex_dbg(priv->adapter, INFO, "info: fw drop data\n");
+		return 0;
+	}
+
+	ret = mwifiex_11n_dispatch_amsdu_pkt(priv, payload);
 	if (!ret)
 		return 0;
 
@@ -921,3 +928,72 @@ void mwifiex_coex_ampdu_rxwinsize(struct mwifiex_adapter *adapter)
 	else
 		mwifiex_update_ampdu_rxwinsize(adapter, false);
 }
+
+/* This function handles rxba_sync event
+ */
+void mwifiex_11n_rxba_sync_event(struct mwifiex_private *priv,
+				 u8 *event_buf, u16 len)
+{
+	struct mwifiex_ie_types_rxba_sync *tlv_rxba = (void *)event_buf;
+	u16 tlv_type, tlv_len;
+	struct mwifiex_rx_reorder_tbl *rx_reor_tbl_ptr;
+	u8 i, j;
+	u16 seq_num, tlv_seq_num, tlv_bitmap_len;
+	int tlv_buf_left = len;
+	int ret;
+	u8 *tmp;
+
+	mwifiex_dbg_dump(priv->adapter, EVT_D, "RXBA_SYNC event:",
+			 event_buf, len);
+	while (tlv_buf_left >= sizeof(*tlv_rxba)) {
+		tlv_type = le16_to_cpu(tlv_rxba->header.type);
+		tlv_len  = le16_to_cpu(tlv_rxba->header.len);
+		if (tlv_type != TLV_TYPE_RXBA_SYNC) {
+			mwifiex_dbg(priv->adapter, ERROR,
+				    "Wrong TLV id=0x%x\n", tlv_type);
+			return;
+		}
+
+		tlv_seq_num = le16_to_cpu(tlv_rxba->seq_num);
+		tlv_bitmap_len = le16_to_cpu(tlv_rxba->bitmap_len);
+		mwifiex_dbg(priv->adapter, INFO,
+			    "%pM tid=%d seq_num=%d bitmap_len=%d\n",
+			    tlv_rxba->mac, tlv_rxba->tid, tlv_seq_num,
+			    tlv_bitmap_len);
+
+		rx_reor_tbl_ptr =
+			mwifiex_11n_get_rx_reorder_tbl(priv, tlv_rxba->tid,
+						       tlv_rxba->mac);
+		if (!rx_reor_tbl_ptr) {
+			mwifiex_dbg(priv->adapter, ERROR,
+				    "Can not find rx_reorder_tbl!");
+			return;
+		}
+
+		for (i = 0; i < tlv_bitmap_len; i++) {
+			for (j = 0 ; j < 8; j++) {
+				if (tlv_rxba->bitmap[i] & (1 << j)) {
+					seq_num = (MAX_TID_VALUE - 1) &
+						(tlv_seq_num + i * 8 + j);
+
+					mwifiex_dbg(priv->adapter, ERROR,
+						    "drop packet,seq=%d\n",
+						    seq_num);
+
+					ret = mwifiex_11n_rx_reorder_pkt
+					(priv, seq_num, tlv_rxba->tid,
+					 tlv_rxba->mac, 0, NULL);
+
+					if (ret)
+						mwifiex_dbg(priv->adapter,
+							    ERROR,
+							    "Fail to drop packet");
+				}
+			}
+		}
+
+		tlv_buf_left -= (sizeof(*tlv_rxba) + tlv_len);
+		tmp = (u8 *)tlv_rxba + tlv_len + sizeof(*tlv_rxba);
+		tlv_rxba = (struct mwifiex_ie_types_rxba_sync *)tmp;
+	}
+}
diff --git a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.h b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.h
index 63ecea89b4ab..22d991f514c8 100644
--- a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.h
+++ b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.h
@@ -81,5 +81,6 @@ struct mwifiex_rx_reorder_tbl *
 mwifiex_11n_get_rx_reorder_tbl(struct mwifiex_private *priv, int tid, u8 *ta);
 void mwifiex_11n_del_rx_reorder_tbl_by_ta(struct mwifiex_private *priv, u8 *ta);
 void mwifiex_update_rxreor_flags(struct mwifiex_adapter *adapter, u8 flags);
-
+void mwifiex_11n_rxba_sync_event(struct mwifiex_private *priv,
+				 u8 *event_buf, u16 len);
 #endif /* _MWIFIEX_11N_RXREORDER_H_ */
diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
index a8ff969c95c2..39ce76ad00bc 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
@@ -484,6 +484,29 @@ mwifiex_cfg80211_add_key(struct wiphy *wiphy, struct net_device *netdev,
 }
 
 /*
+ * CFG802.11 operation handler to set default mgmt key.
+ */
+static int
+mwifiex_cfg80211_set_default_mgmt_key(struct wiphy *wiphy,
+				      struct net_device *netdev,
+				      u8 key_index)
+{
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev);
+	struct mwifiex_ds_encrypt_key encrypt_key;
+
+	wiphy_dbg(wiphy, "set default mgmt key, key index=%d\n", key_index);
+
+	memset(&encrypt_key, 0, sizeof(struct mwifiex_ds_encrypt_key));
+	encrypt_key.key_len = WLAN_KEY_LEN_CCMP;
+	encrypt_key.key_index = key_index;
+	encrypt_key.is_igtk_def_key = true;
+	eth_broadcast_addr(encrypt_key.mac_addr);
+
+	return mwifiex_send_cmd(priv, HostCmd_CMD_802_11_KEY_MATERIAL,
+				HostCmd_ACT_GEN_SET, true, &encrypt_key, true);
+}
+
+/*
  * This function sends domain information to the firmware.
  *
  * The following information are passed to the firmware -
@@ -2012,10 +2035,6 @@ mwifiex_cfg80211_disconnect(struct wiphy *wiphy, struct net_device *dev,
 	if (mwifiex_deauthenticate(priv, NULL))
 		return -EFAULT;
 
-	mwifiex_dbg(priv->adapter, MSG,
-		    "info: successfully disconnected from %pM:\t"
-		    "reason code %d\n", priv->cfg_bssid, reason_code);
-
 	eth_zero_addr(priv->cfg_bssid);
 	priv->hs2_enabled = false;
 
@@ -2485,6 +2504,16 @@ mwifiex_cfg80211_scan(struct wiphy *wiphy,
 
 	priv->scan_request = request;
 
+	if (request->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) {
+		ether_addr_copy(priv->random_mac, request->mac_addr);
+		for (i = 0; i < ETH_ALEN; i++) {
+			priv->random_mac[i] &= request->mac_addr_mask[i];
+			priv->random_mac[i] |= get_random_int() &
+					       ~(request->mac_addr_mask[i]);
+		}
+	}
+
+	ether_addr_copy(user_scan_cfg->random_mac, priv->random_mac);
 	user_scan_cfg->num_ssids = request->n_ssids;
 	user_scan_cfg->ssid_list = request->ssids;
 
@@ -2726,7 +2755,7 @@ mwifiex_setup_ht_caps(struct ieee80211_sta_ht_cap *ht_info,
 		ht_info->cap &= ~IEEE80211_HT_CAP_SGI_40;
 
 	if (adapter->user_dev_mcs_support == HT_STREAM_2X2)
-		ht_info->cap |= 3 << IEEE80211_HT_CAP_RX_STBC_SHIFT;
+		ht_info->cap |= 2 << IEEE80211_HT_CAP_RX_STBC_SHIFT;
 	else
 		ht_info->cap |= 1 << IEEE80211_HT_CAP_RX_STBC_SHIFT;
 
@@ -3913,6 +3942,88 @@ static int mwifiex_cfg80211_get_channel(struct wiphy *wiphy,
 	return ret;
 }
 
+#ifdef CONFIG_NL80211_TESTMODE
+
+enum mwifiex_tm_attr {
+	__MWIFIEX_TM_ATTR_INVALID	= 0,
+	MWIFIEX_TM_ATTR_CMD		= 1,
+	MWIFIEX_TM_ATTR_DATA		= 2,
+
+	/* keep last */
+	__MWIFIEX_TM_ATTR_AFTER_LAST,
+	MWIFIEX_TM_ATTR_MAX		= __MWIFIEX_TM_ATTR_AFTER_LAST - 1,
+};
+
+static const struct nla_policy mwifiex_tm_policy[MWIFIEX_TM_ATTR_MAX + 1] = {
+	[MWIFIEX_TM_ATTR_CMD]		= { .type = NLA_U32 },
+	[MWIFIEX_TM_ATTR_DATA]		= { .type = NLA_BINARY,
+					    .len = MWIFIEX_SIZE_OF_CMD_BUFFER },
+};
+
+enum mwifiex_tm_command {
+	MWIFIEX_TM_CMD_HOSTCMD	= 0,
+};
+
+static int mwifiex_tm_cmd(struct wiphy *wiphy, struct wireless_dev *wdev,
+			  void *data, int len)
+{
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
+	struct mwifiex_ds_misc_cmd *hostcmd;
+	struct nlattr *tb[MWIFIEX_TM_ATTR_MAX + 1];
+	struct mwifiex_adapter *adapter;
+	struct sk_buff *skb;
+	int err;
+
+	if (!priv)
+		return -EINVAL;
+	adapter = priv->adapter;
+
+	err = nla_parse(tb, MWIFIEX_TM_ATTR_MAX, data, len,
+			mwifiex_tm_policy);
+	if (err)
+		return err;
+
+	if (!tb[MWIFIEX_TM_ATTR_CMD])
+		return -EINVAL;
+
+	switch (nla_get_u32(tb[MWIFIEX_TM_ATTR_CMD])) {
+	case MWIFIEX_TM_CMD_HOSTCMD:
+		if (!tb[MWIFIEX_TM_ATTR_DATA])
+			return -EINVAL;
+
+		hostcmd = kzalloc(sizeof(*hostcmd), GFP_KERNEL);
+		if (!hostcmd)
+			return -ENOMEM;
+
+		hostcmd->len = nla_len(tb[MWIFIEX_TM_ATTR_DATA]);
+		memcpy(hostcmd->cmd, nla_data(tb[MWIFIEX_TM_ATTR_DATA]),
+		       hostcmd->len);
+
+		if (mwifiex_send_cmd(priv, 0, 0, 0, hostcmd, true)) {
+			dev_err(priv->adapter->dev, "Failed to process hostcmd\n");
+			return -EFAULT;
+		}
+
+		/* process hostcmd response*/
+		skb = cfg80211_testmode_alloc_reply_skb(wiphy, hostcmd->len);
+		if (!skb)
+			return -ENOMEM;
+		err = nla_put(skb, MWIFIEX_TM_ATTR_DATA,
+			      hostcmd->len, hostcmd->cmd);
+		if (err) {
+			kfree_skb(skb);
+			return -EMSGSIZE;
+		}
+
+		err = cfg80211_testmode_reply(skb);
+		kfree(hostcmd);
+		return err;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+#endif
+
 static int
 mwifiex_cfg80211_start_radar_detection(struct wiphy *wiphy,
 				       struct net_device *dev,
@@ -3994,6 +4105,7 @@ static struct cfg80211_ops mwifiex_cfg80211_ops = {
 	.leave_ibss = mwifiex_cfg80211_leave_ibss,
 	.add_key = mwifiex_cfg80211_add_key,
 	.del_key = mwifiex_cfg80211_del_key,
+	.set_default_mgmt_key = mwifiex_cfg80211_set_default_mgmt_key,
 	.mgmt_tx = mwifiex_cfg80211_mgmt_tx,
 	.mgmt_frame_register = mwifiex_cfg80211_mgmt_frame_register,
 	.remain_on_channel = mwifiex_cfg80211_remain_on_channel,
@@ -4025,6 +4137,7 @@ static struct cfg80211_ops mwifiex_cfg80211_ops = {
 	.tdls_cancel_channel_switch = mwifiex_cfg80211_tdls_cancel_chan_switch,
 	.add_station = mwifiex_cfg80211_add_station,
 	.change_station = mwifiex_cfg80211_change_station,
+	CFG80211_TESTMODE_CMD(mwifiex_tm_cmd)
 	.get_channel = mwifiex_cfg80211_get_channel,
 	.start_radar_detection = mwifiex_cfg80211_start_radar_detection,
 	.channel_switch = mwifiex_cfg80211_channel_switch,
@@ -4135,9 +4248,12 @@ int mwifiex_register_cfg80211(struct mwifiex_adapter *adapter)
 	wiphy->cipher_suites = mwifiex_cipher_suites;
 	wiphy->n_cipher_suites = ARRAY_SIZE(mwifiex_cipher_suites);
 
-	if (adapter->region_code)
-		wiphy->regulatory_flags |= REGULATORY_DISABLE_BEACON_HINTS |
+	if (adapter->regd) {
+		wiphy->regulatory_flags |= REGULATORY_CUSTOM_REG |
+					   REGULATORY_DISABLE_BEACON_HINTS |
 					   REGULATORY_COUNTRY_IE_IGNORE;
+		wiphy_apply_custom_regulatory(wiphy, adapter->regd);
+	}
 
 	ether_addr_copy(wiphy->perm_addr, adapter->perm_addr);
 	wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM;
@@ -4173,7 +4289,10 @@ int mwifiex_register_cfg80211(struct mwifiex_adapter *adapter)
 	wiphy->features |= NL80211_FEATURE_HT_IBSS |
 			   NL80211_FEATURE_INACTIVITY_TIMER |
 			   NL80211_FEATURE_LOW_PRIORITY_SCAN |
-			   NL80211_FEATURE_NEED_OBSS_SCAN;
+			   NL80211_FEATURE_NEED_OBSS_SCAN |
+			   NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR |
+			   NL80211_FEATURE_SCHED_SCAN_RANDOM_MAC_ADDR |
+			   NL80211_FEATURE_ND_RANDOM_MAC_ADDR;
 
 	if (ISSUPP_TDLS_ENABLED(adapter->fw_cap_info))
 		wiphy->features |= NL80211_FEATURE_TDLS_CHANNEL_SWITCH;
@@ -4200,19 +4319,27 @@ int mwifiex_register_cfg80211(struct mwifiex_adapter *adapter)
 		return ret;
 	}
 
-	if (reg_alpha2 && mwifiex_is_valid_alpha2(reg_alpha2)) {
-		mwifiex_dbg(adapter, INFO,
-			    "driver hint alpha2: %2.2s\n", reg_alpha2);
-		regulatory_hint(wiphy, reg_alpha2);
-	} else {
-		if (adapter->region_code == 0x00) {
-			mwifiex_dbg(adapter, WARN, "Ignore world regulatory domain\n");
+	if (!adapter->regd) {
+		if (reg_alpha2 && mwifiex_is_valid_alpha2(reg_alpha2)) {
+			mwifiex_dbg(adapter, INFO,
+				    "driver hint alpha2: %2.2s\n", reg_alpha2);
+			regulatory_hint(wiphy, reg_alpha2);
 		} else {
-			country_code =
-				mwifiex_11d_code_2_region(adapter->region_code);
-			if (country_code &&
-			    regulatory_hint(wiphy, country_code))
-				mwifiex_dbg(priv->adapter, ERROR, "regulatory_hint() failed\n");
+			if (adapter->region_code == 0x00) {
+				mwifiex_dbg(adapter, WARN,
+					    "Ignore world regulatory domain\n");
+			} else {
+				wiphy->regulatory_flags |=
+					REGULATORY_DISABLE_BEACON_HINTS |
+					REGULATORY_COUNTRY_IE_IGNORE;
+				country_code =
+					mwifiex_11d_code_2_region(
+						adapter->region_code);
+				if (country_code &&
+				    regulatory_hint(wiphy, country_code))
+					mwifiex_dbg(priv->adapter, ERROR,
+						    "regulatory_hint() failed\n");
+			}
 		}
 	}
 
diff --git a/drivers/net/wireless/marvell/mwifiex/cmdevt.c b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
index c29f26d8baf2..53477280f39c 100644
--- a/drivers/net/wireless/marvell/mwifiex/cmdevt.c
+++ b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
@@ -480,13 +480,27 @@ int mwifiex_free_cmd_buffer(struct mwifiex_adapter *adapter)
  */
 int mwifiex_process_event(struct mwifiex_adapter *adapter)
 {
-	int ret;
+	int ret, i;
 	struct mwifiex_private *priv =
 		mwifiex_get_priv(adapter, MWIFIEX_BSS_ROLE_ANY);
 	struct sk_buff *skb = adapter->event_skb;
-	u32 eventcause = adapter->event_cause;
+	u32 eventcause;
 	struct mwifiex_rxinfo *rx_info;
 
+	if ((adapter->event_cause & EVENT_ID_MASK) == EVENT_RADAR_DETECTED) {
+		for (i = 0; i < adapter->priv_num; i++) {
+			priv = adapter->priv[i];
+			if (priv && mwifiex_is_11h_active(priv)) {
+				adapter->event_cause |=
+					((priv->bss_num & 0xff) << 16) |
+					((priv->bss_type & 0xff) << 24);
+				break;
+			}
+		}
+	}
+
+	eventcause = adapter->event_cause;
+
 	/* Save the last event to debug log */
 	adapter->dbg.last_event_index =
 			(adapter->dbg.last_event_index + 1) % DBG_CMD_NUM;
@@ -581,6 +595,14 @@ int mwifiex_send_cmd(struct mwifiex_private *priv, u16 cmd_no,
 			return -1;
 		}
 	}
+	/* We don't expect commands in manufacturing mode. They are cooked
+	 * in application and ready to download buffer is passed to the driver
+	 */
+	if (adapter->mfg_mode && cmd_no) {
+		dev_dbg(adapter->dev, "Ignoring commands in manufacturing mode\n");
+		return -1;
+	}
+
 
 	/* Get a new command node */
 	cmd_node = mwifiex_get_cmd_node(adapter);
diff --git a/drivers/net/wireless/marvell/mwifiex/debugfs.c b/drivers/net/wireless/marvell/mwifiex/debugfs.c
index bccf17ad588e..b9284b533294 100644
--- a/drivers/net/wireless/marvell/mwifiex/debugfs.c
+++ b/drivers/net/wireless/marvell/mwifiex/debugfs.c
@@ -118,6 +118,8 @@ mwifiex_info_read(struct file *file, char __user *ubuf,
 		p += sprintf(p, "bssid=\"%pM\"\n", info.bssid);
 		p += sprintf(p, "channel=\"%d\"\n", (int) info.bss_chan);
 		p += sprintf(p, "country_code = \"%s\"\n", info.country_code);
+		p += sprintf(p, "region_code=\"0x%x\"\n",
+			     priv->adapter->region_code);
 
 		netdev_for_each_mc_addr(ha, netdev)
 			p += sprintf(p, "multicast_address[%d]=\"%pM\"\n",
diff --git a/drivers/net/wireless/marvell/mwifiex/fw.h b/drivers/net/wireless/marvell/mwifiex/fw.h
index 5596b6be1898..4b1894b4757f 100644
--- a/drivers/net/wireless/marvell/mwifiex/fw.h
+++ b/drivers/net/wireless/marvell/mwifiex/fw.h
@@ -78,6 +78,7 @@ enum KEY_TYPE_ID {
 	KEY_TYPE_ID_AES,
 	KEY_TYPE_ID_WAPI,
 	KEY_TYPE_ID_AES_CMAC,
+	KEY_TYPE_ID_AES_CMAC_DEF,
 };
 
 #define WPA_PN_SIZE		8
@@ -176,6 +177,7 @@ enum MWIFIEX_802_11_PRIVACY_FILTER {
 #define TLV_TYPE_PWK_CIPHER         (PROPRIETARY_TLV_BASE_ID + 145)
 #define TLV_TYPE_GWK_CIPHER         (PROPRIETARY_TLV_BASE_ID + 146)
 #define TLV_TYPE_TX_PAUSE           (PROPRIETARY_TLV_BASE_ID + 148)
+#define TLV_TYPE_RXBA_SYNC          (PROPRIETARY_TLV_BASE_ID + 153)
 #define TLV_TYPE_COALESCE_RULE      (PROPRIETARY_TLV_BASE_ID + 154)
 #define TLV_TYPE_KEY_PARAM_V2       (PROPRIETARY_TLV_BASE_ID + 156)
 #define TLV_TYPE_REPEAT_COUNT       (PROPRIETARY_TLV_BASE_ID + 176)
@@ -188,6 +190,8 @@ enum MWIFIEX_802_11_PRIVACY_FILTER {
 #define TLV_BTCOEX_WL_AGGR_WINSIZE  (PROPRIETARY_TLV_BASE_ID + 202)
 #define TLV_BTCOEX_WL_SCANTIME      (PROPRIETARY_TLV_BASE_ID + 203)
 #define TLV_TYPE_BSS_MODE           (PROPRIETARY_TLV_BASE_ID + 206)
+#define TLV_TYPE_RANDOM_MAC         (PROPRIETARY_TLV_BASE_ID + 236)
+#define TLV_TYPE_CHAN_ATTR_CFG      (PROPRIETARY_TLV_BASE_ID + 237)
 
 #define MWIFIEX_TX_DATA_BUF_SIZE_2K        2048
 
@@ -208,6 +212,7 @@ enum MWIFIEX_802_11_PRIVACY_FILTER {
 
 #define MWIFIEX_TX_DATA_BUF_SIZE_4K        4096
 #define MWIFIEX_TX_DATA_BUF_SIZE_8K        8192
+#define MWIFIEX_TX_DATA_BUF_SIZE_12K       12288
 
 #define ISSUPP_11NENABLED(FwCapInfo) (FwCapInfo & BIT(11))
 #define ISSUPP_TDLS_ENABLED(FwCapInfo) (FwCapInfo & BIT(14))
@@ -379,6 +384,7 @@ enum MWIFIEX_802_11_PRIVACY_FILTER {
 #define HostCmd_CMD_MC_POLICY                         0x0121
 #define HostCmd_CMD_TDLS_OPER                         0x0122
 #define HostCmd_CMD_SDIO_SP_RX_AGGR_CFG               0x0223
+#define HostCmd_CMD_CHAN_REGION_CFG		      0x0242
 
 #define PROTOCOL_NO_SECURITY        0x01
 #define PROTOCOL_STATIC_WEP         0x02
@@ -411,6 +417,14 @@ enum P2P_MODES {
 	P2P_MODE_CLIENT = 3,
 };
 
+enum mwifiex_channel_flags {
+	MWIFIEX_CHANNEL_PASSIVE = BIT(0),
+	MWIFIEX_CHANNEL_DFS = BIT(1),
+	MWIFIEX_CHANNEL_NOHT40 = BIT(2),
+	MWIFIEX_CHANNEL_NOHT80 = BIT(3),
+	MWIFIEX_CHANNEL_DISABLED = BIT(7),
+};
+
 #define HostCmd_RET_BIT                       0x8000
 #define HostCmd_ACT_GEN_GET                   0x0000
 #define HostCmd_ACT_GEN_SET                   0x0001
@@ -504,6 +518,8 @@ enum P2P_MODES {
 #define EVENT_RSSI_HIGH                 0x0000001c
 #define EVENT_SNR_HIGH                  0x0000001d
 #define EVENT_IBSS_COALESCED            0x0000001e
+#define EVENT_IBSS_STA_CONNECT          0x00000020
+#define EVENT_IBSS_STA_DISCONNECT       0x00000021
 #define EVENT_DATA_RSSI_LOW             0x00000024
 #define EVENT_DATA_SNR_LOW              0x00000025
 #define EVENT_DATA_RSSI_HIGH            0x00000026
@@ -531,6 +547,7 @@ enum P2P_MODES {
 #define EVENT_CHANNEL_REPORT_RDY        0x00000054
 #define EVENT_TX_DATA_PAUSE             0x00000055
 #define EVENT_EXT_SCAN_REPORT           0x00000058
+#define EVENT_RXBA_SYNC                 0x00000059
 #define EVENT_BG_SCAN_STOPPED           0x00000065
 #define EVENT_REMAIN_ON_CHAN_EXPIRED    0x0000005f
 #define EVENT_MULTI_CHAN_INFO           0x0000006a
@@ -734,6 +751,16 @@ struct mwifiex_ie_types_chan_list_param_set {
 	struct mwifiex_chan_scan_param_set chan_scan_param[1];
 } __packed;
 
+struct mwifiex_ie_types_rxba_sync {
+	struct mwifiex_ie_types_header header;
+	u8 mac[ETH_ALEN];
+	u8 tid;
+	u8 reserved;
+	__le16 seq_num;
+	__le16 bitmap_len;
+	u8 bitmap[1];
+} __packed;
+
 struct chan_band_param_set {
 	u8 radio_type;
 	u8 chan_number;
@@ -780,6 +807,11 @@ struct mwifiex_ie_types_scan_chan_gap {
 	__le16 chan_gap;
 } __packed;
 
+struct mwifiex_ie_types_random_mac {
+	struct mwifiex_ie_types_header header;
+	u8 mac[ETH_ALEN];
+} __packed;
+
 struct mwifiex_ietypes_chanstats {
 	struct mwifiex_ie_types_header header;
 	struct mwifiex_fw_chan_stats chanstats[0];
@@ -1464,6 +1496,7 @@ struct mwifiex_user_scan_cfg {
 	/* Variable number (fixed maximum) of channels to scan up */
 	struct mwifiex_user_scan_chan chan_list[MWIFIEX_USER_SCAN_CHAN_MAX];
 	u16 scan_chan_gap;
+	u8 random_mac[ETH_ALEN];
 } __packed;
 
 #define MWIFIEX_BG_SCAN_CHAN_MAX 38
@@ -1646,7 +1679,7 @@ struct mwifiex_ie_types_sta_info {
 };
 
 struct host_cmd_ds_sta_list {
-	u16 sta_count;
+	__le16 sta_count;
 	u8 tlv[0];
 } __packed;
 
@@ -1667,6 +1700,12 @@ struct mwifiex_ie_types_wmm_param_set {
 	u8 wmm_ie[1];
 };
 
+struct mwifiex_ie_types_mgmt_frame {
+	struct mwifiex_ie_types_header header;
+	__le16 frame_control;
+	u8 frame_contents[0];
+};
+
 struct mwifiex_ie_types_wmm_queue_status {
 	struct mwifiex_ie_types_header header;
 	u8 queue_index;
@@ -2034,26 +2073,26 @@ struct host_cmd_ds_set_bss_mode {
 
 struct host_cmd_ds_pcie_details {
 	/* TX buffer descriptor ring address */
-	u32 txbd_addr_lo;
-	u32 txbd_addr_hi;
+	__le32 txbd_addr_lo;
+	__le32 txbd_addr_hi;
 	/* TX buffer descriptor ring count */
-	u32 txbd_count;
+	__le32 txbd_count;
 
 	/* RX buffer descriptor ring address */
-	u32 rxbd_addr_lo;
-	u32 rxbd_addr_hi;
+	__le32 rxbd_addr_lo;
+	__le32 rxbd_addr_hi;
 	/* RX buffer descriptor ring count */
-	u32 rxbd_count;
+	__le32 rxbd_count;
 
 	/* Event buffer descriptor ring address */
-	u32 evtbd_addr_lo;
-	u32 evtbd_addr_hi;
+	__le32 evtbd_addr_lo;
+	__le32 evtbd_addr_hi;
 	/* Event buffer descriptor ring count */
-	u32 evtbd_count;
+	__le32 evtbd_count;
 
 	/* Sleep cookie buffer physical address */
-	u32 sleep_cookie_addr_lo;
-	u32 sleep_cookie_addr_hi;
+	__le32 sleep_cookie_addr_lo;
+	__le32 sleep_cookie_addr_hi;
 } __packed;
 
 struct mwifiex_ie_types_rssi_threshold {
@@ -2093,8 +2132,8 @@ struct mwifiex_ie_types_mc_group_info {
 	u8 chan_buf_weight;
 	u8 band_config;
 	u8 chan_num;
-	u32 chan_time;
-	u32 reserved;
+	__le32 chan_time;
+	__le32 reserved;
 	union {
 		u8 sdio_func_num;
 		u8 usb_ep_num;
@@ -2185,7 +2224,7 @@ struct host_cmd_ds_robust_coex {
 } __packed;
 
 struct host_cmd_ds_wakeup_reason {
-	u16  wakeup_reason;
+	__le16  wakeup_reason;
 } __packed;
 
 struct host_cmd_ds_gtk_rekey_params {
@@ -2196,6 +2235,10 @@ struct host_cmd_ds_gtk_rekey_params {
 	__le32 replay_ctr_high;
 } __packed;
 
+struct host_cmd_ds_chan_region_cfg {
+	__le16 action;
+} __packed;
+
 struct host_cmd_ds_command {
 	__le16 command;
 	__le16 size;
@@ -2270,6 +2313,7 @@ struct host_cmd_ds_command {
 		struct host_cmd_ds_robust_coex coex;
 		struct host_cmd_ds_wakeup_reason hs_wakeup_reason;
 		struct host_cmd_ds_gtk_rekey_params rekey;
+		struct host_cmd_ds_chan_region_cfg reg_cfg;
 	} params;
 } __packed;
 
diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c
index 1489c90192bd..82839d9f079f 100644
--- a/drivers/net/wireless/marvell/mwifiex/init.c
+++ b/drivers/net/wireless/marvell/mwifiex/init.c
@@ -298,6 +298,7 @@ static void mwifiex_init_adapter(struct mwifiex_adapter *adapter)
 	memset(&adapter->arp_filter, 0, sizeof(adapter->arp_filter));
 	adapter->arp_filter_size = 0;
 	adapter->max_mgmt_ie_index = MAX_MGMT_IE_INDEX;
+	adapter->mfg_mode = mfg_mode;
 	adapter->key_api_major_ver = 0;
 	adapter->key_api_minor_ver = 0;
 	eth_broadcast_addr(adapter->perm_addr);
@@ -553,15 +554,22 @@ int mwifiex_init_fw(struct mwifiex_adapter *adapter)
 				return -1;
 		}
 	}
+	if (adapter->mfg_mode) {
+		adapter->hw_status = MWIFIEX_HW_STATUS_READY;
+		ret = -EINPROGRESS;
+	} else {
+		for (i = 0; i < adapter->priv_num; i++) {
+			if (adapter->priv[i]) {
+				ret = mwifiex_sta_init_cmd(adapter->priv[i],
+							   first_sta, true);
+				if (ret == -1)
+					return -1;
+
+				first_sta = false;
+			}
+
 
-	for (i = 0; i < adapter->priv_num; i++) {
-		if (adapter->priv[i]) {
-			ret = mwifiex_sta_init_cmd(adapter->priv[i], first_sta,
-						   true);
-			if (ret == -1)
-				return -1;
 
-			first_sta = false;
 		}
 	}
 
diff --git a/drivers/net/wireless/marvell/mwifiex/ioctl.h b/drivers/net/wireless/marvell/mwifiex/ioctl.h
index 70429815ff53..536ab834b126 100644
--- a/drivers/net/wireless/marvell/mwifiex/ioctl.h
+++ b/drivers/net/wireless/marvell/mwifiex/ioctl.h
@@ -260,6 +260,7 @@ struct mwifiex_ds_encrypt_key {
 	u8 is_igtk_key;
 	u8 is_current_wep_key;
 	u8 is_rx_seq_valid;
+	u8 is_igtk_def_key;
 };
 
 struct mwifiex_power_cfg {
diff --git a/drivers/net/wireless/marvell/mwifiex/join.c b/drivers/net/wireless/marvell/mwifiex/join.c
index 1c7b00630b90..b89596c18b41 100644
--- a/drivers/net/wireless/marvell/mwifiex/join.c
+++ b/drivers/net/wireless/marvell/mwifiex/join.c
@@ -669,9 +669,8 @@ int mwifiex_ret_802_11_associate(struct mwifiex_private *priv,
 	priv->assoc_rsp_size = min(le16_to_cpu(resp->size) - S_DS_GEN,
 				   sizeof(priv->assoc_rsp_buf));
 
-	memcpy(priv->assoc_rsp_buf, &resp->params, priv->assoc_rsp_size);
-
 	assoc_rsp->a_id = cpu_to_le16(aid);
+	memcpy(priv->assoc_rsp_buf, &resp->params, priv->assoc_rsp_size);
 
 	if (status_code) {
 		priv->adapter->dbg.num_cmd_assoc_failure++;
diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c
index db4925db39aa..2478ccd6f2d9 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.c
+++ b/drivers/net/wireless/marvell/mwifiex/main.c
@@ -23,6 +23,7 @@
 #include "11n.h"
 
 #define VERSION	"1.0"
+#define MFG_FIRMWARE	"mwifiex_mfg.bin"
 
 static unsigned int debug_mask = MWIFIEX_DEFAULT_DEBUG_MASK;
 module_param(debug_mask, uint, 0);
@@ -37,6 +38,10 @@ module_param(driver_mode, ushort, 0);
 MODULE_PARM_DESC(driver_mode,
 		 "station=0x1(default), ap-sta=0x3, station-p2p=0x5, ap-sta-p2p=0x7");
 
+bool mfg_mode;
+module_param(mfg_mode, bool, 0);
+MODULE_PARM_DESC(mfg_mode, "manufacturing mode enable:1, disable:0");
+
 /*
  * This function registers the device and performs all the necessary
  * initializations.
@@ -139,6 +144,8 @@ static int mwifiex_unregister(struct mwifiex_adapter *adapter)
 		adapter->nd_info = NULL;
 	}
 
+	kfree(adapter->regd);
+
 	vfree(adapter->chan_stats);
 	kfree(adapter);
 	return 0;
@@ -486,9 +493,11 @@ static void mwifiex_free_adapter(struct mwifiex_adapter *adapter)
  */
 static void mwifiex_terminate_workqueue(struct mwifiex_adapter *adapter)
 {
-	flush_workqueue(adapter->workqueue);
-	destroy_workqueue(adapter->workqueue);
-	adapter->workqueue = NULL;
+	if (adapter->workqueue) {
+		flush_workqueue(adapter->workqueue);
+		destroy_workqueue(adapter->workqueue);
+		adapter->workqueue = NULL;
+	}
 
 	if (adapter->rx_workqueue) {
 		flush_workqueue(adapter->rx_workqueue);
@@ -559,16 +568,21 @@ static void mwifiex_fw_dpc(const struct firmware *firmware, void *context)
 		goto done;
 	}
 	/* Wait for mwifiex_init to complete */
-	wait_event_interruptible(adapter->init_wait_q,
-				 adapter->init_wait_q_woken);
-	if (adapter->hw_status != MWIFIEX_HW_STATUS_READY)
-		goto err_init_fw;
+	if (!adapter->mfg_mode) {
+		wait_event_interruptible(adapter->init_wait_q,
+					 adapter->init_wait_q_woken);
+		if (adapter->hw_status != MWIFIEX_HW_STATUS_READY)
+			goto err_init_fw;
+	}
 
 	priv = adapter->priv[MWIFIEX_BSS_ROLE_STA];
-	if (mwifiex_register_cfg80211(adapter)) {
-		mwifiex_dbg(adapter, ERROR,
-			    "cannot register with cfg80211\n");
-		goto err_init_fw;
+
+	if (!adapter->wiphy) {
+		if (mwifiex_register_cfg80211(adapter)) {
+			mwifiex_dbg(adapter, ERROR,
+				    "cannot register with cfg80211\n");
+			goto err_init_fw;
+		}
 	}
 
 	if (mwifiex_init_channel_scan_gap(adapter)) {
@@ -662,16 +676,41 @@ done:
 /*
  * This function initializes the hardware and gets firmware.
  */
-static int mwifiex_init_hw_fw(struct mwifiex_adapter *adapter)
+static int mwifiex_init_hw_fw(struct mwifiex_adapter *adapter,
+			      bool req_fw_nowait)
 {
 	int ret;
 
-	ret = request_firmware_nowait(THIS_MODULE, 1, adapter->fw_name,
-				      adapter->dev, GFP_KERNEL, adapter,
-				      mwifiex_fw_dpc);
-	if (ret < 0)
-		mwifiex_dbg(adapter, ERROR,
-			    "request_firmware_nowait error %d\n", ret);
+	/* Override default firmware with manufacturing one if
+	 * manufacturing mode is enabled
+	 */
+	if (mfg_mode) {
+		if (strlcpy(adapter->fw_name, MFG_FIRMWARE,
+			    sizeof(adapter->fw_name)) >=
+			    sizeof(adapter->fw_name)) {
+			pr_err("%s: fw_name too long!\n", __func__);
+			return -1;
+		}
+	}
+
+	if (req_fw_nowait) {
+		ret = request_firmware_nowait(THIS_MODULE, 1, adapter->fw_name,
+					      adapter->dev, GFP_KERNEL, adapter,
+					      mwifiex_fw_dpc);
+		if (ret < 0)
+			mwifiex_dbg(adapter, ERROR,
+				    "request_firmware_nowait error %d\n", ret);
+	} else {
+		ret = request_firmware(&adapter->firmware,
+				       adapter->fw_name,
+				       adapter->dev);
+		if (ret < 0)
+			mwifiex_dbg(adapter, ERROR,
+				    "request_firmware error %d\n", ret);
+		else
+			mwifiex_fw_dpc(adapter->firmware, (void *)adapter);
+	}
+
 	return ret;
 }
 
@@ -1321,6 +1360,199 @@ static void mwifiex_main_work_queue(struct work_struct *work)
 }
 
 /*
+ * This function gets called during PCIe function level reset. Required
+ * code is extracted from mwifiex_remove_card()
+ */
+static int
+mwifiex_shutdown_sw(struct mwifiex_adapter *adapter, struct semaphore *sem)
+{
+	struct mwifiex_private *priv;
+	int i;
+
+	if (!adapter)
+		goto exit_return;
+
+	if (down_interruptible(sem))
+		goto exit_sem_err;
+
+	priv = mwifiex_get_priv(adapter, MWIFIEX_BSS_ROLE_ANY);
+	mwifiex_deauthenticate(priv, NULL);
+
+	/* We can no longer handle interrupts once we start doing the teardown
+	 * below.
+	 */
+	if (adapter->if_ops.disable_int)
+		adapter->if_ops.disable_int(adapter);
+
+	adapter->surprise_removed = true;
+	mwifiex_terminate_workqueue(adapter);
+
+	/* Stop data */
+	for (i = 0; i < adapter->priv_num; i++) {
+		priv = adapter->priv[i];
+		if (priv && priv->netdev) {
+			mwifiex_stop_net_dev_queue(priv->netdev, adapter);
+			if (netif_carrier_ok(priv->netdev))
+				netif_carrier_off(priv->netdev);
+			netif_device_detach(priv->netdev);
+		}
+	}
+
+	mwifiex_dbg(adapter, CMD, "cmd: calling mwifiex_shutdown_drv...\n");
+	adapter->init_wait_q_woken = false;
+
+	if (mwifiex_shutdown_drv(adapter) == -EINPROGRESS)
+		wait_event_interruptible(adapter->init_wait_q,
+					 adapter->init_wait_q_woken);
+	if (adapter->if_ops.down_dev)
+		adapter->if_ops.down_dev(adapter);
+
+	mwifiex_dbg(adapter, CMD, "cmd: mwifiex_shutdown_drv done\n");
+	if (atomic_read(&adapter->rx_pending) ||
+	    atomic_read(&adapter->tx_pending) ||
+	    atomic_read(&adapter->cmd_pending)) {
+		mwifiex_dbg(adapter, ERROR,
+			    "rx_pending=%d, tx_pending=%d,\t"
+			    "cmd_pending=%d\n",
+			    atomic_read(&adapter->rx_pending),
+			    atomic_read(&adapter->tx_pending),
+			    atomic_read(&adapter->cmd_pending));
+	}
+
+	for (i = 0; i < adapter->priv_num; i++) {
+		priv = adapter->priv[i];
+		if (!priv)
+			continue;
+		rtnl_lock();
+		if (priv->netdev &&
+		    priv->wdev.iftype != NL80211_IFTYPE_UNSPECIFIED)
+			mwifiex_del_virtual_intf(adapter->wiphy, &priv->wdev);
+		rtnl_unlock();
+	}
+
+	up(sem);
+exit_sem_err:
+	mwifiex_dbg(adapter, INFO, "%s, successful\n", __func__);
+exit_return:
+	return 0;
+}
+
+/* This function gets called during PCIe function level reset. Required
+ * code is extracted from mwifiex_add_card()
+ */
+static int
+mwifiex_reinit_sw(struct mwifiex_adapter *adapter, struct semaphore *sem,
+		  struct mwifiex_if_ops *if_ops, u8 iface_type)
+{
+	char fw_name[32];
+	struct pcie_service_card *card = adapter->card;
+
+	if (down_interruptible(sem))
+		goto exit_sem_err;
+
+	mwifiex_init_lock_list(adapter);
+	if (adapter->if_ops.up_dev)
+		adapter->if_ops.up_dev(adapter);
+
+	adapter->iface_type = iface_type;
+	adapter->card_sem = sem;
+
+	adapter->hw_status = MWIFIEX_HW_STATUS_INITIALIZING;
+	adapter->surprise_removed = false;
+	init_waitqueue_head(&adapter->init_wait_q);
+	adapter->is_suspended = false;
+	adapter->hs_activated = false;
+	init_waitqueue_head(&adapter->hs_activate_wait_q);
+	init_waitqueue_head(&adapter->cmd_wait_q.wait);
+	adapter->cmd_wait_q.status = 0;
+	adapter->scan_wait_q_woken = false;
+
+	if ((num_possible_cpus() > 1) || adapter->iface_type == MWIFIEX_USB)
+		adapter->rx_work_enabled = true;
+
+	adapter->workqueue =
+		alloc_workqueue("MWIFIEX_WORK_QUEUE",
+				WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
+	if (!adapter->workqueue)
+		goto err_kmalloc;
+
+	INIT_WORK(&adapter->main_work, mwifiex_main_work_queue);
+
+	if (adapter->rx_work_enabled) {
+		adapter->rx_workqueue = alloc_workqueue("MWIFIEX_RX_WORK_QUEUE",
+							WQ_HIGHPRI |
+							WQ_MEM_RECLAIM |
+							WQ_UNBOUND, 1);
+		if (!adapter->rx_workqueue)
+			goto err_kmalloc;
+		INIT_WORK(&adapter->rx_work, mwifiex_rx_work_queue);
+	}
+
+	/* Register the device. Fill up the private data structure with
+	 * relevant information from the card. Some code extracted from
+	 * mwifiex_register_dev()
+	 */
+	mwifiex_dbg(adapter, INFO, "%s, mwifiex_init_hw_fw()...\n", __func__);
+	strcpy(fw_name, adapter->fw_name);
+	strcpy(adapter->fw_name, PCIE8997_DEFAULT_WIFIFW_NAME);
+
+	adapter->tx_buf_size = card->pcie.tx_buf_size;
+	adapter->ext_scan = card->pcie.can_ext_scan;
+	if (mwifiex_init_hw_fw(adapter, false)) {
+		strcpy(adapter->fw_name, fw_name);
+		mwifiex_dbg(adapter, ERROR,
+			    "%s: firmware init failed\n", __func__);
+		goto err_init_fw;
+	}
+	strcpy(adapter->fw_name, fw_name);
+	mwifiex_dbg(adapter, INFO, "%s, successful\n", __func__);
+	up(sem);
+	return 0;
+
+err_init_fw:
+	mwifiex_dbg(adapter, ERROR, "info: %s: unregister device\n", __func__);
+	if (adapter->if_ops.unregister_dev)
+		adapter->if_ops.unregister_dev(adapter);
+	if (adapter->hw_status == MWIFIEX_HW_STATUS_READY) {
+		mwifiex_dbg(adapter, ERROR,
+			    "info: %s: shutdown mwifiex\n", __func__);
+		adapter->init_wait_q_woken = false;
+
+		if (mwifiex_shutdown_drv(adapter) == -EINPROGRESS)
+			wait_event_interruptible(adapter->init_wait_q,
+						 adapter->init_wait_q_woken);
+	}
+
+err_kmalloc:
+	mwifiex_terminate_workqueue(adapter);
+	adapter->surprise_removed = true;
+	up(sem);
+exit_sem_err:
+	mwifiex_dbg(adapter, INFO, "%s, error\n", __func__);
+
+	return -1;
+}
+
+/* This function processes pre and post PCIe function level resets.
+ * It performs software cleanup without touching PCIe specific code.
+ * Also, during initialization PCIe stuff is skipped.
+ */
+void mwifiex_do_flr(struct mwifiex_adapter *adapter, bool prepare)
+{
+	struct mwifiex_if_ops if_ops;
+
+	if (!prepare) {
+		mwifiex_reinit_sw(adapter, adapter->card_sem, &if_ops,
+				  adapter->iface_type);
+	} else {
+		memcpy(&if_ops, &adapter->if_ops,
+		       sizeof(struct mwifiex_if_ops));
+		mwifiex_shutdown_sw(adapter, adapter->card_sem);
+	}
+}
+EXPORT_SYMBOL_GPL(mwifiex_do_flr);
+
+/*
  * This function adds the card.
  *
  * This function follows the following major steps to set up the device -
@@ -1391,7 +1623,7 @@ mwifiex_add_card(void *card, struct semaphore *sem,
 		goto err_registerdev;
 	}
 
-	if (mwifiex_init_hw_fw(adapter)) {
+	if (mwifiex_init_hw_fw(adapter, true)) {
 		pr_err("%s: firmware init failed\n", __func__);
 		goto err_init_fw;
 	}
diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h
index 9f6bb400bdae..26df28f4bfb2 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.h
+++ b/drivers/net/wireless/marvell/mwifiex/main.h
@@ -58,6 +58,7 @@
 #include "sdio.h"
 
 extern const char driver_version[];
+extern bool mfg_mode;
 
 struct mwifiex_adapter;
 struct mwifiex_private;
@@ -675,6 +676,7 @@ struct mwifiex_private {
 	struct mwifiex_user_scan_chan hidden_chan[MWIFIEX_USER_SCAN_CHAN_MAX];
 	u8 assoc_resp_ht_param;
 	bool ht_param_present;
+	u8 random_mac[ETH_ALEN];
 };
 
 
@@ -827,6 +829,8 @@ struct mwifiex_if_ops {
 	void (*deaggr_pkt)(struct mwifiex_adapter *, struct sk_buff *);
 	void (*multi_port_resync)(struct mwifiex_adapter *);
 	bool (*is_port_ready)(struct mwifiex_private *);
+	void (*down_dev)(struct mwifiex_adapter *);
+	void (*up_dev)(struct mwifiex_adapter *);
 };
 
 struct mwifiex_adapter {
@@ -989,6 +993,7 @@ struct mwifiex_adapter {
 	u32 drv_info_size;
 	bool scan_chan_gap_enabled;
 	struct sk_buff_head rx_data_q;
+	bool mfg_mode;
 	struct mwifiex_chan_stats *chan_stats;
 	u32 num_in_chan_stats;
 	int survey_idx;
@@ -1004,6 +1009,7 @@ struct mwifiex_adapter {
 	bool usb_mc_status;
 	bool usb_mc_setup;
 	struct cfg80211_wowlan_nd_info *nd_info;
+	struct ieee80211_regdomain *regd;
 };
 
 void mwifiex_process_tx_queue(struct mwifiex_adapter *adapter);
@@ -1625,4 +1631,5 @@ void mwifiex_debugfs_remove(void);
 void mwifiex_dev_debugfs_init(struct mwifiex_private *priv);
 void mwifiex_dev_debugfs_remove(struct mwifiex_private *priv);
 #endif
+void mwifiex_do_flr(struct mwifiex_adapter *adapter, bool prepare);
 #endif /* !_MWIFIEX_MAIN_H_ */
diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c
index 453ab6ad4784..3c3c4f197da8 100644
--- a/drivers/net/wireless/marvell/mwifiex/pcie.c
+++ b/drivers/net/wireless/marvell/mwifiex/pcie.c
@@ -225,7 +225,7 @@ static void mwifiex_pcie_remove(struct pci_dev *pdev)
 	if (!adapter || !adapter->priv_num)
 		return;
 
-	if (user_rmmod) {
+	if (user_rmmod && !adapter->mfg_mode) {
 #ifdef CONFIG_PM_SLEEP
 		if (adapter->is_suspended)
 			mwifiex_pcie_resume(&pdev->dev);
@@ -277,6 +277,52 @@ static const struct pci_device_id mwifiex_ids[] = {
 
 MODULE_DEVICE_TABLE(pci, mwifiex_ids);
 
+static void mwifiex_pcie_reset_notify(struct pci_dev *pdev, bool prepare)
+{
+	struct mwifiex_adapter *adapter;
+	struct pcie_service_card *card;
+
+	if (!pdev) {
+		pr_err("%s: PCIe device is not specified\n", __func__);
+		return;
+	}
+
+	card = (struct pcie_service_card *)pci_get_drvdata(pdev);
+	if (!card || !card->adapter) {
+		pr_err("%s: Card or adapter structure is not valid (%ld)\n",
+		       __func__, (long)card);
+		return;
+	}
+
+	adapter = card->adapter;
+	mwifiex_dbg(adapter, INFO,
+		    "%s: vendor=0x%4.04x device=0x%4.04x rev=%d %s\n",
+		    __func__, pdev->vendor, pdev->device,
+		    pdev->revision,
+		    prepare ? "Pre-FLR" : "Post-FLR");
+
+	if (prepare) {
+		/* Kernel would be performing FLR after this notification.
+		 * Cleanup all software without cleaning anything related to
+		 * PCIe and HW.
+		 */
+		mwifiex_do_flr(adapter, prepare);
+		adapter->surprise_removed = true;
+	} else {
+		/* Kernel stores and restores PCIe function context before and
+		 * after performing FLR respectively. Reconfigure the software
+		 * and firmware including firmware redownload
+		 */
+		adapter->surprise_removed = false;
+		mwifiex_do_flr(adapter, prepare);
+	}
+	mwifiex_dbg(adapter, INFO, "%s, successful\n", __func__);
+}
+
+static const struct pci_error_handlers mwifiex_pcie_err_handler[] = {
+		{ .reset_notify = mwifiex_pcie_reset_notify, },
+};
+
 #ifdef CONFIG_PM_SLEEP
 /* Power Management Hooks */
 static SIMPLE_DEV_PM_OPS(mwifiex_pcie_pm_ops, mwifiex_pcie_suspend,
@@ -295,6 +341,7 @@ static struct pci_driver __refdata mwifiex_pcie = {
 	},
 #endif
 	.shutdown = mwifiex_pcie_shutdown,
+	.err_handler = mwifiex_pcie_err_handler,
 };
 
 /*
@@ -1956,8 +2003,6 @@ static int mwifiex_prog_fw_w_helper(struct mwifiex_adapter *adapter,
 			if (firmware_len - offset < txlen)
 				txlen = firmware_len - offset;
 
-			mwifiex_dbg(adapter, INFO, ".");
-
 			tx_blocks = (txlen + card->pcie.blksz_fw_dl - 1) /
 				    card->pcie.blksz_fw_dl;
 
@@ -2043,6 +2088,10 @@ mwifiex_check_fw_status(struct mwifiex_adapter *adapter, u32 poll_num)
 			ret = -1;
 		else
 			ret = 0;
+
+		mwifiex_dbg(adapter, INFO, "Try %d if FW is ready <%d,%#x>",
+			    tries, ret, firmware_stat);
+
 		if (ret)
 			continue;
 		if (firmware_stat == FIRMWARE_READY_PCIE) {
@@ -2074,8 +2123,7 @@ mwifiex_check_winner_status(struct mwifiex_adapter *adapter)
 		adapter->winner = 1;
 	} else {
 		mwifiex_dbg(adapter, ERROR,
-			    "PCI-E is not the winner <%#x,%d>, exit dnld\n",
-			    ret, adapter->winner);
+			    "PCI-E is not the winner <%#x>", winner);
 	}
 
 	return ret;
@@ -2863,7 +2911,7 @@ static int mwifiex_pcie_request_irq(struct mwifiex_adapter *adapter)
 static void mwifiex_pcie_get_fw_name(struct mwifiex_adapter *adapter)
 {
 	int revision_id = 0;
-	int version;
+	int version, magic;
 	struct pcie_service_card *card = adapter->card;
 
 	switch (card->dev->device) {
@@ -2888,30 +2936,19 @@ static void mwifiex_pcie_get_fw_name(struct mwifiex_adapter *adapter)
 		}
 		break;
 	case PCIE_DEVICE_ID_MARVELL_88W8997:
-		mwifiex_read_reg(adapter, 0x0c48, &revision_id);
+		mwifiex_read_reg(adapter, 0x8, &revision_id);
 		mwifiex_read_reg(adapter, 0x0cd0, &version);
+		mwifiex_read_reg(adapter, 0x0cd4, &magic);
+		revision_id &= 0xff;
 		version &= 0x7;
-		switch (revision_id) {
-		case PCIE8997_V2:
-			if (version == CHIP_VER_PCIEUART)
-				strcpy(adapter->fw_name,
-				       PCIEUART8997_FW_NAME_V2);
-			else
-				strcpy(adapter->fw_name,
-				       PCIEUSB8997_FW_NAME_V2);
-			break;
-		case PCIE8997_Z:
-			if (version == CHIP_VER_PCIEUART)
-				strcpy(adapter->fw_name,
-				       PCIEUART8997_FW_NAME_Z);
-			else
-				strcpy(adapter->fw_name,
-				       PCIEUSB8997_FW_NAME_Z);
-			break;
-		default:
-			strcpy(adapter->fw_name, PCIE8997_DEFAULT_FW_NAME);
-			break;
-		}
+		magic &= 0xff;
+		if (revision_id == PCIE8997_A1 &&
+		    magic == CHIP_MAGIC_VALUE &&
+		    version == CHIP_VER_PCIEUART)
+			strcpy(adapter->fw_name, PCIEUART8997_FW_NAME_V4);
+		else
+			strcpy(adapter->fw_name, PCIEUSB8997_FW_NAME_V4);
+		break;
 	default:
 		break;
 	}
@@ -2952,7 +2989,6 @@ static int mwifiex_register_dev(struct mwifiex_adapter *adapter)
 static void mwifiex_unregister_dev(struct mwifiex_adapter *adapter)
 {
 	struct pcie_service_card *card = adapter->card;
-	const struct mwifiex_pcie_card_reg *reg;
 	struct pci_dev *pdev;
 	int i;
 
@@ -2976,8 +3012,90 @@ static void mwifiex_unregister_dev(struct mwifiex_adapter *adapter)
 			if (card->msi_enable)
 				pci_disable_msi(pdev);
 	       }
+	}
+}
+
+/* This function initializes the PCI-E host memory space, WCB rings, etc.
+ *
+ * The following initializations steps are followed -
+ *      - Allocate TXBD ring buffers
+ *      - Allocate RXBD ring buffers
+ *      - Allocate event BD ring buffers
+ *      - Allocate command response ring buffer
+ *      - Allocate sleep cookie buffer
+ * Part of mwifiex_pcie_init(), not reset the PCIE registers
+ */
+static void mwifiex_pcie_up_dev(struct mwifiex_adapter *adapter)
+{
+	struct pcie_service_card *card = adapter->card;
+	int ret;
+	struct pci_dev *pdev = card->dev;
+	const struct mwifiex_pcie_card_reg *reg = card->pcie.reg;
 
-		reg = card->pcie.reg;
+	card->cmdrsp_buf = NULL;
+	ret = mwifiex_pcie_create_txbd_ring(adapter);
+	if (ret) {
+		mwifiex_dbg(adapter, ERROR, "Failed to create txbd ring\n");
+		goto err_cre_txbd;
+	}
+
+	ret = mwifiex_pcie_create_rxbd_ring(adapter);
+	if (ret) {
+		mwifiex_dbg(adapter, ERROR, "Failed to create rxbd ring\n");
+		goto err_cre_rxbd;
+	}
+
+	ret = mwifiex_pcie_create_evtbd_ring(adapter);
+	if (ret) {
+		mwifiex_dbg(adapter, ERROR, "Failed to create evtbd ring\n");
+		goto err_cre_evtbd;
+	}
+
+	ret = mwifiex_pcie_alloc_cmdrsp_buf(adapter);
+	if (ret) {
+		mwifiex_dbg(adapter, ERROR, "Failed to allocate cmdbuf buffer\n");
+		goto err_alloc_cmdbuf;
+	}
+
+	if (reg->sleep_cookie) {
+		ret = mwifiex_pcie_alloc_sleep_cookie_buf(adapter);
+		if (ret) {
+			mwifiex_dbg(adapter, ERROR, "Failed to allocate sleep_cookie buffer\n");
+			goto err_alloc_cookie;
+		}
+	} else {
+		card->sleep_cookie_vbase = NULL;
+	}
+	return;
+
+err_alloc_cookie:
+	mwifiex_pcie_delete_cmdrsp_buf(adapter);
+err_alloc_cmdbuf:
+	mwifiex_pcie_delete_evtbd_ring(adapter);
+err_cre_evtbd:
+	mwifiex_pcie_delete_rxbd_ring(adapter);
+err_cre_rxbd:
+	mwifiex_pcie_delete_txbd_ring(adapter);
+err_cre_txbd:
+	pci_iounmap(pdev, card->pci_mmap1);
+}
+
+/* This function cleans up the PCI-E host memory space.
+ * Some code is extracted from mwifiex_unregister_dev()
+ *
+ */
+static void mwifiex_pcie_down_dev(struct mwifiex_adapter *adapter)
+{
+	struct pcie_service_card *card = adapter->card;
+	const struct mwifiex_pcie_card_reg *reg = card->pcie.reg;
+
+	if (mwifiex_write_reg(adapter, reg->drv_rdy, 0x00000000))
+		mwifiex_dbg(adapter, ERROR, "Failed to write driver not-ready signature\n");
+
+	adapter->seq_num = 0;
+	adapter->tx_buf_size = MWIFIEX_TX_DATA_BUF_SIZE_4K;
+
+	if (card) {
 		if (reg->sleep_cookie)
 			mwifiex_pcie_delete_sleep_cookie_buf(adapter);
 
@@ -2987,6 +3105,8 @@ static void mwifiex_unregister_dev(struct mwifiex_adapter *adapter)
 		mwifiex_pcie_delete_txbd_ring(adapter);
 		card->cmdrsp_buf = NULL;
 	}
+
+	return;
 }
 
 static struct mwifiex_if_ops pcie_ops = {
@@ -3013,6 +3133,8 @@ static struct mwifiex_if_ops pcie_ops = {
 	.clean_pcie_ring =		mwifiex_clean_pcie_ring_buf,
 	.reg_dump =			mwifiex_pcie_reg_dump,
 	.device_dump =			mwifiex_pcie_device_dump,
+	.down_dev =			mwifiex_pcie_down_dev,
+	.up_dev =			mwifiex_pcie_up_dev,
 };
 
 /*
diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.h b/drivers/net/wireless/marvell/mwifiex/pcie.h
index f05061cea5cd..46f99cae9399 100644
--- a/drivers/net/wireless/marvell/mwifiex/pcie.h
+++ b/drivers/net/wireless/marvell/mwifiex/pcie.h
@@ -32,11 +32,9 @@
 #define PCIE8897_DEFAULT_FW_NAME "mrvl/pcie8897_uapsta.bin"
 #define PCIE8897_A0_FW_NAME "mrvl/pcie8897_uapsta_a0.bin"
 #define PCIE8897_B0_FW_NAME "mrvl/pcie8897_uapsta.bin"
-#define PCIE8997_DEFAULT_FW_NAME "mrvl/pcieusb8997_combo_v2.bin"
-#define PCIEUART8997_FW_NAME_Z "mrvl/pcieuart8997_combo.bin"
-#define PCIEUART8997_FW_NAME_V2 "mrvl/pcieuart8997_combo_v2.bin"
-#define PCIEUSB8997_FW_NAME_Z "mrvl/pcieusb8997_combo.bin"
-#define PCIEUSB8997_FW_NAME_V2 "mrvl/pcieusb8997_combo_v2.bin"
+#define PCIEUART8997_FW_NAME_V4 "mrvl/pcieuart8997_combo_v4.bin"
+#define PCIEUSB8997_FW_NAME_V4 "mrvl/pcieusb8997_combo_v4.bin"
+#define PCIE8997_DEFAULT_WIFIFW_NAME "mrvl/pcie8997_wlan_v4.bin"
 
 #define PCIE_VENDOR_ID_MARVELL              (0x11ab)
 #define PCIE_VENDOR_ID_V2_MARVELL           (0x1b4b)
@@ -46,9 +44,10 @@
 
 #define PCIE8897_A0	0x1100
 #define PCIE8897_B0	0x1200
-#define PCIE8997_Z	0x0
-#define PCIE8997_V2	0x471
+#define PCIE8997_A0	0x10
+#define PCIE8997_A1	0x11
 #define CHIP_VER_PCIEUART	0x3
+#define CHIP_MAGIC_VALUE	0x24
 
 /* Constants for Buffer Descriptor (BD) rings */
 #define MWIFIEX_MAX_TXRX_BD			0x20
diff --git a/drivers/net/wireless/marvell/mwifiex/scan.c b/drivers/net/wireless/marvell/mwifiex/scan.c
index 21ec84794d0c..97c9765b5bc6 100644
--- a/drivers/net/wireless/marvell/mwifiex/scan.c
+++ b/drivers/net/wireless/marvell/mwifiex/scan.c
@@ -820,6 +820,7 @@ mwifiex_config_scan(struct mwifiex_private *priv,
 	struct mwifiex_adapter *adapter = priv->adapter;
 	struct mwifiex_ie_types_num_probes *num_probes_tlv;
 	struct mwifiex_ie_types_scan_chan_gap *chan_gap_tlv;
+	struct mwifiex_ie_types_random_mac *random_mac_tlv;
 	struct mwifiex_ie_types_wildcard_ssid_params *wildcard_ssid_tlv;
 	struct mwifiex_ie_types_bssid_list *bssid_tlv;
 	u8 *tlv_pos;
@@ -835,6 +836,7 @@ mwifiex_config_scan(struct mwifiex_private *priv,
 	u8 ssid_filter;
 	struct mwifiex_ie_types_htcap *ht_cap;
 	struct mwifiex_ie_types_bss_mode *bss_mode;
+	const u8 zero_mac[6] = {0, 0, 0, 0, 0, 0};
 
 	/* The tlv_buf_len is calculated for each scan command.  The TLVs added
 	   in this routine will be preserved since the routine that sends the
@@ -967,6 +969,18 @@ mwifiex_config_scan(struct mwifiex_private *priv,
 			tlv_pos +=
 				  sizeof(struct mwifiex_ie_types_scan_chan_gap);
 		}
+
+		if (!ether_addr_equal(user_scan_in->random_mac, zero_mac)) {
+			random_mac_tlv = (void *)tlv_pos;
+			random_mac_tlv->header.type =
+					 cpu_to_le16(TLV_TYPE_RANDOM_MAC);
+			random_mac_tlv->header.len =
+				    cpu_to_le16(sizeof(random_mac_tlv->mac));
+			ether_addr_copy(random_mac_tlv->mac,
+					user_scan_in->random_mac);
+			tlv_pos +=
+				  sizeof(struct mwifiex_ie_types_random_mac);
+		}
 	} else {
 		scan_cfg_out->bss_mode = (u8) adapter->scan_mode;
 		num_probes = adapter->scan_probes;
@@ -1922,6 +1936,7 @@ mwifiex_active_scan_req_for_passive_chan(struct mwifiex_private *priv)
 	}
 
 	adapter->active_scan_triggered = true;
+	ether_addr_copy(user_scan_cfg->random_mac, priv->random_mac);
 	user_scan_cfg->num_ssids = priv->scan_request->n_ssids;
 	user_scan_cfg->ssid_list = priv->scan_request->ssids;
 
@@ -2179,18 +2194,14 @@ int mwifiex_ret_802_11_scan(struct mwifiex_private *priv,
 
 		if (chan_band_tlv && adapter->nd_info) {
 			adapter->nd_info->matches[idx] =
-				kzalloc(sizeof(*pmatch) +
-				sizeof(u32), GFP_ATOMIC);
+				kzalloc(sizeof(*pmatch) + sizeof(u32),
+					GFP_ATOMIC);
 
 			pmatch = adapter->nd_info->matches[idx];
 
 			if (pmatch) {
-				memset(pmatch, 0, sizeof(*pmatch));
-				if (chan_band_tlv) {
-					pmatch->n_channels = 1;
-					pmatch->channels[0] =
-						chan_band->chan_number;
-				}
+				pmatch->n_channels = 1;
+				pmatch->channels[0] = chan_band->chan_number;
 			}
 		}
 
@@ -2761,6 +2772,7 @@ static int mwifiex_scan_specific_ssid(struct mwifiex_private *priv,
 	if (!scan_cfg)
 		return -ENOMEM;
 
+	ether_addr_copy(scan_cfg->random_mac, priv->random_mac);
 	scan_cfg->ssid_list = req_ssid;
 	scan_cfg->num_ssids = 1;
 
diff --git a/drivers/net/wireless/marvell/mwifiex/sdio.c b/drivers/net/wireless/marvell/mwifiex/sdio.c
index d3e1561ca075..8718950004f3 100644
--- a/drivers/net/wireless/marvell/mwifiex/sdio.c
+++ b/drivers/net/wireless/marvell/mwifiex/sdio.c
@@ -122,9 +122,11 @@ static int mwifiex_sdio_probe_of(struct device *dev, struct sdio_mmc_card *card)
 					       IRQF_TRIGGER_LOW,
 					       "wifi_wake", cfg);
 			if (ret) {
-				dev_err(dev,
+				dev_dbg(dev,
 					"Failed to request irq_wifi %d (%d)\n",
 					cfg->irq_wifi, ret);
+				card->plt_wake_cfg = NULL;
+				return 0;
 			}
 			disable_irq(cfg->irq_wifi);
 		}
@@ -289,7 +291,7 @@ mwifiex_sdio_remove(struct sdio_func *func)
 
 	mwifiex_dbg(adapter, INFO, "info: SDIO func num=%d\n", func->num);
 
-	if (user_rmmod) {
+	if (user_rmmod && !adapter->mfg_mode) {
 		if (adapter->is_suspended)
 			mwifiex_sdio_resume(adapter->dev);
 
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
index 7897037b0992..2a162c33d271 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
@@ -598,6 +598,11 @@ static int mwifiex_set_aes_key_v2(struct mwifiex_private *priv,
 		memcpy(km->key_param_set.key_params.cmac_aes.key,
 		       enc_key->key_material, enc_key->key_len);
 		len += sizeof(struct mwifiex_cmac_aes_param);
+	} else if (enc_key->is_igtk_def_key) {
+		mwifiex_dbg(adapter, INFO,
+			    "%s: Set CMAC default Key index\n", __func__);
+		km->key_param_set.key_type = KEY_TYPE_ID_AES_CMAC_DEF;
+		km->key_param_set.key_idx = enc_key->key_index & KEY_INDEX_MASK;
 	} else {
 		mwifiex_dbg(adapter, INFO,
 			    "%s: Set AES Key\n", __func__);
@@ -706,15 +711,10 @@ mwifiex_cmd_802_11_key_material_v2(struct mwifiex_private *priv,
 				    (priv->wep_key_curr_index & KEY_INDEX_MASK))
 					key_info |= KEY_DEFAULT;
 			} else {
-				if (mac) {
-					if (is_broadcast_ether_addr(mac))
-						key_info |= KEY_MCAST;
-					else
-						key_info |= KEY_UNICAST |
-							    KEY_DEFAULT;
-				} else {
+				if (is_broadcast_ether_addr(mac))
 					key_info |= KEY_MCAST;
-				}
+				else
+					key_info |= KEY_UNICAST | KEY_DEFAULT;
 			}
 		}
 		km->key_param_set.key_info = cpu_to_le16(key_info);
@@ -1244,20 +1244,23 @@ mwifiex_cmd_pcie_host_spec(struct mwifiex_private *priv,
 		return 0;
 
 	/* Send the ring base addresses and count to firmware */
-	host_spec->txbd_addr_lo = (u32)(card->txbd_ring_pbase);
-	host_spec->txbd_addr_hi = (u32)(((u64)card->txbd_ring_pbase)>>32);
-	host_spec->txbd_count = MWIFIEX_MAX_TXRX_BD;
-	host_spec->rxbd_addr_lo = (u32)(card->rxbd_ring_pbase);
-	host_spec->rxbd_addr_hi = (u32)(((u64)card->rxbd_ring_pbase)>>32);
-	host_spec->rxbd_count = MWIFIEX_MAX_TXRX_BD;
-	host_spec->evtbd_addr_lo = (u32)(card->evtbd_ring_pbase);
-	host_spec->evtbd_addr_hi = (u32)(((u64)card->evtbd_ring_pbase)>>32);
-	host_spec->evtbd_count = MWIFIEX_MAX_EVT_BD;
+	host_spec->txbd_addr_lo = cpu_to_le32((u32)(card->txbd_ring_pbase));
+	host_spec->txbd_addr_hi =
+			cpu_to_le32((u32)(((u64)card->txbd_ring_pbase) >> 32));
+	host_spec->txbd_count = cpu_to_le32(MWIFIEX_MAX_TXRX_BD);
+	host_spec->rxbd_addr_lo = cpu_to_le32((u32)(card->rxbd_ring_pbase));
+	host_spec->rxbd_addr_hi =
+			cpu_to_le32((u32)(((u64)card->rxbd_ring_pbase) >> 32));
+	host_spec->rxbd_count = cpu_to_le32(MWIFIEX_MAX_TXRX_BD);
+	host_spec->evtbd_addr_lo = cpu_to_le32((u32)(card->evtbd_ring_pbase));
+	host_spec->evtbd_addr_hi =
+			cpu_to_le32((u32)(((u64)card->evtbd_ring_pbase) >> 32));
+	host_spec->evtbd_count = cpu_to_le32(MWIFIEX_MAX_EVT_BD);
 	if (card->sleep_cookie_vbase) {
 		host_spec->sleep_cookie_addr_lo =
-						(u32)(card->sleep_cookie_pbase);
-		host_spec->sleep_cookie_addr_hi =
-				 (u32)(((u64)(card->sleep_cookie_pbase)) >> 32);
+				cpu_to_le32((u32)(card->sleep_cookie_pbase));
+		host_spec->sleep_cookie_addr_hi = cpu_to_le32((u32)(((u64)
+					(card->sleep_cookie_pbase)) >> 32));
 		mwifiex_dbg(priv->adapter, INFO,
 			    "sleep_cook_lo phy addr: 0x%x\n",
 			    host_spec->sleep_cookie_addr_lo);
@@ -1482,7 +1485,7 @@ int mwifiex_dnld_dt_cfgdata(struct mwifiex_private *priv,
 			continue;
 
 		/* property header is 6 bytes, data must fit in cmd buffer */
-		if (prop && prop->value && prop->length > 6 &&
+		if (prop->value && prop->length > 6 &&
 		    prop->length <= MWIFIEX_SIZE_OF_CMD_BUFFER - S_DS_GEN) {
 			ret = mwifiex_send_cmd(priv, HostCmd_CMD_CFG_DATA,
 					       HostCmd_ACT_GEN_SET, 0,
@@ -1596,6 +1599,21 @@ static int mwifiex_cmd_gtk_rekey_offload(struct mwifiex_private *priv,
 	return 0;
 }
 
+static int mwifiex_cmd_chan_region_cfg(struct mwifiex_private *priv,
+				       struct host_cmd_ds_command *cmd,
+				       u16 cmd_action)
+{
+	struct host_cmd_ds_chan_region_cfg *reg = &cmd->params.reg_cfg;
+
+	cmd->command = cpu_to_le16(HostCmd_CMD_CHAN_REGION_CFG);
+	cmd->size = cpu_to_le16(sizeof(*reg) + S_DS_GEN);
+
+	if (cmd_action == HostCmd_ACT_GEN_GET)
+		reg->action = cpu_to_le16(cmd_action);
+
+	return 0;
+}
+
 static int
 mwifiex_cmd_coalesce_cfg(struct mwifiex_private *priv,
 			 struct host_cmd_ds_command *cmd,
@@ -2136,6 +2154,9 @@ int mwifiex_sta_prepare_cmd(struct mwifiex_private *priv, uint16_t cmd_no,
 		ret = mwifiex_cmd_gtk_rekey_offload(priv, cmd_ptr, cmd_action,
 						    data_buf);
 		break;
+	case HostCmd_CMD_CHAN_REGION_CFG:
+		ret = mwifiex_cmd_chan_region_cfg(priv, cmd_ptr, cmd_action);
+		break;
 	default:
 		mwifiex_dbg(priv->adapter, ERROR,
 			    "PREP_CMD: unknown cmd- %#x\n", cmd_no);
@@ -2273,6 +2294,9 @@ int mwifiex_sta_init_cmd(struct mwifiex_private *priv, u8 first_sta, bool init)
 			if (ret)
 				return -1;
 		}
+
+		mwifiex_send_cmd(priv, HostCmd_CMD_CHAN_REGION_CFG,
+				 HostCmd_ACT_GEN_GET, 0, NULL, true);
 	}
 
 	/* get tx rate */
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c
index ccf54932e321..8548027abf71 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c
@@ -962,7 +962,7 @@ static int mwifiex_ret_uap_sta_list(struct mwifiex_private *priv,
 	int i;
 	struct mwifiex_sta_node *sta_node;
 
-	for (i = 0; i < sta_list->sta_count; i++) {
+	for (i = 0; i < (le16_to_cpu(sta_list->sta_count)); i++) {
 		sta_node = mwifiex_get_sta_entry(priv, sta_info->mac);
 		if (unlikely(!sta_node))
 			continue;
@@ -1022,6 +1022,138 @@ static int mwifiex_ret_robust_coex(struct mwifiex_private *priv,
 	return 0;
 }
 
+static struct ieee80211_regdomain *
+mwifiex_create_custom_regdomain(struct mwifiex_private *priv,
+				u8 *buf, u16 buf_len)
+{
+	u16 num_chan = buf_len / 2;
+	struct ieee80211_regdomain *regd;
+	struct ieee80211_reg_rule *rule;
+	bool new_rule;
+	int regd_size, idx, freq, prev_freq = 0;
+	u32 bw, prev_bw = 0;
+	u8 chflags, prev_chflags = 0, valid_rules = 0;
+
+	if (WARN_ON_ONCE(num_chan > NL80211_MAX_SUPP_REG_RULES))
+		return ERR_PTR(-EINVAL);
+
+	regd_size = sizeof(struct ieee80211_regdomain) +
+		    num_chan * sizeof(struct ieee80211_reg_rule);
+
+	regd = kzalloc(regd_size, GFP_KERNEL);
+	if (!regd)
+		return ERR_PTR(-ENOMEM);
+
+	for (idx = 0; idx < num_chan; idx++) {
+		u8 chan;
+		enum nl80211_band band;
+
+		chan = *buf++;
+		if (!chan) {
+			kfree(regd);
+			return NULL;
+		}
+		chflags = *buf++;
+		band = (chan <= 14) ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ;
+		freq = ieee80211_channel_to_frequency(chan, band);
+		new_rule = false;
+
+		if (chflags & MWIFIEX_CHANNEL_DISABLED)
+			continue;
+
+		if (band == NL80211_BAND_5GHZ) {
+			if (!(chflags & MWIFIEX_CHANNEL_NOHT80))
+				bw = MHZ_TO_KHZ(80);
+			else if (!(chflags & MWIFIEX_CHANNEL_NOHT40))
+				bw = MHZ_TO_KHZ(40);
+			else
+				bw = MHZ_TO_KHZ(20);
+		} else {
+			if (!(chflags & MWIFIEX_CHANNEL_NOHT40))
+				bw = MHZ_TO_KHZ(40);
+			else
+				bw = MHZ_TO_KHZ(20);
+		}
+
+		if (idx == 0 || prev_chflags != chflags || prev_bw != bw ||
+		    freq - prev_freq > 20) {
+			valid_rules++;
+			new_rule = true;
+		}
+
+		rule = &regd->reg_rules[valid_rules - 1];
+
+		rule->freq_range.end_freq_khz = MHZ_TO_KHZ(freq + 10);
+
+		prev_chflags = chflags;
+		prev_freq = freq;
+		prev_bw = bw;
+
+		if (!new_rule)
+			continue;
+
+		rule->freq_range.start_freq_khz = MHZ_TO_KHZ(freq - 10);
+		rule->power_rule.max_eirp = DBM_TO_MBM(19);
+
+		if (chflags & MWIFIEX_CHANNEL_PASSIVE)
+			rule->flags = NL80211_RRF_NO_IR;
+
+		if (chflags & MWIFIEX_CHANNEL_DFS)
+			rule->flags = NL80211_RRF_DFS;
+
+		rule->freq_range.max_bandwidth_khz = bw;
+	}
+
+	regd->n_reg_rules = valid_rules;
+	regd->alpha2[0] = '9';
+	regd->alpha2[1] = '9';
+
+	return regd;
+}
+
+static int mwifiex_ret_chan_region_cfg(struct mwifiex_private *priv,
+				       struct host_cmd_ds_command *resp)
+{
+	struct host_cmd_ds_chan_region_cfg *reg = &resp->params.reg_cfg;
+	u16 action = le16_to_cpu(reg->action);
+	u16 tlv, tlv_buf_len, tlv_buf_left;
+	struct mwifiex_ie_types_header *head;
+	struct ieee80211_regdomain *regd;
+	u8 *tlv_buf;
+
+	if (action != HostCmd_ACT_GEN_GET)
+		return 0;
+
+	tlv_buf = (u8 *)reg + sizeof(*reg);
+	tlv_buf_left = le16_to_cpu(resp->size) - S_DS_GEN - sizeof(*reg);
+
+	while (tlv_buf_left >= sizeof(*head)) {
+		head = (struct mwifiex_ie_types_header *)tlv_buf;
+		tlv = le16_to_cpu(head->type);
+		tlv_buf_len = le16_to_cpu(head->len);
+
+		if (tlv_buf_left < (sizeof(*head) + tlv_buf_len))
+			break;
+
+		switch (tlv) {
+		case TLV_TYPE_CHAN_ATTR_CFG:
+			mwifiex_dbg_dump(priv->adapter, CMD_D, "CHAN:",
+					 (u8 *)head + sizeof(*head),
+					 tlv_buf_len);
+			regd = mwifiex_create_custom_regdomain(priv,
+				(u8 *)head + sizeof(*head), tlv_buf_len);
+			if (!IS_ERR(regd))
+				priv->adapter->regd = regd;
+			break;
+		}
+
+		tlv_buf += (sizeof(*head) + tlv_buf_len);
+		tlv_buf_left -= (sizeof(*head) + tlv_buf_len);
+	}
+
+	return 0;
+}
+
 /*
  * This function handles the command responses.
  *
@@ -1239,6 +1371,9 @@ int mwifiex_process_sta_cmdresp(struct mwifiex_private *priv, u16 cmdresp_no,
 		break;
 	case HostCmd_CMD_GTK_REKEY_OFFLOAD_CFG:
 		break;
+	case HostCmd_CMD_CHAN_REGION_CFG:
+		ret = mwifiex_ret_chan_region_cfg(priv, resp);
+		break;
 	default:
 		mwifiex_dbg(adapter, ERROR,
 			    "CMD_RESP: unknown cmd response %#x\n",
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_event.c b/drivers/net/wireless/marvell/mwifiex/sta_event.c
index a422f3306d4d..9df0c4dc06ed 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_event.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_event.c
@@ -25,6 +25,99 @@
 #include "wmm.h"
 #include "11n.h"
 
+#define MWIFIEX_IBSS_CONNECT_EVT_FIX_SIZE    12
+
+static int mwifiex_check_ibss_peer_capabilties(struct mwifiex_private *priv,
+					       struct mwifiex_sta_node *sta_ptr,
+					       struct sk_buff *event)
+{
+	int evt_len, ele_len;
+	u8 *curr;
+	struct ieee_types_header *ele_hdr;
+	struct mwifiex_ie_types_mgmt_frame *tlv_mgmt_frame;
+	const struct ieee80211_ht_cap *ht_cap;
+	const struct ieee80211_vht_cap *vht_cap;
+
+	skb_pull(event, MWIFIEX_IBSS_CONNECT_EVT_FIX_SIZE);
+	evt_len = event->len;
+	curr = event->data;
+
+	mwifiex_dbg_dump(priv->adapter, EVT_D, "ibss peer capabilties:",
+			 event->data, event->len);
+
+	skb_push(event, MWIFIEX_IBSS_CONNECT_EVT_FIX_SIZE);
+
+	tlv_mgmt_frame = (void *)curr;
+	if (evt_len >= sizeof(*tlv_mgmt_frame) &&
+	    le16_to_cpu(tlv_mgmt_frame->header.type) ==
+	    TLV_TYPE_UAP_MGMT_FRAME) {
+		/* Locate curr pointer to the start of beacon tlv,
+		 * timestamp 8 bytes, beacon intervel 2 bytes,
+		 * capability info 2 bytes, totally 12 byte beacon header
+		 */
+		evt_len = le16_to_cpu(tlv_mgmt_frame->header.len);
+		curr += (sizeof(*tlv_mgmt_frame) + 12);
+	} else {
+		mwifiex_dbg(priv->adapter, MSG,
+			    "management frame tlv not found!\n");
+		return 0;
+	}
+
+	while (evt_len >= sizeof(*ele_hdr)) {
+		ele_hdr = (struct ieee_types_header *)curr;
+		ele_len = ele_hdr->len;
+
+		if (evt_len < ele_len + sizeof(*ele_hdr))
+			break;
+
+		switch (ele_hdr->element_id) {
+		case WLAN_EID_HT_CAPABILITY:
+			sta_ptr->is_11n_enabled = true;
+			ht_cap = (void *)(ele_hdr + 2);
+			sta_ptr->max_amsdu = le16_to_cpu(ht_cap->cap_info) &
+				IEEE80211_HT_CAP_MAX_AMSDU ?
+				MWIFIEX_TX_DATA_BUF_SIZE_8K :
+				MWIFIEX_TX_DATA_BUF_SIZE_4K;
+			mwifiex_dbg(priv->adapter, INFO,
+				    "11n enabled!, max_amsdu : %d\n",
+				    sta_ptr->max_amsdu);
+			break;
+
+		case WLAN_EID_VHT_CAPABILITY:
+			sta_ptr->is_11ac_enabled = true;
+			vht_cap = (void *)(ele_hdr + 2);
+			/* check VHT MAXMPDU capability */
+			switch (le32_to_cpu(vht_cap->vht_cap_info) & 0x3) {
+			case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454:
+				sta_ptr->max_amsdu =
+					MWIFIEX_TX_DATA_BUF_SIZE_12K;
+				break;
+			case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991:
+				sta_ptr->max_amsdu =
+					MWIFIEX_TX_DATA_BUF_SIZE_8K;
+				break;
+			case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895:
+				sta_ptr->max_amsdu =
+					MWIFIEX_TX_DATA_BUF_SIZE_4K;
+			default:
+				break;
+			}
+
+			mwifiex_dbg(priv->adapter, INFO,
+				    "11ac enabled!, max_amsdu : %d\n",
+				    sta_ptr->max_amsdu);
+			break;
+		default:
+			break;
+		}
+
+		curr += (ele_len + sizeof(*ele_hdr));
+		evt_len -= (ele_len + sizeof(*ele_hdr));
+	}
+
+	return 0;
+}
+
 /*
  * This function resets the connection state.
  *
@@ -519,6 +612,8 @@ void mwifiex_bt_coex_wlan_param_update_event(struct mwifiex_private *priv,
  *      - EVENT_LINK_QUALITY
  *      - EVENT_PRE_BEACON_LOST
  *      - EVENT_IBSS_COALESCED
+ *      - EVENT_IBSS_STA_CONNECT
+ *      - EVENT_IBSS_STA_DISCONNECT
  *      - EVENT_WEP_ICV_ERR
  *      - EVENT_BW_CHANGE
  *      - EVENT_HOSTWAKE_STAIE
@@ -547,9 +642,11 @@ void mwifiex_bt_coex_wlan_param_update_event(struct mwifiex_private *priv,
 int mwifiex_process_sta_event(struct mwifiex_private *priv)
 {
 	struct mwifiex_adapter *adapter = priv->adapter;
-	int ret = 0;
+	int ret = 0, i;
 	u32 eventcause = adapter->event_cause;
 	u16 ctrl, reason_code;
+	u8 ibss_sta_addr[ETH_ALEN];
+	struct mwifiex_sta_node *sta_ptr;
 
 	switch (eventcause) {
 	case EVENT_DUMMY_HOST_WAKEUP_SIGNAL:
@@ -708,7 +805,11 @@ int mwifiex_process_sta_event(struct mwifiex_private *priv)
 
 	case EVENT_EXT_SCAN_REPORT:
 		mwifiex_dbg(adapter, EVENT, "event: EXT_SCAN Report\n");
-		if (adapter->ext_scan && !priv->scan_aborting)
+		/* We intend to skip this event during suspend, but handle
+		 * it in interface disabled case
+		 */
+		if (adapter->ext_scan && (!priv->scan_aborting ||
+					  !netif_running(priv->netdev)))
 			ret = mwifiex_handle_event_ext_scan_report(priv,
 						adapter->event_skb->data);
 
@@ -771,6 +872,39 @@ int mwifiex_process_sta_event(struct mwifiex_private *priv)
 				HostCmd_CMD_802_11_IBSS_COALESCING_STATUS,
 				HostCmd_ACT_GEN_GET, 0, NULL, false);
 		break;
+	case EVENT_IBSS_STA_CONNECT:
+		ether_addr_copy(ibss_sta_addr, adapter->event_body + 2);
+		mwifiex_dbg(adapter, EVENT, "event: IBSS_STA_CONNECT %pM\n",
+			    ibss_sta_addr);
+		sta_ptr = mwifiex_add_sta_entry(priv, ibss_sta_addr);
+		if (sta_ptr && adapter->adhoc_11n_enabled) {
+			mwifiex_check_ibss_peer_capabilties(priv, sta_ptr,
+							    adapter->event_skb);
+			if (sta_ptr->is_11n_enabled)
+				for (i = 0; i < MAX_NUM_TID; i++)
+					sta_ptr->ampdu_sta[i] =
+					priv->aggr_prio_tbl[i].ampdu_user;
+			else
+				for (i = 0; i < MAX_NUM_TID; i++)
+					sta_ptr->ampdu_sta[i] =
+						BA_STREAM_NOT_ALLOWED;
+			memset(sta_ptr->rx_seq, 0xff, sizeof(sta_ptr->rx_seq));
+		}
+
+		break;
+	case EVENT_IBSS_STA_DISCONNECT:
+		ether_addr_copy(ibss_sta_addr, adapter->event_body + 2);
+		mwifiex_dbg(adapter, EVENT, "event: IBSS_STA_DISCONNECT %pM\n",
+			    ibss_sta_addr);
+		sta_ptr = mwifiex_get_sta_entry(priv, ibss_sta_addr);
+		if (sta_ptr && sta_ptr->is_11n_enabled) {
+			mwifiex_11n_del_rx_reorder_tbl_by_ta(priv,
+							     ibss_sta_addr);
+			mwifiex_del_tx_ba_stream_tbl_by_ra(priv, ibss_sta_addr);
+		}
+		mwifiex_wmm_del_peer_ra_list(priv, ibss_sta_addr);
+		mwifiex_del_sta_entry(priv, ibss_sta_addr);
+		break;
 	case EVENT_ADDBA:
 		mwifiex_dbg(adapter, EVENT, "event: ADDBA Request\n");
 		mwifiex_send_cmd(priv, HostCmd_CMD_11N_ADDBA_RSP,
@@ -869,6 +1003,12 @@ int mwifiex_process_sta_event(struct mwifiex_private *priv)
 		mwifiex_bt_coex_wlan_param_update_event(priv,
 							adapter->event_skb);
 		break;
+	case EVENT_RXBA_SYNC:
+		dev_dbg(adapter->dev, "EVENT: RXBA_SYNC\n");
+		mwifiex_11n_rxba_sync_event(priv, adapter->event_body,
+					    adapter->event_skb->len -
+					    sizeof(eventcause));
+		break;
 	default:
 		mwifiex_dbg(adapter, ERROR, "event: unknown event id: %#x\n",
 			    eventcause);
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c
index e06647a327b6..644f3a248741 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c
@@ -574,7 +574,7 @@ int mwifiex_enable_hs(struct mwifiex_adapter *adapter)
 
 	adapter->hs_activate_wait_q_woken = false;
 
-	memset(&hscfg, 0, sizeof(struct mwifiex_ds_hs_cfg));
+	memset(&hscfg, 0, sizeof(hscfg));
 	hscfg.is_invoke_hostcmd = true;
 
 	adapter->hs_enabling = true;
@@ -1138,7 +1138,7 @@ int mwifiex_set_encode(struct mwifiex_private *priv, struct key_params *kp,
 {
 	struct mwifiex_ds_encrypt_key encrypt_key;
 
-	memset(&encrypt_key, 0, sizeof(struct mwifiex_ds_encrypt_key));
+	memset(&encrypt_key, 0, sizeof(encrypt_key));
 	encrypt_key.key_len = key_len;
 	encrypt_key.key_index = key_index;
 
@@ -1180,7 +1180,7 @@ mwifiex_get_ver_ext(struct mwifiex_private *priv, u32 version_str_sel)
 {
 	struct mwifiex_ver_ext ver_ext;
 
-	memset(&ver_ext, 0, sizeof(struct host_cmd_ds_version_ext));
+	memset(&ver_ext, 0, sizeof(ver_ext));
 	ver_ext.version_str_sel = version_str_sel;
 	if (mwifiex_send_cmd(priv, HostCmd_CMD_VERSION_EXT,
 			     HostCmd_ACT_GEN_GET, 0, &ver_ext, true))
diff --git a/drivers/net/wireless/marvell/mwifiex/uap_event.c b/drivers/net/wireless/marvell/mwifiex/uap_event.c
index 86ff54296f39..d24eca34ac11 100644
--- a/drivers/net/wireless/marvell/mwifiex/uap_event.c
+++ b/drivers/net/wireless/marvell/mwifiex/uap_event.c
@@ -306,7 +306,12 @@ int mwifiex_process_uap_event(struct mwifiex_private *priv)
 		mwifiex_dbg(adapter, EVENT, "event: multi-chan info\n");
 		mwifiex_process_multi_chan_event(priv, adapter->event_skb);
 		break;
-
+	case EVENT_RXBA_SYNC:
+		dev_dbg(adapter->dev, "EVENT: RXBA_SYNC\n");
+		mwifiex_11n_rxba_sync_event(priv, adapter->event_body,
+					    adapter->event_skb->len -
+					    sizeof(eventcause));
+		break;
 	default:
 		mwifiex_dbg(adapter, EVENT,
 			    "event: unknown event id: %#x\n", eventcause);
diff --git a/drivers/net/wireless/marvell/mwifiex/usb.c b/drivers/net/wireless/marvell/mwifiex/usb.c
index 0857575c5c39..73eb0846db21 100644
--- a/drivers/net/wireless/marvell/mwifiex/usb.c
+++ b/drivers/net/wireless/marvell/mwifiex/usb.c
@@ -273,6 +273,8 @@ static void mwifiex_usb_tx_complete(struct urb *urb)
 	} else {
 		mwifiex_dbg(adapter, DATA,
 			    "%s: DATA\n", __func__);
+		mwifiex_write_data_complete(adapter, context->skb, 0,
+					    urb->status ? -1 : 0);
 		for (i = 0; i < MWIFIEX_TX_DATA_PORT; i++) {
 			port = &card->port[i];
 			if (context->ep == port->tx_data_ep) {
@@ -282,8 +284,6 @@ static void mwifiex_usb_tx_complete(struct urb *urb)
 			}
 		}
 		adapter->data_sent = false;
-		mwifiex_write_data_complete(adapter, context->skb, 0,
-					    urb->status ? -1 : 0);
 	}
 
 	if (card->mc_resync_flag)
@@ -611,7 +611,7 @@ static void mwifiex_usb_disconnect(struct usb_interface *intf)
 	if (!adapter->priv_num)
 		return;
 
-	if (user_rmmod) {
+	if (user_rmmod && !adapter->mfg_mode) {
 #ifdef CONFIG_PM
 		if (adapter->is_suspended)
 			mwifiex_usb_resume(intf);
@@ -657,11 +657,8 @@ static int mwifiex_usb_tx_init(struct mwifiex_adapter *adapter)
 	card->tx_cmd.ep = card->tx_cmd_ep;
 
 	card->tx_cmd.urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!card->tx_cmd.urb) {
-		mwifiex_dbg(adapter, ERROR,
-			    "tx_cmd.urb allocation failed\n");
+	if (!card->tx_cmd.urb)
 		return -ENOMEM;
-	}
 
 	for (i = 0; i < MWIFIEX_TX_DATA_PORT; i++) {
 		port = &card->port[i];
@@ -677,11 +674,8 @@ static int mwifiex_usb_tx_init(struct mwifiex_adapter *adapter)
 			port->tx_data_list[j].ep = port->tx_data_ep;
 			port->tx_data_list[j].urb =
 					usb_alloc_urb(0, GFP_KERNEL);
-			if (!port->tx_data_list[j].urb) {
-				mwifiex_dbg(adapter, ERROR,
-					    "urb allocation failed\n");
+			if (!port->tx_data_list[j].urb)
 				return -ENOMEM;
-			}
 		}
 	}
 
@@ -697,10 +691,8 @@ static int mwifiex_usb_rx_init(struct mwifiex_adapter *adapter)
 	card->rx_cmd.ep = card->rx_cmd_ep;
 
 	card->rx_cmd.urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!card->rx_cmd.urb) {
-		mwifiex_dbg(adapter, ERROR, "rx_cmd.urb allocation failed\n");
+	if (!card->rx_cmd.urb)
 		return -ENOMEM;
-	}
 
 	card->rx_cmd.skb = dev_alloc_skb(MWIFIEX_RX_CMD_BUF_SIZE);
 	if (!card->rx_cmd.skb)
@@ -714,11 +706,8 @@ static int mwifiex_usb_rx_init(struct mwifiex_adapter *adapter)
 		card->rx_data_list[i].ep = card->rx_data_ep;
 
 		card->rx_data_list[i].urb = usb_alloc_urb(0, GFP_KERNEL);
-		if (!card->rx_data_list[i].urb) {
-			mwifiex_dbg(adapter, ERROR,
-				    "rx_data_list[] urb allocation failed\n");
+		if (!card->rx_data_list[i].urb)
 			return -1;
-		}
 		if (mwifiex_usb_submit_rx_urb(&card->rx_data_list[i],
 					      MWIFIEX_RX_DATA_BUF_SIZE))
 			return -1;
@@ -852,7 +841,7 @@ static int mwifiex_usb_host_to_card(struct mwifiex_adapter *adapter, u8 ep,
 	struct usb_tx_data_port *port = NULL;
 	u8 *data = (u8 *)skb->data;
 	struct urb *tx_urb;
-	int idx, ret;
+	int idx, ret = -EINPROGRESS;
 
 	if (adapter->is_suspended) {
 		mwifiex_dbg(adapter, ERROR,
@@ -876,8 +865,9 @@ static int mwifiex_usb_host_to_card(struct mwifiex_adapter *adapter, u8 ep,
 				if (atomic_read(&port->tx_data_urb_pending)
 				    >= MWIFIEX_TX_DATA_URB) {
 					port->block_status = true;
-					ret = -EBUSY;
-					goto done;
+					adapter->data_sent =
+						mwifiex_usb_data_sent(adapter);
+					return -EBUSY;
 				}
 				if (port->tx_data_ix >= MWIFIEX_TX_DATA_URB)
 					port->tx_data_ix = 0;
@@ -908,6 +898,14 @@ static int mwifiex_usb_host_to_card(struct mwifiex_adapter *adapter, u8 ep,
 	else
 		atomic_inc(&port->tx_data_urb_pending);
 
+	if (ep != card->tx_cmd_ep &&
+	    atomic_read(&port->tx_data_urb_pending) ==
+					MWIFIEX_TX_DATA_URB) {
+		port->block_status = true;
+		adapter->data_sent = mwifiex_usb_data_sent(adapter);
+		ret = -ENOSR;
+	}
+
 	if (usb_submit_urb(tx_urb, GFP_ATOMIC)) {
 		mwifiex_dbg(adapter, ERROR,
 			    "%s: usb_submit_urb failed\n", __func__);
@@ -916,29 +914,15 @@ static int mwifiex_usb_host_to_card(struct mwifiex_adapter *adapter, u8 ep,
 		} else {
 			atomic_dec(&port->tx_data_urb_pending);
 			port->block_status = false;
+			adapter->data_sent = false;
 			if (port->tx_data_ix)
 				port->tx_data_ix--;
 			else
 				port->tx_data_ix = MWIFIEX_TX_DATA_URB;
 		}
-
-		return -1;
-	} else {
-		if (ep != card->tx_cmd_ep &&
-		    atomic_read(&port->tx_data_urb_pending) ==
-							MWIFIEX_TX_DATA_URB) {
-			port->block_status = true;
-			ret = -ENOSR;
-			goto done;
-		}
+		ret = -1;
 	}
 
-	return -EINPROGRESS;
-
-done:
-	if (ep != card->tx_cmd_ep)
-		adapter->data_sent = mwifiex_usb_data_sent(adapter);
-
 	return ret;
 }
 
@@ -1037,6 +1021,10 @@ static int mwifiex_prog_fw_w_helper(struct mwifiex_adapter *adapter,
 			dnld_cmd = le32_to_cpu(fwdata->fw_hdr.dnld_cmd);
 			tlen += sizeof(struct fw_header);
 
+			/* Command 7 doesn't have data length field */
+			if (dnld_cmd == FW_CMD_7)
+				dlen = 0;
+
 			memcpy(fwdata->data, &firmware[tlen], dlen);
 
 			fwdata->seq_num = cpu_to_le32(fw_seqnum);
diff --git a/drivers/net/wireless/marvell/mwifiex/usb.h b/drivers/net/wireless/marvell/mwifiex/usb.h
index b4e9246bbcdc..30e8eb8c259d 100644
--- a/drivers/net/wireless/marvell/mwifiex/usb.h
+++ b/drivers/net/wireless/marvell/mwifiex/usb.h
@@ -46,11 +46,12 @@
 #define USB8766_DEFAULT_FW_NAME	"mrvl/usb8766_uapsta.bin"
 #define USB8797_DEFAULT_FW_NAME	"mrvl/usb8797_uapsta.bin"
 #define USB8801_DEFAULT_FW_NAME	"mrvl/usb8801_uapsta.bin"
-#define USB8997_DEFAULT_FW_NAME	"mrvl/usb8997_uapsta.bin"
+#define USB8997_DEFAULT_FW_NAME	"mrvl/usbusb8997_combo_v4.bin"
 
 #define FW_DNLD_TX_BUF_SIZE	620
 #define FW_DNLD_RX_BUF_SIZE	2048
 #define FW_HAS_LAST_BLOCK	0x00000004
+#define FW_CMD_7		0x00000007
 
 #define FW_DATA_XMIT_SIZE \
 	(sizeof(struct fw_header) + dlen + sizeof(u32))
diff --git a/drivers/net/wireless/marvell/mwifiex/util.c b/drivers/net/wireless/marvell/mwifiex/util.c
index 6681be0511c7..18fbb96a46e9 100644
--- a/drivers/net/wireless/marvell/mwifiex/util.c
+++ b/drivers/net/wireless/marvell/mwifiex/util.c
@@ -386,6 +386,7 @@ mwifiex_parse_mgmt_packet(struct mwifiex_private *priv, u8 *payload, u16 len,
 				    "unknown public action frame category %d\n",
 				    category);
 		}
+		break;
 	default:
 		mwifiex_dbg(priv->adapter, INFO,
 		    "unknown mgmt frame subtype %#x\n", stype);
diff --git a/drivers/net/wireless/mediatek/mt7601u/dma.c b/drivers/net/wireless/mediatek/mt7601u/dma.c
index 57a80cfa39b1..a8bc064bc14f 100644
--- a/drivers/net/wireless/mediatek/mt7601u/dma.c
+++ b/drivers/net/wireless/mediatek/mt7601u/dma.c
@@ -103,7 +103,7 @@ static void mt7601u_rx_process_seg(struct mt7601u_dev *dev, u8 *data,
 
 	if (unlikely(rxwi->zero[0] || rxwi->zero[1] || rxwi->zero[2]))
 		dev_err_once(dev->dev, "Error: RXWI zero fields are set\n");
-	if (unlikely(MT76_GET(MT_RXD_INFO_TYPE, fce_info)))
+	if (unlikely(FIELD_GET(MT_RXD_INFO_TYPE, fce_info)))
 		dev_err_once(dev->dev, "Error: RX path seen a non-pkt urb\n");
 
 	trace_mt_rx(dev, rxwi, fce_info);
diff --git a/drivers/net/wireless/mediatek/mt7601u/dma.h b/drivers/net/wireless/mediatek/mt7601u/dma.h
index 978e8a90b87f..270d126880c0 100644
--- a/drivers/net/wireless/mediatek/mt7601u/dma.h
+++ b/drivers/net/wireless/mediatek/mt7601u/dma.h
@@ -18,8 +18,6 @@
 #include <asm/unaligned.h>
 #include <linux/skbuff.h>
 
-#include "util.h"
-
 #define MT_DMA_HDR_LEN			4
 #define MT_RX_INFO_LEN			4
 #define MT_FCE_INFO_LEN			4
@@ -79,9 +77,9 @@ static inline int mt7601u_dma_skb_wrap(struct sk_buff *skb,
 	 */
 
 	info = flags |
-		MT76_SET(MT_TXD_INFO_LEN, round_up(skb->len, 4)) |
-		MT76_SET(MT_TXD_INFO_D_PORT, d_port) |
-		MT76_SET(MT_TXD_INFO_TYPE, type);
+		FIELD_PREP(MT_TXD_INFO_LEN, round_up(skb->len, 4)) |
+		FIELD_PREP(MT_TXD_INFO_D_PORT, d_port) |
+		FIELD_PREP(MT_TXD_INFO_TYPE, type);
 
 	put_unaligned_le32(info, skb_push(skb, sizeof(info)));
 	return skb_put_padto(skb, round_up(skb->len, 4) + 4);
@@ -90,7 +88,7 @@ static inline int mt7601u_dma_skb_wrap(struct sk_buff *skb,
 static inline int
 mt7601u_dma_skb_wrap_pkt(struct sk_buff *skb, enum mt76_qsel qsel, u32 flags)
 {
-	flags |= MT76_SET(MT_TXD_PKT_INFO_QSEL, qsel);
+	flags |= FIELD_PREP(MT_TXD_PKT_INFO_QSEL, qsel);
 	return mt7601u_dma_skb_wrap(skb, WLAN_PORT, DMA_PACKET, flags);
 }
 
diff --git a/drivers/net/wireless/mediatek/mt7601u/eeprom.c b/drivers/net/wireless/mediatek/mt7601u/eeprom.c
index 8d8ee0344f7b..da6faea092d6 100644
--- a/drivers/net/wireless/mediatek/mt7601u/eeprom.c
+++ b/drivers/net/wireless/mediatek/mt7601u/eeprom.c
@@ -45,8 +45,8 @@ mt7601u_efuse_read(struct mt7601u_dev *dev, u16 addr, u8 *data,
 	val = mt76_rr(dev, MT_EFUSE_CTRL);
 	val &= ~(MT_EFUSE_CTRL_AIN |
 		 MT_EFUSE_CTRL_MODE);
-	val |= MT76_SET(MT_EFUSE_CTRL_AIN, addr & ~0xf) |
-	       MT76_SET(MT_EFUSE_CTRL_MODE, mode) |
+	val |= FIELD_PREP(MT_EFUSE_CTRL_AIN, addr & ~0xf) |
+	       FIELD_PREP(MT_EFUSE_CTRL_MODE, mode) |
 	       MT_EFUSE_CTRL_KICK;
 	mt76_wr(dev, MT_EFUSE_CTRL, val);
 
@@ -128,8 +128,8 @@ mt7601u_set_chip_cap(struct mt7601u_dev *dev, u8 *eeprom)
 	if (!field_valid(nic_conf0 >> 8))
 		return;
 
-	if (MT76_GET(MT_EE_NIC_CONF_0_RX_PATH, nic_conf0) > 1 ||
-	    MT76_GET(MT_EE_NIC_CONF_0_TX_PATH, nic_conf0) > 1)
+	if (FIELD_GET(MT_EE_NIC_CONF_0_RX_PATH, nic_conf0) > 1 ||
+	    FIELD_GET(MT_EE_NIC_CONF_0_TX_PATH, nic_conf0) > 1)
 		dev_err(dev->dev,
 			"Error: device has more than 1 RX/TX stream!\n");
 }
@@ -150,7 +150,7 @@ mt7601u_set_macaddr(struct mt7601u_dev *dev, const u8 *eeprom)
 
 	mt76_wr(dev, MT_MAC_ADDR_DW0, get_unaligned_le32(dev->macaddr));
 	mt76_wr(dev, MT_MAC_ADDR_DW1, get_unaligned_le16(dev->macaddr + 4) |
-		MT76_SET(MT_MAC_ADDR_DW1_U2ME_MASK, 0xff));
+		FIELD_PREP(MT_MAC_ADDR_DW1_U2ME_MASK, 0xff));
 
 	return 0;
 }
@@ -176,7 +176,7 @@ mt7601u_set_channel_power(struct mt7601u_dev *dev, u8 *eeprom)
 	u8 max_pwr;
 
 	val = mt7601u_rr(dev, MT_TX_ALC_CFG_0);
-	max_pwr = MT76_GET(MT_TX_ALC_CFG_0_LIMIT_0, val);
+	max_pwr = FIELD_GET(MT_TX_ALC_CFG_0_LIMIT_0, val);
 
 	if (mt7601u_has_tssi(dev, eeprom)) {
 		mt7601u_set_channel_target_power(dev, eeprom, max_pwr);
diff --git a/drivers/net/wireless/mediatek/mt7601u/init.c b/drivers/net/wireless/mediatek/mt7601u/init.c
index 8fa78d7156be..44d46e25db80 100644
--- a/drivers/net/wireless/mediatek/mt7601u/init.c
+++ b/drivers/net/wireless/mediatek/mt7601u/init.c
@@ -108,8 +108,9 @@ static void mt7601u_init_usb_dma(struct mt7601u_dev *dev)
 {
 	u32 val;
 
-	val = MT76_SET(MT_USB_DMA_CFG_RX_BULK_AGG_TOUT, MT_USB_AGGR_TIMEOUT) |
-	      MT76_SET(MT_USB_DMA_CFG_RX_BULK_AGG_LMT, MT_USB_AGGR_SIZE_LIMIT) |
+	val = FIELD_PREP(MT_USB_DMA_CFG_RX_BULK_AGG_TOUT, MT_USB_AGGR_TIMEOUT) |
+	      FIELD_PREP(MT_USB_DMA_CFG_RX_BULK_AGG_LMT,
+			 MT_USB_AGGR_SIZE_LIMIT) |
 	      MT_USB_DMA_CFG_RX_BULK_EN |
 	      MT_USB_DMA_CFG_TX_BULK_EN;
 	if (dev->in_max_packet == 512)
@@ -396,8 +397,9 @@ int mt7601u_init_hardware(struct mt7601u_dev *dev)
 
 	mt7601u_rmw(dev, MT_US_CYC_CFG, MT_US_CYC_CNT, 0x1e);
 
-	mt7601u_wr(dev, MT_TXOP_CTRL_CFG, MT76_SET(MT_TXOP_TRUN_EN, 0x3f) |
-					  MT76_SET(MT_TXOP_EXT_CCA_DLY, 0x58));
+	mt7601u_wr(dev, MT_TXOP_CTRL_CFG,
+		   FIELD_PREP(MT_TXOP_TRUN_EN, 0x3f) |
+		   FIELD_PREP(MT_TXOP_EXT_CCA_DLY, 0x58));
 
 	ret = mt7601u_eeprom_init(dev);
 	if (ret)
diff --git a/drivers/net/wireless/mediatek/mt7601u/mac.c b/drivers/net/wireless/mediatek/mt7601u/mac.c
index e21c53ed09fb..3c576392ed89 100644
--- a/drivers/net/wireless/mediatek/mt7601u/mac.c
+++ b/drivers/net/wireless/mediatek/mt7601u/mac.c
@@ -19,13 +19,13 @@
 static void
 mt76_mac_process_tx_rate(struct ieee80211_tx_rate *txrate, u16 rate)
 {
-	u8 idx = MT76_GET(MT_TXWI_RATE_MCS, rate);
+	u8 idx = FIELD_GET(MT_TXWI_RATE_MCS, rate);
 
 	txrate->idx = 0;
 	txrate->flags = 0;
 	txrate->count = 1;
 
-	switch (MT76_GET(MT_TXWI_RATE_PHY_MODE, rate)) {
+	switch (FIELD_GET(MT_TXWI_RATE_PHY_MODE, rate)) {
 	case MT_PHY_TYPE_OFDM:
 		txrate->idx = idx + 4;
 		return;
@@ -47,7 +47,7 @@ mt76_mac_process_tx_rate(struct ieee80211_tx_rate *txrate, u16 rate)
 		return;
 	}
 
-	if (MT76_GET(MT_TXWI_RATE_BW, rate) == MT_PHY_BW_40)
+	if (FIELD_GET(MT_TXWI_RATE_BW, rate) == MT_PHY_BW_40)
 		txrate->flags |= IEEE80211_TX_RC_40_MHZ_WIDTH;
 
 	if (rate & MT_TXWI_RATE_SGI)
@@ -125,9 +125,9 @@ u16 mt76_mac_tx_rate_val(struct mt7601u_dev *dev,
 		bw = 0;
 	}
 
-	rateval = MT76_SET(MT_RXWI_RATE_MCS, rate_idx);
-	rateval |= MT76_SET(MT_RXWI_RATE_PHY, phy);
-	rateval |= MT76_SET(MT_RXWI_RATE_BW, bw);
+	rateval = FIELD_PREP(MT_RXWI_RATE_MCS, rate_idx);
+	rateval |= FIELD_PREP(MT_RXWI_RATE_PHY, phy);
+	rateval |= FIELD_PREP(MT_RXWI_RATE_BW, bw);
 	if (rate->flags & IEEE80211_TX_RC_SHORT_GI)
 		rateval |= MT_RXWI_RATE_SGI;
 
@@ -156,9 +156,9 @@ struct mt76_tx_status mt7601u_mac_fetch_tx_status(struct mt7601u_dev *dev)
 	stat.success = !!(val & MT_TX_STAT_FIFO_SUCCESS);
 	stat.aggr = !!(val & MT_TX_STAT_FIFO_AGGR);
 	stat.ack_req = !!(val & MT_TX_STAT_FIFO_ACKREQ);
-	stat.pktid = MT76_GET(MT_TX_STAT_FIFO_PID_TYPE, val);
-	stat.wcid = MT76_GET(MT_TX_STAT_FIFO_WCID, val);
-	stat.rate = MT76_GET(MT_TX_STAT_FIFO_RATE, val);
+	stat.pktid = FIELD_GET(MT_TX_STAT_FIFO_PID_TYPE, val);
+	stat.wcid = FIELD_GET(MT_TX_STAT_FIFO_WCID, val);
+	stat.rate = FIELD_GET(MT_TX_STAT_FIFO_RATE, val);
 
 	return stat;
 }
@@ -270,7 +270,7 @@ void mt7601u_mac_config_tsf(struct mt7601u_dev *dev, bool enable, int interval)
 	}
 
 	val &= ~MT_BEACON_TIME_CFG_INTVAL;
-	val |= MT76_SET(MT_BEACON_TIME_CFG_INTVAL, interval << 4) |
+	val |= FIELD_PREP(MT_BEACON_TIME_CFG_INTVAL, interval << 4) |
 		MT_BEACON_TIME_CFG_TIMER_EN |
 		MT_BEACON_TIME_CFG_SYNC_MODE |
 		MT_BEACON_TIME_CFG_TBTT_EN;
@@ -349,8 +349,8 @@ mt7601u_mac_wcid_setup(struct mt7601u_dev *dev, u8 idx, u8 vif_idx, u8 *mac)
 	u8 zmac[ETH_ALEN] = {};
 	u32 attr;
 
-	attr = MT76_SET(MT_WCID_ATTR_BSS_IDX, vif_idx & 7) |
-	       MT76_SET(MT_WCID_ATTR_BSS_IDX_EXT, !!(vif_idx & 8));
+	attr = FIELD_PREP(MT_WCID_ATTR_BSS_IDX, vif_idx & 7) |
+	       FIELD_PREP(MT_WCID_ATTR_BSS_IDX_EXT, !!(vif_idx & 8));
 
 	mt76_wr(dev, MT_WCID_ATTR(idx), attr);
 
@@ -382,15 +382,15 @@ void mt7601u_mac_set_ampdu_factor(struct mt7601u_dev *dev)
 	rcu_read_unlock();
 
 	mt7601u_wr(dev, MT_MAX_LEN_CFG, 0xa0fff |
-		   MT76_SET(MT_MAX_LEN_CFG_AMPDU, min_factor));
+		   FIELD_PREP(MT_MAX_LEN_CFG_AMPDU, min_factor));
 }
 
 static void
 mt76_mac_process_rate(struct ieee80211_rx_status *status, u16 rate)
 {
-	u8 idx = MT76_GET(MT_RXWI_RATE_MCS, rate);
+	u8 idx = FIELD_GET(MT_RXWI_RATE_MCS, rate);
 
-	switch (MT76_GET(MT_RXWI_RATE_PHY, rate)) {
+	switch (FIELD_GET(MT_RXWI_RATE_PHY, rate)) {
 	case MT_PHY_TYPE_OFDM:
 		if (WARN_ON(idx >= 8))
 			idx = 0;
@@ -436,7 +436,7 @@ mt7601u_rx_monitor_beacon(struct mt7601u_dev *dev, struct mt7601u_rxwi *rxwi,
 			  u16 rate, int rssi)
 {
 	dev->bcn_freq_off = rxwi->freq_off;
-	dev->bcn_phy_mode = MT76_GET(MT_RXWI_RATE_PHY, rate);
+	dev->bcn_phy_mode = FIELD_GET(MT_RXWI_RATE_PHY, rate);
 	dev->avg_rssi = (dev->avg_rssi * 15) / 16 + (rssi << 8);
 }
 
@@ -458,7 +458,7 @@ u32 mt76_mac_process_rx(struct mt7601u_dev *dev, struct sk_buff *skb,
 	u16 rate = le16_to_cpu(rxwi->rate);
 	int rssi;
 
-	len = MT76_GET(MT_RXWI_CTL_MPDU_LEN, ctl);
+	len = FIELD_GET(MT_RXWI_CTL_MPDU_LEN, ctl);
 	if (len < 10)
 		return 0;
 
@@ -542,8 +542,8 @@ int mt76_mac_wcid_set_key(struct mt7601u_dev *dev, u8 idx,
 
 	val = mt7601u_rr(dev, MT_WCID_ATTR(idx));
 	val &= ~MT_WCID_ATTR_PKEY_MODE & ~MT_WCID_ATTR_PKEY_MODE_EXT;
-	val |= MT76_SET(MT_WCID_ATTR_PKEY_MODE, cipher & 7) |
-	       MT76_SET(MT_WCID_ATTR_PKEY_MODE_EXT, cipher >> 3);
+	val |= FIELD_PREP(MT_WCID_ATTR_PKEY_MODE, cipher & 7) |
+	       FIELD_PREP(MT_WCID_ATTR_PKEY_MODE_EXT, cipher >> 3);
 	val &= ~MT_WCID_ATTR_PAIRWISE;
 	val |= MT_WCID_ATTR_PAIRWISE *
 		!!(key && key->flags & IEEE80211_KEY_FLAG_PAIRWISE);
diff --git a/drivers/net/wireless/mediatek/mt7601u/main.c b/drivers/net/wireless/mediatek/mt7601u/main.c
index e70dd9523911..43ebd460ba86 100644
--- a/drivers/net/wireless/mediatek/mt7601u/main.c
+++ b/drivers/net/wireless/mediatek/mt7601u/main.c
@@ -15,7 +15,6 @@
 #include "mt7601u.h"
 #include "mac.h"
 #include <linux/etherdevice.h>
-#include <linux/version.h>
 
 static int mt7601u_start(struct ieee80211_hw *hw)
 {
diff --git a/drivers/net/wireless/mediatek/mt7601u/mcu.c b/drivers/net/wireless/mediatek/mt7601u/mcu.c
index 91c4b3427965..dbdfb3f5c507 100644
--- a/drivers/net/wireless/mediatek/mt7601u/mcu.c
+++ b/drivers/net/wireless/mediatek/mt7601u/mcu.c
@@ -43,8 +43,8 @@ static inline void mt7601u_dma_skb_wrap_cmd(struct sk_buff *skb,
 					    u8 seq, enum mcu_cmd cmd)
 {
 	WARN_ON(mt7601u_dma_skb_wrap(skb, CPU_TX_PORT, DMA_COMMAND,
-				     MT76_SET(MT_TXD_CMD_INFO_SEQ, seq) |
-				     MT76_SET(MT_TXD_CMD_INFO_TYPE, cmd)));
+				     FIELD_PREP(MT_TXD_CMD_INFO_SEQ, seq) |
+				     FIELD_PREP(MT_TXD_CMD_INFO_TYPE, cmd)));
 }
 
 static inline void trace_mt_mcu_msg_send_cs(struct mt7601u_dev *dev,
@@ -100,13 +100,13 @@ static int mt7601u_mcu_wait_resp(struct mt7601u_dev *dev, u8 seq)
 			dev_err(dev->dev, "Error: MCU resp urb failed:%d\n",
 				urb_status);
 
-		if (MT76_GET(MT_RXD_CMD_INFO_CMD_SEQ, rxfce) == seq &&
-		    MT76_GET(MT_RXD_CMD_INFO_EVT_TYPE, rxfce) == CMD_DONE)
+		if (FIELD_GET(MT_RXD_CMD_INFO_CMD_SEQ, rxfce) == seq &&
+		    FIELD_GET(MT_RXD_CMD_INFO_EVT_TYPE, rxfce) == CMD_DONE)
 			return 0;
 
-		dev_err(dev->dev, "Error: MCU resp evt:%hhx seq:%hhx-%hhx!\n",
-			MT76_GET(MT_RXD_CMD_INFO_EVT_TYPE, rxfce),
-			seq, MT76_GET(MT_RXD_CMD_INFO_CMD_SEQ, rxfce));
+		dev_err(dev->dev, "Error: MCU resp evt:%lx seq:%hhx-%lx!\n",
+			FIELD_GET(MT_RXD_CMD_INFO_EVT_TYPE, rxfce),
+			seq, FIELD_GET(MT_RXD_CMD_INFO_CMD_SEQ, rxfce));
 	}
 
 	dev_err(dev->dev, "Error: %s timed out\n", __func__);
@@ -291,9 +291,9 @@ static int __mt7601u_dma_fw(struct mt7601u_dev *dev,
 	u32 val;
 	int ret;
 
-	reg = cpu_to_le32(MT76_SET(MT_TXD_INFO_TYPE, DMA_PACKET) |
-			  MT76_SET(MT_TXD_INFO_D_PORT, CPU_TX_PORT) |
-			  MT76_SET(MT_TXD_INFO_LEN, len));
+	reg = cpu_to_le32(FIELD_PREP(MT_TXD_INFO_TYPE, DMA_PACKET) |
+			  FIELD_PREP(MT_TXD_INFO_D_PORT, CPU_TX_PORT) |
+			  FIELD_PREP(MT_TXD_INFO_LEN, len));
 	memcpy(buf.buf, &reg, sizeof(reg));
 	memcpy(buf.buf + sizeof(reg), data, len);
 	memset(buf.buf + sizeof(reg) + len, 0, 8);
diff --git a/drivers/net/wireless/mediatek/mt7601u/mt7601u.h b/drivers/net/wireless/mediatek/mt7601u/mt7601u.h
index 428bd2f10b7b..c7ec40475a5f 100644
--- a/drivers/net/wireless/mediatek/mt7601u/mt7601u.h
+++ b/drivers/net/wireless/mediatek/mt7601u/mt7601u.h
@@ -15,6 +15,7 @@
 #ifndef MT7601U_H
 #define MT7601U_H
 
+#include <linux/bitfield.h>
 #include <linux/kernel.h>
 #include <linux/device.h>
 #include <linux/mutex.h>
@@ -24,7 +25,6 @@
 #include <linux/debugfs.h>
 
 #include "regs.h"
-#include "util.h"
 
 #define MT_CALIBRATE_INTERVAL		(4 * HZ)
 
@@ -299,7 +299,7 @@ bool mt76_poll_msec(struct mt7601u_dev *dev, u32 offset, u32 mask, u32 val,
 
 /* Compatibility with mt76 */
 #define mt76_rmw_field(_dev, _reg, _field, _val)	\
-	mt76_rmw(_dev, _reg, _field, MT76_SET(_field, _val))
+	mt76_rmw(_dev, _reg, _field, FIELD_PREP(_field, _val))
 
 static inline u32 mt76_rr(struct mt7601u_dev *dev, u32 offset)
 {
diff --git a/drivers/net/wireless/mediatek/mt7601u/phy.c b/drivers/net/wireless/mediatek/mt7601u/phy.c
index 1908af6add87..ca09a5d4305e 100644
--- a/drivers/net/wireless/mediatek/mt7601u/phy.c
+++ b/drivers/net/wireless/mediatek/mt7601u/phy.c
@@ -41,11 +41,12 @@ mt7601u_rf_wr(struct mt7601u_dev *dev, u8 bank, u8 offset, u8 value)
 		goto out;
 	}
 
-	mt7601u_wr(dev, MT_RF_CSR_CFG, MT76_SET(MT_RF_CSR_CFG_DATA, value) |
-				       MT76_SET(MT_RF_CSR_CFG_REG_BANK, bank) |
-				       MT76_SET(MT_RF_CSR_CFG_REG_ID, offset) |
-				       MT_RF_CSR_CFG_WR |
-				       MT_RF_CSR_CFG_KICK);
+	mt7601u_wr(dev, MT_RF_CSR_CFG,
+		   FIELD_PREP(MT_RF_CSR_CFG_DATA, value) |
+		   FIELD_PREP(MT_RF_CSR_CFG_REG_BANK, bank) |
+		   FIELD_PREP(MT_RF_CSR_CFG_REG_ID, offset) |
+		   MT_RF_CSR_CFG_WR |
+		   MT_RF_CSR_CFG_KICK);
 	trace_rf_write(dev, bank, offset, value);
 out:
 	mutex_unlock(&dev->reg_atomic_mutex);
@@ -74,17 +75,18 @@ mt7601u_rf_rr(struct mt7601u_dev *dev, u8 bank, u8 offset)
 	if (!mt76_poll(dev, MT_RF_CSR_CFG, MT_RF_CSR_CFG_KICK, 0, 100))
 		goto out;
 
-	mt7601u_wr(dev, MT_RF_CSR_CFG, MT76_SET(MT_RF_CSR_CFG_REG_BANK, bank) |
-				       MT76_SET(MT_RF_CSR_CFG_REG_ID, offset) |
-				       MT_RF_CSR_CFG_KICK);
+	mt7601u_wr(dev, MT_RF_CSR_CFG,
+		   FIELD_PREP(MT_RF_CSR_CFG_REG_BANK, bank) |
+		   FIELD_PREP(MT_RF_CSR_CFG_REG_ID, offset) |
+		   MT_RF_CSR_CFG_KICK);
 
 	if (!mt76_poll(dev, MT_RF_CSR_CFG, MT_RF_CSR_CFG_KICK, 0, 100))
 		goto out;
 
 	val = mt7601u_rr(dev, MT_RF_CSR_CFG);
-	if (MT76_GET(MT_RF_CSR_CFG_REG_ID, val) == offset &&
-	    MT76_GET(MT_RF_CSR_CFG_REG_BANK, val) == bank) {
-		ret = MT76_GET(MT_RF_CSR_CFG_DATA, val);
+	if (FIELD_GET(MT_RF_CSR_CFG_REG_ID, val) == offset &&
+	    FIELD_GET(MT_RF_CSR_CFG_REG_BANK, val) == bank) {
+		ret = FIELD_GET(MT_RF_CSR_CFG_DATA, val);
 		trace_rf_read(dev, bank, offset, ret);
 	}
 out:
@@ -139,8 +141,8 @@ static void mt7601u_bbp_wr(struct mt7601u_dev *dev, u8 offset, u8 val)
 	}
 
 	mt7601u_wr(dev, MT_BBP_CSR_CFG,
-		   MT76_SET(MT_BBP_CSR_CFG_VAL, val) |
-		   MT76_SET(MT_BBP_CSR_CFG_REG_NUM, offset) |
+		   FIELD_PREP(MT_BBP_CSR_CFG_VAL, val) |
+		   FIELD_PREP(MT_BBP_CSR_CFG_REG_NUM, offset) |
 		   MT_BBP_CSR_CFG_RW_MODE | MT_BBP_CSR_CFG_BUSY);
 	trace_bbp_write(dev, offset, val);
 out:
@@ -163,7 +165,7 @@ static int mt7601u_bbp_rr(struct mt7601u_dev *dev, u8 offset)
 		goto out;
 
 	mt7601u_wr(dev, MT_BBP_CSR_CFG,
-		   MT76_SET(MT_BBP_CSR_CFG_REG_NUM, offset) |
+		   FIELD_PREP(MT_BBP_CSR_CFG_REG_NUM, offset) |
 		   MT_BBP_CSR_CFG_RW_MODE | MT_BBP_CSR_CFG_BUSY |
 		   MT_BBP_CSR_CFG_READ);
 
@@ -171,8 +173,8 @@ static int mt7601u_bbp_rr(struct mt7601u_dev *dev, u8 offset)
 		goto out;
 
 	val = mt7601u_rr(dev, MT_BBP_CSR_CFG);
-	if (MT76_GET(MT_BBP_CSR_CFG_REG_NUM, val) == offset) {
-		ret = MT76_GET(MT_BBP_CSR_CFG_VAL, val);
+	if (FIELD_GET(MT_BBP_CSR_CFG_REG_NUM, val) == offset) {
+		ret = FIELD_GET(MT_BBP_CSR_CFG_VAL, val);
 		trace_bbp_read(dev, offset, ret);
 	}
 out:
@@ -249,9 +251,9 @@ int mt7601u_phy_get_rssi(struct mt7601u_dev *dev,
 			/* bw40 */ { -2, 16, 34 }
 		}
 	};
-	int bw = MT76_GET(MT_RXWI_RATE_BW, rate);
-	int aux_lna = MT76_GET(MT_RXWI_ANT_AUX_LNA, rxwi->ant);
-	int lna_id = MT76_GET(MT_RXWI_GAIN_RSSI_LNA_ID, rxwi->gain);
+	int bw = FIELD_GET(MT_RXWI_RATE_BW, rate);
+	int aux_lna = FIELD_GET(MT_RXWI_ANT_AUX_LNA, rxwi->ant);
+	int lna_id = FIELD_GET(MT_RXWI_GAIN_RSSI_LNA_ID, rxwi->gain);
 	int val;
 
 	if (lna_id) /* LNA id can be 0, 2, 3. */
@@ -259,7 +261,7 @@ int mt7601u_phy_get_rssi(struct mt7601u_dev *dev,
 
 	val = 8;
 	val -= lna[aux_lna][bw][lna_id];
-	val -= MT76_GET(MT_RXWI_GAIN_RSSI_VAL, rxwi->gain);
+	val -= FIELD_GET(MT_RXWI_GAIN_RSSI_VAL, rxwi->gain);
 	val -= dev->ee->lna_gain;
 	val -= dev->ee->rssi_offset[0];
 
@@ -939,7 +941,7 @@ static int mt7601u_tssi_cal(struct mt7601u_dev *dev)
 	dev_dbg(dev->dev, "final diff: %08x\n", diff_pwr);
 
 	val = mt7601u_rr(dev, MT_TX_ALC_CFG_1);
-	curr_pwr = s6_to_int(MT76_GET(MT_TX_ALC_CFG_1_TEMP_COMP, val));
+	curr_pwr = s6_to_int(FIELD_GET(MT_TX_ALC_CFG_1_TEMP_COMP, val));
 	diff_pwr += curr_pwr;
 	val = (val & ~MT_TX_ALC_CFG_1_TEMP_COMP) | int_to_s6(diff_pwr);
 	mt7601u_wr(dev, MT_TX_ALC_CFG_1, val);
diff --git a/drivers/net/wireless/mediatek/mt7601u/regs.h b/drivers/net/wireless/mediatek/mt7601u/regs.h
index afd8978e83fa..27a429d90cec 100644
--- a/drivers/net/wireless/mediatek/mt7601u/regs.h
+++ b/drivers/net/wireless/mediatek/mt7601u/regs.h
@@ -17,10 +17,6 @@
 
 #include <linux/bitops.h>
 
-#ifndef GENMASK
-#define GENMASK(h, l)       (((U32_C(1) << ((h) - (l) + 1)) - 1) << (l))
-#endif
-
 #define MT_ASIC_VERSION			0x0000
 
 #define MT76XX_REV_E3		0x22
diff --git a/drivers/net/wireless/mediatek/mt7601u/tx.c b/drivers/net/wireless/mediatek/mt7601u/tx.c
index a0a33dc8f6bc..ad77bec1ba0f 100644
--- a/drivers/net/wireless/mediatek/mt7601u/tx.c
+++ b/drivers/net/wireless/mediatek/mt7601u/tx.c
@@ -175,11 +175,12 @@ mt7601u_push_txwi(struct mt7601u_dev *dev, struct sk_buff *skb,
 		ba_size = min_t(int, 63, ba_size);
 		if (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE)
 			ba_size = 0;
-		txwi->ack_ctl |= MT76_SET(MT_TXWI_ACK_CTL_BA_WINDOW, ba_size);
+		txwi->ack_ctl |= FIELD_PREP(MT_TXWI_ACK_CTL_BA_WINDOW, ba_size);
 
-		txwi->flags = cpu_to_le16(MT_TXWI_FLAGS_AMPDU |
-					  MT76_SET(MT_TXWI_FLAGS_MPDU_DENSITY,
-						   sta->ht_cap.ampdu_density));
+		txwi->flags =
+			cpu_to_le16(MT_TXWI_FLAGS_AMPDU |
+				    FIELD_PREP(MT_TXWI_FLAGS_MPDU_DENSITY,
+					       sta->ht_cap.ampdu_density));
 		if (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE)
 			txwi->flags = 0;
 	}
@@ -188,7 +189,7 @@ mt7601u_push_txwi(struct mt7601u_dev *dev, struct sk_buff *skb,
 
 	is_probe = !!(info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE);
 	pkt_id = mt7601u_tx_pktid_enc(dev, rate_ctl & 0x7, is_probe);
-	pkt_len |= MT76_SET(MT_TXWI_LEN_PKTID, pkt_id);
+	pkt_len |= FIELD_PREP(MT_TXWI_LEN_PKTID, pkt_id);
 	txwi->len_ctl = cpu_to_le16(pkt_len);
 
 	return txwi;
@@ -285,9 +286,9 @@ int mt7601u_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	WARN_ON(cw_min > 0xf);
 	WARN_ON(cw_max > 0xf);
 
-	val = MT76_SET(MT_EDCA_CFG_AIFSN, params->aifs) |
-	      MT76_SET(MT_EDCA_CFG_CWMIN, cw_min) |
-	      MT76_SET(MT_EDCA_CFG_CWMAX, cw_max);
+	val = FIELD_PREP(MT_EDCA_CFG_AIFSN, params->aifs) |
+	      FIELD_PREP(MT_EDCA_CFG_CWMIN, cw_min) |
+	      FIELD_PREP(MT_EDCA_CFG_CWMAX, cw_max);
 	/* TODO: based on user-controlled EnableTxBurst var vendor drv sets
 	 *	 a really long txop on AC0 (see connect.c:2009) but only on
 	 *	 connect? When not connected should be 0.
@@ -295,7 +296,7 @@ int mt7601u_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	if (!hw_q)
 		val |= 0x60;
 	else
-		val |= MT76_SET(MT_EDCA_CFG_TXOP, params->txop);
+		val |= FIELD_PREP(MT_EDCA_CFG_TXOP, params->txop);
 	mt76_wr(dev, MT_EDCA_CFG_AC(hw_q), val);
 
 	val = mt76_rr(dev, MT_WMM_TXOP(hw_q));
diff --git a/drivers/net/wireless/mediatek/mt7601u/util.h b/drivers/net/wireless/mediatek/mt7601u/util.h
deleted file mode 100644
index b89140bf1210..000000000000
--- a/drivers/net/wireless/mediatek/mt7601u/util.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright (C) 2014 Felix Fietkau <nbd@openwrt.org>
- * Copyright (C) 2004 - 2009 Ivo van Doorn <IvDoorn@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2
- * as published by the Free Software Foundation
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef __MT76_UTIL_H
-#define __MT76_UTIL_H
-
-/*
- * Power of two check, this will check
- * if the mask that has been given contains and contiguous set of bits.
- * Note that we cannot use the is_power_of_2() function since this
- * check must be done at compile-time.
- */
-#define is_power_of_two(x)	( !((x) & ((x)-1)) )
-#define low_bit_mask(x)		( ((x)-1) & ~(x) )
-#define is_valid_mask(x)	is_power_of_two(1LU + (x) + low_bit_mask(x))
-
-/*
- * Macros to find first set bit in a variable.
- * These macros behave the same as the __ffs() functions but
- * the most important difference that this is done during
- * compile-time rather then run-time.
- */
-#define compile_ffs2(__x) \
-	__builtin_choose_expr(((__x) & 0x1), 0, 1)
-
-#define compile_ffs4(__x) \
-	__builtin_choose_expr(((__x) & 0x3), \
-			      (compile_ffs2((__x))), \
-			      (compile_ffs2((__x) >> 2) + 2))
-
-#define compile_ffs8(__x) \
-	__builtin_choose_expr(((__x) & 0xf), \
-			      (compile_ffs4((__x))), \
-			      (compile_ffs4((__x) >> 4) + 4))
-
-#define compile_ffs16(__x) \
-	__builtin_choose_expr(((__x) & 0xff), \
-			      (compile_ffs8((__x))), \
-			      (compile_ffs8((__x) >> 8) + 8))
-
-#define compile_ffs32(__x) \
-	__builtin_choose_expr(((__x) & 0xffff), \
-			      (compile_ffs16((__x))), \
-			      (compile_ffs16((__x) >> 16) + 16))
-
-/*
- * This macro will check the requirements for the FIELD{8,16,32} macros
- * The mask should be a constant non-zero contiguous set of bits which
- * does not exceed the given typelimit.
- */
-#define FIELD_CHECK(__mask) \
-	BUILD_BUG_ON(!(__mask) || !is_valid_mask(__mask))
-
-#define MT76_SET(_mask, _val)						\
-	({								\
-		FIELD_CHECK(_mask);					\
-		(((u32) (_val)) << compile_ffs32(_mask)) & _mask;	\
-	})
-
-#define MT76_GET(_mask, _val)						\
-	({								\
-		FIELD_CHECK(_mask);					\
-		(u32) (((_val) & _mask) >> compile_ffs32(_mask));	\
-	})
-
-#endif
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
index 7cf26c6124d1..6005e14213ca 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
@@ -831,8 +831,10 @@ int rt2x00usb_probe(struct usb_interface *usb_intf,
 	rt2x00dev->anchor = devm_kmalloc(&usb_dev->dev,
 					sizeof(struct usb_anchor),
 					GFP_KERNEL);
-	if (!rt2x00dev->anchor)
+	if (!rt2x00dev->anchor) {
+		retval = -ENOMEM;
 		goto exit_free_reg;
+	}
 
 	init_usb_anchor(rt2x00dev->anchor);
 	return 0;
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
index 4341d56805f8..1016628926d2 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
@@ -29,6 +29,7 @@
 #define RTL8XXXU_DEBUG_H2C		0x800
 #define RTL8XXXU_DEBUG_ACTION		0x1000
 #define RTL8XXXU_DEBUG_EFUSE		0x2000
+#define RTL8XXXU_DEBUG_INTERRUPT	0x4000
 
 #define RTW_USB_CONTROL_MSG_TIMEOUT	500
 #define RTL8XXXU_MAX_REG_POLL		500
@@ -43,6 +44,7 @@
 
 #define TX_TOTAL_PAGE_NUM		0xf8
 #define TX_TOTAL_PAGE_NUM_8192E		0xf3
+#define TX_TOTAL_PAGE_NUM_8723B		0xf7
 /* (HPQ + LPQ + NPQ + PUBQ) = TX_TOTAL_PAGE_NUM */
 #define TX_PAGE_NUM_PUBQ		0xe7
 #define TX_PAGE_NUM_HI_PQ		0x0c
@@ -54,6 +56,11 @@
 #define TX_PAGE_NUM_LO_PQ_8192E		0x0c
 #define TX_PAGE_NUM_NORM_PQ_8192E	0x00
 
+#define TX_PAGE_NUM_PUBQ_8723B		0xe7
+#define TX_PAGE_NUM_HI_PQ_8723B		0x0c
+#define TX_PAGE_NUM_LO_PQ_8723B		0x02
+#define TX_PAGE_NUM_NORM_PQ_8723B	0x02
+
 #define RTL_FW_PAGE_SIZE		4096
 #define RTL8XXXU_FIRMWARE_POLL_MAX	1000
 
@@ -1312,7 +1319,7 @@ struct rtl8xxxu_fileops {
 	int (*power_on) (struct rtl8xxxu_priv *priv);
 	void (*power_off) (struct rtl8xxxu_priv *priv);
 	void (*reset_8051) (struct rtl8xxxu_priv *priv);
-	int (*llt_init) (struct rtl8xxxu_priv *priv, u8 last_tx_page);
+	int (*llt_init) (struct rtl8xxxu_priv *priv);
 	void (*init_phy_bb) (struct rtl8xxxu_priv *priv);
 	int (*init_phy_rf) (struct rtl8xxxu_priv *priv);
 	void (*phy_init_antenna_selection) (struct rtl8xxxu_priv *priv);
@@ -1330,11 +1337,17 @@ struct rtl8xxxu_fileops {
 				  u32 ramask, int sgi);
 	void (*report_connect) (struct rtl8xxxu_priv *priv,
 				u8 macid, bool connect);
+	void (*fill_txdesc) (struct ieee80211_hdr *hdr,
+			     struct rtl8xxxu_txdesc32 *tx_desc, u32 rate,
+			     u16 rate_flag, bool sgi, bool short_preamble,
+			     bool ampdu_enable);
 	int writeN_block_size;
 	int rx_agg_buf_size;
 	char tx_desc_size;
 	char rx_desc_size;
-	char has_s0s1;
+	u8 has_s0s1:1;
+	u8 has_tx_report:1;
+	u8 gen2_thermal_meter:1;
 	u32 adda_1t_init;
 	u32 adda_1t_path_on;
 	u32 adda_2t_path_on_a;
@@ -1388,14 +1401,14 @@ int rtl8xxxu_load_firmware(struct rtl8xxxu_priv *priv, char *fw_name);
 void rtl8xxxu_firmware_self_reset(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_power_off(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_reset_8051(struct rtl8xxxu_priv *priv);
-int rtl8xxxu_auto_llt_table(struct rtl8xxxu_priv *priv, u8 last_tx_page);
+int rtl8xxxu_auto_llt_table(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_gen2_prepare_calibrate(struct rtl8xxxu_priv *priv, u8 start);
 int rtl8xxxu_flush_fifo(struct rtl8xxxu_priv *priv);
 int rtl8xxxu_gen2_h2c_cmd(struct rtl8xxxu_priv *priv,
 			  struct h2c_cmd *h2c, int len);
 int rtl8xxxu_active_to_lps(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_disabled_to_emu(struct rtl8xxxu_priv *priv);
-int rtl8xxxu_init_llt_table(struct rtl8xxxu_priv *priv, u8 last_tx_page);
+int rtl8xxxu_init_llt_table(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_gen1_phy_iq_calibrate(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_gen1_init_phy_bb(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_gen1_set_tx_power(struct rtl8xxxu_priv *priv,
@@ -1421,6 +1434,14 @@ int rtl8xxxu_parse_rxdesc24(struct rtl8xxxu_priv *priv, struct sk_buff *skb);
 int rtl8xxxu_gen2_channel_to_group(int channel);
 bool rtl8xxxu_gen2_simularity_compare(struct rtl8xxxu_priv *priv,
 				      int result[][8], int c1, int c2);
+void rtl8xxxu_fill_txdesc_v1(struct ieee80211_hdr *hdr,
+			     struct rtl8xxxu_txdesc32 *tx_desc, u32 rate,
+			     u16 rate_flag, bool sgi, bool short_preamble,
+			     bool ampdu_enable);
+void rtl8xxxu_fill_txdesc_v2(struct ieee80211_hdr *hdr,
+			     struct rtl8xxxu_txdesc32 *tx_desc32, u32 rate,
+			     u16 rate_flag, bool sgi, bool short_preamble,
+			     bool ampdu_enable);
 
 extern struct rtl8xxxu_fileops rtl8192cu_fops;
 extern struct rtl8xxxu_fileops rtl8192eu_fops;
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192c.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192c.c
index 69d1a1453ede..f9e2050812ab 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192c.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192c.c
@@ -567,6 +567,7 @@ struct rtl8xxxu_fileops rtl8192cu_fops = {
 	.set_tx_power = rtl8xxxu_gen1_set_tx_power,
 	.update_rate_mask = rtl8xxxu_update_rate_mask,
 	.report_connect = rtl8xxxu_gen1_report_connect,
+	.fill_txdesc = rtl8xxxu_fill_txdesc_v1,
 	.writeN_block_size = 128,
 	.rx_agg_buf_size = 16000,
 	.tx_desc_size = sizeof(struct rtl8xxxu_txdesc32),
@@ -579,5 +580,9 @@ struct rtl8xxxu_fileops rtl8192cu_fops = {
 	.pbp_rx = PBP_PAGE_SIZE_128,
 	.pbp_tx = PBP_PAGE_SIZE_128,
 	.mactable = rtl8xxxu_gen1_mac_init_table,
+	.total_page_num = TX_TOTAL_PAGE_NUM,
+	.page_num_hi = TX_PAGE_NUM_HI_PQ,
+	.page_num_lo = TX_PAGE_NUM_LO_PQ,
+	.page_num_norm = TX_PAGE_NUM_NORM_PQ,
 };
 #endif
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c
index 9a1994f49b7b..df54d27e7851 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c
@@ -1396,6 +1396,114 @@ exit:
 	return ret;
 }
 
+static int rtl8192eu_active_to_lps(struct rtl8xxxu_priv *priv)
+{
+	struct device *dev = &priv->udev->dev;
+	u8 val8;
+	u16 val16;
+	u32 val32;
+	int retry, retval;
+
+	rtl8xxxu_write8(priv, REG_TXPAUSE, 0xff);
+
+	retry = 100;
+	retval = -EBUSY;
+	/*
+	 * Poll 32 bit wide 0x05f8 for 0x00000000 to ensure no TX is pending.
+	 */
+	do {
+		val32 = rtl8xxxu_read32(priv, REG_SCH_TX_CMD);
+		if (!val32) {
+			retval = 0;
+			break;
+		}
+	} while (retry--);
+
+	if (!retry) {
+		dev_warn(dev, "Failed to flush TX queue\n");
+		retval = -EBUSY;
+		goto out;
+	}
+
+	/* Disable CCK and OFDM, clock gated */
+	val8 = rtl8xxxu_read8(priv, REG_SYS_FUNC);
+	val8 &= ~SYS_FUNC_BBRSTB;
+	rtl8xxxu_write8(priv, REG_SYS_FUNC, val8);
+
+	udelay(2);
+
+	/* Reset whole BB */
+	val8 = rtl8xxxu_read8(priv, REG_SYS_FUNC);
+	val8 &= ~SYS_FUNC_BB_GLB_RSTN;
+	rtl8xxxu_write8(priv, REG_SYS_FUNC, val8);
+
+	/* Reset MAC TRX */
+	val16 = rtl8xxxu_read16(priv, REG_CR);
+	val16 &= 0xff00;
+	val16 |= (CR_HCI_TXDMA_ENABLE | CR_HCI_RXDMA_ENABLE);
+	rtl8xxxu_write16(priv, REG_CR, val16);
+
+	val16 = rtl8xxxu_read16(priv, REG_CR);
+	val16 &= ~CR_SECURITY_ENABLE;
+	rtl8xxxu_write16(priv, REG_CR, val16);
+
+	val8 = rtl8xxxu_read8(priv, REG_DUAL_TSF_RST);
+	val8 |= DUAL_TSF_TX_OK;
+	rtl8xxxu_write8(priv, REG_DUAL_TSF_RST, val8);
+
+out:
+	return retval;
+}
+
+static int rtl8192eu_active_to_emu(struct rtl8xxxu_priv *priv)
+{
+	u8 val8;
+	int count, ret = 0;
+
+	/* Turn off RF */
+	rtl8xxxu_write8(priv, REG_RF_CTRL, 0);
+
+	/* Switch DPDT_SEL_P output from register 0x65[2] */
+	val8 = rtl8xxxu_read8(priv, REG_LEDCFG2);
+	val8 &= ~LEDCFG2_DPDT_SELECT;
+	rtl8xxxu_write8(priv, REG_LEDCFG2, val8);
+
+	/* 0x0005[1] = 1 turn off MAC by HW state machine*/
+	val8 = rtl8xxxu_read8(priv, REG_APS_FSMCO + 1);
+	val8 |= BIT(1);
+	rtl8xxxu_write8(priv, REG_APS_FSMCO + 1, val8);
+
+	for (count = RTL8XXXU_MAX_REG_POLL; count; count--) {
+		val8 = rtl8xxxu_read8(priv, REG_APS_FSMCO + 1);
+		if ((val8 & BIT(1)) == 0)
+			break;
+		udelay(10);
+	}
+
+	if (!count) {
+		dev_warn(&priv->udev->dev, "%s: Disabling MAC timed out\n",
+			 __func__);
+		ret = -EBUSY;
+		goto exit;
+	}
+
+exit:
+	return ret;
+}
+
+static int rtl8192eu_emu_to_disabled(struct rtl8xxxu_priv *priv)
+{
+	u8 val8;
+
+	/* 0x04[12:11] = 01 enable WL suspend */
+	val8 = rtl8xxxu_read8(priv, REG_APS_FSMCO + 1);
+	val8 &= ~(BIT(3) | BIT(4));
+	val8 |= BIT(3);
+	rtl8xxxu_write8(priv, REG_APS_FSMCO + 1, val8);
+
+	return 0;
+}
+
 static int rtl8192eu_power_on(struct rtl8xxxu_priv *priv)
 {
 	u16 val16;
@@ -1446,6 +1554,40 @@ exit:
 	return ret;
 }
 
+void rtl8192eu_power_off(struct rtl8xxxu_priv *priv)
+{
+	u8 val8;
+	u16 val16;
+
+	rtl8xxxu_flush_fifo(priv);
+
+	val8 = rtl8xxxu_read8(priv, REG_TX_REPORT_CTRL);
+	val8 &= ~TX_REPORT_CTRL_TIMER_ENABLE;
+	rtl8xxxu_write8(priv, REG_TX_REPORT_CTRL, val8);
+
+	/* Turn off RF */
+	rtl8xxxu_write8(priv, REG_RF_CTRL, 0x00);
+
+	rtl8192eu_active_to_lps(priv);
+
+	/* Reset Firmware if running in RAM */
+	if (rtl8xxxu_read8(priv, REG_MCU_FW_DL) & MCU_FW_RAM_SEL)
+		rtl8xxxu_firmware_self_reset(priv);
+
+	/* Reset MCU */
+	val16 = rtl8xxxu_read16(priv, REG_SYS_FUNC);
+	val16 &= ~SYS_FUNC_CPU_ENABLE;
+	rtl8xxxu_write16(priv, REG_SYS_FUNC, val16);
+
+	/* Reset MCU ready status */
+	rtl8xxxu_write8(priv, REG_MCU_FW_DL, 0x00);
+
+	rtl8xxxu_reset_8051(priv);
+
+	rtl8192eu_active_to_emu(priv);
+	rtl8192eu_emu_to_disabled(priv);
+}
+
 static void rtl8192e_enable_rf(struct rtl8xxxu_priv *priv)
 {
 	u32 val32;
@@ -1487,7 +1629,7 @@ struct rtl8xxxu_fileops rtl8192eu_fops = {
 	.parse_efuse = rtl8192eu_parse_efuse,
 	.load_firmware = rtl8192eu_load_firmware,
 	.power_on = rtl8192eu_power_on,
-	.power_off = rtl8xxxu_power_off,
+	.power_off = rtl8192eu_power_off,
 	.reset_8051 = rtl8xxxu_reset_8051,
 	.llt_init = rtl8xxxu_auto_llt_table,
 	.init_phy_bb = rtl8192eu_init_phy_bb,
@@ -1501,10 +1643,12 @@ struct rtl8xxxu_fileops rtl8192eu_fops = {
 	.set_tx_power = rtl8192e_set_tx_power,
 	.update_rate_mask = rtl8xxxu_gen2_update_rate_mask,
 	.report_connect = rtl8xxxu_gen2_report_connect,
+	.fill_txdesc = rtl8xxxu_fill_txdesc_v2,
 	.writeN_block_size = 128,
 	.tx_desc_size = sizeof(struct rtl8xxxu_txdesc40),
 	.rx_desc_size = sizeof(struct rtl8xxxu_rxdesc24),
 	.has_s0s1 = 0,
+	.gen2_thermal_meter = 1,
 	.adda_1t_init = 0x0fc01616,
 	.adda_1t_path_on = 0x0fc01616,
 	.adda_2t_path_on_a = 0x0fc01616,
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723a.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723a.c
index 686c551581b1..aef373028155 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723a.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723a.c
@@ -384,6 +384,7 @@ struct rtl8xxxu_fileops rtl8723au_fops = {
 	.set_tx_power = rtl8xxxu_gen1_set_tx_power,
 	.update_rate_mask = rtl8xxxu_update_rate_mask,
 	.report_connect = rtl8xxxu_gen1_report_connect,
+	.fill_txdesc = rtl8xxxu_fill_txdesc_v1,
 	.writeN_block_size = 1024,
 	.rx_agg_buf_size = 16000,
 	.tx_desc_size = sizeof(struct rtl8xxxu_txdesc32),
@@ -396,4 +397,8 @@ struct rtl8xxxu_fileops rtl8723au_fops = {
 	.pbp_rx = PBP_PAGE_SIZE_128,
 	.pbp_tx = PBP_PAGE_SIZE_128,
 	.mactable = rtl8xxxu_gen1_mac_init_table,
+	.total_page_num = TX_TOTAL_PAGE_NUM,
+	.page_num_hi = TX_PAGE_NUM_HI_PQ,
+	.page_num_lo = TX_PAGE_NUM_LO_PQ,
+	.page_num_norm = TX_PAGE_NUM_NORM_PQ,
 };
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c
index 9d45afb0e3fd..6c086b5657e9 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c
@@ -1662,10 +1662,13 @@ struct rtl8xxxu_fileops rtl8723bu_fops = {
 	.set_tx_power = rtl8723b_set_tx_power,
 	.update_rate_mask = rtl8xxxu_gen2_update_rate_mask,
 	.report_connect = rtl8xxxu_gen2_report_connect,
+	.fill_txdesc = rtl8xxxu_fill_txdesc_v2,
 	.writeN_block_size = 1024,
 	.tx_desc_size = sizeof(struct rtl8xxxu_txdesc40),
 	.rx_desc_size = sizeof(struct rtl8xxxu_rxdesc24),
 	.has_s0s1 = 1,
+	.has_tx_report = 1,
+	.gen2_thermal_meter = 1,
 	.adda_1t_init = 0x01c00014,
 	.adda_1t_path_on = 0x01c00014,
 	.adda_2t_path_on_a = 0x01c00014,
@@ -1674,4 +1677,8 @@ struct rtl8xxxu_fileops rtl8723bu_fops = {
 	.pbp_rx = PBP_PAGE_SIZE_256,
 	.pbp_tx = PBP_PAGE_SIZE_256,
 	.mactable = rtl8723b_mac_init_table,
+	.total_page_num = TX_TOTAL_PAGE_NUM_8723B,
+	.page_num_hi = TX_PAGE_NUM_HI_PQ_8723B,
+	.page_num_lo = TX_PAGE_NUM_LO_PQ_8723B,
+	.page_num_norm = TX_PAGE_NUM_NORM_PQ_8723B,
 };
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
index 77048db3b32a..b2d7f6e69667 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
@@ -894,7 +894,7 @@ int rtl8xxxu_write_rfreg(struct rtl8xxxu_priv *priv,
 	return retval;
 }
 
-int
+static int
 rtl8xxxu_gen1_h2c_cmd(struct rtl8xxxu_priv *priv, struct h2c_cmd *h2c, int len)
 {
 	struct device *dev = &priv->udev->dev;
@@ -2472,10 +2472,13 @@ static int rtl8xxxu_llt_write(struct rtl8xxxu_priv *priv, u8 address, u8 data)
 	return ret;
 }
 
-int rtl8xxxu_init_llt_table(struct rtl8xxxu_priv *priv, u8 last_tx_page)
+int rtl8xxxu_init_llt_table(struct rtl8xxxu_priv *priv)
 {
 	int ret;
 	int i;
+	u8 last_tx_page;
+
+	last_tx_page = priv->fops->total_page_num;
 
 	for (i = 0; i < last_tx_page; i++) {
 		ret = rtl8xxxu_llt_write(priv, i, i + 1);
@@ -2503,7 +2506,7 @@ exit:
 	return ret;
 }
 
-int rtl8xxxu_auto_llt_table(struct rtl8xxxu_priv *priv, u8 last_tx_page)
+int rtl8xxxu_auto_llt_table(struct rtl8xxxu_priv *priv)
 {
 	u32 val32;
 	int ret = 0;
@@ -3847,28 +3850,6 @@ void rtl8xxxu_gen2_disable_rf(struct rtl8xxxu_priv *priv)
 	rtl8xxxu_write32(priv, REG_RX_WAIT_CCA, val32);
 }
 
-static void rtl8xxxu_old_init_queue_reserved_page(struct rtl8xxxu_priv *priv)
-{
-	u8 val8;
-	u32 val32;
-
-	if (priv->ep_tx_normal_queue)
-		val8 = TX_PAGE_NUM_NORM_PQ;
-	else
-		val8 = 0;
-
-	rtl8xxxu_write8(priv, REG_RQPN_NPQ, val8);
-
-	val32 = (TX_PAGE_NUM_PUBQ << RQPN_PUB_PQ_SHIFT) | RQPN_LOAD;
-
-	if (priv->ep_tx_high_queue)
-		val32 |= (TX_PAGE_NUM_HI_PQ << RQPN_HI_PQ_SHIFT);
-	if (priv->ep_tx_low_queue)
-		val32 |= (TX_PAGE_NUM_LO_PQ << RQPN_LO_PQ_SHIFT);
-
-	rtl8xxxu_write32(priv, REG_RQPN, val32);
-}
-
 static void rtl8xxxu_init_queue_reserved_page(struct rtl8xxxu_priv *priv)
 {
 	struct rtl8xxxu_fileops *fops = priv->fops;
@@ -3891,7 +3872,7 @@ static void rtl8xxxu_init_queue_reserved_page(struct rtl8xxxu_priv *priv)
 	val32 = (nq << RQPN_NPQ_SHIFT) | (eq << RQPN_EPQ_SHIFT);
 	rtl8xxxu_write32(priv, REG_RQPN_NPQ, val32);
 
-	pubq = fops->total_page_num - hq - lq - nq;
+	pubq = fops->total_page_num - hq - lq - nq - 1;
 
 	val32 = RQPN_LOAD;
 	val32 |= (hq << RQPN_HI_PQ_SHIFT);
@@ -3905,6 +3886,7 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 {
 	struct rtl8xxxu_priv *priv = hw->priv;
 	struct device *dev = &priv->udev->dev;
+	struct rtl8xxxu_fileops *fops = priv->fops;
 	bool macpower;
 	int ret;
 	u8 val8;
@@ -3923,18 +3905,14 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	else
 		macpower = true;
 
-	ret = priv->fops->power_on(priv);
+	ret = fops->power_on(priv);
 	if (ret < 0) {
 		dev_warn(dev, "%s: Failed power on\n", __func__);
 		goto exit;
 	}
 
-	if (!macpower) {
-		if (priv->fops->total_page_num)
-			rtl8xxxu_init_queue_reserved_page(priv);
-		else
-			rtl8xxxu_old_init_queue_reserved_page(priv);
-	}
+	if (!macpower)
+		rtl8xxxu_init_queue_reserved_page(priv);
 
 	ret = rtl8xxxu_init_queue_priority(priv);
 	dev_dbg(dev, "%s: init_queue_priority %i\n", __func__, ret);
@@ -3944,19 +3922,19 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	/*
 	 * Set RX page boundary
 	 */
-	rtl8xxxu_write16(priv, REG_TRXFF_BNDY + 2, priv->fops->trxff_boundary);
+	rtl8xxxu_write16(priv, REG_TRXFF_BNDY + 2, fops->trxff_boundary);
 
 	ret = rtl8xxxu_download_firmware(priv);
-	dev_dbg(dev, "%s: download_fiwmare %i\n", __func__, ret);
+	dev_dbg(dev, "%s: download_firmware %i\n", __func__, ret);
 	if (ret)
 		goto exit;
 	ret = rtl8xxxu_start_firmware(priv);
-	dev_dbg(dev, "%s: start_fiwmare %i\n", __func__, ret);
+	dev_dbg(dev, "%s: start_firmware %i\n", __func__, ret);
 	if (ret)
 		goto exit;
 
-	if (priv->fops->phy_init_antenna_selection)
-		priv->fops->phy_init_antenna_selection(priv);
+	if (fops->phy_init_antenna_selection)
+		fops->phy_init_antenna_selection(priv);
 
 	ret = rtl8xxxu_init_mac(priv);
 
@@ -3969,7 +3947,7 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	if (ret)
 		goto exit;
 
-	ret = priv->fops->init_phy_rf(priv);
+	ret = fops->init_phy_rf(priv);
 	if (ret)
 		goto exit;
 
@@ -3994,13 +3972,7 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 		/*
 		 * Set TX buffer boundary
 		 */
-		if (priv->rtl_chip == RTL8192E)
-			val8 = TX_TOTAL_PAGE_NUM_8192E + 1;
-		else
-			val8 = TX_TOTAL_PAGE_NUM + 1;
-
-		if (priv->rtl_chip == RTL8723B)
-			val8 -= 1;
+		val8 = fops->total_page_num + 1;
 
 		rtl8xxxu_write8(priv, REG_TXPKTBUF_BCNQ_BDNY, val8);
 		rtl8xxxu_write8(priv, REG_TXPKTBUF_MGQ_BDNY, val8);
@@ -4013,14 +3985,14 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	 * The vendor drivers set PBP for all devices, except 8192e.
 	 * There is no explanation for this in any of the sources.
 	 */
-	val8 = (priv->fops->pbp_rx << PBP_PAGE_SIZE_RX_SHIFT) |
-		(priv->fops->pbp_tx << PBP_PAGE_SIZE_TX_SHIFT);
+	val8 = (fops->pbp_rx << PBP_PAGE_SIZE_RX_SHIFT) |
+		(fops->pbp_tx << PBP_PAGE_SIZE_TX_SHIFT);
 	if (priv->rtl_chip != RTL8192E)
 		rtl8xxxu_write8(priv, REG_PBP, val8);
 
 	dev_dbg(dev, "%s: macpower %i\n", __func__, macpower);
 	if (!macpower) {
-		ret = priv->fops->llt_init(priv, TX_TOTAL_PAGE_NUM);
+		ret = fops->llt_init(priv);
 		if (ret) {
 			dev_warn(dev, "%s: LLT table init failed\n", __func__);
 			goto exit;
@@ -4029,13 +4001,12 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 		/*
 		 * Chip specific quirks
 		 */
-		priv->fops->usb_quirks(priv);
+		fops->usb_quirks(priv);
 
 		/*
-		 * Presumably this is for 8188EU as well
-		 * Enable TX report and TX report timer
+		 * Enable TX report and TX report timer for 8723bu/8188eu/...
 		 */
-		if (priv->rtl_chip == RTL8723B) {
+		if (fops->has_tx_report) {
 			val8 = rtl8xxxu_read8(priv, REG_TX_REPORT_CTRL);
 			val8 |= TX_REPORT_CTRL_TIMER_ENABLE;
 			rtl8xxxu_write8(priv, REG_TX_REPORT_CTRL, val8);
@@ -4170,8 +4141,8 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 		rtl8xxxu_write8(priv, REG_RSV_CTRL, val8);
 	}
 
-	if (priv->fops->init_aggregation)
-		priv->fops->init_aggregation(priv);
+	if (fops->init_aggregation)
+		fops->init_aggregation(priv);
 
 	/*
 	 * Enable CCK and OFDM block
@@ -4188,7 +4159,7 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 	/*
 	 * Start out with default power levels for channel 6, 20MHz
 	 */
-	priv->fops->set_tx_power(priv, 1, false);
+	fops->set_tx_power(priv, 1, false);
 
 	/* Let the 8051 take control of antenna setting */
 	if (priv->rtl_chip != RTL8192E) {
@@ -4204,8 +4175,8 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 
 	rtl8xxxu_write16(priv, REG_FAST_EDCA_CTRL, 0);
 
-	if (priv->fops->init_statistics)
-		priv->fops->init_statistics(priv);
+	if (fops->init_statistics)
+		fops->init_statistics(priv);
 
 	if (priv->rtl_chip == RTL8192E) {
 		/*
@@ -4223,12 +4194,12 @@ static int rtl8xxxu_init_device(struct ieee80211_hw *hw)
 
 	rtl8723a_phy_lc_calibrate(priv);
 
-	priv->fops->phy_iq_calibrate(priv);
+	fops->phy_iq_calibrate(priv);
 
 	/*
 	 * This should enable thermal meter
 	 */
-	if (priv->fops->tx_desc_size == sizeof(struct rtl8xxxu_txdesc40))
+	if (fops->gen2_thermal_meter)
 		rtl8xxxu_write_rfreg(priv,
 				     RF_A, RF6052_REG_T_METER_8723B, 0x37cf8);
 	else
@@ -4783,6 +4754,113 @@ static void rtl8xxxu_dump_action(struct device *dev,
 	}
 }
 
+/*
+ * Fill in v1 (gen1) specific TX descriptor bits.
+ * This format is used on 8188cu/8192cu/8723au
+ */
+void
+rtl8xxxu_fill_txdesc_v1(struct ieee80211_hdr *hdr,
+			struct rtl8xxxu_txdesc32 *tx_desc, u32 rate,
+			u16 rate_flag, bool sgi, bool short_preamble,
+			bool ampdu_enable)
+{
+	u16 seq_number;
+
+	seq_number = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
+
+	tx_desc->txdw5 = cpu_to_le32(rate);
+
+	if (ieee80211_is_data(hdr->frame_control))
+		tx_desc->txdw5 |= cpu_to_le32(0x0001ff00);
+
+	tx_desc->txdw3 = cpu_to_le32((u32)seq_number << TXDESC32_SEQ_SHIFT);
+
+	if (ampdu_enable)
+		tx_desc->txdw1 |= cpu_to_le32(TXDESC32_AGG_ENABLE);
+	else
+		tx_desc->txdw1 |= cpu_to_le32(TXDESC32_AGG_BREAK);
+
+	if (ieee80211_is_mgmt(hdr->frame_control)) {
+		tx_desc->txdw5 = cpu_to_le32(rate);
+		tx_desc->txdw4 |= cpu_to_le32(TXDESC32_USE_DRIVER_RATE);
+		tx_desc->txdw5 |= cpu_to_le32(6 << TXDESC32_RETRY_LIMIT_SHIFT);
+		tx_desc->txdw5 |= cpu_to_le32(TXDESC32_RETRY_LIMIT_ENABLE);
+	}
+
+	if (ieee80211_is_data_qos(hdr->frame_control))
+		tx_desc->txdw4 |= cpu_to_le32(TXDESC32_QOS);
+
+	if (short_preamble)
+		tx_desc->txdw4 |= cpu_to_le32(TXDESC32_SHORT_PREAMBLE);
+
+	if (sgi)
+		tx_desc->txdw5 |= cpu_to_le32(TXDESC32_SHORT_GI);
+
+	if (rate_flag & IEEE80211_TX_RC_USE_RTS_CTS) {
+		/*
+		 * Use RTS rate 24M - does the mac80211 tell
+		 * us which to use?
+		 */
+		tx_desc->txdw4 |= cpu_to_le32(DESC_RATE_24M <<
+					      TXDESC32_RTS_RATE_SHIFT);
+		tx_desc->txdw4 |= cpu_to_le32(TXDESC32_RTS_CTS_ENABLE);
+		tx_desc->txdw4 |= cpu_to_le32(TXDESC32_HW_RTS_ENABLE);
+	}
+}
+
+/*
+ * Fill in v2 (gen2) specific TX descriptor bits.
+ * This format is used on 8192eu/8723bu
+ */
+void
+rtl8xxxu_fill_txdesc_v2(struct ieee80211_hdr *hdr,
+			struct rtl8xxxu_txdesc32 *tx_desc32, u32 rate,
+			u16 rate_flag, bool sgi, bool short_preamble,
+			bool ampdu_enable)
+{
+	struct rtl8xxxu_txdesc40 *tx_desc40;
+	u16 seq_number;
+
+	tx_desc40 = (struct rtl8xxxu_txdesc40 *)tx_desc32;
+
+	seq_number = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
+
+	tx_desc40->txdw4 = cpu_to_le32(rate);
+	if (ieee80211_is_data(hdr->frame_control)) {
+		tx_desc40->txdw4 |= cpu_to_le32(0x1f <<
+						TXDESC40_DATA_RATE_FB_SHIFT);
+	}
+
+	tx_desc40->txdw9 = cpu_to_le32((u32)seq_number << TXDESC40_SEQ_SHIFT);
+
+	if (ampdu_enable)
+		tx_desc40->txdw2 |= cpu_to_le32(TXDESC40_AGG_ENABLE);
+	else
+		tx_desc40->txdw2 |= cpu_to_le32(TXDESC40_AGG_BREAK);
+
+	if (ieee80211_is_mgmt(hdr->frame_control)) {
+		tx_desc40->txdw4 = cpu_to_le32(rate);
+		tx_desc40->txdw3 |= cpu_to_le32(TXDESC40_USE_DRIVER_RATE);
+		tx_desc40->txdw4 |=
+			cpu_to_le32(6 << TXDESC40_RETRY_LIMIT_SHIFT);
+		tx_desc40->txdw4 |= cpu_to_le32(TXDESC40_RETRY_LIMIT_ENABLE);
+	}
+
+	if (short_preamble)
+		tx_desc40->txdw5 |= cpu_to_le32(TXDESC40_SHORT_PREAMBLE);
+
+	if (rate_flag & IEEE80211_TX_RC_USE_RTS_CTS) {
+		/*
+		 * Use RTS rate 24M - does the mac80211 tell
+		 * us which to use?
+		 */
+		tx_desc40->txdw4 |= cpu_to_le32(DESC_RATE_24M <<
+						TXDESC40_RTS_RATE_SHIFT);
+		tx_desc40->txdw3 |= cpu_to_le32(TXDESC40_RTS_CTS_ENABLE);
+		tx_desc40->txdw3 |= cpu_to_le32(TXDESC40_HW_RTS_ENABLE);
+	}
+}
+
 static void rtl8xxxu_tx(struct ieee80211_hw *hw,
 			struct ieee80211_tx_control *control,
 			struct sk_buff *skb)
@@ -4792,7 +4870,6 @@ static void rtl8xxxu_tx(struct ieee80211_hw *hw,
 	struct ieee80211_rate *tx_rate = ieee80211_get_tx_rate(hw, tx_info);
 	struct rtl8xxxu_priv *priv = hw->priv;
 	struct rtl8xxxu_txdesc32 *tx_desc;
-	struct rtl8xxxu_txdesc40 *tx_desc40;
 	struct rtl8xxxu_tx_urb *tx_urb;
 	struct ieee80211_sta *sta = NULL;
 	struct ieee80211_vif *vif = tx_info->control.vif;
@@ -4803,7 +4880,7 @@ static void rtl8xxxu_tx(struct ieee80211_hw *hw,
 	u16 rate_flag = tx_info->control.rates[0].flags;
 	int tx_desc_size = priv->fops->tx_desc_size;
 	int ret;
-	bool usedesc40, ampdu_enable;
+	bool usedesc40, ampdu_enable, sgi = false, short_preamble = false;
 
 	if (skb_headroom(skb) < tx_desc_size) {
 		dev_warn(dev,
@@ -4881,107 +4958,26 @@ static void rtl8xxxu_tx(struct ieee80211_hw *hw,
 		}
 	}
 
-	if (rate_flag & IEEE80211_TX_RC_MCS)
+	if (rate_flag & IEEE80211_TX_RC_MCS &&
+	    !ieee80211_is_mgmt(hdr->frame_control))
 		rate = tx_info->control.rates[0].idx + DESC_RATE_MCS0;
 	else
 		rate = tx_rate->hw_value;
 
-	seq_number = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
-	if (!usedesc40) {
-		tx_desc->txdw5 = cpu_to_le32(rate);
-
-		if (ieee80211_is_data(hdr->frame_control))
-			tx_desc->txdw5 |= cpu_to_le32(0x0001ff00);
-
-		tx_desc->txdw3 =
-			cpu_to_le32((u32)seq_number << TXDESC32_SEQ_SHIFT);
-
-		if (ampdu_enable)
-			tx_desc->txdw1 |= cpu_to_le32(TXDESC32_AGG_ENABLE);
-		else
-			tx_desc->txdw1 |= cpu_to_le32(TXDESC32_AGG_BREAK);
-
-		if (ieee80211_is_mgmt(hdr->frame_control)) {
-			tx_desc->txdw5 = cpu_to_le32(tx_rate->hw_value);
-			tx_desc->txdw4 |=
-				cpu_to_le32(TXDESC32_USE_DRIVER_RATE);
-			tx_desc->txdw5 |=
-				cpu_to_le32(6 << TXDESC32_RETRY_LIMIT_SHIFT);
-			tx_desc->txdw5 |=
-				cpu_to_le32(TXDESC32_RETRY_LIMIT_ENABLE);
-		}
-
-		if (ieee80211_is_data_qos(hdr->frame_control))
-			tx_desc->txdw4 |= cpu_to_le32(TXDESC32_QOS);
+	if (rate_flag & IEEE80211_TX_RC_SHORT_GI ||
+	    (ieee80211_is_data_qos(hdr->frame_control) &&
+	     sta && sta->ht_cap.cap &
+	     (IEEE80211_HT_CAP_SGI_40 | IEEE80211_HT_CAP_SGI_20)))
+		sgi = true;
 
-		if (rate_flag & IEEE80211_TX_RC_USE_SHORT_PREAMBLE ||
-		    (sta && vif && vif->bss_conf.use_short_preamble))
-			tx_desc->txdw4 |= cpu_to_le32(TXDESC32_SHORT_PREAMBLE);
+	if (rate_flag & IEEE80211_TX_RC_USE_SHORT_PREAMBLE ||
+	    (sta && vif && vif->bss_conf.use_short_preamble))
+		short_preamble = true;
 
-		if (rate_flag & IEEE80211_TX_RC_SHORT_GI ||
-		    (ieee80211_is_data_qos(hdr->frame_control) &&
-		     sta && sta->ht_cap.cap &
-		     (IEEE80211_HT_CAP_SGI_40 | IEEE80211_HT_CAP_SGI_20))) {
-			tx_desc->txdw5 |= cpu_to_le32(TXDESC32_SHORT_GI);
-		}
-
-		if (rate_flag & IEEE80211_TX_RC_USE_RTS_CTS) {
-			/*
-			 * Use RTS rate 24M - does the mac80211 tell
-			 * us which to use?
-			 */
-			tx_desc->txdw4 |=
-				cpu_to_le32(DESC_RATE_24M <<
-					    TXDESC32_RTS_RATE_SHIFT);
-			tx_desc->txdw4 |=
-				cpu_to_le32(TXDESC32_RTS_CTS_ENABLE);
-			tx_desc->txdw4 |= cpu_to_le32(TXDESC32_HW_RTS_ENABLE);
-		}
-	} else {
-		tx_desc40 = (struct rtl8xxxu_txdesc40 *)tx_desc;
-
-		tx_desc40->txdw4 = cpu_to_le32(rate);
-		if (ieee80211_is_data(hdr->frame_control)) {
-			tx_desc->txdw4 |=
-				cpu_to_le32(0x1f <<
-					    TXDESC40_DATA_RATE_FB_SHIFT);
-		}
-
-		tx_desc40->txdw9 =
-			cpu_to_le32((u32)seq_number << TXDESC40_SEQ_SHIFT);
-
-		if (ampdu_enable)
-			tx_desc40->txdw2 |= cpu_to_le32(TXDESC40_AGG_ENABLE);
-		else
-			tx_desc40->txdw2 |= cpu_to_le32(TXDESC40_AGG_BREAK);
-
-		if (ieee80211_is_mgmt(hdr->frame_control)) {
-			tx_desc40->txdw4 = cpu_to_le32(tx_rate->hw_value);
-			tx_desc40->txdw3 |=
-				cpu_to_le32(TXDESC40_USE_DRIVER_RATE);
-			tx_desc40->txdw4 |=
-				cpu_to_le32(6 << TXDESC40_RETRY_LIMIT_SHIFT);
-			tx_desc40->txdw4 |=
-				cpu_to_le32(TXDESC40_RETRY_LIMIT_ENABLE);
-		}
+	seq_number = IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl));
 
-		if (rate_flag & IEEE80211_TX_RC_USE_SHORT_PREAMBLE ||
-		    (sta && vif && vif->bss_conf.use_short_preamble))
-			tx_desc40->txdw5 |=
-				cpu_to_le32(TXDESC40_SHORT_PREAMBLE);
-
-		if (rate_flag & IEEE80211_TX_RC_USE_RTS_CTS) {
-			/*
-			 * Use RTS rate 24M - does the mac80211 tell
-			 * us which to use?
-			 */
-			tx_desc->txdw4 |=
-				cpu_to_le32(DESC_RATE_24M <<
-					    TXDESC40_RTS_RATE_SHIFT);
-			tx_desc->txdw3 |= cpu_to_le32(TXDESC40_RTS_CTS_ENABLE);
-			tx_desc->txdw3 |= cpu_to_le32(TXDESC40_HW_RTS_ENABLE);
-		}
-	}
+	priv->fops->fill_txdesc(hdr, tx_desc, rate, rate_flag,
+				sgi, short_preamble, ampdu_enable);
 
 	rtl8xxxu_calc_tx_desc_csum(tx_desc);
 
@@ -5379,7 +5375,8 @@ static void rtl8xxxu_int_complete(struct urb *urb)
 	struct device *dev = &priv->udev->dev;
 	int ret;
 
-	dev_dbg(dev, "%s: status %i\n", __func__, urb->status);
+	if (rtl8xxxu_debug & RTL8XXXU_DEBUG_INTERRUPT)
+		dev_dbg(dev, "%s: status %i\n", __func__, urb->status);
 	if (urb->status == 0) {
 		usb_anchor_urb(urb, &priv->int_anchor);
 		ret = usb_submit_urb(urb, GFP_ATOMIC);
@@ -5704,7 +5701,7 @@ rtl8xxxu_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 
 	switch (action) {
 	case IEEE80211_AMPDU_TX_START:
-		dev_info(dev, "%s: IEEE80211_AMPDU_TX_START\n", __func__);
+		dev_dbg(dev, "%s: IEEE80211_AMPDU_TX_START\n", __func__);
 		ampdu_factor = sta->ht_cap.ampdu_factor;
 		ampdu_density = sta->ht_cap.ampdu_density;
 		rtl8xxxu_set_ampdu_factor(priv, ampdu_factor);
@@ -5714,21 +5711,21 @@ rtl8xxxu_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			ampdu_factor, ampdu_density);
 		break;
 	case IEEE80211_AMPDU_TX_STOP_FLUSH:
-		dev_info(dev, "%s: IEEE80211_AMPDU_TX_STOP_FLUSH\n", __func__);
+		dev_dbg(dev, "%s: IEEE80211_AMPDU_TX_STOP_FLUSH\n", __func__);
 		rtl8xxxu_set_ampdu_factor(priv, 0);
 		rtl8xxxu_set_ampdu_min_space(priv, 0);
 		break;
 	case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
-		dev_info(dev, "%s: IEEE80211_AMPDU_TX_STOP_FLUSH_CONT\n",
+		dev_dbg(dev, "%s: IEEE80211_AMPDU_TX_STOP_FLUSH_CONT\n",
 			 __func__);
 		rtl8xxxu_set_ampdu_factor(priv, 0);
 		rtl8xxxu_set_ampdu_min_space(priv, 0);
 		break;
 	case IEEE80211_AMPDU_RX_START:
-		dev_info(dev, "%s: IEEE80211_AMPDU_RX_START\n", __func__);
+		dev_dbg(dev, "%s: IEEE80211_AMPDU_RX_START\n", __func__);
 		break;
 	case IEEE80211_AMPDU_RX_STOP:
-		dev_info(dev, "%s: IEEE80211_AMPDU_RX_STOP\n", __func__);
+		dev_dbg(dev, "%s: IEEE80211_AMPDU_RX_STOP\n", __func__);
 		break;
 	default:
 		break;
@@ -5947,7 +5944,7 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 	struct ieee80211_hw *hw;
 	struct usb_device *udev;
 	struct ieee80211_supported_band *sband;
-	int ret = 0;
+	int ret;
 	int untested = 1;
 
 	udev = usb_get_dev(interface_to_usbdev(interface));
@@ -5971,6 +5968,18 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 		if (id->idProduct == 0x1004)
 			untested = 0;
 		break;
+	case 0x20f4:
+		if (id->idProduct == 0x648b)
+			untested = 0;
+		break;
+	case 0x2001:
+		if (id->idProduct == 0x3308)
+			untested = 0;
+		break;
+	case 0x2357:
+		if (id->idProduct == 0x0109)
+			untested = 0;
+		break;
 	default:
 		break;
 	}
@@ -5987,6 +5996,7 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 	hw = ieee80211_alloc_hw(sizeof(struct rtl8xxxu_priv), &rtl8xxxu_ops);
 	if (!hw) {
 		ret = -ENOMEM;
+		priv = NULL;
 		goto exit;
 	}
 
@@ -6035,6 +6045,8 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 	}
 
 	ret = rtl8xxxu_init_device(hw);
+	if (ret)
+		goto exit;
 
 	hw->wiphy->max_scan_ssids = 1;
 	hw->wiphy->max_scan_ie_len = IEEE80211_MAX_DATA_LEN;
@@ -6085,9 +6097,20 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 		goto exit;
 	}
 
+	return 0;
+
 exit:
-	if (ret < 0)
-		usb_put_dev(udev);
+	usb_set_intfdata(interface, NULL);
+
+	if (priv) {
+		kfree(priv->fw_data);
+		mutex_destroy(&priv->usb_buf_mutex);
+		mutex_destroy(&priv->h2c_mutex);
+	}
+	usb_put_dev(udev);
+
+	ieee80211_free_hw(hw);
+
 	return ret;
 }
 
@@ -6111,6 +6134,11 @@ static void rtl8xxxu_disconnect(struct usb_interface *interface)
 	mutex_destroy(&priv->usb_buf_mutex);
 	mutex_destroy(&priv->h2c_mutex);
 
+	if (priv->udev->state != USB_STATE_NOTATTACHED) {
+		dev_info(&priv->udev->dev,
+			 "Device still attached, trying to reset\n");
+		usb_reset_device(priv->udev);
+	}
 	usb_put_dev(priv->udev);
 	ieee80211_free_hw(hw);
 }
@@ -6124,6 +6152,9 @@ static struct usb_device_id dev_table[] = {
 	.driver_info = (unsigned long)&rtl8723au_fops},
 {USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0x818b, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8192eu_fops},
+/* Tested by Myckel Habets */
+{USB_DEVICE_AND_INTERFACE_INFO(0x2357, 0x0109, 0xff, 0xff, 0xff),
+	.driver_info = (unsigned long)&rtl8192eu_fops},
 {USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0xb720, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8723bu_fops},
 #ifdef CONFIG_RTL8XXXU_UNTESTED
@@ -6140,6 +6171,12 @@ static struct usb_device_id dev_table[] = {
 /* Tested by Andrea Merello */
 {USB_DEVICE_AND_INTERFACE_INFO(0x050d, 0x1004, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8192cu_fops},
+/* Tested by Jocelyn Mayer */
+{USB_DEVICE_AND_INTERFACE_INFO(0x20f4, 0x648b, 0xff, 0xff, 0xff),
+	.driver_info = (unsigned long)&rtl8192cu_fops},
+/* Tested by Stefano Bravi */
+{USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3308, 0xff, 0xff, 0xff),
+	.driver_info = (unsigned long)&rtl8192cu_fops},
 /* Currently untested 8188 series devices */
 {USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0x8191, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8192cu_fops},
@@ -6187,8 +6224,6 @@ static struct usb_device_id dev_table[] = {
 	.driver_info = (unsigned long)&rtl8192cu_fops},
 {USB_DEVICE_AND_INTERFACE_INFO(0x13d3, 0x3357, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8192cu_fops},
-{USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3308, 0xff, 0xff, 0xff),
-	.driver_info = (unsigned long)&rtl8192cu_fops},
 {USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x330b, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8192cu_fops},
 {USB_DEVICE_AND_INTERFACE_INFO(0x2019, 0x4902, 0xff, 0xff, 0xff),
@@ -6199,8 +6234,6 @@ static struct usb_device_id dev_table[] = {
 	.driver_info = (unsigned long)&rtl8192cu_fops},
 {USB_DEVICE_AND_INTERFACE_INFO(0x2019, 0xed17, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8192cu_fops},
-{USB_DEVICE_AND_INTERFACE_INFO(0x20f4, 0x648b, 0xff, 0xff, 0xff),
-	.driver_info = (unsigned long)&rtl8192cu_fops},
 {USB_DEVICE_AND_INTERFACE_INFO(0x4855, 0x0090, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8192cu_fops},
 {USB_DEVICE_AND_INTERFACE_INFO(0x4856, 0x0091, 0xff, 0xff, 0xff),
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_regs.h b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_regs.h
index 921c5653fff2..315ccfb2dff5 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_regs.h
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_regs.h
@@ -213,10 +213,66 @@
 #define REG_HMBOX_EXT_1			0x008a
 #define REG_HMBOX_EXT_2			0x008c
 #define REG_HMBOX_EXT_3			0x008e
+
 /* Interrupt registers for 8192e/8723bu/8812 */
 #define REG_HIMR0			0x00b0
+#define	 IMR0_TXCCK			BIT(30)	/* TXRPT interrupt when CCX bit
+						   of the packet is set */
+#define	 IMR0_PSTIMEOUT			BIT(29)	/* Power Save Time Out Int */
+#define	 IMR0_GTINT4			BIT(28)	/* Set when GTIMER4 expires */
+#define	 IMR0_GTINT3			BIT(27)	/* Set when GTIMER3 expires */
+#define	 IMR0_TBDER			BIT(26)	/* Transmit Beacon0 Error */
+#define	 IMR0_TBDOK			BIT(25)	/* Transmit Beacon0 OK */
+#define	 IMR0_TSF_BIT32_TOGGLE		BIT(24)	/* TSF Timer BIT32 toggle
+						   indication interrupt */
+#define	 IMR0_BCNDMAINT0		BIT(20)	/* Beacon DMA Interrupt 0 */
+#define	 IMR0_BCNDERR0			BIT(16)	/* Beacon Queue DMA Error 0 */
+#define	 IMR0_HSISR_IND_ON_INT		BIT(15)	/* HSISR Indicator (HSIMR &
+						   HSISR is true) */
+#define	 IMR0_BCNDMAINT_E		BIT(14)	/* Beacon DMA Interrupt
+						   Extension for Win7 */
+#define	 IMR0_ATIMEND			BIT(12)	/* CTWidnow End or
+						   ATIM Window End */
+#define	 IMR0_HISR1_IND_INT		BIT(11)	/* HISR1 Indicator
+						   (HISR1 & HIMR1 is true) */
+#define	 IMR0_C2HCMD			BIT(10)	/* CPU to Host Command INT
+						   Status, Write 1 to clear */
+#define	 IMR0_CPWM2			BIT(9)	/* CPU power Mode exchange INT
+						   Status, Write 1 to clear */
+#define	 IMR0_CPWM			BIT(8)	/* CPU power Mode exchange INT
+						   Status, Write 1 to clear */
+#define	 IMR0_HIGHDOK			BIT(7)	/* High Queue DMA OK */
+#define	 IMR0_MGNTDOK			BIT(6)	/* Management Queue DMA OK */
+#define	 IMR0_BKDOK			BIT(5)	/* AC_BK DMA OK */
+#define	 IMR0_BEDOK			BIT(4)	/* AC_BE DMA OK */
+#define	 IMR0_VIDOK			BIT(3)	/* AC_VI DMA OK */
+#define	 IMR0_VODOK			BIT(2)	/* AC_VO DMA OK */
+#define	 IMR0_RDU			BIT(1)	/* Rx Descriptor Unavailable */
+#define	 IMR0_ROK			BIT(0)	/* Receive DMA OK */
 #define REG_HISR0			0x00b4
 #define REG_HIMR1			0x00b8
+#define	 IMR1_BCNDMAINT7		BIT(27)	/* Beacon DMA Interrupt 7 */
+#define	 IMR1_BCNDMAINT6		BIT(26)	/* Beacon DMA Interrupt 6 */
+#define	 IMR1_BCNDMAINT5		BIT(25)	/* Beacon DMA Interrupt 5 */
+#define	 IMR1_BCNDMAINT4		BIT(24)	/* Beacon DMA Interrupt 4 */
+#define	 IMR1_BCNDMAINT3		BIT(23)	/* Beacon DMA Interrupt 3 */
+#define	 IMR1_BCNDMAINT2		BIT(22)	/* Beacon DMA Interrupt 2 */
+#define	 IMR1_BCNDMAINT1		BIT(21)	/* Beacon DMA Interrupt 1 */
+#define	 IMR1_BCNDERR7			BIT(20)	/* Beacon Queue DMA Err Int 7 */
+#define	 IMR1_BCNDERR6			BIT(19)	/* Beacon Queue DMA Err Int 6 */
+#define	 IMR1_BCNDERR5			BIT(18)	/* Beacon Queue DMA Err Int 5 */
+#define	 IMR1_BCNDERR4			BIT(17)	/* Beacon Queue DMA Err Int 4 */
+#define	 IMR1_BCNDERR3			BIT(16)	/* Beacon Queue DMA Err Int 3 */
+#define	 IMR1_BCNDERR2			BIT(15)	/* Beacon Queue DMA Err Int 2 */
+#define	 IMR1_BCNDERR1			BIT(14)	/* Beacon Queue DMA Err Int 1 */
+#define	 IMR1_ATIMEND_E			BIT(13)	/* ATIM Window End Extension
+						   for Win7 */
+#define	 IMR1_TXERR			BIT(11)	/* Tx Error Flag Int Status,
+						   write 1 to clear */
+#define	 IMR1_RXERR			BIT(10)	/* Rx Error Flag Int Status,
+						   write 1 to clear */
+#define	 IMR1_TXFOVW			BIT(9)	/* Transmit FIFO Overflow */
+#define	 IMR1_RXFOVW			BIT(8)	/* Receive FIFO Overflow */
 #define REG_HISR1			0x00bc
 
 /*  Host suspend counter on FPGA platform */
@@ -620,6 +676,7 @@
 #define REG_SCH_TXCMD			0x05d0
 
 /* define REG_FW_TSF_SYNC_CNT		0x04a0 */
+#define REG_SCH_TX_CMD			0x05f8
 #define REG_FW_RESET_TSF_CNT_1		0x05fc
 #define REG_FW_RESET_TSF_CNT_0		0x05fd
 #define REG_FW_BCN_DIS_CNT		0x05fe
@@ -780,6 +837,10 @@
 #define  FPGA_RF_MODE_OFDM		BIT(25)
 
 #define REG_FPGA0_TX_INFO		0x0804
+#define  FPGA0_TX_INFO_OFDM_PATH_A	BIT(0)
+#define  FPGA0_TX_INFO_OFDM_PATH_B	BIT(1)
+#define  FPGA0_TX_INFO_OFDM_PATH_C	BIT(2)
+#define  FPGA0_TX_INFO_OFDM_PATH_D	BIT(3)
 #define REG_FPGA0_PSD_FUNC		0x0808
 #define REG_FPGA0_TX_GAIN		0x080c
 #define REG_FPGA0_RF_TIMING1		0x0810
diff --git a/drivers/net/wireless/realtek/rtlwifi/core.c b/drivers/net/wireless/realtek/rtlwifi/core.c
index 41f77f8a309e..f95760c13c56 100644
--- a/drivers/net/wireless/realtek/rtlwifi/core.c
+++ b/drivers/net/wireless/realtek/rtlwifi/core.c
@@ -765,7 +765,8 @@ static int rtl_op_config(struct ieee80211_hw *hw, u32 changed)
 					mac->bw_40 = false;
 					mac->bw_80 = false;
 					RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-						 "switch case not processed\n");
+						 "switch case %#x not processed\n",
+						 channel_type);
 					break;
 			}
 		}
@@ -1135,7 +1136,7 @@ static void rtl_op_bss_info_changed(struct ieee80211_hw *hw,
 					mac->mode = WIRELESS_MODE_AC_24G;
 			}
 
-			if (vif->type == NL80211_IFTYPE_STATION && sta)
+			if (vif->type == NL80211_IFTYPE_STATION)
 				rtlpriv->cfg->ops->update_rate_tbl(hw, sta, 0);
 			rcu_read_unlock();
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c
index d12586d4f845..0dfa9eac3926 100644
--- a/drivers/net/wireless/realtek/rtlwifi/pci.c
+++ b/drivers/net/wireless/realtek/rtlwifi/pci.c
@@ -179,7 +179,8 @@ static void _rtl_pci_update_default_setting(struct ieee80211_hw *hw)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n",
+			 rtlpci->const_support_pciaspm);
 		break;
 	}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/ps.c b/drivers/net/wireless/realtek/rtlwifi/ps.c
index 9a64f9b703e5..18d979affc18 100644
--- a/drivers/net/wireless/realtek/rtlwifi/ps.c
+++ b/drivers/net/wireless/realtek/rtlwifi/ps.c
@@ -151,7 +151,7 @@ static bool rtl_ps_set_rf_state(struct ieee80211_hw *hw,
 
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", state_toset);
 		break;
 	}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/regd.c b/drivers/net/wireless/realtek/rtlwifi/regd.c
index 3524441fd516..6ee6bf8e7eaf 100644
--- a/drivers/net/wireless/realtek/rtlwifi/regd.c
+++ b/drivers/net/wireless/realtek/rtlwifi/regd.c
@@ -345,9 +345,9 @@ static const struct ieee80211_regdomain *_rtl_regdomain_select(
 		return &rtl_regdom_no_midband;
 	case COUNTRY_CODE_IC:
 		return &rtl_regdom_11;
-	case COUNTRY_CODE_ETSI:
 	case COUNTRY_CODE_TELEC_NETGEAR:
 		return &rtl_regdom_60_64;
+	case COUNTRY_CODE_ETSI:
 	case COUNTRY_CODE_SPAIN:
 	case COUNTRY_CODE_FRANCE:
 	case COUNTRY_CODE_ISRAEL:
@@ -406,6 +406,8 @@ static u8 channel_plan_to_country_code(u8 channelplan)
 		return COUNTRY_CODE_WORLD_WIDE_13;
 	case 0x22:
 		return COUNTRY_CODE_IC;
+	case 0x25:
+		return COUNTRY_CODE_ETSI;
 	case 0x32:
 		return COUNTRY_CODE_TELEC_NETGEAR;
 	case 0x41:
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/fw.c
index 629125658b87..5360d5332359 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/fw.c
@@ -334,7 +334,7 @@ static void _rtl88e_fill_h2c_command(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", boxnum);
 			break;
 		}
 		isfw_read = _rtl88e_check_fw_read_last_h2c(hw, boxnum);
@@ -405,7 +405,7 @@ static void _rtl88e_fill_h2c_command(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", cmd_len);
 			break;
 		}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c
index 4ab6201daf1a..37d6efc3d240 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c
@@ -355,9 +355,11 @@ void rtl88ee_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 
 		*((u64 *)(val)) = tsf;
 		break; }
+	case HAL_DEF_WOWLAN:
+		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process %x\n", variable);
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -571,7 +573,8 @@ void rtl88ee_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 				break;
 			default:
 				RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-					 "switch case not process\n");
+					 "switch case %#x not processed\n",
+					 e_aci);
 				break;
 			}
 		}
@@ -735,7 +738,7 @@ void rtl88ee_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		break; }
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process %x\n", variable);
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -2352,7 +2355,7 @@ void rtl88ee_set_key(struct ieee80211_hw *hw, u32 key_index,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", enc_algo);
 			enc_algo = CAM_TKIP;
 			break;
 		}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/led.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/led.c
index b504bd092fc4..f05c2c674165 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/led.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/led.c
@@ -62,7 +62,7 @@ void rtl88ee_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = true;
@@ -100,7 +100,7 @@ void rtl88ee_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/phy.c
index 7498a1218cba..fffaa92eda81 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/phy.c
@@ -1346,7 +1346,8 @@ static bool _rtl88e_phy_sw_chnl_step_by_step(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n",
+				 currentcmd->cmdid);
 			break;
 		}
 
@@ -2128,7 +2129,7 @@ bool rtl88e_phy_set_io_cmd(struct ieee80211_hw *hw, enum io_type iotype)
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", iotype);
 			break;
 		}
 	} while (false);
@@ -2166,7 +2167,8 @@ static void rtl88e_phy_set_io(struct ieee80211_hw *hw)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n",
+			 rtlphy->current_io_type);
 		break;
 	}
 	rtlphy->set_io_inprogress = false;
@@ -2319,7 +2321,7 @@ static bool _rtl88ee_phy_set_rf_power_state(struct ieee80211_hw *hw,
 		}
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", rfpwr_state);
 		bresult = false;
 		break;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c
index 47e32cb0ec1a..e7b11b40e68d 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c
@@ -280,7 +280,7 @@ static struct rtl_mod_params rtl88ee_mod_params = {
 	.debug = DBG_EMERG,
 };
 
-static struct rtl_hal_cfg rtl88ee_hal_cfg = {
+static const struct rtl_hal_cfg rtl88ee_hal_cfg = {
 	.bar_id = 2,
 	.write_readback = true,
 	.name = "rtl88e_pci",
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192c/fw_common.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192c/fw_common.c
index 43fcb25c885f..7d152466152b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192c/fw_common.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192c/fw_common.c
@@ -352,7 +352,7 @@ static void _rtl92c_fill_h2c_command(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", boxnum);
 			break;
 		}
 
@@ -456,7 +456,7 @@ static void _rtl92c_fill_h2c_command(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", cmd_len);
 			break;
 		}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192c/phy_common.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192c/phy_common.c
index 60ab2ec4f4ef..27e3d5f9ca34 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192c/phy_common.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192c/phy_common.c
@@ -910,7 +910,8 @@ bool _rtl92c_phy_sw_chnl_step_by_step(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n",
+				 currentcmd->cmdid);
 			break;
 		}
 
@@ -1567,7 +1568,7 @@ bool rtl92c_phy_set_io_cmd(struct ieee80211_hw *hw, enum io_type iotype)
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", iotype);
 			break;
 		}
 	} while (false);
@@ -1605,7 +1606,8 @@ void rtl92c_phy_set_io(struct ieee80211_hw *hw)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n",
+			 rtlphy->current_io_type);
 		break;
 	}
 	rtlphy->set_io_inprogress = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
index 244607951e28..a47be73a0980 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
@@ -141,9 +141,11 @@ void rtl92ce_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 
 		break;
 		}
+	case HAL_DEF_WOWLAN:
+		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -367,7 +369,8 @@ void rtl92ce_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 					break;
 				default:
 					RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-						 "switch case not processed\n");
+						 "switch case %#x not processed\n",
+						 e_aci);
 					break;
 				}
 			}
@@ -2154,7 +2157,7 @@ void rtl92ce_set_key(struct ieee80211_hw *hw, u32 key_index,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not processed\n");
+				 "switch case %#x not processed\n", enc_algo);
 			enc_algo = CAM_TKIP;
 			break;
 		}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/led.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/led.c
index 8283e9b27639..24e483ba3fa4 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/led.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/led.c
@@ -62,7 +62,7 @@ void rtl92ce_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = true;
@@ -97,7 +97,7 @@ void rtl92ce_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/phy.c
index 1ee5a6ae9960..46d0d945f283 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/phy.c
@@ -300,12 +300,9 @@ bool rtl92c_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
 		}
 		break;
 	case RF90_PATH_C:
-		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
-		break;
 	case RF90_PATH_D:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", rfpath);
 		break;
 	default:
 		break;
@@ -554,7 +551,7 @@ static bool _rtl92ce_phy_set_rf_power_state(struct ieee80211_hw *hw,
 		}
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", rfpwr_state);
 		bresult = false;
 		break;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c
index 4780bdc63b2b..87aa209ae325 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c
@@ -258,7 +258,7 @@ static struct rtl_mod_params rtl92ce_mod_params = {
 	.debug = DBG_EMERG,
 };
 
-static struct rtl_hal_cfg rtl92ce_hal_cfg = {
+static const struct rtl_hal_cfg rtl92ce_hal_cfg = {
 	.bar_id = 2,
 	.write_readback = true,
 	.name = "rtl92c_pci",
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
index 8789752f8143..ae8f055483fa 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
@@ -1560,7 +1560,7 @@ void rtl92cu_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -1931,7 +1931,7 @@ void rtl92cu_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		}
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/led.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/led.c
index 75a2deb23af1..8514ab652520 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/led.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/led.c
@@ -62,7 +62,7 @@ void rtl92cu_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = true;
@@ -95,7 +95,7 @@ void rtl92cu_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/phy.c
index c972fa50926d..4b2976465905 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/phy.c
@@ -277,12 +277,9 @@ bool rtl92cu_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
 		}
 		break;
 	case RF90_PATH_C:
-		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
-		break;
 	case RF90_PATH_D:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", rfpath);
 		break;
 	default:
 		break;
@@ -517,7 +514,7 @@ static bool _rtl92cu_phy_set_rf_power_state(struct ieee80211_hw *hw,
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", rfpwr_state);
 		bresult = false;
 		break;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/fw.c
index 62ef8209718f..8de29cc3ced0 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/fw.c
@@ -435,7 +435,7 @@ static void _rtl92d_fill_h2c_command(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not processed\n");
+				 "switch case %#x not processed\n", boxnum);
 			break;
 		}
 		isfw_read = _rtl92d_check_fw_read_last_h2c(hw, boxnum);
@@ -512,7 +512,7 @@ static void _rtl92d_fill_h2c_command(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not processed\n");
+				 "switch case %#x not processed\n", cmd_len);
 			break;
 		}
 		bwrite_success = true;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
index b0f632462335..d91f8bbfe7a0 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
@@ -164,9 +164,11 @@ void rtl92de_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 	case HW_VAR_INT_AC:
 		*((bool *)(val)) = rtlpriv->dm.disable_tx_int;
 		break;
+	case HAL_DEF_WOWLAN:
+		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -361,7 +363,8 @@ void rtl92de_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 				break;
 			default:
 				RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-					 "switch case not processed\n");
+					 "switch case %#x not processed\n",
+					 e_aci);
 				break;
 			}
 		}
@@ -502,7 +505,7 @@ void rtl92de_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 	}
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -1757,7 +1760,7 @@ static void _rtl92de_read_adapter_info(struct ieee80211_hw *hw)
 		return;
 
 	if (rtl_get_hwinfo(hw, rtlpriv, HWSET_MAX_SIZE, hwinfo, params))
-		return;
+		goto exit;
 
 	_rtl92de_efuse_update_chip_version(hw);
 	_rtl92de_read_macphymode_and_bandtype(hw, hwinfo);
@@ -1790,6 +1793,7 @@ static void _rtl92de_read_adapter_info(struct ieee80211_hw *hw)
 		break;
 	}
 	rtlefuse->txpwr_fromeprom = true;
+exit:
 	kfree(hwinfo);
 }
 
@@ -2170,7 +2174,7 @@ void rtl92de_set_key(struct ieee80211_hw *hw, u32 key_index,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not processed\n");
+				 "switch case %#x not processed\n", enc_algo);
 			enc_algo = CAM_TKIP;
 			break;
 		}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/led.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/led.c
index 76a57ae4af3e..811ba57eb9bb 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/led.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/led.c
@@ -71,7 +71,7 @@ void rtl92de_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = true;
@@ -106,7 +106,7 @@ void rtl92de_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c
index d334d2a5ea63..2a1edfd21b96 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c
@@ -588,7 +588,7 @@ static bool _rtl92d_phy_config_bb_with_headerfile(struct ieee80211_hw *hw,
 				 * setting. */
 				udelay(1);
 				RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
-					 "The Rtl819XAGCTAB_Array_Table[0] is %ul Rtl819XPHY_REGArray[1] is %ul\n",
+					 "The Rtl819XAGCTAB_Array_Table[0] is %u Rtl819XPHY_REGArray[1] is %u\n",
 					 agctab_array_table[i],
 					 agctab_array_table[i + 1]);
 			}
@@ -604,7 +604,7 @@ static bool _rtl92d_phy_config_bb_with_headerfile(struct ieee80211_hw *hw,
 					 * setting. */
 					udelay(1);
 					RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
-						 "The Rtl819XAGCTAB_Array_Table[0] is %ul Rtl819XPHY_REGArray[1] is %ul\n",
+						 "The Rtl819XAGCTAB_Array_Table[0] is %u Rtl819XPHY_REGArray[1] is %u\n",
 						 agctab_array_table[i],
 						 agctab_array_table[i + 1]);
 				}
@@ -620,7 +620,7 @@ static bool _rtl92d_phy_config_bb_with_headerfile(struct ieee80211_hw *hw,
 					 * setting. */
 					udelay(1);
 					RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
-						 "The Rtl819XAGCTAB_5GArray_Table[0] is %ul Rtl819XPHY_REGArray[1] is %ul\n",
+						 "The Rtl819XAGCTAB_5GArray_Table[0] is %u Rtl819XPHY_REGArray[1] is %u\n",
 						 agctab_5garray_table[i],
 						 agctab_5garray_table[i + 1]);
 				}
@@ -836,12 +836,9 @@ bool rtl92d_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
 		}
 		break;
 	case RF90_PATH_C:
-		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
-		break;
 	case RF90_PATH_D:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", rfpath);
 		break;
 	}
 	return true;
@@ -2850,7 +2847,8 @@ static bool _rtl92d_phy_sw_chnl_step_by_step(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not processed\n");
+				 "switch case %#x not processed\n",
+				 currentcmd->cmdid);
 			break;
 		}
 		break;
@@ -2963,7 +2961,8 @@ static void rtl92d_phy_set_io(struct ieee80211_hw *hw)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n",
+			 rtlphy->current_io_type);
 		break;
 	}
 	rtlphy->set_io_inprogress = false;
@@ -2994,7 +2993,7 @@ bool rtl92d_phy_set_io_cmd(struct ieee80211_hw *hw, enum io_type iotype)
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not processed\n");
+				 "switch case %#x not processed\n", iotype);
 			break;
 		}
 	} while (false);
@@ -3182,7 +3181,7 @@ bool rtl92d_phy_set_rf_power_state(struct ieee80211_hw *hw,
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", rfpwr_state);
 		bresult = false;
 		break;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c
index c6e09a19de1a..0538a4d09568 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c
@@ -262,7 +262,7 @@ static struct rtl_mod_params rtl92de_mod_params = {
 	.debug = DBG_EMERG,
 };
 
-static struct rtl_hal_cfg rtl92de_hal_cfg = {
+static const struct rtl_hal_cfg rtl92de_hal_cfg = {
 	.bar_id = 2,
 	.write_readback = true,
 	.name = "rtl8192de",
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c
index 0708eedd9671..b3f6a9ed15d4 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c
@@ -344,7 +344,7 @@ static void _rtl92ee_fill_h2c_command(struct ieee80211_hw *hw, u8 element_id,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", boxnum);
 			break;
 		}
 
@@ -433,7 +433,7 @@ static void _rtl92ee_fill_h2c_command(struct ieee80211_hw *hw, u8 element_id,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", cmd_len);
 			break;
 		}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
index b07af8d15273..ebf663e1a81a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
@@ -338,9 +338,11 @@ void rtl92ee_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		*((u64 *)(val)) = tsf;
 		}
 		break;
+	case HAL_DEF_WOWLAN:
+		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
-			 "switch case not process %x\n", variable);
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -566,7 +568,8 @@ void rtl92ee_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 				break;
 			default:
 				RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
-					 "switch case not process\n");
+					 "switch case %#x not processed\n",
+					 e_aci);
 				break;
 			}
 		}
@@ -685,7 +688,7 @@ void rtl92ee_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
-			 "switch case not process %x\n", variable);
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -2463,7 +2466,7 @@ void rtl92ee_set_key(struct ieee80211_hw *hw, u32 key_index,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", enc_algo);
 			enc_algo = CAM_TKIP;
 			break;
 		}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/led.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/led.c
index 8388e371c8e2..47da05dd3076 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/led.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/led.c
@@ -61,7 +61,7 @@ void rtl92ee_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = true;
@@ -91,7 +91,7 @@ void rtl92ee_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/phy.c
index beafc9a10ad8..5ad7e753c357 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/phy.c
@@ -1927,7 +1927,8 @@ static bool _rtl92ee_phy_sw_chnl_step_by_step(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n",
+				 currentcmd->cmdid);
 			break;
 		}
 
@@ -3001,7 +3002,7 @@ bool rtl92ee_phy_set_io_cmd(struct ieee80211_hw *hw, enum io_type iotype)
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", iotype);
 			break;
 		}
 	} while (false);
@@ -3041,7 +3042,8 @@ static void rtl92ee_phy_set_io(struct ieee80211_hw *hw)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n",
+			 rtlphy->current_io_type);
 		break;
 	}
 	rtlphy->set_io_inprogress = false;
@@ -3187,7 +3189,7 @@ static bool _rtl92ee_phy_set_rf_power_state(struct ieee80211_hw *hw,
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", rfpwr_state);
 		bresult = false;
 		break;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c
index c31c6bfb536d..ac299cbe59b0 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c
@@ -262,7 +262,7 @@ static struct rtl_mod_params rtl92ee_mod_params = {
 	.debug = DBG_EMERG,
 };
 
-static struct rtl_hal_cfg rtl92ee_hal_cfg = {
+static const struct rtl_hal_cfg rtl92ee_hal_cfg = {
 	.bar_id = 2,
 	.write_readback = true,
 	.name = "rtl92ee_pci",
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
index ddfa0aee5bf8..52e4430edb54 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
@@ -77,9 +77,11 @@ void rtl92se_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 			*((bool *)(val)) = rtlpriv->dm.current_mrc_switch;
 			break;
 		}
+	case HAL_DEF_WOWLAN:
+		break;
 	default: {
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", variable);
 			break;
 		}
 	}
@@ -297,7 +299,8 @@ void rtl92se_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 					break;
 				default:
 					RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-						 "switch case not processed\n");
+						 "switch case %#x not processed\n",
+						 e_aci);
 					break;
 				}
 			}
@@ -433,7 +436,7 @@ void rtl92se_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		break; }
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 
@@ -2465,7 +2468,7 @@ void rtl92se_set_key(struct ieee80211_hw *hw, u32 key_index, u8 *p_macaddr,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not processed\n");
+				 "switch case %#x not processed\n", enc_algo);
 			enc_algo = CAM_TKIP;
 			break;
 		}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/led.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/led.c
index 44949b5cbb87..9849cb988186 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/led.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/led.c
@@ -68,7 +68,7 @@ void rtl92se_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = true;
@@ -104,7 +104,7 @@ void rtl92se_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/phy.c
index 881821f4e243..4bb75581ab38 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/phy.c
@@ -442,7 +442,8 @@ static bool _rtl92s_phy_sw_chnl_step_by_step(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not processed\n");
+				 "switch case %#x not processed\n",
+				 currentcmd->cmdid);
 			break;
 		}
 
@@ -648,7 +649,7 @@ bool rtl92s_phy_set_rf_power_state(struct ieee80211_hw *hw,
 			break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not processed\n");
+			 "switch case %#x not processed\n", rfpwr_state);
 		bresult = false;
 		break;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c
index 31baca41ac2f..5e8e02d5de8a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c
@@ -306,7 +306,7 @@ static struct rtl_mod_params rtl92se_mod_params = {
 
 /* Because memory R/W bursting will cause system hang/crash
  * for 92se, so we don't read back after every write action */
-static struct rtl_hal_cfg rtl92se_hal_cfg = {
+static const struct rtl_hal_cfg rtl92se_hal_cfg = {
 	.bar_id = 1,
 	.write_readback = false,
 	.name = "rtl92s_pci",
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/fw.c
index b7c0d38ee5b5..1186755e55b8 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/fw.c
@@ -124,7 +124,7 @@ static void _rtl8723e_fill_h2c_command(struct ieee80211_hw *hw, u8 element_id,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", boxnum);
 			break;
 		}
 
@@ -230,7 +230,7 @@ static void _rtl8723e_fill_h2c_command(struct ieee80211_hw *hw, u8 element_id,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", cmd_len);
 			break;
 		}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c
index b88c7ee72dbf..f8be0bd7e326 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c
@@ -141,9 +141,11 @@ void rtl8723e_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 
 			break;
 		}
+	case HAL_DEF_WOWLAN:
+		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -366,7 +368,8 @@ void rtl8723e_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 					break;
 				default:
 					RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-						 "switch case not process\n");
+						 "switch case %#x not processed\n",
+						 e_aci);
 					break;
 				}
 			}
@@ -546,7 +549,7 @@ void rtl8723e_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		}
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -1654,7 +1657,7 @@ static void _rtl8723e_read_adapter_info(struct ieee80211_hw *hw,
 			rtlefuse->autoload_failflag, hwinfo);
 
 	if (rtlhal->oem_id != RT_CID_DEFAULT)
-		return;
+		goto exit;
 
 	switch (rtlefuse->eeprom_oemid) {
 	case EEPROM_CID_DEFAULT:
@@ -2225,7 +2228,7 @@ void rtl8723e_set_key(struct ieee80211_hw *hw, u32 key_index,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", enc_algo);
 			enc_algo = CAM_TKIP;
 			break;
 		}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/led.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/led.c
index 13173351cbfd..c7be9342136c 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/led.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/led.c
@@ -63,7 +63,7 @@ void rtl8723e_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = true;
@@ -105,7 +105,7 @@ void rtl8723e_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/phy.c
index 601b78efedfb..17b58cb32d55 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/phy.c
@@ -1023,7 +1023,8 @@ static bool _rtl8723e_phy_sw_chnl_step_by_step(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n",
+				 currentcmd->cmdid);
 			break;
 		}
 
@@ -1499,7 +1500,7 @@ bool rtl8723e_phy_set_io_cmd(struct ieee80211_hw *hw, enum io_type iotype)
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", iotype);
 			break;
 		}
 	} while (false);
@@ -1536,7 +1537,8 @@ static void rtl8723e_phy_set_io(struct ieee80211_hw *hw)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n",
+			 rtlphy->current_io_type);
 		break;
 	}
 	rtlphy->set_io_inprogress = false;
@@ -1682,7 +1684,7 @@ static bool _rtl8723e_phy_set_rf_power_state(struct ieee80211_hw *hw,
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", rfpwr_state);
 		bresult = false;
 		break;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c
index ff49a8c0ff61..89c828ad89f4 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c
@@ -276,7 +276,7 @@ static struct rtl_mod_params rtl8723e_mod_params = {
 	.disable_watchdog = false,
 };
 
-static struct rtl_hal_cfg rtl8723e_hal_cfg = {
+static const struct rtl_hal_cfg rtl8723e_hal_cfg = {
 	.bar_id = 2,
 	.write_readback = true,
 	.name = "rtl8723e_pci",
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c
index d5da0f3c1217..8c5c27ce8e05 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c
@@ -122,7 +122,7 @@ static void _rtl8723be_fill_h2c_command(struct ieee80211_hw *hw, u8 element_id,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", boxnum);
 			break;
 		}
 
@@ -195,7 +195,7 @@ static void _rtl8723be_fill_h2c_command(struct ieee80211_hw *hw, u8 element_id,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", cmd_len);
 			break;
 		}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
index 82e4476cab23..aba60c3145c5 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
@@ -348,9 +348,11 @@ void rtl8723be_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		*((u64 *)(val)) = tsf;
 		}
 		break;
+	case HAL_DEF_WOWLAN:
+		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process %x\n", variable);
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -607,7 +609,8 @@ void rtl8723be_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 				break;
 			default:
 				RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-					 "switch case not process\n");
+					 "switch case %#x not processed\n",
+					 e_aci);
 				break;
 			}
 		}
@@ -723,8 +726,7 @@ void rtl8723be_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process %x\n",
-			 variable);
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -2565,7 +2567,7 @@ void rtl8723be_set_key(struct ieee80211_hw *hw, u32 key_index,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", enc_algo);
 			enc_algo = CAM_TKIP;
 			break;
 		}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/led.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/led.c
index 4196efb723a2..497913eb3b37 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/led.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/led.c
@@ -58,7 +58,7 @@ void rtl8723be_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = true;
@@ -100,7 +100,7 @@ void rtl8723be_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
index 285818df149b..3cc2232f25ca 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
@@ -837,7 +837,7 @@ bool rtl8723be_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
 		break;
 	case RF90_PATH_D:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", rfpath);
 		break;
 	}
 	return true;
@@ -1507,7 +1507,8 @@ static bool _rtl8723be_phy_sw_chnl_step_by_step(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n",
+				 currentcmd->cmdid);
 			break;
 		}
 
@@ -2515,7 +2516,7 @@ bool rtl8723be_phy_set_io_cmd(struct ieee80211_hw *hw, enum io_type iotype)
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", iotype);
 			break;
 		}
 	} while (false);
@@ -2553,7 +2554,8 @@ static void rtl8723be_phy_set_io(struct ieee80211_hw *hw)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n",
+			 rtlphy->current_io_type);
 		break;
 	}
 	rtlphy->set_io_inprogress = false;
@@ -2705,7 +2707,7 @@ static bool _rtl8723be_phy_set_rf_power_state(struct ieee80211_hw *hw,
 
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", rfpwr_state);
 		bresult = false;
 		break;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c
index 2101793438ed..20b53f035483 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c
@@ -276,7 +276,7 @@ static struct rtl_mod_params rtl8723be_mod_params = {
 	.ant_sel = 0,
 };
 
-static struct rtl_hal_cfg rtl8723be_hal_cfg = {
+static const struct rtl_hal_cfg rtl8723be_hal_cfg = {
 	.bar_id = 2,
 	.write_readback = true,
 	.name = "rtl8723be_pci",
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c
index a4fc70e8c9c0..b665446351a4 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c
@@ -392,7 +392,7 @@ static void _rtl8821ae_fill_h2c_command(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", boxnum);
 			break;
 		}
 
@@ -481,7 +481,7 @@ static void _rtl8821ae_fill_h2c_command(struct ieee80211_hw *hw,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", cmd_len);
 			break;
 		}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
index 0cddf1ad0fff..1281ebe0c30a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
@@ -480,7 +480,7 @@ void rtl8821ae_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process %x\n", variable);
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -671,7 +671,8 @@ void rtl8821ae_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 				break;
 			default:
 				RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-					 "switch case not process\n");
+					 "switch case %#x not processed\n",
+					 e_aci);
 				break;
 			}
 		}
@@ -800,7 +801,7 @@ void rtl8821ae_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 		break; }
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process %x\n", variable);
+			 "switch case %#x not processed\n", variable);
 		break;
 	}
 }
@@ -3934,7 +3935,7 @@ void rtl8821ae_set_key(struct ieee80211_hw *hw, u32 key_index,
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", enc_algo);
 			enc_algo = CAM_TKIP;
 			break;
 		}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/led.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/led.c
index ba1946a0280e..fcb3b28c6b8f 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/led.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/led.c
@@ -60,7 +60,7 @@ void rtl8821ae_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = true;
@@ -133,7 +133,7 @@ void rtl8821ae_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", pled->ledpin);
 		break;
 	}
 	pled->ledon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c
index a71bfe38e7e1..5dad402171c2 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c
@@ -2063,12 +2063,9 @@ bool rtl8812ae_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
 		}
 		break;
 	case RF90_PATH_C:
-		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
-		break;
 	case RF90_PATH_D:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", rfpath);
 		break;
 	}
 	return true;
@@ -2133,16 +2130,10 @@ bool rtl8821ae_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
 		break;
 
 	case RF90_PATH_B:
-		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
-		break;
 	case RF90_PATH_C:
-		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
-		break;
 	case RF90_PATH_D:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", rfpath);
 		break;
 	}
 	return true;
@@ -4670,7 +4661,7 @@ bool rtl8821ae_phy_set_io_cmd(struct ieee80211_hw *hw, enum io_type iotype)
 			break;
 		default:
 			RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-				 "switch case not process\n");
+				 "switch case %#x not processed\n", iotype);
 			break;
 		}
 	} while (false);
@@ -4714,7 +4705,8 @@ static void rtl8821ae_phy_set_io(struct ieee80211_hw *hw)
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n",
+			 rtlphy->current_io_type);
 		break;
 	}
 	rtlphy->set_io_inprogress = false;
@@ -4820,7 +4812,7 @@ static bool _rtl8821ae_phy_set_rf_power_state(struct ieee80211_hw *hw,
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_EMERG,
-			 "switch case not process\n");
+			 "switch case %#x not processed\n", rfpwr_state);
 		bresult = false;
 		break;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c
index 4159f9b14db6..22f687b1f133 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c
@@ -316,7 +316,7 @@ static struct rtl_mod_params rtl8821ae_mod_params = {
 	.disable_watchdog = 0,
 };
 
-static struct rtl_hal_cfg rtl8821ae_hal_cfg = {
+static const struct rtl_hal_cfg rtl8821ae_hal_cfg = {
 	.bar_id = 2,
 	.write_readback = true,
 	.name = "rtl8821ae_pci",
diff --git a/drivers/net/wireless/realtek/rtlwifi/usb.c b/drivers/net/wireless/realtek/rtlwifi/usb.c
index 41617b7b0822..32aa5c1d070a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/usb.c
+++ b/drivers/net/wireless/realtek/rtlwifi/usb.c
@@ -739,11 +739,8 @@ static int _rtl_usb_receive(struct ieee80211_hw *hw)
 	for (i = 0; i < rtlusb->rx_urb_num; i++) {
 		err = -ENOMEM;
 		urb = usb_alloc_urb(0, GFP_KERNEL);
-		if (!urb) {
-			RT_TRACE(rtlpriv, COMP_USB, DBG_EMERG,
-				 "Failed to alloc URB!!\n");
+		if (!urb)
 			goto err_out;
-		}
 
 		err = _rtl_prep_rx_urb(hw, rtlusb, urb, GFP_KERNEL);
 		if (err < 0) {
@@ -907,15 +904,12 @@ static void _rtl_tx_complete(struct urb *urb)
 static struct urb *_rtl_usb_tx_urb_setup(struct ieee80211_hw *hw,
 				struct sk_buff *skb, u32 ep_num)
 {
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_usb *rtlusb = rtl_usbdev(rtl_usbpriv(hw));
 	struct urb *_urb;
 
 	WARN_ON(NULL == skb);
 	_urb = usb_alloc_urb(0, GFP_ATOMIC);
 	if (!_urb) {
-		RT_TRACE(rtlpriv, COMP_USB, DBG_EMERG,
-			 "Can't allocate URB for bulk out!\n");
 		kfree_skb(skb);
 		return NULL;
 	}
diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h
index c5086c2229aa..595f7d5d091a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/wifi.h
+++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h
@@ -394,110 +394,110 @@ enum io_type {
 };
 
 enum hw_variables {
-	HW_VAR_ETHER_ADDR,
-	HW_VAR_MULTICAST_REG,
-	HW_VAR_BASIC_RATE,
-	HW_VAR_BSSID,
-	HW_VAR_MEDIA_STATUS,
-	HW_VAR_SECURITY_CONF,
-	HW_VAR_BEACON_INTERVAL,
-	HW_VAR_ATIM_WINDOW,
-	HW_VAR_LISTEN_INTERVAL,
-	HW_VAR_CS_COUNTER,
-	HW_VAR_DEFAULTKEY0,
-	HW_VAR_DEFAULTKEY1,
-	HW_VAR_DEFAULTKEY2,
-	HW_VAR_DEFAULTKEY3,
-	HW_VAR_SIFS,
-	HW_VAR_R2T_SIFS,
-	HW_VAR_DIFS,
-	HW_VAR_EIFS,
-	HW_VAR_SLOT_TIME,
-	HW_VAR_ACK_PREAMBLE,
-	HW_VAR_CW_CONFIG,
-	HW_VAR_CW_VALUES,
-	HW_VAR_RATE_FALLBACK_CONTROL,
-	HW_VAR_CONTENTION_WINDOW,
-	HW_VAR_RETRY_COUNT,
-	HW_VAR_TR_SWITCH,
-	HW_VAR_COMMAND,
-	HW_VAR_WPA_CONFIG,
-	HW_VAR_AMPDU_MIN_SPACE,
-	HW_VAR_SHORTGI_DENSITY,
-	HW_VAR_AMPDU_FACTOR,
-	HW_VAR_MCS_RATE_AVAILABLE,
-	HW_VAR_AC_PARAM,
-	HW_VAR_ACM_CTRL,
-	HW_VAR_DIS_Req_Qsize,
-	HW_VAR_CCX_CHNL_LOAD,
-	HW_VAR_CCX_NOISE_HISTOGRAM,
-	HW_VAR_CCX_CLM_NHM,
-	HW_VAR_TxOPLimit,
-	HW_VAR_TURBO_MODE,
-	HW_VAR_RF_STATE,
-	HW_VAR_RF_OFF_BY_HW,
-	HW_VAR_BUS_SPEED,
-	HW_VAR_SET_DEV_POWER,
-
-	HW_VAR_RCR,
-	HW_VAR_RATR_0,
-	HW_VAR_RRSR,
-	HW_VAR_CPU_RST,
-	HW_VAR_CHECK_BSSID,
-	HW_VAR_LBK_MODE,
-	HW_VAR_AES_11N_FIX,
-	HW_VAR_USB_RX_AGGR,
-	HW_VAR_USER_CONTROL_TURBO_MODE,
-	HW_VAR_RETRY_LIMIT,
-	HW_VAR_INIT_TX_RATE,
-	HW_VAR_TX_RATE_REG,
-	HW_VAR_EFUSE_USAGE,
-	HW_VAR_EFUSE_BYTES,
-	HW_VAR_AUTOLOAD_STATUS,
-	HW_VAR_RF_2R_DISABLE,
-	HW_VAR_SET_RPWM,
-	HW_VAR_H2C_FW_PWRMODE,
-	HW_VAR_H2C_FW_JOINBSSRPT,
-	HW_VAR_H2C_FW_MEDIASTATUSRPT,
-	HW_VAR_H2C_FW_P2P_PS_OFFLOAD,
-	HW_VAR_FW_PSMODE_STATUS,
-	HW_VAR_INIT_RTS_RATE,
-	HW_VAR_RESUME_CLK_ON,
-	HW_VAR_FW_LPS_ACTION,
-	HW_VAR_1X1_RECV_COMBINE,
-	HW_VAR_STOP_SEND_BEACON,
-	HW_VAR_TSF_TIMER,
-	HW_VAR_IO_CMD,
-
-	HW_VAR_RF_RECOVERY,
-	HW_VAR_H2C_FW_UPDATE_GTK,
-	HW_VAR_WF_MASK,
-	HW_VAR_WF_CRC,
-	HW_VAR_WF_IS_MAC_ADDR,
-	HW_VAR_H2C_FW_OFFLOAD,
-	HW_VAR_RESET_WFCRC,
-
-	HW_VAR_HANDLE_FW_C2H,
-	HW_VAR_DL_FW_RSVD_PAGE,
-	HW_VAR_AID,
-	HW_VAR_HW_SEQ_ENABLE,
-	HW_VAR_CORRECT_TSF,
-	HW_VAR_BCN_VALID,
-	HW_VAR_FWLPS_RF_ON,
-	HW_VAR_DUAL_TSF_RST,
-	HW_VAR_SWITCH_EPHY_WoWLAN,
-	HW_VAR_INT_MIGRATION,
-	HW_VAR_INT_AC,
-	HW_VAR_RF_TIMING,
-
-	HAL_DEF_WOWLAN,
-	HW_VAR_MRC,
-	HW_VAR_KEEP_ALIVE,
-	HW_VAR_NAV_UPPER,
-
-	HW_VAR_MGT_FILTER,
-	HW_VAR_CTRL_FILTER,
-	HW_VAR_DATA_FILTER,
+	HW_VAR_ETHER_ADDR = 0x0,
+	HW_VAR_MULTICAST_REG = 0x1,
+	HW_VAR_BASIC_RATE = 0x2,
+	HW_VAR_BSSID = 0x3,
+	HW_VAR_MEDIA_STATUS= 0x4,
+	HW_VAR_SECURITY_CONF= 0x5,
+	HW_VAR_BEACON_INTERVAL = 0x6,
+	HW_VAR_ATIM_WINDOW = 0x7,
+	HW_VAR_LISTEN_INTERVAL = 0x8,
+	HW_VAR_CS_COUNTER = 0x9,
+	HW_VAR_DEFAULTKEY0 = 0xa,
+	HW_VAR_DEFAULTKEY1 = 0xb,
+	HW_VAR_DEFAULTKEY2 = 0xc,
+	HW_VAR_DEFAULTKEY3 = 0xd,
+	HW_VAR_SIFS = 0xe,
+	HW_VAR_R2T_SIFS = 0xf,
+	HW_VAR_DIFS = 0x10,
+	HW_VAR_EIFS = 0x11,
+	HW_VAR_SLOT_TIME = 0x12,
+	HW_VAR_ACK_PREAMBLE = 0x13,
+	HW_VAR_CW_CONFIG = 0x14,
+	HW_VAR_CW_VALUES = 0x15,
+	HW_VAR_RATE_FALLBACK_CONTROL= 0x16,
+	HW_VAR_CONTENTION_WINDOW = 0x17,
+	HW_VAR_RETRY_COUNT = 0x18,
+	HW_VAR_TR_SWITCH = 0x19,
+	HW_VAR_COMMAND = 0x1a,
+	HW_VAR_WPA_CONFIG = 0x1b,
+	HW_VAR_AMPDU_MIN_SPACE = 0x1c,
+	HW_VAR_SHORTGI_DENSITY = 0x1d,
+	HW_VAR_AMPDU_FACTOR = 0x1e,
+	HW_VAR_MCS_RATE_AVAILABLE = 0x1f,
+	HW_VAR_AC_PARAM = 0x20,
+	HW_VAR_ACM_CTRL = 0x21,
+	HW_VAR_DIS_Req_Qsize = 0x22,
+	HW_VAR_CCX_CHNL_LOAD = 0x23,
+	HW_VAR_CCX_NOISE_HISTOGRAM = 0x24,
+	HW_VAR_CCX_CLM_NHM = 0x25,
+	HW_VAR_TxOPLimit = 0x26,
+	HW_VAR_TURBO_MODE = 0x27,
+	HW_VAR_RF_STATE = 0x28,
+	HW_VAR_RF_OFF_BY_HW = 0x29,
+	HW_VAR_BUS_SPEED = 0x2a,
+	HW_VAR_SET_DEV_POWER = 0x2b,
+
+	HW_VAR_RCR = 0x2c,
+	HW_VAR_RATR_0 = 0x2d,
+	HW_VAR_RRSR = 0x2e,
+	HW_VAR_CPU_RST = 0x2f,
+	HW_VAR_CHECK_BSSID = 0x30,
+	HW_VAR_LBK_MODE = 0x31,
+	HW_VAR_AES_11N_FIX = 0x32,
+	HW_VAR_USB_RX_AGGR = 0x33,
+	HW_VAR_USER_CONTROL_TURBO_MODE = 0x34,
+	HW_VAR_RETRY_LIMIT = 0x35,
+	HW_VAR_INIT_TX_RATE = 0x36,
+	HW_VAR_TX_RATE_REG = 0x37,
+	HW_VAR_EFUSE_USAGE = 0x38,
+	HW_VAR_EFUSE_BYTES = 0x39,
+	HW_VAR_AUTOLOAD_STATUS = 0x3a,
+	HW_VAR_RF_2R_DISABLE = 0x3b,
+	HW_VAR_SET_RPWM = 0x3c,
+	HW_VAR_H2C_FW_PWRMODE = 0x3d,
+	HW_VAR_H2C_FW_JOINBSSRPT = 0x3e,
+	HW_VAR_H2C_FW_MEDIASTATUSRPT = 0x3f,
+	HW_VAR_H2C_FW_P2P_PS_OFFLOAD = 0x40,
+	HW_VAR_FW_PSMODE_STATUS = 0x41,
+	HW_VAR_INIT_RTS_RATE = 0x42,
+	HW_VAR_RESUME_CLK_ON = 0x43,
+	HW_VAR_FW_LPS_ACTION = 0x44,
+	HW_VAR_1X1_RECV_COMBINE = 0x45,
+	HW_VAR_STOP_SEND_BEACON = 0x46,
+	HW_VAR_TSF_TIMER = 0x47,
+	HW_VAR_IO_CMD = 0x48,
+
+	HW_VAR_RF_RECOVERY = 0x49,
+	HW_VAR_H2C_FW_UPDATE_GTK = 0x4a,
+	HW_VAR_WF_MASK = 0x4b,
+	HW_VAR_WF_CRC = 0x4c,
+	HW_VAR_WF_IS_MAC_ADDR = 0x4d,
+	HW_VAR_H2C_FW_OFFLOAD = 0x4e,
+	HW_VAR_RESET_WFCRC = 0x4f,
+
+	HW_VAR_HANDLE_FW_C2H = 0x50,
+	HW_VAR_DL_FW_RSVD_PAGE = 0x51,
+	HW_VAR_AID = 0x52,
+	HW_VAR_HW_SEQ_ENABLE = 0x53,
+	HW_VAR_CORRECT_TSF = 0x54,
+	HW_VAR_BCN_VALID = 0x55,
+	HW_VAR_FWLPS_RF_ON = 0x56,
+	HW_VAR_DUAL_TSF_RST = 0x57,
+	HW_VAR_SWITCH_EPHY_WoWLAN = 0x58,
+	HW_VAR_INT_MIGRATION = 0x59,
+	HW_VAR_INT_AC = 0x5a,
+	HW_VAR_RF_TIMING = 0x5b,
+
+	HAL_DEF_WOWLAN = 0x5c,
+	HW_VAR_MRC = 0x5d,
+	HW_VAR_KEEP_ALIVE = 0x5e,
+	HW_VAR_NAV_UPPER = 0x5f,
+
+	HW_VAR_MGT_FILTER = 0x60,
+	HW_VAR_CTRL_FILTER = 0x61,
+	HW_VAR_DATA_FILTER = 0x62,
 };
 
 enum rt_media_status {
diff --git a/drivers/net/wireless/ti/wl18xx/acx.c b/drivers/net/wireless/ti/wl18xx/acx.c
index 4be0409308cb..b5525a38264b 100644
--- a/drivers/net/wireless/ti/wl18xx/acx.c
+++ b/drivers/net/wireless/ti/wl18xx/acx.c
@@ -309,3 +309,32 @@ out:
 	kfree(acx);
 	return ret;
 }
+
+int wl18xx_acx_time_sync_cfg(struct wl1271 *wl)
+{
+	struct acx_time_sync_cfg *acx;
+	int ret;
+
+	wl1271_debug(DEBUG_ACX, "acx time sync cfg: mode %d, addr: %pM",
+		     wl->conf.sg.params[WL18XX_CONF_SG_TIME_SYNC],
+		     wl->zone_master_mac_addr);
+
+	acx = kzalloc(sizeof(*acx), GFP_KERNEL);
+	if (!acx) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	acx->sync_mode = wl->conf.sg.params[WL18XX_CONF_SG_TIME_SYNC];
+	memcpy(acx->zone_mac_addr, wl->zone_master_mac_addr, ETH_ALEN);
+
+	ret = wl1271_cmd_configure(wl, ACX_TIME_SYNC_CFG,
+				   acx, sizeof(*acx));
+	if (ret < 0) {
+		wl1271_warning("acx time sync cfg failed: %d", ret);
+		goto out;
+	}
+out:
+	kfree(acx);
+	return ret;
+}
diff --git a/drivers/net/wireless/ti/wl18xx/acx.h b/drivers/net/wireless/ti/wl18xx/acx.h
index 342a2993ef98..2edbbbfd8421 100644
--- a/drivers/net/wireless/ti/wl18xx/acx.h
+++ b/drivers/net/wireless/ti/wl18xx/acx.h
@@ -37,6 +37,7 @@ enum {
 	ACX_RX_BA_FILTER		 = 0x0058,
 	ACX_AP_SLEEP_CFG                 = 0x0059,
 	ACX_DYNAMIC_TRACES_CFG		 = 0x005A,
+	ACX_TIME_SYNC_CFG		 = 0x005B,
 };
 
 /* numbers of bits the length field takes (add 1 for the actual number) */
@@ -388,6 +389,17 @@ struct acx_dynamic_fw_traces_cfg {
 	__le32 dynamic_fw_traces;
 } __packed;
 
+/*
+ * ACX_TIME_SYNC_CFG
+ * configure the time sync parameters
+ */
+struct acx_time_sync_cfg {
+	struct acx_header header;
+	u8 sync_mode;
+	u8 zone_mac_addr[ETH_ALEN];
+	u8 padding[1];
+} __packed;
+
 int wl18xx_acx_host_if_cfg_bitmap(struct wl1271 *wl, u32 host_cfg_bitmap,
 				  u32 sdio_blk_size, u32 extra_mem_blks,
 				  u32 len_field_size);
@@ -402,5 +414,6 @@ int wl18xx_acx_interrupt_notify_config(struct wl1271 *wl, bool action);
 int wl18xx_acx_rx_ba_filter(struct wl1271 *wl, bool action);
 int wl18xx_acx_ap_sleep(struct wl1271 *wl);
 int wl18xx_acx_dynamic_fw_traces(struct wl1271 *wl);
+int wl18xx_acx_time_sync_cfg(struct wl1271 *wl);
 
 #endif /* __WL18XX_ACX_H__ */
diff --git a/drivers/net/wireless/ti/wl18xx/event.c b/drivers/net/wireless/ti/wl18xx/event.c
index 2c5df43b8ed9..b36ce185c9f2 100644
--- a/drivers/net/wireless/ti/wl18xx/event.c
+++ b/drivers/net/wireless/ti/wl18xx/event.c
@@ -22,6 +22,7 @@
 #include <net/genetlink.h>
 #include "event.h"
 #include "scan.h"
+#include "conf.h"
 #include "../wlcore/cmd.h"
 #include "../wlcore/debug.h"
 #include "../wlcore/vendor_cmd.h"
diff --git a/drivers/net/wireless/ti/wl18xx/main.c b/drivers/net/wireless/ti/wl18xx/main.c
index 00a04dfc03d1..06d6943b257c 100644
--- a/drivers/net/wireless/ti/wl18xx/main.c
+++ b/drivers/net/wireless/ti/wl18xx/main.c
@@ -1397,25 +1397,24 @@ out:
 	return ret;
 }
 
-#define WL18XX_CONF_FILE_NAME "ti-connectivity/wl18xx-conf.bin"
-
 static int wl18xx_load_conf_file(struct device *dev, struct wlcore_conf *conf,
-				 struct wl18xx_priv_conf *priv_conf)
+				 struct wl18xx_priv_conf *priv_conf,
+				 const char *file)
 {
 	struct wlcore_conf_file *conf_file;
 	const struct firmware *fw;
 	int ret;
 
-	ret = request_firmware(&fw, WL18XX_CONF_FILE_NAME, dev);
+	ret = request_firmware(&fw, file, dev);
 	if (ret < 0) {
 		wl1271_error("could not get configuration binary %s: %d",
-			     WL18XX_CONF_FILE_NAME, ret);
+			     file, ret);
 		return ret;
 	}
 
 	if (fw->size != WL18XX_CONF_SIZE) {
-		wl1271_error("configuration binary file size is wrong, expected %zu got %zu",
-			     WL18XX_CONF_SIZE, fw->size);
+		wl1271_error("%s configuration binary size is wrong, expected %zu got %zu",
+			     file, WL18XX_CONF_SIZE, fw->size);
 		ret = -EINVAL;
 		goto out_release;
 	}
@@ -1448,9 +1447,12 @@ out_release:
 
 static int wl18xx_conf_init(struct wl1271 *wl, struct device *dev)
 {
+	struct platform_device *pdev = wl->pdev;
+	struct wlcore_platdev_data *pdata = dev_get_platdata(&pdev->dev);
 	struct wl18xx_priv *priv = wl->priv;
 
-	if (wl18xx_load_conf_file(dev, &wl->conf, &priv->conf) < 0) {
+	if (wl18xx_load_conf_file(dev, &wl->conf, &priv->conf,
+				  pdata->family->cfg_name) < 0) {
 		wl1271_warning("falling back to default config");
 
 		/* apply driver default configuration */
@@ -2141,4 +2143,3 @@ MODULE_PARM_DESC(num_rx_desc_param,
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Luciano Coelho <coelho@ti.com>");
 MODULE_FIRMWARE(WL18XX_FW_NAME);
-MODULE_FIRMWARE(WL18XX_CONF_FILE_NAME);
diff --git a/drivers/net/wireless/ti/wlcore/boot.c b/drivers/net/wireless/ti/wlcore/boot.c
index f75d30444117..f00509ea8aca 100644
--- a/drivers/net/wireless/ti/wlcore/boot.c
+++ b/drivers/net/wireless/ti/wlcore/boot.c
@@ -282,6 +282,9 @@ EXPORT_SYMBOL_GPL(wlcore_boot_upload_firmware);
 
 int wlcore_boot_upload_nvs(struct wl1271 *wl)
 {
+	struct platform_device *pdev = wl->pdev;
+	struct wlcore_platdev_data *pdev_data = dev_get_platdata(&pdev->dev);
+	const char *nvs_name = "unknown";
 	size_t nvs_len, burst_len;
 	int i;
 	u32 dest_addr, val;
@@ -293,6 +296,9 @@ int wlcore_boot_upload_nvs(struct wl1271 *wl)
 		return -ENODEV;
 	}
 
+	if (pdev_data && pdev_data->family)
+		nvs_name = pdev_data->family->nvs_name;
+
 	if (wl->quirks & WLCORE_QUIRK_LEGACY_NVS) {
 		struct wl1271_nvs_file *nvs =
 			(struct wl1271_nvs_file *)wl->nvs;
@@ -310,8 +316,9 @@ int wlcore_boot_upload_nvs(struct wl1271 *wl)
 		if (wl->nvs_len != sizeof(struct wl1271_nvs_file) &&
 		    (wl->nvs_len != WL1271_INI_LEGACY_NVS_FILE_SIZE ||
 		     wl->enable_11a)) {
-			wl1271_error("nvs size is not as expected: %zu != %zu",
-				wl->nvs_len, sizeof(struct wl1271_nvs_file));
+			wl1271_error("%s size is not as expected: %zu != %zu",
+				     nvs_name, wl->nvs_len,
+				     sizeof(struct wl1271_nvs_file));
 			kfree(wl->nvs);
 			wl->nvs = NULL;
 			wl->nvs_len = 0;
@@ -328,8 +335,8 @@ int wlcore_boot_upload_nvs(struct wl1271 *wl)
 			if (nvs->general_params.dual_mode_select)
 				wl->enable_11a = true;
 		} else {
-			wl1271_error("nvs size is not as expected: %zu != %zu",
-				     wl->nvs_len,
+			wl1271_error("%s size is not as expected: %zu != %zu",
+				     nvs_name, wl->nvs_len,
 				     sizeof(struct wl128x_nvs_file));
 			kfree(wl->nvs);
 			wl->nvs = NULL;
diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c
index 1d689169da76..471521a0db7b 100644
--- a/drivers/net/wireless/ti/wlcore/main.c
+++ b/drivers/net/wireless/ti/wlcore/main.c
@@ -4986,7 +4986,6 @@ static int wl12xx_sta_add(struct wl1271 *wl,
 		return ret;
 
 	wl_sta = (struct wl1271_station *)sta->drv_priv;
-	wl_sta->wl = wl;
 	hlid = wl_sta->hlid;
 
 	ret = wl12xx_cmd_add_peer(wl, wlvif, sta, hlid);
@@ -5700,10 +5699,11 @@ out:
 	mutex_unlock(&wl->mutex);
 }
 
-static u32 wlcore_op_get_expected_throughput(struct ieee80211_sta *sta)
+static u32 wlcore_op_get_expected_throughput(struct ieee80211_hw *hw,
+					     struct ieee80211_sta *sta)
 {
 	struct wl1271_station *wl_sta = (struct wl1271_station *)sta->drv_priv;
-	struct wl1271 *wl = wl_sta->wl;
+	struct wl1271 *wl = hw->priv;
 	u8 hlid = wl_sta->hlid;
 
 	/* return in units of Kbps */
@@ -6413,9 +6413,12 @@ static void wlcore_nvs_cb(const struct firmware *fw, void *context)
 			goto out;
 		}
 		wl->nvs_len = fw->size;
-	} else {
+	} else if (pdev_data->family->nvs_name) {
 		wl1271_debug(DEBUG_BOOT, "Could not get nvs file %s",
-			     WL12XX_NVS_NAME);
+			     pdev_data->family->nvs_name);
+		wl->nvs = NULL;
+		wl->nvs_len = 0;
+	} else {
 		wl->nvs = NULL;
 		wl->nvs_len = 0;
 	}
@@ -6510,21 +6513,29 @@ out:
 
 int wlcore_probe(struct wl1271 *wl, struct platform_device *pdev)
 {
-	int ret;
+	struct wlcore_platdev_data *pdev_data = dev_get_platdata(&pdev->dev);
+	const char *nvs_name;
+	int ret = 0;
 
-	if (!wl->ops || !wl->ptable)
+	if (!wl->ops || !wl->ptable || !pdev_data)
 		return -EINVAL;
 
 	wl->dev = &pdev->dev;
 	wl->pdev = pdev;
 	platform_set_drvdata(pdev, wl);
 
-	ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_HOTPLUG,
-				      WL12XX_NVS_NAME, &pdev->dev, GFP_KERNEL,
-				      wl, wlcore_nvs_cb);
-	if (ret < 0) {
-		wl1271_error("request_firmware_nowait failed: %d", ret);
-		complete_all(&wl->nvs_loading_complete);
+	if (pdev_data->family && pdev_data->family->nvs_name) {
+		nvs_name = pdev_data->family->nvs_name;
+		ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_HOTPLUG,
+					      nvs_name, &pdev->dev, GFP_KERNEL,
+					      wl, wlcore_nvs_cb);
+		if (ret < 0) {
+			wl1271_error("request_firmware_nowait failed for %s: %d",
+				     nvs_name, ret);
+			complete_all(&wl->nvs_loading_complete);
+		}
+	} else {
+		wlcore_nvs_cb(NULL, wl);
 	}
 
 	return ret;
@@ -6533,9 +6544,11 @@ EXPORT_SYMBOL_GPL(wlcore_probe);
 
 int wlcore_remove(struct platform_device *pdev)
 {
+	struct wlcore_platdev_data *pdev_data = dev_get_platdata(&pdev->dev);
 	struct wl1271 *wl = platform_get_drvdata(pdev);
 
-	wait_for_completion(&wl->nvs_loading_complete);
+	if (pdev_data->family && pdev_data->family->nvs_name)
+		wait_for_completion(&wl->nvs_loading_complete);
 	if (!wl->initialized)
 		return 0;
 
@@ -6572,4 +6585,3 @@ MODULE_PARM_DESC(no_recovery, "Prevent HW recovery. FW will remain stuck.");
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Luciano Coelho <coelho@ti.com>");
 MODULE_AUTHOR("Juuso Oikarinen <juuso.oikarinen@nokia.com>");
-MODULE_FIRMWARE(WL12XX_NVS_NAME);
diff --git a/drivers/net/wireless/ti/wlcore/sdio.c b/drivers/net/wireless/ti/wlcore/sdio.c
index 5839acbbc782..a6e94b1a12cb 100644
--- a/drivers/net/wireless/ti/wlcore/sdio.c
+++ b/drivers/net/wireless/ti/wlcore/sdio.c
@@ -216,17 +216,33 @@ static struct wl1271_if_operations sdio_ops = {
 };
 
 #ifdef CONFIG_OF
+
+static const struct wilink_family_data wl127x_data = {
+	.name = "wl127x",
+	.nvs_name = "ti-connectivity/wl127x-nvs.bin",
+};
+
+static const struct wilink_family_data wl128x_data = {
+	.name = "wl128x",
+	.nvs_name = "ti-connectivity/wl128x-nvs.bin",
+};
+
+static const struct wilink_family_data wl18xx_data = {
+	.name = "wl18xx",
+	.cfg_name = "ti-connectivity/wl18xx-conf.bin",
+};
+
 static const struct of_device_id wlcore_sdio_of_match_table[] = {
-	{ .compatible = "ti,wl1271" },
-	{ .compatible = "ti,wl1273" },
-	{ .compatible = "ti,wl1281" },
-	{ .compatible = "ti,wl1283" },
-	{ .compatible = "ti,wl1801" },
-	{ .compatible = "ti,wl1805" },
-	{ .compatible = "ti,wl1807" },
-	{ .compatible = "ti,wl1831" },
-	{ .compatible = "ti,wl1835" },
-	{ .compatible = "ti,wl1837" },
+	{ .compatible = "ti,wl1271", .data = &wl127x_data },
+	{ .compatible = "ti,wl1273", .data = &wl127x_data },
+	{ .compatible = "ti,wl1281", .data = &wl128x_data },
+	{ .compatible = "ti,wl1283", .data = &wl128x_data },
+	{ .compatible = "ti,wl1801", .data = &wl18xx_data },
+	{ .compatible = "ti,wl1805", .data = &wl18xx_data },
+	{ .compatible = "ti,wl1807", .data = &wl18xx_data },
+	{ .compatible = "ti,wl1831", .data = &wl18xx_data },
+	{ .compatible = "ti,wl1835", .data = &wl18xx_data },
+	{ .compatible = "ti,wl1837", .data = &wl18xx_data },
 	{ }
 };
 
@@ -234,9 +250,13 @@ static int wlcore_probe_of(struct device *dev, int *irq,
 			   struct wlcore_platdev_data *pdev_data)
 {
 	struct device_node *np = dev->of_node;
+	const struct of_device_id *of_id;
+
+	of_id = of_match_node(wlcore_sdio_of_match_table, np);
+	if (!of_id)
+		return -ENODEV;
 
-	if (!np || !of_match_node(wlcore_sdio_of_match_table, np))
-		return -ENODATA;
+	pdev_data->family = of_id->data;
 
 	*irq = irq_of_parse_and_map(np, 0);
 	if (!*irq) {
@@ -263,7 +283,7 @@ static int wlcore_probe_of(struct device *dev, int *irq,
 static int wl1271_probe(struct sdio_func *func,
 				  const struct sdio_device_id *id)
 {
-	struct wlcore_platdev_data pdev_data;
+	struct wlcore_platdev_data *pdev_data;
 	struct wl12xx_sdio_glue *glue;
 	struct resource res[1];
 	mmc_pm_flag_t mmcflags;
@@ -275,14 +295,15 @@ static int wl1271_probe(struct sdio_func *func,
 	if (func->num != 0x02)
 		return -ENODEV;
 
-	memset(&pdev_data, 0x00, sizeof(pdev_data));
-	pdev_data.if_ops = &sdio_ops;
+	pdev_data = devm_kzalloc(&func->dev, sizeof(*pdev_data), GFP_KERNEL);
+	if (!pdev_data)
+		return -ENOMEM;
 
-	glue = kzalloc(sizeof(*glue), GFP_KERNEL);
-	if (!glue) {
-		dev_err(&func->dev, "can't allocate glue\n");
-		goto out;
-	}
+	pdev_data->if_ops = &sdio_ops;
+
+	glue = devm_kzalloc(&func->dev, sizeof(*glue), GFP_KERNEL);
+	if (!glue)
+		return -ENOMEM;
 
 	glue->dev = &func->dev;
 
@@ -292,16 +313,16 @@ static int wl1271_probe(struct sdio_func *func,
 	/* Use block mode for transferring over one block size of data */
 	func->card->quirks |= MMC_QUIRK_BLKSZ_FOR_BYTE_MODE;
 
-	ret = wlcore_probe_of(&func->dev, &irq, &pdev_data);
+	ret = wlcore_probe_of(&func->dev, &irq, pdev_data);
 	if (ret)
-		goto out_free_glue;
+		goto out;
 
 	/* if sdio can keep power while host is suspended, enable wow */
 	mmcflags = sdio_get_host_pm_caps(func);
 	dev_dbg(glue->dev, "sdio PM caps = 0x%x\n", mmcflags);
 
 	if (mmcflags & MMC_PM_KEEP_POWER)
-		pdev_data.pwr_in_suspend = true;
+		pdev_data->pwr_in_suspend = true;
 
 	sdio_set_drvdata(func, glue);
 
@@ -323,7 +344,7 @@ static int wl1271_probe(struct sdio_func *func,
 	if (!glue->core) {
 		dev_err(glue->dev, "can't allocate platform_device");
 		ret = -ENOMEM;
-		goto out_free_glue;
+		goto out;
 	}
 
 	glue->core->dev.parent = &func->dev;
@@ -341,8 +362,8 @@ static int wl1271_probe(struct sdio_func *func,
 		goto out_dev_put;
 	}
 
-	ret = platform_device_add_data(glue->core, &pdev_data,
-				       sizeof(pdev_data));
+	ret = platform_device_add_data(glue->core, pdev_data,
+				       sizeof(*pdev_data));
 	if (ret) {
 		dev_err(glue->dev, "can't add platform data\n");
 		goto out_dev_put;
@@ -358,9 +379,6 @@ static int wl1271_probe(struct sdio_func *func,
 out_dev_put:
 	platform_device_put(glue->core);
 
-out_free_glue:
-	kfree(glue);
-
 out:
 	return ret;
 }
diff --git a/drivers/net/wireless/ti/wlcore/spi.c b/drivers/net/wireless/ti/wlcore/spi.c
index 6d24040889b8..f949ad2bd898 100644
--- a/drivers/net/wireless/ti/wlcore/spi.c
+++ b/drivers/net/wireless/ti/wlcore/spi.c
@@ -79,19 +79,19 @@
 #define WSPI_MAX_NUM_OF_CHUNKS \
 	((SPI_AGGR_BUFFER_SIZE / WSPI_MAX_CHUNK_SIZE) + 1)
 
-
-struct wilink_familiy_data {
-	char name[8];
+static const struct wilink_family_data wl127x_data = {
+	.name = "wl127x",
+	.nvs_name = "ti-connectivity/wl127x-nvs.bin",
 };
 
-const struct wilink_familiy_data *wilink_data;
-
-static const struct wilink_familiy_data wl18xx_data = {
-	.name = "wl18xx",
+static const struct wilink_family_data wl128x_data = {
+	.name = "wl128x",
+	.nvs_name = "ti-connectivity/wl128x-nvs.bin",
 };
 
-static const struct wilink_familiy_data wl12xx_data = {
-	.name = "wl12xx",
+static const struct wilink_family_data wl18xx_data = {
+	.name = "wl18xx",
+	.cfg_name = "ti-connectivity/wl18xx-conf.bin",
 };
 
 struct wl12xx_spi_glue {
@@ -429,10 +429,10 @@ static struct wl1271_if_operations spi_ops = {
 };
 
 static const struct of_device_id wlcore_spi_of_match_table[] = {
-	{ .compatible = "ti,wl1271", .data = &wl12xx_data},
-	{ .compatible = "ti,wl1273", .data = &wl12xx_data},
-	{ .compatible = "ti,wl1281", .data = &wl12xx_data},
-	{ .compatible = "ti,wl1283", .data = &wl12xx_data},
+	{ .compatible = "ti,wl1271", .data = &wl127x_data},
+	{ .compatible = "ti,wl1273", .data = &wl127x_data},
+	{ .compatible = "ti,wl1281", .data = &wl128x_data},
+	{ .compatible = "ti,wl1283", .data = &wl128x_data},
 	{ .compatible = "ti,wl1801", .data = &wl18xx_data},
 	{ .compatible = "ti,wl1805", .data = &wl18xx_data},
 	{ .compatible = "ti,wl1807", .data = &wl18xx_data},
@@ -460,9 +460,9 @@ static int wlcore_probe_of(struct spi_device *spi, struct wl12xx_spi_glue *glue,
 	if (!of_id)
 		return -ENODEV;
 
-	wilink_data = of_id->data;
-	dev_info(&spi->dev, "selected chip familiy is %s\n",
-		 wilink_data->name);
+	pdev_data->family = of_id->data;
+	dev_info(&spi->dev, "selected chip family is %s\n",
+		 pdev_data->family->name);
 
 	if (of_find_property(dt_node, "clock-xtal", NULL))
 		pdev_data->ref_clock_xtal = true;
@@ -479,13 +479,15 @@ static int wlcore_probe_of(struct spi_device *spi, struct wl12xx_spi_glue *glue,
 static int wl1271_probe(struct spi_device *spi)
 {
 	struct wl12xx_spi_glue *glue;
-	struct wlcore_platdev_data pdev_data;
+	struct wlcore_platdev_data *pdev_data;
 	struct resource res[1];
 	int ret;
 
-	memset(&pdev_data, 0x00, sizeof(pdev_data));
+	pdev_data = devm_kzalloc(&spi->dev, sizeof(*pdev_data), GFP_KERNEL);
+	if (!pdev_data)
+		return -ENOMEM;
 
-	pdev_data.if_ops = &spi_ops;
+	pdev_data->if_ops = &spi_ops;
 
 	glue = devm_kzalloc(&spi->dev, sizeof(*glue), GFP_KERNEL);
 	if (!glue) {
@@ -509,7 +511,7 @@ static int wl1271_probe(struct spi_device *spi)
 		return PTR_ERR(glue->reg);
 	}
 
-	ret = wlcore_probe_of(spi, glue, &pdev_data);
+	ret = wlcore_probe_of(spi, glue, pdev_data);
 	if (ret) {
 		dev_err(glue->dev,
 			"can't get device tree parameters (%d)\n", ret);
@@ -522,7 +524,7 @@ static int wl1271_probe(struct spi_device *spi)
 		return ret;
 	}
 
-	glue->core = platform_device_alloc(wilink_data->name,
+	glue->core = platform_device_alloc(pdev_data->family->name,
 					   PLATFORM_DEVID_AUTO);
 	if (!glue->core) {
 		dev_err(glue->dev, "can't allocate platform_device\n");
@@ -543,8 +545,8 @@ static int wl1271_probe(struct spi_device *spi)
 		goto out_dev_put;
 	}
 
-	ret = platform_device_add_data(glue->core, &pdev_data,
-				       sizeof(pdev_data));
+	ret = platform_device_add_data(glue->core, pdev_data,
+				       sizeof(*pdev_data));
 	if (ret) {
 		dev_err(glue->dev, "can't add platform data\n");
 		goto out_dev_put;
diff --git a/drivers/net/wireless/ti/wlcore/wlcore.h b/drivers/net/wireless/ti/wlcore/wlcore.h
index 8f28aa02230c..1827546ba807 100644
--- a/drivers/net/wireless/ti/wlcore/wlcore.h
+++ b/drivers/net/wireless/ti/wlcore/wlcore.h
@@ -501,6 +501,9 @@ struct wl1271 {
 
 	/* dynamic fw traces */
 	u32 dynamic_fw_traces;
+
+	/* time sync zone master */
+	u8 zone_master_mac_addr[ETH_ALEN];
 };
 
 int wlcore_probe(struct wl1271 *wl, struct platform_device *pdev);
diff --git a/drivers/net/wireless/ti/wlcore/wlcore_i.h b/drivers/net/wireless/ti/wlcore/wlcore_i.h
index 242b4e37b94c..e840985385fc 100644
--- a/drivers/net/wireless/ti/wlcore/wlcore_i.h
+++ b/drivers/net/wireless/ti/wlcore/wlcore_i.h
@@ -35,12 +35,11 @@
 #include "conf.h"
 #include "ini.h"
 
-/*
- * wl127x and wl128x are using the same NVS file name. However, the
- * ini parameters between them are different.  The driver validates
- * the correct NVS size in wl1271_boot_upload_nvs().
- */
-#define WL12XX_NVS_NAME "ti-connectivity/wl1271-nvs.bin"
+struct wilink_family_data {
+	const char *name;
+	const char *nvs_name;	/* wl12xx nvs file */
+	const char *cfg_name;	/* wl18xx cfg file */
+};
 
 #define WL1271_TX_SECURITY_LO16(s) ((u16)((s) & 0xffff))
 #define WL1271_TX_SECURITY_HI32(s) ((u32)(((s) >> 16) & 0xffffffff))
@@ -208,6 +207,7 @@ struct wl1271_if_operations {
 
 struct wlcore_platdev_data {
 	struct wl1271_if_operations *if_ops;
+	const struct wilink_family_data *family;
 
 	bool ref_clock_xtal;	/* specify whether the clock is XTAL or not */
 	u32 ref_clock_freq;	/* in Hertz */
@@ -347,7 +347,6 @@ struct wl1271_station {
 	 * Used in both AP and STA mode.
 	 */
 	u64 total_freed_pkts;
-	struct wl1271 *wl;
 };
 
 struct wl12xx_vif {
diff --git a/drivers/net/wireless/wl3501_cs.c b/drivers/net/wireless/wl3501_cs.c
index 82d94f83b6b4..932f3f81e8cf 100644
--- a/drivers/net/wireless/wl3501_cs.c
+++ b/drivers/net/wireless/wl3501_cs.c
@@ -1258,7 +1258,9 @@ static int wl3501_reset(struct net_device *dev)
 {
 	struct wl3501_card *this = netdev_priv(dev);
 	int rc = -ENODEV;
+	unsigned long flags;
 
+	spin_lock_irqsave(&this->lock, flags);
 	wl3501_block_interrupt(this);
 
 	if (wl3501_init_firmware(this)) {
@@ -1280,20 +1282,17 @@ static int wl3501_reset(struct net_device *dev)
 	pr_debug("%s: device reset", dev->name);
 	rc = 0;
 out:
+	spin_unlock_irqrestore(&this->lock, flags);
 	return rc;
 }
 
 static void wl3501_tx_timeout(struct net_device *dev)
 {
-	struct wl3501_card *this = netdev_priv(dev);
 	struct net_device_stats *stats = &dev->stats;
-	unsigned long flags;
 	int rc;
 
 	stats->tx_errors++;
-	spin_lock_irqsave(&this->lock, flags);
 	rc = wl3501_reset(dev);
-	spin_unlock_irqrestore(&this->lock, flags);
 	if (rc)
 		printk(KERN_ERR "%s: Error %d resetting card on Tx timeout!\n",
 		       dev->name, rc);
diff --git a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c
index a912dc051111..c5effd6c6be9 100644
--- a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c
+++ b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c
@@ -193,7 +193,7 @@ static int upload_code(struct usb_device *udev,
 			0, 0, p, sizeof(ret), 5000 /* ms */);
 		if (r != sizeof(ret)) {
 			dev_err(&udev->dev,
-				"control request firmeware confirmation failed."
+				"control request firmware confirmation failed."
 				" Return value %d\n", r);
 			if (r >= 0)
 				r = -ENODEV;
diff --git a/drivers/net/xen-netback/Makefile b/drivers/net/xen-netback/Makefile
index 11e02be9db1a..d49798a46b51 100644
--- a/drivers/net/xen-netback/Makefile
+++ b/drivers/net/xen-netback/Makefile
@@ -1,3 +1,3 @@
 obj-$(CONFIG_XEN_NETDEV_BACKEND) := xen-netback.o
 
-xen-netback-y := netback.o xenbus.o interface.o hash.o
+xen-netback-y := netback.o xenbus.o interface.o hash.o rx.o
diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h
index 84d6cbdd11b2..cf68149cbb55 100644
--- a/drivers/net/xen-netback/common.h
+++ b/drivers/net/xen-netback/common.h
@@ -91,13 +91,6 @@ struct xenvif_rx_meta {
  */
 #define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1)
 
-/* It's possible for an skb to have a maximal number of frags
- * but still be less than MAX_BUFFER_OFFSET in size. Thus the
- * worst-case number of copy operations is MAX_XEN_SKB_FRAGS per
- * ring slot.
- */
-#define MAX_GRANT_COPY_OPS (MAX_XEN_SKB_FRAGS * XEN_NETIF_RX_RING_SIZE)
-
 #define NETBACK_INVALID_HANDLE -1
 
 /* To avoid confusion, we define XEN_NETBK_LEGACY_SLOTS_MAX indicating
@@ -133,6 +126,15 @@ struct xenvif_stats {
 	unsigned long tx_frag_overflow;
 };
 
+#define COPY_BATCH_SIZE 64
+
+struct xenvif_copy_state {
+	struct gnttab_copy op[COPY_BATCH_SIZE];
+	RING_IDX idx[COPY_BATCH_SIZE];
+	unsigned int num;
+	struct sk_buff_head *completed;
+};
+
 struct xenvif_queue { /* Per-queue data for xenvif */
 	unsigned int id; /* Queue ID, 0-based */
 	char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */
@@ -189,12 +191,7 @@ struct xenvif_queue { /* Per-queue data for xenvif */
 	unsigned long last_rx_time;
 	bool stalled;
 
-	struct gnttab_copy grant_copy_op[MAX_GRANT_COPY_OPS];
-
-	/* We create one meta structure per ring request we consume, so
-	 * the maximum number is the same as the ring size.
-	 */
-	struct xenvif_rx_meta meta[XEN_NETIF_RX_RING_SIZE];
+	struct xenvif_copy_state rx_copy;
 
 	/* Transmit shaping: allow 'credit_bytes' every 'credit_usec'. */
 	unsigned long   credit_bytes;
@@ -260,7 +257,6 @@ struct xenvif {
 
 	/* Frontend feature information. */
 	int gso_mask;
-	int gso_prefix_mask;
 
 	u8 can_sg:1;
 	u8 ip_csum:1;
@@ -292,8 +288,6 @@ struct xenvif {
 #endif
 
 	struct xen_netif_ctrl_back_ring ctrl;
-	struct task_struct *ctrl_task;
-	wait_queue_head_t ctrl_wq;
 	unsigned int ctrl_irq;
 
 	/* Miscellaneous private stuff. */
@@ -359,8 +353,9 @@ void xenvif_kick_thread(struct xenvif_queue *queue);
 
 int xenvif_dealloc_kthread(void *data);
 
-int xenvif_ctrl_kthread(void *data);
+irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data);
 
+void xenvif_rx_action(struct xenvif_queue *queue);
 void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb);
 
 void xenvif_carrier_on(struct xenvif *vif);
diff --git a/drivers/net/xen-netback/hash.c b/drivers/net/xen-netback/hash.c
index fb87cb39a56b..613bac057650 100644
--- a/drivers/net/xen-netback/hash.c
+++ b/drivers/net/xen-netback/hash.c
@@ -32,15 +32,6 @@
 #include <linux/vmalloc.h>
 #include <linux/rculist.h>
 
-static void xenvif_del_hash(struct rcu_head *rcu)
-{
-	struct xenvif_hash_cache_entry *entry;
-
-	entry = container_of(rcu, struct xenvif_hash_cache_entry, rcu);
-
-	kfree(entry);
-}
-
 static void xenvif_add_hash(struct xenvif *vif, const u8 *tag,
 			    unsigned int len, u32 val)
 {
@@ -76,7 +67,7 @@ static void xenvif_add_hash(struct xenvif *vif, const u8 *tag,
 		if (++vif->hash.cache.count > xenvif_hash_cache_size) {
 			list_del_rcu(&oldest->link);
 			vif->hash.cache.count--;
-			call_rcu(&oldest->rcu, xenvif_del_hash);
+			kfree_rcu(oldest, rcu);
 		}
 	}
 
@@ -114,7 +105,7 @@ static void xenvif_flush_hash(struct xenvif *vif)
 	list_for_each_entry_rcu(entry, &vif->hash.cache.list, link) {
 		list_del_rcu(&entry->link);
 		vif->hash.cache.count--;
-		call_rcu(&entry->rcu, xenvif_del_hash);
+		kfree_rcu(entry, rcu);
 	}
 
 	spin_unlock_irqrestore(&vif->hash.cache.lock, flags);
diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c
index 83deeebfc2d1..74dc2bf71428 100644
--- a/drivers/net/xen-netback/interface.c
+++ b/drivers/net/xen-netback/interface.c
@@ -128,15 +128,6 @@ irqreturn_t xenvif_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-irqreturn_t xenvif_ctrl_interrupt(int irq, void *dev_id)
-{
-	struct xenvif *vif = dev_id;
-
-	wake_up(&vif->ctrl_wq);
-
-	return IRQ_HANDLED;
-}
-
 int xenvif_queue_stopped(struct xenvif_queue *queue)
 {
 	struct net_device *dev = queue->vif->dev;
@@ -158,17 +149,8 @@ static u16 xenvif_select_queue(struct net_device *dev, struct sk_buff *skb,
 	struct xenvif *vif = netdev_priv(dev);
 	unsigned int size = vif->hash.size;
 
-	if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE) {
-		u16 index = fallback(dev, skb) % dev->real_num_tx_queues;
-
-		/* Make sure there is no hash information in the socket
-		 * buffer otherwise it would be incorrectly forwarded
-		 * to the frontend.
-		 */
-		skb_clear_hash(skb);
-
-		return index;
-	}
+	if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE)
+		return fallback(dev, skb) % dev->real_num_tx_queues;
 
 	xenvif_set_skb_hash(vif, skb);
 
@@ -217,6 +199,13 @@ static int xenvif_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	cb = XENVIF_RX_CB(skb);
 	cb->expires = jiffies + vif->drain_timeout;
 
+	/* If there is no hash algorithm configured then make sure there
+	 * is no hash information in the socket buffer otherwise it
+	 * would be incorrectly forwarded to the frontend.
+	 */
+	if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE)
+		skb_clear_hash(skb);
+
 	xenvif_rx_queue_tail(queue, skb);
 	xenvif_kick_thread(queue);
 
@@ -328,9 +317,9 @@ static netdev_features_t xenvif_fix_features(struct net_device *dev,
 
 	if (!vif->can_sg)
 		features &= ~NETIF_F_SG;
-	if (~(vif->gso_mask | vif->gso_prefix_mask) & GSO_BIT(TCPV4))
+	if (~(vif->gso_mask) & GSO_BIT(TCPV4))
 		features &= ~NETIF_F_TSO;
-	if (~(vif->gso_mask | vif->gso_prefix_mask) & GSO_BIT(TCPV6))
+	if (~(vif->gso_mask) & GSO_BIT(TCPV6))
 		features &= ~NETIF_F_TSO6;
 	if (!vif->ip_csum)
 		features &= ~NETIF_F_IP_CSUM;
@@ -476,7 +465,7 @@ struct xenvif *xenvif_alloc(struct device *parent, domid_t domid,
 	dev->netdev_ops	= &xenvif_netdev_ops;
 	dev->hw_features = NETIF_F_SG |
 		NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
-		NETIF_F_TSO | NETIF_F_TSO6;
+		NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_FRAGLIST;
 	dev->features = dev->hw_features | NETIF_F_RXCSUM;
 	dev->ethtool_ops = &xenvif_ethtool_ops;
 
@@ -570,8 +559,7 @@ int xenvif_connect_ctrl(struct xenvif *vif, grant_ref_t ring_ref,
 	struct net_device *dev = vif->dev;
 	void *addr;
 	struct xen_netif_ctrl_sring *shared;
-	struct task_struct *task;
-	int err = -ENOMEM;
+	int err;
 
 	err = xenbus_map_ring_valloc(xenvif_to_xenbus_device(vif),
 				     &ring_ref, 1, &addr);
@@ -581,11 +569,7 @@ int xenvif_connect_ctrl(struct xenvif *vif, grant_ref_t ring_ref,
 	shared = (struct xen_netif_ctrl_sring *)addr;
 	BACK_RING_INIT(&vif->ctrl, shared, XEN_PAGE_SIZE);
 
-	init_waitqueue_head(&vif->ctrl_wq);
-
-	err = bind_interdomain_evtchn_to_irqhandler(vif->domid, evtchn,
-						    xenvif_ctrl_interrupt,
-						    0, dev->name, vif);
+	err = bind_interdomain_evtchn_to_irq(vif->domid, evtchn);
 	if (err < 0)
 		goto err_unmap;
 
@@ -593,19 +577,13 @@ int xenvif_connect_ctrl(struct xenvif *vif, grant_ref_t ring_ref,
 
 	xenvif_init_hash(vif);
 
-	task = kthread_create(xenvif_ctrl_kthread, (void *)vif,
-			      "%s-control", dev->name);
-	if (IS_ERR(task)) {
-		pr_warn("Could not allocate kthread for %s\n", dev->name);
-		err = PTR_ERR(task);
+	err = request_threaded_irq(vif->ctrl_irq, NULL, xenvif_ctrl_irq_fn,
+				   IRQF_ONESHOT, "xen-netback-ctrl", vif);
+	if (err) {
+		pr_warn("Could not setup irq handler for %s\n", dev->name);
 		goto err_deinit;
 	}
 
-	get_task_struct(task);
-	vif->ctrl_task = task;
-
-	wake_up_process(vif->ctrl_task);
-
 	return 0;
 
 err_deinit:
@@ -774,12 +752,6 @@ void xenvif_disconnect_data(struct xenvif *vif)
 
 void xenvif_disconnect_ctrl(struct xenvif *vif)
 {
-	if (vif->ctrl_task) {
-		kthread_stop(vif->ctrl_task);
-		put_task_struct(vif->ctrl_task);
-		vif->ctrl_task = NULL;
-	}
-
 	if (vif->ctrl_irq) {
 		xenvif_deinit_hash(vif);
 		unbind_from_irqhandler(vif->ctrl_irq, vif);
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index edbae0b1e8f0..47b481095d77 100644
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -106,13 +106,6 @@ static void push_tx_responses(struct xenvif_queue *queue);
 
 static inline int tx_work_todo(struct xenvif_queue *queue);
 
-static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue,
-					     u16      id,
-					     s8       st,
-					     u16      offset,
-					     u16      size,
-					     u16      flags);
-
 static inline unsigned long idx_to_pfn(struct xenvif_queue *queue,
 				       u16 idx)
 {
@@ -155,571 +148,11 @@ static inline pending_ring_idx_t pending_index(unsigned i)
 	return i & (MAX_PENDING_REQS-1);
 }
 
-static bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue)
-{
-	RING_IDX prod, cons;
-	struct sk_buff *skb;
-	int needed;
-
-	skb = skb_peek(&queue->rx_queue);
-	if (!skb)
-		return false;
-
-	needed = DIV_ROUND_UP(skb->len, XEN_PAGE_SIZE);
-	if (skb_is_gso(skb))
-		needed++;
-	if (skb->sw_hash)
-		needed++;
-
-	do {
-		prod = queue->rx.sring->req_prod;
-		cons = queue->rx.req_cons;
-
-		if (prod - cons >= needed)
-			return true;
-
-		queue->rx.sring->req_event = prod + 1;
-
-		/* Make sure event is visible before we check prod
-		 * again.
-		 */
-		mb();
-	} while (queue->rx.sring->req_prod != prod);
-
-	return false;
-}
-
-void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&queue->rx_queue.lock, flags);
-
-	__skb_queue_tail(&queue->rx_queue, skb);
-
-	queue->rx_queue_len += skb->len;
-	if (queue->rx_queue_len > queue->rx_queue_max)
-		netif_tx_stop_queue(netdev_get_tx_queue(queue->vif->dev, queue->id));
-
-	spin_unlock_irqrestore(&queue->rx_queue.lock, flags);
-}
-
-static struct sk_buff *xenvif_rx_dequeue(struct xenvif_queue *queue)
-{
-	struct sk_buff *skb;
-
-	spin_lock_irq(&queue->rx_queue.lock);
-
-	skb = __skb_dequeue(&queue->rx_queue);
-	if (skb)
-		queue->rx_queue_len -= skb->len;
-
-	spin_unlock_irq(&queue->rx_queue.lock);
-
-	return skb;
-}
-
-static void xenvif_rx_queue_maybe_wake(struct xenvif_queue *queue)
-{
-	spin_lock_irq(&queue->rx_queue.lock);
-
-	if (queue->rx_queue_len < queue->rx_queue_max)
-		netif_tx_wake_queue(netdev_get_tx_queue(queue->vif->dev, queue->id));
-
-	spin_unlock_irq(&queue->rx_queue.lock);
-}
-
-
-static void xenvif_rx_queue_purge(struct xenvif_queue *queue)
-{
-	struct sk_buff *skb;
-	while ((skb = xenvif_rx_dequeue(queue)) != NULL)
-		kfree_skb(skb);
-}
-
-static void xenvif_rx_queue_drop_expired(struct xenvif_queue *queue)
-{
-	struct sk_buff *skb;
-
-	for(;;) {
-		skb = skb_peek(&queue->rx_queue);
-		if (!skb)
-			break;
-		if (time_before(jiffies, XENVIF_RX_CB(skb)->expires))
-			break;
-		xenvif_rx_dequeue(queue);
-		kfree_skb(skb);
-	}
-}
-
-struct netrx_pending_operations {
-	unsigned copy_prod, copy_cons;
-	unsigned meta_prod, meta_cons;
-	struct gnttab_copy *copy;
-	struct xenvif_rx_meta *meta;
-	int copy_off;
-	grant_ref_t copy_gref;
-};
-
-static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif_queue *queue,
-						 struct netrx_pending_operations *npo)
-{
-	struct xenvif_rx_meta *meta;
-	struct xen_netif_rx_request req;
-
-	RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req);
-
-	meta = npo->meta + npo->meta_prod++;
-	meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
-	meta->gso_size = 0;
-	meta->size = 0;
-	meta->id = req.id;
-
-	npo->copy_off = 0;
-	npo->copy_gref = req.gref;
-
-	return meta;
-}
-
-struct gop_frag_copy {
-	struct xenvif_queue *queue;
-	struct netrx_pending_operations *npo;
-	struct xenvif_rx_meta *meta;
-	int head;
-	int gso_type;
-	int protocol;
-	int hash_present;
-
-	struct page *page;
-};
-
-static void xenvif_setup_copy_gop(unsigned long gfn,
-				  unsigned int offset,
-				  unsigned int *len,
-				  struct gop_frag_copy *info)
-{
-	struct gnttab_copy *copy_gop;
-	struct xen_page_foreign *foreign;
-	/* Convenient aliases */
-	struct xenvif_queue *queue = info->queue;
-	struct netrx_pending_operations *npo = info->npo;
-	struct page *page = info->page;
-
-	BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET);
-
-	if (npo->copy_off == MAX_BUFFER_OFFSET)
-		info->meta = get_next_rx_buffer(queue, npo);
-
-	if (npo->copy_off + *len > MAX_BUFFER_OFFSET)
-		*len = MAX_BUFFER_OFFSET - npo->copy_off;
-
-	copy_gop = npo->copy + npo->copy_prod++;
-	copy_gop->flags = GNTCOPY_dest_gref;
-	copy_gop->len = *len;
-
-	foreign = xen_page_foreign(page);
-	if (foreign) {
-		copy_gop->source.domid = foreign->domid;
-		copy_gop->source.u.ref = foreign->gref;
-		copy_gop->flags |= GNTCOPY_source_gref;
-	} else {
-		copy_gop->source.domid = DOMID_SELF;
-		copy_gop->source.u.gmfn = gfn;
-	}
-	copy_gop->source.offset = offset;
-
-	copy_gop->dest.domid = queue->vif->domid;
-	copy_gop->dest.offset = npo->copy_off;
-	copy_gop->dest.u.ref = npo->copy_gref;
-
-	npo->copy_off += *len;
-	info->meta->size += *len;
-
-	if (!info->head)
-		return;
-
-	/* Leave a gap for the GSO descriptor. */
-	if ((1 << info->gso_type) & queue->vif->gso_mask)
-		queue->rx.req_cons++;
-
-	/* Leave a gap for the hash extra segment. */
-	if (info->hash_present)
-		queue->rx.req_cons++;
-
-	info->head = 0; /* There must be something in this buffer now */
-}
-
-static void xenvif_gop_frag_copy_grant(unsigned long gfn,
-				       unsigned offset,
-				       unsigned int len,
-				       void *data)
-{
-	unsigned int bytes;
-
-	while (len) {
-		bytes = len;
-		xenvif_setup_copy_gop(gfn, offset, &bytes, data);
-		offset += bytes;
-		len -= bytes;
-	}
-}
-
-/*
- * Set up the grant operations for this fragment. If it's a flipping
- * interface, we also set up the unmap request from here.
- */
-static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb,
-				 struct netrx_pending_operations *npo,
-				 struct page *page, unsigned long size,
-				 unsigned long offset, int *head)
-{
-	struct gop_frag_copy info = {
-		.queue = queue,
-		.npo = npo,
-		.head = *head,
-		.gso_type = XEN_NETIF_GSO_TYPE_NONE,
-		/* xenvif_set_skb_hash() will have either set a s/w
-		 * hash or cleared the hash depending on
-		 * whether the the frontend wants a hash for this skb.
-		 */
-		.hash_present = skb->sw_hash,
-	};
-	unsigned long bytes;
-
-	if (skb_is_gso(skb)) {
-		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
-			info.gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
-		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
-			info.gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
-	}
-
-	/* Data must not cross a page boundary. */
-	BUG_ON(size + offset > PAGE_SIZE<<compound_order(page));
-
-	info.meta = npo->meta + npo->meta_prod - 1;
-
-	/* Skip unused frames from start of page */
-	page += offset >> PAGE_SHIFT;
-	offset &= ~PAGE_MASK;
-
-	while (size > 0) {
-		BUG_ON(offset >= PAGE_SIZE);
-
-		bytes = PAGE_SIZE - offset;
-		if (bytes > size)
-			bytes = size;
-
-		info.page = page;
-		gnttab_foreach_grant_in_range(page, offset, bytes,
-					      xenvif_gop_frag_copy_grant,
-					      &info);
-		size -= bytes;
-		offset = 0;
-
-		/* Next page */
-		if (size) {
-			BUG_ON(!PageCompound(page));
-			page++;
-		}
-	}
-
-	*head = info.head;
-}
-
-/*
- * Prepare an SKB to be transmitted to the frontend.
- *
- * This function is responsible for allocating grant operations, meta
- * structures, etc.
- *
- * It returns the number of meta structures consumed. The number of
- * ring slots used is always equal to the number of meta slots used
- * plus the number of GSO descriptors used. Currently, we use either
- * zero GSO descriptors (for non-GSO packets) or one descriptor (for
- * frontend-side LRO).
- */
-static int xenvif_gop_skb(struct sk_buff *skb,
-			  struct netrx_pending_operations *npo,
-			  struct xenvif_queue *queue)
-{
-	struct xenvif *vif = netdev_priv(skb->dev);
-	int nr_frags = skb_shinfo(skb)->nr_frags;
-	int i;
-	struct xen_netif_rx_request req;
-	struct xenvif_rx_meta *meta;
-	unsigned char *data;
-	int head = 1;
-	int old_meta_prod;
-	int gso_type;
-
-	old_meta_prod = npo->meta_prod;
-
-	gso_type = XEN_NETIF_GSO_TYPE_NONE;
-	if (skb_is_gso(skb)) {
-		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
-			gso_type = XEN_NETIF_GSO_TYPE_TCPV4;
-		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
-			gso_type = XEN_NETIF_GSO_TYPE_TCPV6;
-	}
-
-	/* Set up a GSO prefix descriptor, if necessary */
-	if ((1 << gso_type) & vif->gso_prefix_mask) {
-		RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req);
-		meta = npo->meta + npo->meta_prod++;
-		meta->gso_type = gso_type;
-		meta->gso_size = skb_shinfo(skb)->gso_size;
-		meta->size = 0;
-		meta->id = req.id;
-	}
-
-	RING_COPY_REQUEST(&queue->rx, queue->rx.req_cons++, &req);
-	meta = npo->meta + npo->meta_prod++;
-
-	if ((1 << gso_type) & vif->gso_mask) {
-		meta->gso_type = gso_type;
-		meta->gso_size = skb_shinfo(skb)->gso_size;
-	} else {
-		meta->gso_type = XEN_NETIF_GSO_TYPE_NONE;
-		meta->gso_size = 0;
-	}
-
-	meta->size = 0;
-	meta->id = req.id;
-	npo->copy_off = 0;
-	npo->copy_gref = req.gref;
-
-	data = skb->data;
-	while (data < skb_tail_pointer(skb)) {
-		unsigned int offset = offset_in_page(data);
-		unsigned int len = PAGE_SIZE - offset;
-
-		if (data + len > skb_tail_pointer(skb))
-			len = skb_tail_pointer(skb) - data;
-
-		xenvif_gop_frag_copy(queue, skb, npo,
-				     virt_to_page(data), len, offset, &head);
-		data += len;
-	}
-
-	for (i = 0; i < nr_frags; i++) {
-		xenvif_gop_frag_copy(queue, skb, npo,
-				     skb_frag_page(&skb_shinfo(skb)->frags[i]),
-				     skb_frag_size(&skb_shinfo(skb)->frags[i]),
-				     skb_shinfo(skb)->frags[i].page_offset,
-				     &head);
-	}
-
-	return npo->meta_prod - old_meta_prod;
-}
-
-/*
- * This is a twin to xenvif_gop_skb.  Assume that xenvif_gop_skb was
- * used to set up the operations on the top of
- * netrx_pending_operations, which have since been done.  Check that
- * they didn't give any errors and advance over them.
- */
-static int xenvif_check_gop(struct xenvif *vif, int nr_meta_slots,
-			    struct netrx_pending_operations *npo)
-{
-	struct gnttab_copy     *copy_op;
-	int status = XEN_NETIF_RSP_OKAY;
-	int i;
-
-	for (i = 0; i < nr_meta_slots; i++) {
-		copy_op = npo->copy + npo->copy_cons++;
-		if (copy_op->status != GNTST_okay) {
-			netdev_dbg(vif->dev,
-				   "Bad status %d from copy to DOM%d.\n",
-				   copy_op->status, vif->domid);
-			status = XEN_NETIF_RSP_ERROR;
-		}
-	}
-
-	return status;
-}
-
-static void xenvif_add_frag_responses(struct xenvif_queue *queue, int status,
-				      struct xenvif_rx_meta *meta,
-				      int nr_meta_slots)
-{
-	int i;
-	unsigned long offset;
-
-	/* No fragments used */
-	if (nr_meta_slots <= 1)
-		return;
-
-	nr_meta_slots--;
-
-	for (i = 0; i < nr_meta_slots; i++) {
-		int flags;
-		if (i == nr_meta_slots - 1)
-			flags = 0;
-		else
-			flags = XEN_NETRXF_more_data;
-
-		offset = 0;
-		make_rx_response(queue, meta[i].id, status, offset,
-				 meta[i].size, flags);
-	}
-}
-
 void xenvif_kick_thread(struct xenvif_queue *queue)
 {
 	wake_up(&queue->wq);
 }
 
-static void xenvif_rx_action(struct xenvif_queue *queue)
-{
-	struct xenvif *vif = queue->vif;
-	s8 status;
-	u16 flags;
-	struct xen_netif_rx_response *resp;
-	struct sk_buff_head rxq;
-	struct sk_buff *skb;
-	LIST_HEAD(notify);
-	int ret;
-	unsigned long offset;
-	bool need_to_notify = false;
-
-	struct netrx_pending_operations npo = {
-		.copy  = queue->grant_copy_op,
-		.meta  = queue->meta,
-	};
-
-	skb_queue_head_init(&rxq);
-
-	while (xenvif_rx_ring_slots_available(queue)
-	       && (skb = xenvif_rx_dequeue(queue)) != NULL) {
-		queue->last_rx_time = jiffies;
-
-		XENVIF_RX_CB(skb)->meta_slots_used = xenvif_gop_skb(skb, &npo, queue);
-
-		__skb_queue_tail(&rxq, skb);
-	}
-
-	BUG_ON(npo.meta_prod > ARRAY_SIZE(queue->meta));
-
-	if (!npo.copy_prod)
-		goto done;
-
-	BUG_ON(npo.copy_prod > MAX_GRANT_COPY_OPS);
-	gnttab_batch_copy(queue->grant_copy_op, npo.copy_prod);
-
-	while ((skb = __skb_dequeue(&rxq)) != NULL) {
-		struct xen_netif_extra_info *extra = NULL;
-
-		if ((1 << queue->meta[npo.meta_cons].gso_type) &
-		    vif->gso_prefix_mask) {
-			resp = RING_GET_RESPONSE(&queue->rx,
-						 queue->rx.rsp_prod_pvt++);
-
-			resp->flags = XEN_NETRXF_gso_prefix | XEN_NETRXF_more_data;
-
-			resp->offset = queue->meta[npo.meta_cons].gso_size;
-			resp->id = queue->meta[npo.meta_cons].id;
-			resp->status = XENVIF_RX_CB(skb)->meta_slots_used;
-
-			npo.meta_cons++;
-			XENVIF_RX_CB(skb)->meta_slots_used--;
-		}
-
-
-		queue->stats.tx_bytes += skb->len;
-		queue->stats.tx_packets++;
-
-		status = xenvif_check_gop(vif,
-					  XENVIF_RX_CB(skb)->meta_slots_used,
-					  &npo);
-
-		if (XENVIF_RX_CB(skb)->meta_slots_used == 1)
-			flags = 0;
-		else
-			flags = XEN_NETRXF_more_data;
-
-		if (skb->ip_summed == CHECKSUM_PARTIAL) /* local packet? */
-			flags |= XEN_NETRXF_csum_blank | XEN_NETRXF_data_validated;
-		else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
-			/* remote but checksummed. */
-			flags |= XEN_NETRXF_data_validated;
-
-		offset = 0;
-		resp = make_rx_response(queue, queue->meta[npo.meta_cons].id,
-					status, offset,
-					queue->meta[npo.meta_cons].size,
-					flags);
-
-		if ((1 << queue->meta[npo.meta_cons].gso_type) &
-		    vif->gso_mask) {
-			extra = (struct xen_netif_extra_info *)
-				RING_GET_RESPONSE(&queue->rx,
-						  queue->rx.rsp_prod_pvt++);
-
-			resp->flags |= XEN_NETRXF_extra_info;
-
-			extra->u.gso.type = queue->meta[npo.meta_cons].gso_type;
-			extra->u.gso.size = queue->meta[npo.meta_cons].gso_size;
-			extra->u.gso.pad = 0;
-			extra->u.gso.features = 0;
-
-			extra->type = XEN_NETIF_EXTRA_TYPE_GSO;
-			extra->flags = 0;
-		}
-
-		if (skb->sw_hash) {
-			/* Since the skb got here via xenvif_select_queue()
-			 * we know that the hash has been re-calculated
-			 * according to a configuration set by the frontend
-			 * and therefore we know that it is legitimate to
-			 * pass it to the frontend.
-			 */
-			if (resp->flags & XEN_NETRXF_extra_info)
-				extra->flags |= XEN_NETIF_EXTRA_FLAG_MORE;
-			else
-				resp->flags |= XEN_NETRXF_extra_info;
-
-			extra = (struct xen_netif_extra_info *)
-				RING_GET_RESPONSE(&queue->rx,
-						  queue->rx.rsp_prod_pvt++);
-
-			extra->u.hash.algorithm =
-				XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ;
-
-			if (skb->l4_hash)
-				extra->u.hash.type =
-					skb->protocol == htons(ETH_P_IP) ?
-					_XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP :
-					_XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP;
-			else
-				extra->u.hash.type =
-					skb->protocol == htons(ETH_P_IP) ?
-					_XEN_NETIF_CTRL_HASH_TYPE_IPV4 :
-					_XEN_NETIF_CTRL_HASH_TYPE_IPV6;
-
-			*(uint32_t *)extra->u.hash.value =
-				skb_get_hash_raw(skb);
-
-			extra->type = XEN_NETIF_EXTRA_TYPE_HASH;
-			extra->flags = 0;
-		}
-
-		xenvif_add_frag_responses(queue, status,
-					  queue->meta + npo.meta_cons + 1,
-					  XENVIF_RX_CB(skb)->meta_slots_used);
-
-		RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->rx, ret);
-
-		need_to_notify |= !!ret;
-
-		npo.meta_cons += XENVIF_RX_CB(skb)->meta_slots_used;
-		dev_kfree_skb(skb);
-	}
-
-done:
-	if (need_to_notify)
-		notify_remote_via_irq(queue->rx_irq);
-}
-
 void xenvif_napi_schedule_or_enable_events(struct xenvif_queue *queue)
 {
 	int more_to_do;
@@ -1951,29 +1384,6 @@ static void push_tx_responses(struct xenvif_queue *queue)
 		notify_remote_via_irq(queue->tx_irq);
 }
 
-static struct xen_netif_rx_response *make_rx_response(struct xenvif_queue *queue,
-					     u16      id,
-					     s8       st,
-					     u16      offset,
-					     u16      size,
-					     u16      flags)
-{
-	RING_IDX i = queue->rx.rsp_prod_pvt;
-	struct xen_netif_rx_response *resp;
-
-	resp = RING_GET_RESPONSE(&queue->rx, i);
-	resp->offset     = offset;
-	resp->flags      = flags;
-	resp->id         = id;
-	resp->status     = (s16)size;
-	if (st < 0)
-		resp->status = (s16)st;
-
-	queue->rx.rsp_prod_pvt = ++i;
-
-	return resp;
-}
-
 void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx)
 {
 	int ret;
@@ -2055,170 +1465,6 @@ err:
 	return err;
 }
 
-static void xenvif_queue_carrier_off(struct xenvif_queue *queue)
-{
-	struct xenvif *vif = queue->vif;
-
-	queue->stalled = true;
-
-	/* At least one queue has stalled? Disable the carrier. */
-	spin_lock(&vif->lock);
-	if (vif->stalled_queues++ == 0) {
-		netdev_info(vif->dev, "Guest Rx stalled");
-		netif_carrier_off(vif->dev);
-	}
-	spin_unlock(&vif->lock);
-}
-
-static void xenvif_queue_carrier_on(struct xenvif_queue *queue)
-{
-	struct xenvif *vif = queue->vif;
-
-	queue->last_rx_time = jiffies; /* Reset Rx stall detection. */
-	queue->stalled = false;
-
-	/* All queues are ready? Enable the carrier. */
-	spin_lock(&vif->lock);
-	if (--vif->stalled_queues == 0) {
-		netdev_info(vif->dev, "Guest Rx ready");
-		netif_carrier_on(vif->dev);
-	}
-	spin_unlock(&vif->lock);
-}
-
-static bool xenvif_rx_queue_stalled(struct xenvif_queue *queue)
-{
-	RING_IDX prod, cons;
-
-	prod = queue->rx.sring->req_prod;
-	cons = queue->rx.req_cons;
-
-	return !queue->stalled && prod - cons < 1
-		&& time_after(jiffies,
-			      queue->last_rx_time + queue->vif->stall_timeout);
-}
-
-static bool xenvif_rx_queue_ready(struct xenvif_queue *queue)
-{
-	RING_IDX prod, cons;
-
-	prod = queue->rx.sring->req_prod;
-	cons = queue->rx.req_cons;
-
-	return queue->stalled && prod - cons >= 1;
-}
-
-static bool xenvif_have_rx_work(struct xenvif_queue *queue)
-{
-	return xenvif_rx_ring_slots_available(queue)
-		|| (queue->vif->stall_timeout &&
-		    (xenvif_rx_queue_stalled(queue)
-		     || xenvif_rx_queue_ready(queue)))
-		|| kthread_should_stop()
-		|| queue->vif->disabled;
-}
-
-static long xenvif_rx_queue_timeout(struct xenvif_queue *queue)
-{
-	struct sk_buff *skb;
-	long timeout;
-
-	skb = skb_peek(&queue->rx_queue);
-	if (!skb)
-		return MAX_SCHEDULE_TIMEOUT;
-
-	timeout = XENVIF_RX_CB(skb)->expires - jiffies;
-	return timeout < 0 ? 0 : timeout;
-}
-
-/* Wait until the guest Rx thread has work.
- *
- * The timeout needs to be adjusted based on the current head of the
- * queue (and not just the head at the beginning).  In particular, if
- * the queue is initially empty an infinite timeout is used and this
- * needs to be reduced when a skb is queued.
- *
- * This cannot be done with wait_event_timeout() because it only
- * calculates the timeout once.
- */
-static void xenvif_wait_for_rx_work(struct xenvif_queue *queue)
-{
-	DEFINE_WAIT(wait);
-
-	if (xenvif_have_rx_work(queue))
-		return;
-
-	for (;;) {
-		long ret;
-
-		prepare_to_wait(&queue->wq, &wait, TASK_INTERRUPTIBLE);
-		if (xenvif_have_rx_work(queue))
-			break;
-		ret = schedule_timeout(xenvif_rx_queue_timeout(queue));
-		if (!ret)
-			break;
-	}
-	finish_wait(&queue->wq, &wait);
-}
-
-int xenvif_kthread_guest_rx(void *data)
-{
-	struct xenvif_queue *queue = data;
-	struct xenvif *vif = queue->vif;
-
-	if (!vif->stall_timeout)
-		xenvif_queue_carrier_on(queue);
-
-	for (;;) {
-		xenvif_wait_for_rx_work(queue);
-
-		if (kthread_should_stop())
-			break;
-
-		/* This frontend is found to be rogue, disable it in
-		 * kthread context. Currently this is only set when
-		 * netback finds out frontend sends malformed packet,
-		 * but we cannot disable the interface in softirq
-		 * context so we defer it here, if this thread is
-		 * associated with queue 0.
-		 */
-		if (unlikely(vif->disabled && queue->id == 0)) {
-			xenvif_carrier_off(vif);
-			break;
-		}
-
-		if (!skb_queue_empty(&queue->rx_queue))
-			xenvif_rx_action(queue);
-
-		/* If the guest hasn't provided any Rx slots for a
-		 * while it's probably not responsive, drop the
-		 * carrier so packets are dropped earlier.
-		 */
-		if (vif->stall_timeout) {
-			if (xenvif_rx_queue_stalled(queue))
-				xenvif_queue_carrier_off(queue);
-			else if (xenvif_rx_queue_ready(queue))
-				xenvif_queue_carrier_on(queue);
-		}
-
-		/* Queued packets may have foreign pages from other
-		 * domains.  These cannot be queued indefinitely as
-		 * this would starve guests of grant refs and transmit
-		 * slots.
-		 */
-		xenvif_rx_queue_drop_expired(queue);
-
-		xenvif_rx_queue_maybe_wake(queue);
-
-		cond_resched();
-	}
-
-	/* Bin any remaining skbs */
-	xenvif_rx_queue_purge(queue);
-
-	return 0;
-}
-
 static bool xenvif_dealloc_kthread_should_stop(struct xenvif_queue *queue)
 {
 	/* Dealloc thread must remain running until all inflight
@@ -2359,24 +1605,14 @@ static bool xenvif_ctrl_work_todo(struct xenvif *vif)
 	return 0;
 }
 
-int xenvif_ctrl_kthread(void *data)
+irqreturn_t xenvif_ctrl_irq_fn(int irq, void *data)
 {
 	struct xenvif *vif = data;
 
-	for (;;) {
-		wait_event_interruptible(vif->ctrl_wq,
-					 xenvif_ctrl_work_todo(vif) ||
-					 kthread_should_stop());
-		if (kthread_should_stop())
-			break;
-
-		while (xenvif_ctrl_work_todo(vif))
-			xenvif_ctrl_action(vif);
+	while (xenvif_ctrl_work_todo(vif))
+		xenvif_ctrl_action(vif);
 
-		cond_resched();
-	}
-
-	return 0;
+	return IRQ_HANDLED;
 }
 
 static int __init netback_init(void)
diff --git a/drivers/net/xen-netback/rx.c b/drivers/net/xen-netback/rx.c
new file mode 100644
index 000000000000..8e9ade6ccf18
--- /dev/null
+++ b/drivers/net/xen-netback/rx.c
@@ -0,0 +1,629 @@
+/*
+ * Copyright (c) 2016 Citrix Systems Inc.
+ * Copyright (c) 2002-2005, K A Fraser
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation; or, when distributed
+ * separately from the Linux kernel or incorporated into other
+ * software packages, subject to the following license:
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this source file (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#include "common.h"
+
+#include <linux/kthread.h>
+
+#include <xen/xen.h>
+#include <xen/events.h>
+
+static bool xenvif_rx_ring_slots_available(struct xenvif_queue *queue)
+{
+	RING_IDX prod, cons;
+	struct sk_buff *skb;
+	int needed;
+
+	skb = skb_peek(&queue->rx_queue);
+	if (!skb)
+		return false;
+
+	needed = DIV_ROUND_UP(skb->len, XEN_PAGE_SIZE);
+	if (skb_is_gso(skb))
+		needed++;
+	if (skb->sw_hash)
+		needed++;
+
+	do {
+		prod = queue->rx.sring->req_prod;
+		cons = queue->rx.req_cons;
+
+		if (prod - cons >= needed)
+			return true;
+
+		queue->rx.sring->req_event = prod + 1;
+
+		/* Make sure event is visible before we check prod
+		 * again.
+		 */
+		mb();
+	} while (queue->rx.sring->req_prod != prod);
+
+	return false;
+}
+
+void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&queue->rx_queue.lock, flags);
+
+	__skb_queue_tail(&queue->rx_queue, skb);
+
+	queue->rx_queue_len += skb->len;
+	if (queue->rx_queue_len > queue->rx_queue_max) {
+		struct net_device *dev = queue->vif->dev;
+
+		netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id));
+	}
+
+	spin_unlock_irqrestore(&queue->rx_queue.lock, flags);
+}
+
+static struct sk_buff *xenvif_rx_dequeue(struct xenvif_queue *queue)
+{
+	struct sk_buff *skb;
+
+	spin_lock_irq(&queue->rx_queue.lock);
+
+	skb = __skb_dequeue(&queue->rx_queue);
+	if (skb) {
+		queue->rx_queue_len -= skb->len;
+		if (queue->rx_queue_len < queue->rx_queue_max) {
+			struct netdev_queue *txq;
+
+			txq = netdev_get_tx_queue(queue->vif->dev, queue->id);
+			netif_tx_wake_queue(txq);
+		}
+	}
+
+	spin_unlock_irq(&queue->rx_queue.lock);
+
+	return skb;
+}
+
+static void xenvif_rx_queue_purge(struct xenvif_queue *queue)
+{
+	struct sk_buff *skb;
+
+	while ((skb = xenvif_rx_dequeue(queue)) != NULL)
+		kfree_skb(skb);
+}
+
+static void xenvif_rx_queue_drop_expired(struct xenvif_queue *queue)
+{
+	struct sk_buff *skb;
+
+	for (;;) {
+		skb = skb_peek(&queue->rx_queue);
+		if (!skb)
+			break;
+		if (time_before(jiffies, XENVIF_RX_CB(skb)->expires))
+			break;
+		xenvif_rx_dequeue(queue);
+		kfree_skb(skb);
+	}
+}
+
+static void xenvif_rx_copy_flush(struct xenvif_queue *queue)
+{
+	unsigned int i;
+	int notify;
+
+	gnttab_batch_copy(queue->rx_copy.op, queue->rx_copy.num);
+
+	for (i = 0; i < queue->rx_copy.num; i++) {
+		struct gnttab_copy *op;
+
+		op = &queue->rx_copy.op[i];
+
+		/* If the copy failed, overwrite the status field in
+		 * the corresponding response.
+		 */
+		if (unlikely(op->status != GNTST_okay)) {
+			struct xen_netif_rx_response *rsp;
+
+			rsp = RING_GET_RESPONSE(&queue->rx,
+						queue->rx_copy.idx[i]);
+			rsp->status = op->status;
+		}
+	}
+
+	queue->rx_copy.num = 0;
+
+	/* Push responses for all completed packets. */
+	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&queue->rx, notify);
+	if (notify)
+		notify_remote_via_irq(queue->rx_irq);
+
+	__skb_queue_purge(queue->rx_copy.completed);
+}
+
+static void xenvif_rx_copy_add(struct xenvif_queue *queue,
+			       struct xen_netif_rx_request *req,
+			       unsigned int offset, void *data, size_t len)
+{
+	struct gnttab_copy *op;
+	struct page *page;
+	struct xen_page_foreign *foreign;
+
+	if (queue->rx_copy.num == COPY_BATCH_SIZE)
+		xenvif_rx_copy_flush(queue);
+
+	op = &queue->rx_copy.op[queue->rx_copy.num];
+
+	page = virt_to_page(data);
+
+	op->flags = GNTCOPY_dest_gref;
+
+	foreign = xen_page_foreign(page);
+	if (foreign) {
+		op->source.domid = foreign->domid;
+		op->source.u.ref = foreign->gref;
+		op->flags |= GNTCOPY_source_gref;
+	} else {
+		op->source.u.gmfn = virt_to_gfn(data);
+		op->source.domid  = DOMID_SELF;
+	}
+
+	op->source.offset = xen_offset_in_page(data);
+	op->dest.u.ref    = req->gref;
+	op->dest.domid    = queue->vif->domid;
+	op->dest.offset   = offset;
+	op->len           = len;
+
+	queue->rx_copy.idx[queue->rx_copy.num] = queue->rx.req_cons;
+	queue->rx_copy.num++;
+}
+
+static unsigned int xenvif_gso_type(struct sk_buff *skb)
+{
+	if (skb_is_gso(skb)) {
+		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
+			return XEN_NETIF_GSO_TYPE_TCPV4;
+		else
+			return XEN_NETIF_GSO_TYPE_TCPV6;
+	}
+	return XEN_NETIF_GSO_TYPE_NONE;
+}
+
+struct xenvif_pkt_state {
+	struct sk_buff *skb;
+	size_t remaining_len;
+	struct sk_buff *frag_iter;
+	int frag; /* frag == -1 => frag_iter->head */
+	unsigned int frag_offset;
+	struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
+	unsigned int extra_count;
+	unsigned int slot;
+};
+
+static void xenvif_rx_next_skb(struct xenvif_queue *queue,
+			       struct xenvif_pkt_state *pkt)
+{
+	struct sk_buff *skb;
+	unsigned int gso_type;
+
+	skb = xenvif_rx_dequeue(queue);
+
+	queue->stats.tx_bytes += skb->len;
+	queue->stats.tx_packets++;
+
+	/* Reset packet state. */
+	memset(pkt, 0, sizeof(struct xenvif_pkt_state));
+
+	pkt->skb = skb;
+	pkt->frag_iter = skb;
+	pkt->remaining_len = skb->len;
+	pkt->frag = -1;
+
+	gso_type = xenvif_gso_type(skb);
+	if ((1 << gso_type) & queue->vif->gso_mask) {
+		struct xen_netif_extra_info *extra;
+
+		extra = &pkt->extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
+
+		extra->u.gso.type = gso_type;
+		extra->u.gso.size = skb_shinfo(skb)->gso_size;
+		extra->u.gso.pad = 0;
+		extra->u.gso.features = 0;
+		extra->type = XEN_NETIF_EXTRA_TYPE_GSO;
+		extra->flags = 0;
+
+		pkt->extra_count++;
+	}
+
+	if (skb->sw_hash) {
+		struct xen_netif_extra_info *extra;
+
+		extra = &pkt->extras[XEN_NETIF_EXTRA_TYPE_HASH - 1];
+
+		extra->u.hash.algorithm =
+			XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ;
+
+		if (skb->l4_hash)
+			extra->u.hash.type =
+				skb->protocol == htons(ETH_P_IP) ?
+				_XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP :
+				_XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP;
+		else
+			extra->u.hash.type =
+				skb->protocol == htons(ETH_P_IP) ?
+				_XEN_NETIF_CTRL_HASH_TYPE_IPV4 :
+				_XEN_NETIF_CTRL_HASH_TYPE_IPV6;
+
+		*(uint32_t *)extra->u.hash.value = skb_get_hash_raw(skb);
+
+		extra->type = XEN_NETIF_EXTRA_TYPE_HASH;
+		extra->flags = 0;
+
+		pkt->extra_count++;
+	}
+}
+
+static void xenvif_rx_complete(struct xenvif_queue *queue,
+			       struct xenvif_pkt_state *pkt)
+{
+	/* All responses are ready to be pushed. */
+	queue->rx.rsp_prod_pvt = queue->rx.req_cons;
+
+	__skb_queue_tail(queue->rx_copy.completed, pkt->skb);
+}
+
+static void xenvif_rx_next_frag(struct xenvif_pkt_state *pkt)
+{
+	struct sk_buff *frag_iter = pkt->frag_iter;
+	unsigned int nr_frags = skb_shinfo(frag_iter)->nr_frags;
+
+	pkt->frag++;
+	pkt->frag_offset = 0;
+
+	if (pkt->frag >= nr_frags) {
+		if (frag_iter == pkt->skb)
+			pkt->frag_iter = skb_shinfo(frag_iter)->frag_list;
+		else
+			pkt->frag_iter = frag_iter->next;
+
+		pkt->frag = -1;
+	}
+}
+
+static void xenvif_rx_next_chunk(struct xenvif_queue *queue,
+				 struct xenvif_pkt_state *pkt,
+				 unsigned int offset, void **data,
+				 size_t *len)
+{
+	struct sk_buff *frag_iter = pkt->frag_iter;
+	void *frag_data;
+	size_t frag_len, chunk_len;
+
+	BUG_ON(!frag_iter);
+
+	if (pkt->frag == -1) {
+		frag_data = frag_iter->data;
+		frag_len = skb_headlen(frag_iter);
+	} else {
+		skb_frag_t *frag = &skb_shinfo(frag_iter)->frags[pkt->frag];
+
+		frag_data = skb_frag_address(frag);
+		frag_len = skb_frag_size(frag);
+	}
+
+	frag_data += pkt->frag_offset;
+	frag_len -= pkt->frag_offset;
+
+	chunk_len = min(frag_len, XEN_PAGE_SIZE - offset);
+	chunk_len = min(chunk_len,
+			XEN_PAGE_SIZE -	xen_offset_in_page(frag_data));
+
+	pkt->frag_offset += chunk_len;
+
+	/* Advance to next frag? */
+	if (frag_len == chunk_len)
+		xenvif_rx_next_frag(pkt);
+
+	*data = frag_data;
+	*len = chunk_len;
+}
+
+static void xenvif_rx_data_slot(struct xenvif_queue *queue,
+				struct xenvif_pkt_state *pkt,
+				struct xen_netif_rx_request *req,
+				struct xen_netif_rx_response *rsp)
+{
+	unsigned int offset = 0;
+	unsigned int flags;
+
+	do {
+		size_t len;
+		void *data;
+
+		xenvif_rx_next_chunk(queue, pkt, offset, &data, &len);
+		xenvif_rx_copy_add(queue, req, offset, data, len);
+
+		offset += len;
+		pkt->remaining_len -= len;
+
+	} while (offset < XEN_PAGE_SIZE && pkt->remaining_len > 0);
+
+	if (pkt->remaining_len > 0)
+		flags = XEN_NETRXF_more_data;
+	else
+		flags = 0;
+
+	if (pkt->slot == 0) {
+		struct sk_buff *skb = pkt->skb;
+
+		if (skb->ip_summed == CHECKSUM_PARTIAL)
+			flags |= XEN_NETRXF_csum_blank |
+				 XEN_NETRXF_data_validated;
+		else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
+			flags |= XEN_NETRXF_data_validated;
+
+		if (pkt->extra_count != 0)
+			flags |= XEN_NETRXF_extra_info;
+	}
+
+	rsp->offset = 0;
+	rsp->flags = flags;
+	rsp->id = req->id;
+	rsp->status = (s16)offset;
+}
+
+static void xenvif_rx_extra_slot(struct xenvif_queue *queue,
+				 struct xenvif_pkt_state *pkt,
+				 struct xen_netif_rx_request *req,
+				 struct xen_netif_rx_response *rsp)
+{
+	struct xen_netif_extra_info *extra = (void *)rsp;
+	unsigned int i;
+
+	pkt->extra_count--;
+
+	for (i = 0; i < ARRAY_SIZE(pkt->extras); i++) {
+		if (pkt->extras[i].type) {
+			*extra = pkt->extras[i];
+
+			if (pkt->extra_count != 0)
+				extra->flags |= XEN_NETIF_EXTRA_FLAG_MORE;
+
+			pkt->extras[i].type = 0;
+			return;
+		}
+	}
+	BUG();
+}
+
+void xenvif_rx_skb(struct xenvif_queue *queue)
+{
+	struct xenvif_pkt_state pkt;
+
+	xenvif_rx_next_skb(queue, &pkt);
+
+	do {
+		struct xen_netif_rx_request *req;
+		struct xen_netif_rx_response *rsp;
+
+		req = RING_GET_REQUEST(&queue->rx, queue->rx.req_cons);
+		rsp = RING_GET_RESPONSE(&queue->rx, queue->rx.req_cons);
+
+		/* Extras must go after the first data slot */
+		if (pkt.slot != 0 && pkt.extra_count != 0)
+			xenvif_rx_extra_slot(queue, &pkt, req, rsp);
+		else
+			xenvif_rx_data_slot(queue, &pkt, req, rsp);
+
+		queue->rx.req_cons++;
+		pkt.slot++;
+	} while (pkt.remaining_len > 0 || pkt.extra_count != 0);
+
+	xenvif_rx_complete(queue, &pkt);
+}
+
+#define RX_BATCH_SIZE 64
+
+void xenvif_rx_action(struct xenvif_queue *queue)
+{
+	struct sk_buff_head completed_skbs;
+	unsigned int work_done = 0;
+
+	__skb_queue_head_init(&completed_skbs);
+	queue->rx_copy.completed = &completed_skbs;
+
+	while (xenvif_rx_ring_slots_available(queue) &&
+	       work_done < RX_BATCH_SIZE) {
+		xenvif_rx_skb(queue);
+		work_done++;
+	}
+
+	/* Flush any pending copies and complete all skbs. */
+	xenvif_rx_copy_flush(queue);
+}
+
+static bool xenvif_rx_queue_stalled(struct xenvif_queue *queue)
+{
+	RING_IDX prod, cons;
+
+	prod = queue->rx.sring->req_prod;
+	cons = queue->rx.req_cons;
+
+	return !queue->stalled &&
+		prod - cons < 1 &&
+		time_after(jiffies,
+			   queue->last_rx_time + queue->vif->stall_timeout);
+}
+
+static bool xenvif_rx_queue_ready(struct xenvif_queue *queue)
+{
+	RING_IDX prod, cons;
+
+	prod = queue->rx.sring->req_prod;
+	cons = queue->rx.req_cons;
+
+	return queue->stalled && prod - cons >= 1;
+}
+
+static bool xenvif_have_rx_work(struct xenvif_queue *queue)
+{
+	return xenvif_rx_ring_slots_available(queue) ||
+		(queue->vif->stall_timeout &&
+		 (xenvif_rx_queue_stalled(queue) ||
+		  xenvif_rx_queue_ready(queue))) ||
+		kthread_should_stop() ||
+		queue->vif->disabled;
+}
+
+static long xenvif_rx_queue_timeout(struct xenvif_queue *queue)
+{
+	struct sk_buff *skb;
+	long timeout;
+
+	skb = skb_peek(&queue->rx_queue);
+	if (!skb)
+		return MAX_SCHEDULE_TIMEOUT;
+
+	timeout = XENVIF_RX_CB(skb)->expires - jiffies;
+	return timeout < 0 ? 0 : timeout;
+}
+
+/* Wait until the guest Rx thread has work.
+ *
+ * The timeout needs to be adjusted based on the current head of the
+ * queue (and not just the head at the beginning).  In particular, if
+ * the queue is initially empty an infinite timeout is used and this
+ * needs to be reduced when a skb is queued.
+ *
+ * This cannot be done with wait_event_timeout() because it only
+ * calculates the timeout once.
+ */
+static void xenvif_wait_for_rx_work(struct xenvif_queue *queue)
+{
+	DEFINE_WAIT(wait);
+
+	if (xenvif_have_rx_work(queue))
+		return;
+
+	for (;;) {
+		long ret;
+
+		prepare_to_wait(&queue->wq, &wait, TASK_INTERRUPTIBLE);
+		if (xenvif_have_rx_work(queue))
+			break;
+		ret = schedule_timeout(xenvif_rx_queue_timeout(queue));
+		if (!ret)
+			break;
+	}
+	finish_wait(&queue->wq, &wait);
+}
+
+static void xenvif_queue_carrier_off(struct xenvif_queue *queue)
+{
+	struct xenvif *vif = queue->vif;
+
+	queue->stalled = true;
+
+	/* At least one queue has stalled? Disable the carrier. */
+	spin_lock(&vif->lock);
+	if (vif->stalled_queues++ == 0) {
+		netdev_info(vif->dev, "Guest Rx stalled");
+		netif_carrier_off(vif->dev);
+	}
+	spin_unlock(&vif->lock);
+}
+
+static void xenvif_queue_carrier_on(struct xenvif_queue *queue)
+{
+	struct xenvif *vif = queue->vif;
+
+	queue->last_rx_time = jiffies; /* Reset Rx stall detection. */
+	queue->stalled = false;
+
+	/* All queues are ready? Enable the carrier. */
+	spin_lock(&vif->lock);
+	if (--vif->stalled_queues == 0) {
+		netdev_info(vif->dev, "Guest Rx ready");
+		netif_carrier_on(vif->dev);
+	}
+	spin_unlock(&vif->lock);
+}
+
+int xenvif_kthread_guest_rx(void *data)
+{
+	struct xenvif_queue *queue = data;
+	struct xenvif *vif = queue->vif;
+
+	if (!vif->stall_timeout)
+		xenvif_queue_carrier_on(queue);
+
+	for (;;) {
+		xenvif_wait_for_rx_work(queue);
+
+		if (kthread_should_stop())
+			break;
+
+		/* This frontend is found to be rogue, disable it in
+		 * kthread context. Currently this is only set when
+		 * netback finds out frontend sends malformed packet,
+		 * but we cannot disable the interface in softirq
+		 * context so we defer it here, if this thread is
+		 * associated with queue 0.
+		 */
+		if (unlikely(vif->disabled && queue->id == 0)) {
+			xenvif_carrier_off(vif);
+			break;
+		}
+
+		if (!skb_queue_empty(&queue->rx_queue))
+			xenvif_rx_action(queue);
+
+		/* If the guest hasn't provided any Rx slots for a
+		 * while it's probably not responsive, drop the
+		 * carrier so packets are dropped earlier.
+		 */
+		if (vif->stall_timeout) {
+			if (xenvif_rx_queue_stalled(queue))
+				xenvif_queue_carrier_off(queue);
+			else if (xenvif_rx_queue_ready(queue))
+				xenvif_queue_carrier_on(queue);
+		}
+
+		/* Queued packets may have foreign pages from other
+		 * domains.  These cannot be queued indefinitely as
+		 * this would starve guests of grant refs and transmit
+		 * slots.
+		 */
+		xenvif_rx_queue_drop_expired(queue);
+
+		cond_resched();
+	}
+
+	/* Bin any remaining skbs */
+	xenvif_rx_queue_purge(queue);
+
+	return 0;
+}
diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c
index 6a31f2610c23..7056404e3cb8 100644
--- a/drivers/net/xen-netback/xenbus.c
+++ b/drivers/net/xen-netback/xenbus.c
@@ -271,6 +271,11 @@ static int netback_probe(struct xenbus_device *dev,
 	be->dev = dev;
 	dev_set_drvdata(&dev->dev, be);
 
+	be->state = XenbusStateInitialising;
+	err = xenbus_switch_state(dev, XenbusStateInitialising);
+	if (err)
+		goto fail;
+
 	sg = 1;
 
 	do {
@@ -383,11 +388,6 @@ static int netback_probe(struct xenbus_device *dev,
 
 	be->hotplug_script = script;
 
-	err = xenbus_switch_state(dev, XenbusStateInitWait);
-	if (err)
-		goto fail;
-
-	be->state = XenbusStateInitWait;
 
 	/* This kicks hotplug scripts, so do it immediately. */
 	err = backend_create_xenvif(be);
@@ -492,20 +492,20 @@ static inline void backend_switch_state(struct backend_info *be,
 
 /* Handle backend state transitions:
  *
- * The backend state starts in InitWait and the following transitions are
+ * The backend state starts in Initialising and the following transitions are
  * allowed.
  *
- * InitWait -> Connected
- *
- *    ^    \         |
- *    |     \        |
- *    |      \       |
- *    |       \      |
- *    |        \     |
- *    |         \    |
- *    |          V   V
+ * Initialising -> InitWait -> Connected
+ *          \
+ *           \        ^    \         |
+ *            \       |     \        |
+ *             \      |      \       |
+ *              \     |       \      |
+ *               \    |        \     |
+ *                \   |         \    |
+ *                 V  |          V   V
  *
- *  Closed  <-> Closing
+ *                  Closed  <-> Closing
  *
  * The state argument specifies the eventual state of the backend and the
  * function transitions to that state via the shortest path.
@@ -515,6 +515,20 @@ static void set_backend_state(struct backend_info *be,
 {
 	while (be->state != state) {
 		switch (be->state) {
+		case XenbusStateInitialising:
+			switch (state) {
+			case XenbusStateInitWait:
+			case XenbusStateConnected:
+			case XenbusStateClosing:
+				backend_switch_state(be, XenbusStateInitWait);
+				break;
+			case XenbusStateClosed:
+				backend_switch_state(be, XenbusStateClosed);
+				break;
+			default:
+				BUG();
+			}
+			break;
 		case XenbusStateClosed:
 			switch (state) {
 			case XenbusStateInitWait:
@@ -1121,7 +1135,6 @@ static int read_xenbus_vif_flags(struct backend_info *be)
 	vif->can_sg = !!val;
 
 	vif->gso_mask = 0;
-	vif->gso_prefix_mask = 0;
 
 	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-gso-tcpv4",
 			 "%d", &val) < 0)
@@ -1129,32 +1142,12 @@ static int read_xenbus_vif_flags(struct backend_info *be)
 	if (val)
 		vif->gso_mask |= GSO_BIT(TCPV4);
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-gso-tcpv4-prefix",
-			 "%d", &val) < 0)
-		val = 0;
-	if (val)
-		vif->gso_prefix_mask |= GSO_BIT(TCPV4);
-
 	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-gso-tcpv6",
 			 "%d", &val) < 0)
 		val = 0;
 	if (val)
 		vif->gso_mask |= GSO_BIT(TCPV6);
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-gso-tcpv6-prefix",
-			 "%d", &val) < 0)
-		val = 0;
-	if (val)
-		vif->gso_prefix_mask |= GSO_BIT(TCPV6);
-
-	if (vif->gso_mask & vif->gso_prefix_mask) {
-		xenbus_dev_fatal(dev, err,
-				 "%s: gso and gso prefix flags are not "
-				 "mutually exclusive",
-				 dev->otherend);
-		return -EOPNOTSUPP;
-	}
-
 	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-no-csum-offload",
 			 "%d", &val) < 0)
 		val = 0;
diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index 96ccd4e943db..e17879dd5d5a 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -565,6 +565,7 @@ static int xennet_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	struct netfront_queue *queue = NULL;
 	unsigned int num_queues = dev->real_num_tx_queues;
 	u16 queue_index;
+	struct sk_buff *nskb;
 
 	/* Drop the packet if no queues are set up */
 	if (num_queues < 1)
@@ -593,6 +594,20 @@ static int xennet_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	page = virt_to_page(skb->data);
 	offset = offset_in_page(skb->data);
+
+	/* The first req should be at least ETH_HLEN size or the packet will be
+	 * dropped by netback.
+	 */
+	if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) {
+		nskb = skb_copy(skb, GFP_ATOMIC);
+		if (!nskb)
+			goto drop;
+		dev_kfree_skb_any(skb);
+		skb = nskb;
+		page = virt_to_page(skb->data);
+		offset = offset_in_page(skb->data);
+	}
+
 	len = skb_headlen(skb);
 
 	spin_lock_irqsave(&queue->tx_lock, flags);
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index d5c5894f252e..8601c10acf74 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -102,13 +102,16 @@ struct ntb_queue_entry {
 	void *buf;
 	unsigned int len;
 	unsigned int flags;
+	int retries;
+	int errors;
+	unsigned int tx_index;
+	unsigned int rx_index;
 
 	struct ntb_transport_qp *qp;
 	union {
 		struct ntb_payload_header __iomem *tx_hdr;
 		struct ntb_payload_header *rx_hdr;
 	};
-	unsigned int index;
 };
 
 struct ntb_rx_info {
@@ -259,6 +262,12 @@ enum {
 static void ntb_transport_rxc_db(unsigned long data);
 static const struct ntb_ctx_ops ntb_transport_ops;
 static struct ntb_client ntb_transport_client;
+static int ntb_async_tx_submit(struct ntb_transport_qp *qp,
+			       struct ntb_queue_entry *entry);
+static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset);
+static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset);
+static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset);
+
 
 static int ntb_transport_bus_match(struct device *dev,
 				   struct device_driver *drv)
@@ -1229,7 +1238,7 @@ static void ntb_complete_rxc(struct ntb_transport_qp *qp)
 			break;
 
 		entry->rx_hdr->flags = 0;
-		iowrite32(entry->index, &qp->rx_info->entry);
+		iowrite32(entry->rx_index, &qp->rx_info->entry);
 
 		cb_data = entry->cb_data;
 		len = entry->len;
@@ -1247,10 +1256,36 @@ static void ntb_complete_rxc(struct ntb_transport_qp *qp)
 	spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags);
 }
 
-static void ntb_rx_copy_callback(void *data)
+static void ntb_rx_copy_callback(void *data,
+				 const struct dmaengine_result *res)
 {
 	struct ntb_queue_entry *entry = data;
 
+	/* we need to check DMA results if we are using DMA */
+	if (res) {
+		enum dmaengine_tx_result dma_err = res->result;
+
+		switch (dma_err) {
+		case DMA_TRANS_READ_FAILED:
+		case DMA_TRANS_WRITE_FAILED:
+			entry->errors++;
+		case DMA_TRANS_ABORTED:
+		{
+			struct ntb_transport_qp *qp = entry->qp;
+			void *offset = qp->rx_buff + qp->rx_max_frame *
+					qp->rx_index;
+
+			ntb_memcpy_rx(entry, offset);
+			qp->rx_memcpy++;
+			return;
+		}
+
+		case DMA_TRANS_NOERROR:
+		default:
+			break;
+		}
+	}
+
 	entry->flags |= DESC_DONE_FLAG;
 
 	ntb_complete_rxc(entry->qp);
@@ -1266,10 +1301,10 @@ static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset)
 	/* Ensure that the data is fully copied out before clearing the flag */
 	wmb();
 
-	ntb_rx_copy_callback(entry);
+	ntb_rx_copy_callback(entry, NULL);
 }
 
-static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
+static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset)
 {
 	struct dma_async_tx_descriptor *txd;
 	struct ntb_transport_qp *qp = entry->qp;
@@ -1282,13 +1317,6 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
 	int retries = 0;
 
 	len = entry->len;
-
-	if (!chan)
-		goto err;
-
-	if (len < copy_bytes)
-		goto err;
-
 	device = chan->device;
 	pay_off = (size_t)offset & ~PAGE_MASK;
 	buff_off = (size_t)buf & ~PAGE_MASK;
@@ -1316,7 +1344,8 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
 	unmap->from_cnt = 1;
 
 	for (retries = 0; retries < DMA_RETRIES; retries++) {
-		txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
+		txd = device->device_prep_dma_memcpy(chan,
+						     unmap->addr[1],
 						     unmap->addr[0], len,
 						     DMA_PREP_INTERRUPT);
 		if (txd)
@@ -1331,7 +1360,7 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
 		goto err_get_unmap;
 	}
 
-	txd->callback = ntb_rx_copy_callback;
+	txd->callback_result = ntb_rx_copy_callback;
 	txd->callback_param = entry;
 	dma_set_unmap(txd, unmap);
 
@@ -1345,13 +1374,38 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
 
 	qp->rx_async++;
 
-	return;
+	return 0;
 
 err_set_unmap:
 	dmaengine_unmap_put(unmap);
 err_get_unmap:
 	dmaengine_unmap_put(unmap);
 err:
+	return -ENXIO;
+}
+
+static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
+{
+	struct ntb_transport_qp *qp = entry->qp;
+	struct dma_chan *chan = qp->rx_dma_chan;
+	int res;
+
+	if (!chan)
+		goto err;
+
+	if (entry->len < copy_bytes)
+		goto err;
+
+	res = ntb_async_rx_submit(entry, offset);
+	if (res < 0)
+		goto err;
+
+	if (!entry->retries)
+		qp->rx_async++;
+
+	return;
+
+err:
 	ntb_memcpy_rx(entry, offset);
 	qp->rx_memcpy++;
 }
@@ -1397,7 +1451,7 @@ static int ntb_process_rxc(struct ntb_transport_qp *qp)
 	}
 
 	entry->rx_hdr = hdr;
-	entry->index = qp->rx_index;
+	entry->rx_index = qp->rx_index;
 
 	if (hdr->len > entry->len) {
 		dev_dbg(&qp->ndev->pdev->dev,
@@ -1467,12 +1521,39 @@ static void ntb_transport_rxc_db(unsigned long data)
 	}
 }
 
-static void ntb_tx_copy_callback(void *data)
+static void ntb_tx_copy_callback(void *data,
+				 const struct dmaengine_result *res)
 {
 	struct ntb_queue_entry *entry = data;
 	struct ntb_transport_qp *qp = entry->qp;
 	struct ntb_payload_header __iomem *hdr = entry->tx_hdr;
 
+	/* we need to check DMA results if we are using DMA */
+	if (res) {
+		enum dmaengine_tx_result dma_err = res->result;
+
+		switch (dma_err) {
+		case DMA_TRANS_READ_FAILED:
+		case DMA_TRANS_WRITE_FAILED:
+			entry->errors++;
+		case DMA_TRANS_ABORTED:
+		{
+			void __iomem *offset =
+				qp->tx_mw + qp->tx_max_frame *
+				entry->tx_index;
+
+			/* resubmit via CPU */
+			ntb_memcpy_tx(entry, offset);
+			qp->tx_memcpy++;
+			return;
+		}
+
+		case DMA_TRANS_NOERROR:
+		default:
+			break;
+		}
+	}
+
 	iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags);
 
 	ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num));
@@ -1507,40 +1588,25 @@ static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset)
 	/* Ensure that the data is fully copied out before setting the flags */
 	wmb();
 
-	ntb_tx_copy_callback(entry);
+	ntb_tx_copy_callback(entry, NULL);
 }
 
-static void ntb_async_tx(struct ntb_transport_qp *qp,
-			 struct ntb_queue_entry *entry)
+static int ntb_async_tx_submit(struct ntb_transport_qp *qp,
+			       struct ntb_queue_entry *entry)
 {
-	struct ntb_payload_header __iomem *hdr;
 	struct dma_async_tx_descriptor *txd;
 	struct dma_chan *chan = qp->tx_dma_chan;
 	struct dma_device *device;
+	size_t len = entry->len;
+	void *buf = entry->buf;
 	size_t dest_off, buff_off;
 	struct dmaengine_unmap_data *unmap;
 	dma_addr_t dest;
 	dma_cookie_t cookie;
-	void __iomem *offset;
-	size_t len = entry->len;
-	void *buf = entry->buf;
 	int retries = 0;
 
-	offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index;
-	hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
-	entry->tx_hdr = hdr;
-
-	iowrite32(entry->len, &hdr->len);
-	iowrite32((u32)qp->tx_pkts, &hdr->ver);
-
-	if (!chan)
-		goto err;
-
-	if (len < copy_bytes)
-		goto err;
-
 	device = chan->device;
-	dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index;
+	dest = qp->tx_mw_phys + qp->tx_max_frame * entry->tx_index;
 	buff_off = (size_t)buf & ~PAGE_MASK;
 	dest_off = (size_t)dest & ~PAGE_MASK;
 
@@ -1560,8 +1626,9 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
 	unmap->to_cnt = 1;
 
 	for (retries = 0; retries < DMA_RETRIES; retries++) {
-		txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0],
-						     len, DMA_PREP_INTERRUPT);
+		txd = device->device_prep_dma_memcpy(chan, dest,
+						     unmap->addr[0], len,
+						     DMA_PREP_INTERRUPT);
 		if (txd)
 			break;
 
@@ -1574,7 +1641,7 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
 		goto err_get_unmap;
 	}
 
-	txd->callback = ntb_tx_copy_callback;
+	txd->callback_result = ntb_tx_copy_callback;
 	txd->callback_param = entry;
 	dma_set_unmap(txd, unmap);
 
@@ -1585,14 +1652,48 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
 	dmaengine_unmap_put(unmap);
 
 	dma_async_issue_pending(chan);
-	qp->tx_async++;
 
-	return;
+	return 0;
 err_set_unmap:
 	dmaengine_unmap_put(unmap);
 err_get_unmap:
 	dmaengine_unmap_put(unmap);
 err:
+	return -ENXIO;
+}
+
+static void ntb_async_tx(struct ntb_transport_qp *qp,
+			 struct ntb_queue_entry *entry)
+{
+	struct ntb_payload_header __iomem *hdr;
+	struct dma_chan *chan = qp->tx_dma_chan;
+	void __iomem *offset;
+	int res;
+
+	entry->tx_index = qp->tx_index;
+	offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index;
+	hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
+	entry->tx_hdr = hdr;
+
+	iowrite32(entry->len, &hdr->len);
+	iowrite32((u32)qp->tx_pkts, &hdr->ver);
+
+	if (!chan)
+		goto err;
+
+	if (entry->len < copy_bytes)
+		goto err;
+
+	res = ntb_async_tx_submit(qp, entry);
+	if (res < 0)
+		goto err;
+
+	if (!entry->retries)
+		qp->tx_async++;
+
+	return;
+
+err:
 	ntb_memcpy_tx(entry, offset);
 	qp->tx_memcpy++;
 }
@@ -1928,6 +2029,9 @@ int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
 	entry->buf = data;
 	entry->len = len;
 	entry->flags = 0;
+	entry->retries = 0;
+	entry->errors = 0;
+	entry->rx_index = 0;
 
 	ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
 
@@ -1970,6 +2074,9 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
 	entry->buf = data;
 	entry->len = len;
 	entry->flags = 0;
+	entry->errors = 0;
+	entry->retries = 0;
+	entry->tx_index = 0;
 
 	rc = ntb_process_tx(qp, entry);
 	if (rc)
diff --git a/drivers/nvdimm/Kconfig b/drivers/nvdimm/Kconfig
index 124c2432ac9c..8b2b740d6679 100644
--- a/drivers/nvdimm/Kconfig
+++ b/drivers/nvdimm/Kconfig
@@ -89,7 +89,7 @@ config NVDIMM_PFN
 	  Select Y if unsure
 
 config NVDIMM_DAX
-	bool "NVDIMM DAX: Raw access to persistent memory"
+	tristate "NVDIMM DAX: Raw access to persistent memory"
 	default LIBNVDIMM
 	depends on NVDIMM_PFN
 	help
diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
index 88e91666f145..368795aad5c9 100644
--- a/drivers/nvdimm/btt.c
+++ b/drivers/nvdimm/btt.c
@@ -1269,6 +1269,7 @@ static int btt_blk_init(struct btt *btt)
 		}
 	}
 	set_capacity(btt->btt_disk, btt->nlba * btt->sector_size >> 9);
+	btt->nd_btt->size = btt->nlba * (u64)btt->sector_size;
 	revalidate_disk(btt->btt_disk);
 
 	return 0;
diff --git a/drivers/nvdimm/btt_devs.c b/drivers/nvdimm/btt_devs.c
index 3fa7919f94a8..97dd2925ed6e 100644
--- a/drivers/nvdimm/btt_devs.c
+++ b/drivers/nvdimm/btt_devs.c
@@ -140,10 +140,30 @@ static ssize_t namespace_store(struct device *dev,
 }
 static DEVICE_ATTR_RW(namespace);
 
+static ssize_t size_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct nd_btt *nd_btt = to_nd_btt(dev);
+	ssize_t rc;
+
+	device_lock(dev);
+	if (dev->driver)
+		rc = sprintf(buf, "%llu\n", nd_btt->size);
+	else {
+		/* no size to convey if the btt instance is disabled */
+		rc = -ENXIO;
+	}
+	device_unlock(dev);
+
+	return rc;
+}
+static DEVICE_ATTR_RO(size);
+
 static struct attribute *nd_btt_attributes[] = {
 	&dev_attr_sector_size.attr,
 	&dev_attr_namespace.attr,
 	&dev_attr_uuid.attr,
+	&dev_attr_size.attr,
 	NULL,
 };
 
diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c
index 458daf927336..a8b6949a8778 100644
--- a/drivers/nvdimm/bus.c
+++ b/drivers/nvdimm/bus.c
@@ -185,8 +185,12 @@ long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
 		return -ENXIO;
 
 	nd_desc = nvdimm_bus->nd_desc;
+	/*
+	 * if ndctl does not exist, it's PMEM_LEGACY and
+	 * we want to just pretend everything is handled.
+	 */
 	if (!nd_desc->ndctl)
-		return -ENXIO;
+		return len;
 
 	memset(&ars_cap, 0, sizeof(ars_cap));
 	ars_cap.address = phys;
@@ -213,6 +217,8 @@ long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
 		return rc;
 	if (cmd_rc < 0)
 		return cmd_rc;
+
+	nvdimm_clear_from_poison_list(nvdimm_bus, phys, len);
 	return clear_err.cleared;
 }
 EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
diff --git a/drivers/nvdimm/core.c b/drivers/nvdimm/core.c
index 715583f69d28..7ceba08774b6 100644
--- a/drivers/nvdimm/core.c
+++ b/drivers/nvdimm/core.c
@@ -99,8 +99,11 @@ static struct nvdimm_map *alloc_nvdimm_map(struct device *dev,
 	nvdimm_map->size = size;
 	kref_init(&nvdimm_map->kref);
 
-	if (!request_mem_region(offset, size, dev_name(&nvdimm_bus->dev)))
+	if (!request_mem_region(offset, size, dev_name(&nvdimm_bus->dev))) {
+		dev_err(&nvdimm_bus->dev, "failed to request %pa + %zd for %s\n",
+				&offset, size, dev_name(dev));
 		goto err_request_region;
+	}
 
 	if (flags)
 		nvdimm_map->mem = memremap(offset, size, flags);
@@ -171,6 +174,9 @@ void *devm_nvdimm_memremap(struct device *dev, resource_size_t offset,
 		kref_get(&nvdimm_map->kref);
 	nvdimm_bus_unlock(dev);
 
+	if (!nvdimm_map)
+		return NULL;
+
 	if (devm_add_action_or_reset(dev, nvdimm_map_put, nvdimm_map))
 		return NULL;
 
@@ -541,11 +547,12 @@ void nvdimm_badblocks_populate(struct nd_region *nd_region,
 }
 EXPORT_SYMBOL_GPL(nvdimm_badblocks_populate);
 
-static int add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
+static int add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length,
+			gfp_t flags)
 {
 	struct nd_poison *pl;
 
-	pl = kzalloc(sizeof(*pl), GFP_KERNEL);
+	pl = kzalloc(sizeof(*pl), flags);
 	if (!pl)
 		return -ENOMEM;
 
@@ -561,7 +568,7 @@ static int bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 	struct nd_poison *pl;
 
 	if (list_empty(&nvdimm_bus->poison_list))
-		return add_poison(nvdimm_bus, addr, length);
+		return add_poison(nvdimm_bus, addr, length, GFP_KERNEL);
 
 	/*
 	 * There is a chance this is a duplicate, check for those first.
@@ -581,7 +588,7 @@ static int bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 	 * as any overlapping ranges will get resolved when the list is consumed
 	 * and converted to badblocks
 	 */
-	return add_poison(nvdimm_bus, addr, length);
+	return add_poison(nvdimm_bus, addr, length, GFP_KERNEL);
 }
 
 int nvdimm_bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
@@ -596,6 +603,70 @@ int nvdimm_bus_add_poison(struct nvdimm_bus *nvdimm_bus, u64 addr, u64 length)
 }
 EXPORT_SYMBOL_GPL(nvdimm_bus_add_poison);
 
+void nvdimm_clear_from_poison_list(struct nvdimm_bus *nvdimm_bus,
+		phys_addr_t start, unsigned int len)
+{
+	struct list_head *poison_list = &nvdimm_bus->poison_list;
+	u64 clr_end = start + len - 1;
+	struct nd_poison *pl, *next;
+
+	nvdimm_bus_lock(&nvdimm_bus->dev);
+	WARN_ON_ONCE(list_empty(poison_list));
+
+	/*
+	 * [start, clr_end] is the poison interval being cleared.
+	 * [pl->start, pl_end] is the poison_list entry we're comparing
+	 * the above interval against. The poison list entry may need
+	 * to be modified (update either start or length), deleted, or
+	 * split into two based on the overlap characteristics
+	 */
+
+	list_for_each_entry_safe(pl, next, poison_list, list) {
+		u64 pl_end = pl->start + pl->length - 1;
+
+		/* Skip intervals with no intersection */
+		if (pl_end < start)
+			continue;
+		if (pl->start >  clr_end)
+			continue;
+		/* Delete completely overlapped poison entries */
+		if ((pl->start >= start) && (pl_end <= clr_end)) {
+			list_del(&pl->list);
+			kfree(pl);
+			continue;
+		}
+		/* Adjust start point of partially cleared entries */
+		if ((start <= pl->start) && (clr_end > pl->start)) {
+			pl->length -= clr_end - pl->start + 1;
+			pl->start = clr_end + 1;
+			continue;
+		}
+		/* Adjust pl->length for partial clearing at the tail end */
+		if ((pl->start < start) && (pl_end <= clr_end)) {
+			/* pl->start remains the same */
+			pl->length = start - pl->start;
+			continue;
+		}
+		/*
+		 * If clearing in the middle of an entry, we split it into
+		 * two by modifying the current entry to represent one half of
+		 * the split, and adding a new entry for the second half.
+		 */
+		if ((pl->start < start) && (pl_end > clr_end)) {
+			u64 new_start = clr_end + 1;
+			u64 new_len = pl_end - new_start + 1;
+
+			/* Add new entry covering the right half */
+			add_poison(nvdimm_bus, new_start, new_len, GFP_NOIO);
+			/* Adjust this entry to cover the left half */
+			pl->length = start - pl->start;
+			continue;
+		}
+	}
+	nvdimm_bus_unlock(&nvdimm_bus->dev);
+}
+EXPORT_SYMBOL_GPL(nvdimm_clear_from_poison_list);
+
 #ifdef CONFIG_BLK_DEV_INTEGRITY
 int nd_integrity_init(struct gendisk *disk, unsigned long meta_size)
 {
diff --git a/drivers/nvdimm/dimm.c b/drivers/nvdimm/dimm.c
index 71d12bb67339..619834e144d1 100644
--- a/drivers/nvdimm/dimm.c
+++ b/drivers/nvdimm/dimm.c
@@ -26,6 +26,14 @@ static int nvdimm_probe(struct device *dev)
 	struct nvdimm_drvdata *ndd;
 	int rc;
 
+	rc = nvdimm_check_config_data(dev);
+	if (rc) {
+		/* not required for non-aliased nvdimm, ex. NVDIMM-N */
+		if (rc == -ENOTTY)
+			rc = 0;
+		return rc;
+	}
+
 	ndd = kzalloc(sizeof(*ndd), GFP_KERNEL);
 	if (!ndd)
 		return -ENOMEM;
@@ -72,6 +80,9 @@ static int nvdimm_remove(struct device *dev)
 {
 	struct nvdimm_drvdata *ndd = dev_get_drvdata(dev);
 
+	if (!ndd)
+		return 0;
+
 	nvdimm_bus_lock(dev);
 	dev_set_drvdata(dev, NULL);
 	nvdimm_bus_unlock(dev);
diff --git a/drivers/nvdimm/dimm_devs.c b/drivers/nvdimm/dimm_devs.c
index d9bba5edd8dc..d614493ad5ac 100644
--- a/drivers/nvdimm/dimm_devs.c
+++ b/drivers/nvdimm/dimm_devs.c
@@ -28,28 +28,30 @@ static DEFINE_IDA(dimm_ida);
  * Retrieve bus and dimm handle and return if this bus supports
  * get_config_data commands
  */
-static int __validate_dimm(struct nvdimm_drvdata *ndd)
+int nvdimm_check_config_data(struct device *dev)
 {
-	struct nvdimm *nvdimm;
-
-	if (!ndd)
-		return -EINVAL;
-
-	nvdimm = to_nvdimm(ndd->dev);
+	struct nvdimm *nvdimm = to_nvdimm(dev);
 
-	if (!nvdimm->cmd_mask)
-		return -ENXIO;
-	if (!test_bit(ND_CMD_GET_CONFIG_DATA, &nvdimm->cmd_mask))
-		return -ENXIO;
+	if (!nvdimm->cmd_mask ||
+	    !test_bit(ND_CMD_GET_CONFIG_DATA, &nvdimm->cmd_mask)) {
+		if (nvdimm->flags & NDD_ALIASING)
+			return -ENXIO;
+		else
+			return -ENOTTY;
+	}
 
 	return 0;
 }
 
 static int validate_dimm(struct nvdimm_drvdata *ndd)
 {
-	int rc = __validate_dimm(ndd);
+	int rc;
 
-	if (rc && ndd)
+	if (!ndd)
+		return -EINVAL;
+
+	rc = nvdimm_check_config_data(ndd->dev);
+	if (rc)
 		dev_dbg(ndd->dev, "%pf: %s error: %d\n",
 				__builtin_return_address(0), __func__, rc);
 	return rc;
@@ -263,6 +265,12 @@ const char *nvdimm_name(struct nvdimm *nvdimm)
 }
 EXPORT_SYMBOL_GPL(nvdimm_name);
 
+struct kobject *nvdimm_kobj(struct nvdimm *nvdimm)
+{
+	return &nvdimm->dev.kobj;
+}
+EXPORT_SYMBOL_GPL(nvdimm_kobj);
+
 unsigned long nvdimm_cmd_mask(struct nvdimm *nvdimm)
 {
 	return nvdimm->cmd_mask;
@@ -378,40 +386,166 @@ struct nvdimm *nvdimm_create(struct nvdimm_bus *nvdimm_bus, void *provider_data,
 }
 EXPORT_SYMBOL_GPL(nvdimm_create);
 
+int alias_dpa_busy(struct device *dev, void *data)
+{
+	resource_size_t map_end, blk_start, new, busy;
+	struct blk_alloc_info *info = data;
+	struct nd_mapping *nd_mapping;
+	struct nd_region *nd_region;
+	struct nvdimm_drvdata *ndd;
+	struct resource *res;
+	int i;
+
+	if (!is_nd_pmem(dev))
+		return 0;
+
+	nd_region = to_nd_region(dev);
+	for (i = 0; i < nd_region->ndr_mappings; i++) {
+		nd_mapping  = &nd_region->mapping[i];
+		if (nd_mapping->nvdimm == info->nd_mapping->nvdimm)
+			break;
+	}
+
+	if (i >= nd_region->ndr_mappings)
+		return 0;
+
+	ndd = to_ndd(nd_mapping);
+	map_end = nd_mapping->start + nd_mapping->size - 1;
+	blk_start = nd_mapping->start;
+
+	/*
+	 * In the allocation case ->res is set to free space that we are
+	 * looking to validate against PMEM aliasing collision rules
+	 * (i.e. BLK is allocated after all aliased PMEM).
+	 */
+	if (info->res) {
+		if (info->res->start >= nd_mapping->start
+				&& info->res->start < map_end)
+			/* pass */;
+		else
+			return 0;
+	}
+
+ retry:
+	/*
+	 * Find the free dpa from the end of the last pmem allocation to
+	 * the end of the interleave-set mapping that is not already
+	 * covered by a blk allocation.
+	 */
+	busy = 0;
+	for_each_dpa_resource(ndd, res) {
+		if ((res->start >= blk_start && res->start < map_end)
+				|| (res->end >= blk_start
+					&& res->end <= map_end)) {
+			if (strncmp(res->name, "pmem", 4) == 0) {
+				new = max(blk_start, min(map_end + 1,
+							res->end + 1));
+				if (new != blk_start) {
+					blk_start = new;
+					goto retry;
+				}
+			} else
+				busy += min(map_end, res->end)
+					- max(nd_mapping->start, res->start) + 1;
+		} else if (nd_mapping->start > res->start
+				&& map_end < res->end) {
+			/* total eclipse of the PMEM region mapping */
+			busy += nd_mapping->size;
+			break;
+		}
+	}
+
+	/* update the free space range with the probed blk_start */
+	if (info->res && blk_start > info->res->start) {
+		info->res->start = max(info->res->start, blk_start);
+		if (info->res->start > info->res->end)
+			info->res->end = info->res->start - 1;
+		return 1;
+	}
+
+	info->available -= blk_start - nd_mapping->start + busy;
+
+	return 0;
+}
+
+static int blk_dpa_busy(struct device *dev, void *data)
+{
+	struct blk_alloc_info *info = data;
+	struct nd_mapping *nd_mapping;
+	struct nd_region *nd_region;
+	resource_size_t map_end;
+	int i;
+
+	if (!is_nd_pmem(dev))
+		return 0;
+
+	nd_region = to_nd_region(dev);
+	for (i = 0; i < nd_region->ndr_mappings; i++) {
+		nd_mapping  = &nd_region->mapping[i];
+		if (nd_mapping->nvdimm == info->nd_mapping->nvdimm)
+			break;
+	}
+
+	if (i >= nd_region->ndr_mappings)
+		return 0;
+
+	map_end = nd_mapping->start + nd_mapping->size - 1;
+	if (info->res->start >= nd_mapping->start
+			&& info->res->start < map_end) {
+		if (info->res->end <= map_end) {
+			info->busy = 0;
+			return 1;
+		} else {
+			info->busy -= info->res->end - map_end;
+			return 0;
+		}
+	} else if (info->res->end >= nd_mapping->start
+			&& info->res->end <= map_end) {
+		info->busy -= nd_mapping->start - info->res->start;
+		return 0;
+	} else {
+		info->busy -= nd_mapping->size;
+		return 0;
+	}
+}
+
 /**
  * nd_blk_available_dpa - account the unused dpa of BLK region
  * @nd_mapping: container of dpa-resource-root + labels
  *
- * Unlike PMEM, BLK namespaces can occupy discontiguous DPA ranges.
+ * Unlike PMEM, BLK namespaces can occupy discontiguous DPA ranges, but
+ * we arrange for them to never start at an lower dpa than the last
+ * PMEM allocation in an aliased region.
  */
-resource_size_t nd_blk_available_dpa(struct nd_mapping *nd_mapping)
+resource_size_t nd_blk_available_dpa(struct nd_region *nd_region)
 {
+	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
+	struct nd_mapping *nd_mapping = &nd_region->mapping[0];
 	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
-	resource_size_t map_end, busy = 0, available;
+	struct blk_alloc_info info = {
+		.nd_mapping = nd_mapping,
+		.available = nd_mapping->size,
+		.res = NULL,
+	};
 	struct resource *res;
 
 	if (!ndd)
 		return 0;
 
-	map_end = nd_mapping->start + nd_mapping->size - 1;
-	for_each_dpa_resource(ndd, res)
-		if (res->start >= nd_mapping->start && res->start < map_end) {
-			resource_size_t end = min(map_end, res->end);
+	device_for_each_child(&nvdimm_bus->dev, &info, alias_dpa_busy);
 
-			busy += end - res->start + 1;
-		} else if (res->end >= nd_mapping->start
-				&& res->end <= map_end) {
-			busy += res->end - nd_mapping->start;
-		} else if (nd_mapping->start > res->start
-				&& nd_mapping->start < res->end) {
-			/* total eclipse of the BLK region mapping */
-			busy += nd_mapping->size;
-		}
+	/* now account for busy blk allocations in unaliased dpa */
+	for_each_dpa_resource(ndd, res) {
+		if (strncmp(res->name, "blk", 3) != 0)
+			continue;
 
-	available = map_end - nd_mapping->start + 1;
-	if (busy < available)
-		return available - busy;
-	return 0;
+		info.res = res;
+		info.busy = resource_size(res);
+		device_for_each_child(&nvdimm_bus->dev, &info, blk_dpa_busy);
+		info.available -= info.busy;
+	}
+
+	return info.available;
 }
 
 /**
@@ -443,21 +577,16 @@ resource_size_t nd_pmem_available_dpa(struct nd_region *nd_region,
 	map_start = nd_mapping->start;
 	map_end = map_start + nd_mapping->size - 1;
 	blk_start = max(map_start, map_end + 1 - *overlap);
-	for_each_dpa_resource(ndd, res)
+	for_each_dpa_resource(ndd, res) {
 		if (res->start >= map_start && res->start < map_end) {
 			if (strncmp(res->name, "blk", 3) == 0)
-				blk_start = min(blk_start, res->start);
-			else if (res->start != map_start) {
+				blk_start = min(blk_start,
+						max(map_start, res->start));
+			else if (res->end > map_end) {
 				reason = "misaligned to iset";
 				goto err;
-			} else {
-				if (busy) {
-					reason = "duplicate overlapping PMEM reservations?";
-					goto err;
-				}
+			} else
 				busy += resource_size(res);
-				continue;
-			}
 		} else if (res->end >= map_start && res->end <= map_end) {
 			if (strncmp(res->name, "blk", 3) == 0) {
 				/*
@@ -466,15 +595,14 @@ resource_size_t nd_pmem_available_dpa(struct nd_region *nd_region,
 				 * be used for BLK.
 				 */
 				blk_start = map_start;
-			} else {
-				reason = "misaligned to iset";
-				goto err;
-			}
+			} else
+				busy += resource_size(res);
 		} else if (map_start > res->start && map_start < res->end) {
 			/* total eclipse of the mapping */
 			busy += nd_mapping->size;
 			blk_start = map_start;
 		}
+	}
 
 	*overlap = map_end + 1 - blk_start;
 	available = blk_start - map_start;
@@ -483,10 +611,6 @@ resource_size_t nd_pmem_available_dpa(struct nd_region *nd_region,
 	return 0;
 
  err:
-	/*
-	 * Something is wrong, PMEM must align with the start of the
-	 * interleave set, and there can only be one allocation per set.
-	 */
 	nd_dbg_dpa(nd_region, ndd, res, "%s\n", reason);
 	return 0;
 }
diff --git a/drivers/nvdimm/label.c b/drivers/nvdimm/label.c
index 96526dcfdd37..fac7cabe8f56 100644
--- a/drivers/nvdimm/label.c
+++ b/drivers/nvdimm/label.c
@@ -494,11 +494,13 @@ static int __pmem_label_update(struct nd_region *nd_region,
 		struct nd_mapping *nd_mapping, struct nd_namespace_pmem *nspm,
 		int pos)
 {
-	u64 cookie = nd_region_interleave_set_cookie(nd_region), rawsize;
+	u64 cookie = nd_region_interleave_set_cookie(nd_region);
 	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
-	struct nd_namespace_label *victim_label;
+	struct nd_label_ent *label_ent, *victim = NULL;
 	struct nd_namespace_label *nd_label;
 	struct nd_namespace_index *nsindex;
+	struct nd_label_id label_id;
+	struct resource *res;
 	unsigned long *free;
 	u32 nslot, slot;
 	size_t offset;
@@ -507,6 +509,16 @@ static int __pmem_label_update(struct nd_region *nd_region,
 	if (!preamble_next(ndd, &nsindex, &free, &nslot))
 		return -ENXIO;
 
+	nd_label_gen_id(&label_id, nspm->uuid, 0);
+	for_each_dpa_resource(ndd, res)
+		if (strcmp(res->name, label_id.id) == 0)
+			break;
+
+	if (!res) {
+		WARN_ON_ONCE(1);
+		return -ENXIO;
+	}
+
 	/* allocate and write the label to the staging (next) index */
 	slot = nd_label_alloc_slot(ndd);
 	if (slot == UINT_MAX)
@@ -522,11 +534,10 @@ static int __pmem_label_update(struct nd_region *nd_region,
 	nd_label->nlabel = __cpu_to_le16(nd_region->ndr_mappings);
 	nd_label->position = __cpu_to_le16(pos);
 	nd_label->isetcookie = __cpu_to_le64(cookie);
-	rawsize = div_u64(resource_size(&nspm->nsio.res),
-			nd_region->ndr_mappings);
-	nd_label->rawsize = __cpu_to_le64(rawsize);
-	nd_label->dpa = __cpu_to_le64(nd_mapping->start);
+	nd_label->rawsize = __cpu_to_le64(resource_size(res));
+	nd_label->dpa = __cpu_to_le64(res->start);
 	nd_label->slot = __cpu_to_le32(slot);
+	nd_dbg_dpa(nd_region, ndd, res, "%s\n", __func__);
 
 	/* update label */
 	offset = nd_label_offset(ndd, nd_label);
@@ -536,38 +547,43 @@ static int __pmem_label_update(struct nd_region *nd_region,
 		return rc;
 
 	/* Garbage collect the previous label */
-	victim_label = nd_mapping->labels[0];
-	if (victim_label) {
-		slot = to_slot(ndd, victim_label);
-		nd_label_free_slot(ndd, slot);
+	mutex_lock(&nd_mapping->lock);
+	list_for_each_entry(label_ent, &nd_mapping->labels, list) {
+		if (!label_ent->label)
+			continue;
+		if (memcmp(nspm->uuid, label_ent->label->uuid,
+					NSLABEL_UUID_LEN) != 0)
+			continue;
+		victim = label_ent;
+		list_move_tail(&victim->list, &nd_mapping->labels);
+		break;
+	}
+	if (victim) {
 		dev_dbg(ndd->dev, "%s: free: %d\n", __func__, slot);
+		slot = to_slot(ndd, victim->label);
+		nd_label_free_slot(ndd, slot);
+		victim->label = NULL;
 	}
 
 	/* update index */
 	rc = nd_label_write_index(ndd, ndd->ns_next,
 			nd_inc_seq(__le32_to_cpu(nsindex->seq)), 0);
-	if (rc < 0)
-		return rc;
-
-	nd_mapping->labels[0] = nd_label;
-
-	return 0;
-}
-
-static void del_label(struct nd_mapping *nd_mapping, int l)
-{
-	struct nd_namespace_label *next_label, *nd_label;
-	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
-	unsigned int slot;
-	int j;
-
-	nd_label = nd_mapping->labels[l];
-	slot = to_slot(ndd, nd_label);
-	dev_vdbg(ndd->dev, "%s: clear: %d\n", __func__, slot);
+	if (rc == 0) {
+		list_for_each_entry(label_ent, &nd_mapping->labels, list)
+			if (!label_ent->label) {
+				label_ent->label = nd_label;
+				nd_label = NULL;
+				break;
+			}
+		dev_WARN_ONCE(&nspm->nsio.common.dev, nd_label,
+				"failed to track label: %d\n",
+				to_slot(ndd, nd_label));
+		if (nd_label)
+			rc = -ENXIO;
+	}
+	mutex_unlock(&nd_mapping->lock);
 
-	for (j = l; (next_label = nd_mapping->labels[j + 1]); j++)
-		nd_mapping->labels[j] = next_label;
-	nd_mapping->labels[j] = NULL;
+	return rc;
 }
 
 static bool is_old_resource(struct resource *res, struct resource **list, int n)
@@ -607,14 +623,16 @@ static int __blk_label_update(struct nd_region *nd_region,
 		struct nd_mapping *nd_mapping, struct nd_namespace_blk *nsblk,
 		int num_labels)
 {
-	int i, l, alloc, victims, nfree, old_num_resources, nlabel, rc = -ENXIO;
+	int i, alloc, victims, nfree, old_num_resources, nlabel, rc = -ENXIO;
 	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
 	struct nd_namespace_label *nd_label;
+	struct nd_label_ent *label_ent, *e;
 	struct nd_namespace_index *nsindex;
 	unsigned long *free, *victim_map = NULL;
 	struct resource *res, **old_res_list;
 	struct nd_label_id label_id;
 	u8 uuid[NSLABEL_UUID_LEN];
+	LIST_HEAD(list);
 	u32 nslot, slot;
 
 	if (!preamble_next(ndd, &nsindex, &free, &nslot))
@@ -736,15 +754,22 @@ static int __blk_label_update(struct nd_region *nd_region,
 	 * entries in nd_mapping->labels
 	 */
 	nlabel = 0;
-	for_each_label(l, nd_label, nd_mapping->labels) {
+	mutex_lock(&nd_mapping->lock);
+	list_for_each_entry_safe(label_ent, e, &nd_mapping->labels, list) {
+		nd_label = label_ent->label;
+		if (!nd_label)
+			continue;
 		nlabel++;
 		memcpy(uuid, nd_label->uuid, NSLABEL_UUID_LEN);
 		if (memcmp(uuid, nsblk->uuid, NSLABEL_UUID_LEN) != 0)
 			continue;
 		nlabel--;
-		del_label(nd_mapping, l);
-		l--; /* retry with the new label at this index */
+		list_move(&label_ent->list, &list);
+		label_ent->label = NULL;
 	}
+	list_splice_tail_init(&list, &nd_mapping->labels);
+	mutex_unlock(&nd_mapping->lock);
+
 	if (nlabel + nsblk->num_resources > num_labels) {
 		/*
 		 * Bug, we can't end up with more resources than
@@ -755,6 +780,15 @@ static int __blk_label_update(struct nd_region *nd_region,
 		goto out;
 	}
 
+	mutex_lock(&nd_mapping->lock);
+	label_ent = list_first_entry_or_null(&nd_mapping->labels,
+			typeof(*label_ent), list);
+	if (!label_ent) {
+		WARN_ON(1);
+		mutex_unlock(&nd_mapping->lock);
+		rc = -ENXIO;
+		goto out;
+	}
 	for_each_clear_bit_le(slot, free, nslot) {
 		nd_label = nd_label_base(ndd) + slot;
 		memcpy(uuid, nd_label->uuid, NSLABEL_UUID_LEN);
@@ -762,11 +796,19 @@ static int __blk_label_update(struct nd_region *nd_region,
 			continue;
 		res = to_resource(ndd, nd_label);
 		res->flags &= ~DPA_RESOURCE_ADJUSTED;
-		dev_vdbg(&nsblk->common.dev, "assign label[%d] slot: %d\n",
-				l, slot);
-		nd_mapping->labels[l++] = nd_label;
+		dev_vdbg(&nsblk->common.dev, "assign label slot: %d\n", slot);
+		list_for_each_entry_from(label_ent, &nd_mapping->labels, list) {
+			if (label_ent->label)
+				continue;
+			label_ent->label = nd_label;
+			nd_label = NULL;
+			break;
+		}
+		if (nd_label)
+			dev_WARN(&nsblk->common.dev,
+					"failed to track label slot%d\n", slot);
 	}
-	nd_mapping->labels[l] = NULL;
+	mutex_unlock(&nd_mapping->lock);
 
  out:
 	kfree(old_res_list);
@@ -788,32 +830,28 @@ static int __blk_label_update(struct nd_region *nd_region,
 
 static int init_labels(struct nd_mapping *nd_mapping, int num_labels)
 {
-	int i, l, old_num_labels = 0;
+	int i, old_num_labels = 0;
+	struct nd_label_ent *label_ent;
 	struct nd_namespace_index *nsindex;
-	struct nd_namespace_label *nd_label;
 	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
-	size_t size = (num_labels + 1) * sizeof(struct nd_namespace_label *);
 
-	for_each_label(l, nd_label, nd_mapping->labels)
+	mutex_lock(&nd_mapping->lock);
+	list_for_each_entry(label_ent, &nd_mapping->labels, list)
 		old_num_labels++;
+	mutex_unlock(&nd_mapping->lock);
 
 	/*
 	 * We need to preserve all the old labels for the mapping so
 	 * they can be garbage collected after writing the new labels.
 	 */
-	if (num_labels > old_num_labels) {
-		struct nd_namespace_label **labels;
-
-		labels = krealloc(nd_mapping->labels, size, GFP_KERNEL);
-		if (!labels)
+	for (i = old_num_labels; i < num_labels; i++) {
+		label_ent = kzalloc(sizeof(*label_ent), GFP_KERNEL);
+		if (!label_ent)
 			return -ENOMEM;
-		nd_mapping->labels = labels;
+		mutex_lock(&nd_mapping->lock);
+		list_add_tail(&label_ent->list, &nd_mapping->labels);
+		mutex_unlock(&nd_mapping->lock);
 	}
-	if (!nd_mapping->labels)
-		return -ENOMEM;
-
-	for (i = old_num_labels; i <= num_labels; i++)
-		nd_mapping->labels[i] = NULL;
 
 	if (ndd->ns_current == -1 || ndd->ns_next == -1)
 		/* pass */;
@@ -837,42 +875,45 @@ static int init_labels(struct nd_mapping *nd_mapping, int num_labels)
 static int del_labels(struct nd_mapping *nd_mapping, u8 *uuid)
 {
 	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
-	struct nd_namespace_label *nd_label;
+	struct nd_label_ent *label_ent, *e;
 	struct nd_namespace_index *nsindex;
 	u8 label_uuid[NSLABEL_UUID_LEN];
-	int l, num_freed = 0;
 	unsigned long *free;
+	LIST_HEAD(list);
 	u32 nslot, slot;
+	int active = 0;
 
 	if (!uuid)
 		return 0;
 
 	/* no index || no labels == nothing to delete */
-	if (!preamble_next(ndd, &nsindex, &free, &nslot)
-			|| !nd_mapping->labels)
+	if (!preamble_next(ndd, &nsindex, &free, &nslot))
 		return 0;
 
-	for_each_label(l, nd_label, nd_mapping->labels) {
+	mutex_lock(&nd_mapping->lock);
+	list_for_each_entry_safe(label_ent, e, &nd_mapping->labels, list) {
+		struct nd_namespace_label *nd_label = label_ent->label;
+
+		if (!nd_label)
+			continue;
+		active++;
 		memcpy(label_uuid, nd_label->uuid, NSLABEL_UUID_LEN);
 		if (memcmp(label_uuid, uuid, NSLABEL_UUID_LEN) != 0)
 			continue;
+		active--;
 		slot = to_slot(ndd, nd_label);
 		nd_label_free_slot(ndd, slot);
 		dev_dbg(ndd->dev, "%s: free: %d\n", __func__, slot);
-		del_label(nd_mapping, l);
-		num_freed++;
-		l--; /* retry with new label at this index */
+		list_move_tail(&label_ent->list, &list);
+		label_ent->label = NULL;
 	}
+	list_splice_tail_init(&list, &nd_mapping->labels);
 
-	if (num_freed > l) {
-		/*
-		 * num_freed will only ever be > l when we delete the last
-		 * label
-		 */
-		kfree(nd_mapping->labels);
-		nd_mapping->labels = NULL;
-		dev_dbg(ndd->dev, "%s: no more labels\n", __func__);
+	if (active == 0) {
+		nd_mapping_free_labels(nd_mapping);
+		dev_dbg(ndd->dev, "%s: no more active labels\n", __func__);
 	}
+	mutex_unlock(&nd_mapping->lock);
 
 	return nd_label_write_index(ndd, ndd->ns_next,
 			nd_inc_seq(__le32_to_cpu(nsindex->seq)), 0);
@@ -885,7 +926,9 @@ int nd_pmem_namespace_label_update(struct nd_region *nd_region,
 
 	for (i = 0; i < nd_region->ndr_mappings; i++) {
 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
-		int rc;
+		struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
+		struct resource *res;
+		int rc, count = 0;
 
 		if (size == 0) {
 			rc = del_labels(nd_mapping, nspm->uuid);
@@ -894,7 +937,12 @@ int nd_pmem_namespace_label_update(struct nd_region *nd_region,
 			continue;
 		}
 
-		rc = init_labels(nd_mapping, 1);
+		for_each_dpa_resource(ndd, res)
+			if (strncmp(res->name, "pmem", 3) == 0)
+				count++;
+		WARN_ON_ONCE(!count);
+
+		rc = init_labels(nd_mapping, count);
 		if (rc < 0)
 			return rc;
 
diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c
index c5e3196c45b0..3509cff68ef9 100644
--- a/drivers/nvdimm/namespace_devs.c
+++ b/drivers/nvdimm/namespace_devs.c
@@ -12,8 +12,10 @@
  */
 #include <linux/module.h>
 #include <linux/device.h>
+#include <linux/sort.h>
 #include <linux/slab.h>
 #include <linux/pmem.h>
+#include <linux/list.h>
 #include <linux/nd.h>
 #include "nd-core.h"
 #include "nd.h"
@@ -28,7 +30,10 @@ static void namespace_io_release(struct device *dev)
 static void namespace_pmem_release(struct device *dev)
 {
 	struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
+	struct nd_region *nd_region = to_nd_region(dev->parent);
 
+	if (nspm->id >= 0)
+		ida_simple_remove(&nd_region->ns_ida, nspm->id);
 	kfree(nspm->alt_name);
 	kfree(nspm->uuid);
 	kfree(nspm);
@@ -62,17 +67,17 @@ static struct device_type namespace_blk_device_type = {
 	.release = namespace_blk_release,
 };
 
-static bool is_namespace_pmem(struct device *dev)
+static bool is_namespace_pmem(const struct device *dev)
 {
 	return dev ? dev->type == &namespace_pmem_device_type : false;
 }
 
-static bool is_namespace_blk(struct device *dev)
+static bool is_namespace_blk(const struct device *dev)
 {
 	return dev ? dev->type == &namespace_blk_device_type : false;
 }
 
-static bool is_namespace_io(struct device *dev)
+static bool is_namespace_io(const struct device *dev)
 {
 	return dev ? dev->type == &namespace_io_device_type : false;
 }
@@ -168,7 +173,21 @@ const char *nvdimm_namespace_disk_name(struct nd_namespace_common *ndns,
 		suffix = "s";
 
 	if (is_namespace_pmem(&ndns->dev) || is_namespace_io(&ndns->dev)) {
-		sprintf(name, "pmem%d%s", nd_region->id, suffix ? suffix : "");
+		int nsidx = 0;
+
+		if (is_namespace_pmem(&ndns->dev)) {
+			struct nd_namespace_pmem *nspm;
+
+			nspm = to_nd_namespace_pmem(&ndns->dev);
+			nsidx = nspm->id;
+		}
+
+		if (nsidx)
+			sprintf(name, "pmem%d.%d%s", nd_region->id, nsidx,
+					suffix ? suffix : "");
+		else
+			sprintf(name, "pmem%d%s", nd_region->id,
+					suffix ? suffix : "");
 	} else if (is_namespace_blk(&ndns->dev)) {
 		struct nd_namespace_blk *nsblk;
 
@@ -294,7 +313,7 @@ static bool __nd_namespace_blk_validate(struct nd_namespace_blk *nsblk)
 		if (strcmp(res->name, label_id.id) != 0)
 			continue;
 		/*
-		 * Resources with unacknoweldged adjustments indicate a
+		 * Resources with unacknowledged adjustments indicate a
 		 * failure to update labels
 		 */
 		if (res->flags & DPA_RESOURCE_ADJUSTED)
@@ -510,19 +529,68 @@ static resource_size_t init_dpa_allocation(struct nd_label_id *label_id,
 	return rc ? n : 0;
 }
 
-static bool space_valid(bool is_pmem, bool is_reserve,
-		struct nd_label_id *label_id, struct resource *res)
+
+/**
+ * space_valid() - validate free dpa space against constraints
+ * @nd_region: hosting region of the free space
+ * @ndd: dimm device data for debug
+ * @label_id: namespace id to allocate space
+ * @prev: potential allocation that precedes free space
+ * @next: allocation that follows the given free space range
+ * @exist: first allocation with same id in the mapping
+ * @n: range that must satisfied for pmem allocations
+ * @valid: free space range to validate
+ *
+ * BLK-space is valid as long as it does not precede a PMEM
+ * allocation in a given region. PMEM-space must be contiguous
+ * and adjacent to an existing existing allocation (if one
+ * exists).  If reserving PMEM any space is valid.
+ */
+static void space_valid(struct nd_region *nd_region, struct nvdimm_drvdata *ndd,
+		struct nd_label_id *label_id, struct resource *prev,
+		struct resource *next, struct resource *exist,
+		resource_size_t n, struct resource *valid)
 {
-	/*
-	 * For BLK-space any space is valid, for PMEM-space, it must be
-	 * contiguous with an existing allocation unless we are
-	 * reserving pmem.
-	 */
-	if (is_reserve || !is_pmem)
-		return true;
-	if (!res || strcmp(res->name, label_id->id) == 0)
-		return true;
-	return false;
+	bool is_reserve = strcmp(label_id->id, "pmem-reserve") == 0;
+	bool is_pmem = strncmp(label_id->id, "pmem", 4) == 0;
+
+	if (valid->start >= valid->end)
+		goto invalid;
+
+	if (is_reserve)
+		return;
+
+	if (!is_pmem) {
+		struct nd_mapping *nd_mapping = &nd_region->mapping[0];
+		struct nvdimm_bus *nvdimm_bus;
+		struct blk_alloc_info info = {
+			.nd_mapping = nd_mapping,
+			.available = nd_mapping->size,
+			.res = valid,
+		};
+
+		WARN_ON(!is_nd_blk(&nd_region->dev));
+		nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
+		device_for_each_child(&nvdimm_bus->dev, &info, alias_dpa_busy);
+		return;
+	}
+
+	/* allocation needs to be contiguous, so this is all or nothing */
+	if (resource_size(valid) < n)
+		goto invalid;
+
+	/* we've got all the space we need and no existing allocation */
+	if (!exist)
+		return;
+
+	/* allocation needs to be contiguous with the existing namespace */
+	if (valid->start == exist->end + 1
+			|| valid->end == exist->start - 1)
+		return;
+
+ invalid:
+	/* truncate @valid size to 0 */
+	valid->end = valid->start - 1;
 }
 
 enum alloc_loc {
@@ -534,18 +602,24 @@ static resource_size_t scan_allocate(struct nd_region *nd_region,
 		resource_size_t n)
 {
 	resource_size_t mapping_end = nd_mapping->start + nd_mapping->size - 1;
-	bool is_reserve = strcmp(label_id->id, "pmem-reserve") == 0;
 	bool is_pmem = strncmp(label_id->id, "pmem", 4) == 0;
 	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
+	struct resource *res, *exist = NULL, valid;
 	const resource_size_t to_allocate = n;
-	struct resource *res;
 	int first;
 
+	for_each_dpa_resource(ndd, res)
+		if (strcmp(label_id->id, res->name) == 0)
+			exist = res;
+
+	valid.start = nd_mapping->start;
+	valid.end = mapping_end;
+	valid.name = "free space";
  retry:
 	first = 0;
 	for_each_dpa_resource(ndd, res) {
-		resource_size_t allocate, available = 0, free_start, free_end;
 		struct resource *next = res->sibling, *new_res = NULL;
+		resource_size_t allocate, available = 0;
 		enum alloc_loc loc = ALLOC_ERR;
 		const char *action;
 		int rc = 0;
@@ -558,32 +632,35 @@ static resource_size_t scan_allocate(struct nd_region *nd_region,
 
 		/* space at the beginning of the mapping */
 		if (!first++ && res->start > nd_mapping->start) {
-			free_start = nd_mapping->start;
-			available = res->start - free_start;
-			if (space_valid(is_pmem, is_reserve, label_id, NULL))
+			valid.start = nd_mapping->start;
+			valid.end = res->start - 1;
+			space_valid(nd_region, ndd, label_id, NULL, next, exist,
+					to_allocate, &valid);
+			available = resource_size(&valid);
+			if (available)
 				loc = ALLOC_BEFORE;
 		}
 
 		/* space between allocations */
 		if (!loc && next) {
-			free_start = res->start + resource_size(res);
-			free_end = min(mapping_end, next->start - 1);
-			if (space_valid(is_pmem, is_reserve, label_id, res)
-					&& free_start < free_end) {
-				available = free_end + 1 - free_start;
+			valid.start = res->start + resource_size(res);
+			valid.end = min(mapping_end, next->start - 1);
+			space_valid(nd_region, ndd, label_id, res, next, exist,
+					to_allocate, &valid);
+			available = resource_size(&valid);
+			if (available)
 				loc = ALLOC_MID;
-			}
 		}
 
 		/* space at the end of the mapping */
 		if (!loc && !next) {
-			free_start = res->start + resource_size(res);
-			free_end = mapping_end;
-			if (space_valid(is_pmem, is_reserve, label_id, res)
-					&& free_start < free_end) {
-				available = free_end + 1 - free_start;
+			valid.start = res->start + resource_size(res);
+			valid.end = mapping_end;
+			space_valid(nd_region, ndd, label_id, res, next, exist,
+					to_allocate, &valid);
+			available = resource_size(&valid);
+			if (available)
 				loc = ALLOC_AFTER;
-			}
 		}
 
 		if (!loc || !available)
@@ -593,8 +670,6 @@ static resource_size_t scan_allocate(struct nd_region *nd_region,
 		case ALLOC_BEFORE:
 			if (strcmp(res->name, label_id->id) == 0) {
 				/* adjust current resource up */
-				if (is_pmem && !is_reserve)
-					return n;
 				rc = adjust_resource(res, res->start - allocate,
 						resource_size(res) + allocate);
 				action = "cur grow up";
@@ -604,8 +679,6 @@ static resource_size_t scan_allocate(struct nd_region *nd_region,
 		case ALLOC_MID:
 			if (strcmp(next->name, label_id->id) == 0) {
 				/* adjust next resource up */
-				if (is_pmem && !is_reserve)
-					return n;
 				rc = adjust_resource(next, next->start
 						- allocate, resource_size(next)
 						+ allocate);
@@ -629,12 +702,10 @@ static resource_size_t scan_allocate(struct nd_region *nd_region,
 		if (strcmp(action, "allocate") == 0) {
 			/* BLK allocate bottom up */
 			if (!is_pmem)
-				free_start += available - allocate;
-			else if (!is_reserve && free_start != nd_mapping->start)
-				return n;
+				valid.start += available - allocate;
 
 			new_res = nvdimm_allocate_dpa(ndd, label_id,
-					free_start, allocate);
+					valid.start, allocate);
 			if (!new_res)
 				rc = -EBUSY;
 		} else if (strcmp(action, "grow down") == 0) {
@@ -832,13 +903,45 @@ static int grow_dpa_allocation(struct nd_region *nd_region,
 	return 0;
 }
 
-static void nd_namespace_pmem_set_size(struct nd_region *nd_region,
+static void nd_namespace_pmem_set_resource(struct nd_region *nd_region,
 		struct nd_namespace_pmem *nspm, resource_size_t size)
 {
 	struct resource *res = &nspm->nsio.res;
+	resource_size_t offset = 0;
 
-	res->start = nd_region->ndr_start;
-	res->end = nd_region->ndr_start + size - 1;
+	if (size && !nspm->uuid) {
+		WARN_ON_ONCE(1);
+		size = 0;
+	}
+
+	if (size && nspm->uuid) {
+		struct nd_mapping *nd_mapping = &nd_region->mapping[0];
+		struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
+		struct nd_label_id label_id;
+		struct resource *res;
+
+		if (!ndd) {
+			size = 0;
+			goto out;
+		}
+
+		nd_label_gen_id(&label_id, nspm->uuid, 0);
+
+		/* calculate a spa offset from the dpa allocation offset */
+		for_each_dpa_resource(ndd, res)
+			if (strcmp(res->name, label_id.id) == 0) {
+				offset = (res->start - nd_mapping->start)
+					* nd_region->ndr_mappings;
+				goto out;
+			}
+
+		WARN_ON_ONCE(1);
+		size = 0;
+	}
+
+ out:
+	res->start = nd_region->ndr_start + offset;
+	res->end = res->start + size - 1;
 }
 
 static bool uuid_not_set(const u8 *uuid, struct device *dev, const char *where)
@@ -929,7 +1032,7 @@ static ssize_t __size_store(struct device *dev, unsigned long long val)
 	if (is_namespace_pmem(dev)) {
 		struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
 
-		nd_namespace_pmem_set_size(nd_region, nspm,
+		nd_namespace_pmem_set_resource(nd_region, nspm,
 				val * nd_region->ndr_mappings);
 	} else if (is_namespace_blk(dev)) {
 		struct nd_namespace_blk *nsblk = to_nd_namespace_blk(dev);
@@ -1031,22 +1134,27 @@ static ssize_t size_show(struct device *dev,
 }
 static DEVICE_ATTR(size, S_IRUGO, size_show, size_store);
 
-static ssize_t uuid_show(struct device *dev,
-		struct device_attribute *attr, char *buf)
+static u8 *namespace_to_uuid(struct device *dev)
 {
-	u8 *uuid;
-
 	if (is_namespace_pmem(dev)) {
 		struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
 
-		uuid = nspm->uuid;
+		return nspm->uuid;
 	} else if (is_namespace_blk(dev)) {
 		struct nd_namespace_blk *nsblk = to_nd_namespace_blk(dev);
 
-		uuid = nsblk->uuid;
+		return nsblk->uuid;
 	} else
-		return -ENXIO;
+		return ERR_PTR(-ENXIO);
+}
 
+static ssize_t uuid_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	u8 *uuid = namespace_to_uuid(dev);
+
+	if (IS_ERR(uuid))
+		return PTR_ERR(uuid);
 	if (uuid)
 		return sprintf(buf, "%pUb\n", uuid);
 	return sprintf(buf, "\n");
@@ -1089,7 +1197,7 @@ static int namespace_update_uuid(struct nd_region *nd_region,
 		 *
 		 * FIXME: can we delete uuid with zero dpa allocated?
 		 */
-		if (nd_mapping->labels)
+		if (list_empty(&nd_mapping->labels))
 			return -EBUSY;
 	}
 
@@ -1491,14 +1599,19 @@ static bool has_uuid_at_pos(struct nd_region *nd_region, u8 *uuid,
 
 	for (i = 0; i < nd_region->ndr_mappings; i++) {
 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
-		struct nd_namespace_label *nd_label;
+		struct nd_label_ent *label_ent;
 		bool found_uuid = false;
-		int l;
 
-		for_each_label(l, nd_label, nd_mapping->labels) {
-			u64 isetcookie = __le64_to_cpu(nd_label->isetcookie);
-			u16 position = __le16_to_cpu(nd_label->position);
-			u16 nlabel = __le16_to_cpu(nd_label->nlabel);
+		list_for_each_entry(label_ent, &nd_mapping->labels, list) {
+			struct nd_namespace_label *nd_label = label_ent->label;
+			u16 position, nlabel;
+			u64 isetcookie;
+
+			if (!nd_label)
+				continue;
+			isetcookie = __le64_to_cpu(nd_label->isetcookie);
+			position = __le16_to_cpu(nd_label->position);
+			nlabel = __le16_to_cpu(nd_label->nlabel);
 
 			if (isetcookie != cookie)
 				continue;
@@ -1528,7 +1641,6 @@ static bool has_uuid_at_pos(struct nd_region *nd_region, u8 *uuid,
 
 static int select_pmem_id(struct nd_region *nd_region, u8 *pmem_id)
 {
-	struct nd_namespace_label *select = NULL;
 	int i;
 
 	if (!pmem_id)
@@ -1536,90 +1648,106 @@ static int select_pmem_id(struct nd_region *nd_region, u8 *pmem_id)
 
 	for (i = 0; i < nd_region->ndr_mappings; i++) {
 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
-		struct nd_namespace_label *nd_label;
+		struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
+		struct nd_namespace_label *nd_label = NULL;
 		u64 hw_start, hw_end, pmem_start, pmem_end;
-		int l;
+		struct nd_label_ent *label_ent;
 
-		for_each_label(l, nd_label, nd_mapping->labels)
+		WARN_ON(!mutex_is_locked(&nd_mapping->lock));
+		list_for_each_entry(label_ent, &nd_mapping->labels, list) {
+			nd_label = label_ent->label;
+			if (!nd_label)
+				continue;
 			if (memcmp(nd_label->uuid, pmem_id, NSLABEL_UUID_LEN) == 0)
 				break;
+			nd_label = NULL;
+		}
 
 		if (!nd_label) {
 			WARN_ON(1);
 			return -EINVAL;
 		}
 
-		select = nd_label;
 		/*
 		 * Check that this label is compliant with the dpa
 		 * range published in NFIT
 		 */
 		hw_start = nd_mapping->start;
 		hw_end = hw_start + nd_mapping->size;
-		pmem_start = __le64_to_cpu(select->dpa);
-		pmem_end = pmem_start + __le64_to_cpu(select->rawsize);
-		if (pmem_start == hw_start && pmem_end <= hw_end)
+		pmem_start = __le64_to_cpu(nd_label->dpa);
+		pmem_end = pmem_start + __le64_to_cpu(nd_label->rawsize);
+		if (pmem_start >= hw_start && pmem_start < hw_end
+				&& pmem_end <= hw_end && pmem_end > hw_start)
 			/* pass */;
-		else
+		else {
+			dev_dbg(&nd_region->dev, "%s invalid label for %pUb\n",
+					dev_name(ndd->dev), nd_label->uuid);
 			return -EINVAL;
+		}
 
-		nd_mapping->labels[0] = select;
-		nd_mapping->labels[1] = NULL;
+		/* move recently validated label to the front of the list */
+		list_move(&label_ent->list, &nd_mapping->labels);
 	}
 	return 0;
 }
 
 /**
- * find_pmem_label_set - validate interleave set labelling, retrieve label0
+ * create_namespace_pmem - validate interleave set labelling, retrieve label0
  * @nd_region: region with mappings to validate
+ * @nspm: target namespace to create
+ * @nd_label: target pmem namespace label to evaluate
  */
-static int find_pmem_label_set(struct nd_region *nd_region,
-		struct nd_namespace_pmem *nspm)
+struct device *create_namespace_pmem(struct nd_region *nd_region,
+		struct nd_namespace_label *nd_label)
 {
 	u64 cookie = nd_region_interleave_set_cookie(nd_region);
-	struct nd_namespace_label *nd_label;
-	u8 select_id[NSLABEL_UUID_LEN];
+	struct nd_label_ent *label_ent;
+	struct nd_namespace_pmem *nspm;
+	struct nd_mapping *nd_mapping;
 	resource_size_t size = 0;
-	u8 *pmem_id = NULL;
-	int rc = -ENODEV, l;
+	struct resource *res;
+	struct device *dev;
+	int rc = 0;
 	u16 i;
 
-	if (cookie == 0)
-		return -ENXIO;
+	if (cookie == 0) {
+		dev_dbg(&nd_region->dev, "invalid interleave-set-cookie\n");
+		return ERR_PTR(-ENXIO);
+	}
 
-	/*
-	 * Find a complete set of labels by uuid.  By definition we can start
-	 * with any mapping as the reference label
-	 */
-	for_each_label(l, nd_label, nd_region->mapping[0].labels) {
-		u64 isetcookie = __le64_to_cpu(nd_label->isetcookie);
+	if (__le64_to_cpu(nd_label->isetcookie) != cookie) {
+		dev_dbg(&nd_region->dev, "invalid cookie in label: %pUb\n",
+				nd_label->uuid);
+		return ERR_PTR(-EAGAIN);
+	}
 
-		if (isetcookie != cookie)
-			continue;
+	nspm = kzalloc(sizeof(*nspm), GFP_KERNEL);
+	if (!nspm)
+		return ERR_PTR(-ENOMEM);
 
-		for (i = 0; nd_region->ndr_mappings; i++)
-			if (!has_uuid_at_pos(nd_region, nd_label->uuid,
-						cookie, i))
-				break;
-		if (i < nd_region->ndr_mappings) {
-			/*
-			 * Give up if we don't find an instance of a
-			 * uuid at each position (from 0 to
-			 * nd_region->ndr_mappings - 1), or if we find a
-			 * dimm with two instances of the same uuid.
-			 */
-			rc = -EINVAL;
-			goto err;
-		} else if (pmem_id) {
-			/*
-			 * If there is more than one valid uuid set, we
-			 * need userspace to clean this up.
-			 */
-			rc = -EBUSY;
-			goto err;
-		}
-		memcpy(select_id, nd_label->uuid, NSLABEL_UUID_LEN);
-		pmem_id = select_id;
+	nspm->id = -1;
+	dev = &nspm->nsio.common.dev;
+	dev->type = &namespace_pmem_device_type;
+	dev->parent = &nd_region->dev;
+	res = &nspm->nsio.res;
+	res->name = dev_name(&nd_region->dev);
+	res->flags = IORESOURCE_MEM;
+
+	for (i = 0; i < nd_region->ndr_mappings; i++)
+		if (!has_uuid_at_pos(nd_region, nd_label->uuid, cookie, i))
+			break;
+	if (i < nd_region->ndr_mappings) {
+		struct nvdimm_drvdata *ndd = to_ndd(&nd_region->mapping[i]);
+
+		/*
+		 * Give up if we don't find an instance of a uuid at each
+		 * position (from 0 to nd_region->ndr_mappings - 1), or if we
+		 * find a dimm with two instances of the same uuid.
+		 */
+		dev_err(&nd_region->dev, "%s missing label for %pUb\n",
+				dev_name(ndd->dev), nd_label->uuid);
+		rc = -EINVAL;
+		goto err;
 	}
 
 	/*
@@ -1630,14 +1758,23 @@ static int find_pmem_label_set(struct nd_region *nd_region,
 	 * the dimm being enabled (i.e. nd_label_reserve_dpa()
 	 * succeeded).
 	 */
-	rc = select_pmem_id(nd_region, pmem_id);
+	rc = select_pmem_id(nd_region, nd_label->uuid);
 	if (rc)
 		goto err;
 
 	/* Calculate total size and populate namespace properties from label0 */
 	for (i = 0; i < nd_region->ndr_mappings; i++) {
-		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
-		struct nd_namespace_label *label0 = nd_mapping->labels[0];
+		struct nd_namespace_label *label0;
+
+		nd_mapping = &nd_region->mapping[i];
+		label_ent = list_first_entry_or_null(&nd_mapping->labels,
+				typeof(*label_ent), list);
+		label0 = label_ent ? label_ent->label : 0;
+
+		if (!label0) {
+			WARN_ON(1);
+			continue;
+		}
 
 		size += __le64_to_cpu(label0->rawsize);
 		if (__le16_to_cpu(label0->position) != 0)
@@ -1654,10 +1791,11 @@ static int find_pmem_label_set(struct nd_region *nd_region,
 		goto err;
 	}
 
-	nd_namespace_pmem_set_size(nd_region, nspm, size);
+	nd_namespace_pmem_set_resource(nd_region, nspm, size);
 
-	return 0;
+	return dev;
  err:
+	namespace_pmem_release(dev);
 	switch (rc) {
 	case -EINVAL:
 		dev_dbg(&nd_region->dev, "%s: invalid label(s)\n", __func__);
@@ -1670,55 +1808,7 @@ static int find_pmem_label_set(struct nd_region *nd_region,
 				__func__, rc);
 		break;
 	}
-	return rc;
-}
-
-static struct device **create_namespace_pmem(struct nd_region *nd_region)
-{
-	struct nd_namespace_pmem *nspm;
-	struct device *dev, **devs;
-	struct resource *res;
-	int rc;
-
-	nspm = kzalloc(sizeof(*nspm), GFP_KERNEL);
-	if (!nspm)
-		return NULL;
-
-	dev = &nspm->nsio.common.dev;
-	dev->type = &namespace_pmem_device_type;
-	dev->parent = &nd_region->dev;
-	res = &nspm->nsio.res;
-	res->name = dev_name(&nd_region->dev);
-	res->flags = IORESOURCE_MEM;
-	rc = find_pmem_label_set(nd_region, nspm);
-	if (rc == -ENODEV) {
-		int i;
-
-		/* Pass, try to permit namespace creation... */
-		for (i = 0; i < nd_region->ndr_mappings; i++) {
-			struct nd_mapping *nd_mapping = &nd_region->mapping[i];
-
-			kfree(nd_mapping->labels);
-			nd_mapping->labels = NULL;
-		}
-
-		/* Publish a zero-sized namespace for userspace to configure. */
-		nd_namespace_pmem_set_size(nd_region, nspm, 0);
-
-		rc = 0;
-	} else if (rc)
-		goto err;
-
-	devs = kcalloc(2, sizeof(struct device *), GFP_KERNEL);
-	if (!devs)
-		goto err;
-
-	devs[0] = dev;
-	return devs;
-
- err:
-	namespace_pmem_release(&nspm->nsio.common.dev);
-	return NULL;
+	return ERR_PTR(rc);
 }
 
 struct resource *nsblk_add_resource(struct nd_region *nd_region,
@@ -1770,16 +1860,58 @@ static struct device *nd_namespace_blk_create(struct nd_region *nd_region)
 	return &nsblk->common.dev;
 }
 
-void nd_region_create_blk_seed(struct nd_region *nd_region)
+static struct device *nd_namespace_pmem_create(struct nd_region *nd_region)
+{
+	struct nd_namespace_pmem *nspm;
+	struct resource *res;
+	struct device *dev;
+
+	if (!is_nd_pmem(&nd_region->dev))
+		return NULL;
+
+	nspm = kzalloc(sizeof(*nspm), GFP_KERNEL);
+	if (!nspm)
+		return NULL;
+
+	dev = &nspm->nsio.common.dev;
+	dev->type = &namespace_pmem_device_type;
+	dev->parent = &nd_region->dev;
+	res = &nspm->nsio.res;
+	res->name = dev_name(&nd_region->dev);
+	res->flags = IORESOURCE_MEM;
+
+	nspm->id = ida_simple_get(&nd_region->ns_ida, 0, 0, GFP_KERNEL);
+	if (nspm->id < 0) {
+		kfree(nspm);
+		return NULL;
+	}
+	dev_set_name(dev, "namespace%d.%d", nd_region->id, nspm->id);
+	dev->parent = &nd_region->dev;
+	dev->groups = nd_namespace_attribute_groups;
+	nd_namespace_pmem_set_resource(nd_region, nspm, 0);
+
+	return dev;
+}
+
+void nd_region_create_ns_seed(struct nd_region *nd_region)
 {
 	WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));
-	nd_region->ns_seed = nd_namespace_blk_create(nd_region);
+
+	if (nd_region_to_nstype(nd_region) == ND_DEVICE_NAMESPACE_IO)
+		return;
+
+	if (is_nd_blk(&nd_region->dev))
+		nd_region->ns_seed = nd_namespace_blk_create(nd_region);
+	else
+		nd_region->ns_seed = nd_namespace_pmem_create(nd_region);
+
 	/*
 	 * Seed creation failures are not fatal, provisioning is simply
 	 * disabled until memory becomes available
 	 */
 	if (!nd_region->ns_seed)
-		dev_err(&nd_region->dev, "failed to create blk namespace\n");
+		dev_err(&nd_region->dev, "failed to create %s namespace\n",
+				is_nd_blk(&nd_region->dev) ? "blk" : "pmem");
 	else
 		nd_device_register(nd_region->ns_seed);
 }
@@ -1820,43 +1952,137 @@ void nd_region_create_btt_seed(struct nd_region *nd_region)
 		dev_err(&nd_region->dev, "failed to create btt namespace\n");
 }
 
-static struct device **create_namespace_blk(struct nd_region *nd_region)
+static int add_namespace_resource(struct nd_region *nd_region,
+		struct nd_namespace_label *nd_label, struct device **devs,
+		int count)
 {
 	struct nd_mapping *nd_mapping = &nd_region->mapping[0];
-	struct nd_namespace_label *nd_label;
-	struct device *dev, **devs = NULL;
+	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
+	int i;
+
+	for (i = 0; i < count; i++) {
+		u8 *uuid = namespace_to_uuid(devs[i]);
+		struct resource *res;
+
+		if (IS_ERR_OR_NULL(uuid)) {
+			WARN_ON(1);
+			continue;
+		}
+
+		if (memcmp(uuid, nd_label->uuid, NSLABEL_UUID_LEN) != 0)
+			continue;
+		if (is_namespace_blk(devs[i])) {
+			res = nsblk_add_resource(nd_region, ndd,
+					to_nd_namespace_blk(devs[i]),
+					__le64_to_cpu(nd_label->dpa));
+			if (!res)
+				return -ENXIO;
+			nd_dbg_dpa(nd_region, ndd, res, "%d assign\n", count);
+		} else {
+			dev_err(&nd_region->dev,
+					"error: conflicting extents for uuid: %pUb\n",
+					nd_label->uuid);
+			return -ENXIO;
+		}
+		break;
+	}
+
+	return i;
+}
+
+struct device *create_namespace_blk(struct nd_region *nd_region,
+		struct nd_namespace_label *nd_label, int count)
+{
+
+	struct nd_mapping *nd_mapping = &nd_region->mapping[0];
+	struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
 	struct nd_namespace_blk *nsblk;
-	struct nvdimm_drvdata *ndd;
-	int i, l, count = 0;
+	char *name[NSLABEL_NAME_LEN];
+	struct device *dev = NULL;
 	struct resource *res;
 
-	if (nd_region->ndr_mappings == 0)
-		return NULL;
+	nsblk = kzalloc(sizeof(*nsblk), GFP_KERNEL);
+	if (!nsblk)
+		return ERR_PTR(-ENOMEM);
+	dev = &nsblk->common.dev;
+	dev->type = &namespace_blk_device_type;
+	dev->parent = &nd_region->dev;
+	nsblk->id = -1;
+	nsblk->lbasize = __le64_to_cpu(nd_label->lbasize);
+	nsblk->uuid = kmemdup(nd_label->uuid, NSLABEL_UUID_LEN,
+			GFP_KERNEL);
+	if (!nsblk->uuid)
+		goto blk_err;
+	memcpy(name, nd_label->name, NSLABEL_NAME_LEN);
+	if (name[0])
+		nsblk->alt_name = kmemdup(name, NSLABEL_NAME_LEN,
+				GFP_KERNEL);
+	res = nsblk_add_resource(nd_region, ndd, nsblk,
+			__le64_to_cpu(nd_label->dpa));
+	if (!res)
+		goto blk_err;
+	nd_dbg_dpa(nd_region, ndd, res, "%d: assign\n", count);
+	return dev;
+ blk_err:
+	namespace_blk_release(dev);
+	return ERR_PTR(-ENXIO);
+}
+
+static int cmp_dpa(const void *a, const void *b)
+{
+	const struct device *dev_a = *(const struct device **) a;
+	const struct device *dev_b = *(const struct device **) b;
+	struct nd_namespace_blk *nsblk_a, *nsblk_b;
+	struct nd_namespace_pmem *nspm_a, *nspm_b;
+
+	if (is_namespace_io(dev_a))
+		return 0;
+
+	if (is_namespace_blk(dev_a)) {
+		nsblk_a = to_nd_namespace_blk(dev_a);
+		nsblk_b = to_nd_namespace_blk(dev_b);
+
+		return memcmp(&nsblk_a->res[0]->start, &nsblk_b->res[0]->start,
+				sizeof(resource_size_t));
+	}
+
+	nspm_a = to_nd_namespace_pmem(dev_a);
+	nspm_b = to_nd_namespace_pmem(dev_b);
+
+	return memcmp(&nspm_a->nsio.res.start, &nspm_b->nsio.res.start,
+			sizeof(resource_size_t));
+}
 
-	ndd = to_ndd(nd_mapping);
-	for_each_label(l, nd_label, nd_mapping->labels) {
-		u32 flags = __le32_to_cpu(nd_label->flags);
-		char *name[NSLABEL_NAME_LEN];
+static struct device **scan_labels(struct nd_region *nd_region)
+{
+	int i, count = 0;
+	struct device *dev, **devs = NULL;
+	struct nd_label_ent *label_ent, *e;
+	struct nd_mapping *nd_mapping = &nd_region->mapping[0];
+	resource_size_t map_end = nd_mapping->start + nd_mapping->size - 1;
+
+	/* "safe" because create_namespace_pmem() might list_move() label_ent */
+	list_for_each_entry_safe(label_ent, e, &nd_mapping->labels, list) {
+		struct nd_namespace_label *nd_label = label_ent->label;
 		struct device **__devs;
+		u32 flags;
 
-		if (flags & NSLABEL_FLAG_LOCAL)
-			/* pass */;
+		if (!nd_label)
+			continue;
+		flags = __le32_to_cpu(nd_label->flags);
+		if (is_nd_blk(&nd_region->dev)
+				== !!(flags & NSLABEL_FLAG_LOCAL))
+			/* pass, region matches label type */;
 		else
 			continue;
 
-		for (i = 0; i < count; i++) {
-			nsblk = to_nd_namespace_blk(devs[i]);
-			if (memcmp(nsblk->uuid, nd_label->uuid,
-						NSLABEL_UUID_LEN) == 0) {
-				res = nsblk_add_resource(nd_region, ndd, nsblk,
-						__le64_to_cpu(nd_label->dpa));
-				if (!res)
-					goto err;
-				nd_dbg_dpa(nd_region, ndd, res, "%s assign\n",
-					dev_name(&nsblk->common.dev));
-				break;
-			}
-		}
+		/* skip labels that describe extents outside of the region */
+		if (nd_label->dpa < nd_mapping->start || nd_label->dpa > map_end)
+			continue;
+
+		i = add_namespace_resource(nd_region, nd_label, devs, count);
+		if (i < 0)
+			goto err;
 		if (i < count)
 			continue;
 		__devs = kcalloc(count + 2, sizeof(dev), GFP_KERNEL);
@@ -1866,67 +2092,126 @@ static struct device **create_namespace_blk(struct nd_region *nd_region)
 		kfree(devs);
 		devs = __devs;
 
-		nsblk = kzalloc(sizeof(*nsblk), GFP_KERNEL);
-		if (!nsblk)
-			goto err;
-		dev = &nsblk->common.dev;
-		dev->type = &namespace_blk_device_type;
-		dev->parent = &nd_region->dev;
-		dev_set_name(dev, "namespace%d.%d", nd_region->id, count);
-		devs[count++] = dev;
-		nsblk->id = -1;
-		nsblk->lbasize = __le64_to_cpu(nd_label->lbasize);
-		nsblk->uuid = kmemdup(nd_label->uuid, NSLABEL_UUID_LEN,
-				GFP_KERNEL);
-		if (!nsblk->uuid)
-			goto err;
-		memcpy(name, nd_label->name, NSLABEL_NAME_LEN);
-		if (name[0])
-			nsblk->alt_name = kmemdup(name, NSLABEL_NAME_LEN,
-					GFP_KERNEL);
-		res = nsblk_add_resource(nd_region, ndd, nsblk,
-				__le64_to_cpu(nd_label->dpa));
-		if (!res)
-			goto err;
-		nd_dbg_dpa(nd_region, ndd, res, "%s assign\n",
-				dev_name(&nsblk->common.dev));
+		if (is_nd_blk(&nd_region->dev)) {
+			dev = create_namespace_blk(nd_region, nd_label, count);
+			if (IS_ERR(dev))
+				goto err;
+			devs[count++] = dev;
+		} else {
+			dev = create_namespace_pmem(nd_region, nd_label);
+			if (IS_ERR(dev)) {
+				switch (PTR_ERR(dev)) {
+				case -EAGAIN:
+					/* skip invalid labels */
+					continue;
+				case -ENODEV:
+					/* fallthrough to seed creation */
+					break;
+				default:
+					goto err;
+				}
+			} else
+				devs[count++] = dev;
+		}
 	}
 
-	dev_dbg(&nd_region->dev, "%s: discovered %d blk namespace%s\n",
-			__func__, count, count == 1 ? "" : "s");
+	dev_dbg(&nd_region->dev, "%s: discovered %d %s namespace%s\n",
+			__func__, count, is_nd_blk(&nd_region->dev)
+			? "blk" : "pmem", count == 1 ? "" : "s");
 
 	if (count == 0) {
 		/* Publish a zero-sized namespace for userspace to configure. */
-		for (i = 0; i < nd_region->ndr_mappings; i++) {
-			struct nd_mapping *nd_mapping = &nd_region->mapping[i];
-
-			kfree(nd_mapping->labels);
-			nd_mapping->labels = NULL;
-		}
+		nd_mapping_free_labels(nd_mapping);
 
 		devs = kcalloc(2, sizeof(dev), GFP_KERNEL);
 		if (!devs)
 			goto err;
-		nsblk = kzalloc(sizeof(*nsblk), GFP_KERNEL);
-		if (!nsblk)
-			goto err;
-		dev = &nsblk->common.dev;
-		dev->type = &namespace_blk_device_type;
+		if (is_nd_blk(&nd_region->dev)) {
+			struct nd_namespace_blk *nsblk;
+
+			nsblk = kzalloc(sizeof(*nsblk), GFP_KERNEL);
+			if (!nsblk)
+				goto err;
+			dev = &nsblk->common.dev;
+			dev->type = &namespace_blk_device_type;
+		} else {
+			struct nd_namespace_pmem *nspm;
+
+			nspm = kzalloc(sizeof(*nspm), GFP_KERNEL);
+			if (!nspm)
+				goto err;
+			dev = &nspm->nsio.common.dev;
+			dev->type = &namespace_pmem_device_type;
+			nd_namespace_pmem_set_resource(nd_region, nspm, 0);
+		}
 		dev->parent = &nd_region->dev;
 		devs[count++] = dev;
+	} else if (is_nd_pmem(&nd_region->dev)) {
+		/* clean unselected labels */
+		for (i = 0; i < nd_region->ndr_mappings; i++) {
+			struct list_head *l, *e;
+			LIST_HEAD(list);
+			int j;
+
+			nd_mapping = &nd_region->mapping[i];
+			if (list_empty(&nd_mapping->labels)) {
+				WARN_ON(1);
+				continue;
+			}
+
+			j = count;
+			list_for_each_safe(l, e, &nd_mapping->labels) {
+				if (!j--)
+					break;
+				list_move_tail(l, &list);
+			}
+			nd_mapping_free_labels(nd_mapping);
+			list_splice_init(&list, &nd_mapping->labels);
+		}
 	}
 
+	if (count > 1)
+		sort(devs, count, sizeof(struct device *), cmp_dpa, NULL);
+
 	return devs;
 
-err:
-	for (i = 0; i < count; i++) {
-		nsblk = to_nd_namespace_blk(devs[i]);
-		namespace_blk_release(&nsblk->common.dev);
-	}
+ err:
+	for (i = 0; devs[i]; i++)
+		if (is_nd_blk(&nd_region->dev))
+			namespace_blk_release(devs[i]);
+		else
+			namespace_pmem_release(devs[i]);
 	kfree(devs);
 	return NULL;
 }
 
+static struct device **create_namespaces(struct nd_region *nd_region)
+{
+	struct nd_mapping *nd_mapping = &nd_region->mapping[0];
+	struct device **devs;
+	int i;
+
+	if (nd_region->ndr_mappings == 0)
+		return NULL;
+
+	/* lock down all mappings while we scan labels */
+	for (i = 0; i < nd_region->ndr_mappings; i++) {
+		nd_mapping = &nd_region->mapping[i];
+		mutex_lock_nested(&nd_mapping->lock, i);
+	}
+
+	devs = scan_labels(nd_region);
+
+	for (i = 0; i < nd_region->ndr_mappings; i++) {
+		int reverse = nd_region->ndr_mappings - 1 - i;
+
+		nd_mapping = &nd_region->mapping[reverse];
+		mutex_unlock(&nd_mapping->lock);
+	}
+
+	return devs;
+}
+
 static int init_active_labels(struct nd_region *nd_region)
 {
 	int i;
@@ -1935,6 +2220,7 @@ static int init_active_labels(struct nd_region *nd_region)
 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
 		struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
 		struct nvdimm *nvdimm = nd_mapping->nvdimm;
+		struct nd_label_ent *label_ent;
 		int count, j;
 
 		/*
@@ -1956,16 +2242,27 @@ static int init_active_labels(struct nd_region *nd_region)
 		dev_dbg(ndd->dev, "%s: %d\n", __func__, count);
 		if (!count)
 			continue;
-		nd_mapping->labels = kcalloc(count + 1, sizeof(void *),
-				GFP_KERNEL);
-		if (!nd_mapping->labels)
-			return -ENOMEM;
 		for (j = 0; j < count; j++) {
 			struct nd_namespace_label *label;
 
+			label_ent = kzalloc(sizeof(*label_ent), GFP_KERNEL);
+			if (!label_ent)
+				break;
 			label = nd_label_active(ndd, j);
-			nd_mapping->labels[j] = label;
+			label_ent->label = label;
+
+			mutex_lock(&nd_mapping->lock);
+			list_add_tail(&label_ent->list, &nd_mapping->labels);
+			mutex_unlock(&nd_mapping->lock);
 		}
+
+		if (j >= count)
+			continue;
+
+		mutex_lock(&nd_mapping->lock);
+		nd_mapping_free_labels(nd_mapping);
+		mutex_unlock(&nd_mapping->lock);
+		return -ENOMEM;
 	}
 
 	return 0;
@@ -1990,10 +2287,8 @@ int nd_region_register_namespaces(struct nd_region *nd_region, int *err)
 		devs = create_namespace_io(nd_region);
 		break;
 	case ND_DEVICE_NAMESPACE_PMEM:
-		devs = create_namespace_pmem(nd_region);
-		break;
 	case ND_DEVICE_NAMESPACE_BLK:
-		devs = create_namespace_blk(nd_region);
+		devs = create_namespaces(nd_region);
 		break;
 	default:
 		break;
@@ -2014,6 +2309,13 @@ int nd_region_register_namespaces(struct nd_region *nd_region, int *err)
 			id = ida_simple_get(&nd_region->ns_ida, 0, 0,
 					GFP_KERNEL);
 			nsblk->id = id;
+		} else if (type == ND_DEVICE_NAMESPACE_PMEM) {
+			struct nd_namespace_pmem *nspm;
+
+			nspm = to_nd_namespace_pmem(dev);
+			id = ida_simple_get(&nd_region->ns_ida, 0, 0,
+					GFP_KERNEL);
+			nspm->id = id;
 		} else
 			id = i;
 
diff --git a/drivers/nvdimm/nd-core.h b/drivers/nvdimm/nd-core.h
index 38ce6bbbc170..8623e57c2ce3 100644
--- a/drivers/nvdimm/nd-core.h
+++ b/drivers/nvdimm/nd-core.h
@@ -44,6 +44,23 @@ struct nvdimm {
 	struct resource *flush_wpq;
 };
 
+/**
+ * struct blk_alloc_info - tracking info for BLK dpa scanning
+ * @nd_mapping: blk region mapping boundaries
+ * @available: decremented in alias_dpa_busy as aliased PMEM is scanned
+ * @busy: decremented in blk_dpa_busy to account for ranges already
+ * 	  handled by alias_dpa_busy
+ * @res: alias_dpa_busy interprets this a free space range that needs to
+ * 	 be truncated to the valid BLK allocation starting DPA, blk_dpa_busy
+ * 	 treats it as a busy range that needs the aliased PMEM ranges
+ * 	 truncated.
+ */
+struct blk_alloc_info {
+	struct nd_mapping *nd_mapping;
+	resource_size_t available, busy;
+	struct resource *res;
+};
+
 bool is_nvdimm(struct device *dev);
 bool is_nd_pmem(struct device *dev);
 bool is_nd_blk(struct device *dev);
@@ -54,7 +71,7 @@ void nvdimm_devs_exit(void);
 void nd_region_devs_exit(void);
 void nd_region_probe_success(struct nvdimm_bus *nvdimm_bus, struct device *dev);
 struct nd_region;
-void nd_region_create_blk_seed(struct nd_region *nd_region);
+void nd_region_create_ns_seed(struct nd_region *nd_region);
 void nd_region_create_btt_seed(struct nd_region *nd_region);
 void nd_region_create_pfn_seed(struct nd_region *nd_region);
 void nd_region_create_dax_seed(struct nd_region *nd_region);
@@ -73,13 +90,14 @@ bool nd_is_uuid_unique(struct device *dev, u8 *uuid);
 struct nd_region;
 struct nvdimm_drvdata;
 struct nd_mapping;
+void nd_mapping_free_labels(struct nd_mapping *nd_mapping);
 resource_size_t nd_pmem_available_dpa(struct nd_region *nd_region,
 		struct nd_mapping *nd_mapping, resource_size_t *overlap);
-resource_size_t nd_blk_available_dpa(struct nd_mapping *nd_mapping);
+resource_size_t nd_blk_available_dpa(struct nd_region *nd_region);
 resource_size_t nd_region_available_dpa(struct nd_region *nd_region);
 resource_size_t nvdimm_allocated_dpa(struct nvdimm_drvdata *ndd,
 		struct nd_label_id *label_id);
-struct nd_mapping;
+int alias_dpa_busy(struct device *dev, void *data);
 struct resource *nsblk_add_resource(struct nd_region *nd_region,
 		struct nvdimm_drvdata *ndd, struct nd_namespace_blk *nsblk,
 		resource_size_t start);
diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h
index 40476399d227..d3b2fca8deec 100644
--- a/drivers/nvdimm/nd.h
+++ b/drivers/nvdimm/nd.h
@@ -52,10 +52,28 @@ struct nvdimm_drvdata {
 struct nd_region_data {
 	int ns_count;
 	int ns_active;
-	unsigned int flush_mask;
-	void __iomem *flush_wpq[0][0];
+	unsigned int hints_shift;
+	void __iomem *flush_wpq[0];
 };
 
+static inline void __iomem *ndrd_get_flush_wpq(struct nd_region_data *ndrd,
+		int dimm, int hint)
+{
+	unsigned int num = 1 << ndrd->hints_shift;
+	unsigned int mask = num - 1;
+
+	return ndrd->flush_wpq[dimm * num + (hint & mask)];
+}
+
+static inline void ndrd_set_flush_wpq(struct nd_region_data *ndrd, int dimm,
+		int hint, void __iomem *flush)
+{
+	unsigned int num = 1 << ndrd->hints_shift;
+	unsigned int mask = num - 1;
+
+	ndrd->flush_wpq[dimm * num + (hint & mask)] = flush;
+}
+
 static inline struct nd_namespace_index *to_namespace_index(
 		struct nvdimm_drvdata *ndd, int i)
 {
@@ -83,9 +101,6 @@ static inline struct nd_namespace_index *to_next_namespace_index(
 		(unsigned long long) (res ? resource_size(res) : 0), \
 		(unsigned long long) (res ? res->start : 0), ##arg)
 
-#define for_each_label(l, label, labels) \
-	for (l = 0; (label = labels ? labels[l] : NULL); l++)
-
 #define for_each_dpa_resource(ndd, res) \
 	for (res = (ndd)->dpa.child; res; res = res->sibling)
 
@@ -98,6 +113,31 @@ struct nd_percpu_lane {
 	spinlock_t lock;
 };
 
+struct nd_label_ent {
+	struct list_head list;
+	struct nd_namespace_label *label;
+};
+
+enum nd_mapping_lock_class {
+	ND_MAPPING_CLASS0,
+	ND_MAPPING_UUID_SCAN,
+};
+
+struct nd_mapping {
+	struct nvdimm *nvdimm;
+	u64 start;
+	u64 size;
+	struct list_head labels;
+	struct mutex lock;
+	/*
+	 * @ndd is for private use at region enable / disable time for
+	 * get_ndd() + put_ndd(), all other nd_mapping to ndd
+	 * conversions use to_ndd() which respects enabled state of the
+	 * nvdimm.
+	 */
+	struct nvdimm_drvdata *ndd;
+};
+
 struct nd_region {
 	struct device dev;
 	struct ida ns_ida;
@@ -143,6 +183,7 @@ struct nd_btt {
 	struct nd_namespace_common *ndns;
 	struct btt *btt;
 	unsigned long lbasize;
+	u64 size;
 	u8 *uuid;
 	int id;
 };
@@ -190,6 +231,7 @@ void nvdimm_exit(void);
 void nd_region_exit(void);
 struct nvdimm;
 struct nvdimm_drvdata *to_ndd(struct nd_mapping *nd_mapping);
+int nvdimm_check_config_data(struct device *dev);
 int nvdimm_init_nsarea(struct nvdimm_drvdata *ndd);
 int nvdimm_init_config_data(struct nvdimm_drvdata *ndd);
 int nvdimm_set_config_data(struct nvdimm_drvdata *ndd, size_t offset,
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index 571a6c7ee2fc..42b3a8217073 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -66,13 +66,32 @@ static void pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset,
 	invalidate_pmem(pmem->virt_addr + offset, len);
 }
 
+static void write_pmem(void *pmem_addr, struct page *page,
+		unsigned int off, unsigned int len)
+{
+	void *mem = kmap_atomic(page);
+
+	memcpy_to_pmem(pmem_addr, mem + off, len);
+	kunmap_atomic(mem);
+}
+
+static int read_pmem(struct page *page, unsigned int off,
+		void *pmem_addr, unsigned int len)
+{
+	int rc;
+	void *mem = kmap_atomic(page);
+
+	rc = memcpy_from_pmem(mem + off, pmem_addr, len);
+	kunmap_atomic(mem);
+	return rc;
+}
+
 static int pmem_do_bvec(struct pmem_device *pmem, struct page *page,
 			unsigned int len, unsigned int off, bool is_write,
 			sector_t sector)
 {
 	int rc = 0;
 	bool bad_pmem = false;
-	void *mem = kmap_atomic(page);
 	phys_addr_t pmem_off = sector * 512 + pmem->data_offset;
 	void *pmem_addr = pmem->virt_addr + pmem_off;
 
@@ -83,7 +102,7 @@ static int pmem_do_bvec(struct pmem_device *pmem, struct page *page,
 		if (unlikely(bad_pmem))
 			rc = -EIO;
 		else {
-			rc = memcpy_from_pmem(mem + off, pmem_addr, len);
+			rc = read_pmem(page, off, pmem_addr, len);
 			flush_dcache_page(page);
 		}
 	} else {
@@ -102,14 +121,13 @@ static int pmem_do_bvec(struct pmem_device *pmem, struct page *page,
 		 * after clear poison.
 		 */
 		flush_dcache_page(page);
-		memcpy_to_pmem(pmem_addr, mem + off, len);
+		write_pmem(pmem_addr, page, off, len);
 		if (unlikely(bad_pmem)) {
 			pmem_clear_poison(pmem, pmem_off, len);
-			memcpy_to_pmem(pmem_addr, mem + off, len);
+			write_pmem(pmem_addr, page, off, len);
 		}
 	}
 
-	kunmap_atomic(mem);
 	return rc;
 }
 
diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index e8d5ba7b29af..6af5e629140c 100644
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -38,7 +38,7 @@ static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm,
 
 	dev_dbg(dev, "%s: map %d flush address%s\n", nvdimm_name(nvdimm),
 			nvdimm->num_flush, nvdimm->num_flush == 1 ? "" : "es");
-	for (i = 0; i < nvdimm->num_flush; i++) {
+	for (i = 0; i < (1 << ndrd->hints_shift); i++) {
 		struct resource *res = &nvdimm->flush_wpq[i];
 		unsigned long pfn = PHYS_PFN(res->start);
 		void __iomem *flush_page;
@@ -54,14 +54,15 @@ static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm,
 
 		if (j < i)
 			flush_page = (void __iomem *) ((unsigned long)
-					ndrd->flush_wpq[dimm][j] & PAGE_MASK);
+					ndrd_get_flush_wpq(ndrd, dimm, j)
+					& PAGE_MASK);
 		else
 			flush_page = devm_nvdimm_ioremap(dev,
-					PHYS_PFN(pfn), PAGE_SIZE);
+					PFN_PHYS(pfn), PAGE_SIZE);
 		if (!flush_page)
 			return -ENXIO;
-		ndrd->flush_wpq[dimm][i] = flush_page
-			+ (res->start & ~PAGE_MASK);
+		ndrd_set_flush_wpq(ndrd, dimm, i, flush_page
+				+ (res->start & ~PAGE_MASK));
 	}
 
 	return 0;
@@ -69,7 +70,7 @@ static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm,
 
 int nd_region_activate(struct nd_region *nd_region)
 {
-	int i, num_flush = 0;
+	int i, j, num_flush = 0;
 	struct nd_region_data *ndrd;
 	struct device *dev = &nd_region->dev;
 	size_t flush_data_size = sizeof(void *);
@@ -93,7 +94,10 @@ int nd_region_activate(struct nd_region *nd_region)
 		return -ENOMEM;
 	dev_set_drvdata(dev, ndrd);
 
-	ndrd->flush_mask = (1 << ilog2(num_flush)) - 1;
+	if (!num_flush)
+		return 0;
+
+	ndrd->hints_shift = ilog2(num_flush);
 	for (i = 0; i < nd_region->ndr_mappings; i++) {
 		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
 		struct nvdimm *nvdimm = nd_mapping->nvdimm;
@@ -103,6 +107,21 @@ int nd_region_activate(struct nd_region *nd_region)
 			return rc;
 	}
 
+	/*
+	 * Clear out entries that are duplicates. This should prevent the
+	 * extra flushings.
+	 */
+	for (i = 0; i < nd_region->ndr_mappings - 1; i++) {
+		/* ignore if NULL already */
+		if (!ndrd_get_flush_wpq(ndrd, i, 0))
+			continue;
+
+		for (j = i + 1; j < nd_region->ndr_mappings; j++)
+			if (ndrd_get_flush_wpq(ndrd, i, 0) ==
+			    ndrd_get_flush_wpq(ndrd, j, 0))
+				ndrd_set_flush_wpq(ndrd, j, 0, NULL);
+	}
+
 	return 0;
 }
 
@@ -294,9 +313,8 @@ resource_size_t nd_region_available_dpa(struct nd_region *nd_region)
 				blk_max_overlap = overlap;
 				goto retry;
 			}
-		} else if (is_nd_blk(&nd_region->dev)) {
-			available += nd_blk_available_dpa(nd_mapping);
-		}
+		} else if (is_nd_blk(&nd_region->dev))
+			available += nd_blk_available_dpa(nd_region);
 	}
 
 	return available;
@@ -487,6 +505,17 @@ u64 nd_region_interleave_set_cookie(struct nd_region *nd_region)
 	return 0;
 }
 
+void nd_mapping_free_labels(struct nd_mapping *nd_mapping)
+{
+	struct nd_label_ent *label_ent, *e;
+
+	WARN_ON(!mutex_is_locked(&nd_mapping->lock));
+	list_for_each_entry_safe(label_ent, e, &nd_mapping->labels, list) {
+		list_del(&label_ent->list);
+		kfree(label_ent);
+	}
+}
+
 /*
  * Upon successful probe/remove, take/release a reference on the
  * associated interleave set (if present), and plant new btt + namespace
@@ -507,8 +536,10 @@ static void nd_region_notify_driver_action(struct nvdimm_bus *nvdimm_bus,
 			struct nvdimm_drvdata *ndd = nd_mapping->ndd;
 			struct nvdimm *nvdimm = nd_mapping->nvdimm;
 
-			kfree(nd_mapping->labels);
-			nd_mapping->labels = NULL;
+			mutex_lock(&nd_mapping->lock);
+			nd_mapping_free_labels(nd_mapping);
+			mutex_unlock(&nd_mapping->lock);
+
 			put_ndd(ndd);
 			nd_mapping->ndd = NULL;
 			if (ndd)
@@ -518,11 +549,12 @@ static void nd_region_notify_driver_action(struct nvdimm_bus *nvdimm_bus,
 		if (is_nd_pmem(dev))
 			return;
 	}
-	if (dev->parent && is_nd_blk(dev->parent) && probe) {
+	if (dev->parent && (is_nd_blk(dev->parent) || is_nd_pmem(dev->parent))
+			&& probe) {
 		nd_region = to_nd_region(dev->parent);
 		nvdimm_bus_lock(dev);
 		if (nd_region->ns_seed == dev)
-			nd_region_create_blk_seed(nd_region);
+			nd_region_create_ns_seed(nd_region);
 		nvdimm_bus_unlock(dev);
 	}
 	if (is_nd_btt(dev) && probe) {
@@ -532,23 +564,30 @@ static void nd_region_notify_driver_action(struct nvdimm_bus *nvdimm_bus,
 		nvdimm_bus_lock(dev);
 		if (nd_region->btt_seed == dev)
 			nd_region_create_btt_seed(nd_region);
-		if (nd_region->ns_seed == &nd_btt->ndns->dev &&
-				is_nd_blk(dev->parent))
-			nd_region_create_blk_seed(nd_region);
+		if (nd_region->ns_seed == &nd_btt->ndns->dev)
+			nd_region_create_ns_seed(nd_region);
 		nvdimm_bus_unlock(dev);
 	}
 	if (is_nd_pfn(dev) && probe) {
+		struct nd_pfn *nd_pfn = to_nd_pfn(dev);
+
 		nd_region = to_nd_region(dev->parent);
 		nvdimm_bus_lock(dev);
 		if (nd_region->pfn_seed == dev)
 			nd_region_create_pfn_seed(nd_region);
+		if (nd_region->ns_seed == &nd_pfn->ndns->dev)
+			nd_region_create_ns_seed(nd_region);
 		nvdimm_bus_unlock(dev);
 	}
 	if (is_nd_dax(dev) && probe) {
+		struct nd_dax *nd_dax = to_nd_dax(dev);
+
 		nd_region = to_nd_region(dev->parent);
 		nvdimm_bus_lock(dev);
 		if (nd_region->dax_seed == dev)
 			nd_region_create_dax_seed(nd_region);
+		if (nd_region->ns_seed == &nd_dax->nd_pfn.ndns->dev)
+			nd_region_create_ns_seed(nd_region);
 		nvdimm_bus_unlock(dev);
 	}
 }
@@ -755,10 +794,10 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
 	int ro = 0;
 
 	for (i = 0; i < ndr_desc->num_mappings; i++) {
-		struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
-		struct nvdimm *nvdimm = nd_mapping->nvdimm;
+		struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
+		struct nvdimm *nvdimm = mapping->nvdimm;
 
-		if ((nd_mapping->start | nd_mapping->size) % SZ_4K) {
+		if ((mapping->start | mapping->size) % SZ_4K) {
 			dev_err(&nvdimm_bus->dev, "%s: %s mapping%d is not 4K aligned\n",
 					caller, dev_name(&nvdimm->dev), i);
 
@@ -809,11 +848,15 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
 		ndl->count = 0;
 	}
 
-	memcpy(nd_region->mapping, ndr_desc->nd_mapping,
-			sizeof(struct nd_mapping) * ndr_desc->num_mappings);
 	for (i = 0; i < ndr_desc->num_mappings; i++) {
-		struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
-		struct nvdimm *nvdimm = nd_mapping->nvdimm;
+		struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
+		struct nvdimm *nvdimm = mapping->nvdimm;
+
+		nd_region->mapping[i].nvdimm = nvdimm;
+		nd_region->mapping[i].start = mapping->start;
+		nd_region->mapping[i].size = mapping->size;
+		INIT_LIST_HEAD(&nd_region->mapping[i].labels);
+		mutex_init(&nd_region->mapping[i].lock);
 
 		get_device(&nvdimm->dev);
 	}
@@ -900,8 +943,8 @@ void nvdimm_flush(struct nd_region *nd_region)
 	 */
 	wmb();
 	for (i = 0; i < nd_region->ndr_mappings; i++)
-		if (ndrd->flush_wpq[i][0])
-			writeq(1, ndrd->flush_wpq[i][idx & ndrd->flush_mask]);
+		if (ndrd_get_flush_wpq(ndrd, i, 0))
+			writeq(1, ndrd_get_flush_wpq(ndrd, i, idx));
 	wmb();
 }
 EXPORT_SYMBOL_GPL(nvdimm_flush);
@@ -925,7 +968,7 @@ int nvdimm_has_flush(struct nd_region *nd_region)
 
 	for (i = 0; i < nd_region->ndr_mappings; i++)
 		/* flush hints present, flushing required */
-		if (ndrd->flush_wpq[i][0])
+		if (ndrd_get_flush_wpq(ndrd, i, 0))
 			return 1;
 
 	/*
diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index db39d53cdfb9..f7d37a62f874 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -30,8 +30,8 @@ config NVME_FABRICS
 
 config NVME_RDMA
 	tristate "NVM Express over Fabrics RDMA host driver"
-	depends on INFINIBAND
-	depends on BLK_DEV_NVME
+	depends on INFINIBAND && BLOCK
+	select NVME_CORE
 	select NVME_FABRICS
 	select SG_POOL
 	help
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 7ff2e820bbf4..329381a28edf 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -81,10 +81,12 @@ EXPORT_SYMBOL_GPL(nvme_cancel_request);
 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 		enum nvme_ctrl_state new_state)
 {
-	enum nvme_ctrl_state old_state = ctrl->state;
+	enum nvme_ctrl_state old_state;
 	bool changed = false;
 
 	spin_lock_irq(&ctrl->lock);
+
+	old_state = ctrl->state;
 	switch (new_state) {
 	case NVME_CTRL_LIVE:
 		switch (old_state) {
@@ -140,11 +142,12 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 	default:
 		break;
 	}
-	spin_unlock_irq(&ctrl->lock);
 
 	if (changed)
 		ctrl->state = new_state;
 
+	spin_unlock_irq(&ctrl->lock);
+
 	return changed;
 }
 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
@@ -153,12 +156,14 @@ static void nvme_free_ns(struct kref *kref)
 {
 	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 
-	if (ns->type == NVME_NS_LIGHTNVM)
-		nvme_nvm_unregister(ns->queue, ns->disk->disk_name);
+	if (ns->ndev)
+		nvme_nvm_unregister(ns);
 
-	spin_lock(&dev_list_lock);
-	ns->disk->private_data = NULL;
-	spin_unlock(&dev_list_lock);
+	if (ns->disk) {
+		spin_lock(&dev_list_lock);
+		ns->disk->private_data = NULL;
+		spin_unlock(&dev_list_lock);
+	}
 
 	put_disk(ns->disk);
 	ida_simple_remove(&ns->ctrl->ns_ida, ns->instance);
@@ -594,7 +599,7 @@ int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
 }
 
 int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
-					dma_addr_t dma_addr, u32 *result)
+		      void *buffer, size_t buflen, u32 *result)
 {
 	struct nvme_command c;
 	struct nvme_completion cqe;
@@ -603,18 +608,17 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
 	memset(&c, 0, sizeof(c));
 	c.features.opcode = nvme_admin_get_features;
 	c.features.nsid = cpu_to_le32(nsid);
-	c.features.dptr.prp1 = cpu_to_le64(dma_addr);
 	c.features.fid = cpu_to_le32(fid);
 
-	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, NULL, 0, 0,
+	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, buffer, buflen, 0,
 			NVME_QID_ANY, 0, 0);
-	if (ret >= 0)
+	if (ret >= 0 && result)
 		*result = le32_to_cpu(cqe.result);
 	return ret;
 }
 
 int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
-					dma_addr_t dma_addr, u32 *result)
+		      void *buffer, size_t buflen, u32 *result)
 {
 	struct nvme_command c;
 	struct nvme_completion cqe;
@@ -622,13 +626,12 @@ int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
 
 	memset(&c, 0, sizeof(c));
 	c.features.opcode = nvme_admin_set_features;
-	c.features.dptr.prp1 = cpu_to_le64(dma_addr);
 	c.features.fid = cpu_to_le32(fid);
 	c.features.dword11 = cpu_to_le32(dword11);
 
-	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, NULL, 0, 0,
-			NVME_QID_ANY, 0, 0);
-	if (ret >= 0)
+	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe,
+			buffer, buflen, 0, NVME_QID_ANY, 0, 0);
+	if (ret >= 0 && result)
 		*result = le32_to_cpu(cqe.result);
 	return ret;
 }
@@ -661,7 +664,7 @@ int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
 	u32 result;
 	int status, nr_io_queues;
 
-	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, 0,
+	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
 			&result);
 	if (status < 0)
 		return status;
@@ -885,42 +888,32 @@ static void nvme_config_discard(struct nvme_ns *ns)
 	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
 }
 
-static int nvme_revalidate_disk(struct gendisk *disk)
+static int nvme_revalidate_ns(struct nvme_ns *ns, struct nvme_id_ns **id)
 {
-	struct nvme_ns *ns = disk->private_data;
-	struct nvme_id_ns *id;
-	u8 lbaf, pi_type;
-	u16 old_ms;
-	unsigned short bs;
-
-	if (test_bit(NVME_NS_DEAD, &ns->flags)) {
-		set_capacity(disk, 0);
-		return -ENODEV;
-	}
-	if (nvme_identify_ns(ns->ctrl, ns->ns_id, &id)) {
-		dev_warn(disk_to_dev(ns->disk), "%s: Identify failure\n",
-				__func__);
-		return -ENODEV;
-	}
-	if (id->ncap == 0) {
-		kfree(id);
+	if (nvme_identify_ns(ns->ctrl, ns->ns_id, id)) {
+		dev_warn(ns->ctrl->dev, "%s: Identify failure\n", __func__);
 		return -ENODEV;
 	}
 
-	if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
-		if (nvme_nvm_register(ns->queue, disk->disk_name)) {
-			dev_warn(disk_to_dev(ns->disk),
-				"%s: LightNVM init failure\n", __func__);
-			kfree(id);
-			return -ENODEV;
-		}
-		ns->type = NVME_NS_LIGHTNVM;
+	if ((*id)->ncap == 0) {
+		kfree(*id);
+		return -ENODEV;
 	}
 
 	if (ns->ctrl->vs >= NVME_VS(1, 1))
-		memcpy(ns->eui, id->eui64, sizeof(ns->eui));
+		memcpy(ns->eui, (*id)->eui64, sizeof(ns->eui));
 	if (ns->ctrl->vs >= NVME_VS(1, 2))
-		memcpy(ns->uuid, id->nguid, sizeof(ns->uuid));
+		memcpy(ns->uuid, (*id)->nguid, sizeof(ns->uuid));
+
+	return 0;
+}
+
+static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
+{
+	struct nvme_ns *ns = disk->private_data;
+	u8 lbaf, pi_type;
+	u16 old_ms;
+	unsigned short bs;
 
 	old_ms = ns->ms;
 	lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
@@ -959,8 +952,26 @@ static int nvme_revalidate_disk(struct gendisk *disk)
 	if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
 		nvme_config_discard(ns);
 	blk_mq_unfreeze_queue(disk->queue);
+}
 
+static int nvme_revalidate_disk(struct gendisk *disk)
+{
+	struct nvme_ns *ns = disk->private_data;
+	struct nvme_id_ns *id = NULL;
+	int ret;
+
+	if (test_bit(NVME_NS_DEAD, &ns->flags)) {
+		set_capacity(disk, 0);
+		return -ENODEV;
+	}
+
+	ret = nvme_revalidate_ns(ns, &id);
+	if (ret)
+		return ret;
+
+	__nvme_revalidate_disk(disk, id);
 	kfree(id);
+
 	return 0;
 }
 
@@ -1422,7 +1433,7 @@ static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
 								char *buf)
 {
-	struct nvme_ns *ns = dev_to_disk(dev)->private_data;
+	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 	struct nvme_ctrl *ctrl = ns->ctrl;
 	int serial_len = sizeof(ctrl->serial);
 	int model_len = sizeof(ctrl->model);
@@ -1446,7 +1457,7 @@ static DEVICE_ATTR(wwid, S_IRUGO, wwid_show, NULL);
 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
 								char *buf)
 {
-	struct nvme_ns *ns = dev_to_disk(dev)->private_data;
+	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 	return sprintf(buf, "%pU\n", ns->uuid);
 }
 static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);
@@ -1454,7 +1465,7 @@ static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);
 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
 								char *buf)
 {
-	struct nvme_ns *ns = dev_to_disk(dev)->private_data;
+	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 	return sprintf(buf, "%8phd\n", ns->eui);
 }
 static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);
@@ -1462,7 +1473,7 @@ static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);
 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
 								char *buf)
 {
-	struct nvme_ns *ns = dev_to_disk(dev)->private_data;
+	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 	return sprintf(buf, "%d\n", ns->ns_id);
 }
 static DEVICE_ATTR(nsid, S_IRUGO, nsid_show, NULL);
@@ -1479,7 +1490,7 @@ static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
 		struct attribute *a, int n)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
-	struct nvme_ns *ns = dev_to_disk(dev)->private_data;
+	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 
 	if (a == &dev_attr_uuid.attr) {
 		if (!memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
@@ -1639,6 +1650,8 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 {
 	struct nvme_ns *ns;
 	struct gendisk *disk;
+	struct nvme_id_ns *id;
+	char disk_name[DISK_NAME_LEN];
 	int node = dev_to_node(ctrl->dev);
 
 	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
@@ -1656,34 +1669,49 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 	ns->queue->queuedata = ns;
 	ns->ctrl = ctrl;
 
-	disk = alloc_disk_node(0, node);
-	if (!disk)
-		goto out_free_queue;
-
 	kref_init(&ns->kref);
 	ns->ns_id = nsid;
-	ns->disk = disk;
 	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
 
-
 	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
 	nvme_set_queue_limits(ctrl, ns->queue);
 
-	disk->fops = &nvme_fops;
-	disk->private_data = ns;
-	disk->queue = ns->queue;
-	disk->flags = GENHD_FL_EXT_DEVT;
-	sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance, ns->instance);
+	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->instance);
 
-	if (nvme_revalidate_disk(ns->disk))
-		goto out_free_disk;
+	if (nvme_revalidate_ns(ns, &id))
+		goto out_free_queue;
+
+	if (nvme_nvm_ns_supported(ns, id)) {
+		if (nvme_nvm_register(ns, disk_name, node,
+							&nvme_ns_attr_group)) {
+			dev_warn(ctrl->dev, "%s: LightNVM init failure\n",
+								__func__);
+			goto out_free_id;
+		}
+	} else {
+		disk = alloc_disk_node(0, node);
+		if (!disk)
+			goto out_free_id;
+
+		disk->fops = &nvme_fops;
+		disk->private_data = ns;
+		disk->queue = ns->queue;
+		disk->flags = GENHD_FL_EXT_DEVT;
+		memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
+		ns->disk = disk;
+
+		__nvme_revalidate_disk(disk, id);
+	}
 
 	mutex_lock(&ctrl->namespaces_mutex);
 	list_add_tail(&ns->list, &ctrl->namespaces);
 	mutex_unlock(&ctrl->namespaces_mutex);
 
 	kref_get(&ctrl->kref);
-	if (ns->type == NVME_NS_LIGHTNVM)
+
+	kfree(id);
+
+	if (ns->ndev)
 		return;
 
 	device_add_disk(ctrl->device, ns->disk);
@@ -1692,8 +1720,8 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 		pr_warn("%s: failed to create sysfs group for identification\n",
 			ns->disk->disk_name);
 	return;
- out_free_disk:
-	kfree(disk);
+ out_free_id:
+	kfree(id);
  out_free_queue:
 	blk_cleanup_queue(ns->queue);
  out_release_instance:
@@ -1707,7 +1735,7 @@ static void nvme_ns_remove(struct nvme_ns *ns)
 	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
 		return;
 
-	if (ns->disk->flags & GENHD_FL_UP) {
+	if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
 		if (blk_get_integrity(ns->disk))
 			blk_integrity_unregister(ns->disk);
 		sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
@@ -1730,7 +1758,7 @@ static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 
 	ns = nvme_find_get_ns(ctrl, nsid);
 	if (ns) {
-		if (revalidate_disk(ns->disk))
+		if (ns->disk && revalidate_disk(ns->disk))
 			nvme_ns_remove(ns);
 		nvme_put_ns(ns);
 	} else
@@ -1823,9 +1851,6 @@ static void nvme_scan_work(struct work_struct *work)
 	list_sort(NULL, &ctrl->namespaces, ns_cmp);
 	mutex_unlock(&ctrl->namespaces_mutex);
 	kfree(id);
-
-	if (ctrl->ops->post_scan)
-		ctrl->ops->post_scan(ctrl);
 }
 
 void nvme_queue_scan(struct nvme_ctrl *ctrl)
@@ -2035,7 +2060,7 @@ void nvme_kill_queues(struct nvme_ctrl *ctrl)
 		 * Revalidating a dead namespace sets capacity to 0. This will
 		 * end buffered writers dirtying pages that can't be synced.
 		 */
-		if (!test_and_set_bit(NVME_NS_DEAD, &ns->flags))
+		if (ns->disk && !test_and_set_bit(NVME_NS_DEAD, &ns->flags))
 			revalidate_disk(ns->disk);
 
 		blk_set_queue_dying(ns->queue);
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index dc996761042f..5a3f008d3480 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -47,8 +47,10 @@ static struct nvmf_host *nvmf_host_add(const char *hostnqn)
 
 	mutex_lock(&nvmf_hosts_mutex);
 	host = __nvmf_host_find(hostnqn);
-	if (host)
+	if (host) {
+		kref_get(&host->ref);
 		goto out_unlock;
+	}
 
 	host = kmalloc(sizeof(*host), GFP_KERNEL);
 	if (!host)
@@ -56,7 +58,7 @@ static struct nvmf_host *nvmf_host_add(const char *hostnqn)
 
 	kref_init(&host->ref);
 	memcpy(host->nqn, hostnqn, NVMF_NQN_SIZE);
-	uuid_le_gen(&host->id);
+	uuid_be_gen(&host->id);
 
 	list_add_tail(&host->list, &nvmf_hosts);
 out_unlock:
@@ -73,9 +75,9 @@ static struct nvmf_host *nvmf_host_default(void)
 		return NULL;
 
 	kref_init(&host->ref);
-	uuid_le_gen(&host->id);
+	uuid_be_gen(&host->id);
 	snprintf(host->nqn, NVMF_NQN_SIZE,
-		"nqn.2014-08.org.nvmexpress:NVMf:uuid:%pUl", &host->id);
+		"nqn.2014-08.org.nvmexpress:NVMf:uuid:%pUb", &host->id);
 
 	mutex_lock(&nvmf_hosts_mutex);
 	list_add_tail(&host->list, &nvmf_hosts);
@@ -109,8 +111,19 @@ static void nvmf_host_put(struct nvmf_host *host)
  */
 int nvmf_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
 {
-	return snprintf(buf, size, "traddr=%s,trsvcid=%s\n",
-			ctrl->opts->traddr, ctrl->opts->trsvcid);
+	int len = 0;
+
+	if (ctrl->opts->mask & NVMF_OPT_TRADDR)
+		len += snprintf(buf, size, "traddr=%s", ctrl->opts->traddr);
+	if (ctrl->opts->mask & NVMF_OPT_TRSVCID)
+		len += snprintf(buf + len, size - len, "%strsvcid=%s",
+				(len) ? "," : "", ctrl->opts->trsvcid);
+	if (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)
+		len += snprintf(buf + len, size - len, "%shost_traddr=%s",
+				(len) ? "," : "", ctrl->opts->host_traddr);
+	len += snprintf(buf + len, size - len, "\n");
+
+	return len;
 }
 EXPORT_SYMBOL_GPL(nvmf_get_address);
 
@@ -363,7 +376,14 @@ int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 	cmd.connect.opcode = nvme_fabrics_command;
 	cmd.connect.fctype = nvme_fabrics_type_connect;
 	cmd.connect.qid = 0;
-	cmd.connect.sqsize = cpu_to_le16(ctrl->sqsize);
+
+	/*
+	 * fabrics spec sets a minimum of depth 32 for admin queue,
+	 * so set the queue with this depth always until
+	 * justification otherwise.
+	 */
+	cmd.connect.sqsize = cpu_to_le16(NVMF_AQ_DEPTH - 1);
+
 	/*
 	 * Set keep-alive timeout in seconds granularity (ms * 1000)
 	 * and add a grace period for controller kato enforcement
@@ -375,7 +395,7 @@ int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 	if (!data)
 		return -ENOMEM;
 
-	memcpy(&data->hostid, &ctrl->opts->host->id, sizeof(uuid_le));
+	memcpy(&data->hostid, &ctrl->opts->host->id, sizeof(uuid_be));
 	data->cntlid = cpu_to_le16(0xffff);
 	strncpy(data->subsysnqn, ctrl->opts->subsysnqn, NVMF_NQN_SIZE);
 	strncpy(data->hostnqn, ctrl->opts->host->nqn, NVMF_NQN_SIZE);
@@ -434,7 +454,7 @@ int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
 	if (!data)
 		return -ENOMEM;
 
-	memcpy(&data->hostid, &ctrl->opts->host->id, sizeof(uuid_le));
+	memcpy(&data->hostid, &ctrl->opts->host->id, sizeof(uuid_be));
 	data->cntlid = cpu_to_le16(ctrl->cntlid);
 	strncpy(data->subsysnqn, ctrl->opts->subsysnqn, NVMF_NQN_SIZE);
 	strncpy(data->hostnqn, ctrl->opts->host->nqn, NVMF_NQN_SIZE);
@@ -510,6 +530,7 @@ static const match_table_t opt_tokens = {
 	{ NVMF_OPT_RECONNECT_DELAY,	"reconnect_delay=%d"	},
 	{ NVMF_OPT_KATO,		"keep_alive_tmo=%d"	},
 	{ NVMF_OPT_HOSTNQN,		"hostnqn=%s"		},
+	{ NVMF_OPT_HOST_TRADDR,		"host_traddr=%s"	},
 	{ NVMF_OPT_ERR,			NULL			}
 };
 
@@ -666,6 +687,14 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 			}
 			opts->reconnect_delay = token;
 			break;
+		case NVMF_OPT_HOST_TRADDR:
+			p = match_strdup(args);
+			if (!p) {
+				ret = -ENOMEM;
+				goto out;
+			}
+			opts->host_traddr = p;
+			break;
 		default:
 			pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n",
 				p);
@@ -732,6 +761,7 @@ void nvmf_free_options(struct nvmf_ctrl_options *opts)
 	kfree(opts->traddr);
 	kfree(opts->trsvcid);
 	kfree(opts->subsysnqn);
+	kfree(opts->host_traddr);
 	kfree(opts);
 }
 EXPORT_SYMBOL_GPL(nvmf_free_options);
diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h
index 89df52c8be97..924145c979f1 100644
--- a/drivers/nvme/host/fabrics.h
+++ b/drivers/nvme/host/fabrics.h
@@ -34,7 +34,7 @@ struct nvmf_host {
 	struct kref		ref;
 	struct list_head	list;
 	char			nqn[NVMF_NQN_SIZE];
-	uuid_le			id;
+	uuid_be			id;
 };
 
 /**
@@ -52,6 +52,7 @@ enum {
 	NVMF_OPT_KATO		= 1 << 7,
 	NVMF_OPT_HOSTNQN	= 1 << 8,
 	NVMF_OPT_RECONNECT_DELAY = 1 << 9,
+	NVMF_OPT_HOST_TRADDR	= 1 << 10,
 };
 
 /**
@@ -64,9 +65,12 @@ enum {
  *		being added.
  * @subsysnqn:	Hold the fully qualified NQN subystem name (format defined
  *		in the NVMe specification, "NVMe Qualified Names").
- * @traddr:	network address that will be used by the host to communicate
- *		to the added NVMe controller.
- * @trsvcid:	network port used for host-controller communication.
+ * @traddr:	The transport-specific TRADDR field for a port on the
+ *              subsystem which is adding a controller.
+ * @trsvcid:	The transport-specific TRSVCID field for a port on the
+ *              subsystem which is adding a controller.
+ * @host_traddr: A transport-specific field identifying the NVME host port
+ *              to use for the connection to the controller.
  * @queue_size: Number of IO queue elements.
  * @nr_io_queues: Number of controller IO queues that will be established.
  * @reconnect_delay: Time between two consecutive reconnect attempts.
@@ -80,6 +84,7 @@ struct nvmf_ctrl_options {
 	char			*subsysnqn;
 	char			*traddr;
 	char			*trsvcid;
+	char			*host_traddr;
 	size_t			queue_size;
 	unsigned int		nr_io_queues;
 	unsigned int		reconnect_delay;
diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c
index 63f483daf930..f5e3011e31fc 100644
--- a/drivers/nvme/host/lightnvm.c
+++ b/drivers/nvme/host/lightnvm.c
@@ -475,7 +475,7 @@ static inline void nvme_nvm_rqtocmd(struct request *rq, struct nvm_rq *rqd,
 
 	if (rqd->opcode == NVM_OP_HBWRITE || rqd->opcode == NVM_OP_HBREAD)
 		c->hb_rw.slba = cpu_to_le64(nvme_block_nr(ns,
-						rqd->bio->bi_iter.bi_sector));
+					rqd->bio->bi_iter.bi_sector));
 }
 
 static void nvme_nvm_end_io(struct request *rq, int error)
@@ -592,14 +592,37 @@ static struct nvm_dev_ops nvme_nvm_dev_ops = {
 	.max_phys_sect		= 64,
 };
 
-int nvme_nvm_register(struct request_queue *q, char *disk_name)
+int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node,
+		      const struct attribute_group *attrs)
 {
-	return nvm_register(q, disk_name, &nvme_nvm_dev_ops);
+	struct request_queue *q = ns->queue;
+	struct nvm_dev *dev;
+	int ret;
+
+	dev = nvm_alloc_dev(node);
+	if (!dev)
+		return -ENOMEM;
+
+	dev->q = q;
+	memcpy(dev->name, disk_name, DISK_NAME_LEN);
+	dev->ops = &nvme_nvm_dev_ops;
+	dev->parent_dev = ns->ctrl->device;
+	dev->private_data = ns;
+	ns->ndev = dev;
+
+	ret = nvm_register(dev);
+
+	ns->lba_shift = ilog2(dev->sec_size) - 9;
+
+	if (sysfs_create_group(&dev->dev.kobj, attrs))
+		pr_warn("%s: failed to create sysfs group for identification\n",
+			disk_name);
+	return ret;
 }
 
-void nvme_nvm_unregister(struct request_queue *q, char *disk_name)
+void nvme_nvm_unregister(struct nvme_ns *ns)
 {
-	nvm_unregister(disk_name);
+	nvm_unregister(ns->ndev);
 }
 
 /* move to shared place when used in multiple places. */
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index ab18b78102bf..d47f5a5d18c7 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -18,6 +18,7 @@
 #include <linux/pci.h>
 #include <linux/kref.h>
 #include <linux/blk-mq.h>
+#include <linux/lightnvm.h>
 
 enum {
 	/*
@@ -154,6 +155,7 @@ struct nvme_ns {
 	struct nvme_ctrl *ctrl;
 	struct request_queue *queue;
 	struct gendisk *disk;
+	struct nvm_dev *ndev;
 	struct kref kref;
 	int instance;
 
@@ -165,7 +167,6 @@ struct nvme_ns {
 	u16 ms;
 	bool ext;
 	u8 pi_type;
-	int type;
 	unsigned long flags;
 
 #define NVME_NS_REMOVING 0
@@ -184,7 +185,6 @@ struct nvme_ctrl_ops {
 	int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
 	int (*reset_ctrl)(struct nvme_ctrl *ctrl);
 	void (*free_ctrl)(struct nvme_ctrl *ctrl);
-	void (*post_scan)(struct nvme_ctrl *ctrl);
 	void (*submit_async_event)(struct nvme_ctrl *ctrl, int aer_idx);
 	int (*delete_ctrl)(struct nvme_ctrl *ctrl);
 	const char *(*get_subsysnqn)(struct nvme_ctrl *ctrl);
@@ -292,9 +292,9 @@ int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
 		struct nvme_id_ns **id);
 int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log);
 int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
-			dma_addr_t dma_addr, u32 *result);
+		      void *buffer, size_t buflen, u32 *result);
 int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
-			dma_addr_t dma_addr, u32 *result);
+		      void *buffer, size_t buflen, u32 *result);
 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
 void nvme_start_keep_alive(struct nvme_ctrl *ctrl);
 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
@@ -307,20 +307,35 @@ int nvme_sg_get_version_num(int __user *ip);
 
 #ifdef CONFIG_NVM
 int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id);
-int nvme_nvm_register(struct request_queue *q, char *disk_name);
-void nvme_nvm_unregister(struct request_queue *q, char *disk_name);
+int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node,
+		      const struct attribute_group *attrs);
+void nvme_nvm_unregister(struct nvme_ns *ns);
+
+static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
+{
+	if (dev->type->devnode)
+		return dev_to_disk(dev)->private_data;
+
+	return (container_of(dev, struct nvm_dev, dev))->private_data;
+}
 #else
-static inline int nvme_nvm_register(struct request_queue *q, char *disk_name)
+static inline int nvme_nvm_register(struct nvme_ns *ns, char *disk_name,
+				    int node,
+				    const struct attribute_group *attrs)
 {
 	return 0;
 }
 
-static inline void nvme_nvm_unregister(struct request_queue *q, char *disk_name) {};
+static inline void nvme_nvm_unregister(struct nvme_ns *ns) {};
 
 static inline int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id)
 {
 	return 0;
 }
+static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
+{
+	return dev_to_disk(dev)->private_data;
+}
 #endif /* CONFIG_NVM */
 
 int __init nvme_core_init(void);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index d7c33f9361aa..0fc99f0f2571 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -16,6 +16,7 @@
 #include <linux/bitops.h>
 #include <linux/blkdev.h>
 #include <linux/blk-mq.h>
+#include <linux/blk-mq-pci.h>
 #include <linux/cpu.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
@@ -88,7 +89,6 @@ struct nvme_dev {
 	unsigned max_qid;
 	int q_depth;
 	u32 db_stride;
-	struct msix_entry *entry;
 	void __iomem *bar;
 	struct work_struct reset_work;
 	struct work_struct remove_work;
@@ -201,6 +201,11 @@ static unsigned int nvme_cmd_size(struct nvme_dev *dev)
 		nvme_iod_alloc_size(dev, NVME_INT_BYTES(dev), NVME_INT_PAGES);
 }
 
+static int nvmeq_irq(struct nvme_queue *nvmeq)
+{
+	return pci_irq_vector(to_pci_dev(nvmeq->dev->dev), nvmeq->cq_vector);
+}
+
 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 				unsigned int hctx_idx)
 {
@@ -263,6 +268,13 @@ static int nvme_init_request(void *data, struct request *req,
 	return 0;
 }
 
+static int nvme_pci_map_queues(struct blk_mq_tag_set *set)
+{
+	struct nvme_dev *dev = set->driver_data;
+
+	return blk_mq_pci_map_queues(set, to_pci_dev(dev->dev));
+}
+
 /**
  * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
  * @nvmeq: The queue to use
@@ -503,7 +515,8 @@ static int nvme_map_data(struct nvme_dev *dev, struct request *req,
 		goto out;
 
 	ret = BLK_MQ_RQ_QUEUE_BUSY;
-	if (!dma_map_sg(dev->dev, iod->sg, iod->nents, dma_dir))
+	if (!dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, dma_dir,
+				DMA_ATTR_NO_WARN))
 		goto out;
 
 	if (!nvme_setup_prps(dev, req, size))
@@ -960,7 +973,7 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq)
 		spin_unlock_irq(&nvmeq->q_lock);
 		return 1;
 	}
-	vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
+	vector = nvmeq_irq(nvmeq);
 	nvmeq->dev->online_queues--;
 	nvmeq->cq_vector = -1;
 	spin_unlock_irq(&nvmeq->q_lock);
@@ -968,7 +981,6 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq)
 	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
 		blk_mq_stop_hw_queues(nvmeq->dev->ctrl.admin_q);
 
-	irq_set_affinity_hint(vector, NULL);
 	free_irq(vector, nvmeq);
 
 	return 0;
@@ -1075,15 +1087,14 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
 	return NULL;
 }
 
-static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
-							const char *name)
+static int queue_request_irq(struct nvme_queue *nvmeq)
 {
 	if (use_threaded_interrupts)
-		return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
-					nvme_irq_check, nvme_irq, IRQF_SHARED,
-					name, nvmeq);
-	return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
-				IRQF_SHARED, name, nvmeq);
+		return request_threaded_irq(nvmeq_irq(nvmeq), nvme_irq_check,
+				nvme_irq, IRQF_SHARED, nvmeq->irqname, nvmeq);
+	else
+		return request_irq(nvmeq_irq(nvmeq), nvme_irq, IRQF_SHARED,
+				nvmeq->irqname, nvmeq);
 }
 
 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
@@ -1114,7 +1125,7 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
 	if (result < 0)
 		goto release_cq;
 
-	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
+	result = queue_request_irq(nvmeq);
 	if (result < 0)
 		goto release_sq;
 
@@ -1131,7 +1142,6 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
 static struct blk_mq_ops nvme_mq_admin_ops = {
 	.queue_rq	= nvme_queue_rq,
 	.complete	= nvme_complete_rq,
-	.map_queue	= blk_mq_map_queue,
 	.init_hctx	= nvme_admin_init_hctx,
 	.exit_hctx      = nvme_admin_exit_hctx,
 	.init_request	= nvme_admin_init_request,
@@ -1141,9 +1151,9 @@ static struct blk_mq_ops nvme_mq_admin_ops = {
 static struct blk_mq_ops nvme_mq_ops = {
 	.queue_rq	= nvme_queue_rq,
 	.complete	= nvme_complete_rq,
-	.map_queue	= blk_mq_map_queue,
 	.init_hctx	= nvme_init_hctx,
 	.init_request	= nvme_init_request,
+	.map_queues	= nvme_pci_map_queues,
 	.timeout	= nvme_timeout,
 	.poll		= nvme_poll,
 };
@@ -1234,7 +1244,7 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev)
 		goto free_nvmeq;
 
 	nvmeq->cq_vector = 0;
-	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
+	result = queue_request_irq(nvmeq);
 	if (result) {
 		nvmeq->cq_vector = -1;
 		goto free_nvmeq;
@@ -1382,7 +1392,7 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 {
 	struct nvme_queue *adminq = dev->queues[0];
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
-	int result, i, vecs, nr_io_queues, size;
+	int result, nr_io_queues, size;
 
 	nr_io_queues = num_online_cpus();
 	result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
@@ -1417,29 +1427,18 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 	}
 
 	/* Deregister the admin queue's interrupt */
-	free_irq(dev->entry[0].vector, adminq);
+	free_irq(pci_irq_vector(pdev, 0), adminq);
 
 	/*
 	 * If we enable msix early due to not intx, disable it again before
 	 * setting up the full range we need.
 	 */
-	if (pdev->msi_enabled)
-		pci_disable_msi(pdev);
-	else if (pdev->msix_enabled)
-		pci_disable_msix(pdev);
-
-	for (i = 0; i < nr_io_queues; i++)
-		dev->entry[i].entry = i;
-	vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
-	if (vecs < 0) {
-		vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
-		if (vecs < 0) {
-			vecs = 1;
-		} else {
-			for (i = 0; i < vecs; i++)
-				dev->entry[i].vector = i + pdev->irq;
-		}
-	}
+	pci_free_irq_vectors(pdev);
+	nr_io_queues = pci_alloc_irq_vectors(pdev, 1, nr_io_queues,
+			PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY);
+	if (nr_io_queues <= 0)
+		return -EIO;
+	dev->max_qid = nr_io_queues;
 
 	/*
 	 * Should investigate if there's a performance win from allocating
@@ -1447,10 +1446,8 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 	 * path to scale better, even if the receive path is limited by the
 	 * number of interrupts.
 	 */
-	nr_io_queues = vecs;
-	dev->max_qid = nr_io_queues;
 
-	result = queue_request_irq(dev, adminq, adminq->irqname);
+	result = queue_request_irq(adminq);
 	if (result) {
 		adminq->cq_vector = -1;
 		goto free_queues;
@@ -1462,23 +1459,6 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
 	return result;
 }
 
-static void nvme_pci_post_scan(struct nvme_ctrl *ctrl)
-{
-	struct nvme_dev *dev = to_nvme_dev(ctrl);
-	struct nvme_queue *nvmeq;
-	int i;
-
-	for (i = 0; i < dev->online_queues; i++) {
-		nvmeq = dev->queues[i];
-
-		if (!nvmeq->tags || !(*nvmeq->tags))
-			continue;
-
-		irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
-					blk_mq_tags_cpumask(*nvmeq->tags));
-	}
-}
-
 static void nvme_del_queue_end(struct request *req, int error)
 {
 	struct nvme_queue *nvmeq = req->end_io_data;
@@ -1543,15 +1523,10 @@ static void nvme_disable_io_queues(struct nvme_dev *dev)
 		reinit_completion(&dev->ioq_wait);
  retry:
 		timeout = ADMIN_TIMEOUT;
-		for (; i > 0; i--) {
-			struct nvme_queue *nvmeq = dev->queues[i];
-
-			if (!pass)
-				nvme_suspend_queue(nvmeq);
-			if (nvme_delete_queue(nvmeq, opcode))
+		for (; i > 0; i--, sent++)
+			if (nvme_delete_queue(dev->queues[i], opcode))
 				break;
-			++sent;
-		}
+
 		while (sent--) {
 			timeout = wait_for_completion_io_timeout(&dev->ioq_wait, timeout);
 			if (timeout == 0)
@@ -1620,15 +1595,9 @@ static int nvme_pci_enable(struct nvme_dev *dev)
 	 * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
 	 * adjust this later.
 	 */
-	if (pci_enable_msix(pdev, dev->entry, 1)) {
-		pci_enable_msi(pdev);
-		dev->entry[0].vector = pdev->irq;
-	}
-
-	if (!dev->entry[0].vector) {
-		result = -ENODEV;
-		goto disable;
-	}
+	result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
+	if (result < 0)
+		return result;
 
 	cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
 
@@ -1670,10 +1639,7 @@ static void nvme_pci_disable(struct nvme_dev *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 
-	if (pdev->msi_enabled)
-		pci_disable_msi(pdev);
-	else if (pdev->msix_enabled)
-		pci_disable_msix(pdev);
+	pci_free_irq_vectors(pdev);
 
 	if (pci_is_enabled(pdev)) {
 		pci_disable_pcie_error_reporting(pdev);
@@ -1693,11 +1659,17 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
 		nvme_stop_queues(&dev->ctrl);
 		csts = readl(dev->bar + NVME_REG_CSTS);
 	}
+
+	for (i = dev->queue_count - 1; i > 0; i--)
+		nvme_suspend_queue(dev->queues[i]);
+
 	if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
-		for (i = dev->queue_count - 1; i >= 0; i--) {
-			struct nvme_queue *nvmeq = dev->queues[i];
-			nvme_suspend_queue(nvmeq);
-		}
+		/* A device might become IO incapable very soon during
+		 * probe, before the admin queue is configured. Thus,
+		 * queue_count can be 0 here.
+		 */
+		if (dev->queue_count)
+			nvme_suspend_queue(dev->queues[0]);
 	} else {
 		nvme_disable_io_queues(dev);
 		nvme_disable_admin_queue(dev, shutdown);
@@ -1742,7 +1714,6 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
 	if (dev->ctrl.admin_q)
 		blk_put_queue(dev->ctrl.admin_q);
 	kfree(dev->queues);
-	kfree(dev->entry);
 	kfree(dev);
 }
 
@@ -1886,7 +1857,6 @@ static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
 	.reg_read64		= nvme_pci_reg_read64,
 	.reset_ctrl		= nvme_pci_reset_ctrl,
 	.free_ctrl		= nvme_pci_free_ctrl,
-	.post_scan		= nvme_pci_post_scan,
 	.submit_async_event	= nvme_pci_submit_async_event,
 };
 
@@ -1919,10 +1889,6 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
 	if (!dev)
 		return -ENOMEM;
-	dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
-							GFP_KERNEL, node);
-	if (!dev->entry)
-		goto free;
 	dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
 							GFP_KERNEL, node);
 	if (!dev->queues)
@@ -1963,7 +1929,6 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	nvme_dev_unmap(dev);
  free:
 	kfree(dev->queues);
-	kfree(dev->entry);
 	kfree(dev);
 	return result;
 }
@@ -2116,6 +2081,8 @@ static const struct pci_device_id nvme_id_table[] = {
 		.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
 	{ PCI_DEVICE(0x1c58, 0x0003),	/* HGST adapter */
 		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
+	{ PCI_DEVICE(0x1c5f, 0x0540),	/* Memblaze Pblaze4 adapter */
+		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
 	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
 	{ 0, }
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 3e3ce2b0424e..5a8388177959 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -12,13 +12,11 @@
  * more details.
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-#include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/err.h>
 #include <linux/string.h>
-#include <linux/jiffies.h>
 #include <linux/atomic.h>
 #include <linux/blk-mq.h>
 #include <linux/types.h>
@@ -26,7 +24,6 @@
 #include <linux/mutex.h>
 #include <linux/scatterlist.h>
 #include <linux/nvme.h>
-#include <linux/t10-pi.h>
 #include <asm/unaligned.h>
 
 #include <rdma/ib_verbs.h>
@@ -46,10 +43,6 @@
 
 #define NVME_RDMA_MAX_INLINE_SEGMENTS	1
 
-#define NVME_RDMA_MAX_PAGES_PER_MR	512
-
-#define NVME_RDMA_DEF_RECONNECT_DELAY	20
-
 /*
  * We handle AEN commands ourselves and don't even let the
  * block layer know about them.
@@ -61,7 +54,6 @@
 struct nvme_rdma_device {
 	struct ib_device       *dev;
 	struct ib_pd	       *pd;
-	struct ib_mr	       *mr;
 	struct kref		ref;
 	struct list_head	entry;
 };
@@ -80,7 +72,6 @@ struct nvme_rdma_request {
 	u32			num_sge;
 	int			nents;
 	bool			inline_data;
-	bool			need_inval;
 	struct ib_reg_wr	reg_wr;
 	struct ib_cqe		reg_cqe;
 	struct nvme_rdma_queue  *queue;
@@ -90,6 +81,8 @@ struct nvme_rdma_request {
 
 enum nvme_rdma_queue_flags {
 	NVME_RDMA_Q_CONNECTED = (1 << 0),
+	NVME_RDMA_IB_QUEUE_ALLOCATED = (1 << 1),
+	NVME_RDMA_Q_DELETING = (1 << 2),
 };
 
 struct nvme_rdma_queue {
@@ -169,7 +162,6 @@ MODULE_PARM_DESC(register_always,
 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
 		struct rdma_cm_event *event);
 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
-static int __nvme_rdma_del_ctrl(struct nvme_rdma_ctrl *ctrl);
 
 /* XXX: really should move to a generic header sooner or later.. */
 static inline void put_unaligned_le24(u32 val, u8 *p)
@@ -290,7 +282,7 @@ static int nvme_rdma_reinit_request(void *data, struct request *rq)
 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 	int ret = 0;
 
-	if (!req->need_inval)
+	if (!req->mr->need_inval)
 		goto out;
 
 	ib_dereg_mr(req->mr);
@@ -300,9 +292,10 @@ static int nvme_rdma_reinit_request(void *data, struct request *rq)
 	if (IS_ERR(req->mr)) {
 		ret = PTR_ERR(req->mr);
 		req->mr = NULL;
+		goto out;
 	}
 
-	req->need_inval = false;
+	req->mr->need_inval = false;
 
 out:
 	return ret;
@@ -414,10 +407,7 @@ static void nvme_rdma_free_dev(struct kref *ref)
 	list_del(&ndev->entry);
 	mutex_unlock(&device_list_mutex);
 
-	if (!register_always)
-		ib_dereg_mr(ndev->mr);
 	ib_dealloc_pd(ndev->pd);
-
 	kfree(ndev);
 }
 
@@ -450,24 +440,16 @@ nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
 	ndev->dev = cm_id->device;
 	kref_init(&ndev->ref);
 
-	ndev->pd = ib_alloc_pd(ndev->dev);
+	ndev->pd = ib_alloc_pd(ndev->dev,
+		register_always ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
 	if (IS_ERR(ndev->pd))
 		goto out_free_dev;
 
-	if (!register_always) {
-		ndev->mr = ib_get_dma_mr(ndev->pd,
-					    IB_ACCESS_LOCAL_WRITE |
-					    IB_ACCESS_REMOTE_READ |
-					    IB_ACCESS_REMOTE_WRITE);
-		if (IS_ERR(ndev->mr))
-			goto out_free_pd;
-	}
-
 	if (!(ndev->dev->attrs.device_cap_flags &
 	      IB_DEVICE_MEM_MGT_EXTENSIONS)) {
 		dev_err(&ndev->dev->dev,
 			"Memory registrations not supported.\n");
-		goto out_free_mr;
+		goto out_free_pd;
 	}
 
 	list_add(&ndev->entry, &device_list);
@@ -475,9 +457,6 @@ out_unlock:
 	mutex_unlock(&device_list_mutex);
 	return ndev;
 
-out_free_mr:
-	if (!register_always)
-		ib_dereg_mr(ndev->mr);
 out_free_pd:
 	ib_dealloc_pd(ndev->pd);
 out_free_dev:
@@ -489,9 +468,14 @@ out_err:
 
 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
 {
-	struct nvme_rdma_device *dev = queue->device;
-	struct ib_device *ibdev = dev->dev;
+	struct nvme_rdma_device *dev;
+	struct ib_device *ibdev;
+
+	if (!test_and_clear_bit(NVME_RDMA_IB_QUEUE_ALLOCATED, &queue->flags))
+		return;
 
+	dev = queue->device;
+	ibdev = dev->dev;
 	rdma_destroy_qp(queue->cm_id);
 	ib_free_cq(queue->ib_cq);
 
@@ -542,6 +526,7 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue,
 		ret = -ENOMEM;
 		goto out_destroy_qp;
 	}
+	set_bit(NVME_RDMA_IB_QUEUE_ALLOCATED, &queue->flags);
 
 	return 0;
 
@@ -594,11 +579,13 @@ static int nvme_rdma_init_queue(struct nvme_rdma_ctrl *ctrl,
 		goto out_destroy_cm_id;
 	}
 
+	clear_bit(NVME_RDMA_Q_DELETING, &queue->flags);
 	set_bit(NVME_RDMA_Q_CONNECTED, &queue->flags);
 
 	return 0;
 
 out_destroy_cm_id:
+	nvme_rdma_destroy_queue_ib(queue);
 	rdma_destroy_id(queue->cm_id);
 	return ret;
 }
@@ -617,7 +604,7 @@ static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
 
 static void nvme_rdma_stop_and_free_queue(struct nvme_rdma_queue *queue)
 {
-	if (!test_and_clear_bit(NVME_RDMA_Q_CONNECTED, &queue->flags))
+	if (test_and_set_bit(NVME_RDMA_Q_DELETING, &queue->flags))
 		return;
 	nvme_rdma_stop_queue(queue);
 	nvme_rdma_free_queue(queue);
@@ -649,7 +636,8 @@ static int nvme_rdma_init_io_queues(struct nvme_rdma_ctrl *ctrl)
 	int i, ret;
 
 	for (i = 1; i < ctrl->queue_count; i++) {
-		ret = nvme_rdma_init_queue(ctrl, i, ctrl->ctrl.sqsize);
+		ret = nvme_rdma_init_queue(ctrl, i,
+					   ctrl->ctrl.opts->queue_size);
 		if (ret) {
 			dev_info(ctrl->ctrl.device,
 				"failed to initialize i/o queue: %d\n", ret);
@@ -660,7 +648,7 @@ static int nvme_rdma_init_io_queues(struct nvme_rdma_ctrl *ctrl)
 	return 0;
 
 out_free_queues:
-	for (; i >= 1; i--)
+	for (i--; i >= 1; i--)
 		nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
 
 	return ret;
@@ -687,11 +675,6 @@ static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
 	list_del(&ctrl->list);
 	mutex_unlock(&nvme_rdma_ctrl_mutex);
 
-	if (ctrl->ctrl.tagset) {
-		blk_cleanup_queue(ctrl->ctrl.connect_q);
-		blk_mq_free_tag_set(&ctrl->tag_set);
-		nvme_rdma_dev_put(ctrl->device);
-	}
 	kfree(ctrl->queues);
 	nvmf_free_options(nctrl->opts);
 free_ctrl:
@@ -748,8 +731,11 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
 	WARN_ON_ONCE(!changed);
 
-	if (ctrl->queue_count > 1)
+	if (ctrl->queue_count > 1) {
 		nvme_start_queues(&ctrl->ctrl);
+		nvme_queue_scan(&ctrl->ctrl);
+		nvme_queue_async_events(&ctrl->ctrl);
+	}
 
 	dev_info(ctrl->ctrl.device, "Successfully reconnected\n");
 
@@ -771,8 +757,13 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)
 {
 	struct nvme_rdma_ctrl *ctrl = container_of(work,
 			struct nvme_rdma_ctrl, err_work);
+	int i;
 
 	nvme_stop_keep_alive(&ctrl->ctrl);
+
+	for (i = 0; i < ctrl->queue_count; i++)
+		clear_bit(NVME_RDMA_Q_CONNECTED, &ctrl->queues[i].flags);
+
 	if (ctrl->queue_count > 1)
 		nvme_stop_queues(&ctrl->ctrl);
 	blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
@@ -855,7 +846,7 @@ static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
 	if (!blk_rq_bytes(rq))
 		return;
 
-	if (req->need_inval) {
+	if (req->mr->need_inval) {
 		res = nvme_rdma_inv_rkey(queue, req);
 		if (res < 0) {
 			dev_err(ctrl->ctrl.device,
@@ -909,7 +900,7 @@ static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,
 
 	sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl));
 	put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length);
-	put_unaligned_le32(queue->device->mr->rkey, sg->key);
+	put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key);
 	sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
 	return 0;
 }
@@ -941,7 +932,7 @@ static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,
 			     IB_ACCESS_REMOTE_READ |
 			     IB_ACCESS_REMOTE_WRITE;
 
-	req->need_inval = true;
+	req->mr->need_inval = true;
 
 	sg->addr = cpu_to_le64(req->mr->iova);
 	put_unaligned_le24(req->mr->length, sg->length);
@@ -964,7 +955,7 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
 
 	req->num_sge = 1;
 	req->inline_data = false;
-	req->need_inval = false;
+	req->mr->need_inval = false;
 
 	c->common.flags |= NVME_CMD_SGL_METABUF;
 
@@ -994,7 +985,7 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
 		    nvme_rdma_queue_idx(queue))
 			return nvme_rdma_map_sg_inline(queue, req, c);
 
-		if (!register_always)
+		if (dev->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)
 			return nvme_rdma_map_sg_single(queue, req, c);
 	}
 
@@ -1151,7 +1142,7 @@ static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
 
 	if ((wc->wc_flags & IB_WC_WITH_INVALIDATE) &&
 	    wc->ex.invalidate_rkey == req->mr->rkey)
-		req->need_inval = false;
+		req->mr->need_inval = false;
 
 	blk_mq_complete_request(rq, status);
 
@@ -1269,7 +1260,7 @@ static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
 {
 	struct nvme_rdma_ctrl *ctrl = queue->ctrl;
 	struct rdma_conn_param param = { };
-	struct nvme_rdma_cm_req priv;
+	struct nvme_rdma_cm_req priv = { };
 	int ret;
 
 	param.qp_num = queue->qp->qp_num;
@@ -1284,8 +1275,22 @@ static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
 
 	priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
 	priv.qid = cpu_to_le16(nvme_rdma_queue_idx(queue));
-	priv.hrqsize = cpu_to_le16(queue->queue_size);
-	priv.hsqsize = cpu_to_le16(queue->queue_size);
+	/*
+	 * set the admin queue depth to the minimum size
+	 * specified by the Fabrics standard.
+	 */
+	if (priv.qid == 0) {
+		priv.hrqsize = cpu_to_le16(NVMF_AQ_DEPTH);
+		priv.hsqsize = cpu_to_le16(NVMF_AQ_DEPTH - 1);
+	} else {
+		/*
+		 * current interpretation of the fabrics spec
+		 * is at minimum you make hrqsize sqsize+1, or a
+		 * 1's based representation of sqsize.
+		 */
+		priv.hrqsize = cpu_to_le16(queue->queue_size);
+		priv.hsqsize = cpu_to_le16(queue->ctrl->ctrl.sqsize);
+	}
 
 	ret = rdma_connect(queue->cm_id, &param);
 	if (ret) {
@@ -1301,56 +1306,6 @@ out_destroy_queue_ib:
 	return ret;
 }
 
-/**
- * nvme_rdma_device_unplug() - Handle RDMA device unplug
- * @queue:      Queue that owns the cm_id that caught the event
- *
- * DEVICE_REMOVAL event notifies us that the RDMA device is about
- * to unplug so we should take care of destroying our RDMA resources.
- * This event will be generated for each allocated cm_id.
- *
- * In our case, the RDMA resources are managed per controller and not
- * only per queue. So the way we handle this is we trigger an implicit
- * controller deletion upon the first DEVICE_REMOVAL event we see, and
- * hold the event inflight until the controller deletion is completed.
- *
- * One exception that we need to handle is the destruction of the cm_id
- * that caught the event. Since we hold the callout until the controller
- * deletion is completed, we'll deadlock if the controller deletion will
- * call rdma_destroy_id on this queue's cm_id. Thus, we claim ownership
- * of destroying this queue before-hand, destroy the queue resources
- * after the controller deletion completed with the exception of destroying
- * the cm_id implicitely by returning a non-zero rc to the callout.
- */
-static int nvme_rdma_device_unplug(struct nvme_rdma_queue *queue)
-{
-	struct nvme_rdma_ctrl *ctrl = queue->ctrl;
-	int ret, ctrl_deleted = 0;
-
-	/* First disable the queue so ctrl delete won't free it */
-	if (!test_and_clear_bit(NVME_RDMA_Q_CONNECTED, &queue->flags))
-		goto out;
-
-	/* delete the controller */
-	ret = __nvme_rdma_del_ctrl(ctrl);
-	if (!ret) {
-		dev_warn(ctrl->ctrl.device,
-			"Got rdma device removal event, deleting ctrl\n");
-		flush_work(&ctrl->delete_work);
-
-		/* Return non-zero so the cm_id will destroy implicitly */
-		ctrl_deleted = 1;
-
-		/* Free this queue ourselves */
-		rdma_disconnect(queue->cm_id);
-		ib_drain_qp(queue->qp);
-		nvme_rdma_destroy_queue_ib(queue);
-	}
-
-out:
-	return ctrl_deleted;
-}
-
 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
 		struct rdma_cm_event *ev)
 {
@@ -1392,8 +1347,8 @@ static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
 		nvme_rdma_error_recovery(queue->ctrl);
 		break;
 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
-		/* return 1 means impliciy CM ID destroy */
-		return nvme_rdma_device_unplug(queue);
+		/* device removal is handled via the ib_client API */
+		break;
 	default:
 		dev_err(queue->ctrl->ctrl.device,
 			"Unexpected RDMA CM event (%d)\n", ev->event);
@@ -1465,7 +1420,7 @@ static int nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
 	if (rq->cmd_type == REQ_TYPE_FS && req_op(rq) == REQ_OP_FLUSH)
 		flush = true;
 	ret = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
-			req->need_inval ? &req->reg_wr.wr : NULL, flush);
+			req->mr->need_inval ? &req->reg_wr.wr : NULL, flush);
 	if (ret) {
 		nvme_rdma_unmap_data(queue, rq);
 		goto err;
@@ -1525,7 +1480,6 @@ static void nvme_rdma_complete_rq(struct request *rq)
 static struct blk_mq_ops nvme_rdma_mq_ops = {
 	.queue_rq	= nvme_rdma_queue_rq,
 	.complete	= nvme_rdma_complete_rq,
-	.map_queue	= blk_mq_map_queue,
 	.init_request	= nvme_rdma_init_request,
 	.exit_request	= nvme_rdma_exit_request,
 	.reinit_request	= nvme_rdma_reinit_request,
@@ -1537,7 +1491,6 @@ static struct blk_mq_ops nvme_rdma_mq_ops = {
 static struct blk_mq_ops nvme_rdma_admin_mq_ops = {
 	.queue_rq	= nvme_rdma_queue_rq,
 	.complete	= nvme_rdma_complete_rq,
-	.map_queue	= blk_mq_map_queue,
 	.init_request	= nvme_rdma_init_admin_request,
 	.exit_request	= nvme_rdma_exit_admin_request,
 	.reinit_request	= nvme_rdma_reinit_request,
@@ -1648,7 +1601,7 @@ static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
 		nvme_rdma_free_io_queues(ctrl);
 	}
 
-	if (ctrl->ctrl.state == NVME_CTRL_LIVE)
+	if (test_bit(NVME_RDMA_Q_CONNECTED, &ctrl->queues[0].flags))
 		nvme_shutdown_ctrl(&ctrl->ctrl);
 
 	blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
@@ -1657,15 +1610,27 @@ static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
 	nvme_rdma_destroy_admin_queue(ctrl);
 }
 
+static void __nvme_rdma_remove_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
+{
+	nvme_uninit_ctrl(&ctrl->ctrl);
+	if (shutdown)
+		nvme_rdma_shutdown_ctrl(ctrl);
+
+	if (ctrl->ctrl.tagset) {
+		blk_cleanup_queue(ctrl->ctrl.connect_q);
+		blk_mq_free_tag_set(&ctrl->tag_set);
+		nvme_rdma_dev_put(ctrl->device);
+	}
+
+	nvme_put_ctrl(&ctrl->ctrl);
+}
+
 static void nvme_rdma_del_ctrl_work(struct work_struct *work)
 {
 	struct nvme_rdma_ctrl *ctrl = container_of(work,
 				struct nvme_rdma_ctrl, delete_work);
 
-	nvme_remove_namespaces(&ctrl->ctrl);
-	nvme_rdma_shutdown_ctrl(ctrl);
-	nvme_uninit_ctrl(&ctrl->ctrl);
-	nvme_put_ctrl(&ctrl->ctrl);
+	__nvme_rdma_remove_ctrl(ctrl, true);
 }
 
 static int __nvme_rdma_del_ctrl(struct nvme_rdma_ctrl *ctrl)
@@ -1682,15 +1647,19 @@ static int __nvme_rdma_del_ctrl(struct nvme_rdma_ctrl *ctrl)
 static int nvme_rdma_del_ctrl(struct nvme_ctrl *nctrl)
 {
 	struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
-	int ret;
+	int ret = 0;
 
+	/*
+	 * Keep a reference until all work is flushed since
+	 * __nvme_rdma_del_ctrl can free the ctrl mem
+	 */
+	if (!kref_get_unless_zero(&ctrl->ctrl.kref))
+		return -EBUSY;
 	ret = __nvme_rdma_del_ctrl(ctrl);
-	if (ret)
-		return ret;
-
-	flush_work(&ctrl->delete_work);
-
-	return 0;
+	if (!ret)
+		flush_work(&ctrl->delete_work);
+	nvme_put_ctrl(&ctrl->ctrl);
+	return ret;
 }
 
 static void nvme_rdma_remove_ctrl_work(struct work_struct *work)
@@ -1698,9 +1667,7 @@ static void nvme_rdma_remove_ctrl_work(struct work_struct *work)
 	struct nvme_rdma_ctrl *ctrl = container_of(work,
 				struct nvme_rdma_ctrl, delete_work);
 
-	nvme_remove_namespaces(&ctrl->ctrl);
-	nvme_uninit_ctrl(&ctrl->ctrl);
-	nvme_put_ctrl(&ctrl->ctrl);
+	__nvme_rdma_remove_ctrl(ctrl, false);
 }
 
 static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
@@ -1739,6 +1706,7 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
 	if (ctrl->queue_count > 1) {
 		nvme_start_queues(&ctrl->ctrl);
 		nvme_queue_scan(&ctrl->ctrl);
+		nvme_queue_async_events(&ctrl->ctrl);
 	}
 
 	return;
@@ -1809,7 +1777,7 @@ static int nvme_rdma_create_io_queues(struct nvme_rdma_ctrl *ctrl)
 
 	memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
 	ctrl->tag_set.ops = &nvme_rdma_mq_ops;
-	ctrl->tag_set.queue_depth = ctrl->ctrl.sqsize;
+	ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
 	ctrl->tag_set.reserved_tags = 1; /* fabric connect */
 	ctrl->tag_set.numa_node = NUMA_NO_NODE;
 	ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
@@ -1907,7 +1875,7 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
 	spin_lock_init(&ctrl->lock);
 
 	ctrl->queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
-	ctrl->ctrl.sqsize = opts->queue_size;
+	ctrl->ctrl.sqsize = opts->queue_size - 1;
 	ctrl->ctrl.kato = opts->kato;
 
 	ret = -ENOMEM;
@@ -1988,27 +1956,57 @@ static struct nvmf_transport_ops nvme_rdma_transport = {
 	.create_ctrl	= nvme_rdma_create_ctrl,
 };
 
+static void nvme_rdma_add_one(struct ib_device *ib_device)
+{
+}
+
+static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
+{
+	struct nvme_rdma_ctrl *ctrl;
+
+	/* Delete all controllers using this device */
+	mutex_lock(&nvme_rdma_ctrl_mutex);
+	list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
+		if (ctrl->device->dev != ib_device)
+			continue;
+		dev_info(ctrl->ctrl.device,
+			"Removing ctrl: NQN \"%s\", addr %pISp\n",
+			ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
+		__nvme_rdma_del_ctrl(ctrl);
+	}
+	mutex_unlock(&nvme_rdma_ctrl_mutex);
+
+	flush_workqueue(nvme_rdma_wq);
+}
+
+static struct ib_client nvme_rdma_ib_client = {
+	.name   = "nvme_rdma",
+	.add = nvme_rdma_add_one,
+	.remove = nvme_rdma_remove_one
+};
+
 static int __init nvme_rdma_init_module(void)
 {
+	int ret;
+
 	nvme_rdma_wq = create_workqueue("nvme_rdma_wq");
 	if (!nvme_rdma_wq)
 		return -ENOMEM;
 
+	ret = ib_register_client(&nvme_rdma_ib_client);
+	if (ret) {
+		destroy_workqueue(nvme_rdma_wq);
+		return ret;
+	}
+
 	nvmf_register_transport(&nvme_rdma_transport);
 	return 0;
 }
 
 static void __exit nvme_rdma_cleanup_module(void)
 {
-	struct nvme_rdma_ctrl *ctrl;
-
 	nvmf_unregister_transport(&nvme_rdma_transport);
-
-	mutex_lock(&nvme_rdma_ctrl_mutex);
-	list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list)
-		__nvme_rdma_del_ctrl(ctrl);
-	mutex_unlock(&nvme_rdma_ctrl_mutex);
-
+	ib_unregister_client(&nvme_rdma_ib_client);
 	destroy_workqueue(nvme_rdma_wq);
 }
 
diff --git a/drivers/nvme/host/scsi.c b/drivers/nvme/host/scsi.c
index e947e298a737..c2a0a1c7d05d 100644
--- a/drivers/nvme/host/scsi.c
+++ b/drivers/nvme/host/scsi.c
@@ -72,15 +72,6 @@ static int sg_version_num = 30534;	/* 2 digits for each component */
 #define ALL_LUNS_RETURNED				0x02
 #define ALL_WELL_KNOWN_LUNS_RETURNED			0x01
 #define RESTRICTED_LUNS_RETURNED			0x00
-#define NVME_POWER_STATE_START_VALID			0x00
-#define NVME_POWER_STATE_ACTIVE				0x01
-#define NVME_POWER_STATE_IDLE				0x02
-#define NVME_POWER_STATE_STANDBY			0x03
-#define NVME_POWER_STATE_LU_CONTROL			0x07
-#define POWER_STATE_0					0
-#define POWER_STATE_1					1
-#define POWER_STATE_2					2
-#define POWER_STATE_3					3
 #define DOWNLOAD_SAVE_ACTIVATE				0x05
 #define DOWNLOAD_SAVE_DEFER_ACTIVATE			0x0E
 #define ACTIVATE_DEFERRED_MICROCODE			0x0F
@@ -915,7 +906,7 @@ static int nvme_trans_log_temperature(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	kfree(smart_log);
 
 	/* Get Features for Temp Threshold */
-	res = nvme_get_features(ns->ctrl, NVME_FEAT_TEMP_THRESH, 0, 0,
+	res = nvme_get_features(ns->ctrl, NVME_FEAT_TEMP_THRESH, 0, NULL, 0,
 								&feature_resp);
 	if (res != NVME_SC_SUCCESS)
 		temp_c_thresh = LOG_TEMP_UNKNOWN;
@@ -1048,7 +1039,7 @@ static int nvme_trans_fill_caching_page(struct nvme_ns *ns,
 	if (len < MODE_PAGE_CACHING_LEN)
 		return -EINVAL;
 
-	nvme_sc = nvme_get_features(ns->ctrl, NVME_FEAT_VOLATILE_WC, 0, 0,
+	nvme_sc = nvme_get_features(ns->ctrl, NVME_FEAT_VOLATILE_WC, 0, NULL, 0,
 								&feature_resp);
 	res = nvme_trans_status_code(hdr, nvme_sc);
 	if (res)
@@ -1229,64 +1220,6 @@ static void nvme_trans_fill_read_cap(u8 *response, struct nvme_id_ns *id_ns,
 
 /* Start Stop Unit Helper Functions */
 
-static int nvme_trans_power_state(struct nvme_ns *ns, struct sg_io_hdr *hdr,
-						u8 pc, u8 pcmod, u8 start)
-{
-	int res;
-	int nvme_sc;
-	struct nvme_id_ctrl *id_ctrl;
-	int lowest_pow_st;	/* max npss = lowest power consumption */
-	unsigned ps_desired = 0;
-
-	nvme_sc = nvme_identify_ctrl(ns->ctrl, &id_ctrl);
-	res = nvme_trans_status_code(hdr, nvme_sc);
-	if (res)
-		return res;
-
-	lowest_pow_st = max(POWER_STATE_0, (int)(id_ctrl->npss - 1));
-	kfree(id_ctrl);
-
-	switch (pc) {
-	case NVME_POWER_STATE_START_VALID:
-		/* Action unspecified if POWER CONDITION MODIFIER != 0 */
-		if (pcmod == 0 && start == 0x1)
-			ps_desired = POWER_STATE_0;
-		if (pcmod == 0 && start == 0x0)
-			ps_desired = lowest_pow_st;
-		break;
-	case NVME_POWER_STATE_ACTIVE:
-		/* Action unspecified if POWER CONDITION MODIFIER != 0 */
-		if (pcmod == 0)
-			ps_desired = POWER_STATE_0;
-		break;
-	case NVME_POWER_STATE_IDLE:
-		/* Action unspecified if POWER CONDITION MODIFIER != [0,1,2] */
-		if (pcmod == 0x0)
-			ps_desired = POWER_STATE_1;
-		else if (pcmod == 0x1)
-			ps_desired = POWER_STATE_2;
-		else if (pcmod == 0x2)
-			ps_desired = POWER_STATE_3;
-		break;
-	case NVME_POWER_STATE_STANDBY:
-		/* Action unspecified if POWER CONDITION MODIFIER != [0,1] */
-		if (pcmod == 0x0)
-			ps_desired = max(POWER_STATE_0, (lowest_pow_st - 2));
-		else if (pcmod == 0x1)
-			ps_desired = max(POWER_STATE_0, (lowest_pow_st - 1));
-		break;
-	case NVME_POWER_STATE_LU_CONTROL:
-	default:
-		res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
-				ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
-				SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
-		break;
-	}
-	nvme_sc = nvme_set_features(ns->ctrl, NVME_FEAT_POWER_MGMT, ps_desired, 0,
-				    NULL);
-	return nvme_trans_status_code(hdr, nvme_sc);
-}
-
 static int nvme_trans_send_activate_fw_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 					u8 buffer_id)
 {
@@ -1395,7 +1328,7 @@ static int nvme_trans_modesel_get_mp(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	case MODE_PAGE_CACHING:
 		dword11 = ((mode_page[2] & CACHING_MODE_PAGE_WCE_MASK) ? 1 : 0);
 		nvme_sc = nvme_set_features(ns->ctrl, NVME_FEAT_VOLATILE_WC,
-					    dword11, 0, NULL);
+					    dword11, NULL, 0, NULL);
 		res = nvme_trans_status_code(hdr, nvme_sc);
 		break;
 	case MODE_PAGE_CONTROL:
@@ -2235,11 +2168,10 @@ static int nvme_trans_synchronize_cache(struct nvme_ns *ns,
 static int nvme_trans_start_stop(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 							u8 *cmd)
 {
-	u8 immed, pcmod, pc, no_flush, start;
+	u8 immed, pcmod, no_flush, start;
 
 	immed = cmd[1] & 0x01;
 	pcmod = cmd[3] & 0x0f;
-	pc = (cmd[4] & 0xf0) >> 4;
 	no_flush = cmd[4] & 0x04;
 	start = cmd[4] & 0x01;
 
@@ -2254,8 +2186,8 @@ static int nvme_trans_start_stop(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 			if (res)
 				return res;
 		}
-		/* Setup the expected power state transition */
-		return nvme_trans_power_state(ns, hdr, pc, pcmod, start);
+
+		return 0;
 	}
 }
 
diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig
index a5c31cbeb481..3a5b9d0576cb 100644
--- a/drivers/nvme/target/Kconfig
+++ b/drivers/nvme/target/Kconfig
@@ -15,8 +15,8 @@ config NVME_TARGET
 
 config NVME_TARGET_LOOP
 	tristate "NVMe loopback device support"
-	depends on BLK_DEV_NVME
 	depends on NVME_TARGET
+	select NVME_CORE
 	select NVME_FABRICS
 	select SG_POOL
 	help
diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index 2fac17a5ad53..7ab9c9381b98 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -13,8 +13,8 @@
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 #include <linux/module.h>
-#include <linux/random.h>
 #include <generated/utsrelease.h>
+#include <asm/unaligned.h>
 #include "nvmet.h"
 
 u32 nvmet_get_log_page_len(struct nvme_command *cmd)
@@ -30,8 +30,84 @@ u32 nvmet_get_log_page_len(struct nvme_command *cmd)
 	return len;
 }
 
+static u16 nvmet_get_smart_log_nsid(struct nvmet_req *req,
+		struct nvme_smart_log *slog)
+{
+	u16 status;
+	struct nvmet_ns *ns;
+	u64 host_reads, host_writes, data_units_read, data_units_written;
+
+	status = NVME_SC_SUCCESS;
+	ns = nvmet_find_namespace(req->sq->ctrl, req->cmd->get_log_page.nsid);
+	if (!ns) {
+		status = NVME_SC_INVALID_NS;
+		pr_err("nvmet : Counld not find namespace id : %d\n",
+				le32_to_cpu(req->cmd->get_log_page.nsid));
+		goto out;
+	}
+
+	host_reads = part_stat_read(ns->bdev->bd_part, ios[READ]);
+	data_units_read = part_stat_read(ns->bdev->bd_part, sectors[READ]);
+	host_writes = part_stat_read(ns->bdev->bd_part, ios[WRITE]);
+	data_units_written = part_stat_read(ns->bdev->bd_part, sectors[WRITE]);
+
+	put_unaligned_le64(host_reads, &slog->host_reads[0]);
+	put_unaligned_le64(data_units_read, &slog->data_units_read[0]);
+	put_unaligned_le64(host_writes, &slog->host_writes[0]);
+	put_unaligned_le64(data_units_written, &slog->data_units_written[0]);
+	nvmet_put_namespace(ns);
+out:
+	return status;
+}
+
+static u16 nvmet_get_smart_log_all(struct nvmet_req *req,
+		struct nvme_smart_log *slog)
+{
+	u16 status;
+	u64 host_reads = 0, host_writes = 0;
+	u64 data_units_read = 0, data_units_written = 0;
+	struct nvmet_ns *ns;
+	struct nvmet_ctrl *ctrl;
+
+	status = NVME_SC_SUCCESS;
+	ctrl = req->sq->ctrl;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link) {
+		host_reads += part_stat_read(ns->bdev->bd_part, ios[READ]);
+		data_units_read +=
+			part_stat_read(ns->bdev->bd_part, sectors[READ]);
+		host_writes += part_stat_read(ns->bdev->bd_part, ios[WRITE]);
+		data_units_written +=
+			part_stat_read(ns->bdev->bd_part, sectors[WRITE]);
+
+	}
+	rcu_read_unlock();
+
+	put_unaligned_le64(host_reads, &slog->host_reads[0]);
+	put_unaligned_le64(data_units_read, &slog->data_units_read[0]);
+	put_unaligned_le64(host_writes, &slog->host_writes[0]);
+	put_unaligned_le64(data_units_written, &slog->data_units_written[0]);
+
+	return status;
+}
+
+static u16 nvmet_get_smart_log(struct nvmet_req *req,
+		struct nvme_smart_log *slog)
+{
+	u16 status;
+
+	WARN_ON(req == NULL || slog == NULL);
+	if (req->cmd->get_log_page.nsid == 0xFFFFFFFF)
+		status = nvmet_get_smart_log_all(req, slog);
+	else
+		status = nvmet_get_smart_log_nsid(req, slog);
+	return status;
+}
+
 static void nvmet_execute_get_log_page(struct nvmet_req *req)
 {
+	struct nvme_smart_log *smart_log;
 	size_t data_len = nvmet_get_log_page_len(req->cmd);
 	void *buf;
 	u16 status = 0;
@@ -60,6 +136,16 @@ static void nvmet_execute_get_log_page(struct nvmet_req *req)
 		 * available (e.g. units or commands read/written) those aren't
 		 * persistent over power loss.
 		 */
+		if (data_len != sizeof(*smart_log)) {
+			status = NVME_SC_INTERNAL;
+			goto err;
+		}
+		smart_log = buf;
+		status = nvmet_get_smart_log(req, smart_log);
+		if (status) {
+			memset(buf, '\0', data_len);
+			goto err;
+		}
 		break;
 	case 0x03:
 		/*
@@ -74,6 +160,7 @@ static void nvmet_execute_get_log_page(struct nvmet_req *req)
 
 	status = nvmet_copy_to_sgl(req, 0, buf, data_len);
 
+err:
 	kfree(buf);
 out:
 	nvmet_req_complete(req, status);
@@ -83,7 +170,6 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 {
 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
 	struct nvme_id_ctrl *id;
-	u64 serial;
 	u16 status = 0;
 
 	id = kzalloc(sizeof(*id), GFP_KERNEL);
@@ -96,10 +182,8 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 	id->vid = 0;
 	id->ssvid = 0;
 
-	/* generate a random serial number as our controllers are ephemeral: */
-	get_random_bytes(&serial, sizeof(serial));
 	memset(id->sn, ' ', sizeof(id->sn));
-	snprintf(id->sn, sizeof(id->sn), "%llx", serial);
+	snprintf(id->sn, sizeof(id->sn), "%llx", ctrl->serial);
 
 	memset(id->mn, ' ', sizeof(id->mn));
 	strncpy((char *)id->mn, "Linux", sizeof(id->mn));
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index 8a891ca53367..6559d5afa7bf 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -13,6 +13,7 @@
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 #include <linux/module.h>
+#include <linux/random.h>
 #include "nvmet.h"
 
 static struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX];
@@ -728,6 +729,9 @@ u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn,
 	memcpy(ctrl->subsysnqn, subsysnqn, NVMF_NQN_SIZE);
 	memcpy(ctrl->hostnqn, hostnqn, NVMF_NQN_SIZE);
 
+	/* generate a random serial number as our controllers are ephemeral: */
+	get_random_bytes(&ctrl->serial, sizeof(ctrl->serial));
+
 	kref_init(&ctrl->ref);
 	ctrl->subsys = subsys;
 
diff --git a/drivers/nvme/target/io-cmd.c b/drivers/nvme/target/io-cmd.c
index 2cd069b691ae..4a96c2049b7b 100644
--- a/drivers/nvme/target/io-cmd.c
+++ b/drivers/nvme/target/io-cmd.c
@@ -58,6 +58,7 @@ static void nvmet_execute_rw(struct nvmet_req *req)
 
 	if (req->cmd->rw.opcode == nvme_cmd_write) {
 		op = REQ_OP_WRITE;
+		op_flags = WRITE_ODIRECT;
 		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
 			op_flags |= REQ_FUA;
 	} else {
@@ -205,7 +206,7 @@ int nvmet_parse_io_cmd(struct nvmet_req *req)
 		return 0;
 	case nvme_cmd_dsm:
 		req->execute = nvmet_execute_dsm;
-		req->data_len = le32_to_cpu(cmd->dsm.nr) *
+		req->data_len = le32_to_cpu(cmd->dsm.nr + 1) *
 			sizeof(struct nvme_dsm_range);
 		return 0;
 	default:
diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c
index 94e782987cc9..d5df77d686b2 100644
--- a/drivers/nvme/target/loop.c
+++ b/drivers/nvme/target/loop.c
@@ -273,7 +273,6 @@ static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 static struct blk_mq_ops nvme_loop_mq_ops = {
 	.queue_rq	= nvme_loop_queue_rq,
 	.complete	= nvme_loop_complete_rq,
-	.map_queue	= blk_mq_map_queue,
 	.init_request	= nvme_loop_init_request,
 	.init_hctx	= nvme_loop_init_hctx,
 	.timeout	= nvme_loop_timeout,
@@ -282,7 +281,6 @@ static struct blk_mq_ops nvme_loop_mq_ops = {
 static struct blk_mq_ops nvme_loop_admin_mq_ops = {
 	.queue_rq	= nvme_loop_queue_rq,
 	.complete	= nvme_loop_complete_rq,
-	.map_queue	= blk_mq_map_queue,
 	.init_request	= nvme_loop_init_admin_request,
 	.init_hctx	= nvme_loop_init_admin_hctx,
 	.timeout	= nvme_loop_timeout,
@@ -414,9 +412,8 @@ static void nvme_loop_del_ctrl_work(struct work_struct *work)
 	struct nvme_loop_ctrl *ctrl = container_of(work,
 				struct nvme_loop_ctrl, delete_work);
 
-	nvme_remove_namespaces(&ctrl->ctrl);
-	nvme_loop_shutdown_ctrl(ctrl);
 	nvme_uninit_ctrl(&ctrl->ctrl);
+	nvme_loop_shutdown_ctrl(ctrl);
 	nvme_put_ctrl(&ctrl->ctrl);
 }
 
@@ -501,7 +498,6 @@ out_free_queues:
 	nvme_loop_destroy_admin_queue(ctrl);
 out_disable:
 	dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
-	nvme_remove_namespaces(&ctrl->ctrl);
 	nvme_uninit_ctrl(&ctrl->ctrl);
 	nvme_put_ctrl(&ctrl->ctrl);
 }
@@ -558,7 +554,7 @@ static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
 
 	memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
 	ctrl->tag_set.ops = &nvme_loop_mq_ops;
-	ctrl->tag_set.queue_depth = ctrl->ctrl.sqsize;
+	ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
 	ctrl->tag_set.reserved_tags = 1; /* fabric connect */
 	ctrl->tag_set.numa_node = NUMA_NO_NODE;
 	ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
@@ -622,7 +618,7 @@ static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev,
 
 	ret = -ENOMEM;
 
-	ctrl->ctrl.sqsize = opts->queue_size;
+	ctrl->ctrl.sqsize = opts->queue_size - 1;
 	ctrl->ctrl.kato = opts->kato;
 
 	ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues),
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 57dd6d834c28..76b6eedccaf9 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -113,6 +113,7 @@ struct nvmet_ctrl {
 
 	struct mutex		lock;
 	u64			cap;
+	u64			serial;
 	u32			cc;
 	u32			csts;
 
diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index e06d504bdf0c..f8d23999e0f2 100644
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -77,6 +77,7 @@ enum nvmet_rdma_queue_state {
 	NVMET_RDMA_Q_CONNECTING,
 	NVMET_RDMA_Q_LIVE,
 	NVMET_RDMA_Q_DISCONNECTING,
+	NVMET_RDMA_IN_DEVICE_REMOVAL,
 };
 
 struct nvmet_rdma_queue {
@@ -615,15 +616,10 @@ static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp,
 	if (!len)
 		return 0;
 
-	/* use the already allocated data buffer if possible */
-	if (len <= NVMET_RDMA_INLINE_DATA_SIZE && rsp->queue->host_qid) {
-		nvmet_rdma_use_inline_sg(rsp, len, 0);
-	} else {
-		status = nvmet_rdma_alloc_sgl(&rsp->req.sg, &rsp->req.sg_cnt,
-				len);
-		if (status)
-			return status;
-	}
+	status = nvmet_rdma_alloc_sgl(&rsp->req.sg, &rsp->req.sg_cnt,
+			len);
+	if (status)
+		return status;
 
 	ret = rdma_rw_ctx_init(&rsp->rw, cm_id->qp, cm_id->port_num,
 			rsp->req.sg, rsp->req.sg_cnt, 0, addr, key,
@@ -852,7 +848,7 @@ nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id)
 	ndev->device = cm_id->device;
 	kref_init(&ndev->ref);
 
-	ndev->pd = ib_alloc_pd(ndev->device);
+	ndev->pd = ib_alloc_pd(ndev->device, 0);
 	if (IS_ERR(ndev->pd))
 		goto out_free_dev;
 
@@ -982,9 +978,13 @@ static void nvmet_rdma_release_queue_work(struct work_struct *w)
 		container_of(w, struct nvmet_rdma_queue, release_work);
 	struct rdma_cm_id *cm_id = queue->cm_id;
 	struct nvmet_rdma_device *dev = queue->dev;
+	enum nvmet_rdma_queue_state state = queue->state;
 
 	nvmet_rdma_free_queue(queue);
-	rdma_destroy_id(cm_id);
+
+	if (state != NVMET_RDMA_IN_DEVICE_REMOVAL)
+		rdma_destroy_id(cm_id);
+
 	kref_put(&dev->ref, nvmet_rdma_free_dev);
 }
 
@@ -1004,10 +1004,10 @@ nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param *conn,
 	queue->host_qid = le16_to_cpu(req->qid);
 
 	/*
-	 * req->hsqsize corresponds to our recv queue size
+	 * req->hsqsize corresponds to our recv queue size plus 1
 	 * req->hrqsize corresponds to our send queue size
 	 */
-	queue->recv_queue_size = le16_to_cpu(req->hsqsize);
+	queue->recv_queue_size = le16_to_cpu(req->hsqsize) + 1;
 	queue->send_queue_size = le16_to_cpu(req->hrqsize);
 
 	if (!queue->host_qid && queue->recv_queue_size > NVMF_AQ_DEPTH)
@@ -1233,8 +1233,9 @@ static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
 	switch (queue->state) {
 	case NVMET_RDMA_Q_CONNECTING:
 	case NVMET_RDMA_Q_LIVE:
-		disconnect = true;
 		queue->state = NVMET_RDMA_Q_DISCONNECTING;
+	case NVMET_RDMA_IN_DEVICE_REMOVAL:
+		disconnect = true;
 		break;
 	case NVMET_RDMA_Q_DISCONNECTING:
 		break;
@@ -1272,6 +1273,62 @@ static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id,
 	schedule_work(&queue->release_work);
 }
 
+/**
+ * nvme_rdma_device_removal() - Handle RDMA device removal
+ * @queue:      nvmet rdma queue (cm id qp_context)
+ * @addr:	nvmet address (cm_id context)
+ *
+ * DEVICE_REMOVAL event notifies us that the RDMA device is about
+ * to unplug so we should take care of destroying our RDMA resources.
+ * This event will be generated for each allocated cm_id.
+ *
+ * Note that this event can be generated on a normal queue cm_id
+ * and/or a device bound listener cm_id (where in this case
+ * queue will be null).
+ *
+ * we claim ownership on destroying the cm_id. For queues we move
+ * the queue state to NVMET_RDMA_IN_DEVICE_REMOVAL and for port
+ * we nullify the priv to prevent double cm_id destruction and destroying
+ * the cm_id implicitely by returning a non-zero rc to the callout.
+ */
+static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id,
+		struct nvmet_rdma_queue *queue)
+{
+	unsigned long flags;
+
+	if (!queue) {
+		struct nvmet_port *port = cm_id->context;
+
+		/*
+		 * This is a listener cm_id. Make sure that
+		 * future remove_port won't invoke a double
+		 * cm_id destroy. use atomic xchg to make sure
+		 * we don't compete with remove_port.
+		 */
+		if (xchg(&port->priv, NULL) != cm_id)
+			return 0;
+	} else {
+		/*
+		 * This is a queue cm_id. Make sure that
+		 * release queue will not destroy the cm_id
+		 * and schedule all ctrl queues removal (only
+		 * if the queue is not disconnecting already).
+		 */
+		spin_lock_irqsave(&queue->state_lock, flags);
+		if (queue->state != NVMET_RDMA_Q_DISCONNECTING)
+			queue->state = NVMET_RDMA_IN_DEVICE_REMOVAL;
+		spin_unlock_irqrestore(&queue->state_lock, flags);
+		nvmet_rdma_queue_disconnect(queue);
+		flush_scheduled_work();
+	}
+
+	/*
+	 * We need to return 1 so that the core will destroy
+	 * it's own ID.  What a great API design..
+	 */
+	return 1;
+}
+
 static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id,
 		struct rdma_cm_event *event)
 {
@@ -1294,20 +1351,11 @@ static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id,
 		break;
 	case RDMA_CM_EVENT_ADDR_CHANGE:
 	case RDMA_CM_EVENT_DISCONNECTED:
-	case RDMA_CM_EVENT_DEVICE_REMOVAL:
 	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
-		/*
-		 * We can get the device removal callback even for a
-		 * CM ID that we aren't actually using.  In that case
-		 * the context pointer is NULL, so we shouldn't try
-		 * to disconnect a non-existing queue.  But we also
-		 * need to return 1 so that the core will destroy
-		 * it's own ID.  What a great API design..
-		 */
-		if (queue)
-			nvmet_rdma_queue_disconnect(queue);
-		else
-			ret = 1;
+		nvmet_rdma_queue_disconnect(queue);
+		break;
+	case RDMA_CM_EVENT_DEVICE_REMOVAL:
+		ret = nvmet_rdma_device_removal(cm_id, queue);
 		break;
 	case RDMA_CM_EVENT_REJECTED:
 	case RDMA_CM_EVENT_UNREACHABLE:
@@ -1396,9 +1444,10 @@ out_destroy_id:
 
 static void nvmet_rdma_remove_port(struct nvmet_port *port)
 {
-	struct rdma_cm_id *cm_id = port->priv;
+	struct rdma_cm_id *cm_id = xchg(&port->priv, NULL);
 
-	rdma_destroy_id(cm_id);
+	if (cm_id)
+		rdma_destroy_id(cm_id);
 }
 
 static struct nvmet_fabrics_ops nvmet_rdma_ops = {
diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig
index f550c4596a7a..ba140eaee5c8 100644
--- a/drivers/nvmem/Kconfig
+++ b/drivers/nvmem/Kconfig
@@ -101,4 +101,14 @@ config NVMEM_VF610_OCOTP
 	  This driver can also be build as a module. If so, the module will
 	  be called nvmem-vf610-ocotp.
 
+config MESON_EFUSE
+	tristate "Amlogic eFuse Support"
+	depends on (ARCH_MESON || COMPILE_TEST) && MESON_SM
+	help
+	  This is a driver to retrieve specific values from the eFuse found on
+	  the Amlogic Meson SoCs.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called nvmem_meson_efuse.
+
 endif
diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile
index 45ab1ae08fa9..8f942a0cdaec 100644
--- a/drivers/nvmem/Makefile
+++ b/drivers/nvmem/Makefile
@@ -22,3 +22,5 @@ obj-$(CONFIG_NVMEM_SUNXI_SID)	+= nvmem_sunxi_sid.o
 nvmem_sunxi_sid-y		:= sunxi_sid.o
 obj-$(CONFIG_NVMEM_VF610_OCOTP)	+= nvmem-vf610-ocotp.o
 nvmem-vf610-ocotp-y		:= vf610-ocotp.o
+obj-$(CONFIG_MESON_EFUSE)	+= nvmem_meson_efuse.o
+nvmem_meson_efuse-y		:= meson-efuse.o
diff --git a/drivers/nvmem/meson-efuse.c b/drivers/nvmem/meson-efuse.c
new file mode 100644
index 000000000000..f207c3b10482
--- /dev/null
+++ b/drivers/nvmem/meson-efuse.c
@@ -0,0 +1,93 @@
+/*
+ * Amlogic eFuse Driver
+ *
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/module.h>
+#include <linux/nvmem-provider.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include <linux/firmware/meson/meson_sm.h>
+
+static int meson_efuse_read(void *context, unsigned int offset,
+			    void *val, size_t bytes)
+{
+	u8 *buf = val;
+	int ret;
+
+	ret = meson_sm_call_read(buf, SM_EFUSE_READ, offset,
+				 bytes, 0, 0, 0);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static struct nvmem_config econfig = {
+	.name = "meson-efuse",
+	.owner = THIS_MODULE,
+	.stride = 1,
+	.word_size = 1,
+	.read_only = true,
+};
+
+static const struct of_device_id meson_efuse_match[] = {
+	{ .compatible = "amlogic,meson-gxbb-efuse", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, meson_efuse_match);
+
+static int meson_efuse_probe(struct platform_device *pdev)
+{
+	struct nvmem_device *nvmem;
+	unsigned int size;
+
+	if (meson_sm_call(SM_EFUSE_USER_MAX, &size, 0, 0, 0, 0, 0) < 0)
+		return -EINVAL;
+
+	econfig.dev = &pdev->dev;
+	econfig.reg_read = meson_efuse_read;
+	econfig.size = size;
+
+	nvmem = nvmem_register(&econfig);
+	if (IS_ERR(nvmem))
+		return PTR_ERR(nvmem);
+
+	platform_set_drvdata(pdev, nvmem);
+
+	return 0;
+}
+
+static int meson_efuse_remove(struct platform_device *pdev)
+{
+	struct nvmem_device *nvmem = platform_get_drvdata(pdev);
+
+	return nvmem_unregister(nvmem);
+}
+
+static struct platform_driver meson_efuse_driver = {
+	.probe = meson_efuse_probe,
+	.remove = meson_efuse_remove,
+	.driver = {
+		.name = "meson-efuse",
+		.of_match_table = meson_efuse_match,
+	},
+};
+
+module_platform_driver(meson_efuse_driver);
+
+MODULE_AUTHOR("Carlo Caione <carlo@endlessm.com>");
+MODULE_DESCRIPTION("Amlogic Meson NVMEM driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/nvmem/rockchip-efuse.c b/drivers/nvmem/rockchip-efuse.c
index 4d3f391f0a0b..423907bdd259 100644
--- a/drivers/nvmem/rockchip-efuse.c
+++ b/drivers/nvmem/rockchip-efuse.c
@@ -22,17 +22,29 @@
 #include <linux/nvmem-provider.h>
 #include <linux/slab.h>
 #include <linux/of.h>
+#include <linux/of_platform.h>
 #include <linux/platform_device.h>
 
-#define EFUSE_A_SHIFT			6
-#define EFUSE_A_MASK			0x3ff
-#define EFUSE_PGENB			BIT(3)
-#define EFUSE_LOAD			BIT(2)
-#define EFUSE_STROBE			BIT(1)
-#define EFUSE_CSB			BIT(0)
-
-#define REG_EFUSE_CTRL			0x0000
-#define REG_EFUSE_DOUT			0x0004
+#define RK3288_A_SHIFT		6
+#define RK3288_A_MASK		0x3ff
+#define RK3288_PGENB		BIT(3)
+#define RK3288_LOAD		BIT(2)
+#define RK3288_STROBE		BIT(1)
+#define RK3288_CSB		BIT(0)
+
+#define RK3399_A_SHIFT		16
+#define RK3399_A_MASK		0x3ff
+#define RK3399_NBYTES		4
+#define RK3399_STROBSFTSEL	BIT(9)
+#define RK3399_RSB		BIT(7)
+#define RK3399_PD		BIT(5)
+#define RK3399_PGENB		BIT(3)
+#define RK3399_LOAD		BIT(2)
+#define RK3399_STROBE		BIT(1)
+#define RK3399_CSB		BIT(0)
+
+#define REG_EFUSE_CTRL		0x0000
+#define REG_EFUSE_DOUT		0x0004
 
 struct rockchip_efuse_chip {
 	struct device *dev;
@@ -40,8 +52,8 @@ struct rockchip_efuse_chip {
 	struct clk *clk;
 };
 
-static int rockchip_efuse_read(void *context, unsigned int offset,
-			       void *val, size_t bytes)
+static int rockchip_rk3288_efuse_read(void *context, unsigned int offset,
+				      void *val, size_t bytes)
 {
 	struct rockchip_efuse_chip *efuse = context;
 	u8 *buf = val;
@@ -53,27 +65,82 @@ static int rockchip_efuse_read(void *context, unsigned int offset,
 		return ret;
 	}
 
-	writel(EFUSE_LOAD | EFUSE_PGENB, efuse->base + REG_EFUSE_CTRL);
+	writel(RK3288_LOAD | RK3288_PGENB, efuse->base + REG_EFUSE_CTRL);
 	udelay(1);
 	while (bytes--) {
 		writel(readl(efuse->base + REG_EFUSE_CTRL) &
-			     (~(EFUSE_A_MASK << EFUSE_A_SHIFT)),
+			     (~(RK3288_A_MASK << RK3288_A_SHIFT)),
 			     efuse->base + REG_EFUSE_CTRL);
 		writel(readl(efuse->base + REG_EFUSE_CTRL) |
-			     ((offset++ & EFUSE_A_MASK) << EFUSE_A_SHIFT),
+			     ((offset++ & RK3288_A_MASK) << RK3288_A_SHIFT),
 			     efuse->base + REG_EFUSE_CTRL);
 		udelay(1);
 		writel(readl(efuse->base + REG_EFUSE_CTRL) |
-			     EFUSE_STROBE, efuse->base + REG_EFUSE_CTRL);
+			     RK3288_STROBE, efuse->base + REG_EFUSE_CTRL);
 		udelay(1);
 		*buf++ = readb(efuse->base + REG_EFUSE_DOUT);
 		writel(readl(efuse->base + REG_EFUSE_CTRL) &
-		     (~EFUSE_STROBE), efuse->base + REG_EFUSE_CTRL);
+		       (~RK3288_STROBE), efuse->base + REG_EFUSE_CTRL);
+		udelay(1);
+	}
+
+	/* Switch to standby mode */
+	writel(RK3288_PGENB | RK3288_CSB, efuse->base + REG_EFUSE_CTRL);
+
+	clk_disable_unprepare(efuse->clk);
+
+	return 0;
+}
+
+static int rockchip_rk3399_efuse_read(void *context, unsigned int offset,
+				      void *val, size_t bytes)
+{
+	struct rockchip_efuse_chip *efuse = context;
+	unsigned int addr_start, addr_end, addr_offset, addr_len;
+	u32 out_value;
+	u8 *buf;
+	int ret, i = 0;
+
+	ret = clk_prepare_enable(efuse->clk);
+	if (ret < 0) {
+		dev_err(efuse->dev, "failed to prepare/enable efuse clk\n");
+		return ret;
+	}
+
+	addr_start = rounddown(offset, RK3399_NBYTES) / RK3399_NBYTES;
+	addr_end = roundup(offset + bytes, RK3399_NBYTES) / RK3399_NBYTES;
+	addr_offset = offset % RK3399_NBYTES;
+	addr_len = addr_end - addr_start;
+
+	buf = kzalloc(sizeof(*buf) * addr_len * RK3399_NBYTES, GFP_KERNEL);
+	if (!buf) {
+		clk_disable_unprepare(efuse->clk);
+		return -ENOMEM;
+	}
+
+	writel(RK3399_LOAD | RK3399_PGENB | RK3399_STROBSFTSEL | RK3399_RSB,
+	       efuse->base + REG_EFUSE_CTRL);
+	udelay(1);
+	while (addr_len--) {
+		writel(readl(efuse->base + REG_EFUSE_CTRL) | RK3399_STROBE |
+		       ((addr_start++ & RK3399_A_MASK) << RK3399_A_SHIFT),
+		       efuse->base + REG_EFUSE_CTRL);
 		udelay(1);
+		out_value = readl(efuse->base + REG_EFUSE_DOUT);
+		writel(readl(efuse->base + REG_EFUSE_CTRL) & (~RK3399_STROBE),
+		       efuse->base + REG_EFUSE_CTRL);
+		udelay(1);
+
+		memcpy(&buf[i], &out_value, RK3399_NBYTES);
+		i += RK3399_NBYTES;
 	}
 
 	/* Switch to standby mode */
-	writel(EFUSE_PGENB | EFUSE_CSB, efuse->base + REG_EFUSE_CTRL);
+	writel(RK3399_PD | RK3399_CSB, efuse->base + REG_EFUSE_CTRL);
+
+	memcpy(val, buf + addr_offset, bytes);
+
+	kfree(buf);
 
 	clk_disable_unprepare(efuse->clk);
 
@@ -89,7 +156,27 @@ static struct nvmem_config econfig = {
 };
 
 static const struct of_device_id rockchip_efuse_match[] = {
-	{ .compatible = "rockchip,rockchip-efuse", },
+	/* deprecated but kept around for dts binding compatibility */
+	{
+		.compatible = "rockchip,rockchip-efuse",
+		.data = (void *)&rockchip_rk3288_efuse_read,
+	},
+	{
+		.compatible = "rockchip,rk3066a-efuse",
+		.data = (void *)&rockchip_rk3288_efuse_read,
+	},
+	{
+		.compatible = "rockchip,rk3188-efuse",
+		.data = (void *)&rockchip_rk3288_efuse_read,
+	},
+	{
+		.compatible = "rockchip,rk3288-efuse",
+		.data = (void *)&rockchip_rk3288_efuse_read,
+	},
+	{
+		.compatible = "rockchip,rk3399-efuse",
+		.data = (void *)&rockchip_rk3399_efuse_read,
+	},
 	{ /* sentinel */},
 };
 MODULE_DEVICE_TABLE(of, rockchip_efuse_match);
@@ -99,6 +186,14 @@ static int rockchip_efuse_probe(struct platform_device *pdev)
 	struct resource *res;
 	struct nvmem_device *nvmem;
 	struct rockchip_efuse_chip *efuse;
+	const struct of_device_id *match;
+	struct device *dev = &pdev->dev;
+
+	match = of_match_device(dev->driver->of_match_table, dev);
+	if (!match || !match->data) {
+		dev_err(dev, "failed to get match data\n");
+		return -EINVAL;
+	}
 
 	efuse = devm_kzalloc(&pdev->dev, sizeof(struct rockchip_efuse_chip),
 			     GFP_KERNEL);
@@ -116,7 +211,7 @@ static int rockchip_efuse_probe(struct platform_device *pdev)
 
 	efuse->dev = &pdev->dev;
 	econfig.size = resource_size(res);
-	econfig.reg_read = rockchip_efuse_read;
+	econfig.reg_read = match->data;
 	econfig.priv = efuse;
 	econfig.dev = efuse->dev;
 	nvmem = nvmem_register(&econfig);
diff --git a/drivers/of/Kconfig b/drivers/of/Kconfig
index bc07ad30c9bf..ba7b034b2b91 100644
--- a/drivers/of/Kconfig
+++ b/drivers/of/Kconfig
@@ -90,10 +90,6 @@ config OF_PCI_IRQ
 	help
 	  OpenFirmware PCI IRQ routing helpers
 
-config OF_MTD
-	depends on MTD
-	def_bool y
-
 config OF_RESERVED_MEM
 	depends on OF_EARLY_FLATTREE
 	bool
diff --git a/drivers/of/base.c b/drivers/of/base.c
index 7792266db259..d687e6de24a0 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -1146,16 +1146,18 @@ EXPORT_SYMBOL_GPL(of_property_count_elems_of_size);
  *
  * @np:		device node from which the property value is to be read.
  * @propname:	name of the property to be searched.
- * @len:	requested length of property value
+ * @min:	minimum allowed length of property value
+ * @max:	maximum allowed length of property value (0 means unlimited)
+ * @len:	if !=NULL, actual length is written to here
  *
  * Search for a property in a device node and valid the requested size.
  * Returns the property value on success, -EINVAL if the property does not
  *  exist, -ENODATA if property does not have a value, and -EOVERFLOW if the
- * property data isn't large enough.
+ * property data is too small or too large.
  *
  */
 static void *of_find_property_value_of_size(const struct device_node *np,
-			const char *propname, u32 len)
+			const char *propname, u32 min, u32 max, size_t *len)
 {
 	struct property *prop = of_find_property(np, propname, NULL);
 
@@ -1163,9 +1165,14 @@ static void *of_find_property_value_of_size(const struct device_node *np,
 		return ERR_PTR(-EINVAL);
 	if (!prop->value)
 		return ERR_PTR(-ENODATA);
-	if (len > prop->length)
+	if (prop->length < min)
+		return ERR_PTR(-EOVERFLOW);
+	if (max && prop->length > max)
 		return ERR_PTR(-EOVERFLOW);
 
+	if (len)
+		*len = prop->length;
+
 	return prop->value;
 }
 
@@ -1189,7 +1196,9 @@ int of_property_read_u32_index(const struct device_node *np,
 				       u32 index, u32 *out_value)
 {
 	const u32 *val = of_find_property_value_of_size(np, propname,
-					((index + 1) * sizeof(*out_value)));
+					((index + 1) * sizeof(*out_value)),
+					0,
+					NULL);
 
 	if (IS_ERR(val))
 		return PTR_ERR(val);
@@ -1200,102 +1209,145 @@ int of_property_read_u32_index(const struct device_node *np,
 EXPORT_SYMBOL_GPL(of_property_read_u32_index);
 
 /**
- * of_property_read_u8_array - Find and read an array of u8 from a property.
+ * of_property_read_variable_u8_array - Find and read an array of u8 from a
+ * property, with bounds on the minimum and maximum array size.
  *
  * @np:		device node from which the property value is to be read.
  * @propname:	name of the property to be searched.
  * @out_values:	pointer to return value, modified only if return value is 0.
- * @sz:		number of array elements to read
+ * @sz_min:	minimum number of array elements to read
+ * @sz_max:	maximum number of array elements to read, if zero there is no
+ *		upper limit on the number of elements in the dts entry but only
+ *		sz_min will be read.
  *
  * Search for a property in a device node and read 8-bit value(s) from
- * it. Returns 0 on success, -EINVAL if the property does not exist,
- * -ENODATA if property does not have a value, and -EOVERFLOW if the
- * property data isn't large enough.
+ * it. Returns number of elements read on success, -EINVAL if the property
+ * does not exist, -ENODATA if property does not have a value, and -EOVERFLOW
+ * if the property data is smaller than sz_min or longer than sz_max.
  *
  * dts entry of array should be like:
  *	property = /bits/ 8 <0x50 0x60 0x70>;
  *
  * The out_values is modified only if a valid u8 value can be decoded.
  */
-int of_property_read_u8_array(const struct device_node *np,
-			const char *propname, u8 *out_values, size_t sz)
+int of_property_read_variable_u8_array(const struct device_node *np,
+					const char *propname, u8 *out_values,
+					size_t sz_min, size_t sz_max)
 {
+	size_t sz, count;
 	const u8 *val = of_find_property_value_of_size(np, propname,
-						(sz * sizeof(*out_values)));
+						(sz_min * sizeof(*out_values)),
+						(sz_max * sizeof(*out_values)),
+						&sz);
 
 	if (IS_ERR(val))
 		return PTR_ERR(val);
 
-	while (sz--)
+	if (!sz_max)
+		sz = sz_min;
+	else
+		sz /= sizeof(*out_values);
+
+	count = sz;
+	while (count--)
 		*out_values++ = *val++;
-	return 0;
+
+	return sz;
 }
-EXPORT_SYMBOL_GPL(of_property_read_u8_array);
+EXPORT_SYMBOL_GPL(of_property_read_variable_u8_array);
 
 /**
- * of_property_read_u16_array - Find and read an array of u16 from a property.
+ * of_property_read_variable_u16_array - Find and read an array of u16 from a
+ * property, with bounds on the minimum and maximum array size.
  *
  * @np:		device node from which the property value is to be read.
  * @propname:	name of the property to be searched.
  * @out_values:	pointer to return value, modified only if return value is 0.
- * @sz:		number of array elements to read
+ * @sz_min:	minimum number of array elements to read
+ * @sz_max:	maximum number of array elements to read, if zero there is no
+ *		upper limit on the number of elements in the dts entry but only
+ *		sz_min will be read.
  *
  * Search for a property in a device node and read 16-bit value(s) from
- * it. Returns 0 on success, -EINVAL if the property does not exist,
- * -ENODATA if property does not have a value, and -EOVERFLOW if the
- * property data isn't large enough.
+ * it. Returns number of elements read on success, -EINVAL if the property
+ * does not exist, -ENODATA if property does not have a value, and -EOVERFLOW
+ * if the property data is smaller than sz_min or longer than sz_max.
  *
  * dts entry of array should be like:
  *	property = /bits/ 16 <0x5000 0x6000 0x7000>;
  *
  * The out_values is modified only if a valid u16 value can be decoded.
  */
-int of_property_read_u16_array(const struct device_node *np,
-			const char *propname, u16 *out_values, size_t sz)
+int of_property_read_variable_u16_array(const struct device_node *np,
+					const char *propname, u16 *out_values,
+					size_t sz_min, size_t sz_max)
 {
+	size_t sz, count;
 	const __be16 *val = of_find_property_value_of_size(np, propname,
-						(sz * sizeof(*out_values)));
+						(sz_min * sizeof(*out_values)),
+						(sz_max * sizeof(*out_values)),
+						&sz);
 
 	if (IS_ERR(val))
 		return PTR_ERR(val);
 
-	while (sz--)
+	if (!sz_max)
+		sz = sz_min;
+	else
+		sz /= sizeof(*out_values);
+
+	count = sz;
+	while (count--)
 		*out_values++ = be16_to_cpup(val++);
-	return 0;
+
+	return sz;
 }
-EXPORT_SYMBOL_GPL(of_property_read_u16_array);
+EXPORT_SYMBOL_GPL(of_property_read_variable_u16_array);
 
 /**
- * of_property_read_u32_array - Find and read an array of 32 bit integers
- * from a property.
+ * of_property_read_variable_u32_array - Find and read an array of 32 bit
+ * integers from a property, with bounds on the minimum and maximum array size.
  *
  * @np:		device node from which the property value is to be read.
  * @propname:	name of the property to be searched.
  * @out_values:	pointer to return value, modified only if return value is 0.
- * @sz:		number of array elements to read
+ * @sz_min:	minimum number of array elements to read
+ * @sz_max:	maximum number of array elements to read, if zero there is no
+ *		upper limit on the number of elements in the dts entry but only
+ *		sz_min will be read.
  *
  * Search for a property in a device node and read 32-bit value(s) from
- * it. Returns 0 on success, -EINVAL if the property does not exist,
- * -ENODATA if property does not have a value, and -EOVERFLOW if the
- * property data isn't large enough.
+ * it. Returns number of elements read on success, -EINVAL if the property
+ * does not exist, -ENODATA if property does not have a value, and -EOVERFLOW
+ * if the property data is smaller than sz_min or longer than sz_max.
  *
  * The out_values is modified only if a valid u32 value can be decoded.
  */
-int of_property_read_u32_array(const struct device_node *np,
+int of_property_read_variable_u32_array(const struct device_node *np,
 			       const char *propname, u32 *out_values,
-			       size_t sz)
+			       size_t sz_min, size_t sz_max)
 {
+	size_t sz, count;
 	const __be32 *val = of_find_property_value_of_size(np, propname,
-						(sz * sizeof(*out_values)));
+						(sz_min * sizeof(*out_values)),
+						(sz_max * sizeof(*out_values)),
+						&sz);
 
 	if (IS_ERR(val))
 		return PTR_ERR(val);
 
-	while (sz--)
+	if (!sz_max)
+		sz = sz_min;
+	else
+		sz /= sizeof(*out_values);
+
+	count = sz;
+	while (count--)
 		*out_values++ = be32_to_cpup(val++);
-	return 0;
+
+	return sz;
 }
-EXPORT_SYMBOL_GPL(of_property_read_u32_array);
+EXPORT_SYMBOL_GPL(of_property_read_variable_u32_array);
 
 /**
  * of_property_read_u64 - Find and read a 64 bit integer from a property
@@ -1314,7 +1366,9 @@ int of_property_read_u64(const struct device_node *np, const char *propname,
 			 u64 *out_value)
 {
 	const __be32 *val = of_find_property_value_of_size(np, propname,
-						sizeof(*out_value));
+						sizeof(*out_value),
+						0,
+						NULL);
 
 	if (IS_ERR(val))
 		return PTR_ERR(val);
@@ -1325,38 +1379,51 @@ int of_property_read_u64(const struct device_node *np, const char *propname,
 EXPORT_SYMBOL_GPL(of_property_read_u64);
 
 /**
- * of_property_read_u64_array - Find and read an array of 64 bit integers
- * from a property.
+ * of_property_read_variable_u64_array - Find and read an array of 64 bit
+ * integers from a property, with bounds on the minimum and maximum array size.
  *
  * @np:		device node from which the property value is to be read.
  * @propname:	name of the property to be searched.
  * @out_values:	pointer to return value, modified only if return value is 0.
- * @sz:		number of array elements to read
+ * @sz_min:	minimum number of array elements to read
+ * @sz_max:	maximum number of array elements to read, if zero there is no
+ *		upper limit on the number of elements in the dts entry but only
+ *		sz_min will be read.
  *
  * Search for a property in a device node and read 64-bit value(s) from
- * it. Returns 0 on success, -EINVAL if the property does not exist,
- * -ENODATA if property does not have a value, and -EOVERFLOW if the
- * property data isn't large enough.
+ * it. Returns number of elements read on success, -EINVAL if the property
+ * does not exist, -ENODATA if property does not have a value, and -EOVERFLOW
+ * if the property data is smaller than sz_min or longer than sz_max.
  *
  * The out_values is modified only if a valid u64 value can be decoded.
  */
-int of_property_read_u64_array(const struct device_node *np,
+int of_property_read_variable_u64_array(const struct device_node *np,
 			       const char *propname, u64 *out_values,
-			       size_t sz)
+			       size_t sz_min, size_t sz_max)
 {
+	size_t sz, count;
 	const __be32 *val = of_find_property_value_of_size(np, propname,
-						(sz * sizeof(*out_values)));
+						(sz_min * sizeof(*out_values)),
+						(sz_max * sizeof(*out_values)),
+						&sz);
 
 	if (IS_ERR(val))
 		return PTR_ERR(val);
 
-	while (sz--) {
+	if (!sz_max)
+		sz = sz_min;
+	else
+		sz /= sizeof(*out_values);
+
+	count = sz;
+	while (count--) {
 		*out_values++ = of_read_number(val, 2);
 		val += 2;
 	}
-	return 0;
+
+	return sz;
 }
-EXPORT_SYMBOL_GPL(of_property_read_u64_array);
+EXPORT_SYMBOL_GPL(of_property_read_variable_u64_array);
 
 /**
  * of_property_read_string - Find and read a string from a property
@@ -1631,8 +1698,7 @@ static int __of_parse_phandle_with_args(const struct device_node *np,
 	 */
 
  err:
-	if (it.node)
-		of_node_put(it.node);
+	of_node_put(it.node);
 	return rc;
 }
 
@@ -2011,6 +2077,8 @@ void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align))
 			name = of_get_property(of_aliases, "stdout", NULL);
 		if (name)
 			of_stdout = of_find_node_opts_by_path(name, &of_stdout_options);
+		if (of_stdout)
+			console_set_by_of();
 	}
 
 	if (!of_aliases)
@@ -2343,20 +2411,13 @@ struct device_node *of_graph_get_endpoint_by_regs(
 	const struct device_node *parent, int port_reg, int reg)
 {
 	struct of_endpoint endpoint;
-	struct device_node *node, *prev_node = NULL;
-
-	while (1) {
-		node = of_graph_get_next_endpoint(parent, prev_node);
-		of_node_put(prev_node);
-		if (!node)
-			break;
+	struct device_node *node = NULL;
 
+	for_each_endpoint_of_node(parent, node) {
 		of_graph_parse_endpoint(node, &endpoint);
 		if (((port_reg == -1) || (endpoint.port == port_reg)) &&
 			((reg == -1) || (endpoint.id == reg)))
 			return node;
-
-		prev_node = node;
 	}
 
 	return NULL;
diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c
index 55f1b8391149..c89d5d231a0e 100644
--- a/drivers/of/fdt.c
+++ b/drivers/of/fdt.c
@@ -517,7 +517,7 @@ static void *__unflatten_device_tree(const void *blob,
 		pr_warning("End of tree marker overwritten: %08x\n",
 			   be32_to_cpup(mem + size));
 
-	if (detached) {
+	if (detached && mynodes) {
 		of_node_set_flag(*mynodes, OF_DETACHED);
 		pr_debug("unflattened tree is detached\n");
 	}
@@ -924,7 +924,7 @@ static inline void early_init_dt_check_for_initrd(unsigned long node)
 
 #ifdef CONFIG_SERIAL_EARLYCON
 
-static int __init early_init_dt_scan_chosen_serial(void)
+int __init early_init_dt_scan_chosen_stdout(void)
 {
 	int offset;
 	const char *p, *q, *options = NULL;
@@ -968,15 +968,6 @@ static int __init early_init_dt_scan_chosen_serial(void)
 	}
 	return -ENODEV;
 }
-
-static int __init setup_of_earlycon(char *buf)
-{
-	if (buf)
-		return 0;
-
-	return early_init_dt_scan_chosen_serial();
-}
-early_param("earlycon", setup_of_earlycon);
 #endif
 
 /**
diff --git a/drivers/of/irq.c b/drivers/of/irq.c
index 89a71c6074fc..393fea85eb4e 100644
--- a/drivers/of/irq.c
+++ b/drivers/of/irq.c
@@ -26,6 +26,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
+#include <linux/of_pci.h>
 #include <linux/string.h>
 #include <linux/slab.h>
 
@@ -544,12 +545,15 @@ void __init of_irq_init(const struct of_device_id *matches)
 
 			list_del(&desc->list);
 
+			of_node_set_flag(desc->dev, OF_POPULATED);
+
 			pr_debug("of_irq_init: init %s (%p), parent %p\n",
 				 desc->dev->full_name,
 				 desc->dev, desc->interrupt_parent);
 			ret = desc->irq_init_cb(desc->dev,
 						desc->interrupt_parent);
 			if (ret) {
+				of_node_clear_flag(desc->dev, OF_POPULATED);
 				kfree(desc);
 				continue;
 			}
@@ -559,8 +563,6 @@ void __init of_irq_init(const struct of_device_id *matches)
 			 * its children can get processed in a subsequent pass.
 			 */
 			list_add_tail(&desc->list, &intc_parent_list);
-
-			of_node_set_flag(desc->dev, OF_POPULATED);
 		}
 
 		/* Get the next pending parent that might have children */
@@ -591,87 +593,16 @@ static u32 __of_msi_map_rid(struct device *dev, struct device_node **np,
 			    u32 rid_in)
 {
 	struct device *parent_dev;
-	struct device_node *msi_controller_node;
-	struct device_node *msi_np = *np;
-	u32 map_mask, masked_rid, rid_base, msi_base, rid_len, phandle;
-	int msi_map_len;
-	bool matched;
 	u32 rid_out = rid_in;
-	const __be32 *msi_map = NULL;
 
 	/*
 	 * Walk up the device parent links looking for one with a
 	 * "msi-map" property.
 	 */
-	for (parent_dev = dev; parent_dev; parent_dev = parent_dev->parent) {
-		if (!parent_dev->of_node)
-			continue;
-
-		msi_map = of_get_property(parent_dev->of_node,
-					  "msi-map", &msi_map_len);
-		if (!msi_map)
-			continue;
-
-		if (msi_map_len % (4 * sizeof(__be32))) {
-			dev_err(parent_dev, "Error: Bad msi-map length: %d\n",
-				msi_map_len);
-			return rid_out;
-		}
-		/* We have a good parent_dev and msi_map, let's use them. */
-		break;
-	}
-	if (!msi_map)
-		return rid_out;
-
-	/* The default is to select all bits. */
-	map_mask = 0xffffffff;
-
-	/*
-	 * Can be overridden by "msi-map-mask" property.  If
-	 * of_property_read_u32() fails, the default is used.
-	 */
-	of_property_read_u32(parent_dev->of_node, "msi-map-mask", &map_mask);
-
-	masked_rid = map_mask & rid_in;
-	matched = false;
-	while (!matched && msi_map_len >= 4 * sizeof(__be32)) {
-		rid_base = be32_to_cpup(msi_map + 0);
-		phandle = be32_to_cpup(msi_map + 1);
-		msi_base = be32_to_cpup(msi_map + 2);
-		rid_len = be32_to_cpup(msi_map + 3);
-
-		if (rid_base & ~map_mask) {
-			dev_err(parent_dev,
-				"Invalid msi-map translation - msi-map-mask (0x%x) ignores rid-base (0x%x)\n",
-				map_mask, rid_base);
-			return rid_out;
-		}
-
-		msi_controller_node = of_find_node_by_phandle(phandle);
-
-		matched = (masked_rid >= rid_base &&
-			   masked_rid < rid_base + rid_len);
-		if (msi_np)
-			matched &= msi_np == msi_controller_node;
-
-		if (matched && !msi_np) {
-			*np = msi_np = msi_controller_node;
+	for (parent_dev = dev; parent_dev; parent_dev = parent_dev->parent)
+		if (!of_pci_map_rid(parent_dev->of_node, rid_in, "msi-map",
+				    "msi-map-mask", np, &rid_out))
 			break;
-		}
-
-		of_node_put(msi_controller_node);
-		msi_map_len -= 4 * sizeof(__be32);
-		msi_map += 4;
-	}
-	if (!matched)
-		return rid_out;
-
-	rid_out = masked_rid - rid_base + msi_base;
-	dev_dbg(dev,
-		"msi-map at: %s, using mask %08x, rid-base: %08x, msi-base: %08x, length: %08x, rid: %08x -> %08x\n",
-		dev_name(parent_dev), map_mask, rid_base, msi_base,
-		rid_len, rid_in, rid_out);
-
 	return rid_out;
 }
 
diff --git a/drivers/of/of_numa.c b/drivers/of/of_numa.c
index ed5a097f0801..f63d4b0deff0 100644
--- a/drivers/of/of_numa.c
+++ b/drivers/of/of_numa.c
@@ -16,6 +16,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define pr_fmt(fmt) "OF: NUMA: " fmt
+
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/nodemask.h>
@@ -49,10 +51,9 @@ static void __init of_numa_parse_cpu_nodes(void)
 		if (r)
 			continue;
 
-		pr_debug("NUMA: CPU on %u\n", nid);
+		pr_debug("CPU on %u\n", nid);
 		if (nid >= MAX_NUMNODES)
-			pr_warn("NUMA: Node id %u exceeds maximum value\n",
-				nid);
+			pr_warn("Node id %u exceeds maximum value\n", nid);
 		else
 			node_set(nid, numa_nodes_parsed);
 	}
@@ -63,13 +64,9 @@ static int __init of_numa_parse_memory_nodes(void)
 	struct device_node *np = NULL;
 	struct resource rsrc;
 	u32 nid;
-	int r = 0;
-
-	for (;;) {
-		np = of_find_node_by_type(np, "memory");
-		if (!np)
-			break;
+	int i, r;
 
+	for_each_node_by_type(np, "memory") {
 		r = of_property_read_u32(np, "numa-node-id", &nid);
 		if (r == -EINVAL)
 			/*
@@ -78,27 +75,23 @@ static int __init of_numa_parse_memory_nodes(void)
 			 * "numa-node-id" property
 			 */
 			continue;
-		else if (r)
-			/* some other error */
-			break;
 
-		r = of_address_to_resource(np, 0, &rsrc);
-		if (r) {
-			pr_err("NUMA: bad reg property in memory node\n");
-			break;
+		if (nid >= MAX_NUMNODES) {
+			pr_warn("Node id %u exceeds maximum value\n", nid);
+			r = -EINVAL;
 		}
 
-		pr_debug("NUMA:  base = %llx len = %llx, node = %u\n",
-			 rsrc.start, rsrc.end - rsrc.start + 1, nid);
-
+		for (i = 0; !r && !of_address_to_resource(np, i, &rsrc); i++)
+			r = numa_add_memblk(nid, rsrc.start, rsrc.end + 1);
 
-		r = numa_add_memblk(nid, rsrc.start, rsrc.end + 1);
-		if (r)
-			break;
+		if (!i || r) {
+			of_node_put(np);
+			pr_err("bad property in memory node\n");
+			return r ? : -EINVAL;
+		}
 	}
-	of_node_put(np);
 
-	return r;
+	return 0;
 }
 
 static int __init of_numa_parse_distance_map_v1(struct device_node *map)
@@ -107,17 +100,17 @@ static int __init of_numa_parse_distance_map_v1(struct device_node *map)
 	int entry_count;
 	int i;
 
-	pr_info("NUMA: parsing numa-distance-map-v1\n");
+	pr_info("parsing numa-distance-map-v1\n");
 
 	matrix = of_get_property(map, "distance-matrix", NULL);
 	if (!matrix) {
-		pr_err("NUMA: No distance-matrix property in distance-map\n");
+		pr_err("No distance-matrix property in distance-map\n");
 		return -EINVAL;
 	}
 
 	entry_count = of_property_count_u32_elems(map, "distance-matrix");
 	if (entry_count <= 0) {
-		pr_err("NUMA: Invalid distance-matrix\n");
+		pr_err("Invalid distance-matrix\n");
 		return -EINVAL;
 	}
 
@@ -132,7 +125,7 @@ static int __init of_numa_parse_distance_map_v1(struct device_node *map)
 		matrix++;
 
 		numa_set_distance(nodea, nodeb, distance);
-		pr_debug("NUMA:  distance[node%d -> node%d] = %d\n",
+		pr_debug("distance[node%d -> node%d] = %d\n",
 			 nodea, nodeb, distance);
 
 		/* Set default distance of node B->A same as A->B */
@@ -166,8 +159,6 @@ int of_node_to_nid(struct device_node *device)
 	np = of_node_get(device);
 
 	while (np) {
-		struct device_node *parent;
-
 		r = of_property_read_u32(np, "numa-node-id", &nid);
 		/*
 		 * -EINVAL indicates the property was not found, and
@@ -178,22 +169,15 @@ int of_node_to_nid(struct device_node *device)
 		if (r != -EINVAL)
 			break;
 
-		parent = of_get_parent(np);
-		of_node_put(np);
-		np = parent;
+		np = of_get_next_parent(np);
 	}
 	if (np && r)
-		pr_warn("NUMA: Invalid \"numa-node-id\" property in node %s\n",
+		pr_warn("Invalid \"numa-node-id\" property in node %s\n",
 			np->name);
 	of_node_put(np);
 
-	if (!r) {
-		if (nid >= MAX_NUMNODES)
-			pr_warn("NUMA: Node id %u exceeds maximum value\n",
-				nid);
-		else
-			return nid;
-	}
+	if (!r)
+		return nid;
 
 	return NUMA_NO_NODE;
 }
diff --git a/drivers/of/of_pci.c b/drivers/of/of_pci.c
index 589b30c68e14..b58be12ab277 100644
--- a/drivers/of/of_pci.c
+++ b/drivers/of/of_pci.c
@@ -308,3 +308,105 @@ struct msi_controller *of_pci_find_msi_chip_by_node(struct device_node *of_node)
 EXPORT_SYMBOL_GPL(of_pci_find_msi_chip_by_node);
 
 #endif /* CONFIG_PCI_MSI */
+
+/**
+ * of_pci_map_rid - Translate a requester ID through a downstream mapping.
+ * @np: root complex device node.
+ * @rid: PCI requester ID to map.
+ * @map_name: property name of the map to use.
+ * @map_mask_name: optional property name of the mask to use.
+ * @target: optional pointer to a target device node.
+ * @id_out: optional pointer to receive the translated ID.
+ *
+ * Given a PCI requester ID, look up the appropriate implementation-defined
+ * platform ID and/or the target device which receives transactions on that
+ * ID, as per the "iommu-map" and "msi-map" bindings. Either of @target or
+ * @id_out may be NULL if only the other is required. If @target points to
+ * a non-NULL device node pointer, only entries targeting that node will be
+ * matched; if it points to a NULL value, it will receive the device node of
+ * the first matching target phandle, with a reference held.
+ *
+ * Return: 0 on success or a standard error code on failure.
+ */
+int of_pci_map_rid(struct device_node *np, u32 rid,
+		   const char *map_name, const char *map_mask_name,
+		   struct device_node **target, u32 *id_out)
+{
+	u32 map_mask, masked_rid;
+	int map_len;
+	const __be32 *map = NULL;
+
+	if (!np || !map_name || (!target && !id_out))
+		return -EINVAL;
+
+	map = of_get_property(np, map_name, &map_len);
+	if (!map) {
+		if (target)
+			return -ENODEV;
+		/* Otherwise, no map implies no translation */
+		*id_out = rid;
+		return 0;
+	}
+
+	if (!map_len || map_len % (4 * sizeof(*map))) {
+		pr_err("%s: Error: Bad %s length: %d\n", np->full_name,
+			map_name, map_len);
+		return -EINVAL;
+	}
+
+	/* The default is to select all bits. */
+	map_mask = 0xffffffff;
+
+	/*
+	 * Can be overridden by "{iommu,msi}-map-mask" property.
+	 * If of_property_read_u32() fails, the default is used.
+	 */
+	if (map_mask_name)
+		of_property_read_u32(np, map_mask_name, &map_mask);
+
+	masked_rid = map_mask & rid;
+	for ( ; map_len > 0; map_len -= 4 * sizeof(*map), map += 4) {
+		struct device_node *phandle_node;
+		u32 rid_base = be32_to_cpup(map + 0);
+		u32 phandle = be32_to_cpup(map + 1);
+		u32 out_base = be32_to_cpup(map + 2);
+		u32 rid_len = be32_to_cpup(map + 3);
+
+		if (rid_base & ~map_mask) {
+			pr_err("%s: Invalid %s translation - %s-mask (0x%x) ignores rid-base (0x%x)\n",
+				np->full_name, map_name, map_name,
+				map_mask, rid_base);
+			return -EFAULT;
+		}
+
+		if (masked_rid < rid_base || masked_rid >= rid_base + rid_len)
+			continue;
+
+		phandle_node = of_find_node_by_phandle(phandle);
+		if (!phandle_node)
+			return -ENODEV;
+
+		if (target) {
+			if (*target)
+				of_node_put(phandle_node);
+			else
+				*target = phandle_node;
+
+			if (*target != phandle_node)
+				continue;
+		}
+
+		if (id_out)
+			*id_out = masked_rid - rid_base + out_base;
+
+		pr_debug("%s: %s, using mask %08x, rid-base: %08x, out-base: %08x, length: %08x, rid: %08x -> %08x\n",
+			np->full_name, map_name, map_mask, rid_base, out_base,
+			rid_len, rid, *id_out);
+		return 0;
+	}
+
+	pr_err("%s: Invalid %s translation - no match for rid 0x%x on %s\n",
+		np->full_name, map_name, rid,
+		target && *target ? (*target)->full_name : "any target");
+	return -EFAULT;
+}
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index 8aa197691074..f811d2796437 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -142,6 +142,7 @@ struct platform_device *of_device_alloc(struct device_node *np,
 	}
 
 	dev->dev.of_node = of_node_get(np);
+	dev->dev.fwnode = &np->fwnode;
 	dev->dev.parent = parent ? : &platform_bus;
 
 	if (bus_id)
@@ -241,6 +242,7 @@ static struct amba_device *of_amba_device_create(struct device_node *node,
 
 	/* setup generic device info */
 	dev->dev.of_node = of_node_get(node);
+	dev->dev.fwnode = &node->fwnode;
 	dev->dev.parent = parent ? : &platform_bus;
 	dev->dev.platform_data = platform_data;
 	if (bus_id)
@@ -497,6 +499,7 @@ int of_platform_default_populate(struct device_node *root,
 }
 EXPORT_SYMBOL_GPL(of_platform_default_populate);
 
+#ifndef CONFIG_PPC
 static int __init of_platform_default_populate_init(void)
 {
 	struct device_node *node;
@@ -521,6 +524,7 @@ static int __init of_platform_default_populate_init(void)
 	return 0;
 }
 arch_initcall_sync(of_platform_default_populate_init);
+#endif
 
 static int of_platform_device_destroy(struct device *dev, void *data)
 {
diff --git a/drivers/oprofile/oprofilefs.c b/drivers/oprofile/oprofilefs.c
index a0e5260bd006..134398e0231b 100644
--- a/drivers/oprofile/oprofilefs.c
+++ b/drivers/oprofile/oprofilefs.c
@@ -30,7 +30,7 @@ static struct inode *oprofilefs_get_inode(struct super_block *sb, int mode)
 	if (inode) {
 		inode->i_ino = get_next_ino();
 		inode->i_mode = mode;
-		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
+		inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
 	}
 	return inode;
 }
diff --git a/drivers/oprofile/timer_int.c b/drivers/oprofile/timer_int.c
index bdef916e5dda..2498a6cd7c24 100644
--- a/drivers/oprofile/timer_int.c
+++ b/drivers/oprofile/timer_int.c
@@ -74,37 +74,39 @@ static void oprofile_hrtimer_stop(void)
 	put_online_cpus();
 }
 
-static int oprofile_cpu_notify(struct notifier_block *self,
-			       unsigned long action, void *hcpu)
+static int oprofile_timer_online(unsigned int cpu)
 {
-	long cpu = (long) hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		smp_call_function_single(cpu, __oprofile_hrtimer_start,
-					 NULL, 1);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		__oprofile_hrtimer_stop(cpu);
-		break;
-	}
-	return NOTIFY_OK;
+	local_irq_disable();
+	__oprofile_hrtimer_start(NULL);
+	local_irq_enable();
+	return 0;
 }
 
-static struct notifier_block __refdata oprofile_cpu_notifier = {
-	.notifier_call = oprofile_cpu_notify,
-};
+static int oprofile_timer_prep_down(unsigned int cpu)
+{
+	__oprofile_hrtimer_stop(cpu);
+	return 0;
+}
+
+static enum cpuhp_state hp_online;
 
 static int oprofile_hrtimer_setup(void)
 {
-	return register_hotcpu_notifier(&oprofile_cpu_notifier);
+	int ret;
+
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"oprofile/timer:online",
+					oprofile_timer_online,
+					oprofile_timer_prep_down);
+	if (ret < 0)
+		return ret;
+	hp_online = ret;
+	return 0;
 }
 
 static void oprofile_hrtimer_shutdown(void)
 {
-	unregister_hotcpu_notifier(&oprofile_cpu_notifier);
+	cpuhp_remove_state_nocalls(hp_online);
 }
 
 int oprofile_timer_init(struct oprofile_operations *ops)
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index 67f9916ff14d..6555eb78d91c 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -25,7 +25,7 @@ config PCI_MSI
 	   If you don't know what to do here, say Y.
 
 config PCI_MSI_IRQ_DOMAIN
-	def_bool ARM || ARM64 || X86
+	def_bool ARC || ARM || ARM64 || X86
 	depends on PCI_MSI
 	select GENERIC_MSI_IRQ_DOMAIN
 
diff --git a/drivers/pci/host-bridge.c b/drivers/pci/host-bridge.c
index 5f4a2e04c8d7..add66236215c 100644
--- a/drivers/pci/host-bridge.c
+++ b/drivers/pci/host-bridge.c
@@ -44,6 +44,7 @@ void pci_set_host_bridge_release(struct pci_host_bridge *bridge,
 	bridge->release_fn = release_fn;
 	bridge->release_data = release_data;
 }
+EXPORT_SYMBOL_GPL(pci_set_host_bridge_release);
 
 void pcibios_resource_to_bus(struct pci_bus *bus, struct pci_bus_region *region,
 			     struct resource *res)
diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig
index 9b485d873b0d..d7e7c0a827c3 100644
--- a/drivers/pci/host/Kconfig
+++ b/drivers/pci/host/Kconfig
@@ -274,4 +274,31 @@ config PCIE_ARTPEC6
 	  Say Y here to enable PCIe controller support on Axis ARTPEC-6
 	  SoCs.  This PCIe controller uses the DesignWare core.
 
+config PCIE_ROCKCHIP
+	bool "Rockchip PCIe controller"
+	depends on ARCH_ROCKCHIP
+	depends on OF
+	depends on PCI_MSI_IRQ_DOMAIN
+	select MFD_SYSCON
+	help
+	  Say Y here if you want internal PCI support on Rockchip SoC.
+	  There is 1 internal PCIe port available to support GEN2 with
+	  4 slots.
+
+config VMD
+	depends on PCI_MSI && X86_64
+	tristate "Intel Volume Management Device Driver"
+	default N
+	---help---
+	  Adds support for the Intel Volume Management Device (VMD). VMD is a
+	  secondary PCI host bridge that allows PCI Express root ports,
+	  and devices attached to them, to be removed from the default
+	  PCI domain and placed within the VMD domain. This provides
+	  more bus resources than are otherwise possible with a
+	  single domain. If you know your system provides one of these and
+	  has devices attached to it, say Y; if you are not sure, say N.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called vmd.
+
 endmenu
diff --git a/drivers/pci/host/Makefile b/drivers/pci/host/Makefile
index 88434101e4c4..084cb4983645 100644
--- a/drivers/pci/host/Makefile
+++ b/drivers/pci/host/Makefile
@@ -31,3 +31,5 @@ obj-$(CONFIG_PCI_HOST_THUNDER_ECAM) += pci-thunder-ecam.o
 obj-$(CONFIG_PCI_HOST_THUNDER_PEM) += pci-thunder-pem.o
 obj-$(CONFIG_PCIE_ARMADA_8K) += pcie-armada8k.o
 obj-$(CONFIG_PCIE_ARTPEC6) += pcie-artpec6.o
+obj-$(CONFIG_PCIE_ROCKCHIP) += pcie-rockchip.o
+obj-$(CONFIG_VMD) += vmd.o
diff --git a/drivers/pci/host/pci-aardvark.c b/drivers/pci/host/pci-aardvark.c
index ef9893fa3176..e4a5b7ee90cf 100644
--- a/drivers/pci/host/pci-aardvark.c
+++ b/drivers/pci/host/pci-aardvark.c
@@ -848,7 +848,7 @@ static int advk_pcie_parse_request_of_pci_ranges(struct advk_pcie *pcie)
 	int err, res_valid = 0;
 	struct device *dev = &pcie->pdev->dev;
 	struct device_node *np = dev->of_node;
-	struct resource_entry *win;
+	struct resource_entry *win, *tmp;
 	resource_size_t iobase;
 
 	INIT_LIST_HEAD(&pcie->resources);
@@ -862,7 +862,7 @@ static int advk_pcie_parse_request_of_pci_ranges(struct advk_pcie *pcie)
 	if (err)
 		goto out_release_res;
 
-	resource_list_for_each_entry(win, &pcie->resources) {
+	resource_list_for_each_entry_safe(win, tmp, &pcie->resources) {
 		struct resource *res = win->res;
 
 		switch (resource_type(res)) {
@@ -874,9 +874,11 @@ static int advk_pcie_parse_request_of_pci_ranges(struct advk_pcie *pcie)
 					     lower_32_bits(res->start),
 					     OB_PCIE_IO);
 			err = pci_remap_iospace(res, iobase);
-			if (err)
+			if (err) {
 				dev_warn(dev, "error %d: failed to map resource %pR\n",
 					 err, res);
+				resource_list_destroy_entry(win);
+			}
 			break;
 		case IORESOURCE_MEM:
 			advk_pcie_set_ob_win(pcie, 0,
@@ -925,10 +927,8 @@ static int advk_pcie_probe(struct platform_device *pdev)
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	pcie->base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(pcie->base)) {
-		dev_err(&pdev->dev, "Failed to map registers\n");
+	if (IS_ERR(pcie->base))
 		return PTR_ERR(pcie->base);
-	}
 
 	irq = platform_get_irq(pdev, 0);
 	ret = devm_request_irq(&pdev->dev, irq, advk_pcie_irq_handler,
diff --git a/drivers/pci/host/pci-dra7xx.c b/drivers/pci/host/pci-dra7xx.c
index 81b3949a26db..19223ed2e619 100644
--- a/drivers/pci/host/pci-dra7xx.c
+++ b/drivers/pci/host/pci-dra7xx.c
@@ -15,7 +15,7 @@
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of_gpio.h>
 #include <linux/pci.h>
 #include <linux/phy/phy.h>
@@ -443,25 +443,6 @@ err_phy:
 	return ret;
 }
 
-static int __exit dra7xx_pcie_remove(struct platform_device *pdev)
-{
-	struct dra7xx_pcie *dra7xx = platform_get_drvdata(pdev);
-	struct pcie_port *pp = &dra7xx->pp;
-	struct device *dev = &pdev->dev;
-	int count = dra7xx->phy_count;
-
-	if (pp->irq_domain)
-		irq_domain_remove(pp->irq_domain);
-	pm_runtime_put(dev);
-	pm_runtime_disable(dev);
-	while (count--) {
-		phy_power_off(dra7xx->phy[count]);
-		phy_exit(dra7xx->phy[count]);
-	}
-
-	return 0;
-}
-
 #ifdef CONFIG_PM_SLEEP
 static int dra7xx_pcie_suspend(struct device *dev)
 {
@@ -545,19 +526,13 @@ static const struct of_device_id of_dra7xx_pcie_match[] = {
 	{ .compatible = "ti,dra7-pcie", },
 	{},
 };
-MODULE_DEVICE_TABLE(of, of_dra7xx_pcie_match);
 
 static struct platform_driver dra7xx_pcie_driver = {
-	.remove		= __exit_p(dra7xx_pcie_remove),
 	.driver = {
 		.name	= "dra7-pcie",
 		.of_match_table = of_dra7xx_pcie_match,
+		.suppress_bind_attrs = true,
 		.pm	= &dra7xx_pcie_pm_ops,
 	},
 };
-
-module_platform_driver_probe(dra7xx_pcie_driver, dra7xx_pcie_probe);
-
-MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
-MODULE_DESCRIPTION("TI PCIe controller driver");
-MODULE_LICENSE("GPL v2");
+builtin_platform_driver_probe(dra7xx_pcie_driver, dra7xx_pcie_probe);
diff --git a/drivers/pci/host/pci-exynos.c b/drivers/pci/host/pci-exynos.c
index 219976103efc..2e2d7f00b9e8 100644
--- a/drivers/pci/host/pci-exynos.c
+++ b/drivers/pci/host/pci-exynos.c
@@ -16,7 +16,7 @@
 #include <linux/gpio.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of_gpio.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
@@ -425,12 +425,15 @@ static void exynos_pcie_enable_interrupts(struct pcie_port *pp)
 		exynos_pcie_msi_init(pp);
 }
 
-static inline void exynos_pcie_readl_rc(struct pcie_port *pp,
-					void __iomem *dbi_base, u32 *val)
+static inline u32 exynos_pcie_readl_rc(struct pcie_port *pp,
+				       void __iomem *dbi_base)
 {
+	u32 val;
+
 	exynos_pcie_sideband_dbi_r_mode(pp, true);
-	*val = readl(dbi_base);
+	val = readl(dbi_base);
 	exynos_pcie_sideband_dbi_r_mode(pp, false);
+	return val;
 }
 
 static inline void exynos_pcie_writel_rc(struct pcie_port *pp,
@@ -624,7 +627,6 @@ static const struct of_device_id exynos_pcie_of_match[] = {
 	{ .compatible = "samsung,exynos5440-pcie", },
 	{},
 };
-MODULE_DEVICE_TABLE(of, exynos_pcie_of_match);
 
 static struct platform_driver exynos_pcie_driver = {
 	.remove		= __exit_p(exynos_pcie_remove),
@@ -641,7 +643,3 @@ static int __init exynos_pcie_init(void)
 	return platform_driver_probe(&exynos_pcie_driver, exynos_pcie_probe);
 }
 subsys_initcall(exynos_pcie_init);
-
-MODULE_AUTHOR("Jingoo Han <jg1.han@samsung.com>");
-MODULE_DESCRIPTION("Samsung PCIe host controller driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/host/pci-host-common.c b/drivers/pci/host/pci-host-common.c
index 9d9d34e959b6..e3c48b5deb93 100644
--- a/drivers/pci/host/pci-host-common.c
+++ b/drivers/pci/host/pci-host-common.c
@@ -1,4 +1,6 @@
 /*
+ * Generic PCI host driver common code
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
@@ -17,7 +19,6 @@
  */
 
 #include <linux/kernel.h>
-#include <linux/module.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
 #include <linux/pci-ecam.h>
@@ -29,7 +30,7 @@ static int gen_pci_parse_request_of_pci_ranges(struct device *dev,
 	int err, res_valid = 0;
 	struct device_node *np = dev->of_node;
 	resource_size_t iobase;
-	struct resource_entry *win;
+	struct resource_entry *win, *tmp;
 
 	err = of_pci_get_host_bridge_resources(np, 0, 0xff, resources, &iobase);
 	if (err)
@@ -39,15 +40,17 @@ static int gen_pci_parse_request_of_pci_ranges(struct device *dev,
 	if (err)
 		return err;
 
-	resource_list_for_each_entry(win, resources) {
+	resource_list_for_each_entry_safe(win, tmp, resources) {
 		struct resource *res = win->res;
 
 		switch (resource_type(res)) {
 		case IORESOURCE_IO:
 			err = pci_remap_iospace(res, iobase);
-			if (err)
+			if (err) {
 				dev_warn(dev, "error %d: failed to map resource %pR\n",
 					 err, res);
+				resource_list_destroy_entry(win);
+			}
 			break;
 		case IORESOURCE_MEM:
 			res_valid |= !(res->flags & IORESOURCE_PREFETCH);
@@ -162,7 +165,3 @@ int pci_host_common_probe(struct platform_device *pdev,
 	pci_bus_add_devices(bus);
 	return 0;
 }
-
-MODULE_DESCRIPTION("Generic PCI host driver common code");
-MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/host/pci-hyperv.c b/drivers/pci/host/pci-hyperv.c
index 6955ffdb89f3..763ff8745828 100644
--- a/drivers/pci/host/pci-hyperv.c
+++ b/drivers/pci/host/pci-hyperv.c
@@ -200,11 +200,11 @@ struct tran_int_desc {
  */
 
 struct pci_message {
-	u32 message_type;
+	u32 type;
 } __packed;
 
 struct pci_child_message {
-	u32 message_type;
+	struct pci_message message_type;
 	union win_slot_encoding wslot;
 } __packed;
 
@@ -222,7 +222,8 @@ struct pci_packet {
 	void (*completion_func)(void *context, struct pci_response *resp,
 				int resp_packet_size);
 	void *compl_ctxt;
-	struct pci_message message;
+
+	struct pci_message message[0];
 };
 
 /*
@@ -258,7 +259,7 @@ struct pci_bus_d0_entry {
 struct pci_bus_relations {
 	struct pci_incoming_message incoming;
 	u32 device_count;
-	struct pci_function_description func[1];
+	struct pci_function_description func[0];
 } __packed;
 
 struct pci_q_res_req_response {
@@ -314,7 +315,7 @@ struct pci_dev_incoming {
 } __packed;
 
 struct pci_eject_response {
-	u32 message_type;
+	struct pci_message message_type;
 	union win_slot_encoding wslot;
 	u32 status;
 } __packed;
@@ -373,7 +374,6 @@ struct hv_pcibus_device {
 
 	struct list_head children;
 	struct list_head dr_list;
-	struct work_struct wrk;
 
 	struct msi_domain_info msi_info;
 	struct msi_controller msi_chip;
@@ -393,7 +393,7 @@ struct hv_dr_work {
 struct hv_dr_state {
 	struct list_head list_entry;
 	u32 device_count;
-	struct pci_function_description func[1];
+	struct pci_function_description func[0];
 };
 
 enum hv_pcichild_state {
@@ -447,15 +447,16 @@ struct hv_pci_compl {
  * for any message for which the completion packet contains a
  * status and nothing else.
  */
-static
-void
-hv_pci_generic_compl(void *context, struct pci_response *resp,
-		     int resp_packet_size)
+static void hv_pci_generic_compl(void *context, struct pci_response *resp,
+				 int resp_packet_size)
 {
 	struct hv_pci_compl *comp_pkt = context;
 
 	if (resp_packet_size >= offsetofend(struct pci_response, status))
 		comp_pkt->completion_status = resp->status;
+	else
+		comp_pkt->completion_status = -1;
+
 	complete(&comp_pkt->host_event);
 }
 
@@ -694,13 +695,12 @@ static void hv_int_desc_free(struct hv_pci_dev *hpdev,
 	struct pci_delete_interrupt *int_pkt;
 	struct {
 		struct pci_packet pkt;
-		u8 buffer[sizeof(struct pci_delete_interrupt) -
-			  sizeof(struct pci_message)];
+		u8 buffer[sizeof(struct pci_delete_interrupt)];
 	} ctxt;
 
 	memset(&ctxt, 0, sizeof(ctxt));
 	int_pkt = (struct pci_delete_interrupt *)&ctxt.pkt.message;
-	int_pkt->message_type.message_type =
+	int_pkt->message_type.type =
 		PCI_DELETE_INTERRUPT_MESSAGE;
 	int_pkt->wslot.slot = hpdev->desc.win_slot.slot;
 	int_pkt->int_desc = *int_desc;
@@ -847,8 +847,7 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 	struct cpumask *affinity;
 	struct {
 		struct pci_packet pkt;
-		u8 buffer[sizeof(struct pci_create_interrupt) -
-			  sizeof(struct pci_message)];
+		u8 buffer[sizeof(struct pci_create_interrupt)];
 	} ctxt;
 	int cpu;
 	int ret;
@@ -876,7 +875,7 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 	ctxt.pkt.completion_func = hv_pci_compose_compl;
 	ctxt.pkt.compl_ctxt = &comp;
 	int_pkt = (struct pci_create_interrupt *)&ctxt.pkt.message;
-	int_pkt->message_type.message_type = PCI_CREATE_INTERRUPT_MESSAGE;
+	int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE;
 	int_pkt->wslot.slot = hpdev->desc.win_slot.slot;
 	int_pkt->int_desc.vector = cfg->vector;
 	int_pkt->int_desc.vector_count = 1;
@@ -897,8 +896,10 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 			       sizeof(*int_pkt), (unsigned long)&ctxt.pkt,
 			       VM_PKT_DATA_INBAND,
 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
-	if (!ret)
-		wait_for_completion(&comp.comp_pkt.host_event);
+	if (ret)
+		goto free_int_desc;
+
+	wait_for_completion(&comp.comp_pkt.host_event);
 
 	if (comp.comp_pkt.completion_status < 0) {
 		dev_err(&hbus->hdev->device,
@@ -1289,7 +1290,7 @@ static struct hv_pci_dev *new_pcichild_device(struct hv_pcibus_device *hbus,
 	pkt.init_packet.compl_ctxt = &comp_pkt;
 	pkt.init_packet.completion_func = q_resource_requirements;
 	res_req = (struct pci_child_message *)&pkt.init_packet.message;
-	res_req->message_type = PCI_QUERY_RESOURCE_REQUIREMENTS;
+	res_req->message_type.type = PCI_QUERY_RESOURCE_REQUIREMENTS;
 	res_req->wslot.slot = desc->win_slot.slot;
 
 	ret = vmbus_sendpacket(hbus->hdev->channel, res_req,
@@ -1466,8 +1467,7 @@ static void pci_devices_present_work(struct work_struct *work)
 			if (hpdev->reported_missing) {
 				found = true;
 				put_pcichild(hpdev, hv_pcidev_ref_childlist);
-				list_del(&hpdev->list_entry);
-				list_add_tail(&hpdev->list_entry, &removed);
+				list_move_tail(&hpdev->list_entry, &removed);
 				break;
 			}
 		}
@@ -1558,8 +1558,7 @@ static void hv_eject_device_work(struct work_struct *work)
 	int wslot;
 	struct {
 		struct pci_packet pkt;
-		u8 buffer[sizeof(struct pci_eject_response) -
-			  sizeof(struct pci_message)];
+		u8 buffer[sizeof(struct pci_eject_response)];
 	} ctxt;
 
 	hpdev = container_of(work, struct hv_pci_dev, wrk);
@@ -1585,7 +1584,7 @@ static void hv_eject_device_work(struct work_struct *work)
 
 	memset(&ctxt, 0, sizeof(ctxt));
 	ejct_pkt = (struct pci_eject_response *)&ctxt.pkt.message;
-	ejct_pkt->message_type = PCI_EJECTION_COMPLETE;
+	ejct_pkt->message_type.type = PCI_EJECTION_COMPLETE;
 	ejct_pkt->wslot.slot = hpdev->desc.win_slot.slot;
 	vmbus_sendpacket(hpdev->hbus->hdev->channel, ejct_pkt,
 			 sizeof(*ejct_pkt), (unsigned long)&ctxt.pkt,
@@ -1688,7 +1687,7 @@ static void hv_pci_onchannelcallback(void *context)
 		case VM_PKT_DATA_INBAND:
 
 			new_message = (struct pci_incoming_message *)buffer;
-			switch (new_message->message_type.message_type) {
+			switch (new_message->message_type.type) {
 			case PCI_BUS_RELATIONS:
 
 				bus_rel = (struct pci_bus_relations *)buffer;
@@ -1719,7 +1718,7 @@ static void hv_pci_onchannelcallback(void *context)
 			default:
 				dev_warn(&hbus->hdev->device,
 					"Unimplemented protocol message %x\n",
-					new_message->message_type.message_type);
+					new_message->message_type.type);
 				break;
 			}
 			break;
@@ -1772,7 +1771,7 @@ static int hv_pci_protocol_negotiation(struct hv_device *hdev)
 	pkt->completion_func = hv_pci_generic_compl;
 	pkt->compl_ctxt = &comp_pkt;
 	version_req = (struct pci_version_request *)&pkt->message;
-	version_req->message_type.message_type = PCI_QUERY_PROTOCOL_VERSION;
+	version_req->message_type.type = PCI_QUERY_PROTOCOL_VERSION;
 	version_req->protocol_version = PCI_PROTOCOL_VERSION_CURRENT;
 
 	ret = vmbus_sendpacket(hdev->channel, version_req,
@@ -1973,7 +1972,7 @@ static int hv_pci_enter_d0(struct hv_device *hdev)
 	pkt->completion_func = hv_pci_generic_compl;
 	pkt->compl_ctxt = &comp_pkt;
 	d0_entry = (struct pci_bus_d0_entry *)&pkt->message;
-	d0_entry->message_type.message_type = PCI_BUS_D0ENTRY;
+	d0_entry->message_type.type = PCI_BUS_D0ENTRY;
 	d0_entry->mmio_base = hbus->mem_config->start;
 
 	ret = vmbus_sendpacket(hdev->channel, d0_entry, sizeof(*d0_entry),
@@ -2019,7 +2018,7 @@ static int hv_pci_query_relations(struct hv_device *hdev)
 		return -ENOTEMPTY;
 
 	memset(&message, 0, sizeof(message));
-	message.message_type = PCI_QUERY_BUS_RELATIONS;
+	message.type = PCI_QUERY_BUS_RELATIONS;
 
 	ret = vmbus_sendpacket(hdev->channel, &message, sizeof(message),
 			       0, VM_PKT_DATA_INBAND, 0);
@@ -2072,8 +2071,8 @@ static int hv_send_resources_allocated(struct hv_device *hdev)
 		init_completion(&comp_pkt.host_event);
 		pkt->completion_func = hv_pci_generic_compl;
 		pkt->compl_ctxt = &comp_pkt;
-		pkt->message.message_type = PCI_RESOURCES_ASSIGNED;
 		res_assigned = (struct pci_resources_assigned *)&pkt->message;
+		res_assigned->message_type.type = PCI_RESOURCES_ASSIGNED;
 		res_assigned->wslot.slot = hpdev->desc.win_slot.slot;
 
 		put_pcichild(hpdev, hv_pcidev_ref_by_slot);
@@ -2123,7 +2122,7 @@ static int hv_send_resources_released(struct hv_device *hdev)
 			continue;
 
 		memset(&pkt, 0, sizeof(pkt));
-		pkt.message_type = PCI_RESOURCES_RELEASED;
+		pkt.message_type.type = PCI_RESOURCES_RELEASED;
 		pkt.wslot.slot = hpdev->desc.win_slot.slot;
 
 		put_pcichild(hpdev, hv_pcidev_ref_by_slot);
@@ -2290,7 +2289,7 @@ static int hv_pci_remove(struct hv_device *hdev)
 	init_completion(&comp_pkt.host_event);
 	pkt.teardown_packet.completion_func = hv_pci_generic_compl;
 	pkt.teardown_packet.compl_ctxt = &comp_pkt;
-	pkt.teardown_packet.message.message_type = PCI_BUS_D0EXIT;
+	pkt.teardown_packet.message[0].type = PCI_BUS_D0EXIT;
 
 	ret = vmbus_sendpacket(hdev->channel, &pkt.teardown_packet.message,
 			       sizeof(struct pci_message),
diff --git a/drivers/pci/host/pci-imx6.c b/drivers/pci/host/pci-imx6.c
index b741a36a67f3..ead4a5c3480b 100644
--- a/drivers/pci/host/pci-imx6.c
+++ b/drivers/pci/host/pci-imx6.c
@@ -739,7 +739,6 @@ static const struct of_device_id imx6_pcie_of_match[] = {
 	{ .compatible = "fsl,imx6qp-pcie", .data = (void *)IMX6QP, },
 	{},
 };
-MODULE_DEVICE_TABLE(of, imx6_pcie_of_match);
 
 static struct platform_driver imx6_pcie_driver = {
 	.driver = {
@@ -749,14 +748,8 @@ static struct platform_driver imx6_pcie_driver = {
 	.shutdown = imx6_pcie_shutdown,
 };
 
-/* Freescale PCIe driver does not allow module unload */
-
 static int __init imx6_pcie_init(void)
 {
 	return platform_driver_probe(&imx6_pcie_driver, imx6_pcie_probe);
 }
-module_init(imx6_pcie_init);
-
-MODULE_AUTHOR("Sean Cross <xobs@kosagi.com>");
-MODULE_DESCRIPTION("Freescale i.MX6 PCIe host controller driver");
-MODULE_LICENSE("GPL v2");
+device_initcall(imx6_pcie_init);
diff --git a/drivers/pci/host/pci-keystone.c b/drivers/pci/host/pci-keystone.c
index 8ba28834d470..82b461b5b08a 100644
--- a/drivers/pci/host/pci-keystone.c
+++ b/drivers/pci/host/pci-keystone.c
@@ -334,8 +334,9 @@ static int __init ks_add_pcie_port(struct keystone_pcie *ks_pcie,
 	if (ks_pcie->error_irq <= 0)
 		dev_info(&pdev->dev, "no error IRQ defined\n");
 	else {
-		if (request_irq(ks_pcie->error_irq, pcie_err_irq_handler,
-				IRQF_SHARED, "pcie-error-irq", ks_pcie) < 0) {
+		ret = request_irq(ks_pcie->error_irq, pcie_err_irq_handler,
+				  IRQF_SHARED, "pcie-error-irq", ks_pcie);
+		if (ret < 0) {
 			dev_err(&pdev->dev, "failed to request error IRQ %d\n",
 				ks_pcie->error_irq);
 			return ret;
diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index 6de0757b11e4..e2a8e4cab22e 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -240,7 +240,7 @@ struct tegra_msi {
 };
 
 /* used to differentiate between Tegra SoC generations */
-struct tegra_pcie_soc_data {
+struct tegra_pcie_soc {
 	unsigned int num_ports;
 	unsigned int msi_base_shift;
 	u32 pads_pll_ctl;
@@ -300,7 +300,7 @@ struct tegra_pcie {
 	struct regulator_bulk_data *supplies;
 	unsigned int num_supplies;
 
-	const struct tegra_pcie_soc_data *soc_data;
+	const struct tegra_pcie_soc *soc;
 	struct dentry *debugfs;
 };
 
@@ -542,8 +542,8 @@ static void tegra_pcie_port_reset(struct tegra_pcie_port *port)
 
 static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
 {
-	const struct tegra_pcie_soc_data *soc = port->pcie->soc_data;
 	unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
+	const struct tegra_pcie_soc *soc = port->pcie->soc;
 	unsigned long value;
 
 	/* enable reference clock */
@@ -562,8 +562,8 @@ static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
 
 static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
 {
-	const struct tegra_pcie_soc_data *soc = port->pcie->soc_data;
 	unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
+	const struct tegra_pcie_soc *soc = port->pcie->soc;
 	unsigned long value;
 
 	/* assert port reset */
@@ -621,7 +621,11 @@ static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)
 	if (err < 0)
 		return err;
 
-	pci_add_resource_offset(&sys->resources, &pcie->pio, sys->io_offset);
+	err = pci_remap_iospace(&pcie->pio, pcie->io.start);
+	if (!err)
+		pci_add_resource_offset(&sys->resources, &pcie->pio,
+					sys->io_offset);
+
 	pci_add_resource_offset(&sys->resources, &pcie->mem, sys->mem_offset);
 	pci_add_resource_offset(&sys->resources, &pcie->prefetch,
 				sys->mem_offset);
@@ -631,7 +635,6 @@ static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)
 	if (err < 0)
 		return err;
 
-	pci_remap_iospace(&pcie->pio, pcie->io.start);
 	return 1;
 }
 
@@ -774,7 +777,7 @@ static void tegra_pcie_setup_translations(struct tegra_pcie *pcie)
 
 static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout)
 {
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 	u32 value;
 
 	timeout = jiffies + msecs_to_jiffies(timeout);
@@ -790,7 +793,7 @@ static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout)
 
 static int tegra_pcie_phy_enable(struct tegra_pcie *pcie)
 {
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 	u32 value;
 	int err;
 
@@ -845,7 +848,7 @@ static int tegra_pcie_phy_enable(struct tegra_pcie *pcie)
 
 static int tegra_pcie_phy_disable(struct tegra_pcie *pcie)
 {
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 	u32 value;
 
 	/* disable TX/RX data */
@@ -906,7 +909,7 @@ static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port)
 
 static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie)
 {
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 	struct tegra_pcie_port *port;
 	int err;
 
@@ -974,7 +977,7 @@ static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie)
 
 static int tegra_pcie_enable_controller(struct tegra_pcie *pcie)
 {
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 	struct tegra_pcie_port *port;
 	unsigned long value;
 	int err;
@@ -1067,7 +1070,7 @@ static void tegra_pcie_power_off(struct tegra_pcie *pcie)
 
 static int tegra_pcie_power_on(struct tegra_pcie *pcie)
 {
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 	int err;
 
 	reset_control_assert(pcie->pcie_xrst);
@@ -1117,7 +1120,7 @@ static int tegra_pcie_power_on(struct tegra_pcie *pcie)
 
 static int tegra_pcie_clocks_get(struct tegra_pcie *pcie)
 {
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 
 	pcie->pex_clk = devm_clk_get(pcie->dev, "pex");
 	if (IS_ERR(pcie->pex_clk))
@@ -1234,7 +1237,7 @@ static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port)
 
 static int tegra_pcie_phys_get(struct tegra_pcie *pcie)
 {
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 	struct device_node *np = pcie->dev->of_node;
 	struct tegra_pcie_port *port;
 	int err;
@@ -1486,7 +1489,7 @@ static const struct irq_domain_ops msi_domain_ops = {
 static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 {
 	struct platform_device *pdev = to_platform_device(pcie->dev);
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 	struct tegra_msi *msi = &pcie->msi;
 	unsigned long base;
 	int err;
@@ -1799,8 +1802,8 @@ static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
 
 static int tegra_pcie_parse_dt(struct tegra_pcie *pcie)
 {
-	const struct tegra_pcie_soc_data *soc = pcie->soc_data;
 	struct device_node *np = pcie->dev->of_node, *port;
+	const struct tegra_pcie_soc *soc = pcie->soc;
 	struct of_pci_range_parser parser;
 	struct of_pci_range range;
 	u32 lanes = 0, mask = 0;
@@ -2043,7 +2046,7 @@ static int tegra_pcie_enable(struct tegra_pcie *pcie)
 	return 0;
 }
 
-static const struct tegra_pcie_soc_data tegra20_pcie_data = {
+static const struct tegra_pcie_soc tegra20_pcie = {
 	.num_ports = 2,
 	.msi_base_shift = 0,
 	.pads_pll_ctl = PADS_PLL_CTL_TEGRA20,
@@ -2056,7 +2059,7 @@ static const struct tegra_pcie_soc_data tegra20_pcie_data = {
 	.has_gen2 = false,
 };
 
-static const struct tegra_pcie_soc_data tegra30_pcie_data = {
+static const struct tegra_pcie_soc tegra30_pcie = {
 	.num_ports = 3,
 	.msi_base_shift = 8,
 	.pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
@@ -2070,7 +2073,7 @@ static const struct tegra_pcie_soc_data tegra30_pcie_data = {
 	.has_gen2 = false,
 };
 
-static const struct tegra_pcie_soc_data tegra124_pcie_data = {
+static const struct tegra_pcie_soc tegra124_pcie = {
 	.num_ports = 2,
 	.msi_base_shift = 8,
 	.pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
@@ -2084,9 +2087,9 @@ static const struct tegra_pcie_soc_data tegra124_pcie_data = {
 };
 
 static const struct of_device_id tegra_pcie_of_match[] = {
-	{ .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie_data },
-	{ .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie_data },
-	{ .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie_data },
+	{ .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
+	{ .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
+	{ .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
 	{ },
 };
 
@@ -2201,21 +2204,16 @@ remove:
 
 static int tegra_pcie_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *match;
 	struct tegra_pcie *pcie;
 	int err;
 
-	match = of_match_device(tegra_pcie_of_match, &pdev->dev);
-	if (!match)
-		return -ENODEV;
-
 	pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL);
 	if (!pcie)
 		return -ENOMEM;
 
+	pcie->soc = of_device_get_match_data(&pdev->dev);
 	INIT_LIST_HEAD(&pcie->buses);
 	INIT_LIST_HEAD(&pcie->ports);
-	pcie->soc_data = match->data;
 	pcie->dev = &pdev->dev;
 
 	err = tegra_pcie_parse_dt(pcie);
diff --git a/drivers/pci/host/pci-versatile.c b/drivers/pci/host/pci-versatile.c
index f234405770ab..b7dc07002f13 100644
--- a/drivers/pci/host/pci-versatile.c
+++ b/drivers/pci/host/pci-versatile.c
@@ -74,7 +74,7 @@ static int versatile_pci_parse_request_of_pci_ranges(struct device *dev,
 	int err, mem = 1, res_valid = 0;
 	struct device_node *np = dev->of_node;
 	resource_size_t iobase;
-	struct resource_entry *win;
+	struct resource_entry *win, *tmp;
 
 	err = of_pci_get_host_bridge_resources(np, 0, 0xff, res, &iobase);
 	if (err)
@@ -84,15 +84,17 @@ static int versatile_pci_parse_request_of_pci_ranges(struct device *dev,
 	if (err)
 		goto out_release_res;
 
-	resource_list_for_each_entry(win, res) {
+	resource_list_for_each_entry_safe(win, tmp, res) {
 		struct resource *res = win->res;
 
 		switch (resource_type(res)) {
 		case IORESOURCE_IO:
 			err = pci_remap_iospace(res, iobase);
-			if (err)
+			if (err) {
 				dev_warn(dev, "error %d: failed to map resource %pR\n",
 					 err, res);
+				resource_list_destroy_entry(win);
+			}
 			break;
 		case IORESOURCE_MEM:
 			res_valid |= !(res->flags & IORESOURCE_PREFETCH);
diff --git a/drivers/pci/host/pcie-altera-msi.c b/drivers/pci/host/pcie-altera-msi.c
index 99177f4ccde2..4e5d628e8cd4 100644
--- a/drivers/pci/host/pcie-altera-msi.c
+++ b/drivers/pci/host/pcie-altera-msi.c
@@ -1,4 +1,8 @@
 /*
+ * Altera PCIe MSI support
+ *
+ * Author: Ley Foon Tan <lftan@altera.com>
+ *
  * Copyright Altera Corporation (C) 2013-2015. All rights reserved
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -16,7 +20,7 @@
 
 #include <linux/interrupt.h>
 #include <linux/irqchip/chained_irq.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/msi.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
@@ -237,11 +241,6 @@ static int altera_msi_probe(struct platform_device *pdev)
 	msi->pdev = pdev;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "csr");
-	if (!res) {
-		dev_err(&pdev->dev, "no csr memory resource defined\n");
-		return -ENODEV;
-	}
-
 	msi->csr_base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(msi->csr_base)) {
 		dev_err(&pdev->dev, "failed to map csr memory\n");
@@ -250,11 +249,6 @@ static int altera_msi_probe(struct platform_device *pdev)
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 					   "vector_slave");
-	if (!res) {
-		dev_err(&pdev->dev, "no vector_slave memory resource defined\n");
-		return -ENODEV;
-	}
-
 	msi->vector_base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(msi->vector_base)) {
 		dev_err(&pdev->dev, "failed to map vector_slave memory\n");
@@ -308,7 +302,3 @@ static int __init altera_msi_init(void)
 	return platform_driver_register(&altera_msi_driver);
 }
 subsys_initcall(altera_msi_init);
-
-MODULE_AUTHOR("Ley Foon Tan <lftan@altera.com>");
-MODULE_DESCRIPTION("Altera PCIe MSI support");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/host/pcie-altera.c b/drivers/pci/host/pcie-altera.c
index 2b7837650db8..c24e96559cbb 100644
--- a/drivers/pci/host/pcie-altera.c
+++ b/drivers/pci/host/pcie-altera.c
@@ -1,6 +1,9 @@
 /*
  * Copyright Altera Corporation (C) 2013-2015. All rights reserved
  *
+ * Author: Ley Foon Tan <lftan@altera.com>
+ * Description: Altera PCIe host controller driver
+ *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
@@ -17,7 +20,7 @@
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/irqchip/chained_irq.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/of_pci.h>
@@ -43,6 +46,7 @@
 #define RP_LTSSM_MASK			0x1f
 #define LTSSM_L0			0xf
 
+#define PCIE_CAP_OFFSET			0x80
 /* TLP configuration type 0 and 1 */
 #define TLP_FMTTYPE_CFGRD0		0x04	/* Configuration Read Type 0 */
 #define TLP_FMTTYPE_CFGWR0		0x44	/* Configuration Write Type 0 */
@@ -61,7 +65,8 @@
 #define TLP_LOOP			500
 #define RP_DEVFN			0
 
-#define LINK_UP_TIMEOUT			5000
+#define LINK_UP_TIMEOUT			HZ
+#define LINK_RETRAIN_TIMEOUT		HZ
 
 #define INTX_NUM			4
 
@@ -99,38 +104,6 @@ static bool altera_pcie_link_is_up(struct altera_pcie *pcie)
 	return !!((cra_readl(pcie, RP_LTSSM) & RP_LTSSM_MASK) == LTSSM_L0);
 }
 
-static void altera_pcie_retrain(struct pci_dev *dev)
-{
-	u16 linkcap, linkstat;
-	struct altera_pcie *pcie = dev->bus->sysdata;
-	int timeout =  0;
-
-	if (!altera_pcie_link_is_up(pcie))
-		return;
-
-	/*
-	 * Set the retrain bit if the PCIe rootport support > 2.5GB/s, but
-	 * current speed is 2.5 GB/s.
-	 */
-	pcie_capability_read_word(dev, PCI_EXP_LNKCAP, &linkcap);
-
-	if ((linkcap & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
-		return;
-
-	pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &linkstat);
-	if ((linkstat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {
-		pcie_capability_set_word(dev, PCI_EXP_LNKCTL,
-					 PCI_EXP_LNKCTL_RL);
-		while (!altera_pcie_link_is_up(pcie)) {
-			timeout++;
-			if (timeout > LINK_UP_TIMEOUT)
-				break;
-			udelay(5);
-		}
-	}
-}
-DECLARE_PCI_FIXUP_EARLY(0x1172, PCI_ANY_ID, altera_pcie_retrain);
-
 /*
  * Altera PCIe port uses BAR0 of RC's configuration space as the translation
  * from PCI bus to native BUS.  Entire DDR region is mapped into PCIe space
@@ -171,13 +144,6 @@ static bool altera_pcie_valid_config(struct altera_pcie *pcie,
 	if (bus->number == pcie->root_bus_nr && dev > 0)
 		return false;
 
-	/*
-	 * Do not read more than one device on the bus directly attached
-	 * to root port, root port can only attach to one downstream port.
-	 */
-	if (bus->primary == pcie->root_bus_nr && dev > 0)
-		return false;
-
 	 return true;
 }
 
@@ -301,22 +267,14 @@ static int tlp_cfg_dword_write(struct altera_pcie *pcie, u8 bus, u32 devfn,
 	return PCIBIOS_SUCCESSFUL;
 }
 
-static int altera_pcie_cfg_read(struct pci_bus *bus, unsigned int devfn,
-				int where, int size, u32 *value)
+static int _altera_pcie_cfg_read(struct altera_pcie *pcie, u8 busno,
+				 unsigned int devfn, int where, int size,
+				 u32 *value)
 {
-	struct altera_pcie *pcie = bus->sysdata;
 	int ret;
 	u32 data;
 	u8 byte_en;
 
-	if (altera_pcie_hide_rc_bar(bus, devfn, where))
-		return PCIBIOS_BAD_REGISTER_NUMBER;
-
-	if (!altera_pcie_valid_config(pcie, bus, PCI_SLOT(devfn))) {
-		*value = 0xffffffff;
-		return PCIBIOS_DEVICE_NOT_FOUND;
-	}
-
 	switch (size) {
 	case 1:
 		byte_en = 1 << (where & 3);
@@ -329,7 +287,7 @@ static int altera_pcie_cfg_read(struct pci_bus *bus, unsigned int devfn,
 		break;
 	}
 
-	ret = tlp_cfg_dword_read(pcie, bus->number, devfn,
+	ret = tlp_cfg_dword_read(pcie, busno, devfn,
 				 (where & ~DWORD_MASK), byte_en, &data);
 	if (ret != PCIBIOS_SUCCESSFUL)
 		return ret;
@@ -349,20 +307,14 @@ static int altera_pcie_cfg_read(struct pci_bus *bus, unsigned int devfn,
 	return PCIBIOS_SUCCESSFUL;
 }
 
-static int altera_pcie_cfg_write(struct pci_bus *bus, unsigned int devfn,
-				 int where, int size, u32 value)
+static int _altera_pcie_cfg_write(struct altera_pcie *pcie, u8 busno,
+				  unsigned int devfn, int where, int size,
+				  u32 value)
 {
-	struct altera_pcie *pcie = bus->sysdata;
 	u32 data32;
 	u32 shift = 8 * (where & 3);
 	u8 byte_en;
 
-	if (altera_pcie_hide_rc_bar(bus, devfn, where))
-		return PCIBIOS_BAD_REGISTER_NUMBER;
-
-	if (!altera_pcie_valid_config(pcie, bus, PCI_SLOT(devfn)))
-		return PCIBIOS_DEVICE_NOT_FOUND;
-
 	switch (size) {
 	case 1:
 		data32 = (value & 0xff) << shift;
@@ -378,8 +330,40 @@ static int altera_pcie_cfg_write(struct pci_bus *bus, unsigned int devfn,
 		break;
 	}
 
-	return tlp_cfg_dword_write(pcie, bus->number, devfn,
-		(where & ~DWORD_MASK), byte_en, data32);
+	return tlp_cfg_dword_write(pcie, busno, devfn, (where & ~DWORD_MASK),
+				   byte_en, data32);
+}
+
+static int altera_pcie_cfg_read(struct pci_bus *bus, unsigned int devfn,
+				int where, int size, u32 *value)
+{
+	struct altera_pcie *pcie = bus->sysdata;
+
+	if (altera_pcie_hide_rc_bar(bus, devfn, where))
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+
+	if (!altera_pcie_valid_config(pcie, bus, PCI_SLOT(devfn))) {
+		*value = 0xffffffff;
+		return PCIBIOS_DEVICE_NOT_FOUND;
+	}
+
+	return _altera_pcie_cfg_read(pcie, bus->number, devfn, where, size,
+				     value);
+}
+
+static int altera_pcie_cfg_write(struct pci_bus *bus, unsigned int devfn,
+				 int where, int size, u32 value)
+{
+	struct altera_pcie *pcie = bus->sysdata;
+
+	if (altera_pcie_hide_rc_bar(bus, devfn, where))
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+
+	if (!altera_pcie_valid_config(pcie, bus, PCI_SLOT(devfn)))
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	return _altera_pcie_cfg_write(pcie, bus->number, devfn, where, size,
+				     value);
 }
 
 static struct pci_ops altera_pcie_ops = {
@@ -387,6 +371,90 @@ static struct pci_ops altera_pcie_ops = {
 	.write = altera_pcie_cfg_write,
 };
 
+static int altera_read_cap_word(struct altera_pcie *pcie, u8 busno,
+				unsigned int devfn, int offset, u16 *value)
+{
+	u32 data;
+	int ret;
+
+	ret = _altera_pcie_cfg_read(pcie, busno, devfn,
+				    PCIE_CAP_OFFSET + offset, sizeof(*value),
+				    &data);
+	*value = data;
+	return ret;
+}
+
+static int altera_write_cap_word(struct altera_pcie *pcie, u8 busno,
+				 unsigned int devfn, int offset, u16 value)
+{
+	return _altera_pcie_cfg_write(pcie, busno, devfn,
+				      PCIE_CAP_OFFSET + offset, sizeof(value),
+				      value);
+}
+
+static void altera_wait_link_retrain(struct altera_pcie *pcie)
+{
+	u16 reg16;
+	unsigned long start_jiffies;
+
+	/* Wait for link training end. */
+	start_jiffies = jiffies;
+	for (;;) {
+		altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN,
+				     PCI_EXP_LNKSTA, &reg16);
+		if (!(reg16 & PCI_EXP_LNKSTA_LT))
+			break;
+
+		if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) {
+			dev_err(&pcie->pdev->dev, "link retrain timeout\n");
+			break;
+		}
+		udelay(100);
+	}
+
+	/* Wait for link is up */
+	start_jiffies = jiffies;
+	for (;;) {
+		if (altera_pcie_link_is_up(pcie))
+			break;
+
+		if (time_after(jiffies, start_jiffies + LINK_UP_TIMEOUT)) {
+			dev_err(&pcie->pdev->dev, "link up timeout\n");
+			break;
+		}
+		udelay(100);
+	}
+}
+
+static void altera_pcie_retrain(struct altera_pcie *pcie)
+{
+	u16 linkcap, linkstat, linkctl;
+
+	if (!altera_pcie_link_is_up(pcie))
+		return;
+
+	/*
+	 * Set the retrain bit if the PCIe rootport support > 2.5GB/s, but
+	 * current speed is 2.5 GB/s.
+	 */
+	altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN, PCI_EXP_LNKCAP,
+			     &linkcap);
+	if ((linkcap & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
+		return;
+
+	altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN, PCI_EXP_LNKSTA,
+			     &linkstat);
+	if ((linkstat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {
+		altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN,
+				     PCI_EXP_LNKCTL, &linkctl);
+		linkctl |= PCI_EXP_LNKCTL_RL;
+		altera_write_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN,
+				      PCI_EXP_LNKCTL, linkctl);
+
+		altera_wait_link_retrain(pcie);
+	}
+}
+
 static int altera_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
 				irq_hw_number_t hwirq)
 {
@@ -508,6 +576,11 @@ static int altera_pcie_parse_dt(struct altera_pcie *pcie)
 	return 0;
 }
 
+static void altera_pcie_host_init(struct altera_pcie *pcie)
+{
+	altera_pcie_retrain(pcie);
+}
+
 static int altera_pcie_probe(struct platform_device *pdev)
 {
 	struct altera_pcie *pcie;
@@ -545,6 +618,7 @@ static int altera_pcie_probe(struct platform_device *pdev)
 	cra_writel(pcie, P2A_INT_STS_ALL, P2A_INT_STATUS);
 	/* enable all interrupts */
 	cra_writel(pcie, P2A_INT_ENA_ALL, P2A_INT_ENABLE);
+	altera_pcie_host_init(pcie);
 
 	bus = pci_scan_root_bus(&pdev->dev, pcie->root_bus_nr, &altera_pcie_ops,
 				pcie, &pcie->resources);
@@ -568,7 +642,6 @@ static const struct of_device_id altera_pcie_of_match[] = {
 	{ .compatible = "altr,pcie-root-port-1.0", },
 	{},
 };
-MODULE_DEVICE_TABLE(of, altera_pcie_of_match);
 
 static struct platform_driver altera_pcie_driver = {
 	.probe		= altera_pcie_probe,
@@ -583,8 +656,4 @@ static int altera_pcie_init(void)
 {
 	return platform_driver_register(&altera_pcie_driver);
 }
-module_init(altera_pcie_init);
-
-MODULE_AUTHOR("Ley Foon Tan <lftan@altera.com>");
-MODULE_DESCRIPTION("Altera PCIe host controller driver");
-MODULE_LICENSE("GPL v2");
+device_initcall(altera_pcie_init);
diff --git a/drivers/pci/host/pcie-artpec6.c b/drivers/pci/host/pcie-artpec6.c
index 16ba70b7ec65..39bf1a6df463 100644
--- a/drivers/pci/host/pcie-artpec6.c
+++ b/drivers/pci/host/pcie-artpec6.c
@@ -191,8 +191,8 @@ static irqreturn_t artpec6_pcie_msi_handler(int irq, void *arg)
 	return dw_handle_msi_irq(pp);
 }
 
-static int __init artpec6_add_pcie_port(struct pcie_port *pp,
-					struct platform_device *pdev)
+static int artpec6_add_pcie_port(struct pcie_port *pp,
+				 struct platform_device *pdev)
 {
 	int ret;
 
diff --git a/drivers/pci/host/pcie-designware-plat.c b/drivers/pci/host/pcie-designware-plat.c
index c8079dc81c10..17da005497a5 100644
--- a/drivers/pci/host/pcie-designware-plat.c
+++ b/drivers/pci/host/pcie-designware-plat.c
@@ -100,9 +100,6 @@ static int dw_plat_pcie_probe(struct platform_device *pdev)
 	pp->dev = &pdev->dev;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENODEV;
-
 	dw_plat_pcie->mem_base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(dw_plat_pcie->mem_base))
 		return PTR_ERR(dw_plat_pcie->mem_base);
diff --git a/drivers/pci/host/pcie-designware.c b/drivers/pci/host/pcie-designware.c
index 12afce19890b..74da71ea544a 100644
--- a/drivers/pci/host/pcie-designware.c
+++ b/drivers/pci/host/pcie-designware.c
@@ -14,7 +14,6 @@
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
-#include <linux/module.h>
 #include <linux/msi.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
@@ -26,7 +25,17 @@
 
 #include "pcie-designware.h"
 
-/* Synopsis specific PCIE configuration registers */
+/* Parameters for the waiting for link up routine */
+#define LINK_WAIT_MAX_RETRIES		10
+#define LINK_WAIT_USLEEP_MIN		90000
+#define LINK_WAIT_USLEEP_MAX		100000
+
+/* Parameters for the waiting for iATU enabled routine */
+#define LINK_WAIT_MAX_IATU_RETRIES	5
+#define LINK_WAIT_IATU_MIN		9000
+#define LINK_WAIT_IATU_MAX		10000
+
+/* Synopsys-specific PCIe configuration registers */
 #define PCIE_PORT_LINK_CONTROL		0x710
 #define PORT_LINK_MODE_MASK		(0x3f << 16)
 #define PORT_LINK_MODE_1_LANES		(0x1 << 16)
@@ -51,6 +60,7 @@
 #define PCIE_ATU_VIEWPORT		0x900
 #define PCIE_ATU_REGION_INBOUND		(0x1 << 31)
 #define PCIE_ATU_REGION_OUTBOUND	(0x0 << 31)
+#define PCIE_ATU_REGION_INDEX2		(0x2 << 0)
 #define PCIE_ATU_REGION_INDEX1		(0x1 << 0)
 #define PCIE_ATU_REGION_INDEX0		(0x0 << 0)
 #define PCIE_ATU_CR1			0x904
@@ -70,10 +80,26 @@
 #define PCIE_ATU_FUNC(x)		(((x) & 0x7) << 16)
 #define PCIE_ATU_UPPER_TARGET		0x91C
 
+/*
+ * iATU Unroll-specific register definitions
+ * From 4.80 core version the address translation will be made by unroll
+ */
+#define PCIE_ATU_UNR_REGION_CTRL1	0x00
+#define PCIE_ATU_UNR_REGION_CTRL2	0x04
+#define PCIE_ATU_UNR_LOWER_BASE		0x08
+#define PCIE_ATU_UNR_UPPER_BASE		0x0C
+#define PCIE_ATU_UNR_LIMIT		0x10
+#define PCIE_ATU_UNR_LOWER_TARGET	0x14
+#define PCIE_ATU_UNR_UPPER_TARGET	0x18
+
+/* Register address builder */
+#define PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(region)  ((0x3 << 20) | (region << 9))
+
 /* PCIe Port Logic registers */
 #define PLR_OFFSET			0x700
 #define PCIE_PHY_DEBUG_R1		(PLR_OFFSET + 0x2c)
-#define PCIE_PHY_DEBUG_R1_LINK_UP	0x00000010
+#define PCIE_PHY_DEBUG_R1_LINK_UP	(0x1 << 4)
+#define PCIE_PHY_DEBUG_R1_LINK_IN_TRAINING	(0x1 << 29)
 
 static struct pci_ops dw_pcie_ops;
 
@@ -115,12 +141,12 @@ int dw_pcie_cfg_write(void __iomem *addr, int size, u32 val)
 	return PCIBIOS_SUCCESSFUL;
 }
 
-static inline void dw_pcie_readl_rc(struct pcie_port *pp, u32 reg, u32 *val)
+static inline u32 dw_pcie_readl_rc(struct pcie_port *pp, u32 reg)
 {
 	if (pp->ops->readl_rc)
-		pp->ops->readl_rc(pp, pp->dbi_base + reg, val);
-	else
-		*val = readl(pp->dbi_base + reg);
+		return pp->ops->readl_rc(pp, pp->dbi_base + reg);
+
+	return readl(pp->dbi_base + reg);
 }
 
 static inline void dw_pcie_writel_rc(struct pcie_port *pp, u32 val, u32 reg)
@@ -131,6 +157,27 @@ static inline void dw_pcie_writel_rc(struct pcie_port *pp, u32 val, u32 reg)
 		writel(val, pp->dbi_base + reg);
 }
 
+static inline u32 dw_pcie_readl_unroll(struct pcie_port *pp, u32 index, u32 reg)
+{
+	u32 offset = PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(index);
+
+	if (pp->ops->readl_rc)
+		return pp->ops->readl_rc(pp, pp->dbi_base + offset + reg);
+
+	return readl(pp->dbi_base + offset + reg);
+}
+
+static inline void dw_pcie_writel_unroll(struct pcie_port *pp, u32 index,
+					 u32 val, u32 reg)
+{
+	u32 offset = PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(index);
+
+	if (pp->ops->writel_rc)
+		pp->ops->writel_rc(pp, val, pp->dbi_base + offset + reg);
+	else
+		writel(val, pp->dbi_base + offset + reg);
+}
+
 static int dw_pcie_rd_own_conf(struct pcie_port *pp, int where, int size,
 			       u32 *val)
 {
@@ -152,24 +199,57 @@ static int dw_pcie_wr_own_conf(struct pcie_port *pp, int where, int size,
 static void dw_pcie_prog_outbound_atu(struct pcie_port *pp, int index,
 		int type, u64 cpu_addr, u64 pci_addr, u32 size)
 {
-	u32 val;
-
-	dw_pcie_writel_rc(pp, PCIE_ATU_REGION_OUTBOUND | index,
-			  PCIE_ATU_VIEWPORT);
-	dw_pcie_writel_rc(pp, lower_32_bits(cpu_addr), PCIE_ATU_LOWER_BASE);
-	dw_pcie_writel_rc(pp, upper_32_bits(cpu_addr), PCIE_ATU_UPPER_BASE);
-	dw_pcie_writel_rc(pp, lower_32_bits(cpu_addr + size - 1),
-			  PCIE_ATU_LIMIT);
-	dw_pcie_writel_rc(pp, lower_32_bits(pci_addr), PCIE_ATU_LOWER_TARGET);
-	dw_pcie_writel_rc(pp, upper_32_bits(pci_addr), PCIE_ATU_UPPER_TARGET);
-	dw_pcie_writel_rc(pp, type, PCIE_ATU_CR1);
-	dw_pcie_writel_rc(pp, PCIE_ATU_ENABLE, PCIE_ATU_CR2);
+	u32 retries, val;
+
+	if (pp->iatu_unroll_enabled) {
+		dw_pcie_writel_unroll(pp, index,
+			lower_32_bits(cpu_addr), PCIE_ATU_UNR_LOWER_BASE);
+		dw_pcie_writel_unroll(pp, index,
+			upper_32_bits(cpu_addr), PCIE_ATU_UNR_UPPER_BASE);
+		dw_pcie_writel_unroll(pp, index,
+			lower_32_bits(cpu_addr + size - 1), PCIE_ATU_UNR_LIMIT);
+		dw_pcie_writel_unroll(pp, index,
+			lower_32_bits(pci_addr), PCIE_ATU_UNR_LOWER_TARGET);
+		dw_pcie_writel_unroll(pp, index,
+			upper_32_bits(pci_addr), PCIE_ATU_UNR_UPPER_TARGET);
+		dw_pcie_writel_unroll(pp, index,
+			type, PCIE_ATU_UNR_REGION_CTRL1);
+		dw_pcie_writel_unroll(pp, index,
+			PCIE_ATU_ENABLE, PCIE_ATU_UNR_REGION_CTRL2);
+	} else {
+		dw_pcie_writel_rc(pp, PCIE_ATU_REGION_OUTBOUND | index,
+						PCIE_ATU_VIEWPORT);
+		dw_pcie_writel_rc(pp, lower_32_bits(cpu_addr),
+						PCIE_ATU_LOWER_BASE);
+		dw_pcie_writel_rc(pp, upper_32_bits(cpu_addr),
+						PCIE_ATU_UPPER_BASE);
+		dw_pcie_writel_rc(pp, lower_32_bits(cpu_addr + size - 1),
+						PCIE_ATU_LIMIT);
+		dw_pcie_writel_rc(pp, lower_32_bits(pci_addr),
+						PCIE_ATU_LOWER_TARGET);
+		dw_pcie_writel_rc(pp, upper_32_bits(pci_addr),
+						PCIE_ATU_UPPER_TARGET);
+		dw_pcie_writel_rc(pp, type, PCIE_ATU_CR1);
+		dw_pcie_writel_rc(pp, PCIE_ATU_ENABLE, PCIE_ATU_CR2);
+	}
 
 	/*
 	 * Make sure ATU enable takes effect before any subsequent config
 	 * and I/O accesses.
 	 */
-	dw_pcie_readl_rc(pp, PCIE_ATU_CR2, &val);
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
+		if (pp->iatu_unroll_enabled)
+			val = dw_pcie_readl_unroll(pp, index,
+						   PCIE_ATU_UNR_REGION_CTRL2);
+		else
+			val = dw_pcie_readl_rc(pp, PCIE_ATU_CR2);
+
+		if (val == PCIE_ATU_ENABLE)
+			return;
+
+		usleep_range(LINK_WAIT_IATU_MIN, LINK_WAIT_IATU_MAX);
+	}
+	dev_err(pp->dev, "iATU is not being enabled\n");
 }
 
 static struct irq_chip dw_msi_irq_chip = {
@@ -412,7 +492,8 @@ int dw_pcie_link_up(struct pcie_port *pp)
 		return pp->ops->link_up(pp);
 
 	val = readl(pp->dbi_base + PCIE_PHY_DEBUG_R1);
-	return val & PCIE_PHY_DEBUG_R1_LINK_UP;
+	return ((val & PCIE_PHY_DEBUG_R1_LINK_UP) &&
+		(!(val & PCIE_PHY_DEBUG_R1_LINK_IN_TRAINING)));
 }
 
 static int dw_pcie_msi_map(struct irq_domain *domain, unsigned int irq,
@@ -428,6 +509,17 @@ static const struct irq_domain_ops msi_domain_ops = {
 	.map = dw_pcie_msi_map,
 };
 
+static u8 dw_pcie_iatu_unroll_enabled(struct pcie_port *pp)
+{
+	u32 val;
+
+	val = dw_pcie_readl_rc(pp, PCIE_ATU_VIEWPORT);
+	if (val == 0xffffffff)
+		return 1;
+
+	return 0;
+}
+
 int dw_pcie_host_init(struct pcie_port *pp)
 {
 	struct device_node *np = pp->dev->of_node;
@@ -436,7 +528,7 @@ int dw_pcie_host_init(struct pcie_port *pp)
 	struct resource *cfg_res;
 	int i, ret;
 	LIST_HEAD(res);
-	struct resource_entry *win;
+	struct resource_entry *win, *tmp;
 
 	cfg_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
 	if (cfg_res) {
@@ -457,17 +549,20 @@ int dw_pcie_host_init(struct pcie_port *pp)
 		goto error;
 
 	/* Get the I/O and memory ranges from DT */
-	resource_list_for_each_entry(win, &res) {
+	resource_list_for_each_entry_safe(win, tmp, &res) {
 		switch (resource_type(win->res)) {
 		case IORESOURCE_IO:
-			pp->io = win->res;
-			pp->io->name = "I/O";
-			pp->io_size = resource_size(pp->io);
-			pp->io_bus_addr = pp->io->start - win->offset;
-			ret = pci_remap_iospace(pp->io, pp->io_base);
-			if (ret)
+			ret = pci_remap_iospace(win->res, pp->io_base);
+			if (ret) {
 				dev_warn(pp->dev, "error %d: failed to map resource %pR\n",
-					 ret, pp->io);
+					 ret, win->res);
+				resource_list_destroy_entry(win);
+			} else {
+				pp->io = win->res;
+				pp->io->name = "I/O";
+				pp->io_size = resource_size(pp->io);
+				pp->io_bus_addr = pp->io->start - win->offset;
+			}
 			break;
 		case IORESOURCE_MEM:
 			pp->mem = win->res;
@@ -524,6 +619,10 @@ int dw_pcie_host_init(struct pcie_port *pp)
 	if (ret)
 		pp->lanes = 0;
 
+	ret = of_property_read_u32(np, "num-viewport", &pp->num_viewport);
+	if (ret)
+		pp->num_viewport = 2;
+
 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
 		if (!pp->ops->msi_host_init) {
 			pp->irq_domain = irq_domain_add_linear(pp->dev->of_node,
@@ -544,6 +643,8 @@ int dw_pcie_host_init(struct pcie_port *pp)
 		}
 	}
 
+	pp->iatu_unroll_enabled = dw_pcie_iatu_unroll_enabled(pp);
+
 	if (pp->ops->host_init)
 		pp->ops->host_init(pp);
 
@@ -609,13 +710,14 @@ static int dw_pcie_rd_other_conf(struct pcie_port *pp, struct pci_bus *bus,
 		va_cfg_base = pp->va_cfg1_base;
 	}
 
-	dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX0,
+	dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX1,
 				  type, cpu_addr,
 				  busdev, cfg_size);
 	ret = dw_pcie_cfg_read(va_cfg_base + where, size, val);
-	dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX0,
-				  PCIE_ATU_TYPE_IO, pp->io_base,
-				  pp->io_bus_addr, pp->io_size);
+	if (pp->num_viewport <= 2)
+		dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX1,
+					  PCIE_ATU_TYPE_IO, pp->io_base,
+					  pp->io_bus_addr, pp->io_size);
 
 	return ret;
 }
@@ -646,13 +748,14 @@ static int dw_pcie_wr_other_conf(struct pcie_port *pp, struct pci_bus *bus,
 		va_cfg_base = pp->va_cfg1_base;
 	}
 
-	dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX0,
+	dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX1,
 				  type, cpu_addr,
 				  busdev, cfg_size);
 	ret = dw_pcie_cfg_write(va_cfg_base + where, size, val);
-	dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX0,
-				  PCIE_ATU_TYPE_IO, pp->io_base,
-				  pp->io_bus_addr, pp->io_size);
+	if (pp->num_viewport <= 2)
+		dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX1,
+					  PCIE_ATU_TYPE_IO, pp->io_base,
+					  pp->io_bus_addr, pp->io_size);
 
 	return ret;
 }
@@ -670,13 +773,6 @@ static int dw_pcie_valid_config(struct pcie_port *pp,
 	if (bus->number == pp->root_bus_nr && dev > 0)
 		return 0;
 
-	/*
-	 * do not read more than one device on the bus directly attached
-	 * to RC's (Virtual Bridge's) DS side.
-	 */
-	if (bus->primary == pp->root_bus_nr && dev > 0)
-		return 0;
-
 	return 1;
 }
 
@@ -720,7 +816,7 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 	u32 val;
 
 	/* set the number of lanes */
-	dw_pcie_readl_rc(pp, PCIE_PORT_LINK_CONTROL, &val);
+	val = dw_pcie_readl_rc(pp, PCIE_PORT_LINK_CONTROL);
 	val &= ~PORT_LINK_MODE_MASK;
 	switch (pp->lanes) {
 	case 1:
@@ -742,7 +838,7 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 	dw_pcie_writel_rc(pp, val, PCIE_PORT_LINK_CONTROL);
 
 	/* set link width speed control register */
-	dw_pcie_readl_rc(pp, PCIE_LINK_WIDTH_SPEED_CONTROL, &val);
+	val = dw_pcie_readl_rc(pp, PCIE_LINK_WIDTH_SPEED_CONTROL);
 	val &= ~PORT_LOGIC_LINK_WIDTH_MASK;
 	switch (pp->lanes) {
 	case 1:
@@ -765,19 +861,19 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 	dw_pcie_writel_rc(pp, 0x00000000, PCI_BASE_ADDRESS_1);
 
 	/* setup interrupt pins */
-	dw_pcie_readl_rc(pp, PCI_INTERRUPT_LINE, &val);
+	val = dw_pcie_readl_rc(pp, PCI_INTERRUPT_LINE);
 	val &= 0xffff00ff;
 	val |= 0x00000100;
 	dw_pcie_writel_rc(pp, val, PCI_INTERRUPT_LINE);
 
 	/* setup bus numbers */
-	dw_pcie_readl_rc(pp, PCI_PRIMARY_BUS, &val);
+	val = dw_pcie_readl_rc(pp, PCI_PRIMARY_BUS);
 	val &= 0xff000000;
 	val |= 0x00010100;
 	dw_pcie_writel_rc(pp, val, PCI_PRIMARY_BUS);
 
 	/* setup command register */
-	dw_pcie_readl_rc(pp, PCI_COMMAND, &val);
+	val = dw_pcie_readl_rc(pp, PCI_COMMAND);
 	val &= 0xffff0000;
 	val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
 		PCI_COMMAND_MASTER | PCI_COMMAND_SERR;
@@ -788,10 +884,15 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 	 * uses its own address translation component rather than ATU, so
 	 * we should not program the ATU here.
 	 */
-	if (!pp->ops->rd_other_conf)
-		dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX1,
+	if (!pp->ops->rd_other_conf) {
+		dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX0,
 					  PCIE_ATU_TYPE_MEM, pp->mem_base,
 					  pp->mem_bus_addr, pp->mem_size);
+		if (pp->num_viewport > 2)
+			dw_pcie_prog_outbound_atu(pp, PCIE_ATU_REGION_INDEX2,
+						  PCIE_ATU_TYPE_IO, pp->io_base,
+						  pp->io_bus_addr, pp->io_size);
+	}
 
 	dw_pcie_wr_own_conf(pp, PCI_BASE_ADDRESS_0, 4, 0);
 
@@ -802,7 +903,3 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 	val |= PORT_LOGIC_SPEED_CHANGE;
 	dw_pcie_wr_own_conf(pp, PCIE_LINK_WIDTH_SPEED_CONTROL, 4, val);
 }
-
-MODULE_AUTHOR("Jingoo Han <jg1.han@samsung.com>");
-MODULE_DESCRIPTION("Designware PCIe host controller driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/host/pcie-designware.h b/drivers/pci/host/pcie-designware.h
index f437f9b5be04..c8e5bc647f49 100644
--- a/drivers/pci/host/pcie-designware.h
+++ b/drivers/pci/host/pcie-designware.h
@@ -22,11 +22,6 @@
 #define MAX_MSI_IRQS			32
 #define MAX_MSI_CTRLS			(MAX_MSI_IRQS / 32)
 
-/* Parameters for the waiting for link up routine */
-#define LINK_WAIT_MAX_RETRIES		10
-#define LINK_WAIT_USLEEP_MIN		90000
-#define LINK_WAIT_USLEEP_MAX		100000
-
 struct pcie_port {
 	struct device		*dev;
 	u8			root_bus_nr;
@@ -49,16 +44,17 @@ struct pcie_port {
 	struct resource		*busn;
 	int			irq;
 	u32			lanes;
+	u32			num_viewport;
 	struct pcie_host_ops	*ops;
 	int			msi_irq;
 	struct irq_domain	*irq_domain;
 	unsigned long		msi_data;
+	u8			iatu_unroll_enabled;
 	DECLARE_BITMAP(msi_irq_in_use, MAX_MSI_IRQS);
 };
 
 struct pcie_host_ops {
-	void (*readl_rc)(struct pcie_port *pp,
-			void __iomem *dbi_base, u32 *val);
+	u32 (*readl_rc)(struct pcie_port *pp, void __iomem *dbi_base);
 	void (*writel_rc)(struct pcie_port *pp,
 			u32 val, void __iomem *dbi_base);
 	int (*rd_own_conf)(struct pcie_port *pp, int where, int size, u32 *val);
diff --git a/drivers/pci/host/pcie-qcom.c b/drivers/pci/host/pcie-qcom.c
index f2f90c50f75d..5ec2d440a6b7 100644
--- a/drivers/pci/host/pcie-qcom.c
+++ b/drivers/pci/host/pcie-qcom.c
@@ -1,7 +1,11 @@
 /*
+ * Qualcomm PCIe root complex driver
+ *
  * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
  * Copyright 2015 Linaro Limited.
  *
+ * Author: Stanimir Varbanov <svarbanov@mm-sol.com>
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
  * only version 2 as published by the Free Software Foundation.
@@ -19,7 +23,7 @@
 #include <linux/io.h>
 #include <linux/iopoll.h>
 #include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of_device.h>
 #include <linux/of_gpio.h>
 #include <linux/pci.h>
@@ -570,37 +574,19 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int qcom_pcie_remove(struct platform_device *pdev)
-{
-	struct qcom_pcie *pcie = platform_get_drvdata(pdev);
-
-	qcom_ep_reset_assert(pcie);
-	phy_power_off(pcie->phy);
-	phy_exit(pcie->phy);
-	pcie->ops->deinit(pcie);
-
-	return 0;
-}
-
 static const struct of_device_id qcom_pcie_match[] = {
 	{ .compatible = "qcom,pcie-ipq8064", .data = &ops_v0 },
 	{ .compatible = "qcom,pcie-apq8064", .data = &ops_v0 },
 	{ .compatible = "qcom,pcie-apq8084", .data = &ops_v1 },
 	{ }
 };
-MODULE_DEVICE_TABLE(of, qcom_pcie_match);
 
 static struct platform_driver qcom_pcie_driver = {
 	.probe = qcom_pcie_probe,
-	.remove = qcom_pcie_remove,
 	.driver = {
 		.name = "qcom-pcie",
+		.suppress_bind_attrs = true,
 		.of_match_table = qcom_pcie_match,
 	},
 };
-
-module_platform_driver(qcom_pcie_driver);
-
-MODULE_AUTHOR("Stanimir Varbanov <svarbanov@mm-sol.com>");
-MODULE_DESCRIPTION("Qualcomm PCIe root complex driver");
-MODULE_LICENSE("GPL v2");
+builtin_platform_driver(qcom_pcie_driver);
diff --git a/drivers/pci/host/pcie-rcar.c b/drivers/pci/host/pcie-rcar.c
index 65db7a221509..e06b1d3b4dea 100644
--- a/drivers/pci/host/pcie-rcar.c
+++ b/drivers/pci/host/pcie-rcar.c
@@ -84,8 +84,18 @@
 #define IDSETR1			0x011004
 #define TLCTLR			0x011048
 #define MACSR			0x011054
+#define  SPCHGFIN		(1 << 4)
+#define  SPCHGFAIL		(1 << 6)
+#define  SPCHGSUC		(1 << 7)
+#define  LINK_SPEED		(0xf << 16)
+#define  LINK_SPEED_2_5GTS	(1 << 16)
+#define  LINK_SPEED_5_0GTS	(2 << 16)
 #define MACCTLR			0x011058
+#define  SPEED_CHANGE		(1 << 24)
 #define  SCRAMBLE_DISABLE	(1 << 27)
+#define MACS2R			0x011078
+#define MACCGSPSETR		0x011084
+#define  SPCNGRSN		(1 << 31)
 
 /* R-Car H1 PHY */
 #define H1_PCIEPHYADRR		0x04000c
@@ -385,11 +395,67 @@ static int rcar_pcie_setup(struct list_head *resource, struct rcar_pcie *pci)
 	return 1;
 }
 
+static void rcar_pcie_force_speedup(struct rcar_pcie *pcie)
+{
+	unsigned int timeout = 1000;
+	u32 macsr;
+
+	if ((rcar_pci_read_reg(pcie, MACS2R) & LINK_SPEED) != LINK_SPEED_5_0GTS)
+		return;
+
+	if (rcar_pci_read_reg(pcie, MACCTLR) & SPEED_CHANGE) {
+		dev_err(pcie->dev, "Speed change already in progress\n");
+		return;
+	}
+
+	macsr = rcar_pci_read_reg(pcie, MACSR);
+	if ((macsr & LINK_SPEED) == LINK_SPEED_5_0GTS)
+		goto done;
+
+	/* Set target link speed to 5.0 GT/s */
+	rcar_rmw32(pcie, EXPCAP(12), PCI_EXP_LNKSTA_CLS,
+		   PCI_EXP_LNKSTA_CLS_5_0GB);
+
+	/* Set speed change reason as intentional factor */
+	rcar_rmw32(pcie, MACCGSPSETR, SPCNGRSN, 0);
+
+	/* Clear SPCHGFIN, SPCHGSUC, and SPCHGFAIL */
+	if (macsr & (SPCHGFIN | SPCHGSUC | SPCHGFAIL))
+		rcar_pci_write_reg(pcie, macsr, MACSR);
+
+	/* Start link speed change */
+	rcar_rmw32(pcie, MACCTLR, SPEED_CHANGE, SPEED_CHANGE);
+
+	while (timeout--) {
+		macsr = rcar_pci_read_reg(pcie, MACSR);
+		if (macsr & SPCHGFIN) {
+			/* Clear the interrupt bits */
+			rcar_pci_write_reg(pcie, macsr, MACSR);
+
+			if (macsr & SPCHGFAIL)
+				dev_err(pcie->dev, "Speed change failed\n");
+
+			goto done;
+		}
+
+		msleep(1);
+	};
+
+	dev_err(pcie->dev, "Speed change timed out\n");
+
+done:
+	dev_info(pcie->dev, "Current link speed is %s GT/s\n",
+		 (macsr & LINK_SPEED) == LINK_SPEED_5_0GTS ? "5" : "2.5");
+}
+
 static int rcar_pcie_enable(struct rcar_pcie *pcie)
 {
 	struct pci_bus *bus, *child;
 	LIST_HEAD(res);
 
+	/* Try setting 5 GT/s link speed */
+	rcar_pcie_force_speedup(pcie);
+
 	rcar_pcie_setup(&res, pcie);
 
 	pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);
@@ -608,6 +674,18 @@ static int rcar_msi_alloc(struct rcar_msi *chip)
 	return msi;
 }
 
+static int rcar_msi_alloc_region(struct rcar_msi *chip, int no_irqs)
+{
+	int msi;
+
+	mutex_lock(&chip->lock);
+	msi = bitmap_find_free_region(chip->used, INT_PCI_MSI_NR,
+				      order_base_2(no_irqs));
+	mutex_unlock(&chip->lock);
+
+	return msi;
+}
+
 static void rcar_msi_free(struct rcar_msi *chip, unsigned long irq)
 {
 	mutex_lock(&chip->lock);
@@ -665,7 +743,7 @@ static int rcar_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
 	if (hwirq < 0)
 		return hwirq;
 
-	irq = irq_create_mapping(msi->domain, hwirq);
+	irq = irq_find_mapping(msi->domain, hwirq);
 	if (!irq) {
 		rcar_msi_free(msi, hwirq);
 		return -EINVAL;
@@ -682,6 +760,58 @@ static int rcar_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
 	return 0;
 }
 
+static int rcar_msi_setup_irqs(struct msi_controller *chip,
+			       struct pci_dev *pdev, int nvec, int type)
+{
+	struct rcar_pcie *pcie = container_of(chip, struct rcar_pcie, msi.chip);
+	struct rcar_msi *msi = to_rcar_msi(chip);
+	struct msi_desc *desc;
+	struct msi_msg msg;
+	unsigned int irq;
+	int hwirq;
+	int i;
+
+	/* MSI-X interrupts are not supported */
+	if (type == PCI_CAP_ID_MSIX)
+		return -EINVAL;
+
+	WARN_ON(!list_is_singular(&pdev->dev.msi_list));
+	desc = list_entry(pdev->dev.msi_list.next, struct msi_desc, list);
+
+	hwirq = rcar_msi_alloc_region(msi, nvec);
+	if (hwirq < 0)
+		return -ENOSPC;
+
+	irq = irq_find_mapping(msi->domain, hwirq);
+	if (!irq)
+		return -ENOSPC;
+
+	for (i = 0; i < nvec; i++) {
+		/*
+		 * irq_create_mapping() called from rcar_pcie_probe() pre-
+		 * allocates descs,  so there is no need to allocate descs here.
+		 * We can therefore assume that if irq_find_mapping() above
+		 * returns non-zero, then the descs are also successfully
+		 * allocated.
+		 */
+		if (irq_set_msi_desc_off(irq, i, desc)) {
+			/* TODO: clear */
+			return -EINVAL;
+		}
+	}
+
+	desc->nvec_used = nvec;
+	desc->msi_attrib.multiple = order_base_2(nvec);
+
+	msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
+	msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
+	msg.data = hwirq;
+
+	pci_write_msi_msg(irq, &msg);
+
+	return 0;
+}
+
 static void rcar_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
 {
 	struct rcar_msi *msi = to_rcar_msi(chip);
@@ -716,12 +846,13 @@ static int rcar_pcie_enable_msi(struct rcar_pcie *pcie)
 	struct platform_device *pdev = to_platform_device(pcie->dev);
 	struct rcar_msi *msi = &pcie->msi;
 	unsigned long base;
-	int err;
+	int err, i;
 
 	mutex_init(&msi->lock);
 
 	msi->chip.dev = pcie->dev;
 	msi->chip.setup_irq = rcar_msi_setup_irq;
+	msi->chip.setup_irqs = rcar_msi_setup_irqs;
 	msi->chip.teardown_irq = rcar_msi_teardown_irq;
 
 	msi->domain = irq_domain_add_linear(pcie->dev->of_node, INT_PCI_MSI_NR,
@@ -731,6 +862,9 @@ static int rcar_pcie_enable_msi(struct rcar_pcie *pcie)
 		return -ENOMEM;
 	}
 
+	for (i = 0; i < INT_PCI_MSI_NR; i++)
+		irq_create_mapping(msi->domain, i);
+
 	/* Two irqs are for MSI, but they are also used for non-MSI irqs */
 	err = devm_request_irq(&pdev->dev, msi->irq1, rcar_pcie_msi_irq,
 			       IRQF_SHARED | IRQF_NO_THREAD,
@@ -775,6 +909,10 @@ static int rcar_pcie_get_resources(struct platform_device *pdev,
 	if (err)
 		return err;
 
+	pcie->base = devm_ioremap_resource(&pdev->dev, &res);
+	if (IS_ERR(pcie->base))
+		return PTR_ERR(pcie->base);
+
 	pcie->clk = devm_clk_get(&pdev->dev, "pcie");
 	if (IS_ERR(pcie->clk)) {
 		dev_err(pcie->dev, "cannot get platform clock\n");
@@ -782,7 +920,7 @@ static int rcar_pcie_get_resources(struct platform_device *pdev,
 	}
 	err = clk_prepare_enable(pcie->clk);
 	if (err)
-		goto fail_clk;
+		return err;
 
 	pcie->bus_clk = devm_clk_get(&pdev->dev, "pcie_bus");
 	if (IS_ERR(pcie->bus_clk)) {
@@ -792,7 +930,7 @@ static int rcar_pcie_get_resources(struct platform_device *pdev,
 	}
 	err = clk_prepare_enable(pcie->bus_clk);
 	if (err)
-		goto err_map_reg;
+		goto fail_clk;
 
 	i = irq_of_parse_and_map(pdev->dev.of_node, 0);
 	if (!i) {
@@ -810,12 +948,6 @@ static int rcar_pcie_get_resources(struct platform_device *pdev,
 	}
 	pcie->msi.irq2 = i;
 
-	pcie->base = devm_ioremap_resource(&pdev->dev, &res);
-	if (IS_ERR(pcie->base)) {
-		err = PTR_ERR(pcie->base);
-		goto err_map_reg;
-	}
-
 	return 0;
 
 err_map_reg:
@@ -865,12 +997,16 @@ static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie,
 		 * Set up 64-bit inbound regions as the range parser doesn't
 		 * distinguish between 32 and 64-bit types.
 		 */
-		rcar_pci_write_reg(pcie, lower_32_bits(pci_addr), PCIEPRAR(idx));
+		rcar_pci_write_reg(pcie, lower_32_bits(pci_addr),
+				   PCIEPRAR(idx));
 		rcar_pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx));
-		rcar_pci_write_reg(pcie, lower_32_bits(mask) | flags, PCIELAMR(idx));
+		rcar_pci_write_reg(pcie, lower_32_bits(mask) | flags,
+				   PCIELAMR(idx));
 
-		rcar_pci_write_reg(pcie, upper_32_bits(pci_addr), PCIEPRAR(idx+1));
-		rcar_pci_write_reg(pcie, upper_32_bits(cpu_addr), PCIELAR(idx+1));
+		rcar_pci_write_reg(pcie, upper_32_bits(pci_addr),
+				   PCIEPRAR(idx + 1));
+		rcar_pci_write_reg(pcie, upper_32_bits(cpu_addr),
+				   PCIELAR(idx + 1));
 		rcar_pci_write_reg(pcie, 0, PCIELAMR(idx + 1));
 
 		pci_addr += size;
@@ -919,6 +1055,7 @@ static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie,
 	/* Get the dma-ranges from DT */
 	for_each_of_pci_range(&parser, &range) {
 		u64 end = range.cpu_addr + range.size - 1;
+
 		dev_dbg(pcie->dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n",
 			range.flags, range.cpu_addr, end, range.pci_addr);
 
@@ -932,9 +1069,12 @@ static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie,
 
 static const struct of_device_id rcar_pcie_of_match[] = {
 	{ .compatible = "renesas,pcie-r8a7779", .data = rcar_pcie_hw_init_h1 },
-	{ .compatible = "renesas,pcie-rcar-gen2", .data = rcar_pcie_hw_init_gen2 },
-	{ .compatible = "renesas,pcie-r8a7790", .data = rcar_pcie_hw_init_gen2 },
-	{ .compatible = "renesas,pcie-r8a7791", .data = rcar_pcie_hw_init_gen2 },
+	{ .compatible = "renesas,pcie-rcar-gen2",
+	  .data = rcar_pcie_hw_init_gen2 },
+	{ .compatible = "renesas,pcie-r8a7790",
+	  .data = rcar_pcie_hw_init_gen2 },
+	{ .compatible = "renesas,pcie-r8a7791",
+	  .data = rcar_pcie_hw_init_gen2 },
 	{ .compatible = "renesas,pcie-r8a7795", .data = rcar_pcie_hw_init },
 	{},
 };
@@ -945,9 +1085,10 @@ static int rcar_pcie_parse_request_of_pci_ranges(struct rcar_pcie *pci)
 	struct device *dev = pci->dev;
 	struct device_node *np = dev->of_node;
 	resource_size_t iobase;
-	struct resource_entry *win;
+	struct resource_entry *win, *tmp;
 
-	err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pci->resources, &iobase);
+	err = of_pci_get_host_bridge_resources(np, 0, 0xff, &pci->resources,
+					       &iobase);
 	if (err)
 		return err;
 
@@ -955,14 +1096,17 @@ static int rcar_pcie_parse_request_of_pci_ranges(struct rcar_pcie *pci)
 	if (err)
 		goto out_release_res;
 
-	resource_list_for_each_entry(win, &pci->resources) {
+	resource_list_for_each_entry_safe(win, tmp, &pci->resources) {
 		struct resource *res = win->res;
 
 		if (resource_type(res) == IORESOURCE_IO) {
 			err = pci_remap_iospace(res, iobase);
-			if (err)
+			if (err) {
 				dev_warn(dev, "error %d: failed to map resource %pR\n",
 					 err, res);
+
+				resource_list_destroy_entry(win);
+			}
 		}
 	}
 
@@ -998,8 +1142,8 @@ static int rcar_pcie_probe(struct platform_device *pdev)
 		return err;
 	}
 
-	 err = rcar_pcie_parse_map_dma_ranges(pcie, pdev->dev.of_node);
-	 if (err)
+	err = rcar_pcie_parse_map_dma_ranges(pcie, pdev->dev.of_node);
+	if (err)
 		return err;
 
 	of_id = of_match_device(rcar_pcie_of_match, pcie->dev);
diff --git a/drivers/pci/host/pcie-rockchip.c b/drivers/pci/host/pcie-rockchip.c
new file mode 100644
index 000000000000..b8c82fc812dc
--- /dev/null
+++ b/drivers/pci/host/pcie-rockchip.c
@@ -0,0 +1,1229 @@
+/*
+ * Rockchip AXI PCIe host controller driver
+ *
+ * Copyright (c) 2016 Rockchip, Inc.
+ *
+ * Author: Shawn Lin <shawn.lin@rock-chips.com>
+ *         Wenrui Li <wenrui.li@rock-chips.com>
+ *
+ * Bits taken from Synopsys Designware Host controller driver and
+ * ARM PCI Host generic driver.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/of_irq.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/regmap.h>
+
+/*
+ * The upper 16 bits of PCIE_CLIENT_CONFIG are a write mask for the lower 16
+ * bits.  This allows atomic updates of the register without locking.
+ */
+#define HIWORD_UPDATE(mask, val)	(((mask) << 16) | (val))
+#define HIWORD_UPDATE_BIT(val)		HIWORD_UPDATE(val, val)
+
+#define ENCODE_LANES(x)			((((x) >> 1) & 3) << 4)
+
+#define PCIE_CLIENT_BASE		0x0
+#define PCIE_CLIENT_CONFIG		(PCIE_CLIENT_BASE + 0x00)
+#define   PCIE_CLIENT_CONF_ENABLE	  HIWORD_UPDATE_BIT(0x0001)
+#define   PCIE_CLIENT_LINK_TRAIN_ENABLE	  HIWORD_UPDATE_BIT(0x0002)
+#define   PCIE_CLIENT_ARI_ENABLE	  HIWORD_UPDATE_BIT(0x0008)
+#define   PCIE_CLIENT_CONF_LANE_NUM(x)	  HIWORD_UPDATE(0x0030, ENCODE_LANES(x))
+#define   PCIE_CLIENT_MODE_RC		  HIWORD_UPDATE_BIT(0x0040)
+#define   PCIE_CLIENT_GEN_SEL_2		  HIWORD_UPDATE_BIT(0x0080)
+#define PCIE_CLIENT_BASIC_STATUS1	(PCIE_CLIENT_BASE + 0x48)
+#define   PCIE_CLIENT_LINK_STATUS_UP		0x00300000
+#define   PCIE_CLIENT_LINK_STATUS_MASK		0x00300000
+#define PCIE_CLIENT_INT_MASK		(PCIE_CLIENT_BASE + 0x4c)
+#define PCIE_CLIENT_INT_STATUS		(PCIE_CLIENT_BASE + 0x50)
+#define   PCIE_CLIENT_INTR_MASK			GENMASK(8, 5)
+#define   PCIE_CLIENT_INTR_SHIFT		5
+#define   PCIE_CLIENT_INT_LEGACY_DONE		BIT(15)
+#define   PCIE_CLIENT_INT_MSG			BIT(14)
+#define   PCIE_CLIENT_INT_HOT_RST		BIT(13)
+#define   PCIE_CLIENT_INT_DPA			BIT(12)
+#define   PCIE_CLIENT_INT_FATAL_ERR		BIT(11)
+#define   PCIE_CLIENT_INT_NFATAL_ERR		BIT(10)
+#define   PCIE_CLIENT_INT_CORR_ERR		BIT(9)
+#define   PCIE_CLIENT_INT_INTD			BIT(8)
+#define   PCIE_CLIENT_INT_INTC			BIT(7)
+#define   PCIE_CLIENT_INT_INTB			BIT(6)
+#define   PCIE_CLIENT_INT_INTA			BIT(5)
+#define   PCIE_CLIENT_INT_LOCAL			BIT(4)
+#define   PCIE_CLIENT_INT_UDMA			BIT(3)
+#define   PCIE_CLIENT_INT_PHY			BIT(2)
+#define   PCIE_CLIENT_INT_HOT_PLUG		BIT(1)
+#define   PCIE_CLIENT_INT_PWR_STCG		BIT(0)
+
+#define PCIE_CLIENT_INT_LEGACY \
+	(PCIE_CLIENT_INT_INTA | PCIE_CLIENT_INT_INTB | \
+	PCIE_CLIENT_INT_INTC | PCIE_CLIENT_INT_INTD)
+
+#define PCIE_CLIENT_INT_CLI \
+	(PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR | \
+	PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA | \
+	PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG | \
+	PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_LEGACY | \
+	PCIE_CLIENT_INT_PHY)
+
+#define PCIE_CORE_CTRL_MGMT_BASE	0x900000
+#define PCIE_CORE_CTRL			(PCIE_CORE_CTRL_MGMT_BASE + 0x000)
+#define   PCIE_CORE_PL_CONF_SPEED_5G		0x00000008
+#define   PCIE_CORE_PL_CONF_SPEED_MASK		0x00000018
+#define   PCIE_CORE_PL_CONF_LANE_MASK		0x00000006
+#define   PCIE_CORE_PL_CONF_LANE_SHIFT		1
+#define PCIE_CORE_CTRL_PLC1		(PCIE_CORE_CTRL_MGMT_BASE + 0x004)
+#define   PCIE_CORE_CTRL_PLC1_FTS_MASK		GENMASK(23, 8)
+#define   PCIE_CORE_CTRL_PLC1_FTS_SHIFT		8
+#define   PCIE_CORE_CTRL_PLC1_FTS_CNT		0xffff
+#define PCIE_CORE_TXCREDIT_CFG1		(PCIE_CORE_CTRL_MGMT_BASE + 0x020)
+#define   PCIE_CORE_TXCREDIT_CFG1_MUI_MASK	0xFFFF0000
+#define   PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT	16
+#define   PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(x) \
+		(((x) >> 3) << PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT)
+#define PCIE_CORE_INT_STATUS		(PCIE_CORE_CTRL_MGMT_BASE + 0x20c)
+#define   PCIE_CORE_INT_PRFPE			BIT(0)
+#define   PCIE_CORE_INT_CRFPE			BIT(1)
+#define   PCIE_CORE_INT_RRPE			BIT(2)
+#define   PCIE_CORE_INT_PRFO			BIT(3)
+#define   PCIE_CORE_INT_CRFO			BIT(4)
+#define   PCIE_CORE_INT_RT			BIT(5)
+#define   PCIE_CORE_INT_RTR			BIT(6)
+#define   PCIE_CORE_INT_PE			BIT(7)
+#define   PCIE_CORE_INT_MTR			BIT(8)
+#define   PCIE_CORE_INT_UCR			BIT(9)
+#define   PCIE_CORE_INT_FCE			BIT(10)
+#define   PCIE_CORE_INT_CT			BIT(11)
+#define   PCIE_CORE_INT_UTC			BIT(18)
+#define   PCIE_CORE_INT_MMVC			BIT(19)
+#define PCIE_CORE_INT_MASK		(PCIE_CORE_CTRL_MGMT_BASE + 0x210)
+#define PCIE_RC_BAR_CONF		(PCIE_CORE_CTRL_MGMT_BASE + 0x300)
+
+#define PCIE_CORE_INT \
+		(PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE | \
+		 PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO | \
+		 PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR | \
+		 PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR | \
+		 PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE | \
+		 PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC | \
+		 PCIE_CORE_INT_MMVC)
+
+#define PCIE_RC_CONFIG_BASE		0xa00000
+#define PCIE_RC_CONFIG_VENDOR		(PCIE_RC_CONFIG_BASE + 0x00)
+#define PCIE_RC_CONFIG_RID_CCR		(PCIE_RC_CONFIG_BASE + 0x08)
+#define   PCIE_RC_CONFIG_SCC_SHIFT		16
+#define PCIE_RC_CONFIG_LCS		(PCIE_RC_CONFIG_BASE + 0xd0)
+#define   PCIE_RC_CONFIG_LCS_RETRAIN_LINK	BIT(5)
+#define   PCIE_RC_CONFIG_LCS_LBMIE		BIT(10)
+#define   PCIE_RC_CONFIG_LCS_LABIE		BIT(11)
+#define   PCIE_RC_CONFIG_LCS_LBMS		BIT(30)
+#define   PCIE_RC_CONFIG_LCS_LAMS		BIT(31)
+#define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c)
+
+#define PCIE_CORE_AXI_CONF_BASE		0xc00000
+#define PCIE_CORE_OB_REGION_ADDR0	(PCIE_CORE_AXI_CONF_BASE + 0x0)
+#define   PCIE_CORE_OB_REGION_ADDR0_NUM_BITS	0x3f
+#define   PCIE_CORE_OB_REGION_ADDR0_LO_ADDR	0xffffff00
+#define PCIE_CORE_OB_REGION_ADDR1	(PCIE_CORE_AXI_CONF_BASE + 0x4)
+#define PCIE_CORE_OB_REGION_DESC0	(PCIE_CORE_AXI_CONF_BASE + 0x8)
+#define PCIE_CORE_OB_REGION_DESC1	(PCIE_CORE_AXI_CONF_BASE + 0xc)
+
+#define PCIE_CORE_AXI_INBOUND_BASE	0xc00800
+#define PCIE_RP_IB_ADDR0		(PCIE_CORE_AXI_INBOUND_BASE + 0x0)
+#define   PCIE_CORE_IB_REGION_ADDR0_NUM_BITS	0x3f
+#define   PCIE_CORE_IB_REGION_ADDR0_LO_ADDR	0xffffff00
+#define PCIE_RP_IB_ADDR1		(PCIE_CORE_AXI_INBOUND_BASE + 0x4)
+
+/* Size of one AXI Region (not Region 0) */
+#define AXI_REGION_SIZE				BIT(20)
+/* Size of Region 0, equal to sum of sizes of other regions */
+#define AXI_REGION_0_SIZE			(32 * (0x1 << 20))
+#define OB_REG_SIZE_SHIFT			5
+#define IB_ROOT_PORT_REG_SIZE_SHIFT		3
+#define AXI_WRAPPER_IO_WRITE			0x6
+#define AXI_WRAPPER_MEM_WRITE			0x2
+
+#define MAX_AXI_IB_ROOTPORT_REGION_NUM		3
+#define MIN_AXI_ADDR_BITS_PASSED		8
+#define ROCKCHIP_VENDOR_ID			0x1d87
+#define PCIE_ECAM_BUS(x)			(((x) & 0xff) << 20)
+#define PCIE_ECAM_DEV(x)			(((x) & 0x1f) << 15)
+#define PCIE_ECAM_FUNC(x)			(((x) & 0x7) << 12)
+#define PCIE_ECAM_REG(x)			(((x) & 0xfff) << 0)
+#define PCIE_ECAM_ADDR(bus, dev, func, reg) \
+	  (PCIE_ECAM_BUS(bus) | PCIE_ECAM_DEV(dev) | \
+	   PCIE_ECAM_FUNC(func) | PCIE_ECAM_REG(reg))
+
+#define RC_REGION_0_ADDR_TRANS_H		0x00000000
+#define RC_REGION_0_ADDR_TRANS_L		0x00000000
+#define RC_REGION_0_PASS_BITS			(25 - 1)
+#define MAX_AXI_WRAPPER_REGION_NUM		33
+
+struct rockchip_pcie {
+	void	__iomem *reg_base;		/* DT axi-base */
+	void	__iomem *apb_base;		/* DT apb-base */
+	struct	phy *phy;
+	struct	reset_control *core_rst;
+	struct	reset_control *mgmt_rst;
+	struct	reset_control *mgmt_sticky_rst;
+	struct	reset_control *pipe_rst;
+	struct	clk *aclk_pcie;
+	struct	clk *aclk_perf_pcie;
+	struct	clk *hclk_pcie;
+	struct	clk *clk_pcie_pm;
+	struct	regulator *vpcie3v3; /* 3.3V power supply */
+	struct	regulator *vpcie1v8; /* 1.8V power supply */
+	struct	regulator *vpcie0v9; /* 0.9V power supply */
+	struct	gpio_desc *ep_gpio;
+	u32	lanes;
+	u8	root_bus_nr;
+	struct	device *dev;
+	struct	irq_domain *irq_domain;
+};
+
+static u32 rockchip_pcie_read(struct rockchip_pcie *rockchip, u32 reg)
+{
+	return readl(rockchip->apb_base + reg);
+}
+
+static void rockchip_pcie_write(struct rockchip_pcie *rockchip, u32 val,
+				u32 reg)
+{
+	writel(val, rockchip->apb_base + reg);
+}
+
+static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
+{
+	u32 status;
+
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+	status |= (PCIE_RC_CONFIG_LCS_LBMIE | PCIE_RC_CONFIG_LCS_LABIE);
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+}
+
+static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
+{
+	u32 status;
+
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+	status |= (PCIE_RC_CONFIG_LCS_LBMS | PCIE_RC_CONFIG_LCS_LAMS);
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+}
+
+static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
+{
+	u32 val;
+
+	/* Update Tx credit maximum update interval */
+	val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
+	val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
+	val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000);	/* ns */
+	rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
+}
+
+static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
+				      struct pci_bus *bus, int dev)
+{
+	/* access only one slot on each root port */
+	if (bus->number == rockchip->root_bus_nr && dev > 0)
+		return 0;
+
+	/*
+	 * do not read more than one device on the bus directly attached
+	 * to RC's downstream side.
+	 */
+	if (bus->primary == rockchip->root_bus_nr && dev > 0)
+		return 0;
+
+	return 1;
+}
+
+static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
+				     int where, int size, u32 *val)
+{
+	void __iomem *addr = rockchip->apb_base + PCIE_RC_CONFIG_BASE + where;
+
+	if (!IS_ALIGNED((uintptr_t)addr, size)) {
+		*val = 0;
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	}
+
+	if (size == 4) {
+		*val = readl(addr);
+	} else if (size == 2) {
+		*val = readw(addr);
+	} else if (size == 1) {
+		*val = readb(addr);
+	} else {
+		*val = 0;
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	}
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
+				     int where, int size, u32 val)
+{
+	u32 mask, tmp, offset;
+
+	offset = where & ~0x3;
+
+	if (size == 4) {
+		writel(val, rockchip->apb_base + PCIE_RC_CONFIG_BASE + offset);
+		return PCIBIOS_SUCCESSFUL;
+	}
+
+	mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
+
+	/*
+	 * N.B. This read/modify/write isn't safe in general because it can
+	 * corrupt RW1C bits in adjacent registers.  But the hardware
+	 * doesn't support smaller writes.
+	 */
+	tmp = readl(rockchip->apb_base + PCIE_RC_CONFIG_BASE + offset) & mask;
+	tmp |= val << ((where & 0x3) * 8);
+	writel(tmp, rockchip->apb_base + PCIE_RC_CONFIG_BASE + offset);
+
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
+				       struct pci_bus *bus, u32 devfn,
+				       int where, int size, u32 *val)
+{
+	u32 busdev;
+
+	busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
+				PCI_FUNC(devfn), where);
+
+	if (!IS_ALIGNED(busdev, size)) {
+		*val = 0;
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	}
+
+	if (size == 4) {
+		*val = readl(rockchip->reg_base + busdev);
+	} else if (size == 2) {
+		*val = readw(rockchip->reg_base + busdev);
+	} else if (size == 1) {
+		*val = readb(rockchip->reg_base + busdev);
+	} else {
+		*val = 0;
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+	}
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
+				       struct pci_bus *bus, u32 devfn,
+				       int where, int size, u32 val)
+{
+	u32 busdev;
+
+	busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
+				PCI_FUNC(devfn), where);
+	if (!IS_ALIGNED(busdev, size))
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+
+	if (size == 4)
+		writel(val, rockchip->reg_base + busdev);
+	else if (size == 2)
+		writew(val, rockchip->reg_base + busdev);
+	else if (size == 1)
+		writeb(val, rockchip->reg_base + busdev);
+	else
+		return PCIBIOS_BAD_REGISTER_NUMBER;
+
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
+				 int size, u32 *val)
+{
+	struct rockchip_pcie *rockchip = bus->sysdata;
+
+	if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) {
+		*val = 0xffffffff;
+		return PCIBIOS_DEVICE_NOT_FOUND;
+	}
+
+	if (bus->number == rockchip->root_bus_nr)
+		return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
+
+	return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, val);
+}
+
+static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
+				 int where, int size, u32 val)
+{
+	struct rockchip_pcie *rockchip = bus->sysdata;
+
+	if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	if (bus->number == rockchip->root_bus_nr)
+		return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
+
+	return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, val);
+}
+
+static struct pci_ops rockchip_pcie_ops = {
+	.read = rockchip_pcie_rd_conf,
+	.write = rockchip_pcie_wr_conf,
+};
+
+/**
+ * rockchip_pcie_init_port - Initialize hardware
+ * @rockchip: PCIe port information
+ */
+static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	int err;
+	u32 status;
+	unsigned long timeout;
+
+	gpiod_set_value(rockchip->ep_gpio, 0);
+
+	err = phy_init(rockchip->phy);
+	if (err < 0) {
+		dev_err(dev, "fail to init phy, err %d\n", err);
+		return err;
+	}
+
+	err = reset_control_assert(rockchip->core_rst);
+	if (err) {
+		dev_err(dev, "assert core_rst err %d\n", err);
+		return err;
+	}
+
+	err = reset_control_assert(rockchip->mgmt_rst);
+	if (err) {
+		dev_err(dev, "assert mgmt_rst err %d\n", err);
+		return err;
+	}
+
+	err = reset_control_assert(rockchip->mgmt_sticky_rst);
+	if (err) {
+		dev_err(dev, "assert mgmt_sticky_rst err %d\n", err);
+		return err;
+	}
+
+	err = reset_control_assert(rockchip->pipe_rst);
+	if (err) {
+		dev_err(dev, "assert pipe_rst err %d\n", err);
+		return err;
+	}
+
+	rockchip_pcie_write(rockchip,
+			    PCIE_CLIENT_CONF_ENABLE |
+			    PCIE_CLIENT_LINK_TRAIN_ENABLE |
+			    PCIE_CLIENT_ARI_ENABLE |
+			    PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes) |
+			    PCIE_CLIENT_MODE_RC |
+			    PCIE_CLIENT_GEN_SEL_2,
+				PCIE_CLIENT_CONFIG);
+
+	err = phy_power_on(rockchip->phy);
+	if (err) {
+		dev_err(dev, "fail to power on phy, err %d\n", err);
+		return err;
+	}
+
+	/*
+	 * Please don't reorder the deassert sequence of the following
+	 * four reset pins.
+	 */
+	err = reset_control_deassert(rockchip->mgmt_sticky_rst);
+	if (err) {
+		dev_err(dev, "deassert mgmt_sticky_rst err %d\n", err);
+		return err;
+	}
+
+	err = reset_control_deassert(rockchip->core_rst);
+	if (err) {
+		dev_err(dev, "deassert core_rst err %d\n", err);
+		return err;
+	}
+
+	err = reset_control_deassert(rockchip->mgmt_rst);
+	if (err) {
+		dev_err(dev, "deassert mgmt_rst err %d\n", err);
+		return err;
+	}
+
+	err = reset_control_deassert(rockchip->pipe_rst);
+	if (err) {
+		dev_err(dev, "deassert pipe_rst err %d\n", err);
+		return err;
+	}
+
+	/*
+	 * We need to read/write PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 before
+	 * enabling ASPM.  Otherwise L1PwrOnSc and L1PwrOnVal isn't
+	 * reliable and enabling ASPM doesn't work.  This is a controller
+	 * bug we need to work around.
+	 */
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2);
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2);
+
+	/* Fix the transmitted FTS count desired to exit from L0s. */
+	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
+	status = (status & PCIE_CORE_CTRL_PLC1_FTS_MASK) |
+		 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
+	rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
+
+	/* Enable Gen1 training */
+	rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
+			    PCIE_CLIENT_CONFIG);
+
+	gpiod_set_value(rockchip->ep_gpio, 1);
+
+	/* 500ms timeout value should be enough for Gen1/2 training */
+	timeout = jiffies + msecs_to_jiffies(500);
+
+	for (;;) {
+		status = rockchip_pcie_read(rockchip,
+					    PCIE_CLIENT_BASIC_STATUS1);
+		if ((status & PCIE_CLIENT_LINK_STATUS_MASK) ==
+		    PCIE_CLIENT_LINK_STATUS_UP) {
+			dev_dbg(dev, "PCIe link training gen1 pass!\n");
+			break;
+		}
+
+		if (time_after(jiffies, timeout)) {
+			dev_err(dev, "PCIe link training gen1 timeout!\n");
+			return -ETIMEDOUT;
+		}
+
+		msleep(20);
+	}
+
+	/*
+	 * Enable retrain for gen2. This should be configured only after
+	 * gen1 finished.
+	 */
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+	status |= PCIE_RC_CONFIG_LCS_RETRAIN_LINK;
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+
+	timeout = jiffies + msecs_to_jiffies(500);
+	for (;;) {
+		status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
+		if ((status & PCIE_CORE_PL_CONF_SPEED_MASK) ==
+		    PCIE_CORE_PL_CONF_SPEED_5G) {
+			dev_dbg(dev, "PCIe link training gen2 pass!\n");
+			break;
+		}
+
+		if (time_after(jiffies, timeout)) {
+			dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
+			break;
+		}
+
+		msleep(20);
+	}
+
+	/* Check the final link width from negotiated lane counter from MGMT */
+	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
+	status =  0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
+			  PCIE_CORE_PL_CONF_LANE_MASK);
+	dev_dbg(dev, "current link width is x%d\n", status);
+
+	rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
+			    PCIE_RC_CONFIG_VENDOR);
+	rockchip_pcie_write(rockchip,
+			    PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
+			    PCIE_RC_CONFIG_RID_CCR);
+	rockchip_pcie_write(rockchip, 0x0, PCIE_RC_BAR_CONF);
+
+	rockchip_pcie_write(rockchip,
+			    (RC_REGION_0_ADDR_TRANS_L + RC_REGION_0_PASS_BITS),
+			    PCIE_CORE_OB_REGION_ADDR0);
+	rockchip_pcie_write(rockchip, RC_REGION_0_ADDR_TRANS_H,
+			    PCIE_CORE_OB_REGION_ADDR1);
+	rockchip_pcie_write(rockchip, 0x0080000a, PCIE_CORE_OB_REGION_DESC0);
+	rockchip_pcie_write(rockchip, 0x0, PCIE_CORE_OB_REGION_DESC1);
+
+	return 0;
+}
+
+static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
+{
+	struct rockchip_pcie *rockchip = arg;
+	struct device *dev = rockchip->dev;
+	u32 reg;
+	u32 sub_reg;
+
+	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
+	if (reg & PCIE_CLIENT_INT_LOCAL) {
+		dev_dbg(dev, "local interrupt received\n");
+		sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
+		if (sub_reg & PCIE_CORE_INT_PRFPE)
+			dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
+
+		if (sub_reg & PCIE_CORE_INT_CRFPE)
+			dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
+
+		if (sub_reg & PCIE_CORE_INT_RRPE)
+			dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
+
+		if (sub_reg & PCIE_CORE_INT_PRFO)
+			dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
+
+		if (sub_reg & PCIE_CORE_INT_CRFO)
+			dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
+
+		if (sub_reg & PCIE_CORE_INT_RT)
+			dev_dbg(dev, "replay timer timed out\n");
+
+		if (sub_reg & PCIE_CORE_INT_RTR)
+			dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
+
+		if (sub_reg & PCIE_CORE_INT_PE)
+			dev_dbg(dev, "phy error detected on receive side\n");
+
+		if (sub_reg & PCIE_CORE_INT_MTR)
+			dev_dbg(dev, "malformed TLP received from the link\n");
+
+		if (sub_reg & PCIE_CORE_INT_UCR)
+			dev_dbg(dev, "malformed TLP received from the link\n");
+
+		if (sub_reg & PCIE_CORE_INT_FCE)
+			dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
+
+		if (sub_reg & PCIE_CORE_INT_CT)
+			dev_dbg(dev, "a request timed out waiting for completion\n");
+
+		if (sub_reg & PCIE_CORE_INT_UTC)
+			dev_dbg(dev, "unmapped TC error\n");
+
+		if (sub_reg & PCIE_CORE_INT_MMVC)
+			dev_dbg(dev, "MSI mask register changes\n");
+
+		rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
+	} else if (reg & PCIE_CLIENT_INT_PHY) {
+		dev_dbg(dev, "phy link changes\n");
+		rockchip_pcie_update_txcredit_mui(rockchip);
+		rockchip_pcie_clr_bw_int(rockchip);
+	}
+
+	rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
+			    PCIE_CLIENT_INT_STATUS);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
+{
+	struct rockchip_pcie *rockchip = arg;
+	struct device *dev = rockchip->dev;
+	u32 reg;
+
+	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
+	if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
+		dev_dbg(dev, "legacy done interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_MSG)
+		dev_dbg(dev, "message done interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_HOT_RST)
+		dev_dbg(dev, "hot reset interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_DPA)
+		dev_dbg(dev, "dpa interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_FATAL_ERR)
+		dev_dbg(dev, "fatal error interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
+		dev_dbg(dev, "no fatal error interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_CORR_ERR)
+		dev_dbg(dev, "correctable error interrupt received\n");
+
+	if (reg & PCIE_CLIENT_INT_PHY)
+		dev_dbg(dev, "phy interrupt received\n");
+
+	rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
+			      PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
+			      PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
+			      PCIE_CLIENT_INT_NFATAL_ERR |
+			      PCIE_CLIENT_INT_CORR_ERR |
+			      PCIE_CLIENT_INT_PHY),
+		   PCIE_CLIENT_INT_STATUS);
+
+	return IRQ_HANDLED;
+}
+
+static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
+{
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
+	struct device *dev = rockchip->dev;
+	u32 reg;
+	u32 hwirq;
+	u32 virq;
+
+	chained_irq_enter(chip, desc);
+
+	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
+	reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
+
+	while (reg) {
+		hwirq = ffs(reg) - 1;
+		reg &= ~BIT(hwirq);
+
+		virq = irq_find_mapping(rockchip->irq_domain, hwirq);
+		if (virq)
+			generic_handle_irq(virq);
+		else
+			dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
+	}
+
+	chained_irq_exit(chip, desc);
+}
+
+
+/**
+ * rockchip_pcie_parse_dt - Parse Device Tree
+ * @rockchip: PCIe port information
+ *
+ * Return: '0' on success and error value on failure
+ */
+static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct device_node *node = dev->of_node;
+	struct resource *regs;
+	int irq;
+	int err;
+
+	regs = platform_get_resource_byname(pdev,
+					    IORESOURCE_MEM,
+					    "axi-base");
+	rockchip->reg_base = devm_ioremap_resource(dev, regs);
+	if (IS_ERR(rockchip->reg_base))
+		return PTR_ERR(rockchip->reg_base);
+
+	regs = platform_get_resource_byname(pdev,
+					    IORESOURCE_MEM,
+					    "apb-base");
+	rockchip->apb_base = devm_ioremap_resource(dev, regs);
+	if (IS_ERR(rockchip->apb_base))
+		return PTR_ERR(rockchip->apb_base);
+
+	rockchip->phy = devm_phy_get(dev, "pcie-phy");
+	if (IS_ERR(rockchip->phy)) {
+		if (PTR_ERR(rockchip->phy) != -EPROBE_DEFER)
+			dev_err(dev, "missing phy\n");
+		return PTR_ERR(rockchip->phy);
+	}
+
+	rockchip->lanes = 1;
+	err = of_property_read_u32(node, "num-lanes", &rockchip->lanes);
+	if (!err && (rockchip->lanes == 0 ||
+		     rockchip->lanes == 3 ||
+		     rockchip->lanes > 4)) {
+		dev_warn(dev, "invalid num-lanes, default to use one lane\n");
+		rockchip->lanes = 1;
+	}
+
+	rockchip->core_rst = devm_reset_control_get(dev, "core");
+	if (IS_ERR(rockchip->core_rst)) {
+		if (PTR_ERR(rockchip->core_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing core reset property in node\n");
+		return PTR_ERR(rockchip->core_rst);
+	}
+
+	rockchip->mgmt_rst = devm_reset_control_get(dev, "mgmt");
+	if (IS_ERR(rockchip->mgmt_rst)) {
+		if (PTR_ERR(rockchip->mgmt_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing mgmt reset property in node\n");
+		return PTR_ERR(rockchip->mgmt_rst);
+	}
+
+	rockchip->mgmt_sticky_rst = devm_reset_control_get(dev, "mgmt-sticky");
+	if (IS_ERR(rockchip->mgmt_sticky_rst)) {
+		if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing mgmt-sticky reset property in node\n");
+		return PTR_ERR(rockchip->mgmt_sticky_rst);
+	}
+
+	rockchip->pipe_rst = devm_reset_control_get(dev, "pipe");
+	if (IS_ERR(rockchip->pipe_rst)) {
+		if (PTR_ERR(rockchip->pipe_rst) != -EPROBE_DEFER)
+			dev_err(dev, "missing pipe reset property in node\n");
+		return PTR_ERR(rockchip->pipe_rst);
+	}
+
+	rockchip->ep_gpio = devm_gpiod_get(dev, "ep", GPIOD_OUT_HIGH);
+	if (IS_ERR(rockchip->ep_gpio)) {
+		dev_err(dev, "missing ep-gpios property in node\n");
+		return PTR_ERR(rockchip->ep_gpio);
+	}
+
+	rockchip->aclk_pcie = devm_clk_get(dev, "aclk");
+	if (IS_ERR(rockchip->aclk_pcie)) {
+		dev_err(dev, "aclk clock not found\n");
+		return PTR_ERR(rockchip->aclk_pcie);
+	}
+
+	rockchip->aclk_perf_pcie = devm_clk_get(dev, "aclk-perf");
+	if (IS_ERR(rockchip->aclk_perf_pcie)) {
+		dev_err(dev, "aclk_perf clock not found\n");
+		return PTR_ERR(rockchip->aclk_perf_pcie);
+	}
+
+	rockchip->hclk_pcie = devm_clk_get(dev, "hclk");
+	if (IS_ERR(rockchip->hclk_pcie)) {
+		dev_err(dev, "hclk clock not found\n");
+		return PTR_ERR(rockchip->hclk_pcie);
+	}
+
+	rockchip->clk_pcie_pm = devm_clk_get(dev, "pm");
+	if (IS_ERR(rockchip->clk_pcie_pm)) {
+		dev_err(dev, "pm clock not found\n");
+		return PTR_ERR(rockchip->clk_pcie_pm);
+	}
+
+	irq = platform_get_irq_byname(pdev, "sys");
+	if (irq < 0) {
+		dev_err(dev, "missing sys IRQ resource\n");
+		return -EINVAL;
+	}
+
+	err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
+			       IRQF_SHARED, "pcie-sys", rockchip);
+	if (err) {
+		dev_err(dev, "failed to request PCIe subsystem IRQ\n");
+		return err;
+	}
+
+	irq = platform_get_irq_byname(pdev, "legacy");
+	if (irq < 0) {
+		dev_err(dev, "missing legacy IRQ resource\n");
+		return -EINVAL;
+	}
+
+	irq_set_chained_handler_and_data(irq,
+					 rockchip_pcie_legacy_int_handler,
+					 rockchip);
+
+	irq = platform_get_irq_byname(pdev, "client");
+	if (irq < 0) {
+		dev_err(dev, "missing client IRQ resource\n");
+		return -EINVAL;
+	}
+
+	err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
+			       IRQF_SHARED, "pcie-client", rockchip);
+	if (err) {
+		dev_err(dev, "failed to request PCIe client IRQ\n");
+		return err;
+	}
+
+	rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
+	if (IS_ERR(rockchip->vpcie3v3)) {
+		if (PTR_ERR(rockchip->vpcie3v3) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(dev, "no vpcie3v3 regulator found\n");
+	}
+
+	rockchip->vpcie1v8 = devm_regulator_get_optional(dev, "vpcie1v8");
+	if (IS_ERR(rockchip->vpcie1v8)) {
+		if (PTR_ERR(rockchip->vpcie1v8) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(dev, "no vpcie1v8 regulator found\n");
+	}
+
+	rockchip->vpcie0v9 = devm_regulator_get_optional(dev, "vpcie0v9");
+	if (IS_ERR(rockchip->vpcie0v9)) {
+		if (PTR_ERR(rockchip->vpcie0v9) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(dev, "no vpcie0v9 regulator found\n");
+	}
+
+	return 0;
+}
+
+static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	int err;
+
+	if (!IS_ERR(rockchip->vpcie3v3)) {
+		err = regulator_enable(rockchip->vpcie3v3);
+		if (err) {
+			dev_err(dev, "fail to enable vpcie3v3 regulator\n");
+			goto err_out;
+		}
+	}
+
+	if (!IS_ERR(rockchip->vpcie1v8)) {
+		err = regulator_enable(rockchip->vpcie1v8);
+		if (err) {
+			dev_err(dev, "fail to enable vpcie1v8 regulator\n");
+			goto err_disable_3v3;
+		}
+	}
+
+	if (!IS_ERR(rockchip->vpcie0v9)) {
+		err = regulator_enable(rockchip->vpcie0v9);
+		if (err) {
+			dev_err(dev, "fail to enable vpcie0v9 regulator\n");
+			goto err_disable_1v8;
+		}
+	}
+
+	return 0;
+
+err_disable_1v8:
+	if (!IS_ERR(rockchip->vpcie1v8))
+		regulator_disable(rockchip->vpcie1v8);
+err_disable_3v3:
+	if (!IS_ERR(rockchip->vpcie3v3))
+		regulator_disable(rockchip->vpcie3v3);
+err_out:
+	return err;
+}
+
+static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
+{
+	rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
+			    (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
+	rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
+			    PCIE_CORE_INT_MASK);
+
+	rockchip_pcie_enable_bw_int(rockchip);
+}
+
+static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
+				  irq_hw_number_t hwirq)
+{
+	irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
+	irq_set_chip_data(irq, domain->host_data);
+
+	return 0;
+}
+
+static const struct irq_domain_ops intx_domain_ops = {
+	.map = rockchip_pcie_intx_map,
+};
+
+static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	struct device_node *intc = of_get_next_child(dev->of_node, NULL);
+
+	if (!intc) {
+		dev_err(dev, "missing child interrupt-controller node\n");
+		return -EINVAL;
+	}
+
+	rockchip->irq_domain = irq_domain_add_linear(intc, 4,
+						    &intx_domain_ops, rockchip);
+	if (!rockchip->irq_domain) {
+		dev_err(dev, "failed to get a INTx IRQ domain\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
+				     int region_no, int type, u8 num_pass_bits,
+				     u32 lower_addr, u32 upper_addr)
+{
+	u32 ob_addr_0;
+	u32 ob_addr_1;
+	u32 ob_desc_0;
+	u32 aw_offset;
+
+	if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
+		return -EINVAL;
+	if (num_pass_bits + 1 < 8)
+		return -EINVAL;
+	if (num_pass_bits > 63)
+		return -EINVAL;
+	if (region_no == 0) {
+		if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
+		return -EINVAL;
+	}
+	if (region_no != 0) {
+		if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
+			return -EINVAL;
+	}
+
+	aw_offset = (region_no << OB_REG_SIZE_SHIFT);
+
+	ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
+	ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
+	ob_addr_1 = upper_addr;
+	ob_desc_0 = (1 << 23 | type);
+
+	rockchip_pcie_write(rockchip, ob_addr_0,
+			    PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
+	rockchip_pcie_write(rockchip, ob_addr_1,
+			    PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
+	rockchip_pcie_write(rockchip, ob_desc_0,
+			    PCIE_CORE_OB_REGION_DESC0 + aw_offset);
+	rockchip_pcie_write(rockchip, 0,
+			    PCIE_CORE_OB_REGION_DESC1 + aw_offset);
+
+	return 0;
+}
+
+static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
+				     int region_no, u8 num_pass_bits,
+				     u32 lower_addr, u32 upper_addr)
+{
+	u32 ib_addr_0;
+	u32 ib_addr_1;
+	u32 aw_offset;
+
+	if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
+		return -EINVAL;
+	if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
+		return -EINVAL;
+	if (num_pass_bits > 63)
+		return -EINVAL;
+
+	aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
+
+	ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
+	ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
+	ib_addr_1 = upper_addr;
+
+	rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
+	rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
+
+	return 0;
+}
+
+static int rockchip_pcie_probe(struct platform_device *pdev)
+{
+	struct rockchip_pcie *rockchip;
+	struct device *dev = &pdev->dev;
+	struct pci_bus *bus, *child;
+	struct resource_entry *win;
+	resource_size_t io_base;
+	struct resource	*mem;
+	struct resource	*io;
+	phys_addr_t io_bus_addr = 0;
+	u32 io_size;
+	phys_addr_t mem_bus_addr = 0;
+	u32 mem_size = 0;
+	int reg_no;
+	int err;
+	int offset;
+
+	LIST_HEAD(res);
+
+	if (!dev->of_node)
+		return -ENODEV;
+
+	rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL);
+	if (!rockchip)
+		return -ENOMEM;
+
+	rockchip->dev = dev;
+
+	err = rockchip_pcie_parse_dt(rockchip);
+	if (err)
+		return err;
+
+	err = clk_prepare_enable(rockchip->aclk_pcie);
+	if (err) {
+		dev_err(dev, "unable to enable aclk_pcie clock\n");
+		goto err_aclk_pcie;
+	}
+
+	err = clk_prepare_enable(rockchip->aclk_perf_pcie);
+	if (err) {
+		dev_err(dev, "unable to enable aclk_perf_pcie clock\n");
+		goto err_aclk_perf_pcie;
+	}
+
+	err = clk_prepare_enable(rockchip->hclk_pcie);
+	if (err) {
+		dev_err(dev, "unable to enable hclk_pcie clock\n");
+		goto err_hclk_pcie;
+	}
+
+	err = clk_prepare_enable(rockchip->clk_pcie_pm);
+	if (err) {
+		dev_err(dev, "unable to enable hclk_pcie clock\n");
+		goto err_pcie_pm;
+	}
+
+	err = rockchip_pcie_set_vpcie(rockchip);
+	if (err) {
+		dev_err(dev, "failed to set vpcie regulator\n");
+		goto err_set_vpcie;
+	}
+
+	err = rockchip_pcie_init_port(rockchip);
+	if (err)
+		goto err_vpcie;
+
+	platform_set_drvdata(pdev, rockchip);
+
+	rockchip_pcie_enable_interrupts(rockchip);
+
+	err = rockchip_pcie_init_irq_domain(rockchip);
+	if (err < 0)
+		goto err_vpcie;
+
+	err = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff,
+					       &res, &io_base);
+	if (err)
+		goto err_vpcie;
+
+	err = devm_request_pci_bus_resources(dev, &res);
+	if (err)
+		goto err_vpcie;
+
+	/* Get the I/O and memory ranges from DT */
+	io_size = 0;
+	resource_list_for_each_entry(win, &res) {
+		switch (resource_type(win->res)) {
+		case IORESOURCE_IO:
+			io = win->res;
+			io->name = "I/O";
+			io_size = resource_size(io);
+			io_bus_addr = io->start - win->offset;
+			err = pci_remap_iospace(io, io_base);
+			if (err) {
+				dev_warn(dev, "error %d: failed to map resource %pR\n",
+					 err, io);
+				continue;
+			}
+			break;
+		case IORESOURCE_MEM:
+			mem = win->res;
+			mem->name = "MEM";
+			mem_size = resource_size(mem);
+			mem_bus_addr = mem->start - win->offset;
+			break;
+		case IORESOURCE_BUS:
+			rockchip->root_bus_nr = win->res->start;
+			break;
+		default:
+			continue;
+		}
+	}
+
+	if (mem_size) {
+		for (reg_no = 0; reg_no < (mem_size >> 20); reg_no++) {
+			err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
+							AXI_WRAPPER_MEM_WRITE,
+							20 - 1,
+							mem_bus_addr +
+							(reg_no << 20),
+							0);
+			if (err) {
+				dev_err(dev, "program RC mem outbound ATU failed\n");
+				goto err_vpcie;
+			}
+		}
+	}
+
+	err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
+	if (err) {
+		dev_err(dev, "program RC mem inbound ATU failed\n");
+		goto err_vpcie;
+	}
+
+	offset = mem_size >> 20;
+
+	if (io_size) {
+		for (reg_no = 0; reg_no < (io_size >> 20); reg_no++) {
+			err = rockchip_pcie_prog_ob_atu(rockchip,
+							reg_no + 1 + offset,
+							AXI_WRAPPER_IO_WRITE,
+							20 - 1,
+							io_bus_addr +
+							(reg_no << 20),
+							0);
+			if (err) {
+				dev_err(dev, "program RC io outbound ATU failed\n");
+				goto err_vpcie;
+			}
+		}
+	}
+
+	bus = pci_scan_root_bus(&pdev->dev, 0, &rockchip_pcie_ops, rockchip, &res);
+	if (!bus) {
+		err = -ENOMEM;
+		goto err_vpcie;
+	}
+
+	pci_bus_size_bridges(bus);
+	pci_bus_assign_resources(bus);
+	list_for_each_entry(child, &bus->children, node)
+		pcie_bus_configure_settings(child);
+
+	pci_bus_add_devices(bus);
+
+	dev_warn(dev, "only 32-bit config accesses supported; smaller writes may corrupt adjacent RW1C fields\n");
+
+	return err;
+
+err_vpcie:
+	if (!IS_ERR(rockchip->vpcie3v3))
+		regulator_disable(rockchip->vpcie3v3);
+	if (!IS_ERR(rockchip->vpcie1v8))
+		regulator_disable(rockchip->vpcie1v8);
+	if (!IS_ERR(rockchip->vpcie0v9))
+		regulator_disable(rockchip->vpcie0v9);
+err_set_vpcie:
+	clk_disable_unprepare(rockchip->clk_pcie_pm);
+err_pcie_pm:
+	clk_disable_unprepare(rockchip->hclk_pcie);
+err_hclk_pcie:
+	clk_disable_unprepare(rockchip->aclk_perf_pcie);
+err_aclk_perf_pcie:
+	clk_disable_unprepare(rockchip->aclk_pcie);
+err_aclk_pcie:
+	return err;
+}
+
+static const struct of_device_id rockchip_pcie_of_match[] = {
+	{ .compatible = "rockchip,rk3399-pcie", },
+	{}
+};
+
+static struct platform_driver rockchip_pcie_driver = {
+	.driver = {
+		.name = "rockchip-pcie",
+		.of_match_table = rockchip_pcie_of_match,
+	},
+	.probe = rockchip_pcie_probe,
+
+};
+builtin_platform_driver(rockchip_pcie_driver);
diff --git a/drivers/pci/host/pcie-spear13xx.c b/drivers/pci/host/pcie-spear13xx.c
index a4060b85ab23..09aed85f275a 100644
--- a/drivers/pci/host/pcie-spear13xx.c
+++ b/drivers/pci/host/pcie-spear13xx.c
@@ -15,7 +15,7 @@
 #include <linux/clk.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 #include <linux/pci.h>
 #include <linux/phy/phy.h>
@@ -355,7 +355,6 @@ static const struct of_device_id spear13xx_pcie_of_match[] = {
 	{ .compatible = "st,spear1340-pcie", },
 	{},
 };
-MODULE_DEVICE_TABLE(of, spear13xx_pcie_of_match);
 
 static struct platform_driver spear13xx_pcie_driver = {
 	.probe		= spear13xx_pcie_probe,
@@ -365,14 +364,8 @@ static struct platform_driver spear13xx_pcie_driver = {
 	},
 };
 
-/* SPEAr13xx PCIe driver does not allow module unload */
-
 static int __init spear13xx_pcie_init(void)
 {
 	return platform_driver_register(&spear13xx_pcie_driver);
 }
-module_init(spear13xx_pcie_init);
-
-MODULE_DESCRIPTION("ST Microelectronics SPEAr13xx PCIe host controller driver");
-MODULE_AUTHOR("Pratyush Anand <pratyush.anand@gmail.com>");
-MODULE_LICENSE("GPL v2");
+device_initcall(spear13xx_pcie_init);
diff --git a/drivers/pci/host/pcie-xilinx-nwl.c b/drivers/pci/host/pcie-xilinx-nwl.c
index 0b597d9190b4..67eae4179290 100644
--- a/drivers/pci/host/pcie-xilinx-nwl.c
+++ b/drivers/pci/host/pcie-xilinx-nwl.c
@@ -15,7 +15,7 @@
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/msi.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
@@ -85,10 +85,15 @@
 #define MSGF_MISC_SR_MASTER_ERR		BIT(5)
 #define MSGF_MISC_SR_I_ADDR_ERR		BIT(6)
 #define MSGF_MISC_SR_E_ADDR_ERR		BIT(7)
-#define MSGF_MISC_SR_UR_DETECT          BIT(20)
-
-#define MSGF_MISC_SR_PCIE_CORE		GENMASK(18, 16)
-#define MSGF_MISC_SR_PCIE_CORE_ERR	GENMASK(31, 22)
+#define MSGF_MISC_SR_FATAL_AER		BIT(16)
+#define MSGF_MISC_SR_NON_FATAL_AER	BIT(17)
+#define MSGF_MISC_SR_CORR_AER		BIT(18)
+#define MSGF_MISC_SR_UR_DETECT		BIT(20)
+#define MSGF_MISC_SR_NON_FATAL_DEV	BIT(22)
+#define MSGF_MISC_SR_FATAL_DEV		BIT(23)
+#define MSGF_MISC_SR_LINK_DOWN		BIT(24)
+#define MSGF_MSIC_SR_LINK_AUTO_BWIDTH	BIT(25)
+#define MSGF_MSIC_SR_LINK_BWIDTH	BIT(26)
 
 #define MSGF_MISC_SR_MASKALL		(MSGF_MISC_SR_RXMSG_AVAIL | \
 					MSGF_MISC_SR_RXMSG_OVER | \
@@ -96,9 +101,15 @@
 					MSGF_MISC_SR_MASTER_ERR | \
 					MSGF_MISC_SR_I_ADDR_ERR | \
 					MSGF_MISC_SR_E_ADDR_ERR | \
+					MSGF_MISC_SR_FATAL_AER | \
+					MSGF_MISC_SR_NON_FATAL_AER | \
+					MSGF_MISC_SR_CORR_AER | \
 					MSGF_MISC_SR_UR_DETECT | \
-					MSGF_MISC_SR_PCIE_CORE | \
-					MSGF_MISC_SR_PCIE_CORE_ERR)
+					MSGF_MISC_SR_NON_FATAL_DEV | \
+					MSGF_MISC_SR_FATAL_DEV | \
+					MSGF_MISC_SR_LINK_DOWN | \
+					MSGF_MSIC_SR_LINK_AUTO_BWIDTH | \
+					MSGF_MSIC_SR_LINK_BWIDTH)
 
 /* Legacy interrupt status mask bits */
 #define MSGF_LEG_SR_INTA		BIT(0)
@@ -109,8 +120,8 @@
 					MSGF_LEG_SR_INTC | MSGF_LEG_SR_INTD)
 
 /* MSI interrupt status mask bits */
-#define MSGF_MSI_SR_LO_MASK		BIT(0)
-#define MSGF_MSI_SR_HI_MASK		BIT(0)
+#define MSGF_MSI_SR_LO_MASK		GENMASK(31, 0)
+#define MSGF_MSI_SR_HI_MASK		GENMASK(31, 0)
 
 #define MSII_PRESENT			BIT(0)
 #define MSII_ENABLE			BIT(0)
@@ -291,8 +302,29 @@ static irqreturn_t nwl_pcie_misc_handler(int irq, void *data)
 		dev_err(pcie->dev,
 			"In Misc Egress address translation error\n");
 
-	if (misc_stat & MSGF_MISC_SR_PCIE_CORE_ERR)
-		dev_err(pcie->dev, "PCIe Core error\n");
+	if (misc_stat & MSGF_MISC_SR_FATAL_AER)
+		dev_err(pcie->dev, "Fatal Error in AER Capability\n");
+
+	if (misc_stat & MSGF_MISC_SR_NON_FATAL_AER)
+		dev_err(pcie->dev, "Non-Fatal Error in AER Capability\n");
+
+	if (misc_stat & MSGF_MISC_SR_CORR_AER)
+		dev_err(pcie->dev, "Correctable Error in AER Capability\n");
+
+	if (misc_stat & MSGF_MISC_SR_UR_DETECT)
+		dev_err(pcie->dev, "Unsupported request Detected\n");
+
+	if (misc_stat & MSGF_MISC_SR_NON_FATAL_DEV)
+		dev_err(pcie->dev, "Non-Fatal Error Detected\n");
+
+	if (misc_stat & MSGF_MISC_SR_FATAL_DEV)
+		dev_err(pcie->dev, "Fatal Error Detected\n");
+
+	if (misc_stat & MSGF_MSIC_SR_LINK_AUTO_BWIDTH)
+		dev_info(pcie->dev, "Link Autonomous Bandwidth Management Status bit set\n");
+
+	if (misc_stat & MSGF_MSIC_SR_LINK_BWIDTH)
+		dev_info(pcie->dev, "Link Bandwidth Management Status bit set\n");
 
 	/* Clear misc interrupt status */
 	nwl_bridge_writel(pcie, misc_stat, MSGF_MISC_STATUS);
@@ -459,40 +491,6 @@ static const struct irq_domain_ops dev_msi_domain_ops = {
 	.free   = nwl_irq_domain_free,
 };
 
-static void nwl_msi_free_irq_domain(struct nwl_pcie *pcie)
-{
-	struct nwl_msi *msi = &pcie->msi;
-
-	if (msi->irq_msi0)
-		irq_set_chained_handler_and_data(msi->irq_msi0, NULL, NULL);
-	if (msi->irq_msi1)
-		irq_set_chained_handler_and_data(msi->irq_msi1, NULL, NULL);
-
-	if (msi->msi_domain)
-		irq_domain_remove(msi->msi_domain);
-	if (msi->dev_domain)
-		irq_domain_remove(msi->dev_domain);
-
-	kfree(msi->bitmap);
-	msi->bitmap = NULL;
-}
-
-static void nwl_pcie_free_irq_domain(struct nwl_pcie *pcie)
-{
-	int i;
-	u32 irq;
-
-	for (i = 0; i < INTX_NUM; i++) {
-		irq = irq_find_mapping(pcie->legacy_irq_domain, i + 1);
-		if (irq > 0)
-			irq_dispose_mapping(irq);
-	}
-	if (pcie->legacy_irq_domain)
-		irq_domain_remove(pcie->legacy_irq_domain);
-
-	nwl_msi_free_irq_domain(pcie);
-}
-
 static int nwl_pcie_init_msi_irq_domain(struct nwl_pcie *pcie)
 {
 #ifdef CONFIG_PCI_MSI
@@ -867,25 +865,12 @@ error:
 	return err;
 }
 
-static int nwl_pcie_remove(struct platform_device *pdev)
-{
-	struct nwl_pcie *pcie = platform_get_drvdata(pdev);
-
-	nwl_pcie_free_irq_domain(pcie);
-	platform_set_drvdata(pdev, NULL);
-	return 0;
-}
-
 static struct platform_driver nwl_pcie_driver = {
 	.driver = {
 		.name = "nwl-pcie",
+		.suppress_bind_attrs = true,
 		.of_match_table = nwl_pcie_of_match,
 	},
 	.probe = nwl_pcie_probe,
-	.remove = nwl_pcie_remove,
 };
-module_platform_driver(nwl_pcie_driver);
-
-MODULE_AUTHOR("Xilinx, Inc");
-MODULE_DESCRIPTION("NWL PCIe driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(nwl_pcie_driver);
diff --git a/drivers/pci/host/pcie-xilinx.c b/drivers/pci/host/pcie-xilinx.c
index a30e01639557..be568039d9d0 100644
--- a/drivers/pci/host/pcie-xilinx.c
+++ b/drivers/pci/host/pcie-xilinx.c
@@ -18,7 +18,7 @@
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/msi.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
@@ -101,7 +101,8 @@
  * @msi_pages: MSI pages
  * @root_busno: Root Bus number
  * @dev: Device pointer
- * @irq_domain: IRQ domain pointer
+ * @msi_domain: MSI IRQ domain pointer
+ * @leg_domain: Legacy IRQ domain pointer
  * @resources: Bus Resources
  */
 struct xilinx_pcie_port {
@@ -110,7 +111,8 @@ struct xilinx_pcie_port {
 	unsigned long msi_pages;
 	u8 root_busno;
 	struct device *dev;
-	struct irq_domain *irq_domain;
+	struct irq_domain *msi_domain;
+	struct irq_domain *leg_domain;
 	struct list_head resources;
 };
 
@@ -168,13 +170,6 @@ static bool xilinx_pcie_valid_device(struct pci_bus *bus, unsigned int devfn)
 	if (bus->number == port->root_busno && devfn > 0)
 		return false;
 
-	/*
-	 * Do not read more than one device on the bus directly attached
-	 * to RC.
-	 */
-	if (bus->primary == port->root_busno && devfn > 0)
-		return false;
-
 	return true;
 }
 
@@ -219,13 +214,15 @@ static void xilinx_pcie_destroy_msi(unsigned int irq)
 {
 	struct msi_desc *msi;
 	struct xilinx_pcie_port *port;
+	struct irq_data *d = irq_get_irq_data(irq);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
 
-	if (!test_bit(irq, msi_irq_in_use)) {
+	if (!test_bit(hwirq, msi_irq_in_use)) {
 		msi = irq_get_msi_desc(irq);
 		port = msi_desc_to_pci_sysdata(msi);
 		dev_err(port->dev, "Trying to free unused MSI#%d\n", irq);
 	} else {
-		clear_bit(irq, msi_irq_in_use);
+		clear_bit(hwirq, msi_irq_in_use);
 	}
 }
 
@@ -257,6 +254,7 @@ static void xilinx_msi_teardown_irq(struct msi_controller *chip,
 				    unsigned int irq)
 {
 	xilinx_pcie_destroy_msi(irq);
+	irq_dispose_mapping(irq);
 }
 
 /**
@@ -281,7 +279,7 @@ static int xilinx_pcie_msi_setup_irq(struct msi_controller *chip,
 	if (hwirq < 0)
 		return hwirq;
 
-	irq = irq_create_mapping(port->irq_domain, hwirq);
+	irq = irq_create_mapping(port->msi_domain, hwirq);
 	if (!irq)
 		return -EINVAL;
 
@@ -432,7 +430,7 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data)
 		/* Check whether interrupt valid */
 		if (!(val & XILINX_PCIE_RPIFR1_INTR_VALID)) {
 			dev_warn(port->dev, "RP Intr FIFO1 read error\n");
-			return IRQ_HANDLED;
+			goto error;
 		}
 
 		if (!(val & XILINX_PCIE_RPIFR1_MSI_INTR)) {
@@ -443,7 +441,7 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data)
 			/* Handle INTx Interrupt */
 			val = ((val & XILINX_PCIE_RPIFR1_INTR_MASK) >>
 				XILINX_PCIE_RPIFR1_INTR_SHIFT) + 1;
-			generic_handle_irq(irq_find_mapping(port->irq_domain,
+			generic_handle_irq(irq_find_mapping(port->leg_domain,
 							    val));
 		}
 	}
@@ -454,7 +452,7 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data)
 
 		if (!(val & XILINX_PCIE_RPIFR1_INTR_VALID)) {
 			dev_warn(port->dev, "RP Intr FIFO1 read error\n");
-			return IRQ_HANDLED;
+			goto error;
 		}
 
 		if (val & XILINX_PCIE_RPIFR1_MSI_INTR) {
@@ -499,6 +497,7 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data)
 	if (status & XILINX_PCIE_INTR_MST_ERRP)
 		dev_warn(port->dev, "Master error poison\n");
 
+error:
 	/* Clear the Interrupt Decode register */
 	pcie_write(port, status, XILINX_PCIE_REG_IDR);
 
@@ -506,35 +505,6 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data)
 }
 
 /**
- * xilinx_pcie_free_irq_domain - Free IRQ domain
- * @port: PCIe port information
- */
-static void xilinx_pcie_free_irq_domain(struct xilinx_pcie_port *port)
-{
-	int i;
-	u32 irq, num_irqs;
-
-	/* Free IRQ Domain */
-	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-
-		free_pages(port->msi_pages, 0);
-
-		num_irqs = XILINX_NUM_MSI_IRQS;
-	} else {
-		/* INTx */
-		num_irqs = 4;
-	}
-
-	for (i = 0; i < num_irqs; i++) {
-		irq = irq_find_mapping(port->irq_domain, i);
-		if (irq > 0)
-			irq_dispose_mapping(irq);
-	}
-
-	irq_domain_remove(port->irq_domain);
-}
-
-/**
  * xilinx_pcie_init_irq_domain - Initialize IRQ domain
  * @port: PCIe port information
  *
@@ -553,21 +523,21 @@ static int xilinx_pcie_init_irq_domain(struct xilinx_pcie_port *port)
 		return -ENODEV;
 	}
 
-	port->irq_domain = irq_domain_add_linear(pcie_intc_node, 4,
+	port->leg_domain = irq_domain_add_linear(pcie_intc_node, 4,
 						 &intx_domain_ops,
 						 port);
-	if (!port->irq_domain) {
+	if (!port->leg_domain) {
 		dev_err(dev, "Failed to get a INTx IRQ domain\n");
 		return -ENODEV;
 	}
 
 	/* Setup MSI */
 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		port->irq_domain = irq_domain_add_linear(node,
+		port->msi_domain = irq_domain_add_linear(node,
 							 XILINX_NUM_MSI_IRQS,
 							 &msi_domain_ops,
 							 &xilinx_pcie_msi_chip);
-		if (!port->irq_domain) {
+		if (!port->msi_domain) {
 			dev_err(dev, "Failed to get a MSI IRQ domain\n");
 			return -ENODEV;
 		}
@@ -724,21 +694,6 @@ error:
 	return err;
 }
 
-/**
- * xilinx_pcie_remove - Remove function
- * @pdev: Platform device pointer
- *
- * Return: '0' always
- */
-static int xilinx_pcie_remove(struct platform_device *pdev)
-{
-	struct xilinx_pcie_port *port = platform_get_drvdata(pdev);
-
-	xilinx_pcie_free_irq_domain(port);
-
-	return 0;
-}
-
 static struct of_device_id xilinx_pcie_of_match[] = {
 	{ .compatible = "xlnx,axi-pcie-host-1.00.a", },
 	{}
@@ -751,10 +706,5 @@ static struct platform_driver xilinx_pcie_driver = {
 		.suppress_bind_attrs = true,
 	},
 	.probe = xilinx_pcie_probe,
-	.remove = xilinx_pcie_remove,
 };
-module_platform_driver(xilinx_pcie_driver);
-
-MODULE_AUTHOR("Xilinx Inc");
-MODULE_DESCRIPTION("Xilinx AXI PCIe driver");
-MODULE_LICENSE("GPL v2");
+builtin_platform_driver(xilinx_pcie_driver);
diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c
new file mode 100644
index 000000000000..37e29b580be3
--- /dev/null
+++ b/drivers/pci/host/vmd.c
@@ -0,0 +1,768 @@
+/*
+ * Volume Management Device driver
+ * Copyright (c) 2015, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/msi.h>
+#include <linux/pci.h>
+#include <linux/rculist.h>
+#include <linux/rcupdate.h>
+
+#include <asm/irqdomain.h>
+#include <asm/device.h>
+#include <asm/msi.h>
+#include <asm/msidef.h>
+
+#define VMD_CFGBAR	0
+#define VMD_MEMBAR1	2
+#define VMD_MEMBAR2	4
+
+/*
+ * Lock for manipulating VMD IRQ lists.
+ */
+static DEFINE_RAW_SPINLOCK(list_lock);
+
+/**
+ * struct vmd_irq - private data to map driver IRQ to the VMD shared vector
+ * @node:	list item for parent traversal.
+ * @rcu:	RCU callback item for freeing.
+ * @irq:	back pointer to parent.
+ * @enabled:	true if driver enabled IRQ
+ * @virq:	the virtual IRQ value provided to the requesting driver.
+ *
+ * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to
+ * a VMD IRQ using this structure.
+ */
+struct vmd_irq {
+	struct list_head	node;
+	struct rcu_head		rcu;
+	struct vmd_irq_list	*irq;
+	bool			enabled;
+	unsigned int		virq;
+};
+
+/**
+ * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector
+ * @irq_list:	the list of irq's the VMD one demuxes to.
+ * @count:	number of child IRQs assigned to this vector; used to track
+ *		sharing.
+ */
+struct vmd_irq_list {
+	struct list_head	irq_list;
+	unsigned int		count;
+};
+
+struct vmd_dev {
+	struct pci_dev		*dev;
+
+	spinlock_t		cfg_lock;
+	char __iomem		*cfgbar;
+
+	int msix_count;
+	struct vmd_irq_list	*irqs;
+
+	struct pci_sysdata	sysdata;
+	struct resource		resources[3];
+	struct irq_domain	*irq_domain;
+	struct pci_bus		*bus;
+
+#ifdef CONFIG_X86_DEV_DMA_OPS
+	struct dma_map_ops	dma_ops;
+	struct dma_domain	dma_domain;
+#endif
+};
+
+static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus)
+{
+	return container_of(bus->sysdata, struct vmd_dev, sysdata);
+}
+
+static inline unsigned int index_from_irqs(struct vmd_dev *vmd,
+					   struct vmd_irq_list *irqs)
+{
+	return irqs - vmd->irqs;
+}
+
+/*
+ * Drivers managing a device in a VMD domain allocate their own IRQs as before,
+ * but the MSI entry for the hardware it's driving will be programmed with a
+ * destination ID for the VMD MSI-X table.  The VMD muxes interrupts in its
+ * domain into one of its own, and the VMD driver de-muxes these for the
+ * handlers sharing that VMD IRQ.  The vmd irq_domain provides the operations
+ * and irq_chip to set this up.
+ */
+static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
+{
+	struct vmd_irq *vmdirq = data->chip_data;
+	struct vmd_irq_list *irq = vmdirq->irq;
+	struct vmd_dev *vmd = irq_data_get_irq_handler_data(data);
+
+	msg->address_hi = MSI_ADDR_BASE_HI;
+	msg->address_lo = MSI_ADDR_BASE_LO |
+			  MSI_ADDR_DEST_ID(index_from_irqs(vmd, irq));
+	msg->data = 0;
+}
+
+/*
+ * We rely on MSI_FLAG_USE_DEF_CHIP_OPS to set the IRQ mask/unmask ops.
+ */
+static void vmd_irq_enable(struct irq_data *data)
+{
+	struct vmd_irq *vmdirq = data->chip_data;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&list_lock, flags);
+	WARN_ON(vmdirq->enabled);
+	list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);
+	vmdirq->enabled = true;
+	raw_spin_unlock_irqrestore(&list_lock, flags);
+
+	data->chip->irq_unmask(data);
+}
+
+static void vmd_irq_disable(struct irq_data *data)
+{
+	struct vmd_irq *vmdirq = data->chip_data;
+	unsigned long flags;
+
+	data->chip->irq_mask(data);
+
+	raw_spin_lock_irqsave(&list_lock, flags);
+	if (vmdirq->enabled) {
+		list_del_rcu(&vmdirq->node);
+		vmdirq->enabled = false;
+	}
+	raw_spin_unlock_irqrestore(&list_lock, flags);
+}
+
+/*
+ * XXX: Stubbed until we develop acceptable way to not create conflicts with
+ * other devices sharing the same vector.
+ */
+static int vmd_irq_set_affinity(struct irq_data *data,
+				const struct cpumask *dest, bool force)
+{
+	return -EINVAL;
+}
+
+static struct irq_chip vmd_msi_controller = {
+	.name			= "VMD-MSI",
+	.irq_enable		= vmd_irq_enable,
+	.irq_disable		= vmd_irq_disable,
+	.irq_compose_msi_msg	= vmd_compose_msi_msg,
+	.irq_set_affinity	= vmd_irq_set_affinity,
+};
+
+static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,
+				     msi_alloc_info_t *arg)
+{
+	return 0;
+}
+
+/*
+ * XXX: We can be even smarter selecting the best IRQ once we solve the
+ * affinity problem.
+ */
+static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)
+{
+	int i, best = 1;
+	unsigned long flags;
+
+	if (!desc->msi_attrib.is_msix || vmd->msix_count == 1)
+		return &vmd->irqs[0];
+
+	raw_spin_lock_irqsave(&list_lock, flags);
+	for (i = 1; i < vmd->msix_count; i++)
+		if (vmd->irqs[i].count < vmd->irqs[best].count)
+			best = i;
+	vmd->irqs[best].count++;
+	raw_spin_unlock_irqrestore(&list_lock, flags);
+
+	return &vmd->irqs[best];
+}
+
+static int vmd_msi_init(struct irq_domain *domain, struct msi_domain_info *info,
+			unsigned int virq, irq_hw_number_t hwirq,
+			msi_alloc_info_t *arg)
+{
+	struct msi_desc *desc = arg->desc;
+	struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);
+	struct vmd_irq *vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);
+	unsigned int index, vector;
+
+	if (!vmdirq)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&vmdirq->node);
+	vmdirq->irq = vmd_next_irq(vmd, desc);
+	vmdirq->virq = virq;
+	index = index_from_irqs(vmd, vmdirq->irq);
+	vector = pci_irq_vector(vmd->dev, index);
+
+	irq_domain_set_info(domain, virq, vector, info->chip, vmdirq,
+			    handle_untracked_irq, vmd, NULL);
+	return 0;
+}
+
+static void vmd_msi_free(struct irq_domain *domain,
+			struct msi_domain_info *info, unsigned int virq)
+{
+	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
+	unsigned long flags;
+
+	synchronize_rcu();
+
+	/* XXX: Potential optimization to rebalance */
+	raw_spin_lock_irqsave(&list_lock, flags);
+	vmdirq->irq->count--;
+	raw_spin_unlock_irqrestore(&list_lock, flags);
+
+	kfree_rcu(vmdirq, rcu);
+}
+
+static int vmd_msi_prepare(struct irq_domain *domain, struct device *dev,
+			   int nvec, msi_alloc_info_t *arg)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
+
+	if (nvec > vmd->msix_count)
+		return vmd->msix_count;
+
+	memset(arg, 0, sizeof(*arg));
+	return 0;
+}
+
+static void vmd_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
+{
+	arg->desc = desc;
+}
+
+static struct msi_domain_ops vmd_msi_domain_ops = {
+	.get_hwirq	= vmd_get_hwirq,
+	.msi_init	= vmd_msi_init,
+	.msi_free	= vmd_msi_free,
+	.msi_prepare	= vmd_msi_prepare,
+	.set_desc	= vmd_set_desc,
+};
+
+static struct msi_domain_info vmd_msi_domain_info = {
+	.flags		= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+			  MSI_FLAG_PCI_MSIX,
+	.ops		= &vmd_msi_domain_ops,
+	.chip		= &vmd_msi_controller,
+};
+
+#ifdef CONFIG_X86_DEV_DMA_OPS
+/*
+ * VMD replaces the requester ID with its own.  DMA mappings for devices in a
+ * VMD domain need to be mapped for the VMD, not the device requiring
+ * the mapping.
+ */
+static struct device *to_vmd_dev(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
+
+	return &vmd->dev->dev;
+}
+
+static struct dma_map_ops *vmd_dma_ops(struct device *dev)
+{
+	return get_dma_ops(to_vmd_dev(dev));
+}
+
+static void *vmd_alloc(struct device *dev, size_t size, dma_addr_t *addr,
+		       gfp_t flag, unsigned long attrs)
+{
+	return vmd_dma_ops(dev)->alloc(to_vmd_dev(dev), size, addr, flag,
+				       attrs);
+}
+
+static void vmd_free(struct device *dev, size_t size, void *vaddr,
+		     dma_addr_t addr, unsigned long attrs)
+{
+	return vmd_dma_ops(dev)->free(to_vmd_dev(dev), size, vaddr, addr,
+				      attrs);
+}
+
+static int vmd_mmap(struct device *dev, struct vm_area_struct *vma,
+		    void *cpu_addr, dma_addr_t addr, size_t size,
+		    unsigned long attrs)
+{
+	return vmd_dma_ops(dev)->mmap(to_vmd_dev(dev), vma, cpu_addr, addr,
+				      size, attrs);
+}
+
+static int vmd_get_sgtable(struct device *dev, struct sg_table *sgt,
+			   void *cpu_addr, dma_addr_t addr, size_t size,
+			   unsigned long attrs)
+{
+	return vmd_dma_ops(dev)->get_sgtable(to_vmd_dev(dev), sgt, cpu_addr,
+					     addr, size, attrs);
+}
+
+static dma_addr_t vmd_map_page(struct device *dev, struct page *page,
+			       unsigned long offset, size_t size,
+			       enum dma_data_direction dir,
+			       unsigned long attrs)
+{
+	return vmd_dma_ops(dev)->map_page(to_vmd_dev(dev), page, offset, size,
+					  dir, attrs);
+}
+
+static void vmd_unmap_page(struct device *dev, dma_addr_t addr, size_t size,
+			   enum dma_data_direction dir, unsigned long attrs)
+{
+	vmd_dma_ops(dev)->unmap_page(to_vmd_dev(dev), addr, size, dir, attrs);
+}
+
+static int vmd_map_sg(struct device *dev, struct scatterlist *sg, int nents,
+		      enum dma_data_direction dir, unsigned long attrs)
+{
+	return vmd_dma_ops(dev)->map_sg(to_vmd_dev(dev), sg, nents, dir, attrs);
+}
+
+static void vmd_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
+			 enum dma_data_direction dir, unsigned long attrs)
+{
+	vmd_dma_ops(dev)->unmap_sg(to_vmd_dev(dev), sg, nents, dir, attrs);
+}
+
+static void vmd_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
+				    size_t size, enum dma_data_direction dir)
+{
+	vmd_dma_ops(dev)->sync_single_for_cpu(to_vmd_dev(dev), addr, size, dir);
+}
+
+static void vmd_sync_single_for_device(struct device *dev, dma_addr_t addr,
+				       size_t size, enum dma_data_direction dir)
+{
+	vmd_dma_ops(dev)->sync_single_for_device(to_vmd_dev(dev), addr, size,
+						 dir);
+}
+
+static void vmd_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
+				int nents, enum dma_data_direction dir)
+{
+	vmd_dma_ops(dev)->sync_sg_for_cpu(to_vmd_dev(dev), sg, nents, dir);
+}
+
+static void vmd_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
+				   int nents, enum dma_data_direction dir)
+{
+	vmd_dma_ops(dev)->sync_sg_for_device(to_vmd_dev(dev), sg, nents, dir);
+}
+
+static int vmd_mapping_error(struct device *dev, dma_addr_t addr)
+{
+	return vmd_dma_ops(dev)->mapping_error(to_vmd_dev(dev), addr);
+}
+
+static int vmd_dma_supported(struct device *dev, u64 mask)
+{
+	return vmd_dma_ops(dev)->dma_supported(to_vmd_dev(dev), mask);
+}
+
+#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
+static u64 vmd_get_required_mask(struct device *dev)
+{
+	return vmd_dma_ops(dev)->get_required_mask(to_vmd_dev(dev));
+}
+#endif
+
+static void vmd_teardown_dma_ops(struct vmd_dev *vmd)
+{
+	struct dma_domain *domain = &vmd->dma_domain;
+
+	if (get_dma_ops(&vmd->dev->dev))
+		del_dma_domain(domain);
+}
+
+#define ASSIGN_VMD_DMA_OPS(source, dest, fn)	\
+	do {					\
+		if (source->fn)			\
+			dest->fn = vmd_##fn;	\
+	} while (0)
+
+static void vmd_setup_dma_ops(struct vmd_dev *vmd)
+{
+	const struct dma_map_ops *source = get_dma_ops(&vmd->dev->dev);
+	struct dma_map_ops *dest = &vmd->dma_ops;
+	struct dma_domain *domain = &vmd->dma_domain;
+
+	domain->domain_nr = vmd->sysdata.domain;
+	domain->dma_ops = dest;
+
+	if (!source)
+		return;
+	ASSIGN_VMD_DMA_OPS(source, dest, alloc);
+	ASSIGN_VMD_DMA_OPS(source, dest, free);
+	ASSIGN_VMD_DMA_OPS(source, dest, mmap);
+	ASSIGN_VMD_DMA_OPS(source, dest, get_sgtable);
+	ASSIGN_VMD_DMA_OPS(source, dest, map_page);
+	ASSIGN_VMD_DMA_OPS(source, dest, unmap_page);
+	ASSIGN_VMD_DMA_OPS(source, dest, map_sg);
+	ASSIGN_VMD_DMA_OPS(source, dest, unmap_sg);
+	ASSIGN_VMD_DMA_OPS(source, dest, sync_single_for_cpu);
+	ASSIGN_VMD_DMA_OPS(source, dest, sync_single_for_device);
+	ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_cpu);
+	ASSIGN_VMD_DMA_OPS(source, dest, sync_sg_for_device);
+	ASSIGN_VMD_DMA_OPS(source, dest, mapping_error);
+	ASSIGN_VMD_DMA_OPS(source, dest, dma_supported);
+#ifdef ARCH_HAS_DMA_GET_REQUIRED_MASK
+	ASSIGN_VMD_DMA_OPS(source, dest, get_required_mask);
+#endif
+	add_dma_domain(domain);
+}
+#undef ASSIGN_VMD_DMA_OPS
+#else
+static void vmd_teardown_dma_ops(struct vmd_dev *vmd) {}
+static void vmd_setup_dma_ops(struct vmd_dev *vmd) {}
+#endif
+
+static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,
+				  unsigned int devfn, int reg, int len)
+{
+	char __iomem *addr = vmd->cfgbar +
+			     (bus->number << 20) + (devfn << 12) + reg;
+
+	if ((addr - vmd->cfgbar) + len >=
+	    resource_size(&vmd->dev->resource[VMD_CFGBAR]))
+		return NULL;
+
+	return addr;
+}
+
+/*
+ * CPU may deadlock if config space is not serialized on some versions of this
+ * hardware, so all config space access is done under a spinlock.
+ */
+static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg,
+			int len, u32 *value)
+{
+	struct vmd_dev *vmd = vmd_from_bus(bus);
+	char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
+	unsigned long flags;
+	int ret = 0;
+
+	if (!addr)
+		return -EFAULT;
+
+	spin_lock_irqsave(&vmd->cfg_lock, flags);
+	switch (len) {
+	case 1:
+		*value = readb(addr);
+		break;
+	case 2:
+		*value = readw(addr);
+		break;
+	case 4:
+		*value = readl(addr);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	spin_unlock_irqrestore(&vmd->cfg_lock, flags);
+	return ret;
+}
+
+/*
+ * VMD h/w converts non-posted config writes to posted memory writes. The
+ * read-back in this function forces the completion so it returns only after
+ * the config space was written, as expected.
+ */
+static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg,
+			 int len, u32 value)
+{
+	struct vmd_dev *vmd = vmd_from_bus(bus);
+	char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
+	unsigned long flags;
+	int ret = 0;
+
+	if (!addr)
+		return -EFAULT;
+
+	spin_lock_irqsave(&vmd->cfg_lock, flags);
+	switch (len) {
+	case 1:
+		writeb(value, addr);
+		readb(addr);
+		break;
+	case 2:
+		writew(value, addr);
+		readw(addr);
+		break;
+	case 4:
+		writel(value, addr);
+		readl(addr);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	spin_unlock_irqrestore(&vmd->cfg_lock, flags);
+	return ret;
+}
+
+static struct pci_ops vmd_ops = {
+	.read		= vmd_pci_read,
+	.write		= vmd_pci_write,
+};
+
+static void vmd_attach_resources(struct vmd_dev *vmd)
+{
+	vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];
+	vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2];
+}
+
+static void vmd_detach_resources(struct vmd_dev *vmd)
+{
+	vmd->dev->resource[VMD_MEMBAR1].child = NULL;
+	vmd->dev->resource[VMD_MEMBAR2].child = NULL;
+}
+
+/*
+ * VMD domains start at 0x1000 to not clash with ACPI _SEG domains.
+ */
+static int vmd_find_free_domain(void)
+{
+	int domain = 0xffff;
+	struct pci_bus *bus = NULL;
+
+	while ((bus = pci_find_next_bus(bus)) != NULL)
+		domain = max_t(int, domain, pci_domain_nr(bus));
+	return domain + 1;
+}
+
+static int vmd_enable_domain(struct vmd_dev *vmd)
+{
+	struct pci_sysdata *sd = &vmd->sysdata;
+	struct resource *res;
+	u32 upper_bits;
+	unsigned long flags;
+	LIST_HEAD(resources);
+
+	res = &vmd->dev->resource[VMD_CFGBAR];
+	vmd->resources[0] = (struct resource) {
+		.name  = "VMD CFGBAR",
+		.start = 0,
+		.end   = (resource_size(res) >> 20) - 1,
+		.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED,
+	};
+
+	/*
+	 * If the window is below 4GB, clear IORESOURCE_MEM_64 so we can
+	 * put 32-bit resources in the window.
+	 *
+	 * There's no hardware reason why a 64-bit window *couldn't*
+	 * contain a 32-bit resource, but pbus_size_mem() computes the
+	 * bridge window size assuming a 64-bit window will contain no
+	 * 32-bit resources.  __pci_assign_resource() enforces that
+	 * artificial restriction to make sure everything will fit.
+	 *
+	 * The only way we could use a 64-bit non-prefechable MEMBAR is
+	 * if its address is <4GB so that we can convert it to a 32-bit
+	 * resource.  To be visible to the host OS, all VMD endpoints must
+	 * be initially configured by platform BIOS, which includes setting
+	 * up these resources.  We can assume the device is configured
+	 * according to the platform needs.
+	 */
+	res = &vmd->dev->resource[VMD_MEMBAR1];
+	upper_bits = upper_32_bits(res->end);
+	flags = res->flags & ~IORESOURCE_SIZEALIGN;
+	if (!upper_bits)
+		flags &= ~IORESOURCE_MEM_64;
+	vmd->resources[1] = (struct resource) {
+		.name  = "VMD MEMBAR1",
+		.start = res->start,
+		.end   = res->end,
+		.flags = flags,
+		.parent = res,
+	};
+
+	res = &vmd->dev->resource[VMD_MEMBAR2];
+	upper_bits = upper_32_bits(res->end);
+	flags = res->flags & ~IORESOURCE_SIZEALIGN;
+	if (!upper_bits)
+		flags &= ~IORESOURCE_MEM_64;
+	vmd->resources[2] = (struct resource) {
+		.name  = "VMD MEMBAR2",
+		.start = res->start + 0x2000,
+		.end   = res->end,
+		.flags = flags,
+		.parent = res,
+	};
+
+	sd->vmd_domain = true;
+	sd->domain = vmd_find_free_domain();
+	if (sd->domain < 0)
+		return sd->domain;
+
+	sd->node = pcibus_to_node(vmd->dev->bus);
+
+	vmd->irq_domain = pci_msi_create_irq_domain(NULL, &vmd_msi_domain_info,
+						    x86_vector_domain);
+	if (!vmd->irq_domain)
+		return -ENODEV;
+
+	pci_add_resource(&resources, &vmd->resources[0]);
+	pci_add_resource(&resources, &vmd->resources[1]);
+	pci_add_resource(&resources, &vmd->resources[2]);
+	vmd->bus = pci_create_root_bus(&vmd->dev->dev, 0, &vmd_ops, sd,
+				       &resources);
+	if (!vmd->bus) {
+		pci_free_resource_list(&resources);
+		irq_domain_remove(vmd->irq_domain);
+		return -ENODEV;
+	}
+
+	vmd_attach_resources(vmd);
+	vmd_setup_dma_ops(vmd);
+	dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain);
+	pci_rescan_bus(vmd->bus);
+
+	WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj,
+			       "domain"), "Can't create symlink to domain\n");
+	return 0;
+}
+
+static irqreturn_t vmd_irq(int irq, void *data)
+{
+	struct vmd_irq_list *irqs = data;
+	struct vmd_irq *vmdirq;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)
+		generic_handle_irq(vmdirq->virq);
+	rcu_read_unlock();
+
+	return IRQ_HANDLED;
+}
+
+static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	struct vmd_dev *vmd;
+	int i, err;
+
+	if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20))
+		return -ENOMEM;
+
+	vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL);
+	if (!vmd)
+		return -ENOMEM;
+
+	vmd->dev = dev;
+	err = pcim_enable_device(dev);
+	if (err < 0)
+		return err;
+
+	vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0);
+	if (!vmd->cfgbar)
+		return -ENOMEM;
+
+	pci_set_master(dev);
+	if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) &&
+	    dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32)))
+		return -ENODEV;
+
+	vmd->msix_count = pci_msix_vec_count(dev);
+	if (vmd->msix_count < 0)
+		return -ENODEV;
+
+	vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count,
+					PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
+	if (vmd->msix_count < 0)
+		return vmd->msix_count;
+
+	vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs),
+				 GFP_KERNEL);
+	if (!vmd->irqs)
+		return -ENOMEM;
+
+	for (i = 0; i < vmd->msix_count; i++) {
+		INIT_LIST_HEAD(&vmd->irqs[i].irq_list);
+		err = devm_request_irq(&dev->dev, pci_irq_vector(dev, i),
+				       vmd_irq, 0, "vmd", &vmd->irqs[i]);
+		if (err)
+			return err;
+	}
+
+	spin_lock_init(&vmd->cfg_lock);
+	pci_set_drvdata(dev, vmd);
+	err = vmd_enable_domain(vmd);
+	if (err)
+		return err;
+
+	dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n",
+		 vmd->sysdata.domain);
+	return 0;
+}
+
+static void vmd_remove(struct pci_dev *dev)
+{
+	struct vmd_dev *vmd = pci_get_drvdata(dev);
+
+	vmd_detach_resources(vmd);
+	pci_set_drvdata(dev, NULL);
+	sysfs_remove_link(&vmd->dev->dev.kobj, "domain");
+	pci_stop_root_bus(vmd->bus);
+	pci_remove_root_bus(vmd->bus);
+	vmd_teardown_dma_ops(vmd);
+	irq_domain_remove(vmd->irq_domain);
+}
+
+#ifdef CONFIG_PM
+static int vmd_suspend(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	pci_save_state(pdev);
+	return 0;
+}
+
+static int vmd_resume(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	pci_restore_state(pdev);
+	return 0;
+}
+#endif
+static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume);
+
+static const struct pci_device_id vmd_ids[] = {
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x201d),},
+	{0,}
+};
+MODULE_DEVICE_TABLE(pci, vmd_ids);
+
+static struct pci_driver vmd_drv = {
+	.name		= "vmd",
+	.id_table	= vmd_ids,
+	.probe		= vmd_probe,
+	.remove		= vmd_remove,
+	.driver		= {
+		.pm	= &vmd_dev_pm_ops,
+	},
+};
+module_pci_driver(vmd_drv);
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION("0.6");
diff --git a/drivers/pci/hotplug/cpci_hotplug.h b/drivers/pci/hotplug/cpci_hotplug.h
index 555bcde3b196..60e66e027ebc 100644
--- a/drivers/pci/hotplug/cpci_hotplug.h
+++ b/drivers/pci/hotplug/cpci_hotplug.h
@@ -101,10 +101,8 @@ int cpci_unconfigure_slot(struct slot *slot);
 
 #ifdef CONFIG_HOTPLUG_PCI_CPCI
 int cpci_hotplug_init(int debug);
-void cpci_hotplug_exit(void);
 #else
 static inline int cpci_hotplug_init(int debug) { return 0; }
-static inline void cpci_hotplug_exit(void) { }
 #endif
 
 #endif	/* _CPCI_HOTPLUG_H */
diff --git a/drivers/pci/hotplug/cpci_hotplug_core.c b/drivers/pci/hotplug/cpci_hotplug_core.c
index 7d3866c47312..7ec8a8f72c69 100644
--- a/drivers/pci/hotplug/cpci_hotplug_core.c
+++ b/drivers/pci/hotplug/cpci_hotplug_core.c
@@ -719,13 +719,3 @@ cpci_hotplug_init(int debug)
 	cpci_debug = debug;
 	return 0;
 }
-
-void __exit
-cpci_hotplug_exit(void)
-{
-	/*
-	 * Clean everything up.
-	 */
-	cpci_hp_stop();
-	cpci_hp_unregister_controller(controller);
-}
diff --git a/drivers/pci/hotplug/pci_hotplug_core.c b/drivers/pci/hotplug/pci_hotplug_core.c
index 9acd1997c6fe..fea0b8b33589 100644
--- a/drivers/pci/hotplug/pci_hotplug_core.c
+++ b/drivers/pci/hotplug/pci_hotplug_core.c
@@ -25,7 +25,7 @@
  *
  */
 
-#include <linux/module.h>
+#include <linux/module.h>	/* try_module_get & module_put */
 #include <linux/moduleparam.h>
 #include <linux/kernel.h>
 #include <linux/types.h>
@@ -537,17 +537,11 @@ static int __init pci_hotplug_init(void)
 	info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
 	return result;
 }
+device_initcall(pci_hotplug_init);
 
-static void __exit pci_hotplug_exit(void)
-{
-	cpci_hotplug_exit();
-}
-
-module_init(pci_hotplug_init);
-module_exit(pci_hotplug_exit);
-
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL");
+/*
+ * not really modular, but the easiest way to keep compat with existing
+ * bootargs behaviour is to continue using module_param here.
+ */
 module_param(debug, bool, 0644);
 MODULE_PARM_DESC(debug, "Debugging mode enabled or not");
diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h
index e764918641ae..37d70b5ad22f 100644
--- a/drivers/pci/hotplug/pciehp.h
+++ b/drivers/pci/hotplug/pciehp.h
@@ -152,6 +152,9 @@ bool pciehp_check_link_active(struct controller *ctrl);
 void pciehp_release_ctrl(struct controller *ctrl);
 int pciehp_reset_slot(struct slot *slot, int probe);
 
+int pciehp_set_raw_indicator_status(struct hotplug_slot *h_slot, u8 status);
+int pciehp_get_raw_indicator_status(struct hotplug_slot *h_slot, u8 *status);
+
 static inline const char *slot_name(struct slot *slot)
 {
 	return hotplug_slot_name(slot->hotplug_slot);
diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c
index ac531e674a05..7d32fa33dcef 100644
--- a/drivers/pci/hotplug/pciehp_core.c
+++ b/drivers/pci/hotplug/pciehp_core.c
@@ -27,7 +27,6 @@
  *
  */
 
-#include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
@@ -47,10 +46,10 @@ static bool pciehp_force;
 #define DRIVER_AUTHOR	"Dan Zink <dan.zink@compaq.com>, Greg Kroah-Hartman <greg@kroah.com>, Dely Sy <dely.l.sy@intel.com>"
 #define DRIVER_DESC	"PCI Express Hot Plug Controller Driver"
 
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL");
-
+/*
+ * not really modular, but the easiest way to keep compat with existing
+ * bootargs behaviour is to continue using module_param here.
+ */
 module_param(pciehp_debug, bool, 0644);
 module_param(pciehp_poll_mode, bool, 0644);
 module_param(pciehp_poll_time, int, 0644);
@@ -114,6 +113,9 @@ static int init_slot(struct controller *ctrl)
 	if (ATTN_LED(ctrl)) {
 		ops->get_attention_status = get_attention_status;
 		ops->set_attention_status = set_attention_status;
+	} else if (ctrl->pcie->port->hotplug_user_indicators) {
+		ops->get_attention_status = pciehp_get_raw_indicator_status;
+		ops->set_attention_status = pciehp_set_raw_indicator_status;
 	}
 
 	/* register this slot with the hotplug pci core */
@@ -337,13 +339,4 @@ static int __init pcied_init(void)
 
 	return retval;
 }
-
-static void __exit pcied_cleanup(void)
-{
-	dbg("unload_pciehpd()\n");
-	pcie_port_service_unregister(&hpdriver_portdrv);
-	info(DRIVER_DESC " version: " DRIVER_VERSION " unloaded\n");
-}
-
-module_init(pcied_init);
-module_exit(pcied_cleanup);
+device_initcall(pcied_init);
diff --git a/drivers/pci/hotplug/pciehp_ctrl.c b/drivers/pci/hotplug/pciehp_ctrl.c
index 880978b6d534..efe69e879455 100644
--- a/drivers/pci/hotplug/pciehp_ctrl.c
+++ b/drivers/pci/hotplug/pciehp_ctrl.c
@@ -106,7 +106,7 @@ static int board_added(struct slot *p_slot)
 
 	/* Check for a power fault */
 	if (ctrl->power_fault_detected || pciehp_query_power_fault(p_slot)) {
-		ctrl_err(ctrl, "Power fault on slot %s\n", slot_name(p_slot));
+		ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(p_slot));
 		retval = -EIO;
 		goto err_exit;
 	}
@@ -120,6 +120,7 @@ static int board_added(struct slot *p_slot)
 	}
 
 	pciehp_green_led_on(p_slot);
+	pciehp_set_attention_status(p_slot, 0);
 	return 0;
 
 err_exit:
@@ -253,11 +254,11 @@ static void handle_button_press_event(struct slot *p_slot)
 		pciehp_get_power_status(p_slot, &getstatus);
 		if (getstatus) {
 			p_slot->state = BLINKINGOFF_STATE;
-			ctrl_info(ctrl, "PCI slot #%s - powering off due to button press\n",
+			ctrl_info(ctrl, "Slot(%s): Powering off due to button press\n",
 				  slot_name(p_slot));
 		} else {
 			p_slot->state = BLINKINGON_STATE;
-			ctrl_info(ctrl, "PCI slot #%s - powering on due to button press\n",
+			ctrl_info(ctrl, "Slot(%s) Powering on due to button press\n",
 				  slot_name(p_slot));
 		}
 		/* blink green LED and turn off amber */
@@ -272,14 +273,14 @@ static void handle_button_press_event(struct slot *p_slot)
 		 * press the attention again before the 5 sec. limit
 		 * expires to cancel hot-add or hot-remove
 		 */
-		ctrl_info(ctrl, "Button cancel on Slot(%s)\n", slot_name(p_slot));
+		ctrl_info(ctrl, "Slot(%s): Button cancel\n", slot_name(p_slot));
 		cancel_delayed_work(&p_slot->work);
 		if (p_slot->state == BLINKINGOFF_STATE)
 			pciehp_green_led_on(p_slot);
 		else
 			pciehp_green_led_off(p_slot);
 		pciehp_set_attention_status(p_slot, 0);
-		ctrl_info(ctrl, "PCI slot #%s - action canceled due to button press\n",
+		ctrl_info(ctrl, "Slot(%s): Action canceled due to button press\n",
 			  slot_name(p_slot));
 		p_slot->state = STATIC_STATE;
 		break;
@@ -290,10 +291,12 @@ static void handle_button_press_event(struct slot *p_slot)
 		 * this means that the previous attention button action
 		 * to hot-add or hot-remove is undergoing
 		 */
-		ctrl_info(ctrl, "Button ignore on Slot(%s)\n", slot_name(p_slot));
+		ctrl_info(ctrl, "Slot(%s): Button ignored\n",
+			  slot_name(p_slot));
 		break;
 	default:
-		ctrl_warn(ctrl, "ignoring invalid state %#x\n", p_slot->state);
+		ctrl_err(ctrl, "Slot(%s): Ignoring invalid state %#x\n",
+			 slot_name(p_slot), p_slot->state);
 		break;
 	}
 }
@@ -301,20 +304,6 @@ static void handle_button_press_event(struct slot *p_slot)
 /*
  * Note: This function must be called with slot->lock held
  */
-static void handle_surprise_event(struct slot *p_slot)
-{
-	u8 getstatus;
-
-	pciehp_get_adapter_status(p_slot, &getstatus);
-	if (!getstatus)
-		pciehp_queue_power_work(p_slot, DISABLE_REQ);
-	else
-		pciehp_queue_power_work(p_slot, ENABLE_REQ);
-}
-
-/*
- * Note: This function must be called with slot->lock held
- */
 static void handle_link_event(struct slot *p_slot, u32 event)
 {
 	struct controller *ctrl = p_slot->ctrl;
@@ -330,31 +319,27 @@ static void handle_link_event(struct slot *p_slot, u32 event)
 		break;
 	case POWERON_STATE:
 		if (event == INT_LINK_UP) {
-			ctrl_info(ctrl,
-				  "Link Up event ignored on slot(%s): already powering on\n",
+			ctrl_info(ctrl, "Slot(%s): Link Up event ignored; already powering on\n",
 				  slot_name(p_slot));
 		} else {
-			ctrl_info(ctrl,
-				  "Link Down event queued on slot(%s): currently getting powered on\n",
+			ctrl_info(ctrl, "Slot(%s): Link Down event queued; currently getting powered on\n",
 				  slot_name(p_slot));
 			pciehp_queue_power_work(p_slot, DISABLE_REQ);
 		}
 		break;
 	case POWEROFF_STATE:
 		if (event == INT_LINK_UP) {
-			ctrl_info(ctrl,
-				  "Link Up event queued on slot(%s): currently getting powered off\n",
+			ctrl_info(ctrl, "Slot(%s): Link Up event queued; currently getting powered off\n",
 				  slot_name(p_slot));
 			pciehp_queue_power_work(p_slot, ENABLE_REQ);
 		} else {
-			ctrl_info(ctrl,
-				  "Link Down event ignored on slot(%s): already powering off\n",
+			ctrl_info(ctrl, "Slot(%s): Link Down event ignored; already powering off\n",
 				  slot_name(p_slot));
 		}
 		break;
 	default:
-		ctrl_err(ctrl, "ignoring invalid state %#x on slot(%s)\n",
-			 p_slot->state, slot_name(p_slot));
+		ctrl_err(ctrl, "Slot(%s): Ignoring invalid state %#x\n",
+			 slot_name(p_slot), p_slot->state);
 		break;
 	}
 }
@@ -377,14 +362,14 @@ static void interrupt_event_handler(struct work_struct *work)
 		pciehp_green_led_off(p_slot);
 		break;
 	case INT_PRESENCE_ON:
-		handle_surprise_event(p_slot);
+		pciehp_queue_power_work(p_slot, ENABLE_REQ);
 		break;
 	case INT_PRESENCE_OFF:
 		/*
 		 * Regardless of surprise capability, we need to
 		 * definitely remove a card that has been pulled out!
 		 */
-		handle_surprise_event(p_slot);
+		pciehp_queue_power_work(p_slot, DISABLE_REQ);
 		break;
 	case INT_LINK_UP:
 	case INT_LINK_DOWN:
@@ -404,18 +389,17 @@ static void interrupt_event_handler(struct work_struct *work)
 int pciehp_enable_slot(struct slot *p_slot)
 {
 	u8 getstatus = 0;
-	int rc;
 	struct controller *ctrl = p_slot->ctrl;
 
 	pciehp_get_adapter_status(p_slot, &getstatus);
 	if (!getstatus) {
-		ctrl_info(ctrl, "No adapter on slot(%s)\n", slot_name(p_slot));
+		ctrl_info(ctrl, "Slot(%s): No adapter\n", slot_name(p_slot));
 		return -ENODEV;
 	}
 	if (MRL_SENS(p_slot->ctrl)) {
 		pciehp_get_latch_status(p_slot, &getstatus);
 		if (getstatus) {
-			ctrl_info(ctrl, "Latch open on slot(%s)\n",
+			ctrl_info(ctrl, "Slot(%s): Latch open\n",
 				  slot_name(p_slot));
 			return -ENODEV;
 		}
@@ -424,19 +408,13 @@ int pciehp_enable_slot(struct slot *p_slot)
 	if (POWER_CTRL(p_slot->ctrl)) {
 		pciehp_get_power_status(p_slot, &getstatus);
 		if (getstatus) {
-			ctrl_info(ctrl, "Already enabled on slot(%s)\n",
+			ctrl_info(ctrl, "Slot(%s): Already enabled\n",
 				  slot_name(p_slot));
 			return -EINVAL;
 		}
 	}
 
-	pciehp_get_latch_status(p_slot, &getstatus);
-
-	rc = board_added(p_slot);
-	if (rc)
-		pciehp_get_latch_status(p_slot, &getstatus);
-
-	return rc;
+	return board_added(p_slot);
 }
 
 /*
@@ -453,7 +431,7 @@ int pciehp_disable_slot(struct slot *p_slot)
 	if (POWER_CTRL(p_slot->ctrl)) {
 		pciehp_get_power_status(p_slot, &getstatus);
 		if (!getstatus) {
-			ctrl_info(ctrl, "Already disabled on slot(%s)\n",
+			ctrl_info(ctrl, "Slot(%s): Already disabled\n",
 				  slot_name(p_slot));
 			return -EINVAL;
 		}
@@ -481,17 +459,17 @@ int pciehp_sysfs_enable_slot(struct slot *p_slot)
 		p_slot->state = STATIC_STATE;
 		break;
 	case POWERON_STATE:
-		ctrl_info(ctrl, "Slot %s is already in powering on state\n",
+		ctrl_info(ctrl, "Slot(%s): Already in powering on state\n",
 			  slot_name(p_slot));
 		break;
 	case BLINKINGOFF_STATE:
 	case POWEROFF_STATE:
-		ctrl_info(ctrl, "Already enabled on slot %s\n",
+		ctrl_info(ctrl, "Slot(%s): Already enabled\n",
 			  slot_name(p_slot));
 		break;
 	default:
-		ctrl_err(ctrl, "invalid state %#x on slot %s\n",
-			 p_slot->state, slot_name(p_slot));
+		ctrl_err(ctrl, "Slot(%s): Invalid state %#x\n",
+			 slot_name(p_slot), p_slot->state);
 		break;
 	}
 	mutex_unlock(&p_slot->lock);
@@ -518,17 +496,17 @@ int pciehp_sysfs_disable_slot(struct slot *p_slot)
 		p_slot->state = STATIC_STATE;
 		break;
 	case POWEROFF_STATE:
-		ctrl_info(ctrl, "Slot %s is already in powering off state\n",
+		ctrl_info(ctrl, "Slot(%s): Already in powering off state\n",
 			  slot_name(p_slot));
 		break;
 	case BLINKINGON_STATE:
 	case POWERON_STATE:
-		ctrl_info(ctrl, "Already disabled on slot %s\n",
+		ctrl_info(ctrl, "Slot(%s): Already disabled\n",
 			  slot_name(p_slot));
 		break;
 	default:
-		ctrl_err(ctrl, "invalid state %#x on slot %s\n",
-			 p_slot->state, slot_name(p_slot));
+		ctrl_err(ctrl, "Slot(%s): Invalid state %#x\n",
+			 slot_name(p_slot), p_slot->state);
 		break;
 	}
 	mutex_unlock(&p_slot->lock);
diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c
index 08e84d61874e..b57fc6d6e28a 100644
--- a/drivers/pci/hotplug/pciehp_hpc.c
+++ b/drivers/pci/hotplug/pciehp_hpc.c
@@ -355,6 +355,18 @@ static int pciehp_link_enable(struct controller *ctrl)
 	return __pciehp_link_set(ctrl, true);
 }
 
+int pciehp_get_raw_indicator_status(struct hotplug_slot *hotplug_slot,
+				    u8 *status)
+{
+	struct slot *slot = hotplug_slot->private;
+	struct pci_dev *pdev = ctrl_dev(slot->ctrl);
+	u16 slot_ctrl;
+
+	pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl);
+	*status = (slot_ctrl & (PCI_EXP_SLTCTL_AIC | PCI_EXP_SLTCTL_PIC)) >> 6;
+	return 0;
+}
+
 void pciehp_get_attention_status(struct slot *slot, u8 *status)
 {
 	struct controller *ctrl = slot->ctrl;
@@ -431,6 +443,17 @@ int pciehp_query_power_fault(struct slot *slot)
 	return !!(slot_status & PCI_EXP_SLTSTA_PFD);
 }
 
+int pciehp_set_raw_indicator_status(struct hotplug_slot *hotplug_slot,
+				    u8 status)
+{
+	struct slot *slot = hotplug_slot->private;
+	struct controller *ctrl = slot->ctrl;
+
+	pcie_write_cmd_nowait(ctrl, status << 6,
+			      PCI_EXP_SLTCTL_AIC | PCI_EXP_SLTCTL_PIC);
+	return 0;
+}
+
 void pciehp_set_attention_status(struct slot *slot, u8 value)
 {
 	struct controller *ctrl = slot->ctrl;
@@ -535,14 +558,14 @@ void pciehp_power_off_slot(struct slot *slot)
 		 PCI_EXP_SLTCTL_PWR_OFF);
 }
 
-static irqreturn_t pcie_isr(int irq, void *dev_id)
+static irqreturn_t pciehp_isr(int irq, void *dev_id)
 {
 	struct controller *ctrl = (struct controller *)dev_id;
 	struct pci_dev *pdev = ctrl_dev(ctrl);
 	struct pci_bus *subordinate = pdev->subordinate;
 	struct pci_dev *dev;
 	struct slot *slot = ctrl->slot;
-	u16 detected, intr_loc;
+	u16 status, events;
 	u8 present;
 	bool link;
 
@@ -550,36 +573,31 @@ static irqreturn_t pcie_isr(int irq, void *dev_id)
 	if (pdev->current_state == PCI_D3cold)
 		return IRQ_NONE;
 
+	pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &status);
+	if (status == (u16) ~0) {
+		ctrl_info(ctrl, "%s: no response from device\n", __func__);
+		return IRQ_NONE;
+	}
+
 	/*
-	 * In order to guarantee that all interrupt events are
-	 * serviced, we need to re-inspect Slot Status register after
-	 * clearing what is presumed to be the last pending interrupt.
+	 * Slot Status contains plain status bits as well as event
+	 * notification bits; right now we only want the event bits.
 	 */
-	intr_loc = 0;
-	do {
-		pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &detected);
-		if (detected == (u16) ~0) {
-			ctrl_info(ctrl, "%s: no response from device\n",
-				  __func__);
-			return IRQ_HANDLED;
-		}
+	events = status & (PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD |
+			   PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_CC |
+			   PCI_EXP_SLTSTA_DLLSC);
+	if (!events)
+		return IRQ_NONE;
 
-		detected &= (PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD |
-			     PCI_EXP_SLTSTA_PDC |
-			     PCI_EXP_SLTSTA_CC | PCI_EXP_SLTSTA_DLLSC);
-		detected &= ~intr_loc;
-		intr_loc |= detected;
-		if (!intr_loc)
-			return IRQ_NONE;
-		if (detected)
-			pcie_capability_write_word(pdev, PCI_EXP_SLTSTA,
-						   intr_loc);
-	} while (detected);
+	/* Capture link status before clearing interrupts */
+	if (events & PCI_EXP_SLTSTA_DLLSC)
+		link = pciehp_check_link_active(ctrl);
 
-	ctrl_dbg(ctrl, "pending interrupts %#06x from Slot Status\n", intr_loc);
+	pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, events);
+	ctrl_dbg(ctrl, "pending interrupts %#06x from Slot Status\n", events);
 
 	/* Check Command Complete Interrupt Pending */
-	if (intr_loc & PCI_EXP_SLTSTA_CC) {
+	if (events & PCI_EXP_SLTSTA_CC) {
 		ctrl->cmd_busy = 0;
 		smp_mb();
 		wake_up(&ctrl->queue);
@@ -589,42 +607,38 @@ static irqreturn_t pcie_isr(int irq, void *dev_id)
 		list_for_each_entry(dev, &subordinate->devices, bus_list) {
 			if (dev->ignore_hotplug) {
 				ctrl_dbg(ctrl, "ignoring hotplug event %#06x (%s requested no hotplug)\n",
-					 intr_loc, pci_name(dev));
+					 events, pci_name(dev));
 				return IRQ_HANDLED;
 			}
 		}
 	}
 
-	if (!(intr_loc & ~PCI_EXP_SLTSTA_CC))
-		return IRQ_HANDLED;
-
 	/* Check Attention Button Pressed */
-	if (intr_loc & PCI_EXP_SLTSTA_ABP) {
-		ctrl_info(ctrl, "Button pressed on Slot(%s)\n",
+	if (events & PCI_EXP_SLTSTA_ABP) {
+		ctrl_info(ctrl, "Slot(%s): Attention button pressed\n",
 			  slot_name(slot));
 		pciehp_queue_interrupt_event(slot, INT_BUTTON_PRESS);
 	}
 
 	/* Check Presence Detect Changed */
-	if (intr_loc & PCI_EXP_SLTSTA_PDC) {
-		pciehp_get_adapter_status(slot, &present);
-		ctrl_info(ctrl, "Card %spresent on Slot(%s)\n",
-			  present ? "" : "not ", slot_name(slot));
+	if (events & PCI_EXP_SLTSTA_PDC) {
+		present = !!(status & PCI_EXP_SLTSTA_PDS);
+		ctrl_info(ctrl, "Slot(%s): Card %spresent\n", slot_name(slot),
+			  present ? "" : "not ");
 		pciehp_queue_interrupt_event(slot, present ? INT_PRESENCE_ON :
 					     INT_PRESENCE_OFF);
 	}
 
 	/* Check Power Fault Detected */
-	if ((intr_loc & PCI_EXP_SLTSTA_PFD) && !ctrl->power_fault_detected) {
+	if ((events & PCI_EXP_SLTSTA_PFD) && !ctrl->power_fault_detected) {
 		ctrl->power_fault_detected = 1;
-		ctrl_err(ctrl, "Power fault on slot %s\n", slot_name(slot));
+		ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(slot));
 		pciehp_queue_interrupt_event(slot, INT_POWER_FAULT);
 	}
 
-	if (intr_loc & PCI_EXP_SLTSTA_DLLSC) {
-		link = pciehp_check_link_active(ctrl);
-		ctrl_info(ctrl, "slot(%s): Link %s event\n",
-			  slot_name(slot), link ? "Up" : "Down");
+	if (events & PCI_EXP_SLTSTA_DLLSC) {
+		ctrl_info(ctrl, "Slot(%s): Link %s\n", slot_name(slot),
+			  link ? "Up" : "Down");
 		pciehp_queue_interrupt_event(slot, link ? INT_LINK_UP :
 					     INT_LINK_DOWN);
 	}
@@ -632,6 +646,25 @@ static irqreturn_t pcie_isr(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t pcie_isr(int irq, void *dev_id)
+{
+	irqreturn_t rc, handled = IRQ_NONE;
+
+	/*
+	 * To guarantee that all interrupt events are serviced, we need to
+	 * re-inspect Slot Status register after clearing what is presumed
+	 * to be the last pending interrupt.
+	 */
+	do {
+		rc = pciehp_isr(irq, dev_id);
+		if (rc == IRQ_HANDLED)
+			handled = IRQ_HANDLED;
+	} while (rc == IRQ_HANDLED);
+
+	/* Return IRQ_HANDLED if we handled one or more events */
+	return handled;
+}
+
 void pcie_enable_notification(struct controller *ctrl)
 {
 	u16 cmd, mask;
@@ -804,6 +837,10 @@ struct controller *pcie_init(struct pcie_device *dev)
 	}
 	ctrl->pcie = dev;
 	pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, &slot_cap);
+
+	if (pdev->hotplug_user_indicators)
+		slot_cap &= ~(PCI_EXP_SLTCAP_AIP | PCI_EXP_SLTCAP_PIP);
+
 	ctrl->slot_cap = slot_cap;
 	mutex_init(&ctrl->ctrl_lock);
 	init_waitqueue_head(&ctrl->queue);
diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c
index e6245b03f0a1..56efaf72d08e 100644
--- a/drivers/pci/hotplug/pnv_php.c
+++ b/drivers/pci/hotplug/pnv_php.c
@@ -22,6 +22,12 @@
 #define DRIVER_AUTHOR	"Gavin Shan, IBM Corporation"
 #define DRIVER_DESC	"PowerPC PowerNV PCI Hotplug Driver"
 
+struct pnv_php_event {
+	bool			added;
+	struct pnv_php_slot	*php_slot;
+	struct work_struct	work;
+};
+
 static LIST_HEAD(pnv_php_slot_list);
 static DEFINE_SPINLOCK(pnv_php_lock);
 
@@ -29,12 +35,40 @@ static void pnv_php_register(struct device_node *dn);
 static void pnv_php_unregister_one(struct device_node *dn);
 static void pnv_php_unregister(struct device_node *dn);
 
+static void pnv_php_disable_irq(struct pnv_php_slot *php_slot)
+{
+	struct pci_dev *pdev = php_slot->pdev;
+	u16 ctrl;
+
+	if (php_slot->irq > 0) {
+		pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &ctrl);
+		ctrl &= ~(PCI_EXP_SLTCTL_HPIE |
+			  PCI_EXP_SLTCTL_PDCE |
+			  PCI_EXP_SLTCTL_DLLSCE);
+		pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, ctrl);
+
+		free_irq(php_slot->irq, php_slot);
+		php_slot->irq = 0;
+	}
+
+	if (php_slot->wq) {
+		destroy_workqueue(php_slot->wq);
+		php_slot->wq = NULL;
+	}
+
+	if (pdev->msix_enabled)
+		pci_disable_msix(pdev);
+	else if (pdev->msi_enabled)
+		pci_disable_msi(pdev);
+}
+
 static void pnv_php_free_slot(struct kref *kref)
 {
 	struct pnv_php_slot *php_slot = container_of(kref,
 					struct pnv_php_slot, kref);
 
 	WARN_ON(!list_empty(&php_slot->children));
+	pnv_php_disable_irq(php_slot);
 	kfree(php_slot->name);
 	kfree(php_slot);
 }
@@ -122,7 +156,7 @@ static void pnv_php_detach_device_nodes(struct device_node *parent)
 
 		of_node_put(dn);
 		refcount = atomic_read(&dn->kobj.kref.refcount);
-		if (unlikely(refcount != 1))
+		if (refcount != 1)
 			pr_warn("Invalid refcount %d on <%s>\n",
 				refcount, of_node_full_name(dn));
 
@@ -184,11 +218,11 @@ static int pnv_php_populate_changeset(struct of_changeset *ocs,
 
 	for_each_child_of_node(dn, child) {
 		ret = of_changeset_attach_node(ocs, child);
-		if (unlikely(ret))
+		if (ret)
 			break;
 
 		ret = pnv_php_populate_changeset(ocs, child);
-		if (unlikely(ret))
+		if (ret)
 			break;
 	}
 
@@ -201,7 +235,7 @@ static void *pnv_php_add_one_pdn(struct device_node *dn, void *data)
 	struct pci_dn *pdn;
 
 	pdn = pci_add_device_node_info(hose, dn);
-	if (unlikely(!pdn))
+	if (!pdn)
 		return ERR_PTR(-ENOMEM);
 
 	return NULL;
@@ -224,21 +258,21 @@ static int pnv_php_add_devtree(struct pnv_php_slot *php_slot)
 	 * fits the real size.
 	 */
 	fdt1 = kzalloc(0x10000, GFP_KERNEL);
-	if (unlikely(!fdt1)) {
+	if (!fdt1) {
 		ret = -ENOMEM;
 		dev_warn(&php_slot->pdev->dev, "Cannot alloc FDT blob\n");
 		goto out;
 	}
 
 	ret = pnv_pci_get_device_tree(php_slot->dn->phandle, fdt1, 0x10000);
-	if (unlikely(ret)) {
+	if (ret) {
 		dev_warn(&php_slot->pdev->dev, "Error %d getting FDT blob\n",
 			 ret);
 		goto free_fdt1;
 	}
 
 	fdt = kzalloc(fdt_totalsize(fdt1), GFP_KERNEL);
-	if (unlikely(!fdt)) {
+	if (!fdt) {
 		ret = -ENOMEM;
 		dev_warn(&php_slot->pdev->dev, "Cannot %d bytes memory\n",
 			 fdt_totalsize(fdt1));
@@ -248,7 +282,7 @@ static int pnv_php_add_devtree(struct pnv_php_slot *php_slot)
 	/* Unflatten device tree blob */
 	memcpy(fdt, fdt1, fdt_totalsize(fdt1));
 	dt = of_fdt_unflatten_tree(fdt, php_slot->dn, NULL);
-	if (unlikely(!dt)) {
+	if (!dt) {
 		ret = -EINVAL;
 		dev_warn(&php_slot->pdev->dev, "Cannot unflatten FDT\n");
 		goto free_fdt;
@@ -258,7 +292,7 @@ static int pnv_php_add_devtree(struct pnv_php_slot *php_slot)
 	of_changeset_init(&php_slot->ocs);
 	pnv_php_reverse_nodes(php_slot->dn);
 	ret = pnv_php_populate_changeset(&php_slot->ocs, php_slot->dn);
-	if (unlikely(ret)) {
+	if (ret) {
 		pnv_php_reverse_nodes(php_slot->dn);
 		dev_warn(&php_slot->pdev->dev, "Error %d populating changeset\n",
 			 ret);
@@ -267,7 +301,7 @@ static int pnv_php_add_devtree(struct pnv_php_slot *php_slot)
 
 	php_slot->dn->child = NULL;
 	ret = of_changeset_apply(&php_slot->ocs);
-	if (unlikely(ret)) {
+	if (ret) {
 		dev_warn(&php_slot->pdev->dev, "Error %d applying changeset\n",
 			 ret);
 		goto destroy_changeset;
@@ -301,7 +335,7 @@ int pnv_php_set_slot_power_state(struct hotplug_slot *slot,
 	int ret;
 
 	ret = pnv_pci_set_power_state(php_slot->id, state, &msg);
-	if (likely(ret > 0)) {
+	if (ret > 0) {
 		if (be64_to_cpu(msg.params[1]) != php_slot->dn->phandle	||
 		    be64_to_cpu(msg.params[2]) != state			||
 		    be64_to_cpu(msg.params[3]) != OPAL_SUCCESS) {
@@ -311,7 +345,7 @@ int pnv_php_set_slot_power_state(struct hotplug_slot *slot,
 				 be64_to_cpu(msg.params[3]));
 			return -ENOMSG;
 		}
-	} else if (unlikely(ret < 0)) {
+	} else if (ret < 0) {
 		dev_warn(&php_slot->pdev->dev, "Error %d powering %s\n",
 			 ret, (state == OPAL_PCI_SLOT_POWER_ON) ? "on" : "off");
 		return ret;
@@ -338,7 +372,7 @@ static int pnv_php_get_power_state(struct hotplug_slot *slot, u8 *state)
 	 * be on.
 	 */
 	ret = pnv_pci_get_power_state(php_slot->id, &power_state);
-	if (unlikely(ret)) {
+	if (ret) {
 		dev_warn(&php_slot->pdev->dev, "Error %d getting power status\n",
 			 ret);
 	} else {
@@ -360,7 +394,7 @@ static int pnv_php_get_adapter_state(struct hotplug_slot *slot, u8 *state)
 	 * get that, it will fail back to be empty.
 	 */
 	ret = pnv_pci_get_presence_state(php_slot->id, &presence);
-	if (likely(ret >= 0)) {
+	if (ret >= 0) {
 		*state = presence;
 		slot->info->adapter_status = presence;
 		ret = 0;
@@ -393,7 +427,7 @@ static int pnv_php_enable(struct pnv_php_slot *php_slot, bool rescan)
 
 	/* Retrieve slot presence status */
 	ret = pnv_php_get_adapter_state(slot, &presence);
-	if (unlikely(ret))
+	if (ret)
 		return ret;
 
 	/* Proceed if there have nothing behind the slot */
@@ -414,7 +448,7 @@ static int pnv_php_enable(struct pnv_php_slot *php_slot, bool rescan)
 		php_slot->power_state_check = true;
 
 		ret = pnv_php_get_power_state(slot, &power_status);
-		if (unlikely(ret))
+		if (ret)
 			return ret;
 
 		if (power_status != OPAL_PCI_SLOT_POWER_ON)
@@ -423,7 +457,7 @@ static int pnv_php_enable(struct pnv_php_slot *php_slot, bool rescan)
 
 	/* Check the power status. Scan the slot if it is already on */
 	ret = pnv_php_get_power_state(slot, &power_status);
-	if (unlikely(ret))
+	if (ret)
 		return ret;
 
 	if (power_status == OPAL_PCI_SLOT_POWER_ON)
@@ -431,7 +465,7 @@ static int pnv_php_enable(struct pnv_php_slot *php_slot, bool rescan)
 
 	/* Power is off, turn it on and then scan the slot */
 	ret = pnv_php_set_slot_power_state(slot, OPAL_PCI_SLOT_POWER_ON);
-	if (unlikely(ret))
+	if (ret)
 		return ret;
 
 scan:
@@ -513,29 +547,30 @@ static struct pnv_php_slot *pnv_php_alloc_slot(struct device_node *dn)
 	struct pci_bus *bus;
 	const char *label;
 	uint64_t id;
+	int ret;
 
-	label = of_get_property(dn, "ibm,slot-label", NULL);
-	if (unlikely(!label))
+	ret = of_property_read_string(dn, "ibm,slot-label", &label);
+	if (ret)
 		return NULL;
 
 	if (pnv_pci_get_slot_id(dn, &id))
 		return NULL;
 
 	bus = pci_find_bus_by_node(dn);
-	if (unlikely(!bus))
+	if (!bus)
 		return NULL;
 
 	php_slot = kzalloc(sizeof(*php_slot), GFP_KERNEL);
-	if (unlikely(!php_slot))
+	if (!php_slot)
 		return NULL;
 
 	php_slot->name = kstrdup(label, GFP_KERNEL);
-	if (unlikely(!php_slot->name)) {
+	if (!php_slot->name) {
 		kfree(php_slot);
 		return NULL;
 	}
 
-	if (likely(dn->child && PCI_DN(dn->child)))
+	if (dn->child && PCI_DN(dn->child))
 		php_slot->slot_no = PCI_SLOT(PCI_DN(dn->child)->devfn);
 	else
 		php_slot->slot_no = -1;   /* Placeholder slot */
@@ -567,7 +602,7 @@ static int pnv_php_register_slot(struct pnv_php_slot *php_slot)
 
 	/* Check if the slot is registered or not */
 	parent = pnv_php_find_slot(php_slot->dn);
-	if (unlikely(parent)) {
+	if (parent) {
 		pnv_php_put_slot(parent);
 		return -EEXIST;
 	}
@@ -575,7 +610,7 @@ static int pnv_php_register_slot(struct pnv_php_slot *php_slot)
 	/* Register PCI slot */
 	ret = pci_hp_register(&php_slot->slot, php_slot->bus,
 			      php_slot->slot_no, php_slot->name);
-	if (unlikely(ret)) {
+	if (ret) {
 		dev_warn(&php_slot->pdev->dev, "Error %d registering slot\n",
 			 ret);
 		return ret;
@@ -609,33 +644,213 @@ static int pnv_php_register_slot(struct pnv_php_slot *php_slot)
 	return 0;
 }
 
+static int pnv_php_enable_msix(struct pnv_php_slot *php_slot)
+{
+	struct pci_dev *pdev = php_slot->pdev;
+	struct msix_entry entry;
+	int nr_entries, ret;
+	u16 pcie_flag;
+
+	/* Get total number of MSIx entries */
+	nr_entries = pci_msix_vec_count(pdev);
+	if (nr_entries < 0)
+		return nr_entries;
+
+	/* Check hotplug MSIx entry is in range */
+	pcie_capability_read_word(pdev, PCI_EXP_FLAGS, &pcie_flag);
+	entry.entry = (pcie_flag & PCI_EXP_FLAGS_IRQ) >> 9;
+	if (entry.entry >= nr_entries)
+		return -ERANGE;
+
+	/* Enable MSIx */
+	ret = pci_enable_msix_exact(pdev, &entry, 1);
+	if (ret) {
+		dev_warn(&pdev->dev, "Error %d enabling MSIx\n", ret);
+		return ret;
+	}
+
+	return entry.vector;
+}
+
+static void pnv_php_event_handler(struct work_struct *work)
+{
+	struct pnv_php_event *event =
+		container_of(work, struct pnv_php_event, work);
+	struct pnv_php_slot *php_slot = event->php_slot;
+
+	if (event->added)
+		pnv_php_enable_slot(&php_slot->slot);
+	else
+		pnv_php_disable_slot(&php_slot->slot);
+
+	kfree(event);
+}
+
+static irqreturn_t pnv_php_interrupt(int irq, void *data)
+{
+	struct pnv_php_slot *php_slot = data;
+	struct pci_dev *pchild, *pdev = php_slot->pdev;
+	struct eeh_dev *edev;
+	struct eeh_pe *pe;
+	struct pnv_php_event *event;
+	u16 sts, lsts;
+	u8 presence;
+	bool added;
+	unsigned long flags;
+	int ret;
+
+	pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &sts);
+	sts &= (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC);
+	pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, sts);
+	if (sts & PCI_EXP_SLTSTA_DLLSC) {
+		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lsts);
+		added = !!(lsts & PCI_EXP_LNKSTA_DLLLA);
+	} else if (sts & PCI_EXP_SLTSTA_PDC) {
+		ret = pnv_pci_get_presence_state(php_slot->id, &presence);
+		if (!ret)
+			return IRQ_HANDLED;
+		added = !!(presence == OPAL_PCI_SLOT_PRESENT);
+	} else {
+		return IRQ_NONE;
+	}
+
+	/* Freeze the removed PE to avoid unexpected error reporting */
+	if (!added) {
+		pchild = list_first_entry_or_null(&php_slot->bus->devices,
+						  struct pci_dev, bus_list);
+		edev = pchild ? pci_dev_to_eeh_dev(pchild) : NULL;
+		pe = edev ? edev->pe : NULL;
+		if (pe) {
+			eeh_serialize_lock(&flags);
+			eeh_pe_state_mark(pe, EEH_PE_ISOLATED);
+			eeh_serialize_unlock(flags);
+			eeh_pe_set_option(pe, EEH_OPT_FREEZE_PE);
+		}
+	}
+
+	/*
+	 * The PE is left in frozen state if the event is missed. It's
+	 * fine as the PCI devices (PE) aren't functional any more.
+	 */
+	event = kzalloc(sizeof(*event), GFP_ATOMIC);
+	if (!event) {
+		dev_warn(&pdev->dev, "PCI slot [%s] missed hotplug event 0x%04x\n",
+			 php_slot->name, sts);
+		return IRQ_HANDLED;
+	}
+
+	dev_info(&pdev->dev, "PCI slot [%s] %s (IRQ: %d)\n",
+		 php_slot->name, added ? "added" : "removed", irq);
+	INIT_WORK(&event->work, pnv_php_event_handler);
+	event->added = added;
+	event->php_slot = php_slot;
+	queue_work(php_slot->wq, &event->work);
+
+	return IRQ_HANDLED;
+}
+
+static void pnv_php_init_irq(struct pnv_php_slot *php_slot, int irq)
+{
+	struct pci_dev *pdev = php_slot->pdev;
+	u16 sts, ctrl;
+	int ret;
+
+	/* Allocate workqueue */
+	php_slot->wq = alloc_workqueue("pciehp-%s", 0, 0, php_slot->name);
+	if (!php_slot->wq) {
+		dev_warn(&pdev->dev, "Cannot alloc workqueue\n");
+		pnv_php_disable_irq(php_slot);
+		return;
+	}
+
+	/* Clear pending interrupts */
+	pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &sts);
+	sts |= (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC);
+	pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, sts);
+
+	/* Request the interrupt */
+	ret = request_irq(irq, pnv_php_interrupt, IRQF_SHARED,
+			  php_slot->name, php_slot);
+	if (ret) {
+		pnv_php_disable_irq(php_slot);
+		dev_warn(&pdev->dev, "Error %d enabling IRQ %d\n", ret, irq);
+		return;
+	}
+
+	/* Enable the interrupts */
+	pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &ctrl);
+	ctrl |= (PCI_EXP_SLTCTL_HPIE |
+		 PCI_EXP_SLTCTL_PDCE |
+		 PCI_EXP_SLTCTL_DLLSCE);
+	pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, ctrl);
+
+	/* The interrupt is initialized successfully when @irq is valid */
+	php_slot->irq = irq;
+}
+
+static void pnv_php_enable_irq(struct pnv_php_slot *php_slot)
+{
+	struct pci_dev *pdev = php_slot->pdev;
+	int irq, ret;
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		dev_warn(&pdev->dev, "Error %d enabling device\n", ret);
+		return;
+	}
+
+	pci_set_master(pdev);
+
+	/* Enable MSIx interrupt */
+	irq = pnv_php_enable_msix(php_slot);
+	if (irq > 0) {
+		pnv_php_init_irq(php_slot, irq);
+		return;
+	}
+
+	/*
+	 * Use MSI if MSIx doesn't work. Fail back to legacy INTx
+	 * if MSI doesn't work either
+	 */
+	ret = pci_enable_msi(pdev);
+	if (!ret || pdev->irq) {
+		irq = pdev->irq;
+		pnv_php_init_irq(php_slot, irq);
+	}
+}
+
 static int pnv_php_register_one(struct device_node *dn)
 {
 	struct pnv_php_slot *php_slot;
-	const __be32 *prop32;
+	u32 prop32;
 	int ret;
 
 	/* Check if it's hotpluggable slot */
-	prop32 = of_get_property(dn, "ibm,slot-pluggable", NULL);
-	if (!prop32 || !of_read_number(prop32, 1))
+	ret = of_property_read_u32(dn, "ibm,slot-pluggable", &prop32);
+	if (ret || !prop32)
 		return -ENXIO;
 
-	prop32 = of_get_property(dn, "ibm,reset-by-firmware", NULL);
-	if (!prop32 || !of_read_number(prop32, 1))
+	ret = of_property_read_u32(dn, "ibm,reset-by-firmware", &prop32);
+	if (ret || !prop32)
 		return -ENXIO;
 
 	php_slot = pnv_php_alloc_slot(dn);
-	if (unlikely(!php_slot))
+	if (!php_slot)
 		return -ENODEV;
 
 	ret = pnv_php_register_slot(php_slot);
-	if (unlikely(ret))
+	if (ret)
 		goto free_slot;
 
 	ret = pnv_php_enable(php_slot, false);
-	if (unlikely(ret))
+	if (ret)
 		goto unregister_slot;
 
+	/* Enable interrupt if the slot supports surprise hotplug */
+	ret = of_property_read_u32(dn, "ibm,slot-surprise-pluggable", &prop32);
+	if (!ret && prop32)
+		pnv_php_enable_irq(php_slot);
+
 	return 0;
 
 unregister_slot:
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 2194b447201d..e30f05c8517f 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -136,7 +136,10 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
 	virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
 	virtfn->vendor = dev->vendor;
 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
-	pci_setup_device(virtfn);
+	rc = pci_setup_device(virtfn);
+	if (rc)
+		goto failed0;
+
 	virtfn->dev.parent = dev->dev.parent;
 	virtfn->physfn = pci_dev_get(dev);
 	virtfn->is_virtfn = 1;
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index a02981efdad5..bfdd0744b686 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -19,6 +19,7 @@
 #include <linux/smp.h>
 #include <linux/errno.h>
 #include <linux/io.h>
+#include <linux/acpi_iort.h>
 #include <linux/slab.h>
 #include <linux/irqdomain.h>
 #include <linux/of_irq.h>
@@ -549,15 +550,23 @@ error_attrs:
 	return ret;
 }
 
-static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec)
+static struct msi_desc *
+msi_setup_entry(struct pci_dev *dev, int nvec, bool affinity)
 {
-	u16 control;
+	struct cpumask *masks = NULL;
 	struct msi_desc *entry;
+	u16 control;
+
+	if (affinity) {
+		masks = irq_create_affinity_masks(dev->irq_affinity, nvec);
+		if (!masks)
+			pr_err("Unable to allocate affinity masks, ignoring\n");
+	}
 
 	/* MSI Entry Initialization */
-	entry = alloc_msi_entry(&dev->dev);
+	entry = alloc_msi_entry(&dev->dev, nvec, masks);
 	if (!entry)
-		return NULL;
+		goto out;
 
 	pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
 
@@ -568,8 +577,6 @@ static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec)
 	entry->msi_attrib.default_irq	= dev->irq;	/* Save IOAPIC IRQ */
 	entry->msi_attrib.multi_cap	= (control & PCI_MSI_FLAGS_QMASK) >> 1;
 	entry->msi_attrib.multiple	= ilog2(__roundup_pow_of_two(nvec));
-	entry->nvec_used		= nvec;
-	entry->affinity			= dev->irq_affinity;
 
 	if (control & PCI_MSI_FLAGS_64BIT)
 		entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64;
@@ -580,6 +587,8 @@ static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec)
 	if (entry->msi_attrib.maskbit)
 		pci_read_config_dword(dev, entry->mask_pos, &entry->masked);
 
+out:
+	kfree(masks);
 	return entry;
 }
 
@@ -608,7 +617,7 @@ static int msi_verify_entries(struct pci_dev *dev)
  * an error, and a positive return value indicates the number of interrupts
  * which could have been allocated.
  */
-static int msi_capability_init(struct pci_dev *dev, int nvec)
+static int msi_capability_init(struct pci_dev *dev, int nvec, bool affinity)
 {
 	struct msi_desc *entry;
 	int ret;
@@ -616,7 +625,7 @@ static int msi_capability_init(struct pci_dev *dev, int nvec)
 
 	pci_msi_set_enable(dev, 0);	/* Disable MSI during set up */
 
-	entry = msi_setup_entry(dev, nvec);
+	entry = msi_setup_entry(dev, nvec, affinity);
 	if (!entry)
 		return -ENOMEM;
 
@@ -679,28 +688,29 @@ static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries)
 }
 
 static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
-			      struct msix_entry *entries, int nvec)
+			      struct msix_entry *entries, int nvec,
+			      bool affinity)
 {
-	const struct cpumask *mask = NULL;
+	struct cpumask *curmsk, *masks = NULL;
 	struct msi_desc *entry;
-	int cpu = -1, i;
-
-	for (i = 0; i < nvec; i++) {
-		if (dev->irq_affinity) {
-			cpu = cpumask_next(cpu, dev->irq_affinity);
-			if (cpu >= nr_cpu_ids)
-				cpu = cpumask_first(dev->irq_affinity);
-			mask = cpumask_of(cpu);
-		}
+	int ret, i;
+
+	if (affinity) {
+		masks = irq_create_affinity_masks(dev->irq_affinity, nvec);
+		if (!masks)
+			pr_err("Unable to allocate affinity masks, ignoring\n");
+	}
 
-		entry = alloc_msi_entry(&dev->dev);
+	for (i = 0, curmsk = masks; i < nvec; i++) {
+		entry = alloc_msi_entry(&dev->dev, 1, curmsk);
 		if (!entry) {
 			if (!i)
 				iounmap(base);
 			else
 				free_msi_irqs(dev);
 			/* No enough memory. Don't try again */
-			return -ENOMEM;
+			ret = -ENOMEM;
+			goto out;
 		}
 
 		entry->msi_attrib.is_msix	= 1;
@@ -711,12 +721,14 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
 			entry->msi_attrib.entry_nr = i;
 		entry->msi_attrib.default_irq	= dev->irq;
 		entry->mask_base		= base;
-		entry->nvec_used		= 1;
-		entry->affinity			= mask;
 
 		list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
+		if (masks)
+			curmsk++;
 	}
-
+	ret = 0;
+out:
+	kfree(masks);
 	return 0;
 }
 
@@ -745,8 +757,8 @@ static void msix_program_entries(struct pci_dev *dev,
  * single MSI-X irq. A return of zero indicates the successful setup of
  * requested MSI-X entries with allocated irqs or non-zero for otherwise.
  **/
-static int msix_capability_init(struct pci_dev *dev,
-				struct msix_entry *entries, int nvec)
+static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
+				int nvec, bool affinity)
 {
 	int ret;
 	u16 control;
@@ -761,7 +773,7 @@ static int msix_capability_init(struct pci_dev *dev,
 	if (!base)
 		return -ENOMEM;
 
-	ret = msix_setup_entries(dev, base, entries, nvec);
+	ret = msix_setup_entries(dev, base, entries, nvec, affinity);
 	if (ret)
 		return ret;
 
@@ -941,22 +953,8 @@ int pci_msix_vec_count(struct pci_dev *dev)
 }
 EXPORT_SYMBOL(pci_msix_vec_count);
 
-/**
- * pci_enable_msix - configure device's MSI-X capability structure
- * @dev: pointer to the pci_dev data structure of MSI-X device function
- * @entries: pointer to an array of MSI-X entries (optional)
- * @nvec: number of MSI-X irqs requested for allocation by device driver
- *
- * Setup the MSI-X capability structure of device function with the number
- * of requested irqs upon its software driver call to request for
- * MSI-X mode enabled on its hardware device function. A return of zero
- * indicates the successful configuration of MSI-X capability structure
- * with new allocated MSI-X irqs. A return of < 0 indicates a failure.
- * Or a return of > 0 indicates that driver request is exceeding the number
- * of irqs or MSI-X vectors available. Driver should use the returned value to
- * re-send its request.
- **/
-int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec)
+static int __pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries,
+			     int nvec, bool affinity)
 {
 	int nr_entries;
 	int i, j;
@@ -988,7 +986,27 @@ int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec)
 		dev_info(&dev->dev, "can't enable MSI-X (MSI IRQ already assigned)\n");
 		return -EINVAL;
 	}
-	return msix_capability_init(dev, entries, nvec);
+	return msix_capability_init(dev, entries, nvec, affinity);
+}
+
+/**
+ * pci_enable_msix - configure device's MSI-X capability structure
+ * @dev: pointer to the pci_dev data structure of MSI-X device function
+ * @entries: pointer to an array of MSI-X entries (optional)
+ * @nvec: number of MSI-X irqs requested for allocation by device driver
+ *
+ * Setup the MSI-X capability structure of device function with the number
+ * of requested irqs upon its software driver call to request for
+ * MSI-X mode enabled on its hardware device function. A return of zero
+ * indicates the successful configuration of MSI-X capability structure
+ * with new allocated MSI-X irqs. A return of < 0 indicates a failure.
+ * Or a return of > 0 indicates that driver request is exceeding the number
+ * of irqs or MSI-X vectors available. Driver should use the returned value to
+ * re-send its request.
+ **/
+int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec)
+{
+	return __pci_enable_msix(dev, entries, nvec, false);
 }
 EXPORT_SYMBOL(pci_enable_msix);
 
@@ -1041,6 +1059,7 @@ EXPORT_SYMBOL(pci_msi_enabled);
 static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
 		unsigned int flags)
 {
+	bool affinity = flags & PCI_IRQ_AFFINITY;
 	int nvec;
 	int rc;
 
@@ -1069,19 +1088,17 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
 		nvec = maxvec;
 
 	for (;;) {
-		if (!(flags & PCI_IRQ_NOAFFINITY)) {
-			dev->irq_affinity = irq_create_affinity_mask(&nvec);
+		if (affinity) {
+			nvec = irq_calc_affinity_vectors(dev->irq_affinity,
+					nvec);
 			if (nvec < minvec)
 				return -ENOSPC;
 		}
 
-		rc = msi_capability_init(dev, nvec);
+		rc = msi_capability_init(dev, nvec, affinity);
 		if (rc == 0)
 			return nvec;
 
-		kfree(dev->irq_affinity);
-		dev->irq_affinity = NULL;
-
 		if (rc < 0)
 			return rc;
 		if (rc < minvec)
@@ -1105,7 +1122,7 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
  **/
 int pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec)
 {
-	return __pci_enable_msi_range(dev, minvec, maxvec, PCI_IRQ_NOAFFINITY);
+	return __pci_enable_msi_range(dev, minvec, maxvec, 0);
 }
 EXPORT_SYMBOL(pci_enable_msi_range);
 
@@ -1113,26 +1130,24 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
 		struct msix_entry *entries, int minvec, int maxvec,
 		unsigned int flags)
 {
-	int nvec = maxvec;
-	int rc;
+	bool affinity = flags & PCI_IRQ_AFFINITY;
+	int rc, nvec = maxvec;
 
 	if (maxvec < minvec)
 		return -ERANGE;
 
 	for (;;) {
-		if (!(flags & PCI_IRQ_NOAFFINITY)) {
-			dev->irq_affinity = irq_create_affinity_mask(&nvec);
+		if (affinity) {
+			nvec = irq_calc_affinity_vectors(dev->irq_affinity,
+					nvec);
 			if (nvec < minvec)
 				return -ENOSPC;
 		}
 
-		rc = pci_enable_msix(dev, entries, nvec);
+		rc = __pci_enable_msix(dev, entries, nvec, affinity);
 		if (rc == 0)
 			return nvec;
 
-		kfree(dev->irq_affinity);
-		dev->irq_affinity = NULL;
-
 		if (rc < 0)
 			return rc;
 		if (rc < minvec)
@@ -1160,8 +1175,7 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
 int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
 		int minvec, int maxvec)
 {
-	return __pci_enable_msix_range(dev, entries, minvec, maxvec,
-			PCI_IRQ_NOAFFINITY);
+	return __pci_enable_msix_range(dev, entries, minvec, maxvec, 0);
 }
 EXPORT_SYMBOL(pci_enable_msix_range);
 
@@ -1187,22 +1201,25 @@ int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs,
 {
 	int vecs = -ENOSPC;
 
-	if (!(flags & PCI_IRQ_NOMSIX)) {
+	if (flags & PCI_IRQ_MSIX) {
 		vecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs,
 				flags);
 		if (vecs > 0)
 			return vecs;
 	}
 
-	if (!(flags & PCI_IRQ_NOMSI)) {
+	if (flags & PCI_IRQ_MSI) {
 		vecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, flags);
 		if (vecs > 0)
 			return vecs;
 	}
 
 	/* use legacy irq if allowed */
-	if (!(flags & PCI_IRQ_NOLEGACY) && min_vecs == 1)
+	if ((flags & PCI_IRQ_LEGACY) && min_vecs == 1) {
+		pci_intx(dev, 1);
 		return 1;
+	}
+
 	return vecs;
 }
 EXPORT_SYMBOL(pci_alloc_irq_vectors);
@@ -1254,6 +1271,37 @@ int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
 }
 EXPORT_SYMBOL(pci_irq_vector);
 
+/**
+ * pci_irq_get_affinity - return the affinity of a particular msi vector
+ * @dev:	PCI device to operate on
+ * @nr:		device-relative interrupt vector index (0-based).
+ */
+const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
+{
+	if (dev->msix_enabled) {
+		struct msi_desc *entry;
+		int i = 0;
+
+		for_each_pci_msi_entry(entry, dev) {
+			if (i == nr)
+				return entry->affinity;
+			i++;
+		}
+		WARN_ON_ONCE(1);
+		return NULL;
+	} else if (dev->msi_enabled) {
+		struct msi_desc *entry = first_pci_msi_entry(dev);
+
+		if (WARN_ON_ONCE(!entry || nr >= entry->nvec_used))
+			return NULL;
+
+		return &entry->affinity[nr];
+	} else {
+		return cpu_possible_mask;
+	}
+}
+EXPORT_SYMBOL(pci_irq_get_affinity);
+
 struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc)
 {
 	return to_pci_dev(desc->dev);
@@ -1411,6 +1459,8 @@ struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
 	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
 		pci_msi_domain_update_chip_ops(info);
 
+	info->flags |= MSI_FLAG_ACTIVATE_EARLY;
+
 	domain = msi_create_irq_domain(fwnode, info, parent);
 	if (!domain)
 		return NULL;
@@ -1498,8 +1548,8 @@ u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
 	pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
 
 	of_node = irq_domain_get_of_node(domain);
-	if (of_node)
-		rid = of_msi_map_rid(&pdev->dev, of_node, rid);
+	rid = of_node ? of_msi_map_rid(&pdev->dev, of_node, rid) :
+			iort_msi_map_rid(&pdev->dev, rid);
 
 	return rid;
 }
@@ -1515,9 +1565,13 @@ u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
  */
 struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
 {
+	struct irq_domain *dom;
 	u32 rid = 0;
 
 	pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
-	return of_msi_map_get_device_domain(&pdev->dev, rid);
+	dom = of_msi_map_get_device_domain(&pdev->dev, rid);
+	if (!dom)
+		dom = iort_get_device_domain(&pdev->dev, rid);
+	return dom;
 }
 #endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */
diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c
index 9a033e8ee9a4..d966d47c9e80 100644
--- a/drivers/pci/pci-acpi.c
+++ b/drivers/pci/pci-acpi.c
@@ -452,6 +452,27 @@ static int acpi_pci_set_power_state(struct pci_dev *dev, pci_power_t state)
 	return error;
 }
 
+static pci_power_t acpi_pci_get_power_state(struct pci_dev *dev)
+{
+	struct acpi_device *adev = ACPI_COMPANION(&dev->dev);
+	static const pci_power_t state_conv[] = {
+		[ACPI_STATE_D0]      = PCI_D0,
+		[ACPI_STATE_D1]      = PCI_D1,
+		[ACPI_STATE_D2]      = PCI_D2,
+		[ACPI_STATE_D3_HOT]  = PCI_D3hot,
+		[ACPI_STATE_D3_COLD] = PCI_D3cold,
+	};
+	int state;
+
+	if (!adev || !acpi_device_power_manageable(adev))
+		return PCI_UNKNOWN;
+
+	if (acpi_device_get_power(adev, &state) || state == ACPI_STATE_UNKNOWN)
+		return PCI_UNKNOWN;
+
+	return state_conv[state];
+}
+
 static bool acpi_pci_can_wakeup(struct pci_dev *dev)
 {
 	struct acpi_device *adev = ACPI_COMPANION(&dev->dev);
@@ -534,6 +555,7 @@ static bool acpi_pci_need_resume(struct pci_dev *dev)
 static const struct pci_platform_pm_ops acpi_pci_platform_pm = {
 	.is_manageable = acpi_pci_power_manageable,
 	.set_state = acpi_pci_set_power_state,
+	.get_state = acpi_pci_get_power_state,
 	.choose_state = acpi_pci_choose_state,
 	.sleep_wake = acpi_pci_sleep_wake,
 	.run_wake = acpi_pci_run_wake,
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c
index e39a67c8ef39..1ccce1cd6aca 100644
--- a/drivers/pci/pci-driver.c
+++ b/drivers/pci/pci-driver.c
@@ -466,7 +466,6 @@ static void pci_device_shutdown(struct device *dev)
 	pci_msi_shutdown(pci_dev);
 	pci_msix_shutdown(pci_dev);
 
-#ifdef CONFIG_KEXEC_CORE
 	/*
 	 * If this is a kexec reboot, turn off Bus Master bit on the
 	 * device to tell it to not continue to do DMA. Don't touch
@@ -476,7 +475,6 @@ static void pci_device_shutdown(struct device *dev)
 	 */
 	if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
 		pci_clear_master(pci_dev);
-#endif
 }
 
 #ifdef CONFIG_PM
@@ -684,8 +682,19 @@ static int pci_pm_prepare(struct device *dev)
 
 static void pci_pm_complete(struct device *dev)
 {
-	pci_dev_complete_resume(to_pci_dev(dev));
-	pm_complete_with_resume_check(dev);
+	struct pci_dev *pci_dev = to_pci_dev(dev);
+
+	pci_dev_complete_resume(pci_dev);
+	pm_generic_complete(dev);
+
+	/* Resume device if platform firmware has put it in reset-power-on */
+	if (dev->power.direct_complete && pm_resume_via_firmware()) {
+		pci_power_t pre_sleep_state = pci_dev->current_state;
+
+		pci_update_current_state(pci_dev, pci_dev->current_state);
+		if (pci_dev->current_state < pre_sleep_state)
+			pm_request_resume(dev);
+	}
 }
 
 #else /* !CONFIG_PM_SLEEP */
diff --git a/drivers/pci/pci-mid.c b/drivers/pci/pci-mid.c
index c878aa71173b..55f453de562e 100644
--- a/drivers/pci/pci-mid.c
+++ b/drivers/pci/pci-mid.c
@@ -60,8 +60,13 @@ static struct pci_platform_pm_ops mid_pci_platform_pm = {
 
 #define ICPU(model)	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, }
 
+/*
+ * This table should be in sync with the one in
+ * arch/x86/platform/intel-mid/pwr.c.
+ */
 static const struct x86_cpu_id lpss_cpu_ids[] = {
-	ICPU(INTEL_FAM6_ATOM_MERRIFIELD1),
+	ICPU(INTEL_FAM6_ATOM_PENWELL),
+	ICPU(INTEL_FAM6_ATOM_MERRIFIELD),
 	{}
 };
 
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index aab9d5115a5f..ba34907538f6 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -480,6 +480,30 @@ struct resource *pci_find_parent_resource(const struct pci_dev *dev,
 EXPORT_SYMBOL(pci_find_parent_resource);
 
 /**
+ * pci_find_resource - Return matching PCI device resource
+ * @dev: PCI device to query
+ * @res: Resource to look for
+ *
+ * Goes over standard PCI resources (BARs) and checks if the given resource
+ * is partially or fully contained in any of them. In that case the
+ * matching resource is returned, %NULL otherwise.
+ */
+struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
+{
+	int i;
+
+	for (i = 0; i < PCI_ROM_RESOURCE; i++) {
+		struct resource *r = &dev->resource[i];
+
+		if (r->start && resource_contains(r, res))
+			return r;
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL(pci_find_resource);
+
+/**
  * pci_find_pcie_root_port - return PCIe Root Port
  * @dev: PCI device to query
  *
@@ -552,8 +576,9 @@ static const struct pci_platform_pm_ops *pci_platform_pm;
 
 int pci_set_platform_pm(const struct pci_platform_pm_ops *ops)
 {
-	if (!ops->is_manageable || !ops->set_state || !ops->choose_state ||
-	    !ops->sleep_wake || !ops->run_wake || !ops->need_resume)
+	if (!ops->is_manageable || !ops->set_state  || !ops->get_state ||
+	    !ops->choose_state  || !ops->sleep_wake || !ops->run_wake  ||
+	    !ops->need_resume)
 		return -EINVAL;
 	pci_platform_pm = ops;
 	return 0;
@@ -570,6 +595,11 @@ static inline int platform_pci_set_power_state(struct pci_dev *dev,
 	return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS;
 }
 
+static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
+{
+	return pci_platform_pm ? pci_platform_pm->get_state(dev) : PCI_UNKNOWN;
+}
+
 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
 {
 	return pci_platform_pm ?
@@ -701,26 +731,25 @@ static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state)
 }
 
 /**
- * pci_update_current_state - Read PCI power state of given device from its
- *                            PCI PM registers and cache it
+ * pci_update_current_state - Read power state of given device and cache it
  * @dev: PCI device to handle.
  * @state: State to cache in case the device doesn't have the PM capability
+ *
+ * The power state is read from the PMCSR register, which however is
+ * inaccessible in D3cold.  The platform firmware is therefore queried first
+ * to detect accessibility of the register.  In case the platform firmware
+ * reports an incorrect state or the device isn't power manageable by the
+ * platform at all, we try to detect D3cold by testing accessibility of the
+ * vendor ID in config space.
  */
 void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
 {
-	if (dev->pm_cap) {
+	if (platform_pci_get_power_state(dev) == PCI_D3cold ||
+	    !pci_device_is_present(dev)) {
+		dev->current_state = PCI_D3cold;
+	} else if (dev->pm_cap) {
 		u16 pmcsr;
 
-		/*
-		 * Configuration space is not accessible for device in
-		 * D3cold, so just keep or set D3cold for safety
-		 */
-		if (dev->current_state == PCI_D3cold)
-			return;
-		if (state == PCI_D3cold) {
-			dev->current_state = PCI_D3cold;
-			return;
-		}
 		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
 		dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
 	} else {
@@ -1959,9 +1988,22 @@ static pci_power_t pci_target_state(struct pci_dev *dev)
 		default:
 			target_state = state;
 		}
-	} else if (!dev->pm_cap) {
+
+		return target_state;
+	}
+
+	if (!dev->pm_cap)
 		target_state = PCI_D0;
-	} else if (device_may_wakeup(&dev->dev)) {
+
+	/*
+	 * If the device is in D3cold even though it's not power-manageable by
+	 * the platform, it may have been powered down by non-standard means.
+	 * Best to let it slumber.
+	 */
+	if (dev->current_state == PCI_D3cold)
+		target_state = PCI_D3cold;
+
+	if (device_may_wakeup(&dev->dev)) {
 		/*
 		 * Find the deepest state from which the device can generate
 		 * wake-up events, make it the target state and enable device
@@ -4959,6 +5001,13 @@ static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev)
 
 	spin_lock(&resource_alignment_lock);
 	p = resource_alignment_param;
+	if (!*p)
+		goto out;
+	if (pci_has_flag(PCI_PROBE_ONLY)) {
+		pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n");
+		goto out;
+	}
+
 	while (*p) {
 		count = 0;
 		if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
@@ -5023,6 +5072,7 @@ static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev)
 		}
 		p++;
 	}
+out:
 	spin_unlock(&resource_alignment_lock);
 	return align;
 }
@@ -5041,6 +5091,15 @@ void pci_reassigndev_resource_alignment(struct pci_dev *dev)
 	resource_size_t align, size;
 	u16 command;
 
+	/*
+	 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec
+	 * 3.4.1.11.  Their resources are allocated from the space
+	 * described by the VF BARx register in the PF's SR-IOV capability.
+	 * We can't influence their alignment here.
+	 */
+	if (dev->is_virtfn)
+		return;
+
 	/* check if specified PCI is target device to reassign */
 	align = pci_specified_resource_alignment(dev);
 	if (!align)
@@ -5063,6 +5122,12 @@ void pci_reassigndev_resource_alignment(struct pci_dev *dev)
 		r = &dev->resource[i];
 		if (!(r->flags & IORESOURCE_MEM))
 			continue;
+		if (r->flags & IORESOURCE_PCI_FIXED) {
+			dev_info(&dev->dev, "Ignoring requested alignment for BAR%d: %pR\n",
+				i, r);
+			continue;
+		}
+
 		size = resource_size(r);
 		if (size < align) {
 			size = align;
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 9730c474b016..451856210e18 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -42,6 +42,8 @@ int pci_probe_reset_function(struct pci_dev *dev);
  *
  * @set_state: invokes the platform firmware to set the device's power state
  *
+ * @get_state: queries the platform firmware for a device's current power state
+ *
  * @choose_state: returns PCI power state of given device preferred by the
  *                platform; to be used during system-wide transitions from a
  *                sleeping state to the working state and vice versa
@@ -62,6 +64,7 @@ int pci_probe_reset_function(struct pci_dev *dev);
 struct pci_platform_pm_ops {
 	bool (*is_manageable)(struct pci_dev *dev);
 	int (*set_state)(struct pci_dev *dev, pci_power_t state);
+	pci_power_t (*get_state)(struct pci_dev *dev);
 	pci_power_t (*choose_state)(struct pci_dev *dev);
 	int (*sleep_wake)(struct pci_dev *dev, bool enable);
 	int (*run_wake)(struct pci_dev *dev, bool enable);
@@ -332,6 +335,12 @@ static inline resource_size_t pci_resource_alignment(struct pci_dev *dev,
 
 void pci_enable_acs(struct pci_dev *dev);
 
+#ifdef CONFIG_PCIE_PTM
+void pci_ptm_init(struct pci_dev *dev);
+#else
+static inline void pci_ptm_init(struct pci_dev *dev) { }
+#endif
+
 struct pci_dev_reset_methods {
 	u16 vendor;
 	u16 device;
diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig
index 7fcea75afa4c..7ce77635e5ad 100644
--- a/drivers/pci/pcie/Kconfig
+++ b/drivers/pci/pcie/Kconfig
@@ -92,3 +92,14 @@ config PCIE_DPC
 	  will be handled by the DPC driver.  If your system doesn't
 	  have this capability or you do not want to use this feature,
 	  it is safe to answer N.
+
+config PCIE_PTM
+	bool "PCIe Precision Time Measurement support"
+	default n
+	depends on PCIEPORTBUS
+	help
+	  This enables PCI Express Precision Time Measurement (PTM)
+	  support.
+
+	  This is only useful if you have devices that support PTM, but it
+	  is safe to enable even if you don't.
diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile
index b24525b3dec1..36e35ea8fde7 100644
--- a/drivers/pci/pcie/Makefile
+++ b/drivers/pci/pcie/Makefile
@@ -16,3 +16,4 @@ obj-$(CONFIG_PCIEAER)		+= aer/
 obj-$(CONFIG_PCIE_PME) += pme.o
 
 obj-$(CONFIG_PCIE_DPC) += pcie-dpc.o
+obj-$(CONFIG_PCIE_PTM) += ptm.o
diff --git a/drivers/pci/pcie/aer/aerdrv.c b/drivers/pci/pcie/aer/aerdrv.c
index 48d21e0edd56..139150b2bdfd 100644
--- a/drivers/pci/pcie/aer/aerdrv.c
+++ b/drivers/pci/pcie/aer/aerdrv.c
@@ -15,7 +15,6 @@
  *
  */
 
-#include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/pci-acpi.h>
 #include <linux/sched.h>
@@ -37,9 +36,6 @@
 #define DRIVER_VERSION "v1.0"
 #define DRIVER_AUTHOR "tom.l.nguyen@intel.com"
 #define DRIVER_DESC "Root Port Advanced Error Reporting Driver"
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL");
 
 static int aer_probe(struct pcie_device *dev);
 static void aer_remove(struct pcie_device *dev);
@@ -70,7 +66,7 @@ static int pcie_aer_disable;
 
 void pci_no_aer(void)
 {
-	pcie_aer_disable = 1;	/* has priority over 'forceload' */
+	pcie_aer_disable = 1;
 }
 
 bool pci_aer_available(void)
@@ -134,7 +130,7 @@ static void aer_enable_rootport(struct aer_rpc *rpc)
 	pcie_capability_clear_word(pdev, PCI_EXP_RTCTL,
 				   SYSTEM_ERROR_INTR_ON_MESG_MASK);
 
-	aer_pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
+	aer_pos = pdev->aer_cap;
 	/* Clear error status */
 	pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, &reg32);
 	pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32);
@@ -173,7 +169,7 @@ static void aer_disable_rootport(struct aer_rpc *rpc)
 	 */
 	set_downstream_devices_error_reporting(pdev, false);
 
-	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
+	pos = pdev->aer_cap;
 	/* Disable Root's interrupt in response to error messages */
 	pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, &reg32);
 	reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;
@@ -200,7 +196,7 @@ irqreturn_t aer_irq(int irq, void *context)
 	unsigned long flags;
 	int pos;
 
-	pos = pci_find_ext_capability(pdev->port, PCI_EXT_CAP_ID_ERR);
+	pos = pdev->port->aer_cap;
 	/*
 	 * Must lock access to Root Error Status Reg, Root Error ID Reg,
 	 * and Root error producer/consumer index
@@ -294,7 +290,6 @@ static void aer_remove(struct pcie_device *dev)
 /**
  * aer_probe - initialize resources
  * @dev: pointer to the pcie_dev data structure
- * @id: pointer to the service id data structure
  *
  * Invoked when PCI Express bus loads AER service driver.
  */
@@ -304,11 +299,6 @@ static int aer_probe(struct pcie_device *dev)
 	struct aer_rpc *rpc;
 	struct device *device = &dev->device;
 
-	/* Init */
-	status = aer_init(dev);
-	if (status)
-		return status;
-
 	/* Alloc rpc data structure */
 	rpc = aer_alloc_rpc(dev);
 	if (!rpc) {
@@ -343,7 +333,7 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev)
 	u32 reg32;
 	int pos;
 
-	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	pos = dev->aer_cap;
 
 	/* Disable Root's interrupt in response to error messages */
 	pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, &reg32);
@@ -396,7 +386,7 @@ static void aer_error_resume(struct pci_dev *dev)
 	pcie_capability_write_word(dev, PCI_EXP_DEVSTA, reg16);
 
 	/* Clean AER Root Error Status */
-	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	pos = dev->aer_cap;
 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 	if (dev->error_state == pci_channel_io_normal)
@@ -417,16 +407,4 @@ static int __init aer_service_init(void)
 		return -ENXIO;
 	return pcie_port_service_register(&aerdriver);
 }
-
-/**
- * aer_service_exit - unregister AER root service driver
- *
- * Invoked when AER root service driver is unloaded.
- */
-static void __exit aer_service_exit(void)
-{
-	pcie_port_service_unregister(&aerdriver);
-}
-
-module_init(aer_service_init);
-module_exit(aer_service_exit);
+device_initcall(aer_service_init);
diff --git a/drivers/pci/pcie/aer/aerdrv.h b/drivers/pci/pcie/aer/aerdrv.h
index 945c939a86c5..d51e4a57b190 100644
--- a/drivers/pci/pcie/aer/aerdrv.h
+++ b/drivers/pci/pcie/aer/aerdrv.h
@@ -60,6 +60,7 @@ struct aer_rpc {
 	struct pcie_device *rpd;	/* Root Port device */
 	struct work_struct dpc_handler;
 	struct aer_err_source e_sources[AER_ERROR_SOURCES_MAX];
+	struct aer_err_info e_info;
 	unsigned short prod_idx;	/* Error Producer Index */
 	unsigned short cons_idx;	/* Error Consumer Index */
 	int isr;
@@ -105,7 +106,6 @@ static inline pci_ers_result_t merge_result(enum pci_ers_result orig,
 }
 
 extern struct bus_type pcie_port_bus_type;
-int aer_init(struct pcie_device *dev);
 void aer_isr(struct work_struct *work);
 void aer_print_error(struct pci_dev *dev, struct aer_err_info *info);
 void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info);
@@ -121,11 +121,4 @@ static inline int pcie_aer_get_firmware_first(struct pci_dev *pci_dev)
 	return 0;
 }
 #endif
-
-static inline void pcie_aer_force_firmware_first(struct pci_dev *pci_dev,
-						 int enable)
-{
-	pci_dev->__aer_firmware_first = !!enable;
-	pci_dev->__aer_firmware_first_valid = 1;
-}
 #endif /* _AERDRV_H_ */
diff --git a/drivers/pci/pcie/aer/aerdrv_core.c b/drivers/pci/pcie/aer/aerdrv_core.c
index 521e39c1b66d..b1303b32053f 100644
--- a/drivers/pci/pcie/aer/aerdrv_core.c
+++ b/drivers/pci/pcie/aer/aerdrv_core.c
@@ -27,11 +27,6 @@
 #include <linux/kfifo.h>
 #include "aerdrv.h"
 
-static bool forceload;
-static bool nosourceid;
-module_param(forceload, bool, 0);
-module_param(nosourceid, bool, 0);
-
 #define	PCI_EXP_AER_FLAGS	(PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \
 				 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE)
 
@@ -40,7 +35,7 @@ int pci_enable_pcie_error_reporting(struct pci_dev *dev)
 	if (pcie_aer_get_firmware_first(dev))
 		return -EIO;
 
-	if (!pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR))
+	if (!dev->aer_cap)
 		return -EIO;
 
 	return pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS);
@@ -62,7 +57,7 @@ int pci_cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 	int pos;
 	u32 status;
 
-	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	pos = dev->aer_cap;
 	if (!pos)
 		return -EIO;
 
@@ -83,7 +78,7 @@ int pci_cleanup_aer_error_status_regs(struct pci_dev *dev)
 	if (!pci_is_pcie(dev))
 		return -ENODEV;
 
-	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	pos = dev->aer_cap;
 	if (!pos)
 		return -EIO;
 
@@ -102,6 +97,12 @@ int pci_cleanup_aer_error_status_regs(struct pci_dev *dev)
 	return 0;
 }
 
+int pci_aer_init(struct pci_dev *dev)
+{
+	dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	return pci_cleanup_aer_error_status_regs(dev);
+}
+
 /**
  * add_error_device - list device to be handled
  * @e_info: pointer to error info
@@ -132,7 +133,8 @@ static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)
 	 * When bus id is equal to 0, it might be a bad id
 	 * reported by root port.
 	 */
-	if (!nosourceid && (PCI_BUS_NUM(e_info->id) != 0)) {
+	if ((PCI_BUS_NUM(e_info->id) != 0) &&
+	    !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) {
 		/* Device ID match? */
 		if (e_info->id == ((dev->bus->number << 8) | dev->devfn))
 			return true;
@@ -144,10 +146,10 @@ static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)
 
 	/*
 	 * When either
-	 *      1) nosourceid==y;
-	 *      2) bus id is equal to 0. Some ports might lose the bus
+	 *      1) bus id is equal to 0. Some ports might lose the bus
 	 *              id of error source id;
-	 *      3) There are multiple errors and prior id comparing fails;
+	 *      2) bus flag PCI_BUS_FLAGS_NO_AERSID is set
+	 *      3) There are multiple errors and prior ID comparing fails;
 	 * We check AER status registers to find possible reporter.
 	 */
 	if (atomic_read(&dev->enable_cnt) == 0)
@@ -158,7 +160,7 @@ static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)
 	if (!(reg16 & PCI_EXP_AER_FLAGS))
 		return false;
 
-	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	pos = dev->aer_cap;
 	if (!pos)
 		return false;
 
@@ -555,7 +557,7 @@ static void handle_error_source(struct pcie_device *aerdev,
 		 * Correctable error does not need software intervention.
 		 * No need to go through error recovery process.
 		 */
-		pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+		pos = dev->aer_cap;
 		if (pos)
 			pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
 					info->status);
@@ -647,7 +649,7 @@ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 	info->status = 0;
 	info->tlp_header_valid = 0;
 
-	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	pos = dev->aer_cap;
 
 	/* The device might not support AER */
 	if (!pos)
@@ -715,15 +717,8 @@ static inline void aer_process_err_devices(struct pcie_device *p_device,
 static void aer_isr_one_error(struct pcie_device *p_device,
 		struct aer_err_source *e_src)
 {
-	struct aer_err_info *e_info;
-
-	/* struct aer_err_info might be big, so we allocate it with slab */
-	e_info = kmalloc(sizeof(struct aer_err_info), GFP_KERNEL);
-	if (!e_info) {
-		dev_printk(KERN_DEBUG, &p_device->port->dev,
-			"Can't allocate mem when processing AER errors\n");
-		return;
-	}
+	struct aer_rpc *rpc = get_service_data(p_device);
+	struct aer_err_info *e_info = &rpc->e_info;
 
 	/*
 	 * There is a possibility that both correctable error and
@@ -762,8 +757,6 @@ static void aer_isr_one_error(struct pcie_device *p_device,
 		if (find_source_device(p_device->port, e_info))
 			aer_process_err_devices(p_device, e_info);
 	}
-
-	kfree(e_info);
 }
 
 /**
@@ -812,19 +805,3 @@ void aer_isr(struct work_struct *work)
 		aer_isr_one_error(p_device, &e_src);
 	mutex_unlock(&rpc->rpc_mutex);
 }
-
-/**
- * aer_init - provide AER initialization
- * @dev: pointer to AER pcie device
- *
- * Invoked when AER service driver is loaded.
- */
-int aer_init(struct pcie_device *dev)
-{
-	if (forceload) {
-		dev_printk(KERN_DEBUG, &dev->device,
-			   "aerdrv forceload requested.\n");
-		pcie_aer_force_firmware_first(dev->port, 0);
-	}
-	return 0;
-}
diff --git a/drivers/pci/pcie/aer/aerdrv_errprint.c b/drivers/pci/pcie/aer/aerdrv_errprint.c
index 167fe411ce2e..54c4b691e51f 100644
--- a/drivers/pci/pcie/aer/aerdrv_errprint.c
+++ b/drivers/pci/pcie/aer/aerdrv_errprint.c
@@ -219,15 +219,13 @@ int cper_severity_to_aer(int cper_severity)
 }
 EXPORT_SYMBOL_GPL(cper_severity_to_aer);
 
-void cper_print_aer(struct pci_dev *dev, int cper_severity,
+void cper_print_aer(struct pci_dev *dev, int aer_severity,
 		    struct aer_capability_regs *aer)
 {
-	int aer_severity, layer, agent, status_strs_size, tlp_header_valid = 0;
+	int layer, agent, status_strs_size, tlp_header_valid = 0;
 	u32 status, mask;
 	const char **status_strs;
 
-	aer_severity = cper_severity_to_aer(cper_severity);
-
 	if (aer_severity == AER_CORRECTABLE) {
 		status = aer->cor_status;
 		mask = aer->cor_mask;
diff --git a/drivers/pci/pcie/pcie-dpc.c b/drivers/pci/pcie/pcie-dpc.c
index 250f87861786..9811b14d9ad8 100644
--- a/drivers/pci/pcie/pcie-dpc.c
+++ b/drivers/pci/pcie/pcie-dpc.c
@@ -1,5 +1,7 @@
 /*
  * PCI Express Downstream Port Containment services driver
+ * Author: Keith Busch <keith.busch@intel.com>
+ *
  * Copyright (C) 2016 Intel Corp.
  *
  * This file is subject to the terms and conditions of the GNU General Public
@@ -9,7 +11,7 @@
 
 #include <linux/delay.h>
 #include <linux/interrupt.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/pci.h>
 #include <linux/pcieport_if.h>
 
@@ -143,16 +145,4 @@ static int __init dpc_service_init(void)
 {
 	return pcie_port_service_register(&dpcdriver);
 }
-
-static void __exit dpc_service_exit(void)
-{
-	pcie_port_service_unregister(&dpcdriver);
-}
-
-MODULE_DESCRIPTION("PCI Express Downstream Port Containment driver");
-MODULE_AUTHOR("Keith Busch <keith.busch@intel.com>");
-MODULE_LICENSE("GPL");
-MODULE_VERSION("0.1");
-
-module_init(dpc_service_init);
-module_exit(dpc_service_exit);
+device_initcall(dpc_service_init);
diff --git a/drivers/pci/pcie/pme.c b/drivers/pci/pcie/pme.c
index 1ae4c73e7a3c..884bad5320f8 100644
--- a/drivers/pci/pcie/pme.c
+++ b/drivers/pci/pcie/pme.c
@@ -10,7 +10,6 @@
  * for more details.
  */
 
-#include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -449,17 +448,6 @@ static int pcie_pme_resume(struct pcie_device *srv)
 	return 0;
 }
 
-/**
- * pcie_pme_remove - Prepare PCIe PME service device for removal.
- * @srv - PCIe service device to remove.
- */
-static void pcie_pme_remove(struct pcie_device *srv)
-{
-	pcie_pme_suspend(srv);
-	free_irq(srv->irq, srv);
-	kfree(get_service_data(srv));
-}
-
 static struct pcie_port_service_driver pcie_pme_driver = {
 	.name		= "pcie_pme",
 	.port_type	= PCI_EXP_TYPE_ROOT_PORT,
@@ -468,7 +456,6 @@ static struct pcie_port_service_driver pcie_pme_driver = {
 	.probe		= pcie_pme_probe,
 	.suspend	= pcie_pme_suspend,
 	.resume		= pcie_pme_resume,
-	.remove		= pcie_pme_remove,
 };
 
 /**
@@ -478,5 +465,4 @@ static int __init pcie_pme_service_init(void)
 {
 	return pcie_port_service_register(&pcie_pme_driver);
 }
-
-module_init(pcie_pme_service_init);
+device_initcall(pcie_pme_service_init);
diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c
index 70d7ad8c6d17..79327cc14e7d 100644
--- a/drivers/pci/pcie/portdrv_pci.c
+++ b/drivers/pci/pcie/portdrv_pci.c
@@ -1,12 +1,13 @@
 /*
  * File:	portdrv_pci.c
  * Purpose:	PCI Express Port Bus Driver
+ * Author:	Tom Nguyen <tom.l.nguyen@intel.com>
+ * Version:	v1.0
  *
  * Copyright (C) 2004 Intel
  * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
  */
 
-#include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -21,16 +22,6 @@
 #include "portdrv.h"
 #include "aer/aerdrv.h"
 
-/*
- * Version Information
- */
-#define DRIVER_VERSION "v1.0"
-#define DRIVER_AUTHOR "tom.l.nguyen@intel.com"
-#define DRIVER_DESC "PCIe Port Bus Driver"
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL");
-
 /* If this switch is set, PCIe port native services should not be enabled. */
 bool pcie_ports_disabled;
 
@@ -341,7 +332,6 @@ static const struct pci_device_id port_pci_ids[] = { {
 	PCI_DEVICE_CLASS(((PCI_CLASS_BRIDGE_PCI << 8) | 0x00), ~0),
 	}, { /* end: all zeroes */ }
 };
-MODULE_DEVICE_TABLE(pci, port_pci_ids);
 
 static const struct pci_error_handlers pcie_portdrv_err_handler = {
 	.error_detected = pcie_portdrv_error_detected,
@@ -406,5 +396,4 @@ static int __init pcie_portdrv_init(void)
  out:
 	return retval;
 }
-
-module_init(pcie_portdrv_init);
+device_initcall(pcie_portdrv_init);
diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c
new file mode 100644
index 000000000000..bab8ac63c4f3
--- /dev/null
+++ b/drivers/pci/pcie/ptm.c
@@ -0,0 +1,142 @@
+/*
+ * PCI Express Precision Time Measurement
+ * Copyright (c) 2016, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/pci.h>
+#include "../pci.h"
+
+static void pci_ptm_info(struct pci_dev *dev)
+{
+	char clock_desc[8];
+
+	switch (dev->ptm_granularity) {
+	case 0:
+		snprintf(clock_desc, sizeof(clock_desc), "unknown");
+		break;
+	case 255:
+		snprintf(clock_desc, sizeof(clock_desc), ">254ns");
+		break;
+	default:
+		snprintf(clock_desc, sizeof(clock_desc), "%udns",
+			 dev->ptm_granularity);
+		break;
+	}
+	dev_info(&dev->dev, "PTM enabled%s, %s granularity\n",
+		 dev->ptm_root ? " (root)" : "", clock_desc);
+}
+
+void pci_ptm_init(struct pci_dev *dev)
+{
+	int pos;
+	u32 cap, ctrl;
+	u8 local_clock;
+	struct pci_dev *ups;
+
+	if (!pci_is_pcie(dev))
+		return;
+
+	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM);
+	if (!pos)
+		return;
+
+	/*
+	 * Enable PTM only on interior devices (root ports, switch ports,
+	 * etc.) on the assumption that it causes no link traffic until an
+	 * endpoint enables it.
+	 */
+	if ((pci_pcie_type(dev) == PCI_EXP_TYPE_ENDPOINT ||
+	     pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END))
+		return;
+
+	pci_read_config_dword(dev, pos + PCI_PTM_CAP, &cap);
+	local_clock = (cap & PCI_PTM_GRANULARITY_MASK) >> 8;
+
+	/*
+	 * There's no point in enabling PTM unless it's enabled in the
+	 * upstream device or this device can be a PTM Root itself.  Per
+	 * the spec recommendation (PCIe r3.1, sec 7.32.3), select the
+	 * furthest upstream Time Source as the PTM Root.
+	 */
+	ups = pci_upstream_bridge(dev);
+	if (ups && ups->ptm_enabled) {
+		ctrl = PCI_PTM_CTRL_ENABLE;
+		if (ups->ptm_granularity == 0)
+			dev->ptm_granularity = 0;
+		else if (ups->ptm_granularity > local_clock)
+			dev->ptm_granularity = ups->ptm_granularity;
+	} else {
+		if (cap & PCI_PTM_CAP_ROOT) {
+			ctrl = PCI_PTM_CTRL_ENABLE | PCI_PTM_CTRL_ROOT;
+			dev->ptm_root = 1;
+			dev->ptm_granularity = local_clock;
+		} else
+			return;
+	}
+
+	ctrl |= dev->ptm_granularity << 8;
+	pci_write_config_dword(dev, pos + PCI_PTM_CTRL, ctrl);
+	dev->ptm_enabled = 1;
+
+	pci_ptm_info(dev);
+}
+
+int pci_enable_ptm(struct pci_dev *dev, u8 *granularity)
+{
+	int pos;
+	u32 cap, ctrl;
+	struct pci_dev *ups;
+
+	if (!pci_is_pcie(dev))
+		return -EINVAL;
+
+	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM);
+	if (!pos)
+		return -EINVAL;
+
+	pci_read_config_dword(dev, pos + PCI_PTM_CAP, &cap);
+	if (!(cap & PCI_PTM_CAP_REQ))
+		return -EINVAL;
+
+	/*
+	 * For a PCIe Endpoint, PTM is only useful if the endpoint can
+	 * issue PTM requests to upstream devices that have PTM enabled.
+	 *
+	 * For Root Complex Integrated Endpoints, there is no upstream
+	 * device, so there must be some implementation-specific way to
+	 * associate the endpoint with a time source.
+	 */
+	if (pci_pcie_type(dev) == PCI_EXP_TYPE_ENDPOINT) {
+		ups = pci_upstream_bridge(dev);
+		if (!ups || !ups->ptm_enabled)
+			return -EINVAL;
+
+		dev->ptm_granularity = ups->ptm_granularity;
+	} else if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
+		dev->ptm_granularity = 0;
+	} else
+		return -EINVAL;
+
+	ctrl = PCI_PTM_CTRL_ENABLE;
+	ctrl |= dev->ptm_granularity << 8;
+	pci_write_config_dword(dev, pos + PCI_PTM_CTRL, ctrl);
+	dev->ptm_enabled = 1;
+
+	pci_ptm_info(dev);
+
+	if (granularity)
+		*granularity = dev->ptm_granularity;
+	return 0;
+}
+EXPORT_SYMBOL(pci_enable_ptm);
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 93f280df3428..ab002671fa60 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -1666,7 +1666,11 @@ static void pci_init_capabilities(struct pci_dev *dev)
 	/* Enable ACS P2P upstream forwarding */
 	pci_enable_acs(dev);
 
-	pci_cleanup_aer_error_status_regs(dev);
+	/* Precision Time Measurement */
+	pci_ptm_init(dev);
+
+	/* Advanced Error Reporting */
+	pci_aer_init(dev);
 }
 
 /*
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index 37ff0158e45f..c232729f5b1b 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -834,6 +834,17 @@ static void quirk_amd_ioapic(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD,	PCI_DEVICE_ID_AMD_VIPER_7410,	quirk_amd_ioapic);
 #endif /* CONFIG_X86_IO_APIC */
 
+#if defined(CONFIG_ARM64) && defined(CONFIG_PCI_ATS)
+
+static void quirk_cavium_sriov_rnm_link(struct pci_dev *dev)
+{
+	/* Fix for improper SRIOV configuration on Cavium cn88xx  RNM device */
+	if (dev->subsystem_device == 0xa118)
+		dev->sriov->link = dev->devfn;
+}
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CAVIUM, 0xa018, quirk_cavium_sriov_rnm_link);
+#endif
+
 /*
  * Some settings of MMRBC can lead to data corruption so block changes.
  * See AMD 8131 HyperTransport PCI-X Tunnel Revision Guide
@@ -3198,6 +3209,7 @@ static void quirk_no_bus_reset(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0030, quirk_no_bus_reset);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0032, quirk_no_bus_reset);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x003c, quirk_no_bus_reset);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0033, quirk_no_bus_reset);
 
 static void quirk_no_pm_reset(struct pci_dev *dev)
 {
@@ -3327,9 +3339,9 @@ static void quirk_apple_wait_for_thunderbolt(struct pci_dev *dev)
 	if (nhi->vendor != PCI_VENDOR_ID_INTEL
 		    || (nhi->device != PCI_DEVICE_ID_INTEL_LIGHT_RIDGE &&
 			nhi->device != PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C &&
+			nhi->device != PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI &&
 			nhi->device != PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI)
-		    || nhi->subsystem_vendor != 0x2222
-		    || nhi->subsystem_device != 0x1111)
+		    || nhi->class != PCI_CLASS_SYSTEM_OTHER << 8)
 		goto out;
 	dev_info(&dev->dev, "quirk: waiting for thunderbolt to reestablish PCI tunnels...\n");
 	device_pm_wait_for_dev(&dev->dev, &nhi->dev);
@@ -3344,6 +3356,9 @@ DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_INTEL,
 			       PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C,
 			       quirk_apple_wait_for_thunderbolt);
 DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_INTEL,
+			       PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_BRIDGE,
+			       quirk_apple_wait_for_thunderbolt);
+DECLARE_PCI_FIXUP_RESUME_EARLY(PCI_VENDOR_ID_INTEL,
 			       PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_BRIDGE,
 			       quirk_apple_wait_for_thunderbolt);
 #endif
@@ -4428,3 +4443,20 @@ static void quirk_intel_qat_vf_cap(struct pci_dev *pdev)
 	}
 }
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap);
+
+/*
+ * VMD-enabled root ports will change the source ID for all messages
+ * to the VMD device. Rather than doing device matching with the source
+ * ID, the AER driver should traverse the child device tree, reading
+ * AER registers to find the faulting device.
+ */
+static void quirk_no_aersid(struct pci_dev *pdev)
+{
+	/* VMD Domain */
+	if (pdev->bus->sysdata && pci_domain_nr(pdev->bus) >= 0x10000)
+		pdev->bus->bus_flags |= PCI_BUS_FLAGS_NO_AERSID;
+}
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2030, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2031, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2032, quirk_no_aersid);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2033, quirk_no_aersid);
diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c
index d1ef7acf6930..f9357e09e9b3 100644
--- a/drivers/pci/remove.c
+++ b/drivers/pci/remove.c
@@ -40,6 +40,7 @@ static void pci_destroy_dev(struct pci_dev *dev)
 	list_del(&dev->bus_list);
 	up_write(&pci_bus_sem);
 
+	pci_bridge_d3_device_removed(dev);
 	pci_free_resources(dev);
 	put_device(&dev->dev);
 }
@@ -96,8 +97,6 @@ static void pci_remove_bus_device(struct pci_dev *dev)
 		dev->subordinate = NULL;
 	}
 
-	pci_bridge_d3_device_removed(dev);
-
 	pci_destroy_dev(dev);
 }
 
diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c
index c74059e10a6d..f30ca75b5b6c 100644
--- a/drivers/pci/setup-bus.c
+++ b/drivers/pci/setup-bus.c
@@ -25,6 +25,7 @@
 #include <linux/ioport.h>
 #include <linux/cache.h>
 #include <linux/slab.h>
+#include <linux/acpi.h>
 #include "pci.h"
 
 unsigned int pci_flags;
@@ -1852,8 +1853,13 @@ void __init pci_assign_unassigned_resources(void)
 {
 	struct pci_bus *root_bus;
 
-	list_for_each_entry(root_bus, &pci_root_buses, node)
+	list_for_each_entry(root_bus, &pci_root_buses, node) {
 		pci_assign_unassigned_root_bus_resources(root_bus);
+
+		/* Make sure the root bridge has a companion ACPI device: */
+		if (ACPI_HANDLE(root_bus->bridge))
+			acpi_ioapic_add(ACPI_HANDLE(root_bus->bridge));
+	}
 }
 
 void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)
diff --git a/drivers/pcmcia/ds.c b/drivers/pcmcia/ds.c
index 489ea1098c96..69b5e811ea2b 100644
--- a/drivers/pcmcia/ds.c
+++ b/drivers/pcmcia/ds.c
@@ -977,7 +977,7 @@ static int pcmcia_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
 
 /************************ runtime PM support ***************************/
 
-static int pcmcia_dev_suspend(struct device *dev, pm_message_t state);
+static int pcmcia_dev_suspend(struct device *dev);
 static int pcmcia_dev_resume(struct device *dev);
 
 static int runtime_suspend(struct device *dev)
@@ -985,7 +985,7 @@ static int runtime_suspend(struct device *dev)
 	int rc;
 
 	device_lock(dev);
-	rc = pcmcia_dev_suspend(dev, PMSG_SUSPEND);
+	rc = pcmcia_dev_suspend(dev);
 	device_unlock(dev);
 	return rc;
 }
@@ -1135,7 +1135,7 @@ ATTRIBUTE_GROUPS(pcmcia_dev);
 
 /* PM support, also needed for reset */
 
-static int pcmcia_dev_suspend(struct device *dev, pm_message_t state)
+static int pcmcia_dev_suspend(struct device *dev)
 {
 	struct pcmcia_device *p_dev = to_pcmcia_dev(dev);
 	struct pcmcia_driver *p_drv = NULL;
@@ -1410,6 +1410,9 @@ static struct class_interface pcmcia_bus_interface __refdata = {
 	.remove_dev = &pcmcia_bus_remove_socket,
 };
 
+static const struct dev_pm_ops pcmcia_bus_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pcmcia_dev_suspend, pcmcia_dev_resume)
+};
 
 struct bus_type pcmcia_bus_type = {
 	.name = "pcmcia",
@@ -1418,8 +1421,7 @@ struct bus_type pcmcia_bus_type = {
 	.dev_groups = pcmcia_dev_groups,
 	.probe = pcmcia_device_probe,
 	.remove = pcmcia_device_remove,
-	.suspend = pcmcia_dev_suspend,
-	.resume = pcmcia_dev_resume,
+	.pm = &pcmcia_bus_pm_ops,
 };
 
 
diff --git a/drivers/pcmcia/pxa2xx_base.c b/drivers/pcmcia/pxa2xx_base.c
index 483f919e0d2e..91b5f5724cba 100644
--- a/drivers/pcmcia/pxa2xx_base.c
+++ b/drivers/pcmcia/pxa2xx_base.c
@@ -214,9 +214,8 @@ pxa2xx_pcmcia_frequency_change(struct soc_pcmcia_socket *skt,
 }
 #endif
 
-void pxa2xx_configure_sockets(struct device *dev)
+void pxa2xx_configure_sockets(struct device *dev, struct pcmcia_low_level *ops)
 {
-	struct pcmcia_low_level *ops = dev->platform_data;
 	/*
 	 * We have at least one socket, so set MECR:CIT
 	 * (Card Is There)
@@ -322,7 +321,7 @@ static int pxa2xx_drv_pcmcia_probe(struct platform_device *dev)
 			goto err1;
 	}
 
-	pxa2xx_configure_sockets(&dev->dev);
+	pxa2xx_configure_sockets(&dev->dev, ops);
 	dev_set_drvdata(&dev->dev, sinfo);
 
 	return 0;
@@ -348,7 +347,9 @@ static int pxa2xx_drv_pcmcia_remove(struct platform_device *dev)
 
 static int pxa2xx_drv_pcmcia_resume(struct device *dev)
 {
-	pxa2xx_configure_sockets(dev);
+	struct pcmcia_low_level *ops = (struct pcmcia_low_level *)dev->platform_data;
+
+	pxa2xx_configure_sockets(dev, ops);
 	return 0;
 }
 
diff --git a/drivers/pcmcia/pxa2xx_base.h b/drivers/pcmcia/pxa2xx_base.h
index b609b45469ed..e58c7a415418 100644
--- a/drivers/pcmcia/pxa2xx_base.h
+++ b/drivers/pcmcia/pxa2xx_base.h
@@ -1,4 +1,4 @@
 int pxa2xx_drv_pcmcia_add_one(struct soc_pcmcia_socket *skt);
 void pxa2xx_drv_pcmcia_ops(struct pcmcia_low_level *ops);
-void pxa2xx_configure_sockets(struct device *dev);
+void pxa2xx_configure_sockets(struct device *dev, struct pcmcia_low_level *ops);
 
diff --git a/drivers/pcmcia/sa1100_assabet.c b/drivers/pcmcia/sa1100_assabet.c
index 44cfc4416e54..71ace6910d7e 100644
--- a/drivers/pcmcia/sa1100_assabet.c
+++ b/drivers/pcmcia/sa1100_assabet.c
@@ -31,13 +31,6 @@ static int assabet_pcmcia_hw_init(struct soc_pcmcia_socket *skt)
 	return 0;
 }
 
-static void
-assabet_pcmcia_socket_state(struct soc_pcmcia_socket *skt, struct pcmcia_state *state)
-{
-	state->vs_3v  = 1; /* Can only apply 3.3V on Assabet. */
-	state->vs_Xv  = 0;
-}
-
 static int
 assabet_pcmcia_configure_socket(struct soc_pcmcia_socket *skt, const socket_state_t *state)
 {
@@ -90,7 +83,7 @@ static void assabet_pcmcia_socket_suspend(struct soc_pcmcia_socket *skt)
 static struct pcmcia_low_level assabet_pcmcia_ops = { 
 	.owner			= THIS_MODULE,
 	.hw_init		= assabet_pcmcia_hw_init,
-	.socket_state		= assabet_pcmcia_socket_state,
+	.socket_state		= soc_common_cf_socket_state,
 	.configure_socket	= assabet_pcmcia_configure_socket,
 	.socket_suspend		= assabet_pcmcia_socket_suspend,
 };
diff --git a/drivers/pcmcia/sa1100_cerf.c b/drivers/pcmcia/sa1100_cerf.c
index b3774e5d0396..c3f67363f6a1 100644
--- a/drivers/pcmcia/sa1100_cerf.c
+++ b/drivers/pcmcia/sa1100_cerf.c
@@ -45,13 +45,6 @@ static void cerf_pcmcia_hw_shutdown(struct soc_pcmcia_socket *skt)
 	gpio_free(CERF_GPIO_CF_RESET);
 }
 
-static void
-cerf_pcmcia_socket_state(struct soc_pcmcia_socket *skt, struct pcmcia_state *state)
-{
-	state->vs_3v	= 1;
-	state->vs_Xv	= 0;
-}
-
 static int
 cerf_pcmcia_configure_socket(struct soc_pcmcia_socket *skt,
 			     const socket_state_t *state)
@@ -77,7 +70,7 @@ static struct pcmcia_low_level cerf_pcmcia_ops = {
 	.owner			= THIS_MODULE,
 	.hw_init		= cerf_pcmcia_hw_init,
 	.hw_shutdown		= cerf_pcmcia_hw_shutdown,
-	.socket_state		= cerf_pcmcia_socket_state,
+	.socket_state		= soc_common_cf_socket_state,
 	.configure_socket	= cerf_pcmcia_configure_socket,
 };
 
diff --git a/drivers/pcmcia/sa1111_badge4.c b/drivers/pcmcia/sa1111_badge4.c
index 12f0dd091477..2f490930430d 100644
--- a/drivers/pcmcia/sa1111_badge4.c
+++ b/drivers/pcmcia/sa1111_badge4.c
@@ -134,20 +134,14 @@ static struct pcmcia_low_level badge4_pcmcia_ops = {
 
 int pcmcia_badge4_init(struct sa1111_dev *dev)
 {
-	int ret = -ENODEV;
-
-	if (machine_is_badge4()) {
-		printk(KERN_INFO
-		       "%s: badge4_pcmvcc=%d, badge4_pcmvpp=%d, badge4_cfvcc=%d\n",
-		       __func__,
-		       badge4_pcmvcc, badge4_pcmvpp, badge4_cfvcc);
-
-		sa11xx_drv_pcmcia_ops(&badge4_pcmcia_ops);
-		ret = sa1111_pcmcia_add(dev, &badge4_pcmcia_ops,
-				sa11xx_drv_pcmcia_add_one);
-	}
-
-	return ret;
+	printk(KERN_INFO
+	       "%s: badge4_pcmvcc=%d, badge4_pcmvpp=%d, badge4_cfvcc=%d\n",
+	       __func__,
+	       badge4_pcmvcc, badge4_pcmvpp, badge4_cfvcc);
+
+	sa11xx_drv_pcmcia_ops(&badge4_pcmcia_ops);
+	return sa1111_pcmcia_add(dev, &badge4_pcmcia_ops,
+				 sa11xx_drv_pcmcia_add_one);
 }
 
 static int __init pcmv_setup(char *s)
diff --git a/drivers/pcmcia/sa1111_generic.c b/drivers/pcmcia/sa1111_generic.c
index a1531feb8460..3d95dffcff7a 100644
--- a/drivers/pcmcia/sa1111_generic.c
+++ b/drivers/pcmcia/sa1111_generic.c
@@ -18,6 +18,7 @@
 
 #include <mach/hardware.h>
 #include <asm/hardware/sa1111.h>
+#include <asm/mach-types.h>
 #include <asm/irq.h>
 
 #include "sa1111_generic.h"
@@ -203,19 +204,30 @@ static int pcmcia_probe(struct sa1111_dev *dev)
 	sa1111_writel(PCSSR_S0_SLEEP | PCSSR_S1_SLEEP, base + PCSSR);
 	sa1111_writel(PCCR_S0_FLT | PCCR_S1_FLT, base + PCCR);
 
+	ret = -ENODEV;
 #ifdef CONFIG_SA1100_BADGE4
-	pcmcia_badge4_init(dev);
+	if (machine_is_badge4())
+		ret = pcmcia_badge4_init(dev);
 #endif
 #ifdef CONFIG_SA1100_JORNADA720
-	pcmcia_jornada720_init(dev);
+	if (machine_is_jornada720())
+		ret = pcmcia_jornada720_init(dev);
 #endif
 #ifdef CONFIG_ARCH_LUBBOCK
-	pcmcia_lubbock_init(dev);
+	if (machine_is_lubbock())
+		ret = pcmcia_lubbock_init(dev);
 #endif
 #ifdef CONFIG_ASSABET_NEPONSET
-	pcmcia_neponset_init(dev);
+	if (machine_is_assabet())
+		ret = pcmcia_neponset_init(dev);
 #endif
-	return 0;
+
+	if (ret) {
+		release_mem_region(dev->res.start, 512);
+		sa1111_disable_device(dev);
+	}
+
+	return ret;
 }
 
 static int pcmcia_remove(struct sa1111_dev *dev)
diff --git a/drivers/pcmcia/sa1111_jornada720.c b/drivers/pcmcia/sa1111_jornada720.c
index c2c30580c83f..480a3ede27c8 100644
--- a/drivers/pcmcia/sa1111_jornada720.c
+++ b/drivers/pcmcia/sa1111_jornada720.c
@@ -94,22 +94,17 @@ static struct pcmcia_low_level jornada720_pcmcia_ops = {
 
 int pcmcia_jornada720_init(struct sa1111_dev *sadev)
 {
-	int ret = -ENODEV;
+	unsigned int pin = GPIO_A0 | GPIO_A1 | GPIO_A2 | GPIO_A3;
 
-	if (machine_is_jornada720()) {
-		unsigned int pin = GPIO_A0 | GPIO_A1 | GPIO_A2 | GPIO_A3;
+	/* Fixme: why messing around with SA11x0's GPIO1? */
+	GRER |= 0x00000002;
 
-		GRER |= 0x00000002;
+	/* Set GPIO_A<3:1> to be outputs for PCMCIA/CF power controller: */
+	sa1111_set_io_dir(sadev, pin, 0, 0);
+	sa1111_set_io(sadev, pin, 0);
+	sa1111_set_sleep_io(sadev, pin, 0);
 
-		/* Set GPIO_A<3:1> to be outputs for PCMCIA/CF power controller: */
-		sa1111_set_io_dir(sadev, pin, 0, 0);
-		sa1111_set_io(sadev, pin, 0);
-		sa1111_set_sleep_io(sadev, pin, 0);
-
-		sa11xx_drv_pcmcia_ops(&jornada720_pcmcia_ops);
-		ret = sa1111_pcmcia_add(sadev, &jornada720_pcmcia_ops,
-				sa11xx_drv_pcmcia_add_one);
-	}
-
-	return ret;
+	sa11xx_drv_pcmcia_ops(&jornada720_pcmcia_ops);
+	return sa1111_pcmcia_add(sadev, &jornada720_pcmcia_ops,
+				 sa11xx_drv_pcmcia_add_one);
 }
diff --git a/drivers/pcmcia/sa1111_lubbock.c b/drivers/pcmcia/sa1111_lubbock.c
index c5caf5790451..e741f499c875 100644
--- a/drivers/pcmcia/sa1111_lubbock.c
+++ b/drivers/pcmcia/sa1111_lubbock.c
@@ -210,27 +210,21 @@ static struct pcmcia_low_level lubbock_pcmcia_ops = {
 
 int pcmcia_lubbock_init(struct sa1111_dev *sadev)
 {
-	int ret = -ENODEV;
-
-	if (machine_is_lubbock()) {
-		/*
-		 * Set GPIO_A<3:0> to be outputs for the MAX1600,
-		 * and switch to standby mode.
-		 */
-		sa1111_set_io_dir(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0, 0);
-		sa1111_set_io(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0);
-		sa1111_set_sleep_io(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0);
-
-		/* Set CF Socket 1 power to standby mode. */
-		lubbock_set_misc_wr((1 << 15) | (1 << 14), 0);
+	/*
+	 * Set GPIO_A<3:0> to be outputs for the MAX1600,
+	 * and switch to standby mode.
+	 */
+	sa1111_set_io_dir(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0, 0);
+	sa1111_set_io(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0);
+	sa1111_set_sleep_io(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0);
 
-		pxa2xx_drv_pcmcia_ops(&lubbock_pcmcia_ops);
-		pxa2xx_configure_sockets(&sadev->dev);
-		ret = sa1111_pcmcia_add(sadev, &lubbock_pcmcia_ops,
-				pxa2xx_drv_pcmcia_add_one);
-	}
+	/* Set CF Socket 1 power to standby mode. */
+	lubbock_set_misc_wr((1 << 15) | (1 << 14), 0);
 
-	return ret;
+	pxa2xx_drv_pcmcia_ops(&lubbock_pcmcia_ops);
+	pxa2xx_configure_sockets(&sadev->dev, &lubbock_pcmcia_ops);
+	return sa1111_pcmcia_add(sadev, &lubbock_pcmcia_ops,
+				 pxa2xx_drv_pcmcia_add_one);
 }
 
 MODULE_LICENSE("GPL");
diff --git a/drivers/pcmcia/sa1111_neponset.c b/drivers/pcmcia/sa1111_neponset.c
index 1d78739c4c07..019c395eb4bf 100644
--- a/drivers/pcmcia/sa1111_neponset.c
+++ b/drivers/pcmcia/sa1111_neponset.c
@@ -110,20 +110,14 @@ static struct pcmcia_low_level neponset_pcmcia_ops = {
 
 int pcmcia_neponset_init(struct sa1111_dev *sadev)
 {
-	int ret = -ENODEV;
-
-	if (machine_is_assabet()) {
-		/*
-		 * Set GPIO_A<3:0> to be outputs for the MAX1600,
-		 * and switch to standby mode.
-		 */
-		sa1111_set_io_dir(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0, 0);
-		sa1111_set_io(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0);
-		sa1111_set_sleep_io(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0);
-		sa11xx_drv_pcmcia_ops(&neponset_pcmcia_ops);
-		ret = sa1111_pcmcia_add(sadev, &neponset_pcmcia_ops,
-				sa11xx_drv_pcmcia_add_one);
-	}
-
-	return ret;
+	/*
+	 * Set GPIO_A<3:0> to be outputs for the MAX1600,
+	 * and switch to standby mode.
+	 */
+	sa1111_set_io_dir(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0, 0);
+	sa1111_set_io(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0);
+	sa1111_set_sleep_io(sadev, GPIO_A0|GPIO_A1|GPIO_A2|GPIO_A3, 0);
+	sa11xx_drv_pcmcia_ops(&neponset_pcmcia_ops);
+	return sa1111_pcmcia_add(sadev, &neponset_pcmcia_ops,
+				 sa11xx_drv_pcmcia_add_one);
 }
diff --git a/drivers/pcmcia/sa11xx_base.c b/drivers/pcmcia/sa11xx_base.c
index 9f6ec87b9f9e..48140ac73ed6 100644
--- a/drivers/pcmcia/sa11xx_base.c
+++ b/drivers/pcmcia/sa11xx_base.c
@@ -144,19 +144,19 @@ static int
 sa1100_pcmcia_show_timing(struct soc_pcmcia_socket *skt, char *buf)
 {
 	struct soc_pcmcia_timing timing;
-	unsigned int clock = clk_get_rate(skt->clk);
+	unsigned int clock = clk_get_rate(skt->clk) / 1000;
 	unsigned long mecr = MECR;
 	char *p = buf;
 
 	soc_common_pcmcia_get_timing(skt, &timing);
 
-	p+=sprintf(p, "I/O      : %u (%u)\n", timing.io,
+	p+=sprintf(p, "I/O      : %uns (%uns)\n", timing.io,
 		   sa1100_pcmcia_cmd_time(clock, MECR_BSIO_GET(mecr, skt->nr)));
 
-	p+=sprintf(p, "attribute: %u (%u)\n", timing.attr,
+	p+=sprintf(p, "attribute: %uns (%uns)\n", timing.attr,
 		   sa1100_pcmcia_cmd_time(clock, MECR_BSA_GET(mecr, skt->nr)));
 
-	p+=sprintf(p, "common   : %u (%u)\n", timing.mem,
+	p+=sprintf(p, "common   : %uns (%uns)\n", timing.mem,
 		   sa1100_pcmcia_cmd_time(clock, MECR_BSM_GET(mecr, skt->nr)));
 
 	return p - buf;
diff --git a/drivers/pcmcia/soc_common.c b/drivers/pcmcia/soc_common.c
index eed5e9c05353..153f3122283d 100644
--- a/drivers/pcmcia/soc_common.c
+++ b/drivers/pcmcia/soc_common.c
@@ -33,6 +33,7 @@
 
 #include <linux/cpufreq.h>
 #include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -42,6 +43,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/mutex.h>
+#include <linux/regulator/consumer.h>
 #include <linux/spinlock.h>
 #include <linux/timer.h>
 
@@ -79,6 +81,41 @@ EXPORT_SYMBOL(soc_pcmcia_debug);
 #define to_soc_pcmcia_socket(x)	\
 	container_of(x, struct soc_pcmcia_socket, socket)
 
+int soc_pcmcia_regulator_set(struct soc_pcmcia_socket *skt,
+	struct soc_pcmcia_regulator *r, int v)
+{
+	bool on;
+	int ret;
+
+	if (!r->reg)
+		return 0;
+
+	on = v != 0;
+	if (r->on == on)
+		return 0;
+
+	if (on) {
+		ret = regulator_set_voltage(r->reg, v * 100000, v * 100000);
+		if (ret) {
+			int vout = regulator_get_voltage(r->reg) / 100000;
+
+			dev_warn(&skt->socket.dev,
+				 "CS requested %s=%u.%uV, applying %u.%uV\n",
+				 r == &skt->vcc ? "Vcc" : "Vpp",
+				 v / 10, v % 10, vout / 10, vout % 10);
+		}
+
+		ret = regulator_enable(r->reg);
+	} else {
+		regulator_disable(r->reg);
+	}
+	if (ret == 0)
+		r->on = on;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(soc_pcmcia_regulator_set);
+
 static unsigned short
 calc_speed(unsigned short *spds, int num, unsigned short dflt)
 {
@@ -111,12 +148,9 @@ static void __soc_pcmcia_hw_shutdown(struct soc_pcmcia_socket *skt,
 {
 	unsigned int i;
 
-	for (i = 0; i < nr; i++) {
+	for (i = 0; i < nr; i++)
 		if (skt->stat[i].irq)
 			free_irq(skt->stat[i].irq, skt);
-		if (gpio_is_valid(skt->stat[i].gpio))
-			gpio_free(skt->stat[i].gpio);
-	}
 
 	if (skt->ops->hw_shutdown)
 		skt->ops->hw_shutdown(skt);
@@ -129,6 +163,30 @@ static void soc_pcmcia_hw_shutdown(struct soc_pcmcia_socket *skt)
 	__soc_pcmcia_hw_shutdown(skt, ARRAY_SIZE(skt->stat));
 }
 
+int soc_pcmcia_request_gpiods(struct soc_pcmcia_socket *skt)
+{
+	struct device *dev = skt->socket.dev.parent;
+	struct gpio_desc *desc;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(skt->stat); i++) {
+		if (!skt->stat[i].name)
+			continue;
+
+		desc = devm_gpiod_get(dev, skt->stat[i].name, GPIOD_IN);
+		if (IS_ERR(desc)) {
+			dev_err(dev, "Failed to get GPIO for %s: %ld\n",
+				skt->stat[i].name, PTR_ERR(desc));
+			return PTR_ERR(desc);
+		}
+
+		skt->stat[i].desc = desc;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(soc_pcmcia_request_gpiods);
+
 static int soc_pcmcia_hw_init(struct soc_pcmcia_socket *skt)
 {
 	int ret = 0, i;
@@ -143,21 +201,32 @@ static int soc_pcmcia_hw_init(struct soc_pcmcia_socket *skt)
 
 	for (i = 0; i < ARRAY_SIZE(skt->stat); i++) {
 		if (gpio_is_valid(skt->stat[i].gpio)) {
-			int irq;
+			unsigned long flags = GPIOF_IN;
 
-			ret = gpio_request_one(skt->stat[i].gpio, GPIOF_IN,
-					       skt->stat[i].name);
+			/* CD is active low by default */
+			if (i == SOC_STAT_CD)
+				flags |= GPIOF_ACTIVE_LOW;
+
+			ret = devm_gpio_request_one(skt->socket.dev.parent,
+						    skt->stat[i].gpio, flags,
+						    skt->stat[i].name);
 			if (ret) {
 				__soc_pcmcia_hw_shutdown(skt, i);
 				return ret;
 			}
 
-			irq = gpio_to_irq(skt->stat[i].gpio);
+			skt->stat[i].desc = gpio_to_desc(skt->stat[i].gpio);
+		}
+
+		if (i < SOC_STAT_VS1 && skt->stat[i].desc) {
+			int irq = gpiod_to_irq(skt->stat[i].desc);
 
-			if (i == SOC_STAT_RDY)
-				skt->socket.pci_irq = irq;
-			else
-				skt->stat[i].irq = irq;
+			if (irq > 0) {
+				if (i == SOC_STAT_RDY)
+					skt->socket.pci_irq = irq;
+				else
+					skt->stat[i].irq = irq;
+			}
 		}
 
 		if (skt->stat[i].irq) {
@@ -166,8 +235,6 @@ static int soc_pcmcia_hw_init(struct soc_pcmcia_socket *skt)
 					  IRQF_TRIGGER_NONE,
 					  skt->stat[i].name, skt);
 			if (ret) {
-				if (gpio_is_valid(skt->stat[i].gpio))
-					gpio_free(skt->stat[i].gpio);
 				__soc_pcmcia_hw_shutdown(skt, i);
 				return ret;
 			}
@@ -197,6 +264,18 @@ static void soc_pcmcia_hw_disable(struct soc_pcmcia_socket *skt)
 			irq_set_irq_type(skt->stat[i].irq, IRQ_TYPE_NONE);
 }
 
+/*
+ * The CF 3.0 specification says that cards tie VS1 to ground and leave
+ * VS2 open.  Many implementations do not wire up the VS signals, so we
+ * provide hard-coded values as per the CF 3.0 spec.
+ */
+void soc_common_cf_socket_state(struct soc_pcmcia_socket *skt,
+	struct pcmcia_state *state)
+{
+	state->vs_3v = 1;
+}
+EXPORT_SYMBOL_GPL(soc_common_cf_socket_state);
+
 static unsigned int soc_common_pcmcia_skt_state(struct soc_pcmcia_socket *skt)
 {
 	struct pcmcia_state state;
@@ -208,17 +287,18 @@ static unsigned int soc_common_pcmcia_skt_state(struct soc_pcmcia_socket *skt)
 	state.bvd1 = 1;
 	state.bvd2 = 1;
 
-	/* CD is active low by default */
-	if (gpio_is_valid(skt->stat[SOC_STAT_CD].gpio))
-		state.detect = !gpio_get_value(skt->stat[SOC_STAT_CD].gpio);
-
-	/* RDY and BVD are active high by default */
-	if (gpio_is_valid(skt->stat[SOC_STAT_RDY].gpio))
-		state.ready = !!gpio_get_value(skt->stat[SOC_STAT_RDY].gpio);
-	if (gpio_is_valid(skt->stat[SOC_STAT_BVD1].gpio))
-		state.bvd1 = !!gpio_get_value(skt->stat[SOC_STAT_BVD1].gpio);
-	if (gpio_is_valid(skt->stat[SOC_STAT_BVD2].gpio))
-		state.bvd2 = !!gpio_get_value(skt->stat[SOC_STAT_BVD2].gpio);
+	if (skt->stat[SOC_STAT_CD].desc)
+		state.detect = !!gpiod_get_value(skt->stat[SOC_STAT_CD].desc);
+	if (skt->stat[SOC_STAT_RDY].desc)
+		state.ready = !!gpiod_get_value(skt->stat[SOC_STAT_RDY].desc);
+	if (skt->stat[SOC_STAT_BVD1].desc)
+		state.bvd1 = !!gpiod_get_value(skt->stat[SOC_STAT_BVD1].desc);
+	if (skt->stat[SOC_STAT_BVD2].desc)
+		state.bvd2 = !!gpiod_get_value(skt->stat[SOC_STAT_BVD2].desc);
+	if (skt->stat[SOC_STAT_VS1].desc)
+		state.vs_3v = !!gpiod_get_value(skt->stat[SOC_STAT_VS1].desc);
+	if (skt->stat[SOC_STAT_VS2].desc)
+		state.vs_Xv = !!gpiod_get_value(skt->stat[SOC_STAT_VS2].desc);
 
 	skt->ops->socket_state(skt, &state);
 
@@ -235,7 +315,7 @@ static unsigned int soc_common_pcmcia_skt_state(struct soc_pcmcia_socket *skt)
 	stat |= skt->cs_state.Vcc ? SS_POWERON : 0;
 
 	if (skt->cs_state.flags & SS_IOCARD)
-		stat |= state.bvd1 ? SS_STSCHG : 0;
+		stat |= state.bvd1 ? 0 : SS_STSCHG;
 	else {
 		if (state.bvd1 == 0)
 			stat |= SS_BATDEAD;
@@ -257,7 +337,30 @@ static int soc_common_pcmcia_config_skt(
 	int ret;
 
 	ret = skt->ops->configure_socket(skt, state);
+	if (ret < 0) {
+		pr_err("soc_common_pcmcia: unable to configure socket %d\n",
+		       skt->nr);
+		/* restore the previous state */
+		WARN_ON(skt->ops->configure_socket(skt, &skt->cs_state));
+		return ret;
+	}
+
 	if (ret == 0) {
+		struct gpio_desc *descs[2];
+		int values[2], n = 0;
+
+		if (skt->gpio_reset) {
+			descs[n] = skt->gpio_reset;
+			values[n++] = !!(state->flags & SS_RESET);
+		}
+		if (skt->gpio_bus_enable) {
+			descs[n] = skt->gpio_bus_enable;
+			values[n++] = !!(state->flags & SS_OUTPUT_ENA);
+		}
+
+		if (n)
+			gpiod_set_array_value_cansleep(n, descs, values);
+
 		/*
 		 * This really needs a better solution.  The IRQ
 		 * may or may not be claimed by the driver.
@@ -274,10 +377,6 @@ static int soc_common_pcmcia_config_skt(
 		skt->cs_state = *state;
 	}
 
-	if (ret < 0)
-		printk(KERN_ERR "soc_common_pcmcia: unable to configure "
-		       "socket %d\n", skt->nr);
-
 	return ret;
 }
 
@@ -637,54 +736,19 @@ static struct pccard_operations soc_common_pcmcia_operations = {
 };
 
 
-static LIST_HEAD(soc_pcmcia_sockets);
-static DEFINE_MUTEX(soc_pcmcia_sockets_lock);
-
 #ifdef CONFIG_CPU_FREQ
-static int
-soc_pcmcia_notifier(struct notifier_block *nb, unsigned long val, void *data)
+static int soc_common_pcmcia_cpufreq_nb(struct notifier_block *nb,
+	unsigned long val, void *data)
 {
-	struct soc_pcmcia_socket *skt;
+	struct soc_pcmcia_socket *skt = container_of(nb, struct soc_pcmcia_socket, cpufreq_nb);
 	struct cpufreq_freqs *freqs = data;
-	int ret = 0;
-
-	mutex_lock(&soc_pcmcia_sockets_lock);
-	list_for_each_entry(skt, &soc_pcmcia_sockets, node)
-		if (skt->ops->frequency_change)
-			ret += skt->ops->frequency_change(skt, val, freqs);
-	mutex_unlock(&soc_pcmcia_sockets_lock);
-
-	return ret;
-}
-
-static struct notifier_block soc_pcmcia_notifier_block = {
-	.notifier_call	= soc_pcmcia_notifier
-};
-
-static int soc_pcmcia_cpufreq_register(void)
-{
-	int ret;
 
-	ret = cpufreq_register_notifier(&soc_pcmcia_notifier_block,
-					CPUFREQ_TRANSITION_NOTIFIER);
-	if (ret < 0)
-		printk(KERN_ERR "Unable to register CPU frequency change "
-				"notifier for PCMCIA (%d)\n", ret);
-	return ret;
+	return skt->ops->frequency_change(skt, val, freqs);
 }
-fs_initcall(soc_pcmcia_cpufreq_register);
-
-static void soc_pcmcia_cpufreq_unregister(void)
-{
-	cpufreq_unregister_notifier(&soc_pcmcia_notifier_block,
-		CPUFREQ_TRANSITION_NOTIFIER);
-}
-module_exit(soc_pcmcia_cpufreq_unregister);
-
 #endif
 
 void soc_pcmcia_init_one(struct soc_pcmcia_socket *skt,
-	struct pcmcia_low_level *ops, struct device *dev)
+	const struct pcmcia_low_level *ops, struct device *dev)
 {
 	int i;
 
@@ -700,19 +764,21 @@ EXPORT_SYMBOL(soc_pcmcia_init_one);
 
 void soc_pcmcia_remove_one(struct soc_pcmcia_socket *skt)
 {
-	mutex_lock(&soc_pcmcia_sockets_lock);
 	del_timer_sync(&skt->poll_timer);
 
 	pcmcia_unregister_socket(&skt->socket);
 
+#ifdef CONFIG_CPU_FREQ
+	if (skt->ops->frequency_change)
+		cpufreq_unregister_notifier(&skt->cpufreq_nb,
+					    CPUFREQ_TRANSITION_NOTIFIER);
+#endif
+
 	soc_pcmcia_hw_shutdown(skt);
 
 	/* should not be required; violates some lowlevel drivers */
 	soc_common_pcmcia_config_skt(skt, &dead_socket);
 
-	list_del(&skt->node);
-	mutex_unlock(&soc_pcmcia_sockets_lock);
-
 	iounmap(skt->virt_io);
 	skt->virt_io = NULL;
 	release_resource(&skt->res_attr);
@@ -726,6 +792,8 @@ int soc_pcmcia_add_one(struct soc_pcmcia_socket *skt)
 {
 	int ret;
 
+	skt->cs_state = dead_socket;
+
 	setup_timer(&skt->poll_timer, soc_common_pcmcia_poll_event,
 		    (unsigned long)skt);
 	skt->poll_timer.expires = jiffies + SOC_PCMCIA_POLL_PERIOD;
@@ -752,10 +820,6 @@ int soc_pcmcia_add_one(struct soc_pcmcia_socket *skt)
 		goto out_err_5;
 	}
 
-	mutex_lock(&soc_pcmcia_sockets_lock);
-
-	list_add(&skt->node, &soc_pcmcia_sockets);
-
 	/*
 	 * We initialize default socket timing here, because
 	 * we are not guaranteed to see a SetIOMap operation at
@@ -776,14 +840,23 @@ int soc_pcmcia_add_one(struct soc_pcmcia_socket *skt)
 
 	skt->status = soc_common_pcmcia_skt_state(skt);
 
+#ifdef CONFIG_CPU_FREQ
+	if (skt->ops->frequency_change) {
+		skt->cpufreq_nb.notifier_call = soc_common_pcmcia_cpufreq_nb;
+
+		ret = cpufreq_register_notifier(&skt->cpufreq_nb,
+						CPUFREQ_TRANSITION_NOTIFIER);
+		if (ret < 0)
+			dev_err(skt->socket.dev.parent,
+				"unable to register CPU frequency change notifier for PCMCIA (%d)\n",
+				ret);
+	}
+#endif
+
 	ret = pcmcia_register_socket(&skt->socket);
 	if (ret)
 		goto out_err_7;
 
-	add_timer(&skt->poll_timer);
-
-	mutex_unlock(&soc_pcmcia_sockets_lock);
-
 	ret = device_create_file(&skt->socket.dev, &dev_attr_status);
 	if (ret)
 		goto out_err_8;
@@ -791,15 +864,12 @@ int soc_pcmcia_add_one(struct soc_pcmcia_socket *skt)
 	return ret;
 
  out_err_8:
-	mutex_lock(&soc_pcmcia_sockets_lock);
 	del_timer_sync(&skt->poll_timer);
 	pcmcia_unregister_socket(&skt->socket);
 
  out_err_7:
 	soc_pcmcia_hw_shutdown(skt);
  out_err_6:
-	list_del(&skt->node);
-	mutex_unlock(&soc_pcmcia_sockets_lock);
 	iounmap(skt->virt_io);
  out_err_5:
 	release_resource(&skt->res_attr);
diff --git a/drivers/pcmcia/soc_common.h b/drivers/pcmcia/soc_common.h
index 94762a54d731..3f3625805353 100644
--- a/drivers/pcmcia/soc_common.h
+++ b/drivers/pcmcia/soc_common.h
@@ -17,7 +17,14 @@
 
 
 struct device;
+struct gpio_desc;
 struct pcmcia_low_level;
+struct regulator;
+
+struct soc_pcmcia_regulator {
+	struct regulator	*reg;
+	bool			on;
+};
 
 /*
  * This structure encapsulates per-socket state which we might need to
@@ -52,18 +59,30 @@ struct soc_pcmcia_socket {
 
 	struct {
 		int		gpio;
+		struct gpio_desc *desc;
 		unsigned int	irq;
 		const char	*name;
-	} stat[4];
+	} stat[6];
 #define SOC_STAT_CD		0	/* Card detect */
 #define SOC_STAT_BVD1		1	/* BATDEAD / IOSTSCHG */
 #define SOC_STAT_BVD2		2	/* BATWARN / IOSPKR */
 #define SOC_STAT_RDY		3	/* Ready / Interrupt */
+#define SOC_STAT_VS1		4	/* Voltage sense 1 */
+#define SOC_STAT_VS2		5	/* Voltage sense 2 */
+
+	struct gpio_desc	*gpio_reset;
+	struct gpio_desc	*gpio_bus_enable;
+	struct soc_pcmcia_regulator vcc;
+	struct soc_pcmcia_regulator vpp;
 
 	unsigned int		irq_state;
 
+#ifdef CONFIG_CPU_FREQ
+	struct notifier_block	cpufreq_nb;
+#endif
 	struct timer_list	poll_timer;
 	struct list_head	node;
+	void *driver_data;
 };
 
 struct skt_dev_info {
@@ -133,10 +152,16 @@ struct soc_pcmcia_timing {
 extern void soc_common_pcmcia_get_timing(struct soc_pcmcia_socket *, struct soc_pcmcia_timing *);
 
 void soc_pcmcia_init_one(struct soc_pcmcia_socket *skt,
-	struct pcmcia_low_level *ops, struct device *dev);
+	const struct pcmcia_low_level *ops, struct device *dev);
 void soc_pcmcia_remove_one(struct soc_pcmcia_socket *skt);
 int soc_pcmcia_add_one(struct soc_pcmcia_socket *skt);
+int soc_pcmcia_request_gpiods(struct soc_pcmcia_socket *skt);
+
+void soc_common_cf_socket_state(struct soc_pcmcia_socket *skt,
+	struct pcmcia_state *state);
 
+int soc_pcmcia_regulator_set(struct soc_pcmcia_socket *skt,
+	struct soc_pcmcia_regulator *r, int v);
 
 #ifdef CONFIG_PCMCIA_DEBUG
 
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 04e2653bb8c0..4d5c5f9f0dbd 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -12,4 +12,11 @@ config ARM_PMU
 	  Say y if you want to use CPU performance monitors on ARM-based
 	  systems.
 
+config XGENE_PMU
+        depends on PERF_EVENTS && ARCH_XGENE
+        bool "APM X-Gene SoC PMU"
+        default n
+        help
+          Say y if you want to use APM X-Gene SoC performance monitors.
+
 endmenu
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index acd2397ded94..b116e982810b 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -1 +1,2 @@
 obj-$(CONFIG_ARM_PMU) += arm_pmu.o
+obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 6ccb994bdfcb..b37b57294566 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -534,6 +534,24 @@ static int armpmu_filter_match(struct perf_event *event)
 	return cpumask_test_cpu(cpu, &armpmu->supported_cpus);
 }
 
+static ssize_t armpmu_cpumask_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	struct arm_pmu *armpmu = to_arm_pmu(dev_get_drvdata(dev));
+	return cpumap_print_to_pagebuf(true, buf, &armpmu->supported_cpus);
+}
+
+static DEVICE_ATTR(cpus, S_IRUGO, armpmu_cpumask_show, NULL);
+
+static struct attribute *armpmu_common_attrs[] = {
+	&dev_attr_cpus.attr,
+	NULL,
+};
+
+static struct attribute_group armpmu_common_attr_group = {
+	.attrs = armpmu_common_attrs,
+};
+
 static void armpmu_init(struct arm_pmu *armpmu)
 {
 	atomic_set(&armpmu->active_events, 0);
@@ -549,7 +567,10 @@ static void armpmu_init(struct arm_pmu *armpmu)
 		.stop		= armpmu_stop,
 		.read		= armpmu_read,
 		.filter_match	= armpmu_filter_match,
+		.attr_groups	= armpmu->attr_groups,
 	};
+	armpmu->attr_groups[ARMPMU_ATTR_GROUP_COMMON] =
+		&armpmu_common_attr_group;
 }
 
 /* Set at runtime when we know what CPU type we are. */
@@ -602,7 +623,7 @@ static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu)
 	irqs = min(pmu_device->num_resources, num_possible_cpus());
 
 	irq = platform_get_irq(pmu_device, 0);
-	if (irq >= 0 && irq_is_percpu(irq)) {
+	if (irq > 0 && irq_is_percpu(irq)) {
 		on_each_cpu_mask(&cpu_pmu->supported_cpus,
 				 cpu_pmu_disable_percpu_irq, &irq, 1);
 		free_percpu_irq(irq, &hw_events->percpu_pmu);
@@ -616,7 +637,7 @@ static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu)
 			if (!cpumask_test_and_clear_cpu(cpu, &cpu_pmu->active_irqs))
 				continue;
 			irq = platform_get_irq(pmu_device, i);
-			if (irq >= 0)
+			if (irq > 0)
 				free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu));
 		}
 	}
@@ -638,7 +659,7 @@ static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)
 	}
 
 	irq = platform_get_irq(pmu_device, 0);
-	if (irq >= 0 && irq_is_percpu(irq)) {
+	if (irq > 0 && irq_is_percpu(irq)) {
 		err = request_percpu_irq(irq, handler, "arm-pmu",
 					 &hw_events->percpu_pmu);
 		if (err) {
@@ -688,28 +709,20 @@ static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)
 	return 0;
 }
 
-static DEFINE_MUTEX(arm_pmu_mutex);
-static LIST_HEAD(arm_pmu_list);
-
 /*
  * PMU hardware loses all context when a CPU goes offline.
  * When a CPU is hotplugged back in, since some hardware registers are
  * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading
  * junk values out of them.
  */
-static int arm_perf_starting_cpu(unsigned int cpu)
+static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node)
 {
-	struct arm_pmu *pmu;
-
-	mutex_lock(&arm_pmu_mutex);
-	list_for_each_entry(pmu, &arm_pmu_list, entry) {
+	struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node);
 
-		if (!cpumask_test_cpu(cpu, &pmu->supported_cpus))
-			continue;
-		if (pmu->reset)
-			pmu->reset(pmu);
-	}
-	mutex_unlock(&arm_pmu_mutex);
+	if (!cpumask_test_cpu(cpu, &pmu->supported_cpus))
+		return 0;
+	if (pmu->reset)
+		pmu->reset(pmu);
 	return 0;
 }
 
@@ -821,9 +834,10 @@ static int cpu_pmu_init(struct arm_pmu *cpu_pmu)
 	if (!cpu_hw_events)
 		return -ENOMEM;
 
-	mutex_lock(&arm_pmu_mutex);
-	list_add_tail(&cpu_pmu->entry, &arm_pmu_list);
-	mutex_unlock(&arm_pmu_mutex);
+	err = cpuhp_state_add_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING,
+					       &cpu_pmu->node);
+	if (err)
+		goto out_free;
 
 	err = cpu_pm_pmu_register(cpu_pmu);
 	if (err)
@@ -859,9 +873,9 @@ static int cpu_pmu_init(struct arm_pmu *cpu_pmu)
 	return 0;
 
 out_unregister:
-	mutex_lock(&arm_pmu_mutex);
-	list_del(&cpu_pmu->entry);
-	mutex_unlock(&arm_pmu_mutex);
+	cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING,
+					    &cpu_pmu->node);
+out_free:
 	free_percpu(cpu_hw_events);
 	return err;
 }
@@ -869,9 +883,8 @@ out_unregister:
 static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu)
 {
 	cpu_pm_pmu_unregister(cpu_pmu);
-	mutex_lock(&arm_pmu_mutex);
-	list_del(&cpu_pmu->entry);
-	mutex_unlock(&arm_pmu_mutex);
+	cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING,
+					    &cpu_pmu->node);
 	free_percpu(cpu_pmu->hw_events);
 }
 
@@ -919,12 +932,13 @@ static int of_pmu_irq_cfg(struct arm_pmu *pmu)
 
 		/* Check the IRQ type and prohibit a mix of PPIs and SPIs */
 		irq = platform_get_irq(pdev, i);
-		if (irq >= 0) {
+		if (irq > 0) {
 			bool spi = !irq_is_percpu(irq);
 
 			if (i > 0 && spi != using_spi) {
 				pr_err("PPI/SPI IRQ type mismatch for %s!\n",
 					dn->name);
+				of_node_put(dn);
 				kfree(irqs);
 				return -EINVAL;
 			}
@@ -967,11 +981,12 @@ static int of_pmu_irq_cfg(struct arm_pmu *pmu)
 
 	/* If we didn't manage to parse anything, try the interrupt affinity */
 	if (cpumask_weight(&pmu->supported_cpus) == 0) {
-		if (!using_spi) {
+		int irq = platform_get_irq(pdev, 0);
+
+		if (irq > 0 && irq_is_percpu(irq)) {
 			/* If using PPIs, check the affinity of the partition */
-			int ret, irq;
+			int ret;
 
-			irq = platform_get_irq(pdev, 0);
 			ret = irq_get_percpu_devid_partition(irq, &pmu->supported_cpus);
 			if (ret) {
 				kfree(irqs);
@@ -1027,7 +1042,7 @@ int arm_pmu_device_probe(struct platform_device *pdev,
 		ret = of_pmu_irq_cfg(pmu);
 		if (!ret)
 			ret = init_fn(pmu);
-	} else {
+	} else if (probe_table) {
 		cpumask_setall(&pmu->supported_cpus);
 		ret = probe_current_pmu(pmu, probe_table);
 	}
@@ -1037,6 +1052,7 @@ int arm_pmu_device_probe(struct platform_device *pdev,
 		goto out_free;
 	}
 
+
 	ret = cpu_pmu_init(pmu);
 	if (ret)
 		goto out_free;
@@ -1067,9 +1083,9 @@ static int arm_pmu_hp_init(void)
 {
 	int ret;
 
-	ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_STARTING,
-					"AP_PERF_ARM_STARTING",
-					arm_perf_starting_cpu, NULL);
+	ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_STARTING,
+				      "AP_PERF_ARM_STARTING",
+				      arm_perf_starting_cpu, NULL);
 	if (ret)
 		pr_err("CPU hotplug notifier for ARM PMU could not be registered: %d\n",
 		       ret);
diff --git a/drivers/perf/xgene_pmu.c b/drivers/perf/xgene_pmu.c
new file mode 100644
index 000000000000..c2ac7646b99f
--- /dev/null
+++ b/drivers/perf/xgene_pmu.c
@@ -0,0 +1,1398 @@
+/*
+ * APM X-Gene SoC PMU (Performance Monitor Unit)
+ *
+ * Copyright (c) 2016, Applied Micro Circuits Corporation
+ * Author: Hoan Tran <hotran@apm.com>
+ *         Tai Nguyen <ttnguyen@apm.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/acpi.h>
+#include <linux/clk.h>
+#include <linux/cpumask.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_address.h>
+#include <linux/of_fdt.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#define CSW_CSWCR                       0x0000
+#define  CSW_CSWCR_DUALMCB_MASK         BIT(0)
+#define MCBADDRMR                       0x0000
+#define  MCBADDRMR_DUALMCU_MODE_MASK    BIT(2)
+
+#define PCPPMU_INTSTATUS_REG	0x000
+#define PCPPMU_INTMASK_REG	0x004
+#define  PCPPMU_INTMASK		0x0000000F
+#define  PCPPMU_INTENMASK	0xFFFFFFFF
+#define  PCPPMU_INTCLRMASK	0xFFFFFFF0
+#define  PCPPMU_INT_MCU		BIT(0)
+#define  PCPPMU_INT_MCB		BIT(1)
+#define  PCPPMU_INT_L3C		BIT(2)
+#define  PCPPMU_INT_IOB		BIT(3)
+
+#define PMU_MAX_COUNTERS	4
+#define PMU_CNT_MAX_PERIOD	0x100000000ULL
+#define PMU_OVERFLOW_MASK	0xF
+#define PMU_PMCR_E		BIT(0)
+#define PMU_PMCR_P		BIT(1)
+
+#define PMU_PMEVCNTR0		0x000
+#define PMU_PMEVCNTR1		0x004
+#define PMU_PMEVCNTR2		0x008
+#define PMU_PMEVCNTR3		0x00C
+#define PMU_PMEVTYPER0		0x400
+#define PMU_PMEVTYPER1		0x404
+#define PMU_PMEVTYPER2		0x408
+#define PMU_PMEVTYPER3		0x40C
+#define PMU_PMAMR0		0xA00
+#define PMU_PMAMR1		0xA04
+#define PMU_PMCNTENSET		0xC00
+#define PMU_PMCNTENCLR		0xC20
+#define PMU_PMINTENSET		0xC40
+#define PMU_PMINTENCLR		0xC60
+#define PMU_PMOVSR		0xC80
+#define PMU_PMCR		0xE04
+
+#define to_pmu_dev(p)     container_of(p, struct xgene_pmu_dev, pmu)
+#define GET_CNTR(ev)      (ev->hw.idx)
+#define GET_EVENTID(ev)   (ev->hw.config & 0xFFULL)
+#define GET_AGENTID(ev)   (ev->hw.config_base & 0xFFFFFFFFUL)
+#define GET_AGENT1ID(ev)  ((ev->hw.config_base >> 32) & 0xFFFFFFFFUL)
+
+struct hw_pmu_info {
+	u32 type;
+	u32 enable_mask;
+	void __iomem *csr;
+};
+
+struct xgene_pmu_dev {
+	struct hw_pmu_info *inf;
+	struct xgene_pmu *parent;
+	struct pmu pmu;
+	u8 max_counters;
+	DECLARE_BITMAP(cntr_assign_mask, PMU_MAX_COUNTERS);
+	u64 max_period;
+	const struct attribute_group **attr_groups;
+	struct perf_event *pmu_counter_event[PMU_MAX_COUNTERS];
+};
+
+struct xgene_pmu {
+	struct device *dev;
+	int version;
+	void __iomem *pcppmu_csr;
+	u32 mcb_active_mask;
+	u32 mc_active_mask;
+	cpumask_t cpu;
+	raw_spinlock_t lock;
+	struct list_head l3cpmus;
+	struct list_head iobpmus;
+	struct list_head mcbpmus;
+	struct list_head mcpmus;
+};
+
+struct xgene_pmu_dev_ctx {
+	char *name;
+	struct list_head next;
+	struct xgene_pmu_dev *pmu_dev;
+	struct hw_pmu_info inf;
+};
+
+struct xgene_pmu_data {
+	int id;
+	u32 data;
+};
+
+enum xgene_pmu_version {
+	PCP_PMU_V1 = 1,
+	PCP_PMU_V2,
+};
+
+enum xgene_pmu_dev_type {
+	PMU_TYPE_L3C = 0,
+	PMU_TYPE_IOB,
+	PMU_TYPE_MCB,
+	PMU_TYPE_MC,
+};
+
+/*
+ * sysfs format attributes
+ */
+static ssize_t xgene_pmu_format_show(struct device *dev,
+				     struct device_attribute *attr, char *buf)
+{
+	struct dev_ext_attribute *eattr;
+
+	eattr = container_of(attr, struct dev_ext_attribute, attr);
+	return sprintf(buf, "%s\n", (char *) eattr->var);
+}
+
+#define XGENE_PMU_FORMAT_ATTR(_name, _config)		\
+	(&((struct dev_ext_attribute[]) {		\
+		{ .attr = __ATTR(_name, S_IRUGO, xgene_pmu_format_show, NULL), \
+		  .var = (void *) _config, }		\
+	})[0].attr.attr)
+
+static struct attribute *l3c_pmu_format_attrs[] = {
+	XGENE_PMU_FORMAT_ATTR(l3c_eventid, "config:0-7"),
+	XGENE_PMU_FORMAT_ATTR(l3c_agentid, "config1:0-9"),
+	NULL,
+};
+
+static struct attribute *iob_pmu_format_attrs[] = {
+	XGENE_PMU_FORMAT_ATTR(iob_eventid, "config:0-7"),
+	XGENE_PMU_FORMAT_ATTR(iob_agentid, "config1:0-63"),
+	NULL,
+};
+
+static struct attribute *mcb_pmu_format_attrs[] = {
+	XGENE_PMU_FORMAT_ATTR(mcb_eventid, "config:0-5"),
+	XGENE_PMU_FORMAT_ATTR(mcb_agentid, "config1:0-9"),
+	NULL,
+};
+
+static struct attribute *mc_pmu_format_attrs[] = {
+	XGENE_PMU_FORMAT_ATTR(mc_eventid, "config:0-28"),
+	NULL,
+};
+
+static const struct attribute_group l3c_pmu_format_attr_group = {
+	.name = "format",
+	.attrs = l3c_pmu_format_attrs,
+};
+
+static const struct attribute_group iob_pmu_format_attr_group = {
+	.name = "format",
+	.attrs = iob_pmu_format_attrs,
+};
+
+static const struct attribute_group mcb_pmu_format_attr_group = {
+	.name = "format",
+	.attrs = mcb_pmu_format_attrs,
+};
+
+static const struct attribute_group mc_pmu_format_attr_group = {
+	.name = "format",
+	.attrs = mc_pmu_format_attrs,
+};
+
+/*
+ * sysfs event attributes
+ */
+static ssize_t xgene_pmu_event_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct dev_ext_attribute *eattr;
+
+	eattr = container_of(attr, struct dev_ext_attribute, attr);
+	return sprintf(buf, "config=0x%lx\n", (unsigned long) eattr->var);
+}
+
+#define XGENE_PMU_EVENT_ATTR(_name, _config)		\
+	(&((struct dev_ext_attribute[]) {		\
+		{ .attr = __ATTR(_name, S_IRUGO, xgene_pmu_event_show, NULL), \
+		  .var = (void *) _config, }		\
+	 })[0].attr.attr)
+
+static struct attribute *l3c_pmu_events_attrs[] = {
+	XGENE_PMU_EVENT_ATTR(cycle-count,			0x00),
+	XGENE_PMU_EVENT_ATTR(cycle-count-div-64,		0x01),
+	XGENE_PMU_EVENT_ATTR(read-hit,				0x02),
+	XGENE_PMU_EVENT_ATTR(read-miss,				0x03),
+	XGENE_PMU_EVENT_ATTR(write-need-replacement,		0x06),
+	XGENE_PMU_EVENT_ATTR(write-not-need-replacement,	0x07),
+	XGENE_PMU_EVENT_ATTR(tq-full,				0x08),
+	XGENE_PMU_EVENT_ATTR(ackq-full,				0x09),
+	XGENE_PMU_EVENT_ATTR(wdb-full,				0x0a),
+	XGENE_PMU_EVENT_ATTR(bank-fifo-full,			0x0b),
+	XGENE_PMU_EVENT_ATTR(odb-full,				0x0c),
+	XGENE_PMU_EVENT_ATTR(wbq-full,				0x0d),
+	XGENE_PMU_EVENT_ATTR(bank-conflict-fifo-issue,		0x0e),
+	XGENE_PMU_EVENT_ATTR(bank-fifo-issue,			0x0f),
+	NULL,
+};
+
+static struct attribute *iob_pmu_events_attrs[] = {
+	XGENE_PMU_EVENT_ATTR(cycle-count,			0x00),
+	XGENE_PMU_EVENT_ATTR(cycle-count-div-64,		0x01),
+	XGENE_PMU_EVENT_ATTR(axi0-read,				0x02),
+	XGENE_PMU_EVENT_ATTR(axi0-read-partial,			0x03),
+	XGENE_PMU_EVENT_ATTR(axi1-read,				0x04),
+	XGENE_PMU_EVENT_ATTR(axi1-read-partial,			0x05),
+	XGENE_PMU_EVENT_ATTR(csw-read-block,			0x06),
+	XGENE_PMU_EVENT_ATTR(csw-read-partial,			0x07),
+	XGENE_PMU_EVENT_ATTR(axi0-write,			0x10),
+	XGENE_PMU_EVENT_ATTR(axi0-write-partial,		0x11),
+	XGENE_PMU_EVENT_ATTR(axi1-write,			0x13),
+	XGENE_PMU_EVENT_ATTR(axi1-write-partial,		0x14),
+	XGENE_PMU_EVENT_ATTR(csw-inbound-dirty,			0x16),
+	NULL,
+};
+
+static struct attribute *mcb_pmu_events_attrs[] = {
+	XGENE_PMU_EVENT_ATTR(cycle-count,			0x00),
+	XGENE_PMU_EVENT_ATTR(cycle-count-div-64,		0x01),
+	XGENE_PMU_EVENT_ATTR(csw-read,				0x02),
+	XGENE_PMU_EVENT_ATTR(csw-write-request,			0x03),
+	XGENE_PMU_EVENT_ATTR(mcb-csw-stall,			0x04),
+	XGENE_PMU_EVENT_ATTR(cancel-read-gack,			0x05),
+	NULL,
+};
+
+static struct attribute *mc_pmu_events_attrs[] = {
+	XGENE_PMU_EVENT_ATTR(cycle-count,			0x00),
+	XGENE_PMU_EVENT_ATTR(cycle-count-div-64,		0x01),
+	XGENE_PMU_EVENT_ATTR(act-cmd-sent,			0x02),
+	XGENE_PMU_EVENT_ATTR(pre-cmd-sent,			0x03),
+	XGENE_PMU_EVENT_ATTR(rd-cmd-sent,			0x04),
+	XGENE_PMU_EVENT_ATTR(rda-cmd-sent,			0x05),
+	XGENE_PMU_EVENT_ATTR(wr-cmd-sent,			0x06),
+	XGENE_PMU_EVENT_ATTR(wra-cmd-sent,			0x07),
+	XGENE_PMU_EVENT_ATTR(pde-cmd-sent,			0x08),
+	XGENE_PMU_EVENT_ATTR(sre-cmd-sent,			0x09),
+	XGENE_PMU_EVENT_ATTR(prea-cmd-sent,			0x0a),
+	XGENE_PMU_EVENT_ATTR(ref-cmd-sent,			0x0b),
+	XGENE_PMU_EVENT_ATTR(rd-rda-cmd-sent,			0x0c),
+	XGENE_PMU_EVENT_ATTR(wr-wra-cmd-sent,			0x0d),
+	XGENE_PMU_EVENT_ATTR(in-rd-collision,			0x0e),
+	XGENE_PMU_EVENT_ATTR(in-wr-collision,			0x0f),
+	XGENE_PMU_EVENT_ATTR(collision-queue-not-empty,		0x10),
+	XGENE_PMU_EVENT_ATTR(collision-queue-full,		0x11),
+	XGENE_PMU_EVENT_ATTR(mcu-request,			0x12),
+	XGENE_PMU_EVENT_ATTR(mcu-rd-request,			0x13),
+	XGENE_PMU_EVENT_ATTR(mcu-hp-rd-request,			0x14),
+	XGENE_PMU_EVENT_ATTR(mcu-wr-request,			0x15),
+	XGENE_PMU_EVENT_ATTR(mcu-rd-proceed-all,		0x16),
+	XGENE_PMU_EVENT_ATTR(mcu-rd-proceed-cancel,		0x17),
+	XGENE_PMU_EVENT_ATTR(mcu-rd-response,			0x18),
+	XGENE_PMU_EVENT_ATTR(mcu-rd-proceed-speculative-all,	0x19),
+	XGENE_PMU_EVENT_ATTR(mcu-rd-proceed-speculative-cancel,	0x1a),
+	XGENE_PMU_EVENT_ATTR(mcu-wr-proceed-all,		0x1b),
+	XGENE_PMU_EVENT_ATTR(mcu-wr-proceed-cancel,		0x1c),
+	NULL,
+};
+
+static const struct attribute_group l3c_pmu_events_attr_group = {
+	.name = "events",
+	.attrs = l3c_pmu_events_attrs,
+};
+
+static const struct attribute_group iob_pmu_events_attr_group = {
+	.name = "events",
+	.attrs = iob_pmu_events_attrs,
+};
+
+static const struct attribute_group mcb_pmu_events_attr_group = {
+	.name = "events",
+	.attrs = mcb_pmu_events_attrs,
+};
+
+static const struct attribute_group mc_pmu_events_attr_group = {
+	.name = "events",
+	.attrs = mc_pmu_events_attrs,
+};
+
+/*
+ * sysfs cpumask attributes
+ */
+static ssize_t xgene_pmu_cpumask_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(dev_get_drvdata(dev));
+
+	return cpumap_print_to_pagebuf(true, buf, &pmu_dev->parent->cpu);
+}
+
+static DEVICE_ATTR(cpumask, S_IRUGO, xgene_pmu_cpumask_show, NULL);
+
+static struct attribute *xgene_pmu_cpumask_attrs[] = {
+	&dev_attr_cpumask.attr,
+	NULL,
+};
+
+static const struct attribute_group pmu_cpumask_attr_group = {
+	.attrs = xgene_pmu_cpumask_attrs,
+};
+
+/*
+ * Per PMU device attribute groups
+ */
+static const struct attribute_group *l3c_pmu_attr_groups[] = {
+	&l3c_pmu_format_attr_group,
+	&pmu_cpumask_attr_group,
+	&l3c_pmu_events_attr_group,
+	NULL
+};
+
+static const struct attribute_group *iob_pmu_attr_groups[] = {
+	&iob_pmu_format_attr_group,
+	&pmu_cpumask_attr_group,
+	&iob_pmu_events_attr_group,
+	NULL
+};
+
+static const struct attribute_group *mcb_pmu_attr_groups[] = {
+	&mcb_pmu_format_attr_group,
+	&pmu_cpumask_attr_group,
+	&mcb_pmu_events_attr_group,
+	NULL
+};
+
+static const struct attribute_group *mc_pmu_attr_groups[] = {
+	&mc_pmu_format_attr_group,
+	&pmu_cpumask_attr_group,
+	&mc_pmu_events_attr_group,
+	NULL
+};
+
+static int get_next_avail_cntr(struct xgene_pmu_dev *pmu_dev)
+{
+	int cntr;
+
+	cntr = find_first_zero_bit(pmu_dev->cntr_assign_mask,
+				pmu_dev->max_counters);
+	if (cntr == pmu_dev->max_counters)
+		return -ENOSPC;
+	set_bit(cntr, pmu_dev->cntr_assign_mask);
+
+	return cntr;
+}
+
+static void clear_avail_cntr(struct xgene_pmu_dev *pmu_dev, int cntr)
+{
+	clear_bit(cntr, pmu_dev->cntr_assign_mask);
+}
+
+static inline void xgene_pmu_mask_int(struct xgene_pmu *xgene_pmu)
+{
+	writel(PCPPMU_INTENMASK, xgene_pmu->pcppmu_csr + PCPPMU_INTMASK_REG);
+}
+
+static inline void xgene_pmu_unmask_int(struct xgene_pmu *xgene_pmu)
+{
+	writel(PCPPMU_INTCLRMASK, xgene_pmu->pcppmu_csr + PCPPMU_INTMASK_REG);
+}
+
+static inline u32 xgene_pmu_read_counter(struct xgene_pmu_dev *pmu_dev, int idx)
+{
+	return readl(pmu_dev->inf->csr + PMU_PMEVCNTR0 + (4 * idx));
+}
+
+static inline void
+xgene_pmu_write_counter(struct xgene_pmu_dev *pmu_dev, int idx, u32 val)
+{
+	writel(val, pmu_dev->inf->csr + PMU_PMEVCNTR0 + (4 * idx));
+}
+
+static inline void
+xgene_pmu_write_evttype(struct xgene_pmu_dev *pmu_dev, int idx, u32 val)
+{
+	writel(val, pmu_dev->inf->csr + PMU_PMEVTYPER0 + (4 * idx));
+}
+
+static inline void
+xgene_pmu_write_agentmsk(struct xgene_pmu_dev *pmu_dev, u32 val)
+{
+	writel(val, pmu_dev->inf->csr + PMU_PMAMR0);
+}
+
+static inline void
+xgene_pmu_write_agent1msk(struct xgene_pmu_dev *pmu_dev, u32 val)
+{
+	writel(val, pmu_dev->inf->csr + PMU_PMAMR1);
+}
+
+static inline void
+xgene_pmu_enable_counter(struct xgene_pmu_dev *pmu_dev, int idx)
+{
+	u32 val;
+
+	val = readl(pmu_dev->inf->csr + PMU_PMCNTENSET);
+	val |= 1 << idx;
+	writel(val, pmu_dev->inf->csr + PMU_PMCNTENSET);
+}
+
+static inline void
+xgene_pmu_disable_counter(struct xgene_pmu_dev *pmu_dev, int idx)
+{
+	u32 val;
+
+	val = readl(pmu_dev->inf->csr + PMU_PMCNTENCLR);
+	val |= 1 << idx;
+	writel(val, pmu_dev->inf->csr + PMU_PMCNTENCLR);
+}
+
+static inline void
+xgene_pmu_enable_counter_int(struct xgene_pmu_dev *pmu_dev, int idx)
+{
+	u32 val;
+
+	val = readl(pmu_dev->inf->csr + PMU_PMINTENSET);
+	val |= 1 << idx;
+	writel(val, pmu_dev->inf->csr + PMU_PMINTENSET);
+}
+
+static inline void
+xgene_pmu_disable_counter_int(struct xgene_pmu_dev *pmu_dev, int idx)
+{
+	u32 val;
+
+	val = readl(pmu_dev->inf->csr + PMU_PMINTENCLR);
+	val |= 1 << idx;
+	writel(val, pmu_dev->inf->csr + PMU_PMINTENCLR);
+}
+
+static inline void xgene_pmu_reset_counters(struct xgene_pmu_dev *pmu_dev)
+{
+	u32 val;
+
+	val = readl(pmu_dev->inf->csr + PMU_PMCR);
+	val |= PMU_PMCR_P;
+	writel(val, pmu_dev->inf->csr + PMU_PMCR);
+}
+
+static inline void xgene_pmu_start_counters(struct xgene_pmu_dev *pmu_dev)
+{
+	u32 val;
+
+	val = readl(pmu_dev->inf->csr + PMU_PMCR);
+	val |= PMU_PMCR_E;
+	writel(val, pmu_dev->inf->csr + PMU_PMCR);
+}
+
+static inline void xgene_pmu_stop_counters(struct xgene_pmu_dev *pmu_dev)
+{
+	u32 val;
+
+	val = readl(pmu_dev->inf->csr + PMU_PMCR);
+	val &= ~PMU_PMCR_E;
+	writel(val, pmu_dev->inf->csr + PMU_PMCR);
+}
+
+static void xgene_perf_pmu_enable(struct pmu *pmu)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(pmu);
+	int enabled = bitmap_weight(pmu_dev->cntr_assign_mask,
+			pmu_dev->max_counters);
+
+	if (!enabled)
+		return;
+
+	xgene_pmu_start_counters(pmu_dev);
+}
+
+static void xgene_perf_pmu_disable(struct pmu *pmu)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(pmu);
+
+	xgene_pmu_stop_counters(pmu_dev);
+}
+
+static int xgene_perf_event_init(struct perf_event *event)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);
+	struct hw_perf_event *hw = &event->hw;
+	struct perf_event *sibling;
+
+	/* Test the event attr type check for PMU enumeration */
+	if (event->attr.type != event->pmu->type)
+		return -ENOENT;
+
+	/*
+	 * SOC PMU counters are shared across all cores.
+	 * Therefore, it does not support per-process mode.
+	 * Also, it does not support event sampling mode.
+	 */
+	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
+		return -EINVAL;
+
+	/* SOC counters do not have usr/os/guest/host bits */
+	if (event->attr.exclude_user || event->attr.exclude_kernel ||
+	    event->attr.exclude_host || event->attr.exclude_guest)
+		return -EINVAL;
+
+	if (event->cpu < 0)
+		return -EINVAL;
+	/*
+	 * Many perf core operations (eg. events rotation) operate on a
+	 * single CPU context. This is obvious for CPU PMUs, where one
+	 * expects the same sets of events being observed on all CPUs,
+	 * but can lead to issues for off-core PMUs, where each
+	 * event could be theoretically assigned to a different CPU. To
+	 * mitigate this, we enforce CPU assignment to one, selected
+	 * processor (the one described in the "cpumask" attribute).
+	 */
+	event->cpu = cpumask_first(&pmu_dev->parent->cpu);
+
+	hw->config = event->attr.config;
+	/*
+	 * Each bit of the config1 field represents an agent from which the
+	 * request of the event come. The event is counted only if it's caused
+	 * by a request of an agent has the bit cleared.
+	 * By default, the event is counted for all agents.
+	 */
+	hw->config_base = event->attr.config1;
+
+	/*
+	 * We must NOT create groups containing mixed PMUs, although software
+	 * events are acceptable
+	 */
+	if (event->group_leader->pmu != event->pmu &&
+			!is_software_event(event->group_leader))
+		return -EINVAL;
+
+	list_for_each_entry(sibling, &event->group_leader->sibling_list,
+			group_entry)
+		if (sibling->pmu != event->pmu &&
+				!is_software_event(sibling))
+			return -EINVAL;
+
+	return 0;
+}
+
+static void xgene_perf_enable_event(struct perf_event *event)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);
+
+	xgene_pmu_write_evttype(pmu_dev, GET_CNTR(event), GET_EVENTID(event));
+	xgene_pmu_write_agentmsk(pmu_dev, ~((u32)GET_AGENTID(event)));
+	if (pmu_dev->inf->type == PMU_TYPE_IOB)
+		xgene_pmu_write_agent1msk(pmu_dev, ~((u32)GET_AGENT1ID(event)));
+
+	xgene_pmu_enable_counter(pmu_dev, GET_CNTR(event));
+	xgene_pmu_enable_counter_int(pmu_dev, GET_CNTR(event));
+}
+
+static void xgene_perf_disable_event(struct perf_event *event)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);
+
+	xgene_pmu_disable_counter(pmu_dev, GET_CNTR(event));
+	xgene_pmu_disable_counter_int(pmu_dev, GET_CNTR(event));
+}
+
+static void xgene_perf_event_set_period(struct perf_event *event)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);
+	struct hw_perf_event *hw = &event->hw;
+	/*
+	 * The X-Gene PMU counters have a period of 2^32. To account for the
+	 * possiblity of extreme interrupt latency we program for a period of
+	 * half that. Hopefully we can handle the interrupt before another 2^31
+	 * events occur and the counter overtakes its previous value.
+	 */
+	u64 val = 1ULL << 31;
+
+	local64_set(&hw->prev_count, val);
+	xgene_pmu_write_counter(pmu_dev, hw->idx, (u32) val);
+}
+
+static void xgene_perf_event_update(struct perf_event *event)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);
+	struct hw_perf_event *hw = &event->hw;
+	u64 delta, prev_raw_count, new_raw_count;
+
+again:
+	prev_raw_count = local64_read(&hw->prev_count);
+	new_raw_count = xgene_pmu_read_counter(pmu_dev, GET_CNTR(event));
+
+	if (local64_cmpxchg(&hw->prev_count, prev_raw_count,
+			    new_raw_count) != prev_raw_count)
+		goto again;
+
+	delta = (new_raw_count - prev_raw_count) & pmu_dev->max_period;
+
+	local64_add(delta, &event->count);
+}
+
+static void xgene_perf_read(struct perf_event *event)
+{
+	xgene_perf_event_update(event);
+}
+
+static void xgene_perf_start(struct perf_event *event, int flags)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);
+	struct hw_perf_event *hw = &event->hw;
+
+	if (WARN_ON_ONCE(!(hw->state & PERF_HES_STOPPED)))
+		return;
+
+	WARN_ON_ONCE(!(hw->state & PERF_HES_UPTODATE));
+	hw->state = 0;
+
+	xgene_perf_event_set_period(event);
+
+	if (flags & PERF_EF_RELOAD) {
+		u64 prev_raw_count =  local64_read(&hw->prev_count);
+
+		xgene_pmu_write_counter(pmu_dev, GET_CNTR(event),
+					(u32) prev_raw_count);
+	}
+
+	xgene_perf_enable_event(event);
+	perf_event_update_userpage(event);
+}
+
+static void xgene_perf_stop(struct perf_event *event, int flags)
+{
+	struct hw_perf_event *hw = &event->hw;
+	u64 config;
+
+	if (hw->state & PERF_HES_UPTODATE)
+		return;
+
+	xgene_perf_disable_event(event);
+	WARN_ON_ONCE(hw->state & PERF_HES_STOPPED);
+	hw->state |= PERF_HES_STOPPED;
+
+	if (hw->state & PERF_HES_UPTODATE)
+		return;
+
+	config = hw->config;
+	xgene_perf_read(event);
+	hw->state |= PERF_HES_UPTODATE;
+}
+
+static int xgene_perf_add(struct perf_event *event, int flags)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);
+	struct hw_perf_event *hw = &event->hw;
+
+	hw->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
+
+	/* Allocate an event counter */
+	hw->idx = get_next_avail_cntr(pmu_dev);
+	if (hw->idx < 0)
+		return -EAGAIN;
+
+	/* Update counter event pointer for Interrupt handler */
+	pmu_dev->pmu_counter_event[hw->idx] = event;
+
+	if (flags & PERF_EF_START)
+		xgene_perf_start(event, PERF_EF_RELOAD);
+
+	return 0;
+}
+
+static void xgene_perf_del(struct perf_event *event, int flags)
+{
+	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu);
+	struct hw_perf_event *hw = &event->hw;
+
+	xgene_perf_stop(event, PERF_EF_UPDATE);
+
+	/* clear the assigned counter */
+	clear_avail_cntr(pmu_dev, GET_CNTR(event));
+
+	perf_event_update_userpage(event);
+	pmu_dev->pmu_counter_event[hw->idx] = NULL;
+}
+
+static int xgene_init_perf(struct xgene_pmu_dev *pmu_dev, char *name)
+{
+	struct xgene_pmu *xgene_pmu;
+
+	pmu_dev->max_period = PMU_CNT_MAX_PERIOD - 1;
+	/* First version PMU supports only single event counter */
+	xgene_pmu = pmu_dev->parent;
+	if (xgene_pmu->version == PCP_PMU_V1)
+		pmu_dev->max_counters = 1;
+	else
+		pmu_dev->max_counters = PMU_MAX_COUNTERS;
+
+	/* Perf driver registration */
+	pmu_dev->pmu = (struct pmu) {
+		.attr_groups	= pmu_dev->attr_groups,
+		.task_ctx_nr	= perf_invalid_context,
+		.pmu_enable	= xgene_perf_pmu_enable,
+		.pmu_disable	= xgene_perf_pmu_disable,
+		.event_init	= xgene_perf_event_init,
+		.add		= xgene_perf_add,
+		.del		= xgene_perf_del,
+		.start		= xgene_perf_start,
+		.stop		= xgene_perf_stop,
+		.read		= xgene_perf_read,
+	};
+
+	/* Hardware counter init */
+	xgene_pmu_stop_counters(pmu_dev);
+	xgene_pmu_reset_counters(pmu_dev);
+
+	return perf_pmu_register(&pmu_dev->pmu, name, -1);
+}
+
+static int
+xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx)
+{
+	struct device *dev = xgene_pmu->dev;
+	struct xgene_pmu_dev *pmu;
+	int rc;
+
+	pmu = devm_kzalloc(dev, sizeof(*pmu), GFP_KERNEL);
+	if (!pmu)
+		return -ENOMEM;
+	pmu->parent = xgene_pmu;
+	pmu->inf = &ctx->inf;
+	ctx->pmu_dev = pmu;
+
+	switch (pmu->inf->type) {
+	case PMU_TYPE_L3C:
+		pmu->attr_groups = l3c_pmu_attr_groups;
+		break;
+	case PMU_TYPE_IOB:
+		pmu->attr_groups = iob_pmu_attr_groups;
+		break;
+	case PMU_TYPE_MCB:
+		if (!(xgene_pmu->mcb_active_mask & pmu->inf->enable_mask))
+			goto dev_err;
+		pmu->attr_groups = mcb_pmu_attr_groups;
+		break;
+	case PMU_TYPE_MC:
+		if (!(xgene_pmu->mc_active_mask & pmu->inf->enable_mask))
+			goto dev_err;
+		pmu->attr_groups = mc_pmu_attr_groups;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	rc = xgene_init_perf(pmu, ctx->name);
+	if (rc) {
+		dev_err(dev, "%s PMU: Failed to init perf driver\n", ctx->name);
+		goto dev_err;
+	}
+
+	dev_info(dev, "%s PMU registered\n", ctx->name);
+
+	return rc;
+
+dev_err:
+	devm_kfree(dev, pmu);
+	return -ENODEV;
+}
+
+static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev)
+{
+	struct xgene_pmu *xgene_pmu = pmu_dev->parent;
+	u32 pmovsr;
+	int idx;
+
+	pmovsr = readl(pmu_dev->inf->csr + PMU_PMOVSR) & PMU_OVERFLOW_MASK;
+	if (!pmovsr)
+		return;
+
+	/* Clear interrupt flag */
+	if (xgene_pmu->version == PCP_PMU_V1)
+		writel(0x0, pmu_dev->inf->csr + PMU_PMOVSR);
+	else
+		writel(pmovsr, pmu_dev->inf->csr + PMU_PMOVSR);
+
+	for (idx = 0; idx < PMU_MAX_COUNTERS; idx++) {
+		struct perf_event *event = pmu_dev->pmu_counter_event[idx];
+		int overflowed = pmovsr & BIT(idx);
+
+		/* Ignore if we don't have an event. */
+		if (!event || !overflowed)
+			continue;
+		xgene_perf_event_update(event);
+		xgene_perf_event_set_period(event);
+	}
+}
+
+static irqreturn_t xgene_pmu_isr(int irq, void *dev_id)
+{
+	struct xgene_pmu_dev_ctx *ctx;
+	struct xgene_pmu *xgene_pmu = dev_id;
+	unsigned long flags;
+	u32 val;
+
+	raw_spin_lock_irqsave(&xgene_pmu->lock, flags);
+
+	/* Get Interrupt PMU source */
+	val = readl(xgene_pmu->pcppmu_csr + PCPPMU_INTSTATUS_REG);
+	if (val & PCPPMU_INT_MCU) {
+		list_for_each_entry(ctx, &xgene_pmu->mcpmus, next) {
+			_xgene_pmu_isr(irq, ctx->pmu_dev);
+		}
+	}
+	if (val & PCPPMU_INT_MCB) {
+		list_for_each_entry(ctx, &xgene_pmu->mcbpmus, next) {
+			_xgene_pmu_isr(irq, ctx->pmu_dev);
+		}
+	}
+	if (val & PCPPMU_INT_L3C) {
+		list_for_each_entry(ctx, &xgene_pmu->l3cpmus, next) {
+			_xgene_pmu_isr(irq, ctx->pmu_dev);
+		}
+	}
+	if (val & PCPPMU_INT_IOB) {
+		list_for_each_entry(ctx, &xgene_pmu->iobpmus, next) {
+			_xgene_pmu_isr(irq, ctx->pmu_dev);
+		}
+	}
+
+	raw_spin_unlock_irqrestore(&xgene_pmu->lock, flags);
+
+	return IRQ_HANDLED;
+}
+
+static int acpi_pmu_probe_active_mcb_mcu(struct xgene_pmu *xgene_pmu,
+					 struct platform_device *pdev)
+{
+	void __iomem *csw_csr, *mcba_csr, *mcbb_csr;
+	struct resource *res;
+	unsigned int reg;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	csw_csr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(csw_csr)) {
+		dev_err(&pdev->dev, "ioremap failed for CSW CSR resource\n");
+		return PTR_ERR(csw_csr);
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
+	mcba_csr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mcba_csr)) {
+		dev_err(&pdev->dev, "ioremap failed for MCBA CSR resource\n");
+		return PTR_ERR(mcba_csr);
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 3);
+	mcbb_csr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mcbb_csr)) {
+		dev_err(&pdev->dev, "ioremap failed for MCBB CSR resource\n");
+		return PTR_ERR(mcbb_csr);
+	}
+
+	reg = readl(csw_csr + CSW_CSWCR);
+	if (reg & CSW_CSWCR_DUALMCB_MASK) {
+		/* Dual MCB active */
+		xgene_pmu->mcb_active_mask = 0x3;
+		/* Probe all active MC(s) */
+		reg = readl(mcbb_csr + CSW_CSWCR);
+		xgene_pmu->mc_active_mask =
+			(reg & MCBADDRMR_DUALMCU_MODE_MASK) ? 0xF : 0x5;
+	} else {
+		/* Single MCB active */
+		xgene_pmu->mcb_active_mask = 0x1;
+		/* Probe all active MC(s) */
+		reg = readl(mcba_csr + CSW_CSWCR);
+		xgene_pmu->mc_active_mask =
+			(reg & MCBADDRMR_DUALMCU_MODE_MASK) ? 0x3 : 0x1;
+	}
+
+	return 0;
+}
+
+static int fdt_pmu_probe_active_mcb_mcu(struct xgene_pmu *xgene_pmu,
+					struct platform_device *pdev)
+{
+	struct regmap *csw_map, *mcba_map, *mcbb_map;
+	struct device_node *np = pdev->dev.of_node;
+	unsigned int reg;
+
+	csw_map = syscon_regmap_lookup_by_phandle(np, "regmap-csw");
+	if (IS_ERR(csw_map)) {
+		dev_err(&pdev->dev, "unable to get syscon regmap csw\n");
+		return PTR_ERR(csw_map);
+	}
+
+	mcba_map = syscon_regmap_lookup_by_phandle(np, "regmap-mcba");
+	if (IS_ERR(mcba_map)) {
+		dev_err(&pdev->dev, "unable to get syscon regmap mcba\n");
+		return PTR_ERR(mcba_map);
+	}
+
+	mcbb_map = syscon_regmap_lookup_by_phandle(np, "regmap-mcbb");
+	if (IS_ERR(mcbb_map)) {
+		dev_err(&pdev->dev, "unable to get syscon regmap mcbb\n");
+		return PTR_ERR(mcbb_map);
+	}
+
+	if (regmap_read(csw_map, CSW_CSWCR, &reg))
+		return -EINVAL;
+
+	if (reg & CSW_CSWCR_DUALMCB_MASK) {
+		/* Dual MCB active */
+		xgene_pmu->mcb_active_mask = 0x3;
+		/* Probe all active MC(s) */
+		if (regmap_read(mcbb_map, MCBADDRMR, &reg))
+			return 0;
+		xgene_pmu->mc_active_mask =
+			(reg & MCBADDRMR_DUALMCU_MODE_MASK) ? 0xF : 0x5;
+	} else {
+		/* Single MCB active */
+		xgene_pmu->mcb_active_mask = 0x1;
+		/* Probe all active MC(s) */
+		if (regmap_read(mcba_map, MCBADDRMR, &reg))
+			return 0;
+		xgene_pmu->mc_active_mask =
+			(reg & MCBADDRMR_DUALMCU_MODE_MASK) ? 0x3 : 0x1;
+	}
+
+	return 0;
+}
+
+static int xgene_pmu_probe_active_mcb_mcu(struct xgene_pmu *xgene_pmu,
+					  struct platform_device *pdev)
+{
+	if (has_acpi_companion(&pdev->dev))
+		return acpi_pmu_probe_active_mcb_mcu(xgene_pmu, pdev);
+	return fdt_pmu_probe_active_mcb_mcu(xgene_pmu, pdev);
+}
+
+static char *xgene_pmu_dev_name(struct device *dev, u32 type, int id)
+{
+	switch (type) {
+	case PMU_TYPE_L3C:
+		return devm_kasprintf(dev, GFP_KERNEL, "l3c%d", id);
+	case PMU_TYPE_IOB:
+		return devm_kasprintf(dev, GFP_KERNEL, "iob%d", id);
+	case PMU_TYPE_MCB:
+		return devm_kasprintf(dev, GFP_KERNEL, "mcb%d", id);
+	case PMU_TYPE_MC:
+		return devm_kasprintf(dev, GFP_KERNEL, "mc%d", id);
+	default:
+		return devm_kasprintf(dev, GFP_KERNEL, "unknown");
+	}
+}
+
+#if defined(CONFIG_ACPI)
+static int acpi_pmu_dev_add_resource(struct acpi_resource *ares, void *data)
+{
+	struct resource *res = data;
+
+	if (ares->type == ACPI_RESOURCE_TYPE_FIXED_MEMORY32)
+		acpi_dev_resource_memory(ares, res);
+
+	/* Always tell the ACPI core to skip this resource */
+	return 1;
+}
+
+static struct
+xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu,
+				       struct acpi_device *adev, u32 type)
+{
+	struct device *dev = xgene_pmu->dev;
+	struct list_head resource_list;
+	struct xgene_pmu_dev_ctx *ctx;
+	const union acpi_object *obj;
+	struct hw_pmu_info *inf;
+	void __iomem *dev_csr;
+	struct resource res;
+	int enable_bit;
+	int rc;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return NULL;
+
+	INIT_LIST_HEAD(&resource_list);
+	rc = acpi_dev_get_resources(adev, &resource_list,
+				    acpi_pmu_dev_add_resource, &res);
+	acpi_dev_free_resource_list(&resource_list);
+	if (rc < 0 || IS_ERR(&res)) {
+		dev_err(dev, "PMU type %d: No resource address found\n", type);
+		goto err;
+	}
+
+	dev_csr = devm_ioremap_resource(dev, &res);
+	if (IS_ERR(dev_csr)) {
+		dev_err(dev, "PMU type %d: Fail to map resource\n", type);
+		goto err;
+	}
+
+	/* A PMU device node without enable-bit-index is always enabled */
+	rc = acpi_dev_get_property(adev, "enable-bit-index",
+				   ACPI_TYPE_INTEGER, &obj);
+	if (rc < 0)
+		enable_bit = 0;
+	else
+		enable_bit = (int) obj->integer.value;
+
+	ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);
+	if (!ctx->name) {
+		dev_err(dev, "PMU type %d: Fail to get device name\n", type);
+		goto err;
+	}
+	inf = &ctx->inf;
+	inf->type = type;
+	inf->csr = dev_csr;
+	inf->enable_mask = 1 << enable_bit;
+
+	return ctx;
+err:
+	devm_kfree(dev, ctx);
+	return NULL;
+}
+
+static acpi_status acpi_pmu_dev_add(acpi_handle handle, u32 level,
+				    void *data, void **return_value)
+{
+	struct xgene_pmu *xgene_pmu = data;
+	struct xgene_pmu_dev_ctx *ctx;
+	struct acpi_device *adev;
+
+	if (acpi_bus_get_device(handle, &adev))
+		return AE_OK;
+	if (acpi_bus_get_status(adev) || !adev->status.present)
+		return AE_OK;
+
+	if (!strcmp(acpi_device_hid(adev), "APMC0D5D"))
+		ctx = acpi_get_pmu_hw_inf(xgene_pmu, adev, PMU_TYPE_L3C);
+	else if (!strcmp(acpi_device_hid(adev), "APMC0D5E"))
+		ctx = acpi_get_pmu_hw_inf(xgene_pmu, adev, PMU_TYPE_IOB);
+	else if (!strcmp(acpi_device_hid(adev), "APMC0D5F"))
+		ctx = acpi_get_pmu_hw_inf(xgene_pmu, adev, PMU_TYPE_MCB);
+	else if (!strcmp(acpi_device_hid(adev), "APMC0D60"))
+		ctx = acpi_get_pmu_hw_inf(xgene_pmu, adev, PMU_TYPE_MC);
+	else
+		ctx = NULL;
+
+	if (!ctx)
+		return AE_OK;
+
+	if (xgene_pmu_dev_add(xgene_pmu, ctx)) {
+		/* Can't add the PMU device, skip it */
+		devm_kfree(xgene_pmu->dev, ctx);
+		return AE_OK;
+	}
+
+	switch (ctx->inf.type) {
+	case PMU_TYPE_L3C:
+		list_add(&ctx->next, &xgene_pmu->l3cpmus);
+		break;
+	case PMU_TYPE_IOB:
+		list_add(&ctx->next, &xgene_pmu->iobpmus);
+		break;
+	case PMU_TYPE_MCB:
+		list_add(&ctx->next, &xgene_pmu->mcbpmus);
+		break;
+	case PMU_TYPE_MC:
+		list_add(&ctx->next, &xgene_pmu->mcpmus);
+		break;
+	}
+	return AE_OK;
+}
+
+static int acpi_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu,
+				  struct platform_device *pdev)
+{
+	struct device *dev = xgene_pmu->dev;
+	acpi_handle handle;
+	acpi_status status;
+
+	handle = ACPI_HANDLE(dev);
+	if (!handle)
+		return -EINVAL;
+
+	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1,
+				     acpi_pmu_dev_add, NULL, xgene_pmu, NULL);
+	if (ACPI_FAILURE(status)) {
+		dev_err(dev, "failed to probe PMU devices\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+#else
+static int acpi_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu,
+				  struct platform_device *pdev)
+{
+	return 0;
+}
+#endif
+
+static struct
+xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu,
+				      struct device_node *np, u32 type)
+{
+	struct device *dev = xgene_pmu->dev;
+	struct xgene_pmu_dev_ctx *ctx;
+	struct hw_pmu_info *inf;
+	void __iomem *dev_csr;
+	struct resource res;
+	int enable_bit;
+	int rc;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return NULL;
+	rc = of_address_to_resource(np, 0, &res);
+	if (rc < 0) {
+		dev_err(dev, "PMU type %d: No resource address found\n", type);
+		goto err;
+	}
+	dev_csr = devm_ioremap_resource(dev, &res);
+	if (IS_ERR(dev_csr)) {
+		dev_err(dev, "PMU type %d: Fail to map resource\n", type);
+		goto err;
+	}
+
+	/* A PMU device node without enable-bit-index is always enabled */
+	if (of_property_read_u32(np, "enable-bit-index", &enable_bit))
+		enable_bit = 0;
+
+	ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);
+	if (!ctx->name) {
+		dev_err(dev, "PMU type %d: Fail to get device name\n", type);
+		goto err;
+	}
+	inf = &ctx->inf;
+	inf->type = type;
+	inf->csr = dev_csr;
+	inf->enable_mask = 1 << enable_bit;
+
+	return ctx;
+err:
+	devm_kfree(dev, ctx);
+	return NULL;
+}
+
+static int fdt_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu,
+				 struct platform_device *pdev)
+{
+	struct xgene_pmu_dev_ctx *ctx;
+	struct device_node *np;
+
+	for_each_child_of_node(pdev->dev.of_node, np) {
+		if (!of_device_is_available(np))
+			continue;
+
+		if (of_device_is_compatible(np, "apm,xgene-pmu-l3c"))
+			ctx = fdt_get_pmu_hw_inf(xgene_pmu, np, PMU_TYPE_L3C);
+		else if (of_device_is_compatible(np, "apm,xgene-pmu-iob"))
+			ctx = fdt_get_pmu_hw_inf(xgene_pmu, np, PMU_TYPE_IOB);
+		else if (of_device_is_compatible(np, "apm,xgene-pmu-mcb"))
+			ctx = fdt_get_pmu_hw_inf(xgene_pmu, np, PMU_TYPE_MCB);
+		else if (of_device_is_compatible(np, "apm,xgene-pmu-mc"))
+			ctx = fdt_get_pmu_hw_inf(xgene_pmu, np, PMU_TYPE_MC);
+		else
+			ctx = NULL;
+
+		if (!ctx)
+			continue;
+
+		if (xgene_pmu_dev_add(xgene_pmu, ctx)) {
+			/* Can't add the PMU device, skip it */
+			devm_kfree(xgene_pmu->dev, ctx);
+			continue;
+		}
+
+		switch (ctx->inf.type) {
+		case PMU_TYPE_L3C:
+			list_add(&ctx->next, &xgene_pmu->l3cpmus);
+			break;
+		case PMU_TYPE_IOB:
+			list_add(&ctx->next, &xgene_pmu->iobpmus);
+			break;
+		case PMU_TYPE_MCB:
+			list_add(&ctx->next, &xgene_pmu->mcbpmus);
+			break;
+		case PMU_TYPE_MC:
+			list_add(&ctx->next, &xgene_pmu->mcpmus);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int xgene_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu,
+				   struct platform_device *pdev)
+{
+	if (has_acpi_companion(&pdev->dev))
+		return acpi_pmu_probe_pmu_dev(xgene_pmu, pdev);
+	return fdt_pmu_probe_pmu_dev(xgene_pmu, pdev);
+}
+
+static const struct xgene_pmu_data xgene_pmu_data = {
+	.id   = PCP_PMU_V1,
+};
+
+static const struct xgene_pmu_data xgene_pmu_v2_data = {
+	.id   = PCP_PMU_V2,
+};
+
+static const struct of_device_id xgene_pmu_of_match[] = {
+	{ .compatible	= "apm,xgene-pmu",	.data = &xgene_pmu_data },
+	{ .compatible	= "apm,xgene-pmu-v2",	.data = &xgene_pmu_v2_data },
+	{},
+};
+MODULE_DEVICE_TABLE(of, xgene_pmu_of_match);
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id xgene_pmu_acpi_match[] = {
+	{"APMC0D5B", PCP_PMU_V1},
+	{"APMC0D5C", PCP_PMU_V2},
+	{},
+};
+MODULE_DEVICE_TABLE(acpi, xgene_pmu_acpi_match);
+#endif
+
+static int xgene_pmu_probe(struct platform_device *pdev)
+{
+	const struct xgene_pmu_data *dev_data;
+	const struct of_device_id *of_id;
+	struct xgene_pmu *xgene_pmu;
+	struct resource *res;
+	int irq, rc;
+	int version;
+
+	xgene_pmu = devm_kzalloc(&pdev->dev, sizeof(*xgene_pmu), GFP_KERNEL);
+	if (!xgene_pmu)
+		return -ENOMEM;
+	xgene_pmu->dev = &pdev->dev;
+	platform_set_drvdata(pdev, xgene_pmu);
+
+	version = -EINVAL;
+	of_id = of_match_device(xgene_pmu_of_match, &pdev->dev);
+	if (of_id) {
+		dev_data = (const struct xgene_pmu_data *) of_id->data;
+		version = dev_data->id;
+	}
+
+#ifdef CONFIG_ACPI
+	if (ACPI_COMPANION(&pdev->dev)) {
+		const struct acpi_device_id *acpi_id;
+
+		acpi_id = acpi_match_device(xgene_pmu_acpi_match, &pdev->dev);
+		if (acpi_id)
+			version = (int) acpi_id->driver_data;
+	}
+#endif
+	if (version < 0)
+		return -ENODEV;
+
+	INIT_LIST_HEAD(&xgene_pmu->l3cpmus);
+	INIT_LIST_HEAD(&xgene_pmu->iobpmus);
+	INIT_LIST_HEAD(&xgene_pmu->mcbpmus);
+	INIT_LIST_HEAD(&xgene_pmu->mcpmus);
+
+	xgene_pmu->version = version;
+	dev_info(&pdev->dev, "X-Gene PMU version %d\n", xgene_pmu->version);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	xgene_pmu->pcppmu_csr = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(xgene_pmu->pcppmu_csr)) {
+		dev_err(&pdev->dev, "ioremap failed for PCP PMU resource\n");
+		rc = PTR_ERR(xgene_pmu->pcppmu_csr);
+		goto err;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "No IRQ resource\n");
+		rc = -EINVAL;
+		goto err;
+	}
+	rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr,
+				IRQF_NOBALANCING | IRQF_NO_THREAD,
+				dev_name(&pdev->dev), xgene_pmu);
+	if (rc) {
+		dev_err(&pdev->dev, "Could not request IRQ %d\n", irq);
+		goto err;
+	}
+
+	raw_spin_lock_init(&xgene_pmu->lock);
+
+	/* Check for active MCBs and MCUs */
+	rc = xgene_pmu_probe_active_mcb_mcu(xgene_pmu, pdev);
+	if (rc) {
+		dev_warn(&pdev->dev, "Unknown MCB/MCU active status\n");
+		xgene_pmu->mcb_active_mask = 0x1;
+		xgene_pmu->mc_active_mask = 0x1;
+	}
+
+	/* Pick one core to use for cpumask attributes */
+	cpumask_set_cpu(smp_processor_id(), &xgene_pmu->cpu);
+
+	/* Make sure that the overflow interrupt is handled by this CPU */
+	rc = irq_set_affinity(irq, &xgene_pmu->cpu);
+	if (rc) {
+		dev_err(&pdev->dev, "Failed to set interrupt affinity!\n");
+		goto err;
+	}
+
+	/* Walk through the tree for all PMU perf devices */
+	rc = xgene_pmu_probe_pmu_dev(xgene_pmu, pdev);
+	if (rc) {
+		dev_err(&pdev->dev, "No PMU perf devices found!\n");
+		goto err;
+	}
+
+	/* Enable interrupt */
+	xgene_pmu_unmask_int(xgene_pmu);
+
+	return 0;
+
+err:
+	if (xgene_pmu->pcppmu_csr)
+		devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr);
+	devm_kfree(&pdev->dev, xgene_pmu);
+
+	return rc;
+}
+
+static void
+xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus)
+{
+	struct xgene_pmu_dev_ctx *ctx;
+	struct device *dev = xgene_pmu->dev;
+	struct xgene_pmu_dev *pmu_dev;
+
+	list_for_each_entry(ctx, pmus, next) {
+		pmu_dev = ctx->pmu_dev;
+		if (pmu_dev->inf->csr)
+			devm_iounmap(dev, pmu_dev->inf->csr);
+		devm_kfree(dev, ctx);
+		devm_kfree(dev, pmu_dev);
+	}
+}
+
+static int xgene_pmu_remove(struct platform_device *pdev)
+{
+	struct xgene_pmu *xgene_pmu = dev_get_drvdata(&pdev->dev);
+
+	xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->l3cpmus);
+	xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->iobpmus);
+	xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcbpmus);
+	xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus);
+
+	if (xgene_pmu->pcppmu_csr)
+		devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr);
+	devm_kfree(&pdev->dev, xgene_pmu);
+
+	return 0;
+}
+
+static struct platform_driver xgene_pmu_driver = {
+	.probe = xgene_pmu_probe,
+	.remove = xgene_pmu_remove,
+	.driver = {
+		.name		= "xgene-pmu",
+		.of_match_table = xgene_pmu_of_match,
+		.acpi_match_table = ACPI_PTR(xgene_pmu_acpi_match),
+	},
+};
+
+builtin_platform_driver(xgene_pmu_driver);
diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig
index 19bff3a10f69..fe00f9134d51 100644
--- a/drivers/phy/Kconfig
+++ b/drivers/phy/Kconfig
@@ -24,6 +24,15 @@ config PHY_BCM_NS_USB2
 	  Enable this to support Broadcom USB 2.0 PHY connected to the USB
 	  controller on Northstar family.
 
+config PHY_BCM_NS_USB3
+	tristate "Broadcom Northstar USB 3.0 PHY Driver"
+	depends on ARCH_BCM_IPROC || COMPILE_TEST
+	depends on HAS_IOMEM && OF
+	select GENERIC_PHY
+	help
+	  Enable this to support Broadcom USB 3.0 PHY connected to the USB
+	  controller on Northstar family.
+
 config PHY_BERLIN_USB
 	tristate "Marvell Berlin USB PHY Driver"
 	depends on ARCH_BERLIN && RESET_CONTROLLER && HAS_IOMEM && OF
@@ -258,7 +267,9 @@ config PHY_SUN4I_USB
 	depends on RESET_CONTROLLER
 	depends on EXTCON
 	depends on POWER_SUPPLY
+	depends on USB_SUPPORT
 	select GENERIC_PHY
+	select USB_COMMON
 	help
 	  Enable this to support the transceiver that is part of Allwinner
 	  sunxi SoCs.
@@ -358,6 +369,14 @@ config PHY_ROCKCHIP_USB
 	help
 	  Enable this to support the Rockchip USB 2.0 PHY.
 
+config PHY_ROCKCHIP_INNO_USB2
+	tristate "Rockchip INNO USB2PHY Driver"
+	depends on (ARCH_ROCKCHIP || COMPILE_TEST) && OF
+	depends on COMMON_CLK
+	select GENERIC_PHY
+	help
+	  Support for Rockchip USB2.0 PHY with Innosilicon IP block.
+
 config PHY_ROCKCHIP_EMMC
 	tristate "Rockchip EMMC PHY Driver"
 	depends on ARCH_ROCKCHIP && OF
@@ -372,6 +391,23 @@ config PHY_ROCKCHIP_DP
 	help
 	  Enable this to support the Rockchip Display Port PHY.
 
+config PHY_ROCKCHIP_PCIE
+	tristate "Rockchip PCIe PHY Driver"
+	depends on (ARCH_ROCKCHIP && OF) || COMPILE_TEST
+	select GENERIC_PHY
+	select MFD_SYSCON
+	help
+	  Enable this to support the Rockchip PCIe PHY.
+
+config PHY_ROCKCHIP_TYPEC
+	tristate "Rockchip TYPEC PHY Driver"
+	depends on OF && (ARCH_ROCKCHIP || COMPILE_TEST)
+	select EXTCON
+	select GENERIC_PHY
+	select RESET_CONTROLLER
+	help
+	  Enable this to support the Rockchip USB TYPEC PHY.
+
 config PHY_ST_SPEAR1310_MIPHY
 	tristate "ST SPEAR1310-MIPHY driver"
 	select GENERIC_PHY
diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile
index 90ae19879b0a..a534cf5be07d 100644
--- a/drivers/phy/Makefile
+++ b/drivers/phy/Makefile
@@ -4,6 +4,7 @@
 
 obj-$(CONFIG_GENERIC_PHY)		+= phy-core.o
 obj-$(CONFIG_PHY_BCM_NS_USB2)		+= phy-bcm-ns-usb2.o
+obj-$(CONFIG_PHY_BCM_NS_USB3)		+= phy-bcm-ns-usb3.o
 obj-$(CONFIG_PHY_BERLIN_USB)		+= phy-berlin-usb.o
 obj-$(CONFIG_PHY_BERLIN_SATA)		+= phy-berlin-sata.o
 obj-$(CONFIG_PHY_DA8XX_USB)		+= phy-da8xx-usb.o
@@ -39,8 +40,11 @@ phy-exynos-usb2-$(CONFIG_PHY_S5PV210_USB2)	+= phy-s5pv210-usb2.o
 obj-$(CONFIG_PHY_EXYNOS5_USBDRD)	+= phy-exynos5-usbdrd.o
 obj-$(CONFIG_PHY_QCOM_APQ8064_SATA)	+= phy-qcom-apq8064-sata.o
 obj-$(CONFIG_PHY_ROCKCHIP_USB) += phy-rockchip-usb.o
+obj-$(CONFIG_PHY_ROCKCHIP_INNO_USB2)	+= phy-rockchip-inno-usb2.o
 obj-$(CONFIG_PHY_ROCKCHIP_EMMC) += phy-rockchip-emmc.o
+obj-$(CONFIG_PHY_ROCKCHIP_PCIE) += phy-rockchip-pcie.o
 obj-$(CONFIG_PHY_ROCKCHIP_DP)		+= phy-rockchip-dp.o
+obj-$(CONFIG_PHY_ROCKCHIP_TYPEC) += phy-rockchip-typec.o
 obj-$(CONFIG_PHY_QCOM_IPQ806X_SATA)	+= phy-qcom-ipq806x-sata.o
 obj-$(CONFIG_PHY_ST_SPEAR1310_MIPHY)	+= phy-spear1310-miphy.o
 obj-$(CONFIG_PHY_ST_SPEAR1340_MIPHY)	+= phy-spear1340-miphy.o
diff --git a/drivers/phy/phy-bcm-ns-usb3.c b/drivers/phy/phy-bcm-ns-usb3.c
new file mode 100644
index 000000000000..f420fa4bebfc
--- /dev/null
+++ b/drivers/phy/phy-bcm-ns-usb3.c
@@ -0,0 +1,274 @@
+/*
+ * Broadcom Northstar USB 3.0 PHY Driver
+ *
+ * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
+ *
+ * All magic values used for initialization (and related comments) were obtained
+ * from Broadcom's SDK:
+ * Copyright (c) Broadcom Corp, 2012
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/bcma/bcma.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/phy/phy.h>
+#include <linux/slab.h>
+
+#define BCM_NS_USB3_MII_MNG_TIMEOUT_US	1000	/* usecs */
+
+enum bcm_ns_family {
+	BCM_NS_UNKNOWN,
+	BCM_NS_AX,
+	BCM_NS_BX,
+};
+
+struct bcm_ns_usb3 {
+	struct device *dev;
+	enum bcm_ns_family family;
+	void __iomem *dmp;
+	void __iomem *ccb_mii;
+	struct phy *phy;
+};
+
+static const struct of_device_id bcm_ns_usb3_id_table[] = {
+	{
+		.compatible = "brcm,ns-ax-usb3-phy",
+		.data = (int *)BCM_NS_AX,
+	},
+	{
+		.compatible = "brcm,ns-bx-usb3-phy",
+		.data = (int *)BCM_NS_BX,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, bcm_ns_usb3_id_table);
+
+static int bcm_ns_usb3_wait_reg(struct bcm_ns_usb3 *usb3, void __iomem *addr,
+				u32 mask, u32 value, unsigned long timeout)
+{
+	unsigned long deadline = jiffies + timeout;
+	u32 val;
+
+	do {
+		val = readl(addr);
+		if ((val & mask) == value)
+			return 0;
+		cpu_relax();
+		udelay(10);
+	} while (!time_after_eq(jiffies, deadline));
+
+	dev_err(usb3->dev, "Timeout waiting for register %p\n", addr);
+
+	return -EBUSY;
+}
+
+static inline int bcm_ns_usb3_mii_mng_wait_idle(struct bcm_ns_usb3 *usb3)
+{
+	return bcm_ns_usb3_wait_reg(usb3, usb3->ccb_mii + BCMA_CCB_MII_MNG_CTL,
+				    0x0100, 0x0000,
+				    usecs_to_jiffies(BCM_NS_USB3_MII_MNG_TIMEOUT_US));
+}
+
+static int bcm_ns_usb3_mii_mng_write32(struct bcm_ns_usb3 *usb3, u32 value)
+{
+	int err;
+
+	err = bcm_ns_usb3_mii_mng_wait_idle(usb3);
+	if (err < 0) {
+		dev_err(usb3->dev, "Couldn't write 0x%08x value\n", value);
+		return err;
+	}
+
+	writel(value, usb3->ccb_mii + BCMA_CCB_MII_MNG_CMD_DATA);
+
+	return 0;
+}
+
+static int bcm_ns_usb3_phy_init_ns_bx(struct bcm_ns_usb3 *usb3)
+{
+	int err;
+
+	/* Enable MDIO. Setting MDCDIV as 26  */
+	writel(0x0000009a, usb3->ccb_mii + BCMA_CCB_MII_MNG_CTL);
+
+	/* Wait for MDIO? */
+	udelay(2);
+
+	/* USB3 PLL Block */
+	err = bcm_ns_usb3_mii_mng_write32(usb3, 0x587e8000);
+	if (err < 0)
+		return err;
+
+	/* Assert Ana_Pllseq start */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x58061000);
+
+	/* Assert CML Divider ratio to 26 */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x582a6400);
+
+	/* Asserting PLL Reset */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x582ec000);
+
+	/* Deaaserting PLL Reset */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x582e8000);
+
+	/* Waiting MII Mgt interface idle */
+	bcm_ns_usb3_mii_mng_wait_idle(usb3);
+
+	/* Deasserting USB3 system reset */
+	writel(0, usb3->dmp + BCMA_RESET_CTL);
+
+	/* PLL frequency monitor enable */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x58069000);
+
+	/* PIPE Block */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x587e8060);
+
+	/* CMPMAX & CMPMINTH setting */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x580af30d);
+
+	/* DEGLITCH MIN & MAX setting */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x580e6302);
+
+	/* TXPMD block */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x587e8040);
+
+	/* Enabling SSC */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x58061003);
+
+	/* Waiting MII Mgt interface idle */
+	bcm_ns_usb3_mii_mng_wait_idle(usb3);
+
+	return 0;
+}
+
+static int bcm_ns_usb3_phy_init_ns_ax(struct bcm_ns_usb3 *usb3)
+{
+	int err;
+
+	/* Enable MDIO. Setting MDCDIV as 26  */
+	writel(0x0000009a, usb3->ccb_mii + BCMA_CCB_MII_MNG_CTL);
+
+	/* Wait for MDIO? */
+	udelay(2);
+
+	/* PLL30 block */
+	err = bcm_ns_usb3_mii_mng_write32(usb3, 0x587e8000);
+	if (err < 0)
+		return err;
+
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x582a6400);
+
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x587e80e0);
+
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x580a009c);
+
+	/* Enable SSC */
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x587e8040);
+
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x580a21d3);
+
+	bcm_ns_usb3_mii_mng_write32(usb3, 0x58061003);
+
+	/* Waiting MII Mgt interface idle */
+	bcm_ns_usb3_mii_mng_wait_idle(usb3);
+
+	/* Deasserting USB3 system reset */
+	writel(0, usb3->dmp + BCMA_RESET_CTL);
+
+	return 0;
+}
+
+static int bcm_ns_usb3_phy_init(struct phy *phy)
+{
+	struct bcm_ns_usb3 *usb3 = phy_get_drvdata(phy);
+	int err;
+
+	/* Perform USB3 system soft reset */
+	writel(BCMA_RESET_CTL_RESET, usb3->dmp + BCMA_RESET_CTL);
+
+	switch (usb3->family) {
+	case BCM_NS_AX:
+		err = bcm_ns_usb3_phy_init_ns_ax(usb3);
+		break;
+	case BCM_NS_BX:
+		err = bcm_ns_usb3_phy_init_ns_bx(usb3);
+		break;
+	default:
+		WARN_ON(1);
+		err = -ENOTSUPP;
+	}
+
+	return err;
+}
+
+static const struct phy_ops ops = {
+	.init		= bcm_ns_usb3_phy_init,
+	.owner		= THIS_MODULE,
+};
+
+static int bcm_ns_usb3_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const struct of_device_id *of_id;
+	struct bcm_ns_usb3 *usb3;
+	struct resource *res;
+	struct phy_provider *phy_provider;
+
+	usb3 = devm_kzalloc(dev, sizeof(*usb3), GFP_KERNEL);
+	if (!usb3)
+		return -ENOMEM;
+
+	usb3->dev = dev;
+
+	of_id = of_match_device(bcm_ns_usb3_id_table, dev);
+	if (!of_id)
+		return -EINVAL;
+	usb3->family = (enum bcm_ns_family)of_id->data;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dmp");
+	usb3->dmp = devm_ioremap_resource(dev, res);
+	if (IS_ERR(usb3->dmp)) {
+		dev_err(dev, "Failed to map DMP regs\n");
+		return PTR_ERR(usb3->dmp);
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ccb-mii");
+	usb3->ccb_mii = devm_ioremap_resource(dev, res);
+	if (IS_ERR(usb3->ccb_mii)) {
+		dev_err(dev, "Failed to map ChipCommon B MII regs\n");
+		return PTR_ERR(usb3->ccb_mii);
+	}
+
+	usb3->phy = devm_phy_create(dev, NULL, &ops);
+	if (IS_ERR(usb3->phy)) {
+		dev_err(dev, "Failed to create PHY\n");
+		return PTR_ERR(usb3->phy);
+	}
+
+	phy_set_drvdata(usb3->phy, usb3);
+	platform_set_drvdata(pdev, usb3);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	if (!IS_ERR(phy_provider))
+		dev_info(dev, "Registered Broadcom Northstar USB 3.0 PHY driver\n");
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static struct platform_driver bcm_ns_usb3_driver = {
+	.probe		= bcm_ns_usb3_probe,
+	.driver = {
+		.name = "bcm_ns_usb3",
+		.of_match_table = bcm_ns_usb3_id_table,
+	},
+};
+module_platform_driver(bcm_ns_usb3_driver);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/phy-bcm-ns2-pcie.c b/drivers/phy/phy-bcm-ns2-pcie.c
index 9513f7ab1eaa..4c7d11d2b378 100644
--- a/drivers/phy/phy-bcm-ns2-pcie.c
+++ b/drivers/phy/phy-bcm-ns2-pcie.c
@@ -18,11 +18,6 @@
 #include <linux/phy.h>
 #include <linux/phy/phy.h>
 
-struct ns2_pci_phy {
-	struct mdio_device *mdiodev;
-	struct phy *phy;
-};
-
 #define BLK_ADDR_REG_OFFSET	0x1f
 #define PLL_AFE1_100MHZ_BLK	0x2100
 #define PLL_CLK_AMP_OFFSET	0x03
@@ -30,17 +25,17 @@ struct ns2_pci_phy {
 
 static int ns2_pci_phy_init(struct phy *p)
 {
-	struct ns2_pci_phy *phy = phy_get_drvdata(p);
+	struct mdio_device *mdiodev = phy_get_drvdata(p);
 	int rc;
 
 	/* select the AFE 100MHz block page */
-	rc = mdiobus_write(phy->mdiodev->bus, phy->mdiodev->addr,
+	rc = mdiobus_write(mdiodev->bus, mdiodev->addr,
 			   BLK_ADDR_REG_OFFSET, PLL_AFE1_100MHZ_BLK);
 	if (rc)
 		goto err;
 
 	/* set the 100 MHz reference clock amplitude to 2.05 v */
-	rc = mdiobus_write(phy->mdiodev->bus, phy->mdiodev->addr,
+	rc = mdiobus_write(mdiodev->bus, mdiodev->addr,
 			   PLL_CLK_AMP_OFFSET, PLL_CLK_AMP_2P05V);
 	if (rc)
 		goto err;
@@ -48,19 +43,19 @@ static int ns2_pci_phy_init(struct phy *p)
 	return 0;
 
 err:
-	dev_err(&phy->mdiodev->dev, "Error %d writing to phy\n", rc);
+	dev_err(&mdiodev->dev, "Error %d writing to phy\n", rc);
 	return rc;
 }
 
-static struct phy_ops ns2_pci_phy_ops = {
+static const struct phy_ops ns2_pci_phy_ops = {
 	.init = ns2_pci_phy_init,
+	.owner = THIS_MODULE,
 };
 
 static int ns2_pci_phy_probe(struct mdio_device *mdiodev)
 {
 	struct device *dev = &mdiodev->dev;
 	struct phy_provider *provider;
-	struct ns2_pci_phy *p;
 	struct phy *phy;
 
 	phy = devm_phy_create(dev, dev->of_node, &ns2_pci_phy_ops);
@@ -69,16 +64,7 @@ static int ns2_pci_phy_probe(struct mdio_device *mdiodev)
 		return PTR_ERR(phy);
 	}
 
-	p = devm_kmalloc(dev, sizeof(struct ns2_pci_phy),
-			 GFP_KERNEL);
-	if (!p)
-		return -ENOMEM;
-
-	p->mdiodev = mdiodev;
-	dev_set_drvdata(dev, p);
-
-	p->phy = phy;
-	phy_set_drvdata(phy, p);
+	phy_set_drvdata(phy, mdiodev);
 
 	provider = devm_of_phy_provider_register(&phy->dev,
 						 of_phy_simple_xlate);
diff --git a/drivers/phy/phy-brcm-sata.c b/drivers/phy/phy-brcm-sata.c
index 18d662610075..8ffc44afdb75 100644
--- a/drivers/phy/phy-brcm-sata.c
+++ b/drivers/phy/phy-brcm-sata.c
@@ -367,7 +367,7 @@ static int brcm_sata_phy_init(struct phy *phy)
 		rc = -ENODEV;
 	};
 
-	return 0;
+	return rc;
 }
 
 static const struct phy_ops phy_ops = {
diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c
index 8eca906b6e70..a268f4d6f3e9 100644
--- a/drivers/phy/phy-core.c
+++ b/drivers/phy/phy-core.c
@@ -357,6 +357,21 @@ int phy_set_mode(struct phy *phy, enum phy_mode mode)
 }
 EXPORT_SYMBOL_GPL(phy_set_mode);
 
+int phy_reset(struct phy *phy)
+{
+	int ret;
+
+	if (!phy || !phy->ops->reset)
+		return 0;
+
+	mutex_lock(&phy->mutex);
+	ret = phy->ops->reset(phy);
+	mutex_unlock(&phy->mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phy_reset);
+
 /**
  * _of_phy_get() - lookup and obtain a reference to a phy by phandle
  * @np: device_node for which to get the phy
diff --git a/drivers/phy/phy-da8xx-usb.c b/drivers/phy/phy-da8xx-usb.c
index b2e59b6170ac..32ae78c8ca17 100644
--- a/drivers/phy/phy-da8xx-usb.c
+++ b/drivers/phy/phy-da8xx-usb.c
@@ -154,7 +154,7 @@ static int da8xx_usb_phy_probe(struct platform_device *pdev)
 		d_phy->regmap = syscon_regmap_lookup_by_compatible(
 							"ti,da830-cfgchip");
 	else
-		d_phy->regmap = syscon_regmap_lookup_by_pdevname("syscon.0");
+		d_phy->regmap = syscon_regmap_lookup_by_pdevname("syscon");
 	if (IS_ERR(d_phy->regmap)) {
 		dev_err(dev, "Failed to get syscon\n");
 		return PTR_ERR(d_phy->regmap);
diff --git a/drivers/phy/phy-exynos5-usbdrd.c b/drivers/phy/phy-exynos5-usbdrd.c
index 20696f53303f..07ed608905ac 100644
--- a/drivers/phy/phy-exynos5-usbdrd.c
+++ b/drivers/phy/phy-exynos5-usbdrd.c
@@ -249,7 +249,7 @@ static void exynos5_usbdrd_phy_isol(struct phy_usb_instance *inst,
 static unsigned int
 exynos5_usbdrd_pipe3_set_refclk(struct phy_usb_instance *inst)
 {
-	static u32 reg;
+	u32 reg;
 	struct exynos5_usbdrd_phy *phy_drd = to_usbdrd_phy(inst);
 
 	/* restore any previous reference clock settings */
@@ -295,7 +295,7 @@ exynos5_usbdrd_pipe3_set_refclk(struct phy_usb_instance *inst)
 static unsigned int
 exynos5_usbdrd_utmi_set_refclk(struct phy_usb_instance *inst)
 {
-	static u32 reg;
+	u32 reg;
 	struct exynos5_usbdrd_phy *phy_drd = to_usbdrd_phy(inst);
 
 	/* restore any previous reference clock settings */
diff --git a/drivers/phy/phy-omap-usb2.c b/drivers/phy/phy-omap-usb2.c
index c134989052f5..fe909fd8144f 100644
--- a/drivers/phy/phy-omap-usb2.c
+++ b/drivers/phy/phy-omap-usb2.c
@@ -133,11 +133,49 @@ static int omap_usb_power_on(struct phy *x)
 	return omap_usb_phy_power(phy, true);
 }
 
+static int omap_usb2_disable_clocks(struct omap_usb *phy)
+{
+	clk_disable(phy->wkupclk);
+	if (!IS_ERR(phy->optclk))
+		clk_disable(phy->optclk);
+
+	return 0;
+}
+
+static int omap_usb2_enable_clocks(struct omap_usb *phy)
+{
+	int ret;
+
+	ret = clk_enable(phy->wkupclk);
+	if (ret < 0) {
+		dev_err(phy->dev, "Failed to enable wkupclk %d\n", ret);
+		goto err0;
+	}
+
+	if (!IS_ERR(phy->optclk)) {
+		ret = clk_enable(phy->optclk);
+		if (ret < 0) {
+			dev_err(phy->dev, "Failed to enable optclk %d\n", ret);
+			goto err1;
+		}
+	}
+
+	return 0;
+
+err1:
+	clk_disable(phy->wkupclk);
+
+err0:
+	return ret;
+}
+
 static int omap_usb_init(struct phy *x)
 {
 	struct omap_usb *phy = phy_get_drvdata(x);
 	u32 val;
 
+	omap_usb2_enable_clocks(phy);
+
 	if (phy->flags & OMAP_USB2_CALIBRATE_FALSE_DISCONNECT) {
 		/*
 		 *
@@ -155,8 +193,16 @@ static int omap_usb_init(struct phy *x)
 	return 0;
 }
 
+static int omap_usb_exit(struct phy *x)
+{
+	struct omap_usb *phy = phy_get_drvdata(x);
+
+	return omap_usb2_disable_clocks(phy);
+}
+
 static const struct phy_ops ops = {
 	.init		= omap_usb_init,
+	.exit		= omap_usb_exit,
 	.power_on	= omap_usb_power_on,
 	.power_off	= omap_usb_power_off,
 	.owner		= THIS_MODULE,
@@ -376,65 +422,11 @@ static int omap_usb2_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM
-
-static int omap_usb2_runtime_suspend(struct device *dev)
-{
-	struct platform_device	*pdev = to_platform_device(dev);
-	struct omap_usb	*phy = platform_get_drvdata(pdev);
-
-	clk_disable(phy->wkupclk);
-	if (!IS_ERR(phy->optclk))
-		clk_disable(phy->optclk);
-
-	return 0;
-}
-
-static int omap_usb2_runtime_resume(struct device *dev)
-{
-	struct platform_device	*pdev = to_platform_device(dev);
-	struct omap_usb	*phy = platform_get_drvdata(pdev);
-	int ret;
-
-	ret = clk_enable(phy->wkupclk);
-	if (ret < 0) {
-		dev_err(phy->dev, "Failed to enable wkupclk %d\n", ret);
-		goto err0;
-	}
-
-	if (!IS_ERR(phy->optclk)) {
-		ret = clk_enable(phy->optclk);
-		if (ret < 0) {
-			dev_err(phy->dev, "Failed to enable optclk %d\n", ret);
-			goto err1;
-		}
-	}
-
-	return 0;
-
-err1:
-	clk_disable(phy->wkupclk);
-
-err0:
-	return ret;
-}
-
-static const struct dev_pm_ops omap_usb2_pm_ops = {
-	SET_RUNTIME_PM_OPS(omap_usb2_runtime_suspend, omap_usb2_runtime_resume,
-		NULL)
-};
-
-#define DEV_PM_OPS     (&omap_usb2_pm_ops)
-#else
-#define DEV_PM_OPS     NULL
-#endif
-
 static struct platform_driver omap_usb2_driver = {
 	.probe		= omap_usb2_probe,
 	.remove		= omap_usb2_remove,
 	.driver		= {
 		.name	= "omap-usb2",
-		.pm	= DEV_PM_OPS,
 		.of_match_table = omap_usb2_id_table,
 	},
 };
diff --git a/drivers/phy/phy-qcom-ufs.c b/drivers/phy/phy-qcom-ufs.c
index 107cb57c3513..18a5b495ad65 100644
--- a/drivers/phy/phy-qcom-ufs.c
+++ b/drivers/phy/phy-qcom-ufs.c
@@ -283,10 +283,8 @@ static int __ufs_qcom_phy_init_vreg(struct phy *phy,
 			err = 0;
 		}
 		snprintf(prop_name, MAX_PROP_NAME, "%s-always-on", name);
-		if (of_get_property(dev->of_node, prop_name, NULL))
-			vreg->is_always_on = true;
-		else
-			vreg->is_always_on = false;
+		vreg->is_always_on = of_property_read_bool(dev->of_node,
+							   prop_name);
 	}
 
 	if (!strcmp(name, "vdda-pll")) {
diff --git a/drivers/phy/phy-rcar-gen3-usb2.c b/drivers/phy/phy-rcar-gen3-usb2.c
index 31156c9c4707..3d97eadd247d 100644
--- a/drivers/phy/phy-rcar-gen3-usb2.c
+++ b/drivers/phy/phy-rcar-gen3-usb2.c
@@ -280,6 +280,7 @@ static irqreturn_t rcar_gen3_phy_usb2_irq(int irq, void *_ch)
 
 static const struct of_device_id rcar_gen3_phy_usb2_match_table[] = {
 	{ .compatible = "renesas,usb2-phy-r8a7795" },
+	{ .compatible = "renesas,usb2-phy-r8a7796" },
 	{ .compatible = "renesas,rcar-gen3-usb2-phy" },
 	{ }
 };
diff --git a/drivers/phy/phy-rockchip-inno-usb2.c b/drivers/phy/phy-rockchip-inno-usb2.c
new file mode 100644
index 000000000000..ac203107b071
--- /dev/null
+++ b/drivers/phy/phy-rockchip-inno-usb2.c
@@ -0,0 +1,707 @@
+/*
+ * Rockchip USB2.0 PHY with Innosilicon IP block driver
+ *
+ * Copyright (C) 2016 Fuzhou Rockchip Electronics Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/gpio/consumer.h>
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+
+#define BIT_WRITEABLE_SHIFT	16
+#define SCHEDULE_DELAY	(60 * HZ)
+
+enum rockchip_usb2phy_port_id {
+	USB2PHY_PORT_OTG,
+	USB2PHY_PORT_HOST,
+	USB2PHY_NUM_PORTS,
+};
+
+enum rockchip_usb2phy_host_state {
+	PHY_STATE_HS_ONLINE	= 0,
+	PHY_STATE_DISCONNECT	= 1,
+	PHY_STATE_CONNECT	= 2,
+	PHY_STATE_FS_LS_ONLINE	= 4,
+};
+
+struct usb2phy_reg {
+	unsigned int	offset;
+	unsigned int	bitend;
+	unsigned int	bitstart;
+	unsigned int	disable;
+	unsigned int	enable;
+};
+
+/**
+ * struct rockchip_usb2phy_port_cfg: usb-phy port configuration.
+ * @phy_sus: phy suspend register.
+ * @ls_det_en: linestate detection enable register.
+ * @ls_det_st: linestate detection state register.
+ * @ls_det_clr: linestate detection clear register.
+ * @utmi_ls: utmi linestate state register.
+ * @utmi_hstdet: utmi host disconnect register.
+ */
+struct rockchip_usb2phy_port_cfg {
+	struct usb2phy_reg	phy_sus;
+	struct usb2phy_reg	ls_det_en;
+	struct usb2phy_reg	ls_det_st;
+	struct usb2phy_reg	ls_det_clr;
+	struct usb2phy_reg	utmi_ls;
+	struct usb2phy_reg	utmi_hstdet;
+};
+
+/**
+ * struct rockchip_usb2phy_cfg: usb-phy configuration.
+ * @reg: the address offset of grf for usb-phy config.
+ * @num_ports: specify how many ports that the phy has.
+ * @clkout_ctl: keep on/turn off output clk of phy.
+ */
+struct rockchip_usb2phy_cfg {
+	unsigned int	reg;
+	unsigned int	num_ports;
+	struct usb2phy_reg	clkout_ctl;
+	const struct rockchip_usb2phy_port_cfg	port_cfgs[USB2PHY_NUM_PORTS];
+};
+
+/**
+ * struct rockchip_usb2phy_port: usb-phy port data.
+ * @port_id: flag for otg port or host port.
+ * @suspended: phy suspended flag.
+ * @ls_irq: IRQ number assigned for linestate detection.
+ * @mutex: for register updating in sm_work.
+ * @sm_work: OTG state machine work.
+ * @phy_cfg: port register configuration, assigned by driver data.
+ */
+struct rockchip_usb2phy_port {
+	struct phy	*phy;
+	unsigned int	port_id;
+	bool		suspended;
+	int		ls_irq;
+	struct mutex	mutex;
+	struct		delayed_work sm_work;
+	const struct	rockchip_usb2phy_port_cfg *port_cfg;
+};
+
+/**
+ * struct rockchip_usb2phy: usb2.0 phy driver data.
+ * @grf: General Register Files regmap.
+ * @clk: clock struct of phy input clk.
+ * @clk480m: clock struct of phy output clk.
+ * @clk_hw: clock struct of phy output clk management.
+ * @phy_cfg: phy register configuration, assigned by driver data.
+ * @ports: phy port instance.
+ */
+struct rockchip_usb2phy {
+	struct device	*dev;
+	struct regmap	*grf;
+	struct clk	*clk;
+	struct clk	*clk480m;
+	struct clk_hw	clk480m_hw;
+	const struct rockchip_usb2phy_cfg	*phy_cfg;
+	struct rockchip_usb2phy_port	ports[USB2PHY_NUM_PORTS];
+};
+
+static inline int property_enable(struct rockchip_usb2phy *rphy,
+				  const struct usb2phy_reg *reg, bool en)
+{
+	unsigned int val, mask, tmp;
+
+	tmp = en ? reg->enable : reg->disable;
+	mask = GENMASK(reg->bitend, reg->bitstart);
+	val = (tmp << reg->bitstart) | (mask << BIT_WRITEABLE_SHIFT);
+
+	return regmap_write(rphy->grf, reg->offset, val);
+}
+
+static inline bool property_enabled(struct rockchip_usb2phy *rphy,
+				    const struct usb2phy_reg *reg)
+{
+	int ret;
+	unsigned int tmp, orig;
+	unsigned int mask = GENMASK(reg->bitend, reg->bitstart);
+
+	ret = regmap_read(rphy->grf, reg->offset, &orig);
+	if (ret)
+		return false;
+
+	tmp = (orig & mask) >> reg->bitstart;
+	return tmp == reg->enable;
+}
+
+static int rockchip_usb2phy_clk480m_enable(struct clk_hw *hw)
+{
+	struct rockchip_usb2phy *rphy =
+		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
+	int ret;
+
+	/* turn on 480m clk output if it is off */
+	if (!property_enabled(rphy, &rphy->phy_cfg->clkout_ctl)) {
+		ret = property_enable(rphy, &rphy->phy_cfg->clkout_ctl, true);
+		if (ret)
+			return ret;
+
+		/* waitting for the clk become stable */
+		mdelay(1);
+	}
+
+	return 0;
+}
+
+static void rockchip_usb2phy_clk480m_disable(struct clk_hw *hw)
+{
+	struct rockchip_usb2phy *rphy =
+		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
+
+	/* turn off 480m clk output */
+	property_enable(rphy, &rphy->phy_cfg->clkout_ctl, false);
+}
+
+static int rockchip_usb2phy_clk480m_enabled(struct clk_hw *hw)
+{
+	struct rockchip_usb2phy *rphy =
+		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
+
+	return property_enabled(rphy, &rphy->phy_cfg->clkout_ctl);
+}
+
+static unsigned long
+rockchip_usb2phy_clk480m_recalc_rate(struct clk_hw *hw,
+				     unsigned long parent_rate)
+{
+	return 480000000;
+}
+
+static const struct clk_ops rockchip_usb2phy_clkout_ops = {
+	.enable = rockchip_usb2phy_clk480m_enable,
+	.disable = rockchip_usb2phy_clk480m_disable,
+	.is_enabled = rockchip_usb2phy_clk480m_enabled,
+	.recalc_rate = rockchip_usb2phy_clk480m_recalc_rate,
+};
+
+static void rockchip_usb2phy_clk480m_unregister(void *data)
+{
+	struct rockchip_usb2phy *rphy = data;
+
+	of_clk_del_provider(rphy->dev->of_node);
+	clk_unregister(rphy->clk480m);
+}
+
+static int
+rockchip_usb2phy_clk480m_register(struct rockchip_usb2phy *rphy)
+{
+	struct device_node *node = rphy->dev->of_node;
+	struct clk_init_data init;
+	const char *clk_name;
+	int ret;
+
+	init.flags = 0;
+	init.name = "clk_usbphy_480m";
+	init.ops = &rockchip_usb2phy_clkout_ops;
+
+	/* optional override of the clockname */
+	of_property_read_string(node, "clock-output-names", &init.name);
+
+	if (rphy->clk) {
+		clk_name = __clk_get_name(rphy->clk);
+		init.parent_names = &clk_name;
+		init.num_parents = 1;
+	} else {
+		init.parent_names = NULL;
+		init.num_parents = 0;
+	}
+
+	rphy->clk480m_hw.init = &init;
+
+	/* register the clock */
+	rphy->clk480m = clk_register(rphy->dev, &rphy->clk480m_hw);
+	if (IS_ERR(rphy->clk480m)) {
+		ret = PTR_ERR(rphy->clk480m);
+		goto err_ret;
+	}
+
+	ret = of_clk_add_provider(node, of_clk_src_simple_get, rphy->clk480m);
+	if (ret < 0)
+		goto err_clk_provider;
+
+	ret = devm_add_action(rphy->dev, rockchip_usb2phy_clk480m_unregister,
+			      rphy);
+	if (ret < 0)
+		goto err_unreg_action;
+
+	return 0;
+
+err_unreg_action:
+	of_clk_del_provider(node);
+err_clk_provider:
+	clk_unregister(rphy->clk480m);
+err_ret:
+	return ret;
+}
+
+static int rockchip_usb2phy_init(struct phy *phy)
+{
+	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(phy->dev.parent);
+	int ret;
+
+	if (rport->port_id == USB2PHY_PORT_HOST) {
+		/* clear linestate and enable linestate detect irq */
+		mutex_lock(&rport->mutex);
+
+		ret = property_enable(rphy, &rport->port_cfg->ls_det_clr, true);
+		if (ret) {
+			mutex_unlock(&rport->mutex);
+			return ret;
+		}
+
+		ret = property_enable(rphy, &rport->port_cfg->ls_det_en, true);
+		if (ret) {
+			mutex_unlock(&rport->mutex);
+			return ret;
+		}
+
+		mutex_unlock(&rport->mutex);
+		schedule_delayed_work(&rport->sm_work, SCHEDULE_DELAY);
+	}
+
+	return 0;
+}
+
+static int rockchip_usb2phy_power_on(struct phy *phy)
+{
+	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(phy->dev.parent);
+	int ret;
+
+	dev_dbg(&rport->phy->dev, "port power on\n");
+
+	if (!rport->suspended)
+		return 0;
+
+	ret = clk_prepare_enable(rphy->clk480m);
+	if (ret)
+		return ret;
+
+	ret = property_enable(rphy, &rport->port_cfg->phy_sus, false);
+	if (ret)
+		return ret;
+
+	rport->suspended = false;
+	return 0;
+}
+
+static int rockchip_usb2phy_power_off(struct phy *phy)
+{
+	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(phy->dev.parent);
+	int ret;
+
+	dev_dbg(&rport->phy->dev, "port power off\n");
+
+	if (rport->suspended)
+		return 0;
+
+	ret = property_enable(rphy, &rport->port_cfg->phy_sus, true);
+	if (ret)
+		return ret;
+
+	rport->suspended = true;
+	clk_disable_unprepare(rphy->clk480m);
+
+	return 0;
+}
+
+static int rockchip_usb2phy_exit(struct phy *phy)
+{
+	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
+
+	if (rport->port_id == USB2PHY_PORT_HOST)
+		cancel_delayed_work_sync(&rport->sm_work);
+
+	return 0;
+}
+
+static const struct phy_ops rockchip_usb2phy_ops = {
+	.init		= rockchip_usb2phy_init,
+	.exit		= rockchip_usb2phy_exit,
+	.power_on	= rockchip_usb2phy_power_on,
+	.power_off	= rockchip_usb2phy_power_off,
+	.owner		= THIS_MODULE,
+};
+
+/*
+ * The function manage host-phy port state and suspend/resume phy port
+ * to save power.
+ *
+ * we rely on utmi_linestate and utmi_hostdisconnect to identify whether
+ * devices is disconnect or not. Besides, we do not need care it is FS/LS
+ * disconnected or HS disconnected, actually, we just only need get the
+ * device is disconnected at last through rearm the delayed work,
+ * to suspend the phy port in _PHY_STATE_DISCONNECT_ case.
+ *
+ * NOTE: It may invoke *phy_powr_off or *phy_power_on which will invoke
+ * some clk related APIs, so do not invoke it from interrupt context directly.
+ */
+static void rockchip_usb2phy_sm_work(struct work_struct *work)
+{
+	struct rockchip_usb2phy_port *rport =
+		container_of(work, struct rockchip_usb2phy_port, sm_work.work);
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
+	unsigned int sh = rport->port_cfg->utmi_hstdet.bitend -
+			  rport->port_cfg->utmi_hstdet.bitstart + 1;
+	unsigned int ul, uhd, state;
+	unsigned int ul_mask, uhd_mask;
+	int ret;
+
+	mutex_lock(&rport->mutex);
+
+	ret = regmap_read(rphy->grf, rport->port_cfg->utmi_ls.offset, &ul);
+	if (ret < 0)
+		goto next_schedule;
+
+	ret = regmap_read(rphy->grf, rport->port_cfg->utmi_hstdet.offset,
+			  &uhd);
+	if (ret < 0)
+		goto next_schedule;
+
+	uhd_mask = GENMASK(rport->port_cfg->utmi_hstdet.bitend,
+			   rport->port_cfg->utmi_hstdet.bitstart);
+	ul_mask = GENMASK(rport->port_cfg->utmi_ls.bitend,
+			  rport->port_cfg->utmi_ls.bitstart);
+
+	/* stitch on utmi_ls and utmi_hstdet as phy state */
+	state = ((uhd & uhd_mask) >> rport->port_cfg->utmi_hstdet.bitstart) |
+		(((ul & ul_mask) >> rport->port_cfg->utmi_ls.bitstart) << sh);
+
+	switch (state) {
+	case PHY_STATE_HS_ONLINE:
+		dev_dbg(&rport->phy->dev, "HS online\n");
+		break;
+	case PHY_STATE_FS_LS_ONLINE:
+		/*
+		 * For FS/LS device, the online state share with connect state
+		 * from utmi_ls and utmi_hstdet register, so we distinguish
+		 * them via suspended flag.
+		 *
+		 * Plus, there are two cases, one is D- Line pull-up, and D+
+		 * line pull-down, the state is 4; another is D+ line pull-up,
+		 * and D- line pull-down, the state is 2.
+		 */
+		if (!rport->suspended) {
+			/* D- line pull-up, D+ line pull-down */
+			dev_dbg(&rport->phy->dev, "FS/LS online\n");
+			break;
+		}
+		/* fall through */
+	case PHY_STATE_CONNECT:
+		if (rport->suspended) {
+			dev_dbg(&rport->phy->dev, "Connected\n");
+			rockchip_usb2phy_power_on(rport->phy);
+			rport->suspended = false;
+		} else {
+			/* D+ line pull-up, D- line pull-down */
+			dev_dbg(&rport->phy->dev, "FS/LS online\n");
+		}
+		break;
+	case PHY_STATE_DISCONNECT:
+		if (!rport->suspended) {
+			dev_dbg(&rport->phy->dev, "Disconnected\n");
+			rockchip_usb2phy_power_off(rport->phy);
+			rport->suspended = true;
+		}
+
+		/*
+		 * activate the linestate detection to get the next device
+		 * plug-in irq.
+		 */
+		property_enable(rphy, &rport->port_cfg->ls_det_clr, true);
+		property_enable(rphy, &rport->port_cfg->ls_det_en, true);
+
+		/*
+		 * we don't need to rearm the delayed work when the phy port
+		 * is suspended.
+		 */
+		mutex_unlock(&rport->mutex);
+		return;
+	default:
+		dev_dbg(&rport->phy->dev, "unknown phy state\n");
+		break;
+	}
+
+next_schedule:
+	mutex_unlock(&rport->mutex);
+	schedule_delayed_work(&rport->sm_work, SCHEDULE_DELAY);
+}
+
+static irqreturn_t rockchip_usb2phy_linestate_irq(int irq, void *data)
+{
+	struct rockchip_usb2phy_port *rport = data;
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
+
+	if (!property_enabled(rphy, &rport->port_cfg->ls_det_st))
+		return IRQ_NONE;
+
+	mutex_lock(&rport->mutex);
+
+	/* disable linestate detect irq and clear its status */
+	property_enable(rphy, &rport->port_cfg->ls_det_en, false);
+	property_enable(rphy, &rport->port_cfg->ls_det_clr, true);
+
+	mutex_unlock(&rport->mutex);
+
+	/*
+	 * In this case for host phy port, a new device is plugged in,
+	 * meanwhile, if the phy port is suspended, we need rearm the work to
+	 * resume it and mange its states; otherwise, we do nothing about that.
+	 */
+	if (rport->suspended && rport->port_id == USB2PHY_PORT_HOST)
+		rockchip_usb2phy_sm_work(&rport->sm_work.work);
+
+	return IRQ_HANDLED;
+}
+
+static int rockchip_usb2phy_host_port_init(struct rockchip_usb2phy *rphy,
+					   struct rockchip_usb2phy_port *rport,
+					   struct device_node *child_np)
+{
+	int ret;
+
+	rport->port_id = USB2PHY_PORT_HOST;
+	rport->port_cfg = &rphy->phy_cfg->port_cfgs[USB2PHY_PORT_HOST];
+	rport->suspended = true;
+
+	mutex_init(&rport->mutex);
+	INIT_DELAYED_WORK(&rport->sm_work, rockchip_usb2phy_sm_work);
+
+	rport->ls_irq = of_irq_get_byname(child_np, "linestate");
+	if (rport->ls_irq < 0) {
+		dev_err(rphy->dev, "no linestate irq provided\n");
+		return rport->ls_irq;
+	}
+
+	ret = devm_request_threaded_irq(rphy->dev, rport->ls_irq, NULL,
+					rockchip_usb2phy_linestate_irq,
+					IRQF_ONESHOT,
+					"rockchip_usb2phy", rport);
+	if (ret) {
+		dev_err(rphy->dev, "failed to request irq handle\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int rockchip_usb2phy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *child_np;
+	struct phy_provider *provider;
+	struct rockchip_usb2phy *rphy;
+	const struct rockchip_usb2phy_cfg *phy_cfgs;
+	const struct of_device_id *match;
+	unsigned int reg;
+	int index, ret;
+
+	rphy = devm_kzalloc(dev, sizeof(*rphy), GFP_KERNEL);
+	if (!rphy)
+		return -ENOMEM;
+
+	match = of_match_device(dev->driver->of_match_table, dev);
+	if (!match || !match->data) {
+		dev_err(dev, "phy configs are not assigned!\n");
+		return -EINVAL;
+	}
+
+	if (!dev->parent || !dev->parent->of_node)
+		return -EINVAL;
+
+	rphy->grf = syscon_node_to_regmap(dev->parent->of_node);
+	if (IS_ERR(rphy->grf))
+		return PTR_ERR(rphy->grf);
+
+	if (of_property_read_u32(np, "reg", &reg)) {
+		dev_err(dev, "the reg property is not assigned in %s node\n",
+			np->name);
+		return -EINVAL;
+	}
+
+	rphy->dev = dev;
+	phy_cfgs = match->data;
+	platform_set_drvdata(pdev, rphy);
+
+	/* find out a proper config which can be matched with dt. */
+	index = 0;
+	while (phy_cfgs[index].reg) {
+		if (phy_cfgs[index].reg == reg) {
+			rphy->phy_cfg = &phy_cfgs[index];
+			break;
+		}
+
+		++index;
+	}
+
+	if (!rphy->phy_cfg) {
+		dev_err(dev, "no phy-config can be matched with %s node\n",
+			np->name);
+		return -EINVAL;
+	}
+
+	rphy->clk = of_clk_get_by_name(np, "phyclk");
+	if (!IS_ERR(rphy->clk)) {
+		clk_prepare_enable(rphy->clk);
+	} else {
+		dev_info(&pdev->dev, "no phyclk specified\n");
+		rphy->clk = NULL;
+	}
+
+	ret = rockchip_usb2phy_clk480m_register(rphy);
+	if (ret) {
+		dev_err(dev, "failed to register 480m output clock\n");
+		goto disable_clks;
+	}
+
+	index = 0;
+	for_each_available_child_of_node(np, child_np) {
+		struct rockchip_usb2phy_port *rport = &rphy->ports[index];
+		struct phy *phy;
+
+		/*
+		 * This driver aim to support both otg-port and host-port,
+		 * but unfortunately, the otg part is not ready in current,
+		 * so this comments and below codes are interim, which should
+		 * be changed after otg-port is supplied soon.
+		 */
+		if (of_node_cmp(child_np->name, "host-port"))
+			goto next_child;
+
+		phy = devm_phy_create(dev, child_np, &rockchip_usb2phy_ops);
+		if (IS_ERR(phy)) {
+			dev_err(dev, "failed to create phy\n");
+			ret = PTR_ERR(phy);
+			goto put_child;
+		}
+
+		rport->phy = phy;
+		phy_set_drvdata(rport->phy, rport);
+
+		ret = rockchip_usb2phy_host_port_init(rphy, rport, child_np);
+		if (ret)
+			goto put_child;
+
+next_child:
+		/* to prevent out of boundary */
+		if (++index >= rphy->phy_cfg->num_ports)
+			break;
+	}
+
+	provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	return PTR_ERR_OR_ZERO(provider);
+
+put_child:
+	of_node_put(child_np);
+disable_clks:
+	if (rphy->clk) {
+		clk_disable_unprepare(rphy->clk);
+		clk_put(rphy->clk);
+	}
+	return ret;
+}
+
+static const struct rockchip_usb2phy_cfg rk3366_phy_cfgs[] = {
+	{
+		.reg = 0x700,
+		.num_ports	= 2,
+		.clkout_ctl	= { 0x0724, 15, 15, 1, 0 },
+		.port_cfgs	= {
+			[USB2PHY_PORT_HOST] = {
+				.phy_sus	= { 0x0728, 15, 0, 0, 0x1d1 },
+				.ls_det_en	= { 0x0680, 4, 4, 0, 1 },
+				.ls_det_st	= { 0x0690, 4, 4, 0, 1 },
+				.ls_det_clr	= { 0x06a0, 4, 4, 0, 1 },
+				.utmi_ls	= { 0x049c, 14, 13, 0, 1 },
+				.utmi_hstdet	= { 0x049c, 12, 12, 0, 1 }
+			}
+		},
+	},
+	{ /* sentinel */ }
+};
+
+static const struct rockchip_usb2phy_cfg rk3399_phy_cfgs[] = {
+	{
+		.reg = 0xe450,
+		.num_ports	= 2,
+		.clkout_ctl	= { 0xe450, 4, 4, 1, 0 },
+		.port_cfgs	= {
+			[USB2PHY_PORT_HOST] = {
+				.phy_sus	= { 0xe458, 1, 0, 0x2, 0x1 },
+				.ls_det_en	= { 0xe3c0, 6, 6, 0, 1 },
+				.ls_det_st	= { 0xe3e0, 6, 6, 0, 1 },
+				.ls_det_clr	= { 0xe3d0, 6, 6, 0, 1 },
+				.utmi_ls	= { 0xe2ac, 22, 21, 0, 1 },
+				.utmi_hstdet	= { 0xe2ac, 23, 23, 0, 1 }
+			}
+		},
+	},
+	{
+		.reg = 0xe460,
+		.num_ports	= 2,
+		.clkout_ctl	= { 0xe460, 4, 4, 1, 0 },
+		.port_cfgs	= {
+			[USB2PHY_PORT_HOST] = {
+				.phy_sus	= { 0xe468, 1, 0, 0x2, 0x1 },
+				.ls_det_en	= { 0xe3c0, 11, 11, 0, 1 },
+				.ls_det_st	= { 0xe3e0, 11, 11, 0, 1 },
+				.ls_det_clr	= { 0xe3d0, 11, 11, 0, 1 },
+				.utmi_ls	= { 0xe2ac, 26, 25, 0, 1 },
+				.utmi_hstdet	= { 0xe2ac, 27, 27, 0, 1 }
+			}
+		},
+	},
+	{ /* sentinel */ }
+};
+
+static const struct of_device_id rockchip_usb2phy_dt_match[] = {
+	{ .compatible = "rockchip,rk3366-usb2phy", .data = &rk3366_phy_cfgs },
+	{ .compatible = "rockchip,rk3399-usb2phy", .data = &rk3399_phy_cfgs },
+	{}
+};
+MODULE_DEVICE_TABLE(of, rockchip_usb2phy_dt_match);
+
+static struct platform_driver rockchip_usb2phy_driver = {
+	.probe		= rockchip_usb2phy_probe,
+	.driver		= {
+		.name	= "rockchip-usb2phy",
+		.of_match_table = rockchip_usb2phy_dt_match,
+	},
+};
+module_platform_driver(rockchip_usb2phy_driver);
+
+MODULE_AUTHOR("Frank Wang <frank.wang@rock-chips.com>");
+MODULE_DESCRIPTION("Rockchip USB2.0 PHY driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/phy-rockchip-pcie.c b/drivers/phy/phy-rockchip-pcie.c
new file mode 100644
index 000000000000..a2b4c6b58aea
--- /dev/null
+++ b/drivers/phy/phy-rockchip-pcie.c
@@ -0,0 +1,357 @@
+/*
+ * Rockchip PCIe PHY driver
+ *
+ * Copyright (C) 2016 Shawn Lin <shawn.lin@rock-chips.com>
+ * Copyright (C) 2016 ROCKCHIP, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+/*
+ * The higher 16-bit of this register is used for write protection
+ * only if BIT(x + 16) set to 1 the BIT(x) can be written.
+ */
+#define HIWORD_UPDATE(val, mask, shift) \
+		((val) << (shift) | (mask) << ((shift) + 16))
+
+#define PHY_MAX_LANE_NUM      4
+#define PHY_CFG_DATA_SHIFT    7
+#define PHY_CFG_ADDR_SHIFT    1
+#define PHY_CFG_DATA_MASK     0xf
+#define PHY_CFG_ADDR_MASK     0x3f
+#define PHY_CFG_RD_MASK       0x3ff
+#define PHY_CFG_WR_ENABLE     1
+#define PHY_CFG_WR_DISABLE    1
+#define PHY_CFG_WR_SHIFT      0
+#define PHY_CFG_WR_MASK       1
+#define PHY_CFG_PLL_LOCK      0x10
+#define PHY_CFG_CLK_TEST      0x10
+#define PHY_CFG_CLK_SCC       0x12
+#define PHY_CFG_SEPE_RATE     BIT(3)
+#define PHY_CFG_PLL_100M      BIT(3)
+#define PHY_PLL_LOCKED        BIT(9)
+#define PHY_PLL_OUTPUT        BIT(10)
+#define PHY_LANE_A_STATUS     0x30
+#define PHY_LANE_B_STATUS     0x31
+#define PHY_LANE_C_STATUS     0x32
+#define PHY_LANE_D_STATUS     0x33
+#define PHY_LANE_RX_DET_SHIFT 11
+#define PHY_LANE_RX_DET_TH    0x1
+#define PHY_LANE_IDLE_OFF     0x1
+#define PHY_LANE_IDLE_MASK    0x1
+#define PHY_LANE_IDLE_A_SHIFT 3
+#define PHY_LANE_IDLE_B_SHIFT 4
+#define PHY_LANE_IDLE_C_SHIFT 5
+#define PHY_LANE_IDLE_D_SHIFT 6
+
+struct rockchip_pcie_data {
+	unsigned int pcie_conf;
+	unsigned int pcie_status;
+	unsigned int pcie_laneoff;
+};
+
+struct rockchip_pcie_phy {
+	struct rockchip_pcie_data *phy_data;
+	struct regmap *reg_base;
+	struct reset_control *phy_rst;
+	struct clk *clk_pciephy_ref;
+};
+
+static inline void phy_wr_cfg(struct rockchip_pcie_phy *rk_phy,
+			      u32 addr, u32 data)
+{
+	regmap_write(rk_phy->reg_base, rk_phy->phy_data->pcie_conf,
+		     HIWORD_UPDATE(data,
+				   PHY_CFG_DATA_MASK,
+				   PHY_CFG_DATA_SHIFT) |
+		     HIWORD_UPDATE(addr,
+				   PHY_CFG_ADDR_MASK,
+				   PHY_CFG_ADDR_SHIFT));
+	udelay(1);
+	regmap_write(rk_phy->reg_base, rk_phy->phy_data->pcie_conf,
+		     HIWORD_UPDATE(PHY_CFG_WR_ENABLE,
+				   PHY_CFG_WR_MASK,
+				   PHY_CFG_WR_SHIFT));
+	udelay(1);
+	regmap_write(rk_phy->reg_base, rk_phy->phy_data->pcie_conf,
+		     HIWORD_UPDATE(PHY_CFG_WR_DISABLE,
+				   PHY_CFG_WR_MASK,
+				   PHY_CFG_WR_SHIFT));
+}
+
+static inline u32 phy_rd_cfg(struct rockchip_pcie_phy *rk_phy,
+			     u32 addr)
+{
+	u32 val;
+
+	regmap_write(rk_phy->reg_base, rk_phy->phy_data->pcie_conf,
+		     HIWORD_UPDATE(addr,
+				   PHY_CFG_RD_MASK,
+				   PHY_CFG_ADDR_SHIFT));
+	regmap_read(rk_phy->reg_base,
+		    rk_phy->phy_data->pcie_status,
+		    &val);
+	return val;
+}
+
+static int rockchip_pcie_phy_power_off(struct phy *phy)
+{
+	struct rockchip_pcie_phy *rk_phy = phy_get_drvdata(phy);
+	int err = 0;
+
+	err = reset_control_assert(rk_phy->phy_rst);
+	if (err) {
+		dev_err(&phy->dev, "assert phy_rst err %d\n", err);
+		return err;
+	}
+
+	return 0;
+}
+
+static int rockchip_pcie_phy_power_on(struct phy *phy)
+{
+	struct rockchip_pcie_phy *rk_phy = phy_get_drvdata(phy);
+	int err = 0;
+	u32 status;
+	unsigned long timeout;
+
+	err = reset_control_deassert(rk_phy->phy_rst);
+	if (err) {
+		dev_err(&phy->dev, "deassert phy_rst err %d\n", err);
+		return err;
+	}
+
+	regmap_write(rk_phy->reg_base, rk_phy->phy_data->pcie_conf,
+		     HIWORD_UPDATE(PHY_CFG_PLL_LOCK,
+				   PHY_CFG_ADDR_MASK,
+				   PHY_CFG_ADDR_SHIFT));
+
+	/*
+	 * No documented timeout value for phy operation below,
+	 * so we make it large enough here. And we use loop-break
+	 * method which should not be harmful.
+	 */
+	timeout = jiffies + msecs_to_jiffies(1000);
+
+	err = -EINVAL;
+	while (time_before(jiffies, timeout)) {
+		regmap_read(rk_phy->reg_base,
+			    rk_phy->phy_data->pcie_status,
+			    &status);
+		if (status & PHY_PLL_LOCKED) {
+			dev_dbg(&phy->dev, "pll locked!\n");
+			err = 0;
+			break;
+		}
+		msleep(20);
+	}
+
+	if (err) {
+		dev_err(&phy->dev, "pll lock timeout!\n");
+		goto err_pll_lock;
+	}
+
+	phy_wr_cfg(rk_phy, PHY_CFG_CLK_TEST, PHY_CFG_SEPE_RATE);
+	phy_wr_cfg(rk_phy, PHY_CFG_CLK_SCC, PHY_CFG_PLL_100M);
+
+	err = -ETIMEDOUT;
+	while (time_before(jiffies, timeout)) {
+		regmap_read(rk_phy->reg_base,
+			    rk_phy->phy_data->pcie_status,
+			    &status);
+		if (!(status & PHY_PLL_OUTPUT)) {
+			dev_dbg(&phy->dev, "pll output enable done!\n");
+			err = 0;
+			break;
+		}
+		msleep(20);
+	}
+
+	if (err) {
+		dev_err(&phy->dev, "pll output enable timeout!\n");
+		goto err_pll_lock;
+	}
+
+	regmap_write(rk_phy->reg_base, rk_phy->phy_data->pcie_conf,
+		     HIWORD_UPDATE(PHY_CFG_PLL_LOCK,
+				   PHY_CFG_ADDR_MASK,
+				   PHY_CFG_ADDR_SHIFT));
+	err = -EINVAL;
+	while (time_before(jiffies, timeout)) {
+		regmap_read(rk_phy->reg_base,
+			    rk_phy->phy_data->pcie_status,
+			    &status);
+		if (status & PHY_PLL_LOCKED) {
+			dev_dbg(&phy->dev, "pll relocked!\n");
+			err = 0;
+			break;
+		}
+		msleep(20);
+	}
+
+	if (err) {
+		dev_err(&phy->dev, "pll relock timeout!\n");
+		goto err_pll_lock;
+	}
+
+	return 0;
+
+err_pll_lock:
+	reset_control_assert(rk_phy->phy_rst);
+	return err;
+}
+
+static int rockchip_pcie_phy_init(struct phy *phy)
+{
+	struct rockchip_pcie_phy *rk_phy = phy_get_drvdata(phy);
+	int err = 0;
+
+	err = clk_prepare_enable(rk_phy->clk_pciephy_ref);
+	if (err) {
+		dev_err(&phy->dev, "Fail to enable pcie ref clock.\n");
+		goto err_refclk;
+	}
+
+	err = reset_control_assert(rk_phy->phy_rst);
+	if (err) {
+		dev_err(&phy->dev, "assert phy_rst err %d\n", err);
+		goto err_reset;
+	}
+
+	return err;
+
+err_reset:
+	clk_disable_unprepare(rk_phy->clk_pciephy_ref);
+err_refclk:
+	return err;
+}
+
+static int rockchip_pcie_phy_exit(struct phy *phy)
+{
+	struct rockchip_pcie_phy *rk_phy = phy_get_drvdata(phy);
+	int err = 0;
+
+	clk_disable_unprepare(rk_phy->clk_pciephy_ref);
+
+	err = reset_control_deassert(rk_phy->phy_rst);
+	if (err) {
+		dev_err(&phy->dev, "deassert phy_rst err %d\n", err);
+		goto err_reset;
+	}
+
+	return err;
+
+err_reset:
+	clk_prepare_enable(rk_phy->clk_pciephy_ref);
+	return err;
+}
+
+static const struct phy_ops ops = {
+	.init		= rockchip_pcie_phy_init,
+	.exit		= rockchip_pcie_phy_exit,
+	.power_on	= rockchip_pcie_phy_power_on,
+	.power_off	= rockchip_pcie_phy_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static const struct rockchip_pcie_data rk3399_pcie_data = {
+	.pcie_conf = 0xe220,
+	.pcie_status = 0xe2a4,
+	.pcie_laneoff = 0xe214,
+};
+
+static const struct of_device_id rockchip_pcie_phy_dt_ids[] = {
+	{
+		.compatible = "rockchip,rk3399-pcie-phy",
+		.data = &rk3399_pcie_data,
+	},
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, rockchip_pcie_phy_dt_ids);
+
+static int rockchip_pcie_phy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rockchip_pcie_phy *rk_phy;
+	struct phy *generic_phy;
+	struct phy_provider *phy_provider;
+	struct regmap *grf;
+	const struct of_device_id *of_id;
+
+	grf = syscon_node_to_regmap(dev->parent->of_node);
+	if (IS_ERR(grf)) {
+		dev_err(dev, "Cannot find GRF syscon\n");
+		return PTR_ERR(grf);
+	}
+
+	rk_phy = devm_kzalloc(dev, sizeof(*rk_phy), GFP_KERNEL);
+	if (!rk_phy)
+		return -ENOMEM;
+
+	of_id = of_match_device(rockchip_pcie_phy_dt_ids, &pdev->dev);
+	if (!of_id)
+		return -EINVAL;
+
+	rk_phy->phy_data = (struct rockchip_pcie_data *)of_id->data;
+	rk_phy->reg_base = grf;
+
+	rk_phy->phy_rst = devm_reset_control_get(dev, "phy");
+	if (IS_ERR(rk_phy->phy_rst)) {
+		if (PTR_ERR(rk_phy->phy_rst) != -EPROBE_DEFER)
+			dev_err(dev,
+				"missing phy property for reset controller\n");
+		return PTR_ERR(rk_phy->phy_rst);
+	}
+
+	rk_phy->clk_pciephy_ref = devm_clk_get(dev, "refclk");
+	if (IS_ERR(rk_phy->clk_pciephy_ref)) {
+		dev_err(dev, "refclk not found.\n");
+		return PTR_ERR(rk_phy->clk_pciephy_ref);
+	}
+
+	generic_phy = devm_phy_create(dev, dev->of_node, &ops);
+	if (IS_ERR(generic_phy)) {
+		dev_err(dev, "failed to create PHY\n");
+		return PTR_ERR(generic_phy);
+	}
+
+	phy_set_drvdata(generic_phy, rk_phy);
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static struct platform_driver rockchip_pcie_driver = {
+	.probe		= rockchip_pcie_phy_probe,
+	.driver		= {
+		.name	= "rockchip-pcie-phy",
+		.of_match_table = rockchip_pcie_phy_dt_ids,
+	},
+};
+
+module_platform_driver(rockchip_pcie_driver);
+
+MODULE_AUTHOR("Shawn Lin <shawn.lin@rock-chips.com>");
+MODULE_DESCRIPTION("Rockchip PCIe PHY driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/phy-rockchip-typec.c b/drivers/phy/phy-rockchip-typec.c
new file mode 100644
index 000000000000..7cfb0f8995de
--- /dev/null
+++ b/drivers/phy/phy-rockchip-typec.c
@@ -0,0 +1,1023 @@
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Chris Zhong <zyw@rock-chips.com>
+ *         Kever Yang <kever.yang@rock-chips.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * The ROCKCHIP Type-C PHY has two PLL clocks. The first PLL clock
+ * is used for USB3, the second PLL clock is used for DP. This Type-C PHY has
+ * 3 working modes: USB3 only mode, DP only mode, and USB3+DP mode.
+ * At USB3 only mode, both PLL clocks need to be initialized, this allows the
+ * PHY to switch mode between USB3 and USB3+DP, without disconnecting the USB
+ * device.
+ * In The DP only mode, only the DP PLL needs to be powered on, and the 4 lanes
+ * are all used for DP.
+ *
+ * This driver gets extcon cable state and property, then decides which mode to
+ * select:
+ *
+ * 1. USB3 only mode:
+ *    EXTCON_USB or EXTCON_USB_HOST state is true, and
+ *    EXTCON_PROP_USB_SS property is true.
+ *    EXTCON_DISP_DP state is false.
+ *
+ * 2. DP only mode:
+ *    EXTCON_DISP_DP state is true, and
+ *    EXTCON_PROP_USB_SS property is false.
+ *    If EXTCON_USB_HOST state is true, it is DP + USB2 mode, since the USB2 phy
+ *    is a separate phy, so this case is still DP only mode.
+ *
+ * 3. USB3+DP mode:
+ *    EXTCON_USB_HOST and EXTCON_DISP_DP are both true, and
+ *    EXTCON_PROP_USB_SS property is true.
+ *
+ * This Type-C PHY driver supports normal and flip orientation. The orientation
+ * is reported by the EXTCON_PROP_USB_TYPEC_POLARITY property: true is flip
+ * orientation, false is normal orientation.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/extcon.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#include <linux/mfd/syscon.h>
+#include <linux/phy/phy.h>
+
+#define CMN_SSM_BANDGAP			(0x21 << 2)
+#define CMN_SSM_BIAS			(0x22 << 2)
+#define CMN_PLLSM0_PLLEN		(0x29 << 2)
+#define CMN_PLLSM0_PLLPRE		(0x2a << 2)
+#define CMN_PLLSM0_PLLVREF		(0x2b << 2)
+#define CMN_PLLSM0_PLLLOCK		(0x2c << 2)
+#define CMN_PLLSM1_PLLEN		(0x31 << 2)
+#define CMN_PLLSM1_PLLPRE		(0x32 << 2)
+#define CMN_PLLSM1_PLLVREF		(0x33 << 2)
+#define CMN_PLLSM1_PLLLOCK		(0x34 << 2)
+#define CMN_PLLSM1_USER_DEF_CTRL	(0x37 << 2)
+#define CMN_ICAL_OVRD			(0xc1 << 2)
+#define CMN_PLL0_VCOCAL_OVRD		(0x83 << 2)
+#define CMN_PLL0_VCOCAL_INIT		(0x84 << 2)
+#define CMN_PLL0_VCOCAL_ITER		(0x85 << 2)
+#define CMN_PLL0_LOCK_REFCNT_START	(0x90 << 2)
+#define CMN_PLL0_LOCK_PLLCNT_START	(0x92 << 2)
+#define CMN_PLL0_LOCK_PLLCNT_THR	(0x93 << 2)
+#define CMN_PLL0_INTDIV			(0x94 << 2)
+#define CMN_PLL0_FRACDIV		(0x95 << 2)
+#define CMN_PLL0_HIGH_THR		(0x96 << 2)
+#define CMN_PLL0_DSM_DIAG		(0x97 << 2)
+#define CMN_PLL0_SS_CTRL1		(0x98 << 2)
+#define CMN_PLL0_SS_CTRL2		(0x99 << 2)
+#define CMN_PLL1_VCOCAL_START		(0xa1 << 2)
+#define CMN_PLL1_VCOCAL_OVRD		(0xa3 << 2)
+#define CMN_PLL1_VCOCAL_INIT		(0xa4 << 2)
+#define CMN_PLL1_VCOCAL_ITER		(0xa5 << 2)
+#define CMN_PLL1_LOCK_REFCNT_START	(0xb0 << 2)
+#define CMN_PLL1_LOCK_PLLCNT_START	(0xb2 << 2)
+#define CMN_PLL1_LOCK_PLLCNT_THR	(0xb3 << 2)
+#define CMN_PLL1_INTDIV			(0xb4 << 2)
+#define CMN_PLL1_FRACDIV		(0xb5 << 2)
+#define CMN_PLL1_HIGH_THR		(0xb6 << 2)
+#define CMN_PLL1_DSM_DIAG		(0xb7 << 2)
+#define CMN_PLL1_SS_CTRL1		(0xb8 << 2)
+#define CMN_PLL1_SS_CTRL2		(0xb9 << 2)
+#define CMN_RXCAL_OVRD			(0xd1 << 2)
+#define CMN_TXPUCAL_CTRL		(0xe0 << 2)
+#define CMN_TXPUCAL_OVRD		(0xe1 << 2)
+#define CMN_TXPDCAL_OVRD		(0xf1 << 2)
+#define CMN_DIAG_PLL0_FBH_OVRD		(0x1c0 << 2)
+#define CMN_DIAG_PLL0_FBL_OVRD		(0x1c1 << 2)
+#define CMN_DIAG_PLL0_OVRD		(0x1c2 << 2)
+#define CMN_DIAG_PLL0_V2I_TUNE		(0x1c5 << 2)
+#define CMN_DIAG_PLL0_CP_TUNE		(0x1c6 << 2)
+#define CMN_DIAG_PLL0_LF_PROG		(0x1c7 << 2)
+#define CMN_DIAG_PLL1_FBH_OVRD		(0x1d0 << 2)
+#define CMN_DIAG_PLL1_FBL_OVRD		(0x1d1 << 2)
+#define CMN_DIAG_PLL1_OVRD		(0x1d2 << 2)
+#define CMN_DIAG_PLL1_V2I_TUNE		(0x1d5 << 2)
+#define CMN_DIAG_PLL1_CP_TUNE		(0x1d6 << 2)
+#define CMN_DIAG_PLL1_LF_PROG		(0x1d7 << 2)
+#define CMN_DIAG_PLL1_PTATIS_TUNE1	(0x1d8 << 2)
+#define CMN_DIAG_PLL1_PTATIS_TUNE2	(0x1d9 << 2)
+#define CMN_DIAG_PLL1_INCLK_CTRL	(0x1da << 2)
+#define CMN_DIAG_HSCLK_SEL		(0x1e0 << 2)
+
+#define XCVR_PSM_RCTRL(n)		((0x4001 | ((n) << 9)) << 2)
+#define XCVR_PSM_CAL_TMR(n)		((0x4002 | ((n) << 9)) << 2)
+#define XCVR_PSM_A0IN_TMR(n)		((0x4003 | ((n) << 9)) << 2)
+#define TX_TXCC_CAL_SCLR_MULT(n)	((0x4047 | ((n) << 9)) << 2)
+#define TX_TXCC_CPOST_MULT_00(n)	((0x404c | ((n) << 9)) << 2)
+#define TX_TXCC_CPOST_MULT_01(n)	((0x404d | ((n) << 9)) << 2)
+#define TX_TXCC_CPOST_MULT_10(n)	((0x404e | ((n) << 9)) << 2)
+#define TX_TXCC_CPOST_MULT_11(n)	((0x404f | ((n) << 9)) << 2)
+#define TX_TXCC_MGNFS_MULT_000(n)	((0x4050 | ((n) << 9)) << 2)
+#define TX_TXCC_MGNFS_MULT_001(n)	((0x4051 | ((n) << 9)) << 2)
+#define TX_TXCC_MGNFS_MULT_010(n)	((0x4052 | ((n) << 9)) << 2)
+#define TX_TXCC_MGNFS_MULT_011(n)	((0x4053 | ((n) << 9)) << 2)
+#define TX_TXCC_MGNFS_MULT_100(n)	((0x4054 | ((n) << 9)) << 2)
+#define TX_TXCC_MGNFS_MULT_101(n)	((0x4055 | ((n) << 9)) << 2)
+#define TX_TXCC_MGNFS_MULT_110(n)	((0x4056 | ((n) << 9)) << 2)
+#define TX_TXCC_MGNFS_MULT_111(n)	((0x4057 | ((n) << 9)) << 2)
+#define XCVR_DIAG_PLLDRC_CTRL(n)	((0x40e0 | ((n) << 9)) << 2)
+#define XCVR_DIAG_BIDI_CTRL(n)		((0x40e8 | ((n) << 9)) << 2)
+#define XCVR_DIAG_LANE_FCM_EN_MGN(n)	((0x40f2 | ((n) << 9)) << 2)
+#define TX_PSC_A0(n)			((0x4100 | ((n) << 9)) << 2)
+#define TX_PSC_A1(n)			((0x4101 | ((n) << 9)) << 2)
+#define TX_PSC_A2(n)			((0x4102 | ((n) << 9)) << 2)
+#define TX_PSC_A3(n)			((0x4103 | ((n) << 9)) << 2)
+#define TX_RCVDET_CTRL(n)		((0x4120 | ((n) << 9)) << 2)
+#define TX_RCVDET_EN_TMR(n)		((0x4122 | ((n) << 9)) << 2)
+#define TX_RCVDET_ST_TMR(n)		((0x4123 | ((n) << 9)) << 2)
+#define TX_DIAG_TX_DRV(n)		((0x41e1 | ((n) << 9)) << 2)
+#define TX_DIAG_BGREF_PREDRV_DELAY	(0x41e7 << 2)
+#define TX_ANA_CTRL_REG_1		(0x5020 << 2)
+#define TX_ANA_CTRL_REG_2		(0x5021 << 2)
+#define TXDA_COEFF_CALC_CTRL		(0x5022 << 2)
+#define TX_DIG_CTRL_REG_2		(0x5024 << 2)
+#define TXDA_CYA_AUXDA_CYA		(0x5025 << 2)
+#define TX_ANA_CTRL_REG_3		(0x5026 << 2)
+#define TX_ANA_CTRL_REG_4		(0x5027 << 2)
+#define TX_ANA_CTRL_REG_5		(0x5029 << 2)
+
+#define RX_PSC_A0(n)			((0x8000 | ((n) << 9)) << 2)
+#define RX_PSC_A1(n)			((0x8001 | ((n) << 9)) << 2)
+#define RX_PSC_A2(n)			((0x8002 | ((n) << 9)) << 2)
+#define RX_PSC_A3(n)			((0x8003 | ((n) << 9)) << 2)
+#define RX_PSC_CAL(n)			((0x8006 | ((n) << 9)) << 2)
+#define RX_PSC_RDY(n)			((0x8007 | ((n) << 9)) << 2)
+#define RX_IQPI_ILL_CAL_OVRD		(0x8023 << 2)
+#define RX_EPI_ILL_CAL_OVRD		(0x8033 << 2)
+#define RX_SDCAL0_OVRD			(0x8041 << 2)
+#define RX_SDCAL1_OVRD			(0x8049 << 2)
+#define RX_SLC_INIT			(0x806d << 2)
+#define RX_SLC_RUN			(0x806e << 2)
+#define RX_CDRLF_CNFG2			(0x8081 << 2)
+#define RX_SIGDET_HL_FILT_TMR(n)	((0x8090 | ((n) << 9)) << 2)
+#define RX_SLC_IOP0_OVRD		(0x8101 << 2)
+#define RX_SLC_IOP1_OVRD		(0x8105 << 2)
+#define RX_SLC_QOP0_OVRD		(0x8109 << 2)
+#define RX_SLC_QOP1_OVRD		(0x810d << 2)
+#define RX_SLC_EOP0_OVRD		(0x8111 << 2)
+#define RX_SLC_EOP1_OVRD		(0x8115 << 2)
+#define RX_SLC_ION0_OVRD		(0x8119 << 2)
+#define RX_SLC_ION1_OVRD		(0x811d << 2)
+#define RX_SLC_QON0_OVRD		(0x8121 << 2)
+#define RX_SLC_QON1_OVRD		(0x8125 << 2)
+#define RX_SLC_EON0_OVRD		(0x8129 << 2)
+#define RX_SLC_EON1_OVRD		(0x812d << 2)
+#define RX_SLC_IEP0_OVRD		(0x8131 << 2)
+#define RX_SLC_IEP1_OVRD		(0x8135 << 2)
+#define RX_SLC_QEP0_OVRD		(0x8139 << 2)
+#define RX_SLC_QEP1_OVRD		(0x813d << 2)
+#define RX_SLC_EEP0_OVRD		(0x8141 << 2)
+#define RX_SLC_EEP1_OVRD		(0x8145 << 2)
+#define RX_SLC_IEN0_OVRD		(0x8149 << 2)
+#define RX_SLC_IEN1_OVRD		(0x814d << 2)
+#define RX_SLC_QEN0_OVRD		(0x8151 << 2)
+#define RX_SLC_QEN1_OVRD		(0x8155 << 2)
+#define RX_SLC_EEN0_OVRD		(0x8159 << 2)
+#define RX_SLC_EEN1_OVRD		(0x815d << 2)
+#define RX_REE_CTRL_DATA_MASK(n)	((0x81bb | ((n) << 9)) << 2)
+#define RX_DIAG_SIGDET_TUNE(n)		((0x81dc | ((n) << 9)) << 2)
+#define RX_DIAG_SC2C_DELAY		(0x81e1 << 2)
+
+#define PMA_LANE_CFG			(0xc000 << 2)
+#define PIPE_CMN_CTRL1			(0xc001 << 2)
+#define PIPE_CMN_CTRL2			(0xc002 << 2)
+#define PIPE_COM_LOCK_CFG1		(0xc003 << 2)
+#define PIPE_COM_LOCK_CFG2		(0xc004 << 2)
+#define PIPE_RCV_DET_INH		(0xc005 << 2)
+#define DP_MODE_CTL			(0xc008 << 2)
+#define DP_CLK_CTL			(0xc009 << 2)
+#define STS				(0xc00F << 2)
+#define PHY_ISO_CMN_CTRL		(0xc010 << 2)
+#define PHY_DP_TX_CTL			(0xc408 << 2)
+#define PMA_CMN_CTRL1			(0xc800 << 2)
+#define PHY_PMA_ISO_CMN_CTRL		(0xc810 << 2)
+#define PHY_ISOLATION_CTRL		(0xc81f << 2)
+#define PHY_PMA_ISO_XCVR_CTRL(n)	((0xcc11 | ((n) << 6)) << 2)
+#define PHY_PMA_ISO_LINK_MODE(n)	((0xcc12 | ((n) << 6)) << 2)
+#define PHY_PMA_ISO_PWRST_CTRL(n)	((0xcc13 | ((n) << 6)) << 2)
+#define PHY_PMA_ISO_TX_DATA_LO(n)	((0xcc14 | ((n) << 6)) << 2)
+#define PHY_PMA_ISO_TX_DATA_HI(n)	((0xcc15 | ((n) << 6)) << 2)
+#define PHY_PMA_ISO_RX_DATA_LO(n)	((0xcc16 | ((n) << 6)) << 2)
+#define PHY_PMA_ISO_RX_DATA_HI(n)	((0xcc17 | ((n) << 6)) << 2)
+#define TX_BIST_CTRL(n)			((0x4140 | ((n) << 9)) << 2)
+#define TX_BIST_UDDWR(n)		((0x4141 | ((n) << 9)) << 2)
+
+/*
+ * Selects which PLL clock will be driven on the analog high speed
+ * clock 0: PLL 0 div 1
+ * clock 1: PLL 1 div 2
+ */
+#define CLK_PLL_CONFIG			0X30
+#define CLK_PLL_MASK			0x33
+
+#define CMN_READY			BIT(0)
+
+#define DP_PLL_CLOCK_ENABLE		BIT(2)
+#define DP_PLL_ENABLE			BIT(0)
+#define DP_PLL_DATA_RATE_RBR		((2 << 12) | (4 << 8))
+#define DP_PLL_DATA_RATE_HBR		((2 << 12) | (4 << 8))
+#define DP_PLL_DATA_RATE_HBR2		((1 << 12) | (2 << 8))
+
+#define DP_MODE_A0			BIT(4)
+#define DP_MODE_A2			BIT(6)
+#define DP_MODE_ENTER_A0		0xc101
+#define DP_MODE_ENTER_A2		0xc104
+
+#define PHY_MODE_SET_TIMEOUT		100000
+
+#define PIN_ASSIGN_C_E			0x51d9
+#define PIN_ASSIGN_D_F			0x5100
+
+#define MODE_DISCONNECT			0
+#define MODE_UFP_USB			BIT(0)
+#define MODE_DFP_USB			BIT(1)
+#define MODE_DFP_DP			BIT(2)
+
+struct usb3phy_reg {
+	u32 offset;
+	u32 enable_bit;
+	u32 write_enable;
+};
+
+struct rockchip_usb3phy_port_cfg {
+	struct usb3phy_reg typec_conn_dir;
+	struct usb3phy_reg usb3tousb2_en;
+	struct usb3phy_reg external_psm;
+	struct usb3phy_reg pipe_status;
+};
+
+struct rockchip_typec_phy {
+	struct device *dev;
+	void __iomem *base;
+	struct extcon_dev *extcon;
+	struct regmap *grf_regs;
+	struct clk *clk_core;
+	struct clk *clk_ref;
+	struct reset_control *uphy_rst;
+	struct reset_control *pipe_rst;
+	struct reset_control *tcphy_rst;
+	struct rockchip_usb3phy_port_cfg port_cfgs;
+	/* mutex to protect access to individual PHYs */
+	struct mutex lock;
+
+	bool flip;
+	u8 mode;
+};
+
+struct phy_reg {
+	u16 value;
+	u32 addr;
+};
+
+struct phy_reg usb3_pll_cfg[] = {
+	{ 0xf0,		CMN_PLL0_VCOCAL_INIT },
+	{ 0x18,		CMN_PLL0_VCOCAL_ITER },
+	{ 0xd0,		CMN_PLL0_INTDIV },
+	{ 0x4a4a,	CMN_PLL0_FRACDIV },
+	{ 0x34,		CMN_PLL0_HIGH_THR },
+	{ 0x1ee,	CMN_PLL0_SS_CTRL1 },
+	{ 0x7f03,	CMN_PLL0_SS_CTRL2 },
+	{ 0x20,		CMN_PLL0_DSM_DIAG },
+	{ 0,		CMN_DIAG_PLL0_OVRD },
+	{ 0,		CMN_DIAG_PLL0_FBH_OVRD },
+	{ 0,		CMN_DIAG_PLL0_FBL_OVRD },
+	{ 0x7,		CMN_DIAG_PLL0_V2I_TUNE },
+	{ 0x45,		CMN_DIAG_PLL0_CP_TUNE },
+	{ 0x8,		CMN_DIAG_PLL0_LF_PROG },
+};
+
+struct phy_reg dp_pll_cfg[] = {
+	{ 0xf0,		CMN_PLL1_VCOCAL_INIT },
+	{ 0x18,		CMN_PLL1_VCOCAL_ITER },
+	{ 0x30b9,	CMN_PLL1_VCOCAL_START },
+	{ 0x21c,	CMN_PLL1_INTDIV },
+	{ 0,		CMN_PLL1_FRACDIV },
+	{ 0x5,		CMN_PLL1_HIGH_THR },
+	{ 0x35,		CMN_PLL1_SS_CTRL1 },
+	{ 0x7f1e,	CMN_PLL1_SS_CTRL2 },
+	{ 0x20,		CMN_PLL1_DSM_DIAG },
+	{ 0,		CMN_PLLSM1_USER_DEF_CTRL },
+	{ 0,		CMN_DIAG_PLL1_OVRD },
+	{ 0,		CMN_DIAG_PLL1_FBH_OVRD },
+	{ 0,		CMN_DIAG_PLL1_FBL_OVRD },
+	{ 0x6,		CMN_DIAG_PLL1_V2I_TUNE },
+	{ 0x45,		CMN_DIAG_PLL1_CP_TUNE },
+	{ 0x8,		CMN_DIAG_PLL1_LF_PROG },
+	{ 0x100,	CMN_DIAG_PLL1_PTATIS_TUNE1 },
+	{ 0x7,		CMN_DIAG_PLL1_PTATIS_TUNE2 },
+	{ 0x4,		CMN_DIAG_PLL1_INCLK_CTRL },
+};
+
+static void tcphy_cfg_24m(struct rockchip_typec_phy *tcphy)
+{
+	u32 i, rdata;
+
+	/*
+	 * cmn_ref_clk_sel = 3, select the 24Mhz for clk parent
+	 * cmn_psm_clk_dig_div = 2, set the clk division to 2
+	 */
+	writel(0x830, tcphy->base + PMA_CMN_CTRL1);
+	for (i = 0; i < 4; i++) {
+		/*
+		 * The following PHY configuration assumes a 24 MHz reference
+		 * clock.
+		 */
+		writel(0x90, tcphy->base + XCVR_DIAG_LANE_FCM_EN_MGN(i));
+		writel(0x960, tcphy->base + TX_RCVDET_EN_TMR(i));
+		writel(0x30, tcphy->base + TX_RCVDET_ST_TMR(i));
+	}
+
+	rdata = readl(tcphy->base + CMN_DIAG_HSCLK_SEL);
+	rdata &= ~CLK_PLL_MASK;
+	rdata |= CLK_PLL_CONFIG;
+	writel(rdata, tcphy->base + CMN_DIAG_HSCLK_SEL);
+}
+
+static void tcphy_cfg_usb3_pll(struct rockchip_typec_phy *tcphy)
+{
+	u32 i;
+
+	/* load the configuration of PLL0 */
+	for (i = 0; i < ARRAY_SIZE(usb3_pll_cfg); i++)
+		writel(usb3_pll_cfg[i].value,
+		       tcphy->base + usb3_pll_cfg[i].addr);
+}
+
+static void tcphy_cfg_dp_pll(struct rockchip_typec_phy *tcphy)
+{
+	u32 i;
+
+	/* set the default mode to RBR */
+	writel(DP_PLL_CLOCK_ENABLE | DP_PLL_ENABLE | DP_PLL_DATA_RATE_RBR,
+	       tcphy->base + DP_CLK_CTL);
+
+	/* load the configuration of PLL1 */
+	for (i = 0; i < ARRAY_SIZE(dp_pll_cfg); i++)
+		writel(dp_pll_cfg[i].value, tcphy->base + dp_pll_cfg[i].addr);
+}
+
+static void tcphy_tx_usb3_cfg_lane(struct rockchip_typec_phy *tcphy, u32 lane)
+{
+	writel(0x7799, tcphy->base + TX_PSC_A0(lane));
+	writel(0x7798, tcphy->base + TX_PSC_A1(lane));
+	writel(0x5098, tcphy->base + TX_PSC_A2(lane));
+	writel(0x5098, tcphy->base + TX_PSC_A3(lane));
+	writel(0, tcphy->base + TX_TXCC_MGNFS_MULT_000(lane));
+	writel(0xbf, tcphy->base + XCVR_DIAG_BIDI_CTRL(lane));
+}
+
+static void tcphy_rx_usb3_cfg_lane(struct rockchip_typec_phy *tcphy, u32 lane)
+{
+	writel(0xa6fd, tcphy->base + RX_PSC_A0(lane));
+	writel(0xa6fd, tcphy->base + RX_PSC_A1(lane));
+	writel(0xa410, tcphy->base + RX_PSC_A2(lane));
+	writel(0x2410, tcphy->base + RX_PSC_A3(lane));
+	writel(0x23ff, tcphy->base + RX_PSC_CAL(lane));
+	writel(0x13, tcphy->base + RX_SIGDET_HL_FILT_TMR(lane));
+	writel(0x03e7, tcphy->base + RX_REE_CTRL_DATA_MASK(lane));
+	writel(0x1004, tcphy->base + RX_DIAG_SIGDET_TUNE(lane));
+	writel(0x2010, tcphy->base + RX_PSC_RDY(lane));
+	writel(0xfb, tcphy->base + XCVR_DIAG_BIDI_CTRL(lane));
+}
+
+static void tcphy_dp_cfg_lane(struct rockchip_typec_phy *tcphy, u32 lane)
+{
+	u16 rdata;
+
+	writel(0xbefc, tcphy->base + XCVR_PSM_RCTRL(lane));
+	writel(0x6799, tcphy->base + TX_PSC_A0(lane));
+	writel(0x6798, tcphy->base + TX_PSC_A1(lane));
+	writel(0x98, tcphy->base + TX_PSC_A2(lane));
+	writel(0x98, tcphy->base + TX_PSC_A3(lane));
+
+	writel(0, tcphy->base + TX_TXCC_MGNFS_MULT_000(lane));
+	writel(0, tcphy->base + TX_TXCC_MGNFS_MULT_001(lane));
+	writel(0, tcphy->base + TX_TXCC_MGNFS_MULT_010(lane));
+	writel(0, tcphy->base + TX_TXCC_MGNFS_MULT_011(lane));
+	writel(0, tcphy->base + TX_TXCC_MGNFS_MULT_100(lane));
+	writel(0, tcphy->base + TX_TXCC_MGNFS_MULT_101(lane));
+	writel(0, tcphy->base + TX_TXCC_MGNFS_MULT_110(lane));
+	writel(0, tcphy->base + TX_TXCC_MGNFS_MULT_111(lane));
+	writel(0, tcphy->base + TX_TXCC_CPOST_MULT_10(lane));
+	writel(0, tcphy->base + TX_TXCC_CPOST_MULT_01(lane));
+	writel(0, tcphy->base + TX_TXCC_CPOST_MULT_00(lane));
+	writel(0, tcphy->base + TX_TXCC_CPOST_MULT_11(lane));
+
+	writel(0x128, tcphy->base + TX_TXCC_CAL_SCLR_MULT(lane));
+	writel(0x400, tcphy->base + TX_DIAG_TX_DRV(lane));
+
+	rdata = readl(tcphy->base + XCVR_DIAG_PLLDRC_CTRL(lane));
+	rdata = (rdata & 0x8fff) | 0x6000;
+	writel(rdata, tcphy->base + XCVR_DIAG_PLLDRC_CTRL(lane));
+}
+
+static inline int property_enable(struct rockchip_typec_phy *tcphy,
+				  const struct usb3phy_reg *reg, bool en)
+{
+	u32 mask = 1 << reg->write_enable;
+	u32 val = en << reg->enable_bit;
+
+	return regmap_write(tcphy->grf_regs, reg->offset, val | mask);
+}
+
+static void tcphy_dp_aux_calibration(struct rockchip_typec_phy *tcphy)
+{
+	u16 rdata, rdata2, val;
+
+	/* disable txda_cal_latch_en for rewrite the calibration values */
+	rdata = readl(tcphy->base + TX_ANA_CTRL_REG_1);
+	val = rdata & 0xdfff;
+	writel(val, tcphy->base + TX_ANA_CTRL_REG_1);
+
+	/*
+	 * read a resistor calibration code from CMN_TXPUCAL_CTRL[6:0] and
+	 * write it to TX_DIG_CTRL_REG_2[6:0], and delay 1ms to make sure it
+	 * works.
+	 */
+	rdata = readl(tcphy->base + TX_DIG_CTRL_REG_2);
+	rdata = rdata & 0xffc0;
+
+	rdata2 = readl(tcphy->base + CMN_TXPUCAL_CTRL);
+	rdata2 = rdata2 & 0x3f;
+
+	val = rdata | rdata2;
+	writel(val, tcphy->base + TX_DIG_CTRL_REG_2);
+	usleep_range(1000, 1050);
+
+	/*
+	 * Enable signal for latch that sample and holds calibration values.
+	 * Activate this signal for 1 clock cycle to sample new calibration
+	 * values.
+	 */
+	rdata = readl(tcphy->base + TX_ANA_CTRL_REG_1);
+	val = rdata | 0x2000;
+	writel(val, tcphy->base + TX_ANA_CTRL_REG_1);
+	usleep_range(150, 200);
+
+	/* set TX Voltage Level and TX Deemphasis to 0 */
+	writel(0, tcphy->base + PHY_DP_TX_CTL);
+	/* re-enable decap */
+	writel(0x100, tcphy->base + TX_ANA_CTRL_REG_2);
+	writel(0x300, tcphy->base + TX_ANA_CTRL_REG_2);
+	writel(0x2008, tcphy->base + TX_ANA_CTRL_REG_1);
+	writel(0x2018, tcphy->base + TX_ANA_CTRL_REG_1);
+
+	writel(0, tcphy->base + TX_ANA_CTRL_REG_5);
+
+	/*
+	 * Programs txda_drv_ldo_prog[15:0], Sets driver LDO
+	 * voltage 16'h1001 for DP-AUX-TX and RX
+	 */
+	writel(0x1001, tcphy->base + TX_ANA_CTRL_REG_4);
+
+	/* re-enables Bandgap reference for LDO */
+	writel(0x2098, tcphy->base + TX_ANA_CTRL_REG_1);
+	writel(0x2198, tcphy->base + TX_ANA_CTRL_REG_1);
+
+	/*
+	 * re-enables the transmitter pre-driver, driver data selection MUX,
+	 * and receiver detect circuits.
+	 */
+	writel(0x301, tcphy->base + TX_ANA_CTRL_REG_2);
+	writel(0x303, tcphy->base + TX_ANA_CTRL_REG_2);
+
+	/*
+	 * BIT 12: Controls auxda_polarity, which selects the polarity of the
+	 * xcvr:
+	 * 1, Reverses the polarity (If TYPEC, Pulls ups aux_p and pull
+	 * down aux_m)
+	 * 0, Normal polarity (if TYPE_C, pulls up aux_m and pulls down
+	 * aux_p)
+	 */
+	val = 0xa078;
+	if (!tcphy->flip)
+		val |= BIT(12);
+	writel(val, tcphy->base + TX_ANA_CTRL_REG_1);
+
+	writel(0, tcphy->base + TX_ANA_CTRL_REG_3);
+	writel(0, tcphy->base + TX_ANA_CTRL_REG_4);
+	writel(0, tcphy->base + TX_ANA_CTRL_REG_5);
+
+	/*
+	 * Controls low_power_swing_en, set the voltage swing of the driver
+	 * to 400mv. The values	below are peak to peak (differential) values.
+	 */
+	writel(4, tcphy->base + TXDA_COEFF_CALC_CTRL);
+	writel(0, tcphy->base + TXDA_CYA_AUXDA_CYA);
+
+	/* Controls tx_high_z_tm_en */
+	val = readl(tcphy->base + TX_DIG_CTRL_REG_2);
+	val |= BIT(15);
+	writel(val, tcphy->base + TX_DIG_CTRL_REG_2);
+}
+
+static int tcphy_phy_init(struct rockchip_typec_phy *tcphy, u8 mode)
+{
+	struct rockchip_usb3phy_port_cfg *cfg = &tcphy->port_cfgs;
+	int ret, i;
+	u32 val;
+
+	ret = clk_prepare_enable(tcphy->clk_core);
+	if (ret) {
+		dev_err(tcphy->dev, "Failed to prepare_enable core clock\n");
+		return ret;
+	}
+
+	ret = clk_prepare_enable(tcphy->clk_ref);
+	if (ret) {
+		dev_err(tcphy->dev, "Failed to prepare_enable ref clock\n");
+		goto err_clk_core;
+	}
+
+	reset_control_deassert(tcphy->tcphy_rst);
+
+	property_enable(tcphy, &cfg->typec_conn_dir, tcphy->flip);
+
+	tcphy_cfg_24m(tcphy);
+
+	if (mode == MODE_DFP_DP) {
+		tcphy_cfg_dp_pll(tcphy);
+		for (i = 0; i < 4; i++)
+			tcphy_dp_cfg_lane(tcphy, i);
+
+		writel(PIN_ASSIGN_C_E, tcphy->base + PMA_LANE_CFG);
+	} else {
+		tcphy_cfg_usb3_pll(tcphy);
+		tcphy_cfg_dp_pll(tcphy);
+		if (tcphy->flip) {
+			tcphy_tx_usb3_cfg_lane(tcphy, 3);
+			tcphy_rx_usb3_cfg_lane(tcphy, 2);
+			tcphy_dp_cfg_lane(tcphy, 0);
+			tcphy_dp_cfg_lane(tcphy, 1);
+		} else {
+			tcphy_tx_usb3_cfg_lane(tcphy, 0);
+			tcphy_rx_usb3_cfg_lane(tcphy, 1);
+			tcphy_dp_cfg_lane(tcphy, 2);
+			tcphy_dp_cfg_lane(tcphy, 3);
+		}
+
+		writel(PIN_ASSIGN_D_F, tcphy->base + PMA_LANE_CFG);
+	}
+
+	writel(DP_MODE_ENTER_A2, tcphy->base + DP_MODE_CTL);
+
+	reset_control_deassert(tcphy->uphy_rst);
+
+	ret = readx_poll_timeout(readl, tcphy->base + PMA_CMN_CTRL1,
+				 val, val & CMN_READY, 10,
+				 PHY_MODE_SET_TIMEOUT);
+	if (ret < 0) {
+		dev_err(tcphy->dev, "wait pma ready timeout\n");
+		ret = -ETIMEDOUT;
+		goto err_wait_pma;
+	}
+
+	reset_control_deassert(tcphy->pipe_rst);
+
+	return 0;
+
+err_wait_pma:
+	reset_control_assert(tcphy->uphy_rst);
+	reset_control_assert(tcphy->tcphy_rst);
+	clk_disable_unprepare(tcphy->clk_ref);
+err_clk_core:
+	clk_disable_unprepare(tcphy->clk_core);
+	return ret;
+}
+
+static void tcphy_phy_deinit(struct rockchip_typec_phy *tcphy)
+{
+	reset_control_assert(tcphy->tcphy_rst);
+	reset_control_assert(tcphy->uphy_rst);
+	reset_control_assert(tcphy->pipe_rst);
+	clk_disable_unprepare(tcphy->clk_core);
+	clk_disable_unprepare(tcphy->clk_ref);
+}
+
+static int tcphy_get_mode(struct rockchip_typec_phy *tcphy)
+{
+	struct extcon_dev *edev = tcphy->extcon;
+	union extcon_property_value property;
+	unsigned int id;
+	bool dfp, ufp, dp;
+	u8 mode;
+	int ret;
+
+	ufp = extcon_get_state(edev, EXTCON_USB);
+	dfp = extcon_get_state(edev, EXTCON_USB_HOST);
+	dp = extcon_get_state(edev, EXTCON_DISP_DP);
+
+	mode = MODE_DFP_USB;
+	id = EXTCON_USB_HOST;
+
+	if (ufp) {
+		mode = MODE_UFP_USB;
+		id = EXTCON_USB;
+	} else if (dp) {
+		mode = MODE_DFP_DP;
+		id = EXTCON_DISP_DP;
+
+		ret = extcon_get_property(edev, id, EXTCON_PROP_USB_SS,
+					  &property);
+		if (ret) {
+			dev_err(tcphy->dev, "get superspeed property failed\n");
+			return ret;
+		}
+
+		if (property.intval)
+			mode |= MODE_DFP_USB;
+	}
+
+	ret = extcon_get_property(edev, id, EXTCON_PROP_USB_TYPEC_POLARITY,
+				  &property);
+	if (ret) {
+		dev_err(tcphy->dev, "get polarity property failed\n");
+		return ret;
+	}
+
+	tcphy->flip = property.intval ? 1 : 0;
+
+	return mode;
+}
+
+static int rockchip_usb3_phy_power_on(struct phy *phy)
+{
+	struct rockchip_typec_phy *tcphy = phy_get_drvdata(phy);
+	struct rockchip_usb3phy_port_cfg *cfg = &tcphy->port_cfgs;
+	const struct usb3phy_reg *reg = &cfg->pipe_status;
+	int timeout, new_mode, ret = 0;
+	u32 val;
+
+	mutex_lock(&tcphy->lock);
+
+	new_mode = tcphy_get_mode(tcphy);
+	if (new_mode < 0) {
+		ret = new_mode;
+		goto unlock_ret;
+	}
+
+	/* DP-only mode; fall back to USB2 */
+	if (!(new_mode & (MODE_DFP_USB | MODE_UFP_USB)))
+		goto unlock_ret;
+
+	if (tcphy->mode == new_mode)
+		goto unlock_ret;
+
+	if (tcphy->mode == MODE_DISCONNECT)
+		tcphy_phy_init(tcphy, new_mode);
+
+	/* wait TCPHY for pipe ready */
+	for (timeout = 0; timeout < 100; timeout++) {
+		regmap_read(tcphy->grf_regs, reg->offset, &val);
+		if (!(val & BIT(reg->enable_bit))) {
+			tcphy->mode |= new_mode & (MODE_DFP_USB | MODE_UFP_USB);
+			goto unlock_ret;
+		}
+		usleep_range(10, 20);
+	}
+
+	if (tcphy->mode == MODE_DISCONNECT)
+		tcphy_phy_deinit(tcphy);
+
+	ret = -ETIMEDOUT;
+
+unlock_ret:
+	mutex_unlock(&tcphy->lock);
+	return ret;
+}
+
+static int rockchip_usb3_phy_power_off(struct phy *phy)
+{
+	struct rockchip_typec_phy *tcphy = phy_get_drvdata(phy);
+
+	mutex_lock(&tcphy->lock);
+
+	if (tcphy->mode == MODE_DISCONNECT)
+		goto unlock;
+
+	tcphy->mode &= ~(MODE_UFP_USB | MODE_DFP_USB);
+	if (tcphy->mode == MODE_DISCONNECT)
+		tcphy_phy_deinit(tcphy);
+
+unlock:
+	mutex_unlock(&tcphy->lock);
+	return 0;
+}
+
+static const struct phy_ops rockchip_usb3_phy_ops = {
+	.power_on	= rockchip_usb3_phy_power_on,
+	.power_off	= rockchip_usb3_phy_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static int rockchip_dp_phy_power_on(struct phy *phy)
+{
+	struct rockchip_typec_phy *tcphy = phy_get_drvdata(phy);
+	int new_mode, ret = 0;
+	u32 val;
+
+	mutex_lock(&tcphy->lock);
+
+	new_mode = tcphy_get_mode(tcphy);
+	if (new_mode < 0) {
+		ret = new_mode;
+		goto unlock_ret;
+	}
+
+	if (!(new_mode & MODE_DFP_DP)) {
+		ret = -ENODEV;
+		goto unlock_ret;
+	}
+
+	if (tcphy->mode == new_mode)
+		goto unlock_ret;
+
+	/*
+	 * If the PHY has been power on, but the mode is not DP only mode,
+	 * re-init the PHY for setting all of 4 lanes to DP.
+	 */
+	if (new_mode == MODE_DFP_DP && tcphy->mode != MODE_DISCONNECT) {
+		tcphy_phy_deinit(tcphy);
+		tcphy_phy_init(tcphy, new_mode);
+	} else if (tcphy->mode == MODE_DISCONNECT) {
+		tcphy_phy_init(tcphy, new_mode);
+	}
+
+	ret = readx_poll_timeout(readl, tcphy->base + DP_MODE_CTL,
+				 val, val & DP_MODE_A2, 1000,
+				 PHY_MODE_SET_TIMEOUT);
+	if (ret < 0) {
+		dev_err(tcphy->dev, "failed to wait TCPHY enter A2\n");
+		goto power_on_finish;
+	}
+
+	tcphy_dp_aux_calibration(tcphy);
+
+	writel(DP_MODE_ENTER_A0, tcphy->base + DP_MODE_CTL);
+
+	ret = readx_poll_timeout(readl, tcphy->base + DP_MODE_CTL,
+				 val, val & DP_MODE_A0, 1000,
+				 PHY_MODE_SET_TIMEOUT);
+	if (ret < 0) {
+		writel(DP_MODE_ENTER_A2, tcphy->base + DP_MODE_CTL);
+		dev_err(tcphy->dev, "failed to wait TCPHY enter A0\n");
+		goto power_on_finish;
+	}
+
+	tcphy->mode |= MODE_DFP_DP;
+
+power_on_finish:
+	if (tcphy->mode == MODE_DISCONNECT)
+		tcphy_phy_deinit(tcphy);
+unlock_ret:
+	mutex_unlock(&tcphy->lock);
+	return ret;
+}
+
+static int rockchip_dp_phy_power_off(struct phy *phy)
+{
+	struct rockchip_typec_phy *tcphy = phy_get_drvdata(phy);
+
+	mutex_lock(&tcphy->lock);
+
+	if (tcphy->mode == MODE_DISCONNECT)
+		goto unlock;
+
+	tcphy->mode &= ~MODE_DFP_DP;
+
+	writel(DP_MODE_ENTER_A2, tcphy->base + DP_MODE_CTL);
+
+	if (tcphy->mode == MODE_DISCONNECT)
+		tcphy_phy_deinit(tcphy);
+
+unlock:
+	mutex_unlock(&tcphy->lock);
+	return 0;
+}
+
+static const struct phy_ops rockchip_dp_phy_ops = {
+	.power_on	= rockchip_dp_phy_power_on,
+	.power_off	= rockchip_dp_phy_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static int tcphy_get_param(struct device *dev,
+			   struct usb3phy_reg *reg,
+			   const char *name)
+{
+	u32 buffer[3];
+	int ret;
+
+	ret = of_property_read_u32_array(dev->of_node, name, buffer, 3);
+	if (ret) {
+		dev_err(dev, "Can not parse %s\n", name);
+		return ret;
+	}
+
+	reg->offset = buffer[0];
+	reg->enable_bit = buffer[1];
+	reg->write_enable = buffer[2];
+	return 0;
+}
+
+static int tcphy_parse_dt(struct rockchip_typec_phy *tcphy,
+			  struct device *dev)
+{
+	struct rockchip_usb3phy_port_cfg *cfg = &tcphy->port_cfgs;
+	int ret;
+
+	ret = tcphy_get_param(dev, &cfg->typec_conn_dir,
+			      "rockchip,typec-conn-dir");
+	if (ret)
+		return ret;
+
+	ret = tcphy_get_param(dev, &cfg->usb3tousb2_en,
+			      "rockchip,usb3tousb2-en");
+	if (ret)
+		return ret;
+
+	ret = tcphy_get_param(dev, &cfg->external_psm,
+			      "rockchip,external-psm");
+	if (ret)
+		return ret;
+
+	ret = tcphy_get_param(dev, &cfg->pipe_status,
+			      "rockchip,pipe-status");
+	if (ret)
+		return ret;
+
+	tcphy->grf_regs = syscon_regmap_lookup_by_phandle(dev->of_node,
+							  "rockchip,grf");
+	if (IS_ERR(tcphy->grf_regs)) {
+		dev_err(dev, "could not find grf dt node\n");
+		return PTR_ERR(tcphy->grf_regs);
+	}
+
+	tcphy->clk_core = devm_clk_get(dev, "tcpdcore");
+	if (IS_ERR(tcphy->clk_core)) {
+		dev_err(dev, "could not get uphy core clock\n");
+		return PTR_ERR(tcphy->clk_core);
+	}
+
+	tcphy->clk_ref = devm_clk_get(dev, "tcpdphy-ref");
+	if (IS_ERR(tcphy->clk_ref)) {
+		dev_err(dev, "could not get uphy ref clock\n");
+		return PTR_ERR(tcphy->clk_ref);
+	}
+
+	tcphy->uphy_rst = devm_reset_control_get(dev, "uphy");
+	if (IS_ERR(tcphy->uphy_rst)) {
+		dev_err(dev, "no uphy_rst reset control found\n");
+		return PTR_ERR(tcphy->uphy_rst);
+	}
+
+	tcphy->pipe_rst = devm_reset_control_get(dev, "uphy-pipe");
+	if (IS_ERR(tcphy->pipe_rst)) {
+		dev_err(dev, "no pipe_rst reset control found\n");
+		return PTR_ERR(tcphy->pipe_rst);
+	}
+
+	tcphy->tcphy_rst = devm_reset_control_get(dev, "uphy-tcphy");
+	if (IS_ERR(tcphy->tcphy_rst)) {
+		dev_err(dev, "no tcphy_rst reset control found\n");
+		return PTR_ERR(tcphy->tcphy_rst);
+	}
+
+	return 0;
+}
+
+static void typec_phy_pre_init(struct rockchip_typec_phy *tcphy)
+{
+	struct rockchip_usb3phy_port_cfg *cfg = &tcphy->port_cfgs;
+
+	reset_control_assert(tcphy->tcphy_rst);
+	reset_control_assert(tcphy->uphy_rst);
+	reset_control_assert(tcphy->pipe_rst);
+
+	/* select external psm clock */
+	property_enable(tcphy, &cfg->external_psm, 1);
+	property_enable(tcphy, &cfg->usb3tousb2_en, 0);
+
+	tcphy->mode = MODE_DISCONNECT;
+}
+
+static int rockchip_typec_phy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *child_np;
+	struct rockchip_typec_phy *tcphy;
+	struct phy_provider *phy_provider;
+	struct resource *res;
+	int ret;
+
+	tcphy = devm_kzalloc(dev, sizeof(*tcphy), GFP_KERNEL);
+	if (!tcphy)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	tcphy->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(tcphy->base))
+		return PTR_ERR(tcphy->base);
+
+	ret = tcphy_parse_dt(tcphy, dev);
+	if (ret)
+		return ret;
+
+	tcphy->dev = dev;
+	platform_set_drvdata(pdev, tcphy);
+	mutex_init(&tcphy->lock);
+
+	typec_phy_pre_init(tcphy);
+
+	tcphy->extcon = extcon_get_edev_by_phandle(dev, 0);
+	if (IS_ERR(tcphy->extcon)) {
+		if (PTR_ERR(tcphy->extcon) != -EPROBE_DEFER)
+			dev_err(dev, "Invalid or missing extcon\n");
+		return PTR_ERR(tcphy->extcon);
+	}
+
+	pm_runtime_enable(dev);
+
+	for_each_available_child_of_node(np, child_np) {
+		struct phy *phy;
+
+		if (!of_node_cmp(child_np->name, "dp-port"))
+			phy = devm_phy_create(dev, child_np,
+					      &rockchip_dp_phy_ops);
+		else if (!of_node_cmp(child_np->name, "usb3-port"))
+			phy = devm_phy_create(dev, child_np,
+					      &rockchip_usb3_phy_ops);
+		else
+			continue;
+
+		if (IS_ERR(phy)) {
+			dev_err(dev, "failed to create phy: %s\n",
+				child_np->name);
+			return PTR_ERR(phy);
+		}
+
+		phy_set_drvdata(phy, tcphy);
+	}
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	if (IS_ERR(phy_provider)) {
+		dev_err(dev, "Failed to register phy provider\n");
+		return PTR_ERR(phy_provider);
+	}
+
+	return 0;
+}
+
+static int rockchip_typec_phy_remove(struct platform_device *pdev)
+{
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static const struct of_device_id rockchip_typec_phy_dt_ids[] = {
+	{ .compatible = "rockchip,rk3399-typec-phy" },
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, rockchip_typec_phy_dt_ids);
+
+static struct platform_driver rockchip_typec_phy_driver = {
+	.probe		= rockchip_typec_phy_probe,
+	.remove		= rockchip_typec_phy_remove,
+	.driver		= {
+		.name	= "rockchip-typec-phy",
+		.of_match_table = rockchip_typec_phy_dt_ids,
+	},
+};
+
+module_platform_driver(rockchip_typec_phy_driver);
+
+MODULE_AUTHOR("Chris Zhong <zyw@rock-chips.com>");
+MODULE_AUTHOR("Kever Yang <kever.yang@rock-chips.com>");
+MODULE_DESCRIPTION("Rockchip USB TYPE-C PHY driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/phy-rockchip-usb.c b/drivers/phy/phy-rockchip-usb.c
index 2a7381f4fe4c..734987fa0ad7 100644
--- a/drivers/phy/phy-rockchip-usb.c
+++ b/drivers/phy/phy-rockchip-usb.c
@@ -29,6 +29,7 @@
 #include <linux/reset.h>
 #include <linux/regmap.h>
 #include <linux/mfd/syscon.h>
+#include <linux/delay.h>
 
 static int enable_usb_uart;
 
@@ -64,6 +65,7 @@ struct rockchip_usb_phy {
 	struct clk_hw	clk480m_hw;
 	struct phy	*phy;
 	bool		uart_enabled;
+	struct reset_control *reset;
 };
 
 static int rockchip_usb_phy_power(struct rockchip_usb_phy *phy,
@@ -144,9 +146,23 @@ static int rockchip_usb_phy_power_on(struct phy *_phy)
 	return clk_prepare_enable(phy->clk480m);
 }
 
+static int rockchip_usb_phy_reset(struct phy *_phy)
+{
+	struct rockchip_usb_phy *phy = phy_get_drvdata(_phy);
+
+	if (phy->reset) {
+		reset_control_assert(phy->reset);
+		udelay(10);
+		reset_control_deassert(phy->reset);
+	}
+
+	return 0;
+}
+
 static const struct phy_ops ops = {
 	.power_on	= rockchip_usb_phy_power_on,
 	.power_off	= rockchip_usb_phy_power_off,
+	.reset		= rockchip_usb_phy_reset,
 	.owner		= THIS_MODULE,
 };
 
@@ -185,6 +201,10 @@ static int rockchip_usb_phy_init(struct rockchip_usb_phy_base *base,
 		return -EINVAL;
 	}
 
+	rk_phy->reset = of_reset_control_get(child, "phy-reset");
+	if (IS_ERR(rk_phy->reset))
+		rk_phy->reset = NULL;
+
 	rk_phy->reg_offset = reg_offset;
 
 	rk_phy->clk = of_clk_get_by_name(child, "phyclk");
diff --git a/drivers/phy/phy-sun4i-usb.c b/drivers/phy/phy-sun4i-usb.c
index 0a45bc6088ae..b9342a2af7b3 100644
--- a/drivers/phy/phy-sun4i-usb.c
+++ b/drivers/phy/phy-sun4i-usb.c
@@ -40,6 +40,8 @@
 #include <linux/power_supply.h>
 #include <linux/regulator/consumer.h>
 #include <linux/reset.h>
+#include <linux/spinlock.h>
+#include <linux/usb/of.h>
 #include <linux/workqueue.h>
 
 #define REG_ISCR			0x00
@@ -49,7 +51,7 @@
 #define REG_PHYCTL_A33			0x10
 #define REG_PHY_UNK_H3			0x20
 
-#define REG_PMU_UNK_H3			0x10
+#define REG_PMU_UNK1			0x10
 
 #define PHYCTL_DATA			BIT(7)
 
@@ -97,6 +99,7 @@ enum sun4i_usb_phy_type {
 	sun6i_a31_phy,
 	sun8i_a33_phy,
 	sun8i_h3_phy,
+	sun50i_a64_phy,
 };
 
 struct sun4i_usb_phy_cfg {
@@ -105,12 +108,14 @@ struct sun4i_usb_phy_cfg {
 	u32 disc_thresh;
 	u8 phyctl_offset;
 	bool dedicated_clocks;
+	bool enable_pmu_unk1;
 };
 
 struct sun4i_usb_phy_data {
 	void __iomem *base;
 	const struct sun4i_usb_phy_cfg *cfg;
-	struct mutex mutex;
+	enum usb_dr_mode dr_mode;
+	spinlock_t reg_lock; /* guard access to phyctl reg */
 	struct sun4i_usb_phy {
 		struct phy *phy;
 		void __iomem *pmu;
@@ -128,6 +133,7 @@ struct sun4i_usb_phy_data {
 	struct power_supply *vbus_power_supply;
 	struct notifier_block vbus_power_nb;
 	bool vbus_power_nb_registered;
+	bool force_session_end;
 	int id_det_irq;
 	int vbus_det_irq;
 	int id_det;
@@ -176,12 +182,14 @@ static void sun4i_usb_phy_write(struct sun4i_usb_phy *phy, u32 addr, u32 data,
 	struct sun4i_usb_phy_data *phy_data = to_sun4i_usb_phy_data(phy);
 	u32 temp, usbc_bit = BIT(phy->index * 2);
 	void __iomem *phyctl = phy_data->base + phy_data->cfg->phyctl_offset;
+	unsigned long flags;
 	int i;
 
-	mutex_lock(&phy_data->mutex);
+	spin_lock_irqsave(&phy_data->reg_lock, flags);
 
-	if (phy_data->cfg->type == sun8i_a33_phy) {
-		/* A33 needs us to set phyctl to 0 explicitly */
+	if (phy_data->cfg->type == sun8i_a33_phy ||
+	    phy_data->cfg->type == sun50i_a64_phy) {
+		/* A33 or A64 needs us to set phyctl to 0 explicitly */
 		writel(0, phyctl);
 	}
 
@@ -215,7 +223,8 @@ static void sun4i_usb_phy_write(struct sun4i_usb_phy *phy, u32 addr, u32 data,
 
 		data >>= 1;
 	}
-	mutex_unlock(&phy_data->mutex);
+
+	spin_unlock_irqrestore(&phy_data->reg_lock, flags);
 }
 
 static void sun4i_usb_phy_passby(struct sun4i_usb_phy *phy, int enable)
@@ -255,14 +264,16 @@ static int sun4i_usb_phy_init(struct phy *_phy)
 		return ret;
 	}
 
+	if (data->cfg->enable_pmu_unk1) {
+		val = readl(phy->pmu + REG_PMU_UNK1);
+		writel(val & ~2, phy->pmu + REG_PMU_UNK1);
+	}
+
 	if (data->cfg->type == sun8i_h3_phy) {
 		if (phy->index == 0) {
 			val = readl(data->base + REG_PHY_UNK_H3);
 			writel(val & ~1, data->base + REG_PHY_UNK_H3);
 		}
-
-		val = readl(phy->pmu + REG_PMU_UNK_H3);
-		writel(val & ~2, phy->pmu + REG_PMU_UNK_H3);
 	} else {
 		/* Enable USB 45 Ohm resistor calibration */
 		if (phy->index == 0)
@@ -285,16 +296,10 @@ static int sun4i_usb_phy_init(struct phy *_phy)
 		sun4i_usb_phy0_update_iscr(_phy, 0, ISCR_DPDM_PULLUP_EN);
 		sun4i_usb_phy0_update_iscr(_phy, 0, ISCR_ID_PULLUP_EN);
 
-		if (data->id_det_gpio) {
-			/* OTG mode, force ISCR and cable state updates */
-			data->id_det = -1;
-			data->vbus_det = -1;
-			queue_delayed_work(system_wq, &data->detect, 0);
-		} else {
-			/* Host only mode */
-			sun4i_usb_phy0_set_id_detect(_phy, 0);
-			sun4i_usb_phy0_set_vbus_detect(_phy, 1);
-		}
+		/* Force ISCR and cable state updates */
+		data->id_det = -1;
+		data->vbus_det = -1;
+		queue_delayed_work(system_wq, &data->detect, 0);
 	}
 
 	return 0;
@@ -319,6 +324,22 @@ static int sun4i_usb_phy_exit(struct phy *_phy)
 	return 0;
 }
 
+static int sun4i_usb_phy0_get_id_det(struct sun4i_usb_phy_data *data)
+{
+	switch (data->dr_mode) {
+	case USB_DR_MODE_OTG:
+		if (data->id_det_gpio)
+			return gpiod_get_value_cansleep(data->id_det_gpio);
+		else
+			return 1; /* Fallback to peripheral mode */
+	case USB_DR_MODE_HOST:
+		return 0;
+	case USB_DR_MODE_PERIPHERAL:
+	default:
+		return 1;
+	}
+}
+
 static int sun4i_usb_phy0_get_vbus_det(struct sun4i_usb_phy_data *data)
 {
 	if (data->vbus_det_gpio)
@@ -372,8 +393,10 @@ static int sun4i_usb_phy_power_on(struct phy *_phy)
 
 	/* For phy0 only turn on Vbus if we don't have an ext. Vbus */
 	if (phy->index == 0 && sun4i_usb_phy0_have_vbus_det(data) &&
-				data->vbus_det)
+				data->vbus_det) {
+		dev_warn(&_phy->dev, "External vbus detected, not enabling our own vbus\n");
 		return 0;
+	}
 
 	ret = regulator_enable(phy->vbus);
 	if (ret)
@@ -409,6 +432,35 @@ static int sun4i_usb_phy_power_off(struct phy *_phy)
 	return 0;
 }
 
+static int sun4i_usb_phy_set_mode(struct phy *_phy, enum phy_mode mode)
+{
+	struct sun4i_usb_phy *phy = phy_get_drvdata(_phy);
+	struct sun4i_usb_phy_data *data = to_sun4i_usb_phy_data(phy);
+
+	if (phy->index != 0)
+		return -EINVAL;
+
+	switch (mode) {
+	case PHY_MODE_USB_HOST:
+		data->dr_mode = USB_DR_MODE_HOST;
+		break;
+	case PHY_MODE_USB_DEVICE:
+		data->dr_mode = USB_DR_MODE_PERIPHERAL;
+		break;
+	case PHY_MODE_USB_OTG:
+		data->dr_mode = USB_DR_MODE_OTG;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	dev_info(&_phy->dev, "Changing dr_mode to %d\n", (int)data->dr_mode);
+	data->force_session_end = true;
+	queue_delayed_work(system_wq, &data->detect, 0);
+
+	return 0;
+}
+
 void sun4i_usb_phy_set_squelch_detect(struct phy *_phy, bool enabled)
 {
 	struct sun4i_usb_phy *phy = phy_get_drvdata(_phy);
@@ -422,6 +474,7 @@ static const struct phy_ops sun4i_usb_phy_ops = {
 	.exit		= sun4i_usb_phy_exit,
 	.power_on	= sun4i_usb_phy_power_on,
 	.power_off	= sun4i_usb_phy_power_off,
+	.set_mode	= sun4i_usb_phy_set_mode,
 	.owner		= THIS_MODULE,
 };
 
@@ -430,9 +483,13 @@ static void sun4i_usb_phy0_id_vbus_det_scan(struct work_struct *work)
 	struct sun4i_usb_phy_data *data =
 		container_of(work, struct sun4i_usb_phy_data, detect.work);
 	struct phy *phy0 = data->phys[0].phy;
-	int id_det, vbus_det, id_notify = 0, vbus_notify = 0;
+	bool force_session_end, id_notify = false, vbus_notify = false;
+	int id_det, vbus_det;
+
+	if (phy0 == NULL)
+		return;
 
-	id_det = gpiod_get_value_cansleep(data->id_det_gpio);
+	id_det = sun4i_usb_phy0_get_id_det(data);
 	vbus_det = sun4i_usb_phy0_get_vbus_det(data);
 
 	mutex_lock(&phy0->mutex);
@@ -442,26 +499,30 @@ static void sun4i_usb_phy0_id_vbus_det_scan(struct work_struct *work)
 		return;
 	}
 
+	force_session_end = data->force_session_end;
+	data->force_session_end = false;
+
 	if (id_det != data->id_det) {
-		/*
-		 * When a host cable (id == 0) gets plugged in on systems
-		 * without vbus detection report vbus low for long enough for
-		 * the musb-ip to end the current device session.
-		 */
-		if (!sun4i_usb_phy0_have_vbus_det(data) && id_det == 0) {
+		/* id-change, force session end if we've no vbus detection */
+		if (data->dr_mode == USB_DR_MODE_OTG &&
+		    !sun4i_usb_phy0_have_vbus_det(data))
+			force_session_end = true;
+
+		/* When entering host mode (id = 0) force end the session now */
+		if (force_session_end && id_det == 0) {
 			sun4i_usb_phy0_set_vbus_detect(phy0, 0);
 			msleep(200);
 			sun4i_usb_phy0_set_vbus_detect(phy0, 1);
 		}
 		sun4i_usb_phy0_set_id_detect(phy0, id_det);
 		data->id_det = id_det;
-		id_notify = 1;
+		id_notify = true;
 	}
 
 	if (vbus_det != data->vbus_det) {
 		sun4i_usb_phy0_set_vbus_detect(phy0, vbus_det);
 		data->vbus_det = vbus_det;
-		vbus_notify = 1;
+		vbus_notify = true;
 	}
 
 	mutex_unlock(&phy0->mutex);
@@ -469,12 +530,8 @@ static void sun4i_usb_phy0_id_vbus_det_scan(struct work_struct *work)
 	if (id_notify) {
 		extcon_set_cable_state_(data->extcon, EXTCON_USB_HOST,
 					!id_det);
-		/*
-		 * When a host cable gets unplugged (id == 1) on systems
-		 * without vbus detection report vbus low for long enough to
-		 * the musb-ip to end the current host session.
-		 */
-		if (!sun4i_usb_phy0_have_vbus_det(data) && id_det == 1) {
+		/* When leaving host mode force end the session here */
+		if (force_session_end && id_det == 1) {
 			mutex_lock(&phy0->mutex);
 			sun4i_usb_phy0_set_vbus_detect(phy0, 0);
 			msleep(1000);
@@ -561,7 +618,7 @@ static int sun4i_usb_phy_probe(struct platform_device *pdev)
 	if (!data)
 		return -ENOMEM;
 
-	mutex_init(&data->mutex);
+	spin_lock_init(&data->reg_lock);
 	INIT_DELAYED_WORK(&data->detect, sun4i_usb_phy0_id_vbus_det_scan);
 	dev_set_drvdata(dev, data);
 	data->cfg = of_device_get_match_data(dev);
@@ -593,23 +650,16 @@ static int sun4i_usb_phy_probe(struct platform_device *pdev)
 			return -EPROBE_DEFER;
 	}
 
-	/* vbus_det without id_det makes no sense, and is not supported */
-	if (sun4i_usb_phy0_have_vbus_det(data) && !data->id_det_gpio) {
-		dev_err(dev, "usb0_id_det missing or invalid\n");
-		return -ENODEV;
-	}
+	data->dr_mode = of_usb_get_dr_mode_by_phy(np, 0);
 
-	if (data->id_det_gpio) {
-		data->extcon = devm_extcon_dev_allocate(dev,
-							sun4i_usb_phy0_cable);
-		if (IS_ERR(data->extcon))
-			return PTR_ERR(data->extcon);
+	data->extcon = devm_extcon_dev_allocate(dev, sun4i_usb_phy0_cable);
+	if (IS_ERR(data->extcon))
+		return PTR_ERR(data->extcon);
 
-		ret = devm_extcon_dev_register(dev, data->extcon);
-		if (ret) {
-			dev_err(dev, "failed to register extcon: %d\n", ret);
-			return ret;
-		}
+	ret = devm_extcon_dev_register(dev, data->extcon);
+	if (ret) {
+		dev_err(dev, "failed to register extcon: %d\n", ret);
+		return ret;
 	}
 
 	for (i = 0; i < data->cfg->num_phys; i++) {
@@ -713,6 +763,7 @@ static const struct sun4i_usb_phy_cfg sun4i_a10_cfg = {
 	.disc_thresh = 3,
 	.phyctl_offset = REG_PHYCTL_A10,
 	.dedicated_clocks = false,
+	.enable_pmu_unk1 = false,
 };
 
 static const struct sun4i_usb_phy_cfg sun5i_a13_cfg = {
@@ -721,6 +772,7 @@ static const struct sun4i_usb_phy_cfg sun5i_a13_cfg = {
 	.disc_thresh = 2,
 	.phyctl_offset = REG_PHYCTL_A10,
 	.dedicated_clocks = false,
+	.enable_pmu_unk1 = false,
 };
 
 static const struct sun4i_usb_phy_cfg sun6i_a31_cfg = {
@@ -729,6 +781,7 @@ static const struct sun4i_usb_phy_cfg sun6i_a31_cfg = {
 	.disc_thresh = 3,
 	.phyctl_offset = REG_PHYCTL_A10,
 	.dedicated_clocks = true,
+	.enable_pmu_unk1 = false,
 };
 
 static const struct sun4i_usb_phy_cfg sun7i_a20_cfg = {
@@ -737,6 +790,7 @@ static const struct sun4i_usb_phy_cfg sun7i_a20_cfg = {
 	.disc_thresh = 2,
 	.phyctl_offset = REG_PHYCTL_A10,
 	.dedicated_clocks = false,
+	.enable_pmu_unk1 = false,
 };
 
 static const struct sun4i_usb_phy_cfg sun8i_a23_cfg = {
@@ -745,6 +799,7 @@ static const struct sun4i_usb_phy_cfg sun8i_a23_cfg = {
 	.disc_thresh = 3,
 	.phyctl_offset = REG_PHYCTL_A10,
 	.dedicated_clocks = true,
+	.enable_pmu_unk1 = false,
 };
 
 static const struct sun4i_usb_phy_cfg sun8i_a33_cfg = {
@@ -753,6 +808,7 @@ static const struct sun4i_usb_phy_cfg sun8i_a33_cfg = {
 	.disc_thresh = 3,
 	.phyctl_offset = REG_PHYCTL_A33,
 	.dedicated_clocks = true,
+	.enable_pmu_unk1 = false,
 };
 
 static const struct sun4i_usb_phy_cfg sun8i_h3_cfg = {
@@ -760,6 +816,16 @@ static const struct sun4i_usb_phy_cfg sun8i_h3_cfg = {
 	.type = sun8i_h3_phy,
 	.disc_thresh = 3,
 	.dedicated_clocks = true,
+	.enable_pmu_unk1 = true,
+};
+
+static const struct sun4i_usb_phy_cfg sun50i_a64_cfg = {
+	.num_phys = 2,
+	.type = sun50i_a64_phy,
+	.disc_thresh = 3,
+	.phyctl_offset = REG_PHYCTL_A33,
+	.dedicated_clocks = true,
+	.enable_pmu_unk1 = true,
 };
 
 static const struct of_device_id sun4i_usb_phy_of_match[] = {
@@ -770,6 +836,8 @@ static const struct of_device_id sun4i_usb_phy_of_match[] = {
 	{ .compatible = "allwinner,sun8i-a23-usb-phy", .data = &sun8i_a23_cfg },
 	{ .compatible = "allwinner,sun8i-a33-usb-phy", .data = &sun8i_a33_cfg },
 	{ .compatible = "allwinner,sun8i-h3-usb-phy", .data = &sun8i_h3_cfg },
+	{ .compatible = "allwinner,sun50i-a64-usb-phy",
+	  .data = &sun50i_a64_cfg},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, sun4i_usb_phy_of_match);
diff --git a/drivers/phy/phy-sun9i-usb.c b/drivers/phy/phy-sun9i-usb.c
index ac4f31abefe3..28fce4bce638 100644
--- a/drivers/phy/phy-sun9i-usb.c
+++ b/drivers/phy/phy-sun9i-usb.c
@@ -141,9 +141,9 @@ static int sun9i_usb_phy_probe(struct platform_device *pdev)
 		}
 
 		phy->hsic_clk = devm_clk_get(dev, "hsic_12M");
-		if (IS_ERR(phy->clk)) {
+		if (IS_ERR(phy->hsic_clk)) {
 			dev_err(dev, "failed to get hsic_12M clock\n");
-			return PTR_ERR(phy->clk);
+			return PTR_ERR(phy->hsic_clk);
 		}
 
 		phy->reset = devm_reset_control_get(dev, "hsic");
diff --git a/drivers/phy/phy-twl4030-usb.c b/drivers/phy/phy-twl4030-usb.c
index d9b10a39a2cf..87e6334eab93 100644
--- a/drivers/phy/phy-twl4030-usb.c
+++ b/drivers/phy/phy-twl4030-usb.c
@@ -172,6 +172,7 @@ struct twl4030_usb {
 	int			irq;
 	enum musb_vbus_id_status linkstat;
 	bool			vbus_supplied;
+	bool			musb_mailbox_pending;
 
 	struct delayed_work	id_workaround_work;
 };
@@ -439,6 +440,17 @@ static int __maybe_unused twl4030_usb_runtime_resume(struct device *dev)
 			  (PHY_CLK_CTRL_CLOCKGATING_EN |
 			   PHY_CLK_CTRL_CLK32K_EN));
 
+	twl4030_i2c_access(twl, 1);
+	twl4030_usb_set_mode(twl, twl->usb_mode);
+	if (twl->usb_mode == T2_USB_MODE_ULPI)
+		twl4030_i2c_access(twl, 0);
+	/*
+	 * According to the TPS65950 TRM, there has to be at least 50ms
+	 * delay between setting POWER_CTRL_OTG_ENAB and enabling charging
+	 * so wait here so that a fully enabled phy can be expected after
+	 * resume
+	 */
+	msleep(50);
 	return 0;
 }
 
@@ -459,11 +471,6 @@ static int twl4030_phy_power_on(struct phy *phy)
 
 	dev_dbg(twl->dev, "%s\n", __func__);
 	pm_runtime_get_sync(twl->dev);
-	twl4030_i2c_access(twl, 1);
-	twl4030_usb_set_mode(twl, twl->usb_mode);
-	if (twl->usb_mode == T2_USB_MODE_ULPI)
-		twl4030_i2c_access(twl, 0);
-	twl->linkstat = MUSB_UNKNOWN;
 	schedule_delayed_work(&twl->id_workaround_work, HZ);
 
 	return 0;
@@ -569,9 +576,12 @@ static irqreturn_t twl4030_usb_irq(int irq, void *_twl)
 			pm_runtime_mark_last_busy(twl->dev);
 			pm_runtime_put_autosuspend(twl->dev);
 		}
+		twl->musb_mailbox_pending = true;
+	}
+	if (twl->musb_mailbox_pending) {
 		err = musb_mailbox(status);
-		if (err)
-			twl->linkstat = MUSB_UNKNOWN;
+		if (!err)
+			twl->musb_mailbox_pending = false;
 	}
 
 	/* don't schedule during sleep - irq works right then */
@@ -676,6 +686,7 @@ static int twl4030_usb_probe(struct platform_device *pdev)
 	twl->irq		= platform_get_irq(pdev, 0);
 	twl->vbus_supplied	= false;
 	twl->linkstat		= MUSB_UNKNOWN;
+	twl->musb_mailbox_pending = false;
 
 	twl->phy.dev		= twl->dev;
 	twl->phy.label		= "twl4030";
diff --git a/drivers/phy/tegra/xusb.c b/drivers/phy/tegra/xusb.c
index ec83dfdbc206..873424ab0e32 100644
--- a/drivers/phy/tegra/xusb.c
+++ b/drivers/phy/tegra/xusb.c
@@ -18,6 +18,7 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/phy/phy.h>
+#include <linux/phy/tegra/xusb.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/consumer.h>
 #include <linux/reset.h>
@@ -101,7 +102,8 @@ tegra_xusb_pad_find_phy_node(struct tegra_xusb_pad *pad, unsigned int index)
 	return of_find_node_by_name(np, pad->soc->lanes[index].name);
 }
 
-int tegra_xusb_lane_lookup_function(struct tegra_xusb_lane *lane,
+static int
+tegra_xusb_lane_lookup_function(struct tegra_xusb_lane *lane,
 				    const char *function)
 {
 	unsigned int i;
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index b3fe1d339632..0e75d94972ba 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -254,6 +254,7 @@ config PINCTRL_ZYNQ
 	help
 	  This selects the pinctrl driver for Xilinx Zynq.
 
+source "drivers/pinctrl/aspeed/Kconfig"
 source "drivers/pinctrl/bcm/Kconfig"
 source "drivers/pinctrl/berlin/Kconfig"
 source "drivers/pinctrl/freescale/Kconfig"
diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile
index 8ebd7b8e1621..11bad373dfe0 100644
--- a/drivers/pinctrl/Makefile
+++ b/drivers/pinctrl/Makefile
@@ -37,6 +37,7 @@ obj-$(CONFIG_PINCTRL_TB10X)	+= pinctrl-tb10x.o
 obj-$(CONFIG_PINCTRL_ST) 	+= pinctrl-st.o
 obj-$(CONFIG_PINCTRL_ZYNQ)	+= pinctrl-zynq.o
 
+obj-$(CONFIG_ARCH_ASPEED)	+= aspeed/
 obj-y				+= bcm/
 obj-$(CONFIG_PINCTRL_BERLIN)	+= berlin/
 obj-y				+= freescale/
diff --git a/drivers/pinctrl/aspeed/Kconfig b/drivers/pinctrl/aspeed/Kconfig
new file mode 100644
index 000000000000..998eabef3a65
--- /dev/null
+++ b/drivers/pinctrl/aspeed/Kconfig
@@ -0,0 +1,24 @@
+config PINCTRL_ASPEED
+	bool
+	depends on (ARCH_ASPEED || COMPILE_TEST) && OF
+	depends on MFD_SYSCON
+	select PINMUX
+	select PINCONF
+	select GENERIC_PINCONF
+	select REGMAP_MMIO
+
+config PINCTRL_ASPEED_G4
+	bool "Aspeed G4 SoC pin control"
+	depends on (MACH_ASPEED_G4 || COMPILE_TEST) && OF
+	select PINCTRL_ASPEED
+	help
+	  Say Y here to enable pin controller support for Aspeed's 4th
+	  generation SoCs. GPIO is provided by a separate GPIO driver.
+
+config PINCTRL_ASPEED_G5
+	bool "Aspeed G5 SoC pin control"
+	depends on (MACH_ASPEED_G5 || COMPILE_TEST) && OF
+	select PINCTRL_ASPEED
+	help
+	  Say Y here to enable pin controller support for Aspeed's 5th
+	  generation SoCs. GPIO is provided by a separate GPIO driver.
diff --git a/drivers/pinctrl/aspeed/Makefile b/drivers/pinctrl/aspeed/Makefile
new file mode 100644
index 000000000000..191ed0fc1804
--- /dev/null
+++ b/drivers/pinctrl/aspeed/Makefile
@@ -0,0 +1,6 @@
+# Aspeed pinctrl support
+
+ccflags-y += -Woverride-init
+obj-$(CONFIG_PINCTRL_ASPEED)	+= pinctrl-aspeed.o
+obj-$(CONFIG_PINCTRL_ASPEED_G4)	+= pinctrl-aspeed-g4.o
+obj-$(CONFIG_PINCTRL_ASPEED_G5)	+= pinctrl-aspeed-g5.o
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed-g4.c b/drivers/pinctrl/aspeed/pinctrl-aspeed-g4.c
new file mode 100644
index 000000000000..a21b071ff290
--- /dev/null
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed-g4.c
@@ -0,0 +1,1231 @@
+/*
+ * Copyright (C) 2016 IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#include <linux/bitops.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+#include "../core.h"
+#include "../pinctrl-utils.h"
+#include "pinctrl-aspeed.h"
+
+/*
+ * Uses undefined macros for symbol naming and references, eg GPIOA0, MAC1LINK,
+ * TIMER3 etc.
+ *
+ * Pins are defined in GPIO bank order:
+ *
+ * GPIOA0: 0
+ * ...
+ * GPIOA7: 7
+ * GPIOB0: 8
+ * ...
+ * GPIOZ7: 207
+ * GPIOAA0: 208
+ * ...
+ * GPIOAB3: 219
+ *
+ * Not all pins have their signals defined (yet).
+ */
+
+#define A4 2
+SSSF_PIN_DECL(A4, GPIOA2, TIMER3, SIG_DESC_SET(SCU80, 2));
+
+#define I2C9_DESC	SIG_DESC_SET(SCU90, 22)
+
+#define C5 4
+SIG_EXPR_LIST_DECL_SINGLE(SCL9, I2C9, I2C9_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(TIMER5, TIMER5, SIG_DESC_SET(SCU80, 4));
+MS_PIN_DECL(C5, GPIOA4, SCL9, TIMER5);
+
+FUNC_GROUP_DECL(TIMER5, C5);
+
+#define B4 5
+SIG_EXPR_LIST_DECL_SINGLE(SDA9, I2C9, I2C9_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(TIMER6, TIMER6, SIG_DESC_SET(SCU80, 5));
+MS_PIN_DECL(B4, GPIOA5, SDA9, TIMER6);
+
+FUNC_GROUP_DECL(TIMER6, B4);
+FUNC_GROUP_DECL(I2C9, C5, B4);
+
+#define MDIO2_DESC	SIG_DESC_SET(SCU90, 2)
+
+#define A3 6
+SIG_EXPR_LIST_DECL_SINGLE(MDC2, MDIO2, MDIO2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(TIMER7, TIMER7, SIG_DESC_SET(SCU80, 6));
+MS_PIN_DECL(A3, GPIOA6, MDC2, TIMER7);
+
+FUNC_GROUP_DECL(TIMER7, A3);
+
+#define D5 7
+SIG_EXPR_LIST_DECL_SINGLE(MDIO2, MDIO2, MDIO2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(TIMER8, TIMER8, SIG_DESC_SET(SCU80, 7));
+MS_PIN_DECL(D5, GPIOA7, MDIO2, TIMER8);
+
+FUNC_GROUP_DECL(TIMER8, D5);
+FUNC_GROUP_DECL(MDIO2, A3, D5);
+
+#define H19 13
+#define H19_DESC        SIG_DESC_SET(SCU80, 13)
+SIG_EXPR_LIST_DECL_SINGLE(LPCPD, LPCPD, H19_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(LPCSMI, LPCSMI, H19_DESC);
+MS_PIN_DECL(H19, GPIOB5, LPCPD, LPCSMI);
+
+FUNC_GROUP_DECL(LPCPD, H19);
+FUNC_GROUP_DECL(LPCSMI, H19);
+
+#define H20 14
+SSSF_PIN_DECL(H20, GPIOB6, LPCPME, SIG_DESC_SET(SCU80, 14));
+
+#define SD1_DESC	SIG_DESC_SET(SCU90, 0)
+#define I2C10_DESC	SIG_DESC_SET(SCU90, 23)
+
+#define C4 16
+SIG_EXPR_LIST_DECL_SINGLE(SD1CLK, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SCL10, I2C10, I2C10_DESC);
+MS_PIN_DECL(C4, GPIOC0, SD1CLK, SCL10);
+
+#define B3 17
+SIG_EXPR_LIST_DECL_SINGLE(SD1CMD, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SDA10, I2C10, I2C10_DESC);
+MS_PIN_DECL(B3, GPIOC1, SD1CMD, SDA10);
+
+FUNC_GROUP_DECL(I2C10, C4, B3);
+
+#define I2C11_DESC	SIG_DESC_SET(SCU90, 24)
+
+#define A2 18
+SIG_EXPR_LIST_DECL_SINGLE(SD1DAT0, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SCL11, I2C11, I2C11_DESC);
+MS_PIN_DECL(A2, GPIOC2, SD1DAT0, SCL11);
+
+#define E5 19
+SIG_EXPR_LIST_DECL_SINGLE(SD1DAT1, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SDA11, I2C11, I2C11_DESC);
+MS_PIN_DECL(E5, GPIOC3, SD1DAT1, SDA11);
+
+FUNC_GROUP_DECL(I2C11, A2, E5);
+
+#define I2C12_DESC	SIG_DESC_SET(SCU90, 25)
+
+#define D4 20
+SIG_EXPR_LIST_DECL_SINGLE(SD1DAT2, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SCL12, I2C12, I2C12_DESC);
+MS_PIN_DECL(D4, GPIOC4, SD1DAT2, SCL12);
+
+#define C3 21
+SIG_EXPR_LIST_DECL_SINGLE(SD1DAT3, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SDA12, I2C12, I2C12_DESC);
+MS_PIN_DECL(C3, GPIOC5, SD1DAT3, SDA12);
+
+FUNC_GROUP_DECL(I2C12, D4, C3);
+
+#define I2C13_DESC	SIG_DESC_SET(SCU90, 26)
+
+#define B2 22
+SIG_EXPR_LIST_DECL_SINGLE(SD1CD, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SCL13, I2C13, I2C13_DESC);
+MS_PIN_DECL(B2, GPIOC6, SD1CD, SCL13);
+
+#define A1 23
+SIG_EXPR_LIST_DECL_SINGLE(SD1WP, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SDA13, I2C13, I2C13_DESC);
+MS_PIN_DECL(A1, GPIOC7, SD1WP, SDA13);
+
+FUNC_GROUP_DECL(I2C13, B2, A1);
+FUNC_GROUP_DECL(SD1, C4, B3, A2, E5, D4, C3, B2, A1);
+
+#define SD2_DESC	SIG_DESC_SET(SCU90, 1)
+#define GPID_DESC       SIG_DESC_SET(HW_STRAP1, 21)
+#define GPID0_DESC	SIG_DESC_SET(SCU8C, 8)
+
+#define A18 24
+SIG_EXPR_LIST_DECL_SINGLE(SD2CLK, SD2, SD2_DESC);
+SIG_EXPR_DECL(GPID0IN, GPID0, GPID0_DESC);
+SIG_EXPR_DECL(GPID0IN, GPID, GPID_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPID0IN, GPID0, GPID);
+MS_PIN_DECL(A18, GPIOD0, SD2CLK, GPID0IN);
+
+#define D16 25
+SIG_EXPR_LIST_DECL_SINGLE(SD2CMD, SD2, SD2_DESC);
+SIG_EXPR_DECL(GPID0OUT, GPID0, GPID0_DESC);
+SIG_EXPR_DECL(GPID0OUT, GPID, GPID_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPID0OUT, GPID0, GPID);
+MS_PIN_DECL(D16, GPIOD1, SD2CMD, GPID0OUT);
+
+FUNC_GROUP_DECL(GPID0, A18, D16);
+
+#define GPIE_DESC       SIG_DESC_SET(HW_STRAP1, 22)
+#define GPIE0_DESC      SIG_DESC_SET(SCU8C, 12)
+#define GPIE2_DESC      SIG_DESC_SET(SCU8C, 13)
+#define GPIE4_DESC      SIG_DESC_SET(SCU8C, 14)
+#define GPIE6_DESC      SIG_DESC_SET(SCU8C, 15)
+
+#define D15 32
+SIG_EXPR_LIST_DECL_SINGLE(NCTS3, NCTS3, SIG_DESC_SET(SCU80, 16));
+SIG_EXPR_DECL(GPIE0IN, GPIE0, GPIE0_DESC);
+SIG_EXPR_DECL(GPIE0IN, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE0IN, GPIE0, GPIE);
+MS_PIN_DECL(D15, GPIOE0, NCTS3, GPIE0IN);
+
+FUNC_GROUP_DECL(NCTS3, D15);
+
+#define C15 33
+SIG_EXPR_LIST_DECL_SINGLE(NDCD3, NDCD3, SIG_DESC_SET(SCU80, 17));
+SIG_EXPR_DECL(GPIE0OUT, GPIE0, GPIE0_DESC);
+SIG_EXPR_DECL(GPIE0OUT, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE0OUT, GPIE0, GPIE);
+MS_PIN_DECL(C15, GPIOE1, NDCD3, GPIE0OUT);
+
+FUNC_GROUP_DECL(NDCD3, C15);
+FUNC_GROUP_DECL(GPIE0, D15, C15);
+
+#define B15 34
+SIG_EXPR_LIST_DECL_SINGLE(NDSR3, NDSR3, SIG_DESC_SET(SCU80, 18));
+SIG_EXPR_DECL(GPIE2IN, GPIE2, GPIE2_DESC);
+SIG_EXPR_DECL(GPIE2IN, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE2IN, GPIE2, GPIE);
+MS_PIN_DECL(B15, GPIOE2, NDSR3, GPIE2IN);
+
+FUNC_GROUP_DECL(NDSR3, B15);
+
+#define A15 35
+SIG_EXPR_LIST_DECL_SINGLE(NRI3, NRI3, SIG_DESC_SET(SCU80, 19));
+SIG_EXPR_DECL(GPIE2OUT, GPIE2, GPIE2_DESC);
+SIG_EXPR_DECL(GPIE2OUT, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE2OUT, GPIE2, GPIE);
+MS_PIN_DECL(A15, GPIOE3, NRI3, GPIE2OUT);
+
+FUNC_GROUP_DECL(NRI3, A15);
+FUNC_GROUP_DECL(GPIE2, B15, A15);
+
+#define E14 36
+SIG_EXPR_LIST_DECL_SINGLE(NDTR3, NDTR3, SIG_DESC_SET(SCU80, 20));
+SIG_EXPR_DECL(GPIE4IN, GPIE4, GPIE4_DESC);
+SIG_EXPR_DECL(GPIE4IN, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE4IN, GPIE4, GPIE);
+MS_PIN_DECL(E14, GPIOE4, NDTR3, GPIE4IN);
+
+FUNC_GROUP_DECL(NDTR3, E14);
+
+#define D14 37
+SIG_EXPR_LIST_DECL_SINGLE(NRTS3, NRTS3, SIG_DESC_SET(SCU80, 21));
+SIG_EXPR_DECL(GPIE4OUT, GPIE4, GPIE4_DESC);
+SIG_EXPR_DECL(GPIE4OUT, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE4OUT, GPIE4, GPIE);
+MS_PIN_DECL(D14, GPIOE5, NRTS3, GPIE4OUT);
+
+FUNC_GROUP_DECL(NRTS3, D14);
+FUNC_GROUP_DECL(GPIE4, E14, D14);
+
+#define C14 38
+SIG_EXPR_LIST_DECL_SINGLE(TXD3, TXD3, SIG_DESC_SET(SCU80, 22));
+SIG_EXPR_DECL(GPIE6IN, GPIE6, GPIE6_DESC);
+SIG_EXPR_DECL(GPIE6IN, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE6IN, GPIE6, GPIE);
+MS_PIN_DECL(C14, GPIOE6, TXD3, GPIE6IN);
+
+FUNC_GROUP_DECL(TXD3, C14);
+
+#define B14 39
+SIG_EXPR_LIST_DECL_SINGLE(RXD3, RXD3, SIG_DESC_SET(SCU80, 23));
+SIG_EXPR_DECL(GPIE6OUT, GPIE6, GPIE6_DESC);
+SIG_EXPR_DECL(GPIE6OUT, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE6OUT, GPIE6, GPIE);
+MS_PIN_DECL(B14, GPIOE7, RXD3, GPIE6OUT);
+
+FUNC_GROUP_DECL(RXD3, B14);
+FUNC_GROUP_DECL(GPIE6, C14, B14);
+
+#define D18 40
+SSSF_PIN_DECL(D18, GPIOF0, NCTS4, SIG_DESC_SET(SCU80, 24));
+
+#define ACPI_DESC       SIG_DESC_BIT(HW_STRAP1, 19, 0)
+
+#define B19 41
+SIG_EXPR_LIST_DECL_SINGLE(NDCD4, NDCD4, SIG_DESC_SET(SCU80, 25));
+SIG_EXPR_DECL(SIOPBI, SIOPBI, SIG_DESC_SET(SCUA4, 12));
+SIG_EXPR_DECL(SIOPBI, ACPI, ACPI_DESC);
+SIG_EXPR_LIST_DECL_DUAL(SIOPBI, SIOPBI, ACPI);
+MS_PIN_DECL(B19, GPIOF1, NDCD4, SIOPBI);
+FUNC_GROUP_DECL(NDCD4, B19);
+FUNC_GROUP_DECL(SIOPBI, B19);
+
+#define D17 43
+SIG_EXPR_LIST_DECL_SINGLE(NRI4, NRI4, SIG_DESC_SET(SCU80, 27));
+SIG_EXPR_DECL(SIOPBO, SIOPBO, SIG_DESC_SET(SCUA4, 14));
+SIG_EXPR_DECL(SIOPBO, ACPI, ACPI_DESC);
+SIG_EXPR_LIST_DECL_DUAL(SIOPBO, SIOPBO, ACPI);
+MS_PIN_DECL(D17, GPIOF3, NRI4, SIOPBO);
+FUNC_GROUP_DECL(NRI4, D17);
+FUNC_GROUP_DECL(SIOPBO, D17);
+
+FUNC_GROUP_DECL(ACPI, B19, D17);
+
+#define E16 46
+SSSF_PIN_DECL(E16, GPIOF6, TXD4, SIG_DESC_SET(SCU80, 30));
+
+#define C17 47
+SSSF_PIN_DECL(C17, GPIOF7, RXD4, SIG_DESC_SET(SCU80, 31));
+
+#define AA22 54
+SSSF_PIN_DECL(AA22, GPIOG6, FLBUSY, SIG_DESC_SET(SCU84, 6));
+
+#define U18 55
+SSSF_PIN_DECL(U18, GPIOG7, FLWP, SIG_DESC_SET(SCU84, 7));
+
+#define UART6_DESC	SIG_DESC_SET(SCU90, 7)
+#define ROM16_DESC	SIG_DESC_SET(SCU90, 6)
+#define FLASH_WIDE	SIG_DESC_SET(HW_STRAP1, 4)
+#define BOOT_SRC_NOR	{ HW_STRAP1, GENMASK(1, 0), 0, 0 }
+
+#define A8 56
+SIG_EXPR_DECL(ROMD8, ROM16, ROM16_DESC);
+SIG_EXPR_DECL(ROMD8, ROM16S, FLASH_WIDE, BOOT_SRC_NOR);
+SIG_EXPR_LIST_DECL_DUAL(ROMD8, ROM16, ROM16S);
+SIG_EXPR_LIST_DECL_SINGLE(NCTS6, NCTS6, UART6_DESC);
+MS_PIN_DECL(A8, GPIOH0, ROMD8, NCTS6);
+
+#define C7 57
+SIG_EXPR_DECL(ROMD9, ROM16, ROM16_DESC);
+SIG_EXPR_DECL(ROMD9, ROM16S, FLASH_WIDE, BOOT_SRC_NOR);
+SIG_EXPR_LIST_DECL_DUAL(ROMD9, ROM16, ROM16S);
+SIG_EXPR_LIST_DECL_SINGLE(NDCD6, NDCD6, UART6_DESC);
+MS_PIN_DECL(C7, GPIOH1, ROMD9, NDCD6);
+
+#define B7 58
+SIG_EXPR_DECL(ROMD10, ROM16, ROM16_DESC);
+SIG_EXPR_DECL(ROMD10, ROM16S, FLASH_WIDE, BOOT_SRC_NOR);
+SIG_EXPR_LIST_DECL_DUAL(ROMD10, ROM16, ROM16S);
+SIG_EXPR_LIST_DECL_SINGLE(NDSR6, NDSR6, UART6_DESC);
+MS_PIN_DECL(B7, GPIOH2, ROMD10, NDSR6);
+
+#define A7 59
+SIG_EXPR_DECL(ROMD11, ROM16, ROM16_DESC);
+SIG_EXPR_DECL(ROMD11, ROM16S, FLASH_WIDE, BOOT_SRC_NOR);
+SIG_EXPR_LIST_DECL_DUAL(ROMD11, ROM16, ROM16S);
+SIG_EXPR_LIST_DECL_SINGLE(NRI6, NRI6, UART6_DESC);
+MS_PIN_DECL(A7, GPIOH3, ROMD11, NRI6);
+
+#define D7 60
+SIG_EXPR_DECL(ROMD12, ROM16, ROM16_DESC);
+SIG_EXPR_DECL(ROMD12, ROM16S, FLASH_WIDE, BOOT_SRC_NOR);
+SIG_EXPR_LIST_DECL_DUAL(ROMD12, ROM16, ROM16S);
+SIG_EXPR_LIST_DECL_SINGLE(NDTR6, NDTR6, UART6_DESC);
+MS_PIN_DECL(D7, GPIOH4, ROMD12, NDTR6);
+
+#define B6 61
+SIG_EXPR_DECL(ROMD13, ROM16, ROM16_DESC);
+SIG_EXPR_DECL(ROMD13, ROM16S, FLASH_WIDE, BOOT_SRC_NOR);
+SIG_EXPR_LIST_DECL_DUAL(ROMD13, ROM16, ROM16S);
+SIG_EXPR_LIST_DECL_SINGLE(NRTS6, NRTS6, UART6_DESC);
+MS_PIN_DECL(B6, GPIOH5, ROMD13, NRTS6);
+
+#define A6 62
+SIG_EXPR_DECL(ROMD14, ROM16, ROM16_DESC);
+SIG_EXPR_DECL(ROMD14, ROM16S, FLASH_WIDE, BOOT_SRC_NOR);
+SIG_EXPR_LIST_DECL_DUAL(ROMD14, ROM16, ROM16S);
+SIG_EXPR_LIST_DECL_SINGLE(TXD6, TXD6, UART6_DESC);
+MS_PIN_DECL(A6, GPIOH6, ROMD14, TXD6);
+
+#define E7 63
+SIG_EXPR_DECL(ROMD15, ROM16, ROM16_DESC);
+SIG_EXPR_DECL(ROMD15, ROM16S, FLASH_WIDE, BOOT_SRC_NOR);
+SIG_EXPR_LIST_DECL_DUAL(ROMD15, ROM16, ROM16S);
+SIG_EXPR_LIST_DECL_SINGLE(RXD6, RXD6, UART6_DESC);
+MS_PIN_DECL(E7, GPIOH7, ROMD15, RXD6);
+
+FUNC_GROUP_DECL(UART6, A8, C7, B7, A7, D7, B6, A6, E7);
+
+#define J3 75
+SSSF_PIN_DECL(J3, GPIOJ3, SGPMI, SIG_DESC_SET(SCU84, 11));
+
+#define T4 76
+SSSF_PIN_DECL(T4, GPIOJ4, VGAHS, SIG_DESC_SET(SCU84, 12));
+
+#define U2 77
+SSSF_PIN_DECL(U2, GPIOJ5, VGAVS, SIG_DESC_SET(SCU84, 13));
+
+#define T2 78
+SSSF_PIN_DECL(T2, GPIOJ6, DDCCLK, SIG_DESC_SET(SCU84, 14));
+
+#define T1 79
+SSSF_PIN_DECL(T1, GPIOJ7, DDCDAT, SIG_DESC_SET(SCU84, 15));
+
+#define I2C5_DESC	SIG_DESC_SET(SCU90, 18)
+
+#define E3 80
+SIG_EXPR_LIST_DECL_SINGLE(SCL5, I2C5, I2C5_DESC);
+SS_PIN_DECL(E3, GPIOK0, SCL5);
+
+#define D2 81
+SIG_EXPR_LIST_DECL_SINGLE(SDA5, I2C5, I2C5_DESC);
+SS_PIN_DECL(D2, GPIOK1, SDA5);
+
+FUNC_GROUP_DECL(I2C5, E3, D2);
+
+#define I2C6_DESC	SIG_DESC_SET(SCU90, 19)
+
+#define C1 82
+SIG_EXPR_LIST_DECL_SINGLE(SCL6, I2C6, I2C6_DESC);
+SS_PIN_DECL(C1, GPIOK2, SCL6);
+
+#define F4 83
+SIG_EXPR_LIST_DECL_SINGLE(SDA6, I2C6, I2C6_DESC);
+SS_PIN_DECL(F4, GPIOK3, SDA6);
+
+FUNC_GROUP_DECL(I2C6, C1, F4);
+
+#define I2C7_DESC	SIG_DESC_SET(SCU90, 20)
+
+#define E2 84
+SIG_EXPR_LIST_DECL_SINGLE(SCL7, I2C7, I2C7_DESC);
+SS_PIN_DECL(E2, GPIOK4, SCL7);
+
+#define D1 85
+SIG_EXPR_LIST_DECL_SINGLE(SDA7, I2C7, I2C7_DESC);
+SS_PIN_DECL(D1, GPIOK5, SDA7);
+
+FUNC_GROUP_DECL(I2C7, E2, D1);
+
+#define I2C8_DESC	SIG_DESC_SET(SCU90, 21)
+
+#define G5 86
+SIG_EXPR_LIST_DECL_SINGLE(SCL8, I2C8, I2C8_DESC);
+SS_PIN_DECL(G5, GPIOK6, SCL8);
+
+#define F3 87
+SIG_EXPR_LIST_DECL_SINGLE(SDA8, I2C8, I2C8_DESC);
+SS_PIN_DECL(F3, GPIOK7, SDA8);
+
+FUNC_GROUP_DECL(I2C8, G5, F3);
+
+#define U1 88
+SSSF_PIN_DECL(U1, GPIOL0, NCTS1, SIG_DESC_SET(SCU84, 16));
+
+#define VPI18_DESC	{ SCU90, GENMASK(5, 4), 1, 0 }
+#define VPI24_DESC	{ SCU90, GENMASK(5, 4), 2, 0 }
+#define VPI30_DESC	{ SCU90, GENMASK(5, 4), 3, 0 }
+
+#define T5 89
+#define T5_DESC         SIG_DESC_SET(SCU84, 17)
+SIG_EXPR_DECL(VPIDE, VPI18, VPI18_DESC, T5_DESC);
+SIG_EXPR_DECL(VPIDE, VPI24, VPI24_DESC, T5_DESC);
+SIG_EXPR_DECL(VPIDE, VPI30, VPI30_DESC, T5_DESC);
+SIG_EXPR_LIST_DECL(VPIDE, SIG_EXPR_PTR(VPIDE, VPI18),
+		SIG_EXPR_PTR(VPIDE, VPI24),
+		SIG_EXPR_PTR(VPIDE, VPI30));
+SIG_EXPR_LIST_DECL_SINGLE(NDCD1, NDCD1, T5_DESC);
+MS_PIN_DECL(T5, GPIOL1, VPIDE, NDCD1);
+FUNC_GROUP_DECL(NDCD1, T5);
+
+#define U3 90
+#define U3_DESC         SIG_DESC_SET(SCU84, 18)
+SIG_EXPR_DECL(VPIODD, VPI18, VPI18_DESC, U3_DESC);
+SIG_EXPR_DECL(VPIODD, VPI24, VPI24_DESC, U3_DESC);
+SIG_EXPR_DECL(VPIODD, VPI30, VPI30_DESC, U3_DESC);
+SIG_EXPR_LIST_DECL(VPIODD, SIG_EXPR_PTR(VPIODD, VPI18),
+		SIG_EXPR_PTR(VPIODD, VPI24),
+		SIG_EXPR_PTR(VPIODD, VPI30));
+SIG_EXPR_LIST_DECL_SINGLE(NDSR1, NDSR1, U3_DESC);
+MS_PIN_DECL(U3, GPIOL2, VPIODD, NDSR1);
+FUNC_GROUP_DECL(NDSR1, U3);
+
+#define V1 91
+#define V1_DESC         SIG_DESC_SET(SCU84, 19)
+SIG_EXPR_DECL(VPIHS, VPI18, VPI18_DESC, V1_DESC);
+SIG_EXPR_DECL(VPIHS, VPI24, VPI24_DESC, V1_DESC);
+SIG_EXPR_DECL(VPIHS, VPI30, VPI30_DESC, V1_DESC);
+SIG_EXPR_LIST_DECL(VPIHS, SIG_EXPR_PTR(VPIHS, VPI18),
+		SIG_EXPR_PTR(VPIHS, VPI24),
+		SIG_EXPR_PTR(VPIHS, VPI30));
+SIG_EXPR_LIST_DECL_SINGLE(NRI1, NRI1, V1_DESC);
+MS_PIN_DECL(V1, GPIOL3, VPIHS, NRI1);
+FUNC_GROUP_DECL(NRI1, V1);
+
+#define U4 92
+#define U4_DESC         SIG_DESC_SET(SCU84, 20)
+SIG_EXPR_DECL(VPIVS, VPI18, VPI18_DESC, U4_DESC);
+SIG_EXPR_DECL(VPIVS, VPI24, VPI24_DESC, U4_DESC);
+SIG_EXPR_DECL(VPIVS, VPI30, VPI30_DESC, U4_DESC);
+SIG_EXPR_LIST_DECL(VPIVS, SIG_EXPR_PTR(VPIVS, VPI18),
+		SIG_EXPR_PTR(VPIVS, VPI24),
+		SIG_EXPR_PTR(VPIVS, VPI30));
+SIG_EXPR_LIST_DECL_SINGLE(NDTR1, NDTR1, U4_DESC);
+MS_PIN_DECL(U4, GPIOL4, VPIVS, NDTR1);
+FUNC_GROUP_DECL(NDTR1, U4);
+
+#define V2 93
+#define V2_DESC         SIG_DESC_SET(SCU84, 21)
+SIG_EXPR_DECL(VPICLK, VPI18, VPI18_DESC, V2_DESC);
+SIG_EXPR_DECL(VPICLK, VPI24, VPI24_DESC, V2_DESC);
+SIG_EXPR_DECL(VPICLK, VPI30, VPI30_DESC, V2_DESC);
+SIG_EXPR_LIST_DECL(VPICLK, SIG_EXPR_PTR(VPICLK, VPI18),
+		SIG_EXPR_PTR(VPICLK, VPI24),
+		SIG_EXPR_PTR(VPICLK, VPI30));
+SIG_EXPR_LIST_DECL_SINGLE(NRTS1, NRTS1, V2_DESC);
+MS_PIN_DECL(V2, GPIOL5, VPICLK, NRTS1);
+FUNC_GROUP_DECL(NRTS1, V2);
+
+#define W1 94
+#define W1_DESC         SIG_DESC_SET(SCU84, 22)
+SIG_EXPR_LIST_DECL_SINGLE(VPIB0, VPI30, VPI30_DESC, W1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(TXD1, TXD1, W1_DESC);
+MS_PIN_DECL(W1, GPIOL6, VPIB0, TXD1);
+FUNC_GROUP_DECL(TXD1, W1);
+
+#define U5 95
+#define U5_DESC         SIG_DESC_SET(SCU84, 23)
+SIG_EXPR_LIST_DECL_SINGLE(VPIB1, VPI30, VPI30_DESC, U5_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RXD1, RXD1, U5_DESC);
+MS_PIN_DECL(U5, GPIOL7, VPIB1, RXD1);
+FUNC_GROUP_DECL(RXD1, U5);
+
+#define W4 104
+#define W4_DESC         SIG_DESC_SET(SCU88, 0)
+SIG_EXPR_LIST_DECL_SINGLE(VPIG0, VPI30, VPI30_DESC, W4_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(PWM0, PWM0, W4_DESC);
+MS_PIN_DECL(W4, GPION0, VPIG0, PWM0);
+FUNC_GROUP_DECL(PWM0, W4);
+
+#define Y3 105
+#define Y3_DESC         SIG_DESC_SET(SCU88, 1)
+SIG_EXPR_LIST_DECL_SINGLE(VPIG1, VPI30, VPI30_DESC, Y3_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(PWM1, PWM1, Y3_DESC);
+MS_PIN_DECL(Y3, GPION1, VPIG1, PWM1);
+FUNC_GROUP_DECL(PWM1, Y3);
+
+#define AA2 106
+#define AA2_DESC        SIG_DESC_SET(SCU88, 2)
+SIG_EXPR_DECL(VPIG2, VPI18, VPI18_DESC, AA2_DESC);
+SIG_EXPR_DECL(VPIG2, VPI24, VPI24_DESC, AA2_DESC);
+SIG_EXPR_DECL(VPIG2, VPI30, VPI30_DESC, AA2_DESC);
+SIG_EXPR_LIST_DECL(VPIG2, SIG_EXPR_PTR(VPIG2, VPI18),
+		SIG_EXPR_PTR(VPIG2, VPI24),
+		SIG_EXPR_PTR(VPIG2, VPI30));
+SIG_EXPR_LIST_DECL_SINGLE(PWM2, PWM2, AA2_DESC);
+MS_PIN_DECL(AA2, GPION2, VPIG2, PWM2);
+FUNC_GROUP_DECL(PWM2, AA2);
+
+#define AB1 107
+#define AB1_DESC        SIG_DESC_SET(SCU88, 3)
+SIG_EXPR_DECL(VPIG3, VPI18, VPI18_DESC, AB1_DESC);
+SIG_EXPR_DECL(VPIG3, VPI24, VPI24_DESC, AB1_DESC);
+SIG_EXPR_DECL(VPIG3, VPI30, VPI30_DESC, AB1_DESC);
+SIG_EXPR_LIST_DECL(VPIG3, SIG_EXPR_PTR(VPIG3, VPI18),
+		SIG_EXPR_PTR(VPIG2, VPI24),
+		SIG_EXPR_PTR(VPIG2, VPI30));
+SIG_EXPR_LIST_DECL_SINGLE(PWM3, PWM3, AB1_DESC);
+MS_PIN_DECL(AB1, GPION3, VPIG3, PWM3);
+FUNC_GROUP_DECL(PWM3, AB1);
+
+#define W5 108
+#define W5_DESC         SIG_DESC_SET(SCU88, 4)
+SIG_EXPR_DECL(VPIG4, VPI18, VPI18_DESC, W5_DESC);
+SIG_EXPR_DECL(VPIG4, VPI24, VPI24_DESC, W5_DESC);
+SIG_EXPR_DECL(VPIG4, VPI30, VPI30_DESC, W5_DESC);
+SIG_EXPR_LIST_DECL(VPIG4, SIG_EXPR_PTR(VPIG4, VPI18),
+		SIG_EXPR_PTR(VPIG2, VPI24),
+		SIG_EXPR_PTR(VPIG2, VPI30));
+SIG_EXPR_LIST_DECL_SINGLE(PWM4, PWM4, W5_DESC);
+MS_PIN_DECL(W5, GPION4, VPIG4, PWM4);
+FUNC_GROUP_DECL(PWM4, W5);
+
+#define Y4 109
+#define Y4_DESC         SIG_DESC_SET(SCU88, 5)
+SIG_EXPR_DECL(VPIG5, VPI18, VPI18_DESC, Y4_DESC);
+SIG_EXPR_DECL(VPIG5, VPI24, VPI24_DESC, Y4_DESC);
+SIG_EXPR_DECL(VPIG5, VPI30, VPI30_DESC, Y4_DESC);
+SIG_EXPR_LIST_DECL(VPIG5, SIG_EXPR_PTR(VPIG5, VPI18),
+		SIG_EXPR_PTR(VPIG2, VPI24),
+		SIG_EXPR_PTR(VPIG2, VPI30));
+SIG_EXPR_LIST_DECL_SINGLE(PWM5, PWM5, Y4_DESC);
+MS_PIN_DECL(Y4, GPION5, VPIG5, PWM5);
+FUNC_GROUP_DECL(PWM5, Y4);
+
+#define AA3 110
+#define AA3_DESC        SIG_DESC_SET(SCU88, 6)
+SIG_EXPR_LIST_DECL_SINGLE(VPIG6, VPI30, VPI30_DESC, AA3_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(PWM6, PWM6, AA3_DESC);
+MS_PIN_DECL(AA3, GPION6, VPIG6, PWM6);
+FUNC_GROUP_DECL(PWM6, AA3);
+
+#define AB2 111
+#define AB2_DESC        SIG_DESC_SET(SCU88, 7)
+SIG_EXPR_LIST_DECL_SINGLE(VPIG7, VPI30, VPI30_DESC, AB2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(PWM7, PWM7, AB2_DESC);
+MS_PIN_DECL(AB2, GPION7, VPIG7, PWM7);
+FUNC_GROUP_DECL(PWM7, AB2);
+
+#define V6 112
+SIG_EXPR_LIST_DECL_SINGLE(VPIG8, VPI24, VPI24_DESC, SIG_DESC_SET(SCU88, 8));
+SS_PIN_DECL(V6, GPIOO0, VPIG8);
+
+#define Y5 113
+SIG_EXPR_LIST_DECL_SINGLE(VPIG9, VPI24, VPI24_DESC, SIG_DESC_SET(SCU88, 9));
+SS_PIN_DECL(Y5, GPIOO1, VPIG9);
+
+FUNC_GROUP_DECL(VPI18, T5, U3, V1, U4, V2, AA22, W5, Y4, AA3, AB2);
+FUNC_GROUP_DECL(VPI24, T5, U3, V1, U4, V2, AA22, W5, Y4, AA3, AB2, V6, Y5);
+FUNC_GROUP_DECL(VPI30, T5, U3, V1, U4, V2, W1, U5, W4, Y3, AA22, W5, Y4, AA3,
+		AB2);
+
+#define Y7 125
+SIG_EXPR_LIST_DECL_SINGLE(GPIOP5, GPIOP5);
+MS_PIN_DECL_(Y7, SIG_EXPR_LIST_PTR(GPIOP5));
+
+#define AA7 126
+SSSF_PIN_DECL(AA7, GPIOP6, BMCINT, SIG_DESC_SET(SCU88, 22));
+
+#define AB7 127
+SSSF_PIN_DECL(AB7, GPIOP7, FLACK, SIG_DESC_SET(SCU88, 23));
+
+#define I2C3_DESC	SIG_DESC_SET(SCU90, 16)
+
+#define D3 128
+SIG_EXPR_LIST_DECL_SINGLE(SCL3, I2C3, I2C3_DESC);
+SS_PIN_DECL(D3, GPIOQ0, SCL3);
+
+#define C2 129
+SIG_EXPR_LIST_DECL_SINGLE(SDA3, I2C3, I2C3_DESC);
+SS_PIN_DECL(C2, GPIOQ1, SDA3);
+
+FUNC_GROUP_DECL(I2C3, D3, C2);
+
+#define I2C4_DESC	SIG_DESC_SET(SCU90, 17)
+
+#define B1 130
+SIG_EXPR_LIST_DECL_SINGLE(SCL4, I2C4, I2C4_DESC);
+SS_PIN_DECL(B1, GPIOQ2, SCL4);
+
+#define F5 131
+SIG_EXPR_LIST_DECL_SINGLE(SDA4, I2C4, I2C4_DESC);
+SS_PIN_DECL(F5, GPIOQ3, SDA4);
+
+FUNC_GROUP_DECL(I2C4, B1, F5);
+
+#define DASH9028_DESC	SIG_DESC_SET(SCU90, 28)
+
+#define H2 134
+SIG_EXPR_LIST_DECL_SINGLE(DASHH2, DASHH2, DASH9028_DESC);
+SS_PIN_DECL(H2, GPIOQ6, DASHH2);
+
+#define H1 135
+SIG_EXPR_LIST_DECL_SINGLE(DASHH1, DASHH1, DASH9028_DESC);
+SS_PIN_DECL(H1, GPIOQ7, DASHH1);
+
+#define V20 136
+SSSF_PIN_DECL(V20, GPIOR0, ROMCS1, SIG_DESC_SET(SCU88, 24));
+
+#define W21 137
+SSSF_PIN_DECL(W21, GPIOR1, ROMCS2, SIG_DESC_SET(SCU88, 25));
+
+#define Y22 138
+SSSF_PIN_DECL(Y22, GPIOR2, ROMCS3, SIG_DESC_SET(SCU88, 26));
+
+#define U19 139
+SSSF_PIN_DECL(U19, GPIOR3, ROMCS4, SIG_DESC_SET(SCU88, 27));
+
+#define VPOOFF0_DESC	{ SCU94, GENMASK(1, 0), 0, 0 }
+#define VPO12_DESC	{ SCU94, GENMASK(1, 0), 1, 0 }
+#define VPO24_DESC	{ SCU94, GENMASK(1, 0), 2, 0 }
+#define VPOOFF1_DESC	{ SCU94, GENMASK(1, 0), 3, 0 }
+#define VPO_OFF_12      { SCU94, 0x2, 0, 0 }
+#define VPO_24_OFF      SIG_DESC_SET(SCU94, 1)
+
+#define V21 140
+#define V21_DESC	SIG_DESC_SET(SCU88, 28)
+SIG_EXPR_DECL(ROMA24, ROM8, V21_DESC, VPO_OFF_12);
+SIG_EXPR_DECL(ROMA24, ROM16, V21_DESC, VPO_OFF_12);
+SIG_EXPR_DECL(ROMA24, ROM16S, V21_DESC, VPO_OFF_12);
+SIG_EXPR_LIST_DECL(ROMA24, SIG_EXPR_PTR(ROMA24, ROM8),
+		SIG_EXPR_PTR(ROMA24, ROM16),
+		SIG_EXPR_PTR(ROMA24, ROM16S));
+SIG_EXPR_LIST_DECL_SINGLE(VPOR6, VPO24, V21_DESC, VPO_24_OFF);
+MS_PIN_DECL(V21, GPIOR4, ROMA24, VPOR6);
+
+#define W22 141
+#define W22_DESC	SIG_DESC_SET(SCU88, 29)
+SIG_EXPR_DECL(ROMA25, ROM8, W22_DESC, VPO_OFF_12);
+SIG_EXPR_DECL(ROMA25, ROM16, W22_DESC, VPO_OFF_12);
+SIG_EXPR_DECL(ROMA25, ROM16S, W22_DESC, VPO_OFF_12);
+SIG_EXPR_LIST_DECL(ROMA25, SIG_EXPR_PTR(ROMA25, ROM8),
+		SIG_EXPR_PTR(ROMA25, ROM16),
+		SIG_EXPR_PTR(ROMA25, ROM16S));
+SIG_EXPR_LIST_DECL_SINGLE(VPOR7, VPO24, W22_DESC, VPO_24_OFF);
+MS_PIN_DECL(W22, GPIOR5, ROMA25, VPOR7);
+
+#define C6 142
+SIG_EXPR_LIST_DECL_SINGLE(MDC1, MDIO1, SIG_DESC_SET(SCU88, 30));
+SS_PIN_DECL(C6, GPIOR6, MDC1);
+
+#define A5 143
+SIG_EXPR_LIST_DECL_SINGLE(MDIO1, MDIO1, SIG_DESC_SET(SCU88, 31));
+SS_PIN_DECL(A5, GPIOR7, MDIO1);
+
+FUNC_GROUP_DECL(MDIO1, C6, A5);
+
+#define U21 144
+#define U21_DESC        SIG_DESC_SET(SCU8C, 0)
+SIG_EXPR_DECL(ROMD4, ROM8, U21_DESC, VPOOFF0_DESC);
+SIG_EXPR_DECL(ROMD4, ROM16, U21_DESC, VPOOFF0_DESC);
+SIG_EXPR_DECL(ROMD4, ROM16S, U21_DESC, VPOOFF0_DESC);
+SIG_EXPR_LIST_DECL(ROMD4, SIG_EXPR_PTR(ROMD4, ROM8),
+		SIG_EXPR_PTR(ROMD4, ROM16),
+		SIG_EXPR_PTR(ROMD4, ROM16S));
+SIG_EXPR_DECL(VPODE, VPO12, U21_DESC, VPO12_DESC);
+SIG_EXPR_DECL(VPODE, VPO24, U21_DESC, VPO12_DESC);
+SIG_EXPR_LIST_DECL_DUAL(VPODE, VPO12, VPO24);
+MS_PIN_DECL(U21, GPIOS0, ROMD4, VPODE);
+
+#define T19 145
+#define T19_DESC        SIG_DESC_SET(SCU8C, 1)
+SIG_EXPR_DECL(ROMD5, ROM8, T19_DESC, VPOOFF0_DESC);
+SIG_EXPR_DECL(ROMD5, ROM16, T19_DESC, VPOOFF0_DESC);
+SIG_EXPR_DECL(ROMD5, ROM16S, T19_DESC, VPOOFF0_DESC);
+SIG_EXPR_LIST_DECL(ROMD5, SIG_EXPR_PTR(ROMD5, ROM8),
+		SIG_EXPR_PTR(ROMD5, ROM16),
+		SIG_EXPR_PTR(ROMD5, ROM16S));
+SIG_EXPR_DECL(VPOHS, VPO12, T19_DESC, VPO12_DESC);
+SIG_EXPR_DECL(VPOHS, VPO24, T19_DESC, VPO24_DESC);
+SIG_EXPR_LIST_DECL_DUAL(VPOHS, VPO12, VPO24);
+MS_PIN_DECL(T19, GPIOS1, ROMD5, VPOHS);
+
+#define V22 146
+#define V22_DESC        SIG_DESC_SET(SCU8C, 2)
+SIG_EXPR_DECL(ROMD6, ROM8, V22_DESC, VPOOFF0_DESC);
+SIG_EXPR_DECL(ROMD6, ROM16, V22_DESC, VPOOFF0_DESC);
+SIG_EXPR_DECL(ROMD6, ROM16S, V22_DESC, VPOOFF0_DESC);
+SIG_EXPR_LIST_DECL(ROMD6, SIG_EXPR_PTR(ROMD6, ROM8),
+		SIG_EXPR_PTR(ROMD6, ROM16),
+		SIG_EXPR_PTR(ROMD6, ROM16S));
+SIG_EXPR_DECL(VPOVS, VPO12, V22_DESC, VPO12_DESC);
+SIG_EXPR_DECL(VPOVS, VPO24, V22_DESC, VPO24_DESC);
+SIG_EXPR_LIST_DECL_DUAL(VPOVS, VPO12, VPO24);
+MS_PIN_DECL(V22, GPIOS2, ROMD6, VPOVS);
+
+#define U20 147
+#define U20_DESC        SIG_DESC_SET(SCU8C, 3)
+SIG_EXPR_DECL(ROMD7, ROM8, U20_DESC, VPOOFF0_DESC);
+SIG_EXPR_DECL(ROMD7, ROM16, U20_DESC, VPOOFF0_DESC);
+SIG_EXPR_DECL(ROMD7, ROM16S, U20_DESC, VPOOFF0_DESC);
+SIG_EXPR_LIST_DECL(ROMD7, SIG_EXPR_PTR(ROMD7, ROM8),
+		SIG_EXPR_PTR(ROMD7, ROM16),
+		SIG_EXPR_PTR(ROMD7, ROM16S));
+SIG_EXPR_DECL(VPOCLK, VPO12, U20_DESC, VPO12_DESC);
+SIG_EXPR_DECL(VPOCLK, VPO24, U20_DESC, VPO24_DESC);
+SIG_EXPR_LIST_DECL_DUAL(VPOCLK, VPO12, VPO24);
+MS_PIN_DECL(U20, GPIOS3, ROMD7, VPOCLK);
+
+#define R18 148
+#define ROMOE_DESC      SIG_DESC_SET(SCU8C, 4)
+SIG_EXPR_LIST_DECL_SINGLE(GPIOS4, GPIOS4);
+SIG_EXPR_DECL(ROMOE, ROM8, ROMOE_DESC);
+SIG_EXPR_DECL(ROMOE, ROM16, ROMOE_DESC);
+SIG_EXPR_DECL(ROMOE, ROM16S, ROMOE_DESC);
+SIG_EXPR_LIST_DECL(ROMOE, SIG_EXPR_PTR(ROMOE, ROM8),
+		SIG_EXPR_PTR(ROMOE, ROM16),
+		SIG_EXPR_PTR(ROMOE, ROM16S));
+MS_PIN_DECL_(R18, SIG_EXPR_LIST_PTR(ROMOE), SIG_EXPR_LIST_PTR(GPIOS4));
+
+#define N21 149
+#define ROMWE_DESC      SIG_DESC_SET(SCU8C, 5)
+SIG_EXPR_LIST_DECL_SINGLE(GPIOS5, GPIOS5);
+SIG_EXPR_DECL(ROMWE, ROM8, ROMWE_DESC);
+SIG_EXPR_DECL(ROMWE, ROM16, ROMWE_DESC);
+SIG_EXPR_DECL(ROMWE, ROM16S, ROMWE_DESC);
+SIG_EXPR_LIST_DECL(ROMWE, SIG_EXPR_PTR(ROMWE, ROM8),
+		SIG_EXPR_PTR(ROMWE, ROM16),
+		SIG_EXPR_PTR(ROMWE, ROM16S));
+MS_PIN_DECL_(N21, SIG_EXPR_LIST_PTR(ROMWE), SIG_EXPR_LIST_PTR(GPIOS5));
+
+#define L22 150
+#define L22_DESC        SIG_DESC_SET(SCU8C, 6)
+SIG_EXPR_DECL(ROMA22, ROM8, L22_DESC, VPO_OFF_12);
+SIG_EXPR_DECL(ROMA22, ROM16, L22_DESC, VPO_OFF_12);
+SIG_EXPR_DECL(ROMA22, ROM16S, L22_DESC, VPO_OFF_12);
+SIG_EXPR_LIST_DECL(ROMA22, SIG_EXPR_PTR(ROMA22, ROM8),
+		SIG_EXPR_PTR(ROMA22, ROM16),
+		SIG_EXPR_PTR(ROMA22, ROM16S));
+SIG_EXPR_LIST_DECL_SINGLE(VPOR4, VPO24, L22_DESC, VPO_24_OFF);
+MS_PIN_DECL(L22, GPIOS6, ROMA22, VPOR4);
+
+#define K18 151
+#define K18_DESC	SIG_DESC_SET(SCU8C, 7)
+SIG_EXPR_DECL(ROMA23, ROM8, K18_DESC, VPO_OFF_12);
+SIG_EXPR_DECL(ROMA23, ROM16, K18_DESC, VPO_OFF_12);
+SIG_EXPR_DECL(ROMA23, ROM16S, K18_DESC, VPO_OFF_12);
+SIG_EXPR_LIST_DECL(ROMA23, SIG_EXPR_PTR(ROMA23, ROM8),
+		SIG_EXPR_PTR(ROMA23, ROM16),
+		SIG_EXPR_PTR(ROMA23, ROM16S));
+SIG_EXPR_LIST_DECL_SINGLE(VPOR5, VPO24, K18_DESC, VPO_24_OFF);
+MS_PIN_DECL(K18, GPIOS7, ROMA23, VPOR5);
+
+FUNC_GROUP_DECL(ROM8, V20, U21, T19, V22, U20, R18, N21, L22, K18, W21, Y22,
+		U19);
+FUNC_GROUP_DECL(ROM16, V20, U21, T19, V22, U20, R18, N21, L22, K18,
+		A8, C7, B7, A7, D7, B6, A6, E7, W21, Y22, U19);
+FUNC_GROUP_DECL(VPO12, U21, T19, V22, U20);
+FUNC_GROUP_DECL(VPO24, U21, T19, V22, U20, L22, K18, V21, W22);
+
+#define RMII1_DESC      SIG_DESC_BIT(HW_STRAP1, 6, 0)
+
+#define A12 152
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT0, GPIOT0, SIG_DESC_SET(SCUA0, 0));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1TXEN, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXCK, RGMII1);
+MS_PIN_DECL_(A12, SIG_EXPR_LIST_PTR(GPIOT0), SIG_EXPR_LIST_PTR(RMII1TXEN),
+		SIG_EXPR_LIST_PTR(RGMII1TXCK));
+
+#define B12 153
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT1, GPIOT1, SIG_DESC_SET(SCUA0, 1));
+SIG_EXPR_LIST_DECL_SINGLE(DASHB12, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXCTL, RGMII1);
+MS_PIN_DECL_(B12, SIG_EXPR_LIST_PTR(GPIOT1), SIG_EXPR_LIST_PTR(DASHB12),
+		SIG_EXPR_LIST_PTR(RGMII1TXCTL));
+
+#define C12 154
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT2, GPIOT2, SIG_DESC_SET(SCUA0, 2));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1TXD0, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXD0, RGMII1);
+MS_PIN_DECL_(C12, SIG_EXPR_LIST_PTR(GPIOT2), SIG_EXPR_LIST_PTR(RMII1TXD0),
+		SIG_EXPR_LIST_PTR(RGMII1TXD0));
+
+#define D12 155
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT3, GPIOT3, SIG_DESC_SET(SCUA0, 3));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1TXD1, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXD1, RGMII1);
+MS_PIN_DECL_(D12, SIG_EXPR_LIST_PTR(GPIOT3), SIG_EXPR_LIST_PTR(RMII1TXD1),
+		SIG_EXPR_LIST_PTR(RGMII1TXD1));
+
+#define E12 156
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT4, GPIOT4, SIG_DESC_SET(SCUA0, 4));
+SIG_EXPR_LIST_DECL_SINGLE(DASHE12, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXD2, RGMII1);
+MS_PIN_DECL_(E12, SIG_EXPR_LIST_PTR(GPIOT4), SIG_EXPR_LIST_PTR(DASHE12),
+		SIG_EXPR_LIST_PTR(RGMII1TXD2));
+
+#define A13 157
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT5, GPIOT5, SIG_DESC_SET(SCUA0, 5));
+SIG_EXPR_LIST_DECL_SINGLE(DASHA13, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXD3, RGMII1);
+MS_PIN_DECL_(A13, SIG_EXPR_LIST_PTR(GPIOT5), SIG_EXPR_LIST_PTR(DASHA13),
+		SIG_EXPR_LIST_PTR(RGMII1TXD3));
+
+#define E11 164
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU4, GPIOU4, SIG_DESC_SET(SCUA0, 12));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1RCLK, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXCK, RGMII1);
+MS_PIN_DECL_(E11, SIG_EXPR_LIST_PTR(GPIOU4), SIG_EXPR_LIST_PTR(RMII1RCLK),
+		SIG_EXPR_LIST_PTR(RGMII1RXCK));
+
+#define D11 165
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU5, GPIOU5, SIG_DESC_SET(SCUA0, 13));
+SIG_EXPR_LIST_DECL_SINGLE(DASHD11, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXCTL, RGMII1);
+MS_PIN_DECL_(D11, SIG_EXPR_LIST_PTR(GPIOU5), SIG_EXPR_LIST_PTR(DASHD11),
+		SIG_EXPR_LIST_PTR(RGMII1RXCTL));
+
+#define C11 166
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU6, GPIOU6, SIG_DESC_SET(SCUA0, 14));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1RXD0, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXD0, RGMII1);
+MS_PIN_DECL_(C11, SIG_EXPR_LIST_PTR(GPIOU6), SIG_EXPR_LIST_PTR(RMII1RXD0),
+		SIG_EXPR_LIST_PTR(RGMII1RXD0));
+
+#define B11 167
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU7, GPIOU7, SIG_DESC_SET(SCUA0, 15));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1RXD1, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXD1, RGMII1);
+MS_PIN_DECL_(B11, SIG_EXPR_LIST_PTR(GPIOU7), SIG_EXPR_LIST_PTR(RMII1RXD1),
+		SIG_EXPR_LIST_PTR(RGMII1RXD1));
+
+#define A11 168
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV0, GPIOV0, SIG_DESC_SET(SCUA0, 16));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1CRSDV, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXD2, RGMII1);
+MS_PIN_DECL_(A11, SIG_EXPR_LIST_PTR(GPIOV0), SIG_EXPR_LIST_PTR(RMII1CRSDV),
+		SIG_EXPR_LIST_PTR(RGMII1RXD2));
+
+#define E10 169
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV1, GPIOV1, SIG_DESC_SET(SCUA0, 17));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1RXER, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXD3, RGMII1);
+MS_PIN_DECL_(E10, SIG_EXPR_LIST_PTR(GPIOV1), SIG_EXPR_LIST_PTR(RMII1RXER),
+		SIG_EXPR_LIST_PTR(RGMII1RXD3));
+
+FUNC_GROUP_DECL(RMII1, A12, B12, C12, D12, E12, A13, E11, D11, C11, B11, A11,
+		E10);
+FUNC_GROUP_DECL(RGMII1, A12, B12, C12, D12, E12, A13, E11, D11, C11, B11, A11,
+		E10);
+
+/* Note we account for GPIOY4-GPIOY7 even though they're not valid, thus 216
+ * pins becomes 220.
+ */
+#define ASPEED_G4_NR_PINS 220
+
+/* Pins, groups and functions are sort(1):ed alphabetically for sanity */
+
+static struct pinctrl_pin_desc aspeed_g4_pins[ASPEED_G4_NR_PINS] = {
+	ASPEED_PINCTRL_PIN(A1),
+	ASPEED_PINCTRL_PIN(A11),
+	ASPEED_PINCTRL_PIN(A12),
+	ASPEED_PINCTRL_PIN(A13),
+	ASPEED_PINCTRL_PIN(A15),
+	ASPEED_PINCTRL_PIN(A18),
+	ASPEED_PINCTRL_PIN(A2),
+	ASPEED_PINCTRL_PIN(A3),
+	ASPEED_PINCTRL_PIN(A4),
+	ASPEED_PINCTRL_PIN(A5),
+	ASPEED_PINCTRL_PIN(A6),
+	ASPEED_PINCTRL_PIN(A7),
+	ASPEED_PINCTRL_PIN(A8),
+	ASPEED_PINCTRL_PIN(AA2),
+	ASPEED_PINCTRL_PIN(AA22),
+	ASPEED_PINCTRL_PIN(AA3),
+	ASPEED_PINCTRL_PIN(AA7),
+	ASPEED_PINCTRL_PIN(AB1),
+	ASPEED_PINCTRL_PIN(AB2),
+	ASPEED_PINCTRL_PIN(AB7),
+	ASPEED_PINCTRL_PIN(B1),
+	ASPEED_PINCTRL_PIN(B11),
+	ASPEED_PINCTRL_PIN(B12),
+	ASPEED_PINCTRL_PIN(B14),
+	ASPEED_PINCTRL_PIN(B15),
+	ASPEED_PINCTRL_PIN(B19),
+	ASPEED_PINCTRL_PIN(B2),
+	ASPEED_PINCTRL_PIN(B3),
+	ASPEED_PINCTRL_PIN(B4),
+	ASPEED_PINCTRL_PIN(B6),
+	ASPEED_PINCTRL_PIN(B7),
+	ASPEED_PINCTRL_PIN(C1),
+	ASPEED_PINCTRL_PIN(C11),
+	ASPEED_PINCTRL_PIN(C12),
+	ASPEED_PINCTRL_PIN(C14),
+	ASPEED_PINCTRL_PIN(C15),
+	ASPEED_PINCTRL_PIN(C17),
+	ASPEED_PINCTRL_PIN(C2),
+	ASPEED_PINCTRL_PIN(C3),
+	ASPEED_PINCTRL_PIN(C4),
+	ASPEED_PINCTRL_PIN(C5),
+	ASPEED_PINCTRL_PIN(C6),
+	ASPEED_PINCTRL_PIN(C7),
+	ASPEED_PINCTRL_PIN(D1),
+	ASPEED_PINCTRL_PIN(D11),
+	ASPEED_PINCTRL_PIN(D12),
+	ASPEED_PINCTRL_PIN(D14),
+	ASPEED_PINCTRL_PIN(D15),
+	ASPEED_PINCTRL_PIN(D16),
+	ASPEED_PINCTRL_PIN(D17),
+	ASPEED_PINCTRL_PIN(D18),
+	ASPEED_PINCTRL_PIN(D2),
+	ASPEED_PINCTRL_PIN(D3),
+	ASPEED_PINCTRL_PIN(D4),
+	ASPEED_PINCTRL_PIN(D5),
+	ASPEED_PINCTRL_PIN(D7),
+	ASPEED_PINCTRL_PIN(E10),
+	ASPEED_PINCTRL_PIN(E11),
+	ASPEED_PINCTRL_PIN(E12),
+	ASPEED_PINCTRL_PIN(E14),
+	ASPEED_PINCTRL_PIN(E16),
+	ASPEED_PINCTRL_PIN(E2),
+	ASPEED_PINCTRL_PIN(E3),
+	ASPEED_PINCTRL_PIN(E5),
+	ASPEED_PINCTRL_PIN(E7),
+	ASPEED_PINCTRL_PIN(F3),
+	ASPEED_PINCTRL_PIN(F4),
+	ASPEED_PINCTRL_PIN(F5),
+	ASPEED_PINCTRL_PIN(G5),
+	ASPEED_PINCTRL_PIN(H1),
+	ASPEED_PINCTRL_PIN(H19),
+	ASPEED_PINCTRL_PIN(H2),
+	ASPEED_PINCTRL_PIN(H20),
+	ASPEED_PINCTRL_PIN(J3),
+	ASPEED_PINCTRL_PIN(K18),
+	ASPEED_PINCTRL_PIN(L22),
+	ASPEED_PINCTRL_PIN(N21),
+	ASPEED_PINCTRL_PIN(R18),
+	ASPEED_PINCTRL_PIN(T1),
+	ASPEED_PINCTRL_PIN(T19),
+	ASPEED_PINCTRL_PIN(T2),
+	ASPEED_PINCTRL_PIN(T4),
+	ASPEED_PINCTRL_PIN(T5),
+	ASPEED_PINCTRL_PIN(U1),
+	ASPEED_PINCTRL_PIN(U18),
+	ASPEED_PINCTRL_PIN(U19),
+	ASPEED_PINCTRL_PIN(U2),
+	ASPEED_PINCTRL_PIN(U20),
+	ASPEED_PINCTRL_PIN(U21),
+	ASPEED_PINCTRL_PIN(U3),
+	ASPEED_PINCTRL_PIN(U4),
+	ASPEED_PINCTRL_PIN(U5),
+	ASPEED_PINCTRL_PIN(V1),
+	ASPEED_PINCTRL_PIN(V2),
+	ASPEED_PINCTRL_PIN(V20),
+	ASPEED_PINCTRL_PIN(V21),
+	ASPEED_PINCTRL_PIN(V22),
+	ASPEED_PINCTRL_PIN(V6),
+	ASPEED_PINCTRL_PIN(W1),
+	ASPEED_PINCTRL_PIN(W21),
+	ASPEED_PINCTRL_PIN(W22),
+	ASPEED_PINCTRL_PIN(W4),
+	ASPEED_PINCTRL_PIN(W5),
+	ASPEED_PINCTRL_PIN(Y22),
+	ASPEED_PINCTRL_PIN(Y3),
+	ASPEED_PINCTRL_PIN(Y4),
+	ASPEED_PINCTRL_PIN(Y5),
+	ASPEED_PINCTRL_PIN(Y7),
+};
+
+static const struct aspeed_pin_group aspeed_g4_groups[] = {
+	ASPEED_PINCTRL_GROUP(ACPI),
+	ASPEED_PINCTRL_GROUP(BMCINT),
+	ASPEED_PINCTRL_GROUP(DDCCLK),
+	ASPEED_PINCTRL_GROUP(DDCDAT),
+	ASPEED_PINCTRL_GROUP(FLACK),
+	ASPEED_PINCTRL_GROUP(FLBUSY),
+	ASPEED_PINCTRL_GROUP(FLWP),
+	ASPEED_PINCTRL_GROUP(GPID0),
+	ASPEED_PINCTRL_GROUP(GPIE0),
+	ASPEED_PINCTRL_GROUP(GPIE2),
+	ASPEED_PINCTRL_GROUP(GPIE4),
+	ASPEED_PINCTRL_GROUP(GPIE6),
+	ASPEED_PINCTRL_GROUP(I2C10),
+	ASPEED_PINCTRL_GROUP(I2C11),
+	ASPEED_PINCTRL_GROUP(I2C12),
+	ASPEED_PINCTRL_GROUP(I2C13),
+	ASPEED_PINCTRL_GROUP(I2C3),
+	ASPEED_PINCTRL_GROUP(I2C4),
+	ASPEED_PINCTRL_GROUP(I2C5),
+	ASPEED_PINCTRL_GROUP(I2C6),
+	ASPEED_PINCTRL_GROUP(I2C7),
+	ASPEED_PINCTRL_GROUP(I2C8),
+	ASPEED_PINCTRL_GROUP(I2C9),
+	ASPEED_PINCTRL_GROUP(LPCPD),
+	ASPEED_PINCTRL_GROUP(LPCPME),
+	ASPEED_PINCTRL_GROUP(LPCPME),
+	ASPEED_PINCTRL_GROUP(LPCSMI),
+	ASPEED_PINCTRL_GROUP(MDIO1),
+	ASPEED_PINCTRL_GROUP(MDIO2),
+	ASPEED_PINCTRL_GROUP(NCTS1),
+	ASPEED_PINCTRL_GROUP(NCTS3),
+	ASPEED_PINCTRL_GROUP(NCTS4),
+	ASPEED_PINCTRL_GROUP(NDCD1),
+	ASPEED_PINCTRL_GROUP(NDCD3),
+	ASPEED_PINCTRL_GROUP(NDCD4),
+	ASPEED_PINCTRL_GROUP(NDSR1),
+	ASPEED_PINCTRL_GROUP(NDSR3),
+	ASPEED_PINCTRL_GROUP(NDTR1),
+	ASPEED_PINCTRL_GROUP(NDTR3),
+	ASPEED_PINCTRL_GROUP(NRI1),
+	ASPEED_PINCTRL_GROUP(NRI3),
+	ASPEED_PINCTRL_GROUP(NRI4),
+	ASPEED_PINCTRL_GROUP(NRTS1),
+	ASPEED_PINCTRL_GROUP(NRTS3),
+	ASPEED_PINCTRL_GROUP(PWM0),
+	ASPEED_PINCTRL_GROUP(PWM1),
+	ASPEED_PINCTRL_GROUP(PWM2),
+	ASPEED_PINCTRL_GROUP(PWM3),
+	ASPEED_PINCTRL_GROUP(PWM4),
+	ASPEED_PINCTRL_GROUP(PWM5),
+	ASPEED_PINCTRL_GROUP(PWM6),
+	ASPEED_PINCTRL_GROUP(PWM7),
+	ASPEED_PINCTRL_GROUP(RGMII1),
+	ASPEED_PINCTRL_GROUP(RMII1),
+	ASPEED_PINCTRL_GROUP(ROM16),
+	ASPEED_PINCTRL_GROUP(ROM8),
+	ASPEED_PINCTRL_GROUP(ROMCS1),
+	ASPEED_PINCTRL_GROUP(ROMCS2),
+	ASPEED_PINCTRL_GROUP(ROMCS3),
+	ASPEED_PINCTRL_GROUP(ROMCS4),
+	ASPEED_PINCTRL_GROUP(RXD1),
+	ASPEED_PINCTRL_GROUP(RXD3),
+	ASPEED_PINCTRL_GROUP(RXD4),
+	ASPEED_PINCTRL_GROUP(SD1),
+	ASPEED_PINCTRL_GROUP(SGPMI),
+	ASPEED_PINCTRL_GROUP(SIOPBI),
+	ASPEED_PINCTRL_GROUP(SIOPBO),
+	ASPEED_PINCTRL_GROUP(TIMER3),
+	ASPEED_PINCTRL_GROUP(TIMER5),
+	ASPEED_PINCTRL_GROUP(TIMER6),
+	ASPEED_PINCTRL_GROUP(TIMER7),
+	ASPEED_PINCTRL_GROUP(TIMER8),
+	ASPEED_PINCTRL_GROUP(TXD1),
+	ASPEED_PINCTRL_GROUP(TXD3),
+	ASPEED_PINCTRL_GROUP(TXD4),
+	ASPEED_PINCTRL_GROUP(UART6),
+	ASPEED_PINCTRL_GROUP(VGAHS),
+	ASPEED_PINCTRL_GROUP(VGAVS),
+	ASPEED_PINCTRL_GROUP(VPI18),
+	ASPEED_PINCTRL_GROUP(VPI24),
+	ASPEED_PINCTRL_GROUP(VPI30),
+	ASPEED_PINCTRL_GROUP(VPO12),
+	ASPEED_PINCTRL_GROUP(VPO24),
+};
+
+static const struct aspeed_pin_function aspeed_g4_functions[] = {
+	ASPEED_PINCTRL_FUNC(ACPI),
+	ASPEED_PINCTRL_FUNC(BMCINT),
+	ASPEED_PINCTRL_FUNC(DDCCLK),
+	ASPEED_PINCTRL_FUNC(DDCDAT),
+	ASPEED_PINCTRL_FUNC(FLACK),
+	ASPEED_PINCTRL_FUNC(FLBUSY),
+	ASPEED_PINCTRL_FUNC(FLWP),
+	ASPEED_PINCTRL_FUNC(GPID0),
+	ASPEED_PINCTRL_FUNC(GPIE0),
+	ASPEED_PINCTRL_FUNC(GPIE2),
+	ASPEED_PINCTRL_FUNC(GPIE4),
+	ASPEED_PINCTRL_FUNC(GPIE6),
+	ASPEED_PINCTRL_FUNC(I2C10),
+	ASPEED_PINCTRL_FUNC(I2C11),
+	ASPEED_PINCTRL_FUNC(I2C12),
+	ASPEED_PINCTRL_FUNC(I2C13),
+	ASPEED_PINCTRL_FUNC(I2C3),
+	ASPEED_PINCTRL_FUNC(I2C4),
+	ASPEED_PINCTRL_FUNC(I2C5),
+	ASPEED_PINCTRL_FUNC(I2C6),
+	ASPEED_PINCTRL_FUNC(I2C7),
+	ASPEED_PINCTRL_FUNC(I2C8),
+	ASPEED_PINCTRL_FUNC(I2C9),
+	ASPEED_PINCTRL_FUNC(LPCPD),
+	ASPEED_PINCTRL_FUNC(LPCPME),
+	ASPEED_PINCTRL_FUNC(LPCSMI),
+	ASPEED_PINCTRL_FUNC(MDIO1),
+	ASPEED_PINCTRL_FUNC(MDIO2),
+	ASPEED_PINCTRL_FUNC(NCTS1),
+	ASPEED_PINCTRL_FUNC(NCTS3),
+	ASPEED_PINCTRL_FUNC(NCTS4),
+	ASPEED_PINCTRL_FUNC(NDCD1),
+	ASPEED_PINCTRL_FUNC(NDCD3),
+	ASPEED_PINCTRL_FUNC(NDCD4),
+	ASPEED_PINCTRL_FUNC(NDSR1),
+	ASPEED_PINCTRL_FUNC(NDSR3),
+	ASPEED_PINCTRL_FUNC(NDTR1),
+	ASPEED_PINCTRL_FUNC(NDTR3),
+	ASPEED_PINCTRL_FUNC(NRI1),
+	ASPEED_PINCTRL_FUNC(NRI3),
+	ASPEED_PINCTRL_FUNC(NRI4),
+	ASPEED_PINCTRL_FUNC(NRTS1),
+	ASPEED_PINCTRL_FUNC(NRTS3),
+	ASPEED_PINCTRL_FUNC(PWM0),
+	ASPEED_PINCTRL_FUNC(PWM1),
+	ASPEED_PINCTRL_FUNC(PWM2),
+	ASPEED_PINCTRL_FUNC(PWM3),
+	ASPEED_PINCTRL_FUNC(PWM4),
+	ASPEED_PINCTRL_FUNC(PWM5),
+	ASPEED_PINCTRL_FUNC(PWM6),
+	ASPEED_PINCTRL_FUNC(PWM7),
+	ASPEED_PINCTRL_FUNC(RGMII1),
+	ASPEED_PINCTRL_FUNC(RMII1),
+	ASPEED_PINCTRL_FUNC(ROM16),
+	ASPEED_PINCTRL_FUNC(ROM8),
+	ASPEED_PINCTRL_FUNC(ROMCS1),
+	ASPEED_PINCTRL_FUNC(ROMCS2),
+	ASPEED_PINCTRL_FUNC(ROMCS3),
+	ASPEED_PINCTRL_FUNC(ROMCS4),
+	ASPEED_PINCTRL_FUNC(RXD1),
+	ASPEED_PINCTRL_FUNC(RXD3),
+	ASPEED_PINCTRL_FUNC(RXD4),
+	ASPEED_PINCTRL_FUNC(SD1),
+	ASPEED_PINCTRL_FUNC(SGPMI),
+	ASPEED_PINCTRL_FUNC(SIOPBI),
+	ASPEED_PINCTRL_FUNC(SIOPBO),
+	ASPEED_PINCTRL_FUNC(TIMER3),
+	ASPEED_PINCTRL_FUNC(TIMER5),
+	ASPEED_PINCTRL_FUNC(TIMER6),
+	ASPEED_PINCTRL_FUNC(TIMER7),
+	ASPEED_PINCTRL_FUNC(TIMER8),
+	ASPEED_PINCTRL_FUNC(TXD1),
+	ASPEED_PINCTRL_FUNC(TXD3),
+	ASPEED_PINCTRL_FUNC(TXD4),
+	ASPEED_PINCTRL_FUNC(UART6),
+	ASPEED_PINCTRL_FUNC(VGAHS),
+	ASPEED_PINCTRL_FUNC(VGAVS),
+	ASPEED_PINCTRL_FUNC(VPI18),
+	ASPEED_PINCTRL_FUNC(VPI24),
+	ASPEED_PINCTRL_FUNC(VPI30),
+	ASPEED_PINCTRL_FUNC(VPO12),
+	ASPEED_PINCTRL_FUNC(VPO24),
+};
+
+static struct aspeed_pinctrl_data aspeed_g4_pinctrl_data = {
+	.pins = aspeed_g4_pins,
+	.npins = ARRAY_SIZE(aspeed_g4_pins),
+	.groups = aspeed_g4_groups,
+	.ngroups = ARRAY_SIZE(aspeed_g4_groups),
+	.functions = aspeed_g4_functions,
+	.nfunctions = ARRAY_SIZE(aspeed_g4_functions),
+};
+
+static struct pinmux_ops aspeed_g4_pinmux_ops = {
+	.get_functions_count = aspeed_pinmux_get_fn_count,
+	.get_function_name = aspeed_pinmux_get_fn_name,
+	.get_function_groups = aspeed_pinmux_get_fn_groups,
+	.set_mux = aspeed_pinmux_set_mux,
+	.gpio_request_enable = aspeed_gpio_request_enable,
+	.strict = true,
+};
+
+static struct pinctrl_ops aspeed_g4_pinctrl_ops = {
+	.get_groups_count = aspeed_pinctrl_get_groups_count,
+	.get_group_name = aspeed_pinctrl_get_group_name,
+	.get_group_pins = aspeed_pinctrl_get_group_pins,
+	.pin_dbg_show = aspeed_pinctrl_pin_dbg_show,
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_pin,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+static struct pinctrl_desc aspeed_g4_pinctrl_desc = {
+	.name = "aspeed-g4-pinctrl",
+	.pins = aspeed_g4_pins,
+	.npins = ARRAY_SIZE(aspeed_g4_pins),
+	.pctlops = &aspeed_g4_pinctrl_ops,
+	.pmxops = &aspeed_g4_pinmux_ops,
+};
+
+static int aspeed_g4_pinctrl_probe(struct platform_device *pdev)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(aspeed_g4_pins); i++)
+		aspeed_g4_pins[i].number = i;
+
+	return aspeed_pinctrl_probe(pdev, &aspeed_g4_pinctrl_desc,
+			&aspeed_g4_pinctrl_data);
+}
+
+static const struct of_device_id aspeed_g4_pinctrl_of_match[] = {
+	{ .compatible = "aspeed,ast2400-pinctrl", },
+	{ .compatible = "aspeed,g4-pinctrl", },
+	{ },
+};
+
+static struct platform_driver aspeed_g4_pinctrl_driver = {
+	.probe = aspeed_g4_pinctrl_probe,
+	.driver = {
+		.name = "aspeed-g4-pinctrl",
+		.of_match_table = aspeed_g4_pinctrl_of_match,
+	},
+};
+
+static int aspeed_g4_pinctrl_init(void)
+{
+	return platform_driver_register(&aspeed_g4_pinctrl_driver);
+}
+
+arch_initcall(aspeed_g4_pinctrl_init);
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed-g5.c b/drivers/pinctrl/aspeed/pinctrl-aspeed-g5.c
new file mode 100644
index 000000000000..e1ab864e1a7f
--- /dev/null
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed-g5.c
@@ -0,0 +1,808 @@
+/*
+ * Copyright (C) 2016 IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#include <linux/bitops.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+#include "../core.h"
+#include "../pinctrl-utils.h"
+#include "pinctrl-aspeed.h"
+
+#define ASPEED_G5_NR_PINS 228
+
+#define COND1		SIG_DESC_BIT(SCU90, 6, 0)
+#define COND2		{ SCU94, GENMASK(1, 0), 0, 0 }
+
+#define B14 0
+SSSF_PIN_DECL(B14, GPIOA0, MAC1LINK, SIG_DESC_SET(SCU80, 0));
+
+#define E13 3
+SSSF_PIN_DECL(E13, GPIOA3, TIMER4, SIG_DESC_SET(SCU80, 3));
+
+#define I2C9_DESC	SIG_DESC_SET(SCU90, 22)
+
+#define C14 4
+SIG_EXPR_LIST_DECL_SINGLE(SCL9, I2C9, I2C9_DESC, COND1);
+SIG_EXPR_LIST_DECL_SINGLE(TIMER5, TIMER5, SIG_DESC_SET(SCU80, 4), COND1);
+MS_PIN_DECL(C14, GPIOA4, SCL9, TIMER5);
+
+FUNC_GROUP_DECL(TIMER5, C14);
+
+#define A13 5
+SIG_EXPR_LIST_DECL_SINGLE(SDA9, I2C9, I2C9_DESC, COND1);
+SIG_EXPR_LIST_DECL_SINGLE(TIMER6, TIMER6, SIG_DESC_SET(SCU80, 5), COND1);
+MS_PIN_DECL(A13, GPIOA5, SDA9, TIMER6);
+
+FUNC_GROUP_DECL(TIMER6, A13);
+
+FUNC_GROUP_DECL(I2C9, C14, A13);
+
+#define MDIO2_DESC	SIG_DESC_SET(SCU90, 2)
+
+#define C13 6
+SIG_EXPR_LIST_DECL_SINGLE(MDC2, MDIO2, MDIO2_DESC, COND1);
+SIG_EXPR_LIST_DECL_SINGLE(TIMER7, TIMER7, SIG_DESC_SET(SCU80, 6), COND1);
+MS_PIN_DECL(C13, GPIOA6, MDC2, TIMER7);
+
+FUNC_GROUP_DECL(TIMER7, C13);
+
+#define B13 7
+SIG_EXPR_LIST_DECL_SINGLE(MDIO2, MDIO2, MDIO2_DESC, COND1);
+SIG_EXPR_LIST_DECL_SINGLE(TIMER8, TIMER8, SIG_DESC_SET(SCU80, 7), COND1);
+MS_PIN_DECL(B13, GPIOA7, MDIO2, TIMER8);
+
+FUNC_GROUP_DECL(TIMER8, B13);
+
+FUNC_GROUP_DECL(MDIO2, C13, B13);
+
+#define H20 15
+GPIO_PIN_DECL(H20, GPIOB7);
+
+#define SD1_DESC	SIG_DESC_SET(SCU90, 0)
+
+#define C12 16
+#define I2C10_DESC	SIG_DESC_SET(SCU90, 23)
+SIG_EXPR_LIST_DECL_SINGLE(SD1CLK, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SCL10, I2C10, I2C10_DESC);
+MS_PIN_DECL(C12, GPIOC0, SD1CLK, SCL10);
+
+#define A12 17
+SIG_EXPR_LIST_DECL_SINGLE(SD1CMD, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SDA10, I2C10, I2C10_DESC);
+MS_PIN_DECL(A12, GPIOC1, SD1CMD, SDA10);
+
+FUNC_GROUP_DECL(I2C10, C12, A12);
+
+#define B12 18
+#define I2C11_DESC	SIG_DESC_SET(SCU90, 24)
+SIG_EXPR_LIST_DECL_SINGLE(SD1DAT0, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SCL11, I2C11, I2C11_DESC);
+MS_PIN_DECL(B12, GPIOC2, SD1DAT0, SCL11);
+
+#define D9  19
+SIG_EXPR_LIST_DECL_SINGLE(SD1DAT1, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SDA11, I2C11, I2C11_DESC);
+MS_PIN_DECL(D9, GPIOC3, SD1DAT1, SDA11);
+
+FUNC_GROUP_DECL(I2C11, B12, D9);
+
+#define D10 20
+#define I2C12_DESC	SIG_DESC_SET(SCU90, 25)
+SIG_EXPR_LIST_DECL_SINGLE(SD1DAT2, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SCL12, I2C12, I2C12_DESC);
+MS_PIN_DECL(D10, GPIOC4, SD1DAT2, SCL12);
+
+#define E12 21
+SIG_EXPR_LIST_DECL_SINGLE(SD1DAT3, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SDA12, I2C12, I2C12_DESC);
+MS_PIN_DECL(E12, GPIOC5, SD1DAT3, SDA12);
+
+FUNC_GROUP_DECL(I2C12, D10, E12);
+
+#define C11 22
+#define I2C13_DESC	SIG_DESC_SET(SCU90, 26)
+SIG_EXPR_LIST_DECL_SINGLE(SD1CD, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SCL13, I2C13, I2C13_DESC);
+MS_PIN_DECL(C11, GPIOC6, SD1CD, SCL13);
+
+#define B11 23
+SIG_EXPR_LIST_DECL_SINGLE(SD1WP, SD1, SD1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(SDA13, I2C13, I2C13_DESC);
+MS_PIN_DECL(B11, GPIOC7, SD1WP, SDA13);
+
+FUNC_GROUP_DECL(I2C13, C11, B11);
+FUNC_GROUP_DECL(SD1, C12, A12, B12, D9, D10, E12, C11, B11);
+
+#define SD2_DESC        SIG_DESC_SET(SCU90, 1)
+#define GPID0_DESC      SIG_DESC_SET(SCU8C, 8)
+#define GPID_DESC       SIG_DESC_SET(HW_STRAP1, 21)
+
+#define F19 24
+SIG_EXPR_LIST_DECL_SINGLE(SD2CLK, SD2, SD2_DESC);
+SIG_EXPR_DECL(GPID0IN, GPID0, GPID0_DESC);
+SIG_EXPR_DECL(GPID0IN, GPID, GPID_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPID0IN, GPID0, GPID);
+MS_PIN_DECL(F19, GPIOD0, SD2CLK, GPID0IN);
+
+#define E21 25
+SIG_EXPR_LIST_DECL_SINGLE(SD2CMD, SD2, SD2_DESC);
+SIG_EXPR_DECL(GPID0OUT, GPID0, GPID0_DESC);
+SIG_EXPR_DECL(GPID0OUT, GPID, GPID_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPID0OUT, GPID0, GPID);
+MS_PIN_DECL(E21, GPIOD1, SD2CMD, GPID0OUT);
+
+FUNC_GROUP_DECL(GPID0, F19, E21);
+
+#define GPID2_DESC      SIG_DESC_SET(SCU8C, 9)
+
+#define D20 26
+SIG_EXPR_LIST_DECL_SINGLE(SD2DAT0, SD2, SD2_DESC);
+SIG_EXPR_DECL(GPID2IN, GPID2, GPID2_DESC);
+SIG_EXPR_DECL(GPID2IN, GPID, GPID_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPID2IN, GPID2, GPID);
+MS_PIN_DECL(D20, GPIOD2, SD2DAT0, GPID2IN);
+
+#define D21 27
+SIG_EXPR_LIST_DECL_SINGLE(SD2DAT1, SD2, SD2_DESC);
+SIG_EXPR_DECL(GPID2OUT, GPID2, GPID2_DESC);
+SIG_EXPR_DECL(GPID2OUT, GPID, GPID_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPID2OUT, GPID2, GPID);
+MS_PIN_DECL(D21, GPIOD3, SD2DAT1, GPID2OUT);
+
+FUNC_GROUP_DECL(GPID2, D20, D21);
+
+#define GPIE_DESC	SIG_DESC_SET(HW_STRAP1, 21)
+#define GPIE0_DESC	SIG_DESC_SET(SCU8C, 12)
+
+#define B20 32
+SIG_EXPR_LIST_DECL_SINGLE(NCTS3, NCTS3, SIG_DESC_SET(SCU80, 16));
+SIG_EXPR_DECL(GPIE0IN, GPIE0, GPIE0_DESC);
+SIG_EXPR_DECL(GPIE0IN, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE0IN, GPIE0, GPIE);
+MS_PIN_DECL(B20, GPIOE0, NCTS3, GPIE0IN);
+
+#define C20 33
+SIG_EXPR_LIST_DECL_SINGLE(NDCD3, NDCD3, SIG_DESC_SET(SCU80, 17));
+SIG_EXPR_DECL(GPIE0OUT, GPIE0, GPIE0_DESC);
+SIG_EXPR_DECL(GPIE0OUT, GPIE, GPIE_DESC);
+SIG_EXPR_LIST_DECL_DUAL(GPIE0OUT, GPIE0, GPIE);
+MS_PIN_DECL(C20, GPIE0, NDCD3, GPIE0OUT);
+
+FUNC_GROUP_DECL(GPIE0, B20, C20);
+
+#define SPI1_DESC	SIG_DESC_SET(HW_STRAP1, 13)
+#define C18 64
+SIG_EXPR_LIST_DECL_SINGLE(SYSCS, SPI1, COND1, SPI1_DESC);
+SS_PIN_DECL(C18, GPIOI0, SYSCS);
+
+#define E15 65
+SIG_EXPR_LIST_DECL_SINGLE(SYSCK, SPI1, COND1, SPI1_DESC);
+SS_PIN_DECL(E15, GPIOI1, SYSCK);
+
+#define A14 66
+SIG_EXPR_LIST_DECL_SINGLE(SYSMOSI, SPI1, COND1, SPI1_DESC);
+SS_PIN_DECL(A14, GPIOI2, SYSMOSI);
+
+#define C16 67
+SIG_EXPR_LIST_DECL_SINGLE(SYSMISO, SPI1, COND1, SPI1_DESC);
+SS_PIN_DECL(C16, GPIOI3, SYSMISO);
+
+FUNC_GROUP_DECL(SPI1, C18, E15, A14, C16);
+
+#define L2 73
+SIG_EXPR_LIST_DECL_SINGLE(SGPMLD, SGPM, SIG_DESC_SET(SCU84, 9));
+SS_PIN_DECL(L2, GPIOJ1, SGPMLD);
+
+#define N3 74
+SIG_EXPR_LIST_DECL_SINGLE(SGPMO, SGPM, SIG_DESC_SET(SCU84, 10));
+SS_PIN_DECL(N3, GPIOJ2, SGPMO);
+
+#define N4 75
+SIG_EXPR_LIST_DECL_SINGLE(SGPMI, SGPM, SIG_DESC_SET(SCU84, 11));
+SS_PIN_DECL(N4, GPIOJ3, SGPMI);
+
+#define I2C5_DESC       SIG_DESC_SET(SCU90, 18)
+
+#define L3 80
+SIG_EXPR_LIST_DECL_SINGLE(SCL5, I2C5, I2C5_DESC);
+SS_PIN_DECL(L3, GPIOK0, SCL5);
+
+#define L4 81
+SIG_EXPR_LIST_DECL_SINGLE(SDA5, I2C5, I2C5_DESC);
+SS_PIN_DECL(L4, GPIOK1, SDA5);
+
+FUNC_GROUP_DECL(I2C5, L3, L4);
+
+#define I2C6_DESC       SIG_DESC_SET(SCU90, 19)
+
+#define L1 82
+SIG_EXPR_LIST_DECL_SINGLE(SCL6, I2C6, I2C6_DESC);
+SS_PIN_DECL(L1, GPIOK2, SCL6);
+
+#define N2 83
+SIG_EXPR_LIST_DECL_SINGLE(SDA6, I2C6, I2C6_DESC);
+SS_PIN_DECL(N2, GPIOK3, SDA6);
+
+FUNC_GROUP_DECL(I2C6, L1, N2);
+
+#define I2C7_DESC       SIG_DESC_SET(SCU90, 20)
+
+#define N1 84
+SIG_EXPR_LIST_DECL_SINGLE(SCL7, I2C7, I2C7_DESC);
+SS_PIN_DECL(N1, GPIOK4, SCL7);
+
+#define P1 85
+SIG_EXPR_LIST_DECL_SINGLE(SDA7, I2C7, I2C7_DESC);
+SS_PIN_DECL(P1, GPIOK5, SDA7);
+
+FUNC_GROUP_DECL(I2C7, N1, P1);
+
+#define I2C8_DESC       SIG_DESC_SET(SCU90, 21)
+
+#define P2 86
+SIG_EXPR_LIST_DECL_SINGLE(SCL8, I2C8, I2C8_DESC);
+SS_PIN_DECL(P2, GPIOK6, SCL8);
+
+#define R1 87
+SIG_EXPR_LIST_DECL_SINGLE(SDA8, I2C8, I2C8_DESC);
+SS_PIN_DECL(R1, GPIOK7, SDA8);
+
+FUNC_GROUP_DECL(I2C8, P2, R1);
+
+#define VPIOFF0_DESC    { SCU90, GENMASK(5, 4), 0, 0 }
+#define VPIOFF1_DESC    { SCU90, GENMASK(5, 4), 1, 0 }
+#define VPI24_DESC      { SCU90, GENMASK(5, 4), 2, 0 }
+#define VPIRSVD_DESC    { SCU90, GENMASK(5, 4), 3, 0 }
+
+#define V2 104
+#define V2_DESC         SIG_DESC_SET(SCU88, 0)
+SIG_EXPR_LIST_DECL_SINGLE(DASHN0, DASHN0, VPIRSVD_DESC, V2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(PWM0, PWM0, V2_DESC, COND2);
+MS_PIN_DECL(V2, GPION0, DASHN0, PWM0);
+FUNC_GROUP_DECL(PWM0, V2);
+
+#define W2 105
+#define W2_DESC         SIG_DESC_SET(SCU88, 1)
+SIG_EXPR_LIST_DECL_SINGLE(DASHN1, DASHN1, VPIRSVD_DESC, W2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(PWM1, PWM1, W2_DESC, COND2);
+MS_PIN_DECL(W2, GPION1, DASHN1, PWM1);
+FUNC_GROUP_DECL(PWM1, W2);
+
+#define V3 106
+#define V3_DESC         SIG_DESC_SET(SCU88, 2)
+SIG_EXPR_DECL(VPIG2, VPI24, VPI24_DESC, V3_DESC, COND2);
+SIG_EXPR_DECL(VPIG2, VPIRSVD, VPIRSVD_DESC, V3_DESC, COND2);
+SIG_EXPR_LIST_DECL_DUAL(VPIG2, VPI24, VPIRSVD);
+SIG_EXPR_LIST_DECL_SINGLE(PWM2, PWM2, V3_DESC, COND2);
+MS_PIN_DECL(V3, GPION2, VPIG2, PWM2);
+FUNC_GROUP_DECL(PWM2, V3);
+
+#define U3 107
+#define U3_DESC         SIG_DESC_SET(SCU88, 3)
+SIG_EXPR_DECL(VPIG3, VPI24, VPI24_DESC, U3_DESC, COND2);
+SIG_EXPR_DECL(VPIG3, VPIRSVD, VPIRSVD_DESC, U3_DESC, COND2);
+SIG_EXPR_LIST_DECL_DUAL(VPIG3, VPI24, VPIRSVD);
+SIG_EXPR_LIST_DECL_SINGLE(PWM3, PWM3, U3_DESC, COND2);
+MS_PIN_DECL(U3, GPION3, VPIG3, PWM3);
+FUNC_GROUP_DECL(PWM3, U3);
+
+#define W3 108
+#define W3_DESC         SIG_DESC_SET(SCU88, 4)
+SIG_EXPR_DECL(VPIG4, VPI24, VPI24_DESC, W3_DESC, COND2);
+SIG_EXPR_DECL(VPIG4, VPIRSVD, VPIRSVD_DESC, W3_DESC, COND2);
+SIG_EXPR_LIST_DECL_DUAL(VPIG4, VPI24, VPIRSVD);
+SIG_EXPR_LIST_DECL_SINGLE(PWM4, PWM4, W3_DESC, COND2);
+MS_PIN_DECL(W3, GPION4, VPIG4, PWM4);
+FUNC_GROUP_DECL(PWM4, W3);
+
+#define AA3 109
+#define AA3_DESC        SIG_DESC_SET(SCU88, 5)
+SIG_EXPR_DECL(VPIG5, VPI24, VPI24_DESC, AA3_DESC, COND2);
+SIG_EXPR_DECL(VPIG5, VPIRSVD, VPIRSVD_DESC, AA3_DESC, COND2);
+SIG_EXPR_LIST_DECL_DUAL(VPIG5, VPI24, VPIRSVD);
+SIG_EXPR_LIST_DECL_SINGLE(PWM5, PWM5, AA3_DESC, COND2);
+MS_PIN_DECL(AA3, GPION5, VPIG5, PWM5);
+FUNC_GROUP_DECL(PWM5, AA3);
+
+#define Y3 110
+#define Y3_DESC         SIG_DESC_SET(SCU88, 6)
+SIG_EXPR_LIST_DECL_SINGLE(VPIG6, VPI24, VPI24_DESC, Y3_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(PWM6, PWM6, Y3_DESC, COND2);
+MS_PIN_DECL(Y3, GPION6, VPIG6, PWM6);
+FUNC_GROUP_DECL(PWM6, Y3);
+
+#define T4 111
+#define T4_DESC         SIG_DESC_SET(SCU88, 7)
+SIG_EXPR_LIST_DECL_SINGLE(VPIG7, VPI24, VPI24_DESC, T4_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(PWM7, PWM7, T4_DESC, COND2);
+MS_PIN_DECL(T4, GPION7, VPIG7, PWM7);
+FUNC_GROUP_DECL(PWM7, T4);
+
+#define V6 127
+SIG_EXPR_LIST_DECL_SINGLE(DASHV6, DASHV6, SIG_DESC_SET(SCU90, 28),
+		SIG_DESC_SET(SCU88, 23));
+SS_PIN_DECL(V6, GPIOP7, DASHV6);
+
+#define I2C3_DESC	SIG_DESC_SET(SCU90, 16)
+
+#define A11 128
+SIG_EXPR_LIST_DECL_SINGLE(SCL3, I2C3, I2C3_DESC);
+SS_PIN_DECL(A11, GPIOQ0, SCL3);
+
+#define A10 129
+SIG_EXPR_LIST_DECL_SINGLE(SDA3, I2C3, I2C3_DESC);
+SS_PIN_DECL(A10, GPIOQ1, SDA3);
+
+FUNC_GROUP_DECL(I2C3, A11, A10);
+
+#define I2C4_DESC	SIG_DESC_SET(SCU90, 17)
+
+#define A9 130
+SIG_EXPR_LIST_DECL_SINGLE(SCL4, I2C4, I2C4_DESC);
+SS_PIN_DECL(A9, GPIOQ2, SCL4);
+
+#define B9 131
+SIG_EXPR_LIST_DECL_SINGLE(SDA4, I2C4, I2C4_DESC);
+SS_PIN_DECL(B9, GPIOQ3, SDA4);
+
+FUNC_GROUP_DECL(I2C4, A9, B9);
+
+#define I2C14_DESC	SIG_DESC_SET(SCU90, 27)
+
+#define N21 132
+SIG_EXPR_LIST_DECL_SINGLE(SCL14, I2C14, I2C14_DESC);
+SS_PIN_DECL(N21, GPIOQ4, SCL14);
+
+#define N22 133
+SIG_EXPR_LIST_DECL_SINGLE(SDA14, I2C14, I2C14_DESC);
+SS_PIN_DECL(N22, GPIOQ5, SDA14);
+
+FUNC_GROUP_DECL(I2C14, N21, N22);
+
+#define B10 134
+SSSF_PIN_DECL(B10, GPIOQ6, OSCCLK, SIG_DESC_SET(SCU2C, 1));
+
+#define N20 135
+SSSF_PIN_DECL(N20, GPIOQ7, PEWAKE, SIG_DESC_SET(SCU2C, 29));
+
+#define D8 142
+SIG_EXPR_LIST_DECL_SINGLE(MDC1, MDIO1, SIG_DESC_SET(SCU88, 30));
+SS_PIN_DECL(D8, GPIOR6, MDC1);
+
+#define E10 143
+SIG_EXPR_LIST_DECL_SINGLE(MDIO1, MDIO1, SIG_DESC_SET(SCU88, 31));
+SS_PIN_DECL(E10, GPIOR7, MDIO1);
+
+FUNC_GROUP_DECL(MDIO1, D8, E10);
+
+/* RGMII1/RMII1 */
+
+#define RMII1_DESC      SIG_DESC_BIT(HW_STRAP1, 6, 0)
+#define RMII2_DESC      SIG_DESC_BIT(HW_STRAP1, 7, 0)
+
+#define B5 152
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT0, GPIOT0, SIG_DESC_SET(SCUA0, 0));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1RCLKO, RMII1, RMII1_DESC,
+		SIG_DESC_SET(SCU48, 29));
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXCK, RGMII1);
+MS_PIN_DECL_(B5, SIG_EXPR_LIST_PTR(GPIOT0), SIG_EXPR_LIST_PTR(RMII1RCLKO),
+		SIG_EXPR_LIST_PTR(RGMII1TXCK));
+
+#define E9 153
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT1, GPIOT1, SIG_DESC_SET(SCUA0, 1));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1TXEN, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXCTL, RGMII1);
+MS_PIN_DECL_(E9, SIG_EXPR_LIST_PTR(GPIOT1), SIG_EXPR_LIST_PTR(RMII1TXEN),
+		SIG_EXPR_LIST_PTR(RGMII1TXCTL));
+
+#define F9 154
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT2, GPIOT2, SIG_DESC_SET(SCUA0, 2));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1TXD0, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXD0, RGMII1);
+MS_PIN_DECL_(F9, SIG_EXPR_LIST_PTR(GPIOT2), SIG_EXPR_LIST_PTR(RMII1TXD0),
+		SIG_EXPR_LIST_PTR(RGMII1TXD0));
+
+#define A5 155
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT3, GPIOT3, SIG_DESC_SET(SCUA0, 3));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1TXD1, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXD1, RGMII1);
+MS_PIN_DECL_(A5, SIG_EXPR_LIST_PTR(GPIOT3), SIG_EXPR_LIST_PTR(RMII1TXD1),
+		SIG_EXPR_LIST_PTR(RGMII1TXD1));
+
+#define E7 156
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT4, GPIOT4, SIG_DESC_SET(SCUA0, 4));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1DASH0, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXD2, RGMII1);
+MS_PIN_DECL_(E7, SIG_EXPR_LIST_PTR(GPIOT4), SIG_EXPR_LIST_PTR(RMII1DASH0),
+		SIG_EXPR_LIST_PTR(RGMII1TXD2));
+
+#define D7 157
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT5, GPIOT5, SIG_DESC_SET(SCUA0, 5));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1DASH1, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1TXD3, RGMII1);
+MS_PIN_DECL_(D7, SIG_EXPR_LIST_PTR(GPIOT5), SIG_EXPR_LIST_PTR(RMII1DASH1),
+		SIG_EXPR_LIST_PTR(RGMII1TXD3));
+
+#define B2 158
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT6, GPIOT6, SIG_DESC_SET(SCUA0, 6));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2RCLKO, RMII2, RMII2_DESC,
+		SIG_DESC_SET(SCU48, 30));
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2TXCK, RGMII2);
+MS_PIN_DECL_(B2, SIG_EXPR_LIST_PTR(GPIOT6), SIG_EXPR_LIST_PTR(RMII2RCLKO),
+		SIG_EXPR_LIST_PTR(RGMII2TXCK));
+
+#define B1 159
+SIG_EXPR_LIST_DECL_SINGLE(GPIOT7, GPIOT7, SIG_DESC_SET(SCUA0, 7));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2TXEN, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2TXCTL, RGMII2);
+MS_PIN_DECL_(B1, SIG_EXPR_LIST_PTR(GPIOT7), SIG_EXPR_LIST_PTR(RMII2TXEN),
+		SIG_EXPR_LIST_PTR(RGMII2TXCTL));
+
+#define A2 160
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU0, GPIOU0, SIG_DESC_SET(SCUA0, 8));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2TXD0, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2TXD0, RGMII2);
+MS_PIN_DECL_(A2, SIG_EXPR_LIST_PTR(GPIOU0), SIG_EXPR_LIST_PTR(RMII2TXD0),
+		SIG_EXPR_LIST_PTR(RGMII2TXD0));
+
+#define B3 161
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU1, GPIOU1, SIG_DESC_SET(SCUA0, 9));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2TXD1, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2TXD1, RGMII2);
+MS_PIN_DECL_(B3, SIG_EXPR_LIST_PTR(GPIOU1), SIG_EXPR_LIST_PTR(RMII2TXD1),
+		SIG_EXPR_LIST_PTR(RGMII2TXD1));
+
+#define D5 162
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU2, GPIOU2, SIG_DESC_SET(SCUA0, 10));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2DASH0, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2TXD2, RGMII2);
+MS_PIN_DECL_(D5, SIG_EXPR_LIST_PTR(GPIOU2), SIG_EXPR_LIST_PTR(RMII2DASH0),
+		SIG_EXPR_LIST_PTR(RGMII2TXD2));
+
+#define D4 163
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU3, GPIOU3, SIG_DESC_SET(SCUA0, 11));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2DASH1, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2TXD3, RGMII2);
+MS_PIN_DECL_(D4, SIG_EXPR_LIST_PTR(GPIOU3), SIG_EXPR_LIST_PTR(RMII2DASH1),
+		SIG_EXPR_LIST_PTR(RGMII2TXD3));
+
+#define B4 164
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU4, GPIOU4, SIG_DESC_SET(SCUA0, 12));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1RCLKI, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXCK, RGMII1);
+MS_PIN_DECL_(B4, SIG_EXPR_LIST_PTR(GPIOU4), SIG_EXPR_LIST_PTR(RMII1RCLKI),
+		SIG_EXPR_LIST_PTR(RGMII1RXCK));
+
+#define A4 165
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU5, GPIOU5, SIG_DESC_SET(SCUA0, 13));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1DASH2, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXCTL, RGMII1);
+MS_PIN_DECL_(A4, SIG_EXPR_LIST_PTR(GPIOU5), SIG_EXPR_LIST_PTR(RMII1DASH2),
+		SIG_EXPR_LIST_PTR(RGMII1RXCTL));
+
+#define A3 166
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU6, GPIOU6, SIG_DESC_SET(SCUA0, 14));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1RXD0, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXD0, RGMII1);
+MS_PIN_DECL_(A3, SIG_EXPR_LIST_PTR(GPIOU6), SIG_EXPR_LIST_PTR(RMII1RXD0),
+		SIG_EXPR_LIST_PTR(RGMII1RXD0));
+
+#define D6 167
+SIG_EXPR_LIST_DECL_SINGLE(GPIOU7, GPIOU7, SIG_DESC_SET(SCUA0, 15));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1RXD1, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXD1, RGMII1);
+MS_PIN_DECL_(D6, SIG_EXPR_LIST_PTR(GPIOU7), SIG_EXPR_LIST_PTR(RMII1RXD1),
+		SIG_EXPR_LIST_PTR(RGMII1RXD1));
+
+#define C5 168
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV0, GPIOV0, SIG_DESC_SET(SCUA0, 16));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1CRSDV, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXD2, RGMII1);
+MS_PIN_DECL_(C5, SIG_EXPR_LIST_PTR(GPIOV0), SIG_EXPR_LIST_PTR(RMII1CRSDV),
+		SIG_EXPR_LIST_PTR(RGMII1RXD2));
+
+#define C4 169
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV1, GPIOV1, SIG_DESC_SET(SCUA0, 17));
+SIG_EXPR_LIST_DECL_SINGLE(RMII1RXER, RMII1, RMII1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII1RXD3, RGMII1);
+MS_PIN_DECL_(C4, SIG_EXPR_LIST_PTR(GPIOV1), SIG_EXPR_LIST_PTR(RMII1RXER),
+		SIG_EXPR_LIST_PTR(RGMII1RXD3));
+
+FUNC_GROUP_DECL(RGMII1, B4, A4, A3, D6, C5, C4, B5, E9, F9, A5, E7, D7);
+FUNC_GROUP_DECL(RMII1, B4, A3, D6, C5, C4, B5, E9, F9, A5);
+
+#define C2 170
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV2, GPIOV2, SIG_DESC_SET(SCUA0, 18));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2RCLKI, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2RXCK, RGMII2);
+MS_PIN_DECL_(C2, SIG_EXPR_LIST_PTR(GPIOV2), SIG_EXPR_LIST_PTR(RMII2RCLKI),
+		SIG_EXPR_LIST_PTR(RGMII2RXCK));
+
+#define C1 171
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV3, GPIOV3, SIG_DESC_SET(SCUA0, 19));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2DASH2, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2RXCTL, RGMII2);
+MS_PIN_DECL_(C1, SIG_EXPR_LIST_PTR(GPIOV3), SIG_EXPR_LIST_PTR(RMII2DASH2),
+		SIG_EXPR_LIST_PTR(RGMII2RXCTL));
+
+#define C3 172
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV4, GPIOV4, SIG_DESC_SET(SCUA0, 20));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2RXD0, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2RXD0, RGMII2);
+MS_PIN_DECL_(C3, SIG_EXPR_LIST_PTR(GPIOV4), SIG_EXPR_LIST_PTR(RMII2RXD0),
+		SIG_EXPR_LIST_PTR(RGMII2RXD0));
+
+#define D1 173
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV5, GPIOV5, SIG_DESC_SET(SCUA0, 21));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2RXD1, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2RXD1, RGMII2);
+MS_PIN_DECL_(D1, SIG_EXPR_LIST_PTR(GPIOV5), SIG_EXPR_LIST_PTR(RMII2RXD1),
+		SIG_EXPR_LIST_PTR(RGMII2RXD1));
+
+#define D2 174
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV6, GPIOV6, SIG_DESC_SET(SCUA0, 22));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2CRSDV, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2RXD2, RGMII2);
+MS_PIN_DECL_(D2, SIG_EXPR_LIST_PTR(GPIOV6), SIG_EXPR_LIST_PTR(RMII2CRSDV),
+		SIG_EXPR_LIST_PTR(RGMII2RXD2));
+
+#define E6 175
+SIG_EXPR_LIST_DECL_SINGLE(GPIOV7, GPIOV7, SIG_DESC_SET(SCUA0, 23));
+SIG_EXPR_LIST_DECL_SINGLE(RMII2RXER, RMII2, RMII2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(RGMII2RXD3, RGMII2);
+MS_PIN_DECL_(E6, SIG_EXPR_LIST_PTR(GPIOV7), SIG_EXPR_LIST_PTR(RMII2RXER),
+		SIG_EXPR_LIST_PTR(RGMII2RXD3));
+
+FUNC_GROUP_DECL(RGMII2, B2, B1, A2, B3, D5, D4, C2, C1, C3, D1, D2, E6);
+FUNC_GROUP_DECL(RMII2, B2, B1, A2, B3, C2, C3, D1, D2, E6);
+
+/* Pins, groups and functions are sort(1):ed alphabetically for sanity */
+
+static struct pinctrl_pin_desc aspeed_g5_pins[ASPEED_G5_NR_PINS] = {
+	ASPEED_PINCTRL_PIN(A10),
+	ASPEED_PINCTRL_PIN(A11),
+	ASPEED_PINCTRL_PIN(A12),
+	ASPEED_PINCTRL_PIN(A13),
+	ASPEED_PINCTRL_PIN(A14),
+	ASPEED_PINCTRL_PIN(A2),
+	ASPEED_PINCTRL_PIN(A3),
+	ASPEED_PINCTRL_PIN(A4),
+	ASPEED_PINCTRL_PIN(A5),
+	ASPEED_PINCTRL_PIN(A9),
+	ASPEED_PINCTRL_PIN(AA3),
+	ASPEED_PINCTRL_PIN(B1),
+	ASPEED_PINCTRL_PIN(B10),
+	ASPEED_PINCTRL_PIN(B11),
+	ASPEED_PINCTRL_PIN(B12),
+	ASPEED_PINCTRL_PIN(B13),
+	ASPEED_PINCTRL_PIN(B14),
+	ASPEED_PINCTRL_PIN(B2),
+	ASPEED_PINCTRL_PIN(B20),
+	ASPEED_PINCTRL_PIN(B3),
+	ASPEED_PINCTRL_PIN(B4),
+	ASPEED_PINCTRL_PIN(B5),
+	ASPEED_PINCTRL_PIN(B9),
+	ASPEED_PINCTRL_PIN(C1),
+	ASPEED_PINCTRL_PIN(C11),
+	ASPEED_PINCTRL_PIN(C12),
+	ASPEED_PINCTRL_PIN(C13),
+	ASPEED_PINCTRL_PIN(C14),
+	ASPEED_PINCTRL_PIN(C16),
+	ASPEED_PINCTRL_PIN(C18),
+	ASPEED_PINCTRL_PIN(C2),
+	ASPEED_PINCTRL_PIN(C20),
+	ASPEED_PINCTRL_PIN(C3),
+	ASPEED_PINCTRL_PIN(C4),
+	ASPEED_PINCTRL_PIN(C5),
+	ASPEED_PINCTRL_PIN(D1),
+	ASPEED_PINCTRL_PIN(D10),
+	ASPEED_PINCTRL_PIN(D2),
+	ASPEED_PINCTRL_PIN(D20),
+	ASPEED_PINCTRL_PIN(D21),
+	ASPEED_PINCTRL_PIN(D4),
+	ASPEED_PINCTRL_PIN(D5),
+	ASPEED_PINCTRL_PIN(D6),
+	ASPEED_PINCTRL_PIN(D7),
+	ASPEED_PINCTRL_PIN(D8),
+	ASPEED_PINCTRL_PIN(D9),
+	ASPEED_PINCTRL_PIN(E10),
+	ASPEED_PINCTRL_PIN(E12),
+	ASPEED_PINCTRL_PIN(E13),
+	ASPEED_PINCTRL_PIN(E15),
+	ASPEED_PINCTRL_PIN(E21),
+	ASPEED_PINCTRL_PIN(E6),
+	ASPEED_PINCTRL_PIN(E7),
+	ASPEED_PINCTRL_PIN(E9),
+	ASPEED_PINCTRL_PIN(F19),
+	ASPEED_PINCTRL_PIN(F9),
+	ASPEED_PINCTRL_PIN(H20),
+	ASPEED_PINCTRL_PIN(L1),
+	ASPEED_PINCTRL_PIN(L2),
+	ASPEED_PINCTRL_PIN(L3),
+	ASPEED_PINCTRL_PIN(L4),
+	ASPEED_PINCTRL_PIN(N1),
+	ASPEED_PINCTRL_PIN(N2),
+	ASPEED_PINCTRL_PIN(N20),
+	ASPEED_PINCTRL_PIN(N21),
+	ASPEED_PINCTRL_PIN(N22),
+	ASPEED_PINCTRL_PIN(N3),
+	ASPEED_PINCTRL_PIN(N4),
+	ASPEED_PINCTRL_PIN(P1),
+	ASPEED_PINCTRL_PIN(P2),
+	ASPEED_PINCTRL_PIN(R1),
+	ASPEED_PINCTRL_PIN(T4),
+	ASPEED_PINCTRL_PIN(U3),
+	ASPEED_PINCTRL_PIN(V2),
+	ASPEED_PINCTRL_PIN(V3),
+	ASPEED_PINCTRL_PIN(V6),
+	ASPEED_PINCTRL_PIN(W2),
+	ASPEED_PINCTRL_PIN(W3),
+	ASPEED_PINCTRL_PIN(Y3),
+};
+
+static const struct aspeed_pin_group aspeed_g5_groups[] = {
+	ASPEED_PINCTRL_GROUP(GPID0),
+	ASPEED_PINCTRL_GROUP(GPID2),
+	ASPEED_PINCTRL_GROUP(GPIE0),
+	ASPEED_PINCTRL_GROUP(I2C10),
+	ASPEED_PINCTRL_GROUP(I2C11),
+	ASPEED_PINCTRL_GROUP(I2C12),
+	ASPEED_PINCTRL_GROUP(I2C13),
+	ASPEED_PINCTRL_GROUP(I2C14),
+	ASPEED_PINCTRL_GROUP(I2C3),
+	ASPEED_PINCTRL_GROUP(I2C4),
+	ASPEED_PINCTRL_GROUP(I2C5),
+	ASPEED_PINCTRL_GROUP(I2C6),
+	ASPEED_PINCTRL_GROUP(I2C7),
+	ASPEED_PINCTRL_GROUP(I2C8),
+	ASPEED_PINCTRL_GROUP(I2C9),
+	ASPEED_PINCTRL_GROUP(MAC1LINK),
+	ASPEED_PINCTRL_GROUP(MDIO1),
+	ASPEED_PINCTRL_GROUP(MDIO2),
+	ASPEED_PINCTRL_GROUP(OSCCLK),
+	ASPEED_PINCTRL_GROUP(PEWAKE),
+	ASPEED_PINCTRL_GROUP(PWM0),
+	ASPEED_PINCTRL_GROUP(PWM1),
+	ASPEED_PINCTRL_GROUP(PWM2),
+	ASPEED_PINCTRL_GROUP(PWM3),
+	ASPEED_PINCTRL_GROUP(PWM4),
+	ASPEED_PINCTRL_GROUP(PWM5),
+	ASPEED_PINCTRL_GROUP(PWM6),
+	ASPEED_PINCTRL_GROUP(PWM7),
+	ASPEED_PINCTRL_GROUP(RGMII1),
+	ASPEED_PINCTRL_GROUP(RGMII2),
+	ASPEED_PINCTRL_GROUP(RMII1),
+	ASPEED_PINCTRL_GROUP(RMII2),
+	ASPEED_PINCTRL_GROUP(SD1),
+	ASPEED_PINCTRL_GROUP(SPI1),
+	ASPEED_PINCTRL_GROUP(TIMER4),
+	ASPEED_PINCTRL_GROUP(TIMER5),
+	ASPEED_PINCTRL_GROUP(TIMER6),
+	ASPEED_PINCTRL_GROUP(TIMER7),
+	ASPEED_PINCTRL_GROUP(TIMER8),
+};
+
+static const struct aspeed_pin_function aspeed_g5_functions[] = {
+	ASPEED_PINCTRL_FUNC(GPID0),
+	ASPEED_PINCTRL_FUNC(GPID2),
+	ASPEED_PINCTRL_FUNC(GPIE0),
+	ASPEED_PINCTRL_FUNC(I2C10),
+	ASPEED_PINCTRL_FUNC(I2C11),
+	ASPEED_PINCTRL_FUNC(I2C12),
+	ASPEED_PINCTRL_FUNC(I2C13),
+	ASPEED_PINCTRL_FUNC(I2C14),
+	ASPEED_PINCTRL_FUNC(I2C3),
+	ASPEED_PINCTRL_FUNC(I2C4),
+	ASPEED_PINCTRL_FUNC(I2C5),
+	ASPEED_PINCTRL_FUNC(I2C6),
+	ASPEED_PINCTRL_FUNC(I2C7),
+	ASPEED_PINCTRL_FUNC(I2C8),
+	ASPEED_PINCTRL_FUNC(I2C9),
+	ASPEED_PINCTRL_FUNC(MAC1LINK),
+	ASPEED_PINCTRL_FUNC(MDIO1),
+	ASPEED_PINCTRL_FUNC(MDIO2),
+	ASPEED_PINCTRL_FUNC(OSCCLK),
+	ASPEED_PINCTRL_FUNC(PEWAKE),
+	ASPEED_PINCTRL_FUNC(PWM0),
+	ASPEED_PINCTRL_FUNC(PWM1),
+	ASPEED_PINCTRL_FUNC(PWM2),
+	ASPEED_PINCTRL_FUNC(PWM3),
+	ASPEED_PINCTRL_FUNC(PWM4),
+	ASPEED_PINCTRL_FUNC(PWM5),
+	ASPEED_PINCTRL_FUNC(PWM6),
+	ASPEED_PINCTRL_FUNC(PWM7),
+	ASPEED_PINCTRL_FUNC(RGMII1),
+	ASPEED_PINCTRL_FUNC(RGMII2),
+	ASPEED_PINCTRL_FUNC(RMII1),
+	ASPEED_PINCTRL_FUNC(RMII2),
+	ASPEED_PINCTRL_FUNC(SD1),
+	ASPEED_PINCTRL_FUNC(SPI1),
+	ASPEED_PINCTRL_FUNC(TIMER4),
+	ASPEED_PINCTRL_FUNC(TIMER5),
+	ASPEED_PINCTRL_FUNC(TIMER6),
+	ASPEED_PINCTRL_FUNC(TIMER7),
+	ASPEED_PINCTRL_FUNC(TIMER8),
+};
+
+static struct aspeed_pinctrl_data aspeed_g5_pinctrl_data = {
+	.pins = aspeed_g5_pins,
+	.npins = ARRAY_SIZE(aspeed_g5_pins),
+	.groups = aspeed_g5_groups,
+	.ngroups = ARRAY_SIZE(aspeed_g5_groups),
+	.functions = aspeed_g5_functions,
+	.nfunctions = ARRAY_SIZE(aspeed_g5_functions),
+};
+
+static struct pinmux_ops aspeed_g5_pinmux_ops = {
+	.get_functions_count = aspeed_pinmux_get_fn_count,
+	.get_function_name = aspeed_pinmux_get_fn_name,
+	.get_function_groups = aspeed_pinmux_get_fn_groups,
+	.set_mux = aspeed_pinmux_set_mux,
+	.gpio_request_enable = aspeed_gpio_request_enable,
+	.strict = true,
+};
+
+static struct pinctrl_ops aspeed_g5_pinctrl_ops = {
+	.get_groups_count = aspeed_pinctrl_get_groups_count,
+	.get_group_name = aspeed_pinctrl_get_group_name,
+	.get_group_pins = aspeed_pinctrl_get_group_pins,
+	.pin_dbg_show = aspeed_pinctrl_pin_dbg_show,
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_pin,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+static struct pinctrl_desc aspeed_g5_pinctrl_desc = {
+	.name = "aspeed-g5-pinctrl",
+	.pins = aspeed_g5_pins,
+	.npins = ARRAY_SIZE(aspeed_g5_pins),
+	.pctlops = &aspeed_g5_pinctrl_ops,
+	.pmxops = &aspeed_g5_pinmux_ops,
+};
+
+static int aspeed_g5_pinctrl_probe(struct platform_device *pdev)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(aspeed_g5_pins); i++)
+		aspeed_g5_pins[i].number = i;
+
+	return aspeed_pinctrl_probe(pdev, &aspeed_g5_pinctrl_desc,
+			&aspeed_g5_pinctrl_data);
+}
+
+static const struct of_device_id aspeed_g5_pinctrl_of_match[] = {
+	{ .compatible = "aspeed,ast2500-pinctrl", },
+	{ .compatible = "aspeed,g5-pinctrl", },
+	{ },
+};
+
+static struct platform_driver aspeed_g5_pinctrl_driver = {
+	.probe = aspeed_g5_pinctrl_probe,
+	.driver = {
+		.name = "aspeed-g5-pinctrl",
+		.of_match_table = aspeed_g5_pinctrl_of_match,
+	},
+};
+
+static int aspeed_g5_pinctrl_init(void)
+{
+	return platform_driver_register(&aspeed_g5_pinctrl_driver);
+}
+
+arch_initcall(aspeed_g5_pinctrl_init);
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed.c b/drivers/pinctrl/aspeed/pinctrl-aspeed.c
new file mode 100644
index 000000000000..0391f9f13f3e
--- /dev/null
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed.c
@@ -0,0 +1,498 @@
+/*
+ * Copyright (C) 2016 IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/mfd/syscon.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include "../core.h"
+#include "pinctrl-aspeed.h"
+
+int aspeed_pinctrl_get_groups_count(struct pinctrl_dev *pctldev)
+{
+	struct aspeed_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev);
+
+	return pdata->ngroups;
+}
+
+const char *aspeed_pinctrl_get_group_name(struct pinctrl_dev *pctldev,
+		unsigned int group)
+{
+	struct aspeed_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev);
+
+	return pdata->groups[group].name;
+}
+
+int aspeed_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
+				  unsigned int group, const unsigned int **pins,
+				  unsigned int *npins)
+{
+	struct aspeed_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev);
+
+	*pins = &pdata->groups[group].pins[0];
+	*npins = pdata->groups[group].npins;
+
+	return 0;
+}
+
+void aspeed_pinctrl_pin_dbg_show(struct pinctrl_dev *pctldev,
+				 struct seq_file *s, unsigned int offset)
+{
+	seq_printf(s, " %s", dev_name(pctldev->dev));
+}
+
+int aspeed_pinmux_get_fn_count(struct pinctrl_dev *pctldev)
+{
+	struct aspeed_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev);
+
+	return pdata->nfunctions;
+}
+
+const char *aspeed_pinmux_get_fn_name(struct pinctrl_dev *pctldev,
+				      unsigned int function)
+{
+	struct aspeed_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev);
+
+	return pdata->functions[function].name;
+}
+
+int aspeed_pinmux_get_fn_groups(struct pinctrl_dev *pctldev,
+				unsigned int function,
+				const char * const **groups,
+				unsigned int * const num_groups)
+{
+	struct aspeed_pinctrl_data *pdata = pinctrl_dev_get_drvdata(pctldev);
+
+	*groups = pdata->functions[function].groups;
+	*num_groups = pdata->functions[function].ngroups;
+
+	return 0;
+}
+
+static inline void aspeed_sig_desc_print_val(
+		const struct aspeed_sig_desc *desc, bool enable, u32 rv)
+{
+	pr_debug("SCU%x[0x%08x]=0x%x, got 0x%x from 0x%08x\n", desc->reg,
+			desc->mask, enable ? desc->enable : desc->disable,
+			(rv & desc->mask) >> __ffs(desc->mask), rv);
+}
+
+/**
+ * Query the enabled or disabled state of a signal descriptor
+ *
+ * @desc: The signal descriptor of interest
+ * @enabled: True to query the enabled state, false to query disabled state
+ * @regmap: The SCU regmap instance
+ *
+ * @return True if the descriptor's bitfield is configured to the state
+ * selected by @enabled, false otherwise
+ *
+ * Evaluation of descriptor state is non-trivial in that it is not a binary
+ * outcome: The bitfields can be greater than one bit in size and thus can take
+ * a value that is neither the enabled nor disabled state recorded in the
+ * descriptor (typically this means a different function to the one of interest
+ * is enabled). Thus we must explicitly test for either condition as required.
+ */
+static bool aspeed_sig_desc_eval(const struct aspeed_sig_desc *desc,
+				 bool enabled, struct regmap *map)
+{
+	unsigned int raw;
+	u32 want;
+
+	if (regmap_read(map, desc->reg, &raw) < 0)
+		return false;
+
+	aspeed_sig_desc_print_val(desc, enabled, raw);
+	want = enabled ? desc->enable : desc->disable;
+
+	return ((raw & desc->mask) >> __ffs(desc->mask)) == want;
+}
+
+/**
+ * Query the enabled or disabled state for a mux function's signal on a pin
+ *
+ * @expr: An expression controlling the signal for a mux function on a pin
+ * @enabled: True to query the enabled state, false to query disabled state
+ * @regmap: The SCU regmap instance
+ *
+ * @return True if the expression composed by @enabled evaluates true, false
+ * otherwise
+ *
+ * A mux function is enabled or disabled if the function's signal expression
+ * for each pin in the function's pin group evaluates true for the desired
+ * state. An signal expression evaluates true if all of its associated signal
+ * descriptors evaluate true for the desired state.
+ *
+ * If an expression's state is described by more than one bit, either through
+ * multi-bit bitfields in a single signal descriptor or through multiple signal
+ * descriptors of a single bit then it is possible for the expression to be in
+ * neither the enabled nor disabled state. Thus we must explicitly test for
+ * either condition as required.
+ */
+static bool aspeed_sig_expr_eval(const struct aspeed_sig_expr *expr,
+				 bool enabled, struct regmap *map)
+{
+	int i;
+
+	for (i = 0; i < expr->ndescs; i++) {
+		const struct aspeed_sig_desc *desc = &expr->descs[i];
+
+		if (!aspeed_sig_desc_eval(desc, enabled, map))
+			return false;
+	}
+
+	return true;
+}
+
+/**
+ * Configure a pin's signal by applying an expression's descriptor state for
+ * all descriptors in the expression.
+ *
+ * @expr: The expression associated with the function whose signal is to be
+ *        configured
+ * @enable: true to enable an function's signal through a pin's signal
+ *          expression, false to disable the function's signal
+ * @map: The SCU's regmap instance for pinmux register access.
+ *
+ * @return true if the expression is configured as requested, false otherwise
+ */
+static bool aspeed_sig_expr_set(const struct aspeed_sig_expr *expr,
+				bool enable, struct regmap *map)
+{
+	int i;
+	bool ret;
+
+	ret = aspeed_sig_expr_eval(expr, enable, map);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < expr->ndescs; i++) {
+		const struct aspeed_sig_desc *desc = &expr->descs[i];
+		u32 pattern = enable ? desc->enable : desc->disable;
+
+		/*
+		 * Strap registers are configured in hardware or by early-boot
+		 * firmware. Treat them as read-only despite that we can write
+		 * them. This may mean that certain functions cannot be
+		 * deconfigured and is the reason we re-evaluate after writing
+		 * all descriptor bits.
+		 */
+		if (desc->reg == HW_STRAP1 || desc->reg == HW_STRAP2)
+			continue;
+
+		ret = regmap_update_bits(map, desc->reg, desc->mask,
+				pattern << __ffs(desc->mask)) == 0;
+
+		if (!ret)
+			return ret;
+	}
+
+	return aspeed_sig_expr_eval(expr, enable, map);
+}
+
+static bool aspeed_sig_expr_enable(const struct aspeed_sig_expr *expr,
+				   struct regmap *map)
+{
+	return aspeed_sig_expr_set(expr, true, map);
+}
+
+static bool aspeed_sig_expr_disable(const struct aspeed_sig_expr *expr,
+				    struct regmap *map)
+{
+	return aspeed_sig_expr_set(expr, false, map);
+}
+
+/**
+ * Disable a signal on a pin by disabling all provided signal expressions.
+ *
+ * @exprs: The list of signal expressions (from a priority level on a pin)
+ * @map: The SCU's regmap instance for pinmux register access.
+ *
+ * @return true if all expressions in the list are successfully disabled, false
+ * otherwise
+ */
+static bool aspeed_disable_sig(const struct aspeed_sig_expr **exprs,
+			       struct regmap *map)
+{
+	bool disabled = true;
+
+	if (!exprs)
+		return true;
+
+	while (*exprs) {
+		bool ret;
+
+		ret = aspeed_sig_expr_disable(*exprs, map);
+		disabled = disabled && ret;
+
+		exprs++;
+	}
+
+	return disabled;
+}
+
+/**
+ * Search for the signal expression needed to enable the pin's signal for the
+ * requested function.
+ *
+ * @exprs: List of signal expressions (haystack)
+ * @name: The name of the requested function (needle)
+ *
+ * @return A pointer to the signal expression whose function tag matches the
+ *         provided name, otherwise NULL.
+ *
+ */
+static const struct aspeed_sig_expr *aspeed_find_expr_by_name(
+		const struct aspeed_sig_expr **exprs, const char *name)
+{
+	while (*exprs) {
+		if (strcmp((*exprs)->function, name) == 0)
+			return *exprs;
+		exprs++;
+	}
+
+	return NULL;
+}
+
+static char *get_defined_attribute(const struct aspeed_pin_desc *pdesc,
+				   const char *(*get)(
+					   const struct aspeed_sig_expr *))
+{
+	char *found = NULL;
+	size_t len = 0;
+	const struct aspeed_sig_expr ***prios, **funcs, *expr;
+
+	prios = pdesc->prios;
+
+	while ((funcs = *prios)) {
+		while ((expr = *funcs)) {
+			const char *str = get(expr);
+			size_t delta = strlen(str) + 2;
+			char *expanded;
+
+			expanded = krealloc(found, len + delta + 1, GFP_KERNEL);
+			if (!expanded) {
+				kfree(found);
+				return expanded;
+			}
+
+			found = expanded;
+			found[len] = '\0';
+			len += delta;
+
+			strcat(found, str);
+			strcat(found, ", ");
+
+			funcs++;
+		}
+		prios++;
+	}
+
+	if (len < 2) {
+		kfree(found);
+		return NULL;
+	}
+
+	found[len - 2] = '\0';
+
+	return found;
+}
+
+static const char *aspeed_sig_expr_function(const struct aspeed_sig_expr *expr)
+{
+	return expr->function;
+}
+
+static char *get_defined_functions(const struct aspeed_pin_desc *pdesc)
+{
+	return get_defined_attribute(pdesc, aspeed_sig_expr_function);
+}
+
+static const char *aspeed_sig_expr_signal(const struct aspeed_sig_expr *expr)
+{
+	return expr->signal;
+}
+
+static char *get_defined_signals(const struct aspeed_pin_desc *pdesc)
+{
+	return get_defined_attribute(pdesc, aspeed_sig_expr_signal);
+}
+
+int aspeed_pinmux_set_mux(struct pinctrl_dev *pctldev, unsigned int function,
+			  unsigned int group)
+{
+	int i;
+	const struct aspeed_pinctrl_data *pdata =
+		pinctrl_dev_get_drvdata(pctldev);
+	const struct aspeed_pin_group *pgroup = &pdata->groups[group];
+	const struct aspeed_pin_function *pfunc =
+		&pdata->functions[function];
+
+	for (i = 0; i < pgroup->npins; i++) {
+		int pin = pgroup->pins[i];
+		const struct aspeed_pin_desc *pdesc = pdata->pins[pin].drv_data;
+		const struct aspeed_sig_expr *expr = NULL;
+		const struct aspeed_sig_expr **funcs;
+		const struct aspeed_sig_expr ***prios;
+
+		if (!pdesc)
+			return -EINVAL;
+
+		prios = pdesc->prios;
+
+		if (!prios)
+			continue;
+
+		/* Disable functions at a higher priority than that requested */
+		while ((funcs = *prios)) {
+			expr = aspeed_find_expr_by_name(funcs, pfunc->name);
+
+			if (expr)
+				break;
+
+			if (!aspeed_disable_sig(funcs, pdata->map))
+				return -EPERM;
+
+			prios++;
+		}
+
+		if (!expr) {
+			char *functions = get_defined_functions(pdesc);
+			char *signals = get_defined_signals(pdesc);
+
+			pr_warn("No function %s found on pin %s (%d). Found signal(s) %s for function(s) %s\n",
+				pfunc->name, pdesc->name, pin, signals,
+				functions);
+			kfree(signals);
+			kfree(functions);
+
+			return -ENXIO;
+		}
+
+		if (!aspeed_sig_expr_enable(expr, pdata->map))
+			return -EPERM;
+	}
+
+	return 0;
+}
+
+static bool aspeed_expr_is_gpio(const struct aspeed_sig_expr *expr)
+{
+	/*
+	 * The signal type is GPIO if the signal name has "GPIO" as a prefix.
+	 * strncmp (rather than strcmp) is used to implement the prefix
+	 * requirement.
+	 *
+	 * expr->signal might look like "GPIOT3" in the GPIO case.
+	 */
+	return strncmp(expr->signal, "GPIO", 4) == 0;
+}
+
+static bool aspeed_gpio_in_exprs(const struct aspeed_sig_expr **exprs)
+{
+	if (!exprs)
+		return false;
+
+	while (*exprs) {
+		if (aspeed_expr_is_gpio(*exprs))
+			return true;
+		exprs++;
+	}
+
+	return false;
+}
+
+int aspeed_gpio_request_enable(struct pinctrl_dev *pctldev,
+			       struct pinctrl_gpio_range *range,
+			       unsigned int offset)
+{
+	const struct aspeed_pinctrl_data *pdata =
+		pinctrl_dev_get_drvdata(pctldev);
+	const struct aspeed_pin_desc *pdesc = pdata->pins[offset].drv_data;
+	const struct aspeed_sig_expr ***prios, **funcs, *expr;
+
+	if (!pdesc)
+		return -EINVAL;
+
+	prios = pdesc->prios;
+
+	if (!prios)
+		return -ENXIO;
+
+	/* Disable any functions of higher priority than GPIO */
+	while ((funcs = *prios)) {
+		if (aspeed_gpio_in_exprs(funcs))
+			break;
+
+		if (!aspeed_disable_sig(funcs, pdata->map))
+			return -EPERM;
+
+		prios++;
+	}
+
+	if (!funcs) {
+		char *signals = get_defined_signals(pdesc);
+
+		pr_warn("No GPIO signal type found on pin %s (%d). Found: %s\n",
+			pdesc->name, offset, signals);
+		kfree(signals);
+
+		return -ENXIO;
+	}
+
+	expr = *funcs;
+
+	/*
+	 * Disabling all higher-priority expressions is enough to enable the
+	 * lowest-priority signal type. As such it has no associated
+	 * expression.
+	 */
+	if (!expr)
+		return 0;
+
+	/*
+	 * If GPIO is not the lowest priority signal type, assume there is only
+	 * one expression defined to enable the GPIO function
+	 */
+	if (!aspeed_sig_expr_enable(expr, pdata->map))
+		return -EPERM;
+
+	return 0;
+}
+
+int aspeed_pinctrl_probe(struct platform_device *pdev,
+			 struct pinctrl_desc *pdesc,
+			 struct aspeed_pinctrl_data *pdata)
+{
+	struct device *parent;
+	struct pinctrl_dev *pctl;
+
+	parent = pdev->dev.parent;
+	if (!parent) {
+		dev_err(&pdev->dev, "No parent for syscon pincontroller\n");
+		return -ENODEV;
+	}
+
+	pdata->map = syscon_node_to_regmap(parent->of_node);
+	if (IS_ERR(pdata->map)) {
+		dev_err(&pdev->dev, "No regmap for syscon pincontroller parent\n");
+		return PTR_ERR(pdata->map);
+	}
+
+	pctl = pinctrl_register(pdesc, &pdev->dev, pdata);
+
+	if (IS_ERR(pctl)) {
+		dev_err(&pdev->dev, "Failed to register pinctrl\n");
+		return PTR_ERR(pctl);
+	}
+
+	platform_set_drvdata(pdev, pdata);
+
+	return 0;
+}
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed.h b/drivers/pinctrl/aspeed/pinctrl-aspeed.h
new file mode 100644
index 000000000000..3e72ef8c54bf
--- /dev/null
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed.h
@@ -0,0 +1,569 @@
+/*
+ * Copyright (C) 2016 IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef PINCTRL_ASPEED
+#define PINCTRL_ASPEED
+
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/regmap.h>
+
+/*
+ * The ASPEED SoCs provide typically more than 200 pins for GPIO and other
+ * functions. The SoC function enabled on a pin is determined on a priority
+ * basis where a given pin can provide a number of different signal types.
+ *
+ * The signal active on a pin is described by both a priority level and
+ * compound logical expressions involving multiple operators, registers and
+ * bits. Some difficulty arises as the pin's function bit masks for each
+ * priority level are frequently not the same (i.e. cannot just flip a bit to
+ * change from a high to low priority signal), or even in the same register.
+ * Further, not all signals can be unmuxed, as some expressions depend on
+ * values in the hardware strapping register (which is treated as read-only).
+ *
+ * SoC Multi-function Pin Expression Examples
+ * ------------------------------------------
+ *
+ * Here are some sample mux configurations from the AST2400 and AST2500
+ * datasheets to illustrate the corner cases, roughly in order of least to most
+ * corner. The signal priorities are in decending order from P0 (highest).
+ *
+ * D6 is a pin with a single function (beside GPIO); a high priority signal
+ * that participates in one function:
+ *
+ * Ball | Default | P0 Signal | P0 Expression               | P1 Signal | P1 Expression | Other
+ * -----+---------+-----------+-----------------------------+-----------+---------------+----------
+ *  D6    GPIOA0    MAC1LINK    SCU80[0]=1                                                GPIOA0
+ * -----+---------+-----------+-----------------------------+-----------+---------------+----------
+ *
+ * C5 is a multi-signal pin (high and low priority signals). Here we touch
+ * different registers for the different functions that enable each signal:
+ *
+ * -----+---------+-----------+-----------------------------+-----------+---------------+----------
+ *  C5    GPIOA4    SCL9        SCU90[22]=1                   TIMER5      SCU80[4]=1      GPIOA4
+ * -----+---------+-----------+-----------------------------+-----------+---------------+----------
+ *
+ * E19 is a single-signal pin with two functions that influence the active
+ * signal. In this case both bits have the same meaning - enable a dedicated
+ * LPC reset pin. However it's not always the case that the bits in the
+ * OR-relationship have the same meaning.
+ *
+ * -----+---------+-----------+-----------------------------+-----------+---------------+----------
+ *  E19   GPIOB4    LPCRST#     SCU80[12]=1 | Strap[14]=1                                 GPIOB4
+ * -----+---------+-----------+-----------------------------+-----------+---------------+----------
+ *
+ * For example, pin B19 has a low-priority signal that's enabled by two
+ * distinct SoC functions: A specific SIOPBI bit in register SCUA4, and an ACPI
+ * bit in the STRAP register. The ACPI bit configures signals on pins in
+ * addition to B19. Both of the low priority functions as well as the high
+ * priority function must be disabled for GPIOF1 to be used.
+ *
+ * Ball | Default | P0 Signal | P0 Expression                           | P1 Signal | P1 Expression                          | Other
+ * -----+---------+-----------+-----------------------------------------+-----------+----------------------------------------+----------
+ *  B19   GPIOF1    NDCD4       SCU80[25]=1                               SIOPBI#     SCUA4[12]=1 | Strap[19]=0                GPIOF1
+ * -----+---------+-----------+-----------------------------------------+-----------+----------------------------------------+----------
+ *
+ * For pin E18, the SoC ANDs the expected state of three bits to determine the
+ * pin's active signal:
+ *
+ * * SCU3C[3]: Enable external SOC reset function
+ * * SCU80[15]: Enable SPICS1# or EXTRST# function pin
+ * * SCU90[31]: Select SPI interface CS# output
+ *
+ * -----+---------+-----------+-----------------------------------------+-----------+----------------------------------------+----------
+ *  E18   GPIOB7    EXTRST#     SCU3C[3]=1 & SCU80[15]=1 & SCU90[31]=0    SPICS1#     SCU3C[3]=1 & SCU80[15]=1 & SCU90[31]=1   GPIOB7
+ * -----+---------+-----------+-----------------------------------------+-----------+----------------------------------------+----------
+ *
+ * (Bits SCU3C[3] and SCU80[15] appear to only be used in the expressions for
+ * selecting the signals on pin E18)
+ *
+ * Pin T5 is a multi-signal pin with a more complex configuration:
+ *
+ * Ball | Default | P0 Signal | P0 Expression                | P1 Signal | P1 Expression | Other
+ * -----+---------+-----------+------------------------------+-----------+---------------+----------
+ *  T5    GPIOL1    VPIDE       SCU90[5:4]!=0 & SCU84[17]=1    NDCD1       SCU84[17]=1     GPIOL1
+ * -----+---------+-----------+------------------------------+-----------+---------------+----------
+ *
+ * The high priority signal configuration is best thought of in terms of its
+ * exploded form, with reference to the SCU90[5:4] bits:
+ *
+ * * SCU90[5:4]=00: disable
+ * * SCU90[5:4]=01: 18 bits (R6/G6/B6) video mode.
+ * * SCU90[5:4]=10: 24 bits (R8/G8/B8) video mode.
+ * * SCU90[5:4]=11: 30 bits (R10/G10/B10) video mode.
+ *
+ * Re-writing:
+ *
+ * -----+---------+-----------+------------------------------+-----------+---------------+----------
+ *  T5    GPIOL1    VPIDE      (SCU90[5:4]=1 & SCU84[17]=1)    NDCD1       SCU84[17]=1     GPIOL1
+ *                             | (SCU90[5:4]=2 & SCU84[17]=1)
+ *                             | (SCU90[5:4]=3 & SCU84[17]=1)
+ * -----+---------+-----------+------------------------------+-----------+---------------+----------
+ *
+ * For reference the SCU84[17] bit configure the "UART1 NDCD1 or Video VPIDE
+ * function pin", where the signal itself is determined by whether SCU94[5:4]
+ * is disabled or in one of the 18, 24 or 30bit video modes.
+ *
+ * Other video-input-related pins require an explicit state in SCU90[5:4], e.g.
+ * W1 and U5:
+ *
+ * -----+---------+-----------+------------------------------+-----------+---------------+----------
+ *  W1    GPIOL6    VPIB0       SCU90[5:4]=3 & SCU84[22]=1     TXD1        SCU84[22]=1     GPIOL6
+ *  U5    GPIOL7    VPIB1       SCU90[5:4]=3 & SCU84[23]=1     RXD1        SCU84[23]=1     GPIOL7
+ * -----+---------+-----------+------------------------------+-----------+---------------+----------
+ *
+ * The examples of T5 and W1 are particularly fertile, as they also demonstrate
+ * that despite operating as part of the video input bus each signal needs to
+ * be enabled individually via it's own SCU84 (in the cases of T5 and W1)
+ * register bit. This is a little crazy if the bus doesn't have optional
+ * signals, but is used to decent effect with some of the UARTs where not all
+ * signals are required. However, this isn't done consistently - UART1 is
+ * enabled on a per-pin basis, and by contrast, all signals for UART6 are
+ * enabled by a single bit.
+ *
+ * Further, the high and low priority signals listed in the table above share
+ * a configuration bit. The VPI signals should operate in concert in a single
+ * function, but the UART signals should retain the ability to be configured
+ * independently. This pushes the implementation down the path of tagging a
+ * signal's expressions with the function they participate in, rather than
+ * defining masks affecting multiple signals per function. The latter approach
+ * fails in this instance where applying the configuration for the UART pin of
+ * interest will stomp on the state of other UART signals when disabling the
+ * VPI functions on the current pin.
+ *
+ * Ball |  Default   | P0 Signal | P0 Expression             | P1 Signal | P1 Expression | Other
+ * -----+------------+-----------+---------------------------+-----------+---------------+------------
+ *  A12   RGMII1TXCK   GPIOT0      SCUA0[0]=1                  RMII1TXEN   Strap[6]=0      RGMII1TXCK
+ *  B12   RGMII1TXCTL  GPIOT1      SCUA0[1]=1                  –           Strap[6]=0      RGMII1TXCTL
+ * -----+------------+-----------+---------------------------+-----------+---------------+------------
+ *
+ * A12 demonstrates that the "Other" signal isn't always GPIO - in this case
+ * GPIOT0 is a high-priority signal and RGMII1TXCK is Other. Thus, GPIO
+ * should be treated like any other signal type with full function expression
+ * requirements, and not assumed to be the default case. Separately, GPIOT0 and
+ * GPIOT1's signal descriptor bits are distinct, therefore we must iterate all
+ * pins in the function's group to disable the higher-priority signals such
+ * that the signal for the function of interest is correctly enabled.
+ *
+ * Finally, three priority levels aren't always enough; the AST2500 brings with
+ * it 18 pins of five priority levels, however the 18 pins only use three of
+ * the five priority levels.
+ *
+ * Ultimately the requirement to control pins in the examples above drive the
+ * design:
+ *
+ * * Pins provide signals according to functions activated in the mux
+ *   configuration
+ *
+ * * Pins provide up to five signal types in a priority order
+ *
+ * * For priorities levels defined on a pin, each priority provides one signal
+ *
+ * * Enabling lower priority signals requires higher priority signals be
+ *   disabled
+ *
+ * * A function represents a set of signals; functions are distinct if their
+ *   sets of signals are not equal
+ *
+ * * Signals participate in one or more functions
+ *
+ * * A function is described by an expression of one or more signal
+ *   descriptors, which compare bit values in a register
+ *
+ * * A signal expression is the smallest set of signal descriptors whose
+ *   comparisons must evaluate 'true' for a signal to be enabled on a pin.
+ *
+ * * A function's signal is active on a pin if evaluating all signal
+ *   descriptors in the pin's signal expression for the function yields a 'true'
+ *   result
+ *
+ * * A signal at a given priority on a given pin is active if any of the
+ *   functions in which the signal participates are active, and no higher
+ *   priority signal on the pin is active
+ *
+ * * GPIO is configured per-pin
+ *
+ * And so:
+ *
+ * * To disable a signal, any function(s) activating the signal must be
+ *   disabled
+ *
+ * * Each pin must know the signal expressions of functions in which it
+ *   participates, for the purpose of enabling the Other function. This is done
+ *   by deactivating all functions that activate higher priority signals on the
+ *   pin.
+ *
+ * As a concrete example:
+ *
+ * * T5 provides three signals types: VPIDE, NDCD1 and GPIO
+ *
+ * * The VPIDE signal participates in 3 functions: VPI18, VPI24 and VPI30
+ *
+ * * The NDCD1 signal participates in just its own NDCD1 function
+ *
+ * * VPIDE is high priority, NDCD1 is low priority, and GPIOL1 is the least
+ *   prioritised
+ *
+ * * The prerequisit for activating the NDCD1 signal is that the VPI18, VPI24
+ *   and VPI30 functions all be disabled
+ *
+ * * Similarly, all of VPI18, VPI24, VPI30 and NDCD1 functions must be disabled
+ *   to provide GPIOL6
+ *
+ * Considerations
+ * --------------
+ *
+ * If pinctrl allows us to allocate a pin we can configure a function without
+ * concern for the function of already allocated pins, if pin groups are
+ * created with respect to the SoC functions in which they participate. This is
+ * intuitive, but it did not feel obvious from the bit/pin relationships.
+ *
+ * Conversely, failing to allocate all pins in a group indicates some bits (as
+ * well as pins) required for the group's configuration will already be in use,
+ * likely in a way that's inconsistent with the requirements of the failed
+ * group.
+ */
+
+/*
+ * The "Multi-function Pins Mapping and Control" table in the SoC datasheet
+ * references registers by the device/offset mnemonic. The register macros
+ * below are named the same way to ease transcription and verification (as
+ * opposed to naming them e.g. PINMUX_CTRL_[0-9]). Further, signal expressions
+ * reference registers beyond those dedicated to pinmux, such as the system
+ * reset control and MAC clock configuration registers. The AST2500 goes a step
+ * further and references registers in the graphics IP block, but that isn't
+ * handled yet.
+ */
+#define SCU2C           0x2C /* Misc. Control Register */
+#define SCU3C           0x3C /* System Reset Control/Status Register */
+#define SCU48           0x48 /* MAC Interface Clock Delay Setting */
+#define HW_STRAP1       0x70 /* AST2400 strapping is 33 bits, is split */
+#define SCU80           0x80 /* Multi-function Pin Control #1 */
+#define SCU84           0x84 /* Multi-function Pin Control #2 */
+#define SCU88           0x88 /* Multi-function Pin Control #3 */
+#define SCU8C           0x8C /* Multi-function Pin Control #4 */
+#define SCU90           0x90 /* Multi-function Pin Control #5 */
+#define SCU94           0x94 /* Multi-function Pin Control #6 */
+#define SCUA0           0xA0 /* Multi-function Pin Control #7 */
+#define SCUA4           0xA4 /* Multi-function Pin Control #8 */
+#define SCUA8           0xA8 /* Multi-function Pin Control #9 */
+#define HW_STRAP2       0xD0 /* Strapping */
+
+ /**
+  * A signal descriptor, which describes the register, bits and the
+  * enable/disable values that should be compared or written.
+  *
+  * @reg: The register offset from base in bytes
+  * @mask: The mask to apply to the register. The lowest set bit of the mask is
+  *        used to derive the shift value.
+  * @enable: The value that enables the function. Value should be in the LSBs,
+  *          not at the position of the mask.
+  * @disable: The value that disables the function. Value should be in the
+  *           LSBs, not at the position of the mask.
+  */
+struct aspeed_sig_desc {
+	unsigned int reg;
+	u32 mask;
+	u32 enable;
+	u32 disable;
+};
+
+/**
+ * Describes a signal expression. The expression is evaluated by ANDing the
+ * evaluation of the descriptors.
+ *
+ * @signal: The signal name for the priority level on the pin. If the signal
+ *          type is GPIO, then the signal name must begin with the string
+ *          "GPIO", e.g. GPIOA0, GPIOT4 etc.
+ * @function: The name of the function the signal participates in for the
+ *            associated expression
+ * @ndescs: The number of signal descriptors in the expression
+ * @descs: Pointer to an array of signal descriptors that comprise the
+ *         function expression
+ */
+struct aspeed_sig_expr {
+	const char *signal;
+	const char *function;
+	int ndescs;
+	const struct aspeed_sig_desc *descs;
+};
+
+/**
+ * A struct capturing the list of expressions enabling signals at each priority
+ * for a given pin. The signal configuration for a priority level is evaluated
+ * by ORing the evaluation of the signal expressions in the respective
+ * priority's list.
+ *
+ * @name: A name for the pin
+ * @prios: A pointer to an array of expression list pointers
+ *
+ */
+struct aspeed_pin_desc {
+	const char *name;
+	const struct aspeed_sig_expr ***prios;
+};
+
+/* Macro hell */
+
+/**
+ * Short-hand macro for describing a configuration enabled by the state of one
+ * bit. The disable value is derived.
+ *
+ * @reg: The signal's associated register, offset from base
+ * @idx: The signal's bit index in the register
+ * @val: The value (0 or 1) that enables the function
+ */
+#define SIG_DESC_BIT(reg, idx, val) \
+	{ reg, BIT_MASK(idx), val, (((val) + 1) & 1) }
+
+/**
+ * A further short-hand macro describing a configuration enabled with a set bit.
+ *
+ * @reg: The configuration's associated register, offset from base
+ * @idx: The configuration's bit index in the register
+ */
+#define SIG_DESC_SET(reg, idx) SIG_DESC_BIT(reg, idx, 1)
+
+#define SIG_DESC_LIST_SYM(sig, func) sig_descs_ ## sig ## _ ## func
+#define SIG_DESC_LIST_DECL(sig, func, ...) \
+	static const struct aspeed_sig_desc SIG_DESC_LIST_SYM(sig, func)[] = \
+		{ __VA_ARGS__ }
+
+#define SIG_EXPR_SYM(sig, func) sig_expr_ ## sig ## _ ## func
+#define SIG_EXPR_DECL_(sig, func) \
+	static const struct aspeed_sig_expr SIG_EXPR_SYM(sig, func) = \
+	{ \
+		.signal = #sig, \
+		.function = #func, \
+		.ndescs = ARRAY_SIZE(SIG_DESC_LIST_SYM(sig, func)), \
+		.descs = &(SIG_DESC_LIST_SYM(sig, func))[0], \
+	}
+
+/**
+ * Declare a signal expression.
+ *
+ * @sig: A macro symbol name for the signal (is subjected to stringification
+ *        and token pasting)
+ * @func: The function in which the signal is participating
+ * @...: Signal descriptors that define the signal expression
+ *
+ * For example, the following declares the ROMD8 signal for the ROM16 function:
+ *
+ *     SIG_EXPR_DECL(ROMD8, ROM16, SIG_DESC_SET(SCU90, 6));
+ *
+ * And with multiple signal descriptors:
+ *
+ *     SIG_EXPR_DECL(ROMD8, ROM16S, SIG_DESC_SET(HW_STRAP1, 4),
+ *              { HW_STRAP1, GENMASK(1, 0), 0, 0 });
+ */
+#define SIG_EXPR_DECL(sig, func, ...) \
+	SIG_DESC_LIST_DECL(sig, func, __VA_ARGS__); \
+	SIG_EXPR_DECL_(sig, func)
+
+/**
+ * Declare a pointer to a signal expression
+ *
+ * @sig: The macro symbol name for the signal (subjected to token pasting)
+ * @func: The macro symbol name for the function (subjected to token pasting)
+ */
+#define SIG_EXPR_PTR(sig, func) (&SIG_EXPR_SYM(sig, func))
+
+#define SIG_EXPR_LIST_SYM(sig) sig_exprs_ ## sig
+
+/**
+ * Declare a signal expression list for reference in a struct aspeed_pin_prio.
+ *
+ * @sig: A macro symbol name for the signal (is subjected to token pasting)
+ * @...: Signal expression structure pointers (use SIG_EXPR_PTR())
+ *
+ * For example, the 16-bit ROM bus can be enabled by one of two possible signal
+ * expressions:
+ *
+ *     SIG_EXPR_DECL(ROMD8, ROM16, SIG_DESC_SET(SCU90, 6));
+ *     SIG_EXPR_DECL(ROMD8, ROM16S, SIG_DESC_SET(HW_STRAP1, 4),
+ *              { HW_STRAP1, GENMASK(1, 0), 0, 0 });
+ *     SIG_EXPR_LIST_DECL(ROMD8, SIG_EXPR_PTR(ROMD8, ROM16),
+ *              SIG_EXPR_PTR(ROMD8, ROM16S));
+ */
+#define SIG_EXPR_LIST_DECL(sig, ...) \
+	static const struct aspeed_sig_expr *SIG_EXPR_LIST_SYM(sig)[] = \
+		{ __VA_ARGS__, NULL }
+
+/**
+ * A short-hand macro for declaring a function expression and an expression
+ * list with a single function.
+ *
+ * @func: A macro symbol name for the function (is subjected to token pasting)
+ * @...: Function descriptors that define the function expression
+ *
+ * For example, signal NCTS6 participates in its own function with one group:
+ *
+ *     SIG_EXPR_LIST_DECL_SINGLE(NCTS6, NCTS6, SIG_DESC_SET(SCU90, 7));
+ */
+#define SIG_EXPR_LIST_DECL_SINGLE(sig, func, ...) \
+	SIG_DESC_LIST_DECL(sig, func, __VA_ARGS__); \
+	SIG_EXPR_DECL_(sig, func); \
+	SIG_EXPR_LIST_DECL(sig, SIG_EXPR_PTR(sig, func))
+
+#define SIG_EXPR_LIST_DECL_DUAL(sig, f0, f1) \
+	SIG_EXPR_LIST_DECL(sig, SIG_EXPR_PTR(sig, f0), SIG_EXPR_PTR(sig, f1))
+
+#define SIG_EXPR_LIST_PTR(sig) (&SIG_EXPR_LIST_SYM(sig)[0])
+
+#define PIN_EXPRS_SYM(pin) pin_exprs_ ## pin
+#define PIN_EXPRS_PTR(pin) (&PIN_EXPRS_SYM(pin)[0])
+#define PIN_SYM(pin) pin_ ## pin
+
+#define MS_PIN_DECL_(pin, ...) \
+	static const struct aspeed_sig_expr **PIN_EXPRS_SYM(pin)[] = \
+		{ __VA_ARGS__, NULL }; \
+	static const struct aspeed_pin_desc PIN_SYM(pin) = \
+		{ #pin, PIN_EXPRS_PTR(pin) }
+
+/**
+ * Declare a multi-signal pin
+ *
+ * @pin: The pin number
+ * @other: Macro name for "other" functionality (subjected to stringification)
+ * @high: Macro name for the highest priority signal functions
+ * @low: Macro name for the low signal functions
+ *
+ * For example:
+ *
+ *     #define A8 56
+ *     SIG_EXPR_DECL(ROMD8, ROM16, SIG_DESC_SET(SCU90, 6));
+ *     SIG_EXPR_DECL(ROMD8, ROM16S, SIG_DESC_SET(HW_STRAP1, 4),
+ *              { HW_STRAP1, GENMASK(1, 0), 0, 0 });
+ *     SIG_EXPR_LIST_DECL(ROMD8, SIG_EXPR_PTR(ROMD8, ROM16),
+ *              SIG_EXPR_PTR(ROMD8, ROM16S));
+ *     SIG_EXPR_LIST_DECL_SINGLE(NCTS6, NCTS6, SIG_DESC_SET(SCU90, 7));
+ *     MS_PIN_DECL(A8, GPIOH0, ROMD8, NCTS6);
+ */
+#define MS_PIN_DECL(pin, other, high, low) \
+	SIG_EXPR_LIST_DECL_SINGLE(other, other); \
+	MS_PIN_DECL_(pin, \
+			SIG_EXPR_LIST_PTR(high), \
+			SIG_EXPR_LIST_PTR(low), \
+			SIG_EXPR_LIST_PTR(other))
+
+#define PIN_GROUP_SYM(func) pins_ ## func
+#define FUNC_GROUP_SYM(func) groups_ ## func
+#define FUNC_GROUP_DECL(func, ...) \
+	static const int PIN_GROUP_SYM(func)[] = { __VA_ARGS__ }; \
+	static const char *FUNC_GROUP_SYM(func)[] = { #func }
+
+/**
+ * Declare a single signal pin
+ *
+ * @pin: The pin number
+ * @other: Macro name for "other" functionality (subjected to stringification)
+ * @sig: Macro name for the signal (subjected to stringification)
+ *
+ * For example:
+ *
+ *     #define E3 80
+ *     SIG_EXPR_LIST_DECL_SINGLE(SCL5, I2C5, I2C5_DESC);
+ *     SS_PIN_DECL(E3, GPIOK0, SCL5);
+ */
+#define SS_PIN_DECL(pin, other, sig) \
+	SIG_EXPR_LIST_DECL_SINGLE(other, other); \
+	MS_PIN_DECL_(pin, SIG_EXPR_LIST_PTR(sig), SIG_EXPR_LIST_PTR(other))
+
+/**
+ * Single signal, single function pin declaration
+ *
+ * @pin: The pin number
+ * @other: Macro name for "other" functionality (subjected to stringification)
+ * @sig: Macro name for the signal (subjected to stringification)
+ * @...: Signal descriptors that define the function expression
+ *
+ * For example:
+ *
+ *    SSSF_PIN_DECL(A4, GPIOA2, TIMER3, SIG_DESC_SET(SCU80, 2));
+ */
+#define SSSF_PIN_DECL(pin, other, sig, ...) \
+	SIG_EXPR_LIST_DECL_SINGLE(sig, sig, __VA_ARGS__); \
+	SIG_EXPR_LIST_DECL_SINGLE(other, other); \
+	MS_PIN_DECL_(pin, SIG_EXPR_LIST_PTR(sig), SIG_EXPR_LIST_PTR(other)); \
+	FUNC_GROUP_DECL(sig, pin)
+
+#define GPIO_PIN_DECL(pin, gpio) \
+	SIG_EXPR_LIST_DECL_SINGLE(gpio, gpio); \
+	MS_PIN_DECL_(pin, SIG_EXPR_LIST_PTR(gpio))
+
+struct aspeed_pinctrl_data {
+	struct regmap *map;
+
+	const struct pinctrl_pin_desc *pins;
+	const unsigned int npins;
+
+	const struct aspeed_pin_group *groups;
+	const unsigned int ngroups;
+
+	const struct aspeed_pin_function *functions;
+	const unsigned int nfunctions;
+};
+
+#define ASPEED_PINCTRL_PIN(name_) \
+	[name_] = { \
+		.number = name_, \
+		.name = #name_, \
+		.drv_data = (void *) &(PIN_SYM(name_)) \
+	}
+
+struct aspeed_pin_group {
+	const char *name;
+	const unsigned int *pins;
+	const unsigned int npins;
+};
+
+#define ASPEED_PINCTRL_GROUP(name_) { \
+	.name = #name_, \
+	.pins = &(PIN_GROUP_SYM(name_))[0], \
+	.npins = ARRAY_SIZE(PIN_GROUP_SYM(name_)), \
+}
+
+struct aspeed_pin_function {
+	const char *name;
+	const char *const *groups;
+	unsigned int ngroups;
+};
+
+#define ASPEED_PINCTRL_FUNC(name_, ...) { \
+	.name = #name_, \
+	.groups = &FUNC_GROUP_SYM(name_)[0], \
+	.ngroups = ARRAY_SIZE(FUNC_GROUP_SYM(name_)), \
+}
+
+int aspeed_pinctrl_get_groups_count(struct pinctrl_dev *pctldev);
+const char *aspeed_pinctrl_get_group_name(struct pinctrl_dev *pctldev,
+		unsigned int group);
+int aspeed_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
+		unsigned int group, const unsigned int **pins,
+		unsigned int *npins);
+void aspeed_pinctrl_pin_dbg_show(struct pinctrl_dev *pctldev,
+		struct seq_file *s, unsigned int offset);
+int aspeed_pinmux_get_fn_count(struct pinctrl_dev *pctldev);
+const char *aspeed_pinmux_get_fn_name(struct pinctrl_dev *pctldev,
+		unsigned int function);
+int aspeed_pinmux_get_fn_groups(struct pinctrl_dev *pctldev,
+		unsigned int function, const char * const **groups,
+		unsigned int * const num_groups);
+int aspeed_pinmux_set_mux(struct pinctrl_dev *pctldev, unsigned int function,
+		unsigned int group);
+int aspeed_gpio_request_enable(struct pinctrl_dev *pctldev,
+		struct pinctrl_gpio_range *range,
+		unsigned int offset);
+int aspeed_pinctrl_probe(struct platform_device *pdev,
+		struct pinctrl_desc *pdesc,
+		struct aspeed_pinctrl_data *pdata);
+
+#endif /* PINCTRL_ASPEED */
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c
index 582f6df446e8..a5331fdfc795 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c
@@ -1018,7 +1018,7 @@ static void bcm281xx_pinctrl_pin_dbg_show(struct pinctrl_dev *pctldev,
 	seq_printf(s, " %s", dev_name(pctldev->dev));
 }
 
-static struct pinctrl_ops bcm281xx_pinctrl_ops = {
+static const struct pinctrl_ops bcm281xx_pinctrl_ops = {
 	.get_groups_count = bcm281xx_pinctrl_get_groups_count,
 	.get_group_name = bcm281xx_pinctrl_get_group_name,
 	.get_group_pins = bcm281xx_pinctrl_get_group_pins,
@@ -1080,7 +1080,7 @@ static int bcm281xx_pinmux_set(struct pinctrl_dev *pctldev,
 	return rc;
 }
 
-static struct pinmux_ops bcm281xx_pinctrl_pinmux_ops = {
+static const struct pinmux_ops bcm281xx_pinctrl_pinmux_ops = {
 	.get_functions_count = bcm281xx_pinctrl_get_fcns_count,
 	.get_function_name = bcm281xx_pinctrl_get_fcn_name,
 	.get_function_groups = bcm281xx_pinctrl_get_fcn_groups,
diff --git a/drivers/pinctrl/bcm/pinctrl-ns2-mux.c b/drivers/pinctrl/bcm/pinctrl-ns2-mux.c
index ca817896ed24..13a4c2774157 100644
--- a/drivers/pinctrl/bcm/pinctrl-ns2-mux.c
+++ b/drivers/pinctrl/bcm/pinctrl-ns2-mux.c
@@ -531,7 +531,7 @@ static void ns2_pin_dbg_show(struct pinctrl_dev *pctrl_dev,
 	seq_printf(s, " %s", dev_name(pctrl_dev->dev));
 }
 
-static struct pinctrl_ops ns2_pinctrl_ops = {
+static const struct pinctrl_ops ns2_pinctrl_ops = {
 	.get_groups_count = ns2_get_groups_count,
 	.get_group_name = ns2_get_group_name,
 	.get_group_pins = ns2_get_group_pins,
@@ -959,7 +959,7 @@ static int ns2_pin_config_set(struct pinctrl_dev *pctrldev, unsigned int pin,
 out:
 	return ret;
 }
-static struct pinmux_ops ns2_pinmux_ops = {
+static const struct pinmux_ops ns2_pinmux_ops = {
 	.get_functions_count = ns2_get_functions_count,
 	.get_function_name = ns2_get_function_name,
 	.get_function_groups = ns2_get_function_groups,
diff --git a/drivers/pinctrl/bcm/pinctrl-nsp-mux.c b/drivers/pinctrl/bcm/pinctrl-nsp-mux.c
index 4149db309c8b..35c17653c694 100644
--- a/drivers/pinctrl/bcm/pinctrl-nsp-mux.c
+++ b/drivers/pinctrl/bcm/pinctrl-nsp-mux.c
@@ -348,7 +348,7 @@ static void nsp_pin_dbg_show(struct pinctrl_dev *pctrl_dev,
 	seq_printf(s, " %s", dev_name(pctrl_dev->dev));
 }
 
-static struct pinctrl_ops nsp_pinctrl_ops = {
+static const struct pinctrl_ops nsp_pinctrl_ops = {
 	.get_groups_count = nsp_get_groups_count,
 	.get_group_name = nsp_get_group_name,
 	.get_group_pins = nsp_get_group_pins,
@@ -518,7 +518,7 @@ static void nsp_gpio_disable_free(struct pinctrl_dev *pctrl_dev,
 	spin_unlock_irqrestore(&pinctrl->lock, flags);
 }
 
-static struct pinmux_ops nsp_pinmux_ops = {
+static const struct pinmux_ops nsp_pinmux_ops = {
 	.get_functions_count = nsp_get_functions_count,
 	.get_function_name = nsp_get_function_name,
 	.get_function_groups = nsp_get_function_groups,
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.c b/drivers/pinctrl/freescale/pinctrl-imx.c
index 71391757938b..47613201269a 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx.c
@@ -315,7 +315,7 @@ static int imx_pmx_gpio_request_enable(struct pinctrl_dev *pctldev,
 
 	/* Currently implementation only for shared mux/conf register */
 	if (!(info->flags & SHARE_MUX_CONF_REG))
-		return -EINVAL;
+		return 0;
 
 	pin_reg = &info->pin_regs[offset];
 	if (pin_reg->mux_reg == -1)
@@ -380,7 +380,7 @@ static int imx_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
 	 * They are part of the shared mux/conf register.
 	 */
 	if (!(info->flags & SHARE_MUX_CONF_REG))
-		return -EINVAL;
+		return 0;
 
 	pin_reg = &info->pin_regs[offset];
 	if (pin_reg->mux_reg == -1)
@@ -501,7 +501,7 @@ static void imx_pinconf_group_dbg_show(struct pinctrl_dev *pctldev,
 		ret = imx_pinconf_get(pctldev, pin->pin, &config);
 		if (ret)
 			return;
-		seq_printf(s, "%s: 0x%lx", name, config);
+		seq_printf(s, "  %s: 0x%lx\n", name, config);
 	}
 }
 
diff --git a/drivers/pinctrl/intel/pinctrl-cherryview.c b/drivers/pinctrl/intel/pinctrl-cherryview.c
index 5749a4eee746..30389f4ccab4 100644
--- a/drivers/pinctrl/intel/pinctrl-cherryview.c
+++ b/drivers/pinctrl/intel/pinctrl-cherryview.c
@@ -134,6 +134,7 @@ struct chv_gpio_pinrange {
  * @gpio_ranges: An array of GPIO ranges in this community
  * @ngpio_ranges: Number of GPIO ranges
  * @ngpios: Total number of GPIOs in this community
+ * @nirqs: Total number of IRQs this community can generate
  */
 struct chv_community {
 	const char *uid;
@@ -146,6 +147,7 @@ struct chv_community {
 	const struct chv_gpio_pinrange *gpio_ranges;
 	size_t ngpio_ranges;
 	size_t ngpios;
+	size_t nirqs;
 };
 
 struct chv_pin_context {
@@ -396,6 +398,12 @@ static const struct chv_community southwest_community = {
 	.gpio_ranges = southwest_gpio_ranges,
 	.ngpio_ranges = ARRAY_SIZE(southwest_gpio_ranges),
 	.ngpios = ARRAY_SIZE(southwest_pins),
+	/*
+	 * Southwest community can benerate GPIO interrupts only for the
+	 * first 8 interrupts. The upper half (8-15) can only be used to
+	 * trigger GPEs.
+	 */
+	.nirqs = 8,
 };
 
 static const struct pinctrl_pin_desc north_pins[] = {
@@ -479,6 +487,12 @@ static const struct chv_community north_community = {
 	.gpio_ranges = north_gpio_ranges,
 	.ngpio_ranges = ARRAY_SIZE(north_gpio_ranges),
 	.ngpios = ARRAY_SIZE(north_pins),
+	/*
+	 * North community can benerate GPIO interrupts only for the first
+	 * 8 interrupts. The upper half (8-15) can only be used to trigger
+	 * GPEs.
+	 */
+	.nirqs = 8,
 };
 
 static const struct pinctrl_pin_desc east_pins[] = {
@@ -521,6 +535,7 @@ static const struct chv_community east_community = {
 	.gpio_ranges = east_gpio_ranges,
 	.ngpio_ranges = ARRAY_SIZE(east_gpio_ranges),
 	.ngpios = ARRAY_SIZE(east_pins),
+	.nirqs = 16,
 };
 
 static const struct pinctrl_pin_desc southeast_pins[] = {
@@ -646,6 +661,7 @@ static const struct chv_community southeast_community = {
 	.gpio_ranges = southeast_gpio_ranges,
 	.ngpio_ranges = ARRAY_SIZE(southeast_gpio_ranges),
 	.ngpios = ARRAY_SIZE(southeast_pins),
+	.nirqs = 16,
 };
 
 static const struct chv_community *chv_communities[] = {
@@ -1497,7 +1513,7 @@ static void chv_gpio_irq_handler(struct irq_desc *desc)
 	chained_irq_enter(chip, desc);
 
 	pending = readl(pctrl->regs + CHV_INTSTAT);
-	for_each_set_bit(intr_line, &pending, 16) {
+	for_each_set_bit(intr_line, &pending, pctrl->community->nirqs) {
 		unsigned irq, offset;
 
 		offset = pctrl->intr_lines[intr_line];
@@ -1520,8 +1536,9 @@ static int chv_gpio_probe(struct chv_pinctrl *pctrl, int irq)
 	chip->label = dev_name(pctrl->dev);
 	chip->parent = pctrl->dev;
 	chip->base = -1;
+	chip->irq_need_valid_mask = true;
 
-	ret = gpiochip_add_data(chip, pctrl);
+	ret = devm_gpiochip_add_data(pctrl->dev, chip, pctrl);
 	if (ret) {
 		dev_err(pctrl->dev, "Failed to register gpiochip\n");
 		return ret;
@@ -1533,31 +1550,40 @@ static int chv_gpio_probe(struct chv_pinctrl *pctrl, int irq)
 					     range->base, range->npins);
 		if (ret) {
 			dev_err(pctrl->dev, "failed to add GPIO pin range\n");
-			goto fail;
+			return ret;
 		}
 
 		offset += range->npins;
 	}
 
-	/* Mask and clear all interrupts */
-	chv_writel(0, pctrl->regs + CHV_INTMASK);
+	/* Do not add GPIOs that can only generate GPEs to the IRQ domain */
+	for (i = 0; i < pctrl->community->npins; i++) {
+		const struct pinctrl_pin_desc *desc;
+		u32 intsel;
+
+		desc = &pctrl->community->pins[i];
+
+		intsel = readl(chv_padreg(pctrl, desc->number, CHV_PADCTRL0));
+		intsel &= CHV_PADCTRL0_INTSEL_MASK;
+		intsel >>= CHV_PADCTRL0_INTSEL_SHIFT;
+
+		if (intsel >= pctrl->community->nirqs)
+			clear_bit(i, chip->irq_valid_mask);
+	}
+
+	/* Clear all interrupts */
 	chv_writel(0xffff, pctrl->regs + CHV_INTSTAT);
 
 	ret = gpiochip_irqchip_add(chip, &chv_gpio_irqchip, 0,
-				   handle_simple_irq, IRQ_TYPE_NONE);
+				   handle_bad_irq, IRQ_TYPE_NONE);
 	if (ret) {
 		dev_err(pctrl->dev, "failed to add IRQ chip\n");
-		goto fail;
+		return ret;
 	}
 
 	gpiochip_set_chained_irqchip(chip, &chv_gpio_irqchip, irq,
 				     chv_gpio_irq_handler);
 	return 0;
-
-fail:
-	gpiochip_remove(chip);
-
-	return ret;
 }
 
 static int chv_pinctrl_probe(struct platform_device *pdev)
@@ -1625,15 +1651,6 @@ static int chv_pinctrl_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int chv_pinctrl_remove(struct platform_device *pdev)
-{
-	struct chv_pinctrl *pctrl = platform_get_drvdata(pdev);
-
-	gpiochip_remove(&pctrl->chip);
-
-	return 0;
-}
-
 #ifdef CONFIG_PM_SLEEP
 static int chv_pinctrl_suspend(struct device *dev)
 {
@@ -1730,7 +1747,6 @@ MODULE_DEVICE_TABLE(acpi, chv_pinctrl_acpi_match);
 
 static struct platform_driver chv_pinctrl_driver = {
 	.probe = chv_pinctrl_probe,
-	.remove = chv_pinctrl_remove,
 	.driver = {
 		.name = "cherryview-pinctrl",
 		.pm = &chv_pinctrl_pm_ops,
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 257cab129692..63387a40b973 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -86,6 +86,7 @@ struct intel_pinctrl_context {
  * @communities: All communities in this pin controller
  * @ncommunities: Number of communities in this pin controller
  * @context: Configuration saved over system sleep
+ * @irq: pinctrl/GPIO chip irq number
  */
 struct intel_pinctrl {
 	struct device *dev;
@@ -97,6 +98,7 @@ struct intel_pinctrl {
 	struct intel_community *communities;
 	size_t ncommunities;
 	struct intel_pinctrl_context context;
+	int irq;
 };
 
 #define pin_to_padno(c, p)	((p) - (c)->pin_base)
@@ -793,38 +795,12 @@ static int intel_gpio_irq_wake(struct irq_data *d, unsigned int on)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct intel_pinctrl *pctrl = gpiochip_get_data(gc);
-	const struct intel_community *community;
 	unsigned pin = irqd_to_hwirq(d);
-	unsigned padno, gpp, gpp_offset;
-	unsigned long flags;
-	u32 gpe_en;
-
-	community = intel_get_community(pctrl, pin);
-	if (!community)
-		return -EINVAL;
-
-	raw_spin_lock_irqsave(&pctrl->lock, flags);
 
-	padno = pin_to_padno(community, pin);
-	gpp = padno / community->gpp_size;
-	gpp_offset = padno % community->gpp_size;
-
-	/* Clear the existing wake status */
-	writel(BIT(gpp_offset), community->regs + GPI_GPE_STS + gpp * 4);
-
-	/*
-	 * The controller will generate wake when GPE of the corresponding
-	 * pad is enabled and it is not routed to SCI (GPIROUTSCI is not
-	 * set).
-	 */
-	gpe_en = readl(community->regs + GPI_GPE_EN + gpp * 4);
 	if (on)
-		gpe_en |= BIT(gpp_offset);
+		enable_irq_wake(pctrl->irq);
 	else
-		gpe_en &= ~BIT(gpp_offset);
-	writel(gpe_en, community->regs + GPI_GPE_EN + gpp * 4);
-
-	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+		disable_irq_wake(pctrl->irq);
 
 	dev_dbg(pctrl->dev, "%sable wake for pin %u\n", on ? "en" : "dis", pin);
 	return 0;
@@ -905,6 +881,7 @@ static int intel_gpio_probe(struct intel_pinctrl *pctrl, int irq)
 	pctrl->chip.label = dev_name(pctrl->dev);
 	pctrl->chip.parent = pctrl->dev;
 	pctrl->chip.base = -1;
+	pctrl->irq = irq;
 
 	ret = gpiochip_add_data(&pctrl->chip, pctrl);
 	if (ret) {
diff --git a/drivers/pinctrl/intel/pinctrl-merrifield.c b/drivers/pinctrl/intel/pinctrl-merrifield.c
index eb4990ff26ca..7826c7f0cb7c 100644
--- a/drivers/pinctrl/intel/pinctrl-merrifield.c
+++ b/drivers/pinctrl/intel/pinctrl-merrifield.c
@@ -11,6 +11,7 @@
 
 #include <linux/bitops.h>
 #include <linux/err.h>
+#include <linux/io.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pinctrl/pinconf.h>
@@ -853,7 +854,7 @@ static int mrfld_pinctrl_probe(struct platform_device *pdev)
 	 */
 	nfamilies = ARRAY_SIZE(mrfld_families),
 	families = devm_kmemdup(&pdev->dev, mrfld_families,
-					    nfamilies * sizeof(mrfld_families),
+					    sizeof(mrfld_families),
 					    GFP_KERNEL);
 	if (!families)
 		return -ENOMEM;
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
index ce554e0d6979..f9aef2ac03a1 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
@@ -15,7 +15,6 @@
 
 #include <linux/io.h>
 #include <linux/gpio/driver.h>
-#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_device.h>
@@ -1055,7 +1054,7 @@ static int mtk_gpio_set_debounce(struct gpio_chip *chip, unsigned offset,
 	return 0;
 }
 
-static struct gpio_chip mtk_gpio_chip = {
+static const struct gpio_chip mtk_gpio_chip = {
 	.owner			= THIS_MODULE,
 	.request		= gpiochip_generic_request,
 	.free			= gpiochip_generic_free,
@@ -1496,7 +1495,3 @@ chip_error:
 	gpiochip_remove(pctl->chip);
 	return ret;
 }
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("MediaTek Pinctrl Driver");
-MODULE_AUTHOR("Hongzhou Yang <hongzhou.yang@mediatek.com>");
diff --git a/drivers/pinctrl/meson/pinctrl-meson-gxbb.c b/drivers/pinctrl/meson/pinctrl-meson-gxbb.c
index cb4d6ad30530..c3928aa3fefa 100644
--- a/drivers/pinctrl/meson/pinctrl-meson-gxbb.c
+++ b/drivers/pinctrl/meson/pinctrl-meson-gxbb.c
@@ -156,6 +156,11 @@ static const unsigned int emmc_clk_pins[] = { PIN(BOOT_8, EE_OFF) };
 static const unsigned int emmc_cmd_pins[] = { PIN(BOOT_10, EE_OFF) };
 static const unsigned int emmc_ds_pins[] = { PIN(BOOT_15, EE_OFF) };
 
+static const unsigned int nor_d_pins[]		= { PIN(BOOT_11, EE_OFF) };
+static const unsigned int nor_q_pins[]		= { PIN(BOOT_12, EE_OFF) };
+static const unsigned int nor_c_pins[]		= { PIN(BOOT_13, EE_OFF) };
+static const unsigned int nor_cs_pins[]		= { PIN(BOOT_15, EE_OFF) };
+
 static const unsigned int sdcard_d0_pins[] = { PIN(CARD_1, EE_OFF) };
 static const unsigned int sdcard_d1_pins[] = { PIN(CARD_0, EE_OFF) };
 static const unsigned int sdcard_d2_pins[] = { PIN(CARD_5, EE_OFF) };
@@ -163,6 +168,23 @@ static const unsigned int sdcard_d3_pins[] = { PIN(CARD_4, EE_OFF) };
 static const unsigned int sdcard_cmd_pins[] = { PIN(CARD_3, EE_OFF) };
 static const unsigned int sdcard_clk_pins[] = { PIN(CARD_2, EE_OFF) };
 
+static const unsigned int sdio_d0_pins[] = { PIN(GPIOX_0, EE_OFF) };
+static const unsigned int sdio_d1_pins[] = { PIN(GPIOX_1, EE_OFF) };
+static const unsigned int sdio_d2_pins[] = { PIN(GPIOX_2, EE_OFF) };
+static const unsigned int sdio_d3_pins[] = { PIN(GPIOX_3, EE_OFF) };
+static const unsigned int sdio_cmd_pins[] = { PIN(GPIOX_4, EE_OFF) };
+static const unsigned int sdio_clk_pins[] = { PIN(GPIOX_5, EE_OFF) };
+static const unsigned int sdio_irq_pins[] = { PIN(GPIOX_7, EE_OFF) };
+
+static const unsigned int nand_ce0_pins[]	= { PIN(BOOT_8, EE_OFF) };
+static const unsigned int nand_ce1_pins[]	= { PIN(BOOT_9, EE_OFF) };
+static const unsigned int nand_rb0_pins[]	= { PIN(BOOT_10, EE_OFF) };
+static const unsigned int nand_ale_pins[]	= { PIN(BOOT_11, EE_OFF) };
+static const unsigned int nand_cle_pins[]	= { PIN(BOOT_12, EE_OFF) };
+static const unsigned int nand_wen_clk_pins[]	= { PIN(BOOT_13, EE_OFF) };
+static const unsigned int nand_ren_wr_pins[]	= { PIN(BOOT_14, EE_OFF) };
+static const unsigned int nand_dqs_pins[]	= { PIN(BOOT_15, EE_OFF) };
+
 static const unsigned int uart_tx_a_pins[]	= { PIN(GPIOX_12, EE_OFF) };
 static const unsigned int uart_rx_a_pins[]	= { PIN(GPIOX_13, EE_OFF) };
 static const unsigned int uart_cts_a_pins[]	= { PIN(GPIOX_14, EE_OFF) };
@@ -178,6 +200,15 @@ static const unsigned int uart_rx_c_pins[]	= { PIN(GPIOY_14, EE_OFF) };
 static const unsigned int uart_cts_c_pins[]	= { PIN(GPIOX_11, EE_OFF) };
 static const unsigned int uart_rts_c_pins[]	= { PIN(GPIOX_12, EE_OFF) };
 
+static const unsigned int i2c_sck_a_pins[]	= { PIN(GPIODV_25, EE_OFF) };
+static const unsigned int i2c_sda_a_pins[]	= { PIN(GPIODV_24, EE_OFF) };
+
+static const unsigned int i2c_sck_b_pins[]	= { PIN(GPIODV_27, EE_OFF) };
+static const unsigned int i2c_sda_b_pins[]	= { PIN(GPIODV_26, EE_OFF) };
+
+static const unsigned int i2c_sck_c_pins[]	= { PIN(GPIODV_29, EE_OFF) };
+static const unsigned int i2c_sda_c_pins[]	= { PIN(GPIODV_28, EE_OFF) };
+
 static const unsigned int eth_mdio_pins[]	= { PIN(GPIOZ_0, EE_OFF) };
 static const unsigned int eth_mdc_pins[]	= { PIN(GPIOZ_1, EE_OFF) };
 static const unsigned int eth_clk_rx_clk_pins[]	= { PIN(GPIOZ_2, EE_OFF) };
@@ -193,6 +224,14 @@ static const unsigned int eth_txd1_pins[]	= { PIN(GPIOZ_11, EE_OFF) };
 static const unsigned int eth_txd2_pins[]	= { PIN(GPIOZ_12, EE_OFF) };
 static const unsigned int eth_txd3_pins[]	= { PIN(GPIOZ_13, EE_OFF) };
 
+static const unsigned int pwm_a_x_pins[]	= { PIN(GPIOX_6, EE_OFF) };
+static const unsigned int pwm_a_y_pins[]	= { PIN(GPIOY_16, EE_OFF) };
+static const unsigned int pwm_b_pins[]		= { PIN(GPIODV_29, EE_OFF) };
+static const unsigned int pwm_d_pins[]		= { PIN(GPIODV_28, EE_OFF) };
+static const unsigned int pwm_e_pins[]		= { PIN(GPIOX_19, EE_OFF) };
+static const unsigned int pwm_f_x_pins[]	= { PIN(GPIOX_7, EE_OFF) };
+static const unsigned int pwm_f_y_pins[]	= { PIN(GPIOY_15, EE_OFF) };
+
 static const struct pinctrl_pin_desc meson_gxbb_aobus_pins[] = {
 	MESON_PIN(GPIOAO_0, 0),
 	MESON_PIN(GPIOAO_1, 0),
@@ -225,6 +264,13 @@ static const unsigned int i2c_sda_ao_pins[] = {PIN(GPIOAO_5, 0) };
 static const unsigned int i2c_slave_sck_ao_pins[] = {PIN(GPIOAO_4, 0) };
 static const unsigned int i2c_slave_sda_ao_pins[] = {PIN(GPIOAO_5, 0) };
 
+static const unsigned int remote_input_ao_pins[] = {PIN(GPIOAO_7, 0) };
+
+static const unsigned int pwm_ao_a_3_pins[]	= { PIN(GPIOAO_3, 0) };
+static const unsigned int pwm_ao_a_6_pins[]	= { PIN(GPIOAO_6, 0) };
+static const unsigned int pwm_ao_a_12_pins[]	= { PIN(GPIOAO_12, 0) };
+static const unsigned int pwm_ao_b_pins[]	= { PIN(GPIOAO_13, 0) };
+
 static struct meson_pmx_group meson_gxbb_periphs_groups[] = {
 	GPIO_GROUP(GPIOZ_0, EE_OFF),
 	GPIO_GROUP(GPIOZ_1, EE_OFF),
@@ -355,16 +401,28 @@ static struct meson_pmx_group meson_gxbb_periphs_groups[] = {
 	GPIO_GROUP(GPIO_TEST_N, EE_OFF),
 
 	/* Bank X */
+	GROUP(sdio_d0,		8,	5),
+	GROUP(sdio_d1,		8,	4),
+	GROUP(sdio_d2,		8,	3),
+	GROUP(sdio_d3,		8,	2),
+	GROUP(sdio_cmd,		8,	1),
+	GROUP(sdio_clk,		8,	0),
+	GROUP(sdio_irq,		8,	11),
 	GROUP(uart_tx_a,	4,	13),
 	GROUP(uart_rx_a,	4,	12),
 	GROUP(uart_cts_a,	4,	11),
 	GROUP(uart_rts_a,	4,	10),
+	GROUP(pwm_a_x,		3,	17),
+	GROUP(pwm_e,		2,	30),
+	GROUP(pwm_f_x,		3,	18),
 
 	/* Bank Y */
 	GROUP(uart_cts_c,	1,	19),
 	GROUP(uart_rts_c,	1,	18),
 	GROUP(uart_tx_c,	1,	17),
 	GROUP(uart_rx_c,	1,	16),
+	GROUP(pwm_a_y,		1,	21),
+	GROUP(pwm_f_y,		1,	20),
 
 	/* Bank Z */
 	GROUP(eth_mdio,		6,	1),
@@ -387,12 +445,32 @@ static struct meson_pmx_group meson_gxbb_periphs_groups[] = {
 	GROUP(uart_rx_b,	2,	28),
 	GROUP(uart_cts_b,	2,	27),
 	GROUP(uart_rts_b,	2,	26),
+	GROUP(pwm_b,		3,	21),
+	GROUP(pwm_d,		3,	20),
+	GROUP(i2c_sck_a,	7,	27),
+	GROUP(i2c_sda_a,	7,	26),
+	GROUP(i2c_sck_b,	7,	25),
+	GROUP(i2c_sda_b,	7,	24),
+	GROUP(i2c_sck_c,	7,	23),
+	GROUP(i2c_sda_c,	7,	22),
 
 	/* Bank BOOT */
 	GROUP(emmc_nand_d07,	4,	30),
 	GROUP(emmc_clk,		4,	18),
 	GROUP(emmc_cmd,		4,	19),
 	GROUP(emmc_ds,		4,	31),
+	GROUP(nor_d,		5,	1),
+	GROUP(nor_q,		5,	3),
+	GROUP(nor_c,		5,	2),
+	GROUP(nor_cs,		5,	0),
+	GROUP(nand_ce0,		4,	26),
+	GROUP(nand_ce1,		4,	27),
+	GROUP(nand_rb0,		4,	25),
+	GROUP(nand_ale,		4,	24),
+	GROUP(nand_cle,		4,	23),
+	GROUP(nand_wen_clk,	4,	22),
+	GROUP(nand_ren_wr,	4,	21),
+	GROUP(nand_dqs,		4,	20),
 
 	/* Bank CARD */
 	GROUP(sdcard_d1,	2,	14),
@@ -432,6 +510,11 @@ static struct meson_pmx_group meson_gxbb_aobus_groups[] = {
 	GROUP(i2c_sda_ao,	0,	5),
 	GROUP(i2c_slave_sck_ao, 0,	2),
 	GROUP(i2c_slave_sda_ao, 0,	1),
+	GROUP(remote_input_ao,	0,	0),
+	GROUP(pwm_ao_a_3,	0,	22),
+	GROUP(pwm_ao_a_6,	0,	18),
+	GROUP(pwm_ao_a_12,	0,	17),
+	GROUP(pwm_ao_b,		0,	3),
 };
 
 static const char * const gpio_periphs_groups[] = {
@@ -475,11 +558,25 @@ static const char * const emmc_groups[] = {
 	"emmc_nand_d07", "emmc_clk", "emmc_cmd", "emmc_ds",
 };
 
+static const char * const nor_groups[] = {
+	"nor_d", "nor_q", "nor_c", "nor_cs",
+};
+
 static const char * const sdcard_groups[] = {
 	"sdcard_d0", "sdcard_d1", "sdcard_d2", "sdcard_d3",
 	"sdcard_cmd", "sdcard_clk",
 };
 
+static const char * const sdio_groups[] = {
+	"sdio_d0", "sdio_d1", "sdio_d2", "sdio_d3",
+	"sdio_cmd", "sdio_clk", "sdio_irq",
+};
+
+static const char * const nand_groups[] = {
+	"nand_ce0", "nand_ce1", "nand_rb0", "nand_ale", "nand_cle",
+	"nand_wen_clk", "nand_ren_wr", "nand_dqs",
+};
+
 static const char * const uart_a_groups[] = {
 	"uart_tx_a", "uart_rx_a", "uart_cts_a", "uart_rts_a",
 };
@@ -492,6 +589,18 @@ static const char * const uart_c_groups[] = {
 	"uart_tx_c", "uart_rx_c", "uart_cts_c", "uart_rts_c",
 };
 
+static const char * const i2c_a_groups[] = {
+	"i2c_sck_a", "i2c_sda_a",
+};
+
+static const char * const i2c_b_groups[] = {
+	"i2c_sck_b", "i2c_sda_b",
+};
+
+static const char * const i2c_c_groups[] = {
+	"i2c_sck_c", "i2c_sda_c",
+};
+
 static const char * const eth_groups[] = {
 	"eth_mdio", "eth_mdc", "eth_clk_rx_clk", "eth_rx_dv",
 	"eth_rxd0", "eth_rxd1", "eth_rxd2", "eth_rxd3",
@@ -499,6 +608,34 @@ static const char * const eth_groups[] = {
 	"eth_txd0", "eth_txd1", "eth_txd2", "eth_txd3",
 };
 
+static const char * const pwm_a_x_groups[] = {
+	"pwm_a_x",
+};
+
+static const char * const pwm_a_y_groups[] = {
+	"pwm_a_y",
+};
+
+static const char * const pwm_b_groups[] = {
+	"pwm_b",
+};
+
+static const char * const pwm_d_groups[] = {
+	"pwm_d",
+};
+
+static const char * const pwm_e_groups[] = {
+	"pwm_e",
+};
+
+static const char * const pwm_f_x_groups[] = {
+	"pwm_f_x",
+};
+
+static const char * const pwm_f_y_groups[] = {
+	"pwm_f_y",
+};
+
 static const char * const gpio_aobus_groups[] = {
 	"GPIOAO_0", "GPIOAO_1", "GPIOAO_2", "GPIOAO_3", "GPIOAO_4",
 	"GPIOAO_5", "GPIOAO_6", "GPIOAO_7", "GPIOAO_8", "GPIOAO_9",
@@ -521,14 +658,47 @@ static const char * const i2c_slave_ao_groups[] = {
 	"i2c_slave_sdk_ao", "i2c_slave_sda_ao",
 };
 
+static const char * const remote_input_ao_groups[] = {
+	"remote_input_ao",
+};
+
+static const char * const pwm_ao_a_3_groups[] = {
+	"pwm_ao_a_3",
+};
+
+static const char * const pwm_ao_a_6_groups[] = {
+	"pwm_ao_a_6",
+};
+
+static const char * const pwm_ao_a_12_groups[] = {
+	"pwm_ao_a_12",
+};
+
+static const char * const pwm_ao_b_groups[] = {
+	"pwm_ao_b",
+};
+
 static struct meson_pmx_func meson_gxbb_periphs_functions[] = {
 	FUNCTION(gpio_periphs),
 	FUNCTION(emmc),
+	FUNCTION(nor),
 	FUNCTION(sdcard),
+	FUNCTION(sdio),
+	FUNCTION(nand),
 	FUNCTION(uart_a),
 	FUNCTION(uart_b),
 	FUNCTION(uart_c),
+	FUNCTION(i2c_a),
+	FUNCTION(i2c_b),
+	FUNCTION(i2c_c),
 	FUNCTION(eth),
+	FUNCTION(pwm_a_x),
+	FUNCTION(pwm_a_y),
+	FUNCTION(pwm_b),
+	FUNCTION(pwm_d),
+	FUNCTION(pwm_e),
+	FUNCTION(pwm_f_x),
+	FUNCTION(pwm_f_y),
 };
 
 static struct meson_pmx_func meson_gxbb_aobus_functions[] = {
@@ -537,6 +707,11 @@ static struct meson_pmx_func meson_gxbb_aobus_functions[] = {
 	FUNCTION(uart_ao_b),
 	FUNCTION(i2c_ao),
 	FUNCTION(i2c_slave_ao),
+	FUNCTION(remote_input_ao),
+	FUNCTION(pwm_ao_a_3),
+	FUNCTION(pwm_ao_a_6),
+	FUNCTION(pwm_ao_a_12),
+	FUNCTION(pwm_ao_b),
 };
 
 static struct meson_bank meson_gxbb_periphs_banks[] = {
@@ -556,38 +731,28 @@ static struct meson_bank meson_gxbb_aobus_banks[] = {
 	BANK("AO",   PIN(GPIOAO_0, 0),  PIN(GPIOAO_13, 0), 0,  0,  0, 16,  0,  0,  0, 16,  1,  0),
 };
 
-static struct meson_domain_data meson_gxbb_periphs_domain_data = {
+struct meson_pinctrl_data meson_gxbb_periphs_pinctrl_data = {
 	.name		= "periphs-banks",
-	.banks		= meson_gxbb_periphs_banks,
-	.num_banks	= ARRAY_SIZE(meson_gxbb_periphs_banks),
 	.pin_base	= 14,
-	.num_pins	= 120,
-};
-
-static struct meson_domain_data meson_gxbb_aobus_domain_data = {
-	.name		= "aobus-banks",
-	.banks		= meson_gxbb_aobus_banks,
-	.num_banks	= ARRAY_SIZE(meson_gxbb_aobus_banks),
-	.pin_base	= 0,
-	.num_pins	= 14,
-};
-
-struct meson_pinctrl_data meson_gxbb_periphs_pinctrl_data = {
 	.pins		= meson_gxbb_periphs_pins,
 	.groups		= meson_gxbb_periphs_groups,
 	.funcs		= meson_gxbb_periphs_functions,
-	.domain_data	= &meson_gxbb_periphs_domain_data,
+	.banks		= meson_gxbb_periphs_banks,
 	.num_pins	= ARRAY_SIZE(meson_gxbb_periphs_pins),
 	.num_groups	= ARRAY_SIZE(meson_gxbb_periphs_groups),
 	.num_funcs	= ARRAY_SIZE(meson_gxbb_periphs_functions),
+	.num_banks	= ARRAY_SIZE(meson_gxbb_periphs_banks),
 };
 
 struct meson_pinctrl_data meson_gxbb_aobus_pinctrl_data = {
+	.name		= "aobus-banks",
+	.pin_base	= 0,
 	.pins		= meson_gxbb_aobus_pins,
 	.groups		= meson_gxbb_aobus_groups,
 	.funcs		= meson_gxbb_aobus_functions,
-	.domain_data	= &meson_gxbb_aobus_domain_data,
+	.banks		= meson_gxbb_aobus_banks,
 	.num_pins	= ARRAY_SIZE(meson_gxbb_aobus_pins),
 	.num_groups	= ARRAY_SIZE(meson_gxbb_aobus_groups),
 	.num_funcs	= ARRAY_SIZE(meson_gxbb_aobus_functions),
+	.num_banks	= ARRAY_SIZE(meson_gxbb_aobus_banks),
 };
diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c
index 11623c6b0cb3..57122eda155a 100644
--- a/drivers/pinctrl/meson/pinctrl-meson.c
+++ b/drivers/pinctrl/meson/pinctrl-meson.c
@@ -21,9 +21,8 @@
  * domain which can't be powered off; the bank also uses a set of
  * registers different from the other banks.
  *
- * For each of the two power domains (regular and always-on) there are
- * 4 different register ranges that control the following properties
- * of the pins:
+ * For each pin controller there are 4 different register ranges that
+ * control the following properties of the pins:
  *  1) pin muxing
  *  2) pull enable/disable
  *  3) pull up/down
@@ -33,8 +32,8 @@
  * direction are the same and thus there are only 3 register ranges.
  *
  * Every pinmux group can be enabled by a specific bit in the first
- * register range of the domain; when all groups for a given pin are
- * disabled the pin acts as a GPIO.
+ * register range; when all groups for a given pin are disabled the
+ * pin acts as a GPIO.
  *
  * For the pull and GPIO configuration every bank uses a contiguous
  * set of bits in the register sets described above; the same register
@@ -66,21 +65,21 @@
 /**
  * meson_get_bank() - find the bank containing a given pin
  *
- * @domain:	the domain containing the pin
+ * @pc:		the pinctrl instance
  * @pin:	the pin number
  * @bank:	the found bank
  *
  * Return:	0 on success, a negative value on error
  */
-static int meson_get_bank(struct meson_domain *domain, unsigned int pin,
+static int meson_get_bank(struct meson_pinctrl *pc, unsigned int pin,
 			  struct meson_bank **bank)
 {
 	int i;
 
-	for (i = 0; i < domain->data->num_banks; i++) {
-		if (pin >= domain->data->banks[i].first &&
-		    pin <= domain->data->banks[i].last) {
-			*bank = &domain->data->banks[i];
+	for (i = 0; i < pc->data->num_banks; i++) {
+		if (pin >= pc->data->banks[i].first &&
+		    pin <= pc->data->banks[i].last) {
+			*bank = &pc->data->banks[i];
 			return 0;
 		}
 	}
@@ -89,33 +88,6 @@ static int meson_get_bank(struct meson_domain *domain, unsigned int pin,
 }
 
 /**
- * meson_get_domain_and_bank() - find domain and bank containing a given pin
- *
- * @pc:		Meson pin controller device
- * @pin:	the pin number
- * @domain:	the found domain
- * @bank:	the found bank
- *
- * Return:	0 on success, a negative value on error
- */
-static int meson_get_domain_and_bank(struct meson_pinctrl *pc, unsigned int pin,
-				     struct meson_domain **domain,
-				     struct meson_bank **bank)
-{
-	struct meson_domain *d;
-
-	d = pc->domain;
-
-	if (pin >= d->data->pin_base &&
-	    pin < d->data->pin_base + d->data->num_pins) {
-		*domain = d;
-		return meson_get_bank(d, pin, bank);
-	}
-
-	return -EINVAL;
-}
-
-/**
  * meson_calc_reg_and_bit() - calculate register and bit for a pin
  *
  * @bank:	the bank containing the pin
@@ -190,7 +162,6 @@ static void meson_pmx_disable_other_groups(struct meson_pinctrl *pc,
 					   unsigned int pin, int sel_group)
 {
 	struct meson_pmx_group *group;
-	struct meson_domain *domain;
 	int i, j;
 
 	for (i = 0; i < pc->data->num_groups; i++) {
@@ -201,8 +172,7 @@ static void meson_pmx_disable_other_groups(struct meson_pinctrl *pc,
 		for (j = 0; j < group->num_pins; j++) {
 			if (group->pins[j] == pin) {
 				/* We have found a group using the pin */
-				domain = pc->domain;
-				regmap_update_bits(domain->reg_mux,
+				regmap_update_bits(pc->reg_mux,
 						   group->reg * 4,
 						   BIT(group->bit), 0);
 			}
@@ -216,7 +186,6 @@ static int meson_pmx_set_mux(struct pinctrl_dev *pcdev, unsigned func_num,
 	struct meson_pinctrl *pc = pinctrl_dev_get_drvdata(pcdev);
 	struct meson_pmx_func *func = &pc->data->funcs[func_num];
 	struct meson_pmx_group *group = &pc->data->groups[group_num];
-	struct meson_domain *domain = pc->domain;
 	int i, ret = 0;
 
 	dev_dbg(pc->dev, "enable function %s, group %s\n", func->name,
@@ -231,7 +200,7 @@ static int meson_pmx_set_mux(struct pinctrl_dev *pcdev, unsigned func_num,
 
 	/* Function 0 (GPIO) doesn't need any additional setting */
 	if (func_num)
-		ret = regmap_update_bits(domain->reg_mux, group->reg * 4,
+		ret = regmap_update_bits(pc->reg_mux, group->reg * 4,
 					 BIT(group->bit), BIT(group->bit));
 
 	return ret;
@@ -287,14 +256,13 @@ static int meson_pinconf_set(struct pinctrl_dev *pcdev, unsigned int pin,
 			     unsigned long *configs, unsigned num_configs)
 {
 	struct meson_pinctrl *pc = pinctrl_dev_get_drvdata(pcdev);
-	struct meson_domain *domain;
 	struct meson_bank *bank;
 	enum pin_config_param param;
 	unsigned int reg, bit;
 	int i, ret;
 	u16 arg;
 
-	ret = meson_get_domain_and_bank(pc, pin, &domain, &bank);
+	ret = meson_get_bank(pc, pin, &bank);
 	if (ret)
 		return ret;
 
@@ -307,7 +275,7 @@ static int meson_pinconf_set(struct pinctrl_dev *pcdev, unsigned int pin,
 			dev_dbg(pc->dev, "pin %u: disable bias\n", pin);
 
 			meson_calc_reg_and_bit(bank, pin, REG_PULL, &reg, &bit);
-			ret = regmap_update_bits(domain->reg_pull, reg,
+			ret = regmap_update_bits(pc->reg_pull, reg,
 						 BIT(bit), 0);
 			if (ret)
 				return ret;
@@ -317,13 +285,13 @@ static int meson_pinconf_set(struct pinctrl_dev *pcdev, unsigned int pin,
 
 			meson_calc_reg_and_bit(bank, pin, REG_PULLEN,
 					       &reg, &bit);
-			ret = regmap_update_bits(domain->reg_pullen, reg,
+			ret = regmap_update_bits(pc->reg_pullen, reg,
 						 BIT(bit), BIT(bit));
 			if (ret)
 				return ret;
 
 			meson_calc_reg_and_bit(bank, pin, REG_PULL, &reg, &bit);
-			ret = regmap_update_bits(domain->reg_pull, reg,
+			ret = regmap_update_bits(pc->reg_pull, reg,
 						 BIT(bit), BIT(bit));
 			if (ret)
 				return ret;
@@ -333,13 +301,13 @@ static int meson_pinconf_set(struct pinctrl_dev *pcdev, unsigned int pin,
 
 			meson_calc_reg_and_bit(bank, pin, REG_PULLEN,
 					       &reg, &bit);
-			ret = regmap_update_bits(domain->reg_pullen, reg,
+			ret = regmap_update_bits(pc->reg_pullen, reg,
 						 BIT(bit), BIT(bit));
 			if (ret)
 				return ret;
 
 			meson_calc_reg_and_bit(bank, pin, REG_PULL, &reg, &bit);
-			ret = regmap_update_bits(domain->reg_pull, reg,
+			ret = regmap_update_bits(pc->reg_pull, reg,
 						 BIT(bit), 0);
 			if (ret)
 				return ret;
@@ -354,18 +322,17 @@ static int meson_pinconf_set(struct pinctrl_dev *pcdev, unsigned int pin,
 
 static int meson_pinconf_get_pull(struct meson_pinctrl *pc, unsigned int pin)
 {
-	struct meson_domain *domain;
 	struct meson_bank *bank;
 	unsigned int reg, bit, val;
 	int ret, conf;
 
-	ret = meson_get_domain_and_bank(pc, pin, &domain, &bank);
+	ret = meson_get_bank(pc, pin, &bank);
 	if (ret)
 		return ret;
 
 	meson_calc_reg_and_bit(bank, pin, REG_PULLEN, &reg, &bit);
 
-	ret = regmap_read(domain->reg_pullen, reg, &val);
+	ret = regmap_read(pc->reg_pullen, reg, &val);
 	if (ret)
 		return ret;
 
@@ -374,7 +341,7 @@ static int meson_pinconf_get_pull(struct meson_pinctrl *pc, unsigned int pin)
 	} else {
 		meson_calc_reg_and_bit(bank, pin, REG_PULL, &reg, &bit);
 
-		ret = regmap_read(domain->reg_pull, reg, &val);
+		ret = regmap_read(pc->reg_pull, reg, &val);
 		if (ret)
 			return ret;
 
@@ -452,82 +419,82 @@ static int meson_gpio_request(struct gpio_chip *chip, unsigned gpio)
 
 static void meson_gpio_free(struct gpio_chip *chip, unsigned gpio)
 {
-	struct meson_domain *domain = gpiochip_get_data(chip);
+	struct meson_pinctrl *pc = gpiochip_get_data(chip);
 
-	pinctrl_free_gpio(domain->data->pin_base + gpio);
+	pinctrl_free_gpio(pc->data->pin_base + gpio);
 }
 
 static int meson_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 {
-	struct meson_domain *domain = gpiochip_get_data(chip);
+	struct meson_pinctrl *pc = gpiochip_get_data(chip);
 	unsigned int reg, bit, pin;
 	struct meson_bank *bank;
 	int ret;
 
-	pin = domain->data->pin_base + gpio;
-	ret = meson_get_bank(domain, pin, &bank);
+	pin = pc->data->pin_base + gpio;
+	ret = meson_get_bank(pc, pin, &bank);
 	if (ret)
 		return ret;
 
 	meson_calc_reg_and_bit(bank, pin, REG_DIR, &reg, &bit);
 
-	return regmap_update_bits(domain->reg_gpio, reg, BIT(bit), BIT(bit));
+	return regmap_update_bits(pc->reg_gpio, reg, BIT(bit), BIT(bit));
 }
 
 static int meson_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
 				       int value)
 {
-	struct meson_domain *domain = gpiochip_get_data(chip);
+	struct meson_pinctrl *pc = gpiochip_get_data(chip);
 	unsigned int reg, bit, pin;
 	struct meson_bank *bank;
 	int ret;
 
-	pin = domain->data->pin_base + gpio;
-	ret = meson_get_bank(domain, pin, &bank);
+	pin = pc->data->pin_base + gpio;
+	ret = meson_get_bank(pc, pin, &bank);
 	if (ret)
 		return ret;
 
 	meson_calc_reg_and_bit(bank, pin, REG_DIR, &reg, &bit);
-	ret = regmap_update_bits(domain->reg_gpio, reg, BIT(bit), 0);
+	ret = regmap_update_bits(pc->reg_gpio, reg, BIT(bit), 0);
 	if (ret)
 		return ret;
 
 	meson_calc_reg_and_bit(bank, pin, REG_OUT, &reg, &bit);
-	return regmap_update_bits(domain->reg_gpio, reg, BIT(bit),
+	return regmap_update_bits(pc->reg_gpio, reg, BIT(bit),
 				  value ? BIT(bit) : 0);
 }
 
 static void meson_gpio_set(struct gpio_chip *chip, unsigned gpio, int value)
 {
-	struct meson_domain *domain = gpiochip_get_data(chip);
+	struct meson_pinctrl *pc = gpiochip_get_data(chip);
 	unsigned int reg, bit, pin;
 	struct meson_bank *bank;
 	int ret;
 
-	pin = domain->data->pin_base + gpio;
-	ret = meson_get_bank(domain, pin, &bank);
+	pin = pc->data->pin_base + gpio;
+	ret = meson_get_bank(pc, pin, &bank);
 	if (ret)
 		return;
 
 	meson_calc_reg_and_bit(bank, pin, REG_OUT, &reg, &bit);
-	regmap_update_bits(domain->reg_gpio, reg, BIT(bit),
+	regmap_update_bits(pc->reg_gpio, reg, BIT(bit),
 			   value ? BIT(bit) : 0);
 }
 
 static int meson_gpio_get(struct gpio_chip *chip, unsigned gpio)
 {
-	struct meson_domain *domain = gpiochip_get_data(chip);
+	struct meson_pinctrl *pc = gpiochip_get_data(chip);
 	unsigned int reg, bit, val, pin;
 	struct meson_bank *bank;
 	int ret;
 
-	pin = domain->data->pin_base + gpio;
-	ret = meson_get_bank(domain, pin, &bank);
+	pin = pc->data->pin_base + gpio;
+	ret = meson_get_bank(pc, pin, &bank);
 	if (ret)
 		return ret;
 
 	meson_calc_reg_and_bit(bank, pin, REG_IN, &reg, &bit);
-	regmap_read(domain->reg_gpio, reg, &val);
+	regmap_read(pc->reg_gpio, reg, &val);
 
 	return !!(val & BIT(bit));
 }
@@ -562,35 +529,32 @@ static const struct of_device_id meson_pinctrl_dt_match[] = {
 
 static int meson_gpiolib_register(struct meson_pinctrl *pc)
 {
-	struct meson_domain *domain;
 	int ret;
 
-	domain = pc->domain;
-
-	domain->chip.label = domain->data->name;
-	domain->chip.parent = pc->dev;
-	domain->chip.request = meson_gpio_request;
-	domain->chip.free = meson_gpio_free;
-	domain->chip.direction_input = meson_gpio_direction_input;
-	domain->chip.direction_output = meson_gpio_direction_output;
-	domain->chip.get = meson_gpio_get;
-	domain->chip.set = meson_gpio_set;
-	domain->chip.base = domain->data->pin_base;
-	domain->chip.ngpio = domain->data->num_pins;
-	domain->chip.can_sleep = false;
-	domain->chip.of_node = domain->of_node;
-	domain->chip.of_gpio_n_cells = 2;
-
-	ret = gpiochip_add_data(&domain->chip, domain);
+	pc->chip.label = pc->data->name;
+	pc->chip.parent = pc->dev;
+	pc->chip.request = meson_gpio_request;
+	pc->chip.free = meson_gpio_free;
+	pc->chip.direction_input = meson_gpio_direction_input;
+	pc->chip.direction_output = meson_gpio_direction_output;
+	pc->chip.get = meson_gpio_get;
+	pc->chip.set = meson_gpio_set;
+	pc->chip.base = pc->data->pin_base;
+	pc->chip.ngpio = pc->data->num_pins;
+	pc->chip.can_sleep = false;
+	pc->chip.of_node = pc->of_node;
+	pc->chip.of_gpio_n_cells = 2;
+
+	ret = gpiochip_add_data(&pc->chip, pc);
 	if (ret) {
 		dev_err(pc->dev, "can't add gpio chip %s\n",
-			domain->data->name);
+			pc->data->name);
 		goto fail;
 	}
 
-	ret = gpiochip_add_pin_range(&domain->chip, dev_name(pc->dev),
-				     0, domain->data->pin_base,
-				     domain->chip.ngpio);
+	ret = gpiochip_add_pin_range(&pc->chip, dev_name(pc->dev),
+				     0, pc->data->pin_base,
+				     pc->chip.ngpio);
 	if (ret) {
 		dev_err(pc->dev, "can't add pin range\n");
 		goto fail;
@@ -598,7 +562,7 @@ static int meson_gpiolib_register(struct meson_pinctrl *pc)
 
 	return 0;
 fail:
-	gpiochip_remove(&pc->domain->chip);
+	gpiochip_remove(&pc->chip);
 
 	return ret;
 }
@@ -637,58 +601,46 @@ static struct regmap *meson_map_resource(struct meson_pinctrl *pc,
 static int meson_pinctrl_parse_dt(struct meson_pinctrl *pc,
 				  struct device_node *node)
 {
-	struct device_node *np;
-	struct meson_domain *domain;
-	int num_domains = 0;
+	struct device_node *np, *gpio_np = NULL;
 
 	for_each_child_of_node(node, np) {
 		if (!of_find_property(np, "gpio-controller", NULL))
 			continue;
-		num_domains++;
+		if (gpio_np) {
+			dev_err(pc->dev, "multiple gpio nodes\n");
+			return -EINVAL;
+		}
+		gpio_np = np;
 	}
 
-	if (num_domains != 1) {
-		dev_err(pc->dev, "wrong number of subnodes\n");
+	if (!gpio_np) {
+		dev_err(pc->dev, "no gpio node found\n");
 		return -EINVAL;
 	}
 
-	pc->domain = devm_kzalloc(pc->dev, sizeof(struct meson_domain), GFP_KERNEL);
-	if (!pc->domain)
-		return -ENOMEM;
-
-	domain = pc->domain;
-	domain->data = pc->data->domain_data;
-
-	for_each_child_of_node(node, np) {
-		if (!of_find_property(np, "gpio-controller", NULL))
-			continue;
+	pc->of_node = gpio_np;
 
-		domain->of_node = np;
-
-		domain->reg_mux = meson_map_resource(pc, np, "mux");
-		if (IS_ERR(domain->reg_mux)) {
-			dev_err(pc->dev, "mux registers not found\n");
-			return PTR_ERR(domain->reg_mux);
-		}
-
-		domain->reg_pull = meson_map_resource(pc, np, "pull");
-		if (IS_ERR(domain->reg_pull)) {
-			dev_err(pc->dev, "pull registers not found\n");
-			return PTR_ERR(domain->reg_pull);
-		}
+	pc->reg_mux = meson_map_resource(pc, gpio_np, "mux");
+	if (IS_ERR(pc->reg_mux)) {
+		dev_err(pc->dev, "mux registers not found\n");
+		return PTR_ERR(pc->reg_mux);
+	}
 
-		domain->reg_pullen = meson_map_resource(pc, np, "pull-enable");
-		/* Use pull region if pull-enable one is not present */
-		if (IS_ERR(domain->reg_pullen))
-			domain->reg_pullen = domain->reg_pull;
+	pc->reg_pull = meson_map_resource(pc, gpio_np, "pull");
+	if (IS_ERR(pc->reg_pull)) {
+		dev_err(pc->dev, "pull registers not found\n");
+		return PTR_ERR(pc->reg_pull);
+	}
 
-		domain->reg_gpio = meson_map_resource(pc, np, "gpio");
-		if (IS_ERR(domain->reg_gpio)) {
-			dev_err(pc->dev, "gpio registers not found\n");
-			return PTR_ERR(domain->reg_gpio);
-		}
+	pc->reg_pullen = meson_map_resource(pc, gpio_np, "pull-enable");
+	/* Use pull region if pull-enable one is not present */
+	if (IS_ERR(pc->reg_pullen))
+		pc->reg_pullen = pc->reg_pull;
 
-		break;
+	pc->reg_gpio = meson_map_resource(pc, gpio_np, "gpio");
+	if (IS_ERR(pc->reg_gpio)) {
+		dev_err(pc->dev, "gpio registers not found\n");
+		return PTR_ERR(pc->reg_gpio);
 	}
 
 	return 0;
@@ -727,13 +679,7 @@ static int meson_pinctrl_probe(struct platform_device *pdev)
 		return PTR_ERR(pc->pcdev);
 	}
 
-	ret = meson_gpiolib_register(pc);
-	if (ret) {
-		pinctrl_unregister(pc->pcdev);
-		return ret;
-	}
-
-	return 0;
+	return meson_gpiolib_register(pc);
 }
 
 static struct platform_driver meson_pinctrl_driver = {
diff --git a/drivers/pinctrl/meson/pinctrl-meson.h b/drivers/pinctrl/meson/pinctrl-meson.h
index d89442ea4a4d..98b5080650c1 100644
--- a/drivers/pinctrl/meson/pinctrl-meson.h
+++ b/drivers/pinctrl/meson/pinctrl-meson.h
@@ -95,54 +95,17 @@ struct meson_bank {
 	struct meson_reg_desc regs[NUM_REG];
 };
 
-/**
- * struct meson_domain_data - domain platform data
- *
- * @name:	name of the domain
- * @banks:	set of banks belonging to the domain
- * @num_banks:	number of banks in the domain
- */
-struct meson_domain_data {
-	const char *name;
-	struct meson_bank *banks;
-	unsigned int num_banks;
-	unsigned int pin_base;
-	unsigned int num_pins;
-};
-
-/**
- * struct meson_domain
- *
- * @reg_mux:	registers for mux settings
- * @reg_pullen:	registers for pull-enable settings
- * @reg_pull:	registers for pull settings
- * @reg_gpio:	registers for gpio settings
- * @chip:	gpio chip associated with the domain
- * @data;	platform data for the domain
- * @node:	device tree node for the domain
- *
- * A domain represents a set of banks controlled by the same set of
- * registers.
- */
-struct meson_domain {
-	struct regmap *reg_mux;
-	struct regmap *reg_pullen;
-	struct regmap *reg_pull;
-	struct regmap *reg_gpio;
-
-	struct gpio_chip chip;
-	struct meson_domain_data *data;
-	struct device_node *of_node;
-};
-
 struct meson_pinctrl_data {
+	const char *name;
 	const struct pinctrl_pin_desc *pins;
 	struct meson_pmx_group *groups;
 	struct meson_pmx_func *funcs;
-	struct meson_domain_data *domain_data;
+	unsigned int pin_base;
 	unsigned int num_pins;
 	unsigned int num_groups;
 	unsigned int num_funcs;
+	struct meson_bank *banks;
+	unsigned int num_banks;
 };
 
 struct meson_pinctrl {
@@ -150,7 +113,12 @@ struct meson_pinctrl {
 	struct pinctrl_dev *pcdev;
 	struct pinctrl_desc desc;
 	struct meson_pinctrl_data *data;
-	struct meson_domain *domain;
+	struct regmap *reg_mux;
+	struct regmap *reg_pullen;
+	struct regmap *reg_pull;
+	struct regmap *reg_gpio;
+	struct gpio_chip chip;
+	struct device_node *of_node;
 };
 
 #define PIN(x, b)	(b + x)
diff --git a/drivers/pinctrl/meson/pinctrl-meson8.c b/drivers/pinctrl/meson/pinctrl-meson8.c
index 32de191e0807..07f1cb21c1b8 100644
--- a/drivers/pinctrl/meson/pinctrl-meson8.c
+++ b/drivers/pinctrl/meson/pinctrl-meson8.c
@@ -931,38 +931,28 @@ static struct meson_bank meson8_aobus_banks[] = {
 	BANK("AO",   PIN(GPIOAO_0, AO_OFF), PIN(GPIO_TEST_N, AO_OFF), 0,  0,  0, 16,  0,  0,  0, 16,  1,  0),
 };
 
-static struct meson_domain_data meson8_cbus_domain_data = {
+struct meson_pinctrl_data meson8_cbus_pinctrl_data = {
 	.name		= "cbus-banks",
-	.banks		= meson8_cbus_banks,
-	.num_banks	= ARRAY_SIZE(meson8_cbus_banks),
 	.pin_base	= 0,
-	.num_pins	= 120,
-};
-
-static struct meson_domain_data meson8_aobus_domain_data = {
-	.name		= "ao-bank",
-	.banks		= meson8_aobus_banks,
-	.num_banks	= ARRAY_SIZE(meson8_aobus_banks),
-	.pin_base	= 120,
-	.num_pins	= 16,
-};
-
-struct meson_pinctrl_data meson8_cbus_pinctrl_data = {
 	.pins		= meson8_cbus_pins,
 	.groups		= meson8_cbus_groups,
 	.funcs		= meson8_cbus_functions,
-	.domain_data	= &meson8_cbus_domain_data,
+	.banks		= meson8_cbus_banks,
 	.num_pins	= ARRAY_SIZE(meson8_cbus_pins),
 	.num_groups	= ARRAY_SIZE(meson8_cbus_groups),
 	.num_funcs	= ARRAY_SIZE(meson8_cbus_functions),
+	.num_banks	= ARRAY_SIZE(meson8_cbus_banks),
 };
 
 struct meson_pinctrl_data meson8_aobus_pinctrl_data = {
+	.name		= "ao-bank",
+	.pin_base	= 120,
 	.pins		= meson8_aobus_pins,
 	.groups		= meson8_aobus_groups,
 	.funcs		= meson8_aobus_functions,
-	.domain_data	= &meson8_aobus_domain_data,
+	.banks		= meson8_aobus_banks,
 	.num_pins	= ARRAY_SIZE(meson8_aobus_pins),
 	.num_groups	= ARRAY_SIZE(meson8_aobus_groups),
 	.num_funcs	= ARRAY_SIZE(meson8_aobus_functions),
+	.num_banks	= ARRAY_SIZE(meson8_aobus_banks),
 };
diff --git a/drivers/pinctrl/meson/pinctrl-meson8b.c b/drivers/pinctrl/meson/pinctrl-meson8b.c
index 874f2edf8be3..76f077f18193 100644
--- a/drivers/pinctrl/meson/pinctrl-meson8b.c
+++ b/drivers/pinctrl/meson/pinctrl-meson8b.c
@@ -896,38 +896,28 @@ static struct meson_bank meson8b_aobus_banks[] = {
 	BANK("AO",   PIN(GPIOAO_0, AO_OFF), PIN(GPIO_TEST_N, AO_OFF), 0,  0,  0, 16,  0,  0,  0, 16,  1,  0),
 };
 
-static struct meson_domain_data meson8b_cbus_domain_data = {
+struct meson_pinctrl_data meson8b_cbus_pinctrl_data = {
 	.name		= "cbus-banks",
-	.banks		= meson8b_cbus_banks,
-	.num_banks	= ARRAY_SIZE(meson8b_cbus_banks),
 	.pin_base	= 0,
-	.num_pins	= 130,
-};
-
-static struct meson_domain_data meson8b_aobus_domain_data = {
-	.name		= "aobus-banks",
-	.banks		= meson8b_aobus_banks,
-	.num_banks	= ARRAY_SIZE(meson8b_aobus_banks),
-	.pin_base	= 130,
-	.num_pins	= 16,
-};
-
-struct meson_pinctrl_data meson8b_cbus_pinctrl_data = {
 	.pins		= meson8b_cbus_pins,
 	.groups		= meson8b_cbus_groups,
 	.funcs		= meson8b_cbus_functions,
-	.domain_data	= &meson8b_cbus_domain_data,
+	.banks		= meson8b_cbus_banks,
 	.num_pins	= ARRAY_SIZE(meson8b_cbus_pins),
 	.num_groups	= ARRAY_SIZE(meson8b_cbus_groups),
 	.num_funcs	= ARRAY_SIZE(meson8b_cbus_functions),
+	.num_banks	= ARRAY_SIZE(meson8b_cbus_banks),
 };
 
 struct meson_pinctrl_data meson8b_aobus_pinctrl_data = {
+	.name		= "aobus-banks",
+	.pin_base	= 130,
 	.pins		= meson8b_aobus_pins,
 	.groups		= meson8b_aobus_groups,
 	.funcs		= meson8b_aobus_functions,
-	.domain_data	= &meson8b_aobus_domain_data,
+	.banks		= meson8b_aobus_banks,
 	.num_pins	= ARRAY_SIZE(meson8b_aobus_pins),
 	.num_groups	= ARRAY_SIZE(meson8b_aobus_groups),
 	.num_funcs	= ARRAY_SIZE(meson8b_aobus_functions),
+	.num_banks	= ARRAY_SIZE(meson8b_aobus_banks),
 };
diff --git a/drivers/pinctrl/mvebu/pinctrl-orion.c b/drivers/pinctrl/mvebu/pinctrl-orion.c
index 345c3df669a0..84e144167b44 100644
--- a/drivers/pinctrl/mvebu/pinctrl-orion.c
+++ b/drivers/pinctrl/mvebu/pinctrl-orion.c
@@ -64,11 +64,11 @@ static int orion_mpp_ctrl_set(unsigned pid, unsigned long config)
 	return 0;
 }
 
-#define V(f5181l, f5182, f5281) \
-	((f5181l << 0) | (f5182 << 1) | (f5281 << 2))
+#define V(f5181, f5182, f5281) \
+	((f5181 << 0) | (f5182 << 1) | (f5281 << 2))
 
 enum orion_variant {
-	V_5181L = V(1, 0, 0),
+	V_5181  = V(1, 0, 0),
 	V_5182  = V(0, 1, 0),
 	V_5281  = V(0, 0, 1),
 	V_ALL   = V(1, 1, 1),
@@ -103,13 +103,13 @@ static struct mvebu_mpp_mode orion_mpp_modes[] = {
 		 MPP_VAR_FUNCTION(0x0, "gpio", NULL,        V_ALL),
 		 MPP_VAR_FUNCTION(0x2, "pci", "req5",       V_ALL),
 		 MPP_VAR_FUNCTION(0x4, "nand", "re0",       V_5182 | V_5281),
-		 MPP_VAR_FUNCTION(0x5, "pci-1", "clk",      V_5181L),
+		 MPP_VAR_FUNCTION(0x5, "pci-1", "clk",      V_5181),
 		 MPP_VAR_FUNCTION(0x5, "sata0", "act",      V_5182)),
 	MPP_MODE(7,
 		 MPP_VAR_FUNCTION(0x0, "gpio", NULL,        V_ALL),
 		 MPP_VAR_FUNCTION(0x2, "pci", "gnt5",       V_ALL),
 		 MPP_VAR_FUNCTION(0x4, "nand", "we0",       V_5182 | V_5281),
-		 MPP_VAR_FUNCTION(0x5, "pci-1", "clk",      V_5181L),
+		 MPP_VAR_FUNCTION(0x5, "pci-1", "clk",      V_5181),
 		 MPP_VAR_FUNCTION(0x5, "sata1", "act",      V_5182)),
 	MPP_MODE(8,
 		 MPP_VAR_FUNCTION(0x0, "gpio", NULL,        V_ALL),
@@ -165,7 +165,7 @@ static struct mvebu_mpp_ctrl orion_mpp_controls[] = {
 	MPP_FUNC_CTRL(0, 19, NULL, orion_mpp_ctrl),
 };
 
-static struct pinctrl_gpio_range mv88f5181l_gpio_ranges[] = {
+static struct pinctrl_gpio_range mv88f5181_gpio_ranges[] = {
 	MPP_GPIO_RANGE(0, 0, 0, 16),
 };
 
@@ -177,14 +177,14 @@ static struct pinctrl_gpio_range mv88f5281_gpio_ranges[] = {
 	MPP_GPIO_RANGE(0, 0, 0, 16),
 };
 
-static struct mvebu_pinctrl_soc_info mv88f5181l_info = {
-	.variant = V_5181L,
+static struct mvebu_pinctrl_soc_info mv88f5181_info = {
+	.variant = V_5181,
 	.controls = orion_mpp_controls,
 	.ncontrols = ARRAY_SIZE(orion_mpp_controls),
 	.modes = orion_mpp_modes,
 	.nmodes = ARRAY_SIZE(orion_mpp_modes),
-	.gpioranges = mv88f5181l_gpio_ranges,
-	.ngpioranges = ARRAY_SIZE(mv88f5181l_gpio_ranges),
+	.gpioranges = mv88f5181_gpio_ranges,
+	.ngpioranges = ARRAY_SIZE(mv88f5181_gpio_ranges),
 };
 
 static struct mvebu_pinctrl_soc_info mv88f5182_info = {
@@ -212,7 +212,8 @@ static struct mvebu_pinctrl_soc_info mv88f5281_info = {
  * muxing, they are identical.
  */
 static const struct of_device_id orion_pinctrl_of_match[] = {
-	{ .compatible = "marvell,88f5181l-pinctrl", .data = &mv88f5181l_info },
+	{ .compatible = "marvell,88f5181-pinctrl", .data = &mv88f5181_info },
+	{ .compatible = "marvell,88f5181l-pinctrl", .data = &mv88f5181_info },
 	{ .compatible = "marvell,88f5182-pinctrl", .data = &mv88f5182_info },
 	{ .compatible = "marvell,88f5281-pinctrl", .data = &mv88f5281_info },
 	{ }
diff --git a/drivers/pinctrl/nomadik/pinctrl-abx500.c b/drivers/pinctrl/nomadik/pinctrl-abx500.c
index 7d343c22c90c..f95001bc1d58 100644
--- a/drivers/pinctrl/nomadik/pinctrl-abx500.c
+++ b/drivers/pinctrl/nomadik/pinctrl-abx500.c
@@ -4,6 +4,8 @@
  * Author: Patrice Chotard <patrice.chotard@st.com>
  * License terms: GNU General Public License (GPL) version 2
  *
+ * Driver allows to use AxB5xx unused pins to be used as GPIO
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
@@ -12,7 +14,6 @@
 #include <linux/types.h>
 #include <linux/slab.h>
 #include <linux/init.h>
-#include <linux/module.h>
 #include <linux/err.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -1269,8 +1270,3 @@ static int __init abx500_gpio_init(void)
 	return platform_driver_register(&abx500_gpio_driver);
 }
 core_initcall(abx500_gpio_init);
-
-MODULE_AUTHOR("Patrice Chotard <patrice.chotard@st.com>");
-MODULE_DESCRIPTION("Driver allows to use AxB5xx unused pins to be used as GPIO");
-MODULE_ALIAS("platform:abx500-gpio");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
index 35f62180db4e..d318ca055489 100644
--- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c
+++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
@@ -11,7 +11,6 @@
  * published by the Free Software Foundation.
  */
 #include <linux/kernel.h>
-#include <linux/module.h>
 #include <linux/init.h>
 #include <linux/device.h>
 #include <linux/platform_device.h>
@@ -1110,10 +1109,8 @@ static int nmk_gpio_probe(struct platform_device *dev)
 		return PTR_ERR(nmk_chip);
 	}
 
-	if (of_get_property(np, "st,supports-sleepmode", NULL))
-		supports_sleepmode = true;
-	else
-		supports_sleepmode = false;
+	supports_sleepmode =
+		of_property_read_bool(np, "st,supports-sleepmode");
 
 	/* Correct platform device ID */
 	dev->id = nmk_chip->bank;
@@ -1180,7 +1177,7 @@ static int nmk_gpio_probe(struct platform_device *dev)
 				   irqchip,
 				   0,
 				   handle_edge_irq,
-				   IRQ_TYPE_EDGE_FALLING);
+				   IRQ_TYPE_NONE);
 	if (ret) {
 		dev_err(&dev->dev, "could not add irqchip\n");
 		gpiochip_remove(&nmk_chip->chip);
@@ -1985,7 +1982,3 @@ static int __init nmk_pinctrl_init(void)
 	return platform_driver_register(&nmk_pinctrl_driver);
 }
 core_initcall(nmk_pinctrl_init);
-
-MODULE_AUTHOR("Prafulla WADASKAR and Alessandro Rubini");
-MODULE_DESCRIPTION("Nomadik GPIO Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
index 634b4d30eefb..aea310a91821 100644
--- a/drivers/pinctrl/pinctrl-amd.c
+++ b/drivers/pinctrl/pinctrl-amd.c
@@ -43,17 +43,6 @@ static int amd_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + offset * 4);
-	/*
-	 * Suppose BIOS or Bootloader sets specific debounce for the
-	 * GPIO. if not, set debounce to be  2.75ms and remove glitch.
-	*/
-	if ((pin_reg & DB_TMR_OUT_MASK) == 0) {
-		pin_reg |= 0xf;
-		pin_reg |= BIT(DB_TMR_OUT_UNIT_OFF);
-		pin_reg |= DB_TYPE_REMOVE_GLITCH << DB_CNTRL_OFF;
-		pin_reg &= ~BIT(DB_TMR_LARGE_OFF);
-	}
-
 	pin_reg &= ~BIT(OUTPUT_ENABLE_OFF);
 	writel(pin_reg, gpio_dev->base + offset * 4);
 	spin_unlock_irqrestore(&gpio_dev->lock, flags);
@@ -326,15 +315,6 @@ static void amd_gpio_irq_enable(struct irq_data *d)
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + (d->hwirq)*4);
-	/*
-		Suppose BIOS or Bootloader sets specific debounce for the
-		GPIO. if not, set debounce to be  2.75ms.
-	*/
-	if ((pin_reg & DB_TMR_OUT_MASK) == 0) {
-		pin_reg |= 0xf;
-		pin_reg |= BIT(DB_TMR_OUT_UNIT_OFF);
-		pin_reg &= ~BIT(DB_TMR_LARGE_OFF);
-	}
 	pin_reg |= BIT(INTERRUPT_ENABLE_OFF);
 	pin_reg |= BIT(INTERRUPT_MASK_OFF);
 	writel(pin_reg, gpio_dev->base + (d->hwirq)*4);
@@ -403,12 +383,26 @@ static int amd_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 	int ret = 0;
 	u32 pin_reg;
 	unsigned long flags;
+	bool level_trig;
+	u32 active_level;
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
 
 	spin_lock_irqsave(&gpio_dev->lock, flags);
 	pin_reg = readl(gpio_dev->base + (d->hwirq)*4);
 
+	/*
+	 * When level_trig is set EDGE and active_level is set HIGH in BIOS
+	 * default settings, ignore incoming settings from client and use
+	 * BIOS settings to configure GPIO register.
+	 */
+	level_trig = !(pin_reg & (LEVEL_TRIGGER << LEVEL_TRIG_OFF));
+	active_level = pin_reg & (ACTIVE_LEVEL_MASK << ACTIVE_LEVEL_OFF);
+
+	if(level_trig &&
+	   ((active_level >> ACTIVE_LEVEL_OFF) == ACTIVE_HIGH))
+		type = IRQ_TYPE_EDGE_FALLING;
+
 	switch (type & IRQ_TYPE_SENSE_MASK) {
 	case IRQ_TYPE_EDGE_RISING:
 		pin_reg &= ~BIT(LEVEL_TRIG_OFF);
diff --git a/drivers/pinctrl/pinctrl-at91.c b/drivers/pinctrl/pinctrl-at91.c
index 80daead3a5a1..9f0904185909 100644
--- a/drivers/pinctrl/pinctrl-at91.c
+++ b/drivers/pinctrl/pinctrl-at91.c
@@ -1614,7 +1614,7 @@ static int at91_gpio_of_irq_setup(struct platform_device *pdev,
 				   &gpio_irqchip,
 				   0,
 				   handle_edge_irq,
-				   IRQ_TYPE_EDGE_BOTH);
+				   IRQ_TYPE_NONE);
 	if (ret) {
 		dev_err(&pdev->dev, "at91_gpio.%d: Couldn't add irqchip to gpiochip.\n",
 			at91_gpio->pioc_idx);
diff --git a/drivers/pinctrl/pinctrl-palmas.c b/drivers/pinctrl/pinctrl-palmas.c
index 8edb3f8c72c8..a30146da7ffd 100644
--- a/drivers/pinctrl/pinctrl-palmas.c
+++ b/drivers/pinctrl/pinctrl-palmas.c
@@ -1004,9 +1004,7 @@ static int palmas_pinctrl_probe(struct platform_device *pdev)
 	bool enable_dvfs2 = false;
 
 	if (pdev->dev.of_node) {
-		const struct of_device_id *match;
-		match = of_match_device(palmas_pinctrl_of_match, &pdev->dev);
-		pinctrl_data = match->data;
+		pinctrl_data = of_device_get_match_data(&pdev->dev);
 		enable_dvfs1 = of_property_read_bool(pdev->dev.of_node,
 					"ti,palmas-enable-dvfs1");
 		enable_dvfs2 = of_property_read_bool(pdev->dev.of_node,
diff --git a/drivers/pinctrl/pinctrl-pistachio.c b/drivers/pinctrl/pinctrl-pistachio.c
index c6d410ef8de0..55375b1b3cc8 100644
--- a/drivers/pinctrl/pinctrl-pistachio.c
+++ b/drivers/pinctrl/pinctrl-pistachio.c
@@ -809,17 +809,17 @@ static const struct pistachio_pin_group pistachio_groups[] = {
 			   PADS_FUNCTION_SELECT2, 12, 0x3),
 	MFIO_MUX_PIN_GROUP(83, MIPS_PLL_LOCK, MIPS_TRACE_DATA, USB_DEBUG,
 			   PADS_FUNCTION_SELECT2, 14, 0x3),
-	MFIO_MUX_PIN_GROUP(84, SYS_PLL_LOCK, MIPS_TRACE_DATA, USB_DEBUG,
+	MFIO_MUX_PIN_GROUP(84, AUDIO_PLL_LOCK, MIPS_TRACE_DATA, USB_DEBUG,
 			   PADS_FUNCTION_SELECT2, 16, 0x3),
-	MFIO_MUX_PIN_GROUP(85, WIFI_PLL_LOCK, MIPS_TRACE_DATA, SDHOST_DEBUG,
+	MFIO_MUX_PIN_GROUP(85, RPU_V_PLL_LOCK, MIPS_TRACE_DATA, SDHOST_DEBUG,
 			   PADS_FUNCTION_SELECT2, 18, 0x3),
-	MFIO_MUX_PIN_GROUP(86, BT_PLL_LOCK, MIPS_TRACE_DATA, SDHOST_DEBUG,
+	MFIO_MUX_PIN_GROUP(86, RPU_L_PLL_LOCK, MIPS_TRACE_DATA, SDHOST_DEBUG,
 			   PADS_FUNCTION_SELECT2, 20, 0x3),
-	MFIO_MUX_PIN_GROUP(87, RPU_V_PLL_LOCK, DREQ2, SOCIF_DEBUG,
+	MFIO_MUX_PIN_GROUP(87, SYS_PLL_LOCK, DREQ2, SOCIF_DEBUG,
 			   PADS_FUNCTION_SELECT2, 22, 0x3),
-	MFIO_MUX_PIN_GROUP(88, RPU_L_PLL_LOCK, DREQ3, SOCIF_DEBUG,
+	MFIO_MUX_PIN_GROUP(88, WIFI_PLL_LOCK, DREQ3, SOCIF_DEBUG,
 			   PADS_FUNCTION_SELECT2, 24, 0x3),
-	MFIO_MUX_PIN_GROUP(89, AUDIO_PLL_LOCK, DREQ4, DREQ5,
+	MFIO_MUX_PIN_GROUP(89, BT_PLL_LOCK, DREQ4, DREQ5,
 			   PADS_FUNCTION_SELECT2, 26, 0x3),
 	PIN_GROUP(TCK, "tck"),
 	PIN_GROUP(TRSTN, "trstn"),
@@ -1432,7 +1432,6 @@ static int pistachio_pinctrl_probe(struct platform_device *pdev)
 {
 	struct pistachio_pinctrl *pctl;
 	struct resource *res;
-	int ret;
 
 	pctl = devm_kzalloc(&pdev->dev, sizeof(*pctl), GFP_KERNEL);
 	if (!pctl)
@@ -1464,13 +1463,7 @@ static int pistachio_pinctrl_probe(struct platform_device *pdev)
 		return PTR_ERR(pctl->pctldev);
 	}
 
-	ret = pistachio_gpio_register(pctl);
-	if (ret < 0) {
-		pinctrl_unregister(pctl->pctldev);
-		return ret;
-	}
-
-	return 0;
+	return pistachio_gpio_register(pctl);
 }
 
 static struct platform_driver pistachio_pinctrl_driver = {
diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c
index 44902c63f507..49bf7dcb7ed8 100644
--- a/drivers/pinctrl/pinctrl-rockchip.c
+++ b/drivers/pinctrl/pinctrl-rockchip.c
@@ -23,7 +23,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/io.h>
 #include <linux/bitops.h>
@@ -2704,7 +2704,6 @@ static const struct of_device_id rockchip_pinctrl_dt_match[] = {
 		.data = (void *)&rk3399_pin_ctrl },
 	{},
 };
-MODULE_DEVICE_TABLE(of, rockchip_pinctrl_dt_match);
 
 static struct platform_driver rockchip_pinctrl_driver = {
 	.probe		= rockchip_pinctrl_probe,
@@ -2720,7 +2719,3 @@ static int __init rockchip_pinctrl_drv_register(void)
 	return platform_driver_register(&rockchip_pinctrl_driver);
 }
 postcore_initcall(rockchip_pinctrl_drv_register);
-
-MODULE_AUTHOR("Heiko Stuebner <heiko@sntech.de>");
-MODULE_DESCRIPTION("Rockchip pinctrl driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/pinctrl-st.c b/drivers/pinctrl/pinctrl-st.c
index 0de1c67dfb94..99da4cf91031 100644
--- a/drivers/pinctrl/pinctrl-st.c
+++ b/drivers/pinctrl/pinctrl-st.c
@@ -335,61 +335,25 @@ struct st_pinctrl {
 };
 
 /* SOC specific data */
-/* STiH415 data */
-static const unsigned int stih415_input_delays[] = {0, 500, 1000, 1500};
-static const unsigned int stih415_output_delays[] = {0, 1000, 2000, 3000};
-
-#define STIH415_PCTRL_COMMON_DATA				\
-	.rt_style	= st_retime_style_packed,		\
-	.input_delays	= stih415_input_delays,			\
-	.ninput_delays	= ARRAY_SIZE(stih415_input_delays),	\
-	.output_delays = stih415_output_delays,			\
-	.noutput_delays = ARRAY_SIZE(stih415_output_delays)
-
-static const struct st_pctl_data  stih415_sbc_data = {
-	STIH415_PCTRL_COMMON_DATA,
-	.alt = 0, .oe = 5, .pu = 7, .od = 9, .rt = 16,
-};
-
-static const struct st_pctl_data  stih415_front_data = {
-	STIH415_PCTRL_COMMON_DATA,
-	.alt = 0, .oe = 8, .pu = 10, .od = 12, .rt = 16,
-};
-
-static const struct st_pctl_data  stih415_rear_data = {
-	STIH415_PCTRL_COMMON_DATA,
-	.alt = 0, .oe = 6, .pu = 8, .od = 10, .rt = 38,
-};
-
-static const struct st_pctl_data  stih415_left_data = {
-	STIH415_PCTRL_COMMON_DATA,
-	.alt = 0, .oe = 3, .pu = 4, .od = 5, .rt = 6,
-};
-
-static const struct st_pctl_data  stih415_right_data = {
-	STIH415_PCTRL_COMMON_DATA,
-	.alt = 0, .oe = 5, .pu = 7, .od = 9, .rt = 11,
-};
 
-/* STiH416 data */
-static const unsigned int stih416_delays[] = {0, 300, 500, 750, 1000, 1250,
+static const unsigned int stih407_delays[] = {0, 300, 500, 750, 1000, 1250,
 			1500, 1750, 2000, 2250, 2500, 2750, 3000, 3250 };
 
-static const struct st_pctl_data  stih416_data = {
-	.rt_style	= st_retime_style_dedicated,
-	.input_delays	= stih416_delays,
-	.ninput_delays	= ARRAY_SIZE(stih416_delays),
-	.output_delays	= stih416_delays,
-	.noutput_delays = ARRAY_SIZE(stih416_delays),
+static const struct st_pctl_data  stih407_data = {
+	.rt_style       = st_retime_style_dedicated,
+	.input_delays   = stih407_delays,
+	.ninput_delays  = ARRAY_SIZE(stih407_delays),
+	.output_delays  = stih407_delays,
+	.noutput_delays = ARRAY_SIZE(stih407_delays),
 	.alt = 0, .oe = 40, .pu = 50, .od = 60, .rt = 100,
 };
 
 static const struct st_pctl_data stih407_flashdata = {
 	.rt_style	= st_retime_style_none,
-	.input_delays	= stih416_delays,
-	.ninput_delays	= ARRAY_SIZE(stih416_delays),
-	.output_delays	= stih416_delays,
-	.noutput_delays = ARRAY_SIZE(stih416_delays),
+	.input_delays	= stih407_delays,
+	.ninput_delays	= ARRAY_SIZE(stih407_delays),
+	.output_delays	= stih407_delays,
+	.noutput_delays = ARRAY_SIZE(stih407_delays),
 	.alt = 0,
 	.oe = -1, /* Not Available */
 	.pu = -1, /* Not Available */
@@ -799,21 +763,6 @@ static int st_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 	return (direction == ST_GPIO_DIRECTION_IN);
 }
 
-static int st_gpio_xlate(struct gpio_chip *gc,
-			const struct of_phandle_args *gpiospec, u32 *flags)
-{
-	if (WARN_ON(gc->of_gpio_n_cells < 1))
-		return -EINVAL;
-
-	if (WARN_ON(gpiospec->args_count < gc->of_gpio_n_cells))
-		return -EINVAL;
-
-	if (gpiospec->args[0] > gc->ngpio)
-		return -EINVAL;
-
-	return gpiospec->args[0];
-}
-
 /* Pinctrl Groups */
 static int st_pctl_get_groups_count(struct pinctrl_dev *pctldev)
 {
@@ -1486,8 +1435,6 @@ static struct gpio_chip st_gpio_template = {
 	.direction_output	= st_gpio_direction_output,
 	.get_direction		= st_gpio_get_direction,
 	.ngpio			= ST_GPIO_PINS_PER_BANK,
-	.of_gpio_n_cells	= 1,
-	.of_xlate		= st_gpio_xlate,
 };
 
 static struct irq_chip st_gpio_irqchip = {
@@ -1579,21 +1526,9 @@ static int st_gpiolib_register_bank(struct st_pinctrl *info,
 }
 
 static const struct of_device_id st_pctl_of_match[] = {
-	{ .compatible = "st,stih415-sbc-pinctrl", .data = &stih415_sbc_data },
-	{ .compatible = "st,stih415-rear-pinctrl", .data = &stih415_rear_data },
-	{ .compatible = "st,stih415-left-pinctrl", .data = &stih415_left_data },
-	{ .compatible = "st,stih415-right-pinctrl",
-		.data = &stih415_right_data },
-	{ .compatible = "st,stih415-front-pinctrl",
-		.data = &stih415_front_data },
-	{ .compatible = "st,stih416-sbc-pinctrl", .data = &stih416_data},
-	{ .compatible = "st,stih416-front-pinctrl", .data = &stih416_data},
-	{ .compatible = "st,stih416-rear-pinctrl", .data = &stih416_data},
-	{ .compatible = "st,stih416-fvdp-fe-pinctrl", .data = &stih416_data},
-	{ .compatible = "st,stih416-fvdp-lite-pinctrl", .data = &stih416_data},
-	{ .compatible = "st,stih407-sbc-pinctrl", .data = &stih416_data},
-	{ .compatible = "st,stih407-front-pinctrl", .data = &stih416_data},
-	{ .compatible = "st,stih407-rear-pinctrl", .data = &stih416_data},
+	{ .compatible = "st,stih407-sbc-pinctrl", .data = &stih407_data},
+	{ .compatible = "st,stih407-front-pinctrl", .data = &stih407_data},
+	{ .compatible = "st,stih407-rear-pinctrl", .data = &stih407_data},
 	{ .compatible = "st,stih407-flash-pinctrl", .data = &stih407_flashdata},
 	{ /* sentinel */ }
 };
diff --git a/drivers/pinctrl/pinctrl-zynq.c b/drivers/pinctrl/pinctrl-zynq.c
index 7afdbede6823..e0ecffcbe11f 100644
--- a/drivers/pinctrl/pinctrl-zynq.c
+++ b/drivers/pinctrl/pinctrl-zynq.c
@@ -233,7 +233,7 @@ static const unsigned int sdio0_2_pins[] = {40, 41, 42, 43, 44, 45};
 static const unsigned int sdio1_0_pins[] = {10, 11, 12, 13, 14, 15};
 static const unsigned int sdio1_1_pins[] = {22, 23, 24, 25, 26, 27};
 static const unsigned int sdio1_2_pins[] = {34, 35, 36, 37, 38, 39};
-static const unsigned int sdio1_3_pins[] = {46, 47, 48, 49, 40, 51};
+static const unsigned int sdio1_3_pins[] = {46, 47, 48, 49, 50, 51};
 static const unsigned int sdio0_emio_wp_pins[] = {54};
 static const unsigned int sdio0_emio_cd_pins[] = {55};
 static const unsigned int sdio1_emio_wp_pins[] = {56};
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c
index 51c42d746883..775c88303017 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm.c
@@ -156,7 +156,7 @@ static int msm_pinmux_set_mux(struct pinctrl_dev *pctldev,
 	spin_lock_irqsave(&pctrl->lock, flags);
 
 	val = readl(pctrl->regs + g->ctl_reg);
-	val &= mask;
+	val &= ~mask;
 	val |= i << g->mux_bit;
 	writel(val, pctrl->regs + g->ctl_reg);
 
diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
index 686accb89f52..664b641fd776 100644
--- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
+++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
@@ -800,6 +800,7 @@ static const struct of_device_id pmic_gpio_of_match[] = {
 	{ .compatible = "qcom,pm8941-gpio" },	/* 36 GPIO's */
 	{ .compatible = "qcom,pm8994-gpio" },	/* 22 GPIO's */
 	{ .compatible = "qcom,pma8084-gpio" },	/* 22 GPIO's */
+	{ .compatible = "qcom,spmi-gpio" }, /* Generic */
 	{ },
 };
 
diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-mpp.c b/drivers/pinctrl/qcom/pinctrl-spmi-mpp.c
index 1735ffef9d5c..6556dbeae65e 100644
--- a/drivers/pinctrl/qcom/pinctrl-spmi-mpp.c
+++ b/drivers/pinctrl/qcom/pinctrl-spmi-mpp.c
@@ -914,6 +914,7 @@ static const struct of_device_id pmic_mpp_of_match[] = {
 	{ .compatible = "qcom,pm8941-mpp" },	/* 8 MPP's */
 	{ .compatible = "qcom,pm8994-mpp" },	/* 8 MPP's */
 	{ .compatible = "qcom,pma8084-mpp" },	/* 8 MPP's */
+	{ .compatible = "qcom,spmi-mpp" },	/* Generic */
 	{ },
 };
 
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.c b/drivers/pinctrl/samsung/pinctrl-exynos.c
index 051b5bf701a8..d32fa2b5ff82 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos.c
@@ -428,14 +428,10 @@ static void exynos_irq_eint0_15(struct irq_desc *desc)
 	int eint_irq;
 
 	chained_irq_enter(chip, desc);
-	chip->irq_mask(&desc->irq_data);
-
-	if (chip->irq_ack)
-		chip->irq_ack(&desc->irq_data);
 
 	eint_irq = irq_linear_revmap(bank->irq_domain, eintd->irq);
 	generic_handle_irq(eint_irq);
-	chip->irq_unmask(&desc->irq_data);
+
 	chained_irq_exit(chip, desc);
 }
 
diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c
index 513fe6b23248..620727fabe64 100644
--- a/drivers/pinctrl/samsung/pinctrl-samsung.c
+++ b/drivers/pinctrl/samsung/pinctrl-samsung.c
@@ -1041,17 +1041,9 @@ static int samsung_pinctrl_probe(struct platform_device *pdev)
 	struct resource *res;
 	int ret;
 
-	if (!dev->of_node) {
-		dev_err(dev, "device tree node not found\n");
-		return -ENODEV;
-	}
-
 	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
-	if (!drvdata) {
-		dev_err(dev, "failed to allocate memory for driver's "
-				"private data\n");
+	if (!drvdata)
 		return -ENOMEM;
-	}
 
 	ctrl = samsung_pinctrl_get_soc_data(drvdata, pdev);
 	if (IS_ERR(ctrl)) {
diff --git a/drivers/pinctrl/sh-pfc/Kconfig b/drivers/pinctrl/sh-pfc/Kconfig
index 415dd8023063..07eca54bdc1c 100644
--- a/drivers/pinctrl/sh-pfc/Kconfig
+++ b/drivers/pinctrl/sh-pfc/Kconfig
@@ -54,6 +54,11 @@ config PINCTRL_PFC_R8A7791
 	depends on ARCH_R8A7791
 	select PINCTRL_SH_PFC
 
+config PINCTRL_PFC_R8A7792
+	def_bool y
+	depends on ARCH_R8A7792
+	select PINCTRL_SH_PFC
+
 config PINCTRL_PFC_R8A7793
 	def_bool y
 	depends on ARCH_R8A7793
@@ -69,6 +74,11 @@ config PINCTRL_PFC_R8A7795
 	depends on ARCH_R8A7795
 	select PINCTRL_SH_PFC
 
+config PINCTRL_PFC_R8A7796
+        def_bool y
+        depends on ARCH_R8A7796
+        select PINCTRL_SH_PFC
+
 config PINCTRL_PFC_SH7203
 	def_bool y
 	depends on CPU_SUBTYPE_SH7203
diff --git a/drivers/pinctrl/sh-pfc/Makefile b/drivers/pinctrl/sh-pfc/Makefile
index 8a2c8710fc93..2dda8c63f3cf 100644
--- a/drivers/pinctrl/sh-pfc/Makefile
+++ b/drivers/pinctrl/sh-pfc/Makefile
@@ -7,9 +7,11 @@ obj-$(CONFIG_PINCTRL_PFC_R8A7778)	+= pfc-r8a7778.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7779)	+= pfc-r8a7779.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7790)	+= pfc-r8a7790.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7791)	+= pfc-r8a7791.o
+obj-$(CONFIG_PINCTRL_PFC_R8A7792)	+= pfc-r8a7792.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7793)	+= pfc-r8a7791.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7794)	+= pfc-r8a7794.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7795)	+= pfc-r8a7795.o
+obj-$(CONFIG_PINCTRL_PFC_R8A7796)	+= pfc-r8a7796.o
 obj-$(CONFIG_PINCTRL_PFC_SH7203)	+= pfc-sh7203.o
 obj-$(CONFIG_PINCTRL_PFC_SH7264)	+= pfc-sh7264.o
 obj-$(CONFIG_PINCTRL_PFC_SH7269)	+= pfc-sh7269.o
diff --git a/drivers/pinctrl/sh-pfc/core.c b/drivers/pinctrl/sh-pfc/core.c
index a3b82041b6a2..f3a8897d4e8f 100644
--- a/drivers/pinctrl/sh-pfc/core.c
+++ b/drivers/pinctrl/sh-pfc/core.c
@@ -494,6 +494,12 @@ static const struct of_device_id sh_pfc_of_table[] = {
 		.data = &r8a7791_pinmux_info,
 	},
 #endif
+#ifdef CONFIG_PINCTRL_PFC_R8A7792
+	{
+		.compatible = "renesas,pfc-r8a7792",
+		.data = &r8a7792_pinmux_info,
+	},
+#endif
 #ifdef CONFIG_PINCTRL_PFC_R8A7793
 	{
 		.compatible = "renesas,pfc-r8a7793",
@@ -512,6 +518,12 @@ static const struct of_device_id sh_pfc_of_table[] = {
 		.data = &r8a7795_pinmux_info,
 	},
 #endif
+#ifdef CONFIG_PINCTRL_PFC_R8A7796
+	{
+		.compatible = "renesas,pfc-r8a7796",
+		.data = &r8a7796_pinmux_info,
+	},
+#endif
 #ifdef CONFIG_PINCTRL_PFC_SH73A0
 	{
 		.compatible = "renesas,pfc-sh73a0",
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7791.c b/drivers/pinctrl/sh-pfc/pfc-r8a7791.c
index 0c1a60c9a844..7ca37c3019ab 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7791.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7791.c
@@ -13,6 +13,10 @@
 
 #include "sh_pfc.h"
 
+/*
+ * Pins 0-23 assigned to GPIO bank 6 can be used for SD interfaces in
+ * which case they support both 3.3V and 1.8V signalling.
+ */
 #define CPU_ALL_PORT(fn, sfx)						\
 	PORT_GP_32(0, fn, sfx),						\
 	PORT_GP_26(1, fn, sfx),						\
@@ -20,7 +24,15 @@
 	PORT_GP_32(3, fn, sfx),						\
 	PORT_GP_32(4, fn, sfx),						\
 	PORT_GP_32(5, fn, sfx),						\
-	PORT_GP_32(6, fn, sfx),						\
+	PORT_GP_CFG_24(6, fn, sfx, SH_PFC_PIN_CFG_IO_VOLTAGE),		\
+	PORT_GP_1(6, 24, fn, sfx),					\
+	PORT_GP_1(6, 25, fn, sfx),					\
+	PORT_GP_1(6, 26, fn, sfx),					\
+	PORT_GP_1(6, 27, fn, sfx),					\
+	PORT_GP_1(6, 28, fn, sfx),					\
+	PORT_GP_1(6, 29, fn, sfx),					\
+	PORT_GP_1(6, 30, fn, sfx),					\
+	PORT_GP_1(6, 31, fn, sfx),					\
 	PORT_GP_26(7, fn, sfx)
 
 enum {
@@ -6404,9 +6416,24 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	{ },
 };
 
+static int r8a7791_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin, u32 *pocctrl)
+{
+	if (pin < RCAR_GP_PIN(6, 0) || pin > RCAR_GP_PIN(6, 23))
+		return -EINVAL;
+
+	*pocctrl = 0xe606008c;
+
+	return 31 - (pin & 0x1f);
+}
+
+static const struct sh_pfc_soc_operations r8a7791_pinmux_ops = {
+	.pin_to_pocctrl = r8a7791_pin_to_pocctrl,
+};
+
 #ifdef CONFIG_PINCTRL_PFC_R8A7791
 const struct sh_pfc_soc_info r8a7791_pinmux_info = {
 	.name = "r8a77910_pfc",
+	.ops = &r8a7791_pinmux_ops,
 	.unlock_reg = 0xe6060000, /* PMMR */
 
 	.function = { PINMUX_FUNCTION_BEGIN, PINMUX_FUNCTION_END },
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7792.c b/drivers/pinctrl/sh-pfc/pfc-r8a7792.c
new file mode 100644
index 000000000000..21badb6166b9
--- /dev/null
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7792.c
@@ -0,0 +1,2739 @@
+/*
+ * r8a7792 processor support - PFC hardware block.
+ *
+ * Copyright (C) 2013-2014 Renesas Electronics Corporation
+ * Copyright (C) 2016 Cogent Embedded, Inc., <source@cogentembedded.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ */
+
+#include <linux/kernel.h>
+
+#include "core.h"
+#include "sh_pfc.h"
+
+#define CPU_ALL_PORT(fn, sfx)						\
+	PORT_GP_29(0, fn, sfx),						\
+	PORT_GP_23(1, fn, sfx),						\
+	PORT_GP_32(2, fn, sfx),						\
+	PORT_GP_28(3, fn, sfx),						\
+	PORT_GP_17(4, fn, sfx),						\
+	PORT_GP_17(5, fn, sfx),						\
+	PORT_GP_17(6, fn, sfx),						\
+	PORT_GP_17(7, fn, sfx),						\
+	PORT_GP_17(8, fn, sfx),						\
+	PORT_GP_17(9, fn, sfx),						\
+	PORT_GP_32(10, fn, sfx),					\
+	PORT_GP_30(11, fn, sfx)
+
+enum {
+	PINMUX_RESERVED = 0,
+
+	PINMUX_DATA_BEGIN,
+	GP_ALL(DATA),
+	PINMUX_DATA_END,
+
+	PINMUX_FUNCTION_BEGIN,
+	GP_ALL(FN),
+
+	/* GPSR0 */
+	FN_IP0_0, FN_IP0_1, FN_IP0_2, FN_IP0_3, FN_IP0_4, FN_IP0_5,
+	FN_IP0_6, FN_IP0_7, FN_IP0_8, FN_IP0_9, FN_IP0_10, FN_IP0_11,
+	FN_IP0_12, FN_IP0_13, FN_IP0_14, FN_IP0_15, FN_IP0_16,
+	FN_IP0_17, FN_IP0_18, FN_IP0_19, FN_IP0_20, FN_IP0_21,
+	FN_IP0_22, FN_IP0_23, FN_IP1_0, FN_IP1_1, FN_IP1_2,
+	FN_IP1_3, FN_IP1_4,
+
+	/* GPSR1 */
+	FN_IP1_5, FN_IP1_6, FN_IP1_7, FN_IP1_8, FN_IP1_9, FN_IP1_10,
+	FN_IP1_11, FN_IP1_12, FN_IP1_13, FN_IP1_14, FN_IP1_15, FN_IP1_16,
+	FN_DU1_DB2_C0_DATA12, FN_DU1_DB3_C1_DATA13, FN_DU1_DB4_C2_DATA14,
+	FN_DU1_DB5_C3_DATA15, FN_DU1_DB6_C4, FN_DU1_DB7_C5,
+	FN_DU1_EXHSYNC_DU1_HSYNC, FN_DU1_EXVSYNC_DU1_VSYNC,
+	FN_DU1_EXODDF_DU1_ODDF_DISP_CDE, FN_DU1_DISP, FN_DU1_CDE,
+
+	/* GPSR2 */
+	FN_D0, FN_D1, FN_D2, FN_D3, FN_D4, FN_D5, FN_D6, FN_D7,
+	FN_D8, FN_D9, FN_D10, FN_D11, FN_D12, FN_D13, FN_D14, FN_D15,
+	FN_A0, FN_A1, FN_A2, FN_A3, FN_A4, FN_A5, FN_A6, FN_A7,
+	FN_A8, FN_A9, FN_A10, FN_A11, FN_A12, FN_A13, FN_A14, FN_A15,
+
+	/* GPSR3 */
+	FN_A16, FN_A17, FN_A18, FN_A19, FN_IP1_17, FN_IP1_18,
+	FN_CS1_N_A26, FN_EX_CS0_N, FN_EX_CS1_N, FN_EX_CS2_N, FN_EX_CS3_N,
+	FN_EX_CS4_N, FN_EX_CS5_N, FN_BS_N, FN_RD_N, FN_RD_WR_N,
+	FN_WE0_N, FN_WE1_N, FN_EX_WAIT0, FN_IRQ0, FN_IRQ1, FN_IRQ2, FN_IRQ3,
+	FN_IP1_19, FN_IP1_20, FN_IP1_21, FN_IP1_22, FN_CS0_N,
+
+	/* GPSR4 */
+	FN_VI0_CLK, FN_VI0_CLKENB, FN_VI0_HSYNC_N, FN_VI0_VSYNC_N,
+	FN_VI0_D0_B0_C0, FN_VI0_D1_B1_C1, FN_VI0_D2_B2_C2, FN_VI0_D3_B3_C3,
+	FN_VI0_D4_B4_C4, FN_VI0_D5_B5_C5, FN_VI0_D6_B6_C6, FN_VI0_D7_B7_C7,
+	FN_VI0_D8_G0_Y0, FN_VI0_D9_G1_Y1, FN_VI0_D10_G2_Y2, FN_VI0_D11_G3_Y3,
+	FN_VI0_FIELD,
+
+	/* GPSR5 */
+	FN_VI1_CLK, FN_VI1_CLKENB, FN_VI1_HSYNC_N, FN_VI1_VSYNC_N,
+	FN_VI1_D0_B0_C0, FN_VI1_D1_B1_C1, FN_VI1_D2_B2_C2, FN_VI1_D3_B3_C3,
+	FN_VI1_D4_B4_C4, FN_VI1_D5_B5_C5, FN_VI1_D6_B6_C6, FN_VI1_D7_B7_C7,
+	FN_VI1_D8_G0_Y0, FN_VI1_D9_G1_Y1, FN_VI1_D10_G2_Y2, FN_VI1_D11_G3_Y3,
+	FN_VI1_FIELD,
+
+	/* GPSR6 */
+	FN_IP2_0, FN_IP2_1, FN_IP2_2, FN_IP2_3, FN_IP2_4, FN_IP2_5, FN_IP2_6,
+	FN_IP2_7, FN_IP2_8, FN_IP2_9, FN_IP2_10, FN_IP2_11, FN_IP2_12,
+	FN_IP2_13, FN_IP2_14, FN_IP2_15, FN_IP2_16,
+
+	/* GPSR7 */
+	FN_IP3_0, FN_IP3_1, FN_IP3_2, FN_IP3_3, FN_IP3_4, FN_IP3_5, FN_IP3_6,
+	FN_IP3_7, FN_IP3_8, FN_IP3_9, FN_IP3_10, FN_IP3_11, FN_IP3_12,
+	FN_IP3_13, FN_VI3_D10_Y2, FN_IP3_14, FN_VI3_FIELD,
+
+	/* GPSR8 */
+	FN_VI4_CLK, FN_IP4_0, FN_IP4_1, FN_IP4_3_2, FN_IP4_4, FN_IP4_6_5,
+	FN_IP4_8_7, FN_IP4_10_9, FN_IP4_12_11, FN_IP4_14_13, FN_IP4_16_15,
+	FN_IP4_18_17, FN_IP4_20_19, FN_IP4_21, FN_IP4_22, FN_IP4_23, FN_IP4_24,
+
+	/* GPSR9 */
+	FN_VI5_CLK, FN_IP5_0, FN_IP5_1, FN_IP5_2, FN_IP5_3, FN_IP5_4, FN_IP5_5,
+	FN_IP5_6, FN_IP5_7, FN_IP5_8, FN_IP5_9, FN_IP5_10, FN_IP5_11,
+	FN_VI5_D9_Y1, FN_VI5_D10_Y2, FN_VI5_D11_Y3, FN_VI5_FIELD,
+
+	/* GPSR10 */
+	FN_IP6_0, FN_IP6_1, FN_HRTS0_N, FN_IP6_2, FN_IP6_3, FN_IP6_4, FN_IP6_5,
+	FN_HCTS1_N, FN_IP6_6, FN_IP6_7,	FN_SCK0, FN_CTS0_N, FN_RTS0_N,
+	FN_TX0, FN_RX0, FN_SCK1, FN_CTS1_N, FN_RTS1_N, FN_TX1, FN_RX1,
+	FN_IP6_9_8, FN_IP6_11_10, FN_IP6_13_12, FN_IP6_15_14, FN_IP6_16,
+	FN_IP6_18_17, FN_SCIF_CLK, FN_CAN0_TX, FN_CAN0_RX, FN_CAN_CLK,
+	FN_CAN1_TX, FN_CAN1_RX,
+
+	/* GPSR11 */
+	FN_IP7_1_0, FN_IP7_3_2, FN_IP7_5_4, FN_IP7_6, FN_IP7_7, FN_SD0_CLK,
+	FN_SD0_CMD, FN_SD0_DAT0, FN_SD0_DAT1, FN_SD0_DAT2, FN_SD0_DAT3,
+	FN_SD0_CD, FN_SD0_WP, FN_IP7_9_8, FN_IP7_11_10, FN_IP7_13_12,
+	FN_IP7_15_14, FN_IP7_16, FN_IP7_17, FN_IP7_18, FN_IP7_19, FN_IP7_20,
+	FN_ADICLK, FN_ADICS_SAMP, FN_ADIDATA, FN_ADICHS0, FN_ADICHS1,
+	FN_ADICHS2, FN_AVS1, FN_AVS2,
+
+	/* IPSR0 */
+	FN_DU0_DR0_DATA0, FN_DU0_DR1_DATA1, FN_DU0_DR2_Y4_DATA2,
+	FN_DU0_DR3_Y5_DATA3, FN_DU0_DR4_Y6_DATA4, FN_DU0_DR5_Y7_DATA5,
+	FN_DU0_DR6_Y8_DATA6, FN_DU0_DR7_Y9_DATA7, FN_DU0_DG0_DATA8,
+	FN_DU0_DG1_DATA9, FN_DU0_DG2_C6_DATA10, FN_DU0_DG3_C7_DATA11,
+	FN_DU0_DG4_Y0_DATA12, FN_DU0_DG5_Y1_DATA13, FN_DU0_DG6_Y2_DATA14,
+	FN_DU0_DG7_Y3_DATA15, FN_DU0_DB0, FN_DU0_DB1, FN_DU0_DB2_C0,
+	FN_DU0_DB3_C1, FN_DU0_DB4_C2, FN_DU0_DB5_C3, FN_DU0_DB6_C4,
+	FN_DU0_DB7_C5,
+
+	/* IPSR1 */
+	FN_DU0_EXHSYNC_DU0_HSYNC, FN_DU0_EXVSYNC_DU0_VSYNC,
+	FN_DU0_EXODDF_DU0_ODDF_DISP_CDE, FN_DU0_DISP, FN_DU0_CDE,
+	FN_DU1_DR2_Y4_DATA0, FN_DU1_DR3_Y5_DATA1, FN_DU1_DR4_Y6_DATA2,
+	FN_DU1_DR5_Y7_DATA3, FN_DU1_DR6_DATA4, FN_DU1_DR7_DATA5,
+	FN_DU1_DG2_C6_DATA6, FN_DU1_DG3_C7_DATA7, FN_DU1_DG4_Y0_DATA8,
+	FN_DU1_DG5_Y1_DATA9, FN_DU1_DG6_Y2_DATA10, FN_DU1_DG7_Y3_DATA11,
+	FN_A20, FN_MOSI_IO0, FN_A21, FN_MISO_IO1, FN_A22, FN_IO2,
+	FN_A23, FN_IO3, FN_A24, FN_SPCLK, FN_A25, FN_SSL,
+
+	/* IPSR2 */
+	FN_VI2_CLK, FN_AVB_RX_CLK, FN_VI2_CLKENB, FN_AVB_RX_DV,
+	FN_VI2_HSYNC_N, FN_AVB_RXD0, FN_VI2_VSYNC_N, FN_AVB_RXD1,
+	FN_VI2_D0_C0, FN_AVB_RXD2, FN_VI2_D1_C1, FN_AVB_RXD3,
+	FN_VI2_D2_C2, FN_AVB_RXD4, FN_VI2_D3_C3, FN_AVB_RXD5,
+	FN_VI2_D4_C4, FN_AVB_RXD6, FN_VI2_D5_C5, FN_AVB_RXD7,
+	FN_VI2_D6_C6, FN_AVB_RX_ER, FN_VI2_D7_C7, FN_AVB_COL,
+	FN_VI2_D8_Y0, FN_AVB_TXD3, FN_VI2_D9_Y1, FN_AVB_TX_EN,
+	FN_VI2_D10_Y2, FN_AVB_TXD0, FN_VI2_D11_Y3, FN_AVB_TXD1,
+	FN_VI2_FIELD, FN_AVB_TXD2,
+
+	/* IPSR3 */
+	FN_VI3_CLK, FN_AVB_TX_CLK, FN_VI3_CLKENB, FN_AVB_TXD4,
+	FN_VI3_HSYNC_N, FN_AVB_TXD5, FN_VI3_VSYNC_N, FN_AVB_TXD6,
+	FN_VI3_D0_C0, FN_AVB_TXD7, FN_VI3_D1_C1, FN_AVB_TX_ER,
+	FN_VI3_D2_C2, FN_AVB_GTX_CLK, FN_VI3_D3_C3, FN_AVB_MDC,
+	FN_VI3_D4_C4, FN_AVB_MDIO, FN_VI3_D5_C5, FN_AVB_LINK,
+	FN_VI3_D6_C6, FN_AVB_MAGIC, FN_VI3_D7_C7, FN_AVB_PHY_INT,
+	FN_VI3_D8_Y0, FN_AVB_CRS, FN_VI3_D9_Y1, FN_AVB_GTXREFCLK,
+	FN_VI3_D11_Y3, FN_AVB_AVTP_MATCH,
+
+	/* IPSR4 */
+	FN_VI4_CLKENB, FN_VI0_D12_G4_Y4, FN_VI4_HSYNC_N, FN_VI0_D13_G5_Y5,
+	FN_VI4_VSYNC_N, FN_VI0_D14_G6_Y6, FN_RDR_CLKOUT,
+	FN_VI4_D0_C0, FN_VI0_D15_G7_Y7,
+	FN_VI4_D1_C1, FN_VI0_D16_R0, FN_VI1_D12_G4_Y4,
+	FN_VI4_D2_C2, FN_VI0_D17_R1, FN_VI1_D13_G5_Y5,
+	FN_VI4_D3_C3, FN_VI0_D18_R2, FN_VI1_D14_G6_Y6,
+	FN_VI4_D4_C4, FN_VI0_D19_R3, FN_VI1_D15_G7_Y7,
+	FN_VI4_D5_C5, FN_VI0_D20_R4, FN_VI2_D12_Y4,
+	FN_VI4_D6_C6, FN_VI0_D21_R5, FN_VI2_D13_Y5,
+	FN_VI4_D7_C7, FN_VI0_D22_R6, FN_VI2_D14_Y6,
+	FN_VI4_D8_Y0, FN_VI0_D23_R7, FN_VI2_D15_Y7,
+	FN_VI4_D9_Y1, FN_VI3_D12_Y4, FN_VI4_D10_Y2, FN_VI3_D13_Y5,
+	FN_VI4_D11_Y3, FN_VI3_D14_Y6, FN_VI4_FIELD, FN_VI3_D15_Y7,
+
+	/* IPSR5 */
+	FN_VI5_CLKENB, FN_VI1_D12_G4_Y4_B, FN_VI5_HSYNC_N, FN_VI1_D13_G5_Y5_B,
+	FN_VI5_VSYNC_N, FN_VI1_D14_G6_Y6_B, FN_VI5_D0_C0, FN_VI1_D15_G7_Y7_B,
+	FN_VI5_D1_C1, FN_VI1_D16_R0, FN_VI5_D2_C2, FN_VI1_D17_R1,
+	FN_VI5_D3_C3, FN_VI1_D18_R2, FN_VI5_D4_C4, FN_VI1_D19_R3,
+	FN_VI5_D5_C5, FN_VI1_D20_R4, FN_VI5_D6_C6, FN_VI1_D21_R5,
+	FN_VI5_D7_C7, FN_VI1_D22_R6, FN_VI5_D8_Y0, FN_VI1_D23_R7,
+
+	/* IPSR6 */
+	FN_MSIOF0_SCK, FN_HSCK0, FN_MSIOF0_SYNC, FN_HCTS0_N,
+	FN_MSIOF0_TXD, FN_HTX0, FN_MSIOF0_RXD, FN_HRX0,
+	FN_MSIOF1_SCK, FN_HSCK1, FN_MSIOF1_SYNC, FN_HRTS1_N,
+	FN_MSIOF1_TXD, FN_HTX1, FN_MSIOF1_RXD, FN_HRX1,
+	FN_DRACK0, FN_SCK2, FN_DACK0, FN_TX2, FN_DREQ0_N, FN_RX2,
+	FN_DACK1, FN_SCK3, FN_TX3, FN_DREQ1_N, FN_RX3,
+
+	/* IPSR7 */
+	FN_PWM0, FN_TCLK1, FN_FSO_CFE_0, FN_PWM1, FN_TCLK2, FN_FSO_CFE_1,
+	FN_PWM2, FN_TCLK3, FN_FSO_TOE, FN_PWM3, FN_PWM4,
+	FN_SSI_SCK34, FN_TPU0TO0, FN_SSI_WS34, FN_TPU0TO1,
+	FN_SSI_SDATA3, FN_TPU0TO2, FN_SSI_SCK4, FN_TPU0TO3,
+	FN_SSI_WS4, FN_SSI_SDATA4, FN_AUDIO_CLKOUT,
+	FN_AUDIO_CLKA, FN_AUDIO_CLKB,
+
+	/* MOD_SEL */
+	FN_SEL_VI1_0, FN_SEL_VI1_1,
+	PINMUX_FUNCTION_END,
+
+	PINMUX_MARK_BEGIN,
+	DU1_DB2_C0_DATA12_MARK, DU1_DB3_C1_DATA13_MARK,
+	DU1_DB4_C2_DATA14_MARK, DU1_DB5_C3_DATA15_MARK,
+	DU1_DB6_C4_MARK, DU1_DB7_C5_MARK, DU1_EXHSYNC_DU1_HSYNC_MARK,
+	DU1_EXVSYNC_DU1_VSYNC_MARK, DU1_EXODDF_DU1_ODDF_DISP_CDE_MARK,
+	DU1_DISP_MARK, DU1_CDE_MARK,
+
+	D0_MARK, D1_MARK, D2_MARK, D3_MARK, D4_MARK, D5_MARK, D6_MARK,
+	D7_MARK, D8_MARK, D9_MARK, D10_MARK, D11_MARK, D12_MARK, D13_MARK,
+	D14_MARK, D15_MARK, A0_MARK, A1_MARK, A2_MARK, A3_MARK, A4_MARK,
+	A5_MARK, A6_MARK, A7_MARK, A8_MARK, A9_MARK, A10_MARK, A11_MARK,
+	A12_MARK, A13_MARK, A14_MARK, A15_MARK,
+
+	A16_MARK, A17_MARK, A18_MARK, A19_MARK, CS1_N_A26_MARK,
+	EX_CS0_N_MARK, EX_CS1_N_MARK, EX_CS2_N_MARK, EX_CS3_N_MARK,
+	EX_CS4_N_MARK, EX_CS5_N_MARK, BS_N_MARK, RD_N_MARK, RD_WR_N_MARK,
+	WE0_N_MARK, WE1_N_MARK, EX_WAIT0_MARK,
+	IRQ0_MARK, IRQ1_MARK, IRQ2_MARK, IRQ3_MARK, CS0_N_MARK,
+
+	VI0_CLK_MARK, VI0_CLKENB_MARK, VI0_HSYNC_N_MARK, VI0_VSYNC_N_MARK,
+	VI0_D0_B0_C0_MARK, VI0_D1_B1_C1_MARK, VI0_D2_B2_C2_MARK,
+	VI0_D3_B3_C3_MARK, VI0_D4_B4_C4_MARK, VI0_D5_B5_C5_MARK,
+	VI0_D6_B6_C6_MARK, VI0_D7_B7_C7_MARK, VI0_D8_G0_Y0_MARK,
+	VI0_D9_G1_Y1_MARK, VI0_D10_G2_Y2_MARK, VI0_D11_G3_Y3_MARK,
+	VI0_FIELD_MARK,
+
+	VI1_CLK_MARK, VI1_CLKENB_MARK, VI1_HSYNC_N_MARK, VI1_VSYNC_N_MARK,
+	VI1_D0_B0_C0_MARK, VI1_D1_B1_C1_MARK, VI1_D2_B2_C2_MARK,
+	VI1_D3_B3_C3_MARK, VI1_D4_B4_C4_MARK, VI1_D5_B5_C5_MARK,
+	VI1_D6_B6_C6_MARK, VI1_D7_B7_C7_MARK, VI1_D8_G0_Y0_MARK,
+	VI1_D9_G1_Y1_MARK, VI1_D10_G2_Y2_MARK, VI1_D11_G3_Y3_MARK,
+	VI1_FIELD_MARK,
+
+	VI3_D10_Y2_MARK, VI3_FIELD_MARK,
+
+	VI4_CLK_MARK,
+
+	VI5_CLK_MARK, VI5_D9_Y1_MARK, VI5_D10_Y2_MARK, VI5_D11_Y3_MARK,
+	VI5_FIELD_MARK,
+
+	HRTS0_N_MARK, HCTS1_N_MARK, SCK0_MARK, CTS0_N_MARK, RTS0_N_MARK,
+	TX0_MARK, RX0_MARK, SCK1_MARK, CTS1_N_MARK, RTS1_N_MARK,
+	TX1_MARK, RX1_MARK, SCIF_CLK_MARK, CAN0_TX_MARK, CAN0_RX_MARK,
+	CAN_CLK_MARK, CAN1_TX_MARK, CAN1_RX_MARK,
+
+	SD0_CLK_MARK, SD0_CMD_MARK, SD0_DAT0_MARK, SD0_DAT1_MARK,
+	SD0_DAT2_MARK, SD0_DAT3_MARK, SD0_CD_MARK, SD0_WP_MARK,
+	ADICLK_MARK, ADICS_SAMP_MARK, ADIDATA_MARK, ADICHS0_MARK,
+	ADICHS1_MARK, ADICHS2_MARK, AVS1_MARK, AVS2_MARK,
+
+	/* IPSR0 */
+	DU0_DR0_DATA0_MARK, DU0_DR1_DATA1_MARK, DU0_DR2_Y4_DATA2_MARK,
+	DU0_DR3_Y5_DATA3_MARK, DU0_DR4_Y6_DATA4_MARK, DU0_DR5_Y7_DATA5_MARK,
+	DU0_DR6_Y8_DATA6_MARK, DU0_DR7_Y9_DATA7_MARK, DU0_DG0_DATA8_MARK,
+	DU0_DG1_DATA9_MARK, DU0_DG2_C6_DATA10_MARK, DU0_DG3_C7_DATA11_MARK,
+	DU0_DG4_Y0_DATA12_MARK, DU0_DG5_Y1_DATA13_MARK, DU0_DG6_Y2_DATA14_MARK,
+	DU0_DG7_Y3_DATA15_MARK, DU0_DB0_MARK, DU0_DB1_MARK,
+	DU0_DB2_C0_MARK, DU0_DB3_C1_MARK, DU0_DB4_C2_MARK, DU0_DB5_C3_MARK,
+	DU0_DB6_C4_MARK, DU0_DB7_C5_MARK,
+
+	/* IPSR1 */
+	DU0_EXHSYNC_DU0_HSYNC_MARK, DU0_EXVSYNC_DU0_VSYNC_MARK,
+	DU0_EXODDF_DU0_ODDF_DISP_CDE_MARK, DU0_DISP_MARK, DU0_CDE_MARK,
+	DU1_DR2_Y4_DATA0_MARK, DU1_DR3_Y5_DATA1_MARK, DU1_DR4_Y6_DATA2_MARK,
+	DU1_DR5_Y7_DATA3_MARK, DU1_DR6_DATA4_MARK, DU1_DR7_DATA5_MARK,
+	DU1_DG2_C6_DATA6_MARK, DU1_DG3_C7_DATA7_MARK, DU1_DG4_Y0_DATA8_MARK,
+	DU1_DG5_Y1_DATA9_MARK, DU1_DG6_Y2_DATA10_MARK, DU1_DG7_Y3_DATA11_MARK,
+	A20_MARK, MOSI_IO0_MARK, A21_MARK, MISO_IO1_MARK, A22_MARK, IO2_MARK,
+	A23_MARK, IO3_MARK, A24_MARK, SPCLK_MARK, A25_MARK, SSL_MARK,
+
+	/* IPSR2 */
+	VI2_CLK_MARK, AVB_RX_CLK_MARK, VI2_CLKENB_MARK, AVB_RX_DV_MARK,
+	VI2_HSYNC_N_MARK, AVB_RXD0_MARK, VI2_VSYNC_N_MARK, AVB_RXD1_MARK,
+	VI2_D0_C0_MARK, AVB_RXD2_MARK, VI2_D1_C1_MARK, AVB_TX_CLK_MARK,
+	VI2_D2_C2_MARK, AVB_RXD4_MARK, VI2_D3_C3_MARK, AVB_RXD5_MARK,
+	VI2_D4_C4_MARK, AVB_RXD6_MARK, VI2_D5_C5_MARK, AVB_RXD7_MARK,
+	VI2_D6_C6_MARK, AVB_RX_ER_MARK, VI2_D7_C7_MARK, AVB_COL_MARK,
+	VI2_D8_Y0_MARK, AVB_RXD3_MARK, VI2_D9_Y1_MARK, AVB_TX_EN_MARK,
+	VI2_D10_Y2_MARK, AVB_TXD0_MARK,
+	VI2_D11_Y3_MARK, AVB_TXD1_MARK, VI2_FIELD_MARK, AVB_TXD2_MARK,
+
+	/* IPSR3 */
+	VI3_CLK_MARK, AVB_TXD3_MARK, VI3_CLKENB_MARK, AVB_TXD4_MARK,
+	VI3_HSYNC_N_MARK, AVB_TXD5_MARK, VI3_VSYNC_N_MARK, AVB_TXD6_MARK,
+	VI3_D0_C0_MARK, AVB_TXD7_MARK, VI3_D1_C1_MARK, AVB_TX_ER_MARK,
+	VI3_D2_C2_MARK, AVB_GTX_CLK_MARK, VI3_D3_C3_MARK, AVB_MDC_MARK,
+	VI3_D4_C4_MARK, AVB_MDIO_MARK, VI3_D5_C5_MARK, AVB_LINK_MARK,
+	VI3_D6_C6_MARK, AVB_MAGIC_MARK, VI3_D7_C7_MARK, AVB_PHY_INT_MARK,
+	VI3_D8_Y0_MARK, AVB_CRS_MARK, VI3_D9_Y1_MARK, AVB_GTXREFCLK_MARK,
+	VI3_D11_Y3_MARK, AVB_AVTP_MATCH_MARK,
+
+	/* IPSR4 */
+	VI4_CLKENB_MARK, VI0_D12_G4_Y4_MARK, VI4_HSYNC_N_MARK,
+	VI0_D13_G5_Y5_MARK, VI4_VSYNC_N_MARK, VI0_D14_G6_Y6_MARK,
+	RDR_CLKOUT_MARK, VI4_D0_C0_MARK, VI0_D15_G7_Y7_MARK, VI4_D1_C1_MARK,
+	VI0_D16_R0_MARK, VI1_D12_G4_Y4_MARK, VI4_D2_C2_MARK, VI0_D17_R1_MARK,
+	VI1_D13_G5_Y5_MARK, VI4_D3_C3_MARK, VI0_D18_R2_MARK, VI1_D14_G6_Y6_MARK,
+	VI4_D4_C4_MARK,	VI0_D19_R3_MARK, VI1_D15_G7_Y7_MARK, VI4_D5_C5_MARK,
+	VI0_D20_R4_MARK, VI2_D12_Y4_MARK, VI4_D6_C6_MARK, VI0_D21_R5_MARK,
+	VI2_D13_Y5_MARK, VI4_D7_C7_MARK, VI0_D22_R6_MARK, VI2_D14_Y6_MARK,
+	VI4_D8_Y0_MARK, VI0_D23_R7_MARK, VI2_D15_Y7_MARK, VI4_D9_Y1_MARK,
+	VI3_D12_Y4_MARK, VI4_D10_Y2_MARK, VI3_D13_Y5_MARK, VI4_D11_Y3_MARK,
+	VI3_D14_Y6_MARK, VI4_FIELD_MARK, VI3_D15_Y7_MARK,
+
+	/* IPSR5 */
+	VI5_CLKENB_MARK, VI1_D12_G4_Y4_B_MARK, VI5_HSYNC_N_MARK,
+	VI1_D13_G5_Y5_B_MARK, VI5_VSYNC_N_MARK, VI1_D14_G6_Y6_B_MARK,
+	VI5_D0_C0_MARK, VI1_D15_G7_Y7_B_MARK, VI5_D1_C1_MARK, VI1_D16_R0_MARK,
+	VI5_D2_C2_MARK, VI1_D17_R1_MARK, VI5_D3_C3_MARK, VI1_D18_R2_MARK,
+	VI5_D4_C4_MARK, VI1_D19_R3_MARK, VI5_D5_C5_MARK, VI1_D20_R4_MARK,
+	VI5_D6_C6_MARK, VI1_D21_R5_MARK, VI5_D7_C7_MARK, VI1_D22_R6_MARK,
+	VI5_D8_Y0_MARK, VI1_D23_R7_MARK,
+
+	/* IPSR6 */
+	MSIOF0_SCK_MARK, HSCK0_MARK, MSIOF0_SYNC_MARK, HCTS0_N_MARK,
+	MSIOF0_TXD_MARK, HTX0_MARK, MSIOF0_RXD_MARK, HRX0_MARK,
+	MSIOF1_SCK_MARK, HSCK1_MARK, MSIOF1_SYNC_MARK, HRTS1_N_MARK,
+	MSIOF1_TXD_MARK, HTX1_MARK, MSIOF1_RXD_MARK, HRX1_MARK,
+	DRACK0_MARK, SCK2_MARK, DACK0_MARK, TX2_MARK, DREQ0_N_MARK,
+	RX2_MARK, DACK1_MARK, SCK3_MARK, TX3_MARK, DREQ1_N_MARK,
+	RX3_MARK,
+
+	/* IPSR7 */
+	PWM0_MARK, TCLK1_MARK, FSO_CFE_0_MARK, PWM1_MARK, TCLK2_MARK,
+	FSO_CFE_1_MARK, PWM2_MARK, TCLK3_MARK, FSO_TOE_MARK, PWM3_MARK,
+	PWM4_MARK, SSI_SCK34_MARK, TPU0TO0_MARK, SSI_WS34_MARK, TPU0TO1_MARK,
+	SSI_SDATA3_MARK, TPU0TO2_MARK, SSI_SCK4_MARK, TPU0TO3_MARK,
+	SSI_WS4_MARK, SSI_SDATA4_MARK, AUDIO_CLKOUT_MARK, AUDIO_CLKA_MARK,
+	AUDIO_CLKB_MARK,
+	PINMUX_MARK_END,
+};
+
+static const u16 pinmux_data[] = {
+	PINMUX_DATA_GP_ALL(), /* PINMUX_DATA(GP_M_N_DATA, GP_M_N_FN...), */
+
+	PINMUX_SINGLE(DU1_DB2_C0_DATA12),
+	PINMUX_SINGLE(DU1_DB3_C1_DATA13),
+	PINMUX_SINGLE(DU1_DB4_C2_DATA14),
+	PINMUX_SINGLE(DU1_DB5_C3_DATA15),
+	PINMUX_SINGLE(DU1_DB6_C4),
+	PINMUX_SINGLE(DU1_DB7_C5),
+	PINMUX_SINGLE(DU1_EXHSYNC_DU1_HSYNC),
+	PINMUX_SINGLE(DU1_EXVSYNC_DU1_VSYNC),
+	PINMUX_SINGLE(DU1_EXODDF_DU1_ODDF_DISP_CDE),
+	PINMUX_SINGLE(DU1_DISP),
+	PINMUX_SINGLE(DU1_CDE),
+	PINMUX_SINGLE(D0),
+	PINMUX_SINGLE(D1),
+	PINMUX_SINGLE(D2),
+	PINMUX_SINGLE(D3),
+	PINMUX_SINGLE(D4),
+	PINMUX_SINGLE(D5),
+	PINMUX_SINGLE(D6),
+	PINMUX_SINGLE(D7),
+	PINMUX_SINGLE(D8),
+	PINMUX_SINGLE(D9),
+	PINMUX_SINGLE(D10),
+	PINMUX_SINGLE(D11),
+	PINMUX_SINGLE(D12),
+	PINMUX_SINGLE(D13),
+	PINMUX_SINGLE(D14),
+	PINMUX_SINGLE(D15),
+	PINMUX_SINGLE(A0),
+	PINMUX_SINGLE(A1),
+	PINMUX_SINGLE(A2),
+	PINMUX_SINGLE(A3),
+	PINMUX_SINGLE(A4),
+	PINMUX_SINGLE(A5),
+	PINMUX_SINGLE(A6),
+	PINMUX_SINGLE(A7),
+	PINMUX_SINGLE(A8),
+	PINMUX_SINGLE(A9),
+	PINMUX_SINGLE(A10),
+	PINMUX_SINGLE(A11),
+	PINMUX_SINGLE(A12),
+	PINMUX_SINGLE(A13),
+	PINMUX_SINGLE(A14),
+	PINMUX_SINGLE(A15),
+	PINMUX_SINGLE(A16),
+	PINMUX_SINGLE(A17),
+	PINMUX_SINGLE(A18),
+	PINMUX_SINGLE(A19),
+	PINMUX_SINGLE(CS1_N_A26),
+	PINMUX_SINGLE(EX_CS0_N),
+	PINMUX_SINGLE(EX_CS1_N),
+	PINMUX_SINGLE(EX_CS2_N),
+	PINMUX_SINGLE(EX_CS3_N),
+	PINMUX_SINGLE(EX_CS4_N),
+	PINMUX_SINGLE(EX_CS5_N),
+	PINMUX_SINGLE(BS_N),
+	PINMUX_SINGLE(RD_N),
+	PINMUX_SINGLE(RD_WR_N),
+	PINMUX_SINGLE(WE0_N),
+	PINMUX_SINGLE(WE1_N),
+	PINMUX_SINGLE(EX_WAIT0),
+	PINMUX_SINGLE(IRQ0),
+	PINMUX_SINGLE(IRQ1),
+	PINMUX_SINGLE(IRQ2),
+	PINMUX_SINGLE(IRQ3),
+	PINMUX_SINGLE(CS0_N),
+	PINMUX_SINGLE(VI0_CLK),
+	PINMUX_SINGLE(VI0_CLKENB),
+	PINMUX_SINGLE(VI0_HSYNC_N),
+	PINMUX_SINGLE(VI0_VSYNC_N),
+	PINMUX_SINGLE(VI0_D0_B0_C0),
+	PINMUX_SINGLE(VI0_D1_B1_C1),
+	PINMUX_SINGLE(VI0_D2_B2_C2),
+	PINMUX_SINGLE(VI0_D3_B3_C3),
+	PINMUX_SINGLE(VI0_D4_B4_C4),
+	PINMUX_SINGLE(VI0_D5_B5_C5),
+	PINMUX_SINGLE(VI0_D6_B6_C6),
+	PINMUX_SINGLE(VI0_D7_B7_C7),
+	PINMUX_SINGLE(VI0_D8_G0_Y0),
+	PINMUX_SINGLE(VI0_D9_G1_Y1),
+	PINMUX_SINGLE(VI0_D10_G2_Y2),
+	PINMUX_SINGLE(VI0_D11_G3_Y3),
+	PINMUX_SINGLE(VI0_FIELD),
+	PINMUX_SINGLE(VI1_CLK),
+	PINMUX_SINGLE(VI1_CLKENB),
+	PINMUX_SINGLE(VI1_HSYNC_N),
+	PINMUX_SINGLE(VI1_VSYNC_N),
+	PINMUX_SINGLE(VI1_D0_B0_C0),
+	PINMUX_SINGLE(VI1_D1_B1_C1),
+	PINMUX_SINGLE(VI1_D2_B2_C2),
+	PINMUX_SINGLE(VI1_D3_B3_C3),
+	PINMUX_SINGLE(VI1_D4_B4_C4),
+	PINMUX_SINGLE(VI1_D5_B5_C5),
+	PINMUX_SINGLE(VI1_D6_B6_C6),
+	PINMUX_SINGLE(VI1_D7_B7_C7),
+	PINMUX_SINGLE(VI1_D8_G0_Y0),
+	PINMUX_SINGLE(VI1_D9_G1_Y1),
+	PINMUX_SINGLE(VI1_D10_G2_Y2),
+	PINMUX_SINGLE(VI1_D11_G3_Y3),
+	PINMUX_SINGLE(VI1_FIELD),
+	PINMUX_SINGLE(VI3_D10_Y2),
+	PINMUX_SINGLE(VI3_FIELD),
+	PINMUX_SINGLE(VI4_CLK),
+	PINMUX_SINGLE(VI5_CLK),
+	PINMUX_SINGLE(VI5_D9_Y1),
+	PINMUX_SINGLE(VI5_D10_Y2),
+	PINMUX_SINGLE(VI5_D11_Y3),
+	PINMUX_SINGLE(VI5_FIELD),
+	PINMUX_SINGLE(HRTS0_N),
+	PINMUX_SINGLE(HCTS1_N),
+	PINMUX_SINGLE(SCK0),
+	PINMUX_SINGLE(CTS0_N),
+	PINMUX_SINGLE(RTS0_N),
+	PINMUX_SINGLE(TX0),
+	PINMUX_SINGLE(RX0),
+	PINMUX_SINGLE(SCK1),
+	PINMUX_SINGLE(CTS1_N),
+	PINMUX_SINGLE(RTS1_N),
+	PINMUX_SINGLE(TX1),
+	PINMUX_SINGLE(RX1),
+	PINMUX_SINGLE(SCIF_CLK),
+	PINMUX_SINGLE(CAN0_TX),
+	PINMUX_SINGLE(CAN0_RX),
+	PINMUX_SINGLE(CAN_CLK),
+	PINMUX_SINGLE(CAN1_TX),
+	PINMUX_SINGLE(CAN1_RX),
+	PINMUX_SINGLE(SD0_CLK),
+	PINMUX_SINGLE(SD0_CMD),
+	PINMUX_SINGLE(SD0_DAT0),
+	PINMUX_SINGLE(SD0_DAT1),
+	PINMUX_SINGLE(SD0_DAT2),
+	PINMUX_SINGLE(SD0_DAT3),
+	PINMUX_SINGLE(SD0_CD),
+	PINMUX_SINGLE(SD0_WP),
+	PINMUX_SINGLE(ADICLK),
+	PINMUX_SINGLE(ADICS_SAMP),
+	PINMUX_SINGLE(ADIDATA),
+	PINMUX_SINGLE(ADICHS0),
+	PINMUX_SINGLE(ADICHS1),
+	PINMUX_SINGLE(ADICHS2),
+	PINMUX_SINGLE(AVS1),
+	PINMUX_SINGLE(AVS2),
+
+	/* IPSR0 */
+	PINMUX_IPSR_GPSR(IP0_0, DU0_DR0_DATA0),
+	PINMUX_IPSR_GPSR(IP0_1, DU0_DR1_DATA1),
+	PINMUX_IPSR_GPSR(IP0_2, DU0_DR2_Y4_DATA2),
+	PINMUX_IPSR_GPSR(IP0_3, DU0_DR3_Y5_DATA3),
+	PINMUX_IPSR_GPSR(IP0_4, DU0_DR4_Y6_DATA4),
+	PINMUX_IPSR_GPSR(IP0_5, DU0_DR5_Y7_DATA5),
+	PINMUX_IPSR_GPSR(IP0_6, DU0_DR6_Y8_DATA6),
+	PINMUX_IPSR_GPSR(IP0_7, DU0_DR7_Y9_DATA7),
+	PINMUX_IPSR_GPSR(IP0_8, DU0_DG0_DATA8),
+	PINMUX_IPSR_GPSR(IP0_9, DU0_DG1_DATA9),
+	PINMUX_IPSR_GPSR(IP0_10, DU0_DG2_C6_DATA10),
+	PINMUX_IPSR_GPSR(IP0_11, DU0_DG3_C7_DATA11),
+	PINMUX_IPSR_GPSR(IP0_12, DU0_DG4_Y0_DATA12),
+	PINMUX_IPSR_GPSR(IP0_13, DU0_DG5_Y1_DATA13),
+	PINMUX_IPSR_GPSR(IP0_14, DU0_DG6_Y2_DATA14),
+	PINMUX_IPSR_GPSR(IP0_15, DU0_DG7_Y3_DATA15),
+	PINMUX_IPSR_GPSR(IP0_16, DU0_DB0),
+	PINMUX_IPSR_GPSR(IP0_17, DU0_DB1),
+	PINMUX_IPSR_GPSR(IP0_18, DU0_DB2_C0),
+	PINMUX_IPSR_GPSR(IP0_19, DU0_DB3_C1),
+	PINMUX_IPSR_GPSR(IP0_20, DU0_DB4_C2),
+	PINMUX_IPSR_GPSR(IP0_21, DU0_DB5_C3),
+	PINMUX_IPSR_GPSR(IP0_22, DU0_DB6_C4),
+	PINMUX_IPSR_GPSR(IP0_23, DU0_DB7_C5),
+
+	/* IPSR1 */
+	PINMUX_IPSR_GPSR(IP1_0, DU0_EXHSYNC_DU0_HSYNC),
+	PINMUX_IPSR_GPSR(IP1_1, DU0_EXVSYNC_DU0_VSYNC),
+	PINMUX_IPSR_GPSR(IP1_2, DU0_EXODDF_DU0_ODDF_DISP_CDE),
+	PINMUX_IPSR_GPSR(IP1_3, DU0_DISP),
+	PINMUX_IPSR_GPSR(IP1_4, DU0_CDE),
+	PINMUX_IPSR_GPSR(IP1_5, DU1_DR2_Y4_DATA0),
+	PINMUX_IPSR_GPSR(IP1_6, DU1_DR3_Y5_DATA1),
+	PINMUX_IPSR_GPSR(IP1_7, DU1_DR4_Y6_DATA2),
+	PINMUX_IPSR_GPSR(IP1_8, DU1_DR5_Y7_DATA3),
+	PINMUX_IPSR_GPSR(IP1_9, DU1_DR6_DATA4),
+	PINMUX_IPSR_GPSR(IP1_10, DU1_DR7_DATA5),
+	PINMUX_IPSR_GPSR(IP1_11, DU1_DG2_C6_DATA6),
+	PINMUX_IPSR_GPSR(IP1_12, DU1_DG3_C7_DATA7),
+	PINMUX_IPSR_GPSR(IP1_13, DU1_DG4_Y0_DATA8),
+	PINMUX_IPSR_GPSR(IP1_14, DU1_DG5_Y1_DATA9),
+	PINMUX_IPSR_GPSR(IP1_15, DU1_DG6_Y2_DATA10),
+	PINMUX_IPSR_GPSR(IP1_16, DU1_DG7_Y3_DATA11),
+	PINMUX_IPSR_GPSR(IP1_17, A20),
+	PINMUX_IPSR_GPSR(IP1_17, MOSI_IO0),
+	PINMUX_IPSR_GPSR(IP1_18, A21),
+	PINMUX_IPSR_GPSR(IP1_18, MISO_IO1),
+	PINMUX_IPSR_GPSR(IP1_19, A22),
+	PINMUX_IPSR_GPSR(IP1_19, IO2),
+	PINMUX_IPSR_GPSR(IP1_20, A23),
+	PINMUX_IPSR_GPSR(IP1_20, IO3),
+	PINMUX_IPSR_GPSR(IP1_21, A24),
+	PINMUX_IPSR_GPSR(IP1_21, SPCLK),
+	PINMUX_IPSR_GPSR(IP1_22, A25),
+	PINMUX_IPSR_GPSR(IP1_22, SSL),
+
+	/* IPSR2 */
+	PINMUX_IPSR_GPSR(IP2_0, VI2_CLK),
+	PINMUX_IPSR_GPSR(IP2_0, AVB_RX_CLK),
+	PINMUX_IPSR_GPSR(IP2_1, VI2_CLKENB),
+	PINMUX_IPSR_GPSR(IP2_1, AVB_RX_DV),
+	PINMUX_IPSR_GPSR(IP2_2, VI2_HSYNC_N),
+	PINMUX_IPSR_GPSR(IP2_2, AVB_RXD0),
+	PINMUX_IPSR_GPSR(IP2_3, VI2_VSYNC_N),
+	PINMUX_IPSR_GPSR(IP2_3, AVB_RXD1),
+	PINMUX_IPSR_GPSR(IP2_4, VI2_D0_C0),
+	PINMUX_IPSR_GPSR(IP2_4, AVB_RXD2),
+	PINMUX_IPSR_GPSR(IP2_5, VI2_D1_C1),
+	PINMUX_IPSR_GPSR(IP2_5, AVB_RXD3),
+	PINMUX_IPSR_GPSR(IP2_6, VI2_D2_C2),
+	PINMUX_IPSR_GPSR(IP2_6, AVB_RXD4),
+	PINMUX_IPSR_GPSR(IP2_7, VI2_D3_C3),
+	PINMUX_IPSR_GPSR(IP2_7, AVB_RXD5),
+	PINMUX_IPSR_GPSR(IP2_8, VI2_D4_C4),
+	PINMUX_IPSR_GPSR(IP2_8, AVB_RXD6),
+	PINMUX_IPSR_GPSR(IP2_9, VI2_D5_C5),
+	PINMUX_IPSR_GPSR(IP2_9, AVB_RXD7),
+	PINMUX_IPSR_GPSR(IP2_10, VI2_D6_C6),
+	PINMUX_IPSR_GPSR(IP2_10, AVB_RX_ER),
+	PINMUX_IPSR_GPSR(IP2_11, VI2_D7_C7),
+	PINMUX_IPSR_GPSR(IP2_11, AVB_COL),
+	PINMUX_IPSR_GPSR(IP2_12, VI2_D8_Y0),
+	PINMUX_IPSR_GPSR(IP2_12, AVB_TXD3),
+	PINMUX_IPSR_GPSR(IP2_13, VI2_D9_Y1),
+	PINMUX_IPSR_GPSR(IP2_13, AVB_TX_EN),
+	PINMUX_IPSR_GPSR(IP2_14, VI2_D10_Y2),
+	PINMUX_IPSR_GPSR(IP2_14, AVB_TXD0),
+	PINMUX_IPSR_GPSR(IP2_15, VI2_D11_Y3),
+	PINMUX_IPSR_GPSR(IP2_15, AVB_TXD1),
+	PINMUX_IPSR_GPSR(IP2_16, VI2_FIELD),
+	PINMUX_IPSR_GPSR(IP2_16, AVB_TXD2),
+
+	/* IPSR3 */
+	PINMUX_IPSR_GPSR(IP3_0, VI3_CLK),
+	PINMUX_IPSR_GPSR(IP3_0, AVB_TX_CLK),
+	PINMUX_IPSR_GPSR(IP3_1, VI3_CLKENB),
+	PINMUX_IPSR_GPSR(IP3_1, AVB_TXD4),
+	PINMUX_IPSR_GPSR(IP3_2, VI3_HSYNC_N),
+	PINMUX_IPSR_GPSR(IP3_2, AVB_TXD5),
+	PINMUX_IPSR_GPSR(IP3_3, VI3_VSYNC_N),
+	PINMUX_IPSR_GPSR(IP3_3, AVB_TXD6),
+	PINMUX_IPSR_GPSR(IP3_4, VI3_D0_C0),
+	PINMUX_IPSR_GPSR(IP3_4, AVB_TXD7),
+	PINMUX_IPSR_GPSR(IP3_5, VI3_D1_C1),
+	PINMUX_IPSR_GPSR(IP3_5, AVB_TX_ER),
+	PINMUX_IPSR_GPSR(IP3_6, VI3_D2_C2),
+	PINMUX_IPSR_GPSR(IP3_6, AVB_GTX_CLK),
+	PINMUX_IPSR_GPSR(IP3_7, VI3_D3_C3),
+	PINMUX_IPSR_GPSR(IP3_7, AVB_MDC),
+	PINMUX_IPSR_GPSR(IP3_8, VI3_D4_C4),
+	PINMUX_IPSR_GPSR(IP3_8, AVB_MDIO),
+	PINMUX_IPSR_GPSR(IP3_9, VI3_D5_C5),
+	PINMUX_IPSR_GPSR(IP3_9, AVB_LINK),
+	PINMUX_IPSR_GPSR(IP3_10, VI3_D6_C6),
+	PINMUX_IPSR_GPSR(IP3_10, AVB_MAGIC),
+	PINMUX_IPSR_GPSR(IP3_11, VI3_D7_C7),
+	PINMUX_IPSR_GPSR(IP3_11, AVB_PHY_INT),
+	PINMUX_IPSR_GPSR(IP3_12, VI3_D8_Y0),
+	PINMUX_IPSR_GPSR(IP3_12, AVB_CRS),
+	PINMUX_IPSR_GPSR(IP3_13, VI3_D9_Y1),
+	PINMUX_IPSR_GPSR(IP3_13, AVB_GTXREFCLK),
+	PINMUX_IPSR_GPSR(IP3_14, VI3_D11_Y3),
+	PINMUX_IPSR_GPSR(IP3_14, AVB_AVTP_MATCH),
+
+	/* IPSR4 */
+	PINMUX_IPSR_GPSR(IP4_0, VI4_CLKENB),
+	PINMUX_IPSR_GPSR(IP4_0, VI0_D12_G4_Y4),
+	PINMUX_IPSR_GPSR(IP4_1, VI4_HSYNC_N),
+	PINMUX_IPSR_GPSR(IP4_1, VI0_D13_G5_Y5),
+	PINMUX_IPSR_GPSR(IP4_3_2, VI4_VSYNC_N),
+	PINMUX_IPSR_GPSR(IP4_3_2, VI0_D14_G6_Y6),
+	PINMUX_IPSR_GPSR(IP4_4, VI4_D0_C0),
+	PINMUX_IPSR_GPSR(IP4_4, VI0_D15_G7_Y7),
+	PINMUX_IPSR_GPSR(IP4_6_5, VI4_D1_C1),
+	PINMUX_IPSR_GPSR(IP4_6_5, VI0_D16_R0),
+	PINMUX_IPSR_MSEL(IP4_6_5, VI1_D12_G4_Y4, SEL_VI1_0),
+	PINMUX_IPSR_GPSR(IP4_8_7, VI4_D2_C2),
+	PINMUX_IPSR_GPSR(IP4_8_7, VI0_D17_R1),
+	PINMUX_IPSR_MSEL(IP4_8_7, VI1_D13_G5_Y5, SEL_VI1_0),
+	PINMUX_IPSR_GPSR(IP4_10_9, VI4_D3_C3),
+	PINMUX_IPSR_GPSR(IP4_10_9, VI0_D18_R2),
+	PINMUX_IPSR_MSEL(IP4_10_9, VI1_D14_G6_Y6, SEL_VI1_0),
+	PINMUX_IPSR_GPSR(IP4_12_11, VI4_D4_C4),
+	PINMUX_IPSR_GPSR(IP4_12_11, VI0_D19_R3),
+	PINMUX_IPSR_MSEL(IP4_12_11, VI1_D15_G7_Y7, SEL_VI1_0),
+	PINMUX_IPSR_GPSR(IP4_14_13, VI4_D5_C5),
+	PINMUX_IPSR_GPSR(IP4_14_13, VI0_D20_R4),
+	PINMUX_IPSR_GPSR(IP4_14_13, VI2_D12_Y4),
+	PINMUX_IPSR_GPSR(IP4_16_15, VI4_D6_C6),
+	PINMUX_IPSR_GPSR(IP4_16_15, VI0_D21_R5),
+	PINMUX_IPSR_GPSR(IP4_16_15, VI2_D13_Y5),
+	PINMUX_IPSR_GPSR(IP4_18_17, VI4_D7_C7),
+	PINMUX_IPSR_GPSR(IP4_18_17, VI0_D22_R6),
+	PINMUX_IPSR_GPSR(IP4_18_17, VI2_D14_Y6),
+	PINMUX_IPSR_GPSR(IP4_20_19, VI4_D8_Y0),
+	PINMUX_IPSR_GPSR(IP4_20_19, VI0_D23_R7),
+	PINMUX_IPSR_GPSR(IP4_20_19, VI2_D15_Y7),
+	PINMUX_IPSR_GPSR(IP4_21, VI4_D9_Y1),
+	PINMUX_IPSR_GPSR(IP4_21, VI3_D12_Y4),
+	PINMUX_IPSR_GPSR(IP4_22, VI4_D10_Y2),
+	PINMUX_IPSR_GPSR(IP4_22, VI3_D13_Y5),
+	PINMUX_IPSR_GPSR(IP4_23, VI4_D11_Y3),
+	PINMUX_IPSR_GPSR(IP4_23, VI3_D14_Y6),
+	PINMUX_IPSR_GPSR(IP4_24, VI4_FIELD),
+	PINMUX_IPSR_GPSR(IP4_24, VI3_D15_Y7),
+
+	/* IPSR5 */
+	PINMUX_IPSR_GPSR(IP5_0, VI5_CLKENB),
+	PINMUX_IPSR_MSEL(IP5_0, VI1_D12_G4_Y4_B, SEL_VI1_1),
+	PINMUX_IPSR_GPSR(IP5_1, VI5_HSYNC_N),
+	PINMUX_IPSR_MSEL(IP5_1, VI1_D13_G5_Y5_B, SEL_VI1_1),
+	PINMUX_IPSR_GPSR(IP5_2, VI5_VSYNC_N),
+	PINMUX_IPSR_MSEL(IP5_2, VI1_D14_G6_Y6_B, SEL_VI1_1),
+	PINMUX_IPSR_GPSR(IP5_3, VI5_D0_C0),
+	PINMUX_IPSR_MSEL(IP5_3, VI1_D15_G7_Y7_B, SEL_VI1_1),
+	PINMUX_IPSR_GPSR(IP5_4, VI5_D1_C1),
+	PINMUX_IPSR_GPSR(IP5_4, VI1_D16_R0),
+	PINMUX_IPSR_GPSR(IP5_5, VI5_D2_C2),
+	PINMUX_IPSR_GPSR(IP5_5, VI1_D17_R1),
+	PINMUX_IPSR_GPSR(IP5_6, VI5_D3_C3),
+	PINMUX_IPSR_GPSR(IP5_6, VI1_D18_R2),
+	PINMUX_IPSR_GPSR(IP5_7, VI5_D4_C4),
+	PINMUX_IPSR_GPSR(IP5_7, VI1_D19_R3),
+	PINMUX_IPSR_GPSR(IP5_8, VI5_D5_C5),
+	PINMUX_IPSR_GPSR(IP5_8, VI1_D20_R4),
+	PINMUX_IPSR_GPSR(IP5_9, VI5_D6_C6),
+	PINMUX_IPSR_GPSR(IP5_9, VI1_D21_R5),
+	PINMUX_IPSR_GPSR(IP5_10, VI5_D7_C7),
+	PINMUX_IPSR_GPSR(IP5_10, VI1_D22_R6),
+	PINMUX_IPSR_GPSR(IP5_11, VI5_D8_Y0),
+	PINMUX_IPSR_GPSR(IP5_11, VI1_D23_R7),
+
+	/* IPSR6 */
+	PINMUX_IPSR_GPSR(IP6_0, MSIOF0_SCK),
+	PINMUX_IPSR_GPSR(IP6_0, HSCK0),
+	PINMUX_IPSR_GPSR(IP6_1, MSIOF0_SYNC),
+	PINMUX_IPSR_GPSR(IP6_1, HCTS0_N),
+	PINMUX_IPSR_GPSR(IP6_2, MSIOF0_TXD),
+	PINMUX_IPSR_GPSR(IP6_2, HTX0),
+	PINMUX_IPSR_GPSR(IP6_3, MSIOF0_RXD),
+	PINMUX_IPSR_GPSR(IP6_3, HRX0),
+	PINMUX_IPSR_GPSR(IP6_4, MSIOF1_SCK),
+	PINMUX_IPSR_GPSR(IP6_4, HSCK1),
+	PINMUX_IPSR_GPSR(IP6_5, MSIOF1_SYNC),
+	PINMUX_IPSR_GPSR(IP6_5, HRTS1_N),
+	PINMUX_IPSR_GPSR(IP6_6, MSIOF1_TXD),
+	PINMUX_IPSR_GPSR(IP6_6, HTX1),
+	PINMUX_IPSR_GPSR(IP6_7, MSIOF1_RXD),
+	PINMUX_IPSR_GPSR(IP6_7, HRX1),
+	PINMUX_IPSR_GPSR(IP6_9_8, DRACK0),
+	PINMUX_IPSR_GPSR(IP6_9_8, SCK2),
+	PINMUX_IPSR_GPSR(IP6_11_10, DACK0),
+	PINMUX_IPSR_GPSR(IP6_11_10, TX2),
+	PINMUX_IPSR_GPSR(IP6_13_12, DREQ0_N),
+	PINMUX_IPSR_GPSR(IP6_13_12, RX2),
+	PINMUX_IPSR_GPSR(IP6_15_14, DACK1),
+	PINMUX_IPSR_GPSR(IP6_15_14, SCK3),
+	PINMUX_IPSR_GPSR(IP6_16, TX3),
+	PINMUX_IPSR_GPSR(IP6_18_17, DREQ1_N),
+	PINMUX_IPSR_GPSR(IP6_18_17, RX3),
+
+	/* IPSR7 */
+	PINMUX_IPSR_GPSR(IP7_1_0, PWM0),
+	PINMUX_IPSR_GPSR(IP7_1_0, TCLK1),
+	PINMUX_IPSR_GPSR(IP7_1_0, FSO_CFE_0),
+	PINMUX_IPSR_GPSR(IP7_3_2, PWM1),
+	PINMUX_IPSR_GPSR(IP7_3_2, TCLK2),
+	PINMUX_IPSR_GPSR(IP7_3_2, FSO_CFE_1),
+	PINMUX_IPSR_GPSR(IP7_5_4, PWM2),
+	PINMUX_IPSR_GPSR(IP7_5_4, TCLK3),
+	PINMUX_IPSR_GPSR(IP7_5_4, FSO_TOE),
+	PINMUX_IPSR_GPSR(IP7_6, PWM3),
+	PINMUX_IPSR_GPSR(IP7_7, PWM4),
+	PINMUX_IPSR_GPSR(IP7_9_8, SSI_SCK34),
+	PINMUX_IPSR_GPSR(IP7_9_8, TPU0TO0),
+	PINMUX_IPSR_GPSR(IP7_11_10, SSI_WS34),
+	PINMUX_IPSR_GPSR(IP7_11_10, TPU0TO1),
+	PINMUX_IPSR_GPSR(IP7_13_12, SSI_SDATA3),
+	PINMUX_IPSR_GPSR(IP7_13_12, TPU0TO2),
+	PINMUX_IPSR_GPSR(IP7_15_14, SSI_SCK4),
+	PINMUX_IPSR_GPSR(IP7_15_14, TPU0TO3),
+	PINMUX_IPSR_GPSR(IP7_16, SSI_WS4),
+	PINMUX_IPSR_GPSR(IP7_17, SSI_SDATA4),
+	PINMUX_IPSR_GPSR(IP7_18, AUDIO_CLKOUT),
+	PINMUX_IPSR_GPSR(IP7_19, AUDIO_CLKA),
+	PINMUX_IPSR_GPSR(IP7_20, AUDIO_CLKB),
+};
+
+static const struct sh_pfc_pin pinmux_pins[] = {
+	PINMUX_GPIO_GP_ALL(),
+};
+
+/* - AVB -------------------------------------------------------------------- */
+static const unsigned int avb_link_pins[] = {
+	RCAR_GP_PIN(7, 9),
+};
+static const unsigned int avb_link_mux[] = {
+	AVB_LINK_MARK,
+};
+static const unsigned int avb_magic_pins[] = {
+	RCAR_GP_PIN(7, 10),
+};
+static const unsigned int avb_magic_mux[] = {
+	AVB_MAGIC_MARK,
+};
+static const unsigned int avb_phy_int_pins[] = {
+	RCAR_GP_PIN(7, 11),
+};
+static const unsigned int avb_phy_int_mux[] = {
+	AVB_PHY_INT_MARK,
+};
+static const unsigned int avb_mdio_pins[] = {
+	RCAR_GP_PIN(7, 7), RCAR_GP_PIN(7, 8),
+};
+static const unsigned int avb_mdio_mux[] = {
+	AVB_MDC_MARK, AVB_MDIO_MARK,
+};
+static const unsigned int avb_mii_pins[] = {
+	RCAR_GP_PIN(6, 14), RCAR_GP_PIN(6, 15), RCAR_GP_PIN(6, 16),
+	RCAR_GP_PIN(6, 12),
+
+	RCAR_GP_PIN(6, 2),  RCAR_GP_PIN(6, 3),  RCAR_GP_PIN(6, 4),
+	RCAR_GP_PIN(6, 5),
+
+	RCAR_GP_PIN(6, 10), RCAR_GP_PIN(6, 0),  RCAR_GP_PIN(6, 1),
+	RCAR_GP_PIN(7, 12), RCAR_GP_PIN(6, 13), RCAR_GP_PIN(7, 5),
+	RCAR_GP_PIN(7, 0),  RCAR_GP_PIN(6, 11),
+};
+static const unsigned int avb_mii_mux[] = {
+	AVB_TXD0_MARK, AVB_TXD1_MARK, AVB_TXD2_MARK,
+	AVB_TXD3_MARK,
+
+	AVB_RXD0_MARK, AVB_RXD1_MARK, AVB_RXD2_MARK,
+	AVB_RXD3_MARK,
+
+	AVB_RX_ER_MARK, AVB_RX_CLK_MARK, AVB_RX_DV_MARK,
+	AVB_CRS_MARK, AVB_TX_EN_MARK, AVB_TX_ER_MARK,
+	AVB_TX_CLK_MARK, AVB_COL_MARK,
+};
+static const unsigned int avb_gmii_pins[] = {
+	RCAR_GP_PIN(6, 14), RCAR_GP_PIN(6, 15), RCAR_GP_PIN(6, 16),
+	RCAR_GP_PIN(6, 12), RCAR_GP_PIN(7, 1),  RCAR_GP_PIN(7, 2),
+	RCAR_GP_PIN(7, 3),  RCAR_GP_PIN(7, 4),
+
+	RCAR_GP_PIN(6, 2),  RCAR_GP_PIN(6, 3), RCAR_GP_PIN(6, 4),
+	RCAR_GP_PIN(6, 5),  RCAR_GP_PIN(6, 6), RCAR_GP_PIN(6, 7),
+	RCAR_GP_PIN(6, 8),  RCAR_GP_PIN(6, 9),
+
+	RCAR_GP_PIN(6, 10), RCAR_GP_PIN(6, 0), RCAR_GP_PIN(6, 1),
+	RCAR_GP_PIN(7, 12), RCAR_GP_PIN(7, 6), RCAR_GP_PIN(7, 13),
+	RCAR_GP_PIN(6, 13), RCAR_GP_PIN(7, 5), RCAR_GP_PIN(7, 0),
+	RCAR_GP_PIN(6, 11),
+};
+static const unsigned int avb_gmii_mux[] = {
+	AVB_TXD0_MARK, AVB_TXD1_MARK, AVB_TXD2_MARK,
+	AVB_TXD3_MARK, AVB_TXD4_MARK, AVB_TXD5_MARK,
+	AVB_TXD6_MARK, AVB_TXD7_MARK,
+
+	AVB_RXD0_MARK, AVB_RXD1_MARK, AVB_RXD2_MARK,
+	AVB_RXD3_MARK, AVB_RXD4_MARK, AVB_RXD5_MARK,
+	AVB_RXD6_MARK, AVB_RXD7_MARK,
+
+	AVB_RX_ER_MARK, AVB_RX_CLK_MARK, AVB_RX_DV_MARK,
+	AVB_CRS_MARK, AVB_GTX_CLK_MARK, AVB_GTXREFCLK_MARK,
+	AVB_TX_EN_MARK, AVB_TX_ER_MARK, AVB_TX_CLK_MARK,
+	AVB_COL_MARK,
+};
+static const unsigned int avb_avtp_match_pins[] = {
+	RCAR_GP_PIN(7, 15),
+};
+static const unsigned int avb_avtp_match_mux[] = {
+	AVB_AVTP_MATCH_MARK,
+};
+/* - CAN -------------------------------------------------------------------- */
+static const unsigned int can0_data_pins[] = {
+	/* TX, RX */
+	RCAR_GP_PIN(10, 27), RCAR_GP_PIN(10, 28),
+};
+static const unsigned int can0_data_mux[] = {
+	CAN0_TX_MARK, CAN0_RX_MARK,
+};
+static const unsigned int can1_data_pins[] = {
+	/* TX, RX */
+	RCAR_GP_PIN(10, 30), RCAR_GP_PIN(10, 31),
+};
+static const unsigned int can1_data_mux[] = {
+	CAN1_TX_MARK, CAN1_RX_MARK,
+};
+static const unsigned int can_clk_pins[] = {
+	/* CAN_CLK */
+	RCAR_GP_PIN(10, 29),
+};
+static const unsigned int can_clk_mux[] = {
+	CAN_CLK_MARK,
+};
+/* - DU --------------------------------------------------------------------- */
+static const unsigned int du0_rgb666_pins[] = {
+	/* R[7:2], G[7:2], B[7:2] */
+	RCAR_GP_PIN(0, 7), RCAR_GP_PIN(0, 6), RCAR_GP_PIN(0, 5),
+	RCAR_GP_PIN(0, 4), RCAR_GP_PIN(0, 3), RCAR_GP_PIN(0, 2),
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14), RCAR_GP_PIN(0, 13),
+	RCAR_GP_PIN(0, 12), RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
+	RCAR_GP_PIN(0, 23), RCAR_GP_PIN(0, 22), RCAR_GP_PIN(0, 21),
+	RCAR_GP_PIN(0, 20), RCAR_GP_PIN(0, 19), RCAR_GP_PIN(0, 18),
+};
+static const unsigned int du0_rgb666_mux[] = {
+	DU0_DR7_Y9_DATA7_MARK, DU0_DR6_Y8_DATA6_MARK, DU0_DR5_Y7_DATA5_MARK,
+	DU0_DR4_Y6_DATA4_MARK, DU0_DR3_Y5_DATA3_MARK, DU0_DR2_Y4_DATA2_MARK,
+	DU0_DG7_Y3_DATA15_MARK, DU0_DG6_Y2_DATA14_MARK, DU0_DG5_Y1_DATA13_MARK,
+	DU0_DG4_Y0_DATA12_MARK, DU0_DG3_C7_DATA11_MARK, DU0_DG2_C6_DATA10_MARK,
+	DU0_DB7_C5_MARK, DU0_DB6_C4_MARK, DU0_DB5_C3_MARK,
+	DU0_DB4_C2_MARK, DU0_DB3_C1_MARK, DU0_DB2_C0_MARK,
+};
+static const unsigned int du0_rgb888_pins[] = {
+	/* R[7:0], G[7:0], B[7:0] */
+	RCAR_GP_PIN(0, 7), RCAR_GP_PIN(0, 6), RCAR_GP_PIN(0, 5),
+	RCAR_GP_PIN(0, 4), RCAR_GP_PIN(0, 3), RCAR_GP_PIN(0, 2),
+	RCAR_GP_PIN(0, 1), RCAR_GP_PIN(0, 0),
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14), RCAR_GP_PIN(0, 13),
+	RCAR_GP_PIN(0, 12), RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
+	RCAR_GP_PIN(0, 9), RCAR_GP_PIN(0, 8),
+	RCAR_GP_PIN(0, 23), RCAR_GP_PIN(0, 22), RCAR_GP_PIN(0, 21),
+	RCAR_GP_PIN(0, 20), RCAR_GP_PIN(0, 19), RCAR_GP_PIN(0, 18),
+	RCAR_GP_PIN(0, 17), RCAR_GP_PIN(0, 16),
+};
+static const unsigned int du0_rgb888_mux[] = {
+	DU0_DR7_Y9_DATA7_MARK, DU0_DR6_Y8_DATA6_MARK, DU0_DR5_Y7_DATA5_MARK,
+	DU0_DR4_Y6_DATA4_MARK, DU0_DR3_Y5_DATA3_MARK, DU0_DR2_Y4_DATA2_MARK,
+	DU0_DR1_DATA1_MARK, DU0_DR0_DATA0_MARK,
+	DU0_DG7_Y3_DATA15_MARK, DU0_DG6_Y2_DATA14_MARK, DU0_DG5_Y1_DATA13_MARK,
+	DU0_DG4_Y0_DATA12_MARK, DU0_DG3_C7_DATA11_MARK, DU0_DG2_C6_DATA10_MARK,
+	DU0_DG1_DATA9_MARK, DU0_DG0_DATA8_MARK,
+	DU0_DB7_C5_MARK, DU0_DB6_C4_MARK, DU0_DB5_C3_MARK,
+	DU0_DB4_C2_MARK, DU0_DB3_C1_MARK, DU0_DB2_C0_MARK,
+	DU0_DB1_MARK, DU0_DB0_MARK,
+};
+static const unsigned int du0_sync_pins[] = {
+	/* EXVSYNC/VSYNC, EXHSYNC/HSYNC */
+	RCAR_GP_PIN(0, 25), RCAR_GP_PIN(0, 24),
+};
+static const unsigned int du0_sync_mux[] = {
+	DU0_EXVSYNC_DU0_VSYNC_MARK, DU0_EXHSYNC_DU0_HSYNC_MARK,
+};
+static const unsigned int du0_oddf_pins[] = {
+	/* EXODDF/ODDF/DISP/CDE */
+	RCAR_GP_PIN(0, 26),
+};
+static const unsigned int du0_oddf_mux[] = {
+	DU0_EXODDF_DU0_ODDF_DISP_CDE_MARK
+};
+static const unsigned int du0_disp_pins[] = {
+	/* DISP */
+	RCAR_GP_PIN(0, 27),
+};
+static const unsigned int du0_disp_mux[] = {
+	DU0_DISP_MARK,
+};
+static const unsigned int du0_cde_pins[] = {
+	/* CDE */
+	RCAR_GP_PIN(0, 28),
+};
+static const unsigned int du0_cde_mux[] = {
+	DU0_CDE_MARK,
+};
+static const unsigned int du1_rgb666_pins[] = {
+	/* R[7:2], G[7:2], B[7:2] */
+	RCAR_GP_PIN(1, 5), RCAR_GP_PIN(1, 4), RCAR_GP_PIN(1, 3),
+	RCAR_GP_PIN(1, 2), RCAR_GP_PIN(1, 1), RCAR_GP_PIN(1, 0),
+	RCAR_GP_PIN(1, 11), RCAR_GP_PIN(1, 10), RCAR_GP_PIN(1, 9),
+	RCAR_GP_PIN(1, 8), RCAR_GP_PIN(1, 7), RCAR_GP_PIN(1, 6),
+	RCAR_GP_PIN(1, 17), RCAR_GP_PIN(1, 16), RCAR_GP_PIN(1, 15),
+	RCAR_GP_PIN(1, 14), RCAR_GP_PIN(1, 13), RCAR_GP_PIN(1, 12),
+};
+static const unsigned int du1_rgb666_mux[] = {
+	DU1_DR7_DATA5_MARK, DU1_DR6_DATA4_MARK, DU1_DR5_Y7_DATA3_MARK,
+	DU1_DR4_Y6_DATA2_MARK, DU1_DR3_Y5_DATA1_MARK, DU1_DR2_Y4_DATA0_MARK,
+	DU1_DG7_Y3_DATA11_MARK, DU1_DG6_Y2_DATA10_MARK, DU1_DG5_Y1_DATA9_MARK,
+	DU1_DG4_Y0_DATA8_MARK, DU1_DG3_C7_DATA7_MARK, DU1_DG2_C6_DATA6_MARK,
+	DU1_DB7_C5_MARK, DU1_DB6_C4_MARK, DU1_DB5_C3_DATA15_MARK,
+	DU1_DB4_C2_DATA14_MARK, DU1_DB3_C1_DATA13_MARK, DU1_DB2_C0_DATA12_MARK,
+};
+static const unsigned int du1_sync_pins[] = {
+	/* EXVSYNC/VSYNC, EXHSYNC/HSYNC */
+	RCAR_GP_PIN(1, 19), RCAR_GP_PIN(1, 18),
+};
+static const unsigned int du1_sync_mux[] = {
+	DU1_EXVSYNC_DU1_VSYNC_MARK, DU1_EXHSYNC_DU1_HSYNC_MARK,
+};
+static const unsigned int du1_oddf_pins[] = {
+	/* EXODDF/ODDF/DISP/CDE */
+	RCAR_GP_PIN(1, 20),
+};
+static const unsigned int du1_oddf_mux[] = {
+	DU1_EXODDF_DU1_ODDF_DISP_CDE_MARK
+};
+static const unsigned int du1_disp_pins[] = {
+	/* DISP */
+	RCAR_GP_PIN(1, 21),
+};
+static const unsigned int du1_disp_mux[] = {
+	DU1_DISP_MARK,
+};
+static const unsigned int du1_cde_pins[] = {
+	/* CDE */
+	RCAR_GP_PIN(1, 22),
+};
+static const unsigned int du1_cde_mux[] = {
+	DU1_CDE_MARK,
+};
+/* - INTC ------------------------------------------------------------------- */
+static const unsigned int intc_irq0_pins[] = {
+	/* IRQ0 */
+	RCAR_GP_PIN(3, 19),
+};
+static const unsigned int intc_irq0_mux[] = {
+	IRQ0_MARK,
+};
+static const unsigned int intc_irq1_pins[] = {
+	/* IRQ1 */
+	RCAR_GP_PIN(3, 20),
+};
+static const unsigned int intc_irq1_mux[] = {
+	IRQ1_MARK,
+};
+static const unsigned int intc_irq2_pins[] = {
+	/* IRQ2 */
+	RCAR_GP_PIN(3, 21),
+};
+static const unsigned int intc_irq2_mux[] = {
+	IRQ2_MARK,
+};
+static const unsigned int intc_irq3_pins[] = {
+	/* IRQ3 */
+	RCAR_GP_PIN(3, 22),
+};
+static const unsigned int intc_irq3_mux[] = {
+	IRQ3_MARK,
+};
+/* - LBSC ------------------------------------------------------------------- */
+static const unsigned int lbsc_cs0_pins[] = {
+	/* CS0# */
+	RCAR_GP_PIN(3, 27),
+};
+static const unsigned int lbsc_cs0_mux[] = {
+	CS0_N_MARK,
+};
+static const unsigned int lbsc_cs1_pins[] = {
+	/* CS1#_A26 */
+	RCAR_GP_PIN(3, 6),
+};
+static const unsigned int lbsc_cs1_mux[] = {
+	CS1_N_A26_MARK,
+};
+static const unsigned int lbsc_ex_cs0_pins[] = {
+	/* EX_CS0# */
+	RCAR_GP_PIN(3, 7),
+};
+static const unsigned int lbsc_ex_cs0_mux[] = {
+	EX_CS0_N_MARK,
+};
+static const unsigned int lbsc_ex_cs1_pins[] = {
+	/* EX_CS1# */
+	RCAR_GP_PIN(3, 8),
+};
+static const unsigned int lbsc_ex_cs1_mux[] = {
+	EX_CS1_N_MARK,
+};
+static const unsigned int lbsc_ex_cs2_pins[] = {
+	/* EX_CS2# */
+	RCAR_GP_PIN(3, 9),
+};
+static const unsigned int lbsc_ex_cs2_mux[] = {
+	EX_CS2_N_MARK,
+};
+static const unsigned int lbsc_ex_cs3_pins[] = {
+	/* EX_CS3# */
+	RCAR_GP_PIN(3, 10),
+};
+static const unsigned int lbsc_ex_cs3_mux[] = {
+	EX_CS3_N_MARK,
+};
+static const unsigned int lbsc_ex_cs4_pins[] = {
+	/* EX_CS4# */
+	RCAR_GP_PIN(3, 11),
+};
+static const unsigned int lbsc_ex_cs4_mux[] = {
+	EX_CS4_N_MARK,
+};
+static const unsigned int lbsc_ex_cs5_pins[] = {
+	/* EX_CS5# */
+	RCAR_GP_PIN(3, 12),
+};
+static const unsigned int lbsc_ex_cs5_mux[] = {
+	EX_CS5_N_MARK,
+};
+/* - MSIOF0 ----------------------------------------------------------------- */
+static const unsigned int msiof0_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(10, 0),
+};
+static const unsigned int msiof0_clk_mux[] = {
+	MSIOF0_SCK_MARK,
+};
+static const unsigned int msiof0_sync_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(10, 1),
+};
+static const unsigned int msiof0_sync_mux[] = {
+	MSIOF0_SYNC_MARK,
+};
+static const unsigned int msiof0_rx_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(10, 4),
+};
+static const unsigned int msiof0_rx_mux[] = {
+	MSIOF0_RXD_MARK,
+};
+static const unsigned int msiof0_tx_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(10, 3),
+};
+static const unsigned int msiof0_tx_mux[] = {
+	MSIOF0_TXD_MARK,
+};
+/* - MSIOF1 ----------------------------------------------------------------- */
+static const unsigned int msiof1_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(10, 5),
+};
+static const unsigned int msiof1_clk_mux[] = {
+	MSIOF1_SCK_MARK,
+};
+static const unsigned int msiof1_sync_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(10, 6),
+};
+static const unsigned int msiof1_sync_mux[] = {
+	MSIOF1_SYNC_MARK,
+};
+static const unsigned int msiof1_rx_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(10, 9),
+};
+static const unsigned int msiof1_rx_mux[] = {
+	MSIOF1_RXD_MARK,
+};
+static const unsigned int msiof1_tx_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(10, 8),
+};
+static const unsigned int msiof1_tx_mux[] = {
+	MSIOF1_TXD_MARK,
+};
+/* - QSPI ------------------------------------------------------------------- */
+static const unsigned int qspi_ctrl_pins[] = {
+	/* SPCLK, SSL */
+	RCAR_GP_PIN(3, 25), RCAR_GP_PIN(3, 26),
+};
+static const unsigned int qspi_ctrl_mux[] = {
+	SPCLK_MARK, SSL_MARK,
+};
+static const unsigned int qspi_data2_pins[] = {
+	/* MOSI_IO0, MISO_IO1 */
+	RCAR_GP_PIN(3, 4), RCAR_GP_PIN(3, 5),
+};
+static const unsigned int qspi_data2_mux[] = {
+	MOSI_IO0_MARK, MISO_IO1_MARK,
+};
+static const unsigned int qspi_data4_pins[] = {
+	/* MOSI_IO0, MISO_IO1, IO2, IO3 */
+	RCAR_GP_PIN(3, 4), RCAR_GP_PIN(3, 5), RCAR_GP_PIN(3, 23),
+	RCAR_GP_PIN(3, 24),
+};
+static const unsigned int qspi_data4_mux[] = {
+	MOSI_IO0_MARK, MISO_IO1_MARK, IO2_MARK,	IO3_MARK,
+};
+/* - SCIF0 ------------------------------------------------------------------ */
+static const unsigned int scif0_data_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(10, 14), RCAR_GP_PIN(10, 13),
+};
+static const unsigned int scif0_data_mux[] = {
+	RX0_MARK, TX0_MARK,
+};
+static const unsigned int scif0_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(10, 10),
+};
+static const unsigned int scif0_clk_mux[] = {
+	SCK0_MARK,
+};
+static const unsigned int scif0_ctrl_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(10, 12), RCAR_GP_PIN(10, 11),
+};
+static const unsigned int scif0_ctrl_mux[] = {
+	RTS0_N_MARK, CTS0_N_MARK,
+};
+/* - SCIF3 ------------------------------------------------------------------ */
+static const unsigned int scif3_data_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(10, 25), RCAR_GP_PIN(10, 24),
+};
+static const unsigned int scif3_data_mux[] = {
+	RX3_MARK, TX3_MARK,
+};
+static const unsigned int scif3_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(10, 23),
+};
+static const unsigned int scif3_clk_mux[] = {
+	SCK3_MARK,
+};
+/* - SDHI0 ------------------------------------------------------------------ */
+static const unsigned int sdhi0_data1_pins[] = {
+	/* DAT0 */
+	RCAR_GP_PIN(11, 7),
+};
+static const unsigned int sdhi0_data1_mux[] = {
+	SD0_DAT0_MARK,
+};
+static const unsigned int sdhi0_data4_pins[] = {
+	/* DAT[0-3] */
+	RCAR_GP_PIN(11, 7), RCAR_GP_PIN(11, 8),
+	RCAR_GP_PIN(11, 9), RCAR_GP_PIN(11, 10),
+};
+static const unsigned int sdhi0_data4_mux[] = {
+	SD0_DAT0_MARK, SD0_DAT1_MARK, SD0_DAT2_MARK, SD0_DAT3_MARK,
+};
+static const unsigned int sdhi0_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(11, 5), RCAR_GP_PIN(11, 6),
+};
+static const unsigned int sdhi0_ctrl_mux[] = {
+	SD0_CLK_MARK, SD0_CMD_MARK,
+};
+static const unsigned int sdhi0_cd_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(11, 11),
+};
+static const unsigned int sdhi0_cd_mux[] = {
+	SD0_CD_MARK,
+};
+static const unsigned int sdhi0_wp_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(11, 12),
+};
+static const unsigned int sdhi0_wp_mux[] = {
+	SD0_WP_MARK,
+};
+/* - VIN0 ------------------------------------------------------------------- */
+static const union vin_data vin0_data_pins = {
+	.data24 = {
+		/* B */
+		RCAR_GP_PIN(4, 4), RCAR_GP_PIN(4, 5),
+		RCAR_GP_PIN(4, 6), RCAR_GP_PIN(4, 7),
+		RCAR_GP_PIN(4, 8), RCAR_GP_PIN(4, 9),
+		RCAR_GP_PIN(4, 10), RCAR_GP_PIN(4, 11),
+		/* G */
+		RCAR_GP_PIN(4, 12), RCAR_GP_PIN(4, 13),
+		RCAR_GP_PIN(4, 14), RCAR_GP_PIN(4, 15),
+		RCAR_GP_PIN(8, 1), RCAR_GP_PIN(8, 2),
+		RCAR_GP_PIN(8, 3), RCAR_GP_PIN(8, 4),
+		/* R */
+		RCAR_GP_PIN(8, 5), RCAR_GP_PIN(8, 6),
+		RCAR_GP_PIN(8, 7), RCAR_GP_PIN(8, 8),
+		RCAR_GP_PIN(8, 9), RCAR_GP_PIN(8, 10),
+		RCAR_GP_PIN(8, 11), RCAR_GP_PIN(8, 12),
+	},
+};
+static const union vin_data vin0_data_mux = {
+	.data24 = {
+		/* B */
+		VI0_D0_B0_C0_MARK, VI0_D1_B1_C1_MARK,
+		VI0_D2_B2_C2_MARK, VI0_D3_B3_C3_MARK,
+		VI0_D4_B4_C4_MARK, VI0_D5_B5_C5_MARK,
+		VI0_D6_B6_C6_MARK, VI0_D7_B7_C7_MARK,
+		/* G */
+		VI0_D8_G0_Y0_MARK, VI0_D9_G1_Y1_MARK,
+		VI0_D10_G2_Y2_MARK, VI0_D11_G3_Y3_MARK,
+		VI0_D12_G4_Y4_MARK, VI0_D13_G5_Y5_MARK,
+		VI0_D14_G6_Y6_MARK, VI0_D15_G7_Y7_MARK,
+		/* R */
+		VI0_D16_R0_MARK, VI0_D17_R1_MARK,
+		VI0_D18_R2_MARK, VI0_D19_R3_MARK,
+		VI0_D20_R4_MARK, VI0_D21_R5_MARK,
+		VI0_D22_R6_MARK, VI0_D23_R7_MARK,
+	},
+};
+static const unsigned int vin0_data18_pins[] = {
+	/* B */
+	RCAR_GP_PIN(4, 6), RCAR_GP_PIN(4, 7),
+	RCAR_GP_PIN(4, 8), RCAR_GP_PIN(4, 9),
+	RCAR_GP_PIN(4, 10), RCAR_GP_PIN(4, 11),
+	/* G */
+	RCAR_GP_PIN(4, 14), RCAR_GP_PIN(4, 15),
+	RCAR_GP_PIN(8, 1), RCAR_GP_PIN(8, 2),
+	RCAR_GP_PIN(8, 3), RCAR_GP_PIN(8, 4),
+	/* R */
+	RCAR_GP_PIN(8, 7), RCAR_GP_PIN(8, 8),
+	RCAR_GP_PIN(8, 9), RCAR_GP_PIN(8, 10),
+	RCAR_GP_PIN(8, 11), RCAR_GP_PIN(8, 12),
+};
+static const unsigned int vin0_data18_mux[] = {
+	/* B */
+	VI0_D2_B2_C2_MARK, VI0_D3_B3_C3_MARK,
+	VI0_D4_B4_C4_MARK, VI0_D5_B5_C5_MARK,
+	VI0_D6_B6_C6_MARK, VI0_D7_B7_C7_MARK,
+	/* G */
+	VI0_D10_G2_Y2_MARK, VI0_D11_G3_Y3_MARK,
+	VI0_D12_G4_Y4_MARK, VI0_D13_G5_Y5_MARK,
+	VI0_D14_G6_Y6_MARK, VI0_D15_G7_Y7_MARK,
+	/* R */
+	VI0_D18_R2_MARK, VI0_D19_R3_MARK,
+	VI0_D20_R4_MARK, VI0_D21_R5_MARK,
+	VI0_D22_R6_MARK, VI0_D23_R7_MARK,
+};
+static const unsigned int vin0_sync_pins[] = {
+	/* HSYNC#, VSYNC# */
+	RCAR_GP_PIN(4, 2), RCAR_GP_PIN(4, 3),
+};
+static const unsigned int vin0_sync_mux[] = {
+	VI0_HSYNC_N_MARK, VI0_VSYNC_N_MARK,
+};
+static const unsigned int vin0_field_pins[] = {
+	RCAR_GP_PIN(4, 16),
+};
+static const unsigned int vin0_field_mux[] = {
+	VI0_FIELD_MARK,
+};
+static const unsigned int vin0_clkenb_pins[] = {
+	RCAR_GP_PIN(4, 1),
+};
+static const unsigned int vin0_clkenb_mux[] = {
+	VI0_CLKENB_MARK,
+};
+static const unsigned int vin0_clk_pins[] = {
+	RCAR_GP_PIN(4, 0),
+};
+static const unsigned int vin0_clk_mux[] = {
+	VI0_CLK_MARK,
+};
+/* - VIN1 ------------------------------------------------------------------- */
+static const union vin_data vin1_data_pins = {
+	.data24 = {
+		/* B */
+		RCAR_GP_PIN(5, 4), RCAR_GP_PIN(5, 5),
+		RCAR_GP_PIN(5, 6), RCAR_GP_PIN(5, 7),
+		RCAR_GP_PIN(5, 8), RCAR_GP_PIN(5, 9),
+		RCAR_GP_PIN(5, 10), RCAR_GP_PIN(5, 11),
+		/* G */
+		RCAR_GP_PIN(5, 12), RCAR_GP_PIN(5, 13),
+		RCAR_GP_PIN(5, 14), RCAR_GP_PIN(5, 15),
+		RCAR_GP_PIN(8, 5), RCAR_GP_PIN(8, 6),
+		RCAR_GP_PIN(8, 7), RCAR_GP_PIN(8, 8),
+		/* R */
+		RCAR_GP_PIN(9, 5), RCAR_GP_PIN(9, 6),
+		RCAR_GP_PIN(9, 7), RCAR_GP_PIN(9, 8),
+		RCAR_GP_PIN(9, 9), RCAR_GP_PIN(9, 10),
+		RCAR_GP_PIN(9, 11), RCAR_GP_PIN(9, 12),
+	},
+};
+static const union vin_data vin1_data_mux = {
+	.data24 = {
+		/* B */
+		VI1_D0_B0_C0_MARK, VI1_D1_B1_C1_MARK,
+		VI1_D2_B2_C2_MARK, VI1_D3_B3_C3_MARK,
+		VI1_D4_B4_C4_MARK, VI1_D5_B5_C5_MARK,
+		VI1_D6_B6_C6_MARK, VI1_D7_B7_C7_MARK,
+		/* G */
+		VI1_D8_G0_Y0_MARK, VI1_D9_G1_Y1_MARK,
+		VI1_D10_G2_Y2_MARK, VI1_D11_G3_Y3_MARK,
+		VI1_D12_G4_Y4_MARK, VI1_D13_G5_Y5_MARK,
+		VI1_D14_G6_Y6_MARK, VI1_D15_G7_Y7_MARK,
+		/* R */
+		VI1_D16_R0_MARK, VI1_D17_R1_MARK,
+		VI1_D18_R2_MARK, VI1_D19_R3_MARK,
+		VI1_D20_R4_MARK, VI1_D21_R5_MARK,
+		VI1_D22_R6_MARK, VI1_D23_R7_MARK,
+	},
+};
+static const unsigned int vin1_data18_pins[] = {
+	/* B */
+	RCAR_GP_PIN(5, 6), RCAR_GP_PIN(5, 7),
+	RCAR_GP_PIN(5, 8), RCAR_GP_PIN(5, 9),
+	RCAR_GP_PIN(5, 10), RCAR_GP_PIN(5, 11),
+	/* G */
+	RCAR_GP_PIN(5, 14), RCAR_GP_PIN(5, 15),
+	RCAR_GP_PIN(8, 5), RCAR_GP_PIN(8, 6),
+	RCAR_GP_PIN(8, 7), RCAR_GP_PIN(8, 8),
+	/* R */
+	RCAR_GP_PIN(9, 7), RCAR_GP_PIN(9, 8),
+	RCAR_GP_PIN(9, 9), RCAR_GP_PIN(9, 10),
+	RCAR_GP_PIN(9, 11), RCAR_GP_PIN(9, 12),
+};
+static const unsigned int vin1_data18_mux[] = {
+	/* B */
+	VI1_D2_B2_C2_MARK, VI1_D3_B3_C3_MARK,
+	VI1_D4_B4_C4_MARK, VI1_D5_B5_C5_MARK,
+	VI1_D6_B6_C6_MARK, VI1_D7_B7_C7_MARK,
+	/* G */
+	VI1_D10_G2_Y2_MARK, VI1_D11_G3_Y3_MARK,
+	VI1_D12_G4_Y4_MARK, VI1_D13_G5_Y5_MARK,
+	VI1_D14_G6_Y6_MARK, VI1_D15_G7_Y7_MARK,
+	/* R */
+	VI1_D18_R2_MARK, VI1_D19_R3_MARK,
+	VI1_D20_R4_MARK, VI1_D21_R5_MARK,
+	VI1_D22_R6_MARK, VI1_D23_R7_MARK,
+};
+static const union vin_data vin1_data_b_pins = {
+	.data24 = {
+		/* B */
+		RCAR_GP_PIN(5, 4), RCAR_GP_PIN(5, 5),
+		RCAR_GP_PIN(5, 6), RCAR_GP_PIN(5, 7),
+		RCAR_GP_PIN(5, 8), RCAR_GP_PIN(5, 9),
+		RCAR_GP_PIN(5, 10), RCAR_GP_PIN(5, 11),
+		/* G */
+		RCAR_GP_PIN(5, 12), RCAR_GP_PIN(5, 13),
+		RCAR_GP_PIN(5, 14), RCAR_GP_PIN(5, 15),
+		RCAR_GP_PIN(9, 1), RCAR_GP_PIN(9, 2),
+		RCAR_GP_PIN(9, 3), RCAR_GP_PIN(9, 4),
+		/* R */
+		RCAR_GP_PIN(9, 5), RCAR_GP_PIN(9, 6),
+		RCAR_GP_PIN(9, 7), RCAR_GP_PIN(9, 8),
+		RCAR_GP_PIN(9, 9), RCAR_GP_PIN(9, 10),
+		RCAR_GP_PIN(9, 11), RCAR_GP_PIN(9, 12),
+	},
+};
+static const union vin_data vin1_data_b_mux = {
+	.data24 = {
+		/* B */
+		VI1_D0_B0_C0_MARK, VI1_D1_B1_C1_MARK,
+		VI1_D2_B2_C2_MARK, VI1_D3_B3_C3_MARK,
+		VI1_D4_B4_C4_MARK, VI1_D5_B5_C5_MARK,
+		VI1_D6_B6_C6_MARK, VI1_D7_B7_C7_MARK,
+		/* G */
+		VI1_D8_G0_Y0_MARK, VI1_D9_G1_Y1_MARK,
+		VI1_D10_G2_Y2_MARK, VI1_D11_G3_Y3_MARK,
+		VI1_D12_G4_Y4_B_MARK, VI1_D13_G5_Y5_B_MARK,
+		VI1_D14_G6_Y6_B_MARK, VI1_D15_G7_Y7_B_MARK,
+		/* R */
+		VI1_D16_R0_MARK, VI1_D17_R1_MARK,
+		VI1_D18_R2_MARK, VI1_D19_R3_MARK,
+		VI1_D20_R4_MARK, VI1_D21_R5_MARK,
+		VI1_D22_R6_MARK, VI1_D23_R7_MARK,
+	},
+};
+static const unsigned int vin1_data18_b_pins[] = {
+	/* B */
+	RCAR_GP_PIN(5, 6), RCAR_GP_PIN(5, 7),
+	RCAR_GP_PIN(5, 8), RCAR_GP_PIN(5, 9),
+	RCAR_GP_PIN(5, 10), RCAR_GP_PIN(5, 11),
+	/* G */
+	RCAR_GP_PIN(5, 14), RCAR_GP_PIN(5, 15),
+	RCAR_GP_PIN(9, 1), RCAR_GP_PIN(9, 2),
+	RCAR_GP_PIN(9, 3), RCAR_GP_PIN(9, 4),
+	/* R */
+	RCAR_GP_PIN(9, 7), RCAR_GP_PIN(9, 8),
+	RCAR_GP_PIN(9, 9), RCAR_GP_PIN(9, 10),
+	RCAR_GP_PIN(9, 11), RCAR_GP_PIN(9, 12),
+};
+static const unsigned int vin1_data18_b_mux[] = {
+	/* B */
+	VI1_D2_B2_C2_MARK, VI1_D3_B3_C3_MARK,
+	VI1_D4_B4_C4_MARK, VI1_D5_B5_C5_MARK,
+	VI1_D6_B6_C6_MARK, VI1_D7_B7_C7_MARK,
+	/* G */
+	VI1_D10_G2_Y2_MARK, VI1_D11_G3_Y3_MARK,
+	VI1_D12_G4_Y4_B_MARK, VI1_D13_G5_Y5_B_MARK,
+	VI1_D14_G6_Y6_B_MARK, VI1_D15_G7_Y7_B_MARK,
+	/* R */
+	VI1_D18_R2_MARK, VI1_D19_R3_MARK,
+	VI1_D20_R4_MARK, VI1_D21_R5_MARK,
+	VI1_D22_R6_MARK, VI1_D23_R7_MARK,
+};
+static const unsigned int vin1_sync_pins[] = {
+	/* HSYNC#, VSYNC# */
+	RCAR_GP_PIN(5, 2), RCAR_GP_PIN(5, 3),
+};
+static const unsigned int vin1_sync_mux[] = {
+	VI1_HSYNC_N_MARK, VI1_VSYNC_N_MARK,
+};
+static const unsigned int vin1_field_pins[] = {
+	RCAR_GP_PIN(5, 16),
+};
+static const unsigned int vin1_field_mux[] = {
+	VI1_FIELD_MARK,
+};
+static const unsigned int vin1_clkenb_pins[] = {
+	RCAR_GP_PIN(5, 1),
+};
+static const unsigned int vin1_clkenb_mux[] = {
+	VI1_CLKENB_MARK,
+};
+static const unsigned int vin1_clk_pins[] = {
+	RCAR_GP_PIN(5, 0),
+};
+static const unsigned int vin1_clk_mux[] = {
+	VI1_CLK_MARK,
+};
+/* - VIN2 ------------------------------------------------------------------- */
+static const union vin_data vin2_data_pins = {
+	.data16 = {
+		RCAR_GP_PIN(6, 4), RCAR_GP_PIN(6, 5),
+		RCAR_GP_PIN(6, 6), RCAR_GP_PIN(6, 7),
+		RCAR_GP_PIN(6, 8), RCAR_GP_PIN(6, 9),
+		RCAR_GP_PIN(6, 10), RCAR_GP_PIN(6, 11),
+		RCAR_GP_PIN(6, 12), RCAR_GP_PIN(6, 13),
+		RCAR_GP_PIN(6, 14), RCAR_GP_PIN(6, 15),
+		RCAR_GP_PIN(8, 9), RCAR_GP_PIN(8, 10),
+		RCAR_GP_PIN(8, 11), RCAR_GP_PIN(8, 12),
+	},
+};
+static const union vin_data vin2_data_mux = {
+	.data16 = {
+		VI2_D0_C0_MARK, VI2_D1_C1_MARK,
+		VI2_D2_C2_MARK,	VI2_D3_C3_MARK,
+		VI2_D4_C4_MARK, VI2_D5_C5_MARK,
+		VI2_D6_C6_MARK, VI2_D7_C7_MARK,
+		VI2_D8_Y0_MARK,	VI2_D9_Y1_MARK,
+		VI2_D10_Y2_MARK, VI2_D11_Y3_MARK,
+		VI2_D12_Y4_MARK, VI2_D13_Y5_MARK,
+		VI2_D14_Y6_MARK, VI2_D15_Y7_MARK,
+	},
+};
+static const unsigned int vin2_sync_pins[] = {
+	/* HSYNC#, VSYNC# */
+	RCAR_GP_PIN(6, 2), RCAR_GP_PIN(6, 3),
+};
+static const unsigned int vin2_sync_mux[] = {
+	VI2_HSYNC_N_MARK, VI2_VSYNC_N_MARK,
+};
+static const unsigned int vin2_field_pins[] = {
+	RCAR_GP_PIN(6, 16),
+};
+static const unsigned int vin2_field_mux[] = {
+	VI2_FIELD_MARK,
+};
+static const unsigned int vin2_clkenb_pins[] = {
+	RCAR_GP_PIN(6, 1),
+};
+static const unsigned int vin2_clkenb_mux[] = {
+	VI2_CLKENB_MARK,
+};
+static const unsigned int vin2_clk_pins[] = {
+	RCAR_GP_PIN(6, 0),
+};
+static const unsigned int vin2_clk_mux[] = {
+	VI2_CLK_MARK,
+};
+/* - VIN3 ------------------------------------------------------------------- */
+static const union vin_data vin3_data_pins = {
+	.data16 = {
+		RCAR_GP_PIN(7, 4), RCAR_GP_PIN(7, 5),
+		RCAR_GP_PIN(7, 6), RCAR_GP_PIN(7, 7),
+		RCAR_GP_PIN(7, 8), RCAR_GP_PIN(7, 9),
+		RCAR_GP_PIN(7, 10), RCAR_GP_PIN(7, 11),
+		RCAR_GP_PIN(7, 12), RCAR_GP_PIN(7, 13),
+		RCAR_GP_PIN(7, 14), RCAR_GP_PIN(7, 15),
+		RCAR_GP_PIN(8, 13), RCAR_GP_PIN(8, 14),
+		RCAR_GP_PIN(8, 15), RCAR_GP_PIN(8, 16),
+	},
+};
+static const union vin_data vin3_data_mux = {
+	.data16 = {
+		VI3_D0_C0_MARK, VI3_D1_C1_MARK,
+		VI3_D2_C2_MARK,	VI3_D3_C3_MARK,
+		VI3_D4_C4_MARK, VI3_D5_C5_MARK,
+		VI3_D6_C6_MARK, VI3_D7_C7_MARK,
+		VI3_D8_Y0_MARK, VI3_D9_Y1_MARK,
+		VI3_D10_Y2_MARK, VI3_D11_Y3_MARK,
+		VI3_D12_Y4_MARK, VI3_D13_Y5_MARK,
+		VI3_D14_Y6_MARK, VI3_D15_Y7_MARK,
+	},
+};
+static const unsigned int vin3_sync_pins[] = {
+	/* HSYNC#, VSYNC# */
+	RCAR_GP_PIN(7, 2), RCAR_GP_PIN(7, 3),
+};
+static const unsigned int vin3_sync_mux[] = {
+	VI3_HSYNC_N_MARK, VI3_VSYNC_N_MARK,
+};
+static const unsigned int vin3_field_pins[] = {
+	RCAR_GP_PIN(7, 16),
+};
+static const unsigned int vin3_field_mux[] = {
+	VI3_FIELD_MARK,
+};
+static const unsigned int vin3_clkenb_pins[] = {
+	RCAR_GP_PIN(7, 1),
+};
+static const unsigned int vin3_clkenb_mux[] = {
+	VI3_CLKENB_MARK,
+};
+static const unsigned int vin3_clk_pins[] = {
+	RCAR_GP_PIN(7, 0),
+};
+static const unsigned int vin3_clk_mux[] = {
+	VI3_CLK_MARK,
+};
+/* - VIN4 ------------------------------------------------------------------- */
+static const union vin_data vin4_data_pins = {
+	.data12 = {
+		RCAR_GP_PIN(8, 4), RCAR_GP_PIN(8, 5),
+		RCAR_GP_PIN(8, 6), RCAR_GP_PIN(8, 7),
+		RCAR_GP_PIN(8, 8), RCAR_GP_PIN(8, 9),
+		RCAR_GP_PIN(8, 10), RCAR_GP_PIN(8, 11),
+		RCAR_GP_PIN(8, 12), RCAR_GP_PIN(8, 13),
+		RCAR_GP_PIN(8, 14), RCAR_GP_PIN(8, 15),
+	},
+};
+static const union vin_data vin4_data_mux = {
+	.data12 = {
+		VI4_D0_C0_MARK, VI4_D1_C1_MARK,
+		VI4_D2_C2_MARK, VI4_D3_C3_MARK,
+		VI4_D4_C4_MARK, VI4_D5_C5_MARK,
+		VI4_D6_C6_MARK, VI4_D7_C7_MARK,
+		VI4_D8_Y0_MARK,	VI4_D9_Y1_MARK,
+		VI4_D10_Y2_MARK, VI4_D11_Y3_MARK,
+	},
+};
+static const unsigned int vin4_sync_pins[] = {
+	 /* HSYNC#, VSYNC# */
+	RCAR_GP_PIN(8, 2), RCAR_GP_PIN(8, 3),
+};
+static const unsigned int vin4_sync_mux[] = {
+	VI4_HSYNC_N_MARK, VI4_VSYNC_N_MARK,
+};
+static const unsigned int vin4_field_pins[] = {
+	RCAR_GP_PIN(8, 16),
+};
+static const unsigned int vin4_field_mux[] = {
+	VI4_FIELD_MARK,
+};
+static const unsigned int vin4_clkenb_pins[] = {
+	RCAR_GP_PIN(8, 1),
+};
+static const unsigned int vin4_clkenb_mux[] = {
+	VI4_CLKENB_MARK,
+};
+static const unsigned int vin4_clk_pins[] = {
+	RCAR_GP_PIN(8, 0),
+};
+static const unsigned int vin4_clk_mux[] = {
+	VI4_CLK_MARK,
+};
+/* - VIN5 ------------------------------------------------------------------- */
+static const union vin_data vin5_data_pins = {
+	.data12 = {
+		RCAR_GP_PIN(9, 4), RCAR_GP_PIN(9, 5),
+		RCAR_GP_PIN(9, 6), RCAR_GP_PIN(9, 7),
+		RCAR_GP_PIN(9, 8), RCAR_GP_PIN(9, 9),
+		RCAR_GP_PIN(9, 10), RCAR_GP_PIN(9, 11),
+		RCAR_GP_PIN(9, 12), RCAR_GP_PIN(9, 13),
+		RCAR_GP_PIN(9, 14), RCAR_GP_PIN(9, 15),
+	},
+};
+static const union vin_data vin5_data_mux = {
+	.data12 = {
+		VI5_D0_C0_MARK, VI5_D1_C1_MARK,
+		VI5_D2_C2_MARK, VI5_D3_C3_MARK,
+		VI5_D4_C4_MARK, VI5_D5_C5_MARK,
+		VI5_D6_C6_MARK, VI5_D7_C7_MARK,
+		VI5_D8_Y0_MARK, VI5_D9_Y1_MARK,
+		VI5_D10_Y2_MARK, VI5_D11_Y3_MARK,
+	},
+};
+static const unsigned int vin5_sync_pins[] = {
+	/* HSYNC#, VSYNC# */
+	RCAR_GP_PIN(9, 2), RCAR_GP_PIN(9, 3),
+};
+static const unsigned int vin5_sync_mux[] = {
+	VI5_HSYNC_N_MARK, VI5_VSYNC_N_MARK,
+};
+static const unsigned int vin5_field_pins[] = {
+	RCAR_GP_PIN(9, 16),
+};
+static const unsigned int vin5_field_mux[] = {
+	VI5_FIELD_MARK,
+};
+static const unsigned int vin5_clkenb_pins[] = {
+	RCAR_GP_PIN(9, 1),
+};
+static const unsigned int vin5_clkenb_mux[] = {
+	VI5_CLKENB_MARK,
+};
+static const unsigned int vin5_clk_pins[] = {
+	RCAR_GP_PIN(9, 0),
+};
+static const unsigned int vin5_clk_mux[] = {
+	VI5_CLK_MARK,
+};
+
+static const struct sh_pfc_pin_group pinmux_groups[] = {
+	SH_PFC_PIN_GROUP(avb_link),
+	SH_PFC_PIN_GROUP(avb_magic),
+	SH_PFC_PIN_GROUP(avb_phy_int),
+	SH_PFC_PIN_GROUP(avb_mdio),
+	SH_PFC_PIN_GROUP(avb_mii),
+	SH_PFC_PIN_GROUP(avb_gmii),
+	SH_PFC_PIN_GROUP(avb_avtp_match),
+	SH_PFC_PIN_GROUP(can0_data),
+	SH_PFC_PIN_GROUP(can1_data),
+	SH_PFC_PIN_GROUP(can_clk),
+	SH_PFC_PIN_GROUP(du0_rgb666),
+	SH_PFC_PIN_GROUP(du0_rgb888),
+	SH_PFC_PIN_GROUP(du0_sync),
+	SH_PFC_PIN_GROUP(du0_oddf),
+	SH_PFC_PIN_GROUP(du0_disp),
+	SH_PFC_PIN_GROUP(du0_cde),
+	SH_PFC_PIN_GROUP(du1_rgb666),
+	SH_PFC_PIN_GROUP(du1_sync),
+	SH_PFC_PIN_GROUP(du1_oddf),
+	SH_PFC_PIN_GROUP(du1_disp),
+	SH_PFC_PIN_GROUP(du1_cde),
+	SH_PFC_PIN_GROUP(intc_irq0),
+	SH_PFC_PIN_GROUP(intc_irq1),
+	SH_PFC_PIN_GROUP(intc_irq2),
+	SH_PFC_PIN_GROUP(intc_irq3),
+	SH_PFC_PIN_GROUP(lbsc_cs0),
+	SH_PFC_PIN_GROUP(lbsc_cs1),
+	SH_PFC_PIN_GROUP(lbsc_ex_cs0),
+	SH_PFC_PIN_GROUP(lbsc_ex_cs1),
+	SH_PFC_PIN_GROUP(lbsc_ex_cs2),
+	SH_PFC_PIN_GROUP(lbsc_ex_cs3),
+	SH_PFC_PIN_GROUP(lbsc_ex_cs4),
+	SH_PFC_PIN_GROUP(lbsc_ex_cs5),
+	SH_PFC_PIN_GROUP(msiof0_clk),
+	SH_PFC_PIN_GROUP(msiof0_sync),
+	SH_PFC_PIN_GROUP(msiof0_rx),
+	SH_PFC_PIN_GROUP(msiof0_tx),
+	SH_PFC_PIN_GROUP(msiof1_clk),
+	SH_PFC_PIN_GROUP(msiof1_sync),
+	SH_PFC_PIN_GROUP(msiof1_rx),
+	SH_PFC_PIN_GROUP(msiof1_tx),
+	SH_PFC_PIN_GROUP(qspi_ctrl),
+	SH_PFC_PIN_GROUP(qspi_data2),
+	SH_PFC_PIN_GROUP(qspi_data4),
+	SH_PFC_PIN_GROUP(scif0_data),
+	SH_PFC_PIN_GROUP(scif0_clk),
+	SH_PFC_PIN_GROUP(scif0_ctrl),
+	SH_PFC_PIN_GROUP(scif3_data),
+	SH_PFC_PIN_GROUP(scif3_clk),
+	SH_PFC_PIN_GROUP(sdhi0_data1),
+	SH_PFC_PIN_GROUP(sdhi0_data4),
+	SH_PFC_PIN_GROUP(sdhi0_ctrl),
+	SH_PFC_PIN_GROUP(sdhi0_cd),
+	SH_PFC_PIN_GROUP(sdhi0_wp),
+	VIN_DATA_PIN_GROUP(vin0_data, 24),
+	VIN_DATA_PIN_GROUP(vin0_data, 20),
+	SH_PFC_PIN_GROUP(vin0_data18),
+	VIN_DATA_PIN_GROUP(vin0_data, 16),
+	VIN_DATA_PIN_GROUP(vin0_data, 12),
+	VIN_DATA_PIN_GROUP(vin0_data, 10),
+	VIN_DATA_PIN_GROUP(vin0_data, 8),
+	SH_PFC_PIN_GROUP(vin0_sync),
+	SH_PFC_PIN_GROUP(vin0_field),
+	SH_PFC_PIN_GROUP(vin0_clkenb),
+	SH_PFC_PIN_GROUP(vin0_clk),
+	VIN_DATA_PIN_GROUP(vin1_data, 24),
+	VIN_DATA_PIN_GROUP(vin1_data, 20),
+	SH_PFC_PIN_GROUP(vin1_data18),
+	VIN_DATA_PIN_GROUP(vin1_data, 16),
+	VIN_DATA_PIN_GROUP(vin1_data, 12),
+	VIN_DATA_PIN_GROUP(vin1_data, 10),
+	VIN_DATA_PIN_GROUP(vin1_data, 8),
+	VIN_DATA_PIN_GROUP(vin1_data_b, 24),
+	VIN_DATA_PIN_GROUP(vin1_data_b, 20),
+	SH_PFC_PIN_GROUP(vin1_data18_b),
+	VIN_DATA_PIN_GROUP(vin1_data_b, 16),
+	SH_PFC_PIN_GROUP(vin1_sync),
+	SH_PFC_PIN_GROUP(vin1_field),
+	SH_PFC_PIN_GROUP(vin1_clkenb),
+	SH_PFC_PIN_GROUP(vin1_clk),
+	VIN_DATA_PIN_GROUP(vin2_data, 16),
+	VIN_DATA_PIN_GROUP(vin2_data, 12),
+	VIN_DATA_PIN_GROUP(vin2_data, 10),
+	VIN_DATA_PIN_GROUP(vin2_data, 8),
+	SH_PFC_PIN_GROUP(vin2_sync),
+	SH_PFC_PIN_GROUP(vin2_field),
+	SH_PFC_PIN_GROUP(vin2_clkenb),
+	SH_PFC_PIN_GROUP(vin2_clk),
+	VIN_DATA_PIN_GROUP(vin3_data, 16),
+	VIN_DATA_PIN_GROUP(vin3_data, 12),
+	VIN_DATA_PIN_GROUP(vin3_data, 10),
+	VIN_DATA_PIN_GROUP(vin3_data, 8),
+	SH_PFC_PIN_GROUP(vin3_sync),
+	SH_PFC_PIN_GROUP(vin3_field),
+	SH_PFC_PIN_GROUP(vin3_clkenb),
+	SH_PFC_PIN_GROUP(vin3_clk),
+	VIN_DATA_PIN_GROUP(vin4_data, 12),
+	VIN_DATA_PIN_GROUP(vin4_data, 10),
+	VIN_DATA_PIN_GROUP(vin4_data, 8),
+	SH_PFC_PIN_GROUP(vin4_sync),
+	SH_PFC_PIN_GROUP(vin4_field),
+	SH_PFC_PIN_GROUP(vin4_clkenb),
+	SH_PFC_PIN_GROUP(vin4_clk),
+	VIN_DATA_PIN_GROUP(vin5_data, 12),
+	VIN_DATA_PIN_GROUP(vin5_data, 10),
+	VIN_DATA_PIN_GROUP(vin5_data, 8),
+	SH_PFC_PIN_GROUP(vin5_sync),
+	SH_PFC_PIN_GROUP(vin5_field),
+	SH_PFC_PIN_GROUP(vin5_clkenb),
+	SH_PFC_PIN_GROUP(vin5_clk),
+};
+
+static const char * const avb_groups[] = {
+	"avb_link",
+	"avb_magic",
+	"avb_phy_int",
+	"avb_mdio",
+	"avb_mii",
+	"avb_gmii",
+	"avb_avtp_match",
+};
+
+static const char * const can0_groups[] = {
+	"can0_data",
+	"can_clk",
+};
+
+static const char * const can1_groups[] = {
+	"can1_data",
+	"can_clk",
+};
+
+static const char * const du0_groups[] = {
+	"du0_rgb666",
+	"du0_rgb888",
+	"du0_sync",
+	"du0_oddf",
+	"du0_disp",
+	"du0_cde",
+};
+
+static const char * const du1_groups[] = {
+	"du1_rgb666",
+	"du1_sync",
+	"du1_oddf",
+	"du1_disp",
+	"du1_cde",
+};
+
+static const char * const intc_groups[] = {
+	"intc_irq0",
+	"intc_irq1",
+	"intc_irq2",
+	"intc_irq3",
+};
+
+static const char * const lbsc_groups[] = {
+	"lbsc_cs0",
+	"lbsc_cs1",
+	"lbsc_ex_cs0",
+	"lbsc_ex_cs1",
+	"lbsc_ex_cs2",
+	"lbsc_ex_cs3",
+	"lbsc_ex_cs4",
+	"lbsc_ex_cs5",
+};
+
+static const char * const msiof0_groups[] = {
+	"msiof0_clk",
+	"msiof0_sync",
+	"msiof0_rx",
+	"msiof0_tx",
+};
+
+static const char * const msiof1_groups[] = {
+	"msiof1_clk",
+	"msiof1_sync",
+	"msiof1_rx",
+	"msiof1_tx",
+};
+
+static const char * const qspi_groups[] = {
+	"qspi_ctrl",
+	"qspi_data2",
+	"qspi_data4",
+};
+
+static const char * const scif0_groups[] = {
+	"scif0_data",
+	"scif0_clk",
+	"scif0_ctrl",
+};
+
+static const char * const scif3_groups[] = {
+	"scif3_data",
+	"scif3_clk",
+};
+
+static const char * const sdhi0_groups[] = {
+	"sdhi0_data1",
+	"sdhi0_data4",
+	"sdhi0_ctrl",
+	"sdhi0_cd",
+	"sdhi0_wp",
+};
+
+static const char * const vin0_groups[] = {
+	"vin0_data24",
+	"vin0_data20",
+	"vin0_data18",
+	"vin0_data16",
+	"vin0_data12",
+	"vin0_data10",
+	"vin0_data8",
+	"vin0_sync",
+	"vin0_field",
+	"vin0_clkenb",
+	"vin0_clk",
+};
+
+static const char * const vin1_groups[] = {
+	"vin1_data24",
+	"vin1_data20",
+	"vin1_data18",
+	"vin1_data16",
+	"vin1_data12",
+	"vin1_data10",
+	"vin1_data8",
+	"vin1_data24_b",
+	"vin1_data20_b",
+	"vin1_data16_b",
+	"vin1_sync",
+	"vin1_field",
+	"vin1_clkenb",
+	"vin1_clk",
+};
+
+static const char * const vin2_groups[] = {
+	"vin2_data16",
+	"vin2_data12",
+	"vin2_data10",
+	"vin2_data8",
+	"vin2_sync",
+	"vin2_field",
+	"vin2_clkenb",
+	"vin2_clk",
+};
+
+static const char * const vin3_groups[] = {
+	"vin3_data16",
+	"vin3_data12",
+	"vin3_data10",
+	"vin3_data8",
+	"vin3_sync",
+	"vin3_field",
+	"vin3_clkenb",
+	"vin3_clk",
+};
+
+static const char * const vin4_groups[] = {
+	"vin4_data12",
+	"vin4_data10",
+	"vin4_data8",
+	"vin4_sync",
+	"vin4_field",
+	"vin4_clkenb",
+	"vin4_clk",
+};
+
+static const char * const vin5_groups[] = {
+	"vin5_data12",
+	"vin5_data10",
+	"vin5_data8",
+	"vin5_sync",
+	"vin5_field",
+	"vin5_clkenb",
+	"vin5_clk",
+};
+
+static const struct sh_pfc_function pinmux_functions[] = {
+	SH_PFC_FUNCTION(avb),
+	SH_PFC_FUNCTION(can0),
+	SH_PFC_FUNCTION(can1),
+	SH_PFC_FUNCTION(du0),
+	SH_PFC_FUNCTION(du1),
+	SH_PFC_FUNCTION(intc),
+	SH_PFC_FUNCTION(lbsc),
+	SH_PFC_FUNCTION(msiof0),
+	SH_PFC_FUNCTION(msiof1),
+	SH_PFC_FUNCTION(qspi),
+	SH_PFC_FUNCTION(scif0),
+	SH_PFC_FUNCTION(scif3),
+	SH_PFC_FUNCTION(sdhi0),
+	SH_PFC_FUNCTION(vin0),
+	SH_PFC_FUNCTION(vin1),
+	SH_PFC_FUNCTION(vin2),
+	SH_PFC_FUNCTION(vin3),
+	SH_PFC_FUNCTION(vin4),
+	SH_PFC_FUNCTION(vin5),
+};
+
+static const struct pinmux_cfg_reg pinmux_config_regs[] = {
+	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_0_28_FN, FN_IP1_4,
+		GP_0_27_FN, FN_IP1_3,
+		GP_0_26_FN, FN_IP1_2,
+		GP_0_25_FN, FN_IP1_1,
+		GP_0_24_FN, FN_IP1_0,
+		GP_0_23_FN, FN_IP0_23,
+		GP_0_22_FN, FN_IP0_22,
+		GP_0_21_FN, FN_IP0_21,
+		GP_0_20_FN, FN_IP0_20,
+		GP_0_19_FN, FN_IP0_19,
+		GP_0_18_FN, FN_IP0_18,
+		GP_0_17_FN, FN_IP0_17,
+		GP_0_16_FN, FN_IP0_16,
+		GP_0_15_FN, FN_IP0_15,
+		GP_0_14_FN, FN_IP0_14,
+		GP_0_13_FN, FN_IP0_13,
+		GP_0_12_FN, FN_IP0_12,
+		GP_0_11_FN, FN_IP0_11,
+		GP_0_10_FN, FN_IP0_10,
+		GP_0_9_FN, FN_IP0_9,
+		GP_0_8_FN, FN_IP0_8,
+		GP_0_7_FN, FN_IP0_7,
+		GP_0_6_FN, FN_IP0_6,
+		GP_0_5_FN, FN_IP0_5,
+		GP_0_4_FN, FN_IP0_4,
+		GP_0_3_FN, FN_IP0_3,
+		GP_0_2_FN, FN_IP0_2,
+		GP_0_1_FN, FN_IP0_1,
+		GP_0_0_FN, FN_IP0_0 }
+	},
+	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_1_22_FN, FN_DU1_CDE,
+		GP_1_21_FN, FN_DU1_DISP,
+		GP_1_20_FN, FN_DU1_EXODDF_DU1_ODDF_DISP_CDE,
+		GP_1_19_FN, FN_DU1_EXVSYNC_DU1_VSYNC,
+		GP_1_18_FN, FN_DU1_EXHSYNC_DU1_HSYNC,
+		GP_1_17_FN, FN_DU1_DB7_C5,
+		GP_1_16_FN, FN_DU1_DB6_C4,
+		GP_1_15_FN, FN_DU1_DB5_C3_DATA15,
+		GP_1_14_FN, FN_DU1_DB4_C2_DATA14,
+		GP_1_13_FN, FN_DU1_DB3_C1_DATA13,
+		GP_1_12_FN, FN_DU1_DB2_C0_DATA12,
+		GP_1_11_FN, FN_IP1_16,
+		GP_1_10_FN, FN_IP1_15,
+		GP_1_9_FN, FN_IP1_14,
+		GP_1_8_FN, FN_IP1_13,
+		GP_1_7_FN, FN_IP1_12,
+		GP_1_6_FN, FN_IP1_11,
+		GP_1_5_FN, FN_IP1_10,
+		GP_1_4_FN, FN_IP1_9,
+		GP_1_3_FN, FN_IP1_8,
+		GP_1_2_FN, FN_IP1_7,
+		GP_1_1_FN, FN_IP1_6,
+		GP_1_0_FN, FN_IP1_5, }
+	},
+	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1) {
+		GP_2_31_FN, FN_A15,
+		GP_2_30_FN, FN_A14,
+		GP_2_29_FN, FN_A13,
+		GP_2_28_FN, FN_A12,
+		GP_2_27_FN, FN_A11,
+		GP_2_26_FN, FN_A10,
+		GP_2_25_FN, FN_A9,
+		GP_2_24_FN, FN_A8,
+		GP_2_23_FN, FN_A7,
+		GP_2_22_FN, FN_A6,
+		GP_2_21_FN, FN_A5,
+		GP_2_20_FN, FN_A4,
+		GP_2_19_FN, FN_A3,
+		GP_2_18_FN, FN_A2,
+		GP_2_17_FN, FN_A1,
+		GP_2_16_FN, FN_A0,
+		GP_2_15_FN, FN_D15,
+		GP_2_14_FN, FN_D14,
+		GP_2_13_FN, FN_D13,
+		GP_2_12_FN, FN_D12,
+		GP_2_11_FN, FN_D11,
+		GP_2_10_FN, FN_D10,
+		GP_2_9_FN, FN_D9,
+		GP_2_8_FN, FN_D8,
+		GP_2_7_FN, FN_D7,
+		GP_2_6_FN, FN_D6,
+		GP_2_5_FN, FN_D5,
+		GP_2_4_FN, FN_D4,
+		GP_2_3_FN, FN_D3,
+		GP_2_2_FN, FN_D2,
+		GP_2_1_FN, FN_D1,
+		GP_2_0_FN, FN_D0 }
+	},
+	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_3_27_FN, FN_CS0_N,
+		GP_3_26_FN, FN_IP1_22,
+		GP_3_25_FN, FN_IP1_21,
+		GP_3_24_FN, FN_IP1_20,
+		GP_3_23_FN, FN_IP1_19,
+		GP_3_22_FN, FN_IRQ3,
+		GP_3_21_FN, FN_IRQ2,
+		GP_3_20_FN, FN_IRQ1,
+		GP_3_19_FN, FN_IRQ0,
+		GP_3_18_FN, FN_EX_WAIT0,
+		GP_3_17_FN, FN_WE1_N,
+		GP_3_16_FN, FN_WE0_N,
+		GP_3_15_FN, FN_RD_WR_N,
+		GP_3_14_FN, FN_RD_N,
+		GP_3_13_FN, FN_BS_N,
+		GP_3_12_FN, FN_EX_CS5_N,
+		GP_3_11_FN, FN_EX_CS4_N,
+		GP_3_10_FN, FN_EX_CS3_N,
+		GP_3_9_FN, FN_EX_CS2_N,
+		GP_3_8_FN, FN_EX_CS1_N,
+		GP_3_7_FN, FN_EX_CS0_N,
+		GP_3_6_FN, FN_CS1_N_A26,
+		GP_3_5_FN, FN_IP1_18,
+		GP_3_4_FN, FN_IP1_17,
+		GP_3_3_FN, FN_A19,
+		GP_3_2_FN, FN_A18,
+		GP_3_1_FN, FN_A17,
+		GP_3_0_FN, FN_A16 }
+	},
+	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_4_16_FN, FN_VI0_FIELD,
+		GP_4_15_FN, FN_VI0_D11_G3_Y3,
+		GP_4_14_FN, FN_VI0_D10_G2_Y2,
+		GP_4_13_FN, FN_VI0_D9_G1_Y1,
+		GP_4_12_FN, FN_VI0_D8_G0_Y0,
+		GP_4_11_FN, FN_VI0_D7_B7_C7,
+		GP_4_10_FN, FN_VI0_D6_B6_C6,
+		GP_4_9_FN, FN_VI0_D5_B5_C5,
+		GP_4_8_FN, FN_VI0_D4_B4_C4,
+		GP_4_7_FN, FN_VI0_D3_B3_C3,
+		GP_4_6_FN, FN_VI0_D2_B2_C2,
+		GP_4_5_FN, FN_VI0_D1_B1_C1,
+		GP_4_4_FN, FN_VI0_D0_B0_C0,
+		GP_4_3_FN, FN_VI0_VSYNC_N,
+		GP_4_2_FN, FN_VI0_HSYNC_N,
+		GP_4_1_FN, FN_VI0_CLKENB,
+		GP_4_0_FN, FN_VI0_CLK }
+	},
+	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_5_16_FN, FN_VI1_FIELD,
+		GP_5_15_FN, FN_VI1_D11_G3_Y3,
+		GP_5_14_FN, FN_VI1_D10_G2_Y2,
+		GP_5_13_FN, FN_VI1_D9_G1_Y1,
+		GP_5_12_FN, FN_VI1_D8_G0_Y0,
+		GP_5_11_FN, FN_VI1_D7_B7_C7,
+		GP_5_10_FN, FN_VI1_D6_B6_C6,
+		GP_5_9_FN, FN_VI1_D5_B5_C5,
+		GP_5_8_FN, FN_VI1_D4_B4_C4,
+		GP_5_7_FN, FN_VI1_D3_B3_C3,
+		GP_5_6_FN, FN_VI1_D2_B2_C2,
+		GP_5_5_FN, FN_VI1_D1_B1_C1,
+		GP_5_4_FN, FN_VI1_D0_B0_C0,
+		GP_5_3_FN, FN_VI1_VSYNC_N,
+		GP_5_2_FN, FN_VI1_HSYNC_N,
+		GP_5_1_FN, FN_VI1_CLKENB,
+		GP_5_0_FN, FN_VI1_CLK }
+	},
+	{ PINMUX_CFG_REG("GPSR6", 0xE606001C, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_6_16_FN, FN_IP2_16,
+		GP_6_15_FN, FN_IP2_15,
+		GP_6_14_FN, FN_IP2_14,
+		GP_6_13_FN, FN_IP2_13,
+		GP_6_12_FN, FN_IP2_12,
+		GP_6_11_FN, FN_IP2_11,
+		GP_6_10_FN, FN_IP2_10,
+		GP_6_9_FN, FN_IP2_9,
+		GP_6_8_FN, FN_IP2_8,
+		GP_6_7_FN, FN_IP2_7,
+		GP_6_6_FN, FN_IP2_6,
+		GP_6_5_FN, FN_IP2_5,
+		GP_6_4_FN, FN_IP2_4,
+		GP_6_3_FN, FN_IP2_3,
+		GP_6_2_FN, FN_IP2_2,
+		GP_6_1_FN, FN_IP2_1,
+		GP_6_0_FN, FN_IP2_0 }
+	},
+	{ PINMUX_CFG_REG("GPSR7", 0xE6060020, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_7_16_FN, FN_VI3_FIELD,
+		GP_7_15_FN, FN_IP3_14,
+		GP_7_14_FN, FN_VI3_D10_Y2,
+		GP_7_13_FN, FN_IP3_13,
+		GP_7_12_FN, FN_IP3_12,
+		GP_7_11_FN, FN_IP3_11,
+		GP_7_10_FN, FN_IP3_10,
+		GP_7_9_FN, FN_IP3_9,
+		GP_7_8_FN, FN_IP3_8,
+		GP_7_7_FN, FN_IP3_7,
+		GP_7_6_FN, FN_IP3_6,
+		GP_7_5_FN, FN_IP3_5,
+		GP_7_4_FN, FN_IP3_4,
+		GP_7_3_FN, FN_IP3_3,
+		GP_7_2_FN, FN_IP3_2,
+		GP_7_1_FN, FN_IP3_1,
+		GP_7_0_FN, FN_IP3_0 }
+	},
+	{ PINMUX_CFG_REG("GPSR8", 0xE6060024, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_8_16_FN, FN_IP4_24,
+		GP_8_15_FN, FN_IP4_23,
+		GP_8_14_FN, FN_IP4_22,
+		GP_8_13_FN, FN_IP4_21,
+		GP_8_12_FN, FN_IP4_20_19,
+		GP_8_11_FN, FN_IP4_18_17,
+		GP_8_10_FN, FN_IP4_16_15,
+		GP_8_9_FN, FN_IP4_14_13,
+		GP_8_8_FN, FN_IP4_12_11,
+		GP_8_7_FN, FN_IP4_10_9,
+		GP_8_6_FN, FN_IP4_8_7,
+		GP_8_5_FN, FN_IP4_6_5,
+		GP_8_4_FN, FN_IP4_4,
+		GP_8_3_FN, FN_IP4_3_2,
+		GP_8_2_FN, FN_IP4_1,
+		GP_8_1_FN, FN_IP4_0,
+		GP_8_0_FN, FN_VI4_CLK }
+	},
+	{ PINMUX_CFG_REG("GPSR9", 0xE6060028, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_9_16_FN, FN_VI5_FIELD,
+		GP_9_15_FN, FN_VI5_D11_Y3,
+		GP_9_14_FN, FN_VI5_D10_Y2,
+		GP_9_13_FN, FN_VI5_D9_Y1,
+		GP_9_12_FN, FN_IP5_11,
+		GP_9_11_FN, FN_IP5_10,
+		GP_9_10_FN, FN_IP5_9,
+		GP_9_9_FN, FN_IP5_8,
+		GP_9_8_FN, FN_IP5_7,
+		GP_9_7_FN, FN_IP5_6,
+		GP_9_6_FN, FN_IP5_5,
+		GP_9_5_FN, FN_IP5_4,
+		GP_9_4_FN, FN_IP5_3,
+		GP_9_3_FN, FN_IP5_2,
+		GP_9_2_FN, FN_IP5_1,
+		GP_9_1_FN, FN_IP5_0,
+		GP_9_0_FN, FN_VI5_CLK }
+	},
+	{ PINMUX_CFG_REG("GPSR10", 0xE606002C, 32, 1) {
+		GP_10_31_FN, FN_CAN1_RX,
+		GP_10_30_FN, FN_CAN1_TX,
+		GP_10_29_FN, FN_CAN_CLK,
+		GP_10_28_FN, FN_CAN0_RX,
+		GP_10_27_FN, FN_CAN0_TX,
+		GP_10_26_FN, FN_SCIF_CLK,
+		GP_10_25_FN, FN_IP6_18_17,
+		GP_10_24_FN, FN_IP6_16,
+		GP_10_23_FN, FN_IP6_15_14,
+		GP_10_22_FN, FN_IP6_13_12,
+		GP_10_21_FN, FN_IP6_11_10,
+		GP_10_20_FN, FN_IP6_9_8,
+		GP_10_19_FN, FN_RX1,
+		GP_10_18_FN, FN_TX1,
+		GP_10_17_FN, FN_RTS1_N,
+		GP_10_16_FN, FN_CTS1_N,
+		GP_10_15_FN, FN_SCK1,
+		GP_10_14_FN, FN_RX0,
+		GP_10_13_FN, FN_TX0,
+		GP_10_12_FN, FN_RTS0_N,
+		GP_10_11_FN, FN_CTS0_N,
+		GP_10_10_FN, FN_SCK0,
+		GP_10_9_FN, FN_IP6_7,
+		GP_10_8_FN, FN_IP6_6,
+		GP_10_7_FN, FN_HCTS1_N,
+		GP_10_6_FN, FN_IP6_5,
+		GP_10_5_FN, FN_IP6_4,
+		GP_10_4_FN, FN_IP6_3,
+		GP_10_3_FN, FN_IP6_2,
+		GP_10_2_FN, FN_HRTS0_N,
+		GP_10_1_FN, FN_IP6_1,
+		GP_10_0_FN, FN_IP6_0 }
+	},
+	{ PINMUX_CFG_REG("GPSR11", 0xE6060030, 32, 1) {
+		0, 0,
+		0, 0,
+		GP_11_29_FN, FN_AVS2,
+		GP_11_28_FN, FN_AVS1,
+		GP_11_27_FN, FN_ADICHS2,
+		GP_11_26_FN, FN_ADICHS1,
+		GP_11_25_FN, FN_ADICHS0,
+		GP_11_24_FN, FN_ADIDATA,
+		GP_11_23_FN, FN_ADICS_SAMP,
+		GP_11_22_FN, FN_ADICLK,
+		GP_11_21_FN, FN_IP7_20,
+		GP_11_20_FN, FN_IP7_19,
+		GP_11_19_FN, FN_IP7_18,
+		GP_11_18_FN, FN_IP7_17,
+		GP_11_17_FN, FN_IP7_16,
+		GP_11_16_FN, FN_IP7_15_14,
+		GP_11_15_FN, FN_IP7_13_12,
+		GP_11_14_FN, FN_IP7_11_10,
+		GP_11_13_FN, FN_IP7_9_8,
+		GP_11_12_FN, FN_SD0_WP,
+		GP_11_11_FN, FN_SD0_CD,
+		GP_11_10_FN, FN_SD0_DAT3,
+		GP_11_9_FN, FN_SD0_DAT2,
+		GP_11_8_FN, FN_SD0_DAT1,
+		GP_11_7_FN, FN_SD0_DAT0,
+		GP_11_6_FN, FN_SD0_CMD,
+		GP_11_5_FN, FN_SD0_CLK,
+		GP_11_4_FN, FN_IP7_7,
+		GP_11_3_FN, FN_IP7_6,
+		GP_11_2_FN, FN_IP7_5_4,
+		GP_11_1_FN, FN_IP7_3_2,
+		GP_11_0_FN, FN_IP7_1_0 }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR0", 0xE6060040, 32,
+			     4, 4,
+			     1, 1, 1, 1, 1, 1, 1, 1,
+			     1, 1, 1, 1, 1, 1, 1, 1,
+			     1, 1, 1, 1, 1, 1, 1, 1) {
+		/* IP0_31_28 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP0_27_24 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0,	0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP0_23 [1] */
+		FN_DU0_DB7_C5, 0,
+		/* IP0_22 [1] */
+		FN_DU0_DB6_C4, 0,
+		/* IP0_21 [1] */
+		FN_DU0_DB5_C3, 0,
+		/* IP0_20 [1] */
+		FN_DU0_DB4_C2, 0,
+		/* IP0_19 [1] */
+		FN_DU0_DB3_C1, 0,
+		/* IP0_18 [1] */
+		FN_DU0_DB2_C0, 0,
+		/* IP0_17 [1] */
+		FN_DU0_DB1, 0,
+		/* IP0_16 [1] */
+		FN_DU0_DB0, 0,
+		/* IP0_15 [1] */
+		FN_DU0_DG7_Y3_DATA15, 0,
+		/* IP0_14 [1] */
+		FN_DU0_DG6_Y2_DATA14, 0,
+		/* IP0_13 [1] */
+		FN_DU0_DG5_Y1_DATA13, 0,
+		/* IP0_12 [1] */
+		FN_DU0_DG4_Y0_DATA12, 0,
+		/* IP0_11 [1] */
+		FN_DU0_DG3_C7_DATA11, 0,
+		/* IP0_10 [1] */
+		FN_DU0_DG2_C6_DATA10, 0,
+		/* IP0_9 [1] */
+		FN_DU0_DG1_DATA9, 0,
+		/* IP0_8 [1] */
+		FN_DU0_DG0_DATA8, 0,
+		/* IP0_7 [1] */
+		FN_DU0_DR7_Y9_DATA7, 0,
+		/* IP0_6 [1] */
+		FN_DU0_DR6_Y8_DATA6, 0,
+		/* IP0_5 [1] */
+		FN_DU0_DR5_Y7_DATA5, 0,
+		/* IP0_4 [1] */
+		FN_DU0_DR4_Y6_DATA4, 0,
+		/* IP0_3 [1] */
+		FN_DU0_DR3_Y5_DATA3, 0,
+		/* IP0_2 [1] */
+		FN_DU0_DR2_Y4_DATA2, 0,
+		/* IP0_1 [1] */
+		FN_DU0_DR1_DATA1, 0,
+		/* IP0_0 [1] */
+		FN_DU0_DR0_DATA0, 0 }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR1", 0xE6060044, 32,
+			     4, 4,
+			     1, 1, 1, 1, 1, 1, 1, 1,
+			     1, 1, 1, 1, 1, 1, 1, 1,
+			     1, 1, 1, 1, 1, 1, 1, 1) {
+		/* IP1_31_28 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP1_27_24 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0,	0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP1_23 [1] */
+		0, 0,
+		/* IP1_22 [1] */
+		FN_A25, FN_SSL,
+		/* IP1_21 [1] */
+		FN_A24, FN_SPCLK,
+		/* IP1_20 [1] */
+		FN_A23, FN_IO3,
+		/* IP1_19 [1] */
+		FN_A22, FN_IO2,
+		/* IP1_18 [1] */
+		FN_A21, FN_MISO_IO1,
+		/* IP1_17 [1] */
+		FN_A20, FN_MOSI_IO0,
+		/* IP1_16 [1] */
+		FN_DU1_DG7_Y3_DATA11, 0,
+		/* IP1_15 [1] */
+		FN_DU1_DG6_Y2_DATA10, 0,
+		/* IP1_14 [1] */
+		FN_DU1_DG5_Y1_DATA9, 0,
+		/* IP1_13 [1] */
+		FN_DU1_DG4_Y0_DATA8, 0,
+		/* IP1_12 [1] */
+		FN_DU1_DG3_C7_DATA7, 0,
+		/* IP1_11 [1] */
+		FN_DU1_DG2_C6_DATA6, 0,
+		/* IP1_10 [1] */
+		FN_DU1_DR7_DATA5, 0,
+		/* IP1_9 [1] */
+		FN_DU1_DR6_DATA4, 0,
+		/* IP1_8 [1] */
+		FN_DU1_DR5_Y7_DATA3, 0,
+		/* IP1_7 [1] */
+		FN_DU1_DR4_Y6_DATA2, 0,
+		/* IP1_6 [1] */
+		FN_DU1_DR3_Y5_DATA1, 0,
+		/* IP1_5 [1] */
+		FN_DU1_DR2_Y4_DATA0, 0,
+		/* IP1_4 [1] */
+		FN_DU0_CDE, 0,
+		/* IP1_3 [1] */
+		FN_DU0_DISP, 0,
+		/* IP1_2 [1] */
+		FN_DU0_EXODDF_DU0_ODDF_DISP_CDE, 0,
+		/* IP1_1 [1] */
+		FN_DU0_EXVSYNC_DU0_VSYNC, 0,
+		/* IP1_0 [1] */
+		FN_DU0_EXHSYNC_DU0_HSYNC, 0 }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR2", 0xE6060048, 32,
+			     4, 4,
+			     4, 3, 1,
+			     1, 1, 1, 1, 1, 1, 1, 1,
+			     1, 1, 1, 1, 1, 1, 1, 1) {
+		/* IP2_31_28 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP2_27_24 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0,	0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP2_23_20 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP2_19_17 [3] */
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP2_16 [1] */
+		FN_VI2_FIELD, FN_AVB_TXD2,
+		/* IP2_15 [1] */
+		FN_VI2_D11_Y3, FN_AVB_TXD1,
+		/* IP2_14 [1] */
+		FN_VI2_D10_Y2, FN_AVB_TXD0,
+		/* IP2_13 [1] */
+		FN_VI2_D9_Y1, FN_AVB_TX_EN,
+		/* IP2_12 [1] */
+		FN_VI2_D8_Y0, FN_AVB_TXD3,
+		/* IP2_11 [1] */
+		FN_VI2_D7_C7, FN_AVB_COL,
+		/* IP2_10 [1] */
+		FN_VI2_D6_C6, FN_AVB_RX_ER,
+		/* IP2_9 [1] */
+		FN_VI2_D5_C5, FN_AVB_RXD7,
+		/* IP2_8 [1] */
+		FN_VI2_D4_C4, FN_AVB_RXD6,
+		/* IP2_7 [1] */
+		FN_VI2_D3_C3, FN_AVB_RXD5,
+		/* IP2_6 [1] */
+		FN_VI2_D2_C2, FN_AVB_RXD4,
+		/* IP2_5 [1] */
+		FN_VI2_D1_C1, FN_AVB_RXD3,
+		/* IP2_4 [1] */
+		FN_VI2_D0_C0, FN_AVB_RXD2,
+		/* IP2_3 [1] */
+		FN_VI2_VSYNC_N, FN_AVB_RXD1,
+		/* IP2_2 [1] */
+		FN_VI2_HSYNC_N, FN_AVB_RXD0,
+		/* IP2_1 [1] */
+		FN_VI2_CLKENB, FN_AVB_RX_DV,
+		/* IP2_0 [1] */
+		FN_VI2_CLK, FN_AVB_RX_CLK }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR3", 0xE606004C, 32,
+			     4, 4,
+			     4, 4,
+			     1, 1, 1, 1, 1, 1, 1, 1,
+			     1, 1, 1, 1, 1, 1, 1, 1) {
+		/* IP3_31_28 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP3_27_24 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0,	0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP3_23_20 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP3_19_16 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP3_15 [1] */
+		0, 0,
+		/* IP3_14 [1] */
+		FN_VI3_D11_Y3, FN_AVB_AVTP_MATCH,
+		/* IP3_13 [1] */
+		FN_VI3_D9_Y1, FN_AVB_GTXREFCLK,
+		/* IP3_12 [1] */
+		FN_VI3_D8_Y0, FN_AVB_CRS,
+		/* IP3_11 [1] */
+		FN_VI3_D7_C7, FN_AVB_PHY_INT,
+		/* IP3_10 [1] */
+		FN_VI3_D6_C6, FN_AVB_MAGIC,
+		/* IP3_9 [1] */
+		FN_VI3_D5_C5, FN_AVB_LINK,
+		/* IP3_8 [1] */
+		FN_VI3_D4_C4, FN_AVB_MDIO,
+		/* IP3_7 [1] */
+		FN_VI3_D3_C3, FN_AVB_MDC,
+		/* IP3_6 [1] */
+		FN_VI3_D2_C2, FN_AVB_GTX_CLK,
+		/* IP3_5 [1] */
+		FN_VI3_D1_C1, FN_AVB_TX_ER,
+		/* IP3_4 [1] */
+		FN_VI3_D0_C0, FN_AVB_TXD7,
+		/* IP3_3 [1] */
+		FN_VI3_VSYNC_N, FN_AVB_TXD6,
+		/* IP3_2 [1] */
+		FN_VI3_HSYNC_N, FN_AVB_TXD5,
+		/* IP3_1 [1] */
+		FN_VI3_CLKENB, FN_AVB_TXD4,
+		/* IP3_0 [1] */
+		FN_VI3_CLK, FN_AVB_TX_CLK }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR4", 0xE6060050, 32,
+			     4, 3, 1,
+			     1, 1, 1, 2, 2, 2,
+			     2, 2, 2, 2, 2, 1, 2, 1, 1) {
+		/* IP4_31_28 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP4_27_25 [3] */
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP4_24 [1] */
+		FN_VI4_FIELD, FN_VI3_D15_Y7,
+		/* IP4_23 [1] */
+		FN_VI4_D11_Y3, FN_VI3_D14_Y6,
+		/* IP4_22 [1] */
+		FN_VI4_D10_Y2, FN_VI3_D13_Y5,
+		/* IP4_21 [1] */
+		FN_VI4_D9_Y1, FN_VI3_D12_Y4,
+		/* IP4_20_19 [2] */
+		FN_VI4_D8_Y0, FN_VI0_D23_R7, FN_VI2_D15_Y7, 0,
+		/* IP4_18_17 [2] */
+		FN_VI4_D7_C7, FN_VI0_D22_R6, FN_VI2_D14_Y6, 0,
+		/* IP4_16_15 [2] */
+		FN_VI4_D6_C6, FN_VI0_D21_R5, FN_VI2_D13_Y5, 0,
+		/* IP4_14_13 [2] */
+		FN_VI4_D5_C5, FN_VI0_D20_R4, FN_VI2_D12_Y4, 0,
+		/* IP4_12_11 [2] */
+		FN_VI4_D4_C4, FN_VI0_D19_R3, FN_VI1_D15_G7_Y7, 0,
+		/* IP4_10_9 [2] */
+		FN_VI4_D3_C3, FN_VI0_D18_R2, FN_VI1_D14_G6_Y6, 0,
+		/* IP4_8_7 [2] */
+		FN_VI4_D2_C2, 0, FN_VI0_D17_R1, FN_VI1_D13_G5_Y5,
+		/* IP4_6_5 [2] */
+		FN_VI4_D1_C1, FN_VI0_D16_R0, FN_VI1_D12_G4_Y4, 0,
+		/* IP4_4 [1] */
+		FN_VI4_D0_C0, FN_VI0_D15_G7_Y7,
+		/* IP4_3_2 [2] */
+		FN_VI4_VSYNC_N, FN_VI0_D14_G6_Y6, 0, 0,
+		/* IP4_1 [1] */
+		FN_VI4_HSYNC_N, FN_VI0_D13_G5_Y5,
+		/* IP4_0 [1] */
+		FN_VI4_CLKENB, FN_VI0_D12_G4_Y4 }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR5", 0xE6060054, 32,
+			     4, 4,
+			     4, 4,
+			     4, 1, 1, 1, 1,
+			     1, 1, 1, 1, 1, 1, 1, 1) {
+		/* IP5_31_28 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP5_27_24 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP5_23_20 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP5_19_16 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP5_15_12 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP5_11 [1] */
+		FN_VI5_D8_Y0, FN_VI1_D23_R7,
+		/* IP5_10 [1] */
+		FN_VI5_D7_C7, FN_VI1_D22_R6,
+		/* IP5_9 [1] */
+		FN_VI5_D6_C6, FN_VI1_D21_R5,
+		/* IP5_8 [1] */
+		FN_VI5_D5_C5, FN_VI1_D20_R4,
+		/* IP5_7 [1] */
+		FN_VI5_D4_C4, FN_VI1_D19_R3,
+		/* IP5_6 [1] */
+		FN_VI5_D3_C3, FN_VI1_D18_R2,
+		/* IP5_5 [1] */
+		FN_VI5_D2_C2, FN_VI1_D17_R1,
+		/* IP5_4 [1] */
+		FN_VI5_D1_C1, FN_VI1_D16_R0,
+		/* IP5_3 [1] */
+		FN_VI5_D0_C0, FN_VI1_D15_G7_Y7_B,
+		/* IP5_2 [1] */
+		FN_VI5_VSYNC_N, FN_VI1_D14_G6_Y6_B,
+		/* IP5_1 [1] */
+		FN_VI5_HSYNC_N, FN_VI1_D13_G5_Y5_B,
+		/* IP5_0 [1] */
+		FN_VI5_CLKENB, FN_VI1_D12_G4_Y4_B }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR6", 0xE6060058, 32,
+			     4, 4,
+			     4, 1, 2, 1,
+			     2, 2, 2, 2,
+			     1, 1, 1, 1, 1, 1, 1, 1) {
+		/* IP6_31_28 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP6_27_24 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP6_23_20 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP6_19 [1] */
+		0, 0,
+		/* IP6_18_17 [2] */
+		FN_DREQ1_N, FN_RX3, 0, 0,
+		/* IP6_16 [1] */
+		FN_TX3, 0,
+		/* IP6_15_14 [2] */
+		FN_DACK1, FN_SCK3, 0, 0,
+		/* IP6_13_12 [2] */
+		FN_DREQ0_N, FN_RX2, 0, 0,
+		/* IP6_11_10 [2] */
+		FN_DACK0, FN_TX2, 0, 0,
+		/* IP6_9_8 [2] */
+		FN_DRACK0, FN_SCK2, 0, 0,
+		/* IP6_7 [1] */
+		FN_MSIOF1_RXD, FN_HRX1,
+		/* IP6_6 [1] */
+		FN_MSIOF1_TXD, FN_HTX1,
+		/* IP6_5 [1] */
+		FN_MSIOF1_SYNC, FN_HRTS1_N,
+		/* IP6_4 [1] */
+		FN_MSIOF1_SCK, FN_HSCK1,
+		/* IP6_3 [1] */
+		FN_MSIOF0_RXD, FN_HRX0,
+		/* IP6_2 [1] */
+		FN_MSIOF0_TXD, FN_HTX0,
+		/* IP6_1 [1] */
+		FN_MSIOF0_SYNC, FN_HCTS0_N,
+		/* IP6_0 [1] */
+		FN_MSIOF0_SCK, FN_HSCK0 }
+	},
+	{ PINMUX_CFG_REG_VAR("IPSR7", 0xE606005C, 32,
+			     4, 4,
+			     3, 1, 1, 1, 1, 1,
+			     2, 2, 2, 2,
+			     1, 1, 2, 2, 2) {
+		/* IP7_31_28 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP7_27_24 [4] */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP7_23_21 [3] */
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP7_20 [1] */
+		FN_AUDIO_CLKB, 0,
+		/* IP7_19 [1] */
+		FN_AUDIO_CLKA, 0,
+		/* IP7_18 [1] */
+		FN_AUDIO_CLKOUT, 0,
+		/* IP7_17 [1] */
+		FN_SSI_SDATA4, 0,
+		/* IP7_16 [1] */
+		FN_SSI_WS4, 0,
+		/* IP7_15_14 [2] */
+		FN_SSI_SCK4, FN_TPU0TO3, 0, 0,
+		/* IP7_13_12 [2] */
+		FN_SSI_SDATA3, FN_TPU0TO2, 0, 0,
+		/* IP7_11_10 [2] */
+		FN_SSI_WS34, FN_TPU0TO1, 0, 0,
+		/* IP7_9_8 [2] */
+		FN_SSI_SCK34, FN_TPU0TO0, 0, 0,
+		/* IP7_7 [1] */
+		FN_PWM4, 0,
+		/* IP7_6 [1] */
+		FN_PWM3, 0,
+		/* IP7_5_4 [2] */
+		FN_PWM2, FN_TCLK3, FN_FSO_TOE, 0,
+		/* IP7_3_2 [2] */
+		FN_PWM1, FN_TCLK2, FN_FSO_CFE_1, 0,
+		/* IP7_1_0 [2] */
+		FN_PWM0, FN_TCLK1, FN_FSO_CFE_0, 0 }
+	},
+	{ },
+};
+
+const struct sh_pfc_soc_info r8a7792_pinmux_info = {
+	.name = "r8a77920_pfc",
+	.unlock_reg = 0xe6060000, /* PMMR */
+
+	.function = { PINMUX_FUNCTION_BEGIN, PINMUX_FUNCTION_END },
+
+	.pins = pinmux_pins,
+	.nr_pins = ARRAY_SIZE(pinmux_pins),
+	.groups = pinmux_groups,
+	.nr_groups = ARRAY_SIZE(pinmux_groups),
+	.functions = pinmux_functions,
+	.nr_functions = ARRAY_SIZE(pinmux_functions),
+
+	.cfg_regs = pinmux_config_regs,
+
+	.pinmux_data = pinmux_data,
+	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
+};
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7794.c b/drivers/pinctrl/sh-pfc/pfc-r8a7794.c
index 8bc2cf0c594e..ed734f560c84 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7794.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7794.c
@@ -22,7 +22,9 @@
 	PORT_GP_32(3, fn, sfx),						\
 	PORT_GP_32(4, fn, sfx),						\
 	PORT_GP_28(5, fn, sfx),						\
-	PORT_GP_26(6, fn, sfx)
+	PORT_GP_CFG_24(6, fn, sfx, SH_PFC_PIN_CFG_IO_VOLTAGE),		\
+	PORT_GP_1(6, 24, fn, sfx),					\
+	PORT_GP_1(6, 25, fn, sfx)
 
 enum {
 	PINMUX_RESERVED = 0,
@@ -5160,8 +5162,32 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	{ },
 };
 
+static int r8a7794_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin, u32 *pocctrl)
+{
+	*pocctrl = 0xe606006c;
+
+	switch (pin & 0x1f) {
+	case 6: return 23;
+	case 7: return 16;
+	case 14: return 15;
+	case 15: return 8;
+	case 0 ... 5:
+	case 8 ... 13:
+		return 22 - (pin & 0x1f);
+	case 16 ... 23:
+		return 47 - (pin & 0x1f);
+	}
+
+	return -EINVAL;
+}
+
+static const struct sh_pfc_soc_operations r8a7794_pinmux_ops = {
+	.pin_to_pocctrl = r8a7794_pin_to_pocctrl,
+};
+
 const struct sh_pfc_soc_info r8a7794_pinmux_info = {
 	.name = "r8a77940_pfc",
+	.ops = &r8a7794_pinmux_ops,
 	.unlock_reg = 0xe6060000, /* PMMR */
 
 	.function = { PINMUX_FUNCTION_BEGIN, PINMUX_FUNCTION_END },
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
index b74cdd310d83..2e8cc2adbed7 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
@@ -1,5 +1,5 @@
 /*
- * R-Car Gen3 processor support - PFC hardware block.
+ * R8A7795 processor support - PFC hardware block.
  *
  * Copyright (C) 2015  Renesas Electronics Corporation
  *
@@ -13,19 +13,23 @@
 #include "core.h"
 #include "sh_pfc.h"
 
+#define CFG_FLAGS (SH_PFC_PIN_CFG_DRIVE_STRENGTH | \
+		   SH_PFC_PIN_CFG_PULL_UP | \
+		   SH_PFC_PIN_CFG_PULL_DOWN)
+
 #define CPU_ALL_PORT(fn, sfx)						\
-	PORT_GP_CFG_16(0, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_28(1, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_15(2, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_12(3, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH | SH_PFC_PIN_CFG_IO_VOLTAGE),	\
-	PORT_GP_CFG_1(3, 12, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_1(3, 13, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_1(3, 14, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_1(3, 15, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_18(4, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH | SH_PFC_PIN_CFG_IO_VOLTAGE),	\
-	PORT_GP_CFG_26(5, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_32(6, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH),	\
-	PORT_GP_CFG_4(7, fn, sfx, SH_PFC_PIN_CFG_DRIVE_STRENGTH)
+	PORT_GP_CFG_16(0, fn, sfx, CFG_FLAGS),	\
+	PORT_GP_CFG_28(1, fn, sfx, CFG_FLAGS),	\
+	PORT_GP_CFG_15(2, fn, sfx, CFG_FLAGS),	\
+	PORT_GP_CFG_12(3, fn, sfx, CFG_FLAGS | SH_PFC_PIN_CFG_IO_VOLTAGE),	\
+	PORT_GP_CFG_1(3, 12, fn, sfx, CFG_FLAGS),	\
+	PORT_GP_CFG_1(3, 13, fn, sfx, CFG_FLAGS),	\
+	PORT_GP_CFG_1(3, 14, fn, sfx, CFG_FLAGS),	\
+	PORT_GP_CFG_1(3, 15, fn, sfx, CFG_FLAGS),	\
+	PORT_GP_CFG_18(4, fn, sfx, CFG_FLAGS | SH_PFC_PIN_CFG_IO_VOLTAGE),	\
+	PORT_GP_CFG_26(5, fn, sfx, CFG_FLAGS),	\
+	PORT_GP_CFG_32(6, fn, sfx, CFG_FLAGS),	\
+	PORT_GP_CFG_4(7, fn, sfx, CFG_FLAGS)
 /*
  * F_() : just information
  * FM() : macro for FN_xxx / xxx_MARK
@@ -1871,6 +1875,86 @@ static const unsigned int drif3_data1_b_mux[] = {
 	RIF3_D1_B_MARK,
 };
 
+/* - DU --------------------------------------------------------------------- */
+static const unsigned int du_rgb666_pins[] = {
+	/* R[7:2], G[7:2], B[7:2] */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14), RCAR_GP_PIN(0, 13),
+	RCAR_GP_PIN(0, 12), RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
+	RCAR_GP_PIN(1, 15), RCAR_GP_PIN(1, 14), RCAR_GP_PIN(1, 13),
+	RCAR_GP_PIN(1, 12), RCAR_GP_PIN(1, 19), RCAR_GP_PIN(1, 18),
+	RCAR_GP_PIN(1, 7),  RCAR_GP_PIN(1, 6),  RCAR_GP_PIN(1, 5),
+	RCAR_GP_PIN(1, 4),  RCAR_GP_PIN(1, 3),  RCAR_GP_PIN(1, 2),
+};
+static const unsigned int du_rgb666_mux[] = {
+	DU_DR7_MARK, DU_DR6_MARK, DU_DR5_MARK, DU_DR4_MARK,
+	DU_DR3_MARK, DU_DR2_MARK,
+	DU_DG7_MARK, DU_DG6_MARK, DU_DG5_MARK, DU_DG4_MARK,
+	DU_DG3_MARK, DU_DG2_MARK,
+	DU_DB7_MARK, DU_DB6_MARK, DU_DB5_MARK, DU_DB4_MARK,
+	DU_DB3_MARK, DU_DB2_MARK,
+};
+static const unsigned int du_rgb888_pins[] = {
+	/* R[7:0], G[7:0], B[7:0] */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14), RCAR_GP_PIN(0, 13),
+	RCAR_GP_PIN(0, 12), RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
+	RCAR_GP_PIN(0, 9),  RCAR_GP_PIN(0, 8),
+	RCAR_GP_PIN(1, 15), RCAR_GP_PIN(1, 14), RCAR_GP_PIN(1, 13),
+	RCAR_GP_PIN(1, 12), RCAR_GP_PIN(1, 19), RCAR_GP_PIN(1, 18),
+	RCAR_GP_PIN(1, 17), RCAR_GP_PIN(1, 16),
+	RCAR_GP_PIN(1, 7),  RCAR_GP_PIN(1, 6),  RCAR_GP_PIN(1, 5),
+	RCAR_GP_PIN(1, 4),  RCAR_GP_PIN(1, 3),  RCAR_GP_PIN(1, 2),
+	RCAR_GP_PIN(1, 1),  RCAR_GP_PIN(1, 0),
+};
+static const unsigned int du_rgb888_mux[] = {
+	DU_DR7_MARK, DU_DR6_MARK, DU_DR5_MARK, DU_DR4_MARK,
+	DU_DR3_MARK, DU_DR2_MARK, DU_DR1_MARK, DU_DR0_MARK,
+	DU_DG7_MARK, DU_DG6_MARK, DU_DG5_MARK, DU_DG4_MARK,
+	DU_DG3_MARK, DU_DG2_MARK, DU_DG1_MARK, DU_DG0_MARK,
+	DU_DB7_MARK, DU_DB6_MARK, DU_DB5_MARK, DU_DB4_MARK,
+	DU_DB3_MARK, DU_DB2_MARK, DU_DB1_MARK, DU_DB0_MARK,
+};
+static const unsigned int du_clk_out_0_pins[] = {
+	/* CLKOUT */
+	RCAR_GP_PIN(1, 27),
+};
+static const unsigned int du_clk_out_0_mux[] = {
+	DU_DOTCLKOUT0_MARK
+};
+static const unsigned int du_clk_out_1_pins[] = {
+	/* CLKOUT */
+	RCAR_GP_PIN(2, 3),
+};
+static const unsigned int du_clk_out_1_mux[] = {
+	DU_DOTCLKOUT1_MARK
+};
+static const unsigned int du_sync_pins[] = {
+	/* EXVSYNC/VSYNC, EXHSYNC/HSYNC */
+	RCAR_GP_PIN(2, 5), RCAR_GP_PIN(2, 4),
+};
+static const unsigned int du_sync_mux[] = {
+	DU_EXVSYNC_DU_VSYNC_MARK, DU_EXHSYNC_DU_HSYNC_MARK
+};
+static const unsigned int du_oddf_pins[] = {
+	/* EXDISP/EXODDF/EXCDE */
+	RCAR_GP_PIN(2, 2),
+};
+static const unsigned int du_oddf_mux[] = {
+	DU_EXODDF_DU_ODDF_DISP_CDE_MARK,
+};
+static const unsigned int du_cde_pins[] = {
+	/* CDE */
+	RCAR_GP_PIN(2, 0),
+};
+static const unsigned int du_cde_mux[] = {
+	DU_CDE_MARK,
+};
+static const unsigned int du_disp_pins[] = {
+	/* DISP */
+	RCAR_GP_PIN(2, 1),
+};
+static const unsigned int du_disp_mux[] = {
+	DU_DISP_MARK,
+};
 /* - HSCIF0 ----------------------------------------------------------------- */
 static const unsigned int hscif0_data_pins[] = {
 	/* RX, TX */
@@ -3593,6 +3677,14 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(drif3_ctrl_b),
 	SH_PFC_PIN_GROUP(drif3_data0_b),
 	SH_PFC_PIN_GROUP(drif3_data1_b),
+	SH_PFC_PIN_GROUP(du_rgb666),
+	SH_PFC_PIN_GROUP(du_rgb888),
+	SH_PFC_PIN_GROUP(du_clk_out_0),
+	SH_PFC_PIN_GROUP(du_clk_out_1),
+	SH_PFC_PIN_GROUP(du_sync),
+	SH_PFC_PIN_GROUP(du_oddf),
+	SH_PFC_PIN_GROUP(du_cde),
+	SH_PFC_PIN_GROUP(du_disp),
 	SH_PFC_PIN_GROUP(hscif0_data),
 	SH_PFC_PIN_GROUP(hscif0_clk),
 	SH_PFC_PIN_GROUP(hscif0_ctrl),
@@ -3918,6 +4010,17 @@ static const char * const drif3_groups[] = {
 	"drif3_data1_b",
 };
 
+static const char * const du_groups[] = {
+	"du_rgb666",
+	"du_rgb888",
+	"du_clk_out_0",
+	"du_clk_out_1",
+	"du_sync",
+	"du_oddf",
+	"du_cde",
+	"du_disp",
+};
+
 static const char * const hscif0_groups[] = {
 	"hscif0_data",
 	"hscif0_clk",
@@ -4265,6 +4368,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(drif1),
 	SH_PFC_FUNCTION(drif2),
 	SH_PFC_FUNCTION(drif3),
+	SH_PFC_FUNCTION(du),
 	SH_PFC_FUNCTION(hscif0),
 	SH_PFC_FUNCTION(hscif1),
 	SH_PFC_FUNCTION(hscif2),
@@ -5073,8 +5177,234 @@ static int r8a7795_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin, u32 *poc
 	return bit;
 }
 
+#define PUEN	0xe6060400
+#define PUD	0xe6060440
+
+#define PU0	0x00
+#define PU1	0x04
+#define PU2	0x08
+#define PU3	0x0c
+#define PU4	0x10
+#define PU5	0x14
+#define PU6	0x18
+
+static const struct {
+	u16 reg : 11;
+	u16 bit : 5;
+} pullups[] = {
+	[RCAR_GP_PIN(2, 11)] = { PU0, 31 },	/* AVB_PHY_INT */
+	[RCAR_GP_PIN(2, 10)] = { PU0, 30 },	/* AVB_MAGIC */
+	[RCAR_GP_PIN(2,  9)] = { PU0, 29 },	/* AVB_MDC */
+
+	[RCAR_GP_PIN(1, 19)] = { PU1, 31 },	/* A19 */
+	[RCAR_GP_PIN(1, 18)] = { PU1, 30 },	/* A18 */
+	[RCAR_GP_PIN(1, 17)] = { PU1, 29 },	/* A17 */
+	[RCAR_GP_PIN(1, 16)] = { PU1, 28 },	/* A16 */
+	[RCAR_GP_PIN(1, 15)] = { PU1, 27 },	/* A15 */
+	[RCAR_GP_PIN(1, 14)] = { PU1, 26 },	/* A14 */
+	[RCAR_GP_PIN(1, 13)] = { PU1, 25 },	/* A13 */
+	[RCAR_GP_PIN(1, 12)] = { PU1, 24 },	/* A12 */
+	[RCAR_GP_PIN(1, 11)] = { PU1, 23 },	/* A11 */
+	[RCAR_GP_PIN(1, 10)] = { PU1, 22 },	/* A10 */
+	[RCAR_GP_PIN(1,  9)] = { PU1, 21 },	/* A9 */
+	[RCAR_GP_PIN(1,  8)] = { PU1, 20 },	/* A8 */
+	[RCAR_GP_PIN(1,  7)] = { PU1, 19 },	/* A7 */
+	[RCAR_GP_PIN(1,  6)] = { PU1, 18 },	/* A6 */
+	[RCAR_GP_PIN(1,  5)] = { PU1, 17 },	/* A5 */
+	[RCAR_GP_PIN(1,  4)] = { PU1, 16 },	/* A4 */
+	[RCAR_GP_PIN(1,  3)] = { PU1, 15 },	/* A3 */
+	[RCAR_GP_PIN(1,  2)] = { PU1, 14 },	/* A2 */
+	[RCAR_GP_PIN(1,  1)] = { PU1, 13 },	/* A1 */
+	[RCAR_GP_PIN(1,  0)] = { PU1, 12 },	/* A0 */
+	[RCAR_GP_PIN(2,  8)] = { PU1, 11 },	/* PWM2_A */
+	[RCAR_GP_PIN(2,  7)] = { PU1, 10 },	/* PWM1_A */
+	[RCAR_GP_PIN(2,  6)] = { PU1,  9 },	/* PWM0 */
+	[RCAR_GP_PIN(2,  5)] = { PU1,  8 },	/* IRQ5 */
+	[RCAR_GP_PIN(2,  4)] = { PU1,  7 },	/* IRQ4 */
+	[RCAR_GP_PIN(2,  3)] = { PU1,  6 },	/* IRQ3 */
+	[RCAR_GP_PIN(2,  2)] = { PU1,  5 },	/* IRQ2 */
+	[RCAR_GP_PIN(2,  1)] = { PU1,  4 },	/* IRQ1 */
+	[RCAR_GP_PIN(2,  0)] = { PU1,  3 },	/* IRQ0 */
+	[RCAR_GP_PIN(2, 14)] = { PU1,  2 },	/* AVB_AVTP_CAPTURE_A */
+	[RCAR_GP_PIN(2, 13)] = { PU1,  1 },	/* AVB_AVTP_MATCH_A */
+	[RCAR_GP_PIN(2, 12)] = { PU1,  0 },	/* AVB_LINK */
+
+	[RCAR_GP_PIN(7,  3)] = { PU2, 29 },	/* HDMI1_CEC */
+	[RCAR_GP_PIN(7,  2)] = { PU2, 28 },	/* HDMI0_CEC */
+	[RCAR_GP_PIN(7,  1)] = { PU2, 27 },	/* AVS2 */
+	[RCAR_GP_PIN(7,  0)] = { PU2, 26 },	/* AVS1 */
+	[RCAR_GP_PIN(0, 15)] = { PU2, 25 },	/* D15 */
+	[RCAR_GP_PIN(0, 14)] = { PU2, 24 },	/* D14 */
+	[RCAR_GP_PIN(0, 13)] = { PU2, 23 },	/* D13 */
+	[RCAR_GP_PIN(0, 12)] = { PU2, 22 },	/* D12 */
+	[RCAR_GP_PIN(0, 11)] = { PU2, 21 },	/* D11 */
+	[RCAR_GP_PIN(0, 10)] = { PU2, 20 },	/* D10 */
+	[RCAR_GP_PIN(0,  9)] = { PU2, 19 },	/* D9 */
+	[RCAR_GP_PIN(0,  8)] = { PU2, 18 },	/* D8 */
+	[RCAR_GP_PIN(0,  7)] = { PU2, 17 },	/* D7 */
+	[RCAR_GP_PIN(0,  6)] = { PU2, 16 },	/* D6 */
+	[RCAR_GP_PIN(0,  5)] = { PU2, 15 },	/* D5 */
+	[RCAR_GP_PIN(0,  4)] = { PU2, 14 },	/* D4 */
+	[RCAR_GP_PIN(0,  3)] = { PU2, 13 },	/* D3 */
+	[RCAR_GP_PIN(0,  2)] = { PU2, 12 },	/* D2 */
+	[RCAR_GP_PIN(0,  1)] = { PU2, 11 },	/* D1 */
+	[RCAR_GP_PIN(0,  0)] = { PU2, 10 },	/* D0 */
+	[RCAR_GP_PIN(1, 27)] = { PU2,  8 },	/* EX_WAIT0_A */
+	[RCAR_GP_PIN(1, 26)] = { PU2,  7 },	/* WE1_N */
+	[RCAR_GP_PIN(1, 25)] = { PU2,  6 },	/* WE0_N */
+	[RCAR_GP_PIN(1, 24)] = { PU2,  5 },	/* RD_WR_N */
+	[RCAR_GP_PIN(1, 23)] = { PU2,  4 },	/* RD_N */
+	[RCAR_GP_PIN(1, 22)] = { PU2,  3 },	/* BS_N */
+	[RCAR_GP_PIN(1, 21)] = { PU2,  2 },	/* CS1_N_A26 */
+	[RCAR_GP_PIN(1, 20)] = { PU2,  1 },	/* CS0_N */
+
+	[RCAR_GP_PIN(4,  9)] = { PU3, 31 },	/* SD3_DAT0 */
+	[RCAR_GP_PIN(4,  8)] = { PU3, 30 },	/* SD3_CMD */
+	[RCAR_GP_PIN(4,  7)] = { PU3, 29 },	/* SD3_CLK */
+	[RCAR_GP_PIN(4,  6)] = { PU3, 28 },	/* SD2_DS */
+	[RCAR_GP_PIN(4,  5)] = { PU3, 27 },	/* SD2_DAT3 */
+	[RCAR_GP_PIN(4,  4)] = { PU3, 26 },	/* SD2_DAT2 */
+	[RCAR_GP_PIN(4,  3)] = { PU3, 25 },	/* SD2_DAT1 */
+	[RCAR_GP_PIN(4,  2)] = { PU3, 24 },	/* SD2_DAT0 */
+	[RCAR_GP_PIN(4,  1)] = { PU3, 23 },	/* SD2_CMD */
+	[RCAR_GP_PIN(4,  0)] = { PU3, 22 },	/* SD2_CLK */
+	[RCAR_GP_PIN(3, 11)] = { PU3, 21 },	/* SD1_DAT3 */
+	[RCAR_GP_PIN(3, 10)] = { PU3, 20 },	/* SD1_DAT2 */
+	[RCAR_GP_PIN(3,  9)] = { PU3, 19 },	/* SD1_DAT1 */
+	[RCAR_GP_PIN(3,  8)] = { PU3, 18 },	/* SD1_DAT0 */
+	[RCAR_GP_PIN(3,  7)] = { PU3, 17 },	/* SD1_CMD */
+	[RCAR_GP_PIN(3,  6)] = { PU3, 16 },	/* SD1_CLK */
+	[RCAR_GP_PIN(3,  5)] = { PU3, 15 },	/* SD0_DAT3 */
+	[RCAR_GP_PIN(3,  4)] = { PU3, 14 },	/* SD0_DAT2 */
+	[RCAR_GP_PIN(3,  3)] = { PU3, 13 },	/* SD0_DAT1 */
+	[RCAR_GP_PIN(3,  2)] = { PU3, 12 },	/* SD0_DAT0 */
+	[RCAR_GP_PIN(3,  1)] = { PU3, 11 },	/* SD0_CMD */
+	[RCAR_GP_PIN(3,  0)] = { PU3, 10 },	/* SD0_CLK */
+
+	[RCAR_GP_PIN(5, 19)] = { PU4, 31 },	/* MSIOF0_SS1 */
+	[RCAR_GP_PIN(5, 18)] = { PU4, 30 },	/* MSIOF0_SYNC */
+	[RCAR_GP_PIN(5, 17)] = { PU4, 29 },	/* MSIOF0_SCK */
+	[RCAR_GP_PIN(5, 16)] = { PU4, 28 },	/* HRTS0_N */
+	[RCAR_GP_PIN(5, 15)] = { PU4, 27 },	/* HCTS0_N */
+	[RCAR_GP_PIN(5, 14)] = { PU4, 26 },	/* HTX0 */
+	[RCAR_GP_PIN(5, 13)] = { PU4, 25 },	/* HRX0 */
+	[RCAR_GP_PIN(5, 12)] = { PU4, 24 },	/* HSCK0 */
+	[RCAR_GP_PIN(5, 11)] = { PU4, 23 },	/* RX2_A */
+	[RCAR_GP_PIN(5, 10)] = { PU4, 22 },	/* TX2_A */
+	[RCAR_GP_PIN(5,  9)] = { PU4, 21 },	/* SCK2 */
+	[RCAR_GP_PIN(5,  8)] = { PU4, 20 },	/* RTS1_N_TANS */
+	[RCAR_GP_PIN(5,  7)] = { PU4, 19 },	/* CTS1_N */
+	[RCAR_GP_PIN(5,  6)] = { PU4, 18 },	/* TX1_A */
+	[RCAR_GP_PIN(5,  5)] = { PU4, 17 },	/* RX1_A */
+	[RCAR_GP_PIN(5,  4)] = { PU4, 16 },	/* RTS0_N_TANS */
+	[RCAR_GP_PIN(5,  3)] = { PU4, 15 },	/* CTS0_N */
+	[RCAR_GP_PIN(5,  2)] = { PU4, 14 },	/* TX0 */
+	[RCAR_GP_PIN(5,  1)] = { PU4, 13 },	/* RX0 */
+	[RCAR_GP_PIN(5,  0)] = { PU4, 12 },	/* SCK0 */
+	[RCAR_GP_PIN(3, 15)] = { PU4, 11 },	/* SD1_WP */
+	[RCAR_GP_PIN(3, 14)] = { PU4, 10 },	/* SD1_CD */
+	[RCAR_GP_PIN(3, 13)] = { PU4,  9 },	/* SD0_WP */
+	[RCAR_GP_PIN(3, 12)] = { PU4,  8 },	/* SD0_CD */
+	[RCAR_GP_PIN(4, 17)] = { PU4,  7 },	/* SD3_DS */
+	[RCAR_GP_PIN(4, 16)] = { PU4,  6 },	/* SD3_DAT7 */
+	[RCAR_GP_PIN(4, 15)] = { PU4,  5 },	/* SD3_DAT6 */
+	[RCAR_GP_PIN(4, 14)] = { PU4,  4 },	/* SD3_DAT5 */
+	[RCAR_GP_PIN(4, 13)] = { PU4,  3 },	/* SD3_DAT4 */
+	[RCAR_GP_PIN(4, 12)] = { PU4,  2 },	/* SD3_DAT3 */
+	[RCAR_GP_PIN(4, 11)] = { PU4,  1 },	/* SD3_DAT2 */
+	[RCAR_GP_PIN(4, 10)] = { PU4,  0 },	/* SD3_DAT1 */
+
+	[RCAR_GP_PIN(6, 24)] = { PU5, 31 },	/* USB0_PWEN */
+	[RCAR_GP_PIN(6, 23)] = { PU5, 30 },	/* AUDIO_CLKB_B */
+	[RCAR_GP_PIN(6, 22)] = { PU5, 29 },	/* AUDIO_CLKA_A */
+	[RCAR_GP_PIN(6, 21)] = { PU5, 28 },	/* SSI_SDATA9_A */
+	[RCAR_GP_PIN(6, 20)] = { PU5, 27 },	/* SSI_SDATA8 */
+	[RCAR_GP_PIN(6, 19)] = { PU5, 26 },	/* SSI_SDATA7 */
+	[RCAR_GP_PIN(6, 18)] = { PU5, 25 },	/* SSI_WS78 */
+	[RCAR_GP_PIN(6, 17)] = { PU5, 24 },	/* SSI_SCK78 */
+	[RCAR_GP_PIN(6, 16)] = { PU5, 23 },	/* SSI_SDATA6 */
+	[RCAR_GP_PIN(6, 15)] = { PU5, 22 },	/* SSI_WS6 */
+	[RCAR_GP_PIN(6, 14)] = { PU5, 21 },	/* SSI_SCK6 */
+	[RCAR_GP_PIN(6, 13)] = { PU5, 20 },	/* SSI_SDATA5 */
+	[RCAR_GP_PIN(6, 12)] = { PU5, 19 },	/* SSI_WS5 */
+	[RCAR_GP_PIN(6, 11)] = { PU5, 18 },	/* SSI_SCK5 */
+	[RCAR_GP_PIN(6, 10)] = { PU5, 17 },	/* SSI_SDATA4 */
+	[RCAR_GP_PIN(6,  9)] = { PU5, 16 },	/* SSI_WS4 */
+	[RCAR_GP_PIN(6,  8)] = { PU5, 15 },	/* SSI_SCK4 */
+	[RCAR_GP_PIN(6,  7)] = { PU5, 14 },	/* SSI_SDATA3 */
+	[RCAR_GP_PIN(6,  6)] = { PU5, 13 },	/* SSI_WS34 */
+	[RCAR_GP_PIN(6,  5)] = { PU5, 12 },	/* SSI_SCK34 */
+	[RCAR_GP_PIN(6,  4)] = { PU5, 11 },	/* SSI_SDATA2_A */
+	[RCAR_GP_PIN(6,  3)] = { PU5, 10 },	/* SSI_SDATA1_A */
+	[RCAR_GP_PIN(6,  2)] = { PU5,  9 },	/* SSI_SDATA0 */
+	[RCAR_GP_PIN(6,  1)] = { PU5,  8 },	/* SSI_WS01239 */
+	[RCAR_GP_PIN(6,  0)] = { PU5,  7 },	/* SSI_SCK01239 */
+	[RCAR_GP_PIN(5, 25)] = { PU5,  5 },	/* MLB_DAT */
+	[RCAR_GP_PIN(5, 24)] = { PU5,  4 },	/* MLB_SIG */
+	[RCAR_GP_PIN(5, 23)] = { PU5,  3 },	/* MLB_CLK */
+	[RCAR_GP_PIN(5, 22)] = { PU5,  2 },	/* MSIOF0_RXD */
+	[RCAR_GP_PIN(5, 21)] = { PU5,  1 },	/* MSIOF0_SS2 */
+	[RCAR_GP_PIN(5, 20)] = { PU5,  0 },	/* MSIOF0_TXD */
+
+	[RCAR_GP_PIN(6, 31)] = { PU6,  6 },	/* USB31_OVC */
+	[RCAR_GP_PIN(6, 30)] = { PU6,  5 },	/* USB31_PWEN */
+	[RCAR_GP_PIN(6, 29)] = { PU6,  4 },	/* USB30_OVC */
+	[RCAR_GP_PIN(6, 28)] = { PU6,  3 },	/* USB30_PWEN */
+	[RCAR_GP_PIN(6, 27)] = { PU6,  2 },	/* USB1_OVC */
+	[RCAR_GP_PIN(6, 26)] = { PU6,  1 },	/* USB1_PWEN */
+	[RCAR_GP_PIN(6, 25)] = { PU6,  0 },	/* USB0_OVC */
+};
+
+static unsigned int r8a7795_pinmux_get_bias(struct sh_pfc *pfc,
+					    unsigned int pin)
+{
+	u32 reg;
+	u32 bit;
+
+	if (WARN_ON_ONCE(!pullups[pin].reg))
+		return PIN_CONFIG_BIAS_DISABLE;
+
+	reg = pullups[pin].reg;
+	bit = BIT(pullups[pin].bit);
+
+	if (sh_pfc_read_reg(pfc, PUEN + reg, 32) & bit) {
+		if (sh_pfc_read_reg(pfc, PUD + reg, 32) & bit)
+			return PIN_CONFIG_BIAS_PULL_UP;
+		else
+			return PIN_CONFIG_BIAS_PULL_DOWN;
+	} else
+		return PIN_CONFIG_BIAS_DISABLE;
+}
+
+static void r8a7795_pinmux_set_bias(struct sh_pfc *pfc, unsigned int pin,
+				   unsigned int bias)
+{
+	u32 enable, updown;
+	u32 reg;
+	u32 bit;
+
+	if (WARN_ON_ONCE(!pullups[pin].reg))
+		return;
+
+	reg = pullups[pin].reg;
+	bit = BIT(pullups[pin].bit);
+
+	enable = sh_pfc_read_reg(pfc, PUEN + reg, 32) & ~bit;
+	if (bias != PIN_CONFIG_BIAS_DISABLE)
+		enable |= bit;
+
+	updown = sh_pfc_read_reg(pfc, PUD + reg, 32) & ~bit;
+	if (bias == PIN_CONFIG_BIAS_PULL_UP)
+		updown |= bit;
+
+	sh_pfc_write_reg(pfc, PUD + reg, 32, updown);
+	sh_pfc_write_reg(pfc, PUEN + reg, 32, enable);
+}
+
 static const struct sh_pfc_soc_operations r8a7795_pinmux_ops = {
 	.pin_to_pocctrl = r8a7795_pin_to_pocctrl,
+	.get_bias = r8a7795_pinmux_get_bias,
+	.set_bias = r8a7795_pinmux_set_bias,
 };
 
 const struct sh_pfc_soc_info r8a7795_pinmux_info = {
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7796.c b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
new file mode 100644
index 000000000000..dc9b671ccf2e
--- /dev/null
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
@@ -0,0 +1,2671 @@
+/*
+ * R8A7796 processor support - PFC hardware block.
+ *
+ * Copyright (C) 2016 Renesas Electronics Corp.
+ *
+ * This file is based on the drivers/pinctrl/sh-pfc/pfc-r8a7795.c
+ *
+ * R-Car Gen3 processor support - PFC hardware block.
+ *
+ * Copyright (C) 2015  Renesas Electronics Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ */
+
+#include <linux/kernel.h>
+
+#include "core.h"
+#include "sh_pfc.h"
+
+#define CPU_ALL_PORT(fn, sfx)						\
+	PORT_GP_16(0, fn, sfx),						\
+	PORT_GP_29(1, fn, sfx),						\
+	PORT_GP_15(2, fn, sfx),						\
+	PORT_GP_CFG_12(3, fn, sfx, SH_PFC_PIN_CFG_IO_VOLTAGE),		\
+	PORT_GP_1(3, 12, fn, sfx),					\
+	PORT_GP_1(3, 13, fn, sfx),					\
+	PORT_GP_1(3, 14, fn, sfx),					\
+	PORT_GP_1(3, 15, fn, sfx),					\
+	PORT_GP_CFG_18(4, fn, sfx, SH_PFC_PIN_CFG_IO_VOLTAGE),		\
+	PORT_GP_26(5, fn, sfx),						\
+	PORT_GP_32(6, fn, sfx),						\
+	PORT_GP_4(7, fn, sfx)
+/*
+ * F_() : just information
+ * FM() : macro for FN_xxx / xxx_MARK
+ */
+
+/* GPSR0 */
+#define GPSR0_15	F_(D15,			IP7_11_8)
+#define GPSR0_14	F_(D14,			IP7_7_4)
+#define GPSR0_13	F_(D13,			IP7_3_0)
+#define GPSR0_12	F_(D12,			IP6_31_28)
+#define GPSR0_11	F_(D11,			IP6_27_24)
+#define GPSR0_10	F_(D10,			IP6_23_20)
+#define GPSR0_9		F_(D9,			IP6_19_16)
+#define GPSR0_8		F_(D8,			IP6_15_12)
+#define GPSR0_7		F_(D7,			IP6_11_8)
+#define GPSR0_6		F_(D6,			IP6_7_4)
+#define GPSR0_5		F_(D5,			IP6_3_0)
+#define GPSR0_4		F_(D4,			IP5_31_28)
+#define GPSR0_3		F_(D3,			IP5_27_24)
+#define GPSR0_2		F_(D2,			IP5_23_20)
+#define GPSR0_1		F_(D1,			IP5_19_16)
+#define GPSR0_0		F_(D0,			IP5_15_12)
+
+/* GPSR1 */
+#define GPSR1_28	FM(CLKOUT)
+#define GPSR1_27	F_(EX_WAIT0_A,		IP5_11_8)
+#define GPSR1_26	F_(WE1_N,		IP5_7_4)
+#define GPSR1_25	F_(WE0_N,		IP5_3_0)
+#define GPSR1_24	F_(RD_WR_N,		IP4_31_28)
+#define GPSR1_23	F_(RD_N,		IP4_27_24)
+#define GPSR1_22	F_(BS_N,		IP4_23_20)
+#define GPSR1_21	F_(CS1_N_A26,		IP4_19_16)
+#define GPSR1_20	F_(CS0_N,		IP4_15_12)
+#define GPSR1_19	F_(A19,			IP4_11_8)
+#define GPSR1_18	F_(A18,			IP4_7_4)
+#define GPSR1_17	F_(A17,			IP4_3_0)
+#define GPSR1_16	F_(A16,			IP3_31_28)
+#define GPSR1_15	F_(A15,			IP3_27_24)
+#define GPSR1_14	F_(A14,			IP3_23_20)
+#define GPSR1_13	F_(A13,			IP3_19_16)
+#define GPSR1_12	F_(A12,			IP3_15_12)
+#define GPSR1_11	F_(A11,			IP3_11_8)
+#define GPSR1_10	F_(A10,			IP3_7_4)
+#define GPSR1_9		F_(A9,			IP3_3_0)
+#define GPSR1_8		F_(A8,			IP2_31_28)
+#define GPSR1_7		F_(A7,			IP2_27_24)
+#define GPSR1_6		F_(A6,			IP2_23_20)
+#define GPSR1_5		F_(A5,			IP2_19_16)
+#define GPSR1_4		F_(A4,			IP2_15_12)
+#define GPSR1_3		F_(A3,			IP2_11_8)
+#define GPSR1_2		F_(A2,			IP2_7_4)
+#define GPSR1_1		F_(A1,			IP2_3_0)
+#define GPSR1_0		F_(A0,			IP1_31_28)
+
+/* GPSR2 */
+#define GPSR2_14	F_(AVB_AVTP_CAPTURE_A,	IP0_23_20)
+#define GPSR2_13	F_(AVB_AVTP_MATCH_A,	IP0_19_16)
+#define GPSR2_12	F_(AVB_LINK,		IP0_15_12)
+#define GPSR2_11	F_(AVB_PHY_INT,		IP0_11_8)
+#define GPSR2_10	F_(AVB_MAGIC,		IP0_7_4)
+#define GPSR2_9		F_(AVB_MDC,		IP0_3_0)
+#define GPSR2_8		F_(PWM2_A,		IP1_27_24)
+#define GPSR2_7		F_(PWM1_A,		IP1_23_20)
+#define GPSR2_6		F_(PWM0,		IP1_19_16)
+#define GPSR2_5		F_(IRQ5,		IP1_15_12)
+#define GPSR2_4		F_(IRQ4,		IP1_11_8)
+#define GPSR2_3		F_(IRQ3,		IP1_7_4)
+#define GPSR2_2		F_(IRQ2,		IP1_3_0)
+#define GPSR2_1		F_(IRQ1,		IP0_31_28)
+#define GPSR2_0		F_(IRQ0,		IP0_27_24)
+
+/* GPSR3 */
+#define GPSR3_15	F_(SD1_WP,		IP11_23_20)
+#define GPSR3_14	F_(SD1_CD,		IP11_19_16)
+#define GPSR3_13	F_(SD0_WP,		IP11_15_12)
+#define GPSR3_12	F_(SD0_CD,		IP11_11_8)
+#define GPSR3_11	F_(SD1_DAT3,		IP8_31_28)
+#define GPSR3_10	F_(SD1_DAT2,		IP8_27_24)
+#define GPSR3_9		F_(SD1_DAT1,		IP8_23_20)
+#define GPSR3_8		F_(SD1_DAT0,		IP8_19_16)
+#define GPSR3_7		F_(SD1_CMD,		IP8_15_12)
+#define GPSR3_6		F_(SD1_CLK,		IP8_11_8)
+#define GPSR3_5		F_(SD0_DAT3,		IP8_7_4)
+#define GPSR3_4		F_(SD0_DAT2,		IP8_3_0)
+#define GPSR3_3		F_(SD0_DAT1,		IP7_31_28)
+#define GPSR3_2		F_(SD0_DAT0,		IP7_27_24)
+#define GPSR3_1		F_(SD0_CMD,		IP7_23_20)
+#define GPSR3_0		F_(SD0_CLK,		IP7_19_16)
+
+/* GPSR4 */
+#define GPSR4_17	F_(SD3_DS,		IP11_11_8)
+#define GPSR4_16	F_(SD3_DAT7,		IP10_7_4)
+#define GPSR4_15	F_(SD3_DAT6,		IP10_3_0)
+#define GPSR4_14	F_(SD3_DAT5,		IP9_31_28)
+#define GPSR4_13	F_(SD3_DAT4,		IP9_27_24)
+#define GPSR4_12	F_(SD3_DAT3,		IP10_19_16)
+#define GPSR4_11	F_(SD3_DAT2,		IP10_15_12)
+#define GPSR4_10	F_(SD3_DAT1,		IP10_11_8)
+#define GPSR4_9		F_(SD3_DAT0,		IP10_7_4)
+#define GPSR4_8		F_(SD3_CMD,		IP10_3_0)
+#define GPSR4_7		F_(SD3_CLK,		IP9_31_28)
+#define GPSR4_6		F_(SD2_DS,		IP9_23_20)
+#define GPSR4_5		F_(SD2_DAT3,		IP9_19_16)
+#define GPSR4_4		F_(SD2_DAT2,		IP9_15_12)
+#define GPSR4_3		F_(SD2_DAT1,		IP9_11_8)
+#define GPSR4_2		F_(SD2_DAT0,		IP9_7_4)
+#define GPSR4_1		F_(SD2_CMD,		IP9_7_4)
+#define GPSR4_0		F_(SD2_CLK,		IP9_3_0)
+
+/* GPSR5 */
+#define GPSR5_25	F_(MLB_DAT,		IP14_19_16)
+#define GPSR5_24	F_(MLB_SIG,		IP14_15_12)
+#define GPSR5_23	F_(MLB_CLK,		IP14_11_8)
+#define GPSR5_22	FM(MSIOF0_RXD)
+#define GPSR5_21	F_(MSIOF0_SS2,		IP14_7_4)
+#define GPSR5_20	FM(MSIOF0_TXD)
+#define GPSR5_19	F_(MSIOF0_SS1,		IP14_3_0)
+#define GPSR5_18	F_(MSIOF0_SYNC,		IP13_31_28)
+#define GPSR5_17	FM(MSIOF0_SCK)
+#define GPSR5_16	F_(HRTS0_N,		IP13_27_24)
+#define GPSR5_15	F_(HCTS0_N,		IP13_23_20)
+#define GPSR5_14	F_(HTX0,		IP13_19_16)
+#define GPSR5_13	F_(HRX0,		IP13_15_12)
+#define GPSR5_12	F_(HSCK0,		IP13_11_8)
+#define GPSR5_11	F_(RX2_A,		IP13_7_4)
+#define GPSR5_10	F_(TX2_A,		IP13_3_0)
+#define GPSR5_9		F_(SCK2,		IP12_31_28)
+#define GPSR5_8		F_(RTS1_N_TANS,		IP12_27_24)
+#define GPSR5_7		F_(CTS1_N,		IP12_23_20)
+#define GPSR5_6		F_(TX1_A,		IP12_19_16)
+#define GPSR5_5		F_(RX1_A,		IP12_15_12)
+#define GPSR5_4		F_(RTS0_N_TANS,		IP12_11_8)
+#define GPSR5_3		F_(CTS0_N,		IP12_7_4)
+#define GPSR5_2		F_(TX0,			IP12_3_0)
+#define GPSR5_1		F_(RX0,			IP11_31_28)
+#define GPSR5_0		F_(SCK0,		IP11_27_24)
+
+/* GPSR6 */
+#define GPSR6_31	F_(GP6_31,		IP18_7_4)
+#define GPSR6_30	F_(GP6_30,		IP18_3_0)
+#define GPSR6_29	F_(USB30_OVC,		IP17_31_28)
+#define GPSR6_28	F_(USB30_PWEN,		IP17_27_24)
+#define GPSR6_27	F_(USB1_OVC,		IP17_23_20)
+#define GPSR6_26	F_(USB1_PWEN,		IP17_19_16)
+#define GPSR6_25	F_(USB0_OVC,		IP17_15_12)
+#define GPSR6_24	F_(USB0_PWEN,		IP17_11_8)
+#define GPSR6_23	F_(AUDIO_CLKB_B,	IP17_7_4)
+#define GPSR6_22	F_(AUDIO_CLKA_A,	IP17_3_0)
+#define GPSR6_21	F_(SSI_SDATA9_A,	IP16_31_28)
+#define GPSR6_20	F_(SSI_SDATA8,		IP16_27_24)
+#define GPSR6_19	F_(SSI_SDATA7,		IP16_23_20)
+#define GPSR6_18	F_(SSI_WS78,		IP16_19_16)
+#define GPSR6_17	F_(SSI_SCK78,		IP16_15_12)
+#define GPSR6_16	F_(SSI_SDATA6,		IP16_11_8)
+#define GPSR6_15	F_(SSI_WS6,		IP16_7_4)
+#define GPSR6_14	F_(SSI_SCK6,		IP16_3_0)
+#define GPSR6_13	FM(SSI_SDATA5)
+#define GPSR6_12	FM(SSI_WS5)
+#define GPSR6_11	FM(SSI_SCK5)
+#define GPSR6_10	F_(SSI_SDATA4,		IP15_31_28)
+#define GPSR6_9		F_(SSI_WS4,		IP15_27_24)
+#define GPSR6_8		F_(SSI_SCK4,		IP15_23_20)
+#define GPSR6_7		F_(SSI_SDATA3,		IP15_19_16)
+#define GPSR6_6		F_(SSI_WS34,		IP15_15_12)
+#define GPSR6_5		F_(SSI_SCK34,		IP15_11_8)
+#define GPSR6_4		F_(SSI_SDATA2_A,	IP15_7_4)
+#define GPSR6_3		F_(SSI_SDATA1_A,	IP15_3_0)
+#define GPSR6_2		F_(SSI_SDATA0,		IP14_31_28)
+#define GPSR6_1		F_(SSI_WS0129,		IP14_27_24)
+#define GPSR6_0		F_(SSI_SCK0129,		IP14_23_20)
+
+/* GPSR7 */
+#define GPSR7_3		FM(GP7_03)
+#define GPSR7_2		FM(HDMI0_CEC)
+#define GPSR7_1		FM(AVS2)
+#define GPSR7_0		FM(AVS1)
+
+
+/* IPSRx */		/* 0 */			/* 1 */		/* 2 */			/* 3 */				/* 4 */		/* 5 */		/* 6 */			/* 7 */		/* 8 */			/* 9 */		/* A */		/* B */		/* C - F */
+#define IP0_3_0		FM(AVB_MDC)		F_(0, 0)	FM(MSIOF2_SS2_C)	F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_7_4		FM(AVB_MAGIC)		F_(0, 0)	FM(MSIOF2_SS1_C)	FM(SCK4_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_11_8	FM(AVB_PHY_INT)		F_(0, 0)	FM(MSIOF2_SYNC_C)	FM(RX4_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_15_12	FM(AVB_LINK)		F_(0, 0)	FM(MSIOF2_SCK_C)	FM(TX4_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_19_16	FM(AVB_AVTP_MATCH_A)	F_(0, 0)	FM(MSIOF2_RXD_C)	FM(CTS4_N_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_23_20	FM(AVB_AVTP_CAPTURE_A)	F_(0, 0)	FM(MSIOF2_TXD_C)	FM(RTS4_N_TANS_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_27_24	FM(IRQ0)		FM(QPOLB)	F_(0, 0)		FM(DU_CDE)			FM(VI4_DATA0_B) FM(CAN0_TX_B)	FM(CANFD0_TX_B)		FM(MSIOF3_SS1_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_31_28	FM(IRQ1)		FM(QPOLA)	F_(0, 0)		FM(DU_DISP)			FM(VI4_DATA1_B) FM(CAN0_RX_B)	FM(CANFD0_RX_B)		FM(MSIOF3_SS2_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_3_0		FM(IRQ2)		FM(QCPV_QDE)	F_(0, 0)		FM(DU_EXODDF_DU_ODDF_DISP_CDE)	FM(VI4_DATA2_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_SYNC_E) F_(0, 0)		FM(PWM3_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_7_4		FM(IRQ3)		FM(QSTVB_QVE)	FM(A25)			FM(DU_DOTCLKOUT1)		FM(VI4_DATA3_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_SCK_E) F_(0, 0)		FM(PWM4_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_11_8	FM(IRQ4)		FM(QSTH_QHS)	FM(A24)			FM(DU_EXHSYNC_DU_HSYNC)		FM(VI4_DATA4_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_RXD_E) F_(0, 0)		FM(PWM5_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_15_12	FM(IRQ5)		FM(QSTB_QHE)	FM(A23)			FM(DU_EXVSYNC_DU_VSYNC)		FM(VI4_DATA5_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_TXD_E) F_(0, 0)		FM(PWM6_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_19_16	FM(PWM0)		FM(AVB_AVTP_PPS)FM(A22)			F_(0, 0)			FM(VI4_DATA6_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		FM(IECLK_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_23_20	FM(PWM1_A)		F_(0, 0)	FM(A21)			FM(HRX3_D)			FM(VI4_DATA7_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		FM(IERX_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_27_24	FM(PWM2_A)		F_(0, 0)	FM(A20)			FM(HTX3_D)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		FM(IETX_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_31_28	FM(A0)			FM(LCDOUT16)	FM(MSIOF3_SYNC_B)	F_(0, 0)			FM(VI4_DATA8)	F_(0, 0)	FM(DU_DB0)		F_(0, 0)	F_(0, 0)		FM(PWM3_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_3_0		FM(A1)			FM(LCDOUT17)	FM(MSIOF3_TXD_B)	F_(0, 0)			FM(VI4_DATA9)	F_(0, 0)	FM(DU_DB1)		F_(0, 0)	F_(0, 0)		FM(PWM4_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_7_4		FM(A2)			FM(LCDOUT18)	FM(MSIOF3_SCK_B)	F_(0, 0)			FM(VI4_DATA10)	F_(0, 0)	FM(DU_DB2)		F_(0, 0)	F_(0, 0)		FM(PWM5_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_11_8	FM(A3)			FM(LCDOUT19)	FM(MSIOF3_RXD_B)	F_(0, 0)			FM(VI4_DATA11)	F_(0, 0)	FM(DU_DB3)		F_(0, 0)	F_(0, 0)		FM(PWM6_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_15_12	FM(A4)			FM(LCDOUT20)	FM(MSIOF3_SS1_B)	F_(0, 0)			FM(VI4_DATA12)	FM(VI5_DATA12)	FM(DU_DB4)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_19_16	FM(A5)			FM(LCDOUT21)	FM(MSIOF3_SS2_B)	FM(SCK4_B)			FM(VI4_DATA13)	FM(VI5_DATA13)	FM(DU_DB5)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_23_20	FM(A6)			FM(LCDOUT22)	FM(MSIOF2_SS1_A)	FM(RX4_B)			FM(VI4_DATA14)	FM(VI5_DATA14)	FM(DU_DB6)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_27_24	FM(A7)			FM(LCDOUT23)	FM(MSIOF2_SS2_A)	FM(TX4_B)			FM(VI4_DATA15)	FM(VI5_DATA15)	FM(DU_DB7)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_31_28	FM(A8)			FM(RX3_B)	FM(MSIOF2_SYNC_A)	FM(HRX4_B)			F_(0, 0)	F_(0, 0)	F_(0, 0)		FM(SDA6_A)	FM(AVB_AVTP_MATCH_B)	FM(PWM1_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_3_0		FM(A9)			F_(0, 0)	FM(MSIOF2_SCK_A)	FM(CTS4_N_B)			F_(0, 0)	FM(VI5_VSYNC_N)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_7_4		FM(A10)			F_(0, 0)	FM(MSIOF2_RXD_A)	FM(RTS4_N_TANS_B)		F_(0, 0)	FM(VI5_HSYNC_N)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_11_8	FM(A11)			FM(TX3_B)	FM(MSIOF2_TXD_A)	FM(HTX4_B)			FM(HSCK4)	FM(VI5_FIELD)	F_(0, 0)		FM(SCL6_A)	FM(AVB_AVTP_CAPTURE_B)	FM(PWM2_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */		/* 2 */			/* 3 */				/* 4 */		/* 5 */		/* 6 */			/* 7 */		/* 8 */			/* 9 */		/* A */		/* B */		/* C - F */
+#define IP3_15_12	FM(A12)			FM(LCDOUT12)	FM(MSIOF3_SCK_C)	F_(0, 0)			FM(HRX4_A)	FM(VI5_DATA8)	FM(DU_DG4)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_19_16	FM(A13)			FM(LCDOUT13)	FM(MSIOF3_SYNC_C)	F_(0, 0)			FM(HTX4_A)	FM(VI5_DATA9)	FM(DU_DG5)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_23_20	FM(A14)			FM(LCDOUT14)	FM(MSIOF3_RXD_C)	F_(0, 0)			FM(HCTS4_N)	FM(VI5_DATA10)	FM(DU_DG6)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_27_24	FM(A15)			FM(LCDOUT15)	FM(MSIOF3_TXD_C)	F_(0, 0)			FM(HRTS4_N)	FM(VI5_DATA11)	FM(DU_DG7)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_31_28	FM(A16)			FM(LCDOUT8)	F_(0, 0)		F_(0, 0)			FM(VI4_FIELD)	F_(0, 0)	FM(DU_DG0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_3_0		FM(A17)			FM(LCDOUT9)	F_(0, 0)		F_(0, 0)			FM(VI4_VSYNC_N)	F_(0, 0)	FM(DU_DG1)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_7_4		FM(A18)			FM(LCDOUT10)	F_(0, 0)		F_(0, 0)			FM(VI4_HSYNC_N)	F_(0, 0)	FM(DU_DG2)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_11_8	FM(A19)			FM(LCDOUT11)	F_(0, 0)		F_(0, 0)			FM(VI4_CLKENB)	F_(0, 0)	FM(DU_DG3)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_15_12	FM(CS0_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(VI5_CLKENB)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_19_16	FM(CS1_N_A26)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(VI5_CLK)	F_(0, 0)		FM(EX_WAIT0_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_23_20	FM(BS_N)		FM(QSTVA_QVS)	FM(MSIOF3_SCK_D)	FM(SCK3)			FM(HSCK3)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN1_TX)		FM(CANFD1_TX)	FM(IETX_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_27_24	FM(RD_N)		F_(0, 0)	FM(MSIOF3_SYNC_D)	FM(RX3_A)			FM(HRX3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN0_TX_A)		FM(CANFD0_TX_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_31_28	FM(RD_WR_N)		F_(0, 0)	FM(MSIOF3_RXD_D)	FM(TX3_A)			FM(HTX3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN0_RX_A)		FM(CANFD0_RX_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_3_0		FM(WE0_N)		F_(0, 0)	FM(MSIOF3_TXD_D)	FM(CTS3_N)			FM(HCTS3_N)	F_(0, 0)	F_(0, 0)		FM(SCL6_B)	FM(CAN_CLK)		F_(0, 0)	FM(IECLK_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_7_4		FM(WE1_N)		F_(0, 0)	FM(MSIOF3_SS1_D)	FM(RTS3_N_TANS)			FM(HRTS3_N)	F_(0, 0)	F_(0, 0)		FM(SDA6_B)	FM(CAN1_RX)		FM(CANFD1_RX)	FM(IERX_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_11_8	FM(EX_WAIT0_A)		FM(QCLK)	F_(0, 0)		F_(0, 0)			FM(VI4_CLK)	F_(0, 0)	FM(DU_DOTCLKOUT0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_15_12	FM(D0)			FM(MSIOF2_SS1_B)FM(MSIOF3_SCK_A)	F_(0, 0)			FM(VI4_DATA16)	FM(VI5_DATA0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_19_16	FM(D1)			FM(MSIOF2_SS2_B)FM(MSIOF3_SYNC_A)	F_(0, 0)			FM(VI4_DATA17)	FM(VI5_DATA1)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_23_20	FM(D2)			F_(0, 0)	FM(MSIOF3_RXD_A)	F_(0, 0)			FM(VI4_DATA18)	FM(VI5_DATA2)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_27_24	FM(D3)			F_(0, 0)	FM(MSIOF3_TXD_A)	F_(0, 0)			FM(VI4_DATA19)	FM(VI5_DATA3)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_31_28	FM(D4)			FM(MSIOF2_SCK_B)F_(0, 0)		F_(0, 0)			FM(VI4_DATA20)	FM(VI5_DATA4)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_3_0		FM(D5)			FM(MSIOF2_SYNC_B)F_(0, 0)		F_(0, 0)			FM(VI4_DATA21)	FM(VI5_DATA5)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_7_4		FM(D6)			FM(MSIOF2_RXD_B)F_(0, 0)		F_(0, 0)			FM(VI4_DATA22)	FM(VI5_DATA6)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_11_8	FM(D7)			FM(MSIOF2_TXD_B)F_(0, 0)		F_(0, 0)			FM(VI4_DATA23)	FM(VI5_DATA7)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_15_12	FM(D8)			FM(LCDOUT0)	FM(MSIOF2_SCK_D)	FM(SCK4_C)			FM(VI4_DATA0_A)	F_(0, 0)	FM(DU_DR0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_19_16	FM(D9)			FM(LCDOUT1)	FM(MSIOF2_SYNC_D)	F_(0, 0)			FM(VI4_DATA1_A)	F_(0, 0)	FM(DU_DR1)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_23_20	FM(D10)			FM(LCDOUT2)	FM(MSIOF2_RXD_D)	FM(HRX3_B)			FM(VI4_DATA2_A)	FM(CTS4_N_C)	FM(DU_DR2)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_27_24	FM(D11)			FM(LCDOUT3)	FM(MSIOF2_TXD_D)	FM(HTX3_B)			FM(VI4_DATA3_A)	FM(RTS4_N_TANS_C)FM(DU_DR3)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_31_28	FM(D12)			FM(LCDOUT4)	FM(MSIOF2_SS1_D)	FM(RX4_C)			FM(VI4_DATA4_A)	F_(0, 0)	FM(DU_DR4)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */		/* 2 */			/* 3 */				/* 4 */		/* 5 */		/* 6 */			/* 7 */		/* 8 */			/* 9 */		/* A */		/* B */		/* C - F */
+#define IP7_3_0		FM(D13)			FM(LCDOUT5)	FM(MSIOF2_SS2_D)	FM(TX4_C)			FM(VI4_DATA5_A)	F_(0, 0)	FM(DU_DR5)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_7_4		FM(D14)			FM(LCDOUT6)	FM(MSIOF3_SS1_A)	FM(HRX3_C)			FM(VI4_DATA6_A)	F_(0, 0)	FM(DU_DR6)		FM(SCL6_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_11_8	FM(D15)			FM(LCDOUT7)	FM(MSIOF3_SS2_A)	FM(HTX3_C)			FM(VI4_DATA7_A)	F_(0, 0)	FM(DU_DR7)		FM(SDA6_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_15_12	FM(FSCLKST)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_19_16	FM(SD0_CLK)		F_(0, 0)	FM(MSIOF1_SCK_E)	F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_OPWM_0_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_23_20	FM(SD0_CMD)		F_(0, 0)	FM(MSIOF1_SYNC_E)	F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_IVCXO27_0_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_27_24	FM(SD0_DAT0)		F_(0, 0)	FM(MSIOF1_RXD_E)	F_(0, 0)			F_(0, 0)	FM(TS_SCK0_B)	FM(STP_ISCLK_0_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_31_28	FM(SD0_DAT1)		F_(0, 0)	FM(MSIOF1_TXD_E)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC0_B)FM(STP_ISSYNC_0_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_3_0		FM(SD0_DAT2)		F_(0, 0)	FM(MSIOF1_SS1_E)	F_(0, 0)			F_(0, 0)	FM(TS_SDAT0_B)	FM(STP_ISD_0_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_7_4		FM(SD0_DAT3)		F_(0, 0)	FM(MSIOF1_SS2_E)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN0_B)	FM(STP_ISEN_0_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_11_8	FM(SD1_CLK)		F_(0, 0)	FM(MSIOF1_SCK_G)	F_(0, 0)			F_(0, 0)	FM(SIM0_CLK_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_15_12	FM(SD1_CMD)		F_(0, 0)	FM(MSIOF1_SYNC_G)	FM(NFCE_N_B)			F_(0, 0)	FM(SIM0_D_A)	FM(STP_IVCXO27_1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_19_16	FM(SD1_DAT0)		FM(SD2_DAT4)	FM(MSIOF1_RXD_G)	FM(NFWP_N_B)			F_(0, 0)	FM(TS_SCK1_B)	FM(STP_ISCLK_1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_23_20	FM(SD1_DAT1)		FM(SD2_DAT5)	FM(MSIOF1_TXD_G)	FM(NFDATA14_B)			F_(0, 0)	FM(TS_SPSYNC1_B)FM(STP_ISSYNC_1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_27_24	FM(SD1_DAT2)		FM(SD2_DAT6)	FM(MSIOF1_SS1_G)	FM(NFDATA15_B)			F_(0, 0)	FM(TS_SDAT1_B)	FM(STP_ISD_1_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_31_28	FM(SD1_DAT3)		FM(SD2_DAT7)	FM(MSIOF1_SS2_G)	FM(NFRB_N_B)			F_(0, 0)	FM(TS_SDEN1_B)	FM(STP_ISEN_1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_3_0		FM(SD2_CLK)		F_(0, 0)	F_(0, 0)		FM(NFDATA8)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_7_4		FM(SD2_CMD)		F_(0, 0)	F_(0, 0)		FM(NFDATA9)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_11_8	FM(SD2_DAT0)		F_(0, 0)	F_(0, 0)		FM(NFDATA10)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_15_12	FM(SD2_DAT1)		F_(0, 0)	F_(0, 0)		FM(NFDATA11)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_19_16	FM(SD2_DAT2)		F_(0, 0)	F_(0, 0)		FM(NFDATA12)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_23_20	FM(SD2_DAT3)		F_(0, 0)	F_(0, 0)		FM(NFDATA13)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_27_24	FM(SD2_DS)		F_(0, 0)	F_(0, 0)		FM(NFALE)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(SATA_DEVSLP_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_31_28	FM(SD3_CLK)		F_(0, 0)	F_(0, 0)		FM(NFWE_N)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_3_0	FM(SD3_CMD)		F_(0, 0)	F_(0, 0)		FM(NFRE_N)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_7_4	FM(SD3_DAT0)		F_(0, 0)	F_(0, 0)		FM(NFDATA0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_11_8	FM(SD3_DAT1)		F_(0, 0)	F_(0, 0)		FM(NFDATA1)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_15_12	FM(SD3_DAT2)		F_(0, 0)	F_(0, 0)		FM(NFDATA2)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_19_16	FM(SD3_DAT3)		F_(0, 0)	F_(0, 0)		FM(NFDATA3)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_23_20	FM(SD3_DAT4)		FM(SD2_CD_A)	F_(0, 0)		FM(NFDATA4)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_27_24	FM(SD3_DAT5)		FM(SD2_WP_A)	F_(0, 0)		FM(NFDATA5)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_31_28	FM(SD3_DAT6)		FM(SD3_CD)	F_(0, 0)		FM(NFDATA6)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_3_0	FM(SD3_DAT7)		FM(SD3_WP)	F_(0, 0)		FM(NFDATA7)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_7_4	FM(SD3_DS)		F_(0, 0)	F_(0, 0)		FM(NFCLE)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_11_8	FM(SD0_CD)		F_(0, 0)	FM(NFDATA14_A)		F_(0, 0)			FM(SCL2_B)	FM(SIM0_RST_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */		/* 2 */			/* 3 */				/* 4 */		/* 5 */		/* 6 */			/* 7 */		/* 8 */			/* 9 */		/* A */		/* B */		/* C - F */
+#define IP11_15_12	FM(SD0_WP)		F_(0, 0)	FM(NFDATA15_A)		F_(0, 0)			FM(SDA2_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_19_16	FM(SD1_CD)		F_(0, 0)	FM(NFRB_N_A)		F_(0, 0)			F_(0, 0)	FM(SIM0_CLK_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_23_20	FM(SD1_WP)		F_(0, 0)	FM(NFCE_N_A)		F_(0, 0)			F_(0, 0)	FM(SIM0_D_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_27_24	FM(SCK0)		FM(HSCK1_B)	FM(MSIOF1_SS2_B)	FM(AUDIO_CLKC_B)		FM(SDA2_A)	FM(SIM0_RST_B)	FM(STP_OPWM_0_C)	FM(RIF0_CLK_B)	F_(0, 0)		FM(ADICHS2)	FM(SCK5_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_31_28	FM(RX0)			FM(HRX1_B)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SCK0_C)	FM(STP_ISCLK_0_C)	FM(RIF0_D0_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_3_0	FM(TX0)			FM(HTX1_B)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC0_C)FM(STP_ISSYNC_0_C)	FM(RIF0_D1_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_7_4	FM(CTS0_N)		FM(HCTS1_N_B)	FM(MSIOF1_SYNC_B)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC1_C)FM(STP_ISSYNC_1_C)	FM(RIF1_SYNC_B)	FM(AUDIO_CLKOUT_C)	FM(ADICS_SAMP)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_11_8	FM(RTS0_N_TANS)		FM(HRTS1_N_B)	FM(MSIOF1_SS1_B)	FM(AUDIO_CLKA_B)		FM(SCL2_A)	F_(0, 0)	FM(STP_IVCXO27_1_C)	FM(RIF0_SYNC_B)	FM(FSO_TOE_A)		FM(ADICHS1)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_15_12	FM(RX1_A)		FM(HRX1_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SDAT0_C)	FM(STP_ISD_0_C)		FM(RIF1_CLK_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_19_16	FM(TX1_A)		FM(HTX1_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SDEN0_C)	FM(STP_ISEN_0_C)	FM(RIF1_D0_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_23_20	FM(CTS1_N)		FM(HCTS1_N_A)	FM(MSIOF1_RXD_B)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN1_C)	FM(STP_ISEN_1_C)	FM(RIF1_D0_B)	F_(0, 0)		FM(ADIDATA)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_27_24	FM(RTS1_N_TANS)		FM(HRTS1_N_A)	FM(MSIOF1_TXD_B)	F_(0, 0)			F_(0, 0)	FM(TS_SDAT1_C)	FM(STP_ISD_1_C)		FM(RIF1_D1_B)	F_(0, 0)		FM(ADICHS0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_31_28	FM(SCK2)		FM(SCIF_CLK_B)	FM(MSIOF1_SCK_B)	F_(0, 0)			F_(0, 0)	FM(TS_SCK1_C)	FM(STP_ISCLK_1_C)	FM(RIF1_CLK_B)	F_(0, 0)		FM(ADICLK)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_3_0	FM(TX2_A)		F_(0, 0)	F_(0, 0)		FM(SD2_CD_B)			FM(SCL1_A)	F_(0, 0)	FM(FMCLK_A)		FM(RIF1_D1_C)	F_(0, 0)		FM(FSO_CFE_0_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_7_4	FM(RX2_A)		F_(0, 0)	F_(0, 0)		FM(SD2_WP_B)			FM(SDA1_A)	F_(0, 0)	FM(FMIN_A)		FM(RIF1_SYNC_C)	F_(0, 0)		FM(FSO_CFE_1_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_11_8	FM(HSCK0)		F_(0, 0)	FM(MSIOF1_SCK_D)	FM(AUDIO_CLKB_A)		FM(SSI_SDATA1_B)FM(TS_SCK0_D)	FM(STP_ISCLK_0_D)	FM(RIF0_CLK_C)	F_(0, 0)		F_(0, 0)	FM(RX5_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_15_12	FM(HRX0)		F_(0, 0)	FM(MSIOF1_RXD_D)	F_(0, 0)			FM(SSI_SDATA2_B)FM(TS_SDEN0_D)	FM(STP_ISEN_0_D)	FM(RIF0_D0_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_19_16	FM(HTX0)		F_(0, 0)	FM(MSIOF1_TXD_D)	F_(0, 0)			FM(SSI_SDATA9_B)FM(TS_SDAT0_D)	FM(STP_ISD_0_D)		FM(RIF0_D1_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_23_20	FM(HCTS0_N)		FM(RX2_B)	FM(MSIOF1_SYNC_D)	F_(0, 0)			FM(SSI_SCK9_A)	FM(TS_SPSYNC0_D)FM(STP_ISSYNC_0_D)	FM(RIF0_SYNC_C)	FM(AUDIO_CLKOUT1_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_27_24	FM(HRTS0_N)		FM(TX2_B)	FM(MSIOF1_SS1_D)	F_(0, 0)			FM(SSI_WS9_A)	F_(0, 0)	FM(STP_IVCXO27_0_D)	FM(BPFCLK_A)	FM(AUDIO_CLKOUT2_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_31_28	FM(MSIOF0_SYNC)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(AUDIO_CLKOUT_A)	F_(0, 0)	FM(TX5_B)	F_(0, 0)	F_(0, 0) FM(BPFCLK_D) F_(0, 0) F_(0, 0)
+#define IP14_3_0	FM(MSIOF0_SS1)		FM(RX5_A)	FM(NFWP_N_A)		FM(AUDIO_CLKA_C)		FM(SSI_SCK2_A)	F_(0, 0)	FM(STP_IVCXO27_0_C)	F_(0, 0)	FM(AUDIO_CLKOUT3_A)	F_(0, 0)	FM(TCLK1_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_7_4	FM(MSIOF0_SS2)		FM(TX5_A)	FM(MSIOF1_SS2_D)	FM(AUDIO_CLKC_A)		FM(SSI_WS2_A)	F_(0, 0)	FM(STP_OPWM_0_D)	F_(0, 0)	FM(AUDIO_CLKOUT_D)	F_(0, 0)	FM(SPEEDIN_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_11_8	FM(MLB_CLK)		F_(0, 0)	FM(MSIOF1_SCK_F)	F_(0, 0)			FM(SCL1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_15_12	FM(MLB_SIG)		FM(RX1_B)	FM(MSIOF1_SYNC_F)	F_(0, 0)			FM(SDA1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_19_16	FM(MLB_DAT)		FM(TX1_B)	FM(MSIOF1_RXD_F)	F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_23_20	FM(SSI_SCK0129)		F_(0, 0)	FM(MSIOF1_TXD_F)	F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP14_27_24	FM(SSI_WS0129)		F_(0, 0)	FM(MSIOF1_SS1_F)	F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */		/* 2 */			/* 3 */				/* 4 */		/* 5 */		/* 6 */			/* 7 */		/* 8 */			/* 9 */		/* A */		/* B */		/* C - F */
+#define IP14_31_28	FM(SSI_SDATA0)		F_(0, 0)	FM(MSIOF1_SS2_F)	F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_3_0	FM(SSI_SDATA1_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_7_4	FM(SSI_SDATA2_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			FM(SSI_SCK1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_11_8	FM(SSI_SCK34)		F_(0, 0)	FM(MSIOF1_SS1_A)	F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_OPWM_0_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_15_12	FM(SSI_WS34)		FM(HCTS2_N_A)	FM(MSIOF1_SS2_A)	F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_IVCXO27_0_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_19_16	FM(SSI_SDATA3)		FM(HRTS2_N_A)	FM(MSIOF1_TXD_A)	F_(0, 0)			F_(0, 0)	FM(TS_SCK0_A)	FM(STP_ISCLK_0_A)	FM(RIF0_D1_A)	FM(RIF2_D0_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_23_20	FM(SSI_SCK4)		FM(HRX2_A)	FM(MSIOF1_SCK_A)	F_(0, 0)			F_(0, 0)	FM(TS_SDAT0_A)	FM(STP_ISD_0_A)		FM(RIF0_CLK_A)	FM(RIF2_CLK_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_27_24	FM(SSI_WS4)		FM(HTX2_A)	FM(MSIOF1_SYNC_A)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN0_A)	FM(STP_ISEN_0_A)	FM(RIF0_SYNC_A)	FM(RIF2_SYNC_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP15_31_28	FM(SSI_SDATA4)		FM(HSCK2_A)	FM(MSIOF1_RXD_A)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC0_A)FM(STP_ISSYNC_0_A)	FM(RIF0_D0_A)	FM(RIF2_D1_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP16_3_0	FM(SSI_SCK6)		F_(0, 0)	F_(0, 0)		FM(SIM0_RST_D)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP16_7_4	FM(SSI_WS6)		F_(0, 0)	F_(0, 0)		FM(SIM0_D_D)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP16_11_8	FM(SSI_SDATA6)		F_(0, 0)	F_(0, 0)		FM(SIM0_CLK_D)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP16_15_12	FM(SSI_SCK78)		FM(HRX2_B)	FM(MSIOF1_SCK_C)	F_(0, 0)			F_(0, 0)	FM(TS_SCK1_A)	FM(STP_ISCLK_1_A)	FM(RIF1_CLK_A)	FM(RIF3_CLK_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP16_19_16	FM(SSI_WS78)		FM(HTX2_B)	FM(MSIOF1_SYNC_C)	F_(0, 0)			F_(0, 0)	FM(TS_SDAT1_A)	FM(STP_ISD_1_A)		FM(RIF1_SYNC_A)	FM(RIF3_SYNC_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP16_23_20	FM(SSI_SDATA7)		FM(HCTS2_N_B)	FM(MSIOF1_RXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN1_A)	FM(STP_ISEN_1_A)	FM(RIF1_D0_A)	FM(RIF3_D0_A)		F_(0, 0)	FM(TCLK2_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP16_27_24	FM(SSI_SDATA8)		FM(HRTS2_N_B)	FM(MSIOF1_TXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC1_A)FM(STP_ISSYNC_1_A)	FM(RIF1_D1_A)	FM(RIF3_D1_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP16_31_28	FM(SSI_SDATA9_A)	FM(HSCK2_B)	FM(MSIOF1_SS1_C)	FM(HSCK1_A)			FM(SSI_WS1_B)	FM(SCK1)	FM(STP_IVCXO27_1_A)	FM(SCK5_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP17_3_0	FM(AUDIO_CLKA_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(CC5_OSCOUT)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP17_7_4	FM(AUDIO_CLKB_B)	FM(SCIF_CLK_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_IVCXO27_1_D)	FM(REMOCON_A)	F_(0, 0)		F_(0, 0)	FM(TCLK1_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP17_11_8	FM(USB0_PWEN)		F_(0, 0)	F_(0, 0)		FM(SIM0_RST_C)			F_(0, 0)	FM(TS_SCK1_D)	FM(STP_ISCLK_1_D)	FM(BPFCLK_B)	FM(RIF3_CLK_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) FM(HSCK2_C) F_(0, 0) F_(0, 0)
+#define IP17_15_12	FM(USB0_OVC)		F_(0, 0)	F_(0, 0)		FM(SIM0_D_C)			F_(0, 0)	FM(TS_SDAT1_D)	FM(STP_ISD_1_D)		F_(0, 0)	FM(RIF3_SYNC_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) FM(HRX2_C) F_(0, 0) F_(0, 0)
+#define IP17_19_16	FM(USB1_PWEN)		F_(0, 0)	F_(0, 0)		FM(SIM0_CLK_C)			FM(SSI_SCK1_A)	FM(TS_SCK0_E)	FM(STP_ISCLK_0_E)	FM(FMCLK_B)	FM(RIF2_CLK_B)		F_(0, 0)	FM(SPEEDIN_A)	F_(0, 0)	F_(0, 0) FM(HTX2_C) F_(0, 0) F_(0, 0)
+#define IP17_23_20	FM(USB1_OVC)		F_(0, 0)	FM(MSIOF1_SS2_C)	F_(0, 0)			FM(SSI_WS1_A)	FM(TS_SDAT0_E)	FM(STP_ISD_0_E)		FM(FMIN_B)	FM(RIF2_SYNC_B)		F_(0, 0)	FM(REMOCON_B)	F_(0, 0)	F_(0, 0) FM(HCTS2_N_C) F_(0, 0) F_(0, 0)
+#define IP17_27_24	FM(USB30_PWEN)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT_B)		FM(SSI_SCK2_B)	FM(TS_SDEN1_D)	FM(STP_ISEN_1_D)	FM(STP_OPWM_0_E)FM(RIF3_D0_B)		F_(0, 0)	FM(TCLK2_B)	FM(TPU0TO0)	FM(BPFCLK_C) FM(HRTS2_N_C) F_(0, 0) F_(0, 0)
+#define IP17_31_28	FM(USB30_OVC)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT1_B)		FM(SSI_WS2_B)	FM(TS_SPSYNC1_D)FM(STP_ISSYNC_1_D)	FM(STP_IVCXO27_0_E)FM(RIF3_D1_B)	F_(0, 0)	FM(FSO_TOE_B)	FM(TPU0TO1)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP18_3_0	FM(GP6_30)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT2_B)		FM(SSI_SCK9_B)	FM(TS_SDEN0_E)	FM(STP_ISEN_0_E)	F_(0, 0)	FM(RIF2_D0_B)		F_(0, 0)	FM(FSO_CFE_0_A)	FM(TPU0TO2)	F_(0, 0) FM(FMCLK_C) FM(FMCLK_D) F_(0, 0)
+#define IP18_7_4	FM(GP6_31)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT3_B)		FM(SSI_WS9_B)	FM(TS_SPSYNC0_E)FM(STP_ISSYNC_0_E)	F_(0, 0)	FM(RIF2_D1_B)		F_(0, 0)	FM(FSO_CFE_1_A)	FM(TPU0TO3)	F_(0, 0) FM(FMIN_C) FM(FMIN_D) F_(0, 0)
+
+#define PINMUX_GPSR	\
+\
+												GPSR6_31 \
+												GPSR6_30 \
+												GPSR6_29 \
+		GPSR1_28									GPSR6_28 \
+		GPSR1_27									GPSR6_27 \
+		GPSR1_26									GPSR6_26 \
+		GPSR1_25							GPSR5_25	GPSR6_25 \
+		GPSR1_24							GPSR5_24	GPSR6_24 \
+		GPSR1_23							GPSR5_23	GPSR6_23 \
+		GPSR1_22							GPSR5_22	GPSR6_22 \
+		GPSR1_21							GPSR5_21	GPSR6_21 \
+		GPSR1_20							GPSR5_20	GPSR6_20 \
+		GPSR1_19							GPSR5_19	GPSR6_19 \
+		GPSR1_18							GPSR5_18	GPSR6_18 \
+		GPSR1_17					GPSR4_17	GPSR5_17	GPSR6_17 \
+		GPSR1_16					GPSR4_16	GPSR5_16	GPSR6_16 \
+GPSR0_15	GPSR1_15			GPSR3_15	GPSR4_15	GPSR5_15	GPSR6_15 \
+GPSR0_14	GPSR1_14	GPSR2_14	GPSR3_14	GPSR4_14	GPSR5_14	GPSR6_14 \
+GPSR0_13	GPSR1_13	GPSR2_13	GPSR3_13	GPSR4_13	GPSR5_13	GPSR6_13 \
+GPSR0_12	GPSR1_12	GPSR2_12	GPSR3_12	GPSR4_12	GPSR5_12	GPSR6_12 \
+GPSR0_11	GPSR1_11	GPSR2_11	GPSR3_11	GPSR4_11	GPSR5_11	GPSR6_11 \
+GPSR0_10	GPSR1_10	GPSR2_10	GPSR3_10	GPSR4_10	GPSR5_10	GPSR6_10 \
+GPSR0_9		GPSR1_9		GPSR2_9		GPSR3_9		GPSR4_9		GPSR5_9		GPSR6_9 \
+GPSR0_8		GPSR1_8		GPSR2_8		GPSR3_8		GPSR4_8		GPSR5_8		GPSR6_8 \
+GPSR0_7		GPSR1_7		GPSR2_7		GPSR3_7		GPSR4_7		GPSR5_7		GPSR6_7 \
+GPSR0_6		GPSR1_6		GPSR2_6		GPSR3_6		GPSR4_6		GPSR5_6		GPSR6_6 \
+GPSR0_5		GPSR1_5		GPSR2_5		GPSR3_5		GPSR4_5		GPSR5_5		GPSR6_5 \
+GPSR0_4		GPSR1_4		GPSR2_4		GPSR3_4		GPSR4_4		GPSR5_4		GPSR6_4 \
+GPSR0_3		GPSR1_3		GPSR2_3		GPSR3_3		GPSR4_3		GPSR5_3		GPSR6_3		GPSR7_3 \
+GPSR0_2		GPSR1_2		GPSR2_2		GPSR3_2		GPSR4_2		GPSR5_2		GPSR6_2		GPSR7_2 \
+GPSR0_1		GPSR1_1		GPSR2_1		GPSR3_1		GPSR4_1		GPSR5_1		GPSR6_1		GPSR7_1 \
+GPSR0_0		GPSR1_0		GPSR2_0		GPSR3_0		GPSR4_0		GPSR5_0		GPSR6_0		GPSR7_0
+
+#define PINMUX_IPSR				\
+\
+FM(IP0_3_0)	IP0_3_0		FM(IP1_3_0)	IP1_3_0		FM(IP2_3_0)	IP2_3_0		FM(IP3_3_0)	IP3_3_0 \
+FM(IP0_7_4)	IP0_7_4		FM(IP1_7_4)	IP1_7_4		FM(IP2_7_4)	IP2_7_4		FM(IP3_7_4)	IP3_7_4 \
+FM(IP0_11_8)	IP0_11_8	FM(IP1_11_8)	IP1_11_8	FM(IP2_11_8)	IP2_11_8	FM(IP3_11_8)	IP3_11_8 \
+FM(IP0_15_12)	IP0_15_12	FM(IP1_15_12)	IP1_15_12	FM(IP2_15_12)	IP2_15_12	FM(IP3_15_12)	IP3_15_12 \
+FM(IP0_19_16)	IP0_19_16	FM(IP1_19_16)	IP1_19_16	FM(IP2_19_16)	IP2_19_16	FM(IP3_19_16)	IP3_19_16 \
+FM(IP0_23_20)	IP0_23_20	FM(IP1_23_20)	IP1_23_20	FM(IP2_23_20)	IP2_23_20	FM(IP3_23_20)	IP3_23_20 \
+FM(IP0_27_24)	IP0_27_24	FM(IP1_27_24)	IP1_27_24	FM(IP2_27_24)	IP2_27_24	FM(IP3_27_24)	IP3_27_24 \
+FM(IP0_31_28)	IP0_31_28	FM(IP1_31_28)	IP1_31_28	FM(IP2_31_28)	IP2_31_28	FM(IP3_31_28)	IP3_31_28 \
+\
+FM(IP4_3_0)	IP4_3_0		FM(IP5_3_0)	IP5_3_0		FM(IP6_3_0)	IP6_3_0		FM(IP7_3_0)	IP7_3_0 \
+FM(IP4_7_4)	IP4_7_4		FM(IP5_7_4)	IP5_7_4		FM(IP6_7_4)	IP6_7_4		FM(IP7_7_4)	IP7_7_4 \
+FM(IP4_11_8)	IP4_11_8	FM(IP5_11_8)	IP5_11_8	FM(IP6_11_8)	IP6_11_8	FM(IP7_11_8)	IP7_11_8 \
+FM(IP4_15_12)	IP4_15_12	FM(IP5_15_12)	IP5_15_12	FM(IP6_15_12)	IP6_15_12	FM(IP7_15_12)	IP7_15_12 \
+FM(IP4_19_16)	IP4_19_16	FM(IP5_19_16)	IP5_19_16	FM(IP6_19_16)	IP6_19_16	FM(IP7_19_16)	IP7_19_16 \
+FM(IP4_23_20)	IP4_23_20	FM(IP5_23_20)	IP5_23_20	FM(IP6_23_20)	IP6_23_20	FM(IP7_23_20)	IP7_23_20 \
+FM(IP4_27_24)	IP4_27_24	FM(IP5_27_24)	IP5_27_24	FM(IP6_27_24)	IP6_27_24	FM(IP7_27_24)	IP7_27_24 \
+FM(IP4_31_28)	IP4_31_28	FM(IP5_31_28)	IP5_31_28	FM(IP6_31_28)	IP6_31_28	FM(IP7_31_28)	IP7_31_28 \
+\
+FM(IP8_3_0)	IP8_3_0		FM(IP9_3_0)	IP9_3_0		FM(IP10_3_0)	IP10_3_0	FM(IP11_3_0)	IP11_3_0 \
+FM(IP8_7_4)	IP8_7_4		FM(IP9_7_4)	IP9_7_4		FM(IP10_7_4)	IP10_7_4	FM(IP11_7_4)	IP11_7_4 \
+FM(IP8_11_8)	IP8_11_8	FM(IP9_11_8)	IP9_11_8	FM(IP10_11_8)	IP10_11_8	FM(IP11_11_8)	IP11_11_8 \
+FM(IP8_15_12)	IP8_15_12	FM(IP9_15_12)	IP9_15_12	FM(IP10_15_12)	IP10_15_12	FM(IP11_15_12)	IP11_15_12 \
+FM(IP8_19_16)	IP8_19_16	FM(IP9_19_16)	IP9_19_16	FM(IP10_19_16)	IP10_19_16	FM(IP11_19_16)	IP11_19_16 \
+FM(IP8_23_20)	IP8_23_20	FM(IP9_23_20)	IP9_23_20	FM(IP10_23_20)	IP10_23_20	FM(IP11_23_20)	IP11_23_20 \
+FM(IP8_27_24)	IP8_27_24	FM(IP9_27_24)	IP9_27_24	FM(IP10_27_24)	IP10_27_24	FM(IP11_27_24)	IP11_27_24 \
+FM(IP8_31_28)	IP8_31_28	FM(IP9_31_28)	IP9_31_28	FM(IP10_31_28)	IP10_31_28	FM(IP11_31_28)	IP11_31_28 \
+\
+FM(IP12_3_0)	IP12_3_0	FM(IP13_3_0)	IP13_3_0	FM(IP14_3_0)	IP14_3_0	FM(IP15_3_0)	IP15_3_0 \
+FM(IP12_7_4)	IP12_7_4	FM(IP13_7_4)	IP13_7_4	FM(IP14_7_4)	IP14_7_4	FM(IP15_7_4)	IP15_7_4 \
+FM(IP12_11_8)	IP12_11_8	FM(IP13_11_8)	IP13_11_8	FM(IP14_11_8)	IP14_11_8	FM(IP15_11_8)	IP15_11_8 \
+FM(IP12_15_12)	IP12_15_12	FM(IP13_15_12)	IP13_15_12	FM(IP14_15_12)	IP14_15_12	FM(IP15_15_12)	IP15_15_12 \
+FM(IP12_19_16)	IP12_19_16	FM(IP13_19_16)	IP13_19_16	FM(IP14_19_16)	IP14_19_16	FM(IP15_19_16)	IP15_19_16 \
+FM(IP12_23_20)	IP12_23_20	FM(IP13_23_20)	IP13_23_20	FM(IP14_23_20)	IP14_23_20	FM(IP15_23_20)	IP15_23_20 \
+FM(IP12_27_24)	IP12_27_24	FM(IP13_27_24)	IP13_27_24	FM(IP14_27_24)	IP14_27_24	FM(IP15_27_24)	IP15_27_24 \
+FM(IP12_31_28)	IP12_31_28	FM(IP13_31_28)	IP13_31_28	FM(IP14_31_28)	IP14_31_28	FM(IP15_31_28)	IP15_31_28 \
+\
+FM(IP16_3_0)	IP16_3_0	FM(IP17_3_0)	IP17_3_0	FM(IP18_3_0)	IP18_3_0 \
+FM(IP16_7_4)	IP16_7_4	FM(IP17_7_4)	IP17_7_4	FM(IP18_7_4)	IP18_7_4 \
+FM(IP16_11_8)	IP16_11_8	FM(IP17_11_8)	IP17_11_8 \
+FM(IP16_15_12)	IP16_15_12	FM(IP17_15_12)	IP17_15_12 \
+FM(IP16_19_16)	IP16_19_16	FM(IP17_19_16)	IP17_19_16 \
+FM(IP16_23_20)	IP16_23_20	FM(IP17_23_20)	IP17_23_20 \
+FM(IP16_27_24)	IP16_27_24	FM(IP17_27_24)	IP17_27_24 \
+FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
+
+/* MOD_SEL0 */			/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */			/* 6 */			/* 7 */
+#define MOD_SEL0_31_30_29	FM(SEL_MSIOF3_0)	FM(SEL_MSIOF3_1)	FM(SEL_MSIOF3_2)	FM(SEL_MSIOF3_3)	FM(SEL_MSIOF3_4)	F_(0, 0)		F_(0, 0)		F_(0, 0)
+#define MOD_SEL0_28_27		FM(SEL_MSIOF2_0)	FM(SEL_MSIOF2_1)	FM(SEL_MSIOF2_2)	FM(SEL_MSIOF2_3)
+#define MOD_SEL0_26_25_24	FM(SEL_MSIOF1_0)	FM(SEL_MSIOF1_1)	FM(SEL_MSIOF1_2)	FM(SEL_MSIOF1_3)	FM(SEL_MSIOF1_4)	FM(SEL_MSIOF1_5)	FM(SEL_MSIOF1_6)	F_(0, 0)
+#define MOD_SEL0_23		FM(SEL_LBSC_0)		FM(SEL_LBSC_1)
+#define MOD_SEL0_22		FM(SEL_IEBUS_0)		FM(SEL_IEBUS_1)
+#define MOD_SEL0_21		FM(SEL_I2C2_0)		FM(SEL_I2C2_1)
+#define MOD_SEL0_20		FM(SEL_I2C1_0)		FM(SEL_I2C1_1)
+#define MOD_SEL0_19		FM(SEL_HSCIF4_0)	FM(SEL_HSCIF4_1)
+#define MOD_SEL0_18_17		FM(SEL_HSCIF3_0)	FM(SEL_HSCIF3_1)	FM(SEL_HSCIF3_2)	FM(SEL_HSCIF3_3)
+#define MOD_SEL0_16		FM(SEL_HSCIF1_0)	FM(SEL_HSCIF1_1)
+#define MOD_SEL0_15		FM(SEL_FSO_0)		FM(SEL_FSO_1)
+#define MOD_SEL0_14_13		FM(SEL_HSCIF2_0)	FM(SEL_HSCIF2_1)	FM(SEL_HSCIF2_2)	F_(0, 0)
+#define MOD_SEL0_12		FM(SEL_ETHERAVB_0)	FM(SEL_ETHERAVB_1)
+#define MOD_SEL0_11		FM(SEL_DRIF3_0)		FM(SEL_DRIF3_1)
+#define MOD_SEL0_10		FM(SEL_DRIF2_0)		FM(SEL_DRIF2_1)
+#define MOD_SEL0_9_8		FM(SEL_DRIF1_0)		FM(SEL_DRIF1_1)		FM(SEL_DRIF1_2)		F_(0, 0)
+#define MOD_SEL0_7_6		FM(SEL_DRIF0_0)		FM(SEL_DRIF0_1)		FM(SEL_DRIF0_2)		F_(0, 0)
+#define MOD_SEL0_5		FM(SEL_CANFD0_0)	FM(SEL_CANFD0_1)
+#define MOD_SEL0_4_3		FM(SEL_ADG_A_0)		FM(SEL_ADG_A_1)		FM(SEL_ADG_A_2)		FM(SEL_ADG_A_3)
+#define MOD_SEL0_2		FM(SEL_5LINE_0)		FM(SEL_5LINE_1)
+
+/* MOD_SEL1 */			/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */			/* 6 */			/* 7 */
+#define MOD_SEL1_31_30		FM(SEL_TSIF1_0)		FM(SEL_TSIF1_1)		FM(SEL_TSIF1_2)		FM(SEL_TSIF1_3)
+#define MOD_SEL1_29_28_27	FM(SEL_TSIF0_0)		FM(SEL_TSIF0_1)		FM(SEL_TSIF0_2)		FM(SEL_TSIF0_3)		FM(SEL_TSIF0_4)		F_(0, 0)		F_(0, 0)		F_(0, 0)
+#define MOD_SEL1_26		FM(SEL_TIMER_TMU_0)	FM(SEL_TIMER_TMU_1)
+#define MOD_SEL1_25_24		FM(SEL_SSP1_1_0)	FM(SEL_SSP1_1_1)	FM(SEL_SSP1_1_2)	FM(SEL_SSP1_1_3)
+#define MOD_SEL1_23_22_21	FM(SEL_SSP1_0_0)	FM(SEL_SSP1_0_1)	FM(SEL_SSP1_0_2)	FM(SEL_SSP1_0_3)	FM(SEL_SSP1_0_4)	F_(0, 0)		F_(0, 0)		F_(0, 0)
+#define MOD_SEL1_20		FM(SEL_SSI_0)		FM(SEL_SSI_1)
+#define MOD_SEL1_19		FM(SEL_SPEED_PULSE_0)	FM(SEL_SPEED_PULSE_1)
+#define MOD_SEL1_18_17		FM(SEL_SIMCARD_0)	FM(SEL_SIMCARD_1)	FM(SEL_SIMCARD_2)	FM(SEL_SIMCARD_3)
+#define MOD_SEL1_16		FM(SEL_SDHI2_0)		FM(SEL_SDHI2_1)
+#define MOD_SEL1_15_14		FM(SEL_SCIF4_0)		FM(SEL_SCIF4_1)		FM(SEL_SCIF4_2)		F_(0, 0)
+#define MOD_SEL1_13		FM(SEL_SCIF3_0)		FM(SEL_SCIF3_1)
+#define MOD_SEL1_12		FM(SEL_SCIF2_0)		FM(SEL_SCIF2_1)
+#define MOD_SEL1_11		FM(SEL_SCIF1_0)		FM(SEL_SCIF1_1)
+#define MOD_SEL1_10		FM(SEL_SATA_0)		FM(SEL_SATA_1)
+#define MOD_SEL1_9		FM(SEL_REMOCON_0)	FM(SEL_REMOCON_1)
+#define MOD_SEL1_6		FM(SEL_RCAN0_0)		FM(SEL_RCAN0_1)
+#define MOD_SEL1_5		FM(SEL_PWM6_0)		FM(SEL_PWM6_1)
+#define MOD_SEL1_4		FM(SEL_PWM5_0)		FM(SEL_PWM5_1)
+#define MOD_SEL1_3		FM(SEL_PWM4_0)		FM(SEL_PWM4_1)
+#define MOD_SEL1_2		FM(SEL_PWM3_0)		FM(SEL_PWM3_1)
+#define MOD_SEL1_1		FM(SEL_PWM2_0)		FM(SEL_PWM2_1)
+#define MOD_SEL1_0		FM(SEL_PWM1_0)		FM(SEL_PWM1_1)
+
+/* MOD_SEL1 */			/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */			/* 6 */			/* 7 */
+#define MOD_SEL2_31		FM(I2C_SEL_5_0)		FM(I2C_SEL_5_1)
+#define MOD_SEL2_30		FM(I2C_SEL_3_0)		FM(I2C_SEL_3_1)
+#define MOD_SEL2_29		FM(I2C_SEL_0_0)		FM(I2C_SEL_0_1)
+#define MOD_SEL2_28_27		FM(SEL_FM_0)		FM(SEL_FM_1)		FM(SEL_FM_2)		FM(SEL_FM_3)
+#define MOD_SEL2_26		FM(SEL_SCIF5_0)		FM(SEL_SCIF5_1)
+#define MOD_SEL2_25_24_23	FM(SEL_I2C6_0)		FM(SEL_I2C6_1)		FM(SEL_I2C6_2)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)
+#define MOD_SEL2_22		FM(SEL_NDF_0)		FM(SEL_NDF_1)
+#define MOD_SEL2_21		FM(SEL_SSI2_0)		FM(SEL_SSI2_1)
+#define MOD_SEL2_20		FM(SEL_SSI9_0)		FM(SEL_SSI9_1)
+#define MOD_SEL2_19		FM(SEL_TIMER_TMU2_0)	FM(SEL_TIMER_TMU2_1)
+#define MOD_SEL2_18		FM(SEL_ADG_B_0)		FM(SEL_ADG_B_1)
+#define MOD_SEL2_17		FM(SEL_ADG_C_0)		FM(SEL_ADG_C_1)
+#define MOD_SEL2_0		FM(SEL_VIN4_0)		FM(SEL_VIN4_1)
+
+#define PINMUX_MOD_SELS	\
+\
+MOD_SEL0_31_30_29	MOD_SEL1_31_30		MOD_SEL2_31 \
+						MOD_SEL2_30 \
+			MOD_SEL1_29_28_27	MOD_SEL2_29 \
+MOD_SEL0_28_27					MOD_SEL2_28_27 \
+MOD_SEL0_26_25_24	MOD_SEL1_26		MOD_SEL2_26 \
+			MOD_SEL1_25_24		MOD_SEL2_25_24_23 \
+MOD_SEL0_23		MOD_SEL1_23_22_21 \
+MOD_SEL0_22					MOD_SEL2_22 \
+MOD_SEL0_21					MOD_SEL2_21 \
+MOD_SEL0_20		MOD_SEL1_20		MOD_SEL2_20 \
+MOD_SEL0_19		MOD_SEL1_19		MOD_SEL2_19 \
+MOD_SEL0_18_17		MOD_SEL1_18_17		MOD_SEL2_18 \
+						MOD_SEL2_17 \
+MOD_SEL0_16		MOD_SEL1_16 \
+MOD_SEL0_15		MOD_SEL1_15_14 \
+MOD_SEL0_14_13 \
+			MOD_SEL1_13 \
+MOD_SEL0_12		MOD_SEL1_12 \
+MOD_SEL0_11		MOD_SEL1_11 \
+MOD_SEL0_10		MOD_SEL1_10 \
+MOD_SEL0_9_8		MOD_SEL1_9 \
+MOD_SEL0_7_6 \
+			MOD_SEL1_6 \
+MOD_SEL0_5		MOD_SEL1_5 \
+MOD_SEL0_4_3		MOD_SEL1_4 \
+			MOD_SEL1_3 \
+MOD_SEL0_2		MOD_SEL1_2 \
+			MOD_SEL1_1 \
+			MOD_SEL1_0		MOD_SEL2_0
+
+enum {
+	PINMUX_RESERVED = 0,
+
+	PINMUX_DATA_BEGIN,
+	GP_ALL(DATA),
+	PINMUX_DATA_END,
+
+#define F_(x, y)
+#define FM(x)	FN_##x,
+	PINMUX_FUNCTION_BEGIN,
+	GP_ALL(FN),
+	PINMUX_GPSR
+	PINMUX_IPSR
+	PINMUX_MOD_SELS
+	PINMUX_FUNCTION_END,
+#undef F_
+#undef FM
+
+#define F_(x, y)
+#define FM(x)	x##_MARK,
+	PINMUX_MARK_BEGIN,
+	PINMUX_GPSR
+	PINMUX_IPSR
+	PINMUX_MOD_SELS
+	PINMUX_MARK_END,
+#undef F_
+#undef FM
+};
+
+static const u16 pinmux_data[] = {
+	PINMUX_DATA_GP_ALL(),
+
+	PINMUX_SINGLE(AVS1),
+	PINMUX_SINGLE(AVS2),
+	PINMUX_SINGLE(CLKOUT),
+	PINMUX_SINGLE(GP7_03),
+	PINMUX_SINGLE(HDMI0_CEC),
+	PINMUX_SINGLE(MSIOF0_RXD),
+	PINMUX_SINGLE(MSIOF0_SCK),
+	PINMUX_SINGLE(MSIOF0_TXD),
+	PINMUX_SINGLE(SSI_SCK5),
+	PINMUX_SINGLE(SSI_SDATA5),
+	PINMUX_SINGLE(SSI_WS5),
+
+	/* IPSR0 */
+	PINMUX_IPSR_GPSR(IP0_3_0,	AVB_MDC),
+	PINMUX_IPSR_MSEL(IP0_3_0,	MSIOF2_SS2_C,		SEL_MSIOF2_2),
+
+	PINMUX_IPSR_GPSR(IP0_7_4,	AVB_MAGIC),
+	PINMUX_IPSR_MSEL(IP0_7_4,	MSIOF2_SS1_C,		SEL_MSIOF2_2),
+	PINMUX_IPSR_MSEL(IP0_7_4,	SCK4_A,			SEL_SCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP0_11_8,	AVB_PHY_INT),
+	PINMUX_IPSR_MSEL(IP0_11_8,	MSIOF2_SYNC_C,		SEL_MSIOF2_2),
+	PINMUX_IPSR_MSEL(IP0_11_8,	RX4_A,			SEL_SCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP0_15_12,	AVB_LINK),
+	PINMUX_IPSR_MSEL(IP0_15_12,	MSIOF2_SCK_C,		SEL_MSIOF2_2),
+	PINMUX_IPSR_MSEL(IP0_15_12,	TX4_A,			SEL_SCIF4_0),
+
+	PINMUX_IPSR_MSEL(IP0_19_16,	AVB_AVTP_MATCH_A,	SEL_ETHERAVB_0),
+	PINMUX_IPSR_MSEL(IP0_19_16,	MSIOF2_RXD_C,		SEL_MSIOF2_2),
+	PINMUX_IPSR_MSEL(IP0_19_16,	CTS4_N_A,		SEL_SCIF4_0),
+
+	PINMUX_IPSR_MSEL(IP0_23_20,	AVB_AVTP_CAPTURE_A,	SEL_ETHERAVB_0),
+	PINMUX_IPSR_MSEL(IP0_23_20,	MSIOF2_TXD_C,		SEL_MSIOF2_2),
+	PINMUX_IPSR_MSEL(IP0_23_20,	RTS4_N_TANS_A,		SEL_SCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP0_27_24,	IRQ0),
+	PINMUX_IPSR_GPSR(IP0_27_24,	QPOLB),
+	PINMUX_IPSR_GPSR(IP0_27_24,	DU_CDE),
+	PINMUX_IPSR_MSEL(IP0_27_24,	VI4_DATA0_B,		SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP0_27_24,	CAN0_TX_B,		SEL_RCAN0_1),
+	PINMUX_IPSR_MSEL(IP0_27_24,	CANFD0_TX_B,		SEL_CANFD0_1),
+	PINMUX_IPSR_MSEL(IP0_27_24,	MSIOF3_SS2_E,		SEL_MSIOF3_4),
+
+	PINMUX_IPSR_GPSR(IP0_31_28,	IRQ1),
+	PINMUX_IPSR_GPSR(IP0_31_28,	QPOLA),
+	PINMUX_IPSR_GPSR(IP0_31_28,	DU_DISP),
+	PINMUX_IPSR_MSEL(IP0_31_28,	VI4_DATA1_B,		SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP0_31_28,	CAN0_RX_B,		SEL_RCAN0_1),
+	PINMUX_IPSR_MSEL(IP0_31_28,	CANFD0_RX_B,		SEL_CANFD0_1),
+	PINMUX_IPSR_MSEL(IP0_27_24,	MSIOF3_SS1_E,		SEL_MSIOF3_4),
+
+	/* IPSR1 */
+	PINMUX_IPSR_GPSR(IP1_3_0,	IRQ2),
+	PINMUX_IPSR_GPSR(IP1_3_0,	QCPV_QDE),
+	PINMUX_IPSR_GPSR(IP1_3_0,	DU_EXODDF_DU_ODDF_DISP_CDE),
+	PINMUX_IPSR_MSEL(IP1_3_0,	VI4_DATA2_B,		SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP1_3_0,	PWM3_B,			SEL_PWM3_1),
+	PINMUX_IPSR_MSEL(IP1_3_0,	MSIOF3_SYNC_E,		SEL_MSIOF3_4),
+
+	PINMUX_IPSR_GPSR(IP1_7_4,	IRQ3),
+	PINMUX_IPSR_GPSR(IP1_7_4,	QSTVB_QVE),
+	PINMUX_IPSR_GPSR(IP1_7_4,	A25),
+	PINMUX_IPSR_GPSR(IP1_7_4,	DU_DOTCLKOUT1),
+	PINMUX_IPSR_MSEL(IP1_7_4,	VI4_DATA3_B,		SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP1_7_4,	PWM4_B,			SEL_PWM4_1),
+	PINMUX_IPSR_MSEL(IP1_7_4,	MSIOF3_SCK_E,		SEL_MSIOF3_4),
+
+	PINMUX_IPSR_GPSR(IP1_11_8,	IRQ4),
+	PINMUX_IPSR_GPSR(IP1_11_8,	QSTH_QHS),
+	PINMUX_IPSR_GPSR(IP1_11_8,	A24),
+	PINMUX_IPSR_GPSR(IP1_11_8,	DU_EXHSYNC_DU_HSYNC),
+	PINMUX_IPSR_MSEL(IP1_11_8,	VI4_DATA4_B,		SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP1_11_8,	PWM5_B,			SEL_PWM5_1),
+	PINMUX_IPSR_MSEL(IP1_11_8,	MSIOF3_RXD_E,		SEL_MSIOF3_4),
+
+	PINMUX_IPSR_GPSR(IP1_15_12,	IRQ5),
+	PINMUX_IPSR_GPSR(IP1_15_12,	QSTB_QHE),
+	PINMUX_IPSR_GPSR(IP1_15_12,	A23),
+	PINMUX_IPSR_GPSR(IP1_15_12,	DU_EXVSYNC_DU_VSYNC),
+	PINMUX_IPSR_MSEL(IP1_15_12,	VI4_DATA5_B,		SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP1_15_12,	PWM6_B,			SEL_PWM6_1),
+	PINMUX_IPSR_MSEL(IP1_15_12,	MSIOF3_TXD_E,		SEL_MSIOF3_4),
+
+	PINMUX_IPSR_GPSR(IP1_19_16,	PWM0),
+	PINMUX_IPSR_GPSR(IP1_19_16,	AVB_AVTP_PPS),
+	PINMUX_IPSR_GPSR(IP1_19_16,	A22),
+	PINMUX_IPSR_MSEL(IP1_19_16,	VI4_DATA6_B,		SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP1_19_16,	IECLK_B,		SEL_IEBUS_1),
+
+	PINMUX_IPSR_MSEL(IP1_23_20,	PWM1_A,			SEL_PWM1_0),
+	PINMUX_IPSR_GPSR(IP1_23_20,	A21),
+	PINMUX_IPSR_MSEL(IP1_23_20,	HRX3_D,			SEL_HSCIF3_3),
+	PINMUX_IPSR_MSEL(IP1_23_20,	VI4_DATA7_B,		SEL_VIN4_1),
+	PINMUX_IPSR_MSEL(IP1_23_20,	IERX_B,			SEL_IEBUS_1),
+
+	PINMUX_IPSR_MSEL(IP1_27_24,	PWM2_A,			SEL_PWM2_0),
+	PINMUX_IPSR_GPSR(IP1_27_24,	A20),
+	PINMUX_IPSR_MSEL(IP1_27_24,	HTX3_D,			SEL_HSCIF3_3),
+	PINMUX_IPSR_MSEL(IP1_27_24,	IETX_B,			SEL_IEBUS_1),
+
+	PINMUX_IPSR_GPSR(IP1_31_28,	A0),
+	PINMUX_IPSR_GPSR(IP1_31_28,	LCDOUT16),
+	PINMUX_IPSR_MSEL(IP1_31_28,	MSIOF3_SYNC_B,		SEL_MSIOF3_1),
+	PINMUX_IPSR_GPSR(IP1_31_28,	VI4_DATA8),
+	PINMUX_IPSR_GPSR(IP1_31_28,	DU_DB0),
+	PINMUX_IPSR_MSEL(IP1_31_28,	PWM3_A,			SEL_PWM3_0),
+
+	/* IPSR2 */
+	PINMUX_IPSR_GPSR(IP2_3_0,	A1),
+	PINMUX_IPSR_GPSR(IP2_3_0,	LCDOUT17),
+	PINMUX_IPSR_MSEL(IP2_3_0,	MSIOF3_TXD_B,		SEL_MSIOF3_1),
+	PINMUX_IPSR_GPSR(IP2_3_0,	VI4_DATA9),
+	PINMUX_IPSR_GPSR(IP2_3_0,	DU_DB1),
+	PINMUX_IPSR_MSEL(IP2_3_0,	PWM4_A,			SEL_PWM4_0),
+
+	PINMUX_IPSR_GPSR(IP2_7_4,	A2),
+	PINMUX_IPSR_GPSR(IP2_7_4,	LCDOUT18),
+	PINMUX_IPSR_MSEL(IP2_7_4,	MSIOF3_SCK_B,		SEL_MSIOF3_1),
+	PINMUX_IPSR_GPSR(IP2_7_4,	VI4_DATA10),
+	PINMUX_IPSR_GPSR(IP2_7_4,	DU_DB2),
+	PINMUX_IPSR_MSEL(IP2_7_4,	PWM5_A,			SEL_PWM5_0),
+
+	PINMUX_IPSR_GPSR(IP2_11_8,	A3),
+	PINMUX_IPSR_GPSR(IP2_11_8,	LCDOUT19),
+	PINMUX_IPSR_MSEL(IP2_11_8,	MSIOF3_RXD_B,		SEL_MSIOF3_1),
+	PINMUX_IPSR_GPSR(IP2_11_8,	VI4_DATA11),
+	PINMUX_IPSR_GPSR(IP2_11_8,	DU_DB3),
+	PINMUX_IPSR_MSEL(IP2_11_8,	PWM6_A,			SEL_PWM6_0),
+
+	PINMUX_IPSR_GPSR(IP2_15_12,	A4),
+	PINMUX_IPSR_GPSR(IP2_15_12,	LCDOUT20),
+	PINMUX_IPSR_MSEL(IP2_15_12,	MSIOF3_SS1_B,		SEL_MSIOF3_1),
+	PINMUX_IPSR_GPSR(IP2_15_12,	VI4_DATA12),
+	PINMUX_IPSR_GPSR(IP2_15_12,	VI5_DATA12),
+	PINMUX_IPSR_GPSR(IP2_15_12,	DU_DB4),
+
+	PINMUX_IPSR_GPSR(IP2_19_16,	A5),
+	PINMUX_IPSR_GPSR(IP2_19_16,	LCDOUT21),
+	PINMUX_IPSR_MSEL(IP2_19_16,	MSIOF3_SS2_B,		SEL_MSIOF3_1),
+	PINMUX_IPSR_MSEL(IP2_19_16,	SCK4_B,			SEL_SCIF4_1),
+	PINMUX_IPSR_GPSR(IP2_19_16,	VI4_DATA13),
+	PINMUX_IPSR_GPSR(IP2_19_16,	VI5_DATA13),
+	PINMUX_IPSR_GPSR(IP2_19_16,	DU_DB5),
+
+	PINMUX_IPSR_GPSR(IP2_23_20,	A6),
+	PINMUX_IPSR_GPSR(IP2_23_20,	LCDOUT22),
+	PINMUX_IPSR_MSEL(IP2_23_20,	MSIOF2_SS1_A,		SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP2_23_20,	RX4_B,			SEL_SCIF4_1),
+	PINMUX_IPSR_GPSR(IP2_23_20,	VI4_DATA14),
+	PINMUX_IPSR_GPSR(IP2_23_20,	VI5_DATA14),
+	PINMUX_IPSR_GPSR(IP2_23_20,	DU_DB6),
+
+	PINMUX_IPSR_GPSR(IP2_27_24,	A7),
+	PINMUX_IPSR_GPSR(IP2_27_24,	LCDOUT23),
+	PINMUX_IPSR_MSEL(IP2_27_24,	MSIOF2_SS2_A,		SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP2_27_24,	TX4_B,			SEL_SCIF4_1),
+	PINMUX_IPSR_GPSR(IP2_27_24,	VI4_DATA15),
+	PINMUX_IPSR_GPSR(IP2_27_24,	VI5_DATA15),
+	PINMUX_IPSR_GPSR(IP2_27_24,	DU_DB7),
+
+	PINMUX_IPSR_GPSR(IP2_31_28,	A8),
+	PINMUX_IPSR_MSEL(IP2_31_28,	RX3_B,			SEL_SCIF3_1),
+	PINMUX_IPSR_MSEL(IP2_31_28,	MSIOF2_SYNC_A,		SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP2_31_28,	HRX4_B,			SEL_HSCIF4_1),
+	PINMUX_IPSR_MSEL(IP2_31_28,	SDA6_A,			SEL_I2C6_0),
+	PINMUX_IPSR_MSEL(IP2_31_28,	AVB_AVTP_MATCH_B,	SEL_ETHERAVB_1),
+	PINMUX_IPSR_MSEL(IP2_31_28,	PWM1_B,			SEL_PWM1_1),
+
+	/* IPSR3 */
+	PINMUX_IPSR_GPSR(IP3_3_0,	A9),
+	PINMUX_IPSR_MSEL(IP3_3_0,	MSIOF2_SCK_A,		SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP3_3_0,	CTS4_N_B,		SEL_SCIF4_1),
+	PINMUX_IPSR_GPSR(IP3_3_0,	VI5_VSYNC_N),
+
+	PINMUX_IPSR_GPSR(IP3_7_4,	A10),
+	PINMUX_IPSR_MSEL(IP3_7_4,	MSIOF2_RXD_A,		SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP3_7_4,	RTS4_N_TANS_B,		SEL_SCIF4_1),
+	PINMUX_IPSR_GPSR(IP3_7_4,	VI5_HSYNC_N),
+
+	PINMUX_IPSR_GPSR(IP3_11_8,	A11),
+	PINMUX_IPSR_MSEL(IP3_11_8,	TX3_B,			SEL_SCIF3_1),
+	PINMUX_IPSR_MSEL(IP3_11_8,	MSIOF2_TXD_A,		SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP3_11_8,	HTX4_B,			SEL_HSCIF4_1),
+	PINMUX_IPSR_GPSR(IP3_11_8,	HSCK4),
+	PINMUX_IPSR_GPSR(IP3_11_8,	VI5_FIELD),
+	PINMUX_IPSR_MSEL(IP3_11_8,	SCL6_A,			SEL_I2C6_0),
+	PINMUX_IPSR_MSEL(IP3_11_8,	AVB_AVTP_CAPTURE_B,	SEL_ETHERAVB_1),
+	PINMUX_IPSR_MSEL(IP3_11_8,	PWM2_B,			SEL_PWM2_1),
+
+	PINMUX_IPSR_GPSR(IP3_15_12,	A12),
+	PINMUX_IPSR_GPSR(IP3_15_12,	LCDOUT12),
+	PINMUX_IPSR_MSEL(IP3_15_12,	MSIOF3_SCK_C,		SEL_MSIOF3_2),
+	PINMUX_IPSR_MSEL(IP3_15_12,	HRX4_A,			SEL_HSCIF4_0),
+	PINMUX_IPSR_GPSR(IP3_15_12,	VI5_DATA8),
+	PINMUX_IPSR_GPSR(IP3_15_12,	DU_DG4),
+
+	PINMUX_IPSR_GPSR(IP3_19_16,	A13),
+	PINMUX_IPSR_GPSR(IP3_19_16,	LCDOUT13),
+	PINMUX_IPSR_MSEL(IP3_19_16,	MSIOF3_SYNC_C,		SEL_MSIOF3_2),
+	PINMUX_IPSR_MSEL(IP3_19_16,	HTX4_A,			SEL_HSCIF4_0),
+	PINMUX_IPSR_GPSR(IP3_19_16,	VI5_DATA9),
+	PINMUX_IPSR_GPSR(IP3_19_16,	DU_DG5),
+
+	PINMUX_IPSR_GPSR(IP3_23_20,	A14),
+	PINMUX_IPSR_GPSR(IP3_23_20,	LCDOUT14),
+	PINMUX_IPSR_MSEL(IP3_23_20,	MSIOF3_RXD_C,		SEL_MSIOF3_2),
+	PINMUX_IPSR_GPSR(IP3_23_20,	HCTS4_N),
+	PINMUX_IPSR_GPSR(IP3_23_20,	VI5_DATA10),
+	PINMUX_IPSR_GPSR(IP3_23_20,	DU_DG6),
+
+	PINMUX_IPSR_GPSR(IP3_27_24,	A15),
+	PINMUX_IPSR_GPSR(IP3_27_24,	LCDOUT15),
+	PINMUX_IPSR_MSEL(IP3_27_24,	MSIOF3_TXD_C,		SEL_MSIOF3_2),
+	PINMUX_IPSR_GPSR(IP3_27_24,	HRTS4_N),
+	PINMUX_IPSR_GPSR(IP3_27_24,	VI5_DATA11),
+	PINMUX_IPSR_GPSR(IP3_27_24,	DU_DG7),
+
+	PINMUX_IPSR_GPSR(IP3_31_28,	A16),
+	PINMUX_IPSR_GPSR(IP3_31_28,	LCDOUT8),
+	PINMUX_IPSR_GPSR(IP3_31_28,	VI4_FIELD),
+	PINMUX_IPSR_GPSR(IP3_31_28,	DU_DG0),
+
+	/* IPSR4 */
+	PINMUX_IPSR_GPSR(IP4_3_0,	A17),
+	PINMUX_IPSR_GPSR(IP4_3_0,	LCDOUT9),
+	PINMUX_IPSR_GPSR(IP4_3_0,	VI4_VSYNC_N),
+	PINMUX_IPSR_GPSR(IP4_3_0,	DU_DG1),
+
+	PINMUX_IPSR_GPSR(IP4_7_4,	A18),
+	PINMUX_IPSR_GPSR(IP4_7_4,	LCDOUT10),
+	PINMUX_IPSR_GPSR(IP4_7_4,	VI4_HSYNC_N),
+	PINMUX_IPSR_GPSR(IP4_7_4,	DU_DG2),
+
+	PINMUX_IPSR_GPSR(IP4_11_8,	A19),
+	PINMUX_IPSR_GPSR(IP4_11_8,	LCDOUT11),
+	PINMUX_IPSR_GPSR(IP4_11_8,	VI4_CLKENB),
+	PINMUX_IPSR_GPSR(IP4_11_8,	DU_DG3),
+
+	PINMUX_IPSR_GPSR(IP4_15_12,	CS0_N),
+	PINMUX_IPSR_GPSR(IP4_15_12,	VI5_CLKENB),
+
+	PINMUX_IPSR_GPSR(IP4_19_16,	CS1_N_A26),
+	PINMUX_IPSR_GPSR(IP4_19_16,	VI5_CLK),
+	PINMUX_IPSR_MSEL(IP4_19_16,	EX_WAIT0_B,		SEL_LBSC_1),
+
+	PINMUX_IPSR_GPSR(IP4_23_20,	BS_N),
+	PINMUX_IPSR_GPSR(IP4_23_20,	QSTVA_QVS),
+	PINMUX_IPSR_MSEL(IP4_23_20,	MSIOF3_SCK_D,		SEL_MSIOF3_3),
+	PINMUX_IPSR_GPSR(IP4_23_20,	SCK3),
+	PINMUX_IPSR_GPSR(IP4_23_20,	HSCK3),
+	PINMUX_IPSR_GPSR(IP4_23_20,	CAN1_TX),
+	PINMUX_IPSR_GPSR(IP4_23_20,	CANFD1_TX),
+	PINMUX_IPSR_MSEL(IP4_23_20,	IETX_A,			SEL_IEBUS_0),
+
+	PINMUX_IPSR_GPSR(IP4_27_24,	RD_N),
+	PINMUX_IPSR_MSEL(IP4_27_24,	MSIOF3_SYNC_D,		SEL_MSIOF3_3),
+	PINMUX_IPSR_MSEL(IP4_27_24,	RX3_A,			SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP4_27_24,	HRX3_A,			SEL_HSCIF3_0),
+	PINMUX_IPSR_MSEL(IP4_27_24,	CAN0_TX_A,		SEL_RCAN0_0),
+	PINMUX_IPSR_MSEL(IP4_27_24,	CANFD0_TX_A,		SEL_CANFD0_0),
+
+	PINMUX_IPSR_GPSR(IP4_31_28,	RD_WR_N),
+	PINMUX_IPSR_MSEL(IP4_31_28,	MSIOF3_RXD_D,		SEL_MSIOF3_3),
+	PINMUX_IPSR_MSEL(IP4_31_28,	TX3_A,			SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP4_31_28,	HTX3_A,			SEL_HSCIF3_0),
+	PINMUX_IPSR_MSEL(IP4_31_28,	CAN0_RX_A,		SEL_RCAN0_0),
+	PINMUX_IPSR_MSEL(IP4_31_28,	CANFD0_RX_A,		SEL_CANFD0_0),
+
+	/* IPSR5 */
+	PINMUX_IPSR_GPSR(IP5_3_0,	WE0_N),
+	PINMUX_IPSR_MSEL(IP5_3_0,	MSIOF3_TXD_D,		SEL_MSIOF3_3),
+	PINMUX_IPSR_GPSR(IP5_3_0,	CTS3_N),
+	PINMUX_IPSR_GPSR(IP5_3_0,	HCTS3_N),
+	PINMUX_IPSR_MSEL(IP5_3_0,	SCL6_B,			SEL_I2C6_1),
+	PINMUX_IPSR_GPSR(IP5_3_0,	CAN_CLK),
+	PINMUX_IPSR_MSEL(IP5_3_0,	IECLK_A,		SEL_IEBUS_0),
+
+	PINMUX_IPSR_GPSR(IP5_7_4,	WE1_N),
+	PINMUX_IPSR_MSEL(IP5_7_4,	MSIOF3_SS1_D,		SEL_MSIOF3_3),
+	PINMUX_IPSR_GPSR(IP5_7_4,	RTS3_N_TANS),
+	PINMUX_IPSR_GPSR(IP5_7_4,	HRTS3_N),
+	PINMUX_IPSR_MSEL(IP5_7_4,	SDA6_B,			SEL_I2C6_1),
+	PINMUX_IPSR_GPSR(IP5_7_4,	CAN1_RX),
+	PINMUX_IPSR_GPSR(IP5_7_4,	CANFD1_RX),
+	PINMUX_IPSR_MSEL(IP5_7_4,	IERX_A,			SEL_IEBUS_0),
+
+	PINMUX_IPSR_MSEL(IP5_11_8,	EX_WAIT0_A,		SEL_LBSC_0),
+	PINMUX_IPSR_GPSR(IP5_11_8,	QCLK),
+	PINMUX_IPSR_GPSR(IP5_11_8,	VI4_CLK),
+	PINMUX_IPSR_GPSR(IP5_11_8,	DU_DOTCLKOUT0),
+
+	PINMUX_IPSR_GPSR(IP5_15_12,	D0),
+	PINMUX_IPSR_MSEL(IP5_15_12,	MSIOF2_SS1_B,		SEL_MSIOF2_1),
+	PINMUX_IPSR_MSEL(IP5_15_12,	MSIOF3_SCK_A,		SEL_MSIOF3_0),
+	PINMUX_IPSR_GPSR(IP5_15_12,	VI4_DATA16),
+	PINMUX_IPSR_GPSR(IP5_15_12,	VI5_DATA0),
+
+	PINMUX_IPSR_GPSR(IP5_19_16,	D1),
+	PINMUX_IPSR_MSEL(IP5_19_16,	MSIOF2_SS2_B,		SEL_MSIOF2_1),
+	PINMUX_IPSR_MSEL(IP5_19_16,	MSIOF3_SYNC_A,		SEL_MSIOF3_0),
+	PINMUX_IPSR_GPSR(IP5_19_16,	VI4_DATA17),
+	PINMUX_IPSR_GPSR(IP5_19_16,	VI5_DATA1),
+
+	PINMUX_IPSR_GPSR(IP5_23_20,	D2),
+	PINMUX_IPSR_MSEL(IP5_23_20,	MSIOF3_RXD_A,		SEL_MSIOF3_0),
+	PINMUX_IPSR_GPSR(IP5_23_20,	VI4_DATA18),
+	PINMUX_IPSR_GPSR(IP5_23_20,	VI5_DATA2),
+
+	PINMUX_IPSR_GPSR(IP5_27_24,	D3),
+	PINMUX_IPSR_MSEL(IP5_27_24,	MSIOF3_TXD_A,		SEL_MSIOF3_0),
+	PINMUX_IPSR_GPSR(IP5_27_24,	VI4_DATA19),
+	PINMUX_IPSR_GPSR(IP5_27_24,	VI5_DATA3),
+
+	PINMUX_IPSR_GPSR(IP5_31_28,	D4),
+	PINMUX_IPSR_MSEL(IP5_31_28,	MSIOF2_SCK_B,		SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP5_31_28,	VI4_DATA20),
+	PINMUX_IPSR_GPSR(IP5_31_28,	VI5_DATA4),
+
+	/* IPSR6 */
+	PINMUX_IPSR_GPSR(IP6_3_0,	D5),
+	PINMUX_IPSR_MSEL(IP6_3_0,	MSIOF2_SYNC_B,		SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP6_3_0,	VI4_DATA21),
+	PINMUX_IPSR_GPSR(IP6_3_0,	VI5_DATA5),
+
+	PINMUX_IPSR_GPSR(IP6_7_4,	D6),
+	PINMUX_IPSR_MSEL(IP6_7_4,	MSIOF2_RXD_B,		SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP6_7_4,	VI4_DATA22),
+	PINMUX_IPSR_GPSR(IP6_7_4,	VI5_DATA6),
+
+	PINMUX_IPSR_GPSR(IP6_11_8,	D7),
+	PINMUX_IPSR_MSEL(IP6_11_8,	MSIOF2_TXD_B,		SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP6_11_8,	VI4_DATA23),
+	PINMUX_IPSR_GPSR(IP6_11_8,	VI5_DATA7),
+
+	PINMUX_IPSR_GPSR(IP6_15_12,	D8),
+	PINMUX_IPSR_GPSR(IP6_15_12,	LCDOUT0),
+	PINMUX_IPSR_MSEL(IP6_15_12,	MSIOF2_SCK_D,		SEL_MSIOF2_3),
+	PINMUX_IPSR_MSEL(IP6_15_12,	SCK4_C,			SEL_SCIF4_2),
+	PINMUX_IPSR_MSEL(IP6_15_12,	VI4_DATA0_A,		SEL_VIN4_0),
+	PINMUX_IPSR_GPSR(IP6_15_12,	DU_DR0),
+
+	PINMUX_IPSR_GPSR(IP6_19_16,	D9),
+	PINMUX_IPSR_GPSR(IP6_19_16,	LCDOUT1),
+	PINMUX_IPSR_MSEL(IP6_19_16,	MSIOF2_SYNC_D,		SEL_MSIOF2_3),
+	PINMUX_IPSR_MSEL(IP6_19_16,	VI4_DATA1_A,		SEL_VIN4_0),
+	PINMUX_IPSR_GPSR(IP6_19_16,	DU_DR1),
+
+	PINMUX_IPSR_GPSR(IP6_23_20,	D10),
+	PINMUX_IPSR_GPSR(IP6_23_20,	LCDOUT2),
+	PINMUX_IPSR_MSEL(IP6_23_20,	MSIOF2_RXD_D,		SEL_MSIOF2_3),
+	PINMUX_IPSR_MSEL(IP6_23_20,	HRX3_B,			SEL_HSCIF3_1),
+	PINMUX_IPSR_MSEL(IP6_23_20,	VI4_DATA2_A,		SEL_VIN4_0),
+	PINMUX_IPSR_MSEL(IP6_23_20,	CTS4_N_C,		SEL_SCIF4_2),
+	PINMUX_IPSR_GPSR(IP6_23_20,	DU_DR2),
+
+	PINMUX_IPSR_GPSR(IP6_27_24,	D11),
+	PINMUX_IPSR_GPSR(IP6_27_24,	LCDOUT3),
+	PINMUX_IPSR_MSEL(IP6_27_24,	MSIOF2_TXD_D,		SEL_MSIOF2_3),
+	PINMUX_IPSR_MSEL(IP6_27_24,	HTX3_B,			SEL_HSCIF3_1),
+	PINMUX_IPSR_MSEL(IP6_27_24,	VI4_DATA3_A,		SEL_VIN4_0),
+	PINMUX_IPSR_MSEL(IP6_27_24,	RTS4_N_TANS_C,		SEL_SCIF4_2),
+	PINMUX_IPSR_GPSR(IP6_27_24,	DU_DR3),
+
+	PINMUX_IPSR_GPSR(IP6_31_28,	D12),
+	PINMUX_IPSR_GPSR(IP6_31_28,	LCDOUT4),
+	PINMUX_IPSR_MSEL(IP6_31_28,	MSIOF2_SS1_D,		SEL_MSIOF2_3),
+	PINMUX_IPSR_MSEL(IP6_31_28,	RX4_C,			SEL_SCIF4_2),
+	PINMUX_IPSR_MSEL(IP6_31_28,	VI4_DATA4_A,		SEL_VIN4_0),
+	PINMUX_IPSR_GPSR(IP6_31_28,	DU_DR4),
+
+	/* IPSR7 */
+	PINMUX_IPSR_GPSR(IP7_3_0,	D13),
+	PINMUX_IPSR_GPSR(IP7_3_0,	LCDOUT5),
+	PINMUX_IPSR_MSEL(IP7_3_0,	MSIOF2_SS2_D,		SEL_MSIOF2_3),
+	PINMUX_IPSR_MSEL(IP7_3_0,	TX4_C,			SEL_SCIF4_2),
+	PINMUX_IPSR_MSEL(IP7_3_0,	VI4_DATA5_A,		SEL_VIN4_0),
+	PINMUX_IPSR_GPSR(IP7_3_0,	DU_DR5),
+
+	PINMUX_IPSR_GPSR(IP7_7_4,	D14),
+	PINMUX_IPSR_GPSR(IP7_7_4,	LCDOUT6),
+	PINMUX_IPSR_MSEL(IP7_7_4,	MSIOF3_SS1_A,		SEL_MSIOF3_0),
+	PINMUX_IPSR_MSEL(IP7_7_4,	HRX3_C,			SEL_HSCIF3_2),
+	PINMUX_IPSR_MSEL(IP7_7_4,	VI4_DATA6_A,		SEL_VIN4_0),
+	PINMUX_IPSR_GPSR(IP7_7_4,	DU_DR6),
+	PINMUX_IPSR_MSEL(IP7_7_4,	SCL6_C,			SEL_I2C6_2),
+
+	PINMUX_IPSR_GPSR(IP7_11_8,	D15),
+	PINMUX_IPSR_GPSR(IP7_11_8,	LCDOUT7),
+	PINMUX_IPSR_MSEL(IP7_11_8,	MSIOF3_SS2_A,		SEL_MSIOF3_0),
+	PINMUX_IPSR_MSEL(IP7_11_8,	HTX3_C,			SEL_HSCIF3_2),
+	PINMUX_IPSR_MSEL(IP7_11_8,	VI4_DATA7_A,		SEL_VIN4_0),
+	PINMUX_IPSR_GPSR(IP7_11_8,	DU_DR7),
+	PINMUX_IPSR_MSEL(IP7_11_8,	SDA6_C,			SEL_I2C6_2),
+
+	PINMUX_IPSR_GPSR(IP7_15_12,	FSCLKST),
+
+	PINMUX_IPSR_GPSR(IP7_19_16,	SD0_CLK),
+	PINMUX_IPSR_MSEL(IP7_19_16,	MSIOF1_SCK_E,		SEL_MSIOF1_4),
+	PINMUX_IPSR_MSEL(IP7_19_16,	STP_OPWM_0_B,		SEL_SSP1_0_1),
+
+	PINMUX_IPSR_GPSR(IP7_23_20,	SD0_CMD),
+	PINMUX_IPSR_MSEL(IP7_23_20,	MSIOF1_SYNC_E,		SEL_MSIOF1_4),
+	PINMUX_IPSR_MSEL(IP7_23_20,	STP_IVCXO27_0_B,	SEL_SSP1_0_1),
+
+	PINMUX_IPSR_GPSR(IP7_27_24,	SD0_DAT0),
+	PINMUX_IPSR_MSEL(IP7_27_24,	MSIOF1_RXD_E,		SEL_MSIOF1_4),
+	PINMUX_IPSR_MSEL(IP7_27_24,	TS_SCK0_B,		SEL_TSIF0_1),
+	PINMUX_IPSR_MSEL(IP7_27_24,	STP_ISCLK_0_B,		SEL_SSP1_0_1),
+
+	PINMUX_IPSR_GPSR(IP7_31_28,	SD0_DAT1),
+	PINMUX_IPSR_MSEL(IP7_31_28,	MSIOF1_TXD_E,		SEL_MSIOF1_4),
+	PINMUX_IPSR_MSEL(IP7_31_28,	TS_SPSYNC0_B,		SEL_TSIF0_1),
+	PINMUX_IPSR_MSEL(IP7_31_28,	STP_ISSYNC_0_B,		SEL_SSP1_0_1),
+
+	/* IPSR8 */
+	PINMUX_IPSR_GPSR(IP8_3_0,	SD0_DAT2),
+	PINMUX_IPSR_MSEL(IP8_3_0,	MSIOF1_SS1_E,		SEL_MSIOF1_4),
+	PINMUX_IPSR_MSEL(IP8_3_0,	TS_SDAT0_B,		SEL_TSIF0_1),
+	PINMUX_IPSR_MSEL(IP8_3_0,	STP_ISD_0_B,		SEL_SSP1_0_1),
+
+	PINMUX_IPSR_GPSR(IP8_7_4,	SD0_DAT3),
+	PINMUX_IPSR_MSEL(IP8_7_4,	MSIOF1_SS2_E,		SEL_MSIOF1_4),
+	PINMUX_IPSR_MSEL(IP8_7_4,	TS_SDEN0_B,		SEL_TSIF0_1),
+	PINMUX_IPSR_MSEL(IP8_7_4,	STP_ISEN_0_B,		SEL_SSP1_0_1),
+
+	PINMUX_IPSR_GPSR(IP8_11_8,	SD1_CLK),
+	PINMUX_IPSR_MSEL(IP8_11_8,	MSIOF1_SCK_G,		SEL_MSIOF1_6),
+	PINMUX_IPSR_MSEL(IP8_11_8,	SIM0_CLK_A,		SEL_SIMCARD_0),
+
+	PINMUX_IPSR_GPSR(IP8_15_12,	SD1_CMD),
+	PINMUX_IPSR_MSEL(IP8_15_12,	MSIOF1_SYNC_G,		SEL_MSIOF1_6),
+	PINMUX_IPSR_MSEL(IP8_15_12,	NFCE_N_B,		SEL_NDF_1),
+	PINMUX_IPSR_MSEL(IP8_15_12,	SIM0_D_A,		SEL_SIMCARD_0),
+	PINMUX_IPSR_MSEL(IP8_15_12,	STP_IVCXO27_1_B,	SEL_SSP1_1_1),
+
+	PINMUX_IPSR_GPSR(IP8_19_16,	SD1_DAT0),
+	PINMUX_IPSR_GPSR(IP8_19_16,	SD2_DAT4),
+	PINMUX_IPSR_MSEL(IP8_19_16,	MSIOF1_RXD_G,		SEL_MSIOF1_6),
+	PINMUX_IPSR_MSEL(IP8_19_16,	NFWP_N_B,		SEL_NDF_1),
+	PINMUX_IPSR_MSEL(IP8_19_16,	TS_SCK1_B,		SEL_TSIF1_1),
+	PINMUX_IPSR_MSEL(IP8_19_16,	STP_ISCLK_1_B,		SEL_SSP1_1_1),
+
+	PINMUX_IPSR_GPSR(IP8_23_20,	SD1_DAT1),
+	PINMUX_IPSR_GPSR(IP8_23_20,	SD2_DAT5),
+	PINMUX_IPSR_MSEL(IP8_23_20,	MSIOF1_TXD_G,		SEL_MSIOF1_6),
+	PINMUX_IPSR_MSEL(IP8_23_20,	NFDATA14_B,		SEL_NDF_1),
+	PINMUX_IPSR_MSEL(IP8_23_20,	TS_SPSYNC1_B,		SEL_TSIF1_1),
+	PINMUX_IPSR_MSEL(IP8_23_20,	STP_ISSYNC_1_B,		SEL_SSP1_1_1),
+
+	PINMUX_IPSR_GPSR(IP8_27_24,	SD1_DAT2),
+	PINMUX_IPSR_GPSR(IP8_27_24,	SD2_DAT6),
+	PINMUX_IPSR_MSEL(IP8_27_24,	MSIOF1_SS1_G,		SEL_MSIOF1_6),
+	PINMUX_IPSR_MSEL(IP8_27_24,	NFDATA15_B,		SEL_NDF_1),
+	PINMUX_IPSR_MSEL(IP8_27_24,	TS_SDAT1_B,		SEL_TSIF1_1),
+	PINMUX_IPSR_MSEL(IP8_27_24,	STP_ISD_1_B,		SEL_SSP1_1_1),
+
+	PINMUX_IPSR_GPSR(IP8_31_28,	SD1_DAT3),
+	PINMUX_IPSR_GPSR(IP8_31_28,	SD2_DAT7),
+	PINMUX_IPSR_MSEL(IP8_31_28,	MSIOF1_SS2_G,		SEL_MSIOF1_6),
+	PINMUX_IPSR_MSEL(IP8_31_28,	NFRB_N_B,		SEL_NDF_1),
+	PINMUX_IPSR_MSEL(IP8_31_28,	TS_SDEN1_B,		SEL_TSIF1_1),
+	PINMUX_IPSR_MSEL(IP8_31_28,	STP_ISEN_1_B,		SEL_SSP1_1_1),
+
+	/* IPSR9 */
+	PINMUX_IPSR_GPSR(IP9_3_0,	SD2_CLK),
+	PINMUX_IPSR_GPSR(IP9_3_0,	NFDATA8),
+
+	PINMUX_IPSR_GPSR(IP9_7_4,	SD2_CMD),
+	PINMUX_IPSR_GPSR(IP9_7_4,	NFDATA9),
+
+	PINMUX_IPSR_GPSR(IP9_11_8,	SD2_DAT0),
+	PINMUX_IPSR_GPSR(IP9_11_8,	NFDATA10),
+
+	PINMUX_IPSR_GPSR(IP9_15_12,	SD2_DAT1),
+	PINMUX_IPSR_GPSR(IP9_15_12,	NFDATA11),
+
+	PINMUX_IPSR_GPSR(IP9_19_16,	SD2_DAT2),
+	PINMUX_IPSR_GPSR(IP9_19_16,	NFDATA12),
+
+	PINMUX_IPSR_GPSR(IP9_23_20,	SD2_DAT3),
+	PINMUX_IPSR_GPSR(IP9_23_20,	NFDATA13),
+
+	PINMUX_IPSR_GPSR(IP9_27_24,	SD2_DS),
+	PINMUX_IPSR_GPSR(IP9_27_24,	NFALE),
+
+	PINMUX_IPSR_GPSR(IP9_31_28,	SD3_CLK),
+	PINMUX_IPSR_GPSR(IP9_31_28,	NFWE_N),
+
+	/* IPSR10 */
+	PINMUX_IPSR_GPSR(IP10_3_0,	SD3_CMD),
+	PINMUX_IPSR_GPSR(IP10_3_0,	NFRE_N),
+
+	PINMUX_IPSR_GPSR(IP10_7_4,	SD3_DAT0),
+	PINMUX_IPSR_GPSR(IP10_7_4,	NFDATA0),
+
+	PINMUX_IPSR_GPSR(IP10_11_8,	SD3_DAT1),
+	PINMUX_IPSR_GPSR(IP10_11_8,	NFDATA1),
+
+	PINMUX_IPSR_GPSR(IP10_15_12,	SD3_DAT2),
+	PINMUX_IPSR_GPSR(IP10_15_12,	NFDATA2),
+
+	PINMUX_IPSR_GPSR(IP10_19_16,	SD3_DAT3),
+	PINMUX_IPSR_GPSR(IP10_19_16,	NFDATA3),
+
+	PINMUX_IPSR_GPSR(IP10_23_20,	SD3_DAT4),
+	PINMUX_IPSR_MSEL(IP10_23_20,	SD2_CD_A,		SEL_SDHI2_0),
+	PINMUX_IPSR_GPSR(IP10_23_20,	NFDATA4),
+
+	PINMUX_IPSR_GPSR(IP10_27_24,	SD3_DAT5),
+	PINMUX_IPSR_MSEL(IP10_27_24,	SD2_WP_A,		SEL_SDHI2_0),
+	PINMUX_IPSR_GPSR(IP10_27_24,	NFDATA5),
+
+	PINMUX_IPSR_GPSR(IP10_31_28,	SD3_DAT6),
+	PINMUX_IPSR_GPSR(IP10_31_28,	SD3_CD),
+	PINMUX_IPSR_GPSR(IP10_31_28,	NFDATA6),
+
+	/* IPSR11 */
+	PINMUX_IPSR_GPSR(IP11_3_0,	SD3_DAT7),
+	PINMUX_IPSR_GPSR(IP11_3_0,	SD3_WP),
+	PINMUX_IPSR_GPSR(IP11_3_0,	NFDATA7),
+
+	PINMUX_IPSR_GPSR(IP11_7_4,	SD3_DS),
+	PINMUX_IPSR_GPSR(IP11_7_4,	NFCLE),
+
+	PINMUX_IPSR_GPSR(IP11_11_8,	SD0_CD),
+	PINMUX_IPSR_MSEL(IP11_11_8,	SCL2_B,			SEL_I2C2_1),
+	PINMUX_IPSR_MSEL(IP11_11_8,	SIM0_RST_A,		SEL_SIMCARD_0),
+
+	PINMUX_IPSR_GPSR(IP11_15_12,	SD0_WP),
+	PINMUX_IPSR_MSEL(IP11_15_12,	SDA2_B,			SEL_I2C2_1),
+
+	PINMUX_IPSR_GPSR(IP11_19_16,	SD1_CD),
+	PINMUX_IPSR_MSEL(IP11_19_16,	SIM0_CLK_B,		SEL_SIMCARD_1),
+
+	PINMUX_IPSR_GPSR(IP11_23_20,	SD1_WP),
+	PINMUX_IPSR_MSEL(IP11_23_20,	SIM0_D_B,		SEL_SIMCARD_1),
+
+	PINMUX_IPSR_GPSR(IP11_27_24,	SCK0),
+	PINMUX_IPSR_MSEL(IP11_27_24,	HSCK1_B,		SEL_HSCIF1_1),
+	PINMUX_IPSR_MSEL(IP11_27_24,	MSIOF1_SS2_B,		SEL_MSIOF1_1),
+	PINMUX_IPSR_MSEL(IP11_27_24,	AUDIO_CLKC_B,		SEL_ADG_C_1),
+	PINMUX_IPSR_MSEL(IP11_27_24,	SDA2_A,			SEL_I2C2_0),
+	PINMUX_IPSR_MSEL(IP11_27_24,	SIM0_RST_B,		SEL_SIMCARD_1),
+	PINMUX_IPSR_MSEL(IP11_27_24,	STP_OPWM_0_C,		SEL_SSP1_0_2),
+	PINMUX_IPSR_MSEL(IP11_27_24,	RIF0_CLK_B,		SEL_DRIF0_1),
+	PINMUX_IPSR_GPSR(IP11_27_24,	ADICHS2),
+	PINMUX_IPSR_MSEL(IP11_27_24,	SCK5_B,			SEL_SCIF5_1),
+
+	PINMUX_IPSR_GPSR(IP11_31_28,	RX0),
+	PINMUX_IPSR_MSEL(IP11_31_28,	HRX1_B,			SEL_HSCIF1_1),
+	PINMUX_IPSR_MSEL(IP11_31_28,	TS_SCK0_C,		SEL_TSIF0_2),
+	PINMUX_IPSR_MSEL(IP11_31_28,	STP_ISCLK_0_C,		SEL_SSP1_0_2),
+	PINMUX_IPSR_MSEL(IP11_31_28,	RIF0_D0_B,		SEL_DRIF0_1),
+
+	/* IPSR12 */
+	PINMUX_IPSR_GPSR(IP12_3_0,	TX0),
+	PINMUX_IPSR_MSEL(IP12_3_0,	HTX1_B,			SEL_HSCIF1_1),
+	PINMUX_IPSR_MSEL(IP12_3_0,	TS_SPSYNC0_C,		SEL_TSIF0_2),
+	PINMUX_IPSR_MSEL(IP12_3_0,	STP_ISSYNC_0_C,		SEL_SSP1_0_2),
+	PINMUX_IPSR_MSEL(IP12_3_0,	RIF0_D1_B,		SEL_DRIF0_1),
+
+	PINMUX_IPSR_GPSR(IP12_7_4,	CTS0_N),
+	PINMUX_IPSR_MSEL(IP12_7_4,	HCTS1_N_B,		SEL_HSCIF1_1),
+	PINMUX_IPSR_MSEL(IP12_7_4,	MSIOF1_SYNC_B,		SEL_MSIOF1_1),
+	PINMUX_IPSR_MSEL(IP12_7_4,	TS_SPSYNC1_C,		SEL_TSIF1_2),
+	PINMUX_IPSR_MSEL(IP12_7_4,	STP_ISSYNC_1_C,		SEL_SSP1_1_2),
+	PINMUX_IPSR_MSEL(IP12_7_4,	RIF1_SYNC_B,		SEL_DRIF1_1),
+	PINMUX_IPSR_GPSR(IP12_7_4,	AUDIO_CLKOUT_C),
+	PINMUX_IPSR_GPSR(IP12_7_4,	ADICS_SAMP),
+
+	PINMUX_IPSR_GPSR(IP12_11_8,	RTS0_N_TANS),
+	PINMUX_IPSR_MSEL(IP12_11_8,	HRTS1_N_B,		SEL_HSCIF1_1),
+	PINMUX_IPSR_MSEL(IP12_11_8,	MSIOF1_SS1_B,		SEL_MSIOF1_1),
+	PINMUX_IPSR_MSEL(IP12_11_8,	AUDIO_CLKA_B,		SEL_ADG_A_1),
+	PINMUX_IPSR_MSEL(IP12_11_8,	SCL2_A,			SEL_I2C2_0),
+	PINMUX_IPSR_MSEL(IP12_11_8,	STP_IVCXO27_1_C,	SEL_SSP1_1_2),
+	PINMUX_IPSR_MSEL(IP12_11_8,	RIF0_SYNC_B,		SEL_DRIF0_1),
+	PINMUX_IPSR_MSEL(IP12_11_8,	FSO_TOE_A,		SEL_FSO_0),
+	PINMUX_IPSR_GPSR(IP12_11_8,	ADICHS1),
+
+	PINMUX_IPSR_MSEL(IP12_15_12,	RX1_A,			SEL_SCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_15_12,	HRX1_A,			SEL_HSCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_15_12,	TS_SDAT0_C,		SEL_TSIF0_2),
+	PINMUX_IPSR_MSEL(IP12_15_12,	STP_ISD_0_C,		SEL_SSP1_0_2),
+	PINMUX_IPSR_MSEL(IP12_15_12,	RIF1_CLK_C,		SEL_DRIF1_2),
+
+	PINMUX_IPSR_MSEL(IP12_19_16,	TX1_A,			SEL_SCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_19_16,	HTX1_A,			SEL_HSCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_19_16,	TS_SDEN0_C,		SEL_TSIF0_2),
+	PINMUX_IPSR_MSEL(IP12_19_16,	STP_ISEN_0_C,		SEL_SSP1_0_2),
+	PINMUX_IPSR_MSEL(IP12_19_16,	RIF1_D0_C,		SEL_DRIF1_2),
+
+	PINMUX_IPSR_GPSR(IP12_23_20,	CTS1_N),
+	PINMUX_IPSR_MSEL(IP12_23_20,	HCTS1_N_A,		SEL_HSCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_23_20,	MSIOF1_RXD_B,		SEL_MSIOF1_1),
+	PINMUX_IPSR_MSEL(IP12_23_20,	TS_SDEN1_C,		SEL_TSIF1_2),
+	PINMUX_IPSR_MSEL(IP12_23_20,	STP_ISEN_1_C,		SEL_SSP1_1_2),
+	PINMUX_IPSR_MSEL(IP12_23_20,	RIF1_D0_B,		SEL_DRIF1_1),
+	PINMUX_IPSR_GPSR(IP12_23_20,	ADIDATA),
+
+	PINMUX_IPSR_GPSR(IP12_27_24,	RTS1_N_TANS),
+	PINMUX_IPSR_MSEL(IP12_27_24,	HRTS1_N_A,		SEL_HSCIF1_0),
+	PINMUX_IPSR_MSEL(IP12_27_24,	MSIOF1_TXD_B,		SEL_MSIOF1_1),
+	PINMUX_IPSR_MSEL(IP12_27_24,	TS_SDAT1_C,		SEL_TSIF1_2),
+	PINMUX_IPSR_MSEL(IP12_27_24,	STP_ISD_1_C,		SEL_SSP1_1_2),
+	PINMUX_IPSR_MSEL(IP12_27_24,	RIF1_D1_B,		SEL_DRIF1_1),
+	PINMUX_IPSR_GPSR(IP12_27_24,	ADICHS0),
+
+	PINMUX_IPSR_GPSR(IP12_31_28,	SCK2),
+	PINMUX_IPSR_MSEL(IP12_31_28,	SCIF_CLK_B,		SEL_SCIF1_1),
+	PINMUX_IPSR_MSEL(IP12_31_28,	MSIOF1_SCK_B,		SEL_MSIOF1_1),
+	PINMUX_IPSR_MSEL(IP12_31_28,	TS_SCK1_C,		SEL_TSIF1_2),
+	PINMUX_IPSR_MSEL(IP12_31_28,	STP_ISCLK_1_C,		SEL_SSP1_1_2),
+	PINMUX_IPSR_MSEL(IP12_31_28,	RIF1_CLK_B,		SEL_DRIF1_1),
+	PINMUX_IPSR_GPSR(IP12_31_28,	ADICLK),
+
+	/* IPSR13 */
+	PINMUX_IPSR_MSEL(IP13_3_0,	TX2_A,			SEL_SCIF2_0),
+	PINMUX_IPSR_MSEL(IP13_3_0,	SD2_CD_B,		SEL_SDHI2_1),
+	PINMUX_IPSR_MSEL(IP13_3_0,	SCL1_A,			SEL_I2C1_0),
+	PINMUX_IPSR_MSEL(IP13_3_0,	FMCLK_A,		SEL_FM_0),
+	PINMUX_IPSR_MSEL(IP13_3_0,	RIF1_D1_C,		SEL_DRIF1_2),
+	PINMUX_IPSR_MSEL(IP13_3_0,	FSO_CFE_0_B,		SEL_FSO_1),
+
+	PINMUX_IPSR_MSEL(IP13_7_4,	RX2_A,			SEL_SCIF2_0),
+	PINMUX_IPSR_MSEL(IP13_7_4,	SD2_WP_B,		SEL_SDHI2_1),
+	PINMUX_IPSR_MSEL(IP13_7_4,	SDA1_A,			SEL_I2C1_0),
+	PINMUX_IPSR_MSEL(IP13_7_4,	FMIN_A,			SEL_FM_0),
+	PINMUX_IPSR_MSEL(IP13_7_4,	RIF1_SYNC_C,		SEL_DRIF1_2),
+	PINMUX_IPSR_MSEL(IP13_7_4,	FSO_CFE_1_B,		SEL_FSO_1),
+
+	PINMUX_IPSR_GPSR(IP13_11_8,	HSCK0),
+	PINMUX_IPSR_MSEL(IP13_11_8,	MSIOF1_SCK_D,		SEL_MSIOF1_3),
+	PINMUX_IPSR_MSEL(IP13_11_8,	AUDIO_CLKB_A,		SEL_ADG_B_0),
+	PINMUX_IPSR_MSEL(IP13_11_8,	SSI_SDATA1_B,		SEL_SSI_1),
+	PINMUX_IPSR_MSEL(IP13_11_8,	TS_SCK0_D,		SEL_TSIF0_3),
+	PINMUX_IPSR_MSEL(IP13_11_8,	STP_ISCLK_0_D,		SEL_SSP1_0_3),
+	PINMUX_IPSR_MSEL(IP13_11_8,	RIF0_CLK_C,		SEL_DRIF0_2),
+	PINMUX_IPSR_MSEL(IP13_11_8,	RX5_B,			SEL_SCIF5_1),
+
+	PINMUX_IPSR_GPSR(IP13_15_12,	HRX0),
+	PINMUX_IPSR_MSEL(IP13_15_12,	MSIOF1_RXD_D,		SEL_MSIOF1_3),
+	PINMUX_IPSR_MSEL(IP13_15_12,	SSI_SDATA2_B,		SEL_SSI_1),
+	PINMUX_IPSR_MSEL(IP13_15_12,	TS_SDEN0_D,		SEL_TSIF0_3),
+	PINMUX_IPSR_MSEL(IP13_15_12,	STP_ISEN_0_D,		SEL_SSP1_0_3),
+	PINMUX_IPSR_MSEL(IP13_15_12,	RIF0_D0_C,		SEL_DRIF0_2),
+
+	PINMUX_IPSR_GPSR(IP13_19_16,	HTX0),
+	PINMUX_IPSR_MSEL(IP13_19_16,	MSIOF1_TXD_D,		SEL_MSIOF1_3),
+	PINMUX_IPSR_MSEL(IP13_19_16,	SSI_SDATA9_B,		SEL_SSI_1),
+	PINMUX_IPSR_MSEL(IP13_19_16,	TS_SDAT0_D,		SEL_TSIF0_3),
+	PINMUX_IPSR_MSEL(IP13_19_16,	STP_ISD_0_D,		SEL_SSP1_0_3),
+	PINMUX_IPSR_MSEL(IP13_19_16,	RIF0_D1_C,		SEL_DRIF0_2),
+
+	PINMUX_IPSR_GPSR(IP13_23_20,	HCTS0_N),
+	PINMUX_IPSR_MSEL(IP13_23_20,	RX2_B,			SEL_SCIF2_1),
+	PINMUX_IPSR_MSEL(IP13_23_20,	MSIOF1_SYNC_D,		SEL_MSIOF1_3),
+	PINMUX_IPSR_MSEL(IP13_23_20,	SSI_SCK9_A,		SEL_SSI_0),
+	PINMUX_IPSR_MSEL(IP13_23_20,	TS_SPSYNC0_D,		SEL_TSIF0_3),
+	PINMUX_IPSR_MSEL(IP13_23_20,	STP_ISSYNC_0_D,		SEL_SSP1_0_3),
+	PINMUX_IPSR_MSEL(IP13_23_20,	RIF0_SYNC_C,		SEL_DRIF0_2),
+	PINMUX_IPSR_GPSR(IP13_23_20,	AUDIO_CLKOUT1_A),
+
+	PINMUX_IPSR_GPSR(IP13_27_24,	HRTS0_N),
+	PINMUX_IPSR_MSEL(IP13_27_24,	TX2_B,			SEL_SCIF2_1),
+	PINMUX_IPSR_MSEL(IP13_27_24,	MSIOF1_SS1_D,		SEL_MSIOF1_3),
+	PINMUX_IPSR_MSEL(IP13_27_24,	SSI_WS9_A,		SEL_SSI_0),
+	PINMUX_IPSR_MSEL(IP13_27_24,	STP_IVCXO27_0_D,	SEL_SSP1_0_3),
+	PINMUX_IPSR_MSEL(IP13_27_24,	BPFCLK_A,		SEL_FM_0),
+	PINMUX_IPSR_GPSR(IP13_27_24,	AUDIO_CLKOUT2_A),
+
+	PINMUX_IPSR_GPSR(IP13_31_28,	MSIOF0_SYNC),
+	PINMUX_IPSR_GPSR(IP13_31_28,	AUDIO_CLKOUT_A),
+	PINMUX_IPSR_MSEL(IP13_31_28,	TX5_B,			SEL_SCIF5_1),
+	PINMUX_IPSR_MSEL(IP13_31_28,	BPFCLK_D,		SEL_FM_3),
+
+	/* IPSR14 */
+	PINMUX_IPSR_GPSR(IP14_3_0,	MSIOF0_SS1),
+	PINMUX_IPSR_MSEL(IP14_3_0,	RX5_A,			SEL_SCIF5_0),
+	PINMUX_IPSR_MSEL(IP14_3_0,	NFWP_N_A,		SEL_NDF_0),
+	PINMUX_IPSR_MSEL(IP14_3_0,	AUDIO_CLKA_C,		SEL_ADG_A_2),
+	PINMUX_IPSR_MSEL(IP14_3_0,	SSI_SCK2_A,		SEL_SSI_0),
+	PINMUX_IPSR_MSEL(IP14_3_0,	STP_IVCXO27_0_C,	SEL_SSP1_0_2),
+	PINMUX_IPSR_GPSR(IP14_3_0,	AUDIO_CLKOUT3_A),
+	PINMUX_IPSR_MSEL(IP14_3_0,	TCLK1_B,		SEL_TIMER_TMU_1),
+
+	PINMUX_IPSR_GPSR(IP14_7_4,	MSIOF0_SS2),
+	PINMUX_IPSR_MSEL(IP14_7_4,	TX5_A,			SEL_SCIF5_0),
+	PINMUX_IPSR_MSEL(IP14_7_4,	MSIOF1_SS2_D,		SEL_MSIOF1_3),
+	PINMUX_IPSR_MSEL(IP14_7_4,	AUDIO_CLKC_A,		SEL_ADG_C_0),
+	PINMUX_IPSR_MSEL(IP14_7_4,	SSI_WS2_A,		SEL_SSI_0),
+	PINMUX_IPSR_MSEL(IP14_7_4,	STP_OPWM_0_D,		SEL_SSP1_0_3),
+	PINMUX_IPSR_GPSR(IP14_7_4,	AUDIO_CLKOUT_D),
+	PINMUX_IPSR_MSEL(IP14_7_4,	SPEEDIN_B,		SEL_SPEED_PULSE_1),
+
+	PINMUX_IPSR_GPSR(IP14_11_8,	MLB_CLK),
+	PINMUX_IPSR_MSEL(IP14_11_8,	MSIOF1_SCK_F,		SEL_MSIOF1_5),
+	PINMUX_IPSR_MSEL(IP14_11_8,	SCL1_B,			SEL_I2C1_1),
+
+	PINMUX_IPSR_GPSR(IP14_15_12,	MLB_SIG),
+	PINMUX_IPSR_MSEL(IP14_15_12,	RX1_B,			SEL_SCIF1_1),
+	PINMUX_IPSR_MSEL(IP14_15_12,	MSIOF1_SYNC_F,		SEL_MSIOF1_5),
+	PINMUX_IPSR_MSEL(IP14_15_12,	SDA1_B,			SEL_I2C1_1),
+
+	PINMUX_IPSR_GPSR(IP14_19_16,	MLB_DAT),
+	PINMUX_IPSR_MSEL(IP14_19_16,	TX1_B,			SEL_SCIF1_1),
+	PINMUX_IPSR_MSEL(IP14_19_16,	MSIOF1_RXD_F,		SEL_MSIOF1_5),
+
+	PINMUX_IPSR_GPSR(IP14_23_20,	SSI_SCK0129),
+	PINMUX_IPSR_MSEL(IP14_23_20,	MSIOF1_TXD_F,		SEL_MSIOF1_5),
+
+	PINMUX_IPSR_GPSR(IP14_27_24,	SSI_WS0129),
+	PINMUX_IPSR_MSEL(IP14_27_24,	MSIOF1_SS1_F,		SEL_MSIOF1_5),
+
+	PINMUX_IPSR_GPSR(IP14_31_28,	SSI_SDATA0),
+	PINMUX_IPSR_MSEL(IP14_31_28,	MSIOF1_SS2_F,		SEL_MSIOF1_5),
+
+	/* IPSR15 */
+	PINMUX_IPSR_MSEL(IP15_3_0,	SSI_SDATA1_A,		SEL_SSI_0),
+
+	PINMUX_IPSR_MSEL(IP15_7_4,	SSI_SDATA2_A,		SEL_SSI_0),
+	PINMUX_IPSR_MSEL(IP15_7_4,	SSI_SCK1_B,		SEL_SSI_1),
+
+	PINMUX_IPSR_GPSR(IP15_11_8,	SSI_SCK34),
+	PINMUX_IPSR_MSEL(IP15_11_8,	MSIOF1_SS1_A,		SEL_MSIOF1_0),
+	PINMUX_IPSR_MSEL(IP15_11_8,	STP_OPWM_0_A,		SEL_SSP1_0_0),
+
+	PINMUX_IPSR_GPSR(IP15_15_12,	SSI_WS34),
+	PINMUX_IPSR_MSEL(IP15_15_12,	HCTS2_N_A,		SEL_HSCIF2_0),
+	PINMUX_IPSR_MSEL(IP15_15_12,	MSIOF1_SS2_A,		SEL_MSIOF1_0),
+	PINMUX_IPSR_MSEL(IP15_15_12,	STP_IVCXO27_0_A,	SEL_SSP1_0_0),
+
+	PINMUX_IPSR_GPSR(IP15_19_16,	SSI_SDATA3),
+	PINMUX_IPSR_MSEL(IP15_19_16,	HRTS2_N_A,		SEL_HSCIF2_0),
+	PINMUX_IPSR_MSEL(IP15_19_16,	MSIOF1_TXD_A,		SEL_MSIOF1_0),
+	PINMUX_IPSR_MSEL(IP15_19_16,	TS_SCK0_A,		SEL_TSIF0_0),
+	PINMUX_IPSR_MSEL(IP15_19_16,	STP_ISCLK_0_A,		SEL_SSP1_0_0),
+	PINMUX_IPSR_MSEL(IP15_19_16,	RIF0_D1_A,		SEL_DRIF0_0),
+	PINMUX_IPSR_MSEL(IP15_19_16,	RIF2_D0_A,		SEL_DRIF2_0),
+
+	PINMUX_IPSR_GPSR(IP15_23_20,	SSI_SCK4),
+	PINMUX_IPSR_MSEL(IP15_23_20,	HRX2_A,			SEL_HSCIF2_0),
+	PINMUX_IPSR_MSEL(IP15_23_20,	MSIOF1_SCK_A,		SEL_MSIOF1_0),
+	PINMUX_IPSR_MSEL(IP15_23_20,	TS_SDAT0_A,		SEL_TSIF0_0),
+	PINMUX_IPSR_MSEL(IP15_23_20,	STP_ISD_0_A,		SEL_SSP1_0_0),
+	PINMUX_IPSR_MSEL(IP15_23_20,	RIF0_CLK_A,		SEL_DRIF0_0),
+	PINMUX_IPSR_MSEL(IP15_23_20,	RIF2_CLK_A,		SEL_DRIF2_0),
+
+	PINMUX_IPSR_GPSR(IP15_27_24,	SSI_WS4),
+	PINMUX_IPSR_MSEL(IP15_27_24,	HTX2_A,			SEL_HSCIF2_0),
+	PINMUX_IPSR_MSEL(IP15_27_24,	MSIOF1_SYNC_A,		SEL_MSIOF1_0),
+	PINMUX_IPSR_MSEL(IP15_27_24,	TS_SDEN0_A,		SEL_TSIF0_0),
+	PINMUX_IPSR_MSEL(IP15_27_24,	STP_ISEN_0_A,		SEL_SSP1_0_0),
+	PINMUX_IPSR_MSEL(IP15_27_24,	RIF0_SYNC_A,		SEL_DRIF0_0),
+	PINMUX_IPSR_MSEL(IP15_27_24,	RIF2_SYNC_A,		SEL_DRIF2_0),
+
+	PINMUX_IPSR_GPSR(IP15_31_28,	SSI_SDATA4),
+	PINMUX_IPSR_MSEL(IP15_31_28,	HSCK2_A,		SEL_HSCIF2_0),
+	PINMUX_IPSR_MSEL(IP15_31_28,	MSIOF1_RXD_A,		SEL_MSIOF1_0),
+	PINMUX_IPSR_MSEL(IP15_31_28,	TS_SPSYNC0_A,		SEL_TSIF0_0),
+	PINMUX_IPSR_MSEL(IP15_31_28,	STP_ISSYNC_0_A,		SEL_SSP1_0_0),
+	PINMUX_IPSR_MSEL(IP15_31_28,	RIF0_D0_A,		SEL_DRIF0_0),
+	PINMUX_IPSR_MSEL(IP15_31_28,	RIF2_D1_A,		SEL_DRIF2_0),
+
+	/* IPSR16 */
+	PINMUX_IPSR_GPSR(IP16_3_0,	SSI_SCK6),
+	PINMUX_IPSR_MSEL(IP16_3_0,	SIM0_RST_D,		SEL_SIMCARD_3),
+
+	PINMUX_IPSR_GPSR(IP16_7_4,	SSI_WS6),
+	PINMUX_IPSR_MSEL(IP16_7_4,	SIM0_D_D,		SEL_SIMCARD_3),
+
+	PINMUX_IPSR_GPSR(IP16_11_8,	SSI_SDATA6),
+	PINMUX_IPSR_MSEL(IP16_11_8,	SIM0_CLK_D,		SEL_SIMCARD_3),
+
+	PINMUX_IPSR_GPSR(IP16_15_12,	SSI_SCK78),
+	PINMUX_IPSR_MSEL(IP16_15_12,	HRX2_B,			SEL_HSCIF2_1),
+	PINMUX_IPSR_MSEL(IP16_15_12,	MSIOF1_SCK_C,		SEL_MSIOF1_2),
+	PINMUX_IPSR_MSEL(IP16_15_12,	TS_SCK1_A,		SEL_TSIF1_0),
+	PINMUX_IPSR_MSEL(IP16_15_12,	STP_ISCLK_1_A,		SEL_SSP1_1_0),
+	PINMUX_IPSR_MSEL(IP16_15_12,	RIF1_CLK_A,		SEL_DRIF1_0),
+	PINMUX_IPSR_MSEL(IP16_15_12,	RIF3_CLK_A,		SEL_DRIF3_0),
+
+	PINMUX_IPSR_GPSR(IP16_19_16,	SSI_WS78),
+	PINMUX_IPSR_MSEL(IP16_19_16,	HTX2_B,			SEL_HSCIF2_1),
+	PINMUX_IPSR_MSEL(IP16_19_16,	MSIOF1_SYNC_C,		SEL_MSIOF1_2),
+	PINMUX_IPSR_MSEL(IP16_19_16,	TS_SDAT1_A,		SEL_TSIF1_0),
+	PINMUX_IPSR_MSEL(IP16_19_16,	STP_ISD_1_A,		SEL_SSP1_1_0),
+	PINMUX_IPSR_MSEL(IP16_19_16,	RIF1_SYNC_A,		SEL_DRIF1_0),
+	PINMUX_IPSR_MSEL(IP16_19_16,	RIF3_SYNC_A,		SEL_DRIF3_0),
+
+	PINMUX_IPSR_GPSR(IP16_23_20,	SSI_SDATA7),
+	PINMUX_IPSR_MSEL(IP16_23_20,	HCTS2_N_B,		SEL_HSCIF2_1),
+	PINMUX_IPSR_MSEL(IP16_23_20,	MSIOF1_RXD_C,		SEL_MSIOF1_2),
+	PINMUX_IPSR_MSEL(IP16_23_20,	TS_SDEN1_A,		SEL_TSIF1_0),
+	PINMUX_IPSR_MSEL(IP16_23_20,	STP_ISEN_1_A,		SEL_SSP1_1_0),
+	PINMUX_IPSR_MSEL(IP16_23_20,	RIF1_D0_A,		SEL_DRIF1_0),
+	PINMUX_IPSR_MSEL(IP16_23_20,	RIF3_D0_A,		SEL_DRIF3_0),
+	PINMUX_IPSR_MSEL(IP16_23_20,	TCLK2_A,		SEL_TIMER_TMU_0),
+
+	PINMUX_IPSR_GPSR(IP16_27_24,	SSI_SDATA8),
+	PINMUX_IPSR_MSEL(IP16_27_24,	HRTS2_N_B,		SEL_HSCIF2_1),
+	PINMUX_IPSR_MSEL(IP16_27_24,	MSIOF1_TXD_C,		SEL_MSIOF1_2),
+	PINMUX_IPSR_MSEL(IP16_27_24,	TS_SPSYNC1_A,		SEL_TSIF1_0),
+	PINMUX_IPSR_MSEL(IP16_27_24,	STP_ISSYNC_1_A,		SEL_SSP1_1_0),
+	PINMUX_IPSR_MSEL(IP16_27_24,	RIF1_D1_A,		SEL_DRIF1_0),
+	PINMUX_IPSR_MSEL(IP16_27_24,	RIF3_D1_A,		SEL_DRIF3_0),
+
+	PINMUX_IPSR_MSEL(IP16_31_28,	SSI_SDATA9_A,		SEL_SSI_0),
+	PINMUX_IPSR_MSEL(IP16_31_28,	HSCK2_B,		SEL_HSCIF2_1),
+	PINMUX_IPSR_MSEL(IP16_31_28,	MSIOF1_SS1_C,		SEL_MSIOF1_2),
+	PINMUX_IPSR_MSEL(IP16_31_28,	HSCK1_A,		SEL_HSCIF1_0),
+	PINMUX_IPSR_MSEL(IP16_31_28,	SSI_WS1_B,		SEL_SSI_1),
+	PINMUX_IPSR_GPSR(IP16_31_28,	SCK1),
+	PINMUX_IPSR_MSEL(IP16_31_28,	STP_IVCXO27_1_A,	SEL_SSP1_1_0),
+	PINMUX_IPSR_GPSR(IP16_31_28,	SCK5_A),
+
+	/* IPSR17 */
+	PINMUX_IPSR_MSEL(IP17_3_0,	AUDIO_CLKA_A,		SEL_ADG_A_0),
+	PINMUX_IPSR_GPSR(IP17_3_0,	CC5_OSCOUT),
+
+	PINMUX_IPSR_MSEL(IP17_7_4,	AUDIO_CLKB_B,		SEL_ADG_B_1),
+	PINMUX_IPSR_MSEL(IP17_7_4,	SCIF_CLK_A,		SEL_SCIF1_0),
+	PINMUX_IPSR_MSEL(IP17_7_4,	STP_IVCXO27_1_D,	SEL_SSP1_1_3),
+	PINMUX_IPSR_MSEL(IP17_7_4,	REMOCON_A,		SEL_REMOCON_0),
+	PINMUX_IPSR_MSEL(IP17_7_4,	TCLK1_A,		SEL_TIMER_TMU_0),
+
+	PINMUX_IPSR_GPSR(IP17_11_8,	USB0_PWEN),
+	PINMUX_IPSR_MSEL(IP17_11_8,	SIM0_RST_C,		SEL_SIMCARD_2),
+	PINMUX_IPSR_MSEL(IP17_11_8,	TS_SCK1_D,		SEL_TSIF1_3),
+	PINMUX_IPSR_MSEL(IP17_11_8,	STP_ISCLK_1_D,		SEL_SSP1_1_3),
+	PINMUX_IPSR_MSEL(IP17_11_8,	BPFCLK_B,		SEL_FM_1),
+	PINMUX_IPSR_MSEL(IP17_11_8,	RIF3_CLK_B,		SEL_DRIF3_1),
+	PINMUX_IPSR_MSEL(IP17_11_8,	HSCK2_C,		SEL_HSCIF2_2),
+
+	PINMUX_IPSR_GPSR(IP17_15_12,	USB0_OVC),
+	PINMUX_IPSR_MSEL(IP17_15_12,	SIM0_D_C,		SEL_SIMCARD_2),
+	PINMUX_IPSR_MSEL(IP17_15_12,	TS_SDAT1_D,		SEL_TSIF1_3),
+	PINMUX_IPSR_MSEL(IP17_15_12,	STP_ISD_1_D,		SEL_SSP1_1_3),
+	PINMUX_IPSR_MSEL(IP17_15_12,	RIF3_SYNC_B,		SEL_DRIF3_1),
+	PINMUX_IPSR_MSEL(IP17_15_12,	HRX2_C,			SEL_HSCIF2_2),
+
+	PINMUX_IPSR_GPSR(IP17_19_16,	USB1_PWEN),
+	PINMUX_IPSR_MSEL(IP17_19_16,	SIM0_CLK_C,		SEL_SIMCARD_2),
+	PINMUX_IPSR_MSEL(IP17_19_16,	SSI_SCK1_A,		SEL_SSI_0),
+	PINMUX_IPSR_MSEL(IP17_19_16,	TS_SCK0_E,		SEL_TSIF0_4),
+	PINMUX_IPSR_MSEL(IP17_19_16,	STP_ISCLK_0_E,		SEL_SSP1_0_4),
+	PINMUX_IPSR_MSEL(IP17_19_16,	FMCLK_B,		SEL_FM_1),
+	PINMUX_IPSR_MSEL(IP17_19_16,	RIF2_CLK_B,		SEL_DRIF2_1),
+	PINMUX_IPSR_MSEL(IP17_19_16,	SPEEDIN_A,		SEL_SPEED_PULSE_0),
+	PINMUX_IPSR_MSEL(IP17_19_16,	HTX2_C,			SEL_HSCIF2_2),
+
+	PINMUX_IPSR_GPSR(IP17_23_20,	USB1_OVC),
+	PINMUX_IPSR_MSEL(IP17_23_20,	MSIOF1_SS2_C,		SEL_MSIOF1_2),
+	PINMUX_IPSR_MSEL(IP17_23_20,	SSI_WS1_A,		SEL_SSI_0),
+	PINMUX_IPSR_MSEL(IP17_23_20,	TS_SDAT0_E,		SEL_TSIF0_4),
+	PINMUX_IPSR_MSEL(IP17_23_20,	STP_ISD_0_E,		SEL_SSP1_0_4),
+	PINMUX_IPSR_MSEL(IP17_23_20,	FMIN_B,			SEL_FM_1),
+	PINMUX_IPSR_MSEL(IP17_23_20,	RIF2_SYNC_B,		SEL_DRIF2_1),
+	PINMUX_IPSR_MSEL(IP17_23_20,	REMOCON_B,		SEL_REMOCON_1),
+	PINMUX_IPSR_MSEL(IP17_23_20,	HCTS2_N_C,		SEL_HSCIF2_2),
+
+	PINMUX_IPSR_GPSR(IP17_27_24,	USB30_PWEN),
+	PINMUX_IPSR_GPSR(IP17_27_24,	AUDIO_CLKOUT_B),
+	PINMUX_IPSR_MSEL(IP17_27_24,	SSI_SCK2_B,		SEL_SSI_1),
+	PINMUX_IPSR_MSEL(IP17_27_24,	TS_SDEN1_D,		SEL_TSIF1_3),
+	PINMUX_IPSR_MSEL(IP17_27_24,	STP_ISEN_1_D,		SEL_SSP1_1_2),
+	PINMUX_IPSR_MSEL(IP17_27_24,	STP_OPWM_0_E,		SEL_SSP1_0_4),
+	PINMUX_IPSR_MSEL(IP17_27_24,	RIF3_D0_B,		SEL_DRIF3_1),
+	PINMUX_IPSR_MSEL(IP17_27_24,	TCLK2_B,		SEL_TIMER_TMU_1),
+	PINMUX_IPSR_GPSR(IP17_27_24,	TPU0TO0),
+	PINMUX_IPSR_MSEL(IP17_27_24,	BPFCLK_C,		SEL_FM_2),
+	PINMUX_IPSR_MSEL(IP17_27_24,	HRTS2_N_C,		SEL_HSCIF2_2),
+
+	PINMUX_IPSR_GPSR(IP17_31_28,	USB30_OVC),
+	PINMUX_IPSR_GPSR(IP17_31_28,	AUDIO_CLKOUT1_B),
+	PINMUX_IPSR_MSEL(IP17_31_28,	SSI_WS2_B,		SEL_SSI_1),
+	PINMUX_IPSR_MSEL(IP17_31_28,	TS_SPSYNC1_D,		SEL_TSIF1_3),
+	PINMUX_IPSR_MSEL(IP17_31_28,	STP_ISSYNC_1_D,		SEL_SSP1_1_3),
+	PINMUX_IPSR_MSEL(IP17_31_28,	STP_IVCXO27_0_E,	SEL_SSP1_0_4),
+	PINMUX_IPSR_MSEL(IP17_31_28,	RIF3_D1_B,		SEL_DRIF3_1),
+	PINMUX_IPSR_MSEL(IP17_31_28,	FSO_TOE_B,		SEL_FSO_1),
+	PINMUX_IPSR_GPSR(IP17_31_28,	TPU0TO1),
+
+	/* IPSR18 */
+	PINMUX_IPSR_GPSR(IP18_3_0,	GP6_30),
+	PINMUX_IPSR_GPSR(IP18_3_0,	AUDIO_CLKOUT2_B),
+	PINMUX_IPSR_MSEL(IP18_3_0,	SSI_SCK9_B,		SEL_SSI_1),
+	PINMUX_IPSR_MSEL(IP18_3_0,	TS_SDEN0_E,		SEL_TSIF0_4),
+	PINMUX_IPSR_MSEL(IP18_3_0,	STP_ISEN_0_E,		SEL_SSP1_0_4),
+	PINMUX_IPSR_MSEL(IP18_3_0,	RIF2_D0_B,		SEL_DRIF2_1),
+	PINMUX_IPSR_GPSR(IP18_3_0,	TPU0TO2),
+	PINMUX_IPSR_MSEL(IP18_3_0,	FSO_CFE_0_A,		SEL_FSO_0),
+	PINMUX_IPSR_MSEL(IP18_3_0,	FMCLK_C,		SEL_FM_2),
+	PINMUX_IPSR_MSEL(IP18_3_0,	FMCLK_D,		SEL_FM_3),
+
+	PINMUX_IPSR_GPSR(IP18_7_4,	GP6_31),
+	PINMUX_IPSR_GPSR(IP18_7_4,	AUDIO_CLKOUT3_B),
+	PINMUX_IPSR_MSEL(IP18_7_4,	SSI_WS9_B,		SEL_SSI_1),
+	PINMUX_IPSR_MSEL(IP18_7_4,	TS_SPSYNC0_E,		SEL_TSIF0_4),
+	PINMUX_IPSR_MSEL(IP18_7_4,	STP_ISSYNC_0_E,		SEL_SSP1_0_4),
+	PINMUX_IPSR_MSEL(IP18_7_4,	RIF2_D1_B,		SEL_DRIF2_1),
+	PINMUX_IPSR_GPSR(IP18_7_4,	TPU0TO3),
+	PINMUX_IPSR_MSEL(IP18_7_4,	FSO_CFE_1_A,		SEL_FSO_0),
+	PINMUX_IPSR_MSEL(IP18_7_4,	FMIN_C,			SEL_FM_2),
+	PINMUX_IPSR_MSEL(IP18_7_4,	FMIN_D,			SEL_FM_3),
+
+	/* I2C */
+	PINMUX_IPSR_NOGP(0,		I2C_SEL_0_1),
+	PINMUX_IPSR_NOGP(0,		I2C_SEL_3_1),
+	PINMUX_IPSR_NOGP(0,		I2C_SEL_5_1),
+};
+
+static const struct sh_pfc_pin pinmux_pins[] = {
+	PINMUX_GPIO_GP_ALL(),
+};
+
+/* - SCIF0 ------------------------------------------------------------------ */
+static const unsigned int scif0_data_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 1), RCAR_GP_PIN(5, 2),
+};
+static const unsigned int scif0_data_mux[] = {
+	RX0_MARK, TX0_MARK,
+};
+static const unsigned int scif0_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 0),
+};
+static const unsigned int scif0_clk_mux[] = {
+	SCK0_MARK,
+};
+static const unsigned int scif0_ctrl_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(5, 4), RCAR_GP_PIN(5, 3),
+};
+static const unsigned int scif0_ctrl_mux[] = {
+	RTS0_N_TANS_MARK, CTS0_N_MARK,
+};
+/* - SCIF1 ------------------------------------------------------------------ */
+static const unsigned int scif1_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 5), RCAR_GP_PIN(5, 6),
+};
+static const unsigned int scif1_data_a_mux[] = {
+	RX1_A_MARK, TX1_A_MARK,
+};
+static const unsigned int scif1_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(6, 21),
+};
+static const unsigned int scif1_clk_mux[] = {
+	SCK1_MARK,
+};
+static const unsigned int scif1_ctrl_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(5, 8), RCAR_GP_PIN(5, 7),
+};
+static const unsigned int scif1_ctrl_mux[] = {
+	RTS1_N_TANS_MARK, CTS1_N_MARK,
+};
+
+static const unsigned int scif1_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 24), RCAR_GP_PIN(5, 25),
+};
+static const unsigned int scif1_data_b_mux[] = {
+	RX1_B_MARK, TX1_B_MARK,
+};
+/* - SCIF2 ------------------------------------------------------------------ */
+static const unsigned int scif2_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 11), RCAR_GP_PIN(5, 10),
+};
+static const unsigned int scif2_data_a_mux[] = {
+	RX2_A_MARK, TX2_A_MARK,
+};
+static const unsigned int scif2_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 9),
+};
+static const unsigned int scif2_clk_mux[] = {
+	SCK2_MARK,
+};
+static const unsigned int scif2_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 15), RCAR_GP_PIN(5, 16),
+};
+static const unsigned int scif2_data_b_mux[] = {
+	RX2_B_MARK, TX2_B_MARK,
+};
+/* - SCIF3 ------------------------------------------------------------------ */
+static const unsigned int scif3_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 23), RCAR_GP_PIN(1, 24),
+};
+static const unsigned int scif3_data_a_mux[] = {
+	RX3_A_MARK, TX3_A_MARK,
+};
+static const unsigned int scif3_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 22),
+};
+static const unsigned int scif3_clk_mux[] = {
+	SCK3_MARK,
+};
+static const unsigned int scif3_ctrl_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(1, 26), RCAR_GP_PIN(1, 25),
+};
+static const unsigned int scif3_ctrl_mux[] = {
+	RTS3_N_TANS_MARK, CTS3_N_MARK,
+};
+static const unsigned int scif3_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 8), RCAR_GP_PIN(1, 11),
+};
+static const unsigned int scif3_data_b_mux[] = {
+	RX3_B_MARK, TX3_B_MARK,
+};
+/* - SCIF4 ------------------------------------------------------------------ */
+static const unsigned int scif4_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 11), RCAR_GP_PIN(2, 12),
+};
+static const unsigned int scif4_data_a_mux[] = {
+	RX4_A_MARK, TX4_A_MARK,
+};
+static const unsigned int scif4_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 10),
+};
+static const unsigned int scif4_clk_a_mux[] = {
+	SCK4_A_MARK,
+};
+static const unsigned int scif4_ctrl_a_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(2, 14), RCAR_GP_PIN(2, 13),
+};
+static const unsigned int scif4_ctrl_a_mux[] = {
+	RTS4_N_TANS_A_MARK, CTS4_N_A_MARK,
+};
+static const unsigned int scif4_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 6), RCAR_GP_PIN(1, 7),
+};
+static const unsigned int scif4_data_b_mux[] = {
+	RX4_B_MARK, TX4_B_MARK,
+};
+static const unsigned int scif4_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 5),
+};
+static const unsigned int scif4_clk_b_mux[] = {
+	SCK4_B_MARK,
+};
+static const unsigned int scif4_ctrl_b_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(1, 10), RCAR_GP_PIN(1, 9),
+};
+static const unsigned int scif4_ctrl_b_mux[] = {
+	RTS4_N_TANS_B_MARK, CTS4_N_B_MARK,
+};
+static const unsigned int scif4_data_c_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(0, 12), RCAR_GP_PIN(0, 13),
+};
+static const unsigned int scif4_data_c_mux[] = {
+	RX4_C_MARK, TX4_C_MARK,
+};
+static const unsigned int scif4_clk_c_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 8),
+};
+static const unsigned int scif4_clk_c_mux[] = {
+	SCK4_C_MARK,
+};
+static const unsigned int scif4_ctrl_c_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
+};
+static const unsigned int scif4_ctrl_c_mux[] = {
+	RTS4_N_TANS_C_MARK, CTS4_N_C_MARK,
+};
+/* - SCIF5 ------------------------------------------------------------------ */
+static const unsigned int scif5_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 19), RCAR_GP_PIN(5, 21),
+};
+static const unsigned int scif5_data_a_mux[] = {
+	RX5_A_MARK, TX5_A_MARK,
+};
+static const unsigned int scif5_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(6, 21),
+};
+static const unsigned int scif5_clk_a_mux[] = {
+	SCK5_A_MARK,
+};
+
+static const unsigned int scif5_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(5, 12), RCAR_GP_PIN(5, 18),
+};
+static const unsigned int scif5_data_b_mux[] = {
+	RX5_B_MARK, TX5_B_MARK,
+};
+static const unsigned int scif5_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 0),
+};
+static const unsigned int scif5_clk_b_mux[] = {
+	SCK5_B_MARK,
+};
+
+/* - SCIF Clock ------------------------------------------------------------- */
+static const unsigned int scif_clk_a_pins[] = {
+	/* SCIF_CLK */
+	RCAR_GP_PIN(6, 23),
+};
+static const unsigned int scif_clk_a_mux[] = {
+	SCIF_CLK_A_MARK,
+};
+static const unsigned int scif_clk_b_pins[] = {
+	/* SCIF_CLK */
+	RCAR_GP_PIN(5, 9),
+};
+static const unsigned int scif_clk_b_mux[] = {
+	SCIF_CLK_B_MARK,
+};
+
+/* - SDHI0 ------------------------------------------------------------------ */
+static const unsigned int sdhi0_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(3, 2),
+};
+static const unsigned int sdhi0_data1_mux[] = {
+	SD0_DAT0_MARK,
+};
+static const unsigned int sdhi0_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(3, 2), RCAR_GP_PIN(3, 3),
+	RCAR_GP_PIN(3, 4), RCAR_GP_PIN(3, 5),
+};
+static const unsigned int sdhi0_data4_mux[] = {
+	SD0_DAT0_MARK, SD0_DAT1_MARK,
+	SD0_DAT2_MARK, SD0_DAT3_MARK,
+};
+static const unsigned int sdhi0_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(3, 0), RCAR_GP_PIN(3, 1),
+};
+static const unsigned int sdhi0_ctrl_mux[] = {
+	SD0_CLK_MARK, SD0_CMD_MARK,
+};
+static const unsigned int sdhi0_cd_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(3, 12),
+};
+static const unsigned int sdhi0_cd_mux[] = {
+	SD0_CD_MARK,
+};
+static const unsigned int sdhi0_wp_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(3, 13),
+};
+static const unsigned int sdhi0_wp_mux[] = {
+	SD0_WP_MARK,
+};
+/* - SDHI1 ------------------------------------------------------------------ */
+static const unsigned int sdhi1_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(3, 8),
+};
+static const unsigned int sdhi1_data1_mux[] = {
+	SD1_DAT0_MARK,
+};
+static const unsigned int sdhi1_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(3, 8),  RCAR_GP_PIN(3, 9),
+	RCAR_GP_PIN(3, 10), RCAR_GP_PIN(3, 11),
+};
+static const unsigned int sdhi1_data4_mux[] = {
+	SD1_DAT0_MARK, SD1_DAT1_MARK,
+	SD1_DAT2_MARK, SD1_DAT3_MARK,
+};
+static const unsigned int sdhi1_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(3, 6), RCAR_GP_PIN(3, 7),
+};
+static const unsigned int sdhi1_ctrl_mux[] = {
+	SD1_CLK_MARK, SD1_CMD_MARK,
+};
+static const unsigned int sdhi1_cd_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(3, 14),
+};
+static const unsigned int sdhi1_cd_mux[] = {
+	SD1_CD_MARK,
+};
+static const unsigned int sdhi1_wp_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(3, 15),
+};
+static const unsigned int sdhi1_wp_mux[] = {
+	SD1_WP_MARK,
+};
+/* - SDHI2 ------------------------------------------------------------------ */
+static const unsigned int sdhi2_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(4, 2),
+};
+static const unsigned int sdhi2_data1_mux[] = {
+	SD2_DAT0_MARK,
+};
+static const unsigned int sdhi2_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(4, 2), RCAR_GP_PIN(4, 3),
+	RCAR_GP_PIN(4, 4), RCAR_GP_PIN(4, 5),
+};
+static const unsigned int sdhi2_data4_mux[] = {
+	SD2_DAT0_MARK, SD2_DAT1_MARK,
+	SD2_DAT2_MARK, SD2_DAT3_MARK,
+};
+static const unsigned int sdhi2_data8_pins[] = {
+	/* D[0:7] */
+	RCAR_GP_PIN(4, 2),  RCAR_GP_PIN(4, 3),
+	RCAR_GP_PIN(4, 4),  RCAR_GP_PIN(4, 5),
+	RCAR_GP_PIN(3, 8),  RCAR_GP_PIN(3, 9),
+	RCAR_GP_PIN(3, 10), RCAR_GP_PIN(3, 11),
+};
+static const unsigned int sdhi2_data8_mux[] = {
+	SD2_DAT0_MARK, SD2_DAT1_MARK,
+	SD2_DAT2_MARK, SD2_DAT3_MARK,
+	SD2_DAT4_MARK, SD2_DAT5_MARK,
+	SD2_DAT6_MARK, SD2_DAT7_MARK,
+};
+static const unsigned int sdhi2_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(4, 0), RCAR_GP_PIN(4, 1),
+};
+static const unsigned int sdhi2_ctrl_mux[] = {
+	SD2_CLK_MARK, SD2_CMD_MARK,
+};
+static const unsigned int sdhi2_cd_a_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(4, 13),
+};
+static const unsigned int sdhi2_cd_a_mux[] = {
+	SD2_CD_A_MARK,
+};
+static const unsigned int sdhi2_cd_b_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(5, 10),
+};
+static const unsigned int sdhi2_cd_b_mux[] = {
+	SD2_CD_B_MARK,
+};
+static const unsigned int sdhi2_wp_a_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(4, 14),
+};
+static const unsigned int sdhi2_wp_a_mux[] = {
+	SD2_WP_A_MARK,
+};
+static const unsigned int sdhi2_wp_b_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(5, 11),
+};
+static const unsigned int sdhi2_wp_b_mux[] = {
+	SD2_WP_B_MARK,
+};
+static const unsigned int sdhi2_ds_pins[] = {
+	/* DS */
+	RCAR_GP_PIN(4, 6),
+};
+static const unsigned int sdhi2_ds_mux[] = {
+	SD2_DS_MARK,
+};
+/* - SDHI3 ------------------------------------------------------------------ */
+static const unsigned int sdhi3_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(4, 9),
+};
+static const unsigned int sdhi3_data1_mux[] = {
+	SD3_DAT0_MARK,
+};
+static const unsigned int sdhi3_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(4, 9),  RCAR_GP_PIN(4, 10),
+	RCAR_GP_PIN(4, 11), RCAR_GP_PIN(4, 12),
+};
+static const unsigned int sdhi3_data4_mux[] = {
+	SD3_DAT0_MARK, SD3_DAT1_MARK,
+	SD3_DAT2_MARK, SD3_DAT3_MARK,
+};
+static const unsigned int sdhi3_data8_pins[] = {
+	/* D[0:7] */
+	RCAR_GP_PIN(4, 9),  RCAR_GP_PIN(4, 10),
+	RCAR_GP_PIN(4, 11), RCAR_GP_PIN(4, 12),
+	RCAR_GP_PIN(4, 13), RCAR_GP_PIN(4, 14),
+	RCAR_GP_PIN(4, 15), RCAR_GP_PIN(4, 16),
+};
+static const unsigned int sdhi3_data8_mux[] = {
+	SD3_DAT0_MARK, SD3_DAT1_MARK,
+	SD3_DAT2_MARK, SD3_DAT3_MARK,
+	SD3_DAT4_MARK, SD3_DAT5_MARK,
+	SD3_DAT6_MARK, SD3_DAT7_MARK,
+};
+static const unsigned int sdhi3_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(4, 7), RCAR_GP_PIN(4, 8),
+};
+static const unsigned int sdhi3_ctrl_mux[] = {
+	SD3_CLK_MARK, SD3_CMD_MARK,
+};
+static const unsigned int sdhi3_cd_pins[] = {
+	/* CD */
+	RCAR_GP_PIN(4, 15),
+};
+static const unsigned int sdhi3_cd_mux[] = {
+	SD3_CD_MARK,
+};
+static const unsigned int sdhi3_wp_pins[] = {
+	/* WP */
+	RCAR_GP_PIN(4, 16),
+};
+static const unsigned int sdhi3_wp_mux[] = {
+	SD3_WP_MARK,
+};
+static const unsigned int sdhi3_ds_pins[] = {
+	/* DS */
+	RCAR_GP_PIN(4, 17),
+};
+static const unsigned int sdhi3_ds_mux[] = {
+	SD3_DS_MARK,
+};
+
+static const struct sh_pfc_pin_group pinmux_groups[] = {
+	SH_PFC_PIN_GROUP(scif0_data),
+	SH_PFC_PIN_GROUP(scif0_clk),
+	SH_PFC_PIN_GROUP(scif0_ctrl),
+	SH_PFC_PIN_GROUP(scif1_data_a),
+	SH_PFC_PIN_GROUP(scif1_clk),
+	SH_PFC_PIN_GROUP(scif1_ctrl),
+	SH_PFC_PIN_GROUP(scif1_data_b),
+	SH_PFC_PIN_GROUP(scif2_data_a),
+	SH_PFC_PIN_GROUP(scif2_clk),
+	SH_PFC_PIN_GROUP(scif2_data_b),
+	SH_PFC_PIN_GROUP(scif3_data_a),
+	SH_PFC_PIN_GROUP(scif3_clk),
+	SH_PFC_PIN_GROUP(scif3_ctrl),
+	SH_PFC_PIN_GROUP(scif3_data_b),
+	SH_PFC_PIN_GROUP(scif4_data_a),
+	SH_PFC_PIN_GROUP(scif4_clk_a),
+	SH_PFC_PIN_GROUP(scif4_ctrl_a),
+	SH_PFC_PIN_GROUP(scif4_data_b),
+	SH_PFC_PIN_GROUP(scif4_clk_b),
+	SH_PFC_PIN_GROUP(scif4_ctrl_b),
+	SH_PFC_PIN_GROUP(scif4_data_c),
+	SH_PFC_PIN_GROUP(scif4_clk_c),
+	SH_PFC_PIN_GROUP(scif4_ctrl_c),
+	SH_PFC_PIN_GROUP(scif5_data_a),
+	SH_PFC_PIN_GROUP(scif5_clk_a),
+	SH_PFC_PIN_GROUP(scif5_data_b),
+	SH_PFC_PIN_GROUP(scif5_clk_b),
+	SH_PFC_PIN_GROUP(scif_clk_a),
+	SH_PFC_PIN_GROUP(scif_clk_b),
+	SH_PFC_PIN_GROUP(sdhi0_data1),
+	SH_PFC_PIN_GROUP(sdhi0_data4),
+	SH_PFC_PIN_GROUP(sdhi0_ctrl),
+	SH_PFC_PIN_GROUP(sdhi0_cd),
+	SH_PFC_PIN_GROUP(sdhi0_wp),
+	SH_PFC_PIN_GROUP(sdhi1_data1),
+	SH_PFC_PIN_GROUP(sdhi1_data4),
+	SH_PFC_PIN_GROUP(sdhi1_ctrl),
+	SH_PFC_PIN_GROUP(sdhi1_cd),
+	SH_PFC_PIN_GROUP(sdhi1_wp),
+	SH_PFC_PIN_GROUP(sdhi2_data1),
+	SH_PFC_PIN_GROUP(sdhi2_data4),
+	SH_PFC_PIN_GROUP(sdhi2_data8),
+	SH_PFC_PIN_GROUP(sdhi2_ctrl),
+	SH_PFC_PIN_GROUP(sdhi2_cd_a),
+	SH_PFC_PIN_GROUP(sdhi2_wp_a),
+	SH_PFC_PIN_GROUP(sdhi2_cd_b),
+	SH_PFC_PIN_GROUP(sdhi2_wp_b),
+	SH_PFC_PIN_GROUP(sdhi2_ds),
+	SH_PFC_PIN_GROUP(sdhi3_data1),
+	SH_PFC_PIN_GROUP(sdhi3_data4),
+	SH_PFC_PIN_GROUP(sdhi3_data8),
+	SH_PFC_PIN_GROUP(sdhi3_ctrl),
+	SH_PFC_PIN_GROUP(sdhi3_cd),
+	SH_PFC_PIN_GROUP(sdhi3_wp),
+	SH_PFC_PIN_GROUP(sdhi3_ds),
+};
+
+static const char * const scif0_groups[] = {
+	"scif0_data",
+	"scif0_clk",
+	"scif0_ctrl",
+};
+
+static const char * const scif1_groups[] = {
+	"scif1_data_a",
+	"scif1_clk",
+	"scif1_ctrl",
+	"scif1_data_b",
+};
+
+static const char * const scif2_groups[] = {
+	"scif2_data_a",
+	"scif2_clk",
+	"scif2_data_b",
+};
+
+static const char * const scif3_groups[] = {
+	"scif3_data_a",
+	"scif3_clk",
+	"scif3_ctrl",
+	"scif3_data_b",
+};
+
+static const char * const scif4_groups[] = {
+	"scif4_data_a",
+	"scif4_clk_a",
+	"scif4_ctrl_a",
+	"scif4_data_b",
+	"scif4_clk_b",
+	"scif4_ctrl_b",
+	"scif4_data_c",
+	"scif4_clk_c",
+	"scif4_ctrl_c",
+};
+
+static const char * const scif5_groups[] = {
+	"scif5_data_a",
+	"scif5_clk_a",
+	"scif5_data_b",
+	"scif5_clk_b",
+};
+
+static const char * const scif_clk_groups[] = {
+	"scif_clk_a",
+	"scif_clk_b",
+};
+
+static const char * const sdhi0_groups[] = {
+	"sdhi0_data1",
+	"sdhi0_data4",
+	"sdhi0_ctrl",
+	"sdhi0_cd",
+	"sdhi0_wp",
+};
+
+static const char * const sdhi1_groups[] = {
+	"sdhi1_data1",
+	"sdhi1_data4",
+	"sdhi1_ctrl",
+	"sdhi1_cd",
+	"sdhi1_wp",
+};
+
+static const char * const sdhi2_groups[] = {
+	"sdhi2_data1",
+	"sdhi2_data4",
+	"sdhi2_data8",
+	"sdhi2_ctrl",
+	"sdhi2_cd_a",
+	"sdhi2_wp_a",
+	"sdhi2_cd_b",
+	"sdhi2_wp_b",
+	"sdhi2_ds",
+};
+
+static const char * const sdhi3_groups[] = {
+	"sdhi3_data1",
+	"sdhi3_data4",
+	"sdhi3_data8",
+	"sdhi3_ctrl",
+	"sdhi3_cd",
+	"sdhi3_wp",
+	"sdhi3_ds",
+};
+
+static const struct sh_pfc_function pinmux_functions[] = {
+	SH_PFC_FUNCTION(scif0),
+	SH_PFC_FUNCTION(scif1),
+	SH_PFC_FUNCTION(scif2),
+	SH_PFC_FUNCTION(scif3),
+	SH_PFC_FUNCTION(scif4),
+	SH_PFC_FUNCTION(scif5),
+	SH_PFC_FUNCTION(scif_clk),
+	SH_PFC_FUNCTION(sdhi0),
+	SH_PFC_FUNCTION(sdhi1),
+	SH_PFC_FUNCTION(sdhi2),
+	SH_PFC_FUNCTION(sdhi3),
+};
+
+static const struct pinmux_cfg_reg pinmux_config_regs[] = {
+#define F_(x, y)	FN_##y
+#define FM(x)		FN_##x
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_0_15_FN,	GPSR0_15,
+		GP_0_14_FN,	GPSR0_14,
+		GP_0_13_FN,	GPSR0_13,
+		GP_0_12_FN,	GPSR0_12,
+		GP_0_11_FN,	GPSR0_11,
+		GP_0_10_FN,	GPSR0_10,
+		GP_0_9_FN,	GPSR0_9,
+		GP_0_8_FN,	GPSR0_8,
+		GP_0_7_FN,	GPSR0_7,
+		GP_0_6_FN,	GPSR0_6,
+		GP_0_5_FN,	GPSR0_5,
+		GP_0_4_FN,	GPSR0_4,
+		GP_0_3_FN,	GPSR0_3,
+		GP_0_2_FN,	GPSR0_2,
+		GP_0_1_FN,	GPSR0_1,
+		GP_0_0_FN,	GPSR0_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_1_28_FN,	GPSR1_28,
+		GP_1_27_FN,	GPSR1_27,
+		GP_1_26_FN,	GPSR1_26,
+		GP_1_25_FN,	GPSR1_25,
+		GP_1_24_FN,	GPSR1_24,
+		GP_1_23_FN,	GPSR1_23,
+		GP_1_22_FN,	GPSR1_22,
+		GP_1_21_FN,	GPSR1_21,
+		GP_1_20_FN,	GPSR1_20,
+		GP_1_19_FN,	GPSR1_19,
+		GP_1_18_FN,	GPSR1_18,
+		GP_1_17_FN,	GPSR1_17,
+		GP_1_16_FN,	GPSR1_16,
+		GP_1_15_FN,	GPSR1_15,
+		GP_1_14_FN,	GPSR1_14,
+		GP_1_13_FN,	GPSR1_13,
+		GP_1_12_FN,	GPSR1_12,
+		GP_1_11_FN,	GPSR1_11,
+		GP_1_10_FN,	GPSR1_10,
+		GP_1_9_FN,	GPSR1_9,
+		GP_1_8_FN,	GPSR1_8,
+		GP_1_7_FN,	GPSR1_7,
+		GP_1_6_FN,	GPSR1_6,
+		GP_1_5_FN,	GPSR1_5,
+		GP_1_4_FN,	GPSR1_4,
+		GP_1_3_FN,	GPSR1_3,
+		GP_1_2_FN,	GPSR1_2,
+		GP_1_1_FN,	GPSR1_1,
+		GP_1_0_FN,	GPSR1_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_2_14_FN,	GPSR2_14,
+		GP_2_13_FN,	GPSR2_13,
+		GP_2_12_FN,	GPSR2_12,
+		GP_2_11_FN,	GPSR2_11,
+		GP_2_10_FN,	GPSR2_10,
+		GP_2_9_FN,	GPSR2_9,
+		GP_2_8_FN,	GPSR2_8,
+		GP_2_7_FN,	GPSR2_7,
+		GP_2_6_FN,	GPSR2_6,
+		GP_2_5_FN,	GPSR2_5,
+		GP_2_4_FN,	GPSR2_4,
+		GP_2_3_FN,	GPSR2_3,
+		GP_2_2_FN,	GPSR2_2,
+		GP_2_1_FN,	GPSR2_1,
+		GP_2_0_FN,	GPSR2_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_3_15_FN,	GPSR3_15,
+		GP_3_14_FN,	GPSR3_14,
+		GP_3_13_FN,	GPSR3_13,
+		GP_3_12_FN,	GPSR3_12,
+		GP_3_11_FN,	GPSR3_11,
+		GP_3_10_FN,	GPSR3_10,
+		GP_3_9_FN,	GPSR3_9,
+		GP_3_8_FN,	GPSR3_8,
+		GP_3_7_FN,	GPSR3_7,
+		GP_3_6_FN,	GPSR3_6,
+		GP_3_5_FN,	GPSR3_5,
+		GP_3_4_FN,	GPSR3_4,
+		GP_3_3_FN,	GPSR3_3,
+		GP_3_2_FN,	GPSR3_2,
+		GP_3_1_FN,	GPSR3_1,
+		GP_3_0_FN,	GPSR3_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_4_17_FN,	GPSR4_17,
+		GP_4_16_FN,	GPSR4_16,
+		GP_4_15_FN,	GPSR4_15,
+		GP_4_14_FN,	GPSR4_14,
+		GP_4_13_FN,	GPSR4_13,
+		GP_4_12_FN,	GPSR4_12,
+		GP_4_11_FN,	GPSR4_11,
+		GP_4_10_FN,	GPSR4_10,
+		GP_4_9_FN,	GPSR4_9,
+		GP_4_8_FN,	GPSR4_8,
+		GP_4_7_FN,	GPSR4_7,
+		GP_4_6_FN,	GPSR4_6,
+		GP_4_5_FN,	GPSR4_5,
+		GP_4_4_FN,	GPSR4_4,
+		GP_4_3_FN,	GPSR4_3,
+		GP_4_2_FN,	GPSR4_2,
+		GP_4_1_FN,	GPSR4_1,
+		GP_4_0_FN,	GPSR4_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_5_25_FN,	GPSR5_25,
+		GP_5_24_FN,	GPSR5_24,
+		GP_5_23_FN,	GPSR5_23,
+		GP_5_22_FN,	GPSR5_22,
+		GP_5_21_FN,	GPSR5_21,
+		GP_5_20_FN,	GPSR5_20,
+		GP_5_19_FN,	GPSR5_19,
+		GP_5_18_FN,	GPSR5_18,
+		GP_5_17_FN,	GPSR5_17,
+		GP_5_16_FN,	GPSR5_16,
+		GP_5_15_FN,	GPSR5_15,
+		GP_5_14_FN,	GPSR5_14,
+		GP_5_13_FN,	GPSR5_13,
+		GP_5_12_FN,	GPSR5_12,
+		GP_5_11_FN,	GPSR5_11,
+		GP_5_10_FN,	GPSR5_10,
+		GP_5_9_FN,	GPSR5_9,
+		GP_5_8_FN,	GPSR5_8,
+		GP_5_7_FN,	GPSR5_7,
+		GP_5_6_FN,	GPSR5_6,
+		GP_5_5_FN,	GPSR5_5,
+		GP_5_4_FN,	GPSR5_4,
+		GP_5_3_FN,	GPSR5_3,
+		GP_5_2_FN,	GPSR5_2,
+		GP_5_1_FN,	GPSR5_1,
+		GP_5_0_FN,	GPSR5_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1) {
+		GP_6_31_FN,	GPSR6_31,
+		GP_6_30_FN,	GPSR6_30,
+		GP_6_29_FN,	GPSR6_29,
+		GP_6_28_FN,	GPSR6_28,
+		GP_6_27_FN,	GPSR6_27,
+		GP_6_26_FN,	GPSR6_26,
+		GP_6_25_FN,	GPSR6_25,
+		GP_6_24_FN,	GPSR6_24,
+		GP_6_23_FN,	GPSR6_23,
+		GP_6_22_FN,	GPSR6_22,
+		GP_6_21_FN,	GPSR6_21,
+		GP_6_20_FN,	GPSR6_20,
+		GP_6_19_FN,	GPSR6_19,
+		GP_6_18_FN,	GPSR6_18,
+		GP_6_17_FN,	GPSR6_17,
+		GP_6_16_FN,	GPSR6_16,
+		GP_6_15_FN,	GPSR6_15,
+		GP_6_14_FN,	GPSR6_14,
+		GP_6_13_FN,	GPSR6_13,
+		GP_6_12_FN,	GPSR6_12,
+		GP_6_11_FN,	GPSR6_11,
+		GP_6_10_FN,	GPSR6_10,
+		GP_6_9_FN,	GPSR6_9,
+		GP_6_8_FN,	GPSR6_8,
+		GP_6_7_FN,	GPSR6_7,
+		GP_6_6_FN,	GPSR6_6,
+		GP_6_5_FN,	GPSR6_5,
+		GP_6_4_FN,	GPSR6_4,
+		GP_6_3_FN,	GPSR6_3,
+		GP_6_2_FN,	GPSR6_2,
+		GP_6_1_FN,	GPSR6_1,
+		GP_6_0_FN,	GPSR6_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR7", 0xe606011c, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_7_3_FN, GPSR7_3,
+		GP_7_2_FN, GPSR7_2,
+		GP_7_1_FN, GPSR7_1,
+		GP_7_0_FN, GPSR7_0, }
+	},
+#undef F_
+#undef FM
+
+#define F_(x, y)	x,
+#define FM(x)		FN_##x,
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+		IP0_31_28
+		IP0_27_24
+		IP0_23_20
+		IP0_19_16
+		IP0_15_12
+		IP0_11_8
+		IP0_7_4
+		IP0_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+		IP1_31_28
+		IP1_27_24
+		IP1_23_20
+		IP1_19_16
+		IP1_15_12
+		IP1_11_8
+		IP1_7_4
+		IP1_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+		IP2_31_28
+		IP2_27_24
+		IP2_23_20
+		IP2_19_16
+		IP2_15_12
+		IP2_11_8
+		IP2_7_4
+		IP2_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+		IP3_31_28
+		IP3_27_24
+		IP3_23_20
+		IP3_19_16
+		IP3_15_12
+		IP3_11_8
+		IP3_7_4
+		IP3_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+		IP4_31_28
+		IP4_27_24
+		IP4_23_20
+		IP4_19_16
+		IP4_15_12
+		IP4_11_8
+		IP4_7_4
+		IP4_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+		IP5_31_28
+		IP5_27_24
+		IP5_23_20
+		IP5_19_16
+		IP5_15_12
+		IP5_11_8
+		IP5_7_4
+		IP5_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+		IP6_31_28
+		IP6_27_24
+		IP6_23_20
+		IP6_19_16
+		IP6_15_12
+		IP6_11_8
+		IP6_7_4
+		IP6_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+		IP7_31_28
+		IP7_27_24
+		IP7_23_20
+		IP7_19_16
+		IP7_15_12
+		IP7_11_8
+		IP7_7_4
+		IP7_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+		IP8_31_28
+		IP8_27_24
+		IP8_23_20
+		IP8_19_16
+		IP8_15_12
+		IP8_11_8
+		IP8_7_4
+		IP8_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+		IP9_31_28
+		IP9_27_24
+		IP9_23_20
+		IP9_19_16
+		IP9_15_12
+		IP9_11_8
+		IP9_7_4
+		IP9_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+		IP10_31_28
+		IP10_27_24
+		IP10_23_20
+		IP10_19_16
+		IP10_15_12
+		IP10_11_8
+		IP10_7_4
+		IP10_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4) {
+		IP11_31_28
+		IP11_27_24
+		IP11_23_20
+		IP11_19_16
+		IP11_15_12
+		IP11_11_8
+		IP11_7_4
+		IP11_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4) {
+		IP12_31_28
+		IP12_27_24
+		IP12_23_20
+		IP12_19_16
+		IP12_15_12
+		IP12_11_8
+		IP12_7_4
+		IP12_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4) {
+		IP13_31_28
+		IP13_27_24
+		IP13_23_20
+		IP13_19_16
+		IP13_15_12
+		IP13_11_8
+		IP13_7_4
+		IP13_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4) {
+		IP14_31_28
+		IP14_27_24
+		IP14_23_20
+		IP14_19_16
+		IP14_15_12
+		IP14_11_8
+		IP14_7_4
+		IP14_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4) {
+		IP15_31_28
+		IP15_27_24
+		IP15_23_20
+		IP15_19_16
+		IP15_15_12
+		IP15_11_8
+		IP15_7_4
+		IP15_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR16", 0xe6060240, 32, 4) {
+		IP16_31_28
+		IP16_27_24
+		IP16_23_20
+		IP16_19_16
+		IP16_15_12
+		IP16_11_8
+		IP16_7_4
+		IP16_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR17", 0xe6060244, 32, 4) {
+		IP17_31_28
+		IP17_27_24
+		IP17_23_20
+		IP17_19_16
+		IP17_15_12
+		IP17_11_8
+		IP17_7_4
+		IP17_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR18", 0xe6060248, 32, 4) {
+		/* IP18_31_28 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP18_27_24 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP18_23_20 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP18_19_16 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP18_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP18_11_8  */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		IP18_7_4
+		IP18_3_0 }
+	},
+#undef F_
+#undef FM
+
+#define F_(x, y)	x,
+#define FM(x)		FN_##x,
+	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
+			     3, 2, 3, 1, 1, 1, 1, 1, 2, 1,
+			     1, 2, 1, 1, 1, 2, 2, 1, 2, 3) {
+		MOD_SEL0_31_30_29
+		MOD_SEL0_28_27
+		MOD_SEL0_26_25_24
+		MOD_SEL0_23
+		MOD_SEL0_22
+		MOD_SEL0_21
+		MOD_SEL0_20
+		MOD_SEL0_19
+		MOD_SEL0_18_17
+		MOD_SEL0_16
+		MOD_SEL0_15
+		MOD_SEL0_14_13
+		MOD_SEL0_12
+		MOD_SEL0_11
+		MOD_SEL0_10
+		MOD_SEL0_9_8
+		MOD_SEL0_7_6
+		MOD_SEL0_5
+		MOD_SEL0_4_3
+		/* RESERVED 2, 1, 0 */
+		0, 0, 0, 0, 0, 0, 0, 0 }
+	},
+	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xe6060504, 32,
+			     2, 3, 1, 2, 3, 1, 1, 2, 1,
+			     2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1) {
+		MOD_SEL1_31_30
+		MOD_SEL1_29_28_27
+		MOD_SEL1_26
+		MOD_SEL1_25_24
+		MOD_SEL1_23_22_21
+		MOD_SEL1_20
+		MOD_SEL1_19
+		MOD_SEL1_18_17
+		MOD_SEL1_16
+		MOD_SEL1_15_14
+		MOD_SEL1_13
+		MOD_SEL1_12
+		MOD_SEL1_11
+		MOD_SEL1_10
+		MOD_SEL1_9
+		0, 0, 0, 0, /* RESERVED 8, 7 */
+		MOD_SEL1_6
+		MOD_SEL1_5
+		MOD_SEL1_4
+		MOD_SEL1_3
+		MOD_SEL1_2
+		MOD_SEL1_1
+		MOD_SEL1_0 }
+	},
+	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xe6060508, 32,
+			     1, 1, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1, 1,
+			     4, 4, 4, 3, 1) {
+		MOD_SEL2_31
+		MOD_SEL2_30
+		MOD_SEL2_29
+		MOD_SEL2_28_27
+		MOD_SEL2_26
+		MOD_SEL2_25_24_23
+		MOD_SEL2_22
+		MOD_SEL2_21
+		MOD_SEL2_20
+		MOD_SEL2_19
+		MOD_SEL2_18
+		MOD_SEL2_17
+		/* RESERVED 16 */
+		0, 0,
+		/* RESERVED 15, 14, 13, 12 */
+		0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* RESERVED 11, 10, 9, 8 */
+		0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* RESERVED 7, 6, 5, 4 */
+		0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0,
+		/* RESERVED 3, 2, 1 */
+		0, 0, 0, 0, 0, 0, 0, 0,
+		MOD_SEL2_0 }
+	},
+	{ },
+};
+
+static int r8a7796_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin, u32 *pocctrl)
+{
+	int bit = -EINVAL;
+
+	*pocctrl = 0xe6060380;
+
+	if (pin >= RCAR_GP_PIN(3, 0) && pin <= RCAR_GP_PIN(3, 11))
+		bit = pin & 0x1f;
+
+	if (pin >= RCAR_GP_PIN(4, 0) && pin <= RCAR_GP_PIN(4, 17))
+		bit = (pin & 0x1f) + 12;
+
+	return bit;
+}
+
+static const struct sh_pfc_soc_operations r8a7796_pinmux_ops = {
+	.pin_to_pocctrl = r8a7796_pin_to_pocctrl,
+};
+
+const struct sh_pfc_soc_info r8a7796_pinmux_info = {
+	.name = "r8a77960_pfc",
+	.ops = &r8a7796_pinmux_ops,
+	.unlock_reg = 0xe6060000, /* PMMR */
+
+	.function = { PINMUX_FUNCTION_BEGIN, PINMUX_FUNCTION_END },
+
+	.pins = pinmux_pins,
+	.nr_pins = ARRAY_SIZE(pinmux_pins),
+	.groups = pinmux_groups,
+	.nr_groups = ARRAY_SIZE(pinmux_groups),
+	.functions = pinmux_functions,
+	.nr_functions = ARRAY_SIZE(pinmux_functions),
+
+	.cfg_regs = pinmux_config_regs,
+
+	.pinmux_data = pinmux_data,
+	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
+};
diff --git a/drivers/pinctrl/sh-pfc/pinctrl.c b/drivers/pinctrl/sh-pfc/pinctrl.c
index e208ee04a9f4..c5772584594c 100644
--- a/drivers/pinctrl/sh-pfc/pinctrl.c
+++ b/drivers/pinctrl/sh-pfc/pinctrl.c
@@ -596,6 +596,7 @@ static int sh_pfc_pinconf_get(struct pinctrl_dev *pctldev, unsigned _pin,
 	struct sh_pfc *pfc = pmx->pfc;
 	enum pin_config_param param = pinconf_to_config_param(*config);
 	unsigned long flags;
+	unsigned int arg;
 
 	if (!sh_pfc_pinconf_validate(pfc, _pin, param))
 		return -ENOTSUPP;
@@ -616,7 +617,7 @@ static int sh_pfc_pinconf_get(struct pinctrl_dev *pctldev, unsigned _pin,
 		if (bias != param)
 			return -EINVAL;
 
-		*config = 0;
+		arg = 0;
 		break;
 	}
 
@@ -627,7 +628,7 @@ static int sh_pfc_pinconf_get(struct pinctrl_dev *pctldev, unsigned _pin,
 		if (ret < 0)
 			return ret;
 
-		*config = ret;
+		arg = ret;
 		break;
 	}
 
@@ -646,7 +647,7 @@ static int sh_pfc_pinconf_get(struct pinctrl_dev *pctldev, unsigned _pin,
 		val = sh_pfc_read_reg(pfc, pocctrl, 32);
 		spin_unlock_irqrestore(&pfc->lock, flags);
 
-		*config = (val & BIT(bit)) ? 3300 : 1800;
+		arg = (val & BIT(bit)) ? 3300 : 1800;
 		break;
 	}
 
@@ -654,6 +655,7 @@ static int sh_pfc_pinconf_get(struct pinctrl_dev *pctldev, unsigned _pin,
 		return -ENOTSUPP;
 	}
 
+	*config = pinconf_to_config_packed(param, arg);
 	return 0;
 }
 
diff --git a/drivers/pinctrl/sh-pfc/sh_pfc.h b/drivers/pinctrl/sh-pfc/sh_pfc.h
index 5e966c09434d..2345421103db 100644
--- a/drivers/pinctrl/sh-pfc/sh_pfc.h
+++ b/drivers/pinctrl/sh-pfc/sh_pfc.h
@@ -257,9 +257,11 @@ extern const struct sh_pfc_soc_info r8a7778_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7779_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7790_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7791_pinmux_info;
+extern const struct sh_pfc_soc_info r8a7792_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7793_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7794_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7795_pinmux_info;
+extern const struct sh_pfc_soc_info r8a7796_pinmux_info;
 extern const struct sh_pfc_soc_info sh7203_pinmux_info;
 extern const struct sh_pfc_soc_info sh7264_pinmux_info;
 extern const struct sh_pfc_soc_info sh7269_pinmux_info;
@@ -354,72 +356,103 @@ extern const struct sh_pfc_soc_info shx3_pinmux_info;
  * GP port style (32 ports banks)
  */
 
-#define PORT_GP_CFG_1(bank, pin, fn, sfx, cfg) fn(bank, pin, GP_##bank##_##pin, sfx, cfg)
+#define PORT_GP_CFG_1(bank, pin, fn, sfx, cfg)				\
+	fn(bank, pin, GP_##bank##_##pin, sfx, cfg)
 #define PORT_GP_1(bank, pin, fn, sfx)	PORT_GP_CFG_1(bank, pin, fn, sfx, 0)
 
-#define PORT_GP_CFG_4(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_1(bank, 0,  fn, sfx, cfg), PORT_GP_CFG_1(bank, 1,  fn, sfx, cfg),	\
-	PORT_GP_CFG_1(bank, 2,  fn, sfx, cfg), PORT_GP_CFG_1(bank, 3,  fn, sfx, cfg)
+#define PORT_GP_CFG_4(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_1(bank, 0,  fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 1,  fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 2,  fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 3,  fn, sfx, cfg)
 #define PORT_GP_4(bank, fn, sfx)	PORT_GP_CFG_4(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_8(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_4(bank, fn, sfx, cfg),						\
-	PORT_GP_CFG_1(bank, 4,  fn, sfx, cfg), PORT_GP_CFG_1(bank, 5,  fn, sfx, cfg),	\
-	PORT_GP_CFG_1(bank, 6,  fn, sfx, cfg), PORT_GP_CFG_1(bank, 7,  fn, sfx, cfg)
+#define PORT_GP_CFG_8(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_4(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 4,  fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 5,  fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 6,  fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 7,  fn, sfx, cfg)
 #define PORT_GP_8(bank, fn, sfx)	PORT_GP_CFG_8(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_9(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_8(bank, fn, sfx, cfg),						\
+#define PORT_GP_CFG_9(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_8(bank, fn, sfx, cfg),				\
 	PORT_GP_CFG_1(bank, 8,  fn, sfx, cfg)
 #define PORT_GP_9(bank, fn, sfx)	PORT_GP_CFG_9(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_12(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_8(bank, fn, sfx, cfg),						\
-	PORT_GP_CFG_1(bank, 8,  fn, sfx, cfg), PORT_GP_CFG_1(bank, 9,  fn, sfx, cfg),	\
-	PORT_GP_CFG_1(bank, 10, fn, sfx, cfg), PORT_GP_CFG_1(bank, 11, fn, sfx, cfg)
+#define PORT_GP_CFG_12(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_9(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 9,  fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 10, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 11, fn, sfx, cfg)
 #define PORT_GP_12(bank, fn, sfx)	PORT_GP_CFG_12(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_14(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_12(bank, fn, sfx, cfg),						\
-	PORT_GP_CFG_1(bank, 12, fn, sfx, cfg), PORT_GP_CFG_1(bank, 13, fn, sfx, cfg)
+#define PORT_GP_CFG_14(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_12(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 12, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 13, fn, sfx, cfg)
 #define PORT_GP_14(bank, fn, sfx)	PORT_GP_CFG_14(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_15(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_14(bank, fn, sfx, cfg),						\
+#define PORT_GP_CFG_15(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_14(bank, fn, sfx, cfg),				\
 	PORT_GP_CFG_1(bank, 14, fn, sfx, cfg)
 #define PORT_GP_15(bank, fn, sfx)	PORT_GP_CFG_15(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_16(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_14(bank, fn, sfx, cfg),						\
-	PORT_GP_CFG_1(bank, 14, fn, sfx, cfg), PORT_GP_CFG_1(bank, 15, fn, sfx, cfg)
+#define PORT_GP_CFG_16(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_15(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 15, fn, sfx, cfg)
 #define PORT_GP_16(bank, fn, sfx)	PORT_GP_CFG_16(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_18(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_16(bank, fn, sfx, cfg),						\
-	PORT_GP_CFG_1(bank, 16, fn, sfx, cfg), PORT_GP_CFG_1(bank, 17, fn, sfx, cfg)
+#define PORT_GP_CFG_17(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_16(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 16, fn, sfx, cfg)
+#define PORT_GP_17(bank, fn, sfx)	PORT_GP_CFG_17(bank, fn, sfx, 0)
+
+#define PORT_GP_CFG_18(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_17(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 17, fn, sfx, cfg)
 #define PORT_GP_18(bank, fn, sfx)	PORT_GP_CFG_18(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_26(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_18(bank, fn, sfx, cfg),						\
-	PORT_GP_CFG_1(bank, 18, fn, sfx, cfg), PORT_GP_CFG_1(bank, 19, fn, sfx, cfg),	\
-	PORT_GP_CFG_1(bank, 20, fn, sfx, cfg), PORT_GP_CFG_1(bank, 21, fn, sfx, cfg),	\
-	PORT_GP_CFG_1(bank, 22, fn, sfx, cfg), PORT_GP_CFG_1(bank, 23, fn, sfx, cfg),	\
-	PORT_GP_CFG_1(bank, 24, fn, sfx, cfg), PORT_GP_CFG_1(bank, 25, fn, sfx, cfg)
+#define PORT_GP_CFG_23(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_18(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 18, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 19, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 20, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 21, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 22, fn, sfx, cfg)
+#define PORT_GP_23(bank, fn, sfx)	PORT_GP_CFG_23(bank, fn, sfx, 0)
+
+#define PORT_GP_CFG_24(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_23(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 23, fn, sfx, cfg)
+#define PORT_GP_24(bank, fn, sfx)	PORT_GP_CFG_24(bank, fn, sfx, 0)
+
+#define PORT_GP_CFG_26(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_24(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 24, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 25, fn, sfx, cfg)
 #define PORT_GP_26(bank, fn, sfx)	PORT_GP_CFG_26(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_28(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_26(bank, fn, sfx, cfg),						\
-	PORT_GP_CFG_1(bank, 26, fn, sfx, cfg), PORT_GP_CFG_1(bank, 27, fn, sfx, cfg)
+#define PORT_GP_CFG_28(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_26(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 26, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 27, fn, sfx, cfg)
 #define PORT_GP_28(bank, fn, sfx)	PORT_GP_CFG_28(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_30(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_28(bank, fn, sfx, cfg),						\
-	PORT_GP_CFG_1(bank, 28, fn, sfx, cfg), PORT_GP_CFG_1(bank, 29, fn, sfx, cfg)
+#define PORT_GP_CFG_29(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_28(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 28, fn, sfx, cfg)
+#define PORT_GP_29(bank, fn, sfx)	PORT_GP_CFG_29(bank, fn, sfx, 0)
+
+#define PORT_GP_CFG_30(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_29(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 29, fn, sfx, cfg)
 #define PORT_GP_30(bank, fn, sfx)	PORT_GP_CFG_30(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_32(bank, fn, sfx, cfg)						\
-	PORT_GP_CFG_30(bank, fn, sfx, cfg),						\
-	PORT_GP_CFG_1(bank, 30, fn, sfx, cfg), PORT_GP_CFG_1(bank, 31, fn, sfx, cfg)
+#define PORT_GP_CFG_32(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_30(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 30, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 31, fn, sfx, cfg)
 #define PORT_GP_32(bank, fn, sfx)	PORT_GP_CFG_32(bank, fn, sfx, 0)
 
 #define PORT_GP_32_REV(bank, fn, sfx)					\
diff --git a/drivers/pinctrl/sirf/pinctrl-atlas7.c b/drivers/pinctrl/sirf/pinctrl-atlas7.c
index 19952f73fa8c..7f3041697813 100644
--- a/drivers/pinctrl/sirf/pinctrl-atlas7.c
+++ b/drivers/pinctrl/sirf/pinctrl-atlas7.c
@@ -7,7 +7,7 @@
  * Licensed under GPLv2 or later.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/io.h>
 #include <linux/bitops.h>
@@ -6158,6 +6158,3 @@ static int __init atlas7_gpio_init(void)
 	return platform_driver_register(&atlas7_gpio_driver);
 }
 subsys_initcall(atlas7_gpio_init);
-
-MODULE_DESCRIPTION("SIRFSOC Atlas7 pin control driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/sirf/pinctrl-sirf.c b/drivers/pinctrl/sirf/pinctrl-sirf.c
index 762c0c9c1278..0df72be60704 100644
--- a/drivers/pinctrl/sirf/pinctrl-sirf.c
+++ b/drivers/pinctrl/sirf/pinctrl-sirf.c
@@ -1,6 +1,11 @@
 /*
  * pinmux driver for CSR SiRFprimaII
  *
+ * Authors:
+ *	Rongjun Ying <rongjun.ying@csr.com>
+ *	Yuping Luo <yuping.luo@csr.com>
+ *	Barry Song <baohua.song@csr.com>
+ *
  * Copyright (c) 2011 - 2014 Cambridge Silicon Radio Limited, a CSR plc group
  * company.
  *
@@ -8,7 +13,6 @@
  */
 
 #include <linux/init.h>
-#include <linux/module.h>
 #include <linux/irq.h>
 #include <linux/platform_device.h>
 #include <linux/io.h>
@@ -884,9 +888,3 @@ static int __init sirfsoc_gpio_init(void)
 	return sirfsoc_gpio_probe(np);
 }
 subsys_initcall(sirfsoc_gpio_init);
-
-MODULE_AUTHOR("Rongjun Ying <rongjun.ying@csr.com>");
-MODULE_AUTHOR("Yuping Luo <yuping.luo@csr.com>");
-MODULE_AUTHOR("Barry Song <baohua.song@csr.com>");
-MODULE_DESCRIPTION("SIRFSOC pin control driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/stm32/Kconfig b/drivers/pinctrl/stm32/Kconfig
index 4c40dae384d1..c03dce7a22df 100644
--- a/drivers/pinctrl/stm32/Kconfig
+++ b/drivers/pinctrl/stm32/Kconfig
@@ -6,16 +6,17 @@ config PINCTRL_STM32
 	select PINMUX
 	select GENERIC_PINCONF
 	select GPIOLIB
+	select MFD_SYSCON
 
 config PINCTRL_STM32F429
 	bool "STMicroelectronics STM32F429 pin control" if COMPILE_TEST && !MACH_STM32F429
-	depends on OF
+	depends on OF && IRQ_DOMAIN_HIERARCHY
 	default MACH_STM32F429
 	select PINCTRL_STM32
 
 config PINCTRL_STM32F746
 	bool "STMicroelectronics STM32F746 pin control" if COMPILE_TEST && !MACH_STM32F746
-	depends on OF
+	depends on OF && IRQ_DOMAIN_HIERARCHY
 	default MACH_STM32F746
 	select PINCTRL_STM32
 
diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c
index 4ae596bf19b5..200667f08c37 100644
--- a/drivers/pinctrl/stm32/pinctrl-stm32.c
+++ b/drivers/pinctrl/stm32/pinctrl-stm32.c
@@ -8,6 +8,8 @@
 #include <linux/clk.h>
 #include <linux/gpio/driver.h>
 #include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
@@ -20,6 +22,7 @@
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/pinctrl/pinmux.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
 #include <linux/reset.h>
 #include <linux/slab.h>
 
@@ -40,6 +43,7 @@
 #define STM32_GPIO_AFRH		0x24
 
 #define STM32_GPIO_PINS_PER_BANK 16
+#define STM32_GPIO_IRQ_LINE	 16
 
 #define gpio_range_to_bank(chip) \
 		container_of(chip, struct stm32_gpio_bank, range)
@@ -65,6 +69,8 @@ struct stm32_gpio_bank {
 	spinlock_t lock;
 	struct gpio_chip gpio_chip;
 	struct pinctrl_gpio_range range;
+	struct fwnode_handle *fwnode;
+	struct irq_domain *domain;
 };
 
 struct stm32_pinctrl {
@@ -77,6 +83,9 @@ struct stm32_pinctrl {
 	struct stm32_gpio_bank *banks;
 	unsigned nbanks;
 	const struct stm32_pinctrl_match_data *match_data;
+	struct irq_domain	*domain;
+	struct regmap		*regmap;
+	struct regmap_field	*irqmux[STM32_GPIO_PINS_PER_BANK];
 };
 
 static inline int stm32_gpio_pin(int gpio)
@@ -174,17 +183,113 @@ static int stm32_gpio_direction_output(struct gpio_chip *chip,
 	return 0;
 }
 
-static struct gpio_chip stm32_gpio_template = {
+
+static int stm32_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
+{
+	struct stm32_gpio_bank *bank = gpiochip_get_data(chip);
+	struct irq_fwspec fwspec;
+
+	fwspec.fwnode = bank->fwnode;
+	fwspec.param_count = 2;
+	fwspec.param[0] = offset;
+	fwspec.param[1] = IRQ_TYPE_NONE;
+
+	return irq_create_fwspec_mapping(&fwspec);
+}
+
+static const struct gpio_chip stm32_gpio_template = {
 	.request		= stm32_gpio_request,
 	.free			= stm32_gpio_free,
 	.get			= stm32_gpio_get,
 	.set			= stm32_gpio_set,
 	.direction_input	= stm32_gpio_direction_input,
 	.direction_output	= stm32_gpio_direction_output,
+	.to_irq			= stm32_gpio_to_irq,
 };
 
-/* Pinctrl functions */
+static struct irq_chip stm32_gpio_irq_chip = {
+	.name           = "stm32gpio",
+	.irq_eoi	= irq_chip_eoi_parent,
+	.irq_mask       = irq_chip_mask_parent,
+	.irq_unmask     = irq_chip_unmask_parent,
+	.irq_set_type   = irq_chip_set_type_parent,
+};
+
+static int stm32_gpio_domain_translate(struct irq_domain *d,
+				       struct irq_fwspec *fwspec,
+				       unsigned long *hwirq,
+				       unsigned int *type)
+{
+	if ((fwspec->param_count != 2) ||
+	    (fwspec->param[0] >= STM32_GPIO_IRQ_LINE))
+		return -EINVAL;
+
+	*hwirq = fwspec->param[0];
+	*type = fwspec->param[1];
+	return 0;
+}
 
+static void stm32_gpio_domain_activate(struct irq_domain *d,
+				       struct irq_data *irq_data)
+{
+	struct stm32_gpio_bank *bank = d->host_data;
+	struct stm32_pinctrl *pctl = dev_get_drvdata(bank->gpio_chip.parent);
+
+	regmap_field_write(pctl->irqmux[irq_data->hwirq], bank->range.id);
+}
+
+static int stm32_gpio_domain_alloc(struct irq_domain *d,
+				   unsigned int virq,
+				   unsigned int nr_irqs, void *data)
+{
+	struct stm32_gpio_bank *bank = d->host_data;
+	struct stm32_pinctrl *pctl = dev_get_drvdata(bank->gpio_chip.parent);
+	struct irq_fwspec *fwspec = data;
+	struct irq_fwspec parent_fwspec;
+	irq_hw_number_t hwirq;
+	int ret;
+
+	hwirq = fwspec->param[0];
+	parent_fwspec.fwnode = d->parent->fwnode;
+	parent_fwspec.param_count = 2;
+	parent_fwspec.param[0] = fwspec->param[0];
+	parent_fwspec.param[1] = fwspec->param[1];
+
+	irq_domain_set_hwirq_and_chip(d, virq, hwirq, &stm32_gpio_irq_chip,
+				      bank);
+
+	ret = gpiochip_lock_as_irq(&bank->gpio_chip, hwirq);
+	if (ret) {
+		dev_err(pctl->dev, "Unable to configure STM32 %s%ld as IRQ\n",
+			bank->gpio_chip.label, hwirq);
+		return ret;
+	}
+
+	ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &parent_fwspec);
+	if (ret)
+		gpiochip_unlock_as_irq(&bank->gpio_chip, hwirq);
+
+	return ret;
+}
+
+static void stm32_gpio_domain_free(struct irq_domain *d, unsigned int virq,
+				   unsigned int nr_irqs)
+{
+	struct stm32_gpio_bank *bank = d->host_data;
+	struct irq_data *data = irq_get_irq_data(virq);
+
+	irq_domain_free_irqs_common(d, virq, nr_irqs);
+	gpiochip_unlock_as_irq(&bank->gpio_chip, data->hwirq);
+}
+
+static const struct irq_domain_ops stm32_gpio_domain_ops = {
+	.translate      = stm32_gpio_domain_translate,
+	.alloc          = stm32_gpio_domain_alloc,
+	.free           = stm32_gpio_domain_free,
+	.activate	= stm32_gpio_domain_activate,
+};
+
+/* Pinctrl functions */
 static struct stm32_pinctrl_group *
 stm32_pctrl_find_group_by_pin(struct stm32_pinctrl *pctl, u32 pin)
 {
@@ -857,6 +962,17 @@ static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl,
 	range->pin_base = range->base = range->id * STM32_GPIO_PINS_PER_BANK;
 	range->npins = bank->gpio_chip.ngpio;
 	range->gc = &bank->gpio_chip;
+
+	/* create irq hierarchical domain */
+	bank->fwnode = of_node_to_fwnode(np);
+
+	bank->domain = irq_domain_create_hierarchy(pctl->domain, 0,
+					STM32_GPIO_IRQ_LINE, bank->fwnode,
+					&stm32_gpio_domain_ops, bank);
+
+	if (!bank->domain)
+		return -ENODEV;
+
 	err = gpiochip_add_data(&bank->gpio_chip, bank);
 	if (err) {
 		dev_err(dev, "Failed to add gpiochip(%d)!\n", bank_nr);
@@ -867,6 +983,47 @@ static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl,
 	return 0;
 }
 
+static int stm32_pctrl_dt_setup_irq(struct platform_device *pdev,
+			   struct stm32_pinctrl *pctl)
+{
+	struct device_node *np = pdev->dev.of_node, *parent;
+	struct device *dev = &pdev->dev;
+	struct regmap *rm;
+	int offset, ret, i;
+
+	parent = of_irq_find_parent(np);
+	if (!parent)
+		return -ENXIO;
+
+	pctl->domain = irq_find_host(parent);
+	if (!pctl->domain)
+		return -ENXIO;
+
+	pctl->regmap = syscon_regmap_lookup_by_phandle(np, "st,syscfg");
+	if (IS_ERR(pctl->regmap))
+		return PTR_ERR(pctl->regmap);
+
+	rm = pctl->regmap;
+
+	ret = of_property_read_u32_index(np, "st,syscfg", 1, &offset);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < STM32_GPIO_PINS_PER_BANK; i++) {
+		struct reg_field mux;
+
+		mux.reg = offset + (i / 4) * 4;
+		mux.lsb = (i % 4) * 4;
+		mux.msb = mux.lsb + 3;
+
+		pctl->irqmux[i] = devm_regmap_field_alloc(dev, rm, mux);
+		if (IS_ERR(pctl->irqmux[i]))
+			return PTR_ERR(pctl->irqmux[i]);
+	}
+
+	return 0;
+}
+
 static int stm32_pctrl_build_state(struct platform_device *pdev)
 {
 	struct stm32_pinctrl *pctl = platform_get_drvdata(pdev);
@@ -935,6 +1092,10 @@ int stm32_pctl_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+	ret = stm32_pctrl_dt_setup_irq(pdev, pctl);
+	if (ret)
+		return ret;
+
 	for_each_child_of_node(np, child)
 		if (of_property_read_bool(child, "gpio-controller"))
 			banks++;
diff --git a/drivers/pinctrl/sunxi/Kconfig b/drivers/pinctrl/sunxi/Kconfig
index aaf075b972f5..bff1ffc6f01e 100644
--- a/drivers/pinctrl/sunxi/Kconfig
+++ b/drivers/pinctrl/sunxi/Kconfig
@@ -17,6 +17,10 @@ config PINCTRL_SUN5I_A13
 	def_bool MACH_SUN5I
 	select PINCTRL_SUNXI
 
+config PINCTRL_GR8
+	def_bool MACH_SUN5I
+	select PINCTRL_SUNXI_COMMON
+
 config PINCTRL_SUN6I_A31
 	def_bool MACH_SUN6I
 	select PINCTRL_SUNXI
diff --git a/drivers/pinctrl/sunxi/Makefile b/drivers/pinctrl/sunxi/Makefile
index 2d8b64e222e0..95f93d0561fc 100644
--- a/drivers/pinctrl/sunxi/Makefile
+++ b/drivers/pinctrl/sunxi/Makefile
@@ -5,6 +5,7 @@ obj-y					+= pinctrl-sunxi.o
 obj-$(CONFIG_PINCTRL_SUN4I_A10)		+= pinctrl-sun4i-a10.o
 obj-$(CONFIG_PINCTRL_SUN5I_A10S)	+= pinctrl-sun5i-a10s.o
 obj-$(CONFIG_PINCTRL_SUN5I_A13)		+= pinctrl-sun5i-a13.o
+obj-$(CONFIG_PINCTRL_GR8)		+= pinctrl-gr8.o
 obj-$(CONFIG_PINCTRL_SUN6I_A31)		+= pinctrl-sun6i-a31.o
 obj-$(CONFIG_PINCTRL_SUN6I_A31S)	+= pinctrl-sun6i-a31s.o
 obj-$(CONFIG_PINCTRL_SUN6I_A31_R)	+= pinctrl-sun6i-a31-r.o
diff --git a/drivers/pinctrl/sunxi/pinctrl-gr8.c b/drivers/pinctrl/sunxi/pinctrl-gr8.c
new file mode 100644
index 000000000000..2904d2b7378b
--- /dev/null
+++ b/drivers/pinctrl/sunxi/pinctrl-gr8.c
@@ -0,0 +1,541 @@
+/*
+ * NextThing GR8 SoCs pinctrl driver.
+ *
+ * Copyright (C) 2016 Mylene Josserand
+ *
+ * Based on pinctrl-sun5i-a13.c
+ *
+ * Mylene Josserand <mylene.josserand@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2.  This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-sunxi.h"
+
+static const struct sunxi_desc_pin sun5i_gr8_pins[] = {
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 0),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2c0")),		/* SCK */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 1),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2c0")),		/* SDA */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 2),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "pwm0"),
+		  SUNXI_FUNCTION(0x3, "spdif"),		/* DO */
+		  SUNXI_FUNCTION_IRQ(0x6, 16)),		/* EINT16 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 3),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ir0"),		/* TX */
+		  SUNXI_FUNCTION_IRQ(0x6, 17)),		/* EINT17 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 4),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ir0"),		/* RX */
+		  SUNXI_FUNCTION_IRQ(0x6, 18)),		/* EINT18 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 5),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2s0"),		/* MCLK */
+		  SUNXI_FUNCTION_IRQ(0x6, 19)),		/* EINT19 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 6),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2s0"),		/* BCLK */
+		  SUNXI_FUNCTION_IRQ(0x6, 20)),		/* EINT20 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 7),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2s0"),		/* LRCK */
+		  SUNXI_FUNCTION_IRQ(0x6, 21)),		/* EINT21 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 8),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2s0"),		/* DO */
+		  SUNXI_FUNCTION_IRQ(0x6, 22)),		/* EINT22 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 9),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2s0"),		/* DI */
+		  SUNXI_FUNCTION(0x3, "spdif"),		/* DI */
+		  SUNXI_FUNCTION_IRQ(0x6, 23)),		/* EINT23 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 10),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi2"),		/* CS1 */
+		  SUNXI_FUNCTION(0x3, "spdif"),		/* DO */
+		  SUNXI_FUNCTION_IRQ(0x6, 24)),		/* EINT24 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 11),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi2"),		/* CS0 */
+		  SUNXI_FUNCTION(0x3, "jtag"),		/* MS0 */
+		  SUNXI_FUNCTION_IRQ(0x6, 25)),		/* EINT25 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 12),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi2"),		/* CLK */
+		  SUNXI_FUNCTION(0x3, "jtag"),		/* CK0 */
+		  SUNXI_FUNCTION_IRQ(0x6, 26)),		/* EINT26 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 13),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi2"),		/* MOSI */
+		  SUNXI_FUNCTION(0x3, "jtag"),		/* DO0 */
+		  SUNXI_FUNCTION_IRQ(0x6, 27)),		/* EINT27 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 14),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi2"),		/* MISO */
+		  SUNXI_FUNCTION(0x3, "jtag"),		/* DI0 */
+		  SUNXI_FUNCTION_IRQ(0x6, 28)),		/* EINT28 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 15),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2c1")),		/* SCK */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 16),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2c1")),		/* SDA */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 17),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2c2")),		/* SCK */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 18),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "i2c2")),		/* SDA */
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 0),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NWE */
+		  SUNXI_FUNCTION(0x3, "spi0")),		/* MOSI */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 1),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NALE */
+		  SUNXI_FUNCTION(0x3, "spi0")),		/* MISO */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 2),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NCLE */
+		  SUNXI_FUNCTION(0x3, "spi0")),		/* CLK */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 3),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NCE1 */
+		  SUNXI_FUNCTION(0x3, "spi0")),		/* CS0 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 4),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0")),	/* NCE0 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 5),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0")),	/* NRE */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 6),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NRB0 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* CMD */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 7),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NRB1 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* CLK */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 8),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ0 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* D0 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 9),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ1 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* D1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 10),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ2 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* D2 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 11),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ3 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* D3 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 12),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ4 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* D4 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 13),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ5 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* D5 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 14),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ6 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* D6 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 15),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ7 */
+		  SUNXI_FUNCTION(0x3, "mmc2")),		/* D7 */
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 19),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQS */
+		  SUNXI_FUNCTION(0x3, "uart2"),		/* RX */
+		  SUNXI_FUNCTION(0x4, "uart3")),	/* RTS */
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 2),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D2 */
+		  SUNXI_FUNCTION(0x3, "uart2")),	/* TX */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 3),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D3 */
+		  SUNXI_FUNCTION(0x3, "uart2")),	/* RX */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 4),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D4 */
+		  SUNXI_FUNCTION(0x3, "uart2")),	/* CTS */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 5),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D5 */
+		  SUNXI_FUNCTION(0x3, "uart2")),	/* RTS */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 6),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D6 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ECRS */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 7),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D7 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ECOL */
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 10),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D10 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ERXD0 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 11),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D11 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ERXD1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 12),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D12 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ERXD2 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 13),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D13 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ERXD3 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 14),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D14 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ERXCK */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 15),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D15 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ERXERR */
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 18),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D18 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ERXDV */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 19),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D19 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ETXD0 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 20),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D20 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ETXD1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 21),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D21 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ETXD2 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 22),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D22 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ETXD3 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 23),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* D23 */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ETXEN */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 24),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* CLK */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ETXCK */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 25),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* DE */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* ETXERR*/
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 26),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* HSYNC */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* EMDC */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 27),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "lcd0"),		/* VSYNC */
+		  SUNXI_FUNCTION(0x3, "emac")),		/* EMDIO */
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 0),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* CLK */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* PCLK */
+		  SUNXI_FUNCTION(0x4, "spi2"),		/* CS0 */
+		  SUNXI_FUNCTION_IRQ(0x6, 14)),		/* EINT14 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 1),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* ERR */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* MCLK */
+		  SUNXI_FUNCTION(0x4, "spi2"),		/* CLK */
+		  SUNXI_FUNCTION_IRQ(0x6, 15)),		/* EINT15 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 2),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* SYNC */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* HSYNC */
+		  SUNXI_FUNCTION(0x4, "spi2")),		/* MOSI */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 3),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* DVLD */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* VSYNC */
+		  SUNXI_FUNCTION(0x4, "spi2")),		/* MISO */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 4),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* D0 */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* D0 */
+		  SUNXI_FUNCTION(0x4, "mmc2")),		/* D0 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 5),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* D1 */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* D1 */
+		  SUNXI_FUNCTION(0x4, "mmc2")),		/* D1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 6),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* D2 */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* D2 */
+		  SUNXI_FUNCTION(0x4, "mmc2")),		/* D2 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 7),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* D3 */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* D3 */
+		  SUNXI_FUNCTION(0x4, "mmc2")),		/* D3 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 8),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* D4 */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* D4 */
+		  SUNXI_FUNCTION(0x4, "mmc2")),		/* CMD */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 9),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* D5 */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* D5 */
+		  SUNXI_FUNCTION(0x4, "mmc2")),		/* CLK */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 10),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* D6 */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* D6 */
+		  SUNXI_FUNCTION(0x4, "uart1")),	/* TX */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(E, 11),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "ts0"),		/* D7 */
+		  SUNXI_FUNCTION(0x3, "csi0"),		/* D7 */
+		  SUNXI_FUNCTION(0x4, "uart1")),	/* RX */
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(F, 0),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc0"),		/* D1 */
+		  SUNXI_FUNCTION(0x4, "jtag")),		/* MS1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(F, 1),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc0"),		/* D0 */
+		  SUNXI_FUNCTION(0x4, "jtag")),		/* DI1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(F, 2),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc0"),		/* CLK */
+		  SUNXI_FUNCTION(0x4, "uart0")),	/* TX */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(F, 3),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc0"),		/* CMD */
+		  SUNXI_FUNCTION(0x4, "jtag")),		/* DO1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(F, 4),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc0"),		/* D3 */
+		  SUNXI_FUNCTION(0x4, "uart0")),	/* RX */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(F, 5),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc0"),		/* D2 */
+		  SUNXI_FUNCTION(0x4, "jtag")),		/* CK1 */
+	/* Hole */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 0),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x2, "gps"),		/* CLK */
+		  SUNXI_FUNCTION_IRQ(0x6, 0)),		/* EINT0 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 1),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x2, "gps"),		/* SIGN */
+		  SUNXI_FUNCTION_IRQ(0x6, 1)),		/* EINT1 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 2),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x2, "gps"),		/* MAG */
+		  SUNXI_FUNCTION_IRQ(0x6, 2)),		/* EINT2 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 3),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc1"),		/* CMD */
+		  SUNXI_FUNCTION(0x3, "ms"),		/* BS */
+		  SUNXI_FUNCTION(0x4, "uart1"),		/* TX */
+		  SUNXI_FUNCTION_IRQ(0x6, 3)),		/* EINT3 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 4),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc1"),		/* CLK */
+		  SUNXI_FUNCTION(0x3, "ms"),		/* CLK */
+		  SUNXI_FUNCTION(0x4, "uart1"),		/* RX */
+		  SUNXI_FUNCTION_IRQ(0x6, 4)),		/* EINT4 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 5),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc1"),		/* D0 */
+		  SUNXI_FUNCTION(0x3, "ms"),		/* D0 */
+		  SUNXI_FUNCTION(0x4, "uart1"),		/* CTS */
+		  SUNXI_FUNCTION_IRQ(0x6, 5)),		/* EINT5 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 6),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc1"),		/* D1 */
+		  SUNXI_FUNCTION(0x3, "ms"),		/* D1 */
+		  SUNXI_FUNCTION(0x4, "uart1"),		/* RTS */
+		  SUNXI_FUNCTION(0x5, "uart2"),		/* RTS */
+		  SUNXI_FUNCTION_IRQ(0x6, 6)),		/* EINT6 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 7),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc1"),		/* D2 */
+		  SUNXI_FUNCTION(0x3, "ms"),		/* D2 */
+		  SUNXI_FUNCTION(0x5, "uart2"),		/* TX */
+		  SUNXI_FUNCTION_IRQ(0x6, 7)),		/* EINT7 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 8),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "mmc1"),		/* D3 */
+		  SUNXI_FUNCTION(0x3, "ms"),		/* D3 */
+		  SUNXI_FUNCTION(0x5, "uart2"),		/* RX */
+		  SUNXI_FUNCTION_IRQ(0x6, 8)),		/* EINT8 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 9),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi1"),		/* CS0 */
+		  SUNXI_FUNCTION(0x3, "uart3"),		/* TX */
+		  SUNXI_FUNCTION_IRQ(0x6, 9)),		/* EINT9 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 10),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi1"),		/* CLK */
+		  SUNXI_FUNCTION(0x3, "uart3"),		/* RX */
+		  SUNXI_FUNCTION_IRQ(0x6, 10)),		/* EINT10 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 11),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi1"),		/* MOSI */
+		  SUNXI_FUNCTION(0x3, "uart3"),		/* CTS */
+		  SUNXI_FUNCTION_IRQ(0x6, 11)),		/* EINT11 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 12),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi1"),		/* MISO */
+		  SUNXI_FUNCTION(0x3, "uart3"),		/* RTS */
+		  SUNXI_FUNCTION_IRQ(0x6, 12)),		/* EINT12 */
+	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 13),
+		  SUNXI_FUNCTION(0x0, "gpio_in"),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		  SUNXI_FUNCTION(0x2, "spi1"),		/* CS1 */
+		  SUNXI_FUNCTION(0x3, "pwm1"),
+		  SUNXI_FUNCTION(0x5, "uart2"),		/* CTS */
+		  SUNXI_FUNCTION_IRQ(0x6, 13)),		/* EINT13 */
+};
+
+static const struct sunxi_pinctrl_desc sun5i_gr8_pinctrl_data = {
+	.pins = sun5i_gr8_pins,
+	.npins = ARRAY_SIZE(sun5i_gr8_pins),
+	.irq_banks = 1,
+};
+
+static int sun5i_gr8_pinctrl_probe(struct platform_device *pdev)
+{
+	return sunxi_pinctrl_init(pdev,
+				  &sun5i_gr8_pinctrl_data);
+}
+
+static const struct of_device_id sun5i_gr8_pinctrl_match[] = {
+	{ .compatible = "nextthing,gr8-pinctrl", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, sun5i_gr8_pinctrl_match);
+
+static struct platform_driver sun5i_gr8_pinctrl_driver = {
+	.probe	= sun5i_gr8_pinctrl_probe,
+	.driver	= {
+		.name		= "gr8-pinctrl",
+		.of_match_table	= sun5i_gr8_pinctrl_match,
+	},
+};
+module_platform_driver(sun5i_gr8_pinctrl_driver);
+
+MODULE_AUTHOR("Mylene Josserand <mylene.josserand@free-electrons.com");
+MODULE_DESCRIPTION("NextThing GR8 pinctrl driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31.c b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31.c
index 022863ab0c58..a70b52957e24 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31.c
@@ -896,10 +896,18 @@ static const struct sunxi_desc_pin sun6i_a31_pins[] = {
 		  SUNXI_FUNCTION(0x1, "gpio_out")),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 27),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
-		  SUNXI_FUNCTION(0x1, "gpio_out")),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		/*
+		 * The SPDIF block is not referenced at all in the A31 user
+		 * manual. However it is described in the code leaked and the
+		 * configuration files supplied by vendors.
+		 */
+		  SUNXI_FUNCTION(0x3, "spdif")),        /* SPDIF IN */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 28),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
-		  SUNXI_FUNCTION(0x1, "gpio_out")),
+		  SUNXI_FUNCTION(0x1, "gpio_out"),
+		/* Undocumented mux function - see above */
+		  SUNXI_FUNCTION(0x3, "spdif")),        /* SPDIF OUT */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 29),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c
index ce483b03a263..f9d661e5c14a 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c
@@ -485,12 +485,12 @@ static const struct sunxi_desc_pin sun8i_a23_pins[] = {
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 8),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "uart2"),		/* RTS */
+		  SUNXI_FUNCTION(0x2, "uart1"),		/* RTS */
 		  SUNXI_FUNCTION_IRQ_BANK(0x4, 2, 8)),	/* PG_EINT8 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 9),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "uart2"),		/* CTS */
+		  SUNXI_FUNCTION(0x2, "uart1"),		/* CTS */
 		  SUNXI_FUNCTION_IRQ_BANK(0x4, 2, 9)),	/* PG_EINT9 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 10),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c
index 3040abe6f73a..3131cac2b76f 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c
@@ -407,12 +407,12 @@ static const struct sunxi_desc_pin sun8i_a33_pins[] = {
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 8),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "uart2"),		/* RTS */
+		  SUNXI_FUNCTION(0x2, "uart1"),		/* RTS */
 		  SUNXI_FUNCTION_IRQ_BANK(0x4, 1, 8)),	/* PG_EINT8 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 9),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "uart2"),		/* CTS */
+		  SUNXI_FUNCTION(0x2, "uart1"),		/* CTS */
 		  SUNXI_FUNCTION_IRQ_BANK(0x4, 1, 9)),	/* PG_EINT9 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 10),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-h3.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-h3.c
index 26a2ad3b651f..518a92df4418 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-h3.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-h3.c
@@ -60,7 +60,6 @@ static const struct sunxi_desc_pin sun8i_h3_pins[] = {
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "sim"),		/* PWREN */
-		  SUNXI_FUNCTION(0x3, "pwm1"),
 		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 6)),	/* PA_EINT6 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 7),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
index 54455af566ec..0facbea5f465 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
@@ -29,7 +29,6 @@
 #include <linux/slab.h>
 
 #include "../core.h"
-#include "../../gpio/gpiolib.h"
 #include "pinctrl-sunxi.h"
 
 static struct irq_chip sunxi_pinctrl_edge_irq_chip;
diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c
index 6c084b266651..04053fe1e980 100644
--- a/drivers/platform/chrome/cros_ec_proto.c
+++ b/drivers/platform/chrome/cros_ec_proto.c
@@ -19,6 +19,7 @@
 #include <linux/device.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <asm/unaligned.h>
 
 #define EC_COMMAND_RETRIES	50
 
@@ -234,11 +235,44 @@ static int cros_ec_host_command_proto_query_v2(struct cros_ec_device *ec_dev)
 	return ret;
 }
 
+static int cros_ec_get_host_command_version_mask(struct cros_ec_device *ec_dev,
+	u16 cmd, u32 *mask)
+{
+	struct ec_params_get_cmd_versions *pver;
+	struct ec_response_get_cmd_versions *rver;
+	struct cros_ec_command *msg;
+	int ret;
+
+	msg = kmalloc(sizeof(*msg) + max(sizeof(*rver), sizeof(*pver)),
+		      GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	msg->version = 0;
+	msg->command = EC_CMD_GET_CMD_VERSIONS;
+	msg->insize = sizeof(*rver);
+	msg->outsize = sizeof(*pver);
+
+	pver = (struct ec_params_get_cmd_versions *)msg->data;
+	pver->cmd = cmd;
+
+	ret = cros_ec_cmd_xfer(ec_dev, msg);
+	if (ret > 0) {
+		rver = (struct ec_response_get_cmd_versions *)msg->data;
+		*mask = rver->version_mask;
+	}
+
+	kfree(msg);
+
+	return ret;
+}
+
 int cros_ec_query_all(struct cros_ec_device *ec_dev)
 {
 	struct device *dev = ec_dev->dev;
 	struct cros_ec_command *proto_msg;
 	struct ec_response_get_protocol_info *proto_info;
+	u32 ver_mask = 0;
 	int ret;
 
 	proto_msg = kzalloc(sizeof(*proto_msg) + sizeof(*proto_info),
@@ -328,6 +362,15 @@ int cros_ec_query_all(struct cros_ec_device *ec_dev)
 		goto exit;
 	}
 
+	/* Probe if MKBP event is supported */
+	ret = cros_ec_get_host_command_version_mask(ec_dev,
+						    EC_CMD_GET_NEXT_EVENT,
+						    &ver_mask);
+	if (ret < 0 || ver_mask == 0)
+		ec_dev->mkbp_event_supported = 0;
+	else
+		ec_dev->mkbp_event_supported = 1;
+
 exit:
 	kfree(proto_msg);
 	return ret;
@@ -397,3 +440,52 @@ int cros_ec_cmd_xfer_status(struct cros_ec_device *ec_dev,
 	return ret;
 }
 EXPORT_SYMBOL(cros_ec_cmd_xfer_status);
+
+static int get_next_event(struct cros_ec_device *ec_dev)
+{
+	u8 buffer[sizeof(struct cros_ec_command) + sizeof(ec_dev->event_data)];
+	struct cros_ec_command *msg = (struct cros_ec_command *)&buffer;
+	int ret;
+
+	msg->version = 0;
+	msg->command = EC_CMD_GET_NEXT_EVENT;
+	msg->insize = sizeof(ec_dev->event_data);
+	msg->outsize = 0;
+
+	ret = cros_ec_cmd_xfer(ec_dev, msg);
+	if (ret > 0) {
+		ec_dev->event_size = ret - 1;
+		memcpy(&ec_dev->event_data, msg->data,
+		       sizeof(ec_dev->event_data));
+	}
+
+	return ret;
+}
+
+static int get_keyboard_state_event(struct cros_ec_device *ec_dev)
+{
+	u8 buffer[sizeof(struct cros_ec_command) +
+		  sizeof(ec_dev->event_data.data)];
+	struct cros_ec_command *msg = (struct cros_ec_command *)&buffer;
+
+	msg->version = 0;
+	msg->command = EC_CMD_MKBP_STATE;
+	msg->insize = sizeof(ec_dev->event_data.data);
+	msg->outsize = 0;
+
+	ec_dev->event_size = cros_ec_cmd_xfer(ec_dev, msg);
+	ec_dev->event_data.event_type = EC_MKBP_EVENT_KEY_MATRIX;
+	memcpy(&ec_dev->event_data.data, msg->data,
+	       sizeof(ec_dev->event_data.data));
+
+	return ec_dev->event_size;
+}
+
+int cros_ec_get_next_event(struct cros_ec_device *ec_dev)
+{
+	if (ec_dev->mkbp_event_supported)
+		return get_next_event(ec_dev);
+	else
+		return get_keyboard_state_event(ec_dev);
+}
+EXPORT_SYMBOL(cros_ec_get_next_event);
diff --git a/drivers/platform/olpc/olpc-ec.c b/drivers/platform/olpc/olpc-ec.c
index f99b183d5296..374a8028fec7 100644
--- a/drivers/platform/olpc/olpc-ec.c
+++ b/drivers/platform/olpc/olpc-ec.c
@@ -1,6 +1,8 @@
 /*
  * Generic driver for the OLPC Embedded Controller.
  *
+ * Author: Andres Salomon <dilinger@queued.net>
+ *
  * Copyright (C) 2011-2012 One Laptop per Child Foundation.
  *
  * Licensed under the GPL v2 or later.
@@ -12,7 +14,7 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/workqueue.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/list.h>
 #include <linux/olpc-ec.h>
 #include <asm/olpc.h>
@@ -326,8 +328,4 @@ static int __init olpc_ec_init_module(void)
 {
 	return platform_driver_register(&olpc_ec_plat_driver);
 }
-
 arch_initcall(olpc_ec_init_module);
-
-MODULE_AUTHOR("Andres Salomon <dilinger@queued.net>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/dell-smo8800.c b/drivers/platform/x86/dell-smo8800.c
index 0aec4fd4c48e..37e646034ef8 100644
--- a/drivers/platform/x86/dell-smo8800.c
+++ b/drivers/platform/x86/dell-smo8800.c
@@ -24,6 +24,7 @@
 #include <linux/acpi.h>
 #include <linux/interrupt.h>
 #include <linux/miscdevice.h>
+#include <linux/uaccess.h>
 
 struct smo8800_device {
 	u32 irq;                     /* acpi device irq */
diff --git a/drivers/platform/x86/dell-wmi.c b/drivers/platform/x86/dell-wmi.c
index d2bc092defd7..da2fe18162e1 100644
--- a/drivers/platform/x86/dell-wmi.c
+++ b/drivers/platform/x86/dell-wmi.c
@@ -110,8 +110,8 @@ static const struct key_entry dell_wmi_keymap_type_0000[] __initconst = {
 	/* BIOS error detected */
 	{ KE_IGNORE, 0xe00d, { KEY_RESERVED } },
 
-	/* Unknown, defined in ACPI DSDT */
-	/* { KE_IGNORE, 0xe00e, { KEY_RESERVED } }, */
+	/* Battery was removed or inserted */
+	{ KE_IGNORE, 0xe00e, { KEY_RESERVED } },
 
 	/* Wifi Catcher */
 	{ KE_KEY,    0xe011, { KEY_PROG2 } },
diff --git a/drivers/platform/x86/intel_pmc_ipc.c b/drivers/platform/x86/intel_pmc_ipc.c
index b86e1bcaa055..a511d518206b 100644
--- a/drivers/platform/x86/intel_pmc_ipc.c
+++ b/drivers/platform/x86/intel_pmc_ipc.c
@@ -651,11 +651,15 @@ static int ipc_create_pmc_devices(void)
 {
 	int ret;
 
-	ret = ipc_create_tco_device();
-	if (ret) {
-		dev_err(ipcdev.dev, "Failed to add tco platform device\n");
-		return ret;
+	/* If we have ACPI based watchdog use that instead */
+	if (!acpi_has_watchdog()) {
+		ret = ipc_create_tco_device();
+		if (ret) {
+			dev_err(ipcdev.dev, "Failed to add tco platform device\n");
+			return ret;
+		}
 	}
+
 	ret = ipc_create_punit_device();
 	if (ret) {
 		dev_err(ipcdev.dev, "Failed to add punit platform device\n");
diff --git a/drivers/platform/x86/intel_pmic_gpio.c b/drivers/platform/x86/intel_pmic_gpio.c
index 63b371d6ee55..91ae58510d92 100644
--- a/drivers/platform/x86/intel_pmic_gpio.c
+++ b/drivers/platform/x86/intel_pmic_gpio.c
@@ -1,6 +1,8 @@
 /* Moorestown PMIC GPIO (access through IPC) driver
  * Copyright (c) 2008 - 2009, Intel Corporation.
  *
+ * Author: Alek Du <alek.du@intel.com>
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
@@ -21,7 +23,6 @@
 
 #define pr_fmt(fmt) "%s: " fmt, __func__
 
-#include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/interrupt.h>
 #include <linux/delay.h>
@@ -322,9 +323,4 @@ static int __init platform_pmic_gpio_init(void)
 {
 	return platform_driver_register(&platform_pmic_gpio_driver);
 }
-
 subsys_initcall(platform_pmic_gpio_init);
-
-MODULE_AUTHOR("Alek Du <alek.du@intel.com>");
-MODULE_DESCRIPTION("Intel Moorestown PMIC GPIO driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c
index 1dba3598cfcb..c890a49587e4 100644
--- a/drivers/platform/x86/sony-laptop.c
+++ b/drivers/platform/x86/sony-laptop.c
@@ -4116,7 +4116,7 @@ static ssize_t sonypi_misc_read(struct file *file, char __user *buf,
 
 	if (ret > 0) {
 		struct inode *inode = file_inode(file);
-		inode->i_atime = current_fs_time(inode->i_sb);
+		inode->i_atime = current_time(inode);
 	}
 
 	return ret;
diff --git a/drivers/pnp/isapnp/core.c b/drivers/pnp/isapnp/core.c
index cf88f9b62445..d8bf5a13aa07 100644
--- a/drivers/pnp/isapnp/core.c
+++ b/drivers/pnp/isapnp/core.c
@@ -34,7 +34,7 @@
  *  2003-08-11	Resource Management Updates - Adam Belay <ambx1@neo.rr.com>
  */
 
-#include <linux/module.h>
+#include <linux/moduleparam.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/delay.h>
@@ -54,8 +54,6 @@ static int isapnp_rdp;		/* Read Data Port */
 static int isapnp_reset = 1;	/* reset all PnP cards (deactivate) */
 static int isapnp_verbose = 1;	/* verbose mode */
 
-MODULE_AUTHOR("Jaroslav Kysela <perex@perex.cz>");
-MODULE_DESCRIPTION("Generic ISA Plug & Play support");
 module_param(isapnp_disable, int, 0);
 MODULE_PARM_DESC(isapnp_disable, "ISA Plug & Play disable");
 module_param(isapnp_rdp, int, 0);
@@ -64,7 +62,6 @@ module_param(isapnp_reset, int, 0);
 MODULE_PARM_DESC(isapnp_reset, "ISA Plug & Play reset all cards");
 module_param(isapnp_verbose, int, 0);
 MODULE_PARM_DESC(isapnp_verbose, "ISA Plug & Play verbose mode");
-MODULE_LICENSE("GPL");
 
 #define _PIDXR		0x279
 #define _PNPWRP		0xa79
diff --git a/drivers/power/Kconfig b/drivers/power/Kconfig
index acd4a1524a1e..63454b5cac27 100644
--- a/drivers/power/Kconfig
+++ b/drivers/power/Kconfig
@@ -1,517 +1,3 @@
-menuconfig POWER_SUPPLY
-	bool "Power supply class support"
-	help
-	  Say Y here to enable power supply class support. This allows
-	  power supply (batteries, AC, USB) monitoring by userspace
-	  via sysfs and uevent (if available) and/or APM kernel interface
-	  (if selected below).
-
-if POWER_SUPPLY
-
-config POWER_SUPPLY_DEBUG
-	bool "Power supply debug"
-	help
-	  Say Y here to enable debugging messages for power supply class
-	  and drivers.
-
-config PDA_POWER
-	tristate "Generic PDA/phone power driver"
-	depends on !S390
-	help
-	  Say Y here to enable generic power driver for PDAs and phones with
-	  one or two external power supplies (AC/USB) connected to main and
-	  backup batteries, and optional builtin charger.
-
-config APM_POWER
-	tristate "APM emulation for class batteries"
-	depends on APM_EMULATION
-	help
-	  Say Y here to enable support APM status emulation using
-	  battery class devices.
-
-config GENERIC_ADC_BATTERY
-	tristate "Generic battery support using IIO"
-	depends on IIO
-	help
-	  Say Y here to enable support for the generic battery driver
-	  which uses IIO framework to read adc.
-
-config MAX8925_POWER
-	tristate "MAX8925 battery charger support"
-	depends on MFD_MAX8925
-	help
-	  Say Y here to enable support for the battery charger in the Maxim
-	  MAX8925 PMIC.
-
-config WM831X_BACKUP
-	tristate "WM831X backup battery charger support"
-	depends on MFD_WM831X
-	help
-	  Say Y here to enable support for the backup battery charger
-	  in the Wolfson Microelectronics WM831x PMICs.
-
-config WM831X_POWER
-	tristate "WM831X PMU support"
-	depends on MFD_WM831X
-	help
-	  Say Y here to enable support for the power management unit
-	  provided by Wolfson Microelectronics WM831x PMICs.
-
-config WM8350_POWER
-        tristate "WM8350 PMU support"
-        depends on MFD_WM8350
-        help
-          Say Y here to enable support for the power management unit
-	  provided by the Wolfson Microelectronics WM8350 PMIC.
-
-config TEST_POWER
-	tristate "Test power driver"
-	help
-	  This driver is used for testing. It's safe to say M here.
-
-config BATTERY_88PM860X
-	tristate "Marvell 88PM860x battery driver"
-	depends on MFD_88PM860X
-	help
-	  Say Y here to enable battery monitor for Marvell 88PM860x chip.
-
-config BATTERY_ACT8945A
-	tristate "Active-semi ACT8945A charger driver"
-	depends on MFD_ACT8945A || COMPILE_TEST
-	help
-	  Say Y here to enable support for power supply provided by
-	  Active-semi ActivePath ACT8945A charger.
-
-config BATTERY_DS2760
-	tristate "DS2760 battery driver (HP iPAQ & others)"
-	depends on W1 && W1_SLAVE_DS2760
-	help
-	  Say Y here to enable support for batteries with ds2760 chip.
-
-config BATTERY_DS2780
-	tristate "DS2780 battery driver"
-	depends on HAS_IOMEM
-	select W1
-	select W1_SLAVE_DS2780
-	help
-	  Say Y here to enable support for batteries with ds2780 chip.
-
-config BATTERY_DS2781
-	tristate "DS2781 battery driver"
-	depends on HAS_IOMEM
-	select W1
-	select W1_SLAVE_DS2781
-	help
-	  If you enable this you will have the DS2781 battery driver support.
-
-	  The battery monitor chip is used in many batteries/devices
-	  as the one who is responsible for charging/discharging/monitoring
-	  Li+ batteries.
-
-	  If you are unsure, say N.
-
-config BATTERY_DS2782
-	tristate "DS2782/DS2786 standalone gas-gauge"
-	depends on I2C
-	help
-	  Say Y here to enable support for the DS2782/DS2786 standalone battery
-	  gas-gauge.
-
-config BATTERY_PMU
-	tristate "Apple PMU battery"
-	depends on PPC32 && ADB_PMU
-	help
-	  Say Y here to expose battery information on Apple machines
-	  through the generic battery class.
-
-config BATTERY_OLPC
-	tristate "One Laptop Per Child battery"
-	depends on X86_32 && OLPC
-	help
-	  Say Y to enable support for the battery on the OLPC laptop.
-
-config BATTERY_TOSA
-	tristate "Sharp SL-6000 (tosa) battery"
-	depends on MACH_TOSA && MFD_TC6393XB && TOUCHSCREEN_WM97XX
-	help
-	  Say Y to enable support for the battery on the Sharp Zaurus
-	  SL-6000 (tosa) models.
-
-config BATTERY_COLLIE
-	tristate "Sharp SL-5500 (collie) battery"
-	depends on SA1100_COLLIE && MCP_UCB1200
-	help
-	  Say Y to enable support for the battery on the Sharp Zaurus
-	  SL-5500 (collie) models.
-
-config BATTERY_IPAQ_MICRO
-	tristate "iPAQ Atmel Micro ASIC battery driver"
-	depends on MFD_IPAQ_MICRO
-	help
-	  Choose this option if you want to monitor battery status on
-	  Compaq/HP iPAQ h3100 and h3600.
-
-config BATTERY_WM97XX
-	bool "WM97xx generic battery driver"
-	depends on TOUCHSCREEN_WM97XX=y
-	help
-	  Say Y to enable support for battery measured by WM97xx aux port.
-
-config BATTERY_SBS
-        tristate "SBS Compliant gas gauge"
-        depends on I2C
-        help
-	  Say Y to include support for SBS battery driver for SBS-compliant
-	  gas gauges.
-
-config BATTERY_BQ27XXX
-	tristate "BQ27xxx battery driver"
-	help
-	  Say Y here to enable support for batteries with BQ27xxx chips.
-
-config BATTERY_BQ27XXX_I2C
-	tristate "BQ27xxx I2C support"
-	depends on BATTERY_BQ27XXX
-	depends on I2C
-	default y
-	help
-	  Say Y here to enable support for batteries with BQ27xxx chips
-	  connected over an I2C bus.
-
-config BATTERY_DA9030
-	tristate "DA9030 battery driver"
-	depends on PMIC_DA903X
-	help
-	  Say Y here to enable support for batteries charger integrated into
-	  DA9030 PMIC.
-
-config BATTERY_DA9052
-	tristate "Dialog DA9052 Battery"
-	depends on PMIC_DA9052
-	help
-	  Say Y here to enable support for batteries charger integrated into
-	  DA9052 PMIC.
-
-config CHARGER_DA9150
-	tristate "Dialog Semiconductor DA9150 Charger support"
-	depends on MFD_DA9150
-	depends on DA9150_GPADC
-	depends on IIO
-	help
-	  Say Y here to enable support for charger unit of the DA9150
-	  Integrated Charger & Fuel-Gauge IC.
-
-	  This driver can also be built as a module. If so, the module will be
-	  called da9150-charger.
-
-config BATTERY_DA9150
-	tristate "Dialog Semiconductor DA9150 Fuel Gauge support"
-	depends on MFD_DA9150
-	help
-	  Say Y here to enable support for the Fuel-Gauge unit of the DA9150
-	  Integrated Charger & Fuel-Gauge IC
-
-	  This driver can also be built as a module. If so, the module will be
-	  called da9150-fg.
-
-config AXP288_CHARGER
-	tristate "X-Powers AXP288 Charger"
-	depends on MFD_AXP20X && EXTCON_AXP288
-	help
-	  Say yes here to have support X-Power AXP288 power management IC (PMIC)
-	  integrated charger.
-
-config AXP288_FUEL_GAUGE
-	tristate "X-Powers AXP288 Fuel Gauge"
-	depends on MFD_AXP20X && IIO
-	help
-	  Say yes here to have support for X-Power power management IC (PMIC)
-	  Fuel Gauge. The device provides battery statistics and status
-	  monitoring as well as alerts for battery over/under voltage and
-	  over/under temperature.
-
-config BATTERY_MAX17040
-	tristate "Maxim MAX17040 Fuel Gauge"
-	depends on I2C
-	help
-	  MAX17040 is fuel-gauge systems for lithium-ion (Li+) batteries
-	  in handheld and portable equipment. The MAX17040 is configured
-	  to operate with a single lithium cell
-
-config BATTERY_MAX17042
-	tristate "Maxim MAX17042/17047/17050/8997/8966 Fuel Gauge"
-	depends on I2C
-	select REGMAP_I2C
-	help
-	  MAX17042 is fuel-gauge systems for lithium-ion (Li+) batteries
-	  in handheld and portable equipment. The MAX17042 is configured
-	  to operate with a single lithium cell. MAX8997 and MAX8966 are
-	  multi-function devices that include fuel gauages that are compatible
-	  with MAX17042. This driver also supports max17047/50 chips which are
-	  improved version of max17042.
-
-config BATTERY_Z2
-	tristate "Z2 battery driver"
-	depends on I2C && MACH_ZIPIT2
-	help
-	  Say Y to include support for the battery on the Zipit Z2.
-
-config BATTERY_S3C_ADC
-	tristate "Battery driver for Samsung ADC based monitoring"
-	depends on S3C_ADC
-	help
-	  Say Y here to enable support for iPAQ h1930/h1940/rx1950 battery
-
-config BATTERY_TWL4030_MADC
-	tristate "TWL4030 MADC battery driver"
-	depends on TWL4030_MADC
-	help
-	  Say Y here to enable this dumb driver for batteries managed
-	  through the TWL4030 MADC.
-
-config CHARGER_88PM860X
-	tristate "Marvell 88PM860x Charger driver"
-	depends on MFD_88PM860X && BATTERY_88PM860X
-	help
-	  Say Y here to enable charger for Marvell 88PM860x chip.
-
-config CHARGER_PCF50633
-	tristate "NXP PCF50633 MBC"
-	depends on MFD_PCF50633
-	help
-	 Say Y to include support for NXP PCF50633 Main Battery Charger.
-
-config BATTERY_JZ4740
-	tristate "Ingenic JZ4740 battery"
-	depends on MACH_JZ4740
-	depends on MFD_JZ4740_ADC
-	help
-	  Say Y to enable support for the battery on Ingenic JZ4740 based
-	  boards.
-
-	  This driver can be build as a module. If so, the module will be
-	  called jz4740-battery.
-
-config BATTERY_INTEL_MID
-	tristate "Battery driver for Intel MID platforms"
-	depends on INTEL_SCU_IPC && SPI
-	help
-	  Say Y here to enable the battery driver on Intel MID
-	  platforms.
-
-config BATTERY_RX51
-	tristate "Nokia RX-51 (N900) battery driver"
-	depends on TWL4030_MADC
-	help
-	  Say Y here to enable support for battery information on Nokia
-	  RX-51, also known as N900 tablet.
-
-config CHARGER_ISP1704
-	tristate "ISP1704 USB Charger Detection"
-	depends on USB_PHY
-	depends on USB_GADGET || !USB_GADGET # if USB_GADGET=m, this can't be 'y'
-	help
-	  Say Y to enable support for USB Charger Detection with
-	  ISP1707/ISP1704 USB transceivers.
-
-config CHARGER_MAX8903
-	tristate "MAX8903 Battery DC-DC Charger for USB and Adapter Power"
-	help
-	  Say Y to enable support for the MAX8903 DC-DC charger and sysfs.
-	  The driver supports controlling charger-enable and current-limit
-	  pins based on the status of charger connections with interrupt
-	  handlers.
-
-config CHARGER_TWL4030
-	tristate "OMAP TWL4030 BCI charger driver"
-	depends on IIO && TWL4030_CORE
-	help
-	  Say Y here to enable support for TWL4030 Battery Charge Interface.
-
-config CHARGER_LP8727
-	tristate "TI/National Semiconductor LP8727 charger driver"
-	depends on I2C
-	help
-	  Say Y here to enable support for LP8727 Charger Driver.
-
-config CHARGER_LP8788
-	tristate "TI LP8788 charger driver"
-	depends on MFD_LP8788
-	depends on LP8788_ADC
-	depends on IIO
-	help
-	  Say Y to enable support for the LP8788 linear charger.
-
-config CHARGER_GPIO
-	tristate "GPIO charger"
-	depends on GPIOLIB || COMPILE_TEST
-	help
-	  Say Y to include support for chargers which report their online status
-	  through a GPIO pin.
-
-	  This driver can be build as a module. If so, the module will be
-	  called gpio-charger.
-
-config CHARGER_MANAGER
-	bool "Battery charger manager for multiple chargers"
-	depends on REGULATOR
-	select EXTCON
-	help
-          Say Y to enable charger-manager support, which allows multiple
-          chargers attached to a battery and multiple batteries attached to a
-          system. The charger-manager also can monitor charging status in
-          runtime and in suspend-to-RAM by waking up the system periodically
-          with help of suspend_again support.
-
-config CHARGER_MAX14577
-	tristate "Maxim MAX14577/77836 battery charger driver"
-	depends on MFD_MAX14577
-	help
-	  Say Y to enable support for the battery charger control sysfs and
-	  platform data of MAX14577/77836 MUICs.
-
-config CHARGER_MAX77693
-	tristate "Maxim MAX77693 battery charger driver"
-	depends on MFD_MAX77693
-	help
-	  Say Y to enable support for the Maxim MAX77693 battery charger.
-
-config CHARGER_MAX8997
-	tristate "Maxim MAX8997/MAX8966 PMIC battery charger driver"
-	depends on MFD_MAX8997 && REGULATOR_MAX8997
-	help
-	  Say Y to enable support for the battery charger control sysfs and
-	  platform data of MAX8997/LP3974 PMICs.
-
-config CHARGER_MAX8998
-	tristate "Maxim MAX8998/LP3974 PMIC battery charger driver"
-	depends on MFD_MAX8998 && REGULATOR_MAX8998
-	help
-	  Say Y to enable support for the battery charger control sysfs and
-	  platform data of MAX8998/LP3974 PMICs.
-
-config CHARGER_QCOM_SMBB
-	tristate "Qualcomm Switch-Mode Battery Charger and Boost"
-	depends on MFD_SPMI_PMIC || COMPILE_TEST
-	depends on OF
-	depends on EXTCON
-	help
-	  Say Y to include support for the Switch-Mode Battery Charger and
-	  Boost (SMBB) hardware found in Qualcomm PM8941 PMICs.  The charger
-	  is an integrated, single-cell lithium-ion battery charger.  DT
-	  configuration is required for loading, see the devicetree
-	  documentation for more detail.  The base name for this driver is
-	  'pm8941_charger'.
-
-config CHARGER_BQ2415X
-	tristate "TI BQ2415x battery charger driver"
-	depends on I2C
-	help
-	  Say Y to enable support for the TI BQ2415x battery charger
-	  PMICs.
-
-	  You'll need this driver to charge batteries on e.g. Nokia
-	  RX-51/N900.
-
-config CHARGER_BQ24190
-	tristate "TI BQ24190 battery charger driver"
-	depends on I2C
-	depends on GPIOLIB || COMPILE_TEST
-	help
-	  Say Y to enable support for the TI BQ24190 battery charger.
-
-config CHARGER_BQ24257
-	tristate "TI BQ24250/24251/24257 battery charger driver"
-	depends on I2C
-	depends on GPIOLIB || COMPILE_TEST
-	depends on REGMAP_I2C
-	help
-	  Say Y to enable support for the TI BQ24250, BQ24251, and BQ24257 battery
-	  chargers.
-
-config CHARGER_BQ24735
-	tristate "TI BQ24735 battery charger support"
-	depends on I2C
-	depends on GPIOLIB || COMPILE_TEST
-	help
-	  Say Y to enable support for the TI BQ24735 battery charger.
-
-config CHARGER_BQ25890
-	tristate "TI BQ25890 battery charger driver"
-	depends on I2C
-	depends on GPIOLIB || COMPILE_TEST
-	select REGMAP_I2C
-	help
-	  Say Y to enable support for the TI BQ25890 battery charger.
-
-config CHARGER_SMB347
-	tristate "Summit Microelectronics SMB347 Battery Charger"
-	depends on I2C
-	select REGMAP_I2C
-	help
-	  Say Y to include support for Summit Microelectronics SMB347
-	  Battery Charger.
-
-config CHARGER_TPS65090
-	tristate "TPS65090 battery charger driver"
-	depends on MFD_TPS65090
-	help
-	 Say Y here to enable support for battery charging with TPS65090
-	 PMIC chips.
-
-config CHARGER_TPS65217
-	tristate "TPS65217 battery charger driver"
-	depends on MFD_TPS65217
-	help
-	 Say Y here to enable support for battery charging with TPS65217
-	 PMIC chips.
-
-config BATTERY_GAUGE_LTC2941
-	tristate "LTC2941/LTC2943 Battery Gauge Driver"
-	depends on I2C
-	help
-	  Say Y here to include support for LTC2941 and LTC2943 Battery
-	  Gauge IC. The driver reports the charge count continuously, and
-	  measures the voltage and temperature every 10 seconds.
-
-config AB8500_BM
-	bool "AB8500 Battery Management Driver"
-	depends on AB8500_CORE && AB8500_GPADC
-	help
-	  Say Y to include support for AB8500 battery management.
-
-config BATTERY_GOLDFISH
-	tristate "Goldfish battery driver"
-	depends on GOLDFISH || COMPILE_TEST
-	depends on HAS_IOMEM
-	help
-	  Say Y to enable support for the battery and AC power in the
-	  Goldfish emulator.
-
-config BATTERY_RT5033
-	tristate "RT5033 fuel gauge support"
-	depends on MFD_RT5033
-	help
-	  This adds support for battery fuel gauge in Richtek RT5033 PMIC.
-	  The fuelgauge calculates and determines the battery state of charge
-	  according to battery open circuit voltage.
-
-config CHARGER_RT9455
-	tristate "Richtek RT9455 battery charger driver"
-	depends on I2C
-	depends on GPIOLIB || COMPILE_TEST
-	select REGMAP_I2C
-	help
-	  Say Y to enable support for Richtek RT9455 battery charger.
-
-config AXP20X_POWER
-	tristate "AXP20x power supply driver"
-	depends on MFD_AXP20X
-	help
-	  This driver provides support for the power supply features of
-	  AXP20x PMIC.
-
-endif # POWER_SUPPLY
-
-source "drivers/power/reset/Kconfig"
 source "drivers/power/avs/Kconfig"
+source "drivers/power/reset/Kconfig"
+source "drivers/power/supply/Kconfig"
diff --git a/drivers/power/Makefile b/drivers/power/Makefile
index e46b75d448a5..ff35c712d824 100644
--- a/drivers/power/Makefile
+++ b/drivers/power/Makefile
@@ -1,76 +1,3 @@
-subdir-ccflags-$(CONFIG_POWER_SUPPLY_DEBUG) := -DDEBUG
-
-power_supply-y				:= power_supply_core.o
-power_supply-$(CONFIG_SYSFS)		+= power_supply_sysfs.o
-power_supply-$(CONFIG_LEDS_TRIGGERS)	+= power_supply_leds.o
-
-obj-$(CONFIG_POWER_SUPPLY)	+= power_supply.o
-obj-$(CONFIG_GENERIC_ADC_BATTERY)	+= generic-adc-battery.o
-
-obj-$(CONFIG_PDA_POWER)		+= pda_power.o
-obj-$(CONFIG_APM_POWER)		+= apm_power.o
-obj-$(CONFIG_AXP20X_POWER)	+= axp20x_usb_power.o
-obj-$(CONFIG_MAX8925_POWER)	+= max8925_power.o
-obj-$(CONFIG_WM831X_BACKUP)	+= wm831x_backup.o
-obj-$(CONFIG_WM831X_POWER)	+= wm831x_power.o
-obj-$(CONFIG_WM8350_POWER)	+= wm8350_power.o
-obj-$(CONFIG_TEST_POWER)	+= test_power.o
-
-obj-$(CONFIG_BATTERY_88PM860X)	+= 88pm860x_battery.o
-obj-$(CONFIG_BATTERY_ACT8945A)	+= act8945a_charger.o
-obj-$(CONFIG_BATTERY_DS2760)	+= ds2760_battery.o
-obj-$(CONFIG_BATTERY_DS2780)	+= ds2780_battery.o
-obj-$(CONFIG_BATTERY_DS2781)	+= ds2781_battery.o
-obj-$(CONFIG_BATTERY_DS2782)	+= ds2782_battery.o
-obj-$(CONFIG_BATTERY_GAUGE_LTC2941)	+= ltc2941-battery-gauge.o
-obj-$(CONFIG_BATTERY_GOLDFISH)	+= goldfish_battery.o
-obj-$(CONFIG_BATTERY_PMU)	+= pmu_battery.o
-obj-$(CONFIG_BATTERY_OLPC)	+= olpc_battery.o
-obj-$(CONFIG_BATTERY_TOSA)	+= tosa_battery.o
-obj-$(CONFIG_BATTERY_COLLIE)	+= collie_battery.o
-obj-$(CONFIG_BATTERY_IPAQ_MICRO) += ipaq_micro_battery.o
-obj-$(CONFIG_BATTERY_WM97XX)	+= wm97xx_battery.o
-obj-$(CONFIG_BATTERY_SBS)	+= sbs-battery.o
-obj-$(CONFIG_BATTERY_BQ27XXX)	+= bq27xxx_battery.o
-obj-$(CONFIG_BATTERY_BQ27XXX_I2C) += bq27xxx_battery_i2c.o
-obj-$(CONFIG_BATTERY_DA9030)	+= da9030_battery.o
-obj-$(CONFIG_BATTERY_DA9052)	+= da9052-battery.o
-obj-$(CONFIG_CHARGER_DA9150)	+= da9150-charger.o
-obj-$(CONFIG_BATTERY_DA9150)	+= da9150-fg.o
-obj-$(CONFIG_BATTERY_MAX17040)	+= max17040_battery.o
-obj-$(CONFIG_BATTERY_MAX17042)	+= max17042_battery.o
-obj-$(CONFIG_BATTERY_Z2)	+= z2_battery.o
-obj-$(CONFIG_BATTERY_RT5033)	+= rt5033_battery.o
-obj-$(CONFIG_CHARGER_RT9455)	+= rt9455_charger.o
-obj-$(CONFIG_BATTERY_S3C_ADC)	+= s3c_adc_battery.o
-obj-$(CONFIG_BATTERY_TWL4030_MADC)	+= twl4030_madc_battery.o
-obj-$(CONFIG_CHARGER_88PM860X)	+= 88pm860x_charger.o
-obj-$(CONFIG_CHARGER_PCF50633)	+= pcf50633-charger.o
-obj-$(CONFIG_BATTERY_JZ4740)	+= jz4740-battery.o
-obj-$(CONFIG_BATTERY_INTEL_MID)	+= intel_mid_battery.o
-obj-$(CONFIG_BATTERY_RX51)	+= rx51_battery.o
-obj-$(CONFIG_AB8500_BM)		+= ab8500_bmdata.o ab8500_charger.o ab8500_fg.o ab8500_btemp.o abx500_chargalg.o pm2301_charger.o
-obj-$(CONFIG_CHARGER_ISP1704)	+= isp1704_charger.o
-obj-$(CONFIG_CHARGER_MAX8903)	+= max8903_charger.o
-obj-$(CONFIG_CHARGER_TWL4030)	+= twl4030_charger.o
-obj-$(CONFIG_CHARGER_LP8727)	+= lp8727_charger.o
-obj-$(CONFIG_CHARGER_LP8788)	+= lp8788-charger.o
-obj-$(CONFIG_CHARGER_GPIO)	+= gpio-charger.o
-obj-$(CONFIG_CHARGER_MANAGER)	+= charger-manager.o
-obj-$(CONFIG_CHARGER_MAX14577)	+= max14577_charger.o
-obj-$(CONFIG_CHARGER_MAX77693)	+= max77693_charger.o
-obj-$(CONFIG_CHARGER_MAX8997)	+= max8997_charger.o
-obj-$(CONFIG_CHARGER_MAX8998)	+= max8998_charger.o
-obj-$(CONFIG_CHARGER_QCOM_SMBB)	+= qcom_smbb.o
-obj-$(CONFIG_CHARGER_BQ2415X)	+= bq2415x_charger.o
-obj-$(CONFIG_CHARGER_BQ24190)	+= bq24190_charger.o
-obj-$(CONFIG_CHARGER_BQ24257)	+= bq24257_charger.o
-obj-$(CONFIG_CHARGER_BQ24735)	+= bq24735-charger.o
-obj-$(CONFIG_CHARGER_BQ25890)	+= bq25890_charger.o
 obj-$(CONFIG_POWER_AVS)		+= avs/
-obj-$(CONFIG_CHARGER_SMB347)	+= smb347-charger.o
-obj-$(CONFIG_CHARGER_TPS65090)	+= tps65090-charger.o
-obj-$(CONFIG_CHARGER_TPS65217)	+= tps65217_charger.o
 obj-$(CONFIG_POWER_RESET)	+= reset/
-obj-$(CONFIG_AXP288_FUEL_GAUGE) += axp288_fuel_gauge.o
-obj-$(CONFIG_AXP288_CHARGER)	+= axp288_charger.o
+obj-$(CONFIG_POWER_SUPPLY)	+= supply/
diff --git a/drivers/power/act8945a_charger.c b/drivers/power/act8945a_charger.c
deleted file mode 100644
index b5c00e45741e..000000000000
--- a/drivers/power/act8945a_charger.c
+++ /dev/null
@@ -1,359 +0,0 @@
-/*
- * Power supply driver for the Active-semi ACT8945A PMIC
- *
- * Copyright (C) 2015 Atmel Corporation
- *
- * Author: Wenyou Yang <wenyou.yang@atmel.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_gpio.h>
-#include <linux/platform_device.h>
-#include <linux/power_supply.h>
-#include <linux/regmap.h>
-
-static const char *act8945a_charger_model = "ACT8945A";
-static const char *act8945a_charger_manufacturer = "Active-semi";
-
-/**
- * ACT8945A Charger Register Map
- */
-
-/* 0x70: Reserved */
-#define ACT8945A_APCH_CFG		0x71
-#define ACT8945A_APCH_STATUS		0x78
-#define ACT8945A_APCH_CTRL		0x79
-#define ACT8945A_APCH_STATE		0x7A
-
-/* ACT8945A_APCH_CFG */
-#define APCH_CFG_OVPSET			(0x3 << 0)
-#define APCH_CFG_OVPSET_6V6		(0x0 << 0)
-#define APCH_CFG_OVPSET_7V		(0x1 << 0)
-#define APCH_CFG_OVPSET_7V5		(0x2 << 0)
-#define APCH_CFG_OVPSET_8V		(0x3 << 0)
-#define APCH_CFG_PRETIMO		(0x3 << 2)
-#define APCH_CFG_PRETIMO_40_MIN		(0x0 << 2)
-#define APCH_CFG_PRETIMO_60_MIN		(0x1 << 2)
-#define APCH_CFG_PRETIMO_80_MIN		(0x2 << 2)
-#define APCH_CFG_PRETIMO_DISABLED	(0x3 << 2)
-#define APCH_CFG_TOTTIMO		(0x3 << 4)
-#define APCH_CFG_TOTTIMO_3_HOUR		(0x0 << 4)
-#define APCH_CFG_TOTTIMO_4_HOUR		(0x1 << 4)
-#define APCH_CFG_TOTTIMO_5_HOUR		(0x2 << 4)
-#define APCH_CFG_TOTTIMO_DISABLED	(0x3 << 4)
-#define APCH_CFG_SUSCHG			(0x1 << 7)
-
-#define APCH_STATUS_CHGDAT		BIT(0)
-#define APCH_STATUS_INDAT		BIT(1)
-#define APCH_STATUS_TEMPDAT		BIT(2)
-#define APCH_STATUS_TIMRDAT		BIT(3)
-#define APCH_STATUS_CHGSTAT		BIT(4)
-#define APCH_STATUS_INSTAT		BIT(5)
-#define APCH_STATUS_TEMPSTAT		BIT(6)
-#define APCH_STATUS_TIMRSTAT		BIT(7)
-
-#define APCH_CTRL_CHGEOCOUT		BIT(0)
-#define APCH_CTRL_INDIS			BIT(1)
-#define APCH_CTRL_TEMPOUT		BIT(2)
-#define APCH_CTRL_TIMRPRE		BIT(3)
-#define APCH_CTRL_CHGEOCIN		BIT(4)
-#define APCH_CTRL_INCON			BIT(5)
-#define APCH_CTRL_TEMPIN		BIT(6)
-#define APCH_CTRL_TIMRTOT		BIT(7)
-
-#define APCH_STATE_ACINSTAT		(0x1 << 1)
-#define APCH_STATE_CSTATE		(0x3 << 4)
-#define APCH_STATE_CSTATE_SHIFT		4
-#define APCH_STATE_CSTATE_DISABLED	0x00
-#define APCH_STATE_CSTATE_EOC		0x01
-#define APCH_STATE_CSTATE_FAST		0x02
-#define APCH_STATE_CSTATE_PRE		0x03
-
-struct act8945a_charger {
-	struct regmap *regmap;
-	bool battery_temperature;
-};
-
-static int act8945a_get_charger_state(struct regmap *regmap, int *val)
-{
-	int ret;
-	unsigned int status, state;
-
-	ret = regmap_read(regmap, ACT8945A_APCH_STATUS, &status);
-	if (ret < 0)
-		return ret;
-
-	ret = regmap_read(regmap, ACT8945A_APCH_STATE, &state);
-	if (ret < 0)
-		return ret;
-
-	state &= APCH_STATE_CSTATE;
-	state >>= APCH_STATE_CSTATE_SHIFT;
-
-	if (state == APCH_STATE_CSTATE_EOC) {
-		if (status & APCH_STATUS_CHGDAT)
-			*val = POWER_SUPPLY_STATUS_FULL;
-		else
-			*val = POWER_SUPPLY_STATUS_NOT_CHARGING;
-	} else if ((state == APCH_STATE_CSTATE_FAST) ||
-		   (state == APCH_STATE_CSTATE_PRE)) {
-		*val = POWER_SUPPLY_STATUS_CHARGING;
-	} else {
-		*val = POWER_SUPPLY_STATUS_NOT_CHARGING;
-	}
-
-	return 0;
-}
-
-static int act8945a_get_charge_type(struct regmap *regmap, int *val)
-{
-	int ret;
-	unsigned int state;
-
-	ret = regmap_read(regmap, ACT8945A_APCH_STATE, &state);
-	if (ret < 0)
-		return ret;
-
-	state &= APCH_STATE_CSTATE;
-	state >>= APCH_STATE_CSTATE_SHIFT;
-
-	switch (state) {
-	case APCH_STATE_CSTATE_PRE:
-		*val = POWER_SUPPLY_CHARGE_TYPE_TRICKLE;
-		break;
-	case APCH_STATE_CSTATE_FAST:
-		*val = POWER_SUPPLY_CHARGE_TYPE_FAST;
-		break;
-	case APCH_STATE_CSTATE_EOC:
-	case APCH_STATE_CSTATE_DISABLED:
-	default:
-		*val = POWER_SUPPLY_CHARGE_TYPE_NONE;
-	}
-
-	return 0;
-}
-
-static int act8945a_get_battery_health(struct act8945a_charger *charger,
-				       struct regmap *regmap, int *val)
-{
-	int ret;
-	unsigned int status;
-
-	ret = regmap_read(regmap, ACT8945A_APCH_STATUS, &status);
-	if (ret < 0)
-		return ret;
-
-	if (charger->battery_temperature && !(status & APCH_STATUS_TEMPDAT))
-		*val = POWER_SUPPLY_HEALTH_OVERHEAT;
-	else if (!(status & APCH_STATUS_INDAT))
-		*val = POWER_SUPPLY_HEALTH_OVERVOLTAGE;
-	else if (status & APCH_STATUS_TIMRDAT)
-		*val = POWER_SUPPLY_HEALTH_SAFETY_TIMER_EXPIRE;
-	else
-		*val = POWER_SUPPLY_HEALTH_GOOD;
-
-	return 0;
-}
-
-static enum power_supply_property act8945a_charger_props[] = {
-	POWER_SUPPLY_PROP_STATUS,
-	POWER_SUPPLY_PROP_CHARGE_TYPE,
-	POWER_SUPPLY_PROP_TECHNOLOGY,
-	POWER_SUPPLY_PROP_HEALTH,
-	POWER_SUPPLY_PROP_MODEL_NAME,
-	POWER_SUPPLY_PROP_MANUFACTURER
-};
-
-static int act8945a_charger_get_property(struct power_supply *psy,
-					 enum power_supply_property prop,
-					 union power_supply_propval *val)
-{
-	struct act8945a_charger *charger = power_supply_get_drvdata(psy);
-	struct regmap *regmap = charger->regmap;
-	int ret = 0;
-
-	switch (prop) {
-	case POWER_SUPPLY_PROP_STATUS:
-		ret = act8945a_get_charger_state(regmap, &val->intval);
-		break;
-	case POWER_SUPPLY_PROP_CHARGE_TYPE:
-		ret = act8945a_get_charge_type(regmap, &val->intval);
-		break;
-	case POWER_SUPPLY_PROP_TECHNOLOGY:
-		val->intval = POWER_SUPPLY_TECHNOLOGY_LION;
-		break;
-	case POWER_SUPPLY_PROP_HEALTH:
-		ret = act8945a_get_battery_health(charger,
-						  regmap, &val->intval);
-		break;
-	case POWER_SUPPLY_PROP_MODEL_NAME:
-		val->strval = act8945a_charger_model;
-		break;
-	case POWER_SUPPLY_PROP_MANUFACTURER:
-		val->strval = act8945a_charger_manufacturer;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-static const struct power_supply_desc act8945a_charger_desc = {
-	.name		= "act8945a-charger",
-	.type		= POWER_SUPPLY_TYPE_BATTERY,
-	.get_property	= act8945a_charger_get_property,
-	.properties	= act8945a_charger_props,
-	.num_properties	= ARRAY_SIZE(act8945a_charger_props),
-};
-
-#define DEFAULT_TOTAL_TIME_OUT		3
-#define DEFAULT_PRE_TIME_OUT		40
-#define DEFAULT_INPUT_OVP_THRESHOLD	6600
-
-static int act8945a_charger_config(struct device *dev,
-				   struct act8945a_charger *charger)
-{
-	struct device_node *np = dev->of_node;
-	enum of_gpio_flags flags;
-	struct regmap *regmap = charger->regmap;
-
-	u32 total_time_out;
-	u32 pre_time_out;
-	u32 input_voltage_threshold;
-	int chglev_pin;
-
-	unsigned int value = 0;
-
-	if (!np) {
-		dev_err(dev, "no charger of node\n");
-		return -EINVAL;
-	}
-
-	charger->battery_temperature = of_property_read_bool(np,
-				"active-semi,check-battery-temperature");
-
-	chglev_pin = of_get_named_gpio_flags(np,
-				"active-semi,chglev-gpios", 0, &flags);
-
-	if (gpio_is_valid(chglev_pin)) {
-		gpio_set_value(chglev_pin,
-			       ((flags == OF_GPIO_ACTIVE_LOW) ? 0 : 1));
-	}
-
-	if (of_property_read_u32(np,
-				 "active-semi,input-voltage-threshold-microvolt",
-				 &input_voltage_threshold))
-		input_voltage_threshold = DEFAULT_INPUT_OVP_THRESHOLD;
-
-	if (of_property_read_u32(np,
-				 "active-semi,precondition-timeout",
-				 &pre_time_out))
-		pre_time_out = DEFAULT_PRE_TIME_OUT;
-
-	if (of_property_read_u32(np, "active-semi,total-timeout",
-				 &total_time_out))
-		total_time_out = DEFAULT_TOTAL_TIME_OUT;
-
-	switch (input_voltage_threshold) {
-	case 8000:
-		value |= APCH_CFG_OVPSET_8V;
-		break;
-	case 7500:
-		value |= APCH_CFG_OVPSET_7V5;
-		break;
-	case 7000:
-		value |= APCH_CFG_OVPSET_7V;
-		break;
-	case 6600:
-	default:
-		value |= APCH_CFG_OVPSET_6V6;
-		break;
-	}
-
-	switch (pre_time_out) {
-	case 60:
-		value |= APCH_CFG_PRETIMO_60_MIN;
-		break;
-	case 80:
-		value |= APCH_CFG_PRETIMO_80_MIN;
-		break;
-	case 0:
-		value |= APCH_CFG_PRETIMO_DISABLED;
-		break;
-	case 40:
-	default:
-		value |= APCH_CFG_PRETIMO_40_MIN;
-		break;
-	}
-
-	switch (total_time_out) {
-	case 4:
-		value |= APCH_CFG_TOTTIMO_4_HOUR;
-		break;
-	case 5:
-		value |= APCH_CFG_TOTTIMO_5_HOUR;
-		break;
-	case 0:
-		value |= APCH_CFG_TOTTIMO_DISABLED;
-		break;
-	case 3:
-	default:
-		value |= APCH_CFG_TOTTIMO_3_HOUR;
-		break;
-	}
-
-	return regmap_write(regmap, ACT8945A_APCH_CFG, value);
-}
-
-static int act8945a_charger_probe(struct platform_device *pdev)
-{
-	struct act8945a_charger *charger;
-	struct power_supply *psy;
-	struct power_supply_config psy_cfg = {};
-	int ret;
-
-	charger = devm_kzalloc(&pdev->dev, sizeof(*charger), GFP_KERNEL);
-	if (!charger)
-		return -ENOMEM;
-
-	charger->regmap = dev_get_regmap(pdev->dev.parent, NULL);
-	if (!charger->regmap) {
-		dev_err(&pdev->dev, "Parent did not provide regmap\n");
-		return -EINVAL;
-	}
-
-	ret = act8945a_charger_config(pdev->dev.parent, charger);
-	if (ret)
-		return ret;
-
-	psy_cfg.of_node	= pdev->dev.parent->of_node;
-	psy_cfg.drv_data = charger;
-
-	psy = devm_power_supply_register(&pdev->dev,
-					 &act8945a_charger_desc,
-					 &psy_cfg);
-	if (IS_ERR(psy)) {
-		dev_err(&pdev->dev, "failed to register power supply\n");
-		return PTR_ERR(psy);
-	}
-
-	return 0;
-}
-
-static struct platform_driver act8945a_charger_driver = {
-	.driver	= {
-		.name = "act8945a-charger",
-	},
-	.probe	= act8945a_charger_probe,
-};
-module_platform_driver(act8945a_charger_driver);
-
-MODULE_DESCRIPTION("Active-semi ACT8945A ActivePath charger driver");
-MODULE_AUTHOR("Wenyou Yang <wenyou.yang@atmel.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/power/avs/smartreflex.c b/drivers/power/avs/smartreflex.c
index db9973bb53f1..fa0f19b975a6 100644
--- a/drivers/power/avs/smartreflex.c
+++ b/drivers/power/avs/smartreflex.c
@@ -136,7 +136,7 @@ static void sr_set_clk_length(struct omap_sr *sr)
 
 	if (IS_ERR(fck)) {
 		dev_err(&sr->pdev->dev, "%s: unable to get fck for device %s\n",
-				__func__, dev_name(&sr->pdev->dev));
+			__func__, dev_name(&sr->pdev->dev));
 		return;
 	}
 
@@ -170,8 +170,8 @@ static void sr_start_vddautocomp(struct omap_sr *sr)
 {
 	if (!sr_class || !(sr_class->enable) || !(sr_class->configure)) {
 		dev_warn(&sr->pdev->dev,
-			"%s: smartreflex class driver not registered\n",
-			__func__);
+			 "%s: smartreflex class driver not registered\n",
+			 __func__);
 		return;
 	}
 
@@ -183,8 +183,8 @@ static void sr_stop_vddautocomp(struct omap_sr *sr)
 {
 	if (!sr_class || !(sr_class->disable)) {
 		dev_warn(&sr->pdev->dev,
-			"%s: smartreflex class driver not registered\n",
-			__func__);
+			 "%s: smartreflex class driver not registered\n",
+			 __func__);
 		return;
 	}
 
@@ -225,9 +225,8 @@ static int sr_late_init(struct omap_sr *sr_info)
 
 error:
 	list_del(&sr_info->node);
-	dev_err(&sr_info->pdev->dev, "%s: ERROR in registering"
-		"interrupt handler. Smartreflex will"
-		"not function as desired\n", __func__);
+	dev_err(&sr_info->pdev->dev, "%s: ERROR in registering interrupt handler. Smartreflex will not function as desired\n",
+		__func__);
 
 	return ret;
 }
@@ -263,7 +262,7 @@ static void sr_v1_disable(struct omap_sr *sr)
 
 	if (timeout >= SR_DISABLE_TIMEOUT)
 		dev_warn(&sr->pdev->dev, "%s: Smartreflex disable timedout\n",
-			__func__);
+			 __func__);
 
 	/* Disable MCUDisableAcknowledge interrupt & clear pending interrupt */
 	sr_modify_reg(sr, ERRCONFIG_V1, ERRCONFIG_MCUDISACKINTEN,
@@ -308,7 +307,7 @@ static void sr_v2_disable(struct omap_sr *sr)
 
 	if (timeout >= SR_DISABLE_TIMEOUT)
 		dev_warn(&sr->pdev->dev, "%s: Smartreflex disable timedout\n",
-			__func__);
+			 __func__);
 
 	/* Disable MCUDisableAcknowledge interrupt & clear pending interrupt */
 	sr_write_reg(sr, IRQENABLE_CLR, IRQENABLE_MCUDISABLEACKINT);
@@ -322,7 +321,7 @@ static struct omap_sr_nvalue_table *sr_retrieve_nvalue_row(
 
 	if (!sr->nvalue_table) {
 		dev_warn(&sr->pdev->dev, "%s: Missing ntarget value table\n",
-			__func__);
+			 __func__);
 		return NULL;
 	}
 
@@ -356,8 +355,8 @@ int sr_configure_errgen(struct omap_sr *sr)
 	u8 senp_shift, senn_shift;
 
 	if (!sr) {
-		pr_warn("%s: NULL omap_sr from %pF\n", __func__,
-			(void *)_RET_IP_);
+		pr_warn("%s: NULL omap_sr from %pF\n",
+			__func__, (void *)_RET_IP_);
 		return -EINVAL;
 	}
 
@@ -387,8 +386,8 @@ int sr_configure_errgen(struct omap_sr *sr)
 		vpboundint_st = ERRCONFIG_VPBOUNDINTST_V2;
 		break;
 	default:
-		dev_err(&sr->pdev->dev, "%s: Trying to Configure smartreflex"
-			"module without specifying the ip\n", __func__);
+		dev_err(&sr->pdev->dev, "%s: Trying to Configure smartreflex module without specifying the ip\n",
+			__func__);
 		return -EINVAL;
 	}
 
@@ -423,8 +422,8 @@ int sr_disable_errgen(struct omap_sr *sr)
 	u32 vpboundint_en, vpboundint_st;
 
 	if (!sr) {
-		pr_warn("%s: NULL omap_sr from %pF\n", __func__,
-			(void *)_RET_IP_);
+		pr_warn("%s: NULL omap_sr from %pF\n",
+			__func__, (void *)_RET_IP_);
 		return -EINVAL;
 	}
 
@@ -440,8 +439,8 @@ int sr_disable_errgen(struct omap_sr *sr)
 		vpboundint_st = ERRCONFIG_VPBOUNDINTST_V2;
 		break;
 	default:
-		dev_err(&sr->pdev->dev, "%s: Trying to Configure smartreflex"
-			"module without specifying the ip\n", __func__);
+		dev_err(&sr->pdev->dev, "%s: Trying to Configure smartreflex module without specifying the ip\n",
+			__func__);
 		return -EINVAL;
 	}
 
@@ -478,8 +477,8 @@ int sr_configure_minmax(struct omap_sr *sr)
 	u8 senp_shift, senn_shift;
 
 	if (!sr) {
-		pr_warn("%s: NULL omap_sr from %pF\n", __func__,
-			(void *)_RET_IP_);
+		pr_warn("%s: NULL omap_sr from %pF\n",
+			__func__, (void *)_RET_IP_);
 		return -EINVAL;
 	}
 
@@ -504,8 +503,8 @@ int sr_configure_minmax(struct omap_sr *sr)
 		senp_shift = SRCONFIG_SENPENABLE_V2_SHIFT;
 		break;
 	default:
-		dev_err(&sr->pdev->dev, "%s: Trying to Configure smartreflex"
-			"module without specifying the ip\n", __func__);
+		dev_err(&sr->pdev->dev, "%s: Trying to Configure smartreflex module without specifying the ip\n",
+			__func__);
 		return -EINVAL;
 	}
 
@@ -537,8 +536,8 @@ int sr_configure_minmax(struct omap_sr *sr)
 			IRQENABLE_MCUBOUNDSINT | IRQENABLE_MCUDISABLEACKINT);
 		break;
 	default:
-		dev_err(&sr->pdev->dev, "%s: Trying to Configure smartreflex"
-			"module without specifying the ip\n", __func__);
+		dev_err(&sr->pdev->dev, "%s: Trying to Configure smartreflex module without specifying the ip\n",
+			__func__);
 		return -EINVAL;
 	}
 
@@ -563,16 +562,16 @@ int sr_enable(struct omap_sr *sr, unsigned long volt)
 	int ret;
 
 	if (!sr) {
-		pr_warn("%s: NULL omap_sr from %pF\n", __func__,
-			(void *)_RET_IP_);
+		pr_warn("%s: NULL omap_sr from %pF\n",
+			__func__, (void *)_RET_IP_);
 		return -EINVAL;
 	}
 
 	volt_data = omap_voltage_get_voltdata(sr->voltdm, volt);
 
 	if (IS_ERR(volt_data)) {
-		dev_warn(&sr->pdev->dev, "%s: Unable to get voltage table"
-			"for nominal voltage %ld\n", __func__, volt);
+		dev_warn(&sr->pdev->dev, "%s: Unable to get voltage table for nominal voltage %ld\n",
+			 __func__, volt);
 		return PTR_ERR(volt_data);
 	}
 
@@ -615,8 +614,8 @@ int sr_enable(struct omap_sr *sr, unsigned long volt)
 void sr_disable(struct omap_sr *sr)
 {
 	if (!sr) {
-		pr_warn("%s: NULL omap_sr from %pF\n", __func__,
-			(void *)_RET_IP_);
+		pr_warn("%s: NULL omap_sr from %pF\n",
+			__func__, (void *)_RET_IP_);
 		return;
 	}
 
@@ -658,13 +657,13 @@ int sr_register_class(struct omap_sr_class_data *class_data)
 	struct omap_sr *sr_info;
 
 	if (!class_data) {
-		pr_warning("%s:, Smartreflex class data passed is NULL\n",
+		pr_warn("%s:, Smartreflex class data passed is NULL\n",
 			__func__);
 		return -EINVAL;
 	}
 
 	if (sr_class) {
-		pr_warning("%s: Smartreflex class driver already registered\n",
+		pr_warn("%s: Smartreflex class driver already registered\n",
 			__func__);
 		return -EBUSY;
 	}
@@ -696,7 +695,7 @@ void omap_sr_enable(struct voltagedomain *voltdm)
 	struct omap_sr *sr = _sr_lookup(voltdm);
 
 	if (IS_ERR(sr)) {
-		pr_warning("%s: omap_sr struct for voltdm not found\n",	__func__);
+		pr_warn("%s: omap_sr struct for voltdm not found\n", __func__);
 		return;
 	}
 
@@ -704,8 +703,8 @@ void omap_sr_enable(struct voltagedomain *voltdm)
 		return;
 
 	if (!sr_class || !(sr_class->enable) || !(sr_class->configure)) {
-		dev_warn(&sr->pdev->dev, "%s: smartreflex class driver not"
-			"registered\n", __func__);
+		dev_warn(&sr->pdev->dev, "%s: smartreflex class driver not registered\n",
+			 __func__);
 		return;
 	}
 
@@ -728,7 +727,7 @@ void omap_sr_disable(struct voltagedomain *voltdm)
 	struct omap_sr *sr = _sr_lookup(voltdm);
 
 	if (IS_ERR(sr)) {
-		pr_warning("%s: omap_sr struct for voltdm not found\n",	__func__);
+		pr_warn("%s: omap_sr struct for voltdm not found\n", __func__);
 		return;
 	}
 
@@ -736,8 +735,8 @@ void omap_sr_disable(struct voltagedomain *voltdm)
 		return;
 
 	if (!sr_class || !(sr_class->disable)) {
-		dev_warn(&sr->pdev->dev, "%s: smartreflex class driver not"
-			"registered\n", __func__);
+		dev_warn(&sr->pdev->dev, "%s: smartreflex class driver not registered\n",
+			 __func__);
 		return;
 	}
 
@@ -760,7 +759,7 @@ void omap_sr_disable_reset_volt(struct voltagedomain *voltdm)
 	struct omap_sr *sr = _sr_lookup(voltdm);
 
 	if (IS_ERR(sr)) {
-		pr_warning("%s: omap_sr struct for voltdm not found\n",	__func__);
+		pr_warn("%s: omap_sr struct for voltdm not found\n", __func__);
 		return;
 	}
 
@@ -768,8 +767,8 @@ void omap_sr_disable_reset_volt(struct voltagedomain *voltdm)
 		return;
 
 	if (!sr_class || !(sr_class->disable)) {
-		dev_warn(&sr->pdev->dev, "%s: smartreflex class driver not"
-			"registered\n", __func__);
+		dev_warn(&sr->pdev->dev, "%s: smartreflex class driver not registered\n",
+			 __func__);
 		return;
 	}
 
@@ -787,8 +786,8 @@ void omap_sr_disable_reset_volt(struct voltagedomain *voltdm)
 void omap_sr_register_pmic(struct omap_sr_pmic_data *pmic_data)
 {
 	if (!pmic_data) {
-		pr_warning("%s: Trying to register NULL PMIC data structure"
-			"with smartreflex\n", __func__);
+		pr_warn("%s: Trying to register NULL PMIC data structure with smartreflex\n",
+			__func__);
 		return;
 	}
 
@@ -801,7 +800,7 @@ static int omap_sr_autocomp_show(void *data, u64 *val)
 	struct omap_sr *sr_info = data;
 
 	if (!sr_info) {
-		pr_warning("%s: omap_sr struct not found\n", __func__);
+		pr_warn("%s: omap_sr struct not found\n", __func__);
 		return -EINVAL;
 	}
 
@@ -815,13 +814,13 @@ static int omap_sr_autocomp_store(void *data, u64 val)
 	struct omap_sr *sr_info = data;
 
 	if (!sr_info) {
-		pr_warning("%s: omap_sr struct not found\n", __func__);
+		pr_warn("%s: omap_sr struct not found\n", __func__);
 		return -EINVAL;
 	}
 
 	/* Sanity check */
 	if (val > 1) {
-		pr_warning("%s: Invalid argument %lld\n", __func__, val);
+		pr_warn("%s: Invalid argument %lld\n", __func__, val);
 		return -EINVAL;
 	}
 
@@ -848,19 +847,13 @@ static int __init omap_sr_probe(struct platform_device *pdev)
 	int i, ret = 0;
 
 	sr_info = devm_kzalloc(&pdev->dev, sizeof(struct omap_sr), GFP_KERNEL);
-	if (!sr_info) {
-		dev_err(&pdev->dev, "%s: unable to allocate sr_info\n",
-			__func__);
+	if (!sr_info)
 		return -ENOMEM;
-	}
 
 	sr_info->name = devm_kzalloc(&pdev->dev,
 				     SMARTREFLEX_NAME_LEN, GFP_KERNEL);
-	if (!sr_info->name) {
-		dev_err(&pdev->dev, "%s: unable to allocate SR instance name\n",
-			__func__);
+	if (!sr_info->name)
 		return -ENOMEM;
-	}
 
 	platform_set_drvdata(pdev, sr_info);
 
@@ -912,7 +905,7 @@ static int __init omap_sr_probe(struct platform_device *pdev)
 	if (sr_class) {
 		ret = sr_late_init(sr_info);
 		if (ret) {
-			pr_warning("%s: Error in SR late init\n", __func__);
+			pr_warn("%s: Error in SR late init\n", __func__);
 			goto err_list_del;
 		}
 	}
@@ -923,7 +916,7 @@ static int __init omap_sr_probe(struct platform_device *pdev)
 		if (IS_ERR_OR_NULL(sr_dbg_dir)) {
 			ret = PTR_ERR(sr_dbg_dir);
 			pr_err("%s:sr debugfs dir creation failed(%d)\n",
-				__func__, ret);
+			       __func__, ret);
 			goto err_list_del;
 		}
 	}
@@ -945,8 +938,8 @@ static int __init omap_sr_probe(struct platform_device *pdev)
 
 	nvalue_dir = debugfs_create_dir("nvalue", sr_info->dbg_dir);
 	if (IS_ERR_OR_NULL(nvalue_dir)) {
-		dev_err(&pdev->dev, "%s: Unable to create debugfs directory"
-			"for n-values\n", __func__);
+		dev_err(&pdev->dev, "%s: Unable to create debugfs directory for n-values\n",
+			__func__);
 		ret = PTR_ERR(nvalue_dir);
 		goto err_debugfs;
 	}
@@ -1053,12 +1046,12 @@ static int __init sr_init(void)
 	if (sr_pmic_data && sr_pmic_data->sr_pmic_init)
 		sr_pmic_data->sr_pmic_init();
 	else
-		pr_warning("%s: No PMIC hook to init smartreflex\n", __func__);
+		pr_warn("%s: No PMIC hook to init smartreflex\n", __func__);
 
 	ret = platform_driver_probe(&smartreflex_driver, omap_sr_probe);
 	if (ret) {
 		pr_err("%s: platform driver register failed for SR\n",
-			__func__);
+		       __func__);
 		return ret;
 	}
 
diff --git a/drivers/power/reset/Kconfig b/drivers/power/reset/Kconfig
index 3bfac539334b..c74c3f67b8da 100644
--- a/drivers/power/reset/Kconfig
+++ b/drivers/power/reset/Kconfig
@@ -200,8 +200,8 @@ config REBOOT_MODE
 config SYSCON_REBOOT_MODE
 	tristate "Generic SYSCON regmap reboot mode driver"
 	depends on OF
+	depends on MFD_SYSCON
 	select REBOOT_MODE
-	select MFD_SYSCON
 	help
 	  Say y here will enable reboot mode driver. This will
 	  get reboot mode arguments and store it in SYSCON mapped
diff --git a/drivers/power/reset/hisi-reboot.c b/drivers/power/reset/hisi-reboot.c
index 9ab7f562a83b..f69387e12c1e 100644
--- a/drivers/power/reset/hisi-reboot.c
+++ b/drivers/power/reset/hisi-reboot.c
@@ -53,13 +53,16 @@ static int hisi_reboot_probe(struct platform_device *pdev)
 
 	if (of_property_read_u32(np, "reboot-offset", &reboot_offset) < 0) {
 		pr_err("failed to find reboot-offset property\n");
+		iounmap(base);
 		return -EINVAL;
 	}
 
 	err = register_restart_handler(&hisi_restart_nb);
-	if (err)
+	if (err) {
 		dev_err(&pdev->dev, "cannot register restart handler (err=%d)\n",
 			err);
+		iounmap(base);
+	}
 
 	return err;
 }
diff --git a/drivers/power/reset/keystone-reset.c b/drivers/power/reset/keystone-reset.c
index c70f1bffe038..09380857a1c5 100644
--- a/drivers/power/reset/keystone-reset.c
+++ b/drivers/power/reset/keystone-reset.c
@@ -139,7 +139,7 @@ static int rsctrl_probe(struct platform_device *pdev)
 		}
 
 		if (val >= WDT_MUX_NUMBER) {
-			dev_err(dev, "ti,wdt-list property can contain"
+			dev_err(dev, "ti,wdt-list property can contain "
 				"only numbers < 4\n");
 			return -EINVAL;
 		}
diff --git a/drivers/power/reset/reboot-mode.c b/drivers/power/reset/reboot-mode.c
index 2dfbbce0f817..fb512183ace3 100644
--- a/drivers/power/reset/reboot-mode.c
+++ b/drivers/power/reset/reboot-mode.c
@@ -135,6 +135,65 @@ int reboot_mode_unregister(struct reboot_mode_driver *reboot)
 }
 EXPORT_SYMBOL_GPL(reboot_mode_unregister);
 
+static void devm_reboot_mode_release(struct device *dev, void *res)
+{
+	reboot_mode_unregister(*(struct reboot_mode_driver **)res);
+}
+
+/**
+ * devm_reboot_mode_register() - resource managed reboot_mode_register()
+ * @dev: device to associate this resource with
+ * @reboot: reboot mode driver
+ *
+ * Returns: 0 on success or a negative error code on failure.
+ */
+int devm_reboot_mode_register(struct device *dev,
+			      struct reboot_mode_driver *reboot)
+{
+	struct reboot_mode_driver **dr;
+	int rc;
+
+	dr = devres_alloc(devm_reboot_mode_release, sizeof(*dr), GFP_KERNEL);
+	if (!dr)
+		return -ENOMEM;
+
+	rc = reboot_mode_register(reboot);
+	if (rc) {
+		devres_free(dr);
+		return rc;
+	}
+
+	*dr = reboot;
+	devres_add(dev, dr);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(devm_reboot_mode_register);
+
+static int devm_reboot_mode_match(struct device *dev, void *res, void *data)
+{
+	struct reboot_mode_driver **p = res;
+
+	if (WARN_ON(!p || !*p))
+		return 0;
+
+	return *p == data;
+}
+
+/**
+ * devm_reboot_mode_unregister() - resource managed reboot_mode_unregister()
+ * @dev: device to associate this resource with
+ * @reboot: reboot mode driver
+ */
+void devm_reboot_mode_unregister(struct device *dev,
+				 struct reboot_mode_driver *reboot)
+{
+	WARN_ON(devres_release(dev,
+			       devm_reboot_mode_release,
+			       devm_reboot_mode_match, reboot));
+}
+EXPORT_SYMBOL_GPL(devm_reboot_mode_unregister);
+
 MODULE_AUTHOR("Andy Yan <andy.yan@rock-chips.com");
 MODULE_DESCRIPTION("System reboot mode core library");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/power/reset/reboot-mode.h b/drivers/power/reset/reboot-mode.h
index 2491bb71f591..75f7fe5c881f 100644
--- a/drivers/power/reset/reboot-mode.h
+++ b/drivers/power/reset/reboot-mode.h
@@ -10,5 +10,9 @@ struct reboot_mode_driver {
 
 int reboot_mode_register(struct reboot_mode_driver *reboot);
 int reboot_mode_unregister(struct reboot_mode_driver *reboot);
+int devm_reboot_mode_register(struct device *dev,
+			      struct reboot_mode_driver *reboot);
+void devm_reboot_mode_unregister(struct device *dev,
+				 struct reboot_mode_driver *reboot);
 
 #endif
diff --git a/drivers/power/reset/st-poweroff.c b/drivers/power/reset/st-poweroff.c
index a488877a3538..2046b31232f7 100644
--- a/drivers/power/reset/st-poweroff.c
+++ b/drivers/power/reset/st-poweroff.c
@@ -28,28 +28,6 @@ struct reset_syscfg {
 	unsigned int mask_rst_msk;
 };
 
-/* STiH415 */
-#define STIH415_SYSCFG_11	0x2c
-#define STIH415_SYSCFG_15	0x3c
-
-static struct reset_syscfg stih415_reset = {
-	.offset_rst = STIH415_SYSCFG_11,
-	.mask_rst = BIT(0),
-	.offset_rst_msk = STIH415_SYSCFG_15,
-	.mask_rst_msk = BIT(0)
-};
-
-/* STiH416 */
-#define STIH416_SYSCFG_500	0x7d0
-#define STIH416_SYSCFG_504	0x7e0
-
-static struct reset_syscfg stih416_reset = {
-	.offset_rst = STIH416_SYSCFG_500,
-	.mask_rst = BIT(0),
-	.offset_rst_msk = STIH416_SYSCFG_504,
-	.mask_rst_msk = BIT(0)
-};
-
 /* STiH407 */
 #define STIH407_SYSCFG_4000	0x0
 #define STIH407_SYSCFG_4008	0x20
@@ -61,16 +39,6 @@ static struct reset_syscfg stih407_reset = {
 	.mask_rst_msk = BIT(0)
 };
 
-/* STiD127 */
-#define STID127_SYSCFG_700	0x0
-#define STID127_SYSCFG_773	0x124
-
-static struct reset_syscfg stid127_reset = {
-	.offset_rst = STID127_SYSCFG_773,
-	.mask_rst = BIT(0),
-	.offset_rst_msk = STID127_SYSCFG_700,
-	.mask_rst_msk = BIT(8)
-};
 
 static struct reset_syscfg *st_restart_syscfg;
 
@@ -99,17 +67,8 @@ static struct notifier_block st_restart_nb = {
 
 static const struct of_device_id st_reset_of_match[] = {
 	{
-		.compatible = "st,stih415-restart",
-		.data = (void *)&stih415_reset,
-	}, {
-		.compatible = "st,stih416-restart",
-		.data = (void *)&stih416_reset,
-	}, {
 		.compatible = "st,stih407-restart",
 		.data = (void *)&stih407_reset,
-	}, {
-		.compatible = "st,stid127-restart",
-		.data = (void *)&stid127_reset,
 	},
 	{}
 };
diff --git a/drivers/power/reset/syscon-reboot-mode.c b/drivers/power/reset/syscon-reboot-mode.c
index 9e1cba5dd58e..1ecb51d67149 100644
--- a/drivers/power/reset/syscon-reboot-mode.c
+++ b/drivers/power/reset/syscon-reboot-mode.c
@@ -53,8 +53,6 @@ static int syscon_reboot_mode_probe(struct platform_device *pdev)
 	syscon_rbm->reboot.write = syscon_reboot_mode_write;
 	syscon_rbm->mask = 0xffffffff;
 
-	dev_set_drvdata(&pdev->dev, syscon_rbm);
-
 	syscon_rbm->map = syscon_node_to_regmap(pdev->dev.parent->of_node);
 	if (IS_ERR(syscon_rbm->map))
 		return PTR_ERR(syscon_rbm->map);
@@ -65,20 +63,13 @@ static int syscon_reboot_mode_probe(struct platform_device *pdev)
 
 	of_property_read_u32(pdev->dev.of_node, "mask", &syscon_rbm->mask);
 
-	ret = reboot_mode_register(&syscon_rbm->reboot);
+	ret = devm_reboot_mode_register(&pdev->dev, &syscon_rbm->reboot);
 	if (ret)
 		dev_err(&pdev->dev, "can't register reboot mode\n");
 
 	return ret;
 }
 
-static int syscon_reboot_mode_remove(struct platform_device *pdev)
-{
-	struct syscon_reboot_mode *syscon_rbm = dev_get_drvdata(&pdev->dev);
-
-	return reboot_mode_unregister(&syscon_rbm->reboot);
-}
-
 static const struct of_device_id syscon_reboot_mode_of_match[] = {
 	{ .compatible = "syscon-reboot-mode" },
 	{}
@@ -86,7 +77,6 @@ static const struct of_device_id syscon_reboot_mode_of_match[] = {
 
 static struct platform_driver syscon_reboot_mode_driver = {
 	.probe = syscon_reboot_mode_probe,
-	.remove = syscon_reboot_mode_remove,
 	.driver = {
 		.name = "syscon-reboot-mode",
 		.of_match_table = syscon_reboot_mode_of_match,
diff --git a/drivers/power/reset/xgene-reboot.c b/drivers/power/reset/xgene-reboot.c
index f07e93c97ba3..73c3d93e5318 100644
--- a/drivers/power/reset/xgene-reboot.c
+++ b/drivers/power/reset/xgene-reboot.c
@@ -81,8 +81,10 @@ static int xgene_reboot_probe(struct platform_device *pdev)
 	ctx->restart_handler.notifier_call = xgene_restart_handler;
 	ctx->restart_handler.priority = 128;
 	err = register_restart_handler(&ctx->restart_handler);
-	if (err)
+	if (err) {
+		iounmap(ctx->csr);
 		dev_err(dev, "cannot register restart handler (err=%d)\n", err);
+	}
 
 	return err;
 }
diff --git a/drivers/power/reset/zx-reboot.c b/drivers/power/reset/zx-reboot.c
index a5b009673d0e..b0b1eb3a78c2 100644
--- a/drivers/power/reset/zx-reboot.c
+++ b/drivers/power/reset/zx-reboot.c
@@ -58,9 +58,12 @@ static int zx_reboot_probe(struct platform_device *pdev)
 	}
 
 	err = register_restart_handler(&zx_restart_nb);
-	if (err)
+	if (err) {
+		iounmap(base);
+		iounmap(pcu_base);
 		dev_err(&pdev->dev, "Register restart handler failed(err=%d)\n",
 			err);
+	}
 
 	return err;
 }
diff --git a/drivers/power/88pm860x_battery.c b/drivers/power/supply/88pm860x_battery.c
index 63c57dc82ac1..63c57dc82ac1 100644
--- a/drivers/power/88pm860x_battery.c
+++ b/drivers/power/supply/88pm860x_battery.c
diff --git a/drivers/power/88pm860x_charger.c b/drivers/power/supply/88pm860x_charger.c
index 2b82e44d9027..2b82e44d9027 100644
--- a/drivers/power/88pm860x_charger.c
+++ b/drivers/power/supply/88pm860x_charger.c
diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig
new file mode 100644
index 000000000000..76806a0be820
--- /dev/null
+++ b/drivers/power/supply/Kconfig
@@ -0,0 +1,514 @@
+menuconfig POWER_SUPPLY
+	bool "Power supply class support"
+	help
+	  Say Y here to enable power supply class support. This allows
+	  power supply (batteries, AC, USB) monitoring by userspace
+	  via sysfs and uevent (if available) and/or APM kernel interface
+	  (if selected below).
+
+if POWER_SUPPLY
+
+config POWER_SUPPLY_DEBUG
+	bool "Power supply debug"
+	help
+	  Say Y here to enable debugging messages for power supply class
+	  and drivers.
+
+config PDA_POWER
+	tristate "Generic PDA/phone power driver"
+	depends on !S390
+	help
+	  Say Y here to enable generic power driver for PDAs and phones with
+	  one or two external power supplies (AC/USB) connected to main and
+	  backup batteries, and optional builtin charger.
+
+config APM_POWER
+	tristate "APM emulation for class batteries"
+	depends on APM_EMULATION
+	help
+	  Say Y here to enable support APM status emulation using
+	  battery class devices.
+
+config GENERIC_ADC_BATTERY
+	tristate "Generic battery support using IIO"
+	depends on IIO
+	help
+	  Say Y here to enable support for the generic battery driver
+	  which uses IIO framework to read adc.
+
+config MAX8925_POWER
+	tristate "MAX8925 battery charger support"
+	depends on MFD_MAX8925
+	help
+	  Say Y here to enable support for the battery charger in the Maxim
+	  MAX8925 PMIC.
+
+config WM831X_BACKUP
+	tristate "WM831X backup battery charger support"
+	depends on MFD_WM831X
+	help
+	  Say Y here to enable support for the backup battery charger
+	  in the Wolfson Microelectronics WM831x PMICs.
+
+config WM831X_POWER
+	tristate "WM831X PMU support"
+	depends on MFD_WM831X
+	help
+	  Say Y here to enable support for the power management unit
+	  provided by Wolfson Microelectronics WM831x PMICs.
+
+config WM8350_POWER
+        tristate "WM8350 PMU support"
+        depends on MFD_WM8350
+        help
+          Say Y here to enable support for the power management unit
+	  provided by the Wolfson Microelectronics WM8350 PMIC.
+
+config TEST_POWER
+	tristate "Test power driver"
+	help
+	  This driver is used for testing. It's safe to say M here.
+
+config BATTERY_88PM860X
+	tristate "Marvell 88PM860x battery driver"
+	depends on MFD_88PM860X
+	help
+	  Say Y here to enable battery monitor for Marvell 88PM860x chip.
+
+config BATTERY_ACT8945A
+	tristate "Active-semi ACT8945A charger driver"
+	depends on MFD_ACT8945A || COMPILE_TEST
+	help
+	  Say Y here to enable support for power supply provided by
+	  Active-semi ActivePath ACT8945A charger.
+
+config BATTERY_DS2760
+	tristate "DS2760 battery driver (HP iPAQ & others)"
+	depends on W1 && W1_SLAVE_DS2760
+	help
+	  Say Y here to enable support for batteries with ds2760 chip.
+
+config BATTERY_DS2780
+	tristate "DS2780 battery driver"
+	depends on HAS_IOMEM
+	select W1
+	select W1_SLAVE_DS2780
+	help
+	  Say Y here to enable support for batteries with ds2780 chip.
+
+config BATTERY_DS2781
+	tristate "DS2781 battery driver"
+	depends on HAS_IOMEM
+	select W1
+	select W1_SLAVE_DS2781
+	help
+	  If you enable this you will have the DS2781 battery driver support.
+
+	  The battery monitor chip is used in many batteries/devices
+	  as the one who is responsible for charging/discharging/monitoring
+	  Li+ batteries.
+
+	  If you are unsure, say N.
+
+config BATTERY_DS2782
+	tristate "DS2782/DS2786 standalone gas-gauge"
+	depends on I2C
+	help
+	  Say Y here to enable support for the DS2782/DS2786 standalone battery
+	  gas-gauge.
+
+config BATTERY_PMU
+	tristate "Apple PMU battery"
+	depends on PPC32 && ADB_PMU
+	help
+	  Say Y here to expose battery information on Apple machines
+	  through the generic battery class.
+
+config BATTERY_OLPC
+	tristate "One Laptop Per Child battery"
+	depends on X86_32 && OLPC
+	help
+	  Say Y to enable support for the battery on the OLPC laptop.
+
+config BATTERY_TOSA
+	tristate "Sharp SL-6000 (tosa) battery"
+	depends on MACH_TOSA && MFD_TC6393XB && TOUCHSCREEN_WM97XX
+	help
+	  Say Y to enable support for the battery on the Sharp Zaurus
+	  SL-6000 (tosa) models.
+
+config BATTERY_COLLIE
+	tristate "Sharp SL-5500 (collie) battery"
+	depends on SA1100_COLLIE && MCP_UCB1200
+	help
+	  Say Y to enable support for the battery on the Sharp Zaurus
+	  SL-5500 (collie) models.
+
+config BATTERY_IPAQ_MICRO
+	tristate "iPAQ Atmel Micro ASIC battery driver"
+	depends on MFD_IPAQ_MICRO
+	help
+	  Choose this option if you want to monitor battery status on
+	  Compaq/HP iPAQ h3100 and h3600.
+
+config BATTERY_WM97XX
+	bool "WM97xx generic battery driver"
+	depends on TOUCHSCREEN_WM97XX=y
+	help
+	  Say Y to enable support for battery measured by WM97xx aux port.
+
+config BATTERY_SBS
+        tristate "SBS Compliant gas gauge"
+        depends on I2C
+        help
+	  Say Y to include support for SBS battery driver for SBS-compliant
+	  gas gauges.
+
+config BATTERY_BQ27XXX
+	tristate "BQ27xxx battery driver"
+	help
+	  Say Y here to enable support for batteries with BQ27xxx chips.
+
+config BATTERY_BQ27XXX_I2C
+	tristate "BQ27xxx I2C support"
+	depends on BATTERY_BQ27XXX
+	depends on I2C
+	default y
+	help
+	  Say Y here to enable support for batteries with BQ27xxx chips
+	  connected over an I2C bus.
+
+config BATTERY_DA9030
+	tristate "DA9030 battery driver"
+	depends on PMIC_DA903X
+	help
+	  Say Y here to enable support for batteries charger integrated into
+	  DA9030 PMIC.
+
+config BATTERY_DA9052
+	tristate "Dialog DA9052 Battery"
+	depends on PMIC_DA9052
+	help
+	  Say Y here to enable support for batteries charger integrated into
+	  DA9052 PMIC.
+
+config CHARGER_DA9150
+	tristate "Dialog Semiconductor DA9150 Charger support"
+	depends on MFD_DA9150
+	depends on DA9150_GPADC
+	depends on IIO
+	help
+	  Say Y here to enable support for charger unit of the DA9150
+	  Integrated Charger & Fuel-Gauge IC.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called da9150-charger.
+
+config BATTERY_DA9150
+	tristate "Dialog Semiconductor DA9150 Fuel Gauge support"
+	depends on MFD_DA9150
+	help
+	  Say Y here to enable support for the Fuel-Gauge unit of the DA9150
+	  Integrated Charger & Fuel-Gauge IC
+
+	  This driver can also be built as a module. If so, the module will be
+	  called da9150-fg.
+
+config AXP288_CHARGER
+	tristate "X-Powers AXP288 Charger"
+	depends on MFD_AXP20X && EXTCON_AXP288
+	help
+	  Say yes here to have support X-Power AXP288 power management IC (PMIC)
+	  integrated charger.
+
+config AXP288_FUEL_GAUGE
+	tristate "X-Powers AXP288 Fuel Gauge"
+	depends on MFD_AXP20X && IIO
+	help
+	  Say yes here to have support for X-Power power management IC (PMIC)
+	  Fuel Gauge. The device provides battery statistics and status
+	  monitoring as well as alerts for battery over/under voltage and
+	  over/under temperature.
+
+config BATTERY_MAX17040
+	tristate "Maxim MAX17040 Fuel Gauge"
+	depends on I2C
+	help
+	  MAX17040 is fuel-gauge systems for lithium-ion (Li+) batteries
+	  in handheld and portable equipment. The MAX17040 is configured
+	  to operate with a single lithium cell
+
+config BATTERY_MAX17042
+	tristate "Maxim MAX17042/17047/17050/8997/8966 Fuel Gauge"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  MAX17042 is fuel-gauge systems for lithium-ion (Li+) batteries
+	  in handheld and portable equipment. The MAX17042 is configured
+	  to operate with a single lithium cell. MAX8997 and MAX8966 are
+	  multi-function devices that include fuel gauages that are compatible
+	  with MAX17042. This driver also supports max17047/50 chips which are
+	  improved version of max17042.
+
+config BATTERY_Z2
+	tristate "Z2 battery driver"
+	depends on I2C && MACH_ZIPIT2
+	help
+	  Say Y to include support for the battery on the Zipit Z2.
+
+config BATTERY_S3C_ADC
+	tristate "Battery driver for Samsung ADC based monitoring"
+	depends on S3C_ADC
+	help
+	  Say Y here to enable support for iPAQ h1930/h1940/rx1950 battery
+
+config BATTERY_TWL4030_MADC
+	tristate "TWL4030 MADC battery driver"
+	depends on TWL4030_MADC
+	help
+	  Say Y here to enable this dumb driver for batteries managed
+	  through the TWL4030 MADC.
+
+config CHARGER_88PM860X
+	tristate "Marvell 88PM860x Charger driver"
+	depends on MFD_88PM860X && BATTERY_88PM860X
+	help
+	  Say Y here to enable charger for Marvell 88PM860x chip.
+
+config CHARGER_PCF50633
+	tristate "NXP PCF50633 MBC"
+	depends on MFD_PCF50633
+	help
+	 Say Y to include support for NXP PCF50633 Main Battery Charger.
+
+config BATTERY_JZ4740
+	tristate "Ingenic JZ4740 battery"
+	depends on MACH_JZ4740
+	depends on MFD_JZ4740_ADC
+	help
+	  Say Y to enable support for the battery on Ingenic JZ4740 based
+	  boards.
+
+	  This driver can be build as a module. If so, the module will be
+	  called jz4740-battery.
+
+config BATTERY_INTEL_MID
+	tristate "Battery driver for Intel MID platforms"
+	depends on INTEL_SCU_IPC && SPI
+	help
+	  Say Y here to enable the battery driver on Intel MID
+	  platforms.
+
+config BATTERY_RX51
+	tristate "Nokia RX-51 (N900) battery driver"
+	depends on TWL4030_MADC
+	help
+	  Say Y here to enable support for battery information on Nokia
+	  RX-51, also known as N900 tablet.
+
+config CHARGER_ISP1704
+	tristate "ISP1704 USB Charger Detection"
+	depends on USB_PHY
+	depends on USB_GADGET || !USB_GADGET # if USB_GADGET=m, this can't be 'y'
+	help
+	  Say Y to enable support for USB Charger Detection with
+	  ISP1707/ISP1704 USB transceivers.
+
+config CHARGER_MAX8903
+	tristate "MAX8903 Battery DC-DC Charger for USB and Adapter Power"
+	help
+	  Say Y to enable support for the MAX8903 DC-DC charger and sysfs.
+	  The driver supports controlling charger-enable and current-limit
+	  pins based on the status of charger connections with interrupt
+	  handlers.
+
+config CHARGER_TWL4030
+	tristate "OMAP TWL4030 BCI charger driver"
+	depends on IIO && TWL4030_CORE
+	help
+	  Say Y here to enable support for TWL4030 Battery Charge Interface.
+
+config CHARGER_LP8727
+	tristate "TI/National Semiconductor LP8727 charger driver"
+	depends on I2C
+	help
+	  Say Y here to enable support for LP8727 Charger Driver.
+
+config CHARGER_LP8788
+	tristate "TI LP8788 charger driver"
+	depends on MFD_LP8788
+	depends on LP8788_ADC
+	depends on IIO
+	help
+	  Say Y to enable support for the LP8788 linear charger.
+
+config CHARGER_GPIO
+	tristate "GPIO charger"
+	depends on GPIOLIB || COMPILE_TEST
+	help
+	  Say Y to include support for chargers which report their online status
+	  through a GPIO pin.
+
+	  This driver can be build as a module. If so, the module will be
+	  called gpio-charger.
+
+config CHARGER_MANAGER
+	bool "Battery charger manager for multiple chargers"
+	depends on REGULATOR
+	select EXTCON
+	help
+          Say Y to enable charger-manager support, which allows multiple
+          chargers attached to a battery and multiple batteries attached to a
+          system. The charger-manager also can monitor charging status in
+          runtime and in suspend-to-RAM by waking up the system periodically
+          with help of suspend_again support.
+
+config CHARGER_MAX14577
+	tristate "Maxim MAX14577/77836 battery charger driver"
+	depends on MFD_MAX14577
+	help
+	  Say Y to enable support for the battery charger control sysfs and
+	  platform data of MAX14577/77836 MUICs.
+
+config CHARGER_MAX77693
+	tristate "Maxim MAX77693 battery charger driver"
+	depends on MFD_MAX77693
+	help
+	  Say Y to enable support for the Maxim MAX77693 battery charger.
+
+config CHARGER_MAX8997
+	tristate "Maxim MAX8997/MAX8966 PMIC battery charger driver"
+	depends on MFD_MAX8997 && REGULATOR_MAX8997
+	help
+	  Say Y to enable support for the battery charger control sysfs and
+	  platform data of MAX8997/LP3974 PMICs.
+
+config CHARGER_MAX8998
+	tristate "Maxim MAX8998/LP3974 PMIC battery charger driver"
+	depends on MFD_MAX8998 && REGULATOR_MAX8998
+	help
+	  Say Y to enable support for the battery charger control sysfs and
+	  platform data of MAX8998/LP3974 PMICs.
+
+config CHARGER_QCOM_SMBB
+	tristate "Qualcomm Switch-Mode Battery Charger and Boost"
+	depends on MFD_SPMI_PMIC || COMPILE_TEST
+	depends on OF
+	depends on EXTCON
+	help
+	  Say Y to include support for the Switch-Mode Battery Charger and
+	  Boost (SMBB) hardware found in Qualcomm PM8941 PMICs.  The charger
+	  is an integrated, single-cell lithium-ion battery charger.  DT
+	  configuration is required for loading, see the devicetree
+	  documentation for more detail.  The base name for this driver is
+	  'pm8941_charger'.
+
+config CHARGER_BQ2415X
+	tristate "TI BQ2415x battery charger driver"
+	depends on I2C
+	help
+	  Say Y to enable support for the TI BQ2415x battery charger
+	  PMICs.
+
+	  You'll need this driver to charge batteries on e.g. Nokia
+	  RX-51/N900.
+
+config CHARGER_BQ24190
+	tristate "TI BQ24190 battery charger driver"
+	depends on I2C
+	depends on GPIOLIB || COMPILE_TEST
+	help
+	  Say Y to enable support for the TI BQ24190 battery charger.
+
+config CHARGER_BQ24257
+	tristate "TI BQ24250/24251/24257 battery charger driver"
+	depends on I2C
+	depends on GPIOLIB || COMPILE_TEST
+	depends on REGMAP_I2C
+	help
+	  Say Y to enable support for the TI BQ24250, BQ24251, and BQ24257 battery
+	  chargers.
+
+config CHARGER_BQ24735
+	tristate "TI BQ24735 battery charger support"
+	depends on I2C
+	depends on GPIOLIB || COMPILE_TEST
+	help
+	  Say Y to enable support for the TI BQ24735 battery charger.
+
+config CHARGER_BQ25890
+	tristate "TI BQ25890 battery charger driver"
+	depends on I2C
+	depends on GPIOLIB || COMPILE_TEST
+	select REGMAP_I2C
+	help
+	  Say Y to enable support for the TI BQ25890 battery charger.
+
+config CHARGER_SMB347
+	tristate "Summit Microelectronics SMB347 Battery Charger"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Say Y to include support for Summit Microelectronics SMB347
+	  Battery Charger.
+
+config CHARGER_TPS65090
+	tristate "TPS65090 battery charger driver"
+	depends on MFD_TPS65090
+	help
+	 Say Y here to enable support for battery charging with TPS65090
+	 PMIC chips.
+
+config CHARGER_TPS65217
+	tristate "TPS65217 battery charger driver"
+	depends on MFD_TPS65217
+	help
+	 Say Y here to enable support for battery charging with TPS65217
+	 PMIC chips.
+
+config BATTERY_GAUGE_LTC2941
+	tristate "LTC2941/LTC2943 Battery Gauge Driver"
+	depends on I2C
+	help
+	  Say Y here to include support for LTC2941 and LTC2943 Battery
+	  Gauge IC. The driver reports the charge count continuously, and
+	  measures the voltage and temperature every 10 seconds.
+
+config AB8500_BM
+	bool "AB8500 Battery Management Driver"
+	depends on AB8500_CORE && AB8500_GPADC
+	help
+	  Say Y to include support for AB8500 battery management.
+
+config BATTERY_GOLDFISH
+	tristate "Goldfish battery driver"
+	depends on GOLDFISH || COMPILE_TEST
+	depends on HAS_IOMEM
+	help
+	  Say Y to enable support for the battery and AC power in the
+	  Goldfish emulator.
+
+config BATTERY_RT5033
+	tristate "RT5033 fuel gauge support"
+	depends on MFD_RT5033
+	help
+	  This adds support for battery fuel gauge in Richtek RT5033 PMIC.
+	  The fuelgauge calculates and determines the battery state of charge
+	  according to battery open circuit voltage.
+
+config CHARGER_RT9455
+	tristate "Richtek RT9455 battery charger driver"
+	depends on I2C
+	depends on GPIOLIB || COMPILE_TEST
+	select REGMAP_I2C
+	help
+	  Say Y to enable support for Richtek RT9455 battery charger.
+
+config AXP20X_POWER
+	tristate "AXP20x power supply driver"
+	depends on MFD_AXP20X
+	help
+	  This driver provides support for the power supply features of
+	  AXP20x PMIC.
+
+endif # POWER_SUPPLY
diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile
new file mode 100644
index 000000000000..36c599d9a495
--- /dev/null
+++ b/drivers/power/supply/Makefile
@@ -0,0 +1,74 @@
+subdir-ccflags-$(CONFIG_POWER_SUPPLY_DEBUG) := -DDEBUG
+
+power_supply-y				:= power_supply_core.o
+power_supply-$(CONFIG_SYSFS)		+= power_supply_sysfs.o
+power_supply-$(CONFIG_LEDS_TRIGGERS)	+= power_supply_leds.o
+
+obj-$(CONFIG_POWER_SUPPLY)	+= power_supply.o
+obj-$(CONFIG_GENERIC_ADC_BATTERY)	+= generic-adc-battery.o
+
+obj-$(CONFIG_PDA_POWER)		+= pda_power.o
+obj-$(CONFIG_APM_POWER)		+= apm_power.o
+obj-$(CONFIG_AXP20X_POWER)	+= axp20x_usb_power.o
+obj-$(CONFIG_MAX8925_POWER)	+= max8925_power.o
+obj-$(CONFIG_WM831X_BACKUP)	+= wm831x_backup.o
+obj-$(CONFIG_WM831X_POWER)	+= wm831x_power.o
+obj-$(CONFIG_WM8350_POWER)	+= wm8350_power.o
+obj-$(CONFIG_TEST_POWER)	+= test_power.o
+
+obj-$(CONFIG_BATTERY_88PM860X)	+= 88pm860x_battery.o
+obj-$(CONFIG_BATTERY_ACT8945A)	+= act8945a_charger.o
+obj-$(CONFIG_BATTERY_DS2760)	+= ds2760_battery.o
+obj-$(CONFIG_BATTERY_DS2780)	+= ds2780_battery.o
+obj-$(CONFIG_BATTERY_DS2781)	+= ds2781_battery.o
+obj-$(CONFIG_BATTERY_DS2782)	+= ds2782_battery.o
+obj-$(CONFIG_BATTERY_GAUGE_LTC2941)	+= ltc2941-battery-gauge.o
+obj-$(CONFIG_BATTERY_GOLDFISH)	+= goldfish_battery.o
+obj-$(CONFIG_BATTERY_PMU)	+= pmu_battery.o
+obj-$(CONFIG_BATTERY_OLPC)	+= olpc_battery.o
+obj-$(CONFIG_BATTERY_TOSA)	+= tosa_battery.o
+obj-$(CONFIG_BATTERY_COLLIE)	+= collie_battery.o
+obj-$(CONFIG_BATTERY_IPAQ_MICRO) += ipaq_micro_battery.o
+obj-$(CONFIG_BATTERY_WM97XX)	+= wm97xx_battery.o
+obj-$(CONFIG_BATTERY_SBS)	+= sbs-battery.o
+obj-$(CONFIG_BATTERY_BQ27XXX)	+= bq27xxx_battery.o
+obj-$(CONFIG_BATTERY_BQ27XXX_I2C) += bq27xxx_battery_i2c.o
+obj-$(CONFIG_BATTERY_DA9030)	+= da9030_battery.o
+obj-$(CONFIG_BATTERY_DA9052)	+= da9052-battery.o
+obj-$(CONFIG_CHARGER_DA9150)	+= da9150-charger.o
+obj-$(CONFIG_BATTERY_DA9150)	+= da9150-fg.o
+obj-$(CONFIG_BATTERY_MAX17040)	+= max17040_battery.o
+obj-$(CONFIG_BATTERY_MAX17042)	+= max17042_battery.o
+obj-$(CONFIG_BATTERY_Z2)	+= z2_battery.o
+obj-$(CONFIG_BATTERY_RT5033)	+= rt5033_battery.o
+obj-$(CONFIG_CHARGER_RT9455)	+= rt9455_charger.o
+obj-$(CONFIG_BATTERY_S3C_ADC)	+= s3c_adc_battery.o
+obj-$(CONFIG_BATTERY_TWL4030_MADC)	+= twl4030_madc_battery.o
+obj-$(CONFIG_CHARGER_88PM860X)	+= 88pm860x_charger.o
+obj-$(CONFIG_CHARGER_PCF50633)	+= pcf50633-charger.o
+obj-$(CONFIG_BATTERY_JZ4740)	+= jz4740-battery.o
+obj-$(CONFIG_BATTERY_INTEL_MID)	+= intel_mid_battery.o
+obj-$(CONFIG_BATTERY_RX51)	+= rx51_battery.o
+obj-$(CONFIG_AB8500_BM)		+= ab8500_bmdata.o ab8500_charger.o ab8500_fg.o ab8500_btemp.o abx500_chargalg.o pm2301_charger.o
+obj-$(CONFIG_CHARGER_ISP1704)	+= isp1704_charger.o
+obj-$(CONFIG_CHARGER_MAX8903)	+= max8903_charger.o
+obj-$(CONFIG_CHARGER_TWL4030)	+= twl4030_charger.o
+obj-$(CONFIG_CHARGER_LP8727)	+= lp8727_charger.o
+obj-$(CONFIG_CHARGER_LP8788)	+= lp8788-charger.o
+obj-$(CONFIG_CHARGER_GPIO)	+= gpio-charger.o
+obj-$(CONFIG_CHARGER_MANAGER)	+= charger-manager.o
+obj-$(CONFIG_CHARGER_MAX14577)	+= max14577_charger.o
+obj-$(CONFIG_CHARGER_MAX77693)	+= max77693_charger.o
+obj-$(CONFIG_CHARGER_MAX8997)	+= max8997_charger.o
+obj-$(CONFIG_CHARGER_MAX8998)	+= max8998_charger.o
+obj-$(CONFIG_CHARGER_QCOM_SMBB)	+= qcom_smbb.o
+obj-$(CONFIG_CHARGER_BQ2415X)	+= bq2415x_charger.o
+obj-$(CONFIG_CHARGER_BQ24190)	+= bq24190_charger.o
+obj-$(CONFIG_CHARGER_BQ24257)	+= bq24257_charger.o
+obj-$(CONFIG_CHARGER_BQ24735)	+= bq24735-charger.o
+obj-$(CONFIG_CHARGER_BQ25890)	+= bq25890_charger.o
+obj-$(CONFIG_CHARGER_SMB347)	+= smb347-charger.o
+obj-$(CONFIG_CHARGER_TPS65090)	+= tps65090-charger.o
+obj-$(CONFIG_CHARGER_TPS65217)	+= tps65217_charger.o
+obj-$(CONFIG_AXP288_FUEL_GAUGE) += axp288_fuel_gauge.o
+obj-$(CONFIG_AXP288_CHARGER)	+= axp288_charger.o
diff --git a/drivers/power/ab8500_bmdata.c b/drivers/power/supply/ab8500_bmdata.c
index d29864533093..d29864533093 100644
--- a/drivers/power/ab8500_bmdata.c
+++ b/drivers/power/supply/ab8500_bmdata.c
diff --git a/drivers/power/ab8500_btemp.c b/drivers/power/supply/ab8500_btemp.c
index bf2e5dd301e7..6ffdc18f2599 100644
--- a/drivers/power/ab8500_btemp.c
+++ b/drivers/power/supply/ab8500_btemp.c
@@ -1095,7 +1095,7 @@ static int ab8500_btemp_probe(struct platform_device *pdev)
 
 	/* Create a work queue for the btemp */
 	di->btemp_wq =
-		create_singlethread_workqueue("ab8500_btemp_wq");
+		alloc_workqueue("ab8500_btemp_wq", WQ_MEM_RECLAIM, 0);
 	if (di->btemp_wq == NULL) {
 		dev_err(di->dev, "failed to create work queue\n");
 		return -ENOMEM;
diff --git a/drivers/power/ab8500_charger.c b/drivers/power/supply/ab8500_charger.c
index 30de5d42b26a..5cee9aa87aa3 100644
--- a/drivers/power/ab8500_charger.c
+++ b/drivers/power/supply/ab8500_charger.c
@@ -3540,8 +3540,8 @@ static int ab8500_charger_probe(struct platform_device *pdev)
 	di->usb_state.usb_current = -1;
 
 	/* Create a work queue for the charger */
-	di->charger_wq =
-		create_singlethread_workqueue("ab8500_charger_wq");
+	di->charger_wq = alloc_ordered_workqueue("ab8500_charger_wq",
+						 WQ_MEM_RECLAIM);
 	if (di->charger_wq == NULL) {
 		dev_err(di->dev, "failed to create work queue\n");
 		return -ENOMEM;
diff --git a/drivers/power/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c
index 5a36cf88578a..2199f673118c 100644
--- a/drivers/power/ab8500_fg.c
+++ b/drivers/power/supply/ab8500_fg.c
@@ -245,13 +245,8 @@ static LIST_HEAD(ab8500_fg_list);
  */
 struct ab8500_fg *ab8500_fg_get(void)
 {
-	struct ab8500_fg *fg;
-
-	if (list_empty(&ab8500_fg_list))
-		return NULL;
-
-	fg = list_first_entry(&ab8500_fg_list, struct ab8500_fg, node);
-	return fg;
+	return list_first_entry_or_null(&ab8500_fg_list, struct ab8500_fg,
+					node);
 }
 
 /* Main battery properties */
@@ -3096,7 +3091,7 @@ static int ab8500_fg_probe(struct platform_device *pdev)
 	ab8500_fg_discharge_state_to(di, AB8500_FG_DISCHARGE_INIT);
 
 	/* Create a work queue for running the FG algorithm */
-	di->fg_wq = create_singlethread_workqueue("ab8500_fg_wq");
+	di->fg_wq = alloc_ordered_workqueue("ab8500_fg_wq", WQ_MEM_RECLAIM);
 	if (di->fg_wq == NULL) {
 		dev_err(di->dev, "failed to create work queue\n");
 		return -ENOMEM;
diff --git a/drivers/power/abx500_chargalg.c b/drivers/power/supply/abx500_chargalg.c
index d9104b1ab7cf..a4411d6bbc96 100644
--- a/drivers/power/abx500_chargalg.c
+++ b/drivers/power/supply/abx500_chargalg.c
@@ -2091,8 +2091,8 @@ static int abx500_chargalg_probe(struct platform_device *pdev)
 		abx500_chargalg_maintenance_timer_expired;
 
 	/* Create a work queue for the chargalg */
-	di->chargalg_wq =
-		create_singlethread_workqueue("abx500_chargalg_wq");
+	di->chargalg_wq = alloc_ordered_workqueue("abx500_chargalg_wq",
+						   WQ_MEM_RECLAIM);
 	if (di->chargalg_wq == NULL) {
 		dev_err(di->dev, "failed to create work queue\n");
 		return -ENOMEM;
diff --git a/drivers/power/supply/act8945a_charger.c b/drivers/power/supply/act8945a_charger.c
new file mode 100644
index 000000000000..d1eb2e359532
--- /dev/null
+++ b/drivers/power/supply/act8945a_charger.c
@@ -0,0 +1,666 @@
+/*
+ * Power supply driver for the Active-semi ACT8945A PMIC
+ *
+ * Copyright (C) 2015 Atmel Corporation
+ *
+ * Author: Wenyou Yang <wenyou.yang@atmel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/power_supply.h>
+#include <linux/regmap.h>
+#include <linux/gpio/consumer.h>
+
+static const char *act8945a_charger_model = "ACT8945A";
+static const char *act8945a_charger_manufacturer = "Active-semi";
+
+/**
+ * ACT8945A Charger Register Map
+ */
+
+/* 0x70: Reserved */
+#define ACT8945A_APCH_CFG		0x71
+#define ACT8945A_APCH_STATUS		0x78
+#define ACT8945A_APCH_CTRL		0x79
+#define ACT8945A_APCH_STATE		0x7A
+
+/* ACT8945A_APCH_CFG */
+#define APCH_CFG_OVPSET			(0x3 << 0)
+#define APCH_CFG_OVPSET_6V6		(0x0 << 0)
+#define APCH_CFG_OVPSET_7V		(0x1 << 0)
+#define APCH_CFG_OVPSET_7V5		(0x2 << 0)
+#define APCH_CFG_OVPSET_8V		(0x3 << 0)
+#define APCH_CFG_PRETIMO		(0x3 << 2)
+#define APCH_CFG_PRETIMO_40_MIN		(0x0 << 2)
+#define APCH_CFG_PRETIMO_60_MIN		(0x1 << 2)
+#define APCH_CFG_PRETIMO_80_MIN		(0x2 << 2)
+#define APCH_CFG_PRETIMO_DISABLED	(0x3 << 2)
+#define APCH_CFG_TOTTIMO		(0x3 << 4)
+#define APCH_CFG_TOTTIMO_3_HOUR		(0x0 << 4)
+#define APCH_CFG_TOTTIMO_4_HOUR		(0x1 << 4)
+#define APCH_CFG_TOTTIMO_5_HOUR		(0x2 << 4)
+#define APCH_CFG_TOTTIMO_DISABLED	(0x3 << 4)
+#define APCH_CFG_SUSCHG			(0x1 << 7)
+
+#define APCH_STATUS_CHGDAT		BIT(0)
+#define APCH_STATUS_INDAT		BIT(1)
+#define APCH_STATUS_TEMPDAT		BIT(2)
+#define APCH_STATUS_TIMRDAT		BIT(3)
+#define APCH_STATUS_CHGSTAT		BIT(4)
+#define APCH_STATUS_INSTAT		BIT(5)
+#define APCH_STATUS_TEMPSTAT		BIT(6)
+#define APCH_STATUS_TIMRSTAT		BIT(7)
+
+#define APCH_CTRL_CHGEOCOUT		BIT(0)
+#define APCH_CTRL_INDIS			BIT(1)
+#define APCH_CTRL_TEMPOUT		BIT(2)
+#define APCH_CTRL_TIMRPRE		BIT(3)
+#define APCH_CTRL_CHGEOCIN		BIT(4)
+#define APCH_CTRL_INCON			BIT(5)
+#define APCH_CTRL_TEMPIN		BIT(6)
+#define APCH_CTRL_TIMRTOT		BIT(7)
+
+#define APCH_STATE_ACINSTAT		(0x1 << 1)
+#define APCH_STATE_CSTATE		(0x3 << 4)
+#define APCH_STATE_CSTATE_SHIFT		4
+#define APCH_STATE_CSTATE_DISABLED	0x00
+#define APCH_STATE_CSTATE_EOC		0x01
+#define APCH_STATE_CSTATE_FAST		0x02
+#define APCH_STATE_CSTATE_PRE		0x03
+
+struct act8945a_charger {
+	struct power_supply *psy;
+	struct power_supply_desc desc;
+	struct regmap *regmap;
+	struct work_struct work;
+
+	bool init_done;
+	struct gpio_desc *lbo_gpio;
+	struct gpio_desc *chglev_gpio;
+};
+
+static int act8945a_get_charger_state(struct regmap *regmap, int *val)
+{
+	int ret;
+	unsigned int status, state;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATUS, &status);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATE, &state);
+	if (ret < 0)
+		return ret;
+
+	state &= APCH_STATE_CSTATE;
+	state >>= APCH_STATE_CSTATE_SHIFT;
+
+	switch (state) {
+	case APCH_STATE_CSTATE_PRE:
+	case APCH_STATE_CSTATE_FAST:
+		*val = POWER_SUPPLY_STATUS_CHARGING;
+		break;
+	case APCH_STATE_CSTATE_EOC:
+		if (status & APCH_STATUS_CHGDAT)
+			*val = POWER_SUPPLY_STATUS_FULL;
+		else
+			*val = POWER_SUPPLY_STATUS_CHARGING;
+		break;
+	case APCH_STATE_CSTATE_DISABLED:
+	default:
+		if (!(status & APCH_STATUS_INDAT))
+			*val = POWER_SUPPLY_STATUS_DISCHARGING;
+		else
+			*val = POWER_SUPPLY_STATUS_NOT_CHARGING;
+		break;
+	}
+
+	return 0;
+}
+
+static int act8945a_get_charge_type(struct regmap *regmap, int *val)
+{
+	int ret;
+	unsigned int status, state;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATUS, &status);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATE, &state);
+	if (ret < 0)
+		return ret;
+
+	state &= APCH_STATE_CSTATE;
+	state >>= APCH_STATE_CSTATE_SHIFT;
+
+	switch (state) {
+	case APCH_STATE_CSTATE_PRE:
+		*val = POWER_SUPPLY_CHARGE_TYPE_TRICKLE;
+		break;
+	case APCH_STATE_CSTATE_FAST:
+		*val = POWER_SUPPLY_CHARGE_TYPE_FAST;
+		break;
+	case APCH_STATE_CSTATE_EOC:
+		*val = POWER_SUPPLY_CHARGE_TYPE_NONE;
+		break;
+	case APCH_STATE_CSTATE_DISABLED:
+	default:
+		if (!(status & APCH_STATUS_INDAT))
+			*val = POWER_SUPPLY_CHARGE_TYPE_NONE;
+		else
+			*val = POWER_SUPPLY_CHARGE_TYPE_UNKNOWN;
+		break;
+	}
+
+	return 0;
+}
+
+static int act8945a_get_battery_health(struct regmap *regmap, int *val)
+{
+	int ret;
+	unsigned int status, state, config;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATUS, &status);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_CFG, &config);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATE, &state);
+	if (ret < 0)
+		return ret;
+
+	state &= APCH_STATE_CSTATE;
+	state >>= APCH_STATE_CSTATE_SHIFT;
+
+	switch (state) {
+	case APCH_STATE_CSTATE_DISABLED:
+		if (config & APCH_CFG_SUSCHG) {
+			*val = POWER_SUPPLY_HEALTH_UNKNOWN;
+		} else if (status & APCH_STATUS_INDAT) {
+			if (!(status & APCH_STATUS_TEMPDAT))
+				*val = POWER_SUPPLY_HEALTH_OVERHEAT;
+			else if (status & APCH_STATUS_TIMRDAT)
+				*val = POWER_SUPPLY_HEALTH_SAFETY_TIMER_EXPIRE;
+			else
+				*val = POWER_SUPPLY_HEALTH_OVERVOLTAGE;
+		} else {
+			*val = POWER_SUPPLY_HEALTH_GOOD;
+		}
+		break;
+	case APCH_STATE_CSTATE_PRE:
+	case APCH_STATE_CSTATE_FAST:
+	case APCH_STATE_CSTATE_EOC:
+	default:
+		*val = POWER_SUPPLY_HEALTH_GOOD;
+		break;
+	}
+
+	return 0;
+}
+
+static int act8945a_get_capacity_level(struct act8945a_charger *charger,
+				       struct regmap *regmap, int *val)
+{
+	int ret;
+	unsigned int status, state, config;
+	int lbo_level = gpiod_get_value(charger->lbo_gpio);
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATUS, &status);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_CFG, &config);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATE, &state);
+	if (ret < 0)
+		return ret;
+
+	state &= APCH_STATE_CSTATE;
+	state >>= APCH_STATE_CSTATE_SHIFT;
+
+	switch (state) {
+	case APCH_STATE_CSTATE_PRE:
+		*val = POWER_SUPPLY_CAPACITY_LEVEL_LOW;
+		break;
+	case APCH_STATE_CSTATE_FAST:
+		if (lbo_level)
+			*val = POWER_SUPPLY_CAPACITY_LEVEL_HIGH;
+		else
+			*val = POWER_SUPPLY_CAPACITY_LEVEL_LOW;
+		break;
+	case APCH_STATE_CSTATE_EOC:
+		if (status & APCH_STATUS_CHGDAT)
+			*val = POWER_SUPPLY_CAPACITY_LEVEL_FULL;
+		else
+			*val = POWER_SUPPLY_CAPACITY_LEVEL_NORMAL;
+		break;
+	case APCH_STATE_CSTATE_DISABLED:
+	default:
+		if (config & APCH_CFG_SUSCHG) {
+			*val = POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN;
+		} else {
+			*val = POWER_SUPPLY_CAPACITY_LEVEL_NORMAL;
+			if (!(status & APCH_STATUS_INDAT)) {
+				if (!lbo_level)
+					*val = POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL;
+			}
+		}
+		break;
+	}
+
+	return 0;
+}
+
+#define MAX_CURRENT_USB_HIGH	450000
+#define MAX_CURRENT_USB_LOW	90000
+#define MAX_CURRENT_USB_PRE	45000
+/*
+ * Riset(K) = 2336 * (1V/Ichg(mA)) - 0.205
+ * Riset = 2.43K
+ */
+#define MAX_CURRENT_AC_HIGH		886527
+#define MAX_CURRENT_AC_LOW		117305
+#define MAX_CURRENT_AC_HIGH_PRE		88653
+#define MAX_CURRENT_AC_LOW_PRE		11731
+
+static int act8945a_get_current_max(struct act8945a_charger *charger,
+				    struct regmap *regmap, int *val)
+{
+	int ret;
+	unsigned int status, state;
+	unsigned int acin_state;
+	int chgin_level = gpiod_get_value(charger->chglev_gpio);
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATUS, &status);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(regmap, ACT8945A_APCH_STATE, &state);
+	if (ret < 0)
+		return ret;
+
+	acin_state = (state & APCH_STATE_ACINSTAT) >> 1;
+
+	state &= APCH_STATE_CSTATE;
+	state >>= APCH_STATE_CSTATE_SHIFT;
+
+	switch (state) {
+	case APCH_STATE_CSTATE_PRE:
+		if (acin_state) {
+			if (chgin_level)
+				*val = MAX_CURRENT_AC_HIGH_PRE;
+			else
+				*val = MAX_CURRENT_AC_LOW_PRE;
+		} else {
+			*val = MAX_CURRENT_USB_PRE;
+		}
+		break;
+	case APCH_STATE_CSTATE_FAST:
+		if (acin_state) {
+			if (chgin_level)
+				*val = MAX_CURRENT_AC_HIGH;
+			else
+				*val = MAX_CURRENT_AC_LOW;
+		} else {
+			if (chgin_level)
+				*val = MAX_CURRENT_USB_HIGH;
+			else
+				*val = MAX_CURRENT_USB_LOW;
+		}
+		break;
+	case APCH_STATE_CSTATE_EOC:
+	case APCH_STATE_CSTATE_DISABLED:
+	default:
+		*val = 0;
+		break;
+	}
+
+	return 0;
+}
+
+static enum power_supply_property act8945a_charger_props[] = {
+	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_CHARGE_TYPE,
+	POWER_SUPPLY_PROP_TECHNOLOGY,
+	POWER_SUPPLY_PROP_HEALTH,
+	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
+	POWER_SUPPLY_PROP_CURRENT_MAX,
+	POWER_SUPPLY_PROP_MODEL_NAME,
+	POWER_SUPPLY_PROP_MANUFACTURER
+};
+
+static int act8945a_charger_get_property(struct power_supply *psy,
+					 enum power_supply_property prop,
+					 union power_supply_propval *val)
+{
+	struct act8945a_charger *charger = power_supply_get_drvdata(psy);
+	struct regmap *regmap = charger->regmap;
+	int ret = 0;
+
+	switch (prop) {
+	case POWER_SUPPLY_PROP_STATUS:
+		ret = act8945a_get_charger_state(regmap, &val->intval);
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_TYPE:
+		ret = act8945a_get_charge_type(regmap, &val->intval);
+		break;
+	case POWER_SUPPLY_PROP_TECHNOLOGY:
+		val->intval = POWER_SUPPLY_TECHNOLOGY_LION;
+		break;
+	case POWER_SUPPLY_PROP_HEALTH:
+		ret = act8945a_get_battery_health(regmap, &val->intval);
+		break;
+	case POWER_SUPPLY_PROP_CAPACITY_LEVEL:
+		ret = act8945a_get_capacity_level(charger,
+						  regmap, &val->intval);
+		break;
+	case POWER_SUPPLY_PROP_CURRENT_MAX:
+		ret = act8945a_get_current_max(charger,
+					       regmap, &val->intval);
+		break;
+	case POWER_SUPPLY_PROP_MODEL_NAME:
+		val->strval = act8945a_charger_model;
+		break;
+	case POWER_SUPPLY_PROP_MANUFACTURER:
+		val->strval = act8945a_charger_manufacturer;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int act8945a_enable_interrupt(struct act8945a_charger *charger)
+{
+	struct regmap *regmap = charger->regmap;
+	unsigned char ctrl;
+	int ret;
+
+	ctrl = APCH_CTRL_CHGEOCOUT | APCH_CTRL_CHGEOCIN |
+	       APCH_CTRL_INDIS | APCH_CTRL_INCON |
+	       APCH_CTRL_TEMPOUT | APCH_CTRL_TEMPIN |
+	       APCH_CTRL_TIMRPRE | APCH_CTRL_TIMRTOT;
+	ret = regmap_write(regmap, ACT8945A_APCH_CTRL, ctrl);
+	if (ret)
+		return ret;
+
+	ctrl = APCH_STATUS_CHGSTAT | APCH_STATUS_INSTAT |
+	       APCH_STATUS_TEMPSTAT | APCH_STATUS_TIMRSTAT;
+	ret = regmap_write(regmap, ACT8945A_APCH_STATUS, ctrl);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static unsigned int act8945a_set_supply_type(struct act8945a_charger *charger,
+					     unsigned int *type)
+{
+	unsigned int status, state;
+	int ret;
+
+	ret = regmap_read(charger->regmap, ACT8945A_APCH_STATUS, &status);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(charger->regmap, ACT8945A_APCH_STATE, &state);
+	if (ret < 0)
+		return ret;
+
+	if (status & APCH_STATUS_INDAT) {
+		if (state & APCH_STATE_ACINSTAT)
+			*type = POWER_SUPPLY_TYPE_MAINS;
+		else
+			*type = POWER_SUPPLY_TYPE_USB;
+	} else {
+		*type = POWER_SUPPLY_TYPE_BATTERY;
+	}
+
+	return 0;
+}
+
+static void act8945a_work(struct work_struct *work)
+{
+	struct act8945a_charger *charger =
+			container_of(work, struct act8945a_charger, work);
+
+	act8945a_set_supply_type(charger, &charger->desc.type);
+
+	power_supply_changed(charger->psy);
+}
+
+static irqreturn_t act8945a_status_changed(int irq, void *dev_id)
+{
+	struct act8945a_charger *charger = dev_id;
+
+	if (charger->init_done)
+		schedule_work(&charger->work);
+
+	return IRQ_HANDLED;
+}
+
+#define DEFAULT_TOTAL_TIME_OUT		3
+#define DEFAULT_PRE_TIME_OUT		40
+#define DEFAULT_INPUT_OVP_THRESHOLD	6600
+
+static int act8945a_charger_config(struct device *dev,
+				   struct act8945a_charger *charger)
+{
+	struct device_node *np = dev->of_node;
+	struct regmap *regmap = charger->regmap;
+
+	u32 total_time_out;
+	u32 pre_time_out;
+	u32 input_voltage_threshold;
+	int err, ret;
+
+	unsigned int tmp;
+	unsigned int value = 0;
+
+	if (!np) {
+		dev_err(dev, "no charger of node\n");
+		return -EINVAL;
+	}
+
+	ret = regmap_read(regmap, ACT8945A_APCH_CFG, &tmp);
+	if (ret)
+		return ret;
+
+	if (tmp & APCH_CFG_SUSCHG) {
+		value |= APCH_CFG_SUSCHG;
+		dev_info(dev, "have been suspended\n");
+	}
+
+	charger->lbo_gpio = devm_gpiod_get_optional(dev, "active-semi,lbo",
+						    GPIOD_IN);
+	if (IS_ERR(charger->lbo_gpio)) {
+		err = PTR_ERR(charger->lbo_gpio);
+		dev_err(dev, "unable to claim gpio \"lbo\": %d\n", err);
+		return err;
+	}
+
+	ret = devm_request_irq(dev, gpiod_to_irq(charger->lbo_gpio),
+			       act8945a_status_changed,
+			       (IRQF_TRIGGER_FALLING | IRQF_TRIGGER_RISING),
+			       "act8945a_lbo_detect", charger);
+	if (ret)
+		dev_info(dev, "failed to request gpio \"lbo\" IRQ\n");
+
+	charger->chglev_gpio = devm_gpiod_get_optional(dev,
+						       "active-semi,chglev",
+						       GPIOD_IN);
+	if (IS_ERR(charger->chglev_gpio)) {
+		err = PTR_ERR(charger->chglev_gpio);
+		dev_err(dev, "unable to claim gpio \"chglev\": %d\n", err);
+		return err;
+	}
+
+	if (of_property_read_u32(np,
+				 "active-semi,input-voltage-threshold-microvolt",
+				 &input_voltage_threshold))
+		input_voltage_threshold = DEFAULT_INPUT_OVP_THRESHOLD;
+
+	if (of_property_read_u32(np,
+				 "active-semi,precondition-timeout",
+				 &pre_time_out))
+		pre_time_out = DEFAULT_PRE_TIME_OUT;
+
+	if (of_property_read_u32(np, "active-semi,total-timeout",
+				 &total_time_out))
+		total_time_out = DEFAULT_TOTAL_TIME_OUT;
+
+	switch (input_voltage_threshold) {
+	case 8000:
+		value |= APCH_CFG_OVPSET_8V;
+		break;
+	case 7500:
+		value |= APCH_CFG_OVPSET_7V5;
+		break;
+	case 7000:
+		value |= APCH_CFG_OVPSET_7V;
+		break;
+	case 6600:
+	default:
+		value |= APCH_CFG_OVPSET_6V6;
+		break;
+	}
+
+	switch (pre_time_out) {
+	case 60:
+		value |= APCH_CFG_PRETIMO_60_MIN;
+		break;
+	case 80:
+		value |= APCH_CFG_PRETIMO_80_MIN;
+		break;
+	case 0:
+		value |= APCH_CFG_PRETIMO_DISABLED;
+		break;
+	case 40:
+	default:
+		value |= APCH_CFG_PRETIMO_40_MIN;
+		break;
+	}
+
+	switch (total_time_out) {
+	case 4:
+		value |= APCH_CFG_TOTTIMO_4_HOUR;
+		break;
+	case 5:
+		value |= APCH_CFG_TOTTIMO_5_HOUR;
+		break;
+	case 0:
+		value |= APCH_CFG_TOTTIMO_DISABLED;
+		break;
+	case 3:
+	default:
+		value |= APCH_CFG_TOTTIMO_3_HOUR;
+		break;
+	}
+
+	return regmap_write(regmap, ACT8945A_APCH_CFG, value);
+}
+
+static int act8945a_charger_probe(struct platform_device *pdev)
+{
+	struct act8945a_charger *charger;
+	struct power_supply_config psy_cfg = {};
+	int irq, ret;
+
+	charger = devm_kzalloc(&pdev->dev, sizeof(*charger), GFP_KERNEL);
+	if (!charger)
+		return -ENOMEM;
+
+	charger->regmap = dev_get_regmap(pdev->dev.parent, NULL);
+	if (!charger->regmap) {
+		dev_err(&pdev->dev, "Parent did not provide regmap\n");
+		return -EINVAL;
+	}
+
+	ret = act8945a_charger_config(&pdev->dev, charger);
+	if (ret)
+		return ret;
+
+	irq = of_irq_get(pdev->dev.of_node, 0);
+	if (irq == -EPROBE_DEFER) {
+		dev_err(&pdev->dev, "failed to find IRQ number\n");
+		return -EPROBE_DEFER;
+	}
+
+	ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed,
+			       IRQF_TRIGGER_FALLING, "act8945a_interrupt",
+			       charger);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to request nIRQ pin IRQ\n");
+		return ret;
+	}
+
+	charger->desc.name = "act8945a-charger";
+	charger->desc.get_property = act8945a_charger_get_property;
+	charger->desc.properties = act8945a_charger_props;
+	charger->desc.num_properties = ARRAY_SIZE(act8945a_charger_props);
+
+	ret = act8945a_set_supply_type(charger, &charger->desc.type);
+	if (ret)
+		return -EINVAL;
+
+	psy_cfg.of_node	= pdev->dev.of_node;
+	psy_cfg.drv_data = charger;
+
+	charger->psy = devm_power_supply_register(&pdev->dev,
+						  &charger->desc,
+						  &psy_cfg);
+	if (IS_ERR(charger->psy)) {
+		dev_err(&pdev->dev, "failed to register power supply\n");
+		return PTR_ERR(charger->psy);
+	}
+
+	platform_set_drvdata(pdev, charger);
+
+	INIT_WORK(&charger->work, act8945a_work);
+
+	ret = act8945a_enable_interrupt(charger);
+	if (ret)
+		return -EIO;
+
+	charger->init_done = true;
+
+	return 0;
+}
+
+static int act8945a_charger_remove(struct platform_device *pdev)
+{
+	struct act8945a_charger *charger = platform_get_drvdata(pdev);
+
+	charger->init_done = false;
+	cancel_work_sync(&charger->work);
+
+	return 0;
+}
+
+static struct platform_driver act8945a_charger_driver = {
+	.driver	= {
+		.name = "act8945a-charger",
+	},
+	.probe	= act8945a_charger_probe,
+	.remove = act8945a_charger_remove,
+};
+module_platform_driver(act8945a_charger_driver);
+
+MODULE_DESCRIPTION("Active-semi ACT8945A ActivePath charger driver");
+MODULE_AUTHOR("Wenyou Yang <wenyou.yang@atmel.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/power/apm_power.c b/drivers/power/supply/apm_power.c
index 9d1a7fbcaed4..9d1a7fbcaed4 100644
--- a/drivers/power/apm_power.c
+++ b/drivers/power/supply/apm_power.c
diff --git a/drivers/power/axp20x_usb_power.c b/drivers/power/supply/axp20x_usb_power.c
index 6af6feb7058d..6af6feb7058d 100644
--- a/drivers/power/axp20x_usb_power.c
+++ b/drivers/power/supply/axp20x_usb_power.c
diff --git a/drivers/power/axp288_charger.c b/drivers/power/supply/axp288_charger.c
index 4030eeb7cf65..75b8e0c7402b 100644
--- a/drivers/power/axp288_charger.c
+++ b/drivers/power/supply/axp288_charger.c
@@ -23,7 +23,6 @@
 #include <linux/usb/otg.h>
 #include <linux/notifier.h>
 #include <linux/power_supply.h>
-#include <linux/notifier.h>
 #include <linux/property.h>
 #include <linux/mfd/axp20x.h>
 #include <linux/extcon.h>
diff --git a/drivers/power/axp288_fuel_gauge.c b/drivers/power/supply/axp288_fuel_gauge.c
index 50c0110d6b58..5bdde692f724 100644
--- a/drivers/power/axp288_fuel_gauge.c
+++ b/drivers/power/supply/axp288_fuel_gauge.c
@@ -22,7 +22,6 @@
 #include <linux/regmap.h>
 #include <linux/jiffies.h>
 #include <linux/interrupt.h>
-#include <linux/device.h>
 #include <linux/workqueue.h>
 #include <linux/mfd/axp20x.h>
 #include <linux/platform_device.h>
diff --git a/drivers/power/bq2415x_charger.c b/drivers/power/supply/bq2415x_charger.c
index 73e2f0b79dd4..73e2f0b79dd4 100644
--- a/drivers/power/bq2415x_charger.c
+++ b/drivers/power/supply/bq2415x_charger.c
diff --git a/drivers/power/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c
index f5746b9f4e83..f5746b9f4e83 100644
--- a/drivers/power/bq24190_charger.c
+++ b/drivers/power/supply/bq24190_charger.c
diff --git a/drivers/power/bq24257_charger.c b/drivers/power/supply/bq24257_charger.c
index 1fea2c7ef97f..6fc31bdc639b 100644
--- a/drivers/power/bq24257_charger.c
+++ b/drivers/power/supply/bq24257_charger.c
@@ -1068,6 +1068,12 @@ static int bq24257_probe(struct i2c_client *client,
 		return ret;
 	}
 
+	ret = bq24257_power_supply_init(bq);
+	if (ret < 0) {
+		dev_err(dev, "Failed to register power supply\n");
+		return ret;
+	}
+
 	ret = devm_request_threaded_irq(dev, client->irq, NULL,
 					bq24257_irq_handler_thread,
 					IRQF_TRIGGER_FALLING |
@@ -1078,12 +1084,6 @@ static int bq24257_probe(struct i2c_client *client,
 		return ret;
 	}
 
-	ret = bq24257_power_supply_init(bq);
-	if (ret < 0) {
-		dev_err(dev, "Failed to register power supply\n");
-		return ret;
-	}
-
 	ret = sysfs_create_group(&bq->charger->dev.kobj, &bq24257_attr_group);
 	if (ret < 0) {
 		dev_err(dev, "Can't create sysfs entries\n");
diff --git a/drivers/power/bq24735-charger.c b/drivers/power/supply/bq24735-charger.c
index fa454c19ce17..eb7783b42e0a 100644
--- a/drivers/power/bq24735-charger.c
+++ b/drivers/power/supply/bq24735-charger.c
@@ -25,7 +25,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/power_supply.h>
 #include <linux/slab.h>
 
@@ -49,6 +49,7 @@ struct bq24735 {
 	struct i2c_client		*client;
 	struct bq24735_platform		*pdata;
 	struct mutex			lock;
+	struct gpio_desc		*status_gpio;
 	bool				charging;
 };
 
@@ -177,12 +178,8 @@ static int bq24735_config_charger(struct bq24735 *charger)
 
 static bool bq24735_charger_is_present(struct bq24735 *charger)
 {
-	struct bq24735_platform *pdata = charger->pdata;
-	int ret;
-
-	if (pdata->status_gpio_valid) {
-		ret = gpio_get_value_cansleep(pdata->status_gpio);
-		return ret ^= pdata->status_gpio_active_low == 0;
+	if (charger->status_gpio) {
+		return !gpiod_get_value_cansleep(charger->status_gpio);
 	} else {
 		int ac = 0;
 
@@ -201,8 +198,12 @@ static bool bq24735_charger_is_present(struct bq24735 *charger)
 
 static int bq24735_charger_is_charging(struct bq24735 *charger)
 {
-	int ret = bq24735_read_word(charger->client, BQ24735_CHG_OPT);
+	int ret;
 
+	if (!bq24735_charger_is_present(charger))
+		return 0;
+
+	ret  = bq24735_read_word(charger->client, BQ24735_CHG_OPT);
 	if (ret < 0)
 		return ret;
 
@@ -304,7 +305,6 @@ static struct bq24735_platform *bq24735_parse_dt_data(struct i2c_client *client)
 	struct device_node *np = client->dev.of_node;
 	u32 val;
 	int ret;
-	enum of_gpio_flags flags;
 
 	pdata = devm_kzalloc(&client->dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata) {
@@ -313,12 +313,6 @@ static struct bq24735_platform *bq24735_parse_dt_data(struct i2c_client *client)
 		return NULL;
 	}
 
-	pdata->status_gpio = of_get_named_gpio_flags(np, "ti,ac-detect-gpios",
-						     0, &flags);
-
-	if (flags & OF_GPIO_ACTIVE_LOW)
-		pdata->status_gpio_active_low = 1;
-
 	ret = of_property_read_u32(np, "ti,charge-current", &val);
 	if (!ret)
 		pdata->charge_current = val;
@@ -392,21 +386,16 @@ static int bq24735_charger_probe(struct i2c_client *client,
 
 	i2c_set_clientdata(client, charger);
 
-	if (gpio_is_valid(charger->pdata->status_gpio)) {
-		ret = devm_gpio_request(&client->dev,
-					charger->pdata->status_gpio,
-					name);
-		if (ret) {
-			dev_err(&client->dev,
-				"Failed GPIO request for GPIO %d: %d\n",
-				charger->pdata->status_gpio, ret);
-		}
-
-		charger->pdata->status_gpio_valid = !ret;
+	charger->status_gpio = devm_gpiod_get_optional(&client->dev,
+						       "ti,ac-detect",
+						       GPIOD_IN);
+	if (IS_ERR(charger->status_gpio)) {
+		ret = PTR_ERR(charger->status_gpio);
+		dev_err(&client->dev, "Getting gpio failed: %d\n", ret);
+		return ret;
 	}
 
-	if (!charger->pdata->status_gpio_valid
-	    || bq24735_charger_is_present(charger)) {
+	if (!charger->status_gpio || bq24735_charger_is_present(charger)) {
 		ret = bq24735_read_word(client, BQ24735_MANUFACTURER_ID);
 		if (ret < 0) {
 			dev_err(&client->dev, "Failed to read manufacturer id : %d\n",
diff --git a/drivers/power/bq25890_charger.c b/drivers/power/supply/bq25890_charger.c
index f993a55cde20..f993a55cde20 100644
--- a/drivers/power/bq25890_charger.c
+++ b/drivers/power/supply/bq25890_charger.c
diff --git a/drivers/power/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c
index 323d05a12f9b..3b0dbc689d72 100644
--- a/drivers/power/bq27xxx_battery.c
+++ b/drivers/power/supply/bq27xxx_battery.c
@@ -39,6 +39,7 @@
 
 #include <linux/device.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/param.h>
 #include <linux/jiffies.h>
 #include <linux/workqueue.h>
@@ -390,8 +391,35 @@ static struct {
 	BQ27XXX_PROP(BQ27421, bq27421_battery_props),
 };
 
+static DEFINE_MUTEX(bq27xxx_list_lock);
+static LIST_HEAD(bq27xxx_battery_devices);
+
+static int poll_interval_param_set(const char *val, const struct kernel_param *kp)
+{
+	struct bq27xxx_device_info *di;
+	int ret;
+
+	ret = param_set_uint(val, kp);
+	if (ret < 0)
+		return ret;
+
+	mutex_lock(&bq27xxx_list_lock);
+	list_for_each_entry(di, &bq27xxx_battery_devices, list) {
+		cancel_delayed_work_sync(&di->work);
+		schedule_delayed_work(&di->work, 0);
+	}
+	mutex_unlock(&bq27xxx_list_lock);
+
+	return ret;
+}
+
+static const struct kernel_param_ops param_ops_poll_interval = {
+	.get = param_get_uint,
+	.set = poll_interval_param_set,
+};
+
 static unsigned int poll_interval = 360;
-module_param(poll_interval, uint, 0644);
+module_param_cb(poll_interval, &param_ops_poll_interval, &poll_interval, 0644);
 MODULE_PARM_DESC(poll_interval,
 		 "battery poll interval in seconds - 0 disables polling");
 
@@ -644,8 +672,9 @@ static bool bq27xxx_battery_dead(struct bq27xxx_device_info *di, u16 flags)
 static int bq27xxx_battery_read_health(struct bq27xxx_device_info *di)
 {
 	int flags;
+	bool has_singe_flag = di->chip == BQ27000 || di->chip == BQ27010;
 
-	flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, false);
+	flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, has_singe_flag);
 	if (flags < 0) {
 		dev_err(di->dev, "error reading flag register:%d\n", flags);
 		return flags;
@@ -745,7 +774,7 @@ static int bq27xxx_battery_current(struct bq27xxx_device_info *di,
 	}
 
 	if (di->chip == BQ27000 || di->chip == BQ27010) {
-		flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, false);
+		flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, true);
 		if (flags & BQ27000_FLAG_CHGS) {
 			dev_dbg(di->dev, "negative current!\n");
 			curr = -curr;
@@ -971,6 +1000,10 @@ int bq27xxx_battery_setup(struct bq27xxx_device_info *di)
 
 	bq27xxx_battery_update(di);
 
+	mutex_lock(&bq27xxx_list_lock);
+	list_add(&di->list, &bq27xxx_battery_devices);
+	mutex_unlock(&bq27xxx_list_lock);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(bq27xxx_battery_setup);
@@ -989,6 +1022,10 @@ void bq27xxx_battery_teardown(struct bq27xxx_device_info *di)
 
 	power_supply_unregister(di->bat);
 
+	mutex_lock(&bq27xxx_list_lock);
+	list_del(&di->list);
+	mutex_unlock(&bq27xxx_list_lock);
+
 	mutex_destroy(&di->lock);
 }
 EXPORT_SYMBOL_GPL(bq27xxx_battery_teardown);
diff --git a/drivers/power/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c
index 85d4ea2a9c20..85d4ea2a9c20 100644
--- a/drivers/power/bq27xxx_battery_i2c.c
+++ b/drivers/power/supply/bq27xxx_battery_i2c.c
diff --git a/drivers/power/charger-manager.c b/drivers/power/supply/charger-manager.c
index e664ca7c0afd..e664ca7c0afd 100644
--- a/drivers/power/charger-manager.c
+++ b/drivers/power/supply/charger-manager.c
diff --git a/drivers/power/collie_battery.c b/drivers/power/supply/collie_battery.c
index 3a0bc608d4b5..3a0bc608d4b5 100644
--- a/drivers/power/collie_battery.c
+++ b/drivers/power/supply/collie_battery.c
diff --git a/drivers/power/da9030_battery.c b/drivers/power/supply/da9030_battery.c
index 5ca0f4d90792..5ca0f4d90792 100644
--- a/drivers/power/da9030_battery.c
+++ b/drivers/power/supply/da9030_battery.c
diff --git a/drivers/power/da9052-battery.c b/drivers/power/supply/da9052-battery.c
index 830ec46fe7d0..830ec46fe7d0 100644
--- a/drivers/power/da9052-battery.c
+++ b/drivers/power/supply/da9052-battery.c
diff --git a/drivers/power/da9150-charger.c b/drivers/power/supply/da9150-charger.c
index 60099815296e..60099815296e 100644
--- a/drivers/power/da9150-charger.c
+++ b/drivers/power/supply/da9150-charger.c
diff --git a/drivers/power/da9150-fg.c b/drivers/power/supply/da9150-fg.c
index 8b8ce978656a..8b8ce978656a 100644
--- a/drivers/power/da9150-fg.c
+++ b/drivers/power/supply/da9150-fg.c
diff --git a/drivers/power/ds2760_battery.c b/drivers/power/supply/ds2760_battery.c
index 80f73ccb77ab..17225689e3f6 100644
--- a/drivers/power/ds2760_battery.c
+++ b/drivers/power/supply/ds2760_battery.c
@@ -28,8 +28,8 @@
 #include <linux/platform_device.h>
 #include <linux/power_supply.h>
 
-#include "../w1/w1.h"
-#include "../w1/slaves/w1_ds2760.h"
+#include "../../w1/w1.h"
+#include "../../w1/slaves/w1_ds2760.h"
 
 struct ds2760_device_info {
 	struct device *dev;
@@ -566,7 +566,8 @@ static int ds2760_battery_probe(struct platform_device *pdev)
 	INIT_DELAYED_WORK(&di->monitor_work, ds2760_battery_work);
 	INIT_DELAYED_WORK(&di->set_charged_work,
 			  ds2760_battery_set_charged_work);
-	di->monitor_wqueue = create_singlethread_workqueue(dev_name(&pdev->dev));
+	di->monitor_wqueue = alloc_ordered_workqueue(dev_name(&pdev->dev),
+						     WQ_MEM_RECLAIM);
 	if (!di->monitor_wqueue) {
 		retval = -ESRCH;
 		goto workqueue_failed;
diff --git a/drivers/power/ds2780_battery.c b/drivers/power/supply/ds2780_battery.c
index d3743d0ad55b..1b3b6fa89c28 100644
--- a/drivers/power/ds2780_battery.c
+++ b/drivers/power/supply/ds2780_battery.c
@@ -21,8 +21,8 @@
 #include <linux/power_supply.h>
 #include <linux/idr.h>
 
-#include "../w1/w1.h"
-#include "../w1/slaves/w1_ds2780.h"
+#include "../../w1/w1.h"
+#include "../../w1/slaves/w1_ds2780.h"
 
 /* Current unit measurement in uA for a 1 milli-ohm sense resistor */
 #define DS2780_CURRENT_UNITS	1563
diff --git a/drivers/power/ds2781_battery.c b/drivers/power/supply/ds2781_battery.c
index c3680024f399..cc0149131f89 100644
--- a/drivers/power/ds2781_battery.c
+++ b/drivers/power/supply/ds2781_battery.c
@@ -19,8 +19,8 @@
 #include <linux/power_supply.h>
 #include <linux/idr.h>
 
-#include "../w1/w1.h"
-#include "../w1/slaves/w1_ds2781.h"
+#include "../../w1/w1.h"
+#include "../../w1/slaves/w1_ds2781.h"
 
 /* Current unit measurement in uA for a 1 milli-ohm sense resistor */
 #define DS2781_CURRENT_UNITS	1563
diff --git a/drivers/power/ds2782_battery.c b/drivers/power/supply/ds2782_battery.c
index a1b7e0592245..a1b7e0592245 100644
--- a/drivers/power/ds2782_battery.c
+++ b/drivers/power/supply/ds2782_battery.c
diff --git a/drivers/power/generic-adc-battery.c b/drivers/power/supply/generic-adc-battery.c
index edb36bf781b0..edb36bf781b0 100644
--- a/drivers/power/generic-adc-battery.c
+++ b/drivers/power/supply/generic-adc-battery.c
diff --git a/drivers/power/goldfish_battery.c b/drivers/power/supply/goldfish_battery.c
index f5c525e4482a..f5c525e4482a 100644
--- a/drivers/power/goldfish_battery.c
+++ b/drivers/power/supply/goldfish_battery.c
diff --git a/drivers/power/gpio-charger.c b/drivers/power/supply/gpio-charger.c
index c5869b1941ac..c5869b1941ac 100644
--- a/drivers/power/gpio-charger.c
+++ b/drivers/power/supply/gpio-charger.c
diff --git a/drivers/power/intel_mid_battery.c b/drivers/power/supply/intel_mid_battery.c
index 9fa4acc107ca..dc7feef1bea4 100644
--- a/drivers/power/intel_mid_battery.c
+++ b/drivers/power/supply/intel_mid_battery.c
@@ -689,8 +689,7 @@ static int probe(int irq, struct device *dev)
 	/* initialize all required framework before enabling interrupts */
 	INIT_WORK(&pbi->handler, pmic_battery_handle_intrpt);
 	INIT_DELAYED_WORK(&pbi->monitor_battery, pmic_battery_monitor);
-	pbi->monitor_wqueue =
-			create_singlethread_workqueue(dev_name(dev));
+	pbi->monitor_wqueue = alloc_workqueue(dev_name(dev), WQ_MEM_RECLAIM, 0);
 	if (!pbi->monitor_wqueue) {
 		dev_err(dev, "%s(): wqueue init failed\n", __func__);
 		retval = -ESRCH;
diff --git a/drivers/power/ipaq_micro_battery.c b/drivers/power/supply/ipaq_micro_battery.c
index 35b01c7d775b..4af7b770f293 100644
--- a/drivers/power/ipaq_micro_battery.c
+++ b/drivers/power/supply/ipaq_micro_battery.c
@@ -235,7 +235,7 @@ static int micro_batt_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	mb->micro = dev_get_drvdata(pdev->dev.parent);
-	mb->wq = create_singlethread_workqueue("ipaq-battery-wq");
+	mb->wq = alloc_workqueue("ipaq-battery-wq", WQ_MEM_RECLAIM, 0);
 	if (!mb->wq)
 		return -ENOMEM;
 
diff --git a/drivers/power/isp1704_charger.c b/drivers/power/supply/isp1704_charger.c
index 4cd6899b961e..4cd6899b961e 100644
--- a/drivers/power/isp1704_charger.c
+++ b/drivers/power/supply/isp1704_charger.c
diff --git a/drivers/power/jz4740-battery.c b/drivers/power/supply/jz4740-battery.c
index 88f04f4d1a70..88f04f4d1a70 100644
--- a/drivers/power/jz4740-battery.c
+++ b/drivers/power/supply/jz4740-battery.c
diff --git a/drivers/power/lp8727_charger.c b/drivers/power/supply/lp8727_charger.c
index 042fb3dacb46..042fb3dacb46 100644
--- a/drivers/power/lp8727_charger.c
+++ b/drivers/power/supply/lp8727_charger.c
diff --git a/drivers/power/lp8788-charger.c b/drivers/power/supply/lp8788-charger.c
index 7321b727d484..7321b727d484 100644
--- a/drivers/power/lp8788-charger.c
+++ b/drivers/power/supply/lp8788-charger.c
diff --git a/drivers/power/ltc2941-battery-gauge.c b/drivers/power/supply/ltc2941-battery-gauge.c
index 4adf2ba021ce..4adf2ba021ce 100644
--- a/drivers/power/ltc2941-battery-gauge.c
+++ b/drivers/power/supply/ltc2941-battery-gauge.c
diff --git a/drivers/power/max14577_charger.c b/drivers/power/supply/max14577_charger.c
index a36bcaf62dd4..449fc56f09eb 100644
--- a/drivers/power/max14577_charger.c
+++ b/drivers/power/supply/max14577_charger.c
@@ -2,7 +2,7 @@
  * max14577_charger.c - Battery charger driver for the Maxim 14577/77836
  *
  * Copyright (C) 2013,2014 Samsung Electronics
- * Krzysztof Kozlowski <k.kozlowski@samsung.com>
+ * Krzysztof Kozlowski <krzk@kernel.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -643,6 +643,6 @@ static struct platform_driver max14577_charger_driver = {
 };
 module_platform_driver(max14577_charger_driver);
 
-MODULE_AUTHOR("Krzysztof Kozlowski <k.kozlowski@samsung.com>");
+MODULE_AUTHOR("Krzysztof Kozlowski <krzk@kernel.org>");
 MODULE_DESCRIPTION("Maxim 14577/77836 charger driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/power/max17040_battery.c b/drivers/power/supply/max17040_battery.c
index 8689c80202b5..8689c80202b5 100644
--- a/drivers/power/max17040_battery.c
+++ b/drivers/power/supply/max17040_battery.c
diff --git a/drivers/power/max17042_battery.c b/drivers/power/supply/max17042_battery.c
index 9c65f134d447..da7a75f82489 100644
--- a/drivers/power/max17042_battery.c
+++ b/drivers/power/supply/max17042_battery.c
@@ -457,13 +457,16 @@ static inline void max17042_write_model_data(struct max17042_chip *chip,
 }
 
 static inline void max17042_read_model_data(struct max17042_chip *chip,
-					u8 addr, u32 *data, int size)
+					u8 addr, u16 *data, int size)
 {
 	struct regmap *map = chip->regmap;
 	int i;
+	u32 tmp;
 
-	for (i = 0; i < size; i++)
-		regmap_read(map, addr + i, &data[i]);
+	for (i = 0; i < size; i++) {
+		regmap_read(map, addr + i, &tmp);
+		data[i] = (u16)tmp;
+	}
 }
 
 static inline int max17042_model_data_compare(struct max17042_chip *chip,
@@ -486,7 +489,7 @@ static int max17042_init_model(struct max17042_chip *chip)
 {
 	int ret;
 	int table_size = ARRAY_SIZE(chip->pdata->config_data->cell_char_tbl);
-	u32 *temp_data;
+	u16 *temp_data;
 
 	temp_data = kcalloc(table_size, sizeof(*temp_data), GFP_KERNEL);
 	if (!temp_data)
@@ -501,7 +504,7 @@ static int max17042_init_model(struct max17042_chip *chip)
 	ret = max17042_model_data_compare(
 		chip,
 		chip->pdata->config_data->cell_char_tbl,
-		(u16 *)temp_data,
+		temp_data,
 		table_size);
 
 	max10742_lock_model(chip);
@@ -514,7 +517,7 @@ static int max17042_verify_model_lock(struct max17042_chip *chip)
 {
 	int i;
 	int table_size = ARRAY_SIZE(chip->pdata->config_data->cell_char_tbl);
-	u32 *temp_data;
+	u16 *temp_data;
 	int ret = 0;
 
 	temp_data = kcalloc(table_size, sizeof(*temp_data), GFP_KERNEL);
diff --git a/drivers/power/max77693_charger.c b/drivers/power/supply/max77693_charger.c
index 060cab5ae3aa..6c78884bad5e 100644
--- a/drivers/power/max77693_charger.c
+++ b/drivers/power/supply/max77693_charger.c
@@ -2,7 +2,7 @@
  * max77693_charger.c - Battery charger driver for the Maxim 77693
  *
  * Copyright (C) 2014 Samsung Electronics
- * Krzysztof Kozlowski <k.kozlowski@samsung.com>
+ * Krzysztof Kozlowski <krzk@kernel.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -766,6 +766,6 @@ static struct platform_driver max77693_charger_driver = {
 };
 module_platform_driver(max77693_charger_driver);
 
-MODULE_AUTHOR("Krzysztof Kozlowski <k.kozlowski@samsung.com>");
+MODULE_AUTHOR("Krzysztof Kozlowski <krzk@kernel.org>");
 MODULE_DESCRIPTION("Maxim 77693 charger driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/power/max8903_charger.c b/drivers/power/supply/max8903_charger.c
index fdc73d686153..fdc73d686153 100644
--- a/drivers/power/max8903_charger.c
+++ b/drivers/power/supply/max8903_charger.c
diff --git a/drivers/power/max8925_power.c b/drivers/power/supply/max8925_power.c
index 3b94620ce5c1..3b94620ce5c1 100644
--- a/drivers/power/max8925_power.c
+++ b/drivers/power/supply/max8925_power.c
diff --git a/drivers/power/max8997_charger.c b/drivers/power/supply/max8997_charger.c
index 0b2eab571528..0b2eab571528 100644
--- a/drivers/power/max8997_charger.c
+++ b/drivers/power/supply/max8997_charger.c
diff --git a/drivers/power/max8998_charger.c b/drivers/power/supply/max8998_charger.c
index b64cf0f14142..b64cf0f14142 100644
--- a/drivers/power/max8998_charger.c
+++ b/drivers/power/supply/max8998_charger.c
diff --git a/drivers/power/olpc_battery.c b/drivers/power/supply/olpc_battery.c
index 9e29b1321648..9e29b1321648 100644
--- a/drivers/power/olpc_battery.c
+++ b/drivers/power/supply/olpc_battery.c
diff --git a/drivers/power/pcf50633-charger.c b/drivers/power/supply/pcf50633-charger.c
index d05597b4e40f..d05597b4e40f 100644
--- a/drivers/power/pcf50633-charger.c
+++ b/drivers/power/supply/pcf50633-charger.c
diff --git a/drivers/power/pda_power.c b/drivers/power/supply/pda_power.c
index dfe1ee89f7c7..dfe1ee89f7c7 100644
--- a/drivers/power/pda_power.c
+++ b/drivers/power/supply/pda_power.c
diff --git a/drivers/power/pm2301_charger.c b/drivers/power/supply/pm2301_charger.c
index fb62ed3fc38c..78561b6884fc 100644
--- a/drivers/power/pm2301_charger.c
+++ b/drivers/power/supply/pm2301_charger.c
@@ -1054,7 +1054,8 @@ static int pm2xxx_wall_charger_probe(struct i2c_client *i2c_client,
 	pm2->ac_chg.external = true;
 
 	/* Create a work queue for the charger */
-	pm2->charger_wq = create_singlethread_workqueue("pm2xxx_charger_wq");
+	pm2->charger_wq = alloc_ordered_workqueue("pm2xxx_charger_wq",
+						  WQ_MEM_RECLAIM);
 	if (pm2->charger_wq == NULL) {
 		ret = -ENOMEM;
 		dev_err(pm2->dev, "failed to create work queue\n");
diff --git a/drivers/power/pm2301_charger.h b/drivers/power/supply/pm2301_charger.h
index 24181cf9717b..24181cf9717b 100644
--- a/drivers/power/pm2301_charger.h
+++ b/drivers/power/supply/pm2301_charger.h
diff --git a/drivers/power/pmu_battery.c b/drivers/power/supply/pmu_battery.c
index 9c8d5253812c..9c8d5253812c 100644
--- a/drivers/power/pmu_battery.c
+++ b/drivers/power/supply/pmu_battery.c
diff --git a/drivers/power/power_supply.h b/drivers/power/supply/power_supply.h
index cc439fd89d8d..cc439fd89d8d 100644
--- a/drivers/power/power_supply.h
+++ b/drivers/power/supply/power_supply.h
diff --git a/drivers/power/power_supply_core.c b/drivers/power/supply/power_supply_core.c
index a74d8ca383a1..a74d8ca383a1 100644
--- a/drivers/power/power_supply_core.c
+++ b/drivers/power/supply/power_supply_core.c
diff --git a/drivers/power/power_supply_leds.c b/drivers/power/supply/power_supply_leds.c
index 2277ad9c2f68..2277ad9c2f68 100644
--- a/drivers/power/power_supply_leds.c
+++ b/drivers/power/supply/power_supply_leds.c
diff --git a/drivers/power/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c
index bcde8d13476a..bcde8d13476a 100644
--- a/drivers/power/power_supply_sysfs.c
+++ b/drivers/power/supply/power_supply_sysfs.c
diff --git a/drivers/power/qcom_smbb.c b/drivers/power/supply/qcom_smbb.c
index b5896ba2a602..b5896ba2a602 100644
--- a/drivers/power/qcom_smbb.c
+++ b/drivers/power/supply/qcom_smbb.c
diff --git a/drivers/power/rt5033_battery.c b/drivers/power/supply/rt5033_battery.c
index bcdd83048492..bcdd83048492 100644
--- a/drivers/power/rt5033_battery.c
+++ b/drivers/power/supply/rt5033_battery.c
diff --git a/drivers/power/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c
index cfdbde9daf94..cfdbde9daf94 100644
--- a/drivers/power/rt9455_charger.c
+++ b/drivers/power/supply/rt9455_charger.c
diff --git a/drivers/power/rx51_battery.c b/drivers/power/supply/rx51_battery.c
index af9383d23d12..af9383d23d12 100644
--- a/drivers/power/rx51_battery.c
+++ b/drivers/power/supply/rx51_battery.c
diff --git a/drivers/power/s3c_adc_battery.c b/drivers/power/supply/s3c_adc_battery.c
index 0ffe5cd3abf6..0ffe5cd3abf6 100644
--- a/drivers/power/s3c_adc_battery.c
+++ b/drivers/power/supply/s3c_adc_battery.c
diff --git a/drivers/power/sbs-battery.c b/drivers/power/supply/sbs-battery.c
index 768b9fcb58ea..8bb2eb38eb1c 100644
--- a/drivers/power/sbs-battery.c
+++ b/drivers/power/supply/sbs-battery.c
@@ -26,7 +26,7 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/of.h>
 #include <linux/stat.h>
 
@@ -41,6 +41,7 @@ enum {
 	REG_TIME_TO_EMPTY,
 	REG_TIME_TO_FULL,
 	REG_STATUS,
+	REG_CAPACITY_LEVEL,
 	REG_CYCLE_COUNT,
 	REG_SERIAL_NUMBER,
 	REG_REMAINING_CAPACITY,
@@ -68,6 +69,7 @@ enum sbs_battery_mode {
 #define MANUFACTURER_ACCESS_SLEEP	0x0011
 
 /* battery status value bits */
+#define BATTERY_INITIALIZED		0x80
 #define BATTERY_DISCHARGING		0x40
 #define BATTERY_FULL_CHARGED		0x20
 #define BATTERY_FULL_DISCHARGED		0x10
@@ -110,6 +112,8 @@ static const struct chip_data {
 		SBS_DATA(POWER_SUPPLY_PROP_TIME_TO_FULL_AVG, 0x13, 0, 65535),
 	[REG_STATUS] =
 		SBS_DATA(POWER_SUPPLY_PROP_STATUS, 0x16, 0, 65535),
+	[REG_CAPACITY_LEVEL] =
+		SBS_DATA(POWER_SUPPLY_PROP_CAPACITY_LEVEL, 0x16, 0, 65535),
 	[REG_CYCLE_COUNT] =
 		SBS_DATA(POWER_SUPPLY_PROP_CYCLE_COUNT, 0x17, 0, 65535),
 	[REG_DESIGN_CAPACITY] =
@@ -131,6 +135,7 @@ static const struct chip_data {
 
 static enum power_supply_property sbs_properties[] = {
 	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
 	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
@@ -158,13 +163,13 @@ static enum power_supply_property sbs_properties[] = {
 struct sbs_info {
 	struct i2c_client		*client;
 	struct power_supply		*power_supply;
-	struct sbs_platform_data	*pdata;
 	bool				is_present;
-	bool				gpio_detect;
+	struct gpio_desc		*gpio_detect;
 	bool				enable_detection;
-	int				irq;
 	int				last_state;
 	int				poll_time;
+	u32				i2c_retry_count;
+	u32				poll_retry_count;
 	struct delayed_work		work;
 	int				ignore_changes;
 };
@@ -179,8 +184,7 @@ static int sbs_read_word_data(struct i2c_client *client, u8 address)
 	s32 ret = 0;
 	int retries = 1;
 
-	if (chip->pdata)
-		retries = max(chip->pdata->i2c_retry_count + 1, 1);
+	retries = chip->i2c_retry_count;
 
 	while (retries > 0) {
 		ret = i2c_smbus_read_word_data(client, address);
@@ -207,10 +211,8 @@ static int sbs_read_string_data(struct i2c_client *client, u8 address,
 	int retries_length = 1, retries_block = 1;
 	u8 block_buffer[I2C_SMBUS_BLOCK_MAX + 1];
 
-	if (chip->pdata) {
-		retries_length = max(chip->pdata->i2c_retry_count + 1, 1);
-		retries_block = max(chip->pdata->i2c_retry_count + 1, 1);
-	}
+	retries_length = chip->i2c_retry_count;
+	retries_block = chip->i2c_retry_count;
 
 	/* Adapter needs to support these two functions */
 	if (!i2c_check_functionality(client->adapter,
@@ -274,8 +276,7 @@ static int sbs_write_word_data(struct i2c_client *client, u8 address,
 	s32 ret = 0;
 	int retries = 1;
 
-	if (chip->pdata)
-		retries = max(chip->pdata->i2c_retry_count + 1, 1);
+	retries = chip->i2c_retry_count;
 
 	while (retries > 0) {
 		ret = i2c_smbus_write_word_data(client, address,
@@ -302,32 +303,31 @@ static int sbs_get_battery_presence_and_health(
 	s32 ret;
 	struct sbs_info *chip = i2c_get_clientdata(client);
 
-	if (psp == POWER_SUPPLY_PROP_PRESENT &&
-		chip->gpio_detect) {
-		ret = gpio_get_value(chip->pdata->battery_detect);
-		if (ret == chip->pdata->battery_detect_present)
-			val->intval = 1;
-		else
-			val->intval = 0;
+	if (psp == POWER_SUPPLY_PROP_PRESENT && chip->gpio_detect) {
+		ret = gpiod_get_value_cansleep(chip->gpio_detect);
+		if (ret < 0)
+			return ret;
+		val->intval = ret;
 		chip->is_present = val->intval;
 		return ret;
 	}
 
-	/* Write to ManufacturerAccess with
-	 * ManufacturerAccess command and then
-	 * read the status */
-	ret = sbs_write_word_data(client, sbs_data[REG_MANUFACTURER_DATA].addr,
-					MANUFACTURER_ACCESS_STATUS);
+	/*
+	 * Write to ManufacturerAccess with ManufacturerAccess command
+	 * and then read the status. Do not check for error on the write
+	 * since not all batteries implement write access to this command,
+	 * while others mandate it.
+	 */
+	sbs_write_word_data(client, sbs_data[REG_MANUFACTURER_DATA].addr,
+			    MANUFACTURER_ACCESS_STATUS);
+
+	ret = sbs_read_word_data(client, sbs_data[REG_MANUFACTURER_DATA].addr);
 	if (ret < 0) {
 		if (psp == POWER_SUPPLY_PROP_PRESENT)
 			val->intval = 0; /* battery removed */
 		return ret;
 	}
 
-	ret = sbs_read_word_data(client, sbs_data[REG_MANUFACTURER_DATA].addr);
-	if (ret < 0)
-		return ret;
-
 	if (ret < sbs_data[REG_MANUFACTURER_DATA].min_value ||
 	    ret > sbs_data[REG_MANUFACTURER_DATA].max_value) {
 		val->intval = 0;
@@ -377,8 +377,23 @@ static int sbs_get_battery_property(struct i2c_client *client,
 	if (ret >= sbs_data[reg_offset].min_value &&
 	    ret <= sbs_data[reg_offset].max_value) {
 		val->intval = ret;
-		if (psp != POWER_SUPPLY_PROP_STATUS)
+		if (psp == POWER_SUPPLY_PROP_CAPACITY_LEVEL) {
+			if (!(ret & BATTERY_INITIALIZED))
+				val->intval =
+					POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN;
+			else if (ret & BATTERY_FULL_CHARGED)
+				val->intval =
+					POWER_SUPPLY_CAPACITY_LEVEL_FULL;
+			else if (ret & BATTERY_FULL_DISCHARGED)
+				val->intval =
+					POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL;
+			else
+				val->intval =
+					POWER_SUPPLY_CAPACITY_LEVEL_NORMAL;
+			return 0;
+		} else if (psp != POWER_SUPPLY_PROP_STATUS) {
 			return 0;
+		}
 
 		if (ret & BATTERY_FULL_CHARGED)
 			val->intval = POWER_SUPPLY_STATUS_FULL;
@@ -590,6 +605,7 @@ static int sbs_get_property(struct power_supply *psy,
 		break;
 
 	case POWER_SUPPLY_PROP_STATUS:
+	case POWER_SUPPLY_PROP_CAPACITY_LEVEL:
 	case POWER_SUPPLY_PROP_CYCLE_COUNT:
 	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
 	case POWER_SUPPLY_PROP_CURRENT_NOW:
@@ -661,8 +677,14 @@ done:
 
 static irqreturn_t sbs_irq(int irq, void *devid)
 {
-	struct power_supply *battery = devid;
+	struct sbs_info *chip = devid;
+	struct power_supply *battery = chip->power_supply;
+	int ret;
 
+	ret = gpiod_get_value_cansleep(chip->gpio_detect);
+	if (ret < 0)
+		return ret;
+	chip->is_present = ret;
 	power_supply_changed(battery);
 
 	return IRQ_HANDLED;
@@ -681,7 +703,7 @@ static void sbs_external_power_changed(struct power_supply *psy)
 	cancel_delayed_work_sync(&chip->work);
 
 	schedule_delayed_work(&chip->work, HZ);
-	chip->poll_time = chip->pdata->poll_retry_count;
+	chip->poll_time = chip->poll_retry_count;
 }
 
 static void sbs_delayed_work(struct work_struct *work)
@@ -717,80 +739,6 @@ static void sbs_delayed_work(struct work_struct *work)
 	}
 }
 
-#if defined(CONFIG_OF)
-
-#include <linux/of_device.h>
-#include <linux/of_gpio.h>
-
-static const struct of_device_id sbs_dt_ids[] = {
-	{ .compatible = "sbs,sbs-battery" },
-	{ .compatible = "ti,bq20z75" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, sbs_dt_ids);
-
-static struct sbs_platform_data *sbs_of_populate_pdata(
-		struct i2c_client *client)
-{
-	struct device_node *of_node = client->dev.of_node;
-	struct sbs_platform_data *pdata = client->dev.platform_data;
-	enum of_gpio_flags gpio_flags;
-	int rc;
-	u32 prop;
-
-	/* verify this driver matches this device */
-	if (!of_node)
-		return NULL;
-
-	/* if platform data is set, honor it */
-	if (pdata)
-		return pdata;
-
-	/* first make sure at least one property is set, otherwise
-	 * it won't change behavior from running without pdata.
-	 */
-	if (!of_get_property(of_node, "sbs,i2c-retry-count", NULL) &&
-		!of_get_property(of_node, "sbs,poll-retry-count", NULL) &&
-		!of_get_property(of_node, "sbs,battery-detect-gpios", NULL))
-		goto of_out;
-
-	pdata = devm_kzalloc(&client->dev, sizeof(struct sbs_platform_data),
-				GFP_KERNEL);
-	if (!pdata)
-		goto of_out;
-
-	rc = of_property_read_u32(of_node, "sbs,i2c-retry-count", &prop);
-	if (!rc)
-		pdata->i2c_retry_count = prop;
-
-	rc = of_property_read_u32(of_node, "sbs,poll-retry-count", &prop);
-	if (!rc)
-		pdata->poll_retry_count = prop;
-
-	if (!of_get_property(of_node, "sbs,battery-detect-gpios", NULL)) {
-		pdata->battery_detect = -1;
-		goto of_out;
-	}
-
-	pdata->battery_detect = of_get_named_gpio_flags(of_node,
-			"sbs,battery-detect-gpios", 0, &gpio_flags);
-
-	if (gpio_flags & OF_GPIO_ACTIVE_LOW)
-		pdata->battery_detect_present = 0;
-	else
-		pdata->battery_detect_present = 1;
-
-of_out:
-	return pdata;
-}
-#else
-static struct sbs_platform_data *sbs_of_populate_pdata(
-	struct i2c_client *client)
-{
-	return client->dev.platform_data;
-}
-#endif
-
 static const struct power_supply_desc sbs_default_desc = {
 	.type = POWER_SUPPLY_TYPE_BATTERY,
 	.properties = sbs_properties,
@@ -819,13 +767,12 @@ static int sbs_probe(struct i2c_client *client,
 	if (!sbs_desc->name)
 		return -ENOMEM;
 
-	chip = kzalloc(sizeof(struct sbs_info), GFP_KERNEL);
+	chip = devm_kzalloc(&client->dev, sizeof(struct sbs_info), GFP_KERNEL);
 	if (!chip)
 		return -ENOMEM;
 
 	chip->client = client;
 	chip->enable_detection = false;
-	chip->gpio_detect = false;
 	psy_cfg.of_node = client->dev.of_node;
 	psy_cfg.drv_data = chip;
 	/* ignore first notification of external change, it is generated
@@ -834,11 +781,31 @@ static int sbs_probe(struct i2c_client *client,
 	chip->ignore_changes = 1;
 	chip->last_state = POWER_SUPPLY_STATUS_UNKNOWN;
 
-	pdata = sbs_of_populate_pdata(client);
+	/* use pdata if available, fall back to DT properties,
+	 * or hardcoded defaults if not
+	 */
+	rc = of_property_read_u32(client->dev.of_node, "sbs,i2c-retry-count",
+				  &chip->i2c_retry_count);
+	if (rc)
+		chip->i2c_retry_count = 0;
+
+	rc = of_property_read_u32(client->dev.of_node, "sbs,poll-retry-count",
+				  &chip->poll_retry_count);
+	if (rc)
+		chip->poll_retry_count = 0;
 
 	if (pdata) {
-		chip->gpio_detect = gpio_is_valid(pdata->battery_detect);
-		chip->pdata = pdata;
+		chip->poll_retry_count = pdata->poll_retry_count;
+		chip->i2c_retry_count  = pdata->i2c_retry_count;
+	}
+	chip->i2c_retry_count = chip->i2c_retry_count + 1;
+
+	chip->gpio_detect = devm_gpiod_get_optional(&client->dev,
+			"sbs,battery-detect", GPIOD_IN);
+	if (IS_ERR(chip->gpio_detect)) {
+		dev_err(&client->dev, "Failed to get gpio: %ld\n",
+			PTR_ERR(chip->gpio_detect));
+		return PTR_ERR(chip->gpio_detect);
 	}
 
 	i2c_set_clientdata(client, chip);
@@ -846,47 +813,26 @@ static int sbs_probe(struct i2c_client *client,
 	if (!chip->gpio_detect)
 		goto skip_gpio;
 
-	rc = gpio_request(pdata->battery_detect, dev_name(&client->dev));
-	if (rc) {
-		dev_warn(&client->dev, "Failed to request gpio: %d\n", rc);
-		chip->gpio_detect = false;
-		goto skip_gpio;
-	}
-
-	rc = gpio_direction_input(pdata->battery_detect);
-	if (rc) {
-		dev_warn(&client->dev, "Failed to get gpio as input: %d\n", rc);
-		gpio_free(pdata->battery_detect);
-		chip->gpio_detect = false;
-		goto skip_gpio;
-	}
-
-	irq = gpio_to_irq(pdata->battery_detect);
+	irq = gpiod_to_irq(chip->gpio_detect);
 	if (irq <= 0) {
 		dev_warn(&client->dev, "Failed to get gpio as irq: %d\n", irq);
-		gpio_free(pdata->battery_detect);
-		chip->gpio_detect = false;
 		goto skip_gpio;
 	}
 
-	rc = request_irq(irq, sbs_irq,
+	rc = devm_request_threaded_irq(&client->dev, irq, NULL, sbs_irq,
 		IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING,
-		dev_name(&client->dev), chip->power_supply);
+		dev_name(&client->dev), chip);
 	if (rc) {
 		dev_warn(&client->dev, "Failed to request irq: %d\n", rc);
-		gpio_free(pdata->battery_detect);
-		chip->gpio_detect = false;
 		goto skip_gpio;
 	}
 
-	chip->irq = irq;
-
 skip_gpio:
 	/*
 	 * Before we register, we might need to make sure we can actually talk
 	 * to the battery.
 	 */
-	if (!force_load) {
+	if (!(force_load || chip->gpio_detect)) {
 		rc = sbs_read_word_data(client, sbs_data[REG_STATUS].addr);
 
 		if (rc < 0) {
@@ -896,7 +842,7 @@ skip_gpio:
 		}
 	}
 
-	chip->power_supply = power_supply_register(&client->dev, sbs_desc,
+	chip->power_supply = devm_power_supply_register(&client->dev, sbs_desc,
 						   &psy_cfg);
 	if (IS_ERR(chip->power_supply)) {
 		dev_err(&client->dev,
@@ -915,13 +861,6 @@ skip_gpio:
 	return 0;
 
 exit_psupply:
-	if (chip->irq)
-		free_irq(chip->irq, chip->power_supply);
-	if (chip->gpio_detect)
-		gpio_free(pdata->battery_detect);
-
-	kfree(chip);
-
 	return rc;
 }
 
@@ -929,18 +868,8 @@ static int sbs_remove(struct i2c_client *client)
 {
 	struct sbs_info *chip = i2c_get_clientdata(client);
 
-	if (chip->irq)
-		free_irq(chip->irq, chip->power_supply);
-	if (chip->gpio_detect)
-		gpio_free(chip->pdata->battery_detect);
-
-	power_supply_unregister(chip->power_supply);
-
 	cancel_delayed_work_sync(&chip->work);
 
-	kfree(chip);
-	chip = NULL;
-
 	return 0;
 }
 
@@ -950,16 +879,16 @@ static int sbs_suspend(struct device *dev)
 {
 	struct i2c_client *client = to_i2c_client(dev);
 	struct sbs_info *chip = i2c_get_clientdata(client);
-	s32 ret;
 
 	if (chip->poll_time > 0)
 		cancel_delayed_work_sync(&chip->work);
 
-	/* write to manufacturer access with sleep command */
-	ret = sbs_write_word_data(client, sbs_data[REG_MANUFACTURER_DATA].addr,
+	/*
+	 * Write to manufacturer access with sleep command.
+	 * Support is manufacturer dependend, so ignore errors.
+	 */
+	sbs_write_word_data(client, sbs_data[REG_MANUFACTURER_DATA].addr,
 		MANUFACTURER_ACCESS_SLEEP);
-	if (chip->is_present && ret < 0)
-		return ret;
 
 	return 0;
 }
@@ -978,13 +907,20 @@ static const struct i2c_device_id sbs_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, sbs_id);
 
+static const struct of_device_id sbs_dt_ids[] = {
+	{ .compatible = "sbs,sbs-battery" },
+	{ .compatible = "ti,bq20z75" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sbs_dt_ids);
+
 static struct i2c_driver sbs_battery_driver = {
 	.probe		= sbs_probe,
 	.remove		= sbs_remove,
 	.id_table	= sbs_id,
 	.driver = {
 		.name	= "sbs-battery",
-		.of_match_table = of_match_ptr(sbs_dt_ids),
+		.of_match_table = sbs_dt_ids,
 		.pm	= SBS_PM_OPS,
 	},
 };
diff --git a/drivers/power/smb347-charger.c b/drivers/power/supply/smb347-charger.c
index 072c5189bd6d..072c5189bd6d 100644
--- a/drivers/power/smb347-charger.c
+++ b/drivers/power/supply/smb347-charger.c
diff --git a/drivers/power/test_power.c b/drivers/power/supply/test_power.c
index 57246cdbd042..57246cdbd042 100644
--- a/drivers/power/test_power.c
+++ b/drivers/power/supply/test_power.c
diff --git a/drivers/power/tosa_battery.c b/drivers/power/supply/tosa_battery.c
index 6e88c1b37945..6e88c1b37945 100644
--- a/drivers/power/tosa_battery.c
+++ b/drivers/power/supply/tosa_battery.c
diff --git a/drivers/power/tps65090-charger.c b/drivers/power/supply/tps65090-charger.c
index 1b4b5e09538e..1b4b5e09538e 100644
--- a/drivers/power/tps65090-charger.c
+++ b/drivers/power/supply/tps65090-charger.c
diff --git a/drivers/power/tps65217_charger.c b/drivers/power/supply/tps65217_charger.c
index 73dfae41def8..9fd019f9b88c 100644
--- a/drivers/power/tps65217_charger.c
+++ b/drivers/power/supply/tps65217_charger.c
@@ -46,6 +46,8 @@ struct tps65217_charger {
 	int	prev_ac_online;
 
 	struct task_struct	*poll_task;
+
+	int	irq;
 };
 
 static enum power_supply_property tps65217_ac_props[] = {
@@ -198,6 +200,7 @@ static int tps65217_charger_probe(struct platform_device *pdev)
 	struct tps65217 *tps = dev_get_drvdata(pdev->dev.parent);
 	struct tps65217_charger *charger;
 	struct power_supply_config cfg = {};
+	int irq;
 	int ret;
 
 	dev_dbg(&pdev->dev, "%s\n", __func__);
@@ -206,6 +209,7 @@ static int tps65217_charger_probe(struct platform_device *pdev)
 	if (!charger)
 		return -ENOMEM;
 
+	platform_set_drvdata(pdev, charger);
 	charger->tps = tps;
 	charger->dev = &pdev->dev;
 
@@ -220,18 +224,40 @@ static int tps65217_charger_probe(struct platform_device *pdev)
 		return PTR_ERR(charger->ac);
 	}
 
+	irq = platform_get_irq_byname(pdev, "AC");
+	if (irq < 0)
+		irq = -ENXIO;
+	charger->irq = irq;
+
 	ret = tps65217_config_charger(charger);
 	if (ret < 0) {
 		dev_err(charger->dev, "charger config failed, err %d\n", ret);
 		return ret;
 	}
 
-	charger->poll_task = kthread_run(tps65217_charger_poll_task,
-				      charger, "ktps65217charger");
-	if (IS_ERR(charger->poll_task)) {
-		ret = PTR_ERR(charger->poll_task);
-		dev_err(charger->dev, "Unable to run kthread err %d\n", ret);
-		return ret;
+	if (irq != -ENXIO) {
+		ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+						tps65217_charger_irq,
+						0, "tps65217-charger",
+						charger);
+		if (ret) {
+			dev_err(charger->dev,
+				"Unable to register irq %d err %d\n", irq,
+				ret);
+			return ret;
+		}
+
+		/* Check current state */
+		tps65217_charger_irq(irq, charger);
+	} else {
+		charger->poll_task = kthread_run(tps65217_charger_poll_task,
+						charger, "ktps65217charger");
+		if (IS_ERR(charger->poll_task)) {
+			ret = PTR_ERR(charger->poll_task);
+			dev_err(charger->dev,
+				"Unable to run kthread err %d\n", ret);
+			return ret;
+		}
 	}
 
 	return 0;
@@ -241,7 +267,8 @@ static int tps65217_charger_remove(struct platform_device *pdev)
 {
 	struct tps65217_charger *charger = platform_get_drvdata(pdev);
 
-	kthread_stop(charger->poll_task);
+	if (charger->irq == -ENXIO)
+		kthread_stop(charger->poll_task);
 
 	return 0;
 }
diff --git a/drivers/power/twl4030_charger.c b/drivers/power/supply/twl4030_charger.c
index bcd4dc304f27..bcd4dc304f27 100644
--- a/drivers/power/twl4030_charger.c
+++ b/drivers/power/supply/twl4030_charger.c
diff --git a/drivers/power/twl4030_madc_battery.c b/drivers/power/supply/twl4030_madc_battery.c
index f5817e422d64..f5817e422d64 100644
--- a/drivers/power/twl4030_madc_battery.c
+++ b/drivers/power/supply/twl4030_madc_battery.c
diff --git a/drivers/power/wm831x_backup.c b/drivers/power/supply/wm831x_backup.c
index 2e33109ca8c7..2e33109ca8c7 100644
--- a/drivers/power/wm831x_backup.c
+++ b/drivers/power/supply/wm831x_backup.c
diff --git a/drivers/power/wm831x_power.c b/drivers/power/supply/wm831x_power.c
index 7082301da945..7082301da945 100644
--- a/drivers/power/wm831x_power.c
+++ b/drivers/power/supply/wm831x_power.c
diff --git a/drivers/power/wm8350_power.c b/drivers/power/supply/wm8350_power.c
index 5c5880664e09..5c5880664e09 100644
--- a/drivers/power/wm8350_power.c
+++ b/drivers/power/supply/wm8350_power.c
diff --git a/drivers/power/wm97xx_battery.c b/drivers/power/supply/wm97xx_battery.c
index c2f09ed35050..6285626d142a 100644
--- a/drivers/power/wm97xx_battery.c
+++ b/drivers/power/supply/wm97xx_battery.c
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/power/wm97xx_battery.c
- *
  * Battery measurement code for WM97xx
  *
  * based on tosa_battery.c
diff --git a/drivers/power/z2_battery.c b/drivers/power/supply/z2_battery.c
index b201e3facf73..8a43b49cfd35 100644
--- a/drivers/power/z2_battery.c
+++ b/drivers/power/supply/z2_battery.c
@@ -317,7 +317,6 @@ MODULE_DEVICE_TABLE(i2c, z2_batt_id);
 static struct i2c_driver z2_batt_driver = {
 	.driver	= {
 		.name	= "z2-battery",
-		.owner	= THIS_MODULE,
 		.pm	= Z2_BATTERY_PM_OPS
 	},
 	.probe		= z2_batt_probe,
diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c
index fbab29dfa793..243b233ff31b 100644
--- a/drivers/powercap/intel_rapl.c
+++ b/drivers/powercap/intel_rapl.c
@@ -1154,8 +1154,8 @@ static const struct x86_cpu_id rapl_ids[] __initconst = {
 
 	RAPL_CPU(INTEL_FAM6_ATOM_SILVERMONT1,	rapl_defaults_byt),
 	RAPL_CPU(INTEL_FAM6_ATOM_AIRMONT,	rapl_defaults_cht),
-	RAPL_CPU(INTEL_FAM6_ATOM_MERRIFIELD1,	rapl_defaults_tng),
-	RAPL_CPU(INTEL_FAM6_ATOM_MERRIFIELD2,	rapl_defaults_ann),
+	RAPL_CPU(INTEL_FAM6_ATOM_MERRIFIELD,	rapl_defaults_tng),
+	RAPL_CPU(INTEL_FAM6_ATOM_MOOREFIELD,	rapl_defaults_ann),
 	RAPL_CPU(INTEL_FAM6_ATOM_GOLDMONT,	rapl_defaults_core),
 	RAPL_CPU(INTEL_FAM6_ATOM_DENVERTON,	rapl_defaults_core),
 
diff --git a/drivers/pps/Kconfig b/drivers/pps/Kconfig
index 7512e98e9311..564a51abeece 100644
--- a/drivers/pps/Kconfig
+++ b/drivers/pps/Kconfig
@@ -31,7 +31,7 @@ config PPS_DEBUG
 
 config NTP_PPS
 	bool "PPS kernel consumer support"
-	depends on !NO_HZ
+	depends on !NO_HZ_COMMON
 	help
 	  This option adds support for direct in-kernel time
 	  synchronization using an external PPS signal.
diff --git a/drivers/ps3/ps3-vuart.c b/drivers/ps3/ps3-vuart.c
index 632701a1d993..b7f300b79ffd 100644
--- a/drivers/ps3/ps3-vuart.c
+++ b/drivers/ps3/ps3-vuart.c
@@ -958,7 +958,7 @@ static int ps3_vuart_bus_interrupt_get(void)
 
 fail_request_irq:
 	ps3_vuart_irq_destroy(vuart_bus_priv.virq);
-	vuart_bus_priv.virq = NO_IRQ;
+	vuart_bus_priv.virq = 0;
 fail_alloc_irq:
 	kfree(vuart_bus_priv.bmp);
 	vuart_bus_priv.bmp = NULL;
@@ -982,7 +982,7 @@ static int ps3_vuart_bus_interrupt_put(void)
 	free_irq(vuart_bus_priv.virq, &vuart_bus_priv);
 
 	ps3_vuart_irq_destroy(vuart_bus_priv.virq);
-	vuart_bus_priv.virq = NO_IRQ;
+	vuart_bus_priv.virq = 0;
 
 	kfree(vuart_bus_priv.bmp);
 	vuart_bus_priv.bmp = NULL;
diff --git a/drivers/ptp/ptp_clock.c b/drivers/ptp/ptp_clock.c
index 2e481b9e8ea5..86280b7e41f3 100644
--- a/drivers/ptp/ptp_clock.c
+++ b/drivers/ptp/ptp_clock.c
@@ -263,6 +263,7 @@ no_sysfs:
 no_device:
 	mutex_destroy(&ptp->tsevq_mux);
 	mutex_destroy(&ptp->pincfg_mux);
+	ida_simple_remove(&ptp_clocks_map, index);
 no_slot:
 	kfree(ptp);
 no_memory:
diff --git a/drivers/ptp/ptp_ixp46x.c b/drivers/ptp/ptp_ixp46x.c
index ee4f183ef9ee..344a3bac210b 100644
--- a/drivers/ptp/ptp_ixp46x.c
+++ b/drivers/ptp/ptp_ixp46x.c
@@ -268,18 +268,19 @@ static int setup_interrupt(int gpio)
 		return err;
 
 	irq = gpio_to_irq(gpio);
+	if (irq < 0)
+		return irq;
 
-	if (NO_IRQ == irq)
-		return NO_IRQ;
-
-	if (irq_set_irq_type(irq, IRQF_TRIGGER_FALLING)) {
+	err = irq_set_irq_type(irq, IRQF_TRIGGER_FALLING);
+	if (err) {
 		pr_err("cannot set trigger type for irq %d\n", irq);
-		return NO_IRQ;
+		return err;
 	}
 
-	if (request_irq(irq, isr, 0, DRIVER, &ixp_clock)) {
+	err = request_irq(irq, isr, 0, DRIVER, &ixp_clock);
+	if (err) {
 		pr_err("request_irq failed for irq %d\n", irq);
-		return NO_IRQ;
+		return err;
 	}
 
 	return irq;
diff --git a/drivers/rapidio/devices/tsi721.c b/drivers/rapidio/devices/tsi721.c
index 32f0f014a067..9d19b9a62011 100644
--- a/drivers/rapidio/devices/tsi721.c
+++ b/drivers/rapidio/devices/tsi721.c
@@ -1161,7 +1161,7 @@ static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart,
 		} else if (ibw_start < (ib_win->rstart + ib_win->size) &&
 			   (ibw_start + ibw_size) > ib_win->rstart) {
 			/* Return error if address translation involved */
-			if (direct && ib_win->xlat) {
+			if (!direct || ib_win->xlat) {
 				ret = -EFAULT;
 				break;
 			}
diff --git a/drivers/rapidio/rio_cm.c b/drivers/rapidio/rio_cm.c
index cecc15a880de..bad0e0ea4f30 100644
--- a/drivers/rapidio/rio_cm.c
+++ b/drivers/rapidio/rio_cm.c
@@ -1080,8 +1080,8 @@ static int riocm_send_ack(struct rio_channel *ch)
 static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
 					   long timeout)
 {
-	struct rio_channel *ch = NULL;
-	struct rio_channel *new_ch = NULL;
+	struct rio_channel *ch;
+	struct rio_channel *new_ch;
 	struct conn_req *req;
 	struct cm_peer *peer;
 	int found = 0;
@@ -1155,6 +1155,7 @@ static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
 
 	spin_unlock_bh(&ch->lock);
 	riocm_put_channel(ch);
+	ch = NULL;
 	kfree(req);
 
 	down_read(&rdev_sem);
@@ -1172,7 +1173,7 @@ static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
 	if (!found) {
 		/* If peer device object not found, simply ignore the request */
 		err = -ENODEV;
-		goto err_nodev;
+		goto err_put_new_ch;
 	}
 
 	new_ch->rdev = peer->rdev;
@@ -1184,15 +1185,16 @@ static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
 
 	*new_ch_id = new_ch->id;
 	return new_ch;
+
+err_put_new_ch:
+	spin_lock_bh(&idr_lock);
+	idr_remove(&ch_idr, new_ch->id);
+	spin_unlock_bh(&idr_lock);
+	riocm_put_channel(new_ch);
+
 err_put:
-	riocm_put_channel(ch);
-err_nodev:
-	if (new_ch) {
-		spin_lock_bh(&idr_lock);
-		idr_remove(&ch_idr, new_ch->id);
-		spin_unlock_bh(&idr_lock);
-		riocm_put_channel(new_ch);
-	}
+	if (ch)
+		riocm_put_channel(ch);
 	*new_ch_id = 0;
 	return ERR_PTR(err);
 }
@@ -1839,24 +1841,19 @@ static int cm_chan_msg_send(void __user *arg)
 {
 	struct rio_cm_msg msg;
 	void *buf;
-	int ret = 0;
+	int ret;
 
 	if (copy_from_user(&msg, arg, sizeof(msg)))
 		return -EFAULT;
 	if (msg.size > RIO_MAX_MSG_SIZE)
 		return -EINVAL;
 
-	buf = kmalloc(msg.size, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	if (copy_from_user(buf, (void __user *)(uintptr_t)msg.msg, msg.size)) {
-		ret = -EFAULT;
-		goto out;
-	}
+	buf = memdup_user((void __user *)(uintptr_t)msg.msg, msg.size);
+	if (IS_ERR(buf))
+		return PTR_ERR(buf);
 
 	ret = riocm_ch_send(msg.ch_num, buf, msg.size);
-out:
+
 	kfree(buf);
 	return ret;
 }
@@ -2245,17 +2242,30 @@ static int rio_cm_shutdown(struct notifier_block *nb, unsigned long code,
 {
 	struct rio_channel *ch;
 	unsigned int i;
+	LIST_HEAD(list);
 
 	riocm_debug(EXIT, ".");
 
+	/*
+	 * If there are any channels left in connected state send
+	 * close notification to the connection partner.
+	 * First build a list of channels that require a closing
+	 * notification because function riocm_send_close() should
+	 * be called outside of spinlock protected code.
+	 */
 	spin_lock_bh(&idr_lock);
 	idr_for_each_entry(&ch_idr, ch, i) {
-		riocm_debug(EXIT, "close ch %d", ch->id);
-		if (ch->state == RIO_CM_CONNECTED)
-			riocm_send_close(ch);
+		if (ch->state == RIO_CM_CONNECTED) {
+			riocm_debug(EXIT, "close ch %d", ch->id);
+			idr_remove(&ch_idr, ch->id);
+			list_add(&ch->ch_node, &list);
+		}
 	}
 	spin_unlock_bh(&idr_lock);
 
+	list_for_each_entry(ch, &list, ch_node)
+		riocm_send_close(ch);
+
 	return NOTIFY_DONE;
 }
 
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index 6c88e31c01f7..936f7ccc9736 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -323,7 +323,7 @@ config REGULATOR_LP872X
 
 config REGULATOR_LP873X
 	tristate "TI LP873X Power regulators"
-	depends on MFD_LP873X && OF
+	depends on MFD_TI_LP873X && OF
 	help
 	  This driver supports LP873X voltage regulator chips. LP873X
 	  provides two step-down converters and two general-purpose LDO
@@ -353,6 +353,14 @@ config REGULATOR_LTC3589
 	  This enables support for the LTC3589, LTC3589-1, and LTC3589-2
 	  8-output regulators controlled via I2C.
 
+config REGULATOR_LTC3676
+	tristate "LTC3676 8-output voltage regulator"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  This enables support for the LTC3676
+	  8-output regulators controlled via I2C.
+
 config REGULATOR_MAX14577
 	tristate "Maxim 14577/77836 regulator"
 	depends on MFD_MAX14577
@@ -635,11 +643,11 @@ config REGULATOR_RC5T583
 	  outputs which can be controlled by i2c communication.
 
 config REGULATOR_RK808
-	tristate "Rockchip RK808 Power regulators"
+	tristate "Rockchip RK808/RK818 Power regulators"
 	depends on MFD_RK808
 	help
 	  Select this option to enable the power regulator of ROCKCHIP
-	  PMIC RK808.
+	  PMIC RK808 and RK818.
 	  This driver supports the control of different power rails of device
 	  through regulator interface. The device supports multiple DCDC/LDO
 	  outputs which can be controlled by i2c communication.
@@ -820,7 +828,7 @@ config REGULATOR_TPS65912
 	    This driver supports TPS65912 voltage regulator chip.
 
 config REGULATOR_TPS80031
-	tristate "TI TPS80031/TPS80032 power regualtor driver"
+	tristate "TI TPS80031/TPS80032 power regulator driver"
 	depends on MFD_TPS80031
 	help
 	  TPS80031/ TPS80032 Fully Integrated Power Management with Power
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index f3da9eea9ab6..2142a5d3fc08 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -47,6 +47,7 @@ obj-$(CONFIG_REGULATOR_LP8788) += lp8788-buck.o
 obj-$(CONFIG_REGULATOR_LP8788) += lp8788-ldo.o
 obj-$(CONFIG_REGULATOR_LP8755) += lp8755.o
 obj-$(CONFIG_REGULATOR_LTC3589) += ltc3589.o
+obj-$(CONFIG_REGULATOR_LTC3676) += ltc3676.o
 obj-$(CONFIG_REGULATOR_MAX14577) += max14577-regulator.o
 obj-$(CONFIG_REGULATOR_MAX1586) += max1586.o
 obj-$(CONFIG_REGULATOR_MAX77620) += max77620-regulator.o
diff --git a/drivers/regulator/axp20x-regulator.c b/drivers/regulator/axp20x-regulator.c
index 6d9ac76a772f..54382ef902c6 100644
--- a/drivers/regulator/axp20x-regulator.c
+++ b/drivers/regulator/axp20x-regulator.c
@@ -244,16 +244,64 @@ static const struct regulator_desc axp22x_drivevbus_regulator = {
 	.ops		= &axp20x_ops_sw,
 };
 
-static const struct regulator_linear_range axp809_dcdc4_ranges[] = {
-	REGULATOR_LINEAR_RANGE(600000, 0x0, 0x2f, 20000),
-	REGULATOR_LINEAR_RANGE(1800000, 0x30, 0x38, 100000),
+static const struct regulator_linear_range axp806_dcdca_ranges[] = {
+	REGULATOR_LINEAR_RANGE(600000, 0x0, 0x32, 10000),
+	REGULATOR_LINEAR_RANGE(1120000, 0x33, 0x47, 20000),
 };
 
-static const struct regulator_linear_range axp809_dldo1_ranges[] = {
+static const struct regulator_linear_range axp806_dcdcd_ranges[] = {
+	REGULATOR_LINEAR_RANGE(600000, 0x0, 0x2d, 20000),
+	REGULATOR_LINEAR_RANGE(1600000, 0x2e, 0x3f, 100000),
+};
+
+static const struct regulator_linear_range axp806_cldo2_ranges[] = {
 	REGULATOR_LINEAR_RANGE(700000, 0x0, 0x1a, 100000),
 	REGULATOR_LINEAR_RANGE(3400000, 0x1b, 0x1f, 200000),
 };
 
+static const struct regulator_desc axp806_regulators[] = {
+	AXP_DESC_RANGES(AXP806, DCDCA, "dcdca", "vina", axp806_dcdca_ranges,
+			72, AXP806_DCDCA_V_CTRL, 0x7f, AXP806_PWR_OUT_CTRL1,
+			BIT(0)),
+	AXP_DESC(AXP806, DCDCB, "dcdcb", "vinb", 1000, 2550, 50,
+		 AXP806_DCDCB_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL1, BIT(1)),
+	AXP_DESC_RANGES(AXP806, DCDCC, "dcdcc", "vinc", axp806_dcdca_ranges,
+			72, AXP806_DCDCC_V_CTRL, 0x7f, AXP806_PWR_OUT_CTRL1,
+			BIT(2)),
+	AXP_DESC_RANGES(AXP806, DCDCD, "dcdcd", "vind", axp806_dcdcd_ranges,
+			64, AXP806_DCDCD_V_CTRL, 0x3f, AXP806_PWR_OUT_CTRL1,
+			BIT(3)),
+	AXP_DESC(AXP806, DCDCE, "dcdce", "vine", 1100, 3400, 100,
+		 AXP806_DCDCB_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL1, BIT(4)),
+	AXP_DESC(AXP806, ALDO1, "aldo1", "aldoin", 700, 3300, 100,
+		 AXP806_ALDO1_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL1, BIT(5)),
+	AXP_DESC(AXP806, ALDO2, "aldo2", "aldoin", 700, 3400, 100,
+		 AXP806_ALDO2_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL1, BIT(6)),
+	AXP_DESC(AXP806, ALDO3, "aldo3", "aldoin", 700, 3300, 100,
+		 AXP806_ALDO3_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL1, BIT(7)),
+	AXP_DESC(AXP806, BLDO1, "bldo1", "bldoin", 700, 1900, 100,
+		 AXP806_BLDO1_V_CTRL, 0x0f, AXP806_PWR_OUT_CTRL2, BIT(0)),
+	AXP_DESC(AXP806, BLDO2, "bldo2", "bldoin", 700, 1900, 100,
+		 AXP806_BLDO2_V_CTRL, 0x0f, AXP806_PWR_OUT_CTRL2, BIT(1)),
+	AXP_DESC(AXP806, BLDO3, "bldo3", "bldoin", 700, 1900, 100,
+		 AXP806_BLDO3_V_CTRL, 0x0f, AXP806_PWR_OUT_CTRL2, BIT(2)),
+	AXP_DESC(AXP806, BLDO4, "bldo4", "bldoin", 700, 1900, 100,
+		 AXP806_BLDO4_V_CTRL, 0x0f, AXP806_PWR_OUT_CTRL2, BIT(3)),
+	AXP_DESC(AXP806, CLDO1, "cldo1", "cldoin", 700, 3300, 100,
+		 AXP806_CLDO1_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL2, BIT(4)),
+	AXP_DESC_RANGES(AXP806, CLDO2, "cldo2", "cldoin", axp806_cldo2_ranges,
+			32, AXP806_CLDO2_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL2,
+			BIT(5)),
+	AXP_DESC(AXP806, CLDO3, "cldo3", "cldoin", 700, 3300, 100,
+		 AXP806_CLDO3_V_CTRL, 0x1f, AXP806_PWR_OUT_CTRL2, BIT(6)),
+	AXP_DESC_SW(AXP806, SW, "sw", "swin", AXP806_PWR_OUT_CTRL2, BIT(7)),
+};
+
+static const struct regulator_linear_range axp809_dcdc4_ranges[] = {
+	REGULATOR_LINEAR_RANGE(600000, 0x0, 0x2f, 20000),
+	REGULATOR_LINEAR_RANGE(1800000, 0x30, 0x38, 100000),
+};
+
 static const struct regulator_desc axp809_regulators[] = {
 	AXP_DESC(AXP809, DCDC1, "dcdc1", "vin1", 1600, 3400, 100,
 		 AXP22X_DCDC1_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL1, BIT(1)),
@@ -278,7 +326,7 @@ static const struct regulator_desc axp809_regulators[] = {
 		 AXP22X_ALDO2_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL1, BIT(7)),
 	AXP_DESC(AXP809, ALDO3, "aldo3", "aldoin", 700, 3300, 100,
 		 AXP22X_ALDO3_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL2, BIT(5)),
-	AXP_DESC_RANGES(AXP809, DLDO1, "dldo1", "dldoin", axp809_dldo1_ranges,
+	AXP_DESC_RANGES(AXP809, DLDO1, "dldo1", "dldoin", axp806_cldo2_ranges,
 			32, AXP22X_DLDO1_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL2,
 			BIT(3)),
 	AXP_DESC(AXP809, DLDO2, "dldo2", "dldoin", 700, 3300, 100,
@@ -302,6 +350,7 @@ static const struct regulator_desc axp809_regulators[] = {
 static int axp20x_set_dcdc_freq(struct platform_device *pdev, u32 dcdcfreq)
 {
 	struct axp20x_dev *axp20x = dev_get_drvdata(pdev->dev.parent);
+	unsigned int reg = AXP20X_DCDC_FREQ;
 	u32 min, max, def, step;
 
 	switch (axp20x->variant) {
@@ -312,6 +361,14 @@ static int axp20x_set_dcdc_freq(struct platform_device *pdev, u32 dcdcfreq)
 		def = 1500;
 		step = 75;
 		break;
+	case AXP806_ID:
+		/*
+		 * AXP806 DCDC work frequency setting has the same range and
+		 * step as AXP22X, but at a different register.
+		 * Fall through to the check below.
+		 * (See include/linux/mfd/axp20x.h)
+		 */
+		reg = AXP806_DCDC_FREQ_CTRL;
 	case AXP221_ID:
 	case AXP223_ID:
 	case AXP809_ID:
@@ -343,7 +400,7 @@ static int axp20x_set_dcdc_freq(struct platform_device *pdev, u32 dcdcfreq)
 
 	dcdcfreq = (dcdcfreq - min) / step;
 
-	return regmap_update_bits(axp20x->regmap, AXP20X_DCDC_FREQ,
+	return regmap_update_bits(axp20x->regmap, reg,
 				  AXP20X_FREQ_DCDC_MASK, dcdcfreq);
 }
 
@@ -377,6 +434,7 @@ static int axp20x_regulator_parse_dt(struct platform_device *pdev)
 static int axp20x_set_dcdc_workmode(struct regulator_dev *rdev, int id, u32 workmode)
 {
 	struct axp20x_dev *axp20x = rdev_get_drvdata(rdev);
+	unsigned int reg = AXP20X_DCDC_MODE;
 	unsigned int mask;
 
 	switch (axp20x->variant) {
@@ -392,6 +450,13 @@ static int axp20x_set_dcdc_workmode(struct regulator_dev *rdev, int id, u32 work
 		workmode <<= ffs(mask) - 1;
 		break;
 
+	case AXP806_ID:
+		reg = AXP806_DCDC_MODE_CTRL2;
+		/*
+		 * AXP806 DCDC regulator IDs have the same range as AXP22X.
+		 * Fall through to the check below.
+		 * (See include/linux/mfd/axp20x.h)
+		 */
 	case AXP221_ID:
 	case AXP223_ID:
 	case AXP809_ID:
@@ -408,7 +473,34 @@ static int axp20x_set_dcdc_workmode(struct regulator_dev *rdev, int id, u32 work
 		return -EINVAL;
 	}
 
-	return regmap_update_bits(rdev->regmap, AXP20X_DCDC_MODE, mask, workmode);
+	return regmap_update_bits(rdev->regmap, reg, mask, workmode);
+}
+
+/*
+ * This function checks whether a regulator is part of a poly-phase
+ * output setup based on the registers settings. Returns true if it is.
+ */
+static bool axp20x_is_polyphase_slave(struct axp20x_dev *axp20x, int id)
+{
+	u32 reg = 0;
+
+	/* Only AXP806 has poly-phase outputs */
+	if (axp20x->variant != AXP806_ID)
+		return false;
+
+	regmap_read(axp20x->regmap, AXP806_DCDC_MODE_CTRL2, &reg);
+
+	switch (id) {
+	case AXP806_DCDCB:
+		return (((reg & GENMASK(7, 6)) == BIT(6)) ||
+			((reg & GENMASK(7, 6)) == BIT(7)));
+	case AXP806_DCDCC:
+		return ((reg & GENMASK(7, 6)) == BIT(7));
+	case AXP806_DCDCE:
+		return !!(reg & BIT(5));
+	}
+
+	return false;
 }
 
 static int axp20x_regulator_probe(struct platform_device *pdev)
@@ -440,6 +532,10 @@ static int axp20x_regulator_probe(struct platform_device *pdev)
 		drivevbus = of_property_read_bool(pdev->dev.parent->of_node,
 						  "x-powers,drive-vbus-en");
 		break;
+	case AXP806_ID:
+		regulators = axp806_regulators;
+		nregulators = AXP806_REG_ID_MAX;
+		break;
 	case AXP809_ID:
 		regulators = axp809_regulators;
 		nregulators = AXP809_REG_ID_MAX;
@@ -458,6 +554,14 @@ static int axp20x_regulator_probe(struct platform_device *pdev)
 		struct regulator_desc *new_desc;
 
 		/*
+		 * If this regulator is a slave in a poly-phase setup,
+		 * skip it, as its controls are bound to the master
+		 * regulator and won't work.
+		 */
+		if (axp20x_is_polyphase_slave(axp20x, i))
+			continue;
+
+		/*
 		 * Regulators DC1SW and DC5LDO are connected internally,
 		 * so we have to handle their supply names separately.
 		 *
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index db320e8fa865..67426c0477d3 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -679,24 +679,6 @@ static int drms_uA_update(struct regulator_dev *rdev)
 	    !rdev->desc->ops->set_load)
 		return -EINVAL;
 
-	/* get output voltage */
-	output_uV = _regulator_get_voltage(rdev);
-	if (output_uV <= 0) {
-		rdev_err(rdev, "invalid output voltage found\n");
-		return -EINVAL;
-	}
-
-	/* get input voltage */
-	input_uV = 0;
-	if (rdev->supply)
-		input_uV = regulator_get_voltage(rdev->supply);
-	if (input_uV <= 0)
-		input_uV = rdev->constraints->input_uV;
-	if (input_uV <= 0) {
-		rdev_err(rdev, "invalid input voltage found\n");
-		return -EINVAL;
-	}
-
 	/* calc total requested load */
 	list_for_each_entry(sibling, &rdev->consumer_list, list)
 		current_uA += sibling->uA_load;
@@ -709,6 +691,24 @@ static int drms_uA_update(struct regulator_dev *rdev)
 		if (err < 0)
 			rdev_err(rdev, "failed to set load %d\n", current_uA);
 	} else {
+		/* get output voltage */
+		output_uV = _regulator_get_voltage(rdev);
+		if (output_uV <= 0) {
+			rdev_err(rdev, "invalid output voltage found\n");
+			return -EINVAL;
+		}
+
+		/* get input voltage */
+		input_uV = 0;
+		if (rdev->supply)
+			input_uV = regulator_get_voltage(rdev->supply);
+		if (input_uV <= 0)
+			input_uV = rdev->constraints->input_uV;
+		if (input_uV <= 0) {
+			rdev_err(rdev, "invalid input voltage found\n");
+			return -EINVAL;
+		}
+
 		/* now get the optimum mode for our new total regulator load */
 		mode = rdev->desc->ops->get_optimum_mode(rdev, input_uV,
 							 output_uV, current_uA);
@@ -2743,6 +2743,24 @@ static int _regulator_call_set_voltage_sel(struct regulator_dev *rdev,
 	return ret;
 }
 
+static int _regulator_set_voltage_time(struct regulator_dev *rdev,
+				       int old_uV, int new_uV)
+{
+	unsigned int ramp_delay = 0;
+
+	if (rdev->constraints->ramp_delay)
+		ramp_delay = rdev->constraints->ramp_delay;
+	else if (rdev->desc->ramp_delay)
+		ramp_delay = rdev->desc->ramp_delay;
+
+	if (ramp_delay == 0) {
+		rdev_warn(rdev, "ramp_delay not set\n");
+		return 0;
+	}
+
+	return DIV_ROUND_UP(abs(new_uV - old_uV), ramp_delay);
+}
+
 static int _regulator_do_set_voltage(struct regulator_dev *rdev,
 				     int min_uV, int max_uV)
 {
@@ -2751,6 +2769,8 @@ static int _regulator_do_set_voltage(struct regulator_dev *rdev,
 	int best_val = 0;
 	unsigned int selector;
 	int old_selector = -1;
+	const struct regulator_ops *ops = rdev->desc->ops;
+	int old_uV = _regulator_get_voltage(rdev);
 
 	trace_regulator_set_voltage(rdev_get_name(rdev), min_uV, max_uV);
 
@@ -2762,29 +2782,28 @@ static int _regulator_do_set_voltage(struct regulator_dev *rdev,
 	 * info to call set_voltage_time_sel().
 	 */
 	if (_regulator_is_enabled(rdev) &&
-	    rdev->desc->ops->set_voltage_time_sel &&
-	    rdev->desc->ops->get_voltage_sel) {
-		old_selector = rdev->desc->ops->get_voltage_sel(rdev);
+	    ops->set_voltage_time_sel && ops->get_voltage_sel) {
+		old_selector = ops->get_voltage_sel(rdev);
 		if (old_selector < 0)
 			return old_selector;
 	}
 
-	if (rdev->desc->ops->set_voltage) {
+	if (ops->set_voltage) {
 		ret = _regulator_call_set_voltage(rdev, min_uV, max_uV,
 						  &selector);
 
 		if (ret >= 0) {
-			if (rdev->desc->ops->list_voltage)
-				best_val = rdev->desc->ops->list_voltage(rdev,
-									 selector);
+			if (ops->list_voltage)
+				best_val = ops->list_voltage(rdev,
+							     selector);
 			else
 				best_val = _regulator_get_voltage(rdev);
 		}
 
-	} else if (rdev->desc->ops->set_voltage_sel) {
+	} else if (ops->set_voltage_sel) {
 		ret = regulator_map_voltage(rdev, min_uV, max_uV);
 		if (ret >= 0) {
-			best_val = rdev->desc->ops->list_voltage(rdev, ret);
+			best_val = ops->list_voltage(rdev, ret);
 			if (min_uV <= best_val && max_uV >= best_val) {
 				selector = ret;
 				if (old_selector == selector)
@@ -2800,34 +2819,50 @@ static int _regulator_do_set_voltage(struct regulator_dev *rdev,
 		ret = -EINVAL;
 	}
 
-	/* Call set_voltage_time_sel if successfully obtained old_selector */
-	if (ret == 0 && !rdev->constraints->ramp_disable && old_selector >= 0
-		&& old_selector != selector) {
+	if (ret)
+		goto out;
 
-		delay = rdev->desc->ops->set_voltage_time_sel(rdev,
-						old_selector, selector);
-		if (delay < 0) {
-			rdev_warn(rdev, "set_voltage_time_sel() failed: %d\n",
-				  delay);
-			delay = 0;
+	if (ops->set_voltage_time_sel) {
+		/*
+		 * Call set_voltage_time_sel if successfully obtained
+		 * old_selector
+		 */
+		if (old_selector >= 0 && old_selector != selector)
+			delay = ops->set_voltage_time_sel(rdev, old_selector,
+							  selector);
+	} else {
+		if (old_uV != best_val) {
+			if (ops->set_voltage_time)
+				delay = ops->set_voltage_time(rdev, old_uV,
+							      best_val);
+			else
+				delay = _regulator_set_voltage_time(rdev,
+								    old_uV,
+								    best_val);
 		}
+	}
 
-		/* Insert any necessary delays */
-		if (delay >= 1000) {
-			mdelay(delay / 1000);
-			udelay(delay % 1000);
-		} else if (delay) {
-			udelay(delay);
-		}
+	if (delay < 0) {
+		rdev_warn(rdev, "failed to get delay: %d\n", delay);
+		delay = 0;
 	}
 
-	if (ret == 0 && best_val >= 0) {
+	/* Insert any necessary delays */
+	if (delay >= 1000) {
+		mdelay(delay / 1000);
+		udelay(delay % 1000);
+	} else if (delay) {
+		udelay(delay);
+	}
+
+	if (best_val >= 0) {
 		unsigned long data = best_val;
 
 		_notifier_call_chain(rdev, REGULATOR_EVENT_VOLTAGE_CHANGE,
 				     (void *)data);
 	}
 
+out:
 	trace_regulator_set_voltage_complete(rdev_get_name(rdev), best_val);
 
 	return ret;
@@ -2998,9 +3033,13 @@ int regulator_set_voltage_time(struct regulator *regulator,
 	int voltage;
 	int i;
 
+	if (ops->set_voltage_time)
+		return ops->set_voltage_time(rdev, old_uV, new_uV);
+	else if (!ops->set_voltage_time_sel)
+		return _regulator_set_voltage_time(rdev, old_uV, new_uV);
+
 	/* Currently requires operations to do this */
-	if (!ops->list_voltage || !ops->set_voltage_time_sel
-	    || !rdev->desc->n_voltages)
+	if (!ops->list_voltage || !rdev->desc->n_voltages)
 		return -EINVAL;
 
 	for (i = 0; i < rdev->desc->n_voltages; i++) {
@@ -3039,19 +3078,8 @@ int regulator_set_voltage_time_sel(struct regulator_dev *rdev,
 				   unsigned int old_selector,
 				   unsigned int new_selector)
 {
-	unsigned int ramp_delay = 0;
 	int old_volt, new_volt;
 
-	if (rdev->constraints->ramp_delay)
-		ramp_delay = rdev->constraints->ramp_delay;
-	else if (rdev->desc->ramp_delay)
-		ramp_delay = rdev->desc->ramp_delay;
-
-	if (ramp_delay == 0) {
-		rdev_warn(rdev, "ramp_delay not set\n");
-		return 0;
-	}
-
 	/* sanity check */
 	if (!rdev->desc->ops->list_voltage)
 		return -EINVAL;
@@ -3059,7 +3087,11 @@ int regulator_set_voltage_time_sel(struct regulator_dev *rdev,
 	old_volt = rdev->desc->ops->list_voltage(rdev, old_selector);
 	new_volt = rdev->desc->ops->list_voltage(rdev, new_selector);
 
-	return DIV_ROUND_UP(abs(new_volt - old_volt), ramp_delay);
+	if (rdev->desc->ops->set_voltage_time)
+		return rdev->desc->ops->set_voltage_time(rdev, old_volt,
+							 new_volt);
+	else
+		return _regulator_set_voltage_time(rdev, old_volt, new_volt);
 }
 EXPORT_SYMBOL_GPL(regulator_set_voltage_time_sel);
 
@@ -3483,10 +3515,8 @@ int regulator_bulk_get(struct device *dev, int num_consumers,
 		consumers[i].consumer = NULL;
 
 	for (i = 0; i < num_consumers; i++) {
-		consumers[i].consumer = _regulator_get(dev,
-						       consumers[i].supply,
-						       false,
-						       !consumers[i].optional);
+		consumers[i].consumer = regulator_get(dev,
+						      consumers[i].supply);
 		if (IS_ERR(consumers[i].consumer)) {
 			ret = PTR_ERR(consumers[i].consumer);
 			dev_err(dev, "Failed to get supply '%s': %d\n",
diff --git a/drivers/regulator/dbx500-prcmu.c b/drivers/regulator/dbx500-prcmu.c
index 3963dfad766c..8976141c1438 100644
--- a/drivers/regulator/dbx500-prcmu.c
+++ b/drivers/regulator/dbx500-prcmu.c
@@ -75,24 +75,6 @@ static struct ux500_regulator_debug {
 	u8 *state_after_suspend;
 } rdebug;
 
-void ux500_regulator_suspend_debug(void)
-{
-	int i;
-
-	for (i = 0; i < rdebug.num_regulators; i++)
-		rdebug.state_before_suspend[i] =
-			rdebug.regulator_array[i].is_enabled;
-}
-
-void ux500_regulator_resume_debug(void)
-{
-	int i;
-
-	for (i = 0; i < rdebug.num_regulators; i++)
-		rdebug.state_after_suspend[i] =
-			rdebug.regulator_array[i].is_enabled;
-}
-
 static int ux500_regulator_power_state_cnt_print(struct seq_file *s, void *p)
 {
 	/* print power state count */
diff --git a/drivers/regulator/devres.c b/drivers/regulator/devres.c
index 6ad8ab4c578d..6ec1d400adae 100644
--- a/drivers/regulator/devres.c
+++ b/drivers/regulator/devres.c
@@ -164,11 +164,8 @@ int devm_regulator_bulk_get(struct device *dev, int num_consumers,
 		consumers[i].consumer = NULL;
 
 	for (i = 0; i < num_consumers; i++) {
-		consumers[i].consumer = _devm_regulator_get(dev,
-							    consumers[i].supply,
-							    consumers[i].optional ?
-								OPTIONAL_GET :
-								NORMAL_GET);
+		consumers[i].consumer = devm_regulator_get(dev,
+							   consumers[i].supply);
 		if (IS_ERR(consumers[i].consumer)) {
 			ret = PTR_ERR(consumers[i].consumer);
 			dev_err(dev, "Failed to get supply '%s': %d\n",
diff --git a/drivers/regulator/hi6421-regulator.c b/drivers/regulator/hi6421-regulator.c
index 42dc5fb8c899..62c5f5445d44 100644
--- a/drivers/regulator/hi6421-regulator.c
+++ b/drivers/regulator/hi6421-regulator.c
@@ -477,7 +477,8 @@ static int hi6421_regulator_buck_set_mode(struct regulator_dev *rdev,
 	return 0;
 }
 
-unsigned int hi6421_regulator_ldo_get_optimum_mode(struct regulator_dev *rdev,
+static unsigned int
+hi6421_regulator_ldo_get_optimum_mode(struct regulator_dev *rdev,
 			int input_uV, int output_uV, int load_uA)
 {
 	struct hi6421_regulator_info *info = rdev_get_drvdata(rdev);
diff --git a/drivers/regulator/ltc3676.c b/drivers/regulator/ltc3676.c
new file mode 100644
index 000000000000..e2b476ca2b4d
--- /dev/null
+++ b/drivers/regulator/ltc3676.c
@@ -0,0 +1,420 @@
+/*
+ * Copyright (C) 2016 Gateworks Corporation, Inc. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/regulator/of_regulator.h>
+
+#define DRIVER_NAME		"ltc3676"
+
+/* LTC3676 Registers */
+#define LTC3676_BUCK1     0x01
+#define LTC3676_BUCK2     0x02
+#define LTC3676_BUCK3     0x03
+#define LTC3676_BUCK4     0x04
+#define LTC3676_LDOA      0x05
+#define LTC3676_LDOB      0x06
+#define LTC3676_SQD1      0x07
+#define LTC3676_SQD2      0x08
+#define LTC3676_CNTRL     0x09
+#define LTC3676_DVB1A     0x0A
+#define LTC3676_DVB1B     0x0B
+#define LTC3676_DVB2A     0x0C
+#define LTC3676_DVB2B     0x0D
+#define LTC3676_DVB3A     0x0E
+#define LTC3676_DVB3B     0x0F
+#define LTC3676_DVB4A     0x10
+#define LTC3676_DVB4B     0x11
+#define LTC3676_MSKIRQ    0x12
+#define LTC3676_MSKPG     0x13
+#define LTC3676_USER      0x14
+#define LTC3676_IRQSTAT   0x15
+#define LTC3676_PGSTATL   0x16
+#define LTC3676_PGSTATRT  0x17
+#define LTC3676_HRST      0x1E
+#define LTC3676_CLIRQ     0x1F
+
+#define LTC3676_DVBxA_REF_SELECT	BIT(5)
+
+#define LTC3676_IRQSTAT_PGOOD_TIMEOUT	BIT(3)
+#define LTC3676_IRQSTAT_UNDERVOLT_WARN	BIT(4)
+#define LTC3676_IRQSTAT_UNDERVOLT_FAULT	BIT(5)
+#define LTC3676_IRQSTAT_THERMAL_WARN	BIT(6)
+#define LTC3676_IRQSTAT_THERMAL_FAULT	BIT(7)
+
+enum ltc3676_reg {
+	LTC3676_SW1,
+	LTC3676_SW2,
+	LTC3676_SW3,
+	LTC3676_SW4,
+	LTC3676_LDO1,
+	LTC3676_LDO2,
+	LTC3676_LDO3,
+	LTC3676_LDO4,
+	LTC3676_NUM_REGULATORS,
+};
+
+struct ltc3676 {
+	struct regmap *regmap;
+	struct device *dev;
+	struct regulator_desc regulator_descs[LTC3676_NUM_REGULATORS];
+	struct regulator_dev *regulators[LTC3676_NUM_REGULATORS];
+};
+
+static int ltc3676_set_suspend_voltage(struct regulator_dev *rdev, int uV)
+{
+	struct ltc3676 *ltc3676 = rdev_get_drvdata(rdev);
+	struct device *dev = ltc3676->dev;
+	int dcdc = rdev_get_id(rdev);
+	int sel;
+
+	dev_dbg(dev, "%s id=%d uV=%d\n", __func__, dcdc, uV);
+	sel = regulator_map_voltage_linear(rdev, uV, uV);
+	if (sel < 0)
+		return sel;
+
+	/* DVBB register follows right after the corresponding DVBA register */
+	return regmap_update_bits(ltc3676->regmap, rdev->desc->vsel_reg + 1,
+				  rdev->desc->vsel_mask, sel);
+}
+
+static int ltc3676_set_suspend_mode(struct regulator_dev *rdev,
+				    unsigned int mode)
+{
+	struct ltc3676 *ltc3676= rdev_get_drvdata(rdev);
+	struct device *dev = ltc3676->dev;
+	int mask, val;
+	int dcdc = rdev_get_id(rdev);
+
+	dev_dbg(dev, "%s id=%d mode=%d\n", __func__, dcdc, mode);
+
+	mask = LTC3676_DVBxA_REF_SELECT;
+	switch (mode) {
+	case REGULATOR_MODE_STANDBY:
+		val = 0; /* select DVBxA */
+		break;
+	case REGULATOR_MODE_NORMAL:
+		val = LTC3676_DVBxA_REF_SELECT; /* select DVBxB */
+		break;
+	default:
+		dev_warn(&rdev->dev, "%s: regulator mode: 0x%x not supported\n",
+			 rdev->desc->name, mode);
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(ltc3676->regmap, rdev->desc->vsel_reg,
+				  mask, val);
+}
+
+static inline unsigned int ltc3676_scale(unsigned int uV, u32 r1, u32 r2)
+{
+	uint64_t tmp;
+	if (uV == 0)
+		return 0;
+	tmp = (uint64_t)uV * r1;
+	do_div(tmp, r2);
+	return uV + (unsigned int)tmp;
+}
+
+static int ltc3676_of_parse_cb(struct device_node *np,
+			       const struct regulator_desc *desc,
+			       struct regulator_config *config)
+{
+	struct ltc3676 *ltc3676 = config->driver_data;
+	struct regulator_desc *rdesc = &ltc3676->regulator_descs[desc->id];
+	u32 r[2];
+	int ret;
+
+	/* LDO3 has a fixed output */
+	if (desc->id == LTC3676_LDO3)
+		return 0;
+
+	ret = of_property_read_u32_array(np, "lltc,fb-voltage-divider", r, 2);
+	if (ret) {
+		dev_err(ltc3676->dev, "Failed to parse voltage divider: %d\n",
+			ret);
+		return ret;
+	}
+
+	rdesc->min_uV = ltc3676_scale(desc->min_uV, r[0], r[1]);
+	rdesc->uV_step = ltc3676_scale(desc->uV_step, r[0], r[1]);
+	rdesc->fixed_uV = ltc3676_scale(desc->fixed_uV, r[0], r[1]);
+
+	return 0;
+}
+
+/* SW1, SW2, SW3, SW4 linear 0.8V-3.3V with scalar via R1/R2 feeback res */
+static struct regulator_ops ltc3676_linear_regulator_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_linear,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_suspend_voltage = ltc3676_set_suspend_voltage,
+	.set_suspend_mode = ltc3676_set_suspend_mode,
+};
+
+/* LDO1 always on fixed 0.8V-3.3V via scalar via R1/R2 feeback res */
+static struct regulator_ops ltc3676_fixed_standby_regulator_ops = {
+};
+
+/* LDO2, LDO3 fixed (LDO2 has external scalar via R1/R2 feedback res) */
+static struct regulator_ops ltc3676_fixed_regulator_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+};
+
+#define LTC3676_REG(_id, _name, _ops, en_reg, en_bit, dvba_reg, dvb_mask)   \
+	[LTC3676_ ## _id] = {                                        \
+		.name = #_name,                                \
+		.of_match = of_match_ptr(#_name),              \
+		.regulators_node = of_match_ptr("regulators"), \
+		.of_parse_cb = ltc3676_of_parse_cb,            \
+		.n_voltages = (dvb_mask) + 1,                  \
+		.min_uV = (dvba_reg) ? 412500 : 0,             \
+		.uV_step = (dvba_reg) ? 12500 : 0,             \
+		.ramp_delay = (dvba_reg) ? 800 : 0,            \
+		.fixed_uV = (dvb_mask) ? 0 : 725000,           \
+		.ops = &ltc3676_ ## _ops ## _regulator_ops,    \
+		.type = REGULATOR_VOLTAGE,                     \
+		.id = LTC3676_ ## _id,                         \
+		.owner = THIS_MODULE,                          \
+		.vsel_reg = (dvba_reg),                        \
+		.vsel_mask = (dvb_mask),                       \
+		.enable_reg = (en_reg),                        \
+		.enable_mask = (1 << en_bit),                  \
+	}
+
+#define LTC3676_LINEAR_REG(_id, _name, _en, _dvba)                     \
+	LTC3676_REG(_id, _name, linear,                                \
+		    LTC3676_ ## _en, 7,                                \
+		    LTC3676_ ## _dvba, 0x1f)
+
+#define LTC3676_FIXED_REG(_id, _name, _en_reg, _en_bit)                \
+	LTC3676_REG(_id, _name, fixed, LTC3676_ ## _en_reg, _en_bit, 0, 0)
+
+static struct regulator_desc ltc3676_regulators[LTC3676_NUM_REGULATORS] = {
+	LTC3676_LINEAR_REG(SW1, sw1, BUCK1, DVB1A),
+	LTC3676_LINEAR_REG(SW2, sw2, BUCK2, DVB2A),
+	LTC3676_LINEAR_REG(SW3, sw3, BUCK3, DVB3A),
+	LTC3676_LINEAR_REG(SW4, sw4, BUCK4, DVB4A),
+	LTC3676_REG(LDO1, ldo1, fixed_standby, 0, 0, 0, 0),
+	LTC3676_FIXED_REG(LDO2, ldo2, LDOA, 2),
+	LTC3676_FIXED_REG(LDO3, ldo3, LDOA, 5),
+	LTC3676_FIXED_REG(LDO4, ldo4, LDOB, 2),
+};
+
+static bool ltc3676_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case LTC3676_IRQSTAT:
+	case LTC3676_BUCK1:
+	case LTC3676_BUCK2:
+	case LTC3676_BUCK3:
+	case LTC3676_BUCK4:
+	case LTC3676_LDOA:
+	case LTC3676_LDOB:
+	case LTC3676_SQD1:
+	case LTC3676_SQD2:
+	case LTC3676_CNTRL:
+	case LTC3676_DVB1A:
+	case LTC3676_DVB1B:
+	case LTC3676_DVB2A:
+	case LTC3676_DVB2B:
+	case LTC3676_DVB3A:
+	case LTC3676_DVB3B:
+	case LTC3676_DVB4A:
+	case LTC3676_DVB4B:
+	case LTC3676_MSKIRQ:
+	case LTC3676_MSKPG:
+	case LTC3676_USER:
+	case LTC3676_HRST:
+	case LTC3676_CLIRQ:
+		return true;
+	}
+	return false;
+}
+
+static bool ltc3676_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case LTC3676_IRQSTAT:
+	case LTC3676_BUCK1:
+	case LTC3676_BUCK2:
+	case LTC3676_BUCK3:
+	case LTC3676_BUCK4:
+	case LTC3676_LDOA:
+	case LTC3676_LDOB:
+	case LTC3676_SQD1:
+	case LTC3676_SQD2:
+	case LTC3676_CNTRL:
+	case LTC3676_DVB1A:
+	case LTC3676_DVB1B:
+	case LTC3676_DVB2A:
+	case LTC3676_DVB2B:
+	case LTC3676_DVB3A:
+	case LTC3676_DVB3B:
+	case LTC3676_DVB4A:
+	case LTC3676_DVB4B:
+	case LTC3676_MSKIRQ:
+	case LTC3676_MSKPG:
+	case LTC3676_USER:
+	case LTC3676_HRST:
+	case LTC3676_CLIRQ:
+		return true;
+	}
+	return false;
+}
+
+static bool ltc3676_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case LTC3676_IRQSTAT:
+	case LTC3676_PGSTATL:
+	case LTC3676_PGSTATRT:
+		return true;
+	}
+	return false;
+}
+
+static const struct regmap_config ltc3676_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.writeable_reg = ltc3676_writeable_reg,
+	.readable_reg = ltc3676_readable_reg,
+	.volatile_reg = ltc3676_volatile_reg,
+	.max_register = LTC3676_CLIRQ,
+	.use_single_rw = true,
+	.cache_type = REGCACHE_RBTREE,
+};
+
+static irqreturn_t ltc3676_isr(int irq, void *dev_id)
+{
+	struct ltc3676 *ltc3676 = dev_id;
+	struct device *dev = ltc3676->dev;
+	unsigned int i, irqstat, event;
+
+	regmap_read(ltc3676->regmap, LTC3676_IRQSTAT, &irqstat);
+
+	dev_dbg(dev, "irq%d irqstat=0x%02x\n", irq, irqstat);
+	if (irqstat & LTC3676_IRQSTAT_THERMAL_WARN) {
+		dev_warn(dev, "Over-temperature Warning\n");
+		event = REGULATOR_EVENT_OVER_TEMP;
+		for (i = 0; i < LTC3676_NUM_REGULATORS; i++)
+			regulator_notifier_call_chain(ltc3676->regulators[i],
+						      event, NULL);
+	}
+
+	if (irqstat & LTC3676_IRQSTAT_UNDERVOLT_WARN) {
+		dev_info(dev, "Undervoltage Warning\n");
+		event = REGULATOR_EVENT_UNDER_VOLTAGE;
+		for (i = 0; i < LTC3676_NUM_REGULATORS; i++)
+			regulator_notifier_call_chain(ltc3676->regulators[i],
+						      event, NULL);
+	}
+
+	/* Clear warning condition */
+	regmap_write(ltc3676->regmap, LTC3676_CLIRQ, 0);
+
+	return IRQ_HANDLED;
+}
+
+static int ltc3676_regulator_probe(struct i2c_client *client,
+				    const struct i2c_device_id *id)
+{
+	struct device *dev = &client->dev;
+	struct regulator_init_data *init_data = dev_get_platdata(dev);
+	struct regulator_desc *descs;
+	struct ltc3676 *ltc3676;
+	int i, ret;
+
+	ltc3676 = devm_kzalloc(dev, sizeof(*ltc3676), GFP_KERNEL);
+	if (!ltc3676)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, ltc3676);
+	ltc3676->dev = dev;
+
+	descs = ltc3676->regulator_descs;
+	memcpy(descs, ltc3676_regulators, sizeof(ltc3676_regulators));
+	descs[LTC3676_LDO3].fixed_uV = 1800000; /* LDO3 is fixed 1.8V */
+
+	ltc3676->regmap = devm_regmap_init_i2c(client, &ltc3676_regmap_config);
+	if (IS_ERR(ltc3676->regmap)) {
+		ret = PTR_ERR(ltc3676->regmap);
+		dev_err(dev, "failed to initialize regmap: %d\n", ret);
+		return ret;
+	}
+
+	for (i = 0; i < LTC3676_NUM_REGULATORS; i++) {
+		struct regulator_desc *desc = &ltc3676->regulator_descs[i];
+		struct regulator_config config = { };
+
+		if (init_data)
+			config.init_data = &init_data[i];
+
+		config.dev = dev;
+		config.driver_data = ltc3676;
+
+		ltc3676->regulators[i] = devm_regulator_register(dev, desc,
+								 &config);
+		if (IS_ERR(ltc3676->regulators[i])) {
+			ret = PTR_ERR(ltc3676->regulators[i]);
+			dev_err(dev, "failed to register regulator %s: %d\n",
+				desc->name, ret);
+			return ret;
+		}
+	}
+
+	regmap_write(ltc3676->regmap, LTC3676_CLIRQ, 0);
+	if (client->irq) {
+		ret = devm_request_threaded_irq(dev, client->irq, NULL,
+						ltc3676_isr,
+						IRQF_TRIGGER_LOW | IRQF_ONESHOT,
+						client->name, ltc3676);
+		if (ret) {
+			dev_err(dev, "Failed to request IRQ: %d\n", ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static const struct i2c_device_id ltc3676_i2c_id[] = {
+	{ "ltc3676" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, ltc3676_i2c_id);
+
+static struct i2c_driver ltc3676_driver = {
+	.driver = {
+		.name = DRIVER_NAME,
+	},
+	.probe = ltc3676_regulator_probe,
+	.id_table = ltc3676_i2c_id,
+};
+module_i2c_driver(ltc3676_driver);
+
+MODULE_AUTHOR("Tim Harvey <tharvey@gateworks.com>");
+MODULE_DESCRIPTION("Regulator driver for Linear Technology LTC1376");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/regulator/max14577-regulator.c b/drivers/regulator/max14577-regulator.c
index b2daa6641417..c9ff26199711 100644
--- a/drivers/regulator/max14577-regulator.c
+++ b/drivers/regulator/max14577-regulator.c
@@ -2,7 +2,7 @@
  * max14577.c - Regulator driver for the Maxim 14577/77836
  *
  * Copyright (C) 2013,2014 Samsung Electronics
- * Krzysztof Kozlowski <k.kozlowski@samsung.com>
+ * Krzysztof Kozlowski <krzk@kernel.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -331,7 +331,7 @@ static void __exit max14577_regulator_exit(void)
 }
 module_exit(max14577_regulator_exit);
 
-MODULE_AUTHOR("Krzysztof Kozlowski <k.kozlowski@samsung.com>");
+MODULE_AUTHOR("Krzysztof Kozlowski <krzk@kernel.org>");
 MODULE_DESCRIPTION("Maxim 14577/77836 regulator driver");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:max14577-regulator");
diff --git a/drivers/regulator/max77693-regulator.c b/drivers/regulator/max77693-regulator.c
index de730fd3f8a5..cfbb9512e486 100644
--- a/drivers/regulator/max77693-regulator.c
+++ b/drivers/regulator/max77693-regulator.c
@@ -3,7 +3,7 @@
  *
  * Copyright (C) 2013-2015 Samsung Electronics
  * Jonghwa Lee <jonghwa3.lee@samsung.com>
- * Krzysztof Kozlowski <k.kozlowski.k@gmail.com>
+ * Krzysztof Kozlowski <krzk@kernel.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -314,5 +314,5 @@ module_exit(max77693_pmic_cleanup);
 
 MODULE_DESCRIPTION("MAXIM 77693/77843 regulator driver");
 MODULE_AUTHOR("Jonghwa Lee <jonghwa3.lee@samsung.com>");
-MODULE_AUTHOR("Krzysztof Kozlowski <k.kozlowski.k@gmail.com>");
+MODULE_AUTHOR("Krzysztof Kozlowski <krzk@kernel.org>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/pv88080-regulator.c b/drivers/regulator/pv88080-regulator.c
index 81950bdb1cc4..954a20eeb26f 100644
--- a/drivers/regulator/pv88080-regulator.c
+++ b/drivers/regulator/pv88080-regulator.c
@@ -16,6 +16,7 @@
 #include <linux/err.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/regulator/driver.h>
@@ -26,7 +27,7 @@
 #include <linux/regulator/of_regulator.h>
 #include "pv88080-regulator.h"
 
-#define PV88080_MAX_REGULATORS	3
+#define PV88080_MAX_REGULATORS	4
 
 /* PV88080 REGULATOR IDs */
 enum {
@@ -34,6 +35,12 @@ enum {
 	PV88080_ID_BUCK1,
 	PV88080_ID_BUCK2,
 	PV88080_ID_BUCK3,
+	PV88080_ID_HVBUCK,
+};
+
+enum pv88080_types {
+	TYPE_PV88080_AA,
+	TYPE_PV88080_BA,
 };
 
 struct pv88080_regulator {
@@ -42,7 +49,8 @@ struct pv88080_regulator {
 	unsigned int n_current_limits;
 	const int *current_limits;
 	unsigned int limit_mask;
-	unsigned int conf;
+	unsigned int mode_reg;
+	unsigned int limit_reg;
 	unsigned int conf2;
 	unsigned int conf5;
 };
@@ -51,6 +59,8 @@ struct pv88080 {
 	struct device *dev;
 	struct regmap *regmap;
 	struct regulator_dev *rdev[PV88080_MAX_REGULATORS];
+	unsigned long type;
+	const struct pv88080_compatible_regmap *regmap_config;
 };
 
 struct pv88080_buck_voltage {
@@ -59,6 +69,30 @@ struct pv88080_buck_voltage {
 	int uV_step;
 };
 
+struct pv88080_buck_regmap {
+	/* REGS */
+	int buck_enable_reg;
+	int buck_vsel_reg;
+	int buck_mode_reg;
+	int buck_limit_reg;
+	int buck_vdac_range_reg;
+	int buck_vrange_gain_reg;
+	/* MASKS */
+	int buck_enable_mask;
+	int buck_vsel_mask;
+	int buck_limit_mask;
+};
+
+struct pv88080_compatible_regmap {
+	/* BUCK1, 2, 3 */
+	struct pv88080_buck_regmap buck_regmap[PV88080_MAX_REGULATORS-1];
+	/* HVBUCK */
+	int hvbuck_enable_reg;
+	int hvbuck_vsel_reg;
+	int hvbuck_enable_mask;
+	int hvbuck_vsel_mask;
+};
+
 static const struct regmap_config pv88080_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
@@ -89,13 +123,111 @@ static const struct pv88080_buck_voltage pv88080_buck_vol[2] = {
 	},
 };
 
+static const struct pv88080_compatible_regmap pv88080_aa_regs = {
+	/* BUCK1 */
+	.buck_regmap[0] = {
+		.buck_enable_reg      = PV88080AA_REG_BUCK1_CONF0,
+		.buck_vsel_reg        = PV88080AA_REG_BUCK1_CONF0,
+		.buck_mode_reg        = PV88080AA_REG_BUCK1_CONF1,
+		.buck_limit_reg       = PV88080AA_REG_BUCK1_CONF1,
+		.buck_vdac_range_reg  = PV88080AA_REG_BUCK1_CONF2,
+		.buck_vrange_gain_reg = PV88080AA_REG_BUCK1_CONF5,
+		.buck_enable_mask     = PV88080_BUCK1_EN,
+		.buck_vsel_mask       = PV88080_VBUCK1_MASK,
+		.buck_limit_mask      = PV88080_BUCK1_ILIM_MASK,
+	},
+	/* BUCK2 */
+	.buck_regmap[1] = {
+		.buck_enable_reg      = PV88080AA_REG_BUCK2_CONF0,
+		.buck_vsel_reg        = PV88080AA_REG_BUCK2_CONF0,
+		.buck_mode_reg        = PV88080AA_REG_BUCK2_CONF1,
+		.buck_limit_reg	      = PV88080AA_REG_BUCK2_CONF1,
+		.buck_vdac_range_reg  = PV88080AA_REG_BUCK2_CONF2,
+		.buck_vrange_gain_reg = PV88080AA_REG_BUCK2_CONF5,
+		.buck_enable_mask	  = PV88080_BUCK2_EN,
+		.buck_vsel_mask       = PV88080_VBUCK2_MASK,
+		.buck_limit_mask      = PV88080_BUCK2_ILIM_MASK,
+	},
+	/* BUCK3 */
+	.buck_regmap[2] = {
+		.buck_enable_reg	  = PV88080AA_REG_BUCK3_CONF0,
+		.buck_vsel_reg        = PV88080AA_REG_BUCK3_CONF0,
+		.buck_mode_reg        = PV88080AA_REG_BUCK3_CONF1,
+		.buck_limit_reg	      = PV88080AA_REG_BUCK3_CONF1,
+		.buck_vdac_range_reg  = PV88080AA_REG_BUCK3_CONF2,
+		.buck_vrange_gain_reg = PV88080AA_REG_BUCK3_CONF5,
+		.buck_enable_mask	  = PV88080_BUCK3_EN,
+		.buck_vsel_mask       = PV88080_VBUCK3_MASK,
+		.buck_limit_mask      = PV88080_BUCK3_ILIM_MASK,
+	},
+	/* HVBUCK */
+	.hvbuck_enable_reg	      = PV88080AA_REG_HVBUCK_CONF2,
+	.hvbuck_vsel_reg          = PV88080AA_REG_HVBUCK_CONF1,
+	.hvbuck_enable_mask       = PV88080_HVBUCK_EN,
+	.hvbuck_vsel_mask         = PV88080_VHVBUCK_MASK,
+};
+
+static const struct pv88080_compatible_regmap pv88080_ba_regs = {
+	/* BUCK1 */
+	.buck_regmap[0] = {
+		.buck_enable_reg	  = PV88080BA_REG_BUCK1_CONF0,
+		.buck_vsel_reg        = PV88080BA_REG_BUCK1_CONF0,
+		.buck_mode_reg        = PV88080BA_REG_BUCK1_CONF1,
+		.buck_limit_reg       = PV88080BA_REG_BUCK1_CONF1,
+		.buck_vdac_range_reg  = PV88080BA_REG_BUCK1_CONF2,
+		.buck_vrange_gain_reg = PV88080BA_REG_BUCK1_CONF5,
+		.buck_enable_mask     = PV88080_BUCK1_EN,
+		.buck_vsel_mask       = PV88080_VBUCK1_MASK,
+		.buck_limit_mask	  = PV88080_BUCK1_ILIM_MASK,
+	},
+	/* BUCK2 */
+	.buck_regmap[1] = {
+		.buck_enable_reg	  = PV88080BA_REG_BUCK2_CONF0,
+		.buck_vsel_reg        = PV88080BA_REG_BUCK2_CONF0,
+		.buck_mode_reg        = PV88080BA_REG_BUCK2_CONF1,
+		.buck_limit_reg	      = PV88080BA_REG_BUCK2_CONF1,
+		.buck_vdac_range_reg  = PV88080BA_REG_BUCK2_CONF2,
+		.buck_vrange_gain_reg = PV88080BA_REG_BUCK2_CONF5,
+		.buck_enable_mask	  = PV88080_BUCK2_EN,
+		.buck_vsel_mask       = PV88080_VBUCK2_MASK,
+		.buck_limit_mask	  = PV88080_BUCK2_ILIM_MASK,
+	},
+	/* BUCK3 */
+	.buck_regmap[2] = {
+		.buck_enable_reg	  = PV88080BA_REG_BUCK3_CONF0,
+		.buck_vsel_reg        = PV88080BA_REG_BUCK3_CONF0,
+		.buck_mode_reg        = PV88080BA_REG_BUCK3_CONF1,
+		.buck_limit_reg	      = PV88080BA_REG_BUCK3_CONF1,
+		.buck_vdac_range_reg  = PV88080BA_REG_BUCK3_CONF2,
+		.buck_vrange_gain_reg = PV88080BA_REG_BUCK3_CONF5,
+		.buck_enable_mask	  = PV88080_BUCK3_EN,
+		.buck_vsel_mask       = PV88080_VBUCK3_MASK,
+		.buck_limit_mask	  = PV88080_BUCK3_ILIM_MASK,
+	},
+	/* HVBUCK */
+	.hvbuck_enable_reg	      = PV88080BA_REG_HVBUCK_CONF2,
+	.hvbuck_vsel_reg          = PV88080BA_REG_HVBUCK_CONF1,
+	.hvbuck_enable_mask       = PV88080_HVBUCK_EN,
+	.hvbuck_vsel_mask		  = PV88080_VHVBUCK_MASK,
+};
+
+#ifdef CONFIG_OF
+static const struct of_device_id pv88080_dt_ids[] = {
+	{ .compatible = "pvs,pv88080",    .data = (void *)TYPE_PV88080_AA },
+	{ .compatible = "pvs,pv88080-aa", .data = (void *)TYPE_PV88080_AA },
+	{ .compatible = "pvs,pv88080-ba", .data = (void *)TYPE_PV88080_BA },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pv88080_dt_ids);
+#endif
+
 static unsigned int pv88080_buck_get_mode(struct regulator_dev *rdev)
 {
 	struct pv88080_regulator *info = rdev_get_drvdata(rdev);
 	unsigned int data;
 	int ret, mode = 0;
 
-	ret = regmap_read(rdev->regmap, info->conf, &data);
+	ret = regmap_read(rdev->regmap, info->mode_reg, &data);
 	if (ret < 0)
 		return ret;
 
@@ -136,7 +268,7 @@ static int pv88080_buck_set_mode(struct regulator_dev *rdev,
 		return -EINVAL;
 	}
 
-	return regmap_update_bits(rdev->regmap, info->conf,
+	return regmap_update_bits(rdev->regmap, info->mode_reg,
 					PV88080_BUCK1_MODE_MASK, val);
 }
 
@@ -151,7 +283,7 @@ static int pv88080_set_current_limit(struct regulator_dev *rdev, int min,
 		if (min <= info->current_limits[i]
 			&& max >= info->current_limits[i]) {
 				return regmap_update_bits(rdev->regmap,
-					info->conf,
+					info->limit_reg,
 					info->limit_mask,
 					i << PV88080_BUCK1_ILIM_SHIFT);
 		}
@@ -166,7 +298,7 @@ static int pv88080_get_current_limit(struct regulator_dev *rdev)
 	unsigned int data;
 	int ret;
 
-	ret = regmap_read(rdev->regmap, info->conf, &data);
+	ret = regmap_read(rdev->regmap, info->limit_reg, &data);
 	if (ret < 0)
 		return ret;
 
@@ -187,6 +319,15 @@ static struct regulator_ops pv88080_buck_ops = {
 	.get_current_limit = pv88080_get_current_limit,
 };
 
+static struct regulator_ops pv88080_hvbuck_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.list_voltage = regulator_list_voltage_linear,
+};
+
 #define PV88080_BUCK(chip, regl_name, min, step, max, limits_array) \
 {\
 	.desc	=	{\
@@ -200,17 +341,25 @@ static struct regulator_ops pv88080_buck_ops = {
 		.min_uV = min, \
 		.uV_step = step, \
 		.n_voltages = ((max) - (min))/(step) + 1, \
-		.enable_reg = PV88080_REG_##regl_name##_CONF0, \
-		.enable_mask = PV88080_##regl_name##_EN, \
-		.vsel_reg = PV88080_REG_##regl_name##_CONF0, \
-		.vsel_mask = PV88080_V##regl_name##_MASK, \
 	},\
 	.current_limits = limits_array, \
 	.n_current_limits = ARRAY_SIZE(limits_array), \
-	.limit_mask = PV88080_##regl_name##_ILIM_MASK, \
-	.conf = PV88080_REG_##regl_name##_CONF1, \
-	.conf2 = PV88080_REG_##regl_name##_CONF2, \
-	.conf5 = PV88080_REG_##regl_name##_CONF5, \
+}
+
+#define PV88080_HVBUCK(chip, regl_name, min, step, max) \
+{\
+	.desc	=	{\
+		.id = chip##_ID_##regl_name,\
+		.name = __stringify(chip##_##regl_name),\
+		.of_match = of_match_ptr(#regl_name),\
+		.regulators_node = of_match_ptr("regulators"),\
+		.type = REGULATOR_VOLTAGE,\
+		.owner = THIS_MODULE,\
+		.ops = &pv88080_hvbuck_ops,\
+		.min_uV = min, \
+		.uV_step = step, \
+		.n_voltages = ((max) - (min))/(step) + 1, \
+	},\
 }
 
 static struct pv88080_regulator pv88080_regulator_info[] = {
@@ -220,6 +369,7 @@ static struct pv88080_regulator pv88080_regulator_info[] = {
 		pv88080_buck23_limits),
 	PV88080_BUCK(PV88080, BUCK3, 600000, 6250, 1393750,
 		pv88080_buck23_limits),
+	PV88080_HVBUCK(PV88080, HVBUCK, 0, 5000, 1275000),
 };
 
 static irqreturn_t pv88080_irq_handler(int irq, void *data)
@@ -280,6 +430,8 @@ static int pv88080_i2c_probe(struct i2c_client *i2c,
 {
 	struct regulator_init_data *init_data = dev_get_platdata(&i2c->dev);
 	struct pv88080 *chip;
+	const struct pv88080_compatible_regmap *regmap_config;
+	const struct of_device_id *match;
 	struct regulator_config config = { };
 	int i, error, ret;
 	unsigned int conf2, conf5;
@@ -297,6 +449,17 @@ static int pv88080_i2c_probe(struct i2c_client *i2c,
 		return error;
 	}
 
+	if (i2c->dev.of_node) {
+		match = of_match_node(pv88080_dt_ids, i2c->dev.of_node);
+		if (!match) {
+			dev_err(chip->dev, "Failed to get of_match_node\n");
+			return -EINVAL;
+		}
+		chip->type = (unsigned long)match->data;
+	} else {
+		chip->type = id->driver_data;
+	}
+
 	i2c_set_clientdata(i2c, chip);
 
 	if (i2c->irq != 0) {
@@ -336,31 +499,58 @@ static int pv88080_i2c_probe(struct i2c_client *i2c,
 				"Failed to update mask reg: %d\n", ret);
 			return ret;
 		}
-
 	} else {
 		dev_warn(chip->dev, "No IRQ configured\n");
 	}
 
+	switch (chip->type) {
+	case TYPE_PV88080_AA:
+		chip->regmap_config = &pv88080_aa_regs;
+		break;
+	case TYPE_PV88080_BA:
+		chip->regmap_config = &pv88080_ba_regs;
+		break;
+	}
+
+	regmap_config = chip->regmap_config;
 	config.dev = chip->dev;
 	config.regmap = chip->regmap;
 
-	for (i = 0; i < PV88080_MAX_REGULATORS; i++) {
+	/* Registeration for BUCK1, 2, 3 */
+	for (i = 0; i < PV88080_MAX_REGULATORS-1; i++) {
 		if (init_data)
 			config.init_data = &init_data[i];
 
+		pv88080_regulator_info[i].limit_reg
+			= regmap_config->buck_regmap[i].buck_limit_reg;
+		pv88080_regulator_info[i].limit_mask
+			= regmap_config->buck_regmap[i].buck_limit_mask;
+		pv88080_regulator_info[i].mode_reg
+			= regmap_config->buck_regmap[i].buck_mode_reg;
+		pv88080_regulator_info[i].conf2
+			= regmap_config->buck_regmap[i].buck_vdac_range_reg;
+		pv88080_regulator_info[i].conf5
+			= regmap_config->buck_regmap[i].buck_vrange_gain_reg;
+		pv88080_regulator_info[i].desc.enable_reg
+			= regmap_config->buck_regmap[i].buck_enable_reg;
+		pv88080_regulator_info[i].desc.enable_mask
+			= regmap_config->buck_regmap[i].buck_enable_mask;
+		pv88080_regulator_info[i].desc.vsel_reg
+			= regmap_config->buck_regmap[i].buck_vsel_reg;
+		pv88080_regulator_info[i].desc.vsel_mask
+			= regmap_config->buck_regmap[i].buck_vsel_mask;
+
 		ret = regmap_read(chip->regmap,
-			pv88080_regulator_info[i].conf2, &conf2);
+				pv88080_regulator_info[i].conf2, &conf2);
 		if (ret < 0)
 			return ret;
-
 		conf2 = ((conf2 >> PV88080_BUCK_VDAC_RANGE_SHIFT) &
 			PV88080_BUCK_VDAC_RANGE_MASK);
 
 		ret = regmap_read(chip->regmap,
-			pv88080_regulator_info[i].conf5, &conf5);
+				pv88080_regulator_info[i].conf5, &conf5);
 		if (ret < 0)
 			return ret;
-
 		conf5 = ((conf5 >> PV88080_BUCK_VRANGE_GAIN_SHIFT) &
 			PV88080_BUCK_VRANGE_GAIN_MASK);
 
@@ -383,23 +573,38 @@ static int pv88080_i2c_probe(struct i2c_client *i2c,
 		}
 	}
 
+	pv88080_regulator_info[PV88080_ID_HVBUCK].desc.enable_reg
+		= regmap_config->hvbuck_enable_reg;
+	pv88080_regulator_info[PV88080_ID_HVBUCK].desc.enable_mask
+		= regmap_config->hvbuck_enable_mask;
+	pv88080_regulator_info[PV88080_ID_HVBUCK].desc.vsel_reg
+		= regmap_config->hvbuck_vsel_reg;
+	pv88080_regulator_info[PV88080_ID_HVBUCK].desc.vsel_mask
+		= regmap_config->hvbuck_vsel_mask;
+
+	/* Registeration for HVBUCK */
+	if (init_data)
+		config.init_data = &init_data[PV88080_ID_HVBUCK];
+
+	config.driver_data = (void *)&pv88080_regulator_info[PV88080_ID_HVBUCK];
+	chip->rdev[PV88080_ID_HVBUCK] = devm_regulator_register(chip->dev,
+		&pv88080_regulator_info[PV88080_ID_HVBUCK].desc, &config);
+	if (IS_ERR(chip->rdev[PV88080_ID_HVBUCK])) {
+		dev_err(chip->dev, "Failed to register PV88080 regulator\n");
+		return PTR_ERR(chip->rdev[PV88080_ID_HVBUCK]);
+	}
+
 	return 0;
 }
 
 static const struct i2c_device_id pv88080_i2c_id[] = {
-	{"pv88080", 0},
+	{ "pv88080",    TYPE_PV88080_AA },
+	{ "pv88080-aa", TYPE_PV88080_AA },
+	{ "pv88080-ba", TYPE_PV88080_BA },
 	{},
 };
 MODULE_DEVICE_TABLE(i2c, pv88080_i2c_id);
 
-#ifdef CONFIG_OF
-static const struct of_device_id pv88080_dt_ids[] = {
-	{ .compatible = "pvs,pv88080", .data = &pv88080_i2c_id[0] },
-	{},
-};
-MODULE_DEVICE_TABLE(of, pv88080_dt_ids);
-#endif
-
 static struct i2c_driver pv88080_regulator_driver = {
 	.driver = {
 		.name = "pv88080",
diff --git a/drivers/regulator/pv88080-regulator.h b/drivers/regulator/pv88080-regulator.h
index 5e9afde606f4..ae25ff360e3d 100644
--- a/drivers/regulator/pv88080-regulator.h
+++ b/drivers/regulator/pv88080-regulator.h
@@ -17,55 +17,75 @@
 #define __PV88080_REGISTERS_H__
 
 /* System Control and Event Registers */
-#define	PV88080_REG_EVENT_A			0x04
-#define	PV88080_REG_MASK_A			0x09
-#define	PV88080_REG_MASK_B			0x0a
-#define	PV88080_REG_MASK_C			0x0b
-
-/* Regulator Registers */
-#define	PV88080_REG_BUCK1_CONF0			0x27
-#define	PV88080_REG_BUCK1_CONF1			0x28
-#define	PV88080_REG_BUCK1_CONF2			0x59
-#define	PV88080_REG_BUCK1_CONF5			0x5c
-#define	PV88080_REG_BUCK2_CONF0			0x29
-#define	PV88080_REG_BUCK2_CONF1			0x2a
-#define	PV88080_REG_BUCK2_CONF2			0x61
-#define	PV88080_REG_BUCK2_CONF5			0x64
-#define	PV88080_REG_BUCK3_CONF0			0x2b
-#define	PV88080_REG_BUCK3_CONF1			0x2c
-#define	PV88080_REG_BUCK3_CONF2			0x69
-#define	PV88080_REG_BUCK3_CONF5			0x6c
+#define	PV88080_REG_EVENT_A				0x04
+#define	PV88080_REG_MASK_A				0x09
+#define	PV88080_REG_MASK_B				0x0A
+#define	PV88080_REG_MASK_C				0x0B
+
+/* Regulator Registers - rev. AA */
+#define PV88080AA_REG_HVBUCK_CONF1		0x2D
+#define PV88080AA_REG_HVBUCK_CONF2		0x2E
+#define	PV88080AA_REG_BUCK1_CONF0		0x27
+#define	PV88080AA_REG_BUCK1_CONF1		0x28
+#define	PV88080AA_REG_BUCK1_CONF2		0x59
+#define	PV88080AA_REG_BUCK1_CONF5		0x5C
+#define	PV88080AA_REG_BUCK2_CONF0		0x29
+#define	PV88080AA_REG_BUCK2_CONF1		0x2A
+#define	PV88080AA_REG_BUCK2_CONF2		0x61
+#define	PV88080AA_REG_BUCK2_CONF5		0x64
+#define	PV88080AA_REG_BUCK3_CONF0		0x2B
+#define	PV88080AA_REG_BUCK3_CONF1		0x2C
+#define	PV88080AA_REG_BUCK3_CONF2		0x69
+#define	PV88080AA_REG_BUCK3_CONF5		0x6C
+
+/* Regulator Registers - rev. BA */
+#define	PV88080BA_REG_HVBUCK_CONF1		0x33
+#define	PV88080BA_REG_HVBUCK_CONF2		0x34
+#define	PV88080BA_REG_BUCK1_CONF0		0x2A
+#define	PV88080BA_REG_BUCK1_CONF1		0x2C
+#define	PV88080BA_REG_BUCK1_CONF2		0x5A
+#define	PV88080BA_REG_BUCK1_CONF5		0x5D
+#define	PV88080BA_REG_BUCK2_CONF0		0x2D
+#define	PV88080BA_REG_BUCK2_CONF1		0x2F
+#define	PV88080BA_REG_BUCK2_CONF2		0x63
+#define	PV88080BA_REG_BUCK2_CONF5		0x66
+#define	PV88080BA_REG_BUCK3_CONF0		0x30
+#define	PV88080BA_REG_BUCK3_CONF1		0x32
+#define	PV88080BA_REG_BUCK3_CONF2		0x6C
+#define	PV88080BA_REG_BUCK3_CONF5		0x6F
 
 /* PV88080_REG_EVENT_A (addr=0x04) */
 #define	PV88080_E_VDD_FLT				0x01
-#define	PV88080_E_OVER_TEMP			0x02
+#define	PV88080_E_OVER_TEMP				0x02
 
 /* PV88080_REG_MASK_A (addr=0x09) */
 #define	PV88080_M_VDD_FLT				0x01
-#define	PV88080_M_OVER_TEMP			0x02
+#define	PV88080_M_OVER_TEMP				0x02
 
-/* PV88080_REG_BUCK1_CONF0 (addr=0x27) */
+/* PV88080_REG_BUCK1_CONF0 (addr=0x27|0x2A) */
 #define	PV88080_BUCK1_EN				0x80
-#define PV88080_VBUCK1_MASK			0x7F
-/* PV88080_REG_BUCK2_CONF0 (addr=0x29) */
+#define PV88080_VBUCK1_MASK				0x7F
+
+/* PV88080_REG_BUCK2_CONF0 (addr=0x29|0x2D) */
 #define	PV88080_BUCK2_EN				0x80
-#define PV88080_VBUCK2_MASK			0x7F
-/* PV88080_REG_BUCK3_CONF0 (addr=0x2b) */
+#define PV88080_VBUCK2_MASK				0x7F
+
+/* PV88080_REG_BUCK3_CONF0 (addr=0x2B|0x30) */
 #define	PV88080_BUCK3_EN				0x80
-#define PV88080_VBUCK3_MASK			0x7F
+#define PV88080_VBUCK3_MASK				0x7F
 
-/* PV88080_REG_BUCK1_CONF1 (addr=0x28) */
-#define PV88080_BUCK1_ILIM_SHIFT			2
+/* PV88080_REG_BUCK1_CONF1 (addr=0x28|0x2C) */
+#define PV88080_BUCK1_ILIM_SHIFT		2
 #define PV88080_BUCK1_ILIM_MASK			0x0C
 #define PV88080_BUCK1_MODE_MASK			0x03
 
-/* PV88080_REG_BUCK2_CONF1 (addr=0x2a) */
-#define PV88080_BUCK2_ILIM_SHIFT			2
+/* PV88080_REG_BUCK2_CONF1 (addr=0x2A|0x2F) */
+#define PV88080_BUCK2_ILIM_SHIFT		2
 #define PV88080_BUCK2_ILIM_MASK			0x0C
 #define PV88080_BUCK2_MODE_MASK			0x03
 
-/* PV88080_REG_BUCK3_CONF1 (addr=0x2c) */
-#define PV88080_BUCK3_ILIM_SHIFT			2
+/* PV88080_REG_BUCK3_CONF1 (addr=0x2C|0x32) */
+#define PV88080_BUCK3_ILIM_SHIFT		2
 #define PV88080_BUCK3_ILIM_MASK			0x0C
 #define PV88080_BUCK3_MODE_MASK			0x03
 
@@ -73,20 +93,26 @@
 #define	PV88080_BUCK_MODE_AUTO			0x01
 #define	PV88080_BUCK_MODE_SYNC			0x02
 
-/* PV88080_REG_BUCK2_CONF2 (addr=0x61) */
-/* PV88080_REG_BUCK3_CONF2 (addr=0x69) */
-#define PV88080_BUCK_VDAC_RANGE_SHIFT			7
-#define PV88080_BUCK_VDAC_RANGE_MASK			0x01
+/* PV88080_REG_HVBUCK_CONF1 (addr=0x2D|0x33) */
+#define PV88080_VHVBUCK_MASK			0xFF
+
+/* PV88080_REG_HVBUCK_CONF1 (addr=0x2E|0x34) */
+#define PV88080_HVBUCK_EN				0x01
+
+/* PV88080_REG_BUCK2_CONF2 (addr=0x61|0x63) */
+/* PV88080_REG_BUCK3_CONF2 (addr=0x69|0x6C) */
+#define PV88080_BUCK_VDAC_RANGE_SHIFT	7
+#define PV88080_BUCK_VDAC_RANGE_MASK	0x01
 
-#define PV88080_BUCK_VDAC_RANGE_1			0x00
-#define PV88080_BUCK_VDAC_RANGE_2			0x01
+#define PV88080_BUCK_VDAC_RANGE_1		0x00
+#define PV88080_BUCK_VDAC_RANGE_2		0x01
 
-/* PV88080_REG_BUCK2_CONF5 (addr=0x64) */
-/* PV88080_REG_BUCK3_CONF5 (addr=0x6c) */
-#define PV88080_BUCK_VRANGE_GAIN_SHIFT			0
-#define PV88080_BUCK_VRANGE_GAIN_MASK			0x01
+/* PV88080_REG_BUCK2_CONF5 (addr=0x64|0x66) */
+/* PV88080_REG_BUCK3_CONF5 (addr=0x6C|0x6F) */
+#define PV88080_BUCK_VRANGE_GAIN_SHIFT	0
+#define PV88080_BUCK_VRANGE_GAIN_MASK	0x01
 
-#define PV88080_BUCK_VRANGE_GAIN_1			0x00
-#define PV88080_BUCK_VRANGE_GAIN_2			0x01
+#define PV88080_BUCK_VRANGE_GAIN_1		0x00
+#define PV88080_BUCK_VRANGE_GAIN_2		0x01
 
 #endif	/* __PV88080_REGISTERS_H__ */
diff --git a/drivers/regulator/pwm-regulator.c b/drivers/regulator/pwm-regulator.c
index c24524242da2..1b88e0e15a70 100644
--- a/drivers/regulator/pwm-regulator.c
+++ b/drivers/regulator/pwm-regulator.c
@@ -10,7 +10,6 @@
  * published by the Free Software Foundation.
  */
 
-#include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/err.h>
@@ -194,12 +193,10 @@ static int pwm_regulator_set_voltage(struct regulator_dev *rdev,
 	unsigned int min_uV_duty = drvdata->continuous.min_uV_dutycycle;
 	unsigned int max_uV_duty = drvdata->continuous.max_uV_dutycycle;
 	unsigned int duty_unit = drvdata->continuous.dutycycle_unit;
-	unsigned int ramp_delay = rdev->constraints->ramp_delay;
 	int min_uV = rdev->constraints->min_uV;
 	int max_uV = rdev->constraints->max_uV;
 	int diff_uV = max_uV - min_uV;
 	struct pwm_state pstate;
-	int old_uV = pwm_regulator_get_voltage(rdev);
 	unsigned int diff_duty;
 	unsigned int dutycycle;
 	int ret;
@@ -233,13 +230,6 @@ static int pwm_regulator_set_voltage(struct regulator_dev *rdev,
 		return ret;
 	}
 
-	if ((ramp_delay == 0) || !pwm_regulator_is_enabled(rdev))
-		return 0;
-
-	/* Ramp delay is in uV/uS. Adjust to uS and delay */
-	ramp_delay = DIV_ROUND_UP(abs(req_min_uV - old_uV), ramp_delay);
-	usleep_range(ramp_delay, ramp_delay + DIV_ROUND_UP(ramp_delay, 10));
-
 	return 0;
 }
 
diff --git a/drivers/regulator/qcom_rpm-regulator.c b/drivers/regulator/qcom_rpm-regulator.c
index e254272585b2..1b2acc43fea1 100644
--- a/drivers/regulator/qcom_rpm-regulator.c
+++ b/drivers/regulator/qcom_rpm-regulator.c
@@ -448,6 +448,44 @@ static struct regulator_ops switch_ops = {
 };
 
 /*
+ * PM8018 regulators
+ */
+static const struct qcom_rpm_reg pm8018_pldo = {
+	.desc.linear_ranges = pldo_ranges,
+	.desc.n_linear_ranges = ARRAY_SIZE(pldo_ranges),
+	.desc.n_voltages = 161,
+	.desc.ops = &uV_ops,
+	.parts = &rpm8960_ldo_parts,
+	.supports_force_mode_auto = false,
+	.supports_force_mode_bypass = false,
+};
+
+static const struct qcom_rpm_reg pm8018_nldo = {
+	.desc.linear_ranges = nldo_ranges,
+	.desc.n_linear_ranges = ARRAY_SIZE(nldo_ranges),
+	.desc.n_voltages = 64,
+	.desc.ops = &uV_ops,
+	.parts = &rpm8960_ldo_parts,
+	.supports_force_mode_auto = false,
+	.supports_force_mode_bypass = false,
+};
+
+static const struct qcom_rpm_reg pm8018_smps = {
+	.desc.linear_ranges = smps_ranges,
+	.desc.n_linear_ranges = ARRAY_SIZE(smps_ranges),
+	.desc.n_voltages = 154,
+	.desc.ops = &uV_ops,
+	.parts = &rpm8960_smps_parts,
+	.supports_force_mode_auto = false,
+	.supports_force_mode_bypass = false,
+};
+
+static const struct qcom_rpm_reg pm8018_switch = {
+	.desc.ops = &switch_ops,
+	.parts = &rpm8960_switch_parts,
+};
+
+/*
  * PM8058 regulators
  */
 static const struct qcom_rpm_reg pm8058_pldo = {
@@ -755,6 +793,32 @@ struct rpm_regulator_data {
 	const char *supply;
 };
 
+static const struct rpm_regulator_data rpm_pm8018_regulators[] = {
+	{ "s1",  QCOM_RPM_PM8018_SMPS1, &pm8018_smps, "vdd_s1" },
+	{ "s2",  QCOM_RPM_PM8018_SMPS2, &pm8018_smps, "vdd_s2" },
+	{ "s3",  QCOM_RPM_PM8018_SMPS3, &pm8018_smps, "vdd_s3" },
+	{ "s4",  QCOM_RPM_PM8018_SMPS4, &pm8018_smps, "vdd_s4" },
+	{ "s5",  QCOM_RPM_PM8018_SMPS5, &pm8018_smps, "vdd_s5" },
+
+	{ "l2",  QCOM_RPM_PM8018_LDO2,  &pm8018_pldo, "vdd_l2" },
+	{ "l3",  QCOM_RPM_PM8018_LDO3,  &pm8018_pldo, "vdd_l3" },
+	{ "l4",  QCOM_RPM_PM8018_LDO4,  &pm8018_pldo, "vdd_l4" },
+	{ "l5",  QCOM_RPM_PM8018_LDO5,  &pm8018_pldo, "vdd_l5" },
+	{ "l6",  QCOM_RPM_PM8018_LDO6,  &pm8018_pldo, "vdd_l7" },
+	{ "l7",  QCOM_RPM_PM8018_LDO7,  &pm8018_pldo, "vdd_l7" },
+	{ "l8",  QCOM_RPM_PM8018_LDO8,  &pm8018_nldo, "vdd_l8" },
+	{ "l9",  QCOM_RPM_PM8018_LDO9,  &pm8921_nldo1200,
+						      "vdd_l9_l10_l11_l12" },
+	{ "l10", QCOM_RPM_PM8018_LDO10, &pm8018_nldo, "vdd_l9_l10_l11_l12" },
+	{ "l11", QCOM_RPM_PM8018_LDO11, &pm8018_nldo, "vdd_l9_l10_l11_l12" },
+	{ "l12", QCOM_RPM_PM8018_LDO12, &pm8018_nldo, "vdd_l9_l10_l11_l12" },
+	{ "l14", QCOM_RPM_PM8018_LDO14, &pm8018_pldo, "vdd_l14" },
+
+	{ "lvs1", QCOM_RPM_PM8018_LVS1, &pm8018_switch, "lvs1_in" },
+
+	{ }
+};
+
 static const struct rpm_regulator_data rpm_pm8058_regulators[] = {
 	{ "l0",   QCOM_RPM_PM8058_LDO0,   &pm8058_nldo, "vdd_l0_l1_lvs"	},
 	{ "l1",   QCOM_RPM_PM8058_LDO1,   &pm8058_nldo, "vdd_l0_l1_lvs" },
@@ -870,6 +934,8 @@ static const struct rpm_regulator_data rpm_pm8921_regulators[] = {
 };
 
 static const struct of_device_id rpm_of_match[] = {
+	{ .compatible = "qcom,rpm-pm8018-regulators",
+		.data = &rpm_pm8018_regulators },
 	{ .compatible = "qcom,rpm-pm8058-regulators", .data = &rpm_pm8058_regulators },
 	{ .compatible = "qcom,rpm-pm8901-regulators", .data = &rpm_pm8901_regulators },
 	{ .compatible = "qcom,rpm-pm8921-regulators", .data = &rpm_pm8921_regulators },
diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c
index 5022fa8d10c6..8ed46a9a55c8 100644
--- a/drivers/regulator/qcom_smd-regulator.c
+++ b/drivers/regulator/qcom_smd-regulator.c
@@ -178,20 +178,21 @@ static const struct regulator_desc pma8084_hfsmps = {
 static const struct regulator_desc pma8084_ftsmps = {
 	.linear_ranges = (struct regulator_linear_range[]) {
 		REGULATOR_LINEAR_RANGE(350000,  0, 184, 5000),
-		REGULATOR_LINEAR_RANGE(700000, 185, 339, 10000),
+		REGULATOR_LINEAR_RANGE(1280000, 185, 261, 10000),
 	},
 	.n_linear_ranges = 2,
-	.n_voltages = 340,
+	.n_voltages = 262,
 	.ops = &rpm_smps_ldo_ops,
 };
 
 static const struct regulator_desc pma8084_pldo = {
 	.linear_ranges = (struct regulator_linear_range[]) {
-		REGULATOR_LINEAR_RANGE(750000,  0,  30, 25000),
-		REGULATOR_LINEAR_RANGE(1500000, 31, 99, 50000),
+		REGULATOR_LINEAR_RANGE( 750000,  0,  63, 12500),
+		REGULATOR_LINEAR_RANGE(1550000, 64, 126, 25000),
+		REGULATOR_LINEAR_RANGE(3100000, 127, 163, 50000),
 	},
-	.n_linear_ranges = 2,
-	.n_voltages = 100,
+	.n_linear_ranges = 3,
+	.n_voltages = 164,
 	.ops = &rpm_smps_ldo_ops,
 };
 
@@ -221,29 +222,30 @@ static const struct regulator_desc pm8x41_hfsmps = {
 static const struct regulator_desc pm8841_ftsmps = {
 	.linear_ranges = (struct regulator_linear_range[]) {
 		REGULATOR_LINEAR_RANGE(350000,  0, 184, 5000),
-		REGULATOR_LINEAR_RANGE(700000, 185, 339, 10000),
+		REGULATOR_LINEAR_RANGE(1280000, 185, 261, 10000),
 	},
 	.n_linear_ranges = 2,
-	.n_voltages = 340,
+	.n_voltages = 262,
 	.ops = &rpm_smps_ldo_ops,
 };
 
 static const struct regulator_desc pm8941_boost = {
 	.linear_ranges = (struct regulator_linear_range[]) {
-		REGULATOR_LINEAR_RANGE(4000000, 0, 15, 100000),
+		REGULATOR_LINEAR_RANGE(4000000, 0, 30, 50000),
 	},
 	.n_linear_ranges = 1,
-	.n_voltages = 16,
+	.n_voltages = 31,
 	.ops = &rpm_smps_ldo_ops,
 };
 
 static const struct regulator_desc pm8941_pldo = {
 	.linear_ranges = (struct regulator_linear_range[]) {
-		REGULATOR_LINEAR_RANGE( 750000,  0,  30, 25000),
-		REGULATOR_LINEAR_RANGE(1500000, 31, 99, 50000),
+		REGULATOR_LINEAR_RANGE( 750000,  0,  63, 12500),
+		REGULATOR_LINEAR_RANGE(1550000, 64, 126, 25000),
+		REGULATOR_LINEAR_RANGE(3100000, 127, 163, 50000),
 	},
-	.n_linear_ranges = 2,
-	.n_voltages = 100,
+	.n_linear_ranges = 3,
+	.n_voltages = 164,
 	.ops = &rpm_smps_ldo_ops,
 };
 
diff --git a/drivers/regulator/rk808-regulator.c b/drivers/regulator/rk808-regulator.c
index 40d07ba036e7..3314bf299a51 100644
--- a/drivers/regulator/rk808-regulator.c
+++ b/drivers/regulator/rk808-regulator.c
@@ -1,11 +1,15 @@
 /*
- * Regulator driver for Rockchip RK808
+ * Regulator driver for Rockchip RK808/RK818
  *
  * Copyright (c) 2014, Fuzhou Rockchip Electronics Co., Ltd
  *
  * Author: Chris Zhong <zyw@rock-chips.com>
  * Author: Zhang Qing <zhangqing@rock-chips.com>
  *
+ * Copyright (C) 2016 PHYTEC Messtechnik GmbH
+ *
+ * Author: Wadim Egorov <w.egorov@phytec.de>
+ *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
@@ -32,6 +36,12 @@
 #define RK808_BUCK4_VSEL_MASK	0xf
 #define RK808_LDO_VSEL_MASK	0x1f
 
+#define RK818_BUCK_VSEL_MASK		0x3f
+#define RK818_BUCK4_VSEL_MASK		0x1f
+#define RK818_LDO_VSEL_MASK		0x1f
+#define RK818_LDO3_ON_VSEL_MASK		0xf
+#define RK818_BOOST_ON_VSEL_MASK	0xe0
+
 /* Ramp rate definitions for buck1 / buck2 only */
 #define RK808_RAMP_RATE_OFFSET		3
 #define RK808_RAMP_RATE_MASK		(3 << RK808_RAMP_RATE_OFFSET)
@@ -454,6 +464,108 @@ static const struct regulator_desc rk808_reg[] = {
 		RK808_DCDC_EN_REG, BIT(6)),
 };
 
+static const struct regulator_desc rk818_reg[] = {
+	{
+		.name = "DCDC_REG1",
+		.supply_name = "vcc1",
+		.of_match = of_match_ptr("DCDC_REG1"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = RK818_ID_DCDC1,
+		.ops = &rk808_reg_ops,
+		.type = REGULATOR_VOLTAGE,
+		.min_uV = 712500,
+		.uV_step = 12500,
+		.n_voltages = 64,
+		.vsel_reg = RK818_BUCK1_ON_VSEL_REG,
+		.vsel_mask = RK818_BUCK_VSEL_MASK,
+		.enable_reg = RK818_DCDC_EN_REG,
+		.enable_mask = BIT(0),
+		.owner = THIS_MODULE,
+	}, {
+		.name = "DCDC_REG2",
+		.supply_name = "vcc2",
+		.of_match = of_match_ptr("DCDC_REG2"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = RK818_ID_DCDC2,
+		.ops = &rk808_reg_ops,
+		.type = REGULATOR_VOLTAGE,
+		.min_uV = 712500,
+		.uV_step = 12500,
+		.n_voltages = 64,
+		.vsel_reg = RK818_BUCK2_ON_VSEL_REG,
+		.vsel_mask = RK818_BUCK_VSEL_MASK,
+		.enable_reg = RK818_DCDC_EN_REG,
+		.enable_mask = BIT(1),
+		.owner = THIS_MODULE,
+	}, {
+		.name = "DCDC_REG3",
+		.supply_name = "vcc3",
+		.of_match = of_match_ptr("DCDC_REG3"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = RK818_ID_DCDC3,
+		.ops = &rk808_switch_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = 1,
+		.enable_reg = RK818_DCDC_EN_REG,
+		.enable_mask = BIT(2),
+		.owner = THIS_MODULE,
+	},
+	RK8XX_DESC(RK818_ID_DCDC4, "DCDC_REG4", "vcc4", 1800, 3600, 100,
+		RK818_BUCK4_ON_VSEL_REG, RK818_BUCK4_VSEL_MASK,
+		RK818_DCDC_EN_REG, BIT(3), 0),
+	RK8XX_DESC(RK818_ID_BOOST, "DCDC_BOOST", "boost", 4700, 5400, 100,
+		RK818_BOOST_LDO9_ON_VSEL_REG, RK818_BOOST_ON_VSEL_MASK,
+		RK818_DCDC_EN_REG, BIT(4), 0),
+	RK8XX_DESC(RK818_ID_LDO1, "LDO_REG1", "vcc6", 1800, 3400, 100,
+		RK818_LDO1_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK818_LDO_EN_REG,
+		BIT(0), 400),
+	RK8XX_DESC(RK818_ID_LDO2, "LDO_REG2", "vcc6", 1800, 3400, 100,
+		RK818_LDO1_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK818_LDO_EN_REG,
+		BIT(1), 400),
+	{
+		.name = "LDO_REG3",
+		.supply_name = "vcc7",
+		.of_match = of_match_ptr("LDO_REG3"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = RK818_ID_LDO3,
+		.ops = &rk808_reg_ops_ranges,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = 16,
+		.linear_ranges = rk808_ldo3_voltage_ranges,
+		.n_linear_ranges = ARRAY_SIZE(rk808_ldo3_voltage_ranges),
+		.vsel_reg = RK818_LDO3_ON_VSEL_REG,
+		.vsel_mask = RK818_LDO3_ON_VSEL_MASK,
+		.enable_reg = RK818_LDO_EN_REG,
+		.enable_mask = BIT(2),
+		.enable_time = 400,
+		.owner = THIS_MODULE,
+	},
+	RK8XX_DESC(RK818_ID_LDO4, "LDO_REG4", "vcc8", 1800, 3400, 100,
+		RK818_LDO4_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK818_LDO_EN_REG,
+		BIT(3), 400),
+	RK8XX_DESC(RK818_ID_LDO5, "LDO_REG5", "vcc7", 1800, 3400, 100,
+		RK818_LDO5_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK818_LDO_EN_REG,
+		BIT(4), 400),
+	RK8XX_DESC(RK818_ID_LDO6, "LDO_REG6", "vcc8", 800, 2500, 100,
+		RK818_LDO6_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK818_LDO_EN_REG,
+		BIT(5), 400),
+	RK8XX_DESC(RK818_ID_LDO7, "LDO_REG7", "vcc7", 800, 2500, 100,
+		RK818_LDO7_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK818_LDO_EN_REG,
+		BIT(6), 400),
+	RK8XX_DESC(RK818_ID_LDO8, "LDO_REG8", "vcc8", 1800, 3400, 100,
+		RK818_LDO8_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK818_LDO_EN_REG,
+		BIT(7), 400),
+	RK8XX_DESC(RK818_ID_LDO9, "LDO_REG9", "vcc9", 1800, 3400, 100,
+		RK818_BOOST_LDO9_ON_VSEL_REG, RK818_LDO_VSEL_MASK,
+		RK818_DCDC_EN_REG, BIT(5), 400),
+	RK8XX_DESC_SWITCH(RK818_ID_SWITCH, "SWITCH_REG", "vcc9",
+		RK818_DCDC_EN_REG, BIT(6)),
+	RK8XX_DESC_SWITCH(RK818_ID_HDMI_SWITCH, "HDMI_SWITCH", "h_5v",
+		RK818_H5V_EN_REG, BIT(0)),
+	RK8XX_DESC_SWITCH(RK818_ID_OTG_SWITCH, "OTG_SWITCH", "usb",
+		RK818_DCDC_EN_REG, BIT(7)),
+};
+
 static int rk808_regulator_dt_parse_pdata(struct device *dev,
 				   struct device *client_dev,
 				   struct regmap *map,
@@ -499,7 +611,8 @@ static int rk808_regulator_probe(struct platform_device *pdev)
 	struct regulator_config config = {};
 	struct regulator_dev *rk808_rdev;
 	struct rk808_regulator_data *pdata;
-	int ret, i;
+	const struct regulator_desc *regulators;
+	int ret, i, nregulators;
 
 	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata)
@@ -512,14 +625,29 @@ static int rk808_regulator_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, pdata);
 
+	switch (rk808->variant) {
+	case RK808_ID:
+		regulators = rk808_reg;
+		nregulators = RK808_NUM_REGULATORS;
+		break;
+	case RK818_ID:
+		regulators = rk818_reg;
+		nregulators = RK818_NUM_REGULATORS;
+		break;
+	default:
+		dev_err(&client->dev, "unsupported RK8XX ID %lu\n",
+			rk808->variant);
+		return -EINVAL;
+	}
+
 	config.dev = &client->dev;
 	config.driver_data = pdata;
 	config.regmap = rk808->regmap;
 
 	/* Instantiate the regulators */
-	for (i = 0; i < RK808_NUM_REGULATORS; i++) {
+	for (i = 0; i < nregulators; i++) {
 		rk808_rdev = devm_regulator_register(&pdev->dev,
-						     &rk808_reg[i], &config);
+						     &regulators[i], &config);
 		if (IS_ERR(rk808_rdev)) {
 			dev_err(&client->dev,
 				"failed to register %d regulator\n", i);
@@ -533,15 +661,15 @@ static int rk808_regulator_probe(struct platform_device *pdev)
 static struct platform_driver rk808_regulator_driver = {
 	.probe = rk808_regulator_probe,
 	.driver = {
-		.name = "rk808-regulator",
-		.owner = THIS_MODULE,
+		.name = "rk808-regulator"
 	},
 };
 
 module_platform_driver(rk808_regulator_driver);
 
-MODULE_DESCRIPTION("regulator driver for the rk808 series PMICs");
-MODULE_AUTHOR("Chris Zhong<zyw@rock-chips.com>");
-MODULE_AUTHOR("Zhang Qing<zhangqing@rock-chips.com>");
+MODULE_DESCRIPTION("regulator driver for the RK808/RK818 series PMICs");
+MODULE_AUTHOR("Chris Zhong <zyw@rock-chips.com>");
+MODULE_AUTHOR("Zhang Qing <zhangqing@rock-chips.com>");
+MODULE_AUTHOR("Wadim Egorov <w.egorov@phytec.de>");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:rk808-regulator");
diff --git a/drivers/regulator/tps65218-regulator.c b/drivers/regulator/tps65218-regulator.c
index d1e631d64a20..eb0f5b13841a 100644
--- a/drivers/regulator/tps65218-regulator.c
+++ b/drivers/regulator/tps65218-regulator.c
@@ -180,6 +180,14 @@ static int tps65218_pmic_set_suspend_disable(struct regulator_dev *dev)
 	if (rid < TPS65218_DCDC_1 || rid > TPS65218_LDO_1)
 		return -EINVAL;
 
+	/*
+	 * Certain revisions of TPS65218 will need to have DCDC3 regulator
+	 * enabled always, otherwise an immediate system reboot will occur
+	 * during poweroff.
+	 */
+	if (rid == TPS65218_DCDC_3 && tps->rev == TPS65218_REV_2_1)
+		return 0;
+
 	if (!tps->info[rid]->strobe) {
 		if (rid == TPS65218_DCDC_3)
 			tps->info[rid]->strobe = 3;
diff --git a/drivers/regulator/tps65910-regulator.c b/drivers/regulator/tps65910-regulator.c
index fb991ec76423..696116ebdf50 100644
--- a/drivers/regulator/tps65910-regulator.c
+++ b/drivers/regulator/tps65910-regulator.c
@@ -1111,6 +1111,12 @@ static int tps65910_probe(struct platform_device *pdev)
 		pmic->num_regulators = ARRAY_SIZE(tps65910_regs);
 		pmic->ext_sleep_control = tps65910_ext_sleep_control;
 		info = tps65910_regs;
+		/* Work around silicon erratum SWCZ010: output programmed
+		 * voltage level can go higher than expected or crash
+		 * Workaround: use no synchronization of DCDC clocks
+		 */
+		tps65910_reg_clear_bits(pmic->mfd, TPS65910_DCDCCTRL,
+					DCDCCTRL_DCDCCKSYNC_MASK);
 		break;
 	case TPS65911:
 		pmic->get_ctrl_reg = &tps65911_get_ctrl_register;
diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index 1a8bf76a925f..f396bfef5d42 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -17,7 +17,7 @@ config OMAP_REMOTEPROC
 	select REMOTEPROC
 	select MAILBOX
 	select OMAP2PLUS_MBOX
-	select RPMSG
+	select RPMSG_VIRTIO
 	help
 	  Say y here to support OMAP's remote processors (dual M3
 	  and DSP on OMAP4) via the remote processor framework.
@@ -59,7 +59,7 @@ config DA8XX_REMOTEPROC
 	depends on ARCH_DAVINCI_DA8XX
 	select CMA if MMU
 	select REMOTEPROC
-	select RPMSG
+	select RPMSG_VIRTIO
 	help
 	  Say y here to support DA8xx/OMAP-L13x remote processors via the
 	  remote processor framework.
@@ -91,6 +91,22 @@ config QCOM_Q6V5_PIL
 	  Say y here to support the Qualcomm Peripherial Image Loader for the
 	  Hexagon V5 based remote processors.
 
+config QCOM_WCNSS_IRIS
+	tristate
+	depends on OF && ARCH_QCOM
+
+config QCOM_WCNSS_PIL
+	tristate "Qualcomm WCNSS Peripheral Image Loader"
+	depends on OF && ARCH_QCOM
+	depends on QCOM_SMEM
+	select QCOM_MDT_LOADER
+	select QCOM_SCM
+	select QCOM_WCNSS_IRIS
+	select REMOTEPROC
+	help
+	  Say y here to support the Peripheral Image Loader for the Qualcomm
+	  Wireless Connectivity Subsystem.
+
 config ST_REMOTEPROC
 	tristate "ST remoteproc support"
 	depends on ARCH_STI
diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
index 92d3758bd15c..6dfb62ed643f 100644
--- a/drivers/remoteproc/Makefile
+++ b/drivers/remoteproc/Makefile
@@ -13,4 +13,6 @@ obj-$(CONFIG_WKUP_M3_RPROC)		+= wkup_m3_rproc.o
 obj-$(CONFIG_DA8XX_REMOTEPROC)		+= da8xx_remoteproc.o
 obj-$(CONFIG_QCOM_MDT_LOADER)		+= qcom_mdt_loader.o
 obj-$(CONFIG_QCOM_Q6V5_PIL)		+= qcom_q6v5_pil.o
+obj-$(CONFIG_QCOM_WCNSS_IRIS)		+= qcom_wcnss_iris.o
+obj-$(CONFIG_QCOM_WCNSS_PIL)		+= qcom_wcnss.o
 obj-$(CONFIG_ST_REMOTEPROC)		+= st_remoteproc.o
diff --git a/drivers/remoteproc/da8xx_remoteproc.c b/drivers/remoteproc/da8xx_remoteproc.c
index 009e56f67de2..1afac8f31be0 100644
--- a/drivers/remoteproc/da8xx_remoteproc.c
+++ b/drivers/remoteproc/da8xx_remoteproc.c
@@ -147,7 +147,7 @@ static void da8xx_rproc_kick(struct rproc *rproc, int vqid)
 {
 	struct da8xx_rproc *drproc = (struct da8xx_rproc *)rproc->priv;
 
-	/* Interupt remote proc */
+	/* Interrupt remote proc */
 	writel(SYSCFG_CHIPSIG2, drproc->chipsig);
 }
 
@@ -261,7 +261,7 @@ static int da8xx_rproc_probe(struct platform_device *pdev)
 	return 0;
 
 free_rproc:
-	rproc_put(rproc);
+	rproc_free(rproc);
 
 	return ret;
 }
@@ -290,7 +290,7 @@ static int da8xx_rproc_remove(struct platform_device *pdev)
 	disable_irq(drproc->irq);
 
 	rproc_del(rproc);
-	rproc_put(rproc);
+	rproc_free(rproc);
 
 	return 0;
 }
diff --git a/drivers/remoteproc/omap_remoteproc.c b/drivers/remoteproc/omap_remoteproc.c
index b74368a91235..fa63bf2eb885 100644
--- a/drivers/remoteproc/omap_remoteproc.c
+++ b/drivers/remoteproc/omap_remoteproc.c
@@ -96,7 +96,8 @@ static void omap_rproc_kick(struct rproc *rproc, int vqid)
 	/* send the index of the triggered virtqueue in the mailbox payload */
 	ret = mbox_send_message(oproc->mbox, (void *)vqid);
 	if (ret < 0)
-		dev_err(dev, "omap_mbox_msg_send failed: %d\n", ret);
+		dev_err(dev, "failed to send mailbox message, status = %d\n",
+			ret);
 }
 
 /*
@@ -196,7 +197,7 @@ static int omap_rproc_probe(struct platform_device *pdev)
 	}
 
 	rproc = rproc_alloc(&pdev->dev, pdata->name, &omap_rproc_ops,
-				pdata->firmware, sizeof(*oproc));
+			    pdata->firmware, sizeof(*oproc));
 	if (!rproc)
 		return -ENOMEM;
 
@@ -214,7 +215,7 @@ static int omap_rproc_probe(struct platform_device *pdev)
 	return 0;
 
 free_rproc:
-	rproc_put(rproc);
+	rproc_free(rproc);
 	return ret;
 }
 
@@ -223,7 +224,7 @@ static int omap_rproc_remove(struct platform_device *pdev)
 	struct rproc *rproc = platform_get_drvdata(pdev);
 
 	rproc_del(rproc);
-	rproc_put(rproc);
+	rproc_free(rproc);
 
 	return 0;
 }
diff --git a/drivers/remoteproc/qcom_q6v5_pil.c b/drivers/remoteproc/qcom_q6v5_pil.c
index 2a1b2c7d8f2c..2e0caaaa766a 100644
--- a/drivers/remoteproc/qcom_q6v5_pil.c
+++ b/drivers/remoteproc/qcom_q6v5_pil.c
@@ -863,8 +863,10 @@ static int q6v5_probe(struct platform_device *pdev)
 		goto free_rproc;
 
 	qproc->state = qcom_smem_state_get(&pdev->dev, "stop", &qproc->stop_bit);
-	if (IS_ERR(qproc->state))
+	if (IS_ERR(qproc->state)) {
+		ret = PTR_ERR(qproc->state);
 		goto free_rproc;
+	}
 
 	ret = rproc_add(rproc);
 	if (ret)
@@ -873,7 +875,7 @@ static int q6v5_probe(struct platform_device *pdev)
 	return 0;
 
 free_rproc:
-	rproc_put(rproc);
+	rproc_free(rproc);
 
 	return ret;
 }
@@ -883,7 +885,7 @@ static int q6v5_remove(struct platform_device *pdev)
 	struct q6v5 *qproc = platform_get_drvdata(pdev);
 
 	rproc_del(qproc->rproc);
-	rproc_put(qproc->rproc);
+	rproc_free(qproc->rproc);
 
 	return 0;
 }
diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c
new file mode 100644
index 000000000000..f5cedeaafba1
--- /dev/null
+++ b/drivers/remoteproc/qcom_wcnss.c
@@ -0,0 +1,624 @@
+/*
+ * Qualcomm Wireless Connectivity Subsystem Peripheral Image Loader
+ *
+ * Copyright (C) 2016 Linaro Ltd
+ * Copyright (C) 2014 Sony Mobile Communications AB
+ * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/qcom_scm.h>
+#include <linux/regulator/consumer.h>
+#include <linux/remoteproc.h>
+#include <linux/soc/qcom/smem.h>
+#include <linux/soc/qcom/smem_state.h>
+
+#include "qcom_mdt_loader.h"
+#include "remoteproc_internal.h"
+#include "qcom_wcnss.h"
+
+#define WCNSS_CRASH_REASON_SMEM		422
+#define WCNSS_FIRMWARE_NAME		"wcnss.mdt"
+#define WCNSS_PAS_ID			6
+
+#define WCNSS_SPARE_NVBIN_DLND		BIT(25)
+
+#define WCNSS_PMU_IRIS_XO_CFG		BIT(3)
+#define WCNSS_PMU_IRIS_XO_EN		BIT(4)
+#define WCNSS_PMU_GC_BUS_MUX_SEL_TOP	BIT(5)
+#define WCNSS_PMU_IRIS_XO_CFG_STS	BIT(6) /* 1: in progress, 0: done */
+
+#define WCNSS_PMU_IRIS_RESET		BIT(7)
+#define WCNSS_PMU_IRIS_RESET_STS	BIT(8) /* 1: in progress, 0: done */
+#define WCNSS_PMU_IRIS_XO_READ		BIT(9)
+#define WCNSS_PMU_IRIS_XO_READ_STS	BIT(10)
+
+#define WCNSS_PMU_XO_MODE_MASK		GENMASK(2, 1)
+#define WCNSS_PMU_XO_MODE_19p2		0
+#define WCNSS_PMU_XO_MODE_48		3
+
+struct wcnss_data {
+	size_t pmu_offset;
+	size_t spare_offset;
+
+	const struct wcnss_vreg_info *vregs;
+	size_t num_vregs;
+};
+
+struct qcom_wcnss {
+	struct device *dev;
+	struct rproc *rproc;
+
+	void __iomem *pmu_cfg;
+	void __iomem *spare_out;
+
+	bool use_48mhz_xo;
+
+	int wdog_irq;
+	int fatal_irq;
+	int ready_irq;
+	int handover_irq;
+	int stop_ack_irq;
+
+	struct qcom_smem_state *state;
+	unsigned stop_bit;
+
+	struct mutex iris_lock;
+	struct qcom_iris *iris;
+
+	struct regulator_bulk_data *vregs;
+	size_t num_vregs;
+
+	struct completion start_done;
+	struct completion stop_done;
+
+	phys_addr_t mem_phys;
+	phys_addr_t mem_reloc;
+	void *mem_region;
+	size_t mem_size;
+};
+
+static const struct wcnss_data riva_data = {
+	.pmu_offset = 0x28,
+	.spare_offset = 0xb4,
+
+	.vregs = (struct wcnss_vreg_info[]) {
+		{ "vddmx",  1050000, 1150000, 0 },
+		{ "vddcx",  1050000, 1150000, 0 },
+		{ "vddpx",  1800000, 1800000, 0 },
+	},
+	.num_vregs = 3,
+};
+
+static const struct wcnss_data pronto_v1_data = {
+	.pmu_offset = 0x1004,
+	.spare_offset = 0x1088,
+
+	.vregs = (struct wcnss_vreg_info[]) {
+		{ "vddmx", 950000, 1150000, 0 },
+		{ "vddcx", .super_turbo = true},
+		{ "vddpx", 1800000, 1800000, 0 },
+	},
+	.num_vregs = 3,
+};
+
+static const struct wcnss_data pronto_v2_data = {
+	.pmu_offset = 0x1004,
+	.spare_offset = 0x1088,
+
+	.vregs = (struct wcnss_vreg_info[]) {
+		{ "vddmx", 1287500, 1287500, 0 },
+		{ "vddcx", .super_turbo = true },
+		{ "vddpx", 1800000, 1800000, 0 },
+	},
+	.num_vregs = 3,
+};
+
+void qcom_wcnss_assign_iris(struct qcom_wcnss *wcnss,
+			    struct qcom_iris *iris,
+			    bool use_48mhz_xo)
+{
+	mutex_lock(&wcnss->iris_lock);
+
+	wcnss->iris = iris;
+	wcnss->use_48mhz_xo = use_48mhz_xo;
+
+	mutex_unlock(&wcnss->iris_lock);
+}
+EXPORT_SYMBOL_GPL(qcom_wcnss_assign_iris);
+
+static int wcnss_load(struct rproc *rproc, const struct firmware *fw)
+{
+	struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
+	phys_addr_t fw_addr;
+	size_t fw_size;
+	bool relocate;
+	int ret;
+
+	ret = qcom_scm_pas_init_image(WCNSS_PAS_ID, fw->data, fw->size);
+	if (ret) {
+		dev_err(&rproc->dev, "invalid firmware metadata\n");
+		return ret;
+	}
+
+	ret = qcom_mdt_parse(fw, &fw_addr, &fw_size, &relocate);
+	if (ret) {
+		dev_err(&rproc->dev, "failed to parse mdt header\n");
+		return ret;
+	}
+
+	if (relocate) {
+		wcnss->mem_reloc = fw_addr;
+
+		ret = qcom_scm_pas_mem_setup(WCNSS_PAS_ID, wcnss->mem_phys, fw_size);
+		if (ret) {
+			dev_err(&rproc->dev, "unable to setup memory for image\n");
+			return ret;
+		}
+	}
+
+	return qcom_mdt_load(rproc, fw, rproc->firmware);
+}
+
+static const struct rproc_fw_ops wcnss_fw_ops = {
+	.find_rsc_table = qcom_mdt_find_rsc_table,
+	.load = wcnss_load,
+};
+
+static void wcnss_indicate_nv_download(struct qcom_wcnss *wcnss)
+{
+	u32 val;
+
+	/* Indicate NV download capability */
+	val = readl(wcnss->spare_out);
+	val |= WCNSS_SPARE_NVBIN_DLND;
+	writel(val, wcnss->spare_out);
+}
+
+static void wcnss_configure_iris(struct qcom_wcnss *wcnss)
+{
+	u32 val;
+
+	/* Clear PMU cfg register */
+	writel(0, wcnss->pmu_cfg);
+
+	val = WCNSS_PMU_GC_BUS_MUX_SEL_TOP | WCNSS_PMU_IRIS_XO_EN;
+	writel(val, wcnss->pmu_cfg);
+
+	/* Clear XO_MODE */
+	val &= ~WCNSS_PMU_XO_MODE_MASK;
+	if (wcnss->use_48mhz_xo)
+		val |= WCNSS_PMU_XO_MODE_48 << 1;
+	else
+		val |= WCNSS_PMU_XO_MODE_19p2 << 1;
+	writel(val, wcnss->pmu_cfg);
+
+	/* Reset IRIS */
+	val |= WCNSS_PMU_IRIS_RESET;
+	writel(val, wcnss->pmu_cfg);
+
+	/* Wait for PMU.iris_reg_reset_sts */
+	while (readl(wcnss->pmu_cfg) & WCNSS_PMU_IRIS_RESET_STS)
+		cpu_relax();
+
+	/* Clear IRIS reset */
+	val &= ~WCNSS_PMU_IRIS_RESET;
+	writel(val, wcnss->pmu_cfg);
+
+	/* Start IRIS XO configuration */
+	val |= WCNSS_PMU_IRIS_XO_CFG;
+	writel(val, wcnss->pmu_cfg);
+
+	/* Wait for XO configuration to finish */
+	while (readl(wcnss->pmu_cfg) & WCNSS_PMU_IRIS_XO_CFG_STS)
+		cpu_relax();
+
+	/* Stop IRIS XO configuration */
+	val &= ~WCNSS_PMU_GC_BUS_MUX_SEL_TOP;
+	val &= ~WCNSS_PMU_IRIS_XO_CFG;
+	writel(val, wcnss->pmu_cfg);
+
+	/* Add some delay for XO to settle */
+	msleep(20);
+}
+
+static int wcnss_start(struct rproc *rproc)
+{
+	struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
+	int ret;
+
+	mutex_lock(&wcnss->iris_lock);
+	if (!wcnss->iris) {
+		dev_err(wcnss->dev, "no iris registered\n");
+		ret = -EINVAL;
+		goto release_iris_lock;
+	}
+
+	ret = regulator_bulk_enable(wcnss->num_vregs, wcnss->vregs);
+	if (ret)
+		goto release_iris_lock;
+
+	ret = qcom_iris_enable(wcnss->iris);
+	if (ret)
+		goto disable_regulators;
+
+	wcnss_indicate_nv_download(wcnss);
+	wcnss_configure_iris(wcnss);
+
+	ret = qcom_scm_pas_auth_and_reset(WCNSS_PAS_ID);
+	if (ret) {
+		dev_err(wcnss->dev,
+			"failed to authenticate image and release reset\n");
+		goto disable_iris;
+	}
+
+	ret = wait_for_completion_timeout(&wcnss->start_done,
+					  msecs_to_jiffies(5000));
+	if (wcnss->ready_irq > 0 && ret == 0) {
+		/* We have a ready_irq, but it didn't fire in time. */
+		dev_err(wcnss->dev, "start timed out\n");
+		qcom_scm_pas_shutdown(WCNSS_PAS_ID);
+		ret = -ETIMEDOUT;
+		goto disable_iris;
+	}
+
+	ret = 0;
+
+disable_iris:
+	qcom_iris_disable(wcnss->iris);
+disable_regulators:
+	regulator_bulk_disable(wcnss->num_vregs, wcnss->vregs);
+release_iris_lock:
+	mutex_unlock(&wcnss->iris_lock);
+
+	return ret;
+}
+
+static int wcnss_stop(struct rproc *rproc)
+{
+	struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
+	int ret;
+
+	if (wcnss->state) {
+		qcom_smem_state_update_bits(wcnss->state,
+					    BIT(wcnss->stop_bit),
+					    BIT(wcnss->stop_bit));
+
+		ret = wait_for_completion_timeout(&wcnss->stop_done,
+						  msecs_to_jiffies(5000));
+		if (ret == 0)
+			dev_err(wcnss->dev, "timed out on wait\n");
+
+		qcom_smem_state_update_bits(wcnss->state,
+					    BIT(wcnss->stop_bit),
+					    0);
+	}
+
+	ret = qcom_scm_pas_shutdown(WCNSS_PAS_ID);
+	if (ret)
+		dev_err(wcnss->dev, "failed to shutdown: %d\n", ret);
+
+	return ret;
+}
+
+static void *wcnss_da_to_va(struct rproc *rproc, u64 da, int len)
+{
+	struct qcom_wcnss *wcnss = (struct qcom_wcnss *)rproc->priv;
+	int offset;
+
+	offset = da - wcnss->mem_reloc;
+	if (offset < 0 || offset + len > wcnss->mem_size)
+		return NULL;
+
+	return wcnss->mem_region + offset;
+}
+
+static const struct rproc_ops wcnss_ops = {
+	.start = wcnss_start,
+	.stop = wcnss_stop,
+	.da_to_va = wcnss_da_to_va,
+};
+
+static irqreturn_t wcnss_wdog_interrupt(int irq, void *dev)
+{
+	struct qcom_wcnss *wcnss = dev;
+
+	rproc_report_crash(wcnss->rproc, RPROC_WATCHDOG);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t wcnss_fatal_interrupt(int irq, void *dev)
+{
+	struct qcom_wcnss *wcnss = dev;
+	size_t len;
+	char *msg;
+
+	msg = qcom_smem_get(QCOM_SMEM_HOST_ANY, WCNSS_CRASH_REASON_SMEM, &len);
+	if (!IS_ERR(msg) && len > 0 && msg[0])
+		dev_err(wcnss->dev, "fatal error received: %s\n", msg);
+
+	rproc_report_crash(wcnss->rproc, RPROC_FATAL_ERROR);
+
+	if (!IS_ERR(msg))
+		msg[0] = '\0';
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t wcnss_ready_interrupt(int irq, void *dev)
+{
+	struct qcom_wcnss *wcnss = dev;
+
+	complete(&wcnss->start_done);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t wcnss_handover_interrupt(int irq, void *dev)
+{
+	/*
+	 * XXX: At this point we're supposed to release the resources that we
+	 * have been holding on behalf of the WCNSS. Unfortunately this
+	 * interrupt comes way before the other side seems to be done.
+	 *
+	 * So we're currently relying on the ready interrupt firing later then
+	 * this and we just disable the resources at the end of wcnss_start().
+	 */
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t wcnss_stop_ack_interrupt(int irq, void *dev)
+{
+	struct qcom_wcnss *wcnss = dev;
+
+	complete(&wcnss->stop_done);
+
+	return IRQ_HANDLED;
+}
+
+static int wcnss_init_regulators(struct qcom_wcnss *wcnss,
+				 const struct wcnss_vreg_info *info,
+				 int num_vregs)
+{
+	struct regulator_bulk_data *bulk;
+	int ret;
+	int i;
+
+	bulk = devm_kcalloc(wcnss->dev,
+			    num_vregs, sizeof(struct regulator_bulk_data),
+			    GFP_KERNEL);
+	if (!bulk)
+		return -ENOMEM;
+
+	for (i = 0; i < num_vregs; i++)
+		bulk[i].supply = info[i].name;
+
+	ret = devm_regulator_bulk_get(wcnss->dev, num_vregs, bulk);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < num_vregs; i++) {
+		if (info[i].max_voltage)
+			regulator_set_voltage(bulk[i].consumer,
+					      info[i].min_voltage,
+					      info[i].max_voltage);
+
+		if (info[i].load_uA)
+			regulator_set_load(bulk[i].consumer, info[i].load_uA);
+	}
+
+	wcnss->vregs = bulk;
+	wcnss->num_vregs = num_vregs;
+
+	return 0;
+}
+
+static int wcnss_request_irq(struct qcom_wcnss *wcnss,
+			     struct platform_device *pdev,
+			     const char *name,
+			     bool optional,
+			     irq_handler_t thread_fn)
+{
+	int ret;
+
+	ret = platform_get_irq_byname(pdev, name);
+	if (ret < 0 && optional) {
+		dev_dbg(&pdev->dev, "no %s IRQ defined, ignoring\n", name);
+		return 0;
+	} else if (ret < 0) {
+		dev_err(&pdev->dev, "no %s IRQ defined\n", name);
+		return ret;
+	}
+
+	ret = devm_request_threaded_irq(&pdev->dev, ret,
+					NULL, thread_fn,
+					IRQF_TRIGGER_RISING | IRQF_ONESHOT,
+					"wcnss", wcnss);
+	if (ret)
+		dev_err(&pdev->dev, "request %s IRQ failed\n", name);
+
+	return ret;
+}
+
+static int wcnss_alloc_memory_region(struct qcom_wcnss *wcnss)
+{
+	struct device_node *node;
+	struct resource r;
+	int ret;
+
+	node = of_parse_phandle(wcnss->dev->of_node, "memory-region", 0);
+	if (!node) {
+		dev_err(wcnss->dev, "no memory-region specified\n");
+		return -EINVAL;
+	}
+
+	ret = of_address_to_resource(node, 0, &r);
+	if (ret)
+		return ret;
+
+	wcnss->mem_phys = wcnss->mem_reloc = r.start;
+	wcnss->mem_size = resource_size(&r);
+	wcnss->mem_region = devm_ioremap_wc(wcnss->dev, wcnss->mem_phys, wcnss->mem_size);
+	if (!wcnss->mem_region) {
+		dev_err(wcnss->dev, "unable to map memory region: %pa+%zx\n",
+			&r.start, wcnss->mem_size);
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+static int wcnss_probe(struct platform_device *pdev)
+{
+	const struct wcnss_data *data;
+	struct qcom_wcnss *wcnss;
+	struct resource *res;
+	struct rproc *rproc;
+	void __iomem *mmio;
+	int ret;
+
+	data = of_device_get_match_data(&pdev->dev);
+
+	if (!qcom_scm_is_available())
+		return -EPROBE_DEFER;
+
+	if (!qcom_scm_pas_supported(WCNSS_PAS_ID)) {
+		dev_err(&pdev->dev, "PAS is not available for WCNSS\n");
+		return -ENXIO;
+	}
+
+	rproc = rproc_alloc(&pdev->dev, pdev->name, &wcnss_ops,
+			    WCNSS_FIRMWARE_NAME, sizeof(*wcnss));
+	if (!rproc) {
+		dev_err(&pdev->dev, "unable to allocate remoteproc\n");
+		return -ENOMEM;
+	}
+
+	rproc->fw_ops = &wcnss_fw_ops;
+
+	wcnss = (struct qcom_wcnss *)rproc->priv;
+	wcnss->dev = &pdev->dev;
+	wcnss->rproc = rproc;
+	platform_set_drvdata(pdev, wcnss);
+
+	init_completion(&wcnss->start_done);
+	init_completion(&wcnss->stop_done);
+
+	mutex_init(&wcnss->iris_lock);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pmu");
+	mmio = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mmio)) {
+		ret = PTR_ERR(mmio);
+		goto free_rproc;
+	};
+
+	ret = wcnss_alloc_memory_region(wcnss);
+	if (ret)
+		goto free_rproc;
+
+	wcnss->pmu_cfg = mmio + data->pmu_offset;
+	wcnss->spare_out = mmio + data->spare_offset;
+
+	ret = wcnss_init_regulators(wcnss, data->vregs, data->num_vregs);
+	if (ret)
+		goto free_rproc;
+
+	ret = wcnss_request_irq(wcnss, pdev, "wdog", false, wcnss_wdog_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	wcnss->wdog_irq = ret;
+
+	ret = wcnss_request_irq(wcnss, pdev, "fatal", false, wcnss_fatal_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	wcnss->fatal_irq = ret;
+
+	ret = wcnss_request_irq(wcnss, pdev, "ready", true, wcnss_ready_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	wcnss->ready_irq = ret;
+
+	ret = wcnss_request_irq(wcnss, pdev, "handover", true, wcnss_handover_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	wcnss->handover_irq = ret;
+
+	ret = wcnss_request_irq(wcnss, pdev, "stop-ack", true, wcnss_stop_ack_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	wcnss->stop_ack_irq = ret;
+
+	if (wcnss->stop_ack_irq) {
+		wcnss->state = qcom_smem_state_get(&pdev->dev, "stop",
+						   &wcnss->stop_bit);
+		if (IS_ERR(wcnss->state)) {
+			ret = PTR_ERR(wcnss->state);
+			goto free_rproc;
+		}
+	}
+
+	ret = rproc_add(rproc);
+	if (ret)
+		goto free_rproc;
+
+	return of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
+
+free_rproc:
+	rproc_free(rproc);
+
+	return ret;
+}
+
+static int wcnss_remove(struct platform_device *pdev)
+{
+	struct qcom_wcnss *wcnss = platform_get_drvdata(pdev);
+
+	of_platform_depopulate(&pdev->dev);
+
+	qcom_smem_state_put(wcnss->state);
+	rproc_del(wcnss->rproc);
+	rproc_free(wcnss->rproc);
+
+	return 0;
+}
+
+static const struct of_device_id wcnss_of_match[] = {
+	{ .compatible = "qcom,riva-pil", &riva_data },
+	{ .compatible = "qcom,pronto-v1-pil", &pronto_v1_data },
+	{ .compatible = "qcom,pronto-v2-pil", &pronto_v2_data },
+	{ },
+};
+
+static struct platform_driver wcnss_driver = {
+	.probe = wcnss_probe,
+	.remove = wcnss_remove,
+	.driver = {
+		.name = "qcom-wcnss-pil",
+		.of_match_table = wcnss_of_match,
+	},
+};
+
+module_platform_driver(wcnss_driver);
+MODULE_DESCRIPTION("Qualcomm Peripherial Image Loader for Wireless Subsystem");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/remoteproc/qcom_wcnss.h b/drivers/remoteproc/qcom_wcnss.h
new file mode 100644
index 000000000000..9dc4a9fe41e1
--- /dev/null
+++ b/drivers/remoteproc/qcom_wcnss.h
@@ -0,0 +1,22 @@
+#ifndef __QCOM_WNCSS_H__
+#define __QCOM_WNCSS_H__
+
+struct qcom_iris;
+struct qcom_wcnss;
+
+struct wcnss_vreg_info {
+	const char * const name;
+	int min_voltage;
+	int max_voltage;
+
+	int load_uA;
+
+	bool super_turbo;
+};
+
+int qcom_iris_enable(struct qcom_iris *iris);
+void qcom_iris_disable(struct qcom_iris *iris);
+
+void qcom_wcnss_assign_iris(struct qcom_wcnss *wcnss, struct qcom_iris *iris, bool use_48mhz_xo);
+
+#endif
diff --git a/drivers/remoteproc/qcom_wcnss_iris.c b/drivers/remoteproc/qcom_wcnss_iris.c
new file mode 100644
index 000000000000..f0ca24a8dd0b
--- /dev/null
+++ b/drivers/remoteproc/qcom_wcnss_iris.c
@@ -0,0 +1,188 @@
+/*
+ * Qualcomm Wireless Connectivity Subsystem Iris driver
+ *
+ * Copyright (C) 2016 Linaro Ltd
+ * Copyright (C) 2014 Sony Mobile Communications AB
+ * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+
+#include "qcom_wcnss.h"
+
+struct qcom_iris {
+	struct device *dev;
+
+	struct clk *xo_clk;
+
+	struct regulator_bulk_data *vregs;
+	size_t num_vregs;
+};
+
+struct iris_data {
+	const struct wcnss_vreg_info *vregs;
+	size_t num_vregs;
+
+	bool use_48mhz_xo;
+};
+
+static const struct iris_data wcn3620_data = {
+	.vregs = (struct wcnss_vreg_info[]) {
+		{ "vddxo",  1800000, 1800000, 10000 },
+		{ "vddrfa", 1300000, 1300000, 100000 },
+		{ "vddpa",  3300000, 3300000, 515000 },
+		{ "vdddig", 1800000, 1800000, 10000 },
+	},
+	.num_vregs = 4,
+	.use_48mhz_xo = false,
+};
+
+static const struct iris_data wcn3660_data = {
+	.vregs = (struct wcnss_vreg_info[]) {
+		{ "vddxo",  1800000, 1800000, 10000 },
+		{ "vddrfa", 1300000, 1300000, 100000 },
+		{ "vddpa",  2900000, 3000000, 515000 },
+		{ "vdddig", 1200000, 1225000, 10000 },
+	},
+	.num_vregs = 4,
+	.use_48mhz_xo = true,
+};
+
+static const struct iris_data wcn3680_data = {
+	.vregs = (struct wcnss_vreg_info[]) {
+		{ "vddxo",  1800000, 1800000, 10000 },
+		{ "vddrfa", 1300000, 1300000, 100000 },
+		{ "vddpa",  3300000, 3300000, 515000 },
+		{ "vdddig", 1800000, 1800000, 10000 },
+	},
+	.num_vregs = 4,
+	.use_48mhz_xo = true,
+};
+
+int qcom_iris_enable(struct qcom_iris *iris)
+{
+	int ret;
+
+	ret = regulator_bulk_enable(iris->num_vregs, iris->vregs);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(iris->xo_clk);
+	if (ret) {
+		dev_err(iris->dev, "failed to enable xo clk\n");
+		goto disable_regulators;
+	}
+
+	return 0;
+
+disable_regulators:
+	regulator_bulk_disable(iris->num_vregs, iris->vregs);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(qcom_iris_enable);
+
+void qcom_iris_disable(struct qcom_iris *iris)
+{
+	clk_disable_unprepare(iris->xo_clk);
+	regulator_bulk_disable(iris->num_vregs, iris->vregs);
+}
+EXPORT_SYMBOL_GPL(qcom_iris_disable);
+
+static int qcom_iris_probe(struct platform_device *pdev)
+{
+	const struct iris_data *data;
+	struct qcom_wcnss *wcnss;
+	struct qcom_iris *iris;
+	int ret;
+	int i;
+
+	iris = devm_kzalloc(&pdev->dev, sizeof(struct qcom_iris), GFP_KERNEL);
+	if (!iris)
+		return -ENOMEM;
+
+	data = of_device_get_match_data(&pdev->dev);
+	wcnss = dev_get_drvdata(pdev->dev.parent);
+
+	iris->xo_clk = devm_clk_get(&pdev->dev, "xo");
+	if (IS_ERR(iris->xo_clk)) {
+		if (PTR_ERR(iris->xo_clk) != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to acquire xo clk\n");
+		return PTR_ERR(iris->xo_clk);
+	}
+
+	iris->num_vregs = data->num_vregs;
+	iris->vregs = devm_kcalloc(&pdev->dev,
+				   iris->num_vregs,
+				   sizeof(struct regulator_bulk_data),
+				   GFP_KERNEL);
+	if (!iris->vregs)
+		return -ENOMEM;
+
+	for (i = 0; i < iris->num_vregs; i++)
+		iris->vregs[i].supply = data->vregs[i].name;
+
+	ret = devm_regulator_bulk_get(&pdev->dev, iris->num_vregs, iris->vregs);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to get regulators\n");
+		return ret;
+	}
+
+	for (i = 0; i < iris->num_vregs; i++) {
+		if (data->vregs[i].max_voltage)
+			regulator_set_voltage(iris->vregs[i].consumer,
+					      data->vregs[i].min_voltage,
+					      data->vregs[i].max_voltage);
+
+		if (data->vregs[i].load_uA)
+			regulator_set_load(iris->vregs[i].consumer,
+					   data->vregs[i].load_uA);
+	}
+
+	qcom_wcnss_assign_iris(wcnss, iris, data->use_48mhz_xo);
+
+	return 0;
+}
+
+static int qcom_iris_remove(struct platform_device *pdev)
+{
+	struct qcom_wcnss *wcnss = dev_get_drvdata(pdev->dev.parent);
+
+	qcom_wcnss_assign_iris(wcnss, NULL, false);
+
+	return 0;
+}
+
+static const struct of_device_id iris_of_match[] = {
+	{ .compatible = "qcom,wcn3620", .data = &wcn3620_data },
+	{ .compatible = "qcom,wcn3660", .data = &wcn3660_data },
+	{ .compatible = "qcom,wcn3680", .data = &wcn3680_data },
+	{}
+};
+
+static struct platform_driver wcnss_driver = {
+	.probe = qcom_iris_probe,
+	.remove = qcom_iris_remove,
+	.driver = {
+		.name = "qcom-iris",
+		.of_match_table = iris_of_match,
+	},
+};
+
+module_platform_driver(wcnss_driver);
+MODULE_DESCRIPTION("Qualcomm Wireless Subsystem Iris driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
index fe0539ed9cb5..c6bfb3496684 100644
--- a/drivers/remoteproc/remoteproc_core.c
+++ b/drivers/remoteproc/remoteproc_core.c
@@ -78,7 +78,7 @@ static const char *rproc_crash_to_string(enum rproc_crash_type type)
  * will try to access an unmapped device address.
  */
 static int rproc_iommu_fault(struct iommu_domain *domain, struct device *dev,
-		unsigned long iova, int flags, void *token)
+			     unsigned long iova, int flags, void *token)
 {
 	struct rproc *rproc = token;
 
@@ -236,8 +236,8 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i)
 	}
 	notifyid = ret;
 
-	dev_dbg(dev, "vring%d: va %p dma %llx size %x idr %d\n", i, va,
-				(unsigned long long)dma, size, notifyid);
+	dev_dbg(dev, "vring%d: va %p dma %pad size 0x%x idr %d\n",
+		i, va, &dma, size, notifyid);
 
 	rvring->va = va;
 	rvring->dma = dma;
@@ -263,19 +263,13 @@ rproc_parse_vring(struct rproc_vdev *rvdev, struct fw_rsc_vdev *rsc, int i)
 	struct fw_rsc_vdev_vring *vring = &rsc->vring[i];
 	struct rproc_vring *rvring = &rvdev->vring[i];
 
-	dev_dbg(dev, "vdev rsc: vring%d: da %x, qsz %d, align %d\n",
-				i, vring->da, vring->num, vring->align);
-
-	/* make sure reserved bytes are zeroes */
-	if (vring->reserved) {
-		dev_err(dev, "vring rsc has non zero reserved bytes\n");
-		return -EINVAL;
-	}
+	dev_dbg(dev, "vdev rsc: vring%d: da 0x%x, qsz %d, align %d\n",
+		i, vring->da, vring->num, vring->align);
 
 	/* verify queue size and vring alignment are sane */
 	if (!vring->num || !vring->align) {
 		dev_err(dev, "invalid qsz (%d) or alignment (%d)\n",
-						vring->num, vring->align);
+			vring->num, vring->align);
 		return -EINVAL;
 	}
 
@@ -330,7 +324,7 @@ void rproc_free_vring(struct rproc_vring *rvring)
  * Returns 0 on success, or an appropriate error code otherwise
  */
 static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc,
-							int offset, int avail)
+			     int offset, int avail)
 {
 	struct device *dev = &rproc->dev;
 	struct rproc_vdev *rvdev;
@@ -349,7 +343,7 @@ static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc,
 		return -EINVAL;
 	}
 
-	dev_dbg(dev, "vdev rsc: id %d, dfeatures %x, cfg len %d, %d vrings\n",
+	dev_dbg(dev, "vdev rsc: id %d, dfeatures 0x%x, cfg len %d, %d vrings\n",
 		rsc->id, rsc->dfeatures, rsc->config_len, rsc->num_of_vrings);
 
 	/* we currently support only two vrings per rvdev */
@@ -358,7 +352,7 @@ static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc,
 		return -EINVAL;
 	}
 
-	rvdev = kzalloc(sizeof(struct rproc_vdev), GFP_KERNEL);
+	rvdev = kzalloc(sizeof(*rvdev), GFP_KERNEL);
 	if (!rvdev)
 		return -ENOMEM;
 
@@ -407,7 +401,7 @@ free_rvdev:
  * Returns 0 on success, or an appropriate error code otherwise
  */
 static int rproc_handle_trace(struct rproc *rproc, struct fw_rsc_trace *rsc,
-							int offset, int avail)
+			      int offset, int avail)
 {
 	struct rproc_mem_entry *trace;
 	struct device *dev = &rproc->dev;
@@ -455,8 +449,8 @@ static int rproc_handle_trace(struct rproc *rproc, struct fw_rsc_trace *rsc,
 
 	rproc->num_traces++;
 
-	dev_dbg(dev, "%s added: va %p, da 0x%x, len 0x%x\n", name, ptr,
-						rsc->da, rsc->len);
+	dev_dbg(dev, "%s added: va %p, da 0x%x, len 0x%x\n",
+		name, ptr, rsc->da, rsc->len);
 
 	return 0;
 }
@@ -487,7 +481,7 @@ static int rproc_handle_trace(struct rproc *rproc, struct fw_rsc_trace *rsc,
  * are outside those ranges.
  */
 static int rproc_handle_devmem(struct rproc *rproc, struct fw_rsc_devmem *rsc,
-							int offset, int avail)
+			       int offset, int avail)
 {
 	struct rproc_mem_entry *mapping;
 	struct device *dev = &rproc->dev;
@@ -530,7 +524,7 @@ static int rproc_handle_devmem(struct rproc *rproc, struct fw_rsc_devmem *rsc,
 	list_add_tail(&mapping->node, &rproc->mappings);
 
 	dev_dbg(dev, "mapped devmem pa 0x%x, da 0x%x, len 0x%x\n",
-					rsc->pa, rsc->da, rsc->len);
+		rsc->pa, rsc->da, rsc->len);
 
 	return 0;
 
@@ -558,9 +552,8 @@ out:
  * pressure is important; it may have a substantial impact on performance.
  */
 static int rproc_handle_carveout(struct rproc *rproc,
-						struct fw_rsc_carveout *rsc,
-						int offset, int avail)
-
+				 struct fw_rsc_carveout *rsc,
+				 int offset, int avail)
 {
 	struct rproc_mem_entry *carveout, *mapping;
 	struct device *dev = &rproc->dev;
@@ -579,8 +572,8 @@ static int rproc_handle_carveout(struct rproc *rproc,
 		return -EINVAL;
 	}
 
-	dev_dbg(dev, "carveout rsc: da %x, pa %x, len %x, flags %x\n",
-			rsc->da, rsc->pa, rsc->len, rsc->flags);
+	dev_dbg(dev, "carveout rsc: name: %s, da 0x%x, pa 0x%x, len 0x%x, flags 0x%x\n",
+		rsc->name, rsc->da, rsc->pa, rsc->len, rsc->flags);
 
 	carveout = kzalloc(sizeof(*carveout), GFP_KERNEL);
 	if (!carveout)
@@ -588,13 +581,14 @@ static int rproc_handle_carveout(struct rproc *rproc,
 
 	va = dma_alloc_coherent(dev->parent, rsc->len, &dma, GFP_KERNEL);
 	if (!va) {
-		dev_err(dev->parent, "dma_alloc_coherent err: %d\n", rsc->len);
+		dev_err(dev->parent,
+			"failed to allocate dma memory: len 0x%x\n", rsc->len);
 		ret = -ENOMEM;
 		goto free_carv;
 	}
 
-	dev_dbg(dev, "carveout va %p, dma %llx, len 0x%x\n", va,
-					(unsigned long long)dma, rsc->len);
+	dev_dbg(dev, "carveout va %p, dma %pad, len 0x%x\n",
+		va, &dma, rsc->len);
 
 	/*
 	 * Ok, this is non-standard.
@@ -616,13 +610,12 @@ static int rproc_handle_carveout(struct rproc *rproc,
 	if (rproc->domain) {
 		mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
 		if (!mapping) {
-			dev_err(dev, "kzalloc mapping failed\n");
 			ret = -ENOMEM;
 			goto dma_free;
 		}
 
 		ret = iommu_map(rproc->domain, rsc->da, dma, rsc->len,
-								rsc->flags);
+				rsc->flags);
 		if (ret) {
 			dev_err(dev, "iommu_map failed: %d\n", ret);
 			goto free_mapping;
@@ -639,8 +632,8 @@ static int rproc_handle_carveout(struct rproc *rproc,
 		mapping->len = rsc->len;
 		list_add_tail(&mapping->node, &rproc->mappings);
 
-		dev_dbg(dev, "carveout mapped 0x%x to 0x%llx\n",
-					rsc->da, (unsigned long long)dma);
+		dev_dbg(dev, "carveout mapped 0x%x to %pad\n",
+			rsc->da, &dma);
 	}
 
 	/*
@@ -697,17 +690,13 @@ static rproc_handle_resource_t rproc_loading_handlers[RSC_LAST] = {
 	[RSC_CARVEOUT] = (rproc_handle_resource_t)rproc_handle_carveout,
 	[RSC_DEVMEM] = (rproc_handle_resource_t)rproc_handle_devmem,
 	[RSC_TRACE] = (rproc_handle_resource_t)rproc_handle_trace,
-	[RSC_VDEV] = NULL, /* VDEVs were handled upon registrarion */
+	[RSC_VDEV] = (rproc_handle_resource_t)rproc_count_vrings,
 };
 
 static rproc_handle_resource_t rproc_vdev_handler[RSC_LAST] = {
 	[RSC_VDEV] = (rproc_handle_resource_t)rproc_handle_vdev,
 };
 
-static rproc_handle_resource_t rproc_count_vrings_handler[RSC_LAST] = {
-	[RSC_VDEV] = (rproc_handle_resource_t)rproc_count_vrings,
-};
-
 /* handle firmware resource entries before booting the remote processor */
 static int rproc_handle_resources(struct rproc *rproc, int len,
 				  rproc_handle_resource_t handlers[RSC_LAST])
@@ -757,6 +746,7 @@ static int rproc_handle_resources(struct rproc *rproc, int len,
 static void rproc_resource_cleanup(struct rproc *rproc)
 {
 	struct rproc_mem_entry *entry, *tmp;
+	struct rproc_vdev *rvdev, *rvtmp;
 	struct device *dev = &rproc->dev;
 
 	/* clean up debugfs trace entries */
@@ -775,7 +765,7 @@ static void rproc_resource_cleanup(struct rproc *rproc)
 		if (unmapped != entry->len) {
 			/* nothing much to do besides complaining */
 			dev_err(dev, "failed to unmap %u/%zu\n", entry->len,
-								unmapped);
+				unmapped);
 		}
 
 		list_del(&entry->node);
@@ -789,6 +779,10 @@ static void rproc_resource_cleanup(struct rproc *rproc)
 		list_del(&entry->node);
 		kfree(entry);
 	}
+
+	/* clean up remote vdev entries */
+	list_for_each_entry_safe(rvdev, rvtmp, &rproc->rvdevs, node)
+		rproc_remove_virtio_dev(rvdev);
 }
 
 /*
@@ -801,9 +795,6 @@ static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 	struct resource_table *table, *loaded_table;
 	int ret, tablesz;
 
-	if (!rproc->table_ptr)
-		return -ENOMEM;
-
 	ret = rproc_fw_sanity_check(rproc, fw);
 	if (ret)
 		return ret;
@@ -830,9 +821,25 @@ static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 		goto clean_up;
 	}
 
-	/* Verify that resource table in loaded fw is unchanged */
-	if (rproc->table_csum != crc32(0, table, tablesz)) {
-		dev_err(dev, "resource checksum failed, fw changed?\n");
+	/*
+	 * Create a copy of the resource table. When a virtio device starts
+	 * and calls vring_new_virtqueue() the address of the allocated vring
+	 * will be stored in the cached_table. Before the device is started,
+	 * cached_table will be copied into device memory.
+	 */
+	rproc->cached_table = kmemdup(table, tablesz, GFP_KERNEL);
+	if (!rproc->cached_table)
+		goto clean_up;
+
+	rproc->table_ptr = rproc->cached_table;
+
+	/* reset max_notifyid */
+	rproc->max_notifyid = -1;
+
+	/* look for virtio devices and register them */
+	ret = rproc_handle_resources(rproc, tablesz, rproc_vdev_handler);
+	if (ret) {
+		dev_err(dev, "Failed to handle vdev resources: %d\n", ret);
 		goto clean_up;
 	}
 
@@ -840,49 +847,50 @@ static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 	ret = rproc_handle_resources(rproc, tablesz, rproc_loading_handlers);
 	if (ret) {
 		dev_err(dev, "Failed to process resources: %d\n", ret);
-		goto clean_up;
+		goto clean_up_resources;
 	}
 
 	/* load the ELF segments to memory */
 	ret = rproc_load_segments(rproc, fw);
 	if (ret) {
 		dev_err(dev, "Failed to load program segments: %d\n", ret);
-		goto clean_up;
+		goto clean_up_resources;
 	}
 
 	/*
 	 * The starting device has been given the rproc->cached_table as the
 	 * resource table. The address of the vring along with the other
 	 * allocated resources (carveouts etc) is stored in cached_table.
-	 * In order to pass this information to the remote device we must
-	 * copy this information to device memory.
+	 * In order to pass this information to the remote device we must copy
+	 * this information to device memory. We also update the table_ptr so
+	 * that any subsequent changes will be applied to the loaded version.
 	 */
 	loaded_table = rproc_find_loaded_rsc_table(rproc, fw);
-	if (loaded_table)
+	if (loaded_table) {
 		memcpy(loaded_table, rproc->cached_table, tablesz);
+		rproc->table_ptr = loaded_table;
+	}
 
 	/* power up the remote processor */
 	ret = rproc->ops->start(rproc);
 	if (ret) {
 		dev_err(dev, "can't start rproc %s: %d\n", rproc->name, ret);
-		goto clean_up;
+		goto clean_up_resources;
 	}
 
-	/*
-	 * Update table_ptr so that all subsequent vring allocations and
-	 * virtio fields manipulation update the actual loaded resource table
-	 * in device memory.
-	 */
-	rproc->table_ptr = loaded_table;
-
 	rproc->state = RPROC_RUNNING;
 
 	dev_info(dev, "remote processor %s is now up\n", rproc->name);
 
 	return 0;
 
-clean_up:
+clean_up_resources:
 	rproc_resource_cleanup(rproc);
+clean_up:
+	kfree(rproc->cached_table);
+	rproc->cached_table = NULL;
+	rproc->table_ptr = NULL;
+
 	rproc_disable_iommu(rproc);
 	return ret;
 }
@@ -898,42 +906,11 @@ clean_up:
 static void rproc_fw_config_virtio(const struct firmware *fw, void *context)
 {
 	struct rproc *rproc = context;
-	struct resource_table *table;
-	int ret, tablesz;
-
-	if (rproc_fw_sanity_check(rproc, fw) < 0)
-		goto out;
-
-	/* look for the resource table */
-	table = rproc_find_rsc_table(rproc, fw,  &tablesz);
-	if (!table)
-		goto out;
-
-	rproc->table_csum = crc32(0, table, tablesz);
-
-	/*
-	 * Create a copy of the resource table. When a virtio device starts
-	 * and calls vring_new_virtqueue() the address of the allocated vring
-	 * will be stored in the cached_table. Before the device is started,
-	 * cached_table will be copied into devic memory.
-	 */
-	rproc->cached_table = kmemdup(table, tablesz, GFP_KERNEL);
-	if (!rproc->cached_table)
-		goto out;
-
-	rproc->table_ptr = rproc->cached_table;
 
-	/* count the number of notify-ids */
-	rproc->max_notifyid = -1;
-	ret = rproc_handle_resources(rproc, tablesz,
-				     rproc_count_vrings_handler);
-	if (ret)
-		goto out;
-
-	/* look for virtio devices and register them */
-	ret = rproc_handle_resources(rproc, tablesz, rproc_vdev_handler);
+	/* if rproc is marked always-on, request it to boot */
+	if (rproc->auto_boot)
+		rproc_boot_nowait(rproc);
 
-out:
 	release_firmware(fw);
 	/* allow rproc_del() contexts, if any, to proceed */
 	complete_all(&rproc->firmware_loading_complete);
@@ -969,7 +946,7 @@ static int rproc_add_virtio_devices(struct rproc *rproc)
  * rproc_trigger_recovery() - recover a remoteproc
  * @rproc: the remote processor
  *
- * The recovery is done by reseting all the virtio devices, that way all the
+ * The recovery is done by resetting all the virtio devices, that way all the
  * rpmsg drivers will be reseted along with the remote processor making the
  * remoteproc functional again.
  *
@@ -977,23 +954,23 @@ static int rproc_add_virtio_devices(struct rproc *rproc)
  */
 int rproc_trigger_recovery(struct rproc *rproc)
 {
-	struct rproc_vdev *rvdev, *rvtmp;
-
 	dev_err(&rproc->dev, "recovering %s\n", rproc->name);
 
 	init_completion(&rproc->crash_comp);
 
-	/* clean up remote vdev entries */
-	list_for_each_entry_safe(rvdev, rvtmp, &rproc->rvdevs, node)
-		rproc_remove_virtio_dev(rvdev);
+	/* shut down the remote */
+	/* TODO: make sure this works with rproc->power > 1 */
+	rproc_shutdown(rproc);
 
 	/* wait until there is no more rproc users */
 	wait_for_completion(&rproc->crash_comp);
 
-	/* Free the copy of the resource table */
-	kfree(rproc->cached_table);
+	/*
+	 * boot the remote processor up again
+	 */
+	rproc_boot(rproc);
 
-	return rproc_add_virtio_devices(rproc);
+	return 0;
 }
 
 /**
@@ -1058,20 +1035,6 @@ static int __rproc_boot(struct rproc *rproc, bool wait)
 		return ret;
 	}
 
-	/* loading a firmware is required */
-	if (!rproc->firmware) {
-		dev_err(dev, "%s: no firmware to load\n", __func__);
-		ret = -EINVAL;
-		goto unlock_mutex;
-	}
-
-	/* prevent underlying implementation from being removed */
-	if (!try_module_get(dev->parent->driver->owner)) {
-		dev_err(dev, "%s: can't get owner\n", __func__);
-		ret = -EINVAL;
-		goto unlock_mutex;
-	}
-
 	/* skip the boot process if rproc is already powered up */
 	if (atomic_inc_return(&rproc->power) > 1) {
 		ret = 0;
@@ -1096,10 +1059,8 @@ static int __rproc_boot(struct rproc *rproc, bool wait)
 	release_firmware(firmware_p);
 
 downref_rproc:
-	if (ret) {
-		module_put(dev->parent->driver->owner);
+	if (ret)
 		atomic_dec(&rproc->power);
-	}
 unlock_mutex:
 	mutex_unlock(&rproc->lock);
 	return ret;
@@ -1173,8 +1134,10 @@ void rproc_shutdown(struct rproc *rproc)
 
 	rproc_disable_iommu(rproc);
 
-	/* Give the next start a clean resource table */
-	rproc->table_ptr = rproc->cached_table;
+	/* Free the copy of the resource table */
+	kfree(rproc->cached_table);
+	rproc->cached_table = NULL;
+	rproc->table_ptr = NULL;
 
 	/* if in crash state, unlock crash handler */
 	if (rproc->state == RPROC_CRASHED)
@@ -1186,8 +1149,6 @@ void rproc_shutdown(struct rproc *rproc)
 
 out:
 	mutex_unlock(&rproc->lock);
-	if (!ret)
-		module_put(dev->parent->driver->owner);
 }
 EXPORT_SYMBOL(rproc_shutdown);
 
@@ -1216,6 +1177,12 @@ struct rproc *rproc_get_by_phandle(phandle phandle)
 	mutex_lock(&rproc_list_mutex);
 	list_for_each_entry(r, &rproc_list, node) {
 		if (r->dev.parent && r->dev.parent->of_node == np) {
+			/* prevent underlying implementation from being removed */
+			if (!try_module_get(r->dev.parent->driver->owner)) {
+				dev_err(&r->dev, "can't get owner\n");
+				break;
+			}
+
 			rproc = r;
 			get_device(&rproc->dev);
 			break;
@@ -1335,11 +1302,11 @@ static struct device_type rproc_type = {
  * On success the new rproc is returned, and on failure, NULL.
  *
  * Note: _never_ directly deallocate @rproc, even if it was not registered
- * yet. Instead, when you need to unroll rproc_alloc(), use rproc_put().
+ * yet. Instead, when you need to unroll rproc_alloc(), use rproc_free().
  */
 struct rproc *rproc_alloc(struct device *dev, const char *name,
-				const struct rproc_ops *ops,
-				const char *firmware, int len)
+			  const struct rproc_ops *ops,
+			  const char *firmware, int len)
 {
 	struct rproc *rproc;
 	char *p, *template = "rproc-%s-fw";
@@ -1359,7 +1326,7 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 		 */
 		name_len = strlen(name) + strlen(template) - 2 + 1;
 
-	rproc = kzalloc(sizeof(struct rproc) + len + name_len, GFP_KERNEL);
+	rproc = kzalloc(sizeof(*rproc) + len + name_len, GFP_KERNEL);
 	if (!rproc)
 		return NULL;
 
@@ -1374,6 +1341,7 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 	rproc->name = name;
 	rproc->ops = ops;
 	rproc->priv = &rproc[1];
+	rproc->auto_boot = true;
 
 	device_initialize(&rproc->dev);
 	rproc->dev.parent = dev;
@@ -1413,7 +1381,22 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 EXPORT_SYMBOL(rproc_alloc);
 
 /**
- * rproc_put() - unroll rproc_alloc()
+ * rproc_free() - unroll rproc_alloc()
+ * @rproc: the remote processor handle
+ *
+ * This function decrements the rproc dev refcount.
+ *
+ * If no one holds any reference to rproc anymore, then its refcount would
+ * now drop to zero, and it would be freed.
+ */
+void rproc_free(struct rproc *rproc)
+{
+	put_device(&rproc->dev);
+}
+EXPORT_SYMBOL(rproc_free);
+
+/**
+ * rproc_put() - release rproc reference
  * @rproc: the remote processor handle
  *
  * This function decrements the rproc dev refcount.
@@ -1423,6 +1406,7 @@ EXPORT_SYMBOL(rproc_alloc);
  */
 void rproc_put(struct rproc *rproc)
 {
+	module_put(rproc->dev.parent->driver->owner);
 	put_device(&rproc->dev);
 }
 EXPORT_SYMBOL(rproc_put);
@@ -1438,7 +1422,7 @@ EXPORT_SYMBOL(rproc_put);
  *
  * After rproc_del() returns, @rproc isn't freed yet, because
  * of the outstanding reference created by rproc_alloc. To decrement that
- * one last refcount, one still needs to call rproc_put().
+ * one last refcount, one still needs to call rproc_free().
  *
  * Returns 0 on success and -EINVAL if @rproc isn't valid.
  */
@@ -1452,13 +1436,15 @@ int rproc_del(struct rproc *rproc)
 	/* if rproc is just being registered, wait */
 	wait_for_completion(&rproc->firmware_loading_complete);
 
+	/* if rproc is marked always-on, rproc_add() booted it */
+	/* TODO: make sure this works with rproc->power > 1 */
+	if (rproc->auto_boot)
+		rproc_shutdown(rproc);
+
 	/* clean up remote vdev entries */
 	list_for_each_entry_safe(rvdev, tmp, &rproc->rvdevs, node)
 		rproc_remove_virtio_dev(rvdev);
 
-	/* Free the copy of the resource table */
-	kfree(rproc->cached_table);
-
 	/* the rproc is downref'ed as soon as it's removed from the klist */
 	mutex_lock(&rproc_list_mutex);
 	list_del(&rproc->node);
diff --git a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c
index 74a120b6e206..374797206c79 100644
--- a/drivers/remoteproc/remoteproc_debugfs.c
+++ b/drivers/remoteproc/remoteproc_debugfs.c
@@ -45,7 +45,7 @@ static struct dentry *rproc_dbg;
  * as it provides very early tracing with little to no dependencies at all.
  */
 static ssize_t rproc_trace_read(struct file *filp, char __user *userbuf,
-						size_t count, loff_t *ppos)
+				size_t count, loff_t *ppos)
 {
 	struct rproc_mem_entry *trace = filp->private_data;
 	int len = strnlen(trace->va, trace->len);
@@ -73,7 +73,7 @@ static const char * const rproc_state_string[] = {
 
 /* expose the state of the remote processor via debugfs */
 static ssize_t rproc_state_read(struct file *filp, char __user *userbuf,
-						size_t count, loff_t *ppos)
+				size_t count, loff_t *ppos)
 {
 	struct rproc *rproc = filp->private_data;
 	unsigned int state;
@@ -83,7 +83,7 @@ static ssize_t rproc_state_read(struct file *filp, char __user *userbuf,
 	state = rproc->state > RPROC_LAST ? RPROC_LAST : rproc->state;
 
 	i = scnprintf(buf, 30, "%.28s (%d)\n", rproc_state_string[state],
-							rproc->state);
+		      rproc->state);
 
 	return simple_read_from_buffer(userbuf, count, ppos, buf, i);
 }
@@ -130,7 +130,7 @@ static const struct file_operations rproc_state_ops = {
 
 /* expose the name of the remote processor via debugfs */
 static ssize_t rproc_name_read(struct file *filp, char __user *userbuf,
-						size_t count, loff_t *ppos)
+			       size_t count, loff_t *ppos)
 {
 	struct rproc *rproc = filp->private_data;
 	/* need room for the name, a newline and a terminating null */
@@ -230,12 +230,12 @@ void rproc_remove_trace_file(struct dentry *tfile)
 }
 
 struct dentry *rproc_create_trace_file(const char *name, struct rproc *rproc,
-					struct rproc_mem_entry *trace)
+				       struct rproc_mem_entry *trace)
 {
 	struct dentry *tfile;
 
-	tfile = debugfs_create_file(name, 0400, rproc->dbg_dir,
-						trace, &trace_rproc_ops);
+	tfile = debugfs_create_file(name, 0400, rproc->dbg_dir, trace,
+				    &trace_rproc_ops);
 	if (!tfile) {
 		dev_err(&rproc->dev, "failed to create debugfs trace entry\n");
 		return NULL;
@@ -264,11 +264,11 @@ void rproc_create_debug_dir(struct rproc *rproc)
 		return;
 
 	debugfs_create_file("name", 0400, rproc->dbg_dir,
-					rproc, &rproc_name_ops);
+			    rproc, &rproc_name_ops);
 	debugfs_create_file("state", 0400, rproc->dbg_dir,
-					rproc, &rproc_state_ops);
+			    rproc, &rproc_state_ops);
 	debugfs_create_file("recovery", 0400, rproc->dbg_dir,
-					rproc, &rproc_recovery_ops);
+			    rproc, &rproc_recovery_ops);
 }
 
 void __init rproc_init_debugfs(void)
diff --git a/drivers/remoteproc/remoteproc_elf_loader.c b/drivers/remoteproc/remoteproc_elf_loader.c
index ce283a5b42a1..c523983a4aec 100644
--- a/drivers/remoteproc/remoteproc_elf_loader.c
+++ b/drivers/remoteproc/remoteproc_elf_loader.c
@@ -166,18 +166,18 @@ rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
 			continue;
 
 		dev_dbg(dev, "phdr: type %d da 0x%x memsz 0x%x filesz 0x%x\n",
-					phdr->p_type, da, memsz, filesz);
+			phdr->p_type, da, memsz, filesz);
 
 		if (filesz > memsz) {
 			dev_err(dev, "bad phdr filesz 0x%x memsz 0x%x\n",
-							filesz, memsz);
+				filesz, memsz);
 			ret = -EINVAL;
 			break;
 		}
 
 		if (offset + filesz > fw->size) {
 			dev_err(dev, "truncated fw: need 0x%x avail 0x%zx\n",
-					offset + filesz, fw->size);
+				offset + filesz, fw->size);
 			ret = -EINVAL;
 			break;
 		}
diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h
index 57e1de59bec8..4cf93ca2816e 100644
--- a/drivers/remoteproc/remoteproc_internal.h
+++ b/drivers/remoteproc/remoteproc_internal.h
@@ -36,10 +36,10 @@ struct rproc;
  */
 struct rproc_fw_ops {
 	struct resource_table *(*find_rsc_table)(struct rproc *rproc,
-						const struct firmware *fw,
-						int *tablesz);
-	struct resource_table *(*find_loaded_rsc_table)(struct rproc *rproc,
-						const struct firmware *fw);
+						 const struct firmware *fw,
+						 int *tablesz);
+	struct resource_table *(*find_loaded_rsc_table)(
+				struct rproc *rproc, const struct firmware *fw);
 	int (*load)(struct rproc *rproc, const struct firmware *fw);
 	int (*sanity_check)(struct rproc *rproc, const struct firmware *fw);
 	u32 (*get_boot_addr)(struct rproc *rproc, const struct firmware *fw);
@@ -57,7 +57,7 @@ void rproc_remove_virtio_dev(struct rproc_vdev *rvdev);
 /* from remoteproc_debugfs.c */
 void rproc_remove_trace_file(struct dentry *tfile);
 struct dentry *rproc_create_trace_file(const char *name, struct rproc *rproc,
-					struct rproc_mem_entry *trace);
+				       struct rproc_mem_entry *trace);
 void rproc_delete_debug_dir(struct rproc *rproc);
 void rproc_create_debug_dir(struct rproc *rproc);
 void rproc_init_debugfs(void);
@@ -98,7 +98,8 @@ int rproc_load_segments(struct rproc *rproc, const struct firmware *fw)
 
 static inline
 struct resource_table *rproc_find_rsc_table(struct rproc *rproc,
-				const struct firmware *fw, int *tablesz)
+					    const struct firmware *fw,
+					    int *tablesz)
 {
 	if (rproc->fw_ops->find_rsc_table)
 		return rproc->fw_ops->find_rsc_table(rproc, fw, tablesz);
@@ -108,7 +109,7 @@ struct resource_table *rproc_find_rsc_table(struct rproc *rproc,
 
 static inline
 struct resource_table *rproc_find_loaded_rsc_table(struct rproc *rproc,
-						const struct firmware *fw)
+						   const struct firmware *fw)
 {
 	if (rproc->fw_ops->find_loaded_rsc_table)
 		return rproc->fw_ops->find_loaded_rsc_table(rproc, fw);
diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
index cc91556313e1..01870a16d6d2 100644
--- a/drivers/remoteproc/remoteproc_virtio.c
+++ b/drivers/remoteproc/remoteproc_virtio.c
@@ -69,7 +69,7 @@ irqreturn_t rproc_vq_interrupt(struct rproc *rproc, int notifyid)
 EXPORT_SYMBOL(rproc_vq_interrupt);
 
 static struct virtqueue *rp_find_vq(struct virtio_device *vdev,
-				    unsigned id,
+				    unsigned int id,
 				    void (*callback)(struct virtqueue *vq),
 				    const char *name)
 {
@@ -101,14 +101,14 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev,
 	memset(addr, 0, size);
 
 	dev_dbg(dev, "vring%d: va %p qsz %d notifyid %d\n",
-					id, addr, len, rvring->notifyid);
+		id, addr, len, rvring->notifyid);
 
 	/*
 	 * Create the new vq, and tell virtio we're not interested in
 	 * the 'weak' smp barriers, since we're talking with a real device.
 	 */
 	vq = vring_new_virtqueue(id, len, rvring->align, vdev, false, addr,
-					rproc_virtio_notify, callback, name);
+				 rproc_virtio_notify, callback, name);
 	if (!vq) {
 		dev_err(dev, "vring_new_virtqueue %s failed\n", name);
 		rproc_free_vring(rvring);
@@ -136,20 +136,14 @@ static void __rproc_virtio_del_vqs(struct virtio_device *vdev)
 
 static void rproc_virtio_del_vqs(struct virtio_device *vdev)
 {
-	struct rproc *rproc = vdev_to_rproc(vdev);
-
-	/* power down the remote processor before deleting vqs */
-	rproc_shutdown(rproc);
-
 	__rproc_virtio_del_vqs(vdev);
 }
 
-static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs,
-		       struct virtqueue *vqs[],
-		       vq_callback_t *callbacks[],
-		       const char * const names[])
+static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
+				 struct virtqueue *vqs[],
+				 vq_callback_t *callbacks[],
+				 const char * const names[])
 {
-	struct rproc *rproc = vdev_to_rproc(vdev);
 	int i, ret;
 
 	for (i = 0; i < nvqs; ++i) {
@@ -160,13 +154,6 @@ static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 		}
 	}
 
-	/* now that the vqs are all set, boot the remote processor */
-	ret = rproc_boot_nowait(rproc);
-	if (ret) {
-		dev_err(&rproc->dev, "rproc_boot() failed %d\n", ret);
-		goto error;
-	}
-
 	return 0;
 
 error:
@@ -239,8 +226,8 @@ static int rproc_virtio_finalize_features(struct virtio_device *vdev)
 	return 0;
 }
 
-static void rproc_virtio_get(struct virtio_device *vdev, unsigned offset,
-							void *buf, unsigned len)
+static void rproc_virtio_get(struct virtio_device *vdev, unsigned int offset,
+			     void *buf, unsigned int len)
 {
 	struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
 	struct fw_rsc_vdev *rsc;
@@ -257,8 +244,8 @@ static void rproc_virtio_get(struct virtio_device *vdev, unsigned offset,
 	memcpy(buf, cfg + offset, len);
 }
 
-static void rproc_virtio_set(struct virtio_device *vdev, unsigned offset,
-		      const void *buf, unsigned len)
+static void rproc_virtio_set(struct virtio_device *vdev, unsigned int offset,
+			     const void *buf, unsigned int len)
 {
 	struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
 	struct fw_rsc_vdev *rsc;
diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c
index 6f056caa8a56..ae8963fcc8c8 100644
--- a/drivers/remoteproc/st_remoteproc.c
+++ b/drivers/remoteproc/st_remoteproc.c
@@ -262,7 +262,7 @@ static int st_rproc_probe(struct platform_device *pdev)
 	return 0;
 
 free_rproc:
-	rproc_put(rproc);
+	rproc_free(rproc);
 	return ret;
 }
 
@@ -277,7 +277,7 @@ static int st_rproc_remove(struct platform_device *pdev)
 
 	of_reserved_mem_device_release(&pdev->dev);
 
-	rproc_put(rproc);
+	rproc_free(rproc);
 
 	return 0;
 }
diff --git a/drivers/remoteproc/ste_modem_rproc.c b/drivers/remoteproc/ste_modem_rproc.c
index 53dc17bdd54e..03d69a9a3c5b 100644
--- a/drivers/remoteproc/ste_modem_rproc.c
+++ b/drivers/remoteproc/ste_modem_rproc.c
@@ -257,7 +257,7 @@ static int sproc_drv_remove(struct platform_device *pdev)
 	rproc_del(sproc->rproc);
 	dma_free_coherent(sproc->rproc->dev.parent, SPROC_FW_SIZE,
 			  sproc->fw_addr, sproc->fw_dma_addr);
-	rproc_put(sproc->rproc);
+	rproc_free(sproc->rproc);
 
 	mdev->drv_data = NULL;
 
@@ -325,7 +325,7 @@ free_mem:
 free_rproc:
 	/* Reset device data upon error */
 	mdev->drv_data = NULL;
-	rproc_put(rproc);
+	rproc_free(rproc);
 	return err;
 }
 
diff --git a/drivers/remoteproc/wkup_m3_rproc.c b/drivers/remoteproc/wkup_m3_rproc.c
index 02d271d101b4..18175d0331fd 100644
--- a/drivers/remoteproc/wkup_m3_rproc.c
+++ b/drivers/remoteproc/wkup_m3_rproc.c
@@ -167,6 +167,8 @@ static int wkup_m3_rproc_probe(struct platform_device *pdev)
 		goto err;
 	}
 
+	rproc->auto_boot = false;
+
 	wkupm3 = rproc->priv;
 	wkupm3->rproc = rproc;
 	wkupm3->pdev = pdev;
@@ -206,7 +208,7 @@ static int wkup_m3_rproc_probe(struct platform_device *pdev)
 	return 0;
 
 err_put_rproc:
-	rproc_put(rproc);
+	rproc_free(rproc);
 err:
 	pm_runtime_put_noidle(dev);
 	pm_runtime_disable(dev);
@@ -218,7 +220,7 @@ static int wkup_m3_rproc_remove(struct platform_device *pdev)
 	struct rproc *rproc = platform_get_drvdata(pdev);
 
 	rproc_del(rproc);
-	rproc_put(rproc);
+	rproc_free(rproc);
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig
index 4be1b8c21f6f..06d9fa2f3bc0 100644
--- a/drivers/reset/Kconfig
+++ b/drivers/reset/Kconfig
@@ -14,9 +14,58 @@ menuconfig RESET_CONTROLLER
 
 if RESET_CONTROLLER
 
+config RESET_ATH79
+	bool "AR71xx Reset Driver" if COMPILE_TEST
+	default ATH79
+	help
+	  This enables the ATH79 reset controller driver that supports the
+	  AR71xx SoC reset controller.
+
+config RESET_BERLIN
+	bool "Berlin Reset Driver" if COMPILE_TEST
+	default ARCH_BERLIN
+	help
+	  This enables the reset controller driver for Marvell Berlin SoCs.
+
+config RESET_LPC18XX
+	bool "LPC18xx/43xx Reset Driver" if COMPILE_TEST
+	default ARCH_LPC18XX
+	help
+	  This enables the reset controller driver for NXP LPC18xx/43xx SoCs.
+
+config RESET_MESON
+	bool "Meson Reset Driver" if COMPILE_TEST
+	default ARCH_MESON
+	help
+	  This enables the reset driver for Amlogic Meson SoCs.
+
 config RESET_OXNAS
 	bool
 
+config RESET_PISTACHIO
+	bool "Pistachio Reset Driver" if COMPILE_TEST
+	default MACH_PISTACHIO
+	help
+	  This enables the reset driver for ImgTec Pistachio SoCs.
+
+config RESET_SOCFPGA
+	bool "SoCFPGA Reset Driver" if COMPILE_TEST
+	default ARCH_SOCFPGA
+	help
+	  This enables the reset controller driver for Altera SoCFPGAs.
+
+config RESET_STM32
+	bool "STM32 Reset Driver" if COMPILE_TEST
+	default ARCH_STM32
+	help
+	  This enables the RCC reset controller driver for STM32 MCUs.
+
+config RESET_SUNXI
+	bool "Allwinner SoCs Reset Driver" if COMPILE_TEST && !ARCH_SUNXI
+	default ARCH_SUNXI
+	help
+	  This enables the reset driver for Allwinner SoCs.
+
 config TI_SYSCON_RESET
 	tristate "TI SYSCON Reset Driver"
 	depends on HAS_IOMEM
@@ -27,6 +76,22 @@ config TI_SYSCON_RESET
 	  you wish to use the reset framework for such memory-mapped devices,
 	  say Y here. Otherwise, say N.
 
+config RESET_UNIPHIER
+	tristate "Reset controller driver for UniPhier SoCs"
+	depends on ARCH_UNIPHIER || COMPILE_TEST
+	depends on OF && MFD_SYSCON
+	default ARCH_UNIPHIER
+	help
+	  Support for reset controllers on UniPhier SoCs.
+	  Say Y if you want to control reset signals provided by System Control
+	  block, Media I/O block, Peripheral Block.
+
+config RESET_ZYNQ
+	bool "ZYNQ Reset Driver" if COMPILE_TEST
+	default ARCH_ZYNQ
+	help
+	  This enables the reset controller driver for Xilinx Zynq SoCs.
+
 source "drivers/reset/sti/Kconfig"
 source "drivers/reset/hisilicon/Kconfig"
 
diff --git a/drivers/reset/Makefile b/drivers/reset/Makefile
index 5d65a93d3c43..bbe7026617fc 100644
--- a/drivers/reset/Makefile
+++ b/drivers/reset/Makefile
@@ -1,13 +1,15 @@
 obj-y += core.o
-obj-$(CONFIG_ARCH_LPC18XX) += reset-lpc18xx.o
-obj-$(CONFIG_ARCH_SOCFPGA) += reset-socfpga.o
-obj-$(CONFIG_ARCH_BERLIN) += reset-berlin.o
-obj-$(CONFIG_MACH_PISTACHIO) += reset-pistachio.o
-obj-$(CONFIG_ARCH_MESON) += reset-meson.o
-obj-$(CONFIG_ARCH_SUNXI) += reset-sunxi.o
+obj-y += hisilicon/
 obj-$(CONFIG_ARCH_STI) += sti/
-obj-$(CONFIG_ARCH_HISI) += hisilicon/
-obj-$(CONFIG_ARCH_ZYNQ) += reset-zynq.o
-obj-$(CONFIG_ATH79) += reset-ath79.o
+obj-$(CONFIG_RESET_ATH79) += reset-ath79.o
+obj-$(CONFIG_RESET_BERLIN) += reset-berlin.o
+obj-$(CONFIG_RESET_LPC18XX) += reset-lpc18xx.o
+obj-$(CONFIG_RESET_MESON) += reset-meson.o
 obj-$(CONFIG_RESET_OXNAS) += reset-oxnas.o
+obj-$(CONFIG_RESET_PISTACHIO) += reset-pistachio.o
+obj-$(CONFIG_RESET_SOCFPGA) += reset-socfpga.o
+obj-$(CONFIG_RESET_STM32) += reset-stm32.o
+obj-$(CONFIG_RESET_SUNXI) += reset-sunxi.o
 obj-$(CONFIG_TI_SYSCON_RESET) += reset-ti-syscon.o
+obj-$(CONFIG_RESET_UNIPHIER) += reset-uniphier.o
+obj-$(CONFIG_RESET_ZYNQ) += reset-zynq.o
diff --git a/drivers/reset/core.c b/drivers/reset/core.c
index 395dc9ce492e..b8ae1dbd4c17 100644
--- a/drivers/reset/core.c
+++ b/drivers/reset/core.c
@@ -138,7 +138,8 @@ EXPORT_SYMBOL_GPL(devm_reset_controller_register);
  */
 int reset_control_reset(struct reset_control *rstc)
 {
-	if (WARN_ON(rstc->shared))
+	if (WARN_ON(IS_ERR_OR_NULL(rstc)) ||
+	    WARN_ON(rstc->shared))
 		return -EINVAL;
 
 	if (rstc->rcdev->ops->reset)
@@ -161,6 +162,9 @@ EXPORT_SYMBOL_GPL(reset_control_reset);
  */
 int reset_control_assert(struct reset_control *rstc)
 {
+	if (WARN_ON(IS_ERR_OR_NULL(rstc)))
+		return -EINVAL;
+
 	if (!rstc->rcdev->ops->assert)
 		return -ENOTSUPP;
 
@@ -184,6 +188,9 @@ EXPORT_SYMBOL_GPL(reset_control_assert);
  */
 int reset_control_deassert(struct reset_control *rstc)
 {
+	if (WARN_ON(IS_ERR_OR_NULL(rstc)))
+		return -EINVAL;
+
 	if (!rstc->rcdev->ops->deassert)
 		return -ENOTSUPP;
 
@@ -204,6 +211,9 @@ EXPORT_SYMBOL_GPL(reset_control_deassert);
  */
 int reset_control_status(struct reset_control *rstc)
 {
+	if (WARN_ON(IS_ERR_OR_NULL(rstc)))
+		return -EINVAL;
+
 	if (rstc->rcdev->ops->status)
 		return rstc->rcdev->ops->status(rstc->rcdev, rstc->id);
 
diff --git a/drivers/reset/hisilicon/Kconfig b/drivers/reset/hisilicon/Kconfig
index 26bf95a83a8e..1ff8b0c80980 100644
--- a/drivers/reset/hisilicon/Kconfig
+++ b/drivers/reset/hisilicon/Kconfig
@@ -1,5 +1,6 @@
 config COMMON_RESET_HI6220
 	tristate "Hi6220 Reset Driver"
-	depends on (ARCH_HISI && RESET_CONTROLLER)
+	depends on ARCH_HISI || COMPILE_TEST
+	default ARCH_HISI
 	help
 	  Build the Hisilicon Hi6220 reset driver.
diff --git a/drivers/reset/reset-ath79.c b/drivers/reset/reset-ath79.c
index 16d410cd6146..6b97631f5489 100644
--- a/drivers/reset/reset-ath79.c
+++ b/drivers/reset/reset-ath79.c
@@ -12,6 +12,7 @@
  * GNU General Public License for more details.
  */
 
+#include <linux/io.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/reset-controller.h>
diff --git a/drivers/reset/reset-socfpga.c b/drivers/reset/reset-socfpga.c
index 12add9b0fa49..78ebf8424375 100644
--- a/drivers/reset/reset-socfpga.c
+++ b/drivers/reset/reset-socfpga.c
@@ -28,7 +28,6 @@
 struct socfpga_reset_data {
 	spinlock_t			lock;
 	void __iomem			*membase;
-	u32				modrst_offset;
 	struct reset_controller_dev	rcdev;
 };
 
@@ -45,9 +44,8 @@ static int socfpga_reset_assert(struct reset_controller_dev *rcdev,
 
 	spin_lock_irqsave(&data->lock, flags);
 
-	reg = readl(data->membase + data->modrst_offset + (bank * NR_BANKS));
-	writel(reg | BIT(offset), data->membase + data->modrst_offset +
-				 (bank * NR_BANKS));
+	reg = readl(data->membase + (bank * NR_BANKS));
+	writel(reg | BIT(offset), data->membase + (bank * NR_BANKS));
 	spin_unlock_irqrestore(&data->lock, flags);
 
 	return 0;
@@ -67,9 +65,8 @@ static int socfpga_reset_deassert(struct reset_controller_dev *rcdev,
 
 	spin_lock_irqsave(&data->lock, flags);
 
-	reg = readl(data->membase + data->modrst_offset + (bank * NR_BANKS));
-	writel(reg & ~BIT(offset), data->membase + data->modrst_offset +
-				  (bank * NR_BANKS));
+	reg = readl(data->membase + (bank * NR_BANKS));
+	writel(reg & ~BIT(offset), data->membase + (bank * NR_BANKS));
 
 	spin_unlock_irqrestore(&data->lock, flags);
 
@@ -85,7 +82,7 @@ static int socfpga_reset_status(struct reset_controller_dev *rcdev,
 	int offset = id % BITS_PER_LONG;
 	u32 reg;
 
-	reg = readl(data->membase + data->modrst_offset + (bank * NR_BANKS));
+	reg = readl(data->membase + (bank * NR_BANKS));
 
 	return !(reg & BIT(offset));
 }
@@ -102,6 +99,7 @@ static int socfpga_reset_probe(struct platform_device *pdev)
 	struct resource *res;
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
+	u32 modrst_offset;
 
 	/*
 	 * The binding was mainlined without the required property.
@@ -122,10 +120,11 @@ static int socfpga_reset_probe(struct platform_device *pdev)
 	if (IS_ERR(data->membase))
 		return PTR_ERR(data->membase);
 
-	if (of_property_read_u32(np, "altr,modrst-offset", &data->modrst_offset)) {
+	if (of_property_read_u32(np, "altr,modrst-offset", &modrst_offset)) {
 		dev_warn(dev, "missing altr,modrst-offset property, assuming 0x10!\n");
-		data->modrst_offset = 0x10;
+		modrst_offset = 0x10;
 	}
+	data->membase += modrst_offset;
 
 	spin_lock_init(&data->lock);
 
diff --git a/drivers/reset/reset-stm32.c b/drivers/reset/reset-stm32.c
new file mode 100644
index 000000000000..3a7c8527e66a
--- /dev/null
+++ b/drivers/reset/reset-stm32.c
@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) Maxime Coquelin 2015
+ * Author:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ *
+ * Heavily based on sunxi driver from Maxime Ripard.
+ */
+
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/reset-controller.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+struct stm32_reset_data {
+	spinlock_t			lock;
+	void __iomem			*membase;
+	struct reset_controller_dev	rcdev;
+};
+
+static int stm32_reset_assert(struct reset_controller_dev *rcdev,
+			      unsigned long id)
+{
+	struct stm32_reset_data *data = container_of(rcdev,
+						     struct stm32_reset_data,
+						     rcdev);
+	int bank = id / BITS_PER_LONG;
+	int offset = id % BITS_PER_LONG;
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&data->lock, flags);
+
+	reg = readl(data->membase + (bank * 4));
+	writel(reg | BIT(offset), data->membase + (bank * 4));
+
+	spin_unlock_irqrestore(&data->lock, flags);
+
+	return 0;
+}
+
+static int stm32_reset_deassert(struct reset_controller_dev *rcdev,
+				unsigned long id)
+{
+	struct stm32_reset_data *data = container_of(rcdev,
+						     struct stm32_reset_data,
+						     rcdev);
+	int bank = id / BITS_PER_LONG;
+	int offset = id % BITS_PER_LONG;
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&data->lock, flags);
+
+	reg = readl(data->membase + (bank * 4));
+	writel(reg & ~BIT(offset), data->membase + (bank * 4));
+
+	spin_unlock_irqrestore(&data->lock, flags);
+
+	return 0;
+}
+
+static const struct reset_control_ops stm32_reset_ops = {
+	.assert		= stm32_reset_assert,
+	.deassert	= stm32_reset_deassert,
+};
+
+static const struct of_device_id stm32_reset_dt_ids[] = {
+	 { .compatible = "st,stm32-rcc", },
+	 { /* sentinel */ },
+};
+
+static int stm32_reset_probe(struct platform_device *pdev)
+{
+	struct stm32_reset_data *data;
+	struct resource *res;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	data->membase = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(data->membase))
+		return PTR_ERR(data->membase);
+
+	spin_lock_init(&data->lock);
+
+	data->rcdev.owner = THIS_MODULE;
+	data->rcdev.nr_resets = resource_size(res) * 8;
+	data->rcdev.ops = &stm32_reset_ops;
+	data->rcdev.of_node = pdev->dev.of_node;
+
+	return devm_reset_controller_register(&pdev->dev, &data->rcdev);
+}
+
+static struct platform_driver stm32_reset_driver = {
+	.probe	= stm32_reset_probe,
+	.driver = {
+		.name		= "stm32-rcc-reset",
+		.of_match_table	= stm32_reset_dt_ids,
+	},
+};
+builtin_platform_driver(stm32_reset_driver);
diff --git a/drivers/reset/reset-uniphier.c b/drivers/reset/reset-uniphier.c
new file mode 100644
index 000000000000..8b2558e7363e
--- /dev/null
+++ b/drivers/reset/reset-uniphier.c
@@ -0,0 +1,440 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+
+struct uniphier_reset_data {
+	unsigned int id;
+	unsigned int reg;
+	unsigned int bit;
+	unsigned int flags;
+#define UNIPHIER_RESET_ACTIVE_LOW		BIT(0)
+};
+
+#define UNIPHIER_RESET_ID_END		(unsigned int)(-1)
+
+#define UNIPHIER_RESET_END				\
+	{ .id = UNIPHIER_RESET_ID_END }
+
+#define UNIPHIER_RESET(_id, _reg, _bit)			\
+	{						\
+		.id = (_id),				\
+		.reg = (_reg),				\
+		.bit = (_bit),				\
+	}
+
+#define UNIPHIER_RESETX(_id, _reg, _bit)		\
+	{						\
+		.id = (_id),				\
+		.reg = (_reg),				\
+		.bit = (_bit),				\
+		.flags = UNIPHIER_RESET_ACTIVE_LOW,	\
+	}
+
+/* System reset data */
+#define UNIPHIER_SLD3_SYS_RESET_STDMAC(id)		\
+	UNIPHIER_RESETX((id), 0x2000, 10)
+
+#define UNIPHIER_LD11_SYS_RESET_STDMAC(id)		\
+	UNIPHIER_RESETX((id), 0x200c, 8)
+
+#define UNIPHIER_PRO4_SYS_RESET_GIO(id)			\
+	UNIPHIER_RESETX((id), 0x2000, 6)
+
+#define UNIPHIER_LD20_SYS_RESET_GIO(id)			\
+	UNIPHIER_RESETX((id), 0x200c, 5)
+
+#define UNIPHIER_PRO4_SYS_RESET_USB3(id, ch)		\
+	UNIPHIER_RESETX((id), 0x2000 + 0x4 * (ch), 17)
+
+const struct uniphier_reset_data uniphier_sld3_sys_reset_data[] = {
+	UNIPHIER_SLD3_SYS_RESET_STDMAC(8),	/* Ether, HSC, MIO */
+	UNIPHIER_RESET_END,
+};
+
+const struct uniphier_reset_data uniphier_pro4_sys_reset_data[] = {
+	UNIPHIER_SLD3_SYS_RESET_STDMAC(8),	/* HSC, MIO, RLE */
+	UNIPHIER_PRO4_SYS_RESET_GIO(12),	/* Ether, SATA, USB3 */
+	UNIPHIER_PRO4_SYS_RESET_USB3(14, 0),
+	UNIPHIER_PRO4_SYS_RESET_USB3(15, 1),
+	UNIPHIER_RESET_END,
+};
+
+const struct uniphier_reset_data uniphier_pro5_sys_reset_data[] = {
+	UNIPHIER_SLD3_SYS_RESET_STDMAC(8),	/* HSC */
+	UNIPHIER_PRO4_SYS_RESET_GIO(12),	/* PCIe, USB3 */
+	UNIPHIER_PRO4_SYS_RESET_USB3(14, 0),
+	UNIPHIER_PRO4_SYS_RESET_USB3(15, 1),
+	UNIPHIER_RESET_END,
+};
+
+const struct uniphier_reset_data uniphier_pxs2_sys_reset_data[] = {
+	UNIPHIER_SLD3_SYS_RESET_STDMAC(8),	/* HSC, RLE */
+	UNIPHIER_PRO4_SYS_RESET_USB3(14, 0),
+	UNIPHIER_PRO4_SYS_RESET_USB3(15, 1),
+	UNIPHIER_RESETX(16, 0x2014, 4),		/* USB30-PHY0 */
+	UNIPHIER_RESETX(17, 0x2014, 0),		/* USB30-PHY1 */
+	UNIPHIER_RESETX(18, 0x2014, 2),		/* USB30-PHY2 */
+	UNIPHIER_RESETX(20, 0x2014, 5),		/* USB31-PHY0 */
+	UNIPHIER_RESETX(21, 0x2014, 1),		/* USB31-PHY1 */
+	UNIPHIER_RESETX(28, 0x2014, 12),	/* SATA */
+	UNIPHIER_RESET(29, 0x2014, 8),		/* SATA-PHY (active high) */
+	UNIPHIER_RESET_END,
+};
+
+const struct uniphier_reset_data uniphier_ld11_sys_reset_data[] = {
+	UNIPHIER_LD11_SYS_RESET_STDMAC(8),	/* HSC, MIO */
+	UNIPHIER_RESET_END,
+};
+
+const struct uniphier_reset_data uniphier_ld20_sys_reset_data[] = {
+	UNIPHIER_LD11_SYS_RESET_STDMAC(8),	/* HSC */
+	UNIPHIER_LD20_SYS_RESET_GIO(12),	/* PCIe, USB3 */
+	UNIPHIER_RESETX(16, 0x200c, 12),	/* USB30-PHY0 */
+	UNIPHIER_RESETX(17, 0x200c, 13),	/* USB30-PHY1 */
+	UNIPHIER_RESETX(18, 0x200c, 14),	/* USB30-PHY2 */
+	UNIPHIER_RESETX(19, 0x200c, 15),	/* USB30-PHY3 */
+	UNIPHIER_RESET_END,
+};
+
+/* Media I/O reset data */
+#define UNIPHIER_MIO_RESET_SD(id, ch)			\
+	UNIPHIER_RESETX((id), 0x110 + 0x200 * (ch), 0)
+
+#define UNIPHIER_MIO_RESET_SD_BRIDGE(id, ch)		\
+	UNIPHIER_RESETX((id), 0x110 + 0x200 * (ch), 26)
+
+#define UNIPHIER_MIO_RESET_EMMC_HW_RESET(id, ch)	\
+	UNIPHIER_RESETX((id), 0x80 + 0x200 * (ch), 0)
+
+#define UNIPHIER_MIO_RESET_USB2(id, ch)			\
+	UNIPHIER_RESETX((id), 0x114 + 0x200 * (ch), 0)
+
+#define UNIPHIER_MIO_RESET_USB2_BRIDGE(id, ch)		\
+	UNIPHIER_RESETX((id), 0x110 + 0x200 * (ch), 24)
+
+#define UNIPHIER_MIO_RESET_DMAC(id)			\
+	UNIPHIER_RESETX((id), 0x110, 17)
+
+const struct uniphier_reset_data uniphier_sld3_mio_reset_data[] = {
+	UNIPHIER_MIO_RESET_SD(0, 0),
+	UNIPHIER_MIO_RESET_SD(1, 1),
+	UNIPHIER_MIO_RESET_SD(2, 2),
+	UNIPHIER_MIO_RESET_SD_BRIDGE(3, 0),
+	UNIPHIER_MIO_RESET_SD_BRIDGE(4, 1),
+	UNIPHIER_MIO_RESET_SD_BRIDGE(5, 2),
+	UNIPHIER_MIO_RESET_EMMC_HW_RESET(6, 1),
+	UNIPHIER_MIO_RESET_DMAC(7),
+	UNIPHIER_MIO_RESET_USB2(8, 0),
+	UNIPHIER_MIO_RESET_USB2(9, 1),
+	UNIPHIER_MIO_RESET_USB2(10, 2),
+	UNIPHIER_MIO_RESET_USB2(11, 3),
+	UNIPHIER_MIO_RESET_USB2_BRIDGE(12, 0),
+	UNIPHIER_MIO_RESET_USB2_BRIDGE(13, 1),
+	UNIPHIER_MIO_RESET_USB2_BRIDGE(14, 2),
+	UNIPHIER_MIO_RESET_USB2_BRIDGE(15, 3),
+	UNIPHIER_RESET_END,
+};
+
+const struct uniphier_reset_data uniphier_pro5_mio_reset_data[] = {
+	UNIPHIER_MIO_RESET_SD(0, 0),
+	UNIPHIER_MIO_RESET_SD(1, 1),
+	UNIPHIER_MIO_RESET_EMMC_HW_RESET(6, 1),
+	UNIPHIER_RESET_END,
+};
+
+/* Peripheral reset data */
+#define UNIPHIER_PERI_RESET_UART(id, ch)		\
+	UNIPHIER_RESETX((id), 0x114, 19 + (ch))
+
+#define UNIPHIER_PERI_RESET_I2C(id, ch)			\
+	UNIPHIER_RESETX((id), 0x114, 5 + (ch))
+
+#define UNIPHIER_PERI_RESET_FI2C(id, ch)		\
+	UNIPHIER_RESETX((id), 0x114, 24 + (ch))
+
+const struct uniphier_reset_data uniphier_ld4_peri_reset_data[] = {
+	UNIPHIER_PERI_RESET_UART(0, 0),
+	UNIPHIER_PERI_RESET_UART(1, 1),
+	UNIPHIER_PERI_RESET_UART(2, 2),
+	UNIPHIER_PERI_RESET_UART(3, 3),
+	UNIPHIER_PERI_RESET_I2C(4, 0),
+	UNIPHIER_PERI_RESET_I2C(5, 1),
+	UNIPHIER_PERI_RESET_I2C(6, 2),
+	UNIPHIER_PERI_RESET_I2C(7, 3),
+	UNIPHIER_PERI_RESET_I2C(8, 4),
+	UNIPHIER_RESET_END,
+};
+
+const struct uniphier_reset_data uniphier_pro4_peri_reset_data[] = {
+	UNIPHIER_PERI_RESET_UART(0, 0),
+	UNIPHIER_PERI_RESET_UART(1, 1),
+	UNIPHIER_PERI_RESET_UART(2, 2),
+	UNIPHIER_PERI_RESET_UART(3, 3),
+	UNIPHIER_PERI_RESET_FI2C(4, 0),
+	UNIPHIER_PERI_RESET_FI2C(5, 1),
+	UNIPHIER_PERI_RESET_FI2C(6, 2),
+	UNIPHIER_PERI_RESET_FI2C(7, 3),
+	UNIPHIER_PERI_RESET_FI2C(8, 4),
+	UNIPHIER_PERI_RESET_FI2C(9, 5),
+	UNIPHIER_PERI_RESET_FI2C(10, 6),
+	UNIPHIER_RESET_END,
+};
+
+/* core implementaton */
+struct uniphier_reset_priv {
+	struct reset_controller_dev rcdev;
+	struct device *dev;
+	struct regmap *regmap;
+	const struct uniphier_reset_data *data;
+};
+
+#define to_uniphier_reset_priv(_rcdev) \
+			container_of(_rcdev, struct uniphier_reset_priv, rcdev)
+
+static int uniphier_reset_update(struct reset_controller_dev *rcdev,
+				 unsigned long id, int assert)
+{
+	struct uniphier_reset_priv *priv = to_uniphier_reset_priv(rcdev);
+	const struct uniphier_reset_data *p;
+
+	for (p = priv->data; p->id != UNIPHIER_RESET_ID_END; p++) {
+		unsigned int mask, val;
+
+		if (p->id != id)
+			continue;
+
+		mask = BIT(p->bit);
+
+		if (assert)
+			val = mask;
+		else
+			val = ~mask;
+
+		if (p->flags & UNIPHIER_RESET_ACTIVE_LOW)
+			val = ~val;
+
+		return regmap_write_bits(priv->regmap, p->reg, mask, val);
+	}
+
+	dev_err(priv->dev, "reset_id=%lu was not handled\n", id);
+	return -EINVAL;
+}
+
+static int uniphier_reset_assert(struct reset_controller_dev *rcdev,
+				 unsigned long id)
+{
+	return uniphier_reset_update(rcdev, id, 1);
+}
+
+static int uniphier_reset_deassert(struct reset_controller_dev *rcdev,
+				   unsigned long id)
+{
+	return uniphier_reset_update(rcdev, id, 0);
+}
+
+static int uniphier_reset_status(struct reset_controller_dev *rcdev,
+				 unsigned long id)
+{
+	struct uniphier_reset_priv *priv = to_uniphier_reset_priv(rcdev);
+	const struct uniphier_reset_data *p;
+
+	for (p = priv->data; p->id != UNIPHIER_RESET_ID_END; p++) {
+		unsigned int val;
+		int ret, asserted;
+
+		if (p->id != id)
+			continue;
+
+		ret = regmap_read(priv->regmap, p->reg, &val);
+		if (ret)
+			return ret;
+
+		asserted = !!(val & BIT(p->bit));
+
+		if (p->flags & UNIPHIER_RESET_ACTIVE_LOW)
+			asserted = !asserted;
+
+		return asserted;
+	}
+
+	dev_err(priv->dev, "reset_id=%lu was not found\n", id);
+	return -EINVAL;
+}
+
+static const struct reset_control_ops uniphier_reset_ops = {
+	.assert = uniphier_reset_assert,
+	.deassert = uniphier_reset_deassert,
+	.status = uniphier_reset_status,
+};
+
+static int uniphier_reset_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct uniphier_reset_priv *priv;
+	const struct uniphier_reset_data *p, *data;
+	struct regmap *regmap;
+	struct device_node *parent;
+	unsigned int nr_resets = 0;
+
+	data = of_device_get_match_data(dev);
+	if (WARN_ON(!data))
+		return -EINVAL;
+
+	parent = of_get_parent(dev->of_node); /* parent should be syscon node */
+	regmap = syscon_node_to_regmap(parent);
+	of_node_put(parent);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "failed to get regmap (error %ld)\n",
+			PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	for (p = data; p->id != UNIPHIER_RESET_ID_END; p++)
+		nr_resets = max(nr_resets, p->id + 1);
+
+	priv->rcdev.ops = &uniphier_reset_ops;
+	priv->rcdev.owner = dev->driver->owner;
+	priv->rcdev.of_node = dev->of_node;
+	priv->rcdev.nr_resets = nr_resets;
+	priv->dev = dev;
+	priv->regmap = regmap;
+	priv->data = data;
+
+	return devm_reset_controller_register(&pdev->dev, &priv->rcdev);
+}
+
+static const struct of_device_id uniphier_reset_match[] = {
+	/* System reset */
+	{
+		.compatible = "socionext,uniphier-sld3-reset",
+		.data = uniphier_sld3_sys_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld4-reset",
+		.data = uniphier_sld3_sys_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro4-reset",
+		.data = uniphier_pro4_sys_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-sld8-reset",
+		.data = uniphier_sld3_sys_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro5-reset",
+		.data = uniphier_pro5_sys_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pxs2-reset",
+		.data = uniphier_pxs2_sys_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld11-reset",
+		.data = uniphier_ld11_sys_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld20-reset",
+		.data = uniphier_ld20_sys_reset_data,
+	},
+	/* Media I/O reset */
+	{
+		.compatible = "socionext,uniphier-sld3-mio-reset",
+		.data = uniphier_sld3_mio_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld4-mio-reset",
+		.data = uniphier_sld3_mio_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro4-mio-reset",
+		.data = uniphier_sld3_mio_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-sld8-mio-reset",
+		.data = uniphier_sld3_mio_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro5-mio-reset",
+		.data = uniphier_pro5_mio_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pxs2-mio-reset",
+		.data = uniphier_pro5_mio_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld11-mio-reset",
+		.data = uniphier_sld3_mio_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld20-mio-reset",
+		.data = uniphier_pro5_mio_reset_data,
+	},
+	/* Peripheral reset */
+	{
+		.compatible = "socionext,uniphier-ld4-peri-reset",
+		.data = uniphier_ld4_peri_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro4-peri-reset",
+		.data = uniphier_pro4_peri_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-sld8-peri-reset",
+		.data = uniphier_ld4_peri_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pro5-peri-reset",
+		.data = uniphier_pro4_peri_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pxs2-peri-reset",
+		.data = uniphier_pro4_peri_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld11-peri-reset",
+		.data = uniphier_pro4_peri_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld20-peri-reset",
+		.data = uniphier_pro4_peri_reset_data,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, uniphier_reset_match);
+
+static struct platform_driver uniphier_reset_driver = {
+	.probe = uniphier_reset_probe,
+	.driver = {
+		.name = "uniphier-reset",
+		.of_match_table = uniphier_reset_match,
+	},
+};
+module_platform_driver(uniphier_reset_driver);
+
+MODULE_AUTHOR("Masahiro Yamada <yamada.masahiro@socionext.com>");
+MODULE_DESCRIPTION("UniPhier Reset Controller Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/rpmsg/Kconfig b/drivers/rpmsg/Kconfig
index 69a219387582..de31c5f14dd9 100644
--- a/drivers/rpmsg/Kconfig
+++ b/drivers/rpmsg/Kconfig
@@ -3,6 +3,20 @@ menu "Rpmsg drivers"
 # RPMSG always gets selected by whoever wants it
 config RPMSG
 	tristate
+
+config RPMSG_QCOM_SMD
+	tristate "Qualcomm Shared Memory Driver (SMD)"
+	depends on QCOM_SMEM
+	depends on QCOM_SMD=n
+	select RPMSG
+	help
+	  Say y here to enable support for the Qualcomm Shared Memory Driver
+	  providing communication channels to remote processors in Qualcomm
+	  platforms.
+
+config RPMSG_VIRTIO
+	tristate
+	select RPMSG
 	select VIRTIO
 	select VIRTUALIZATION
 
diff --git a/drivers/rpmsg/Makefile b/drivers/rpmsg/Makefile
index 7617fcb8259f..ae9c9132cf76 100644
--- a/drivers/rpmsg/Makefile
+++ b/drivers/rpmsg/Makefile
@@ -1 +1,3 @@
-obj-$(CONFIG_RPMSG)	+= virtio_rpmsg_bus.o
+obj-$(CONFIG_RPMSG)		+= rpmsg_core.o
+obj-$(CONFIG_RPMSG_QCOM_SMD)	+= qcom_smd.o
+obj-$(CONFIG_RPMSG_VIRTIO)	+= virtio_rpmsg_bus.o
diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c
new file mode 100644
index 000000000000..06fef2b4c814
--- /dev/null
+++ b/drivers/rpmsg/qcom_smd.c
@@ -0,0 +1,1434 @@
+/*
+ * Copyright (c) 2015, Sony Mobile Communications AB.
+ * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/soc/qcom/smem.h>
+#include <linux/wait.h>
+#include <linux/rpmsg.h>
+
+#include "rpmsg_internal.h"
+
+/*
+ * The Qualcomm Shared Memory communication solution provides point-to-point
+ * channels for clients to send and receive streaming or packet based data.
+ *
+ * Each channel consists of a control item (channel info) and a ring buffer
+ * pair. The channel info carry information related to channel state, flow
+ * control and the offsets within the ring buffer.
+ *
+ * All allocated channels are listed in an allocation table, identifying the
+ * pair of items by name, type and remote processor.
+ *
+ * Upon creating a new channel the remote processor allocates channel info and
+ * ring buffer items from the smem heap and populate the allocation table. An
+ * interrupt is sent to the other end of the channel and a scan for new
+ * channels should be done. A channel never goes away, it will only change
+ * state.
+ *
+ * The remote processor signals it intent for bring up the communication
+ * channel by setting the state of its end of the channel to "opening" and
+ * sends out an interrupt. We detect this change and register a smd device to
+ * consume the channel. Upon finding a consumer we finish the handshake and the
+ * channel is up.
+ *
+ * Upon closing a channel, the remote processor will update the state of its
+ * end of the channel and signal us, we will then unregister any attached
+ * device and close our end of the channel.
+ *
+ * Devices attached to a channel can use the qcom_smd_send function to push
+ * data to the channel, this is done by copying the data into the tx ring
+ * buffer, updating the pointers in the channel info and signaling the remote
+ * processor.
+ *
+ * The remote processor does the equivalent when it transfer data and upon
+ * receiving the interrupt we check the channel info for new data and delivers
+ * this to the attached device. If the device is not ready to receive the data
+ * we leave it in the ring buffer for now.
+ */
+
+struct smd_channel_info;
+struct smd_channel_info_pair;
+struct smd_channel_info_word;
+struct smd_channel_info_word_pair;
+
+static const struct rpmsg_endpoint_ops qcom_smd_endpoint_ops;
+
+#define SMD_ALLOC_TBL_COUNT	2
+#define SMD_ALLOC_TBL_SIZE	64
+
+/*
+ * This lists the various smem heap items relevant for the allocation table and
+ * smd channel entries.
+ */
+static const struct {
+	unsigned alloc_tbl_id;
+	unsigned info_base_id;
+	unsigned fifo_base_id;
+} smem_items[SMD_ALLOC_TBL_COUNT] = {
+	{
+		.alloc_tbl_id = 13,
+		.info_base_id = 14,
+		.fifo_base_id = 338
+	},
+	{
+		.alloc_tbl_id = 266,
+		.info_base_id = 138,
+		.fifo_base_id = 202,
+	},
+};
+
+/**
+ * struct qcom_smd_edge - representing a remote processor
+ * @of_node:		of_node handle for information related to this edge
+ * @edge_id:		identifier of this edge
+ * @remote_pid:		identifier of remote processor
+ * @irq:		interrupt for signals on this edge
+ * @ipc_regmap:		regmap handle holding the outgoing ipc register
+ * @ipc_offset:		offset within @ipc_regmap of the register for ipc
+ * @ipc_bit:		bit in the register at @ipc_offset of @ipc_regmap
+ * @channels:		list of all channels detected on this edge
+ * @channels_lock:	guard for modifications of @channels
+ * @allocated:		array of bitmaps representing already allocated channels
+ * @smem_available:	last available amount of smem triggering a channel scan
+ * @scan_work:		work item for discovering new channels
+ * @state_work:		work item for edge state changes
+ */
+struct qcom_smd_edge {
+	struct device dev;
+
+	struct device_node *of_node;
+	unsigned edge_id;
+	unsigned remote_pid;
+
+	int irq;
+
+	struct regmap *ipc_regmap;
+	int ipc_offset;
+	int ipc_bit;
+
+	struct list_head channels;
+	spinlock_t channels_lock;
+
+	DECLARE_BITMAP(allocated[SMD_ALLOC_TBL_COUNT], SMD_ALLOC_TBL_SIZE);
+
+	unsigned smem_available;
+
+	wait_queue_head_t new_channel_event;
+
+	struct work_struct scan_work;
+	struct work_struct state_work;
+};
+
+/*
+ * SMD channel states.
+ */
+enum smd_channel_state {
+	SMD_CHANNEL_CLOSED,
+	SMD_CHANNEL_OPENING,
+	SMD_CHANNEL_OPENED,
+	SMD_CHANNEL_FLUSHING,
+	SMD_CHANNEL_CLOSING,
+	SMD_CHANNEL_RESET,
+	SMD_CHANNEL_RESET_OPENING
+};
+
+struct qcom_smd_device {
+	struct rpmsg_device rpdev;
+
+	struct qcom_smd_edge *edge;
+};
+
+struct qcom_smd_endpoint {
+	struct rpmsg_endpoint ept;
+
+	struct qcom_smd_channel *qsch;
+};
+
+#define to_smd_device(_rpdev)	container_of(_rpdev, struct qcom_smd_device, rpdev)
+#define to_smd_edge(d)		container_of(d, struct qcom_smd_edge, dev)
+#define to_smd_endpoint(ept)	container_of(ept, struct qcom_smd_endpoint, ept)
+
+/**
+ * struct qcom_smd_channel - smd channel struct
+ * @edge:		qcom_smd_edge this channel is living on
+ * @qsdev:		reference to a associated smd client device
+ * @name:		name of the channel
+ * @state:		local state of the channel
+ * @remote_state:	remote state of the channel
+ * @info:		byte aligned outgoing/incoming channel info
+ * @info_word:		word aligned outgoing/incoming channel info
+ * @tx_lock:		lock to make writes to the channel mutually exclusive
+ * @fblockread_event:	wakeup event tied to tx fBLOCKREADINTR
+ * @tx_fifo:		pointer to the outgoing ring buffer
+ * @rx_fifo:		pointer to the incoming ring buffer
+ * @fifo_size:		size of each ring buffer
+ * @bounce_buffer:	bounce buffer for reading wrapped packets
+ * @cb:			callback function registered for this channel
+ * @recv_lock:		guard for rx info modifications and cb pointer
+ * @pkt_size:		size of the currently handled packet
+ * @list:		lite entry for @channels in qcom_smd_edge
+ */
+struct qcom_smd_channel {
+	struct qcom_smd_edge *edge;
+
+	struct qcom_smd_endpoint *qsept;
+	bool registered;
+
+	char *name;
+	enum smd_channel_state state;
+	enum smd_channel_state remote_state;
+
+	struct smd_channel_info_pair *info;
+	struct smd_channel_info_word_pair *info_word;
+
+	struct mutex tx_lock;
+	wait_queue_head_t fblockread_event;
+
+	void *tx_fifo;
+	void *rx_fifo;
+	int fifo_size;
+
+	void *bounce_buffer;
+
+	spinlock_t recv_lock;
+
+	int pkt_size;
+
+	void *drvdata;
+
+	struct list_head list;
+};
+
+/*
+ * Format of the smd_info smem items, for byte aligned channels.
+ */
+struct smd_channel_info {
+	__le32 state;
+	u8  fDSR;
+	u8  fCTS;
+	u8  fCD;
+	u8  fRI;
+	u8  fHEAD;
+	u8  fTAIL;
+	u8  fSTATE;
+	u8  fBLOCKREADINTR;
+	__le32 tail;
+	__le32 head;
+};
+
+struct smd_channel_info_pair {
+	struct smd_channel_info tx;
+	struct smd_channel_info rx;
+};
+
+/*
+ * Format of the smd_info smem items, for word aligned channels.
+ */
+struct smd_channel_info_word {
+	__le32 state;
+	__le32 fDSR;
+	__le32 fCTS;
+	__le32 fCD;
+	__le32 fRI;
+	__le32 fHEAD;
+	__le32 fTAIL;
+	__le32 fSTATE;
+	__le32 fBLOCKREADINTR;
+	__le32 tail;
+	__le32 head;
+};
+
+struct smd_channel_info_word_pair {
+	struct smd_channel_info_word tx;
+	struct smd_channel_info_word rx;
+};
+
+#define GET_RX_CHANNEL_FLAG(channel, param)				     \
+	({								     \
+		BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u8)); \
+		channel->info_word ?					     \
+			le32_to_cpu(channel->info_word->rx.param) :	     \
+			channel->info->rx.param;			     \
+	})
+
+#define GET_RX_CHANNEL_INFO(channel, param)				      \
+	({								      \
+		BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u32)); \
+		le32_to_cpu(channel->info_word ?			      \
+			channel->info_word->rx.param :			      \
+			channel->info->rx.param);			      \
+	})
+
+#define SET_RX_CHANNEL_FLAG(channel, param, value)			     \
+	({								     \
+		BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u8)); \
+		if (channel->info_word)					     \
+			channel->info_word->rx.param = cpu_to_le32(value);   \
+		else							     \
+			channel->info->rx.param = value;		     \
+	})
+
+#define SET_RX_CHANNEL_INFO(channel, param, value)			      \
+	({								      \
+		BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u32)); \
+		if (channel->info_word)					      \
+			channel->info_word->rx.param = cpu_to_le32(value);    \
+		else							      \
+			channel->info->rx.param = cpu_to_le32(value);	      \
+	})
+
+#define GET_TX_CHANNEL_FLAG(channel, param)				     \
+	({								     \
+		BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u8)); \
+		channel->info_word ?					     \
+			le32_to_cpu(channel->info_word->tx.param) :          \
+			channel->info->tx.param;			     \
+	})
+
+#define GET_TX_CHANNEL_INFO(channel, param)				      \
+	({								      \
+		BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u32)); \
+		le32_to_cpu(channel->info_word ?			      \
+			channel->info_word->tx.param :			      \
+			channel->info->tx.param);			      \
+	})
+
+#define SET_TX_CHANNEL_FLAG(channel, param, value)			     \
+	({								     \
+		BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u8)); \
+		if (channel->info_word)					     \
+			channel->info_word->tx.param = cpu_to_le32(value);   \
+		else							     \
+			channel->info->tx.param = value;		     \
+	})
+
+#define SET_TX_CHANNEL_INFO(channel, param, value)			      \
+	({								      \
+		BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u32)); \
+		if (channel->info_word)					      \
+			channel->info_word->tx.param = cpu_to_le32(value);   \
+		else							      \
+			channel->info->tx.param = cpu_to_le32(value);	      \
+	})
+
+/**
+ * struct qcom_smd_alloc_entry - channel allocation entry
+ * @name:	channel name
+ * @cid:	channel index
+ * @flags:	channel flags and edge id
+ * @ref_count:	reference count of the channel
+ */
+struct qcom_smd_alloc_entry {
+	u8 name[20];
+	__le32 cid;
+	__le32 flags;
+	__le32 ref_count;
+} __packed;
+
+#define SMD_CHANNEL_FLAGS_EDGE_MASK	0xff
+#define SMD_CHANNEL_FLAGS_STREAM	BIT(8)
+#define SMD_CHANNEL_FLAGS_PACKET	BIT(9)
+
+/*
+ * Each smd packet contains a 20 byte header, with the first 4 being the length
+ * of the packet.
+ */
+#define SMD_PACKET_HEADER_LEN	20
+
+/*
+ * Signal the remote processor associated with 'channel'.
+ */
+static void qcom_smd_signal_channel(struct qcom_smd_channel *channel)
+{
+	struct qcom_smd_edge *edge = channel->edge;
+
+	regmap_write(edge->ipc_regmap, edge->ipc_offset, BIT(edge->ipc_bit));
+}
+
+/*
+ * Initialize the tx channel info
+ */
+static void qcom_smd_channel_reset(struct qcom_smd_channel *channel)
+{
+	SET_TX_CHANNEL_INFO(channel, state, SMD_CHANNEL_CLOSED);
+	SET_TX_CHANNEL_FLAG(channel, fDSR, 0);
+	SET_TX_CHANNEL_FLAG(channel, fCTS, 0);
+	SET_TX_CHANNEL_FLAG(channel, fCD, 0);
+	SET_TX_CHANNEL_FLAG(channel, fRI, 0);
+	SET_TX_CHANNEL_FLAG(channel, fHEAD, 0);
+	SET_TX_CHANNEL_FLAG(channel, fTAIL, 0);
+	SET_TX_CHANNEL_FLAG(channel, fSTATE, 1);
+	SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 1);
+	SET_TX_CHANNEL_INFO(channel, head, 0);
+	SET_RX_CHANNEL_INFO(channel, tail, 0);
+
+	qcom_smd_signal_channel(channel);
+
+	channel->state = SMD_CHANNEL_CLOSED;
+	channel->pkt_size = 0;
+}
+
+/*
+ * Set the callback for a channel, with appropriate locking
+ */
+static void qcom_smd_channel_set_callback(struct qcom_smd_channel *channel,
+					  rpmsg_rx_cb_t cb)
+{
+	struct rpmsg_endpoint *ept = &channel->qsept->ept;
+	unsigned long flags;
+
+	spin_lock_irqsave(&channel->recv_lock, flags);
+	ept->cb = cb;
+	spin_unlock_irqrestore(&channel->recv_lock, flags);
+};
+
+/*
+ * Calculate the amount of data available in the rx fifo
+ */
+static size_t qcom_smd_channel_get_rx_avail(struct qcom_smd_channel *channel)
+{
+	unsigned head;
+	unsigned tail;
+
+	head = GET_RX_CHANNEL_INFO(channel, head);
+	tail = GET_RX_CHANNEL_INFO(channel, tail);
+
+	return (head - tail) & (channel->fifo_size - 1);
+}
+
+/*
+ * Set tx channel state and inform the remote processor
+ */
+static void qcom_smd_channel_set_state(struct qcom_smd_channel *channel,
+				       int state)
+{
+	struct qcom_smd_edge *edge = channel->edge;
+	bool is_open = state == SMD_CHANNEL_OPENED;
+
+	if (channel->state == state)
+		return;
+
+	dev_dbg(&edge->dev, "set_state(%s, %d)\n", channel->name, state);
+
+	SET_TX_CHANNEL_FLAG(channel, fDSR, is_open);
+	SET_TX_CHANNEL_FLAG(channel, fCTS, is_open);
+	SET_TX_CHANNEL_FLAG(channel, fCD, is_open);
+
+	SET_TX_CHANNEL_INFO(channel, state, state);
+	SET_TX_CHANNEL_FLAG(channel, fSTATE, 1);
+
+	channel->state = state;
+	qcom_smd_signal_channel(channel);
+}
+
+/*
+ * Copy count bytes of data using 32bit accesses, if that's required.
+ */
+static void smd_copy_to_fifo(void __iomem *dst,
+			     const void *src,
+			     size_t count,
+			     bool word_aligned)
+{
+	if (word_aligned) {
+		__iowrite32_copy(dst, src, count / sizeof(u32));
+	} else {
+		memcpy_toio(dst, src, count);
+	}
+}
+
+/*
+ * Copy count bytes of data using 32bit accesses, if that is required.
+ */
+static void smd_copy_from_fifo(void *dst,
+			       const void __iomem *src,
+			       size_t count,
+			       bool word_aligned)
+{
+	if (word_aligned) {
+		__ioread32_copy(dst, src, count / sizeof(u32));
+	} else {
+		memcpy_fromio(dst, src, count);
+	}
+}
+
+/*
+ * Read count bytes of data from the rx fifo into buf, but don't advance the
+ * tail.
+ */
+static size_t qcom_smd_channel_peek(struct qcom_smd_channel *channel,
+				    void *buf, size_t count)
+{
+	bool word_aligned;
+	unsigned tail;
+	size_t len;
+
+	word_aligned = channel->info_word;
+	tail = GET_RX_CHANNEL_INFO(channel, tail);
+
+	len = min_t(size_t, count, channel->fifo_size - tail);
+	if (len) {
+		smd_copy_from_fifo(buf,
+				   channel->rx_fifo + tail,
+				   len,
+				   word_aligned);
+	}
+
+	if (len != count) {
+		smd_copy_from_fifo(buf + len,
+				   channel->rx_fifo,
+				   count - len,
+				   word_aligned);
+	}
+
+	return count;
+}
+
+/*
+ * Advance the rx tail by count bytes.
+ */
+static void qcom_smd_channel_advance(struct qcom_smd_channel *channel,
+				     size_t count)
+{
+	unsigned tail;
+
+	tail = GET_RX_CHANNEL_INFO(channel, tail);
+	tail += count;
+	tail &= (channel->fifo_size - 1);
+	SET_RX_CHANNEL_INFO(channel, tail, tail);
+}
+
+/*
+ * Read out a single packet from the rx fifo and deliver it to the device
+ */
+static int qcom_smd_channel_recv_single(struct qcom_smd_channel *channel)
+{
+	struct rpmsg_endpoint *ept = &channel->qsept->ept;
+	unsigned tail;
+	size_t len;
+	void *ptr;
+	int ret;
+
+	tail = GET_RX_CHANNEL_INFO(channel, tail);
+
+	/* Use bounce buffer if the data wraps */
+	if (tail + channel->pkt_size >= channel->fifo_size) {
+		ptr = channel->bounce_buffer;
+		len = qcom_smd_channel_peek(channel, ptr, channel->pkt_size);
+	} else {
+		ptr = channel->rx_fifo + tail;
+		len = channel->pkt_size;
+	}
+
+	ret = ept->cb(ept->rpdev, ptr, len, ept->priv, RPMSG_ADDR_ANY);
+	if (ret < 0)
+		return ret;
+
+	/* Only forward the tail if the client consumed the data */
+	qcom_smd_channel_advance(channel, len);
+
+	channel->pkt_size = 0;
+
+	return 0;
+}
+
+/*
+ * Per channel interrupt handling
+ */
+static bool qcom_smd_channel_intr(struct qcom_smd_channel *channel)
+{
+	bool need_state_scan = false;
+	int remote_state;
+	__le32 pktlen;
+	int avail;
+	int ret;
+
+	/* Handle state changes */
+	remote_state = GET_RX_CHANNEL_INFO(channel, state);
+	if (remote_state != channel->remote_state) {
+		channel->remote_state = remote_state;
+		need_state_scan = true;
+	}
+	/* Indicate that we have seen any state change */
+	SET_RX_CHANNEL_FLAG(channel, fSTATE, 0);
+
+	/* Signal waiting qcom_smd_send() about the interrupt */
+	if (!GET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR))
+		wake_up_interruptible(&channel->fblockread_event);
+
+	/* Don't consume any data until we've opened the channel */
+	if (channel->state != SMD_CHANNEL_OPENED)
+		goto out;
+
+	/* Indicate that we've seen the new data */
+	SET_RX_CHANNEL_FLAG(channel, fHEAD, 0);
+
+	/* Consume data */
+	for (;;) {
+		avail = qcom_smd_channel_get_rx_avail(channel);
+
+		if (!channel->pkt_size && avail >= SMD_PACKET_HEADER_LEN) {
+			qcom_smd_channel_peek(channel, &pktlen, sizeof(pktlen));
+			qcom_smd_channel_advance(channel, SMD_PACKET_HEADER_LEN);
+			channel->pkt_size = le32_to_cpu(pktlen);
+		} else if (channel->pkt_size && avail >= channel->pkt_size) {
+			ret = qcom_smd_channel_recv_single(channel);
+			if (ret)
+				break;
+		} else {
+			break;
+		}
+	}
+
+	/* Indicate that we have seen and updated tail */
+	SET_RX_CHANNEL_FLAG(channel, fTAIL, 1);
+
+	/* Signal the remote that we've consumed the data (if requested) */
+	if (!GET_RX_CHANNEL_FLAG(channel, fBLOCKREADINTR)) {
+		/* Ensure ordering of channel info updates */
+		wmb();
+
+		qcom_smd_signal_channel(channel);
+	}
+
+out:
+	return need_state_scan;
+}
+
+/*
+ * The edge interrupts are triggered by the remote processor on state changes,
+ * channel info updates or when new channels are created.
+ */
+static irqreturn_t qcom_smd_edge_intr(int irq, void *data)
+{
+	struct qcom_smd_edge *edge = data;
+	struct qcom_smd_channel *channel;
+	unsigned available;
+	bool kick_scanner = false;
+	bool kick_state = false;
+
+	/*
+	 * Handle state changes or data on each of the channels on this edge
+	 */
+	spin_lock(&edge->channels_lock);
+	list_for_each_entry(channel, &edge->channels, list) {
+		spin_lock(&channel->recv_lock);
+		kick_state |= qcom_smd_channel_intr(channel);
+		spin_unlock(&channel->recv_lock);
+	}
+	spin_unlock(&edge->channels_lock);
+
+	/*
+	 * Creating a new channel requires allocating an smem entry, so we only
+	 * have to scan if the amount of available space in smem have changed
+	 * since last scan.
+	 */
+	available = qcom_smem_get_free_space(edge->remote_pid);
+	if (available != edge->smem_available) {
+		edge->smem_available = available;
+		kick_scanner = true;
+	}
+
+	if (kick_scanner)
+		schedule_work(&edge->scan_work);
+	if (kick_state)
+		schedule_work(&edge->state_work);
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * Calculate how much space is available in the tx fifo.
+ */
+static size_t qcom_smd_get_tx_avail(struct qcom_smd_channel *channel)
+{
+	unsigned head;
+	unsigned tail;
+	unsigned mask = channel->fifo_size - 1;
+
+	head = GET_TX_CHANNEL_INFO(channel, head);
+	tail = GET_TX_CHANNEL_INFO(channel, tail);
+
+	return mask - ((head - tail) & mask);
+}
+
+/*
+ * Write count bytes of data into channel, possibly wrapping in the ring buffer
+ */
+static int qcom_smd_write_fifo(struct qcom_smd_channel *channel,
+			       const void *data,
+			       size_t count)
+{
+	bool word_aligned;
+	unsigned head;
+	size_t len;
+
+	word_aligned = channel->info_word;
+	head = GET_TX_CHANNEL_INFO(channel, head);
+
+	len = min_t(size_t, count, channel->fifo_size - head);
+	if (len) {
+		smd_copy_to_fifo(channel->tx_fifo + head,
+				 data,
+				 len,
+				 word_aligned);
+	}
+
+	if (len != count) {
+		smd_copy_to_fifo(channel->tx_fifo,
+				 data + len,
+				 count - len,
+				 word_aligned);
+	}
+
+	head += count;
+	head &= (channel->fifo_size - 1);
+	SET_TX_CHANNEL_INFO(channel, head, head);
+
+	return count;
+}
+
+/**
+ * qcom_smd_send - write data to smd channel
+ * @channel:	channel handle
+ * @data:	buffer of data to write
+ * @len:	number of bytes to write
+ *
+ * This is a blocking write of len bytes into the channel's tx ring buffer and
+ * signal the remote end. It will sleep until there is enough space available
+ * in the tx buffer, utilizing the fBLOCKREADINTR signaling mechanism to avoid
+ * polling.
+ */
+static int __qcom_smd_send(struct qcom_smd_channel *channel, const void *data,
+			   int len, bool wait)
+{
+	__le32 hdr[5] = { cpu_to_le32(len), };
+	int tlen = sizeof(hdr) + len;
+	int ret;
+
+	/* Word aligned channels only accept word size aligned data */
+	if (channel->info_word && len % 4)
+		return -EINVAL;
+
+	/* Reject packets that are too big */
+	if (tlen >= channel->fifo_size)
+		return -EINVAL;
+
+	ret = mutex_lock_interruptible(&channel->tx_lock);
+	if (ret)
+		return ret;
+
+	while (qcom_smd_get_tx_avail(channel) < tlen) {
+		if (!wait) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		if (channel->state != SMD_CHANNEL_OPENED) {
+			ret = -EPIPE;
+			goto out;
+		}
+
+		SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 0);
+
+		ret = wait_event_interruptible(channel->fblockread_event,
+				       qcom_smd_get_tx_avail(channel) >= tlen ||
+				       channel->state != SMD_CHANNEL_OPENED);
+		if (ret)
+			goto out;
+
+		SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 1);
+	}
+
+	SET_TX_CHANNEL_FLAG(channel, fTAIL, 0);
+
+	qcom_smd_write_fifo(channel, hdr, sizeof(hdr));
+	qcom_smd_write_fifo(channel, data, len);
+
+	SET_TX_CHANNEL_FLAG(channel, fHEAD, 1);
+
+	/* Ensure ordering of channel info updates */
+	wmb();
+
+	qcom_smd_signal_channel(channel);
+
+out:
+	mutex_unlock(&channel->tx_lock);
+
+	return ret;
+}
+
+/*
+ * Helper for opening a channel
+ */
+static int qcom_smd_channel_open(struct qcom_smd_channel *channel,
+				 rpmsg_rx_cb_t cb)
+{
+	size_t bb_size;
+
+	/*
+	 * Packets are maximum 4k, but reduce if the fifo is smaller
+	 */
+	bb_size = min(channel->fifo_size, SZ_4K);
+	channel->bounce_buffer = kmalloc(bb_size, GFP_KERNEL);
+	if (!channel->bounce_buffer)
+		return -ENOMEM;
+
+	qcom_smd_channel_set_callback(channel, cb);
+	qcom_smd_channel_set_state(channel, SMD_CHANNEL_OPENING);
+	qcom_smd_channel_set_state(channel, SMD_CHANNEL_OPENED);
+
+	return 0;
+}
+
+/*
+ * Helper for closing and resetting a channel
+ */
+static void qcom_smd_channel_close(struct qcom_smd_channel *channel)
+{
+	qcom_smd_channel_set_callback(channel, NULL);
+
+	kfree(channel->bounce_buffer);
+	channel->bounce_buffer = NULL;
+
+	qcom_smd_channel_set_state(channel, SMD_CHANNEL_CLOSED);
+	qcom_smd_channel_reset(channel);
+}
+
+static struct qcom_smd_channel *
+qcom_smd_find_channel(struct qcom_smd_edge *edge, const char *name)
+{
+	struct qcom_smd_channel *channel;
+	struct qcom_smd_channel *ret = NULL;
+	unsigned long flags;
+	unsigned state;
+
+	spin_lock_irqsave(&edge->channels_lock, flags);
+	list_for_each_entry(channel, &edge->channels, list) {
+		if (strcmp(channel->name, name))
+			continue;
+
+		state = GET_RX_CHANNEL_INFO(channel, state);
+		if (state != SMD_CHANNEL_OPENING &&
+		    state != SMD_CHANNEL_OPENED)
+			continue;
+
+		ret = channel;
+		break;
+	}
+	spin_unlock_irqrestore(&edge->channels_lock, flags);
+
+	return ret;
+}
+
+static void __ept_release(struct kref *kref)
+{
+	struct rpmsg_endpoint *ept = container_of(kref, struct rpmsg_endpoint,
+						  refcount);
+	kfree(to_smd_endpoint(ept));
+}
+
+static struct rpmsg_endpoint *qcom_smd_create_ept(struct rpmsg_device *rpdev,
+						  rpmsg_rx_cb_t cb, void *priv,
+						  struct rpmsg_channel_info chinfo)
+{
+	struct qcom_smd_endpoint *qsept;
+	struct qcom_smd_channel *channel;
+	struct qcom_smd_device *qsdev = to_smd_device(rpdev);
+	struct qcom_smd_edge *edge = qsdev->edge;
+	struct rpmsg_endpoint *ept;
+	const char *name = chinfo.name;
+	int ret;
+
+	/* Wait up to HZ for the channel to appear */
+	ret = wait_event_interruptible_timeout(edge->new_channel_event,
+			(channel = qcom_smd_find_channel(edge, name)) != NULL,
+			HZ);
+	if (!ret)
+		return NULL;
+
+	if (channel->state != SMD_CHANNEL_CLOSED) {
+		dev_err(&rpdev->dev, "channel %s is busy\n", channel->name);
+		return NULL;
+	}
+
+	qsept = kzalloc(sizeof(*qsept), GFP_KERNEL);
+	if (!qsept)
+		return NULL;
+
+	ept = &qsept->ept;
+
+	kref_init(&ept->refcount);
+
+	ept->rpdev = rpdev;
+	ept->cb = cb;
+	ept->priv = priv;
+	ept->ops = &qcom_smd_endpoint_ops;
+
+	channel->qsept = qsept;
+	qsept->qsch = channel;
+
+	ret = qcom_smd_channel_open(channel, cb);
+	if (ret)
+		goto free_ept;
+
+	return ept;
+
+free_ept:
+	channel->qsept = NULL;
+	kref_put(&ept->refcount, __ept_release);
+	return NULL;
+}
+
+static void qcom_smd_destroy_ept(struct rpmsg_endpoint *ept)
+{
+	struct qcom_smd_endpoint *qsept = to_smd_endpoint(ept);
+	struct qcom_smd_channel *ch = qsept->qsch;
+
+	qcom_smd_channel_close(ch);
+	ch->qsept = NULL;
+	kref_put(&ept->refcount, __ept_release);
+}
+
+static int qcom_smd_send(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	struct qcom_smd_endpoint *qsept = to_smd_endpoint(ept);
+
+	return __qcom_smd_send(qsept->qsch, data, len, true);
+}
+
+static int qcom_smd_trysend(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	struct qcom_smd_endpoint *qsept = to_smd_endpoint(ept);
+
+	return __qcom_smd_send(qsept->qsch, data, len, false);
+}
+
+/*
+ * Finds the device_node for the smd child interested in this channel.
+ */
+static struct device_node *qcom_smd_match_channel(struct device_node *edge_node,
+						  const char *channel)
+{
+	struct device_node *child;
+	const char *name;
+	const char *key;
+	int ret;
+
+	for_each_available_child_of_node(edge_node, child) {
+		key = "qcom,smd-channels";
+		ret = of_property_read_string(child, key, &name);
+		if (ret)
+			continue;
+
+		if (strcmp(name, channel) == 0)
+			return child;
+	}
+
+	return NULL;
+}
+
+static const struct rpmsg_device_ops qcom_smd_device_ops = {
+	.create_ept = qcom_smd_create_ept,
+};
+
+static const struct rpmsg_endpoint_ops qcom_smd_endpoint_ops = {
+	.destroy_ept = qcom_smd_destroy_ept,
+	.send = qcom_smd_send,
+	.trysend = qcom_smd_trysend,
+};
+
+/*
+ * Create a smd client device for channel that is being opened.
+ */
+static int qcom_smd_create_device(struct qcom_smd_channel *channel)
+{
+	struct qcom_smd_device *qsdev;
+	struct rpmsg_device *rpdev;
+	struct qcom_smd_edge *edge = channel->edge;
+
+	dev_dbg(&edge->dev, "registering '%s'\n", channel->name);
+
+	qsdev = kzalloc(sizeof(*qsdev), GFP_KERNEL);
+	if (!qsdev)
+		return -ENOMEM;
+
+	/* Link qsdev to our SMD edge */
+	qsdev->edge = edge;
+
+	/* Assign callbacks for rpmsg_device */
+	qsdev->rpdev.ops = &qcom_smd_device_ops;
+
+	/* Assign public information to the rpmsg_device */
+	rpdev = &qsdev->rpdev;
+	strncpy(rpdev->id.name, channel->name, RPMSG_NAME_SIZE);
+	rpdev->src = RPMSG_ADDR_ANY;
+	rpdev->dst = RPMSG_ADDR_ANY;
+
+	rpdev->dev.of_node = qcom_smd_match_channel(edge->of_node, channel->name);
+	rpdev->dev.parent = &edge->dev;
+
+	return rpmsg_register_device(rpdev);
+}
+
+/*
+ * Allocate the qcom_smd_channel object for a newly found smd channel,
+ * retrieving and validating the smem items involved.
+ */
+static struct qcom_smd_channel *qcom_smd_create_channel(struct qcom_smd_edge *edge,
+							unsigned smem_info_item,
+							unsigned smem_fifo_item,
+							char *name)
+{
+	struct qcom_smd_channel *channel;
+	size_t fifo_size;
+	size_t info_size;
+	void *fifo_base;
+	void *info;
+	int ret;
+
+	channel = devm_kzalloc(&edge->dev, sizeof(*channel), GFP_KERNEL);
+	if (!channel)
+		return ERR_PTR(-ENOMEM);
+
+	channel->edge = edge;
+	channel->name = devm_kstrdup(&edge->dev, name, GFP_KERNEL);
+	if (!channel->name)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&channel->tx_lock);
+	spin_lock_init(&channel->recv_lock);
+	init_waitqueue_head(&channel->fblockread_event);
+
+	info = qcom_smem_get(edge->remote_pid, smem_info_item, &info_size);
+	if (IS_ERR(info)) {
+		ret = PTR_ERR(info);
+		goto free_name_and_channel;
+	}
+
+	/*
+	 * Use the size of the item to figure out which channel info struct to
+	 * use.
+	 */
+	if (info_size == 2 * sizeof(struct smd_channel_info_word)) {
+		channel->info_word = info;
+	} else if (info_size == 2 * sizeof(struct smd_channel_info)) {
+		channel->info = info;
+	} else {
+		dev_err(&edge->dev,
+			"channel info of size %zu not supported\n", info_size);
+		ret = -EINVAL;
+		goto free_name_and_channel;
+	}
+
+	fifo_base = qcom_smem_get(edge->remote_pid, smem_fifo_item, &fifo_size);
+	if (IS_ERR(fifo_base)) {
+		ret =  PTR_ERR(fifo_base);
+		goto free_name_and_channel;
+	}
+
+	/* The channel consist of a rx and tx fifo of equal size */
+	fifo_size /= 2;
+
+	dev_dbg(&edge->dev, "new channel '%s' info-size: %zu fifo-size: %zu\n",
+			  name, info_size, fifo_size);
+
+	channel->tx_fifo = fifo_base;
+	channel->rx_fifo = fifo_base + fifo_size;
+	channel->fifo_size = fifo_size;
+
+	qcom_smd_channel_reset(channel);
+
+	return channel;
+
+free_name_and_channel:
+	devm_kfree(&edge->dev, channel->name);
+	devm_kfree(&edge->dev, channel);
+
+	return ERR_PTR(ret);
+}
+
+/*
+ * Scans the allocation table for any newly allocated channels, calls
+ * qcom_smd_create_channel() to create representations of these and add
+ * them to the edge's list of channels.
+ */
+static void qcom_channel_scan_worker(struct work_struct *work)
+{
+	struct qcom_smd_edge *edge = container_of(work, struct qcom_smd_edge, scan_work);
+	struct qcom_smd_alloc_entry *alloc_tbl;
+	struct qcom_smd_alloc_entry *entry;
+	struct qcom_smd_channel *channel;
+	unsigned long flags;
+	unsigned fifo_id;
+	unsigned info_id;
+	int tbl;
+	int i;
+	u32 eflags, cid;
+
+	for (tbl = 0; tbl < SMD_ALLOC_TBL_COUNT; tbl++) {
+		alloc_tbl = qcom_smem_get(edge->remote_pid,
+				    smem_items[tbl].alloc_tbl_id, NULL);
+		if (IS_ERR(alloc_tbl))
+			continue;
+
+		for (i = 0; i < SMD_ALLOC_TBL_SIZE; i++) {
+			entry = &alloc_tbl[i];
+			eflags = le32_to_cpu(entry->flags);
+			if (test_bit(i, edge->allocated[tbl]))
+				continue;
+
+			if (entry->ref_count == 0)
+				continue;
+
+			if (!entry->name[0])
+				continue;
+
+			if (!(eflags & SMD_CHANNEL_FLAGS_PACKET))
+				continue;
+
+			if ((eflags & SMD_CHANNEL_FLAGS_EDGE_MASK) != edge->edge_id)
+				continue;
+
+			cid = le32_to_cpu(entry->cid);
+			info_id = smem_items[tbl].info_base_id + cid;
+			fifo_id = smem_items[tbl].fifo_base_id + cid;
+
+			channel = qcom_smd_create_channel(edge, info_id, fifo_id, entry->name);
+			if (IS_ERR(channel))
+				continue;
+
+			spin_lock_irqsave(&edge->channels_lock, flags);
+			list_add(&channel->list, &edge->channels);
+			spin_unlock_irqrestore(&edge->channels_lock, flags);
+
+			dev_dbg(&edge->dev, "new channel found: '%s'\n", channel->name);
+			set_bit(i, edge->allocated[tbl]);
+
+			wake_up_interruptible(&edge->new_channel_event);
+		}
+	}
+
+	schedule_work(&edge->state_work);
+}
+
+/*
+ * This per edge worker scans smem for any new channels and register these. It
+ * then scans all registered channels for state changes that should be handled
+ * by creating or destroying smd client devices for the registered channels.
+ *
+ * LOCKING: edge->channels_lock only needs to cover the list operations, as the
+ * worker is killed before any channels are deallocated
+ */
+static void qcom_channel_state_worker(struct work_struct *work)
+{
+	struct qcom_smd_channel *channel;
+	struct qcom_smd_edge *edge = container_of(work,
+						  struct qcom_smd_edge,
+						  state_work);
+	struct rpmsg_channel_info chinfo;
+	unsigned remote_state;
+	unsigned long flags;
+
+	/*
+	 * Register a device for any closed channel where the remote processor
+	 * is showing interest in opening the channel.
+	 */
+	spin_lock_irqsave(&edge->channels_lock, flags);
+	list_for_each_entry(channel, &edge->channels, list) {
+		if (channel->state != SMD_CHANNEL_CLOSED)
+			continue;
+
+		remote_state = GET_RX_CHANNEL_INFO(channel, state);
+		if (remote_state != SMD_CHANNEL_OPENING &&
+		    remote_state != SMD_CHANNEL_OPENED)
+			continue;
+
+		if (channel->registered)
+			continue;
+
+		spin_unlock_irqrestore(&edge->channels_lock, flags);
+		qcom_smd_create_device(channel);
+		channel->registered = true;
+		spin_lock_irqsave(&edge->channels_lock, flags);
+
+		channel->registered = true;
+	}
+
+	/*
+	 * Unregister the device for any channel that is opened where the
+	 * remote processor is closing the channel.
+	 */
+	list_for_each_entry(channel, &edge->channels, list) {
+		if (channel->state != SMD_CHANNEL_OPENING &&
+		    channel->state != SMD_CHANNEL_OPENED)
+			continue;
+
+		remote_state = GET_RX_CHANNEL_INFO(channel, state);
+		if (remote_state == SMD_CHANNEL_OPENING ||
+		    remote_state == SMD_CHANNEL_OPENED)
+			continue;
+
+		spin_unlock_irqrestore(&edge->channels_lock, flags);
+
+		strncpy(chinfo.name, channel->name, sizeof(chinfo.name));
+		chinfo.src = RPMSG_ADDR_ANY;
+		chinfo.dst = RPMSG_ADDR_ANY;
+		rpmsg_unregister_device(&edge->dev, &chinfo);
+		channel->registered = false;
+		spin_lock_irqsave(&edge->channels_lock, flags);
+	}
+	spin_unlock_irqrestore(&edge->channels_lock, flags);
+}
+
+/*
+ * Parses an of_node describing an edge.
+ */
+static int qcom_smd_parse_edge(struct device *dev,
+			       struct device_node *node,
+			       struct qcom_smd_edge *edge)
+{
+	struct device_node *syscon_np;
+	const char *key;
+	int irq;
+	int ret;
+
+	INIT_LIST_HEAD(&edge->channels);
+	spin_lock_init(&edge->channels_lock);
+
+	INIT_WORK(&edge->scan_work, qcom_channel_scan_worker);
+	INIT_WORK(&edge->state_work, qcom_channel_state_worker);
+
+	edge->of_node = of_node_get(node);
+
+	key = "qcom,smd-edge";
+	ret = of_property_read_u32(node, key, &edge->edge_id);
+	if (ret) {
+		dev_err(dev, "edge missing %s property\n", key);
+		return -EINVAL;
+	}
+
+	edge->remote_pid = QCOM_SMEM_HOST_ANY;
+	key = "qcom,remote-pid";
+	of_property_read_u32(node, key, &edge->remote_pid);
+
+	syscon_np = of_parse_phandle(node, "qcom,ipc", 0);
+	if (!syscon_np) {
+		dev_err(dev, "no qcom,ipc node\n");
+		return -ENODEV;
+	}
+
+	edge->ipc_regmap = syscon_node_to_regmap(syscon_np);
+	if (IS_ERR(edge->ipc_regmap))
+		return PTR_ERR(edge->ipc_regmap);
+
+	key = "qcom,ipc";
+	ret = of_property_read_u32_index(node, key, 1, &edge->ipc_offset);
+	if (ret < 0) {
+		dev_err(dev, "no offset in %s\n", key);
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32_index(node, key, 2, &edge->ipc_bit);
+	if (ret < 0) {
+		dev_err(dev, "no bit in %s\n", key);
+		return -EINVAL;
+	}
+
+	irq = irq_of_parse_and_map(node, 0);
+	if (irq < 0) {
+		dev_err(dev, "required smd interrupt missing\n");
+		return -EINVAL;
+	}
+
+	ret = devm_request_irq(dev, irq,
+			       qcom_smd_edge_intr, IRQF_TRIGGER_RISING,
+			       node->name, edge);
+	if (ret) {
+		dev_err(dev, "failed to request smd irq\n");
+		return ret;
+	}
+
+	edge->irq = irq;
+
+	return 0;
+}
+
+/*
+ * Release function for an edge.
+  * Reset the state of each associated channel and free the edge context.
+ */
+static void qcom_smd_edge_release(struct device *dev)
+{
+	struct qcom_smd_channel *channel;
+	struct qcom_smd_edge *edge = to_smd_edge(dev);
+
+	list_for_each_entry(channel, &edge->channels, list) {
+		SET_RX_CHANNEL_INFO(channel, state, SMD_CHANNEL_CLOSED);
+		SET_RX_CHANNEL_INFO(channel, head, 0);
+		SET_RX_CHANNEL_INFO(channel, tail, 0);
+	}
+
+	kfree(edge);
+}
+
+/**
+ * qcom_smd_register_edge() - register an edge based on an device_node
+ * @parent:    parent device for the edge
+ * @node:      device_node describing the edge
+ *
+ * Returns an edge reference, or negative ERR_PTR() on failure.
+ */
+struct qcom_smd_edge *qcom_smd_register_edge(struct device *parent,
+					     struct device_node *node)
+{
+	struct qcom_smd_edge *edge;
+	int ret;
+
+	edge = kzalloc(sizeof(*edge), GFP_KERNEL);
+	if (!edge)
+		return ERR_PTR(-ENOMEM);
+
+	init_waitqueue_head(&edge->new_channel_event);
+
+	edge->dev.parent = parent;
+	edge->dev.release = qcom_smd_edge_release;
+	dev_set_name(&edge->dev, "%s:%s", dev_name(parent), node->name);
+	ret = device_register(&edge->dev);
+	if (ret) {
+		pr_err("failed to register smd edge\n");
+		return ERR_PTR(ret);
+	}
+
+	ret = qcom_smd_parse_edge(&edge->dev, node, edge);
+	if (ret) {
+		dev_err(&edge->dev, "failed to parse smd edge\n");
+		goto unregister_dev;
+	}
+
+	schedule_work(&edge->scan_work);
+
+	return edge;
+
+unregister_dev:
+	put_device(&edge->dev);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(qcom_smd_register_edge);
+
+static int qcom_smd_remove_device(struct device *dev, void *data)
+{
+	device_unregister(dev);
+
+	return 0;
+}
+
+/**
+ * qcom_smd_unregister_edge() - release an edge and its children
+ * @edge:      edge reference acquired from qcom_smd_register_edge
+ */
+int qcom_smd_unregister_edge(struct qcom_smd_edge *edge)
+{
+	int ret;
+
+	disable_irq(edge->irq);
+	cancel_work_sync(&edge->scan_work);
+	cancel_work_sync(&edge->state_work);
+
+	ret = device_for_each_child(&edge->dev, NULL, qcom_smd_remove_device);
+	if (ret)
+		dev_warn(&edge->dev, "can't remove smd device: %d\n", ret);
+
+	device_unregister(&edge->dev);
+
+	return 0;
+}
+EXPORT_SYMBOL(qcom_smd_unregister_edge);
+
+static int qcom_smd_probe(struct platform_device *pdev)
+{
+	struct device_node *node;
+	void *p;
+
+	/* Wait for smem */
+	p = qcom_smem_get(QCOM_SMEM_HOST_ANY, smem_items[0].alloc_tbl_id, NULL);
+	if (PTR_ERR(p) == -EPROBE_DEFER)
+		return PTR_ERR(p);
+
+	for_each_available_child_of_node(pdev->dev.of_node, node)
+		qcom_smd_register_edge(&pdev->dev, node);
+
+	return 0;
+}
+
+static int qcom_smd_remove_edge(struct device *dev, void *data)
+{
+	struct qcom_smd_edge *edge = to_smd_edge(dev);
+
+	return qcom_smd_unregister_edge(edge);
+}
+
+/*
+ * Shut down all smd clients by making sure that each edge stops processing
+ * events and scanning for new channels, then call destroy on the devices.
+ */
+static int qcom_smd_remove(struct platform_device *pdev)
+{
+	int ret;
+
+	ret = device_for_each_child(&pdev->dev, NULL, qcom_smd_remove_edge);
+	if (ret)
+		dev_warn(&pdev->dev, "can't remove smd device: %d\n", ret);
+
+	return ret;
+}
+
+static const struct of_device_id qcom_smd_of_match[] = {
+	{ .compatible = "qcom,smd" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, qcom_smd_of_match);
+
+static struct platform_driver qcom_smd_driver = {
+	.probe = qcom_smd_probe,
+	.remove = qcom_smd_remove,
+	.driver = {
+		.name = "qcom-smd",
+		.of_match_table = qcom_smd_of_match,
+	},
+};
+
+static int __init qcom_smd_init(void)
+{
+	return platform_driver_register(&qcom_smd_driver);
+}
+subsys_initcall(qcom_smd_init);
+
+static void __exit qcom_smd_exit(void)
+{
+	platform_driver_unregister(&qcom_smd_driver);
+}
+module_exit(qcom_smd_exit);
+
+MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@sonymobile.com>");
+MODULE_DESCRIPTION("Qualcomm Shared Memory Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
new file mode 100644
index 000000000000..b6ea9ffa7381
--- /dev/null
+++ b/drivers/rpmsg/rpmsg_core.c
@@ -0,0 +1,498 @@
+/*
+ * remote processor messaging bus
+ *
+ * Copyright (C) 2011 Texas Instruments, Inc.
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * Ohad Ben-Cohen <ohad@wizery.com>
+ * Brian Swetland <swetland@google.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) "%s: " fmt, __func__
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/rpmsg.h>
+#include <linux/of_device.h>
+#include <linux/slab.h>
+
+#include "rpmsg_internal.h"
+
+/**
+ * rpmsg_create_ept() - create a new rpmsg_endpoint
+ * @rpdev: rpmsg channel device
+ * @cb: rx callback handler
+ * @priv: private data for the driver's use
+ * @chinfo: channel_info with the local rpmsg address to bind with @cb
+ *
+ * Every rpmsg address in the system is bound to an rx callback (so when
+ * inbound messages arrive, they are dispatched by the rpmsg bus using the
+ * appropriate callback handler) by means of an rpmsg_endpoint struct.
+ *
+ * This function allows drivers to create such an endpoint, and by that,
+ * bind a callback, and possibly some private data too, to an rpmsg address
+ * (either one that is known in advance, or one that will be dynamically
+ * assigned for them).
+ *
+ * Simple rpmsg drivers need not call rpmsg_create_ept, because an endpoint
+ * is already created for them when they are probed by the rpmsg bus
+ * (using the rx callback provided when they registered to the rpmsg bus).
+ *
+ * So things should just work for simple drivers: they already have an
+ * endpoint, their rx callback is bound to their rpmsg address, and when
+ * relevant inbound messages arrive (i.e. messages which their dst address
+ * equals to the src address of their rpmsg channel), the driver's handler
+ * is invoked to process it.
+ *
+ * That said, more complicated drivers might do need to allocate
+ * additional rpmsg addresses, and bind them to different rx callbacks.
+ * To accomplish that, those drivers need to call this function.
+ *
+ * Drivers should provide their @rpdev channel (so the new endpoint would belong
+ * to the same remote processor their channel belongs to), an rx callback
+ * function, an optional private data (which is provided back when the
+ * rx callback is invoked), and an address they want to bind with the
+ * callback. If @addr is RPMSG_ADDR_ANY, then rpmsg_create_ept will
+ * dynamically assign them an available rpmsg address (drivers should have
+ * a very good reason why not to always use RPMSG_ADDR_ANY here).
+ *
+ * Returns a pointer to the endpoint on success, or NULL on error.
+ */
+struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_device *rpdev,
+					rpmsg_rx_cb_t cb, void *priv,
+					struct rpmsg_channel_info chinfo)
+{
+	return rpdev->ops->create_ept(rpdev, cb, priv, chinfo);
+}
+EXPORT_SYMBOL(rpmsg_create_ept);
+
+/**
+ * rpmsg_destroy_ept() - destroy an existing rpmsg endpoint
+ * @ept: endpoing to destroy
+ *
+ * Should be used by drivers to destroy an rpmsg endpoint previously
+ * created with rpmsg_create_ept().
+ */
+void rpmsg_destroy_ept(struct rpmsg_endpoint *ept)
+{
+	ept->ops->destroy_ept(ept);
+}
+EXPORT_SYMBOL(rpmsg_destroy_ept);
+
+/**
+ * rpmsg_send() - send a message across to the remote processor
+ * @ept: the rpmsg endpoint
+ * @data: payload of message
+ * @len: length of payload
+ *
+ * This function sends @data of length @len on the @ept endpoint.
+ * The message will be sent to the remote processor which the @ept
+ * endpoint belongs to, using @ept's address and its associated rpmsg
+ * device destination addresses.
+ * In case there are no TX buffers available, the function will block until
+ * one becomes available, or a timeout of 15 seconds elapses. When the latter
+ * happens, -ERESTARTSYS is returned.
+ *
+ * Can only be called from process context (for now).
+ *
+ * Returns 0 on success and an appropriate error value on failure.
+ */
+int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	return ept->ops->send(ept, data, len);
+}
+EXPORT_SYMBOL(rpmsg_send);
+
+/**
+ * rpmsg_sendto() - send a message across to the remote processor, specify dst
+ * @ept: the rpmsg endpoint
+ * @data: payload of message
+ * @len: length of payload
+ * @dst: destination address
+ *
+ * This function sends @data of length @len to the remote @dst address.
+ * The message will be sent to the remote processor which the @ept
+ * endpoint belongs to, using @ept's address as source.
+ * In case there are no TX buffers available, the function will block until
+ * one becomes available, or a timeout of 15 seconds elapses. When the latter
+ * happens, -ERESTARTSYS is returned.
+ *
+ * Can only be called from process context (for now).
+ *
+ * Returns 0 on success and an appropriate error value on failure.
+ */
+int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
+{
+	return ept->ops->sendto(ept, data, len, dst);
+}
+EXPORT_SYMBOL(rpmsg_sendto);
+
+/**
+ * rpmsg_send_offchannel() - send a message using explicit src/dst addresses
+ * @ept: the rpmsg endpoint
+ * @src: source address
+ * @dst: destination address
+ * @data: payload of message
+ * @len: length of payload
+ *
+ * This function sends @data of length @len to the remote @dst address,
+ * and uses @src as the source address.
+ * The message will be sent to the remote processor which the @ept
+ * endpoint belongs to.
+ * In case there are no TX buffers available, the function will block until
+ * one becomes available, or a timeout of 15 seconds elapses. When the latter
+ * happens, -ERESTARTSYS is returned.
+ *
+ * Can only be called from process context (for now).
+ *
+ * Returns 0 on success and an appropriate error value on failure.
+ */
+int rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst,
+			  void *data, int len)
+{
+	return ept->ops->send_offchannel(ept, src, dst, data, len);
+}
+EXPORT_SYMBOL(rpmsg_send_offchannel);
+
+/**
+ * rpmsg_send() - send a message across to the remote processor
+ * @ept: the rpmsg endpoint
+ * @data: payload of message
+ * @len: length of payload
+ *
+ * This function sends @data of length @len on the @ept endpoint.
+ * The message will be sent to the remote processor which the @ept
+ * endpoint belongs to, using @ept's address as source and its associated
+ * rpdev's address as destination.
+ * In case there are no TX buffers available, the function will immediately
+ * return -ENOMEM without waiting until one becomes available.
+ *
+ * Can only be called from process context (for now).
+ *
+ * Returns 0 on success and an appropriate error value on failure.
+ */
+int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	return ept->ops->trysend(ept, data, len);
+}
+EXPORT_SYMBOL(rpmsg_trysend);
+
+/**
+ * rpmsg_sendto() - send a message across to the remote processor, specify dst
+ * @ept: the rpmsg endpoint
+ * @data: payload of message
+ * @len: length of payload
+ * @dst: destination address
+ *
+ * This function sends @data of length @len to the remote @dst address.
+ * The message will be sent to the remote processor which the @ept
+ * endpoint belongs to, using @ept's address as source.
+ * In case there are no TX buffers available, the function will immediately
+ * return -ENOMEM without waiting until one becomes available.
+ *
+ * Can only be called from process context (for now).
+ *
+ * Returns 0 on success and an appropriate error value on failure.
+ */
+int rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
+{
+	return ept->ops->trysendto(ept, data, len, dst);
+}
+EXPORT_SYMBOL(rpmsg_trysendto);
+
+/**
+ * rpmsg_send_offchannel() - send a message using explicit src/dst addresses
+ * @ept: the rpmsg endpoint
+ * @src: source address
+ * @dst: destination address
+ * @data: payload of message
+ * @len: length of payload
+ *
+ * This function sends @data of length @len to the remote @dst address,
+ * and uses @src as the source address.
+ * The message will be sent to the remote processor which the @ept
+ * endpoint belongs to.
+ * In case there are no TX buffers available, the function will immediately
+ * return -ENOMEM without waiting until one becomes available.
+ *
+ * Can only be called from process context (for now).
+ *
+ * Returns 0 on success and an appropriate error value on failure.
+ */
+int rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst,
+			     void *data, int len)
+{
+	return ept->ops->trysend_offchannel(ept, src, dst, data, len);
+}
+EXPORT_SYMBOL(rpmsg_trysend_offchannel);
+
+/*
+ * match an rpmsg channel with a channel info struct.
+ * this is used to make sure we're not creating rpmsg devices for channels
+ * that already exist.
+ */
+static int rpmsg_device_match(struct device *dev, void *data)
+{
+	struct rpmsg_channel_info *chinfo = data;
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+
+	if (chinfo->src != RPMSG_ADDR_ANY && chinfo->src != rpdev->src)
+		return 0;
+
+	if (chinfo->dst != RPMSG_ADDR_ANY && chinfo->dst != rpdev->dst)
+		return 0;
+
+	if (strncmp(chinfo->name, rpdev->id.name, RPMSG_NAME_SIZE))
+		return 0;
+
+	/* found a match ! */
+	return 1;
+}
+
+struct device *rpmsg_find_device(struct device *parent,
+				 struct rpmsg_channel_info *chinfo)
+{
+	return device_find_child(parent, chinfo, rpmsg_device_match);
+
+}
+EXPORT_SYMBOL(rpmsg_find_device);
+
+/* sysfs show configuration fields */
+#define rpmsg_show_attr(field, path, format_string)			\
+static ssize_t								\
+field##_show(struct device *dev,					\
+			struct device_attribute *attr, char *buf)	\
+{									\
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);		\
+									\
+	return sprintf(buf, format_string, rpdev->path);		\
+}
+
+/* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */
+rpmsg_show_attr(name, id.name, "%s\n");
+rpmsg_show_attr(src, src, "0x%x\n");
+rpmsg_show_attr(dst, dst, "0x%x\n");
+rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n");
+
+static ssize_t modalias_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+
+	return sprintf(buf, RPMSG_DEVICE_MODALIAS_FMT "\n", rpdev->id.name);
+}
+
+static struct device_attribute rpmsg_dev_attrs[] = {
+	__ATTR_RO(name),
+	__ATTR_RO(modalias),
+	__ATTR_RO(dst),
+	__ATTR_RO(src),
+	__ATTR_RO(announce),
+	__ATTR_NULL
+};
+
+/* rpmsg devices and drivers are matched using the service name */
+static inline int rpmsg_id_match(const struct rpmsg_device *rpdev,
+				  const struct rpmsg_device_id *id)
+{
+	return strncmp(id->name, rpdev->id.name, RPMSG_NAME_SIZE) == 0;
+}
+
+/* match rpmsg channel and rpmsg driver */
+static int rpmsg_dev_match(struct device *dev, struct device_driver *drv)
+{
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+	struct rpmsg_driver *rpdrv = to_rpmsg_driver(drv);
+	const struct rpmsg_device_id *ids = rpdrv->id_table;
+	unsigned int i;
+
+	if (ids)
+		for (i = 0; ids[i].name[0]; i++)
+			if (rpmsg_id_match(rpdev, &ids[i]))
+				return 1;
+
+	return of_driver_match_device(dev, drv);
+}
+
+static int rpmsg_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+
+	return add_uevent_var(env, "MODALIAS=" RPMSG_DEVICE_MODALIAS_FMT,
+					rpdev->id.name);
+}
+
+/*
+ * when an rpmsg driver is probed with a channel, we seamlessly create
+ * it an endpoint, binding its rx callback to a unique local rpmsg
+ * address.
+ *
+ * if we need to, we also announce about this channel to the remote
+ * processor (needed in case the driver is exposing an rpmsg service).
+ */
+static int rpmsg_dev_probe(struct device *dev)
+{
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+	struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver);
+	struct rpmsg_channel_info chinfo = {};
+	struct rpmsg_endpoint *ept;
+	int err;
+
+	strncpy(chinfo.name, rpdev->id.name, RPMSG_NAME_SIZE);
+	chinfo.src = rpdev->src;
+	chinfo.dst = RPMSG_ADDR_ANY;
+
+	ept = rpmsg_create_ept(rpdev, rpdrv->callback, NULL, chinfo);
+	if (!ept) {
+		dev_err(dev, "failed to create endpoint\n");
+		err = -ENOMEM;
+		goto out;
+	}
+
+	rpdev->ept = ept;
+	rpdev->src = ept->addr;
+
+	err = rpdrv->probe(rpdev);
+	if (err) {
+		dev_err(dev, "%s: failed: %d\n", __func__, err);
+		rpmsg_destroy_ept(ept);
+		goto out;
+	}
+
+	if (rpdev->ops->announce_create)
+		err = rpdev->ops->announce_create(rpdev);
+out:
+	return err;
+}
+
+static int rpmsg_dev_remove(struct device *dev)
+{
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+	struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver);
+	int err = 0;
+
+	if (rpdev->ops->announce_destroy)
+		err = rpdev->ops->announce_destroy(rpdev);
+
+	rpdrv->remove(rpdev);
+
+	rpmsg_destroy_ept(rpdev->ept);
+
+	return err;
+}
+
+static struct bus_type rpmsg_bus = {
+	.name		= "rpmsg",
+	.match		= rpmsg_dev_match,
+	.dev_attrs	= rpmsg_dev_attrs,
+	.uevent		= rpmsg_uevent,
+	.probe		= rpmsg_dev_probe,
+	.remove		= rpmsg_dev_remove,
+};
+
+static void rpmsg_release_device(struct device *dev)
+{
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+
+	kfree(rpdev);
+}
+
+int rpmsg_register_device(struct rpmsg_device *rpdev)
+{
+	struct device *dev = &rpdev->dev;
+	int ret;
+
+	dev_set_name(&rpdev->dev, "%s:%s",
+		     dev_name(dev->parent), rpdev->id.name);
+
+	rpdev->dev.bus = &rpmsg_bus;
+	rpdev->dev.release = rpmsg_release_device;
+
+	ret = device_register(&rpdev->dev);
+	if (ret) {
+		dev_err(dev, "device_register failed: %d\n", ret);
+		put_device(&rpdev->dev);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(rpmsg_register_device);
+
+/*
+ * find an existing channel using its name + address properties,
+ * and destroy it
+ */
+int rpmsg_unregister_device(struct device *parent,
+			    struct rpmsg_channel_info *chinfo)
+{
+	struct device *dev;
+
+	dev = rpmsg_find_device(parent, chinfo);
+	if (!dev)
+		return -EINVAL;
+
+	device_unregister(dev);
+
+	put_device(dev);
+
+	return 0;
+}
+EXPORT_SYMBOL(rpmsg_unregister_device);
+
+/**
+ * __register_rpmsg_driver() - register an rpmsg driver with the rpmsg bus
+ * @rpdrv: pointer to a struct rpmsg_driver
+ * @owner: owning module/driver
+ *
+ * Returns 0 on success, and an appropriate error value on failure.
+ */
+int __register_rpmsg_driver(struct rpmsg_driver *rpdrv, struct module *owner)
+{
+	rpdrv->drv.bus = &rpmsg_bus;
+	rpdrv->drv.owner = owner;
+	return driver_register(&rpdrv->drv);
+}
+EXPORT_SYMBOL(__register_rpmsg_driver);
+
+/**
+ * unregister_rpmsg_driver() - unregister an rpmsg driver from the rpmsg bus
+ * @rpdrv: pointer to a struct rpmsg_driver
+ *
+ * Returns 0 on success, and an appropriate error value on failure.
+ */
+void unregister_rpmsg_driver(struct rpmsg_driver *rpdrv)
+{
+	driver_unregister(&rpdrv->drv);
+}
+EXPORT_SYMBOL(unregister_rpmsg_driver);
+
+
+static int __init rpmsg_init(void)
+{
+	int ret;
+
+	ret = bus_register(&rpmsg_bus);
+	if (ret)
+		pr_err("failed to register rpmsg bus: %d\n", ret);
+
+	return ret;
+}
+postcore_initcall(rpmsg_init);
+
+static void __exit rpmsg_fini(void)
+{
+	bus_unregister(&rpmsg_bus);
+}
+module_exit(rpmsg_fini);
+
+MODULE_DESCRIPTION("remote processor messaging bus");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/rpmsg/rpmsg_internal.h b/drivers/rpmsg/rpmsg_internal.h
new file mode 100644
index 000000000000..8075a20f919b
--- /dev/null
+++ b/drivers/rpmsg/rpmsg_internal.h
@@ -0,0 +1,82 @@
+/*
+ * remote processor messaging bus internals
+ *
+ * Copyright (C) 2011 Texas Instruments, Inc.
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * Ohad Ben-Cohen <ohad@wizery.com>
+ * Brian Swetland <swetland@google.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __RPMSG_INTERNAL_H__
+#define __RPMSG_INTERNAL_H__
+
+#include <linux/rpmsg.h>
+
+#define to_rpmsg_device(d) container_of(d, struct rpmsg_device, dev)
+#define to_rpmsg_driver(d) container_of(d, struct rpmsg_driver, drv)
+
+/**
+ * struct rpmsg_device_ops - indirection table for the rpmsg_device operations
+ * @create_ept:		create backend-specific endpoint, requried
+ * @announce_create:	announce presence of new channel, optional
+ * @announce_destroy:	announce destruction of channel, optional
+ *
+ * Indirection table for the operations that a rpmsg backend should implement.
+ * @announce_create and @announce_destroy are optional as the backend might
+ * advertise new channels implicitly by creating the endpoints.
+ */
+struct rpmsg_device_ops {
+	struct rpmsg_endpoint *(*create_ept)(struct rpmsg_device *rpdev,
+					    rpmsg_rx_cb_t cb, void *priv,
+					    struct rpmsg_channel_info chinfo);
+
+	int (*announce_create)(struct rpmsg_device *ept);
+	int (*announce_destroy)(struct rpmsg_device *ept);
+};
+
+/**
+ * struct rpmsg_endpoint_ops - indirection table for rpmsg_endpoint operations
+ * @destroy_ept:	destroy the given endpoint, required
+ * @send:		see @rpmsg_send(), required
+ * @sendto:		see @rpmsg_sendto(), optional
+ * @send_offchannel:	see @rpmsg_send_offchannel(), optional
+ * @trysend:		see @rpmsg_trysend(), required
+ * @trysendto:		see @rpmsg_trysendto(), optional
+ * @trysend_offchannel:	see @rpmsg_trysend_offchannel(), optional
+ *
+ * Indirection table for the operations that a rpmsg backend should implement.
+ * In addition to @destroy_ept, the backend must at least implement @send and
+ * @trysend, while the variants sending data off-channel are optional.
+ */
+struct rpmsg_endpoint_ops {
+	void (*destroy_ept)(struct rpmsg_endpoint *ept);
+
+	int (*send)(struct rpmsg_endpoint *ept, void *data, int len);
+	int (*sendto)(struct rpmsg_endpoint *ept, void *data, int len, u32 dst);
+	int (*send_offchannel)(struct rpmsg_endpoint *ept, u32 src, u32 dst,
+				  void *data, int len);
+
+	int (*trysend)(struct rpmsg_endpoint *ept, void *data, int len);
+	int (*trysendto)(struct rpmsg_endpoint *ept, void *data, int len, u32 dst);
+	int (*trysend_offchannel)(struct rpmsg_endpoint *ept, u32 src, u32 dst,
+			     void *data, int len);
+};
+
+int rpmsg_register_device(struct rpmsg_device *rpdev);
+int rpmsg_unregister_device(struct device *parent,
+			    struct rpmsg_channel_info *chinfo);
+
+struct device *rpmsg_find_device(struct device *parent,
+				 struct rpmsg_channel_info *chinfo);
+
+#endif
diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c
index fe03b2aef450..3090b0d3072f 100644
--- a/drivers/rpmsg/virtio_rpmsg_bus.c
+++ b/drivers/rpmsg/virtio_rpmsg_bus.c
@@ -33,6 +33,9 @@
 #include <linux/wait.h>
 #include <linux/rpmsg.h>
 #include <linux/mutex.h>
+#include <linux/of_device.h>
+
+#include "rpmsg_internal.h"
 
 /**
  * struct virtproc_info - virtual remote processor state
@@ -72,20 +75,69 @@ struct virtproc_info {
 	struct rpmsg_endpoint *ns_ept;
 };
 
+/* The feature bitmap for virtio rpmsg */
+#define VIRTIO_RPMSG_F_NS	0 /* RP supports name service notifications */
+
 /**
- * struct rpmsg_channel_info - internal channel info representation
- * @name: name of service
- * @src: local address
+ * struct rpmsg_hdr - common header for all rpmsg messages
+ * @src: source address
  * @dst: destination address
+ * @reserved: reserved for future use
+ * @len: length of payload (in bytes)
+ * @flags: message flags
+ * @data: @len bytes of message payload data
+ *
+ * Every message sent(/received) on the rpmsg bus begins with this header.
  */
-struct rpmsg_channel_info {
-	char name[RPMSG_NAME_SIZE];
+struct rpmsg_hdr {
 	u32 src;
 	u32 dst;
+	u32 reserved;
+	u16 len;
+	u16 flags;
+	u8 data[0];
+} __packed;
+
+/**
+ * struct rpmsg_ns_msg - dynamic name service announcement message
+ * @name: name of remote service that is published
+ * @addr: address of remote service that is published
+ * @flags: indicates whether service is created or destroyed
+ *
+ * This message is sent across to publish a new service, or announce
+ * about its removal. When we receive these messages, an appropriate
+ * rpmsg channel (i.e device) is created/destroyed. In turn, the ->probe()
+ * or ->remove() handler of the appropriate rpmsg driver will be invoked
+ * (if/as-soon-as one is registered).
+ */
+struct rpmsg_ns_msg {
+	char name[RPMSG_NAME_SIZE];
+	u32 addr;
+	u32 flags;
+} __packed;
+
+/**
+ * enum rpmsg_ns_flags - dynamic name service announcement flags
+ *
+ * @RPMSG_NS_CREATE: a new remote service was just created
+ * @RPMSG_NS_DESTROY: a known remote service was just destroyed
+ */
+enum rpmsg_ns_flags {
+	RPMSG_NS_CREATE		= 0,
+	RPMSG_NS_DESTROY	= 1,
+};
+
+/**
+ * @vrp: the remote processor this channel belongs to
+ */
+struct virtio_rpmsg_channel {
+	struct rpmsg_device rpdev;
+
+	struct virtproc_info *vrp;
 };
 
-#define to_rpmsg_channel(d) container_of(d, struct rpmsg_channel, dev)
-#define to_rpmsg_driver(d) container_of(d, struct rpmsg_driver, drv)
+#define to_virtio_rpmsg_channel(_rpdev) \
+	container_of(_rpdev, struct virtio_rpmsg_channel, rpdev)
 
 /*
  * We're allocating buffers of 512 bytes each for communications. The
@@ -118,78 +170,28 @@ struct rpmsg_channel_info {
 /* Address 53 is reserved for advertising remote services */
 #define RPMSG_NS_ADDR			(53)
 
-/* sysfs show configuration fields */
-#define rpmsg_show_attr(field, path, format_string)			\
-static ssize_t								\
-field##_show(struct device *dev,					\
-			struct device_attribute *attr, char *buf)	\
-{									\
-	struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);		\
-									\
-	return sprintf(buf, format_string, rpdev->path);		\
-}
-
-/* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */
-rpmsg_show_attr(name, id.name, "%s\n");
-rpmsg_show_attr(src, src, "0x%x\n");
-rpmsg_show_attr(dst, dst, "0x%x\n");
-rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n");
-
-/*
- * Unique (and free running) index for rpmsg devices.
- *
- * Yeah, we're not recycling those numbers (yet?). will be easy
- * to change if/when we want to.
- */
-static unsigned int rpmsg_dev_index;
-
-static ssize_t modalias_show(struct device *dev,
-			     struct device_attribute *attr, char *buf)
-{
-	struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);
-
-	return sprintf(buf, RPMSG_DEVICE_MODALIAS_FMT "\n", rpdev->id.name);
-}
-
-static struct device_attribute rpmsg_dev_attrs[] = {
-	__ATTR_RO(name),
-	__ATTR_RO(modalias),
-	__ATTR_RO(dst),
-	__ATTR_RO(src),
-	__ATTR_RO(announce),
-	__ATTR_NULL
+static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept);
+static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len);
+static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len,
+			       u32 dst);
+static int virtio_rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src,
+					u32 dst, void *data, int len);
+static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len);
+static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data,
+				  int len, u32 dst);
+static int virtio_rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src,
+					   u32 dst, void *data, int len);
+
+static const struct rpmsg_endpoint_ops virtio_endpoint_ops = {
+	.destroy_ept = virtio_rpmsg_destroy_ept,
+	.send = virtio_rpmsg_send,
+	.sendto = virtio_rpmsg_sendto,
+	.send_offchannel = virtio_rpmsg_send_offchannel,
+	.trysend = virtio_rpmsg_trysend,
+	.trysendto = virtio_rpmsg_trysendto,
+	.trysend_offchannel = virtio_rpmsg_trysend_offchannel,
 };
 
-/* rpmsg devices and drivers are matched using the service name */
-static inline int rpmsg_id_match(const struct rpmsg_channel *rpdev,
-				  const struct rpmsg_device_id *id)
-{
-	return strncmp(id->name, rpdev->id.name, RPMSG_NAME_SIZE) == 0;
-}
-
-/* match rpmsg channel and rpmsg driver */
-static int rpmsg_dev_match(struct device *dev, struct device_driver *drv)
-{
-	struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);
-	struct rpmsg_driver *rpdrv = to_rpmsg_driver(drv);
-	const struct rpmsg_device_id *ids = rpdrv->id_table;
-	unsigned int i;
-
-	for (i = 0; ids[i].name[0]; i++)
-		if (rpmsg_id_match(rpdev, &ids[i]))
-			return 1;
-
-	return 0;
-}
-
-static int rpmsg_uevent(struct device *dev, struct kobj_uevent_env *env)
-{
-	struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);
-
-	return add_uevent_var(env, "MODALIAS=" RPMSG_DEVICE_MODALIAS_FMT,
-					rpdev->id.name);
-}
-
 /**
  * __ept_release() - deallocate an rpmsg endpoint
  * @kref: the ept's reference count
@@ -212,18 +214,17 @@ static void __ept_release(struct kref *kref)
 
 /* for more info, see below documentation of rpmsg_create_ept() */
 static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp,
-		struct rpmsg_channel *rpdev, rpmsg_rx_cb_t cb,
-		void *priv, u32 addr)
+						 struct rpmsg_device *rpdev,
+						 rpmsg_rx_cb_t cb,
+						 void *priv, u32 addr)
 {
 	int id_min, id_max, id;
 	struct rpmsg_endpoint *ept;
 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;
 
 	ept = kzalloc(sizeof(*ept), GFP_KERNEL);
-	if (!ept) {
-		dev_err(dev, "failed to kzalloc a new ept\n");
+	if (!ept)
 		return NULL;
-	}
 
 	kref_init(&ept->refcount);
 	mutex_init(&ept->cb_lock);
@@ -231,6 +232,7 @@ static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp,
 	ept->rpdev = rpdev;
 	ept->cb = cb;
 	ept->priv = priv;
+	ept->ops = &virtio_endpoint_ops;
 
 	/* do we need to allocate a local address ? */
 	if (addr == RPMSG_ADDR_ANY) {
@@ -261,52 +263,15 @@ free_ept:
 	return NULL;
 }
 
-/**
- * rpmsg_create_ept() - create a new rpmsg_endpoint
- * @rpdev: rpmsg channel device
- * @cb: rx callback handler
- * @priv: private data for the driver's use
- * @addr: local rpmsg address to bind with @cb
- *
- * Every rpmsg address in the system is bound to an rx callback (so when
- * inbound messages arrive, they are dispatched by the rpmsg bus using the
- * appropriate callback handler) by means of an rpmsg_endpoint struct.
- *
- * This function allows drivers to create such an endpoint, and by that,
- * bind a callback, and possibly some private data too, to an rpmsg address
- * (either one that is known in advance, or one that will be dynamically
- * assigned for them).
- *
- * Simple rpmsg drivers need not call rpmsg_create_ept, because an endpoint
- * is already created for them when they are probed by the rpmsg bus
- * (using the rx callback provided when they registered to the rpmsg bus).
- *
- * So things should just work for simple drivers: they already have an
- * endpoint, their rx callback is bound to their rpmsg address, and when
- * relevant inbound messages arrive (i.e. messages which their dst address
- * equals to the src address of their rpmsg channel), the driver's handler
- * is invoked to process it.
- *
- * That said, more complicated drivers might do need to allocate
- * additional rpmsg addresses, and bind them to different rx callbacks.
- * To accomplish that, those drivers need to call this function.
- *
- * Drivers should provide their @rpdev channel (so the new endpoint would belong
- * to the same remote processor their channel belongs to), an rx callback
- * function, an optional private data (which is provided back when the
- * rx callback is invoked), and an address they want to bind with the
- * callback. If @addr is RPMSG_ADDR_ANY, then rpmsg_create_ept will
- * dynamically assign them an available rpmsg address (drivers should have
- * a very good reason why not to always use RPMSG_ADDR_ANY here).
- *
- * Returns a pointer to the endpoint on success, or NULL on error.
- */
-struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_channel *rpdev,
-				rpmsg_rx_cb_t cb, void *priv, u32 addr)
+static struct rpmsg_endpoint *virtio_rpmsg_create_ept(struct rpmsg_device *rpdev,
+						      rpmsg_rx_cb_t cb,
+						      void *priv,
+						      struct rpmsg_channel_info chinfo)
 {
-	return __rpmsg_create_ept(rpdev->vrp, rpdev, cb, priv, addr);
+	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);
+
+	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);
 }
-EXPORT_SYMBOL(rpmsg_create_ept);
 
 /**
  * __rpmsg_destroy_ept() - destroy an existing rpmsg endpoint
@@ -334,178 +299,82 @@ __rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept)
 	kref_put(&ept->refcount, __ept_release);
 }
 
-/**
- * rpmsg_destroy_ept() - destroy an existing rpmsg endpoint
- * @ept: endpoing to destroy
- *
- * Should be used by drivers to destroy an rpmsg endpoint previously
- * created with rpmsg_create_ept().
- */
-void rpmsg_destroy_ept(struct rpmsg_endpoint *ept)
+static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept)
 {
-	__rpmsg_destroy_ept(ept->rpdev->vrp, ept);
+	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(ept->rpdev);
+
+	__rpmsg_destroy_ept(vch->vrp, ept);
 }
-EXPORT_SYMBOL(rpmsg_destroy_ept);
 
-/*
- * when an rpmsg driver is probed with a channel, we seamlessly create
- * it an endpoint, binding its rx callback to a unique local rpmsg
- * address.
- *
- * if we need to, we also announce about this channel to the remote
- * processor (needed in case the driver is exposing an rpmsg service).
- */
-static int rpmsg_dev_probe(struct device *dev)
+static int virtio_rpmsg_announce_create(struct rpmsg_device *rpdev)
 {
-	struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);
-	struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver);
-	struct virtproc_info *vrp = rpdev->vrp;
-	struct rpmsg_endpoint *ept;
-	int err;
-
-	ept = rpmsg_create_ept(rpdev, rpdrv->callback, NULL, rpdev->src);
-	if (!ept) {
-		dev_err(dev, "failed to create endpoint\n");
-		err = -ENOMEM;
-		goto out;
-	}
-
-	rpdev->ept = ept;
-	rpdev->src = ept->addr;
-
-	err = rpdrv->probe(rpdev);
-	if (err) {
-		dev_err(dev, "%s: failed: %d\n", __func__, err);
-		rpmsg_destroy_ept(ept);
-		goto out;
-	}
+	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);
+	struct virtproc_info *vrp = vch->vrp;
+	struct device *dev = &rpdev->dev;
+	int err = 0;
 
 	/* need to tell remote processor's name service about this channel ? */
 	if (rpdev->announce &&
-			virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {
+	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {
 		struct rpmsg_ns_msg nsm;
 
 		strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE);
-		nsm.addr = rpdev->src;
+		nsm.addr = rpdev->ept->addr;
 		nsm.flags = RPMSG_NS_CREATE;
 
-		err = rpmsg_sendto(rpdev, &nsm, sizeof(nsm), RPMSG_NS_ADDR);
+		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);
 		if (err)
 			dev_err(dev, "failed to announce service %d\n", err);
 	}
 
-out:
 	return err;
 }
 
-static int rpmsg_dev_remove(struct device *dev)
+static int virtio_rpmsg_announce_destroy(struct rpmsg_device *rpdev)
 {
-	struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);
-	struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver);
-	struct virtproc_info *vrp = rpdev->vrp;
+	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);
+	struct virtproc_info *vrp = vch->vrp;
+	struct device *dev = &rpdev->dev;
 	int err = 0;
 
 	/* tell remote processor's name service we're removing this channel */
 	if (rpdev->announce &&
-			virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {
+	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {
 		struct rpmsg_ns_msg nsm;
 
 		strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE);
 		nsm.addr = rpdev->src;
 		nsm.flags = RPMSG_NS_DESTROY;
 
-		err = rpmsg_sendto(rpdev, &nsm, sizeof(nsm), RPMSG_NS_ADDR);
+		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);
 		if (err)
 			dev_err(dev, "failed to announce service %d\n", err);
 	}
 
-	rpdrv->remove(rpdev);
-
-	rpmsg_destroy_ept(rpdev->ept);
-
 	return err;
 }
 
-static struct bus_type rpmsg_bus = {
-	.name		= "rpmsg",
-	.match		= rpmsg_dev_match,
-	.dev_attrs	= rpmsg_dev_attrs,
-	.uevent		= rpmsg_uevent,
-	.probe		= rpmsg_dev_probe,
-	.remove		= rpmsg_dev_remove,
+static const struct rpmsg_device_ops virtio_rpmsg_ops = {
+	.create_ept = virtio_rpmsg_create_ept,
+	.announce_create = virtio_rpmsg_announce_create,
+	.announce_destroy = virtio_rpmsg_announce_destroy,
 };
 
-/**
- * __register_rpmsg_driver() - register an rpmsg driver with the rpmsg bus
- * @rpdrv: pointer to a struct rpmsg_driver
- * @owner: owning module/driver
- *
- * Returns 0 on success, and an appropriate error value on failure.
- */
-int __register_rpmsg_driver(struct rpmsg_driver *rpdrv, struct module *owner)
-{
-	rpdrv->drv.bus = &rpmsg_bus;
-	rpdrv->drv.owner = owner;
-	return driver_register(&rpdrv->drv);
-}
-EXPORT_SYMBOL(__register_rpmsg_driver);
-
-/**
- * unregister_rpmsg_driver() - unregister an rpmsg driver from the rpmsg bus
- * @rpdrv: pointer to a struct rpmsg_driver
- *
- * Returns 0 on success, and an appropriate error value on failure.
- */
-void unregister_rpmsg_driver(struct rpmsg_driver *rpdrv)
-{
-	driver_unregister(&rpdrv->drv);
-}
-EXPORT_SYMBOL(unregister_rpmsg_driver);
-
-static void rpmsg_release_device(struct device *dev)
-{
-	struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);
-
-	kfree(rpdev);
-}
-
-/*
- * match an rpmsg channel with a channel info struct.
- * this is used to make sure we're not creating rpmsg devices for channels
- * that already exist.
- */
-static int rpmsg_channel_match(struct device *dev, void *data)
-{
-	struct rpmsg_channel_info *chinfo = data;
-	struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);
-
-	if (chinfo->src != RPMSG_ADDR_ANY && chinfo->src != rpdev->src)
-		return 0;
-
-	if (chinfo->dst != RPMSG_ADDR_ANY && chinfo->dst != rpdev->dst)
-		return 0;
-
-	if (strncmp(chinfo->name, rpdev->id.name, RPMSG_NAME_SIZE))
-		return 0;
-
-	/* found a match ! */
-	return 1;
-}
-
 /*
  * create an rpmsg channel using its name and address info.
  * this function will be used to create both static and dynamic
  * channels.
  */
-static struct rpmsg_channel *rpmsg_create_channel(struct virtproc_info *vrp,
-				struct rpmsg_channel_info *chinfo)
+static struct rpmsg_device *rpmsg_create_channel(struct virtproc_info *vrp,
+						 struct rpmsg_channel_info *chinfo)
 {
-	struct rpmsg_channel *rpdev;
+	struct virtio_rpmsg_channel *vch;
+	struct rpmsg_device *rpdev;
 	struct device *tmp, *dev = &vrp->vdev->dev;
 	int ret;
 
 	/* make sure a similar channel doesn't already exist */
-	tmp = device_find_child(dev, chinfo, rpmsg_channel_match);
+	tmp = rpmsg_find_device(dev, chinfo);
 	if (tmp) {
 		/* decrement the matched device's refcount back */
 		put_device(tmp);
@@ -514,62 +383,38 @@ static struct rpmsg_channel *rpmsg_create_channel(struct virtproc_info *vrp,
 		return NULL;
 	}
 
-	rpdev = kzalloc(sizeof(struct rpmsg_channel), GFP_KERNEL);
-	if (!rpdev) {
-		pr_err("kzalloc failed\n");
+	vch = kzalloc(sizeof(*vch), GFP_KERNEL);
+	if (!vch)
 		return NULL;
-	}
 
-	rpdev->vrp = vrp;
+	/* Link the channel to our vrp */
+	vch->vrp = vrp;
+
+	/* Assign callbacks for rpmsg_channel */
+	vch->rpdev.ops = &virtio_rpmsg_ops;
+
+	/* Assign public information to the rpmsg_device */
+	rpdev = &vch->rpdev;
 	rpdev->src = chinfo->src;
 	rpdev->dst = chinfo->dst;
+	rpdev->ops = &virtio_rpmsg_ops;
 
 	/*
 	 * rpmsg server channels has predefined local address (for now),
 	 * and their existence needs to be announced remotely
 	 */
-	rpdev->announce = rpdev->src != RPMSG_ADDR_ANY ? true : false;
+	rpdev->announce = rpdev->src != RPMSG_ADDR_ANY;
 
 	strncpy(rpdev->id.name, chinfo->name, RPMSG_NAME_SIZE);
 
-	/* very simple device indexing plumbing which is enough for now */
-	dev_set_name(&rpdev->dev, "rpmsg%d", rpmsg_dev_index++);
-
 	rpdev->dev.parent = &vrp->vdev->dev;
-	rpdev->dev.bus = &rpmsg_bus;
-	rpdev->dev.release = rpmsg_release_device;
-
-	ret = device_register(&rpdev->dev);
-	if (ret) {
-		dev_err(dev, "device_register failed: %d\n", ret);
-		put_device(&rpdev->dev);
+	ret = rpmsg_register_device(rpdev);
+	if (ret)
 		return NULL;
-	}
 
 	return rpdev;
 }
 
-/*
- * find an existing channel using its name + address properties,
- * and destroy it
- */
-static int rpmsg_destroy_channel(struct virtproc_info *vrp,
-					struct rpmsg_channel_info *chinfo)
-{
-	struct virtio_device *vdev = vrp->vdev;
-	struct device *dev;
-
-	dev = device_find_child(&vdev->dev, chinfo, rpmsg_channel_match);
-	if (!dev)
-		return -EINVAL;
-
-	device_unregister(dev);
-
-	put_device(dev);
-
-	return 0;
-}
-
 /* super simple buffer "allocator" that is just enough for now */
 static void *get_a_tx_buf(struct virtproc_info *vrp)
 {
@@ -684,10 +529,12 @@ static void rpmsg_downref_sleepers(struct virtproc_info *vrp)
  *
  * Returns 0 on success and an appropriate error value on failure.
  */
-int rpmsg_send_offchannel_raw(struct rpmsg_channel *rpdev, u32 src, u32 dst,
-					void *data, int len, bool wait)
+static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev,
+				     u32 src, u32 dst,
+				     void *data, int len, bool wait)
 {
-	struct virtproc_info *vrp = rpdev->vrp;
+	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);
+	struct virtproc_info *vrp = vch->vrp;
 	struct device *dev = &rpdev->dev;
 	struct scatterlist sg;
 	struct rpmsg_hdr *msg;
@@ -751,10 +598,11 @@ int rpmsg_send_offchannel_raw(struct rpmsg_channel *rpdev, u32 src, u32 dst,
 	memcpy(msg->data, data, len);
 
 	dev_dbg(dev, "TX From 0x%x, To 0x%x, Len %d, Flags %d, Reserved %d\n",
-					msg->src, msg->dst, msg->len,
-					msg->flags, msg->reserved);
-	print_hex_dump(KERN_DEBUG, "rpmsg_virtio TX: ", DUMP_PREFIX_NONE, 16, 1,
-					msg, sizeof(*msg) + msg->len, true);
+		msg->src, msg->dst, msg->len, msg->flags, msg->reserved);
+#if defined(CONFIG_DYNAMIC_DEBUG)
+	dynamic_hex_dump("rpmsg_virtio TX: ", DUMP_PREFIX_NONE, 16, 1,
+			 msg, sizeof(*msg) + msg->len, true);
+#endif
 
 	sg_init_one(&sg, msg, sizeof(*msg) + len);
 
@@ -780,6 +628,56 @@ out:
 }
 EXPORT_SYMBOL(rpmsg_send_offchannel_raw);
 
+static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	struct rpmsg_device *rpdev = ept->rpdev;
+	u32 src = ept->addr, dst = rpdev->dst;
+
+	return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true);
+}
+
+static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len,
+			       u32 dst)
+{
+	struct rpmsg_device *rpdev = ept->rpdev;
+	u32 src = ept->addr;
+
+	return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true);
+}
+
+static int virtio_rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src,
+					u32 dst, void *data, int len)
+{
+	struct rpmsg_device *rpdev = ept->rpdev;
+
+	return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true);
+}
+
+static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	struct rpmsg_device *rpdev = ept->rpdev;
+	u32 src = ept->addr, dst = rpdev->dst;
+
+	return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false);
+}
+
+static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data,
+				  int len, u32 dst)
+{
+	struct rpmsg_device *rpdev = ept->rpdev;
+	u32 src = ept->addr;
+
+	return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false);
+}
+
+static int virtio_rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src,
+					   u32 dst, void *data, int len)
+{
+	struct rpmsg_device *rpdev = ept->rpdev;
+
+	return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false);
+}
+
 static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,
 			     struct rpmsg_hdr *msg, unsigned int len)
 {
@@ -788,17 +686,18 @@ static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,
 	int err;
 
 	dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Reserved: %d\n",
-					msg->src, msg->dst, msg->len,
-					msg->flags, msg->reserved);
-	print_hex_dump(KERN_DEBUG, "rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1,
-					msg, sizeof(*msg) + msg->len, true);
+		msg->src, msg->dst, msg->len, msg->flags, msg->reserved);
+#if defined(CONFIG_DYNAMIC_DEBUG)
+	dynamic_hex_dump("rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1,
+			 msg, sizeof(*msg) + msg->len, true);
+#endif
 
 	/*
 	 * We currently use fixed-sized buffers, so trivially sanitize
 	 * the reported payload length.
 	 */
 	if (len > RPMSG_BUF_SIZE ||
-		msg->len > (len - sizeof(struct rpmsg_hdr))) {
+	    msg->len > (len - sizeof(struct rpmsg_hdr))) {
 		dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg->len);
 		return -EINVAL;
 	}
@@ -865,7 +764,7 @@ static void rpmsg_recv_done(struct virtqueue *rvq)
 		msgs_received++;
 
 		msg = virtqueue_get_buf(rvq, &len);
-	};
+	}
 
 	dev_dbg(dev, "Received %u messages\n", msgs_received);
 
@@ -892,23 +791,24 @@ static void rpmsg_xmit_done(struct virtqueue *svq)
 }
 
 /* invoked when a name service announcement arrives */
-static void rpmsg_ns_cb(struct rpmsg_channel *rpdev, void *data, int len,
-							void *priv, u32 src)
+static int rpmsg_ns_cb(struct rpmsg_device *rpdev, void *data, int len,
+		       void *priv, u32 src)
 {
 	struct rpmsg_ns_msg *msg = data;
-	struct rpmsg_channel *newch;
+	struct rpmsg_device *newch;
 	struct rpmsg_channel_info chinfo;
 	struct virtproc_info *vrp = priv;
 	struct device *dev = &vrp->vdev->dev;
 	int ret;
 
-	print_hex_dump(KERN_DEBUG, "NS announcement: ",
-			DUMP_PREFIX_NONE, 16, 1,
-			data, len, true);
+#if defined(CONFIG_DYNAMIC_DEBUG)
+	dynamic_hex_dump("NS announcement: ", DUMP_PREFIX_NONE, 16, 1,
+			 data, len, true);
+#endif
 
 	if (len != sizeof(*msg)) {
 		dev_err(dev, "malformed ns msg (%d)\n", len);
-		return;
+		return -EINVAL;
 	}
 
 	/*
@@ -919,22 +819,22 @@ static void rpmsg_ns_cb(struct rpmsg_channel *rpdev, void *data, int len,
 	 */
 	if (rpdev) {
 		dev_err(dev, "anomaly: ns ept has an rpdev handle\n");
-		return;
+		return -EINVAL;
 	}
 
 	/* don't trust the remote processor for null terminating the name */
 	msg->name[RPMSG_NAME_SIZE - 1] = '\0';
 
 	dev_info(dev, "%sing channel %s addr 0x%x\n",
-			msg->flags & RPMSG_NS_DESTROY ? "destroy" : "creat",
-			msg->name, msg->addr);
+		 msg->flags & RPMSG_NS_DESTROY ? "destroy" : "creat",
+		 msg->name, msg->addr);
 
 	strncpy(chinfo.name, msg->name, sizeof(chinfo.name));
 	chinfo.src = RPMSG_ADDR_ANY;
 	chinfo.dst = msg->addr;
 
 	if (msg->flags & RPMSG_NS_DESTROY) {
-		ret = rpmsg_destroy_channel(vrp, &chinfo);
+		ret = rpmsg_unregister_device(&vrp->vdev->dev, &chinfo);
 		if (ret)
 			dev_err(dev, "rpmsg_destroy_channel failed: %d\n", ret);
 	} else {
@@ -942,6 +842,8 @@ static void rpmsg_ns_cb(struct rpmsg_channel *rpdev, void *data, int len,
 		if (!newch)
 			dev_err(dev, "rpmsg_create_channel failed\n");
 	}
+
+	return 0;
 }
 
 static int rpmsg_probe(struct virtio_device *vdev)
@@ -995,8 +897,8 @@ static int rpmsg_probe(struct virtio_device *vdev)
 		goto vqs_del;
 	}
 
-	dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va,
-					(unsigned long long)vrp->bufs_dma);
+	dev_dbg(&vdev->dev, "buffers: va %p, dma %pad\n",
+		bufs_va, &vrp->bufs_dma);
 
 	/* half of the buffers is dedicated for RX */
 	vrp->rbufs = bufs_va;
@@ -1012,7 +914,7 @@ static int rpmsg_probe(struct virtio_device *vdev)
 		sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE);
 
 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,
-								GFP_KERNEL);
+					  GFP_KERNEL);
 		WARN_ON(err); /* sanity check; this can't really happen */
 	}
 
@@ -1119,17 +1021,9 @@ static int __init rpmsg_init(void)
 {
 	int ret;
 
-	ret = bus_register(&rpmsg_bus);
-	if (ret) {
-		pr_err("failed to register rpmsg bus: %d\n", ret);
-		return ret;
-	}
-
 	ret = register_virtio_driver(&virtio_ipc_driver);
-	if (ret) {
+	if (ret)
 		pr_err("failed to register virtio driver: %d\n", ret);
-		bus_unregister(&rpmsg_bus);
-	}
 
 	return ret;
 }
@@ -1138,7 +1032,6 @@ subsys_initcall(rpmsg_init);
 static void __exit rpmsg_fini(void)
 {
 	unregister_virtio_driver(&virtio_ipc_driver);
-	bus_unregister(&rpmsg_bus);
 }
 module_exit(rpmsg_fini);
 
diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig
index e215f50794b6..d1e080701264 100644
--- a/drivers/rtc/Kconfig
+++ b/drivers/rtc/Kconfig
@@ -187,6 +187,16 @@ config RTC_DRV_ABX80X
 	  This driver can also be built as a module. If so, the module
 	  will be called rtc-abx80x.
 
+config RTC_DRV_AC100
+	tristate "X-Powers AC100"
+	depends on MFD_AC100
+	help
+	  If you say yes here you get support for the real-time clock found
+	  in X-Powers AC100 family peripheral ICs.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called rtc-ac100.
+
 config RTC_DRV_AS3722
 	tristate "ams AS3722 RTC driver"
 	depends on MFD_AS3722
@@ -328,11 +338,11 @@ config RTC_DRV_MAX77686
 	  will be called rtc-max77686.
 
 config RTC_DRV_RK808
-	tristate "Rockchip RK808 RTC"
+	tristate "Rockchip RK808/RK818 RTC"
 	depends on MFD_RK808
 	help
 	  If you say yes here you will get support for the
-	  RTC of RK808 PMIC.
+	  RTC of RK808 and RK818 PMIC.
 
 	  This driver can also be built as a module. If so, the module
 	  will be called rk808-rtc.
diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile
index 7cf7ad559c79..8fb994bacdf7 100644
--- a/drivers/rtc/Makefile
+++ b/drivers/rtc/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_RTC_DRV_AB3100)	+= rtc-ab3100.o
 obj-$(CONFIG_RTC_DRV_AB8500)	+= rtc-ab8500.o
 obj-$(CONFIG_RTC_DRV_ABB5ZES3)	+= rtc-ab-b5ze-s3.o
 obj-$(CONFIG_RTC_DRV_ABX80X)	+= rtc-abx80x.o
+obj-$(CONFIG_RTC_DRV_AC100)	+= rtc-ac100.o
 obj-$(CONFIG_RTC_DRV_ARMADA38X)	+= rtc-armada38x.o
 obj-$(CONFIG_RTC_DRV_AS3722)	+= rtc-as3722.o
 obj-$(CONFIG_RTC_DRV_ASM9260)	+= rtc-asm9260.o
diff --git a/drivers/rtc/rtc-ac100.c b/drivers/rtc/rtc-ac100.c
new file mode 100644
index 000000000000..70b4fd0f6122
--- /dev/null
+++ b/drivers/rtc/rtc-ac100.c
@@ -0,0 +1,627 @@
+/*
+ * RTC Driver for X-Powers AC100
+ *
+ * Copyright (c) 2016 Chen-Yu Tsai
+ *
+ * Chen-Yu Tsai <wens@csie.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/bcd.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/ac100.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/rtc.h>
+#include <linux/types.h>
+
+/* Control register */
+#define AC100_RTC_CTRL_24HOUR	BIT(0)
+
+/* Clock output register bits */
+#define AC100_CLKOUT_PRE_DIV_SHIFT	5
+#define AC100_CLKOUT_PRE_DIV_WIDTH	3
+#define AC100_CLKOUT_MUX_SHIFT		4
+#define AC100_CLKOUT_MUX_WIDTH		1
+#define AC100_CLKOUT_DIV_SHIFT		1
+#define AC100_CLKOUT_DIV_WIDTH		3
+#define AC100_CLKOUT_EN			BIT(0)
+
+/* RTC */
+#define AC100_RTC_SEC_MASK	GENMASK(6, 0)
+#define AC100_RTC_MIN_MASK	GENMASK(6, 0)
+#define AC100_RTC_HOU_MASK	GENMASK(5, 0)
+#define AC100_RTC_WEE_MASK	GENMASK(2, 0)
+#define AC100_RTC_DAY_MASK	GENMASK(5, 0)
+#define AC100_RTC_MON_MASK	GENMASK(4, 0)
+#define AC100_RTC_YEA_MASK	GENMASK(7, 0)
+#define AC100_RTC_YEA_LEAP	BIT(15)
+#define AC100_RTC_UPD_TRIGGER	BIT(15)
+
+/* Alarm (wall clock) */
+#define AC100_ALM_INT_ENABLE	BIT(0)
+
+#define AC100_ALM_SEC_MASK	GENMASK(6, 0)
+#define AC100_ALM_MIN_MASK	GENMASK(6, 0)
+#define AC100_ALM_HOU_MASK	GENMASK(5, 0)
+#define AC100_ALM_WEE_MASK	GENMASK(2, 0)
+#define AC100_ALM_DAY_MASK	GENMASK(5, 0)
+#define AC100_ALM_MON_MASK	GENMASK(4, 0)
+#define AC100_ALM_YEA_MASK	GENMASK(7, 0)
+#define AC100_ALM_ENABLE_FLAG	BIT(15)
+#define AC100_ALM_UPD_TRIGGER	BIT(15)
+
+/*
+ * The year parameter passed to the driver is usually an offset relative to
+ * the year 1900. This macro is used to convert this offset to another one
+ * relative to the minimum year allowed by the hardware.
+ *
+ * The year range is 1970 - 2069. This range is selected to match Allwinner's
+ * driver.
+ */
+#define AC100_YEAR_MIN				1970
+#define AC100_YEAR_MAX				2069
+#define AC100_YEAR_OFF				(AC100_YEAR_MIN - 1900)
+
+struct ac100_clkout {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	u8 offset;
+};
+
+#define to_ac100_clkout(_hw) container_of(_hw, struct ac100_clkout, hw)
+
+#define AC100_RTC_32K_NAME	"ac100-rtc-32k"
+#define AC100_RTC_32K_RATE	32768
+#define AC100_CLKOUT_NUM	3
+
+static const char * const ac100_clkout_names[AC100_CLKOUT_NUM] = {
+	"ac100-cko1-rtc",
+	"ac100-cko2-rtc",
+	"ac100-cko3-rtc",
+};
+
+struct ac100_rtc_dev {
+	struct rtc_device *rtc;
+	struct device *dev;
+	struct regmap *regmap;
+	int irq;
+	unsigned long alarm;
+
+	struct clk_hw *rtc_32k_clk;
+	struct ac100_clkout clks[AC100_CLKOUT_NUM];
+	struct clk_hw_onecell_data *clk_data;
+};
+
+/**
+ * Clock controls for 3 clock output pins
+ */
+
+static const struct clk_div_table ac100_clkout_prediv[] = {
+	{ .val = 0, .div = 1 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 4 },
+	{ .val = 3, .div = 8 },
+	{ .val = 4, .div = 16 },
+	{ .val = 5, .div = 32 },
+	{ .val = 6, .div = 64 },
+	{ .val = 7, .div = 122 },
+	{ },
+};
+
+/* Abuse the fact that one parent is 32768 Hz, and the other is 4 MHz */
+static unsigned long ac100_clkout_recalc_rate(struct clk_hw *hw,
+					      unsigned long prate)
+{
+	struct ac100_clkout *clk = to_ac100_clkout(hw);
+	unsigned int reg, div;
+
+	regmap_read(clk->regmap, clk->offset, &reg);
+
+	/* Handle pre-divider first */
+	if (prate != AC100_RTC_32K_RATE) {
+		div = (reg >> AC100_CLKOUT_PRE_DIV_SHIFT) &
+			((1 << AC100_CLKOUT_PRE_DIV_WIDTH) - 1);
+		prate = divider_recalc_rate(hw, prate, div,
+					    ac100_clkout_prediv, 0);
+	}
+
+	div = (reg >> AC100_CLKOUT_DIV_SHIFT) &
+		(BIT(AC100_CLKOUT_DIV_WIDTH) - 1);
+	return divider_recalc_rate(hw, prate, div, NULL,
+				   CLK_DIVIDER_POWER_OF_TWO);
+}
+
+static long ac100_clkout_round_rate(struct clk_hw *hw, unsigned long rate,
+				    unsigned long prate)
+{
+	unsigned long best_rate = 0, tmp_rate, tmp_prate;
+	int i;
+
+	if (prate == AC100_RTC_32K_RATE)
+		return divider_round_rate(hw, rate, &prate, NULL,
+					  AC100_CLKOUT_DIV_WIDTH,
+					  CLK_DIVIDER_POWER_OF_TWO);
+
+	for (i = 0; ac100_clkout_prediv[i].div; i++) {
+		tmp_prate = DIV_ROUND_UP(prate, ac100_clkout_prediv[i].val);
+		tmp_rate = divider_round_rate(hw, rate, &tmp_prate, NULL,
+					      AC100_CLKOUT_DIV_WIDTH,
+					      CLK_DIVIDER_POWER_OF_TWO);
+
+		if (tmp_rate > rate)
+			continue;
+		if (rate - tmp_rate < best_rate - tmp_rate)
+			best_rate = tmp_rate;
+	}
+
+	return best_rate;
+}
+
+static int ac100_clkout_determine_rate(struct clk_hw *hw,
+				       struct clk_rate_request *req)
+{
+	struct clk_hw *best_parent;
+	unsigned long best = 0;
+	int i, num_parents = clk_hw_get_num_parents(hw);
+
+	for (i = 0; i < num_parents; i++) {
+		struct clk_hw *parent = clk_hw_get_parent_by_index(hw, i);
+		unsigned long tmp, prate = clk_hw_get_rate(parent);
+
+		tmp = ac100_clkout_round_rate(hw, req->rate, prate);
+
+		if (tmp > req->rate)
+			continue;
+		if (req->rate - tmp < req->rate - best) {
+			best = tmp;
+			best_parent = parent;
+		}
+	}
+
+	if (!best)
+		return -EINVAL;
+
+	req->best_parent_hw = best_parent;
+	req->best_parent_rate = best;
+	req->rate = best;
+
+	return 0;
+}
+
+static int ac100_clkout_set_rate(struct clk_hw *hw, unsigned long rate,
+				 unsigned long prate)
+{
+	struct ac100_clkout *clk = to_ac100_clkout(hw);
+	int div = 0, pre_div = 0;
+
+	do {
+		div = divider_get_val(rate * ac100_clkout_prediv[pre_div].div,
+				      prate, NULL, AC100_CLKOUT_DIV_WIDTH,
+				      CLK_DIVIDER_POWER_OF_TWO);
+		if (div >= 0)
+			break;
+	} while (prate != AC100_RTC_32K_RATE &&
+		 ac100_clkout_prediv[++pre_div].div);
+
+	if (div < 0)
+		return div;
+
+	pre_div = ac100_clkout_prediv[pre_div].val;
+
+	regmap_update_bits(clk->regmap, clk->offset,
+			   ((1 << AC100_CLKOUT_DIV_WIDTH) - 1) << AC100_CLKOUT_DIV_SHIFT |
+			   ((1 << AC100_CLKOUT_PRE_DIV_WIDTH) - 1) << AC100_CLKOUT_PRE_DIV_SHIFT,
+			   (div - 1) << AC100_CLKOUT_DIV_SHIFT |
+			   (pre_div - 1) << AC100_CLKOUT_PRE_DIV_SHIFT);
+
+	return 0;
+}
+
+static int ac100_clkout_prepare(struct clk_hw *hw)
+{
+	struct ac100_clkout *clk = to_ac100_clkout(hw);
+
+	return regmap_update_bits(clk->regmap, clk->offset, AC100_CLKOUT_EN,
+				  AC100_CLKOUT_EN);
+}
+
+static void ac100_clkout_unprepare(struct clk_hw *hw)
+{
+	struct ac100_clkout *clk = to_ac100_clkout(hw);
+
+	regmap_update_bits(clk->regmap, clk->offset, AC100_CLKOUT_EN, 0);
+}
+
+static int ac100_clkout_is_prepared(struct clk_hw *hw)
+{
+	struct ac100_clkout *clk = to_ac100_clkout(hw);
+	unsigned int reg;
+
+	regmap_read(clk->regmap, clk->offset, &reg);
+
+	return reg & AC100_CLKOUT_EN;
+}
+
+static u8 ac100_clkout_get_parent(struct clk_hw *hw)
+{
+	struct ac100_clkout *clk = to_ac100_clkout(hw);
+	unsigned int reg;
+
+	regmap_read(clk->regmap, clk->offset, &reg);
+
+	return (reg >> AC100_CLKOUT_MUX_SHIFT) & 0x1;
+}
+
+static int ac100_clkout_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct ac100_clkout *clk = to_ac100_clkout(hw);
+
+	return regmap_update_bits(clk->regmap, clk->offset,
+				  BIT(AC100_CLKOUT_MUX_SHIFT),
+				  index ? BIT(AC100_CLKOUT_MUX_SHIFT) : 0);
+}
+
+static const struct clk_ops ac100_clkout_ops = {
+	.prepare	= ac100_clkout_prepare,
+	.unprepare	= ac100_clkout_unprepare,
+	.is_prepared	= ac100_clkout_is_prepared,
+	.recalc_rate	= ac100_clkout_recalc_rate,
+	.determine_rate	= ac100_clkout_determine_rate,
+	.get_parent	= ac100_clkout_get_parent,
+	.set_parent	= ac100_clkout_set_parent,
+	.set_rate	= ac100_clkout_set_rate,
+};
+
+static int ac100_rtc_register_clks(struct ac100_rtc_dev *chip)
+{
+	struct device_node *np = chip->dev->of_node;
+	const char *parents[2] = {AC100_RTC_32K_NAME};
+	int i, ret;
+
+	chip->clk_data = devm_kzalloc(chip->dev, sizeof(*chip->clk_data) +
+						 sizeof(*chip->clk_data->hws) *
+						 AC100_CLKOUT_NUM,
+						 GFP_KERNEL);
+	if (!chip->clk_data)
+		return -ENOMEM;
+
+	chip->rtc_32k_clk = clk_hw_register_fixed_rate(chip->dev,
+						       AC100_RTC_32K_NAME,
+						       NULL, 0,
+						       AC100_RTC_32K_RATE);
+	if (IS_ERR(chip->rtc_32k_clk)) {
+		ret = PTR_ERR(chip->rtc_32k_clk);
+		dev_err(chip->dev, "Failed to register RTC-32k clock: %d\n",
+			ret);
+		return ret;
+	}
+
+	parents[1] = of_clk_get_parent_name(np, 0);
+	if (!parents[1]) {
+		dev_err(chip->dev, "Failed to get ADDA 4M clock\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < AC100_CLKOUT_NUM; i++) {
+		struct ac100_clkout *clk = &chip->clks[i];
+		struct clk_init_data init = {
+			.name = ac100_clkout_names[i],
+			.ops = &ac100_clkout_ops,
+			.parent_names = parents,
+			.num_parents = ARRAY_SIZE(parents),
+			.flags = 0,
+		};
+
+		clk->regmap = chip->regmap;
+		clk->offset = AC100_CLKOUT_CTRL1 + i;
+		clk->hw.init = &init;
+
+		ret = devm_clk_hw_register(chip->dev, &clk->hw);
+		if (ret) {
+			dev_err(chip->dev, "Failed to register clk '%s': %d\n",
+				init.name, ret);
+			goto err_unregister_rtc_32k;
+		}
+
+		chip->clk_data->hws[i] = &clk->hw;
+	}
+
+	chip->clk_data->num = i;
+	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, chip->clk_data);
+	if (ret)
+		goto err_unregister_rtc_32k;
+
+	return 0;
+
+err_unregister_rtc_32k:
+	clk_unregister_fixed_rate(chip->rtc_32k_clk->clk);
+
+	return ret;
+}
+
+static void ac100_rtc_unregister_clks(struct ac100_rtc_dev *chip)
+{
+	of_clk_del_provider(chip->dev->of_node);
+	clk_unregister_fixed_rate(chip->rtc_32k_clk->clk);
+}
+
+/**
+ * RTC related bits
+ */
+static int ac100_rtc_get_time(struct device *dev, struct rtc_time *rtc_tm)
+{
+	struct ac100_rtc_dev *chip = dev_get_drvdata(dev);
+	struct regmap *regmap = chip->regmap;
+	u16 reg[7];
+	int ret;
+
+	ret = regmap_bulk_read(regmap, AC100_RTC_SEC, reg, 7);
+	if (ret)
+		return ret;
+
+	rtc_tm->tm_sec  = bcd2bin(reg[0] & AC100_RTC_SEC_MASK);
+	rtc_tm->tm_min  = bcd2bin(reg[1] & AC100_RTC_MIN_MASK);
+	rtc_tm->tm_hour = bcd2bin(reg[2] & AC100_RTC_HOU_MASK);
+	rtc_tm->tm_wday = bcd2bin(reg[3] & AC100_RTC_WEE_MASK);
+	rtc_tm->tm_mday = bcd2bin(reg[4] & AC100_RTC_DAY_MASK);
+	rtc_tm->tm_mon  = bcd2bin(reg[5] & AC100_RTC_MON_MASK) - 1;
+	rtc_tm->tm_year = bcd2bin(reg[6] & AC100_RTC_YEA_MASK) +
+			  AC100_YEAR_OFF;
+
+	return rtc_valid_tm(rtc_tm);
+}
+
+static int ac100_rtc_set_time(struct device *dev, struct rtc_time *rtc_tm)
+{
+	struct ac100_rtc_dev *chip = dev_get_drvdata(dev);
+	struct regmap *regmap = chip->regmap;
+	int year;
+	u16 reg[8];
+
+	/* our RTC has a limited year range... */
+	year = rtc_tm->tm_year - AC100_YEAR_OFF;
+	if (year < 0 || year > (AC100_YEAR_MAX - 1900)) {
+		dev_err(dev, "rtc only supports year in range %d - %d\n",
+			AC100_YEAR_MIN, AC100_YEAR_MAX);
+		return -EINVAL;
+	}
+
+	/* convert to BCD */
+	reg[0] = bin2bcd(rtc_tm->tm_sec)     & AC100_RTC_SEC_MASK;
+	reg[1] = bin2bcd(rtc_tm->tm_min)     & AC100_RTC_MIN_MASK;
+	reg[2] = bin2bcd(rtc_tm->tm_hour)    & AC100_RTC_HOU_MASK;
+	reg[3] = bin2bcd(rtc_tm->tm_wday)    & AC100_RTC_WEE_MASK;
+	reg[4] = bin2bcd(rtc_tm->tm_mday)    & AC100_RTC_DAY_MASK;
+	reg[5] = bin2bcd(rtc_tm->tm_mon + 1) & AC100_RTC_MON_MASK;
+	reg[6] = bin2bcd(year)		     & AC100_RTC_YEA_MASK;
+	/* trigger write */
+	reg[7] = AC100_RTC_UPD_TRIGGER;
+
+	/* Is it a leap year? */
+	if (is_leap_year(year + AC100_YEAR_OFF + 1900))
+		reg[6] |= AC100_RTC_YEA_LEAP;
+
+	return regmap_bulk_write(regmap, AC100_RTC_SEC, reg, 8);
+}
+
+static int ac100_rtc_alarm_irq_enable(struct device *dev, unsigned int en)
+{
+	struct ac100_rtc_dev *chip = dev_get_drvdata(dev);
+	struct regmap *regmap = chip->regmap;
+	unsigned int val;
+
+	val = en ? AC100_ALM_INT_ENABLE : 0;
+
+	return regmap_write(regmap, AC100_ALM_INT_ENA, val);
+}
+
+static int ac100_rtc_get_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct ac100_rtc_dev *chip = dev_get_drvdata(dev);
+	struct regmap *regmap = chip->regmap;
+	struct rtc_time *alrm_tm = &alrm->time;
+	u16 reg[7];
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(regmap, AC100_ALM_INT_ENA, &val);
+	if (ret)
+		return ret;
+
+	alrm->enabled = !!(val & AC100_ALM_INT_ENABLE);
+
+	ret = regmap_bulk_read(regmap, AC100_ALM_SEC, reg, 7);
+	if (ret)
+		return ret;
+
+	alrm_tm->tm_sec  = bcd2bin(reg[0] & AC100_ALM_SEC_MASK);
+	alrm_tm->tm_min  = bcd2bin(reg[1] & AC100_ALM_MIN_MASK);
+	alrm_tm->tm_hour = bcd2bin(reg[2] & AC100_ALM_HOU_MASK);
+	alrm_tm->tm_wday = bcd2bin(reg[3] & AC100_ALM_WEE_MASK);
+	alrm_tm->tm_mday = bcd2bin(reg[4] & AC100_ALM_DAY_MASK);
+	alrm_tm->tm_mon  = bcd2bin(reg[5] & AC100_ALM_MON_MASK) - 1;
+	alrm_tm->tm_year = bcd2bin(reg[6] & AC100_ALM_YEA_MASK) +
+			   AC100_YEAR_OFF;
+
+	return 0;
+}
+
+static int ac100_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct ac100_rtc_dev *chip = dev_get_drvdata(dev);
+	struct regmap *regmap = chip->regmap;
+	struct rtc_time *alrm_tm = &alrm->time;
+	u16 reg[8];
+	int year;
+	int ret;
+
+	/* our alarm has a limited year range... */
+	year = alrm_tm->tm_year - AC100_YEAR_OFF;
+	if (year < 0 || year > (AC100_YEAR_MAX - 1900)) {
+		dev_err(dev, "alarm only supports year in range %d - %d\n",
+			AC100_YEAR_MIN, AC100_YEAR_MAX);
+		return -EINVAL;
+	}
+
+	/* convert to BCD */
+	reg[0] = (bin2bcd(alrm_tm->tm_sec)  & AC100_ALM_SEC_MASK) |
+			AC100_ALM_ENABLE_FLAG;
+	reg[1] = (bin2bcd(alrm_tm->tm_min)  & AC100_ALM_MIN_MASK) |
+			AC100_ALM_ENABLE_FLAG;
+	reg[2] = (bin2bcd(alrm_tm->tm_hour) & AC100_ALM_HOU_MASK) |
+			AC100_ALM_ENABLE_FLAG;
+	/* Do not enable weekday alarm */
+	reg[3] = bin2bcd(alrm_tm->tm_wday) & AC100_ALM_WEE_MASK;
+	reg[4] = (bin2bcd(alrm_tm->tm_mday) & AC100_ALM_DAY_MASK) |
+			AC100_ALM_ENABLE_FLAG;
+	reg[5] = (bin2bcd(alrm_tm->tm_mon + 1)  & AC100_ALM_MON_MASK) |
+			AC100_ALM_ENABLE_FLAG;
+	reg[6] = (bin2bcd(year) & AC100_ALM_YEA_MASK) |
+			AC100_ALM_ENABLE_FLAG;
+	/* trigger write */
+	reg[7] = AC100_ALM_UPD_TRIGGER;
+
+	ret = regmap_bulk_write(regmap, AC100_ALM_SEC, reg, 8);
+	if (ret)
+		return ret;
+
+	return ac100_rtc_alarm_irq_enable(dev, alrm->enabled);
+}
+
+static irqreturn_t ac100_rtc_irq(int irq, void *data)
+{
+	struct ac100_rtc_dev *chip = data;
+	struct regmap *regmap = chip->regmap;
+	unsigned int val = 0;
+	int ret;
+
+	mutex_lock(&chip->rtc->ops_lock);
+
+	/* read status */
+	ret = regmap_read(regmap, AC100_ALM_INT_STA, &val);
+	if (ret)
+		goto out;
+
+	if (val & AC100_ALM_INT_ENABLE) {
+		/* signal rtc framework */
+		rtc_update_irq(chip->rtc, 1, RTC_AF | RTC_IRQF);
+
+		/* clear status */
+		ret = regmap_write(regmap, AC100_ALM_INT_STA, val);
+		if (ret)
+			goto out;
+
+		/* disable interrupt */
+		ret = ac100_rtc_alarm_irq_enable(chip->dev, 0);
+		if (ret)
+			goto out;
+	}
+
+out:
+	mutex_unlock(&chip->rtc->ops_lock);
+	return IRQ_HANDLED;
+}
+
+static const struct rtc_class_ops ac100_rtc_ops = {
+	.read_time	  = ac100_rtc_get_time,
+	.set_time	  = ac100_rtc_set_time,
+	.read_alarm	  = ac100_rtc_get_alarm,
+	.set_alarm	  = ac100_rtc_set_alarm,
+	.alarm_irq_enable = ac100_rtc_alarm_irq_enable,
+};
+
+static int ac100_rtc_probe(struct platform_device *pdev)
+{
+	struct ac100_dev *ac100 = dev_get_drvdata(pdev->dev.parent);
+	struct ac100_rtc_dev *chip;
+	int ret;
+
+	chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL);
+	platform_set_drvdata(pdev, chip);
+	chip->dev = &pdev->dev;
+	chip->regmap = ac100->regmap;
+
+	chip->irq = platform_get_irq(pdev, 0);
+	if (chip->irq < 0) {
+		dev_err(&pdev->dev, "No IRQ resource\n");
+		return chip->irq;
+	}
+
+	ret = devm_request_threaded_irq(&pdev->dev, chip->irq, NULL,
+					ac100_rtc_irq,
+					IRQF_SHARED | IRQF_ONESHOT,
+					dev_name(&pdev->dev), chip);
+	if (ret) {
+		dev_err(&pdev->dev, "Could not request IRQ\n");
+		return ret;
+	}
+
+	/* always use 24 hour mode */
+	regmap_write_bits(chip->regmap, AC100_RTC_CTRL, AC100_RTC_CTRL_24HOUR,
+			  AC100_RTC_CTRL_24HOUR);
+
+	/* disable counter alarm interrupt */
+	regmap_write(chip->regmap, AC100_ALM_INT_ENA, 0);
+
+	/* clear counter alarm pending interrupts */
+	regmap_write(chip->regmap, AC100_ALM_INT_STA, AC100_ALM_INT_ENABLE);
+
+	chip->rtc = devm_rtc_device_register(&pdev->dev, "rtc-ac100",
+					     &ac100_rtc_ops, THIS_MODULE);
+	if (IS_ERR(chip->rtc)) {
+		dev_err(&pdev->dev, "unable to register device\n");
+		return PTR_ERR(chip->rtc);
+	}
+
+	ret = ac100_rtc_register_clks(chip);
+	if (ret)
+		return ret;
+
+	dev_info(&pdev->dev, "RTC enabled\n");
+
+	return 0;
+}
+
+static int ac100_rtc_remove(struct platform_device *pdev)
+{
+	struct ac100_rtc_dev *chip = platform_get_drvdata(pdev);
+
+	ac100_rtc_unregister_clks(chip);
+
+	return 0;
+}
+
+static const struct of_device_id ac100_rtc_match[] = {
+	{ .compatible = "x-powers,ac100-rtc" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ac100_rtc_match);
+
+static struct platform_driver ac100_rtc_driver = {
+	.probe		= ac100_rtc_probe,
+	.remove		= ac100_rtc_remove,
+	.driver		= {
+		.name		= "ac100-rtc",
+		.of_match_table	= of_match_ptr(ac100_rtc_match),
+	},
+};
+module_platform_driver(ac100_rtc_driver);
+
+MODULE_DESCRIPTION("X-Powers AC100 RTC driver");
+MODULE_AUTHOR("Chen-Yu Tsai <wens@csie.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/rtc/rtc-pm8xxx.c b/drivers/rtc/rtc-pm8xxx.c
index 795fcbd02ea3..fac835530671 100644
--- a/drivers/rtc/rtc-pm8xxx.c
+++ b/drivers/rtc/rtc-pm8xxx.c
@@ -428,6 +428,7 @@ static const struct pm8xxx_rtc_regs pm8941_regs = {
  */
 static const struct of_device_id pm8xxx_id_table[] = {
 	{ .compatible = "qcom,pm8921-rtc", .data = &pm8921_regs },
+	{ .compatible = "qcom,pm8018-rtc", .data = &pm8921_regs },
 	{ .compatible = "qcom,pm8058-rtc", .data = &pm8058_regs },
 	{ .compatible = "qcom,pm8941-rtc", .data = &pm8941_regs },
 	{ },
diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c
index 8973d34ce5ba..1de089019268 100644
--- a/drivers/s390/block/dasd.c
+++ b/drivers/s390/block/dasd.c
@@ -212,16 +212,6 @@ static int dasd_state_known_to_new(struct dasd_device *device)
 {
 	/* Disable extended error reporting for this device. */
 	dasd_eer_disable(device);
-	/* Forget the discipline information. */
-	if (device->discipline) {
-		if (device->discipline->uncheck_device)
-			device->discipline->uncheck_device(device);
-		module_put(device->discipline->owner);
-	}
-	device->discipline = NULL;
-	if (device->base_discipline)
-		module_put(device->base_discipline->owner);
-	device->base_discipline = NULL;
 	device->state = DASD_STATE_NEW;
 
 	if (device->block)
@@ -336,6 +326,7 @@ static int dasd_state_basic_to_ready(struct dasd_device *device)
 {
 	int rc;
 	struct dasd_block *block;
+	struct gendisk *disk;
 
 	rc = 0;
 	block = device->block;
@@ -346,6 +337,9 @@ static int dasd_state_basic_to_ready(struct dasd_device *device)
 		if (rc) {
 			if (rc != -EAGAIN) {
 				device->state = DASD_STATE_UNFMT;
+				disk = device->block->gdp;
+				kobject_uevent(&disk_to_dev(disk)->kobj,
+					       KOBJ_CHANGE);
 				goto out;
 			}
 			return rc;
@@ -1643,9 +1637,18 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
 	u8 *sense = NULL;
 	int expires;
 
+	cqr = (struct dasd_ccw_req *) intparm;
 	if (IS_ERR(irb)) {
 		switch (PTR_ERR(irb)) {
 		case -EIO:
+			if (cqr && cqr->status == DASD_CQR_CLEAR_PENDING) {
+				device = (struct dasd_device *) cqr->startdev;
+				cqr->status = DASD_CQR_CLEARED;
+				dasd_device_clear_timer(device);
+				wake_up(&dasd_flush_wq);
+				dasd_schedule_device_bh(device);
+				return;
+			}
 			break;
 		case -ETIMEDOUT:
 			DBF_EVENT_DEVID(DBF_WARNING, cdev, "%s: "
@@ -1661,7 +1664,6 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
 	}
 
 	now = get_tod_clock();
-	cqr = (struct dasd_ccw_req *) intparm;
 	/* check for conditions that should be handled immediately */
 	if (!cqr ||
 	    !(scsw_dstat(&irb->scsw) == (DEV_STAT_CHN_END | DEV_STAT_DEV_END) &&
@@ -2266,6 +2268,15 @@ static int _dasd_sleep_on(struct dasd_ccw_req *maincqr, int interruptible)
 			continue;
 		}
 		/*
+		 * Don't try to start requests if device is in
+		 * offline processing, it might wait forever
+		 */
+		if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) {
+			cqr->status = DASD_CQR_FAILED;
+			cqr->intrc = -ENODEV;
+			continue;
+		}
+		/*
 		 * Don't try to start requests if device is stopped
 		 * except path verification requests
 		 */
@@ -3356,6 +3367,22 @@ int dasd_generic_probe(struct ccw_device *cdev,
 }
 EXPORT_SYMBOL_GPL(dasd_generic_probe);
 
+void dasd_generic_free_discipline(struct dasd_device *device)
+{
+	/* Forget the discipline information. */
+	if (device->discipline) {
+		if (device->discipline->uncheck_device)
+			device->discipline->uncheck_device(device);
+		module_put(device->discipline->owner);
+		device->discipline = NULL;
+	}
+	if (device->base_discipline) {
+		module_put(device->base_discipline->owner);
+		device->base_discipline = NULL;
+	}
+}
+EXPORT_SYMBOL_GPL(dasd_generic_free_discipline);
+
 /*
  * This will one day be called from a global not_oper handler.
  * It is also used by driver_unregister during module unload.
diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c
index 3cdbce45e464..15a1a70cace9 100644
--- a/drivers/s390/block/dasd_devmap.c
+++ b/drivers/s390/block/dasd_devmap.c
@@ -617,6 +617,7 @@ dasd_delete_device(struct dasd_device *device)
 	/* Wait for reference counter to drop to zero. */
 	wait_event(dasd_delete_wq, atomic_read(&device->ref_count) == 0);
 
+	dasd_generic_free_discipline(device);
 	/* Disconnect dasd_device structure from ccw_device structure. */
 	cdev = device->cdev;
 	device->cdev = NULL;
diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c
index fd2eff440098..831935af7389 100644
--- a/drivers/s390/block/dasd_eckd.c
+++ b/drivers/s390/block/dasd_eckd.c
@@ -5078,6 +5078,8 @@ static int dasd_eckd_read_message_buffer(struct dasd_device *device,
 		return PTR_ERR(cqr);
 	}
 
+	cqr->lpm = lpum;
+retry:
 	cqr->startdev = device;
 	cqr->memdev = device;
 	cqr->block = NULL;
@@ -5122,6 +5124,14 @@ static int dasd_eckd_read_message_buffer(struct dasd_device *device,
 			(prssdp + 1);
 		memcpy(messages, message_buf,
 		       sizeof(struct dasd_rssd_messages));
+	} else if (cqr->lpm) {
+		/*
+		 * on z/VM we might not be able to do I/O on the requested path
+		 * but instead we get the required information on any path
+		 * so retry with open path mask
+		 */
+		cqr->lpm = 0;
+		goto retry;
 	} else
 		DBF_EVENT_DEVID(DBF_WARNING, device->cdev,
 				"Reading messages failed with rc=%d\n"
@@ -5191,7 +5201,7 @@ static int dasd_eckd_query_host_access(struct dasd_device *device,
 
 	cqr->buildclk = get_tod_clock();
 	cqr->status = DASD_CQR_FILLED;
-	rc = dasd_sleep_on(cqr);
+	rc = dasd_sleep_on_interruptible(cqr);
 	if (rc == 0) {
 		*data = *host_access;
 	} else {
diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c
index e1e88486b2b4..d138d0116734 100644
--- a/drivers/s390/block/dasd_erp.c
+++ b/drivers/s390/block/dasd_erp.c
@@ -169,12 +169,12 @@ dasd_log_sense(struct dasd_ccw_req *cqr, struct irb *irb)
 	device = cqr->startdev;
 	if (cqr->intrc == -ETIMEDOUT) {
 		dev_err(&device->cdev->dev,
-			"A timeout error occurred for cqr %p", cqr);
+			"A timeout error occurred for cqr %p\n", cqr);
 		return;
 	}
 	if (cqr->intrc == -ENOLINK) {
 		dev_err(&device->cdev->dev,
-			"A transport error occurred for cqr %p", cqr);
+			"A transport error occurred for cqr %p\n", cqr);
 		return;
 	}
 	/* dump sense data */
diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h
index ac7027e6d52b..87ff6cef872f 100644
--- a/drivers/s390/block/dasd_int.h
+++ b/drivers/s390/block/dasd_int.h
@@ -725,6 +725,7 @@ void dasd_block_clear_timer(struct dasd_block *);
 int  dasd_cancel_req(struct dasd_ccw_req *);
 int dasd_flush_device_queue(struct dasd_device *);
 int dasd_generic_probe (struct ccw_device *, struct dasd_discipline *);
+void dasd_generic_free_discipline(struct dasd_device *);
 void dasd_generic_remove (struct ccw_device *cdev);
 int dasd_generic_set_online(struct ccw_device *, struct dasd_discipline *);
 int dasd_generic_set_offline (struct ccw_device *cdev);
diff --git a/drivers/s390/char/con3270.c b/drivers/s390/char/con3270.c
index 6b1577c73fe7..285b4006f44b 100644
--- a/drivers/s390/char/con3270.c
+++ b/drivers/s390/char/con3270.c
@@ -124,7 +124,12 @@ con3270_create_status(struct con3270 *cp)
 static void
 con3270_update_string(struct con3270 *cp, struct string *s, int nr)
 {
-	if (s->len >= cp->view.cols - 5)
+	if (s->len < 4) {
+		/* This indicates a bug, but printing a warning would
+		 * cause a deadlock. */
+		return;
+	}
+	if (s->string[s->len - 4] != TO_RA)
 		return;
 	raw3270_buffer_address(cp->view.dev, s->string + s->len - 3,
 			       cp->view.cols * (nr + 1));
@@ -460,11 +465,11 @@ con3270_cline_end(struct con3270 *cp)
 		cp->cline->len + 4 : cp->view.cols;
 	s = con3270_alloc_string(cp, size);
 	memcpy(s->string, cp->cline->string, cp->cline->len);
-	if (s->len < cp->view.cols - 5) {
+	if (cp->cline->len < cp->view.cols - 5) {
 		s->string[s->len - 4] = TO_RA;
 		s->string[s->len - 1] = 0;
 	} else {
-		while (--size > cp->cline->len)
+		while (--size >= cp->cline->len)
 			s->string[size] = cp->view.ascebc[' '];
 	}
 	/* Replace cline with allocated line s and reset cline. */
diff --git a/drivers/s390/char/sclp_ctl.c b/drivers/s390/char/sclp_ctl.c
index ea607a4a1bdd..554eaa1e347d 100644
--- a/drivers/s390/char/sclp_ctl.c
+++ b/drivers/s390/char/sclp_ctl.c
@@ -126,21 +126,4 @@ static struct miscdevice sclp_ctl_device = {
 	.name = "sclp",
 	.fops = &sclp_ctl_fops,
 };
-
-/*
- * Register sclp_ctl misc device
- */
-static int __init sclp_ctl_init(void)
-{
-	return misc_register(&sclp_ctl_device);
-}
-module_init(sclp_ctl_init);
-
-/*
- * Deregister sclp_ctl misc device
- */
-static void __exit sclp_ctl_exit(void)
-{
-	misc_deregister(&sclp_ctl_device);
-}
-module_exit(sclp_ctl_exit);
+module_misc_device(sclp_ctl_device);
diff --git a/drivers/s390/char/tape_3590.c b/drivers/s390/char/tape_3590.c
index d3d1936057b4..e352047ed9f7 100644
--- a/drivers/s390/char/tape_3590.c
+++ b/drivers/s390/char/tape_3590.c
@@ -312,15 +312,10 @@ static int tape_3592_ioctl_kekl_set(struct tape_device *device,
 		return -ENOSYS;
 	if (!crypt_enabled(device))
 		return -EUNATCH;
-	ext_kekls = kmalloc(sizeof(*ext_kekls), GFP_KERNEL);
-	if (!ext_kekls)
-		return -ENOMEM;
-	if (copy_from_user(ext_kekls, (char __user *)arg, sizeof(*ext_kekls))) {
-		rc = -EFAULT;
-		goto out;
-	}
+	ext_kekls = memdup_user((char __user *)arg, sizeof(*ext_kekls));
+	if (IS_ERR(ext_kekls))
+		return PTR_ERR(ext_kekls);
 	rc = tape_3592_kekl_set(device, ext_kekls);
-out:
 	kfree(ext_kekls);
 	return rc;
 }
diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c
index 6c30e93ab8fa..ff18f373af9a 100644
--- a/drivers/s390/char/vmur.c
+++ b/drivers/s390/char/vmur.c
@@ -306,10 +306,11 @@ static void ur_int_handler(struct ccw_device *cdev, unsigned long intparm,
 {
 	struct urdev *urd;
 
-	TRACE("ur_int_handler: intparm=0x%lx cstat=%02x dstat=%02x res=%u\n",
-	      intparm, irb->scsw.cmd.cstat, irb->scsw.cmd.dstat,
-	      irb->scsw.cmd.count);
-
+	if (!IS_ERR(irb)) {
+		TRACE("ur_int_handler: intparm=0x%lx cstat=%02x dstat=%02x res=%u\n",
+		      intparm, irb->scsw.cmd.cstat, irb->scsw.cmd.dstat,
+		      irb->scsw.cmd.count);
+	}
 	if (!intparm) {
 		TRACE("ur_int_handler: unsolicited interrupt\n");
 		return;
diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c
index 940e725bde1e..11674698b36d 100644
--- a/drivers/s390/cio/chsc.c
+++ b/drivers/s390/cio/chsc.c
@@ -95,12 +95,13 @@ struct chsc_ssd_area {
 int chsc_get_ssd_info(struct subchannel_id schid, struct chsc_ssd_info *ssd)
 {
 	struct chsc_ssd_area *ssd_area;
+	unsigned long flags;
 	int ccode;
 	int ret;
 	int i;
 	int mask;
 
-	spin_lock_irq(&chsc_page_lock);
+	spin_lock_irqsave(&chsc_page_lock, flags);
 	memset(chsc_page, 0, PAGE_SIZE);
 	ssd_area = chsc_page;
 	ssd_area->request.length = 0x0010;
@@ -144,7 +145,7 @@ int chsc_get_ssd_info(struct subchannel_id schid, struct chsc_ssd_info *ssd)
 			ssd->fla[i] = ssd_area->fla[i];
 	}
 out:
-	spin_unlock_irq(&chsc_page_lock);
+	spin_unlock_irqrestore(&chsc_page_lock, flags);
 	return ret;
 }
 
@@ -832,9 +833,10 @@ int __chsc_do_secm(struct channel_subsystem *css, int enable)
 		u32 fmt : 4;
 		u32 : 16;
 	} __attribute__ ((packed)) *secm_area;
+	unsigned long flags;
 	int ret, ccode;
 
-	spin_lock_irq(&chsc_page_lock);
+	spin_lock_irqsave(&chsc_page_lock, flags);
 	memset(chsc_page, 0, PAGE_SIZE);
 	secm_area = chsc_page;
 	secm_area->request.length = 0x0050;
@@ -864,7 +866,7 @@ int __chsc_do_secm(struct channel_subsystem *css, int enable)
 		CIO_CRW_EVENT(2, "chsc: secm failed (rc=%04x)\n",
 			      secm_area->response.code);
 out:
-	spin_unlock_irq(&chsc_page_lock);
+	spin_unlock_irqrestore(&chsc_page_lock, flags);
 	return ret;
 }
 
@@ -992,6 +994,7 @@ chsc_initialize_cmg_chars(struct channel_path *chp, u8 cmcv,
 
 int chsc_get_channel_measurement_chars(struct channel_path *chp)
 {
+	unsigned long flags;
 	int ccode, ret;
 
 	struct {
@@ -1021,7 +1024,7 @@ int chsc_get_channel_measurement_chars(struct channel_path *chp)
 	if (!css_chsc_characteristics.scmc || !css_chsc_characteristics.secm)
 		return -EINVAL;
 
-	spin_lock_irq(&chsc_page_lock);
+	spin_lock_irqsave(&chsc_page_lock, flags);
 	memset(chsc_page, 0, PAGE_SIZE);
 	scmc_area = chsc_page;
 	scmc_area->request.length = 0x0010;
@@ -1053,7 +1056,7 @@ int chsc_get_channel_measurement_chars(struct channel_path *chp)
 	chsc_initialize_cmg_chars(chp, scmc_area->cmcv,
 				  (struct cmg_chars *) &scmc_area->data);
 out:
-	spin_unlock_irq(&chsc_page_lock);
+	spin_unlock_irqrestore(&chsc_page_lock, flags);
 	return ret;
 }
 
@@ -1134,6 +1137,7 @@ struct css_chsc_char css_chsc_characteristics;
 int __init
 chsc_determine_css_characteristics(void)
 {
+	unsigned long flags;
 	int result;
 	struct {
 		struct chsc_header request;
@@ -1146,7 +1150,7 @@ chsc_determine_css_characteristics(void)
 		u32 chsc_char[508];
 	} __attribute__ ((packed)) *scsc_area;
 
-	spin_lock_irq(&chsc_page_lock);
+	spin_lock_irqsave(&chsc_page_lock, flags);
 	memset(chsc_page, 0, PAGE_SIZE);
 	scsc_area = chsc_page;
 	scsc_area->request.length = 0x0010;
@@ -1168,7 +1172,7 @@ chsc_determine_css_characteristics(void)
 		CIO_CRW_EVENT(2, "chsc: scsc failed (rc=%04x)\n",
 			      scsc_area->response.code);
 exit:
-	spin_unlock_irq(&chsc_page_lock);
+	spin_unlock_irqrestore(&chsc_page_lock, flags);
 	return result;
 }
 
diff --git a/drivers/s390/cio/cio.h b/drivers/s390/cio/cio.h
index 93de0b46b489..f0e57aefb5f2 100644
--- a/drivers/s390/cio/cio.h
+++ b/drivers/s390/cio/cio.h
@@ -127,7 +127,6 @@ extern int cio_resume (struct subchannel *);
 extern int cio_halt (struct subchannel *);
 extern int cio_start (struct subchannel *, struct ccw1 *, __u8);
 extern int cio_start_key (struct subchannel *, struct ccw1 *, __u8, __u8);
-extern int cio_cancel (struct subchannel *);
 extern int cio_set_options (struct subchannel *, int);
 extern int cio_update_schib(struct subchannel *sch);
 extern int cio_commit_config(struct subchannel *sch);
diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c
index 7ada078ffdd0..6a58bc8f46e2 100644
--- a/drivers/s390/cio/device.c
+++ b/drivers/s390/cio/device.c
@@ -762,7 +762,6 @@ static int io_subchannel_initialize_dev(struct subchannel *sch,
 	priv->state = DEV_STATE_NOT_OPER;
 	priv->dev_id.devno = sch->schib.pmcw.dev;
 	priv->dev_id.ssid = sch->schid.ssid;
-	priv->schid = sch->schid;
 
 	INIT_WORK(&priv->todo_work, ccw_device_todo);
 	INIT_LIST_HEAD(&priv->cmb_list);
@@ -1000,7 +999,6 @@ static int ccw_device_move_to_sch(struct ccw_device *cdev,
 	put_device(&old_sch->dev);
 	/* Initialize new subchannel. */
 	spin_lock_irq(sch->lock);
-	cdev->private->schid = sch->schid;
 	cdev->ccwlock = sch->lock;
 	if (!sch_is_pseudo_sch(sch))
 		sch_set_cdev(sch, cdev);
diff --git a/drivers/s390/cio/device_status.c b/drivers/s390/cio/device_status.c
index 15b56a15db15..9bc3512374c9 100644
--- a/drivers/s390/cio/device_status.c
+++ b/drivers/s390/cio/device_status.c
@@ -26,6 +26,7 @@
 static void
 ccw_device_msg_control_check(struct ccw_device *cdev, struct irb *irb)
 {
+	struct subchannel *sch = to_subchannel(cdev->dev.parent);
 	char dbf_text[15];
 
 	if (!scsw_is_valid_cstat(&irb->scsw) ||
@@ -36,10 +37,10 @@ ccw_device_msg_control_check(struct ccw_device *cdev, struct irb *irb)
 		      "received"
 		      " ... device %04x on subchannel 0.%x.%04x, dev_stat "
 		      ": %02X sch_stat : %02X\n",
-		      cdev->private->dev_id.devno, cdev->private->schid.ssid,
-		      cdev->private->schid.sch_no,
+		      cdev->private->dev_id.devno, sch->schid.ssid,
+		      sch->schid.sch_no,
 		      scsw_dstat(&irb->scsw), scsw_cstat(&irb->scsw));
-	sprintf(dbf_text, "chk%x", cdev->private->schid.sch_no);
+	sprintf(dbf_text, "chk%x", sch->schid.sch_no);
 	CIO_TRACE_EVENT(0, dbf_text);
 	CIO_HEX_EVENT(0, irb, sizeof(struct irb));
 }
diff --git a/drivers/s390/cio/io_sch.h b/drivers/s390/cio/io_sch.h
index 8975060af96c..220f49145b2f 100644
--- a/drivers/s390/cio/io_sch.h
+++ b/drivers/s390/cio/io_sch.h
@@ -120,7 +120,6 @@ struct ccw_device_private {
 	int state;		/* device state */
 	atomic_t onoff;
 	struct ccw_dev_id dev_id;	/* device id */
-	struct subchannel_id schid;	/* subchannel number */
 	struct ccw_request req;		/* internal I/O request */
 	int iretry;
 	u8 pgid_valid_mask;	/* mask of valid PGIDs */
diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c
index 4bb5262f7aee..71bf9bded485 100644
--- a/drivers/s390/cio/qdio_main.c
+++ b/drivers/s390/cio/qdio_main.c
@@ -686,6 +686,15 @@ static void qdio_kick_handler(struct qdio_q *q)
 	q->qdio_error = 0;
 }
 
+static inline int qdio_tasklet_schedule(struct qdio_q *q)
+{
+	if (likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE)) {
+		tasklet_schedule(&q->tasklet);
+		return 0;
+	}
+	return -EPERM;
+}
+
 static void __qdio_inbound_processing(struct qdio_q *q)
 {
 	qperf_inc(q, tasklet_inbound);
@@ -698,10 +707,8 @@ static void __qdio_inbound_processing(struct qdio_q *q)
 	if (!qdio_inbound_q_done(q)) {
 		/* means poll time is not yet over */
 		qperf_inc(q, tasklet_inbound_resched);
-		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
-			tasklet_schedule(&q->tasklet);
+		if (!qdio_tasklet_schedule(q))
 			return;
-		}
 	}
 
 	qdio_stop_polling(q);
@@ -711,8 +718,7 @@ static void __qdio_inbound_processing(struct qdio_q *q)
 	 */
 	if (!qdio_inbound_q_done(q)) {
 		qperf_inc(q, tasklet_inbound_resched2);
-		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
-			tasklet_schedule(&q->tasklet);
+		qdio_tasklet_schedule(q);
 	}
 }
 
@@ -869,16 +875,15 @@ static void __qdio_outbound_processing(struct qdio_q *q)
 	 * is noticed and outbound_handler is called after some time.
 	 */
 	if (qdio_outbound_q_done(q))
-		del_timer(&q->u.out.timer);
+		del_timer_sync(&q->u.out.timer);
 	else
-		if (!timer_pending(&q->u.out.timer))
+		if (!timer_pending(&q->u.out.timer) &&
+		    likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE))
 			mod_timer(&q->u.out.timer, jiffies + 10 * HZ);
 	return;
 
 sched:
-	if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
-		return;
-	tasklet_schedule(&q->tasklet);
+	qdio_tasklet_schedule(q);
 }
 
 /* outbound tasklet */
@@ -892,9 +897,7 @@ void qdio_outbound_timer(unsigned long data)
 {
 	struct qdio_q *q = (struct qdio_q *)data;
 
-	if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
-		return;
-	tasklet_schedule(&q->tasklet);
+	qdio_tasklet_schedule(q);
 }
 
 static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
@@ -907,7 +910,7 @@ static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
 
 	for_each_output_queue(q->irq_ptr, out, i)
 		if (!qdio_outbound_q_done(out))
-			tasklet_schedule(&out->tasklet);
+			qdio_tasklet_schedule(out);
 }
 
 static void __tiqdio_inbound_processing(struct qdio_q *q)
@@ -929,10 +932,8 @@ static void __tiqdio_inbound_processing(struct qdio_q *q)
 
 	if (!qdio_inbound_q_done(q)) {
 		qperf_inc(q, tasklet_inbound_resched);
-		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
-			tasklet_schedule(&q->tasklet);
+		if (!qdio_tasklet_schedule(q))
 			return;
-		}
 	}
 
 	qdio_stop_polling(q);
@@ -942,8 +943,7 @@ static void __tiqdio_inbound_processing(struct qdio_q *q)
 	 */
 	if (!qdio_inbound_q_done(q)) {
 		qperf_inc(q, tasklet_inbound_resched2);
-		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
-			tasklet_schedule(&q->tasklet);
+		qdio_tasklet_schedule(q);
 	}
 }
 
@@ -977,7 +977,7 @@ static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
 	int i;
 	struct qdio_q *q;
 
-	if (unlikely(irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
+	if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
 		return;
 
 	for_each_input_queue(irq_ptr, q, i) {
@@ -1003,7 +1003,7 @@ static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
 			continue;
 		if (need_siga_sync(q) && need_siga_sync_out_after_pci(q))
 			qdio_siga_sync_q(q);
-		tasklet_schedule(&q->tasklet);
+		qdio_tasklet_schedule(q);
 	}
 }
 
@@ -1066,10 +1066,12 @@ void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
 		      struct irb *irb)
 {
 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
+	struct subchannel_id schid;
 	int cstat, dstat;
 
 	if (!intparm || !irq_ptr) {
-		DBF_ERROR("qint:%4x", cdev->private->schid.sch_no);
+		ccw_device_get_schid(cdev, &schid);
+		DBF_ERROR("qint:%4x", schid.sch_no);
 		return;
 	}
 
@@ -1122,12 +1124,14 @@ void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
 int qdio_get_ssqd_desc(struct ccw_device *cdev,
 		       struct qdio_ssqd_desc *data)
 {
+	struct subchannel_id schid;
 
 	if (!cdev || !cdev->private)
 		return -EINVAL;
 
-	DBF_EVENT("get ssqd:%4x", cdev->private->schid.sch_no);
-	return qdio_setup_get_ssqd(NULL, &cdev->private->schid, data);
+	ccw_device_get_schid(cdev, &schid);
+	DBF_EVENT("get ssqd:%4x", schid.sch_no);
+	return qdio_setup_get_ssqd(NULL, &schid, data);
 }
 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
 
@@ -1141,7 +1145,7 @@ static void qdio_shutdown_queues(struct ccw_device *cdev)
 		tasklet_kill(&q->tasklet);
 
 	for_each_output_queue(irq_ptr, q, i) {
-		del_timer(&q->u.out.timer);
+		del_timer_sync(&q->u.out.timer);
 		tasklet_kill(&q->tasklet);
 	}
 }
@@ -1154,14 +1158,15 @@ static void qdio_shutdown_queues(struct ccw_device *cdev)
 int qdio_shutdown(struct ccw_device *cdev, int how)
 {
 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
+	struct subchannel_id schid;
 	int rc;
-	unsigned long flags;
 
 	if (!irq_ptr)
 		return -ENODEV;
 
 	WARN_ON_ONCE(irqs_disabled());
-	DBF_EVENT("qshutdown:%4x", cdev->private->schid.sch_no);
+	ccw_device_get_schid(cdev, &schid);
+	DBF_EVENT("qshutdown:%4x", schid.sch_no);
 
 	mutex_lock(&irq_ptr->setup_mutex);
 	/*
@@ -1184,7 +1189,7 @@ int qdio_shutdown(struct ccw_device *cdev, int how)
 	qdio_shutdown_debug_entries(irq_ptr);
 
 	/* cleanup subchannel */
-	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
+	spin_lock_irq(get_ccwdev_lock(cdev));
 
 	if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
 		rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
@@ -1198,12 +1203,12 @@ int qdio_shutdown(struct ccw_device *cdev, int how)
 	}
 
 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
-	spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
+	spin_unlock_irq(get_ccwdev_lock(cdev));
 	wait_event_interruptible_timeout(cdev->private->wait_q,
 		irq_ptr->state == QDIO_IRQ_STATE_INACTIVE ||
 		irq_ptr->state == QDIO_IRQ_STATE_ERR,
 		10 * HZ);
-	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
+	spin_lock_irq(get_ccwdev_lock(cdev));
 
 no_cleanup:
 	qdio_shutdown_thinint(irq_ptr);
@@ -1211,7 +1216,7 @@ no_cleanup:
 	/* restore interrupt handler */
 	if ((void *)cdev->handler == (void *)qdio_int_handler)
 		cdev->handler = irq_ptr->orig_handler;
-	spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
+	spin_unlock_irq(get_ccwdev_lock(cdev));
 
 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
 	mutex_unlock(&irq_ptr->setup_mutex);
@@ -1228,11 +1233,13 @@ EXPORT_SYMBOL_GPL(qdio_shutdown);
 int qdio_free(struct ccw_device *cdev)
 {
 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
+	struct subchannel_id schid;
 
 	if (!irq_ptr)
 		return -ENODEV;
 
-	DBF_EVENT("qfree:%4x", cdev->private->schid.sch_no);
+	ccw_device_get_schid(cdev, &schid);
+	DBF_EVENT("qfree:%4x", schid.sch_no);
 	DBF_DEV_EVENT(DBF_ERR, irq_ptr, "dbf abandoned");
 	mutex_lock(&irq_ptr->setup_mutex);
 
@@ -1251,9 +1258,11 @@ EXPORT_SYMBOL_GPL(qdio_free);
  */
 int qdio_allocate(struct qdio_initialize *init_data)
 {
+	struct subchannel_id schid;
 	struct qdio_irq *irq_ptr;
 
-	DBF_EVENT("qallocate:%4x", init_data->cdev->private->schid.sch_no);
+	ccw_device_get_schid(init_data->cdev, &schid);
+	DBF_EVENT("qallocate:%4x", schid.sch_no);
 
 	if ((init_data->no_input_qs && !init_data->input_handler) ||
 	    (init_data->no_output_qs && !init_data->output_handler))
@@ -1331,20 +1340,18 @@ static void qdio_detect_hsicq(struct qdio_irq *irq_ptr)
  */
 int qdio_establish(struct qdio_initialize *init_data)
 {
-	struct qdio_irq *irq_ptr;
 	struct ccw_device *cdev = init_data->cdev;
-	unsigned long saveflags;
+	struct subchannel_id schid;
+	struct qdio_irq *irq_ptr;
 	int rc;
 
-	DBF_EVENT("qestablish:%4x", cdev->private->schid.sch_no);
+	ccw_device_get_schid(cdev, &schid);
+	DBF_EVENT("qestablish:%4x", schid.sch_no);
 
 	irq_ptr = cdev->private->qdio_data;
 	if (!irq_ptr)
 		return -ENODEV;
 
-	if (cdev->private->state != DEV_STATE_ONLINE)
-		return -EINVAL;
-
 	mutex_lock(&irq_ptr->setup_mutex);
 	qdio_setup_irq(init_data);
 
@@ -1361,17 +1368,14 @@ int qdio_establish(struct qdio_initialize *init_data)
 	irq_ptr->ccw.count = irq_ptr->equeue.count;
 	irq_ptr->ccw.cda = (u32)((addr_t)irq_ptr->qdr);
 
-	spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
+	spin_lock_irq(get_ccwdev_lock(cdev));
 	ccw_device_set_options_mask(cdev, 0);
 
 	rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0);
+	spin_unlock_irq(get_ccwdev_lock(cdev));
 	if (rc) {
 		DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
 		DBF_ERROR("rc:%4x", rc);
-	}
-	spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
-
-	if (rc) {
 		mutex_unlock(&irq_ptr->setup_mutex);
 		qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
 		return rc;
@@ -1407,19 +1411,17 @@ EXPORT_SYMBOL_GPL(qdio_establish);
  */
 int qdio_activate(struct ccw_device *cdev)
 {
+	struct subchannel_id schid;
 	struct qdio_irq *irq_ptr;
 	int rc;
-	unsigned long saveflags;
 
-	DBF_EVENT("qactivate:%4x", cdev->private->schid.sch_no);
+	ccw_device_get_schid(cdev, &schid);
+	DBF_EVENT("qactivate:%4x", schid.sch_no);
 
 	irq_ptr = cdev->private->qdio_data;
 	if (!irq_ptr)
 		return -ENODEV;
 
-	if (cdev->private->state != DEV_STATE_ONLINE)
-		return -EINVAL;
-
 	mutex_lock(&irq_ptr->setup_mutex);
 	if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
 		rc = -EBUSY;
@@ -1431,19 +1433,17 @@ int qdio_activate(struct ccw_device *cdev)
 	irq_ptr->ccw.count = irq_ptr->aqueue.count;
 	irq_ptr->ccw.cda = 0;
 
-	spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
+	spin_lock_irq(get_ccwdev_lock(cdev));
 	ccw_device_set_options(cdev, CCWDEV_REPORT_ALL);
 
 	rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ACTIVATE,
 			      0, DOIO_DENY_PREFETCH);
+	spin_unlock_irq(get_ccwdev_lock(cdev));
 	if (rc) {
 		DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no);
 		DBF_ERROR("rc:%4x", rc);
-	}
-	spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
-
-	if (rc)
 		goto out;
+	}
 
 	if (is_thinint_irq(irq_ptr))
 		tiqdio_add_input_queues(irq_ptr);
@@ -1585,10 +1585,11 @@ static int handle_outbound(struct qdio_q *q, unsigned int callflags,
 
 	/* in case of SIGA errors we must process the error immediately */
 	if (used >= q->u.out.scan_threshold || rc)
-		tasklet_schedule(&q->tasklet);
+		qdio_tasklet_schedule(q);
 	else
 		/* free the SBALs in case of no further traffic */
-		if (!timer_pending(&q->u.out.timer))
+		if (!timer_pending(&q->u.out.timer) &&
+		    likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE))
 			mod_timer(&q->u.out.timer, jiffies + HZ);
 	return rc;
 }
diff --git a/drivers/s390/net/qeth_core.h b/drivers/s390/net/qeth_core.h
index bf40063de202..6d4b68c483f3 100644
--- a/drivers/s390/net/qeth_core.h
+++ b/drivers/s390/net/qeth_core.h
@@ -999,6 +999,7 @@ struct qeth_cmd_buffer *qeth_get_setassparms_cmd(struct qeth_card *,
 						 __u16, __u16,
 						 enum qeth_prot_versions);
 int qeth_set_features(struct net_device *, netdev_features_t);
+int qeth_recover_features(struct net_device *);
 netdev_features_t qeth_fix_features(struct net_device *, netdev_features_t);
 
 /* exports for OSN */
diff --git a/drivers/s390/net/qeth_core_main.c b/drivers/s390/net/qeth_core_main.c
index 7dba6c8537a1..20cf29613043 100644
--- a/drivers/s390/net/qeth_core_main.c
+++ b/drivers/s390/net/qeth_core_main.c
@@ -3619,7 +3619,8 @@ static void qeth_qdio_cq_handler(struct qeth_card *card,
 		int e;
 
 		e = 0;
-		while (buffer->element[e].addr) {
+		while ((e < QDIO_MAX_ELEMENTS_PER_BUFFER) &&
+		       buffer->element[e].addr) {
 			unsigned long phys_aob_addr;
 
 			phys_aob_addr = (unsigned long) buffer->element[e].addr;
@@ -6131,6 +6132,35 @@ static int qeth_set_ipa_tso(struct qeth_card *card, int on)
 	return rc;
 }
 
+/* try to restore device features on a device after recovery */
+int qeth_recover_features(struct net_device *dev)
+{
+	struct qeth_card *card = dev->ml_priv;
+	netdev_features_t recover = dev->features;
+
+	if (recover & NETIF_F_IP_CSUM) {
+		if (qeth_set_ipa_csum(card, 1, IPA_OUTBOUND_CHECKSUM))
+			recover ^= NETIF_F_IP_CSUM;
+	}
+	if (recover & NETIF_F_RXCSUM) {
+		if (qeth_set_ipa_csum(card, 1, IPA_INBOUND_CHECKSUM))
+			recover ^= NETIF_F_RXCSUM;
+	}
+	if (recover & NETIF_F_TSO) {
+		if (qeth_set_ipa_tso(card, 1))
+			recover ^= NETIF_F_TSO;
+	}
+
+	if (recover == dev->features)
+		return 0;
+
+	dev_warn(&card->gdev->dev,
+		 "Device recovery failed to restore all offload features\n");
+	dev->features = recover;
+	return -EIO;
+}
+EXPORT_SYMBOL_GPL(qeth_recover_features);
+
 int qeth_set_features(struct net_device *dev, netdev_features_t features)
 {
 	struct qeth_card *card = dev->ml_priv;
diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c
index 7bc20c5188bc..bb27058fa9f0 100644
--- a/drivers/s390/net/qeth_l2_main.c
+++ b/drivers/s390/net/qeth_l2_main.c
@@ -1124,14 +1124,11 @@ static int qeth_l2_setup_netdev(struct qeth_card *card)
 			card->dev->hw_features |= NETIF_F_RXCSUM;
 			card->dev->vlan_features |= NETIF_F_RXCSUM;
 		}
-		/* Turn on SG per default */
-		card->dev->features |= NETIF_F_SG;
 	}
 	card->info.broadcast_capable = 1;
 	qeth_l2_request_initial_mac(card);
 	card->dev->gso_max_size = (QETH_MAX_BUFFER_ELEMENTS(card) - 1) *
 				  PAGE_SIZE;
-	card->dev->gso_max_segs = (QETH_MAX_BUFFER_ELEMENTS(card) - 1);
 	SET_NETDEV_DEV(card->dev, &card->gdev->dev);
 	netif_napi_add(card->dev, &card->napi, qeth_l2_poll, QETH_NAPI_WEIGHT);
 	netif_carrier_off(card->dev);
@@ -1246,6 +1243,9 @@ contin:
 		}
 		/* this also sets saved unicast addresses */
 		qeth_l2_set_rx_mode(card->dev);
+		rtnl_lock();
+		qeth_recover_features(card->dev);
+		rtnl_unlock();
 	}
 	/* let user_space know that device is online */
 	kobject_uevent(&gdev->dev.kobj, KOBJ_CHANGE);
diff --git a/drivers/s390/net/qeth_l3_main.c b/drivers/s390/net/qeth_l3_main.c
index 72934666fedf..272d9e7419be 100644
--- a/drivers/s390/net/qeth_l3_main.c
+++ b/drivers/s390/net/qeth_l3_main.c
@@ -257,6 +257,11 @@ int qeth_l3_delete_ip(struct qeth_card *card, struct qeth_ipaddr *tmp_addr)
 	if (addr->in_progress)
 		return -EINPROGRESS;
 
+	if (!qeth_card_hw_is_reachable(card)) {
+		addr->disp_flag = QETH_DISP_ADDR_DELETE;
+		return 0;
+	}
+
 	rc = qeth_l3_deregister_addr_entry(card, addr);
 
 	hash_del(&addr->hnode);
@@ -296,6 +301,11 @@ int qeth_l3_add_ip(struct qeth_card *card, struct qeth_ipaddr *tmp_addr)
 		hash_add(card->ip_htable, &addr->hnode,
 				qeth_l3_ipaddr_hash(addr));
 
+		if (!qeth_card_hw_is_reachable(card)) {
+			addr->disp_flag = QETH_DISP_ADDR_ADD;
+			return 0;
+		}
+
 		/* qeth_l3_register_addr_entry can go to sleep
 		 * if we add a IPV4 addr. It is caused by the reason
 		 * that SETIP ipa cmd starts ARP staff for IPV4 addr.
@@ -390,12 +400,16 @@ static void qeth_l3_recover_ip(struct qeth_card *card)
 	int i;
 	int rc;
 
-	QETH_CARD_TEXT(card, 4, "recoverip");
+	QETH_CARD_TEXT(card, 4, "recovrip");
 
 	spin_lock_bh(&card->ip_lock);
 
 	hash_for_each_safe(card->ip_htable, i, tmp, addr, hnode) {
-		if (addr->disp_flag == QETH_DISP_ADDR_ADD) {
+		if (addr->disp_flag == QETH_DISP_ADDR_DELETE) {
+			qeth_l3_deregister_addr_entry(card, addr);
+			hash_del(&addr->hnode);
+			kfree(addr);
+		} else if (addr->disp_flag == QETH_DISP_ADDR_ADD) {
 			if (addr->proto == QETH_PROT_IPV4) {
 				addr->in_progress = 1;
 				spin_unlock_bh(&card->ip_lock);
@@ -407,10 +421,8 @@ static void qeth_l3_recover_ip(struct qeth_card *card)
 
 			if (!rc) {
 				addr->disp_flag = QETH_DISP_ADDR_DO_NOTHING;
-				if (addr->ref_counter < 1) {
+				if (addr->ref_counter < 1)
 					qeth_l3_delete_ip(card, addr);
-					kfree(addr);
-				}
 			} else {
 				hash_del(&addr->hnode);
 				kfree(addr);
@@ -689,7 +701,7 @@ int qeth_l3_add_vipa(struct qeth_card *card, enum qeth_prot_versions proto,
 
 	spin_lock_bh(&card->ip_lock);
 
-	if (!qeth_l3_ip_from_hash(card, ipaddr))
+	if (qeth_l3_ip_from_hash(card, ipaddr))
 		rc = -EEXIST;
 	else
 		qeth_l3_add_ip(card, ipaddr);
@@ -757,7 +769,7 @@ int qeth_l3_add_rxip(struct qeth_card *card, enum qeth_prot_versions proto,
 
 	spin_lock_bh(&card->ip_lock);
 
-	if (!qeth_l3_ip_from_hash(card, ipaddr))
+	if (qeth_l3_ip_from_hash(card, ipaddr))
 		rc = -EEXIST;
 	else
 		qeth_l3_add_ip(card, ipaddr);
@@ -3108,7 +3120,6 @@ static int qeth_l3_setup_netdev(struct qeth_card *card)
 				card->dev->vlan_features = NETIF_F_SG |
 					NETIF_F_RXCSUM | NETIF_F_IP_CSUM |
 					NETIF_F_TSO;
-				card->dev->features = NETIF_F_SG;
 			}
 		}
 	} else if (card->info.type == QETH_CARD_TYPE_IQD) {
@@ -3136,7 +3147,6 @@ static int qeth_l3_setup_netdev(struct qeth_card *card)
 	netif_keep_dst(card->dev);
 	card->dev->gso_max_size = (QETH_MAX_BUFFER_ELEMENTS(card) - 1) *
 				  PAGE_SIZE;
-	card->dev->gso_max_segs = (QETH_MAX_BUFFER_ELEMENTS(card) - 1);
 
 	SET_NETDEV_DEV(card->dev, &card->gdev->dev);
 	netif_napi_add(card->dev, &card->napi, qeth_l3_poll, QETH_NAPI_WEIGHT);
@@ -3269,6 +3279,7 @@ contin:
 		else
 			dev_open(card->dev);
 		qeth_l3_set_multicast_list(card->dev);
+		qeth_recover_features(card->dev);
 		rtnl_unlock();
 	}
 	qeth_trace_features(card);
diff --git a/drivers/s390/net/qeth_l3_sys.c b/drivers/s390/net/qeth_l3_sys.c
index 65645b11fc19..0e00a5ce0f00 100644
--- a/drivers/s390/net/qeth_l3_sys.c
+++ b/drivers/s390/net/qeth_l3_sys.c
@@ -297,7 +297,9 @@ static ssize_t qeth_l3_dev_hsuid_store(struct device *dev,
 		addr->u.a6.pfxlen = 0;
 		addr->type = QETH_IP_TYPE_NORMAL;
 
+		spin_lock_bh(&card->ip_lock);
 		qeth_l3_delete_ip(card, addr);
+		spin_unlock_bh(&card->ip_lock);
 		kfree(addr);
 	}
 
@@ -329,7 +331,10 @@ static ssize_t qeth_l3_dev_hsuid_store(struct device *dev,
 		addr->type = QETH_IP_TYPE_NORMAL;
 	} else
 		return -ENOMEM;
+
+	spin_lock_bh(&card->ip_lock);
 	qeth_l3_add_ip(card, addr);
+	spin_unlock_bh(&card->ip_lock);
 	kfree(addr);
 
 	return count;
diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c
index c00ac4650dce..bcc8f3dfd4c4 100644
--- a/drivers/s390/scsi/zfcp_aux.c
+++ b/drivers/s390/scsi/zfcp_aux.c
@@ -310,7 +310,7 @@ static int zfcp_setup_adapter_work_queue(struct zfcp_adapter *adapter)
 
 	snprintf(name, sizeof(name), "zfcp_q_%s",
 		 dev_name(&adapter->ccw_device->dev));
-	adapter->work_queue = create_singlethread_workqueue(name);
+	adapter->work_queue = alloc_ordered_workqueue(name, WQ_MEM_RECLAIM);
 
 	if (adapter->work_queue)
 		return 0;
diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c
index 5d7fbe4e907e..637cf8973c9e 100644
--- a/drivers/s390/scsi/zfcp_dbf.c
+++ b/drivers/s390/scsi/zfcp_dbf.c
@@ -3,7 +3,7 @@
  *
  * Debug traces for zfcp.
  *
- * Copyright IBM Corp. 2002, 2013
+ * Copyright IBM Corp. 2002, 2016
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -65,7 +65,7 @@ void zfcp_dbf_pl_write(struct zfcp_dbf *dbf, void *data, u16 length, char *area,
  * @tag: tag indicating which kind of unsolicited status has been received
  * @req: request for which a response was received
  */
-void zfcp_dbf_hba_fsf_res(char *tag, struct zfcp_fsf_req *req)
+void zfcp_dbf_hba_fsf_res(char *tag, int level, struct zfcp_fsf_req *req)
 {
 	struct zfcp_dbf *dbf = req->adapter->dbf;
 	struct fsf_qtcb_prefix *q_pref = &req->qtcb->prefix;
@@ -85,6 +85,8 @@ void zfcp_dbf_hba_fsf_res(char *tag, struct zfcp_fsf_req *req)
 	rec->u.res.req_issued = req->issued;
 	rec->u.res.prot_status = q_pref->prot_status;
 	rec->u.res.fsf_status = q_head->fsf_status;
+	rec->u.res.port_handle = q_head->port_handle;
+	rec->u.res.lun_handle = q_head->lun_handle;
 
 	memcpy(rec->u.res.prot_status_qual, &q_pref->prot_status_qual,
 	       FSF_PROT_STATUS_QUAL_SIZE);
@@ -97,7 +99,7 @@ void zfcp_dbf_hba_fsf_res(char *tag, struct zfcp_fsf_req *req)
 				  rec->pl_len, "fsf_res", req->req_id);
 	}
 
-	debug_event(dbf->hba, 1, rec, sizeof(*rec));
+	debug_event(dbf->hba, level, rec, sizeof(*rec));
 	spin_unlock_irqrestore(&dbf->hba_lock, flags);
 }
 
@@ -241,7 +243,8 @@ static void zfcp_dbf_set_common(struct zfcp_dbf_rec *rec,
 	if (sdev) {
 		rec->lun_status = atomic_read(&sdev_to_zfcp(sdev)->status);
 		rec->lun = zfcp_scsi_dev_lun(sdev);
-	}
+	} else
+		rec->lun = ZFCP_DBF_INVALID_LUN;
 }
 
 /**
@@ -320,13 +323,48 @@ void zfcp_dbf_rec_run(char *tag, struct zfcp_erp_action *erp)
 	spin_unlock_irqrestore(&dbf->rec_lock, flags);
 }
 
+/**
+ * zfcp_dbf_rec_run_wka - trace wka port event with info like running recovery
+ * @tag: identifier for event
+ * @wka_port: well known address port
+ * @req_id: request ID to correlate with potential HBA trace record
+ */
+void zfcp_dbf_rec_run_wka(char *tag, struct zfcp_fc_wka_port *wka_port,
+			  u64 req_id)
+{
+	struct zfcp_dbf *dbf = wka_port->adapter->dbf;
+	struct zfcp_dbf_rec *rec = &dbf->rec_buf;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dbf->rec_lock, flags);
+	memset(rec, 0, sizeof(*rec));
+
+	rec->id = ZFCP_DBF_REC_RUN;
+	memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN);
+	rec->port_status = wka_port->status;
+	rec->d_id = wka_port->d_id;
+	rec->lun = ZFCP_DBF_INVALID_LUN;
+
+	rec->u.run.fsf_req_id = req_id;
+	rec->u.run.rec_status = ~0;
+	rec->u.run.rec_step = ~0;
+	rec->u.run.rec_action = ~0;
+	rec->u.run.rec_count = ~0;
+
+	debug_event(dbf->rec, 1, rec, sizeof(*rec));
+	spin_unlock_irqrestore(&dbf->rec_lock, flags);
+}
+
 static inline
-void zfcp_dbf_san(char *tag, struct zfcp_dbf *dbf, void *data, u8 id, u16 len,
-		  u64 req_id, u32 d_id)
+void zfcp_dbf_san(char *tag, struct zfcp_dbf *dbf,
+		  char *paytag, struct scatterlist *sg, u8 id, u16 len,
+		  u64 req_id, u32 d_id, u16 cap_len)
 {
 	struct zfcp_dbf_san *rec = &dbf->san_buf;
 	u16 rec_len;
 	unsigned long flags;
+	struct zfcp_dbf_pay *payload = &dbf->pay_buf;
+	u16 pay_sum = 0;
 
 	spin_lock_irqsave(&dbf->san_lock, flags);
 	memset(rec, 0, sizeof(*rec));
@@ -334,10 +372,41 @@ void zfcp_dbf_san(char *tag, struct zfcp_dbf *dbf, void *data, u8 id, u16 len,
 	rec->id = id;
 	rec->fsf_req_id = req_id;
 	rec->d_id = d_id;
-	rec_len = min(len, (u16)ZFCP_DBF_SAN_MAX_PAYLOAD);
-	memcpy(rec->payload, data, rec_len);
 	memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN);
+	rec->pl_len = len; /* full length even if we cap pay below */
+	if (!sg)
+		goto out;
+	rec_len = min_t(unsigned int, sg->length, ZFCP_DBF_SAN_MAX_PAYLOAD);
+	memcpy(rec->payload, sg_virt(sg), rec_len); /* part of 1st sg entry */
+	if (len <= rec_len)
+		goto out; /* skip pay record if full content in rec->payload */
+
+	/* if (len > rec_len):
+	 * dump data up to cap_len ignoring small duplicate in rec->payload
+	 */
+	spin_lock_irqsave(&dbf->pay_lock, flags);
+	memset(payload, 0, sizeof(*payload));
+	memcpy(payload->area, paytag, ZFCP_DBF_TAG_LEN);
+	payload->fsf_req_id = req_id;
+	payload->counter = 0;
+	for (; sg && pay_sum < cap_len; sg = sg_next(sg)) {
+		u16 pay_len, offset = 0;
+
+		while (offset < sg->length && pay_sum < cap_len) {
+			pay_len = min((u16)ZFCP_DBF_PAY_MAX_REC,
+				      (u16)(sg->length - offset));
+			/* cap_len <= pay_sum < cap_len+ZFCP_DBF_PAY_MAX_REC */
+			memcpy(payload->data, sg_virt(sg) + offset, pay_len);
+			debug_event(dbf->pay, 1, payload,
+				    zfcp_dbf_plen(pay_len));
+			payload->counter++;
+			offset += pay_len;
+			pay_sum += pay_len;
+		}
+	}
+	spin_unlock(&dbf->pay_lock);
 
+out:
 	debug_event(dbf->san, 1, rec, sizeof(*rec));
 	spin_unlock_irqrestore(&dbf->san_lock, flags);
 }
@@ -354,9 +423,62 @@ void zfcp_dbf_san_req(char *tag, struct zfcp_fsf_req *fsf, u32 d_id)
 	struct zfcp_fsf_ct_els *ct_els = fsf->data;
 	u16 length;
 
-	length = (u16)(ct_els->req->length + FC_CT_HDR_LEN);
-	zfcp_dbf_san(tag, dbf, sg_virt(ct_els->req), ZFCP_DBF_SAN_REQ, length,
-		     fsf->req_id, d_id);
+	length = (u16)zfcp_qdio_real_bytes(ct_els->req);
+	zfcp_dbf_san(tag, dbf, "san_req", ct_els->req, ZFCP_DBF_SAN_REQ,
+		     length, fsf->req_id, d_id, length);
+}
+
+static u16 zfcp_dbf_san_res_cap_len_if_gpn_ft(char *tag,
+					      struct zfcp_fsf_req *fsf,
+					      u16 len)
+{
+	struct zfcp_fsf_ct_els *ct_els = fsf->data;
+	struct fc_ct_hdr *reqh = sg_virt(ct_els->req);
+	struct fc_ns_gid_ft *reqn = (struct fc_ns_gid_ft *)(reqh + 1);
+	struct scatterlist *resp_entry = ct_els->resp;
+	struct fc_gpn_ft_resp *acc;
+	int max_entries, x, last = 0;
+
+	if (!(memcmp(tag, "fsscth2", 7) == 0
+	      && ct_els->d_id == FC_FID_DIR_SERV
+	      && reqh->ct_rev == FC_CT_REV
+	      && reqh->ct_in_id[0] == 0
+	      && reqh->ct_in_id[1] == 0
+	      && reqh->ct_in_id[2] == 0
+	      && reqh->ct_fs_type == FC_FST_DIR
+	      && reqh->ct_fs_subtype == FC_NS_SUBTYPE
+	      && reqh->ct_options == 0
+	      && reqh->_ct_resvd1 == 0
+	      && reqh->ct_cmd == FC_NS_GPN_FT
+	      /* reqh->ct_mr_size can vary so do not match but read below */
+	      && reqh->_ct_resvd2 == 0
+	      && reqh->ct_reason == 0
+	      && reqh->ct_explan == 0
+	      && reqh->ct_vendor == 0
+	      && reqn->fn_resvd == 0
+	      && reqn->fn_domain_id_scope == 0
+	      && reqn->fn_area_id_scope == 0
+	      && reqn->fn_fc4_type == FC_TYPE_FCP))
+		return len; /* not GPN_FT response so do not cap */
+
+	acc = sg_virt(resp_entry);
+	max_entries = (reqh->ct_mr_size * 4 / sizeof(struct fc_gpn_ft_resp))
+		+ 1 /* zfcp_fc_scan_ports: bytes correct, entries off-by-one
+		     * to account for header as 1st pseudo "entry" */;
+
+	/* the basic CT_IU preamble is the same size as one entry in the GPN_FT
+	 * response, allowing us to skip special handling for it - just skip it
+	 */
+	for (x = 1; x < max_entries && !last; x++) {
+		if (x % (ZFCP_FC_GPN_FT_ENT_PAGE + 1))
+			acc++;
+		else
+			acc = sg_virt(++resp_entry);
+
+		last = acc->fp_flags & FC_NS_FID_LAST;
+	}
+	len = min(len, (u16)(x * sizeof(struct fc_gpn_ft_resp)));
+	return len; /* cap after last entry */
 }
 
 /**
@@ -370,9 +492,10 @@ void zfcp_dbf_san_res(char *tag, struct zfcp_fsf_req *fsf)
 	struct zfcp_fsf_ct_els *ct_els = fsf->data;
 	u16 length;
 
-	length = (u16)(ct_els->resp->length + FC_CT_HDR_LEN);
-	zfcp_dbf_san(tag, dbf, sg_virt(ct_els->resp), ZFCP_DBF_SAN_RES, length,
-		     fsf->req_id, 0);
+	length = (u16)zfcp_qdio_real_bytes(ct_els->resp);
+	zfcp_dbf_san(tag, dbf, "san_res", ct_els->resp, ZFCP_DBF_SAN_RES,
+		     length, fsf->req_id, ct_els->d_id,
+		     zfcp_dbf_san_res_cap_len_if_gpn_ft(tag, fsf, length));
 }
 
 /**
@@ -386,11 +509,13 @@ void zfcp_dbf_san_in_els(char *tag, struct zfcp_fsf_req *fsf)
 	struct fsf_status_read_buffer *srb =
 		(struct fsf_status_read_buffer *) fsf->data;
 	u16 length;
+	struct scatterlist sg;
 
 	length = (u16)(srb->length -
 			offsetof(struct fsf_status_read_buffer, payload));
-	zfcp_dbf_san(tag, dbf, srb->payload.data, ZFCP_DBF_SAN_ELS, length,
-		     fsf->req_id, ntoh24(srb->d_id));
+	sg_init_one(&sg, srb->payload.data, length);
+	zfcp_dbf_san(tag, dbf, "san_els", &sg, ZFCP_DBF_SAN_ELS, length,
+		     fsf->req_id, ntoh24(srb->d_id), length);
 }
 
 /**
@@ -399,7 +524,8 @@ void zfcp_dbf_san_in_els(char *tag, struct zfcp_fsf_req *fsf)
  * @sc: pointer to struct scsi_cmnd
  * @fsf: pointer to struct zfcp_fsf_req
  */
-void zfcp_dbf_scsi(char *tag, struct scsi_cmnd *sc, struct zfcp_fsf_req *fsf)
+void zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *sc,
+		   struct zfcp_fsf_req *fsf)
 {
 	struct zfcp_adapter *adapter =
 		(struct zfcp_adapter *) sc->device->host->hostdata[0];
@@ -442,7 +568,7 @@ void zfcp_dbf_scsi(char *tag, struct scsi_cmnd *sc, struct zfcp_fsf_req *fsf)
 		}
 	}
 
-	debug_event(dbf->scsi, 1, rec, sizeof(*rec));
+	debug_event(dbf->scsi, level, rec, sizeof(*rec));
 	spin_unlock_irqrestore(&dbf->scsi_lock, flags);
 }
 
diff --git a/drivers/s390/scsi/zfcp_dbf.h b/drivers/s390/scsi/zfcp_dbf.h
index 0be3d48681ae..36d07584271d 100644
--- a/drivers/s390/scsi/zfcp_dbf.h
+++ b/drivers/s390/scsi/zfcp_dbf.h
@@ -2,7 +2,7 @@
  * zfcp device driver
  * debug feature declarations
  *
- * Copyright IBM Corp. 2008, 2010
+ * Copyright IBM Corp. 2008, 2015
  */
 
 #ifndef ZFCP_DBF_H
@@ -17,6 +17,11 @@
 
 #define ZFCP_DBF_INVALID_LUN	0xFFFFFFFFFFFFFFFFull
 
+enum zfcp_dbf_pseudo_erp_act_type {
+	ZFCP_PSEUDO_ERP_ACTION_RPORT_ADD = 0xff,
+	ZFCP_PSEUDO_ERP_ACTION_RPORT_DEL = 0xfe,
+};
+
 /**
  * struct zfcp_dbf_rec_trigger - trace record for triggered recovery action
  * @ready: number of ready recovery actions
@@ -110,6 +115,7 @@ struct zfcp_dbf_san {
 	u32 d_id;
 #define ZFCP_DBF_SAN_MAX_PAYLOAD (FC_CT_HDR_LEN + 32)
 	char payload[ZFCP_DBF_SAN_MAX_PAYLOAD];
+	u16 pl_len;
 } __packed;
 
 /**
@@ -126,6 +132,8 @@ struct zfcp_dbf_hba_res {
 	u8  prot_status_qual[FSF_PROT_STATUS_QUAL_SIZE];
 	u32 fsf_status;
 	u8  fsf_status_qual[FSF_STATUS_QUALIFIER_SIZE];
+	u32 port_handle;
+	u32 lun_handle;
 } __packed;
 
 /**
@@ -279,7 +287,7 @@ static inline
 void zfcp_dbf_hba_fsf_resp(char *tag, int level, struct zfcp_fsf_req *req)
 {
 	if (debug_level_enabled(req->adapter->dbf->hba, level))
-		zfcp_dbf_hba_fsf_res(tag, req);
+		zfcp_dbf_hba_fsf_res(tag, level, req);
 }
 
 /**
@@ -318,7 +326,7 @@ void _zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *scmd,
 					scmd->device->host->hostdata[0];
 
 	if (debug_level_enabled(adapter->dbf->scsi, level))
-		zfcp_dbf_scsi(tag, scmd, req);
+		zfcp_dbf_scsi(tag, level, scmd, req);
 }
 
 /**
diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c
index 3fb410977014..a59d678125bd 100644
--- a/drivers/s390/scsi/zfcp_erp.c
+++ b/drivers/s390/scsi/zfcp_erp.c
@@ -3,7 +3,7 @@
  *
  * Error Recovery Procedures (ERP).
  *
- * Copyright IBM Corp. 2002, 2010
+ * Copyright IBM Corp. 2002, 2015
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -1217,8 +1217,14 @@ static void zfcp_erp_action_cleanup(struct zfcp_erp_action *act, int result)
 		break;
 
 	case ZFCP_ERP_ACTION_REOPEN_PORT:
-		if (result == ZFCP_ERP_SUCCEEDED)
-			zfcp_scsi_schedule_rport_register(port);
+		/* This switch case might also happen after a forced reopen
+		 * was successfully done and thus overwritten with a new
+		 * non-forced reopen at `ersfs_2'. In this case, we must not
+		 * do the clean-up of the non-forced version.
+		 */
+		if (act->step != ZFCP_ERP_STEP_UNINITIALIZED)
+			if (result == ZFCP_ERP_SUCCEEDED)
+				zfcp_scsi_schedule_rport_register(port);
 		/* fall through */
 	case ZFCP_ERP_ACTION_REOPEN_PORT_FORCED:
 		put_device(&port->dev);
diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h
index 5b500652572b..c8fed9fa1cca 100644
--- a/drivers/s390/scsi/zfcp_ext.h
+++ b/drivers/s390/scsi/zfcp_ext.h
@@ -3,7 +3,7 @@
  *
  * External function declarations.
  *
- * Copyright IBM Corp. 2002, 2010
+ * Copyright IBM Corp. 2002, 2015
  */
 
 #ifndef ZFCP_EXT_H
@@ -35,8 +35,9 @@ extern void zfcp_dbf_adapter_unregister(struct zfcp_adapter *);
 extern void zfcp_dbf_rec_trig(char *, struct zfcp_adapter *,
 			      struct zfcp_port *, struct scsi_device *, u8, u8);
 extern void zfcp_dbf_rec_run(char *, struct zfcp_erp_action *);
+extern void zfcp_dbf_rec_run_wka(char *, struct zfcp_fc_wka_port *, u64);
 extern void zfcp_dbf_hba_fsf_uss(char *, struct zfcp_fsf_req *);
-extern void zfcp_dbf_hba_fsf_res(char *, struct zfcp_fsf_req *);
+extern void zfcp_dbf_hba_fsf_res(char *, int, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_bit_err(char *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_berr(struct zfcp_dbf *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_def_err(struct zfcp_adapter *, u64, u16, void **);
@@ -44,7 +45,8 @@ extern void zfcp_dbf_hba_basic(char *, struct zfcp_adapter *);
 extern void zfcp_dbf_san_req(char *, struct zfcp_fsf_req *, u32);
 extern void zfcp_dbf_san_res(char *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_san_in_els(char *, struct zfcp_fsf_req *);
-extern void zfcp_dbf_scsi(char *, struct scsi_cmnd *, struct zfcp_fsf_req *);
+extern void zfcp_dbf_scsi(char *, int, struct scsi_cmnd *,
+			  struct zfcp_fsf_req *);
 
 /* zfcp_erp.c */
 extern void zfcp_erp_set_adapter_status(struct zfcp_adapter *, u32);
diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c
index 522a633c866a..75f820ca17b7 100644
--- a/drivers/s390/scsi/zfcp_fsf.c
+++ b/drivers/s390/scsi/zfcp_fsf.c
@@ -3,7 +3,7 @@
  *
  * Implementation of FSF commands.
  *
- * Copyright IBM Corp. 2002, 2013
+ * Copyright IBM Corp. 2002, 2015
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -508,7 +508,10 @@ static int zfcp_fsf_exchange_config_evaluate(struct zfcp_fsf_req *req)
 		fc_host_port_type(shost) = FC_PORTTYPE_PTP;
 		break;
 	case FSF_TOPO_FABRIC:
-		fc_host_port_type(shost) = FC_PORTTYPE_NPORT;
+		if (bottom->connection_features & FSF_FEATURE_NPIV_MODE)
+			fc_host_port_type(shost) = FC_PORTTYPE_NPIV;
+		else
+			fc_host_port_type(shost) = FC_PORTTYPE_NPORT;
 		break;
 	case FSF_TOPO_AL:
 		fc_host_port_type(shost) = FC_PORTTYPE_NLPORT;
@@ -613,7 +616,6 @@ static void zfcp_fsf_exchange_port_evaluate(struct zfcp_fsf_req *req)
 
 	if (adapter->connection_features & FSF_FEATURE_NPIV_MODE) {
 		fc_host_permanent_port_name(shost) = bottom->wwpn;
-		fc_host_port_type(shost) = FC_PORTTYPE_NPIV;
 	} else
 		fc_host_permanent_port_name(shost) = fc_host_port_name(shost);
 	fc_host_maxframe_size(shost) = bottom->maximum_frame_size;
@@ -982,8 +984,12 @@ static int zfcp_fsf_setup_ct_els_sbals(struct zfcp_fsf_req *req,
 	if (zfcp_adapter_multi_buffer_active(adapter)) {
 		if (zfcp_qdio_sbals_from_sg(qdio, &req->qdio_req, sg_req))
 			return -EIO;
+		qtcb->bottom.support.req_buf_length =
+			zfcp_qdio_real_bytes(sg_req);
 		if (zfcp_qdio_sbals_from_sg(qdio, &req->qdio_req, sg_resp))
 			return -EIO;
+		qtcb->bottom.support.resp_buf_length =
+			zfcp_qdio_real_bytes(sg_resp);
 
 		zfcp_qdio_set_data_div(qdio, &req->qdio_req,
 					zfcp_qdio_sbale_count(sg_req));
@@ -1073,6 +1079,7 @@ int zfcp_fsf_send_ct(struct zfcp_fc_wka_port *wka_port,
 
 	req->handler = zfcp_fsf_send_ct_handler;
 	req->qtcb->header.port_handle = wka_port->handle;
+	ct->d_id = wka_port->d_id;
 	req->data = ct;
 
 	zfcp_dbf_san_req("fssct_1", req, wka_port->d_id);
@@ -1169,6 +1176,7 @@ int zfcp_fsf_send_els(struct zfcp_adapter *adapter, u32 d_id,
 
 	hton24(req->qtcb->bottom.support.d_id, d_id);
 	req->handler = zfcp_fsf_send_els_handler;
+	els->d_id = d_id;
 	req->data = els;
 
 	zfcp_dbf_san_req("fssels1", req, d_id);
@@ -1575,7 +1583,7 @@ out:
 int zfcp_fsf_open_wka_port(struct zfcp_fc_wka_port *wka_port)
 {
 	struct zfcp_qdio *qdio = wka_port->adapter->qdio;
-	struct zfcp_fsf_req *req;
+	struct zfcp_fsf_req *req = NULL;
 	int retval = -EIO;
 
 	spin_lock_irq(&qdio->req_q_lock);
@@ -1604,6 +1612,8 @@ int zfcp_fsf_open_wka_port(struct zfcp_fc_wka_port *wka_port)
 		zfcp_fsf_req_free(req);
 out:
 	spin_unlock_irq(&qdio->req_q_lock);
+	if (req && !IS_ERR(req))
+		zfcp_dbf_rec_run_wka("fsowp_1", wka_port, req->req_id);
 	return retval;
 }
 
@@ -1628,7 +1638,7 @@ static void zfcp_fsf_close_wka_port_handler(struct zfcp_fsf_req *req)
 int zfcp_fsf_close_wka_port(struct zfcp_fc_wka_port *wka_port)
 {
 	struct zfcp_qdio *qdio = wka_port->adapter->qdio;
-	struct zfcp_fsf_req *req;
+	struct zfcp_fsf_req *req = NULL;
 	int retval = -EIO;
 
 	spin_lock_irq(&qdio->req_q_lock);
@@ -1657,6 +1667,8 @@ int zfcp_fsf_close_wka_port(struct zfcp_fc_wka_port *wka_port)
 		zfcp_fsf_req_free(req);
 out:
 	spin_unlock_irq(&qdio->req_q_lock);
+	if (req && !IS_ERR(req))
+		zfcp_dbf_rec_run_wka("fscwp_1", wka_port, req->req_id);
 	return retval;
 }
 
diff --git a/drivers/s390/scsi/zfcp_fsf.h b/drivers/s390/scsi/zfcp_fsf.h
index 57ae3ae1046d..be1c04b334c5 100644
--- a/drivers/s390/scsi/zfcp_fsf.h
+++ b/drivers/s390/scsi/zfcp_fsf.h
@@ -3,7 +3,7 @@
  *
  * Interface to the FSF support functions.
  *
- * Copyright IBM Corp. 2002, 2010
+ * Copyright IBM Corp. 2002, 2015
  */
 
 #ifndef FSF_H
@@ -436,6 +436,7 @@ struct zfcp_blk_drv_data {
  * @handler_data: data passed to handler function
  * @port: Optional pointer to port for zfcp internal ELS (only test link ADISC)
  * @status: used to pass error status to calling function
+ * @d_id: Destination ID of either open WKA port for CT or of D_ID for ELS
  */
 struct zfcp_fsf_ct_els {
 	struct scatterlist *req;
@@ -444,6 +445,7 @@ struct zfcp_fsf_ct_els {
 	void *handler_data;
 	struct zfcp_port *port;
 	int status;
+	u32 d_id;
 };
 
 #endif				/* FSF_H */
diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c
index b3c6ff49103b..9069f98a1817 100644
--- a/drivers/s390/scsi/zfcp_scsi.c
+++ b/drivers/s390/scsi/zfcp_scsi.c
@@ -3,7 +3,7 @@
  *
  * Interface to Linux SCSI midlayer.
  *
- * Copyright IBM Corp. 2002, 2013
+ * Copyright IBM Corp. 2002, 2015
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -556,6 +556,9 @@ static void zfcp_scsi_rport_register(struct zfcp_port *port)
 	ids.port_id = port->d_id;
 	ids.roles = FC_RPORT_ROLE_FCP_TARGET;
 
+	zfcp_dbf_rec_trig("scpaddy", port->adapter, port, NULL,
+			  ZFCP_PSEUDO_ERP_ACTION_RPORT_ADD,
+			  ZFCP_PSEUDO_ERP_ACTION_RPORT_ADD);
 	rport = fc_remote_port_add(port->adapter->scsi_host, 0, &ids);
 	if (!rport) {
 		dev_err(&port->adapter->ccw_device->dev,
@@ -577,6 +580,9 @@ static void zfcp_scsi_rport_block(struct zfcp_port *port)
 	struct fc_rport *rport = port->rport;
 
 	if (rport) {
+		zfcp_dbf_rec_trig("scpdely", port->adapter, port, NULL,
+				  ZFCP_PSEUDO_ERP_ACTION_RPORT_DEL,
+				  ZFCP_PSEUDO_ERP_ACTION_RPORT_DEL);
 		fc_remote_port_delete(rport);
 		port->rport = NULL;
 	}
diff --git a/drivers/s390/virtio/Makefile b/drivers/s390/virtio/Makefile
index 241891a57caf..df40692a9011 100644
--- a/drivers/s390/virtio/Makefile
+++ b/drivers/s390/virtio/Makefile
@@ -6,4 +6,8 @@
 # it under the terms of the GNU General Public License (version 2 only)
 # as published by the Free Software Foundation.
 
-obj-$(CONFIG_S390_GUEST) += kvm_virtio.o virtio_ccw.o
+s390-virtio-objs := virtio_ccw.o
+ifdef CONFIG_S390_GUEST_OLD_TRANSPORT
+s390-virtio-objs += kvm_virtio.o
+endif
+obj-$(CONFIG_S390_GUEST) += $(s390-virtio-objs)
diff --git a/drivers/s390/virtio/kvm_virtio.c b/drivers/s390/virtio/kvm_virtio.c
index 1d060fd293a3..5e5c11f37b24 100644
--- a/drivers/s390/virtio/kvm_virtio.c
+++ b/drivers/s390/virtio/kvm_virtio.c
@@ -458,6 +458,8 @@ static int __init kvm_devices_init(void)
 	if (test_devices_support(total_memory_size) < 0)
 		return -ENODEV;
 
+	pr_warn("The s390-virtio transport is deprecated. Please switch to a modern host providing virtio-ccw.\n");
+
 	rc = vmem_add_mapping(total_memory_size, PAGE_SIZE);
 	if (rc)
 		return rc;
@@ -482,7 +484,7 @@ static int __init kvm_devices_init(void)
 }
 
 /* code for early console output with virtio_console */
-static __init int early_put_chars(u32 vtermno, const char *buf, int count)
+static int early_put_chars(u32 vtermno, const char *buf, int count)
 {
 	char scratch[17];
 	unsigned int len = count;
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index 7d1b4317eccc..3e2bdb90813c 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -396,18 +396,6 @@ config SCSI_3W_SAS
 	  Please read the comments at the top of
 	  <file:drivers/scsi/3w-sas.c>.
 
-config SCSI_7000FASST
-	tristate "7000FASST SCSI support"
-	depends on ISA && SCSI && ISA_DMA_API
-	select CHECK_SIGNATURE
-	help
-	  This driver supports the Western Digital 7000 SCSI host adapter
-	  family.  Some information is in the source:
-	  <file:drivers/scsi/wd7000.c>.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called wd7000.
-
 config SCSI_ACARD
 	tristate "ACARD SCSI support"
 	depends on PCI && SCSI
@@ -512,18 +500,6 @@ config SCSI_ADVANSYS
 	  To compile this driver as a module, choose M here: the
 	  module will be called advansys.
 
-config SCSI_IN2000
-	tristate "Always IN2000 SCSI support"
-	depends on ISA && SCSI
-	help
-	  This is support for an ISA bus SCSI host adapter.  You'll find more
-	  information in <file:Documentation/scsi/in2000.txt>. If it doesn't work
-	  out of the box, you may have to change the jumpers for IRQ or
-	  address selection.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called in2000.
-
 config SCSI_ARCMSR
 	tristate "ARECA (ARC11xx/12xx/13xx/16xx) SATA/SAS RAID Host Adapter"
 	depends on PCI && SCSI
@@ -540,6 +516,7 @@ config SCSI_ARCMSR
 source "drivers/scsi/esas2r/Kconfig"
 source "drivers/scsi/megaraid/Kconfig.megaraid"
 source "drivers/scsi/mpt3sas/Kconfig"
+source "drivers/scsi/smartpqi/Kconfig"
 source "drivers/scsi/ufs/Kconfig"
 
 config SCSI_HPTIOP
@@ -660,20 +637,6 @@ config SCSI_DMX3191D
 	  To compile this driver as a module, choose M here: the
 	  module will be called dmx3191d.
 
-config SCSI_DTC3280
-	tristate "DTC3180/3280 SCSI support"
-	depends on ISA && SCSI
-	select SCSI_SPI_ATTRS
-	select CHECK_SIGNATURE
-	help
-	  This is support for DTC 3180/3280 SCSI Host Adapters.  Please read
-	  the SCSI-HOWTO, available from
-	  <http://www.tldp.org/docs.html#howto>, and the file
-	  <file:Documentation/scsi/dtc3x80.txt>.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called dtc.
-
 config SCSI_EATA
 	tristate "EATA ISA/EISA/PCI (DPT and generic EATA/DMA-compliant boards) support"
 	depends on (ISA || EISA || PCI) && SCSI && ISA_DMA_API
@@ -1248,20 +1211,6 @@ config SCSI_NCR53C8XX_NO_DISCONNECT
 	  not allow targets to disconnect is not reasonable if there is more
 	  than 1 device on a SCSI bus. The normal answer therefore is N.
 
-config SCSI_PAS16
-	tristate "PAS16 SCSI support"
-	depends on ISA && SCSI
-	select SCSI_SPI_ATTRS
-	---help---
-	  This is support for a SCSI host adapter.  It is explained in section
-	  3.10 of the SCSI-HOWTO, available from
-	  <http://www.tldp.org/docs.html#howto>.  If it doesn't work out
-	  of the box, you may have to change some settings in
-	  <file:drivers/scsi/pas16.h>.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called pas16.
-
 config SCSI_QLOGIC_FAS
 	tristate "Qlogic FAS SCSI support"
 	depends on ISA && SCSI
@@ -1382,89 +1331,6 @@ config SCSI_AM53C974
 	  To compile this driver as a module, choose M here: the
 	  module will be called am53c974.
 
-config SCSI_T128
-	tristate "Trantor T128/T128F/T228 SCSI support"
-	depends on ISA && SCSI
-	select SCSI_SPI_ATTRS
-	select CHECK_SIGNATURE
-	---help---
-	  This is support for a SCSI host adapter. It is explained in section
-	  3.11 of the SCSI-HOWTO, available from
-	  <http://www.tldp.org/docs.html#howto>.  If it doesn't work out
-	  of the box, you may have to change some settings in
-	  <file:drivers/scsi/t128.h>.  Note that Trantor was purchased by
-	  Adaptec, and some former Trantor products are being sold under the
-	  Adaptec name.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called t128.
-
-config SCSI_U14_34F
-	tristate "UltraStor 14F/34F support"
-	depends on ISA && SCSI && ISA_DMA_API
-	---help---
-	  This is support for the UltraStor 14F and 34F SCSI-2 host adapters.
-	  The source at <file:drivers/scsi/u14-34f.c> contains some
-	  information about this hardware.  If the driver doesn't work out of
-	  the box, you may have to change some settings in
-	  <file: drivers/scsi/u14-34f.c>.  Read the SCSI-HOWTO, available from
-	  <http://www.tldp.org/docs.html#howto>.  Note that there is also
-	  another driver for the same hardware: "UltraStor SCSI support",
-	  below.  You should say Y to both only if you want 24F support as
-	  well.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called u14-34f.
-
-config SCSI_U14_34F_TAGGED_QUEUE
-	bool "enable tagged command queueing"
-	depends on SCSI_U14_34F
-	help
-	  This is a feature of SCSI-2 which improves performance: the host
-	  adapter can send several SCSI commands to a device's queue even if
-	  previous commands haven't finished yet.
-	  This is equivalent to the "u14-34f=tc:y" boot option.
-
-config SCSI_U14_34F_LINKED_COMMANDS
-	bool "enable elevator sorting"
-	depends on SCSI_U14_34F
-	help
-	  This option enables elevator sorting for all probed SCSI disks and
-	  CD-ROMs. It definitely reduces the average seek distance when doing
-	  random seeks, but this does not necessarily result in a noticeable
-	  performance improvement: your mileage may vary...
-	  This is equivalent to the "u14-34f=lc:y" boot option.
-
-config SCSI_U14_34F_MAX_TAGS
-	int "maximum number of queued commands"
-	depends on SCSI_U14_34F
-	default "8"
-	help
-	  This specifies how many SCSI commands can be maximally queued for
-	  each probed SCSI device. You should reduce the default value of 8
-	  only if you have disks with buggy or limited tagged command support.
-	  Minimum is 2 and maximum is 14. This value is also the window size
-	  used by the elevator sorting option above. The effective value used
-	  by the driver for each probed SCSI device is reported at boot time.
-	  This is equivalent to the "u14-34f=mq:8" boot option.
-
-config SCSI_ULTRASTOR
-	tristate "UltraStor SCSI support"
-	depends on X86 && ISA && SCSI && ISA_DMA_API
-	---help---
-	  This is support for the UltraStor 14F, 24F and 34F SCSI-2 host
-	  adapter family.  This driver is explained in section 3.12 of the
-	  SCSI-HOWTO, available from
-	  <http://www.tldp.org/docs.html#howto>.  If it doesn't work out
-	  of the box, you may have to change some settings in
-	  <file:drivers/scsi/ultrastor.h>.
-
-	  Note that there is also another driver for the same hardware:
-	  "UltraStor 14F/34F support", above.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called ultrastor.
-
 config SCSI_NSP32
 	tristate "Workbit NinjaSCSI-32Bi/UDE support"
 	depends on PCI && SCSI && !64BIT
diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile
index d5397987e731..38d938d7fe67 100644
--- a/drivers/scsi/Makefile
+++ b/drivers/scsi/Makefile
@@ -61,9 +61,7 @@ obj-$(CONFIG_SCSI_SIM710)	+= 53c700.o	sim710.o
 obj-$(CONFIG_SCSI_ADVANSYS)	+= advansys.o
 obj-$(CONFIG_SCSI_BUSLOGIC)	+= BusLogic.o
 obj-$(CONFIG_SCSI_DPT_I2O)	+= dpt_i2o.o
-obj-$(CONFIG_SCSI_U14_34F)	+= u14-34f.o
 obj-$(CONFIG_SCSI_ARCMSR)	+= arcmsr/
-obj-$(CONFIG_SCSI_ULTRASTOR)	+= ultrastor.o
 obj-$(CONFIG_SCSI_AHA152X)	+= aha152x.o
 obj-$(CONFIG_SCSI_AHA1542)	+= aha1542.o
 obj-$(CONFIG_SCSI_AHA1740)	+= aha1740.o
@@ -75,7 +73,6 @@ obj-$(CONFIG_SCSI_PM8001)	+= pm8001/
 obj-$(CONFIG_SCSI_ISCI)		+= isci/
 obj-$(CONFIG_SCSI_IPS)		+= ips.o
 obj-$(CONFIG_SCSI_FUTURE_DOMAIN)+= fdomain.o
-obj-$(CONFIG_SCSI_IN2000)	+= in2000.o
 obj-$(CONFIG_SCSI_GENERIC_NCR5380) += g_NCR5380.o
 obj-$(CONFIG_SCSI_GENERIC_NCR5380_MMIO) += g_NCR5380_mmio.o
 obj-$(CONFIG_SCSI_NCR53C406A)	+= NCR53c406a.o
@@ -90,15 +87,12 @@ obj-$(CONFIG_SCSI_QLA_ISCSI)	+= libiscsi.o qla4xxx/
 obj-$(CONFIG_SCSI_LPFC)		+= lpfc/
 obj-$(CONFIG_SCSI_BFA_FC)	+= bfa/
 obj-$(CONFIG_SCSI_CHELSIO_FCOE)	+= csiostor/
-obj-$(CONFIG_SCSI_PAS16)	+= pas16.o
-obj-$(CONFIG_SCSI_T128)		+= t128.o
 obj-$(CONFIG_SCSI_DMX3191D)	+= dmx3191d.o
 obj-$(CONFIG_SCSI_HPSA)		+= hpsa.o
-obj-$(CONFIG_SCSI_DTC3280)	+= dtc.o
+obj-$(CONFIG_SCSI_SMARTPQI)	+= smartpqi/
 obj-$(CONFIG_SCSI_SYM53C8XX_2)	+= sym53c8xx_2/
 obj-$(CONFIG_SCSI_ZALON)	+= zalon7xx.o
 obj-$(CONFIG_SCSI_EATA_PIO)	+= eata_pio.o
-obj-$(CONFIG_SCSI_7000FASST)	+= wd7000.o
 obj-$(CONFIG_SCSI_EATA)		+= eata.o
 obj-$(CONFIG_SCSI_DC395x)	+= dc395x.o
 obj-$(CONFIG_SCSI_AM53C974)	+= esp_scsi.o	am53c974.o
diff --git a/drivers/scsi/NCR5380.c b/drivers/scsi/NCR5380.c
index 43908bbb3b23..db2739079cbb 100644
--- a/drivers/scsi/NCR5380.c
+++ b/drivers/scsi/NCR5380.c
@@ -230,13 +230,6 @@ static int NCR5380_poll_politely2(struct Scsi_Host *instance,
 	return -ETIMEDOUT;
 }
 
-static inline int NCR5380_poll_politely(struct Scsi_Host *instance,
-                                        int reg, int bit, int val, int wait)
-{
-	return NCR5380_poll_politely2(instance, reg, bit, val,
-	                                        reg, bit, val, wait);
-}
-
 #if NDEBUG
 static struct {
 	unsigned char mask;
@@ -1854,11 +1847,11 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 						/* XXX - need to source or sink data here, as appropriate */
 					}
 				} else {
-					/* Break up transfer into 3 ms chunks,
-					 * presuming 6 accesses per handshake.
+					/* Transfer a small chunk so that the
+					 * irq mode lock is not held too long.
 					 */
-					transfersize = min((unsigned long)cmd->SCp.this_residual,
-					                   hostdata->accesses_per_ms / 2);
+					transfersize = min(cmd->SCp.this_residual,
+							   NCR5380_PIO_CHUNK_SIZE);
 					len = transfersize;
 					NCR5380_transfer_pio(instance, &phase, &len,
 					                     (unsigned char **)&cmd->SCp.ptr);
diff --git a/drivers/scsi/NCR5380.h b/drivers/scsi/NCR5380.h
index c60728785d89..965d92339455 100644
--- a/drivers/scsi/NCR5380.h
+++ b/drivers/scsi/NCR5380.h
@@ -250,6 +250,8 @@ struct NCR5380_cmd {
 
 #define NCR5380_CMD_SIZE		(sizeof(struct NCR5380_cmd))
 
+#define NCR5380_PIO_CHUNK_SIZE		256
+
 static inline struct scsi_cmnd *NCR5380_to_scmd(struct NCR5380_cmd *ncmd_ptr)
 {
 	return ((struct scsi_cmnd *)ncmd_ptr) - 1;
@@ -292,8 +294,14 @@ static void NCR5380_reselect(struct Scsi_Host *instance);
 static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *, struct scsi_cmnd *);
 static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data);
 static int NCR5380_transfer_pio(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data);
-static int NCR5380_poll_politely(struct Scsi_Host *, int, int, int, int);
 static int NCR5380_poll_politely2(struct Scsi_Host *, int, int, int, int, int, int, int);
 
+static inline int NCR5380_poll_politely(struct Scsi_Host *instance,
+					int reg, int bit, int val, int wait)
+{
+	return NCR5380_poll_politely2(instance, reg, bit, val,
+						reg, bit, val, wait);
+}
+
 #endif				/* __KERNEL__ */
 #endif				/* NCR5380_H */
diff --git a/drivers/scsi/aacraid/commctrl.c b/drivers/scsi/aacraid/commctrl.c
index b381b3718a98..5648b715fed9 100644
--- a/drivers/scsi/aacraid/commctrl.c
+++ b/drivers/scsi/aacraid/commctrl.c
@@ -63,7 +63,7 @@ static int ioctl_send_fib(struct aac_dev * dev, void __user *arg)
 	struct fib *fibptr;
 	struct hw_fib * hw_fib = (struct hw_fib *)0;
 	dma_addr_t hw_fib_pa = (dma_addr_t)0LL;
-	unsigned size;
+	unsigned int size, osize;
 	int retval;
 
 	if (dev->in_reset) {
@@ -87,7 +87,8 @@ static int ioctl_send_fib(struct aac_dev * dev, void __user *arg)
 	 *	will not overrun the buffer when we copy the memory. Return
 	 *	an error if we would.
 	 */
-	size = le16_to_cpu(kfib->header.Size) + sizeof(struct aac_fibhdr);
+	osize = size = le16_to_cpu(kfib->header.Size) +
+		sizeof(struct aac_fibhdr);
 	if (size < le16_to_cpu(kfib->header.SenderSize))
 		size = le16_to_cpu(kfib->header.SenderSize);
 	if (size > dev->max_fib_size) {
@@ -118,6 +119,14 @@ static int ioctl_send_fib(struct aac_dev * dev, void __user *arg)
 		goto cleanup;
 	}
 
+	/* Sanity check the second copy */
+	if ((osize != le16_to_cpu(kfib->header.Size) +
+		sizeof(struct aac_fibhdr))
+		|| (size < le16_to_cpu(kfib->header.SenderSize))) {
+		retval = -EINVAL;
+		goto cleanup;
+	}
+
 	if (kfib->header.Command == cpu_to_le16(TakeABreakPt)) {
 		aac_adapter_interrupt(dev);
 		/*
diff --git a/drivers/scsi/aacraid/src.c b/drivers/scsi/aacraid/src.c
index 28f8b8a1b8a4..0c453880f214 100644
--- a/drivers/scsi/aacraid/src.c
+++ b/drivers/scsi/aacraid/src.c
@@ -613,7 +613,7 @@ static int aac_src_restart_adapter(struct aac_dev *dev, int bled)
  *	@dev: Adapter
  *	@comm: communications method
  */
-int aac_src_select_comm(struct aac_dev *dev, int comm)
+static int aac_src_select_comm(struct aac_dev *dev, int comm)
 {
 	switch (comm) {
 	case AAC_COMM_MESSAGE:
diff --git a/drivers/scsi/aic94xx/aic94xx_hwi.c b/drivers/scsi/aic94xx/aic94xx_hwi.c
index 0fdc98bc2338..7c713f797535 100644
--- a/drivers/scsi/aic94xx/aic94xx_hwi.c
+++ b/drivers/scsi/aic94xx/aic94xx_hwi.c
@@ -632,7 +632,7 @@ int asd_init_hw(struct asd_ha_struct *asd_ha)
 			   pci_name(asd_ha->pcidev));
 		return err;
 	}
-	pci_write_config_dword(asd_ha->pcidev, PCIC_HSTPCIX_CNTRL,
+	err = pci_write_config_dword(asd_ha->pcidev, PCIC_HSTPCIX_CNTRL,
 					v | SC_TMR_DIS);
 	if (err) {
 		asd_printk("couldn't disable split completion timer of %s\n",
diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c
index 7640498964a5..3d53d636b17b 100644
--- a/drivers/scsi/arcmsr/arcmsr_hba.c
+++ b/drivers/scsi/arcmsr/arcmsr_hba.c
@@ -2388,15 +2388,23 @@ static int arcmsr_iop_message_xfer(struct AdapterControlBlock *acb,
 	}
 	case ARCMSR_MESSAGE_WRITE_WQBUFFER: {
 		unsigned char *ver_addr;
-		int32_t user_len, cnt2end;
+		uint32_t user_len;
+		int32_t cnt2end;
 		uint8_t *pQbuffer, *ptmpuserbuffer;
+
+		user_len = pcmdmessagefld->cmdmessage.Length;
+		if (user_len > ARCMSR_API_DATA_BUFLEN) {
+			retvalue = ARCMSR_MESSAGE_FAIL;
+			goto message_out;
+		}
+
 		ver_addr = kmalloc(ARCMSR_API_DATA_BUFLEN, GFP_ATOMIC);
 		if (!ver_addr) {
 			retvalue = ARCMSR_MESSAGE_FAIL;
 			goto message_out;
 		}
 		ptmpuserbuffer = ver_addr;
-		user_len = pcmdmessagefld->cmdmessage.Length;
+
 		memcpy(ptmpuserbuffer,
 			pcmdmessagefld->messagedatabuffer, user_len);
 		spin_lock_irqsave(&acb->wqbuffer_lock, flags);
diff --git a/drivers/scsi/be2iscsi/be.h b/drivers/scsi/be2iscsi/be.h
index ee5ace873535..b1d0fdc5d5e1 100644
--- a/drivers/scsi/be2iscsi/be.h
+++ b/drivers/scsi/be2iscsi/be.h
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -8,7 +8,7 @@
  * Public License is included in this distribution in the file called COPYING.
  *
  * Contact Information:
- * linux-drivers@avagotech.com
+ * linux-drivers@broadcom.com
  *
  * Emulex
  * 3333 Susan Street
@@ -89,7 +89,7 @@ struct be_aic_obj {		/* Adaptive interrupt coalescing (AIC) info */
 	u32 max_eqd;		/* in usecs */
 	u32 prev_eqd;		/* in usecs */
 	u32 et_eqd;		/* configured val when aic is off */
-	ulong jiffs;
+	ulong jiffies;
 	u64 eq_prev;		/* Used to calculate eqe */
 };
 
@@ -100,7 +100,7 @@ struct be_eq_obj {
 	struct be_queue_info q;
 	struct beiscsi_hba *phba;
 	struct be_queue_info *cq;
-	struct work_struct work_cqs; /* Work Item */
+	struct work_struct mcc_work; /* Work Item */
 	struct irq_poll	iopoll;
 };
 
@@ -111,8 +111,11 @@ struct be_mcc_obj {
 
 struct beiscsi_mcc_tag_state {
 	unsigned long tag_state;
-#define MCC_TAG_STATE_RUNNING	1
-#define MCC_TAG_STATE_TIMEOUT	2
+#define MCC_TAG_STATE_RUNNING	0
+#define MCC_TAG_STATE_TIMEOUT	1
+#define MCC_TAG_STATE_ASYNC	2
+#define MCC_TAG_STATE_IGNORE	3
+	void (*cbfn)(struct beiscsi_hba *, unsigned int);
 	struct be_dma_mem tag_mem_state;
 };
 
diff --git a/drivers/scsi/be2iscsi/be_cmds.c b/drivers/scsi/be2iscsi/be_cmds.c
index a55eaeea37e7..be65da2988fb 100644
--- a/drivers/scsi/be2iscsi/be_cmds.c
+++ b/drivers/scsi/be2iscsi/be_cmds.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -8,7 +8,7 @@
  * Public License is included in this distribution in the file called COPYING.
  *
  * Contact Information:
- * linux-drivers@avagotech.com
+ * linux-drivers@broadcom.com
  *
  * Emulex
  * 3333 Susan Street
@@ -21,110 +21,77 @@
 #include "be.h"
 #include "be_mgmt.h"
 
-int beiscsi_pci_soft_reset(struct beiscsi_hba *phba)
-{
-	u32 sreset;
-	u8 *pci_reset_offset = 0;
-	u8 *pci_online0_offset = 0;
-	u8 *pci_online1_offset = 0;
-	u32 pconline0 = 0;
-	u32 pconline1 = 0;
-	u32 i;
-
-	pci_reset_offset = (u8 *)phba->pci_va + BE2_SOFT_RESET;
-	pci_online0_offset = (u8 *)phba->pci_va + BE2_PCI_ONLINE0;
-	pci_online1_offset = (u8 *)phba->pci_va + BE2_PCI_ONLINE1;
-	sreset = readl((void *)pci_reset_offset);
-	sreset |= BE2_SET_RESET;
-	writel(sreset, (void *)pci_reset_offset);
-
-	i = 0;
-	while (sreset & BE2_SET_RESET) {
-		if (i > 64)
-			break;
-		msleep(100);
-		sreset = readl((void *)pci_reset_offset);
-		i++;
-	}
-
-	if (sreset & BE2_SET_RESET) {
-		printk(KERN_ERR DRV_NAME
-		       " Soft Reset  did not deassert\n");
-		return -EIO;
-	}
-	pconline1 = BE2_MPU_IRAM_ONLINE;
-	writel(pconline0, (void *)pci_online0_offset);
-	writel(pconline1, (void *)pci_online1_offset);
-
-	sreset |= BE2_SET_RESET;
-	writel(sreset, (void *)pci_reset_offset);
-
-	i = 0;
-	while (sreset & BE2_SET_RESET) {
-		if (i > 64)
-			break;
-		msleep(1);
-		sreset = readl((void *)pci_reset_offset);
-		i++;
-	}
-	if (sreset & BE2_SET_RESET) {
-		printk(KERN_ERR DRV_NAME
-		       " MPU Online Soft Reset did not deassert\n");
-		return -EIO;
-	}
-	return 0;
-}
-
-int be_chk_reset_complete(struct beiscsi_hba *phba)
-{
-	unsigned int num_loop;
-	u8 *mpu_sem = 0;
-	u32 status;
-
-	num_loop = 1000;
-	mpu_sem = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
-	msleep(5000);
-
-	while (num_loop) {
-		status = readl((void *)mpu_sem);
-
-		if ((status & 0x80000000) || (status & 0x0000FFFF) == 0xC000)
-			break;
-		msleep(60);
-		num_loop--;
-	}
-
-	if ((status & 0x80000000) || (!num_loop)) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BC_%d : Failed in be_chk_reset_complete"
-			    "status = 0x%x\n", status);
-		return -EIO;
-	}
-
-	return 0;
-}
-
-unsigned int alloc_mcc_tag(struct beiscsi_hba *phba)
-{
-	unsigned int tag = 0;
+/* UE Status Low CSR */
+static const char * const desc_ue_status_low[] = {
+	"CEV",
+	"CTX",
+	"DBUF",
+	"ERX",
+	"Host",
+	"MPU",
+	"NDMA",
+	"PTC ",
+	"RDMA ",
+	"RXF ",
+	"RXIPS ",
+	"RXULP0 ",
+	"RXULP1 ",
+	"RXULP2 ",
+	"TIM ",
+	"TPOST ",
+	"TPRE ",
+	"TXIPS ",
+	"TXULP0 ",
+	"TXULP1 ",
+	"UC ",
+	"WDMA ",
+	"TXULP2 ",
+	"HOST1 ",
+	"P0_OB_LINK ",
+	"P1_OB_LINK ",
+	"HOST_GPIO ",
+	"MBOX ",
+	"AXGMAC0",
+	"AXGMAC1",
+	"JTAG",
+	"MPU_INTPEND"
+};
 
-	spin_lock(&phba->ctrl.mcc_lock);
-	if (phba->ctrl.mcc_tag_available) {
-		tag = phba->ctrl.mcc_tag[phba->ctrl.mcc_alloc_index];
-		phba->ctrl.mcc_tag[phba->ctrl.mcc_alloc_index] = 0;
-		phba->ctrl.mcc_tag_status[tag] = 0;
-		phba->ctrl.ptag_state[tag].tag_state = 0;
-	}
-	if (tag) {
-		phba->ctrl.mcc_tag_available--;
-		if (phba->ctrl.mcc_alloc_index == (MAX_MCC_CMD - 1))
-			phba->ctrl.mcc_alloc_index = 0;
-		else
-			phba->ctrl.mcc_alloc_index++;
-	}
-	spin_unlock(&phba->ctrl.mcc_lock);
-	return tag;
-}
+/* UE Status High CSR */
+static const char * const desc_ue_status_hi[] = {
+	"LPCMEMHOST",
+	"MGMT_MAC",
+	"PCS0ONLINE",
+	"MPU_IRAM",
+	"PCS1ONLINE",
+	"PCTL0",
+	"PCTL1",
+	"PMEM",
+	"RR",
+	"TXPB",
+	"RXPP",
+	"XAUI",
+	"TXP",
+	"ARM",
+	"IPC",
+	"HOST2",
+	"HOST3",
+	"HOST4",
+	"HOST5",
+	"HOST6",
+	"HOST7",
+	"HOST8",
+	"HOST9",
+	"NETC",
+	"Unknown",
+	"Unknown",
+	"Unknown",
+	"Unknown",
+	"Unknown",
+	"Unknown",
+	"Unknown",
+	"Unknown"
+};
 
 struct be_mcc_wrb *alloc_mcc_wrb(struct beiscsi_hba *phba,
 				 unsigned int *ref_tag)
@@ -133,7 +100,7 @@ struct be_mcc_wrb *alloc_mcc_wrb(struct beiscsi_hba *phba,
 	struct be_mcc_wrb *wrb = NULL;
 	unsigned int tag;
 
-	spin_lock_bh(&phba->ctrl.mcc_lock);
+	spin_lock(&phba->ctrl.mcc_lock);
 	if (mccq->used == mccq->len) {
 		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT |
 			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
@@ -160,6 +127,7 @@ struct be_mcc_wrb *alloc_mcc_wrb(struct beiscsi_hba *phba,
 	phba->ctrl.mcc_tag[phba->ctrl.mcc_alloc_index] = 0;
 	phba->ctrl.mcc_tag_status[tag] = 0;
 	phba->ctrl.ptag_state[tag].tag_state = 0;
+	phba->ctrl.ptag_state[tag].cbfn = NULL;
 	phba->ctrl.mcc_tag_available--;
 	if (phba->ctrl.mcc_alloc_index == (MAX_MCC_CMD - 1))
 		phba->ctrl.mcc_alloc_index = 0;
@@ -174,7 +142,7 @@ struct be_mcc_wrb *alloc_mcc_wrb(struct beiscsi_hba *phba,
 	mccq->used++;
 
 alloc_failed:
-	spin_unlock_bh(&phba->ctrl.mcc_lock);
+	spin_unlock(&phba->ctrl.mcc_lock);
 	return wrb;
 }
 
@@ -182,7 +150,7 @@ void free_mcc_wrb(struct be_ctrl_info *ctrl, unsigned int tag)
 {
 	struct be_queue_info *mccq = &ctrl->mcc_obj.q;
 
-	spin_lock_bh(&ctrl->mcc_lock);
+	spin_lock(&ctrl->mcc_lock);
 	tag = tag & MCC_Q_CMD_TAG_MASK;
 	ctrl->mcc_tag[ctrl->mcc_free_index] = tag;
 	if (ctrl->mcc_free_index == (MAX_MCC_CMD - 1))
@@ -191,16 +159,71 @@ void free_mcc_wrb(struct be_ctrl_info *ctrl, unsigned int tag)
 		ctrl->mcc_free_index++;
 	ctrl->mcc_tag_available++;
 	mccq->used--;
-	spin_unlock_bh(&ctrl->mcc_lock);
+	spin_unlock(&ctrl->mcc_lock);
 }
 
-/**
- * beiscsi_fail_session(): Closing session with appropriate error
- * @cls_session: ptr to session
- **/
-void beiscsi_fail_session(struct iscsi_cls_session *cls_session)
+/*
+ * beiscsi_mcc_compl_status - Return the status of MCC completion
+ * @phba: Driver private structure
+ * @tag: Tag for the MBX Command
+ * @wrb: the WRB used for the MBX Command
+ * @mbx_cmd_mem: ptr to memory allocated for MBX Cmd
+ *
+ * return
+ * Success: 0
+ * Failure: Non-Zero
+ */
+int __beiscsi_mcc_compl_status(struct beiscsi_hba *phba,
+			       unsigned int tag,
+			       struct be_mcc_wrb **wrb,
+			       struct be_dma_mem *mbx_cmd_mem)
 {
-	iscsi_session_failure(cls_session->dd_data, ISCSI_ERR_CONN_FAILED);
+	struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
+	uint16_t status = 0, addl_status = 0, wrb_num = 0;
+	struct be_cmd_resp_hdr *mbx_resp_hdr;
+	struct be_cmd_req_hdr *mbx_hdr;
+	struct be_mcc_wrb *temp_wrb;
+	uint32_t mcc_tag_status;
+	int rc = 0;
+
+	mcc_tag_status = phba->ctrl.mcc_tag_status[tag];
+	status = (mcc_tag_status & CQE_STATUS_MASK);
+	addl_status = ((mcc_tag_status & CQE_STATUS_ADDL_MASK) >>
+			CQE_STATUS_ADDL_SHIFT);
+
+	if (mbx_cmd_mem) {
+		mbx_hdr = (struct be_cmd_req_hdr *)mbx_cmd_mem->va;
+	} else {
+		wrb_num = (mcc_tag_status & CQE_STATUS_WRB_MASK) >>
+			  CQE_STATUS_WRB_SHIFT;
+		temp_wrb = (struct be_mcc_wrb *)queue_get_wrb(mccq, wrb_num);
+		mbx_hdr = embedded_payload(temp_wrb);
+
+		if (wrb)
+			*wrb = temp_wrb;
+	}
+
+	if (status || addl_status) {
+		beiscsi_log(phba, KERN_WARNING,
+			    BEISCSI_LOG_INIT | BEISCSI_LOG_EH |
+			    BEISCSI_LOG_CONFIG,
+			    "BC_%d : MBX Cmd Failed for Subsys : %d Opcode : %d with Status : %d and Extd_Status : %d\n",
+			    mbx_hdr->subsystem, mbx_hdr->opcode,
+			    status, addl_status);
+		rc = -EIO;
+		if (status == MCC_STATUS_INSUFFICIENT_BUFFER) {
+			mbx_resp_hdr = (struct be_cmd_resp_hdr *)mbx_hdr;
+			beiscsi_log(phba, KERN_WARNING,
+				    BEISCSI_LOG_INIT | BEISCSI_LOG_EH |
+				    BEISCSI_LOG_CONFIG,
+				    "BC_%d : Insufficient Buffer Error Resp_Len : %d Actual_Resp_Len : %d\n",
+				    mbx_resp_hdr->response_length,
+				    mbx_resp_hdr->actual_resp_len);
+			rc = -EAGAIN;
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -217,26 +240,34 @@ void beiscsi_fail_session(struct iscsi_cls_session *cls_session)
  * Failure: Non-Zero
  **/
 int beiscsi_mccq_compl_wait(struct beiscsi_hba *phba,
-			    uint32_t tag, struct be_mcc_wrb **wrb,
+			    unsigned int tag,
+			    struct be_mcc_wrb **wrb,
 			    struct be_dma_mem *mbx_cmd_mem)
 {
 	int rc = 0;
-	uint32_t mcc_tag_status;
-	uint16_t status = 0, addl_status = 0, wrb_num = 0;
-	struct be_mcc_wrb *temp_wrb;
-	struct be_cmd_req_hdr *mbx_hdr;
-	struct be_cmd_resp_hdr *mbx_resp_hdr;
-	struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
 
-	if (beiscsi_error(phba))
-		return -EPERM;
+	if (beiscsi_hba_in_error(phba)) {
+		clear_bit(MCC_TAG_STATE_RUNNING,
+			  &phba->ctrl.ptag_state[tag].tag_state);
+		return -EIO;
+	}
 
 	/* wait for the mccq completion */
-	rc = wait_event_interruptible_timeout(
-				phba->ctrl.mcc_wait[tag],
-				phba->ctrl.mcc_tag_status[tag],
-				msecs_to_jiffies(
-				BEISCSI_HOST_MBX_TIMEOUT));
+	rc = wait_event_interruptible_timeout(phba->ctrl.mcc_wait[tag],
+					      phba->ctrl.mcc_tag_status[tag],
+					      msecs_to_jiffies(
+						BEISCSI_HOST_MBX_TIMEOUT));
+	/**
+	 * Return EIO if port is being disabled. Associated DMA memory, if any,
+	 * is freed by the caller. When port goes offline, MCCQ is cleaned up
+	 * so does WRB.
+	 */
+	if (!test_bit(BEISCSI_HBA_ONLINE, &phba->state)) {
+		clear_bit(MCC_TAG_STATE_RUNNING,
+			  &phba->ctrl.ptag_state[tag].tag_state);
+		return -EIO;
+	}
+
 	/**
 	 * If MBOX cmd timeout expired, tag and resource allocated
 	 * for cmd is not freed until FW returns completion.
@@ -270,47 +301,7 @@ int beiscsi_mccq_compl_wait(struct beiscsi_hba *phba,
 		return -EBUSY;
 	}
 
-	rc = 0;
-	mcc_tag_status = phba->ctrl.mcc_tag_status[tag];
-	status = (mcc_tag_status & CQE_STATUS_MASK);
-	addl_status = ((mcc_tag_status & CQE_STATUS_ADDL_MASK) >>
-			CQE_STATUS_ADDL_SHIFT);
-
-	if (mbx_cmd_mem) {
-		mbx_hdr = (struct be_cmd_req_hdr *)mbx_cmd_mem->va;
-	} else {
-		wrb_num = (mcc_tag_status & CQE_STATUS_WRB_MASK) >>
-			   CQE_STATUS_WRB_SHIFT;
-		temp_wrb = (struct be_mcc_wrb *)queue_get_wrb(mccq, wrb_num);
-		mbx_hdr = embedded_payload(temp_wrb);
-
-		if (wrb)
-			*wrb = temp_wrb;
-	}
-
-	if (status || addl_status) {
-		beiscsi_log(phba, KERN_WARNING,
-			    BEISCSI_LOG_INIT | BEISCSI_LOG_EH |
-			    BEISCSI_LOG_CONFIG,
-			    "BC_%d : MBX Cmd Failed for "
-			    "Subsys : %d Opcode : %d with "
-			    "Status : %d and Extd_Status : %d\n",
-			    mbx_hdr->subsystem,
-			    mbx_hdr->opcode,
-			    status, addl_status);
-		rc = -EIO;
-		if (status == MCC_STATUS_INSUFFICIENT_BUFFER) {
-			mbx_resp_hdr = (struct be_cmd_resp_hdr *) mbx_hdr;
-			beiscsi_log(phba, KERN_WARNING,
-				    BEISCSI_LOG_INIT | BEISCSI_LOG_EH |
-				    BEISCSI_LOG_CONFIG,
-				    "BC_%d : Insufficient Buffer Error "
-				    "Resp_Len : %d Actual_Resp_Len : %d\n",
-				    mbx_resp_hdr->response_length,
-				    mbx_resp_hdr->actual_resp_len);
-			rc = -EAGAIN;
-		}
-	}
+	rc = __beiscsi_mcc_compl_status(phba, tag, wrb, mbx_cmd_mem);
 
 	free_mcc_wrb(&phba->ctrl, tag);
 	return rc;
@@ -330,11 +321,10 @@ int beiscsi_mccq_compl_wait(struct beiscsi_hba *phba,
 static int beiscsi_process_mbox_compl(struct be_ctrl_info *ctrl,
 				      struct be_mcc_compl *compl)
 {
-	u16 compl_status, extd_status;
 	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
 	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
 	struct be_cmd_req_hdr *hdr = embedded_payload(wrb);
-	struct be_cmd_resp_hdr *resp_hdr;
+	u16 compl_status, extd_status;
 
 	/**
 	 * To check if valid bit is set, check the entire word as we don't know
@@ -368,14 +358,7 @@ static int beiscsi_process_mbox_compl(struct be_ctrl_info *ctrl,
 	beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
 		    "BC_%d : error in cmd completion: Subsystem : %d Opcode : %d status(compl/extd)=%d/%d\n",
 		    hdr->subsystem, hdr->opcode, compl_status, extd_status);
-
-	if (compl_status == MCC_STATUS_INSUFFICIENT_BUFFER) {
-		/* if status is insufficient buffer, check the length */
-		resp_hdr = (struct be_cmd_resp_hdr *) hdr;
-		if (resp_hdr->response_length)
-			return 0;
-	}
-	return -EINVAL;
+	return compl_status;
 }
 
 static void beiscsi_process_async_link(struct beiscsi_hba *phba,
@@ -391,18 +374,19 @@ static void beiscsi_process_async_link(struct beiscsi_hba *phba,
 	 * This has been newly introduced in SKH-R Firmware 10.0.338.45.
 	 **/
 	if (evt->port_link_status & BE_ASYNC_LINK_UP_MASK) {
-		phba->state = BE_ADAPTER_LINK_UP | BE_ADAPTER_CHECK_BOOT;
-		phba->get_boot = BE_GET_BOOT_RETRIES;
+		set_bit(BEISCSI_HBA_LINK_UP, &phba->state);
+		if (test_bit(BEISCSI_HBA_BOOT_FOUND, &phba->state))
+			beiscsi_start_boot_work(phba, BE_BOOT_INVALID_SHANDLE);
 		__beiscsi_log(phba, KERN_ERR,
 			      "BC_%d : Link Up on Port %d tag 0x%x\n",
 			      evt->physical_port, evt->event_tag);
 	} else {
-		phba->state = BE_ADAPTER_LINK_DOWN;
+		clear_bit(BEISCSI_HBA_LINK_UP, &phba->state);
 		__beiscsi_log(phba, KERN_ERR,
 			      "BC_%d : Link Down on Port %d tag 0x%x\n",
 			      evt->physical_port, evt->event_tag);
 		iscsi_host_for_each_session(phba->shost,
-					    beiscsi_fail_session);
+					    beiscsi_session_fail);
 	}
 }
 
@@ -482,8 +466,8 @@ void beiscsi_process_async_event(struct beiscsi_hba *phba,
 		beiscsi_process_async_link(phba, compl);
 		break;
 	case ASYNC_EVENT_CODE_ISCSI:
-		phba->state |= BE_ADAPTER_CHECK_BOOT;
-		phba->get_boot = BE_GET_BOOT_RETRIES;
+		if (test_bit(BEISCSI_HBA_BOOT_FOUND, &phba->state))
+			beiscsi_start_boot_work(phba, BE_BOOT_INVALID_SHANDLE);
 		sev = KERN_ERR;
 		break;
 	case ASYNC_EVENT_CODE_SLI:
@@ -519,6 +503,9 @@ int beiscsi_process_mcc_compl(struct be_ctrl_info *ctrl,
 		return 0;
 	}
 
+	/* end MCC with this tag */
+	clear_bit(MCC_TAG_STATE_RUNNING, &ctrl->ptag_state[tag].tag_state);
+
 	if (test_bit(MCC_TAG_STATE_TIMEOUT, &ctrl->ptag_state[tag].tag_state)) {
 		beiscsi_log(phba, KERN_WARNING,
 			    BEISCSI_LOG_MBOX | BEISCSI_LOG_INIT |
@@ -529,9 +516,11 @@ int beiscsi_process_mcc_compl(struct be_ctrl_info *ctrl,
 		 * Only for non-embedded cmd, PCI resource is allocated.
 		 **/
 		tag_mem = &ctrl->ptag_state[tag].tag_mem_state;
-		if (tag_mem->size)
+		if (tag_mem->size) {
 			pci_free_consistent(ctrl->pdev, tag_mem->size,
 					tag_mem->va, tag_mem->dma);
+			tag_mem->size = 0;
+		}
 		free_mcc_wrb(ctrl, tag);
 		return 0;
 	}
@@ -550,57 +539,25 @@ int beiscsi_process_mcc_compl(struct be_ctrl_info *ctrl,
 				     CQE_STATUS_ADDL_MASK;
 	ctrl->mcc_tag_status[tag] |= (compl_status & CQE_STATUS_MASK);
 
-	/* write ordering forced in wake_up_interruptible */
-	clear_bit(MCC_TAG_STATE_RUNNING, &ctrl->ptag_state[tag].tag_state);
-	wake_up_interruptible(&ctrl->mcc_wait[tag]);
-	return 0;
-}
-
-/*
- * be_mcc_compl_poll()- Wait for MBX completion
- * @phba: driver private structure
- *
- * Wait till no more pending mcc requests are present
- *
- * return
- * Success: 0
- * Failure: Non-Zero
- *
- **/
-int be_mcc_compl_poll(struct beiscsi_hba *phba, unsigned int tag)
-{
-	struct be_ctrl_info *ctrl = &phba->ctrl;
-	int i;
-
-	if (!test_bit(MCC_TAG_STATE_RUNNING,
-		      &ctrl->ptag_state[tag].tag_state)) {
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
-			    "BC_%d: tag %u state not running\n", tag);
+	if (test_bit(MCC_TAG_STATE_ASYNC, &ctrl->ptag_state[tag].tag_state)) {
+		if (ctrl->ptag_state[tag].cbfn)
+			ctrl->ptag_state[tag].cbfn(phba, tag);
+		else
+			__beiscsi_log(phba, KERN_ERR,
+				      "BC_%d : MBX ASYNC command with no callback\n");
+		free_mcc_wrb(ctrl, tag);
 		return 0;
 	}
-	for (i = 0; i < mcc_timeout; i++) {
-		if (beiscsi_error(phba))
-			return -EIO;
 
-		beiscsi_process_mcc_cq(phba);
-		/* after polling, wrb and tag need to be released */
-		if (!test_bit(MCC_TAG_STATE_RUNNING,
-			      &ctrl->ptag_state[tag].tag_state)) {
-			free_mcc_wrb(ctrl, tag);
-			break;
-		}
-		udelay(100);
-	}
-
-	if (i < mcc_timeout)
+	if (test_bit(MCC_TAG_STATE_IGNORE, &ctrl->ptag_state[tag].tag_state)) {
+		/* just check completion status and free wrb */
+		__beiscsi_mcc_compl_status(phba, tag, NULL, NULL);
+		free_mcc_wrb(ctrl, tag);
 		return 0;
+	}
 
-	beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
-		    "BC_%d : FW Timed Out\n");
-	phba->fw_timeout = true;
-	beiscsi_ue_detect(phba);
-	return -EBUSY;
+	wake_up_interruptible(&ctrl->mcc_wait[tag]);
+	return 0;
 }
 
 void be_mcc_notify(struct beiscsi_hba *phba, unsigned int tag)
@@ -642,7 +599,7 @@ static int be_mbox_db_ready_poll(struct be_ctrl_info *ctrl)
 	 */
 	timeout = jiffies + msecs_to_jiffies(BEISCSI_MBX_RDY_BIT_TIMEOUT);
 	do {
-		if (beiscsi_error(phba))
+		if (beiscsi_hba_in_error(phba))
 			return -EIO;
 
 		ready = ioread32(db);
@@ -655,16 +612,14 @@ static int be_mbox_db_ready_poll(struct be_ctrl_info *ctrl)
 
 		if (time_after(jiffies, timeout))
 			break;
-		msleep(20);
+		/* 1ms sleep is enough in most cases */
+		schedule_timeout_uninterruptible(msecs_to_jiffies(1));
 	} while (!ready);
 
 	beiscsi_log(phba, KERN_ERR,
 			BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
 			"BC_%d : FW Timed Out\n");
-
-	phba->fw_timeout = true;
-	beiscsi_ue_detect(phba);
-
+	set_bit(BEISCSI_HBA_FW_TIMEOUT, &phba->state);
 	return -EBUSY;
 }
 
@@ -679,7 +634,7 @@ static int be_mbox_db_ready_poll(struct be_ctrl_info *ctrl)
  * Success: 0
  * Failure: Non-Zero
  **/
-int be_mbox_notify(struct be_ctrl_info *ctrl)
+static int be_mbox_notify(struct be_ctrl_info *ctrl)
 {
 	int status;
 	u32 val = 0;
@@ -819,87 +774,6 @@ int beiscsi_cmd_eq_create(struct be_ctrl_info *ctrl,
 	return status;
 }
 
-/**
- * be_cmd_fw_initialize()- Initialize FW
- * @ctrl: Pointer to function control structure
- *
- * Send FW initialize pattern for the function.
- *
- * return
- * Success: 0
- * Failure: Non-Zero value
- **/
-int be_cmd_fw_initialize(struct be_ctrl_info *ctrl)
-{
-	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
-	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
-	int status;
-	u8 *endian_check;
-
-	mutex_lock(&ctrl->mbox_lock);
-	memset(wrb, 0, sizeof(*wrb));
-
-	endian_check = (u8 *) wrb;
-	*endian_check++ = 0xFF;
-	*endian_check++ = 0x12;
-	*endian_check++ = 0x34;
-	*endian_check++ = 0xFF;
-	*endian_check++ = 0xFF;
-	*endian_check++ = 0x56;
-	*endian_check++ = 0x78;
-	*endian_check++ = 0xFF;
-	be_dws_cpu_to_le(wrb, sizeof(*wrb));
-
-	status = be_mbox_notify(ctrl);
-	if (status)
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BC_%d : be_cmd_fw_initialize Failed\n");
-
-	mutex_unlock(&ctrl->mbox_lock);
-	return status;
-}
-
-/**
- * be_cmd_fw_uninit()- Uinitialize FW
- * @ctrl: Pointer to function control structure
- *
- * Send FW uninitialize pattern for the function
- *
- * return
- * Success: 0
- * Failure: Non-Zero value
- **/
-int be_cmd_fw_uninit(struct be_ctrl_info *ctrl)
-{
-	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
-	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
-	int status;
-	u8 *endian_check;
-
-	mutex_lock(&ctrl->mbox_lock);
-	memset(wrb, 0, sizeof(*wrb));
-
-	endian_check = (u8 *) wrb;
-	*endian_check++ = 0xFF;
-	*endian_check++ = 0xAA;
-	*endian_check++ = 0xBB;
-	*endian_check++ = 0xFF;
-	*endian_check++ = 0xFF;
-	*endian_check++ = 0xCC;
-	*endian_check++ = 0xDD;
-	*endian_check = 0xFF;
-
-	be_dws_cpu_to_le(wrb, sizeof(*wrb));
-
-	status = be_mbox_notify(ctrl);
-	if (status)
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BC_%d : be_cmd_fw_uninit Failed\n");
-
-	mutex_unlock(&ctrl->mbox_lock);
-	return status;
-}
-
 int beiscsi_cmd_cq_create(struct be_ctrl_info *ctrl,
 			  struct be_queue_info *cq, struct be_queue_info *eq,
 			  bool sol_evts, bool no_delay, int coalesce_wm)
@@ -1343,25 +1217,6 @@ error:
 	return status;
 }
 
-int beiscsi_cmd_reset_function(struct beiscsi_hba  *phba)
-{
-	struct be_ctrl_info *ctrl = &phba->ctrl;
-	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
-	struct be_post_sgl_pages_req *req = embedded_payload(wrb);
-	int status;
-
-	mutex_lock(&ctrl->mbox_lock);
-
-	req = embedded_payload(wrb);
-	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
-	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_COMMON,
-			   OPCODE_COMMON_FUNCTION_RESET, sizeof(*req));
-	status = be_mbox_notify(ctrl);
-
-	mutex_unlock(&ctrl->mbox_lock);
-	return status;
-}
-
 /**
  * be_cmd_set_vlan()- Configure VLAN paramters on the adapter
  * @phba: device priv structure instance
@@ -1402,3 +1257,564 @@ int be_cmd_set_vlan(struct beiscsi_hba *phba,
 
 	return tag;
 }
+
+int beiscsi_check_supported_fw(struct be_ctrl_info *ctrl,
+			       struct beiscsi_hba *phba)
+{
+	struct be_dma_mem nonemb_cmd;
+	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
+	struct be_mgmt_controller_attributes *req;
+	struct be_sge *sge = nonembedded_sgl(wrb);
+	int status = 0;
+
+	nonemb_cmd.va = pci_alloc_consistent(ctrl->pdev,
+				sizeof(struct be_mgmt_controller_attributes),
+				&nonemb_cmd.dma);
+	if (nonemb_cmd.va == NULL) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BG_%d : pci_alloc_consistent failed in %s\n",
+			    __func__);
+		return -ENOMEM;
+	}
+	nonemb_cmd.size = sizeof(struct be_mgmt_controller_attributes);
+	req = nonemb_cmd.va;
+	memset(req, 0, sizeof(*req));
+	mutex_lock(&ctrl->mbox_lock);
+	memset(wrb, 0, sizeof(*wrb));
+	be_wrb_hdr_prepare(wrb, sizeof(*req), false, 1);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_COMMON,
+			   OPCODE_COMMON_GET_CNTL_ATTRIBUTES, sizeof(*req));
+	sge->pa_hi = cpu_to_le32(upper_32_bits(nonemb_cmd.dma));
+	sge->pa_lo = cpu_to_le32(nonemb_cmd.dma & 0xFFFFFFFF);
+	sge->len = cpu_to_le32(nonemb_cmd.size);
+	status = be_mbox_notify(ctrl);
+	if (!status) {
+		struct be_mgmt_controller_attributes_resp *resp = nonemb_cmd.va;
+
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BG_%d : Firmware Version of CMD : %s\n"
+			    "Firmware Version is : %s\n"
+			    "Developer Build, not performing version check...\n",
+			    resp->params.hba_attribs
+			    .flashrom_version_string,
+			    resp->params.hba_attribs.
+			    firmware_version_string);
+
+		phba->fw_config.iscsi_features =
+				resp->params.hba_attribs.iscsi_features;
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : phba->fw_config.iscsi_features = %d\n",
+			    phba->fw_config.iscsi_features);
+		memcpy(phba->fw_ver_str, resp->params.hba_attribs.
+		       firmware_version_string, BEISCSI_VER_STRLEN);
+	} else
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BG_%d :  Failed in beiscsi_check_supported_fw\n");
+	mutex_unlock(&ctrl->mbox_lock);
+	if (nonemb_cmd.va)
+		pci_free_consistent(ctrl->pdev, nonemb_cmd.size,
+				    nonemb_cmd.va, nonemb_cmd.dma);
+
+	return status;
+}
+
+/**
+ * beiscsi_get_fw_config()- Get the FW config for the function
+ * @ctrl: ptr to Ctrl Info
+ * @phba: ptr to the dev priv structure
+ *
+ * Get the FW config and resources available for the function.
+ * The resources are created based on the count received here.
+ *
+ * return
+ *	Success: 0
+ *	Failure: Non-Zero Value
+ **/
+int beiscsi_get_fw_config(struct be_ctrl_info *ctrl,
+			  struct beiscsi_hba *phba)
+{
+	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
+	struct be_fw_cfg *pfw_cfg = embedded_payload(wrb);
+	uint32_t cid_count, icd_count;
+	int status = -EINVAL;
+	uint8_t ulp_num = 0;
+
+	mutex_lock(&ctrl->mbox_lock);
+	memset(wrb, 0, sizeof(*wrb));
+	be_wrb_hdr_prepare(wrb, sizeof(*pfw_cfg), true, 0);
+
+	be_cmd_hdr_prepare(&pfw_cfg->hdr, CMD_SUBSYSTEM_COMMON,
+			   OPCODE_COMMON_QUERY_FIRMWARE_CONFIG,
+			   EMBED_MBX_MAX_PAYLOAD_SIZE);
+
+	if (be_mbox_notify(ctrl)) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BG_%d : Failed in beiscsi_get_fw_config\n");
+		goto fail_init;
+	}
+
+	/* FW response formats depend on port id */
+	phba->fw_config.phys_port = pfw_cfg->phys_port;
+	if (phba->fw_config.phys_port >= BEISCSI_PHYS_PORT_MAX) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BG_%d : invalid physical port id %d\n",
+			    phba->fw_config.phys_port);
+		goto fail_init;
+	}
+
+	/* populate and check FW config against min and max values */
+	if (!is_chip_be2_be3r(phba)) {
+		phba->fw_config.eqid_count = pfw_cfg->eqid_count;
+		phba->fw_config.cqid_count = pfw_cfg->cqid_count;
+		if (phba->fw_config.eqid_count == 0 ||
+		    phba->fw_config.eqid_count > 2048) {
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BG_%d : invalid EQ count %d\n",
+				    phba->fw_config.eqid_count);
+			goto fail_init;
+		}
+		if (phba->fw_config.cqid_count == 0 ||
+		    phba->fw_config.cqid_count > 4096) {
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BG_%d : invalid CQ count %d\n",
+				    phba->fw_config.cqid_count);
+			goto fail_init;
+		}
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BG_%d : EQ_Count : %d CQ_Count : %d\n",
+			    phba->fw_config.eqid_count,
+			    phba->fw_config.cqid_count);
+	}
+
+	/**
+	 * Check on which all ULP iSCSI Protocol is loaded.
+	 * Set the Bit for those ULP. This set flag is used
+	 * at all places in the code to check on which ULP
+	 * iSCSi Protocol is loaded
+	 **/
+	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
+		if (pfw_cfg->ulp[ulp_num].ulp_mode &
+		    BEISCSI_ULP_ISCSI_INI_MODE) {
+			set_bit(ulp_num, &phba->fw_config.ulp_supported);
+
+			/* Get the CID, ICD and Chain count for each ULP */
+			phba->fw_config.iscsi_cid_start[ulp_num] =
+				pfw_cfg->ulp[ulp_num].sq_base;
+			phba->fw_config.iscsi_cid_count[ulp_num] =
+				pfw_cfg->ulp[ulp_num].sq_count;
+
+			phba->fw_config.iscsi_icd_start[ulp_num] =
+				pfw_cfg->ulp[ulp_num].icd_base;
+			phba->fw_config.iscsi_icd_count[ulp_num] =
+				pfw_cfg->ulp[ulp_num].icd_count;
+
+			phba->fw_config.iscsi_chain_start[ulp_num] =
+				pfw_cfg->chain_icd[ulp_num].chain_base;
+			phba->fw_config.iscsi_chain_count[ulp_num] =
+				pfw_cfg->chain_icd[ulp_num].chain_count;
+
+			beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+				    "BG_%d : Function loaded on ULP : %d\n"
+				    "\tiscsi_cid_count : %d\n"
+				    "\tiscsi_cid_start : %d\n"
+				    "\t iscsi_icd_count : %d\n"
+				    "\t iscsi_icd_start : %d\n",
+				    ulp_num,
+				    phba->fw_config.
+				    iscsi_cid_count[ulp_num],
+				    phba->fw_config.
+				    iscsi_cid_start[ulp_num],
+				    phba->fw_config.
+				    iscsi_icd_count[ulp_num],
+				    phba->fw_config.
+				    iscsi_icd_start[ulp_num]);
+		}
+	}
+
+	if (phba->fw_config.ulp_supported == 0) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BG_%d : iSCSI initiator mode not set: ULP0 %x ULP1 %x\n",
+			    pfw_cfg->ulp[BEISCSI_ULP0].ulp_mode,
+			    pfw_cfg->ulp[BEISCSI_ULP1].ulp_mode);
+		goto fail_init;
+	}
+
+	/**
+	 * ICD is shared among ULPs. Use icd_count of any one loaded ULP
+	 **/
+	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
+		if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
+			break;
+	icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
+	if (icd_count == 0 || icd_count > 65536) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BG_%d: invalid ICD count %d\n", icd_count);
+		goto fail_init;
+	}
+
+	cid_count = BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP0) +
+		    BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP1);
+	if (cid_count == 0 || cid_count > 4096) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BG_%d: invalid CID count %d\n", cid_count);
+		goto fail_init;
+	}
+
+	/**
+	 * Check FW is dual ULP aware i.e. can handle either
+	 * of the protocols.
+	 */
+	phba->fw_config.dual_ulp_aware = (pfw_cfg->function_mode &
+					  BEISCSI_FUNC_DUA_MODE);
+
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BG_%d : DUA Mode : 0x%x\n",
+		    phba->fw_config.dual_ulp_aware);
+
+	/* all set, continue using this FW config */
+	status = 0;
+fail_init:
+	mutex_unlock(&ctrl->mbox_lock);
+	return status;
+}
+
+/**
+ * beiscsi_get_port_name()- Get port name for the function
+ * @ctrl: ptr to Ctrl Info
+ * @phba: ptr to the dev priv structure
+ *
+ * Get the alphanumeric character for port
+ *
+ **/
+int beiscsi_get_port_name(struct be_ctrl_info *ctrl, struct beiscsi_hba *phba)
+{
+	int ret = 0;
+	struct be_mcc_wrb *wrb;
+	struct be_cmd_get_port_name *ioctl;
+
+	mutex_lock(&ctrl->mbox_lock);
+	wrb = wrb_from_mbox(&ctrl->mbox_mem);
+	memset(wrb, 0, sizeof(*wrb));
+	ioctl = embedded_payload(wrb);
+
+	be_wrb_hdr_prepare(wrb, sizeof(*ioctl), true, 0);
+	be_cmd_hdr_prepare(&ioctl->h.req_hdr, CMD_SUBSYSTEM_COMMON,
+			   OPCODE_COMMON_GET_PORT_NAME,
+			   EMBED_MBX_MAX_PAYLOAD_SIZE);
+	ret = be_mbox_notify(ctrl);
+	phba->port_name = 0;
+	if (!ret) {
+		phba->port_name = ioctl->p.resp.port_names >>
+				  (phba->fw_config.phys_port * 8) & 0xff;
+	} else {
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BG_%d : GET_PORT_NAME ret 0x%x status 0x%x\n",
+			    ret, ioctl->h.resp_hdr.status);
+	}
+
+	if (phba->port_name == 0)
+		phba->port_name = '?';
+
+	mutex_unlock(&ctrl->mbox_lock);
+	return ret;
+}
+
+int beiscsi_set_uer_feature(struct beiscsi_hba *phba)
+{
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct be_cmd_set_features *ioctl;
+	struct be_mcc_wrb *wrb;
+	int ret = 0;
+
+	mutex_lock(&ctrl->mbox_lock);
+	wrb = wrb_from_mbox(&ctrl->mbox_mem);
+	memset(wrb, 0, sizeof(*wrb));
+	ioctl = embedded_payload(wrb);
+
+	be_wrb_hdr_prepare(wrb, sizeof(*ioctl), true, 0);
+	be_cmd_hdr_prepare(&ioctl->h.req_hdr, CMD_SUBSYSTEM_COMMON,
+			   OPCODE_COMMON_SET_FEATURES,
+			   EMBED_MBX_MAX_PAYLOAD_SIZE);
+	ioctl->feature = BE_CMD_SET_FEATURE_UER;
+	ioctl->param_len = sizeof(ioctl->param.req);
+	ioctl->param.req.uer = BE_CMD_UER_SUPP_BIT;
+	ret = be_mbox_notify(ctrl);
+	if (!ret) {
+		phba->ue2rp = ioctl->param.resp.ue2rp;
+		set_bit(BEISCSI_HBA_UER_SUPP, &phba->state);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BG_%d : HBA error recovery supported\n");
+	} else {
+		/**
+		 * Check "MCC_STATUS_INVALID_LENGTH" for SKH.
+		 * Older FW versions return this error.
+		 */
+		if (ret == MCC_STATUS_ILLEGAL_REQUEST ||
+		    ret == MCC_STATUS_INVALID_LENGTH)
+			__beiscsi_log(phba, KERN_INFO,
+				      "BG_%d : HBA error recovery not supported\n");
+	}
+
+	mutex_unlock(&ctrl->mbox_lock);
+	return ret;
+}
+
+static u32 beiscsi_get_post_stage(struct beiscsi_hba *phba)
+{
+	u32 sem;
+
+	if (is_chip_be2_be3r(phba))
+		sem = ioread32(phba->csr_va + SLIPORT_SEMAPHORE_OFFSET_BEx);
+	else
+		pci_read_config_dword(phba->pcidev,
+				      SLIPORT_SEMAPHORE_OFFSET_SH, &sem);
+	return sem;
+}
+
+int beiscsi_check_fw_rdy(struct beiscsi_hba *phba)
+{
+	u32 loop, post, rdy = 0;
+
+	loop = 1000;
+	while (loop--) {
+		post = beiscsi_get_post_stage(phba);
+		if (post & POST_ERROR_BIT)
+			break;
+		if ((post & POST_STAGE_MASK) == POST_STAGE_ARMFW_RDY) {
+			rdy = 1;
+			break;
+		}
+		msleep(60);
+	}
+
+	if (!rdy) {
+		__beiscsi_log(phba, KERN_ERR,
+			      "BC_%d : FW not ready 0x%x\n", post);
+	}
+
+	return rdy;
+}
+
+int beiscsi_cmd_function_reset(struct beiscsi_hba *phba)
+{
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
+	struct be_post_sgl_pages_req *req = embedded_payload(wrb);
+	int status;
+
+	mutex_lock(&ctrl->mbox_lock);
+
+	req = embedded_payload(wrb);
+	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_COMMON,
+			   OPCODE_COMMON_FUNCTION_RESET, sizeof(*req));
+	status = be_mbox_notify(ctrl);
+
+	mutex_unlock(&ctrl->mbox_lock);
+	return status;
+}
+
+int beiscsi_cmd_special_wrb(struct be_ctrl_info *ctrl, u32 load)
+{
+	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
+	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
+	u8 *endian_check;
+	int status;
+
+	mutex_lock(&ctrl->mbox_lock);
+	memset(wrb, 0, sizeof(*wrb));
+
+	endian_check = (u8 *) wrb;
+	if (load) {
+		/* to start communicating */
+		*endian_check++ = 0xFF;
+		*endian_check++ = 0x12;
+		*endian_check++ = 0x34;
+		*endian_check++ = 0xFF;
+		*endian_check++ = 0xFF;
+		*endian_check++ = 0x56;
+		*endian_check++ = 0x78;
+		*endian_check++ = 0xFF;
+	} else {
+		/* to stop communicating */
+		*endian_check++ = 0xFF;
+		*endian_check++ = 0xAA;
+		*endian_check++ = 0xBB;
+		*endian_check++ = 0xFF;
+		*endian_check++ = 0xFF;
+		*endian_check++ = 0xCC;
+		*endian_check++ = 0xDD;
+		*endian_check = 0xFF;
+	}
+	be_dws_cpu_to_le(wrb, sizeof(*wrb));
+
+	status = be_mbox_notify(ctrl);
+	if (status)
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BC_%d : special WRB message failed\n");
+	mutex_unlock(&ctrl->mbox_lock);
+	return status;
+}
+
+int beiscsi_init_sliport(struct beiscsi_hba *phba)
+{
+	int status;
+
+	/* check POST stage before talking to FW */
+	status = beiscsi_check_fw_rdy(phba);
+	if (!status)
+		return -EIO;
+
+	/* clear all error states after checking FW rdy */
+	phba->state &= ~BEISCSI_HBA_IN_ERR;
+
+	/* check again UER support */
+	phba->state &= ~BEISCSI_HBA_UER_SUPP;
+
+	/*
+	 * SLI COMMON_FUNCTION_RESET completion is indicated by BMBX RDY bit.
+	 * It should clean up any stale info in FW for this fn.
+	 */
+	status = beiscsi_cmd_function_reset(phba);
+	if (status) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BC_%d : SLI Function Reset failed\n");
+		return status;
+	}
+
+	/* indicate driver is loading */
+	return beiscsi_cmd_special_wrb(&phba->ctrl, 1);
+}
+
+/**
+ * beiscsi_cmd_iscsi_cleanup()- Inform FW to cleanup EP data structures.
+ * @phba: pointer to dev priv structure
+ * @ulp: ULP number.
+ *
+ * return
+ *	Success: 0
+ *	Failure: Non-Zero Value
+ **/
+int beiscsi_cmd_iscsi_cleanup(struct beiscsi_hba *phba, unsigned short ulp)
+{
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct iscsi_cleanup_req_v1 *req_v1;
+	struct iscsi_cleanup_req *req;
+	struct be_mcc_wrb *wrb;
+	int status;
+
+	mutex_lock(&ctrl->mbox_lock);
+	wrb = wrb_from_mbox(&ctrl->mbox_mem);
+	req = embedded_payload(wrb);
+	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI,
+			   OPCODE_COMMON_ISCSI_CLEANUP, sizeof(*req));
+
+       /**
+	* TODO: Check with FW folks the chute value to be set.
+	* For now, use the ULP_MASK as the chute value.
+	*/
+	if (is_chip_be2_be3r(phba)) {
+		req->chute = (1 << ulp);
+		req->hdr_ring_id = HWI_GET_DEF_HDRQ_ID(phba, ulp);
+		req->data_ring_id = HWI_GET_DEF_BUFQ_ID(phba, ulp);
+	} else {
+		req_v1 = (struct iscsi_cleanup_req_v1 *)req;
+		req_v1->hdr.version = 1;
+		req_v1->hdr_ring_id = cpu_to_le16(HWI_GET_DEF_HDRQ_ID(phba,
+								      ulp));
+		req_v1->data_ring_id = cpu_to_le16(HWI_GET_DEF_BUFQ_ID(phba,
+								       ulp));
+	}
+
+	status = be_mbox_notify(ctrl);
+	if (status)
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BG_%d : %s failed %d\n", __func__, ulp);
+	mutex_unlock(&ctrl->mbox_lock);
+	return status;
+}
+
+/*
+ * beiscsi_detect_ue()- Detect Unrecoverable Error on adapter
+ * @phba: Driver priv structure
+ *
+ * Read registers linked to UE and check for the UE status
+ **/
+int beiscsi_detect_ue(struct beiscsi_hba *phba)
+{
+	uint32_t ue_mask_hi = 0, ue_mask_lo = 0;
+	uint32_t ue_hi = 0, ue_lo = 0;
+	uint8_t i = 0;
+	int ret = 0;
+
+	pci_read_config_dword(phba->pcidev,
+			      PCICFG_UE_STATUS_LOW, &ue_lo);
+	pci_read_config_dword(phba->pcidev,
+			      PCICFG_UE_STATUS_MASK_LOW,
+			      &ue_mask_lo);
+	pci_read_config_dword(phba->pcidev,
+			      PCICFG_UE_STATUS_HIGH,
+			      &ue_hi);
+	pci_read_config_dword(phba->pcidev,
+			      PCICFG_UE_STATUS_MASK_HI,
+			      &ue_mask_hi);
+
+	ue_lo = (ue_lo & ~ue_mask_lo);
+	ue_hi = (ue_hi & ~ue_mask_hi);
+
+
+	if (ue_lo || ue_hi) {
+		set_bit(BEISCSI_HBA_IN_UE, &phba->state);
+		__beiscsi_log(phba, KERN_ERR,
+			      "BC_%d : HBA error detected\n");
+		ret = 1;
+	}
+
+	if (ue_lo) {
+		for (i = 0; ue_lo; ue_lo >>= 1, i++) {
+			if (ue_lo & 1)
+				__beiscsi_log(phba, KERN_ERR,
+					      "BC_%d : UE_LOW %s bit set\n",
+					      desc_ue_status_low[i]);
+		}
+	}
+
+	if (ue_hi) {
+		for (i = 0; ue_hi; ue_hi >>= 1, i++) {
+			if (ue_hi & 1)
+				__beiscsi_log(phba, KERN_ERR,
+					      "BC_%d : UE_HIGH %s bit set\n",
+					      desc_ue_status_hi[i]);
+		}
+	}
+	return ret;
+}
+
+/*
+ * beiscsi_detect_tpe()- Detect Transient Parity Error on adapter
+ * @phba: Driver priv structure
+ *
+ * Read SLIPORT SEMAPHORE register to check for UER
+ *
+ **/
+int beiscsi_detect_tpe(struct beiscsi_hba *phba)
+{
+	u32 post, status;
+	int ret = 0;
+
+	post = beiscsi_get_post_stage(phba);
+	status = post & POST_STAGE_MASK;
+	if ((status & POST_ERR_RECOVERY_CODE_MASK) ==
+	    POST_STAGE_RECOVERABLE_ERR) {
+		set_bit(BEISCSI_HBA_IN_TPE, &phba->state);
+		__beiscsi_log(phba, KERN_INFO,
+			      "BC_%d : HBA error recoverable: 0x%x\n", post);
+		ret = 1;
+	} else {
+		__beiscsi_log(phba, KERN_INFO,
+			      "BC_%d : HBA in UE: 0x%x\n", post);
+	}
+
+	return ret;
+}
diff --git a/drivers/scsi/be2iscsi/be_cmds.h b/drivers/scsi/be2iscsi/be_cmds.h
index deeb951e6874..328fb5b973cd 100644
--- a/drivers/scsi/be2iscsi/be_cmds.h
+++ b/drivers/scsi/be2iscsi/be_cmds.h
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -8,7 +8,7 @@
  * Public License is included in this distribution in the file called COPYING.
  *
  * Contact Information:
- * linux-drivers@avagotech.com
+ * linux-drivers@broadcom.com
  *
  * Emulex
  * 3333 Susan Street
@@ -57,6 +57,7 @@ struct be_mcc_wrb {
 #define MCC_STATUS_ILLEGAL_REQUEST 0x2
 #define MCC_STATUS_ILLEGAL_FIELD 0x3
 #define MCC_STATUS_INSUFFICIENT_BUFFER 0x4
+#define MCC_STATUS_INVALID_LENGTH 0x74
 
 #define CQE_STATUS_COMPL_MASK	0xFFFF
 #define CQE_STATUS_COMPL_SHIFT	0		/* bits 0 - 15 */
@@ -97,11 +98,23 @@ struct be_mcc_compl {
 #define MPU_MAILBOX_DB_RDY_MASK	0x1	/* bit 0 */
 #define MPU_MAILBOX_DB_HI_MASK	0x2	/* bit 1 */
 
-/********** MPU semphore ******************/
-#define MPU_EP_SEMAPHORE_OFFSET 0xac
-#define EP_SEMAPHORE_POST_STAGE_MASK 0x0000FFFF
-#define EP_SEMAPHORE_POST_ERR_MASK 0x1
-#define EP_SEMAPHORE_POST_ERR_SHIFT 31
+/********** MPU semphore: used for SH & BE ******************/
+#define SLIPORT_SOFTRESET_OFFSET		0x5c	/* CSR BAR offset */
+#define SLIPORT_SEMAPHORE_OFFSET_BEx		0xac	/* CSR BAR offset */
+#define SLIPORT_SEMAPHORE_OFFSET_SH		0x94	/* PCI-CFG offset */
+#define POST_STAGE_MASK				0x0000FFFF
+#define POST_ERROR_BIT				0x80000000
+#define POST_ERR_RECOVERY_CODE_MASK		0xF000
+
+/* Soft Reset register masks */
+#define SLIPORT_SOFTRESET_SR_MASK		0x00000080	/* SR bit */
+
+/* MPU semphore POST stage values */
+#define POST_STAGE_AWAITING_HOST_RDY	0x1 /* FW awaiting goahead from host */
+#define POST_STAGE_HOST_RDY		0x2 /* Host has given go-ahed to FW */
+#define POST_STAGE_BE_RESET		0x3 /* Host wants to reset chip */
+#define POST_STAGE_ARMFW_RDY		0xC000 /* FW is done with POST */
+#define POST_STAGE_RECOVERABLE_ERR	0xE000 /* Recoverable err detected */
 
 /********** MCC door bell ************/
 #define DB_MCCQ_OFFSET 0x140
@@ -109,9 +122,6 @@ struct be_mcc_compl {
 /* Number of entries posted */
 #define DB_MCCQ_NUM_POSTED_SHIFT 16		/* bits 16 - 29 */
 
-/* MPU semphore POST stage values */
-#define POST_STAGE_ARMFW_RDY		0xc000	/* FW is done with POST */
-
 /**
  * When the async bit of mcc_compl is set, the last 4 bytes of
  * mcc_compl is interpreted as follows:
@@ -217,6 +227,7 @@ struct be_mcc_mailbox {
 #define OPCODE_COMMON_QUERY_FIRMWARE_CONFIG		58
 #define OPCODE_COMMON_FUNCTION_RESET			61
 #define OPCODE_COMMON_GET_PORT_NAME			77
+#define OPCODE_COMMON_SET_FEATURES			191
 
 /**
  * LIST of opcodes that are common between Initiator and Target
@@ -345,8 +356,8 @@ struct be_cmd_req_logout_fw_sess {
 
 struct be_cmd_resp_logout_fw_sess {
 	struct be_cmd_resp_hdr hdr;	/* dw[4] */
-#define BEISCSI_MGMT_SESSION_CLOSE 0x20
 	uint32_t session_status;
+#define BE_SESS_STATUS_CLOSE		0x20
 } __packed;
 
 struct mgmt_conn_login_options {
@@ -365,6 +376,14 @@ struct ip_addr_format {
 	u16 size_of_structure;
 	u8 reserved;
 	u8 ip_type;
+#define BEISCSI_IP_TYPE_V4		0x1
+#define BEISCSI_IP_TYPE_STATIC_V4	0x3
+#define BEISCSI_IP_TYPE_DHCP_V4		0x5
+/* type v4 values < type v6 values */
+#define BEISCSI_IP_TYPE_V6		0x10
+#define BEISCSI_IP_TYPE_ROUTABLE_V6	0x30
+#define BEISCSI_IP_TYPE_LINK_LOCAL_V6	0x50
+#define BEISCSI_IP_TYPE_AUTO_V6		0x90
 	u8 addr[16];
 	u32 rsvd0;
 } __packed;
@@ -430,8 +449,13 @@ struct be_cmd_get_boot_target_req {
 
 struct be_cmd_get_boot_target_resp {
 	struct be_cmd_resp_hdr hdr;
-	u32  boot_session_count;
-	int  boot_session_handle;
+	u32 boot_session_count;
+	u32 boot_session_handle;
+/**
+ * FW returns 0xffffffff if it couldn't establish connection with
+ * configured boot target.
+ */
+#define BE_BOOT_INVALID_SHANDLE	0xffffffff
 };
 
 struct be_cmd_reopen_session_req {
@@ -699,16 +723,59 @@ struct be_cmd_get_nic_conf_resp {
 	u8 mac_address[ETH_ALEN];
 } __packed;
 
-#define BEISCSI_ALIAS_LEN 32
+/******************** Get HBA NAME *******************/
 
 struct be_cmd_hba_name {
 	struct be_cmd_req_hdr hdr;
 	u16 flags;
 	u16 rsvd0;
 	u8 initiator_name[ISCSI_NAME_LEN];
-	u8 initiator_alias[BEISCSI_ALIAS_LEN];
+#define BE_INI_ALIAS_LEN 32
+	u8 initiator_alias[BE_INI_ALIAS_LEN];
 } __packed;
 
+/******************** COMMON SET Features *******************/
+#define BE_CMD_SET_FEATURE_UER	0x10
+#define BE_CMD_UER_SUPP_BIT	0x1
+struct be_uer_req {
+	u32 uer;
+	u32 rsvd;
+};
+
+struct be_uer_resp {
+	u32 uer;
+	u16 ue2rp;
+	u16 ue2sr;
+};
+
+struct be_cmd_set_features {
+	union {
+		struct be_cmd_req_hdr req_hdr;
+		struct be_cmd_resp_hdr resp_hdr;
+	} h;
+	u32 feature;
+	u32 param_len;
+	union {
+		struct be_uer_req req;
+		struct be_uer_resp resp;
+		u32 rsvd[2];
+	} param;
+} __packed;
+
+int beiscsi_cmd_function_reset(struct beiscsi_hba *phba);
+
+int beiscsi_cmd_special_wrb(struct be_ctrl_info *ctrl, u32 load);
+
+int beiscsi_check_fw_rdy(struct beiscsi_hba *phba);
+
+int beiscsi_init_sliport(struct beiscsi_hba *phba);
+
+int beiscsi_cmd_iscsi_cleanup(struct beiscsi_hba *phba, unsigned short ulp_num);
+
+int beiscsi_detect_ue(struct beiscsi_hba *phba);
+
+int beiscsi_detect_tpe(struct beiscsi_hba *phba);
+
 int beiscsi_cmd_eq_create(struct be_ctrl_info *ctrl,
 			  struct be_queue_info *eq, int eq_delay);
 
@@ -723,24 +790,21 @@ int beiscsi_cmd_mccq_create(struct beiscsi_hba *phba,
 			struct be_queue_info *mccq,
 			struct be_queue_info *cq);
 
-int be_poll_mcc(struct be_ctrl_info *ctrl);
-int mgmt_check_supported_fw(struct be_ctrl_info *ctrl,
-				      struct beiscsi_hba *phba);
 unsigned int be_cmd_get_initname(struct beiscsi_hba *phba);
 
 void free_mcc_wrb(struct be_ctrl_info *ctrl, unsigned int tag);
 
-int be_cmd_modify_eq_delay(struct beiscsi_hba *phba, struct be_set_eqd *,
+int beiscsi_modify_eq_delay(struct beiscsi_hba *phba, struct be_set_eqd *,
 			    int num);
 int beiscsi_mccq_compl_wait(struct beiscsi_hba *phba,
-			    uint32_t tag, struct be_mcc_wrb **wrb,
+			    unsigned int tag,
+			    struct be_mcc_wrb **wrb,
 			    struct be_dma_mem *mbx_cmd_mem);
-/*ISCSI Functuions */
-int be_cmd_fw_initialize(struct be_ctrl_info *ctrl);
-int be_cmd_fw_uninit(struct be_ctrl_info *ctrl);
-
+int __beiscsi_mcc_compl_status(struct beiscsi_hba *phba,
+			       unsigned int tag,
+			       struct be_mcc_wrb **wrb,
+			       struct be_dma_mem *mbx_cmd_mem);
 struct be_mcc_wrb *wrb_from_mbox(struct be_dma_mem *mbox_mem);
-int be_mcc_compl_poll(struct beiscsi_hba *phba, unsigned int tag);
 void be_mcc_notify(struct beiscsi_hba *phba, unsigned int tag);
 struct be_mcc_wrb *alloc_mcc_wrb(struct beiscsi_hba *phba,
 				 unsigned int *ref_tag);
@@ -749,9 +813,6 @@ void beiscsi_process_async_event(struct beiscsi_hba *phba,
 int beiscsi_process_mcc_compl(struct be_ctrl_info *ctrl,
 			      struct be_mcc_compl *compl);
 
-
-int be_mbox_notify(struct be_ctrl_info *ctrl);
-
 int be_cmd_create_default_pdu_queue(struct be_ctrl_info *ctrl,
 				    struct be_queue_info *cq,
 				    struct be_queue_info *dq, int length,
@@ -767,8 +828,6 @@ int be_cmd_iscsi_post_sgl_pages(struct be_ctrl_info *ctrl,
 				struct be_dma_mem *q_mem, u32 page_offset,
 				u32 num_pages);
 
-int beiscsi_cmd_reset_function(struct beiscsi_hba *phba);
-
 int be_cmd_wrbq_create(struct be_ctrl_info *ctrl, struct be_dma_mem *q_mem,
 		       struct be_queue_info *wrbq,
 		       struct hwi_wrb_context *pwrb_context,
@@ -777,6 +836,15 @@ int be_cmd_wrbq_create(struct be_ctrl_info *ctrl, struct be_dma_mem *q_mem,
 /* Configuration Functions */
 int be_cmd_set_vlan(struct beiscsi_hba *phba, uint16_t vlan_tag);
 
+int beiscsi_check_supported_fw(struct be_ctrl_info *ctrl,
+			       struct beiscsi_hba *phba);
+
+int beiscsi_get_fw_config(struct be_ctrl_info *ctrl, struct beiscsi_hba *phba);
+
+int beiscsi_get_port_name(struct be_ctrl_info *ctrl, struct beiscsi_hba *phba);
+
+int beiscsi_set_uer_feature(struct beiscsi_hba *phba);
+
 struct be_default_pdu_context {
 	u32 dw[4];
 } __packed;
@@ -999,7 +1067,16 @@ struct iscsi_cleanup_req {
 	u16 chute;
 	u8 hdr_ring_id;
 	u8 data_ring_id;
+} __packed;
 
+struct iscsi_cleanup_req_v1 {
+	struct be_cmd_req_hdr hdr;
+	u16 chute;
+	u16 rsvd1;
+	u16 hdr_ring_id;
+	u16 rsvd2;
+	u16 data_ring_id;
+	u16 rsvd3;
 } __packed;
 
 struct eq_delay {
@@ -1368,14 +1445,9 @@ struct be_cmd_get_port_name {
 						 * the cxn
 						 */
 
-int beiscsi_pci_soft_reset(struct beiscsi_hba *phba);
-int be_chk_reset_complete(struct beiscsi_hba *phba);
-
 void be_wrb_hdr_prepare(struct be_mcc_wrb *wrb, int payload_len,
 			bool embedded, u8 sge_cnt);
 
 void be_cmd_hdr_prepare(struct be_cmd_req_hdr *req_hdr,
 			u8 subsystem, u8 opcode, int cmd_len);
-
-void beiscsi_fail_session(struct iscsi_cls_session *cls_session);
 #endif /* !BEISCSI_CMDS_H */
diff --git a/drivers/scsi/be2iscsi/be_iscsi.c b/drivers/scsi/be2iscsi/be_iscsi.c
index 09f89a3eaa87..ba258217614e 100644
--- a/drivers/scsi/be2iscsi/be_iscsi.c
+++ b/drivers/scsi/be2iscsi/be_iscsi.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -7,10 +7,10 @@
  * as published by the Free Software Foundation.  The full GNU General
  * Public License is included in this distribution in the file called COPYING.
  *
- * Written by: Jayamohan Kallickal (jayamohan.kallickal@avagotech.com)
+ * Written by: Jayamohan Kallickal (jayamohan.kallickal@broadcom.com)
  *
  * Contact Information:
- * linux-drivers@avagotech.com
+ * linux-drivers@broadcom.com
  *
  * Emulex
  * 3333 Susan Street
@@ -52,22 +52,20 @@ struct iscsi_cls_session *beiscsi_session_create(struct iscsi_endpoint *ep,
 
 
 	if (!ep) {
-		printk(KERN_ERR
-		       "beiscsi_session_create: invalid ep\n");
+		pr_err("beiscsi_session_create: invalid ep\n");
 		return NULL;
 	}
 	beiscsi_ep = ep->dd_data;
 	phba = beiscsi_ep->phba;
 
-	if (phba->state & BE_ADAPTER_PCI_ERR) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : PCI_ERROR Recovery\n");
-		return NULL;
-	} else {
+	if (!beiscsi_hba_is_online(phba)) {
 		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
-			    "BS_%d : In beiscsi_session_create\n");
+			    "BS_%d : HBA in error 0x%lx\n", phba->state);
+		return NULL;
 	}
 
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_session_create\n");
 	if (cmds_max > beiscsi_ep->phba->params.wrbs_per_cxn) {
 		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
 			    "BS_%d : Cannot handle %d cmds."
@@ -120,6 +118,16 @@ void beiscsi_session_destroy(struct iscsi_cls_session *cls_session)
 }
 
 /**
+ * beiscsi_session_fail(): Closing session with appropriate error
+ * @cls_session: ptr to session
+ **/
+void beiscsi_session_fail(struct iscsi_cls_session *cls_session)
+{
+	iscsi_session_failure(cls_session->dd_data, ISCSI_ERR_CONN_FAILED);
+}
+
+
+/**
  * beiscsi_conn_create - create an instance of iscsi connection
  * @cls_session: ptr to iscsi_cls_session
  * @cid: iscsi cid
@@ -237,7 +245,7 @@ int beiscsi_conn_bind(struct iscsi_cls_session *cls_session,
 	return beiscsi_bindconn_cid(phba, beiscsi_conn, beiscsi_ep->ep_cid);
 }
 
-static int beiscsi_create_ipv4_iface(struct beiscsi_hba *phba)
+static int beiscsi_iface_create_ipv4(struct beiscsi_hba *phba)
 {
 	if (phba->ipv4_iface)
 		return 0;
@@ -256,7 +264,7 @@ static int beiscsi_create_ipv4_iface(struct beiscsi_hba *phba)
 	return 0;
 }
 
-static int beiscsi_create_ipv6_iface(struct beiscsi_hba *phba)
+static int beiscsi_iface_create_ipv6(struct beiscsi_hba *phba)
 {
 	if (phba->ipv6_iface)
 		return 0;
@@ -275,79 +283,31 @@ static int beiscsi_create_ipv6_iface(struct beiscsi_hba *phba)
 	return 0;
 }
 
-void beiscsi_create_def_ifaces(struct beiscsi_hba *phba)
+void beiscsi_iface_create_default(struct beiscsi_hba *phba)
 {
 	struct be_cmd_get_if_info_resp *if_info;
 
-	if (!mgmt_get_if_info(phba, BE2_IPV4, &if_info)) {
-		beiscsi_create_ipv4_iface(phba);
+	if (!beiscsi_if_get_info(phba, BEISCSI_IP_TYPE_V4, &if_info)) {
+		beiscsi_iface_create_ipv4(phba);
 		kfree(if_info);
 	}
 
-	if (!mgmt_get_if_info(phba, BE2_IPV6, &if_info)) {
-		beiscsi_create_ipv6_iface(phba);
+	if (!beiscsi_if_get_info(phba, BEISCSI_IP_TYPE_V6, &if_info)) {
+		beiscsi_iface_create_ipv6(phba);
 		kfree(if_info);
 	}
 }
 
-void beiscsi_destroy_def_ifaces(struct beiscsi_hba *phba)
+void beiscsi_iface_destroy_default(struct beiscsi_hba *phba)
 {
-	if (phba->ipv6_iface)
+	if (phba->ipv6_iface) {
 		iscsi_destroy_iface(phba->ipv6_iface);
-	if (phba->ipv4_iface)
-		iscsi_destroy_iface(phba->ipv4_iface);
-}
-
-static int
-beiscsi_set_static_ip(struct Scsi_Host *shost,
-		struct iscsi_iface_param_info *iface_param,
-		void *data, uint32_t dt_len)
-{
-	struct beiscsi_hba *phba = iscsi_host_priv(shost);
-	struct iscsi_iface_param_info *iface_ip = NULL;
-	struct iscsi_iface_param_info *iface_subnet = NULL;
-	struct nlattr *nla;
-	int ret;
-
-
-	switch (iface_param->param) {
-	case ISCSI_NET_PARAM_IPV4_BOOTPROTO:
-		nla = nla_find(data, dt_len, ISCSI_NET_PARAM_IPV4_ADDR);
-		if (nla)
-			iface_ip = nla_data(nla);
-
-		nla = nla_find(data, dt_len, ISCSI_NET_PARAM_IPV4_SUBNET);
-		if (nla)
-			iface_subnet = nla_data(nla);
-		break;
-	case ISCSI_NET_PARAM_IPV4_ADDR:
-		iface_ip = iface_param;
-		nla = nla_find(data, dt_len, ISCSI_NET_PARAM_IPV4_SUBNET);
-		if (nla)
-			iface_subnet = nla_data(nla);
-		break;
-	case ISCSI_NET_PARAM_IPV4_SUBNET:
-		iface_subnet = iface_param;
-		nla = nla_find(data, dt_len, ISCSI_NET_PARAM_IPV4_ADDR);
-		if (nla)
-			iface_ip = nla_data(nla);
-		break;
-	default:
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : Unsupported param %d\n",
-			    iface_param->param);
+		phba->ipv6_iface = NULL;
 	}
-
-	if (!iface_ip || !iface_subnet) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : IP and Subnet Mask required\n");
-		return -EINVAL;
+	if (phba->ipv4_iface) {
+		iscsi_destroy_iface(phba->ipv4_iface);
+		phba->ipv4_iface = NULL;
 	}
-
-	ret = mgmt_set_ip(phba, iface_ip, iface_subnet,
-			ISCSI_BOOTPROTO_STATIC);
-
-	return ret;
 }
 
 /**
@@ -363,137 +323,141 @@ beiscsi_set_static_ip(struct Scsi_Host *shost,
  *	Failure: Non-Zero Value
  **/
 static int
-beiscsi_set_vlan_tag(struct Scsi_Host *shost,
-		      struct iscsi_iface_param_info *iface_param)
+beiscsi_iface_config_vlan(struct Scsi_Host *shost,
+			  struct iscsi_iface_param_info *iface_param)
 {
 	struct beiscsi_hba *phba = iscsi_host_priv(shost);
-	int ret;
-
-	/* Get the Interface Handle */
-	ret = mgmt_get_all_if_id(phba);
-	if (ret) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : Getting Interface Handle Failed\n");
-		return ret;
-	}
+	int ret = -EPERM;
 
 	switch (iface_param->param) {
 	case ISCSI_NET_PARAM_VLAN_ENABLED:
+		ret = 0;
 		if (iface_param->value[0] != ISCSI_VLAN_ENABLE)
-			ret = mgmt_set_vlan(phba, BEISCSI_VLAN_DISABLE);
+			ret = beiscsi_if_set_vlan(phba, BEISCSI_VLAN_DISABLE);
 		break;
 	case ISCSI_NET_PARAM_VLAN_TAG:
-		ret = mgmt_set_vlan(phba,
-				    *((uint16_t *)iface_param->value));
+		ret = beiscsi_if_set_vlan(phba,
+					  *((uint16_t *)iface_param->value));
 		break;
-	default:
-		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
-			    "BS_%d : Unknown Param Type : %d\n",
-			    iface_param->param);
-		return -ENOSYS;
 	}
 	return ret;
 }
 
 
 static int
-beiscsi_set_ipv4(struct Scsi_Host *shost,
-		struct iscsi_iface_param_info *iface_param,
-		void *data, uint32_t dt_len)
+beiscsi_iface_config_ipv4(struct Scsi_Host *shost,
+			  struct iscsi_iface_param_info *info,
+			  void *data, uint32_t dt_len)
 {
 	struct beiscsi_hba *phba = iscsi_host_priv(shost);
-	int ret = 0;
+	u8 *ip = NULL, *subnet = NULL, *gw;
+	struct nlattr *nla;
+	int ret = -EPERM;
 
 	/* Check the param */
-	switch (iface_param->param) {
+	switch (info->param) {
+	case ISCSI_NET_PARAM_IFACE_ENABLE:
+		if (info->value[0] == ISCSI_IFACE_ENABLE)
+			ret = beiscsi_iface_create_ipv4(phba);
+		else {
+			iscsi_destroy_iface(phba->ipv4_iface);
+			phba->ipv4_iface = NULL;
+		}
+		break;
 	case ISCSI_NET_PARAM_IPV4_GW:
-		ret = mgmt_set_gateway(phba, iface_param);
+		gw = info->value;
+		ret = beiscsi_if_set_gw(phba, BEISCSI_IP_TYPE_V4, gw);
 		break;
 	case ISCSI_NET_PARAM_IPV4_BOOTPROTO:
-		if (iface_param->value[0] == ISCSI_BOOTPROTO_DHCP)
-			ret = mgmt_set_ip(phba, iface_param,
-					NULL, ISCSI_BOOTPROTO_DHCP);
-		else if (iface_param->value[0] == ISCSI_BOOTPROTO_STATIC)
-			ret = beiscsi_set_static_ip(shost, iface_param,
-						    data, dt_len);
+		if (info->value[0] == ISCSI_BOOTPROTO_DHCP)
+			ret = beiscsi_if_en_dhcp(phba, BEISCSI_IP_TYPE_V4);
+		else if (info->value[0] == ISCSI_BOOTPROTO_STATIC)
+			/* release DHCP IP address */
+			ret = beiscsi_if_en_static(phba, BEISCSI_IP_TYPE_V4,
+						   NULL, NULL);
 		else
 			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
 				    "BS_%d : Invalid BOOTPROTO: %d\n",
-				    iface_param->value[0]);
+				    info->value[0]);
 		break;
-	case ISCSI_NET_PARAM_IFACE_ENABLE:
-		if (iface_param->value[0] == ISCSI_IFACE_ENABLE)
-			ret = beiscsi_create_ipv4_iface(phba);
-		else
-			iscsi_destroy_iface(phba->ipv4_iface);
-		break;
-	case ISCSI_NET_PARAM_IPV4_SUBNET:
 	case ISCSI_NET_PARAM_IPV4_ADDR:
-		ret = beiscsi_set_static_ip(shost, iface_param,
-					    data, dt_len);
+		ip = info->value;
+		nla = nla_find(data, dt_len, ISCSI_NET_PARAM_IPV4_SUBNET);
+		if (nla) {
+			info = nla_data(nla);
+			subnet = info->value;
+		}
+		ret = beiscsi_if_en_static(phba, BEISCSI_IP_TYPE_V4,
+					   ip, subnet);
 		break;
-	case ISCSI_NET_PARAM_VLAN_ENABLED:
-	case ISCSI_NET_PARAM_VLAN_TAG:
-		ret = beiscsi_set_vlan_tag(shost, iface_param);
+	case ISCSI_NET_PARAM_IPV4_SUBNET:
+		/*
+		 * OPCODE_COMMON_ISCSI_NTWK_MODIFY_IP_ADDR ioctl needs IP
+		 * and subnet both. Find IP to be applied for this subnet.
+		 */
+		subnet = info->value;
+		nla = nla_find(data, dt_len, ISCSI_NET_PARAM_IPV4_ADDR);
+		if (nla) {
+			info = nla_data(nla);
+			ip = info->value;
+		}
+		ret = beiscsi_if_en_static(phba, BEISCSI_IP_TYPE_V4,
+					   ip, subnet);
 		break;
-	default:
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : Param %d not supported\n",
-			    iface_param->param);
 	}
 
 	return ret;
 }
 
 static int
-beiscsi_set_ipv6(struct Scsi_Host *shost,
-		struct iscsi_iface_param_info *iface_param,
-		void *data, uint32_t dt_len)
+beiscsi_iface_config_ipv6(struct Scsi_Host *shost,
+			  struct iscsi_iface_param_info *iface_param,
+			  void *data, uint32_t dt_len)
 {
 	struct beiscsi_hba *phba = iscsi_host_priv(shost);
-	int ret = 0;
+	int ret = -EPERM;
 
 	switch (iface_param->param) {
 	case ISCSI_NET_PARAM_IFACE_ENABLE:
 		if (iface_param->value[0] == ISCSI_IFACE_ENABLE)
-			ret = beiscsi_create_ipv6_iface(phba);
+			ret = beiscsi_iface_create_ipv6(phba);
 		else {
 			iscsi_destroy_iface(phba->ipv6_iface);
-			ret = 0;
+			phba->ipv6_iface = NULL;
 		}
 		break;
 	case ISCSI_NET_PARAM_IPV6_ADDR:
-		ret = mgmt_set_ip(phba, iface_param, NULL,
-				  ISCSI_BOOTPROTO_STATIC);
+		ret = beiscsi_if_en_static(phba, BEISCSI_IP_TYPE_V6,
+					   iface_param->value, NULL);
 		break;
-	case ISCSI_NET_PARAM_VLAN_ENABLED:
-	case ISCSI_NET_PARAM_VLAN_TAG:
-		ret = beiscsi_set_vlan_tag(shost, iface_param);
-		break;
-	default:
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : Param %d not supported\n",
-			    iface_param->param);
 	}
 
 	return ret;
 }
 
-int be2iscsi_iface_set_param(struct Scsi_Host *shost,
-		void *data, uint32_t dt_len)
+int beiscsi_iface_set_param(struct Scsi_Host *shost,
+			    void *data, uint32_t dt_len)
 {
 	struct iscsi_iface_param_info *iface_param = NULL;
 	struct beiscsi_hba *phba = iscsi_host_priv(shost);
 	struct nlattr *attrib;
 	uint32_t rm_len = dt_len;
-	int ret = 0 ;
+	int ret;
 
-	if (phba->state & BE_ADAPTER_PCI_ERR) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : In PCI_ERROR Recovery\n");
+	if (!beiscsi_hba_is_online(phba)) {
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : HBA in error 0x%lx\n", phba->state);
 		return -EBUSY;
 	}
 
+	/* update interface_handle */
+	ret = beiscsi_if_get_handle(phba);
+	if (ret) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Getting Interface Handle Failed\n");
+		return ret;
+	}
+
 	nla_for_each_attr(attrib, data, dt_len, rm_len) {
 		iface_param = nla_data(attrib);
 
@@ -512,40 +476,58 @@ int be2iscsi_iface_set_param(struct Scsi_Host *shost,
 			return -EINVAL;
 		}
 
-		switch (iface_param->iface_type) {
-		case ISCSI_IFACE_TYPE_IPV4:
-			ret = beiscsi_set_ipv4(shost, iface_param,
-					       data, dt_len);
-			break;
-		case ISCSI_IFACE_TYPE_IPV6:
-			ret = beiscsi_set_ipv6(shost, iface_param,
-					       data, dt_len);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : %s.0 set param %d",
+			    (iface_param->iface_type == ISCSI_IFACE_TYPE_IPV4) ?
+			    "ipv4" : "ipv6", iface_param->param);
+
+		ret = -EPERM;
+		switch (iface_param->param) {
+		case ISCSI_NET_PARAM_VLAN_ENABLED:
+		case ISCSI_NET_PARAM_VLAN_TAG:
+			ret = beiscsi_iface_config_vlan(shost, iface_param);
 			break;
 		default:
-			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-				    "BS_%d : Invalid iface type :%d passed\n",
-				    iface_param->iface_type);
-			break;
+			switch (iface_param->iface_type) {
+			case ISCSI_IFACE_TYPE_IPV4:
+				ret = beiscsi_iface_config_ipv4(shost,
+								iface_param,
+								data, dt_len);
+				break;
+			case ISCSI_IFACE_TYPE_IPV6:
+				ret = beiscsi_iface_config_ipv6(shost,
+								iface_param,
+								data, dt_len);
+				break;
+			}
 		}
 
+		if (ret == -EPERM) {
+			__beiscsi_log(phba, KERN_ERR,
+				      "BS_%d : %s.0 set param %d not permitted",
+				      (iface_param->iface_type ==
+				       ISCSI_IFACE_TYPE_IPV4) ? "ipv4" : "ipv6",
+				      iface_param->param);
+			ret = 0;
+		}
 		if (ret)
-			return ret;
+			break;
 	}
 
 	return ret;
 }
 
-static int be2iscsi_get_if_param(struct beiscsi_hba *phba,
-		struct iscsi_iface *iface, int param,
-		char *buf)
+static int __beiscsi_iface_get_param(struct beiscsi_hba *phba,
+				     struct iscsi_iface *iface,
+				     int param, char *buf)
 {
 	struct be_cmd_get_if_info_resp *if_info;
-	int len, ip_type = BE2_IPV4;
+	int len, ip_type = BEISCSI_IP_TYPE_V4;
 
 	if (iface->iface_type == ISCSI_IFACE_TYPE_IPV6)
-		ip_type = BE2_IPV6;
+		ip_type = BEISCSI_IP_TYPE_V6;
 
-	len = mgmt_get_if_info(phba, ip_type, &if_info);
+	len = beiscsi_if_get_info(phba, ip_type, &if_info);
 	if (len)
 		return len;
 
@@ -567,24 +549,24 @@ static int be2iscsi_get_if_param(struct beiscsi_hba *phba,
 		break;
 	case ISCSI_NET_PARAM_VLAN_ENABLED:
 		len = sprintf(buf, "%s\n",
-			     (if_info->vlan_priority == BEISCSI_VLAN_DISABLE)
-			     ? "Disabled\n" : "Enabled\n");
+			      (if_info->vlan_priority == BEISCSI_VLAN_DISABLE) ?
+			      "disable" : "enable");
 		break;
 	case ISCSI_NET_PARAM_VLAN_ID:
 		if (if_info->vlan_priority == BEISCSI_VLAN_DISABLE)
 			len = -EINVAL;
 		else
 			len = sprintf(buf, "%d\n",
-				     (if_info->vlan_priority &
-				     ISCSI_MAX_VLAN_ID));
+				      (if_info->vlan_priority &
+				       ISCSI_MAX_VLAN_ID));
 		break;
 	case ISCSI_NET_PARAM_VLAN_PRIORITY:
 		if (if_info->vlan_priority == BEISCSI_VLAN_DISABLE)
 			len = -EINVAL;
 		else
 			len = sprintf(buf, "%d\n",
-				     ((if_info->vlan_priority >> 13) &
-				     ISCSI_MAX_VLAN_PRIORITY));
+				      ((if_info->vlan_priority >> 13) &
+				       ISCSI_MAX_VLAN_PRIORITY));
 		break;
 	default:
 		WARN_ON(1);
@@ -594,18 +576,20 @@ static int be2iscsi_get_if_param(struct beiscsi_hba *phba,
 	return len;
 }
 
-int be2iscsi_iface_get_param(struct iscsi_iface *iface,
-		enum iscsi_param_type param_type,
-		int param, char *buf)
+int beiscsi_iface_get_param(struct iscsi_iface *iface,
+			    enum iscsi_param_type param_type,
+			    int param, char *buf)
 {
 	struct Scsi_Host *shost = iscsi_iface_to_shost(iface);
 	struct beiscsi_hba *phba = iscsi_host_priv(shost);
 	struct be_cmd_get_def_gateway_resp gateway;
-	int len = -ENOSYS;
+	int len = -EPERM;
 
-	if (phba->state & BE_ADAPTER_PCI_ERR) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : In PCI_ERROR Recovery\n");
+	if (param_type != ISCSI_NET_PARAM)
+		return 0;
+	if (!beiscsi_hba_is_online(phba)) {
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : HBA in error 0x%lx\n", phba->state);
 		return -EBUSY;
 	}
 
@@ -617,19 +601,22 @@ int be2iscsi_iface_get_param(struct iscsi_iface *iface,
 	case ISCSI_NET_PARAM_VLAN_ENABLED:
 	case ISCSI_NET_PARAM_VLAN_ID:
 	case ISCSI_NET_PARAM_VLAN_PRIORITY:
-		len = be2iscsi_get_if_param(phba, iface, param, buf);
+		len = __beiscsi_iface_get_param(phba, iface, param, buf);
 		break;
 	case ISCSI_NET_PARAM_IFACE_ENABLE:
-		len = sprintf(buf, "enabled\n");
+		if (iface->iface_type == ISCSI_IFACE_TYPE_IPV4)
+			len = sprintf(buf, "%s\n",
+				      phba->ipv4_iface ? "enable" : "disable");
+		else if (iface->iface_type == ISCSI_IFACE_TYPE_IPV6)
+			len = sprintf(buf, "%s\n",
+				      phba->ipv6_iface ? "enable" : "disable");
 		break;
 	case ISCSI_NET_PARAM_IPV4_GW:
 		memset(&gateway, 0, sizeof(gateway));
-		len = mgmt_get_gateway(phba, BE2_IPV4, &gateway);
+		len = beiscsi_if_get_gw(phba, BEISCSI_IP_TYPE_V4, &gateway);
 		if (!len)
 			len = sprintf(buf, "%pI4\n", &gateway.ip_addr.addr);
 		break;
-	default:
-		len = -ENOSYS;
 	}
 
 	return len;
@@ -647,7 +634,7 @@ int beiscsi_ep_get_param(struct iscsi_endpoint *ep,
 			   enum iscsi_param param, char *buf)
 {
 	struct beiscsi_endpoint *beiscsi_ep = ep->dd_data;
-	int len = 0;
+	int len;
 
 	beiscsi_log(beiscsi_ep->phba, KERN_INFO,
 		    BEISCSI_LOG_CONFIG,
@@ -659,13 +646,13 @@ int beiscsi_ep_get_param(struct iscsi_endpoint *ep,
 		len = sprintf(buf, "%hu\n", beiscsi_ep->dst_tcpport);
 		break;
 	case ISCSI_PARAM_CONN_ADDRESS:
-		if (beiscsi_ep->ip_type == BE2_IPV4)
+		if (beiscsi_ep->ip_type == BEISCSI_IP_TYPE_V4)
 			len = sprintf(buf, "%pI4\n", &beiscsi_ep->dst_addr);
 		else
 			len = sprintf(buf, "%pI6\n", &beiscsi_ep->dst6_addr);
 		break;
 	default:
-		return -ENOSYS;
+		len = -EPERM;
 	}
 	return len;
 }
@@ -758,7 +745,7 @@ static void beiscsi_get_port_state(struct Scsi_Host *shost)
 	struct beiscsi_hba *phba = iscsi_host_priv(shost);
 	struct iscsi_cls_host *ihost = shost->shost_data;
 
-	ihost->port_state = (phba->state & BE_ADAPTER_LINK_UP) ?
+	ihost->port_state = test_bit(BEISCSI_HBA_LINK_UP, &phba->state) ?
 		ISCSI_PORT_STATE_UP : ISCSI_PORT_STATE_DOWN;
 }
 
@@ -810,16 +797,13 @@ int beiscsi_get_host_param(struct Scsi_Host *shost,
 	struct beiscsi_hba *phba = iscsi_host_priv(shost);
 	int status = 0;
 
-
-	if (phba->state & BE_ADAPTER_PCI_ERR) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : In PCI_ERROR Recovery\n");
-		return -EBUSY;
-	} else {
+	if (!beiscsi_hba_is_online(phba)) {
 		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
-			    "BS_%d : In beiscsi_get_host_param,"
-			    " param = %d\n", param);
+			    "BS_%d : HBA in error 0x%lx\n", phba->state);
+		return -EBUSY;
 	}
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_get_host_param, param = %d\n", param);
 
 	switch (param) {
 	case ISCSI_HOST_PARAM_HWADDRESS:
@@ -961,15 +945,13 @@ int beiscsi_conn_start(struct iscsi_cls_conn *cls_conn)
 
 	phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
 
-	if (phba->state & BE_ADAPTER_PCI_ERR) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : In PCI_ERROR Recovery\n");
+	if (!beiscsi_hba_is_online(phba)) {
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : HBA in error 0x%lx\n", phba->state);
 		return -EBUSY;
-	} else {
-		beiscsi_log(beiscsi_conn->phba, KERN_INFO,
-			    BEISCSI_LOG_CONFIG,
-			    "BS_%d : In beiscsi_conn_start\n");
 	}
+	beiscsi_log(beiscsi_conn->phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_conn_start\n");
 
 	memset(&params, 0, sizeof(struct beiscsi_offload_params));
 	beiscsi_ep = beiscsi_conn->ep;
@@ -1186,28 +1168,20 @@ beiscsi_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,
 	struct iscsi_endpoint *ep;
 	int ret;
 
-	if (shost)
-		phba = iscsi_host_priv(shost);
-	else {
+	if (!shost) {
 		ret = -ENXIO;
-		printk(KERN_ERR
-		       "beiscsi_ep_connect shost is NULL\n");
+		pr_err("beiscsi_ep_connect shost is NULL\n");
 		return ERR_PTR(ret);
 	}
 
-	if (beiscsi_error(phba)) {
+	phba = iscsi_host_priv(shost);
+	if (!beiscsi_hba_is_online(phba)) {
 		ret = -EIO;
-		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
-			    "BS_%d : The FW state Not Stable!!!\n");
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : HBA in error 0x%lx\n", phba->state);
 		return ERR_PTR(ret);
 	}
-
-	if (phba->state & BE_ADAPTER_PCI_ERR) {
-		ret = -EBUSY;
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : In PCI_ERROR Recovery\n");
-		return ERR_PTR(ret);
-	} else if (phba->state & BE_ADAPTER_LINK_DOWN) {
+	if (!test_bit(BEISCSI_HBA_LINK_UP, &phba->state)) {
 		ret = -EBUSY;
 		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
 			    "BS_%d : The Adapter Port state is Down!!!\n");
@@ -1361,9 +1335,9 @@ void beiscsi_ep_disconnect(struct iscsi_endpoint *ep)
 		tcp_upload_flag = CONNECTION_UPLOAD_ABORT;
 	}
 
-	if (phba->state & BE_ADAPTER_PCI_ERR) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
-			    "BS_%d : PCI_ERROR Recovery\n");
+	if (!beiscsi_hba_is_online(phba)) {
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : HBA in error 0x%lx\n", phba->state);
 		goto free_ep;
 	}
 
@@ -1386,7 +1360,7 @@ free_ep:
 	iscsi_destroy_endpoint(beiscsi_ep->openiscsi_ep);
 }
 
-umode_t be2iscsi_attr_is_visible(int param_type, int param)
+umode_t beiscsi_attr_is_visible(int param_type, int param)
 {
 	switch (param_type) {
 	case ISCSI_NET_PARAM:
diff --git a/drivers/scsi/be2iscsi/be_iscsi.h b/drivers/scsi/be2iscsi/be_iscsi.h
index 0c84e1c0763a..e4d67dfea4cb 100644
--- a/drivers/scsi/be2iscsi/be_iscsi.h
+++ b/drivers/scsi/be2iscsi/be_iscsi.h
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2005 - 2015 Avago Technologies
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -7,10 +7,10 @@
  * as published by the Free Software Foundation.  The full GNU General
  * Public License is included in this distribution in the file called COPYING.
  *
- * Written by: Jayamohan Kallickal (jayamohan.kallickal@avagotech.com)
+ * Written by: Jayamohan Kallickal (jayamohan.kallickal@broadcom.com)
  *
  * Contact Information:
- * linux-drivers@avagotech.com
+ * linux-drivers@broadcom.com
  *
  * Avago Technologies
  * 3333 Susan Street
@@ -23,25 +23,18 @@
 #include "be_main.h"
 #include "be_mgmt.h"
 
-#define BE2_IPV4  0x1
-#define BE2_IPV6  0x10
-#define BE2_DHCP_V4 0x05
+void beiscsi_iface_create_default(struct beiscsi_hba *phba);
 
-#define NON_BLOCKING 0x0
-#define BLOCKING 0x1
+void beiscsi_iface_destroy_default(struct beiscsi_hba *phba);
 
-void beiscsi_create_def_ifaces(struct beiscsi_hba *phba);
-
-void beiscsi_destroy_def_ifaces(struct beiscsi_hba *phba);
-
-int be2iscsi_iface_get_param(struct iscsi_iface *iface,
+int beiscsi_iface_get_param(struct iscsi_iface *iface,
 			     enum iscsi_param_type param_type,
 			     int param, char *buf);
 
-int be2iscsi_iface_set_param(struct Scsi_Host *shost,
+int beiscsi_iface_set_param(struct Scsi_Host *shost,
 			     void *data, uint32_t count);
 
-umode_t be2iscsi_attr_is_visible(int param_type, int param);
+umode_t beiscsi_attr_is_visible(int param_type, int param);
 
 void beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
 				struct beiscsi_offload_params *params);
@@ -57,6 +50,8 @@ struct iscsi_cls_session *beiscsi_session_create(struct iscsi_endpoint *ep,
 
 void beiscsi_session_destroy(struct iscsi_cls_session *cls_session);
 
+void beiscsi_session_fail(struct iscsi_cls_session *cls_session);
+
 struct iscsi_cls_conn *beiscsi_conn_create(struct iscsi_cls_session
 					   *cls_session, uint32_t cid);
 
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index f05e7737107d..6a6906f847db 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -7,10 +7,10 @@
  * as published by the Free Software Foundation.  The full GNU General
  * Public License is included in this distribution in the file called COPYING.
  *
- * Written by: Jayamohan Kallickal (jayamohan.kallickal@avagotech.com)
+ * Written by: Jayamohan Kallickal (jayamohan.kallickal@broadcom.com)
  *
  * Contact Information:
- * linux-drivers@avagotech.com
+ * linux-drivers@broadcom.com
  *
  * Emulex
  * 3333 Susan Street
@@ -374,170 +374,6 @@ static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 	return iscsi_eh_device_reset(sc);
 }
 
-static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
-{
-	struct beiscsi_hba *phba = data;
-	struct mgmt_session_info *boot_sess = &phba->boot_sess;
-	struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
-	char *str = buf;
-	int rc;
-
-	switch (type) {
-	case ISCSI_BOOT_TGT_NAME:
-		rc = sprintf(buf, "%.*s\n",
-			    (int)strlen(boot_sess->target_name),
-			    (char *)&boot_sess->target_name);
-		break;
-	case ISCSI_BOOT_TGT_IP_ADDR:
-		if (boot_conn->dest_ipaddr.ip_type == 0x1)
-			rc = sprintf(buf, "%pI4\n",
-				(char *)&boot_conn->dest_ipaddr.addr);
-		else
-			rc = sprintf(str, "%pI6\n",
-				(char *)&boot_conn->dest_ipaddr.addr);
-		break;
-	case ISCSI_BOOT_TGT_PORT:
-		rc = sprintf(str, "%d\n", boot_conn->dest_port);
-		break;
-
-	case ISCSI_BOOT_TGT_CHAP_NAME:
-		rc = sprintf(str,  "%.*s\n",
-			     boot_conn->negotiated_login_options.auth_data.chap.
-			     target_chap_name_length,
-			     (char *)&boot_conn->negotiated_login_options.
-			     auth_data.chap.target_chap_name);
-		break;
-	case ISCSI_BOOT_TGT_CHAP_SECRET:
-		rc = sprintf(str,  "%.*s\n",
-			     boot_conn->negotiated_login_options.auth_data.chap.
-			     target_secret_length,
-			     (char *)&boot_conn->negotiated_login_options.
-			     auth_data.chap.target_secret);
-		break;
-	case ISCSI_BOOT_TGT_REV_CHAP_NAME:
-		rc = sprintf(str,  "%.*s\n",
-			     boot_conn->negotiated_login_options.auth_data.chap.
-			     intr_chap_name_length,
-			     (char *)&boot_conn->negotiated_login_options.
-			     auth_data.chap.intr_chap_name);
-		break;
-	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
-		rc = sprintf(str,  "%.*s\n",
-			     boot_conn->negotiated_login_options.auth_data.chap.
-			     intr_secret_length,
-			     (char *)&boot_conn->negotiated_login_options.
-			     auth_data.chap.intr_secret);
-		break;
-	case ISCSI_BOOT_TGT_FLAGS:
-		rc = sprintf(str, "2\n");
-		break;
-	case ISCSI_BOOT_TGT_NIC_ASSOC:
-		rc = sprintf(str, "0\n");
-		break;
-	default:
-		rc = -ENOSYS;
-		break;
-	}
-	return rc;
-}
-
-static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
-{
-	struct beiscsi_hba *phba = data;
-	char *str = buf;
-	int rc;
-
-	switch (type) {
-	case ISCSI_BOOT_INI_INITIATOR_NAME:
-		rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
-		break;
-	default:
-		rc = -ENOSYS;
-		break;
-	}
-	return rc;
-}
-
-static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
-{
-	struct beiscsi_hba *phba = data;
-	char *str = buf;
-	int rc;
-
-	switch (type) {
-	case ISCSI_BOOT_ETH_FLAGS:
-		rc = sprintf(str, "2\n");
-		break;
-	case ISCSI_BOOT_ETH_INDEX:
-		rc = sprintf(str, "0\n");
-		break;
-	case ISCSI_BOOT_ETH_MAC:
-		rc  = beiscsi_get_macaddr(str, phba);
-		break;
-	default:
-		rc = -ENOSYS;
-		break;
-	}
-	return rc;
-}
-
-
-static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
-{
-	umode_t rc;
-
-	switch (type) {
-	case ISCSI_BOOT_TGT_NAME:
-	case ISCSI_BOOT_TGT_IP_ADDR:
-	case ISCSI_BOOT_TGT_PORT:
-	case ISCSI_BOOT_TGT_CHAP_NAME:
-	case ISCSI_BOOT_TGT_CHAP_SECRET:
-	case ISCSI_BOOT_TGT_REV_CHAP_NAME:
-	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
-	case ISCSI_BOOT_TGT_NIC_ASSOC:
-	case ISCSI_BOOT_TGT_FLAGS:
-		rc = S_IRUGO;
-		break;
-	default:
-		rc = 0;
-		break;
-	}
-	return rc;
-}
-
-static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
-{
-	umode_t rc;
-
-	switch (type) {
-	case ISCSI_BOOT_INI_INITIATOR_NAME:
-		rc = S_IRUGO;
-		break;
-	default:
-		rc = 0;
-		break;
-	}
-	return rc;
-}
-
-
-static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
-{
-	umode_t rc;
-
-	switch (type) {
-	case ISCSI_BOOT_ETH_FLAGS:
-	case ISCSI_BOOT_ETH_MAC:
-	case ISCSI_BOOT_ETH_INDEX:
-		rc = S_IRUGO;
-		break;
-	default:
-		rc = 0;
-		break;
-	}
-	return rc;
-}
-
 /*------------------- PCI Driver operations and data ----------------- */
 static const struct pci_device_id beiscsi_pci_id_table[] = {
 	{ PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
@@ -850,12 +686,11 @@ static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 {
 	struct beiscsi_hba *phba;
-	struct be_eq_entry *eqe = NULL;
+	struct be_eq_entry *eqe;
 	struct be_queue_info *eq;
 	struct be_queue_info *mcc;
-	unsigned int num_eq_processed;
+	unsigned int mcc_events;
 	struct be_eq_obj *pbe_eq;
-	unsigned long flags;
 
 	pbe_eq = dev_id;
 	eq = &pbe_eq->q;
@@ -863,27 +698,23 @@ static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 	mcc = &phba->ctrl.mcc_obj.cq;
 	eqe = queue_tail_node(eq);
 
-	num_eq_processed = 0;
-
+	mcc_events = 0;
 	while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 				& EQE_VALID_MASK) {
 		if (((eqe->dw[offsetof(struct amap_eq_entry,
 		     resource_id) / 32] &
 		     EQE_RESID_MASK) >> 16) == mcc->id) {
-			spin_lock_irqsave(&phba->isr_lock, flags);
-			pbe_eq->todo_mcc_cq = true;
-			spin_unlock_irqrestore(&phba->isr_lock, flags);
+			mcc_events++;
 		}
 		AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 		queue_tail_inc(eq);
 		eqe = queue_tail_node(eq);
-		num_eq_processed++;
 	}
-	if (pbe_eq->todo_mcc_cq)
-		queue_work(phba->wq, &pbe_eq->work_cqs);
-	if (num_eq_processed)
-		hwi_ring_eq_db(phba, eq->id, 1,	num_eq_processed, 1, 1);
 
+	if (mcc_events) {
+		queue_work(phba->wq, &pbe_eq->mcc_work);
+		hwi_ring_eq_db(phba, eq->id, 1,	mcc_events, 1, 1);
+	}
 	return IRQ_HANDLED;
 }
 
@@ -902,7 +733,6 @@ static irqreturn_t be_isr_msix(int irq, void *dev_id)
 	eq = &pbe_eq->q;
 
 	phba = pbe_eq->phba;
-
 	/* disable interrupt till iopoll completes */
 	hwi_ring_eq_db(phba, eq->id, 1,	0, 0, 1);
 	irq_poll_sched(&pbe_eq->iopoll);
@@ -920,14 +750,13 @@ static irqreturn_t be_isr(int irq, void *dev_id)
 	struct beiscsi_hba *phba;
 	struct hwi_controller *phwi_ctrlr;
 	struct hwi_context_memory *phwi_context;
-	struct be_eq_entry *eqe = NULL;
+	struct be_eq_entry *eqe;
 	struct be_queue_info *eq;
 	struct be_queue_info *mcc;
-	unsigned long flags, index;
-	unsigned int num_mcceq_processed, num_ioeq_processed;
+	unsigned int mcc_events, io_events;
 	struct be_ctrl_info *ctrl;
 	struct be_eq_obj *pbe_eq;
-	int isr;
+	int isr, rearm;
 
 	phba = dev_id;
 	ctrl = &phba->ctrl;
@@ -942,44 +771,35 @@ static irqreturn_t be_isr(int irq, void *dev_id)
 
 	eq = &phwi_context->be_eq[0].q;
 	mcc = &phba->ctrl.mcc_obj.cq;
-	index = 0;
 	eqe = queue_tail_node(eq);
 
-	num_ioeq_processed = 0;
-	num_mcceq_processed = 0;
+	io_events = 0;
+	mcc_events = 0;
 	while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 				& EQE_VALID_MASK) {
 		if (((eqe->dw[offsetof(struct amap_eq_entry,
-		     resource_id) / 32] &
-		     EQE_RESID_MASK) >> 16) == mcc->id) {
-			spin_lock_irqsave(&phba->isr_lock, flags);
-			pbe_eq->todo_mcc_cq = true;
-			spin_unlock_irqrestore(&phba->isr_lock, flags);
-			num_mcceq_processed++;
-		} else {
-			irq_poll_sched(&pbe_eq->iopoll);
-			num_ioeq_processed++;
-		}
+		      resource_id) / 32] & EQE_RESID_MASK) >> 16) == mcc->id)
+			mcc_events++;
+		else
+			io_events++;
 		AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 		queue_tail_inc(eq);
 		eqe = queue_tail_node(eq);
 	}
-	if (num_ioeq_processed || num_mcceq_processed) {
-		if (pbe_eq->todo_mcc_cq)
-			queue_work(phba->wq, &pbe_eq->work_cqs);
-
-		if ((num_mcceq_processed) && (!num_ioeq_processed))
-			hwi_ring_eq_db(phba, eq->id, 0,
-				      (num_ioeq_processed +
-				       num_mcceq_processed) , 1, 1);
-		else
-			hwi_ring_eq_db(phba, eq->id, 0,
-				       (num_ioeq_processed +
-					num_mcceq_processed), 0, 1);
-
-		return IRQ_HANDLED;
-	} else
+	if (!io_events && !mcc_events)
 		return IRQ_NONE;
+
+	/* no need to rearm if interrupt is only for IOs */
+	rearm = 0;
+	if (mcc_events) {
+		queue_work(phba->wq, &pbe_eq->mcc_work);
+		/* rearm for MCCQ */
+		rearm = 1;
+	}
+	if (io_events)
+		irq_poll_sched(&pbe_eq->iopoll);
+	hwi_ring_eq_db(phba, eq->id, 0, (io_events + mcc_events), rearm, 1);
+	return IRQ_HANDLED;
 }
 
 
@@ -1077,57 +897,6 @@ void hwi_ring_cq_db(struct beiscsi_hba *phba,
 	iowrite32(val, phba->db_va + DB_CQ_OFFSET);
 }
 
-static unsigned int
-beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
-			  struct beiscsi_hba *phba,
-			  struct pdu_base *ppdu,
-			  unsigned long pdu_len,
-			  void *pbuffer, unsigned long buf_len)
-{
-	struct iscsi_conn *conn = beiscsi_conn->conn;
-	struct iscsi_session *session = conn->session;
-	struct iscsi_task *task;
-	struct beiscsi_io_task *io_task;
-	struct iscsi_hdr *login_hdr;
-
-	switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
-						PDUBASE_OPCODE_MASK) {
-	case ISCSI_OP_NOOP_IN:
-		pbuffer = NULL;
-		buf_len = 0;
-		break;
-	case ISCSI_OP_ASYNC_EVENT:
-		break;
-	case ISCSI_OP_REJECT:
-		WARN_ON(!pbuffer);
-		WARN_ON(!(buf_len == 48));
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
-			    "BM_%d : In ISCSI_OP_REJECT\n");
-		break;
-	case ISCSI_OP_LOGIN_RSP:
-	case ISCSI_OP_TEXT_RSP:
-		task = conn->login_task;
-		io_task = task->dd_data;
-		login_hdr = (struct iscsi_hdr *)ppdu;
-		login_hdr->itt = io_task->libiscsi_itt;
-		break;
-	default:
-		beiscsi_log(phba, KERN_WARNING,
-			    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
-			    "BM_%d : Unrecognized opcode 0x%x in async msg\n",
-			    (ppdu->
-			     dw[offsetof(struct amap_pdu_base, opcode) / 32]
-			     & PDUBASE_OPCODE_MASK));
-		return 1;
-	}
-
-	spin_lock_bh(&session->back_lock);
-	__iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
-	spin_unlock_bh(&session->back_lock);
-	return 0;
-}
-
 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 {
 	struct sgl_handle *psgl_handle;
@@ -1199,6 +968,9 @@ beiscsi_get_wrb_handle(struct hwi_wrb_context *pwrb_context,
 		pwrb_context->alloc_index++;
 	spin_unlock_bh(&pwrb_context->wrb_lock);
 
+	if (pwrb_handle)
+		memset(pwrb_handle->pwrb, 0, sizeof(*pwrb_handle->pwrb));
+
 	return pwrb_handle;
 }
 
@@ -1440,11 +1212,10 @@ hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
 		       struct beiscsi_hba *phba, struct sol_cqe *psol)
 {
 	struct hwi_wrb_context *pwrb_context;
-	struct wrb_handle *pwrb_handle = NULL;
+	uint16_t wrb_index, cid, cri_index;
 	struct hwi_controller *phwi_ctrlr;
+	struct wrb_handle *pwrb_handle;
 	struct iscsi_task *task;
-	struct beiscsi_io_task *io_task;
-	uint16_t wrb_index, cid, cri_index;
 
 	phwi_ctrlr = phba->phwi_ctrlr;
 	if (is_chip_be2_be3r(phba)) {
@@ -1463,9 +1234,6 @@ hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
 	pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
 	pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];
 	task = pwrb_handle->pio_handle;
-
-	io_task = task->dd_data;
-	memset(io_task->pwrb_handle->pwrb, 0, sizeof(struct iscsi_wrb));
 	iscsi_put_task(task);
 }
 
@@ -1614,431 +1382,428 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
 	spin_unlock_bh(&session->back_lock);
 }
 
-static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
-					  *pasync_ctx, unsigned int is_header,
-					  unsigned int host_write_ptr)
+/**
+ * ASYNC PDUs include
+ * a. Unsolicited NOP-In (target initiated NOP-In)
+ * b. ASYNC Messages
+ * c. Reject PDU
+ * d. Login response
+ * These headers arrive unprocessed by the EP firmware.
+ * iSCSI layer processes them.
+ */
+static unsigned int
+beiscsi_complete_pdu(struct beiscsi_conn *beiscsi_conn,
+		struct pdu_base *phdr, void *pdata, unsigned int dlen)
 {
-	if (is_header)
-		return &pasync_ctx->async_entry[host_write_ptr].
-		    header_busy_list;
-	else
-		return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
+	struct beiscsi_hba *phba = beiscsi_conn->phba;
+	struct iscsi_conn *conn = beiscsi_conn->conn;
+	struct beiscsi_io_task *io_task;
+	struct iscsi_hdr *login_hdr;
+	struct iscsi_task *task;
+	u8 code;
+
+	code = AMAP_GET_BITS(struct amap_pdu_base, opcode, phdr);
+	switch (code) {
+	case ISCSI_OP_NOOP_IN:
+		pdata = NULL;
+		dlen = 0;
+		break;
+	case ISCSI_OP_ASYNC_EVENT:
+		break;
+	case ISCSI_OP_REJECT:
+		WARN_ON(!pdata);
+		WARN_ON(!(dlen == 48));
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
+			    "BM_%d : In ISCSI_OP_REJECT\n");
+		break;
+	case ISCSI_OP_LOGIN_RSP:
+	case ISCSI_OP_TEXT_RSP:
+		task = conn->login_task;
+		io_task = task->dd_data;
+		login_hdr = (struct iscsi_hdr *)phdr;
+		login_hdr->itt = io_task->libiscsi_itt;
+		break;
+	default:
+		beiscsi_log(phba, KERN_WARNING,
+			    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+			    "BM_%d : unrecognized async PDU opcode 0x%x\n",
+			    code);
+		return 1;
+	}
+	__iscsi_complete_pdu(conn, (struct iscsi_hdr *)phdr, pdata, dlen);
+	return 0;
+}
+
+static inline void
+beiscsi_hdl_put_handle(struct hd_async_context *pasync_ctx,
+			 struct hd_async_handle *pasync_handle)
+{
+	if (pasync_handle->is_header) {
+		list_add_tail(&pasync_handle->link,
+				&pasync_ctx->async_header.free_list);
+		pasync_ctx->async_header.free_entries++;
+	} else {
+		list_add_tail(&pasync_handle->link,
+				&pasync_ctx->async_data.free_list);
+		pasync_ctx->async_data.free_entries++;
+	}
 }
 
-static struct async_pdu_handle *
-hwi_get_async_handle(struct beiscsi_hba *phba,
-		     struct beiscsi_conn *beiscsi_conn,
-		     struct hwi_async_pdu_context *pasync_ctx,
-		     struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
+static struct hd_async_handle *
+beiscsi_hdl_get_handle(struct beiscsi_conn *beiscsi_conn,
+		       struct hd_async_context *pasync_ctx,
+		       struct i_t_dpdu_cqe *pdpdu_cqe)
 {
+	struct beiscsi_hba *phba = beiscsi_conn->phba;
+	struct hd_async_handle *pasync_handle;
 	struct be_bus_address phys_addr;
-	struct list_head *pbusy_list;
-	struct async_pdu_handle *pasync_handle = NULL;
-	unsigned char is_header = 0;
-	unsigned int index, dpl;
+	u8 final, error = 0;
+	u16 cid, code, ci;
+	u32 dpl;
 
+	cid = beiscsi_conn->beiscsi_conn_cid;
+	/**
+	 * This function is invoked to get the right async_handle structure
+	 * from a given DEF PDU CQ entry.
+	 *
+	 * - index in CQ entry gives the vertical index
+	 * - address in CQ entry is the offset where the DMA last ended
+	 * - final - no more notifications for this PDU
+	 */
 	if (is_chip_be2_be3r(phba)) {
 		dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
 				    dpl, pdpdu_cqe);
-		index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
+		ci = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
 				      index, pdpdu_cqe);
+		final = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
+				      final, pdpdu_cqe);
 	} else {
 		dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
 				    dpl, pdpdu_cqe);
-		index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
+		ci = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
 				      index, pdpdu_cqe);
+		final = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
+				      final, pdpdu_cqe);
 	}
 
-	phys_addr.u.a32.address_lo =
-		(pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
-					db_addr_lo) / 32] - dpl);
-	phys_addr.u.a32.address_hi =
-		pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
-				       db_addr_hi) / 32];
-
-	phys_addr.u.a64.address =
-			*((unsigned long long *)(&phys_addr.u.a64.address));
-
-	switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
-			& PDUCQE_CODE_MASK) {
+	/**
+	 * DB addr Hi/Lo is same for BE and SKH.
+	 * Subtract the dataplacementlength to get to the base.
+	 */
+	phys_addr.u.a32.address_lo = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
+						   db_addr_lo, pdpdu_cqe);
+	phys_addr.u.a32.address_lo -= dpl;
+	phys_addr.u.a32.address_hi = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
+						   db_addr_hi, pdpdu_cqe);
+
+	code = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, code, pdpdu_cqe);
+	switch (code) {
 	case UNSOL_HDR_NOTIFY:
-		is_header = 1;
-
-		 pbusy_list = hwi_get_async_busy_list(pasync_ctx,
-						      is_header, index);
+		pasync_handle = pasync_ctx->async_entry[ci].header;
 		break;
+	case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
+		error = 1;
 	case UNSOL_DATA_NOTIFY:
-		 pbusy_list = hwi_get_async_busy_list(pasync_ctx,
-						      is_header, index);
+		pasync_handle = pasync_ctx->async_entry[ci].data;
 		break;
+	/* called only for above codes */
 	default:
-		pbusy_list = NULL;
-		beiscsi_log(phba, KERN_WARNING,
-			    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
-			    "BM_%d : Unexpected code=%d\n",
-			    pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
-			    code) / 32] & PDUCQE_CODE_MASK);
-		return NULL;
+		pasync_handle = NULL;
+		break;
 	}
 
-	WARN_ON(list_empty(pbusy_list));
-	list_for_each_entry(pasync_handle, pbusy_list, link) {
-		if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
-			break;
+	if (!pasync_handle) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
+			    "BM_%d : cid %d async PDU handle not found - code %d ci %d addr %llx\n",
+			    cid, code, ci, phys_addr.u.a64.address);
+		return pasync_handle;
 	}
 
-	WARN_ON(!pasync_handle);
+	if (pasync_handle->pa.u.a64.address != phys_addr.u.a64.address ||
+	    pasync_handle->index != ci) {
+		/* driver bug - if ci does not match async handle index */
+		error = 1;
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
+			    "BM_%d : cid %u async PDU handle mismatch - addr in %cQE %llx at %u:addr in CQE %llx ci %u\n",
+			    cid, pasync_handle->is_header ? 'H' : 'D',
+			    pasync_handle->pa.u.a64.address,
+			    pasync_handle->index,
+			    phys_addr.u.a64.address, ci);
+		/* FW has stale address - attempt continuing by dropping */
+	}
 
-	pasync_handle->cri = BE_GET_ASYNC_CRI_FROM_CID(
-			     beiscsi_conn->beiscsi_conn_cid);
-	pasync_handle->is_header = is_header;
+	/**
+	 * Each CID is associated with unique CRI.
+	 * ASYNC_CRI_FROM_CID mapping and CRI_FROM_CID are totaly different.
+	 **/
+	pasync_handle->cri = BE_GET_ASYNC_CRI_FROM_CID(cid);
+	pasync_handle->is_final = final;
 	pasync_handle->buffer_len = dpl;
-	*pcq_index = index;
+	/* empty the slot */
+	if (pasync_handle->is_header)
+		pasync_ctx->async_entry[ci].header = NULL;
+	else
+		pasync_ctx->async_entry[ci].data = NULL;
 
+	/**
+	 * DEF PDU header and data buffers with errors should be simply
+	 * dropped as there are no consumers for it.
+	 */
+	if (error) {
+		beiscsi_hdl_put_handle(pasync_ctx, pasync_handle);
+		pasync_handle = NULL;
+	}
 	return pasync_handle;
 }
 
-static unsigned int
-hwi_update_async_writables(struct beiscsi_hba *phba,
-			    struct hwi_async_pdu_context *pasync_ctx,
-			    unsigned int is_header, unsigned int cq_index)
+static void
+beiscsi_hdl_purge_handles(struct beiscsi_hba *phba,
+			  struct hd_async_context *pasync_ctx,
+			  u16 cri)
 {
-	struct list_head *pbusy_list;
-	struct async_pdu_handle *pasync_handle;
-	unsigned int num_entries, writables = 0;
-	unsigned int *pep_read_ptr, *pwritables;
-
-	num_entries = pasync_ctx->num_entries;
-	if (is_header) {
-		pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
-		pwritables = &pasync_ctx->async_header.writables;
-	} else {
-		pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
-		pwritables = &pasync_ctx->async_data.writables;
-	}
-
-	while ((*pep_read_ptr) != cq_index) {
-		(*pep_read_ptr)++;
-		*pep_read_ptr = (*pep_read_ptr) % num_entries;
-
-		pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
-						     *pep_read_ptr);
-		if (writables == 0)
-			WARN_ON(list_empty(pbusy_list));
-
-		if (!list_empty(pbusy_list)) {
-			pasync_handle = list_entry(pbusy_list->next,
-						   struct async_pdu_handle,
-						   link);
-			WARN_ON(!pasync_handle);
-			pasync_handle->consumed = 1;
-		}
-
-		writables++;
-	}
+	struct hd_async_handle *pasync_handle, *tmp_handle;
+	struct list_head *plist;
 
-	if (!writables) {
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
-			    "BM_%d : Duplicate notification received - index 0x%x!!\n",
-			    cq_index);
-		WARN_ON(1);
+	plist  = &pasync_ctx->async_entry[cri].wq.list;
+	list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
+		list_del(&pasync_handle->link);
+		beiscsi_hdl_put_handle(pasync_ctx, pasync_handle);
 	}
 
-	*pwritables = *pwritables + writables;
-	return 0;
+	INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wq.list);
+	pasync_ctx->async_entry[cri].wq.hdr_len = 0;
+	pasync_ctx->async_entry[cri].wq.bytes_received = 0;
+	pasync_ctx->async_entry[cri].wq.bytes_needed = 0;
 }
 
-static void hwi_free_async_msg(struct beiscsi_hba *phba,
-			       struct hwi_async_pdu_context *pasync_ctx,
-			       unsigned int cri)
+static unsigned int
+beiscsi_hdl_fwd_pdu(struct beiscsi_conn *beiscsi_conn,
+		    struct hd_async_context *pasync_ctx,
+		    u16 cri)
 {
-	struct async_pdu_handle *pasync_handle, *tmp_handle;
+	struct iscsi_session *session = beiscsi_conn->conn->session;
+	struct hd_async_handle *pasync_handle, *plast_handle;
+	struct beiscsi_hba *phba = beiscsi_conn->phba;
+	void *phdr = NULL, *pdata = NULL;
+	u32 dlen = 0, status = 0;
 	struct list_head *plist;
 
-	plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
-	list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
-		list_del(&pasync_handle->link);
-
-		if (pasync_handle->is_header) {
-			list_add_tail(&pasync_handle->link,
-				      &pasync_ctx->async_header.free_list);
-			pasync_ctx->async_header.free_entries++;
-		} else {
-			list_add_tail(&pasync_handle->link,
-				      &pasync_ctx->async_data.free_list);
-			pasync_ctx->async_data.free_entries++;
+	plist = &pasync_ctx->async_entry[cri].wq.list;
+	plast_handle = NULL;
+	list_for_each_entry(pasync_handle, plist, link) {
+		plast_handle = pasync_handle;
+		/* get the header, the first entry */
+		if (!phdr) {
+			phdr = pasync_handle->pbuffer;
+			continue;
+		}
+		/* use first buffer to collect all the data */
+		if (!pdata) {
+			pdata = pasync_handle->pbuffer;
+			dlen = pasync_handle->buffer_len;
+			continue;
 		}
+		memcpy(pdata + dlen, pasync_handle->pbuffer,
+		       pasync_handle->buffer_len);
+		dlen += pasync_handle->buffer_len;
 	}
 
-	INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
-	pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
-	pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
+	if (!plast_handle->is_final) {
+		/* last handle should have final PDU notification from FW */
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
+			    "BM_%d : cid %u %p fwd async PDU with last handle missing - HL%u:DN%u:DR%u\n",
+			    beiscsi_conn->beiscsi_conn_cid, plast_handle,
+			    pasync_ctx->async_entry[cri].wq.hdr_len,
+			    pasync_ctx->async_entry[cri].wq.bytes_needed,
+			    pasync_ctx->async_entry[cri].wq.bytes_received);
+	}
+	spin_lock_bh(&session->back_lock);
+	status = beiscsi_complete_pdu(beiscsi_conn, phdr, pdata, dlen);
+	spin_unlock_bh(&session->back_lock);
+	beiscsi_hdl_purge_handles(phba, pasync_ctx, cri);
+	return status;
 }
 
-static struct phys_addr *
-hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
-		     unsigned int is_header, unsigned int host_write_ptr)
+static unsigned int
+beiscsi_hdl_gather_pdu(struct beiscsi_conn *beiscsi_conn,
+		       struct hd_async_context *pasync_ctx,
+		       struct hd_async_handle *pasync_handle)
 {
-	struct phys_addr *pasync_sge = NULL;
+	unsigned int bytes_needed = 0, status = 0;
+	u16 cri = pasync_handle->cri;
+	struct cri_wait_queue *wq;
+	struct beiscsi_hba *phba;
+	struct pdu_base *ppdu;
+	char *err = "";
 
-	if (is_header)
-		pasync_sge = pasync_ctx->async_header.ring_base;
-	else
-		pasync_sge = pasync_ctx->async_data.ring_base;
+	phba = beiscsi_conn->phba;
+	wq = &pasync_ctx->async_entry[cri].wq;
+	if (pasync_handle->is_header) {
+		/* check if PDU hdr is rcv'd when old hdr not completed */
+		if (wq->hdr_len) {
+			err = "incomplete";
+			goto drop_pdu;
+		}
+		ppdu = pasync_handle->pbuffer;
+		bytes_needed = AMAP_GET_BITS(struct amap_pdu_base,
+					     data_len_hi, ppdu);
+		bytes_needed <<= 16;
+		bytes_needed |= be16_to_cpu(AMAP_GET_BITS(struct amap_pdu_base,
+							  data_len_lo, ppdu));
+		wq->hdr_len = pasync_handle->buffer_len;
+		wq->bytes_received = 0;
+		wq->bytes_needed = bytes_needed;
+		list_add_tail(&pasync_handle->link, &wq->list);
+		if (!bytes_needed)
+			status = beiscsi_hdl_fwd_pdu(beiscsi_conn,
+						     pasync_ctx, cri);
+	} else {
+		/* check if data received has header and is needed */
+		if (!wq->hdr_len || !wq->bytes_needed) {
+			err = "header less";
+			goto drop_pdu;
+		}
+		wq->bytes_received += pasync_handle->buffer_len;
+		/* Something got overwritten? Better catch it here. */
+		if (wq->bytes_received > wq->bytes_needed) {
+			err = "overflow";
+			goto drop_pdu;
+		}
+		list_add_tail(&pasync_handle->link, &wq->list);
+		if (wq->bytes_received == wq->bytes_needed)
+			status = beiscsi_hdl_fwd_pdu(beiscsi_conn,
+						     pasync_ctx, cri);
+	}
+	return status;
 
-	return pasync_sge + host_write_ptr;
+drop_pdu:
+	beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
+		    "BM_%d : cid %u async PDU %s - def-%c:HL%u:DN%u:DR%u\n",
+		    beiscsi_conn->beiscsi_conn_cid, err,
+		    pasync_handle->is_header ? 'H' : 'D',
+		    wq->hdr_len, wq->bytes_needed,
+		    pasync_handle->buffer_len);
+	/* discard this handle */
+	beiscsi_hdl_put_handle(pasync_ctx, pasync_handle);
+	/* free all the other handles in cri_wait_queue */
+	beiscsi_hdl_purge_handles(phba, pasync_ctx, cri);
+	/* try continuing */
+	return status;
 }
 
-static void hwi_post_async_buffers(struct beiscsi_hba *phba,
-				    unsigned int is_header, uint8_t ulp_num)
+static void
+beiscsi_hdq_post_handles(struct beiscsi_hba *phba,
+			 u8 header, u8 ulp_num)
 {
+	struct hd_async_handle *pasync_handle, *tmp, **slot;
+	struct hd_async_context *pasync_ctx;
 	struct hwi_controller *phwi_ctrlr;
-	struct hwi_async_pdu_context *pasync_ctx;
-	struct async_pdu_handle *pasync_handle;
-	struct list_head *pfree_link, *pbusy_list;
+	struct list_head *hfree_list;
 	struct phys_addr *pasync_sge;
-	unsigned int ring_id, num_entries;
-	unsigned int host_write_num, doorbell_offset;
-	unsigned int writables;
-	unsigned int i = 0;
-	u32 doorbell = 0;
+	u32 ring_id, doorbell = 0;
+	u16 index, num_entries;
+	u32 doorbell_offset;
+	u16 prod = 0, cons;
 
 	phwi_ctrlr = phba->phwi_ctrlr;
 	pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num);
 	num_entries = pasync_ctx->num_entries;
-
-	if (is_header) {
-		writables = min(pasync_ctx->async_header.writables,
-				pasync_ctx->async_header.free_entries);
-		pfree_link = pasync_ctx->async_header.free_list.next;
-		host_write_num = pasync_ctx->async_header.host_write_ptr;
+	if (header) {
+		cons = pasync_ctx->async_header.free_entries;
+		hfree_list = &pasync_ctx->async_header.free_list;
 		ring_id = phwi_ctrlr->default_pdu_hdr[ulp_num].id;
 		doorbell_offset = phwi_ctrlr->default_pdu_hdr[ulp_num].
-				  doorbell_offset;
+					doorbell_offset;
 	} else {
-		writables = min(pasync_ctx->async_data.writables,
-				pasync_ctx->async_data.free_entries);
-		pfree_link = pasync_ctx->async_data.free_list.next;
-		host_write_num = pasync_ctx->async_data.host_write_ptr;
+		cons = pasync_ctx->async_data.free_entries;
+		hfree_list = &pasync_ctx->async_data.free_list;
 		ring_id = phwi_ctrlr->default_pdu_data[ulp_num].id;
 		doorbell_offset = phwi_ctrlr->default_pdu_data[ulp_num].
-				  doorbell_offset;
+					doorbell_offset;
 	}
+	/* number of entries posted must be in multiples of 8 */
+	if (cons % 8)
+		return;
 
-	writables = (writables / 8) * 8;
-	if (writables) {
-		for (i = 0; i < writables; i++) {
-			pbusy_list =
-			    hwi_get_async_busy_list(pasync_ctx, is_header,
-						    host_write_num);
-			pasync_handle =
-			    list_entry(pfree_link, struct async_pdu_handle,
-								link);
-			WARN_ON(!pasync_handle);
-			pasync_handle->consumed = 0;
-
-			pfree_link = pfree_link->next;
-
-			pasync_sge = hwi_get_ring_address(pasync_ctx,
-						is_header, host_write_num);
-
-			pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
-			pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
-
-			list_move(&pasync_handle->link, pbusy_list);
-
-			host_write_num++;
-			host_write_num = host_write_num % num_entries;
-		}
-
-		if (is_header) {
-			pasync_ctx->async_header.host_write_ptr =
-							host_write_num;
-			pasync_ctx->async_header.free_entries -= writables;
-			pasync_ctx->async_header.writables -= writables;
-			pasync_ctx->async_header.busy_entries += writables;
-		} else {
-			pasync_ctx->async_data.host_write_ptr = host_write_num;
-			pasync_ctx->async_data.free_entries -= writables;
-			pasync_ctx->async_data.writables -= writables;
-			pasync_ctx->async_data.busy_entries += writables;
-		}
-
-		doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
-		doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
-		doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
-		doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
-					<< DB_DEF_PDU_CQPROC_SHIFT;
-
-		iowrite32(doorbell, phba->db_va + doorbell_offset);
-	}
-}
-
-static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
-					 struct beiscsi_conn *beiscsi_conn,
-					 struct i_t_dpdu_cqe *pdpdu_cqe)
-{
-	struct hwi_controller *phwi_ctrlr;
-	struct hwi_async_pdu_context *pasync_ctx;
-	struct async_pdu_handle *pasync_handle = NULL;
-	unsigned int cq_index = -1;
-	uint16_t cri_index = BE_GET_CRI_FROM_CID(
-			     beiscsi_conn->beiscsi_conn_cid);
-
-	phwi_ctrlr = phba->phwi_ctrlr;
-	pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
-		     BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
-		     cri_index));
-
-	pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
-					     pdpdu_cqe, &cq_index);
-	BUG_ON(pasync_handle->is_header != 0);
-	if (pasync_handle->consumed == 0)
-		hwi_update_async_writables(phba, pasync_ctx,
-					   pasync_handle->is_header, cq_index);
-
-	hwi_free_async_msg(phba, pasync_ctx, pasync_handle->cri);
-	hwi_post_async_buffers(phba, pasync_handle->is_header,
-			       BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
-			       cri_index));
-}
-
-static unsigned int
-hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
-		  struct beiscsi_hba *phba,
-		  struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
-{
-	struct list_head *plist;
-	struct async_pdu_handle *pasync_handle;
-	void *phdr = NULL;
-	unsigned int hdr_len = 0, buf_len = 0;
-	unsigned int status, index = 0, offset = 0;
-	void *pfirst_buffer = NULL;
-	unsigned int num_buf = 0;
-
-	plist = &pasync_ctx->async_entry[cri].wait_queue.list;
+	list_for_each_entry_safe(pasync_handle, tmp, hfree_list, link) {
+		list_del_init(&pasync_handle->link);
+		pasync_handle->is_final = 0;
+		pasync_handle->buffer_len = 0;
 
-	list_for_each_entry(pasync_handle, plist, link) {
-		if (index == 0) {
-			phdr = pasync_handle->pbuffer;
-			hdr_len = pasync_handle->buffer_len;
-		} else {
-			buf_len = pasync_handle->buffer_len;
-			if (!num_buf) {
-				pfirst_buffer = pasync_handle->pbuffer;
-				num_buf++;
-			}
-			memcpy(pfirst_buffer + offset,
-			       pasync_handle->pbuffer, buf_len);
-			offset += buf_len;
+		/* handles can be consumed out of order, use index in handle */
+		index = pasync_handle->index;
+		WARN_ON(pasync_handle->is_header != header);
+		if (header)
+			slot = &pasync_ctx->async_entry[index].header;
+		else
+			slot = &pasync_ctx->async_entry[index].data;
+		/**
+		 * The slot just tracks handle's hold and release, so
+		 * overwriting at the same index won't do any harm but
+		 * needs to be caught.
+		 */
+		if (*slot != NULL) {
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
+				    "BM_%d : async PDU %s slot at %u not empty\n",
+				    header ? "header" : "data", index);
 		}
-		index++;
+		/**
+		 * We use same freed index as in completion to post so this
+		 * operation is not required for refills. Its required only
+		 * for ring creation.
+		 */
+		if (header)
+			pasync_sge = pasync_ctx->async_header.ring_base;
+		else
+			pasync_sge = pasync_ctx->async_data.ring_base;
+		pasync_sge += index;
+		/* if its a refill then address is same; hi is lo */
+		WARN_ON(pasync_sge->hi &&
+			pasync_sge->hi != pasync_handle->pa.u.a32.address_lo);
+		WARN_ON(pasync_sge->lo &&
+			pasync_sge->lo != pasync_handle->pa.u.a32.address_hi);
+		pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
+		pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
+
+		*slot = pasync_handle;
+		if (++prod == cons)
+			break;
 	}
+	if (header)
+		pasync_ctx->async_header.free_entries -= prod;
+	else
+		pasync_ctx->async_data.free_entries -= prod;
 
-	status = beiscsi_process_async_pdu(beiscsi_conn, phba,
-					    phdr, hdr_len, pfirst_buffer,
-					    offset);
-
-	hwi_free_async_msg(phba, pasync_ctx, cri);
-	return 0;
-}
-
-static unsigned int
-hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
-		     struct beiscsi_hba *phba,
-		     struct async_pdu_handle *pasync_handle)
-{
-	struct hwi_async_pdu_context *pasync_ctx;
-	struct hwi_controller *phwi_ctrlr;
-	unsigned int bytes_needed = 0, status = 0;
-	unsigned short cri = pasync_handle->cri;
-	struct pdu_base *ppdu;
-
-	phwi_ctrlr = phba->phwi_ctrlr;
-	pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
-		     BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
-		     BE_GET_CRI_FROM_CID(beiscsi_conn->
-				 beiscsi_conn_cid)));
-
-	list_del(&pasync_handle->link);
-	if (pasync_handle->is_header) {
-		pasync_ctx->async_header.busy_entries--;
-		if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
-			hwi_free_async_msg(phba, pasync_ctx, cri);
-			BUG();
-		}
-
-		pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
-		pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
-		pasync_ctx->async_entry[cri].wait_queue.hdr_len =
-				(unsigned short)pasync_handle->buffer_len;
-		list_add_tail(&pasync_handle->link,
-			      &pasync_ctx->async_entry[cri].wait_queue.list);
-
-		ppdu = pasync_handle->pbuffer;
-		bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
-			data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
-			0xFFFF0000) | ((be16_to_cpu((ppdu->
-			dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
-			& PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
-
-		if (status == 0) {
-			pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
-			    bytes_needed;
-
-			if (bytes_needed == 0)
-				status = hwi_fwd_async_msg(beiscsi_conn, phba,
-							   pasync_ctx, cri);
-		}
-	} else {
-		pasync_ctx->async_data.busy_entries--;
-		if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
-			list_add_tail(&pasync_handle->link,
-				      &pasync_ctx->async_entry[cri].wait_queue.
-				      list);
-			pasync_ctx->async_entry[cri].wait_queue.
-				bytes_received +=
-				(unsigned short)pasync_handle->buffer_len;
-
-			if (pasync_ctx->async_entry[cri].wait_queue.
-			    bytes_received >=
-			    pasync_ctx->async_entry[cri].wait_queue.
-			    bytes_needed)
-				status = hwi_fwd_async_msg(beiscsi_conn, phba,
-							   pasync_ctx, cri);
-		}
-	}
-	return status;
+	doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
+	doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
+	doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
+	doorbell |= (prod & DB_DEF_PDU_CQPROC_MASK) << DB_DEF_PDU_CQPROC_SHIFT;
+	iowrite32(doorbell, phba->db_va + doorbell_offset);
 }
 
-static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
-					 struct beiscsi_hba *phba,
-					 struct i_t_dpdu_cqe *pdpdu_cqe)
+static void
+beiscsi_hdq_process_compl(struct beiscsi_conn *beiscsi_conn,
+			  struct i_t_dpdu_cqe *pdpdu_cqe)
 {
+	struct beiscsi_hba *phba = beiscsi_conn->phba;
+	struct hd_async_handle *pasync_handle = NULL;
+	struct hd_async_context *pasync_ctx;
 	struct hwi_controller *phwi_ctrlr;
-	struct hwi_async_pdu_context *pasync_ctx;
-	struct async_pdu_handle *pasync_handle = NULL;
-	unsigned int cq_index = -1;
-	uint16_t cri_index = BE_GET_CRI_FROM_CID(
-			     beiscsi_conn->beiscsi_conn_cid);
+	u16 cid_cri;
+	u8 ulp_num;
 
 	phwi_ctrlr = phba->phwi_ctrlr;
-	pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
-		     BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
-		     cri_index));
-
-	pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
-					     pdpdu_cqe, &cq_index);
-
-	if (pasync_handle->consumed == 0)
-		hwi_update_async_writables(phba, pasync_ctx,
-					   pasync_handle->is_header, cq_index);
+	cid_cri = BE_GET_CRI_FROM_CID(beiscsi_conn->beiscsi_conn_cid);
+	ulp_num = BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr, cid_cri);
+	pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num);
+	pasync_handle = beiscsi_hdl_get_handle(beiscsi_conn, pasync_ctx,
+					       pdpdu_cqe);
+	if (!pasync_handle)
+		return;
 
-	hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
-	hwi_post_async_buffers(phba, pasync_handle->is_header,
-			       BEISCSI_GET_ULP_FROM_CRI(
-			       phwi_ctrlr, cri_index));
+	beiscsi_hdl_gather_pdu(beiscsi_conn, pasync_ctx, pasync_handle);
+	beiscsi_hdq_post_handles(phba, pasync_handle->is_header, ulp_num);
 }
 
 void beiscsi_process_mcc_cq(struct beiscsi_hba *phba)
@@ -2051,6 +1816,9 @@ void beiscsi_process_mcc_cq(struct beiscsi_hba *phba)
 	mcc_compl = queue_tail_node(mcc_cq);
 	mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
 	while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
+		if (beiscsi_hba_in_error(phba))
+			return;
+
 		if (num_processed >= 32) {
 			hwi_ring_cq_db(phba, mcc_cq->id,
 					num_processed, 0);
@@ -2073,6 +1841,19 @@ void beiscsi_process_mcc_cq(struct beiscsi_hba *phba)
 		hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1);
 }
 
+static void beiscsi_mcc_work(struct work_struct *work)
+{
+	struct be_eq_obj *pbe_eq;
+	struct beiscsi_hba *phba;
+
+	pbe_eq = container_of(work, struct be_eq_obj, mcc_work);
+	phba = pbe_eq->phba;
+	beiscsi_process_mcc_cq(phba);
+	/* rearm EQ for further interrupts */
+	if (!beiscsi_hba_in_error(phba))
+		hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
+}
+
 /**
  * beiscsi_process_cq()- Process the Completion Queue
  * @pbe_eq: Event Q on which the Completion has come
@@ -2101,6 +1882,9 @@ unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq, int budget)
 
 	while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
 	       CQE_VALID_MASK) {
+		if (beiscsi_hba_in_error(phba))
+			return 0;
+
 		be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
 
 		 code = (sol->dw[offsetof(struct amap_sol_cqe, code) /
@@ -2165,8 +1949,8 @@ unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq, int budget)
 				    cqe_desc[code], code, cid);
 
 			spin_lock_bh(&phba->async_pdu_lock);
-			hwi_process_default_pdu_ring(beiscsi_conn, phba,
-					     (struct i_t_dpdu_cqe *)sol);
+			beiscsi_hdq_process_compl(beiscsi_conn,
+						  (struct i_t_dpdu_cqe *)sol);
 			spin_unlock_bh(&phba->async_pdu_lock);
 			break;
 		case UNSOL_DATA_NOTIFY:
@@ -2176,8 +1960,8 @@ unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq, int budget)
 				    cqe_desc[code], code, cid);
 
 			spin_lock_bh(&phba->async_pdu_lock);
-			hwi_process_default_pdu_ring(beiscsi_conn, phba,
-					     (struct i_t_dpdu_cqe *)sol);
+			beiscsi_hdq_process_compl(beiscsi_conn,
+						  (struct i_t_dpdu_cqe *)sol);
 			spin_unlock_bh(&phba->async_pdu_lock);
 			break;
 		case CXN_INVALIDATE_INDEX_NOTIFY:
@@ -2213,8 +1997,9 @@ unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq, int budget)
 				    "BM_%d :  Dropping %s[%d] on DPDU ring on CID : %d\n",
 				    cqe_desc[code], code, cid);
 			spin_lock_bh(&phba->async_pdu_lock);
-			hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
-					     (struct i_t_dpdu_cqe *) sol);
+			/* driver consumes the entry and drops the contents */
+			beiscsi_hdq_process_compl(beiscsi_conn,
+						  (struct i_t_dpdu_cqe *)sol);
 			spin_unlock_bh(&phba->async_pdu_lock);
 			break;
 		case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
@@ -2262,60 +2047,32 @@ proc_next_cqe:
 	return total;
 }
 
-void beiscsi_process_all_cqs(struct work_struct *work)
-{
-	unsigned long flags;
-	struct hwi_controller *phwi_ctrlr;
-	struct hwi_context_memory *phwi_context;
-	struct beiscsi_hba *phba;
-	struct be_eq_obj *pbe_eq =
-	    container_of(work, struct be_eq_obj, work_cqs);
-
-	phba = pbe_eq->phba;
-	phwi_ctrlr = phba->phwi_ctrlr;
-	phwi_context = phwi_ctrlr->phwi_ctxt;
-
-	if (pbe_eq->todo_mcc_cq) {
-		spin_lock_irqsave(&phba->isr_lock, flags);
-		pbe_eq->todo_mcc_cq = false;
-		spin_unlock_irqrestore(&phba->isr_lock, flags);
-		beiscsi_process_mcc_cq(phba);
-	}
-
-	if (pbe_eq->todo_cq) {
-		spin_lock_irqsave(&phba->isr_lock, flags);
-		pbe_eq->todo_cq = false;
-		spin_unlock_irqrestore(&phba->isr_lock, flags);
-		beiscsi_process_cq(pbe_eq, BE2_MAX_NUM_CQ_PROC);
-	}
-
-	/* rearm EQ for further interrupts */
-	hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
-}
-
 static int be_iopoll(struct irq_poll *iop, int budget)
 {
-	unsigned int ret, num_eq_processed;
+	unsigned int ret, io_events;
 	struct beiscsi_hba *phba;
 	struct be_eq_obj *pbe_eq;
 	struct be_eq_entry *eqe = NULL;
 	struct be_queue_info *eq;
 
-	num_eq_processed = 0;
 	pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
 	phba = pbe_eq->phba;
+	if (beiscsi_hba_in_error(phba)) {
+		irq_poll_complete(iop);
+		return 0;
+	}
+
+	io_events = 0;
 	eq = &pbe_eq->q;
 	eqe = queue_tail_node(eq);
-
 	while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] &
 			EQE_VALID_MASK) {
 		AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 		queue_tail_inc(eq);
 		eqe = queue_tail_node(eq);
-		num_eq_processed++;
+		io_events++;
 	}
-
-	hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
+	hwi_ring_eq_db(phba, eq->id, 1, io_events, 0, 1);
 
 	ret = beiscsi_process_cq(pbe_eq, budget);
 	pbe_eq->cq_count += ret;
@@ -2325,7 +2082,8 @@ static int be_iopoll(struct irq_poll *iop, int budget)
 			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
 			    "BM_%d : rearm pbe_eq->q.id =%d ret %d\n",
 			    pbe_eq->q.id, ret);
-		hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
+		if (!beiscsi_hba_in_error(phba))
+			hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
 	}
 	return ret;
 }
@@ -2691,20 +2449,20 @@ static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
 					  (ulp_num * MEM_DESCR_OFFSET));
 			phba->mem_req[mem_descr_index] =
 					  BEISCSI_GET_CID_COUNT(phba, ulp_num) *
-					  sizeof(struct async_pdu_handle);
+					  sizeof(struct hd_async_handle);
 
 			mem_descr_index = (HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
 					  (ulp_num * MEM_DESCR_OFFSET));
 			phba->mem_req[mem_descr_index] =
 					  BEISCSI_GET_CID_COUNT(phba, ulp_num) *
-					  sizeof(struct async_pdu_handle);
+					  sizeof(struct hd_async_handle);
 
 			mem_descr_index = (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
 					  (ulp_num * MEM_DESCR_OFFSET));
 			phba->mem_req[mem_descr_index] =
-					  sizeof(struct hwi_async_pdu_context) +
+					  sizeof(struct hd_async_context) +
 					 (BEISCSI_GET_CID_COUNT(phba, ulp_num) *
-					  sizeof(struct hwi_async_entry));
+					  sizeof(struct hd_async_entry));
 		}
 	}
 }
@@ -2963,35 +2721,34 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 	uint8_t ulp_num;
 	struct hwi_controller *phwi_ctrlr;
 	struct hba_parameters *p = &phba->params;
-	struct hwi_async_pdu_context *pasync_ctx;
-	struct async_pdu_handle *pasync_header_h, *pasync_data_h;
+	struct hd_async_context *pasync_ctx;
+	struct hd_async_handle *pasync_header_h, *pasync_data_h;
 	unsigned int index, idx, num_per_mem, num_async_data;
 	struct be_mem_descriptor *mem_descr;
 
 	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
-
+			 /* get async_ctx for each ULP */
 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 			mem_descr += (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
 				     (ulp_num * MEM_DESCR_OFFSET));
 
 			phwi_ctrlr = phba->phwi_ctrlr;
 			phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num] =
-				(struct hwi_async_pdu_context *)
+				(struct hd_async_context *)
 				 mem_descr->mem_array[0].virtual_address;
 
 			pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
 			memset(pasync_ctx, 0, sizeof(*pasync_ctx));
 
 			pasync_ctx->async_entry =
-					(struct hwi_async_entry *)
+					(struct hd_async_entry *)
 					((long unsigned int)pasync_ctx +
-					sizeof(struct hwi_async_pdu_context));
+					sizeof(struct hd_async_context));
 
 			pasync_ctx->num_entries = BEISCSI_GET_CID_COUNT(phba,
 						  ulp_num);
-			pasync_ctx->buffer_size = p->defpdu_hdr_sz;
-
+			/* setup header buffers */
 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 			mem_descr += HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
 				(ulp_num * MEM_DESCR_OFFSET);
@@ -3008,6 +2765,7 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 					    "BM_%d : No Virtual address for ULP : %d\n",
 					    ulp_num);
 
+			pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
 			pasync_ctx->async_header.va_base =
 				mem_descr->mem_array[0].virtual_address;
 
@@ -3015,6 +2773,7 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 				mem_descr->mem_array[0].
 				bus_address.u.a64.address;
 
+			/* setup header buffer sgls */
 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 			mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
 				     (ulp_num * MEM_DESCR_OFFSET);
@@ -3034,6 +2793,7 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 			pasync_ctx->async_header.ring_base =
 				mem_descr->mem_array[0].virtual_address;
 
+			/* setup header buffer handles */
 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 			mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
 				     (ulp_num * MEM_DESCR_OFFSET);
@@ -3052,9 +2812,9 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 
 			pasync_ctx->async_header.handle_base =
 				mem_descr->mem_array[0].virtual_address;
-			pasync_ctx->async_header.writables = 0;
 			INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
 
+			/* setup data buffer sgls */
 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 			mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
 				     (ulp_num * MEM_DESCR_OFFSET);
@@ -3074,6 +2834,7 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 			pasync_ctx->async_data.ring_base =
 				mem_descr->mem_array[0].virtual_address;
 
+			/* setup data buffer handles */
 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 			mem_descr += HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
 				     (ulp_num * MEM_DESCR_OFFSET);
@@ -3085,16 +2846,16 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 
 			pasync_ctx->async_data.handle_base =
 				mem_descr->mem_array[0].virtual_address;
-			pasync_ctx->async_data.writables = 0;
 			INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
 
 			pasync_header_h =
-				(struct async_pdu_handle *)
+				(struct hd_async_handle *)
 				pasync_ctx->async_header.handle_base;
 			pasync_data_h =
-				(struct async_pdu_handle *)
+				(struct hd_async_handle *)
 				pasync_ctx->async_data.handle_base;
 
+			/* setup data buffers */
 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 			mem_descr += HWI_MEM_ASYNC_DATA_BUF_ULP0 +
 				     (ulp_num * MEM_DESCR_OFFSET);
@@ -3112,6 +2873,7 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 					    ulp_num);
 
 			idx = 0;
+			pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
 			pasync_ctx->async_data.va_base =
 				mem_descr->mem_array[idx].virtual_address;
 			pasync_ctx->async_data.pa_base.u.a64.address =
@@ -3125,7 +2887,8 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 			for (index = 0;	index < BEISCSI_GET_CID_COUNT
 					(phba, ulp_num); index++) {
 				pasync_header_h->cri = -1;
-				pasync_header_h->index = (char)index;
+				pasync_header_h->is_header = 1;
+				pasync_header_h->index = index;
 				INIT_LIST_HEAD(&pasync_header_h->link);
 				pasync_header_h->pbuffer =
 					(void *)((unsigned long)
@@ -3142,14 +2905,13 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 					      free_list);
 				pasync_header_h++;
 				pasync_ctx->async_header.free_entries++;
-				pasync_ctx->async_header.writables++;
-
 				INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
-					       wait_queue.list);
-				INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
-					       header_busy_list);
+						wq.list);
+				pasync_ctx->async_entry[index].header = NULL;
+
 				pasync_data_h->cri = -1;
-				pasync_data_h->index = (char)index;
+				pasync_data_h->is_header = 0;
+				pasync_data_h->index = index;
 				INIT_LIST_HEAD(&pasync_data_h->link);
 
 				if (!num_async_data) {
@@ -3184,16 +2946,8 @@ static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 					      free_list);
 				pasync_data_h++;
 				pasync_ctx->async_data.free_entries++;
-				pasync_ctx->async_data.writables++;
-
-				INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
-					       data_busy_list);
+				pasync_ctx->async_entry[index].data = NULL;
 			}
-
-			pasync_ctx->async_header.host_write_ptr = 0;
-			pasync_ctx->async_header.ep_read_ptr = -1;
-			pasync_ctx->async_data.host_write_ptr = 0;
-			pasync_ctx->async_data.ep_read_ptr = -1;
 		}
 	}
 
@@ -3265,8 +3019,8 @@ static int be_fill_queue(struct be_queue_info *q,
 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
 			     struct hwi_context_memory *phwi_context)
 {
+	int ret = -ENOMEM, eq_for_mcc;
 	unsigned int i, num_eq_pages;
-	int ret = 0, eq_for_mcc;
 	struct be_queue_info *eq;
 	struct be_dma_mem *mem;
 	void *eq_vaddress;
@@ -3284,8 +3038,8 @@ static int beiscsi_create_eqs(struct beiscsi_hba *phba,
 		mem = &eq->dma_mem;
 		phwi_context->be_eq[i].phba = phba;
 		eq_vaddress = pci_alloc_consistent(phba->pcidev,
-						     num_eq_pages * PAGE_SIZE,
-						     &paddr);
+						   num_eq_pages * PAGE_SIZE,
+						   &paddr);
 		if (!eq_vaddress)
 			goto create_eq_error;
 
@@ -3313,6 +3067,7 @@ static int beiscsi_create_eqs(struct beiscsi_hba *phba,
 			    phwi_context->be_eq[i].q.id);
 	}
 	return 0;
+
 create_eq_error:
 	for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
 		eq = &phwi_context->be_eq[i].q;
@@ -3329,11 +3084,11 @@ static int beiscsi_create_cqs(struct beiscsi_hba *phba,
 			     struct hwi_context_memory *phwi_context)
 {
 	unsigned int i, num_cq_pages;
-	int ret = 0;
 	struct be_queue_info *cq, *eq;
 	struct be_dma_mem *mem;
 	struct be_eq_obj *pbe_eq;
 	void *cq_vaddress;
+	int ret = -ENOMEM;
 	dma_addr_t paddr;
 
 	num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
@@ -3347,10 +3102,11 @@ static int beiscsi_create_cqs(struct beiscsi_hba *phba,
 		pbe_eq->phba = phba;
 		mem = &cq->dma_mem;
 		cq_vaddress = pci_alloc_consistent(phba->pcidev,
-						     num_cq_pages * PAGE_SIZE,
-						     &paddr);
+						   num_cq_pages * PAGE_SIZE,
+						   &paddr);
 		if (!cq_vaddress)
 			goto create_cq_error;
+
 		ret = be_fill_queue(cq, phba->params.num_cq_entries,
 				    sizeof(struct sol_cqe), cq_vaddress);
 		if (ret) {
@@ -3385,7 +3141,6 @@ create_cq_error:
 					    mem->va, mem->dma);
 	}
 	return ret;
-
 }
 
 static int
@@ -3437,7 +3192,6 @@ beiscsi_create_def_hdr(struct beiscsi_hba *phba,
 		    "BM_%d : iscsi hdr def pdu id for ULP : %d is %d\n",
 		    ulp_num,
 		    phwi_context->be_def_hdrq[ulp_num].id);
-	hwi_post_async_buffers(phba, BEISCSI_DEFQ_HDR, ulp_num);
 	return 0;
 }
 
@@ -3492,11 +3246,9 @@ beiscsi_create_def_data(struct beiscsi_hba *phba,
 		    ulp_num,
 		    phwi_context->be_def_dataq[ulp_num].id);
 
-	hwi_post_async_buffers(phba, BEISCSI_DEFQ_DATA, ulp_num);
 	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
 		    "BM_%d : DEFAULT PDU DATA RING CREATED"
 		    "on ULP : %d\n", ulp_num);
-
 	return 0;
 }
 
@@ -3716,10 +3468,53 @@ static void free_wrb_handles(struct beiscsi_hba *phba)
 
 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
 {
-	struct be_queue_info *q;
 	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct be_dma_mem *ptag_mem;
+	struct be_queue_info *q;
+	int i, tag;
 
 	q = &phba->ctrl.mcc_obj.q;
+	for (i = 0; i < MAX_MCC_CMD; i++) {
+		tag = i + 1;
+		if (!test_bit(MCC_TAG_STATE_RUNNING,
+			      &ctrl->ptag_state[tag].tag_state))
+			continue;
+
+		if (test_bit(MCC_TAG_STATE_TIMEOUT,
+			     &ctrl->ptag_state[tag].tag_state)) {
+			ptag_mem = &ctrl->ptag_state[tag].tag_mem_state;
+			if (ptag_mem->size) {
+				pci_free_consistent(ctrl->pdev,
+						    ptag_mem->size,
+						    ptag_mem->va,
+						    ptag_mem->dma);
+				ptag_mem->size = 0;
+			}
+			continue;
+		}
+		/**
+		 * If MCC is still active and waiting then wake up the process.
+		 * We are here only because port is going offline. The process
+		 * sees that (BEISCSI_HBA_ONLINE is cleared) and EIO error is
+		 * returned for the operation and allocated memory cleaned up.
+		 */
+		if (waitqueue_active(&ctrl->mcc_wait[tag])) {
+			ctrl->mcc_tag_status[tag] = MCC_STATUS_FAILED;
+			ctrl->mcc_tag_status[tag] |= CQE_VALID_MASK;
+			wake_up_interruptible(&ctrl->mcc_wait[tag]);
+			/*
+			 * Control tag info gets reinitialized in enable
+			 * so wait for the process to clear running state.
+			 */
+			while (test_bit(MCC_TAG_STATE_RUNNING,
+					&ctrl->ptag_state[tag].tag_state))
+				schedule_timeout_uninterruptible(HZ);
+		}
+		/**
+		 * For MCC with tag_states MCC_TAG_STATE_ASYNC and
+		 * MCC_TAG_STATE_IGNORE nothing needs to done.
+		 */
+	}
 	if (q->created) {
 		beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
 		be_queue_free(phba, q);
@@ -3732,68 +3527,6 @@ static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
 	}
 }
 
-static void hwi_cleanup(struct beiscsi_hba *phba)
-{
-	struct be_queue_info *q;
-	struct be_ctrl_info *ctrl = &phba->ctrl;
-	struct hwi_controller *phwi_ctrlr;
-	struct hwi_context_memory *phwi_context;
-	struct hwi_async_pdu_context *pasync_ctx;
-	int i, eq_for_mcc, ulp_num;
-
-	phwi_ctrlr = phba->phwi_ctrlr;
-	phwi_context = phwi_ctrlr->phwi_ctxt;
-
-	be_cmd_iscsi_remove_template_hdr(ctrl);
-
-	for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
-		q = &phwi_context->be_wrbq[i];
-		if (q->created)
-			beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
-	}
-	kfree(phwi_context->be_wrbq);
-	free_wrb_handles(phba);
-
-	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
-		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
-
-			q = &phwi_context->be_def_hdrq[ulp_num];
-			if (q->created)
-				beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
-
-			q = &phwi_context->be_def_dataq[ulp_num];
-			if (q->created)
-				beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
-
-			pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
-		}
-	}
-
-	beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
-
-	for (i = 0; i < (phba->num_cpus); i++) {
-		q = &phwi_context->be_cq[i];
-		if (q->created) {
-			be_queue_free(phba, q);
-			beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
-		}
-	}
-
-	be_mcc_queues_destroy(phba);
-	if (phba->msix_enabled)
-		eq_for_mcc = 1;
-	else
-		eq_for_mcc = 0;
-	for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
-		q = &phwi_context->be_eq[i].q;
-		if (q->created) {
-			be_queue_free(phba, q);
-			beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
-		}
-	}
-	be_cmd_fw_uninit(ctrl);
-}
-
 static int be_mcc_queues_create(struct beiscsi_hba *phba,
 				struct hwi_context_memory *phwi_context)
 {
@@ -3875,6 +3608,118 @@ static void find_num_cpus(struct beiscsi_hba *phba)
 	}
 }
 
+static void hwi_purge_eq(struct beiscsi_hba *phba)
+{
+	struct hwi_controller *phwi_ctrlr;
+	struct hwi_context_memory *phwi_context;
+	struct be_queue_info *eq;
+	struct be_eq_entry *eqe = NULL;
+	int i, eq_msix;
+	unsigned int num_processed;
+
+	if (beiscsi_hba_in_error(phba))
+		return;
+
+	phwi_ctrlr = phba->phwi_ctrlr;
+	phwi_context = phwi_ctrlr->phwi_ctxt;
+	if (phba->msix_enabled)
+		eq_msix = 1;
+	else
+		eq_msix = 0;
+
+	for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
+		eq = &phwi_context->be_eq[i].q;
+		eqe = queue_tail_node(eq);
+		num_processed = 0;
+		while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
+					& EQE_VALID_MASK) {
+			AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
+			queue_tail_inc(eq);
+			eqe = queue_tail_node(eq);
+			num_processed++;
+		}
+
+		if (num_processed)
+			hwi_ring_eq_db(phba, eq->id, 1,	num_processed, 1, 1);
+	}
+}
+
+static void hwi_cleanup_port(struct beiscsi_hba *phba)
+{
+	struct be_queue_info *q;
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct hwi_controller *phwi_ctrlr;
+	struct hwi_context_memory *phwi_context;
+	struct hd_async_context *pasync_ctx;
+	int i, eq_for_mcc, ulp_num;
+
+	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
+		if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
+			beiscsi_cmd_iscsi_cleanup(phba, ulp_num);
+
+	/**
+	 * Purge all EQ entries that may have been left out. This is to
+	 * workaround a problem we've seen occasionally where driver gets an
+	 * interrupt with EQ entry bit set after stopping the controller.
+	 */
+	hwi_purge_eq(phba);
+
+	phwi_ctrlr = phba->phwi_ctrlr;
+	phwi_context = phwi_ctrlr->phwi_ctxt;
+
+	be_cmd_iscsi_remove_template_hdr(ctrl);
+
+	for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
+		q = &phwi_context->be_wrbq[i];
+		if (q->created)
+			beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
+	}
+	kfree(phwi_context->be_wrbq);
+	free_wrb_handles(phba);
+
+	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
+		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
+
+			q = &phwi_context->be_def_hdrq[ulp_num];
+			if (q->created)
+				beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
+
+			q = &phwi_context->be_def_dataq[ulp_num];
+			if (q->created)
+				beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
+
+			pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
+		}
+	}
+
+	beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
+
+	for (i = 0; i < (phba->num_cpus); i++) {
+		q = &phwi_context->be_cq[i];
+		if (q->created) {
+			be_queue_free(phba, q);
+			beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
+		}
+	}
+
+	be_mcc_queues_destroy(phba);
+	if (phba->msix_enabled)
+		eq_for_mcc = 1;
+	else
+		eq_for_mcc = 0;
+	for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
+		q = &phwi_context->be_eq[i].q;
+		if (q->created) {
+			be_queue_free(phba, q);
+			beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
+		}
+	}
+	/* this ensures complete FW cleanup */
+	beiscsi_cmd_function_reset(phba);
+	/* last communication, indicate driver is unloading */
+	beiscsi_cmd_special_wrb(&phba->ctrl, 0);
+}
+
 static int hwi_init_port(struct beiscsi_hba *phba)
 {
 	struct hwi_controller *phwi_ctrlr;
@@ -3887,9 +3732,8 @@ static int hwi_init_port(struct beiscsi_hba *phba)
 	phwi_context = phwi_ctrlr->phwi_ctxt;
 	phwi_context->max_eqd = 128;
 	phwi_context->min_eqd = 0;
-	phwi_context->cur_eqd = 0;
-	be_cmd_fw_initialize(&phba->ctrl);
-	/* set optic state to unknown */
+	phwi_context->cur_eqd = 32;
+	/* set port optic state to unknown */
 	phba->optic_state = 0xff;
 
 	status = beiscsi_create_eqs(phba, phwi_context);
@@ -3903,7 +3747,7 @@ static int hwi_init_port(struct beiscsi_hba *phba)
 	if (status != 0)
 		goto error;
 
-	status = mgmt_check_supported_fw(ctrl, phba);
+	status = beiscsi_check_supported_fw(ctrl, phba);
 	if (status != 0) {
 		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 			    "BM_%d : Unsupported fw version\n");
@@ -3919,7 +3763,6 @@ static int hwi_init_port(struct beiscsi_hba *phba)
 
 	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
-
 			def_pdu_ring_sz =
 				BEISCSI_GET_CID_COUNT(phba, ulp_num) *
 				sizeof(struct phys_addr);
@@ -3945,6 +3788,15 @@ static int hwi_init_port(struct beiscsi_hba *phba)
 					    ulp_num);
 				goto error;
 			}
+			/**
+			 * Now that the default PDU rings have been created,
+			 * let EP know about it.
+			 * Call beiscsi_cmd_iscsi_cleanup before posting?
+			 */
+			beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_HDR,
+						 ulp_num);
+			beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_DATA,
+						 ulp_num);
 		}
 	}
 
@@ -3973,7 +3825,7 @@ static int hwi_init_port(struct beiscsi_hba *phba)
 
 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
 			uint16_t cri = 0;
-			struct hwi_async_pdu_context *pasync_ctx;
+			struct hd_async_context *pasync_ctx;
 
 			pasync_ctx = HWI_GET_ASYNC_PDU_CTX(
 				     phwi_ctrlr, ulp_num);
@@ -3985,6 +3837,14 @@ static int hwi_init_port(struct beiscsi_hba *phba)
 					phwi_ctrlr->wrb_context[cri].cid] =
 					async_arr_idx++;
 			}
+			/**
+			 * Now that the default PDU rings have been created,
+			 * let EP know about it.
+			 */
+			beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_HDR,
+						 ulp_num);
+			beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_DATA,
+						 ulp_num);
 		}
 	}
 
@@ -3995,7 +3855,7 @@ static int hwi_init_port(struct beiscsi_hba *phba)
 error:
 	beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 		    "BM_%d : hwi_init_port failed");
-	hwi_cleanup(phba);
+	hwi_cleanup_port(phba);
 	return status;
 }
 
@@ -4354,149 +4214,6 @@ static void hwi_disable_intr(struct beiscsi_hba *phba)
 			    "BM_%d : In hwi_disable_intr, Already Disabled\n");
 }
 
-/**
- * beiscsi_get_boot_info()- Get the boot session info
- * @phba: The device priv structure instance
- *
- * Get the boot target info and store in driver priv structure
- *
- * return values
- *	Success: 0
- *	Failure: Non-Zero Value
- **/
-static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
-{
-	struct be_cmd_get_session_resp *session_resp;
-	struct be_dma_mem nonemb_cmd;
-	unsigned int tag;
-	unsigned int s_handle;
-	int ret = -ENOMEM;
-
-	/* Get the session handle of the boot target */
-	ret = be_mgmt_get_boot_shandle(phba, &s_handle);
-	if (ret) {
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-			    "BM_%d : No boot session\n");
-
-		if (ret == -ENXIO)
-			phba->get_boot = 0;
-
-
-		return ret;
-	}
-	phba->get_boot = 0;
-	nonemb_cmd.va = pci_zalloc_consistent(phba->ctrl.pdev,
-					      sizeof(*session_resp),
-					      &nonemb_cmd.dma);
-	if (nonemb_cmd.va == NULL) {
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-			    "BM_%d : Failed to allocate memory for"
-			    "beiscsi_get_session_info\n");
-
-		return -ENOMEM;
-	}
-
-	tag = mgmt_get_session_info(phba, s_handle,
-				    &nonemb_cmd);
-	if (!tag) {
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-			    "BM_%d : beiscsi_get_session_info"
-			    " Failed\n");
-
-		goto boot_freemem;
-	}
-
-	ret = beiscsi_mccq_compl_wait(phba, tag, NULL, &nonemb_cmd);
-	if (ret) {
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-			    "BM_%d : beiscsi_get_session_info Failed");
-
-		if (ret != -EBUSY)
-			goto boot_freemem;
-		else
-			return ret;
-	}
-
-	session_resp = nonemb_cmd.va ;
-
-	memcpy(&phba->boot_sess, &session_resp->session_info,
-	       sizeof(struct mgmt_session_info));
-
-	 beiscsi_logout_fw_sess(phba,
-				phba->boot_sess.session_handle);
-	ret = 0;
-
-boot_freemem:
-	pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
-		    nonemb_cmd.va, nonemb_cmd.dma);
-	return ret;
-}
-
-static void beiscsi_boot_release(void *data)
-{
-	struct beiscsi_hba *phba = data;
-
-	scsi_host_put(phba->shost);
-}
-
-static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
-{
-	struct iscsi_boot_kobj *boot_kobj;
-
-	/* it has been created previously */
-	if (phba->boot_kset)
-		return 0;
-
-	/* get boot info using mgmt cmd */
-	if (beiscsi_get_boot_info(phba))
-		/* Try to see if we can carry on without this */
-		return 0;
-
-	phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
-	if (!phba->boot_kset)
-		return -ENOMEM;
-
-	/* get a ref because the show function will ref the phba */
-	if (!scsi_host_get(phba->shost))
-		goto free_kset;
-	boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
-					     beiscsi_show_boot_tgt_info,
-					     beiscsi_tgt_get_attr_visibility,
-					     beiscsi_boot_release);
-	if (!boot_kobj)
-		goto put_shost;
-
-	if (!scsi_host_get(phba->shost))
-		goto free_kset;
-	boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
-						beiscsi_show_boot_ini_info,
-						beiscsi_ini_get_attr_visibility,
-						beiscsi_boot_release);
-	if (!boot_kobj)
-		goto put_shost;
-
-	if (!scsi_host_get(phba->shost))
-		goto free_kset;
-	boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
-					       beiscsi_show_boot_eth_info,
-					       beiscsi_eth_get_attr_visibility,
-					       beiscsi_boot_release);
-	if (!boot_kobj)
-		goto put_shost;
-	return 0;
-
-put_shost:
-	scsi_host_put(phba->shost);
-free_kset:
-	iscsi_boot_destroy_kset(phba->boot_kset);
-	phba->boot_kset = NULL;
-	return -ENOMEM;
-}
-
 static int beiscsi_init_port(struct beiscsi_hba *phba)
 {
 	int ret;
@@ -4516,7 +4233,8 @@ static int beiscsi_init_port(struct beiscsi_hba *phba)
 		goto do_cleanup_ctrlr;
 	}
 
-	if (hba_setup_cid_tbls(phba)) {
+	ret = hba_setup_cid_tbls(phba);
+	if (ret < 0) {
 		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 			    "BM_%d : Failed in hba_setup_cid_tbls\n");
 		kfree(phba->io_sgl_hndl_base);
@@ -4527,61 +4245,15 @@ static int beiscsi_init_port(struct beiscsi_hba *phba)
 	return ret;
 
 do_cleanup_ctrlr:
-	hwi_cleanup(phba);
+	hwi_cleanup_port(phba);
 	return ret;
 }
 
-static void hwi_purge_eq(struct beiscsi_hba *phba)
-{
-	struct hwi_controller *phwi_ctrlr;
-	struct hwi_context_memory *phwi_context;
-	struct be_queue_info *eq;
-	struct be_eq_entry *eqe = NULL;
-	int i, eq_msix;
-	unsigned int num_processed;
-
-	phwi_ctrlr = phba->phwi_ctrlr;
-	phwi_context = phwi_ctrlr->phwi_ctxt;
-	if (phba->msix_enabled)
-		eq_msix = 1;
-	else
-		eq_msix = 0;
-
-	for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
-		eq = &phwi_context->be_eq[i].q;
-		eqe = queue_tail_node(eq);
-		num_processed = 0;
-		while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
-					& EQE_VALID_MASK) {
-			AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
-			queue_tail_inc(eq);
-			eqe = queue_tail_node(eq);
-			num_processed++;
-		}
-
-		if (num_processed)
-			hwi_ring_eq_db(phba, eq->id, 1,	num_processed, 1, 1);
-	}
-}
-
-static void beiscsi_clean_port(struct beiscsi_hba *phba)
+static void beiscsi_cleanup_port(struct beiscsi_hba *phba)
 {
-	int mgmt_status, ulp_num;
 	struct ulp_cid_info *ptr_cid_info = NULL;
+	int ulp_num;
 
-	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
-		if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
-			mgmt_status = mgmt_epfw_cleanup(phba, ulp_num);
-			if (mgmt_status)
-				beiscsi_log(phba, KERN_WARNING,
-					    BEISCSI_LOG_INIT,
-					    "BM_%d : mgmt_epfw_cleanup FAILED"
-					    " for ULP_%d\n", ulp_num);
-		}
-	}
-
-	hwi_purge_eq(phba);
-	hwi_cleanup(phba);
 	kfree(phba->io_sgl_hndl_base);
 	kfree(phba->eh_sgl_hndl_base);
 	kfree(phba->ep_array);
@@ -4598,7 +4270,6 @@ static void beiscsi_clean_port(struct beiscsi_hba *phba)
 			}
 		}
 	}
-
 }
 
 /**
@@ -4625,16 +4296,12 @@ beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn,
 	io_task = task->dd_data;
 
 	if (io_task->pwrb_handle) {
-		memset(io_task->pwrb_handle->pwrb, 0,
-		       sizeof(struct iscsi_wrb));
-		free_wrb_handle(phba, pwrb_context,
-				io_task->pwrb_handle);
+		free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
 		io_task->pwrb_handle = NULL;
 	}
 
 	if (io_task->psgl_handle) {
-		free_mgmt_sgl_handle(phba,
-				     io_task->psgl_handle);
+		free_mgmt_sgl_handle(phba, io_task->psgl_handle);
 		io_task->psgl_handle = NULL;
 	}
 
@@ -4671,6 +4338,7 @@ static void beiscsi_cleanup_task(struct iscsi_task *task)
 		pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
 			      io_task->bhs_pa.u.a64.address);
 		io_task->cmd_bhs = NULL;
+		task->hdr = NULL;
 	}
 
 	if (task->sc) {
@@ -4686,7 +4354,8 @@ static void beiscsi_cleanup_task(struct iscsi_task *task)
 		}
 
 		if (io_task->scsi_cmnd) {
-			scsi_dma_unmap(io_task->scsi_cmnd);
+			if (io_task->num_sg)
+				scsi_dma_unmap(io_task->scsi_cmnd);
 			io_task->scsi_cmnd = NULL;
 		}
 	} else {
@@ -5051,7 +4720,6 @@ static int beiscsi_mtask(struct iscsi_task *task)
 
 	cid = beiscsi_conn->beiscsi_conn_cid;
 	pwrb = io_task->pwrb_handle->pwrb;
-	memset(pwrb, 0, sizeof(*pwrb));
 
 	if (is_chip_be2_be3r(phba)) {
 		AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
@@ -5165,6 +4833,15 @@ static int beiscsi_task_xmit(struct iscsi_task *task)
 	int num_sg;
 	unsigned int  writedir = 0, xferlen = 0;
 
+	phba = io_task->conn->phba;
+	/**
+	 * HBA in error includes BEISCSI_HBA_FW_TIMEOUT. IO path might be
+	 * operational if FW still gets heartbeat from EP FW. Is management
+	 * path really needed to continue further?
+	 */
+	if (!beiscsi_hba_is_online(phba))
+		return -EIO;
+
 	if (!io_task->conn->login_in_progress)
 		task->hdr->exp_statsn = 0;
 
@@ -5172,8 +4849,8 @@ static int beiscsi_task_xmit(struct iscsi_task *task)
 		return beiscsi_mtask(task);
 
 	io_task->scsi_cmnd = sc;
+	io_task->num_sg = 0;
 	num_sg = scsi_dma_map(sc);
-	phba = io_task->conn->phba;
 	if (num_sg < 0) {
 		beiscsi_log(phba, KERN_ERR,
 			    BEISCSI_LOG_IO | BEISCSI_LOG_ISCSI,
@@ -5184,6 +4861,11 @@ static int beiscsi_task_xmit(struct iscsi_task *task)
 
 		return num_sg;
 	}
+	/**
+	 * For scsi cmd task, check num_sg before unmapping in cleanup_task.
+	 * For management task, cleanup_task checks mtask_addr before unmapping.
+	 */
+	io_task->num_sg = num_sg;
 	xferlen = scsi_bufflen(sc);
 	sg = scsi_sglist(sc);
 	if (sc->sc_data_direction == DMA_TO_DEVICE)
@@ -5213,6 +4895,12 @@ static int beiscsi_bsg_request(struct bsg_job *job)
 	shost = iscsi_job_to_shost(job);
 	phba = iscsi_host_priv(shost);
 
+	if (!beiscsi_hba_is_online(phba)) {
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BM_%d : HBA in error 0x%lx\n", phba->state);
+		return -ENXIO;
+	}
+
 	switch (bsg_req->msgcode) {
 	case ISCSI_BSG_HST_VENDOR:
 		nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
@@ -5240,6 +4928,14 @@ static int beiscsi_bsg_request(struct bsg_job *job)
 					phba->ctrl.mcc_tag_status[tag],
 					msecs_to_jiffies(
 					BEISCSI_HOST_MBX_TIMEOUT));
+
+		if (!test_bit(BEISCSI_HBA_ONLINE, &phba->state)) {
+			clear_bit(MCC_TAG_STATE_RUNNING,
+				  &phba->ctrl.ptag_state[tag].tag_state);
+			pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
+					    nonemb_cmd.va, nonemb_cmd.dma);
+			return -EIO;
+		}
 		extd_status = (phba->ctrl.mcc_tag_status[tag] &
 			       CQE_STATUS_ADDL_MASK) >> CQE_STATUS_ADDL_SHIFT;
 		status = phba->ctrl.mcc_tag_status[tag] & CQE_STATUS_MASK;
@@ -5283,106 +4979,294 @@ void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
 	beiscsi_log_enable_init(phba, beiscsi_log_enable);
 }
 
-/*
- * beiscsi_quiesce()- Cleanup Driver resources
- * @phba: Instance Priv structure
- * @unload_state:i Clean or EEH unload state
- *
- * Free the OS and HW resources held by the driver
- **/
-static void beiscsi_quiesce(struct beiscsi_hba *phba,
-		uint32_t unload_state)
+void beiscsi_start_boot_work(struct beiscsi_hba *phba, unsigned int s_handle)
 {
-	struct hwi_controller *phwi_ctrlr;
-	struct hwi_context_memory *phwi_context;
-	struct be_eq_obj *pbe_eq;
-	unsigned int i, msix_vec;
+	if (phba->boot_struct.boot_kset)
+		return;
 
-	phwi_ctrlr = phba->phwi_ctrlr;
-	phwi_context = phwi_ctrlr->phwi_ctxt;
-	hwi_disable_intr(phba);
-	if (phba->msix_enabled) {
-		for (i = 0; i <= phba->num_cpus; i++) {
-			msix_vec = phba->msix_entries[i].vector;
-			free_irq(msix_vec, &phwi_context->be_eq[i]);
-			kfree(phba->msi_name[i]);
-		}
-	} else
-		if (phba->pcidev->irq)
-			free_irq(phba->pcidev->irq, phba);
-	pci_disable_msix(phba->pcidev);
-	cancel_delayed_work_sync(&phba->beiscsi_hw_check_task);
+	/* skip if boot work is already in progress */
+	if (test_and_set_bit(BEISCSI_HBA_BOOT_WORK, &phba->state))
+		return;
 
-	for (i = 0; i < phba->num_cpus; i++) {
-		pbe_eq = &phwi_context->be_eq[i];
-		irq_poll_disable(&pbe_eq->iopoll);
+	phba->boot_struct.retry = 3;
+	phba->boot_struct.tag = 0;
+	phba->boot_struct.s_handle = s_handle;
+	phba->boot_struct.action = BEISCSI_BOOT_GET_SHANDLE;
+	schedule_work(&phba->boot_work);
+}
+
+static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
+{
+	struct beiscsi_hba *phba = data;
+	struct mgmt_session_info *boot_sess = &phba->boot_struct.boot_sess;
+	struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
+	char *str = buf;
+	int rc = -EPERM;
+
+	switch (type) {
+	case ISCSI_BOOT_TGT_NAME:
+		rc = sprintf(buf, "%.*s\n",
+			    (int)strlen(boot_sess->target_name),
+			    (char *)&boot_sess->target_name);
+		break;
+	case ISCSI_BOOT_TGT_IP_ADDR:
+		if (boot_conn->dest_ipaddr.ip_type == BEISCSI_IP_TYPE_V4)
+			rc = sprintf(buf, "%pI4\n",
+				(char *)&boot_conn->dest_ipaddr.addr);
+		else
+			rc = sprintf(str, "%pI6\n",
+				(char *)&boot_conn->dest_ipaddr.addr);
+		break;
+	case ISCSI_BOOT_TGT_PORT:
+		rc = sprintf(str, "%d\n", boot_conn->dest_port);
+		break;
+
+	case ISCSI_BOOT_TGT_CHAP_NAME:
+		rc = sprintf(str,  "%.*s\n",
+			     boot_conn->negotiated_login_options.auth_data.chap.
+			     target_chap_name_length,
+			     (char *)&boot_conn->negotiated_login_options.
+			     auth_data.chap.target_chap_name);
+		break;
+	case ISCSI_BOOT_TGT_CHAP_SECRET:
+		rc = sprintf(str,  "%.*s\n",
+			     boot_conn->negotiated_login_options.auth_data.chap.
+			     target_secret_length,
+			     (char *)&boot_conn->negotiated_login_options.
+			     auth_data.chap.target_secret);
+		break;
+	case ISCSI_BOOT_TGT_REV_CHAP_NAME:
+		rc = sprintf(str,  "%.*s\n",
+			     boot_conn->negotiated_login_options.auth_data.chap.
+			     intr_chap_name_length,
+			     (char *)&boot_conn->negotiated_login_options.
+			     auth_data.chap.intr_chap_name);
+		break;
+	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
+		rc = sprintf(str,  "%.*s\n",
+			     boot_conn->negotiated_login_options.auth_data.chap.
+			     intr_secret_length,
+			     (char *)&boot_conn->negotiated_login_options.
+			     auth_data.chap.intr_secret);
+		break;
+	case ISCSI_BOOT_TGT_FLAGS:
+		rc = sprintf(str, "2\n");
+		break;
+	case ISCSI_BOOT_TGT_NIC_ASSOC:
+		rc = sprintf(str, "0\n");
+		break;
 	}
+	return rc;
+}
 
-	if (unload_state == BEISCSI_CLEAN_UNLOAD) {
-		destroy_workqueue(phba->wq);
-		beiscsi_clean_port(phba);
-		beiscsi_free_mem(phba);
+static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
+{
+	struct beiscsi_hba *phba = data;
+	char *str = buf;
+	int rc = -EPERM;
 
-		beiscsi_unmap_pci_function(phba);
-		pci_free_consistent(phba->pcidev,
-				    phba->ctrl.mbox_mem_alloced.size,
-				    phba->ctrl.mbox_mem_alloced.va,
-				    phba->ctrl.mbox_mem_alloced.dma);
-	} else {
-		hwi_purge_eq(phba);
-		hwi_cleanup(phba);
+	switch (type) {
+	case ISCSI_BOOT_INI_INITIATOR_NAME:
+		rc = sprintf(str, "%s\n",
+			     phba->boot_struct.boot_sess.initiator_iscsiname);
+		break;
 	}
+	return rc;
+}
+
+static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
+{
+	struct beiscsi_hba *phba = data;
+	char *str = buf;
+	int rc = -EPERM;
 
+	switch (type) {
+	case ISCSI_BOOT_ETH_FLAGS:
+		rc = sprintf(str, "2\n");
+		break;
+	case ISCSI_BOOT_ETH_INDEX:
+		rc = sprintf(str, "0\n");
+		break;
+	case ISCSI_BOOT_ETH_MAC:
+		rc  = beiscsi_get_macaddr(str, phba);
+		break;
+	}
+	return rc;
 }
 
-static void beiscsi_remove(struct pci_dev *pcidev)
+static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
 {
-	struct beiscsi_hba *phba = NULL;
+	umode_t rc = 0;
 
-	phba = pci_get_drvdata(pcidev);
-	if (!phba) {
-		dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
-		return;
+	switch (type) {
+	case ISCSI_BOOT_TGT_NAME:
+	case ISCSI_BOOT_TGT_IP_ADDR:
+	case ISCSI_BOOT_TGT_PORT:
+	case ISCSI_BOOT_TGT_CHAP_NAME:
+	case ISCSI_BOOT_TGT_CHAP_SECRET:
+	case ISCSI_BOOT_TGT_REV_CHAP_NAME:
+	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
+	case ISCSI_BOOT_TGT_NIC_ASSOC:
+	case ISCSI_BOOT_TGT_FLAGS:
+		rc = S_IRUGO;
+		break;
 	}
+	return rc;
+}
 
-	beiscsi_destroy_def_ifaces(phba);
-	iscsi_boot_destroy_kset(phba->boot_kset);
-	iscsi_host_remove(phba->shost);
-	beiscsi_quiesce(phba, BEISCSI_CLEAN_UNLOAD);
-	pci_dev_put(phba->pcidev);
-	iscsi_host_free(phba->shost);
-	pci_disable_pcie_error_reporting(pcidev);
-	pci_set_drvdata(pcidev, NULL);
-	pci_release_regions(pcidev);
-	pci_disable_device(pcidev);
+static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
+{
+	umode_t rc = 0;
+
+	switch (type) {
+	case ISCSI_BOOT_INI_INITIATOR_NAME:
+		rc = S_IRUGO;
+		break;
+	}
+	return rc;
 }
 
-static void beiscsi_msix_enable(struct beiscsi_hba *phba)
+static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
 {
-	int i, status;
+	umode_t rc = 0;
 
-	for (i = 0; i <= phba->num_cpus; i++)
-		phba->msix_entries[i].entry = i;
+	switch (type) {
+	case ISCSI_BOOT_ETH_FLAGS:
+	case ISCSI_BOOT_ETH_MAC:
+	case ISCSI_BOOT_ETH_INDEX:
+		rc = S_IRUGO;
+		break;
+	}
+	return rc;
+}
 
-	status = pci_enable_msix_range(phba->pcidev, phba->msix_entries,
-				       phba->num_cpus + 1, phba->num_cpus + 1);
-	if (status > 0)
-		phba->msix_enabled = true;
+static void beiscsi_boot_kobj_release(void *data)
+{
+	struct beiscsi_hba *phba = data;
 
-	return;
+	scsi_host_put(phba->shost);
 }
 
-static void be_eqd_update(struct beiscsi_hba *phba)
+static int beiscsi_boot_create_kset(struct beiscsi_hba *phba)
 {
+	struct boot_struct *bs = &phba->boot_struct;
+	struct iscsi_boot_kobj *boot_kobj;
+
+	if (bs->boot_kset) {
+		__beiscsi_log(phba, KERN_ERR,
+			      "BM_%d: boot_kset already created\n");
+		return 0;
+	}
+
+	bs->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
+	if (!bs->boot_kset) {
+		__beiscsi_log(phba, KERN_ERR,
+			      "BM_%d: boot_kset alloc failed\n");
+		return -ENOMEM;
+	}
+
+	/* get shost ref because the show function will refer phba */
+	if (!scsi_host_get(phba->shost))
+		goto free_kset;
+
+	boot_kobj = iscsi_boot_create_target(bs->boot_kset, 0, phba,
+					     beiscsi_show_boot_tgt_info,
+					     beiscsi_tgt_get_attr_visibility,
+					     beiscsi_boot_kobj_release);
+	if (!boot_kobj)
+		goto put_shost;
+
+	if (!scsi_host_get(phba->shost))
+		goto free_kset;
+
+	boot_kobj = iscsi_boot_create_initiator(bs->boot_kset, 0, phba,
+						beiscsi_show_boot_ini_info,
+						beiscsi_ini_get_attr_visibility,
+						beiscsi_boot_kobj_release);
+	if (!boot_kobj)
+		goto put_shost;
+
+	if (!scsi_host_get(phba->shost))
+		goto free_kset;
+
+	boot_kobj = iscsi_boot_create_ethernet(bs->boot_kset, 0, phba,
+					       beiscsi_show_boot_eth_info,
+					       beiscsi_eth_get_attr_visibility,
+					       beiscsi_boot_kobj_release);
+	if (!boot_kobj)
+		goto put_shost;
+
+	return 0;
+
+put_shost:
+	scsi_host_put(phba->shost);
+free_kset:
+	iscsi_boot_destroy_kset(bs->boot_kset);
+	bs->boot_kset = NULL;
+	return -ENOMEM;
+}
+
+static void beiscsi_boot_work(struct work_struct *work)
+{
+	struct beiscsi_hba *phba =
+		container_of(work, struct beiscsi_hba, boot_work);
+	struct boot_struct *bs = &phba->boot_struct;
+	unsigned int tag = 0;
+
+	if (!beiscsi_hba_is_online(phba))
+		return;
+
+	beiscsi_log(phba, KERN_INFO,
+		    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+		    "BM_%d : %s action %d\n",
+		    __func__, phba->boot_struct.action);
+
+	switch (phba->boot_struct.action) {
+	case BEISCSI_BOOT_REOPEN_SESS:
+		tag = beiscsi_boot_reopen_sess(phba);
+		break;
+	case BEISCSI_BOOT_GET_SHANDLE:
+		tag = __beiscsi_boot_get_shandle(phba, 1);
+		break;
+	case BEISCSI_BOOT_GET_SINFO:
+		tag = beiscsi_boot_get_sinfo(phba);
+		break;
+	case BEISCSI_BOOT_LOGOUT_SESS:
+		tag = beiscsi_boot_logout_sess(phba);
+		break;
+	case BEISCSI_BOOT_CREATE_KSET:
+		beiscsi_boot_create_kset(phba);
+		/**
+		 * updated boot_kset is made visible to all before
+		 * ending the boot work.
+		 */
+		mb();
+		clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state);
+		return;
+	}
+	if (!tag) {
+		if (bs->retry--)
+			schedule_work(&phba->boot_work);
+		else
+			clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state);
+	}
+}
+
+static void beiscsi_eqd_update_work(struct work_struct *work)
+{
+	struct hwi_context_memory *phwi_context;
 	struct be_set_eqd set_eqd[MAX_CPUS];
-	struct be_aic_obj *aic;
-	struct be_eq_obj *pbe_eq;
 	struct hwi_controller *phwi_ctrlr;
-	struct hwi_context_memory *phwi_context;
+	struct be_eq_obj *pbe_eq;
+	struct beiscsi_hba *phba;
+	unsigned int pps, delta;
+	struct be_aic_obj *aic;
 	int eqd, i, num = 0;
-	ulong now;
-	u32 pps, delta;
-	unsigned int tag;
+	unsigned long now;
+
+	phba = container_of(work, struct beiscsi_hba, eqd_update.work);
+	if (!beiscsi_hba_is_online(phba))
+		return;
 
 	phwi_ctrlr = phba->phwi_ctrlr;
 	phwi_context = phwi_ctrlr->phwi_ctxt;
@@ -5391,13 +5275,13 @@ static void be_eqd_update(struct beiscsi_hba *phba)
 		aic = &phba->aic_obj[i];
 		pbe_eq = &phwi_context->be_eq[i];
 		now = jiffies;
-		if (!aic->jiffs || time_before(now, aic->jiffs) ||
+		if (!aic->jiffies || time_before(now, aic->jiffies) ||
 		    pbe_eq->cq_count < aic->eq_prev) {
-			aic->jiffs = now;
+			aic->jiffies = now;
 			aic->eq_prev = pbe_eq->cq_count;
 			continue;
 		}
-		delta = jiffies_to_msecs(now - aic->jiffs);
+		delta = jiffies_to_msecs(now - aic->jiffies);
 		pps = (((u32)(pbe_eq->cq_count - aic->eq_prev) * 1000) / delta);
 		eqd = (pps / 1500) << 2;
 
@@ -5406,7 +5290,7 @@ static void be_eqd_update(struct beiscsi_hba *phba)
 		eqd = min_t(u32, eqd, phwi_context->max_eqd);
 		eqd = max_t(u32, eqd, phwi_context->min_eqd);
 
-		aic->jiffs = now;
+		aic->jiffies = now;
 		aic->eq_prev = pbe_eq->cq_count;
 
 		if (eqd != aic->prev_eqd) {
@@ -5416,53 +5300,242 @@ static void be_eqd_update(struct beiscsi_hba *phba)
 			num++;
 		}
 	}
-	if (num) {
-		tag = be_cmd_modify_eq_delay(phba, set_eqd, num);
-		if (tag)
-			beiscsi_mccq_compl_wait(phba, tag, NULL, NULL);
+	if (num)
+		/* completion of this is ignored */
+		beiscsi_modify_eq_delay(phba, set_eqd, num);
+
+	schedule_delayed_work(&phba->eqd_update,
+			      msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL));
+}
+
+static void beiscsi_msix_enable(struct beiscsi_hba *phba)
+{
+	int i, status;
+
+	for (i = 0; i <= phba->num_cpus; i++)
+		phba->msix_entries[i].entry = i;
+
+	status = pci_enable_msix_range(phba->pcidev, phba->msix_entries,
+				       phba->num_cpus + 1, phba->num_cpus + 1);
+	if (status > 0)
+		phba->msix_enabled = true;
+}
+
+static void beiscsi_hw_tpe_check(unsigned long ptr)
+{
+	struct beiscsi_hba *phba;
+	u32 wait;
+
+	phba = (struct beiscsi_hba *)ptr;
+	/* if not TPE, do nothing */
+	if (!beiscsi_detect_tpe(phba))
+		return;
+
+	/* wait default 4000ms before recovering */
+	wait = 4000;
+	if (phba->ue2rp > BEISCSI_UE_DETECT_INTERVAL)
+		wait = phba->ue2rp - BEISCSI_UE_DETECT_INTERVAL;
+	queue_delayed_work(phba->wq, &phba->recover_port,
+			   msecs_to_jiffies(wait));
+}
+
+static void beiscsi_hw_health_check(unsigned long ptr)
+{
+	struct beiscsi_hba *phba;
+
+	phba = (struct beiscsi_hba *)ptr;
+	beiscsi_detect_ue(phba);
+	if (beiscsi_detect_ue(phba)) {
+		__beiscsi_log(phba, KERN_ERR,
+			      "BM_%d : port in error: %lx\n", phba->state);
+		/* sessions are no longer valid, so first fail the sessions */
+		queue_work(phba->wq, &phba->sess_work);
+
+		/* detect UER supported */
+		if (!test_bit(BEISCSI_HBA_UER_SUPP, &phba->state))
+			return;
+		/* modify this timer to check TPE */
+		phba->hw_check.function = beiscsi_hw_tpe_check;
 	}
+
+	mod_timer(&phba->hw_check,
+		  jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL));
 }
 
-static void be_check_boot_session(struct beiscsi_hba *phba)
+/*
+ * beiscsi_enable_port()- Enables the disabled port.
+ * Only port resources freed in disable function are reallocated.
+ * This is called in HBA error handling path.
+ *
+ * @phba: Instance of driver private structure
+ *
+ **/
+static int beiscsi_enable_port(struct beiscsi_hba *phba)
 {
-	if (beiscsi_setup_boot_info(phba))
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BM_%d : Could not set up "
-			    "iSCSI boot info on async event.\n");
+	struct hwi_context_memory *phwi_context;
+	struct hwi_controller *phwi_ctrlr;
+	struct be_eq_obj *pbe_eq;
+	int ret, i;
+
+	if (test_bit(BEISCSI_HBA_ONLINE, &phba->state)) {
+		__beiscsi_log(phba, KERN_ERR,
+			      "BM_%d : %s : port is online %lx\n",
+			      __func__, phba->state);
+		return 0;
+	}
+
+	ret = beiscsi_init_sliport(phba);
+	if (ret)
+		return ret;
+
+	if (enable_msix)
+		find_num_cpus(phba);
+	else
+		phba->num_cpus = 1;
+	if (enable_msix) {
+		beiscsi_msix_enable(phba);
+		if (!phba->msix_enabled)
+			phba->num_cpus = 1;
+	}
+
+	beiscsi_get_params(phba);
+	/* Re-enable UER. If different TPE occurs then it is recoverable. */
+	beiscsi_set_uer_feature(phba);
+
+	phba->shost->max_id = phba->params.cxns_per_ctrl;
+	phba->shost->can_queue = phba->params.ios_per_ctrl;
+	ret = hwi_init_controller(phba);
+	if (ret) {
+		__beiscsi_log(phba, KERN_ERR,
+			      "BM_%d : init controller failed %d\n", ret);
+		goto disable_msix;
+	}
+
+	for (i = 0; i < MAX_MCC_CMD; i++) {
+		init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
+		phba->ctrl.mcc_tag[i] = i + 1;
+		phba->ctrl.mcc_tag_status[i + 1] = 0;
+		phba->ctrl.mcc_tag_available++;
+	}
+
+	phwi_ctrlr = phba->phwi_ctrlr;
+	phwi_context = phwi_ctrlr->phwi_ctxt;
+	for (i = 0; i < phba->num_cpus; i++) {
+		pbe_eq = &phwi_context->be_eq[i];
+		irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll);
+	}
+
+	i = (phba->msix_enabled) ? i : 0;
+	/* Work item for MCC handling */
+	pbe_eq = &phwi_context->be_eq[i];
+	INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work);
+
+	ret = beiscsi_init_irqs(phba);
+	if (ret < 0) {
+		__beiscsi_log(phba, KERN_ERR,
+			      "BM_%d : setup IRQs failed %d\n", ret);
+		goto cleanup_port;
+	}
+	hwi_enable_intr(phba);
+	/* port operational: clear all error bits */
+	set_bit(BEISCSI_HBA_ONLINE, &phba->state);
+	__beiscsi_log(phba, KERN_INFO,
+		      "BM_%d : port online: 0x%lx\n", phba->state);
+
+	/* start hw_check timer and eqd_update work */
+	schedule_delayed_work(&phba->eqd_update,
+			      msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL));
+
+	/**
+	 * Timer function gets modified for TPE detection.
+	 * Always reinit to do health check first.
+	 */
+	phba->hw_check.function = beiscsi_hw_health_check;
+	mod_timer(&phba->hw_check,
+		  jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL));
+	return 0;
+
+cleanup_port:
+	for (i = 0; i < phba->num_cpus; i++) {
+		pbe_eq = &phwi_context->be_eq[i];
+		irq_poll_disable(&pbe_eq->iopoll);
+	}
+	hwi_cleanup_port(phba);
+
+disable_msix:
+	if (phba->msix_enabled)
+		pci_disable_msix(phba->pcidev);
+
+	return ret;
 }
 
 /*
- * beiscsi_hw_health_check()- Check adapter health
- * @work: work item to check HW health
+ * beiscsi_disable_port()- Disable port and cleanup driver resources.
+ * This is called in HBA error handling and driver removal.
+ * @phba: Instance Priv structure
+ * @unload: indicate driver is unloading
  *
- * Check if adapter in an unrecoverable state or not.
+ * Free the OS and HW resources held by the driver
  **/
-static void
-beiscsi_hw_health_check(struct work_struct *work)
+static void beiscsi_disable_port(struct beiscsi_hba *phba, int unload)
 {
-	struct beiscsi_hba *phba =
-		container_of(work, struct beiscsi_hba,
-			     beiscsi_hw_check_task.work);
+	struct hwi_context_memory *phwi_context;
+	struct hwi_controller *phwi_ctrlr;
+	struct be_eq_obj *pbe_eq;
+	unsigned int i, msix_vec;
 
-	be_eqd_update(phba);
+	if (!test_and_clear_bit(BEISCSI_HBA_ONLINE, &phba->state))
+		return;
 
-	if (phba->state & BE_ADAPTER_CHECK_BOOT) {
-		if ((phba->get_boot > 0) && (!phba->boot_kset)) {
-			phba->get_boot--;
-			if (!(phba->get_boot % BE_GET_BOOT_TO))
-				be_check_boot_session(phba);
-		} else {
-			phba->state &= ~BE_ADAPTER_CHECK_BOOT;
-			phba->get_boot = 0;
+	phwi_ctrlr = phba->phwi_ctrlr;
+	phwi_context = phwi_ctrlr->phwi_ctxt;
+	hwi_disable_intr(phba);
+	if (phba->msix_enabled) {
+		for (i = 0; i <= phba->num_cpus; i++) {
+			msix_vec = phba->msix_entries[i].vector;
+			free_irq(msix_vec, &phwi_context->be_eq[i]);
+			kfree(phba->msi_name[i]);
 		}
+	} else
+		if (phba->pcidev->irq)
+			free_irq(phba->pcidev->irq, phba);
+	pci_disable_msix(phba->pcidev);
+
+	for (i = 0; i < phba->num_cpus; i++) {
+		pbe_eq = &phwi_context->be_eq[i];
+		irq_poll_disable(&pbe_eq->iopoll);
 	}
+	cancel_delayed_work_sync(&phba->eqd_update);
+	cancel_work_sync(&phba->boot_work);
+	/* WQ might be running cancel queued mcc_work if we are not exiting */
+	if (!unload && beiscsi_hba_in_error(phba)) {
+		pbe_eq = &phwi_context->be_eq[i];
+		cancel_work_sync(&pbe_eq->mcc_work);
+	}
+	hwi_cleanup_port(phba);
+}
 
-	beiscsi_ue_detect(phba);
+static void beiscsi_sess_work(struct work_struct *work)
+{
+	struct beiscsi_hba *phba;
 
-	schedule_delayed_work(&phba->beiscsi_hw_check_task,
-			      msecs_to_jiffies(1000));
+	phba = container_of(work, struct beiscsi_hba, sess_work);
+	/*
+	 * This work gets scheduled only in case of HBA error.
+	 * Old sessions are gone so need to be re-established.
+	 * iscsi_session_failure needs process context hence this work.
+	 */
+	iscsi_host_for_each_session(phba->shost, beiscsi_session_fail);
 }
 
+static void beiscsi_recover_port(struct work_struct *work)
+{
+	struct beiscsi_hba *phba;
+
+	phba = container_of(work, struct beiscsi_hba, recover_port.work);
+	beiscsi_disable_port(phba, 0);
+	beiscsi_enable_port(phba);
+}
 
 static pci_ers_result_t beiscsi_eeh_err_detected(struct pci_dev *pdev,
 		pci_channel_state_t state)
@@ -5470,12 +5543,18 @@ static pci_ers_result_t beiscsi_eeh_err_detected(struct pci_dev *pdev,
 	struct beiscsi_hba *phba = NULL;
 
 	phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
-	phba->state |= BE_ADAPTER_PCI_ERR;
+	set_bit(BEISCSI_HBA_PCI_ERR, &phba->state);
 
 	beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 		    "BM_%d : EEH error detected\n");
 
-	beiscsi_quiesce(phba, BEISCSI_EEH_UNLOAD);
+	/* first stop UE detection when PCI error detected */
+	del_timer_sync(&phba->hw_check);
+	cancel_delayed_work_sync(&phba->recover_port);
+
+	/* sessions are no longer valid, so first fail the sessions */
+	iscsi_host_for_each_session(phba->shost, beiscsi_session_fail);
+	beiscsi_disable_port(phba, 0);
 
 	if (state == pci_channel_io_perm_failure) {
 		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
@@ -5515,9 +5594,8 @@ static pci_ers_result_t beiscsi_eeh_reset(struct pci_dev *pdev)
 	pci_set_power_state(pdev, PCI_D0);
 	pci_restore_state(pdev);
 
-	/* Wait for the CHIP Reset to complete */
-	status = be_chk_reset_complete(phba);
-	if (!status) {
+	status = beiscsi_check_fw_rdy(phba);
+	if (status) {
 		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
 			    "BM_%d : EEH Reset Completed\n");
 	} else {
@@ -5532,87 +5610,16 @@ static pci_ers_result_t beiscsi_eeh_reset(struct pci_dev *pdev)
 
 static void beiscsi_eeh_resume(struct pci_dev *pdev)
 {
-	int ret = 0, i;
-	struct be_eq_obj *pbe_eq;
-	struct beiscsi_hba *phba = NULL;
-	struct hwi_controller *phwi_ctrlr;
-	struct hwi_context_memory *phwi_context;
+	struct beiscsi_hba *phba;
+	int ret;
 
 	phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
 	pci_save_state(pdev);
 
-	if (enable_msix)
-		find_num_cpus(phba);
-	else
-		phba->num_cpus = 1;
-
-	if (enable_msix) {
-		beiscsi_msix_enable(phba);
-		if (!phba->msix_enabled)
-			phba->num_cpus = 1;
-	}
-
-	ret = beiscsi_cmd_reset_function(phba);
-	if (ret) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BM_%d : Reset Failed\n");
-		goto ret_err;
-	}
-
-	ret = be_chk_reset_complete(phba);
-	if (ret) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BM_%d : Failed to get out of reset.\n");
-		goto ret_err;
-	}
-
-	beiscsi_get_params(phba);
-	phba->shost->max_id = phba->params.cxns_per_ctrl;
-	phba->shost->can_queue = phba->params.ios_per_ctrl;
-	ret = hwi_init_controller(phba);
-	if (ret) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BM_%d : beiscsi_eeh_resume -"
-			     "Failed to initialize beiscsi_hba.\n");
-		goto ret_err;
-	}
-
-	for (i = 0; i < MAX_MCC_CMD; i++) {
-		init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
-		phba->ctrl.mcc_tag[i] = i + 1;
-		phba->ctrl.mcc_tag_status[i + 1] = 0;
-		phba->ctrl.mcc_tag_available++;
-	}
-
-	phwi_ctrlr = phba->phwi_ctrlr;
-	phwi_context = phwi_ctrlr->phwi_ctxt;
-
-	for (i = 0; i < phba->num_cpus; i++) {
-		pbe_eq = &phwi_context->be_eq[i];
-		irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget,
-				be_iopoll);
-	}
-
-	i = (phba->msix_enabled) ? i : 0;
-	/* Work item for MCC handling */
-	pbe_eq = &phwi_context->be_eq[i];
-	INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
-
-	ret = beiscsi_init_irqs(phba);
-	if (ret < 0) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BM_%d : beiscsi_eeh_resume - "
-			    "Failed to beiscsi_init_irqs\n");
-		goto ret_err;
-	}
-
-	hwi_enable_intr(phba);
-	phba->state &= ~BE_ADAPTER_PCI_ERR;
-
-	return;
-ret_err:
-	beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-		    "BM_%d : AER EEH Resume Failed\n");
+	ret = beiscsi_enable_port(phba);
+	if (ret)
+		__beiscsi_log(phba, KERN_ERR,
+			      "BM_%d : AER EEH resume failed\n");
 }
 
 static int beiscsi_dev_probe(struct pci_dev *pcidev,
@@ -5622,7 +5629,8 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
 	struct hwi_controller *phwi_ctrlr;
 	struct hwi_context_memory *phwi_context;
 	struct be_eq_obj *pbe_eq;
-	int ret = 0, i;
+	unsigned int s_handle;
+	int ret, i;
 
 	ret = beiscsi_enable_pci(pcidev);
 	if (ret < 0) {
@@ -5635,6 +5643,7 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
 	if (!phba) {
 		dev_err(&pcidev->dev,
 			"beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
+		ret = -ENOMEM;
 		goto disable_pci;
 	}
 
@@ -5650,10 +5659,8 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
 	/* Initialize Driver configuration Paramters */
 	beiscsi_hba_attrs_init(phba);
 
-	phba->fw_timeout = false;
 	phba->mac_addr_set = false;
 
-
 	switch (pcidev->device) {
 	case BE_DEVICE_ID1:
 	case OC_DEVICE_ID1:
@@ -5677,39 +5684,26 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
 	ret = be_ctrl_init(phba, pcidev);
 	if (ret) {
 		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BM_%d : beiscsi_dev_probe-"
-			    "Failed in be_ctrl_init\n");
+			    "BM_%d : be_ctrl_init failed\n");
 		goto hba_free;
 	}
 
-	/*
-	 * FUNCTION_RESET should clean up any stale info in FW for this fn
-	 */
-	ret = beiscsi_cmd_reset_function(phba);
-	if (ret) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BM_%d : Reset Failed\n");
-		goto hba_free;
-	}
-	ret = be_chk_reset_complete(phba);
-	if (ret) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BM_%d : Failed to get out of reset.\n");
+	ret = beiscsi_init_sliport(phba);
+	if (ret)
 		goto hba_free;
-	}
 
 	spin_lock_init(&phba->io_sgl_lock);
 	spin_lock_init(&phba->mgmt_sgl_lock);
-	spin_lock_init(&phba->isr_lock);
 	spin_lock_init(&phba->async_pdu_lock);
-	ret = mgmt_get_fw_config(&phba->ctrl, phba);
+	ret = beiscsi_get_fw_config(&phba->ctrl, phba);
 	if (ret != 0) {
 		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 			    "BM_%d : Error getting fw config\n");
 		goto free_port;
 	}
-	mgmt_get_port_name(&phba->ctrl, phba);
+	beiscsi_get_port_name(&phba->ctrl, phba);
 	beiscsi_get_params(phba);
+	beiscsi_set_uer_feature(phba);
 
 	if (enable_msix)
 		find_num_cpus(phba);
@@ -5754,25 +5748,24 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
 		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 			    "BM_%d : beiscsi_dev_probe-"
 			    "Failed to allocate work queue\n");
+		ret = -ENOMEM;
 		goto free_twq;
 	}
 
-	INIT_DELAYED_WORK(&phba->beiscsi_hw_check_task,
-			  beiscsi_hw_health_check);
+	INIT_DELAYED_WORK(&phba->eqd_update, beiscsi_eqd_update_work);
 
 	phwi_ctrlr = phba->phwi_ctrlr;
 	phwi_context = phwi_ctrlr->phwi_ctxt;
 
 	for (i = 0; i < phba->num_cpus; i++) {
 		pbe_eq = &phwi_context->be_eq[i];
-		irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget,
-				be_iopoll);
+		irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll);
 	}
 
 	i = (phba->msix_enabled) ? i : 0;
 	/* Work item for MCC handling */
 	pbe_eq = &phwi_context->be_eq[i];
-	INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
+	INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work);
 
 	ret = beiscsi_init_irqs(phba);
 	if (ret < 0) {
@@ -5783,22 +5776,42 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
 	}
 	hwi_enable_intr(phba);
 
-	if (iscsi_host_add(phba->shost, &phba->pcidev->dev))
+	ret = iscsi_host_add(phba->shost, &phba->pcidev->dev);
+	if (ret)
 		goto free_blkenbld;
 
-	if (beiscsi_setup_boot_info(phba))
-		/*
-		 * log error but continue, because we may not be using
-		 * iscsi boot.
+	/* set online bit after port is operational */
+	set_bit(BEISCSI_HBA_ONLINE, &phba->state);
+	__beiscsi_log(phba, KERN_INFO,
+		      "BM_%d : port online: 0x%lx\n", phba->state);
+
+	INIT_WORK(&phba->boot_work, beiscsi_boot_work);
+	ret = beiscsi_boot_get_shandle(phba, &s_handle);
+	if (ret > 0) {
+		beiscsi_start_boot_work(phba, s_handle);
+		/**
+		 * Set this bit after starting the work to let
+		 * probe handle it first.
+		 * ASYNC event can too schedule this work.
 		 */
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BM_%d : Could not set up "
-			    "iSCSI boot info.\n");
+		set_bit(BEISCSI_HBA_BOOT_FOUND, &phba->state);
+	}
 
-	beiscsi_create_def_ifaces(phba);
-	schedule_delayed_work(&phba->beiscsi_hw_check_task,
-			      msecs_to_jiffies(1000));
+	beiscsi_iface_create_default(phba);
+	schedule_delayed_work(&phba->eqd_update,
+			      msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL));
 
+	INIT_WORK(&phba->sess_work, beiscsi_sess_work);
+	INIT_DELAYED_WORK(&phba->recover_port, beiscsi_recover_port);
+	/**
+	 * Start UE detection here. UE before this will cause stall in probe
+	 * and eventually fail the probe.
+	 */
+	init_timer(&phba->hw_check);
+	phba->hw_check.function = beiscsi_hw_health_check;
+	phba->hw_check.data = (unsigned long)phba;
+	mod_timer(&phba->hw_check,
+		  jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL));
 	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
 		    "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
 	return 0;
@@ -5810,7 +5823,8 @@ free_blkenbld:
 		irq_poll_disable(&pbe_eq->iopoll);
 	}
 free_twq:
-	beiscsi_clean_port(phba);
+	hwi_cleanup_port(phba);
+	beiscsi_cleanup_port(phba);
 	beiscsi_free_mem(phba);
 free_port:
 	pci_free_consistent(phba->pcidev,
@@ -5830,6 +5844,49 @@ disable_pci:
 	return ret;
 }
 
+static void beiscsi_remove(struct pci_dev *pcidev)
+{
+	struct beiscsi_hba *phba = NULL;
+
+	phba = pci_get_drvdata(pcidev);
+	if (!phba) {
+		dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
+		return;
+	}
+
+	/* first stop UE detection before unloading */
+	del_timer_sync(&phba->hw_check);
+	cancel_delayed_work_sync(&phba->recover_port);
+	cancel_work_sync(&phba->sess_work);
+
+	beiscsi_iface_destroy_default(phba);
+	iscsi_host_remove(phba->shost);
+	beiscsi_disable_port(phba, 1);
+
+	/* after cancelling boot_work */
+	iscsi_boot_destroy_kset(phba->boot_struct.boot_kset);
+
+	/* free all resources */
+	destroy_workqueue(phba->wq);
+	beiscsi_cleanup_port(phba);
+	beiscsi_free_mem(phba);
+
+	/* ctrl uninit */
+	beiscsi_unmap_pci_function(phba);
+	pci_free_consistent(phba->pcidev,
+			    phba->ctrl.mbox_mem_alloced.size,
+			    phba->ctrl.mbox_mem_alloced.va,
+			    phba->ctrl.mbox_mem_alloced.dma);
+
+	pci_dev_put(phba->pcidev);
+	iscsi_host_free(phba->shost);
+	pci_disable_pcie_error_reporting(pcidev);
+	pci_set_drvdata(pcidev, NULL);
+	pci_release_regions(pcidev);
+	pci_disable_device(pcidev);
+}
+
+
 static struct pci_error_handlers beiscsi_eeh_handlers = {
 	.error_detected = beiscsi_eeh_err_detected,
 	.slot_reset = beiscsi_eeh_reset,
@@ -5846,9 +5903,9 @@ struct iscsi_transport beiscsi_iscsi_transport = {
 	.create_conn = beiscsi_conn_create,
 	.bind_conn = beiscsi_conn_bind,
 	.destroy_conn = iscsi_conn_teardown,
-	.attr_is_visible = be2iscsi_attr_is_visible,
-	.set_iface_param = be2iscsi_iface_set_param,
-	.get_iface_param = be2iscsi_iface_get_param,
+	.attr_is_visible = beiscsi_attr_is_visible,
+	.set_iface_param = beiscsi_iface_set_param,
+	.get_iface_param = beiscsi_iface_get_param,
 	.set_param = beiscsi_set_param,
 	.get_conn_param = iscsi_conn_get_param,
 	.get_session_param = iscsi_session_get_param,
@@ -5877,7 +5934,6 @@ static struct pci_driver beiscsi_pci_driver = {
 	.err_handler = &beiscsi_eeh_handlers
 };
 
-
 static int __init beiscsi_module_init(void)
 {
 	int ret;
diff --git a/drivers/scsi/be2iscsi/be_main.h b/drivers/scsi/be2iscsi/be_main.h
index 30a4606d9a3b..6376657e45f7 100644
--- a/drivers/scsi/be2iscsi/be_main.h
+++ b/drivers/scsi/be2iscsi/be_main.h
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -7,10 +7,10 @@
  * as published by the Free Software Foundation.  The full GNU General
  * Public License is included in this distribution in the file called COPYING.
  *
- * Written by: Jayamohan Kallickal (jayamohan.kallickal@avagotech.com)
+ * Written by: Jayamohan Kallickal (jayamohan.kallickal@broadcom.com)
  *
  * Contact Information:
- * linux-drivers@avagotech.com
+ * linux-drivers@broadcom.com
  *
  * Emulex
  * 3333 Susan Street
@@ -36,7 +36,7 @@
 #include <scsi/scsi_transport_iscsi.h>
 
 #define DRV_NAME		"be2iscsi"
-#define BUILD_STR		"11.0.0.0"
+#define BUILD_STR		"11.2.0.0"
 #define BE_NAME			"Emulex OneConnect" \
 				"Open-iSCSI Driver version" BUILD_STR
 #define DRV_DESC		BE_NAME " " "Driver"
@@ -82,36 +82,12 @@
 #define BEISCSI_MAX_FRAGS_INIT	192
 #define BE_NUM_MSIX_ENTRIES	1
 
-#define MPU_EP_CONTROL          0
-#define MPU_EP_SEMAPHORE        0xac
-#define BE2_SOFT_RESET          0x5c
-#define BE2_PCI_ONLINE0         0xb0
-#define BE2_PCI_ONLINE1         0xb4
-#define BE2_SET_RESET           0x80
-#define BE2_MPU_IRAM_ONLINE     0x00000080
-
 #define BE_SENSE_INFO_SIZE		258
 #define BE_ISCSI_PDU_HEADER_SIZE	64
 #define BE_MIN_MEM_SIZE			16384
 #define MAX_CMD_SZ			65536
 #define IIOC_SCSI_DATA                  0x05	/* Write Operation */
 
-#define INVALID_SESS_HANDLE	0xFFFFFFFF
-
-/**
- * Adapter States
- **/
-#define BE_ADAPTER_LINK_UP	0x001
-#define BE_ADAPTER_LINK_DOWN	0x002
-#define BE_ADAPTER_PCI_ERR	0x004
-#define BE_ADAPTER_CHECK_BOOT	0x008
-
-
-#define BEISCSI_CLEAN_UNLOAD	0x01
-#define BEISCSI_EEH_UNLOAD	0x02
-
-#define BE_GET_BOOT_RETRIES	45
-#define BE_GET_BOOT_TO		20
 /**
  * hardware needs the async PDU buffers to be posted in multiples of 8
  * So have atleast 8 of them by default
@@ -378,7 +354,6 @@ struct beiscsi_hba {
 	struct sgl_handle **eh_sgl_hndl_base;
 	spinlock_t io_sgl_lock;
 	spinlock_t mgmt_sgl_lock;
-	spinlock_t isr_lock;
 	spinlock_t async_pdu_lock;
 	unsigned int age;
 	struct list_head hba_queue;
@@ -390,7 +365,6 @@ struct beiscsi_hba {
 	struct ulp_cid_info *cid_array_info[BEISCSI_ULP_COUNT];
 	struct iscsi_endpoint **ep_array;
 	struct beiscsi_conn **conn_table;
-	struct iscsi_boot_kset *boot_kset;
 	struct Scsi_Host *shost;
 	struct iscsi_iface *ipv4_iface;
 	struct iscsi_iface *ipv6_iface;
@@ -418,12 +392,33 @@ struct beiscsi_hba {
 		unsigned long ulp_supported;
 	} fw_config;
 
-	unsigned int state;
+	unsigned long state;
+#define BEISCSI_HBA_ONLINE	0
+#define BEISCSI_HBA_LINK_UP	1
+#define BEISCSI_HBA_BOOT_FOUND	2
+#define BEISCSI_HBA_BOOT_WORK	3
+#define BEISCSI_HBA_UER_SUPP	4
+#define BEISCSI_HBA_PCI_ERR	5
+#define BEISCSI_HBA_FW_TIMEOUT	6
+#define BEISCSI_HBA_IN_UE	7
+#define BEISCSI_HBA_IN_TPE	8
+
+/* error bits */
+#define BEISCSI_HBA_IN_ERR	((1 << BEISCSI_HBA_PCI_ERR) | \
+				 (1 << BEISCSI_HBA_FW_TIMEOUT) | \
+				 (1 << BEISCSI_HBA_IN_UE) | \
+				 (1 << BEISCSI_HBA_IN_TPE))
+
 	u8 optic_state;
-	int get_boot;
-	bool fw_timeout;
-	bool ue_detected;
-	struct delayed_work beiscsi_hw_check_task;
+	struct delayed_work eqd_update;
+	/* update EQ delay timer every 1000ms */
+#define BEISCSI_EQD_UPDATE_INTERVAL	1000
+	struct timer_list hw_check;
+	/* check for UE every 1000ms */
+#define BEISCSI_UE_DETECT_INTERVAL	1000
+	u32 ue2rp;
+	struct delayed_work recover_port;
+	struct work_struct sess_work;
 
 	bool mac_addr_set;
 	u8 mac_address[ETH_ALEN];
@@ -435,7 +430,6 @@ struct beiscsi_hba {
 	struct be_ctrl_info ctrl;
 	unsigned int generation;
 	unsigned int interface_handle;
-	struct mgmt_session_info boot_sess;
 	struct invalidate_command_table inv_tbl[128];
 
 	struct be_aic_obj aic_obj[MAX_CPUS];
@@ -444,8 +438,29 @@ struct beiscsi_hba {
 			struct scatterlist *sg,
 			uint32_t num_sg, uint32_t xferlen,
 			uint32_t writedir);
+	struct boot_struct {
+		int retry;
+		unsigned int tag;
+		unsigned int s_handle;
+		struct be_dma_mem nonemb_cmd;
+		enum {
+			BEISCSI_BOOT_REOPEN_SESS = 1,
+			BEISCSI_BOOT_GET_SHANDLE,
+			BEISCSI_BOOT_GET_SINFO,
+			BEISCSI_BOOT_LOGOUT_SESS,
+			BEISCSI_BOOT_CREATE_KSET,
+		} action;
+		struct mgmt_session_info boot_sess;
+		struct iscsi_boot_kset *boot_kset;
+	} boot_struct;
+	struct work_struct boot_work;
 };
 
+#define beiscsi_hba_in_error(phba) ((phba)->state & BEISCSI_HBA_IN_ERR)
+#define beiscsi_hba_is_online(phba) \
+	(!beiscsi_hba_in_error((phba)) && \
+	 test_bit(BEISCSI_HBA_ONLINE, &phba->state))
+
 struct beiscsi_session {
 	struct pci_pool *bhs_pool;
 };
@@ -508,6 +523,7 @@ struct beiscsi_io_task {
 	struct sgl_handle *psgl_handle;
 	struct beiscsi_conn *conn;
 	struct scsi_cmnd *scsi_cmnd;
+	int num_sg;
 	struct hwi_wrb_context *pwrb_context;
 	unsigned int cmd_sn;
 	unsigned int flags;
@@ -592,80 +608,81 @@ struct amap_beiscsi_offload_params {
 	u8 max_recv_data_segment_length[32];
 };
 
-/* void hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
-		struct beiscsi_hba *phba, struct sol_cqe *psol);*/
-
-struct async_pdu_handle {
+struct hd_async_handle {
 	struct list_head link;
 	struct be_bus_address pa;
 	void *pbuffer;
-	unsigned int consumed;
-	unsigned char index;
-	unsigned char is_header;
-	unsigned short cri;
-	unsigned long buffer_len;
+	u32 buffer_len;
+	u16 index;
+	u16 cri;
+	u8 is_header;
+	u8 is_final;
 };
 
-struct hwi_async_entry {
-	struct {
-		unsigned char hdr_received;
-		unsigned char hdr_len;
-		unsigned short bytes_received;
+/**
+ * This has list of async PDUs that are waiting to be processed.
+ * Buffers live in this list for a brief duration before they get
+ * processed and posted back to hardware.
+ * Note that we don't really need one cri_wait_queue per async_entry.
+ * We need one cri_wait_queue per CRI. Its easier to manage if this
+ * is tagged along with the async_entry.
+ */
+struct hd_async_entry {
+	struct cri_wait_queue {
+		unsigned short hdr_len;
+		unsigned int bytes_received;
 		unsigned int bytes_needed;
 		struct list_head list;
-	} wait_queue;
-
-	struct list_head header_busy_list;
-	struct list_head data_busy_list;
+	} wq;
+	/* handles posted to FW resides here */
+	struct hd_async_handle *header;
+	struct hd_async_handle *data;
 };
 
-struct hwi_async_pdu_context {
-	struct {
-		struct be_bus_address pa_base;
-		void *va_base;
-		void *ring_base;
-		struct async_pdu_handle *handle_base;
-
-		unsigned int host_write_ptr;
-		unsigned int ep_read_ptr;
-		unsigned int writables;
-
-		unsigned int free_entries;
-		unsigned int busy_entries;
-
-		struct list_head free_list;
-	} async_header;
+struct hd_async_buf_context {
+	struct be_bus_address pa_base;
+	void *va_base;
+	void *ring_base;
+	struct hd_async_handle *handle_base;
+	u16 free_entries;
+	u32 buffer_size;
+	/**
+	 * Once iSCSI layer finishes processing an async PDU, the
+	 * handles used for the PDU are added to this list.
+	 * They are posted back to FW in groups of 8.
+	 */
+	struct list_head free_list;
+};
 
-	struct {
-		struct be_bus_address pa_base;
-		void *va_base;
-		void *ring_base;
-		struct async_pdu_handle *handle_base;
-
-		unsigned int host_write_ptr;
-		unsigned int ep_read_ptr;
-		unsigned int writables;
-
-		unsigned int free_entries;
-		unsigned int busy_entries;
-		struct list_head free_list;
-	} async_data;
-
-	unsigned int buffer_size;
-	unsigned int num_entries;
+/**
+ * hd_async_context is declared for each ULP supporting iSCSI function.
+ */
+struct hd_async_context {
+	struct hd_async_buf_context async_header;
+	struct hd_async_buf_context async_data;
+	u16 num_entries;
+	/**
+	 * When unsol PDU is in, it needs to be chained till all the bytes are
+	 * received and then processing is done. hd_async_entry is created
+	 * based on the cid_count for each ULP. When unsol PDU comes in based
+	 * on the conn_id it needs to be added to the correct async_entry wq.
+	 * Below defined cid_to_async_cri_map is used to reterive the
+	 * async_cri_map for a particular connection.
+	 *
+	 * This array is initialized after beiscsi_create_wrb_rings returns.
+	 *
+	 * - this method takes more memory space, fixed to 2K
+	 * - any support for connections greater than this the array size needs
+	 * to be incremented
+	 */
 #define BE_GET_ASYNC_CRI_FROM_CID(cid) (pasync_ctx->cid_to_async_cri_map[cid])
 	unsigned short cid_to_async_cri_map[BE_MAX_SESSION];
 	/**
-	 * This is a varying size list! Do not add anything
-	 * after this entry!!
+	 * This is a variable size array. Don`t add anything after this field!!
 	 */
-	struct hwi_async_entry *async_entry;
+	struct hd_async_entry *async_entry;
 };
 
-#define PDUCQE_CODE_MASK	0x0000003F
-#define PDUCQE_DPL_MASK		0xFFFF0000
-#define PDUCQE_INDEX_MASK	0x0000FFFF
-
 struct i_t_dpdu_cqe {
 	u32 dw[4];
 } __packed;
@@ -845,7 +862,6 @@ struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid,
 void
 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle);
 
-void beiscsi_process_all_cqs(struct work_struct *work);
 void beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn,
 				     struct iscsi_task *task);
 
@@ -856,11 +872,6 @@ void hwi_ring_cq_db(struct beiscsi_hba *phba,
 unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq, int budget);
 void beiscsi_process_mcc_cq(struct beiscsi_hba *phba);
 
-static inline bool beiscsi_error(struct beiscsi_hba *phba)
-{
-	return phba->ue_detected || phba->fw_timeout;
-}
-
 struct pdu_nop_out {
 	u32 dw[12];
 };
@@ -1067,11 +1078,18 @@ struct hwi_context_memory {
 	struct be_queue_info be_cq[MAX_CPUS - 1];
 
 	struct be_queue_info *be_wrbq;
+	/**
+	 * Create array of ULP number for below entries as DEFQ
+	 * will be created for both ULP if iSCSI Protocol is
+	 * loaded on both ULP.
+	 */
 	struct be_queue_info be_def_hdrq[BEISCSI_ULP_COUNT];
 	struct be_queue_info be_def_dataq[BEISCSI_ULP_COUNT];
-	struct hwi_async_pdu_context *pasync_ctx[BEISCSI_ULP_COUNT];
+	struct hd_async_context *pasync_ctx[BEISCSI_ULP_COUNT];
 };
 
+void beiscsi_start_boot_work(struct beiscsi_hba *phba, unsigned int s_handle);
+
 /* Logging related definitions */
 #define BEISCSI_LOG_INIT	0x0001	/* Initialization events */
 #define BEISCSI_LOG_MBOX	0x0002	/* Mailbox Events */
diff --git a/drivers/scsi/be2iscsi/be_mgmt.c b/drivers/scsi/be2iscsi/be_mgmt.c
index 83926e221f1e..aebc4ddb3060 100644
--- a/drivers/scsi/be2iscsi/be_mgmt.c
+++ b/drivers/scsi/be2iscsi/be_mgmt.c
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -7,10 +7,10 @@
  * as published by the Free Software Foundation.  The full GNU General
  * Public License is included in this distribution in the file called COPYING.
  *
- * Written by: Jayamohan Kallickal (jayamohan.kallickal@avagotech.com)
+ * Written by: Jayamohan Kallickal (jayamohan.kallickal@broadcom.com)
  *
  * Contact Information:
- * linux-drivers@avagotech.com
+ * linux-drivers@broadcom.com
  *
  * Emulex
  * 3333 Susan Street
@@ -24,139 +24,9 @@
 #include "be_iscsi.h"
 #include "be_main.h"
 
-/* UE Status Low CSR */
-static const char * const desc_ue_status_low[] = {
-	"CEV",
-	"CTX",
-	"DBUF",
-	"ERX",
-	"Host",
-	"MPU",
-	"NDMA",
-	"PTC ",
-	"RDMA ",
-	"RXF ",
-	"RXIPS ",
-	"RXULP0 ",
-	"RXULP1 ",
-	"RXULP2 ",
-	"TIM ",
-	"TPOST ",
-	"TPRE ",
-	"TXIPS ",
-	"TXULP0 ",
-	"TXULP1 ",
-	"UC ",
-	"WDMA ",
-	"TXULP2 ",
-	"HOST1 ",
-	"P0_OB_LINK ",
-	"P1_OB_LINK ",
-	"HOST_GPIO ",
-	"MBOX ",
-	"AXGMAC0",
-	"AXGMAC1",
-	"JTAG",
-	"MPU_INTPEND"
-};
-
-/* UE Status High CSR */
-static const char * const desc_ue_status_hi[] = {
-	"LPCMEMHOST",
-	"MGMT_MAC",
-	"PCS0ONLINE",
-	"MPU_IRAM",
-	"PCS1ONLINE",
-	"PCTL0",
-	"PCTL1",
-	"PMEM",
-	"RR",
-	"TXPB",
-	"RXPP",
-	"XAUI",
-	"TXP",
-	"ARM",
-	"IPC",
-	"HOST2",
-	"HOST3",
-	"HOST4",
-	"HOST5",
-	"HOST6",
-	"HOST7",
-	"HOST8",
-	"HOST9",
-	"NETC",
-	"Unknown",
-	"Unknown",
-	"Unknown",
-	"Unknown",
-	"Unknown",
-	"Unknown",
-	"Unknown",
-	"Unknown"
-};
-
-/*
- * beiscsi_ue_detec()- Detect Unrecoverable Error on adapter
- * @phba: Driver priv structure
- *
- * Read registers linked to UE and check for the UE status
- **/
-void beiscsi_ue_detect(struct beiscsi_hba *phba)
-{
-	uint32_t ue_hi = 0, ue_lo = 0;
-	uint32_t ue_mask_hi = 0, ue_mask_lo = 0;
-	uint8_t i = 0;
-
-	if (phba->ue_detected)
-		return;
-
-	pci_read_config_dword(phba->pcidev,
-			      PCICFG_UE_STATUS_LOW, &ue_lo);
-	pci_read_config_dword(phba->pcidev,
-			      PCICFG_UE_STATUS_MASK_LOW,
-			      &ue_mask_lo);
-	pci_read_config_dword(phba->pcidev,
-			      PCICFG_UE_STATUS_HIGH,
-			      &ue_hi);
-	pci_read_config_dword(phba->pcidev,
-			      PCICFG_UE_STATUS_MASK_HI,
-			      &ue_mask_hi);
-
-	ue_lo = (ue_lo & ~ue_mask_lo);
-	ue_hi = (ue_hi & ~ue_mask_hi);
-
-
-	if (ue_lo || ue_hi) {
-		phba->ue_detected = true;
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
-			    "BG_%d : Error detected on the adapter\n");
-	}
-
-	if (ue_lo) {
-		for (i = 0; ue_lo; ue_lo >>= 1, i++) {
-			if (ue_lo & 1)
-				beiscsi_log(phba, KERN_ERR,
-					    BEISCSI_LOG_CONFIG,
-					    "BG_%d : UE_LOW %s bit set\n",
-					    desc_ue_status_low[i]);
-		}
-	}
-
-	if (ue_hi) {
-		for (i = 0; ue_hi; ue_hi >>= 1, i++) {
-			if (ue_hi & 1)
-				beiscsi_log(phba, KERN_ERR,
-					    BEISCSI_LOG_CONFIG,
-					    "BG_%d : UE_HIGH %s bit set\n",
-					    desc_ue_status_hi[i]);
-		}
-	}
-}
-
-int be_cmd_modify_eq_delay(struct beiscsi_hba *phba,
-		 struct be_set_eqd *set_eqd, int num)
+int beiscsi_modify_eq_delay(struct beiscsi_hba *phba,
+			    struct be_set_eqd *set_eqd,
+			    int num)
 {
 	struct be_ctrl_info *ctrl = &phba->ctrl;
 	struct be_mcc_wrb *wrb;
@@ -174,7 +44,7 @@ int be_cmd_modify_eq_delay(struct beiscsi_hba *phba,
 	req = embedded_payload(wrb);
 	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
 	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_COMMON,
-		OPCODE_COMMON_MODIFY_EQ_DELAY, sizeof(*req));
+			   OPCODE_COMMON_MODIFY_EQ_DELAY, sizeof(*req));
 
 	req->num_eq = cpu_to_le32(num);
 	for (i = 0; i < num; i++) {
@@ -184,386 +54,13 @@ int be_cmd_modify_eq_delay(struct beiscsi_hba *phba,
 				cpu_to_le32(set_eqd[i].delay_multiplier);
 	}
 
+	/* ignore the completion of this mbox command */
+	set_bit(MCC_TAG_STATE_IGNORE, &ctrl->ptag_state[tag].tag_state);
 	be_mcc_notify(phba, tag);
 	mutex_unlock(&ctrl->mbox_lock);
 	return tag;
 }
 
-/**
- * mgmt_reopen_session()- Reopen a session based on reopen_type
- * @phba: Device priv structure instance
- * @reopen_type: Type of reopen_session FW should do.
- * @sess_handle: Session Handle of the session to be re-opened
- *
- * return
- *	the TAG used for MBOX Command
- *
- **/
-unsigned int mgmt_reopen_session(struct beiscsi_hba *phba,
-				  unsigned int reopen_type,
-				  unsigned int sess_handle)
-{
-	struct be_ctrl_info *ctrl = &phba->ctrl;
-	struct be_mcc_wrb *wrb;
-	struct be_cmd_reopen_session_req *req;
-	unsigned int tag;
-
-	beiscsi_log(phba, KERN_INFO,
-		    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
-		    "BG_%d : In bescsi_get_boot_target\n");
-
-	mutex_lock(&ctrl->mbox_lock);
-	wrb = alloc_mcc_wrb(phba, &tag);
-	if (!wrb) {
-		mutex_unlock(&ctrl->mbox_lock);
-		return 0;
-	}
-
-	req = embedded_payload(wrb);
-	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
-	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI_INI,
-			   OPCODE_ISCSI_INI_DRIVER_REOPEN_ALL_SESSIONS,
-			   sizeof(struct be_cmd_reopen_session_resp));
-
-	/* set the reopen_type,sess_handle */
-	req->reopen_type = reopen_type;
-	req->session_handle = sess_handle;
-
-	be_mcc_notify(phba, tag);
-	mutex_unlock(&ctrl->mbox_lock);
-	return tag;
-}
-
-unsigned int mgmt_get_boot_target(struct beiscsi_hba *phba)
-{
-	struct be_ctrl_info *ctrl = &phba->ctrl;
-	struct be_mcc_wrb *wrb;
-	struct be_cmd_get_boot_target_req *req;
-	unsigned int tag;
-
-	beiscsi_log(phba, KERN_INFO,
-		    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
-		    "BG_%d : In bescsi_get_boot_target\n");
-
-	mutex_lock(&ctrl->mbox_lock);
-	wrb = alloc_mcc_wrb(phba, &tag);
-	if (!wrb) {
-		mutex_unlock(&ctrl->mbox_lock);
-		return 0;
-	}
-
-	req = embedded_payload(wrb);
-	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
-	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI_INI,
-			   OPCODE_ISCSI_INI_BOOT_GET_BOOT_TARGET,
-			   sizeof(struct be_cmd_get_boot_target_resp));
-
-	be_mcc_notify(phba, tag);
-	mutex_unlock(&ctrl->mbox_lock);
-	return tag;
-}
-
-unsigned int mgmt_get_session_info(struct beiscsi_hba *phba,
-				   u32 boot_session_handle,
-				   struct be_dma_mem *nonemb_cmd)
-{
-	struct be_ctrl_info *ctrl = &phba->ctrl;
-	struct be_mcc_wrb *wrb;
-	unsigned int tag;
-	struct  be_cmd_get_session_req *req;
-	struct be_cmd_get_session_resp *resp;
-	struct be_sge *sge;
-
-	beiscsi_log(phba, KERN_INFO,
-		    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
-		    "BG_%d : In beiscsi_get_session_info\n");
-
-	mutex_lock(&ctrl->mbox_lock);
-	wrb = alloc_mcc_wrb(phba, &tag);
-	if (!wrb) {
-		mutex_unlock(&ctrl->mbox_lock);
-		return 0;
-	}
-
-	nonemb_cmd->size = sizeof(*resp);
-	req = nonemb_cmd->va;
-	memset(req, 0, sizeof(*req));
-	sge = nonembedded_sgl(wrb);
-	be_wrb_hdr_prepare(wrb, sizeof(*req), false, 1);
-	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI_INI,
-			   OPCODE_ISCSI_INI_SESSION_GET_A_SESSION,
-			   sizeof(*resp));
-	req->session_handle = boot_session_handle;
-	sge->pa_hi = cpu_to_le32(upper_32_bits(nonemb_cmd->dma));
-	sge->pa_lo = cpu_to_le32(nonemb_cmd->dma & 0xFFFFFFFF);
-	sge->len = cpu_to_le32(nonemb_cmd->size);
-
-	be_mcc_notify(phba, tag);
-	mutex_unlock(&ctrl->mbox_lock);
-	return tag;
-}
-
-/**
- * mgmt_get_port_name()- Get port name for the function
- * @ctrl: ptr to Ctrl Info
- * @phba: ptr to the dev priv structure
- *
- * Get the alphanumeric character for port
- *
- **/
-int mgmt_get_port_name(struct be_ctrl_info *ctrl,
-		       struct beiscsi_hba *phba)
-{
-	int ret = 0;
-	struct be_mcc_wrb *wrb;
-	struct be_cmd_get_port_name *ioctl;
-
-	mutex_lock(&ctrl->mbox_lock);
-	wrb = wrb_from_mbox(&ctrl->mbox_mem);
-	memset(wrb, 0, sizeof(*wrb));
-	ioctl = embedded_payload(wrb);
-
-	be_wrb_hdr_prepare(wrb, sizeof(*ioctl), true, 0);
-	be_cmd_hdr_prepare(&ioctl->h.req_hdr, CMD_SUBSYSTEM_COMMON,
-			   OPCODE_COMMON_GET_PORT_NAME,
-			   EMBED_MBX_MAX_PAYLOAD_SIZE);
-	ret = be_mbox_notify(ctrl);
-	phba->port_name = 0;
-	if (!ret) {
-		phba->port_name = ioctl->p.resp.port_names >>
-				  (phba->fw_config.phys_port * 8) & 0xff;
-	} else {
-		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
-			    "BG_%d : GET_PORT_NAME ret 0x%x status 0x%x\n",
-			    ret, ioctl->h.resp_hdr.status);
-	}
-
-	if (phba->port_name == 0)
-		phba->port_name = '?';
-
-	mutex_unlock(&ctrl->mbox_lock);
-	return ret;
-}
-
-/**
- * mgmt_get_fw_config()- Get the FW config for the function
- * @ctrl: ptr to Ctrl Info
- * @phba: ptr to the dev priv structure
- *
- * Get the FW config and resources available for the function.
- * The resources are created based on the count received here.
- *
- * return
- *	Success: 0
- *	Failure: Non-Zero Value
- **/
-int mgmt_get_fw_config(struct be_ctrl_info *ctrl,
-				struct beiscsi_hba *phba)
-{
-	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
-	struct be_fw_cfg *pfw_cfg = embedded_payload(wrb);
-	uint32_t cid_count, icd_count;
-	int status = -EINVAL;
-	uint8_t ulp_num = 0;
-
-	mutex_lock(&ctrl->mbox_lock);
-	memset(wrb, 0, sizeof(*wrb));
-	be_wrb_hdr_prepare(wrb, sizeof(*pfw_cfg), true, 0);
-
-	be_cmd_hdr_prepare(&pfw_cfg->hdr, CMD_SUBSYSTEM_COMMON,
-			   OPCODE_COMMON_QUERY_FIRMWARE_CONFIG,
-			   EMBED_MBX_MAX_PAYLOAD_SIZE);
-
-	if (be_mbox_notify(ctrl)) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BG_%d : Failed in mgmt_get_fw_config\n");
-		goto fail_init;
-	}
-
-	/* FW response formats depend on port id */
-	phba->fw_config.phys_port = pfw_cfg->phys_port;
-	if (phba->fw_config.phys_port >= BEISCSI_PHYS_PORT_MAX) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BG_%d : invalid physical port id %d\n",
-			    phba->fw_config.phys_port);
-		goto fail_init;
-	}
-
-	/* populate and check FW config against min and max values */
-	if (!is_chip_be2_be3r(phba)) {
-		phba->fw_config.eqid_count = pfw_cfg->eqid_count;
-		phba->fw_config.cqid_count = pfw_cfg->cqid_count;
-		if (phba->fw_config.eqid_count == 0 ||
-		    phba->fw_config.eqid_count > 2048) {
-			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-				    "BG_%d : invalid EQ count %d\n",
-				    phba->fw_config.eqid_count);
-			goto fail_init;
-		}
-		if (phba->fw_config.cqid_count == 0 ||
-		    phba->fw_config.cqid_count > 4096) {
-			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-				    "BG_%d : invalid CQ count %d\n",
-				    phba->fw_config.cqid_count);
-			goto fail_init;
-		}
-		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
-			    "BG_%d : EQ_Count : %d CQ_Count : %d\n",
-			    phba->fw_config.eqid_count,
-			    phba->fw_config.cqid_count);
-	}
-
-	/**
-	 * Check on which all ULP iSCSI Protocol is loaded.
-	 * Set the Bit for those ULP. This set flag is used
-	 * at all places in the code to check on which ULP
-	 * iSCSi Protocol is loaded
-	 **/
-	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
-		if (pfw_cfg->ulp[ulp_num].ulp_mode &
-		    BEISCSI_ULP_ISCSI_INI_MODE) {
-			set_bit(ulp_num, &phba->fw_config.ulp_supported);
-
-			/* Get the CID, ICD and Chain count for each ULP */
-			phba->fw_config.iscsi_cid_start[ulp_num] =
-				pfw_cfg->ulp[ulp_num].sq_base;
-			phba->fw_config.iscsi_cid_count[ulp_num] =
-				pfw_cfg->ulp[ulp_num].sq_count;
-
-			phba->fw_config.iscsi_icd_start[ulp_num] =
-				pfw_cfg->ulp[ulp_num].icd_base;
-			phba->fw_config.iscsi_icd_count[ulp_num] =
-				pfw_cfg->ulp[ulp_num].icd_count;
-
-			phba->fw_config.iscsi_chain_start[ulp_num] =
-				pfw_cfg->chain_icd[ulp_num].chain_base;
-			phba->fw_config.iscsi_chain_count[ulp_num] =
-				pfw_cfg->chain_icd[ulp_num].chain_count;
-
-			beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
-				    "BG_%d : Function loaded on ULP : %d\n"
-				    "\tiscsi_cid_count : %d\n"
-				    "\tiscsi_cid_start : %d\n"
-				    "\t iscsi_icd_count : %d\n"
-				    "\t iscsi_icd_start : %d\n",
-				    ulp_num,
-				    phba->fw_config.
-				    iscsi_cid_count[ulp_num],
-				    phba->fw_config.
-				    iscsi_cid_start[ulp_num],
-				    phba->fw_config.
-				    iscsi_icd_count[ulp_num],
-				    phba->fw_config.
-				    iscsi_icd_start[ulp_num]);
-		}
-	}
-
-	if (phba->fw_config.ulp_supported == 0) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BG_%d : iSCSI initiator mode not set: ULP0 %x ULP1 %x\n",
-			    pfw_cfg->ulp[BEISCSI_ULP0].ulp_mode,
-			    pfw_cfg->ulp[BEISCSI_ULP1].ulp_mode);
-		goto fail_init;
-	}
-
-	/**
-	 * ICD is shared among ULPs. Use icd_count of any one loaded ULP
-	 **/
-	for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
-		if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
-			break;
-	icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
-	if (icd_count == 0 || icd_count > 65536) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BG_%d: invalid ICD count %d\n", icd_count);
-		goto fail_init;
-	}
-
-	cid_count = BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP0) +
-		    BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP1);
-	if (cid_count == 0 || cid_count > 4096) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BG_%d: invalid CID count %d\n", cid_count);
-		goto fail_init;
-	}
-
-	/**
-	 * Check FW is dual ULP aware i.e. can handle either
-	 * of the protocols.
-	 */
-	phba->fw_config.dual_ulp_aware = (pfw_cfg->function_mode &
-					  BEISCSI_FUNC_DUA_MODE);
-
-	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
-		    "BG_%d : DUA Mode : 0x%x\n",
-		    phba->fw_config.dual_ulp_aware);
-
-	/* all set, continue using this FW config */
-	status = 0;
-fail_init:
-	mutex_unlock(&ctrl->mbox_lock);
-	return status;
-}
-
-int mgmt_check_supported_fw(struct be_ctrl_info *ctrl,
-				      struct beiscsi_hba *phba)
-{
-	struct be_dma_mem nonemb_cmd;
-	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
-	struct be_mgmt_controller_attributes *req;
-	struct be_sge *sge = nonembedded_sgl(wrb);
-	int status = 0;
-
-	nonemb_cmd.va = pci_alloc_consistent(ctrl->pdev,
-				sizeof(struct be_mgmt_controller_attributes),
-				&nonemb_cmd.dma);
-	if (nonemb_cmd.va == NULL) {
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BG_%d : Failed to allocate memory for "
-			    "mgmt_check_supported_fw\n");
-		return -ENOMEM;
-	}
-	nonemb_cmd.size = sizeof(struct be_mgmt_controller_attributes);
-	req = nonemb_cmd.va;
-	memset(req, 0, sizeof(*req));
-	mutex_lock(&ctrl->mbox_lock);
-	memset(wrb, 0, sizeof(*wrb));
-	be_wrb_hdr_prepare(wrb, sizeof(*req), false, 1);
-	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_COMMON,
-			   OPCODE_COMMON_GET_CNTL_ATTRIBUTES, sizeof(*req));
-	sge->pa_hi = cpu_to_le32(upper_32_bits(nonemb_cmd.dma));
-	sge->pa_lo = cpu_to_le32(nonemb_cmd.dma & 0xFFFFFFFF);
-	sge->len = cpu_to_le32(nonemb_cmd.size);
-	status = be_mbox_notify(ctrl);
-	if (!status) {
-		struct be_mgmt_controller_attributes_resp *resp = nonemb_cmd.va;
-		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
-			    "BG_%d : Firmware Version of CMD : %s\n"
-			    "Firmware Version is : %s\n"
-			    "Developer Build, not performing version check...\n",
-			    resp->params.hba_attribs
-			    .flashrom_version_string,
-			    resp->params.hba_attribs.
-			    firmware_version_string);
-
-		phba->fw_config.iscsi_features =
-				resp->params.hba_attribs.iscsi_features;
-		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
-			    "BM_%d : phba->fw_config.iscsi_features = %d\n",
-			    phba->fw_config.iscsi_features);
-		memcpy(phba->fw_ver_str, resp->params.hba_attribs.
-		       firmware_version_string, BEISCSI_VER_STRLEN);
-	} else
-		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
-			    "BG_%d :  Failed in mgmt_check_supported_fw\n");
-	mutex_unlock(&ctrl->mbox_lock);
-	if (nonemb_cmd.va)
-		pci_free_consistent(ctrl->pdev, nonemb_cmd.size,
-				    nonemb_cmd.va, nonemb_cmd.dma);
-
-	return status;
-}
-
 unsigned int mgmt_vendor_specific_fw_cmd(struct be_ctrl_info *ctrl,
 					 struct beiscsi_hba *phba,
 					 struct bsg_job *job,
@@ -609,7 +106,7 @@ unsigned int mgmt_vendor_specific_fw_cmd(struct be_ctrl_info *ctrl,
 			    bsg_req->rqst_data.h_vendor.vendor_cmd[0]);
 
 		mutex_unlock(&ctrl->mbox_lock);
-		return -ENOSYS;
+		return -EPERM;
 	}
 
 	wrb = alloc_mcc_wrb(phba, &tag);
@@ -631,48 +128,6 @@ unsigned int mgmt_vendor_specific_fw_cmd(struct be_ctrl_info *ctrl,
 	return tag;
 }
 
-/**
- * mgmt_epfw_cleanup()- Inform FW to cleanup data structures.
- * @phba: pointer to dev priv structure
- * @ulp_num: ULP number.
- *
- * return
- *	Success: 0
- *	Failure: Non-Zero Value
- **/
-int mgmt_epfw_cleanup(struct beiscsi_hba *phba, unsigned short ulp_num)
-{
-	struct be_ctrl_info *ctrl = &phba->ctrl;
-	struct be_mcc_wrb *wrb;
-	struct iscsi_cleanup_req *req;
-	unsigned int tag;
-	int status;
-
-	mutex_lock(&ctrl->mbox_lock);
-	wrb = alloc_mcc_wrb(phba, &tag);
-	if (!wrb) {
-		mutex_unlock(&ctrl->mbox_lock);
-		return -EBUSY;
-	}
-
-	req = embedded_payload(wrb);
-	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
-	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI,
-			   OPCODE_COMMON_ISCSI_CLEANUP, sizeof(*req));
-
-	req->chute = (1 << ulp_num);
-	req->hdr_ring_id = cpu_to_le16(HWI_GET_DEF_HDRQ_ID(phba, ulp_num));
-	req->data_ring_id = cpu_to_le16(HWI_GET_DEF_BUFQ_ID(phba, ulp_num));
-
-	be_mcc_notify(phba, tag);
-	status = be_mcc_compl_poll(phba, tag);
-	if (status)
-		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
-			    "BG_%d : mgmt_epfw_cleanup , FAILED\n");
-	mutex_unlock(&ctrl->mbox_lock);
-	return status;
-}
-
 unsigned int  mgmt_invalidate_icds(struct beiscsi_hba *phba,
 				struct invalidate_command_table *inv_tbl,
 				unsigned int num_invalidate, unsigned int cid,
@@ -844,7 +299,7 @@ int mgmt_open_connection(struct beiscsi_hba *phba,
 			   nonemb_cmd->size);
 	if (dst_addr->sa_family == PF_INET) {
 		__be32 s_addr = daddr_in->sin_addr.s_addr;
-		req->ip_address.ip_type = BE2_IPV4;
+		req->ip_address.ip_type = BEISCSI_IP_TYPE_V4;
 		req->ip_address.addr[0] = s_addr & 0x000000ff;
 		req->ip_address.addr[1] = (s_addr & 0x0000ff00) >> 8;
 		req->ip_address.addr[2] = (s_addr & 0x00ff0000) >> 16;
@@ -852,17 +307,17 @@ int mgmt_open_connection(struct beiscsi_hba *phba,
 		req->tcp_port = ntohs(daddr_in->sin_port);
 		beiscsi_ep->dst_addr = daddr_in->sin_addr.s_addr;
 		beiscsi_ep->dst_tcpport = ntohs(daddr_in->sin_port);
-		beiscsi_ep->ip_type = BE2_IPV4;
+		beiscsi_ep->ip_type = BEISCSI_IP_TYPE_V4;
 	} else {
 		/* else its PF_INET6 family */
-		req->ip_address.ip_type = BE2_IPV6;
+		req->ip_address.ip_type = BEISCSI_IP_TYPE_V6;
 		memcpy(&req->ip_address.addr,
 		       &daddr_in6->sin6_addr.in6_u.u6_addr8, 16);
 		req->tcp_port = ntohs(daddr_in6->sin6_port);
 		beiscsi_ep->dst_tcpport = ntohs(daddr_in6->sin6_port);
 		memcpy(&beiscsi_ep->dst6_addr,
 		       &daddr_in6->sin6_addr.in6_u.u6_addr8, 16);
-		beiscsi_ep->ip_type = BE2_IPV6;
+		beiscsi_ep->ip_type = BEISCSI_IP_TYPE_V6;
 	}
 	req->cid = cid;
 	i = phba->nxt_cqid++;
@@ -883,7 +338,7 @@ int mgmt_open_connection(struct beiscsi_hba *phba,
 
 	if (!is_chip_be2_be3r(phba)) {
 		req->hdr.version = MBX_CMD_VER1;
-		req->tcp_window_size = 0;
+		req->tcp_window_size = 0x8000;
 		req->tcp_window_scale_count = 2;
 	}
 
@@ -892,44 +347,6 @@ int mgmt_open_connection(struct beiscsi_hba *phba,
 	return tag;
 }
 
-unsigned int mgmt_get_all_if_id(struct beiscsi_hba *phba)
-{
-	struct be_ctrl_info *ctrl = &phba->ctrl;
-	struct be_mcc_wrb *wrb;
-	struct be_cmd_get_all_if_id_req *req;
-	struct be_cmd_get_all_if_id_req *pbe_allid;
-	unsigned int tag;
-	int status = 0;
-
-	if (mutex_lock_interruptible(&ctrl->mbox_lock))
-		return -EINTR;
-	wrb = alloc_mcc_wrb(phba, &tag);
-	if (!wrb) {
-		mutex_unlock(&ctrl->mbox_lock);
-		return -ENOMEM;
-	}
-
-	req = embedded_payload(wrb);
-	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
-	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI,
-			   OPCODE_COMMON_ISCSI_NTWK_GET_ALL_IF_ID,
-			   sizeof(*req));
-	be_mcc_notify(phba, tag);
-	mutex_unlock(&ctrl->mbox_lock);
-
-	status = beiscsi_mccq_compl_wait(phba, tag, &wrb, NULL);
-	if (status) {
-		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
-			    "BG_%d : Failed in mgmt_get_all_if_id\n");
-		return -EBUSY;
-	}
-
-	pbe_allid = embedded_payload(wrb);
-	phba->interface_handle = pbe_allid->if_hndl_list[0];
-
-	return status;
-}
-
 /*
  * mgmt_exec_nonemb_cmd()- Execute Non Embedded MBX Cmd
  * @phba: Driver priv structure
@@ -1001,72 +418,68 @@ static int mgmt_alloc_cmd_data(struct beiscsi_hba *phba, struct be_dma_mem *cmd,
 	}
 	cmd->size = size;
 	be_cmd_hdr_prepare(cmd->va, CMD_SUBSYSTEM_ISCSI, iscsi_cmd, size);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BG_%d : subsystem iSCSI cmd %d size %d\n",
+		    iscsi_cmd, size);
 	return 0;
 }
 
-static int
-mgmt_static_ip_modify(struct beiscsi_hba *phba,
-		      struct be_cmd_get_if_info_resp *if_info,
-		      struct iscsi_iface_param_info *ip_param,
-		      struct iscsi_iface_param_info *subnet_param,
-		      uint32_t ip_action)
+unsigned int beiscsi_if_get_handle(struct beiscsi_hba *phba)
 {
-	struct be_cmd_set_ip_addr_req *req;
-	struct be_dma_mem nonemb_cmd;
-	uint32_t ip_type;
-	int rc;
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct be_mcc_wrb *wrb;
+	struct be_cmd_get_all_if_id_req *req;
+	struct be_cmd_get_all_if_id_req *pbe_allid;
+	unsigned int tag;
+	int status = 0;
 
-	rc = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
-				 OPCODE_COMMON_ISCSI_NTWK_MODIFY_IP_ADDR,
-				 sizeof(*req));
-	if (rc)
-		return rc;
+	if (mutex_lock_interruptible(&ctrl->mbox_lock))
+		return -EINTR;
+	wrb = alloc_mcc_wrb(phba, &tag);
+	if (!wrb) {
+		mutex_unlock(&ctrl->mbox_lock);
+		return -ENOMEM;
+	}
 
-	ip_type = (ip_param->param == ISCSI_NET_PARAM_IPV6_ADDR) ?
-		BE2_IPV6 : BE2_IPV4 ;
+	req = embedded_payload(wrb);
+	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI,
+			   OPCODE_COMMON_ISCSI_NTWK_GET_ALL_IF_ID,
+			   sizeof(*req));
+	be_mcc_notify(phba, tag);
+	mutex_unlock(&ctrl->mbox_lock);
 
-	req = nonemb_cmd.va;
-	req->ip_params.record_entry_count = 1;
-	req->ip_params.ip_record.action = ip_action;
-	req->ip_params.ip_record.interface_hndl =
-		phba->interface_handle;
-	req->ip_params.ip_record.ip_addr.size_of_structure =
-		sizeof(struct be_ip_addr_subnet_format);
-	req->ip_params.ip_record.ip_addr.ip_type = ip_type;
+	status = beiscsi_mccq_compl_wait(phba, tag, &wrb, NULL);
+	if (status) {
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BG_%d : %s failed: %d\n", __func__, status);
+		return -EBUSY;
+	}
 
-	if (ip_action == IP_ACTION_ADD) {
-		memcpy(req->ip_params.ip_record.ip_addr.addr, ip_param->value,
-		       sizeof(req->ip_params.ip_record.ip_addr.addr));
+	pbe_allid = embedded_payload(wrb);
+	/* we now support only one interface per function */
+	phba->interface_handle = pbe_allid->if_hndl_list[0];
 
-		if (subnet_param)
-			memcpy(req->ip_params.ip_record.ip_addr.subnet_mask,
-			       subnet_param->value,
-			       sizeof(req->ip_params.ip_record.ip_addr.subnet_mask));
-	} else {
-		memcpy(req->ip_params.ip_record.ip_addr.addr,
-		       if_info->ip_addr.addr,
-		       sizeof(req->ip_params.ip_record.ip_addr.addr));
+	return status;
+}
 
-		memcpy(req->ip_params.ip_record.ip_addr.subnet_mask,
-		       if_info->ip_addr.subnet_mask,
-		       sizeof(req->ip_params.ip_record.ip_addr.subnet_mask));
-	}
+static inline bool beiscsi_if_zero_ip(u8 *ip, u32 ip_type)
+{
+	u32 len;
 
-	rc = mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
-	if (rc < 0)
-		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
-			    "BG_%d : Failed to Modify existing IP Address\n");
-	return rc;
+	len = (ip_type < BEISCSI_IP_TYPE_V6) ? IP_V4_LEN : IP_V6_LEN;
+	while (len && !ip[len - 1])
+		len--;
+	return (len == 0);
 }
 
-static int mgmt_modify_gateway(struct beiscsi_hba *phba, uint8_t *gt_addr,
-			       uint32_t gtway_action, uint32_t param_len)
+static int beiscsi_if_mod_gw(struct beiscsi_hba *phba,
+			     u32 action, u32 ip_type, u8 *gw)
 {
 	struct be_cmd_set_def_gateway_req *req;
 	struct be_dma_mem nonemb_cmd;
 	int rt_val;
 
-
 	rt_val = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
 				OPCODE_COMMON_ISCSI_NTWK_MODIFY_DEFAULT_GATEWAY,
 				sizeof(*req));
@@ -1074,200 +487,300 @@ static int mgmt_modify_gateway(struct beiscsi_hba *phba, uint8_t *gt_addr,
 		return rt_val;
 
 	req = nonemb_cmd.va;
-	req->action = gtway_action;
-	req->ip_addr.ip_type = BE2_IPV4;
+	req->action = action;
+	req->ip_addr.ip_type = ip_type;
+	memcpy(req->ip_addr.addr, gw,
+	       (ip_type < BEISCSI_IP_TYPE_V6) ? IP_V4_LEN : IP_V6_LEN);
+	return mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
+}
 
-	memcpy(req->ip_addr.addr, gt_addr, sizeof(req->ip_addr.addr));
+int beiscsi_if_set_gw(struct beiscsi_hba *phba, u32 ip_type, u8 *gw)
+{
+	struct be_cmd_get_def_gateway_resp gw_resp;
+	int rt_val;
 
-	return mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
+	memset(&gw_resp, 0, sizeof(gw_resp));
+	rt_val = beiscsi_if_get_gw(phba, ip_type, &gw_resp);
+	if (rt_val) {
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BG_%d : Failed to Get Gateway Addr\n");
+		return rt_val;
+	}
+
+	if (!beiscsi_if_zero_ip(gw_resp.ip_addr.addr, ip_type)) {
+		rt_val = beiscsi_if_mod_gw(phba, IP_ACTION_DEL, ip_type,
+					   gw_resp.ip_addr.addr);
+		if (rt_val) {
+			beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+				    "BG_%d : Failed to clear Gateway Addr Set\n");
+			return rt_val;
+		}
+	}
+
+	rt_val = beiscsi_if_mod_gw(phba, IP_ACTION_ADD, ip_type, gw);
+	if (rt_val)
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BG_%d : Failed to Set Gateway Addr\n");
+
+	return rt_val;
 }
 
-int mgmt_set_ip(struct beiscsi_hba *phba,
-		struct iscsi_iface_param_info *ip_param,
-		struct iscsi_iface_param_info *subnet_param,
-		uint32_t boot_proto)
+int beiscsi_if_get_gw(struct beiscsi_hba *phba, u32 ip_type,
+		      struct be_cmd_get_def_gateway_resp *resp)
 {
-	struct be_cmd_get_def_gateway_resp gtway_addr_set;
-	struct be_cmd_get_if_info_resp *if_info;
-	struct be_cmd_set_dhcp_req *dhcpreq;
-	struct be_cmd_rel_dhcp_req *reldhcp;
+	struct be_cmd_get_def_gateway_req *req;
 	struct be_dma_mem nonemb_cmd;
-	uint8_t *gtway_addr;
-	uint32_t ip_type;
 	int rc;
 
-	rc = mgmt_get_all_if_id(phba);
+	rc = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
+				 OPCODE_COMMON_ISCSI_NTWK_GET_DEFAULT_GATEWAY,
+				 sizeof(*resp));
 	if (rc)
 		return rc;
 
-	ip_type = (ip_param->param == ISCSI_NET_PARAM_IPV6_ADDR) ?
-		BE2_IPV6 : BE2_IPV4 ;
+	req = nonemb_cmd.va;
+	req->ip_type = ip_type;
+
+	return mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, resp,
+				    sizeof(*resp));
+}
+
+static int
+beiscsi_if_clr_ip(struct beiscsi_hba *phba,
+		  struct be_cmd_get_if_info_resp *if_info)
+{
+	struct be_cmd_set_ip_addr_req *req;
+	struct be_dma_mem nonemb_cmd;
+	int rc;
 
-	rc = mgmt_get_if_info(phba, ip_type, &if_info);
+	rc = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
+				 OPCODE_COMMON_ISCSI_NTWK_MODIFY_IP_ADDR,
+				 sizeof(*req));
 	if (rc)
 		return rc;
 
-	if (boot_proto == ISCSI_BOOTPROTO_DHCP) {
-		if (if_info->dhcp_state) {
-			beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
-				    "BG_%d : DHCP Already Enabled\n");
-			goto exit;
-		}
-		/* The ip_param->len is 1 in DHCP case. Setting
-		   proper IP len as this it is used while
-		   freeing the Static IP.
-		 */
-		ip_param->len = (ip_param->param == ISCSI_NET_PARAM_IPV6_ADDR) ?
-				IP_V6_LEN : IP_V4_LEN;
-
-	} else {
-		if (if_info->dhcp_state) {
+	req = nonemb_cmd.va;
+	req->ip_params.record_entry_count = 1;
+	req->ip_params.ip_record.action = IP_ACTION_DEL;
+	req->ip_params.ip_record.interface_hndl =
+		phba->interface_handle;
+	req->ip_params.ip_record.ip_addr.size_of_structure =
+		sizeof(struct be_ip_addr_subnet_format);
+	req->ip_params.ip_record.ip_addr.ip_type = if_info->ip_addr.ip_type;
+	memcpy(req->ip_params.ip_record.ip_addr.addr,
+	       if_info->ip_addr.addr,
+	       sizeof(if_info->ip_addr.addr));
+	memcpy(req->ip_params.ip_record.ip_addr.subnet_mask,
+	       if_info->ip_addr.subnet_mask,
+	       sizeof(if_info->ip_addr.subnet_mask));
+	rc = mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
+	if (rc < 0 || req->ip_params.ip_record.status) {
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BG_%d : failed to clear IP: rc %d status %d\n",
+			    rc, req->ip_params.ip_record.status);
+	}
+	return rc;
+}
 
-			memset(if_info, 0, sizeof(*if_info));
-			rc = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
-				OPCODE_COMMON_ISCSI_NTWK_REL_STATELESS_IP_ADDR,
-				sizeof(*reldhcp));
+static int
+beiscsi_if_set_ip(struct beiscsi_hba *phba, u8 *ip,
+		  u8 *subnet, u32 ip_type)
+{
+	struct be_cmd_set_ip_addr_req *req;
+	struct be_dma_mem nonemb_cmd;
+	uint32_t ip_len;
+	int rc;
 
-			if (rc)
-				goto exit;
+	rc = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
+				 OPCODE_COMMON_ISCSI_NTWK_MODIFY_IP_ADDR,
+				 sizeof(*req));
+	if (rc)
+		return rc;
 
-			reldhcp = nonemb_cmd.va;
-			reldhcp->interface_hndl = phba->interface_handle;
-			reldhcp->ip_type = ip_type;
+	req = nonemb_cmd.va;
+	req->ip_params.record_entry_count = 1;
+	req->ip_params.ip_record.action = IP_ACTION_ADD;
+	req->ip_params.ip_record.interface_hndl =
+		phba->interface_handle;
+	req->ip_params.ip_record.ip_addr.size_of_structure =
+		sizeof(struct be_ip_addr_subnet_format);
+	req->ip_params.ip_record.ip_addr.ip_type = ip_type;
+	ip_len = (ip_type < BEISCSI_IP_TYPE_V6) ? IP_V4_LEN : IP_V6_LEN;
+	memcpy(req->ip_params.ip_record.ip_addr.addr, ip, ip_len);
+	if (subnet)
+		memcpy(req->ip_params.ip_record.ip_addr.subnet_mask,
+		       subnet, ip_len);
 
-			rc = mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
-			if (rc < 0) {
-				beiscsi_log(phba, KERN_WARNING,
-					    BEISCSI_LOG_CONFIG,
-					    "BG_%d : Failed to Delete existing dhcp\n");
-				goto exit;
-			}
-		}
+	rc = mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
+	/**
+	 * In some cases, host needs to look into individual record status
+	 * even though FW reported success for that IOCTL.
+	 */
+	if (rc < 0 || req->ip_params.ip_record.status) {
+		__beiscsi_log(phba, KERN_ERR,
+			    "BG_%d : failed to set IP: rc %d status %d\n",
+			    rc, req->ip_params.ip_record.status);
+		if (req->ip_params.ip_record.status)
+			rc = -EINVAL;
 	}
+	return rc;
+}
 
-	/* Delete the Static IP Set */
-	if (if_info->ip_addr.addr[0]) {
-		rc = mgmt_static_ip_modify(phba, if_info, ip_param, NULL,
-					   IP_ACTION_DEL);
+int beiscsi_if_en_static(struct beiscsi_hba *phba, u32 ip_type,
+			 u8 *ip, u8 *subnet)
+{
+	struct be_cmd_get_if_info_resp *if_info;
+	struct be_cmd_rel_dhcp_req *reldhcp;
+	struct be_dma_mem nonemb_cmd;
+	int rc;
+
+	rc = beiscsi_if_get_info(phba, ip_type, &if_info);
+	if (rc)
+		return rc;
+
+	if (if_info->dhcp_state) {
+		rc = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
+				OPCODE_COMMON_ISCSI_NTWK_REL_STATELESS_IP_ADDR,
+				sizeof(*reldhcp));
 		if (rc)
 			goto exit;
-	}
 
-	/* Delete the Gateway settings if mode change is to DHCP */
-	if (boot_proto == ISCSI_BOOTPROTO_DHCP) {
-		memset(&gtway_addr_set, 0, sizeof(gtway_addr_set));
-		rc = mgmt_get_gateway(phba, BE2_IPV4, &gtway_addr_set);
-		if (rc) {
+		reldhcp = nonemb_cmd.va;
+		reldhcp->interface_hndl = phba->interface_handle;
+		reldhcp->ip_type = ip_type;
+		rc = mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
+		if (rc < 0) {
 			beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
-				    "BG_%d : Failed to Get Gateway Addr\n");
+				    "BG_%d : failed to release existing DHCP: %d\n",
+				    rc);
 			goto exit;
 		}
-
-		if (gtway_addr_set.ip_addr.addr[0]) {
-			gtway_addr = (uint8_t *)&gtway_addr_set.ip_addr.addr;
-			rc = mgmt_modify_gateway(phba, gtway_addr,
-						 IP_ACTION_DEL, IP_V4_LEN);
-
-			if (rc) {
-				beiscsi_log(phba, KERN_WARNING,
-					    BEISCSI_LOG_CONFIG,
-					    "BG_%d : Failed to clear Gateway Addr Set\n");
-				goto exit;
-			}
-		}
 	}
 
-	/* Set Adapter to DHCP/Static Mode */
-	if (boot_proto == ISCSI_BOOTPROTO_DHCP) {
-		rc = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
-			OPCODE_COMMON_ISCSI_NTWK_CONFIG_STATELESS_IP_ADDR,
-			sizeof(*dhcpreq));
+	/* first delete any IP set */
+	if (!beiscsi_if_zero_ip(if_info->ip_addr.addr, ip_type)) {
+		rc = beiscsi_if_clr_ip(phba, if_info);
 		if (rc)
 			goto exit;
-
-		dhcpreq = nonemb_cmd.va;
-		dhcpreq->flags = BLOCKING;
-		dhcpreq->retry_count = 1;
-		dhcpreq->interface_hndl = phba->interface_handle;
-		dhcpreq->ip_type = BE2_DHCP_V4;
-
-		rc = mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
-	} else {
-		rc = mgmt_static_ip_modify(phba, if_info, ip_param,
-					     subnet_param, IP_ACTION_ADD);
 	}
 
+	/* if ip == NULL then this is called just to release DHCP IP */
+	if (ip)
+		rc = beiscsi_if_set_ip(phba, ip, subnet, ip_type);
 exit:
 	kfree(if_info);
 	return rc;
 }
 
-int mgmt_set_gateway(struct beiscsi_hba *phba,
-		     struct iscsi_iface_param_info *gateway_param)
+int beiscsi_if_en_dhcp(struct beiscsi_hba *phba, u32 ip_type)
 {
-	struct be_cmd_get_def_gateway_resp gtway_addr_set;
-	uint8_t *gtway_addr;
-	int rt_val;
+	struct be_cmd_get_def_gateway_resp gw_resp;
+	struct be_cmd_get_if_info_resp *if_info;
+	struct be_cmd_set_dhcp_req *dhcpreq;
+	struct be_dma_mem nonemb_cmd;
+	u8 *gw;
+	int rc;
 
-	memset(&gtway_addr_set, 0, sizeof(gtway_addr_set));
-	rt_val = mgmt_get_gateway(phba, BE2_IPV4, &gtway_addr_set);
-	if (rt_val) {
+	rc = beiscsi_if_get_info(phba, ip_type, &if_info);
+	if (rc)
+		return rc;
+
+	if (if_info->dhcp_state) {
 		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
-			    "BG_%d : Failed to Get Gateway Addr\n");
-		return rt_val;
+				"BG_%d : DHCP Already Enabled\n");
+		goto exit;
 	}
 
-	if (gtway_addr_set.ip_addr.addr[0]) {
-		gtway_addr = (uint8_t *)&gtway_addr_set.ip_addr.addr;
-		rt_val = mgmt_modify_gateway(phba, gtway_addr, IP_ACTION_DEL,
-					     gateway_param->len);
-		if (rt_val) {
+	/* first delete any IP set */
+	if (!beiscsi_if_zero_ip(if_info->ip_addr.addr, ip_type)) {
+		rc = beiscsi_if_clr_ip(phba, if_info);
+		if (rc)
+			goto exit;
+	}
+
+	/* delete gateway settings if mode change is to DHCP */
+	memset(&gw_resp, 0, sizeof(gw_resp));
+	/* use ip_type provided in if_info */
+	rc = beiscsi_if_get_gw(phba, if_info->ip_addr.ip_type, &gw_resp);
+	if (rc) {
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BG_%d : Failed to Get Gateway Addr\n");
+		goto exit;
+	}
+	gw = (u8 *)&gw_resp.ip_addr.addr;
+	if (!beiscsi_if_zero_ip(gw, if_info->ip_addr.ip_type)) {
+		rc = beiscsi_if_mod_gw(phba, IP_ACTION_DEL,
+				       if_info->ip_addr.ip_type, gw);
+		if (rc) {
 			beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
 				    "BG_%d : Failed to clear Gateway Addr Set\n");
-			return rt_val;
+			goto exit;
 		}
 	}
 
-	gtway_addr = (uint8_t *)&gateway_param->value;
-	rt_val = mgmt_modify_gateway(phba, gtway_addr, IP_ACTION_ADD,
-				     gateway_param->len);
+	rc = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
+			OPCODE_COMMON_ISCSI_NTWK_CONFIG_STATELESS_IP_ADDR,
+			sizeof(*dhcpreq));
+	if (rc)
+		goto exit;
 
-	if (rt_val)
-		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
-			    "BG_%d : Failed to Set Gateway Addr\n");
+	dhcpreq = nonemb_cmd.va;
+	dhcpreq->flags = 1; /* 1 - blocking; 0 - non-blocking */
+	dhcpreq->retry_count = 1;
+	dhcpreq->interface_hndl = phba->interface_handle;
+	dhcpreq->ip_type = ip_type;
+	rc = mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
 
-	return rt_val;
+exit:
+	kfree(if_info);
+	return rc;
 }
 
-int mgmt_get_gateway(struct beiscsi_hba *phba, int ip_type,
-		     struct be_cmd_get_def_gateway_resp *gateway)
+/**
+ * beiscsi_if_set_vlan()- Issue and wait for CMD completion
+ * @phba: device private structure instance
+ * @vlan_tag: VLAN tag
+ *
+ * Issue the MBX Cmd and wait for the completion of the
+ * command.
+ *
+ * returns
+ *	Success: 0
+ *	Failure: Non-Xero Value
+ **/
+int beiscsi_if_set_vlan(struct beiscsi_hba *phba, uint16_t vlan_tag)
 {
-	struct be_cmd_get_def_gateway_req *req;
-	struct be_dma_mem nonemb_cmd;
 	int rc;
+	unsigned int tag;
 
-	rc = mgmt_alloc_cmd_data(phba, &nonemb_cmd,
-				 OPCODE_COMMON_ISCSI_NTWK_GET_DEFAULT_GATEWAY,
-				 sizeof(*gateway));
-	if (rc)
-		return rc;
-
-	req = nonemb_cmd.va;
-	req->ip_type = ip_type;
+	tag = be_cmd_set_vlan(phba, vlan_tag);
+	if (!tag) {
+		beiscsi_log(phba, KERN_ERR,
+			    (BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX),
+			    "BG_%d : VLAN Setting Failed\n");
+		return -EBUSY;
+	}
 
-	return mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, gateway,
-				    sizeof(*gateway));
+	rc = beiscsi_mccq_compl_wait(phba, tag, NULL, NULL);
+	if (rc) {
+		beiscsi_log(phba, KERN_ERR,
+			    (BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX),
+			    "BS_%d : VLAN MBX Cmd Failed\n");
+		return rc;
+	}
+	return rc;
 }
 
-int mgmt_get_if_info(struct beiscsi_hba *phba, int ip_type,
-		     struct be_cmd_get_if_info_resp **if_info)
+
+int beiscsi_if_get_info(struct beiscsi_hba *phba, int ip_type,
+			struct be_cmd_get_if_info_resp **if_info)
 {
 	struct be_cmd_get_if_info_req *req;
 	struct be_dma_mem nonemb_cmd;
 	uint32_t ioctl_size = sizeof(struct be_cmd_get_if_info_resp);
 	int rc;
 
-	rc = mgmt_get_all_if_id(phba);
+	rc = beiscsi_if_get_handle(phba);
 	if (rc)
 		return rc;
 
@@ -1364,123 +877,317 @@ unsigned int be_cmd_get_initname(struct beiscsi_hba *phba)
 	return tag;
 }
 
+static void beiscsi_boot_process_compl(struct beiscsi_hba *phba,
+				       unsigned int tag)
+{
+	struct be_cmd_get_boot_target_resp *boot_resp;
+	struct be_cmd_resp_logout_fw_sess *logo_resp;
+	struct be_cmd_get_session_resp *sess_resp;
+	struct be_mcc_wrb *wrb;
+	struct boot_struct *bs;
+	int boot_work, status;
+
+	if (!test_bit(BEISCSI_HBA_BOOT_WORK, &phba->state)) {
+		__beiscsi_log(phba, KERN_ERR,
+			      "BG_%d : %s no boot work %lx\n",
+			      __func__, phba->state);
+		return;
+	}
+
+	if (phba->boot_struct.tag != tag) {
+		__beiscsi_log(phba, KERN_ERR,
+			      "BG_%d : %s tag mismatch %d:%d\n",
+			      __func__, tag, phba->boot_struct.tag);
+		return;
+	}
+	bs = &phba->boot_struct;
+	boot_work = 1;
+	status = 0;
+	switch (bs->action) {
+	case BEISCSI_BOOT_REOPEN_SESS:
+		status = __beiscsi_mcc_compl_status(phba, tag, NULL, NULL);
+		if (!status)
+			bs->action = BEISCSI_BOOT_GET_SHANDLE;
+		else
+			bs->retry--;
+		break;
+	case BEISCSI_BOOT_GET_SHANDLE:
+		status = __beiscsi_mcc_compl_status(phba, tag, &wrb, NULL);
+		if (!status) {
+			boot_resp = embedded_payload(wrb);
+			bs->s_handle = boot_resp->boot_session_handle;
+		}
+		if (bs->s_handle == BE_BOOT_INVALID_SHANDLE) {
+			bs->action = BEISCSI_BOOT_REOPEN_SESS;
+			bs->retry--;
+		} else {
+			bs->action = BEISCSI_BOOT_GET_SINFO;
+		}
+		break;
+	case BEISCSI_BOOT_GET_SINFO:
+		status = __beiscsi_mcc_compl_status(phba, tag, NULL,
+						    &bs->nonemb_cmd);
+		if (!status) {
+			sess_resp = bs->nonemb_cmd.va;
+			memcpy(&bs->boot_sess, &sess_resp->session_info,
+			       sizeof(struct mgmt_session_info));
+			bs->action = BEISCSI_BOOT_LOGOUT_SESS;
+		} else {
+			__beiscsi_log(phba, KERN_ERR,
+				      "BG_%d : get boot session info error : 0x%x\n",
+				      status);
+			boot_work = 0;
+		}
+		pci_free_consistent(phba->ctrl.pdev, bs->nonemb_cmd.size,
+				    bs->nonemb_cmd.va, bs->nonemb_cmd.dma);
+		bs->nonemb_cmd.va = NULL;
+		break;
+	case BEISCSI_BOOT_LOGOUT_SESS:
+		status = __beiscsi_mcc_compl_status(phba, tag, &wrb, NULL);
+		if (!status) {
+			logo_resp = embedded_payload(wrb);
+			if (logo_resp->session_status != BE_SESS_STATUS_CLOSE) {
+				__beiscsi_log(phba, KERN_ERR,
+					      "BG_%d : FW boot session logout error : 0x%x\n",
+					      logo_resp->session_status);
+			}
+		}
+		/* continue to create boot_kset even if logout failed? */
+		bs->action = BEISCSI_BOOT_CREATE_KSET;
+		break;
+	default:
+		break;
+	}
+
+	/* clear the tag so no other completion matches this tag */
+	bs->tag = 0;
+	if (!bs->retry) {
+		boot_work = 0;
+		__beiscsi_log(phba, KERN_ERR,
+			      "BG_%d : failed to setup boot target: status %d action %d\n",
+			      status, bs->action);
+	}
+	if (!boot_work) {
+		/* wait for next event to start boot_work */
+		clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state);
+		return;
+	}
+	schedule_work(&phba->boot_work);
+}
+
 /**
- * be_mgmt_get_boot_shandle()- Get the session handle
- * @phba: device priv structure instance
- * @s_handle: session handle returned for boot session.
+ * beiscsi_boot_logout_sess()- Logout from boot FW session
+ * @phba: Device priv structure instance
  *
- * Get the boot target session handle. In case of
- * crashdump mode driver has to issue and MBX Cmd
- * for FW to login to boot target
+ * return
+ *	the TAG used for MBOX Command
+ *
+ */
+unsigned int beiscsi_boot_logout_sess(struct beiscsi_hba *phba)
+{
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct be_mcc_wrb *wrb;
+	struct be_cmd_req_logout_fw_sess *req;
+	unsigned int tag;
+
+	mutex_lock(&ctrl->mbox_lock);
+	wrb = alloc_mcc_wrb(phba, &tag);
+	if (!wrb) {
+		mutex_unlock(&ctrl->mbox_lock);
+		return 0;
+	}
+
+	req = embedded_payload(wrb);
+	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI_INI,
+			   OPCODE_ISCSI_INI_SESSION_LOGOUT_TARGET,
+			   sizeof(struct be_cmd_req_logout_fw_sess));
+	/* Use the session handle copied into boot_sess */
+	req->session_handle = phba->boot_struct.boot_sess.session_handle;
+
+	phba->boot_struct.tag = tag;
+	set_bit(MCC_TAG_STATE_ASYNC, &ctrl->ptag_state[tag].tag_state);
+	ctrl->ptag_state[tag].cbfn = beiscsi_boot_process_compl;
+
+	be_mcc_notify(phba, tag);
+	mutex_unlock(&ctrl->mbox_lock);
+
+	return tag;
+}
+/**
+ * beiscsi_boot_reopen_sess()- Reopen boot session
+ * @phba: Device priv structure instance
  *
  * return
- *	Success: 0
- *	Failure: Non-Zero value
+ *	the TAG used for MBOX Command
  *
  **/
-int be_mgmt_get_boot_shandle(struct beiscsi_hba *phba,
-			      unsigned int *s_handle)
+unsigned int beiscsi_boot_reopen_sess(struct beiscsi_hba *phba)
 {
-	struct be_cmd_get_boot_target_resp *boot_resp;
+	struct be_ctrl_info *ctrl = &phba->ctrl;
 	struct be_mcc_wrb *wrb;
+	struct be_cmd_reopen_session_req *req;
 	unsigned int tag;
-	uint8_t boot_retry = 3;
-	int rc;
 
-	do {
-		/* Get the Boot Target Session Handle and Count*/
-		tag = mgmt_get_boot_target(phba);
-		if (!tag) {
-			beiscsi_log(phba, KERN_ERR,
-				    BEISCSI_LOG_CONFIG | BEISCSI_LOG_INIT,
-				    "BG_%d : Getting Boot Target Info Failed\n");
-			return -EAGAIN;
-		}
+	mutex_lock(&ctrl->mbox_lock);
+	wrb = alloc_mcc_wrb(phba, &tag);
+	if (!wrb) {
+		mutex_unlock(&ctrl->mbox_lock);
+		return 0;
+	}
 
-		rc = beiscsi_mccq_compl_wait(phba, tag, &wrb, NULL);
-		if (rc) {
-			beiscsi_log(phba, KERN_ERR,
-				    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-				    "BG_%d : MBX CMD get_boot_target Failed\n");
-			return -EBUSY;
-		}
+	req = embedded_payload(wrb);
+	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI_INI,
+			   OPCODE_ISCSI_INI_DRIVER_REOPEN_ALL_SESSIONS,
+			   sizeof(struct be_cmd_reopen_session_resp));
+	req->reopen_type = BE_REOPEN_BOOT_SESSIONS;
+	req->session_handle = BE_BOOT_INVALID_SHANDLE;
 
-		boot_resp = embedded_payload(wrb);
+	phba->boot_struct.tag = tag;
+	set_bit(MCC_TAG_STATE_ASYNC, &ctrl->ptag_state[tag].tag_state);
+	ctrl->ptag_state[tag].cbfn = beiscsi_boot_process_compl;
 
-		/* Check if the there are any Boot targets configured */
-		if (!boot_resp->boot_session_count) {
-			beiscsi_log(phba, KERN_INFO,
-				    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-				    "BG_%d  ;No boot targets configured\n");
-			return -ENXIO;
-		}
+	be_mcc_notify(phba, tag);
+	mutex_unlock(&ctrl->mbox_lock);
+	return tag;
+}
 
-		/* FW returns the session handle of the boot session */
-		if (boot_resp->boot_session_handle != INVALID_SESS_HANDLE) {
-			*s_handle = boot_resp->boot_session_handle;
-			return 0;
-		}
 
-		/* Issue MBX Cmd to FW to login to the boot target */
-		tag = mgmt_reopen_session(phba, BE_REOPEN_BOOT_SESSIONS,
-					  INVALID_SESS_HANDLE);
-		if (!tag) {
-			beiscsi_log(phba, KERN_ERR,
-				    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-				    "BG_%d : mgmt_reopen_session Failed\n");
-			return -EAGAIN;
-		}
+/**
+ * beiscsi_boot_get_sinfo()- Get boot session info
+ * @phba: device priv structure instance
+ *
+ * Fetches the boot_struct.s_handle info from FW.
+ * return
+ *	the TAG used for MBOX Command
+ *
+ **/
+unsigned int beiscsi_boot_get_sinfo(struct beiscsi_hba *phba)
+{
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct be_cmd_get_session_resp *resp;
+	struct be_cmd_get_session_req *req;
+	struct be_dma_mem *nonemb_cmd;
+	struct be_mcc_wrb *wrb;
+	struct be_sge *sge;
+	unsigned int tag;
 
-		rc = beiscsi_mccq_compl_wait(phba, tag, NULL, NULL);
-		if (rc) {
-			beiscsi_log(phba, KERN_ERR,
-				    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-				    "BG_%d : mgmt_reopen_session Failed");
-			return rc;
-		}
-	} while (--boot_retry);
+	mutex_lock(&ctrl->mbox_lock);
+	wrb = alloc_mcc_wrb(phba, &tag);
+	if (!wrb) {
+		mutex_unlock(&ctrl->mbox_lock);
+		return 0;
+	}
 
-	/* Couldn't log into the boot target */
-	beiscsi_log(phba, KERN_ERR,
-		    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-		    "BG_%d : Login to Boot Target Failed\n");
-	return -ENXIO;
+	nonemb_cmd = &phba->boot_struct.nonemb_cmd;
+	nonemb_cmd->size = sizeof(*resp);
+	nonemb_cmd->va = pci_alloc_consistent(phba->ctrl.pdev,
+					      sizeof(nonemb_cmd->size),
+					      &nonemb_cmd->dma);
+	if (!nonemb_cmd->va) {
+		mutex_unlock(&ctrl->mbox_lock);
+		return 0;
+	}
+
+	req = nonemb_cmd->va;
+	memset(req, 0, sizeof(*req));
+	sge = nonembedded_sgl(wrb);
+	be_wrb_hdr_prepare(wrb, sizeof(*req), false, 1);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI_INI,
+			   OPCODE_ISCSI_INI_SESSION_GET_A_SESSION,
+			   sizeof(*resp));
+	req->session_handle = phba->boot_struct.s_handle;
+	sge->pa_hi = cpu_to_le32(upper_32_bits(nonemb_cmd->dma));
+	sge->pa_lo = cpu_to_le32(nonemb_cmd->dma & 0xFFFFFFFF);
+	sge->len = cpu_to_le32(nonemb_cmd->size);
+
+	phba->boot_struct.tag = tag;
+	set_bit(MCC_TAG_STATE_ASYNC, &ctrl->ptag_state[tag].tag_state);
+	ctrl->ptag_state[tag].cbfn = beiscsi_boot_process_compl;
+
+	be_mcc_notify(phba, tag);
+	mutex_unlock(&ctrl->mbox_lock);
+	return tag;
+}
+
+unsigned int __beiscsi_boot_get_shandle(struct beiscsi_hba *phba, int async)
+{
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct be_mcc_wrb *wrb;
+	struct be_cmd_get_boot_target_req *req;
+	unsigned int tag;
+
+	mutex_lock(&ctrl->mbox_lock);
+	wrb = alloc_mcc_wrb(phba, &tag);
+	if (!wrb) {
+		mutex_unlock(&ctrl->mbox_lock);
+		return 0;
+	}
+
+	req = embedded_payload(wrb);
+	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI_INI,
+			   OPCODE_ISCSI_INI_BOOT_GET_BOOT_TARGET,
+			   sizeof(struct be_cmd_get_boot_target_resp));
+
+	if (async) {
+		phba->boot_struct.tag = tag;
+		set_bit(MCC_TAG_STATE_ASYNC, &ctrl->ptag_state[tag].tag_state);
+		ctrl->ptag_state[tag].cbfn = beiscsi_boot_process_compl;
+	}
+
+	be_mcc_notify(phba, tag);
+	mutex_unlock(&ctrl->mbox_lock);
+	return tag;
 }
 
 /**
- * mgmt_set_vlan()- Issue and wait for CMD completion
- * @phba: device private structure instance
- * @vlan_tag: VLAN tag
+ * beiscsi_boot_get_shandle()- Get boot session handle
+ * @phba: device priv structure instance
+ * @s_handle: session handle returned for boot session.
  *
- * Issue the MBX Cmd and wait for the completion of the
- * command.
+ * return
+ *	Success: 1
+ *	Failure: negative
  *
- * returns
- *	Success: 0
- *	Failure: Non-Xero Value
  **/
-int mgmt_set_vlan(struct beiscsi_hba *phba,
-		   uint16_t vlan_tag)
+int beiscsi_boot_get_shandle(struct beiscsi_hba *phba, unsigned int *s_handle)
 {
-	int rc;
+	struct be_cmd_get_boot_target_resp *boot_resp;
+	struct be_mcc_wrb *wrb;
 	unsigned int tag;
+	int rc;
 
-	tag = be_cmd_set_vlan(phba, vlan_tag);
+	*s_handle = BE_BOOT_INVALID_SHANDLE;
+	/* get configured boot session count and handle */
+	tag = __beiscsi_boot_get_shandle(phba, 0);
 	if (!tag) {
 		beiscsi_log(phba, KERN_ERR,
-			    (BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX),
-			    "BG_%d : VLAN Setting Failed\n");
-		return -EBUSY;
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_INIT,
+			    "BG_%d : Getting Boot Target Info Failed\n");
+		return -EAGAIN;
 	}
 
-	rc = beiscsi_mccq_compl_wait(phba, tag, NULL, NULL);
+	rc = beiscsi_mccq_compl_wait(phba, tag, &wrb, NULL);
 	if (rc) {
 		beiscsi_log(phba, KERN_ERR,
-			    (BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX),
-			    "BS_%d : VLAN MBX Cmd Failed\n");
-		return rc;
+			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+			    "BG_%d : MBX CMD get_boot_target Failed\n");
+		return -EBUSY;
 	}
-	return rc;
+
+	boot_resp = embedded_payload(wrb);
+	/* check if there are any boot targets configured */
+	if (!boot_resp->boot_session_count) {
+		__beiscsi_log(phba, KERN_INFO,
+			      "BG_%d : No boot targets configured\n");
+		return -ENXIO;
+	}
+
+	/* only if FW has logged in to the boot target, s_handle is valid */
+	*s_handle = boot_resp->boot_session_handle;
+	return 1;
 }
 
 /**
@@ -1645,7 +1352,6 @@ void beiscsi_offload_cxn_v0(struct beiscsi_offload_params *params,
 {
 	struct iscsi_wrb *pwrb = pwrb_handle->pwrb;
 
-	memset(pwrb, 0, sizeof(*pwrb));
 	AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
 		      max_send_data_segment_length, pwrb,
 		      params->dw[offsetof(struct amap_beiscsi_offload_params,
@@ -1717,8 +1423,6 @@ void beiscsi_offload_cxn_v2(struct beiscsi_offload_params *params,
 {
 	struct iscsi_wrb *pwrb = pwrb_handle->pwrb;
 
-	memset(pwrb, 0, sizeof(*pwrb));
-
 	AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb_v2,
 		      max_burst_length, pwrb, params->dw[offsetof
 		      (struct amap_beiscsi_offload_params,
@@ -1790,70 +1494,3 @@ void beiscsi_offload_cxn_v2(struct beiscsi_offload_params *params,
 		     (params->dw[offsetof(struct amap_beiscsi_offload_params,
 		      exp_statsn) / 32] + 1));
 }
-
-/**
- * beiscsi_logout_fw_sess()- Firmware Session Logout
- * @phba: Device priv structure instance
- * @fw_sess_handle: FW session handle
- *
- * Logout from the FW established sessions.
- * returns
- *  Success: 0
- *  Failure: Non-Zero Value
- *
- */
-int beiscsi_logout_fw_sess(struct beiscsi_hba *phba,
-		uint32_t fw_sess_handle)
-{
-	struct be_ctrl_info *ctrl = &phba->ctrl;
-	struct be_mcc_wrb *wrb;
-	struct be_cmd_req_logout_fw_sess *req;
-	struct be_cmd_resp_logout_fw_sess *resp;
-	unsigned int tag;
-	int rc;
-
-	beiscsi_log(phba, KERN_INFO,
-		    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
-		    "BG_%d : In bescsi_logout_fwboot_sess\n");
-
-	mutex_lock(&ctrl->mbox_lock);
-	wrb = alloc_mcc_wrb(phba, &tag);
-	if (!wrb) {
-		mutex_unlock(&ctrl->mbox_lock);
-		beiscsi_log(phba, KERN_INFO,
-			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
-			    "BG_%d : MBX Tag Failure\n");
-		return -EINVAL;
-	}
-
-	req = embedded_payload(wrb);
-	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
-	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI_INI,
-			   OPCODE_ISCSI_INI_SESSION_LOGOUT_TARGET,
-			   sizeof(struct be_cmd_req_logout_fw_sess));
-
-	/* Set the session handle */
-	req->session_handle = fw_sess_handle;
-	be_mcc_notify(phba, tag);
-	mutex_unlock(&ctrl->mbox_lock);
-
-	rc = beiscsi_mccq_compl_wait(phba, tag, &wrb, NULL);
-	if (rc) {
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-			    "BG_%d : MBX CMD FW_SESSION_LOGOUT_TARGET Failed\n");
-		return -EBUSY;
-	}
-
-	resp = embedded_payload(wrb);
-	if (resp->session_status !=
-		BEISCSI_MGMT_SESSION_CLOSE) {
-		beiscsi_log(phba, KERN_ERR,
-			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
-			    "BG_%d : FW_SESSION_LOGOUT_TARGET resp : 0x%x\n",
-			    resp->session_status);
-		rc = -EINVAL;
-	}
-
-	return rc;
-}
diff --git a/drivers/scsi/be2iscsi/be_mgmt.h b/drivers/scsi/be2iscsi/be_mgmt.h
index f3a48a04b2ca..b897cfd57c72 100644
--- a/drivers/scsi/be2iscsi/be_mgmt.h
+++ b/drivers/scsi/be2iscsi/be_mgmt.h
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2005 - 2015 Emulex
+ * Copyright (C) 2005 - 2016 Broadcom
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or
@@ -7,10 +7,10 @@
  * as published by the Free Software Foundation.  The full GNU General
  * Public License is included in this distribution in the file called COPYING.
  *
- * Written by: Jayamohan Kallickal (jayamohan.kallickal@avagotech.com)
+ * Written by: Jayamohan Kallickal (jayamohan.kallickal@broadcom.com)
  *
  * Contact Information:
- * linux-drivers@avagotech.com
+ * linux-drivers@broadcom.com
  *
  * Emulex
  * 3333 Susan Street
@@ -96,7 +96,6 @@ struct mcc_wrb {
 	struct mcc_wrb_payload payload;
 };
 
-int mgmt_epfw_cleanup(struct beiscsi_hba *phba, unsigned short chute);
 int mgmt_open_connection(struct beiscsi_hba *phba,
 			 struct sockaddr *dst_addr,
 			 struct beiscsi_endpoint *beiscsi_ep,
@@ -266,50 +265,41 @@ struct beiscsi_endpoint {
 	u16 cid_vld;
 };
 
-int mgmt_get_fw_config(struct be_ctrl_info *ctrl,
-				 struct beiscsi_hba *phba);
-int mgmt_get_port_name(struct be_ctrl_info *ctrl,
-		       struct beiscsi_hba *phba);
-
 unsigned int mgmt_invalidate_connection(struct beiscsi_hba *phba,
 					 struct beiscsi_endpoint *beiscsi_ep,
 					 unsigned short cid,
 					 unsigned short issue_reset,
 					 unsigned short savecfg_flag);
 
-int mgmt_set_ip(struct beiscsi_hba *phba,
-		struct iscsi_iface_param_info *ip_param,
-		struct iscsi_iface_param_info *subnet_param,
-		uint32_t boot_proto);
+int beiscsi_if_en_dhcp(struct beiscsi_hba *phba, u32 ip_type);
 
-unsigned int mgmt_get_boot_target(struct beiscsi_hba *phba);
+int beiscsi_if_en_static(struct beiscsi_hba *phba, u32 ip_type,
+			 u8 *ip, u8 *subnet);
 
-unsigned int mgmt_reopen_session(struct beiscsi_hba *phba,
-				  unsigned int reopen_type,
-				  unsigned sess_handle);
+int beiscsi_if_set_gw(struct beiscsi_hba *phba, u32 ip_type, u8 *gw);
 
-unsigned int mgmt_get_session_info(struct beiscsi_hba *phba,
-				   u32 boot_session_handle,
-				   struct be_dma_mem *nonemb_cmd);
+int beiscsi_if_get_gw(struct beiscsi_hba *phba, u32 ip_type,
+		      struct be_cmd_get_def_gateway_resp *resp);
 
 int mgmt_get_nic_conf(struct beiscsi_hba *phba,
 		      struct be_cmd_get_nic_conf_resp *mac);
 
-int mgmt_get_if_info(struct beiscsi_hba *phba, int ip_type,
-		     struct be_cmd_get_if_info_resp **if_info);
+int beiscsi_if_get_info(struct beiscsi_hba *phba, int ip_type,
+			struct be_cmd_get_if_info_resp **if_info);
+
+unsigned int beiscsi_if_get_handle(struct beiscsi_hba *phba);
+
+int beiscsi_if_set_vlan(struct beiscsi_hba *phba, uint16_t vlan_tag);
 
-int mgmt_get_gateway(struct beiscsi_hba *phba, int ip_type,
-		     struct be_cmd_get_def_gateway_resp *gateway);
+unsigned int beiscsi_boot_logout_sess(struct beiscsi_hba *phba);
 
-int mgmt_set_gateway(struct beiscsi_hba *phba,
-		     struct iscsi_iface_param_info *gateway_param);
+unsigned int beiscsi_boot_reopen_sess(struct beiscsi_hba *phba);
 
-int be_mgmt_get_boot_shandle(struct beiscsi_hba *phba,
-			      unsigned int *s_handle);
+unsigned int beiscsi_boot_get_sinfo(struct beiscsi_hba *phba);
 
-unsigned int mgmt_get_all_if_id(struct beiscsi_hba *phba);
+unsigned int __beiscsi_boot_get_shandle(struct beiscsi_hba *phba, int async);
 
-int mgmt_set_vlan(struct beiscsi_hba *phba, uint16_t vlan_tag);
+int beiscsi_boot_get_shandle(struct beiscsi_hba *phba, unsigned int *s_handle);
 
 ssize_t beiscsi_drvr_ver_disp(struct device *dev,
 			       struct device_attribute *attr, char *buf);
@@ -339,7 +329,6 @@ void beiscsi_offload_cxn_v2(struct beiscsi_offload_params *params,
 			     struct wrb_handle *pwrb_handle,
 			     struct hwi_wrb_context *pwrb_context);
 
-void beiscsi_ue_detect(struct beiscsi_hba *phba);
 int be_cmd_modify_eq_delay(struct beiscsi_hba *phba,
 			 struct be_set_eqd *, int num);
 
diff --git a/drivers/scsi/bfa/bfa_fcs_lport.c b/drivers/scsi/bfa/bfa_fcs_lport.c
index 7733ad5305d4..4ddda72f60e6 100644
--- a/drivers/scsi/bfa/bfa_fcs_lport.c
+++ b/drivers/scsi/bfa/bfa_fcs_lport.c
@@ -5827,13 +5827,13 @@ bfa_fcs_lport_get_rport_max_speed(bfa_fcs_lport_t *port)
 	bfa_port_speed_t max_speed = 0;
 	struct bfa_port_attr_s port_attr;
 	bfa_port_speed_t port_speed, rport_speed;
-	bfa_boolean_t trl_enabled = bfa_fcport_is_ratelim(port->fcs->bfa);
-
+	bfa_boolean_t trl_enabled;
 
 	if (port == NULL)
 		return 0;
 
 	fcs = port->fcs;
+	trl_enabled = bfa_fcport_is_ratelim(port->fcs->bfa);
 
 	/* Get Physical port's current speed */
 	bfa_fcport_get_attr(port->fcs->bfa, &port_attr);
diff --git a/drivers/scsi/bnx2fc/bnx2fc_els.c b/drivers/scsi/bnx2fc/bnx2fc_els.c
index 5beea776b9f5..68ca518d34b0 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_els.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_els.c
@@ -254,7 +254,7 @@ int bnx2fc_send_rls(struct bnx2fc_rport *tgt, struct fc_frame *fp)
 	return rc;
 }
 
-void bnx2fc_srr_compl(struct bnx2fc_els_cb_arg *cb_arg)
+static void bnx2fc_srr_compl(struct bnx2fc_els_cb_arg *cb_arg)
 {
 	struct bnx2fc_mp_req *mp_req;
 	struct fc_frame_header *fc_hdr, *fh;
@@ -364,7 +364,7 @@ srr_compl_done:
 	kref_put(&orig_io_req->refcount, bnx2fc_cmd_release);
 }
 
-void bnx2fc_rec_compl(struct bnx2fc_els_cb_arg *cb_arg)
+static void bnx2fc_rec_compl(struct bnx2fc_els_cb_arg *cb_arg)
 {
 	struct bnx2fc_cmd *orig_io_req, *new_io_req;
 	struct bnx2fc_cmd *rec_req;
diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
index a5052dd8d7e6..f9ddb6156f14 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
@@ -625,7 +625,7 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
  *
  * @arg:	ptr to bnx2fc_percpu_info structure
  */
-int bnx2fc_percpu_io_thread(void *arg)
+static int bnx2fc_percpu_io_thread(void *arg)
 {
 	struct bnx2fc_percpu_s *p = arg;
 	struct bnx2fc_work *work, *tmp;
@@ -1410,9 +1410,10 @@ bind_err:
 	return NULL;
 }
 
-struct bnx2fc_interface *bnx2fc_interface_create(struct bnx2fc_hba *hba,
-				      struct net_device *netdev,
-				      enum fip_state fip_mode)
+static struct bnx2fc_interface *
+bnx2fc_interface_create(struct bnx2fc_hba *hba,
+			struct net_device *netdev,
+			enum fip_state fip_mode)
 {
 	struct fcoe_ctlr_device *ctlr_dev;
 	struct bnx2fc_interface *interface;
@@ -2765,8 +2766,7 @@ static void __exit bnx2fc_mod_exit(void)
 	 * held.
 	 */
 	mutex_lock(&bnx2fc_dev_lock);
-	list_splice(&adapter_list, &to_be_deleted);
-	INIT_LIST_HEAD(&adapter_list);
+	list_splice_init(&adapter_list, &to_be_deleted);
 	adapter_count = 0;
 	mutex_unlock(&bnx2fc_dev_lock);
 
diff --git a/drivers/scsi/bnx2fc/bnx2fc_hwi.c b/drivers/scsi/bnx2fc/bnx2fc_hwi.c
index 28c671b609b2..5ff9f89c17c7 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_hwi.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_hwi.c
@@ -994,7 +994,7 @@ void bnx2fc_arm_cq(struct bnx2fc_rport *tgt)
 
 }
 
-struct bnx2fc_work *bnx2fc_alloc_work(struct bnx2fc_rport *tgt, u16 wqe)
+static struct bnx2fc_work *bnx2fc_alloc_work(struct bnx2fc_rport *tgt, u16 wqe)
 {
 	struct bnx2fc_work *work;
 	work = kzalloc(sizeof(struct bnx2fc_work), GFP_ATOMIC);
diff --git a/drivers/scsi/bnx2fc/bnx2fc_io.c b/drivers/scsi/bnx2fc/bnx2fc_io.c
index 8f24d60f09d7..f501095f91ac 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_io.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_io.c
@@ -1079,7 +1079,7 @@ int bnx2fc_eh_device_reset(struct scsi_cmnd *sc_cmd)
 	return bnx2fc_initiate_tmf(sc_cmd, FCP_TMF_LUN_RESET);
 }
 
-int bnx2fc_abts_cleanup(struct bnx2fc_cmd *io_req)
+static int bnx2fc_abts_cleanup(struct bnx2fc_cmd *io_req)
 {
 	struct bnx2fc_rport *tgt = io_req->tgt;
 	int rc = SUCCESS;
diff --git a/drivers/scsi/constants.c b/drivers/scsi/constants.c
index 83458f7a2824..6dc96c8dfe75 100644
--- a/drivers/scsi/constants.c
+++ b/drivers/scsi/constants.c
@@ -361,8 +361,9 @@ static const char * const snstext[] = {
 
 /* Get sense key string or NULL if not available */
 const char *
-scsi_sense_key_string(unsigned char key) {
-	if (key <= 0xE)
+scsi_sense_key_string(unsigned char key)
+{
+	if (key < ARRAY_SIZE(snstext))
 		return snstext[key];
 	return NULL;
 }
diff --git a/drivers/scsi/csiostor/csio_scsi.c b/drivers/scsi/csiostor/csio_scsi.c
index c2a6f9f29427..89a52b941ea8 100644
--- a/drivers/scsi/csiostor/csio_scsi.c
+++ b/drivers/scsi/csiostor/csio_scsi.c
@@ -1721,7 +1721,7 @@ out:
 
 	/* Wake up waiting threads */
 	csio_scsi_cmnd(req) = NULL;
-	complete_all(&req->cmplobj);
+	complete(&req->cmplobj);
 }
 
 /*
@@ -1945,6 +1945,7 @@ csio_eh_abort_handler(struct scsi_cmnd *cmnd)
 	ready = csio_is_lnode_ready(ln);
 	tmo = CSIO_SCSI_ABRT_TMO_MS;
 
+	reinit_completion(&ioreq->cmplobj);
 	spin_lock_irq(&hw->lock);
 	rv = csio_do_abrt_cls(hw, ioreq, (ready ? SCSI_ABORT : SCSI_CLOSE));
 	spin_unlock_irq(&hw->lock);
@@ -1964,8 +1965,6 @@ csio_eh_abort_handler(struct scsi_cmnd *cmnd)
 		goto inval_scmnd;
 	}
 
-	/* Wait for completion */
-	init_completion(&ioreq->cmplobj);
 	wait_for_completion_timeout(&ioreq->cmplobj, msecs_to_jiffies(tmo));
 
 	/* FW didnt respond to abort within our timeout */
diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
index e4ba2d2616cd..7c0d7af0d3b7 100644
--- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
+++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
@@ -84,6 +84,9 @@ static inline int send_tx_flowc_wr(struct cxgbi_sock *);
 
 static const struct cxgb4_uld_info cxgb4i_uld_info = {
 	.name = DRV_MODULE_NAME,
+	.nrxq = MAX_ULD_QSETS,
+	.rxq_size = 1024,
+	.lro = false,
 	.add = t4_uld_add,
 	.rx_handler = t4_uld_rx_handler,
 	.state_change = t4_uld_state_change,
diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c
index 661bb94e2548..b301655f91cd 100644
--- a/drivers/scsi/cxlflash/main.c
+++ b/drivers/scsi/cxlflash/main.c
@@ -823,17 +823,6 @@ static void notify_shutdown(struct cxlflash_cfg *cfg, bool wait)
 }
 
 /**
- * cxlflash_shutdown() - shutdown handler
- * @pdev:	PCI device associated with the host.
- */
-static void cxlflash_shutdown(struct pci_dev *pdev)
-{
-	struct cxlflash_cfg *cfg = pci_get_drvdata(pdev);
-
-	notify_shutdown(cfg, false);
-}
-
-/**
  * cxlflash_remove() - PCI entry point to tear down host
  * @pdev:	PCI device associated with the host.
  *
@@ -844,6 +833,11 @@ static void cxlflash_remove(struct pci_dev *pdev)
 	struct cxlflash_cfg *cfg = pci_get_drvdata(pdev);
 	ulong lock_flags;
 
+	if (!pci_is_enabled(pdev)) {
+		pr_debug("%s: Device is disabled\n", __func__);
+		return;
+	}
+
 	/* If a Task Management Function is active, wait for it to complete
 	 * before continuing with remove.
 	 */
@@ -1046,6 +1040,8 @@ static int wait_port_online(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry)
 	do {
 		msleep(delay_us / 1000);
 		status = readq_be(&fc_regs[FC_MTIP_STATUS / 8]);
+		if (status == U64_MAX)
+			nretry /= 2;
 	} while ((status & FC_MTIP_STATUS_MASK) != FC_MTIP_STATUS_ONLINE &&
 		 nretry--);
 
@@ -1077,6 +1073,8 @@ static int wait_port_offline(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry)
 	do {
 		msleep(delay_us / 1000);
 		status = readq_be(&fc_regs[FC_MTIP_STATUS / 8]);
+		if (status == U64_MAX)
+			nretry /= 2;
 	} while ((status & FC_MTIP_STATUS_MASK) != FC_MTIP_STATUS_OFFLINE &&
 		 nretry--);
 
@@ -1095,42 +1093,25 @@ static int wait_port_offline(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry)
  * online. This toggling action can cause this routine to delay up to a few
  * seconds. When configured to use the internal LUN feature of the AFU, a
  * failure to come online is overridden.
- *
- * Return:
- *	0 when the WWPN is successfully written and the port comes back online
- *	-1 when the port fails to go offline or come back up online
  */
-static int afu_set_wwpn(struct afu *afu, int port, __be64 __iomem *fc_regs,
-			u64 wwpn)
+static void afu_set_wwpn(struct afu *afu, int port, __be64 __iomem *fc_regs,
+			 u64 wwpn)
 {
-	int rc = 0;
-
 	set_port_offline(fc_regs);
-
 	if (!wait_port_offline(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US,
 			       FC_PORT_STATUS_RETRY_CNT)) {
 		pr_debug("%s: wait on port %d to go offline timed out\n",
 			 __func__, port);
-		rc = -1; /* but continue on to leave the port back online */
 	}
 
-	if (rc == 0)
-		writeq_be(wwpn, &fc_regs[FC_PNAME / 8]);
-
-	/* Always return success after programming WWPN */
-	rc = 0;
+	writeq_be(wwpn, &fc_regs[FC_PNAME / 8]);
 
 	set_port_online(fc_regs);
-
 	if (!wait_port_online(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US,
 			      FC_PORT_STATUS_RETRY_CNT)) {
-		pr_err("%s: wait on port %d to go online timed out\n",
-		       __func__, port);
+		pr_debug("%s: wait on port %d to go online timed out\n",
+			 __func__, port);
 	}
-
-	pr_debug("%s: returning rc=%d\n", __func__, rc);
-
-	return rc;
 }
 
 /**
@@ -1187,7 +1168,7 @@ static const struct asyc_intr_info ainfo[] = {
 	{SISL_ASTATUS_FC0_LOGI_F, "login failed", 0, CLR_FC_ERROR},
 	{SISL_ASTATUS_FC0_LOGI_S, "login succeeded", 0, SCAN_HOST},
 	{SISL_ASTATUS_FC0_LINK_DN, "link down", 0, 0},
-	{SISL_ASTATUS_FC0_LINK_UP, "link up", 0, SCAN_HOST},
+	{SISL_ASTATUS_FC0_LINK_UP, "link up", 0, 0},
 	{SISL_ASTATUS_FC1_OTHER, "other error", 1, CLR_FC_ERROR | LINK_RESET},
 	{SISL_ASTATUS_FC1_LOGO, "target initiated LOGO", 1, 0},
 	{SISL_ASTATUS_FC1_CRC_T, "CRC threshold exceeded", 1, LINK_RESET},
@@ -1195,7 +1176,7 @@ static const struct asyc_intr_info ainfo[] = {
 	{SISL_ASTATUS_FC1_LOGI_F, "login failed", 1, CLR_FC_ERROR},
 	{SISL_ASTATUS_FC1_LOGI_S, "login succeeded", 1, SCAN_HOST},
 	{SISL_ASTATUS_FC1_LINK_DN, "link down", 1, 0},
-	{SISL_ASTATUS_FC1_LINK_UP, "link up", 1, SCAN_HOST},
+	{SISL_ASTATUS_FC1_LINK_UP, "link up", 1, 0},
 	{0x0, "", 0, 0}		/* terminator */
 };
 
@@ -1631,15 +1612,10 @@ static int init_global(struct cxlflash_cfg *cfg)
 			  [FC_CRC_THRESH / 8]);
 
 		/* Set WWPNs. If already programmed, wwpn[i] is 0 */
-		if (wwpn[i] != 0 &&
-		    afu_set_wwpn(afu, i,
-				 &afu->afu_map->global.fc_regs[i][0],
-				 wwpn[i])) {
-			dev_err(dev, "%s: failed to set WWPN on port %d\n",
-			       __func__, i);
-			rc = -EIO;
-			goto out;
-		}
+		if (wwpn[i] != 0)
+			afu_set_wwpn(afu, i,
+				     &afu->afu_map->global.fc_regs[i][0],
+				     wwpn[i]);
 		/* Programming WWPN back to back causes additional
 		 * offline/online transitions and a PLOGI
 		 */
@@ -2048,6 +2024,11 @@ retry:
  * cxlflash_eh_host_reset_handler() - reset the host adapter
  * @scp:	SCSI command from stack identifying host.
  *
+ * Following a reset, the state is evaluated again in case an EEH occurred
+ * during the reset. In such a scenario, the host reset will either yield
+ * until the EEH recovery is complete or return success or failure based
+ * upon the current device state.
+ *
  * Return:
  *	SUCCESS as defined in scsi/scsi.h
  *	FAILED as defined in scsi/scsi.h
@@ -2080,7 +2061,8 @@ static int cxlflash_eh_host_reset_handler(struct scsi_cmnd *scp)
 		} else
 			cfg->state = STATE_NORMAL;
 		wake_up_all(&cfg->reset_waitq);
-		break;
+		ssleep(1);
+		/* fall through */
 	case STATE_RESET:
 		wait_event(cfg->reset_waitq, cfg->state != STATE_RESET);
 		if (cfg->state == STATE_NORMAL)
@@ -2596,6 +2578,9 @@ out_remove:
  * @pdev:	PCI device struct.
  * @state:	PCI channel state.
  *
+ * When an EEH occurs during an active reset, wait until the reset is
+ * complete and then take action based upon the device state.
+ *
  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT
  */
 static pci_ers_result_t cxlflash_pci_error_detected(struct pci_dev *pdev,
@@ -2609,6 +2594,10 @@ static pci_ers_result_t cxlflash_pci_error_detected(struct pci_dev *pdev,
 
 	switch (state) {
 	case pci_channel_io_frozen:
+		wait_event(cfg->reset_waitq, cfg->state != STATE_RESET);
+		if (cfg->state == STATE_FAILTERM)
+			return PCI_ERS_RESULT_DISCONNECT;
+
 		cfg->state = STATE_RESET;
 		scsi_block_requests(cfg->host);
 		drain_ioctls(cfg);
@@ -2685,7 +2674,7 @@ static struct pci_driver cxlflash_driver = {
 	.id_table = cxlflash_pci_table,
 	.probe = cxlflash_probe,
 	.remove = cxlflash_remove,
-	.shutdown = cxlflash_shutdown,
+	.shutdown = cxlflash_remove,
 	.err_handler = &cxlflash_err_handler,
 };
 
diff --git a/drivers/scsi/cxlflash/superpipe.c b/drivers/scsi/cxlflash/superpipe.c
index ce1507023132..9636970d9611 100644
--- a/drivers/scsi/cxlflash/superpipe.c
+++ b/drivers/scsi/cxlflash/superpipe.c
@@ -709,14 +709,13 @@ int cxlflash_disk_release(struct scsi_device *sdev,
  * @cfg:	Internal structure associated with the host.
  * @ctxi:	Context to release.
  *
- * This routine is safe to be called with a a non-initialized context
- * and is tolerant of being called with the context's mutex held (it
- * will be unlocked if necessary before freeing). Also note that the
- * routine conditionally checks for the existence of the context control
- * map before clearing the RHT registers and context capabilities because
- * it is possible to destroy a context while the context is in the error
- * state (previous mapping was removed [so there is no need to worry about
- * clearing] and context is waiting for a new mapping).
+ * This routine is safe to be called with a a non-initialized context.
+ * Also note that the routine conditionally checks for the existence
+ * of the context control map before clearing the RHT registers and
+ * context capabilities because it is possible to destroy a context
+ * while the context is in the error state (previous mapping was
+ * removed [so there is no need to worry about clearing] and context
+ * is waiting for a new mapping).
  */
 static void destroy_context(struct cxlflash_cfg *cfg,
 			    struct ctx_info *ctxi)
@@ -732,9 +731,6 @@ static void destroy_context(struct cxlflash_cfg *cfg,
 			writeq_be(0, &ctxi->ctrl_map->rht_cnt_id);
 			writeq_be(0, &ctxi->ctrl_map->ctx_cap);
 		}
-
-		if (mutex_is_locked(&ctxi->mutex))
-			mutex_unlock(&ctxi->mutex);
 	}
 
 	/* Free memory associated with context */
@@ -792,32 +788,58 @@ err:
  * @cfg:	Internal structure associated with the host.
  * @ctx:	Previously obtained CXL context reference.
  * @ctxid:	Previously obtained process element associated with CXL context.
- * @adap_fd:	Previously obtained adapter fd associated with CXL context.
  * @file:	Previously obtained file associated with CXL context.
  * @perms:	User-specified permissions.
- *
- * Upon return, the context is marked as initialized and the context's mutex
- * is locked.
  */
 static void init_context(struct ctx_info *ctxi, struct cxlflash_cfg *cfg,
-			 struct cxl_context *ctx, int ctxid, int adap_fd,
-			 struct file *file, u32 perms)
+			 struct cxl_context *ctx, int ctxid, struct file *file,
+			 u32 perms)
 {
 	struct afu *afu = cfg->afu;
 
 	ctxi->rht_perms = perms;
 	ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;
 	ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
-	ctxi->lfd = adap_fd;
 	ctxi->pid = current->tgid; /* tgid = pid */
 	ctxi->ctx = ctx;
+	ctxi->cfg = cfg;
 	ctxi->file = file;
 	ctxi->initialized = true;
 	mutex_init(&ctxi->mutex);
+	kref_init(&ctxi->kref);
 	INIT_LIST_HEAD(&ctxi->luns);
 	INIT_LIST_HEAD(&ctxi->list); /* initialize for list_empty() */
+}
 
+/**
+ * remove_context() - context kref release handler
+ * @kref:	Kernel reference associated with context to be removed.
+ *
+ * When a context no longer has any references it can safely be removed
+ * from global access and destroyed. Note that it is assumed the thread
+ * relinquishing access to the context holds its mutex.
+ */
+static void remove_context(struct kref *kref)
+{
+	struct ctx_info *ctxi = container_of(kref, struct ctx_info, kref);
+	struct cxlflash_cfg *cfg = ctxi->cfg;
+	u64 ctxid = DECODE_CTXID(ctxi->ctxid);
+
+	/* Remove context from table/error list */
+	WARN_ON(!mutex_is_locked(&ctxi->mutex));
+	ctxi->unavail = true;
+	mutex_unlock(&ctxi->mutex);
+	mutex_lock(&cfg->ctx_tbl_list_mutex);
 	mutex_lock(&ctxi->mutex);
+
+	if (!list_empty(&ctxi->list))
+		list_del(&ctxi->list);
+	cfg->ctx_tbl[ctxid] = NULL;
+	mutex_unlock(&cfg->ctx_tbl_list_mutex);
+	mutex_unlock(&ctxi->mutex);
+
+	/* Context now completely uncoupled/unreachable */
+	destroy_context(cfg, ctxi);
 }
 
 /**
@@ -845,7 +867,6 @@ static int _cxlflash_disk_detach(struct scsi_device *sdev,
 
 	int i;
 	int rc = 0;
-	int lfd;
 	u64 ctxid = DECODE_CTXID(detach->context_id),
 	    rctxid = detach->context_id;
 
@@ -887,40 +908,13 @@ static int _cxlflash_disk_detach(struct scsi_device *sdev,
 			break;
 		}
 
-	/* Tear down context following last LUN cleanup */
-	if (list_empty(&ctxi->luns)) {
-		ctxi->unavail = true;
-		mutex_unlock(&ctxi->mutex);
-		mutex_lock(&cfg->ctx_tbl_list_mutex);
-		mutex_lock(&ctxi->mutex);
-
-		/* Might not have been in error list so conditionally remove */
-		if (!list_empty(&ctxi->list))
-			list_del(&ctxi->list);
-		cfg->ctx_tbl[ctxid] = NULL;
-		mutex_unlock(&cfg->ctx_tbl_list_mutex);
-		mutex_unlock(&ctxi->mutex);
-
-		lfd = ctxi->lfd;
-		destroy_context(cfg, ctxi);
-		ctxi = NULL;
+	/*
+	 * Release the context reference and the sdev reference that
+	 * bound this LUN to the context.
+	 */
+	if (kref_put(&ctxi->kref, remove_context))
 		put_ctx = false;
-
-		/*
-		 * As a last step, clean up external resources when not
-		 * already on an external cleanup thread, i.e.: close(adap_fd).
-		 *
-		 * NOTE: this will free up the context from the CXL services,
-		 * allowing it to dole out the same context_id on a future
-		 * (or even currently in-flight) disk_attach operation.
-		 */
-		if (lfd != -1)
-			sys_close(lfd);
-	}
-
-	/* Release the sdev reference that bound this LUN to the context */
 	scsi_device_put(sdev);
-
 out:
 	if (put_ctx)
 		put_context(ctxi);
@@ -941,34 +935,18 @@ static int cxlflash_disk_detach(struct scsi_device *sdev,
  *
  * This routine is the release handler for the fops registered with
  * the CXL services on an initial attach for a context. It is called
- * when a close is performed on the adapter file descriptor returned
- * to the user. Programmatically, the user is not required to perform
- * the close, as it is handled internally via the detach ioctl when
- * a context is being removed. Note that nothing prevents the user
- * from performing a close, but the user should be aware that doing
- * so is considered catastrophic and subsequent usage of the superpipe
- * API with previously saved off tokens will fail.
- *
- * When initiated from an external close (either by the user or via
- * a process tear down), the routine derives the context reference
- * and calls detach for each LUN associated with the context. The
- * final detach operation will cause the context itself to be freed.
- * Note that the saved off lfd is reset prior to calling detach to
- * signify that the final detach should not perform a close.
- *
- * When initiated from a detach operation as part of the tear down
- * of a context, the context is first completely freed and then the
- * close is performed. This routine will fail to derive the context
- * reference (due to the context having already been freed) and then
- * call into the CXL release entry point.
+ * when a close (explicity by the user or as part of a process tear
+ * down) is performed on the adapter file descriptor returned to the
+ * user. The user should be aware that explicitly performing a close
+ * considered catastrophic and subsequent usage of the superpipe API
+ * with previously saved off tokens will fail.
  *
- * Thus, with exception to when the CXL process element (context id)
- * lookup fails (a case that should theoretically never occur), every
- * call into this routine results in a complete freeing of a context.
- *
- * As part of the detach, all per-context resources associated with the LUN
- * are cleaned up. When detaching the last LUN for a context, the context
- * itself is cleaned up and released.
+ * This routine derives the context reference and calls detach for
+ * each LUN associated with the context.The final detach operation
+ * causes the context itself to be freed. With exception to when the
+ * CXL process element (context id) lookup fails (a case that should
+ * theoretically never occur), every call into this routine results
+ * in a complete freeing of a context.
  *
  * Return: 0 on success
  */
@@ -1006,11 +984,8 @@ static int cxlflash_cxl_release(struct inode *inode, struct file *file)
 		goto out;
 	}
 
-	dev_dbg(dev, "%s: close(%d) for context %d\n",
-		__func__, ctxi->lfd, ctxid);
+	dev_dbg(dev, "%s: close for context %d\n", __func__, ctxid);
 
-	/* Reset the file descriptor to indicate we're on a close() thread */
-	ctxi->lfd = -1;
 	detach.context_id = ctxi->ctxid;
 	list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
 		_cxlflash_disk_detach(lun_access->sdev, ctxi, &detach);
@@ -1110,8 +1085,7 @@ static int cxlflash_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 		goto err;
 	}
 
-	dev_dbg(dev, "%s: fault(%d) for context %d\n",
-		__func__, ctxi->lfd, ctxid);
+	dev_dbg(dev, "%s: fault for context %d\n", __func__, ctxid);
 
 	if (likely(!ctxi->err_recovery_active)) {
 		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
@@ -1186,8 +1160,7 @@ static int cxlflash_cxl_mmap(struct file *file, struct vm_area_struct *vma)
 		goto out;
 	}
 
-	dev_dbg(dev, "%s: mmap(%d) for context %d\n",
-		__func__, ctxi->lfd, ctxid);
+	dev_dbg(dev, "%s: mmap for context %d\n", __func__, ctxid);
 
 	rc = cxl_fd_mmap(file, vma);
 	if (likely(!rc)) {
@@ -1377,12 +1350,12 @@ static int cxlflash_disk_attach(struct scsi_device *sdev,
 	lun_access->lli = lli;
 	lun_access->sdev = sdev;
 
-	/* Non-NULL context indicates reuse */
+	/* Non-NULL context indicates reuse (another context reference) */
 	if (ctxi) {
 		dev_dbg(dev, "%s: Reusing context for LUN! (%016llX)\n",
 			__func__, rctxid);
+		kref_get(&ctxi->kref);
 		list_add(&lun_access->list, &ctxi->luns);
-		fd = ctxi->lfd;
 		goto out_attach;
 	}
 
@@ -1430,7 +1403,7 @@ static int cxlflash_disk_attach(struct scsi_device *sdev,
 	perms = SISL_RHT_PERM(attach->hdr.flags + 1);
 
 	/* Context mutex is locked upon return */
-	init_context(ctxi, cfg, ctx, ctxid, fd, file, perms);
+	init_context(ctxi, cfg, ctx, ctxid, file, perms);
 
 	rc = afu_attach(cfg, ctxi);
 	if (unlikely(rc)) {
@@ -1445,7 +1418,6 @@ static int cxlflash_disk_attach(struct scsi_device *sdev,
 	 * knows about us yet; we can be the only one holding our mutex.
 	 */
 	list_add(&lun_access->list, &ctxi->luns);
-	mutex_unlock(&ctxi->mutex);
 	mutex_lock(&cfg->ctx_tbl_list_mutex);
 	mutex_lock(&ctxi->mutex);
 	cfg->ctx_tbl[ctxid] = ctxi;
@@ -1453,7 +1425,11 @@ static int cxlflash_disk_attach(struct scsi_device *sdev,
 	fd_install(fd, file);
 
 out_attach:
-	attach->hdr.return_flags = 0;
+	if (fd != -1)
+		attach->hdr.return_flags = DK_CXLFLASH_APP_CLOSE_ADAP_FD;
+	else
+		attach->hdr.return_flags = 0;
+
 	attach->context_id = ctxi->ctxid;
 	attach->block_size = gli->blk_len;
 	attach->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
@@ -1494,7 +1470,7 @@ err:
 		file = NULL;
 	}
 
-	/* Cleanup our context; safe to call even with mutex locked */
+	/* Cleanup our context */
 	if (ctxi) {
 		destroy_context(cfg, ctxi);
 		ctxi = NULL;
@@ -1509,16 +1485,19 @@ err:
  * recover_context() - recovers a context in error
  * @cfg:	Internal structure associated with the host.
  * @ctxi:	Context to release.
+ * @adap_fd:	Adapter file descriptor associated with new/recovered context.
  *
  * Restablishes the state for a context-in-error.
  *
  * Return: 0 on success, -errno on failure
  */
-static int recover_context(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
+static int recover_context(struct cxlflash_cfg *cfg,
+			   struct ctx_info *ctxi,
+			   int *adap_fd)
 {
 	struct device *dev = &cfg->dev->dev;
 	int rc = 0;
-	int old_fd, fd = -1;
+	int fd = -1;
 	int ctxid = -1;
 	struct file *file;
 	struct cxl_context *ctx;
@@ -1566,9 +1545,7 @@ static int recover_context(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
 	 * No error paths after this point. Once the fd is installed it's
 	 * visible to user space and can't be undone safely on this thread.
 	 */
-	old_fd = ctxi->lfd;
 	ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
-	ctxi->lfd = fd;
 	ctxi->ctx = ctx;
 	ctxi->file = file;
 
@@ -1585,9 +1562,7 @@ static int recover_context(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
 	cfg->ctx_tbl[ctxid] = ctxi;
 	mutex_unlock(&cfg->ctx_tbl_list_mutex);
 	fd_install(fd, file);
-
-	/* Release the original adapter fd and associated CXL resources */
-	sys_close(old_fd);
+	*adap_fd = fd;
 out:
 	dev_dbg(dev, "%s: returning ctxid=%d fd=%d rc=%d\n",
 		__func__, ctxid, fd, rc);
@@ -1646,6 +1621,7 @@ static int cxlflash_afu_recover(struct scsi_device *sdev,
 	    rctxid = recover->context_id;
 	long reg;
 	int lretry = 20; /* up to 2 seconds */
+	int new_adap_fd = -1;
 	int rc = 0;
 
 	atomic_inc(&cfg->recovery_threads);
@@ -1675,7 +1651,7 @@ retry:
 
 	if (ctxi->err_recovery_active) {
 retry_recover:
-		rc = recover_context(cfg, ctxi);
+		rc = recover_context(cfg, ctxi, &new_adap_fd);
 		if (unlikely(rc)) {
 			dev_err(dev, "%s: Recovery failed for context %llu (rc=%d)\n",
 				__func__, ctxid, rc);
@@ -1697,9 +1673,9 @@ retry_recover:
 
 		ctxi->err_recovery_active = false;
 		recover->context_id = ctxi->ctxid;
-		recover->adap_fd = ctxi->lfd;
+		recover->adap_fd = new_adap_fd;
 		recover->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
-		recover->hdr.return_flags |=
+		recover->hdr.return_flags = DK_CXLFLASH_APP_CLOSE_ADAP_FD |
 			DK_CXLFLASH_RECOVER_AFU_CONTEXT_RESET;
 		goto out;
 	}
diff --git a/drivers/scsi/cxlflash/superpipe.h b/drivers/scsi/cxlflash/superpipe.h
index 5f9a091fda95..9e62ff304e4b 100644
--- a/drivers/scsi/cxlflash/superpipe.h
+++ b/drivers/scsi/cxlflash/superpipe.h
@@ -100,13 +100,14 @@ struct ctx_info {
 
 	struct cxl_ioctl_start_work work;
 	u64 ctxid;
-	int lfd;
 	pid_t pid;
 	bool initialized;
 	bool unavail;
 	bool err_recovery_active;
 	struct mutex mutex; /* Context protection */
+	struct kref kref;
 	struct cxl_context *ctx;
+	struct cxlflash_cfg *cfg;
 	struct list_head luns;	/* LUNs attached to this context */
 	const struct vm_operations_struct *cxl_mmap_vmops;
 	struct file *file;
diff --git a/drivers/scsi/cxlflash/vlun.c b/drivers/scsi/cxlflash/vlun.c
index 50f8e9300770..90c5d7f5278e 100644
--- a/drivers/scsi/cxlflash/vlun.c
+++ b/drivers/scsi/cxlflash/vlun.c
@@ -1135,14 +1135,13 @@ int cxlflash_disk_clone(struct scsi_device *sdev,
 	    ctxid_dst = DECODE_CTXID(clone->context_id_dst),
 	    rctxid_src = clone->context_id_src,
 	    rctxid_dst = clone->context_id_dst;
-	int adap_fd_src = clone->adap_fd_src;
 	int i, j;
 	int rc = 0;
 	bool found;
 	LIST_HEAD(sidecar);
 
-	pr_debug("%s: ctxid_src=%llu ctxid_dst=%llu adap_fd_src=%d\n",
-		 __func__, ctxid_src, ctxid_dst, adap_fd_src);
+	pr_debug("%s: ctxid_src=%llu ctxid_dst=%llu\n",
+		 __func__, ctxid_src, ctxid_dst);
 
 	/* Do not clone yourself */
 	if (unlikely(rctxid_src == rctxid_dst)) {
@@ -1166,13 +1165,6 @@ int cxlflash_disk_clone(struct scsi_device *sdev,
 		goto out;
 	}
 
-	if (unlikely(adap_fd_src != ctxi_src->lfd)) {
-		pr_debug("%s: Invalid source adapter fd! (%d)\n",
-			 __func__, adap_fd_src);
-		rc = -EINVAL;
-		goto out;
-	}
-
 	/* Verify there is no open resource handle in the destination context */
 	for (i = 0; i < MAX_RHT_PER_CONTEXT; i++)
 		if (ctxi_dst->rht_start[i].nmask != 0) {
@@ -1257,7 +1249,6 @@ int cxlflash_disk_clone(struct scsi_device *sdev,
 
 out_success:
 	list_splice(&sidecar, &ctxi_dst->luns);
-	sys_close(adap_fd_src);
 
 	/* fall through */
 out:
diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index 752b5c9d1ab2..241829e59668 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -583,6 +583,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_port_group *pg)
 			sdev_printk(KERN_ERR, sdev, "%s: rtpg retry\n",
 				    ALUA_DH_NAME);
 			scsi_print_sense_hdr(sdev, ALUA_DH_NAME, &sense_hdr);
+			kfree(buff);
 			return err;
 		}
 		sdev_printk(KERN_ERR, sdev, "%s: rtpg failed\n",
diff --git a/drivers/scsi/dtc.c b/drivers/scsi/dtc.c
deleted file mode 100644
index 459863f94e46..000000000000
--- a/drivers/scsi/dtc.c
+++ /dev/null
@@ -1,447 +0,0 @@
-/*
- * DTC 3180/3280 driver, by
- *	Ray Van Tassle	rayvt@comm.mot.com
- *
- *	taken from ...
- *	Trantor T128/T128F/T228 driver by...
- *
- * 	Drew Eckhardt
- *	Visionary Computing
- *	(Unix and Linux consulting and custom programming)
- *	drew@colorado.edu
- *      +1 (303) 440-4894
- */
-
-/*
- * The card is detected and initialized in one of several ways : 
- * 1.  Autoprobe (default) - since the board is memory mapped, 
- *     a BIOS signature is scanned for to locate the registers.
- *     An interrupt is triggered to autoprobe for the interrupt
- *     line.
- *
- * 2.  With command line overrides - dtc=address,irq may be 
- *     used on the LILO command line to override the defaults.
- * 
-*/
-
-/*----------------------------------------------------------------*/
-/* the following will set the monitor border color (useful to find
- where something crashed or gets stuck at */
-/* 1 = blue
- 2 = green
- 3 = cyan
- 4 = red
- 5 = magenta
- 6 = yellow
- 7 = white
-*/
-#if 0
-#define rtrc(i) {inb(0x3da); outb(0x31, 0x3c0); outb((i), 0x3c0);}
-#else
-#define rtrc(i) {}
-#endif
-
-
-#include <linux/module.h>
-#include <linux/blkdev.h>
-#include <linux/string.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <scsi/scsi_host.h>
-
-#include "dtc.h"
-#include "NCR5380.h"
-
-/*
- * The DTC3180 & 3280 boards are memory mapped.
- * 
- */
-
-/*
- */
-/* Offset from DTC_5380_OFFSET */
-#define DTC_CONTROL_REG		0x100	/* rw */
-#define D_CR_ACCESS		0x80	/* ro set=can access 3280 registers */
-#define CSR_DIR_READ		0x40	/* rw direction, 1 = read 0 = write */
-
-#define CSR_RESET              0x80	/* wo  Resets 53c400 */
-#define CSR_5380_REG           0x80	/* ro  5380 registers can be accessed */
-#define CSR_TRANS_DIR          0x40	/* rw  Data transfer direction */
-#define CSR_SCSI_BUFF_INTR     0x20	/* rw  Enable int on transfer ready */
-#define CSR_5380_INTR          0x10	/* rw  Enable 5380 interrupts */
-#define CSR_SHARED_INTR        0x08	/* rw  Interrupt sharing */
-#define CSR_HOST_BUF_NOT_RDY   0x04	/* ro  Host buffer not ready */
-#define CSR_SCSI_BUF_RDY       0x02	/* ro  SCSI buffer ready */
-#define CSR_GATED_5380_IRQ     0x01	/* ro  Last block xferred */
-#define CSR_INT_BASE (CSR_SCSI_BUFF_INTR | CSR_5380_INTR)
-
-
-#define DTC_BLK_CNT		0x101	/* rw 
-					 * # of 128-byte blocks to transfer */
-
-
-#define D_CR_ACCESS             0x80	/* ro set=can access 3280 registers */
-
-#define DTC_SWITCH_REG		0x3982	/* ro - DIP switches */
-#define DTC_RESUME_XFER		0x3982	/* wo - resume data xfer 
-					 * after disconnect/reconnect*/
-
-#define DTC_5380_OFFSET		0x3880	/* 8 registers here, see NCR5380.h */
-
-/*!!!! for dtc, it's a 128 byte buffer at 3900 !!! */
-#define DTC_DATA_BUF		0x3900	/* rw 128 bytes long */
-
-static struct override {
-	unsigned int address;
-	int irq;
-} overrides
-#ifdef OVERRIDE
-[] __initdata = OVERRIDE;
-#else
-[4] __initdata = {
-	{ 0, IRQ_AUTO }, { 0, IRQ_AUTO }, { 0, IRQ_AUTO }, { 0, IRQ_AUTO }
-};
-#endif
-
-#define NO_OVERRIDES ARRAY_SIZE(overrides)
-
-static struct base {
-	unsigned long address;
-	int noauto;
-} bases[] __initdata = {
-	{ 0xcc000, 0 },
-	{ 0xc8000, 0 },
-	{ 0xdc000, 0 },
-	{ 0xd8000, 0 }
-};
-
-#define NO_BASES ARRAY_SIZE(bases)
-
-static const struct signature {
-	const char *string;
-	int offset;
-} signatures[] = {
-	{"DATA TECHNOLOGY CORPORATION BIOS", 0x25},
-};
-
-#define NO_SIGNATURES ARRAY_SIZE(signatures)
-
-#ifndef MODULE
-/*
- * Function : dtc_setup(char *str, int *ints)
- *
- * Purpose : LILO command line initialization of the overrides array,
- *
- * Inputs : str - unused, ints - array of integer parameters with ints[0]
- *	equal to the number of ints.
- *
- */
-
-static int __init dtc_setup(char *str)
-{
-	static int commandline_current;
-	int i;
-	int ints[10];
-
-	get_options(str, ARRAY_SIZE(ints), ints);
-	if (ints[0] != 2)
-		printk("dtc_setup: usage dtc=address,irq\n");
-	else if (commandline_current < NO_OVERRIDES) {
-		overrides[commandline_current].address = ints[1];
-		overrides[commandline_current].irq = ints[2];
-		for (i = 0; i < NO_BASES; ++i)
-			if (bases[i].address == ints[1]) {
-				bases[i].noauto = 1;
-				break;
-			}
-		++commandline_current;
-	}
-	return 1;
-}
-
-__setup("dtc=", dtc_setup);
-#endif
-
-/* 
- * Function : int dtc_detect(struct scsi_host_template * tpnt)
- *
- * Purpose : detects and initializes DTC 3180/3280 controllers
- *	that were autoprobed, overridden on the LILO command line, 
- *	or specified at compile time.
- *
- * Inputs : tpnt - template for this SCSI adapter.
- * 
- * Returns : 1 if a host adapter was found, 0 if not.
- *
-*/
-
-static int __init dtc_detect(struct scsi_host_template * tpnt)
-{
-	static int current_override, current_base;
-	struct Scsi_Host *instance;
-	unsigned int addr;
-	void __iomem *base;
-	int sig, count;
-
-	for (count = 0; current_override < NO_OVERRIDES; ++current_override) {
-		addr = 0;
-		base = NULL;
-
-		if (overrides[current_override].address) {
-			addr = overrides[current_override].address;
-			base = ioremap(addr, 0x2000);
-			if (!base)
-				addr = 0;
-		} else
-			for (; !addr && (current_base < NO_BASES); ++current_base) {
-				dprintk(NDEBUG_INIT, "dtc: probing address 0x%08x\n",
-				        (unsigned int)bases[current_base].address);
-				if (bases[current_base].noauto)
-					continue;
-				base = ioremap(bases[current_base].address, 0x2000);
-				if (!base)
-					continue;
-				for (sig = 0; sig < NO_SIGNATURES; ++sig) {
-					if (check_signature(base + signatures[sig].offset, signatures[sig].string, strlen(signatures[sig].string))) {
-						addr = bases[current_base].address;
-						dprintk(NDEBUG_INIT, "dtc: detected board\n");
-						goto found;
-					}
-				}
-				iounmap(base);
-			}
-
-		dprintk(NDEBUG_INIT, "dtc: addr = 0x%08x\n", addr);
-
-		if (!addr)
-			break;
-
-found:
-		instance = scsi_register(tpnt, sizeof(struct NCR5380_hostdata));
-		if (instance == NULL)
-			goto out_unmap;
-
-		instance->base = addr;
-		((struct NCR5380_hostdata *)(instance)->hostdata)->base = base;
-
-		if (NCR5380_init(instance, FLAG_LATE_DMA_SETUP))
-			goto out_unregister;
-
-		NCR5380_maybe_reset_bus(instance);
-
-		NCR5380_write(DTC_CONTROL_REG, CSR_5380_INTR);	/* Enable int's */
-		if (overrides[current_override].irq != IRQ_AUTO)
-			instance->irq = overrides[current_override].irq;
-		else
-			instance->irq = NCR5380_probe_irq(instance, DTC_IRQS);
-
-		/* Compatibility with documented NCR5380 kernel parameters */
-		if (instance->irq == 255)
-			instance->irq = NO_IRQ;
-
-		/* With interrupts enabled, it will sometimes hang when doing heavy
-		 * reads. So better not enable them until I finger it out. */
-		instance->irq = NO_IRQ;
-
-		if (instance->irq != NO_IRQ)
-			if (request_irq(instance->irq, dtc_intr, 0,
-					"dtc", instance)) {
-				printk(KERN_ERR "scsi%d : IRQ%d not free, interrupts disabled\n", instance->host_no, instance->irq);
-				instance->irq = NO_IRQ;
-			}
-
-		if (instance->irq == NO_IRQ) {
-			printk(KERN_WARNING "scsi%d : interrupts not enabled. for better interactive performance,\n", instance->host_no);
-			printk(KERN_WARNING "scsi%d : please jumper the board for a free IRQ.\n", instance->host_no);
-		}
-
-		dprintk(NDEBUG_INIT, "scsi%d : irq = %d\n",
-		        instance->host_no, instance->irq);
-
-		++current_override;
-		++count;
-	}
-	return count;
-
-out_unregister:
-	scsi_unregister(instance);
-out_unmap:
-	iounmap(base);
-	return count;
-}
-
-/*
- * Function : int dtc_biosparam(Disk * disk, struct block_device *dev, int *ip)
- *
- * Purpose : Generates a BIOS / DOS compatible H-C-S mapping for 
- *	the specified device / size.
- * 
- * Inputs : size = size of device in sectors (512 bytes), dev = block device
- *	major / minor, ip[] = {heads, sectors, cylinders}  
- *
- * Returns : always 0 (success), initializes ip
- *	
-*/
-
-/* 
- * XXX Most SCSI boards use this mapping, I could be incorrect.  Some one
- * using hard disks on a trantor should verify that this mapping corresponds
- * to that used by the BIOS / ASPI driver by running the linux fdisk program
- * and matching the H_C_S coordinates to what DOS uses.
-*/
-
-static int dtc_biosparam(struct scsi_device *sdev, struct block_device *dev,
-			 sector_t capacity, int *ip)
-{
-	int size = capacity;
-
-	ip[0] = 64;
-	ip[1] = 32;
-	ip[2] = size >> 11;
-	return 0;
-}
-
-
-/****************************************************************
- * Function : int NCR5380_pread (struct Scsi_Host *instance, 
- *	unsigned char *dst, int len)
- *
- * Purpose : Fast 5380 pseudo-dma read function, reads len bytes to 
- *	dst
- * 
- * Inputs : dst = destination, len = length in bytes
- *
- * Returns : 0 on success, non zero on a failure such as a watchdog 
- * 	timeout.
-*/
-
-static inline int dtc_pread(struct Scsi_Host *instance,
-                            unsigned char *dst, int len)
-{
-	unsigned char *d = dst;
-	int i;			/* For counting time spent in the poll-loop */
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-
-	i = 0;
-	if (instance->irq == NO_IRQ)
-		NCR5380_write(DTC_CONTROL_REG, CSR_DIR_READ);
-	else
-		NCR5380_write(DTC_CONTROL_REG, CSR_DIR_READ | CSR_INT_BASE);
-	NCR5380_write(DTC_BLK_CNT, len >> 7);	/* Block count */
-	rtrc(1);
-	while (len > 0) {
-		rtrc(2);
-		while (NCR5380_read(DTC_CONTROL_REG) & CSR_HOST_BUF_NOT_RDY)
-			++i;
-		rtrc(3);
-		memcpy_fromio(d, hostdata->base + DTC_DATA_BUF, 128);
-		d += 128;
-		len -= 128;
-		rtrc(7);
-		/*** with int's on, it sometimes hangs after here.
-		 * Looks like something makes HBNR go away. */
-	}
-	rtrc(4);
-	while (!(NCR5380_read(DTC_CONTROL_REG) & D_CR_ACCESS))
-		++i;
-	rtrc(0);
-	return (0);
-}
-
-/****************************************************************
- * Function : int NCR5380_pwrite (struct Scsi_Host *instance, 
- *	unsigned char *src, int len)
- *
- * Purpose : Fast 5380 pseudo-dma write function, transfers len bytes from
- *	src
- * 
- * Inputs : src = source, len = length in bytes
- *
- * Returns : 0 on success, non zero on a failure such as a watchdog 
- * 	timeout.
-*/
-
-static inline int dtc_pwrite(struct Scsi_Host *instance,
-                             unsigned char *src, int len)
-{
-	int i;
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-
-	if (instance->irq == NO_IRQ)
-		NCR5380_write(DTC_CONTROL_REG, 0);
-	else
-		NCR5380_write(DTC_CONTROL_REG, CSR_5380_INTR);
-	NCR5380_write(DTC_BLK_CNT, len >> 7);	/* Block count */
-	for (i = 0; len > 0; ++i) {
-		rtrc(5);
-		/* Poll until the host buffer can accept data. */
-		while (NCR5380_read(DTC_CONTROL_REG) & CSR_HOST_BUF_NOT_RDY)
-			++i;
-		rtrc(3);
-		memcpy_toio(hostdata->base + DTC_DATA_BUF, src, 128);
-		src += 128;
-		len -= 128;
-	}
-	rtrc(4);
-	while (!(NCR5380_read(DTC_CONTROL_REG) & D_CR_ACCESS))
-		++i;
-	rtrc(6);
-	/* Wait until the last byte has been sent to the disk */
-	while (!(NCR5380_read(TARGET_COMMAND_REG) & TCR_LAST_BYTE_SENT))
-		++i;
-	rtrc(7);
-	/* Check for parity error here. fixme. */
-	rtrc(0);
-	return (0);
-}
-
-static int dtc_dma_xfer_len(struct scsi_cmnd *cmd)
-{
-	int transfersize = cmd->transfersize;
-
-	/* Limit transfers to 32K, for xx400 & xx406
-	 * pseudoDMA that transfers in 128 bytes blocks.
-	 */
-	if (transfersize > 32 * 1024 && cmd->SCp.this_residual &&
-	    !(cmd->SCp.this_residual % transfersize))
-		transfersize = 32 * 1024;
-
-	return transfersize;
-}
-
-MODULE_LICENSE("GPL");
-
-#include "NCR5380.c"
-
-static int dtc_release(struct Scsi_Host *shost)
-{
-	struct NCR5380_hostdata *hostdata = shost_priv(shost);
-
-	if (shost->irq != NO_IRQ)
-		free_irq(shost->irq, shost);
-	NCR5380_exit(shost);
-	scsi_unregister(shost);
-	iounmap(hostdata->base);
-	return 0;
-}
-
-static struct scsi_host_template driver_template = {
-	.name			= "DTC 3180/3280",
-	.detect			= dtc_detect,
-	.release		= dtc_release,
-	.proc_name		= "dtc3x80",
-	.info			= dtc_info,
-	.queuecommand		= dtc_queue_command,
-	.eh_abort_handler	= dtc_abort,
-	.eh_bus_reset_handler	= dtc_bus_reset,
-	.bios_param		= dtc_biosparam,
-	.can_queue		= 32,
-	.this_id		= 7,
-	.sg_tablesize		= SG_ALL,
-	.cmd_per_lun		= 2,
-	.use_clustering		= DISABLE_CLUSTERING,
-	.cmd_size		= NCR5380_CMD_SIZE,
-	.max_sectors		= 128,
-};
-#include "scsi_module.c"
diff --git a/drivers/scsi/dtc.h b/drivers/scsi/dtc.h
deleted file mode 100644
index fcb0a8ea7bda..000000000000
--- a/drivers/scsi/dtc.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * DTC controller, taken from T128 driver by...
- * Copyright 1993, Drew Eckhardt
- *	Visionary Computing
- *	(Unix and Linux consulting and custom programming)
- *	drew@colorado.edu
- *      +1 (303) 440-4894
- */
-
-#ifndef DTC3280_H
-#define DTC3280_H
-
-#define NCR5380_implementation_fields \
-    void __iomem *base
-
-#define DTC_address(reg) \
-	(((struct NCR5380_hostdata *)shost_priv(instance))->base + DTC_5380_OFFSET + reg)
-
-#define NCR5380_read(reg) (readb(DTC_address(reg)))
-#define NCR5380_write(reg, value) (writeb(value, DTC_address(reg)))
-
-#define NCR5380_dma_xfer_len(instance, cmd, phase) \
-        dtc_dma_xfer_len(cmd)
-#define NCR5380_dma_recv_setup		dtc_pread
-#define NCR5380_dma_send_setup		dtc_pwrite
-#define NCR5380_dma_residual(instance)	(0)
-
-#define NCR5380_intr			dtc_intr
-#define NCR5380_queue_command		dtc_queue_command
-#define NCR5380_abort			dtc_abort
-#define NCR5380_bus_reset		dtc_bus_reset
-#define NCR5380_info			dtc_info
-
-#define NCR5380_io_delay(x)		udelay(x)
-
-/* 15 12 11 10
-   1001 1100 0000 0000 */
-
-#define DTC_IRQS 0x9c00
-
-
-#endif /* DTC3280_H */
diff --git a/drivers/scsi/esas2r/esas2r_init.c b/drivers/scsi/esas2r/esas2r_init.c
index 78ce4d61a69b..d6e53aee2295 100644
--- a/drivers/scsi/esas2r/esas2r_init.c
+++ b/drivers/scsi/esas2r/esas2r_init.c
@@ -963,10 +963,6 @@ bool esas2r_init_adapter_struct(struct esas2r_adapter *a,
 
 	/* initialize the allocated memory */
 	if (test_bit(AF_FIRST_INIT, &a->flags)) {
-		memset(a->req_table, 0,
-		       (num_requests + num_ae_requests +
-			1) * sizeof(struct esas2r_request *));
-
 		esas2r_targ_db_initialize(a);
 
 		/* prime parts of the inbound list */
diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c
index 2aca4d16f39e..5092c821d088 100644
--- a/drivers/scsi/esas2r/esas2r_main.c
+++ b/drivers/scsi/esas2r/esas2r_main.c
@@ -194,7 +194,7 @@ static ssize_t write_hw(struct file *file, struct kobject *kobj,
 	int length = min(sizeof(struct atto_ioctl), count);
 
 	if (!a->local_atto_ioctl) {
-		a->local_atto_ioctl = kzalloc(sizeof(struct atto_ioctl),
+		a->local_atto_ioctl = kmalloc(sizeof(struct atto_ioctl),
 					      GFP_KERNEL);
 		if (a->local_atto_ioctl == NULL) {
 			esas2r_log(ESAS2R_LOG_WARN,
diff --git a/drivers/scsi/fcoe/fcoe_ctlr.c b/drivers/scsi/fcoe/fcoe_ctlr.c
index a569c65f22b1..dcf36537a767 100644
--- a/drivers/scsi/fcoe/fcoe_ctlr.c
+++ b/drivers/scsi/fcoe/fcoe_ctlr.c
@@ -2923,7 +2923,7 @@ static int fcoe_ctlr_vlan_recv(struct fcoe_ctlr *fip, struct sk_buff *skb)
 	mutex_unlock(&fip->ctlr_mutex);
 
 drop:
-	kfree(skb);
+	kfree_skb(skb);
 	return rc;
 }
 
diff --git a/drivers/scsi/fcoe/fcoe_transport.c b/drivers/scsi/fcoe/fcoe_transport.c
index 7028dd37e5dd..375c536cbc68 100644
--- a/drivers/scsi/fcoe/fcoe_transport.c
+++ b/drivers/scsi/fcoe/fcoe_transport.c
@@ -83,6 +83,41 @@ static struct notifier_block libfcoe_notifier = {
 	.notifier_call = libfcoe_device_notification,
 };
 
+static const struct {
+	u32 fc_port_speed;
+#define SPEED_2000	2000
+#define SPEED_4000	4000
+#define SPEED_8000	8000
+#define SPEED_16000	16000
+#define SPEED_32000	32000
+	u32 eth_port_speed;
+} fcoe_port_speed_mapping[] = {
+	{ FC_PORTSPEED_1GBIT,   SPEED_1000   },
+	{ FC_PORTSPEED_2GBIT,   SPEED_2000   },
+	{ FC_PORTSPEED_4GBIT,   SPEED_4000   },
+	{ FC_PORTSPEED_8GBIT,   SPEED_8000   },
+	{ FC_PORTSPEED_10GBIT,  SPEED_10000  },
+	{ FC_PORTSPEED_16GBIT,  SPEED_16000  },
+	{ FC_PORTSPEED_20GBIT,  SPEED_20000  },
+	{ FC_PORTSPEED_25GBIT,  SPEED_25000  },
+	{ FC_PORTSPEED_32GBIT,  SPEED_32000  },
+	{ FC_PORTSPEED_40GBIT,  SPEED_40000  },
+	{ FC_PORTSPEED_50GBIT,  SPEED_50000  },
+	{ FC_PORTSPEED_100GBIT, SPEED_100000 },
+};
+
+static inline u32 eth2fc_speed(u32 eth_port_speed)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(fcoe_port_speed_mapping); i++) {
+		if (fcoe_port_speed_mapping[i].eth_port_speed == eth_port_speed)
+			return fcoe_port_speed_mapping[i].fc_port_speed;
+	}
+
+	return FC_PORTSPEED_UNKNOWN;
+}
+
 /**
  * fcoe_link_speed_update() - Update the supported and actual link speeds
  * @lport: The local port to update speeds for
@@ -126,23 +161,7 @@ int fcoe_link_speed_update(struct fc_lport *lport)
 			    SUPPORTED_40000baseLR4_Full))
 			lport->link_supported_speeds |= FC_PORTSPEED_40GBIT;
 
-		switch (ecmd.base.speed) {
-		case SPEED_1000:
-			lport->link_speed = FC_PORTSPEED_1GBIT;
-			break;
-		case SPEED_10000:
-			lport->link_speed = FC_PORTSPEED_10GBIT;
-			break;
-		case SPEED_20000:
-			lport->link_speed = FC_PORTSPEED_20GBIT;
-			break;
-		case SPEED_40000:
-			lport->link_speed = FC_PORTSPEED_40GBIT;
-			break;
-		default:
-			lport->link_speed = FC_PORTSPEED_UNKNOWN;
-			break;
-		}
+		lport->link_speed = eth2fc_speed(ecmd.base.speed);
 		return 0;
 	}
 	return -1;
diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h
index 4731d3241323..72c98522bd26 100644
--- a/drivers/scsi/hisi_sas/hisi_sas.h
+++ b/drivers/scsi/hisi_sas/hisi_sas.h
@@ -23,7 +23,7 @@
 #include <scsi/sas_ata.h>
 #include <scsi/libsas.h>
 
-#define DRV_VERSION "v1.5"
+#define DRV_VERSION "v1.6"
 
 #define HISI_SAS_MAX_PHYS	9
 #define HISI_SAS_MAX_QUEUES	32
@@ -56,6 +56,11 @@ enum dev_status {
 	HISI_SAS_DEV_EH,
 };
 
+enum {
+	HISI_SAS_INT_ABT_CMD = 0,
+	HISI_SAS_INT_ABT_DEV = 1,
+};
+
 enum hisi_sas_dev_type {
 	HISI_SAS_DEV_TYPE_STP = 0,
 	HISI_SAS_DEV_TYPE_SSP,
@@ -89,6 +94,13 @@ struct hisi_sas_port {
 
 struct hisi_sas_cq {
 	struct hisi_hba *hisi_hba;
+	int	rd_point;
+	int	id;
+};
+
+struct hisi_sas_dq {
+	struct hisi_hba *hisi_hba;
+	int	wr_point;
 	int	id;
 };
 
@@ -146,6 +158,9 @@ struct hisi_sas_hw {
 			struct hisi_sas_slot *slot);
 	int (*prep_stp)(struct hisi_hba *hisi_hba,
 			struct hisi_sas_slot *slot);
+	int (*prep_abort)(struct hisi_hba *hisi_hba,
+			  struct hisi_sas_slot *slot,
+			  int device_id, int abort_flag, int tag_to_abort);
 	int (*slot_complete)(struct hisi_hba *hisi_hba,
 			     struct hisi_sas_slot *slot, int abort);
 	void (*phy_enable)(struct hisi_hba *hisi_hba, int phy_no);
@@ -185,6 +200,7 @@ struct hisi_hba {
 	struct Scsi_Host *shost;
 
 	struct hisi_sas_cq cq[HISI_SAS_MAX_QUEUES];
+	struct hisi_sas_dq dq[HISI_SAS_MAX_QUEUES];
 	struct hisi_sas_phy phy[HISI_SAS_MAX_PHYS];
 	struct hisi_sas_port port[HISI_SAS_MAX_PHYS];
 
diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
index 18dd5ea2c721..2f872f784e10 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_main.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
@@ -17,6 +17,10 @@
 
 static int hisi_sas_debug_issue_ssp_tmf(struct domain_device *device,
 				u8 *lun, struct hisi_sas_tmf_task *tmf);
+static int
+hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,
+			     struct domain_device *device,
+			     int abort_flag, int tag);
 
 static struct hisi_hba *dev_to_hisi_hba(struct domain_device *device)
 {
@@ -93,7 +97,7 @@ void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task,
 	slot->task = NULL;
 	slot->port = NULL;
 	hisi_sas_slot_index_free(hisi_hba, slot->idx);
-	memset(slot, 0, sizeof(*slot));
+	/* slot memory is fully zeroed when it is reused */
 }
 EXPORT_SYMBOL_GPL(hisi_sas_slot_task_free);
 
@@ -116,6 +120,14 @@ static int hisi_sas_task_prep_ata(struct hisi_hba *hisi_hba,
 	return hisi_hba->hw->prep_stp(hisi_hba, slot);
 }
 
+static int hisi_sas_task_prep_abort(struct hisi_hba *hisi_hba,
+		struct hisi_sas_slot *slot,
+		int device_id, int abort_flag, int tag_to_abort)
+{
+	return hisi_hba->hw->prep_abort(hisi_hba, slot,
+			device_id, abort_flag, tag_to_abort);
+}
+
 /*
  * This function will issue an abort TMF regardless of whether the
  * task is in the sdev or not. Then it will do the task complete
@@ -192,27 +204,13 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_hba *hisi_hba,
 		return rc;
 	}
 	port = device->port->lldd_port;
-	if (port && !port->port_attached && !tmf) {
-		if (sas_protocol_ata(task->task_proto)) {
-			struct task_status_struct *ts = &task->task_status;
-
-			dev_info(dev,
-				 "task prep: SATA/STP port%d not attach device\n",
-				 device->port->id);
-			ts->resp = SAS_TASK_COMPLETE;
-			ts->stat = SAS_PHY_DOWN;
-			task->task_done(task);
-		} else {
-			struct task_status_struct *ts = &task->task_status;
-
-			dev_info(dev,
-				 "task prep: SAS port%d does not attach device\n",
-				 device->port->id);
-			ts->resp = SAS_TASK_UNDELIVERED;
-			ts->stat = SAS_PHY_DOWN;
-			task->task_done(task);
-		}
-		return 0;
+	if (port && !port->port_attached) {
+		dev_info(dev, "task prep: %s port%d not attach device\n",
+			 (sas_protocol_ata(task->task_proto)) ?
+			 "SATA/STP" : "SAS",
+			 device->port->id);
+
+		return SAS_PHY_DOWN;
 	}
 
 	if (!sas_protocol_ata(task->task_proto)) {
@@ -609,6 +607,9 @@ static void hisi_sas_dev_gone(struct domain_device *device)
 	dev_info(dev, "found dev[%lld:%x] is gone\n",
 		 sas_dev->device_id, sas_dev->dev_type);
 
+	hisi_sas_internal_task_abort(hisi_hba, device,
+				     HISI_SAS_INT_ABT_DEV, 0);
+
 	hisi_hba->hw->free_device(hisi_hba, sas_dev);
 	device->lldd_dev = NULL;
 	memset(sas_dev, 0, sizeof(*sas_dev));
@@ -729,6 +730,12 @@ static int hisi_sas_exec_internal_tmf_task(struct domain_device *device,
 		}
 
 		if (task->task_status.resp == SAS_TASK_COMPLETE &&
+			task->task_status.stat == TMF_RESP_FUNC_SUCC) {
+			res = TMF_RESP_FUNC_SUCC;
+			break;
+		}
+
+		if (task->task_status.resp == SAS_TASK_COMPLETE &&
 		      task->task_status.stat == SAS_DATA_UNDERRUN) {
 			/* no error, but return the number of bytes of
 			 * underrun
@@ -826,18 +833,22 @@ static int hisi_sas_abort_task(struct sas_task *task)
 			}
 		}
 
+		hisi_sas_internal_task_abort(hisi_hba, device,
+					     HISI_SAS_INT_ABT_CMD, tag);
 	} else if (task->task_proto & SAS_PROTOCOL_SATA ||
 		task->task_proto & SAS_PROTOCOL_STP) {
 		if (task->dev->dev_type == SAS_SATA_DEV) {
-			struct hisi_slot_info *slot = task->lldd_task;
-
-			dev_notice(dev, "abort task: hba=%p task=%p slot=%p\n",
-				   hisi_hba, task, slot);
-			task->task_state_flags |= SAS_TASK_STATE_ABORTED;
+			hisi_sas_internal_task_abort(hisi_hba, device,
+						     HISI_SAS_INT_ABT_DEV, 0);
 			rc = TMF_RESP_FUNC_COMPLETE;
-			goto out;
 		}
+	} else if (task->task_proto & SAS_PROTOCOL_SMP) {
+		/* SMP */
+		struct hisi_sas_slot *slot = task->lldd_task;
+		u32 tag = slot->idx;
 
+		hisi_sas_internal_task_abort(hisi_hba, device,
+					     HISI_SAS_INT_ABT_CMD, tag);
 	}
 
 out:
@@ -954,6 +965,157 @@ static int hisi_sas_query_task(struct sas_task *task)
 	return rc;
 }
 
+static int
+hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, u64 device_id,
+				  struct sas_task *task, int abort_flag,
+				  int task_tag)
+{
+	struct domain_device *device = task->dev;
+	struct hisi_sas_device *sas_dev = device->lldd_dev;
+	struct device *dev = &hisi_hba->pdev->dev;
+	struct hisi_sas_port *port;
+	struct hisi_sas_slot *slot;
+	struct hisi_sas_cmd_hdr *cmd_hdr_base;
+	int dlvry_queue_slot, dlvry_queue, n_elem = 0, rc, slot_idx;
+
+	if (!device->port)
+		return -1;
+
+	port = device->port->lldd_port;
+
+	/* simply get a slot and send abort command */
+	rc = hisi_sas_slot_index_alloc(hisi_hba, &slot_idx);
+	if (rc)
+		goto err_out;
+	rc = hisi_hba->hw->get_free_slot(hisi_hba, &dlvry_queue,
+					 &dlvry_queue_slot);
+	if (rc)
+		goto err_out_tag;
+
+	slot = &hisi_hba->slot_info[slot_idx];
+	memset(slot, 0, sizeof(struct hisi_sas_slot));
+
+	slot->idx = slot_idx;
+	slot->n_elem = n_elem;
+	slot->dlvry_queue = dlvry_queue;
+	slot->dlvry_queue_slot = dlvry_queue_slot;
+	cmd_hdr_base = hisi_hba->cmd_hdr[dlvry_queue];
+	slot->cmd_hdr = &cmd_hdr_base[dlvry_queue_slot];
+	slot->task = task;
+	slot->port = port;
+	task->lldd_task = slot;
+
+	memset(slot->cmd_hdr, 0, sizeof(struct hisi_sas_cmd_hdr));
+
+	rc = hisi_sas_task_prep_abort(hisi_hba, slot, device_id,
+				      abort_flag, task_tag);
+	if (rc)
+		goto err_out_tag;
+
+	/* Port structure is static for the HBA, so
+	*  even if the port is deformed it is ok
+	*  to reference.
+	*/
+	list_add_tail(&slot->entry, &port->list);
+	spin_lock(&task->task_state_lock);
+	task->task_state_flags |= SAS_TASK_AT_INITIATOR;
+	spin_unlock(&task->task_state_lock);
+
+	hisi_hba->slot_prep = slot;
+
+	sas_dev->running_req++;
+	/* send abort command to our chip */
+	hisi_hba->hw->start_delivery(hisi_hba);
+
+	return 0;
+
+err_out_tag:
+	hisi_sas_slot_index_free(hisi_hba, slot_idx);
+err_out:
+	dev_err(dev, "internal abort task prep: failed[%d]!\n", rc);
+
+	return rc;
+}
+
+/**
+ * hisi_sas_internal_task_abort -- execute an internal
+ * abort command for single IO command or a device
+ * @hisi_hba: host controller struct
+ * @device: domain device
+ * @abort_flag: mode of operation, device or single IO
+ * @tag: tag of IO to be aborted (only relevant to single
+ *       IO mode)
+ */
+static int
+hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,
+			     struct domain_device *device,
+			     int abort_flag, int tag)
+{
+	struct sas_task *task;
+	struct hisi_sas_device *sas_dev = device->lldd_dev;
+	struct device *dev = &hisi_hba->pdev->dev;
+	int res;
+	unsigned long flags;
+
+	if (!hisi_hba->hw->prep_abort)
+		return -EOPNOTSUPP;
+
+	task = sas_alloc_slow_task(GFP_KERNEL);
+	if (!task)
+		return -ENOMEM;
+
+	task->dev = device;
+	task->task_proto = device->tproto;
+	task->task_done = hisi_sas_task_done;
+	task->slow_task->timer.data = (unsigned long)task;
+	task->slow_task->timer.function = hisi_sas_tmf_timedout;
+	task->slow_task->timer.expires = jiffies + 20*HZ;
+	add_timer(&task->slow_task->timer);
+
+	/* Lock as we are alloc'ing a slot, which cannot be interrupted */
+	spin_lock_irqsave(&hisi_hba->lock, flags);
+	res = hisi_sas_internal_abort_task_exec(hisi_hba, sas_dev->device_id,
+						task, abort_flag, tag);
+	spin_unlock_irqrestore(&hisi_hba->lock, flags);
+	if (res) {
+		del_timer(&task->slow_task->timer);
+		dev_err(dev, "internal task abort: executing internal task failed: %d\n",
+			res);
+		goto exit;
+	}
+	wait_for_completion(&task->slow_task->completion);
+	res = TMF_RESP_FUNC_FAILED;
+
+	if (task->task_status.resp == SAS_TASK_COMPLETE &&
+		task->task_status.stat == TMF_RESP_FUNC_COMPLETE) {
+		res = TMF_RESP_FUNC_COMPLETE;
+		goto exit;
+	}
+
+	/* TMF timed out, return direct. */
+	if ((task->task_state_flags & SAS_TASK_STATE_ABORTED)) {
+		if (!(task->task_state_flags & SAS_TASK_STATE_DONE)) {
+			dev_err(dev, "internal task abort: timeout.\n");
+			if (task->lldd_task) {
+				struct hisi_sas_slot *slot = task->lldd_task;
+
+				hisi_sas_slot_task_free(hisi_hba, task, slot);
+			}
+		}
+	}
+
+exit:
+	dev_info(dev, "internal task abort: task to dev %016llx task=%p "
+		"resp: 0x%x sts 0x%x\n",
+		SAS_ADDR(device->sas_addr),
+		task,
+		task->task_status.resp, /* 0 is complete, -1 is undelivered */
+		task->task_status.stat);
+	sas_free_task(task);
+
+	return res;
+}
+
 static void hisi_sas_port_formed(struct asd_sas_phy *sas_phy)
 {
 	hisi_sas_port_notify_formed(sas_phy);
@@ -1063,11 +1225,16 @@ static int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost)
 
 	for (i = 0; i < hisi_hba->queue_count; i++) {
 		struct hisi_sas_cq *cq = &hisi_hba->cq[i];
+		struct hisi_sas_dq *dq = &hisi_hba->dq[i];
 
 		/* Completion queue structure */
 		cq->id = i;
 		cq->hisi_hba = hisi_hba;
 
+		/* Delivery queue structure */
+		dq->id = i;
+		dq->hisi_hba = hisi_hba;
+
 		/* Delivery queue */
 		s = sizeof(struct hisi_sas_cmd_hdr) * HISI_SAS_QUEUE_SLOTS;
 		hisi_hba->cmd_hdr[i] = dma_alloc_coherent(dev, s,
@@ -1128,7 +1295,7 @@ static int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost)
 	memset(hisi_hba->breakpoint, 0, s);
 
 	hisi_hba->slot_index_count = max_command_entries;
-	s = hisi_hba->slot_index_count / sizeof(unsigned long);
+	s = hisi_hba->slot_index_count / BITS_PER_BYTE;
 	hisi_hba->slot_index_tags = devm_kzalloc(dev, s, GFP_KERNEL);
 	if (!hisi_hba->slot_index_tags)
 		goto err_out;
@@ -1272,6 +1439,12 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev,
 				     &hisi_hba->queue_count))
 		goto err_out;
 
+	if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)) &&
+	    dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32))) {
+		dev_err(dev, "No usable DMA addressing method\n");
+		goto err_out;
+	}
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	hisi_hba->regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR(hisi_hba->regs))
@@ -1319,13 +1492,6 @@ int hisi_sas_probe(struct platform_device *pdev,
 	hisi_hba = shost_priv(shost);
 	platform_set_drvdata(pdev, sha);
 
-	if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)) &&
-	    dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32))) {
-		dev_err(dev, "No usable DMA addressing method\n");
-		rc = -EIO;
-		goto err_out_ha;
-	}
-
 	phy_nr = port_nr = hisi_hba->n_phy;
 
 	arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
index 1abbc2e162df..c0ac49d8bc8d 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
@@ -490,25 +490,17 @@ static void config_id_frame_v1_hw(struct hisi_hba *hisi_hba, int phy_no)
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
 			__swab32(identify_buffer[0]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
-			identify_buffer[2]);
+			__swab32(identify_buffer[1]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
-			identify_buffer[1]);
+			__swab32(identify_buffer[2]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
-			identify_buffer[4]);
+			__swab32(identify_buffer[3]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
-			identify_buffer[3]);
+			__swab32(identify_buffer[4]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
 			__swab32(identify_buffer[5]));
 }
 
-static void init_id_frame_v1_hw(struct hisi_hba *hisi_hba)
-{
-	int i;
-
-	for (i = 0; i < hisi_hba->n_phy; i++)
-		config_id_frame_v1_hw(hisi_hba, i);
-}
-
 static void setup_itct_v1_hw(struct hisi_hba *hisi_hba,
 			     struct hisi_sas_device *sas_dev)
 {
@@ -774,8 +766,6 @@ static int hw_init_v1_hw(struct hisi_hba *hisi_hba)
 	msleep(100);
 	init_reg_v1_hw(hisi_hba);
 
-	init_id_frame_v1_hw(hisi_hba);
-
 	return 0;
 }
 
@@ -875,12 +865,13 @@ static int get_wideport_bitmap_v1_hw(struct hisi_hba *hisi_hba, int port_id)
 static int get_free_slot_v1_hw(struct hisi_hba *hisi_hba, int *q, int *s)
 {
 	struct device *dev = &hisi_hba->pdev->dev;
+	struct hisi_sas_dq *dq;
 	u32 r, w;
 	int queue = hisi_hba->queue;
 
 	while (1) {
-		w = hisi_sas_read32_relaxed(hisi_hba,
-				    DLVRY_Q_0_WR_PTR + (queue * 0x14));
+		dq = &hisi_hba->dq[queue];
+		w = dq->wr_point;
 		r = hisi_sas_read32_relaxed(hisi_hba,
 				    DLVRY_Q_0_RD_PTR + (queue * 0x14));
 		if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
@@ -903,10 +894,11 @@ static void start_delivery_v1_hw(struct hisi_hba *hisi_hba)
 {
 	int dlvry_queue = hisi_hba->slot_prep->dlvry_queue;
 	int dlvry_queue_slot = hisi_hba->slot_prep->dlvry_queue_slot;
+	struct hisi_sas_dq *dq = &hisi_hba->dq[dlvry_queue];
 
-	hisi_sas_write32(hisi_hba,
-			 DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
-			 ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS);
+	dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
+	hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
+			 dq->wr_point);
 }
 
 static int prep_prd_sge_v1_hw(struct hisi_hba *hisi_hba,
@@ -1565,14 +1557,11 @@ static irqreturn_t cq_interrupt_v1_hw(int irq, void *p)
 	struct hisi_sas_complete_v1_hdr *complete_queue =
 			(struct hisi_sas_complete_v1_hdr *)
 			hisi_hba->complete_hdr[queue];
-	u32 irq_value, rd_point, wr_point;
+	u32 irq_value, rd_point = cq->rd_point, wr_point;
 
 	irq_value = hisi_sas_read32(hisi_hba, OQ_INT_SRC);
 
 	hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue);
-
-	rd_point = hisi_sas_read32(hisi_hba,
-			COMPL_Q_0_RD_PTR + (0x14 * queue));
 	wr_point = hisi_sas_read32(hisi_hba,
 			COMPL_Q_0_WR_PTR + (0x14 * queue));
 
@@ -1600,6 +1589,7 @@ static irqreturn_t cq_interrupt_v1_hw(int irq, void *p)
 	}
 
 	/* update rd_point */
+	cq->rd_point = rd_point;
 	hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
 
 	return IRQ_HANDLED;
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
index f96560431cf1..9825a3f49f53 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
@@ -117,6 +117,8 @@
 #define SL_CONTROL			(PORT_BASE + 0x94)
 #define SL_CONTROL_NOTIFY_EN_OFF	0
 #define SL_CONTROL_NOTIFY_EN_MSK	(0x1 << SL_CONTROL_NOTIFY_EN_OFF)
+#define SL_CONTROL_CTA_OFF		17
+#define SL_CONTROL_CTA_MSK		(0x1 << SL_CONTROL_CTA_OFF)
 #define TX_ID_DWORD0			(PORT_BASE + 0x9c)
 #define TX_ID_DWORD1			(PORT_BASE + 0xa0)
 #define TX_ID_DWORD2			(PORT_BASE + 0xa4)
@@ -124,6 +126,9 @@
 #define TX_ID_DWORD4			(PORT_BASE + 0xaC)
 #define TX_ID_DWORD5			(PORT_BASE + 0xb0)
 #define TX_ID_DWORD6			(PORT_BASE + 0xb4)
+#define TXID_AUTO			(PORT_BASE + 0xb8)
+#define TXID_AUTO_CT3_OFF		1
+#define TXID_AUTO_CT3_MSK		(0x1 << TXID_AUTO_CT3_OFF)
 #define RX_IDAF_DWORD0			(PORT_BASE + 0xc4)
 #define RX_IDAF_DWORD1			(PORT_BASE + 0xc8)
 #define RX_IDAF_DWORD2			(PORT_BASE + 0xcc)
@@ -174,6 +179,10 @@
 /* HW dma structures */
 /* Delivery queue header */
 /* dw0 */
+#define CMD_HDR_ABORT_FLAG_OFF		0
+#define CMD_HDR_ABORT_FLAG_MSK		(0x3 << CMD_HDR_ABORT_FLAG_OFF)
+#define CMD_HDR_ABORT_DEVICE_TYPE_OFF	2
+#define CMD_HDR_ABORT_DEVICE_TYPE_MSK	(0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF)
 #define CMD_HDR_RESP_REPORT_OFF		5
 #define CMD_HDR_RESP_REPORT_MSK		(0x1 << CMD_HDR_RESP_REPORT_OFF)
 #define CMD_HDR_TLR_CTRL_OFF		6
@@ -214,6 +223,8 @@
 #define CMD_HDR_DIF_SGL_LEN_MSK		(0xffff << CMD_HDR_DIF_SGL_LEN_OFF)
 #define CMD_HDR_DATA_SGL_LEN_OFF	16
 #define CMD_HDR_DATA_SGL_LEN_MSK	(0xffff << CMD_HDR_DATA_SGL_LEN_OFF)
+#define CMD_HDR_ABORT_IPTT_OFF		16
+#define CMD_HDR_ABORT_IPTT_MSK		(0xffff << CMD_HDR_ABORT_IPTT_OFF)
 
 /* Completion header */
 /* dw0 */
@@ -221,6 +232,13 @@
 #define CMPLT_HDR_RSPNS_XFRD_MSK	(0x1 << CMPLT_HDR_RSPNS_XFRD_OFF)
 #define CMPLT_HDR_ERX_OFF		12
 #define CMPLT_HDR_ERX_MSK		(0x1 << CMPLT_HDR_ERX_OFF)
+#define CMPLT_HDR_ABORT_STAT_OFF	13
+#define CMPLT_HDR_ABORT_STAT_MSK	(0x7 << CMPLT_HDR_ABORT_STAT_OFF)
+/* abort_stat */
+#define STAT_IO_NOT_VALID		0x1
+#define STAT_IO_NO_DEVICE		0x2
+#define STAT_IO_COMPLETE		0x3
+#define STAT_IO_ABORTED			0x4
 /* dw1 */
 #define CMPLT_HDR_IPTT_OFF		0
 #define CMPLT_HDR_IPTT_MSK		(0xffff << CMPLT_HDR_IPTT_OFF)
@@ -549,25 +567,17 @@ static void config_id_frame_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
 			__swab32(identify_buffer[0]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
-			identify_buffer[2]);
+			__swab32(identify_buffer[1]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
-			identify_buffer[1]);
+			__swab32(identify_buffer[2]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
-			identify_buffer[4]);
+			__swab32(identify_buffer[3]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
-			identify_buffer[3]);
+			__swab32(identify_buffer[4]));
 	hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
 			__swab32(identify_buffer[5]));
 }
 
-static void init_id_frame_v2_hw(struct hisi_hba *hisi_hba)
-{
-	int i;
-
-	for (i = 0; i < hisi_hba->n_phy; i++)
-		config_id_frame_v2_hw(hisi_hba, i);
-}
-
 static void setup_itct_v2_hw(struct hisi_hba *hisi_hba,
 			     struct hisi_sas_device *sas_dev)
 {
@@ -589,6 +599,7 @@ static void setup_itct_v2_hw(struct hisi_hba *hisi_hba,
 		qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF;
 		break;
 	case SAS_SATA_DEV:
+	case SAS_SATA_PENDING:
 		if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
 			qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF;
 		else
@@ -672,9 +683,7 @@ static int reset_hw_v2_hw(struct hisi_hba *hisi_hba)
 	else
 		reset_val = 0x7ffff;
 
-	/* Disable all of the DQ */
-	for (i = 0; i < HISI_SAS_MAX_QUEUES; i++)
-		hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
+	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
 
 	/* Disable all of the PHYs */
 	for (i = 0; i < hisi_hba->n_phy; i++) {
@@ -810,6 +819,8 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba)
 		hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, 0x855);
 		hisi_sas_phy_write32(hisi_hba, i, SAS_PHY_CTRL, 0x30b9908);
 		hisi_sas_phy_write32(hisi_hba, i, SL_TOUT_CFG, 0x7d7d7d7d);
+		hisi_sas_phy_write32(hisi_hba, i, SL_CONTROL, 0x0);
+		hisi_sas_phy_write32(hisi_hba, i, TXID_AUTO, 0x2);
 		hisi_sas_phy_write32(hisi_hba, i, DONE_RECEIVED_TIME, 0x10);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
@@ -901,8 +912,6 @@ static int hw_init_v2_hw(struct hisi_hba *hisi_hba)
 	msleep(100);
 	init_reg_v2_hw(hisi_hba);
 
-	init_id_frame_v2_hw(hisi_hba);
-
 	return 0;
 }
 
@@ -952,14 +961,8 @@ static void start_phys_v2_hw(unsigned long data)
 
 static void phys_init_v2_hw(struct hisi_hba *hisi_hba)
 {
-	int i;
 	struct timer_list *timer = &hisi_hba->timer;
 
-	for (i = 0; i < hisi_hba->n_phy; i++) {
-		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0x6a);
-		hisi_sas_phy_read32(hisi_hba, i, CHL_INT2_MSK);
-	}
-
 	setup_timer(timer, start_phys_v2_hw, (unsigned long)hisi_hba);
 	mod_timer(timer, jiffies + HZ);
 }
@@ -1010,12 +1013,13 @@ static int get_wideport_bitmap_v2_hw(struct hisi_hba *hisi_hba, int port_id)
 static int get_free_slot_v2_hw(struct hisi_hba *hisi_hba, int *q, int *s)
 {
 	struct device *dev = &hisi_hba->pdev->dev;
+	struct hisi_sas_dq *dq;
 	u32 r, w;
 	int queue = hisi_hba->queue;
 
 	while (1) {
-		w = hisi_sas_read32_relaxed(hisi_hba,
-					    DLVRY_Q_0_WR_PTR + (queue * 0x14));
+		dq = &hisi_hba->dq[queue];
+		w = dq->wr_point;
 		r = hisi_sas_read32_relaxed(hisi_hba,
 					    DLVRY_Q_0_RD_PTR + (queue * 0x14));
 		if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
@@ -1038,9 +1042,11 @@ static void start_delivery_v2_hw(struct hisi_hba *hisi_hba)
 {
 	int dlvry_queue = hisi_hba->slot_prep->dlvry_queue;
 	int dlvry_queue_slot = hisi_hba->slot_prep->dlvry_queue_slot;
+	struct hisi_sas_dq *dq = &hisi_hba->dq[dlvry_queue];
 
+	dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
 	hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
-			 ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS);
+			 dq->wr_point);
 }
 
 static int prep_prd_sge_v2_hw(struct hisi_hba *hisi_hba,
@@ -1563,6 +1569,30 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot,
 		goto out;
 	}
 
+	/* Use SAS+TMF status codes */
+	switch ((complete_hdr->dw0 & CMPLT_HDR_ABORT_STAT_MSK)
+			>> CMPLT_HDR_ABORT_STAT_OFF) {
+	case STAT_IO_ABORTED:
+		/* this io has been aborted by abort command */
+		ts->stat = SAS_ABORTED_TASK;
+		goto out;
+	case STAT_IO_COMPLETE:
+		/* internal abort command complete */
+		ts->stat = TMF_RESP_FUNC_COMPLETE;
+		goto out;
+	case STAT_IO_NO_DEVICE:
+		ts->stat = TMF_RESP_FUNC_COMPLETE;
+		goto out;
+	case STAT_IO_NOT_VALID:
+		/* abort single io, controller don't find
+		 * the io need to abort
+		 */
+		ts->stat = TMF_RESP_FUNC_FAILED;
+		goto out;
+	default:
+		break;
+	}
+
 	if ((complete_hdr->dw0 & CMPLT_HDR_ERX_MSK) &&
 		(!(complete_hdr->dw0 & CMPLT_HDR_RSPNS_XFRD_MSK))) {
 
@@ -1775,6 +1805,32 @@ static int prep_ata_v2_hw(struct hisi_hba *hisi_hba,
 	return 0;
 }
 
+static int prep_abort_v2_hw(struct hisi_hba *hisi_hba,
+		struct hisi_sas_slot *slot,
+		int device_id, int abort_flag, int tag_to_abort)
+{
+	struct sas_task *task = slot->task;
+	struct domain_device *dev = task->dev;
+	struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
+	struct hisi_sas_port *port = slot->port;
+
+	/* dw0 */
+	hdr->dw0 = cpu_to_le32((5 << CMD_HDR_CMD_OFF) | /*abort*/
+			       (port->id << CMD_HDR_PORT_OFF) |
+			       ((dev_is_sata(dev) ? 1:0) <<
+				CMD_HDR_ABORT_DEVICE_TYPE_OFF) |
+			       (abort_flag << CMD_HDR_ABORT_FLAG_OFF));
+
+	/* dw1 */
+	hdr->dw1 = cpu_to_le32(device_id << CMD_HDR_DEV_ID_OFF);
+
+	/* dw7 */
+	hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
+	hdr->transfer_tags = cpu_to_le32(slot->idx);
+
+	return 0;
+}
+
 static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
 {
 	int i, res = 0;
@@ -1818,9 +1874,6 @@ static int phy_up_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
 		frame_rcvd[i] = __swab32(idaf);
 	}
 
-	/* Get the linkrates */
-	link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE);
-	link_rate = (link_rate >> (phy_no * 4)) & 0xf;
 	sas_phy->linkrate = link_rate;
 	hard_phy_linkrate = hisi_sas_phy_read32(hisi_hba, phy_no,
 						HARD_PHY_LINKRATE);
@@ -1855,16 +1908,21 @@ end:
 static int phy_down_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
 {
 	int res = 0;
-	u32 phy_cfg, phy_state;
+	u32 phy_state, sl_ctrl, txid_auto;
 
 	hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1);
 
-	phy_cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
-
 	phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
-
 	hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0);
 
+	sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
+	hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL,
+			     sl_ctrl & ~SL_CONTROL_CTA_MSK);
+
+	txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
+	hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
+			     txid_auto | TXID_AUTO_CT3_MSK);
+
 	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
 	hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0);
 
@@ -1986,7 +2044,7 @@ static irqreturn_t cq_interrupt_v2_hw(int irq_no, void *p)
 	struct hisi_sas_slot *slot;
 	struct hisi_sas_itct *itct;
 	struct hisi_sas_complete_v2_hdr *complete_queue;
-	u32 irq_value, rd_point, wr_point, dev_id;
+	u32 irq_value, rd_point = cq->rd_point, wr_point, dev_id;
 	int queue = cq->id;
 
 	complete_queue = hisi_hba->complete_hdr[queue];
@@ -1994,8 +2052,6 @@ static irqreturn_t cq_interrupt_v2_hw(int irq_no, void *p)
 
 	hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue);
 
-	rd_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_RD_PTR +
-				   (0x14 * queue));
 	wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
 				   (0x14 * queue));
 
@@ -2043,6 +2099,7 @@ static irqreturn_t cq_interrupt_v2_hw(int irq_no, void *p)
 	}
 
 	/* update rd_point */
+	cq->rd_point = rd_point;
 	hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
 	return IRQ_HANDLED;
 }
@@ -2239,6 +2296,7 @@ static const struct hisi_sas_hw hisi_sas_v2_hw = {
 	.prep_smp = prep_smp_v2_hw,
 	.prep_ssp = prep_ssp_v2_hw,
 	.prep_stp = prep_ata_v2_hw,
+	.prep_abort = prep_abort_v2_hw,
 	.get_free_slot = get_free_slot_v2_hw,
 	.start_delivery = start_delivery_v2_hw,
 	.slot_complete = slot_complete_v2_hw,
diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index ba9af4a2bd2a..258a3f9a2519 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -246,10 +246,6 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
 
 	shost->dma_dev = dma_dev;
 
-	error = device_add(&shost->shost_gendev);
-	if (error)
-		goto out_destroy_freelist;
-
 	/*
 	 * Increase usage count temporarily here so that calling
 	 * scsi_autopm_put_host() will trigger runtime idle if there is
@@ -260,6 +256,10 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
 	pm_runtime_enable(&shost->shost_gendev);
 	device_enable_async_suspend(&shost->shost_gendev);
 
+	error = device_add(&shost->shost_gendev);
+	if (error)
+		goto out_destroy_freelist;
+
 	scsi_host_set_state(shost, SHOST_RUNNING);
 	get_device(shost->shost_gendev.parent);
 
@@ -309,6 +309,10 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
  out_del_gendev:
 	device_del(&shost->shost_gendev);
  out_destroy_freelist:
+	device_disable_async_suspend(&shost->shost_gendev);
+	pm_runtime_disable(&shost->shost_gendev);
+	pm_runtime_set_suspended(&shost->shost_gendev);
+	pm_runtime_put_noidle(&shost->shost_gendev);
 	scsi_destroy_command_freelist(shost);
  out_destroy_tags:
 	if (shost_use_blk_mq(shost))
@@ -486,6 +490,8 @@ struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize)
 	else
 		shost->dma_boundary = 0xffffffff;
 
+	shost->use_blk_mq = scsi_use_blk_mq;
+
 	device_initialize(&shost->shost_gendev);
 	dev_set_name(&shost->shost_gendev, "host%d", shost->host_no);
 	shost->shost_gendev.bus = &scsi_bus_type;
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index 030d0023e1d2..d007ec18179a 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -293,6 +293,8 @@ static int detect_controller_lockup(struct ctlr_info *h);
 static void hpsa_disable_rld_caching(struct ctlr_info *h);
 static inline int hpsa_scsi_do_report_phys_luns(struct ctlr_info *h,
 	struct ReportExtendedLUNdata *buf, int bufsize);
+static bool hpsa_vpd_page_supported(struct ctlr_info *h,
+	unsigned char scsi3addr[], u8 page);
 static int hpsa_luns_changed(struct ctlr_info *h);
 static bool hpsa_cmd_dev_match(struct ctlr_info *h, struct CommandList *c,
 			       struct hpsa_scsi_dev_t *dev,
@@ -2388,7 +2390,8 @@ static void hpsa_cmd_free_and_done(struct ctlr_info *h,
 		struct CommandList *c, struct scsi_cmnd *cmd)
 {
 	hpsa_cmd_resolve_and_free(h, c);
-	cmd->scsi_done(cmd);
+	if (cmd && cmd->scsi_done)
+		cmd->scsi_done(cmd);
 }
 
 static void hpsa_retry_cmd(struct ctlr_info *h, struct CommandList *c)
@@ -2489,7 +2492,17 @@ static void complete_scsi_command(struct CommandList *cp)
 	ei = cp->err_info;
 	cmd = cp->scsi_cmd;
 	h = cp->h;
+
+	if (!cmd->device) {
+		cmd->result = DID_NO_CONNECT << 16;
+		return hpsa_cmd_free_and_done(h, cp, cmd);
+	}
+
 	dev = cmd->device->hostdata;
+	if (!dev) {
+		cmd->result = DID_NO_CONNECT << 16;
+		return hpsa_cmd_free_and_done(h, cp, cmd);
+	}
 	c2 = &h->ioaccel2_cmd_pool[cp->cmdindex];
 
 	scsi_dma_unmap(cmd); /* undo the DMA mappings */
@@ -2504,8 +2517,15 @@ static void complete_scsi_command(struct CommandList *cp)
 	cmd->result = (DID_OK << 16); 		/* host byte */
 	cmd->result |= (COMMAND_COMPLETE << 8);	/* msg byte */
 
-	if (cp->cmd_type == CMD_IOACCEL2 || cp->cmd_type == CMD_IOACCEL1)
-		atomic_dec(&cp->phys_disk->ioaccel_cmds_out);
+	if (cp->cmd_type == CMD_IOACCEL2 || cp->cmd_type == CMD_IOACCEL1) {
+		if (dev->physical_device && dev->expose_device &&
+			dev->removed) {
+			cmd->result = DID_NO_CONNECT << 16;
+			return hpsa_cmd_free_and_done(h, cp, cmd);
+		}
+		if (likely(cp->phys_disk != NULL))
+			atomic_dec(&cp->phys_disk->ioaccel_cmds_out);
+	}
 
 	/*
 	 * We check for lockup status here as it may be set for
@@ -3074,11 +3094,19 @@ static void hpsa_get_raid_level(struct ctlr_info *h,
 	buf = kzalloc(64, GFP_KERNEL);
 	if (!buf)
 		return;
-	rc = hpsa_scsi_do_inquiry(h, scsi3addr, VPD_PAGE | 0xC1, buf, 64);
+
+	if (!hpsa_vpd_page_supported(h, scsi3addr,
+		HPSA_VPD_LV_DEVICE_GEOMETRY))
+		goto exit;
+
+	rc = hpsa_scsi_do_inquiry(h, scsi3addr, VPD_PAGE |
+		HPSA_VPD_LV_DEVICE_GEOMETRY, buf, 64);
+
 	if (rc == 0)
 		*raid_level = buf[8];
 	if (*raid_level > RAID_UNKNOWN)
 		*raid_level = RAID_UNKNOWN;
+exit:
 	kfree(buf);
 	return;
 }
@@ -3436,7 +3464,7 @@ static void hpsa_get_sas_address(struct ctlr_info *h, unsigned char *scsi3addr,
 }
 
 /* Get a device id from inquiry page 0x83 */
-static int hpsa_vpd_page_supported(struct ctlr_info *h,
+static bool hpsa_vpd_page_supported(struct ctlr_info *h,
 	unsigned char scsi3addr[], u8 page)
 {
 	int rc;
@@ -3446,7 +3474,7 @@ static int hpsa_vpd_page_supported(struct ctlr_info *h,
 
 	buf = kzalloc(256, GFP_KERNEL);
 	if (!buf)
-		return 0;
+		return false;
 
 	/* Get the size of the page list first */
 	rc = hpsa_scsi_do_inquiry(h, scsi3addr,
@@ -3473,10 +3501,10 @@ static int hpsa_vpd_page_supported(struct ctlr_info *h,
 			goto exit_supported;
 exit_unsupported:
 	kfree(buf);
-	return 0;
+	return false;
 exit_supported:
 	kfree(buf);
-	return 1;
+	return true;
 }
 
 static void hpsa_get_ioaccel_status(struct ctlr_info *h,
@@ -3525,18 +3553,25 @@ static int hpsa_get_device_id(struct ctlr_info *h, unsigned char *scsi3addr,
 	int rc;
 	unsigned char *buf;
 
-	if (buflen > 16)
-		buflen = 16;
+	/* Does controller have VPD for device id? */
+	if (!hpsa_vpd_page_supported(h, scsi3addr, HPSA_VPD_LV_DEVICE_ID))
+		return 1; /* not supported */
+
 	buf = kzalloc(64, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
-	rc = hpsa_scsi_do_inquiry(h, scsi3addr, VPD_PAGE | 0x83, buf, 64);
-	if (rc == 0)
-		memcpy(device_id, &buf[index], buflen);
+
+	rc = hpsa_scsi_do_inquiry(h, scsi3addr, VPD_PAGE |
+					HPSA_VPD_LV_DEVICE_ID, buf, 64);
+	if (rc == 0) {
+		if (buflen > 16)
+			buflen = 16;
+		memcpy(device_id, &buf[8], buflen);
+	}
 
 	kfree(buf);
 
-	return rc != 0;
+	return rc; /*0 - got id,  otherwise, didn't */
 }
 
 static int hpsa_scsi_do_report_luns(struct ctlr_info *h, int logical,
@@ -3807,8 +3842,15 @@ static int hpsa_update_device_info(struct ctlr_info *h,
 		sizeof(this_device->model));
 	memset(this_device->device_id, 0,
 		sizeof(this_device->device_id));
-	hpsa_get_device_id(h, scsi3addr, this_device->device_id, 8,
-		sizeof(this_device->device_id));
+	if (hpsa_get_device_id(h, scsi3addr, this_device->device_id, 8,
+		sizeof(this_device->device_id)))
+		dev_err(&h->pdev->dev,
+			"hpsa%d: %s: can't get device id for host %d:C0:T%d:L%d\t%s\t%.16s\n",
+			h->ctlr, __func__,
+			h->scsi_host->host_no,
+			this_device->target, this_device->lun,
+			scsi_device_type(this_device->devtype),
+			this_device->model);
 
 	if ((this_device->devtype == TYPE_DISK ||
 		this_device->devtype == TYPE_ZBC) &&
@@ -4034,7 +4076,17 @@ static void hpsa_get_ioaccel_drive_info(struct ctlr_info *h,
 		struct bmic_identify_physical_device *id_phys)
 {
 	int rc;
-	struct ext_report_lun_entry *rle = &rlep->LUN[rle_index];
+	struct ext_report_lun_entry *rle;
+
+	/*
+	 * external targets don't support BMIC
+	 */
+	if (dev->external) {
+		dev->queue_depth = 7;
+		return;
+	}
+
+	rle = &rlep->LUN[rle_index];
 
 	dev->ioaccel_handle = rle->ioaccel_handle;
 	if ((rle->device_flags & 0x08) && dev->ioaccel_handle)
@@ -4270,6 +4322,11 @@ static void hpsa_update_scsi_devices(struct ctlr_info *h)
 		lunaddrbytes = figure_lunaddrbytes(h, raid_ctlr_position,
 			i, nphysicals, nlogicals, physdev_list, logdev_list);
 
+		/* Determine if this is a lun from an external target array */
+		tmpdevice->external =
+			figure_external_status(h, raid_ctlr_position, i,
+						nphysicals, nlocal_logicals);
+
 		/*
 		 * Skip over some devices such as a spare.
 		 */
@@ -4295,11 +4352,6 @@ static void hpsa_update_scsi_devices(struct ctlr_info *h)
 			continue;
 		}
 
-		/* Determine if this is a lun from an external target array */
-		tmpdevice->external =
-			figure_external_status(h, raid_ctlr_position, i,
-						nphysicals, nlocal_logicals);
-
 		figure_bus_target_lun(h, lunaddrbytes, tmpdevice);
 		hpsa_update_device_supports_aborts(h, tmpdevice, lunaddrbytes);
 		this_device = currentsd[ncurrent];
@@ -4513,7 +4565,9 @@ static int fixup_ioaccel_cdb(u8 *cdb, int *cdb_len)
 	case READ_6:
 	case READ_12:
 		if (*cdb_len == 6) {
-			block = get_unaligned_be16(&cdb[2]);
+			block = (((cdb[1] & 0x1F) << 16) |
+				(cdb[2] << 8) |
+				cdb[3]);
 			block_cnt = cdb[4];
 			if (block_cnt == 0)
 				block_cnt = 256;
@@ -4638,6 +4692,9 @@ static int hpsa_scsi_ioaccel_direct_map(struct ctlr_info *h,
 	struct scsi_cmnd *cmd = c->scsi_cmd;
 	struct hpsa_scsi_dev_t *dev = cmd->device->hostdata;
 
+	if (!dev)
+		return -1;
+
 	c->phys_disk = dev;
 
 	return hpsa_scsi_ioaccel_queue_command(h, c, dev->ioaccel_handle,
@@ -4670,9 +4727,11 @@ static void set_encrypt_ioaccel2(struct ctlr_info *h,
 	 */
 	switch (cmd->cmnd[0]) {
 	/* Required? 6-byte cdbs eliminated by fixup_ioaccel_cdb */
-	case WRITE_6:
 	case READ_6:
-		first_block = get_unaligned_be16(&cmd->cmnd[2]);
+	case WRITE_6:
+		first_block = (((cmd->cmnd[1] & 0x1F) << 16) |
+				(cmd->cmnd[2] << 8) |
+				cmd->cmnd[3]);
 		break;
 	case WRITE_10:
 	case READ_10:
@@ -4714,6 +4773,12 @@ static int hpsa_scsi_ioaccel2_queue_command(struct ctlr_info *h,
 	u32 len;
 	u32 total_len = 0;
 
+	if (!cmd->device)
+		return -1;
+
+	if (!cmd->device->hostdata)
+		return -1;
+
 	BUG_ON(scsi_sg_count(cmd) > h->maxsgentries);
 
 	if (fixup_ioaccel_cdb(cdb, &cdb_len)) {
@@ -4822,6 +4887,12 @@ static int hpsa_scsi_ioaccel_queue_command(struct ctlr_info *h,
 	struct CommandList *c, u32 ioaccel_handle, u8 *cdb, int cdb_len,
 	u8 *scsi3addr, struct hpsa_scsi_dev_t *phys_disk)
 {
+	if (!c->scsi_cmd->device)
+		return -1;
+
+	if (!c->scsi_cmd->device->hostdata)
+		return -1;
+
 	/* Try to honor the device's queue depth */
 	if (atomic_inc_return(&phys_disk->ioaccel_cmds_out) >
 					phys_disk->queue_depth) {
@@ -4902,12 +4973,17 @@ static int hpsa_scsi_ioaccel_raid_map(struct ctlr_info *h,
 #endif
 	int offload_to_mirror;
 
+	if (!dev)
+		return -1;
+
 	/* check for valid opcode, get LBA and block count */
 	switch (cmd->cmnd[0]) {
 	case WRITE_6:
 		is_write = 1;
 	case READ_6:
-		first_block = get_unaligned_be16(&cmd->cmnd[2]);
+		first_block = (((cmd->cmnd[1] & 0x1F) << 16) |
+				(cmd->cmnd[2] << 8) |
+				cmd->cmnd[3]);
 		block_cnt = cmd->cmnd[4];
 		if (block_cnt == 0)
 			block_cnt = 256;
@@ -5314,6 +5390,9 @@ static int hpsa_ioaccel_submit(struct ctlr_info *h,
 	struct hpsa_scsi_dev_t *dev = cmd->device->hostdata;
 	int rc = IO_ACCEL_INELIGIBLE;
 
+	if (!dev)
+		return SCSI_MLQUEUE_HOST_BUSY;
+
 	cmd->host_scribble = (unsigned char *) c;
 
 	if (dev->offload_enabled) {
@@ -5852,6 +5931,9 @@ static void setup_ioaccel2_abort_cmd(struct CommandList *c, struct ctlr_info *h,
 	struct scsi_cmnd *scmd = command_to_abort->scsi_cmd;
 	struct hpsa_scsi_dev_t *dev = scmd->device->hostdata;
 
+	if (!dev)
+		return;
+
 	/*
 	 * We're overlaying struct hpsa_tmf_struct on top of something which
 	 * was allocated as a struct io_accel2_cmd, so we better be sure it
@@ -5935,7 +6017,7 @@ static int hpsa_send_reset_as_abort_ioaccel2(struct ctlr_info *h,
 			"Reset as abort: Resetting physical device at scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
 			psa[0], psa[1], psa[2], psa[3],
 			psa[4], psa[5], psa[6], psa[7]);
-	rc = hpsa_do_reset(h, dev, psa, HPSA_RESET_TYPE_TARGET, reply_queue);
+	rc = hpsa_do_reset(h, dev, psa, HPSA_PHYS_TARGET_RESET, reply_queue);
 	if (rc != 0) {
 		dev_warn(&h->pdev->dev,
 			"Reset as abort: Failed on physical device at scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
@@ -5972,6 +6054,9 @@ static int hpsa_send_abort_ioaccel2(struct ctlr_info *h,
 	struct io_accel2_cmd *c2;
 
 	dev = abort->scsi_cmd->device->hostdata;
+	if (!dev)
+		return -1;
+
 	if (!dev->offload_enabled && !dev->hba_ioaccel_enabled)
 		return -1;
 
diff --git a/drivers/scsi/hpsa.h b/drivers/scsi/hpsa.h
index a1487e67f7a1..82cdfad874f3 100644
--- a/drivers/scsi/hpsa.h
+++ b/drivers/scsi/hpsa.h
@@ -312,7 +312,6 @@ struct offline_device_entry {
 #define HPSA_DEVICE_RESET_MSG 1
 #define HPSA_RESET_TYPE_CONTROLLER 0x00
 #define HPSA_RESET_TYPE_BUS 0x01
-#define HPSA_RESET_TYPE_TARGET 0x03
 #define HPSA_RESET_TYPE_LUN 0x04
 #define HPSA_PHYS_TARGET_RESET 0x99 /* not defined by cciss spec */
 #define HPSA_MSG_SEND_RETRY_LIMIT 10
diff --git a/drivers/scsi/hpsa_cmd.h b/drivers/scsi/hpsa_cmd.h
index a5be153d92d4..a584cdf07058 100644
--- a/drivers/scsi/hpsa_cmd.h
+++ b/drivers/scsi/hpsa_cmd.h
@@ -157,6 +157,7 @@
 
 /* VPD Inquiry types */
 #define HPSA_VPD_SUPPORTED_PAGES        0x00
+#define HPSA_VPD_LV_DEVICE_ID           0x83
 #define HPSA_VPD_LV_DEVICE_GEOMETRY     0xC1
 #define HPSA_VPD_LV_IOACCEL_STATUS      0xC2
 #define HPSA_VPD_LV_STATUS		0xC3
diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c
index ab67ec4b6bd6..7e487c78279c 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc.c
+++ b/drivers/scsi/ibmvscsi/ibmvfc.c
@@ -52,6 +52,7 @@ static unsigned int max_requests = IBMVFC_MAX_REQUESTS_DEFAULT;
 static unsigned int disc_threads = IBMVFC_MAX_DISC_THREADS;
 static unsigned int ibmvfc_debug = IBMVFC_DEBUG;
 static unsigned int log_level = IBMVFC_DEFAULT_LOG_LEVEL;
+static unsigned int cls3_error = IBMVFC_CLS3_ERROR;
 static LIST_HEAD(ibmvfc_head);
 static DEFINE_SPINLOCK(ibmvfc_driver_lock);
 static struct scsi_transport_template *ibmvfc_transport_template;
@@ -86,6 +87,9 @@ MODULE_PARM_DESC(debug, "Enable driver debug information. "
 module_param_named(log_level, log_level, uint, 0);
 MODULE_PARM_DESC(log_level, "Set to 0 - 4 for increasing verbosity of device driver. "
 		 "[Default=" __stringify(IBMVFC_DEFAULT_LOG_LEVEL) "]");
+module_param_named(cls3_error, cls3_error, uint, 0);
+MODULE_PARM_DESC(cls3_error, "Enable FC Class 3 Error Recovery. "
+		 "[Default=" __stringify(IBMVFC_CLS3_ERROR) "]");
 
 static const struct {
 	u16 status;
@@ -717,7 +721,6 @@ static int ibmvfc_reset_crq(struct ibmvfc_host *vhost)
 	spin_lock_irqsave(vhost->host->host_lock, flags);
 	vhost->state = IBMVFC_NO_CRQ;
 	vhost->logged_in = 0;
-	ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_NONE);
 
 	/* Clean out the queue */
 	memset(crq->msgs, 0, PAGE_SIZE);
@@ -1335,6 +1338,9 @@ static int ibmvfc_map_sg_data(struct scsi_cmnd *scmd,
 	struct srp_direct_buf *data = &vfc_cmd->ioba;
 	struct ibmvfc_host *vhost = dev_get_drvdata(dev);
 
+	if (cls3_error)
+		vfc_cmd->flags |= cpu_to_be16(IBMVFC_CLASS_3_ERR);
+
 	sg_mapped = scsi_dma_map(scmd);
 	if (!sg_mapped) {
 		vfc_cmd->flags |= cpu_to_be16(IBMVFC_NO_MEM_DESC);
@@ -3381,6 +3387,10 @@ static void ibmvfc_tgt_send_prli(struct ibmvfc_target *tgt)
 	prli->parms.type = IBMVFC_SCSI_FCP_TYPE;
 	prli->parms.flags = cpu_to_be16(IBMVFC_PRLI_EST_IMG_PAIR);
 	prli->parms.service_parms = cpu_to_be32(IBMVFC_PRLI_INITIATOR_FUNC);
+	prli->parms.service_parms |= cpu_to_be32(IBMVFC_PRLI_READ_FCP_XFER_RDY_DISABLED);
+
+	if (cls3_error)
+		prli->parms.service_parms |= cpu_to_be32(IBMVFC_PRLI_RETRY);
 
 	ibmvfc_set_tgt_action(tgt, IBMVFC_TGT_ACTION_INIT_WAIT);
 	if (ibmvfc_send_event(evt, vhost, default_timeout)) {
diff --git a/drivers/scsi/ibmvscsi/ibmvfc.h b/drivers/scsi/ibmvscsi/ibmvfc.h
index 5c70a52ad346..9a0696f68f37 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc.h
+++ b/drivers/scsi/ibmvscsi/ibmvfc.h
@@ -54,6 +54,7 @@
 #define IBMVFC_DEV_LOSS_TMO		(5 * 60)
 #define IBMVFC_DEFAULT_LOG_LEVEL	2
 #define IBMVFC_MAX_CDB_LEN		16
+#define IBMVFC_CLS3_ERROR		0
 
 /*
  * Ensure we have resources for ERP and initialization:
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
index b29fef9d0f27..642b739ad0da 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
@@ -1606,8 +1606,6 @@ static void ibmvscsis_send_messages(struct scsi_info *vscsi)
 
 	if (!(vscsi->flags & RESPONSE_Q_DOWN)) {
 		list_for_each_entry_safe(cmd, nxt, &vscsi->waiting_rsp, list) {
-			pr_debug("send_messages cmd %p\n", cmd);
-
 			iue = cmd->iue;
 
 			crq->valid = VALID_CMD_RESP_EL;
@@ -1934,6 +1932,8 @@ static int ibmvscsis_drop_nexus(struct ibmvscsis_tport *tport)
 	/*
 	 * Release the SCSI I_T Nexus to the emulated ibmvscsis Target Port
 	 */
+	target_wait_for_sess_cmds(se_sess);
+	transport_deregister_session_configfs(se_sess);
 	transport_deregister_session(se_sess);
 	tport->ibmv_nexus = NULL;
 	kfree(nexus);
@@ -1978,7 +1978,7 @@ static long ibmvscsis_srp_login(struct scsi_info *vscsi,
 		reason = SRP_LOGIN_REJ_MULTI_CHANNEL_UNSUPPORTED;
 	else if (fmt->buffers & (~SUPPORTED_FORMATS))
 		reason = SRP_LOGIN_REJ_UNSUPPORTED_DESCRIPTOR_FMT;
-	else if ((fmt->buffers | SUPPORTED_FORMATS) == 0)
+	else if ((fmt->buffers & SUPPORTED_FORMATS) == 0)
 		reason = SRP_LOGIN_REJ_UNSUPPORTED_DESCRIPTOR_FMT;
 
 	if (vscsi->state == SRP_PROCESSING)
@@ -2554,10 +2554,6 @@ static void ibmvscsis_parse_cmd(struct scsi_info *vscsi,
 
 	srp->lun.scsi_lun[0] &= 0x3f;
 
-	pr_debug("calling submit_cmd, se_cmd %p, lun 0x%llx, cdb 0x%x, attr:%d\n",
-		 &cmd->se_cmd, scsilun_to_int(&srp->lun), (int)srp->cdb[0],
-		 attr);
-
 	rc = target_submit_cmd(&cmd->se_cmd, nexus->se_sess, srp->cdb,
 			       cmd->sense_buf, scsilun_to_int(&srp->lun),
 			       data_len, attr, dir, 0);
@@ -3142,8 +3138,6 @@ static int ibmvscsis_rdma(struct ibmvscsis_cmd *cmd, struct scatterlist *sg,
 	long tx_len;
 	long rc = 0;
 
-	pr_debug("rdma: dir %d, bytes 0x%x\n", dir, bytes);
-
 	if (bytes == 0)
 		return 0;
 
@@ -3192,12 +3186,6 @@ static int ibmvscsis_rdma(struct ibmvscsis_cmd *cmd, struct scatterlist *sg,
 					 vscsi->dds.window[LOCAL].liobn,
 					 server_ioba);
 		} else {
-			/* write to client */
-			struct srp_cmd *srp = (struct srp_cmd *)iue->sbuf->buf;
-
-			if (!READ_CMD(srp->cdb))
-				print_hex_dump_bytes(" data:", DUMP_PREFIX_NONE,
-						     sg_virt(sgp), buf_len);
 			/* The h_copy_rdma will cause phyp, running in another
 			 * partition, to read memory, so we need to make sure
 			 * the data has been written out, hence these syncs.
@@ -3322,12 +3310,9 @@ cmd_work:
 				rc = ibmvscsis_trans_event(vscsi, crq);
 			} else if (vscsi->flags & TRANS_EVENT) {
 				/*
-				 * if a tranport event has occurred leave
+				 * if a transport event has occurred leave
 				 * everything but transport events on the queue
-				 */
-				pr_debug("handle_crq, ignoring\n");
-
-				/*
+				 *
 				 * need to decrement the queue index so we can
 				 * look at the elment again
 				 */
@@ -3461,6 +3446,7 @@ static int ibmvscsis_probe(struct vio_dev *vdev,
 	vscsi->map_ioba = dma_map_single(&vdev->dev, vscsi->map_buf, PAGE_SIZE,
 					 DMA_BIDIRECTIONAL);
 	if (dma_mapping_error(&vdev->dev, vscsi->map_ioba)) {
+		rc = -ENOMEM;
 		dev_err(&vscsi->dev, "probe: error mapping command buffer\n");
 		goto free_buf;
 	}
@@ -3693,12 +3679,9 @@ static void ibmvscsis_release_cmd(struct se_cmd *se_cmd)
 						 se_cmd);
 	struct scsi_info *vscsi = cmd->adapter;
 
-	pr_debug("release_cmd %p, flags %d\n", se_cmd, cmd->flags);
-
 	spin_lock_bh(&vscsi->intr_lock);
 	/* Remove from active_q */
-	list_del(&cmd->list);
-	list_add_tail(&cmd->list, &vscsi->waiting_rsp);
+	list_move_tail(&cmd->list, &vscsi->waiting_rsp);
 	ibmvscsis_send_messages(vscsi);
 	spin_unlock_bh(&vscsi->intr_lock);
 }
@@ -3715,9 +3698,6 @@ static int ibmvscsis_write_pending(struct se_cmd *se_cmd)
 	struct iu_entry *iue = cmd->iue;
 	int rc;
 
-	pr_debug("write_pending, se_cmd %p, length 0x%x\n",
-		 se_cmd, se_cmd->data_length);
-
 	rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma,
 			       1, 1);
 	if (rc) {
@@ -3756,9 +3736,6 @@ static int ibmvscsis_queue_data_in(struct se_cmd *se_cmd)
 	uint len = 0;
 	int rc;
 
-	pr_debug("queue_data_in, se_cmd %p, length 0x%x\n",
-		 se_cmd, se_cmd->data_length);
-
 	rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma, 1,
 			       1);
 	if (rc) {
diff --git a/drivers/scsi/in2000.c b/drivers/scsi/in2000.c
deleted file mode 100644
index 3882d9f519c8..000000000000
--- a/drivers/scsi/in2000.c
+++ /dev/null
@@ -1,2302 +0,0 @@
-/*
- *    in2000.c -  Linux device driver for the
- *                Always IN2000 ISA SCSI card.
- *
- * Copyright (c) 1996 John Shifflett, GeoLog Consulting
- *    john@geolog.com
- *    jshiffle@netcom.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * For the avoidance of doubt the "preferred form" of this code is one which
- * is in an open non patent encumbered format. Where cryptographic key signing
- * forms part of the process of creating an executable the information
- * including keys needed to generate an equivalently functional executable
- * are deemed to be part of the source code.
- *
- * Drew Eckhardt's excellent 'Generic NCR5380' sources provided
- * much of the inspiration and some of the code for this driver.
- * The Linux IN2000 driver distributed in the Linux kernels through
- * version 1.2.13 was an extremely valuable reference on the arcane
- * (and still mysterious) workings of the IN2000's fifo. It also
- * is where I lifted in2000_biosparam(), the gist of the card
- * detection scheme, and other bits of code. Many thanks to the
- * talented and courageous people who wrote, contributed to, and
- * maintained that driver (including Brad McLean, Shaun Savage,
- * Bill Earnest, Larry Doolittle, Roger Sunshine, John Luckey,
- * Matt Postiff, Peter Lu, zerucha@shell.portal.com, and Eric
- * Youngdale). I should also mention the driver written by
- * Hamish Macdonald for the (GASP!) Amiga A2091 card, included
- * in the Linux-m68k distribution; it gave me a good initial
- * understanding of the proper way to run a WD33c93 chip, and I
- * ended up stealing lots of code from it.
- *
- * _This_ driver is (I feel) an improvement over the old one in
- * several respects:
- *    -  All problems relating to the data size of a SCSI request are
- *          gone (as far as I know). The old driver couldn't handle
- *          swapping to partitions because that involved 4k blocks, nor
- *          could it deal with the st.c tape driver unmodified, because
- *          that usually involved 4k - 32k blocks. The old driver never
- *          quite got away from a morbid dependence on 2k block sizes -
- *          which of course is the size of the card's fifo.
- *
- *    -  Target Disconnection/Reconnection is now supported. Any
- *          system with more than one device active on the SCSI bus
- *          will benefit from this. The driver defaults to what I'm
- *          calling 'adaptive disconnect' - meaning that each command
- *          is evaluated individually as to whether or not it should
- *          be run with the option to disconnect/reselect (if the
- *          device chooses), or as a "SCSI-bus-hog".
- *
- *    -  Synchronous data transfers are now supported. Because there
- *          are a few devices (and many improperly terminated systems)
- *          that choke when doing sync, the default is sync DISABLED
- *          for all devices. This faster protocol can (and should!)
- *          be enabled on selected devices via the command-line.
- *
- *    -  Runtime operating parameters can now be specified through
- *       either the LILO or the 'insmod' command line. For LILO do:
- *          "in2000=blah,blah,blah"
- *       and with insmod go like:
- *          "insmod /usr/src/linux/modules/in2000.o setup_strings=blah,blah"
- *       The defaults should be good for most people. See the comment
- *       for 'setup_strings' below for more details.
- *
- *    -  The old driver relied exclusively on what the Western Digital
- *          docs call "Combination Level 2 Commands", which are a great
- *          idea in that the CPU is relieved of a lot of interrupt
- *          overhead. However, by accepting a certain (user-settable)
- *          amount of additional interrupts, this driver achieves
- *          better control over the SCSI bus, and data transfers are
- *          almost as fast while being much easier to define, track,
- *          and debug.
- *
- *    -  You can force detection of a card whose BIOS has been disabled.
- *
- *    -  Multiple IN2000 cards might almost be supported. I've tried to
- *       keep it in mind, but have no way to test...
- *
- *
- * TODO:
- *       tagged queuing. multiple cards.
- *
- *
- * NOTE:
- *       When using this or any other SCSI driver as a module, you'll
- *       find that with the stock kernel, at most _two_ SCSI hard
- *       drives will be linked into the device list (ie, usable).
- *       If your IN2000 card has more than 2 disks on its bus, you
- *       might want to change the define of 'SD_EXTRA_DEVS' in the
- *       'hosts.h' file from 2 to whatever is appropriate. It took
- *       me a while to track down this surprisingly obscure and
- *       undocumented little "feature".
- *
- *
- * People with bug reports, wish-lists, complaints, comments,
- * or improvements are asked to pah-leeez email me (John Shifflett)
- * at john@geolog.com or jshiffle@netcom.com! I'm anxious to get
- * this thing into as good a shape as possible, and I'm positive
- * there are lots of lurking bugs and "Stupid Places".
- *
- * Updated for Linux 2.5 by Alan Cox <alan@lxorguk.ukuu.org.uk>
- *	- Using new_eh handler
- *	- Hopefully got all the locking right again
- *	See "FIXME" notes for items that could do with more work
- */
-
-#include <linux/module.h>
-#include <linux/blkdev.h>
-#include <linux/interrupt.h>
-#include <linux/string.h>
-#include <linux/delay.h>
-#include <linux/proc_fs.h>
-#include <linux/ioport.h>
-#include <linux/stat.h>
-
-#include <asm/io.h>
-
-#include "scsi.h"
-#include <scsi/scsi_host.h>
-
-#define IN2000_VERSION    "1.33-2.5"
-#define IN2000_DATE       "2002/11/03"
-
-#include "in2000.h"
-
-
-/*
- * 'setup_strings' is a single string used to pass operating parameters and
- * settings from the kernel/module command-line to the driver. 'setup_args[]'
- * is an array of strings that define the compile-time default values for
- * these settings. If Linux boots with a LILO or insmod command-line, those
- * settings are combined with 'setup_args[]'. Note that LILO command-lines
- * are prefixed with "in2000=" while insmod uses a "setup_strings=" prefix.
- * The driver recognizes the following keywords (lower case required) and
- * arguments:
- *
- * -  ioport:addr    -Where addr is IO address of a (usually ROM-less) card.
- * -  noreset        -No optional args. Prevents SCSI bus reset at boot time.
- * -  nosync:x       -x is a bitmask where the 1st 7 bits correspond with
- *                    the 7 possible SCSI devices (bit 0 for device #0, etc).
- *                    Set a bit to PREVENT sync negotiation on that device.
- *                    The driver default is sync DISABLED on all devices.
- * -  period:ns      -ns is the minimum # of nanoseconds in a SCSI data transfer
- *                    period. Default is 500; acceptable values are 250 - 1000.
- * -  disconnect:x   -x = 0 to never allow disconnects, 2 to always allow them.
- *                    x = 1 does 'adaptive' disconnects, which is the default
- *                    and generally the best choice.
- * -  debug:x        -If 'DEBUGGING_ON' is defined, x is a bitmask that causes
- *                    various types of debug output to printed - see the DB_xxx
- *                    defines in in2000.h
- * -  proc:x         -If 'PROC_INTERFACE' is defined, x is a bitmask that
- *                    determines how the /proc interface works and what it
- *                    does - see the PR_xxx defines in in2000.h
- *
- * Syntax Notes:
- * -  Numeric arguments can be decimal or the '0x' form of hex notation. There
- *    _must_ be a colon between a keyword and its numeric argument, with no
- *    spaces.
- * -  Keywords are separated by commas, no spaces, in the standard kernel
- *    command-line manner.
- * -  A keyword in the 'nth' comma-separated command-line member will overwrite
- *    the 'nth' element of setup_args[]. A blank command-line member (in
- *    other words, a comma with no preceding keyword) will _not_ overwrite
- *    the corresponding setup_args[] element.
- *
- * A few LILO examples (for insmod, use 'setup_strings' instead of 'in2000'):
- * -  in2000=ioport:0x220,noreset
- * -  in2000=period:250,disconnect:2,nosync:0x03
- * -  in2000=debug:0x1e
- * -  in2000=proc:3
- */
-
-/* Normally, no defaults are specified... */
-static char *setup_args[] = { "", "", "", "", "", "", "", "", "" };
-
-/* filled in by 'insmod' */
-static char *setup_strings;
-
-module_param(setup_strings, charp, 0);
-
-static inline uchar read_3393(struct IN2000_hostdata *hostdata, uchar reg_num)
-{
-	write1_io(reg_num, IO_WD_ADDR);
-	return read1_io(IO_WD_DATA);
-}
-
-
-#define READ_AUX_STAT() read1_io(IO_WD_ASR)
-
-
-static inline void write_3393(struct IN2000_hostdata *hostdata, uchar reg_num, uchar value)
-{
-	write1_io(reg_num, IO_WD_ADDR);
-	write1_io(value, IO_WD_DATA);
-}
-
-
-static inline void write_3393_cmd(struct IN2000_hostdata *hostdata, uchar cmd)
-{
-/*   while (READ_AUX_STAT() & ASR_CIP)
-      printk("|");*/
-	write1_io(WD_COMMAND, IO_WD_ADDR);
-	write1_io(cmd, IO_WD_DATA);
-}
-
-
-static uchar read_1_byte(struct IN2000_hostdata *hostdata)
-{
-	uchar asr, x = 0;
-
-	write_3393(hostdata, WD_CONTROL, CTRL_IDI | CTRL_EDI | CTRL_POLLED);
-	write_3393_cmd(hostdata, WD_CMD_TRANS_INFO | 0x80);
-	do {
-		asr = READ_AUX_STAT();
-		if (asr & ASR_DBR)
-			x = read_3393(hostdata, WD_DATA);
-	} while (!(asr & ASR_INT));
-	return x;
-}
-
-
-static void write_3393_count(struct IN2000_hostdata *hostdata, unsigned long value)
-{
-	write1_io(WD_TRANSFER_COUNT_MSB, IO_WD_ADDR);
-	write1_io((value >> 16), IO_WD_DATA);
-	write1_io((value >> 8), IO_WD_DATA);
-	write1_io(value, IO_WD_DATA);
-}
-
-
-static unsigned long read_3393_count(struct IN2000_hostdata *hostdata)
-{
-	unsigned long value;
-
-	write1_io(WD_TRANSFER_COUNT_MSB, IO_WD_ADDR);
-	value = read1_io(IO_WD_DATA) << 16;
-	value |= read1_io(IO_WD_DATA) << 8;
-	value |= read1_io(IO_WD_DATA);
-	return value;
-}
-
-
-/* The 33c93 needs to be told which direction a command transfers its
- * data; we use this function to figure it out. Returns true if there
- * will be a DATA_OUT phase with this command, false otherwise.
- * (Thanks to Joerg Dorchain for the research and suggestion.)
- */
-static int is_dir_out(Scsi_Cmnd * cmd)
-{
-	switch (cmd->cmnd[0]) {
-	case WRITE_6:
-	case WRITE_10:
-	case WRITE_12:
-	case WRITE_LONG:
-	case WRITE_SAME:
-	case WRITE_BUFFER:
-	case WRITE_VERIFY:
-	case WRITE_VERIFY_12:
-	case COMPARE:
-	case COPY:
-	case COPY_VERIFY:
-	case SEARCH_EQUAL:
-	case SEARCH_HIGH:
-	case SEARCH_LOW:
-	case SEARCH_EQUAL_12:
-	case SEARCH_HIGH_12:
-	case SEARCH_LOW_12:
-	case FORMAT_UNIT:
-	case REASSIGN_BLOCKS:
-	case RESERVE:
-	case MODE_SELECT:
-	case MODE_SELECT_10:
-	case LOG_SELECT:
-	case SEND_DIAGNOSTIC:
-	case CHANGE_DEFINITION:
-	case UPDATE_BLOCK:
-	case SET_WINDOW:
-	case MEDIUM_SCAN:
-	case SEND_VOLUME_TAG:
-	case 0xea:
-		return 1;
-	default:
-		return 0;
-	}
-}
-
-
-
-static struct sx_period sx_table[] = {
-	{1, 0x20},
-	{252, 0x20},
-	{376, 0x30},
-	{500, 0x40},
-	{624, 0x50},
-	{752, 0x60},
-	{876, 0x70},
-	{1000, 0x00},
-	{0, 0}
-};
-
-static int round_period(unsigned int period)
-{
-	int x;
-
-	for (x = 1; sx_table[x].period_ns; x++) {
-		if ((period <= sx_table[x - 0].period_ns) && (period > sx_table[x - 1].period_ns)) {
-			return x;
-		}
-	}
-	return 7;
-}
-
-static uchar calc_sync_xfer(unsigned int period, unsigned int offset)
-{
-	uchar result;
-
-	period *= 4;		/* convert SDTR code to ns */
-	result = sx_table[round_period(period)].reg_value;
-	result |= (offset < OPTIMUM_SX_OFF) ? offset : OPTIMUM_SX_OFF;
-	return result;
-}
-
-
-
-static void in2000_execute(struct Scsi_Host *instance);
-
-static int in2000_queuecommand_lck(Scsi_Cmnd * cmd, void (*done) (Scsi_Cmnd *))
-{
-	struct Scsi_Host *instance;
-	struct IN2000_hostdata *hostdata;
-	Scsi_Cmnd *tmp;
-
-	instance = cmd->device->host;
-	hostdata = (struct IN2000_hostdata *) instance->hostdata;
-
-	DB(DB_QUEUE_COMMAND, scmd_printk(KERN_DEBUG, cmd, "Q-%02x(", cmd->cmnd[0]))
-
-/* Set up a few fields in the Scsi_Cmnd structure for our own use:
- *  - host_scribble is the pointer to the next cmd in the input queue
- *  - scsi_done points to the routine we call when a cmd is finished
- *  - result is what you'd expect
- */
-	    cmd->host_scribble = NULL;
-	cmd->scsi_done = done;
-	cmd->result = 0;
-
-/* We use the Scsi_Pointer structure that's included with each command
- * as a scratchpad (as it's intended to be used!). The handy thing about
- * the SCp.xxx fields is that they're always associated with a given
- * cmd, and are preserved across disconnect-reselect. This means we
- * can pretty much ignore SAVE_POINTERS and RESTORE_POINTERS messages
- * if we keep all the critical pointers and counters in SCp:
- *  - SCp.ptr is the pointer into the RAM buffer
- *  - SCp.this_residual is the size of that buffer
- *  - SCp.buffer points to the current scatter-gather buffer
- *  - SCp.buffers_residual tells us how many S.G. buffers there are
- *  - SCp.have_data_in helps keep track of >2048 byte transfers
- *  - SCp.sent_command is not used
- *  - SCp.phase records this command's SRCID_ER bit setting
- */
-
-	if (scsi_bufflen(cmd)) {
-		cmd->SCp.buffer = scsi_sglist(cmd);
-		cmd->SCp.buffers_residual = scsi_sg_count(cmd) - 1;
-		cmd->SCp.ptr = sg_virt(cmd->SCp.buffer);
-		cmd->SCp.this_residual = cmd->SCp.buffer->length;
-	} else {
-		cmd->SCp.buffer = NULL;
-		cmd->SCp.buffers_residual = 0;
-		cmd->SCp.ptr = NULL;
-		cmd->SCp.this_residual = 0;
-	}
-	cmd->SCp.have_data_in = 0;
-
-/* We don't set SCp.phase here - that's done in in2000_execute() */
-
-/* WD docs state that at the conclusion of a "LEVEL2" command, the
- * status byte can be retrieved from the LUN register. Apparently,
- * this is the case only for *uninterrupted* LEVEL2 commands! If
- * there are any unexpected phases entered, even if they are 100%
- * legal (different devices may choose to do things differently),
- * the LEVEL2 command sequence is exited. This often occurs prior
- * to receiving the status byte, in which case the driver does a
- * status phase interrupt and gets the status byte on its own.
- * While such a command can then be "resumed" (ie restarted to
- * finish up as a LEVEL2 command), the LUN register will NOT be
- * a valid status byte at the command's conclusion, and we must
- * use the byte obtained during the earlier interrupt. Here, we
- * preset SCp.Status to an illegal value (0xff) so that when
- * this command finally completes, we can tell where the actual
- * status byte is stored.
- */
-
-	cmd->SCp.Status = ILLEGAL_STATUS_BYTE;
-
-/* We need to disable interrupts before messing with the input
- * queue and calling in2000_execute().
- */
-
-	/*
-	 * Add the cmd to the end of 'input_Q'. Note that REQUEST_SENSE
-	 * commands are added to the head of the queue so that the desired
-	 * sense data is not lost before REQUEST_SENSE executes.
-	 */
-
-	if (!(hostdata->input_Q) || (cmd->cmnd[0] == REQUEST_SENSE)) {
-		cmd->host_scribble = (uchar *) hostdata->input_Q;
-		hostdata->input_Q = cmd;
-	} else {		/* find the end of the queue */
-		for (tmp = (Scsi_Cmnd *) hostdata->input_Q; tmp->host_scribble; tmp = (Scsi_Cmnd *) tmp->host_scribble);
-		tmp->host_scribble = (uchar *) cmd;
-	}
-
-/* We know that there's at least one command in 'input_Q' now.
- * Go see if any of them are runnable!
- */
-
-	in2000_execute(cmd->device->host);
-
-	DB(DB_QUEUE_COMMAND, printk(")Q "))
-	    return 0;
-}
-
-static DEF_SCSI_QCMD(in2000_queuecommand)
-
-
-
-/*
- * This routine attempts to start a scsi command. If the host_card is
- * already connected, we give up immediately. Otherwise, look through
- * the input_Q, using the first command we find that's intended
- * for a currently non-busy target/lun.
- * Note that this function is always called with interrupts already
- * disabled (either from in2000_queuecommand() or in2000_intr()).
- */
-static void in2000_execute(struct Scsi_Host *instance)
-{
-	struct IN2000_hostdata *hostdata;
-	Scsi_Cmnd *cmd, *prev;
-	int i;
-	unsigned short *sp;
-	unsigned short f;
-	unsigned short flushbuf[16];
-
-
-	hostdata = (struct IN2000_hostdata *) instance->hostdata;
-
-	DB(DB_EXECUTE, printk("EX("))
-
-	    if (hostdata->selecting || hostdata->connected) {
-
-		DB(DB_EXECUTE, printk(")EX-0 "))
-
-		    return;
-	}
-
-	/*
-	 * Search through the input_Q for a command destined
-	 * for an idle target/lun.
-	 */
-
-	cmd = (Scsi_Cmnd *) hostdata->input_Q;
-	prev = NULL;
-	while (cmd) {
-		if (!(hostdata->busy[cmd->device->id] & (1 << cmd->device->lun)))
-			break;
-		prev = cmd;
-		cmd = (Scsi_Cmnd *) cmd->host_scribble;
-	}
-
-	/* quit if queue empty or all possible targets are busy */
-
-	if (!cmd) {
-
-		DB(DB_EXECUTE, printk(")EX-1 "))
-
-		    return;
-	}
-
-	/*  remove command from queue */
-
-	if (prev)
-		prev->host_scribble = cmd->host_scribble;
-	else
-		hostdata->input_Q = (Scsi_Cmnd *) cmd->host_scribble;
-
-#ifdef PROC_STATISTICS
-	hostdata->cmd_cnt[cmd->device->id]++;
-#endif
-
-/*
- * Start the selection process
- */
-
-	if (is_dir_out(cmd))
-		write_3393(hostdata, WD_DESTINATION_ID, cmd->device->id);
-	else
-		write_3393(hostdata, WD_DESTINATION_ID, cmd->device->id | DSTID_DPD);
-
-/* Now we need to figure out whether or not this command is a good
- * candidate for disconnect/reselect. We guess to the best of our
- * ability, based on a set of hierarchical rules. When several
- * devices are operating simultaneously, disconnects are usually
- * an advantage. In a single device system, or if only 1 device
- * is being accessed, transfers usually go faster if disconnects
- * are not allowed:
- *
- * + Commands should NEVER disconnect if hostdata->disconnect =
- *   DIS_NEVER (this holds for tape drives also), and ALWAYS
- *   disconnect if hostdata->disconnect = DIS_ALWAYS.
- * + Tape drive commands should always be allowed to disconnect.
- * + Disconnect should be allowed if disconnected_Q isn't empty.
- * + Commands should NOT disconnect if input_Q is empty.
- * + Disconnect should be allowed if there are commands in input_Q
- *   for a different target/lun. In this case, the other commands
- *   should be made disconnect-able, if not already.
- *
- * I know, I know - this code would flunk me out of any
- * "C Programming 101" class ever offered. But it's easy
- * to change around and experiment with for now.
- */
-
-	cmd->SCp.phase = 0;	/* assume no disconnect */
-	if (hostdata->disconnect == DIS_NEVER)
-		goto no;
-	if (hostdata->disconnect == DIS_ALWAYS)
-		goto yes;
-	if (cmd->device->type == 1)	/* tape drive? */
-		goto yes;
-	if (hostdata->disconnected_Q)	/* other commands disconnected? */
-		goto yes;
-	if (!(hostdata->input_Q))	/* input_Q empty? */
-		goto no;
-	for (prev = (Scsi_Cmnd *) hostdata->input_Q; prev; prev = (Scsi_Cmnd *) prev->host_scribble) {
-		if ((prev->device->id != cmd->device->id) || (prev->device->lun != cmd->device->lun)) {
-			for (prev = (Scsi_Cmnd *) hostdata->input_Q; prev; prev = (Scsi_Cmnd *) prev->host_scribble)
-				prev->SCp.phase = 1;
-			goto yes;
-		}
-	}
-	goto no;
-
-      yes:
-	cmd->SCp.phase = 1;
-
-#ifdef PROC_STATISTICS
-	hostdata->disc_allowed_cnt[cmd->device->id]++;
-#endif
-
-      no:
-	write_3393(hostdata, WD_SOURCE_ID, ((cmd->SCp.phase) ? SRCID_ER : 0));
-
-	write_3393(hostdata, WD_TARGET_LUN, cmd->device->lun);
-	write_3393(hostdata, WD_SYNCHRONOUS_TRANSFER, hostdata->sync_xfer[cmd->device->id]);
-	hostdata->busy[cmd->device->id] |= (1 << cmd->device->lun);
-
-	if ((hostdata->level2 <= L2_NONE) || (hostdata->sync_stat[cmd->device->id] == SS_UNSET)) {
-
-		/*
-		 * Do a 'Select-With-ATN' command. This will end with
-		 * one of the following interrupts:
-		 *    CSR_RESEL_AM:  failure - can try again later.
-		 *    CSR_TIMEOUT:   failure - give up.
-		 *    CSR_SELECT:    success - proceed.
-		 */
-
-		hostdata->selecting = cmd;
-
-/* Every target has its own synchronous transfer setting, kept in
- * the sync_xfer array, and a corresponding status byte in sync_stat[].
- * Each target's sync_stat[] entry is initialized to SS_UNSET, and its
- * sync_xfer[] entry is initialized to the default/safe value. SS_UNSET
- * means that the parameters are undetermined as yet, and that we
- * need to send an SDTR message to this device after selection is
- * complete. We set SS_FIRST to tell the interrupt routine to do so,
- * unless we don't want to even _try_ synchronous transfers: In this
- * case we set SS_SET to make the defaults final.
- */
-		if (hostdata->sync_stat[cmd->device->id] == SS_UNSET) {
-			if (hostdata->sync_off & (1 << cmd->device->id))
-				hostdata->sync_stat[cmd->device->id] = SS_SET;
-			else
-				hostdata->sync_stat[cmd->device->id] = SS_FIRST;
-		}
-		hostdata->state = S_SELECTING;
-		write_3393_count(hostdata, 0);	/* this guarantees a DATA_PHASE interrupt */
-		write_3393_cmd(hostdata, WD_CMD_SEL_ATN);
-	}
-
-	else {
-
-		/*
-		 * Do a 'Select-With-ATN-Xfer' command. This will end with
-		 * one of the following interrupts:
-		 *    CSR_RESEL_AM:  failure - can try again later.
-		 *    CSR_TIMEOUT:   failure - give up.
-		 *    anything else: success - proceed.
-		 */
-
-		hostdata->connected = cmd;
-		write_3393(hostdata, WD_COMMAND_PHASE, 0);
-
-		/* copy command_descriptor_block into WD chip
-		 * (take advantage of auto-incrementing)
-		 */
-
-		write1_io(WD_CDB_1, IO_WD_ADDR);
-		for (i = 0; i < cmd->cmd_len; i++)
-			write1_io(cmd->cmnd[i], IO_WD_DATA);
-
-		/* The wd33c93 only knows about Group 0, 1, and 5 commands when
-		 * it's doing a 'select-and-transfer'. To be safe, we write the
-		 * size of the CDB into the OWN_ID register for every case. This
-		 * way there won't be problems with vendor-unique, audio, etc.
-		 */
-
-		write_3393(hostdata, WD_OWN_ID, cmd->cmd_len);
-
-		/* When doing a non-disconnect command, we can save ourselves a DATA
-		 * phase interrupt later by setting everything up now. With writes we
-		 * need to pre-fill the fifo; if there's room for the 32 flush bytes,
-		 * put them in there too - that'll avoid a fifo interrupt. Reads are
-		 * somewhat simpler.
-		 * KLUDGE NOTE: It seems that you can't completely fill the fifo here:
-		 * This results in the IO_FIFO_COUNT register rolling over to zero,
-		 * and apparently the gate array logic sees this as empty, not full,
-		 * so the 3393 chip is never signalled to start reading from the
-		 * fifo. Or maybe it's seen as a permanent fifo interrupt condition.
-		 * Regardless, we fix this by temporarily pretending that the fifo
-		 * is 16 bytes smaller. (I see now that the old driver has a comment
-		 * about "don't fill completely" in an analogous place - must be the
-		 * same deal.) This results in CDROM, swap partitions, and tape drives
-		 * needing an extra interrupt per write command - I think we can live
-		 * with that!
-		 */
-
-		if (!(cmd->SCp.phase)) {
-			write_3393_count(hostdata, cmd->SCp.this_residual);
-			write_3393(hostdata, WD_CONTROL, CTRL_IDI | CTRL_EDI | CTRL_BUS);
-			write1_io(0, IO_FIFO_WRITE);	/* clear fifo counter, write mode */
-
-			if (is_dir_out(cmd)) {
-				hostdata->fifo = FI_FIFO_WRITING;
-				if ((i = cmd->SCp.this_residual) > (IN2000_FIFO_SIZE - 16))
-					i = IN2000_FIFO_SIZE - 16;
-				cmd->SCp.have_data_in = i;	/* this much data in fifo */
-				i >>= 1;	/* Gulp. Assuming modulo 2. */
-				sp = (unsigned short *) cmd->SCp.ptr;
-				f = hostdata->io_base + IO_FIFO;
-
-#ifdef FAST_WRITE_IO
-
-				FAST_WRITE2_IO();
-#else
-				while (i--)
-					write2_io(*sp++, IO_FIFO);
-
-#endif
-
-				/* Is there room for the flush bytes? */
-
-				if (cmd->SCp.have_data_in <= ((IN2000_FIFO_SIZE - 16) - 32)) {
-					sp = flushbuf;
-					i = 16;
-
-#ifdef FAST_WRITE_IO
-
-					FAST_WRITE2_IO();
-#else
-					while (i--)
-						write2_io(0, IO_FIFO);
-
-#endif
-
-				}
-			}
-
-			else {
-				write1_io(0, IO_FIFO_READ);	/* put fifo in read mode */
-				hostdata->fifo = FI_FIFO_READING;
-				cmd->SCp.have_data_in = 0;	/* nothing transferred yet */
-			}
-
-		} else {
-			write_3393_count(hostdata, 0);	/* this guarantees a DATA_PHASE interrupt */
-		}
-		hostdata->state = S_RUNNING_LEVEL2;
-		write_3393_cmd(hostdata, WD_CMD_SEL_ATN_XFER);
-	}
-
-	/*
-	 * Since the SCSI bus can handle only 1 connection at a time,
-	 * we get out of here now. If the selection fails, or when
-	 * the command disconnects, we'll come back to this routine
-	 * to search the input_Q again...
-	 */
-
-	DB(DB_EXECUTE, printk("%s)EX-2 ", (cmd->SCp.phase) ? "d:" : ""))
-
-}
-
-
-
-static void transfer_pio(uchar * buf, int cnt, int data_in_dir, struct IN2000_hostdata *hostdata)
-{
-	uchar asr;
-
-	DB(DB_TRANSFER, printk("(%p,%d,%s)", buf, cnt, data_in_dir ? "in" : "out"))
-
-	    write_3393(hostdata, WD_CONTROL, CTRL_IDI | CTRL_EDI | CTRL_POLLED);
-	write_3393_count(hostdata, cnt);
-	write_3393_cmd(hostdata, WD_CMD_TRANS_INFO);
-	if (data_in_dir) {
-		do {
-			asr = READ_AUX_STAT();
-			if (asr & ASR_DBR)
-				*buf++ = read_3393(hostdata, WD_DATA);
-		} while (!(asr & ASR_INT));
-	} else {
-		do {
-			asr = READ_AUX_STAT();
-			if (asr & ASR_DBR)
-				write_3393(hostdata, WD_DATA, *buf++);
-		} while (!(asr & ASR_INT));
-	}
-
-	/* Note: we are returning with the interrupt UN-cleared.
-	 * Since (presumably) an entire I/O operation has
-	 * completed, the bus phase is probably different, and
-	 * the interrupt routine will discover this when it
-	 * responds to the uncleared int.
-	 */
-
-}
-
-
-
-static void transfer_bytes(Scsi_Cmnd * cmd, int data_in_dir)
-{
-	struct IN2000_hostdata *hostdata;
-	unsigned short *sp;
-	unsigned short f;
-	int i;
-
-	hostdata = (struct IN2000_hostdata *) cmd->device->host->hostdata;
-
-/* Normally, you'd expect 'this_residual' to be non-zero here.
- * In a series of scatter-gather transfers, however, this
- * routine will usually be called with 'this_residual' equal
- * to 0 and 'buffers_residual' non-zero. This means that a
- * previous transfer completed, clearing 'this_residual', and
- * now we need to setup the next scatter-gather buffer as the
- * source or destination for THIS transfer.
- */
-	if (!cmd->SCp.this_residual && cmd->SCp.buffers_residual) {
-		++cmd->SCp.buffer;
-		--cmd->SCp.buffers_residual;
-		cmd->SCp.this_residual = cmd->SCp.buffer->length;
-		cmd->SCp.ptr = sg_virt(cmd->SCp.buffer);
-	}
-
-/* Set up hardware registers */
-
-	write_3393(hostdata, WD_SYNCHRONOUS_TRANSFER, hostdata->sync_xfer[cmd->device->id]);
-	write_3393_count(hostdata, cmd->SCp.this_residual);
-	write_3393(hostdata, WD_CONTROL, CTRL_IDI | CTRL_EDI | CTRL_BUS);
-	write1_io(0, IO_FIFO_WRITE);	/* zero counter, assume write */
-
-/* Reading is easy. Just issue the command and return - we'll
- * get an interrupt later when we have actual data to worry about.
- */
-
-	if (data_in_dir) {
-		write1_io(0, IO_FIFO_READ);
-		if ((hostdata->level2 >= L2_DATA) || (hostdata->level2 == L2_BASIC && cmd->SCp.phase == 0)) {
-			write_3393(hostdata, WD_COMMAND_PHASE, 0x45);
-			write_3393_cmd(hostdata, WD_CMD_SEL_ATN_XFER);
-			hostdata->state = S_RUNNING_LEVEL2;
-		} else
-			write_3393_cmd(hostdata, WD_CMD_TRANS_INFO);
-		hostdata->fifo = FI_FIFO_READING;
-		cmd->SCp.have_data_in = 0;
-		return;
-	}
-
-/* Writing is more involved - we'll start the WD chip and write as
- * much data to the fifo as we can right now. Later interrupts will
- * write any bytes that don't make it at this stage.
- */
-
-	if ((hostdata->level2 >= L2_DATA) || (hostdata->level2 == L2_BASIC && cmd->SCp.phase == 0)) {
-		write_3393(hostdata, WD_COMMAND_PHASE, 0x45);
-		write_3393_cmd(hostdata, WD_CMD_SEL_ATN_XFER);
-		hostdata->state = S_RUNNING_LEVEL2;
-	} else
-		write_3393_cmd(hostdata, WD_CMD_TRANS_INFO);
-	hostdata->fifo = FI_FIFO_WRITING;
-	sp = (unsigned short *) cmd->SCp.ptr;
-
-	if ((i = cmd->SCp.this_residual) > IN2000_FIFO_SIZE)
-		i = IN2000_FIFO_SIZE;
-	cmd->SCp.have_data_in = i;
-	i >>= 1;		/* Gulp. We assume this_residual is modulo 2 */
-	f = hostdata->io_base + IO_FIFO;
-
-#ifdef FAST_WRITE_IO
-
-	FAST_WRITE2_IO();
-#else
-	while (i--)
-		write2_io(*sp++, IO_FIFO);
-
-#endif
-
-}
-
-
-/* We need to use spin_lock_irqsave() & spin_unlock_irqrestore() in this
- * function in order to work in an SMP environment. (I'd be surprised
- * if the driver is ever used by anyone on a real multi-CPU motherboard,
- * but it _does_ need to be able to compile and run in an SMP kernel.)
- */
-
-static irqreturn_t in2000_intr(int irqnum, void *dev_id)
-{
-	struct Scsi_Host *instance = dev_id;
-	struct IN2000_hostdata *hostdata;
-	Scsi_Cmnd *patch, *cmd;
-	uchar asr, sr, phs, id, lun, *ucp, msg;
-	int i, j;
-	unsigned long length;
-	unsigned short *sp;
-	unsigned short f;
-	unsigned long flags;
-
-	hostdata = (struct IN2000_hostdata *) instance->hostdata;
-
-/* Get the spin_lock and disable further ints, for SMP */
-
-	spin_lock_irqsave(instance->host_lock, flags);
-
-#ifdef PROC_STATISTICS
-	hostdata->int_cnt++;
-#endif
-
-/* The IN2000 card has 2 interrupt sources OR'ed onto its IRQ line - the
- * WD3393 chip and the 2k fifo (which is actually a dual-port RAM combined
- * with a big logic array, so it's a little different than what you might
- * expect). As far as I know, there's no reason that BOTH can't be active
- * at the same time, but there's a problem: while we can read the 3393
- * to tell if _it_ wants an interrupt, I don't know of a way to ask the
- * fifo the same question. The best we can do is check the 3393 and if
- * it _isn't_ the source of the interrupt, then we can be pretty sure
- * that the fifo is the culprit.
- *  UPDATE: I have it on good authority (Bill Earnest) that bit 0 of the
- *          IO_FIFO_COUNT register mirrors the fifo interrupt state. I
- *          assume that bit clear means interrupt active. As it turns
- *          out, the driver really doesn't need to check for this after
- *          all, so my remarks above about a 'problem' can safely be
- *          ignored. The way the logic is set up, there's no advantage
- *          (that I can see) to worrying about it.
- *
- * It seems that the fifo interrupt signal is negated when we extract
- * bytes during read or write bytes during write.
- *  - fifo will interrupt when data is moving from it to the 3393, and
- *    there are 31 (or less?) bytes left to go. This is sort of short-
- *    sighted: what if you don't WANT to do more? In any case, our
- *    response is to push more into the fifo - either actual data or
- *    dummy bytes if need be. Note that we apparently have to write at
- *    least 32 additional bytes to the fifo after an interrupt in order
- *    to get it to release the ones it was holding on to - writing fewer
- *    than 32 will result in another fifo int.
- *  UPDATE: Again, info from Bill Earnest makes this more understandable:
- *          32 bytes = two counts of the fifo counter register. He tells
- *          me that the fifo interrupt is a non-latching signal derived
- *          from a straightforward boolean interpretation of the 7
- *          highest bits of the fifo counter and the fifo-read/fifo-write
- *          state. Who'd a thought?
- */
-
-	write1_io(0, IO_LED_ON);
-	asr = READ_AUX_STAT();
-	if (!(asr & ASR_INT)) {	/* no WD33c93 interrupt? */
-
-/* Ok. This is definitely a FIFO-only interrupt.
- *
- * If FI_FIFO_READING is set, there are up to 2048 bytes waiting to be read,
- * maybe more to come from the SCSI bus. Read as many as we can out of the
- * fifo and into memory at the location of SCp.ptr[SCp.have_data_in], and
- * update have_data_in afterwards.
- *
- * If we have FI_FIFO_WRITING, the FIFO has almost run out of bytes to move
- * into the WD3393 chip (I think the interrupt happens when there are 31
- * bytes left, but it may be fewer...). The 3393 is still waiting, so we
- * shove some more into the fifo, which gets things moving again. If the
- * original SCSI command specified more than 2048 bytes, there may still
- * be some of that data left: fine - use it (from SCp.ptr[SCp.have_data_in]).
- * Don't forget to update have_data_in. If we've already written out the
- * entire buffer, feed 32 dummy bytes to the fifo - they're needed to
- * push out the remaining real data.
- *    (Big thanks to Bill Earnest for getting me out of the mud in here.)
- */
-
-		cmd = (Scsi_Cmnd *) hostdata->connected;	/* assume we're connected */
-		CHECK_NULL(cmd, "fifo_int")
-
-		    if (hostdata->fifo == FI_FIFO_READING) {
-
-			DB(DB_FIFO, printk("{R:%02x} ", read1_io(IO_FIFO_COUNT)))
-
-			    sp = (unsigned short *) (cmd->SCp.ptr + cmd->SCp.have_data_in);
-			i = read1_io(IO_FIFO_COUNT) & 0xfe;
-			i <<= 2;	/* # of words waiting in the fifo */
-			f = hostdata->io_base + IO_FIFO;
-
-#ifdef FAST_READ_IO
-
-			FAST_READ2_IO();
-#else
-			while (i--)
-				*sp++ = read2_io(IO_FIFO);
-
-#endif
-
-			i = sp - (unsigned short *) (cmd->SCp.ptr + cmd->SCp.have_data_in);
-			i <<= 1;
-			cmd->SCp.have_data_in += i;
-		}
-
-		else if (hostdata->fifo == FI_FIFO_WRITING) {
-
-			DB(DB_FIFO, printk("{W:%02x} ", read1_io(IO_FIFO_COUNT)))
-
-/* If all bytes have been written to the fifo, flush out the stragglers.
- * Note that while writing 16 dummy words seems arbitrary, we don't
- * have another choice that I can see. What we really want is to read
- * the 3393 transfer count register (that would tell us how many bytes
- * needed flushing), but the TRANSFER_INFO command hasn't completed
- * yet (not enough bytes!) and that register won't be accessible. So,
- * we use 16 words - a number obtained through trial and error.
- *  UPDATE: Bill says this is exactly what Always does, so there.
- *          More thanks due him for help in this section.
- */
-			    if (cmd->SCp.this_residual == cmd->SCp.have_data_in) {
-				i = 16;
-				while (i--)	/* write 32 dummy bytes */
-					write2_io(0, IO_FIFO);
-			}
-
-/* If there are still bytes left in the SCSI buffer, write as many as we
- * can out to the fifo.
- */
-
-			else {
-				sp = (unsigned short *) (cmd->SCp.ptr + cmd->SCp.have_data_in);
-				i = cmd->SCp.this_residual - cmd->SCp.have_data_in;	/* bytes yet to go */
-				j = read1_io(IO_FIFO_COUNT) & 0xfe;
-				j <<= 2;	/* how many words the fifo has room for */
-				if ((j << 1) > i)
-					j = (i >> 1);
-				while (j--)
-					write2_io(*sp++, IO_FIFO);
-
-				i = sp - (unsigned short *) (cmd->SCp.ptr + cmd->SCp.have_data_in);
-				i <<= 1;
-				cmd->SCp.have_data_in += i;
-			}
-		}
-
-		else {
-			printk("*** Spurious FIFO interrupt ***");
-		}
-
-		write1_io(0, IO_LED_OFF);
-
-/* release the SMP spin_lock and restore irq state */
-		spin_unlock_irqrestore(instance->host_lock, flags);
-		return IRQ_HANDLED;
-	}
-
-/* This interrupt was triggered by the WD33c93 chip. The fifo interrupt
- * may also be asserted, but we don't bother to check it: we get more
- * detailed info from FIFO_READING and FIFO_WRITING (see below).
- */
-
-	cmd = (Scsi_Cmnd *) hostdata->connected;	/* assume we're connected */
-	sr = read_3393(hostdata, WD_SCSI_STATUS);	/* clear the interrupt */
-	phs = read_3393(hostdata, WD_COMMAND_PHASE);
-
-	if (!cmd && (sr != CSR_RESEL_AM && sr != CSR_TIMEOUT && sr != CSR_SELECT)) {
-		printk("\nNR:wd-intr-1\n");
-		write1_io(0, IO_LED_OFF);
-
-/* release the SMP spin_lock and restore irq state */
-		spin_unlock_irqrestore(instance->host_lock, flags);
-		return IRQ_HANDLED;
-	}
-
-	DB(DB_INTR, printk("{%02x:%02x-", asr, sr))
-
-/* After starting a FIFO-based transfer, the next _WD3393_ interrupt is
- * guaranteed to be in response to the completion of the transfer.
- * If we were reading, there's probably data in the fifo that needs
- * to be copied into RAM - do that here. Also, we have to update
- * 'this_residual' and 'ptr' based on the contents of the
- * TRANSFER_COUNT register, in case the device decided to do an
- * intermediate disconnect (a device may do this if it has to
- * do a seek,  or just to be nice and let other devices have
- * some bus time during long transfers).
- * After doing whatever is necessary with the fifo, we go on and
- * service the WD3393 interrupt normally.
- */
-	    if (hostdata->fifo == FI_FIFO_READING) {
-
-/* buffer index = start-of-buffer + #-of-bytes-already-read */
-
-		sp = (unsigned short *) (cmd->SCp.ptr + cmd->SCp.have_data_in);
-
-/* bytes remaining in fifo = (total-wanted - #-not-got) - #-already-read */
-
-		i = (cmd->SCp.this_residual - read_3393_count(hostdata)) - cmd->SCp.have_data_in;
-		i >>= 1;	/* Gulp. We assume this will always be modulo 2 */
-		f = hostdata->io_base + IO_FIFO;
-
-#ifdef FAST_READ_IO
-
-		FAST_READ2_IO();
-#else
-		while (i--)
-			*sp++ = read2_io(IO_FIFO);
-
-#endif
-
-		hostdata->fifo = FI_FIFO_UNUSED;
-		length = cmd->SCp.this_residual;
-		cmd->SCp.this_residual = read_3393_count(hostdata);
-		cmd->SCp.ptr += (length - cmd->SCp.this_residual);
-
-		DB(DB_TRANSFER, printk("(%p,%d)", cmd->SCp.ptr, cmd->SCp.this_residual))
-
-	}
-
-	else if (hostdata->fifo == FI_FIFO_WRITING) {
-		hostdata->fifo = FI_FIFO_UNUSED;
-		length = cmd->SCp.this_residual;
-		cmd->SCp.this_residual = read_3393_count(hostdata);
-		cmd->SCp.ptr += (length - cmd->SCp.this_residual);
-
-		DB(DB_TRANSFER, printk("(%p,%d)", cmd->SCp.ptr, cmd->SCp.this_residual))
-
-	}
-
-/* Respond to the specific WD3393 interrupt - there are quite a few! */
-
-	switch (sr) {
-
-	case CSR_TIMEOUT:
-		DB(DB_INTR, printk("TIMEOUT"))
-
-		    if (hostdata->state == S_RUNNING_LEVEL2)
-			hostdata->connected = NULL;
-		else {
-			cmd = (Scsi_Cmnd *) hostdata->selecting;	/* get a valid cmd */
-			CHECK_NULL(cmd, "csr_timeout")
-			    hostdata->selecting = NULL;
-		}
-
-		cmd->result = DID_NO_CONNECT << 16;
-		hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-		hostdata->state = S_UNCONNECTED;
-		cmd->scsi_done(cmd);
-
-/* We are not connected to a target - check to see if there
- * are commands waiting to be executed.
- */
-
-		in2000_execute(instance);
-		break;
-
-
-/* Note: this interrupt should not occur in a LEVEL2 command */
-
-	case CSR_SELECT:
-		DB(DB_INTR, printk("SELECT"))
-		    hostdata->connected = cmd = (Scsi_Cmnd *) hostdata->selecting;
-		CHECK_NULL(cmd, "csr_select")
-		    hostdata->selecting = NULL;
-
-		/* construct an IDENTIFY message with correct disconnect bit */
-
-		hostdata->outgoing_msg[0] = (0x80 | 0x00 | cmd->device->lun);
-		if (cmd->SCp.phase)
-			hostdata->outgoing_msg[0] |= 0x40;
-
-		if (hostdata->sync_stat[cmd->device->id] == SS_FIRST) {
-#ifdef SYNC_DEBUG
-			printk(" sending SDTR ");
-#endif
-
-			hostdata->sync_stat[cmd->device->id] = SS_WAITING;
-
-			/* tack on a 2nd message to ask about synchronous transfers */
-
-			hostdata->outgoing_msg[1] = EXTENDED_MESSAGE;
-			hostdata->outgoing_msg[2] = 3;
-			hostdata->outgoing_msg[3] = EXTENDED_SDTR;
-			hostdata->outgoing_msg[4] = OPTIMUM_SX_PER / 4;
-			hostdata->outgoing_msg[5] = OPTIMUM_SX_OFF;
-			hostdata->outgoing_len = 6;
-		} else
-			hostdata->outgoing_len = 1;
-
-		hostdata->state = S_CONNECTED;
-		break;
-
-
-	case CSR_XFER_DONE | PHS_DATA_IN:
-	case CSR_UNEXP | PHS_DATA_IN:
-	case CSR_SRV_REQ | PHS_DATA_IN:
-		DB(DB_INTR, printk("IN-%d.%d", cmd->SCp.this_residual, cmd->SCp.buffers_residual))
-		    transfer_bytes(cmd, DATA_IN_DIR);
-		if (hostdata->state != S_RUNNING_LEVEL2)
-			hostdata->state = S_CONNECTED;
-		break;
-
-
-	case CSR_XFER_DONE | PHS_DATA_OUT:
-	case CSR_UNEXP | PHS_DATA_OUT:
-	case CSR_SRV_REQ | PHS_DATA_OUT:
-		DB(DB_INTR, printk("OUT-%d.%d", cmd->SCp.this_residual, cmd->SCp.buffers_residual))
-		    transfer_bytes(cmd, DATA_OUT_DIR);
-		if (hostdata->state != S_RUNNING_LEVEL2)
-			hostdata->state = S_CONNECTED;
-		break;
-
-
-/* Note: this interrupt should not occur in a LEVEL2 command */
-
-	case CSR_XFER_DONE | PHS_COMMAND:
-	case CSR_UNEXP | PHS_COMMAND:
-	case CSR_SRV_REQ | PHS_COMMAND:
-		DB(DB_INTR, printk("CMND-%02x", cmd->cmnd[0]))
-		    transfer_pio(cmd->cmnd, cmd->cmd_len, DATA_OUT_DIR, hostdata);
-		hostdata->state = S_CONNECTED;
-		break;
-
-
-	case CSR_XFER_DONE | PHS_STATUS:
-	case CSR_UNEXP | PHS_STATUS:
-	case CSR_SRV_REQ | PHS_STATUS:
-		DB(DB_INTR, printk("STATUS="))
-
-		    cmd->SCp.Status = read_1_byte(hostdata);
-		DB(DB_INTR, printk("%02x", cmd->SCp.Status))
-		    if (hostdata->level2 >= L2_BASIC) {
-			sr = read_3393(hostdata, WD_SCSI_STATUS);	/* clear interrupt */
-			hostdata->state = S_RUNNING_LEVEL2;
-			write_3393(hostdata, WD_COMMAND_PHASE, 0x50);
-			write_3393_cmd(hostdata, WD_CMD_SEL_ATN_XFER);
-		} else {
-			hostdata->state = S_CONNECTED;
-		}
-		break;
-
-
-	case CSR_XFER_DONE | PHS_MESS_IN:
-	case CSR_UNEXP | PHS_MESS_IN:
-	case CSR_SRV_REQ | PHS_MESS_IN:
-		DB(DB_INTR, printk("MSG_IN="))
-
-		    msg = read_1_byte(hostdata);
-		sr = read_3393(hostdata, WD_SCSI_STATUS);	/* clear interrupt */
-
-		hostdata->incoming_msg[hostdata->incoming_ptr] = msg;
-		if (hostdata->incoming_msg[0] == EXTENDED_MESSAGE)
-			msg = EXTENDED_MESSAGE;
-		else
-			hostdata->incoming_ptr = 0;
-
-		cmd->SCp.Message = msg;
-		switch (msg) {
-
-		case COMMAND_COMPLETE:
-			DB(DB_INTR, printk("CCMP"))
-			    write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-			hostdata->state = S_PRE_CMP_DISC;
-			break;
-
-		case SAVE_POINTERS:
-			DB(DB_INTR, printk("SDP"))
-			    write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-			hostdata->state = S_CONNECTED;
-			break;
-
-		case RESTORE_POINTERS:
-			DB(DB_INTR, printk("RDP"))
-			    if (hostdata->level2 >= L2_BASIC) {
-				write_3393(hostdata, WD_COMMAND_PHASE, 0x45);
-				write_3393_cmd(hostdata, WD_CMD_SEL_ATN_XFER);
-				hostdata->state = S_RUNNING_LEVEL2;
-			} else {
-				write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-				hostdata->state = S_CONNECTED;
-			}
-			break;
-
-		case DISCONNECT:
-			DB(DB_INTR, printk("DIS"))
-			    cmd->device->disconnect = 1;
-			write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-			hostdata->state = S_PRE_TMP_DISC;
-			break;
-
-		case MESSAGE_REJECT:
-			DB(DB_INTR, printk("REJ"))
-#ifdef SYNC_DEBUG
-			    printk("-REJ-");
-#endif
-			if (hostdata->sync_stat[cmd->device->id] == SS_WAITING)
-				hostdata->sync_stat[cmd->device->id] = SS_SET;
-			write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-			hostdata->state = S_CONNECTED;
-			break;
-
-		case EXTENDED_MESSAGE:
-			DB(DB_INTR, printk("EXT"))
-
-			    ucp = hostdata->incoming_msg;
-
-#ifdef SYNC_DEBUG
-			printk("%02x", ucp[hostdata->incoming_ptr]);
-#endif
-			/* Is this the last byte of the extended message? */
-
-			if ((hostdata->incoming_ptr >= 2) && (hostdata->incoming_ptr == (ucp[1] + 1))) {
-
-				switch (ucp[2]) {	/* what's the EXTENDED code? */
-				case EXTENDED_SDTR:
-					id = calc_sync_xfer(ucp[3], ucp[4]);
-					if (hostdata->sync_stat[cmd->device->id] != SS_WAITING) {
-
-/* A device has sent an unsolicited SDTR message; rather than go
- * through the effort of decoding it and then figuring out what
- * our reply should be, we're just gonna say that we have a
- * synchronous fifo depth of 0. This will result in asynchronous
- * transfers - not ideal but so much easier.
- * Actually, this is OK because it assures us that if we don't
- * specifically ask for sync transfers, we won't do any.
- */
-
-						write_3393_cmd(hostdata, WD_CMD_ASSERT_ATN);	/* want MESS_OUT */
-						hostdata->outgoing_msg[0] = EXTENDED_MESSAGE;
-						hostdata->outgoing_msg[1] = 3;
-						hostdata->outgoing_msg[2] = EXTENDED_SDTR;
-						hostdata->outgoing_msg[3] = hostdata->default_sx_per / 4;
-						hostdata->outgoing_msg[4] = 0;
-						hostdata->outgoing_len = 5;
-						hostdata->sync_xfer[cmd->device->id] = calc_sync_xfer(hostdata->default_sx_per / 4, 0);
-					} else {
-						hostdata->sync_xfer[cmd->device->id] = id;
-					}
-#ifdef SYNC_DEBUG
-					printk("sync_xfer=%02x", hostdata->sync_xfer[cmd->device->id]);
-#endif
-					hostdata->sync_stat[cmd->device->id] = SS_SET;
-					write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-					hostdata->state = S_CONNECTED;
-					break;
-				case EXTENDED_WDTR:
-					write_3393_cmd(hostdata, WD_CMD_ASSERT_ATN);	/* want MESS_OUT */
-					printk("sending WDTR ");
-					hostdata->outgoing_msg[0] = EXTENDED_MESSAGE;
-					hostdata->outgoing_msg[1] = 2;
-					hostdata->outgoing_msg[2] = EXTENDED_WDTR;
-					hostdata->outgoing_msg[3] = 0;	/* 8 bit transfer width */
-					hostdata->outgoing_len = 4;
-					write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-					hostdata->state = S_CONNECTED;
-					break;
-				default:
-					write_3393_cmd(hostdata, WD_CMD_ASSERT_ATN);	/* want MESS_OUT */
-					printk("Rejecting Unknown Extended Message(%02x). ", ucp[2]);
-					hostdata->outgoing_msg[0] = MESSAGE_REJECT;
-					hostdata->outgoing_len = 1;
-					write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-					hostdata->state = S_CONNECTED;
-					break;
-				}
-				hostdata->incoming_ptr = 0;
-			}
-
-			/* We need to read more MESS_IN bytes for the extended message */
-
-			else {
-				hostdata->incoming_ptr++;
-				write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-				hostdata->state = S_CONNECTED;
-			}
-			break;
-
-		default:
-			printk("Rejecting Unknown Message(%02x) ", msg);
-			write_3393_cmd(hostdata, WD_CMD_ASSERT_ATN);	/* want MESS_OUT */
-			hostdata->outgoing_msg[0] = MESSAGE_REJECT;
-			hostdata->outgoing_len = 1;
-			write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-			hostdata->state = S_CONNECTED;
-		}
-		break;
-
-
-/* Note: this interrupt will occur only after a LEVEL2 command */
-
-	case CSR_SEL_XFER_DONE:
-
-/* Make sure that reselection is enabled at this point - it may
- * have been turned off for the command that just completed.
- */
-
-		write_3393(hostdata, WD_SOURCE_ID, SRCID_ER);
-		if (phs == 0x60) {
-			DB(DB_INTR, printk("SX-DONE"))
-			    cmd->SCp.Message = COMMAND_COMPLETE;
-			lun = read_3393(hostdata, WD_TARGET_LUN);
-			DB(DB_INTR, printk(":%d.%d", cmd->SCp.Status, lun))
-			    hostdata->connected = NULL;
-			hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-			hostdata->state = S_UNCONNECTED;
-			if (cmd->SCp.Status == ILLEGAL_STATUS_BYTE)
-				cmd->SCp.Status = lun;
-			if (cmd->cmnd[0] == REQUEST_SENSE && cmd->SCp.Status != GOOD)
-				cmd->result = (cmd->result & 0x00ffff) | (DID_ERROR << 16);
-			else
-				cmd->result = cmd->SCp.Status | (cmd->SCp.Message << 8);
-			cmd->scsi_done(cmd);
-
-/* We are no longer connected to a target - check to see if
- * there are commands waiting to be executed.
- */
-
-			in2000_execute(instance);
-		} else {
-			printk("%02x:%02x:%02x: Unknown SEL_XFER_DONE phase!!---", asr, sr, phs);
-		}
-		break;
-
-
-/* Note: this interrupt will occur only after a LEVEL2 command */
-
-	case CSR_SDP:
-		DB(DB_INTR, printk("SDP"))
-		    hostdata->state = S_RUNNING_LEVEL2;
-		write_3393(hostdata, WD_COMMAND_PHASE, 0x41);
-		write_3393_cmd(hostdata, WD_CMD_SEL_ATN_XFER);
-		break;
-
-
-	case CSR_XFER_DONE | PHS_MESS_OUT:
-	case CSR_UNEXP | PHS_MESS_OUT:
-	case CSR_SRV_REQ | PHS_MESS_OUT:
-		DB(DB_INTR, printk("MSG_OUT="))
-
-/* To get here, we've probably requested MESSAGE_OUT and have
- * already put the correct bytes in outgoing_msg[] and filled
- * in outgoing_len. We simply send them out to the SCSI bus.
- * Sometimes we get MESSAGE_OUT phase when we're not expecting
- * it - like when our SDTR message is rejected by a target. Some
- * targets send the REJECT before receiving all of the extended
- * message, and then seem to go back to MESSAGE_OUT for a byte
- * or two. Not sure why, or if I'm doing something wrong to
- * cause this to happen. Regardless, it seems that sending
- * NOP messages in these situations results in no harm and
- * makes everyone happy.
- */
-		    if (hostdata->outgoing_len == 0) {
-			hostdata->outgoing_len = 1;
-			hostdata->outgoing_msg[0] = NOP;
-		}
-		transfer_pio(hostdata->outgoing_msg, hostdata->outgoing_len, DATA_OUT_DIR, hostdata);
-		DB(DB_INTR, printk("%02x", hostdata->outgoing_msg[0]))
-		    hostdata->outgoing_len = 0;
-		hostdata->state = S_CONNECTED;
-		break;
-
-
-	case CSR_UNEXP_DISC:
-
-/* I think I've seen this after a request-sense that was in response
- * to an error condition, but not sure. We certainly need to do
- * something when we get this interrupt - the question is 'what?'.
- * Let's think positively, and assume some command has finished
- * in a legal manner (like a command that provokes a request-sense),
- * so we treat it as a normal command-complete-disconnect.
- */
-
-
-/* Make sure that reselection is enabled at this point - it may
- * have been turned off for the command that just completed.
- */
-
-		write_3393(hostdata, WD_SOURCE_ID, SRCID_ER);
-		if (cmd == NULL) {
-			printk(" - Already disconnected! ");
-			hostdata->state = S_UNCONNECTED;
-
-/* release the SMP spin_lock and restore irq state */
-			spin_unlock_irqrestore(instance->host_lock, flags);
-			return IRQ_HANDLED;
-		}
-		DB(DB_INTR, printk("UNEXP_DISC"))
-		    hostdata->connected = NULL;
-		hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-		hostdata->state = S_UNCONNECTED;
-		if (cmd->cmnd[0] == REQUEST_SENSE && cmd->SCp.Status != GOOD)
-			cmd->result = (cmd->result & 0x00ffff) | (DID_ERROR << 16);
-		else
-			cmd->result = cmd->SCp.Status | (cmd->SCp.Message << 8);
-		cmd->scsi_done(cmd);
-
-/* We are no longer connected to a target - check to see if
- * there are commands waiting to be executed.
- */
-
-		in2000_execute(instance);
-		break;
-
-
-	case CSR_DISC:
-
-/* Make sure that reselection is enabled at this point - it may
- * have been turned off for the command that just completed.
- */
-
-		write_3393(hostdata, WD_SOURCE_ID, SRCID_ER);
-		DB(DB_INTR, printk("DISC"))
-		    if (cmd == NULL) {
-			printk(" - Already disconnected! ");
-			hostdata->state = S_UNCONNECTED;
-		}
-		switch (hostdata->state) {
-		case S_PRE_CMP_DISC:
-			hostdata->connected = NULL;
-			hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-			hostdata->state = S_UNCONNECTED;
-			DB(DB_INTR, printk(":%d", cmd->SCp.Status))
-			    if (cmd->cmnd[0] == REQUEST_SENSE && cmd->SCp.Status != GOOD)
-				cmd->result = (cmd->result & 0x00ffff) | (DID_ERROR << 16);
-			else
-				cmd->result = cmd->SCp.Status | (cmd->SCp.Message << 8);
-			cmd->scsi_done(cmd);
-			break;
-		case S_PRE_TMP_DISC:
-		case S_RUNNING_LEVEL2:
-			cmd->host_scribble = (uchar *) hostdata->disconnected_Q;
-			hostdata->disconnected_Q = cmd;
-			hostdata->connected = NULL;
-			hostdata->state = S_UNCONNECTED;
-
-#ifdef PROC_STATISTICS
-			hostdata->disc_done_cnt[cmd->device->id]++;
-#endif
-
-			break;
-		default:
-			printk("*** Unexpected DISCONNECT interrupt! ***");
-			hostdata->state = S_UNCONNECTED;
-		}
-
-/* We are no longer connected to a target - check to see if
- * there are commands waiting to be executed.
- */
-
-		in2000_execute(instance);
-		break;
-
-
-	case CSR_RESEL_AM:
-		DB(DB_INTR, printk("RESEL"))
-
-		    /* First we have to make sure this reselection didn't */
-		    /* happen during Arbitration/Selection of some other device. */
-		    /* If yes, put losing command back on top of input_Q. */
-		    if (hostdata->level2 <= L2_NONE) {
-
-			if (hostdata->selecting) {
-				cmd = (Scsi_Cmnd *) hostdata->selecting;
-				hostdata->selecting = NULL;
-				hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-				cmd->host_scribble = (uchar *) hostdata->input_Q;
-				hostdata->input_Q = cmd;
-			}
-		}
-
-		else {
-
-			if (cmd) {
-				if (phs == 0x00) {
-					hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-					cmd->host_scribble = (uchar *) hostdata->input_Q;
-					hostdata->input_Q = cmd;
-				} else {
-					printk("---%02x:%02x:%02x-TROUBLE: Intrusive ReSelect!---", asr, sr, phs);
-					while (1)
-						printk("\r");
-				}
-			}
-
-		}
-
-		/* OK - find out which device reselected us. */
-
-		id = read_3393(hostdata, WD_SOURCE_ID);
-		id &= SRCID_MASK;
-
-		/* and extract the lun from the ID message. (Note that we don't
-		 * bother to check for a valid message here - I guess this is
-		 * not the right way to go, but....)
-		 */
-
-		lun = read_3393(hostdata, WD_DATA);
-		if (hostdata->level2 < L2_RESELECT)
-			write_3393_cmd(hostdata, WD_CMD_NEGATE_ACK);
-		lun &= 7;
-
-		/* Now we look for the command that's reconnecting. */
-
-		cmd = (Scsi_Cmnd *) hostdata->disconnected_Q;
-		patch = NULL;
-		while (cmd) {
-			if (id == cmd->device->id && lun == cmd->device->lun)
-				break;
-			patch = cmd;
-			cmd = (Scsi_Cmnd *) cmd->host_scribble;
-		}
-
-		/* Hmm. Couldn't find a valid command.... What to do? */
-
-		if (!cmd) {
-			printk("---TROUBLE: target %d.%d not in disconnect queue---", id, lun);
-			break;
-		}
-
-		/* Ok, found the command - now start it up again. */
-
-		if (patch)
-			patch->host_scribble = cmd->host_scribble;
-		else
-			hostdata->disconnected_Q = (Scsi_Cmnd *) cmd->host_scribble;
-		hostdata->connected = cmd;
-
-		/* We don't need to worry about 'initialize_SCp()' or 'hostdata->busy[]'
-		 * because these things are preserved over a disconnect.
-		 * But we DO need to fix the DPD bit so it's correct for this command.
-		 */
-
-		if (is_dir_out(cmd))
-			write_3393(hostdata, WD_DESTINATION_ID, cmd->device->id);
-		else
-			write_3393(hostdata, WD_DESTINATION_ID, cmd->device->id | DSTID_DPD);
-		if (hostdata->level2 >= L2_RESELECT) {
-			write_3393_count(hostdata, 0);	/* we want a DATA_PHASE interrupt */
-			write_3393(hostdata, WD_COMMAND_PHASE, 0x45);
-			write_3393_cmd(hostdata, WD_CMD_SEL_ATN_XFER);
-			hostdata->state = S_RUNNING_LEVEL2;
-		} else
-			hostdata->state = S_CONNECTED;
-
-		    break;
-
-	default:
-		printk("--UNKNOWN INTERRUPT:%02x:%02x:%02x--", asr, sr, phs);
-	}
-
-	write1_io(0, IO_LED_OFF);
-
-	DB(DB_INTR, printk("} "))
-
-/* release the SMP spin_lock and restore irq state */
-	    spin_unlock_irqrestore(instance->host_lock, flags);
-	return IRQ_HANDLED;
-}
-
-
-
-#define RESET_CARD         0
-#define RESET_CARD_AND_BUS 1
-#define B_FLAG 0x80
-
-/*
- *	Caller must hold instance lock!
- */
-
-static int reset_hardware(struct Scsi_Host *instance, int type)
-{
-	struct IN2000_hostdata *hostdata;
-	int qt, x;
-
-	hostdata = (struct IN2000_hostdata *) instance->hostdata;
-
-	write1_io(0, IO_LED_ON);
-	if (type == RESET_CARD_AND_BUS) {
-		write1_io(0, IO_CARD_RESET);
-		x = read1_io(IO_HARDWARE);
-	}
-	x = read_3393(hostdata, WD_SCSI_STATUS);	/* clear any WD intrpt */
-	write_3393(hostdata, WD_OWN_ID, instance->this_id | OWNID_EAF | OWNID_RAF | OWNID_FS_8);
-	write_3393(hostdata, WD_CONTROL, CTRL_IDI | CTRL_EDI | CTRL_POLLED);
-	write_3393(hostdata, WD_SYNCHRONOUS_TRANSFER, calc_sync_xfer(hostdata->default_sx_per / 4, DEFAULT_SX_OFF));
-
-	write1_io(0, IO_FIFO_WRITE);	/* clear fifo counter */
-	write1_io(0, IO_FIFO_READ);	/* start fifo out in read mode */
-	write_3393(hostdata, WD_COMMAND, WD_CMD_RESET);
-	/* FIXME: timeout ?? */
-	while (!(READ_AUX_STAT() & ASR_INT))
-		cpu_relax();	/* wait for RESET to complete */
-
-	x = read_3393(hostdata, WD_SCSI_STATUS);	/* clear interrupt */
-
-	write_3393(hostdata, WD_QUEUE_TAG, 0xa5);	/* any random number */
-	qt = read_3393(hostdata, WD_QUEUE_TAG);
-	if (qt == 0xa5) {
-		x |= B_FLAG;
-		write_3393(hostdata, WD_QUEUE_TAG, 0);
-	}
-	write_3393(hostdata, WD_TIMEOUT_PERIOD, TIMEOUT_PERIOD_VALUE);
-	write_3393(hostdata, WD_CONTROL, CTRL_IDI | CTRL_EDI | CTRL_POLLED);
-	write1_io(0, IO_LED_OFF);
-	return x;
-}
-
-
-
-static int in2000_bus_reset(Scsi_Cmnd * cmd)
-{
-	struct Scsi_Host *instance;
-	struct IN2000_hostdata *hostdata;
-	int x;
-	unsigned long flags;
-
-	instance = cmd->device->host;
-	hostdata = (struct IN2000_hostdata *) instance->hostdata;
-
-	printk(KERN_WARNING "scsi%d: Reset. ", instance->host_no);
-
-	spin_lock_irqsave(instance->host_lock, flags);
-
-	/* do scsi-reset here */
-	reset_hardware(instance, RESET_CARD_AND_BUS);
-	for (x = 0; x < 8; x++) {
-		hostdata->busy[x] = 0;
-		hostdata->sync_xfer[x] = calc_sync_xfer(DEFAULT_SX_PER / 4, DEFAULT_SX_OFF);
-		hostdata->sync_stat[x] = SS_UNSET;	/* using default sync values */
-	}
-	hostdata->input_Q = NULL;
-	hostdata->selecting = NULL;
-	hostdata->connected = NULL;
-	hostdata->disconnected_Q = NULL;
-	hostdata->state = S_UNCONNECTED;
-	hostdata->fifo = FI_FIFO_UNUSED;
-	hostdata->incoming_ptr = 0;
-	hostdata->outgoing_len = 0;
-
-	cmd->result = DID_RESET << 16;
-
-	spin_unlock_irqrestore(instance->host_lock, flags);
-	return SUCCESS;
-}
-
-static int __in2000_abort(Scsi_Cmnd * cmd)
-{
-	struct Scsi_Host *instance;
-	struct IN2000_hostdata *hostdata;
-	Scsi_Cmnd *tmp, *prev;
-	uchar sr, asr;
-	unsigned long timeout;
-
-	instance = cmd->device->host;
-	hostdata = (struct IN2000_hostdata *) instance->hostdata;
-
-	printk(KERN_DEBUG "scsi%d: Abort-", instance->host_no);
-	printk("(asr=%02x,count=%ld,resid=%d,buf_resid=%d,have_data=%d,FC=%02x)- ", READ_AUX_STAT(), read_3393_count(hostdata), cmd->SCp.this_residual, cmd->SCp.buffers_residual, cmd->SCp.have_data_in, read1_io(IO_FIFO_COUNT));
-
-/*
- * Case 1 : If the command hasn't been issued yet, we simply remove it
- *     from the inout_Q.
- */
-
-	tmp = (Scsi_Cmnd *) hostdata->input_Q;
-	prev = NULL;
-	while (tmp) {
-		if (tmp == cmd) {
-			if (prev)
-				prev->host_scribble = cmd->host_scribble;
-			cmd->host_scribble = NULL;
-			cmd->result = DID_ABORT << 16;
-			printk(KERN_WARNING "scsi%d: Abort - removing command from input_Q. ", instance->host_no);
-			cmd->scsi_done(cmd);
-			return SUCCESS;
-		}
-		prev = tmp;
-		tmp = (Scsi_Cmnd *) tmp->host_scribble;
-	}
-
-/*
- * Case 2 : If the command is connected, we're going to fail the abort
- *     and let the high level SCSI driver retry at a later time or
- *     issue a reset.
- *
- *     Timeouts, and therefore aborted commands, will be highly unlikely
- *     and handling them cleanly in this situation would make the common
- *     case of noresets less efficient, and would pollute our code.  So,
- *     we fail.
- */
-
-	if (hostdata->connected == cmd) {
-
-		printk(KERN_WARNING "scsi%d: Aborting connected command - ", instance->host_no);
-
-		printk("sending wd33c93 ABORT command - ");
-		write_3393(hostdata, WD_CONTROL, CTRL_IDI | CTRL_EDI | CTRL_POLLED);
-		write_3393_cmd(hostdata, WD_CMD_ABORT);
-
-/* Now we have to attempt to flush out the FIFO... */
-
-		printk("flushing fifo - ");
-		timeout = 1000000;
-		do {
-			asr = READ_AUX_STAT();
-			if (asr & ASR_DBR)
-				read_3393(hostdata, WD_DATA);
-		} while (!(asr & ASR_INT) && timeout-- > 0);
-		sr = read_3393(hostdata, WD_SCSI_STATUS);
-		printk("asr=%02x, sr=%02x, %ld bytes un-transferred (timeout=%ld) - ", asr, sr, read_3393_count(hostdata), timeout);
-
-		/*
-		 * Abort command processed.
-		 * Still connected.
-		 * We must disconnect.
-		 */
-
-		printk("sending wd33c93 DISCONNECT command - ");
-		write_3393_cmd(hostdata, WD_CMD_DISCONNECT);
-
-		timeout = 1000000;
-		asr = READ_AUX_STAT();
-		while ((asr & ASR_CIP) && timeout-- > 0)
-			asr = READ_AUX_STAT();
-		sr = read_3393(hostdata, WD_SCSI_STATUS);
-		printk("asr=%02x, sr=%02x.", asr, sr);
-
-		hostdata->busy[cmd->device->id] &= ~(1 << cmd->device->lun);
-		hostdata->connected = NULL;
-		hostdata->state = S_UNCONNECTED;
-		cmd->result = DID_ABORT << 16;
-		cmd->scsi_done(cmd);
-
-		in2000_execute(instance);
-
-		return SUCCESS;
-	}
-
-/*
- * Case 3: If the command is currently disconnected from the bus,
- * we're not going to expend much effort here: Let's just return
- * an ABORT_SNOOZE and hope for the best...
- */
-
-	for (tmp = (Scsi_Cmnd *) hostdata->disconnected_Q; tmp; tmp = (Scsi_Cmnd *) tmp->host_scribble)
-		if (cmd == tmp) {
-			printk(KERN_DEBUG "scsi%d: unable to abort disconnected command.\n", instance->host_no);
-			return FAILED;
-		}
-
-/*
- * Case 4 : If we reached this point, the command was not found in any of
- *     the queues.
- *
- * We probably reached this point because of an unlikely race condition
- * between the command completing successfully and the abortion code,
- * so we won't panic, but we will notify the user in case something really
- * broke.
- */
-
-	in2000_execute(instance);
-
-	printk("scsi%d: warning : SCSI command probably completed successfully" "         before abortion. ", instance->host_no);
-	return SUCCESS;
-}
-
-static int in2000_abort(Scsi_Cmnd * cmd)
-{
-	int rc;
-
-	spin_lock_irq(cmd->device->host->host_lock);
-	rc = __in2000_abort(cmd);
-	spin_unlock_irq(cmd->device->host->host_lock);
-
-	return rc;
-}
-
-
-#define MAX_IN2000_HOSTS 3
-#define MAX_SETUP_ARGS ARRAY_SIZE(setup_args)
-#define SETUP_BUFFER_SIZE 200
-static char setup_buffer[SETUP_BUFFER_SIZE];
-static char setup_used[MAX_SETUP_ARGS];
-static int done_setup = 0;
-
-static void __init in2000_setup(char *str, int *ints)
-{
-	int i;
-	char *p1, *p2;
-
-	strlcpy(setup_buffer, str, SETUP_BUFFER_SIZE);
-	p1 = setup_buffer;
-	i = 0;
-	while (*p1 && (i < MAX_SETUP_ARGS)) {
-		p2 = strchr(p1, ',');
-		if (p2) {
-			*p2 = '\0';
-			if (p1 != p2)
-				setup_args[i] = p1;
-			p1 = p2 + 1;
-			i++;
-		} else {
-			setup_args[i] = p1;
-			break;
-		}
-	}
-	for (i = 0; i < MAX_SETUP_ARGS; i++)
-		setup_used[i] = 0;
-	done_setup = 1;
-}
-
-
-/* check_setup_args() returns index if key found, 0 if not
- */
-
-static int __init check_setup_args(char *key, int *val, char *buf)
-{
-	int x;
-	char *cp;
-
-	for (x = 0; x < MAX_SETUP_ARGS; x++) {
-		if (setup_used[x])
-			continue;
-		if (!strncmp(setup_args[x], key, strlen(key)))
-			break;
-	}
-	if (x == MAX_SETUP_ARGS)
-		return 0;
-	setup_used[x] = 1;
-	cp = setup_args[x] + strlen(key);
-	*val = -1;
-	if (*cp != ':')
-		return ++x;
-	cp++;
-	if ((*cp >= '0') && (*cp <= '9')) {
-		*val = simple_strtoul(cp, NULL, 0);
-	}
-	return ++x;
-}
-
-
-
-/* The "correct" (ie portable) way to access memory-mapped hardware
- * such as the IN2000 EPROM and dip switch is through the use of
- * special macros declared in 'asm/io.h'. We use readb() and readl()
- * when reading from the card's BIOS area in in2000_detect().
- */
-static u32 bios_tab[] in2000__INITDATA = {
-	0xc8000,
-	0xd0000,
-	0xd8000,
-	0
-};
-
-static unsigned short base_tab[] in2000__INITDATA = {
-	0x220,
-	0x200,
-	0x110,
-	0x100,
-};
-
-static int int_tab[] in2000__INITDATA = {
-	15,
-	14,
-	11,
-	10
-};
-
-static int probe_bios(u32 addr, u32 *s1, uchar *switches)
-{
-	void __iomem *p = ioremap(addr, 0x34);
-	if (!p)
-		return 0;
-	*s1 = readl(p + 0x10);
-	if (*s1 == 0x41564f4e || readl(p + 0x30) == 0x61776c41) {
-		/* Read the switch image that's mapped into EPROM space */
-		*switches = ~readb(p + 0x20);
-		iounmap(p);
-		return 1;
-	}
-	iounmap(p);
-	return 0;
-}
-
-static int __init in2000_detect(struct scsi_host_template * tpnt)
-{
-	struct Scsi_Host *instance;
-	struct IN2000_hostdata *hostdata;
-	int detect_count;
-	int bios;
-	int x;
-	unsigned short base;
-	uchar switches;
-	uchar hrev;
-	unsigned long flags;
-	int val;
-	char buf[32];
-
-/* Thanks to help from Bill Earnest, probing for IN2000 cards is a
- * pretty straightforward and fool-proof operation. There are 3
- * possible locations for the IN2000 EPROM in memory space - if we
- * find a BIOS signature, we can read the dip switch settings from
- * the byte at BIOS+32 (shadowed in by logic on the card). From 2
- * of the switch bits we get the card's address in IO space. There's
- * an image of the dip switch there, also, so we have a way to back-
- * check that this really is an IN2000 card. Very nifty. Use the
- * 'ioport:xx' command-line parameter if your BIOS EPROM is absent
- * or disabled.
- */
-
-	if (!done_setup && setup_strings)
-		in2000_setup(setup_strings, NULL);
-
-	detect_count = 0;
-	for (bios = 0; bios_tab[bios]; bios++) {
-		u32 s1 = 0;
-		if (check_setup_args("ioport", &val, buf)) {
-			base = val;
-			switches = ~inb(base + IO_SWITCHES) & 0xff;
-			printk("Forcing IN2000 detection at IOport 0x%x ", base);
-			bios = 2;
-		}
-/*
- * There have been a couple of BIOS versions with different layouts
- * for the obvious ID strings. We look for the 2 most common ones and
- * hope that they cover all the cases...
- */
-		else if (probe_bios(bios_tab[bios], &s1, &switches)) {
-			printk("Found IN2000 BIOS at 0x%x ", (unsigned int) bios_tab[bios]);
-
-/* Find out where the IO space is */
-
-			x = switches & (SW_ADDR0 | SW_ADDR1);
-			base = base_tab[x];
-
-/* Check for the IN2000 signature in IO space. */
-
-			x = ~inb(base + IO_SWITCHES) & 0xff;
-			if (x != switches) {
-				printk("Bad IO signature: %02x vs %02x.\n", x, switches);
-				continue;
-			}
-		} else
-			continue;
-
-/* OK. We have a base address for the IO ports - run a few safety checks */
-
-		if (!(switches & SW_BIT7)) {	/* I _think_ all cards do this */
-			printk("There is no IN-2000 SCSI card at IOport 0x%03x!\n", base);
-			continue;
-		}
-
-/* Let's assume any hardware version will work, although the driver
- * has only been tested on 0x21, 0x22, 0x25, 0x26, and 0x27. We'll
- * print out the rev number for reference later, but accept them all.
- */
-
-		hrev = inb(base + IO_HARDWARE);
-
-		/* Bit 2 tells us if interrupts are disabled */
-		if (switches & SW_DISINT) {
-			printk("The IN-2000 SCSI card at IOport 0x%03x ", base);
-			printk("is not configured for interrupt operation!\n");
-			printk("This driver requires an interrupt: cancelling detection.\n");
-			continue;
-		}
-
-/* Ok. We accept that there's an IN2000 at ioaddr 'base'. Now
- * initialize it.
- */
-
-		tpnt->proc_name = "in2000";
-		instance = scsi_register(tpnt, sizeof(struct IN2000_hostdata));
-		if (instance == NULL)
-			continue;
-		detect_count++;
-		hostdata = (struct IN2000_hostdata *) instance->hostdata;
-		instance->io_port = hostdata->io_base = base;
-		hostdata->dip_switch = switches;
-		hostdata->hrev = hrev;
-
-		write1_io(0, IO_FIFO_WRITE);	/* clear fifo counter */
-		write1_io(0, IO_FIFO_READ);	/* start fifo out in read mode */
-		write1_io(0, IO_INTR_MASK);	/* allow all ints */
-		x = int_tab[(switches & (SW_INT0 | SW_INT1)) >> SW_INT_SHIFT];
-		if (request_irq(x, in2000_intr, 0, "in2000", instance)) {
-			printk("in2000_detect: Unable to allocate IRQ.\n");
-			detect_count--;
-			continue;
-		}
-		instance->irq = x;
-		instance->n_io_port = 13;
-		request_region(base, 13, "in2000");	/* lock in this IO space for our use */
-
-		for (x = 0; x < 8; x++) {
-			hostdata->busy[x] = 0;
-			hostdata->sync_xfer[x] = calc_sync_xfer(DEFAULT_SX_PER / 4, DEFAULT_SX_OFF);
-			hostdata->sync_stat[x] = SS_UNSET;	/* using default sync values */
-#ifdef PROC_STATISTICS
-			hostdata->cmd_cnt[x] = 0;
-			hostdata->disc_allowed_cnt[x] = 0;
-			hostdata->disc_done_cnt[x] = 0;
-#endif
-		}
-		hostdata->input_Q = NULL;
-		hostdata->selecting = NULL;
-		hostdata->connected = NULL;
-		hostdata->disconnected_Q = NULL;
-		hostdata->state = S_UNCONNECTED;
-		hostdata->fifo = FI_FIFO_UNUSED;
-		hostdata->level2 = L2_BASIC;
-		hostdata->disconnect = DIS_ADAPTIVE;
-		hostdata->args = DEBUG_DEFAULTS;
-		hostdata->incoming_ptr = 0;
-		hostdata->outgoing_len = 0;
-		hostdata->default_sx_per = DEFAULT_SX_PER;
-
-/* Older BIOS's had a 'sync on/off' switch - use its setting */
-
-		if (s1 == 0x41564f4e && (switches & SW_SYNC_DOS5))
-			hostdata->sync_off = 0x00;	/* sync defaults to on */
-		else
-			hostdata->sync_off = 0xff;	/* sync defaults to off */
-
-#ifdef PROC_INTERFACE
-		hostdata->proc = PR_VERSION | PR_INFO | PR_STATISTICS | PR_CONNECTED | PR_INPUTQ | PR_DISCQ | PR_STOP;
-#ifdef PROC_STATISTICS
-		hostdata->int_cnt = 0;
-#endif
-#endif
-
-		if (check_setup_args("nosync", &val, buf))
-			hostdata->sync_off = val;
-
-		if (check_setup_args("period", &val, buf))
-			hostdata->default_sx_per = sx_table[round_period((unsigned int) val)].period_ns;
-
-		if (check_setup_args("disconnect", &val, buf)) {
-			if ((val >= DIS_NEVER) && (val <= DIS_ALWAYS))
-				hostdata->disconnect = val;
-			else
-				hostdata->disconnect = DIS_ADAPTIVE;
-		}
-
-		if (check_setup_args("noreset", &val, buf))
-			hostdata->args ^= A_NO_SCSI_RESET;
-
-		if (check_setup_args("level2", &val, buf))
-			hostdata->level2 = val;
-
-		if (check_setup_args("debug", &val, buf))
-			hostdata->args = (val & DB_MASK);
-
-#ifdef PROC_INTERFACE
-		if (check_setup_args("proc", &val, buf))
-			hostdata->proc = val;
-#endif
-
-
-		/* FIXME: not strictly needed I think but the called code expects
-		   to be locked */
-		spin_lock_irqsave(instance->host_lock, flags);
-		x = reset_hardware(instance, (hostdata->args & A_NO_SCSI_RESET) ? RESET_CARD : RESET_CARD_AND_BUS);
-		spin_unlock_irqrestore(instance->host_lock, flags);
-
-		hostdata->microcode = read_3393(hostdata, WD_CDB_1);
-		if (x & 0x01) {
-			if (x & B_FLAG)
-				hostdata->chip = C_WD33C93B;
-			else
-				hostdata->chip = C_WD33C93A;
-		} else
-			hostdata->chip = C_WD33C93;
-
-		printk("dip_switch=%02x irq=%d ioport=%02x floppy=%s sync/DOS5=%s ", (switches & 0x7f), instance->irq, hostdata->io_base, (switches & SW_FLOPPY) ? "Yes" : "No", (switches & SW_SYNC_DOS5) ? "Yes" : "No");
-		printk("hardware_ver=%02x chip=%s microcode=%02x\n", hrev, (hostdata->chip == C_WD33C93) ? "WD33c93" : (hostdata->chip == C_WD33C93A) ? "WD33c93A" : (hostdata->chip == C_WD33C93B) ? "WD33c93B" : "unknown", hostdata->microcode);
-#ifdef DEBUGGING_ON
-		printk("setup_args = ");
-		for (x = 0; x < MAX_SETUP_ARGS; x++)
-			printk("%s,", setup_args[x]);
-		printk("\n");
-#endif
-		if (hostdata->sync_off == 0xff)
-			printk("Sync-transfer DISABLED on all devices: ENABLE from command-line\n");
-		printk("IN2000 driver version %s - %s\n", IN2000_VERSION, IN2000_DATE);
-	}
-
-	return detect_count;
-}
-
-static int in2000_release(struct Scsi_Host *shost)
-{
-	if (shost->irq)
-		free_irq(shost->irq, shost);
-	if (shost->io_port && shost->n_io_port)
-		release_region(shost->io_port, shost->n_io_port);
-	return 0;
-}
-
-/* NOTE: I lifted this function straight out of the old driver,
- *       and have not tested it. Presumably it does what it's
- *       supposed to do...
- */
-
-static int in2000_biosparam(struct scsi_device *sdev, struct block_device *bdev, sector_t capacity, int *iinfo)
-{
-	int size;
-
-	size = capacity;
-	iinfo[0] = 64;
-	iinfo[1] = 32;
-	iinfo[2] = size >> 11;
-
-/* This should approximate the large drive handling that the DOS ASPI manager
-   uses.  Drives very near the boundaries may not be handled correctly (i.e.
-   near 2.0 Gb and 4.0 Gb) */
-
-	if (iinfo[2] > 1024) {
-		iinfo[0] = 64;
-		iinfo[1] = 63;
-		iinfo[2] = (unsigned long) capacity / (iinfo[0] * iinfo[1]);
-	}
-	if (iinfo[2] > 1024) {
-		iinfo[0] = 128;
-		iinfo[1] = 63;
-		iinfo[2] = (unsigned long) capacity / (iinfo[0] * iinfo[1]);
-	}
-	if (iinfo[2] > 1024) {
-		iinfo[0] = 255;
-		iinfo[1] = 63;
-		iinfo[2] = (unsigned long) capacity / (iinfo[0] * iinfo[1]);
-	}
-	return 0;
-}
-
-
-static int in2000_write_info(struct Scsi_Host *instance, char *buf, int len)
-{
-
-#ifdef PROC_INTERFACE
-
-	char *bp;
-	struct IN2000_hostdata *hd;
-	int x, i;
-
-	hd = (struct IN2000_hostdata *) instance->hostdata;
-
-	buf[len] = '\0';
-	bp = buf;
-	if (!strncmp(bp, "debug:", 6)) {
-		bp += 6;
-		hd->args = simple_strtoul(bp, NULL, 0) & DB_MASK;
-	} else if (!strncmp(bp, "disconnect:", 11)) {
-		bp += 11;
-		x = simple_strtoul(bp, NULL, 0);
-		if (x < DIS_NEVER || x > DIS_ALWAYS)
-			x = DIS_ADAPTIVE;
-		hd->disconnect = x;
-	} else if (!strncmp(bp, "period:", 7)) {
-		bp += 7;
-		x = simple_strtoul(bp, NULL, 0);
-		hd->default_sx_per = sx_table[round_period((unsigned int) x)].period_ns;
-	} else if (!strncmp(bp, "resync:", 7)) {
-		bp += 7;
-		x = simple_strtoul(bp, NULL, 0);
-		for (i = 0; i < 7; i++)
-			if (x & (1 << i))
-				hd->sync_stat[i] = SS_UNSET;
-	} else if (!strncmp(bp, "proc:", 5)) {
-		bp += 5;
-		hd->proc = simple_strtoul(bp, NULL, 0);
-	} else if (!strncmp(bp, "level2:", 7)) {
-		bp += 7;
-		hd->level2 = simple_strtoul(bp, NULL, 0);
-	}
-#endif
-	return len;
-}
-
-static int in2000_show_info(struct seq_file *m, struct Scsi_Host *instance)
-{
-
-#ifdef PROC_INTERFACE
-	unsigned long flags;
-	struct IN2000_hostdata *hd;
-	Scsi_Cmnd *cmd;
-	int x;
-
-	hd = (struct IN2000_hostdata *) instance->hostdata;
-
-	spin_lock_irqsave(instance->host_lock, flags);
-	if (hd->proc & PR_VERSION)
-		seq_printf(m, "\nVersion %s - %s.", IN2000_VERSION, IN2000_DATE);
-
-	if (hd->proc & PR_INFO) {
-		seq_printf(m, "\ndip_switch=%02x: irq=%d io=%02x floppy=%s sync/DOS5=%s", (hd->dip_switch & 0x7f), instance->irq, hd->io_base, (hd->dip_switch & 0x40) ? "Yes" : "No", (hd->dip_switch & 0x20) ? "Yes" : "No");
-		seq_puts(m, "\nsync_xfer[] =       ");
-		for (x = 0; x < 7; x++)
-			seq_printf(m, "\t%02x", hd->sync_xfer[x]);
-		seq_puts(m, "\nsync_stat[] =       ");
-		for (x = 0; x < 7; x++)
-			seq_printf(m, "\t%02x", hd->sync_stat[x]);
-	}
-#ifdef PROC_STATISTICS
-	if (hd->proc & PR_STATISTICS) {
-		seq_puts(m, "\ncommands issued:    ");
-		for (x = 0; x < 7; x++)
-			seq_printf(m, "\t%ld", hd->cmd_cnt[x]);
-		seq_puts(m, "\ndisconnects allowed:");
-		for (x = 0; x < 7; x++)
-			seq_printf(m, "\t%ld", hd->disc_allowed_cnt[x]);
-		seq_puts(m, "\ndisconnects done:   ");
-		for (x = 0; x < 7; x++)
-			seq_printf(m, "\t%ld", hd->disc_done_cnt[x]);
-		seq_printf(m, "\ninterrupts:      \t%ld", hd->int_cnt);
-	}
-#endif
-	if (hd->proc & PR_CONNECTED) {
-		seq_puts(m, "\nconnected:     ");
-		if (hd->connected) {
-			cmd = (Scsi_Cmnd *) hd->connected;
-			seq_printf(m, " %d:%llu(%02x)", cmd->device->id, cmd->device->lun, cmd->cmnd[0]);
-		}
-	}
-	if (hd->proc & PR_INPUTQ) {
-		seq_puts(m, "\ninput_Q:       ");
-		cmd = (Scsi_Cmnd *) hd->input_Q;
-		while (cmd) {
-			seq_printf(m, " %d:%llu(%02x)", cmd->device->id, cmd->device->lun, cmd->cmnd[0]);
-			cmd = (Scsi_Cmnd *) cmd->host_scribble;
-		}
-	}
-	if (hd->proc & PR_DISCQ) {
-		seq_puts(m, "\ndisconnected_Q:");
-		cmd = (Scsi_Cmnd *) hd->disconnected_Q;
-		while (cmd) {
-			seq_printf(m, " %d:%llu(%02x)", cmd->device->id, cmd->device->lun, cmd->cmnd[0]);
-			cmd = (Scsi_Cmnd *) cmd->host_scribble;
-		}
-	}
-	if (hd->proc & PR_TEST) {
-		;		/* insert your own custom function here */
-	}
-	seq_putc(m, '\n');
-	spin_unlock_irqrestore(instance->host_lock, flags);
-#endif				/* PROC_INTERFACE */
-	return 0;
-}
-
-MODULE_LICENSE("GPL");
-
-
-static struct scsi_host_template driver_template = {
-	.proc_name       		= "in2000",
-	.write_info       		= in2000_write_info,
-	.show_info       		= in2000_show_info,
-	.name            		= "Always IN2000",
-	.detect          		= in2000_detect, 
-	.release			= in2000_release,
-	.queuecommand    		= in2000_queuecommand,
-	.eh_abort_handler		= in2000_abort,
-	.eh_bus_reset_handler		= in2000_bus_reset,
-	.bios_param      		= in2000_biosparam, 
-	.can_queue       		= IN2000_CAN_Q,
-	.this_id         		= IN2000_HOST_ID,
-	.sg_tablesize    		= IN2000_SG,
-	.cmd_per_lun     		= IN2000_CPL,
-	.use_clustering  		= DISABLE_CLUSTERING,
-};
-#include "scsi_module.c"
diff --git a/drivers/scsi/in2000.h b/drivers/scsi/in2000.h
deleted file mode 100644
index 5821e1fbce08..000000000000
--- a/drivers/scsi/in2000.h
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- *    in2000.h -  Linux device driver definitions for the
- *                Always IN2000 ISA SCSI card.
- *
- *    IMPORTANT: This file is for version 1.33 - 26/Aug/1998
- *
- * Copyright (c) 1996 John Shifflett, GeoLog Consulting
- *    john@geolog.com
- *    jshiffle@netcom.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- */
-
-#ifndef IN2000_H
-#define IN2000_H
-
-#include <asm/io.h>
-
-#define PROC_INTERFACE     /* add code for /proc/scsi/in2000/xxx interface */
-#ifdef  PROC_INTERFACE
-#define PROC_STATISTICS    /* add code for keeping various real time stats */
-#endif
-
-#define SYNC_DEBUG         /* extra info on sync negotiation printed */
-#define DEBUGGING_ON       /* enable command-line debugging bitmask */
-#define DEBUG_DEFAULTS 0   /* default bitmask - change from command-line */
-
-#ifdef __i386__
-#define FAST_READ_IO       /* No problems with these on my machine */
-#define FAST_WRITE_IO
-#endif
-
-#ifdef DEBUGGING_ON
-#define DB(f,a) if (hostdata->args & (f)) a;
-#define CHECK_NULL(p,s) /* if (!(p)) {printk("\n"); while (1) printk("NP:%s\r",(s));} */
-#else
-#define DB(f,a)
-#define CHECK_NULL(p,s)
-#endif
-
-#define uchar unsigned char
-
-#define read1_io(a)     (inb(hostdata->io_base+(a)))
-#define read2_io(a)     (inw(hostdata->io_base+(a)))
-#define write1_io(b,a)  (outb((b),hostdata->io_base+(a)))
-#define write2_io(w,a)  (outw((w),hostdata->io_base+(a)))
-
-#ifdef __i386__
-/* These inline assembly defines are derived from a patch
- * sent to me by Bill Earnest. He's done a lot of very
- * valuable thinking, testing, and coding during his effort
- * to squeeze more speed out of this driver. I really think
- * that we are doing IO at close to the maximum now with
- * the fifo. (And yes, insw uses 'edi' while outsw uses
- * 'esi'. Thanks Bill!)
- */
-
-#define FAST_READ2_IO()    \
-({ \
-int __dummy_1,__dummy_2; \
-   __asm__ __volatile__ ("\n \
-   cld                    \n \
-   orl %%ecx, %%ecx       \n \
-   jz 1f                  \n \
-   rep                    \n \
-   insw (%%dx),%%es:(%%edi) \n \
-1: "                       \
-   : "=D" (sp) ,"=c" (__dummy_1) ,"=d" (__dummy_2)  /* output */   \
-   : "2" (f), "0" (sp), "1" (i)  /* input */    \
-   );       /* trashed */ \
-})
-
-#define FAST_WRITE2_IO()   \
-({ \
-int __dummy_1,__dummy_2; \
-   __asm__ __volatile__ ("\n \
-   cld                    \n \
-   orl %%ecx, %%ecx       \n \
-   jz 1f                  \n \
-   rep                    \n \
-   outsw %%ds:(%%esi),(%%dx) \n \
-1: "                       \
-   : "=S" (sp) ,"=c" (__dummy_1) ,"=d" (__dummy_2)/* output */   \
-   : "2" (f), "0" (sp), "1" (i)  /* input */    \
-   );       /* trashed */ \
-})
-#endif
-
-/* IN2000 io_port offsets */
-#define IO_WD_ASR       0x00     /* R - 3393 auxstat reg */
-#define     ASR_INT        0x80
-#define     ASR_LCI        0x40
-#define     ASR_BSY        0x20
-#define     ASR_CIP        0x10
-#define     ASR_PE         0x02
-#define     ASR_DBR        0x01
-#define IO_WD_ADDR      0x00     /* W - 3393 address reg */
-#define IO_WD_DATA      0x01     /* R/W - rest of 3393 regs */
-#define IO_FIFO         0x02     /* R/W - in2000 dual-port fifo (16 bits) */
-#define IN2000_FIFO_SIZE   2048  /*    fifo capacity in bytes */
-#define IO_CARD_RESET   0x03     /* W - in2000 start master reset */
-#define IO_FIFO_COUNT   0x04     /* R - in2000 fifo counter */
-#define IO_FIFO_WRITE   0x05     /* W - clear fifo counter, start write */
-#define IO_FIFO_READ    0x07     /* W - start fifo read */
-#define IO_LED_OFF      0x08     /* W - turn off in2000 activity LED */
-#define IO_SWITCHES     0x08     /* R - read in2000 dip switch */
-#define     SW_ADDR0       0x01  /*    bit 0 = bit 0 of index to io addr */
-#define     SW_ADDR1       0x02  /*    bit 1 = bit 1 of index io addr */
-#define     SW_DISINT      0x04  /*    bit 2 true if ints disabled */
-#define     SW_INT0        0x08  /*    bit 3 = bit 0 of index to interrupt */
-#define     SW_INT1        0x10  /*    bit 4 = bit 1 of index to interrupt */
-#define     SW_INT_SHIFT   3     /*    shift right this amount to right justify int bits */
-#define     SW_SYNC_DOS5   0x20  /*    bit 5 used by Always BIOS */
-#define     SW_FLOPPY      0x40  /*    bit 6 true if floppy enabled */
-#define     SW_BIT7        0x80  /*    bit 7 hardwired true (ground) */
-#define IO_LED_ON       0x09     /* W - turn on in2000 activity LED */
-#define IO_HARDWARE     0x0a     /* R - read in2000 hardware rev, stop reset */
-#define IO_INTR_MASK    0x0c     /* W - in2000 interrupt mask reg */
-#define     IMASK_WD       0x01  /*    WD33c93 interrupt mask */
-#define     IMASK_FIFO     0x02  /*    FIFO interrupt mask */
-
-/* wd register names */
-#define WD_OWN_ID    0x00
-#define WD_CONTROL   0x01
-#define WD_TIMEOUT_PERIOD  0x02
-#define WD_CDB_1     0x03
-#define WD_CDB_2     0x04
-#define WD_CDB_3     0x05
-#define WD_CDB_4     0x06
-#define WD_CDB_5     0x07
-#define WD_CDB_6     0x08
-#define WD_CDB_7     0x09
-#define WD_CDB_8     0x0a
-#define WD_CDB_9     0x0b
-#define WD_CDB_10    0x0c
-#define WD_CDB_11    0x0d
-#define WD_CDB_12    0x0e
-#define WD_TARGET_LUN      0x0f
-#define WD_COMMAND_PHASE   0x10
-#define WD_SYNCHRONOUS_TRANSFER  0x11
-#define WD_TRANSFER_COUNT_MSB 0x12
-#define WD_TRANSFER_COUNT  0x13
-#define WD_TRANSFER_COUNT_LSB 0x14
-#define WD_DESTINATION_ID  0x15
-#define WD_SOURCE_ID    0x16
-#define WD_SCSI_STATUS     0x17
-#define WD_COMMAND      0x18
-#define WD_DATA      0x19
-#define WD_QUEUE_TAG    0x1a
-#define WD_AUXILIARY_STATUS   0x1f
-
-/* WD commands */
-#define WD_CMD_RESET    0x00
-#define WD_CMD_ABORT    0x01
-#define WD_CMD_ASSERT_ATN  0x02
-#define WD_CMD_NEGATE_ACK  0x03
-#define WD_CMD_DISCONNECT  0x04
-#define WD_CMD_RESELECT    0x05
-#define WD_CMD_SEL_ATN     0x06
-#define WD_CMD_SEL      0x07
-#define WD_CMD_SEL_ATN_XFER   0x08
-#define WD_CMD_SEL_XFER    0x09
-#define WD_CMD_RESEL_RECEIVE  0x0a
-#define WD_CMD_RESEL_SEND  0x0b
-#define WD_CMD_WAIT_SEL_RECEIVE 0x0c
-#define WD_CMD_TRANS_ADDR  0x18
-#define WD_CMD_TRANS_INFO  0x20
-#define WD_CMD_TRANSFER_PAD   0x21
-#define WD_CMD_SBT_MODE    0x80
-
-/* SCSI Bus Phases */
-#define PHS_DATA_OUT    0x00
-#define PHS_DATA_IN     0x01
-#define PHS_COMMAND     0x02
-#define PHS_STATUS      0x03
-#define PHS_MESS_OUT    0x06
-#define PHS_MESS_IN     0x07
-
-/* Command Status Register definitions */
-
-  /* reset state interrupts */
-#define CSR_RESET    0x00
-#define CSR_RESET_AF    0x01
-
-  /* successful completion interrupts */
-#define CSR_RESELECT    0x10
-#define CSR_SELECT      0x11
-#define CSR_SEL_XFER_DONE  0x16
-#define CSR_XFER_DONE      0x18
-
-  /* paused or aborted interrupts */
-#define CSR_MSGIN    0x20
-#define CSR_SDP         0x21
-#define CSR_SEL_ABORT      0x22
-#define CSR_RESEL_ABORT    0x25
-#define CSR_RESEL_ABORT_AM 0x27
-#define CSR_ABORT    0x28
-
-  /* terminated interrupts */
-#define CSR_INVALID     0x40
-#define CSR_UNEXP_DISC     0x41
-#define CSR_TIMEOUT     0x42
-#define CSR_PARITY      0x43
-#define CSR_PARITY_ATN     0x44
-#define CSR_BAD_STATUS     0x45
-#define CSR_UNEXP    0x48
-
-  /* service required interrupts */
-#define CSR_RESEL    0x80
-#define CSR_RESEL_AM    0x81
-#define CSR_DISC     0x85
-#define CSR_SRV_REQ     0x88
-
-   /* Own ID/CDB Size register */
-#define OWNID_EAF    0x08
-#define OWNID_EHP    0x10
-#define OWNID_RAF    0x20
-#define OWNID_FS_8   0x00
-#define OWNID_FS_12  0x40
-#define OWNID_FS_16  0x80
-
-   /* Control register */
-#define CTRL_HSP     0x01
-#define CTRL_HA      0x02
-#define CTRL_IDI     0x04
-#define CTRL_EDI     0x08
-#define CTRL_HHP     0x10
-#define CTRL_POLLED  0x00
-#define CTRL_BURST   0x20
-#define CTRL_BUS     0x40
-#define CTRL_DMA     0x80
-
-   /* Timeout Period register */
-#define TIMEOUT_PERIOD_VALUE  20    /* results in 200 ms. */
-
-   /* Synchronous Transfer Register */
-#define STR_FSS      0x80
-
-   /* Destination ID register */
-#define DSTID_DPD    0x40
-#define DATA_OUT_DIR 0
-#define DATA_IN_DIR  1
-#define DSTID_SCC    0x80
-
-   /* Source ID register */
-#define SRCID_MASK   0x07
-#define SRCID_SIV    0x08
-#define SRCID_DSP    0x20
-#define SRCID_ES     0x40
-#define SRCID_ER     0x80
-
-
-
-#define ILLEGAL_STATUS_BYTE   0xff
-
-
-#define DEFAULT_SX_PER     500   /* (ns) fairly safe */
-#define DEFAULT_SX_OFF     0     /* aka async */
-
-#define OPTIMUM_SX_PER     252   /* (ns) best we can do (mult-of-4) */
-#define OPTIMUM_SX_OFF     12    /* size of in2000 fifo */
-
-struct sx_period {
-   unsigned int   period_ns;
-   uchar          reg_value;
-   };
-
-
-struct IN2000_hostdata {
-    struct Scsi_Host *next;
-    uchar            chip;             /* what kind of wd33c93 chip? */
-    uchar            microcode;        /* microcode rev if 'B' */
-    unsigned short   io_base;          /* IO port base */
-    unsigned int     dip_switch;       /* dip switch settings */
-    unsigned int     hrev;             /* hardware revision of card */
-    volatile uchar   busy[8];          /* index = target, bit = lun */
-    volatile Scsi_Cmnd *input_Q;       /* commands waiting to be started */
-    volatile Scsi_Cmnd *selecting;     /* trying to select this command */
-    volatile Scsi_Cmnd *connected;     /* currently connected command */
-    volatile Scsi_Cmnd *disconnected_Q;/* commands waiting for reconnect */
-    uchar            state;            /* what we are currently doing */
-    uchar            fifo;             /* what the FIFO is up to */
-    uchar            level2;           /* extent to which Level-2 commands are used */
-    uchar            disconnect;       /* disconnect/reselect policy */
-    unsigned int     args;             /* set from command-line argument */
-    uchar            incoming_msg[8];  /* filled during message_in phase */
-    int              incoming_ptr;     /* mainly used with EXTENDED messages */
-    uchar            outgoing_msg[8];  /* send this during next message_out */
-    int              outgoing_len;     /* length of outgoing message */
-    unsigned int     default_sx_per;   /* default transfer period for SCSI bus */
-    uchar            sync_xfer[8];     /* sync_xfer reg settings per target */
-    uchar            sync_stat[8];     /* status of sync negotiation per target */
-    uchar            sync_off;         /* bit mask: don't use sync with these targets */
-#ifdef PROC_INTERFACE
-    uchar            proc;             /* bit mask: what's in proc output */
-#ifdef PROC_STATISTICS
-    unsigned long    cmd_cnt[8];       /* # of commands issued per target */
-    unsigned long    int_cnt;          /* # of interrupts serviced */
-    unsigned long    disc_allowed_cnt[8]; /* # of disconnects allowed per target */
-    unsigned long    disc_done_cnt[8]; /* # of disconnects done per target*/
-#endif
-#endif
-    };
-
-
-/* defines for hostdata->chip */
-
-#define C_WD33C93       0
-#define C_WD33C93A      1
-#define C_WD33C93B      2
-#define C_UNKNOWN_CHIP  100
-
-/* defines for hostdata->state */
-
-#define S_UNCONNECTED         0
-#define S_SELECTING           1
-#define S_RUNNING_LEVEL2      2
-#define S_CONNECTED           3
-#define S_PRE_TMP_DISC        4
-#define S_PRE_CMP_DISC        5
-
-/* defines for hostdata->fifo */
-
-#define FI_FIFO_UNUSED        0
-#define FI_FIFO_READING       1
-#define FI_FIFO_WRITING       2
-
-/* defines for hostdata->level2 */
-/* NOTE: only the first 3 are trustworthy at this point -
- * having trouble when more than 1 device is reading/writing
- * at the same time...
- */
-
-#define L2_NONE      0  /* no combination commands - we get lots of ints */
-#define L2_SELECT    1  /* start with SEL_ATN_XFER, but never resume it */
-#define L2_BASIC     2  /* resume after STATUS ints & RDP messages */
-#define L2_DATA      3  /* resume after DATA_IN/OUT ints */
-#define L2_MOST      4  /* resume after anything except a RESELECT int */
-#define L2_RESELECT  5  /* resume after everything, including RESELECT ints */
-#define L2_ALL       6  /* always resume */
-
-/* defines for hostdata->disconnect */
-
-#define DIS_NEVER    0
-#define DIS_ADAPTIVE 1
-#define DIS_ALWAYS   2
-
-/* defines for hostdata->args */
-
-#define DB_TEST               1<<0
-#define DB_FIFO               1<<1
-#define DB_QUEUE_COMMAND      1<<2
-#define DB_EXECUTE            1<<3
-#define DB_INTR               1<<4
-#define DB_TRANSFER           1<<5
-#define DB_MASK               0x3f
-
-#define A_NO_SCSI_RESET       1<<15
-
-
-/* defines for hostdata->sync_xfer[] */
-
-#define SS_UNSET     0
-#define SS_FIRST     1
-#define SS_WAITING   2
-#define SS_SET       3
-
-/* defines for hostdata->proc */
-
-#define PR_VERSION   1<<0
-#define PR_INFO      1<<1
-#define PR_STATISTICS 1<<2
-#define PR_CONNECTED 1<<3
-#define PR_INPUTQ    1<<4
-#define PR_DISCQ     1<<5
-#define PR_TEST      1<<6
-#define PR_STOP      1<<7
-
-
-# include <linux/init.h>
-# include <linux/spinlock.h>
-# define in2000__INITFUNC(function) __initfunc(function)
-# define in2000__INIT __init
-# define in2000__INITDATA __initdata
-# define CLISPIN_LOCK(host,flags)   spin_lock_irqsave(host->host_lock, flags)
-# define CLISPIN_UNLOCK(host,flags) spin_unlock_irqrestore(host->host_lock, \
-							   flags)
-
-static int in2000_detect(struct scsi_host_template *) in2000__INIT;
-static int in2000_queuecommand(struct Scsi_Host *, struct scsi_cmnd *);
-static int in2000_abort(Scsi_Cmnd *);
-static void in2000_setup(char *, int *) in2000__INIT;
-static int in2000_biosparam(struct scsi_device *, struct block_device *,
-		sector_t, int *);
-static int in2000_bus_reset(Scsi_Cmnd *);
-
-
-#define IN2000_CAN_Q    16
-#define IN2000_SG       SG_ALL
-#define IN2000_CPL      2
-#define IN2000_HOST_ID  7
-
-#endif /* IN2000_H */
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index bf85974be862..a8762a3efeef 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -493,15 +493,15 @@ struct ipr_error_table_t ipr_error_table[] = {
 	"9072: Link not operational transition"},
 	{0x066B8200, 0, IPR_DEFAULT_LOG_LEVEL,
 	"9032: Array exposed but still protected"},
-	{0x066B8300, 0, IPR_DEFAULT_LOG_LEVEL + 1,
+	{0x066B8300, 0, IPR_DEBUG_LOG_LEVEL,
 	"70DD: Device forced failed by disrupt device command"},
 	{0x066B9100, 0, IPR_DEFAULT_LOG_LEVEL,
 	"4061: Multipath redundancy level got better"},
 	{0x066B9200, 0, IPR_DEFAULT_LOG_LEVEL,
 	"4060: Multipath redundancy level got worse"},
-	{0x06808100, 0, IPR_DEFAULT_LOG_LEVEL,
+	{0x06808100, 0, IPR_DEBUG_LOG_LEVEL,
 	"9083: Device raw mode enabled"},
-	{0x06808200, 0, IPR_DEFAULT_LOG_LEVEL,
+	{0x06808200, 0, IPR_DEBUG_LOG_LEVEL,
 	"9084: Device raw mode disabled"},
 	{0x07270000, 0, 0,
 	"Failure due to other device"},
@@ -1473,7 +1473,7 @@ static void ipr_process_ccn(struct ipr_cmnd *ipr_cmd)
 	struct ipr_hostrcb *hostrcb = ipr_cmd->u.hostrcb;
 	u32 ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc);
 
-	list_del(&hostrcb->queue);
+	list_del_init(&hostrcb->queue);
 	list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q);
 
 	if (ioasc) {
@@ -2552,6 +2552,23 @@ static void ipr_handle_log_data(struct ipr_ioa_cfg *ioa_cfg,
 	}
 }
 
+static struct ipr_hostrcb *ipr_get_free_hostrcb(struct ipr_ioa_cfg *ioa)
+{
+	struct ipr_hostrcb *hostrcb;
+
+	hostrcb = list_first_entry_or_null(&ioa->hostrcb_free_q,
+					struct ipr_hostrcb, queue);
+
+	if (unlikely(!hostrcb)) {
+		dev_info(&ioa->pdev->dev, "Reclaiming async error buffers.");
+		hostrcb = list_first_entry_or_null(&ioa->hostrcb_report_q,
+						struct ipr_hostrcb, queue);
+	}
+
+	list_del_init(&hostrcb->queue);
+	return hostrcb;
+}
+
 /**
  * ipr_process_error - Op done function for an adapter error log.
  * @ipr_cmd:	ipr command struct
@@ -2569,13 +2586,14 @@ static void ipr_process_error(struct ipr_cmnd *ipr_cmd)
 	struct ipr_hostrcb *hostrcb = ipr_cmd->u.hostrcb;
 	u32 ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc);
 	u32 fd_ioasc;
+	char *envp[] = { "ASYNC_ERR_LOG=1", NULL };
 
 	if (ioa_cfg->sis64)
 		fd_ioasc = be32_to_cpu(hostrcb->hcam.u.error64.fd_ioasc);
 	else
 		fd_ioasc = be32_to_cpu(hostrcb->hcam.u.error.fd_ioasc);
 
-	list_del(&hostrcb->queue);
+	list_del_init(&hostrcb->queue);
 	list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q);
 
 	if (!ioasc) {
@@ -2588,6 +2606,10 @@ static void ipr_process_error(struct ipr_cmnd *ipr_cmd)
 			"Host RCB failed with IOASC: 0x%08X\n", ioasc);
 	}
 
+	list_add_tail(&hostrcb->queue, &ioa_cfg->hostrcb_report_q);
+	hostrcb = ipr_get_free_hostrcb(ioa_cfg);
+	kobject_uevent_env(&ioa_cfg->host->shost_dev.kobj, KOBJ_CHANGE, envp);
+
 	ipr_send_hcam(ioa_cfg, IPR_HCAM_CDB_OP_CODE_LOG_DATA, hostrcb);
 }
 
@@ -4095,6 +4117,64 @@ static struct device_attribute ipr_ioa_fw_type_attr = {
 	.show = ipr_show_fw_type
 };
 
+static ssize_t ipr_read_async_err_log(struct file *filep, struct kobject *kobj,
+				struct bin_attribute *bin_attr, char *buf,
+				loff_t off, size_t count)
+{
+	struct device *cdev = container_of(kobj, struct device, kobj);
+	struct Scsi_Host *shost = class_to_shost(cdev);
+	struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *)shost->hostdata;
+	struct ipr_hostrcb *hostrcb;
+	unsigned long lock_flags = 0;
+	int ret;
+
+	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	hostrcb = list_first_entry_or_null(&ioa_cfg->hostrcb_report_q,
+					struct ipr_hostrcb, queue);
+	if (!hostrcb) {
+		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		return 0;
+	}
+	ret = memory_read_from_buffer(buf, count, &off, &hostrcb->hcam,
+				sizeof(hostrcb->hcam));
+	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	return ret;
+}
+
+static ssize_t ipr_next_async_err_log(struct file *filep, struct kobject *kobj,
+				struct bin_attribute *bin_attr, char *buf,
+				loff_t off, size_t count)
+{
+	struct device *cdev = container_of(kobj, struct device, kobj);
+	struct Scsi_Host *shost = class_to_shost(cdev);
+	struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *)shost->hostdata;
+	struct ipr_hostrcb *hostrcb;
+	unsigned long lock_flags = 0;
+
+	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	hostrcb = list_first_entry_or_null(&ioa_cfg->hostrcb_report_q,
+					struct ipr_hostrcb, queue);
+	if (!hostrcb) {
+		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		return count;
+	}
+
+	/* Reclaim hostrcb before exit */
+	list_move_tail(&hostrcb->queue, &ioa_cfg->hostrcb_free_q);
+	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	return count;
+}
+
+static struct bin_attribute ipr_ioa_async_err_log = {
+	.attr = {
+		.name =		"async_err_log",
+		.mode =		S_IRUGO | S_IWUSR,
+	},
+	.size = 0,
+	.read = ipr_read_async_err_log,
+	.write = ipr_next_async_err_log
+};
+
 static struct device_attribute *ipr_ioa_attrs[] = {
 	&ipr_fw_version_attr,
 	&ipr_log_level_attr,
@@ -7026,8 +7106,7 @@ static int ipr_ioa_reset_done(struct ipr_cmnd *ipr_cmd)
 {
 	struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg;
 	struct ipr_resource_entry *res;
-	struct ipr_hostrcb *hostrcb, *temp;
-	int i = 0, j;
+	int j;
 
 	ENTER;
 	ioa_cfg->in_reset_reload = 0;
@@ -7048,12 +7127,16 @@ static int ipr_ioa_reset_done(struct ipr_cmnd *ipr_cmd)
 	}
 	schedule_work(&ioa_cfg->work_q);
 
-	list_for_each_entry_safe(hostrcb, temp, &ioa_cfg->hostrcb_free_q, queue) {
-		list_del(&hostrcb->queue);
-		if (i++ < IPR_NUM_LOG_HCAMS)
-			ipr_send_hcam(ioa_cfg, IPR_HCAM_CDB_OP_CODE_LOG_DATA, hostrcb);
+	for (j = 0; j < IPR_NUM_HCAMS; j++) {
+		list_del_init(&ioa_cfg->hostrcb[j]->queue);
+		if (j < IPR_NUM_LOG_HCAMS)
+			ipr_send_hcam(ioa_cfg,
+				IPR_HCAM_CDB_OP_CODE_LOG_DATA,
+				ioa_cfg->hostrcb[j]);
 		else
-			ipr_send_hcam(ioa_cfg, IPR_HCAM_CDB_OP_CODE_CONFIG_CHANGE, hostrcb);
+			ipr_send_hcam(ioa_cfg,
+				IPR_HCAM_CDB_OP_CODE_CONFIG_CHANGE,
+				ioa_cfg->hostrcb[j]);
 	}
 
 	scsi_report_bus_reset(ioa_cfg->host, IPR_VSET_BUS);
@@ -7966,7 +8049,8 @@ static int ipr_ioafp_identify_hrrq(struct ipr_cmnd *ipr_cmd)
 
 	ENTER;
 	ipr_cmd->job_step = ipr_ioafp_std_inquiry;
-	dev_info(&ioa_cfg->pdev->dev, "Starting IOA initialization sequence.\n");
+	if (ioa_cfg->identify_hrrq_index == 0)
+		dev_info(&ioa_cfg->pdev->dev, "Starting IOA initialization sequence.\n");
 
 	if (ioa_cfg->identify_hrrq_index < ioa_cfg->hrrq_num) {
 		hrrq = &ioa_cfg->hrrq[ioa_cfg->identify_hrrq_index];
@@ -8335,7 +8419,7 @@ static void ipr_get_unit_check_buffer(struct ipr_ioa_cfg *ioa_cfg)
 
 	hostrcb = list_entry(ioa_cfg->hostrcb_free_q.next,
 			     struct ipr_hostrcb, queue);
-	list_del(&hostrcb->queue);
+	list_del_init(&hostrcb->queue);
 	memset(&hostrcb->hcam, 0, sizeof(hostrcb->hcam));
 
 	rc = ipr_get_ldump_data_section(ioa_cfg,
@@ -9332,7 +9416,7 @@ static void ipr_free_mem(struct ipr_ioa_cfg *ioa_cfg)
 	dma_free_coherent(&ioa_cfg->pdev->dev, ioa_cfg->cfg_table_size,
 			  ioa_cfg->u.cfg_table, ioa_cfg->cfg_table_dma);
 
-	for (i = 0; i < IPR_NUM_HCAMS; i++) {
+	for (i = 0; i < IPR_MAX_HCAMS; i++) {
 		dma_free_coherent(&ioa_cfg->pdev->dev,
 				  sizeof(struct ipr_hostrcb),
 				  ioa_cfg->hostrcb[i],
@@ -9572,7 +9656,7 @@ static int ipr_alloc_mem(struct ipr_ioa_cfg *ioa_cfg)
 	if (!ioa_cfg->u.cfg_table)
 		goto out_free_host_rrq;
 
-	for (i = 0; i < IPR_NUM_HCAMS; i++) {
+	for (i = 0; i < IPR_MAX_HCAMS; i++) {
 		ioa_cfg->hostrcb[i] = dma_alloc_coherent(&pdev->dev,
 							 sizeof(struct ipr_hostrcb),
 							 &ioa_cfg->hostrcb_dma[i],
@@ -9714,6 +9798,7 @@ static void ipr_init_ioa_cfg(struct ipr_ioa_cfg *ioa_cfg,
 
 	INIT_LIST_HEAD(&ioa_cfg->hostrcb_free_q);
 	INIT_LIST_HEAD(&ioa_cfg->hostrcb_pending_q);
+	INIT_LIST_HEAD(&ioa_cfg->hostrcb_report_q);
 	INIT_LIST_HEAD(&ioa_cfg->free_res_q);
 	INIT_LIST_HEAD(&ioa_cfg->used_res_q);
 	INIT_WORK(&ioa_cfg->work_q, ipr_worker_thread);
@@ -10352,6 +10437,8 @@ static void ipr_remove(struct pci_dev *pdev)
 			      &ipr_trace_attr);
 	ipr_remove_dump_file(&ioa_cfg->host->shost_dev.kobj,
 			     &ipr_dump_attr);
+	sysfs_remove_bin_file(&ioa_cfg->host->shost_dev.kobj,
+			&ipr_ioa_async_err_log);
 	scsi_remove_host(ioa_cfg->host);
 
 	__ipr_remove(pdev);
@@ -10400,18 +10487,36 @@ static int ipr_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id)
 		return rc;
 	}
 
+	rc = sysfs_create_bin_file(&ioa_cfg->host->shost_dev.kobj,
+			&ipr_ioa_async_err_log);
+
+	if (rc) {
+		ipr_remove_dump_file(&ioa_cfg->host->shost_dev.kobj,
+				&ipr_dump_attr);
+		ipr_remove_trace_file(&ioa_cfg->host->shost_dev.kobj,
+				&ipr_trace_attr);
+		scsi_remove_host(ioa_cfg->host);
+		__ipr_remove(pdev);
+		return rc;
+	}
+
 	rc = ipr_create_dump_file(&ioa_cfg->host->shost_dev.kobj,
 				   &ipr_dump_attr);
 
 	if (rc) {
+		sysfs_remove_bin_file(&ioa_cfg->host->shost_dev.kobj,
+				      &ipr_ioa_async_err_log);
 		ipr_remove_trace_file(&ioa_cfg->host->shost_dev.kobj,
 				      &ipr_trace_attr);
 		scsi_remove_host(ioa_cfg->host);
 		__ipr_remove(pdev);
 		return rc;
 	}
+	spin_lock_irqsave(ioa_cfg->host->host_lock, flags);
+	ioa_cfg->scan_enabled = 1;
+	schedule_work(&ioa_cfg->work_q);
+	spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags);
 
-	scsi_scan_host(ioa_cfg->host);
 	ioa_cfg->iopoll_weight = ioa_cfg->chip_cfg->iopoll_weight;
 
 	if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
@@ -10421,10 +10526,8 @@ static int ipr_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id)
 		}
 	}
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, flags);
-	ioa_cfg->scan_enabled = 1;
-	schedule_work(&ioa_cfg->work_q);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags);
+	scsi_scan_host(ioa_cfg->host);
+
 	return 0;
 }
 
diff --git a/drivers/scsi/ipr.h b/drivers/scsi/ipr.h
index cdb51960b53c..8995053d01b3 100644
--- a/drivers/scsi/ipr.h
+++ b/drivers/scsi/ipr.h
@@ -154,7 +154,9 @@
 #define IPR_DEFAULT_MAX_ERROR_DUMP			984
 #define IPR_NUM_LOG_HCAMS				2
 #define IPR_NUM_CFG_CHG_HCAMS				2
+#define IPR_NUM_HCAM_QUEUE				12
 #define IPR_NUM_HCAMS	(IPR_NUM_LOG_HCAMS + IPR_NUM_CFG_CHG_HCAMS)
+#define IPR_MAX_HCAMS	(IPR_NUM_HCAMS + IPR_NUM_HCAM_QUEUE)
 
 #define IPR_MAX_SIS64_TARGETS_PER_BUS			1024
 #define IPR_MAX_SIS64_LUNS_PER_TARGET			0xffffffff
@@ -1504,6 +1506,7 @@ struct ipr_ioa_cfg {
 	u8 log_level;
 #define IPR_MAX_LOG_LEVEL			4
 #define IPR_DEFAULT_LOG_LEVEL		2
+#define IPR_DEBUG_LOG_LEVEL		3
 
 #define IPR_NUM_TRACE_INDEX_BITS	8
 #define IPR_NUM_TRACE_ENTRIES		(1 << IPR_NUM_TRACE_INDEX_BITS)
@@ -1532,10 +1535,11 @@ struct ipr_ioa_cfg {
 
 	char ipr_hcam_label[8];
 #define IPR_HCAM_LABEL			"hcams"
-	struct ipr_hostrcb *hostrcb[IPR_NUM_HCAMS];
-	dma_addr_t hostrcb_dma[IPR_NUM_HCAMS];
+	struct ipr_hostrcb *hostrcb[IPR_MAX_HCAMS];
+	dma_addr_t hostrcb_dma[IPR_MAX_HCAMS];
 	struct list_head hostrcb_free_q;
 	struct list_head hostrcb_pending_q;
+	struct list_head hostrcb_report_q;
 
 	struct ipr_hrr_queue hrrq[IPR_MAX_HRRQ_NUM];
 	u32 hrrq_num;
diff --git a/drivers/scsi/libfc/fc_exch.c b/drivers/scsi/libfc/fc_exch.c
index e72673b0a8fb..16ca31ad5ec0 100644
--- a/drivers/scsi/libfc/fc_exch.c
+++ b/drivers/scsi/libfc/fc_exch.c
@@ -1837,7 +1837,6 @@ static void fc_exch_reset(struct fc_exch *ep)
 	int rc = 1;
 
 	spin_lock_bh(&ep->ex_lock);
-	fc_exch_abort_locked(ep, 0);
 	ep->state |= FC_EX_RST_CLEANUP;
 	fc_exch_timer_cancel(ep);
 	if (ep->esb_stat & ESB_ST_REC_QUAL)
diff --git a/drivers/scsi/libfc/fc_rport.c b/drivers/scsi/libfc/fc_rport.c
index 93f596182145..97aeaddd600d 100644
--- a/drivers/scsi/libfc/fc_rport.c
+++ b/drivers/scsi/libfc/fc_rport.c
@@ -457,6 +457,9 @@ static void fc_rport_enter_delete(struct fc_rport_priv *rdata,
  */
 static int fc_rport_logoff(struct fc_rport_priv *rdata)
 {
+	struct fc_lport *lport = rdata->local_port;
+	u32 port_id = rdata->ids.port_id;
+
 	mutex_lock(&rdata->rp_mutex);
 
 	FC_RPORT_DBG(rdata, "Remove port\n");
@@ -466,6 +469,15 @@ static int fc_rport_logoff(struct fc_rport_priv *rdata)
 		FC_RPORT_DBG(rdata, "Port in Delete state, not removing\n");
 		goto out;
 	}
+	/*
+	 * FC-LS states:
+	 * To explicitly Logout, the initiating Nx_Port shall terminate
+	 * other open Sequences that it initiated with the destination
+	 * Nx_Port prior to performing Logout.
+	 */
+	lport->tt.exch_mgr_reset(lport, 0, port_id);
+	lport->tt.exch_mgr_reset(lport, port_id, 0);
+
 	fc_rport_enter_logo(rdata);
 
 	/*
@@ -547,16 +559,24 @@ static void fc_rport_timeout(struct work_struct *work)
  */
 static void fc_rport_error(struct fc_rport_priv *rdata, struct fc_frame *fp)
 {
+	struct fc_lport *lport = rdata->local_port;
+
 	FC_RPORT_DBG(rdata, "Error %ld in state %s, retries %d\n",
 		     IS_ERR(fp) ? -PTR_ERR(fp) : 0,
 		     fc_rport_state(rdata), rdata->retries);
 
 	switch (rdata->rp_state) {
 	case RPORT_ST_FLOGI:
-	case RPORT_ST_PLOGI:
 		rdata->flags &= ~FC_RP_STARTED;
 		fc_rport_enter_delete(rdata, RPORT_EV_FAILED);
 		break;
+	case RPORT_ST_PLOGI:
+		if (lport->point_to_multipoint) {
+			rdata->flags &= ~FC_RP_STARTED;
+			fc_rport_enter_delete(rdata, RPORT_EV_FAILED);
+		} else
+			fc_rport_enter_logo(rdata);
+		break;
 	case RPORT_ST_RTV:
 		fc_rport_enter_ready(rdata);
 		break;
@@ -1877,7 +1897,7 @@ static void fc_rport_recv_prlo_req(struct fc_rport_priv *rdata,
 	spp->spp_type_ext = rspp->spp_type_ext;
 	spp->spp_flags = FC_SPP_RESP_ACK;
 
-	fc_rport_enter_delete(rdata, RPORT_EV_LOGO);
+	fc_rport_enter_prli(rdata);
 
 	fc_fill_reply_hdr(fp, rx_fp, FC_RCTL_ELS_REP, 0);
 	lport->tt.frame_send(lport, fp);
@@ -1915,7 +1935,7 @@ static void fc_rport_recv_logo_req(struct fc_lport *lport, struct fc_frame *fp)
 		FC_RPORT_DBG(rdata, "Received LOGO request while in state %s\n",
 			     fc_rport_state(rdata));
 
-		fc_rport_enter_delete(rdata, RPORT_EV_LOGO);
+		fc_rport_enter_delete(rdata, RPORT_EV_STOP);
 		mutex_unlock(&rdata->rp_mutex);
 		kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy);
 	} else
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index 63e48d4277b0..4ac03b16d17f 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -1535,7 +1535,7 @@ lpfc_fdmi_num_disc_check(struct lpfc_vport *vport)
 }
 
 /* Routines for all individual HBA attributes */
-int
+static int
 lpfc_fdmi_hba_attr_wwnn(struct lpfc_vport *vport, struct lpfc_fdmi_attr_def *ad)
 {
 	struct lpfc_fdmi_attr_entry *ae;
@@ -1551,7 +1551,7 @@ lpfc_fdmi_hba_attr_wwnn(struct lpfc_vport *vport, struct lpfc_fdmi_attr_def *ad)
 	ad->AttrType = cpu_to_be16(RHBA_NODENAME);
 	return size;
 }
-int
+static int
 lpfc_fdmi_hba_attr_manufacturer(struct lpfc_vport *vport,
 				struct lpfc_fdmi_attr_def *ad)
 {
@@ -1573,7 +1573,7 @@ lpfc_fdmi_hba_attr_manufacturer(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_sn(struct lpfc_vport *vport, struct lpfc_fdmi_attr_def *ad)
 {
 	struct lpfc_hba *phba = vport->phba;
@@ -1594,7 +1594,7 @@ lpfc_fdmi_hba_attr_sn(struct lpfc_vport *vport, struct lpfc_fdmi_attr_def *ad)
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_model(struct lpfc_vport *vport,
 			 struct lpfc_fdmi_attr_def *ad)
 {
@@ -1615,7 +1615,7 @@ lpfc_fdmi_hba_attr_model(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_description(struct lpfc_vport *vport,
 			       struct lpfc_fdmi_attr_def *ad)
 {
@@ -1637,7 +1637,7 @@ lpfc_fdmi_hba_attr_description(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_hdw_ver(struct lpfc_vport *vport,
 			   struct lpfc_fdmi_attr_def *ad)
 {
@@ -1669,7 +1669,7 @@ lpfc_fdmi_hba_attr_hdw_ver(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_drvr_ver(struct lpfc_vport *vport,
 			    struct lpfc_fdmi_attr_def *ad)
 {
@@ -1690,7 +1690,7 @@ lpfc_fdmi_hba_attr_drvr_ver(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_rom_ver(struct lpfc_vport *vport,
 			   struct lpfc_fdmi_attr_def *ad)
 {
@@ -1715,7 +1715,7 @@ lpfc_fdmi_hba_attr_rom_ver(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_fmw_ver(struct lpfc_vport *vport,
 			   struct lpfc_fdmi_attr_def *ad)
 {
@@ -1736,7 +1736,7 @@ lpfc_fdmi_hba_attr_fmw_ver(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_os_ver(struct lpfc_vport *vport,
 			  struct lpfc_fdmi_attr_def *ad)
 {
@@ -1759,7 +1759,7 @@ lpfc_fdmi_hba_attr_os_ver(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_ct_len(struct lpfc_vport *vport,
 			  struct lpfc_fdmi_attr_def *ad)
 {
@@ -1775,7 +1775,7 @@ lpfc_fdmi_hba_attr_ct_len(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_symbolic_name(struct lpfc_vport *vport,
 				 struct lpfc_fdmi_attr_def *ad)
 {
@@ -1794,7 +1794,7 @@ lpfc_fdmi_hba_attr_symbolic_name(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_vendor_info(struct lpfc_vport *vport,
 			       struct lpfc_fdmi_attr_def *ad)
 {
@@ -1811,7 +1811,7 @@ lpfc_fdmi_hba_attr_vendor_info(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_num_ports(struct lpfc_vport *vport,
 			     struct lpfc_fdmi_attr_def *ad)
 {
@@ -1828,7 +1828,7 @@ lpfc_fdmi_hba_attr_num_ports(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_fabric_wwnn(struct lpfc_vport *vport,
 			       struct lpfc_fdmi_attr_def *ad)
 {
@@ -1846,7 +1846,7 @@ lpfc_fdmi_hba_attr_fabric_wwnn(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_bios_ver(struct lpfc_vport *vport,
 			    struct lpfc_fdmi_attr_def *ad)
 {
@@ -1867,7 +1867,7 @@ lpfc_fdmi_hba_attr_bios_ver(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_bios_state(struct lpfc_vport *vport,
 			      struct lpfc_fdmi_attr_def *ad)
 {
@@ -1884,7 +1884,7 @@ lpfc_fdmi_hba_attr_bios_state(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_hba_attr_vendor_id(struct lpfc_vport *vport,
 			     struct lpfc_fdmi_attr_def *ad)
 {
@@ -1906,7 +1906,7 @@ lpfc_fdmi_hba_attr_vendor_id(struct lpfc_vport *vport,
 }
 
 /* Routines for all individual PORT attributes */
-int
+static int
 lpfc_fdmi_port_attr_fc4type(struct lpfc_vport *vport,
 			    struct lpfc_fdmi_attr_def *ad)
 {
@@ -1925,7 +1925,7 @@ lpfc_fdmi_port_attr_fc4type(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_support_speed(struct lpfc_vport *vport,
 				  struct lpfc_fdmi_attr_def *ad)
 {
@@ -1975,7 +1975,7 @@ lpfc_fdmi_port_attr_support_speed(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_speed(struct lpfc_vport *vport,
 			  struct lpfc_fdmi_attr_def *ad)
 {
@@ -2039,7 +2039,7 @@ lpfc_fdmi_port_attr_speed(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_max_frame(struct lpfc_vport *vport,
 			      struct lpfc_fdmi_attr_def *ad)
 {
@@ -2059,7 +2059,7 @@ lpfc_fdmi_port_attr_max_frame(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_os_devname(struct lpfc_vport *vport,
 			       struct lpfc_fdmi_attr_def *ad)
 {
@@ -2081,7 +2081,7 @@ lpfc_fdmi_port_attr_os_devname(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_host_name(struct lpfc_vport *vport,
 			      struct lpfc_fdmi_attr_def *ad)
 {
@@ -2102,7 +2102,7 @@ lpfc_fdmi_port_attr_host_name(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_wwnn(struct lpfc_vport *vport,
 			 struct lpfc_fdmi_attr_def *ad)
 {
@@ -2120,7 +2120,7 @@ lpfc_fdmi_port_attr_wwnn(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_wwpn(struct lpfc_vport *vport,
 			 struct lpfc_fdmi_attr_def *ad)
 {
@@ -2138,7 +2138,7 @@ lpfc_fdmi_port_attr_wwpn(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_symbolic_name(struct lpfc_vport *vport,
 				  struct lpfc_fdmi_attr_def *ad)
 {
@@ -2156,7 +2156,7 @@ lpfc_fdmi_port_attr_symbolic_name(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_port_type(struct lpfc_vport *vport,
 			      struct lpfc_fdmi_attr_def *ad)
 {
@@ -2175,7 +2175,7 @@ lpfc_fdmi_port_attr_port_type(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_class(struct lpfc_vport *vport,
 			  struct lpfc_fdmi_attr_def *ad)
 {
@@ -2190,7 +2190,7 @@ lpfc_fdmi_port_attr_class(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_fabric_wwpn(struct lpfc_vport *vport,
 				struct lpfc_fdmi_attr_def *ad)
 {
@@ -2208,7 +2208,7 @@ lpfc_fdmi_port_attr_fabric_wwpn(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_active_fc4type(struct lpfc_vport *vport,
 				   struct lpfc_fdmi_attr_def *ad)
 {
@@ -2227,7 +2227,7 @@ lpfc_fdmi_port_attr_active_fc4type(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_port_state(struct lpfc_vport *vport,
 			       struct lpfc_fdmi_attr_def *ad)
 {
@@ -2243,7 +2243,7 @@ lpfc_fdmi_port_attr_port_state(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_num_disc(struct lpfc_vport *vport,
 			     struct lpfc_fdmi_attr_def *ad)
 {
@@ -2259,7 +2259,7 @@ lpfc_fdmi_port_attr_num_disc(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_port_attr_nportid(struct lpfc_vport *vport,
 			    struct lpfc_fdmi_attr_def *ad)
 {
@@ -2274,7 +2274,7 @@ lpfc_fdmi_port_attr_nportid(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_smart_attr_service(struct lpfc_vport *vport,
 			     struct lpfc_fdmi_attr_def *ad)
 {
@@ -2295,7 +2295,7 @@ lpfc_fdmi_smart_attr_service(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_smart_attr_guid(struct lpfc_vport *vport,
 			  struct lpfc_fdmi_attr_def *ad)
 {
@@ -2316,7 +2316,7 @@ lpfc_fdmi_smart_attr_guid(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_smart_attr_version(struct lpfc_vport *vport,
 			     struct lpfc_fdmi_attr_def *ad)
 {
@@ -2337,7 +2337,7 @@ lpfc_fdmi_smart_attr_version(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_smart_attr_model(struct lpfc_vport *vport,
 			   struct lpfc_fdmi_attr_def *ad)
 {
@@ -2358,7 +2358,7 @@ lpfc_fdmi_smart_attr_model(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_smart_attr_port_info(struct lpfc_vport *vport,
 			       struct lpfc_fdmi_attr_def *ad)
 {
@@ -2378,7 +2378,7 @@ lpfc_fdmi_smart_attr_port_info(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_smart_attr_qos(struct lpfc_vport *vport,
 			 struct lpfc_fdmi_attr_def *ad)
 {
@@ -2393,7 +2393,7 @@ lpfc_fdmi_smart_attr_qos(struct lpfc_vport *vport,
 	return size;
 }
 
-int
+static int
 lpfc_fdmi_smart_attr_security(struct lpfc_vport *vport,
 			      struct lpfc_fdmi_attr_def *ad)
 {
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index c0af32f24954..b7d54bfb1df9 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -4617,7 +4617,7 @@ lpfc_els_disc_plogi(struct lpfc_vport *vport)
 	return sentplogi;
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_link_service(struct fc_rdp_link_service_desc *desc,
 		uint32_t word0)
 {
@@ -4629,7 +4629,7 @@ lpfc_rdp_res_link_service(struct fc_rdp_link_service_desc *desc,
 	return sizeof(struct fc_rdp_link_service_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_sfp_desc(struct fc_rdp_sfp_desc *desc,
 		uint8_t *page_a0, uint8_t *page_a2)
 {
@@ -4694,7 +4694,7 @@ lpfc_rdp_res_sfp_desc(struct fc_rdp_sfp_desc *desc,
 	return sizeof(struct fc_rdp_sfp_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_link_error(struct fc_rdp_link_error_status_desc *desc,
 		READ_LNK_VAR *stat)
 {
@@ -4723,7 +4723,7 @@ lpfc_rdp_res_link_error(struct fc_rdp_link_error_status_desc *desc,
 	return sizeof(struct fc_rdp_link_error_status_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_bbc_desc(struct fc_rdp_bbc_desc *desc, READ_LNK_VAR *stat,
 		      struct lpfc_vport *vport)
 {
@@ -4748,7 +4748,7 @@ lpfc_rdp_res_bbc_desc(struct fc_rdp_bbc_desc *desc, READ_LNK_VAR *stat,
 	return sizeof(struct fc_rdp_bbc_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_oed_temp_desc(struct lpfc_hba *phba,
 			   struct fc_rdp_oed_sfp_desc *desc, uint8_t *page_a2)
 {
@@ -4776,7 +4776,7 @@ lpfc_rdp_res_oed_temp_desc(struct lpfc_hba *phba,
 	return sizeof(struct fc_rdp_oed_sfp_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_oed_voltage_desc(struct lpfc_hba *phba,
 			      struct fc_rdp_oed_sfp_desc *desc,
 			      uint8_t *page_a2)
@@ -4805,7 +4805,7 @@ lpfc_rdp_res_oed_voltage_desc(struct lpfc_hba *phba,
 	return sizeof(struct fc_rdp_oed_sfp_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_oed_txbias_desc(struct lpfc_hba *phba,
 			     struct fc_rdp_oed_sfp_desc *desc,
 			     uint8_t *page_a2)
@@ -4834,7 +4834,7 @@ lpfc_rdp_res_oed_txbias_desc(struct lpfc_hba *phba,
 	return sizeof(struct fc_rdp_oed_sfp_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_oed_txpower_desc(struct lpfc_hba *phba,
 			      struct fc_rdp_oed_sfp_desc *desc,
 			      uint8_t *page_a2)
@@ -4864,7 +4864,7 @@ lpfc_rdp_res_oed_txpower_desc(struct lpfc_hba *phba,
 }
 
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_oed_rxpower_desc(struct lpfc_hba *phba,
 			      struct fc_rdp_oed_sfp_desc *desc,
 			      uint8_t *page_a2)
@@ -4893,7 +4893,7 @@ lpfc_rdp_res_oed_rxpower_desc(struct lpfc_hba *phba,
 	return sizeof(struct fc_rdp_oed_sfp_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_opd_desc(struct fc_rdp_opd_sfp_desc *desc,
 		      uint8_t *page_a0, struct lpfc_vport *vport)
 {
@@ -4907,7 +4907,7 @@ lpfc_rdp_res_opd_desc(struct fc_rdp_opd_sfp_desc *desc,
 	return sizeof(struct fc_rdp_opd_sfp_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_fec_desc(struct fc_fec_rdp_desc *desc, READ_LNK_VAR *stat)
 {
 	if (bf_get(lpfc_read_link_stat_gec2, stat) == 0)
@@ -4924,7 +4924,7 @@ lpfc_rdp_res_fec_desc(struct fc_fec_rdp_desc *desc, READ_LNK_VAR *stat)
 	return sizeof(struct fc_fec_rdp_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_speed(struct fc_rdp_port_speed_desc *desc, struct lpfc_hba *phba)
 {
 	uint16_t rdp_cap = 0;
@@ -4986,7 +4986,7 @@ lpfc_rdp_res_speed(struct fc_rdp_port_speed_desc *desc, struct lpfc_hba *phba)
 	return sizeof(struct fc_rdp_port_speed_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_diag_port_names(struct fc_rdp_port_name_desc *desc,
 		struct lpfc_hba *phba)
 {
@@ -5003,7 +5003,7 @@ lpfc_rdp_res_diag_port_names(struct fc_rdp_port_name_desc *desc,
 	return sizeof(struct fc_rdp_port_name_desc);
 }
 
-uint32_t
+static uint32_t
 lpfc_rdp_res_attach_port_names(struct fc_rdp_port_name_desc *desc,
 		struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 {
@@ -5027,7 +5027,7 @@ lpfc_rdp_res_attach_port_names(struct fc_rdp_port_name_desc *desc,
 	return sizeof(struct fc_rdp_port_name_desc);
 }
 
-void
+static void
 lpfc_els_rdp_cmpl(struct lpfc_hba *phba, struct lpfc_rdp_context *rdp_context,
 		int status)
 {
@@ -5165,7 +5165,7 @@ free_rdp_context:
 	kfree(rdp_context);
 }
 
-int
+static int
 lpfc_get_rdp_info(struct lpfc_hba *phba, struct lpfc_rdp_context *rdp_context)
 {
 	LPFC_MBOXQ_t *mbox = NULL;
@@ -7995,7 +7995,7 @@ lpfc_els_unsol_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	}
 }
 
-void
+static void
 lpfc_start_fdmi(struct lpfc_vport *vport)
 {
 	struct lpfc_hba *phba = vport->phba;
diff --git a/drivers/scsi/lpfc/lpfc_mbox.c b/drivers/scsi/lpfc/lpfc_mbox.c
index 12dbe99ccc50..b234c50c255f 100644
--- a/drivers/scsi/lpfc/lpfc_mbox.c
+++ b/drivers/scsi/lpfc/lpfc_mbox.c
@@ -2260,7 +2260,7 @@ lpfc_sli4_dump_cfg_rg23(struct lpfc_hba *phba, struct lpfcMboxq *mbox)
 	return 0;
 }
 
-void
+static void
 lpfc_mbx_cmpl_rdp_link_stat(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 {
 	MAILBOX_t *mb;
@@ -2281,7 +2281,7 @@ mbx_failed:
 	rdp_context->cmpl(phba, rdp_context, rc);
 }
 
-void
+static void
 lpfc_mbx_cmpl_rdp_page_a2(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox)
 {
 	struct lpfc_dmabuf *mp = (struct lpfc_dmabuf *) mbox->context1;
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index 7080ce2920fd..c5326055beee 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -5689,7 +5689,7 @@ lpfc_sli4_dealloc_extent(struct lpfc_hba *phba, uint16_t type)
 	return rc;
 }
 
-void
+static void
 lpfc_set_features(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox,
 		  uint32_t feature)
 {
@@ -8968,7 +8968,7 @@ lpfc_sli_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp)
  * Since ABORTS must go on the same WQ of the command they are
  * aborting, we use command's fcp_wqidx.
  */
-int
+static int
 lpfc_sli_calc_ring(struct lpfc_hba *phba, uint32_t ring_number,
 		    struct lpfc_iocbq *piocb)
 {
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index 2dab3dc2aa69..9ff57dee72d7 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -189,25 +189,12 @@ u32
 megasas_build_and_issue_cmd(struct megasas_instance *instance,
 			    struct scsi_cmnd *scmd);
 static void megasas_complete_cmd_dpc(unsigned long instance_addr);
-void
-megasas_release_fusion(struct megasas_instance *instance);
-int
-megasas_ioc_init_fusion(struct megasas_instance *instance);
-void
-megasas_free_cmds_fusion(struct megasas_instance *instance);
-u8
-megasas_get_map_info(struct megasas_instance *instance);
-int
-megasas_sync_map_info(struct megasas_instance *instance);
 int
 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
 	int seconds);
-void megasas_reset_reply_desc(struct megasas_instance *instance);
 void megasas_fusion_ocr_wq(struct work_struct *work);
 static int megasas_get_ld_vf_affiliation(struct megasas_instance *instance,
 					 int initial);
-int megasas_check_mpio_paths(struct megasas_instance *instance,
-			     struct scsi_cmnd *scmd);
 
 int
 megasas_issue_dcmd(struct megasas_instance *instance, struct megasas_cmd *cmd)
@@ -5036,8 +5023,8 @@ static int megasas_init_fw(struct megasas_instance *instance)
 
 	/* Find first memory bar */
 	bar_list = pci_select_bars(instance->pdev, IORESOURCE_MEM);
-	instance->bar = find_first_bit(&bar_list, sizeof(unsigned long));
-	if (pci_request_selected_regions(instance->pdev, instance->bar,
+	instance->bar = find_first_bit(&bar_list, BITS_PER_LONG);
+	if (pci_request_selected_regions(instance->pdev, 1<<instance->bar,
 					 "megasas: LSI")) {
 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "IO memory region busy!\n");
 		return -EBUSY;
@@ -5339,7 +5326,7 @@ fail_ready_state:
 	iounmap(instance->reg_set);
 
       fail_ioremap:
-	pci_release_selected_regions(instance->pdev, instance->bar);
+	pci_release_selected_regions(instance->pdev, 1<<instance->bar);
 
 	return -EINVAL;
 }
@@ -5360,7 +5347,7 @@ static void megasas_release_mfi(struct megasas_instance *instance)
 
 	iounmap(instance->reg_set);
 
-	pci_release_selected_regions(instance->pdev, instance->bar);
+	pci_release_selected_regions(instance->pdev, 1<<instance->bar);
 }
 
 /**
@@ -5782,7 +5769,7 @@ static int megasas_probe_one(struct pci_dev *pdev,
 					     &instance->consumer_h);
 
 		if (!instance->producer || !instance->consumer) {
-			dev_printk(KERN_DEBUG, &pdev->dev, "Failed to allocate"
+			dev_printk(KERN_DEBUG, &pdev->dev, "Failed to allocate "
 			       "memory for producer, consumer\n");
 			goto fail_alloc_dma_buf;
 		}
@@ -6711,14 +6698,9 @@ static int megasas_mgmt_ioctl_fw(struct file *file, unsigned long arg)
 	unsigned long flags;
 	u32 wait_time = MEGASAS_RESET_WAIT_TIME;
 
-	ioc = kmalloc(sizeof(*ioc), GFP_KERNEL);
-	if (!ioc)
-		return -ENOMEM;
-
-	if (copy_from_user(ioc, user_ioc, sizeof(*ioc))) {
-		error = -EFAULT;
-		goto out_kfree_ioc;
-	}
+	ioc = memdup_user(user_ioc, sizeof(*ioc));
+	if (IS_ERR(ioc))
+		return PTR_ERR(ioc);
 
 	instance = megasas_lookup_instance(ioc->host_no);
 	if (!instance) {
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index ec837544f784..52d8bbf7feb5 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -2603,7 +2603,7 @@ megasas_release_fusion(struct megasas_instance *instance)
 
 	iounmap(instance->reg_set);
 
-	pci_release_selected_regions(instance->pdev, instance->bar);
+	pci_release_selected_regions(instance->pdev, 1<<instance->bar);
 }
 
 /**
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.h b/drivers/scsi/megaraid/megaraid_sas_fusion.h
index 80eaee22f5bc..e3bee04c1eb1 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.h
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.h
@@ -991,5 +991,14 @@ union desc_value {
 	} u;
 };
 
+void megasas_free_cmds_fusion(struct megasas_instance *instance);
+int megasas_ioc_init_fusion(struct megasas_instance *instance);
+u8 megasas_get_map_info(struct megasas_instance *instance);
+int megasas_sync_map_info(struct megasas_instance *instance);
+void megasas_release_fusion(struct megasas_instance *instance);
+void megasas_reset_reply_desc(struct megasas_instance *instance);
+int megasas_check_mpio_paths(struct megasas_instance *instance,
+			      struct scsi_cmnd *scmd);
+void megasas_fusion_ocr_wq(struct work_struct *work);
 
 #endif /* _MEGARAID_SAS_FUSION_H_ */
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index 751f13edece0..a1a5ceb42ce6 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -98,7 +98,7 @@ MODULE_PARM_DESC(mpt3sas_fwfault_debug,
 	" enable detection of firmware fault and halt firmware - (default=0)");
 
 static int
-_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc, int sleep_flag);
+_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc);
 
 /**
  * _scsih_set_fwfault_debug - global setting of ioc->fwfault_debug.
@@ -218,8 +218,7 @@ _base_fault_reset_work(struct work_struct *work)
 	ioc->non_operational_loop = 0;
 
 	if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) {
-		rc = mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 		pr_warn(MPT3SAS_FMT "%s: hard reset: %s\n", ioc->name,
 		    __func__, (rc == 0) ? "success" : "failed");
 		doorbell = mpt3sas_base_get_iocstate(ioc, 0);
@@ -2040,7 +2039,7 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
  * mpt3sas_base_unmap_resources - free controller resources
  * @ioc: per adapter object
  */
-void
+static void
 mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)
 {
 	struct pci_dev *pdev = ioc->pdev;
@@ -2145,7 +2144,7 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
 
 	_base_mask_interrupts(ioc);
 
-	r = _base_get_ioc_facts(ioc, CAN_SLEEP);
+	r = _base_get_ioc_facts(ioc);
 	if (r)
 		goto out_fail;
 
@@ -2188,6 +2187,17 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
 	} else
 		ioc->msix96_vector = 0;
 
+	if (ioc->is_warpdrive) {
+		ioc->reply_post_host_index[0] = (resource_size_t __iomem *)
+		    &ioc->chip->ReplyPostHostIndex;
+
+		for (i = 1; i < ioc->cpu_msix_table_sz; i++)
+			ioc->reply_post_host_index[i] =
+			(resource_size_t __iomem *)
+			((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1)
+			* 4)));
+	}
+
 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list)
 		pr_info(MPT3SAS_FMT "%s: IRQ %d\n",
 		    reply_q->name,  ((ioc->msix_enable) ? "PCI-MSI-X enabled" :
@@ -3172,12 +3182,11 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 /**
  * _base_allocate_memory_pools - allocate start of day memory pools
  * @ioc: per adapter object
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 success, anything else error
  */
 static int
-_base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc,  int sleep_flag)
+_base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 {
 	struct mpt3sas_facts *facts;
 	u16 max_sge_elements;
@@ -3647,29 +3656,25 @@ mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked)
  * _base_wait_on_iocstate - waiting on a particular ioc state
  * @ioc_state: controller state { READY, OPERATIONAL, or RESET }
  * @timeout: timeout in second
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout,
-	int sleep_flag)
+_base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout)
 {
 	u32 count, cntdn;
 	u32 current_state;
 
 	count = 0;
-	cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
+	cntdn = 1000 * timeout;
 	do {
 		current_state = mpt3sas_base_get_iocstate(ioc, 1);
 		if (current_state == ioc_state)
 			return 0;
 		if (count && current_state == MPI2_IOC_STATE_FAULT)
 			break;
-		if (sleep_flag == CAN_SLEEP)
-			usleep_range(1000, 1500);
-		else
-			udelay(500);
+
+		usleep_range(1000, 1500);
 		count++;
 	} while (--cntdn);
 
@@ -3681,24 +3686,22 @@ _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout,
  * a write to the doorbell)
  * @ioc: per adapter object
  * @timeout: timeout in second
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  *
  * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell.
  */
 static int
-_base_diag_reset(struct MPT3SAS_ADAPTER *ioc, int sleep_flag);
+_base_diag_reset(struct MPT3SAS_ADAPTER *ioc);
 
 static int
-_base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout,
-	int sleep_flag)
+_base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
 {
 	u32 cntdn, count;
 	u32 int_status;
 
 	count = 0;
-	cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
+	cntdn = 1000 * timeout;
 	do {
 		int_status = readl(&ioc->chip->HostInterruptStatus);
 		if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
@@ -3707,10 +3710,35 @@ _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout,
 				ioc->name, __func__, count, timeout));
 			return 0;
 		}
-		if (sleep_flag == CAN_SLEEP)
-			usleep_range(1000, 1500);
-		else
-			udelay(500);
+
+		usleep_range(1000, 1500);
+		count++;
+	} while (--cntdn);
+
+	pr_err(MPT3SAS_FMT
+		"%s: failed due to timeout count(%d), int_status(%x)!\n",
+		ioc->name, __func__, count, int_status);
+	return -EFAULT;
+}
+
+static int
+_base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
+{
+	u32 cntdn, count;
+	u32 int_status;
+
+	count = 0;
+	cntdn = 2000 * timeout;
+	do {
+		int_status = readl(&ioc->chip->HostInterruptStatus);
+		if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
+			dhsprintk(ioc, pr_info(MPT3SAS_FMT
+				"%s: successful count(%d), timeout(%d)\n",
+				ioc->name, __func__, count, timeout));
+			return 0;
+		}
+
+		udelay(500);
 		count++;
 	} while (--cntdn);
 
@@ -3718,13 +3746,13 @@ _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout,
 		"%s: failed due to timeout count(%d), int_status(%x)!\n",
 		ioc->name, __func__, count, int_status);
 	return -EFAULT;
+
 }
 
 /**
  * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell.
  * @ioc: per adapter object
  * @timeout: timeout in second
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  *
@@ -3732,15 +3760,14 @@ _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout,
  * doorbell.
  */
 static int
-_base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout,
-	int sleep_flag)
+_base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout)
 {
 	u32 cntdn, count;
 	u32 int_status;
 	u32 doorbell;
 
 	count = 0;
-	cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
+	cntdn = 1000 * timeout;
 	do {
 		int_status = readl(&ioc->chip->HostInterruptStatus);
 		if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
@@ -3758,10 +3785,7 @@ _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout,
 		} else if (int_status == 0xFFFFFFFF)
 			goto out;
 
-		if (sleep_flag == CAN_SLEEP)
-			usleep_range(1000, 1500);
-		else
-			udelay(500);
+		usleep_range(1000, 1500);
 		count++;
 	} while (--cntdn);
 
@@ -3776,20 +3800,18 @@ _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout,
  * _base_wait_for_doorbell_not_used - waiting for doorbell to not be in use
  * @ioc: per adapter object
  * @timeout: timeout in second
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  *
  */
 static int
-_base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout,
-	int sleep_flag)
+_base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout)
 {
 	u32 cntdn, count;
 	u32 doorbell_reg;
 
 	count = 0;
-	cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout;
+	cntdn = 1000 * timeout;
 	do {
 		doorbell_reg = readl(&ioc->chip->Doorbell);
 		if (!(doorbell_reg & MPI2_DOORBELL_USED)) {
@@ -3798,10 +3820,8 @@ _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout,
 				ioc->name, __func__, count, timeout));
 			return 0;
 		}
-		if (sleep_flag == CAN_SLEEP)
-			usleep_range(1000, 1500);
-		else
-			udelay(500);
+
+		usleep_range(1000, 1500);
 		count++;
 	} while (--cntdn);
 
@@ -3816,13 +3836,11 @@ _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout,
  * @ioc: per adapter object
  * @reset_type: currently only supports: MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET
  * @timeout: timeout in second
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout,
-	int sleep_flag)
+_base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout)
 {
 	u32 ioc_state;
 	int r = 0;
@@ -3841,12 +3859,11 @@ _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout,
 
 	writel(reset_type << MPI2_DOORBELL_FUNCTION_SHIFT,
 	    &ioc->chip->Doorbell);
-	if ((_base_wait_for_doorbell_ack(ioc, 15, sleep_flag))) {
+	if ((_base_wait_for_doorbell_ack(ioc, 15))) {
 		r = -EFAULT;
 		goto out;
 	}
-	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY,
-	    timeout, sleep_flag);
+	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
 	if (ioc_state) {
 		pr_err(MPT3SAS_FMT
 			"%s: failed going to ready state (ioc_state=0x%x)\n",
@@ -3868,18 +3885,16 @@ _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout,
  * @reply_bytes: reply length
  * @reply: pointer to reply payload
  * @timeout: timeout in second
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
 _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
-	u32 *request, int reply_bytes, u16 *reply, int timeout, int sleep_flag)
+	u32 *request, int reply_bytes, u16 *reply, int timeout)
 {
 	MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply;
 	int i;
 	u8 failed;
-	u16 dummy;
 	__le32 *mfp;
 
 	/* make sure doorbell is not in use */
@@ -3900,7 +3915,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 	    ((request_bytes/4)<<MPI2_DOORBELL_ADD_DWORDS_SHIFT)),
 	    &ioc->chip->Doorbell);
 
-	if ((_base_wait_for_doorbell_int(ioc, 5, NO_SLEEP))) {
+	if ((_base_spin_on_doorbell_int(ioc, 5))) {
 		pr_err(MPT3SAS_FMT
 			"doorbell handshake int failed (line=%d)\n",
 			ioc->name, __LINE__);
@@ -3908,7 +3923,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 	}
 	writel(0, &ioc->chip->HostInterruptStatus);
 
-	if ((_base_wait_for_doorbell_ack(ioc, 5, sleep_flag))) {
+	if ((_base_wait_for_doorbell_ack(ioc, 5))) {
 		pr_err(MPT3SAS_FMT
 			"doorbell handshake ack failed (line=%d)\n",
 			ioc->name, __LINE__);
@@ -3918,7 +3933,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 	/* send message 32-bits at a time */
 	for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) {
 		writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);
-		if ((_base_wait_for_doorbell_ack(ioc, 5, sleep_flag)))
+		if ((_base_wait_for_doorbell_ack(ioc, 5)))
 			failed = 1;
 	}
 
@@ -3930,7 +3945,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 	}
 
 	/* now wait for the reply */
-	if ((_base_wait_for_doorbell_int(ioc, timeout, sleep_flag))) {
+	if ((_base_wait_for_doorbell_int(ioc, timeout))) {
 		pr_err(MPT3SAS_FMT
 			"doorbell handshake int failed (line=%d)\n",
 			ioc->name, __LINE__);
@@ -3941,7 +3956,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 	reply[0] = le16_to_cpu(readl(&ioc->chip->Doorbell)
 	    & MPI2_DOORBELL_DATA_MASK);
 	writel(0, &ioc->chip->HostInterruptStatus);
-	if ((_base_wait_for_doorbell_int(ioc, 5, sleep_flag))) {
+	if ((_base_wait_for_doorbell_int(ioc, 5))) {
 		pr_err(MPT3SAS_FMT
 			"doorbell handshake int failed (line=%d)\n",
 			ioc->name, __LINE__);
@@ -3952,22 +3967,22 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
 	writel(0, &ioc->chip->HostInterruptStatus);
 
 	for (i = 2; i < default_reply->MsgLength * 2; i++)  {
-		if ((_base_wait_for_doorbell_int(ioc, 5, sleep_flag))) {
+		if ((_base_wait_for_doorbell_int(ioc, 5))) {
 			pr_err(MPT3SAS_FMT
 				"doorbell handshake int failed (line=%d)\n",
 				ioc->name, __LINE__);
 			return -EFAULT;
 		}
 		if (i >=  reply_bytes/2) /* overflow case */
-			dummy = readl(&ioc->chip->Doorbell);
+			readl(&ioc->chip->Doorbell);
 		else
 			reply[i] = le16_to_cpu(readl(&ioc->chip->Doorbell)
 			    & MPI2_DOORBELL_DATA_MASK);
 		writel(0, &ioc->chip->HostInterruptStatus);
 	}
 
-	_base_wait_for_doorbell_int(ioc, 5, sleep_flag);
-	if (_base_wait_for_doorbell_not_used(ioc, 5, sleep_flag) != 0) {
+	_base_wait_for_doorbell_int(ioc, 5);
+	if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) {
 		dhsprintk(ioc, pr_info(MPT3SAS_FMT
 			"doorbell is in use (line=%d)\n", ioc->name, __LINE__));
 	}
@@ -4004,7 +4019,6 @@ mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
 {
 	u16 smid;
 	u32 ioc_state;
-	unsigned long timeleft;
 	bool issue_reset = false;
 	int rc;
 	void *request;
@@ -4057,7 +4071,7 @@ mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
 		ioc->ioc_link_reset_in_progress = 1;
 	init_completion(&ioc->base_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->base_cmds.done,
+	wait_for_completion_timeout(&ioc->base_cmds.done,
 	    msecs_to_jiffies(10000));
 	if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
 	    mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) &&
@@ -4082,8 +4096,7 @@ mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
 
  issue_host_reset:
 	if (issue_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
 	rc = -EFAULT;
  out:
@@ -4108,7 +4121,6 @@ mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
 {
 	u16 smid;
 	u32 ioc_state;
-	unsigned long timeleft;
 	bool issue_reset = false;
 	int rc;
 	void *request;
@@ -4159,7 +4171,7 @@ mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
 	memcpy(request, mpi_request, sizeof(Mpi2SepReply_t));
 	init_completion(&ioc->base_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->base_cmds.done,
+	wait_for_completion_timeout(&ioc->base_cmds.done,
 	    msecs_to_jiffies(10000));
 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
@@ -4180,8 +4192,7 @@ mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
 
  issue_host_reset:
 	if (issue_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
 	rc = -EFAULT;
  out:
@@ -4192,12 +4203,11 @@ mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
 /**
  * _base_get_port_facts - obtain port facts reply and save in ioc
  * @ioc: per adapter object
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port, int sleep_flag)
+_base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port)
 {
 	Mpi2PortFactsRequest_t mpi_request;
 	Mpi2PortFactsReply_t mpi_reply;
@@ -4213,7 +4223,7 @@ _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port, int sleep_flag)
 	mpi_request.Function = MPI2_FUNCTION_PORT_FACTS;
 	mpi_request.PortNumber = port;
 	r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
-	    (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5, CAN_SLEEP);
+	    (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
 
 	if (r != 0) {
 		pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
@@ -4236,13 +4246,11 @@ _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port, int sleep_flag)
  * _base_wait_for_iocstate - Wait until the card is in READY or OPERATIONAL
  * @ioc: per adapter object
  * @timeout:
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout,
-	int sleep_flag)
+_base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout)
 {
 	u32 ioc_state;
 	int rc;
@@ -4276,8 +4284,7 @@ _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout,
 		goto issue_diag_reset;
 	}
 
-	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY,
-	    timeout, sleep_flag);
+	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
 	if (ioc_state) {
 		dfailprintk(ioc, printk(MPT3SAS_FMT
 		    "%s: failed going to ready state (ioc_state=0x%x)\n",
@@ -4286,19 +4293,18 @@ _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout,
 	}
 
  issue_diag_reset:
-	rc = _base_diag_reset(ioc, sleep_flag);
+	rc = _base_diag_reset(ioc);
 	return rc;
 }
 
 /**
  * _base_get_ioc_facts - obtain ioc facts reply and save in ioc
  * @ioc: per adapter object
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
+_base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc)
 {
 	Mpi2IOCFactsRequest_t mpi_request;
 	Mpi2IOCFactsReply_t mpi_reply;
@@ -4308,7 +4314,7 @@ _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 	dinitprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
 	    __func__));
 
-	r = _base_wait_for_iocstate(ioc, 10, sleep_flag);
+	r = _base_wait_for_iocstate(ioc, 10);
 	if (r) {
 		dfailprintk(ioc, printk(MPT3SAS_FMT
 		    "%s: failed getting to correct state\n",
@@ -4320,7 +4326,7 @@ _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 	memset(&mpi_request, 0, mpi_request_sz);
 	mpi_request.Function = MPI2_FUNCTION_IOC_FACTS;
 	r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
-	    (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5, CAN_SLEEP);
+	    (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
 
 	if (r != 0) {
 		pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
@@ -4380,12 +4386,11 @@ _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 /**
  * _base_send_ioc_init - send ioc_init to firmware
  * @ioc: per adapter object
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
+_base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
 {
 	Mpi2IOCInitRequest_t mpi_request;
 	Mpi2IOCInitReply_t mpi_reply;
@@ -4468,8 +4473,7 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 
 	r = _base_handshake_req_reply_wait(ioc,
 	    sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request,
-	    sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 10,
-	    sleep_flag);
+	    sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 10);
 
 	if (r != 0) {
 		pr_err(MPT3SAS_FMT "%s: handshake failed (r=%d)\n",
@@ -4544,16 +4548,14 @@ mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
 /**
  * _base_send_port_enable - send port_enable(discovery stuff) to firmware
  * @ioc: per adapter object
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_send_port_enable(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
+_base_send_port_enable(struct MPT3SAS_ADAPTER *ioc)
 {
 	Mpi2PortEnableRequest_t *mpi_request;
 	Mpi2PortEnableReply_t *mpi_reply;
-	unsigned long timeleft;
 	int r = 0;
 	u16 smid;
 	u16 ioc_status;
@@ -4581,8 +4583,7 @@ _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 
 	init_completion(&ioc->port_enable_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->port_enable_cmds.done,
-	    300*HZ);
+	wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ);
 	if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
 		    ioc->name, __func__);
@@ -4726,15 +4727,13 @@ _base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event)
 /**
  * _base_event_notification - send event notification
  * @ioc: per adapter object
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_event_notification(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
+_base_event_notification(struct MPT3SAS_ADAPTER *ioc)
 {
 	Mpi2EventNotificationRequest_t *mpi_request;
-	unsigned long timeleft;
 	u16 smid;
 	int r = 0;
 	int i;
@@ -4766,7 +4765,7 @@ _base_event_notification(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 		    cpu_to_le32(ioc->event_masks[i]);
 	init_completion(&ioc->base_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);
+	wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);
 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
 		    ioc->name, __func__);
@@ -4816,19 +4815,18 @@ mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type)
 		return;
 
 	mutex_lock(&ioc->base_cmds.mutex);
-	_base_event_notification(ioc, CAN_SLEEP);
+	_base_event_notification(ioc);
 	mutex_unlock(&ioc->base_cmds.mutex);
 }
 
 /**
  * _base_diag_reset - the "big hammer" start of day reset
  * @ioc: per adapter object
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_diag_reset(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
+_base_diag_reset(struct MPT3SAS_ADAPTER *ioc)
 {
 	u32 host_diagnostic;
 	u32 ioc_state;
@@ -4856,10 +4854,7 @@ _base_diag_reset(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 		writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence);
 
 		/* wait 100 msec */
-		if (sleep_flag == CAN_SLEEP)
-			msleep(100);
-		else
-			mdelay(100);
+		msleep(100);
 
 		if (count++ > 20)
 			goto out;
@@ -4879,10 +4874,7 @@ _base_diag_reset(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 	     &ioc->chip->HostDiagnostic);
 
 	/*This delay allows the chip PCIe hardware time to finish reset tasks*/
-	if (sleep_flag == CAN_SLEEP)
-		msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
-	else
-		mdelay(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
+	msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
 
 	/* Approximately 300 second max wait */
 	for (count = 0; count < (300000000 /
@@ -4895,13 +4887,7 @@ _base_diag_reset(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 		if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER))
 			break;
 
-		/* Wait to pass the second read delay window */
-		if (sleep_flag == CAN_SLEEP)
-			msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC
-								/ 1000);
-		else
-			mdelay(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC
-								/ 1000);
+		msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC / 1000);
 	}
 
 	if (host_diagnostic & MPI2_DIAG_HCB_MODE) {
@@ -4930,8 +4916,7 @@ _base_diag_reset(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 
 	drsprintk(ioc, pr_info(MPT3SAS_FMT
 		"Wait for FW to go to the READY state\n", ioc->name));
-	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20,
-	    sleep_flag);
+	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20);
 	if (ioc_state) {
 		pr_err(MPT3SAS_FMT
 			"%s: failed going to ready state (ioc_state=0x%x)\n",
@@ -4950,14 +4935,12 @@ _base_diag_reset(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 /**
  * _base_make_ioc_ready - put controller in READY state
  * @ioc: per adapter object
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  * @type: FORCE_BIG_HAMMER or SOFT_RESET
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
-	enum reset_type type)
+_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type)
 {
 	u32 ioc_state;
 	int rc;
@@ -4984,10 +4967,7 @@ _base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
 				    ioc->name, __func__, ioc_state);
 				return -EFAULT;
 			}
-			if (sleep_flag == CAN_SLEEP)
-				ssleep(1);
-			else
-				mdelay(1000);
+			ssleep(1);
 			ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
 		}
 	}
@@ -5013,24 +4993,23 @@ _base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
 
 	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
 		if (!(_base_send_ioc_reset(ioc,
-		    MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15, CAN_SLEEP))) {
+		    MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15))) {
 			return 0;
 	}
 
  issue_diag_reset:
-	rc = _base_diag_reset(ioc, CAN_SLEEP);
+	rc = _base_diag_reset(ioc);
 	return rc;
 }
 
 /**
  * _base_make_ioc_operational - put controller in OPERATIONAL state
  * @ioc: per adapter object
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * Returns 0 for success, non-zero for failure.
  */
 static int
-_base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
+_base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
 {
 	int r, i, index;
 	unsigned long	flags;
@@ -5149,7 +5128,7 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 	}
  skip_init_reply_post_free_queue:
 
-	r = _base_send_ioc_init(ioc, sleep_flag);
+	r = _base_send_ioc_init(ioc);
 	if (r)
 		return r;
 
@@ -5175,13 +5154,11 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
  skip_init_reply_post_host_index:
 
 	_base_unmask_interrupts(ioc);
-	r = _base_event_notification(ioc, sleep_flag);
+	r = _base_event_notification(ioc);
 	if (r)
 		return r;
 
-	if (sleep_flag == CAN_SLEEP)
-		_base_static_config_pages(ioc);
-
+	_base_static_config_pages(ioc);
 
 	if (ioc->is_driver_loading) {
 
@@ -5200,7 +5177,7 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 		return r; /* scan_start and scan_finished support */
 	}
 
-	r = _base_send_port_enable(ioc, sleep_flag);
+	r = _base_send_port_enable(ioc);
 	if (r)
 		return r;
 
@@ -5224,7 +5201,7 @@ mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc)
 	if (ioc->chip_phys && ioc->chip) {
 		_base_mask_interrupts(ioc);
 		ioc->shost_recovery = 1;
-		_base_make_ioc_ready(ioc, CAN_SLEEP, SOFT_RESET);
+		_base_make_ioc_ready(ioc, SOFT_RESET);
 		ioc->shost_recovery = 0;
 	}
 
@@ -5280,19 +5257,8 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 	if (r)
 		goto out_free_resources;
 
-	if (ioc->is_warpdrive) {
-		ioc->reply_post_host_index[0] = (resource_size_t __iomem *)
-		    &ioc->chip->ReplyPostHostIndex;
-
-		for (i = 1; i < ioc->cpu_msix_table_sz; i++)
-			ioc->reply_post_host_index[i] =
-			(resource_size_t __iomem *)
-			((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1)
-			* 4)));
-	}
-
 	pci_set_drvdata(ioc->pdev, ioc->shost);
-	r = _base_get_ioc_facts(ioc, CAN_SLEEP);
+	r = _base_get_ioc_facts(ioc);
 	if (r)
 		goto out_free_resources;
 
@@ -5326,7 +5292,7 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 	ioc->build_sg_mpi = &_base_build_sg;
 	ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge;
 
-	r = _base_make_ioc_ready(ioc, CAN_SLEEP, SOFT_RESET);
+	r = _base_make_ioc_ready(ioc, SOFT_RESET);
 	if (r)
 		goto out_free_resources;
 
@@ -5338,12 +5304,12 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 	}
 
 	for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) {
-		r = _base_get_port_facts(ioc, i, CAN_SLEEP);
+		r = _base_get_port_facts(ioc, i);
 		if (r)
 			goto out_free_resources;
 	}
 
-	r = _base_allocate_memory_pools(ioc, CAN_SLEEP);
+	r = _base_allocate_memory_pools(ioc);
 	if (r)
 		goto out_free_resources;
 
@@ -5429,7 +5395,7 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 	if (ioc->hba_mpi_version_belonged == MPI26_VERSION)
 		_base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
 
-	r = _base_make_ioc_operational(ioc, CAN_SLEEP);
+	r = _base_make_ioc_operational(ioc);
 	if (r)
 		goto out_free_resources;
 
@@ -5565,21 +5531,18 @@ _base_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase)
 /**
  * _wait_for_commands_to_complete - reset controller
  * @ioc: Pointer to MPT_ADAPTER structure
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  *
  * This function waiting(3s) for all pending commands to complete
  * prior to putting controller in reset.
  */
 static void
-_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
+_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc)
 {
 	u32 ioc_state;
 	unsigned long flags;
 	u16 i;
 
 	ioc->pending_io_count = 0;
-	if (sleep_flag != CAN_SLEEP)
-		return;
 
 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
 	if ((ioc_state & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL)
@@ -5602,13 +5565,12 @@ _wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc, int sleep_flag)
 /**
  * mpt3sas_base_hard_reset_handler - reset controller
  * @ioc: Pointer to MPT_ADAPTER structure
- * @sleep_flag: CAN_SLEEP or NO_SLEEP
  * @type: FORCE_BIG_HAMMER or SOFT_RESET
  *
  * Returns 0 for success, non-zero for failure.
  */
 int
-mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
+mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,
 	enum reset_type type)
 {
 	int r;
@@ -5629,13 +5591,6 @@ mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
 	if (mpt3sas_fwfault_debug)
 		mpt3sas_halt_firmware(ioc);
 
-	/* TODO - What we really should be doing is pulling
-	 * out all the code associated with NO_SLEEP; its never used.
-	 * That is legacy code from mpt fusion driver, ported over.
-	 * I will leave this BUG_ON here for now till its been resolved.
-	 */
-	BUG_ON(sleep_flag == NO_SLEEP);
-
 	/* wait for an active reset in progress to complete */
 	if (!mutex_trylock(&ioc->reset_in_progress_mutex)) {
 		do {
@@ -5660,9 +5615,9 @@ mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
 			is_fault = 1;
 	}
 	_base_reset_handler(ioc, MPT3_IOC_PRE_RESET);
-	_wait_for_commands_to_complete(ioc, sleep_flag);
+	_wait_for_commands_to_complete(ioc);
 	_base_mask_interrupts(ioc);
-	r = _base_make_ioc_ready(ioc, sleep_flag, type);
+	r = _base_make_ioc_ready(ioc, type);
 	if (r)
 		goto out;
 	_base_reset_handler(ioc, MPT3_IOC_AFTER_RESET);
@@ -5675,7 +5630,7 @@ mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
 		r = -EFAULT;
 		goto out;
 	}
-	r = _base_get_ioc_facts(ioc, CAN_SLEEP);
+	r = _base_get_ioc_facts(ioc);
 	if (r)
 		goto out;
 
@@ -5684,7 +5639,7 @@ mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
 		      "Please reboot the system and ensure that the correct"
 		      " firmware version is running\n", ioc->name);
 
-	r = _base_make_ioc_operational(ioc, sleep_flag);
+	r = _base_make_ioc_operational(ioc);
 	if (!r)
 		_base_reset_handler(ioc, MPT3_IOC_DONE_RESET);
 
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index 892c9be008b5..3e71bc1b4a80 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -119,10 +119,6 @@
 
 #define MPT_MAX_CALLBACKS		32
 
-
-#define	 CAN_SLEEP			1
-#define  NO_SLEEP			0
-
 #define INTERNAL_CMDS_COUNT		10	/* reserved cmds */
 /* reserved for issuing internally framed scsi io cmds */
 #define INTERNAL_SCSIIO_CMDS_COUNT	3
@@ -478,7 +474,7 @@ struct _sas_device {
 	u8	pfa_led_on;
 	u8	pend_sas_rphy_add;
 	u8	enclosure_level;
-	u8	connector_name[4];
+	u8	connector_name[5];
 	struct kref refcount;
 };
 
@@ -794,16 +790,6 @@ struct reply_post_struct {
 	dma_addr_t			reply_post_free_dma;
 };
 
-/**
- * enum mutex_type - task management mutex type
- * @TM_MUTEX_OFF: mutex is not required becuase calling function is acquiring it
- * @TM_MUTEX_ON: mutex is required
- */
-enum mutex_type {
-	TM_MUTEX_OFF = 0,
-	TM_MUTEX_ON = 1,
-};
-
 typedef void (*MPT3SAS_FLUSH_RUNNING_CMDS)(struct MPT3SAS_ADAPTER *ioc);
 /**
  * struct MPT3SAS_ADAPTER - per adapter struct
@@ -1229,7 +1215,7 @@ int mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc);
 void mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc);
 int mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc);
 void mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc);
-int mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, int sleep_flag,
+int mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,
 	enum reset_type type);
 
 void *mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid);
@@ -1291,7 +1277,11 @@ void mpt3sas_scsih_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase);
 
 int mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 	uint channel, uint id, uint lun, u8 type, u16 smid_task,
-	ulong timeout, enum mutex_type m_type);
+	ulong timeout);
+int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
+	uint channel, uint id, uint lun, u8 type, u16 smid_task,
+	ulong timeout);
+
 void mpt3sas_scsih_set_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle);
 void mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle);
 void mpt3sas_expander_remove(struct MPT3SAS_ADAPTER *ioc, u64 sas_address);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_config.c b/drivers/scsi/mpt3sas/mpt3sas_config.c
index a6914ec99cc0..cebfd734fd76 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_config.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_config.c
@@ -285,7 +285,6 @@ _config_request(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigRequest_t
 {
 	u16 smid;
 	u32 ioc_state;
-	unsigned long timeleft;
 	Mpi2ConfigRequest_t *config_request;
 	int r;
 	u8 retry_count, issue_host_reset = 0;
@@ -386,8 +385,7 @@ _config_request(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigRequest_t
 	_config_display_some_debug(ioc, smid, "config_request", NULL);
 	init_completion(&ioc->config_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->config_cmds.done,
-	    timeout*HZ);
+	wait_for_completion_timeout(&ioc->config_cmds.done, timeout*HZ);
 	if (!(ioc->config_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
 		    ioc->name, __func__);
@@ -491,8 +489,7 @@ _config_request(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigRequest_t
 	mutex_unlock(&ioc->config_cmds.mutex);
 
 	if (issue_host_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 	return r;
 }
 
diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
index 7d00f09666b6..26cdc127ac89 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
@@ -518,7 +518,7 @@ mpt3sas_ctl_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase)
  *
  * Called when application request fasyn callback handler.
  */
-int
+static int
 _ctl_fasync(int fd, struct file *filep, int mode)
 {
 	return fasync_helper(fd, filep, mode, &async_queue);
@@ -530,7 +530,7 @@ _ctl_fasync(int fd, struct file *filep, int mode)
  * @wait -
  *
  */
-unsigned int
+static unsigned int
 _ctl_poll(struct file *filep, poll_table *wait)
 {
 	struct MPT3SAS_ADAPTER *ioc;
@@ -641,9 +641,8 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 	MPI2RequestHeader_t *mpi_request = NULL, *request;
 	MPI2DefaultReply_t *mpi_reply;
 	u32 ioc_state;
-	u16 ioc_status;
 	u16 smid;
-	unsigned long timeout, timeleft;
+	unsigned long timeout;
 	u8 issue_reset;
 	u32 sz;
 	void *psge;
@@ -914,8 +913,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 		timeout = MPT3_IOCTL_DEFAULT_TIMEOUT;
 	else
 		timeout = karg.timeout;
-	timeleft = wait_for_completion_timeout(&ioc->ctl_cmds.done,
-	    timeout*HZ);
+	wait_for_completion_timeout(&ioc->ctl_cmds.done, timeout*HZ);
 	if (mpi_request->Function == MPI2_FUNCTION_SCSI_TASK_MGMT) {
 		Mpi2SCSITaskManagementRequest_t *tm_request =
 		    (Mpi2SCSITaskManagementRequest_t *)mpi_request;
@@ -938,7 +936,6 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 	}
 
 	mpi_reply = ioc->ctl_cmds.reply;
-	ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
 
 	if (mpi_reply->Function == MPI2_FUNCTION_SCSI_TASK_MGMT &&
 	    (ioc->logging_level & MPT_DEBUG_TM)) {
@@ -1001,13 +998,11 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 				ioc->name,
 				le16_to_cpu(mpi_request->FunctionDependent1));
 			mpt3sas_halt_firmware(ioc);
-			mpt3sas_scsih_issue_tm(ioc,
+			mpt3sas_scsih_issue_locked_tm(ioc,
 			    le16_to_cpu(mpi_request->FunctionDependent1), 0, 0,
-			    0, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 30,
-			    TM_MUTEX_ON);
+			    0, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 30);
 		} else
-			mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-			    FORCE_BIG_HAMMER);
+			mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 	}
 
  out:
@@ -1220,8 +1215,7 @@ _ctl_do_reset(struct MPT3SAS_ADAPTER *ioc, void __user *arg)
 	dctlprintk(ioc, pr_info(MPT3SAS_FMT "%s: enter\n", ioc->name,
 	    __func__));
 
-	retval = mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-	    FORCE_BIG_HAMMER);
+	retval = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 	pr_info(MPT3SAS_FMT "host reset: %s\n",
 	    ioc->name, ((!retval) ? "SUCCESS" : "FAILED"));
 	return 0;
@@ -1381,7 +1375,6 @@ _ctl_diag_register_2(struct MPT3SAS_ADAPTER *ioc,
 	Mpi2DiagBufferPostRequest_t *mpi_request;
 	Mpi2DiagBufferPostReply_t *mpi_reply;
 	u8 buffer_type;
-	unsigned long timeleft;
 	u16 smid;
 	u16 ioc_status;
 	u32 ioc_state;
@@ -1499,7 +1492,7 @@ _ctl_diag_register_2(struct MPT3SAS_ADAPTER *ioc,
 
 	init_completion(&ioc->ctl_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->ctl_cmds.done,
+	wait_for_completion_timeout(&ioc->ctl_cmds.done,
 	    MPT3_IOCTL_DEFAULT_TIMEOUT*HZ);
 
 	if (!(ioc->ctl_cmds.status & MPT3_CMD_COMPLETE)) {
@@ -1538,8 +1531,7 @@ _ctl_diag_register_2(struct MPT3SAS_ADAPTER *ioc,
 
  issue_host_reset:
 	if (issue_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 
  out:
 
@@ -1800,7 +1792,6 @@ mpt3sas_send_diag_release(struct MPT3SAS_ADAPTER *ioc, u8 buffer_type,
 	u16 ioc_status;
 	u32 ioc_state;
 	int rc;
-	unsigned long timeleft;
 
 	dctlprintk(ioc, pr_info(MPT3SAS_FMT "%s\n", ioc->name,
 	    __func__));
@@ -1848,7 +1839,7 @@ mpt3sas_send_diag_release(struct MPT3SAS_ADAPTER *ioc, u8 buffer_type,
 
 	init_completion(&ioc->ctl_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->ctl_cmds.done,
+	wait_for_completion_timeout(&ioc->ctl_cmds.done,
 	    MPT3_IOCTL_DEFAULT_TIMEOUT*HZ);
 
 	if (!(ioc->ctl_cmds.status & MPT3_CMD_COMPLETE)) {
@@ -1974,8 +1965,7 @@ _ctl_diag_release(struct MPT3SAS_ADAPTER *ioc, void __user *arg)
 	rc = mpt3sas_send_diag_release(ioc, buffer_type, &issue_reset);
 
 	if (issue_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 
 	return rc;
 }
@@ -1995,7 +1985,7 @@ _ctl_diag_read_buffer(struct MPT3SAS_ADAPTER *ioc, void __user *arg)
 	Mpi2DiagBufferPostReply_t *mpi_reply;
 	int rc, i;
 	u8 buffer_type;
-	unsigned long timeleft, request_size, copy_size;
+	unsigned long request_size, copy_size;
 	u16 smid;
 	u16 ioc_status;
 	u8 issue_reset = 0;
@@ -2116,7 +2106,7 @@ _ctl_diag_read_buffer(struct MPT3SAS_ADAPTER *ioc, void __user *arg)
 
 	init_completion(&ioc->ctl_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->ctl_cmds.done,
+	wait_for_completion_timeout(&ioc->ctl_cmds.done,
 	    MPT3_IOCTL_DEFAULT_TIMEOUT*HZ);
 
 	if (!(ioc->ctl_cmds.status & MPT3_CMD_COMPLETE)) {
@@ -2155,8 +2145,7 @@ _ctl_diag_read_buffer(struct MPT3SAS_ADAPTER *ioc, void __user *arg)
 
  issue_host_reset:
 	if (issue_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 
  out:
 
@@ -2352,7 +2341,7 @@ out_unlock_pciaccess:
  * @cmd - ioctl opcode
  * @arg -
  */
-long
+static long
 _ctl_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	long ret;
@@ -2372,7 +2361,7 @@ _ctl_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  * @cmd - ioctl opcode
  * @arg -
  */
-long
+static long
 _ctl_mpt2_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	long ret;
@@ -2392,7 +2381,7 @@ _ctl_mpt2_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  *
  * This routine handles 32 bit applications in 64bit os.
  */
-long
+static long
 _ctl_ioctl_compat(struct file *file, unsigned cmd, unsigned long arg)
 {
 	long ret;
@@ -2410,7 +2399,7 @@ _ctl_ioctl_compat(struct file *file, unsigned cmd, unsigned long arg)
  *
  * This routine handles 32 bit applications in 64bit os.
  */
-long
+static long
 _ctl_mpt2_ioctl_compat(struct file *file, unsigned cmd, unsigned long arg)
 {
 	long ret;
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index cd91a684c945..209a969a979d 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -1195,7 +1195,7 @@ _scsih_scsi_lookup_find_by_lun(struct MPT3SAS_ADAPTER *ioc, int id,
  *
  * Returns queue depth.
  */
-int
+static int
 scsih_change_queue_depth(struct scsi_device *sdev, int qdepth)
 {
 	struct Scsi_Host *shost = sdev->host;
@@ -1244,7 +1244,7 @@ scsih_change_queue_depth(struct scsi_device *sdev, int qdepth)
  * Returns 0 if ok. Any other return is assumed to be an error and
  * the device is ignored.
  */
-int
+static int
 scsih_target_alloc(struct scsi_target *starget)
 {
 	struct Scsi_Host *shost = dev_to_shost(&starget->dev);
@@ -1311,7 +1311,7 @@ scsih_target_alloc(struct scsi_target *starget)
  *
  * Returns nothing.
  */
-void
+static void
 scsih_target_destroy(struct scsi_target *starget)
 {
 	struct Scsi_Host *shost = dev_to_shost(&starget->dev);
@@ -1320,7 +1320,6 @@ scsih_target_destroy(struct scsi_target *starget)
 	struct _sas_device *sas_device;
 	struct _raid_device *raid_device;
 	unsigned long flags;
-	struct sas_rphy *rphy;
 
 	sas_target_priv_data = starget->hostdata;
 	if (!sas_target_priv_data)
@@ -1339,7 +1338,6 @@ scsih_target_destroy(struct scsi_target *starget)
 	}
 
 	spin_lock_irqsave(&ioc->sas_device_lock, flags);
-	rphy = dev_to_rphy(starget->dev.parent);
 	sas_device = __mpt3sas_get_sdev_from_target(ioc, sas_target_priv_data);
 	if (sas_device && (sas_device->starget == starget) &&
 	    (sas_device->id == starget->id) &&
@@ -1369,7 +1367,7 @@ scsih_target_destroy(struct scsi_target *starget)
  * Returns 0 if ok. Any other return is assumed to be an error and
  * the device is ignored.
  */
-int
+static int
 scsih_slave_alloc(struct scsi_device *sdev)
 {
 	struct Scsi_Host *shost;
@@ -1434,7 +1432,7 @@ scsih_slave_alloc(struct scsi_device *sdev)
  *
  * Returns nothing.
  */
-void
+static void
 scsih_slave_destroy(struct scsi_device *sdev)
 {
 	struct MPT3SAS_TARGET *sas_target_priv_data;
@@ -1527,7 +1525,7 @@ _scsih_display_sata_capabilities(struct MPT3SAS_ADAPTER *ioc,
  * scsih_is_raid - return boolean indicating device is raid volume
  * @dev the device struct object
  */
-int
+static int
 scsih_is_raid(struct device *dev)
 {
 	struct scsi_device *sdev = to_scsi_device(dev);
@@ -1542,7 +1540,7 @@ scsih_is_raid(struct device *dev)
  * scsih_get_resync - get raid volume resync percent complete
  * @dev the device struct object
  */
-void
+static void
 scsih_get_resync(struct device *dev)
 {
 	struct scsi_device *sdev = to_scsi_device(dev);
@@ -1603,7 +1601,7 @@ scsih_get_resync(struct device *dev)
  * scsih_get_state - get raid volume level
  * @dev the device struct object
  */
-void
+static void
 scsih_get_state(struct device *dev)
 {
 	struct scsi_device *sdev = to_scsi_device(dev);
@@ -1805,7 +1803,7 @@ _scsih_enable_tlr(struct MPT3SAS_ADAPTER *ioc, struct scsi_device *sdev)
  * Returns 0 if ok. Any other return is assumed to be an error and
  * the device is ignored.
  */
-int
+static int
 scsih_slave_configure(struct scsi_device *sdev)
 {
 	struct Scsi_Host *shost = sdev->host;
@@ -2021,7 +2019,7 @@ scsih_slave_configure(struct scsi_device *sdev)
  *
  * Return nothing.
  */
-int
+static int
 scsih_bios_param(struct scsi_device *sdev, struct block_device *bdev,
 	sector_t capacity, int params[])
 {
@@ -2201,7 +2199,6 @@ mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
  * @smid_task: smid assigned to the task
  * @timeout: timeout in seconds
- * @m_type: TM_MUTEX_ON or TM_MUTEX_OFF
  * Context: user
  *
  * A generic API for sending task management requests to firmware.
@@ -2212,60 +2209,51 @@ mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
  */
 int
 mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel,
-	uint id, uint lun, u8 type, u16 smid_task, ulong timeout,
-	enum mutex_type m_type)
+	uint id, uint lun, u8 type, u16 smid_task, ulong timeout)
 {
 	Mpi2SCSITaskManagementRequest_t *mpi_request;
 	Mpi2SCSITaskManagementReply_t *mpi_reply;
 	u16 smid = 0;
 	u32 ioc_state;
-	unsigned long timeleft;
 	struct scsiio_tracker *scsi_lookup = NULL;
 	int rc;
 	u16 msix_task = 0;
 
-	if (m_type == TM_MUTEX_ON)
-		mutex_lock(&ioc->tm_cmds.mutex);
+	lockdep_assert_held(&ioc->tm_cmds.mutex);
+
 	if (ioc->tm_cmds.status != MPT3_CMD_NOT_USED) {
 		pr_info(MPT3SAS_FMT "%s: tm_cmd busy!!!\n",
 		    __func__, ioc->name);
-		rc = FAILED;
-		goto err_out;
+		return FAILED;
 	}
 
 	if (ioc->shost_recovery || ioc->remove_host ||
 	    ioc->pci_error_recovery) {
 		pr_info(MPT3SAS_FMT "%s: host reset in progress!\n",
 		    __func__, ioc->name);
-		rc = FAILED;
-		goto err_out;
+		return FAILED;
 	}
 
 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
 	if (ioc_state & MPI2_DOORBELL_USED) {
 		dhsprintk(ioc, pr_info(MPT3SAS_FMT
 			"unexpected doorbell active!\n", ioc->name));
-		rc = mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
-		rc = (!rc) ? SUCCESS : FAILED;
-		goto err_out;
+		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
+		return (!rc) ? SUCCESS : FAILED;
 	}
 
 	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
 		mpt3sas_base_fault_info(ioc, ioc_state &
 		    MPI2_DOORBELL_DATA_MASK);
-		rc = mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
-		rc = (!rc) ? SUCCESS : FAILED;
-		goto err_out;
+		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
+		return (!rc) ? SUCCESS : FAILED;
 	}
 
 	smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_cb_idx);
 	if (!smid) {
 		pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
 		    ioc->name, __func__);
-		rc = FAILED;
-		goto err_out;
+		return FAILED;
 	}
 
 	if (type == MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK)
@@ -2292,19 +2280,17 @@ mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel,
 	else
 		msix_task = 0;
 	mpt3sas_base_put_smid_hi_priority(ioc, smid, msix_task);
-	timeleft = wait_for_completion_timeout(&ioc->tm_cmds.done, timeout*HZ);
+	wait_for_completion_timeout(&ioc->tm_cmds.done, timeout*HZ);
 	if (!(ioc->tm_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
 		    ioc->name, __func__);
 		_debug_dump_mf(mpi_request,
 		    sizeof(Mpi2SCSITaskManagementRequest_t)/4);
 		if (!(ioc->tm_cmds.status & MPT3_CMD_RESET)) {
-			rc = mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-			    FORCE_BIG_HAMMER);
+			rc = mpt3sas_base_hard_reset_handler(ioc,
+					FORCE_BIG_HAMMER);
 			rc = (!rc) ? SUCCESS : FAILED;
-			ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
-			mpt3sas_scsih_clear_tm_flag(ioc, handle);
-			goto err_out;
+			goto out;
 		}
 	}
 
@@ -2356,17 +2342,23 @@ mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel,
 		break;
 	}
 
+out:
 	mpt3sas_scsih_clear_tm_flag(ioc, handle);
 	ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
-	if (m_type == TM_MUTEX_ON)
-		mutex_unlock(&ioc->tm_cmds.mutex);
-
 	return rc;
+}
 
- err_out:
-	if (m_type == TM_MUTEX_ON)
-		mutex_unlock(&ioc->tm_cmds.mutex);
-	return rc;
+int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
+	uint channel, uint id, uint lun, u8 type, u16 smid_task, ulong timeout)
+{
+	int ret;
+
+	mutex_lock(&ioc->tm_cmds.mutex);
+	ret = mpt3sas_scsih_issue_tm(ioc, handle, channel, id, lun, type,
+			smid_task, timeout);
+	mutex_unlock(&ioc->tm_cmds.mutex);
+
+	return ret;
 }
 
 /**
@@ -2439,7 +2431,7 @@ _scsih_tm_display_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd)
  *
  * Returns SUCCESS if command aborted else FAILED
  */
-int
+static int
 scsih_abort(struct scsi_cmnd *scmd)
 {
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
@@ -2482,9 +2474,9 @@ scsih_abort(struct scsi_cmnd *scmd)
 	mpt3sas_halt_firmware(ioc);
 
 	handle = sas_device_priv_data->sas_target->handle;
-	r = mpt3sas_scsih_issue_tm(ioc, handle, scmd->device->channel,
+	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,
 	    scmd->device->id, scmd->device->lun,
-	    MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK, smid, 30, TM_MUTEX_ON);
+	    MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK, smid, 30);
 
  out:
 	sdev_printk(KERN_INFO, scmd->device, "task abort: %s scmd(%p)\n",
@@ -2498,7 +2490,7 @@ scsih_abort(struct scsi_cmnd *scmd)
  *
  * Returns SUCCESS if command aborted else FAILED
  */
-int
+static int
 scsih_dev_reset(struct scsi_cmnd *scmd)
 {
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
@@ -2541,9 +2533,9 @@ scsih_dev_reset(struct scsi_cmnd *scmd)
 		goto out;
 	}
 
-	r = mpt3sas_scsih_issue_tm(ioc, handle, scmd->device->channel,
+	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,
 	    scmd->device->id, scmd->device->lun,
-	    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, 0, 30, TM_MUTEX_ON);
+	    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, 0, 30);
 
  out:
 	sdev_printk(KERN_INFO, scmd->device, "device reset: %s scmd(%p)\n",
@@ -2561,7 +2553,7 @@ scsih_dev_reset(struct scsi_cmnd *scmd)
  *
  * Returns SUCCESS if command aborted else FAILED
  */
-int
+static int
 scsih_target_reset(struct scsi_cmnd *scmd)
 {
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
@@ -2603,9 +2595,9 @@ scsih_target_reset(struct scsi_cmnd *scmd)
 		goto out;
 	}
 
-	r = mpt3sas_scsih_issue_tm(ioc, handle, scmd->device->channel,
+	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,
 	    scmd->device->id, 0, MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0,
-	    30, TM_MUTEX_ON);
+	    30);
 
  out:
 	starget_printk(KERN_INFO, starget, "target reset: %s scmd(%p)\n",
@@ -2624,7 +2616,7 @@ scsih_target_reset(struct scsi_cmnd *scmd)
  *
  * Returns SUCCESS if command aborted else FAILED
  */
-int
+static int
 scsih_host_reset(struct scsi_cmnd *scmd)
 {
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
@@ -2641,8 +2633,7 @@ scsih_host_reset(struct scsi_cmnd *scmd)
 		goto out;
 	}
 
-	retval = mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-	    FORCE_BIG_HAMMER);
+	retval = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 	r = (retval < 0) ? FAILED : SUCCESS;
 out:
 	pr_info(MPT3SAS_FMT "host reset: %s scmd(%p)\n",
@@ -3455,7 +3446,7 @@ _scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
  *
  * Context - processed in interrupt context.
  */
-void
+static void
 _scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 event,
 				u32 event_context)
 {
@@ -3494,7 +3485,7 @@ _scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 event,
  *
  * Context - processed in interrupt context.
  */
-void
+static void
 _scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc,
 					u16 smid, u16 handle)
 	{
@@ -4032,7 +4023,7 @@ _scsih_eedp_error_handling(struct scsi_cmnd *scmd, u16 ioc_status)
  * SCSI_MLQUEUE_DEVICE_BUSY if the device queue is full, or
  * SCSI_MLQUEUE_HOST_BUSY if the entire host queue is full
  */
-int
+static int
 scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
 {
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
@@ -4701,7 +4692,7 @@ _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
 			    le16_to_cpu(mpi_reply->DevHandle));
 		mpt3sas_trigger_scsi(ioc, data.skey, data.asc, data.ascq);
 
-		if (!(ioc->logging_level & MPT_DEBUG_REPLY) &&
+		if ((ioc->logging_level & MPT_DEBUG_REPLY) &&
 		     ((scmd->sense_buffer[2] == UNIT_ATTENTION) ||
 		     (scmd->sense_buffer[2] == MEDIUM_ERROR) ||
 		     (scmd->sense_buffer[2] == HARDWARE_ERROR)))
@@ -5380,8 +5371,9 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
 		     MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
 			sas_device->enclosure_level =
 				le16_to_cpu(sas_device_pg0.EnclosureLevel);
-			memcpy(&sas_device->connector_name[0],
-				&sas_device_pg0.ConnectorName[0], 4);
+			memcpy(sas_device->connector_name,
+				sas_device_pg0.ConnectorName, 4);
+			sas_device->connector_name[4] = '\0';
 		} else {
 			sas_device->enclosure_level = 0;
 			sas_device->connector_name[0] = '\0';
@@ -5508,8 +5500,9 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
 	if (sas_device_pg0.Flags & MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
 		sas_device->enclosure_level =
 			le16_to_cpu(sas_device_pg0.EnclosureLevel);
-		memcpy(&sas_device->connector_name[0],
-			&sas_device_pg0.ConnectorName[0], 4);
+		memcpy(sas_device->connector_name,
+			sas_device_pg0.ConnectorName, 4);
+		sas_device->connector_name[4] = '\0';
 	} else {
 		sas_device->enclosure_level = 0;
 		sas_device->connector_name[0] = '\0';
@@ -6087,8 +6080,7 @@ _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
 
 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
 		r = mpt3sas_scsih_issue_tm(ioc, handle, 0, 0, lun,
-		    MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK, smid, 30,
-		    TM_MUTEX_OFF);
+		    MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK, smid, 30);
 		if (r == FAILED) {
 			sdev_printk(KERN_WARNING, sdev,
 			    "mpt3sas_scsih_issue_tm: FAILED when sending "
@@ -6128,8 +6120,8 @@ _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
 			goto out_no_lock;
 
 		r = mpt3sas_scsih_issue_tm(ioc, handle, sdev->channel, sdev->id,
-		    sdev->lun, MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK, smid, 30,
-		    TM_MUTEX_OFF);
+		    sdev->lun, MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK, smid,
+		    30);
 		if (r == FAILED) {
 			sdev_printk(KERN_WARNING, sdev,
 			    "mpt3sas_scsih_issue_tm: ABORT_TASK: FAILED : "
@@ -6297,8 +6289,7 @@ _scsih_ir_fastpath(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phys_disk_num)
 	mutex_unlock(&ioc->scsih_cmds.mutex);
 
 	if (issue_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 	return rc;
 }
 
@@ -6311,11 +6302,10 @@ _scsih_ir_fastpath(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phys_disk_num)
 static void
 _scsih_reprobe_lun(struct scsi_device *sdev, void *no_uld_attach)
 {
-	int rc;
 	sdev->no_uld_attach = no_uld_attach ? 1 : 0;
 	sdev_printk(KERN_INFO, sdev, "%s raid component\n",
 	    sdev->no_uld_attach ? "hidding" : "exposing");
-	rc = scsi_device_reprobe(sdev);
+	WARN_ON(scsi_device_reprobe(sdev));
 }
 
 /**
@@ -8137,7 +8127,7 @@ _scsih_ir_shutdown(struct MPT3SAS_ADAPTER *ioc)
  * Routine called when unloading the driver.
  * Return nothing.
  */
-void scsih_remove(struct pci_dev *pdev)
+static void scsih_remove(struct pci_dev *pdev)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
@@ -8210,7 +8200,7 @@ void scsih_remove(struct pci_dev *pdev)
  *
  * Return nothing.
  */
-void
+static void
 scsih_shutdown(struct pci_dev *pdev)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
@@ -8451,7 +8441,7 @@ _scsih_probe_devices(struct MPT3SAS_ADAPTER *ioc)
  * of scanning the entire bus.  In our implemention, we will kick off
  * firmware discovery.
  */
-void
+static void
 scsih_scan_start(struct Scsi_Host *shost)
 {
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
@@ -8478,7 +8468,7 @@ scsih_scan_start(struct Scsi_Host *shost)
  * scsi_host and the elapsed time of the scan in jiffies. In our implemention,
  * we wait for firmware discovery to complete, then return 1.
  */
-int
+static int
 scsih_scan_finished(struct Scsi_Host *shost, unsigned long time)
 {
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
@@ -8608,7 +8598,7 @@ static struct raid_function_template mpt3sas_raid_functions = {
  *	MPI25_VERSION for SAS 3.0 HBA devices, and
  *	MPI26 VERSION for Cutlass & Invader SAS 3.0 HBA devices
  */
-u16
+static u16
 _scsih_determine_hba_mpi_version(struct pci_dev *pdev)
 {
 
@@ -8660,7 +8650,7 @@ _scsih_determine_hba_mpi_version(struct pci_dev *pdev)
  *
  * Returns 0 success, anything else error.
  */
-int
+static int
 _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	struct MPT3SAS_ADAPTER *ioc;
@@ -8869,7 +8859,7 @@ out_add_shost_fail:
  *
  * Returns 0 success, anything else error.
  */
-int
+static int
 scsih_suspend(struct pci_dev *pdev, pm_message_t state)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
@@ -8896,7 +8886,7 @@ scsih_suspend(struct pci_dev *pdev, pm_message_t state)
  *
  * Returns 0 success, anything else error.
  */
-int
+static int
 scsih_resume(struct pci_dev *pdev)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
@@ -8916,7 +8906,7 @@ scsih_resume(struct pci_dev *pdev)
 	if (r)
 		return r;
 
-	mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP, SOFT_RESET);
+	mpt3sas_base_hard_reset_handler(ioc, SOFT_RESET);
 	scsi_unblock_requests(shost);
 	mpt3sas_base_start_watchdog(ioc);
 	return 0;
@@ -8933,7 +8923,7 @@ scsih_resume(struct pci_dev *pdev)
  * Return value:
  *      PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT
  */
-pci_ers_result_t
+static pci_ers_result_t
 scsih_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
@@ -8970,7 +8960,7 @@ scsih_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
  * code after the PCI slot has been reset, just before we
  * should resume normal operations.
  */
-pci_ers_result_t
+static pci_ers_result_t
 scsih_pci_slot_reset(struct pci_dev *pdev)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
@@ -8987,8 +8977,7 @@ scsih_pci_slot_reset(struct pci_dev *pdev)
 	if (rc)
 		return PCI_ERS_RESULT_DISCONNECT;
 
-	rc = mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-	    FORCE_BIG_HAMMER);
+	rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 
 	pr_warn(MPT3SAS_FMT "hard reset: %s\n", ioc->name,
 	    (rc == 0) ? "success" : "failed");
@@ -9007,7 +8996,7 @@ scsih_pci_slot_reset(struct pci_dev *pdev)
  * OK to resume normal operation. Use completion to allow
  * halted scsi ops to resume.
  */
-void
+static void
 scsih_pci_resume(struct pci_dev *pdev)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
@@ -9024,7 +9013,7 @@ scsih_pci_resume(struct pci_dev *pdev)
  * scsih_pci_mmio_enabled - Enable MMIO and dump debug registers
  * @pdev: pointer to PCI device
  */
-pci_ers_result_t
+static pci_ers_result_t
 scsih_pci_mmio_enabled(struct pci_dev *pdev)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
@@ -9152,7 +9141,7 @@ static struct pci_driver mpt3sas_driver = {
  *
  * Returns 0 success, anything else error.
  */
-int
+static int
 scsih_init(void)
 {
 	mpt2_ids = 0;
@@ -9202,7 +9191,7 @@ scsih_init(void)
  *
  * Returns 0 success, anything else error.
  */
-void
+static void
 scsih_exit(void)
 {
 
diff --git a/drivers/scsi/mpt3sas/mpt3sas_transport.c b/drivers/scsi/mpt3sas/mpt3sas_transport.c
index ff93286bc32f..b74faf1a69b2 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_transport.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_transport.c
@@ -300,7 +300,6 @@ _transport_expander_report_manufacture(struct MPT3SAS_ADAPTER *ioc,
 	int rc;
 	u16 smid;
 	u32 ioc_state;
-	unsigned long timeleft;
 	void *psge;
 	u8 issue_reset = 0;
 	void *data_out = NULL;
@@ -394,8 +393,7 @@ _transport_expander_report_manufacture(struct MPT3SAS_ADAPTER *ioc,
 		ioc->name, (unsigned long long)sas_address));
 	init_completion(&ioc->transport_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->transport_cmds.done,
-	    10*HZ);
+	wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ);
 
 	if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
@@ -446,8 +444,7 @@ _transport_expander_report_manufacture(struct MPT3SAS_ADAPTER *ioc,
 
  issue_host_reset:
 	if (issue_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
  out:
 	ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
 	if (data_out)
@@ -1107,7 +1104,6 @@ _transport_get_expander_phy_error_log(struct MPT3SAS_ADAPTER *ioc,
 	int rc;
 	u16 smid;
 	u32 ioc_state;
-	unsigned long timeleft;
 	void *psge;
 	u8 issue_reset = 0;
 	void *data_out = NULL;
@@ -1203,8 +1199,7 @@ _transport_get_expander_phy_error_log(struct MPT3SAS_ADAPTER *ioc,
 		phy->number));
 	init_completion(&ioc->transport_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->transport_cmds.done,
-	    10*HZ);
+	wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ);
 
 	if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
@@ -1253,8 +1248,7 @@ _transport_get_expander_phy_error_log(struct MPT3SAS_ADAPTER *ioc,
 
  issue_host_reset:
 	if (issue_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
  out:
 	ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
 	if (data_out)
@@ -1421,7 +1415,6 @@ _transport_expander_phy_control(struct MPT3SAS_ADAPTER *ioc,
 	int rc;
 	u16 smid;
 	u32 ioc_state;
-	unsigned long timeleft;
 	void *psge;
 	u8 issue_reset = 0;
 	void *data_out = NULL;
@@ -1522,8 +1515,7 @@ _transport_expander_phy_control(struct MPT3SAS_ADAPTER *ioc,
 		phy->number, phy_operation));
 	init_completion(&ioc->transport_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->transport_cmds.done,
-	    10*HZ);
+	wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ);
 
 	if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
@@ -1564,8 +1556,7 @@ _transport_expander_phy_control(struct MPT3SAS_ADAPTER *ioc,
 
  issue_host_reset:
 	if (issue_reset)
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
  out:
 	ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
 	if (data_out)
@@ -1899,7 +1890,6 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 	int rc;
 	u16 smid;
 	u32 ioc_state;
-	unsigned long timeleft;
 	void *psge;
 	u8 issue_reset = 0;
 	dma_addr_t dma_addr_in = 0;
@@ -2043,8 +2033,7 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 
 	init_completion(&ioc->transport_cmds.done);
 	mpt3sas_base_put_smid_default(ioc, smid);
-	timeleft = wait_for_completion_timeout(&ioc->transport_cmds.done,
-	    10*HZ);
+	wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ);
 
 	if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s : timeout\n",
@@ -2103,8 +2092,7 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 
  issue_host_reset:
 	if (issue_reset) {
-		mpt3sas_base_hard_reset_handler(ioc, CAN_SLEEP,
-		    FORCE_BIG_HAMMER);
+		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
 		rc = -ETIMEDOUT;
 	}
 
diff --git a/drivers/scsi/mvsas/mv_64xx.c b/drivers/scsi/mvsas/mv_64xx.c
index 8bb06995adfb..b757d389e32f 100644
--- a/drivers/scsi/mvsas/mv_64xx.c
+++ b/drivers/scsi/mvsas/mv_64xx.c
@@ -136,7 +136,8 @@ static void mvs_64xx_phy_reset(struct mvs_info *mvi, u32 phy_id, int hard)
 	}
 }
 
-void mvs_64xx_clear_srs_irq(struct mvs_info *mvi, u8 reg_set, u8 clear_all)
+static void
+mvs_64xx_clear_srs_irq(struct mvs_info *mvi, u8 reg_set, u8 clear_all)
 {
 	void __iomem *regs = mvi->regs;
 	u32 tmp;
@@ -563,7 +564,7 @@ static u8 mvs_64xx_assign_reg_set(struct mvs_info *mvi, u8 *tfs)
 	return MVS_ID_NOT_MAPPED;
 }
 
-void mvs_64xx_make_prd(struct scatterlist *scatter, int nr, void *prd)
+static void mvs_64xx_make_prd(struct scatterlist *scatter, int nr, void *prd)
 {
 	int i;
 	struct scatterlist *sg;
@@ -633,7 +634,7 @@ static void mvs_64xx_phy_work_around(struct mvs_info *mvi, int i)
 	mvs_write_port_vsr_data(mvi, i, tmp);
 }
 
-void mvs_64xx_phy_set_link_rate(struct mvs_info *mvi, u32 phy_id,
+static void mvs_64xx_phy_set_link_rate(struct mvs_info *mvi, u32 phy_id,
 			struct sas_phy_linkrates *rates)
 {
 	u32 lrmin = 0, lrmax = 0;
@@ -668,20 +669,20 @@ static void mvs_64xx_clear_active_cmds(struct mvs_info *mvi)
 }
 
 
-u32 mvs_64xx_spi_read_data(struct mvs_info *mvi)
+static u32 mvs_64xx_spi_read_data(struct mvs_info *mvi)
 {
 	void __iomem *regs = mvi->regs_ex;
 	return ior32(SPI_DATA_REG_64XX);
 }
 
-void mvs_64xx_spi_write_data(struct mvs_info *mvi, u32 data)
+static void mvs_64xx_spi_write_data(struct mvs_info *mvi, u32 data)
 {
 	void __iomem *regs = mvi->regs_ex;
 	 iow32(SPI_DATA_REG_64XX, data);
 }
 
 
-int mvs_64xx_spi_buildcmd(struct mvs_info *mvi,
+static int mvs_64xx_spi_buildcmd(struct mvs_info *mvi,
 			u32      *dwCmd,
 			u8       cmd,
 			u8       read,
@@ -705,7 +706,7 @@ int mvs_64xx_spi_buildcmd(struct mvs_info *mvi,
 }
 
 
-int mvs_64xx_spi_issuecmd(struct mvs_info *mvi, u32 cmd)
+static int mvs_64xx_spi_issuecmd(struct mvs_info *mvi, u32 cmd)
 {
 	void __iomem *regs = mvi->regs_ex;
 	int     retry;
@@ -720,7 +721,7 @@ int mvs_64xx_spi_issuecmd(struct mvs_info *mvi, u32 cmd)
 	return 0;
 }
 
-int mvs_64xx_spi_waitdataready(struct mvs_info *mvi, u32 timeout)
+static int mvs_64xx_spi_waitdataready(struct mvs_info *mvi, u32 timeout)
 {
 	void __iomem *regs = mvi->regs_ex;
 	u32 i, dwTmp;
@@ -735,7 +736,7 @@ int mvs_64xx_spi_waitdataready(struct mvs_info *mvi, u32 timeout)
 	return -1;
 }
 
-void mvs_64xx_fix_dma(struct mvs_info *mvi, u32 phy_mask,
+static void mvs_64xx_fix_dma(struct mvs_info *mvi, u32 phy_mask,
 				int buf_len, int from, void *prd)
 {
 	int i;
diff --git a/drivers/scsi/mvsas/mv_94xx.c b/drivers/scsi/mvsas/mv_94xx.c
index f6fc4a705924..4c57d9abce7b 100644
--- a/drivers/scsi/mvsas/mv_94xx.c
+++ b/drivers/scsi/mvsas/mv_94xx.c
@@ -48,8 +48,8 @@ static void mvs_94xx_detect_porttype(struct mvs_info *mvi, int i)
 	}
 }
 
-void set_phy_tuning(struct mvs_info *mvi, int phy_id,
-			struct phy_tuning phy_tuning)
+static void set_phy_tuning(struct mvs_info *mvi, int phy_id,
+			   struct phy_tuning phy_tuning)
 {
 	u32 tmp, setting_0 = 0, setting_1 = 0;
 	u8 i;
@@ -110,8 +110,8 @@ void set_phy_tuning(struct mvs_info *mvi, int phy_id,
 	}
 }
 
-void set_phy_ffe_tuning(struct mvs_info *mvi, int phy_id,
-				struct ffe_control ffe)
+static void set_phy_ffe_tuning(struct mvs_info *mvi, int phy_id,
+			       struct ffe_control ffe)
 {
 	u32 tmp;
 
@@ -177,7 +177,7 @@ void set_phy_ffe_tuning(struct mvs_info *mvi, int phy_id,
 }
 
 /*Notice: this function must be called when phy is disabled*/
-void set_phy_rate(struct mvs_info *mvi, int phy_id, u8 rate)
+static void set_phy_rate(struct mvs_info *mvi, int phy_id, u8 rate)
 {
 	union reg_phy_cfg phy_cfg, phy_cfg_tmp;
 	mvs_write_port_vsr_addr(mvi, phy_id, VSR_PHY_MODE2);
@@ -679,7 +679,8 @@ static void mvs_94xx_command_active(struct mvs_info *mvi, u32 slot_idx)
 	}
 }
 
-void mvs_94xx_clear_srs_irq(struct mvs_info *mvi, u8 reg_set, u8 clear_all)
+static void
+mvs_94xx_clear_srs_irq(struct mvs_info *mvi, u8 reg_set, u8 clear_all)
 {
 	void __iomem *regs = mvi->regs;
 	u32 tmp;
@@ -906,8 +907,8 @@ static void mvs_94xx_fix_phy_info(struct mvs_info *mvi, int i,
 
 }
 
-void mvs_94xx_phy_set_link_rate(struct mvs_info *mvi, u32 phy_id,
-			struct sas_phy_linkrates *rates)
+static void mvs_94xx_phy_set_link_rate(struct mvs_info *mvi, u32 phy_id,
+				       struct sas_phy_linkrates *rates)
 {
 	u32 lrmax = 0;
 	u32 tmp;
@@ -936,25 +937,25 @@ static void mvs_94xx_clear_active_cmds(struct mvs_info *mvi)
 }
 
 
-u32 mvs_94xx_spi_read_data(struct mvs_info *mvi)
+static u32 mvs_94xx_spi_read_data(struct mvs_info *mvi)
 {
 	void __iomem *regs = mvi->regs_ex - 0x10200;
 	return mr32(SPI_RD_DATA_REG_94XX);
 }
 
-void mvs_94xx_spi_write_data(struct mvs_info *mvi, u32 data)
+static void mvs_94xx_spi_write_data(struct mvs_info *mvi, u32 data)
 {
 	void __iomem *regs = mvi->regs_ex - 0x10200;
 	 mw32(SPI_RD_DATA_REG_94XX, data);
 }
 
 
-int mvs_94xx_spi_buildcmd(struct mvs_info *mvi,
-				u32      *dwCmd,
-				u8       cmd,
-				u8       read,
-				u8       length,
-				u32      addr
+static int mvs_94xx_spi_buildcmd(struct mvs_info *mvi,
+				 u32      *dwCmd,
+				 u8       cmd,
+				 u8       read,
+				 u8       length,
+				 u32      addr
 				)
 {
 	void __iomem *regs = mvi->regs_ex - 0x10200;
@@ -974,7 +975,7 @@ int mvs_94xx_spi_buildcmd(struct mvs_info *mvi,
 }
 
 
-int mvs_94xx_spi_issuecmd(struct mvs_info *mvi, u32 cmd)
+static int mvs_94xx_spi_issuecmd(struct mvs_info *mvi, u32 cmd)
 {
 	void __iomem *regs = mvi->regs_ex - 0x10200;
 	mw32(SPI_CTRL_REG_94XX, cmd | SPI_CTRL_SpiStart_94XX);
@@ -982,7 +983,7 @@ int mvs_94xx_spi_issuecmd(struct mvs_info *mvi, u32 cmd)
 	return 0;
 }
 
-int mvs_94xx_spi_waitdataready(struct mvs_info *mvi, u32 timeout)
+static int mvs_94xx_spi_waitdataready(struct mvs_info *mvi, u32 timeout)
 {
 	void __iomem *regs = mvi->regs_ex - 0x10200;
 	u32   i, dwTmp;
@@ -997,8 +998,8 @@ int mvs_94xx_spi_waitdataready(struct mvs_info *mvi, u32 timeout)
 	return -1;
 }
 
-void mvs_94xx_fix_dma(struct mvs_info *mvi, u32 phy_mask,
-				int buf_len, int from, void *prd)
+static void mvs_94xx_fix_dma(struct mvs_info *mvi, u32 phy_mask,
+			     int buf_len, int from, void *prd)
 {
 	int i;
 	struct mvs_prd *buf_prd = prd;
diff --git a/drivers/scsi/mvsas/mv_sas.c b/drivers/scsi/mvsas/mv_sas.c
index 5b9fcff6cd94..86eb19902bac 100644
--- a/drivers/scsi/mvsas/mv_sas.c
+++ b/drivers/scsi/mvsas/mv_sas.c
@@ -74,7 +74,7 @@ void mvs_tag_init(struct mvs_info *mvi)
 		mvs_tag_clear(mvi, i);
 }
 
-struct mvs_info *mvs_find_dev_mvi(struct domain_device *dev)
+static struct mvs_info *mvs_find_dev_mvi(struct domain_device *dev)
 {
 	unsigned long i = 0, j = 0, hi = 0;
 	struct sas_ha_struct *sha = dev->port->ha;
@@ -102,7 +102,7 @@ struct mvs_info *mvs_find_dev_mvi(struct domain_device *dev)
 
 }
 
-int mvs_find_dev_phyno(struct domain_device *dev, int *phyno)
+static int mvs_find_dev_phyno(struct domain_device *dev, int *phyno)
 {
 	unsigned long i = 0, j = 0, n = 0, num = 0;
 	struct mvs_device *mvi_dev = (struct mvs_device *)dev->lldd_dev;
@@ -1158,7 +1158,7 @@ void mvs_port_deformed(struct asd_sas_phy *sas_phy)
 	mvs_port_notify_deformed(sas_phy, 1);
 }
 
-struct mvs_device *mvs_alloc_dev(struct mvs_info *mvi)
+static struct mvs_device *mvs_alloc_dev(struct mvs_info *mvi)
 {
 	u32 dev;
 	for (dev = 0; dev < MVS_MAX_DEVICES; dev++) {
@@ -1175,7 +1175,7 @@ struct mvs_device *mvs_alloc_dev(struct mvs_info *mvi)
 	return NULL;
 }
 
-void mvs_free_dev(struct mvs_device *mvi_dev)
+static void mvs_free_dev(struct mvs_device *mvi_dev)
 {
 	u32 id = mvi_dev->device_id;
 	memset(mvi_dev, 0, sizeof(*mvi_dev));
@@ -1185,7 +1185,7 @@ void mvs_free_dev(struct mvs_device *mvi_dev)
 	mvi_dev->taskfileset = MVS_ID_NOT_MAPPED;
 }
 
-int mvs_dev_found_notify(struct domain_device *dev, int lock)
+static int mvs_dev_found_notify(struct domain_device *dev, int lock)
 {
 	unsigned long flags = 0;
 	int res = 0;
@@ -1241,7 +1241,7 @@ int mvs_dev_found(struct domain_device *dev)
 	return mvs_dev_found_notify(dev, 1);
 }
 
-void mvs_dev_gone_notify(struct domain_device *dev)
+static void mvs_dev_gone_notify(struct domain_device *dev)
 {
 	unsigned long flags = 0;
 	struct mvs_device *mvi_dev = dev->lldd_dev;
@@ -1611,7 +1611,7 @@ static int mvs_sata_done(struct mvs_info *mvi, struct sas_task *task,
 	return stat;
 }
 
-void mvs_set_sense(u8 *buffer, int len, int d_sense,
+static void mvs_set_sense(u8 *buffer, int len, int d_sense,
 		int key, int asc, int ascq)
 {
 	memset(buffer, 0, len);
@@ -1650,7 +1650,7 @@ void mvs_set_sense(u8 *buffer, int len, int d_sense,
 	return;
 }
 
-void mvs_fill_ssp_resp_iu(struct ssp_response_iu *iu,
+static void mvs_fill_ssp_resp_iu(struct ssp_response_iu *iu,
 				u8 key, u8 asc, u8 asc_q)
 {
 	iu->datapres = 2;
diff --git a/drivers/scsi/pas16.c b/drivers/scsi/pas16.c
deleted file mode 100644
index 2f689ae7a803..000000000000
--- a/drivers/scsi/pas16.c
+++ /dev/null
@@ -1,565 +0,0 @@
-/*
- * This driver adapted from Drew Eckhardt's Trantor T128 driver
- *
- * Copyright 1993, Drew Eckhardt
- *	Visionary Computing
- *	(Unix and Linux consulting and custom programming)
- *	drew@colorado.edu
- *      +1 (303) 666-5836
- *
- *  ( Based on T128 - DISTRIBUTION RELEASE 3. ) 
- *
- * Modified to work with the Pro Audio Spectrum/Studio 16
- * by John Weidman.
- *
- *
- * For more information, please consult 
- *
- * Media Vision
- * (510) 770-8600
- * (800) 348-7116
- */
-
-/*
- * The card is detected and initialized in one of several ways : 
- * 1.  Autoprobe (default) - There are many different models of
- *     the Pro Audio Spectrum/Studio 16, and I only have one of
- *     them, so this may require a little tweaking.  An interrupt
- *     is triggered to autoprobe for the interrupt line.  Note:
- *     with the newer model boards, the interrupt is set via
- *     software after reset using the default_irq for the
- *     current board number.
- *
- * 2.  With command line overrides - pas16=port,irq may be 
- *     used on the LILO command line to override the defaults.
- *
- * 3.  With the PAS16_OVERRIDE compile time define.  This is 
- *     specified as an array of address, irq tuples.  Ie, for
- *     one board at the default 0x388 address, IRQ10, I could say 
- *     -DPAS16_OVERRIDE={{0x388, 10}}
- *     NOTE:  Untested.
- *	
- * 4.  When included as a module, with arguments passed on the command line:
- *         pas16_irq=xx		the interrupt
- *         pas16_addr=xx	the port
- *     e.g. "modprobe pas16 pas16_addr=0x388 pas16_irq=5"
- *
- *     Note that if the override methods are used, place holders must
- *     be specified for other boards in the system.
- *
- *
- * Configuration notes :
- *   The current driver does not support interrupt sharing with the
- *   sound portion of the card.  If you use the same irq for the
- *   scsi port and sound you will have problems.  Either use
- *   a different irq for the scsi port or don't use interrupts
- *   for the scsi port.
- *
- *   If you have problems with your card not being recognized, use
- *   the LILO command line override.  Try to get it recognized without
- *   interrupts.  Ie, for a board at the default 0x388 base port,
- *   boot: linux pas16=0x388,0
- *
- *   NO_IRQ (0) should be specified for no interrupt,
- *   IRQ_AUTO (254) to autoprobe for an IRQ line if overridden
- *   on the command line.
- */
- 
-#include <linux/module.h>
-
-#include <asm/io.h>
-#include <asm/dma.h>
-#include <linux/blkdev.h>
-#include <linux/interrupt.h>
-#include <linux/init.h>
-
-#include <scsi/scsi_host.h>
-#include "pas16.h"
-#include "NCR5380.h"
-
-
-static unsigned short pas16_addr;
-static int pas16_irq;
- 
-
-static const int scsi_irq_translate[] =
-	{ 0,  0,  1,  2,  3,  4,  5,  6, 0,  0,  7,  8,  9,  0, 10, 11 };
-
-/* The default_irqs array contains values used to set the irq into the
- * board via software (as must be done on newer model boards without
- * irq jumpers on the board).  The first value in the array will be
- * assigned to logical board 0, the next to board 1, etc.
- */
-static int default_irqs[] __initdata =
-	{  PAS16_DEFAULT_BOARD_1_IRQ,
-	   PAS16_DEFAULT_BOARD_2_IRQ,
-	   PAS16_DEFAULT_BOARD_3_IRQ,
-	   PAS16_DEFAULT_BOARD_4_IRQ
-	};
-
-static struct override {
-    unsigned short io_port;
-    int  irq;
-} overrides
-#ifdef PAS16_OVERRIDE
-    [] __initdata = PAS16_OVERRIDE;
-#else
-    [4] __initdata = {{0,IRQ_AUTO}, {0,IRQ_AUTO}, {0,IRQ_AUTO},
-	{0,IRQ_AUTO}};
-#endif
-
-#define NO_OVERRIDES ARRAY_SIZE(overrides)
-
-static struct base {
-    unsigned short io_port;
-    int noauto;
-} bases[] __initdata =
-	{ {PAS16_DEFAULT_BASE_1, 0},
-	  {PAS16_DEFAULT_BASE_2, 0},
-	  {PAS16_DEFAULT_BASE_3, 0},
-	  {PAS16_DEFAULT_BASE_4, 0}
-	};
-
-#define NO_BASES ARRAY_SIZE(bases)
-
-static const unsigned short  pas16_offset[ 8 ] =
-    {
-	0x1c00,    /* OUTPUT_DATA_REG */
-	0x1c01,    /* INITIATOR_COMMAND_REG */
-	0x1c02,    /* MODE_REG */
-	0x1c03,    /* TARGET_COMMAND_REG */
-	0x3c00,    /* STATUS_REG ro, SELECT_ENABLE_REG wo */
-	0x3c01,    /* BUS_AND_STATUS_REG ro, START_DMA_SEND_REG wo */
-	0x3c02,    /* INPUT_DATA_REGISTER ro, (N/A on PAS16 ?)
-		    * START_DMA_TARGET_RECEIVE_REG wo
-		    */
-	0x3c03,    /* RESET_PARITY_INTERRUPT_REG ro,
-		    * START_DMA_INITIATOR_RECEIVE_REG wo
-		    */
-    };
-
-
-/*
- * Function : enable_board( int  board_num, unsigned short port )
- *
- * Purpose :  set address in new model board
- *
- * Inputs : board_num - logical board number 0-3, port - base address
- *
- */
-
-static void __init
-	enable_board( int  board_num,  unsigned short port )
-{
-    outb( 0xbc + board_num, MASTER_ADDRESS_PTR );
-    outb( port >> 2, MASTER_ADDRESS_PTR );
-}
-
-
-
-/*
- * Function : init_board( unsigned short port, int irq )
- *
- * Purpose :  Set the board up to handle the SCSI interface
- *
- * Inputs : port - base address of the board,
- *	    irq - irq to assign to the SCSI port
- *	    force_irq - set it even if it conflicts with sound driver
- *
- */
-
-static void __init 
-	init_board( unsigned short io_port, int irq, int force_irq )
-{
-	unsigned int	tmp;
-	unsigned int	pas_irq_code;
-
-	/* Initialize the SCSI part of the board */
-
-	outb( 0x30, io_port + P_TIMEOUT_COUNTER_REG );  /* Timeout counter */
-	outb( 0x01, io_port + P_TIMEOUT_STATUS_REG_OFFSET );   /* Reset TC */
-	outb( 0x01, io_port + WAIT_STATE );   /* 1 Wait state */
-
-	inb(io_port + pas16_offset[RESET_PARITY_INTERRUPT_REG]);
-
-	/* Set the SCSI interrupt pointer without mucking up the sound
-	 * interrupt pointer in the same byte.
-	 */
-	pas_irq_code = ( irq < 16 ) ? scsi_irq_translate[irq] : 0;
-	tmp = inb( io_port + IO_CONFIG_3 );
-
-	if( (( tmp & 0x0f ) == pas_irq_code) && pas_irq_code > 0 
-	    && !force_irq )
-	{
-	    printk( "pas16: WARNING: Can't use same irq as sound "
-		    "driver -- interrupts disabled\n" );
-	    /* Set up the drive parameters, disable 5380 interrupts */
-	    outb( 0x4d, io_port + SYS_CONFIG_4 );
-	}
-	else
-	{
-	    tmp = (  tmp & 0x0f ) | ( pas_irq_code << 4 );
-	    outb( tmp, io_port + IO_CONFIG_3 );
-
-	    /* Set up the drive parameters and enable 5380 interrupts */
-	    outb( 0x6d, io_port + SYS_CONFIG_4 );
-	}
-}
-
-
-/*
- * Function : pas16_hw_detect( unsigned short board_num )
- *
- * Purpose : determine if a pas16 board is present
- * 
- * Inputs : board_num - logical board number ( 0 - 3 )
- *
- * Returns : 0 if board not found, 1 if found.
- */
-
-static int __init 
-     pas16_hw_detect( unsigned short  board_num )
-{
-    unsigned char	board_rev, tmp;
-    unsigned short	io_port = bases[ board_num ].io_port;
-
-    /* See if we can find a PAS16 board at the address associated
-     * with this logical board number.
-     */
-
-    /* First, attempt to take a newer model board out of reset and
-     * give it a base address.  This shouldn't affect older boards.
-     */
-    enable_board( board_num, io_port );
-
-    /* Now see if it looks like a PAS16 board */
-    board_rev = inb( io_port + PCB_CONFIG );
-
-    if( board_rev == 0xff )
-	return 0;
-
-    tmp = board_rev ^ 0xe0;
-
-    outb( tmp, io_port + PCB_CONFIG );
-    tmp = inb( io_port + PCB_CONFIG );
-    outb( board_rev, io_port + PCB_CONFIG );
-
-    if( board_rev != tmp ) 	/* Not a PAS-16 */
-	return 0;
-
-    if( ( inb( io_port + OPERATION_MODE_1 ) & 0x03 ) != 0x03 ) 
-	return 0;  	/* return if no SCSI interface found */
-
-    /* Mediavision has some new model boards that return ID bits
-     * that indicate a SCSI interface, but they're not (LMS).  We'll
-     * put in an additional test to try to weed them out.
-     */
-
-	outb(0x01, io_port + WAIT_STATE);             /* 1 Wait state */
-	outb(0x20, io_port + pas16_offset[MODE_REG]); /* Is it really SCSI? */
-	if (inb(io_port + pas16_offset[MODE_REG]) != 0x20) /* Write to a reg. */
-		return 0;                                  /* and try to read */
-	outb(0x00, io_port + pas16_offset[MODE_REG]);      /* it back. */
-	if (inb(io_port + pas16_offset[MODE_REG]) != 0x00)
-		return 0;
-
-    return 1;
-}
-
-
-#ifndef MODULE
-/*
- * Function : pas16_setup(char *str, int *ints)
- *
- * Purpose : LILO command line initialization of the overrides array,
- * 
- * Inputs : str - unused, ints - array of integer parameters with ints[0]
- *	equal to the number of ints.
- *
- */
-
-static int __init pas16_setup(char *str)
-{
-	static int commandline_current;
-    int i;
-    int ints[10];
-
-    get_options(str, ARRAY_SIZE(ints), ints);
-    if (ints[0] != 2) 
-	printk("pas16_setup : usage pas16=io_port,irq\n");
-    else 
-	if (commandline_current < NO_OVERRIDES) {
-	    overrides[commandline_current].io_port = (unsigned short) ints[1];
-	    overrides[commandline_current].irq = ints[2];
-	    for (i = 0; i < NO_BASES; ++i)
-		if (bases[i].io_port == (unsigned short) ints[1]) {
- 		    bases[i].noauto = 1;
-		    break;
-		}
-	    ++commandline_current;
-	}
-    return 1;
-}
-
-__setup("pas16=", pas16_setup);
-#endif
-
-/* 
- * Function : int pas16_detect(struct scsi_host_template * tpnt)
- *
- * Purpose : detects and initializes PAS16 controllers
- *	that were autoprobed, overridden on the LILO command line, 
- *	or specified at compile time.
- *
- * Inputs : tpnt - template for this SCSI adapter.
- * 
- * Returns : 1 if a host adapter was found, 0 if not.
- *
- */
-
-static int __init pas16_detect(struct scsi_host_template *tpnt)
-{
-	static int current_override;
-	static unsigned short current_base;
-    struct Scsi_Host *instance;
-    unsigned short io_port;
-    int  count;
-
-    if (pas16_addr != 0) {
-	overrides[0].io_port = pas16_addr;
-	/*
-	*  This is how we avoid seeing more than
-	*  one host adapter at the same I/O port.
-	*  Cribbed shamelessly from pas16_setup().
-	*/
-	for (count = 0; count < NO_BASES; ++count)
-	    if (bases[count].io_port == pas16_addr) {
- 		    bases[count].noauto = 1;
-		    break;
-	}
-    }
-    if (pas16_irq != 0)
-	overrides[0].irq = pas16_irq;
-
-    for (count = 0; current_override < NO_OVERRIDES; ++current_override) {
-	io_port = 0;
-
-	if (overrides[current_override].io_port)
-	{
-	    io_port = overrides[current_override].io_port;
-	    enable_board( current_override, io_port );
-	    init_board( io_port, overrides[current_override].irq, 1 );
-	}
-	else
-	    for (; !io_port && (current_base < NO_BASES); ++current_base) {
-		dprintk(NDEBUG_INIT, "pas16: probing io_port 0x%04x\n",
-		        (unsigned int)bases[current_base].io_port);
-		if ( !bases[current_base].noauto &&
-		     pas16_hw_detect( current_base ) ){
-			io_port = bases[current_base].io_port;
-			init_board( io_port, default_irqs[ current_base ], 0 ); 
-			dprintk(NDEBUG_INIT, "pas16: detected board\n");
-		}
-    }
-
-	dprintk(NDEBUG_INIT, "pas16: io_port = 0x%04x\n",
-	        (unsigned int)io_port);
-
-	if (!io_port)
-	    break;
-
-	instance = scsi_register (tpnt, sizeof(struct NCR5380_hostdata));
-	if(instance == NULL)
-		goto out;
-		
-	instance->io_port = io_port;
-
-	if (NCR5380_init(instance, FLAG_DMA_FIXUP | FLAG_LATE_DMA_SETUP))
-		goto out_unregister;
-
-	NCR5380_maybe_reset_bus(instance);
-
-	if (overrides[current_override].irq != IRQ_AUTO)
-	    instance->irq = overrides[current_override].irq;
-	else 
-	    instance->irq = NCR5380_probe_irq(instance, PAS16_IRQS);
-
-	/* Compatibility with documented NCR5380 kernel parameters */
-	if (instance->irq == 255)
-		instance->irq = NO_IRQ;
-
-	if (instance->irq != NO_IRQ)
-	    if (request_irq(instance->irq, pas16_intr, 0,
-			    "pas16", instance)) {
-		printk("scsi%d : IRQ%d not free, interrupts disabled\n", 
-		    instance->host_no, instance->irq);
-		instance->irq = NO_IRQ;
-	    } 
-
-	if (instance->irq == NO_IRQ) {
-	    printk("scsi%d : interrupts not enabled. for better interactive performance,\n", instance->host_no);
-	    printk("scsi%d : please jumper the board for a free IRQ.\n", instance->host_no);
-	    /* Disable 5380 interrupts, leave drive params the same */
-	    outb( 0x4d, io_port + SYS_CONFIG_4 );
-	    outb( (inb(io_port + IO_CONFIG_3) & 0x0f), io_port + IO_CONFIG_3 );
-	}
-
-	dprintk(NDEBUG_INIT, "scsi%d : irq = %d\n",
-	        instance->host_no, instance->irq);
-
-	++current_override;
-	++count;
-    }
-    return count;
-
-out_unregister:
-	scsi_unregister(instance);
-out:
-	return count;
-}
-
-/*
- * Function : int pas16_biosparam(Disk *disk, struct block_device *dev, int *ip)
- *
- * Purpose : Generates a BIOS / DOS compatible H-C-S mapping for 
- *	the specified device / size.
- * 
- * Inputs : size = size of device in sectors (512 bytes), dev = block device
- *	major / minor, ip[] = {heads, sectors, cylinders}  
- *
- * Returns : always 0 (success), initializes ip
- *	
- */
-
-/* 
- * XXX Most SCSI boards use this mapping, I could be incorrect.  Some one
- * using hard disks on a trantor should verify that this mapping corresponds
- * to that used by the BIOS / ASPI driver by running the linux fdisk program
- * and matching the H_C_S coordinates to what DOS uses.
- */
-
-static int pas16_biosparam(struct scsi_device *sdev, struct block_device *dev,
-                           sector_t capacity, int *ip)
-{
-  int size = capacity;
-  ip[0] = 64;
-  ip[1] = 32;
-  ip[2] = size >> 11;		/* I think I have it as /(32*64) */
-  if( ip[2] > 1024 ) {		/* yes, >, not >= */
-	ip[0]=255;
-	ip[1]=63;
-	ip[2]=size/(63*255);
-	if( ip[2] > 1023 )	/* yes >1023... */
-		ip[2] = 1023;
-  }
-
-  return 0;
-}
-
-/*
- * Function : int pas16_pread (struct Scsi_Host *instance,
- *	unsigned char *dst, int len)
- *
- * Purpose : Fast 5380 pseudo-dma read function, transfers len bytes to 
- *	dst
- * 
- * Inputs : dst = destination, len = length in bytes
- *
- * Returns : 0 on success, non zero on a failure such as a watchdog 
- * 	timeout.
- */
-
-static inline int pas16_pread(struct Scsi_Host *instance,
-                              unsigned char *dst, int len)
-{
-    register unsigned char  *d = dst;
-    register unsigned short reg = (unsigned short) (instance->io_port + 
-	P_DATA_REG_OFFSET);
-    register int i = len;
-    int ii = 0;
-
-    while ( !(inb(instance->io_port + P_STATUS_REG_OFFSET) & P_ST_RDY) )
-	 ++ii;
-
-    insb( reg, d, i );
-
-    if ( inb(instance->io_port + P_TIMEOUT_STATUS_REG_OFFSET) & P_TS_TIM) {
-	outb( P_TS_CT, instance->io_port + P_TIMEOUT_STATUS_REG_OFFSET);
-	printk("scsi%d : watchdog timer fired in NCR5380_pread()\n",
-	    instance->host_no);
-	return -1;
-    }
-    return 0;
-}
-
-/*
- * Function : int pas16_pwrite (struct Scsi_Host *instance,
- *	unsigned char *src, int len)
- *
- * Purpose : Fast 5380 pseudo-dma write function, transfers len bytes from
- *	src
- * 
- * Inputs : src = source, len = length in bytes
- *
- * Returns : 0 on success, non zero on a failure such as a watchdog 
- * 	timeout.
- */
-
-static inline int pas16_pwrite(struct Scsi_Host *instance,
-                               unsigned char *src, int len)
-{
-    register unsigned char *s = src;
-    register unsigned short reg = (instance->io_port + P_DATA_REG_OFFSET);
-    register int i = len;
-    int ii = 0;
-
-    while ( !((inb(instance->io_port + P_STATUS_REG_OFFSET)) & P_ST_RDY) )
-	 ++ii;
- 
-    outsb( reg, s, i );
-
-    if (inb(instance->io_port + P_TIMEOUT_STATUS_REG_OFFSET) & P_TS_TIM) {
-	outb( P_TS_CT, instance->io_port + P_TIMEOUT_STATUS_REG_OFFSET);
-	printk("scsi%d : watchdog timer fired in NCR5380_pwrite()\n",
-	    instance->host_no);
-	return -1;
-    }
-    return 0;
-}
-
-#include "NCR5380.c"
-
-static int pas16_release(struct Scsi_Host *shost)
-{
-	if (shost->irq != NO_IRQ)
-		free_irq(shost->irq, shost);
-	NCR5380_exit(shost);
-	scsi_unregister(shost);
-	return 0;
-}
-
-static struct scsi_host_template driver_template = {
-	.name			= "Pro Audio Spectrum-16 SCSI",
-	.detect			= pas16_detect,
-	.release		= pas16_release,
-	.proc_name		= "pas16",
-	.info			= pas16_info,
-	.queuecommand		= pas16_queue_command,
-	.eh_abort_handler	= pas16_abort,
-	.eh_bus_reset_handler	= pas16_bus_reset,
-	.bios_param		= pas16_biosparam,
-	.can_queue		= 32,
-	.this_id		= 7,
-	.sg_tablesize		= SG_ALL,
-	.cmd_per_lun		= 2,
-	.use_clustering		= DISABLE_CLUSTERING,
-	.cmd_size		= NCR5380_CMD_SIZE,
-	.max_sectors		= 128,
-};
-#include "scsi_module.c"
-
-#ifdef MODULE
-module_param(pas16_addr, ushort, 0);
-module_param(pas16_irq, int, 0);
-#endif
-MODULE_LICENSE("GPL");
diff --git a/drivers/scsi/pas16.h b/drivers/scsi/pas16.h
deleted file mode 100644
index 9fe7f33660b4..000000000000
--- a/drivers/scsi/pas16.h
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * This driver adapted from Drew Eckhardt's Trantor T128 driver
- *
- * Copyright 1993, Drew Eckhardt
- *	Visionary Computing
- *	(Unix and Linux consulting and custom programming)
- *	drew@colorado.edu
- *      +1 (303) 666-5836
- *
- *  ( Based on T128 - DISTRIBUTION RELEASE 3. ) 
- *
- * Modified to work with the Pro Audio Spectrum/Studio 16
- * by John Weidman.
- *
- *
- * For more information, please consult 
- *
- * Media Vision
- * (510) 770-8600
- * (800) 348-7116
- */
-
-
-#ifndef PAS16_H
-#define PAS16_H
-
-#define PAS16_DEFAULT_BASE_1  0x388
-#define PAS16_DEFAULT_BASE_2  0x384
-#define PAS16_DEFAULT_BASE_3  0x38c
-#define PAS16_DEFAULT_BASE_4  0x288
-
-#define PAS16_DEFAULT_BOARD_1_IRQ 10
-#define PAS16_DEFAULT_BOARD_2_IRQ 12
-#define PAS16_DEFAULT_BOARD_3_IRQ 14
-#define PAS16_DEFAULT_BOARD_4_IRQ 15
-
-
-/*
- * The Pro Audio Spectrum boards are I/O mapped. They use a Zilog 5380
- * SCSI controller, which is the equivalent of NCR's 5380.  "Pseudo-DMA"
- * architecture is used, where a PAL drives the DMA signals on the 5380
- * allowing fast, blind transfers with proper handshaking. 
- */
-
-
-/* The Time-out Counter register is used to safe-guard against a stuck
- * bus (in the case of RDY driven handshake) or a stuck byte (if 16-Bit
- * DMA conversion is used).  The counter uses a 28.224MHz clock
- * divided by 14 as its clock source.  In the case of a stuck byte in
- * the holding register, an interrupt is generated (and mixed with the
- * one with the drive) using the CD-ROM interrupt pointer.
- */
- 
-#define P_TIMEOUT_COUNTER_REG	0x4000
-#define P_TC_DISABLE	0x80	/* Set to 0 to enable timeout int. */
-				/* Bits D6-D0 contain timeout count */
-
-
-#define P_TIMEOUT_STATUS_REG_OFFSET	0x4001
-#define P_TS_TIM		0x80	/* check timeout status */
-					/* Bits D6-D4 N/U */
-#define P_TS_ARM_DRQ_INT	0x08	/* Arm DRQ Int.  When set high,
-					 * the next rising edge will
-					 * cause a CD-ROM interrupt.
-					 * When set low, the interrupt
-					 * will be cleared.  There is
-					 * no status available for
-					 * this interrupt.
-					 */
-#define P_TS_ENABLE_TO_ERR_INTERRUPT	/* Enable timeout error int. */
-#define P_TS_ENABLE_WAIT		/* Enable Wait */
-
-#define P_TS_CT			0x01	/* clear timeout. Note: writing
-					 * to this register clears the
-					 * timeout error int. or status
-					 */
-
-
-/*
- * The data register reads/writes to/from the 5380 in pseudo-DMA mode
- */ 
-
-#define P_DATA_REG_OFFSET	0x5c00	/* rw */
-
-#define P_STATUS_REG_OFFSET	0x5c01	/* ro */
-#define P_ST_RDY		0x80	/* 5380 DDRQ Status */
-
-#define P_IRQ_STATUS		0x5c03
-#define P_IS_IRQ		0x80	/* DIRQ status */
-
-#define PCB_CONFIG 0x803
-#define MASTER_ADDRESS_PTR 0x9a01  /* Fixed position - no relo */
-#define SYS_CONFIG_4 0x8003
-#define WAIT_STATE 0xbc00
-#define OPERATION_MODE_1 0xec03
-#define IO_CONFIG_3 0xf002
-
-#define NCR5380_implementation_fields /* none */
-
-#define PAS16_io_port(reg) (instance->io_port + pas16_offset[(reg)])
-
-#define NCR5380_read(reg) ( inb(PAS16_io_port(reg)) )
-#define NCR5380_write(reg, value) ( outb((value),PAS16_io_port(reg)) )
-
-#define NCR5380_dma_xfer_len(instance, cmd, phase)	(cmd->transfersize)
-#define NCR5380_dma_recv_setup		pas16_pread
-#define NCR5380_dma_send_setup		pas16_pwrite
-#define NCR5380_dma_residual(instance)	(0)
-
-#define NCR5380_intr pas16_intr
-#define NCR5380_queue_command pas16_queue_command
-#define NCR5380_abort pas16_abort
-#define NCR5380_bus_reset pas16_bus_reset
-#define NCR5380_info pas16_info
-
-/* 15 14 12 10 7 5 3 
-   1101 0100 1010 1000 */
-   
-#define PAS16_IRQS 0xd4a8 
-
-#endif /* PAS16_H */
diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c
index 04e67a190652..10546faac58c 100644
--- a/drivers/scsi/pm8001/pm8001_hwi.c
+++ b/drivers/scsi/pm8001/pm8001_hwi.c
@@ -4492,8 +4492,8 @@ pm8001_chip_phy_start_req(struct pm8001_hba_info *pm8001_ha, u8 phy_id)
  * @num: the inbound queue number
  * @phy_id: the phy id which we wanted to start up.
  */
-int pm8001_chip_phy_stop_req(struct pm8001_hba_info *pm8001_ha,
-	u8 phy_id)
+static int pm8001_chip_phy_stop_req(struct pm8001_hba_info *pm8001_ha,
+				    u8 phy_id)
 {
 	struct phy_stop_req payload;
 	struct inbound_queue_table *circularQ;
diff --git a/drivers/scsi/pm8001/pm8001_sas.c b/drivers/scsi/pm8001/pm8001_sas.c
index dc33dfa8f994..ce584c31d36e 100644
--- a/drivers/scsi/pm8001/pm8001_sas.c
+++ b/drivers/scsi/pm8001/pm8001_sas.c
@@ -527,7 +527,7 @@ void pm8001_ccb_task_free(struct pm8001_hba_info *pm8001_ha,
   * pm8001_alloc_dev - find a empty pm8001_device
   * @pm8001_ha: our hba card information
   */
-struct pm8001_device *pm8001_alloc_dev(struct pm8001_hba_info *pm8001_ha)
+static struct pm8001_device *pm8001_alloc_dev(struct pm8001_hba_info *pm8001_ha)
 {
 	u32 dev;
 	for (dev = 0; dev < PM8001_MAX_DEVICES; dev++) {
diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c
index b2a88200fe54..68a5c347fae9 100644
--- a/drivers/scsi/pmcraid.c
+++ b/drivers/scsi/pmcraid.c
@@ -306,7 +306,7 @@ static int pmcraid_change_queue_depth(struct scsi_device *scsi_dev, int depth)
  * Return Value
  *	 None
  */
-void pmcraid_init_cmdblk(struct pmcraid_cmd *cmd, int index)
+static void pmcraid_init_cmdblk(struct pmcraid_cmd *cmd, int index)
 {
 	struct pmcraid_ioarcb *ioarcb = &(cmd->ioa_cb->ioarcb);
 	dma_addr_t dma_addr = cmd->ioa_cb_bus_addr;
@@ -401,7 +401,7 @@ static struct pmcraid_cmd *pmcraid_get_free_cmd(
  * Return Value:
  *	nothing
  */
-void pmcraid_return_cmd(struct pmcraid_cmd *cmd)
+static void pmcraid_return_cmd(struct pmcraid_cmd *cmd)
 {
 	struct pmcraid_instance *pinstance = cmd->drv_inst;
 	unsigned long lock_flags;
@@ -1710,7 +1710,7 @@ static struct pmcraid_ioasc_error *pmcraid_get_error_info(u32 ioasc)
  * @ioasc: ioasc code
  * @cmd: pointer to command that resulted in 'ioasc'
  */
-void pmcraid_ioasc_logger(u32 ioasc, struct pmcraid_cmd *cmd)
+static void pmcraid_ioasc_logger(u32 ioasc, struct pmcraid_cmd *cmd)
 {
 	struct pmcraid_ioasc_error *error_info = pmcraid_get_error_info(ioasc);
 
@@ -3137,7 +3137,7 @@ static int pmcraid_eh_host_reset_handler(struct scsi_cmnd *scmd)
  *   returns pointer pmcraid_ioadl_desc, initialized to point to internal
  *   or external IOADLs
  */
-struct pmcraid_ioadl_desc *
+static struct pmcraid_ioadl_desc *
 pmcraid_init_ioadls(struct pmcraid_cmd *cmd, int sgcount)
 {
 	struct pmcraid_ioadl_desc *ioadl;
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index ae4a74756128..73b12e41d992 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -279,16 +279,6 @@ struct req_que;
 struct qla_tgt_sess;
 
 /*
- * (sd.h is not exported, hence local inclusion)
- * Data Integrity Field tuple.
- */
-struct sd_dif_tuple {
-	__be16 guard_tag;	/* Checksum */
-	__be16 app_tag;		/* Opaque storage */
-	__be32 ref_tag;		/* Target LBA or indirect LBA */
-};
-
-/*
  * SCSI Request Block
  */
 struct srb_cmd {
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 987f1c729e9c..068c4e47fac9 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -1828,7 +1828,7 @@ qla2x00_handle_dif_error(srb_t *sp, struct sts_entry_24xx *sts24)
 		if (scsi_prot_sg_count(cmd)) {
 			uint32_t i, j = 0, k = 0, num_ent;
 			struct scatterlist *sg;
-			struct sd_dif_tuple *spt;
+			struct t10_pi_tuple *spt;
 
 			/* Patch the corresponding protection tags */
 			scsi_for_each_prot_sg(cmd, sg,
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 2674f4c16bc3..ace65db1d2a2 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -899,12 +899,12 @@ qla2x00_wait_for_hba_ready(scsi_qla_host_t *vha)
 	struct qla_hw_data *ha = vha->hw;
 	scsi_qla_host_t *base_vha = pci_get_drvdata(ha->pdev);
 
-	while (((qla2x00_reset_active(vha)) || ha->dpc_active ||
-	    ha->flags.mbox_busy) ||
-		test_bit(FX00_RESET_RECOVERY, &vha->dpc_flags) ||
-		test_bit(FX00_TARGET_SCAN, &vha->dpc_flags)) {
-			if (test_bit(UNLOADING, &base_vha->dpc_flags))
-				break;
+	while ((qla2x00_reset_active(vha) || ha->dpc_active ||
+		ha->flags.mbox_busy) ||
+	       test_bit(FX00_RESET_RECOVERY, &vha->dpc_flags) ||
+	       test_bit(FX00_TARGET_SCAN, &vha->dpc_flags)) {
+		if (test_bit(UNLOADING, &base_vha->dpc_flags))
+			break;
 		msleep(1000);
 	}
 }
@@ -4694,7 +4694,7 @@ retry_unlock:
 			qla83xx_wait_logic();
 			retry++;
 			ql_dbg(ql_dbg_p3p, base_vha, 0xb064,
-			    "Failed to release IDC lock, retyring=%d\n", retry);
+			    "Failed to release IDC lock, retrying=%d\n", retry);
 			goto retry_unlock;
 		}
 	} else if (retry < 10) {
@@ -4702,7 +4702,7 @@ retry_unlock:
 		qla83xx_wait_logic();
 		retry++;
 		ql_dbg(ql_dbg_p3p, base_vha, 0xb065,
-		    "Failed to read drv-lockid, retyring=%d\n", retry);
+		    "Failed to read drv-lockid, retrying=%d\n", retry);
 		goto retry_unlock;
 	}
 
@@ -4718,7 +4718,7 @@ retry_unlock2:
 			qla83xx_wait_logic();
 			retry++;
 			ql_dbg(ql_dbg_p3p, base_vha, 0xb066,
-			    "Failed to release IDC lock, retyring=%d\n", retry);
+			    "Failed to release IDC lock, retrying=%d\n", retry);
 			goto retry_unlock2;
 		}
 	}
diff --git a/drivers/scsi/qla4xxx/ql4_nx.c b/drivers/scsi/qla4xxx/ql4_nx.c
index ae87d6c19f17..06ddd13cb7cc 100644
--- a/drivers/scsi/qla4xxx/ql4_nx.c
+++ b/drivers/scsi/qla4xxx/ql4_nx.c
@@ -1843,7 +1843,7 @@ static uint32_t ql4_84xx_poll_wait_for_ready(struct scsi_qla_host *ha,
 	return rval;
 }
 
-uint32_t ql4_84xx_ipmdio_rd_reg(struct scsi_qla_host *ha, uint32_t addr1,
+static uint32_t ql4_84xx_ipmdio_rd_reg(struct scsi_qla_host *ha, uint32_t addr1,
 				uint32_t addr3, uint32_t mask, uint32_t addr,
 				uint32_t *data_ptr)
 {
diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
index 1f36aca44394..1deb6adc411f 100644
--- a/drivers/scsi/scsi.c
+++ b/drivers/scsi/scsi.c
@@ -1160,7 +1160,6 @@ bool scsi_use_blk_mq = true;
 bool scsi_use_blk_mq = false;
 #endif
 module_param_named(use_blk_mq, scsi_use_blk_mq, bool, S_IWUSR | S_IRUGO);
-EXPORT_SYMBOL_GPL(scsi_use_blk_mq);
 
 static int __init init_scsi(void)
 {
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 6a219a0844d3..c905709707f0 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -42,6 +42,7 @@
 #include <linux/atomic.h>
 #include <linux/hrtimer.h>
 #include <linux/uuid.h>
+#include <linux/t10-pi.h>
 
 #include <net/checksum.h>
 
@@ -627,7 +628,7 @@ static LIST_HEAD(sdebug_host_list);
 static DEFINE_SPINLOCK(sdebug_host_list_lock);
 
 static unsigned char *fake_storep;	/* ramdisk storage */
-static struct sd_dif_tuple *dif_storep;	/* protection info */
+static struct t10_pi_tuple *dif_storep;	/* protection info */
 static void *map_storep;		/* provisioning map */
 
 static unsigned long map_size;
@@ -682,7 +683,7 @@ static void *fake_store(unsigned long long lba)
 	return fake_storep + lba * sdebug_sector_size;
 }
 
-static struct sd_dif_tuple *dif_store(sector_t sector)
+static struct t10_pi_tuple *dif_store(sector_t sector)
 {
 	sector = sector_div(sector, sdebug_store_sectors);
 
@@ -1349,7 +1350,7 @@ static int resp_inquiry(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		} else if (0x86 == cmd[2]) { /* extended inquiry */
 			arr[1] = cmd[2];	/*sanity */
 			arr[3] = 0x3c;	/* number of following entries */
-			if (sdebug_dif == SD_DIF_TYPE3_PROTECTION)
+			if (sdebug_dif == T10_PI_TYPE3_PROTECTION)
 				arr[4] = 0x4;	/* SPT: GRD_CHK:1 */
 			else if (have_dif_prot)
 				arr[4] = 0x5;   /* SPT: GRD_CHK:1, REF_CHK:1 */
@@ -2430,7 +2431,7 @@ static __be16 dif_compute_csum(const void *buf, int len)
 	return csum;
 }
 
-static int dif_verify(struct sd_dif_tuple *sdt, const void *data,
+static int dif_verify(struct t10_pi_tuple *sdt, const void *data,
 		      sector_t sector, u32 ei_lba)
 {
 	__be16 csum = dif_compute_csum(data, sdebug_sector_size);
@@ -2442,13 +2443,13 @@ static int dif_verify(struct sd_dif_tuple *sdt, const void *data,
 			be16_to_cpu(csum));
 		return 0x01;
 	}
-	if (sdebug_dif == SD_DIF_TYPE1_PROTECTION &&
+	if (sdebug_dif == T10_PI_TYPE1_PROTECTION &&
 	    be32_to_cpu(sdt->ref_tag) != (sector & 0xffffffff)) {
 		pr_err("REF check failed on sector %lu\n",
 			(unsigned long)sector);
 		return 0x03;
 	}
-	if (sdebug_dif == SD_DIF_TYPE2_PROTECTION &&
+	if (sdebug_dif == T10_PI_TYPE2_PROTECTION &&
 	    be32_to_cpu(sdt->ref_tag) != ei_lba) {
 		pr_err("REF check failed on sector %lu\n",
 			(unsigned long)sector);
@@ -2504,7 +2505,7 @@ static int prot_verify_read(struct scsi_cmnd *SCpnt, sector_t start_sec,
 			    unsigned int sectors, u32 ei_lba)
 {
 	unsigned int i;
-	struct sd_dif_tuple *sdt;
+	struct t10_pi_tuple *sdt;
 	sector_t sector;
 
 	for (i = 0; i < sectors; i++, ei_lba++) {
@@ -2580,13 +2581,13 @@ static int resp_read_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		break;
 	}
 	if (unlikely(have_dif_prot && check_prot)) {
-		if (sdebug_dif == SD_DIF_TYPE2_PROTECTION &&
+		if (sdebug_dif == T10_PI_TYPE2_PROTECTION &&
 		    (cmd[1] & 0xe0)) {
 			mk_sense_invalid_opcode(scp);
 			return check_condition_result;
 		}
-		if ((sdebug_dif == SD_DIF_TYPE1_PROTECTION ||
-		     sdebug_dif == SD_DIF_TYPE3_PROTECTION) &&
+		if ((sdebug_dif == T10_PI_TYPE1_PROTECTION ||
+		     sdebug_dif == T10_PI_TYPE3_PROTECTION) &&
 		    (cmd[1] & 0xe0) == 0)
 			sdev_printk(KERN_ERR, scp->device, "Unprotected RD "
 				    "to DIF device\n");
@@ -2696,7 +2697,7 @@ static int prot_verify_write(struct scsi_cmnd *SCpnt, sector_t start_sec,
 			     unsigned int sectors, u32 ei_lba)
 {
 	int ret;
-	struct sd_dif_tuple *sdt;
+	struct t10_pi_tuple *sdt;
 	void *daddr;
 	sector_t sector = start_sec;
 	int ppage_offset;
@@ -2722,7 +2723,7 @@ static int prot_verify_write(struct scsi_cmnd *SCpnt, sector_t start_sec,
 		}
 
 		for (ppage_offset = 0; ppage_offset < piter.length;
-		     ppage_offset += sizeof(struct sd_dif_tuple)) {
+		     ppage_offset += sizeof(struct t10_pi_tuple)) {
 			/* If we're at the end of the current
 			 * data page advance to the next one
 			 */
@@ -2893,13 +2894,13 @@ static int resp_write_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		break;
 	}
 	if (unlikely(have_dif_prot && check_prot)) {
-		if (sdebug_dif == SD_DIF_TYPE2_PROTECTION &&
+		if (sdebug_dif == T10_PI_TYPE2_PROTECTION &&
 		    (cmd[1] & 0xe0)) {
 			mk_sense_invalid_opcode(scp);
 			return check_condition_result;
 		}
-		if ((sdebug_dif == SD_DIF_TYPE1_PROTECTION ||
-		     sdebug_dif == SD_DIF_TYPE3_PROTECTION) &&
+		if ((sdebug_dif == T10_PI_TYPE1_PROTECTION ||
+		     sdebug_dif == T10_PI_TYPE3_PROTECTION) &&
 		    (cmd[1] & 0xe0) == 0)
 			sdev_printk(KERN_ERR, scp->device, "Unprotected WR "
 				    "to DIF device\n");
@@ -3135,13 +3136,13 @@ static int resp_comp_write(struct scsi_cmnd *scp,
 	num = cmd[13];		/* 1 to a maximum of 255 logical blocks */
 	if (0 == num)
 		return 0;	/* degenerate case, not an error */
-	if (sdebug_dif == SD_DIF_TYPE2_PROTECTION &&
+	if (sdebug_dif == T10_PI_TYPE2_PROTECTION &&
 	    (cmd[1] & 0xe0)) {
 		mk_sense_invalid_opcode(scp);
 		return check_condition_result;
 	}
-	if ((sdebug_dif == SD_DIF_TYPE1_PROTECTION ||
-	     sdebug_dif == SD_DIF_TYPE3_PROTECTION) &&
+	if ((sdebug_dif == T10_PI_TYPE1_PROTECTION ||
+	     sdebug_dif == T10_PI_TYPE3_PROTECTION) &&
 	    (cmd[1] & 0xe0) == 0)
 		sdev_printk(KERN_ERR, scp->device, "Unprotected WR "
 			    "to DIF device\n");
@@ -4939,12 +4940,11 @@ static int __init scsi_debug_init(void)
 	}
 
 	switch (sdebug_dif) {
-
-	case SD_DIF_TYPE0_PROTECTION:
+	case T10_PI_TYPE0_PROTECTION:
 		break;
-	case SD_DIF_TYPE1_PROTECTION:
-	case SD_DIF_TYPE2_PROTECTION:
-	case SD_DIF_TYPE3_PROTECTION:
+	case T10_PI_TYPE1_PROTECTION:
+	case T10_PI_TYPE2_PROTECTION:
+	case T10_PI_TYPE3_PROTECTION:
 		have_dif_prot = true;
 		break;
 
@@ -5026,7 +5026,7 @@ static int __init scsi_debug_init(void)
 	if (sdebug_dix) {
 		int dif_size;
 
-		dif_size = sdebug_store_sectors * sizeof(struct sd_dif_tuple);
+		dif_size = sdebug_store_sectors * sizeof(struct t10_pi_tuple);
 		dif_storep = vmalloc(dif_size);
 
 		pr_err("dif_storep %u bytes @ %p\n", dif_size, dif_storep);
@@ -5480,19 +5480,19 @@ static int sdebug_driver_probe(struct device * dev)
 
 	switch (sdebug_dif) {
 
-	case SD_DIF_TYPE1_PROTECTION:
+	case T10_PI_TYPE1_PROTECTION:
 		hprot = SHOST_DIF_TYPE1_PROTECTION;
 		if (sdebug_dix)
 			hprot |= SHOST_DIX_TYPE1_PROTECTION;
 		break;
 
-	case SD_DIF_TYPE2_PROTECTION:
+	case T10_PI_TYPE2_PROTECTION:
 		hprot = SHOST_DIF_TYPE2_PROTECTION;
 		if (sdebug_dix)
 			hprot |= SHOST_DIX_TYPE2_PROTECTION;
 		break;
 
-	case SD_DIF_TYPE3_PROTECTION:
+	case T10_PI_TYPE3_PROTECTION:
 		hprot = SHOST_DIF_TYPE3_PROTECTION;
 		if (sdebug_dix)
 			hprot |= SHOST_DIX_TYPE3_PROTECTION;
diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c
index eaccd651ccda..246456925335 100644
--- a/drivers/scsi/scsi_devinfo.c
+++ b/drivers/scsi/scsi_devinfo.c
@@ -246,6 +246,10 @@ static struct {
 	{"IBM", "Universal Xport", "*", BLIST_NO_ULD_ATTACH},
 	{"SUN", "Universal Xport", "*", BLIST_NO_ULD_ATTACH},
 	{"DELL", "Universal Xport", "*", BLIST_NO_ULD_ATTACH},
+	{"STK", "Universal Xport", "*", BLIST_NO_ULD_ATTACH},
+	{"NETAPP", "Universal Xport", "*", BLIST_NO_ULD_ATTACH},
+	{"LSI", "Universal Xport", "*", BLIST_NO_ULD_ATTACH},
+	{"ENGENIO", "Universal Xport", "*", BLIST_NO_ULD_ATTACH},
 	{"SMSC", "USB 2 HS-CF", NULL, BLIST_SPARSELUN | BLIST_INQUIRY_36},
 	{"SONY", "CD-ROM CDU-8001", NULL, BLIST_BORKEN},
 	{"SONY", "TSL", NULL, BLIST_FORCELUN},		/* DDS3 & DDS4 autoloaders */
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index c71344aebdbb..2cca9cffc63f 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -2077,7 +2077,6 @@ struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
 }
 
 static struct blk_mq_ops scsi_mq_ops = {
-	.map_queue	= blk_mq_map_queue,
 	.queue_rq	= scsi_queue_rq,
 	.complete	= scsi_softirq_done,
 	.timeout	= scsi_timeout,
diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h
index 57a4b9973320..193636a59adf 100644
--- a/drivers/scsi/scsi_priv.h
+++ b/drivers/scsi/scsi_priv.h
@@ -29,6 +29,7 @@ extern int scsi_init_hosts(void);
 extern void scsi_exit_hosts(void);
 
 /* scsi.c */
+extern bool scsi_use_blk_mq;
 extern int scsi_setup_command_freelist(struct Scsi_Host *shost);
 extern void scsi_destroy_command_freelist(struct Scsi_Host *shost);
 #ifdef CONFIG_SCSI_LOGGING
@@ -85,12 +86,14 @@ extern void scsi_device_unbusy(struct scsi_device *sdev);
 extern void scsi_queue_insert(struct scsi_cmnd *cmd, int reason);
 extern void scsi_io_completion(struct scsi_cmnd *, unsigned int);
 extern void scsi_run_host_queues(struct Scsi_Host *shost);
+extern void scsi_requeue_run_queue(struct work_struct *work);
 extern struct request_queue *scsi_alloc_queue(struct scsi_device *sdev);
 extern struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev);
 extern int scsi_mq_setup_tags(struct Scsi_Host *shost);
 extern void scsi_mq_destroy_tags(struct Scsi_Host *shost);
 extern int scsi_init_queue(void);
 extern void scsi_exit_queue(void);
+extern void scsi_evt_thread(struct work_struct *work);
 struct request_queue;
 struct request;
 extern struct kmem_cache *scsi_sdb_cache;
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index e0a78f53d809..212e98d940bc 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -217,8 +217,6 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget,
 	struct scsi_device *sdev;
 	int display_failure_msg = 1, ret;
 	struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
-	extern void scsi_evt_thread(struct work_struct *work);
-	extern void scsi_requeue_run_queue(struct work_struct *work);
 
 	sdev = kzalloc(sizeof(*sdev) + shost->transportt->device_size,
 		       GFP_ATOMIC);
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index 3f0ff072184b..60b651bfaa01 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -341,22 +341,6 @@ static int do_sas_phy_delete(struct device *dev, void *data)
 }
 
 /**
- * is_sas_attached - check if device is SAS attached
- * @sdev: scsi device to check
- *
- * returns true if the device is SAS attached
- */
-int is_sas_attached(struct scsi_device *sdev)
-{
-	struct Scsi_Host *shost = sdev->host;
-
-	return shost->transportt->host_attrs.ac.class ==
-		&sas_host_class.class;
-}
-EXPORT_SYMBOL(is_sas_attached);
-
-
-/**
  * sas_remove_children  -  tear down a devices SAS data structures
  * @dev:	device belonging to the sas object
  *
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index d3e852ad5aa3..51e56296f465 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -52,6 +52,7 @@
 #include <linux/slab.h>
 #include <linux/pm_runtime.h>
 #include <linux/pr.h>
+#include <linux/t10-pi.h>
 #include <asm/uaccess.h>
 #include <asm/unaligned.h>
 
@@ -314,7 +315,7 @@ protection_type_store(struct device *dev, struct device_attribute *attr,
 	if (err)
 		return err;
 
-	if (val >= 0 && val <= SD_DIF_TYPE3_PROTECTION)
+	if (val >= 0 && val <= T10_PI_TYPE3_PROTECTION)
 		sdkp->protection_type = val;
 
 	return count;
@@ -332,7 +333,7 @@ protection_mode_show(struct device *dev, struct device_attribute *attr,
 	dif = scsi_host_dif_capable(sdp->host, sdkp->protection_type);
 	dix = scsi_host_dix_capable(sdp->host, sdkp->protection_type);
 
-	if (!dix && scsi_host_dix_capable(sdp->host, SD_DIF_TYPE0_PROTECTION)) {
+	if (!dix && scsi_host_dix_capable(sdp->host, T10_PI_TYPE0_PROTECTION)) {
 		dif = 0;
 		dix = 1;
 	}
@@ -608,7 +609,7 @@ static unsigned char sd_setup_protect_cmnd(struct scsi_cmnd *scmd,
 			scmd->prot_flags |= SCSI_PROT_GUARD_CHECK;
 	}
 
-	if (dif != SD_DIF_TYPE3_PROTECTION) {	/* DIX/DIF Type 0, 1, 2 */
+	if (dif != T10_PI_TYPE3_PROTECTION) {	/* DIX/DIF Type 0, 1, 2 */
 		scmd->prot_flags |= SCSI_PROT_REF_INCREMENT;
 
 		if (bio_integrity_flagged(bio, BIP_CTRL_NOCHECK) == false)
@@ -1031,7 +1032,7 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt)
 	else
 		protect = 0;
 
-	if (protect && sdkp->protection_type == SD_DIF_TYPE2_PROTECTION) {
+	if (protect && sdkp->protection_type == T10_PI_TYPE2_PROTECTION) {
 		SCpnt->cmnd = mempool_alloc(sd_cdb_pool, GFP_ATOMIC);
 
 		if (unlikely(SCpnt->cmnd == NULL)) {
@@ -1997,7 +1998,7 @@ static int sd_read_protection_type(struct scsi_disk *sdkp, unsigned char *buffer
 
 	type = ((buffer[12] >> 1) & 7) + 1; /* P_TYPE 0 = Type 1 */
 
-	if (type > SD_DIF_TYPE3_PROTECTION)
+	if (type > T10_PI_TYPE3_PROTECTION)
 		ret = -ENODEV;
 	else if (scsi_host_dif_capable(sdp->host, type))
 		ret = 1;
diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h
index 765a6f1ac1b7..c8d986368da9 100644
--- a/drivers/scsi/sd.h
+++ b/drivers/scsi/sd.h
@@ -157,27 +157,6 @@ static inline unsigned int logical_to_bytes(struct scsi_device *sdev, sector_t b
 }
 
 /*
- * A DIF-capable target device can be formatted with different
- * protection schemes.  Currently 0 through 3 are defined:
- *
- * Type 0 is regular (unprotected) I/O
- *
- * Type 1 defines the contents of the guard and reference tags
- *
- * Type 2 defines the contents of the guard and reference tags and
- * uses 32-byte commands to seed the latter
- *
- * Type 3 defines the contents of the guard tag only
- */
-
-enum sd_dif_target_protection_types {
-	SD_DIF_TYPE0_PROTECTION = 0x0,
-	SD_DIF_TYPE1_PROTECTION = 0x1,
-	SD_DIF_TYPE2_PROTECTION = 0x2,
-	SD_DIF_TYPE3_PROTECTION = 0x3,
-};
-
-/*
  * Look up the DIX operation based on whether the command is read or
  * write and whether dix and dif are enabled.
  */
@@ -239,15 +218,6 @@ static inline unsigned int sd_prot_flag_mask(unsigned int prot_op)
 	return flag_mask[prot_op];
 }
 
-/*
- * Data Integrity Field tuple.
- */
-struct sd_dif_tuple {
-       __be16 guard_tag;	/* Checksum */
-       __be16 app_tag;		/* Opaque storage */
-       __be32 ref_tag;		/* Target LBA or indirect LBA */
-};
-
 #ifdef CONFIG_BLK_DEV_INTEGRITY
 
 extern void sd_dif_config_host(struct scsi_disk *);
diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c
index 987bf392c336..9035380c0dda 100644
--- a/drivers/scsi/sd_dif.c
+++ b/drivers/scsi/sd_dif.c
@@ -60,14 +60,14 @@ void sd_dif_config_host(struct scsi_disk *sdkp)
 
 	/* Enable DMA of protection information */
 	if (scsi_host_get_guard(sdkp->device->host) & SHOST_DIX_GUARD_IP) {
-		if (type == SD_DIF_TYPE3_PROTECTION)
+		if (type == T10_PI_TYPE3_PROTECTION)
 			bi.profile = &t10_pi_type3_ip;
 		else
 			bi.profile = &t10_pi_type1_ip;
 
 		bi.flags |= BLK_INTEGRITY_IP_CHECKSUM;
 	} else
-		if (type == SD_DIF_TYPE3_PROTECTION)
+		if (type == T10_PI_TYPE3_PROTECTION)
 			bi.profile = &t10_pi_type3_crc;
 		else
 			bi.profile = &t10_pi_type1_crc;
@@ -82,7 +82,7 @@ void sd_dif_config_host(struct scsi_disk *sdkp)
 		if (!sdkp->ATO)
 			goto out;
 
-		if (type == SD_DIF_TYPE3_PROTECTION)
+		if (type == T10_PI_TYPE3_PROTECTION)
 			bi.tag_size = sizeof(u16) + sizeof(u32);
 		else
 			bi.tag_size = sizeof(u16);
@@ -121,7 +121,7 @@ void sd_dif_prepare(struct scsi_cmnd *scmd)
 
 	sdkp = scsi_disk(scmd->request->rq_disk);
 
-	if (sdkp->protection_type == SD_DIF_TYPE3_PROTECTION)
+	if (sdkp->protection_type == T10_PI_TYPE3_PROTECTION)
 		return;
 
 	phys = scsi_prot_ref_tag(scmd);
@@ -172,7 +172,7 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes)
 
 	sdkp = scsi_disk(scmd->request->rq_disk);
 
-	if (sdkp->protection_type == SD_DIF_TYPE3_PROTECTION || good_bytes == 0)
+	if (sdkp->protection_type == T10_PI_TYPE3_PROTECTION || good_bytes == 0)
 		return;
 
 	intervals = good_bytes / scsi_prot_interval(scmd);
diff --git a/drivers/scsi/ses.c b/drivers/scsi/ses.c
index 53ef1cb6418e..8c9a35c91705 100644
--- a/drivers/scsi/ses.c
+++ b/drivers/scsi/ses.c
@@ -587,7 +587,7 @@ static void ses_match_to_enclosure(struct enclosure_device *edev,
 
 	ses_enclosure_data_process(edev, to_scsi_device(edev->edev.parent), 0);
 
-	if (is_sas_attached(sdev))
+	if (scsi_is_sas_rphy(&sdev->sdev_gendev))
 		efd.addr = sas_get_address(sdev);
 
 	if (efd.addr) {
@@ -778,6 +778,8 @@ static void ses_intf_remove_enclosure(struct scsi_device *sdev)
 	if (!edev)
 		return;
 
+	enclosure_unregister(edev);
+
 	ses_dev = edev->scratch;
 	edev->scratch = NULL;
 
@@ -789,7 +791,6 @@ static void ses_intf_remove_enclosure(struct scsi_device *sdev)
 	kfree(edev->component[0].scratch);
 
 	put_device(&edev->edev);
-	enclosure_unregister(edev);
 }
 
 static void ses_intf_remove(struct device *cdev,
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index ae7d9bdf409c..070332eb41f3 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -79,18 +79,7 @@ static void sg_proc_cleanup(void);
  */
 #define SG_MAX_CDB_SIZE 252
 
-/*
- * Suppose you want to calculate the formula muldiv(x,m,d)=int(x * m / d)
- * Then when using 32 bit integers x * m may overflow during the calculation.
- * Replacing muldiv(x) by muldiv(x)=((x % d) * m) / d + int(x / d) * m
- * calculates the same, but prevents the overflow when both m and d
- * are "small" numbers (like HZ and USER_HZ).
- * Of course an overflow is inavoidable if the result of muldiv doesn't fit
- * in 32 bits.
- */
-#define MULDIV(X,MUL,DIV) ((((X % DIV) * MUL) / DIV) + ((X / DIV) * MUL))
-
-#define SG_DEFAULT_TIMEOUT MULDIV(SG_DEFAULT_TIMEOUT_USER, HZ, USER_HZ)
+#define SG_DEFAULT_TIMEOUT mult_frac(SG_DEFAULT_TIMEOUT_USER, HZ, USER_HZ)
 
 int sg_big_buff = SG_DEF_RESERVED_SIZE;
 /* N.B. This variable is readable and writeable via
@@ -884,10 +873,11 @@ sg_ioctl(struct file *filp, unsigned int cmd_in, unsigned long arg)
 			return result;
 		if (val < 0)
 			return -EIO;
-		if (val >= MULDIV (INT_MAX, USER_HZ, HZ))
-		    val = MULDIV (INT_MAX, USER_HZ, HZ);
+		if (val >= mult_frac((s64)INT_MAX, USER_HZ, HZ))
+			val = min_t(s64, mult_frac((s64)INT_MAX, USER_HZ, HZ),
+				    INT_MAX);
 		sfp->timeout_user = val;
-		sfp->timeout = MULDIV (val, HZ, USER_HZ);
+		sfp->timeout = mult_frac(val, HZ, USER_HZ);
 
 		return 0;
 	case SG_GET_TIMEOUT:	/* N.B. User receives timeout as return value */
diff --git a/drivers/scsi/smartpqi/Kconfig b/drivers/scsi/smartpqi/Kconfig
new file mode 100644
index 000000000000..97e159c2cecd
--- /dev/null
+++ b/drivers/scsi/smartpqi/Kconfig
@@ -0,0 +1,54 @@
+#
+# Kernel configuration file for the SMARTPQI
+#
+# Copyright (c) 2016 Microsemi Corporation
+# Copyright (c) 2016 PMC-Sierra, Inc.
+#  (mailto:esc.storagedev@microsemi.com)
+
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# NO WARRANTY
+# THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
+# LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
+# MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
+# solely responsible for determining the appropriateness of using and
+# distributing the Program and assumes all risks associated with its
+# exercise of rights under this Agreement, including but not limited to
+# the risks and costs of program errors, damage to or loss of data,
+# programs or equipment, and unavailability or interruption of operations.
+
+# DISCLAIMER OF LIABILITY
+# NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
+# HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
+
+config SCSI_SMARTPQI
+	tristate "Microsemi PQI Driver"
+	depends on PCI && SCSI && !S390
+	select SCSI_SAS_ATTRS
+	select RAID_ATTRS
+	---help---
+	This driver supports Microsemi PQI controllers.
+
+	<http://www.microsemi.com>
+
+	To compile this driver as a module, choose M here: the
+	module will be called smartpqi.
+
+        Note: the aacraid driver will not manage a smartpqi
+              controller. You need to enable smartpqi for smartpqi
+              controllers. For more information, please see
+              Documentation/scsi/smartpqi.txt
diff --git a/drivers/scsi/smartpqi/Makefile b/drivers/scsi/smartpqi/Makefile
new file mode 100644
index 000000000000..0f42a225a664
--- /dev/null
+++ b/drivers/scsi/smartpqi/Makefile
@@ -0,0 +1,3 @@
+ccflags-y += -I.
+obj-m		+= smartpqi.o
+smartpqi-objs := smartpqi_init.o smartpqi_sis.o smartpqi_sas_transport.o
diff --git a/drivers/scsi/smartpqi/smartpqi.h b/drivers/scsi/smartpqi/smartpqi.h
new file mode 100644
index 000000000000..07b6444d3e0a
--- /dev/null
+++ b/drivers/scsi/smartpqi/smartpqi.h
@@ -0,0 +1,1136 @@
+/*
+ *    driver for Microsemi PQI-based storage controllers
+ *    Copyright (c) 2016 Microsemi Corporation
+ *    Copyright (c) 2016 PMC-Sierra, Inc.
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; version 2 of the License.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *    NON INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *
+ */
+
+#if !defined(_SMARTPQI_H)
+#define _SMARTPQI_H
+
+#pragma pack(1)
+
+#define PQI_DEVICE_SIGNATURE	"PQI DREG"
+
+/* This structure is defined by the PQI specification. */
+struct pqi_device_registers {
+	__le64	signature;
+	u8	function_and_status_code;
+	u8	reserved[7];
+	u8	max_admin_iq_elements;
+	u8	max_admin_oq_elements;
+	u8	admin_iq_element_length;	/* in 16-byte units */
+	u8	admin_oq_element_length;	/* in 16-byte units */
+	__le16	max_reset_timeout;		/* in 100-millisecond units */
+	u8	reserved1[2];
+	__le32	legacy_intx_status;
+	__le32	legacy_intx_mask_set;
+	__le32	legacy_intx_mask_clear;
+	u8	reserved2[28];
+	__le32	device_status;
+	u8	reserved3[4];
+	__le64	admin_iq_pi_offset;
+	__le64	admin_oq_ci_offset;
+	__le64	admin_iq_element_array_addr;
+	__le64	admin_oq_element_array_addr;
+	__le64	admin_iq_ci_addr;
+	__le64	admin_oq_pi_addr;
+	u8	admin_iq_num_elements;
+	u8	admin_oq_num_elements;
+	__le16	admin_queue_int_msg_num;
+	u8	reserved4[4];
+	__le32	device_error;
+	u8	reserved5[4];
+	__le64	error_details;
+	__le32	device_reset;
+	__le32	power_action;
+	u8	reserved6[104];
+};
+
+/*
+ * controller registers
+ *
+ * These are defined by the PMC implementation.
+ *
+ * Some registers (those named sis_*) are only used when in
+ * legacy SIS mode before we transition the controller into
+ * PQI mode.  There are a number of other SIS mode registers,
+ * but we don't use them, so only the SIS registers that we
+ * care about are defined here.  The offsets mentioned in the
+ * comments are the offsets from the PCIe BAR 0.
+ */
+struct pqi_ctrl_registers {
+	u8	reserved[0x20];
+	__le32	sis_host_to_ctrl_doorbell;		/* 20h */
+	u8	reserved1[0x34 - (0x20 + sizeof(__le32))];
+	__le32	sis_interrupt_mask;			/* 34h */
+	u8	reserved2[0x9c - (0x34 + sizeof(__le32))];
+	__le32	sis_ctrl_to_host_doorbell;		/* 9Ch */
+	u8	reserved3[0xa0 - (0x9c + sizeof(__le32))];
+	__le32	sis_ctrl_to_host_doorbell_clear;	/* A0h */
+	u8	reserved4[0xb0 - (0xa0 + sizeof(__le32))];
+	__le32	sis_driver_scratch;			/* B0h */
+	u8	reserved5[0xbc - (0xb0 + sizeof(__le32))];
+	__le32	sis_firmware_status;			/* BCh */
+	u8	reserved6[0x1000 - (0xbc + sizeof(__le32))];
+	__le32	sis_mailbox[8];				/* 1000h */
+	u8	reserved7[0x4000 - (0x1000 + (sizeof(__le32) * 8))];
+	/*
+	 * The PQI spec states that the PQI registers should be at
+	 * offset 0 from the PCIe BAR 0.  However, we can't map
+	 * them at offset 0 because that would break compatibility
+	 * with the SIS registers.  So we map them at offset 4000h.
+	 */
+	struct pqi_device_registers pqi_registers;	/* 4000h */
+};
+
+#define PQI_DEVICE_REGISTERS_OFFSET	0x4000
+
+enum pqi_io_path {
+	RAID_PATH = 0,
+	AIO_PATH = 1
+};
+
+struct pqi_sg_descriptor {
+	__le64	address;
+	__le32	length;
+	__le32	flags;
+};
+
+/* manifest constants for the flags field of pqi_sg_descriptor */
+#define CISS_SG_LAST	0x40000000
+#define CISS_SG_CHAIN	0x80000000
+
+struct pqi_iu_header {
+	u8	iu_type;
+	u8	reserved;
+	__le16	iu_length;	/* in bytes - does not include the length */
+				/* of this header */
+	__le16	response_queue_id;	/* specifies the OQ where the */
+					/*   response IU is to be delivered */
+	u8	work_area[2];	/* reserved for driver use */
+};
+
+/*
+ * According to the PQI spec, the IU header is only the first 4 bytes of our
+ * pqi_iu_header structure.
+ */
+#define PQI_REQUEST_HEADER_LENGTH	4
+
+struct pqi_general_admin_request {
+	struct pqi_iu_header header;
+	__le16	request_id;
+	u8	function_code;
+	union {
+		struct {
+			u8	reserved[33];
+			__le32	buffer_length;
+			struct pqi_sg_descriptor sg_descriptor;
+		} report_device_capability;
+
+		struct {
+			u8	reserved;
+			__le16	queue_id;
+			u8	reserved1[2];
+			__le64	element_array_addr;
+			__le64	ci_addr;
+			__le16	num_elements;
+			__le16	element_length;
+			u8	queue_protocol;
+			u8	reserved2[23];
+			__le32	vendor_specific;
+		} create_operational_iq;
+
+		struct {
+			u8	reserved;
+			__le16	queue_id;
+			u8	reserved1[2];
+			__le64	element_array_addr;
+			__le64	pi_addr;
+			__le16	num_elements;
+			__le16	element_length;
+			u8	queue_protocol;
+			u8	reserved2[3];
+			__le16	int_msg_num;
+			__le16	coalescing_count;
+			__le32	min_coalescing_time;
+			__le32	max_coalescing_time;
+			u8	reserved3[8];
+			__le32	vendor_specific;
+		} create_operational_oq;
+
+		struct {
+			u8	reserved;
+			__le16	queue_id;
+			u8	reserved1[50];
+		} delete_operational_queue;
+
+		struct {
+			u8	reserved;
+			__le16	queue_id;
+			u8	reserved1[46];
+			__le32	vendor_specific;
+		} change_operational_iq_properties;
+
+	} data;
+};
+
+struct pqi_general_admin_response {
+	struct pqi_iu_header header;
+	__le16	request_id;
+	u8	function_code;
+	u8	status;
+	union {
+		struct {
+			u8	status_descriptor[4];
+			__le64	iq_pi_offset;
+			u8	reserved[40];
+		} create_operational_iq;
+
+		struct {
+			u8	status_descriptor[4];
+			__le64	oq_ci_offset;
+			u8	reserved[40];
+		} create_operational_oq;
+	} data;
+};
+
+struct pqi_iu_layer_descriptor {
+	u8	inbound_spanning_supported : 1;
+	u8	reserved : 7;
+	u8	reserved1[5];
+	__le16	max_inbound_iu_length;
+	u8	outbound_spanning_supported : 1;
+	u8	reserved2 : 7;
+	u8	reserved3[5];
+	__le16	max_outbound_iu_length;
+};
+
+struct pqi_device_capability {
+	__le16	data_length;
+	u8	reserved[6];
+	u8	iq_arbitration_priority_support_bitmask;
+	u8	maximum_aw_a;
+	u8	maximum_aw_b;
+	u8	maximum_aw_c;
+	u8	max_arbitration_burst : 3;
+	u8	reserved1 : 4;
+	u8	iqa : 1;
+	u8	reserved2[2];
+	u8	iq_freeze : 1;
+	u8	reserved3 : 7;
+	__le16	max_inbound_queues;
+	__le16	max_elements_per_iq;
+	u8	reserved4[4];
+	__le16	max_iq_element_length;
+	__le16	min_iq_element_length;
+	u8	reserved5[2];
+	__le16	max_outbound_queues;
+	__le16	max_elements_per_oq;
+	__le16	intr_coalescing_time_granularity;
+	__le16	max_oq_element_length;
+	__le16	min_oq_element_length;
+	u8	reserved6[24];
+	struct pqi_iu_layer_descriptor iu_layer_descriptors[32];
+};
+
+#define PQI_MAX_EMBEDDED_SG_DESCRIPTORS		4
+
+struct pqi_raid_path_request {
+	struct pqi_iu_header header;
+	__le16	request_id;
+	__le16	nexus_id;
+	__le32	buffer_length;
+	u8	lun_number[8];
+	__le16	protocol_specific;
+	u8	data_direction : 2;
+	u8	partial : 1;
+	u8	reserved1 : 4;
+	u8	fence : 1;
+	__le16	error_index;
+	u8	reserved2;
+	u8	task_attribute : 3;
+	u8	command_priority : 4;
+	u8	reserved3 : 1;
+	u8	reserved4 : 2;
+	u8	additional_cdb_bytes_usage : 3;
+	u8	reserved5 : 3;
+	u8	cdb[32];
+	struct pqi_sg_descriptor
+		sg_descriptors[PQI_MAX_EMBEDDED_SG_DESCRIPTORS];
+};
+
+struct pqi_aio_path_request {
+	struct pqi_iu_header header;
+	__le16	request_id;
+	u8	reserved1[2];
+	__le32	nexus_id;
+	__le32	buffer_length;
+	u8	data_direction : 2;
+	u8	partial : 1;
+	u8	memory_type : 1;
+	u8	fence : 1;
+	u8	encryption_enable : 1;
+	u8	reserved2 : 2;
+	u8	task_attribute : 3;
+	u8	command_priority : 4;
+	u8	reserved3 : 1;
+	__le16	data_encryption_key_index;
+	__le32	encrypt_tweak_lower;
+	__le32	encrypt_tweak_upper;
+	u8	cdb[16];
+	__le16	error_index;
+	u8	num_sg_descriptors;
+	u8	cdb_length;
+	u8	lun_number[8];
+	u8	reserved4[4];
+	struct pqi_sg_descriptor
+		sg_descriptors[PQI_MAX_EMBEDDED_SG_DESCRIPTORS];
+};
+
+struct pqi_io_response {
+	struct pqi_iu_header header;
+	__le16	request_id;
+	__le16	error_index;
+	u8	reserved2[4];
+};
+
+struct pqi_general_management_request {
+	struct pqi_iu_header header;
+	__le16	request_id;
+	union {
+		struct {
+			u8	reserved[2];
+			__le32	buffer_length;
+			struct pqi_sg_descriptor sg_descriptors[3];
+		} report_event_configuration;
+
+		struct {
+			__le16	global_event_oq_id;
+			__le32	buffer_length;
+			struct pqi_sg_descriptor sg_descriptors[3];
+		} set_event_configuration;
+	} data;
+};
+
+struct pqi_event_descriptor {
+	u8	event_type;
+	u8	reserved;
+	__le16	oq_id;
+};
+
+struct pqi_event_config {
+	u8	reserved[2];
+	u8	num_event_descriptors;
+	u8	reserved1;
+	struct pqi_event_descriptor descriptors[1];
+};
+
+#define PQI_MAX_EVENT_DESCRIPTORS	255
+
+struct pqi_event_response {
+	struct pqi_iu_header header;
+	u8	event_type;
+	u8	reserved2 : 7;
+	u8	request_acknowlege : 1;
+	__le16	event_id;
+	__le32	additional_event_id;
+	u8	data[16];
+};
+
+struct pqi_event_acknowledge_request {
+	struct pqi_iu_header header;
+	u8	event_type;
+	u8	reserved2;
+	__le16	event_id;
+	__le32	additional_event_id;
+};
+
+struct pqi_task_management_request {
+	struct pqi_iu_header header;
+	__le16	request_id;
+	__le16	nexus_id;
+	u8	reserved[4];
+	u8	lun_number[8];
+	__le16	protocol_specific;
+	__le16	outbound_queue_id_to_manage;
+	__le16	request_id_to_manage;
+	u8	task_management_function;
+	u8	reserved2 : 7;
+	u8	fence : 1;
+};
+
+#define SOP_TASK_MANAGEMENT_LUN_RESET	0x8
+
+struct pqi_task_management_response {
+	struct pqi_iu_header header;
+	__le16	request_id;
+	__le16	nexus_id;
+	u8	additional_response_info[3];
+	u8	response_code;
+};
+
+struct pqi_aio_error_info {
+	u8	status;
+	u8	service_response;
+	u8	data_present;
+	u8	reserved;
+	__le32	residual_count;
+	__le16	data_length;
+	__le16	reserved1;
+	u8	data[256];
+};
+
+struct pqi_raid_error_info {
+	u8	data_in_result;
+	u8	data_out_result;
+	u8	reserved[3];
+	u8	status;
+	__le16	status_qualifier;
+	__le16	sense_data_length;
+	__le16	response_data_length;
+	__le32	data_in_transferred;
+	__le32	data_out_transferred;
+	u8	data[256];
+};
+
+#define PQI_REQUEST_IU_TASK_MANAGEMENT			0x13
+#define PQI_REQUEST_IU_RAID_PATH_IO			0x14
+#define PQI_REQUEST_IU_AIO_PATH_IO			0x15
+#define PQI_REQUEST_IU_GENERAL_ADMIN			0x60
+#define PQI_REQUEST_IU_REPORT_VENDOR_EVENT_CONFIG	0x72
+#define PQI_REQUEST_IU_SET_VENDOR_EVENT_CONFIG		0x73
+#define PQI_REQUEST_IU_ACKNOWLEDGE_VENDOR_EVENT		0xf6
+
+#define PQI_RESPONSE_IU_GENERAL_MANAGEMENT		0x81
+#define PQI_RESPONSE_IU_TASK_MANAGEMENT			0x93
+#define PQI_RESPONSE_IU_GENERAL_ADMIN			0xe0
+#define PQI_RESPONSE_IU_RAID_PATH_IO_SUCCESS		0xf0
+#define PQI_RESPONSE_IU_AIO_PATH_IO_SUCCESS		0xf1
+#define PQI_RESPONSE_IU_RAID_PATH_IO_ERROR		0xf2
+#define PQI_RESPONSE_IU_AIO_PATH_IO_ERROR		0xf3
+#define PQI_RESPONSE_IU_AIO_PATH_DISABLED		0xf4
+#define PQI_RESPONSE_IU_VENDOR_EVENT			0xf5
+
+#define PQI_GENERAL_ADMIN_FUNCTION_REPORT_DEVICE_CAPABILITY	0x0
+#define PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ			0x10
+#define PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ			0x11
+#define PQI_GENERAL_ADMIN_FUNCTION_DELETE_IQ			0x12
+#define PQI_GENERAL_ADMIN_FUNCTION_DELETE_OQ			0x13
+#define PQI_GENERAL_ADMIN_FUNCTION_CHANGE_IQ_PROPERTY		0x14
+
+#define PQI_GENERAL_ADMIN_STATUS_SUCCESS	0x0
+
+#define PQI_IQ_PROPERTY_IS_AIO_QUEUE	0x1
+
+#define PQI_GENERAL_ADMIN_IU_LENGTH		0x3c
+#define PQI_PROTOCOL_SOP			0x0
+
+#define PQI_DATA_IN_OUT_GOOD					0x0
+#define PQI_DATA_IN_OUT_UNDERFLOW				0x1
+#define PQI_DATA_IN_OUT_BUFFER_ERROR				0x40
+#define PQI_DATA_IN_OUT_BUFFER_OVERFLOW				0x41
+#define PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA		0x42
+#define PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE			0x43
+#define PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR			0x60
+#define PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT			0x61
+#define PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED		0x62
+#define PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED	0x63
+#define PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED			0x64
+#define PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST		0x65
+#define PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION			0x66
+#define PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED			0x67
+#define PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ		0x6F
+#define PQI_DATA_IN_OUT_ERROR					0xf0
+#define PQI_DATA_IN_OUT_PROTOCOL_ERROR				0xf1
+#define PQI_DATA_IN_OUT_HARDWARE_ERROR				0xf2
+#define PQI_DATA_IN_OUT_UNSOLICITED_ABORT			0xf3
+#define PQI_DATA_IN_OUT_ABORTED					0xf4
+#define PQI_DATA_IN_OUT_TIMEOUT					0xf5
+
+#define CISS_CMD_STATUS_SUCCESS			0x0
+#define CISS_CMD_STATUS_TARGET_STATUS		0x1
+#define CISS_CMD_STATUS_DATA_UNDERRUN		0x2
+#define CISS_CMD_STATUS_DATA_OVERRUN		0x3
+#define CISS_CMD_STATUS_INVALID			0x4
+#define CISS_CMD_STATUS_PROTOCOL_ERROR		0x5
+#define CISS_CMD_STATUS_HARDWARE_ERROR		0x6
+#define CISS_CMD_STATUS_CONNECTION_LOST		0x7
+#define CISS_CMD_STATUS_ABORTED			0x8
+#define CISS_CMD_STATUS_ABORT_FAILED		0x9
+#define CISS_CMD_STATUS_UNSOLICITED_ABORT	0xa
+#define CISS_CMD_STATUS_TIMEOUT			0xb
+#define CISS_CMD_STATUS_UNABORTABLE		0xc
+#define CISS_CMD_STATUS_TMF			0xd
+#define CISS_CMD_STATUS_AIO_DISABLED		0xe
+
+#define PQI_NUM_EVENT_QUEUE_ELEMENTS	32
+#define PQI_EVENT_OQ_ELEMENT_LENGTH	sizeof(struct pqi_event_response)
+
+#define PQI_EVENT_TYPE_HOTPLUG			0x1
+#define PQI_EVENT_TYPE_HARDWARE			0x2
+#define PQI_EVENT_TYPE_PHYSICAL_DEVICE		0x4
+#define PQI_EVENT_TYPE_LOGICAL_DEVICE		0x5
+#define PQI_EVENT_TYPE_AIO_STATE_CHANGE		0xfd
+#define PQI_EVENT_TYPE_AIO_CONFIG_CHANGE	0xfe
+#define PQI_EVENT_TYPE_HEARTBEAT		0xff
+
+#pragma pack()
+
+#define PQI_ERROR_BUFFER_ELEMENT_LENGTH		\
+	sizeof(struct pqi_raid_error_info)
+
+/* these values are based on our implementation */
+#define PQI_ADMIN_IQ_NUM_ELEMENTS		8
+#define PQI_ADMIN_OQ_NUM_ELEMENTS		20
+#define PQI_ADMIN_IQ_ELEMENT_LENGTH		64
+#define PQI_ADMIN_OQ_ELEMENT_LENGTH		64
+
+#define PQI_OPERATIONAL_IQ_ELEMENT_LENGTH	128
+#define PQI_OPERATIONAL_OQ_ELEMENT_LENGTH	16
+
+#define PQI_MIN_MSIX_VECTORS		1
+#define PQI_MAX_MSIX_VECTORS		64
+
+/* these values are defined by the PQI spec */
+#define PQI_MAX_NUM_ELEMENTS_ADMIN_QUEUE	255
+#define PQI_MAX_NUM_ELEMENTS_OPERATIONAL_QUEUE	65535
+#define PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT	64
+#define PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT	16
+#define PQI_ADMIN_INDEX_ALIGNMENT		64
+#define PQI_OPERATIONAL_INDEX_ALIGNMENT		4
+
+#define PQI_MIN_OPERATIONAL_QUEUE_ID		1
+#define PQI_MAX_OPERATIONAL_QUEUE_ID		65535
+
+#define PQI_AIO_SERV_RESPONSE_COMPLETE		0
+#define PQI_AIO_SERV_RESPONSE_FAILURE		1
+#define PQI_AIO_SERV_RESPONSE_TMF_COMPLETE	2
+#define PQI_AIO_SERV_RESPONSE_TMF_SUCCEEDED	3
+#define PQI_AIO_SERV_RESPONSE_TMF_REJECTED	4
+#define PQI_AIO_SERV_RESPONSE_TMF_INCORRECT_LUN	5
+
+#define PQI_AIO_STATUS_IO_ERROR			0x1
+#define PQI_AIO_STATUS_IO_ABORTED		0x2
+#define PQI_AIO_STATUS_NO_PATH_TO_DEVICE	0x3
+#define PQI_AIO_STATUS_INVALID_DEVICE		0x4
+#define PQI_AIO_STATUS_AIO_PATH_DISABLED	0xe
+#define PQI_AIO_STATUS_UNDERRUN			0x51
+#define PQI_AIO_STATUS_OVERRUN			0x75
+
+typedef u32 pqi_index_t;
+
+/* SOP data direction flags */
+#define SOP_NO_DIRECTION_FLAG	0
+#define SOP_WRITE_FLAG		1	/* host writes data to Data-Out */
+					/* buffer */
+#define SOP_READ_FLAG		2	/* host receives data from Data-In */
+					/* buffer */
+#define SOP_BIDIRECTIONAL	3	/* data is transferred from the */
+					/* Data-Out buffer and data is */
+					/* transferred to the Data-In buffer */
+
+#define SOP_TASK_ATTRIBUTE_SIMPLE		0
+#define SOP_TASK_ATTRIBUTE_HEAD_OF_QUEUE	1
+#define SOP_TASK_ATTRIBUTE_ORDERED		2
+#define SOP_TASK_ATTRIBUTE_ACA			4
+
+#define SOP_TMF_COMPLETE		0x0
+#define SOP_TMF_FUNCTION_SUCCEEDED	0x8
+
+/* additional CDB bytes usage field codes */
+#define SOP_ADDITIONAL_CDB_BYTES_0	0	/* 16-byte CDB */
+#define SOP_ADDITIONAL_CDB_BYTES_4	1	/* 20-byte CDB */
+#define SOP_ADDITIONAL_CDB_BYTES_8	2	/* 24-byte CDB */
+#define SOP_ADDITIONAL_CDB_BYTES_12	3	/* 28-byte CDB */
+#define SOP_ADDITIONAL_CDB_BYTES_16	4	/* 32-byte CDB */
+
+/*
+ * The purpose of this structure is to obtain proper alignment of objects in
+ * an admin queue pair.
+ */
+struct pqi_admin_queues_aligned {
+	__aligned(PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT)
+		u8	iq_element_array[PQI_ADMIN_IQ_ELEMENT_LENGTH]
+					[PQI_ADMIN_IQ_NUM_ELEMENTS];
+	__aligned(PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT)
+		u8	oq_element_array[PQI_ADMIN_OQ_ELEMENT_LENGTH]
+					[PQI_ADMIN_OQ_NUM_ELEMENTS];
+	__aligned(PQI_ADMIN_INDEX_ALIGNMENT) pqi_index_t iq_ci;
+	__aligned(PQI_ADMIN_INDEX_ALIGNMENT) pqi_index_t oq_pi;
+};
+
+struct pqi_admin_queues {
+	void		*iq_element_array;
+	void		*oq_element_array;
+	volatile pqi_index_t *iq_ci;
+	volatile pqi_index_t *oq_pi;
+	dma_addr_t	iq_element_array_bus_addr;
+	dma_addr_t	oq_element_array_bus_addr;
+	dma_addr_t	iq_ci_bus_addr;
+	dma_addr_t	oq_pi_bus_addr;
+	__le32 __iomem	*iq_pi;
+	pqi_index_t	iq_pi_copy;
+	__le32 __iomem	*oq_ci;
+	pqi_index_t	oq_ci_copy;
+	struct task_struct *task;
+	u16		int_msg_num;
+};
+
+struct pqi_queue_group {
+	struct pqi_ctrl_info *ctrl_info;	/* backpointer */
+	u16		iq_id[2];
+	u16		oq_id;
+	u16		int_msg_num;
+	void		*iq_element_array[2];
+	void		*oq_element_array;
+	dma_addr_t	iq_element_array_bus_addr[2];
+	dma_addr_t	oq_element_array_bus_addr;
+	__le32 __iomem	*iq_pi[2];
+	pqi_index_t	iq_pi_copy[2];
+	volatile pqi_index_t *iq_ci[2];
+	volatile pqi_index_t *oq_pi;
+	dma_addr_t	iq_ci_bus_addr[2];
+	dma_addr_t	oq_pi_bus_addr;
+	__le32 __iomem	*oq_ci;
+	pqi_index_t	oq_ci_copy;
+	spinlock_t	submit_lock[2];	/* protect submission queue */
+	struct list_head request_list[2];
+};
+
+struct pqi_event_queue {
+	u16		oq_id;
+	u16		int_msg_num;
+	void		*oq_element_array;
+	volatile pqi_index_t *oq_pi;
+	dma_addr_t	oq_element_array_bus_addr;
+	dma_addr_t	oq_pi_bus_addr;
+	__le32 __iomem	*oq_ci;
+	pqi_index_t	oq_ci_copy;
+};
+
+#define PQI_DEFAULT_QUEUE_GROUP		0
+#define PQI_MAX_QUEUE_GROUPS		PQI_MAX_MSIX_VECTORS
+
+struct pqi_encryption_info {
+	u16	data_encryption_key_index;
+	u32	encrypt_tweak_lower;
+	u32	encrypt_tweak_upper;
+};
+
+#define PQI_MAX_OUTSTANDING_REQUESTS	((u32)~0)
+#define PQI_MAX_TRANSFER_SIZE		(4 * 1024U * 1024U)
+
+#define RAID_MAP_MAX_ENTRIES		1024
+
+#define PQI_PHYSICAL_DEVICE_BUS		0
+#define PQI_RAID_VOLUME_BUS		1
+#define PQI_HBA_BUS			2
+#define PQI_MAX_BUS			PQI_HBA_BUS
+
+#pragma pack(1)
+
+struct report_lun_header {
+	__be32	list_length;
+	u8	extended_response;
+	u8	reserved[3];
+};
+
+struct report_log_lun_extended_entry {
+	u8	lunid[8];
+	u8	volume_id[16];
+};
+
+struct report_log_lun_extended {
+	struct report_lun_header header;
+	struct report_log_lun_extended_entry lun_entries[1];
+};
+
+struct report_phys_lun_extended_entry {
+	u8	lunid[8];
+	__be64	wwid;
+	u8	device_type;
+	u8	device_flags;
+	u8	lun_count;	/* number of LUNs in a multi-LUN device */
+	u8	redundant_paths;
+	u32	aio_handle;
+};
+
+/* for device_flags field of struct report_phys_lun_extended_entry */
+#define REPORT_PHYS_LUN_DEV_FLAG_NON_DISK	0x1
+#define REPORT_PHYS_LUN_DEV_FLAG_AIO_ENABLED	0x8
+
+struct report_phys_lun_extended {
+	struct report_lun_header header;
+	struct report_phys_lun_extended_entry lun_entries[1];
+};
+
+struct raid_map_disk_data {
+	u32	aio_handle;
+	u8	xor_mult[2];
+	u8	reserved[2];
+};
+
+/* constants for flags field of RAID map */
+#define RAID_MAP_ENCRYPTION_ENABLED	0x1
+
+struct raid_map {
+	__le32	structure_size;		/* size of entire structure in bytes */
+	__le32	volume_blk_size;	/* bytes / block in the volume */
+	__le64	volume_blk_cnt;		/* logical blocks on the volume */
+	u8	phys_blk_shift;		/* shift factor to convert between */
+					/* units of logical blocks and */
+					/* physical disk blocks */
+	u8	parity_rotation_shift;	/* shift factor to convert between */
+					/* units of logical stripes and */
+					/* physical stripes */
+	__le16	strip_size;		/* blocks used on each disk / stripe */
+	__le64	disk_starting_blk;	/* first disk block used in volume */
+	__le64	disk_blk_cnt;		/* disk blocks used by volume / disk */
+	__le16	data_disks_per_row;	/* data disk entries / row in the map */
+	__le16	metadata_disks_per_row;	/* mirror/parity disk entries / row */
+					/* in the map */
+	__le16	row_cnt;		/* rows in each layout map */
+	__le16	layout_map_count;	/* layout maps (1 map per */
+					/* mirror parity group) */
+	__le16	flags;
+	__le16	data_encryption_key_index;
+	u8	reserved[16];
+	struct raid_map_disk_data disk_data[RAID_MAP_MAX_ENTRIES];
+};
+
+#pragma pack()
+
+#define RAID_CTLR_LUNID		"\0\0\0\0\0\0\0\0"
+
+struct pqi_scsi_dev {
+	int	devtype;		/* as reported by INQUIRY commmand */
+	u8	device_type;		/* as reported by */
+					/* BMIC_IDENTIFY_PHYSICAL_DEVICE */
+					/* only valid for devtype = TYPE_DISK */
+	int	bus;
+	int	target;
+	int	lun;
+	u8	scsi3addr[8];
+	__be64	wwid;
+	u8	volume_id[16];
+	u8	is_physical_device : 1;
+	u8	target_lun_valid : 1;
+	u8	expose_device : 1;
+	u8	no_uld_attach : 1;
+	u8	aio_enabled : 1;	/* only valid for physical disks */
+	u8	device_gone : 1;
+	u8	new_device : 1;
+	u8	keep_device : 1;
+	u8	volume_offline : 1;
+	u8	vendor[8];		/* bytes 8-15 of inquiry data */
+	u8	model[16];		/* bytes 16-31 of inquiry data */
+	u64	sas_address;
+	u8	raid_level;
+	u16	queue_depth;		/* max. queue_depth for this device */
+	u16	advertised_queue_depth;
+	u32	aio_handle;
+	u8	volume_status;
+	u8	active_path_index;
+	u8	path_map;
+	u8	bay;
+	u8	box[8];
+	u16	phys_connector[8];
+	int	offload_configured;	/* I/O accel RAID offload configured */
+	int	offload_enabled;	/* I/O accel RAID offload enabled */
+	int	offload_enabled_pending;
+	int	offload_to_mirror;	/* Send next I/O accelerator RAID */
+					/* offload request to mirror drive. */
+	struct raid_map *raid_map;	/* I/O accelerator RAID map */
+
+	struct pqi_sas_port *sas_port;
+	struct scsi_device *sdev;
+
+	struct list_head scsi_device_list_entry;
+	struct list_head new_device_list_entry;
+	struct list_head add_list_entry;
+	struct list_head delete_list_entry;
+};
+
+/* VPD inquiry pages */
+#define SCSI_VPD_SUPPORTED_PAGES	0x0	/* standard page */
+#define SCSI_VPD_DEVICE_ID		0x83	/* standard page */
+#define CISS_VPD_LV_DEVICE_GEOMETRY	0xc1	/* vendor-specific page */
+#define CISS_VPD_LV_OFFLOAD_STATUS	0xc2	/* vendor-specific page */
+#define CISS_VPD_LV_STATUS		0xc3	/* vendor-specific page */
+
+#define VPD_PAGE	(1 << 8)
+
+#pragma pack(1)
+
+/* structure for CISS_VPD_LV_STATUS */
+struct ciss_vpd_logical_volume_status {
+	u8	peripheral_info;
+	u8	page_code;
+	u8	reserved;
+	u8	page_length;
+	u8	volume_status;
+	u8	reserved2[3];
+	__be32	flags;
+};
+
+#pragma pack()
+
+/* constants for volume_status field of ciss_vpd_logical_volume_status */
+#define CISS_LV_OK					0
+#define CISS_LV_FAILED					1
+#define CISS_LV_NOT_CONFIGURED				2
+#define CISS_LV_DEGRADED				3
+#define CISS_LV_READY_FOR_RECOVERY			4
+#define CISS_LV_UNDERGOING_RECOVERY			5
+#define CISS_LV_WRONG_PHYSICAL_DRIVE_REPLACED		6
+#define CISS_LV_PHYSICAL_DRIVE_CONNECTION_PROBLEM	7
+#define CISS_LV_HARDWARE_OVERHEATING			8
+#define CISS_LV_HARDWARE_HAS_OVERHEATED			9
+#define CISS_LV_UNDERGOING_EXPANSION			10
+#define CISS_LV_NOT_AVAILABLE				11
+#define CISS_LV_QUEUED_FOR_EXPANSION			12
+#define CISS_LV_DISABLED_SCSI_ID_CONFLICT		13
+#define CISS_LV_EJECTED					14
+#define CISS_LV_UNDERGOING_ERASE			15
+/* state 16 not used */
+#define CISS_LV_READY_FOR_PREDICTIVE_SPARE_REBUILD	17
+#define CISS_LV_UNDERGOING_RPI				18
+#define CISS_LV_PENDING_RPI				19
+#define CISS_LV_ENCRYPTED_NO_KEY			20
+/* state 21 not used */
+#define CISS_LV_UNDERGOING_ENCRYPTION			22
+#define CISS_LV_UNDERGOING_ENCRYPTION_REKEYING		23
+#define CISS_LV_ENCRYPTED_IN_NON_ENCRYPTED_CONTROLLER	24
+#define CISS_LV_PENDING_ENCRYPTION			25
+#define CISS_LV_PENDING_ENCRYPTION_REKEYING		26
+#define CISS_LV_NOT_SUPPORTED				27
+#define CISS_LV_STATUS_UNAVAILABLE			255
+
+/* constants for flags field of ciss_vpd_logical_volume_status */
+#define CISS_LV_FLAGS_NO_HOST_IO	0x1	/* volume not available for */
+						/* host I/O */
+
+/* for SAS hosts and SAS expanders */
+struct pqi_sas_node {
+	struct device *parent_dev;
+	struct list_head port_list_head;
+};
+
+struct pqi_sas_port {
+	struct list_head port_list_entry;
+	u64	sas_address;
+	struct sas_port *port;
+	int	next_phy_index;
+	struct list_head phy_list_head;
+	struct pqi_sas_node *parent_node;
+	struct sas_rphy *rphy;
+};
+
+struct pqi_sas_phy {
+	struct list_head phy_list_entry;
+	struct sas_phy *phy;
+	struct pqi_sas_port *parent_port;
+	bool	added_to_port;
+};
+
+struct pqi_io_request {
+	atomic_t	refcount;
+	u16		index;
+	void (*io_complete_callback)(struct pqi_io_request *io_request,
+		void *context);
+	void		*context;
+	int		status;
+	struct scsi_cmnd *scmd;
+	void		*error_info;
+	struct pqi_sg_descriptor *sg_chain_buffer;
+	dma_addr_t	sg_chain_buffer_dma_handle;
+	void		*iu;
+	struct list_head request_list_entry;
+};
+
+/* for indexing into the pending_events[] field of struct pqi_ctrl_info */
+#define PQI_EVENT_HEARTBEAT		0
+#define PQI_EVENT_HOTPLUG		1
+#define PQI_EVENT_HARDWARE		2
+#define PQI_EVENT_PHYSICAL_DEVICE	3
+#define PQI_EVENT_LOGICAL_DEVICE	4
+#define PQI_EVENT_AIO_STATE_CHANGE	5
+#define PQI_EVENT_AIO_CONFIG_CHANGE	6
+#define PQI_NUM_SUPPORTED_EVENTS	7
+
+struct pqi_event {
+	bool	pending;
+	u8	event_type;
+	__le16	event_id;
+	__le32	additional_event_id;
+};
+
+#define PQI_RESERVED_IO_SLOTS_LUN_RESET			1
+#define PQI_RESERVED_IO_SLOTS_EVENT_ACK			PQI_NUM_SUPPORTED_EVENTS
+#define PQI_RESERVED_IO_SLOTS_SYNCHRONOUS_REQUESTS	3
+#define PQI_RESERVED_IO_SLOTS				\
+	(PQI_RESERVED_IO_SLOTS_LUN_RESET + PQI_RESERVED_IO_SLOTS_EVENT_ACK + \
+	PQI_RESERVED_IO_SLOTS_SYNCHRONOUS_REQUESTS)
+
+struct pqi_ctrl_info {
+	unsigned int	ctrl_id;
+	struct pci_dev	*pci_dev;
+	char		firmware_version[11];
+	void __iomem	*iomem_base;
+	struct pqi_ctrl_registers __iomem *registers;
+	struct pqi_device_registers __iomem *pqi_registers;
+	u32		max_sg_entries;
+	u32		config_table_offset;
+	u32		config_table_length;
+	u16		max_inbound_queues;
+	u16		max_elements_per_iq;
+	u16		max_iq_element_length;
+	u16		max_outbound_queues;
+	u16		max_elements_per_oq;
+	u16		max_oq_element_length;
+	u32		max_transfer_size;
+	u32		max_outstanding_requests;
+	u32		max_io_slots;
+	unsigned int	scsi_ml_can_queue;
+	unsigned short	sg_tablesize;
+	unsigned int	max_sectors;
+	u32		error_buffer_length;
+	void		*error_buffer;
+	dma_addr_t	error_buffer_dma_handle;
+	size_t		sg_chain_buffer_length;
+	unsigned int	num_queue_groups;
+	unsigned int	num_active_queue_groups;
+	u16		num_elements_per_iq;
+	u16		num_elements_per_oq;
+	u16		max_inbound_iu_length_per_firmware;
+	u16		max_inbound_iu_length;
+	unsigned int	max_sg_per_iu;
+	void		*admin_queue_memory_base;
+	u32		admin_queue_memory_length;
+	dma_addr_t	admin_queue_memory_base_dma_handle;
+	void		*queue_memory_base;
+	u32		queue_memory_length;
+	dma_addr_t	queue_memory_base_dma_handle;
+	struct pqi_admin_queues admin_queues;
+	struct pqi_queue_group queue_groups[PQI_MAX_QUEUE_GROUPS];
+	struct pqi_event_queue event_queue;
+	int		max_msix_vectors;
+	int		num_msix_vectors_enabled;
+	int		num_msix_vectors_initialized;
+	u32		msix_vectors[PQI_MAX_MSIX_VECTORS];
+	void		*intr_data[PQI_MAX_MSIX_VECTORS];
+	int		event_irq;
+	struct Scsi_Host *scsi_host;
+
+	struct mutex	scan_mutex;
+	u8		inbound_spanning_supported : 1;
+	u8		outbound_spanning_supported : 1;
+	u8		pqi_mode_enabled : 1;
+	u8		controller_online : 1;
+	u8		heartbeat_timer_started : 1;
+
+	struct list_head scsi_device_list;
+	spinlock_t	scsi_device_list_lock;
+
+	struct delayed_work rescan_work;
+	struct delayed_work update_time_work;
+
+	struct pqi_sas_node *sas_host;
+	u64		sas_address;
+
+	struct pqi_io_request *io_request_pool;
+	u16		next_io_request_slot;
+
+	struct pqi_event pending_events[PQI_NUM_SUPPORTED_EVENTS];
+	struct work_struct event_work;
+
+	atomic_t	num_interrupts;
+	int		previous_num_interrupts;
+	unsigned int	num_heartbeats_requested;
+	struct timer_list heartbeat_timer;
+
+	struct semaphore sync_request_sem;
+	struct semaphore lun_reset_sem;
+};
+
+enum pqi_ctrl_mode {
+	UNKNOWN,
+	PQI_MODE
+};
+
+/*
+ * assume worst case: SATA queue depth of 31 minus 4 internal firmware commands
+ */
+#define PQI_PHYSICAL_DISK_DEFAULT_MAX_QUEUE_DEPTH	27
+
+/* 0 = no limit */
+#define PQI_LOGICAL_DRIVE_DEFAULT_MAX_QUEUE_DEPTH	0
+
+/* CISS commands */
+#define CISS_READ		0xc0
+#define CISS_REPORT_LOG		0xc2	/* Report Logical LUNs */
+#define CISS_REPORT_PHYS	0xc3	/* Report Physical LUNs */
+#define CISS_GET_RAID_MAP	0xc8
+
+/* constants for CISS_REPORT_LOG/CISS_REPORT_PHYS commands */
+#define CISS_REPORT_LOG_EXTENDED		0x1
+#define CISS_REPORT_PHYS_EXTENDED		0x2
+
+/* BMIC commands */
+#define BMIC_IDENTIFY_CONTROLLER		0x11
+#define BMIC_IDENTIFY_PHYSICAL_DEVICE		0x15
+#define BMIC_READ				0x26
+#define BMIC_WRITE				0x27
+#define BMIC_SENSE_CONTROLLER_PARAMETERS	0x64
+#define BMIC_SENSE_SUBSYSTEM_INFORMATION	0x66
+#define BMIC_WRITE_HOST_WELLNESS		0xa5
+#define BMIC_CACHE_FLUSH			0xc2
+
+#define SA_CACHE_FLUSH				0x01
+
+#define MASKED_DEVICE(lunid)			((lunid)[3] & 0xc0)
+#define CISS_GET_BUS(lunid)			((lunid)[7] & 0x3f)
+#define CISS_GET_LEVEL_2_TARGET(lunid)		((lunid)[6])
+#define CISS_GET_DRIVE_NUMBER(lunid)		\
+	(((CISS_GET_BUS((lunid)) - 1) << 8) +	\
+	CISS_GET_LEVEL_2_TARGET((lunid)))
+
+#define NO_TIMEOUT		((unsigned long) -1)
+
+#pragma pack(1)
+
+struct bmic_identify_controller {
+	u8	configured_logical_drive_count;
+	__le32	configuration_signature;
+	u8	firmware_version[4];
+	u8	reserved[145];
+	__le16	extended_logical_unit_count;
+	u8	reserved1[34];
+	__le16	firmware_build_number;
+	u8	reserved2[100];
+	u8	controller_mode;
+	u8	reserved3[32];
+};
+
+struct bmic_identify_physical_device {
+	u8	scsi_bus;		/* SCSI Bus number on controller */
+	u8	scsi_id;		/* SCSI ID on this bus */
+	__le16	block_size;		/* sector size in bytes */
+	__le32	total_blocks;		/* number for sectors on drive */
+	__le32	reserved_blocks;	/* controller reserved (RIS) */
+	u8	model[40];		/* Physical Drive Model */
+	u8	serial_number[40];	/* Drive Serial Number */
+	u8	firmware_revision[8];	/* drive firmware revision */
+	u8	scsi_inquiry_bits;	/* inquiry byte 7 bits */
+	u8	compaq_drive_stamp;	/* 0 means drive not stamped */
+	u8	last_failure_reason;
+	u8	flags;
+	u8	more_flags;
+	u8	scsi_lun;		/* SCSI LUN for phys drive */
+	u8	yet_more_flags;
+	u8	even_more_flags;
+	__le32	spi_speed_rules;
+	u8	phys_connector[2];	/* connector number on controller */
+	u8	phys_box_on_bus;	/* phys enclosure this drive resides */
+	u8	phys_bay_in_box;	/* phys drv bay this drive resides */
+	__le32	rpm;			/* drive rotational speed in RPM */
+	u8	device_type;		/* type of drive */
+	u8	sata_version;		/* only valid when device_type = */
+					/* BMIC_DEVICE_TYPE_SATA */
+	__le64	big_total_block_count;
+	__le64	ris_starting_lba;
+	__le32	ris_size;
+	u8	wwid[20];
+	u8	controller_phy_map[32];
+	__le16	phy_count;
+	u8	phy_connected_dev_type[256];
+	u8	phy_to_drive_bay_num[256];
+	__le16	phy_to_attached_dev_index[256];
+	u8	box_index;
+	u8	reserved;
+	__le16	extra_physical_drive_flags;
+	u8	negotiated_link_rate[256];
+	u8	phy_to_phy_map[256];
+	u8	redundant_path_present_map;
+	u8	redundant_path_failure_map;
+	u8	active_path_number;
+	__le16	alternate_paths_phys_connector[8];
+	u8	alternate_paths_phys_box_on_port[8];
+	u8	multi_lun_device_lun_count;
+	u8	minimum_good_fw_revision[8];
+	u8	unique_inquiry_bytes[20];
+	u8	current_temperature_degreesC;
+	u8	temperature_threshold_degreesC;
+	u8	max_temperature_degreesC;
+	u8	logical_blocks_per_phys_block_exp;
+	__le16	current_queue_depth_limit;
+	u8	switch_name[10];
+	__le16	switch_port;
+	u8	alternate_paths_switch_name[40];
+	u8	alternate_paths_switch_port[8];
+	__le16	power_on_hours;
+	__le16	percent_endurance_used;
+	u8	drive_authentication;
+	u8	smart_carrier_authentication;
+	u8	smart_carrier_app_fw_version;
+	u8	smart_carrier_bootloader_fw_version;
+	u8	encryption_key_name[64];
+	__le32	misc_drive_flags;
+	__le16	dek_index;
+	u8	padding[112];
+};
+
+#pragma pack()
+
+int pqi_add_sas_host(struct Scsi_Host *shost, struct pqi_ctrl_info *ctrl_info);
+void pqi_delete_sas_host(struct pqi_ctrl_info *ctrl_info);
+int pqi_add_sas_device(struct pqi_sas_node *pqi_sas_node,
+	struct pqi_scsi_dev *device);
+void pqi_remove_sas_device(struct pqi_scsi_dev *device);
+struct pqi_scsi_dev *pqi_find_device_by_sas_rphy(
+	struct pqi_ctrl_info *ctrl_info, struct sas_rphy *rphy);
+
+extern struct sas_function_template pqi_sas_transport_functions;
+
+#if !defined(readq)
+#define readq readq
+static inline u64 readq(const volatile void __iomem *addr)
+{
+	u32 lower32;
+	u32 upper32;
+
+	lower32 = readl(addr);
+	upper32 = readl(addr + 4);
+
+	return ((u64)upper32 << 32) | lower32;
+}
+#endif
+
+#if !defined(writeq)
+#define writeq writeq
+static inline void writeq(u64 value, volatile void __iomem *addr)
+{
+	u32 lower32;
+	u32 upper32;
+
+	lower32 = lower_32_bits(value);
+	upper32 = upper_32_bits(value);
+
+	writel(lower32, addr);
+	writel(upper32, addr + 4);
+}
+#endif
+
+#endif /* _SMARTPQI_H */
diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
new file mode 100644
index 000000000000..a535b2661f38
--- /dev/null
+++ b/drivers/scsi/smartpqi/smartpqi_init.c
@@ -0,0 +1,6303 @@
+/*
+ *    driver for Microsemi PQI-based storage controllers
+ *    Copyright (c) 2016 Microsemi Corporation
+ *    Copyright (c) 2016 PMC-Sierra, Inc.
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; version 2 of the License.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *    NON INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/sched.h>
+#include <linux/rtc.h>
+#include <linux/bcd.h>
+#include <linux/cciss_ioctl.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_eh.h>
+#include <scsi/scsi_transport_sas.h>
+#include <asm/unaligned.h>
+#include "smartpqi.h"
+#include "smartpqi_sis.h"
+
+#if !defined(BUILD_TIMESTAMP)
+#define BUILD_TIMESTAMP
+#endif
+
+#define DRIVER_VERSION		"0.9.13-370"
+#define DRIVER_MAJOR		0
+#define DRIVER_MINOR		9
+#define DRIVER_RELEASE		13
+#define DRIVER_REVISION		370
+
+#define DRIVER_NAME		"Microsemi PQI Driver (v" DRIVER_VERSION ")"
+#define DRIVER_NAME_SHORT	"smartpqi"
+
+MODULE_AUTHOR("Microsemi");
+MODULE_DESCRIPTION("Driver for Microsemi Smart Family Controller version "
+	DRIVER_VERSION);
+MODULE_SUPPORTED_DEVICE("Microsemi Smart Family Controllers");
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL");
+
+#define PQI_ENABLE_MULTI_QUEUE_SUPPORT	0
+
+static char *hpe_branded_controller = "HPE Smart Array Controller";
+static char *microsemi_branded_controller = "Microsemi Smart Family Controller";
+
+static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info);
+static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
+static void pqi_scan_start(struct Scsi_Host *shost);
+static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_queue_group *queue_group, enum pqi_io_path path,
+	struct pqi_io_request *io_request);
+static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_iu_header *request, unsigned int flags,
+	struct pqi_raid_error_info *error_info, unsigned long timeout_msecs);
+static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
+	struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
+	unsigned int cdb_length, struct pqi_queue_group *queue_group,
+	struct pqi_encryption_info *encryption_info);
+
+/* for flags argument to pqi_submit_raid_request_synchronous() */
+#define PQI_SYNC_FLAGS_INTERRUPTABLE	0x1
+
+static struct scsi_transport_template *pqi_sas_transport_template;
+
+static atomic_t pqi_controller_count = ATOMIC_INIT(0);
+
+static int pqi_disable_device_id_wildcards;
+module_param_named(disable_device_id_wildcards,
+	pqi_disable_device_id_wildcards, int, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(disable_device_id_wildcards,
+	"Disable device ID wildcards.");
+
+static char *raid_levels[] = {
+	"RAID-0",
+	"RAID-4",
+	"RAID-1(1+0)",
+	"RAID-5",
+	"RAID-5+1",
+	"RAID-ADG",
+	"RAID-1(ADM)",
+};
+
+static char *pqi_raid_level_to_string(u8 raid_level)
+{
+	if (raid_level < ARRAY_SIZE(raid_levels))
+		return raid_levels[raid_level];
+
+	return "";
+}
+
+#define SA_RAID_0		0
+#define SA_RAID_4		1
+#define SA_RAID_1		2	/* also used for RAID 10 */
+#define SA_RAID_5		3	/* also used for RAID 50 */
+#define SA_RAID_51		4
+#define SA_RAID_6		5	/* also used for RAID 60 */
+#define SA_RAID_ADM		6	/* also used for RAID 1+0 ADM */
+#define SA_RAID_MAX		SA_RAID_ADM
+#define SA_RAID_UNKNOWN		0xff
+
+static inline void pqi_scsi_done(struct scsi_cmnd *scmd)
+{
+	scmd->scsi_done(scmd);
+}
+
+static inline bool pqi_scsi3addr_equal(u8 *scsi3addr1, u8 *scsi3addr2)
+{
+	return memcmp(scsi3addr1, scsi3addr2, 8) == 0;
+}
+
+static inline struct pqi_ctrl_info *shost_to_hba(struct Scsi_Host *shost)
+{
+	void *hostdata = shost_priv(shost);
+
+	return *((struct pqi_ctrl_info **)hostdata);
+}
+
+static inline bool pqi_is_logical_device(struct pqi_scsi_dev *device)
+{
+	return !device->is_physical_device;
+}
+
+static inline bool pqi_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
+{
+	return !ctrl_info->controller_online;
+}
+
+static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)
+{
+	if (ctrl_info->controller_online)
+		if (!sis_is_firmware_running(ctrl_info))
+			pqi_take_ctrl_offline(ctrl_info);
+}
+
+static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
+{
+	return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
+}
+
+static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(
+	struct pqi_ctrl_info *ctrl_info)
+{
+	return sis_read_driver_scratch(ctrl_info);
+}
+
+static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
+	enum pqi_ctrl_mode mode)
+{
+	sis_write_driver_scratch(ctrl_info, mode);
+}
+
+#define PQI_RESCAN_WORK_INTERVAL	(10 * HZ)
+
+static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)
+{
+	schedule_delayed_work(&ctrl_info->rescan_work,
+		PQI_RESCAN_WORK_INTERVAL);
+}
+
+static int pqi_map_single(struct pci_dev *pci_dev,
+	struct pqi_sg_descriptor *sg_descriptor, void *buffer,
+	size_t buffer_length, int data_direction)
+{
+	dma_addr_t bus_address;
+
+	if (!buffer || buffer_length == 0 || data_direction == PCI_DMA_NONE)
+		return 0;
+
+	bus_address = pci_map_single(pci_dev, buffer, buffer_length,
+		data_direction);
+	if (pci_dma_mapping_error(pci_dev, bus_address))
+		return -ENOMEM;
+
+	put_unaligned_le64((u64)bus_address, &sg_descriptor->address);
+	put_unaligned_le32(buffer_length, &sg_descriptor->length);
+	put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
+
+	return 0;
+}
+
+static void pqi_pci_unmap(struct pci_dev *pci_dev,
+	struct pqi_sg_descriptor *descriptors, int num_descriptors,
+	int data_direction)
+{
+	int i;
+
+	if (data_direction == PCI_DMA_NONE)
+		return;
+
+	for (i = 0; i < num_descriptors; i++)
+		pci_unmap_single(pci_dev,
+			(dma_addr_t)get_unaligned_le64(&descriptors[i].address),
+			get_unaligned_le32(&descriptors[i].length),
+			data_direction);
+}
+
+static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_raid_path_request *request, u8 cmd,
+	u8 *scsi3addr, void *buffer, size_t buffer_length,
+	u16 vpd_page, int *pci_direction)
+{
+	u8 *cdb;
+	int pci_dir;
+
+	memset(request, 0, sizeof(*request));
+
+	request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
+	put_unaligned_le16(offsetof(struct pqi_raid_path_request,
+		sg_descriptors[1]) - PQI_REQUEST_HEADER_LENGTH,
+		&request->header.iu_length);
+	put_unaligned_le32(buffer_length, &request->buffer_length);
+	memcpy(request->lun_number, scsi3addr, sizeof(request->lun_number));
+	request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
+	request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
+
+	cdb = request->cdb;
+
+	switch (cmd) {
+	case INQUIRY:
+		request->data_direction = SOP_READ_FLAG;
+		cdb[0] = INQUIRY;
+		if (vpd_page & VPD_PAGE) {
+			cdb[1] = 0x1;
+			cdb[2] = (u8)vpd_page;
+		}
+		cdb[4] = (u8)buffer_length;
+		break;
+	case CISS_REPORT_LOG:
+	case CISS_REPORT_PHYS:
+		request->data_direction = SOP_READ_FLAG;
+		cdb[0] = cmd;
+		if (cmd == CISS_REPORT_PHYS)
+			cdb[1] = CISS_REPORT_PHYS_EXTENDED;
+		else
+			cdb[1] = CISS_REPORT_LOG_EXTENDED;
+		put_unaligned_be32(buffer_length, &cdb[6]);
+		break;
+	case CISS_GET_RAID_MAP:
+		request->data_direction = SOP_READ_FLAG;
+		cdb[0] = CISS_READ;
+		cdb[1] = CISS_GET_RAID_MAP;
+		put_unaligned_be32(buffer_length, &cdb[6]);
+		break;
+	case SA_CACHE_FLUSH:
+		request->data_direction = SOP_WRITE_FLAG;
+		cdb[0] = BMIC_WRITE;
+		cdb[6] = BMIC_CACHE_FLUSH;
+		put_unaligned_be16(buffer_length, &cdb[7]);
+		break;
+	case BMIC_IDENTIFY_CONTROLLER:
+	case BMIC_IDENTIFY_PHYSICAL_DEVICE:
+		request->data_direction = SOP_READ_FLAG;
+		cdb[0] = BMIC_READ;
+		cdb[6] = cmd;
+		put_unaligned_be16(buffer_length, &cdb[7]);
+		break;
+	case BMIC_WRITE_HOST_WELLNESS:
+		request->data_direction = SOP_WRITE_FLAG;
+		cdb[0] = BMIC_WRITE;
+		cdb[6] = cmd;
+		put_unaligned_be16(buffer_length, &cdb[7]);
+		break;
+	default:
+		dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n",
+			cmd);
+		WARN_ON(cmd);
+		break;
+	}
+
+	switch (request->data_direction) {
+	case SOP_READ_FLAG:
+		pci_dir = PCI_DMA_FROMDEVICE;
+		break;
+	case SOP_WRITE_FLAG:
+		pci_dir = PCI_DMA_TODEVICE;
+		break;
+	case SOP_NO_DIRECTION_FLAG:
+		pci_dir = PCI_DMA_NONE;
+		break;
+	default:
+		pci_dir = PCI_DMA_BIDIRECTIONAL;
+		break;
+	}
+
+	*pci_direction = pci_dir;
+
+	return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],
+		buffer, buffer_length, pci_dir);
+}
+
+static struct pqi_io_request *pqi_alloc_io_request(
+	struct pqi_ctrl_info *ctrl_info)
+{
+	struct pqi_io_request *io_request;
+	u16 i = ctrl_info->next_io_request_slot;	/* benignly racy */
+
+	while (1) {
+		io_request = &ctrl_info->io_request_pool[i];
+		if (atomic_inc_return(&io_request->refcount) == 1)
+			break;
+		atomic_dec(&io_request->refcount);
+		i = (i + 1) % ctrl_info->max_io_slots;
+	}
+
+	/* benignly racy */
+	ctrl_info->next_io_request_slot = (i + 1) % ctrl_info->max_io_slots;
+
+	io_request->scmd = NULL;
+	io_request->status = 0;
+	io_request->error_info = NULL;
+
+	return io_request;
+}
+
+static void pqi_free_io_request(struct pqi_io_request *io_request)
+{
+	atomic_dec(&io_request->refcount);
+}
+
+static int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,
+	struct bmic_identify_controller *buffer)
+{
+	int rc;
+	int pci_direction;
+	struct pqi_raid_path_request request;
+
+	rc = pqi_build_raid_path_request(ctrl_info, &request,
+		BMIC_IDENTIFY_CONTROLLER, RAID_CTLR_LUNID, buffer,
+		sizeof(*buffer), 0, &pci_direction);
+	if (rc)
+		return rc;
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
+		NULL, NO_TIMEOUT);
+
+	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
+		pci_direction);
+
+	return rc;
+}
+
+static int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,
+	u8 *scsi3addr, u16 vpd_page, void *buffer, size_t buffer_length)
+{
+	int rc;
+	int pci_direction;
+	struct pqi_raid_path_request request;
+
+	rc = pqi_build_raid_path_request(ctrl_info, &request,
+		INQUIRY, scsi3addr, buffer, buffer_length, vpd_page,
+		&pci_direction);
+	if (rc)
+		return rc;
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
+		NULL, NO_TIMEOUT);
+
+	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
+		pci_direction);
+
+	return rc;
+}
+
+static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device,
+	struct bmic_identify_physical_device *buffer,
+	size_t buffer_length)
+{
+	int rc;
+	int pci_direction;
+	u16 bmic_device_index;
+	struct pqi_raid_path_request request;
+
+	rc = pqi_build_raid_path_request(ctrl_info, &request,
+		BMIC_IDENTIFY_PHYSICAL_DEVICE, RAID_CTLR_LUNID, buffer,
+		buffer_length, 0, &pci_direction);
+	if (rc)
+		return rc;
+
+	bmic_device_index = CISS_GET_DRIVE_NUMBER(device->scsi3addr);
+	request.cdb[2] = (u8)bmic_device_index;
+	request.cdb[9] = (u8)(bmic_device_index >> 8);
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
+		0, NULL, NO_TIMEOUT);
+
+	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
+		pci_direction);
+
+	return rc;
+}
+
+#define SA_CACHE_FLUSH_BUFFER_LENGTH	4
+
+static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	struct pqi_raid_path_request request;
+	int pci_direction;
+	u8 *buffer;
+
+	/*
+	 * Don't bother trying to flush the cache if the controller is
+	 * locked up.
+	 */
+	if (pqi_ctrl_offline(ctrl_info))
+		return -ENXIO;
+
+	buffer = kzalloc(SA_CACHE_FLUSH_BUFFER_LENGTH, GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	rc = pqi_build_raid_path_request(ctrl_info, &request,
+		SA_CACHE_FLUSH, RAID_CTLR_LUNID, buffer,
+		SA_CACHE_FLUSH_BUFFER_LENGTH, 0, &pci_direction);
+	if (rc)
+		goto out;
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
+		0, NULL, NO_TIMEOUT);
+
+	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
+		pci_direction);
+
+out:
+	kfree(buffer);
+
+	return rc;
+}
+
+static int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,
+	void *buffer, size_t buffer_length)
+{
+	int rc;
+	struct pqi_raid_path_request request;
+	int pci_direction;
+
+	rc = pqi_build_raid_path_request(ctrl_info, &request,
+		BMIC_WRITE_HOST_WELLNESS, RAID_CTLR_LUNID, buffer,
+		buffer_length, 0, &pci_direction);
+	if (rc)
+		return rc;
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
+		0, NULL, NO_TIMEOUT);
+
+	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
+		pci_direction);
+
+	return rc;
+}
+
+#pragma pack(1)
+
+struct bmic_host_wellness_driver_version {
+	u8	start_tag[4];
+	u8	driver_version_tag[2];
+	__le16	driver_version_length;
+	char	driver_version[32];
+	u8	end_tag[2];
+};
+
+#pragma pack()
+
+static int pqi_write_driver_version_to_host_wellness(
+	struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	struct bmic_host_wellness_driver_version *buffer;
+	size_t buffer_length;
+
+	buffer_length = sizeof(*buffer);
+
+	buffer = kmalloc(buffer_length, GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	buffer->start_tag[0] = '<';
+	buffer->start_tag[1] = 'H';
+	buffer->start_tag[2] = 'W';
+	buffer->start_tag[3] = '>';
+	buffer->driver_version_tag[0] = 'D';
+	buffer->driver_version_tag[1] = 'V';
+	put_unaligned_le16(sizeof(buffer->driver_version),
+		&buffer->driver_version_length);
+	strncpy(buffer->driver_version, DRIVER_VERSION,
+		sizeof(buffer->driver_version) - 1);
+	buffer->driver_version[sizeof(buffer->driver_version) - 1] = '\0';
+	buffer->end_tag[0] = 'Z';
+	buffer->end_tag[1] = 'Z';
+
+	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
+
+	kfree(buffer);
+
+	return rc;
+}
+
+#pragma pack(1)
+
+struct bmic_host_wellness_time {
+	u8	start_tag[4];
+	u8	time_tag[2];
+	__le16	time_length;
+	u8	time[8];
+	u8	dont_write_tag[2];
+	u8	end_tag[2];
+};
+
+#pragma pack()
+
+static int pqi_write_current_time_to_host_wellness(
+	struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	struct bmic_host_wellness_time *buffer;
+	size_t buffer_length;
+	time64_t local_time;
+	unsigned int year;
+	struct timeval time;
+	struct rtc_time tm;
+
+	buffer_length = sizeof(*buffer);
+
+	buffer = kmalloc(buffer_length, GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	buffer->start_tag[0] = '<';
+	buffer->start_tag[1] = 'H';
+	buffer->start_tag[2] = 'W';
+	buffer->start_tag[3] = '>';
+	buffer->time_tag[0] = 'T';
+	buffer->time_tag[1] = 'D';
+	put_unaligned_le16(sizeof(buffer->time),
+		&buffer->time_length);
+
+	do_gettimeofday(&time);
+	local_time = time.tv_sec - (sys_tz.tz_minuteswest * 60);
+	rtc_time64_to_tm(local_time, &tm);
+	year = tm.tm_year + 1900;
+
+	buffer->time[0] = bin2bcd(tm.tm_hour);
+	buffer->time[1] = bin2bcd(tm.tm_min);
+	buffer->time[2] = bin2bcd(tm.tm_sec);
+	buffer->time[3] = 0;
+	buffer->time[4] = bin2bcd(tm.tm_mon + 1);
+	buffer->time[5] = bin2bcd(tm.tm_mday);
+	buffer->time[6] = bin2bcd(year / 100);
+	buffer->time[7] = bin2bcd(year % 100);
+
+	buffer->dont_write_tag[0] = 'D';
+	buffer->dont_write_tag[1] = 'W';
+	buffer->end_tag[0] = 'Z';
+	buffer->end_tag[1] = 'Z';
+
+	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
+
+	kfree(buffer);
+
+	return rc;
+}
+
+#define PQI_UPDATE_TIME_WORK_INTERVAL	(24UL * 60 * 60 * HZ)
+
+static void pqi_update_time_worker(struct work_struct *work)
+{
+	int rc;
+	struct pqi_ctrl_info *ctrl_info;
+
+	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
+		update_time_work);
+
+	rc = pqi_write_current_time_to_host_wellness(ctrl_info);
+	if (rc)
+		dev_warn(&ctrl_info->pci_dev->dev,
+			"error updating time on controller\n");
+
+	schedule_delayed_work(&ctrl_info->update_time_work,
+		PQI_UPDATE_TIME_WORK_INTERVAL);
+}
+
+static inline void pqi_schedule_update_time_worker(
+	struct pqi_ctrl_info *ctrl_info)
+{
+	schedule_delayed_work(&ctrl_info->update_time_work, 0);
+}
+
+static int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,
+	void *buffer, size_t buffer_length)
+{
+	int rc;
+	int pci_direction;
+	struct pqi_raid_path_request request;
+
+	rc = pqi_build_raid_path_request(ctrl_info, &request,
+		cmd, RAID_CTLR_LUNID, buffer, buffer_length, 0, &pci_direction);
+	if (rc)
+		return rc;
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
+		NULL, NO_TIMEOUT);
+
+	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
+		pci_direction);
+
+	return rc;
+}
+
+static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,
+	void **buffer)
+{
+	int rc;
+	size_t lun_list_length;
+	size_t lun_data_length;
+	size_t new_lun_list_length;
+	void *lun_data = NULL;
+	struct report_lun_header *report_lun_header;
+
+	report_lun_header = kmalloc(sizeof(*report_lun_header), GFP_KERNEL);
+	if (!report_lun_header) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	rc = pqi_report_luns(ctrl_info, cmd, report_lun_header,
+		sizeof(*report_lun_header));
+	if (rc)
+		goto out;
+
+	lun_list_length = get_unaligned_be32(&report_lun_header->list_length);
+
+again:
+	lun_data_length = sizeof(struct report_lun_header) + lun_list_length;
+
+	lun_data = kmalloc(lun_data_length, GFP_KERNEL);
+	if (!lun_data) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	if (lun_list_length == 0) {
+		memcpy(lun_data, report_lun_header, sizeof(*report_lun_header));
+		goto out;
+	}
+
+	rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);
+	if (rc)
+		goto out;
+
+	new_lun_list_length = get_unaligned_be32(
+		&((struct report_lun_header *)lun_data)->list_length);
+
+	if (new_lun_list_length > lun_list_length) {
+		lun_list_length = new_lun_list_length;
+		kfree(lun_data);
+		goto again;
+	}
+
+out:
+	kfree(report_lun_header);
+
+	if (rc) {
+		kfree(lun_data);
+		lun_data = NULL;
+	}
+
+	*buffer = lun_data;
+
+	return rc;
+}
+
+static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info,
+	void **buffer)
+{
+	return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS,
+		buffer);
+}
+
+static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info,
+	void **buffer)
+{
+	return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);
+}
+
+static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,
+	struct report_phys_lun_extended **physdev_list,
+	struct report_log_lun_extended **logdev_list)
+{
+	int rc;
+	size_t logdev_list_length;
+	size_t logdev_data_length;
+	struct report_log_lun_extended *internal_logdev_list;
+	struct report_log_lun_extended *logdev_data;
+	struct report_lun_header report_lun_header;
+
+	rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);
+	if (rc)
+		dev_err(&ctrl_info->pci_dev->dev,
+			"report physical LUNs failed\n");
+
+	rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);
+	if (rc)
+		dev_err(&ctrl_info->pci_dev->dev,
+			"report logical LUNs failed\n");
+
+	/*
+	 * Tack the controller itself onto the end of the logical device list.
+	 */
+
+	logdev_data = *logdev_list;
+
+	if (logdev_data) {
+		logdev_list_length =
+			get_unaligned_be32(&logdev_data->header.list_length);
+	} else {
+		memset(&report_lun_header, 0, sizeof(report_lun_header));
+		logdev_data =
+			(struct report_log_lun_extended *)&report_lun_header;
+		logdev_list_length = 0;
+	}
+
+	logdev_data_length = sizeof(struct report_lun_header) +
+		logdev_list_length;
+
+	internal_logdev_list = kmalloc(logdev_data_length +
+		sizeof(struct report_log_lun_extended), GFP_KERNEL);
+	if (!internal_logdev_list) {
+		kfree(*logdev_list);
+		*logdev_list = NULL;
+		return -ENOMEM;
+	}
+
+	memcpy(internal_logdev_list, logdev_data, logdev_data_length);
+	memset((u8 *)internal_logdev_list + logdev_data_length, 0,
+		sizeof(struct report_log_lun_extended_entry));
+	put_unaligned_be32(logdev_list_length +
+		sizeof(struct report_log_lun_extended_entry),
+		&internal_logdev_list->header.list_length);
+
+	kfree(*logdev_list);
+	*logdev_list = internal_logdev_list;
+
+	return 0;
+}
+
+static inline void pqi_set_bus_target_lun(struct pqi_scsi_dev *device,
+	int bus, int target, int lun)
+{
+	device->bus = bus;
+	device->target = target;
+	device->lun = lun;
+}
+
+static void pqi_assign_bus_target_lun(struct pqi_scsi_dev *device)
+{
+	u8 *scsi3addr;
+	u32 lunid;
+
+	scsi3addr = device->scsi3addr;
+	lunid = get_unaligned_le32(scsi3addr);
+
+	if (pqi_is_hba_lunid(scsi3addr)) {
+		/* The specified device is the controller. */
+		pqi_set_bus_target_lun(device, PQI_HBA_BUS, 0, lunid & 0x3fff);
+		device->target_lun_valid = true;
+		return;
+	}
+
+	if (pqi_is_logical_device(device)) {
+		pqi_set_bus_target_lun(device, PQI_RAID_VOLUME_BUS, 0,
+			lunid & 0x3fff);
+		device->target_lun_valid = true;
+		return;
+	}
+
+	/*
+	 * Defer target and LUN assignment for non-controller physical devices
+	 * because the SAS transport layer will make these assignments later.
+	 */
+	pqi_set_bus_target_lun(device, PQI_PHYSICAL_DEVICE_BUS, 0, 0);
+}
+
+static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	int rc;
+	u8 raid_level;
+	u8 *buffer;
+
+	raid_level = SA_RAID_UNKNOWN;
+
+	buffer = kmalloc(64, GFP_KERNEL);
+	if (buffer) {
+		rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
+			VPD_PAGE | CISS_VPD_LV_DEVICE_GEOMETRY, buffer, 64);
+		if (rc == 0) {
+			raid_level = buffer[8];
+			if (raid_level > SA_RAID_MAX)
+				raid_level = SA_RAID_UNKNOWN;
+		}
+		kfree(buffer);
+	}
+
+	device->raid_level = raid_level;
+}
+
+static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device, struct raid_map *raid_map)
+{
+	char *err_msg;
+	u32 raid_map_size;
+	u32 r5or6_blocks_per_row;
+	unsigned int num_phys_disks;
+	unsigned int num_raid_map_entries;
+
+	raid_map_size = get_unaligned_le32(&raid_map->structure_size);
+
+	if (raid_map_size < offsetof(struct raid_map, disk_data)) {
+		err_msg = "RAID map too small";
+		goto bad_raid_map;
+	}
+
+	if (raid_map_size > sizeof(*raid_map)) {
+		err_msg = "RAID map too large";
+		goto bad_raid_map;
+	}
+
+	num_phys_disks = get_unaligned_le16(&raid_map->layout_map_count) *
+		(get_unaligned_le16(&raid_map->data_disks_per_row) +
+		get_unaligned_le16(&raid_map->metadata_disks_per_row));
+	num_raid_map_entries = num_phys_disks *
+		get_unaligned_le16(&raid_map->row_cnt);
+
+	if (num_raid_map_entries > RAID_MAP_MAX_ENTRIES) {
+		err_msg = "invalid number of map entries in RAID map";
+		goto bad_raid_map;
+	}
+
+	if (device->raid_level == SA_RAID_1) {
+		if (get_unaligned_le16(&raid_map->layout_map_count) != 2) {
+			err_msg = "invalid RAID-1 map";
+			goto bad_raid_map;
+		}
+	} else if (device->raid_level == SA_RAID_ADM) {
+		if (get_unaligned_le16(&raid_map->layout_map_count) != 3) {
+			err_msg = "invalid RAID-1(ADM) map";
+			goto bad_raid_map;
+		}
+	} else if ((device->raid_level == SA_RAID_5 ||
+		device->raid_level == SA_RAID_6) &&
+		get_unaligned_le16(&raid_map->layout_map_count) > 1) {
+		/* RAID 50/60 */
+		r5or6_blocks_per_row =
+			get_unaligned_le16(&raid_map->strip_size) *
+			get_unaligned_le16(&raid_map->data_disks_per_row);
+		if (r5or6_blocks_per_row == 0) {
+			err_msg = "invalid RAID-5 or RAID-6 map";
+			goto bad_raid_map;
+		}
+	}
+
+	return 0;
+
+bad_raid_map:
+	dev_warn(&ctrl_info->pci_dev->dev, "%s\n", err_msg);
+
+	return -EINVAL;
+}
+
+static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	int rc;
+	int pci_direction;
+	struct pqi_raid_path_request request;
+	struct raid_map *raid_map;
+
+	raid_map = kmalloc(sizeof(*raid_map), GFP_KERNEL);
+	if (!raid_map)
+		return -ENOMEM;
+
+	rc = pqi_build_raid_path_request(ctrl_info, &request,
+		CISS_GET_RAID_MAP, device->scsi3addr, raid_map,
+		sizeof(*raid_map), 0, &pci_direction);
+	if (rc)
+		goto error;
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
+		NULL, NO_TIMEOUT);
+
+	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
+		pci_direction);
+
+	if (rc)
+		goto error;
+
+	rc = pqi_validate_raid_map(ctrl_info, device, raid_map);
+	if (rc)
+		goto error;
+
+	device->raid_map = raid_map;
+
+	return 0;
+
+error:
+	kfree(raid_map);
+
+	return rc;
+}
+
+static void pqi_get_offload_status(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	int rc;
+	u8 *buffer;
+	u8 offload_status;
+
+	buffer = kmalloc(64, GFP_KERNEL);
+	if (!buffer)
+		return;
+
+	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
+		VPD_PAGE | CISS_VPD_LV_OFFLOAD_STATUS, buffer, 64);
+	if (rc)
+		goto out;
+
+#define OFFLOAD_STATUS_BYTE	4
+#define OFFLOAD_CONFIGURED_BIT	0x1
+#define OFFLOAD_ENABLED_BIT	0x2
+
+	offload_status = buffer[OFFLOAD_STATUS_BYTE];
+	device->offload_configured =
+		!!(offload_status & OFFLOAD_CONFIGURED_BIT);
+	if (device->offload_configured) {
+		device->offload_enabled_pending =
+			!!(offload_status & OFFLOAD_ENABLED_BIT);
+		if (pqi_get_raid_map(ctrl_info, device))
+			device->offload_enabled_pending = false;
+	}
+
+out:
+	kfree(buffer);
+}
+
+/*
+ * Use vendor-specific VPD to determine online/offline status of a volume.
+ */
+
+static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	int rc;
+	size_t page_length;
+	u8 volume_status = CISS_LV_STATUS_UNAVAILABLE;
+	bool volume_offline = true;
+	u32 volume_flags;
+	struct ciss_vpd_logical_volume_status *vpd;
+
+	vpd = kmalloc(sizeof(*vpd), GFP_KERNEL);
+	if (!vpd)
+		goto no_buffer;
+
+	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
+		VPD_PAGE | CISS_VPD_LV_STATUS, vpd, sizeof(*vpd));
+	if (rc)
+		goto out;
+
+	page_length = offsetof(struct ciss_vpd_logical_volume_status,
+		volume_status) + vpd->page_length;
+	if (page_length < sizeof(*vpd))
+		goto out;
+
+	volume_status = vpd->volume_status;
+	volume_flags = get_unaligned_be32(&vpd->flags);
+	volume_offline = (volume_flags & CISS_LV_FLAGS_NO_HOST_IO) != 0;
+
+out:
+	kfree(vpd);
+no_buffer:
+	device->volume_status = volume_status;
+	device->volume_offline = volume_offline;
+}
+
+static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	int rc;
+	u8 *buffer;
+
+	buffer = kmalloc(64, GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	/* Send an inquiry to the device to see what it is. */
+	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0, buffer, 64);
+	if (rc)
+		goto out;
+
+	scsi_sanitize_inquiry_string(&buffer[8], 8);
+	scsi_sanitize_inquiry_string(&buffer[16], 16);
+
+	device->devtype = buffer[0] & 0x1f;
+	memcpy(device->vendor, &buffer[8],
+		sizeof(device->vendor));
+	memcpy(device->model, &buffer[16],
+		sizeof(device->model));
+
+	if (pqi_is_logical_device(device) && device->devtype == TYPE_DISK) {
+		pqi_get_raid_level(ctrl_info, device);
+		pqi_get_offload_status(ctrl_info, device);
+		pqi_get_volume_status(ctrl_info, device);
+	}
+
+out:
+	kfree(buffer);
+
+	return rc;
+}
+
+static void pqi_get_physical_disk_info(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device,
+	struct bmic_identify_physical_device *id_phys)
+{
+	int rc;
+
+	memset(id_phys, 0, sizeof(*id_phys));
+
+	rc = pqi_identify_physical_device(ctrl_info, device,
+		id_phys, sizeof(*id_phys));
+	if (rc) {
+		device->queue_depth = PQI_PHYSICAL_DISK_DEFAULT_MAX_QUEUE_DEPTH;
+		return;
+	}
+
+	device->queue_depth =
+		get_unaligned_le16(&id_phys->current_queue_depth_limit);
+	device->device_type = id_phys->device_type;
+	device->active_path_index = id_phys->active_path_number;
+	device->path_map = id_phys->redundant_path_present_map;
+	memcpy(&device->box,
+		&id_phys->alternate_paths_phys_box_on_port,
+		sizeof(device->box));
+	memcpy(&device->phys_connector,
+		&id_phys->alternate_paths_phys_connector,
+		sizeof(device->phys_connector));
+	device->bay = id_phys->phys_bay_in_box;
+}
+
+static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	char *status;
+	static const char unknown_state_str[] =
+		"Volume is in an unknown state (%u)";
+	char unknown_state_buffer[sizeof(unknown_state_str) + 10];
+
+	switch (device->volume_status) {
+	case CISS_LV_OK:
+		status = "Volume online";
+		break;
+	case CISS_LV_FAILED:
+		status = "Volume failed";
+		break;
+	case CISS_LV_NOT_CONFIGURED:
+		status = "Volume not configured";
+		break;
+	case CISS_LV_DEGRADED:
+		status = "Volume degraded";
+		break;
+	case CISS_LV_READY_FOR_RECOVERY:
+		status = "Volume ready for recovery operation";
+		break;
+	case CISS_LV_UNDERGOING_RECOVERY:
+		status = "Volume undergoing recovery";
+		break;
+	case CISS_LV_WRONG_PHYSICAL_DRIVE_REPLACED:
+		status = "Wrong physical drive was replaced";
+		break;
+	case CISS_LV_PHYSICAL_DRIVE_CONNECTION_PROBLEM:
+		status = "A physical drive not properly connected";
+		break;
+	case CISS_LV_HARDWARE_OVERHEATING:
+		status = "Hardware is overheating";
+		break;
+	case CISS_LV_HARDWARE_HAS_OVERHEATED:
+		status = "Hardware has overheated";
+		break;
+	case CISS_LV_UNDERGOING_EXPANSION:
+		status = "Volume undergoing expansion";
+		break;
+	case CISS_LV_NOT_AVAILABLE:
+		status = "Volume waiting for transforming volume";
+		break;
+	case CISS_LV_QUEUED_FOR_EXPANSION:
+		status = "Volume queued for expansion";
+		break;
+	case CISS_LV_DISABLED_SCSI_ID_CONFLICT:
+		status = "Volume disabled due to SCSI ID conflict";
+		break;
+	case CISS_LV_EJECTED:
+		status = "Volume has been ejected";
+		break;
+	case CISS_LV_UNDERGOING_ERASE:
+		status = "Volume undergoing background erase";
+		break;
+	case CISS_LV_READY_FOR_PREDICTIVE_SPARE_REBUILD:
+		status = "Volume ready for predictive spare rebuild";
+		break;
+	case CISS_LV_UNDERGOING_RPI:
+		status = "Volume undergoing rapid parity initialization";
+		break;
+	case CISS_LV_PENDING_RPI:
+		status = "Volume queued for rapid parity initialization";
+		break;
+	case CISS_LV_ENCRYPTED_NO_KEY:
+		status = "Encrypted volume inaccessible - key not present";
+		break;
+	case CISS_LV_UNDERGOING_ENCRYPTION:
+		status = "Volume undergoing encryption process";
+		break;
+	case CISS_LV_UNDERGOING_ENCRYPTION_REKEYING:
+		status = "Volume undergoing encryption re-keying process";
+		break;
+	case CISS_LV_ENCRYPTED_IN_NON_ENCRYPTED_CONTROLLER:
+		status =
+			"Encrypted volume inaccessible - disabled on ctrl";
+		break;
+	case CISS_LV_PENDING_ENCRYPTION:
+		status = "Volume pending migration to encrypted state";
+		break;
+	case CISS_LV_PENDING_ENCRYPTION_REKEYING:
+		status = "Volume pending encryption rekeying";
+		break;
+	case CISS_LV_NOT_SUPPORTED:
+		status = "Volume not supported on this controller";
+		break;
+	case CISS_LV_STATUS_UNAVAILABLE:
+		status = "Volume status not available";
+		break;
+	default:
+		snprintf(unknown_state_buffer, sizeof(unknown_state_buffer),
+			unknown_state_str, device->volume_status);
+		status = unknown_state_buffer;
+		break;
+	}
+
+	dev_info(&ctrl_info->pci_dev->dev,
+		"scsi %d:%d:%d:%d %s\n",
+		ctrl_info->scsi_host->host_no,
+		device->bus, device->target, device->lun, status);
+}
+
+static struct pqi_scsi_dev *pqi_find_disk_by_aio_handle(
+	struct pqi_ctrl_info *ctrl_info, u32 aio_handle)
+{
+	struct pqi_scsi_dev *device;
+
+	list_for_each_entry(device, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry) {
+		if (device->devtype != TYPE_DISK && device->devtype != TYPE_ZBC)
+			continue;
+		if (pqi_is_logical_device(device))
+			continue;
+		if (device->aio_handle == aio_handle)
+			return device;
+	}
+
+	return NULL;
+}
+
+static void pqi_update_logical_drive_queue_depth(
+	struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *logical_drive)
+{
+	unsigned int i;
+	struct raid_map *raid_map;
+	struct raid_map_disk_data *disk_data;
+	struct pqi_scsi_dev *phys_disk;
+	unsigned int num_phys_disks;
+	unsigned int num_raid_map_entries;
+	unsigned int queue_depth;
+
+	logical_drive->queue_depth = PQI_LOGICAL_DRIVE_DEFAULT_MAX_QUEUE_DEPTH;
+
+	raid_map = logical_drive->raid_map;
+	if (!raid_map)
+		return;
+
+	disk_data = raid_map->disk_data;
+	num_phys_disks = get_unaligned_le16(&raid_map->layout_map_count) *
+		(get_unaligned_le16(&raid_map->data_disks_per_row) +
+		get_unaligned_le16(&raid_map->metadata_disks_per_row));
+	num_raid_map_entries = num_phys_disks *
+		get_unaligned_le16(&raid_map->row_cnt);
+
+	queue_depth = 0;
+	for (i = 0; i < num_raid_map_entries; i++) {
+		phys_disk = pqi_find_disk_by_aio_handle(ctrl_info,
+			disk_data[i].aio_handle);
+
+		if (!phys_disk) {
+			dev_warn(&ctrl_info->pci_dev->dev,
+				"failed to find physical disk for logical drive %016llx\n",
+				get_unaligned_be64(logical_drive->scsi3addr));
+			logical_drive->offload_enabled = false;
+			logical_drive->offload_enabled_pending = false;
+			kfree(raid_map);
+			logical_drive->raid_map = NULL;
+			return;
+		}
+
+		queue_depth += phys_disk->queue_depth;
+	}
+
+	logical_drive->queue_depth = queue_depth;
+}
+
+static void pqi_update_all_logical_drive_queue_depths(
+	struct pqi_ctrl_info *ctrl_info)
+{
+	struct pqi_scsi_dev *device;
+
+	list_for_each_entry(device, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry) {
+		if (device->devtype != TYPE_DISK && device->devtype != TYPE_ZBC)
+			continue;
+		if (!pqi_is_logical_device(device))
+			continue;
+		pqi_update_logical_drive_queue_depth(ctrl_info, device);
+	}
+}
+
+static void pqi_rescan_worker(struct work_struct *work)
+{
+	struct pqi_ctrl_info *ctrl_info;
+
+	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
+		rescan_work);
+
+	pqi_scan_scsi_devices(ctrl_info);
+}
+
+static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	int rc;
+
+	if (pqi_is_logical_device(device))
+		rc = scsi_add_device(ctrl_info->scsi_host, device->bus,
+			device->target, device->lun);
+	else
+		rc = pqi_add_sas_device(ctrl_info->sas_host, device);
+
+	return rc;
+}
+
+static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	if (pqi_is_logical_device(device))
+		scsi_remove_device(device->sdev);
+	else
+		pqi_remove_sas_device(device);
+}
+
+/* Assumes the SCSI device list lock is held. */
+
+static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,
+	int bus, int target, int lun)
+{
+	struct pqi_scsi_dev *device;
+
+	list_for_each_entry(device, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry)
+		if (device->bus == bus && device->target == target &&
+			device->lun == lun)
+			return device;
+
+	return NULL;
+}
+
+static inline bool pqi_device_equal(struct pqi_scsi_dev *dev1,
+	struct pqi_scsi_dev *dev2)
+{
+	if (dev1->is_physical_device != dev2->is_physical_device)
+		return false;
+
+	if (dev1->is_physical_device)
+		return dev1->wwid == dev2->wwid;
+
+	return memcmp(dev1->volume_id, dev2->volume_id,
+		sizeof(dev1->volume_id)) == 0;
+}
+
+enum pqi_find_result {
+	DEVICE_NOT_FOUND,
+	DEVICE_CHANGED,
+	DEVICE_SAME,
+};
+
+static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device_to_find,
+	struct pqi_scsi_dev **matching_device)
+{
+	struct pqi_scsi_dev *device;
+
+	list_for_each_entry(device, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry) {
+		if (pqi_scsi3addr_equal(device_to_find->scsi3addr,
+			device->scsi3addr)) {
+			*matching_device = device;
+			if (pqi_device_equal(device_to_find, device)) {
+				if (device_to_find->volume_offline)
+					return DEVICE_CHANGED;
+				return DEVICE_SAME;
+			}
+			return DEVICE_CHANGED;
+		}
+	}
+
+	return DEVICE_NOT_FOUND;
+}
+
+static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,
+	char *action, struct pqi_scsi_dev *device)
+{
+	dev_info(&ctrl_info->pci_dev->dev,
+		"%s scsi %d:%d:%d:%d: %s %.8s %.16s %-12s SSDSmartPathCap%c En%c Exp%c qd=%d\n",
+		action,
+		ctrl_info->scsi_host->host_no,
+		device->bus,
+		device->target,
+		device->lun,
+		scsi_device_type(device->devtype),
+		device->vendor,
+		device->model,
+		pqi_raid_level_to_string(device->raid_level),
+		device->offload_configured ? '+' : '-',
+		device->offload_enabled_pending ? '+' : '-',
+		device->expose_device ? '+' : '-',
+		device->queue_depth);
+}
+
+/* Assumes the SCSI device list lock is held. */
+
+static void pqi_scsi_update_device(struct pqi_scsi_dev *existing_device,
+	struct pqi_scsi_dev *new_device)
+{
+	existing_device->devtype = new_device->devtype;
+	existing_device->device_type = new_device->device_type;
+	existing_device->bus = new_device->bus;
+	if (new_device->target_lun_valid) {
+		existing_device->target = new_device->target;
+		existing_device->lun = new_device->lun;
+		existing_device->target_lun_valid = true;
+	}
+
+	/* By definition, the scsi3addr and wwid fields are already the same. */
+
+	existing_device->is_physical_device = new_device->is_physical_device;
+	existing_device->expose_device = new_device->expose_device;
+	existing_device->no_uld_attach = new_device->no_uld_attach;
+	existing_device->aio_enabled = new_device->aio_enabled;
+	memcpy(existing_device->vendor, new_device->vendor,
+		sizeof(existing_device->vendor));
+	memcpy(existing_device->model, new_device->model,
+		sizeof(existing_device->model));
+	existing_device->sas_address = new_device->sas_address;
+	existing_device->raid_level = new_device->raid_level;
+	existing_device->queue_depth = new_device->queue_depth;
+	existing_device->aio_handle = new_device->aio_handle;
+	existing_device->volume_status = new_device->volume_status;
+	existing_device->active_path_index = new_device->active_path_index;
+	existing_device->path_map = new_device->path_map;
+	existing_device->bay = new_device->bay;
+	memcpy(existing_device->box, new_device->box,
+		sizeof(existing_device->box));
+	memcpy(existing_device->phys_connector, new_device->phys_connector,
+		sizeof(existing_device->phys_connector));
+	existing_device->offload_configured = new_device->offload_configured;
+	existing_device->offload_enabled = false;
+	existing_device->offload_enabled_pending =
+		new_device->offload_enabled_pending;
+	existing_device->offload_to_mirror = 0;
+	kfree(existing_device->raid_map);
+	existing_device->raid_map = new_device->raid_map;
+
+	/* To prevent this from being freed later. */
+	new_device->raid_map = NULL;
+}
+
+static inline void pqi_free_device(struct pqi_scsi_dev *device)
+{
+	if (device) {
+		kfree(device->raid_map);
+		kfree(device);
+	}
+}
+
+/*
+ * Called when exposing a new device to the OS fails in order to re-adjust
+ * our internal SCSI device list to match the SCSI ML's view.
+ */
+
+static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
+	list_del(&device->scsi_device_list_entry);
+	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
+
+	/* Allow the device structure to be freed later. */
+	device->keep_device = false;
+}
+
+static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *new_device_list[], unsigned int num_new_devices)
+{
+	int rc;
+	unsigned int i;
+	unsigned long flags;
+	enum pqi_find_result find_result;
+	struct pqi_scsi_dev *device;
+	struct pqi_scsi_dev *next;
+	struct pqi_scsi_dev *matching_device;
+	struct list_head add_list;
+	struct list_head delete_list;
+
+	INIT_LIST_HEAD(&add_list);
+	INIT_LIST_HEAD(&delete_list);
+
+	/*
+	 * The idea here is to do as little work as possible while holding the
+	 * spinlock.  That's why we go to great pains to defer anything other
+	 * than updating the internal device list until after we release the
+	 * spinlock.
+	 */
+
+	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
+
+	/* Assume that all devices in the existing list have gone away. */
+	list_for_each_entry(device, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry)
+		device->device_gone = true;
+
+	for (i = 0; i < num_new_devices; i++) {
+		device = new_device_list[i];
+
+		find_result = pqi_scsi_find_entry(ctrl_info, device,
+						&matching_device);
+
+		switch (find_result) {
+		case DEVICE_SAME:
+			/*
+			 * The newly found device is already in the existing
+			 * device list.
+			 */
+			device->new_device = false;
+			matching_device->device_gone = false;
+			pqi_scsi_update_device(matching_device, device);
+			break;
+		case DEVICE_NOT_FOUND:
+			/*
+			 * The newly found device is NOT in the existing device
+			 * list.
+			 */
+			device->new_device = true;
+			break;
+		case DEVICE_CHANGED:
+			/*
+			 * The original device has gone away and we need to add
+			 * the new device.
+			 */
+			device->new_device = true;
+			break;
+		default:
+			WARN_ON(find_result);
+			break;
+		}
+	}
+
+	/* Process all devices that have gone away. */
+	list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry) {
+		if (device->device_gone) {
+			list_del(&device->scsi_device_list_entry);
+			list_add_tail(&device->delete_list_entry, &delete_list);
+		}
+	}
+
+	/* Process all new devices. */
+	for (i = 0; i < num_new_devices; i++) {
+		device = new_device_list[i];
+		if (!device->new_device)
+			continue;
+		if (device->volume_offline)
+			continue;
+		list_add_tail(&device->scsi_device_list_entry,
+			&ctrl_info->scsi_device_list);
+		list_add_tail(&device->add_list_entry, &add_list);
+		/* To prevent this device structure from being freed later. */
+		device->keep_device = true;
+	}
+
+	pqi_update_all_logical_drive_queue_depths(ctrl_info);
+
+	list_for_each_entry(device, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry)
+		device->offload_enabled =
+			device->offload_enabled_pending;
+
+	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
+
+	/* Remove all devices that have gone away. */
+	list_for_each_entry_safe(device, next, &delete_list,
+		delete_list_entry) {
+		if (device->sdev)
+			pqi_remove_device(ctrl_info, device);
+		if (device->volume_offline) {
+			pqi_dev_info(ctrl_info, "offline", device);
+			pqi_show_volume_status(ctrl_info, device);
+		} else {
+			pqi_dev_info(ctrl_info, "removed", device);
+		}
+		list_del(&device->delete_list_entry);
+		pqi_free_device(device);
+	}
+
+	/*
+	 * Notify the SCSI ML if the queue depth of any existing device has
+	 * changed.
+	 */
+	list_for_each_entry(device, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry) {
+		if (device->sdev && device->queue_depth !=
+			device->advertised_queue_depth) {
+			device->advertised_queue_depth = device->queue_depth;
+			scsi_change_queue_depth(device->sdev,
+				device->advertised_queue_depth);
+		}
+	}
+
+	/* Expose any new devices. */
+	list_for_each_entry_safe(device, next, &add_list, add_list_entry) {
+		if (device->expose_device && !device->sdev) {
+			rc = pqi_add_device(ctrl_info, device);
+			if (rc) {
+				dev_warn(&ctrl_info->pci_dev->dev,
+					"scsi %d:%d:%d:%d addition failed, device not added\n",
+					ctrl_info->scsi_host->host_no,
+					device->bus, device->target,
+					device->lun);
+				pqi_fixup_botched_add(ctrl_info, device);
+				continue;
+			}
+		}
+		pqi_dev_info(ctrl_info, "added", device);
+	}
+}
+
+static bool pqi_is_supported_device(struct pqi_scsi_dev *device)
+{
+	bool is_supported = false;
+
+	switch (device->devtype) {
+	case TYPE_DISK:
+	case TYPE_ZBC:
+	case TYPE_TAPE:
+	case TYPE_MEDIUM_CHANGER:
+	case TYPE_ENCLOSURE:
+		is_supported = true;
+		break;
+	case TYPE_RAID:
+		/*
+		 * Only support the HBA controller itself as a RAID
+		 * controller.  If it's a RAID controller other than
+		 * the HBA itself (an external RAID controller, MSA500
+		 * or similar), we don't support it.
+		 */
+		if (pqi_is_hba_lunid(device->scsi3addr))
+			is_supported = true;
+		break;
+	}
+
+	return is_supported;
+}
+
+static inline bool pqi_skip_device(u8 *scsi3addr,
+	struct report_phys_lun_extended_entry *phys_lun_ext_entry)
+{
+	u8 device_flags;
+
+	if (!MASKED_DEVICE(scsi3addr))
+		return false;
+
+	/* The device is masked. */
+
+	device_flags = phys_lun_ext_entry->device_flags;
+
+	if (device_flags & REPORT_PHYS_LUN_DEV_FLAG_NON_DISK) {
+		/*
+		 * It's a non-disk device.  We ignore all devices of this type
+		 * when they're masked.
+		 */
+		return true;
+	}
+
+	return false;
+}
+
+static inline bool pqi_expose_device(struct pqi_scsi_dev *device)
+{
+	/* Expose all devices except for physical devices that are masked. */
+	if (device->is_physical_device && MASKED_DEVICE(device->scsi3addr))
+		return false;
+
+	return true;
+}
+
+static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)
+{
+	int i;
+	int rc;
+	struct list_head new_device_list_head;
+	struct report_phys_lun_extended *physdev_list = NULL;
+	struct report_log_lun_extended *logdev_list = NULL;
+	struct report_phys_lun_extended_entry *phys_lun_ext_entry;
+	struct report_log_lun_extended_entry *log_lun_ext_entry;
+	struct bmic_identify_physical_device *id_phys = NULL;
+	u32 num_physicals;
+	u32 num_logicals;
+	struct pqi_scsi_dev **new_device_list = NULL;
+	struct pqi_scsi_dev *device;
+	struct pqi_scsi_dev *next;
+	unsigned int num_new_devices;
+	unsigned int num_valid_devices;
+	bool is_physical_device;
+	u8 *scsi3addr;
+	static char *out_of_memory_msg =
+		"out of memory, device discovery stopped";
+
+	INIT_LIST_HEAD(&new_device_list_head);
+
+	rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);
+	if (rc)
+		goto out;
+
+	if (physdev_list)
+		num_physicals =
+			get_unaligned_be32(&physdev_list->header.list_length)
+				/ sizeof(physdev_list->lun_entries[0]);
+	else
+		num_physicals = 0;
+
+	if (logdev_list)
+		num_logicals =
+			get_unaligned_be32(&logdev_list->header.list_length)
+				/ sizeof(logdev_list->lun_entries[0]);
+	else
+		num_logicals = 0;
+
+	if (num_physicals) {
+		/*
+		 * We need this buffer for calls to pqi_get_physical_disk_info()
+		 * below.  We allocate it here instead of inside
+		 * pqi_get_physical_disk_info() because it's a fairly large
+		 * buffer.
+		 */
+		id_phys = kmalloc(sizeof(*id_phys), GFP_KERNEL);
+		if (!id_phys) {
+			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
+				out_of_memory_msg);
+			rc = -ENOMEM;
+			goto out;
+		}
+	}
+
+	num_new_devices = num_physicals + num_logicals;
+
+	new_device_list = kmalloc(sizeof(*new_device_list) *
+		num_new_devices, GFP_KERNEL);
+	if (!new_device_list) {
+		dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	for (i = 0; i < num_new_devices; i++) {
+		device = kzalloc(sizeof(*device), GFP_KERNEL);
+		if (!device) {
+			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
+				out_of_memory_msg);
+			rc = -ENOMEM;
+			goto out;
+		}
+		list_add_tail(&device->new_device_list_entry,
+			&new_device_list_head);
+	}
+
+	device = NULL;
+	num_valid_devices = 0;
+
+	for (i = 0; i < num_new_devices; i++) {
+
+		if (i < num_physicals) {
+			is_physical_device = true;
+			phys_lun_ext_entry = &physdev_list->lun_entries[i];
+			log_lun_ext_entry = NULL;
+			scsi3addr = phys_lun_ext_entry->lunid;
+		} else {
+			is_physical_device = false;
+			phys_lun_ext_entry = NULL;
+			log_lun_ext_entry =
+				&logdev_list->lun_entries[i - num_physicals];
+			scsi3addr = log_lun_ext_entry->lunid;
+		}
+
+		if (is_physical_device &&
+			pqi_skip_device(scsi3addr, phys_lun_ext_entry))
+			continue;
+
+		if (device)
+			device = list_next_entry(device, new_device_list_entry);
+		else
+			device = list_first_entry(&new_device_list_head,
+				struct pqi_scsi_dev, new_device_list_entry);
+
+		memcpy(device->scsi3addr, scsi3addr, sizeof(device->scsi3addr));
+		device->is_physical_device = is_physical_device;
+		device->raid_level = SA_RAID_UNKNOWN;
+
+		/* Gather information about the device. */
+		rc = pqi_get_device_info(ctrl_info, device);
+		if (rc == -ENOMEM) {
+			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
+				out_of_memory_msg);
+			goto out;
+		}
+		if (rc) {
+			dev_warn(&ctrl_info->pci_dev->dev,
+				"obtaining device info failed, skipping device %016llx\n",
+				get_unaligned_be64(device->scsi3addr));
+			rc = 0;
+			continue;
+		}
+
+		if (!pqi_is_supported_device(device))
+			continue;
+
+		pqi_assign_bus_target_lun(device);
+
+		device->expose_device = pqi_expose_device(device);
+
+		if (device->is_physical_device) {
+			device->wwid = phys_lun_ext_entry->wwid;
+			if ((phys_lun_ext_entry->device_flags &
+				REPORT_PHYS_LUN_DEV_FLAG_AIO_ENABLED) &&
+				phys_lun_ext_entry->aio_handle)
+				device->aio_enabled = true;
+		} else {
+			memcpy(device->volume_id, log_lun_ext_entry->volume_id,
+				sizeof(device->volume_id));
+		}
+
+		switch (device->devtype) {
+		case TYPE_DISK:
+		case TYPE_ZBC:
+		case TYPE_ENCLOSURE:
+			if (device->is_physical_device) {
+				device->sas_address =
+					get_unaligned_be64(&device->wwid);
+				if (device->devtype == TYPE_DISK ||
+					device->devtype == TYPE_ZBC) {
+					device->aio_handle =
+						phys_lun_ext_entry->aio_handle;
+					pqi_get_physical_disk_info(ctrl_info,
+						device, id_phys);
+				}
+			}
+			break;
+		}
+
+		new_device_list[num_valid_devices++] = device;
+	}
+
+	pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);
+
+out:
+	list_for_each_entry_safe(device, next, &new_device_list_head,
+		new_device_list_entry) {
+		if (device->keep_device)
+			continue;
+		list_del(&device->new_device_list_entry);
+		pqi_free_device(device);
+	}
+
+	kfree(new_device_list);
+	kfree(physdev_list);
+	kfree(logdev_list);
+	kfree(id_phys);
+
+	return rc;
+}
+
+static void pqi_remove_all_scsi_devices(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned long flags;
+	struct pqi_scsi_dev *device;
+	struct pqi_scsi_dev *next;
+
+	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
+
+	list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry) {
+		if (device->sdev)
+			pqi_remove_device(ctrl_info, device);
+		list_del(&device->scsi_device_list_entry);
+		pqi_free_device(device);
+	}
+
+	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
+}
+
+static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+
+	if (pqi_ctrl_offline(ctrl_info))
+		return -ENXIO;
+
+	mutex_lock(&ctrl_info->scan_mutex);
+
+	rc = pqi_update_scsi_devices(ctrl_info);
+	if (rc)
+		pqi_schedule_rescan_worker(ctrl_info);
+
+	mutex_unlock(&ctrl_info->scan_mutex);
+
+	return rc;
+}
+
+static void pqi_scan_start(struct Scsi_Host *shost)
+{
+	pqi_scan_scsi_devices(shost_to_hba(shost));
+}
+
+/* Returns TRUE if scan is finished. */
+
+static int pqi_scan_finished(struct Scsi_Host *shost,
+	unsigned long elapsed_time)
+{
+	struct pqi_ctrl_info *ctrl_info;
+
+	ctrl_info = shost_priv(shost);
+
+	return !mutex_is_locked(&ctrl_info->scan_mutex);
+}
+
+static inline void pqi_set_encryption_info(
+	struct pqi_encryption_info *encryption_info, struct raid_map *raid_map,
+	u64 first_block)
+{
+	u32 volume_blk_size;
+
+	/*
+	 * Set the encryption tweak values based on logical block address.
+	 * If the block size is 512, the tweak value is equal to the LBA.
+	 * For other block sizes, tweak value is (LBA * block size) / 512.
+	 */
+	volume_blk_size = get_unaligned_le32(&raid_map->volume_blk_size);
+	if (volume_blk_size != 512)
+		first_block = (first_block * volume_blk_size) / 512;
+
+	encryption_info->data_encryption_key_index =
+		get_unaligned_le16(&raid_map->data_encryption_key_index);
+	encryption_info->encrypt_tweak_lower = lower_32_bits(first_block);
+	encryption_info->encrypt_tweak_upper = upper_32_bits(first_block);
+}
+
+/*
+ * Attempt to perform offload RAID mapping for a logical volume I/O.
+ */
+
+#define PQI_RAID_BYPASS_INELIGIBLE	1
+
+static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
+	struct pqi_queue_group *queue_group)
+{
+	struct raid_map *raid_map;
+	bool is_write = false;
+	u32 map_index;
+	u64 first_block;
+	u64 last_block;
+	u32 block_cnt;
+	u32 blocks_per_row;
+	u64 first_row;
+	u64 last_row;
+	u32 first_row_offset;
+	u32 last_row_offset;
+	u32 first_column;
+	u32 last_column;
+	u64 r0_first_row;
+	u64 r0_last_row;
+	u32 r5or6_blocks_per_row;
+	u64 r5or6_first_row;
+	u64 r5or6_last_row;
+	u32 r5or6_first_row_offset;
+	u32 r5or6_last_row_offset;
+	u32 r5or6_first_column;
+	u32 r5or6_last_column;
+	u16 data_disks_per_row;
+	u32 total_disks_per_row;
+	u16 layout_map_count;
+	u32 stripesize;
+	u16 strip_size;
+	u32 first_group;
+	u32 last_group;
+	u32 current_group;
+	u32 map_row;
+	u32 aio_handle;
+	u64 disk_block;
+	u32 disk_block_cnt;
+	u8 cdb[16];
+	u8 cdb_length;
+	int offload_to_mirror;
+	struct pqi_encryption_info *encryption_info_ptr;
+	struct pqi_encryption_info encryption_info;
+#if BITS_PER_LONG == 32
+	u64 tmpdiv;
+#endif
+
+	/* Check for valid opcode, get LBA and block count. */
+	switch (scmd->cmnd[0]) {
+	case WRITE_6:
+		is_write = true;
+		/* fall through */
+	case READ_6:
+		first_block = (u64)(((scmd->cmnd[1] & 0x1f) << 16) |
+			(scmd->cmnd[2] << 8) | scmd->cmnd[3]);
+		block_cnt = (u32)scmd->cmnd[4];
+		if (block_cnt == 0)
+			block_cnt = 256;
+		break;
+	case WRITE_10:
+		is_write = true;
+		/* fall through */
+	case READ_10:
+		first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
+		block_cnt = (u32)get_unaligned_be16(&scmd->cmnd[7]);
+		break;
+	case WRITE_12:
+		is_write = true;
+		/* fall through */
+	case READ_12:
+		first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
+		block_cnt = get_unaligned_be32(&scmd->cmnd[6]);
+		break;
+	case WRITE_16:
+		is_write = true;
+		/* fall through */
+	case READ_16:
+		first_block = get_unaligned_be64(&scmd->cmnd[2]);
+		block_cnt = get_unaligned_be32(&scmd->cmnd[10]);
+		break;
+	default:
+		/* Process via normal I/O path. */
+		return PQI_RAID_BYPASS_INELIGIBLE;
+	}
+
+	/* Check for write to non-RAID-0. */
+	if (is_write && device->raid_level != SA_RAID_0)
+		return PQI_RAID_BYPASS_INELIGIBLE;
+
+	if (unlikely(block_cnt == 0))
+		return PQI_RAID_BYPASS_INELIGIBLE;
+
+	last_block = first_block + block_cnt - 1;
+	raid_map = device->raid_map;
+
+	/* Check for invalid block or wraparound. */
+	if (last_block >= get_unaligned_le64(&raid_map->volume_blk_cnt) ||
+		last_block < first_block)
+		return PQI_RAID_BYPASS_INELIGIBLE;
+
+	data_disks_per_row = get_unaligned_le16(&raid_map->data_disks_per_row);
+	strip_size = get_unaligned_le16(&raid_map->strip_size);
+	layout_map_count = get_unaligned_le16(&raid_map->layout_map_count);
+
+	/* Calculate stripe information for the request. */
+	blocks_per_row = data_disks_per_row * strip_size;
+#if BITS_PER_LONG == 32
+	tmpdiv = first_block;
+	do_div(tmpdiv, blocks_per_row);
+	first_row = tmpdiv;
+	tmpdiv = last_block;
+	do_div(tmpdiv, blocks_per_row);
+	last_row = tmpdiv;
+	first_row_offset = (u32)(first_block - (first_row * blocks_per_row));
+	last_row_offset = (u32)(last_block - (last_row * blocks_per_row));
+	tmpdiv = first_row_offset;
+	do_div(tmpdiv, strip_size);
+	first_column = tmpdiv;
+	tmpdiv = last_row_offset;
+	do_div(tmpdiv, strip_size);
+	last_column = tmpdiv;
+#else
+	first_row = first_block / blocks_per_row;
+	last_row = last_block / blocks_per_row;
+	first_row_offset = (u32)(first_block - (first_row * blocks_per_row));
+	last_row_offset = (u32)(last_block - (last_row * blocks_per_row));
+	first_column = first_row_offset / strip_size;
+	last_column = last_row_offset / strip_size;
+#endif
+
+	/* If this isn't a single row/column then give to the controller. */
+	if (first_row != last_row || first_column != last_column)
+		return PQI_RAID_BYPASS_INELIGIBLE;
+
+	/* Proceeding with driver mapping. */
+	total_disks_per_row = data_disks_per_row +
+		get_unaligned_le16(&raid_map->metadata_disks_per_row);
+	map_row = ((u32)(first_row >> raid_map->parity_rotation_shift)) %
+		get_unaligned_le16(&raid_map->row_cnt);
+	map_index = (map_row * total_disks_per_row) + first_column;
+
+	/* RAID 1 */
+	if (device->raid_level == SA_RAID_1) {
+		if (device->offload_to_mirror)
+			map_index += data_disks_per_row;
+		device->offload_to_mirror = !device->offload_to_mirror;
+	} else if (device->raid_level == SA_RAID_ADM) {
+		/* RAID ADM */
+		/*
+		 * Handles N-way mirrors  (R1-ADM) and R10 with # of drives
+		 * divisible by 3.
+		 */
+		offload_to_mirror = device->offload_to_mirror;
+		if (offload_to_mirror == 0)  {
+			/* use physical disk in the first mirrored group. */
+			map_index %= data_disks_per_row;
+		} else {
+			do {
+				/*
+				 * Determine mirror group that map_index
+				 * indicates.
+				 */
+				current_group = map_index / data_disks_per_row;
+
+				if (offload_to_mirror != current_group) {
+					if (current_group <
+						layout_map_count - 1) {
+						/*
+						 * Select raid index from
+						 * next group.
+						 */
+						map_index += data_disks_per_row;
+						current_group++;
+					} else {
+						/*
+						 * Select raid index from first
+						 * group.
+						 */
+						map_index %= data_disks_per_row;
+						current_group = 0;
+					}
+				}
+			} while (offload_to_mirror != current_group);
+		}
+
+		/* Set mirror group to use next time. */
+		offload_to_mirror =
+			(offload_to_mirror >= layout_map_count - 1) ?
+				0 : offload_to_mirror + 1;
+		WARN_ON(offload_to_mirror >= layout_map_count);
+		device->offload_to_mirror = offload_to_mirror;
+		/*
+		 * Avoid direct use of device->offload_to_mirror within this
+		 * function since multiple threads might simultaneously
+		 * increment it beyond the range of device->layout_map_count -1.
+		 */
+	} else if ((device->raid_level == SA_RAID_5 ||
+		device->raid_level == SA_RAID_6) && layout_map_count > 1) {
+		/* RAID 50/60 */
+		/* Verify first and last block are in same RAID group */
+		r5or6_blocks_per_row = strip_size * data_disks_per_row;
+		stripesize = r5or6_blocks_per_row * layout_map_count;
+#if BITS_PER_LONG == 32
+		tmpdiv = first_block;
+		first_group = do_div(tmpdiv, stripesize);
+		tmpdiv = first_group;
+		do_div(tmpdiv, r5or6_blocks_per_row);
+		first_group = tmpdiv;
+		tmpdiv = last_block;
+		last_group = do_div(tmpdiv, stripesize);
+		tmpdiv = last_group;
+		do_div(tmpdiv, r5or6_blocks_per_row);
+		last_group = tmpdiv;
+#else
+		first_group = (first_block % stripesize) / r5or6_blocks_per_row;
+		last_group = (last_block % stripesize) / r5or6_blocks_per_row;
+#endif
+		if (first_group != last_group)
+			return PQI_RAID_BYPASS_INELIGIBLE;
+
+		/* Verify request is in a single row of RAID 5/6 */
+#if BITS_PER_LONG == 32
+		tmpdiv = first_block;
+		do_div(tmpdiv, stripesize);
+		first_row = r5or6_first_row = r0_first_row = tmpdiv;
+		tmpdiv = last_block;
+		do_div(tmpdiv, stripesize);
+		r5or6_last_row = r0_last_row = tmpdiv;
+#else
+		first_row = r5or6_first_row = r0_first_row =
+			first_block / stripesize;
+		r5or6_last_row = r0_last_row = last_block / stripesize;
+#endif
+		if (r5or6_first_row != r5or6_last_row)
+			return PQI_RAID_BYPASS_INELIGIBLE;
+
+		/* Verify request is in a single column */
+#if BITS_PER_LONG == 32
+		tmpdiv = first_block;
+		first_row_offset = do_div(tmpdiv, stripesize);
+		tmpdiv = first_row_offset;
+		first_row_offset = (u32)do_div(tmpdiv, r5or6_blocks_per_row);
+		r5or6_first_row_offset = first_row_offset;
+		tmpdiv = last_block;
+		r5or6_last_row_offset = do_div(tmpdiv, stripesize);
+		tmpdiv = r5or6_last_row_offset;
+		r5or6_last_row_offset = do_div(tmpdiv, r5or6_blocks_per_row);
+		tmpdiv = r5or6_first_row_offset;
+		do_div(tmpdiv, strip_size);
+		first_column = r5or6_first_column = tmpdiv;
+		tmpdiv = r5or6_last_row_offset;
+		do_div(tmpdiv, strip_size);
+		r5or6_last_column = tmpdiv;
+#else
+		first_row_offset = r5or6_first_row_offset =
+			(u32)((first_block % stripesize) %
+			r5or6_blocks_per_row);
+
+		r5or6_last_row_offset =
+			(u32)((last_block % stripesize) %
+			r5or6_blocks_per_row);
+
+		first_column = r5or6_first_row_offset / strip_size;
+		r5or6_first_column = first_column;
+		r5or6_last_column = r5or6_last_row_offset / strip_size;
+#endif
+		if (r5or6_first_column != r5or6_last_column)
+			return PQI_RAID_BYPASS_INELIGIBLE;
+
+		/* Request is eligible */
+		map_row =
+			((u32)(first_row >> raid_map->parity_rotation_shift)) %
+			get_unaligned_le16(&raid_map->row_cnt);
+
+		map_index = (first_group *
+			(get_unaligned_le16(&raid_map->row_cnt) *
+			total_disks_per_row)) +
+			(map_row * total_disks_per_row) + first_column;
+	}
+
+	if (unlikely(map_index >= RAID_MAP_MAX_ENTRIES))
+		return PQI_RAID_BYPASS_INELIGIBLE;
+
+	aio_handle = raid_map->disk_data[map_index].aio_handle;
+	disk_block = get_unaligned_le64(&raid_map->disk_starting_blk) +
+		first_row * strip_size +
+		(first_row_offset - first_column * strip_size);
+	disk_block_cnt = block_cnt;
+
+	/* Handle differing logical/physical block sizes. */
+	if (raid_map->phys_blk_shift) {
+		disk_block <<= raid_map->phys_blk_shift;
+		disk_block_cnt <<= raid_map->phys_blk_shift;
+	}
+
+	if (unlikely(disk_block_cnt > 0xffff))
+		return PQI_RAID_BYPASS_INELIGIBLE;
+
+	/* Build the new CDB for the physical disk I/O. */
+	if (disk_block > 0xffffffff) {
+		cdb[0] = is_write ? WRITE_16 : READ_16;
+		cdb[1] = 0;
+		put_unaligned_be64(disk_block, &cdb[2]);
+		put_unaligned_be32(disk_block_cnt, &cdb[10]);
+		cdb[14] = 0;
+		cdb[15] = 0;
+		cdb_length = 16;
+	} else {
+		cdb[0] = is_write ? WRITE_10 : READ_10;
+		cdb[1] = 0;
+		put_unaligned_be32((u32)disk_block, &cdb[2]);
+		cdb[6] = 0;
+		put_unaligned_be16((u16)disk_block_cnt, &cdb[7]);
+		cdb[9] = 0;
+		cdb_length = 10;
+	}
+
+	if (get_unaligned_le16(&raid_map->flags) &
+		RAID_MAP_ENCRYPTION_ENABLED) {
+		pqi_set_encryption_info(&encryption_info, raid_map,
+			first_block);
+		encryption_info_ptr = &encryption_info;
+	} else {
+		encryption_info_ptr = NULL;
+	}
+
+	return pqi_aio_submit_io(ctrl_info, scmd, aio_handle,
+		cdb, cdb_length, queue_group, encryption_info_ptr);
+}
+
+#define PQI_STATUS_IDLE		0x0
+
+#define PQI_CREATE_ADMIN_QUEUE_PAIR	1
+#define PQI_DELETE_ADMIN_QUEUE_PAIR	2
+
+#define PQI_DEVICE_STATE_POWER_ON_AND_RESET		0x0
+#define PQI_DEVICE_STATE_STATUS_AVAILABLE		0x1
+#define PQI_DEVICE_STATE_ALL_REGISTERS_READY		0x2
+#define PQI_DEVICE_STATE_ADMIN_QUEUE_PAIR_READY		0x3
+#define PQI_DEVICE_STATE_ERROR				0x4
+
+#define PQI_MODE_READY_TIMEOUT_SECS		30
+#define PQI_MODE_READY_POLL_INTERVAL_MSECS	1
+
+static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)
+{
+	struct pqi_device_registers __iomem *pqi_registers;
+	unsigned long timeout;
+	u64 signature;
+	u8 status;
+
+	pqi_registers = ctrl_info->pqi_registers;
+	timeout = (PQI_MODE_READY_TIMEOUT_SECS * HZ) + jiffies;
+
+	while (1) {
+		signature = readq(&pqi_registers->signature);
+		if (memcmp(&signature, PQI_DEVICE_SIGNATURE,
+			sizeof(signature)) == 0)
+			break;
+		if (time_after(jiffies, timeout)) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"timed out waiting for PQI signature\n");
+			return -ETIMEDOUT;
+		}
+		msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
+	}
+
+	while (1) {
+		status = readb(&pqi_registers->function_and_status_code);
+		if (status == PQI_STATUS_IDLE)
+			break;
+		if (time_after(jiffies, timeout)) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"timed out waiting for PQI IDLE\n");
+			return -ETIMEDOUT;
+		}
+		msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
+	}
+
+	while (1) {
+		if (readl(&pqi_registers->device_status) ==
+			PQI_DEVICE_STATE_ALL_REGISTERS_READY)
+			break;
+		if (time_after(jiffies, timeout)) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"timed out waiting for PQI all registers ready\n");
+			return -ETIMEDOUT;
+		}
+		msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
+	}
+
+	return 0;
+}
+
+static inline void pqi_aio_path_disabled(struct pqi_io_request *io_request)
+{
+	struct pqi_scsi_dev *device;
+
+	device = io_request->scmd->device->hostdata;
+	device->offload_enabled = false;
+}
+
+static inline void pqi_take_device_offline(struct scsi_device *sdev)
+{
+	struct pqi_ctrl_info *ctrl_info;
+	struct pqi_scsi_dev *device;
+
+	if (scsi_device_online(sdev)) {
+		scsi_device_set_state(sdev, SDEV_OFFLINE);
+		ctrl_info = shost_to_hba(sdev->host);
+		schedule_delayed_work(&ctrl_info->rescan_work, 0);
+		device = sdev->hostdata;
+		dev_err(&ctrl_info->pci_dev->dev, "offlined scsi %d:%d:%d:%d\n",
+			ctrl_info->scsi_host->host_no, device->bus,
+			device->target, device->lun);
+	}
+}
+
+static void pqi_process_raid_io_error(struct pqi_io_request *io_request)
+{
+	u8 scsi_status;
+	u8 host_byte;
+	struct scsi_cmnd *scmd;
+	struct pqi_raid_error_info *error_info;
+	size_t sense_data_length;
+	int residual_count;
+	int xfer_count;
+	struct scsi_sense_hdr sshdr;
+
+	scmd = io_request->scmd;
+	if (!scmd)
+		return;
+
+	error_info = io_request->error_info;
+	scsi_status = error_info->status;
+	host_byte = DID_OK;
+
+	if (error_info->data_out_result == PQI_DATA_IN_OUT_UNDERFLOW) {
+		xfer_count =
+			get_unaligned_le32(&error_info->data_out_transferred);
+		residual_count = scsi_bufflen(scmd) - xfer_count;
+		scsi_set_resid(scmd, residual_count);
+		if (xfer_count < scmd->underflow)
+			host_byte = DID_SOFT_ERROR;
+	}
+
+	sense_data_length = get_unaligned_le16(&error_info->sense_data_length);
+	if (sense_data_length == 0)
+		sense_data_length =
+			get_unaligned_le16(&error_info->response_data_length);
+	if (sense_data_length) {
+		if (sense_data_length > sizeof(error_info->data))
+			sense_data_length = sizeof(error_info->data);
+
+		if (scsi_status == SAM_STAT_CHECK_CONDITION &&
+			scsi_normalize_sense(error_info->data,
+				sense_data_length, &sshdr) &&
+				sshdr.sense_key == HARDWARE_ERROR &&
+				sshdr.asc == 0x3e &&
+				sshdr.ascq == 0x1) {
+			pqi_take_device_offline(scmd->device);
+			host_byte = DID_NO_CONNECT;
+		}
+
+		if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
+			sense_data_length = SCSI_SENSE_BUFFERSIZE;
+		memcpy(scmd->sense_buffer, error_info->data,
+			sense_data_length);
+	}
+
+	scmd->result = scsi_status;
+	set_host_byte(scmd, host_byte);
+}
+
+static void pqi_process_aio_io_error(struct pqi_io_request *io_request)
+{
+	u8 scsi_status;
+	u8 host_byte;
+	struct scsi_cmnd *scmd;
+	struct pqi_aio_error_info *error_info;
+	size_t sense_data_length;
+	int residual_count;
+	int xfer_count;
+	bool device_offline;
+
+	scmd = io_request->scmd;
+	error_info = io_request->error_info;
+	host_byte = DID_OK;
+	sense_data_length = 0;
+	device_offline = false;
+
+	switch (error_info->service_response) {
+	case PQI_AIO_SERV_RESPONSE_COMPLETE:
+		scsi_status = error_info->status;
+		break;
+	case PQI_AIO_SERV_RESPONSE_FAILURE:
+		switch (error_info->status) {
+		case PQI_AIO_STATUS_IO_ABORTED:
+			scsi_status = SAM_STAT_TASK_ABORTED;
+			break;
+		case PQI_AIO_STATUS_UNDERRUN:
+			scsi_status = SAM_STAT_GOOD;
+			residual_count = get_unaligned_le32(
+						&error_info->residual_count);
+			scsi_set_resid(scmd, residual_count);
+			xfer_count = scsi_bufflen(scmd) - residual_count;
+			if (xfer_count < scmd->underflow)
+				host_byte = DID_SOFT_ERROR;
+			break;
+		case PQI_AIO_STATUS_OVERRUN:
+			scsi_status = SAM_STAT_GOOD;
+			break;
+		case PQI_AIO_STATUS_AIO_PATH_DISABLED:
+			pqi_aio_path_disabled(io_request);
+			scsi_status = SAM_STAT_GOOD;
+			io_request->status = -EAGAIN;
+			break;
+		case PQI_AIO_STATUS_NO_PATH_TO_DEVICE:
+		case PQI_AIO_STATUS_INVALID_DEVICE:
+			device_offline = true;
+			pqi_take_device_offline(scmd->device);
+			host_byte = DID_NO_CONNECT;
+			scsi_status = SAM_STAT_CHECK_CONDITION;
+			break;
+		case PQI_AIO_STATUS_IO_ERROR:
+		default:
+			scsi_status = SAM_STAT_CHECK_CONDITION;
+			break;
+		}
+		break;
+	case PQI_AIO_SERV_RESPONSE_TMF_COMPLETE:
+	case PQI_AIO_SERV_RESPONSE_TMF_SUCCEEDED:
+		scsi_status = SAM_STAT_GOOD;
+		break;
+	case PQI_AIO_SERV_RESPONSE_TMF_REJECTED:
+	case PQI_AIO_SERV_RESPONSE_TMF_INCORRECT_LUN:
+	default:
+		scsi_status = SAM_STAT_CHECK_CONDITION;
+		break;
+	}
+
+	if (error_info->data_present) {
+		sense_data_length =
+			get_unaligned_le16(&error_info->data_length);
+		if (sense_data_length) {
+			if (sense_data_length > sizeof(error_info->data))
+				sense_data_length = sizeof(error_info->data);
+			if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
+				sense_data_length = SCSI_SENSE_BUFFERSIZE;
+			memcpy(scmd->sense_buffer, error_info->data,
+				sense_data_length);
+		}
+	}
+
+	if (device_offline && sense_data_length == 0)
+		scsi_build_sense_buffer(0, scmd->sense_buffer, HARDWARE_ERROR,
+			0x3e, 0x1);
+
+	scmd->result = scsi_status;
+	set_host_byte(scmd, host_byte);
+}
+
+static void pqi_process_io_error(unsigned int iu_type,
+	struct pqi_io_request *io_request)
+{
+	switch (iu_type) {
+	case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
+		pqi_process_raid_io_error(io_request);
+		break;
+	case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
+		pqi_process_aio_io_error(io_request);
+		break;
+	}
+}
+
+static int pqi_interpret_task_management_response(
+	struct pqi_task_management_response *response)
+{
+	int rc;
+
+	switch (response->response_code) {
+	case SOP_TMF_COMPLETE:
+	case SOP_TMF_FUNCTION_SUCCEEDED:
+		rc = 0;
+		break;
+	default:
+		rc = -EIO;
+		break;
+	}
+
+	return rc;
+}
+
+static unsigned int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_queue_group *queue_group)
+{
+	unsigned int num_responses;
+	pqi_index_t oq_pi;
+	pqi_index_t oq_ci;
+	struct pqi_io_request *io_request;
+	struct pqi_io_response *response;
+	u16 request_id;
+
+	num_responses = 0;
+	oq_ci = queue_group->oq_ci_copy;
+
+	while (1) {
+		oq_pi = *queue_group->oq_pi;
+		if (oq_pi == oq_ci)
+			break;
+
+		num_responses++;
+		response = queue_group->oq_element_array +
+			(oq_ci * PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
+
+		request_id = get_unaligned_le16(&response->request_id);
+		WARN_ON(request_id >= ctrl_info->max_io_slots);
+
+		io_request = &ctrl_info->io_request_pool[request_id];
+		WARN_ON(atomic_read(&io_request->refcount) == 0);
+
+		switch (response->header.iu_type) {
+		case PQI_RESPONSE_IU_RAID_PATH_IO_SUCCESS:
+		case PQI_RESPONSE_IU_AIO_PATH_IO_SUCCESS:
+		case PQI_RESPONSE_IU_GENERAL_MANAGEMENT:
+			break;
+		case PQI_RESPONSE_IU_TASK_MANAGEMENT:
+			io_request->status =
+				pqi_interpret_task_management_response(
+					(void *)response);
+			break;
+		case PQI_RESPONSE_IU_AIO_PATH_DISABLED:
+			pqi_aio_path_disabled(io_request);
+			io_request->status = -EAGAIN;
+			break;
+		case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
+		case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
+			io_request->error_info = ctrl_info->error_buffer +
+				(get_unaligned_le16(&response->error_index) *
+				PQI_ERROR_BUFFER_ELEMENT_LENGTH);
+			pqi_process_io_error(response->header.iu_type,
+				io_request);
+			break;
+		default:
+			dev_err(&ctrl_info->pci_dev->dev,
+				"unexpected IU type: 0x%x\n",
+				response->header.iu_type);
+			WARN_ON(response->header.iu_type);
+			break;
+		}
+
+		io_request->io_complete_callback(io_request,
+			io_request->context);
+
+		/*
+		 * Note that the I/O request structure CANNOT BE TOUCHED after
+		 * returning from the I/O completion callback!
+		 */
+
+		oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;
+	}
+
+	if (num_responses) {
+		queue_group->oq_ci_copy = oq_ci;
+		writel(oq_ci, queue_group->oq_ci);
+	}
+
+	return num_responses;
+}
+
+static inline unsigned int pqi_num_elements_free(unsigned int pi,
+	unsigned int ci, unsigned int elements_in_queue)
+{
+	unsigned int num_elements_used;
+
+	if (pi >= ci)
+		num_elements_used = pi - ci;
+	else
+		num_elements_used = elements_in_queue - ci + pi;
+
+	return elements_in_queue - num_elements_used - 1;
+}
+
+#define PQI_EVENT_ACK_TIMEOUT	30
+
+static void pqi_start_event_ack(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_event_acknowledge_request *iu, size_t iu_length)
+{
+	pqi_index_t iq_pi;
+	pqi_index_t iq_ci;
+	unsigned long flags;
+	void *next_element;
+	unsigned long timeout;
+	struct pqi_queue_group *queue_group;
+
+	queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];
+	put_unaligned_le16(queue_group->oq_id, &iu->header.response_queue_id);
+
+	timeout = (PQI_EVENT_ACK_TIMEOUT * HZ) + jiffies;
+
+	while (1) {
+		spin_lock_irqsave(&queue_group->submit_lock[RAID_PATH], flags);
+
+		iq_pi = queue_group->iq_pi_copy[RAID_PATH];
+		iq_ci = *queue_group->iq_ci[RAID_PATH];
+
+		if (pqi_num_elements_free(iq_pi, iq_ci,
+			ctrl_info->num_elements_per_iq))
+			break;
+
+		spin_unlock_irqrestore(
+			&queue_group->submit_lock[RAID_PATH], flags);
+
+		if (time_after(jiffies, timeout)) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"sending event acknowledge timed out\n");
+			return;
+		}
+	}
+
+	next_element = queue_group->iq_element_array[RAID_PATH] +
+		(iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
+
+	memcpy(next_element, iu, iu_length);
+
+	iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;
+
+	queue_group->iq_pi_copy[RAID_PATH] = iq_pi;
+
+	/*
+	 * This write notifies the controller that an IU is available to be
+	 * processed.
+	 */
+	writel(iq_pi, queue_group->iq_pi[RAID_PATH]);
+
+	spin_unlock_irqrestore(&queue_group->submit_lock[RAID_PATH], flags);
+}
+
+static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_event *event)
+{
+	struct pqi_event_acknowledge_request request;
+
+	memset(&request, 0, sizeof(request));
+
+	request.header.iu_type = PQI_REQUEST_IU_ACKNOWLEDGE_VENDOR_EVENT;
+	put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
+		&request.header.iu_length);
+	request.event_type = event->event_type;
+	request.event_id = event->event_id;
+	request.additional_event_id = event->additional_event_id;
+
+	pqi_start_event_ack(ctrl_info, &request, sizeof(request));
+}
+
+static void pqi_event_worker(struct work_struct *work)
+{
+	unsigned int i;
+	struct pqi_ctrl_info *ctrl_info;
+	struct pqi_event *pending_event;
+	bool got_non_heartbeat_event = false;
+
+	ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);
+
+	pending_event = ctrl_info->pending_events;
+	for (i = 0; i < PQI_NUM_SUPPORTED_EVENTS; i++) {
+		if (pending_event->pending) {
+			pending_event->pending = false;
+			pqi_acknowledge_event(ctrl_info, pending_event);
+			if (i != PQI_EVENT_HEARTBEAT)
+				got_non_heartbeat_event = true;
+		}
+		pending_event++;
+	}
+
+	if (got_non_heartbeat_event)
+		pqi_schedule_rescan_worker(ctrl_info);
+}
+
+static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned int i;
+	unsigned int path;
+	struct pqi_queue_group *queue_group;
+	unsigned long flags;
+	struct pqi_io_request *io_request;
+	struct pqi_io_request *next;
+	struct scsi_cmnd *scmd;
+
+	ctrl_info->controller_online = false;
+	dev_err(&ctrl_info->pci_dev->dev, "controller offline\n");
+
+	for (i = 0; i < ctrl_info->num_queue_groups; i++) {
+		queue_group = &ctrl_info->queue_groups[i];
+
+		for (path = 0; path < 2; path++) {
+			spin_lock_irqsave(
+				&queue_group->submit_lock[path], flags);
+
+			list_for_each_entry_safe(io_request, next,
+				&queue_group->request_list[path],
+				request_list_entry) {
+
+				scmd = io_request->scmd;
+				if (scmd) {
+					set_host_byte(scmd, DID_NO_CONNECT);
+					pqi_scsi_done(scmd);
+				}
+
+				list_del(&io_request->request_list_entry);
+			}
+
+			spin_unlock_irqrestore(
+				&queue_group->submit_lock[path], flags);
+		}
+	}
+}
+
+#define PQI_HEARTBEAT_TIMER_INTERVAL	(5 * HZ)
+#define PQI_MAX_HEARTBEAT_REQUESTS	5
+
+static void pqi_heartbeat_timer_handler(unsigned long data)
+{
+	int num_interrupts;
+	struct pqi_ctrl_info *ctrl_info = (struct pqi_ctrl_info *)data;
+
+	num_interrupts = atomic_read(&ctrl_info->num_interrupts);
+
+	if (num_interrupts == ctrl_info->previous_num_interrupts) {
+		ctrl_info->num_heartbeats_requested++;
+		if (ctrl_info->num_heartbeats_requested >
+			PQI_MAX_HEARTBEAT_REQUESTS) {
+			pqi_take_ctrl_offline(ctrl_info);
+			return;
+		}
+		ctrl_info->pending_events[PQI_EVENT_HEARTBEAT].pending = true;
+		schedule_work(&ctrl_info->event_work);
+	} else {
+		ctrl_info->num_heartbeats_requested = 0;
+	}
+
+	ctrl_info->previous_num_interrupts = num_interrupts;
+	mod_timer(&ctrl_info->heartbeat_timer,
+		jiffies + PQI_HEARTBEAT_TIMER_INTERVAL);
+}
+
+static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
+{
+	ctrl_info->previous_num_interrupts =
+		atomic_read(&ctrl_info->num_interrupts);
+
+	init_timer(&ctrl_info->heartbeat_timer);
+	ctrl_info->heartbeat_timer.expires =
+		jiffies + PQI_HEARTBEAT_TIMER_INTERVAL;
+	ctrl_info->heartbeat_timer.data = (unsigned long)ctrl_info;
+	ctrl_info->heartbeat_timer.function = pqi_heartbeat_timer_handler;
+	add_timer(&ctrl_info->heartbeat_timer);
+	ctrl_info->heartbeat_timer_started = true;
+}
+
+static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
+{
+	if (ctrl_info->heartbeat_timer_started)
+		del_timer_sync(&ctrl_info->heartbeat_timer);
+}
+
+static int pqi_event_type_to_event_index(unsigned int event_type)
+{
+	int index;
+
+	switch (event_type) {
+	case PQI_EVENT_TYPE_HEARTBEAT:
+		index = PQI_EVENT_HEARTBEAT;
+		break;
+	case PQI_EVENT_TYPE_HOTPLUG:
+		index = PQI_EVENT_HOTPLUG;
+		break;
+	case PQI_EVENT_TYPE_HARDWARE:
+		index = PQI_EVENT_HARDWARE;
+		break;
+	case PQI_EVENT_TYPE_PHYSICAL_DEVICE:
+		index = PQI_EVENT_PHYSICAL_DEVICE;
+		break;
+	case PQI_EVENT_TYPE_LOGICAL_DEVICE:
+		index = PQI_EVENT_LOGICAL_DEVICE;
+		break;
+	case PQI_EVENT_TYPE_AIO_STATE_CHANGE:
+		index = PQI_EVENT_AIO_STATE_CHANGE;
+		break;
+	case PQI_EVENT_TYPE_AIO_CONFIG_CHANGE:
+		index = PQI_EVENT_AIO_CONFIG_CHANGE;
+		break;
+	default:
+		index = -1;
+		break;
+	}
+
+	return index;
+}
+
+static unsigned int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned int num_events;
+	pqi_index_t oq_pi;
+	pqi_index_t oq_ci;
+	struct pqi_event_queue *event_queue;
+	struct pqi_event_response *response;
+	struct pqi_event *pending_event;
+	bool need_delayed_work;
+	int event_index;
+
+	event_queue = &ctrl_info->event_queue;
+	num_events = 0;
+	need_delayed_work = false;
+	oq_ci = event_queue->oq_ci_copy;
+
+	while (1) {
+		oq_pi = *event_queue->oq_pi;
+		if (oq_pi == oq_ci)
+			break;
+
+		num_events++;
+		response = event_queue->oq_element_array +
+			(oq_ci * PQI_EVENT_OQ_ELEMENT_LENGTH);
+
+		event_index =
+			pqi_event_type_to_event_index(response->event_type);
+
+		if (event_index >= 0) {
+			if (response->request_acknowlege) {
+				pending_event =
+					&ctrl_info->pending_events[event_index];
+				pending_event->event_type =
+					response->event_type;
+				pending_event->event_id = response->event_id;
+				pending_event->additional_event_id =
+					response->additional_event_id;
+				if (event_index != PQI_EVENT_HEARTBEAT) {
+					pending_event->pending = true;
+					need_delayed_work = true;
+				}
+			}
+		}
+
+		oq_ci = (oq_ci + 1) % PQI_NUM_EVENT_QUEUE_ELEMENTS;
+	}
+
+	if (num_events) {
+		event_queue->oq_ci_copy = oq_ci;
+		writel(oq_ci, event_queue->oq_ci);
+
+		if (need_delayed_work)
+			schedule_work(&ctrl_info->event_work);
+	}
+
+	return num_events;
+}
+
+static irqreturn_t pqi_irq_handler(int irq, void *data)
+{
+	struct pqi_ctrl_info *ctrl_info;
+	struct pqi_queue_group *queue_group;
+	unsigned int num_responses_handled;
+
+	queue_group = data;
+	ctrl_info = queue_group->ctrl_info;
+
+	if (!ctrl_info || !queue_group->oq_ci)
+		return IRQ_NONE;
+
+	num_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);
+
+	if (irq == ctrl_info->event_irq)
+		num_responses_handled += pqi_process_event_intr(ctrl_info);
+
+	if (num_responses_handled)
+		atomic_inc(&ctrl_info->num_interrupts);
+
+	pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);
+	pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);
+
+	return IRQ_HANDLED;
+}
+
+static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)
+{
+	int i;
+	int rc;
+
+	ctrl_info->event_irq = ctrl_info->msix_vectors[0];
+
+	for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {
+		rc = request_irq(ctrl_info->msix_vectors[i],
+			pqi_irq_handler, 0,
+			DRIVER_NAME_SHORT, ctrl_info->intr_data[i]);
+		if (rc) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"irq %u init failed with error %d\n",
+				ctrl_info->msix_vectors[i], rc);
+			return rc;
+		}
+		ctrl_info->num_msix_vectors_initialized++;
+	}
+
+	return 0;
+}
+
+static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)
+{
+	int i;
+
+	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)
+		free_irq(ctrl_info->msix_vectors[i],
+			ctrl_info->intr_data[i]);
+}
+
+static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned int i;
+	int max_vectors;
+	int num_vectors_enabled;
+	struct msix_entry msix_entries[PQI_MAX_MSIX_VECTORS];
+
+	max_vectors = ctrl_info->num_queue_groups;
+
+	for (i = 0; i < max_vectors; i++)
+		msix_entries[i].entry = i;
+
+	num_vectors_enabled = pci_enable_msix_range(ctrl_info->pci_dev,
+		msix_entries, PQI_MIN_MSIX_VECTORS, max_vectors);
+
+	if (num_vectors_enabled < 0) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"MSI-X init failed with error %d\n",
+			num_vectors_enabled);
+		return num_vectors_enabled;
+	}
+
+	ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;
+	for (i = 0; i < num_vectors_enabled; i++) {
+		ctrl_info->msix_vectors[i] = msix_entries[i].vector;
+		ctrl_info->intr_data[i] = &ctrl_info->queue_groups[i];
+	}
+
+	return 0;
+}
+
+static void pqi_irq_set_affinity_hint(struct pqi_ctrl_info *ctrl_info)
+{
+	int i;
+	int rc;
+	int cpu;
+
+	cpu = cpumask_first(cpu_online_mask);
+	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++) {
+		rc = irq_set_affinity_hint(ctrl_info->msix_vectors[i],
+			get_cpu_mask(cpu));
+		if (rc)
+			dev_err(&ctrl_info->pci_dev->dev,
+				"error %d setting affinity hint for irq vector %u\n",
+				rc, ctrl_info->msix_vectors[i]);
+		cpu = cpumask_next(cpu, cpu_online_mask);
+	}
+}
+
+static void pqi_irq_unset_affinity_hint(struct pqi_ctrl_info *ctrl_info)
+{
+	int i;
+
+	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)
+		irq_set_affinity_hint(ctrl_info->msix_vectors[i], NULL);
+}
+
+static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned int i;
+	size_t alloc_length;
+	size_t element_array_length_per_iq;
+	size_t element_array_length_per_oq;
+	void *element_array;
+	void *next_queue_index;
+	void *aligned_pointer;
+	unsigned int num_inbound_queues;
+	unsigned int num_outbound_queues;
+	unsigned int num_queue_indexes;
+	struct pqi_queue_group *queue_group;
+
+	element_array_length_per_iq =
+		PQI_OPERATIONAL_IQ_ELEMENT_LENGTH *
+		ctrl_info->num_elements_per_iq;
+	element_array_length_per_oq =
+		PQI_OPERATIONAL_OQ_ELEMENT_LENGTH *
+		ctrl_info->num_elements_per_oq;
+	num_inbound_queues = ctrl_info->num_queue_groups * 2;
+	num_outbound_queues = ctrl_info->num_queue_groups;
+	num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;
+
+	aligned_pointer = NULL;
+
+	for (i = 0; i < num_inbound_queues; i++) {
+		aligned_pointer = PTR_ALIGN(aligned_pointer,
+			PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
+		aligned_pointer += element_array_length_per_iq;
+	}
+
+	for (i = 0; i < num_outbound_queues; i++) {
+		aligned_pointer = PTR_ALIGN(aligned_pointer,
+			PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
+		aligned_pointer += element_array_length_per_oq;
+	}
+
+	aligned_pointer = PTR_ALIGN(aligned_pointer,
+		PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
+	aligned_pointer += PQI_NUM_EVENT_QUEUE_ELEMENTS *
+		PQI_EVENT_OQ_ELEMENT_LENGTH;
+
+	for (i = 0; i < num_queue_indexes; i++) {
+		aligned_pointer = PTR_ALIGN(aligned_pointer,
+			PQI_OPERATIONAL_INDEX_ALIGNMENT);
+		aligned_pointer += sizeof(pqi_index_t);
+	}
+
+	alloc_length = (size_t)aligned_pointer +
+		PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
+
+	ctrl_info->queue_memory_base =
+		dma_zalloc_coherent(&ctrl_info->pci_dev->dev,
+			alloc_length,
+			&ctrl_info->queue_memory_base_dma_handle, GFP_KERNEL);
+
+	if (!ctrl_info->queue_memory_base) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"failed to allocate memory for PQI admin queues\n");
+		return -ENOMEM;
+	}
+
+	ctrl_info->queue_memory_length = alloc_length;
+
+	element_array = PTR_ALIGN(ctrl_info->queue_memory_base,
+		PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
+
+	for (i = 0; i < ctrl_info->num_queue_groups; i++) {
+		queue_group = &ctrl_info->queue_groups[i];
+		queue_group->iq_element_array[RAID_PATH] = element_array;
+		queue_group->iq_element_array_bus_addr[RAID_PATH] =
+			ctrl_info->queue_memory_base_dma_handle +
+				(element_array - ctrl_info->queue_memory_base);
+		element_array += element_array_length_per_iq;
+		element_array = PTR_ALIGN(element_array,
+			PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
+		queue_group->iq_element_array[AIO_PATH] = element_array;
+		queue_group->iq_element_array_bus_addr[AIO_PATH] =
+			ctrl_info->queue_memory_base_dma_handle +
+			(element_array - ctrl_info->queue_memory_base);
+		element_array += element_array_length_per_iq;
+		element_array = PTR_ALIGN(element_array,
+			PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
+	}
+
+	for (i = 0; i < ctrl_info->num_queue_groups; i++) {
+		queue_group = &ctrl_info->queue_groups[i];
+		queue_group->oq_element_array = element_array;
+		queue_group->oq_element_array_bus_addr =
+			ctrl_info->queue_memory_base_dma_handle +
+			(element_array - ctrl_info->queue_memory_base);
+		element_array += element_array_length_per_oq;
+		element_array = PTR_ALIGN(element_array,
+			PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
+	}
+
+	ctrl_info->event_queue.oq_element_array = element_array;
+	ctrl_info->event_queue.oq_element_array_bus_addr =
+		ctrl_info->queue_memory_base_dma_handle +
+		(element_array - ctrl_info->queue_memory_base);
+	element_array += PQI_NUM_EVENT_QUEUE_ELEMENTS *
+		PQI_EVENT_OQ_ELEMENT_LENGTH;
+
+	next_queue_index = PTR_ALIGN(element_array,
+		PQI_OPERATIONAL_INDEX_ALIGNMENT);
+
+	for (i = 0; i < ctrl_info->num_queue_groups; i++) {
+		queue_group = &ctrl_info->queue_groups[i];
+		queue_group->iq_ci[RAID_PATH] = next_queue_index;
+		queue_group->iq_ci_bus_addr[RAID_PATH] =
+			ctrl_info->queue_memory_base_dma_handle +
+			(next_queue_index - ctrl_info->queue_memory_base);
+		next_queue_index += sizeof(pqi_index_t);
+		next_queue_index = PTR_ALIGN(next_queue_index,
+			PQI_OPERATIONAL_INDEX_ALIGNMENT);
+		queue_group->iq_ci[AIO_PATH] = next_queue_index;
+		queue_group->iq_ci_bus_addr[AIO_PATH] =
+			ctrl_info->queue_memory_base_dma_handle +
+			(next_queue_index - ctrl_info->queue_memory_base);
+		next_queue_index += sizeof(pqi_index_t);
+		next_queue_index = PTR_ALIGN(next_queue_index,
+			PQI_OPERATIONAL_INDEX_ALIGNMENT);
+		queue_group->oq_pi = next_queue_index;
+		queue_group->oq_pi_bus_addr =
+			ctrl_info->queue_memory_base_dma_handle +
+			(next_queue_index - ctrl_info->queue_memory_base);
+		next_queue_index += sizeof(pqi_index_t);
+		next_queue_index = PTR_ALIGN(next_queue_index,
+			PQI_OPERATIONAL_INDEX_ALIGNMENT);
+	}
+
+	ctrl_info->event_queue.oq_pi = next_queue_index;
+	ctrl_info->event_queue.oq_pi_bus_addr =
+		ctrl_info->queue_memory_base_dma_handle +
+		(next_queue_index - ctrl_info->queue_memory_base);
+
+	return 0;
+}
+
+static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned int i;
+	u16 next_iq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
+	u16 next_oq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
+
+	/*
+	 * Initialize the backpointers to the controller structure in
+	 * each operational queue group structure.
+	 */
+	for (i = 0; i < ctrl_info->num_queue_groups; i++)
+		ctrl_info->queue_groups[i].ctrl_info = ctrl_info;
+
+	/*
+	 * Assign IDs to all operational queues.  Note that the IDs
+	 * assigned to operational IQs are independent of the IDs
+	 * assigned to operational OQs.
+	 */
+	ctrl_info->event_queue.oq_id = next_oq_id++;
+	for (i = 0; i < ctrl_info->num_queue_groups; i++) {
+		ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;
+		ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;
+		ctrl_info->queue_groups[i].oq_id = next_oq_id++;
+	}
+
+	/*
+	 * Assign MSI-X table entry indexes to all queues.  Note that the
+	 * interrupt for the event queue is shared with the first queue group.
+	 */
+	ctrl_info->event_queue.int_msg_num = 0;
+	for (i = 0; i < ctrl_info->num_queue_groups; i++)
+		ctrl_info->queue_groups[i].int_msg_num = i;
+
+	for (i = 0; i < ctrl_info->num_queue_groups; i++) {
+		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);
+		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);
+		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);
+		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);
+	}
+}
+
+static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)
+{
+	size_t alloc_length;
+	struct pqi_admin_queues_aligned *admin_queues_aligned;
+	struct pqi_admin_queues *admin_queues;
+
+	alloc_length = sizeof(struct pqi_admin_queues_aligned) +
+		PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
+
+	ctrl_info->admin_queue_memory_base =
+		dma_zalloc_coherent(&ctrl_info->pci_dev->dev,
+			alloc_length,
+			&ctrl_info->admin_queue_memory_base_dma_handle,
+			GFP_KERNEL);
+
+	if (!ctrl_info->admin_queue_memory_base)
+		return -ENOMEM;
+
+	ctrl_info->admin_queue_memory_length = alloc_length;
+
+	admin_queues = &ctrl_info->admin_queues;
+	admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,
+		PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
+	admin_queues->iq_element_array =
+		&admin_queues_aligned->iq_element_array;
+	admin_queues->oq_element_array =
+		&admin_queues_aligned->oq_element_array;
+	admin_queues->iq_ci = &admin_queues_aligned->iq_ci;
+	admin_queues->oq_pi = &admin_queues_aligned->oq_pi;
+
+	admin_queues->iq_element_array_bus_addr =
+		ctrl_info->admin_queue_memory_base_dma_handle +
+		(admin_queues->iq_element_array -
+		ctrl_info->admin_queue_memory_base);
+	admin_queues->oq_element_array_bus_addr =
+		ctrl_info->admin_queue_memory_base_dma_handle +
+		(admin_queues->oq_element_array -
+		ctrl_info->admin_queue_memory_base);
+	admin_queues->iq_ci_bus_addr =
+		ctrl_info->admin_queue_memory_base_dma_handle +
+		((void *)admin_queues->iq_ci -
+		ctrl_info->admin_queue_memory_base);
+	admin_queues->oq_pi_bus_addr =
+		ctrl_info->admin_queue_memory_base_dma_handle +
+		((void *)admin_queues->oq_pi -
+		ctrl_info->admin_queue_memory_base);
+
+	return 0;
+}
+
+#define PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES		HZ
+#define PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS	1
+
+static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)
+{
+	struct pqi_device_registers __iomem *pqi_registers;
+	struct pqi_admin_queues *admin_queues;
+	unsigned long timeout;
+	u8 status;
+	u32 reg;
+
+	pqi_registers = ctrl_info->pqi_registers;
+	admin_queues = &ctrl_info->admin_queues;
+
+	writeq((u64)admin_queues->iq_element_array_bus_addr,
+		&pqi_registers->admin_iq_element_array_addr);
+	writeq((u64)admin_queues->oq_element_array_bus_addr,
+		&pqi_registers->admin_oq_element_array_addr);
+	writeq((u64)admin_queues->iq_ci_bus_addr,
+		&pqi_registers->admin_iq_ci_addr);
+	writeq((u64)admin_queues->oq_pi_bus_addr,
+		&pqi_registers->admin_oq_pi_addr);
+
+	reg = PQI_ADMIN_IQ_NUM_ELEMENTS |
+		(PQI_ADMIN_OQ_NUM_ELEMENTS) << 8 |
+		(admin_queues->int_msg_num << 16);
+	writel(reg, &pqi_registers->admin_iq_num_elements);
+	writel(PQI_CREATE_ADMIN_QUEUE_PAIR,
+		&pqi_registers->function_and_status_code);
+
+	timeout = PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES + jiffies;
+	while (1) {
+		status = readb(&pqi_registers->function_and_status_code);
+		if (status == PQI_STATUS_IDLE)
+			break;
+		if (time_after(jiffies, timeout))
+			return -ETIMEDOUT;
+		msleep(PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS);
+	}
+
+	/*
+	 * The offset registers are not initialized to the correct
+	 * offsets until *after* the create admin queue pair command
+	 * completes successfully.
+	 */
+	admin_queues->iq_pi = ctrl_info->iomem_base +
+		PQI_DEVICE_REGISTERS_OFFSET +
+		readq(&pqi_registers->admin_iq_pi_offset);
+	admin_queues->oq_ci = ctrl_info->iomem_base +
+		PQI_DEVICE_REGISTERS_OFFSET +
+		readq(&pqi_registers->admin_oq_ci_offset);
+
+	return 0;
+}
+
+static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_general_admin_request *request)
+{
+	struct pqi_admin_queues *admin_queues;
+	void *next_element;
+	pqi_index_t iq_pi;
+
+	admin_queues = &ctrl_info->admin_queues;
+	iq_pi = admin_queues->iq_pi_copy;
+
+	next_element = admin_queues->iq_element_array +
+		(iq_pi * PQI_ADMIN_IQ_ELEMENT_LENGTH);
+
+	memcpy(next_element, request, sizeof(*request));
+
+	iq_pi = (iq_pi + 1) % PQI_ADMIN_IQ_NUM_ELEMENTS;
+	admin_queues->iq_pi_copy = iq_pi;
+
+	/*
+	 * This write notifies the controller that an IU is available to be
+	 * processed.
+	 */
+	writel(iq_pi, admin_queues->iq_pi);
+}
+
+static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_general_admin_response *response)
+{
+	struct pqi_admin_queues *admin_queues;
+	pqi_index_t oq_pi;
+	pqi_index_t oq_ci;
+	unsigned long timeout;
+
+	admin_queues = &ctrl_info->admin_queues;
+	oq_ci = admin_queues->oq_ci_copy;
+
+	timeout = (3 * HZ) + jiffies;
+
+	while (1) {
+		oq_pi = *admin_queues->oq_pi;
+		if (oq_pi != oq_ci)
+			break;
+		if (time_after(jiffies, timeout)) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"timed out waiting for admin response\n");
+			return -ETIMEDOUT;
+		}
+		usleep_range(1000, 2000);
+	}
+
+	memcpy(response, admin_queues->oq_element_array +
+		(oq_ci * PQI_ADMIN_OQ_ELEMENT_LENGTH), sizeof(*response));
+
+	oq_ci = (oq_ci + 1) % PQI_ADMIN_OQ_NUM_ELEMENTS;
+	admin_queues->oq_ci_copy = oq_ci;
+	writel(oq_ci, admin_queues->oq_ci);
+
+	return 0;
+}
+
+static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_queue_group *queue_group, enum pqi_io_path path,
+	struct pqi_io_request *io_request)
+{
+	struct pqi_io_request *next;
+	void *next_element;
+	pqi_index_t iq_pi;
+	pqi_index_t iq_ci;
+	size_t iu_length;
+	unsigned long flags;
+	unsigned int num_elements_needed;
+	unsigned int num_elements_to_end_of_queue;
+	size_t copy_count;
+	struct pqi_iu_header *request;
+
+	spin_lock_irqsave(&queue_group->submit_lock[path], flags);
+
+	if (io_request)
+		list_add_tail(&io_request->request_list_entry,
+			&queue_group->request_list[path]);
+
+	iq_pi = queue_group->iq_pi_copy[path];
+
+	list_for_each_entry_safe(io_request, next,
+		&queue_group->request_list[path], request_list_entry) {
+
+		request = io_request->iu;
+
+		iu_length = get_unaligned_le16(&request->iu_length) +
+			PQI_REQUEST_HEADER_LENGTH;
+		num_elements_needed =
+			DIV_ROUND_UP(iu_length,
+				PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
+
+		iq_ci = *queue_group->iq_ci[path];
+
+		if (num_elements_needed > pqi_num_elements_free(iq_pi, iq_ci,
+			ctrl_info->num_elements_per_iq))
+			break;
+
+		put_unaligned_le16(queue_group->oq_id,
+			&request->response_queue_id);
+
+		next_element = queue_group->iq_element_array[path] +
+			(iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
+
+		num_elements_to_end_of_queue =
+			ctrl_info->num_elements_per_iq - iq_pi;
+
+		if (num_elements_needed <= num_elements_to_end_of_queue) {
+			memcpy(next_element, request, iu_length);
+		} else {
+			copy_count = num_elements_to_end_of_queue *
+				PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
+			memcpy(next_element, request, copy_count);
+			memcpy(queue_group->iq_element_array[path],
+				(u8 *)request + copy_count,
+				iu_length - copy_count);
+		}
+
+		iq_pi = (iq_pi + num_elements_needed) %
+			ctrl_info->num_elements_per_iq;
+
+		list_del(&io_request->request_list_entry);
+	}
+
+	if (iq_pi != queue_group->iq_pi_copy[path]) {
+		queue_group->iq_pi_copy[path] = iq_pi;
+		/*
+		 * This write notifies the controller that one or more IUs are
+		 * available to be processed.
+		 */
+		writel(iq_pi, queue_group->iq_pi[path]);
+	}
+
+	spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);
+}
+
+static void pqi_raid_synchronous_complete(struct pqi_io_request *io_request,
+	void *context)
+{
+	struct completion *waiting = context;
+
+	complete(waiting);
+}
+
+static int pqi_submit_raid_request_synchronous_with_io_request(
+	struct pqi_ctrl_info *ctrl_info, struct pqi_io_request *io_request,
+	unsigned long timeout_msecs)
+{
+	int rc = 0;
+	DECLARE_COMPLETION_ONSTACK(wait);
+
+	io_request->io_complete_callback = pqi_raid_synchronous_complete;
+	io_request->context = &wait;
+
+	pqi_start_io(ctrl_info,
+		&ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
+		io_request);
+
+	if (timeout_msecs == NO_TIMEOUT) {
+		wait_for_completion_io(&wait);
+	} else {
+		if (!wait_for_completion_io_timeout(&wait,
+			msecs_to_jiffies(timeout_msecs))) {
+			dev_warn(&ctrl_info->pci_dev->dev,
+				"command timed out\n");
+			rc = -ETIMEDOUT;
+		}
+	}
+
+	return rc;
+}
+
+static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_iu_header *request, unsigned int flags,
+	struct pqi_raid_error_info *error_info, unsigned long timeout_msecs)
+{
+	int rc;
+	struct pqi_io_request *io_request;
+	unsigned long start_jiffies;
+	unsigned long msecs_blocked;
+	size_t iu_length;
+
+	/*
+	 * Note that specifying PQI_SYNC_FLAGS_INTERRUPTABLE and a timeout value
+	 * are mutually exclusive.
+	 */
+
+	if (flags & PQI_SYNC_FLAGS_INTERRUPTABLE) {
+		if (down_interruptible(&ctrl_info->sync_request_sem))
+			return -ERESTARTSYS;
+	} else {
+		if (timeout_msecs == NO_TIMEOUT) {
+			down(&ctrl_info->sync_request_sem);
+		} else {
+			start_jiffies = jiffies;
+			if (down_timeout(&ctrl_info->sync_request_sem,
+				msecs_to_jiffies(timeout_msecs)))
+				return -ETIMEDOUT;
+			msecs_blocked =
+				jiffies_to_msecs(jiffies - start_jiffies);
+			if (msecs_blocked >= timeout_msecs)
+				return -ETIMEDOUT;
+			timeout_msecs -= msecs_blocked;
+		}
+	}
+
+	io_request = pqi_alloc_io_request(ctrl_info);
+
+	put_unaligned_le16(io_request->index,
+		&(((struct pqi_raid_path_request *)request)->request_id));
+
+	if (request->iu_type == PQI_REQUEST_IU_RAID_PATH_IO)
+		((struct pqi_raid_path_request *)request)->error_index =
+			((struct pqi_raid_path_request *)request)->request_id;
+
+	iu_length = get_unaligned_le16(&request->iu_length) +
+		PQI_REQUEST_HEADER_LENGTH;
+	memcpy(io_request->iu, request, iu_length);
+
+	rc = pqi_submit_raid_request_synchronous_with_io_request(ctrl_info,
+		io_request, timeout_msecs);
+
+	if (error_info) {
+		if (io_request->error_info)
+			memcpy(error_info, io_request->error_info,
+				sizeof(*error_info));
+		else
+			memset(error_info, 0, sizeof(*error_info));
+	} else if (rc == 0 && io_request->error_info) {
+		u8 scsi_status;
+		struct pqi_raid_error_info *raid_error_info;
+
+		raid_error_info = io_request->error_info;
+		scsi_status = raid_error_info->status;
+
+		if (scsi_status == SAM_STAT_CHECK_CONDITION &&
+			raid_error_info->data_out_result ==
+			PQI_DATA_IN_OUT_UNDERFLOW)
+			scsi_status = SAM_STAT_GOOD;
+
+		if (scsi_status != SAM_STAT_GOOD)
+			rc = -EIO;
+	}
+
+	pqi_free_io_request(io_request);
+
+	up(&ctrl_info->sync_request_sem);
+
+	return rc;
+}
+
+static int pqi_validate_admin_response(
+	struct pqi_general_admin_response *response, u8 expected_function_code)
+{
+	if (response->header.iu_type != PQI_RESPONSE_IU_GENERAL_ADMIN)
+		return -EINVAL;
+
+	if (get_unaligned_le16(&response->header.iu_length) !=
+		PQI_GENERAL_ADMIN_IU_LENGTH)
+		return -EINVAL;
+
+	if (response->function_code != expected_function_code)
+		return -EINVAL;
+
+	if (response->status != PQI_GENERAL_ADMIN_STATUS_SUCCESS)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int pqi_submit_admin_request_synchronous(
+	struct pqi_ctrl_info *ctrl_info,
+	struct pqi_general_admin_request *request,
+	struct pqi_general_admin_response *response)
+{
+	int rc;
+
+	pqi_submit_admin_request(ctrl_info, request);
+
+	rc = pqi_poll_for_admin_response(ctrl_info, response);
+
+	if (rc == 0)
+		rc = pqi_validate_admin_response(response,
+			request->function_code);
+
+	return rc;
+}
+
+static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	struct pqi_general_admin_request request;
+	struct pqi_general_admin_response response;
+	struct pqi_device_capability *capability;
+	struct pqi_iu_layer_descriptor *sop_iu_layer_descriptor;
+
+	capability = kmalloc(sizeof(*capability), GFP_KERNEL);
+	if (!capability)
+		return -ENOMEM;
+
+	memset(&request, 0, sizeof(request));
+
+	request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
+	put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
+		&request.header.iu_length);
+	request.function_code =
+		PQI_GENERAL_ADMIN_FUNCTION_REPORT_DEVICE_CAPABILITY;
+	put_unaligned_le32(sizeof(*capability),
+		&request.data.report_device_capability.buffer_length);
+
+	rc = pqi_map_single(ctrl_info->pci_dev,
+		&request.data.report_device_capability.sg_descriptor,
+		capability, sizeof(*capability),
+		PCI_DMA_FROMDEVICE);
+	if (rc)
+		goto out;
+
+	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
+		&response);
+
+	pqi_pci_unmap(ctrl_info->pci_dev,
+		&request.data.report_device_capability.sg_descriptor, 1,
+		PCI_DMA_FROMDEVICE);
+
+	if (rc)
+		goto out;
+
+	if (response.status != PQI_GENERAL_ADMIN_STATUS_SUCCESS) {
+		rc = -EIO;
+		goto out;
+	}
+
+	ctrl_info->max_inbound_queues =
+		get_unaligned_le16(&capability->max_inbound_queues);
+	ctrl_info->max_elements_per_iq =
+		get_unaligned_le16(&capability->max_elements_per_iq);
+	ctrl_info->max_iq_element_length =
+		get_unaligned_le16(&capability->max_iq_element_length)
+		* 16;
+	ctrl_info->max_outbound_queues =
+		get_unaligned_le16(&capability->max_outbound_queues);
+	ctrl_info->max_elements_per_oq =
+		get_unaligned_le16(&capability->max_elements_per_oq);
+	ctrl_info->max_oq_element_length =
+		get_unaligned_le16(&capability->max_oq_element_length)
+		* 16;
+
+	sop_iu_layer_descriptor =
+		&capability->iu_layer_descriptors[PQI_PROTOCOL_SOP];
+
+	ctrl_info->max_inbound_iu_length_per_firmware =
+		get_unaligned_le16(
+			&sop_iu_layer_descriptor->max_inbound_iu_length);
+	ctrl_info->inbound_spanning_supported =
+		sop_iu_layer_descriptor->inbound_spanning_supported;
+	ctrl_info->outbound_spanning_supported =
+		sop_iu_layer_descriptor->outbound_spanning_supported;
+
+out:
+	kfree(capability);
+
+	return rc;
+}
+
+static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)
+{
+	if (ctrl_info->max_iq_element_length <
+		PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"max. inbound queue element length of %d is less than the required length of %d\n",
+			ctrl_info->max_iq_element_length,
+			PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
+		return -EINVAL;
+	}
+
+	if (ctrl_info->max_oq_element_length <
+		PQI_OPERATIONAL_OQ_ELEMENT_LENGTH) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"max. outbound queue element length of %d is less than the required length of %d\n",
+			ctrl_info->max_oq_element_length,
+			PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
+		return -EINVAL;
+	}
+
+	if (ctrl_info->max_inbound_iu_length_per_firmware <
+		PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"max. inbound IU length of %u is less than the min. required length of %d\n",
+			ctrl_info->max_inbound_iu_length_per_firmware,
+			PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
+		return -EINVAL;
+	}
+
+	if (!ctrl_info->inbound_spanning_supported) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"the controller does not support inbound spanning\n");
+		return -EINVAL;
+	}
+
+	if (ctrl_info->outbound_spanning_supported) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"the controller supports outbound spanning but this driver does not\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int pqi_delete_operational_queue(struct pqi_ctrl_info *ctrl_info,
+	bool inbound_queue, u16 queue_id)
+{
+	struct pqi_general_admin_request request;
+	struct pqi_general_admin_response response;
+
+	memset(&request, 0, sizeof(request));
+	request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
+	put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
+		&request.header.iu_length);
+	if (inbound_queue)
+		request.function_code =
+			PQI_GENERAL_ADMIN_FUNCTION_DELETE_IQ;
+	else
+		request.function_code =
+			PQI_GENERAL_ADMIN_FUNCTION_DELETE_OQ;
+	put_unaligned_le16(queue_id,
+		&request.data.delete_operational_queue.queue_id);
+
+	return pqi_submit_admin_request_synchronous(ctrl_info, &request,
+		&response);
+}
+
+static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	struct pqi_event_queue *event_queue;
+	struct pqi_general_admin_request request;
+	struct pqi_general_admin_response response;
+
+	event_queue = &ctrl_info->event_queue;
+
+	/*
+	 * Create OQ (Outbound Queue - device to host queue) to dedicate
+	 * to events.
+	 */
+	memset(&request, 0, sizeof(request));
+	request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
+	put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
+		&request.header.iu_length);
+	request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
+	put_unaligned_le16(event_queue->oq_id,
+		&request.data.create_operational_oq.queue_id);
+	put_unaligned_le64((u64)event_queue->oq_element_array_bus_addr,
+		&request.data.create_operational_oq.element_array_addr);
+	put_unaligned_le64((u64)event_queue->oq_pi_bus_addr,
+		&request.data.create_operational_oq.pi_addr);
+	put_unaligned_le16(PQI_NUM_EVENT_QUEUE_ELEMENTS,
+		&request.data.create_operational_oq.num_elements);
+	put_unaligned_le16(PQI_EVENT_OQ_ELEMENT_LENGTH / 16,
+		&request.data.create_operational_oq.element_length);
+	request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
+	put_unaligned_le16(event_queue->int_msg_num,
+		&request.data.create_operational_oq.int_msg_num);
+
+	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
+		&response);
+	if (rc)
+		return rc;
+
+	event_queue->oq_ci = ctrl_info->iomem_base +
+		PQI_DEVICE_REGISTERS_OFFSET +
+		get_unaligned_le64(
+			&response.data.create_operational_oq.oq_ci_offset);
+
+	return 0;
+}
+
+static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned int i;
+	int rc;
+	struct pqi_queue_group *queue_group;
+	struct pqi_general_admin_request request;
+	struct pqi_general_admin_response response;
+
+	i = ctrl_info->num_active_queue_groups;
+	queue_group = &ctrl_info->queue_groups[i];
+
+	/*
+	 * Create IQ (Inbound Queue - host to device queue) for
+	 * RAID path.
+	 */
+	memset(&request, 0, sizeof(request));
+	request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
+	put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
+		&request.header.iu_length);
+	request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
+	put_unaligned_le16(queue_group->iq_id[RAID_PATH],
+		&request.data.create_operational_iq.queue_id);
+	put_unaligned_le64(
+		(u64)queue_group->iq_element_array_bus_addr[RAID_PATH],
+		&request.data.create_operational_iq.element_array_addr);
+	put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[RAID_PATH],
+		&request.data.create_operational_iq.ci_addr);
+	put_unaligned_le16(ctrl_info->num_elements_per_iq,
+		&request.data.create_operational_iq.num_elements);
+	put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
+		&request.data.create_operational_iq.element_length);
+	request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
+
+	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
+		&response);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error creating inbound RAID queue\n");
+		return rc;
+	}
+
+	queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +
+		PQI_DEVICE_REGISTERS_OFFSET +
+		get_unaligned_le64(
+			&response.data.create_operational_iq.iq_pi_offset);
+
+	/*
+	 * Create IQ (Inbound Queue - host to device queue) for
+	 * Advanced I/O (AIO) path.
+	 */
+	memset(&request, 0, sizeof(request));
+	request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
+	put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
+		&request.header.iu_length);
+	request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
+	put_unaligned_le16(queue_group->iq_id[AIO_PATH],
+		&request.data.create_operational_iq.queue_id);
+	put_unaligned_le64((u64)queue_group->
+		iq_element_array_bus_addr[AIO_PATH],
+		&request.data.create_operational_iq.element_array_addr);
+	put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[AIO_PATH],
+		&request.data.create_operational_iq.ci_addr);
+	put_unaligned_le16(ctrl_info->num_elements_per_iq,
+		&request.data.create_operational_iq.num_elements);
+	put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
+		&request.data.create_operational_iq.element_length);
+	request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
+
+	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
+		&response);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error creating inbound AIO queue\n");
+		goto delete_inbound_queue_raid;
+	}
+
+	queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +
+		PQI_DEVICE_REGISTERS_OFFSET +
+		get_unaligned_le64(
+			&response.data.create_operational_iq.iq_pi_offset);
+
+	/*
+	 * Designate the 2nd IQ as the AIO path.  By default, all IQs are
+	 * assumed to be for RAID path I/O unless we change the queue's
+	 * property.
+	 */
+	memset(&request, 0, sizeof(request));
+	request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
+	put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
+		&request.header.iu_length);
+	request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CHANGE_IQ_PROPERTY;
+	put_unaligned_le16(queue_group->iq_id[AIO_PATH],
+		&request.data.change_operational_iq_properties.queue_id);
+	put_unaligned_le32(PQI_IQ_PROPERTY_IS_AIO_QUEUE,
+		&request.data.change_operational_iq_properties.vendor_specific);
+
+	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
+		&response);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error changing queue property\n");
+		goto delete_inbound_queue_aio;
+	}
+
+	/*
+	 * Create OQ (Outbound Queue - device to host queue).
+	 */
+	memset(&request, 0, sizeof(request));
+	request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
+	put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
+		&request.header.iu_length);
+	request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
+	put_unaligned_le16(queue_group->oq_id,
+		&request.data.create_operational_oq.queue_id);
+	put_unaligned_le64((u64)queue_group->oq_element_array_bus_addr,
+		&request.data.create_operational_oq.element_array_addr);
+	put_unaligned_le64((u64)queue_group->oq_pi_bus_addr,
+		&request.data.create_operational_oq.pi_addr);
+	put_unaligned_le16(ctrl_info->num_elements_per_oq,
+		&request.data.create_operational_oq.num_elements);
+	put_unaligned_le16(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH / 16,
+		&request.data.create_operational_oq.element_length);
+	request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
+	put_unaligned_le16(queue_group->int_msg_num,
+		&request.data.create_operational_oq.int_msg_num);
+
+	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
+		&response);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error creating outbound queue\n");
+		goto delete_inbound_queue_aio;
+	}
+
+	queue_group->oq_ci = ctrl_info->iomem_base +
+		PQI_DEVICE_REGISTERS_OFFSET +
+		get_unaligned_le64(
+			&response.data.create_operational_oq.oq_ci_offset);
+
+	ctrl_info->num_active_queue_groups++;
+
+	return 0;
+
+delete_inbound_queue_aio:
+	pqi_delete_operational_queue(ctrl_info, true,
+		queue_group->iq_id[AIO_PATH]);
+
+delete_inbound_queue_raid:
+	pqi_delete_operational_queue(ctrl_info, true,
+		queue_group->iq_id[RAID_PATH]);
+
+	return rc;
+}
+
+static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	unsigned int i;
+
+	rc = pqi_create_event_queue(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error creating event queue\n");
+		return rc;
+	}
+
+	for (i = 0; i < ctrl_info->num_queue_groups; i++) {
+		rc = pqi_create_queue_group(ctrl_info);
+		if (rc) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"error creating queue group number %u/%u\n",
+				i, ctrl_info->num_queue_groups);
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+#define PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH	\
+	(offsetof(struct pqi_event_config, descriptors) + \
+	(PQI_MAX_EVENT_DESCRIPTORS * sizeof(struct pqi_event_descriptor)))
+
+static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	unsigned int i;
+	struct pqi_event_config *event_config;
+	struct pqi_general_management_request request;
+
+	event_config = kmalloc(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
+		GFP_KERNEL);
+	if (!event_config)
+		return -ENOMEM;
+
+	memset(&request, 0, sizeof(request));
+
+	request.header.iu_type = PQI_REQUEST_IU_REPORT_VENDOR_EVENT_CONFIG;
+	put_unaligned_le16(offsetof(struct pqi_general_management_request,
+		data.report_event_configuration.sg_descriptors[1]) -
+		PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
+	put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
+		&request.data.report_event_configuration.buffer_length);
+
+	rc = pqi_map_single(ctrl_info->pci_dev,
+		request.data.report_event_configuration.sg_descriptors,
+		event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
+		PCI_DMA_FROMDEVICE);
+	if (rc)
+		goto out;
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
+		0, NULL, NO_TIMEOUT);
+
+	pqi_pci_unmap(ctrl_info->pci_dev,
+		request.data.report_event_configuration.sg_descriptors, 1,
+		PCI_DMA_FROMDEVICE);
+
+	if (rc)
+		goto out;
+
+	for (i = 0; i < event_config->num_event_descriptors; i++)
+		put_unaligned_le16(ctrl_info->event_queue.oq_id,
+			&event_config->descriptors[i].oq_id);
+
+	memset(&request, 0, sizeof(request));
+
+	request.header.iu_type = PQI_REQUEST_IU_SET_VENDOR_EVENT_CONFIG;
+	put_unaligned_le16(offsetof(struct pqi_general_management_request,
+		data.report_event_configuration.sg_descriptors[1]) -
+		PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
+	put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
+		&request.data.report_event_configuration.buffer_length);
+
+	rc = pqi_map_single(ctrl_info->pci_dev,
+		request.data.report_event_configuration.sg_descriptors,
+		event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
+		PCI_DMA_TODEVICE);
+	if (rc)
+		goto out;
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
+		NULL, NO_TIMEOUT);
+
+	pqi_pci_unmap(ctrl_info->pci_dev,
+		request.data.report_event_configuration.sg_descriptors, 1,
+		PCI_DMA_TODEVICE);
+
+out:
+	kfree(event_config);
+
+	return rc;
+}
+
+static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned int i;
+	struct device *dev;
+	size_t sg_chain_buffer_length;
+	struct pqi_io_request *io_request;
+
+	if (!ctrl_info->io_request_pool)
+		return;
+
+	dev = &ctrl_info->pci_dev->dev;
+	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
+	io_request = ctrl_info->io_request_pool;
+
+	for (i = 0; i < ctrl_info->max_io_slots; i++) {
+		kfree(io_request->iu);
+		if (!io_request->sg_chain_buffer)
+			break;
+		dma_free_coherent(dev, sg_chain_buffer_length,
+			io_request->sg_chain_buffer,
+			io_request->sg_chain_buffer_dma_handle);
+		io_request++;
+	}
+
+	kfree(ctrl_info->io_request_pool);
+	ctrl_info->io_request_pool = NULL;
+}
+
+static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)
+{
+	ctrl_info->error_buffer = dma_zalloc_coherent(&ctrl_info->pci_dev->dev,
+		ctrl_info->error_buffer_length,
+		&ctrl_info->error_buffer_dma_handle, GFP_KERNEL);
+
+	if (!ctrl_info->error_buffer)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned int i;
+	void *sg_chain_buffer;
+	size_t sg_chain_buffer_length;
+	dma_addr_t sg_chain_buffer_dma_handle;
+	struct device *dev;
+	struct pqi_io_request *io_request;
+
+	ctrl_info->io_request_pool = kzalloc(ctrl_info->max_io_slots *
+		sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL);
+
+	if (!ctrl_info->io_request_pool) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"failed to allocate I/O request pool\n");
+		goto error;
+	}
+
+	dev = &ctrl_info->pci_dev->dev;
+	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
+	io_request = ctrl_info->io_request_pool;
+
+	for (i = 0; i < ctrl_info->max_io_slots; i++) {
+		io_request->iu =
+			kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);
+
+		if (!io_request->iu) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"failed to allocate IU buffers\n");
+			goto error;
+		}
+
+		sg_chain_buffer = dma_alloc_coherent(dev,
+			sg_chain_buffer_length, &sg_chain_buffer_dma_handle,
+			GFP_KERNEL);
+
+		if (!sg_chain_buffer) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"failed to allocate PQI scatter-gather chain buffers\n");
+			goto error;
+		}
+
+		io_request->index = i;
+		io_request->sg_chain_buffer = sg_chain_buffer;
+		io_request->sg_chain_buffer_dma_handle =
+			sg_chain_buffer_dma_handle;
+		io_request++;
+	}
+
+	return 0;
+
+error:
+	pqi_free_all_io_requests(ctrl_info);
+
+	return -ENOMEM;
+}
+
+/*
+ * Calculate required resources that are sized based on max. outstanding
+ * requests and max. transfer size.
+ */
+
+static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)
+{
+	u32 max_transfer_size;
+	u32 max_sg_entries;
+
+	ctrl_info->scsi_ml_can_queue =
+		ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;
+	ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;
+
+	ctrl_info->error_buffer_length =
+		ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;
+
+	max_transfer_size =
+		min(ctrl_info->max_transfer_size, PQI_MAX_TRANSFER_SIZE);
+
+	max_sg_entries = max_transfer_size / PAGE_SIZE;
+
+	/* +1 to cover when the buffer is not page-aligned. */
+	max_sg_entries++;
+
+	max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);
+
+	max_transfer_size = (max_sg_entries - 1) * PAGE_SIZE;
+
+	ctrl_info->sg_chain_buffer_length =
+		max_sg_entries * sizeof(struct pqi_sg_descriptor);
+	ctrl_info->sg_tablesize = max_sg_entries;
+	ctrl_info->max_sectors = max_transfer_size / 512;
+}
+
+static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)
+{
+	int num_cpus;
+	int max_queue_groups;
+	int num_queue_groups;
+	u16 num_elements_per_iq;
+	u16 num_elements_per_oq;
+
+	max_queue_groups = min(ctrl_info->max_inbound_queues / 2,
+		ctrl_info->max_outbound_queues - 1);
+	max_queue_groups = min(max_queue_groups, PQI_MAX_QUEUE_GROUPS);
+
+	num_cpus = num_online_cpus();
+	num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors);
+	num_queue_groups = min(num_queue_groups, max_queue_groups);
+
+	ctrl_info->num_queue_groups = num_queue_groups;
+
+	/*
+	 * Make sure that the max. inbound IU length is an even multiple
+	 * of our inbound element length.
+	 */
+	ctrl_info->max_inbound_iu_length =
+		(ctrl_info->max_inbound_iu_length_per_firmware /
+		PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) *
+		PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
+
+	num_elements_per_iq =
+		(ctrl_info->max_inbound_iu_length /
+		PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
+
+	/* Add one because one element in each queue is unusable. */
+	num_elements_per_iq++;
+
+	num_elements_per_iq = min(num_elements_per_iq,
+		ctrl_info->max_elements_per_iq);
+
+	num_elements_per_oq = ((num_elements_per_iq - 1) * 2) + 1;
+	num_elements_per_oq = min(num_elements_per_oq,
+		ctrl_info->max_elements_per_oq);
+
+	ctrl_info->num_elements_per_iq = num_elements_per_iq;
+	ctrl_info->num_elements_per_oq = num_elements_per_oq;
+
+	ctrl_info->max_sg_per_iu =
+		((ctrl_info->max_inbound_iu_length -
+		PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) /
+		sizeof(struct pqi_sg_descriptor)) +
+		PQI_MAX_EMBEDDED_SG_DESCRIPTORS;
+}
+
+static inline void pqi_set_sg_descriptor(
+	struct pqi_sg_descriptor *sg_descriptor, struct scatterlist *sg)
+{
+	u64 address = (u64)sg_dma_address(sg);
+	unsigned int length = sg_dma_len(sg);
+
+	put_unaligned_le64(address, &sg_descriptor->address);
+	put_unaligned_le32(length, &sg_descriptor->length);
+	put_unaligned_le32(0, &sg_descriptor->flags);
+}
+
+static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_raid_path_request *request, struct scsi_cmnd *scmd,
+	struct pqi_io_request *io_request)
+{
+	int i;
+	u16 iu_length;
+	int sg_count;
+	bool chained;
+	unsigned int num_sg_in_iu;
+	unsigned int max_sg_per_iu;
+	struct scatterlist *sg;
+	struct pqi_sg_descriptor *sg_descriptor;
+
+	sg_count = scsi_dma_map(scmd);
+	if (sg_count < 0)
+		return sg_count;
+
+	iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
+		PQI_REQUEST_HEADER_LENGTH;
+
+	if (sg_count == 0)
+		goto out;
+
+	sg = scsi_sglist(scmd);
+	sg_descriptor = request->sg_descriptors;
+	max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;
+	chained = false;
+	num_sg_in_iu = 0;
+	i = 0;
+
+	while (1) {
+		pqi_set_sg_descriptor(sg_descriptor, sg);
+		if (!chained)
+			num_sg_in_iu++;
+		i++;
+		if (i == sg_count)
+			break;
+		sg_descriptor++;
+		if (i == max_sg_per_iu) {
+			put_unaligned_le64(
+				(u64)io_request->sg_chain_buffer_dma_handle,
+				&sg_descriptor->address);
+			put_unaligned_le32((sg_count - num_sg_in_iu)
+				* sizeof(*sg_descriptor),
+				&sg_descriptor->length);
+			put_unaligned_le32(CISS_SG_CHAIN,
+				&sg_descriptor->flags);
+			chained = true;
+			num_sg_in_iu++;
+			sg_descriptor = io_request->sg_chain_buffer;
+		}
+		sg = sg_next(sg);
+	}
+
+	put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
+	request->partial = chained;
+	iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
+
+out:
+	put_unaligned_le16(iu_length, &request->header.iu_length);
+
+	return 0;
+}
+
+static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_aio_path_request *request, struct scsi_cmnd *scmd,
+	struct pqi_io_request *io_request)
+{
+	int i;
+	u16 iu_length;
+	int sg_count;
+	bool chained;
+	unsigned int num_sg_in_iu;
+	unsigned int max_sg_per_iu;
+	struct scatterlist *sg;
+	struct pqi_sg_descriptor *sg_descriptor;
+
+	sg_count = scsi_dma_map(scmd);
+	if (sg_count < 0)
+		return sg_count;
+
+	iu_length = offsetof(struct pqi_aio_path_request, sg_descriptors) -
+		PQI_REQUEST_HEADER_LENGTH;
+	num_sg_in_iu = 0;
+
+	if (sg_count == 0)
+		goto out;
+
+	sg = scsi_sglist(scmd);
+	sg_descriptor = request->sg_descriptors;
+	max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;
+	chained = false;
+	i = 0;
+
+	while (1) {
+		pqi_set_sg_descriptor(sg_descriptor, sg);
+		if (!chained)
+			num_sg_in_iu++;
+		i++;
+		if (i == sg_count)
+			break;
+		sg_descriptor++;
+		if (i == max_sg_per_iu) {
+			put_unaligned_le64(
+				(u64)io_request->sg_chain_buffer_dma_handle,
+				&sg_descriptor->address);
+			put_unaligned_le32((sg_count - num_sg_in_iu)
+				* sizeof(*sg_descriptor),
+				&sg_descriptor->length);
+			put_unaligned_le32(CISS_SG_CHAIN,
+				&sg_descriptor->flags);
+			chained = true;
+			num_sg_in_iu++;
+			sg_descriptor = io_request->sg_chain_buffer;
+		}
+		sg = sg_next(sg);
+	}
+
+	put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
+	request->partial = chained;
+	iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
+
+out:
+	put_unaligned_le16(iu_length, &request->header.iu_length);
+	request->num_sg_descriptors = num_sg_in_iu;
+
+	return 0;
+}
+
+static void pqi_raid_io_complete(struct pqi_io_request *io_request,
+	void *context)
+{
+	struct scsi_cmnd *scmd;
+
+	scmd = io_request->scmd;
+	pqi_free_io_request(io_request);
+	scsi_dma_unmap(scmd);
+	pqi_scsi_done(scmd);
+}
+
+static int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
+	struct pqi_queue_group *queue_group)
+{
+	int rc;
+	size_t cdb_length;
+	struct pqi_io_request *io_request;
+	struct pqi_raid_path_request *request;
+
+	io_request = pqi_alloc_io_request(ctrl_info);
+	io_request->io_complete_callback = pqi_raid_io_complete;
+	io_request->scmd = scmd;
+
+	scmd->host_scribble = (unsigned char *)io_request;
+
+	request = io_request->iu;
+	memset(request, 0,
+		offsetof(struct pqi_raid_path_request, sg_descriptors));
+
+	request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
+	put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
+	request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
+	put_unaligned_le16(io_request->index, &request->request_id);
+	request->error_index = request->request_id;
+	memcpy(request->lun_number, device->scsi3addr,
+		sizeof(request->lun_number));
+
+	cdb_length = min_t(size_t, scmd->cmd_len, sizeof(request->cdb));
+	memcpy(request->cdb, scmd->cmnd, cdb_length);
+
+	switch (cdb_length) {
+	case 6:
+	case 10:
+	case 12:
+	case 16:
+		/* No bytes in the Additional CDB bytes field */
+		request->additional_cdb_bytes_usage =
+			SOP_ADDITIONAL_CDB_BYTES_0;
+		break;
+	case 20:
+		/* 4 bytes in the Additional cdb field */
+		request->additional_cdb_bytes_usage =
+			SOP_ADDITIONAL_CDB_BYTES_4;
+		break;
+	case 24:
+		/* 8 bytes in the Additional cdb field */
+		request->additional_cdb_bytes_usage =
+			SOP_ADDITIONAL_CDB_BYTES_8;
+		break;
+	case 28:
+		/* 12 bytes in the Additional cdb field */
+		request->additional_cdb_bytes_usage =
+			SOP_ADDITIONAL_CDB_BYTES_12;
+		break;
+	case 32:
+	default:
+		/* 16 bytes in the Additional cdb field */
+		request->additional_cdb_bytes_usage =
+			SOP_ADDITIONAL_CDB_BYTES_16;
+		break;
+	}
+
+	switch (scmd->sc_data_direction) {
+	case DMA_TO_DEVICE:
+		request->data_direction = SOP_READ_FLAG;
+		break;
+	case DMA_FROM_DEVICE:
+		request->data_direction = SOP_WRITE_FLAG;
+		break;
+	case DMA_NONE:
+		request->data_direction = SOP_NO_DIRECTION_FLAG;
+		break;
+	case DMA_BIDIRECTIONAL:
+		request->data_direction = SOP_BIDIRECTIONAL;
+		break;
+	default:
+		dev_err(&ctrl_info->pci_dev->dev,
+			"unknown data direction: %d\n",
+			scmd->sc_data_direction);
+		WARN_ON(scmd->sc_data_direction);
+		break;
+	}
+
+	rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);
+	if (rc) {
+		pqi_free_io_request(io_request);
+		return SCSI_MLQUEUE_HOST_BUSY;
+	}
+
+	pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);
+
+	return 0;
+}
+
+static void pqi_aio_io_complete(struct pqi_io_request *io_request,
+	void *context)
+{
+	struct scsi_cmnd *scmd;
+
+	scmd = io_request->scmd;
+	scsi_dma_unmap(scmd);
+	if (io_request->status == -EAGAIN)
+		set_host_byte(scmd, DID_IMM_RETRY);
+	pqi_free_io_request(io_request);
+	pqi_scsi_done(scmd);
+}
+
+static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
+	struct pqi_queue_group *queue_group)
+{
+	return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,
+		scmd->cmnd, scmd->cmd_len, queue_group, NULL);
+}
+
+static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
+	struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
+	unsigned int cdb_length, struct pqi_queue_group *queue_group,
+	struct pqi_encryption_info *encryption_info)
+{
+	int rc;
+	struct pqi_io_request *io_request;
+	struct pqi_aio_path_request *request;
+
+	io_request = pqi_alloc_io_request(ctrl_info);
+	io_request->io_complete_callback = pqi_aio_io_complete;
+	io_request->scmd = scmd;
+
+	scmd->host_scribble = (unsigned char *)io_request;
+
+	request = io_request->iu;
+	memset(request, 0,
+		offsetof(struct pqi_raid_path_request, sg_descriptors));
+
+	request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_IO;
+	put_unaligned_le32(aio_handle, &request->nexus_id);
+	put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
+	request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
+	put_unaligned_le16(io_request->index, &request->request_id);
+	request->error_index = request->request_id;
+	if (cdb_length > sizeof(request->cdb))
+		cdb_length = sizeof(request->cdb);
+	request->cdb_length = cdb_length;
+	memcpy(request->cdb, cdb, cdb_length);
+
+	switch (scmd->sc_data_direction) {
+	case DMA_TO_DEVICE:
+		request->data_direction = SOP_READ_FLAG;
+		break;
+	case DMA_FROM_DEVICE:
+		request->data_direction = SOP_WRITE_FLAG;
+		break;
+	case DMA_NONE:
+		request->data_direction = SOP_NO_DIRECTION_FLAG;
+		break;
+	case DMA_BIDIRECTIONAL:
+		request->data_direction = SOP_BIDIRECTIONAL;
+		break;
+	default:
+		dev_err(&ctrl_info->pci_dev->dev,
+			"unknown data direction: %d\n",
+			scmd->sc_data_direction);
+		WARN_ON(scmd->sc_data_direction);
+		break;
+	}
+
+	if (encryption_info) {
+		request->encryption_enable = true;
+		put_unaligned_le16(encryption_info->data_encryption_key_index,
+			&request->data_encryption_key_index);
+		put_unaligned_le32(encryption_info->encrypt_tweak_lower,
+			&request->encrypt_tweak_lower);
+		put_unaligned_le32(encryption_info->encrypt_tweak_upper,
+			&request->encrypt_tweak_upper);
+	}
+
+	rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);
+	if (rc) {
+		pqi_free_io_request(io_request);
+		return SCSI_MLQUEUE_HOST_BUSY;
+	}
+
+	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
+
+	return 0;
+}
+
+static int pqi_scsi_queue_command(struct Scsi_Host *shost,
+	struct scsi_cmnd *scmd)
+{
+	int rc;
+	struct pqi_ctrl_info *ctrl_info;
+	struct pqi_scsi_dev *device;
+	u16 hwq;
+	struct pqi_queue_group *queue_group;
+	bool raid_bypassed;
+
+	device = scmd->device->hostdata;
+	ctrl_info = shost_to_hba(shost);
+
+	if (pqi_ctrl_offline(ctrl_info)) {
+		set_host_byte(scmd, DID_NO_CONNECT);
+		pqi_scsi_done(scmd);
+		return 0;
+	}
+
+	/*
+	 * This is necessary because the SML doesn't zero out this field during
+	 * error recovery.
+	 */
+	scmd->result = 0;
+
+	hwq = blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(scmd->request));
+	if (hwq >= ctrl_info->num_queue_groups)
+		hwq = 0;
+
+	queue_group = &ctrl_info->queue_groups[hwq];
+
+	if (pqi_is_logical_device(device)) {
+		raid_bypassed = false;
+		if (device->offload_enabled &&
+			scmd->request->cmd_type == REQ_TYPE_FS) {
+			rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device,
+				scmd, queue_group);
+			if (rc == 0 ||
+				rc == SCSI_MLQUEUE_HOST_BUSY ||
+				rc == SAM_STAT_CHECK_CONDITION ||
+				rc == SAM_STAT_RESERVATION_CONFLICT)
+				raid_bypassed = true;
+		}
+		if (!raid_bypassed)
+			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd,
+				queue_group);
+	} else {
+		if (device->aio_enabled)
+			rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd,
+				queue_group);
+		else
+			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd,
+				queue_group);
+	}
+
+	return rc;
+}
+
+static void pqi_lun_reset_complete(struct pqi_io_request *io_request,
+	void *context)
+{
+	struct completion *waiting = context;
+
+	complete(waiting);
+}
+
+#define PQI_LUN_RESET_TIMEOUT_SECS	10
+
+static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device, struct completion *wait)
+{
+	int rc;
+	unsigned int wait_secs = 0;
+
+	while (1) {
+		if (wait_for_completion_io_timeout(wait,
+			PQI_LUN_RESET_TIMEOUT_SECS * HZ)) {
+			rc = 0;
+			break;
+		}
+
+		pqi_check_ctrl_health(ctrl_info);
+		if (pqi_ctrl_offline(ctrl_info)) {
+			rc = -ETIMEDOUT;
+			break;
+		}
+
+		wait_secs += PQI_LUN_RESET_TIMEOUT_SECS;
+
+		dev_err(&ctrl_info->pci_dev->dev,
+			"resetting scsi %d:%d:%d:%d - waiting %u seconds\n",
+			ctrl_info->scsi_host->host_no, device->bus,
+			device->target, device->lun, wait_secs);
+	}
+
+	return rc;
+}
+
+static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	int rc;
+	struct pqi_io_request *io_request;
+	DECLARE_COMPLETION_ONSTACK(wait);
+	struct pqi_task_management_request *request;
+
+	down(&ctrl_info->lun_reset_sem);
+
+	io_request = pqi_alloc_io_request(ctrl_info);
+	io_request->io_complete_callback = pqi_lun_reset_complete;
+	io_request->context = &wait;
+
+	request = io_request->iu;
+	memset(request, 0, sizeof(*request));
+
+	request->header.iu_type = PQI_REQUEST_IU_TASK_MANAGEMENT;
+	put_unaligned_le16(sizeof(*request) - PQI_REQUEST_HEADER_LENGTH,
+		&request->header.iu_length);
+	put_unaligned_le16(io_request->index, &request->request_id);
+	memcpy(request->lun_number, device->scsi3addr,
+		sizeof(request->lun_number));
+	request->task_management_function = SOP_TASK_MANAGEMENT_LUN_RESET;
+
+	pqi_start_io(ctrl_info,
+		&ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
+		io_request);
+
+	rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, &wait);
+	if (rc == 0)
+		rc = io_request->status;
+
+	pqi_free_io_request(io_request);
+	up(&ctrl_info->lun_reset_sem);
+
+	return rc;
+}
+
+/* Performs a reset at the LUN level. */
+
+static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info,
+	struct pqi_scsi_dev *device)
+{
+	int rc;
+
+	pqi_check_ctrl_health(ctrl_info);
+	if (pqi_ctrl_offline(ctrl_info))
+		return FAILED;
+
+	rc = pqi_lun_reset(ctrl_info, device);
+
+	return rc == 0 ? SUCCESS : FAILED;
+}
+
+static int pqi_eh_device_reset_handler(struct scsi_cmnd *scmd)
+{
+	int rc;
+	struct pqi_ctrl_info *ctrl_info;
+	struct pqi_scsi_dev *device;
+
+	ctrl_info = shost_to_hba(scmd->device->host);
+	device = scmd->device->hostdata;
+
+	dev_err(&ctrl_info->pci_dev->dev,
+		"resetting scsi %d:%d:%d:%d\n",
+		ctrl_info->scsi_host->host_no,
+		device->bus, device->target, device->lun);
+
+	rc = pqi_device_reset(ctrl_info, device);
+
+	dev_err(&ctrl_info->pci_dev->dev,
+		"reset of scsi %d:%d:%d:%d: %s\n",
+		ctrl_info->scsi_host->host_no,
+		device->bus, device->target, device->lun,
+		rc == SUCCESS ? "SUCCESS" : "FAILED");
+
+	return rc;
+}
+
+static int pqi_slave_alloc(struct scsi_device *sdev)
+{
+	struct pqi_scsi_dev *device;
+	unsigned long flags;
+	struct pqi_ctrl_info *ctrl_info;
+	struct scsi_target *starget;
+	struct sas_rphy *rphy;
+
+	ctrl_info = shost_to_hba(sdev->host);
+
+	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
+
+	if (sdev_channel(sdev) == PQI_PHYSICAL_DEVICE_BUS) {
+		starget = scsi_target(sdev);
+		rphy = target_to_rphy(starget);
+		device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);
+		if (device) {
+			device->target = sdev_id(sdev);
+			device->lun = sdev->lun;
+			device->target_lun_valid = true;
+		}
+	} else {
+		device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),
+			sdev_id(sdev), sdev->lun);
+	}
+
+	if (device && device->expose_device) {
+		sdev->hostdata = device;
+		device->sdev = sdev;
+		if (device->queue_depth) {
+			device->advertised_queue_depth = device->queue_depth;
+			scsi_change_queue_depth(sdev,
+				device->advertised_queue_depth);
+		}
+	}
+
+	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
+
+	return 0;
+}
+
+static int pqi_slave_configure(struct scsi_device *sdev)
+{
+	struct pqi_scsi_dev *device;
+
+	device = sdev->hostdata;
+	if (!device->expose_device)
+		sdev->no_uld_attach = true;
+
+	return 0;
+}
+
+static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info,
+	void __user *arg)
+{
+	struct pci_dev *pci_dev;
+	u32 subsystem_vendor;
+	u32 subsystem_device;
+	cciss_pci_info_struct pciinfo;
+
+	if (!arg)
+		return -EINVAL;
+
+	pci_dev = ctrl_info->pci_dev;
+
+	pciinfo.domain = pci_domain_nr(pci_dev->bus);
+	pciinfo.bus = pci_dev->bus->number;
+	pciinfo.dev_fn = pci_dev->devfn;
+	subsystem_vendor = pci_dev->subsystem_vendor;
+	subsystem_device = pci_dev->subsystem_device;
+	pciinfo.board_id = ((subsystem_device << 16) & 0xffff0000) |
+		subsystem_vendor;
+
+	if (copy_to_user(arg, &pciinfo, sizeof(pciinfo)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int pqi_getdrivver_ioctl(void __user *arg)
+{
+	u32 version;
+
+	if (!arg)
+		return -EINVAL;
+
+	version = (DRIVER_MAJOR << 28) | (DRIVER_MINOR << 24) |
+		(DRIVER_RELEASE << 16) | DRIVER_REVISION;
+
+	if (copy_to_user(arg, &version, sizeof(version)))
+		return -EFAULT;
+
+	return 0;
+}
+
+struct ciss_error_info {
+	u8	scsi_status;
+	int	command_status;
+	size_t	sense_data_length;
+};
+
+static void pqi_error_info_to_ciss(struct pqi_raid_error_info *pqi_error_info,
+	struct ciss_error_info *ciss_error_info)
+{
+	int ciss_cmd_status;
+	size_t sense_data_length;
+
+	switch (pqi_error_info->data_out_result) {
+	case PQI_DATA_IN_OUT_GOOD:
+		ciss_cmd_status = CISS_CMD_STATUS_SUCCESS;
+		break;
+	case PQI_DATA_IN_OUT_UNDERFLOW:
+		ciss_cmd_status = CISS_CMD_STATUS_DATA_UNDERRUN;
+		break;
+	case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
+		ciss_cmd_status = CISS_CMD_STATUS_DATA_OVERRUN;
+		break;
+	case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
+	case PQI_DATA_IN_OUT_BUFFER_ERROR:
+	case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
+	case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
+	case PQI_DATA_IN_OUT_ERROR:
+		ciss_cmd_status = CISS_CMD_STATUS_PROTOCOL_ERROR;
+		break;
+	case PQI_DATA_IN_OUT_HARDWARE_ERROR:
+	case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
+	case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
+	case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
+	case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
+	case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
+	case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
+	case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
+	case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
+	case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
+		ciss_cmd_status = CISS_CMD_STATUS_HARDWARE_ERROR;
+		break;
+	case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
+		ciss_cmd_status = CISS_CMD_STATUS_UNSOLICITED_ABORT;
+		break;
+	case PQI_DATA_IN_OUT_ABORTED:
+		ciss_cmd_status = CISS_CMD_STATUS_ABORTED;
+		break;
+	case PQI_DATA_IN_OUT_TIMEOUT:
+		ciss_cmd_status = CISS_CMD_STATUS_TIMEOUT;
+		break;
+	default:
+		ciss_cmd_status = CISS_CMD_STATUS_TARGET_STATUS;
+		break;
+	}
+
+	sense_data_length =
+		get_unaligned_le16(&pqi_error_info->sense_data_length);
+	if (sense_data_length == 0)
+		sense_data_length =
+		get_unaligned_le16(&pqi_error_info->response_data_length);
+	if (sense_data_length)
+		if (sense_data_length > sizeof(pqi_error_info->data))
+			sense_data_length = sizeof(pqi_error_info->data);
+
+	ciss_error_info->scsi_status = pqi_error_info->status;
+	ciss_error_info->command_status = ciss_cmd_status;
+	ciss_error_info->sense_data_length = sense_data_length;
+}
+
+static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
+{
+	int rc;
+	char *kernel_buffer = NULL;
+	u16 iu_length;
+	size_t sense_data_length;
+	IOCTL_Command_struct iocommand;
+	struct pqi_raid_path_request request;
+	struct pqi_raid_error_info pqi_error_info;
+	struct ciss_error_info ciss_error_info;
+
+	if (pqi_ctrl_offline(ctrl_info))
+		return -ENXIO;
+	if (!arg)
+		return -EINVAL;
+	if (!capable(CAP_SYS_RAWIO))
+		return -EPERM;
+	if (copy_from_user(&iocommand, arg, sizeof(iocommand)))
+		return -EFAULT;
+	if (iocommand.buf_size < 1 &&
+		iocommand.Request.Type.Direction != XFER_NONE)
+		return -EINVAL;
+	if (iocommand.Request.CDBLen > sizeof(request.cdb))
+		return -EINVAL;
+	if (iocommand.Request.Type.Type != TYPE_CMD)
+		return -EINVAL;
+
+	switch (iocommand.Request.Type.Direction) {
+	case XFER_NONE:
+	case XFER_WRITE:
+	case XFER_READ:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (iocommand.buf_size > 0) {
+		kernel_buffer = kmalloc(iocommand.buf_size, GFP_KERNEL);
+		if (!kernel_buffer)
+			return -ENOMEM;
+		if (iocommand.Request.Type.Direction & XFER_WRITE) {
+			if (copy_from_user(kernel_buffer, iocommand.buf,
+				iocommand.buf_size)) {
+				rc = -EFAULT;
+				goto out;
+			}
+		} else {
+			memset(kernel_buffer, 0, iocommand.buf_size);
+		}
+	}
+
+	memset(&request, 0, sizeof(request));
+
+	request.header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
+	iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
+		PQI_REQUEST_HEADER_LENGTH;
+	memcpy(request.lun_number, iocommand.LUN_info.LunAddrBytes,
+		sizeof(request.lun_number));
+	memcpy(request.cdb, iocommand.Request.CDB, iocommand.Request.CDBLen);
+	request.additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
+
+	switch (iocommand.Request.Type.Direction) {
+	case XFER_NONE:
+		request.data_direction = SOP_NO_DIRECTION_FLAG;
+		break;
+	case XFER_WRITE:
+		request.data_direction = SOP_WRITE_FLAG;
+		break;
+	case XFER_READ:
+		request.data_direction = SOP_READ_FLAG;
+		break;
+	}
+
+	request.task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
+
+	if (iocommand.buf_size > 0) {
+		put_unaligned_le32(iocommand.buf_size, &request.buffer_length);
+
+		rc = pqi_map_single(ctrl_info->pci_dev,
+			&request.sg_descriptors[0], kernel_buffer,
+			iocommand.buf_size, PCI_DMA_BIDIRECTIONAL);
+		if (rc)
+			goto out;
+
+		iu_length += sizeof(request.sg_descriptors[0]);
+	}
+
+	put_unaligned_le16(iu_length, &request.header.iu_length);
+
+	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
+		PQI_SYNC_FLAGS_INTERRUPTABLE, &pqi_error_info, NO_TIMEOUT);
+
+	if (iocommand.buf_size > 0)
+		pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
+			PCI_DMA_BIDIRECTIONAL);
+
+	memset(&iocommand.error_info, 0, sizeof(iocommand.error_info));
+
+	if (rc == 0) {
+		pqi_error_info_to_ciss(&pqi_error_info, &ciss_error_info);
+		iocommand.error_info.ScsiStatus = ciss_error_info.scsi_status;
+		iocommand.error_info.CommandStatus =
+			ciss_error_info.command_status;
+		sense_data_length = ciss_error_info.sense_data_length;
+		if (sense_data_length) {
+			if (sense_data_length >
+				sizeof(iocommand.error_info.SenseInfo))
+				sense_data_length =
+					sizeof(iocommand.error_info.SenseInfo);
+			memcpy(iocommand.error_info.SenseInfo,
+				pqi_error_info.data, sense_data_length);
+			iocommand.error_info.SenseLen = sense_data_length;
+		}
+	}
+
+	if (copy_to_user(arg, &iocommand, sizeof(iocommand))) {
+		rc = -EFAULT;
+		goto out;
+	}
+
+	if (rc == 0 && iocommand.buf_size > 0 &&
+		(iocommand.Request.Type.Direction & XFER_READ)) {
+		if (copy_to_user(iocommand.buf, kernel_buffer,
+			iocommand.buf_size)) {
+			rc = -EFAULT;
+		}
+	}
+
+out:
+	kfree(kernel_buffer);
+
+	return rc;
+}
+
+static int pqi_ioctl(struct scsi_device *sdev, int cmd, void __user *arg)
+{
+	int rc;
+	struct pqi_ctrl_info *ctrl_info;
+
+	ctrl_info = shost_to_hba(sdev->host);
+
+	switch (cmd) {
+	case CCISS_DEREGDISK:
+	case CCISS_REGNEWDISK:
+	case CCISS_REGNEWD:
+		rc = pqi_scan_scsi_devices(ctrl_info);
+		break;
+	case CCISS_GETPCIINFO:
+		rc = pqi_getpciinfo_ioctl(ctrl_info, arg);
+		break;
+	case CCISS_GETDRIVVER:
+		rc = pqi_getdrivver_ioctl(arg);
+		break;
+	case CCISS_PASSTHRU:
+		rc = pqi_passthru_ioctl(ctrl_info, arg);
+		break;
+	default:
+		rc = -EINVAL;
+		break;
+	}
+
+	return rc;
+}
+
+static ssize_t pqi_version_show(struct device *dev,
+	struct device_attribute *attr, char *buffer)
+{
+	ssize_t count = 0;
+	struct Scsi_Host *shost;
+	struct pqi_ctrl_info *ctrl_info;
+
+	shost = class_to_shost(dev);
+	ctrl_info = shost_to_hba(shost);
+
+	count += snprintf(buffer + count, PAGE_SIZE - count,
+		"  driver: %s\n", DRIVER_VERSION BUILD_TIMESTAMP);
+
+	count += snprintf(buffer + count, PAGE_SIZE - count,
+		"firmware: %s\n", ctrl_info->firmware_version);
+
+	return count;
+}
+
+static ssize_t pqi_host_rescan_store(struct device *dev,
+	struct device_attribute *attr, const char *buffer, size_t count)
+{
+	struct Scsi_Host *shost = class_to_shost(dev);
+
+	pqi_scan_start(shost);
+
+	return count;
+}
+
+static DEVICE_ATTR(version, S_IRUGO, pqi_version_show, NULL);
+static DEVICE_ATTR(rescan, S_IWUSR, NULL, pqi_host_rescan_store);
+
+static struct device_attribute *pqi_shost_attrs[] = {
+	&dev_attr_version,
+	&dev_attr_rescan,
+	NULL
+};
+
+static ssize_t pqi_sas_address_show(struct device *dev,
+	struct device_attribute *attr, char *buffer)
+{
+	struct pqi_ctrl_info *ctrl_info;
+	struct scsi_device *sdev;
+	struct pqi_scsi_dev *device;
+	unsigned long flags;
+	u64 sas_address;
+
+	sdev = to_scsi_device(dev);
+	ctrl_info = shost_to_hba(sdev->host);
+
+	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
+
+	device = sdev->hostdata;
+	if (pqi_is_logical_device(device)) {
+		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,
+			flags);
+		return -ENODEV;
+	}
+	sas_address = device->sas_address;
+
+	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
+
+	return snprintf(buffer, PAGE_SIZE, "0x%016llx\n", sas_address);
+}
+
+static ssize_t pqi_ssd_smart_path_enabled_show(struct device *dev,
+	struct device_attribute *attr, char *buffer)
+{
+	struct pqi_ctrl_info *ctrl_info;
+	struct scsi_device *sdev;
+	struct pqi_scsi_dev *device;
+	unsigned long flags;
+
+	sdev = to_scsi_device(dev);
+	ctrl_info = shost_to_hba(sdev->host);
+
+	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
+
+	device = sdev->hostdata;
+	buffer[0] = device->offload_enabled ? '1' : '0';
+	buffer[1] = '\n';
+	buffer[2] = '\0';
+
+	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
+
+	return 2;
+}
+
+static DEVICE_ATTR(sas_address, S_IRUGO, pqi_sas_address_show, NULL);
+static DEVICE_ATTR(ssd_smart_path_enabled, S_IRUGO,
+	pqi_ssd_smart_path_enabled_show, NULL);
+
+static struct device_attribute *pqi_sdev_attrs[] = {
+	&dev_attr_sas_address,
+	&dev_attr_ssd_smart_path_enabled,
+	NULL
+};
+
+static struct scsi_host_template pqi_driver_template = {
+	.module = THIS_MODULE,
+	.name = DRIVER_NAME_SHORT,
+	.proc_name = DRIVER_NAME_SHORT,
+	.queuecommand = pqi_scsi_queue_command,
+	.scan_start = pqi_scan_start,
+	.scan_finished = pqi_scan_finished,
+	.this_id = -1,
+	.use_clustering = ENABLE_CLUSTERING,
+	.eh_device_reset_handler = pqi_eh_device_reset_handler,
+	.ioctl = pqi_ioctl,
+	.slave_alloc = pqi_slave_alloc,
+	.slave_configure = pqi_slave_configure,
+	.sdev_attrs = pqi_sdev_attrs,
+	.shost_attrs = pqi_shost_attrs,
+};
+
+static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	struct Scsi_Host *shost;
+
+	shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));
+	if (!shost) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"scsi_host_alloc failed for controller %u\n",
+			ctrl_info->ctrl_id);
+		return -ENOMEM;
+	}
+
+	shost->io_port = 0;
+	shost->n_io_port = 0;
+	shost->this_id = -1;
+	shost->max_channel = PQI_MAX_BUS;
+	shost->max_cmd_len = MAX_COMMAND_SIZE;
+	shost->max_lun = ~0;
+	shost->max_id = ~0;
+	shost->max_sectors = ctrl_info->max_sectors;
+	shost->can_queue = ctrl_info->scsi_ml_can_queue;
+	shost->cmd_per_lun = shost->can_queue;
+	shost->sg_tablesize = ctrl_info->sg_tablesize;
+	shost->transportt = pqi_sas_transport_template;
+	shost->irq = ctrl_info->msix_vectors[0];
+	shost->unique_id = shost->irq;
+	shost->nr_hw_queues = ctrl_info->num_queue_groups;
+	shost->hostdata[0] = (unsigned long)ctrl_info;
+
+	rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"scsi_add_host failed for controller %u\n",
+			ctrl_info->ctrl_id);
+		goto free_host;
+	}
+
+	rc = pqi_add_sas_host(shost, ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"add SAS host failed for controller %u\n",
+			ctrl_info->ctrl_id);
+		goto remove_host;
+	}
+
+	ctrl_info->scsi_host = shost;
+
+	return 0;
+
+remove_host:
+	scsi_remove_host(shost);
+free_host:
+	scsi_host_put(shost);
+
+	return rc;
+}
+
+static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)
+{
+	struct Scsi_Host *shost;
+
+	pqi_delete_sas_host(ctrl_info);
+
+	shost = ctrl_info->scsi_host;
+	if (!shost)
+		return;
+
+	scsi_remove_host(shost);
+	scsi_host_put(shost);
+}
+
+#define PQI_RESET_ACTION_RESET		0x1
+
+#define PQI_RESET_TYPE_NO_RESET		0x0
+#define PQI_RESET_TYPE_SOFT_RESET	0x1
+#define PQI_RESET_TYPE_FIRM_RESET	0x2
+#define PQI_RESET_TYPE_HARD_RESET	0x3
+
+static int pqi_reset(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	u32 reset_params;
+
+	reset_params = (PQI_RESET_ACTION_RESET << 5) |
+		PQI_RESET_TYPE_HARD_RESET;
+
+	writel(reset_params,
+		&ctrl_info->pqi_registers->device_reset);
+
+	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
+	if (rc)
+		dev_err(&ctrl_info->pci_dev->dev,
+			"PQI reset failed\n");
+
+	return rc;
+}
+
+static int pqi_get_ctrl_firmware_version(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	struct bmic_identify_controller *identify;
+
+	identify = kmalloc(sizeof(*identify), GFP_KERNEL);
+	if (!identify)
+		return -ENOMEM;
+
+	rc = pqi_identify_controller(ctrl_info, identify);
+	if (rc)
+		goto out;
+
+	memcpy(ctrl_info->firmware_version, identify->firmware_version,
+		sizeof(identify->firmware_version));
+	ctrl_info->firmware_version[sizeof(identify->firmware_version)] = '\0';
+	snprintf(ctrl_info->firmware_version +
+		strlen(ctrl_info->firmware_version),
+		sizeof(ctrl_info->firmware_version),
+		"-%u", get_unaligned_le16(&identify->firmware_build_number));
+
+out:
+	kfree(identify);
+
+	return rc;
+}
+
+static int pqi_kdump_init(struct pqi_ctrl_info *ctrl_info)
+{
+	if (!sis_is_firmware_running(ctrl_info))
+		return -ENXIO;
+
+	if (pqi_get_ctrl_mode(ctrl_info) == PQI_MODE) {
+		sis_disable_msix(ctrl_info);
+		if (pqi_reset(ctrl_info) == 0)
+			sis_reenable_sis_mode(ctrl_info);
+	}
+
+	return 0;
+}
+
+static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+
+	if (reset_devices) {
+		rc = pqi_kdump_init(ctrl_info);
+		if (rc)
+			return rc;
+	}
+
+	/*
+	 * When the controller comes out of reset, it is always running
+	 * in legacy SIS mode.  This is so that it can be compatible
+	 * with legacy drivers shipped with OSes.  So we have to talk
+	 * to it using SIS commands at first.  Once we are satisified
+	 * that the controller supports PQI, we transition it into PQI
+	 * mode.
+	 */
+
+	/*
+	 * Wait until the controller is ready to start accepting SIS
+	 * commands.
+	 */
+	rc = sis_wait_for_ctrl_ready(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error initializing SIS interface\n");
+		return rc;
+	}
+
+	/*
+	 * Get the controller properties.  This allows us to determine
+	 * whether or not it supports PQI mode.
+	 */
+	rc = sis_get_ctrl_properties(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error obtaining controller properties\n");
+		return rc;
+	}
+
+	rc = sis_get_pqi_capabilities(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error obtaining controller capabilities\n");
+		return rc;
+	}
+
+	if (ctrl_info->max_outstanding_requests > PQI_MAX_OUTSTANDING_REQUESTS)
+		ctrl_info->max_outstanding_requests =
+			PQI_MAX_OUTSTANDING_REQUESTS;
+
+	pqi_calculate_io_resources(ctrl_info);
+
+	rc = pqi_alloc_error_buffer(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"failed to allocate PQI error buffer\n");
+		return rc;
+	}
+
+	/*
+	 * If the function we are about to call succeeds, the
+	 * controller will transition from legacy SIS mode
+	 * into PQI mode.
+	 */
+	rc = sis_init_base_struct_addr(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error initializing PQI mode\n");
+		return rc;
+	}
+
+	/* Wait for the controller to complete the SIS -> PQI transition. */
+	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"transition to PQI mode failed\n");
+		return rc;
+	}
+
+	/* From here on, we are running in PQI mode. */
+	ctrl_info->pqi_mode_enabled = true;
+	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
+
+	rc = pqi_alloc_admin_queues(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error allocating admin queues\n");
+		return rc;
+	}
+
+	rc = pqi_create_admin_queues(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error creating admin queues\n");
+		return rc;
+	}
+
+	rc = pqi_report_device_capability(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"obtaining device capability failed\n");
+		return rc;
+	}
+
+	rc = pqi_validate_device_capability(ctrl_info);
+	if (rc)
+		return rc;
+
+	pqi_calculate_queue_resources(ctrl_info);
+
+	rc = pqi_enable_msix_interrupts(ctrl_info);
+	if (rc)
+		return rc;
+
+	if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {
+		ctrl_info->max_msix_vectors =
+			ctrl_info->num_msix_vectors_enabled;
+		pqi_calculate_queue_resources(ctrl_info);
+	}
+
+	rc = pqi_alloc_io_resources(ctrl_info);
+	if (rc)
+		return rc;
+
+	rc = pqi_alloc_operational_queues(ctrl_info);
+	if (rc)
+		return rc;
+
+	pqi_init_operational_queues(ctrl_info);
+
+	rc = pqi_request_irqs(ctrl_info);
+	if (rc)
+		return rc;
+
+	pqi_irq_set_affinity_hint(ctrl_info);
+
+	rc = pqi_create_queues(ctrl_info);
+	if (rc)
+		return rc;
+
+	sis_enable_msix(ctrl_info);
+
+	rc = pqi_configure_events(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error configuring events\n");
+		return rc;
+	}
+
+	pqi_start_heartbeat_timer(ctrl_info);
+
+	ctrl_info->controller_online = true;
+
+	/* Register with the SCSI subsystem. */
+	rc = pqi_register_scsi(ctrl_info);
+	if (rc)
+		return rc;
+
+	rc = pqi_get_ctrl_firmware_version(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error obtaining firmware version\n");
+		return rc;
+	}
+
+	rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"error updating host wellness\n");
+		return rc;
+	}
+
+	pqi_schedule_update_time_worker(ctrl_info);
+
+	pqi_scan_scsi_devices(ctrl_info);
+
+	return 0;
+}
+
+static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	u64 mask;
+
+	rc = pci_enable_device(ctrl_info->pci_dev);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"failed to enable PCI device\n");
+		return rc;
+	}
+
+	if (sizeof(dma_addr_t) > 4)
+		mask = DMA_BIT_MASK(64);
+	else
+		mask = DMA_BIT_MASK(32);
+
+	rc = dma_set_mask(&ctrl_info->pci_dev->dev, mask);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");
+		goto disable_device;
+	}
+
+	rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"failed to obtain PCI resources\n");
+		goto disable_device;
+	}
+
+	ctrl_info->iomem_base = ioremap_nocache(pci_resource_start(
+		ctrl_info->pci_dev, 0),
+		sizeof(struct pqi_ctrl_registers));
+	if (!ctrl_info->iomem_base) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"failed to map memory for controller registers\n");
+		rc = -ENOMEM;
+		goto release_regions;
+	}
+
+	ctrl_info->registers = ctrl_info->iomem_base;
+	ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;
+
+	/* Enable bus mastering. */
+	pci_set_master(ctrl_info->pci_dev);
+
+	pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);
+
+	return 0;
+
+release_regions:
+	pci_release_regions(ctrl_info->pci_dev);
+disable_device:
+	pci_disable_device(ctrl_info->pci_dev);
+
+	return rc;
+}
+
+static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)
+{
+	iounmap(ctrl_info->iomem_base);
+	pci_release_regions(ctrl_info->pci_dev);
+	pci_disable_device(ctrl_info->pci_dev);
+	pci_set_drvdata(ctrl_info->pci_dev, NULL);
+}
+
+static struct pqi_ctrl_info *pqi_alloc_ctrl_info(int numa_node)
+{
+	struct pqi_ctrl_info *ctrl_info;
+
+	ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),
+			GFP_KERNEL, numa_node);
+	if (!ctrl_info)
+		return NULL;
+
+	mutex_init(&ctrl_info->scan_mutex);
+
+	INIT_LIST_HEAD(&ctrl_info->scsi_device_list);
+	spin_lock_init(&ctrl_info->scsi_device_list_lock);
+
+	INIT_WORK(&ctrl_info->event_work, pqi_event_worker);
+	atomic_set(&ctrl_info->num_interrupts, 0);
+
+	INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);
+	INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);
+
+	sema_init(&ctrl_info->sync_request_sem,
+		PQI_RESERVED_IO_SLOTS_SYNCHRONOUS_REQUESTS);
+	sema_init(&ctrl_info->lun_reset_sem, PQI_RESERVED_IO_SLOTS_LUN_RESET);
+
+	ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;
+	ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;
+
+	return ctrl_info;
+}
+
+static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)
+{
+	kfree(ctrl_info);
+}
+
+static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)
+{
+	pqi_irq_unset_affinity_hint(ctrl_info);
+	pqi_free_irqs(ctrl_info);
+	if (ctrl_info->num_msix_vectors_enabled)
+		pci_disable_msix(ctrl_info->pci_dev);
+}
+
+static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)
+{
+	pqi_stop_heartbeat_timer(ctrl_info);
+	pqi_free_interrupts(ctrl_info);
+	if (ctrl_info->queue_memory_base)
+		dma_free_coherent(&ctrl_info->pci_dev->dev,
+			ctrl_info->queue_memory_length,
+			ctrl_info->queue_memory_base,
+			ctrl_info->queue_memory_base_dma_handle);
+	if (ctrl_info->admin_queue_memory_base)
+		dma_free_coherent(&ctrl_info->pci_dev->dev,
+			ctrl_info->admin_queue_memory_length,
+			ctrl_info->admin_queue_memory_base,
+			ctrl_info->admin_queue_memory_base_dma_handle);
+	pqi_free_all_io_requests(ctrl_info);
+	if (ctrl_info->error_buffer)
+		dma_free_coherent(&ctrl_info->pci_dev->dev,
+			ctrl_info->error_buffer_length,
+			ctrl_info->error_buffer,
+			ctrl_info->error_buffer_dma_handle);
+	if (ctrl_info->iomem_base)
+		pqi_cleanup_pci_init(ctrl_info);
+	pqi_free_ctrl_info(ctrl_info);
+}
+
+static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)
+{
+	cancel_delayed_work_sync(&ctrl_info->rescan_work);
+	cancel_delayed_work_sync(&ctrl_info->update_time_work);
+	pqi_remove_all_scsi_devices(ctrl_info);
+	pqi_unregister_scsi(ctrl_info);
+
+	if (ctrl_info->pqi_mode_enabled) {
+		sis_disable_msix(ctrl_info);
+		if (pqi_reset(ctrl_info) == 0)
+			sis_reenable_sis_mode(ctrl_info);
+	}
+	pqi_free_ctrl_resources(ctrl_info);
+}
+
+static void pqi_print_ctrl_info(struct pci_dev *pdev,
+	const struct pci_device_id *id)
+{
+	char *ctrl_description;
+
+	if (id->driver_data) {
+		ctrl_description = (char *)id->driver_data;
+	} else {
+		switch (id->subvendor) {
+		case PCI_VENDOR_ID_HP:
+			ctrl_description = hpe_branded_controller;
+			break;
+		case PCI_VENDOR_ID_ADAPTEC2:
+		default:
+			ctrl_description = microsemi_branded_controller;
+			break;
+		}
+	}
+
+	dev_info(&pdev->dev, "%s found\n", ctrl_description);
+}
+
+static int pqi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	int rc;
+	int node;
+	struct pqi_ctrl_info *ctrl_info;
+
+	pqi_print_ctrl_info(pdev, id);
+
+	if (pqi_disable_device_id_wildcards &&
+		id->subvendor == PCI_ANY_ID &&
+		id->subdevice == PCI_ANY_ID) {
+		dev_warn(&pdev->dev,
+			"controller not probed because device ID wildcards are disabled\n");
+		return -ENODEV;
+	}
+
+	if (id->subvendor == PCI_ANY_ID || id->subdevice == PCI_ANY_ID)
+		dev_warn(&pdev->dev,
+			"controller device ID matched using wildcards\n");
+
+	node = dev_to_node(&pdev->dev);
+	if (node == NUMA_NO_NODE)
+		set_dev_node(&pdev->dev, 0);
+
+	ctrl_info = pqi_alloc_ctrl_info(node);
+	if (!ctrl_info) {
+		dev_err(&pdev->dev,
+			"failed to allocate controller info block\n");
+		return -ENOMEM;
+	}
+
+	ctrl_info->pci_dev = pdev;
+
+	rc = pqi_pci_init(ctrl_info);
+	if (rc)
+		goto error;
+
+	rc = pqi_ctrl_init(ctrl_info);
+	if (rc)
+		goto error;
+
+	return 0;
+
+error:
+	pqi_remove_ctrl(ctrl_info);
+
+	return rc;
+}
+
+static void pqi_pci_remove(struct pci_dev *pdev)
+{
+	struct pqi_ctrl_info *ctrl_info;
+
+	ctrl_info = pci_get_drvdata(pdev);
+	if (!ctrl_info)
+		return;
+
+	pqi_remove_ctrl(ctrl_info);
+}
+
+static void pqi_shutdown(struct pci_dev *pdev)
+{
+	int rc;
+	struct pqi_ctrl_info *ctrl_info;
+
+	ctrl_info = pci_get_drvdata(pdev);
+	if (!ctrl_info)
+		goto error;
+
+	/*
+	 * Write all data in the controller's battery-backed cache to
+	 * storage.
+	 */
+	rc = pqi_flush_cache(ctrl_info);
+	if (rc == 0)
+		return;
+
+error:
+	dev_warn(&pdev->dev,
+		"unable to flush controller cache\n");
+}
+
+/* Define the PCI IDs for the controllers that we support. */
+static const struct pci_device_id pqi_pci_id_table[] = {
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0110)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0600)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0601)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0602)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0603)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0650)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0651)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0652)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0653)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0654)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0655)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0700)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x0701)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0800)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0801)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0802)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0803)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0804)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0805)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0900)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0901)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0902)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0903)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0904)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0905)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0906)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x1001)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x1100)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x1101)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x1102)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_HP, 0x1150)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_ANY_ID, PCI_ANY_ID)
+	},
+	{ 0 }
+};
+
+MODULE_DEVICE_TABLE(pci, pqi_pci_id_table);
+
+static struct pci_driver pqi_pci_driver = {
+	.name = DRIVER_NAME_SHORT,
+	.id_table = pqi_pci_id_table,
+	.probe = pqi_pci_probe,
+	.remove = pqi_pci_remove,
+	.shutdown = pqi_shutdown,
+};
+
+static int __init pqi_init(void)
+{
+	int rc;
+
+	pr_info(DRIVER_NAME "\n");
+
+	pqi_sas_transport_template =
+		sas_attach_transport(&pqi_sas_transport_functions);
+	if (!pqi_sas_transport_template)
+		return -ENODEV;
+
+	rc = pci_register_driver(&pqi_pci_driver);
+	if (rc)
+		sas_release_transport(pqi_sas_transport_template);
+
+	return rc;
+}
+
+static void __exit pqi_cleanup(void)
+{
+	pci_unregister_driver(&pqi_pci_driver);
+	sas_release_transport(pqi_sas_transport_template);
+}
+
+module_init(pqi_init);
+module_exit(pqi_cleanup);
+
+static void __attribute__((unused)) verify_structures(void)
+{
+	BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
+		sis_host_to_ctrl_doorbell) != 0x20);
+	BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
+		sis_interrupt_mask) != 0x34);
+	BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
+		sis_ctrl_to_host_doorbell) != 0x9c);
+	BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
+		sis_ctrl_to_host_doorbell_clear) != 0xa0);
+	BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
+		sis_driver_scratch) != 0xb0);
+	BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
+		sis_firmware_status) != 0xbc);
+	BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
+		sis_mailbox) != 0x1000);
+	BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
+		pqi_registers) != 0x4000);
+
+	BUILD_BUG_ON(offsetof(struct pqi_iu_header,
+		iu_type) != 0x0);
+	BUILD_BUG_ON(offsetof(struct pqi_iu_header,
+		iu_length) != 0x2);
+	BUILD_BUG_ON(offsetof(struct pqi_iu_header,
+		response_queue_id) != 0x4);
+	BUILD_BUG_ON(offsetof(struct pqi_iu_header,
+		work_area) != 0x6);
+	BUILD_BUG_ON(sizeof(struct pqi_iu_header) != 0x8);
+
+	BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
+		status) != 0x0);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
+		service_response) != 0x1);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
+		data_present) != 0x2);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
+		reserved) != 0x3);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
+		residual_count) != 0x4);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
+		data_length) != 0x8);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
+		reserved1) != 0xa);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
+		data) != 0xc);
+	BUILD_BUG_ON(sizeof(struct pqi_aio_error_info) != 0x10c);
+
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		data_in_result) != 0x0);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		data_out_result) != 0x1);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		reserved) != 0x2);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		status) != 0x5);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		status_qualifier) != 0x6);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		sense_data_length) != 0x8);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		response_data_length) != 0xa);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		data_in_transferred) != 0xc);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		data_out_transferred) != 0x10);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
+		data) != 0x14);
+	BUILD_BUG_ON(sizeof(struct pqi_raid_error_info) != 0x114);
+
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		signature) != 0x0);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		function_and_status_code) != 0x8);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		max_admin_iq_elements) != 0x10);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		max_admin_oq_elements) != 0x11);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_iq_element_length) != 0x12);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_oq_element_length) != 0x13);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		max_reset_timeout) != 0x14);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		legacy_intx_status) != 0x18);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		legacy_intx_mask_set) != 0x1c);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		legacy_intx_mask_clear) != 0x20);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		device_status) != 0x40);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_iq_pi_offset) != 0x48);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_oq_ci_offset) != 0x50);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_iq_element_array_addr) != 0x58);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_oq_element_array_addr) != 0x60);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_iq_ci_addr) != 0x68);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_oq_pi_addr) != 0x70);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_iq_num_elements) != 0x78);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_oq_num_elements) != 0x79);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		admin_queue_int_msg_num) != 0x7a);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		device_error) != 0x80);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		error_details) != 0x88);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		device_reset) != 0x90);
+	BUILD_BUG_ON(offsetof(struct pqi_device_registers,
+		power_action) != 0x94);
+	BUILD_BUG_ON(sizeof(struct pqi_device_registers) != 0x100);
+
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		header.work_area) != 6);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		request_id) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		function_code) != 10);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.report_device_capability.buffer_length) != 44);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.report_device_capability.sg_descriptor) != 48);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_iq.queue_id) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_iq.element_array_addr) != 16);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_iq.ci_addr) != 24);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_iq.num_elements) != 32);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_iq.element_length) != 34);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_iq.queue_protocol) != 36);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.queue_id) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.element_array_addr) != 16);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.pi_addr) != 24);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.num_elements) != 32);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.element_length) != 34);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.queue_protocol) != 36);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.int_msg_num) != 40);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.coalescing_count) != 42);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.min_coalescing_time) != 44);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.create_operational_oq.max_coalescing_time) != 48);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
+		data.delete_operational_queue.queue_id) != 12);
+	BUILD_BUG_ON(sizeof(struct pqi_general_admin_request) != 64);
+	BUILD_BUG_ON(FIELD_SIZEOF(struct pqi_general_admin_request,
+		data.create_operational_iq) != 64 - 11);
+	BUILD_BUG_ON(FIELD_SIZEOF(struct pqi_general_admin_request,
+		data.create_operational_oq) != 64 - 11);
+	BUILD_BUG_ON(FIELD_SIZEOF(struct pqi_general_admin_request,
+		data.delete_operational_queue) != 64 - 11);
+
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		header.work_area) != 6);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		request_id) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		function_code) != 10);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		status) != 11);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		data.create_operational_iq.status_descriptor) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		data.create_operational_iq.iq_pi_offset) != 16);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		data.create_operational_oq.status_descriptor) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
+		data.create_operational_oq.oq_ci_offset) != 16);
+	BUILD_BUG_ON(sizeof(struct pqi_general_admin_response) != 64);
+
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		header.response_queue_id) != 4);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		header.work_area) != 6);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		request_id) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		nexus_id) != 10);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		buffer_length) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		lun_number) != 16);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		protocol_specific) != 24);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		error_index) != 27);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		cdb) != 32);
+	BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
+		sg_descriptors) != 64);
+	BUILD_BUG_ON(sizeof(struct pqi_raid_path_request) !=
+		PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
+
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		header.response_queue_id) != 4);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		header.work_area) != 6);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		request_id) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		nexus_id) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		buffer_length) != 16);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		data_encryption_key_index) != 22);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		encrypt_tweak_lower) != 24);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		encrypt_tweak_upper) != 28);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		cdb) != 32);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		error_index) != 48);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		num_sg_descriptors) != 50);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		cdb_length) != 51);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		lun_number) != 52);
+	BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
+		sg_descriptors) != 64);
+	BUILD_BUG_ON(sizeof(struct pqi_aio_path_request) !=
+		PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
+
+	BUILD_BUG_ON(offsetof(struct pqi_io_response,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_io_response,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_io_response,
+		request_id) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_io_response,
+		error_index) != 10);
+
+	BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
+		header.response_queue_id) != 4);
+	BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
+		request_id) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
+		data.report_event_configuration.buffer_length) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
+		data.report_event_configuration.sg_descriptors) != 16);
+	BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
+		data.set_event_configuration.global_event_oq_id) != 10);
+	BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
+		data.set_event_configuration.buffer_length) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
+		data.set_event_configuration.sg_descriptors) != 16);
+
+	BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
+		max_inbound_iu_length) != 6);
+	BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
+		max_outbound_iu_length) != 14);
+	BUILD_BUG_ON(sizeof(struct pqi_iu_layer_descriptor) != 16);
+
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		data_length) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		iq_arbitration_priority_support_bitmask) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		maximum_aw_a) != 9);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		maximum_aw_b) != 10);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		maximum_aw_c) != 11);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		max_inbound_queues) != 16);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		max_elements_per_iq) != 18);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		max_iq_element_length) != 24);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		min_iq_element_length) != 26);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		max_outbound_queues) != 30);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		max_elements_per_oq) != 32);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		intr_coalescing_time_granularity) != 34);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		max_oq_element_length) != 36);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		min_oq_element_length) != 38);
+	BUILD_BUG_ON(offsetof(struct pqi_device_capability,
+		iu_layer_descriptors) != 64);
+	BUILD_BUG_ON(sizeof(struct pqi_device_capability) != 576);
+
+	BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
+		event_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
+		oq_id) != 2);
+	BUILD_BUG_ON(sizeof(struct pqi_event_descriptor) != 4);
+
+	BUILD_BUG_ON(offsetof(struct pqi_event_config,
+		num_event_descriptors) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_event_config,
+		descriptors) != 4);
+
+	BUILD_BUG_ON(offsetof(struct pqi_event_response,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_event_response,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_event_response,
+		event_type) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_event_response,
+		event_id) != 10);
+	BUILD_BUG_ON(offsetof(struct pqi_event_response,
+		additional_event_id) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_event_response,
+		data) != 16);
+	BUILD_BUG_ON(sizeof(struct pqi_event_response) != 32);
+
+	BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
+		event_type) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
+		event_id) != 10);
+	BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
+		additional_event_id) != 12);
+	BUILD_BUG_ON(sizeof(struct pqi_event_acknowledge_request) != 16);
+
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
+		request_id) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
+		nexus_id) != 10);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
+		lun_number) != 16);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
+		protocol_specific) != 24);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
+		outbound_queue_id_to_manage) != 26);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
+		request_id_to_manage) != 28);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
+		task_management_function) != 30);
+	BUILD_BUG_ON(sizeof(struct pqi_task_management_request) != 32);
+
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
+		header.iu_type) != 0);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
+		header.iu_length) != 2);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
+		request_id) != 8);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
+		nexus_id) != 10);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
+		additional_response_info) != 12);
+	BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
+		response_code) != 15);
+	BUILD_BUG_ON(sizeof(struct pqi_task_management_response) != 16);
+
+	BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
+		configured_logical_drive_count) != 0);
+	BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
+		configuration_signature) != 1);
+	BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
+		firmware_version) != 5);
+	BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
+		extended_logical_unit_count) != 154);
+	BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
+		firmware_build_number) != 190);
+	BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
+		controller_mode) != 292);
+
+	BUILD_BUG_ON(PQI_ADMIN_IQ_NUM_ELEMENTS > 255);
+	BUILD_BUG_ON(PQI_ADMIN_OQ_NUM_ELEMENTS > 255);
+	BUILD_BUG_ON(PQI_ADMIN_IQ_ELEMENT_LENGTH %
+		PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
+	BUILD_BUG_ON(PQI_ADMIN_OQ_ELEMENT_LENGTH %
+		PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
+	BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH > 1048560);
+	BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH %
+		PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
+	BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH > 1048560);
+	BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH %
+		PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
+
+	BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >= PQI_MAX_OUTSTANDING_REQUESTS);
+}
diff --git a/drivers/scsi/smartpqi/smartpqi_sas_transport.c b/drivers/scsi/smartpqi/smartpqi_sas_transport.c
new file mode 100644
index 000000000000..52ca4f93f1b2
--- /dev/null
+++ b/drivers/scsi/smartpqi/smartpqi_sas_transport.c
@@ -0,0 +1,350 @@
+/*
+ *    driver for Microsemi PQI-based storage controllers
+ *    Copyright (c) 2016 Microsemi Corporation
+ *    Copyright (c) 2016 PMC-Sierra, Inc.
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; version 2 of the License.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *    NON INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *
+ */
+
+#include <linux/kernel.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_transport_sas.h>
+#include "smartpqi.h"
+
+static struct pqi_sas_phy *pqi_alloc_sas_phy(struct pqi_sas_port *pqi_sas_port)
+{
+	struct pqi_sas_phy *pqi_sas_phy;
+	struct sas_phy *phy;
+
+	pqi_sas_phy = kzalloc(sizeof(*pqi_sas_phy), GFP_KERNEL);
+	if (!pqi_sas_phy)
+		return NULL;
+
+	phy = sas_phy_alloc(pqi_sas_port->parent_node->parent_dev,
+		pqi_sas_port->next_phy_index);
+	if (!phy) {
+		kfree(pqi_sas_phy);
+		return NULL;
+	}
+
+	pqi_sas_port->next_phy_index++;
+	pqi_sas_phy->phy = phy;
+	pqi_sas_phy->parent_port = pqi_sas_port;
+
+	return pqi_sas_phy;
+}
+
+static void pqi_free_sas_phy(struct pqi_sas_phy *pqi_sas_phy)
+{
+	struct sas_phy *phy = pqi_sas_phy->phy;
+
+	sas_port_delete_phy(pqi_sas_phy->parent_port->port, phy);
+	sas_phy_free(phy);
+	if (pqi_sas_phy->added_to_port)
+		list_del(&pqi_sas_phy->phy_list_entry);
+	kfree(pqi_sas_phy);
+}
+
+static int pqi_sas_port_add_phy(struct pqi_sas_phy *pqi_sas_phy)
+{
+	int rc;
+	struct pqi_sas_port *pqi_sas_port;
+	struct sas_phy *phy;
+	struct sas_identify *identify;
+
+	pqi_sas_port = pqi_sas_phy->parent_port;
+	phy = pqi_sas_phy->phy;
+
+	identify = &phy->identify;
+	memset(identify, 0, sizeof(*identify));
+	identify->sas_address = pqi_sas_port->sas_address;
+	identify->device_type = SAS_END_DEVICE;
+	identify->initiator_port_protocols = SAS_PROTOCOL_STP;
+	identify->target_port_protocols = SAS_PROTOCOL_STP;
+	phy->minimum_linkrate_hw = SAS_LINK_RATE_UNKNOWN;
+	phy->maximum_linkrate_hw = SAS_LINK_RATE_UNKNOWN;
+	phy->minimum_linkrate = SAS_LINK_RATE_UNKNOWN;
+	phy->maximum_linkrate = SAS_LINK_RATE_UNKNOWN;
+	phy->negotiated_linkrate = SAS_LINK_RATE_UNKNOWN;
+
+	rc = sas_phy_add(pqi_sas_phy->phy);
+	if (rc)
+		return rc;
+
+	sas_port_add_phy(pqi_sas_port->port, pqi_sas_phy->phy);
+	list_add_tail(&pqi_sas_phy->phy_list_entry,
+		&pqi_sas_port->phy_list_head);
+	pqi_sas_phy->added_to_port = true;
+
+	return 0;
+}
+
+static int pqi_sas_port_add_rphy(struct pqi_sas_port *pqi_sas_port,
+	struct sas_rphy *rphy)
+{
+	struct sas_identify *identify;
+
+	identify = &rphy->identify;
+	identify->sas_address = pqi_sas_port->sas_address;
+	identify->initiator_port_protocols = SAS_PROTOCOL_STP;
+	identify->target_port_protocols = SAS_PROTOCOL_STP;
+
+	return sas_rphy_add(rphy);
+}
+
+static struct pqi_sas_port *pqi_alloc_sas_port(
+	struct pqi_sas_node *pqi_sas_node, u64 sas_address)
+{
+	int rc;
+	struct pqi_sas_port *pqi_sas_port;
+	struct sas_port *port;
+
+	pqi_sas_port = kzalloc(sizeof(*pqi_sas_port), GFP_KERNEL);
+	if (!pqi_sas_port)
+		return NULL;
+
+	INIT_LIST_HEAD(&pqi_sas_port->phy_list_head);
+	pqi_sas_port->parent_node = pqi_sas_node;
+
+	port = sas_port_alloc_num(pqi_sas_node->parent_dev);
+	if (!port)
+		goto free_pqi_port;
+
+	rc = sas_port_add(port);
+	if (rc)
+		goto free_sas_port;
+
+	pqi_sas_port->port = port;
+	pqi_sas_port->sas_address = sas_address;
+	list_add_tail(&pqi_sas_port->port_list_entry,
+		&pqi_sas_node->port_list_head);
+
+	return pqi_sas_port;
+
+free_sas_port:
+	sas_port_free(port);
+free_pqi_port:
+	kfree(pqi_sas_port);
+
+	return NULL;
+}
+
+static void pqi_free_sas_port(struct pqi_sas_port *pqi_sas_port)
+{
+	struct pqi_sas_phy *pqi_sas_phy;
+	struct pqi_sas_phy *next;
+
+	list_for_each_entry_safe(pqi_sas_phy, next,
+			&pqi_sas_port->phy_list_head, phy_list_entry)
+		pqi_free_sas_phy(pqi_sas_phy);
+
+	sas_port_delete(pqi_sas_port->port);
+	list_del(&pqi_sas_port->port_list_entry);
+	kfree(pqi_sas_port);
+}
+
+static struct pqi_sas_node *pqi_alloc_sas_node(struct device *parent_dev)
+{
+	struct pqi_sas_node *pqi_sas_node;
+
+	pqi_sas_node = kzalloc(sizeof(*pqi_sas_node), GFP_KERNEL);
+	if (pqi_sas_node) {
+		pqi_sas_node->parent_dev = parent_dev;
+		INIT_LIST_HEAD(&pqi_sas_node->port_list_head);
+	}
+
+	return pqi_sas_node;
+}
+
+static void pqi_free_sas_node(struct pqi_sas_node *pqi_sas_node)
+{
+	struct pqi_sas_port *pqi_sas_port;
+	struct pqi_sas_port *next;
+
+	if (!pqi_sas_node)
+		return;
+
+	list_for_each_entry_safe(pqi_sas_port, next,
+			&pqi_sas_node->port_list_head, port_list_entry)
+		pqi_free_sas_port(pqi_sas_port);
+
+	kfree(pqi_sas_node);
+}
+
+struct pqi_scsi_dev *pqi_find_device_by_sas_rphy(
+	struct pqi_ctrl_info *ctrl_info, struct sas_rphy *rphy)
+{
+	struct pqi_scsi_dev *device;
+
+	list_for_each_entry(device, &ctrl_info->scsi_device_list,
+		scsi_device_list_entry) {
+		if (!device->sas_port)
+			continue;
+		if (device->sas_port->rphy == rphy)
+			return device;
+	}
+
+	return NULL;
+}
+
+int pqi_add_sas_host(struct Scsi_Host *shost, struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	struct device *parent_dev;
+	struct pqi_sas_node *pqi_sas_node;
+	struct pqi_sas_port *pqi_sas_port;
+	struct pqi_sas_phy *pqi_sas_phy;
+
+	parent_dev = &shost->shost_gendev;
+
+	pqi_sas_node = pqi_alloc_sas_node(parent_dev);
+	if (!pqi_sas_node)
+		return -ENOMEM;
+
+	pqi_sas_port = pqi_alloc_sas_port(pqi_sas_node, ctrl_info->sas_address);
+	if (!pqi_sas_port) {
+		rc = -ENODEV;
+		goto free_sas_node;
+	}
+
+	pqi_sas_phy = pqi_alloc_sas_phy(pqi_sas_port);
+	if (!pqi_sas_phy) {
+		rc = -ENODEV;
+		goto free_sas_port;
+	}
+
+	rc = pqi_sas_port_add_phy(pqi_sas_phy);
+	if (rc)
+		goto free_sas_phy;
+
+	ctrl_info->sas_host = pqi_sas_node;
+
+	return 0;
+
+free_sas_phy:
+	pqi_free_sas_phy(pqi_sas_phy);
+free_sas_port:
+	pqi_free_sas_port(pqi_sas_port);
+free_sas_node:
+	pqi_free_sas_node(pqi_sas_node);
+
+	return rc;
+}
+
+void pqi_delete_sas_host(struct pqi_ctrl_info *ctrl_info)
+{
+	pqi_free_sas_node(ctrl_info->sas_host);
+}
+
+int pqi_add_sas_device(struct pqi_sas_node *pqi_sas_node,
+	struct pqi_scsi_dev *device)
+{
+	int rc;
+	struct pqi_sas_port *pqi_sas_port;
+	struct sas_rphy *rphy;
+
+	pqi_sas_port = pqi_alloc_sas_port(pqi_sas_node, device->sas_address);
+	if (!pqi_sas_port)
+		return -ENOMEM;
+
+	rphy = sas_end_device_alloc(pqi_sas_port->port);
+	if (!rphy) {
+		rc = -ENODEV;
+		goto free_sas_port;
+	}
+
+	pqi_sas_port->rphy = rphy;
+	device->sas_port = pqi_sas_port;
+
+	rc = pqi_sas_port_add_rphy(pqi_sas_port, rphy);
+	if (rc)
+		goto free_sas_port;
+
+	return 0;
+
+free_sas_port:
+	pqi_free_sas_port(pqi_sas_port);
+	device->sas_port = NULL;
+
+	return rc;
+}
+
+void pqi_remove_sas_device(struct pqi_scsi_dev *device)
+{
+	if (device->sas_port) {
+		pqi_free_sas_port(device->sas_port);
+		device->sas_port = NULL;
+	}
+}
+
+static int pqi_sas_get_linkerrors(struct sas_phy *phy)
+{
+	return 0;
+}
+
+static int pqi_sas_get_enclosure_identifier(struct sas_rphy *rphy,
+	u64 *identifier)
+{
+	return 0;
+}
+
+static int pqi_sas_get_bay_identifier(struct sas_rphy *rphy)
+{
+	return -ENXIO;
+}
+
+static int pqi_sas_phy_reset(struct sas_phy *phy, int hard_reset)
+{
+	return 0;
+}
+
+static int pqi_sas_phy_enable(struct sas_phy *phy, int enable)
+{
+	return 0;
+}
+
+static int pqi_sas_phy_setup(struct sas_phy *phy)
+{
+	return 0;
+}
+
+static void pqi_sas_phy_release(struct sas_phy *phy)
+{
+}
+
+static int pqi_sas_phy_speed(struct sas_phy *phy,
+	struct sas_phy_linkrates *rates)
+{
+	return -EINVAL;
+}
+
+/* SMP = Serial Management Protocol */
+
+static int pqi_sas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
+	struct request *req)
+{
+	return -EINVAL;
+}
+
+struct sas_function_template pqi_sas_transport_functions = {
+	.get_linkerrors = pqi_sas_get_linkerrors,
+	.get_enclosure_identifier = pqi_sas_get_enclosure_identifier,
+	.get_bay_identifier = pqi_sas_get_bay_identifier,
+	.phy_reset = pqi_sas_phy_reset,
+	.phy_enable = pqi_sas_phy_enable,
+	.phy_setup = pqi_sas_phy_setup,
+	.phy_release = pqi_sas_phy_release,
+	.set_phy_speed = pqi_sas_phy_speed,
+	.smp_handler = pqi_sas_smp_handler,
+};
diff --git a/drivers/scsi/smartpqi/smartpqi_sis.c b/drivers/scsi/smartpqi/smartpqi_sis.c
new file mode 100644
index 000000000000..71408f9e8f75
--- /dev/null
+++ b/drivers/scsi/smartpqi/smartpqi_sis.c
@@ -0,0 +1,404 @@
+/*
+ *    driver for Microsemi PQI-based storage controllers
+ *    Copyright (c) 2016 Microsemi Corporation
+ *    Copyright (c) 2016 PMC-Sierra, Inc.
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; version 2 of the License.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *    NON INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/pci.h>
+#include <scsi/scsi_device.h>
+#include <asm/unaligned.h>
+#include "smartpqi.h"
+#include "smartpqi_sis.h"
+
+/* legacy SIS interface commands */
+#define SIS_CMD_GET_ADAPTER_PROPERTIES		0x19
+#define SIS_CMD_INIT_BASE_STRUCT_ADDRESS	0x1b
+#define SIS_CMD_GET_PQI_CAPABILITIES		0x3000
+
+/* for submission of legacy SIS commands */
+#define SIS_REENABLE_SIS_MODE			0x1
+#define SIS_ENABLE_MSIX				0x40
+#define SIS_SOFT_RESET				0x100
+#define SIS_CMD_READY				0x200
+#define SIS_CMD_COMPLETE			0x1000
+#define SIS_CLEAR_CTRL_TO_HOST_DOORBELL		0x1000
+#define SIS_CMD_STATUS_SUCCESS			0x1
+#define SIS_CMD_COMPLETE_TIMEOUT_SECS		30
+#define SIS_CMD_COMPLETE_POLL_INTERVAL_MSECS	10
+
+/* used with SIS_CMD_GET_ADAPTER_PROPERTIES command */
+#define SIS_EXTENDED_PROPERTIES_SUPPORTED	0x800000
+#define SIS_SMARTARRAY_FEATURES_SUPPORTED	0x2
+#define SIS_PQI_MODE_SUPPORTED			0x4
+#define SIS_REQUIRED_EXTENDED_PROPERTIES	\
+	(SIS_SMARTARRAY_FEATURES_SUPPORTED | SIS_PQI_MODE_SUPPORTED)
+
+/* used with SIS_CMD_INIT_BASE_STRUCT_ADDRESS command */
+#define SIS_BASE_STRUCT_REVISION		9
+#define SIS_BASE_STRUCT_ALIGNMENT		16
+
+#define SIS_CTRL_KERNEL_UP			0x80
+#define SIS_CTRL_KERNEL_PANIC			0x100
+#define SIS_CTRL_READY_TIMEOUT_SECS		30
+#define SIS_CTRL_READY_POLL_INTERVAL_MSECS	10
+
+#pragma pack(1)
+
+/* for use with SIS_CMD_INIT_BASE_STRUCT_ADDRESS command */
+struct sis_base_struct {
+	__le32	revision;		/* revision of this structure */
+	__le32	flags;			/* reserved */
+	__le32	error_buffer_paddr_low;	/* lower 32 bits of physical memory */
+					/* buffer for PQI error response */
+					/* data */
+	__le32	error_buffer_paddr_high;	/* upper 32 bits of physical */
+						/* memory buffer for PQI */
+						/* error response data */
+	__le32	error_buffer_element_length;	/* length of each PQI error */
+						/* response buffer element */
+						/*   in bytes */
+	__le32	error_buffer_num_elements;	/* total number of PQI error */
+						/* response buffers available */
+};
+
+#pragma pack()
+
+int sis_wait_for_ctrl_ready(struct pqi_ctrl_info *ctrl_info)
+{
+	unsigned long timeout;
+	u32 status;
+
+	timeout = (SIS_CTRL_READY_TIMEOUT_SECS * HZ) + jiffies;
+
+	while (1) {
+		status = readl(&ctrl_info->registers->sis_firmware_status);
+		if (status != ~0) {
+			if (status & SIS_CTRL_KERNEL_PANIC) {
+				dev_err(&ctrl_info->pci_dev->dev,
+					"controller is offline: status code 0x%x\n",
+					readl(
+					&ctrl_info->registers->sis_mailbox[7]));
+				return -ENODEV;
+			}
+			if (status & SIS_CTRL_KERNEL_UP)
+				break;
+		}
+		if (time_after(jiffies, timeout))
+			return -ETIMEDOUT;
+		msleep(SIS_CTRL_READY_POLL_INTERVAL_MSECS);
+	}
+
+	return 0;
+}
+
+bool sis_is_firmware_running(struct pqi_ctrl_info *ctrl_info)
+{
+	bool running;
+	u32 status;
+
+	status = readl(&ctrl_info->registers->sis_firmware_status);
+
+	if (status & SIS_CTRL_KERNEL_PANIC)
+		running = false;
+	else
+		running = true;
+
+	if (!running)
+		dev_err(&ctrl_info->pci_dev->dev,
+			"controller is offline: status code 0x%x\n",
+			readl(&ctrl_info->registers->sis_mailbox[7]));
+
+	return running;
+}
+
+/* used for passing command parameters/results when issuing SIS commands */
+struct sis_sync_cmd_params {
+	u32	mailbox[6];	/* mailboxes 0-5 */
+};
+
+static int sis_send_sync_cmd(struct pqi_ctrl_info *ctrl_info,
+	u32 cmd, struct sis_sync_cmd_params *params)
+{
+	struct pqi_ctrl_registers __iomem *registers;
+	unsigned int i;
+	unsigned long timeout;
+	u32 doorbell;
+	u32 cmd_status;
+
+	registers = ctrl_info->registers;
+
+	/* Write the command to mailbox 0. */
+	writel(cmd, &registers->sis_mailbox[0]);
+
+	/*
+	 * Write the command parameters to mailboxes 1-4 (mailbox 5 is not used
+	 * when sending a command to the controller).
+	 */
+	for (i = 1; i <= 4; i++)
+		writel(params->mailbox[i], &registers->sis_mailbox[i]);
+
+	/* Clear the command doorbell. */
+	writel(SIS_CLEAR_CTRL_TO_HOST_DOORBELL,
+		&registers->sis_ctrl_to_host_doorbell_clear);
+
+	/* Disable doorbell interrupts by masking all interrupts. */
+	writel(~0, &registers->sis_interrupt_mask);
+
+	/*
+	 * Force the completion of the interrupt mask register write before
+	 * submitting the command.
+	 */
+	readl(&registers->sis_interrupt_mask);
+
+	/* Submit the command to the controller. */
+	writel(SIS_CMD_READY, &registers->sis_host_to_ctrl_doorbell);
+
+	/*
+	 * Poll for command completion.  Note that the call to msleep() is at
+	 * the top of the loop in order to give the controller time to start
+	 * processing the command before we start polling.
+	 */
+	timeout = (SIS_CMD_COMPLETE_TIMEOUT_SECS * HZ) + jiffies;
+	while (1) {
+		msleep(SIS_CMD_COMPLETE_POLL_INTERVAL_MSECS);
+		doorbell = readl(&registers->sis_ctrl_to_host_doorbell);
+		if (doorbell & SIS_CMD_COMPLETE)
+			break;
+		if (time_after(jiffies, timeout))
+			return -ETIMEDOUT;
+	}
+
+	/* Read the command status from mailbox 0. */
+	cmd_status = readl(&registers->sis_mailbox[0]);
+	if (cmd_status != SIS_CMD_STATUS_SUCCESS) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"SIS command failed for command 0x%x: status = 0x%x\n",
+			cmd, cmd_status);
+		return -EINVAL;
+	}
+
+	/*
+	 * The command completed successfully, so save the command status and
+	 * read the values returned in mailboxes 1-5.
+	 */
+	params->mailbox[0] = cmd_status;
+	for (i = 1; i < ARRAY_SIZE(params->mailbox); i++)
+		params->mailbox[i] = readl(&registers->sis_mailbox[i]);
+
+	return 0;
+}
+
+/*
+ * This function verifies that we are talking to a controller that speaks PQI.
+ */
+
+int sis_get_ctrl_properties(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	u32 properties;
+	u32 extended_properties;
+	struct sis_sync_cmd_params params;
+
+	memset(&params, 0, sizeof(params));
+
+	rc = sis_send_sync_cmd(ctrl_info, SIS_CMD_GET_ADAPTER_PROPERTIES,
+		&params);
+	if (rc)
+		return rc;
+
+	properties = params.mailbox[1];
+
+	if (!(properties & SIS_EXTENDED_PROPERTIES_SUPPORTED))
+		return -ENODEV;
+
+	extended_properties = params.mailbox[4];
+
+	if ((extended_properties & SIS_REQUIRED_EXTENDED_PROPERTIES) !=
+		SIS_REQUIRED_EXTENDED_PROPERTIES)
+		return -ENODEV;
+
+	return 0;
+}
+
+int sis_get_pqi_capabilities(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	struct sis_sync_cmd_params params;
+
+	memset(&params, 0, sizeof(params));
+
+	rc = sis_send_sync_cmd(ctrl_info, SIS_CMD_GET_PQI_CAPABILITIES,
+		&params);
+	if (rc)
+		return rc;
+
+	ctrl_info->max_sg_entries = params.mailbox[1];
+	ctrl_info->max_transfer_size = params.mailbox[2];
+	ctrl_info->max_outstanding_requests = params.mailbox[3];
+	ctrl_info->config_table_offset = params.mailbox[4];
+	ctrl_info->config_table_length = params.mailbox[5];
+
+	return 0;
+}
+
+int sis_init_base_struct_addr(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	void *base_struct_unaligned;
+	struct sis_base_struct *base_struct;
+	struct sis_sync_cmd_params params;
+	unsigned long error_buffer_paddr;
+	dma_addr_t bus_address;
+
+	base_struct_unaligned = kzalloc(sizeof(*base_struct)
+		+ SIS_BASE_STRUCT_ALIGNMENT - 1, GFP_KERNEL);
+	if (!base_struct_unaligned)
+		return -ENOMEM;
+
+	base_struct = PTR_ALIGN(base_struct_unaligned,
+		SIS_BASE_STRUCT_ALIGNMENT);
+	error_buffer_paddr = (unsigned long)ctrl_info->error_buffer_dma_handle;
+
+	put_unaligned_le32(SIS_BASE_STRUCT_REVISION, &base_struct->revision);
+	put_unaligned_le32(lower_32_bits(error_buffer_paddr),
+		&base_struct->error_buffer_paddr_low);
+	put_unaligned_le32(upper_32_bits(error_buffer_paddr),
+		&base_struct->error_buffer_paddr_high);
+	put_unaligned_le32(PQI_ERROR_BUFFER_ELEMENT_LENGTH,
+		&base_struct->error_buffer_element_length);
+	put_unaligned_le32(ctrl_info->max_io_slots,
+		&base_struct->error_buffer_num_elements);
+
+	bus_address = pci_map_single(ctrl_info->pci_dev, base_struct,
+		sizeof(*base_struct), PCI_DMA_TODEVICE);
+	if (pci_dma_mapping_error(ctrl_info->pci_dev, bus_address)) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	memset(&params, 0, sizeof(params));
+	params.mailbox[1] = lower_32_bits((u64)bus_address);
+	params.mailbox[2] = upper_32_bits((u64)bus_address);
+	params.mailbox[3] = sizeof(*base_struct);
+
+	rc = sis_send_sync_cmd(ctrl_info, SIS_CMD_INIT_BASE_STRUCT_ADDRESS,
+		&params);
+
+	pci_unmap_single(ctrl_info->pci_dev, bus_address, sizeof(*base_struct),
+		PCI_DMA_TODEVICE);
+
+out:
+	kfree(base_struct_unaligned);
+
+	return rc;
+}
+
+/* Enable MSI-X interrupts on the controller. */
+
+void sis_enable_msix(struct pqi_ctrl_info *ctrl_info)
+{
+	u32 doorbell_register;
+
+	doorbell_register =
+		readl(&ctrl_info->registers->sis_host_to_ctrl_doorbell);
+	doorbell_register |= SIS_ENABLE_MSIX;
+
+	writel(doorbell_register,
+		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
+}
+
+/* Disable MSI-X interrupts on the controller. */
+
+void sis_disable_msix(struct pqi_ctrl_info *ctrl_info)
+{
+	u32 doorbell_register;
+
+	doorbell_register =
+		readl(&ctrl_info->registers->sis_host_to_ctrl_doorbell);
+	doorbell_register &= ~SIS_ENABLE_MSIX;
+
+	writel(doorbell_register,
+		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
+}
+
+void sis_soft_reset(struct pqi_ctrl_info *ctrl_info)
+{
+	writel(SIS_SOFT_RESET,
+		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
+}
+
+#define SIS_MODE_READY_TIMEOUT_SECS	30
+
+int sis_reenable_sis_mode(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	unsigned long timeout;
+	struct pqi_ctrl_registers __iomem *registers;
+	u32 doorbell;
+
+	registers = ctrl_info->registers;
+
+	writel(SIS_REENABLE_SIS_MODE,
+		&registers->sis_host_to_ctrl_doorbell);
+
+	rc = 0;
+	timeout = (SIS_MODE_READY_TIMEOUT_SECS * HZ) + jiffies;
+
+	while (1) {
+		doorbell = readl(&registers->sis_ctrl_to_host_doorbell);
+		if ((doorbell & SIS_REENABLE_SIS_MODE) == 0)
+			break;
+		if (time_after(jiffies, timeout)) {
+			rc = -ETIMEDOUT;
+			break;
+		}
+	}
+
+	if (rc)
+		dev_err(&ctrl_info->pci_dev->dev,
+			"re-enabling SIS mode failed\n");
+
+	return rc;
+}
+
+void sis_write_driver_scratch(struct pqi_ctrl_info *ctrl_info, u32 value)
+{
+	writel(value, &ctrl_info->registers->sis_driver_scratch);
+}
+
+u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info)
+{
+	return readl(&ctrl_info->registers->sis_driver_scratch);
+}
+
+static void __attribute__((unused)) verify_structures(void)
+{
+	BUILD_BUG_ON(offsetof(struct sis_base_struct,
+		revision) != 0x0);
+	BUILD_BUG_ON(offsetof(struct sis_base_struct,
+		flags) != 0x4);
+	BUILD_BUG_ON(offsetof(struct sis_base_struct,
+		error_buffer_paddr_low) != 0x8);
+	BUILD_BUG_ON(offsetof(struct sis_base_struct,
+		error_buffer_paddr_high) != 0xc);
+	BUILD_BUG_ON(offsetof(struct sis_base_struct,
+		error_buffer_element_length) != 0x10);
+	BUILD_BUG_ON(offsetof(struct sis_base_struct,
+		error_buffer_num_elements) != 0x14);
+	BUILD_BUG_ON(sizeof(struct sis_base_struct) != 0x18);
+}
diff --git a/drivers/scsi/smartpqi/smartpqi_sis.h b/drivers/scsi/smartpqi/smartpqi_sis.h
new file mode 100644
index 000000000000..bd6e7b08338e
--- /dev/null
+++ b/drivers/scsi/smartpqi/smartpqi_sis.h
@@ -0,0 +1,34 @@
+/*
+ *    driver for Microsemi PQI-based storage controllers
+ *    Copyright (c) 2016 Microsemi Corporation
+ *    Copyright (c) 2016 PMC-Sierra, Inc.
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; version 2 of the License.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *    NON INFRINGEMENT.  See the GNU General Public License for more details.
+ *
+ *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *
+ */
+
+#if !defined(_SMARTPQI_SIS_H)
+#define _SMARTPQI_SIS_H
+
+int sis_wait_for_ctrl_ready(struct pqi_ctrl_info *ctrl_info);
+bool sis_is_firmware_running(struct pqi_ctrl_info *ctrl_info);
+int sis_get_ctrl_properties(struct pqi_ctrl_info *ctrl_info);
+int sis_get_pqi_capabilities(struct pqi_ctrl_info *ctrl_info);
+int sis_init_base_struct_addr(struct pqi_ctrl_info *ctrl_info);
+void sis_enable_msix(struct pqi_ctrl_info *ctrl_info);
+void sis_disable_msix(struct pqi_ctrl_info *ctrl_info);
+void sis_soft_reset(struct pqi_ctrl_info *ctrl_info);
+int sis_reenable_sis_mode(struct pqi_ctrl_info *ctrl_info);
+void sis_write_driver_scratch(struct pqi_ctrl_info *ctrl_info, u32 value);
+u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info);
+
+#endif	/* _SMARTPQI_SIS_H */
diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c
index ed179348de80..bed2bbd6b923 100644
--- a/drivers/scsi/sr.c
+++ b/drivers/scsi/sr.c
@@ -83,7 +83,7 @@ static int sr_init_command(struct scsi_cmnd *SCpnt);
 static int sr_done(struct scsi_cmnd *);
 static int sr_runtime_suspend(struct device *dev);
 
-static struct dev_pm_ops sr_pm_ops = {
+static const struct dev_pm_ops sr_pm_ops = {
 	.runtime_suspend	= sr_runtime_suspend,
 };
 
diff --git a/drivers/scsi/sym53c8xx_2/sym_glue.c b/drivers/scsi/sym53c8xx_2/sym_glue.c
index 5d00e514ff28..d32e3ba8863e 100644
--- a/drivers/scsi/sym53c8xx_2/sym_glue.c
+++ b/drivers/scsi/sym53c8xx_2/sym_glue.c
@@ -1874,7 +1874,7 @@ static void sym2_io_resume(struct pci_dev *pdev)
 
 	spin_lock_irq(shost->host_lock);
 	if (sym_data->io_reset)
-		complete_all(sym_data->io_reset);
+		complete(sym_data->io_reset);
 	spin_unlock_irq(shost->host_lock);
 }
 
diff --git a/drivers/scsi/t128.c b/drivers/scsi/t128.c
deleted file mode 100644
index 8a8608ac62e6..000000000000
--- a/drivers/scsi/t128.c
+++ /dev/null
@@ -1,407 +0,0 @@
-/*
- * Trantor T128/T128F/T228 driver
- *	Note : architecturally, the T100 and T130 are different and won't 
- * 	work
- *
- * Copyright 1993, Drew Eckhardt
- *	Visionary Computing
- *	(Unix and Linux consulting and custom programming)
- *	drew@colorado.edu
- *      +1 (303) 440-4894
- *
- * For more information, please consult 
- *
- * Trantor Systems, Ltd.
- * T128/T128F/T228 SCSI Host Adapter
- * Hardware Specifications
- * 
- * Trantor Systems, Ltd. 
- * 5415 Randall Place
- * Fremont, CA 94538
- * 1+ (415) 770-1400, FAX 1+ (415) 770-9910
- */
-
-/*
- * The card is detected and initialized in one of several ways : 
- * 1.  Autoprobe (default) - since the board is memory mapped, 
- *     a BIOS signature is scanned for to locate the registers.
- *     An interrupt is triggered to autoprobe for the interrupt
- *     line.
- *
- * 2.  With command line overrides - t128=address,irq may be 
- *     used on the LILO command line to override the defaults.
- *
- * 3.  With the T128_OVERRIDE compile time define.  This is 
- *     specified as an array of address, irq tuples.  Ie, for
- *     one board at the default 0xcc000 address, IRQ5, I could say 
- *     -DT128_OVERRIDE={{0xcc000, 5}}
- *	
- *     Note that if the override methods are used, place holders must
- *     be specified for other boards in the system.
- * 
- * T128/T128F jumper/dipswitch settings (note : on my sample, the switches 
- * were epoxy'd shut, meaning I couldn't change the 0xcc000 base address) :
- *
- * T128    Sw7 Sw8 Sw6 = 0ws Sw5 = boot 
- * T128F   Sw6 Sw7 Sw5 = 0ws Sw4 = boot Sw8 = floppy disable
- * cc000   off off      
- * c8000   off on
- * dc000   on  off
- * d8000   on  on
- *
- * 
- * Interrupts 
- * There is a 12 pin jumper block, jp1, numbered as follows : 
- *   T128 (JP1)  	 T128F (J5)
- * 2 4 6 8 10 12	11 9  7 5 3 1
- * 1 3 5 7 9  11	12 10 8 6 4 2
- *
- * 3   2-4
- * 5   1-3
- * 7   3-5
- * T128F only 
- * 10 8-10
- * 12 7-9
- * 14 10-12
- * 15 9-11
- */
- 
-#include <linux/io.h>
-#include <linux/blkdev.h>
-#include <linux/interrupt.h>
-#include <linux/init.h>
-#include <linux/module.h>
-
-#include <scsi/scsi_host.h>
-#include "t128.h"
-#include "NCR5380.h"
-
-static struct override {
-    unsigned long address;
-    int irq;
-} overrides
-#ifdef T128_OVERRIDE
-    [] __initdata = T128_OVERRIDE;
-#else
-    [4] __initdata = {{0, IRQ_AUTO}, {0, IRQ_AUTO},
-        {0 ,IRQ_AUTO}, {0, IRQ_AUTO}};
-#endif
-
-#define NO_OVERRIDES ARRAY_SIZE(overrides)
-
-static struct base {
-    unsigned int address;
-    int noauto;
-} bases[] __initdata = {
-    { 0xcc000, 0}, { 0xc8000, 0}, { 0xdc000, 0}, { 0xd8000, 0}
-};
-
-#define NO_BASES ARRAY_SIZE(bases)
-
-static struct signature {
-    const char *string;
-    int offset;
-} signatures[] __initdata = {
-{"TSROM: SCSI BIOS, Version 1.12", 0x36},
-};
-
-#define NO_SIGNATURES ARRAY_SIZE(signatures)
-
-#ifndef MODULE
-/*
- * Function : t128_setup(char *str, int *ints)
- *
- * Purpose : LILO command line initialization of the overrides array,
- * 
- * Inputs : str - unused, ints - array of integer parameters with ints[0]
- *	equal to the number of ints.
- *
- */
-
-static int __init t128_setup(char *str)
-{
-	static int commandline_current;
-    int i;
-    int ints[10];
-
-    get_options(str, ARRAY_SIZE(ints), ints);
-    if (ints[0] != 2) 
-	printk("t128_setup : usage t128=address,irq\n");
-    else 
-	if (commandline_current < NO_OVERRIDES) {
-	    overrides[commandline_current].address = ints[1];
-	    overrides[commandline_current].irq = ints[2];
-	    for (i = 0; i < NO_BASES; ++i)
-		if (bases[i].address == ints[1]) {
-		    bases[i].noauto = 1;
-		    break;
-		}
-	    ++commandline_current;
-	}
-    return 1;
-}
-
-__setup("t128=", t128_setup);
-#endif
-
-/* 
- * Function : int t128_detect(struct scsi_host_template * tpnt)
- *
- * Purpose : detects and initializes T128,T128F, or T228 controllers
- *	that were autoprobed, overridden on the LILO command line, 
- *	or specified at compile time.
- *
- * Inputs : tpnt - template for this SCSI adapter.
- * 
- * Returns : 1 if a host adapter was found, 0 if not.
- *
- */
-
-static int __init t128_detect(struct scsi_host_template *tpnt)
-{
-	static int current_override, current_base;
-    struct Scsi_Host *instance;
-    unsigned long base;
-    void __iomem *p;
-    int sig, count;
-
-    for (count = 0; current_override < NO_OVERRIDES; ++current_override) {
-	base = 0;
-	p = NULL;
-
-	if (overrides[current_override].address) {
-	    base = overrides[current_override].address;
-	    p = ioremap(bases[current_base].address, 0x2000);
-	    if (!p)
-		base = 0;
-	} else 
-	    for (; !base && (current_base < NO_BASES); ++current_base) {
-		dprintk(NDEBUG_INIT, "t128: probing address 0x%08x\n",
-		        bases[current_base].address);
-		if (bases[current_base].noauto)
-			continue;
-		p = ioremap(bases[current_base].address, 0x2000);
-		if (!p)
-			continue;
-		for (sig = 0; sig < NO_SIGNATURES; ++sig) 
-		    if (check_signature(p + signatures[sig].offset,
-					signatures[sig].string,
-					strlen(signatures[sig].string))) {
-			base = bases[current_base].address;
-			dprintk(NDEBUG_INIT, "t128: detected board\n");
-			goto found;
-		    }
-		iounmap(p);
-	    }
-
-	dprintk(NDEBUG_INIT, "t128: base = 0x%08x\n", (unsigned int)base);
-
-	if (!base)
-	    break;
-
-found:
-	instance = scsi_register (tpnt, sizeof(struct NCR5380_hostdata));
-	if(instance == NULL)
-		goto out_unmap;
-
-	instance->base = base;
-	((struct NCR5380_hostdata *)instance->hostdata)->base = p;
-
-	if (NCR5380_init(instance, FLAG_DMA_FIXUP | FLAG_LATE_DMA_SETUP))
-		goto out_unregister;
-
-	NCR5380_maybe_reset_bus(instance);
-
-	if (overrides[current_override].irq != IRQ_AUTO)
-	    instance->irq = overrides[current_override].irq;
-	else 
-	    instance->irq = NCR5380_probe_irq(instance, T128_IRQS);
-
-	/* Compatibility with documented NCR5380 kernel parameters */
-	if (instance->irq == 255)
-		instance->irq = NO_IRQ;
-
-	if (instance->irq != NO_IRQ)
-	    if (request_irq(instance->irq, t128_intr, 0, "t128",
-			    instance)) {
-		printk("scsi%d : IRQ%d not free, interrupts disabled\n", 
-		    instance->host_no, instance->irq);
-		instance->irq = NO_IRQ;
-	    } 
-
-	if (instance->irq == NO_IRQ) {
-	    printk("scsi%d : interrupts not enabled. for better interactive performance,\n", instance->host_no);
-	    printk("scsi%d : please jumper the board for a free IRQ.\n", instance->host_no);
-	}
-
-	dprintk(NDEBUG_INIT, "scsi%d: irq = %d\n",
-	        instance->host_no, instance->irq);
-
-	++current_override;
-	++count;
-    }
-    return count;
-
-out_unregister:
-	scsi_unregister(instance);
-out_unmap:
-	iounmap(p);
-	return count;
-}
-
-static int t128_release(struct Scsi_Host *shost)
-{
-	struct NCR5380_hostdata *hostdata = shost_priv(shost);
-
-	if (shost->irq != NO_IRQ)
-		free_irq(shost->irq, shost);
-	NCR5380_exit(shost);
-	scsi_unregister(shost);
-	iounmap(hostdata->base);
-	return 0;
-}
-
-/*
- * Function : int t128_biosparam(Disk * disk, struct block_device *dev, int *ip)
- *
- * Purpose : Generates a BIOS / DOS compatible H-C-S mapping for 
- *	the specified device / size.
- * 
- * Inputs : size = size of device in sectors (512 bytes), dev = block device
- *	major / minor, ip[] = {heads, sectors, cylinders}  
- *
- * Returns : always 0 (success), initializes ip
- *	
- */
-
-/* 
- * XXX Most SCSI boards use this mapping, I could be incorrect.  Some one
- * using hard disks on a trantor should verify that this mapping corresponds
- * to that used by the BIOS / ASPI driver by running the linux fdisk program
- * and matching the H_C_S coordinates to what DOS uses.
- */
-
-static int t128_biosparam(struct scsi_device *sdev, struct block_device *bdev,
-                          sector_t capacity, int *ip)
-{
-  ip[0] = 64;
-  ip[1] = 32;
-  ip[2] = capacity >> 11;
-  return 0;
-}
-
-/*
- * Function : int t128_pread (struct Scsi_Host *instance,
- *	unsigned char *dst, int len)
- *
- * Purpose : Fast 5380 pseudo-dma read function, transfers len bytes to 
- *	dst
- * 
- * Inputs : dst = destination, len = length in bytes
- *
- * Returns : 0 on success, non zero on a failure such as a watchdog 
- * 	timeout.
- */
-
-static inline int t128_pread(struct Scsi_Host *instance,
-                             unsigned char *dst, int len)
-{
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-	void __iomem *reg, *base = hostdata->base;
-    unsigned char *d = dst;
-    register int i = len;
-
-    reg = base + T_DATA_REG_OFFSET;
-
-#if 0
-    for (; i; --i) {
-	while (!(readb(base+T_STATUS_REG_OFFSET) & T_ST_RDY)) barrier();
-#else
-    while (!(readb(base+T_STATUS_REG_OFFSET) & T_ST_RDY)) barrier();
-    for (; i; --i) {
-#endif
-	*d++ = readb(reg);
-    }
-
-    if (readb(base + T_STATUS_REG_OFFSET) & T_ST_TIM) {
-	unsigned char tmp;
-	void __iomem *foo = base + T_CONTROL_REG_OFFSET;
-	tmp = readb(foo);
-	writeb(tmp | T_CR_CT, foo);
-	writeb(tmp, foo);
-	printk("scsi%d : watchdog timer fired in NCR5380_pread()\n",
-	    instance->host_no);
-	return -1;
-    } else
-	return 0;
-}
-
-/*
- * Function : int t128_pwrite (struct Scsi_Host *instance,
- *	unsigned char *src, int len)
- *
- * Purpose : Fast 5380 pseudo-dma write function, transfers len bytes from
- *	src
- * 
- * Inputs : src = source, len = length in bytes
- *
- * Returns : 0 on success, non zero on a failure such as a watchdog 
- * 	timeout.
- */
-
-static inline int t128_pwrite(struct Scsi_Host *instance,
-                              unsigned char *src, int len)
-{
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-	void __iomem *reg, *base = hostdata->base;
-    unsigned char *s = src;
-    register int i = len;
-
-    reg = base + T_DATA_REG_OFFSET;
-
-#if 0
-    for (; i; --i) {
-	while (!(readb(base+T_STATUS_REG_OFFSET) & T_ST_RDY)) barrier();
-#else
-    while (!(readb(base+T_STATUS_REG_OFFSET) & T_ST_RDY)) barrier();
-    for (; i; --i) {
-#endif
-	writeb(*s++, reg);
-    }
-
-    if (readb(base + T_STATUS_REG_OFFSET) & T_ST_TIM) {
-	unsigned char tmp;
-	void __iomem *foo = base + T_CONTROL_REG_OFFSET;
-	tmp = readb(foo);
-	writeb(tmp | T_CR_CT, foo);
-	writeb(tmp, foo);
-	printk("scsi%d : watchdog timer fired in NCR5380_pwrite()\n",
-	    instance->host_no);
-	return -1;
-    } else 
-	return 0;
-}
-
-MODULE_LICENSE("GPL");
-
-#include "NCR5380.c"
-
-static struct scsi_host_template driver_template = {
-	.name			= "Trantor T128/T128F/T228",
-	.detect			= t128_detect,
-	.release		= t128_release,
-	.proc_name		= "t128",
-	.info			= t128_info,
-	.queuecommand		= t128_queue_command,
-	.eh_abort_handler	= t128_abort,
-	.eh_bus_reset_handler	= t128_bus_reset,
-	.bios_param		= t128_biosparam,
-	.can_queue		= 32,
-	.this_id		= 7,
-	.sg_tablesize		= SG_ALL,
-	.cmd_per_lun		= 2,
-	.use_clustering		= DISABLE_CLUSTERING,
-	.cmd_size		= NCR5380_CMD_SIZE,
-	.max_sectors		= 128,
-};
-#include "scsi_module.c"
diff --git a/drivers/scsi/t128.h b/drivers/scsi/t128.h
deleted file mode 100644
index c95bcd839109..000000000000
--- a/drivers/scsi/t128.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Trantor T128/T128F/T228 defines
- *	Note : architecturally, the T100 and T128 are different and won't work
- *
- * Copyright 1993, Drew Eckhardt
- *	Visionary Computing
- *	(Unix and Linux consulting and custom programming)
- *	drew@colorado.edu
- *      +1 (303) 440-4894
- *
- * For more information, please consult
- *
- * Trantor Systems, Ltd.
- * T128/T128F/T228 SCSI Host Adapter
- * Hardware Specifications
- *
- * Trantor Systems, Ltd.
- * 5415 Randall Place
- * Fremont, CA 94538
- * 1+ (415) 770-1400, FAX 1+ (415) 770-9910
- */
-
-#ifndef T128_H
-#define T128_H
-
-/*
- * The trantor boards are memory mapped. They use an NCR5380 or
- * equivalent (my sample board had part second sourced from ZILOG).
- * NCR's recommended "Pseudo-DMA" architecture is used, where
- * a PAL drives the DMA signals on the 5380 allowing fast, blind
- * transfers with proper handshaking.
- */
-
-/*
- * Note : a boot switch is provided for the purpose of informing the
- * firmware to boot or not boot from attached SCSI devices.  So, I imagine
- * there are fewer people who've yanked the ROM like they do on the Seagate
- * to make bootup faster, and I'll probably use this for autodetection.
- */
-#define T_ROM_OFFSET		0
-
-/*
- * Note : my sample board *WAS NOT* populated with the SRAM, so this
- * can't be used for autodetection without a ROM present.
- */
-#define T_RAM_OFFSET		0x1800
-
-/*
- * All of the registers are allocated 32 bytes of address space, except
- * for the data register (read/write to/from the 5380 in pseudo-DMA mode)
- */ 
-#define T_CONTROL_REG_OFFSET	0x1c00	/* rw */
-#define T_CR_INT		0x10	/* Enable interrupts */
-#define T_CR_CT			0x02	/* Reset watchdog timer */
-
-#define T_STATUS_REG_OFFSET	0x1c20	/* ro */
-#define T_ST_BOOT		0x80	/* Boot switch */
-#define T_ST_S3			0x40	/* User settable switches, */
-#define T_ST_S2			0x20	/* read 0 when switch is on, 1 off */
-#define T_ST_S1			0x10
-#define T_ST_PS2		0x08	/* Set for Microchannel 228 */
-#define T_ST_RDY		0x04	/* 5380 DRQ */
-#define T_ST_TIM		0x02	/* indicates 40us watchdog timer fired */
-#define T_ST_ZERO		0x01	/* Always zero */
-
-#define T_5380_OFFSET		0x1d00	/* 8 registers here, see NCR5380.h */
-
-#define T_DATA_REG_OFFSET	0x1e00	/* rw 512 bytes long */
-
-#define NCR5380_implementation_fields \
-    void __iomem *base
-
-#define T128_address(reg) \
-	(((struct NCR5380_hostdata *)shost_priv(instance))->base + T_5380_OFFSET + ((reg) * 0x20))
-
-#define NCR5380_read(reg) readb(T128_address(reg))
-#define NCR5380_write(reg, value) writeb((value),(T128_address(reg)))
-
-#define NCR5380_dma_xfer_len(instance, cmd, phase)	(cmd->transfersize)
-#define NCR5380_dma_recv_setup		t128_pread
-#define NCR5380_dma_send_setup		t128_pwrite
-#define NCR5380_dma_residual(instance)	(0)
-
-#define NCR5380_intr t128_intr
-#define NCR5380_queue_command t128_queue_command
-#define NCR5380_abort t128_abort
-#define NCR5380_bus_reset t128_bus_reset
-#define NCR5380_info t128_info
-
-#define NCR5380_io_delay(x)		udelay(x)
-
-/* 15 14 12 10 7 5 3
-   1101 0100 1010 1000 */
-
-#define T128_IRQS 0xc4a8
-
-#endif /* T128_H */
diff --git a/drivers/scsi/u14-34f.c b/drivers/scsi/u14-34f.c
deleted file mode 100644
index 14eb50b95a1e..000000000000
--- a/drivers/scsi/u14-34f.c
+++ /dev/null
@@ -1,1971 +0,0 @@
-/*
- *      u14-34f.c - Low-level driver for UltraStor 14F/34F SCSI host adapters.
- *
- *      03 Jun 2003 Rev. 8.10 for linux-2.5.70
- *        + Update for new IRQ API.
- *        + Use "goto" when appropriate.
- *        + Drop u14-34f.h.
- *        + Update for new module_param API.
- *        + Module parameters  can now be specified only in the
- *          same format as the kernel boot options.
- *
- *             boot option    old module param 
- *             -----------    ------------------
- *             addr,...       io_port=addr,...
- *             lc:[y|n]       linked_comm=[1|0]
- *             mq:xx          max_queue_depth=xx
- *             tm:[0|1|2]     tag_mode=[0|1|2]
- *             et:[y|n]       ext_tran=[1|0]
- *             of:[y|n]       have_old_firmware=[1|0]
- *
- *          A valid example using the new parameter format is:
- *          modprobe u14-34f "u14-34f=0x340,0x330,lc:y,tm:0,mq:4"
- *
- *          which is equivalent to the old format:
- *          modprobe u14-34f io_port=0x340,0x330 linked_comm=1 tag_mode=0 \
- *                        max_queue_depth=4
- *
- *          With actual module code, u14-34f and u14_34f are equivalent
- *          as module parameter names.
- *
- *      12 Feb 2003 Rev. 8.04 for linux 2.5.60
- *        + Release irq before calling scsi_register.
- *
- *      12 Nov 2002 Rev. 8.02 for linux 2.5.47
- *        + Release driver_lock before calling scsi_register.
- *
- *      11 Nov 2002 Rev. 8.01 for linux 2.5.47
- *        + Fixed bios_param and scsicam_bios_param calling parameters.
- *
- *      28 Oct 2002 Rev. 8.00 for linux 2.5.44-ac4
- *        + Use new tcq and adjust_queue_depth api.
- *        + New command line option (tm:[0-2]) to choose the type of tags:
- *          0 -> disable tagging ; 1 -> simple tags  ; 2 -> ordered tags.
- *          Default is tm:0 (tagged commands disabled).
- *          For compatibility the "tc:" option is an alias of the "tm:"
- *          option; tc:n is equivalent to tm:0 and tc:y is equivalent to
- *          tm:1.
- *
- *      10 Oct 2002 Rev. 7.70 for linux 2.5.42
- *        + Foreport from revision 6.70.
- *
- *      25 Jun 2002 Rev. 6.70 for linux 2.4.19
- *        + Fixed endian-ness problem due to bitfields.
- *
- *      21 Feb 2002 Rev. 6.52 for linux 2.4.18
- *        + Backport from rev. 7.22 (use io_request_lock).
- *
- *      20 Feb 2002 Rev. 7.22 for linux 2.5.5
- *        + Remove any reference to virt_to_bus().
- *        + Fix pio hang while detecting multiple HBAs.
- *
- *      01 Jan 2002 Rev. 7.20 for linux 2.5.1
- *        + Use the dynamic DMA mapping API.
- *
- *      19 Dec 2001 Rev. 7.02 for linux 2.5.1
- *        + Use SCpnt->sc_data_direction if set.
- *        + Use sglist.page instead of sglist.address.
- *
- *      11 Dec 2001 Rev. 7.00 for linux 2.5.1
- *        + Use host->host_lock instead of io_request_lock.
- *
- *       1 May 2001 Rev. 6.05 for linux 2.4.4
- *        + Fix data transfer direction for opcode SEND_CUE_SHEET (0x5d)
- *
- *      25 Jan 2001 Rev. 6.03 for linux 2.4.0
- *        + "check_region" call replaced by "request_region".
- *
- *      22 Nov 2000 Rev. 6.02 for linux 2.4.0-test11
- *        + Removed old scsi error handling support.
- *        + The obsolete boot option flag eh:n is silently ignored.
- *        + Removed error messages while a disk drive is powered up at
- *          boot time.
- *        + Improved boot messages: all tagged capable device are
- *          indicated as "tagged".
- *
- *      16 Sep 1999 Rev. 5.11 for linux 2.2.12 and 2.3.18
- *        + Updated to the new __setup interface for boot command line options.
- *        + When loaded as a module, accepts the new parameter boot_options
- *          which value is a string with the same format of the kernel boot
- *          command line options. A valid example is:
- *          modprobe u14-34f 'boot_options="0x230,0x340,lc:y,mq:4"'
- *
- *      22 Jul 1999 Rev. 5.00 for linux 2.2.10 and 2.3.11
- *        + Removed pre-2.2 source code compatibility.
- *
- *      26 Jul 1998 Rev. 4.33 for linux 2.0.35 and 2.1.111
- *          Added command line option (et:[y|n]) to use the existing
- *          translation (returned by scsicam_bios_param) as disk geometry.
- *          The default is et:n, which uses the disk geometry jumpered
- *          on the board.
- *          The default value et:n is compatible with all previous revisions
- *          of this driver.
- *
- *      28 May 1998 Rev. 4.32 for linux 2.0.33 and 2.1.104
- *          Increased busy timeout from 10 msec. to 200 msec. while
- *          processing interrupts.
- *
- *      18 May 1998 Rev. 4.31 for linux 2.0.33 and 2.1.102
- *          Improved abort handling during the eh recovery process.
- *
- *      13 May 1998 Rev. 4.30 for linux 2.0.33 and 2.1.101
- *          The driver is now fully SMP safe, including the
- *          abort and reset routines.
- *          Added command line options (eh:[y|n]) to choose between
- *          new_eh_code and the old scsi code.
- *          If linux version >= 2.1.101 the default is eh:y, while the eh
- *          option is ignored for previous releases and the old scsi code
- *          is used.
- *
- *      18 Apr 1998 Rev. 4.20 for linux 2.0.33 and 2.1.97
- *          Reworked interrupt handler.
- *
- *      11 Apr 1998 rev. 4.05 for linux 2.0.33 and 2.1.95
- *          Major reliability improvement: when a batch with overlapping
- *          requests is detected, requests are queued one at a time
- *          eliminating any possible board or drive reordering.
- *
- *      10 Apr 1998 rev. 4.04 for linux 2.0.33 and 2.1.95
- *          Improved SMP support (if linux version >= 2.1.95).
- *
- *       9 Apr 1998 rev. 4.03 for linux 2.0.33 and 2.1.94
- *          Performance improvement: when sequential i/o is detected,
- *          always use direct sort instead of reverse sort.
- *
- *       4 Apr 1998 rev. 4.02 for linux 2.0.33 and 2.1.92
- *          io_port is now unsigned long.
- *
- *      17 Mar 1998 rev. 4.01 for linux 2.0.33 and 2.1.88
- *          Use new scsi error handling code (if linux version >= 2.1.88).
- *          Use new interrupt code.
- *
- *      12 Sep 1997 rev. 3.11 for linux 2.0.30 and 2.1.55
- *          Use of udelay inside the wait loops to avoid timeout
- *          problems with fast cpus.
- *          Removed check about useless calls to the interrupt service
- *          routine (reported on SMP systems only).
- *          At initialization time "sorted/unsorted" is displayed instead
- *          of "linked/unlinked" to reinforce the fact that "linking" is
- *          nothing but "elevator sorting" in the actual implementation.
- *
- *      17 May 1997 rev. 3.10 for linux 2.0.30 and 2.1.38
- *          Use of serial_number_at_timeout in abort and reset processing.
- *          Use of the __initfunc and __initdata macro in setup code.
- *          Minor cleanups in the list_statistics code.
- *
- *      24 Feb 1997 rev. 3.00 for linux 2.0.29 and 2.1.26
- *          When loading as a module, parameter passing is now supported
- *          both in 2.0 and in 2.1 style.
- *          Fixed data transfer direction for some SCSI opcodes.
- *          Immediate acknowledge to request sense commands.
- *          Linked commands to each disk device are now reordered by elevator
- *          sorting. Rare cases in which reordering of write requests could
- *          cause wrong results are managed.
- *
- *      18 Jan 1997 rev. 2.60 for linux 2.1.21 and 2.0.28
- *          Added command line options to enable/disable linked commands
- *          (lc:[y|n]), old firmware support (of:[y|n]) and to set the max
- *          queue depth (mq:xx). Default is "u14-34f=lc:n,of:n,mq:8".
- *          Improved command linking.
- *
- *       8 Jan 1997 rev. 2.50 for linux 2.1.20 and 2.0.27
- *          Added linked command support.
- *
- *       3 Dec 1996 rev. 2.40 for linux 2.1.14 and 2.0.27
- *          Added queue depth adjustment.
- *
- *      22 Nov 1996 rev. 2.30 for linux 2.1.12 and 2.0.26
- *          The list of i/o ports to be probed can be overwritten by the
- *          "u14-34f=port0,port1,...." boot command line option.
- *          Scatter/gather lists are now allocated by a number of kmalloc
- *          calls, in order to avoid the previous size limit of 64Kb.
- *
- *      16 Nov 1996 rev. 2.20 for linux 2.1.10 and 2.0.25
- *          Added multichannel support.
- *
- *      27 Sep 1996 rev. 2.12 for linux 2.1.0
- *          Portability cleanups (virtual/bus addressing, little/big endian
- *          support).
- *
- *      09 Jul 1996 rev. 2.11 for linux 2.0.4
- *          "Data over/under-run" no longer implies a redo on all targets.
- *          Number of internal retries is now limited.
- *
- *      16 Apr 1996 rev. 2.10 for linux 1.3.90
- *          New argument "reset_flags" to the reset routine.
- *
- *      21 Jul 1995 rev. 2.02 for linux 1.3.11
- *          Fixed Data Transfer Direction for some SCSI commands.
- *
- *      13 Jun 1995 rev. 2.01 for linux 1.2.10
- *          HAVE_OLD_UX4F_FIRMWARE should be defined for U34F boards when
- *          the firmware prom is not the latest one (28008-006).
- *
- *      11 Mar 1995 rev. 2.00 for linux 1.2.0
- *          Fixed a bug which prevented media change detection for removable
- *          disk drives.
- *
- *      23 Feb 1995 rev. 1.18 for linux 1.1.94
- *          Added a check for scsi_register returning NULL.
- *
- *      11 Feb 1995 rev. 1.17 for linux 1.1.91
- *          U14F qualified to run with 32 sglists.
- *          Now DEBUG_RESET is disabled by default.
- *
- *       9 Feb 1995 rev. 1.16 for linux 1.1.90
- *          Use host->wish_block instead of host->block.
- *
- *       8 Feb 1995 rev. 1.15 for linux 1.1.89
- *          Cleared target_time_out counter while performing a reset.
- *
- *      28 Jan 1995 rev. 1.14 for linux 1.1.86
- *          Added module support.
- *          Log and do a retry when a disk drive returns a target status
- *          different from zero on a recovered error.
- *          Auto detects if U14F boards have an old firmware revision.
- *          Max number of scatter/gather lists set to 16 for all boards
- *          (most installation run fine using 33 sglists, while other
- *          has problems when using more than 16).
- *
- *      16 Jan 1995 rev. 1.13 for linux 1.1.81
- *          Display a message if check_region detects a port address
- *          already in use.
- *
- *      15 Dec 1994 rev. 1.12 for linux 1.1.74
- *          The host->block flag is set for all the detected ISA boards.
- *
- *      30 Nov 1994 rev. 1.11 for linux 1.1.68
- *          Redo i/o on target status CHECK_CONDITION for TYPE_DISK only.
- *          Added optional support for using a single board at a time.
- *
- *      14 Nov 1994 rev. 1.10 for linux 1.1.63
- *
- *      28 Oct 1994 rev. 1.09 for linux 1.1.58  Final BETA release.
- *      16 Jul 1994 rev. 1.00 for linux 1.1.29  Initial ALPHA release.
- *
- *          This driver is a total replacement of the original UltraStor
- *          scsi driver, but it supports ONLY the 14F and 34F boards.
- *          It can be configured in the same kernel in which the original
- *          ultrastor driver is configured to allow the original U24F
- *          support.
- *
- *          Multiple U14F and/or U34F host adapters are supported.
- *
- *  Copyright (C) 1994-2003 Dario Ballabio (ballabio_dario@emc.com)
- *
- *  Alternate email: dario.ballabio@inwind.it, dario.ballabio@tiscalinet.it
- *
- *  Redistribution and use in source and binary forms, with or without
- *  modification, are permitted provided that redistributions of source
- *  code retain the above copyright notice and this comment without
- *  modification.
- *
- *      WARNING: if your 14/34F board has an old firmware revision (see below)
- *               you must change "#undef" into "#define" in the following
- *               statement.
- */
-#undef HAVE_OLD_UX4F_FIRMWARE
-/*
- *  The UltraStor 14F, 24F, and 34F are a family of intelligent, high
- *  performance SCSI-2 host adapters.
- *  Here is the scoop on the various models:
- *
- *  14F - ISA first-party DMA HA with floppy support and WD1003 emulation.
- *  24F - EISA Bus Master HA with floppy support and WD1003 emulation.
- *  34F - VESA Local-Bus Bus Master HA (no WD1003 emulation).
- *
- *  This code has been tested with up to two U14F boards, using both
- *  firmware 28004-005/38004-004 (BIOS rev. 2.00) and the latest firmware
- *  28004-006/38004-005 (BIOS rev. 2.01).
- *
- *  The latest firmware is required in order to get reliable operations when
- *  clustering is enabled. ENABLE_CLUSTERING provides a performance increase
- *  up to 50% on sequential access.
- *
- *  Since the struct scsi_host_template structure is shared among all 14F and 34F,
- *  the last setting of use_clustering is in effect for all of these boards.
- *
- *  Here a sample configuration using two U14F boards:
- *
- U14F0: ISA 0x330, BIOS 0xc8000, IRQ 11, DMA 5, SG 32, MB 16, of:n, lc:y, mq:8.
- U14F1: ISA 0x340, BIOS 0x00000, IRQ 10, DMA 6, SG 32, MB 16, of:n, lc:y, mq:8.
- *
- *  The boot controller must have its BIOS enabled, while other boards can
- *  have their BIOS disabled, or enabled to an higher address.
- *  Boards are named Ux4F0, Ux4F1..., according to the port address order in
- *  the io_port[] array.
- *
- *  The following facts are based on real testing results (not on
- *  documentation) on the above U14F board.
- *
- *  - The U14F board should be jumpered for bus on time less or equal to 7
- *    microseconds, while the default is 11 microseconds. This is order to
- *    get acceptable performance while using floppy drive and hard disk
- *    together. The jumpering for 7 microseconds is: JP13 pin 15-16,
- *    JP14 pin 7-8 and pin 9-10.
- *    The reduction has a little impact on scsi performance.
- *
- *  - If scsi bus length exceeds 3m., the scsi bus speed needs to be reduced
- *    from 10Mhz to 5Mhz (do this by inserting a jumper on JP13 pin 7-8).
- *
- *  - If U14F on board firmware is older than 28004-006/38004-005,
- *    the U14F board is unable to provide reliable operations if the scsi
- *    request length exceeds 16Kbyte. When this length is exceeded the
- *    behavior is:
- *    - adapter_status equal 0x96 or 0xa3 or 0x93 or 0x94;
- *    - adapter_status equal 0 and target_status equal 2 on for all targets
- *      in the next operation following the reset.
- *    This sequence takes a long time (>3 seconds), so in the meantime
- *    the SD_TIMEOUT in sd.c could expire giving rise to scsi aborts
- *    (SD_TIMEOUT has been increased from 3 to 6 seconds in 1.1.31).
- *    Because of this I had to DISABLE_CLUSTERING and to work around the
- *    bus reset in the interrupt service routine, returning DID_BUS_BUSY
- *    so that the operations are retried without complains from the scsi.c
- *    code.
- *    Any reset of the scsi bus is going to kill tape operations, since
- *    no retry is allowed for tapes. Bus resets are more likely when the
- *    scsi bus is under heavy load.
- *    Requests using scatter/gather have a maximum length of 16 x 1024 bytes
- *    when DISABLE_CLUSTERING is in effect, but unscattered requests could be
- *    larger than 16Kbyte.
- *
- *    The new firmware has fixed all the above problems.
- *
- *  For U34F boards the latest bios prom is 38008-002 (BIOS rev. 2.01),
- *  the latest firmware prom is 28008-006. Older firmware 28008-005 has
- *  problems when using more than 16 scatter/gather lists.
- *
- *  The list of i/o ports to be probed can be totally replaced by the
- *  boot command line option: "u14-34f=port0,port1,port2,...", where the
- *  port0, port1... arguments are ISA/VESA addresses to be probed.
- *  For example using "u14-34f=0x230,0x340", the driver probes only the two
- *  addresses 0x230 and 0x340 in this order; "u14-34f=0" totally disables
- *  this driver.
- *
- *  After the optional list of detection probes, other possible command line
- *  options are:
- *
- *  et:y  use disk geometry returned by scsicam_bios_param;
- *  et:n  use disk geometry jumpered on the board;
- *  lc:y  enables linked commands;
- *  lc:n  disables linked commands;
- *  tm:0  disables tagged commands (same as tc:n);
- *  tm:1  use simple queue tags (same as tc:y);
- *  tm:2  use ordered queue tags (same as tc:2);
- *  of:y  enables old firmware support;
- *  of:n  disables old firmware support;
- *  mq:xx set the max queue depth to the value xx (2 <= xx <= 8).
- *
- *  The default value is: "u14-34f=lc:n,of:n,mq:8,tm:0,et:n".
- *  An example using the list of detection probes could be:
- *  "u14-34f=0x230,0x340,lc:y,tm:2,of:n,mq:4,et:n".
- *
- *  When loading as a module, parameters can be specified as well.
- *  The above example would be (use 1 in place of y and 0 in place of n):
- *
- *  modprobe u14-34f io_port=0x230,0x340 linked_comm=1 have_old_firmware=0 \
- *                max_queue_depth=4 ext_tran=0 tag_mode=2
- *
- *  ----------------------------------------------------------------------------
- *  In this implementation, linked commands are designed to work with any DISK
- *  or CD-ROM, since this linking has only the intent of clustering (time-wise)
- *  and reordering by elevator sorting commands directed to each device,
- *  without any relation with the actual SCSI protocol between the controller
- *  and the device.
- *  If Q is the queue depth reported at boot time for each device (also named
- *  cmds/lun) and Q > 2, whenever there is already an active command to the
- *  device all other commands to the same device  (up to Q-1) are kept waiting
- *  in the elevator sorting queue. When the active command completes, the
- *  commands in this queue are sorted by sector address. The sort is chosen
- *  between increasing or decreasing by minimizing the seek distance between
- *  the sector of the commands just completed and the sector of the first
- *  command in the list to be sorted.
- *  Trivial math assures that the unsorted average seek distance when doing
- *  random seeks over S sectors is S/3.
- *  When (Q-1) requests are uniformly distributed over S sectors, the average
- *  distance between two adjacent requests is S/((Q-1) + 1), so the sorted
- *  average seek distance for (Q-1) random requests over S sectors is S/Q.
- *  The elevator sorting hence divides the seek distance by a factor Q/3.
- *  The above pure geometric remarks are valid in all cases and the
- *  driver effectively reduces the seek distance by the predicted factor
- *  when there are Q concurrent read i/o operations on the device, but this
- *  does not necessarily results in a noticeable performance improvement:
- *  your mileage may vary....
- *
- *  Note: command reordering inside a batch of queued commands could cause
- *        wrong results only if there is at least one write request and the
- *        intersection (sector-wise) of all requests is not empty.
- *        When the driver detects a batch including overlapping requests
- *        (a really rare event) strict serial (pid) order is enforced.
- *  ----------------------------------------------------------------------------
- *
- *  The boards are named Ux4F0, Ux4F1,... according to the detection order.
- *
- *  In order to support multiple ISA boards in a reliable way,
- *  the driver sets host->wish_block = TRUE for all ISA boards.
- */
-
-#include <linux/string.h>
-#include <linux/kernel.h>
-#include <linux/ioport.h>
-#include <linux/delay.h>
-#include <asm/io.h>
-#include <asm/byteorder.h>
-#include <linux/proc_fs.h>
-#include <linux/blkdev.h>
-#include <linux/interrupt.h>
-#include <linux/stat.h>
-#include <linux/pci.h>
-#include <linux/init.h>
-#include <linux/ctype.h>
-#include <linux/spinlock.h>
-#include <linux/slab.h>
-#include <asm/dma.h>
-#include <asm/irq.h>
-
-#include <scsi/scsi.h>
-#include <scsi/scsi_cmnd.h>
-#include <scsi/scsi_device.h>
-#include <scsi/scsi_host.h>
-#include <scsi/scsi_tcq.h>
-#include <scsi/scsicam.h>
-
-static int u14_34f_detect(struct scsi_host_template *);
-static int u14_34f_release(struct Scsi_Host *);
-static int u14_34f_queuecommand(struct Scsi_Host *, struct scsi_cmnd *);
-static int u14_34f_eh_abort(struct scsi_cmnd *);
-static int u14_34f_eh_host_reset(struct scsi_cmnd *);
-static int u14_34f_bios_param(struct scsi_device *, struct block_device *,
-                              sector_t, int *);
-static int u14_34f_slave_configure(struct scsi_device *);
-
-static struct scsi_host_template driver_template = {
-                .name                    = "UltraStor 14F/34F rev. 8.10.00 ",
-                .detect                  = u14_34f_detect,
-                .release                 = u14_34f_release,
-                .queuecommand            = u14_34f_queuecommand,
-                .eh_abort_handler        = u14_34f_eh_abort,
-                .eh_host_reset_handler   = u14_34f_eh_host_reset,
-                .bios_param              = u14_34f_bios_param,
-                .slave_configure         = u14_34f_slave_configure,
-                .this_id                 = 7,
-                .unchecked_isa_dma       = 1,
-                .use_clustering          = ENABLE_CLUSTERING,
-                };
-
-#if !defined(__BIG_ENDIAN_BITFIELD) && !defined(__LITTLE_ENDIAN_BITFIELD)
-#error "Adjust your <asm/byteorder.h> defines"
-#endif
-
-/* Values for the PRODUCT_ID ports for the 14/34F */
-#define PRODUCT_ID1  0x56
-#define PRODUCT_ID2  0x40        /* NOTE: Only upper nibble is used */
-
-/* Subversion values */
-#define ISA  0
-#define ESA 1
-
-#define OP_HOST_ADAPTER   0x1
-#define OP_SCSI           0x2
-#define OP_RESET          0x4
-#define DTD_SCSI          0x0
-#define DTD_IN            0x1
-#define DTD_OUT           0x2
-#define DTD_NONE          0x3
-#define HA_CMD_INQUIRY    0x1
-#define HA_CMD_SELF_DIAG  0x2
-#define HA_CMD_READ_BUFF  0x3
-#define HA_CMD_WRITE_BUFF 0x4
-
-#undef  DEBUG_LINKED_COMMANDS
-#undef  DEBUG_DETECT
-#undef  DEBUG_INTERRUPT
-#undef  DEBUG_RESET
-#undef  DEBUG_GENERATE_ERRORS
-#undef  DEBUG_GENERATE_ABORTS
-#undef  DEBUG_GEOMETRY
-
-#define MAX_ISA 3
-#define MAX_VESA 1
-#define MAX_EISA 0
-#define MAX_PCI 0
-#define MAX_BOARDS (MAX_ISA + MAX_VESA + MAX_EISA + MAX_PCI)
-#define MAX_CHANNEL 1
-#define MAX_LUN 8
-#define MAX_TARGET 8
-#define MAX_MAILBOXES 16
-#define MAX_SGLIST 32
-#define MAX_SAFE_SGLIST 16
-#define MAX_INTERNAL_RETRIES 64
-#define MAX_CMD_PER_LUN 2
-#define MAX_TAGGED_CMD_PER_LUN (MAX_MAILBOXES - MAX_CMD_PER_LUN)
-
-#define SKIP ULONG_MAX
-#define FALSE 0
-#define TRUE 1
-#define FREE 0
-#define IN_USE   1
-#define LOCKED   2
-#define IN_RESET 3
-#define IGNORE   4
-#define READY    5
-#define ABORTING 6
-#define NO_DMA  0xff
-#define MAXLOOP  10000
-#define TAG_DISABLED 0
-#define TAG_SIMPLE   1
-#define TAG_ORDERED  2
-
-#define REG_LCL_MASK      0
-#define REG_LCL_INTR      1
-#define REG_SYS_MASK      2
-#define REG_SYS_INTR      3
-#define REG_PRODUCT_ID1   4
-#define REG_PRODUCT_ID2   5
-#define REG_CONFIG1       6
-#define REG_CONFIG2       7
-#define REG_OGM           8
-#define REG_ICM           12
-#define REGION_SIZE       13UL
-#define BSY_ASSERTED      0x01
-#define IRQ_ASSERTED      0x01
-#define CMD_RESET         0xc0
-#define CMD_OGM_INTR      0x01
-#define CMD_CLR_INTR      0x01
-#define CMD_ENA_INTR      0x81
-#define ASOK              0x00
-#define ASST              0x91
-
-#define YESNO(a) ((a) ? 'y' : 'n')
-#define TLDEV(type) ((type) == TYPE_DISK || (type) == TYPE_ROM)
-
-#define PACKED          __attribute__((packed))
-
-struct sg_list {
-   unsigned int address;                /* Segment Address */
-   unsigned int num_bytes;              /* Segment Length */
-   };
-
-/* MailBox SCSI Command Packet */
-struct mscp {
-
-#if defined(__BIG_ENDIAN_BITFIELD)
-   unsigned char sg:1, ca:1, dcn:1, xdir:2, opcode:3;
-   unsigned char lun: 3, channel:2, target:3;
-#else
-   unsigned char opcode: 3,             /* type of command */
-                 xdir: 2,               /* data transfer direction */
-                 dcn: 1,                /* disable disconnect */
-                 ca: 1,                 /* use cache (if available) */
-                 sg: 1;                 /* scatter/gather operation */
-   unsigned char target: 3,             /* SCSI target id */
-                 channel: 2,            /* SCSI channel number */
-                 lun: 3;                /* SCSI logical unit number */
-#endif
-
-   unsigned int data_address PACKED;    /* transfer data pointer */
-   unsigned int data_len PACKED;        /* length in bytes */
-   unsigned int link_address PACKED;    /* for linking command chains */
-   unsigned char clink_id;              /* identifies command in chain */
-   unsigned char use_sg;                /* (if sg is set) 8 bytes per list */
-   unsigned char sense_len;
-   unsigned char cdb_len;               /* 6, 10, or 12 */
-   unsigned char cdb[12];               /* SCSI Command Descriptor Block */
-   unsigned char adapter_status;        /* non-zero indicates HA error */
-   unsigned char target_status;         /* non-zero indicates target error */
-   unsigned int sense_addr PACKED;
-
-   /* Additional fields begin here. */
-   struct scsi_cmnd *SCpnt;
-   unsigned int cpp_index;              /* cp index */
-
-   /* All the cp structure is zero filled by queuecommand except the
-      following CP_TAIL_SIZE bytes, initialized by detect */
-   dma_addr_t cp_dma_addr; /* dma handle for this cp structure */
-   struct sg_list *sglist; /* pointer to the allocated SG list */
-   };
-
-#define CP_TAIL_SIZE (sizeof(struct sglist *) + sizeof(dma_addr_t))
-
-struct hostdata {
-   struct mscp cp[MAX_MAILBOXES];       /* Mailboxes for this board */
-   unsigned int cp_stat[MAX_MAILBOXES]; /* FREE, IN_USE, LOCKED, IN_RESET */
-   unsigned int last_cp_used;           /* Index of last mailbox used */
-   unsigned int iocount;                /* Total i/o done for this board */
-   int board_number;                    /* Number of this board */
-   char board_name[16];                 /* Name of this board */
-   int in_reset;                        /* True if board is doing a reset */
-   int target_to[MAX_TARGET][MAX_CHANNEL]; /* N. of timeout errors on target */
-   int target_redo[MAX_TARGET][MAX_CHANNEL]; /* If TRUE redo i/o on target */
-   unsigned int retries;                /* Number of internal retries */
-   unsigned long last_retried_pid;      /* Pid of last retried command */
-   unsigned char subversion;            /* Bus type, either ISA or ESA */
-   struct pci_dev *pdev;                /* Always NULL */
-   unsigned char heads;
-   unsigned char sectors;
-   char board_id[256];                  /* data from INQUIRY on this board */
-   };
-
-static struct Scsi_Host *sh[MAX_BOARDS + 1];
-static const char *driver_name = "Ux4F";
-static char sha[MAX_BOARDS];
-static DEFINE_SPINLOCK(driver_lock);
-
-/* Initialize num_boards so that ihdlr can work while detect is in progress */
-static unsigned int num_boards = MAX_BOARDS;
-
-static unsigned long io_port[] = {
-
-   /* Space for MAX_INT_PARAM ports usable while loading as a module */
-   SKIP,    SKIP,   SKIP,   SKIP,   SKIP,   SKIP,   SKIP,   SKIP,
-   SKIP,    SKIP,
-
-   /* Possible ISA/VESA ports */
-   0x330, 0x340, 0x230, 0x240, 0x210, 0x130, 0x140,
-
-   /* End of list */
-   0x0
-   };
-
-#define HD(board) ((struct hostdata *) &sh[board]->hostdata)
-#define BN(board) (HD(board)->board_name)
-
-/* Device is Little Endian */
-#define H2DEV(x) cpu_to_le32(x)
-#define DEV2H(x) le32_to_cpu(x)
-
-static irqreturn_t do_interrupt_handler(int, void *);
-static void flush_dev(struct scsi_device *, unsigned long, unsigned int, unsigned int);
-static int do_trace = FALSE;
-static int setup_done = FALSE;
-static int link_statistics;
-static int ext_tran = FALSE;
-
-#if defined(HAVE_OLD_UX4F_FIRMWARE)
-static int have_old_firmware = TRUE;
-#else
-static int have_old_firmware = FALSE;
-#endif
-
-#if defined(CONFIG_SCSI_U14_34F_TAGGED_QUEUE)
-static int tag_mode = TAG_SIMPLE;
-#else
-static int tag_mode = TAG_DISABLED;
-#endif
-
-#if defined(CONFIG_SCSI_U14_34F_LINKED_COMMANDS)
-static int linked_comm = TRUE;
-#else
-static int linked_comm = FALSE;
-#endif
-
-#if defined(CONFIG_SCSI_U14_34F_MAX_TAGS)
-static int max_queue_depth = CONFIG_SCSI_U14_34F_MAX_TAGS;
-#else
-static int max_queue_depth = MAX_CMD_PER_LUN;
-#endif
-
-#define MAX_INT_PARAM 10
-#define MAX_BOOT_OPTIONS_SIZE 256
-static char boot_options[MAX_BOOT_OPTIONS_SIZE];
-
-#if defined(MODULE)
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-
-module_param_string(u14_34f, boot_options, MAX_BOOT_OPTIONS_SIZE, 0);
-MODULE_PARM_DESC(u14_34f, " equivalent to the \"u14-34f=...\" kernel boot " \
-"option." \
-"      Example: modprobe u14-34f \"u14_34f=0x340,0x330,lc:y,tm:0,mq:4\"");
-MODULE_AUTHOR("Dario Ballabio");
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("UltraStor 14F/34F SCSI Driver");
-
-#endif
-
-static int u14_34f_slave_configure(struct scsi_device *dev) {
-   int j, tqd, utqd;
-   char *tag_suffix, *link_suffix;
-   struct Scsi_Host *host = dev->host;
-
-   j = ((struct hostdata *) host->hostdata)->board_number;
-
-   utqd = MAX_CMD_PER_LUN;
-   tqd = max_queue_depth;
-
-   if (TLDEV(dev->type) && dev->tagged_supported)
-
-      if (tag_mode == TAG_SIMPLE) {
-         scsi_change_queue_depth(dev, tqd);
-         tag_suffix = ", simple tags";
-         }
-      else if (tag_mode == TAG_ORDERED) {
-         scsi_change_queue_depth(dev, tqd);
-         tag_suffix = ", ordered tags";
-         }
-      else {
-         scsi_change_queue_depth(dev, tqd);
-         tag_suffix = ", no tags";
-         }
-
-   else if (TLDEV(dev->type) && linked_comm) {
-      scsi_change_queue_depth(dev, tqd);
-      tag_suffix = ", untagged";
-      }
-
-   else {
-      scsi_change_queue_depth(dev, utqd);
-      tag_suffix = "";
-      }
-
-   if (TLDEV(dev->type) && linked_comm && dev->queue_depth > 2)
-      link_suffix = ", sorted";
-   else if (TLDEV(dev->type))
-      link_suffix = ", unsorted";
-   else
-      link_suffix = "";
-
-   sdev_printk(KERN_INFO, dev, "cmds/lun %d%s%s.\n",
-          dev->queue_depth, link_suffix, tag_suffix);
-
-   return FALSE;
-}
-
-static int wait_on_busy(unsigned long iobase, unsigned int loop) {
-
-   while (inb(iobase + REG_LCL_INTR) & BSY_ASSERTED) {
-      udelay(1L);
-      if (--loop == 0) return TRUE;
-      }
-
-   return FALSE;
-}
-
-static int board_inquiry(unsigned int j) {
-   struct mscp *cpp;
-   dma_addr_t id_dma_addr;
-   unsigned int limit = 0;
-   unsigned long time;
-
-   id_dma_addr = pci_map_single(HD(j)->pdev, HD(j)->board_id,
-                    sizeof(HD(j)->board_id), PCI_DMA_BIDIRECTIONAL);
-   cpp = &HD(j)->cp[0];
-   cpp->cp_dma_addr = pci_map_single(HD(j)->pdev, cpp, sizeof(struct mscp),
-                                     PCI_DMA_BIDIRECTIONAL);
-   memset(cpp, 0, sizeof(struct mscp) - CP_TAIL_SIZE);
-   cpp->opcode = OP_HOST_ADAPTER;
-   cpp->xdir = DTD_IN;
-   cpp->data_address = H2DEV(id_dma_addr);
-   cpp->data_len = H2DEV(sizeof(HD(j)->board_id));
-   cpp->cdb_len = 6;
-   cpp->cdb[0] = HA_CMD_INQUIRY;
-
-   if (wait_on_busy(sh[j]->io_port, MAXLOOP)) {
-      printk("%s: board_inquiry, adapter busy.\n", BN(j));
-      return TRUE;
-      }
-
-   HD(j)->cp_stat[0] = IGNORE;
-
-   /* Clear the interrupt indication */
-   outb(CMD_CLR_INTR, sh[j]->io_port + REG_SYS_INTR);
-
-   /* Store pointer in OGM address bytes */
-   outl(H2DEV(cpp->cp_dma_addr), sh[j]->io_port + REG_OGM);
-
-   /* Issue OGM interrupt */
-   outb(CMD_OGM_INTR, sh[j]->io_port + REG_LCL_INTR);
-
-   spin_unlock_irq(&driver_lock);
-   time = jiffies;
-   while ((jiffies - time) < HZ && limit++ < 20000) udelay(100L);
-   spin_lock_irq(&driver_lock);
-
-   if (cpp->adapter_status || HD(j)->cp_stat[0] != FREE) {
-      HD(j)->cp_stat[0] = FREE;
-      printk("%s: board_inquiry, err 0x%x.\n", BN(j), cpp->adapter_status);
-      return TRUE;
-      }
-
-   pci_unmap_single(HD(j)->pdev, cpp->cp_dma_addr, sizeof(struct mscp),
-                    PCI_DMA_BIDIRECTIONAL);
-   pci_unmap_single(HD(j)->pdev, id_dma_addr, sizeof(HD(j)->board_id),
-                    PCI_DMA_BIDIRECTIONAL);
-   return FALSE;
-}
-
-static int port_detect \
-      (unsigned long port_base, unsigned int j, struct scsi_host_template *tpnt) {
-   unsigned char irq, dma_channel, subversion, i;
-   unsigned char in_byte;
-   char *bus_type, dma_name[16];
-
-   /* Allowed BIOS base addresses (NULL indicates reserved) */
-   unsigned long bios_segment_table[8] = {
-      0,
-      0xc4000, 0xc8000, 0xcc000, 0xd0000,
-      0xd4000, 0xd8000, 0xdc000
-      };
-
-   /* Allowed IRQs */
-   unsigned char interrupt_table[4] = { 15, 14, 11, 10 };
-
-   /* Allowed DMA channels for ISA (0 indicates reserved) */
-   unsigned char dma_channel_table[4] = { 5, 6, 7, 0 };
-
-   /* Head/sector mappings */
-   struct {
-      unsigned char heads;
-      unsigned char sectors;
-      } mapping_table[4] = {
-           { 16, 63 }, { 64, 32 }, { 64, 63 }, { 64, 32 }
-           };
-
-   struct config_1 {
-
-#if defined(__BIG_ENDIAN_BITFIELD)
-      unsigned char dma_channel: 2, interrupt:2,
-                    removable_disks_as_fixed:1, bios_segment: 3;
-#else
-      unsigned char bios_segment: 3, removable_disks_as_fixed: 1,
-                    interrupt: 2, dma_channel: 2;
-#endif
-
-      } config_1;
-
-   struct config_2 {
-
-#if defined(__BIG_ENDIAN_BITFIELD)
-      unsigned char tfr_port: 2, bios_drive_number: 1,
-                    mapping_mode: 2, ha_scsi_id: 3;
-#else
-      unsigned char ha_scsi_id: 3, mapping_mode: 2,
-                    bios_drive_number: 1, tfr_port: 2;
-#endif
-
-      } config_2;
-
-   char name[16];
-
-   sprintf(name, "%s%d", driver_name, j);
-
-   if (!request_region(port_base, REGION_SIZE, driver_name)) {
-#if defined(DEBUG_DETECT)
-      printk("%s: address 0x%03lx in use, skipping probe.\n", name, port_base);
-#endif
-      goto fail;
-      }
-
-   spin_lock_irq(&driver_lock);
-
-   if (inb(port_base + REG_PRODUCT_ID1) != PRODUCT_ID1) goto freelock;
-
-   in_byte = inb(port_base + REG_PRODUCT_ID2);
-
-   if ((in_byte & 0xf0) != PRODUCT_ID2) goto freelock;
-
-   *(char *)&config_1 = inb(port_base + REG_CONFIG1);
-   *(char *)&config_2 = inb(port_base + REG_CONFIG2);
-
-   irq = interrupt_table[config_1.interrupt];
-   dma_channel = dma_channel_table[config_1.dma_channel];
-   subversion = (in_byte & 0x0f);
-
-   /* Board detected, allocate its IRQ */
-   if (request_irq(irq, do_interrupt_handler,
-             (subversion == ESA) ? IRQF_SHARED : 0,
-             driver_name, (void *) &sha[j])) {
-      printk("%s: unable to allocate IRQ %u, detaching.\n", name, irq);
-      goto freelock;
-      }
-
-   if (subversion == ISA && request_dma(dma_channel, driver_name)) {
-      printk("%s: unable to allocate DMA channel %u, detaching.\n",
-             name, dma_channel);
-      goto freeirq;
-      }
-
-   if (have_old_firmware) tpnt->use_clustering = DISABLE_CLUSTERING;
-
-   spin_unlock_irq(&driver_lock);
-   sh[j] = scsi_register(tpnt, sizeof(struct hostdata));
-   spin_lock_irq(&driver_lock);
-
-   if (sh[j] == NULL) {
-      printk("%s: unable to register host, detaching.\n", name);
-      goto freedma;
-      }
-
-   sh[j]->io_port = port_base;
-   sh[j]->unique_id = port_base;
-   sh[j]->n_io_port = REGION_SIZE;
-   sh[j]->base = bios_segment_table[config_1.bios_segment];
-   sh[j]->irq = irq;
-   sh[j]->sg_tablesize = MAX_SGLIST;
-   sh[j]->this_id = config_2.ha_scsi_id;
-   sh[j]->can_queue = MAX_MAILBOXES;
-   sh[j]->cmd_per_lun = MAX_CMD_PER_LUN;
-
-#if defined(DEBUG_DETECT)
-   {
-   unsigned char sys_mask, lcl_mask;
-
-   sys_mask = inb(sh[j]->io_port + REG_SYS_MASK);
-   lcl_mask = inb(sh[j]->io_port + REG_LCL_MASK);
-   printk("SYS_MASK 0x%x, LCL_MASK 0x%x.\n", sys_mask, lcl_mask);
-   }
-#endif
-
-   /* Probably a bogus host scsi id, set it to the dummy value */
-   if (sh[j]->this_id == 0) sh[j]->this_id = -1;
-
-   /* If BIOS is disabled, force enable interrupts */
-   if (sh[j]->base == 0) outb(CMD_ENA_INTR, sh[j]->io_port + REG_SYS_MASK);
-
-   memset(HD(j), 0, sizeof(struct hostdata));
-   HD(j)->heads = mapping_table[config_2.mapping_mode].heads;
-   HD(j)->sectors = mapping_table[config_2.mapping_mode].sectors;
-   HD(j)->subversion = subversion;
-   HD(j)->pdev = NULL;
-   HD(j)->board_number = j;
-
-   if (have_old_firmware) sh[j]->sg_tablesize = MAX_SAFE_SGLIST;
-
-   if (HD(j)->subversion == ESA) {
-      sh[j]->unchecked_isa_dma = FALSE;
-      sh[j]->dma_channel = NO_DMA;
-      sprintf(BN(j), "U34F%d", j);
-      bus_type = "VESA";
-      }
-   else {
-      unsigned long flags;
-      sh[j]->unchecked_isa_dma = TRUE;
-
-      flags=claim_dma_lock();
-      disable_dma(dma_channel);
-      clear_dma_ff(dma_channel);
-      set_dma_mode(dma_channel, DMA_MODE_CASCADE);
-      enable_dma(dma_channel);
-      release_dma_lock(flags);
-
-      sh[j]->dma_channel = dma_channel;
-      sprintf(BN(j), "U14F%d", j);
-      bus_type = "ISA";
-      }
-
-   sh[j]->max_channel = MAX_CHANNEL - 1;
-   sh[j]->max_id = MAX_TARGET;
-   sh[j]->max_lun = MAX_LUN;
-
-   if (HD(j)->subversion == ISA && !board_inquiry(j)) {
-      HD(j)->board_id[40] = 0;
-
-      if (strcmp(&HD(j)->board_id[32], "06000600")) {
-         printk("%s: %s.\n", BN(j), &HD(j)->board_id[8]);
-         printk("%s: firmware %s is outdated, FW PROM should be 28004-006.\n",
-                BN(j), &HD(j)->board_id[32]);
-         sh[j]->hostt->use_clustering = DISABLE_CLUSTERING;
-         sh[j]->sg_tablesize = MAX_SAFE_SGLIST;
-         }
-      }
-
-   if (dma_channel == NO_DMA) sprintf(dma_name, "%s", "BMST");
-   else                       sprintf(dma_name, "DMA %u", dma_channel);
-
-   spin_unlock_irq(&driver_lock);
-
-   for (i = 0; i < sh[j]->can_queue; i++)
-      HD(j)->cp[i].cp_dma_addr = pci_map_single(HD(j)->pdev,
-            &HD(j)->cp[i], sizeof(struct mscp), PCI_DMA_BIDIRECTIONAL);
-
-   for (i = 0; i < sh[j]->can_queue; i++)
-      if (! ((&HD(j)->cp[i])->sglist = kmalloc(
-            sh[j]->sg_tablesize * sizeof(struct sg_list),
-            (sh[j]->unchecked_isa_dma ? GFP_DMA : 0) | GFP_ATOMIC))) {
-         printk("%s: kmalloc SGlist failed, mbox %d, detaching.\n", BN(j), i);
-         goto release;
-         }
-
-   if (max_queue_depth > MAX_TAGGED_CMD_PER_LUN)
-       max_queue_depth = MAX_TAGGED_CMD_PER_LUN;
-
-   if (max_queue_depth < MAX_CMD_PER_LUN) max_queue_depth = MAX_CMD_PER_LUN;
-
-   if (tag_mode != TAG_DISABLED && tag_mode != TAG_SIMPLE)
-      tag_mode = TAG_ORDERED;
-
-   if (j == 0) {
-      printk("UltraStor 14F/34F: Copyright (C) 1994-2003 Dario Ballabio.\n");
-      printk("%s config options -> of:%c, tm:%d, lc:%c, mq:%d, et:%c.\n",
-             driver_name, YESNO(have_old_firmware), tag_mode,
-             YESNO(linked_comm), max_queue_depth, YESNO(ext_tran));
-      }
-
-   printk("%s: %s 0x%03lx, BIOS 0x%05x, IRQ %u, %s, SG %d, MB %d.\n",
-          BN(j), bus_type, (unsigned long)sh[j]->io_port, (int)sh[j]->base,
-          sh[j]->irq, dma_name, sh[j]->sg_tablesize, sh[j]->can_queue);
-
-   if (sh[j]->max_id > 8 || sh[j]->max_lun > 8)
-      printk("%s: wide SCSI support enabled, max_id %u, max_lun %llu.\n",
-             BN(j), sh[j]->max_id, sh[j]->max_lun);
-
-   for (i = 0; i <= sh[j]->max_channel; i++)
-      printk("%s: SCSI channel %u enabled, host target ID %d.\n",
-             BN(j), i, sh[j]->this_id);
-
-   return TRUE;
-
-freedma:
-   if (subversion == ISA) free_dma(dma_channel);
-freeirq:
-   free_irq(irq, &sha[j]);
-freelock:
-   spin_unlock_irq(&driver_lock);
-   release_region(port_base, REGION_SIZE);
-fail:
-   return FALSE;
-
-release:
-   u14_34f_release(sh[j]);
-   return FALSE;
-}
-
-static void internal_setup(char *str, int *ints) {
-   int i, argc = ints[0];
-   char *cur = str, *pc;
-
-   if (argc > 0) {
-
-      if (argc > MAX_INT_PARAM) argc = MAX_INT_PARAM;
-
-      for (i = 0; i < argc; i++) io_port[i] = ints[i + 1];
-
-      io_port[i] = 0;
-      setup_done = TRUE;
-      }
-
-   while (cur && (pc = strchr(cur, ':'))) {
-      int val = 0, c = *++pc;
-
-      if (c == 'n' || c == 'N') val = FALSE;
-      else if (c == 'y' || c == 'Y') val = TRUE;
-      else val = (int) simple_strtoul(pc, NULL, 0);
-
-      if (!strncmp(cur, "lc:", 3)) linked_comm = val;
-      else if (!strncmp(cur, "of:", 3)) have_old_firmware = val;
-      else if (!strncmp(cur, "tm:", 3)) tag_mode = val;
-      else if (!strncmp(cur, "tc:", 3)) tag_mode = val;
-      else if (!strncmp(cur, "mq:", 3))  max_queue_depth = val;
-      else if (!strncmp(cur, "ls:", 3))  link_statistics = val;
-      else if (!strncmp(cur, "et:", 3))  ext_tran = val;
-
-      if ((cur = strchr(cur, ','))) ++cur;
-      }
-
-   return;
-}
-
-static int option_setup(char *str) {
-   int ints[MAX_INT_PARAM];
-   char *cur = str;
-   int i = 1;
-
-   while (cur && isdigit(*cur) && i < MAX_INT_PARAM) {
-      ints[i++] = simple_strtoul(cur, NULL, 0);
-
-      if ((cur = strchr(cur, ',')) != NULL) cur++;
-   }
-
-   ints[0] = i - 1;
-   internal_setup(cur, ints);
-   return 1;
-}
-
-static int u14_34f_detect(struct scsi_host_template *tpnt) {
-   unsigned int j = 0, k;
-
-   tpnt->proc_name = "u14-34f";
-
-   if(strlen(boot_options)) option_setup(boot_options);
-
-#if defined(MODULE)
-   /* io_port could have been modified when loading as a module */
-   if(io_port[0] != SKIP) {
-      setup_done = TRUE;
-      io_port[MAX_INT_PARAM] = 0;
-      }
-#endif
-
-   for (k = 0; k < MAX_BOARDS + 1; k++) sh[k] = NULL;
-
-   for (k = 0; io_port[k]; k++) {
-
-      if (io_port[k] == SKIP) continue;
-
-      if (j < MAX_BOARDS && port_detect(io_port[k], j, tpnt)) j++;
-      }
-
-   num_boards = j;
-   return j;
-}
-
-static void map_dma(unsigned int i, unsigned int j) {
-   unsigned int data_len = 0;
-   unsigned int k, pci_dir;
-   int count;
-   struct scatterlist *sg;
-   struct mscp *cpp;
-   struct scsi_cmnd *SCpnt;
-
-   cpp = &HD(j)->cp[i]; SCpnt = cpp->SCpnt;
-   pci_dir = SCpnt->sc_data_direction;
-
-   if (SCpnt->sense_buffer)
-      cpp->sense_addr = H2DEV(pci_map_single(HD(j)->pdev, SCpnt->sense_buffer,
-                           SCSI_SENSE_BUFFERSIZE, PCI_DMA_FROMDEVICE));
-
-   cpp->sense_len = SCSI_SENSE_BUFFERSIZE;
-
-   if (scsi_bufflen(SCpnt)) {
-	   count = scsi_dma_map(SCpnt);
-	   BUG_ON(count < 0);
-
-	   scsi_for_each_sg(SCpnt, sg, count, k) {
-		   cpp->sglist[k].address = H2DEV(sg_dma_address(sg));
-		   cpp->sglist[k].num_bytes = H2DEV(sg_dma_len(sg));
-		   data_len += sg->length;
-	   }
-
-	   cpp->sg = TRUE;
-	   cpp->use_sg = scsi_sg_count(SCpnt);
-	   cpp->data_address =
-		   H2DEV(pci_map_single(HD(j)->pdev, cpp->sglist,
-					cpp->use_sg * sizeof(struct sg_list),
-					pci_dir));
-	   cpp->data_len = H2DEV(data_len);
-
-   } else {
-	   pci_dir = PCI_DMA_BIDIRECTIONAL;
-	   cpp->data_len = H2DEV(scsi_bufflen(SCpnt));
-   }
-}
-
-static void unmap_dma(unsigned int i, unsigned int j) {
-   unsigned int pci_dir;
-   struct mscp *cpp;
-   struct scsi_cmnd *SCpnt;
-
-   cpp = &HD(j)->cp[i]; SCpnt = cpp->SCpnt;
-   pci_dir = SCpnt->sc_data_direction;
-
-   if (DEV2H(cpp->sense_addr))
-      pci_unmap_single(HD(j)->pdev, DEV2H(cpp->sense_addr),
-                       DEV2H(cpp->sense_len), PCI_DMA_FROMDEVICE);
-
-   scsi_dma_unmap(SCpnt);
-
-   if (!DEV2H(cpp->data_len)) pci_dir = PCI_DMA_BIDIRECTIONAL;
-
-   if (DEV2H(cpp->data_address))
-      pci_unmap_single(HD(j)->pdev, DEV2H(cpp->data_address),
-                       DEV2H(cpp->data_len), pci_dir);
-}
-
-static void sync_dma(unsigned int i, unsigned int j) {
-   unsigned int pci_dir;
-   struct mscp *cpp;
-   struct scsi_cmnd *SCpnt;
-
-   cpp = &HD(j)->cp[i]; SCpnt = cpp->SCpnt;
-   pci_dir = SCpnt->sc_data_direction;
-
-   if (DEV2H(cpp->sense_addr))
-      pci_dma_sync_single_for_cpu(HD(j)->pdev, DEV2H(cpp->sense_addr),
-                          DEV2H(cpp->sense_len), PCI_DMA_FROMDEVICE);
-
-   if (scsi_sg_count(SCpnt))
-	   pci_dma_sync_sg_for_cpu(HD(j)->pdev, scsi_sglist(SCpnt),
-				   scsi_sg_count(SCpnt), pci_dir);
-
-   if (!DEV2H(cpp->data_len)) pci_dir = PCI_DMA_BIDIRECTIONAL;
-
-   if (DEV2H(cpp->data_address))
-      pci_dma_sync_single_for_cpu(HD(j)->pdev, DEV2H(cpp->data_address),
-                       DEV2H(cpp->data_len), pci_dir);
-}
-
-static void scsi_to_dev_dir(unsigned int i, unsigned int j) {
-   unsigned int k;
-
-   static const unsigned char data_out_cmds[] = {
-      0x0a, 0x2a, 0x15, 0x55, 0x04, 0x07, 0x18, 0x1d, 0x24, 0x2e,
-      0x30, 0x31, 0x32, 0x38, 0x39, 0x3a, 0x3b, 0x3d, 0x3f, 0x40,
-      0x41, 0x4c, 0xaa, 0xae, 0xb0, 0xb1, 0xb2, 0xb6, 0xea, 0x1b, 0x5d
-      };
-
-   static const unsigned char data_none_cmds[] = {
-      0x01, 0x0b, 0x10, 0x11, 0x13, 0x16, 0x17, 0x19, 0x2b, 0x1e,
-      0x2c, 0xac, 0x2f, 0xaf, 0x33, 0xb3, 0x35, 0x36, 0x45, 0x47,
-      0x48, 0x49, 0xa9, 0x4b, 0xa5, 0xa6, 0xb5, 0x00
-      };
-
-   struct mscp *cpp;
-   struct scsi_cmnd *SCpnt;
-
-   cpp = &HD(j)->cp[i]; SCpnt = cpp->SCpnt;
-
-   if (SCpnt->sc_data_direction == DMA_FROM_DEVICE) {
-      cpp->xdir = DTD_IN;
-      return;
-      }
-   else if (SCpnt->sc_data_direction == DMA_TO_DEVICE) {
-      cpp->xdir = DTD_OUT;
-      return;
-      }
-   else if (SCpnt->sc_data_direction == DMA_NONE) {
-      cpp->xdir = DTD_NONE;
-      return;
-      }
-
-   if (SCpnt->sc_data_direction != DMA_BIDIRECTIONAL)
-      panic("%s: qcomm, invalid SCpnt->sc_data_direction.\n", BN(j));
-
-   cpp->xdir = DTD_IN;
-
-   for (k = 0; k < ARRAY_SIZE(data_out_cmds); k++)
-      if (SCpnt->cmnd[0] == data_out_cmds[k]) {
-         cpp->xdir = DTD_OUT;
-         break;
-         }
-
-   if (cpp->xdir == DTD_IN)
-      for (k = 0; k < ARRAY_SIZE(data_none_cmds); k++)
-         if (SCpnt->cmnd[0] == data_none_cmds[k]) {
-            cpp->xdir = DTD_NONE;
-            break;
-            }
-
-}
-
-static int u14_34f_queuecommand_lck(struct scsi_cmnd *SCpnt, void (*done)(struct scsi_cmnd *)) {
-   unsigned int i, j, k;
-   struct mscp *cpp;
-
-   /* j is the board number */
-   j = ((struct hostdata *) SCpnt->device->host->hostdata)->board_number;
-
-   if (SCpnt->host_scribble)
-      panic("%s: qcomm, SCpnt %p already active.\n",
-            BN(j), SCpnt);
-
-   /* i is the mailbox number, look for the first free mailbox
-      starting from last_cp_used */
-   i = HD(j)->last_cp_used + 1;
-
-   for (k = 0; k < sh[j]->can_queue; k++, i++) {
-
-      if (i >= sh[j]->can_queue) i = 0;
-
-      if (HD(j)->cp_stat[i] == FREE) {
-         HD(j)->last_cp_used = i;
-         break;
-         }
-      }
-
-   if (k == sh[j]->can_queue) {
-      printk("%s: qcomm, no free mailbox.\n", BN(j));
-      return 1;
-      }
-
-   /* Set pointer to control packet structure */
-   cpp = &HD(j)->cp[i];
-
-   memset(cpp, 0, sizeof(struct mscp) - CP_TAIL_SIZE);
-   SCpnt->scsi_done = done;
-   cpp->cpp_index = i;
-   SCpnt->host_scribble = (unsigned char *) &cpp->cpp_index;
-
-   if (do_trace) printk("%s: qcomm, mbox %d, target %d.%d:%u.\n",
-                        BN(j), i, SCpnt->device->channel, SCpnt->device->id,
-                        (u8)SCpnt->device->lun);
-
-   cpp->opcode = OP_SCSI;
-   cpp->channel = SCpnt->device->channel;
-   cpp->target = SCpnt->device->id;
-   cpp->lun = (u8)SCpnt->device->lun;
-   cpp->SCpnt = SCpnt;
-   cpp->cdb_len = SCpnt->cmd_len;
-   memcpy(cpp->cdb, SCpnt->cmnd, SCpnt->cmd_len);
-
-   /* Use data transfer direction SCpnt->sc_data_direction */
-   scsi_to_dev_dir(i, j);
-
-   /* Map DMA buffers and SG list */
-   map_dma(i, j);
-
-   if (linked_comm && SCpnt->device->queue_depth > 2
-                                     && TLDEV(SCpnt->device->type)) {
-      HD(j)->cp_stat[i] = READY;
-      flush_dev(SCpnt->device, blk_rq_pos(SCpnt->request), j, FALSE);
-      return 0;
-      }
-
-   if (wait_on_busy(sh[j]->io_port, MAXLOOP)) {
-      unmap_dma(i, j);
-      SCpnt->host_scribble = NULL;
-      scmd_printk(KERN_INFO, SCpnt,
-      		"qcomm, adapter busy.\n");
-      return 1;
-      }
-
-   /* Store pointer in OGM address bytes */
-   outl(H2DEV(cpp->cp_dma_addr), sh[j]->io_port + REG_OGM);
-
-   /* Issue OGM interrupt */
-   outb(CMD_OGM_INTR, sh[j]->io_port + REG_LCL_INTR);
-
-   HD(j)->cp_stat[i] = IN_USE;
-   return 0;
-}
-
-static DEF_SCSI_QCMD(u14_34f_queuecommand)
-
-static int u14_34f_eh_abort(struct scsi_cmnd *SCarg) {
-   unsigned int i, j;
-
-   j = ((struct hostdata *) SCarg->device->host->hostdata)->board_number;
-
-   if (SCarg->host_scribble == NULL) {
-      scmd_printk(KERN_INFO, SCarg, "abort, command inactive.\n");
-      return SUCCESS;
-      }
-
-   i = *(unsigned int *)SCarg->host_scribble;
-   scmd_printk(KERN_INFO, SCarg, "abort, mbox %d.\n", i);
-
-   if (i >= sh[j]->can_queue)
-      panic("%s: abort, invalid SCarg->host_scribble.\n", BN(j));
-
-   if (wait_on_busy(sh[j]->io_port, MAXLOOP)) {
-      printk("%s: abort, timeout error.\n", BN(j));
-      return FAILED;
-      }
-
-   if (HD(j)->cp_stat[i] == FREE) {
-      printk("%s: abort, mbox %d is free.\n", BN(j), i);
-      return SUCCESS;
-      }
-
-   if (HD(j)->cp_stat[i] == IN_USE) {
-      printk("%s: abort, mbox %d is in use.\n", BN(j), i);
-
-      if (SCarg != HD(j)->cp[i].SCpnt)
-         panic("%s: abort, mbox %d, SCarg %p, cp SCpnt %p.\n",
-               BN(j), i, SCarg, HD(j)->cp[i].SCpnt);
-
-      if (inb(sh[j]->io_port + REG_SYS_INTR) & IRQ_ASSERTED)
-         printk("%s: abort, mbox %d, interrupt pending.\n", BN(j), i);
-
-      return FAILED;
-      }
-
-   if (HD(j)->cp_stat[i] == IN_RESET) {
-      printk("%s: abort, mbox %d is in reset.\n", BN(j), i);
-      return FAILED;
-      }
-
-   if (HD(j)->cp_stat[i] == LOCKED) {
-      printk("%s: abort, mbox %d is locked.\n", BN(j), i);
-      return SUCCESS;
-      }
-
-   if (HD(j)->cp_stat[i] == READY || HD(j)->cp_stat[i] == ABORTING) {
-      unmap_dma(i, j);
-      SCarg->result = DID_ABORT << 16;
-      SCarg->host_scribble = NULL;
-      HD(j)->cp_stat[i] = FREE;
-      printk("%s, abort, mbox %d ready, DID_ABORT, done.\n", BN(j), i);
-      SCarg->scsi_done(SCarg);
-      return SUCCESS;
-      }
-
-   panic("%s: abort, mbox %d, invalid cp_stat.\n", BN(j), i);
-}
-
-static int u14_34f_eh_host_reset(struct scsi_cmnd *SCarg) {
-   unsigned int i, j, k, c, limit = 0;
-   unsigned long time;
-   int arg_done = FALSE;
-   struct scsi_cmnd *SCpnt;
-
-   j = ((struct hostdata *) SCarg->device->host->hostdata)->board_number;
-   scmd_printk(KERN_INFO, SCarg, "reset, enter.\n");
-
-   spin_lock_irq(sh[j]->host_lock);
-
-   if (SCarg->host_scribble == NULL)
-      printk("%s: reset, inactive.\n", BN(j));
-
-   if (HD(j)->in_reset) {
-      printk("%s: reset, exit, already in reset.\n", BN(j));
-      spin_unlock_irq(sh[j]->host_lock);
-      return FAILED;
-      }
-
-   if (wait_on_busy(sh[j]->io_port, MAXLOOP)) {
-      printk("%s: reset, exit, timeout error.\n", BN(j));
-      spin_unlock_irq(sh[j]->host_lock);
-      return FAILED;
-      }
-
-   HD(j)->retries = 0;
-
-   for (c = 0; c <= sh[j]->max_channel; c++)
-      for (k = 0; k < sh[j]->max_id; k++) {
-         HD(j)->target_redo[k][c] = TRUE;
-         HD(j)->target_to[k][c] = 0;
-         }
-
-   for (i = 0; i < sh[j]->can_queue; i++) {
-
-      if (HD(j)->cp_stat[i] == FREE) continue;
-
-      if (HD(j)->cp_stat[i] == LOCKED) {
-         HD(j)->cp_stat[i] = FREE;
-         printk("%s: reset, locked mbox %d forced free.\n", BN(j), i);
-         continue;
-         }
-
-      if (!(SCpnt = HD(j)->cp[i].SCpnt))
-         panic("%s: reset, mbox %d, SCpnt == NULL.\n", BN(j), i);
-
-      if (HD(j)->cp_stat[i] == READY || HD(j)->cp_stat[i] == ABORTING) {
-         HD(j)->cp_stat[i] = ABORTING;
-         printk("%s: reset, mbox %d aborting.\n", BN(j), i);
-         }
-
-      else {
-         HD(j)->cp_stat[i] = IN_RESET;
-         printk("%s: reset, mbox %d in reset.\n", BN(j), i);
-         }
-
-      if (SCpnt->host_scribble == NULL)
-         panic("%s: reset, mbox %d, garbled SCpnt.\n", BN(j), i);
-
-      if (*(unsigned int *)SCpnt->host_scribble != i)
-         panic("%s: reset, mbox %d, index mismatch.\n", BN(j), i);
-
-      if (SCpnt->scsi_done == NULL)
-         panic("%s: reset, mbox %d, SCpnt->scsi_done == NULL.\n", BN(j), i);
-
-      if (SCpnt == SCarg) arg_done = TRUE;
-      }
-
-   if (wait_on_busy(sh[j]->io_port, MAXLOOP)) {
-      printk("%s: reset, cannot reset, timeout error.\n", BN(j));
-      spin_unlock_irq(sh[j]->host_lock);
-      return FAILED;
-      }
-
-   outb(CMD_RESET, sh[j]->io_port + REG_LCL_INTR);
-   printk("%s: reset, board reset done, enabling interrupts.\n", BN(j));
-
-#if defined(DEBUG_RESET)
-   do_trace = TRUE;
-#endif
-
-   HD(j)->in_reset = TRUE;
-
-   spin_unlock_irq(sh[j]->host_lock);
-   time = jiffies;
-   while ((jiffies - time) < (10 * HZ) && limit++ < 200000) udelay(100L);
-   spin_lock_irq(sh[j]->host_lock);
-
-   printk("%s: reset, interrupts disabled, loops %d.\n", BN(j), limit);
-
-   for (i = 0; i < sh[j]->can_queue; i++) {
-
-      if (HD(j)->cp_stat[i] == IN_RESET) {
-         SCpnt = HD(j)->cp[i].SCpnt;
-         unmap_dma(i, j);
-         SCpnt->result = DID_RESET << 16;
-         SCpnt->host_scribble = NULL;
-
-         /* This mailbox is still waiting for its interrupt */
-         HD(j)->cp_stat[i] = LOCKED;
-
-         printk("%s, reset, mbox %d locked, DID_RESET, done.\n", BN(j), i);
-         }
-
-      else if (HD(j)->cp_stat[i] == ABORTING) {
-         SCpnt = HD(j)->cp[i].SCpnt;
-         unmap_dma(i, j);
-         SCpnt->result = DID_RESET << 16;
-         SCpnt->host_scribble = NULL;
-
-         /* This mailbox was never queued to the adapter */
-         HD(j)->cp_stat[i] = FREE;
-
-         printk("%s, reset, mbox %d aborting, DID_RESET, done.\n", BN(j), i);
-         }
-
-      else
-
-         /* Any other mailbox has already been set free by interrupt */
-         continue;
-
-      SCpnt->scsi_done(SCpnt);
-      }
-
-   HD(j)->in_reset = FALSE;
-   do_trace = FALSE;
-
-   if (arg_done) printk("%s: reset, exit, done.\n", BN(j));
-   else          printk("%s: reset, exit.\n", BN(j));
-
-   spin_unlock_irq(sh[j]->host_lock);
-   return SUCCESS;
-}
-
-static int u14_34f_bios_param(struct scsi_device *disk,
-                 struct block_device *bdev, sector_t capacity, int *dkinfo) {
-   unsigned int j = 0;
-   unsigned int size = capacity;
-
-   dkinfo[0] = HD(j)->heads;
-   dkinfo[1] = HD(j)->sectors;
-   dkinfo[2] = size / (HD(j)->heads * HD(j)->sectors);
-
-   if (ext_tran && (scsicam_bios_param(bdev, capacity, dkinfo) < 0)) {
-      dkinfo[0] = 255;
-      dkinfo[1] = 63;
-      dkinfo[2] = size / (dkinfo[0] * dkinfo[1]);
-      }
-
-#if defined (DEBUG_GEOMETRY)
-   printk ("%s: bios_param, head=%d, sec=%d, cyl=%d.\n", driver_name,
-           dkinfo[0], dkinfo[1], dkinfo[2]);
-#endif
-
-   return FALSE;
-}
-
-static void sort(unsigned long sk[], unsigned int da[], unsigned int n,
-                 unsigned int rev) {
-   unsigned int i, j, k, y;
-   unsigned long x;
-
-   for (i = 0; i < n - 1; i++) {
-      k = i;
-
-      for (j = k + 1; j < n; j++)
-         if (rev) {
-            if (sk[j] > sk[k]) k = j;
-            }
-         else {
-            if (sk[j] < sk[k]) k = j;
-            }
-
-      if (k != i) {
-         x = sk[k]; sk[k] = sk[i]; sk[i] = x;
-         y = da[k]; da[k] = da[i]; da[i] = y;
-         }
-      }
-
-   return;
-   }
-
-static int reorder(unsigned int j, unsigned long cursec,
-                 unsigned int ihdlr, unsigned int il[], unsigned int n_ready) {
-   struct scsi_cmnd *SCpnt;
-   struct mscp *cpp;
-   unsigned int k, n;
-   unsigned int rev = FALSE, s = TRUE, r = TRUE;
-   unsigned int input_only = TRUE, overlap = FALSE;
-   unsigned long sl[n_ready], pl[n_ready], ll[n_ready];
-   unsigned long maxsec = 0, minsec = ULONG_MAX, seek = 0, iseek = 0;
-   unsigned long ioseek = 0;
-
-   static unsigned int flushcount = 0, batchcount = 0, sortcount = 0;
-   static unsigned int readycount = 0, ovlcount = 0, inputcount = 0;
-   static unsigned int readysorted = 0, revcount = 0;
-   static unsigned long seeksorted = 0, seeknosort = 0;
-
-   if (link_statistics && !(++flushcount % link_statistics))
-      printk("fc %d bc %d ic %d oc %d rc %d rs %d sc %d re %d"\
-             " av %ldK as %ldK.\n", flushcount, batchcount, inputcount,
-             ovlcount, readycount, readysorted, sortcount, revcount,
-             seeknosort / (readycount + 1),
-             seeksorted / (readycount + 1));
-
-   if (n_ready <= 1) return FALSE;
-
-   for (n = 0; n < n_ready; n++) {
-      k = il[n]; cpp = &HD(j)->cp[k]; SCpnt = cpp->SCpnt;
-
-      if (!(cpp->xdir == DTD_IN)) input_only = FALSE;
-
-      if (blk_rq_pos(SCpnt->request) < minsec)
-	 minsec = blk_rq_pos(SCpnt->request);
-      if (blk_rq_pos(SCpnt->request) > maxsec)
-	 maxsec = blk_rq_pos(SCpnt->request);
-
-      sl[n] = blk_rq_pos(SCpnt->request);
-      ioseek += blk_rq_sectors(SCpnt->request);
-
-      if (!n) continue;
-
-      if (sl[n] < sl[n - 1]) s = FALSE;
-      if (sl[n] > sl[n - 1]) r = FALSE;
-
-      if (link_statistics) {
-         if (sl[n] > sl[n - 1])
-            seek += sl[n] - sl[n - 1];
-         else
-            seek += sl[n - 1] - sl[n];
-         }
-
-      }
-
-   if (link_statistics) {
-      if (cursec > sl[0]) seek += cursec - sl[0]; else seek += sl[0] - cursec;
-      }
-
-   if (cursec > ((maxsec + minsec) / 2)) rev = TRUE;
-
-   if (ioseek > ((maxsec - minsec) / 2)) rev = FALSE;
-
-   if (!((rev && r) || (!rev && s))) sort(sl, il, n_ready, rev);
-
-   if (!input_only) for (n = 0; n < n_ready; n++) {
-      k = il[n]; cpp = &HD(j)->cp[k]; SCpnt = cpp->SCpnt;
-      ll[n] = blk_rq_sectors(SCpnt->request); pl[n] = SCpnt->serial_number;
-
-      if (!n) continue;
-
-      if ((sl[n] == sl[n - 1]) || (!rev && ((sl[n - 1] + ll[n - 1]) > sl[n]))
-          || (rev && ((sl[n] + ll[n]) > sl[n - 1]))) overlap = TRUE;
-      }
-
-   if (overlap) sort(pl, il, n_ready, FALSE);
-
-   if (link_statistics) {
-      if (cursec > sl[0]) iseek = cursec - sl[0]; else iseek = sl[0] - cursec;
-      batchcount++; readycount += n_ready; seeknosort += seek / 1024;
-      if (input_only) inputcount++;
-      if (overlap) { ovlcount++; seeksorted += iseek / 1024; }
-      else seeksorted += (iseek + maxsec - minsec) / 1024;
-      if (rev && !r)     {  revcount++; readysorted += n_ready; }
-      if (!rev && !s)    { sortcount++; readysorted += n_ready; }
-      }
-
-#if defined(DEBUG_LINKED_COMMANDS)
-   if (link_statistics && (overlap || !(flushcount % link_statistics)))
-      for (n = 0; n < n_ready; n++) {
-         k = il[n]; cpp = &HD(j)->cp[k]; SCpnt = cpp->SCpnt;
-         printk("%s %d.%d:%llu mb %d fc %d nr %d sec %ld ns %u"\
-                " cur %ld s:%c r:%c rev:%c in:%c ov:%c xd %d.\n",
-                (ihdlr ? "ihdlr" : "qcomm"), SCpnt->channel, SCpnt->target,
-                (u8)SCpnt->lun, k, flushcount, n_ready,
-                blk_rq_pos(SCpnt->request), blk_rq_sectors(SCpnt->request),
-		cursec, YESNO(s), YESNO(r), YESNO(rev), YESNO(input_only),
-                YESNO(overlap), cpp->xdir);
-         }
-#endif
-   return overlap;
-}
-
-static void flush_dev(struct scsi_device *dev, unsigned long cursec, unsigned int j,
-                      unsigned int ihdlr) {
-   struct scsi_cmnd *SCpnt;
-   struct mscp *cpp;
-   unsigned int k, n, n_ready = 0, il[MAX_MAILBOXES];
-
-   for (k = 0; k < sh[j]->can_queue; k++) {
-
-      if (HD(j)->cp_stat[k] != READY && HD(j)->cp_stat[k] != IN_USE) continue;
-
-      cpp = &HD(j)->cp[k]; SCpnt = cpp->SCpnt;
-
-      if (SCpnt->device != dev) continue;
-
-      if (HD(j)->cp_stat[k] == IN_USE) return;
-
-      il[n_ready++] = k;
-      }
-
-   if (reorder(j, cursec, ihdlr, il, n_ready)) n_ready = 1;
-
-   for (n = 0; n < n_ready; n++) {
-      k = il[n]; cpp = &HD(j)->cp[k]; SCpnt = cpp->SCpnt;
-
-      if (wait_on_busy(sh[j]->io_port, MAXLOOP)) {
-         scmd_printk(KERN_INFO, SCpnt,
-	 	"%s, mbox %d, adapter"
-                " busy, will abort.\n", (ihdlr ? "ihdlr" : "qcomm"),
-                k);
-         HD(j)->cp_stat[k] = ABORTING;
-         continue;
-         }
-
-      outl(H2DEV(cpp->cp_dma_addr), sh[j]->io_port + REG_OGM);
-      outb(CMD_OGM_INTR, sh[j]->io_port + REG_LCL_INTR);
-      HD(j)->cp_stat[k] = IN_USE;
-      }
-
-}
-
-static irqreturn_t ihdlr(unsigned int j)
-{
-   struct scsi_cmnd *SCpnt;
-   unsigned int i, k, c, status, tstatus, reg, ret;
-   struct mscp *spp, *cpp;
-   int irq = sh[j]->irq;
-
-   /* Check if this board need to be serviced */
-   if (!((reg = inb(sh[j]->io_port + REG_SYS_INTR)) & IRQ_ASSERTED)) goto none;
-
-   HD(j)->iocount++;
-
-   if (do_trace) printk("%s: ihdlr, enter, irq %d, count %d.\n", BN(j), irq,
-                        HD(j)->iocount);
-
-   /* Check if this board is still busy */
-   if (wait_on_busy(sh[j]->io_port, 20 * MAXLOOP)) {
-      outb(CMD_CLR_INTR, sh[j]->io_port + REG_SYS_INTR);
-      printk("%s: ihdlr, busy timeout error,  irq %d, reg 0x%x, count %d.\n",
-             BN(j), irq, reg, HD(j)->iocount);
-      goto none;
-      }
-
-   ret = inl(sh[j]->io_port + REG_ICM);
-
-   /* Clear interrupt pending flag */
-   outb(CMD_CLR_INTR, sh[j]->io_port + REG_SYS_INTR);
-
-   /* Find the mailbox to be serviced on this board */
-   for (i = 0; i < sh[j]->can_queue; i++)
-      if (H2DEV(HD(j)->cp[i].cp_dma_addr) == ret) break;
-
-   if (i >= sh[j]->can_queue)
-      panic("%s: ihdlr, invalid mscp bus address %p, cp0 %p.\n", BN(j),
-            (void *)ret, (void *)H2DEV(HD(j)->cp[0].cp_dma_addr));
-
-   cpp = &(HD(j)->cp[i]);
-   spp = cpp;
-
-#if defined(DEBUG_GENERATE_ABORTS)
-   if ((HD(j)->iocount > 500) && ((HD(j)->iocount % 500) < 3)) goto handled;
-#endif
-
-   if (HD(j)->cp_stat[i] == IGNORE) {
-      HD(j)->cp_stat[i] = FREE;
-      goto handled;
-      }
-   else if (HD(j)->cp_stat[i] == LOCKED) {
-      HD(j)->cp_stat[i] = FREE;
-      printk("%s: ihdlr, mbox %d unlocked, count %d.\n", BN(j), i,
-             HD(j)->iocount);
-      goto handled;
-      }
-   else if (HD(j)->cp_stat[i] == FREE) {
-      printk("%s: ihdlr, mbox %d is free, count %d.\n", BN(j), i,
-             HD(j)->iocount);
-      goto handled;
-      }
-   else if (HD(j)->cp_stat[i] == IN_RESET)
-      printk("%s: ihdlr, mbox %d is in reset.\n", BN(j), i);
-   else if (HD(j)->cp_stat[i] != IN_USE)
-      panic("%s: ihdlr, mbox %d, invalid cp_stat: %d.\n",
-            BN(j), i, HD(j)->cp_stat[i]);
-
-   HD(j)->cp_stat[i] = FREE;
-   SCpnt = cpp->SCpnt;
-
-   if (SCpnt == NULL) panic("%s: ihdlr, mbox %d, SCpnt == NULL.\n", BN(j), i);
-
-   if (SCpnt->host_scribble == NULL)
-      panic("%s: ihdlr, mbox %d, SCpnt %p garbled.\n", BN(j), i,
-            SCpnt);
-
-   if (*(unsigned int *)SCpnt->host_scribble != i)
-      panic("%s: ihdlr, mbox %d, index mismatch %d.\n",
-            BN(j), i, *(unsigned int *)SCpnt->host_scribble);
-
-   sync_dma(i, j);
-
-   if (linked_comm && SCpnt->device->queue_depth > 2
-                                     && TLDEV(SCpnt->device->type))
-      flush_dev(SCpnt->device, blk_rq_pos(SCpnt->request), j, TRUE);
-
-   tstatus = status_byte(spp->target_status);
-
-#if defined(DEBUG_GENERATE_ERRORS)
-   if ((HD(j)->iocount > 500) && ((HD(j)->iocount % 200) < 2))
-                                           spp->adapter_status = 0x01;
-#endif
-
-   switch (spp->adapter_status) {
-      case ASOK:     /* status OK */
-
-         /* Forces a reset if a disk drive keeps returning BUSY */
-         if (tstatus == BUSY && SCpnt->device->type != TYPE_TAPE)
-            status = DID_ERROR << 16;
-
-         /* If there was a bus reset, redo operation on each target */
-         else if (tstatus != GOOD && SCpnt->device->type == TYPE_DISK
-                  && HD(j)->target_redo[scmd_id(SCpnt)][scmd_channel(SCpnt)])
-            status = DID_BUS_BUSY << 16;
-
-         /* Works around a flaw in scsi.c */
-         else if (tstatus == CHECK_CONDITION
-                  && SCpnt->device->type == TYPE_DISK
-                  && (SCpnt->sense_buffer[2] & 0xf) == RECOVERED_ERROR)
-            status = DID_BUS_BUSY << 16;
-
-         else
-            status = DID_OK << 16;
-
-         if (tstatus == GOOD)
-            HD(j)->target_redo[scmd_id(SCpnt)][scmd_channel(SCpnt)] = FALSE;
-
-         if (spp->target_status && SCpnt->device->type == TYPE_DISK &&
-             (!(tstatus == CHECK_CONDITION && HD(j)->iocount <= 1000 &&
-               (SCpnt->sense_buffer[2] & 0xf) == NOT_READY)))
-            scmd_printk(KERN_INFO, SCpnt,
-	    	"ihdlr, target_status 0x%x, sense key 0x%x.\n",
-                   spp->target_status,
-                   SCpnt->sense_buffer[2]);
-
-         HD(j)->target_to[scmd_id(SCpnt)][scmd_channel(SCpnt)] = 0;
-
-         if (HD(j)->last_retried_pid == SCpnt->serial_number) HD(j)->retries = 0;
-
-         break;
-      case ASST:     /* Selection Time Out */
-
-         if (HD(j)->target_to[scmd_id(SCpnt)][scmd_channel(SCpnt)] > 1)
-            status = DID_ERROR << 16;
-         else {
-            status = DID_TIME_OUT << 16;
-            HD(j)->target_to[scmd_id(SCpnt)][scmd_channel(SCpnt)]++;
-            }
-
-         break;
-
-      /* Perform a limited number of internal retries */
-      case 0x93:     /* Unexpected bus free */
-      case 0x94:     /* Target bus phase sequence failure */
-      case 0x96:     /* Illegal SCSI command */
-      case 0xa3:     /* SCSI bus reset error */
-
-         for (c = 0; c <= sh[j]->max_channel; c++)
-            for (k = 0; k < sh[j]->max_id; k++)
-               HD(j)->target_redo[k][c] = TRUE;
-
-
-      case 0x92:     /* Data over/under-run */
-
-         if (SCpnt->device->type != TYPE_TAPE
-             && HD(j)->retries < MAX_INTERNAL_RETRIES) {
-
-#if defined(DID_SOFT_ERROR)
-            status = DID_SOFT_ERROR << 16;
-#else
-            status = DID_BUS_BUSY << 16;
-#endif
-
-            HD(j)->retries++;
-            HD(j)->last_retried_pid = SCpnt->serial_number;
-            }
-         else
-            status = DID_ERROR << 16;
-
-         break;
-      case 0x01:     /* Invalid command */
-      case 0x02:     /* Invalid parameters */
-      case 0x03:     /* Invalid data list */
-      case 0x84:     /* SCSI bus abort error */
-      case 0x9b:     /* Auto request sense error */
-      case 0x9f:     /* Unexpected command complete message error */
-      case 0xff:     /* Invalid parameter in the S/G list */
-      default:
-         status = DID_ERROR << 16;
-         break;
-      }
-
-   SCpnt->result = status | spp->target_status;
-
-#if defined(DEBUG_INTERRUPT)
-   if (SCpnt->result || do_trace)
-#else
-   if ((spp->adapter_status != ASOK && HD(j)->iocount >  1000) ||
-       (spp->adapter_status != ASOK &&
-        spp->adapter_status != ASST && HD(j)->iocount <= 1000) ||
-        do_trace || msg_byte(spp->target_status))
-#endif
-      scmd_printk(KERN_INFO, SCpnt, "ihdlr, mbox %2d, err 0x%x:%x,"\
-             " reg 0x%x, count %d.\n",
-             i, spp->adapter_status, spp->target_status,
-             reg, HD(j)->iocount);
-
-   unmap_dma(i, j);
-
-   /* Set the command state to inactive */
-   SCpnt->host_scribble = NULL;
-
-   SCpnt->scsi_done(SCpnt);
-
-   if (do_trace) printk("%s: ihdlr, exit, irq %d, count %d.\n", BN(j), irq,
-                        HD(j)->iocount);
-
-handled:
-   return IRQ_HANDLED;
-none:
-   return IRQ_NONE;
-}
-
-static irqreturn_t do_interrupt_handler(int irq, void *shap) {
-   unsigned int j;
-   unsigned long spin_flags;
-   irqreturn_t ret;
-
-   /* Check if the interrupt must be processed by this handler */
-   if ((j = (unsigned int)((char *)shap - sha)) >= num_boards) return IRQ_NONE;
-
-   spin_lock_irqsave(sh[j]->host_lock, spin_flags);
-   ret = ihdlr(j);
-   spin_unlock_irqrestore(sh[j]->host_lock, spin_flags);
-   return ret;
-}
-
-static int u14_34f_release(struct Scsi_Host *shpnt) {
-   unsigned int i, j;
-
-   for (j = 0; sh[j] != NULL && sh[j] != shpnt; j++);
-
-   if (sh[j] == NULL)
-      panic("%s: release, invalid Scsi_Host pointer.\n", driver_name);
-
-   for (i = 0; i < sh[j]->can_queue; i++)
-      kfree((&HD(j)->cp[i])->sglist);
-
-   for (i = 0; i < sh[j]->can_queue; i++)
-      pci_unmap_single(HD(j)->pdev, HD(j)->cp[i].cp_dma_addr,
-                     sizeof(struct mscp), PCI_DMA_BIDIRECTIONAL);
-
-   free_irq(sh[j]->irq, &sha[j]);
-
-   if (sh[j]->dma_channel != NO_DMA)
-      free_dma(sh[j]->dma_channel);
-
-   release_region(sh[j]->io_port, sh[j]->n_io_port);
-   scsi_unregister(sh[j]);
-   return FALSE;
-}
-
-#include "scsi_module.c"
-
-#ifndef MODULE
-__setup("u14-34f=", option_setup);
-#endif /* end MODULE */
diff --git a/drivers/scsi/ufs/tc-dwc-g210.c b/drivers/scsi/ufs/tc-dwc-g210.c
index 70db6d999ca3..dc03e47f7c58 100644
--- a/drivers/scsi/ufs/tc-dwc-g210.c
+++ b/drivers/scsi/ufs/tc-dwc-g210.c
@@ -15,6 +15,7 @@
 
 #include "ufshcd-dwc.h"
 #include "ufshci-dwc.h"
+#include "tc-dwc-g210.h"
 
 /**
  * tc_dwc_g210_setup_40bit_rmmi()
diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h
index b291fa6ed2ad..845b874e2977 100644
--- a/drivers/scsi/ufs/ufs.h
+++ b/drivers/scsi/ufs/ufs.h
@@ -327,6 +327,7 @@ enum {
 	MASK_QUERY_DATA_SEG_LEN         = 0xFFFF,
 	MASK_RSP_UPIU_DATA_SEG_LEN	= 0xFFFF,
 	MASK_RSP_EXCEPTION_EVENT        = 0x10000,
+	MASK_TM_SERVICE_RESP		= 0xFF,
 };
 
 /* Task management service response */
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index f08d41a2d70b..37f3c51e9d92 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -2568,7 +2568,7 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
 	status = ufshcd_get_upmcrs(hba);
 	if (status != PWR_LOCAL) {
 		dev_err(hba->dev,
-			"pwr ctrl cmd 0x%0x failed, host umpcrs:0x%x\n",
+			"pwr ctrl cmd 0x%0x failed, host upmcrs:0x%x\n",
 			cmd->command, status);
 		ret = (status != PWR_OK) ? status : -1;
 	}
@@ -3364,8 +3364,8 @@ static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index, u8 *resp)
 	if (ocs_value == OCS_SUCCESS) {
 		task_rsp_upiup = (struct utp_upiu_task_rsp *)
 				task_req_descp[index].task_rsp_upiu;
-		task_result = be32_to_cpu(task_rsp_upiup->header.dword_1);
-		task_result = ((task_result & MASK_TASK_RESPONSE) >> 8);
+		task_result = be32_to_cpu(task_rsp_upiup->output_param1);
+		task_result = task_result & MASK_TM_SERVICE_RESP;
 		if (resp)
 			*resp = (u8)task_result;
 	} else {
diff --git a/drivers/scsi/ultrastor.c b/drivers/scsi/ultrastor.c
deleted file mode 100644
index 14e0c40a68c9..000000000000
--- a/drivers/scsi/ultrastor.c
+++ /dev/null
@@ -1,1210 +0,0 @@
-/*
- *	ultrastor.c	Copyright (C) 1992 David B. Gentzel
- *	Low-level SCSI driver for UltraStor 14F, 24F, and 34F
- *	by David B. Gentzel, Whitfield Software Services, Carnegie, PA
- *	    (gentzel@nova.enet.dec.com)
- *  scatter/gather added by Scott Taylor (n217cg@tamuts.tamu.edu)
- *  24F and multiple command support by John F. Carr (jfc@athena.mit.edu)
- *    John's work modified by Caleb Epstein (cae@jpmorgan.com) and 
- *    Eric Youngdale (ericy@cais.com).
- *	Thanks to UltraStor for providing the necessary documentation
- *
- *  This is an old driver, for the 14F and 34F you should be using the
- *  u14-34f driver instead.
- */
-
-/*
- * TODO:
- *	1. Find out why scatter/gather is limited to 16 requests per command.
- *         This is fixed, at least on the 24F, as of version 1.12 - CAE.
- *	2. Look at command linking (mscp.command_link and
- *	   mscp.command_link_id).  (Does not work with many disks, 
- *				and no performance increase.  ERY).
- *	3. Allow multiple adapters.
- */
-
-/*
- * NOTES:
- *    The UltraStor 14F, 24F, and 34F are a family of intelligent, high
- *    performance SCSI-2 host adapters.  They all support command queueing
- *    and scatter/gather I/O.  Some of them can also emulate the standard
- *    WD1003 interface for use with OS's which don't support SCSI.  Here
- *    is the scoop on the various models:
- *	14F - ISA first-party DMA HA with floppy support and WD1003 emulation.
- *	14N - ISA HA with floppy support.  I think that this is a non-DMA
- *	      HA.  Nothing further known.
- *	24F - EISA Bus Master HA with floppy support and WD1003 emulation.
- *	34F - VL-Bus Bus Master HA with floppy support (no WD1003 emulation).
- *
- *    The 14F, 24F, and 34F are supported by this driver.
- *
- *    Places flagged with a triple question-mark are things which are either
- *    unfinished, questionable, or wrong.
- */
-
-/* Changes from version 1.11 alpha to 1.12
- *
- * Increased the size of the scatter-gather list to 33 entries for
- * the 24F adapter (it was 16).  I don't have the specs for the 14F
- * or the 34F, so they may support larger s-g lists as well.
- *
- * Caleb Epstein <cae@jpmorgan.com>
- */
-
-/* Changes from version 1.9 to 1.11
- *
- * Patches to bring this driver up to speed with the default kernel
- * driver which supports only the 14F and 34F adapters.  This version
- * should compile cleanly into 0.99.13, 0.99.12 and probably 0.99.11.
- *
- * Fixes from Eric Youngdale to fix a few possible race conditions and
- * several problems with bit testing operations (insufficient
- * parentheses).
- *
- * Removed the ultrastor_abort() and ultrastor_reset() functions
- * (enclosed them in #if 0 / #endif).  These functions, at least on
- * the 24F, cause the SCSI bus to do odd things and generally lead to
- * kernel panics and machine hangs.  This is like the Adaptec code.
- *
- * Use check/snarf_region for 14f, 34f to avoid I/O space address conflicts.
- */
-
-/* Changes from version 1.8 to version 1.9
- *
- *  0.99.11 patches (cae@jpmorgan.com) */
-
-/* Changes from version 1.7 to version 1.8
- *
- * Better error reporting.
- */
-
-/* Changes from version 1.6 to version 1.7
- *
- * Removed CSIR command code.
- *
- * Better race condition avoidance (xchgb function added).
- *
- * Set ICM and OGM status to zero at probe (24F)
- *
- * reset sends soft reset to UltraStor adapter
- *
- * reset adapter if adapter interrupts with an invalid MSCP address
- *
- * handle aborted command interrupt (24F)
- *
- */
-
-/* Changes from version 1.5 to version 1.6:
- *
- * Read MSCP address from ICM _before_ clearing the interrupt flag.
- * This fixes a race condition.
- */
-
-/* Changes from version 1.4 to version 1.5:
- *
- * Abort now calls done when multiple commands are enabled.
- *
- * Clear busy when aborted command finishes, not when abort is called.
- *
- * More debugging messages for aborts.
- */
-
-/* Changes from version 1.3 to version 1.4:
- *
- * Enable automatic request of sense data on error (requires newer version
- * of scsi.c to be useful).
- *
- * Fix PORT_OVERRIDE for 14F.
- *
- * Fix abort and reset to work properly (config.aborted wasn't cleared
- * after it was tested, so after a command abort no further commands would
- * work).
- *
- * Boot time test to enable SCSI bus reset (defaults to not allowing reset).
- *
- * Fix test for OGM busy -- the busy bit is in different places on the 24F.
- *
- * Release ICM slot by clearing first byte on 24F.
- */
-
-#include <linux/module.h>
-#include <linux/blkdev.h>
-#include <linux/interrupt.h>
-#include <linux/stddef.h>
-#include <linux/string.h>
-#include <linux/kernel.h>
-#include <linux/ioport.h>
-#include <linux/proc_fs.h>
-#include <linux/spinlock.h>
-#include <linux/stat.h>
-#include <linux/bitops.h>
-#include <linux/delay.h>
-
-#include <asm/io.h>
-#include <asm/dma.h>
-
-#define ULTRASTOR_PRIVATE	/* Get the private stuff from ultrastor.h */
-#include "scsi.h"
-#include <scsi/scsi_host.h>
-#include "ultrastor.h"
-
-#define FALSE 0
-#define TRUE 1
-
-#ifndef ULTRASTOR_DEBUG
-#define ULTRASTOR_DEBUG (UD_ABORT|UD_CSIR|UD_RESET)
-#endif
-
-#define VERSION "1.12"
-
-#define PACKED		__attribute__((packed))
-#define ALIGNED(x)	__attribute__((aligned(x)))
-
-
-/* The 14F uses an array of 4-byte ints for its scatter/gather list.
-   The data can be unaligned, but need not be.  It's easier to give
-   the list normal alignment since it doesn't need to fit into a
-   packed structure.  */
-
-typedef struct {
-  u32 address;
-  u32 num_bytes;
-} ultrastor_sg_list;
-
-
-/* MailBox SCSI Command Packet.  Basic command structure for communicating
-   with controller. */
-struct mscp {
-  unsigned char opcode: 3;		/* type of command */
-  unsigned char xdir: 2;		/* data transfer direction */
-  unsigned char dcn: 1;		/* disable disconnect */
-  unsigned char ca: 1;		/* use cache (if available) */
-  unsigned char sg: 1;		/* scatter/gather operation */
-  unsigned char target_id: 3;		/* target SCSI id */
-  unsigned char ch_no: 2;		/* SCSI channel (always 0 for 14f) */
-  unsigned char lun: 3;		/* logical unit number */
-  unsigned int transfer_data PACKED;	/* transfer data pointer */
-  unsigned int transfer_data_length PACKED;	/* length in bytes */
-  unsigned int command_link PACKED;	/* for linking command chains */
-  unsigned char scsi_command_link_id;	/* identifies command in chain */
-  unsigned char number_of_sg_list;	/* (if sg is set) 8 bytes per list */
-  unsigned char length_of_sense_byte;
-  unsigned char length_of_scsi_cdbs;	/* 6, 10, or 12 */
-  unsigned char scsi_cdbs[12];	/* SCSI commands */
-  unsigned char adapter_status;	/* non-zero indicates HA error */
-  unsigned char target_status;	/* non-zero indicates target error */
-  u32 sense_data PACKED;
-  /* The following fields are for software only.  They are included in
-     the MSCP structure because they are associated with SCSI requests.  */
-  void (*done) (struct scsi_cmnd *);
-  struct scsi_cmnd *SCint;
-  ultrastor_sg_list sglist[ULTRASTOR_24F_MAX_SG]; /* use larger size for 24F */
-};
-
-
-/* Port addresses (relative to the base address) */
-#define U14F_PRODUCT_ID(port) ((port) + 0x4)
-#define CONFIG(port) ((port) + 0x6)
-
-/* Port addresses relative to the doorbell base address.  */
-#define LCL_DOORBELL_MASK(port) ((port) + 0x0)
-#define LCL_DOORBELL_INTR(port) ((port) + 0x1)
-#define SYS_DOORBELL_MASK(port) ((port) + 0x2)
-#define SYS_DOORBELL_INTR(port) ((port) + 0x3)
-
-
-/* Used to store configuration info read from config i/o registers.  Most of
-   this is not used yet, but might as well save it.
-   
-   This structure also holds port addresses that are not at the same offset
-   on the 14F and 24F.
-   
-   This structure holds all data that must be duplicated to support multiple
-   adapters.  */
-
-static struct ultrastor_config
-{
-  unsigned short port_address;		/* base address of card */
-  unsigned short doorbell_address;	/* base address of doorbell CSRs */
-  unsigned short ogm_address;		/* base address of OGM */
-  unsigned short icm_address;		/* base address of ICM */
-  const void *bios_segment;
-  unsigned char interrupt: 4;
-  unsigned char dma_channel: 3;
-  unsigned char bios_drive_number: 1;
-  unsigned char heads;
-  unsigned char sectors;
-  unsigned char ha_scsi_id: 3;
-  unsigned char subversion: 4;
-  unsigned char revision;
-  /* The slot number is used to distinguish the 24F (slot != 0) from
-     the 14F and 34F (slot == 0). */
-  unsigned char slot;
-
-#ifdef PRINT_U24F_VERSION
-  volatile int csir_done;
-#endif
-
-  /* A pool of MSCP structures for this adapter, and a bitmask of
-     busy structures.  (If ULTRASTOR_14F_MAX_CMDS == 1, a 1 byte
-     busy flag is used instead.)  */
-
-#if ULTRASTOR_MAX_CMDS == 1
-  unsigned char mscp_busy;
-#else
-  unsigned long mscp_free;
-#endif
-  volatile unsigned char aborted[ULTRASTOR_MAX_CMDS];
-  struct mscp mscp[ULTRASTOR_MAX_CMDS];
-} config = {0};
-
-/* Set this to 1 to reset the SCSI bus on error.  */
-static int ultrastor_bus_reset;
-
-
-/* Allowed BIOS base addresses (NULL indicates reserved) */
-static const void *const bios_segment_table[8] = {
-  NULL,	     (void *)0xC4000, (void *)0xC8000, (void *)0xCC000,
-  (void *)0xD0000, (void *)0xD4000, (void *)0xD8000, (void *)0xDC000,
-};
-
-/* Allowed IRQs for 14f */
-static const unsigned char interrupt_table_14f[4] = { 15, 14, 11, 10 };
-
-/* Allowed DMA channels for 14f (0 indicates reserved) */
-static const unsigned char dma_channel_table_14f[4] = { 5, 6, 7, 0 };
-
-/* Head/sector mappings allowed by 14f */
-static const struct {
-  unsigned char heads;
-  unsigned char sectors;
-} mapping_table[4] = { { 16, 63 }, { 64, 32 }, { 64, 63 }, { 64, 32 } };
-
-#ifndef PORT_OVERRIDE
-/* ??? A probe of address 0x310 screws up NE2000 cards */
-static const unsigned short ultrastor_ports_14f[] = {
-  0x330, 0x340, /*0x310,*/ 0x230, 0x240, 0x210, 0x130, 0x140,
-};
-#endif
-
-static void ultrastor_interrupt(void *);
-static irqreturn_t do_ultrastor_interrupt(int, void *);
-static inline void build_sg_list(struct mscp *, struct scsi_cmnd *SCpnt);
-
-
-/* Always called with host lock held */
-
-static inline int find_and_clear_bit_16(unsigned long *field)
-{
-  int rv;
-
-  if (*field == 0)
-    panic("No free mscp");
-
-  asm volatile (
-	"xorl %0,%0\n\t"
-	"0: bsfw %1,%w0\n\t"
-	"btr %0,%1\n\t"
-	"jnc 0b"
-	: "=&r" (rv), "+m" (*field) :);
-
-  return rv;
-}
-
-/* This has been re-implemented with the help of Richard Earnshaw,
-   <rwe@pegasus.esprit.ec.org> and works with gcc-2.5.8 and gcc-2.6.0.
-   The instability noted by jfc below appears to be a bug in
-   gcc-2.5.x when compiling w/o optimization.  --Caleb
-
-   This asm is fragile: it doesn't work without the casts and it may
-   not work without optimization.  Maybe I should add a swap builtin
-   to gcc.  --jfc  */
-static inline unsigned char xchgb(unsigned char reg,
-				  volatile unsigned char *mem)
-{
-  __asm__ ("xchgb %0,%1" : "=q" (reg), "=m" (*mem) : "0" (reg));
-  return reg;
-}
-
-#if ULTRASTOR_DEBUG & (UD_COMMAND | UD_ABORT)
-
-/* Always called with the host lock held */
-static void log_ultrastor_abort(struct ultrastor_config *config,
-				int command)
-{
-  static char fmt[80] = "abort %d (%x); MSCP free pool: %x;";
-  int i;
-
-  for (i = 0; i < ULTRASTOR_MAX_CMDS; i++)
-    {
-      fmt[20 + i*2] = ' ';
-      if (! (config->mscp_free & (1 << i)))
-	fmt[21 + i*2] = '0' + config->mscp[i].target_id;
-      else
-	fmt[21 + i*2] = '-';
-    }
-  fmt[20 + ULTRASTOR_MAX_CMDS * 2] = '\n';
-  fmt[21 + ULTRASTOR_MAX_CMDS * 2] = 0;
-  printk(fmt, command, &config->mscp[command], config->mscp_free);
-
-}
-#endif
-
-static int ultrastor_14f_detect(struct scsi_host_template * tpnt)
-{
-    size_t i;
-    unsigned char in_byte, version_byte = 0;
-    struct config_1 {
-      unsigned char bios_segment: 3;
-      unsigned char removable_disks_as_fixed: 1;
-      unsigned char interrupt: 2;
-    unsigned char dma_channel: 2;
-    } config_1;
-    struct config_2 {
-      unsigned char ha_scsi_id: 3;
-      unsigned char mapping_mode: 2;
-      unsigned char bios_drive_number: 1;
-      unsigned char tfr_port: 2;
-    } config_2;
-
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-    printk("US14F: detect: called\n");
-#endif
-
-    /* If a 24F has already been configured, don't look for a 14F.  */
-    if (config.bios_segment)
-	return FALSE;
-
-#ifdef PORT_OVERRIDE
-    if(!request_region(PORT_OVERRIDE, 0xc, "ultrastor")) {
-      printk("Ultrastor I/O space already in use\n");
-      return FALSE;
-    };
-    config.port_address = PORT_OVERRIDE;
-#else
-    for (i = 0; i < ARRAY_SIZE(ultrastor_ports_14f); i++) {
-      if(!request_region(ultrastor_ports_14f[i], 0x0c, "ultrastor")) continue;
-      config.port_address = ultrastor_ports_14f[i];
-#endif
-
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-	printk("US14F: detect: testing port address %03X\n", config.port_address);
-#endif
-
-	in_byte = inb(U14F_PRODUCT_ID(config.port_address));
-	if (in_byte != US14F_PRODUCT_ID_0) {
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-# ifdef PORT_OVERRIDE
-	    printk("US14F: detect: wrong product ID 0 - %02X\n", in_byte);
-# else
-	    printk("US14F: detect: no adapter at port %03X\n", config.port_address);
-# endif
-#endif
-#ifdef PORT_OVERRIDE
-	    goto out_release_port;
-#else
-	    release_region(config.port_address, 0x0c);
-	    continue;
-#endif
-	}
-	in_byte = inb(U14F_PRODUCT_ID(config.port_address) + 1);
-	/* Only upper nibble is significant for Product ID 1 */
-	if ((in_byte & 0xF0) != US14F_PRODUCT_ID_1) {
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-# ifdef PORT_OVERRIDE
-	    printk("US14F: detect: wrong product ID 1 - %02X\n", in_byte);
-# else
-	    printk("US14F: detect: no adapter at port %03X\n", config.port_address);
-# endif
-#endif
-#ifdef PORT_OVERRIDE
-	    goto out_release_port;
-#else
-	    release_region(config.port_address, 0x0c);
-	    continue;
-#endif
-	}
-	version_byte = in_byte;
-#ifndef PORT_OVERRIDE
-	break;
-    }
-    if (i == ARRAY_SIZE(ultrastor_ports_14f)) {
-# if (ULTRASTOR_DEBUG & UD_DETECT)
-	printk("US14F: detect: no port address found!\n");
-# endif
-	/* all ports probed already released - we can just go straight out */
-	return FALSE;
-    }
-#endif
-
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-    printk("US14F: detect: adapter found at port address %03X\n",
-	   config.port_address);
-#endif
-
-    /* Set local doorbell mask to disallow bus reset unless
-       ultrastor_bus_reset is true.  */
-    outb(ultrastor_bus_reset ? 0xc2 : 0x82, LCL_DOORBELL_MASK(config.port_address));
-
-    /* All above tests passed, must be the right thing.  Get some useful
-       info. */
-
-    /* Register the I/O space that we use */
-
-    *(char *)&config_1 = inb(CONFIG(config.port_address + 0));
-    *(char *)&config_2 = inb(CONFIG(config.port_address + 1));
-    config.bios_segment = bios_segment_table[config_1.bios_segment];
-    config.doorbell_address = config.port_address;
-    config.ogm_address = config.port_address + 0x8;
-    config.icm_address = config.port_address + 0xC;
-    config.interrupt = interrupt_table_14f[config_1.interrupt];
-    config.ha_scsi_id = config_2.ha_scsi_id;
-    config.heads = mapping_table[config_2.mapping_mode].heads;
-    config.sectors = mapping_table[config_2.mapping_mode].sectors;
-    config.bios_drive_number = config_2.bios_drive_number;
-    config.subversion = (version_byte & 0x0F);
-    if (config.subversion == U34F)
-	config.dma_channel = 0;
-    else
-	config.dma_channel = dma_channel_table_14f[config_1.dma_channel];
-
-    if (!config.bios_segment) {
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-	printk("US14F: detect: not detected.\n");
-#endif
-	goto out_release_port;
-    }
-
-    /* Final consistency check, verify previous info. */
-    if (config.subversion != U34F)
-	if (!config.dma_channel || !(config_2.tfr_port & 0x2)) {
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-	    printk("US14F: detect: consistency check failed\n");
-#endif
-           goto out_release_port;
-	}
-
-    /* If we were TRULY paranoid, we could issue a host adapter inquiry
-       command here and verify the data returned.  But frankly, I'm
-       exhausted! */
-
-    /* Finally!  Now I'm satisfied... */
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-    printk("US14F: detect: detect succeeded\n"
-	   "  Port address: %03X\n"
-	   "  BIOS segment: %05X\n"
-	   "  Interrupt: %u\n"
-	   "  DMA channel: %u\n"
-	   "  H/A SCSI ID: %u\n"
-	   "  Subversion: %u\n",
-	   config.port_address, config.bios_segment, config.interrupt,
-	   config.dma_channel, config.ha_scsi_id, config.subversion);
-#endif
-    tpnt->this_id = config.ha_scsi_id;
-    tpnt->unchecked_isa_dma = (config.subversion != U34F);
-
-#if ULTRASTOR_MAX_CMDS > 1
-    config.mscp_free = ~0;
-#endif
-
-    /*
-     * Brrr, &config.mscp[0].SCint->host) it is something magical....
-     * XXX and FIXME
-     */
-    if (request_irq(config.interrupt, do_ultrastor_interrupt, 0, "Ultrastor", &config.mscp[0].SCint->device->host)) {
-	printk("Unable to allocate IRQ%u for UltraStor controller.\n",
-	       config.interrupt);
-	goto out_release_port;
-    }
-    if (config.dma_channel && request_dma(config.dma_channel,"Ultrastor")) {
-	printk("Unable to allocate DMA channel %u for UltraStor controller.\n",
-	       config.dma_channel);
-	free_irq(config.interrupt, NULL);
-	goto out_release_port;
-    }
-    tpnt->sg_tablesize = ULTRASTOR_14F_MAX_SG;
-    printk("UltraStor driver version" VERSION ".  Using %d SG lists.\n",
-	   ULTRASTOR_14F_MAX_SG);
-
-    return TRUE;
-out_release_port:
-    release_region(config.port_address, 0x0c);
-    return FALSE;
-}
-
-static int ultrastor_24f_detect(struct scsi_host_template * tpnt)
-{
-  int i;
-  struct Scsi_Host * shpnt = NULL;
-
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-  printk("US24F: detect");
-#endif
-
-  /* probe each EISA slot at slot address C80 */
-  for (i = 1; i < 15; i++)
-    {
-      unsigned char config_1, config_2;
-      unsigned short addr = (i << 12) | ULTRASTOR_24F_PORT;
-
-      if (inb(addr) != US24F_PRODUCT_ID_0 &&
-	  inb(addr+1) != US24F_PRODUCT_ID_1 &&
-	  inb(addr+2) != US24F_PRODUCT_ID_2)
-	continue;
-
-      config.revision = inb(addr+3);
-      config.slot = i;
-      if (! (inb(addr+4) & 1))
-	{
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-	  printk("U24F: found disabled card in slot %u\n", i);
-#endif
-	  continue;
-	}
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-      printk("U24F: found card in slot %u\n", i);
-#endif
-      config_1 = inb(addr + 5);
-      config.bios_segment = bios_segment_table[config_1 & 7];
-      switch(config_1 >> 4)
-	{
-	case 1:
-	  config.interrupt = 15;
-	  break;
-	case 2:
-	  config.interrupt = 14;
-	  break;
-	case 4:
-	  config.interrupt = 11;
-	  break;
-	case 8:
-	  config.interrupt = 10;
-	  break;
-	default:
-	  printk("U24F: invalid IRQ\n");
-	  return FALSE;
-	}
-
-      /* BIOS addr set */
-      /* base port set */
-      config.port_address = addr;
-      config.doorbell_address = addr + 12;
-      config.ogm_address = addr + 0x17;
-      config.icm_address = addr + 0x1C;
-      config_2 = inb(addr + 7);
-      config.ha_scsi_id = config_2 & 7;
-      config.heads = mapping_table[(config_2 >> 3) & 3].heads;
-      config.sectors = mapping_table[(config_2 >> 3) & 3].sectors;
-#if (ULTRASTOR_DEBUG & UD_DETECT)
-      printk("US24F: detect: detect succeeded\n"
-	     "  Port address: %03X\n"
-	     "  BIOS segment: %05X\n"
-	     "  Interrupt: %u\n"
-	     "  H/A SCSI ID: %u\n",
-	     config.port_address, config.bios_segment,
-	     config.interrupt, config.ha_scsi_id);
-#endif
-      tpnt->this_id = config.ha_scsi_id;
-      tpnt->unchecked_isa_dma = 0;
-      tpnt->sg_tablesize = ULTRASTOR_24F_MAX_SG;
-
-      shpnt = scsi_register(tpnt, 0);
-      if (!shpnt) {
-             printk(KERN_WARNING "(ultrastor:) Could not register scsi device. Aborting registration.\n");
-             free_irq(config.interrupt, do_ultrastor_interrupt);
-             return FALSE;
-      }
-      
-      if (request_irq(config.interrupt, do_ultrastor_interrupt, 0, "Ultrastor", shpnt))
-	{
-	  printk("Unable to allocate IRQ%u for UltraStor controller.\n",
-		 config.interrupt);
-	  return FALSE;
-	}
-
-      shpnt->irq = config.interrupt;
-      shpnt->dma_channel = config.dma_channel;
-      shpnt->io_port = config.port_address;
-
-#if ULTRASTOR_MAX_CMDS > 1
-      config.mscp_free = ~0;
-#endif
-      /* Mark ICM and OGM free */
-      outb(0, addr + 0x16);
-      outb(0, addr + 0x1B);
-
-      /* Set local doorbell mask to disallow bus reset unless
-	 ultrastor_bus_reset is true.  */
-      outb(ultrastor_bus_reset ? 0xc2 : 0x82, LCL_DOORBELL_MASK(addr+12));
-      outb(0x02, SYS_DOORBELL_MASK(addr+12));
-      printk("UltraStor driver version " VERSION ".  Using %d SG lists.\n",
-	     tpnt->sg_tablesize);
-      return TRUE;
-    }
-  return FALSE;
-}
-
-static int ultrastor_detect(struct scsi_host_template * tpnt)
-{
-	tpnt->proc_name = "ultrastor";
-	return ultrastor_14f_detect(tpnt) || ultrastor_24f_detect(tpnt);
-}
-
-static int ultrastor_release(struct Scsi_Host *shost)
-{
-	if (shost->irq)
-		free_irq(shost->irq, NULL);
-	if (shost->dma_channel != 0xff)
-		free_dma(shost->dma_channel);
-	if (shost->io_port && shost->n_io_port)
-		release_region(shost->io_port, shost->n_io_port);
-	scsi_unregister(shost);
-	return 0;
-}
-
-static const char *ultrastor_info(struct Scsi_Host * shpnt)
-{
-    static char buf[64];
-
-    if (config.slot)
-      sprintf(buf, "UltraStor 24F SCSI @ Slot %u IRQ%u",
-	      config.slot, config.interrupt);
-    else if (config.subversion)
-      sprintf(buf, "UltraStor 34F SCSI @ Port %03X BIOS %05X IRQ%u",
-	      config.port_address, (int)config.bios_segment,
-	      config.interrupt);
-    else
-      sprintf(buf, "UltraStor 14F SCSI @ Port %03X BIOS %05X IRQ%u DMA%u",
-	      config.port_address, (int)config.bios_segment,
-	      config.interrupt, config.dma_channel);
-    return buf;
-}
-
-static inline void build_sg_list(struct mscp *mscp, struct scsi_cmnd *SCpnt)
-{
-	struct scatterlist *sg;
-	long transfer_length = 0;
-	int i, max;
-
-	max = scsi_sg_count(SCpnt);
-	scsi_for_each_sg(SCpnt, sg, max, i) {
-		mscp->sglist[i].address = isa_page_to_bus(sg_page(sg)) + sg->offset;
-		mscp->sglist[i].num_bytes = sg->length;
-		transfer_length += sg->length;
-	}
-	mscp->number_of_sg_list = max;
-	mscp->transfer_data = isa_virt_to_bus(mscp->sglist);
-	/* ??? May not be necessary.  Docs are unclear as to whether transfer
-	   length field is ignored or whether it should be set to the total
-	   number of bytes of the transfer.  */
-	mscp->transfer_data_length = transfer_length;
-}
-
-static int ultrastor_queuecommand_lck(struct scsi_cmnd *SCpnt,
-				void (*done) (struct scsi_cmnd *))
-{
-    struct mscp *my_mscp;
-#if ULTRASTOR_MAX_CMDS > 1
-    int mscp_index;
-#endif
-    unsigned int status;
-
-    /* Next test is for debugging; "can't happen" */
-    if ((config.mscp_free & ((1U << ULTRASTOR_MAX_CMDS) - 1)) == 0)
-	panic("ultrastor_queuecommand: no free MSCP\n");
-    mscp_index = find_and_clear_bit_16(&config.mscp_free);
-
-    /* Has the command been aborted?  */
-    if (xchgb(0xff, &config.aborted[mscp_index]) != 0)
-      {
-	status = DID_ABORT << 16;
-	goto aborted;
-      }
-
-    my_mscp = &config.mscp[mscp_index];
-
-    *(unsigned char *)my_mscp = OP_SCSI | (DTD_SCSI << 3);
-
-    /* Tape drives don't work properly if the cache is used.  The SCSI
-       READ command for a tape doesn't have a block offset, and the adapter
-       incorrectly assumes that all reads from the tape read the same
-       blocks.  Results will depend on read buffer size and other disk
-       activity. 
-
-       ???  Which other device types should never use the cache?   */
-    my_mscp->ca = SCpnt->device->type != TYPE_TAPE;
-    my_mscp->target_id = SCpnt->device->id;
-    my_mscp->ch_no = 0;
-    my_mscp->lun = SCpnt->device->lun;
-    if (scsi_sg_count(SCpnt)) {
-	/* Set scatter/gather flag in SCSI command packet */
-	my_mscp->sg = TRUE;
-	build_sg_list(my_mscp, SCpnt);
-    } else {
-	/* Unset scatter/gather flag in SCSI command packet */
-	my_mscp->sg = FALSE;
-	my_mscp->transfer_data = isa_virt_to_bus(scsi_sglist(SCpnt));
-	my_mscp->transfer_data_length = scsi_bufflen(SCpnt);
-    }
-    my_mscp->command_link = 0;		/*???*/
-    my_mscp->scsi_command_link_id = 0;	/*???*/
-    my_mscp->length_of_sense_byte = SCSI_SENSE_BUFFERSIZE;
-    my_mscp->length_of_scsi_cdbs = SCpnt->cmd_len;
-    memcpy(my_mscp->scsi_cdbs, SCpnt->cmnd, my_mscp->length_of_scsi_cdbs);
-    my_mscp->adapter_status = 0;
-    my_mscp->target_status = 0;
-    my_mscp->sense_data = isa_virt_to_bus(&SCpnt->sense_buffer);
-    my_mscp->done = done;
-    my_mscp->SCint = SCpnt;
-    SCpnt->host_scribble = (unsigned char *)my_mscp;
-
-    /* Find free OGM slot.  On 24F, look for OGM status byte == 0.
-       On 14F and 34F, wait for local interrupt pending flag to clear. 
-       
-       FIXME: now we are using new_eh we should punt here and let the
-       midlayer sort it out */
-
-retry:
-    if (config.slot)
-	while (inb(config.ogm_address - 1) != 0 && config.aborted[mscp_index] == 0xff)
-		barrier();
-
-    /* else??? */
-
-    while ((inb(LCL_DOORBELL_INTR(config.doorbell_address)) & (config.slot ? 2 : 1))  && config.aborted[mscp_index] == 0xff)
-    	barrier();
-
-    /* To avoid race conditions, keep the code to write to the adapter
-       atomic.  This simplifies the abort code.  Right now the
-       scsi mid layer has the host_lock already held
-     */
-
-    if (inb(LCL_DOORBELL_INTR(config.doorbell_address)) & (config.slot ? 2 : 1))
-      goto retry;
-
-    status = xchgb(0, &config.aborted[mscp_index]);
-    if (status != 0xff) {
-
-#if ULTRASTOR_DEBUG & (UD_COMMAND | UD_ABORT)
-	printk("USx4F: queuecommand: aborted\n");
-#if ULTRASTOR_MAX_CMDS > 1
-	log_ultrastor_abort(&config, mscp_index);
-#endif
-#endif
-	status <<= 16;
-
-      aborted:
-	set_bit(mscp_index, &config.mscp_free);
-	/* If the driver queues commands, call the done proc here.  Otherwise
-	   return an error.  */
-#if ULTRASTOR_MAX_CMDS > 1
-	SCpnt->result = status;
-	done(SCpnt);
-	return 0;
-#else
-	return status;
-#endif
-    }
-
-    /* Store pointer in OGM address bytes */
-    outl(isa_virt_to_bus(my_mscp), config.ogm_address);
-
-    /* Issue OGM interrupt */
-    if (config.slot) {
-	/* Write OGM command register on 24F */
-	outb(1, config.ogm_address - 1);
-	outb(0x2, LCL_DOORBELL_INTR(config.doorbell_address));
-    } else {
-	outb(0x1, LCL_DOORBELL_INTR(config.doorbell_address));
-    }
-
-#if (ULTRASTOR_DEBUG & UD_COMMAND)
-    printk("USx4F: queuecommand: returning\n");
-#endif
-
-    return 0;
-}
-
-static DEF_SCSI_QCMD(ultrastor_queuecommand)
-
-/* This code must deal with 2 cases:
-
-   1. The command has not been written to the OGM.  In this case, set
-   the abort flag and return.
-
-   2. The command has been written to the OGM and is stuck somewhere in
-   the adapter.
-
-   2a.  On a 24F, ask the adapter to abort the command.  It will interrupt
-   when it does.
-
-   2b.  Call the command's done procedure.
-
- */
-
-static int ultrastor_abort(struct scsi_cmnd *SCpnt)
-{
-#if ULTRASTOR_DEBUG & UD_ABORT
-    char out[108];
-    unsigned char icm_status = 0, ogm_status = 0;
-    unsigned int icm_addr = 0, ogm_addr = 0;
-#endif
-    unsigned int mscp_index;
-    unsigned char old_aborted;
-    unsigned long flags;
-    void (*done)(struct scsi_cmnd *);
-    struct Scsi_Host *host = SCpnt->device->host;
-
-    if(config.slot) 
-      return FAILED;  /* Do not attempt an abort for the 24f */
-      
-    /* Simple consistency checking */
-    if(!SCpnt->host_scribble)
-      return FAILED;
-
-    mscp_index = ((struct mscp *)SCpnt->host_scribble) - config.mscp;
-    if (mscp_index >= ULTRASTOR_MAX_CMDS)
-	panic("Ux4F aborting invalid MSCP");
-
-#if ULTRASTOR_DEBUG & UD_ABORT
-    if (config.slot)
-      {
-	int port0 = (config.slot << 12) | 0xc80;
-	int i;
-	unsigned long flags;
-	
-	spin_lock_irqsave(host->host_lock, flags);
-	strcpy(out, "OGM %d:%x ICM %d:%x ports:  ");
-	for (i = 0; i < 16; i++)
-	  {
-	    unsigned char p = inb(port0 + i);
-	    out[28 + i * 3] = "0123456789abcdef"[p >> 4];
-	    out[29 + i * 3] = "0123456789abcdef"[p & 15];
-	    out[30 + i * 3] = ' ';
-	  }
-	out[28 + i * 3] = '\n';
-	out[29 + i * 3] = 0;
-	ogm_status = inb(port0 + 22);
-	ogm_addr = (unsigned int)isa_bus_to_virt(inl(port0 + 23));
-	icm_status = inb(port0 + 27);
-	icm_addr = (unsigned int)isa_bus_to_virt(inl(port0 + 28));
-	spin_unlock_irqrestore(host->host_lock, flags);
-      }
-
-    /* First check to see if an interrupt is pending.  I suspect the SiS
-       chipset loses interrupts.  (I also suspect is mangles data, but
-       one bug at a time... */
-    if (config.slot ? inb(config.icm_address - 1) == 2 :
-	(inb(SYS_DOORBELL_INTR(config.doorbell_address)) & 1))
-      {
-	printk("Ux4F: abort while completed command pending\n");
-	
-	spin_lock_irqsave(host->host_lock, flags);
-	/* FIXME: Ewww... need to think about passing host around properly */
-	ultrastor_interrupt(NULL);
-	spin_unlock_irqrestore(host->host_lock, flags);
-	return SUCCESS;
-      }
-#endif
-
-    old_aborted = xchgb(DID_ABORT, &config.aborted[mscp_index]);
-
-    /* aborted == 0xff is the signal that queuecommand has not yet sent
-       the command.  It will notice the new abort flag and fail.  */
-    if (old_aborted == 0xff)
-	return SUCCESS;
-
-    /* On 24F, send an abort MSCP request.  The adapter will interrupt
-       and the interrupt handler will call done.  */
-    if (config.slot && inb(config.ogm_address - 1) == 0)
-      {
-	unsigned long flags;
-
-	spin_lock_irqsave(host->host_lock, flags);
-	outl(isa_virt_to_bus(&config.mscp[mscp_index]), config.ogm_address);
-	udelay(8);
-	outb(0x80, config.ogm_address - 1);
-	outb(0x2, LCL_DOORBELL_INTR(config.doorbell_address));
-#if ULTRASTOR_DEBUG & UD_ABORT
-	log_ultrastor_abort(&config, mscp_index);
-	printk(out, ogm_status, ogm_addr, icm_status, icm_addr);
-#endif
-	spin_unlock_irqrestore(host->host_lock, flags);
-	/* FIXME: add a wait for the abort to complete */
-	return SUCCESS;
-      }
-
-#if ULTRASTOR_DEBUG & UD_ABORT
-    log_ultrastor_abort(&config, mscp_index);
-#endif
-
-    /* Can't request a graceful abort.  Either this is not a 24F or
-       the OGM is busy.  Don't free the command -- the adapter might
-       still be using it.  Setting SCint = 0 causes the interrupt
-       handler to ignore the command.  */
-
-    /* FIXME - devices that implement soft resets will still be running
-       the command after a bus reset.  We would probably rather leave
-       the command in the queue.  The upper level code will automatically
-       leave the command in the active state instead of requeueing it. ERY */
-
-#if ULTRASTOR_DEBUG & UD_ABORT
-    if (config.mscp[mscp_index].SCint != SCpnt)
-	printk("abort: command mismatch, %p != %p\n",
-	       config.mscp[mscp_index].SCint, SCpnt);
-#endif
-    if (config.mscp[mscp_index].SCint == NULL)
-	return FAILED;
-
-    if (config.mscp[mscp_index].SCint != SCpnt) panic("Bad abort");
-    config.mscp[mscp_index].SCint = NULL;
-    done = config.mscp[mscp_index].done;
-    config.mscp[mscp_index].done = NULL;
-    SCpnt->result = DID_ABORT << 16;
-    
-    /* Take the host lock to guard against scsi layer re-entry */
-    done(SCpnt);
-
-    /* Need to set a timeout here in case command never completes.  */
-    return SUCCESS;
-}
-
-static int ultrastor_host_reset(struct scsi_cmnd * SCpnt)
-{
-    unsigned long flags;
-    int i;
-    struct Scsi_Host *host = SCpnt->device->host;
-    
-#if (ULTRASTOR_DEBUG & UD_RESET)
-    printk("US14F: reset: called\n");
-#endif
-
-    if(config.slot)
-    	return FAILED;
-
-    spin_lock_irqsave(host->host_lock, flags);
-    /* Reset the adapter and SCSI bus.  The SCSI bus reset can be
-       inhibited by clearing ultrastor_bus_reset before probe.  */
-    outb(0xc0, LCL_DOORBELL_INTR(config.doorbell_address));
-    if (config.slot)
-      {
-	outb(0, config.ogm_address - 1);
-	outb(0, config.icm_address - 1);
-      }
-
-#if ULTRASTOR_MAX_CMDS == 1
-    if (config.mscp_busy && config.mscp->done && config.mscp->SCint)
-      {
-	config.mscp->SCint->result = DID_RESET << 16;
-	config.mscp->done(config.mscp->SCint);
-      }
-    config.mscp->SCint = 0;
-#else
-    for (i = 0; i < ULTRASTOR_MAX_CMDS; i++)
-      {
-	if (! (config.mscp_free & (1 << i)) &&
-	    config.mscp[i].done && config.mscp[i].SCint)
-	  {
-	    config.mscp[i].SCint->result = DID_RESET << 16;
-	    config.mscp[i].done(config.mscp[i].SCint);
-	    config.mscp[i].done = NULL;
-	  }
-	config.mscp[i].SCint = NULL;
-      }
-#endif
-
-    /* FIXME - if the device implements soft resets, then the command
-       will still be running.  ERY  
-       
-       Even bigger deal with new_eh! 
-     */
-
-    memset((unsigned char *)config.aborted, 0, sizeof config.aborted);
-#if ULTRASTOR_MAX_CMDS == 1
-    config.mscp_busy = 0;
-#else
-    config.mscp_free = ~0;
-#endif
-
-    spin_unlock_irqrestore(host->host_lock, flags);
-    return SUCCESS;
-
-}
-
-int ultrastor_biosparam(struct scsi_device *sdev, struct block_device *bdev,
-		sector_t capacity, int * dkinfo)
-{
-    int size = capacity;
-    unsigned int s = config.heads * config.sectors;
-
-    dkinfo[0] = config.heads;
-    dkinfo[1] = config.sectors;
-    dkinfo[2] = size / s;	/* Ignore partial cylinders */
-#if 0
-    if (dkinfo[2] > 1024)
-	dkinfo[2] = 1024;
-#endif
-    return 0;
-}
-
-static void ultrastor_interrupt(void *dev_id)
-{
-    unsigned int status;
-#if ULTRASTOR_MAX_CMDS > 1
-    unsigned int mscp_index;
-#endif
-    struct mscp *mscp;
-    void (*done) (struct scsi_cmnd *);
-    struct scsi_cmnd *SCtmp;
-
-#if ULTRASTOR_MAX_CMDS == 1
-    mscp = &config.mscp[0];
-#else
-    mscp = (struct mscp *)isa_bus_to_virt(inl(config.icm_address));
-    mscp_index = mscp - config.mscp;
-    if (mscp_index >= ULTRASTOR_MAX_CMDS) {
-	printk("Ux4F interrupt: bad MSCP address %x\n", (unsigned int) mscp);
-	/* A command has been lost.  Reset and report an error
-	   for all commands.  */
-	ultrastor_host_reset(dev_id);
-	return;
-    }
-#endif
-
-    /* Clean ICM slot (set ICMINT bit to 0) */
-    if (config.slot) {
-	unsigned char icm_status = inb(config.icm_address - 1);
-#if ULTRASTOR_DEBUG & (UD_INTERRUPT|UD_ERROR|UD_ABORT)
-	if (icm_status != 1 && icm_status != 2)
-	    printk("US24F: ICM status %x for MSCP %d (%x)\n", icm_status,
-		   mscp_index, (unsigned int) mscp);
-#endif
-	/* The manual says clear interrupt then write 0 to ICM status.
-	   This seems backwards, but I'll do it anyway.  --jfc */
-	outb(2, SYS_DOORBELL_INTR(config.doorbell_address));
-	outb(0, config.icm_address - 1);
-	if (icm_status == 4) {
-	    printk("UltraStor abort command failed\n");
-	    return;
-	}
-	if (icm_status == 3) {
-	    void (*done)(struct scsi_cmnd *) = mscp->done;
-	    if (done) {
-		mscp->done = NULL;
-		mscp->SCint->result = DID_ABORT << 16;
-		done(mscp->SCint);
-	    }
-	    return;
-	}
-    } else {
-	outb(1, SYS_DOORBELL_INTR(config.doorbell_address));
-    }
-
-    SCtmp = mscp->SCint;
-    mscp->SCint = NULL;
-
-    if (!SCtmp)
-      {
-#if ULTRASTOR_DEBUG & (UD_ABORT|UD_INTERRUPT)
-	printk("MSCP %d (%x): no command\n", mscp_index, (unsigned int) mscp);
-#endif	
-#if ULTRASTOR_MAX_CMDS == 1
-	config.mscp_busy = FALSE;
-#else
-	set_bit(mscp_index, &config.mscp_free);
-#endif
-	config.aborted[mscp_index] = 0;
-	return;
-      }
-
-    /* Save done locally and zero before calling.  This is needed as
-       once we call done, we may get another command queued before this
-       interrupt service routine can return. */
-    done = mscp->done;
-    mscp->done = NULL;
-
-    /* Let the higher levels know that we're done */
-    switch (mscp->adapter_status)
-      {
-      case 0:
-	status = DID_OK << 16;
-	break;
-      case 0x01:	/* invalid command */
-      case 0x02:	/* invalid parameters */
-      case 0x03:	/* invalid data list */
-      default:
-	status = DID_ERROR << 16;
-	break;
-      case 0x84:	/* SCSI bus abort */
-	status = DID_ABORT << 16;
-	break;
-      case 0x91:
-	status = DID_TIME_OUT << 16;
-	break;
-      }
-
-    SCtmp->result = status | mscp->target_status;
-
-    SCtmp->host_scribble = NULL;
-
-    /* Free up mscp block for next command */
-#if ULTRASTOR_MAX_CMDS == 1
-    config.mscp_busy = FALSE;
-#else
-    set_bit(mscp_index, &config.mscp_free);
-#endif
-
-#if ULTRASTOR_DEBUG & (UD_ABORT|UD_INTERRUPT)
-    if (config.aborted[mscp_index])
-	printk("Ux4 interrupt: MSCP %d (%x) aborted = %d\n",
-	       mscp_index, (unsigned int) mscp, config.aborted[mscp_index]);
-#endif
-    config.aborted[mscp_index] = 0;
-
-    if (done)
-	done(SCtmp);
-    else
-	printk("US14F: interrupt: unexpected interrupt\n");
-
-    if (config.slot ? inb(config.icm_address - 1) :
-       (inb(SYS_DOORBELL_INTR(config.doorbell_address)) & 1))
-#if (ULTRASTOR_DEBUG & UD_MULTI_CMD)
-      printk("Ux4F: multiple commands completed\n");
-#else
-      ;
-#endif
-
-#if (ULTRASTOR_DEBUG & UD_INTERRUPT)
-    printk("USx4F: interrupt: returning\n");
-#endif
-}
-
-static irqreturn_t do_ultrastor_interrupt(int irq, void *dev_id)
-{
-    unsigned long flags;
-    struct Scsi_Host *dev = dev_id;
-    
-    spin_lock_irqsave(dev->host_lock, flags);
-    ultrastor_interrupt(dev_id);
-    spin_unlock_irqrestore(dev->host_lock, flags);
-    return IRQ_HANDLED;
-}
-
-MODULE_LICENSE("GPL");
-
-static struct scsi_host_template driver_template = {
-	.name              = "UltraStor 14F/24F/34F",
-	.detect            = ultrastor_detect,
-	.release	   = ultrastor_release,
-	.info              = ultrastor_info,
-	.queuecommand      = ultrastor_queuecommand,
-	.eh_abort_handler  = ultrastor_abort,
-	.eh_host_reset_handler  = ultrastor_host_reset,	
-	.bios_param        = ultrastor_biosparam,
-	.can_queue         = ULTRASTOR_MAX_CMDS,
-	.sg_tablesize      = ULTRASTOR_14F_MAX_SG,
-	.cmd_per_lun       = ULTRASTOR_MAX_CMDS_PER_LUN,
-	.unchecked_isa_dma = 1,
-	.use_clustering    = ENABLE_CLUSTERING,
-};
-#include "scsi_module.c"
diff --git a/drivers/scsi/ultrastor.h b/drivers/scsi/ultrastor.h
deleted file mode 100644
index 165c18b5cf5f..000000000000
--- a/drivers/scsi/ultrastor.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- *	ultrastor.c	(C) 1991 David B. Gentzel
- *	Low-level scsi driver for UltraStor 14F
- *	by David B. Gentzel, Whitfield Software Services, Carnegie, PA
- *	    (gentzel@nova.enet.dec.com)
- *  scatter/gather added by Scott Taylor (n217cg@tamuts.tamu.edu)
- *  24F support by John F. Carr (jfc@athena.mit.edu)
- *    John's work modified by Caleb Epstein (cae@jpmorgan.com) and 
- *    Eric Youngdale (eric@tantalus.nrl.navy.mil).
- *	Thanks to UltraStor for providing the necessary documentation
- */
-
-#ifndef _ULTRASTOR_H
-#define _ULTRASTOR_H
-
-static int ultrastor_detect(struct scsi_host_template *);
-static const char *ultrastor_info(struct Scsi_Host *shpnt);
-static int ultrastor_queuecommand(struct Scsi_Host *, struct scsi_cmnd *);
-static int ultrastor_abort(struct scsi_cmnd *);
-static int ultrastor_host_reset(struct scsi_cmnd *);
-static int ultrastor_biosparam(struct scsi_device *, struct block_device *,
-				sector_t, int *);
-
-
-#define ULTRASTOR_14F_MAX_SG 16
-#define ULTRASTOR_24F_MAX_SG 33
-
-#define ULTRASTOR_MAX_CMDS_PER_LUN 5
-#define ULTRASTOR_MAX_CMDS 16
-
-#define ULTRASTOR_24F_PORT 0xC80
-
-
-#ifdef ULTRASTOR_PRIVATE
-
-#define UD_ABORT	0x0001
-#define UD_COMMAND	0x0002
-#define UD_DETECT	0x0004
-#define UD_INTERRUPT	0x0008
-#define UD_RESET	0x0010
-#define UD_MULTI_CMD	0x0020
-#define UD_CSIR		0x0040
-#define UD_ERROR	0x0080
-
-/* #define PORT_OVERRIDE 0x330 */
-
-/* Values for the PRODUCT_ID ports for the 14F */
-#define US14F_PRODUCT_ID_0 0x56
-#define US14F_PRODUCT_ID_1 0x40		/* NOTE: Only upper nibble is used */
-
-#define US24F_PRODUCT_ID_0 0x56
-#define US24F_PRODUCT_ID_1 0x63
-#define US24F_PRODUCT_ID_2 0x02
-
-/* Subversion values */
-#define U14F 0
-#define U34F 1
-
-/* MSCP field values */
-
-/* Opcode */
-#define OP_HOST_ADAPTER 0x1
-#define OP_SCSI 0x2
-#define OP_RESET 0x4
-
-/* Date Transfer Direction */
-#define DTD_SCSI 0x0
-#define DTD_IN 0x1
-#define DTD_OUT 0x2
-#define DTD_NONE 0x3
-
-/* Host Adapter command subcodes */
-#define HA_CMD_INQUIRY 0x1
-#define HA_CMD_SELF_DIAG 0x2
-#define HA_CMD_READ_BUFF 0x3
-#define HA_CMD_WRITE_BUFF 0x4
-
-#endif
-
-#endif
diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
index 7dbbb29d24c6..ec91bd07f00a 100644
--- a/drivers/scsi/virtio_scsi.c
+++ b/drivers/scsi/virtio_scsi.c
@@ -107,8 +107,8 @@ struct virtio_scsi {
 	/* If the affinity hint is set for virtqueues */
 	bool affinity_hint_set;
 
-	/* CPU hotplug notifier */
-	struct notifier_block nb;
+	struct hlist_node node;
+	struct hlist_node node_dead;
 
 	/* Protected by event_vq lock */
 	bool stop_events;
@@ -118,6 +118,7 @@ struct virtio_scsi {
 	struct virtio_scsi_vq req_vqs[];
 };
 
+static enum cpuhp_state virtioscsi_online;
 static struct kmem_cache *virtscsi_cmd_cache;
 static mempool_t *virtscsi_cmd_pool;
 
@@ -258,7 +259,7 @@ static void virtscsi_complete_free(struct virtio_scsi *vscsi, void *buf)
 	struct virtio_scsi_cmd *cmd = buf;
 
 	if (cmd->comp)
-		complete_all(cmd->comp);
+		complete(cmd->comp);
 }
 
 static void virtscsi_ctrl_done(struct virtqueue *vq)
@@ -852,21 +853,33 @@ static void virtscsi_set_affinity(struct virtio_scsi *vscsi, bool affinity)
 	put_online_cpus();
 }
 
-static int virtscsi_cpu_callback(struct notifier_block *nfb,
-				 unsigned long action, void *hcpu)
+static int virtscsi_cpu_online(unsigned int cpu, struct hlist_node *node)
 {
-	struct virtio_scsi *vscsi = container_of(nfb, struct virtio_scsi, nb);
-	switch(action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		__virtscsi_set_affinity(vscsi, true);
-		break;
-	default:
-		break;
-	}
-	return NOTIFY_OK;
+	struct virtio_scsi *vscsi = hlist_entry_safe(node, struct virtio_scsi,
+						     node);
+	__virtscsi_set_affinity(vscsi, true);
+	return 0;
+}
+
+static int virtscsi_cpu_notif_add(struct virtio_scsi *vi)
+{
+	int ret;
+
+	ret = cpuhp_state_add_instance(virtioscsi_online, &vi->node);
+	if (ret)
+		return ret;
+
+	ret = cpuhp_state_add_instance(CPUHP_VIRT_SCSI_DEAD, &vi->node_dead);
+	if (ret)
+		cpuhp_state_remove_instance(virtioscsi_online, &vi->node);
+	return ret;
+}
+
+static void virtscsi_cpu_notif_remove(struct virtio_scsi *vi)
+{
+	cpuhp_state_remove_instance_nocalls(virtioscsi_online, &vi->node);
+	cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_SCSI_DEAD,
+					    &vi->node_dead);
 }
 
 static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq,
@@ -929,8 +942,6 @@ static int virtscsi_init(struct virtio_device *vdev,
 		virtscsi_init_vq(&vscsi->req_vqs[i - VIRTIO_SCSI_VQ_BASE],
 				 vqs[i]);
 
-	virtscsi_set_affinity(vscsi, true);
-
 	virtscsi_config_set(vdev, cdb_size, VIRTIO_SCSI_CDB_SIZE);
 	virtscsi_config_set(vdev, sense_size, VIRTIO_SCSI_SENSE_SIZE);
 
@@ -987,12 +998,9 @@ static int virtscsi_probe(struct virtio_device *vdev)
 	if (err)
 		goto virtscsi_init_failed;
 
-	vscsi->nb.notifier_call = &virtscsi_cpu_callback;
-	err = register_hotcpu_notifier(&vscsi->nb);
-	if (err) {
-		pr_err("registering cpu notifier failed\n");
+	err = virtscsi_cpu_notif_add(vscsi);
+	if (err)
 		goto scsi_add_host_failed;
-	}
 
 	cmd_per_lun = virtscsi_config_get(vdev, cmd_per_lun) ?: 1;
 	shost->cmd_per_lun = min_t(u32, cmd_per_lun, shost->can_queue);
@@ -1049,7 +1057,7 @@ static void virtscsi_remove(struct virtio_device *vdev)
 
 	scsi_remove_host(shost);
 
-	unregister_hotcpu_notifier(&vscsi->nb);
+	virtscsi_cpu_notif_remove(vscsi);
 
 	virtscsi_remove_vqs(vdev);
 	scsi_host_put(shost);
@@ -1061,7 +1069,7 @@ static int virtscsi_freeze(struct virtio_device *vdev)
 	struct Scsi_Host *sh = virtio_scsi_host(vdev);
 	struct virtio_scsi *vscsi = shost_priv(sh);
 
-	unregister_hotcpu_notifier(&vscsi->nb);
+	virtscsi_cpu_notif_remove(vscsi);
 	virtscsi_remove_vqs(vdev);
 	return 0;
 }
@@ -1076,12 +1084,11 @@ static int virtscsi_restore(struct virtio_device *vdev)
 	if (err)
 		return err;
 
-	err = register_hotcpu_notifier(&vscsi->nb);
+	err = virtscsi_cpu_notif_add(vscsi);
 	if (err) {
 		vdev->config->del_vqs(vdev);
 		return err;
 	}
-
 	virtio_device_ready(vdev);
 
 	if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG))
@@ -1136,6 +1143,16 @@ static int __init init(void)
 		pr_err("mempool_create() for virtscsi_cmd_pool failed\n");
 		goto error;
 	}
+	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+				      "scsi/virtio:online",
+				      virtscsi_cpu_online, NULL);
+	if (ret < 0)
+		goto error;
+	virtioscsi_online = ret;
+	ret = cpuhp_setup_state_multi(CPUHP_VIRT_SCSI_DEAD, "scsi/virtio:dead",
+				      NULL, virtscsi_cpu_online);
+	if (ret)
+		goto error;
 	ret = register_virtio_driver(&virtio_scsi_driver);
 	if (ret < 0)
 		goto error;
@@ -1151,12 +1168,17 @@ error:
 		kmem_cache_destroy(virtscsi_cmd_cache);
 		virtscsi_cmd_cache = NULL;
 	}
+	if (virtioscsi_online)
+		cpuhp_remove_multi_state(virtioscsi_online);
+	cpuhp_remove_multi_state(CPUHP_VIRT_SCSI_DEAD);
 	return ret;
 }
 
 static void __exit fini(void)
 {
 	unregister_virtio_driver(&virtio_scsi_driver);
+	cpuhp_remove_multi_state(virtioscsi_online);
+	cpuhp_remove_multi_state(CPUHP_VIRT_SCSI_DEAD);
 	mempool_destroy(virtscsi_cmd_pool);
 	kmem_cache_destroy(virtscsi_cmd_cache);
 }
diff --git a/drivers/scsi/wd7000.c b/drivers/scsi/wd7000.c
deleted file mode 100644
index 409f959845c4..000000000000
--- a/drivers/scsi/wd7000.c
+++ /dev/null
@@ -1,1657 +0,0 @@
-/* $Id: $
- *  linux/drivers/scsi/wd7000.c
- *
- *  Copyright (C) 1992  Thomas Wuensche
- *	closely related to the aha1542 driver from Tommy Thorn
- *	( as close as different hardware allows on a lowlevel-driver :-) )
- *
- *  Revised (and renamed) by John Boyd <boyd@cis.ohio-state.edu> to
- *  accommodate Eric Youngdale's modifications to scsi.c.  Nov 1992.
- *
- *  Additional changes to support scatter/gather.  Dec. 1992.  tw/jb
- *
- *  No longer tries to reset SCSI bus at boot (it wasn't working anyway).
- *  Rewritten to support multiple host adapters.
- *  Miscellaneous cleanup.
- *  So far, still doesn't do reset or abort correctly, since I have no idea
- *  how to do them with this board (8^(.                      Jan 1994 jb
- *
- * This driver now supports both of the two standard configurations (per
- * the 3.36 Owner's Manual, my latest reference) by the same method as
- * before; namely, by looking for a BIOS signature.  Thus, the location of
- * the BIOS signature determines the board configuration.  Until I have
- * time to do something more flexible, users should stick to one of the
- * following:
- *
- * Standard configuration for single-adapter systems:
- *    - BIOS at CE00h
- *    - I/O base address 350h
- *    - IRQ level 15
- *    - DMA channel 6
- * Standard configuration for a second adapter in a system:
- *    - BIOS at C800h
- *    - I/O base address 330h
- *    - IRQ level 11
- *    - DMA channel 5
- *
- * Anyone who can recompile the kernel is welcome to add others as need
- * arises, but unpredictable results may occur if there are conflicts.
- * In any event, if there are multiple adapters in a system, they MUST
- * use different I/O bases, IRQ levels, and DMA channels, since they will be
- * indistinguishable (and in direct conflict) otherwise.
- *
- *   As a point of information, the NO_OP command toggles the CMD_RDY bit
- * of the status port, and this fact could be used as a test for the I/O
- * base address (or more generally, board detection).  There is an interrupt
- * status port, so IRQ probing could also be done.  I suppose the full
- * DMA diagnostic could be used to detect the DMA channel being used.  I
- * haven't done any of this, though, because I think there's too much of
- * a chance that such explorations could be destructive, if some other
- * board's resources are used inadvertently.  So, call me a wimp, but I
- * don't want to try it.  The only kind of exploration I trust is memory
- * exploration, since it's more certain that reading memory won't be
- * destructive.
- *
- * More to my liking would be a LILO boot command line specification, such
- * as is used by the aha152x driver (and possibly others).  I'll look into
- * it, as I have time...
- *
- *   I get mail occasionally from people who either are using or are
- * considering using a WD7000 with Linux.  There is a variety of
- * nomenclature describing WD7000's.  To the best of my knowledge, the
- * following is a brief summary (from an old WD doc - I don't work for
- * them or anything like that):
- *
- * WD7000-FASST2: This is a WD7000 board with the real-mode SST ROM BIOS
- *        installed.  Last I heard, the BIOS was actually done by Columbia
- *        Data Products.  The BIOS is only used by this driver (and thus
- *        by Linux) to identify the board; none of it can be executed under
- *        Linux.
- *
- * WD7000-ASC: This is the original adapter board, with or without BIOS.
- *        The board uses a WD33C93 or WD33C93A SBIC, which in turn is
- *        controlled by an onboard Z80 processor.  The board interface
- *        visible to the host CPU is defined effectively by the Z80's
- *        firmware, and it is this firmware's revision level that is
- *        determined and reported by this driver.  (The version of the
- *        on-board BIOS is of no interest whatsoever.)  The host CPU has
- *        no access to the SBIC; hence the fact that it is a WD33C93 is
- *        also of no interest to this driver.
- *
- * WD7000-AX:
- * WD7000-MX:
- * WD7000-EX: These are newer versions of the WD7000-ASC.  The -ASC is
- *        largely built from discrete components; these boards use more
- *        integration.  The -AX is an ISA bus board (like the -ASC),
- *        the -MX is an MCA (i.e., PS/2) bus board), and the -EX is an
- *        EISA bus board.
- *
- *  At the time of my documentation, the -?X boards were "future" products,
- *  and were not yet available.  However, I vaguely recall that Thomas
- *  Wuensche had an -AX, so I believe at least it is supported by this
- *  driver.  I have no personal knowledge of either -MX or -EX boards.
- *
- *  P.S. Just recently, I've discovered (directly from WD and Future
- *  Domain) that all but the WD7000-EX have been out of production for
- *  two years now.  FD has production rights to the 7000-EX, and are
- *  producing it under a new name, and with a new BIOS.  If anyone has
- *  one of the FD boards, it would be nice to come up with a signature
- *  for it.
- *                                                           J.B. Jan 1994.
- *
- *
- *  Revisions by Miroslav Zagorac <zaga@fly.cc.fer.hr>
- *
- *  08/24/1996.
- *
- *  Enhancement for wd7000_detect function has been made, so you don't have
- *  to enter BIOS ROM address in initialisation data (see struct Config).
- *  We cannot detect IRQ, DMA and I/O base address for now, so we have to
- *  enter them as arguments while wd_7000 is detected. If someone has IRQ,
- *  DMA or I/O base address set to some other value, he can enter them in
- *  configuration without any problem. Also I wrote a function wd7000_setup,
- *  so now you can enter WD-7000 definition as kernel arguments,
- *  as in lilo.conf:
- *
- *     append="wd7000=IRQ,DMA,IO"
- *
- *  PS: If card BIOS ROM is disabled, function wd7000_detect now will recognize
- *      adapter, unlike the old one. Anyway, BIOS ROM from WD7000 adapter is
- *      useless for Linux. B^)
- *
- *
- *  09/06/1996.
- *
- *  Autodetecting of I/O base address from wd7000_detect function is removed,
- *  some little bugs removed, etc...
- *
- *  Thanks to Roger Scott for driver debugging.
- *
- *  06/07/1997
- *
- *  Added support for /proc file system (/proc/scsi/wd7000/[0...] files).
- *  Now, driver can handle hard disks with capacity >1GB.
- *
- *  01/15/1998
- *
- *  Added support for BUS_ON and BUS_OFF parameters in config line.
- *  Miscellaneous cleanup.
- *
- *  03/01/1998
- *
- *  WD7000 driver now work on kernels >= 2.1.x
- *
- *
- * 12/31/2001 - Arnaldo Carvalho de Melo <acme@conectiva.com.br>
- *
- * use host->host_lock, not io_request_lock, cleanups
- *
- * 2002/10/04 - Alan Cox <alan@lxorguk.ukuu.org.uk>
- *
- * Use dev_id for interrupts, kill __func__ pasting
- * Add a lock for the scb pool, clean up all other cli/sti usage stuff
- * Use the adapter lock for the other places we had the cli's
- *
- * 2002/10/06 - Alan Cox <alan@lxorguk.ukuu.org.uk>
- *
- * Switch to new style error handling
- * Clean up delay to udelay, and yielding sleeps
- * Make host reset actually reset the card
- * Make everything static
- *
- * 2003/02/12 - Christoph Hellwig <hch@infradead.org>
- *
- * Cleaned up host template definition
- * Removed now obsolete wd7000.h
- */
-
-#include <linux/delay.h>
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/string.h>
-#include <linux/spinlock.h>
-#include <linux/ioport.h>
-#include <linux/proc_fs.h>
-#include <linux/blkdev.h>
-#include <linux/init.h>
-#include <linux/stat.h>
-#include <linux/io.h>
-
-#include <asm/dma.h>
-
-#include <scsi/scsi.h>
-#include <scsi/scsi_cmnd.h>
-#include <scsi/scsi_device.h>
-#include <scsi/scsi_host.h>
-#include <scsi/scsicam.h>
-
-
-#undef  WD7000_DEBUG		/* general debug                */
-#ifdef WD7000_DEBUG
-#define dprintk printk
-#else
-#define dprintk	no_printk
-#endif
-
-/*
- *  Mailbox structure sizes.
- *  I prefer to keep the number of ICMBs much larger than the number of
- *  OGMBs.  OGMBs are used very quickly by the driver to start one or
- *  more commands, while ICMBs are used by the host adapter per command.
- */
-#define OGMB_CNT	16
-#define ICMB_CNT	32
-
-/*
- *  Scb's are shared by all active adapters.  So, if they all become busy,
- *  callers may be made to wait in alloc_scbs for them to free.  That can
- *  be avoided by setting MAX_SCBS to NUM_CONFIG * WD7000_Q.  If you'd
- *  rather conserve memory, use a smaller number (> 0, of course) - things
- *  will should still work OK.
- */
-#define MAX_SCBS        32
-
-/*
- *  In this version, sg_tablesize now defaults to WD7000_SG, and will
- *  be set to SG_NONE for older boards.  This is the reverse of the
- *  previous default, and was changed so that the driver-level
- *  scsi_host_template would reflect the driver's support for scatter/
- *  gather.
- *
- *  Also, it has been reported that boards at Revision 6 support scatter/
- *  gather, so the new definition of an "older" board has been changed
- *  accordingly.
- */
-#define WD7000_Q	16
-#define WD7000_SG	16
-
-
-/*
- *  WD7000-specific mailbox structure
- *
- */
-typedef volatile struct mailbox {
-	unchar status;
-	unchar scbptr[3];	/* SCSI-style - MSB first (big endian) */
-} Mailbox;
-
-/*
- *  This structure should contain all per-adapter global data.  I.e., any
- *  new global per-adapter data should put in here.
- */
-typedef struct adapter {
-	struct Scsi_Host *sh;	/* Pointer to Scsi_Host structure    */
-	int iobase;		/* This adapter's I/O base address   */
-	int irq;		/* This adapter's IRQ level          */
-	int dma;		/* This adapter's DMA channel        */
-	int int_counter;	/* This adapter's interrupt counter  */
-	int bus_on;		/* This adapter's BUS_ON time        */
-	int bus_off;		/* This adapter's BUS_OFF time       */
-	struct {		/* This adapter's mailboxes          */
-		Mailbox ogmb[OGMB_CNT];	/* Outgoing mailboxes                */
-		Mailbox icmb[ICMB_CNT];	/* Incoming mailboxes                */
-	} mb;
-	int next_ogmb;		/* to reduce contention at mailboxes */
-	unchar control;		/* shadows CONTROL port value        */
-	unchar rev1, rev2;	/* filled in by wd7000_revision      */
-} Adapter;
-
-/*
- * (linear) base address for ROM BIOS
- */
-static const long wd7000_biosaddr[] = {
-	0xc0000, 0xc2000, 0xc4000, 0xc6000, 0xc8000, 0xca000, 0xcc000, 0xce000,
-	0xd0000, 0xd2000, 0xd4000, 0xd6000, 0xd8000, 0xda000, 0xdc000, 0xde000
-};
-#define NUM_ADDRS ARRAY_SIZE(wd7000_biosaddr)
-
-static const unsigned short wd7000_iobase[] = {
-	0x0300, 0x0308, 0x0310, 0x0318, 0x0320, 0x0328, 0x0330, 0x0338,
-	0x0340, 0x0348, 0x0350, 0x0358, 0x0360, 0x0368, 0x0370, 0x0378,
-	0x0380, 0x0388, 0x0390, 0x0398, 0x03a0, 0x03a8, 0x03b0, 0x03b8,
-	0x03c0, 0x03c8, 0x03d0, 0x03d8, 0x03e0, 0x03e8, 0x03f0, 0x03f8
-};
-#define NUM_IOPORTS ARRAY_SIZE(wd7000_iobase)
-
-static const short wd7000_irq[] = { 3, 4, 5, 7, 9, 10, 11, 12, 14, 15 };
-#define NUM_IRQS ARRAY_SIZE(wd7000_irq)
-
-static const short wd7000_dma[] = { 5, 6, 7 };
-#define NUM_DMAS ARRAY_SIZE(wd7000_dma)
-
-/*
- * The following is set up by wd7000_detect, and used thereafter for
- * proc and other global ookups
- */
-
-#define UNITS	8
-static struct Scsi_Host *wd7000_host[UNITS];
-
-#define BUS_ON    64		/* x 125ns = 8000ns (BIOS default) */
-#define BUS_OFF   15		/* x 125ns = 1875ns (BIOS default) */
-
-/*
- *  Standard Adapter Configurations - used by wd7000_detect
- */
-typedef struct {
-	short irq;		/* IRQ level                                  */
-	short dma;		/* DMA channel                                */
-	unsigned iobase;	/* I/O base address                           */
-	short bus_on;		/* Time that WD7000 spends on the AT-bus when */
-	/* transferring data. BIOS default is 8000ns. */
-	short bus_off;		/* Time that WD7000 spends OFF THE BUS after  */
-	/* while it is transferring data.             */
-	/* BIOS default is 1875ns                     */
-} Config;
-
-/*
- * Add here your configuration...
- */
-static Config configs[] = {
-	{15, 6, 0x350, BUS_ON, BUS_OFF},	/* defaults for single adapter */
-	{11, 5, 0x320, BUS_ON, BUS_OFF},	/* defaults for second adapter */
-	{7, 6, 0x350, BUS_ON, BUS_OFF},	/* My configuration (Zaga)     */
-	{-1, -1, 0x0, BUS_ON, BUS_OFF}	/* Empty slot                  */
-};
-#define NUM_CONFIGS ARRAY_SIZE(configs)
-
-/*
- *  The following list defines strings to look for in the BIOS that identify
- *  it as the WD7000-FASST2 SST BIOS.  I suspect that something should be
- *  added for the Future Domain version.
- */
-typedef struct signature {
-	const char *sig;	/* String to look for            */
-	unsigned long ofs;	/* offset from BIOS base address */
-	unsigned len;		/* length of string              */
-} Signature;
-
-static const Signature signatures[] = {
-	{"SSTBIOS", 0x0000d, 7}	/* "SSTBIOS" @ offset 0x0000d */
-};
-#define NUM_SIGNATURES ARRAY_SIZE(signatures)
-
-
-/*
- *  I/O Port Offsets and Bit Definitions
- *  4 addresses are used.  Those not defined here are reserved.
- */
-#define ASC_STAT        0	/* Status,  Read          */
-#define ASC_COMMAND     0	/* Command, Write         */
-#define ASC_INTR_STAT   1	/* Interrupt Status, Read */
-#define ASC_INTR_ACK    1	/* Acknowledge, Write     */
-#define ASC_CONTROL     2	/* Control, Write         */
-
-/*
- * ASC Status Port
- */
-#define INT_IM		0x80	/* Interrupt Image Flag           */
-#define CMD_RDY		0x40	/* Command Port Ready             */
-#define CMD_REJ		0x20	/* Command Port Byte Rejected     */
-#define ASC_INIT        0x10	/* ASC Initialized Flag           */
-#define ASC_STATMASK    0xf0	/* The lower 4 Bytes are reserved */
-
-/*
- * COMMAND opcodes
- *
- *  Unfortunately, I have no idea how to properly use some of these commands,
- *  as the OEM manual does not make it clear.  I have not been able to use
- *  enable/disable unsolicited interrupts or the reset commands with any
- *  discernible effect whatsoever.  I think they may be related to certain
- *  ICB commands, but again, the OEM manual doesn't make that clear.
- */
-#define NO_OP             0	/* NO-OP toggles CMD_RDY bit in ASC_STAT  */
-#define INITIALIZATION    1	/* initialization (10 bytes)              */
-#define DISABLE_UNS_INTR  2	/* disable unsolicited interrupts         */
-#define ENABLE_UNS_INTR   3	/* enable unsolicited interrupts          */
-#define INTR_ON_FREE_OGMB 4	/* interrupt on free OGMB                 */
-#define SOFT_RESET        5	/* SCSI bus soft reset                    */
-#define HARD_RESET_ACK    6	/* SCSI bus hard reset acknowledge        */
-#define START_OGMB        0x80	/* start command in OGMB (n)              */
-#define SCAN_OGMBS        0xc0	/* start multiple commands, signature (n) */
-				/*    where (n) = lower 6 bits            */
-/*
- * For INITIALIZATION:
- */
-typedef struct initCmd {
-	unchar op;		/* command opcode (= 1)                    */
-	unchar ID;		/* Adapter's SCSI ID                       */
-	unchar bus_on;		/* Bus on time, x 125ns (see below)        */
-	unchar bus_off;		/* Bus off time, ""         ""             */
-	unchar rsvd;		/* Reserved                                */
-	unchar mailboxes[3];	/* Address of Mailboxes, MSB first         */
-	unchar ogmbs;		/* Number of outgoing MBs, max 64, 0,1 = 1 */
-	unchar icmbs;		/* Number of incoming MBs,   ""       ""   */
-} InitCmd;
-
-/*
- * Interrupt Status Port - also returns diagnostic codes at ASC reset
- *
- * if msb is zero, the lower bits are diagnostic status
- * Diagnostics:
- * 01   No diagnostic error occurred
- * 02   RAM failure
- * 03   FIFO R/W failed
- * 04   SBIC register read/write failed
- * 05   Initialization D-FF failed
- * 06   Host IRQ D-FF failed
- * 07   ROM checksum error
- * Interrupt status (bitwise):
- * 10NNNNNN   outgoing mailbox NNNNNN is free
- * 11NNNNNN   incoming mailbox NNNNNN needs service
- */
-#define MB_INTR    0xC0		/* Mailbox Service possible/required */
-#define IMB_INTR   0x40		/* 1 Incoming / 0 Outgoing           */
-#define MB_MASK    0x3f		/* mask for mailbox number           */
-
-/*
- * CONTROL port bits
- */
-#define INT_EN     0x08		/* Interrupt Enable */
-#define DMA_EN     0x04		/* DMA Enable       */
-#define SCSI_RES   0x02		/* SCSI Reset       */
-#define ASC_RES    0x01		/* ASC Reset        */
-
-/*
- * Driver data structures:
- *   - mb and scbs are required for interfacing with the host adapter.
- *     An SCB has extra fields not visible to the adapter; mb's
- *     _cannot_ do this, since the adapter assumes they are contiguous in
- *     memory, 4 bytes each, with ICMBs following OGMBs, and uses this fact
- *     to access them.
- *   - An icb is for host-only (non-SCSI) commands.  ICBs are 16 bytes each;
- *     the additional bytes are used only by the driver.
- *   - For now, a pool of SCBs are kept in global storage by this driver,
- *     and are allocated and freed as needed.
- *
- *  The 7000-FASST2 marks OGMBs empty as soon as it has _started_ a command,
- *  not when it has finished.  Since the SCB must be around for completion,
- *  problems arise when SCBs correspond to OGMBs, which may be reallocated
- *  earlier (or delayed unnecessarily until a command completes).
- *  Mailboxes are used as transient data structures, simply for
- *  carrying SCB addresses to/from the 7000-FASST2.
- *
- *  Note also since SCBs are not "permanently" associated with mailboxes,
- *  there is no need to keep a global list of scsi_cmnd pointers indexed
- *  by OGMB.   Again, SCBs reference their scsi_cmnds directly, so mailbox
- *  indices need not be involved.
- */
-
-/*
- *  WD7000-specific scatter/gather element structure
- */
-typedef struct sgb {
-	unchar len[3];
-	unchar ptr[3];		/* Also SCSI-style - MSB first */
-} Sgb;
-
-typedef struct scb {		/* Command Control Block 5.4.1               */
-	unchar op;		/* Command Control Block Operation Code      */
-	unchar idlun;		/* op=0,2:Target Id, op=1:Initiator Id       */
-	/* Outbound data transfer, length is checked */
-	/* Inbound data transfer, length is checked  */
-	/* Logical Unit Number                       */
-	unchar cdb[12];		/* SCSI Command Block                        */
-	volatile unchar status;	/* SCSI Return Status                        */
-	volatile unchar vue;	/* Vendor Unique Error Code                  */
-	unchar maxlen[3];	/* Maximum Data Transfer Length              */
-	unchar dataptr[3];	/* SCSI Data Block Pointer                   */
-	unchar linkptr[3];	/* Next Command Link Pointer                 */
-	unchar direc;		/* Transfer Direction                        */
-	unchar reserved2[6];	/* SCSI Command Descriptor Block             */
-	/* end of hardware SCB                       */
-	struct scsi_cmnd *SCpnt;/* scsi_cmnd using this SCB                  */
-	Sgb sgb[WD7000_SG];	/* Scatter/gather list for this SCB          */
-	Adapter *host;		/* host adapter                              */
-	struct scb *next;	/* for lists of scbs                         */
-} Scb;
-
-/*
- *  This driver is written to allow host-only commands to be executed.
- *  These use a 16-byte block called an ICB.  The format is extended by the
- *  driver to 18 bytes, to support the status returned in the ICMB and
- *  an execution phase code.
- *
- *  There are other formats besides these; these are the ones I've tried
- *  to use.  Formats for some of the defined ICB opcodes are not defined
- *  (notably, get/set unsolicited interrupt status) in my copy of the OEM
- *  manual, and others are ambiguous/hard to follow.
- */
-#define ICB_OP_MASK           0x80	/* distinguishes scbs from icbs        */
-#define ICB_OP_OPEN_RBUF      0x80	/* open receive buffer                 */
-#define ICB_OP_RECV_CMD       0x81	/* receive command from initiator      */
-#define ICB_OP_RECV_DATA      0x82	/* receive data from initiator         */
-#define ICB_OP_RECV_SDATA     0x83	/* receive data with status from init. */
-#define ICB_OP_SEND_DATA      0x84	/* send data with status to initiator  */
-#define ICB_OP_SEND_STAT      0x86	/* send command status to initiator    */
-					/* 0x87 is reserved                    */
-#define ICB_OP_READ_INIT      0x88	/* read initialization bytes           */
-#define ICB_OP_READ_ID        0x89	/* read adapter's SCSI ID              */
-#define ICB_OP_SET_UMASK      0x8A	/* set unsolicited interrupt mask      */
-#define ICB_OP_GET_UMASK      0x8B	/* read unsolicited interrupt mask     */
-#define ICB_OP_GET_REVISION   0x8C	/* read firmware revision level        */
-#define ICB_OP_DIAGNOSTICS    0x8D	/* execute diagnostics                 */
-#define ICB_OP_SET_EPARMS     0x8E	/* set execution parameters            */
-#define ICB_OP_GET_EPARMS     0x8F	/* read execution parameters           */
-
-typedef struct icbRecvCmd {
-	unchar op;
-	unchar IDlun;		/* Initiator SCSI ID/lun     */
-	unchar len[3];		/* command buffer length     */
-	unchar ptr[3];		/* command buffer address    */
-	unchar rsvd[7];		/* reserved                  */
-	volatile unchar vue;	/* vendor-unique error code  */
-	volatile unchar status;	/* returned (icmb) status    */
-	volatile unchar phase;	/* used by interrupt handler */
-} IcbRecvCmd;
-
-typedef struct icbSendStat {
-	unchar op;
-	unchar IDlun;		/* Target SCSI ID/lun                  */
-	unchar stat;		/* (outgoing) completion status byte 1 */
-	unchar rsvd[12];	/* reserved                            */
-	volatile unchar vue;	/* vendor-unique error code            */
-	volatile unchar status;	/* returned (icmb) status              */
-	volatile unchar phase;	/* used by interrupt handler           */
-} IcbSendStat;
-
-typedef struct icbRevLvl {
-	unchar op;
-	volatile unchar primary;	/* primary revision level (returned)   */
-	volatile unchar secondary;	/* secondary revision level (returned) */
-	unchar rsvd[12];	/* reserved                            */
-	volatile unchar vue;	/* vendor-unique error code            */
-	volatile unchar status;	/* returned (icmb) status              */
-	volatile unchar phase;	/* used by interrupt handler           */
-} IcbRevLvl;
-
-typedef struct icbUnsMask {	/* I'm totally guessing here */
-	unchar op;
-	volatile unchar mask[14];	/* mask bits                 */
-#if 0
-	unchar rsvd[12];	/* reserved                  */
-#endif
-	volatile unchar vue;	/* vendor-unique error code  */
-	volatile unchar status;	/* returned (icmb) status    */
-	volatile unchar phase;	/* used by interrupt handler */
-} IcbUnsMask;
-
-typedef struct icbDiag {
-	unchar op;
-	unchar type;		/* diagnostics type code (0-3) */
-	unchar len[3];		/* buffer length               */
-	unchar ptr[3];		/* buffer address              */
-	unchar rsvd[7];		/* reserved                    */
-	volatile unchar vue;	/* vendor-unique error code    */
-	volatile unchar status;	/* returned (icmb) status      */
-	volatile unchar phase;	/* used by interrupt handler   */
-} IcbDiag;
-
-#define ICB_DIAG_POWERUP   0	/* Power-up diags only       */
-#define ICB_DIAG_WALKING   1	/* walking 1's pattern       */
-#define ICB_DIAG_DMA       2	/* DMA - system memory diags */
-#define ICB_DIAG_FULL      3	/* do both 1 & 2             */
-
-typedef struct icbParms {
-	unchar op;
-	unchar rsvd1;		/* reserved                  */
-	unchar len[3];		/* parms buffer length       */
-	unchar ptr[3];		/* parms buffer address      */
-	unchar idx[2];		/* index (MSB-LSB)           */
-	unchar rsvd2[5];	/* reserved                  */
-	volatile unchar vue;	/* vendor-unique error code  */
-	volatile unchar status;	/* returned (icmb) status    */
-	volatile unchar phase;	/* used by interrupt handler */
-} IcbParms;
-
-typedef struct icbAny {
-	unchar op;
-	unchar data[14];	/* format-specific data      */
-	volatile unchar vue;	/* vendor-unique error code  */
-	volatile unchar status;	/* returned (icmb) status    */
-	volatile unchar phase;	/* used by interrupt handler */
-} IcbAny;
-
-typedef union icb {
-	unchar op;		/* ICB opcode                     */
-	IcbRecvCmd recv_cmd;	/* format for receive command     */
-	IcbSendStat send_stat;	/* format for send status         */
-	IcbRevLvl rev_lvl;	/* format for get revision level  */
-	IcbDiag diag;		/* format for execute diagnostics */
-	IcbParms eparms;	/* format for get/set exec parms  */
-	IcbAny icb;		/* generic format                 */
-	unchar data[18];
-} Icb;
-
-#ifdef MODULE
-static char *wd7000;
-module_param(wd7000, charp, 0);
-#endif
-
-/*
- *  Driver SCB structure pool.
- *
- *  The SCBs declared here are shared by all host adapters; hence, this
- *  structure is not part of the Adapter structure.
- */
-static Scb scbs[MAX_SCBS];
-static Scb *scbfree;		/* free list         */
-static int freescbs = MAX_SCBS;	/* free list counter */
-static spinlock_t scbpool_lock;	/* guards the scb free list and count */
-
-/*
- *  END of data/declarations - code follows.
- */
-static void __init setup_error(char *mesg, int *ints)
-{
-	if (ints[0] == 3)
-		printk(KERN_ERR "wd7000_setup: \"wd7000=%d,%d,0x%x\" -> %s\n", ints[1], ints[2], ints[3], mesg);
-	else if (ints[0] == 4)
-		printk(KERN_ERR "wd7000_setup: \"wd7000=%d,%d,0x%x,%d\" -> %s\n", ints[1], ints[2], ints[3], ints[4], mesg);
-	else
-		printk(KERN_ERR "wd7000_setup: \"wd7000=%d,%d,0x%x,%d,%d\" -> %s\n", ints[1], ints[2], ints[3], ints[4], ints[5], mesg);
-}
-
-
-/*
- * Note: You can now set these options from the kernel's "command line".
- * The syntax is:
- *
- *     wd7000=<IRQ>,<DMA>,<IO>[,<BUS_ON>[,<BUS_OFF>]]
- *
- * , where BUS_ON and BUS_OFF are in nanoseconds. BIOS default values
- * are 8000ns for BUS_ON and 1875ns for BUS_OFF.
- * eg:
- *     wd7000=7,6,0x350
- *
- * will configure the driver for a WD-7000 controller
- * using IRQ 15 with a DMA channel 6, at IO base address 0x350.
- */
-static int __init wd7000_setup(char *str)
-{
-	static short wd7000_card_num;	/* .bss will zero this */
-	short i;
-	int ints[6];
-
-	(void) get_options(str, ARRAY_SIZE(ints), ints);
-
-	if (wd7000_card_num >= NUM_CONFIGS) {
-		printk(KERN_ERR "%s: Too many \"wd7000=\" configurations in " "command line!\n", __func__);
-		return 0;
-	}
-
-	if ((ints[0] < 3) || (ints[0] > 5)) {
-		printk(KERN_ERR "%s: Error in command line!  " "Usage: wd7000=<IRQ>,<DMA>,IO>[,<BUS_ON>" "[,<BUS_OFF>]]\n", __func__);
-	} else {
-		for (i = 0; i < NUM_IRQS; i++)
-			if (ints[1] == wd7000_irq[i])
-				break;
-
-		if (i == NUM_IRQS) {
-			setup_error("invalid IRQ.", ints);
-			return 0;
-		} else
-			configs[wd7000_card_num].irq = ints[1];
-
-		for (i = 0; i < NUM_DMAS; i++)
-			if (ints[2] == wd7000_dma[i])
-				break;
-
-		if (i == NUM_DMAS) {
-			setup_error("invalid DMA channel.", ints);
-			return 0;
-		} else
-			configs[wd7000_card_num].dma = ints[2];
-
-		for (i = 0; i < NUM_IOPORTS; i++)
-			if (ints[3] == wd7000_iobase[i])
-				break;
-
-		if (i == NUM_IOPORTS) {
-			setup_error("invalid I/O base address.", ints);
-			return 0;
-		} else
-			configs[wd7000_card_num].iobase = ints[3];
-
-		if (ints[0] > 3) {
-			if ((ints[4] < 500) || (ints[4] > 31875)) {
-				setup_error("BUS_ON value is out of range (500" " to 31875 nanoseconds)!", ints);
-				configs[wd7000_card_num].bus_on = BUS_ON;
-			} else
-				configs[wd7000_card_num].bus_on = ints[4] / 125;
-		} else
-			configs[wd7000_card_num].bus_on = BUS_ON;
-
-		if (ints[0] > 4) {
-			if ((ints[5] < 500) || (ints[5] > 31875)) {
-				setup_error("BUS_OFF value is out of range (500" " to 31875 nanoseconds)!", ints);
-				configs[wd7000_card_num].bus_off = BUS_OFF;
-			} else
-				configs[wd7000_card_num].bus_off = ints[5] / 125;
-		} else
-			configs[wd7000_card_num].bus_off = BUS_OFF;
-
-		if (wd7000_card_num) {
-			for (i = 0; i < (wd7000_card_num - 1); i++) {
-				int j = i + 1;
-
-				for (; j < wd7000_card_num; j++)
-					if (configs[i].irq == configs[j].irq) {
-						setup_error("duplicated IRQ!", ints);
-						return 0;
-					}
-				if (configs[i].dma == configs[j].dma) {
-					setup_error("duplicated DMA " "channel!", ints);
-					return 0;
-				}
-				if (configs[i].iobase == configs[j].iobase) {
-					setup_error("duplicated I/O " "base address!", ints);
-					return 0;
-				}
-			}
-		}
-
-		dprintk(KERN_DEBUG "wd7000_setup: IRQ=%d, DMA=%d, I/O=0x%x, "
-			"BUS_ON=%dns, BUS_OFF=%dns\n", configs[wd7000_card_num].irq, configs[wd7000_card_num].dma, configs[wd7000_card_num].iobase, configs[wd7000_card_num].bus_on * 125, configs[wd7000_card_num].bus_off * 125);
-
-		wd7000_card_num++;
-	}
-	return 1;
-}
-
-__setup("wd7000=", wd7000_setup);
-
-static inline void any2scsi(unchar * scsi, int any)
-{
-	*scsi++ = (unsigned)any >> 16;
-	*scsi++ = (unsigned)any >> 8;
-	*scsi++ = any;
-}
-
-static inline int scsi2int(unchar * scsi)
-{
-	return (scsi[0] << 16) | (scsi[1] << 8) | scsi[2];
-}
-
-static inline void wd7000_enable_intr(Adapter * host)
-{
-	host->control |= INT_EN;
-	outb(host->control, host->iobase + ASC_CONTROL);
-}
-
-
-static inline void wd7000_enable_dma(Adapter * host)
-{
-	unsigned long flags;
-	host->control |= DMA_EN;
-	outb(host->control, host->iobase + ASC_CONTROL);
-
-	flags = claim_dma_lock();
-	set_dma_mode(host->dma, DMA_MODE_CASCADE);
-	enable_dma(host->dma);
-	release_dma_lock(flags);
-
-}
-
-
-#define WAITnexttimeout 200	/* 2 seconds */
-
-static inline short WAIT(unsigned port, unsigned mask, unsigned allof, unsigned noneof)
-{
-	unsigned WAITbits;
-	unsigned long WAITtimeout = jiffies + WAITnexttimeout;
-
-	while (time_before_eq(jiffies, WAITtimeout)) {
-		WAITbits = inb(port) & mask;
-
-		if (((WAITbits & allof) == allof) && ((WAITbits & noneof) == 0))
-			return (0);
-	}
-
-	return (1);
-}
-
-
-static inline int command_out(Adapter * host, unchar * cmd, int len)
-{
-	if (!WAIT(host->iobase + ASC_STAT, ASC_STATMASK, CMD_RDY, 0)) {
-		while (len--) {
-			do {
-				outb(*cmd, host->iobase + ASC_COMMAND);
-				WAIT(host->iobase + ASC_STAT, ASC_STATMASK, CMD_RDY, 0);
-			} while (inb(host->iobase + ASC_STAT) & CMD_REJ);
-
-			cmd++;
-		}
-
-		return (1);
-	}
-
-	printk(KERN_WARNING "wd7000 command_out: WAIT failed(%d)\n", len + 1);
-
-	return (0);
-}
-
-
-/*
- *  This version of alloc_scbs is in preparation for supporting multiple
- *  commands per lun and command chaining, by queueing pending commands.
- *  We will need to allocate Scbs in blocks since they will wait to be
- *  executed so there is the possibility of deadlock otherwise.
- *  Also, to keep larger requests from being starved by smaller requests,
- *  we limit access to this routine with an internal busy flag, so that
- *  the satisfiability of a request is not dependent on the size of the
- *  request.
- */
-static inline Scb *alloc_scbs(struct Scsi_Host *host, int needed)
-{
-	Scb *scb, *p = NULL;
-	unsigned long flags;
-	unsigned long timeout = jiffies + WAITnexttimeout;
-	unsigned long now;
-	int i;
-
-	if (needed <= 0)
-		return (NULL);	/* sanity check */
-
-	spin_unlock_irq(host->host_lock);
-
-      retry:
-	while (freescbs < needed) {
-		timeout = jiffies + WAITnexttimeout;
-		do {
-			/* FIXME: can we actually just yield here ?? */
-			for (now = jiffies; now == jiffies;)
-				cpu_relax();	/* wait a jiffy */
-		} while (freescbs < needed && time_before_eq(jiffies, timeout));
-		/*
-		 *  If we get here with enough free Scbs, we can take them.
-		 *  Otherwise, we timed out and didn't get enough.
-		 */
-		if (freescbs < needed) {
-			printk(KERN_ERR "wd7000: can't get enough free SCBs.\n");
-			return (NULL);
-		}
-	}
-
-	/* Take the lock, then check we didn't get beaten, if so try again */
-	spin_lock_irqsave(&scbpool_lock, flags);
-	if (freescbs < needed) {
-		spin_unlock_irqrestore(&scbpool_lock, flags);
-		goto retry;
-	}
-
-	scb = scbfree;
-	freescbs -= needed;
-	for (i = 0; i < needed; i++) {
-		p = scbfree;
-		scbfree = p->next;
-	}
-	p->next = NULL;
-
-	spin_unlock_irqrestore(&scbpool_lock, flags);
-
-	spin_lock_irq(host->host_lock);
-	return (scb);
-}
-
-
-static inline void free_scb(Scb * scb)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&scbpool_lock, flags);
-
-	memset(scb, 0, sizeof(Scb));
-	scb->next = scbfree;
-	scbfree = scb;
-	freescbs++;
-
-	spin_unlock_irqrestore(&scbpool_lock, flags);
-}
-
-
-static inline void init_scbs(void)
-{
-	int i;
-
-	spin_lock_init(&scbpool_lock);
-
-	/* This is only ever called before the SCB pool is active */
-
-	scbfree = &(scbs[0]);
-	memset(scbs, 0, sizeof(scbs));
-	for (i = 0; i < MAX_SCBS - 1; i++) {
-		scbs[i].next = &(scbs[i + 1]);
-		scbs[i].SCpnt = NULL;
-	}
-	scbs[MAX_SCBS - 1].next = NULL;
-	scbs[MAX_SCBS - 1].SCpnt = NULL;
-}
-
-
-static int mail_out(Adapter * host, Scb * scbptr)
-/*
- *  Note: this can also be used for ICBs; just cast to the parm type.
- */
-{
-	int i, ogmb;
-	unsigned long flags;
-	unchar start_ogmb;
-	Mailbox *ogmbs = host->mb.ogmb;
-	int *next_ogmb = &(host->next_ogmb);
-
-	dprintk("wd7000_mail_out: 0x%06lx", (long) scbptr);
-
-	/* We first look for a free outgoing mailbox */
-	spin_lock_irqsave(host->sh->host_lock, flags);
-	ogmb = *next_ogmb;
-	for (i = 0; i < OGMB_CNT; i++) {
-		if (ogmbs[ogmb].status == 0) {
-			dprintk(" using OGMB 0x%x", ogmb);
-			ogmbs[ogmb].status = 1;
-			any2scsi((unchar *) ogmbs[ogmb].scbptr, (int) scbptr);
-
-			*next_ogmb = (ogmb + 1) % OGMB_CNT;
-			break;
-		} else
-			ogmb = (ogmb + 1) % OGMB_CNT;
-	}
-	spin_unlock_irqrestore(host->sh->host_lock, flags);
-
-	dprintk(", scb is 0x%06lx", (long) scbptr);
-
-	if (i >= OGMB_CNT) {
-		/*
-		 *  Alternatively, we might issue the "interrupt on free OGMB",
-		 *  and sleep, but it must be ensured that it isn't the init
-		 *  task running.  Instead, this version assumes that the caller
-		 *  will be persistent, and try again.  Since it's the adapter
-		 *  that marks OGMB's free, waiting even with interrupts off
-		 *  should work, since they are freed very quickly in most cases.
-		 */
-		dprintk(", no free OGMBs.\n");
-		return (0);
-	}
-
-	wd7000_enable_intr(host);
-
-	start_ogmb = START_OGMB | ogmb;
-	command_out(host, &start_ogmb, 1);
-
-	dprintk(", awaiting interrupt.\n");
-
-	return (1);
-}
-
-
-static int make_code(unsigned hosterr, unsigned scsierr)
-{
-#ifdef WD7000_DEBUG
-	int in_error = hosterr;
-#endif
-
-	switch ((hosterr >> 8) & 0xff) {
-	case 0:		/* Reserved */
-		hosterr = DID_ERROR;
-		break;
-	case 1:		/* Command Complete, no errors */
-		hosterr = DID_OK;
-		break;
-	case 2:		/* Command complete, error logged in scb status (scsierr) */
-		hosterr = DID_OK;
-		break;
-	case 4:		/* Command failed to complete - timeout */
-		hosterr = DID_TIME_OUT;
-		break;
-	case 5:		/* Command terminated; Bus reset by external device */
-		hosterr = DID_RESET;
-		break;
-	case 6:		/* Unexpected Command Received w/ host as target */
-		hosterr = DID_BAD_TARGET;
-		break;
-	case 80:		/* Unexpected Reselection */
-	case 81:		/* Unexpected Selection */
-		hosterr = DID_BAD_INTR;
-		break;
-	case 82:		/* Abort Command Message  */
-		hosterr = DID_ABORT;
-		break;
-	case 83:		/* SCSI Bus Software Reset */
-	case 84:		/* SCSI Bus Hardware Reset */
-		hosterr = DID_RESET;
-		break;
-	default:		/* Reserved */
-		hosterr = DID_ERROR;
-	}
-#ifdef WD7000_DEBUG
-	if (scsierr || hosterr)
-		dprintk("\nSCSI command error: SCSI 0x%02x host 0x%04x return %d\n", scsierr, in_error, hosterr);
-#endif
-	return (scsierr | (hosterr << 16));
-}
-
-#define wd7000_intr_ack(host)   outb (0, host->iobase + ASC_INTR_ACK)
-
-
-static irqreturn_t wd7000_intr(int irq, void *dev_id)
-{
-	Adapter *host = (Adapter *) dev_id;
-	int flag, icmb, errstatus, icmb_status;
-	int host_error, scsi_error;
-	Scb *scb;	/* for SCSI commands */
-	IcbAny *icb;	/* for host commands */
-	struct scsi_cmnd *SCpnt;
-	Mailbox *icmbs = host->mb.icmb;
-	unsigned long flags;
-
-	spin_lock_irqsave(host->sh->host_lock, flags);
-	host->int_counter++;
-
-	dprintk("wd7000_intr: irq = %d, host = 0x%06lx\n", irq, (long) host);
-
-	flag = inb(host->iobase + ASC_INTR_STAT);
-
-	dprintk("wd7000_intr: intr stat = 0x%02x\n", flag);
-
-	if (!(inb(host->iobase + ASC_STAT) & INT_IM)) {
-		/* NB: these are _very_ possible if IRQ 15 is being used, since
-		 * it's the "garbage collector" on the 2nd 8259 PIC.  Specifically,
-		 * any interrupt signal into the 8259 which can't be identified
-		 * comes out as 7 from the 8259, which is 15 to the host.  Thus, it
-		 * is a good thing the WD7000 has an interrupt status port, so we
-		 * can sort these out.  Otherwise, electrical noise and other such
-		 * problems would be indistinguishable from valid interrupts...
-		 */
-		dprintk("wd7000_intr: phantom interrupt...\n");
-		goto ack;
-	}
-
-	if (!(flag & MB_INTR))
-		goto ack;
-
-	/* The interrupt is for a mailbox */
-	if (!(flag & IMB_INTR)) {
-		dprintk("wd7000_intr: free outgoing mailbox\n");
-		/*
-		 * If sleep_on() and the "interrupt on free OGMB" command are
-		 * used in mail_out(), wake_up() should correspondingly be called
-		 * here.  For now, we don't need to do anything special.
-		 */
-		goto ack;
-	}
-
-	/* The interrupt is for an incoming mailbox */
-	icmb = flag & MB_MASK;
-	icmb_status = icmbs[icmb].status;
-	if (icmb_status & 0x80) {	/* unsolicited - result in ICMB */
-		dprintk("wd7000_intr: unsolicited interrupt 0x%02x\n", icmb_status);
-		goto ack;
-	}
-
-	/* Aaaargh! (Zaga) */
-	scb = isa_bus_to_virt(scsi2int((unchar *) icmbs[icmb].scbptr));
-	icmbs[icmb].status = 0;
-	if (scb->op & ICB_OP_MASK) {	/* an SCB is done */
-		icb = (IcbAny *) scb;
-		icb->status = icmb_status;
-		icb->phase = 0;
-		goto ack;
-	}
-
-	SCpnt = scb->SCpnt;
-	if (--(SCpnt->SCp.phase) <= 0) {	/* all scbs are done */
-		host_error = scb->vue | (icmb_status << 8);
-		scsi_error = scb->status;
-		errstatus = make_code(host_error, scsi_error);
-		SCpnt->result = errstatus;
-
-		free_scb(scb);
-
-		SCpnt->scsi_done(SCpnt);
-	}
-
- ack:
-	dprintk("wd7000_intr: return from interrupt handler\n");
-	wd7000_intr_ack(host);
-
-	spin_unlock_irqrestore(host->sh->host_lock, flags);
-	return IRQ_HANDLED;
-}
-
-static int wd7000_queuecommand_lck(struct scsi_cmnd *SCpnt,
-		void (*done)(struct scsi_cmnd *))
-{
-	Scb *scb;
-	Sgb *sgb;
-	unchar *cdb = (unchar *) SCpnt->cmnd;
-	unchar idlun;
-	short cdblen;
-	int nseg;
-	Adapter *host = (Adapter *) SCpnt->device->host->hostdata;
-
-	cdblen = SCpnt->cmd_len;
-	idlun = ((SCpnt->device->id << 5) & 0xe0) | (SCpnt->device->lun & 7);
-	SCpnt->scsi_done = done;
-	SCpnt->SCp.phase = 1;
-	scb = alloc_scbs(SCpnt->device->host, 1);
-	scb->idlun = idlun;
-	memcpy(scb->cdb, cdb, cdblen);
-	scb->direc = 0x40;	/* Disable direction check */
-
-	scb->SCpnt = SCpnt;	/* so we can find stuff later */
-	SCpnt->host_scribble = (unchar *) scb;
-	scb->host = host;
-
-	nseg = scsi_sg_count(SCpnt);
-	if (nseg > 1) {
-		struct scatterlist *sg;
-		unsigned i;
-
-		dprintk("Using scatter/gather with %d elements.\n", nseg);
-
-		sgb = scb->sgb;
-		scb->op = 1;
-		any2scsi(scb->dataptr, (int) sgb);
-		any2scsi(scb->maxlen, nseg * sizeof(Sgb));
-
-		scsi_for_each_sg(SCpnt, sg, nseg, i) {
-			any2scsi(sgb[i].ptr, isa_page_to_bus(sg_page(sg)) + sg->offset);
-			any2scsi(sgb[i].len, sg->length);
-		}
-	} else {
-		scb->op = 0;
-		if (nseg) {
-			struct scatterlist *sg = scsi_sglist(SCpnt);
-			any2scsi(scb->dataptr, isa_page_to_bus(sg_page(sg)) + sg->offset);
-		}
-		any2scsi(scb->maxlen, scsi_bufflen(SCpnt));
-	}
-
-	/* FIXME: drop lock and yield here ? */
-
-	while (!mail_out(host, scb))
-		cpu_relax();	/* keep trying */
-
-	return 0;
-}
-
-static DEF_SCSI_QCMD(wd7000_queuecommand)
-
-static int wd7000_diagnostics(Adapter * host, int code)
-{
-	static IcbDiag icb = { ICB_OP_DIAGNOSTICS };
-	static unchar buf[256];
-	unsigned long timeout;
-
-	icb.type = code;
-	any2scsi(icb.len, sizeof(buf));
-	any2scsi(icb.ptr, (int) &buf);
-	icb.phase = 1;
-	/*
-	 * This routine is only called at init, so there should be OGMBs
-	 * available.  I'm assuming so here.  If this is going to
-	 * fail, I can just let the timeout catch the failure.
-	 */
-	mail_out(host, (struct scb *) &icb);
-	timeout = jiffies + WAITnexttimeout;	/* wait up to 2 seconds */
-	while (icb.phase && time_before(jiffies, timeout)) {
-		cpu_relax();	/* wait for completion */
-		barrier();
-	}
-
-	if (icb.phase) {
-		printk("wd7000_diagnostics: timed out.\n");
-		return (0);
-	}
-	if (make_code(icb.vue | (icb.status << 8), 0)) {
-		printk("wd7000_diagnostics: failed (0x%02x,0x%02x)\n", icb.vue, icb.status);
-		return (0);
-	}
-
-	return (1);
-}
-
-
-static int wd7000_adapter_reset(Adapter * host)
-{
-	InitCmd init_cmd = {
-		INITIALIZATION,
-		7,
-		host->bus_on,
-		host->bus_off,
-		0,
-		{0, 0, 0},
-		OGMB_CNT,
-		ICMB_CNT
-	};
-	int diag;
-	/*
-	 *  Reset the adapter - only.  The SCSI bus was initialized at power-up,
-	 *  and we need to do this just so we control the mailboxes, etc.
-	 */
-	outb(ASC_RES, host->iobase + ASC_CONTROL);
-	udelay(40);		/* reset pulse: this is 40us, only need 25us */
-	outb(0, host->iobase + ASC_CONTROL);
-	host->control = 0;	/* this must always shadow ASC_CONTROL */
-
-	if (WAIT(host->iobase + ASC_STAT, ASC_STATMASK, CMD_RDY, 0)) {
-		printk(KERN_ERR "wd7000_init: WAIT timed out.\n");
-		return -1;	/* -1 = not ok */
-	}
-
-	if ((diag = inb(host->iobase + ASC_INTR_STAT)) != 1) {
-		printk("wd7000_init: ");
-
-		switch (diag) {
-		case 2:
-			printk(KERN_ERR "RAM failure.\n");
-			break;
-		case 3:
-			printk(KERN_ERR "FIFO R/W failed\n");
-			break;
-		case 4:
-			printk(KERN_ERR "SBIC register R/W failed\n");
-			break;
-		case 5:
-			printk(KERN_ERR "Initialization D-FF failed.\n");
-			break;
-		case 6:
-			printk(KERN_ERR "Host IRQ D-FF failed.\n");
-			break;
-		case 7:
-			printk(KERN_ERR "ROM checksum error.\n");
-			break;
-		default:
-			printk(KERN_ERR "diagnostic code 0x%02Xh received.\n", diag);
-		}
-		return -1;
-	}
-	/* Clear mailboxes */
-	memset(&(host->mb), 0, sizeof(host->mb));
-
-	/* Execute init command */
-	any2scsi((unchar *) & (init_cmd.mailboxes), (int) &(host->mb));
-	if (!command_out(host, (unchar *) & init_cmd, sizeof(init_cmd))) {
-		printk(KERN_ERR "wd7000_adapter_reset: adapter initialization failed.\n");
-		return -1;
-	}
-
-	if (WAIT(host->iobase + ASC_STAT, ASC_STATMASK, ASC_INIT, 0)) {
-		printk("wd7000_adapter_reset: WAIT timed out.\n");
-		return -1;
-	}
-	return 0;
-}
-
-static int wd7000_init(Adapter * host)
-{
-	if (wd7000_adapter_reset(host) == -1)
-		return 0;
-
-
-	if (request_irq(host->irq, wd7000_intr, 0, "wd7000", host)) {
-		printk("wd7000_init: can't get IRQ %d.\n", host->irq);
-		return (0);
-	}
-	if (request_dma(host->dma, "wd7000")) {
-		printk("wd7000_init: can't get DMA channel %d.\n", host->dma);
-		free_irq(host->irq, host);
-		return (0);
-	}
-	wd7000_enable_dma(host);
-	wd7000_enable_intr(host);
-
-	if (!wd7000_diagnostics(host, ICB_DIAG_FULL)) {
-		free_dma(host->dma);
-		free_irq(host->irq, NULL);
-		return (0);
-	}
-
-	return (1);
-}
-
-
-static void wd7000_revision(Adapter * host)
-{
-	static IcbRevLvl icb = { ICB_OP_GET_REVISION };
-
-	icb.phase = 1;
-	/*
-	 * Like diagnostics, this is only done at init time, in fact, from
-	 * wd7000_detect, so there should be OGMBs available.  If it fails,
-	 * the only damage will be that the revision will show up as 0.0,
-	 * which in turn means that scatter/gather will be disabled.
-	 */
-	mail_out(host, (struct scb *) &icb);
-	while (icb.phase) {
-		cpu_relax();	/* wait for completion */
-		barrier();
-	}
-	host->rev1 = icb.primary;
-	host->rev2 = icb.secondary;
-}
-
-
-static int wd7000_set_info(struct Scsi_Host *host, char *buffer, int length)
-{
-	dprintk("Buffer = <%.*s>, length = %d\n", length, buffer, length);
-
-	/*
-	 * Currently this is a no-op
-	 */
-	dprintk("Sorry, this function is currently out of order...\n");
-	return (length);
-}
-
-
-static int wd7000_show_info(struct seq_file *m, struct Scsi_Host *host)
-{
-	Adapter *adapter = (Adapter *)host->hostdata;
-	unsigned long flags;
-#ifdef WD7000_DEBUG
-	Mailbox *ogmbs, *icmbs;
-	short count;
-#endif
-
-	spin_lock_irqsave(host->host_lock, flags);
-	seq_printf(m, "Host scsi%d: Western Digital WD-7000 (rev %d.%d)\n", host->host_no, adapter->rev1, adapter->rev2);
-	seq_printf(m, "  IO base:      0x%x\n", adapter->iobase);
-	seq_printf(m, "  IRQ:          %d\n", adapter->irq);
-	seq_printf(m, "  DMA channel:  %d\n", adapter->dma);
-	seq_printf(m, "  Interrupts:   %d\n", adapter->int_counter);
-	seq_printf(m, "  BUS_ON time:  %d nanoseconds\n", adapter->bus_on * 125);
-	seq_printf(m, "  BUS_OFF time: %d nanoseconds\n", adapter->bus_off * 125);
-
-#ifdef WD7000_DEBUG
-	ogmbs = adapter->mb.ogmb;
-	icmbs = adapter->mb.icmb;
-
-	seq_printf(m, "\nControl port value: 0x%x\n", adapter->control);
-	seq_puts(m, "Incoming mailbox:\n");
-	seq_printf(m, "  size: %d\n", ICMB_CNT);
-	seq_puts(m, "  queued messages: ");
-
-	for (i = count = 0; i < ICMB_CNT; i++)
-		if (icmbs[i].status) {
-			count++;
-			seq_printf(m, "0x%x ", i);
-		}
-
-	seq_puts(m, count ? "\n" : "none\n");
-
-	seq_puts(m, "Outgoing mailbox:\n");
-	seq_printf(m, "  size: %d\n", OGMB_CNT);
-	seq_printf(m, "  next message: 0x%x\n", adapter->next_ogmb);
-	seq_puts(m, "  queued messages: ");
-
-	for (i = count = 0; i < OGMB_CNT; i++)
-		if (ogmbs[i].status) {
-			count++;
-			seq_printf(m, "0x%x ", i);
-		}
-
-	seq_puts(m, count ? "\n" : "none\n");
-#endif
-
-	spin_unlock_irqrestore(host->host_lock, flags);
-
-	return 0;
-}
-
-
-/*
- *  Returns the number of adapters this driver is supporting.
- *
- *  The source for hosts.c says to wait to call scsi_register until 100%
- *  sure about an adapter.  We need to do it a little sooner here; we
- *  need the storage set up by scsi_register before wd7000_init, and
- *  changing the location of an Adapter structure is more trouble than
- *  calling scsi_unregister.
- *
- */
-
-static __init int wd7000_detect(struct scsi_host_template *tpnt)
-{
-	short present = 0, biosaddr_ptr, sig_ptr, i, pass;
-	short biosptr[NUM_CONFIGS];
-	unsigned iobase;
-	Adapter *host = NULL;
-	struct Scsi_Host *sh;
-	int unit = 0;
-
-	dprintk("wd7000_detect: started\n");
-
-#ifdef MODULE
-	if (wd7000)
-		wd7000_setup(wd7000);
-#endif
-
-	for (i = 0; i < UNITS; wd7000_host[i++] = NULL);
-	for (i = 0; i < NUM_CONFIGS; biosptr[i++] = -1);
-
-	tpnt->proc_name = "wd7000";
-	tpnt->show_info = &wd7000_show_info;
-	tpnt->write_info = wd7000_set_info;
-
-	/*
-	 * Set up SCB free list, which is shared by all adapters
-	 */
-	init_scbs();
-
-	for (pass = 0; pass < NUM_CONFIGS; pass++) {
-		/*
-		 * First, search for BIOS SIGNATURE...
-		 */
-		for (biosaddr_ptr = 0; biosaddr_ptr < NUM_ADDRS; biosaddr_ptr++)
-			for (sig_ptr = 0; sig_ptr < NUM_SIGNATURES; sig_ptr++) {
-				for (i = 0; i < pass; i++)
-					if (biosptr[i] == biosaddr_ptr)
-						break;
-
-				if (i == pass) {
-					void __iomem *biosaddr = ioremap(wd7000_biosaddr[biosaddr_ptr] + signatures[sig_ptr].ofs,
-								 signatures[sig_ptr].len);
-					short bios_match = 1;
-
-					if (biosaddr)
-						bios_match = check_signature(biosaddr, signatures[sig_ptr].sig, signatures[sig_ptr].len);
-
-					iounmap(biosaddr);
-
-					if (bios_match)
-						goto bios_matched;
-				}
-			}
-
-	      bios_matched:
-		/*
-		 * BIOS SIGNATURE has been found.
-		 */
-#ifdef WD7000_DEBUG
-		dprintk("wd7000_detect: pass %d\n", pass + 1);
-
-		if (biosaddr_ptr == NUM_ADDRS)
-			dprintk("WD-7000 SST BIOS not detected...\n");
-		else
-			dprintk("WD-7000 SST BIOS detected at 0x%lx: checking...\n", wd7000_biosaddr[biosaddr_ptr]);
-#endif
-
-		if (configs[pass].irq < 0)
-			continue;
-
-		if (unit == UNITS)
-			continue;
-
-		iobase = configs[pass].iobase;
-
-		dprintk("wd7000_detect: check IO 0x%x region...\n", iobase);
-
-		if (request_region(iobase, 4, "wd7000")) {
-
-			dprintk("wd7000_detect: ASC reset (IO 0x%x) ...", iobase);
-			/*
-			 * ASC reset...
-			 */
-			outb(ASC_RES, iobase + ASC_CONTROL);
-			msleep(10);
-			outb(0, iobase + ASC_CONTROL);
-
-			if (WAIT(iobase + ASC_STAT, ASC_STATMASK, CMD_RDY, 0)) {
-				dprintk("failed!\n");
-				goto err_release;
-			} else
-				dprintk("ok!\n");
-
-			if (inb(iobase + ASC_INTR_STAT) == 1) {
-				/*
-				 *  We register here, to get a pointer to the extra space,
-				 *  which we'll use as the Adapter structure (host) for
-				 *  this adapter.  It is located just after the registered
-				 *  Scsi_Host structure (sh), and is located by the empty
-				 *  array hostdata.
-				 */
-				sh = scsi_register(tpnt, sizeof(Adapter));
-				if (sh == NULL)
-					goto err_release;
-
-				host = (Adapter *) sh->hostdata;
-
-				dprintk("wd7000_detect: adapter allocated at 0x%x\n", (int) host);
-				memset(host, 0, sizeof(Adapter));
-
-				host->irq = configs[pass].irq;
-				host->dma = configs[pass].dma;
-				host->iobase = iobase;
-				host->int_counter = 0;
-				host->bus_on = configs[pass].bus_on;
-				host->bus_off = configs[pass].bus_off;
-				host->sh = wd7000_host[unit] = sh;
-				unit++;
-
-				dprintk("wd7000_detect: Trying init WD-7000 card at IO " "0x%x, IRQ %d, DMA %d...\n", host->iobase, host->irq, host->dma);
-
-				if (!wd7000_init(host))	/* Initialization failed */
-					goto err_unregister;
-
-				/*
-				 *  OK from here - we'll use this adapter/configuration.
-				 */
-				wd7000_revision(host);	/* important for scatter/gather */
-
-				/*
-				 *  For boards before rev 6.0, scatter/gather isn't supported.
-				 */
-				if (host->rev1 < 6)
-					sh->sg_tablesize = 1;
-
-				present++;	/* count it */
-
-				if (biosaddr_ptr != NUM_ADDRS)
-					biosptr[pass] = biosaddr_ptr;
-
-				printk(KERN_INFO "Western Digital WD-7000 (rev %d.%d) ", host->rev1, host->rev2);
-				printk("using IO 0x%x, IRQ %d, DMA %d.\n", host->iobase, host->irq, host->dma);
-				printk("  BUS_ON time: %dns, BUS_OFF time: %dns\n", host->bus_on * 125, host->bus_off * 125);
-			}
-		} else
-			dprintk("wd7000_detect: IO 0x%x region already allocated!\n", iobase);
-
-		continue;
-
-	      err_unregister:
-		scsi_unregister(sh);
-	      err_release:
-		release_region(iobase, 4);
-
-	}
-
-	if (!present)
-		printk("Failed initialization of WD-7000 SCSI card!\n");
-
-	return (present);
-}
-
-static int wd7000_release(struct Scsi_Host *shost)
-{
-	if (shost->irq)
-		free_irq(shost->irq, NULL);
-	if (shost->io_port && shost->n_io_port)
-		release_region(shost->io_port, shost->n_io_port);
-	scsi_unregister(shost);
-	return 0;
-}
-
-#if 0
-/*
- *  I have absolutely NO idea how to do an abort with the WD7000...
- */
-static int wd7000_abort(Scsi_Cmnd * SCpnt)
-{
-	Adapter *host = (Adapter *) SCpnt->device->host->hostdata;
-
-	if (inb(host->iobase + ASC_STAT) & INT_IM) {
-		printk("wd7000_abort: lost interrupt\n");
-		wd7000_intr_handle(host->irq, NULL, NULL);
-		return FAILED;
-	}
-	return FAILED;
-}
-#endif
-
-/*
- *  Last resort. Reinitialize the board.
- */
-
-static int wd7000_host_reset(struct scsi_cmnd *SCpnt)
-{
-	Adapter *host = (Adapter *) SCpnt->device->host->hostdata;
-
-	spin_lock_irq(SCpnt->device->host->host_lock);
-
-	if (wd7000_adapter_reset(host) < 0) {
-		spin_unlock_irq(SCpnt->device->host->host_lock);
-		return FAILED;
-	}
-
-	wd7000_enable_intr(host);
-
-	spin_unlock_irq(SCpnt->device->host->host_lock);
-	return SUCCESS;
-}
-
-/*
- *  This was borrowed directly from aha1542.c. (Zaga)
- */
-
-static int wd7000_biosparam(struct scsi_device *sdev,
-		struct block_device *bdev, sector_t capacity, int *ip)
-{
-	char b[BDEVNAME_SIZE];
-
-	dprintk("wd7000_biosparam: dev=%s, size=%llu, ",
-		bdevname(bdev, b), (u64)capacity);
-	(void)b;	/* unused var warning? */
-
-	/*
-	 * try default translation
-	 */
-	ip[0] = 64;
-	ip[1] = 32;
-	ip[2] = capacity >> 11;
-
-	/*
-	 * for disks >1GB do some guessing
-	 */
-	if (ip[2] >= 1024) {
-		int info[3];
-
-		/*
-		 * try to figure out the geometry from the partition table
-		 */
-		if ((scsicam_bios_param(bdev, capacity, info) < 0) || !(((info[0] == 64) && (info[1] == 32)) || ((info[0] == 255) && (info[1] == 63)))) {
-			printk("wd7000_biosparam: unable to verify geometry for disk with >1GB.\n" "                  using extended translation.\n");
-
-			ip[0] = 255;
-			ip[1] = 63;
-			ip[2] = (unsigned long) capacity / (255 * 63);
-		} else {
-			ip[0] = info[0];
-			ip[1] = info[1];
-			ip[2] = info[2];
-
-			if (info[0] == 255)
-				printk(KERN_INFO "%s: current partition table is " "using extended translation.\n", __func__);
-		}
-	}
-
-	dprintk("bios geometry: head=%d, sec=%d, cyl=%d\n", ip[0], ip[1], ip[2]);
-	dprintk("WARNING: check, if the bios geometry is correct.\n");
-
-	return (0);
-}
-
-MODULE_AUTHOR("Thomas Wuensche, John Boyd, Miroslav Zagorac");
-MODULE_DESCRIPTION("Driver for the WD7000 series ISA controllers");
-MODULE_LICENSE("GPL");
-
-static struct scsi_host_template driver_template = {
-	.proc_name		= "wd7000",
-	.show_info		= wd7000_show_info,
-	.write_info		= wd7000_set_info,
-	.name			= "Western Digital WD-7000",
-	.detect			= wd7000_detect,
-	.release		= wd7000_release,
-	.queuecommand		= wd7000_queuecommand,
-	.eh_host_reset_handler	= wd7000_host_reset,
-	.bios_param		= wd7000_biosparam,
-	.can_queue		= WD7000_Q,
-	.this_id		= 7,
-	.sg_tablesize		= WD7000_SG,
-	.unchecked_isa_dma	= 1,
-	.use_clustering		= ENABLE_CLUSTERING,
-};
-
-#include "scsi_module.c"
diff --git a/drivers/scsi/wd719x.c b/drivers/scsi/wd719x.c
index e3da1a2fdb66..2a9da2e0ea6b 100644
--- a/drivers/scsi/wd719x.c
+++ b/drivers/scsi/wd719x.c
@@ -962,7 +962,7 @@ static void wd719x_pci_remove(struct pci_dev *pdev)
 	scsi_host_put(sh);
 }
 
-static DEFINE_PCI_DEVICE_TABLE(wd719x_pci_table) = {
+static const struct pci_device_id wd719x_pci_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_WD, 0x3296) },
 	{}
 };
diff --git a/drivers/soc/mediatek/mtk-pmic-wrap.c b/drivers/soc/mediatek/mtk-pmic-wrap.c
index a003ba26ca6e..a5f10936fb9c 100644
--- a/drivers/soc/mediatek/mtk-pmic-wrap.c
+++ b/drivers/soc/mediatek/mtk-pmic-wrap.c
@@ -583,7 +583,7 @@ static int pwrap_wait_for_state(struct pmic_wrapper *wrp,
 {
 	unsigned long timeout;
 
-	timeout = jiffies + usecs_to_jiffies(255);
+	timeout = jiffies + usecs_to_jiffies(10000);
 
 	do {
 		if (time_after(jiffies, timeout))
diff --git a/drivers/soc/qcom/smd.c b/drivers/soc/qcom/smd.c
index ac1957dfdf24..322034ab9d37 100644
--- a/drivers/soc/qcom/smd.c
+++ b/drivers/soc/qcom/smd.c
@@ -95,7 +95,7 @@ static const struct {
 
 /**
  * struct qcom_smd_edge - representing a remote processor
- * @smd:		handle to qcom_smd
+ * @dev:		device for this edge
  * @of_node:		of_node handle for information related to this edge
  * @edge_id:		identifier of this edge
  * @remote_pid:		identifier of remote processor
@@ -111,7 +111,8 @@ static const struct {
  * @state_work:		work item for edge state changes
  */
 struct qcom_smd_edge {
-	struct qcom_smd *smd;
+	struct device dev;
+
 	struct device_node *of_node;
 	unsigned edge_id;
 	unsigned remote_pid;
@@ -135,6 +136,8 @@ struct qcom_smd_edge {
 	struct work_struct state_work;
 };
 
+#define to_smd_edge(d) container_of(d, struct qcom_smd_edge, dev)
+
 /*
  * SMD channel states.
  */
@@ -197,20 +200,6 @@ struct qcom_smd_channel {
 	void *drvdata;
 
 	struct list_head list;
-	struct list_head dev_list;
-};
-
-/**
- * struct qcom_smd - smd struct
- * @dev:	device struct
- * @num_edges:	number of entries in @edges
- * @edges:	array of edges to be handled
- */
-struct qcom_smd {
-	struct device *dev;
-
-	unsigned num_edges;
-	struct qcom_smd_edge edges[0];
 };
 
 /*
@@ -374,7 +363,7 @@ static void qcom_smd_channel_reset(struct qcom_smd_channel *channel)
 	SET_TX_CHANNEL_FLAG(channel, fSTATE, 1);
 	SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 1);
 	SET_TX_CHANNEL_INFO(channel, head, 0);
-	SET_TX_CHANNEL_INFO(channel, tail, 0);
+	SET_RX_CHANNEL_INFO(channel, tail, 0);
 
 	qcom_smd_signal_channel(channel);
 
@@ -421,7 +410,7 @@ static void qcom_smd_channel_set_state(struct qcom_smd_channel *channel,
 	if (channel->state == state)
 		return;
 
-	dev_dbg(edge->smd->dev, "set_state(%s, %d)\n", channel->name, state);
+	dev_dbg(&edge->dev, "set_state(%s, %d)\n", channel->name, state);
 
 	SET_TX_CHANNEL_FLAG(channel, fDSR, is_open);
 	SET_TX_CHANNEL_FLAG(channel, fCTS, is_open);
@@ -891,8 +880,6 @@ static int qcom_smd_dev_remove(struct device *dev)
 	struct qcom_smd_device *qsdev = to_smd_device(dev);
 	struct qcom_smd_driver *qsdrv = to_smd_driver(dev);
 	struct qcom_smd_channel *channel = qsdev->channel;
-	struct qcom_smd_channel *tmp;
-	struct qcom_smd_channel *ch;
 
 	qcom_smd_channel_set_state(channel, SMD_CHANNEL_CLOSING);
 
@@ -911,15 +898,9 @@ static int qcom_smd_dev_remove(struct device *dev)
 	if (qsdrv->remove)
 		qsdrv->remove(qsdev);
 
-	/*
-	 * The client is now gone, close and release all channels associated
-	 * with this sdev
-	 */
-	list_for_each_entry_safe(ch, tmp, &channel->dev_list, dev_list) {
-		qcom_smd_channel_close(ch);
-		list_del(&ch->dev_list);
-		ch->qsdev = NULL;
-	}
+	/* The client is now gone, close the primary channel */
+	qcom_smd_channel_close(channel);
+	channel->qsdev = NULL;
 
 	return 0;
 }
@@ -973,13 +954,12 @@ static int qcom_smd_create_device(struct qcom_smd_channel *channel)
 	struct qcom_smd_device *qsdev;
 	struct qcom_smd_edge *edge = channel->edge;
 	struct device_node *node;
-	struct qcom_smd *smd = edge->smd;
 	int ret;
 
 	if (channel->qsdev)
 		return -EEXIST;
 
-	dev_dbg(smd->dev, "registering '%s'\n", channel->name);
+	dev_dbg(&edge->dev, "registering '%s'\n", channel->name);
 
 	qsdev = kzalloc(sizeof(*qsdev), GFP_KERNEL);
 	if (!qsdev)
@@ -990,7 +970,7 @@ static int qcom_smd_create_device(struct qcom_smd_channel *channel)
 		     edge->of_node->name,
 		     node ? node->name : channel->name);
 
-	qsdev->dev.parent = smd->dev;
+	qsdev->dev.parent = &edge->dev;
 	qsdev->dev.bus = &qcom_smd_bus;
 	qsdev->dev.release = qcom_smd_release_device;
 	qsdev->dev.of_node = node;
@@ -1001,7 +981,7 @@ static int qcom_smd_create_device(struct qcom_smd_channel *channel)
 
 	ret = device_register(&qsdev->dev);
 	if (ret) {
-		dev_err(smd->dev, "device_register failed: %d\n", ret);
+		dev_err(&edge->dev, "device_register failed: %d\n", ret);
 		put_device(&qsdev->dev);
 	}
 
@@ -1091,6 +1071,8 @@ qcom_smd_find_channel(struct qcom_smd_edge *edge, const char *name)
  *
  * Returns a channel handle on success, or -EPROBE_DEFER if the channel isn't
  * ready.
+ *
+ * Any channels returned must be closed with a call to qcom_smd_close_channel()
  */
 struct qcom_smd_channel *qcom_smd_open_channel(struct qcom_smd_channel *parent,
 					       const char *name,
@@ -1120,15 +1102,21 @@ struct qcom_smd_channel *qcom_smd_open_channel(struct qcom_smd_channel *parent,
 		return ERR_PTR(ret);
 	}
 
-	/*
-	 * Append the list of channel to the channels associated with the sdev
-	 */
-	list_add_tail(&channel->dev_list, &sdev->channel->dev_list);
-
 	return channel;
 }
 EXPORT_SYMBOL(qcom_smd_open_channel);
 
+/**
+ * qcom_smd_close_channel() - close an additionally opened channel
+ * @channel:	channel handle, returned by qcom_smd_open_channel()
+ */
+void qcom_smd_close_channel(struct qcom_smd_channel *channel)
+{
+	qcom_smd_channel_close(channel);
+	channel->qsdev = NULL;
+}
+EXPORT_SYMBOL(qcom_smd_close_channel);
+
 /*
  * Allocate the qcom_smd_channel object for a newly found smd channel,
  * retrieving and validating the smem items involved.
@@ -1139,20 +1127,18 @@ static struct qcom_smd_channel *qcom_smd_create_channel(struct qcom_smd_edge *ed
 							char *name)
 {
 	struct qcom_smd_channel *channel;
-	struct qcom_smd *smd = edge->smd;
 	size_t fifo_size;
 	size_t info_size;
 	void *fifo_base;
 	void *info;
 	int ret;
 
-	channel = devm_kzalloc(smd->dev, sizeof(*channel), GFP_KERNEL);
+	channel = devm_kzalloc(&edge->dev, sizeof(*channel), GFP_KERNEL);
 	if (!channel)
 		return ERR_PTR(-ENOMEM);
 
-	INIT_LIST_HEAD(&channel->dev_list);
 	channel->edge = edge;
-	channel->name = devm_kstrdup(smd->dev, name, GFP_KERNEL);
+	channel->name = devm_kstrdup(&edge->dev, name, GFP_KERNEL);
 	if (!channel->name)
 		return ERR_PTR(-ENOMEM);
 
@@ -1175,7 +1161,7 @@ static struct qcom_smd_channel *qcom_smd_create_channel(struct qcom_smd_edge *ed
 	} else if (info_size == 2 * sizeof(struct smd_channel_info)) {
 		channel->info = info;
 	} else {
-		dev_err(smd->dev,
+		dev_err(&edge->dev,
 			"channel info of size %zu not supported\n", info_size);
 		ret = -EINVAL;
 		goto free_name_and_channel;
@@ -1190,7 +1176,7 @@ static struct qcom_smd_channel *qcom_smd_create_channel(struct qcom_smd_edge *ed
 	/* The channel consist of a rx and tx fifo of equal size */
 	fifo_size /= 2;
 
-	dev_dbg(smd->dev, "new channel '%s' info-size: %zu fifo-size: %zu\n",
+	dev_dbg(&edge->dev, "new channel '%s' info-size: %zu fifo-size: %zu\n",
 			  name, info_size, fifo_size);
 
 	channel->tx_fifo = fifo_base;
@@ -1202,8 +1188,8 @@ static struct qcom_smd_channel *qcom_smd_create_channel(struct qcom_smd_edge *ed
 	return channel;
 
 free_name_and_channel:
-	devm_kfree(smd->dev, channel->name);
-	devm_kfree(smd->dev, channel);
+	devm_kfree(&edge->dev, channel->name);
+	devm_kfree(&edge->dev, channel);
 
 	return ERR_PTR(ret);
 }
@@ -1219,7 +1205,6 @@ static void qcom_channel_scan_worker(struct work_struct *work)
 	struct qcom_smd_alloc_entry *alloc_tbl;
 	struct qcom_smd_alloc_entry *entry;
 	struct qcom_smd_channel *channel;
-	struct qcom_smd *smd = edge->smd;
 	unsigned long flags;
 	unsigned fifo_id;
 	unsigned info_id;
@@ -1263,7 +1248,7 @@ static void qcom_channel_scan_worker(struct work_struct *work)
 			list_add(&channel->list, &edge->channels);
 			spin_unlock_irqrestore(&edge->channels_lock, flags);
 
-			dev_dbg(smd->dev, "new channel found: '%s'\n", channel->name);
+			dev_dbg(&edge->dev, "new channel found: '%s'\n", channel->name);
 			set_bit(i, edge->allocated[tbl]);
 
 			wake_up_interruptible(&edge->new_channel_event);
@@ -1350,22 +1335,6 @@ static int qcom_smd_parse_edge(struct device *dev,
 
 	edge->of_node = of_node_get(node);
 
-	irq = irq_of_parse_and_map(node, 0);
-	if (irq < 0) {
-		dev_err(dev, "required smd interrupt missing\n");
-		return -EINVAL;
-	}
-
-	ret = devm_request_irq(dev, irq,
-			       qcom_smd_edge_intr, IRQF_TRIGGER_RISING,
-			       node->name, edge);
-	if (ret) {
-		dev_err(dev, "failed to request smd irq\n");
-		return ret;
-	}
-
-	edge->irq = irq;
-
 	key = "qcom,smd-edge";
 	ret = of_property_read_u32(node, key, &edge->edge_id);
 	if (ret) {
@@ -1400,18 +1369,121 @@ static int qcom_smd_parse_edge(struct device *dev,
 		return -EINVAL;
 	}
 
+	irq = irq_of_parse_and_map(node, 0);
+	if (irq < 0) {
+		dev_err(dev, "required smd interrupt missing\n");
+		return -EINVAL;
+	}
+
+	ret = devm_request_irq(dev, irq,
+			       qcom_smd_edge_intr, IRQF_TRIGGER_RISING,
+			       node->name, edge);
+	if (ret) {
+		dev_err(dev, "failed to request smd irq\n");
+		return ret;
+	}
+
+	edge->irq = irq;
+
 	return 0;
 }
 
-static int qcom_smd_probe(struct platform_device *pdev)
+/*
+ * Release function for an edge.
+ * Reset the state of each associated channel and free the edge context.
+ */
+static void qcom_smd_edge_release(struct device *dev)
+{
+	struct qcom_smd_channel *channel;
+	struct qcom_smd_edge *edge = to_smd_edge(dev);
+
+	list_for_each_entry(channel, &edge->channels, list) {
+		SET_RX_CHANNEL_INFO(channel, state, SMD_CHANNEL_CLOSED);
+		SET_RX_CHANNEL_INFO(channel, head, 0);
+		SET_RX_CHANNEL_INFO(channel, tail, 0);
+	}
+
+	kfree(edge);
+}
+
+/**
+ * qcom_smd_register_edge() - register an edge based on an device_node
+ * @parent:	parent device for the edge
+ * @node:	device_node describing the edge
+ *
+ * Returns an edge reference, or negative ERR_PTR() on failure.
+ */
+struct qcom_smd_edge *qcom_smd_register_edge(struct device *parent,
+					     struct device_node *node)
 {
 	struct qcom_smd_edge *edge;
-	struct device_node *node;
-	struct qcom_smd *smd;
-	size_t array_size;
-	int num_edges;
 	int ret;
-	int i = 0;
+
+	edge = kzalloc(sizeof(*edge), GFP_KERNEL);
+	if (!edge)
+		return ERR_PTR(-ENOMEM);
+
+	init_waitqueue_head(&edge->new_channel_event);
+
+	edge->dev.parent = parent;
+	edge->dev.release = qcom_smd_edge_release;
+	dev_set_name(&edge->dev, "%s:%s", dev_name(parent), node->name);
+	ret = device_register(&edge->dev);
+	if (ret) {
+		pr_err("failed to register smd edge\n");
+		return ERR_PTR(ret);
+	}
+
+	ret = qcom_smd_parse_edge(&edge->dev, node, edge);
+	if (ret) {
+		dev_err(&edge->dev, "failed to parse smd edge\n");
+		goto unregister_dev;
+	}
+
+	schedule_work(&edge->scan_work);
+
+	return edge;
+
+unregister_dev:
+	put_device(&edge->dev);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(qcom_smd_register_edge);
+
+static int qcom_smd_remove_device(struct device *dev, void *data)
+{
+	device_unregister(dev);
+	of_node_put(dev->of_node);
+	put_device(dev);
+
+	return 0;
+}
+
+/**
+ * qcom_smd_unregister_edge() - release an edge and its children
+ * @edge:	edge reference acquired from qcom_smd_register_edge
+ */
+int qcom_smd_unregister_edge(struct qcom_smd_edge *edge)
+{
+	int ret;
+
+	disable_irq(edge->irq);
+	cancel_work_sync(&edge->scan_work);
+	cancel_work_sync(&edge->state_work);
+
+	ret = device_for_each_child(&edge->dev, NULL, qcom_smd_remove_device);
+	if (ret)
+		dev_warn(&edge->dev, "can't remove smd device: %d\n", ret);
+
+	device_unregister(&edge->dev);
+
+	return 0;
+}
+EXPORT_SYMBOL(qcom_smd_unregister_edge);
+
+static int qcom_smd_probe(struct platform_device *pdev)
+{
+	struct device_node *node;
 	void *p;
 
 	/* Wait for smem */
@@ -1419,29 +1491,17 @@ static int qcom_smd_probe(struct platform_device *pdev)
 	if (PTR_ERR(p) == -EPROBE_DEFER)
 		return PTR_ERR(p);
 
-	num_edges = of_get_available_child_count(pdev->dev.of_node);
-	array_size = sizeof(*smd) + num_edges * sizeof(struct qcom_smd_edge);
-	smd = devm_kzalloc(&pdev->dev, array_size, GFP_KERNEL);
-	if (!smd)
-		return -ENOMEM;
-	smd->dev = &pdev->dev;
-
-	smd->num_edges = num_edges;
-	for_each_available_child_of_node(pdev->dev.of_node, node) {
-		edge = &smd->edges[i++];
-		edge->smd = smd;
-		init_waitqueue_head(&edge->new_channel_event);
-
-		ret = qcom_smd_parse_edge(&pdev->dev, node, edge);
-		if (ret)
-			continue;
+	for_each_available_child_of_node(pdev->dev.of_node, node)
+		qcom_smd_register_edge(&pdev->dev, node);
 
-		schedule_work(&edge->scan_work);
-	}
+	return 0;
+}
 
-	platform_set_drvdata(pdev, smd);
+static int qcom_smd_remove_edge(struct device *dev, void *data)
+{
+	struct qcom_smd_edge *edge = to_smd_edge(dev);
 
-	return 0;
+	return qcom_smd_unregister_edge(edge);
 }
 
 /*
@@ -1450,28 +1510,13 @@ static int qcom_smd_probe(struct platform_device *pdev)
  */
 static int qcom_smd_remove(struct platform_device *pdev)
 {
-	struct qcom_smd_channel *channel;
-	struct qcom_smd_edge *edge;
-	struct qcom_smd *smd = platform_get_drvdata(pdev);
-	int i;
-
-	for (i = 0; i < smd->num_edges; i++) {
-		edge = &smd->edges[i];
-
-		disable_irq(edge->irq);
-		cancel_work_sync(&edge->scan_work);
-		cancel_work_sync(&edge->state_work);
-
-		/* No need to lock here, because the writer is gone */
-		list_for_each_entry(channel, &edge->channels, list) {
-			if (!channel->qsdev)
-				continue;
+	int ret;
 
-			qcom_smd_destroy_device(channel);
-		}
-	}
+	ret = device_for_each_child(&pdev->dev, NULL, qcom_smd_remove_edge);
+	if (ret)
+		dev_warn(&pdev->dev, "can't remove smd device: %d\n", ret);
 
-	return 0;
+	return ret;
 }
 
 static const struct of_device_id qcom_smd_of_match[] = {
diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c
index 2e1aa9f130f4..18ec52f2078a 100644
--- a/drivers/soc/qcom/smem.c
+++ b/drivers/soc/qcom/smem.c
@@ -740,7 +740,8 @@ static int qcom_smem_probe(struct platform_device *pdev)
 
 	hwlock_id = of_hwspin_lock_get_id(pdev->dev.of_node, 0);
 	if (hwlock_id < 0) {
-		dev_err(&pdev->dev, "failed to retrieve hwlock\n");
+		if (hwlock_id != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to retrieve hwlock\n");
 		return hwlock_id;
 	}
 
diff --git a/drivers/soc/rockchip/pm_domains.c b/drivers/soc/rockchip/pm_domains.c
index 44842a205e4b..7acd1517dd37 100644
--- a/drivers/soc/rockchip/pm_domains.c
+++ b/drivers/soc/rockchip/pm_domains.c
@@ -27,6 +27,7 @@ struct rockchip_domain_info {
 	int req_mask;
 	int idle_mask;
 	int ack_mask;
+	bool active_wakeup;
 };
 
 struct rockchip_pmu_info {
@@ -75,23 +76,24 @@ struct rockchip_pmu {
 
 #define to_rockchip_pd(gpd) container_of(gpd, struct rockchip_pm_domain, genpd)
 
-#define DOMAIN(pwr, status, req, idle, ack)	\
+#define DOMAIN(pwr, status, req, idle, ack, wakeup)	\
 {						\
 	.pwr_mask = (pwr >= 0) ? BIT(pwr) : 0,		\
 	.status_mask = (status >= 0) ? BIT(status) : 0,	\
 	.req_mask = (req >= 0) ? BIT(req) : 0,		\
 	.idle_mask = (idle >= 0) ? BIT(idle) : 0,	\
 	.ack_mask = (ack >= 0) ? BIT(ack) : 0,		\
+	.active_wakeup = wakeup,			\
 }
 
-#define DOMAIN_RK3288(pwr, status, req)		\
-	DOMAIN(pwr, status, req, req, (req) + 16)
+#define DOMAIN_RK3288(pwr, status, req, wakeup)		\
+	DOMAIN(pwr, status, req, req, (req) + 16, wakeup)
 
-#define DOMAIN_RK3368(pwr, status, req)		\
-	DOMAIN(pwr, status, req, (req) + 16, req)
+#define DOMAIN_RK3368(pwr, status, req, wakeup)		\
+	DOMAIN(pwr, status, req, (req) + 16, req, wakeup)
 
-#define DOMAIN_RK3399(pwr, status, req)                \
-	DOMAIN(pwr, status, req, req, req)
+#define DOMAIN_RK3399(pwr, status, req, wakeup)		\
+	DOMAIN(pwr, status, req, req, req, wakeup)
 
 static bool rockchip_pmu_domain_is_idle(struct rockchip_pm_domain *pd)
 {
@@ -295,6 +297,17 @@ static void rockchip_pd_detach_dev(struct generic_pm_domain *genpd,
 	pm_clk_destroy(dev);
 }
 
+static bool rockchip_active_wakeup(struct device *dev)
+{
+	struct generic_pm_domain *genpd;
+	struct rockchip_pm_domain *pd;
+
+	genpd = pd_to_genpd(dev->pm_domain);
+	pd = container_of(genpd, struct rockchip_pm_domain, genpd);
+
+	return pd->info->active_wakeup;
+}
+
 static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu,
 				      struct device_node *node)
 {
@@ -415,6 +428,7 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu,
 	pd->genpd.power_on = rockchip_pd_power_on;
 	pd->genpd.attach_dev = rockchip_pd_attach_dev;
 	pd->genpd.detach_dev = rockchip_pd_detach_dev;
+	pd->genpd.dev_ops.active_wakeup = rockchip_active_wakeup;
 	pd->genpd.flags = GENPD_FLAG_PM_CLK;
 	pm_genpd_init(&pd->genpd, NULL, false);
 
@@ -623,48 +637,48 @@ err_out:
 }
 
 static const struct rockchip_domain_info rk3288_pm_domains[] = {
-	[RK3288_PD_VIO]		= DOMAIN_RK3288(7, 7, 4),
-	[RK3288_PD_HEVC]	= DOMAIN_RK3288(14, 10, 9),
-	[RK3288_PD_VIDEO]	= DOMAIN_RK3288(8, 8, 3),
-	[RK3288_PD_GPU]		= DOMAIN_RK3288(9, 9, 2),
+	[RK3288_PD_VIO]		= DOMAIN_RK3288(7, 7, 4, false),
+	[RK3288_PD_HEVC]	= DOMAIN_RK3288(14, 10, 9, false),
+	[RK3288_PD_VIDEO]	= DOMAIN_RK3288(8, 8, 3, false),
+	[RK3288_PD_GPU]		= DOMAIN_RK3288(9, 9, 2, false),
 };
 
 static const struct rockchip_domain_info rk3368_pm_domains[] = {
-	[RK3368_PD_PERI]	= DOMAIN_RK3368(13, 12, 6),
-	[RK3368_PD_VIO]		= DOMAIN_RK3368(15, 14, 8),
-	[RK3368_PD_VIDEO]	= DOMAIN_RK3368(14, 13, 7),
-	[RK3368_PD_GPU_0]	= DOMAIN_RK3368(16, 15, 2),
-	[RK3368_PD_GPU_1]	= DOMAIN_RK3368(17, 16, 2),
+	[RK3368_PD_PERI]	= DOMAIN_RK3368(13, 12, 6, true),
+	[RK3368_PD_VIO]		= DOMAIN_RK3368(15, 14, 8, false),
+	[RK3368_PD_VIDEO]	= DOMAIN_RK3368(14, 13, 7, false),
+	[RK3368_PD_GPU_0]	= DOMAIN_RK3368(16, 15, 2, false),
+	[RK3368_PD_GPU_1]	= DOMAIN_RK3368(17, 16, 2, false),
 };
 
 static const struct rockchip_domain_info rk3399_pm_domains[] = {
-	[RK3399_PD_TCPD0]	= DOMAIN_RK3399(8, 8, -1),
-	[RK3399_PD_TCPD1]	= DOMAIN_RK3399(9, 9, -1),
-	[RK3399_PD_CCI]		= DOMAIN_RK3399(10, 10, -1),
-	[RK3399_PD_CCI0]	= DOMAIN_RK3399(-1, -1, 15),
-	[RK3399_PD_CCI1]	= DOMAIN_RK3399(-1, -1, 16),
-	[RK3399_PD_PERILP]	= DOMAIN_RK3399(11, 11, 1),
-	[RK3399_PD_PERIHP]	= DOMAIN_RK3399(12, 12, 2),
-	[RK3399_PD_CENTER]	= DOMAIN_RK3399(13, 13, 14),
-	[RK3399_PD_VIO]		= DOMAIN_RK3399(14, 14, 17),
-	[RK3399_PD_GPU]		= DOMAIN_RK3399(15, 15, 0),
-	[RK3399_PD_VCODEC]	= DOMAIN_RK3399(16, 16, 3),
-	[RK3399_PD_VDU]		= DOMAIN_RK3399(17, 17, 4),
-	[RK3399_PD_RGA]		= DOMAIN_RK3399(18, 18, 5),
-	[RK3399_PD_IEP]		= DOMAIN_RK3399(19, 19, 6),
-	[RK3399_PD_VO]		= DOMAIN_RK3399(20, 20, -1),
-	[RK3399_PD_VOPB]	= DOMAIN_RK3399(-1, -1, 7),
-	[RK3399_PD_VOPL]	= DOMAIN_RK3399(-1, -1, 8),
-	[RK3399_PD_ISP0]	= DOMAIN_RK3399(22, 22, 9),
-	[RK3399_PD_ISP1]	= DOMAIN_RK3399(23, 23, 10),
-	[RK3399_PD_HDCP]	= DOMAIN_RK3399(24, 24, 11),
-	[RK3399_PD_GMAC]	= DOMAIN_RK3399(25, 25, 23),
-	[RK3399_PD_EMMC]	= DOMAIN_RK3399(26, 26, 24),
-	[RK3399_PD_USB3]	= DOMAIN_RK3399(27, 27, 12),
-	[RK3399_PD_EDP]		= DOMAIN_RK3399(28, 28, 22),
-	[RK3399_PD_GIC]		= DOMAIN_RK3399(29, 29, 27),
-	[RK3399_PD_SD]		= DOMAIN_RK3399(30, 30, 28),
-	[RK3399_PD_SDIOAUDIO]	= DOMAIN_RK3399(31, 31, 29),
+	[RK3399_PD_TCPD0]	= DOMAIN_RK3399(8, 8, -1, false),
+	[RK3399_PD_TCPD1]	= DOMAIN_RK3399(9, 9, -1, false),
+	[RK3399_PD_CCI]		= DOMAIN_RK3399(10, 10, -1, true),
+	[RK3399_PD_CCI0]	= DOMAIN_RK3399(-1, -1, 15, true),
+	[RK3399_PD_CCI1]	= DOMAIN_RK3399(-1, -1, 16, true),
+	[RK3399_PD_PERILP]	= DOMAIN_RK3399(11, 11, 1, true),
+	[RK3399_PD_PERIHP]	= DOMAIN_RK3399(12, 12, 2, true),
+	[RK3399_PD_CENTER]	= DOMAIN_RK3399(13, 13, 14, true),
+	[RK3399_PD_VIO]		= DOMAIN_RK3399(14, 14, 17, false),
+	[RK3399_PD_GPU]		= DOMAIN_RK3399(15, 15, 0, false),
+	[RK3399_PD_VCODEC]	= DOMAIN_RK3399(16, 16, 3, false),
+	[RK3399_PD_VDU]		= DOMAIN_RK3399(17, 17, 4, false),
+	[RK3399_PD_RGA]		= DOMAIN_RK3399(18, 18, 5, false),
+	[RK3399_PD_IEP]		= DOMAIN_RK3399(19, 19, 6, false),
+	[RK3399_PD_VO]		= DOMAIN_RK3399(20, 20, -1, false),
+	[RK3399_PD_VOPB]	= DOMAIN_RK3399(-1, -1, 7, false),
+	[RK3399_PD_VOPL]	= DOMAIN_RK3399(-1, -1, 8, false),
+	[RK3399_PD_ISP0]	= DOMAIN_RK3399(22, 22, 9, false),
+	[RK3399_PD_ISP1]	= DOMAIN_RK3399(23, 23, 10, false),
+	[RK3399_PD_HDCP]	= DOMAIN_RK3399(24, 24, 11, false),
+	[RK3399_PD_GMAC]	= DOMAIN_RK3399(25, 25, 23, true),
+	[RK3399_PD_EMMC]	= DOMAIN_RK3399(26, 26, 24, true),
+	[RK3399_PD_USB3]	= DOMAIN_RK3399(27, 27, 12, true),
+	[RK3399_PD_EDP]		= DOMAIN_RK3399(28, 28, 22, false),
+	[RK3399_PD_GIC]		= DOMAIN_RK3399(29, 29, 27, true),
+	[RK3399_PD_SD]		= DOMAIN_RK3399(30, 30, 28, true),
+	[RK3399_PD_SDIOAUDIO]	= DOMAIN_RK3399(31, 31, 29, true),
 };
 
 static const struct rockchip_pmu_info rk3288_pmu = {
diff --git a/drivers/soc/samsung/pm_domains.c b/drivers/soc/samsung/pm_domains.c
index 4822346aadc6..7112004b8032 100644
--- a/drivers/soc/samsung/pm_domains.c
+++ b/drivers/soc/samsung/pm_domains.c
@@ -215,29 +215,22 @@ no_clk:
 
 	/* Assign the child power domains to their parents */
 	for_each_matching_node(np, exynos_pm_domain_of_match) {
-		struct generic_pm_domain *child_domain, *parent_domain;
-		struct of_phandle_args args;
+		struct of_phandle_args child, parent;
 
-		args.np = np;
-		args.args_count = 0;
-		child_domain = of_genpd_get_from_provider(&args);
-		if (IS_ERR(child_domain))
-			continue;
+		child.np = np;
+		child.args_count = 0;
 
 		if (of_parse_phandle_with_args(np, "power-domains",
-					 "#power-domain-cells", 0, &args) != 0)
-			continue;
-
-		parent_domain = of_genpd_get_from_provider(&args);
-		if (IS_ERR(parent_domain))
+					       "#power-domain-cells", 0,
+					       &parent) != 0)
 			continue;
 
-		if (pm_genpd_add_subdomain(parent_domain, child_domain))
+		if (of_genpd_add_subdomain(&parent, &child))
 			pr_warn("%s failed to add subdomain: %s\n",
-				parent_domain->name, child_domain->name);
+				parent.np->name, child.np->name);
 		else
 			pr_info("%s has as child subdomain: %s.\n",
-				parent_domain->name, child_domain->name);
+				parent.np->name, child.np->name);
 	}
 
 	return 0;
diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
index 71c834f3847e..7792ed88d80b 100644
--- a/drivers/soc/tegra/pmc.c
+++ b/drivers/soc/tegra/pmc.c
@@ -967,8 +967,8 @@ static void tegra_io_rail_unprepare(void)
 
 int tegra_io_rail_power_on(unsigned int id)
 {
-	unsigned long request, status, value;
-	unsigned int bit, mask;
+	unsigned long request, status;
+	unsigned int bit;
 	int err;
 
 	mutex_lock(&pmc->powergates_lock);
@@ -977,15 +977,9 @@ int tegra_io_rail_power_on(unsigned int id)
 	if (err)
 		goto error;
 
-	mask = 1 << bit;
+	tegra_pmc_writel(IO_DPD_REQ_CODE_OFF | BIT(bit), request);
 
-	value = tegra_pmc_readl(request);
-	value |= mask;
-	value &= ~IO_DPD_REQ_CODE_MASK;
-	value |= IO_DPD_REQ_CODE_OFF;
-	tegra_pmc_writel(value, request);
-
-	err = tegra_io_rail_poll(status, mask, 0, 250);
+	err = tegra_io_rail_poll(status, BIT(bit), 0, 250);
 	if (err) {
 		pr_info("tegra_io_rail_poll() failed: %d\n", err);
 		goto error;
@@ -1002,8 +996,8 @@ EXPORT_SYMBOL(tegra_io_rail_power_on);
 
 int tegra_io_rail_power_off(unsigned int id)
 {
-	unsigned long request, status, value;
-	unsigned int bit, mask;
+	unsigned long request, status;
+	unsigned int bit;
 	int err;
 
 	mutex_lock(&pmc->powergates_lock);
@@ -1014,15 +1008,9 @@ int tegra_io_rail_power_off(unsigned int id)
 		goto error;
 	}
 
-	mask = 1 << bit;
-
-	value = tegra_pmc_readl(request);
-	value |= mask;
-	value &= ~IO_DPD_REQ_CODE_MASK;
-	value |= IO_DPD_REQ_CODE_ON;
-	tegra_pmc_writel(value, request);
+	tegra_pmc_writel(IO_DPD_REQ_CODE_ON | BIT(bit), request);
 
-	err = tegra_io_rail_poll(status, mask, mask, 250);
+	err = tegra_io_rail_poll(status, BIT(bit), BIT(bit), 250);
 	if (err)
 		goto error;
 
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index d6fb8d4b7786..b7995474148c 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -153,6 +153,16 @@ config SPI_BCM63XX_HSSPI
 	  This enables support for the High Speed SPI controller present on
 	  newer Broadcom BCM63XX SoCs.
 
+config SPI_BCM_QSPI
+	tristate "Broadcom BSPI and MSPI controller support"
+	depends on ARCH_BRCMSTB || ARCH_BCM || ARCH_BCM_IPROC || COMPILE_TEST
+	default ARCH_BCM_IPROC
+	help
+	  Enables support for the Broadcom SPI flash and MSPI controller.
+	  Select this option for any one of BRCMSTB, iProc NSP and NS2 SoCs
+	  based platforms. This driver works for both SPI master for spi-nor
+	  flash device as well as MSPI device.
+
 config SPI_BITBANG
 	tristate "Utilities for Bitbanging SPI masters"
 	help
@@ -285,6 +295,13 @@ config SPI_IMX
 	  This enables using the Freescale i.MX SPI controllers in master
 	  mode.
 
+config SPI_JCORE
+	tristate "J-Core SPI Master"
+	depends on OF && (SUPERH || COMPILE_TEST)
+	help
+	  This enables support for the SPI master controller in the J-Core
+	  synthesizable, open source SoC.
+
 config SPI_LM70_LLP
 	tristate "Parallel port adapter for LM70 eval board (DEVELOPMENT)"
 	depends on PARPORT
@@ -549,7 +566,7 @@ config SPI_SC18IS602
 config SPI_SH_MSIOF
 	tristate "SuperH MSIOF SPI controller"
 	depends on HAVE_CLK && HAS_DMA
-	depends on SUPERH || ARCH_RENESAS || COMPILE_TEST
+	depends on ARCH_SHMOBILE || ARCH_RENESAS || COMPILE_TEST
 	help
 	  SPI driver for SuperH and SH Mobile MSIOF blocks.
 
@@ -631,6 +648,13 @@ config SPI_TEGRA20_SLINK
 	help
 	  SPI driver for Nvidia Tegra20/Tegra30 SLINK Controller interface.
 
+config SPI_THUNDERX
+	tristate "Cavium ThunderX SPI controller"
+	depends on PCI && 64BIT && (ARM64 || COMPILE_TEST)
+	help
+	  SPI host driver for the hardware found on Cavium ThunderX
+	  SOCs.
+
 config SPI_TOPCLIFF_PCH
 	tristate "Intel EG20T PCH/LAPIS Semicon IOH(ML7213/ML7223/ML7831) SPI"
 	depends on PCI && (X86_32 || MIPS || COMPILE_TEST)
diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile
index 185367ef6576..aa939d955521 100644
--- a/drivers/spi/Makefile
+++ b/drivers/spi/Makefile
@@ -21,6 +21,7 @@ obj-$(CONFIG_SPI_BCM2835AUX)		+= spi-bcm2835aux.o
 obj-$(CONFIG_SPI_BCM53XX)		+= spi-bcm53xx.o
 obj-$(CONFIG_SPI_BCM63XX)		+= spi-bcm63xx.o
 obj-$(CONFIG_SPI_BCM63XX_HSSPI)		+= spi-bcm63xx-hsspi.o
+obj-$(CONFIG_SPI_BCM_QSPI)		+= spi-iproc-qspi.o spi-brcmstb-qspi.o spi-bcm-qspi.o
 obj-$(CONFIG_SPI_BFIN5XX)		+= spi-bfin5xx.o
 obj-$(CONFIG_SPI_ADI_V3)                += spi-adi-v3.o
 obj-$(CONFIG_SPI_BFIN_SPORT)		+= spi-bfin-sport.o
@@ -46,6 +47,7 @@ obj-$(CONFIG_SPI_FSL_SPI)		+= spi-fsl-spi.o
 obj-$(CONFIG_SPI_GPIO)			+= spi-gpio.o
 obj-$(CONFIG_SPI_IMG_SPFI)		+= spi-img-spfi.o
 obj-$(CONFIG_SPI_IMX)			+= spi-imx.o
+obj-$(CONFIG_SPI_JCORE)			+= spi-jcore.o
 obj-$(CONFIG_SPI_LM70_LLP)		+= spi-lm70llp.o
 obj-$(CONFIG_SPI_LP8841_RTC)		+= spi-lp8841-rtc.o
 obj-$(CONFIG_SPI_MESON_SPIFC)		+= spi-meson-spifc.o
@@ -91,6 +93,8 @@ obj-$(CONFIG_SPI_TEGRA114)		+= spi-tegra114.o
 obj-$(CONFIG_SPI_TEGRA20_SFLASH)	+= spi-tegra20-sflash.o
 obj-$(CONFIG_SPI_TEGRA20_SLINK)		+= spi-tegra20-slink.o
 obj-$(CONFIG_SPI_TLE62X0)		+= spi-tle62x0.o
+spi-thunderx-objs			:= spi-cavium.o spi-cavium-thunderx.o
+obj-$(CONFIG_SPI_THUNDERX)		+= spi-thunderx.o
 obj-$(CONFIG_SPI_TOPCLIFF_PCH)		+= spi-topcliff-pch.o
 obj-$(CONFIG_SPI_TXX9)			+= spi-txx9.o
 obj-$(CONFIG_SPI_XCOMM)		+= spi-xcomm.o
diff --git a/drivers/spi/spi-bcm-qspi.c b/drivers/spi/spi-bcm-qspi.c
new file mode 100644
index 000000000000..14f9dea3173f
--- /dev/null
+++ b/drivers/spi/spi-bcm-qspi.c
@@ -0,0 +1,1397 @@
+/*
+ * Driver for Broadcom BRCMSTB, NSP,  NS2, Cygnus SPI Controllers
+ *
+ * Copyright 2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation (the "GPL").
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 (GPLv2) for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 (GPLv2) along with this source code.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mtd/spi-nor.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/spi/spi.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include "spi-bcm-qspi.h"
+
+#define DRIVER_NAME "bcm_qspi"
+
+
+/* BSPI register offsets */
+#define BSPI_REVISION_ID			0x000
+#define BSPI_SCRATCH				0x004
+#define BSPI_MAST_N_BOOT_CTRL			0x008
+#define BSPI_BUSY_STATUS			0x00c
+#define BSPI_INTR_STATUS			0x010
+#define BSPI_B0_STATUS				0x014
+#define BSPI_B0_CTRL				0x018
+#define BSPI_B1_STATUS				0x01c
+#define BSPI_B1_CTRL				0x020
+#define BSPI_STRAP_OVERRIDE_CTRL		0x024
+#define BSPI_FLEX_MODE_ENABLE			0x028
+#define BSPI_BITS_PER_CYCLE			0x02c
+#define BSPI_BITS_PER_PHASE			0x030
+#define BSPI_CMD_AND_MODE_BYTE			0x034
+#define BSPI_BSPI_FLASH_UPPER_ADDR_BYTE	0x038
+#define BSPI_BSPI_XOR_VALUE			0x03c
+#define BSPI_BSPI_XOR_ENABLE			0x040
+#define BSPI_BSPI_PIO_MODE_ENABLE		0x044
+#define BSPI_BSPI_PIO_IODIR			0x048
+#define BSPI_BSPI_PIO_DATA			0x04c
+
+/* RAF register offsets */
+#define BSPI_RAF_START_ADDR			0x100
+#define BSPI_RAF_NUM_WORDS			0x104
+#define BSPI_RAF_CTRL				0x108
+#define BSPI_RAF_FULLNESS			0x10c
+#define BSPI_RAF_WATERMARK			0x110
+#define BSPI_RAF_STATUS			0x114
+#define BSPI_RAF_READ_DATA			0x118
+#define BSPI_RAF_WORD_CNT			0x11c
+#define BSPI_RAF_CURR_ADDR			0x120
+
+/* Override mode masks */
+#define BSPI_STRAP_OVERRIDE_CTRL_OVERRIDE	BIT(0)
+#define BSPI_STRAP_OVERRIDE_CTRL_DATA_DUAL	BIT(1)
+#define BSPI_STRAP_OVERRIDE_CTRL_ADDR_4BYTE	BIT(2)
+#define BSPI_STRAP_OVERRIDE_CTRL_DATA_QUAD	BIT(3)
+#define BSPI_STRAP_OVERRIDE_CTRL_ENDAIN_MODE	BIT(4)
+
+#define BSPI_ADDRLEN_3BYTES			3
+#define BSPI_ADDRLEN_4BYTES			4
+
+#define BSPI_RAF_STATUS_FIFO_EMPTY_MASK	BIT(1)
+
+#define BSPI_RAF_CTRL_START_MASK		BIT(0)
+#define BSPI_RAF_CTRL_CLEAR_MASK		BIT(1)
+
+#define BSPI_BPP_MODE_SELECT_MASK		BIT(8)
+#define BSPI_BPP_ADDR_SELECT_MASK		BIT(16)
+
+#define BSPI_READ_LENGTH			256
+
+/* MSPI register offsets */
+#define MSPI_SPCR0_LSB				0x000
+#define MSPI_SPCR0_MSB				0x004
+#define MSPI_SPCR1_LSB				0x008
+#define MSPI_SPCR1_MSB				0x00c
+#define MSPI_NEWQP				0x010
+#define MSPI_ENDQP				0x014
+#define MSPI_SPCR2				0x018
+#define MSPI_MSPI_STATUS			0x020
+#define MSPI_CPTQP				0x024
+#define MSPI_SPCR3				0x028
+#define MSPI_TXRAM				0x040
+#define MSPI_RXRAM				0x0c0
+#define MSPI_CDRAM				0x140
+#define MSPI_WRITE_LOCK			0x180
+
+#define MSPI_MASTER_BIT			BIT(7)
+
+#define MSPI_NUM_CDRAM				16
+#define MSPI_CDRAM_CONT_BIT			BIT(7)
+#define MSPI_CDRAM_BITSE_BIT			BIT(6)
+#define MSPI_CDRAM_PCS				0xf
+
+#define MSPI_SPCR2_SPE				BIT(6)
+#define MSPI_SPCR2_CONT_AFTER_CMD		BIT(7)
+
+#define MSPI_MSPI_STATUS_SPIF			BIT(0)
+
+#define INTR_BASE_BIT_SHIFT			0x02
+#define INTR_COUNT				0x07
+
+#define NUM_CHIPSELECT				4
+#define QSPI_SPBR_MIN				8U
+#define QSPI_SPBR_MAX				255U
+
+#define OPCODE_DIOR				0xBB
+#define OPCODE_QIOR				0xEB
+#define OPCODE_DIOR_4B				0xBC
+#define OPCODE_QIOR_4B				0xEC
+
+#define MAX_CMD_SIZE				6
+
+#define ADDR_4MB_MASK				GENMASK(22, 0)
+
+/* stop at end of transfer, no other reason */
+#define TRANS_STATUS_BREAK_NONE		0
+/* stop at end of spi_message */
+#define TRANS_STATUS_BREAK_EOM			1
+/* stop at end of spi_transfer if delay */
+#define TRANS_STATUS_BREAK_DELAY		2
+/* stop at end of spi_transfer if cs_change */
+#define TRANS_STATUS_BREAK_CS_CHANGE		4
+/* stop if we run out of bytes */
+#define TRANS_STATUS_BREAK_NO_BYTES		8
+
+/* events that make us stop filling TX slots */
+#define TRANS_STATUS_BREAK_TX (TRANS_STATUS_BREAK_EOM |		\
+			       TRANS_STATUS_BREAK_DELAY |		\
+			       TRANS_STATUS_BREAK_CS_CHANGE)
+
+/* events that make us deassert CS */
+#define TRANS_STATUS_BREAK_DESELECT (TRANS_STATUS_BREAK_EOM |		\
+				     TRANS_STATUS_BREAK_CS_CHANGE)
+
+struct bcm_qspi_parms {
+	u32 speed_hz;
+	u8 mode;
+	u8 bits_per_word;
+};
+
+struct bcm_xfer_mode {
+	bool flex_mode;
+	unsigned int width;
+	unsigned int addrlen;
+	unsigned int hp;
+};
+
+enum base_type {
+	MSPI,
+	BSPI,
+	CHIP_SELECT,
+	BASEMAX,
+};
+
+enum irq_source {
+	SINGLE_L2,
+	MUXED_L1,
+};
+
+struct bcm_qspi_irq {
+	const char *irq_name;
+	const irq_handler_t irq_handler;
+	int irq_source;
+	u32 mask;
+};
+
+struct bcm_qspi_dev_id {
+	const struct bcm_qspi_irq *irqp;
+	void *dev;
+};
+
+struct qspi_trans {
+	struct spi_transfer *trans;
+	int byte;
+};
+
+struct bcm_qspi {
+	struct platform_device *pdev;
+	struct spi_master *master;
+	struct clk *clk;
+	u32 base_clk;
+	u32 max_speed_hz;
+	void __iomem *base[BASEMAX];
+
+	/* Some SoCs provide custom interrupt status register(s) */
+	struct bcm_qspi_soc_intc	*soc_intc;
+
+	struct bcm_qspi_parms last_parms;
+	struct qspi_trans  trans_pos;
+	int curr_cs;
+	int bspi_maj_rev;
+	int bspi_min_rev;
+	int bspi_enabled;
+	struct spi_flash_read_message *bspi_rf_msg;
+	u32 bspi_rf_msg_idx;
+	u32 bspi_rf_msg_len;
+	u32 bspi_rf_msg_status;
+	struct bcm_xfer_mode xfer_mode;
+	u32 s3_strap_override_ctrl;
+	bool bspi_mode;
+	bool big_endian;
+	int num_irqs;
+	struct bcm_qspi_dev_id *dev_ids;
+	struct completion mspi_done;
+	struct completion bspi_done;
+};
+
+static inline bool has_bspi(struct bcm_qspi *qspi)
+{
+	return qspi->bspi_mode;
+}
+
+/* Read qspi controller register*/
+static inline u32 bcm_qspi_read(struct bcm_qspi *qspi, enum base_type type,
+				unsigned int offset)
+{
+	return bcm_qspi_readl(qspi->big_endian, qspi->base[type] + offset);
+}
+
+/* Write qspi controller register*/
+static inline void bcm_qspi_write(struct bcm_qspi *qspi, enum base_type type,
+				  unsigned int offset, unsigned int data)
+{
+	bcm_qspi_writel(qspi->big_endian, data, qspi->base[type] + offset);
+}
+
+/* BSPI helpers */
+static int bcm_qspi_bspi_busy_poll(struct bcm_qspi *qspi)
+{
+	int i;
+
+	/* this should normally finish within 10us */
+	for (i = 0; i < 1000; i++) {
+		if (!(bcm_qspi_read(qspi, BSPI, BSPI_BUSY_STATUS) & 1))
+			return 0;
+		udelay(1);
+	}
+	dev_warn(&qspi->pdev->dev, "timeout waiting for !busy_status\n");
+	return -EIO;
+}
+
+static inline bool bcm_qspi_bspi_ver_three(struct bcm_qspi *qspi)
+{
+	if (qspi->bspi_maj_rev < 4)
+		return true;
+	return false;
+}
+
+static void bcm_qspi_bspi_flush_prefetch_buffers(struct bcm_qspi *qspi)
+{
+	bcm_qspi_bspi_busy_poll(qspi);
+	/* Force rising edge for the b0/b1 'flush' field */
+	bcm_qspi_write(qspi, BSPI, BSPI_B0_CTRL, 1);
+	bcm_qspi_write(qspi, BSPI, BSPI_B1_CTRL, 1);
+	bcm_qspi_write(qspi, BSPI, BSPI_B0_CTRL, 0);
+	bcm_qspi_write(qspi, BSPI, BSPI_B1_CTRL, 0);
+}
+
+static int bcm_qspi_bspi_lr_is_fifo_empty(struct bcm_qspi *qspi)
+{
+	return (bcm_qspi_read(qspi, BSPI, BSPI_RAF_STATUS) &
+				BSPI_RAF_STATUS_FIFO_EMPTY_MASK);
+}
+
+static inline u32 bcm_qspi_bspi_lr_read_fifo(struct bcm_qspi *qspi)
+{
+	u32 data = bcm_qspi_read(qspi, BSPI, BSPI_RAF_READ_DATA);
+
+	/* BSPI v3 LR is LE only, convert data to host endianness */
+	if (bcm_qspi_bspi_ver_three(qspi))
+		data = le32_to_cpu(data);
+
+	return data;
+}
+
+static inline void bcm_qspi_bspi_lr_start(struct bcm_qspi *qspi)
+{
+	bcm_qspi_bspi_busy_poll(qspi);
+	bcm_qspi_write(qspi, BSPI, BSPI_RAF_CTRL,
+		       BSPI_RAF_CTRL_START_MASK);
+}
+
+static inline void bcm_qspi_bspi_lr_clear(struct bcm_qspi *qspi)
+{
+	bcm_qspi_write(qspi, BSPI, BSPI_RAF_CTRL,
+		       BSPI_RAF_CTRL_CLEAR_MASK);
+	bcm_qspi_bspi_flush_prefetch_buffers(qspi);
+}
+
+static void bcm_qspi_bspi_lr_data_read(struct bcm_qspi *qspi)
+{
+	u32 *buf = (u32 *)qspi->bspi_rf_msg->buf;
+	u32 data = 0;
+
+	dev_dbg(&qspi->pdev->dev, "xfer %p rx %p rxlen %d\n", qspi->bspi_rf_msg,
+		qspi->bspi_rf_msg->buf, qspi->bspi_rf_msg_len);
+	while (!bcm_qspi_bspi_lr_is_fifo_empty(qspi)) {
+		data = bcm_qspi_bspi_lr_read_fifo(qspi);
+		if (likely(qspi->bspi_rf_msg_len >= 4) &&
+		    IS_ALIGNED((uintptr_t)buf, 4)) {
+			buf[qspi->bspi_rf_msg_idx++] = data;
+			qspi->bspi_rf_msg_len -= 4;
+		} else {
+			/* Read out remaining bytes, make sure*/
+			u8 *cbuf = (u8 *)&buf[qspi->bspi_rf_msg_idx];
+
+			data = cpu_to_le32(data);
+			while (qspi->bspi_rf_msg_len) {
+				*cbuf++ = (u8)data;
+				data >>= 8;
+				qspi->bspi_rf_msg_len--;
+			}
+		}
+	}
+}
+
+static void bcm_qspi_bspi_set_xfer_params(struct bcm_qspi *qspi, u8 cmd_byte,
+					  int bpp, int bpc, int flex_mode)
+{
+	bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE, 0);
+	bcm_qspi_write(qspi, BSPI, BSPI_BITS_PER_CYCLE, bpc);
+	bcm_qspi_write(qspi, BSPI, BSPI_BITS_PER_PHASE, bpp);
+	bcm_qspi_write(qspi, BSPI, BSPI_CMD_AND_MODE_BYTE, cmd_byte);
+	bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE, flex_mode);
+}
+
+static int bcm_qspi_bspi_set_flex_mode(struct bcm_qspi *qspi, int width,
+				       int addrlen, int hp)
+{
+	int bpc = 0, bpp = 0;
+	u8 command = SPINOR_OP_READ_FAST;
+	int flex_mode = 1, rv = 0;
+	bool spans_4byte = false;
+
+	dev_dbg(&qspi->pdev->dev, "set flex mode w %x addrlen %x hp %d\n",
+		width, addrlen, hp);
+
+	if (addrlen == BSPI_ADDRLEN_4BYTES) {
+		bpp = BSPI_BPP_ADDR_SELECT_MASK;
+		spans_4byte = true;
+	}
+
+	bpp |= 8;
+
+	switch (width) {
+	case SPI_NBITS_SINGLE:
+		if (addrlen == BSPI_ADDRLEN_3BYTES)
+			/* default mode, does not need flex_cmd */
+			flex_mode = 0;
+		else
+			command = SPINOR_OP_READ4_FAST;
+		break;
+	case SPI_NBITS_DUAL:
+		bpc = 0x00000001;
+		if (hp) {
+			bpc |= 0x00010100; /* address and mode are 2-bit */
+			bpp = BSPI_BPP_MODE_SELECT_MASK;
+			command = OPCODE_DIOR;
+			if (spans_4byte)
+				command = OPCODE_DIOR_4B;
+		} else {
+			command = SPINOR_OP_READ_1_1_2;
+			if (spans_4byte)
+				command = SPINOR_OP_READ4_1_1_2;
+		}
+		break;
+	case SPI_NBITS_QUAD:
+		bpc = 0x00000002;
+		if (hp) {
+			bpc |= 0x00020200; /* address and mode are 4-bit */
+			bpp = 4; /* dummy cycles */
+			bpp |= BSPI_BPP_ADDR_SELECT_MASK;
+			command = OPCODE_QIOR;
+			if (spans_4byte)
+				command = OPCODE_QIOR_4B;
+		} else {
+			command = SPINOR_OP_READ_1_1_4;
+			if (spans_4byte)
+				command = SPINOR_OP_READ4_1_1_4;
+		}
+		break;
+	default:
+		rv = -EINVAL;
+		break;
+	}
+
+	if (rv == 0)
+		bcm_qspi_bspi_set_xfer_params(qspi, command, bpp, bpc,
+					      flex_mode);
+
+	return rv;
+}
+
+static int bcm_qspi_bspi_set_override(struct bcm_qspi *qspi, int width,
+				      int addrlen, int hp)
+{
+	u32 data = bcm_qspi_read(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL);
+
+	dev_dbg(&qspi->pdev->dev, "set override mode w %x addrlen %x hp %d\n",
+		width, addrlen, hp);
+
+	switch (width) {
+	case SPI_NBITS_SINGLE:
+		/* clear quad/dual mode */
+		data &= ~(BSPI_STRAP_OVERRIDE_CTRL_DATA_QUAD |
+			  BSPI_STRAP_OVERRIDE_CTRL_DATA_DUAL);
+		break;
+
+	case SPI_NBITS_QUAD:
+		/* clear dual mode and set quad mode */
+		data &= ~BSPI_STRAP_OVERRIDE_CTRL_DATA_DUAL;
+		data |= BSPI_STRAP_OVERRIDE_CTRL_DATA_QUAD;
+		break;
+	case SPI_NBITS_DUAL:
+		/* clear quad mode set dual mode */
+		data &= ~BSPI_STRAP_OVERRIDE_CTRL_DATA_QUAD;
+		data |= BSPI_STRAP_OVERRIDE_CTRL_DATA_DUAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (addrlen == BSPI_ADDRLEN_4BYTES)
+		/* set 4byte mode*/
+		data |= BSPI_STRAP_OVERRIDE_CTRL_ADDR_4BYTE;
+	else
+		/* clear 4 byte mode */
+		data &= ~BSPI_STRAP_OVERRIDE_CTRL_ADDR_4BYTE;
+
+	/* set the override mode */
+	data |=	BSPI_STRAP_OVERRIDE_CTRL_OVERRIDE;
+	bcm_qspi_write(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL, data);
+	bcm_qspi_bspi_set_xfer_params(qspi, SPINOR_OP_READ_FAST, 0, 0, 0);
+
+	return 0;
+}
+
+static int bcm_qspi_bspi_set_mode(struct bcm_qspi *qspi,
+				  int width, int addrlen, int hp)
+{
+	int error = 0;
+
+	/* default mode */
+	qspi->xfer_mode.flex_mode = true;
+
+	if (!bcm_qspi_bspi_ver_three(qspi)) {
+		u32 val, mask;
+
+		val = bcm_qspi_read(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL);
+		mask = BSPI_STRAP_OVERRIDE_CTRL_OVERRIDE;
+		if (val & mask || qspi->s3_strap_override_ctrl & mask) {
+			qspi->xfer_mode.flex_mode = false;
+			bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE,
+				       0);
+
+			if ((val | qspi->s3_strap_override_ctrl) &
+			    BSPI_STRAP_OVERRIDE_CTRL_DATA_DUAL)
+				width = SPI_NBITS_DUAL;
+			else if ((val |  qspi->s3_strap_override_ctrl) &
+				 BSPI_STRAP_OVERRIDE_CTRL_DATA_QUAD)
+				width = SPI_NBITS_QUAD;
+
+			error = bcm_qspi_bspi_set_override(qspi, width, addrlen,
+							   hp);
+		}
+	}
+
+	if (qspi->xfer_mode.flex_mode)
+		error = bcm_qspi_bspi_set_flex_mode(qspi, width, addrlen, hp);
+
+	if (error) {
+		dev_warn(&qspi->pdev->dev,
+			 "INVALID COMBINATION: width=%d addrlen=%d hp=%d\n",
+			 width, addrlen, hp);
+	} else if (qspi->xfer_mode.width != width ||
+		   qspi->xfer_mode.addrlen != addrlen ||
+		   qspi->xfer_mode.hp != hp) {
+		qspi->xfer_mode.width = width;
+		qspi->xfer_mode.addrlen = addrlen;
+		qspi->xfer_mode.hp = hp;
+		dev_dbg(&qspi->pdev->dev,
+			"cs:%d %d-lane output, %d-byte address%s\n",
+			qspi->curr_cs,
+			qspi->xfer_mode.width,
+			qspi->xfer_mode.addrlen,
+			qspi->xfer_mode.hp != -1 ? ", hp mode" : "");
+	}
+
+	return error;
+}
+
+static void bcm_qspi_enable_bspi(struct bcm_qspi *qspi)
+{
+	if (!has_bspi(qspi) || (qspi->bspi_enabled))
+		return;
+
+	qspi->bspi_enabled = 1;
+	if ((bcm_qspi_read(qspi, BSPI, BSPI_MAST_N_BOOT_CTRL) & 1) == 0)
+		return;
+
+	bcm_qspi_bspi_flush_prefetch_buffers(qspi);
+	udelay(1);
+	bcm_qspi_write(qspi, BSPI, BSPI_MAST_N_BOOT_CTRL, 0);
+	udelay(1);
+}
+
+static void bcm_qspi_disable_bspi(struct bcm_qspi *qspi)
+{
+	if (!has_bspi(qspi) || (!qspi->bspi_enabled))
+		return;
+
+	qspi->bspi_enabled = 0;
+	if ((bcm_qspi_read(qspi, BSPI, BSPI_MAST_N_BOOT_CTRL) & 1))
+		return;
+
+	bcm_qspi_bspi_busy_poll(qspi);
+	bcm_qspi_write(qspi, BSPI, BSPI_MAST_N_BOOT_CTRL, 1);
+	udelay(1);
+}
+
+static void bcm_qspi_chip_select(struct bcm_qspi *qspi, int cs)
+{
+	u32 data = 0;
+
+	if (qspi->curr_cs == cs)
+		return;
+	if (qspi->base[CHIP_SELECT]) {
+		data = bcm_qspi_read(qspi, CHIP_SELECT, 0);
+		data = (data & ~0xff) | (1 << cs);
+		bcm_qspi_write(qspi, CHIP_SELECT, 0, data);
+		usleep_range(10, 20);
+	}
+	qspi->curr_cs = cs;
+}
+
+/* MSPI helpers */
+static void bcm_qspi_hw_set_parms(struct bcm_qspi *qspi,
+				  const struct bcm_qspi_parms *xp)
+{
+	u32 spcr, spbr = 0;
+
+	if (xp->speed_hz)
+		spbr = qspi->base_clk / (2 * xp->speed_hz);
+
+	spcr = clamp_val(spbr, QSPI_SPBR_MIN, QSPI_SPBR_MAX);
+	bcm_qspi_write(qspi, MSPI, MSPI_SPCR0_LSB, spcr);
+
+	spcr = MSPI_MASTER_BIT;
+	/* for 16 bit the data should be zero */
+	if (xp->bits_per_word != 16)
+		spcr |= xp->bits_per_word << 2;
+	spcr |= xp->mode & 3;
+	bcm_qspi_write(qspi, MSPI, MSPI_SPCR0_MSB, spcr);
+
+	qspi->last_parms = *xp;
+}
+
+static void bcm_qspi_update_parms(struct bcm_qspi *qspi,
+				  struct spi_device *spi,
+				  struct spi_transfer *trans)
+{
+	struct bcm_qspi_parms xp;
+
+	xp.speed_hz = trans->speed_hz;
+	xp.bits_per_word = trans->bits_per_word;
+	xp.mode = spi->mode;
+
+	bcm_qspi_hw_set_parms(qspi, &xp);
+}
+
+static int bcm_qspi_setup(struct spi_device *spi)
+{
+	struct bcm_qspi_parms *xp;
+
+	if (spi->bits_per_word > 16)
+		return -EINVAL;
+
+	xp = spi_get_ctldata(spi);
+	if (!xp) {
+		xp = kzalloc(sizeof(*xp), GFP_KERNEL);
+		if (!xp)
+			return -ENOMEM;
+		spi_set_ctldata(spi, xp);
+	}
+	xp->speed_hz = spi->max_speed_hz;
+	xp->mode = spi->mode;
+
+	if (spi->bits_per_word)
+		xp->bits_per_word = spi->bits_per_word;
+	else
+		xp->bits_per_word = 8;
+
+	return 0;
+}
+
+static int update_qspi_trans_byte_count(struct bcm_qspi *qspi,
+					struct qspi_trans *qt, int flags)
+{
+	int ret = TRANS_STATUS_BREAK_NONE;
+
+	/* count the last transferred bytes */
+	if (qt->trans->bits_per_word <= 8)
+		qt->byte++;
+	else
+		qt->byte += 2;
+
+	if (qt->byte >= qt->trans->len) {
+		/* we're at the end of the spi_transfer */
+
+		/* in TX mode, need to pause for a delay or CS change */
+		if (qt->trans->delay_usecs &&
+		    (flags & TRANS_STATUS_BREAK_DELAY))
+			ret |= TRANS_STATUS_BREAK_DELAY;
+		if (qt->trans->cs_change &&
+		    (flags & TRANS_STATUS_BREAK_CS_CHANGE))
+			ret |= TRANS_STATUS_BREAK_CS_CHANGE;
+		if (ret)
+			goto done;
+
+		dev_dbg(&qspi->pdev->dev, "advance msg exit\n");
+		if (spi_transfer_is_last(qspi->master, qt->trans))
+			ret = TRANS_STATUS_BREAK_EOM;
+		else
+			ret = TRANS_STATUS_BREAK_NO_BYTES;
+
+		qt->trans = NULL;
+	}
+
+done:
+	dev_dbg(&qspi->pdev->dev, "trans %p len %d byte %d ret %x\n",
+		qt->trans, qt->trans ? qt->trans->len : 0, qt->byte, ret);
+	return ret;
+}
+
+static inline u8 read_rxram_slot_u8(struct bcm_qspi *qspi, int slot)
+{
+	u32 slot_offset = MSPI_RXRAM + (slot << 3) + 0x4;
+
+	/* mask out reserved bits */
+	return bcm_qspi_read(qspi, MSPI, slot_offset) & 0xff;
+}
+
+static inline u16 read_rxram_slot_u16(struct bcm_qspi *qspi, int slot)
+{
+	u32 reg_offset = MSPI_RXRAM;
+	u32 lsb_offset = reg_offset + (slot << 3) + 0x4;
+	u32 msb_offset = reg_offset + (slot << 3);
+
+	return (bcm_qspi_read(qspi, MSPI, lsb_offset) & 0xff) |
+		((bcm_qspi_read(qspi, MSPI, msb_offset) & 0xff) << 8);
+}
+
+static void read_from_hw(struct bcm_qspi *qspi, int slots)
+{
+	struct qspi_trans tp;
+	int slot;
+
+	bcm_qspi_disable_bspi(qspi);
+
+	if (slots > MSPI_NUM_CDRAM) {
+		/* should never happen */
+		dev_err(&qspi->pdev->dev, "%s: too many slots!\n", __func__);
+		return;
+	}
+
+	tp = qspi->trans_pos;
+
+	for (slot = 0; slot < slots; slot++) {
+		if (tp.trans->bits_per_word <= 8) {
+			u8 *buf = tp.trans->rx_buf;
+
+			if (buf)
+				buf[tp.byte] = read_rxram_slot_u8(qspi, slot);
+			dev_dbg(&qspi->pdev->dev, "RD %02x\n",
+				buf ? buf[tp.byte] : 0xff);
+		} else {
+			u16 *buf = tp.trans->rx_buf;
+
+			if (buf)
+				buf[tp.byte / 2] = read_rxram_slot_u16(qspi,
+								      slot);
+			dev_dbg(&qspi->pdev->dev, "RD %04x\n",
+				buf ? buf[tp.byte] : 0xffff);
+		}
+
+		update_qspi_trans_byte_count(qspi, &tp,
+					     TRANS_STATUS_BREAK_NONE);
+	}
+
+	qspi->trans_pos = tp;
+}
+
+static inline void write_txram_slot_u8(struct bcm_qspi *qspi, int slot,
+				       u8 val)
+{
+	u32 reg_offset = MSPI_TXRAM + (slot << 3);
+
+	/* mask out reserved bits */
+	bcm_qspi_write(qspi, MSPI, reg_offset, val);
+}
+
+static inline void write_txram_slot_u16(struct bcm_qspi *qspi, int slot,
+					u16 val)
+{
+	u32 reg_offset = MSPI_TXRAM;
+	u32 msb_offset = reg_offset + (slot << 3);
+	u32 lsb_offset = reg_offset + (slot << 3) + 0x4;
+
+	bcm_qspi_write(qspi, MSPI, msb_offset, (val >> 8));
+	bcm_qspi_write(qspi, MSPI, lsb_offset, (val & 0xff));
+}
+
+static inline u32 read_cdram_slot(struct bcm_qspi *qspi, int slot)
+{
+	return bcm_qspi_read(qspi, MSPI, MSPI_CDRAM + (slot << 2));
+}
+
+static inline void write_cdram_slot(struct bcm_qspi *qspi, int slot, u32 val)
+{
+	bcm_qspi_write(qspi, MSPI, (MSPI_CDRAM + (slot << 2)), val);
+}
+
+/* Return number of slots written */
+static int write_to_hw(struct bcm_qspi *qspi, struct spi_device *spi)
+{
+	struct qspi_trans tp;
+	int slot = 0, tstatus = 0;
+	u32 mspi_cdram = 0;
+
+	bcm_qspi_disable_bspi(qspi);
+	tp = qspi->trans_pos;
+	bcm_qspi_update_parms(qspi, spi, tp.trans);
+
+	/* Run until end of transfer or reached the max data */
+	while (!tstatus && slot < MSPI_NUM_CDRAM) {
+		if (tp.trans->bits_per_word <= 8) {
+			const u8 *buf = tp.trans->tx_buf;
+			u8 val = buf ? buf[tp.byte] : 0xff;
+
+			write_txram_slot_u8(qspi, slot, val);
+			dev_dbg(&qspi->pdev->dev, "WR %02x\n", val);
+		} else {
+			const u16 *buf = tp.trans->tx_buf;
+			u16 val = buf ? buf[tp.byte / 2] : 0xffff;
+
+			write_txram_slot_u16(qspi, slot, val);
+			dev_dbg(&qspi->pdev->dev, "WR %04x\n", val);
+		}
+		mspi_cdram = MSPI_CDRAM_CONT_BIT;
+		mspi_cdram |= (~(1 << spi->chip_select) &
+			       MSPI_CDRAM_PCS);
+		mspi_cdram |= ((tp.trans->bits_per_word <= 8) ? 0 :
+				MSPI_CDRAM_BITSE_BIT);
+
+		write_cdram_slot(qspi, slot, mspi_cdram);
+
+		tstatus = update_qspi_trans_byte_count(qspi, &tp,
+						       TRANS_STATUS_BREAK_TX);
+		slot++;
+	}
+
+	if (!slot) {
+		dev_err(&qspi->pdev->dev, "%s: no data to send?", __func__);
+		goto done;
+	}
+
+	dev_dbg(&qspi->pdev->dev, "submitting %d slots\n", slot);
+	bcm_qspi_write(qspi, MSPI, MSPI_NEWQP, 0);
+	bcm_qspi_write(qspi, MSPI, MSPI_ENDQP, slot - 1);
+
+	if (tstatus & TRANS_STATUS_BREAK_DESELECT) {
+		mspi_cdram = read_cdram_slot(qspi, slot - 1) &
+			~MSPI_CDRAM_CONT_BIT;
+		write_cdram_slot(qspi, slot - 1, mspi_cdram);
+	}
+
+	if (has_bspi(qspi))
+		bcm_qspi_write(qspi, MSPI, MSPI_WRITE_LOCK, 1);
+
+	/* Must flush previous writes before starting MSPI operation */
+	mb();
+	/* Set cont | spe | spifie */
+	bcm_qspi_write(qspi, MSPI, MSPI_SPCR2, 0xe0);
+
+done:
+	return slot;
+}
+
+static int bcm_qspi_bspi_flash_read(struct spi_device *spi,
+				    struct spi_flash_read_message *msg)
+{
+	struct bcm_qspi *qspi = spi_master_get_devdata(spi->master);
+	u32 addr = 0, len, len_words;
+	int ret = 0;
+	unsigned long timeo = msecs_to_jiffies(100);
+	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;
+
+	if (bcm_qspi_bspi_ver_three(qspi))
+		if (msg->addr_width == BSPI_ADDRLEN_4BYTES)
+			return -EIO;
+
+	bcm_qspi_chip_select(qspi, spi->chip_select);
+	bcm_qspi_write(qspi, MSPI, MSPI_WRITE_LOCK, 0);
+
+	/*
+	 * when using flex mode mode we need to send
+	 * the upper address byte to bspi
+	 */
+	if (bcm_qspi_bspi_ver_three(qspi) == false) {
+		addr = msg->from & 0xff000000;
+		bcm_qspi_write(qspi, BSPI,
+			       BSPI_BSPI_FLASH_UPPER_ADDR_BYTE, addr);
+	}
+
+	if (!qspi->xfer_mode.flex_mode)
+		addr = msg->from;
+	else
+		addr = msg->from & 0x00ffffff;
+
+	/* set BSPI RAF buffer max read length */
+	len = msg->len;
+	if (len > BSPI_READ_LENGTH)
+		len = BSPI_READ_LENGTH;
+
+	if (bcm_qspi_bspi_ver_three(qspi) == true)
+		addr = (addr + 0xc00000) & 0xffffff;
+
+	reinit_completion(&qspi->bspi_done);
+	bcm_qspi_enable_bspi(qspi);
+	len_words = (len + 3) >> 2;
+	qspi->bspi_rf_msg = msg;
+	qspi->bspi_rf_msg_status = 0;
+	qspi->bspi_rf_msg_idx = 0;
+	qspi->bspi_rf_msg_len = len;
+	dev_dbg(&qspi->pdev->dev, "bspi xfr addr 0x%x len 0x%x", addr, len);
+
+	bcm_qspi_write(qspi, BSPI, BSPI_RAF_START_ADDR, addr);
+	bcm_qspi_write(qspi, BSPI, BSPI_RAF_NUM_WORDS, len_words);
+	bcm_qspi_write(qspi, BSPI, BSPI_RAF_WATERMARK, 0);
+
+	if (qspi->soc_intc) {
+		/*
+		 * clear soc MSPI and BSPI interrupts and enable
+		 * BSPI interrupts.
+		 */
+		soc_intc->bcm_qspi_int_ack(soc_intc, MSPI_BSPI_DONE);
+		soc_intc->bcm_qspi_int_set(soc_intc, BSPI_DONE, true);
+	}
+
+	/* Must flush previous writes before starting BSPI operation */
+	mb();
+
+	bcm_qspi_bspi_lr_start(qspi);
+	if (!wait_for_completion_timeout(&qspi->bspi_done, timeo)) {
+		dev_err(&qspi->pdev->dev, "timeout waiting for BSPI\n");
+		ret = -ETIMEDOUT;
+	} else {
+		/* set the return length for the caller */
+		msg->retlen = len;
+	}
+
+	return ret;
+}
+
+static int bcm_qspi_flash_read(struct spi_device *spi,
+			       struct spi_flash_read_message *msg)
+{
+	struct bcm_qspi *qspi = spi_master_get_devdata(spi->master);
+	int ret = 0;
+	bool mspi_read = false;
+	u32 io_width, addrlen, addr, len;
+	u_char *buf;
+
+	buf = msg->buf;
+	addr = msg->from;
+	len = msg->len;
+
+	if (bcm_qspi_bspi_ver_three(qspi) == true) {
+		/*
+		 * The address coming into this function is a raw flash offset.
+		 * But for BSPI <= V3, we need to convert it to a remapped BSPI
+		 * address. If it crosses a 4MB boundary, just revert back to
+		 * using MSPI.
+		 */
+		addr = (addr + 0xc00000) & 0xffffff;
+
+		if ((~ADDR_4MB_MASK & addr) ^
+		    (~ADDR_4MB_MASK & (addr + len - 1)))
+			mspi_read = true;
+	}
+
+	/* non-aligned and very short transfers are handled by MSPI */
+	if (!IS_ALIGNED((uintptr_t)addr, 4) || !IS_ALIGNED((uintptr_t)buf, 4) ||
+	    len < 4)
+		mspi_read = true;
+
+	if (mspi_read)
+		/* this will make the m25p80 read to fallback to mspi read */
+		return -EAGAIN;
+
+	io_width = msg->data_nbits ? msg->data_nbits : SPI_NBITS_SINGLE;
+	addrlen = msg->addr_width;
+	ret = bcm_qspi_bspi_set_mode(qspi, io_width, addrlen, -1);
+
+	if (!ret)
+		ret = bcm_qspi_bspi_flash_read(spi, msg);
+
+	return ret;
+}
+
+static int bcm_qspi_transfer_one(struct spi_master *master,
+				 struct spi_device *spi,
+				 struct spi_transfer *trans)
+{
+	struct bcm_qspi *qspi = spi_master_get_devdata(master);
+	int slots;
+	unsigned long timeo = msecs_to_jiffies(100);
+
+	bcm_qspi_chip_select(qspi, spi->chip_select);
+	qspi->trans_pos.trans = trans;
+	qspi->trans_pos.byte = 0;
+
+	while (qspi->trans_pos.byte < trans->len) {
+		reinit_completion(&qspi->mspi_done);
+
+		slots = write_to_hw(qspi, spi);
+		if (!wait_for_completion_timeout(&qspi->mspi_done, timeo)) {
+			dev_err(&qspi->pdev->dev, "timeout waiting for MSPI\n");
+			return -ETIMEDOUT;
+		}
+
+		read_from_hw(qspi, slots);
+	}
+
+	return 0;
+}
+
+static void bcm_qspi_cleanup(struct spi_device *spi)
+{
+	struct bcm_qspi_parms *xp = spi_get_ctldata(spi);
+
+	kfree(xp);
+}
+
+static irqreturn_t bcm_qspi_mspi_l2_isr(int irq, void *dev_id)
+{
+	struct bcm_qspi_dev_id *qspi_dev_id = dev_id;
+	struct bcm_qspi *qspi = qspi_dev_id->dev;
+	u32 status = bcm_qspi_read(qspi, MSPI, MSPI_MSPI_STATUS);
+
+	if (status & MSPI_MSPI_STATUS_SPIF) {
+		struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;
+		/* clear interrupt */
+		status &= ~MSPI_MSPI_STATUS_SPIF;
+		bcm_qspi_write(qspi, MSPI, MSPI_MSPI_STATUS, status);
+		if (qspi->soc_intc)
+			soc_intc->bcm_qspi_int_ack(soc_intc, MSPI_DONE);
+		complete(&qspi->mspi_done);
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+static irqreturn_t bcm_qspi_bspi_lr_l2_isr(int irq, void *dev_id)
+{
+	struct bcm_qspi_dev_id *qspi_dev_id = dev_id;
+	struct bcm_qspi *qspi = qspi_dev_id->dev;
+	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;
+	u32 status = qspi_dev_id->irqp->mask;
+
+	if (qspi->bspi_enabled && qspi->bspi_rf_msg) {
+		bcm_qspi_bspi_lr_data_read(qspi);
+		if (qspi->bspi_rf_msg_len == 0) {
+			qspi->bspi_rf_msg = NULL;
+			if (qspi->soc_intc) {
+				/* disable soc BSPI interrupt */
+				soc_intc->bcm_qspi_int_set(soc_intc, BSPI_DONE,
+							   false);
+				/* indicate done */
+				status = INTR_BSPI_LR_SESSION_DONE_MASK;
+			}
+
+			if (qspi->bspi_rf_msg_status)
+				bcm_qspi_bspi_lr_clear(qspi);
+			else
+				bcm_qspi_bspi_flush_prefetch_buffers(qspi);
+		}
+
+		if (qspi->soc_intc)
+			/* clear soc BSPI interrupt */
+			soc_intc->bcm_qspi_int_ack(soc_intc, BSPI_DONE);
+	}
+
+	status &= INTR_BSPI_LR_SESSION_DONE_MASK;
+	if (qspi->bspi_enabled && status && qspi->bspi_rf_msg_len == 0)
+		complete(&qspi->bspi_done);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t bcm_qspi_bspi_lr_err_l2_isr(int irq, void *dev_id)
+{
+	struct bcm_qspi_dev_id *qspi_dev_id = dev_id;
+	struct bcm_qspi *qspi = qspi_dev_id->dev;
+	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;
+
+	dev_err(&qspi->pdev->dev, "BSPI INT error\n");
+	qspi->bspi_rf_msg_status = -EIO;
+	if (qspi->soc_intc)
+		/* clear soc interrupt */
+		soc_intc->bcm_qspi_int_ack(soc_intc, BSPI_ERR);
+
+	complete(&qspi->bspi_done);
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t bcm_qspi_l1_isr(int irq, void *dev_id)
+{
+	struct bcm_qspi_dev_id *qspi_dev_id = dev_id;
+	struct bcm_qspi *qspi = qspi_dev_id->dev;
+	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;
+	irqreturn_t ret = IRQ_NONE;
+
+	if (soc_intc) {
+		u32 status = soc_intc->bcm_qspi_get_int_status(soc_intc);
+
+		if (status & MSPI_DONE)
+			ret = bcm_qspi_mspi_l2_isr(irq, dev_id);
+		else if (status & BSPI_DONE)
+			ret = bcm_qspi_bspi_lr_l2_isr(irq, dev_id);
+		else if (status & BSPI_ERR)
+			ret = bcm_qspi_bspi_lr_err_l2_isr(irq, dev_id);
+	}
+
+	return ret;
+}
+
+static const struct bcm_qspi_irq qspi_irq_tab[] = {
+	{
+		.irq_name = "spi_lr_fullness_reached",
+		.irq_handler = bcm_qspi_bspi_lr_l2_isr,
+		.mask = INTR_BSPI_LR_FULLNESS_REACHED_MASK,
+	},
+	{
+		.irq_name = "spi_lr_session_aborted",
+		.irq_handler = bcm_qspi_bspi_lr_err_l2_isr,
+		.mask = INTR_BSPI_LR_SESSION_ABORTED_MASK,
+	},
+	{
+		.irq_name = "spi_lr_impatient",
+		.irq_handler = bcm_qspi_bspi_lr_err_l2_isr,
+		.mask = INTR_BSPI_LR_IMPATIENT_MASK,
+	},
+	{
+		.irq_name = "spi_lr_session_done",
+		.irq_handler = bcm_qspi_bspi_lr_l2_isr,
+		.mask = INTR_BSPI_LR_SESSION_DONE_MASK,
+	},
+#ifdef QSPI_INT_DEBUG
+	/* this interrupt is for debug purposes only, dont request irq */
+	{
+		.irq_name = "spi_lr_overread",
+		.irq_handler = bcm_qspi_bspi_lr_err_l2_isr,
+		.mask = INTR_BSPI_LR_OVERREAD_MASK,
+	},
+#endif
+	{
+		.irq_name = "mspi_done",
+		.irq_handler = bcm_qspi_mspi_l2_isr,
+		.mask = INTR_MSPI_DONE_MASK,
+	},
+	{
+		.irq_name = "mspi_halted",
+		.irq_handler = bcm_qspi_mspi_l2_isr,
+		.mask = INTR_MSPI_HALTED_MASK,
+	},
+	{
+		/* single muxed L1 interrupt source */
+		.irq_name = "spi_l1_intr",
+		.irq_handler = bcm_qspi_l1_isr,
+		.irq_source = MUXED_L1,
+		.mask = QSPI_INTERRUPTS_ALL,
+	},
+};
+
+static void bcm_qspi_bspi_init(struct bcm_qspi *qspi)
+{
+	u32 val = 0;
+
+	val = bcm_qspi_read(qspi, BSPI, BSPI_REVISION_ID);
+	qspi->bspi_maj_rev = (val >> 8) & 0xff;
+	qspi->bspi_min_rev = val & 0xff;
+	if (!(bcm_qspi_bspi_ver_three(qspi))) {
+		/* Force mapping of BSPI address -> flash offset */
+		bcm_qspi_write(qspi, BSPI, BSPI_BSPI_XOR_VALUE, 0);
+		bcm_qspi_write(qspi, BSPI, BSPI_BSPI_XOR_ENABLE, 1);
+	}
+	qspi->bspi_enabled = 1;
+	bcm_qspi_disable_bspi(qspi);
+	bcm_qspi_write(qspi, BSPI, BSPI_B0_CTRL, 0);
+	bcm_qspi_write(qspi, BSPI, BSPI_B1_CTRL, 0);
+}
+
+static void bcm_qspi_hw_init(struct bcm_qspi *qspi)
+{
+	struct bcm_qspi_parms parms;
+
+	bcm_qspi_write(qspi, MSPI, MSPI_SPCR1_LSB, 0);
+	bcm_qspi_write(qspi, MSPI, MSPI_SPCR1_MSB, 0);
+	bcm_qspi_write(qspi, MSPI, MSPI_NEWQP, 0);
+	bcm_qspi_write(qspi, MSPI, MSPI_ENDQP, 0);
+	bcm_qspi_write(qspi, MSPI, MSPI_SPCR2, 0x20);
+
+	parms.mode = SPI_MODE_3;
+	parms.bits_per_word = 8;
+	parms.speed_hz = qspi->max_speed_hz;
+	bcm_qspi_hw_set_parms(qspi, &parms);
+
+	if (has_bspi(qspi))
+		bcm_qspi_bspi_init(qspi);
+}
+
+static void bcm_qspi_hw_uninit(struct bcm_qspi *qspi)
+{
+	bcm_qspi_write(qspi, MSPI, MSPI_SPCR2, 0);
+	if (has_bspi(qspi))
+		bcm_qspi_write(qspi, MSPI, MSPI_WRITE_LOCK, 0);
+
+}
+
+static const struct of_device_id bcm_qspi_of_match[] = {
+	{ .compatible = "brcm,spi-bcm-qspi" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, bcm_qspi_of_match);
+
+int bcm_qspi_probe(struct platform_device *pdev,
+		   struct bcm_qspi_soc_intc *soc_intc)
+{
+	struct device *dev = &pdev->dev;
+	struct bcm_qspi *qspi;
+	struct spi_master *master;
+	struct resource *res;
+	int irq, ret = 0, num_ints = 0;
+	u32 val;
+	const char *name = NULL;
+	int num_irqs = ARRAY_SIZE(qspi_irq_tab);
+
+	/* We only support device-tree instantiation */
+	if (!dev->of_node)
+		return -ENODEV;
+
+	if (!of_match_node(bcm_qspi_of_match, dev->of_node))
+		return -ENODEV;
+
+	master = spi_alloc_master(dev, sizeof(struct bcm_qspi));
+	if (!master) {
+		dev_err(dev, "error allocating spi_master\n");
+		return -ENOMEM;
+	}
+
+	qspi = spi_master_get_devdata(master);
+	qspi->pdev = pdev;
+	qspi->trans_pos.trans = NULL;
+	qspi->trans_pos.byte = 0;
+	qspi->master = master;
+
+	master->bus_num = -1;
+	master->mode_bits = SPI_CPHA | SPI_CPOL | SPI_RX_DUAL | SPI_RX_QUAD;
+	master->setup = bcm_qspi_setup;
+	master->transfer_one = bcm_qspi_transfer_one;
+	master->spi_flash_read = bcm_qspi_flash_read;
+	master->cleanup = bcm_qspi_cleanup;
+	master->dev.of_node = dev->of_node;
+	master->num_chipselect = NUM_CHIPSELECT;
+
+	qspi->big_endian = of_device_is_big_endian(dev->of_node);
+
+	if (!of_property_read_u32(dev->of_node, "num-cs", &val))
+		master->num_chipselect = val;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "hif_mspi");
+	if (!res)
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   "mspi");
+
+	if (res) {
+		qspi->base[MSPI]  = devm_ioremap_resource(dev, res);
+		if (IS_ERR(qspi->base[MSPI])) {
+			ret = PTR_ERR(qspi->base[MSPI]);
+			goto qspi_probe_err;
+		}
+	} else {
+		goto qspi_probe_err;
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "bspi");
+	if (res) {
+		qspi->base[BSPI]  = devm_ioremap_resource(dev, res);
+		if (IS_ERR(qspi->base[BSPI])) {
+			ret = PTR_ERR(qspi->base[BSPI]);
+			goto qspi_probe_err;
+		}
+		qspi->bspi_mode = true;
+	} else {
+		qspi->bspi_mode = false;
+	}
+
+	dev_info(dev, "using %smspi mode\n", qspi->bspi_mode ? "bspi-" : "");
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs_reg");
+	if (res) {
+		qspi->base[CHIP_SELECT]  = devm_ioremap_resource(dev, res);
+		if (IS_ERR(qspi->base[CHIP_SELECT])) {
+			ret = PTR_ERR(qspi->base[CHIP_SELECT]);
+			goto qspi_probe_err;
+		}
+	}
+
+	qspi->dev_ids = kcalloc(num_irqs, sizeof(struct bcm_qspi_dev_id),
+				GFP_KERNEL);
+	if (!qspi->dev_ids) {
+		ret = -ENOMEM;
+		goto qspi_probe_err;
+	}
+
+	for (val = 0; val < num_irqs; val++) {
+		irq = -1;
+		name = qspi_irq_tab[val].irq_name;
+		if (qspi_irq_tab[val].irq_source == SINGLE_L2) {
+			/* get the l2 interrupts */
+			irq = platform_get_irq_byname(pdev, name);
+		} else if (!num_ints && soc_intc) {
+			/* all mspi, bspi intrs muxed to one L1 intr */
+			irq = platform_get_irq(pdev, 0);
+		}
+
+		if (irq  >= 0) {
+			ret = devm_request_irq(&pdev->dev, irq,
+					       qspi_irq_tab[val].irq_handler, 0,
+					       name,
+					       &qspi->dev_ids[val]);
+			if (ret < 0) {
+				dev_err(&pdev->dev, "IRQ %s not found\n", name);
+				goto qspi_probe_err;
+			}
+
+			qspi->dev_ids[val].dev = qspi;
+			qspi->dev_ids[val].irqp = &qspi_irq_tab[val];
+			num_ints++;
+			dev_dbg(&pdev->dev, "registered IRQ %s %d\n",
+				qspi_irq_tab[val].irq_name,
+				irq);
+		}
+	}
+
+	if (!num_ints) {
+		dev_err(&pdev->dev, "no IRQs registered, cannot init driver\n");
+		ret = -EINVAL;
+		goto qspi_probe_err;
+	}
+
+	/*
+	 * Some SoCs integrate spi controller (e.g., its interrupt bits)
+	 * in specific ways
+	 */
+	if (soc_intc) {
+		qspi->soc_intc = soc_intc;
+		soc_intc->bcm_qspi_int_set(soc_intc, MSPI_DONE, true);
+	} else {
+		qspi->soc_intc = NULL;
+	}
+
+	qspi->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(qspi->clk)) {
+		dev_warn(dev, "unable to get clock\n");
+		ret = PTR_ERR(qspi->clk);
+		goto qspi_probe_err;
+	}
+
+	ret = clk_prepare_enable(qspi->clk);
+	if (ret) {
+		dev_err(dev, "failed to prepare clock\n");
+		goto qspi_probe_err;
+	}
+
+	qspi->base_clk = clk_get_rate(qspi->clk);
+	qspi->max_speed_hz = qspi->base_clk / (QSPI_SPBR_MIN * 2);
+
+	bcm_qspi_hw_init(qspi);
+	init_completion(&qspi->mspi_done);
+	init_completion(&qspi->bspi_done);
+	qspi->curr_cs = -1;
+
+	platform_set_drvdata(pdev, qspi);
+
+	qspi->xfer_mode.width = -1;
+	qspi->xfer_mode.addrlen = -1;
+	qspi->xfer_mode.hp = -1;
+
+	ret = devm_spi_register_master(&pdev->dev, master);
+	if (ret < 0) {
+		dev_err(dev, "can't register master\n");
+		goto qspi_reg_err;
+	}
+
+	return 0;
+
+qspi_reg_err:
+	bcm_qspi_hw_uninit(qspi);
+	clk_disable_unprepare(qspi->clk);
+qspi_probe_err:
+	spi_master_put(master);
+	kfree(qspi->dev_ids);
+	return ret;
+}
+/* probe function to be called by SoC specific platform driver probe */
+EXPORT_SYMBOL_GPL(bcm_qspi_probe);
+
+int bcm_qspi_remove(struct platform_device *pdev)
+{
+	struct bcm_qspi *qspi = platform_get_drvdata(pdev);
+
+	platform_set_drvdata(pdev, NULL);
+	bcm_qspi_hw_uninit(qspi);
+	clk_disable_unprepare(qspi->clk);
+	kfree(qspi->dev_ids);
+	spi_unregister_master(qspi->master);
+
+	return 0;
+}
+/* function to be called by SoC specific platform driver remove() */
+EXPORT_SYMBOL_GPL(bcm_qspi_remove);
+
+static int __maybe_unused bcm_qspi_suspend(struct device *dev)
+{
+	struct bcm_qspi *qspi = dev_get_drvdata(dev);
+
+	spi_master_suspend(qspi->master);
+	clk_disable(qspi->clk);
+	bcm_qspi_hw_uninit(qspi);
+
+	return 0;
+};
+
+static int __maybe_unused bcm_qspi_resume(struct device *dev)
+{
+	struct bcm_qspi *qspi = dev_get_drvdata(dev);
+	int ret = 0;
+
+	bcm_qspi_hw_init(qspi);
+	bcm_qspi_chip_select(qspi, qspi->curr_cs);
+	if (qspi->soc_intc)
+		/* enable MSPI interrupt */
+		qspi->soc_intc->bcm_qspi_int_set(qspi->soc_intc, MSPI_DONE,
+						 true);
+
+	ret = clk_enable(qspi->clk);
+	if (!ret)
+		spi_master_resume(qspi->master);
+
+	return ret;
+}
+
+SIMPLE_DEV_PM_OPS(bcm_qspi_pm_ops, bcm_qspi_suspend, bcm_qspi_resume);
+
+/* pm_ops to be called by SoC specific platform driver */
+EXPORT_SYMBOL_GPL(bcm_qspi_pm_ops);
+
+MODULE_AUTHOR("Kamal Dasu");
+MODULE_DESCRIPTION("Broadcom QSPI driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:" DRIVER_NAME);
diff --git a/drivers/spi/spi-bcm-qspi.h b/drivers/spi/spi-bcm-qspi.h
new file mode 100644
index 000000000000..7abfc75a3860
--- /dev/null
+++ b/drivers/spi/spi-bcm-qspi.h
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation (the "GPL").
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 (GPLv2) for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 (GPLv2) along with this source code.
+ */
+
+#ifndef __SPI_BCM_QSPI_H__
+#define __SPI_BCM_QSPI_H__
+
+#include <linux/types.h>
+#include <linux/io.h>
+
+/* BSPI interrupt masks */
+#define INTR_BSPI_LR_OVERREAD_MASK		BIT(4)
+#define INTR_BSPI_LR_SESSION_DONE_MASK		BIT(3)
+#define INTR_BSPI_LR_IMPATIENT_MASK		BIT(2)
+#define INTR_BSPI_LR_SESSION_ABORTED_MASK	BIT(1)
+#define INTR_BSPI_LR_FULLNESS_REACHED_MASK	BIT(0)
+
+#define BSPI_LR_INTERRUPTS_DATA		       \
+	(INTR_BSPI_LR_SESSION_DONE_MASK |	       \
+	 INTR_BSPI_LR_FULLNESS_REACHED_MASK)
+
+#define BSPI_LR_INTERRUPTS_ERROR               \
+	(INTR_BSPI_LR_OVERREAD_MASK |	       \
+	 INTR_BSPI_LR_IMPATIENT_MASK |	       \
+	 INTR_BSPI_LR_SESSION_ABORTED_MASK)
+
+#define BSPI_LR_INTERRUPTS_ALL                 \
+	(BSPI_LR_INTERRUPTS_ERROR |	       \
+	 BSPI_LR_INTERRUPTS_DATA)
+
+/* MSPI Interrupt masks */
+#define INTR_MSPI_HALTED_MASK			BIT(6)
+#define INTR_MSPI_DONE_MASK			BIT(5)
+
+#define MSPI_INTERRUPTS_ALL		       \
+	(INTR_MSPI_DONE_MASK |		       \
+	 INTR_MSPI_HALTED_MASK)
+
+#define QSPI_INTERRUPTS_ALL                    \
+	(MSPI_INTERRUPTS_ALL |		       \
+	 BSPI_LR_INTERRUPTS_ALL)
+
+struct platform_device;
+struct dev_pm_ops;
+
+enum {
+	MSPI_DONE = 0x1,
+	BSPI_DONE = 0x2,
+	BSPI_ERR = 0x4,
+	MSPI_BSPI_DONE = 0x7
+};
+
+struct bcm_qspi_soc_intc {
+	void (*bcm_qspi_int_ack)(struct bcm_qspi_soc_intc *soc_intc, int type);
+	void (*bcm_qspi_int_set)(struct bcm_qspi_soc_intc *soc_intc, int type,
+				 bool en);
+	u32 (*bcm_qspi_get_int_status)(struct bcm_qspi_soc_intc *soc_intc);
+};
+
+/* Read controller register*/
+static inline u32 bcm_qspi_readl(bool be, void __iomem *addr)
+{
+	if (be)
+		return ioread32be(addr);
+	else
+		return readl_relaxed(addr);
+}
+
+/* Write controller register*/
+static inline void bcm_qspi_writel(bool be,
+				   unsigned int data, void __iomem *addr)
+{
+	if (be)
+		iowrite32be(data, addr);
+	else
+		writel_relaxed(data, addr);
+}
+
+static inline u32 get_qspi_mask(int type)
+{
+	switch (type) {
+	case MSPI_DONE:
+		return INTR_MSPI_DONE_MASK;
+	case BSPI_DONE:
+		return BSPI_LR_INTERRUPTS_ALL;
+	case MSPI_BSPI_DONE:
+		return QSPI_INTERRUPTS_ALL;
+	case BSPI_ERR:
+		return BSPI_LR_INTERRUPTS_ERROR;
+	}
+
+	return 0;
+}
+
+/* The common driver functions to be called by the SoC platform driver */
+int bcm_qspi_probe(struct platform_device *pdev,
+		   struct bcm_qspi_soc_intc *soc_intc);
+int bcm_qspi_remove(struct platform_device *pdev);
+
+/* pm_ops used by the SoC platform driver called on PM suspend/resume */
+extern const struct dev_pm_ops bcm_qspi_pm_ops;
+
+#endif /* __SPI_BCM_QSPI_H__ */
diff --git a/drivers/spi/spi-brcmstb-qspi.c b/drivers/spi/spi-brcmstb-qspi.c
new file mode 100644
index 000000000000..c7df92e7cf6f
--- /dev/null
+++ b/drivers/spi/spi-brcmstb-qspi.c
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation (the "GPL").
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 (GPLv2) for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 (GPLv2) along with this source code.
+ */
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include "spi-bcm-qspi.h"
+
+static const struct of_device_id brcmstb_qspi_of_match[] = {
+	{ .compatible = "brcm,spi-brcmstb-qspi" },
+	{ .compatible = "brcm,spi-brcmstb-mspi" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, brcmstb_qspi_of_match);
+
+static int brcmstb_qspi_probe(struct platform_device *pdev)
+{
+	return bcm_qspi_probe(pdev, NULL);
+}
+
+static int brcmstb_qspi_remove(struct platform_device *pdev)
+{
+	return bcm_qspi_remove(pdev);
+}
+
+static struct platform_driver brcmstb_qspi_driver = {
+	.probe			= brcmstb_qspi_probe,
+	.remove			= brcmstb_qspi_remove,
+	.driver = {
+		.name		= "brcmstb_qspi",
+		.pm		= &bcm_qspi_pm_ops,
+		.of_match_table = brcmstb_qspi_of_match,
+	}
+};
+module_platform_driver(brcmstb_qspi_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Kamal Dasu");
+MODULE_DESCRIPTION("Broadcom SPI driver for settop SoC");
diff --git a/drivers/spi/spi-cavium-thunderx.c b/drivers/spi/spi-cavium-thunderx.c
new file mode 100644
index 000000000000..877937706240
--- /dev/null
+++ b/drivers/spi/spi-cavium-thunderx.c
@@ -0,0 +1,120 @@
+/*
+ * Cavium ThunderX SPI driver.
+ *
+ * Copyright (C) 2016 Cavium Inc.
+ * Authors: Jan Glauber <jglauber@cavium.com>
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/spi/spi.h>
+
+#include "spi-cavium.h"
+
+#define DRV_NAME "spi-thunderx"
+
+#define SYS_FREQ_DEFAULT 700000000 /* 700 Mhz */
+
+static int thunderx_spi_probe(struct pci_dev *pdev,
+			      const struct pci_device_id *ent)
+{
+	struct device *dev = &pdev->dev;
+	struct spi_master *master;
+	struct octeon_spi *p;
+	int ret;
+
+	master = spi_alloc_master(dev, sizeof(struct octeon_spi));
+	if (!master)
+		return -ENOMEM;
+
+	p = spi_master_get_devdata(master);
+
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		goto error;
+
+	ret = pci_request_regions(pdev, DRV_NAME);
+	if (ret)
+		goto error;
+
+	p->register_base = pcim_iomap(pdev, 0, pci_resource_len(pdev, 0));
+	if (!p->register_base) {
+		ret = -EINVAL;
+		goto error;
+	}
+
+	p->regs.config = 0x1000;
+	p->regs.status = 0x1008;
+	p->regs.tx = 0x1010;
+	p->regs.data = 0x1080;
+
+	p->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(p->clk)) {
+		ret = PTR_ERR(p->clk);
+		goto error;
+	}
+
+	ret = clk_prepare_enable(p->clk);
+	if (ret)
+		goto error;
+
+	p->sys_freq = clk_get_rate(p->clk);
+	if (!p->sys_freq)
+		p->sys_freq = SYS_FREQ_DEFAULT;
+	dev_info(dev, "Set system clock to %u\n", p->sys_freq);
+
+	master->num_chipselect = 4;
+	master->mode_bits = SPI_CPHA | SPI_CPOL | SPI_CS_HIGH |
+			    SPI_LSB_FIRST | SPI_3WIRE;
+	master->transfer_one_message = octeon_spi_transfer_one_message;
+	master->bits_per_word_mask = SPI_BPW_MASK(8);
+	master->max_speed_hz = OCTEON_SPI_MAX_CLOCK_HZ;
+	master->dev.of_node = pdev->dev.of_node;
+
+	pci_set_drvdata(pdev, master);
+
+	ret = devm_spi_register_master(dev, master);
+	if (ret)
+		goto error;
+
+	return 0;
+
+error:
+	clk_disable_unprepare(p->clk);
+	spi_master_put(master);
+	return ret;
+}
+
+static void thunderx_spi_remove(struct pci_dev *pdev)
+{
+	struct spi_master *master = pci_get_drvdata(pdev);
+	struct octeon_spi *p;
+
+	p = spi_master_get_devdata(master);
+	if (!p)
+		return;
+
+	clk_disable_unprepare(p->clk);
+	/* Put everything in a known state. */
+	writeq(0, p->register_base + OCTEON_SPI_CFG(p));
+}
+
+static const struct pci_device_id thunderx_spi_pci_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa00b) },
+	{ 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, thunderx_spi_pci_id_table);
+
+static struct pci_driver thunderx_spi_driver = {
+	.name		= DRV_NAME,
+	.id_table	= thunderx_spi_pci_id_table,
+	.probe		= thunderx_spi_probe,
+	.remove		= thunderx_spi_remove,
+};
+
+module_pci_driver(thunderx_spi_driver);
+
+MODULE_DESCRIPTION("Cavium, Inc. ThunderX SPI bus driver");
+MODULE_AUTHOR("Jan Glauber");
+MODULE_LICENSE("GPL");
diff --git a/drivers/spi/spi-cavium.h b/drivers/spi/spi-cavium.h
index 88c5f36e7ea7..1f91d61b745b 100644
--- a/drivers/spi/spi-cavium.h
+++ b/drivers/spi/spi-cavium.h
@@ -1,6 +1,8 @@
 #ifndef __SPI_CAVIUM_H
 #define __SPI_CAVIUM_H
 
+#include <linux/clk.h>
+
 #define OCTEON_SPI_MAX_BYTES 9
 #define OCTEON_SPI_MAX_CLOCK_HZ 16000000
 
@@ -17,6 +19,7 @@ struct octeon_spi {
 	u64 cs_enax;
 	int sys_freq;
 	struct octeon_spi_regs regs;
+	struct clk *clk;
 };
 
 #define OCTEON_SPI_CFG(x)	(x->regs.config)
diff --git a/drivers/spi/spi-dw.c b/drivers/spi/spi-dw.c
index c09bb745693a..27960e46135d 100644
--- a/drivers/spi/spi-dw.c
+++ b/drivers/spi/spi-dw.c
@@ -283,7 +283,6 @@ static int dw_spi_transfer_one(struct spi_master *master,
 	struct chip_data *chip = spi_get_ctldata(spi);
 	u8 imask = 0;
 	u16 txlevel = 0;
-	u16 clk_div;
 	u32 cr0;
 	int ret;
 
@@ -298,13 +297,13 @@ static int dw_spi_transfer_one(struct spi_master *master,
 	spi_enable_chip(dws, 0);
 
 	/* Handle per transfer options for bpw and speed */
-	if (transfer->speed_hz != chip->speed_hz) {
-		/* clk_div doesn't support odd number */
-		clk_div = (dws->max_freq / transfer->speed_hz + 1) & 0xfffe;
-
-		chip->speed_hz = transfer->speed_hz;
-		chip->clk_div = clk_div;
-
+	if (transfer->speed_hz != dws->current_freq) {
+		if (transfer->speed_hz != chip->speed_hz) {
+			/* clk_div doesn't support odd number */
+			chip->clk_div = (DIV_ROUND_UP(dws->max_freq, transfer->speed_hz) + 1) & 0xfffe;
+			chip->speed_hz = transfer->speed_hz;
+		}
+		dws->current_freq = transfer->speed_hz;
 		spi_set_clk(dws, chip->clk_div);
 	}
 	if (transfer->bits_per_word == 8) {
diff --git a/drivers/spi/spi-dw.h b/drivers/spi/spi-dw.h
index 61bc3cbab38d..c21ca02f8ec5 100644
--- a/drivers/spi/spi-dw.h
+++ b/drivers/spi/spi-dw.h
@@ -123,6 +123,7 @@ struct dw_spi {
 	u8			n_bytes;	/* current is a 1/2 bytes op */
 	u32			dma_width;
 	irqreturn_t		(*transfer_handler)(struct dw_spi *dws);
+	u32			current_freq;	/* frequency in hz */
 
 	/* DMA info */
 	int			dma_inited;
diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c
index 9e9dadb52b3d..35c0dd945668 100644
--- a/drivers/spi/spi-fsl-dspi.c
+++ b/drivers/spi/spi-fsl-dspi.c
@@ -159,7 +159,7 @@ struct fsl_dspi {
 	u8			cs;
 	u16			void_write_data;
 	u32			cs_change;
-	struct fsl_dspi_devtype_data *devtype_data;
+	const struct fsl_dspi_devtype_data *devtype_data;
 
 	wait_queue_head_t	waitq;
 	u32			waitflags;
@@ -624,10 +624,13 @@ static int dspi_resume(struct device *dev)
 {
 	struct spi_master *master = dev_get_drvdata(dev);
 	struct fsl_dspi *dspi = spi_master_get_devdata(master);
+	int ret;
 
 	pinctrl_pm_select_default_state(dev);
 
-	clk_prepare_enable(dspi->clk);
+	ret = clk_prepare_enable(dspi->clk);
+	if (ret)
+		return ret;
 	spi_master_resume(master);
 
 	return 0;
@@ -651,8 +654,6 @@ static int dspi_probe(struct platform_device *pdev)
 	struct resource *res;
 	void __iomem *base;
 	int ret = 0, cs_num, bus_num;
-	const struct of_device_id *of_id =
-			of_match_device(fsl_dspi_dt_ids, &pdev->dev);
 
 	master = spi_alloc_master(&pdev->dev, sizeof(struct fsl_dspi));
 	if (!master)
@@ -686,7 +687,7 @@ static int dspi_probe(struct platform_device *pdev)
 	}
 	master->bus_num = bus_num;
 
-	dspi->devtype_data = (struct fsl_dspi_devtype_data *)of_id->data;
+	dspi->devtype_data = of_device_get_match_data(&pdev->dev);
 	if (!dspi->devtype_data) {
 		dev_err(&pdev->dev, "can't get devtype_data\n");
 		ret = -EFAULT;
@@ -728,7 +729,9 @@ static int dspi_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "unable to get clock\n");
 		goto out_master_put;
 	}
-	clk_prepare_enable(dspi->clk);
+	ret = clk_prepare_enable(dspi->clk);
+	if (ret)
+		goto out_master_put;
 
 	master->max_speed_hz =
 		clk_get_rate(dspi->clk) / dspi->devtype_data->max_clock_factor;
@@ -760,7 +763,6 @@ static int dspi_remove(struct platform_device *pdev)
 	/* Disconnect from the SPI framework */
 	clk_disable_unprepare(dspi->clk);
 	spi_unregister_master(dspi->master);
-	spi_master_put(dspi->master);
 
 	return 0;
 }
diff --git a/drivers/spi/spi-fsl-espi.c b/drivers/spi/spi-fsl-espi.c
index 8d85a3c343da..7451585a080e 100644
--- a/drivers/spi/spi-fsl-espi.c
+++ b/drivers/spi/spi-fsl-espi.c
@@ -12,7 +12,6 @@
 #include <linux/err.h>
 #include <linux/fsl_devices.h>
 #include <linux/interrupt.h>
-#include <linux/irq.h>
 #include <linux/module.h>
 #include <linux/mm.h>
 #include <linux/of.h>
@@ -27,40 +26,29 @@
 #include "spi-fsl-lib.h"
 
 /* eSPI Controller registers */
-struct fsl_espi_reg {
-	__be32 mode;		/* 0x000 - eSPI mode register */
-	__be32 event;		/* 0x004 - eSPI event register */
-	__be32 mask;		/* 0x008 - eSPI mask register */
-	__be32 command;		/* 0x00c - eSPI command register */
-	__be32 transmit;	/* 0x010 - eSPI transmit FIFO access register*/
-	__be32 receive;		/* 0x014 - eSPI receive FIFO access register*/
-	u8 res[8];		/* 0x018 - 0x01c reserved */
-	__be32 csmode[4];	/* 0x020 - 0x02c eSPI cs mode register */
-};
+#define ESPI_SPMODE	0x00	/* eSPI mode register */
+#define ESPI_SPIE	0x04	/* eSPI event register */
+#define ESPI_SPIM	0x08	/* eSPI mask register */
+#define ESPI_SPCOM	0x0c	/* eSPI command register */
+#define ESPI_SPITF	0x10	/* eSPI transmit FIFO access register*/
+#define ESPI_SPIRF	0x14	/* eSPI receive FIFO access register*/
+#define ESPI_SPMODE0	0x20	/* eSPI cs0 mode register */
 
-struct fsl_espi_transfer {
-	const void *tx_buf;
-	void *rx_buf;
-	unsigned len;
-	unsigned n_tx;
-	unsigned n_rx;
-	unsigned actual_length;
-	int status;
-};
+#define ESPI_SPMODEx(x)	(ESPI_SPMODE0 + (x) * 4)
 
 /* eSPI Controller mode register definitions */
-#define SPMODE_ENABLE		(1 << 31)
-#define SPMODE_LOOP		(1 << 30)
+#define SPMODE_ENABLE		BIT(31)
+#define SPMODE_LOOP		BIT(30)
 #define SPMODE_TXTHR(x)		((x) << 8)
 #define SPMODE_RXTHR(x)		((x) << 0)
 
 /* eSPI Controller CS mode register definitions */
-#define CSMODE_CI_INACTIVEHIGH	(1 << 31)
-#define CSMODE_CP_BEGIN_EDGECLK	(1 << 30)
-#define CSMODE_REV		(1 << 29)
-#define CSMODE_DIV16		(1 << 28)
+#define CSMODE_CI_INACTIVEHIGH	BIT(31)
+#define CSMODE_CP_BEGIN_EDGECLK	BIT(30)
+#define CSMODE_REV		BIT(29)
+#define CSMODE_DIV16		BIT(28)
 #define CSMODE_PM(x)		((x) << 24)
-#define CSMODE_POL_1		(1 << 20)
+#define CSMODE_POL_1		BIT(20)
 #define CSMODE_LEN(x)		((x) << 16)
 #define CSMODE_BEF(x)		((x) << 12)
 #define CSMODE_AFT(x)		((x) << 8)
@@ -72,29 +60,114 @@ struct fsl_espi_transfer {
 		| CSMODE_AFT(0) | CSMODE_CG(1))
 
 /* SPIE register values */
-#define	SPIE_NE		0x00000200	/* Not empty */
-#define	SPIE_NF		0x00000100	/* Not full */
-
-/* SPIM register values */
-#define	SPIM_NE		0x00000200	/* Not empty */
-#define	SPIM_NF		0x00000100	/* Not full */
 #define SPIE_RXCNT(reg)     ((reg >> 24) & 0x3F)
 #define SPIE_TXCNT(reg)     ((reg >> 16) & 0x3F)
+#define	SPIE_TXE		BIT(15)	/* TX FIFO empty */
+#define	SPIE_DON		BIT(14)	/* TX done */
+#define	SPIE_RXT		BIT(13)	/* RX FIFO threshold */
+#define	SPIE_RXF		BIT(12)	/* RX FIFO full */
+#define	SPIE_TXT		BIT(11)	/* TX FIFO threshold*/
+#define	SPIE_RNE		BIT(9)	/* RX FIFO not empty */
+#define	SPIE_TNF		BIT(8)	/* TX FIFO not full */
+
+/* SPIM register values */
+#define	SPIM_TXE		BIT(15)	/* TX FIFO empty */
+#define	SPIM_DON		BIT(14)	/* TX done */
+#define	SPIM_RXT		BIT(13)	/* RX FIFO threshold */
+#define	SPIM_RXF		BIT(12)	/* RX FIFO full */
+#define	SPIM_TXT		BIT(11)	/* TX FIFO threshold*/
+#define	SPIM_RNE		BIT(9)	/* RX FIFO not empty */
+#define	SPIM_TNF		BIT(8)	/* TX FIFO not full */
 
 /* SPCOM register values */
 #define SPCOM_CS(x)		((x) << 30)
+#define SPCOM_DO		BIT(28) /* Dual output */
+#define SPCOM_TO		BIT(27) /* TX only */
+#define SPCOM_RXSKIP(x)		((x) << 16)
 #define SPCOM_TRANLEN(x)	((x) << 0)
+
 #define	SPCOM_TRANLEN_MAX	0x10000	/* Max transaction length */
 
 #define AUTOSUSPEND_TIMEOUT 2000
 
+static inline u32 fsl_espi_read_reg(struct mpc8xxx_spi *mspi, int offset)
+{
+	return ioread32be(mspi->reg_base + offset);
+}
+
+static inline u8 fsl_espi_read_reg8(struct mpc8xxx_spi *mspi, int offset)
+{
+	return ioread8(mspi->reg_base + offset);
+}
+
+static inline void fsl_espi_write_reg(struct mpc8xxx_spi *mspi, int offset,
+				      u32 val)
+{
+	iowrite32be(val, mspi->reg_base + offset);
+}
+
+static inline void fsl_espi_write_reg8(struct mpc8xxx_spi *mspi, int offset,
+				       u8 val)
+{
+	iowrite8(val, mspi->reg_base + offset);
+}
+
+static void fsl_espi_copy_to_buf(struct spi_message *m,
+				 struct mpc8xxx_spi *mspi)
+{
+	struct spi_transfer *t;
+	u8 *buf = mspi->local_buf;
+
+	list_for_each_entry(t, &m->transfers, transfer_list) {
+		if (t->tx_buf)
+			memcpy(buf, t->tx_buf, t->len);
+		else
+			memset(buf, 0, t->len);
+		buf += t->len;
+	}
+}
+
+static void fsl_espi_copy_from_buf(struct spi_message *m,
+				   struct mpc8xxx_spi *mspi)
+{
+	struct spi_transfer *t;
+	u8 *buf = mspi->local_buf;
+
+	list_for_each_entry(t, &m->transfers, transfer_list) {
+		if (t->rx_buf)
+			memcpy(t->rx_buf, buf, t->len);
+		buf += t->len;
+	}
+}
+
+static int fsl_espi_check_message(struct spi_message *m)
+{
+	struct mpc8xxx_spi *mspi = spi_master_get_devdata(m->spi->master);
+	struct spi_transfer *t, *first;
+
+	if (m->frame_length > SPCOM_TRANLEN_MAX) {
+		dev_err(mspi->dev, "message too long, size is %u bytes\n",
+			m->frame_length);
+		return -EMSGSIZE;
+	}
+
+	first = list_first_entry(&m->transfers, struct spi_transfer,
+				 transfer_list);
+	list_for_each_entry(t, &m->transfers, transfer_list) {
+		if (first->bits_per_word != t->bits_per_word ||
+		    first->speed_hz != t->speed_hz) {
+			dev_err(mspi->dev, "bits_per_word/speed_hz should be the same for all transfers\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
 static void fsl_espi_change_mode(struct spi_device *spi)
 {
 	struct mpc8xxx_spi *mspi = spi_master_get_devdata(spi->master);
 	struct spi_mpc8xxx_cs *cs = spi->controller_state;
-	struct fsl_espi_reg *reg_base = mspi->reg_base;
-	__be32 __iomem *mode = &reg_base->csmode[spi->chip_select];
-	__be32 __iomem *espi_mode = &reg_base->mode;
 	u32 tmp;
 	unsigned long flags;
 
@@ -102,10 +175,11 @@ static void fsl_espi_change_mode(struct spi_device *spi)
 	local_irq_save(flags);
 
 	/* Turn off SPI unit prior changing mode */
-	tmp = mpc8xxx_spi_read_reg(espi_mode);
-	mpc8xxx_spi_write_reg(espi_mode, tmp & ~SPMODE_ENABLE);
-	mpc8xxx_spi_write_reg(mode, cs->hw_mode);
-	mpc8xxx_spi_write_reg(espi_mode, tmp);
+	tmp = fsl_espi_read_reg(mspi, ESPI_SPMODE);
+	fsl_espi_write_reg(mspi, ESPI_SPMODE, tmp & ~SPMODE_ENABLE);
+	fsl_espi_write_reg(mspi, ESPI_SPMODEx(spi->chip_select),
+			      cs->hw_mode);
+	fsl_espi_write_reg(mspi, ESPI_SPMODE, tmp);
 
 	local_irq_restore(flags);
 }
@@ -131,27 +205,15 @@ static u32 fsl_espi_tx_buf_lsb(struct mpc8xxx_spi *mpc8xxx_spi)
 	return data;
 }
 
-static int fsl_espi_setup_transfer(struct spi_device *spi,
+static void fsl_espi_setup_transfer(struct spi_device *spi,
 					struct spi_transfer *t)
 {
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(spi->master);
-	int bits_per_word = 0;
+	int bits_per_word = t ? t->bits_per_word : spi->bits_per_word;
+	u32 hz = t ? t->speed_hz : spi->max_speed_hz;
 	u8 pm;
-	u32 hz = 0;
 	struct spi_mpc8xxx_cs *cs = spi->controller_state;
 
-	if (t) {
-		bits_per_word = t->bits_per_word;
-		hz = t->speed_hz;
-	}
-
-	/* spi_transfer level calls that work per-word */
-	if (!bits_per_word)
-		bits_per_word = spi->bits_per_word;
-
-	if (!hz)
-		hz = spi->max_speed_hz;
-
 	cs->rx_shift = 0;
 	cs->tx_shift = 0;
 	cs->get_rx = mpc8xxx_spi_rx_buf_u32;
@@ -169,12 +231,10 @@ static int fsl_espi_setup_transfer(struct spi_device *spi,
 	mpc8xxx_spi->get_rx = cs->get_rx;
 	mpc8xxx_spi->get_tx = cs->get_tx;
 
-	bits_per_word = bits_per_word - 1;
-
 	/* mask out bits we are going to set */
 	cs->hw_mode &= ~(CSMODE_LEN(0xF) | CSMODE_DIV16 | CSMODE_PM(0xF));
 
-	cs->hw_mode |= CSMODE_LEN(bits_per_word);
+	cs->hw_mode |= CSMODE_LEN(bits_per_word - 1);
 
 	if ((mpc8xxx_spi->spibrg / hz) > 64) {
 		cs->hw_mode |= CSMODE_DIV16;
@@ -196,36 +256,16 @@ static int fsl_espi_setup_transfer(struct spi_device *spi,
 	cs->hw_mode |= CSMODE_PM(pm);
 
 	fsl_espi_change_mode(spi);
-	return 0;
-}
-
-static int fsl_espi_cpu_bufs(struct mpc8xxx_spi *mspi, struct spi_transfer *t,
-		unsigned int len)
-{
-	u32 word;
-	struct fsl_espi_reg *reg_base = mspi->reg_base;
-
-	mspi->count = len;
-
-	/* enable rx ints */
-	mpc8xxx_spi_write_reg(&reg_base->mask, SPIM_NE);
-
-	/* transmit word */
-	word = mspi->get_tx(mspi);
-	mpc8xxx_spi_write_reg(&reg_base->transmit, word);
-
-	return 0;
 }
 
 static int fsl_espi_bufs(struct spi_device *spi, struct spi_transfer *t)
 {
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(spi->master);
-	struct fsl_espi_reg *reg_base = mpc8xxx_spi->reg_base;
-	unsigned int len = t->len;
+	u32 word;
 	int ret;
 
 	mpc8xxx_spi->len = t->len;
-	len = roundup(len, 4) / 4;
+	mpc8xxx_spi->count = roundup(t->len, 4) / 4;
 
 	mpc8xxx_spi->tx = t->tx_buf;
 	mpc8xxx_spi->rx = t->rx_buf;
@@ -233,17 +273,15 @@ static int fsl_espi_bufs(struct spi_device *spi, struct spi_transfer *t)
 	reinit_completion(&mpc8xxx_spi->done);
 
 	/* Set SPCOM[CS] and SPCOM[TRANLEN] field */
-	if (t->len > SPCOM_TRANLEN_MAX) {
-		dev_err(mpc8xxx_spi->dev, "Transaction length (%d)"
-				" beyond the SPCOM[TRANLEN] field\n", t->len);
-		return -EINVAL;
-	}
-	mpc8xxx_spi_write_reg(&reg_base->command,
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPCOM,
 		(SPCOM_CS(spi->chip_select) | SPCOM_TRANLEN(t->len - 1)));
 
-	ret = fsl_espi_cpu_bufs(mpc8xxx_spi, t, len);
-	if (ret)
-		return ret;
+	/* enable rx ints */
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, SPIM_RNE);
+
+	/* transmit word */
+	word = mpc8xxx_spi->get_tx(mpc8xxx_spi);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPITF, word);
 
 	/* Won't hang up forever, SPI bus sometimes got lost interrupts... */
 	ret = wait_for_completion_timeout(&mpc8xxx_spi->done, 2 * HZ);
@@ -253,230 +291,76 @@ static int fsl_espi_bufs(struct spi_device *spi, struct spi_transfer *t)
 			mpc8xxx_spi->count);
 
 	/* disable rx ints */
-	mpc8xxx_spi_write_reg(&reg_base->mask, 0);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, 0);
 
-	return mpc8xxx_spi->count;
+	return mpc8xxx_spi->count > 0 ? -EMSGSIZE : 0;
 }
 
-static inline void fsl_espi_addr2cmd(unsigned int addr, u8 *cmd)
-{
-	if (cmd) {
-		cmd[1] = (u8)(addr >> 16);
-		cmd[2] = (u8)(addr >> 8);
-		cmd[3] = (u8)(addr >> 0);
-	}
-}
-
-static inline unsigned int fsl_espi_cmd2addr(u8 *cmd)
-{
-	if (cmd)
-		return cmd[1] << 16 | cmd[2] << 8 | cmd[3] << 0;
-
-	return 0;
-}
-
-static void fsl_espi_do_trans(struct spi_message *m,
-				struct fsl_espi_transfer *tr)
+static int fsl_espi_trans(struct spi_message *m, struct spi_transfer *trans)
 {
+	struct mpc8xxx_spi *mspi = spi_master_get_devdata(m->spi->master);
 	struct spi_device *spi = m->spi;
-	struct mpc8xxx_spi *mspi = spi_master_get_devdata(spi->master);
-	struct fsl_espi_transfer *espi_trans = tr;
-	struct spi_message message;
-	struct spi_transfer *t, *first, trans;
-	int status = 0;
-
-	spi_message_init(&message);
-	memset(&trans, 0, sizeof(trans));
-
-	first = list_first_entry(&m->transfers, struct spi_transfer,
-			transfer_list);
-	list_for_each_entry(t, &m->transfers, transfer_list) {
-		if ((first->bits_per_word != t->bits_per_word) ||
-			(first->speed_hz != t->speed_hz)) {
-			espi_trans->status = -EINVAL;
-			dev_err(mspi->dev,
-				"bits_per_word/speed_hz should be same for the same SPI transfer\n");
-			return;
-		}
-
-		trans.speed_hz = t->speed_hz;
-		trans.bits_per_word = t->bits_per_word;
-		trans.delay_usecs = max(first->delay_usecs, t->delay_usecs);
-	}
-
-	trans.len = espi_trans->len;
-	trans.tx_buf = espi_trans->tx_buf;
-	trans.rx_buf = espi_trans->rx_buf;
-	spi_message_add_tail(&trans, &message);
-
-	list_for_each_entry(t, &message.transfers, transfer_list) {
-		if (t->bits_per_word || t->speed_hz) {
-			status = -EINVAL;
-
-			status = fsl_espi_setup_transfer(spi, t);
-			if (status < 0)
-				break;
-		}
+	int ret;
 
-		if (t->len)
-			status = fsl_espi_bufs(spi, t);
+	fsl_espi_copy_to_buf(m, mspi);
+	fsl_espi_setup_transfer(spi, trans);
 
-		if (status) {
-			status = -EMSGSIZE;
-			break;
-		}
+	ret = fsl_espi_bufs(spi, trans);
 
-		if (t->delay_usecs)
-			udelay(t->delay_usecs);
-	}
+	if (trans->delay_usecs)
+		udelay(trans->delay_usecs);
 
-	espi_trans->status = status;
 	fsl_espi_setup_transfer(spi, NULL);
-}
-
-static void fsl_espi_cmd_trans(struct spi_message *m,
-				struct fsl_espi_transfer *trans, u8 *rx_buff)
-{
-	struct spi_transfer *t;
-	u8 *local_buf;
-	int i = 0;
-	struct fsl_espi_transfer *espi_trans = trans;
-
-	local_buf = kzalloc(SPCOM_TRANLEN_MAX, GFP_KERNEL);
-	if (!local_buf) {
-		espi_trans->status = -ENOMEM;
-		return;
-	}
-
-	list_for_each_entry(t, &m->transfers, transfer_list) {
-		if (t->tx_buf) {
-			memcpy(local_buf + i, t->tx_buf, t->len);
-			i += t->len;
-		}
-	}
-
-	espi_trans->tx_buf = local_buf;
-	espi_trans->rx_buf = local_buf;
-	fsl_espi_do_trans(m, espi_trans);
 
-	espi_trans->actual_length = espi_trans->len;
-	kfree(local_buf);
-}
-
-static void fsl_espi_rw_trans(struct spi_message *m,
-				struct fsl_espi_transfer *trans, u8 *rx_buff)
-{
-	struct fsl_espi_transfer *espi_trans = trans;
-	unsigned int total_len = espi_trans->len;
-	struct spi_transfer *t;
-	u8 *local_buf;
-	u8 *rx_buf = rx_buff;
-	unsigned int trans_len;
-	unsigned int addr;
-	unsigned int tx_only;
-	unsigned int rx_pos = 0;
-	unsigned int pos;
-	int i, loop;
-
-	local_buf = kzalloc(SPCOM_TRANLEN_MAX, GFP_KERNEL);
-	if (!local_buf) {
-		espi_trans->status = -ENOMEM;
-		return;
-	}
-
-	for (pos = 0, loop = 0; pos < total_len; pos += trans_len, loop++) {
-		trans_len = total_len - pos;
-
-		i = 0;
-		tx_only = 0;
-		list_for_each_entry(t, &m->transfers, transfer_list) {
-			if (t->tx_buf) {
-				memcpy(local_buf + i, t->tx_buf, t->len);
-				i += t->len;
-				if (!t->rx_buf)
-					tx_only += t->len;
-			}
-		}
-
-		/* Add additional TX bytes to compensate SPCOM_TRANLEN_MAX */
-		if (loop > 0)
-			trans_len += tx_only;
-
-		if (trans_len > SPCOM_TRANLEN_MAX)
-			trans_len = SPCOM_TRANLEN_MAX;
-
-		/* Update device offset */
-		if (pos > 0) {
-			addr = fsl_espi_cmd2addr(local_buf);
-			addr += rx_pos;
-			fsl_espi_addr2cmd(addr, local_buf);
-		}
-
-		espi_trans->len = trans_len;
-		espi_trans->tx_buf = local_buf;
-		espi_trans->rx_buf = local_buf;
-		fsl_espi_do_trans(m, espi_trans);
-
-		/* If there is at least one RX byte then copy it to rx_buf */
-		if (tx_only < SPCOM_TRANLEN_MAX)
-			memcpy(rx_buf + rx_pos, espi_trans->rx_buf + tx_only,
-					trans_len - tx_only);
-
-		rx_pos += trans_len - tx_only;
-
-		if (loop > 0)
-			espi_trans->actual_length += espi_trans->len - tx_only;
-		else
-			espi_trans->actual_length += espi_trans->len;
-	}
+	if (!ret)
+		fsl_espi_copy_from_buf(m, mspi);
 
-	kfree(local_buf);
+	return ret;
 }
 
 static int fsl_espi_do_one_msg(struct spi_master *master,
 			       struct spi_message *m)
 {
-	struct spi_transfer *t;
-	u8 *rx_buf = NULL;
-	unsigned int n_tx = 0;
-	unsigned int n_rx = 0;
-	unsigned int xfer_len = 0;
-	struct fsl_espi_transfer espi_trans;
+	struct mpc8xxx_spi *mspi = spi_master_get_devdata(m->spi->master);
+	unsigned int delay_usecs = 0;
+	struct spi_transfer *t, trans = {};
+	int ret;
+
+	ret = fsl_espi_check_message(m);
+	if (ret)
+		goto out;
 
 	list_for_each_entry(t, &m->transfers, transfer_list) {
-		if (t->tx_buf)
-			n_tx += t->len;
-		if (t->rx_buf) {
-			n_rx += t->len;
-			rx_buf = t->rx_buf;
-		}
-		if ((t->tx_buf) || (t->rx_buf))
-			xfer_len += t->len;
+		if (t->delay_usecs > delay_usecs)
+			delay_usecs = t->delay_usecs;
 	}
 
-	espi_trans.n_tx = n_tx;
-	espi_trans.n_rx = n_rx;
-	espi_trans.len = xfer_len;
-	espi_trans.actual_length = 0;
-	espi_trans.status = 0;
+	t = list_first_entry(&m->transfers, struct spi_transfer,
+			     transfer_list);
+
+	trans.len = m->frame_length;
+	trans.speed_hz = t->speed_hz;
+	trans.bits_per_word = t->bits_per_word;
+	trans.delay_usecs = delay_usecs;
+	trans.tx_buf = mspi->local_buf;
+	trans.rx_buf = mspi->local_buf;
+
+	if (trans.len)
+		ret = fsl_espi_trans(m, &trans);
 
-	if (!rx_buf)
-		fsl_espi_cmd_trans(m, &espi_trans, NULL);
-	else
-		fsl_espi_rw_trans(m, &espi_trans, rx_buf);
+	m->actual_length = ret ? 0 : trans.len;
+out:
+	if (m->status == -EINPROGRESS)
+		m->status = ret;
 
-	m->actual_length = espi_trans.actual_length;
-	m->status = espi_trans.status;
 	spi_finalize_current_message(master);
-	return 0;
+
+	return ret;
 }
 
 static int fsl_espi_setup(struct spi_device *spi)
 {
 	struct mpc8xxx_spi *mpc8xxx_spi;
-	struct fsl_espi_reg *reg_base;
-	int retval;
-	u32 hw_mode;
 	u32 loop_mode;
 	struct spi_mpc8xxx_cs *cs = spi_get_ctldata(spi);
 
@@ -491,13 +375,11 @@ static int fsl_espi_setup(struct spi_device *spi)
 	}
 
 	mpc8xxx_spi = spi_master_get_devdata(spi->master);
-	reg_base = mpc8xxx_spi->reg_base;
 
 	pm_runtime_get_sync(mpc8xxx_spi->dev);
 
-	hw_mode = cs->hw_mode; /* Save original settings */
-	cs->hw_mode = mpc8xxx_spi_read_reg(
-			&reg_base->csmode[spi->chip_select]);
+	cs->hw_mode = fsl_espi_read_reg(mpc8xxx_spi,
+					   ESPI_SPMODEx(spi->chip_select));
 	/* mask out bits we are going to set */
 	cs->hw_mode &= ~(CSMODE_CP_BEGIN_EDGECLK | CSMODE_CI_INACTIVEHIGH
 			 | CSMODE_REV);
@@ -510,21 +392,17 @@ static int fsl_espi_setup(struct spi_device *spi)
 		cs->hw_mode |= CSMODE_REV;
 
 	/* Handle the loop mode */
-	loop_mode = mpc8xxx_spi_read_reg(&reg_base->mode);
+	loop_mode = fsl_espi_read_reg(mpc8xxx_spi, ESPI_SPMODE);
 	loop_mode &= ~SPMODE_LOOP;
 	if (spi->mode & SPI_LOOP)
 		loop_mode |= SPMODE_LOOP;
-	mpc8xxx_spi_write_reg(&reg_base->mode, loop_mode);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, loop_mode);
 
-	retval = fsl_espi_setup_transfer(spi, NULL);
+	fsl_espi_setup_transfer(spi, NULL);
 
 	pm_runtime_mark_last_busy(mpc8xxx_spi->dev);
 	pm_runtime_put_autosuspend(mpc8xxx_spi->dev);
 
-	if (retval < 0) {
-		cs->hw_mode = hw_mode; /* Restore settings */
-		return retval;
-	}
 	return 0;
 }
 
@@ -536,12 +414,10 @@ static void fsl_espi_cleanup(struct spi_device *spi)
 	spi_set_ctldata(spi, NULL);
 }
 
-void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events)
+static void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events)
 {
-	struct fsl_espi_reg *reg_base = mspi->reg_base;
-
 	/* We need handle RX first */
-	if (events & SPIE_NE) {
+	if (events & SPIE_RNE) {
 		u32 rx_data, tmp;
 		u8 rx_data_8;
 		int rx_nr_bytes = 4;
@@ -551,7 +427,7 @@ void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events)
 		if (SPIE_RXCNT(events) < min(4, mspi->len)) {
 			ret = spin_event_timeout(
 				!(SPIE_RXCNT(events =
-				mpc8xxx_spi_read_reg(&reg_base->event)) <
+				fsl_espi_read_reg(mspi, ESPI_SPIE)) <
 						min(4, mspi->len)),
 						10000, 0); /* 10 msec */
 			if (!ret)
@@ -560,10 +436,10 @@ void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events)
 		}
 
 		if (mspi->len >= 4) {
-			rx_data = mpc8xxx_spi_read_reg(&reg_base->receive);
+			rx_data = fsl_espi_read_reg(mspi, ESPI_SPIRF);
 		} else if (mspi->len <= 0) {
 			dev_err(mspi->dev,
-				"unexpected RX(SPIE_NE) interrupt occurred,\n"
+				"unexpected RX(SPIE_RNE) interrupt occurred,\n"
 				"(local rxlen %d bytes, reg rxlen %d bytes)\n",
 				min(4, mspi->len), SPIE_RXCNT(events));
 			rx_nr_bytes = 0;
@@ -572,7 +448,8 @@ void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events)
 			tmp = mspi->len;
 			rx_data = 0;
 			while (tmp--) {
-				rx_data_8 = in_8((u8 *)&reg_base->receive);
+				rx_data_8 = fsl_espi_read_reg8(mspi,
+							       ESPI_SPIRF);
 				rx_data |= (rx_data_8 << (tmp * 8));
 			}
 
@@ -585,30 +462,24 @@ void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events)
 			mspi->get_rx(rx_data, mspi);
 	}
 
-	if (!(events & SPIE_NF)) {
+	if (!(events & SPIE_TNF)) {
 		int ret;
 
 		/* spin until TX is done */
-		ret = spin_event_timeout(((events = mpc8xxx_spi_read_reg(
-				&reg_base->event)) & SPIE_NF), 1000, 0);
+		ret = spin_event_timeout(((events = fsl_espi_read_reg(
+				mspi, ESPI_SPIE)) & SPIE_TNF), 1000, 0);
 		if (!ret) {
-			dev_err(mspi->dev, "tired waiting for SPIE_NF\n");
-
-			/* Clear the SPIE bits */
-			mpc8xxx_spi_write_reg(&reg_base->event, events);
+			dev_err(mspi->dev, "tired waiting for SPIE_TNF\n");
 			complete(&mspi->done);
 			return;
 		}
 	}
 
-	/* Clear the events */
-	mpc8xxx_spi_write_reg(&reg_base->event, events);
-
 	mspi->count -= 1;
 	if (mspi->count) {
 		u32 word = mspi->get_tx(mspi);
 
-		mpc8xxx_spi_write_reg(&reg_base->transmit, word);
+		fsl_espi_write_reg(mspi, ESPI_SPITF, word);
 	} else {
 		complete(&mspi->done);
 	}
@@ -617,20 +488,21 @@ void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events)
 static irqreturn_t fsl_espi_irq(s32 irq, void *context_data)
 {
 	struct mpc8xxx_spi *mspi = context_data;
-	struct fsl_espi_reg *reg_base = mspi->reg_base;
-	irqreturn_t ret = IRQ_NONE;
 	u32 events;
 
 	/* Get interrupt events(tx/rx) */
-	events = mpc8xxx_spi_read_reg(&reg_base->event);
-	if (events)
-		ret = IRQ_HANDLED;
+	events = fsl_espi_read_reg(mspi, ESPI_SPIE);
+	if (!events)
+		return IRQ_NONE;
 
 	dev_vdbg(mspi->dev, "%s: events %x\n", __func__, events);
 
 	fsl_espi_cpu_irq(mspi, events);
 
-	return ret;
+	/* Clear the events */
+	fsl_espi_write_reg(mspi, ESPI_SPIE, events);
+
+	return IRQ_HANDLED;
 }
 
 #ifdef CONFIG_PM
@@ -638,12 +510,11 @@ static int fsl_espi_runtime_suspend(struct device *dev)
 {
 	struct spi_master *master = dev_get_drvdata(dev);
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master);
-	struct fsl_espi_reg *reg_base = mpc8xxx_spi->reg_base;
 	u32 regval;
 
-	regval = mpc8xxx_spi_read_reg(&reg_base->mode);
+	regval = fsl_espi_read_reg(mpc8xxx_spi, ESPI_SPMODE);
 	regval &= ~SPMODE_ENABLE;
-	mpc8xxx_spi_write_reg(&reg_base->mode, regval);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval);
 
 	return 0;
 }
@@ -652,39 +523,35 @@ static int fsl_espi_runtime_resume(struct device *dev)
 {
 	struct spi_master *master = dev_get_drvdata(dev);
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master);
-	struct fsl_espi_reg *reg_base = mpc8xxx_spi->reg_base;
 	u32 regval;
 
-	regval = mpc8xxx_spi_read_reg(&reg_base->mode);
+	regval = fsl_espi_read_reg(mpc8xxx_spi, ESPI_SPMODE);
 	regval |= SPMODE_ENABLE;
-	mpc8xxx_spi_write_reg(&reg_base->mode, regval);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval);
 
 	return 0;
 }
 #endif
 
-static size_t fsl_espi_max_transfer_size(struct spi_device *spi)
+static size_t fsl_espi_max_message_size(struct spi_device *spi)
 {
 	return SPCOM_TRANLEN_MAX;
 }
 
-static struct spi_master * fsl_espi_probe(struct device *dev,
-		struct resource *mem, unsigned int irq)
+static int fsl_espi_probe(struct device *dev, struct resource *mem,
+			  unsigned int irq)
 {
 	struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
 	struct spi_master *master;
 	struct mpc8xxx_spi *mpc8xxx_spi;
-	struct fsl_espi_reg *reg_base;
 	struct device_node *nc;
 	const __be32 *prop;
 	u32 regval, csmode;
-	int i, len, ret = 0;
+	int i, len, ret;
 
 	master = spi_alloc_master(dev, sizeof(struct mpc8xxx_spi));
-	if (!master) {
-		ret = -ENOMEM;
-		goto err;
-	}
+	if (!master)
+		return -ENOMEM;
 
 	dev_set_drvdata(dev, master);
 
@@ -695,18 +562,23 @@ static struct spi_master * fsl_espi_probe(struct device *dev,
 	master->cleanup = fsl_espi_cleanup;
 	master->transfer_one_message = fsl_espi_do_one_msg;
 	master->auto_runtime_pm = true;
-	master->max_transfer_size = fsl_espi_max_transfer_size;
+	master->max_message_size = fsl_espi_max_message_size;
 
 	mpc8xxx_spi = spi_master_get_devdata(master);
 
+	mpc8xxx_spi->local_buf =
+		devm_kmalloc(dev, SPCOM_TRANLEN_MAX, GFP_KERNEL);
+	if (!mpc8xxx_spi->local_buf) {
+		ret = -ENOMEM;
+		goto err_probe;
+	}
+
 	mpc8xxx_spi->reg_base = devm_ioremap_resource(dev, mem);
 	if (IS_ERR(mpc8xxx_spi->reg_base)) {
 		ret = PTR_ERR(mpc8xxx_spi->reg_base);
 		goto err_probe;
 	}
 
-	reg_base = mpc8xxx_spi->reg_base;
-
 	/* Register for SPI Interrupt */
 	ret = devm_request_irq(dev, mpc8xxx_spi->irq, fsl_espi_irq,
 			  0, "fsl_espi", mpc8xxx_spi);
@@ -719,10 +591,10 @@ static struct spi_master * fsl_espi_probe(struct device *dev,
 	}
 
 	/* SPI controller initializations */
-	mpc8xxx_spi_write_reg(&reg_base->mode, 0);
-	mpc8xxx_spi_write_reg(&reg_base->mask, 0);
-	mpc8xxx_spi_write_reg(&reg_base->command, 0);
-	mpc8xxx_spi_write_reg(&reg_base->event, 0xffffffff);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, 0);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, 0);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPCOM, 0);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIE, 0xffffffff);
 
 	/* Init eSPI CS mode register */
 	for_each_available_child_of_node(master->dev.of_node, nc) {
@@ -747,7 +619,7 @@ static struct spi_master * fsl_espi_probe(struct device *dev,
 			csmode &= ~(CSMODE_AFT(0xf));
 			csmode |= CSMODE_AFT(be32_to_cpup(prop));
 		}
-		mpc8xxx_spi_write_reg(&reg_base->csmode[i], csmode);
+		fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODEx(i), csmode);
 
 		dev_info(dev, "cs=%d, init_csmode=0x%x\n", i, csmode);
 	}
@@ -755,7 +627,7 @@ static struct spi_master * fsl_espi_probe(struct device *dev,
 	/* Enable SPI interface */
 	regval = pdata->initial_spmode | SPMODE_INIT_VAL | SPMODE_ENABLE;
 
-	mpc8xxx_spi_write_reg(&reg_base->mode, regval);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval);
 
 	pm_runtime_set_autosuspend_delay(dev, AUTOSUSPEND_TIMEOUT);
 	pm_runtime_use_autosuspend(dev);
@@ -767,12 +639,13 @@ static struct spi_master * fsl_espi_probe(struct device *dev,
 	if (ret < 0)
 		goto err_pm;
 
-	dev_info(dev, "at 0x%p (irq = %d)\n", reg_base, mpc8xxx_spi->irq);
+	dev_info(dev, "at 0x%p (irq = %d)\n", mpc8xxx_spi->reg_base,
+		 mpc8xxx_spi->irq);
 
 	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
 
-	return master;
+	return 0;
 
 err_pm:
 	pm_runtime_put_noidle(dev);
@@ -780,8 +653,7 @@ err_pm:
 	pm_runtime_set_suspended(dev);
 err_probe:
 	spi_master_put(master);
-err:
-	return ERR_PTR(ret);
+	return ret;
 }
 
 static int of_fsl_espi_get_chipselects(struct device *dev)
@@ -807,10 +679,9 @@ static int of_fsl_espi_probe(struct platform_device *ofdev)
 {
 	struct device *dev = &ofdev->dev;
 	struct device_node *np = ofdev->dev.of_node;
-	struct spi_master *master;
 	struct resource mem;
 	unsigned int irq;
-	int ret = -ENOMEM;
+	int ret;
 
 	ret = of_mpc8xxx_spi_probe(ofdev);
 	if (ret)
@@ -818,28 +689,17 @@ static int of_fsl_espi_probe(struct platform_device *ofdev)
 
 	ret = of_fsl_espi_get_chipselects(dev);
 	if (ret)
-		goto err;
+		return ret;
 
 	ret = of_address_to_resource(np, 0, &mem);
 	if (ret)
-		goto err;
+		return ret;
 
 	irq = irq_of_parse_and_map(np, 0);
-	if (!irq) {
-		ret = -EINVAL;
-		goto err;
-	}
-
-	master = fsl_espi_probe(dev, &mem, irq);
-	if (IS_ERR(master)) {
-		ret = PTR_ERR(master);
-		goto err;
-	}
-
-	return 0;
+	if (!irq)
+		return -EINVAL;
 
-err:
-	return ret;
+	return fsl_espi_probe(dev, &mem, irq);
 }
 
 static int of_fsl_espi_remove(struct platform_device *dev)
@@ -873,27 +733,26 @@ static int of_fsl_espi_resume(struct device *dev)
 	struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
 	struct spi_master *master = dev_get_drvdata(dev);
 	struct mpc8xxx_spi *mpc8xxx_spi;
-	struct fsl_espi_reg *reg_base;
 	u32 regval;
 	int i, ret;
 
 	mpc8xxx_spi = spi_master_get_devdata(master);
-	reg_base = mpc8xxx_spi->reg_base;
 
 	/* SPI controller initializations */
-	mpc8xxx_spi_write_reg(&reg_base->mode, 0);
-	mpc8xxx_spi_write_reg(&reg_base->mask, 0);
-	mpc8xxx_spi_write_reg(&reg_base->command, 0);
-	mpc8xxx_spi_write_reg(&reg_base->event, 0xffffffff);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, 0);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, 0);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPCOM, 0);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIE, 0xffffffff);
 
 	/* Init eSPI CS mode register */
 	for (i = 0; i < pdata->max_chipselect; i++)
-		mpc8xxx_spi_write_reg(&reg_base->csmode[i], CSMODE_INIT_VAL);
+		fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODEx(i),
+				      CSMODE_INIT_VAL);
 
 	/* Enable SPI interface */
 	regval = pdata->initial_spmode | SPMODE_INIT_VAL | SPMODE_ENABLE;
 
-	mpc8xxx_spi_write_reg(&reg_base->mode, regval);
+	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval);
 
 	ret = pm_runtime_force_resume(dev);
 	if (ret < 0)
diff --git a/drivers/spi/spi-fsl-lib.h b/drivers/spi/spi-fsl-lib.h
index 84f5dcb7a897..2925c8089fd9 100644
--- a/drivers/spi/spi-fsl-lib.h
+++ b/drivers/spi/spi-fsl-lib.h
@@ -23,13 +23,14 @@
 /* SPI/eSPI Controller driver's private data. */
 struct mpc8xxx_spi {
 	struct device *dev;
-	void *reg_base;
+	void __iomem *reg_base;
 
 	/* rx & tx bufs from the spi_transfer */
 	const void *tx;
 	void *rx;
 #if IS_ENABLED(CONFIG_SPI_FSL_ESPI)
 	int len;
+	u8 *local_buf;
 #endif
 
 	int subblock;
diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c
index 823cbc92d1e7..7a37090dabbe 100644
--- a/drivers/spi/spi-img-spfi.c
+++ b/drivers/spi/spi-img-spfi.c
@@ -720,8 +720,6 @@ static int img_spfi_remove(struct platform_device *pdev)
 		clk_disable_unprepare(spfi->sys_clk);
 	}
 
-	spi_master_put(master);
-
 	return 0;
 }
 
diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c
index f63cb30f9010..deb782f6556c 100644
--- a/drivers/spi/spi-imx.c
+++ b/drivers/spi/spi-imx.c
@@ -186,17 +186,19 @@ static unsigned int spi_imx_clkdiv_1(unsigned int fin,
 
 /* MX1, MX31, MX35, MX51 CSPI */
 static unsigned int spi_imx_clkdiv_2(unsigned int fin,
-		unsigned int fspi)
+		unsigned int fspi, unsigned int *fres)
 {
 	int i, div = 4;
 
 	for (i = 0; i < 7; i++) {
 		if (fspi * div >= fin)
-			return i;
+			goto out;
 		div <<= 1;
 	}
 
-	return 7;
+out:
+	*fres = fin / div;
+	return i;
 }
 
 static int spi_imx_bytes_per_word(const int bpw)
@@ -453,6 +455,9 @@ static void mx51_ecspi_reset(struct spi_imx_data *spi_imx)
 #define MX31_CSPISTATUS		0x14
 #define MX31_STATUS_RR		(1 << 3)
 
+#define MX31_CSPI_TESTREG	0x1C
+#define MX31_TEST_LBC		(1 << 14)
+
 /* These functions also work for the i.MX35, but be aware that
  * the i.MX35 has a slightly different register layout for bits
  * we do not use here.
@@ -482,9 +487,11 @@ static int mx31_config(struct spi_device *spi, struct spi_imx_config *config)
 {
 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
 	unsigned int reg = MX31_CSPICTRL_ENABLE | MX31_CSPICTRL_MASTER;
+	unsigned int clk;
 
-	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, config->speed_hz) <<
+	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, config->speed_hz, &clk) <<
 		MX31_CSPICTRL_DR_SHIFT;
+	spi_imx->spi_bus_clk = clk;
 
 	if (is_imx35_cspi(spi_imx)) {
 		reg |= (config->bpw - 1) << MX35_CSPICTRL_BL_SHIFT;
@@ -506,6 +513,13 @@ static int mx31_config(struct spi_device *spi, struct spi_imx_config *config)
 
 	writel(reg, spi_imx->base + MXC_CSPICTRL);
 
+	reg = readl(spi_imx->base + MX31_CSPI_TESTREG);
+	if (spi->mode & SPI_LOOP)
+		reg |= MX31_TEST_LBC;
+	else
+		reg &= ~MX31_TEST_LBC;
+	writel(reg, spi_imx->base + MX31_CSPI_TESTREG);
+
 	return 0;
 }
 
@@ -625,9 +639,12 @@ static int mx1_config(struct spi_device *spi, struct spi_imx_config *config)
 {
 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
 	unsigned int reg = MX1_CSPICTRL_ENABLE | MX1_CSPICTRL_MASTER;
+	unsigned int clk;
 
-	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, config->speed_hz) <<
+	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, config->speed_hz, &clk) <<
 		MX1_CSPICTRL_DR_SHIFT;
+	spi_imx->spi_bus_clk = clk;
+
 	reg |= config->bpw - 1;
 
 	if (spi->mode & SPI_CPHA)
@@ -1179,7 +1196,7 @@ static int spi_imx_probe(struct platform_device *pdev)
 	spi_imx->bitbang.master->prepare_message = spi_imx_prepare_message;
 	spi_imx->bitbang.master->unprepare_message = spi_imx_unprepare_message;
 	spi_imx->bitbang.master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
-	if (is_imx51_ecspi(spi_imx))
+	if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx))
 		spi_imx->bitbang.master->mode_bits |= SPI_LOOP;
 
 	init_completion(&spi_imx->xfer_done);
@@ -1251,6 +1268,12 @@ static int spi_imx_probe(struct platform_device *pdev)
 		goto out_clk_put;
 	}
 
+	if (!master->cs_gpios) {
+		dev_err(&pdev->dev, "No CS GPIOs available\n");
+		ret = -EINVAL;
+		goto out_clk_put;
+	}
+
 	for (i = 0; i < master->num_chipselect; i++) {
 		if (!gpio_is_valid(master->cs_gpios[i]))
 			continue;
diff --git a/drivers/spi/spi-iproc-qspi.c b/drivers/spi/spi-iproc-qspi.c
new file mode 100644
index 000000000000..be6ccb204a66
--- /dev/null
+++ b/drivers/spi/spi-iproc-qspi.c
@@ -0,0 +1,163 @@
+/*
+ * Copyright 2016 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "spi-bcm-qspi.h"
+
+#define INTR_BASE_BIT_SHIFT			0x02
+#define INTR_COUNT				0x07
+
+struct bcm_iproc_intc {
+	struct bcm_qspi_soc_intc soc_intc;
+	struct platform_device *pdev;
+	void __iomem *int_reg;
+	void __iomem *int_status_reg;
+	spinlock_t soclock;
+	bool big_endian;
+};
+
+static u32 bcm_iproc_qspi_get_l2_int_status(struct bcm_qspi_soc_intc *soc_intc)
+{
+	struct bcm_iproc_intc *priv =
+			container_of(soc_intc, struct bcm_iproc_intc, soc_intc);
+	void __iomem *mmio = priv->int_status_reg;
+	int i;
+	u32 val = 0, sts = 0;
+
+	for (i = 0; i < INTR_COUNT; i++) {
+		if (bcm_qspi_readl(priv->big_endian, mmio + (i * 4)))
+			val |= 1UL << i;
+	}
+
+	if (val & INTR_MSPI_DONE_MASK)
+		sts |= MSPI_DONE;
+
+	if (val & BSPI_LR_INTERRUPTS_ALL)
+		sts |= BSPI_DONE;
+
+	if (val & BSPI_LR_INTERRUPTS_ERROR)
+		sts |= BSPI_ERR;
+
+	return sts;
+}
+
+static void bcm_iproc_qspi_int_ack(struct bcm_qspi_soc_intc *soc_intc, int type)
+{
+	struct bcm_iproc_intc *priv =
+			container_of(soc_intc, struct bcm_iproc_intc, soc_intc);
+	void __iomem *mmio = priv->int_status_reg;
+	u32 mask = get_qspi_mask(type);
+	int i;
+
+	for (i = 0; i < INTR_COUNT; i++) {
+		if (mask & (1UL << i))
+			bcm_qspi_writel(priv->big_endian, 1, mmio + (i * 4));
+	}
+}
+
+static void bcm_iproc_qspi_int_set(struct bcm_qspi_soc_intc *soc_intc, int type,
+				   bool en)
+{
+	struct bcm_iproc_intc *priv =
+			container_of(soc_intc, struct bcm_iproc_intc, soc_intc);
+	void __iomem *mmio = priv->int_reg;
+	u32 mask = get_qspi_mask(type);
+	u32 val;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->soclock, flags);
+
+	val = bcm_qspi_readl(priv->big_endian, mmio);
+
+	if (en)
+		val = val | (mask << INTR_BASE_BIT_SHIFT);
+	else
+		val = val & ~(mask << INTR_BASE_BIT_SHIFT);
+
+	bcm_qspi_writel(priv->big_endian, val, mmio);
+
+	spin_unlock_irqrestore(&priv->soclock, flags);
+}
+
+static int bcm_iproc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct bcm_iproc_intc *priv;
+	struct bcm_qspi_soc_intc *soc_intc;
+	struct resource *res;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+	soc_intc = &priv->soc_intc;
+	priv->pdev = pdev;
+
+	spin_lock_init(&priv->soclock);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "intr_regs");
+	priv->int_reg = devm_ioremap_resource(dev, res);
+	if (IS_ERR(priv->int_reg))
+		return PTR_ERR(priv->int_reg);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+					   "intr_status_reg");
+	priv->int_status_reg = devm_ioremap_resource(dev, res);
+	if (IS_ERR(priv->int_status_reg))
+		return PTR_ERR(priv->int_status_reg);
+
+	priv->big_endian = of_device_is_big_endian(dev->of_node);
+
+	bcm_iproc_qspi_int_ack(soc_intc, MSPI_BSPI_DONE);
+	bcm_iproc_qspi_int_set(soc_intc, MSPI_BSPI_DONE, false);
+
+	soc_intc->bcm_qspi_int_ack = bcm_iproc_qspi_int_ack;
+	soc_intc->bcm_qspi_int_set = bcm_iproc_qspi_int_set;
+	soc_intc->bcm_qspi_get_int_status = bcm_iproc_qspi_get_l2_int_status;
+
+	return bcm_qspi_probe(pdev, soc_intc);
+}
+
+static int bcm_iproc_remove(struct platform_device *pdev)
+{
+	return bcm_qspi_remove(pdev);
+}
+
+static const struct of_device_id bcm_iproc_of_match[] = {
+	{ .compatible = "brcm,spi-nsp-qspi" },
+	{ .compatible = "brcm,spi-ns2-qspi" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, bcm_iproc_of_match);
+
+static struct platform_driver bcm_iproc_driver = {
+	.probe			= bcm_iproc_probe,
+	.remove			= bcm_iproc_remove,
+	.driver = {
+		.name		= "bcm_iproc",
+		.pm		= &bcm_qspi_pm_ops,
+		.of_match_table = bcm_iproc_of_match,
+	}
+};
+module_platform_driver(bcm_iproc_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Kamal Dasu");
+MODULE_DESCRIPTION("SPI flash driver for Broadcom iProc SoCs");
diff --git a/drivers/spi/spi-jcore.c b/drivers/spi/spi-jcore.c
new file mode 100644
index 000000000000..f8117b80fa22
--- /dev/null
+++ b/drivers/spi/spi-jcore.c
@@ -0,0 +1,231 @@
+/*
+ * J-Core SPI controller driver
+ *
+ * Copyright (C) 2012-2016 Smart Energy Instruments, Inc.
+ *
+ * Current version by Rich Felker
+ * Based loosely on initial version by Oleksandr G Zhadan
+ *
+ */
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/errno.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/spi/spi.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/delay.h>
+
+#define DRV_NAME	"jcore_spi"
+
+#define CTRL_REG	0x0
+#define DATA_REG	0x4
+
+#define JCORE_SPI_CTRL_XMIT		0x02
+#define JCORE_SPI_STAT_BUSY		0x02
+#define JCORE_SPI_CTRL_LOOP		0x08
+#define JCORE_SPI_CTRL_CS_BITS		0x15
+
+#define JCORE_SPI_WAIT_RDY_MAX_LOOP	2000000
+
+struct jcore_spi {
+	struct spi_master *master;
+	void __iomem *base;
+	unsigned int cs_reg;
+	unsigned int speed_reg;
+	unsigned int speed_hz;
+	unsigned int clock_freq;
+};
+
+static int jcore_spi_wait(void __iomem *ctrl_reg)
+{
+	unsigned timeout = JCORE_SPI_WAIT_RDY_MAX_LOOP;
+
+	do {
+		if (!(readl(ctrl_reg) & JCORE_SPI_STAT_BUSY))
+			return 0;
+		cpu_relax();
+	} while (--timeout);
+
+	return -EBUSY;
+}
+
+static void jcore_spi_program(struct jcore_spi *hw)
+{
+	void __iomem *ctrl_reg = hw->base + CTRL_REG;
+
+	if (jcore_spi_wait(ctrl_reg))
+		dev_err(hw->master->dev.parent,
+			"timeout waiting to program ctrl reg.\n");
+
+	writel(hw->cs_reg | hw->speed_reg, ctrl_reg);
+}
+
+static void jcore_spi_chipsel(struct spi_device *spi, bool value)
+{
+	struct jcore_spi *hw = spi_master_get_devdata(spi->master);
+	u32 csbit = 1U << (2 * spi->chip_select);
+
+	dev_dbg(hw->master->dev.parent, "chipselect %d\n", spi->chip_select);
+
+	if (value)
+		hw->cs_reg |= csbit;
+	else
+		hw->cs_reg &= ~csbit;
+
+	jcore_spi_program(hw);
+}
+
+static void jcore_spi_baudrate(struct jcore_spi *hw, int speed)
+{
+	if (speed == hw->speed_hz) return;
+	hw->speed_hz = speed;
+	if (speed >= hw->clock_freq / 2)
+		hw->speed_reg = 0;
+	else
+		hw->speed_reg = ((hw->clock_freq / 2 / speed) - 1) << 27;
+	jcore_spi_program(hw);
+	dev_dbg(hw->master->dev.parent, "speed=%d reg=0x%x\n",
+		speed, hw->speed_reg);
+}
+
+static int jcore_spi_txrx(struct spi_master *master, struct spi_device *spi,
+			  struct spi_transfer *t)
+{
+	struct jcore_spi *hw = spi_master_get_devdata(master);
+
+	void __iomem *ctrl_reg = hw->base + CTRL_REG;
+	void __iomem *data_reg = hw->base + DATA_REG;
+	u32 xmit;
+
+	/* data buffers */
+	const unsigned char *tx;
+	unsigned char *rx;
+	unsigned int len;
+	unsigned int count;
+
+	jcore_spi_baudrate(hw, t->speed_hz);
+
+	xmit = hw->cs_reg | hw->speed_reg | JCORE_SPI_CTRL_XMIT;
+	tx = t->tx_buf;
+	rx = t->rx_buf;
+	len = t->len;
+
+	for (count = 0; count < len; count++) {
+		if (jcore_spi_wait(ctrl_reg))
+			break;
+
+		writel(tx ? *tx++ : 0, data_reg);
+		writel(xmit, ctrl_reg);
+
+		if (jcore_spi_wait(ctrl_reg))
+			break;
+
+		if (rx)
+			*rx++ = readl(data_reg);
+	}
+
+	spi_finalize_current_transfer(master);
+
+	if (count < len)
+		return -EREMOTEIO;
+
+	return 0;
+}
+
+static int jcore_spi_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct jcore_spi *hw;
+	struct spi_master *master;
+	struct resource *res;
+	u32 clock_freq;
+	struct clk *clk;
+	int err = -ENODEV;
+
+	master = spi_alloc_master(&pdev->dev, sizeof(struct jcore_spi));
+	if (!master)
+		return err;
+
+	/* Setup the master state. */
+	master->num_chipselect = 3;
+	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
+	master->transfer_one = jcore_spi_txrx;
+	master->set_cs = jcore_spi_chipsel;
+	master->dev.of_node = node;
+	master->bus_num = pdev->id;
+
+	hw = spi_master_get_devdata(master);
+	hw->master = master;
+	platform_set_drvdata(pdev, hw);
+
+	/* Find and map our resources */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		goto exit_busy;
+	if (!devm_request_mem_region(&pdev->dev, res->start,
+				     resource_size(res), pdev->name))
+		goto exit_busy;
+	hw->base = devm_ioremap_nocache(&pdev->dev, res->start,
+					resource_size(res));
+	if (!hw->base)
+		goto exit_busy;
+
+	/*
+	 * The SPI clock rate controlled via a configurable clock divider
+	 * which is applied to the reference clock. A 50 MHz reference is
+	 * most suitable for obtaining standard SPI clock rates, but some
+	 * designs may have a different reference clock, and the DT must
+	 * make the driver aware so that it can properly program the
+	 * requested rate. If the clock is omitted, 50 MHz is assumed.
+	 */
+	clock_freq = 50000000;
+	clk = devm_clk_get(&pdev->dev, "ref_clk");
+	if (!IS_ERR_OR_NULL(clk)) {
+		if (clk_enable(clk) == 0)
+			clock_freq = clk_get_rate(clk);
+		else
+			dev_warn(&pdev->dev, "could not enable ref_clk\n");
+	}
+	hw->clock_freq = clock_freq;
+
+	/* Initialize all CS bits to high. */
+	hw->cs_reg = JCORE_SPI_CTRL_CS_BITS;
+	jcore_spi_baudrate(hw, 400000);
+
+	/* Register our spi controller */
+	err = devm_spi_register_master(&pdev->dev, master);
+	if (err)
+		goto exit;
+
+	return 0;
+
+exit_busy:
+	err = -EBUSY;
+exit:
+	spi_master_put(master);
+	return err;
+}
+
+static const struct of_device_id jcore_spi_of_match[] = {
+	{ .compatible = "jcore,spi2" },
+	{},
+};
+
+static struct platform_driver jcore_spi_driver = {
+	.probe = jcore_spi_probe,
+	.driver = {
+		.name = DRV_NAME,
+		.of_match_table = jcore_spi_of_match,
+	},
+};
+
+module_platform_driver(jcore_spi_driver);
+
+MODULE_DESCRIPTION("J-Core SPI driver");
+MODULE_AUTHOR("Rich Felker <dalias@libc.org>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:" DRV_NAME);
diff --git a/drivers/spi/spi-loopback-test.c b/drivers/spi/spi-loopback-test.c
index faa1b133b0f1..50e620f4e8fe 100644
--- a/drivers/spi/spi-loopback-test.c
+++ b/drivers/spi/spi-loopback-test.c
@@ -405,7 +405,7 @@ struct rx_ranges {
 	u8 *end;
 };
 
-int rx_ranges_cmp(void *priv, struct list_head *a, struct list_head *b)
+static int rx_ranges_cmp(void *priv, struct list_head *a, struct list_head *b)
 {
 	struct rx_ranges *rx_a = list_entry(a, struct rx_ranges, list);
 	struct rx_ranges *rx_b = list_entry(b, struct rx_ranges, list);
diff --git a/drivers/spi/spi-meson-spifc.c b/drivers/spi/spi-meson-spifc.c
index 2465259f6241..616566e793c6 100644
--- a/drivers/spi/spi-meson-spifc.c
+++ b/drivers/spi/spi-meson-spifc.c
@@ -442,6 +442,7 @@ static const struct dev_pm_ops meson_spifc_pm_ops = {
 
 static const struct of_device_id meson_spifc_dt_match[] = {
 	{ .compatible = "amlogic,meson6-spifc", },
+	{ .compatible = "amlogic,meson-gxbb-spifc", },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, meson_spifc_dt_match);
diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c
index 0be89e052428..899d7a8f0889 100644
--- a/drivers/spi/spi-mt65xx.c
+++ b/drivers/spi/spi-mt65xx.c
@@ -685,7 +685,6 @@ static int mtk_spi_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 
 	mtk_spi_reset(mdata);
-	spi_master_put(master);
 
 	return 0;
 }
diff --git a/drivers/spi/spi-pic32-sqi.c b/drivers/spi/spi-pic32-sqi.c
index c41abddab318..bd1c6b53283f 100644
--- a/drivers/spi/spi-pic32-sqi.c
+++ b/drivers/spi/spi-pic32-sqi.c
@@ -253,15 +253,13 @@ static struct ring_desc *ring_desc_get(struct pic32_sqi *sqi)
 		return NULL;
 
 	rdesc = list_first_entry(&sqi->bd_list_free, struct ring_desc, list);
-	list_del(&rdesc->list);
-	list_add_tail(&rdesc->list, &sqi->bd_list_used);
+	list_move_tail(&rdesc->list, &sqi->bd_list_used);
 	return rdesc;
 }
 
 static void ring_desc_put(struct pic32_sqi *sqi, struct ring_desc *rdesc)
 {
-	list_del(&rdesc->list);
-	list_add(&rdesc->list, &sqi->bd_list_free);
+	list_move(&rdesc->list, &sqi->bd_list_free);
 }
 
 static int pic32_sqi_one_transfer(struct pic32_sqi *sqi,
diff --git a/drivers/spi/spi-pxa2xx-dma.c b/drivers/spi/spi-pxa2xx-dma.c
index db3ae1dd829e..04f3eecf5cf3 100644
--- a/drivers/spi/spi-pxa2xx-dma.c
+++ b/drivers/spi/spi-pxa2xx-dma.c
@@ -23,7 +23,7 @@
 static void pxa2xx_spi_dma_transfer_complete(struct driver_data *drv_data,
 					     bool error)
 {
-	struct spi_message *msg = drv_data->cur_msg;
+	struct spi_message *msg = drv_data->master->cur_msg;
 
 	/*
 	 * It is possible that one CPU is handling ROR interrupt and other
@@ -76,7 +76,8 @@ static struct dma_async_tx_descriptor *
 pxa2xx_spi_dma_prepare_one(struct driver_data *drv_data,
 			   enum dma_transfer_direction dir)
 {
-	struct chip_data *chip = drv_data->cur_chip;
+	struct chip_data *chip =
+		spi_get_ctldata(drv_data->master->cur_msg->spi);
 	struct spi_transfer *xfer = drv_data->cur_transfer;
 	enum dma_slave_buswidth width;
 	struct dma_slave_config cfg;
@@ -146,7 +147,7 @@ irqreturn_t pxa2xx_spi_dma_transfer(struct driver_data *drv_data)
 int pxa2xx_spi_dma_prepare(struct driver_data *drv_data, u32 dma_burst)
 {
 	struct dma_async_tx_descriptor *tx_desc, *rx_desc;
-	int err = 0;
+	int err;
 
 	tx_desc = pxa2xx_spi_dma_prepare_one(drv_data, DMA_MEM_TO_DEV);
 	if (!tx_desc) {
diff --git a/drivers/spi/spi-pxa2xx-pci.c b/drivers/spi/spi-pxa2xx-pci.c
index f3df522db93b..58d2d48e16a5 100644
--- a/drivers/spi/spi-pxa2xx-pci.c
+++ b/drivers/spi/spi-pxa2xx-pci.c
@@ -214,6 +214,7 @@ static int pxa2xx_spi_pci_probe(struct pci_dev *dev,
 		return PTR_ERR(ssp->clk);
 
 	memset(&pi, 0, sizeof(pi));
+	pi.fwnode = dev->dev.fwnode;
 	pi.parent = &dev->dev;
 	pi.name = "pxa2xx-spi";
 	pi.id = ssp->port_id;
diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c
index 87150a1049bd..dd7b5b47291d 100644
--- a/drivers/spi/spi-pxa2xx.c
+++ b/drivers/spi/spi-pxa2xx.c
@@ -28,6 +28,7 @@
 #include <linux/spi/spi.h>
 #include <linux/delay.h>
 #include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/slab.h>
 #include <linux/clk.h>
 #include <linux/pm_runtime.h>
@@ -62,6 +63,13 @@ MODULE_ALIAS("platform:pxa2xx-spi");
 				| QUARK_X1000_SSCR1_TFT		\
 				| SSCR1_SPH | SSCR1_SPO | SSCR1_LBM)
 
+#define CE4100_SSCR1_CHANGE_MASK (SSCR1_TTELP | SSCR1_TTE | SSCR1_SCFR \
+				| SSCR1_ECRA | SSCR1_ECRB | SSCR1_SCLKDIR \
+				| SSCR1_SFRMDIR | SSCR1_RWOT | SSCR1_TRAIL \
+				| SSCR1_IFS | SSCR1_STRF | SSCR1_EFWR \
+				| CE4100_SSCR1_RFT | CE4100_SSCR1_TFT | SSCR1_MWDS \
+				| SSCR1_SPH | SSCR1_SPO | SSCR1_LBM)
+
 #define LPSS_GENERAL_REG_RXTO_HOLDOFF_DISABLE	BIT(24)
 #define LPSS_CS_CONTROL_SW_MODE			BIT(0)
 #define LPSS_CS_CONTROL_CS_HIGH			BIT(1)
@@ -175,6 +183,8 @@ static u32 pxa2xx_spi_get_ssrc1_change_mask(const struct driver_data *drv_data)
 	switch (drv_data->ssp_type) {
 	case QUARK_X1000_SSP:
 		return QUARK_X1000_SSCR1_CHANGE_MASK;
+	case CE4100_SSP:
+		return CE4100_SSCR1_CHANGE_MASK;
 	default:
 		return SSCR1_CHANGE_MASK;
 	}
@@ -186,6 +196,8 @@ pxa2xx_spi_get_rx_default_thre(const struct driver_data *drv_data)
 	switch (drv_data->ssp_type) {
 	case QUARK_X1000_SSP:
 		return RX_THRESH_QUARK_X1000_DFLT;
+	case CE4100_SSP:
+		return RX_THRESH_CE4100_DFLT;
 	default:
 		return RX_THRESH_DFLT;
 	}
@@ -199,6 +211,9 @@ static bool pxa2xx_spi_txfifo_full(const struct driver_data *drv_data)
 	case QUARK_X1000_SSP:
 		mask = QUARK_X1000_SSSR_TFL_MASK;
 		break;
+	case CE4100_SSP:
+		mask = CE4100_SSSR_TFL_MASK;
+		break;
 	default:
 		mask = SSSR_TFL_MASK;
 		break;
@@ -216,6 +231,9 @@ static void pxa2xx_spi_clear_rx_thre(const struct driver_data *drv_data,
 	case QUARK_X1000_SSP:
 		mask = QUARK_X1000_SSCR1_RFT;
 		break;
+	case CE4100_SSP:
+		mask = CE4100_SSCR1_RFT;
+		break;
 	default:
 		mask = SSCR1_RFT;
 		break;
@@ -230,6 +248,9 @@ static void pxa2xx_spi_set_rx_thre(const struct driver_data *drv_data,
 	case QUARK_X1000_SSP:
 		*sccr1_reg |= QUARK_X1000_SSCR1_RxTresh(threshold);
 		break;
+	case CE4100_SSP:
+		*sccr1_reg |= CE4100_SSCR1_RxTresh(threshold);
+		break;
 	default:
 		*sccr1_reg |= SSCR1_RxTresh(threshold);
 		break;
@@ -316,7 +337,7 @@ static void lpss_ssp_select_cs(struct driver_data *drv_data,
 
 	value = __lpss_ssp_read_priv(drv_data, config->reg_cs_ctrl);
 
-	cs = drv_data->cur_msg->spi->chip_select;
+	cs = drv_data->master->cur_msg->spi->chip_select;
 	cs <<= config->cs_sel_shift;
 	if (cs != (value & config->cs_sel_mask)) {
 		/*
@@ -355,10 +376,11 @@ static void lpss_ssp_cs_control(struct driver_data *drv_data, bool enable)
 
 static void cs_assert(struct driver_data *drv_data)
 {
-	struct chip_data *chip = drv_data->cur_chip;
+	struct chip_data *chip =
+		spi_get_ctldata(drv_data->master->cur_msg->spi);
 
 	if (drv_data->ssp_type == CE4100_SSP) {
-		pxa2xx_spi_write(drv_data, SSSR, drv_data->cur_chip->frm);
+		pxa2xx_spi_write(drv_data, SSSR, chip->frm);
 		return;
 	}
 
@@ -378,7 +400,8 @@ static void cs_assert(struct driver_data *drv_data)
 
 static void cs_deassert(struct driver_data *drv_data)
 {
-	struct chip_data *chip = drv_data->cur_chip;
+	struct chip_data *chip =
+		spi_get_ctldata(drv_data->master->cur_msg->spi);
 
 	if (drv_data->ssp_type == CE4100_SSP)
 		return;
@@ -508,7 +531,7 @@ static int u32_reader(struct driver_data *drv_data)
 
 void *pxa2xx_spi_next_transfer(struct driver_data *drv_data)
 {
-	struct spi_message *msg = drv_data->cur_msg;
+	struct spi_message *msg = drv_data->master->cur_msg;
 	struct spi_transfer *trans = drv_data->cur_transfer;
 
 	/* Move to next transfer */
@@ -529,8 +552,7 @@ static void giveback(struct driver_data *drv_data)
 	struct spi_message *msg;
 	unsigned long timeout;
 
-	msg = drv_data->cur_msg;
-	drv_data->cur_msg = NULL;
+	msg = drv_data->master->cur_msg;
 	drv_data->cur_transfer = NULL;
 
 	last_transfer = list_last_entry(&msg->transfers, struct spi_transfer,
@@ -575,13 +597,13 @@ static void giveback(struct driver_data *drv_data)
 			cs_deassert(drv_data);
 	}
 
-	drv_data->cur_chip = NULL;
 	spi_finalize_current_message(drv_data->master);
 }
 
 static void reset_sccr1(struct driver_data *drv_data)
 {
-	struct chip_data *chip = drv_data->cur_chip;
+	struct chip_data *chip =
+		spi_get_ctldata(drv_data->master->cur_msg->spi);
 	u32 sccr1_reg;
 
 	sccr1_reg = pxa2xx_spi_read(drv_data, SSCR1) & ~drv_data->int_cr1;
@@ -589,6 +611,9 @@ static void reset_sccr1(struct driver_data *drv_data)
 	case QUARK_X1000_SSP:
 		sccr1_reg &= ~QUARK_X1000_SSCR1_RFT;
 		break;
+	case CE4100_SSP:
+		sccr1_reg &= ~CE4100_SSCR1_RFT;
+		break;
 	default:
 		sccr1_reg &= ~SSCR1_RFT;
 		break;
@@ -610,7 +635,7 @@ static void int_error_stop(struct driver_data *drv_data, const char* msg)
 
 	dev_err(&drv_data->pdev->dev, "%s\n", msg);
 
-	drv_data->cur_msg->state = ERROR_STATE;
+	drv_data->master->cur_msg->state = ERROR_STATE;
 	tasklet_schedule(&drv_data->pump_transfers);
 }
 
@@ -623,7 +648,7 @@ static void int_transfer_complete(struct driver_data *drv_data)
 		pxa2xx_spi_write(drv_data, SSTO, 0);
 
 	/* Update total byte transferred return count actual bytes read */
-	drv_data->cur_msg->actual_length += drv_data->len -
+	drv_data->master->cur_msg->actual_length += drv_data->len -
 				(drv_data->rx_end - drv_data->rx);
 
 	/* Transfer delays and chip select release are
@@ -631,7 +656,7 @@ static void int_transfer_complete(struct driver_data *drv_data)
 	 */
 
 	/* Move to next transfer */
-	drv_data->cur_msg->state = pxa2xx_spi_next_transfer(drv_data);
+	drv_data->master->cur_msg->state = pxa2xx_spi_next_transfer(drv_data);
 
 	/* Schedule transfer tasklet */
 	tasklet_schedule(&drv_data->pump_transfers);
@@ -746,7 +771,7 @@ static irqreturn_t ssp_int(int irq, void *dev_id)
 	if (!(status & mask))
 		return IRQ_NONE;
 
-	if (!drv_data->cur_msg) {
+	if (!drv_data->master->cur_msg) {
 
 		pxa2xx_spi_write(drv_data, SSCR0,
 				 pxa2xx_spi_read(drv_data, SSCR0)
@@ -905,7 +930,8 @@ static unsigned int ssp_get_clk_div(struct driver_data *drv_data, int rate)
 static unsigned int pxa2xx_ssp_get_clk_div(struct driver_data *drv_data,
 					   int rate)
 {
-	struct chip_data *chip = drv_data->cur_chip;
+	struct chip_data *chip =
+		spi_get_ctldata(drv_data->master->cur_msg->spi);
 	unsigned int clk_div;
 
 	switch (drv_data->ssp_type) {
@@ -934,25 +960,23 @@ static void pump_transfers(unsigned long data)
 {
 	struct driver_data *drv_data = (struct driver_data *)data;
 	struct spi_master *master = drv_data->master;
-	struct spi_message *message = NULL;
-	struct spi_transfer *transfer = NULL;
-	struct spi_transfer *previous = NULL;
-	struct chip_data *chip = NULL;
-	u32 clk_div = 0;
-	u8 bits = 0;
-	u32 speed = 0;
+	struct spi_message *message = master->cur_msg;
+	struct chip_data *chip = spi_get_ctldata(message->spi);
+	u32 dma_thresh = chip->dma_threshold;
+	u32 dma_burst = chip->dma_burst_size;
+	u32 change_mask = pxa2xx_spi_get_ssrc1_change_mask(drv_data);
+	struct spi_transfer *transfer;
+	struct spi_transfer *previous;
+	u32 clk_div;
+	u8 bits;
+	u32 speed;
 	u32 cr0;
 	u32 cr1;
-	u32 dma_thresh = drv_data->cur_chip->dma_threshold;
-	u32 dma_burst = drv_data->cur_chip->dma_burst_size;
-	u32 change_mask = pxa2xx_spi_get_ssrc1_change_mask(drv_data);
 	int err;
 	int dma_mapped;
 
 	/* Get current state information */
-	message = drv_data->cur_msg;
 	transfer = drv_data->cur_transfer;
-	chip = drv_data->cur_chip;
 
 	/* Handle for abort */
 	if (message->state == ERROR_STATE) {
@@ -1146,17 +1170,12 @@ static int pxa2xx_spi_transfer_one_message(struct spi_master *master,
 {
 	struct driver_data *drv_data = spi_master_get_devdata(master);
 
-	drv_data->cur_msg = msg;
 	/* Initial message state*/
-	drv_data->cur_msg->state = START_STATE;
-	drv_data->cur_transfer = list_entry(drv_data->cur_msg->transfers.next,
+	msg->state = START_STATE;
+	drv_data->cur_transfer = list_entry(msg->transfers.next,
 						struct spi_transfer,
 						transfer_list);
 
-	/* prepare to setup the SSP, in pump_transfers, using the per
-	 * chip configuration */
-	drv_data->cur_chip = spi_get_ctldata(drv_data->cur_msg->spi);
-
 	/* Mark as busy and launch transfers */
 	tasklet_schedule(&drv_data->pump_transfers);
 	return 0;
@@ -1176,9 +1195,26 @@ static int pxa2xx_spi_unprepare_transfer(struct spi_master *master)
 static int setup_cs(struct spi_device *spi, struct chip_data *chip,
 		    struct pxa2xx_spi_chip *chip_info)
 {
+	struct driver_data *drv_data = spi_master_get_devdata(spi->master);
 	int err = 0;
 
-	if (chip == NULL || chip_info == NULL)
+	if (chip == NULL)
+		return 0;
+
+	if (drv_data->cs_gpiods) {
+		struct gpio_desc *gpiod;
+
+		gpiod = drv_data->cs_gpiods[spi->chip_select];
+		if (gpiod) {
+			chip->gpio_cs = desc_to_gpio(gpiod);
+			chip->gpio_cs_inverted = spi->mode & SPI_CS_HIGH;
+			gpiod_set_value(gpiod, chip->gpio_cs_inverted);
+		}
+
+		return 0;
+	}
+
+	if (chip_info == NULL)
 		return 0;
 
 	/* NOTE: setup() can be called multiple times, possibly with
@@ -1213,7 +1249,7 @@ static int setup_cs(struct spi_device *spi, struct chip_data *chip,
 
 static int setup(struct spi_device *spi)
 {
-	struct pxa2xx_spi_chip *chip_info = NULL;
+	struct pxa2xx_spi_chip *chip_info;
 	struct chip_data *chip;
 	const struct lpss_config *config;
 	struct driver_data *drv_data = spi_master_get_devdata(spi->master);
@@ -1225,6 +1261,11 @@ static int setup(struct spi_device *spi)
 		tx_hi_thres = 0;
 		rx_thres = RX_THRESH_QUARK_X1000_DFLT;
 		break;
+	case CE4100_SSP:
+		tx_thres = TX_THRESH_CE4100_DFLT;
+		tx_hi_thres = 0;
+		rx_thres = RX_THRESH_CE4100_DFLT;
+		break;
 	case LPSS_LPT_SSP:
 	case LPSS_BYT_SSP:
 	case LPSS_BSW_SSP:
@@ -1309,6 +1350,10 @@ static int setup(struct spi_device *spi)
 				   | (QUARK_X1000_SSCR1_TxTresh(tx_thres)
 				   & QUARK_X1000_SSCR1_TFT);
 		break;
+	case CE4100_SSP:
+		chip->threshold = (CE4100_SSCR1_RxTresh(rx_thres) & CE4100_SSCR1_RFT) |
+			(CE4100_SSCR1_TxTresh(tx_thres) & CE4100_SSCR1_TFT);
+		break;
 	default:
 		chip->threshold = (SSCR1_RxTresh(rx_thres) & SSCR1_RFT) |
 			(SSCR1_TxTresh(tx_thres) & SSCR1_TFT);
@@ -1352,7 +1397,8 @@ static void cleanup(struct spi_device *spi)
 	if (!chip)
 		return;
 
-	if (drv_data->ssp_type != CE4100_SSP && gpio_is_valid(chip->gpio_cs))
+	if (drv_data->ssp_type != CE4100_SSP && !drv_data->cs_gpiods &&
+	    gpio_is_valid(chip->gpio_cs))
 		gpio_free(chip->gpio_cs);
 
 	kfree(chip);
@@ -1530,7 +1576,7 @@ static int pxa2xx_spi_probe(struct platform_device *pdev)
 	struct driver_data *drv_data;
 	struct ssp_device *ssp;
 	const struct lpss_config *config;
-	int status;
+	int status, count;
 	u32 tmp;
 
 	platform_info = dev_get_platdata(dev);
@@ -1630,15 +1676,20 @@ static int pxa2xx_spi_probe(struct platform_device *pdev)
 	pxa2xx_spi_write(drv_data, SSCR0, 0);
 	switch (drv_data->ssp_type) {
 	case QUARK_X1000_SSP:
-		tmp = QUARK_X1000_SSCR1_RxTresh(RX_THRESH_QUARK_X1000_DFLT)
-		      | QUARK_X1000_SSCR1_TxTresh(TX_THRESH_QUARK_X1000_DFLT);
+		tmp = QUARK_X1000_SSCR1_RxTresh(RX_THRESH_QUARK_X1000_DFLT) |
+		      QUARK_X1000_SSCR1_TxTresh(TX_THRESH_QUARK_X1000_DFLT);
 		pxa2xx_spi_write(drv_data, SSCR1, tmp);
 
 		/* using the Motorola SPI protocol and use 8 bit frame */
-		pxa2xx_spi_write(drv_data, SSCR0,
-				 QUARK_X1000_SSCR0_Motorola
-				 | QUARK_X1000_SSCR0_DataSize(8));
+		tmp = QUARK_X1000_SSCR0_Motorola | QUARK_X1000_SSCR0_DataSize(8);
+		pxa2xx_spi_write(drv_data, SSCR0, tmp);
 		break;
+	case CE4100_SSP:
+		tmp = CE4100_SSCR1_RxTresh(RX_THRESH_CE4100_DFLT) |
+		      CE4100_SSCR1_TxTresh(TX_THRESH_CE4100_DFLT);
+		pxa2xx_spi_write(drv_data, SSCR1, tmp);
+		tmp = SSCR0_SCR(2) | SSCR0_Motorola | SSCR0_DataSize(8);
+		pxa2xx_spi_write(drv_data, SSCR0, tmp);
 	default:
 		tmp = SSCR1_RxTresh(RX_THRESH_DFLT) |
 		      SSCR1_TxTresh(TX_THRESH_DFLT);
@@ -1669,6 +1720,39 @@ static int pxa2xx_spi_probe(struct platform_device *pdev)
 	}
 	master->num_chipselect = platform_info->num_chipselect;
 
+	count = gpiod_count(&pdev->dev, "cs");
+	if (count > 0) {
+		int i;
+
+		master->num_chipselect = max_t(int, count,
+			master->num_chipselect);
+
+		drv_data->cs_gpiods = devm_kcalloc(&pdev->dev,
+			master->num_chipselect, sizeof(struct gpio_desc *),
+			GFP_KERNEL);
+		if (!drv_data->cs_gpiods) {
+			status = -ENOMEM;
+			goto out_error_clock_enabled;
+		}
+
+		for (i = 0; i < master->num_chipselect; i++) {
+			struct gpio_desc *gpiod;
+
+			gpiod = devm_gpiod_get_index(dev, "cs", i,
+						     GPIOD_OUT_HIGH);
+			if (IS_ERR(gpiod)) {
+				/* Means use native chip select */
+				if (PTR_ERR(gpiod) == -ENOENT)
+					continue;
+
+				status = (int)PTR_ERR(gpiod);
+				goto out_error_clock_enabled;
+			} else {
+				drv_data->cs_gpiods[i] = gpiod;
+			}
+		}
+	}
+
 	tasklet_init(&drv_data->pump_transfers, pump_transfers,
 		     (unsigned long)drv_data);
 
@@ -1742,7 +1826,7 @@ static int pxa2xx_spi_suspend(struct device *dev)
 {
 	struct driver_data *drv_data = dev_get_drvdata(dev);
 	struct ssp_device *ssp = drv_data->ssp;
-	int status = 0;
+	int status;
 
 	status = spi_master_suspend(drv_data->master);
 	if (status != 0)
@@ -1759,7 +1843,7 @@ static int pxa2xx_spi_resume(struct device *dev)
 {
 	struct driver_data *drv_data = dev_get_drvdata(dev);
 	struct ssp_device *ssp = drv_data->ssp;
-	int status = 0;
+	int status;
 
 	/* Enable the SSP clock */
 	if (!pm_runtime_suspended(dev))
diff --git a/drivers/spi/spi-pxa2xx.h b/drivers/spi/spi-pxa2xx.h
index d217ad55cc12..ce31b8199bb3 100644
--- a/drivers/spi/spi-pxa2xx.h
+++ b/drivers/spi/spi-pxa2xx.h
@@ -53,9 +53,7 @@ struct driver_data {
 	atomic_t dma_running;
 
 	/* Current message transfer state info */
-	struct spi_message *cur_msg;
 	struct spi_transfer *cur_transfer;
-	struct chip_data *cur_chip;
 	size_t len;
 	void *tx;
 	void *tx_end;
@@ -68,6 +66,9 @@ struct driver_data {
 	void (*cs_control)(u32 command);
 
 	void __iomem *lpss_base;
+
+	/* GPIOs for chip selects */
+	struct gpio_desc **cs_gpiods;
 };
 
 struct chip_data {
diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c
index c338ef1136f6..1bfa889b8427 100644
--- a/drivers/spi/spi-qup.c
+++ b/drivers/spi/spi-qup.c
@@ -982,8 +982,10 @@ static int spi_qup_suspend(struct device *device)
 	if (ret)
 		return ret;
 
-	clk_disable_unprepare(controller->cclk);
-	clk_disable_unprepare(controller->iclk);
+	if (!pm_runtime_suspended(device)) {
+		clk_disable_unprepare(controller->cclk);
+		clk_disable_unprepare(controller->iclk);
+	}
 	return 0;
 }
 
@@ -1030,7 +1032,6 @@ static int spi_qup_remove(struct platform_device *pdev)
 
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-	spi_master_put(master);
 
 	return 0;
 }
diff --git a/drivers/spi/spi-rspi.c b/drivers/spi/spi-rspi.c
index 818843336932..a816f07e168e 100644
--- a/drivers/spi/spi-rspi.c
+++ b/drivers/spi/spi-rspi.c
@@ -295,14 +295,24 @@ static int rspi_set_config_register(struct rspi_data *rspi, int access_size)
 static int rspi_rz_set_config_register(struct rspi_data *rspi, int access_size)
 {
 	int spbr;
+	int div = 0;
+	unsigned long clksrc;
 
 	/* Sets output mode, MOSI signal, and (optionally) loopback */
 	rspi_write8(rspi, rspi->sppcr, RSPI_SPPCR);
 
+	clksrc = clk_get_rate(rspi->clk);
+	while (div < 3) {
+		if (rspi->max_speed_hz >= clksrc/4) /* 4=(CLK/2)/2 */
+			break;
+		div++;
+		clksrc /= 2;
+	}
+
 	/* Sets transfer bit rate */
-	spbr = DIV_ROUND_UP(clk_get_rate(rspi->clk),
-			    2 * rspi->max_speed_hz) - 1;
+	spbr = DIV_ROUND_UP(clksrc, 2 * rspi->max_speed_hz) - 1;
 	rspi_write8(rspi, clamp(spbr, 0, 255), RSPI_SPBR);
+	rspi->spcmd |= div << 2;
 
 	/* Disable dummy transmission, set byte access */
 	rspi_write8(rspi, SPDCR_SPLBYTE, RSPI_SPDCR);
diff --git a/drivers/spi/spi-sc18is602.c b/drivers/spi/spi-sc18is602.c
index 36af4d48a700..f63714ffb62f 100644
--- a/drivers/spi/spi-sc18is602.c
+++ b/drivers/spi/spi-sc18is602.c
@@ -23,6 +23,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/of.h>
 #include <linux/platform_data/sc18is602.h>
+#include <linux/gpio/consumer.h>
 
 enum chips { sc18is602, sc18is602b, sc18is603 };
 
@@ -50,6 +51,8 @@ struct sc18is602 {
 	u8			buffer[SC18IS602_BUFSIZ + 1];
 	int			tlen;	/* Data queued for tx in buffer */
 	int			rindex;	/* Receive data index in buffer */
+
+	struct gpio_desc	*reset;
 };
 
 static int sc18is602_wait_ready(struct sc18is602 *hw, int len)
@@ -257,6 +260,12 @@ static int sc18is602_probe(struct i2c_client *client,
 	hw = spi_master_get_devdata(master);
 	i2c_set_clientdata(client, hw);
 
+	/* assert reset and then release */
+	hw->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(hw->reset))
+		return PTR_ERR(hw->reset);
+	gpiod_set_value_cansleep(hw->reset, 0);
+
 	hw->master = master;
 	hw->client = client;
 	hw->dev = dev;
diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c
index 0f83ad1d5a58..1de3a772eb7d 100644
--- a/drivers/spi/spi-sh-msiof.c
+++ b/drivers/spi/spi-sh-msiof.c
@@ -262,6 +262,9 @@ static void sh_msiof_spi_set_clk_regs(struct sh_msiof_spi_priv *p,
 
 	for (k = 0; k < ARRAY_SIZE(sh_msiof_spi_div_table); k++) {
 		brps = DIV_ROUND_UP(div, sh_msiof_spi_div_table[k].div);
+		/* SCR_BRDV_DIV_1 is valid only if BRPS is x 1/1 or x 1/2 */
+		if (sh_msiof_spi_div_table[k].div == 1 && brps > 2)
+			continue;
 		if (brps <= 32) /* max of brdv is 32 */
 			break;
 	}
diff --git a/drivers/spi/spi-st-ssc4.c b/drivers/spi/spi-st-ssc4.c
index a56eca0e95a6..e54b59638458 100644
--- a/drivers/spi/spi-st-ssc4.c
+++ b/drivers/spi/spi-st-ssc4.c
@@ -175,10 +175,7 @@ static int spi_st_transfer_one(struct spi_master *master,
 
 static void spi_st_cleanup(struct spi_device *spi)
 {
-	int cs = spi->cs_gpio;
-
-	if (gpio_is_valid(cs))
-		devm_gpio_free(&spi->dev, cs);
+	gpio_free(spi->cs_gpio);
 }
 
 /* the spi->mode bits understood by this driver: */
@@ -201,14 +198,15 @@ static int spi_st_setup(struct spi_device *spi)
 		return -EINVAL;
 	}
 
-	if (devm_gpio_request(&spi->dev, cs, dev_name(&spi->dev))) {
+	ret = gpio_request(cs, dev_name(&spi->dev));
+	if (ret) {
 		dev_err(&spi->dev, "could not request gpio:%d\n", cs);
-		return -EINVAL;
+		return ret;
 	}
 
 	ret = gpio_direction_output(cs, spi->mode & SPI_CS_HIGH);
 	if (ret)
-		return ret;
+		goto out_free_gpio;
 
 	spi_st_clk = clk_get_rate(spi_st->clk);
 
@@ -217,7 +215,8 @@ static int spi_st_setup(struct spi_device *spi)
 	if (sscbrg < 0x07 || sscbrg > BIT(16)) {
 		dev_err(&spi->dev,
 			"baudrate %d outside valid range %d\n", sscbrg, hz);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out_free_gpio;
 	}
 
 	spi_st->baud = spi_st_clk / (2 * sscbrg);
@@ -266,6 +265,10 @@ static int spi_st_setup(struct spi_device *spi)
 	 readl_relaxed(spi_st->base + SSC_RBUF);
 
 	 return 0;
+
+out_free_gpio:
+	gpio_free(cs);
+	return ret;
 }
 
 /* Interrupt fired when TX shift register becomes empty */
diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c
index ac0b072815a3..caeac66a3977 100644
--- a/drivers/spi/spi-ti-qspi.c
+++ b/drivers/spi/spi-ti-qspi.c
@@ -41,6 +41,8 @@ struct ti_qspi_regs {
 };
 
 struct ti_qspi {
+	struct completion	transfer_complete;
+
 	/* list synchronization */
 	struct mutex            list_lock;
 
@@ -54,6 +56,9 @@ struct ti_qspi {
 
 	struct ti_qspi_regs     ctx_reg;
 
+	dma_addr_t		mmap_phys_base;
+	struct dma_chan		*rx_chan;
+
 	u32 spi_max_frequency;
 	u32 cmd;
 	u32 dc;
@@ -379,6 +384,72 @@ static int qspi_transfer_msg(struct ti_qspi *qspi, struct spi_transfer *t,
 	return 0;
 }
 
+static void ti_qspi_dma_callback(void *param)
+{
+	struct ti_qspi *qspi = param;
+
+	complete(&qspi->transfer_complete);
+}
+
+static int ti_qspi_dma_xfer(struct ti_qspi *qspi, dma_addr_t dma_dst,
+			    dma_addr_t dma_src, size_t len)
+{
+	struct dma_chan *chan = qspi->rx_chan;
+	struct dma_device *dma_dev = chan->device;
+	dma_cookie_t cookie;
+	enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
+	struct dma_async_tx_descriptor *tx;
+	int ret;
+
+	tx = dma_dev->device_prep_dma_memcpy(chan, dma_dst, dma_src,
+					     len, flags);
+	if (!tx) {
+		dev_err(qspi->dev, "device_prep_dma_memcpy error\n");
+		return -EIO;
+	}
+
+	tx->callback = ti_qspi_dma_callback;
+	tx->callback_param = qspi;
+	cookie = tx->tx_submit(tx);
+
+	ret = dma_submit_error(cookie);
+	if (ret) {
+		dev_err(qspi->dev, "dma_submit_error %d\n", cookie);
+		return -EIO;
+	}
+
+	dma_async_issue_pending(chan);
+	ret = wait_for_completion_timeout(&qspi->transfer_complete,
+					  msecs_to_jiffies(len));
+	if (ret <= 0) {
+		dmaengine_terminate_sync(chan);
+		dev_err(qspi->dev, "DMA wait_for_completion_timeout\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int ti_qspi_dma_xfer_sg(struct ti_qspi *qspi, struct sg_table rx_sg,
+			       loff_t from)
+{
+	struct scatterlist *sg;
+	dma_addr_t dma_src = qspi->mmap_phys_base + from;
+	dma_addr_t dma_dst;
+	int i, len, ret;
+
+	for_each_sg(rx_sg.sgl, sg, rx_sg.nents, i) {
+		dma_dst = sg_dma_address(sg);
+		len = sg_dma_len(sg);
+		ret = ti_qspi_dma_xfer(qspi, dma_dst, dma_src, len);
+		if (ret)
+			return ret;
+		dma_src += len;
+	}
+
+	return 0;
+}
+
 static void ti_qspi_enable_memory_map(struct spi_device *spi)
 {
 	struct ti_qspi  *qspi = spi_master_get_devdata(spi->master);
@@ -426,7 +497,7 @@ static void ti_qspi_setup_mmap_read(struct spi_device *spi,
 		      QSPI_SPI_SETUP_REG(spi->chip_select));
 }
 
-static int ti_qspi_spi_flash_read(struct  spi_device *spi,
+static int ti_qspi_spi_flash_read(struct spi_device *spi,
 				  struct spi_flash_read_message *msg)
 {
 	struct ti_qspi *qspi = spi_master_get_devdata(spi->master);
@@ -437,9 +508,23 @@ static int ti_qspi_spi_flash_read(struct  spi_device *spi,
 	if (!qspi->mmap_enabled)
 		ti_qspi_enable_memory_map(spi);
 	ti_qspi_setup_mmap_read(spi, msg);
-	memcpy_fromio(msg->buf, qspi->mmap_base + msg->from, msg->len);
+
+	if (qspi->rx_chan) {
+		if (msg->cur_msg_mapped) {
+			ret = ti_qspi_dma_xfer_sg(qspi, msg->rx_sg, msg->from);
+			if (ret)
+				goto err_unlock;
+		} else {
+			dev_err(qspi->dev, "Invalid address for DMA\n");
+			ret = -EIO;
+			goto err_unlock;
+		}
+	} else {
+		memcpy_fromio(msg->buf, qspi->mmap_base + msg->from, msg->len);
+	}
 	msg->retlen = msg->len;
 
+err_unlock:
 	mutex_unlock(&qspi->list_lock);
 
 	return ret;
@@ -536,6 +621,7 @@ static int ti_qspi_probe(struct platform_device *pdev)
 	struct device_node *np = pdev->dev.of_node;
 	u32 max_freq;
 	int ret = 0, num_cs, irq;
+	dma_cap_mask_t mask;
 
 	master = spi_alloc_master(&pdev->dev, sizeof(*qspi));
 	if (!master)
@@ -550,6 +636,7 @@ static int ti_qspi_probe(struct platform_device *pdev)
 	master->dev.of_node = pdev->dev.of_node;
 	master->bits_per_word_mask = SPI_BPW_MASK(32) | SPI_BPW_MASK(16) |
 				     SPI_BPW_MASK(8);
+	master->spi_flash_read = ti_qspi_spi_flash_read;
 
 	if (!of_property_read_u32(np, "num-cs", &num_cs))
 		master->num_chipselect = num_cs;
@@ -592,17 +679,6 @@ static int ti_qspi_probe(struct platform_device *pdev)
 		goto free_master;
 	}
 
-	if (res_mmap) {
-		qspi->mmap_base = devm_ioremap_resource(&pdev->dev,
-							res_mmap);
-		master->spi_flash_read = ti_qspi_spi_flash_read;
-		if (IS_ERR(qspi->mmap_base)) {
-			dev_err(&pdev->dev,
-				"falling back to PIO mode\n");
-			master->spi_flash_read = NULL;
-		}
-	}
-	qspi->mmap_enabled = false;
 
 	if (of_property_read_bool(np, "syscon-chipselects")) {
 		qspi->ctrl_base =
@@ -633,11 +709,37 @@ static int ti_qspi_probe(struct platform_device *pdev)
 	if (!of_property_read_u32(np, "spi-max-frequency", &max_freq))
 		qspi->spi_max_frequency = max_freq;
 
-	ret = devm_spi_register_master(&pdev->dev, master);
-	if (ret)
-		goto free_master;
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_MEMCPY, mask);
 
-	return 0;
+	qspi->rx_chan = dma_request_chan_by_mask(&mask);
+	if (!qspi->rx_chan) {
+		dev_err(qspi->dev,
+			"No Rx DMA available, trying mmap mode\n");
+		ret = 0;
+		goto no_dma;
+	}
+	master->dma_rx = qspi->rx_chan;
+	init_completion(&qspi->transfer_complete);
+	if (res_mmap)
+		qspi->mmap_phys_base = (dma_addr_t)res_mmap->start;
+
+no_dma:
+	if (!qspi->rx_chan && res_mmap) {
+		qspi->mmap_base = devm_ioremap_resource(&pdev->dev, res_mmap);
+		if (IS_ERR(qspi->mmap_base)) {
+			dev_info(&pdev->dev,
+				 "mmap failed with error %ld using PIO mode\n",
+				 PTR_ERR(qspi->mmap_base));
+			qspi->mmap_base = NULL;
+			master->spi_flash_read = NULL;
+		}
+	}
+	qspi->mmap_enabled = false;
+
+	ret = devm_spi_register_master(&pdev->dev, master);
+	if (!ret)
+		return 0;
 
 free_master:
 	spi_master_put(master);
@@ -656,6 +758,9 @@ static int ti_qspi_remove(struct platform_device *pdev)
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
+	if (qspi->rx_chan)
+		dma_release_channel(qspi->rx_chan);
+
 	return 0;
 }
 
diff --git a/drivers/spi/spi-txx9.c b/drivers/spi/spi-txx9.c
index 7492ea346b43..51759d3fd45f 100644
--- a/drivers/spi/spi-txx9.c
+++ b/drivers/spi/spi-txx9.c
@@ -346,7 +346,7 @@ static int txx9spi_probe(struct platform_device *dev)
 		c->clk = NULL;
 		goto exit;
 	}
-	ret = clk_enable(c->clk);
+	ret = clk_prepare_enable(c->clk);
 	if (ret) {
 		c->clk = NULL;
 		goto exit;
@@ -395,7 +395,7 @@ static int txx9spi_probe(struct platform_device *dev)
 exit_busy:
 	ret = -EBUSY;
 exit:
-	clk_disable(c->clk);
+	clk_disable_unprepare(c->clk);
 	spi_master_put(master);
 	return ret;
 }
@@ -406,7 +406,7 @@ static int txx9spi_remove(struct platform_device *dev)
 	struct txx9spi *c = spi_master_get_devdata(master);
 
 	flush_work(&c->work);
-	clk_disable(c->clk);
+	clk_disable_unprepare(c->clk);
 	return 0;
 }
 
diff --git a/drivers/spi/spi-xlp.c b/drivers/spi/spi-xlp.c
index 8f04feca6ee3..4071a729eb2f 100644
--- a/drivers/spi/spi-xlp.c
+++ b/drivers/spi/spi-xlp.c
@@ -11,6 +11,7 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  */
+#include <linux/acpi.h>
 #include <linux/clk.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -405,8 +406,9 @@ static int xlp_spi_probe(struct platform_device *pdev)
 	clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(clk)) {
 		dev_err(&pdev->dev, "could not get spi clock\n");
-		return -ENODEV;
+		return PTR_ERR(clk);
 	}
+
 	xspi->spi_clk = clk_get_rate(clk);
 
 	master = spi_alloc_master(&pdev->dev, 0);
@@ -437,6 +439,14 @@ static int xlp_spi_probe(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id xlp_spi_acpi_match[] = {
+	{ "BRCM900D", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(acpi, xlp_spi_acpi_match);
+#endif
+
 static const struct of_device_id xlp_spi_dt_id[] = {
 	{ .compatible = "netlogic,xlp832-spi" },
 	{ },
@@ -447,6 +457,7 @@ static struct platform_driver xlp_spi_driver = {
 	.driver = {
 		.name	= "xlp-spi",
 		.of_match_table = xlp_spi_dt_id,
+		.acpi_match_table = ACPI_PTR(xlp_spi_acpi_match),
 	},
 };
 module_platform_driver(xlp_spi_driver);
diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
index 51ad42fad567..5787b723b593 100644
--- a/drivers/spi/spi.c
+++ b/drivers/spi/spi.c
@@ -37,6 +37,7 @@
 #include <linux/kthread.h>
 #include <linux/ioport.h>
 #include <linux/acpi.h>
+#include <linux/highmem.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/spi.h>
@@ -709,6 +710,13 @@ static int spi_map_buf(struct spi_master *master, struct device *dev,
 {
 	const bool vmalloced_buf = is_vmalloc_addr(buf);
 	unsigned int max_seg_size = dma_get_max_seg_size(dev);
+#ifdef CONFIG_HIGHMEM
+	const bool kmap_buf = ((unsigned long)buf >= PKMAP_BASE &&
+				(unsigned long)buf < (PKMAP_BASE +
+					(LAST_PKMAP * PAGE_SIZE)));
+#else
+	const bool kmap_buf = false;
+#endif
 	int desc_len;
 	int sgs;
 	struct page *vm_page;
@@ -716,7 +724,7 @@ static int spi_map_buf(struct spi_master *master, struct device *dev,
 	size_t min;
 	int i, ret;
 
-	if (vmalloced_buf) {
+	if (vmalloced_buf || kmap_buf) {
 		desc_len = min_t(int, max_seg_size, PAGE_SIZE);
 		sgs = DIV_ROUND_UP(len + offset_in_page(buf), desc_len);
 	} else if (virt_addr_valid(buf)) {
@@ -732,10 +740,13 @@ static int spi_map_buf(struct spi_master *master, struct device *dev,
 
 	for (i = 0; i < sgs; i++) {
 
-		if (vmalloced_buf) {
+		if (vmalloced_buf || kmap_buf) {
 			min = min_t(size_t,
 				    len, desc_len - offset_in_page(buf));
-			vm_page = vmalloc_to_page(buf);
+			if (vmalloced_buf)
+				vm_page = vmalloc_to_page(buf);
+			else
+				vm_page = kmap_to_page(buf);
 			if (!vm_page) {
 				sg_free_table(sgt);
 				return -ENOMEM;
@@ -960,7 +971,7 @@ static int spi_transfer_one_message(struct spi_master *master,
 	struct spi_transfer *xfer;
 	bool keep_cs = false;
 	int ret = 0;
-	unsigned long ms = 1;
+	unsigned long long ms = 1;
 	struct spi_statistics *statm = &master->statistics;
 	struct spi_statistics *stats = &msg->spi->statistics;
 
@@ -991,9 +1002,13 @@ static int spi_transfer_one_message(struct spi_master *master,
 
 			if (ret > 0) {
 				ret = 0;
-				ms = xfer->len * 8 * 1000 / xfer->speed_hz;
+				ms = 8LL * 1000LL * xfer->len;
+				do_div(ms, xfer->speed_hz);
 				ms += ms + 100; /* some tolerance */
 
+				if (ms > UINT_MAX)
+					ms = UINT_MAX;
+
 				ms = wait_for_completion_timeout(&master->xfer_completion,
 								 msecs_to_jiffies(ms));
 			}
@@ -1097,7 +1112,7 @@ static void __spi_pump_messages(struct spi_master *master, bool in_kthread)
 
 	/* If another context is idling the device then defer */
 	if (master->idling) {
-		queue_kthread_work(&master->kworker, &master->pump_messages);
+		kthread_queue_work(&master->kworker, &master->pump_messages);
 		spin_unlock_irqrestore(&master->queue_lock, flags);
 		return;
 	}
@@ -1111,7 +1126,7 @@ static void __spi_pump_messages(struct spi_master *master, bool in_kthread)
 
 		/* Only do teardown in the thread */
 		if (!in_kthread) {
-			queue_kthread_work(&master->kworker,
+			kthread_queue_work(&master->kworker,
 					   &master->pump_messages);
 			spin_unlock_irqrestore(&master->queue_lock, flags);
 			return;
@@ -1159,6 +1174,7 @@ static void __spi_pump_messages(struct spi_master *master, bool in_kthread)
 		if (ret < 0) {
 			dev_err(&master->dev, "Failed to power device: %d\n",
 				ret);
+			mutex_unlock(&master->io_mutex);
 			return;
 		}
 	}
@@ -1174,6 +1190,7 @@ static void __spi_pump_messages(struct spi_master *master, bool in_kthread)
 
 			if (master->auto_runtime_pm)
 				pm_runtime_put(master->dev.parent);
+			mutex_unlock(&master->io_mutex);
 			return;
 		}
 	}
@@ -1233,7 +1250,7 @@ static int spi_init_queue(struct spi_master *master)
 	master->running = false;
 	master->busy = false;
 
-	init_kthread_worker(&master->kworker);
+	kthread_init_worker(&master->kworker);
 	master->kworker_task = kthread_run(kthread_worker_fn,
 					   &master->kworker, "%s",
 					   dev_name(&master->dev));
@@ -1241,7 +1258,7 @@ static int spi_init_queue(struct spi_master *master)
 		dev_err(&master->dev, "failed to create message pump task\n");
 		return PTR_ERR(master->kworker_task);
 	}
-	init_kthread_work(&master->pump_messages, spi_pump_messages);
+	kthread_init_work(&master->pump_messages, spi_pump_messages);
 
 	/*
 	 * Master config will indicate if this controller should run the
@@ -1314,7 +1331,7 @@ void spi_finalize_current_message(struct spi_master *master)
 	spin_lock_irqsave(&master->queue_lock, flags);
 	master->cur_msg = NULL;
 	master->cur_msg_prepared = false;
-	queue_kthread_work(&master->kworker, &master->pump_messages);
+	kthread_queue_work(&master->kworker, &master->pump_messages);
 	spin_unlock_irqrestore(&master->queue_lock, flags);
 
 	trace_spi_message_done(mesg);
@@ -1340,7 +1357,7 @@ static int spi_start_queue(struct spi_master *master)
 	master->cur_msg = NULL;
 	spin_unlock_irqrestore(&master->queue_lock, flags);
 
-	queue_kthread_work(&master->kworker, &master->pump_messages);
+	kthread_queue_work(&master->kworker, &master->pump_messages);
 
 	return 0;
 }
@@ -1387,7 +1404,7 @@ static int spi_destroy_queue(struct spi_master *master)
 	ret = spi_stop_queue(master);
 
 	/*
-	 * flush_kthread_worker will block until all work is done.
+	 * kthread_flush_worker will block until all work is done.
 	 * If the reason that stop_queue timed out is that the work will never
 	 * finish, then it does no good to call flush/stop thread, so
 	 * return anyway.
@@ -1397,7 +1414,7 @@ static int spi_destroy_queue(struct spi_master *master)
 		return ret;
 	}
 
-	flush_kthread_worker(&master->kworker);
+	kthread_flush_worker(&master->kworker);
 	kthread_stop(master->kworker_task);
 
 	return 0;
@@ -1421,7 +1438,7 @@ static int __spi_queued_transfer(struct spi_device *spi,
 
 	list_add_tail(&msg->queue, &master->queue);
 	if (!master->busy && need_pump)
-		queue_kthread_work(&master->kworker, &master->pump_messages);
+		kthread_queue_work(&master->kworker, &master->pump_messages);
 
 	spin_unlock_irqrestore(&master->queue_lock, flags);
 	return 0;
diff --git a/drivers/spmi/spmi-pmic-arb.c b/drivers/spmi/spmi-pmic-arb.c
index aca282d45421..5ec3a595dc7d 100644
--- a/drivers/spmi/spmi-pmic-arb.c
+++ b/drivers/spmi/spmi-pmic-arb.c
@@ -954,6 +954,7 @@ static int spmi_pmic_arb_probe(struct platform_device *pdev)
 	if (channel > 5) {
 		dev_err(&pdev->dev, "invalid channel (%u) specified.\n",
 			channel);
+		err = -EINVAL;
 		goto err_put_ctrl;
 	}
 
diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig
index af9476460023..58a7b3504b82 100644
--- a/drivers/staging/Kconfig
+++ b/drivers/staging/Kconfig
@@ -40,8 +40,6 @@ source "drivers/staging/rtl8712/Kconfig"
 
 source "drivers/staging/rtl8188eu/Kconfig"
 
-source "drivers/staging/rtl8723au/Kconfig"
-
 source "drivers/staging/rts5208/Kconfig"
 
 source "drivers/staging/octeon/Kconfig"
@@ -104,4 +102,8 @@ source "drivers/staging/i4l/Kconfig"
 
 source "drivers/staging/ks7010/Kconfig"
 
+source "drivers/staging/greybus/Kconfig"
+
+source "drivers/staging/vc04_services/Kconfig"
+
 endif # STAGING
diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile
index 9f6009dcafa8..2fa9745db614 100644
--- a/drivers/staging/Makefile
+++ b/drivers/staging/Makefile
@@ -9,7 +9,6 @@ obj-$(CONFIG_RTL8192U)		+= rtl8192u/
 obj-$(CONFIG_RTL8192E)		+= rtl8192e/
 obj-$(CONFIG_R8712U)		+= rtl8712/
 obj-$(CONFIG_R8188EU)		+= rtl8188eu/
-obj-$(CONFIG_R8723AU)		+= rtl8723au/
 obj-$(CONFIG_RTS5208)		+= rts5208/
 obj-$(CONFIG_NETLOGIC_XLR_NET)	+= netlogic/
 obj-$(CONFIG_OCTEON_ETHERNET)	+= octeon/
@@ -41,3 +40,5 @@ obj-$(CONFIG_WILC1000)		+= wilc1000/
 obj-$(CONFIG_MOST)		+= most/
 obj-$(CONFIG_ISDN_I4L)		+= i4l/
 obj-$(CONFIG_KS7010)		+= ks7010/
+obj-$(CONFIG_GREYBUS)		+= greybus/
+obj-$(CONFIG_BCM2708_VCHIQ)	+= vc04_services/
diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig
index 06e41d24ec62..6c00d6f765c6 100644
--- a/drivers/staging/android/Kconfig
+++ b/drivers/staging/android/Kconfig
@@ -24,19 +24,6 @@ config ANDROID_LOW_MEMORY_KILLER
 	  scripts (/init.rc), and it defines priority values with minimum free memory size
 	  for each priority.
 
-config SW_SYNC
-	bool "Software synchronization framework"
-	default n
-	depends on SYNC_FILE
-	depends on DEBUG_FS
-	---help---
-	  A sync object driver that uses a 32bit counter to coordinate
-	  synchronization.  Useful when there is no hardware primitive backing
-	  the synchronization.
-
-	  WARNING: improper use of this can result in deadlocking kernel
-	  drivers from userspace. Intended for test and debug only.
-
 source "drivers/staging/android/ion/Kconfig"
 
 endif # if ANDROID
diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile
index 7ca61b77a8d4..7ed1be798909 100644
--- a/drivers/staging/android/Makefile
+++ b/drivers/staging/android/Makefile
@@ -4,4 +4,3 @@ obj-y					+= ion/
 
 obj-$(CONFIG_ASHMEM)			+= ashmem.o
 obj-$(CONFIG_ANDROID_LOW_MEMORY_KILLER)	+= lowmemorykiller.o
-obj-$(CONFIG_SW_SYNC)			+= sw_sync.o sync_debug.o
diff --git a/drivers/staging/android/ion/Kconfig b/drivers/staging/android/ion/Kconfig
index 19c1572f1525..c8fb4134c55d 100644
--- a/drivers/staging/android/ion/Kconfig
+++ b/drivers/staging/android/ion/Kconfig
@@ -36,7 +36,19 @@ config ION_TEGRA
 config ION_HISI
 	tristate "Ion for Hisilicon"
 	depends on ARCH_HISI && ION
+	select ION_OF
 	help
 	  Choose this option if you wish to use ion on Hisilicon Platform.
 
 source "drivers/staging/android/ion/hisilicon/Kconfig"
+
+config ION_OF
+	bool "Devicetree support for Ion"
+	depends on ION && OF_ADDRESS
+	help
+	  Provides base support for defining Ion heaps in devicetree
+	  and setting them up. Also includes functions for platforms
+	  to parse the devicetree and expand for their own custom
+	  extensions
+
+	  If using Ion and devicetree, you should say Y here
diff --git a/drivers/staging/android/ion/Makefile b/drivers/staging/android/ion/Makefile
index 18cc2aa593c2..5d630a088381 100644
--- a/drivers/staging/android/ion/Makefile
+++ b/drivers/staging/android/ion/Makefile
@@ -1,4 +1,5 @@
-obj-$(CONFIG_ION) +=	ion.o ion_heap.o ion_page_pool.o ion_system_heap.o \
+obj-$(CONFIG_ION) +=	ion.o ion-ioctl.o ion_heap.o \
+			ion_page_pool.o ion_system_heap.o \
 			ion_carveout_heap.o ion_chunk_heap.o ion_cma_heap.o
 obj-$(CONFIG_ION_TEST) += ion_test.o
 ifdef CONFIG_COMPAT
@@ -8,4 +9,5 @@ endif
 obj-$(CONFIG_ION_DUMMY) += ion_dummy_driver.o
 obj-$(CONFIG_ION_TEGRA) += tegra/
 obj-$(CONFIG_ION_HISI) += hisilicon/
+obj-$(CONFIG_ION_OF) += ion_of.o
 
diff --git a/drivers/staging/android/ion/devicetree.txt b/drivers/staging/android/ion/devicetree.txt
new file mode 100644
index 000000000000..168715271f06
--- /dev/null
+++ b/drivers/staging/android/ion/devicetree.txt
@@ -0,0 +1,51 @@
+Ion Memory Manager
+
+Ion is a memory manager that allows for sharing of buffers via dma-buf.
+Ion allows for different types of allocation via an abstraction called
+a 'heap'. A heap represents a specific type of memory. Each heap has
+a different type. There can be multiple instances of the same heap
+type.
+
+Specific heap instances are tied to heap IDs. Heap IDs are not to be specified
+in the devicetree.
+
+Required properties for Ion
+
+- compatible: "linux,ion" PLUS a compatible property for the device
+
+All child nodes of a linux,ion node are interpreted as heaps
+
+required properties for heaps
+
+- compatible: compatible string for a heap type PLUS a compatible property
+for the specific instance of the heap. Current heap types
+-- linux,ion-heap-system
+-- linux,ion-heap-system-contig
+-- linux,ion-heap-carveout
+-- linux,ion-heap-chunk
+-- linux,ion-heap-dma
+-- linux,ion-heap-custom
+
+Optional properties
+- memory-region: A phandle to a memory region. Required for DMA heap type
+(see reserved-memory.txt for details on the reservation)
+
+Example:
+
+	ion {
+		compatbile = "hisilicon,ion", "linux,ion";
+
+		ion-system-heap {
+			compatbile = "hisilicon,system-heap", "linux,ion-heap-system"
+		};
+
+		ion-camera-region {
+			compatible = "hisilicon,camera-heap", "linux,ion-heap-dma"
+			memory-region = <&camera_region>;
+		};
+
+		ion-fb-region {
+			compatbile = "hisilicon,fb-heap", "linux,ion-heap-dma"
+			memory-region = <&fb_region>;
+		};
+	}
diff --git a/drivers/staging/android/ion/hisilicon/hi6220_ion.c b/drivers/staging/android/ion/hisilicon/hi6220_ion.c
index fe9f0fd210cd..0de7897fd4bf 100644
--- a/drivers/staging/android/ion/hisilicon/hi6220_ion.c
+++ b/drivers/staging/android/ion/hisilicon/hi6220_ion.c
@@ -19,181 +19,74 @@
 #include <linux/mm.h>
 #include "../ion_priv.h"
 #include "../ion.h"
+#include "../ion_of.h"
 
-struct hi6220_ion_type_table {
-	const char *name;
-	enum ion_heap_type type;
+struct hisi_ion_dev {
+	struct ion_heap	**heaps;
+	struct ion_device *idev;
+	struct ion_platform_data *data;
 };
 
-static struct hi6220_ion_type_table ion_type_table[] = {
-	{"ion_system", ION_HEAP_TYPE_SYSTEM},
-	{"ion_system_contig", ION_HEAP_TYPE_SYSTEM_CONTIG},
-	{"ion_carveout", ION_HEAP_TYPE_CARVEOUT},
-	{"ion_chunk", ION_HEAP_TYPE_CHUNK},
-	{"ion_dma", ION_HEAP_TYPE_DMA},
-	{"ion_custom", ION_HEAP_TYPE_CUSTOM},
+static struct ion_of_heap hisi_heaps[] = {
+	PLATFORM_HEAP("hisilicon,sys_user", 0,
+		      ION_HEAP_TYPE_SYSTEM, "sys_user"),
+	PLATFORM_HEAP("hisilicon,sys_contig", 1,
+		      ION_HEAP_TYPE_SYSTEM_CONTIG, "sys_contig"),
+	PLATFORM_HEAP("hisilicon,cma", ION_HEAP_TYPE_DMA, ION_HEAP_TYPE_DMA,
+		      "cma"),
+	{}
 };
 
-static struct ion_device *idev;
-static int num_heaps;
-static struct ion_heap **heaps;
-static struct ion_platform_heap **heaps_data;
-
-static int get_type_by_name(const char *name, enum ion_heap_type *type)
+static int hi6220_ion_probe(struct platform_device *pdev)
 {
+	struct hisi_ion_dev *ipdev;
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(ion_type_table); i++) {
-		if (strncmp(name, ion_type_table[i].name, strlen(name)))
-			continue;
-
-		*type = ion_type_table[i].type;
-		return 0;
-	}
-
-	return -EINVAL;
-}
-
-static int hi6220_set_platform_data(struct platform_device *pdev)
-{
-	unsigned int base;
-	unsigned int size;
-	unsigned int id;
-	const char *heap_name;
-	const char *type_name;
-	enum ion_heap_type type;
-	int ret;
-	struct device_node *np;
-	struct ion_platform_heap *p_data;
-	const struct device_node *dt_node = pdev->dev.of_node;
-	int index = 0;
-
-	for_each_child_of_node(dt_node, np)
-		num_heaps++;
-
-	heaps_data = devm_kzalloc(&pdev->dev,
-				  sizeof(struct ion_platform_heap *) *
-				  num_heaps,
-				  GFP_KERNEL);
-	if (!heaps_data)
+	ipdev = devm_kzalloc(&pdev->dev, sizeof(*ipdev), GFP_KERNEL);
+	if (!ipdev)
 		return -ENOMEM;
 
-	for_each_child_of_node(dt_node, np) {
-		ret = of_property_read_string(np, "heap-name", &heap_name);
-		if (ret < 0) {
-			pr_err("check the name of node %s\n", np->name);
-			continue;
-		}
+	platform_set_drvdata(pdev, ipdev);
 
-		ret = of_property_read_u32(np, "heap-id", &id);
-		if (ret < 0) {
-			pr_err("check the id %s\n", np->name);
-			continue;
-		}
+	ipdev->idev = ion_device_create(NULL);
+	if (IS_ERR(ipdev->idev))
+		return PTR_ERR(ipdev->idev);
 
-		ret = of_property_read_u32(np, "heap-base", &base);
-		if (ret < 0) {
-			pr_err("check the base of node %s\n", np->name);
-			continue;
-		}
-
-		ret = of_property_read_u32(np, "heap-size", &size);
-		if (ret < 0) {
-			pr_err("check the size of node %s\n", np->name);
-			continue;
-		}
-
-		ret = of_property_read_string(np, "heap-type", &type_name);
-		if (ret < 0) {
-			pr_err("check the type of node %s\n", np->name);
-			continue;
-		}
+	ipdev->data = ion_parse_dt(pdev, hisi_heaps);
+	if (IS_ERR(ipdev->data))
+		return PTR_ERR(ipdev->data);
 
-		ret = get_type_by_name(type_name, &type);
-		if (ret < 0) {
-			pr_err("type name error %s!\n", type_name);
-			continue;
-		}
-		pr_info("heap index %d : name %s base 0x%x size 0x%x id %d type %d\n",
-			index, heap_name, base, size, id, type);
+	ipdev->heaps = devm_kzalloc(&pdev->dev,
+				sizeof(struct ion_heap) * ipdev->data->nr,
+				GFP_KERNEL);
+	if (!ipdev->heaps) {
+		ion_destroy_platform_data(ipdev->data);
+		return -ENOMEM;
+	}
 
-		p_data = devm_kzalloc(&pdev->dev,
-				      sizeof(struct ion_platform_heap),
-				      GFP_KERNEL);
-		if (!p_data)
+	for (i = 0; i < ipdev->data->nr; i++) {
+		ipdev->heaps[i] = ion_heap_create(&ipdev->data->heaps[i]);
+		if (!ipdev->heaps) {
+			ion_destroy_platform_data(ipdev->data);
 			return -ENOMEM;
-
-		p_data->name = heap_name;
-		p_data->base = base;
-		p_data->size = size;
-		p_data->id = id;
-		p_data->type = type;
-
-		heaps_data[index] = p_data;
-		index++;
+		}
+		ion_device_add_heap(ipdev->idev, ipdev->heaps[i]);
 	}
 	return 0;
 }
 
-static int hi6220_ion_probe(struct platform_device *pdev)
+static int hi6220_ion_remove(struct platform_device *pdev)
 {
+	struct hisi_ion_dev *ipdev;
 	int i;
-	int err;
-	static struct ion_platform_heap *p_heap;
-
-	idev = ion_device_create(NULL);
-	err = hi6220_set_platform_data(pdev);
-	if (err) {
-		pr_err("ion set platform data error!\n");
-		goto err_free_idev;
-	}
-	heaps = devm_kzalloc(&pdev->dev,
-			     sizeof(struct ion_heap *) * num_heaps,
-			     GFP_KERNEL);
-	if (!heaps) {
-		err = -ENOMEM;
-		goto err_free_idev;
-	}
-
-	/*
-	 * create the heaps as specified in the dts file
-	 */
-	for (i = 0; i < num_heaps; i++) {
-		p_heap = heaps_data[i];
-		heaps[i] = ion_heap_create(p_heap);
-		if (IS_ERR_OR_NULL(heaps[i])) {
-			err = PTR_ERR(heaps[i]);
-			goto err_free_heaps;
-		}
-
-		ion_device_add_heap(idev, heaps[i]);
 
-		pr_info("%s: adding heap %s of type %d with %lx@%lx\n",
-			__func__, p_heap->name, p_heap->type,
-			p_heap->base, (unsigned long)p_heap->size);
-	}
-	return err;
+	ipdev = platform_get_drvdata(pdev);
 
-err_free_heaps:
-	for (i = 0; i < num_heaps; ++i) {
-		ion_heap_destroy(heaps[i]);
-		heaps[i] = NULL;
-	}
-err_free_idev:
-	ion_device_destroy(idev);
+	for (i = 0; i < ipdev->data->nr; i++)
+		ion_heap_destroy(ipdev->heaps[i]);
 
-	return err;
-}
-
-static int hi6220_ion_remove(struct platform_device *pdev)
-{
-	int i;
-
-	for (i = 0; i < num_heaps; i++) {
-		ion_heap_destroy(heaps[i]);
-		heaps[i] = NULL;
-	}
-	ion_device_destroy(idev);
+	ion_destroy_platform_data(ipdev->data);
+	ion_device_destroy(ipdev->idev);
 
 	return 0;
 }
diff --git a/drivers/staging/android/ion/ion-ioctl.c b/drivers/staging/android/ion/ion-ioctl.c
new file mode 100644
index 000000000000..7e7431d8d49f
--- /dev/null
+++ b/drivers/staging/android/ion/ion-ioctl.c
@@ -0,0 +1,177 @@
+/*
+ *
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+
+#include "ion.h"
+#include "ion_priv.h"
+#include "compat_ion.h"
+
+union ion_ioctl_arg {
+	struct ion_fd_data fd;
+	struct ion_allocation_data allocation;
+	struct ion_handle_data handle;
+	struct ion_custom_data custom;
+	struct ion_heap_query query;
+};
+
+static int validate_ioctl_arg(unsigned int cmd, union ion_ioctl_arg *arg)
+{
+	int ret = 0;
+
+	switch (cmd) {
+	case ION_IOC_HEAP_QUERY:
+		ret = arg->query.reserved0 != 0;
+		ret |= arg->query.reserved1 != 0;
+		ret |= arg->query.reserved2 != 0;
+		break;
+	default:
+		break;
+	}
+
+	return ret ? -EINVAL : 0;
+}
+
+/* fix up the cases where the ioctl direction bits are incorrect */
+static unsigned int ion_ioctl_dir(unsigned int cmd)
+{
+	switch (cmd) {
+	case ION_IOC_SYNC:
+	case ION_IOC_FREE:
+	case ION_IOC_CUSTOM:
+		return _IOC_WRITE;
+	default:
+		return _IOC_DIR(cmd);
+	}
+}
+
+long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	struct ion_client *client = filp->private_data;
+	struct ion_device *dev = client->dev;
+	struct ion_handle *cleanup_handle = NULL;
+	int ret = 0;
+	unsigned int dir;
+	union ion_ioctl_arg data;
+
+	dir = ion_ioctl_dir(cmd);
+
+	if (_IOC_SIZE(cmd) > sizeof(data))
+		return -EINVAL;
+
+	/*
+	 * The copy_from_user is unconditional here for both read and write
+	 * to do the validate. If there is no write for the ioctl, the
+	 * buffer is cleared
+	 */
+	if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd)))
+		return -EFAULT;
+
+	ret = validate_ioctl_arg(cmd, &data);
+	if (WARN_ON_ONCE(ret))
+		return ret;
+
+	if (!(dir & _IOC_WRITE))
+		memset(&data, 0, sizeof(data));
+
+	switch (cmd) {
+	case ION_IOC_ALLOC:
+	{
+		struct ion_handle *handle;
+
+		handle = ion_alloc(client, data.allocation.len,
+						data.allocation.align,
+						data.allocation.heap_id_mask,
+						data.allocation.flags);
+		if (IS_ERR(handle))
+			return PTR_ERR(handle);
+
+		data.allocation.handle = handle->id;
+
+		cleanup_handle = handle;
+		break;
+	}
+	case ION_IOC_FREE:
+	{
+		struct ion_handle *handle;
+
+		mutex_lock(&client->lock);
+		handle = ion_handle_get_by_id_nolock(client, data.handle.handle);
+		if (IS_ERR(handle)) {
+			mutex_unlock(&client->lock);
+			return PTR_ERR(handle);
+		}
+		ion_free_nolock(client, handle);
+		ion_handle_put_nolock(handle);
+		mutex_unlock(&client->lock);
+		break;
+	}
+	case ION_IOC_SHARE:
+	case ION_IOC_MAP:
+	{
+		struct ion_handle *handle;
+
+		handle = ion_handle_get_by_id(client, data.handle.handle);
+		if (IS_ERR(handle))
+			return PTR_ERR(handle);
+		data.fd.fd = ion_share_dma_buf_fd(client, handle);
+		ion_handle_put(handle);
+		if (data.fd.fd < 0)
+			ret = data.fd.fd;
+		break;
+	}
+	case ION_IOC_IMPORT:
+	{
+		struct ion_handle *handle;
+
+		handle = ion_import_dma_buf_fd(client, data.fd.fd);
+		if (IS_ERR(handle))
+			ret = PTR_ERR(handle);
+		else
+			data.handle.handle = handle->id;
+		break;
+	}
+	case ION_IOC_SYNC:
+	{
+		ret = ion_sync_for_device(client, data.fd.fd);
+		break;
+	}
+	case ION_IOC_CUSTOM:
+	{
+		if (!dev->custom_ioctl)
+			return -ENOTTY;
+		ret = dev->custom_ioctl(client, data.custom.cmd,
+						data.custom.arg);
+		break;
+	}
+	case ION_IOC_HEAP_QUERY:
+		ret = ion_query_heaps(client, &data.query);
+		break;
+	default:
+		return -ENOTTY;
+	}
+
+	if (dir & _IOC_READ) {
+		if (copy_to_user((void __user *)arg, &data, _IOC_SIZE(cmd))) {
+			if (cleanup_handle)
+				ion_free(client, cleanup_handle);
+			return -EFAULT;
+		}
+	}
+	return ret;
+}
diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c
index a2cf93b59016..396ded52ab70 100644
--- a/drivers/staging/android/ion/ion.c
+++ b/drivers/staging/android/ion/ion.c
@@ -41,80 +41,6 @@
 #include "ion_priv.h"
 #include "compat_ion.h"
 
-/**
- * struct ion_device - the metadata of the ion device node
- * @dev:		the actual misc device
- * @buffers:		an rb tree of all the existing buffers
- * @buffer_lock:	lock protecting the tree of buffers
- * @lock:		rwsem protecting the tree of heaps and clients
- * @heaps:		list of all the heaps in the system
- * @user_clients:	list of all the clients created from userspace
- */
-struct ion_device {
-	struct miscdevice dev;
-	struct rb_root buffers;
-	struct mutex buffer_lock;
-	struct rw_semaphore lock;
-	struct plist_head heaps;
-	long (*custom_ioctl)(struct ion_client *client, unsigned int cmd,
-			     unsigned long arg);
-	struct rb_root clients;
-	struct dentry *debug_root;
-	struct dentry *heaps_debug_root;
-	struct dentry *clients_debug_root;
-};
-
-/**
- * struct ion_client - a process/hw block local address space
- * @node:		node in the tree of all clients
- * @dev:		backpointer to ion device
- * @handles:		an rb tree of all the handles in this client
- * @idr:		an idr space for allocating handle ids
- * @lock:		lock protecting the tree of handles
- * @name:		used for debugging
- * @display_name:	used for debugging (unique version of @name)
- * @display_serial:	used for debugging (to make display_name unique)
- * @task:		used for debugging
- *
- * A client represents a list of buffers this client may access.
- * The mutex stored here is used to protect both handles tree
- * as well as the handles themselves, and should be held while modifying either.
- */
-struct ion_client {
-	struct rb_node node;
-	struct ion_device *dev;
-	struct rb_root handles;
-	struct idr idr;
-	struct mutex lock;
-	const char *name;
-	char *display_name;
-	int display_serial;
-	struct task_struct *task;
-	pid_t pid;
-	struct dentry *debug_root;
-};
-
-/**
- * ion_handle - a client local reference to a buffer
- * @ref:		reference count
- * @client:		back pointer to the client the buffer resides in
- * @buffer:		pointer to the buffer
- * @node:		node in the client's handle rbtree
- * @kmap_cnt:		count of times this client has mapped to kernel
- * @id:			client-unique id allocated by client->idr
- *
- * Modifications to node, map_cnt or mapping should be protected by the
- * lock in the client.  Other fields are never changed after initialization.
- */
-struct ion_handle {
-	struct kref ref;
-	struct ion_client *client;
-	struct ion_buffer *buffer;
-	struct rb_node node;
-	unsigned int kmap_cnt;
-	int id;
-};
-
 bool ion_buffer_fault_user_mappings(struct ion_buffer *buffer)
 {
 	return (buffer->flags & ION_FLAG_CACHED) &&
@@ -174,10 +100,10 @@ static void ion_buffer_add(struct ion_device *dev,
 
 /* this function should only be called while dev->lock is held */
 static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
-				     struct ion_device *dev,
-				     unsigned long len,
-				     unsigned long align,
-				     unsigned long flags)
+					    struct ion_device *dev,
+					    unsigned long len,
+					    unsigned long align,
+					    unsigned long flags)
 {
 	struct ion_buffer *buffer;
 	struct sg_table *table;
@@ -205,19 +131,16 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
 			goto err2;
 	}
 
-	buffer->dev = dev;
-	buffer->size = len;
-
-	table = heap->ops->map_dma(heap, buffer);
-	if (WARN_ONCE(table == NULL,
-			"heap->ops->map_dma should return ERR_PTR on error"))
-		table = ERR_PTR(-EINVAL);
-	if (IS_ERR(table)) {
+	if (buffer->sg_table == NULL) {
+		WARN_ONCE(1, "This heap needs to set the sgtable");
 		ret = -EINVAL;
 		goto err1;
 	}
 
-	buffer->sg_table = table;
+	table = buffer->sg_table;
+	buffer->dev = dev;
+	buffer->size = len;
+
 	if (ion_buffer_fault_user_mappings(buffer)) {
 		int num_pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;
 		struct scatterlist *sg;
@@ -226,7 +149,7 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
 		buffer->pages = vmalloc(sizeof(struct page *) * num_pages);
 		if (!buffer->pages) {
 			ret = -ENOMEM;
-			goto err;
+			goto err1;
 		}
 
 		for_each_sg(table->sgl, sg, table->nents, i) {
@@ -260,8 +183,6 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
 	mutex_unlock(&dev->buffer_lock);
 	return buffer;
 
-err:
-	heap->ops->unmap_dma(heap, buffer);
 err1:
 	heap->ops->free(buffer);
 err2:
@@ -273,7 +194,6 @@ void ion_buffer_destroy(struct ion_buffer *buffer)
 {
 	if (WARN_ON(buffer->kmap_cnt > 0))
 		buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
-	buffer->heap->ops->unmap_dma(buffer->heap, buffer);
 	buffer->heap->ops->free(buffer);
 	vfree(buffer->pages);
 	kfree(buffer);
@@ -337,7 +257,7 @@ static void ion_buffer_remove_from_handle(struct ion_buffer *buffer)
 }
 
 static struct ion_handle *ion_handle_create(struct ion_client *client,
-				     struct ion_buffer *buffer)
+					    struct ion_buffer *buffer)
 {
 	struct ion_handle *handle;
 
@@ -377,26 +297,17 @@ static void ion_handle_destroy(struct kref *kref)
 	kfree(handle);
 }
 
-struct ion_buffer *ion_handle_buffer(struct ion_handle *handle)
-{
-	return handle->buffer;
-}
-
 static void ion_handle_get(struct ion_handle *handle)
 {
 	kref_get(&handle->ref);
 }
 
-static int ion_handle_put_nolock(struct ion_handle *handle)
+int ion_handle_put_nolock(struct ion_handle *handle)
 {
-	int ret;
-
-	ret = kref_put(&handle->ref, ion_handle_destroy);
-
-	return ret;
+	return kref_put(&handle->ref, ion_handle_destroy);
 }
 
-static int ion_handle_put(struct ion_handle *handle)
+int ion_handle_put(struct ion_handle *handle)
 {
 	struct ion_client *client = handle->client;
 	int ret;
@@ -426,8 +337,8 @@ static struct ion_handle *ion_handle_lookup(struct ion_client *client,
 	return ERR_PTR(-EINVAL);
 }
 
-static struct ion_handle *ion_handle_get_by_id_nolock(struct ion_client *client,
-						int id)
+struct ion_handle *ion_handle_get_by_id_nolock(struct ion_client *client,
+					       int id)
 {
 	struct ion_handle *handle;
 
@@ -438,7 +349,7 @@ static struct ion_handle *ion_handle_get_by_id_nolock(struct ion_client *client,
 	return handle ? handle : ERR_PTR(-EINVAL);
 }
 
-static struct ion_handle *ion_handle_get_by_id(struct ion_client *client,
+struct ion_handle *ion_handle_get_by_id(struct ion_client *client,
 					       int id)
 {
 	struct ion_handle *handle;
@@ -551,15 +462,10 @@ struct ion_handle *ion_alloc(struct ion_client *client, size_t len,
 }
 EXPORT_SYMBOL(ion_alloc);
 
-static void ion_free_nolock(struct ion_client *client, struct ion_handle *handle)
+void ion_free_nolock(struct ion_client *client,
+		     struct ion_handle *handle)
 {
-	bool valid_handle;
-
-	BUG_ON(client != handle->client);
-
-	valid_handle = ion_handle_validate(client, handle);
-
-	if (!valid_handle) {
+	if (!ion_handle_validate(client, handle)) {
 		WARN(1, "%s: invalid handle passed to free.\n", __func__);
 		return;
 	}
@@ -576,32 +482,6 @@ void ion_free(struct ion_client *client, struct ion_handle *handle)
 }
 EXPORT_SYMBOL(ion_free);
 
-int ion_phys(struct ion_client *client, struct ion_handle *handle,
-	     ion_phys_addr_t *addr, size_t *len)
-{
-	struct ion_buffer *buffer;
-	int ret;
-
-	mutex_lock(&client->lock);
-	if (!ion_handle_validate(client, handle)) {
-		mutex_unlock(&client->lock);
-		return -EINVAL;
-	}
-
-	buffer = handle->buffer;
-
-	if (!buffer->heap->ops->phys) {
-		pr_err("%s: ion_phys is not implemented by this heap (name=%s, type=%d).\n",
-			__func__, buffer->heap->name, buffer->heap->type);
-		mutex_unlock(&client->lock);
-		return -ENODEV;
-	}
-	mutex_unlock(&client->lock);
-	ret = buffer->heap->ops->phys(buffer->heap, buffer, addr, len);
-	return ret;
-}
-EXPORT_SYMBOL(ion_phys);
-
 static void *ion_buffer_kmap_get(struct ion_buffer *buffer)
 {
 	void *vaddr;
@@ -612,7 +492,7 @@ static void *ion_buffer_kmap_get(struct ion_buffer *buffer)
 	}
 	vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer);
 	if (WARN_ONCE(vaddr == NULL,
-			"heap->ops->map_kernel should return ERR_PTR on error"))
+		      "heap->ops->map_kernel should return ERR_PTR on error"))
 		return ERR_PTR(-EINVAL);
 	if (IS_ERR(vaddr))
 		return vaddr;
@@ -781,14 +661,14 @@ static const struct file_operations debug_client_fops = {
 };
 
 static int ion_get_client_serial(const struct rb_root *root,
-					const unsigned char *name)
+				 const unsigned char *name)
 {
 	int serial = -1;
 	struct rb_node *node;
 
 	for (node = rb_first(root); node; node = rb_next(node)) {
 		struct ion_client *client = rb_entry(node, struct ion_client,
-						node);
+						     node);
 
 		if (strcmp(client->name, name))
 			continue;
@@ -863,14 +743,14 @@ struct ion_client *ion_client_create(struct ion_device *dev,
 	rb_insert_color(&client->node, &dev->clients);
 
 	client->debug_root = debugfs_create_file(client->display_name, 0664,
-						dev->clients_debug_root,
-						client, &debug_client_fops);
+						 dev->clients_debug_root,
+						 client, &debug_client_fops);
 	if (!client->debug_root) {
 		char buf[256], *path;
 
 		path = dentry_path(dev->clients_debug_root, buf, 256);
 		pr_err("Failed to create client debugfs at %s/%s\n",
-			path, client->display_name);
+		       path, client->display_name);
 	}
 
 	up_write(&dev->lock);
@@ -917,26 +797,6 @@ void ion_client_destroy(struct ion_client *client)
 }
 EXPORT_SYMBOL(ion_client_destroy);
 
-struct sg_table *ion_sg_table(struct ion_client *client,
-			      struct ion_handle *handle)
-{
-	struct ion_buffer *buffer;
-	struct sg_table *table;
-
-	mutex_lock(&client->lock);
-	if (!ion_handle_validate(client, handle)) {
-		pr_err("%s: invalid handle passed to map_dma.\n",
-		       __func__);
-		mutex_unlock(&client->lock);
-		return ERR_PTR(-EINVAL);
-	}
-	buffer = handle->buffer;
-	table = buffer->sg_table;
-	mutex_unlock(&client->lock);
-	return table;
-}
-EXPORT_SYMBOL(ion_sg_table);
-
 static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
 				       struct device *dev,
 				       enum dma_data_direction direction);
@@ -958,7 +818,7 @@ static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment,
 }
 
 void ion_pages_sync_for_device(struct device *dev, struct page *page,
-		size_t size, enum dma_data_direction dir)
+			       size_t size, enum dma_data_direction dir)
 {
 	struct scatterlist sg;
 
@@ -998,7 +858,7 @@ static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
 
 		if (ion_buffer_page_is_dirty(page))
 			ion_pages_sync_for_device(dev, ion_buffer_page(page),
-							PAGE_SIZE, dir);
+						  PAGE_SIZE, dir);
 
 		ion_buffer_page_clean(buffer->pages + i);
 	}
@@ -1076,7 +936,7 @@ static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
 
 	if (!buffer->heap->ops->map_user) {
 		pr_err("%s: this heap does not define a method for mapping to userspace\n",
-			__func__);
+		       __func__);
 		return -EINVAL;
 	}
 
@@ -1167,7 +1027,7 @@ static struct dma_buf_ops dma_buf_ops = {
 };
 
 struct dma_buf *ion_share_dma_buf(struct ion_client *client,
-						struct ion_handle *handle)
+				  struct ion_handle *handle)
 {
 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 	struct ion_buffer *buffer;
@@ -1275,7 +1135,7 @@ struct ion_handle *ion_import_dma_buf_fd(struct ion_client *client, int fd)
 }
 EXPORT_SYMBOL(ion_import_dma_buf_fd);
 
-static int ion_sync_for_device(struct ion_client *client, int fd)
+int ion_sync_for_device(struct ion_client *client, int fd)
 {
 	struct dma_buf *dmabuf;
 	struct ion_buffer *buffer;
@@ -1299,124 +1159,45 @@ static int ion_sync_for_device(struct ion_client *client, int fd)
 	return 0;
 }
 
-/* fix up the cases where the ioctl direction bits are incorrect */
-static unsigned int ion_ioctl_dir(unsigned int cmd)
-{
-	switch (cmd) {
-	case ION_IOC_SYNC:
-	case ION_IOC_FREE:
-	case ION_IOC_CUSTOM:
-		return _IOC_WRITE;
-	default:
-		return _IOC_DIR(cmd);
-	}
-}
-
-static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+int ion_query_heaps(struct ion_client *client, struct ion_heap_query *query)
 {
-	struct ion_client *client = filp->private_data;
 	struct ion_device *dev = client->dev;
-	struct ion_handle *cleanup_handle = NULL;
-	int ret = 0;
-	unsigned int dir;
-
-	union {
-		struct ion_fd_data fd;
-		struct ion_allocation_data allocation;
-		struct ion_handle_data handle;
-		struct ion_custom_data custom;
-	} data;
+	struct ion_heap_data __user *buffer = u64_to_user_ptr(query->heaps);
+	int ret = -EINVAL, cnt = 0, max_cnt;
+	struct ion_heap *heap;
+	struct ion_heap_data hdata;
 
-	dir = ion_ioctl_dir(cmd);
+	memset(&hdata, 0, sizeof(hdata));
 
-	if (_IOC_SIZE(cmd) > sizeof(data))
-		return -EINVAL;
+	down_read(&dev->lock);
+	if (!buffer) {
+		query->cnt = dev->heap_cnt;
+		ret = 0;
+		goto out;
+	}
 
-	if (dir & _IOC_WRITE)
-		if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd)))
-			return -EFAULT;
+	if (query->cnt <= 0)
+		goto out;
 
-	switch (cmd) {
-	case ION_IOC_ALLOC:
-	{
-		struct ion_handle *handle;
+	max_cnt = query->cnt;
 
-		handle = ion_alloc(client, data.allocation.len,
-						data.allocation.align,
-						data.allocation.heap_id_mask,
-						data.allocation.flags);
-		if (IS_ERR(handle))
-			return PTR_ERR(handle);
+	plist_for_each_entry(heap, &dev->heaps, node) {
+		strncpy(hdata.name, heap->name, MAX_HEAP_NAME);
+		hdata.name[sizeof(hdata.name) - 1] = '\0';
+		hdata.type = heap->type;
+		hdata.heap_id = heap->id;
 
-		data.allocation.handle = handle->id;
+		ret = copy_to_user(&buffer[cnt],
+				   &hdata, sizeof(hdata));
 
-		cleanup_handle = handle;
-		break;
-	}
-	case ION_IOC_FREE:
-	{
-		struct ion_handle *handle;
-
-		mutex_lock(&client->lock);
-		handle = ion_handle_get_by_id_nolock(client, data.handle.handle);
-		if (IS_ERR(handle)) {
-			mutex_unlock(&client->lock);
-			return PTR_ERR(handle);
-		}
-		ion_free_nolock(client, handle);
-		ion_handle_put_nolock(handle);
-		mutex_unlock(&client->lock);
-		break;
-	}
-	case ION_IOC_SHARE:
-	case ION_IOC_MAP:
-	{
-		struct ion_handle *handle;
-
-		handle = ion_handle_get_by_id(client, data.handle.handle);
-		if (IS_ERR(handle))
-			return PTR_ERR(handle);
-		data.fd.fd = ion_share_dma_buf_fd(client, handle);
-		ion_handle_put(handle);
-		if (data.fd.fd < 0)
-			ret = data.fd.fd;
-		break;
-	}
-	case ION_IOC_IMPORT:
-	{
-		struct ion_handle *handle;
-
-		handle = ion_import_dma_buf_fd(client, data.fd.fd);
-		if (IS_ERR(handle))
-			ret = PTR_ERR(handle);
-		else
-			data.handle.handle = handle->id;
-		break;
-	}
-	case ION_IOC_SYNC:
-	{
-		ret = ion_sync_for_device(client, data.fd.fd);
-		break;
-	}
-	case ION_IOC_CUSTOM:
-	{
-		if (!dev->custom_ioctl)
-			return -ENOTTY;
-		ret = dev->custom_ioctl(client, data.custom.cmd,
-						data.custom.arg);
-		break;
-	}
-	default:
-		return -ENOTTY;
+		cnt++;
+		if (cnt >= max_cnt)
+			break;
 	}
 
-	if (dir & _IOC_READ) {
-		if (copy_to_user((void __user *)arg, &data, _IOC_SIZE(cmd))) {
-			if (cleanup_handle)
-				ion_free(client, cleanup_handle);
-			return -EFAULT;
-		}
-	}
+	query->cnt = cnt;
+out:
+	up_read(&dev->lock);
 	return ret;
 }
 
@@ -1528,7 +1309,7 @@ static int ion_debug_heap_show(struct seq_file *s, void *unused)
 	seq_printf(s, "%16s %16zu\n", "total ", total_size);
 	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
 		seq_printf(s, "%16s %16zu\n", "deferred free",
-				heap->free_list_size);
+			   heap->free_list_size);
 	seq_puts(s, "----------------------------------------------------\n");
 
 	if (heap->debug_show)
@@ -1588,8 +1369,7 @@ void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)
 {
 	struct dentry *debug_file;
 
-	if (!heap->ops->allocate || !heap->ops->free || !heap->ops->map_dma ||
-	    !heap->ops->unmap_dma)
+	if (!heap->ops->allocate || !heap->ops->free)
 		pr_err("%s: can not add heap with invalid ops struct.\n",
 		       __func__);
 
@@ -1611,15 +1391,15 @@ void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)
 	plist_node_init(&heap->node, -heap->id);
 	plist_add(&heap->node, &dev->heaps);
 	debug_file = debugfs_create_file(heap->name, 0664,
-					dev->heaps_debug_root, heap,
-					&debug_heap_fops);
+					 dev->heaps_debug_root, heap,
+					 &debug_heap_fops);
 
 	if (!debug_file) {
 		char buf[256], *path;
 
 		path = dentry_path(dev->heaps_debug_root, buf, 256);
 		pr_err("Failed to create heap debugfs at %s/%s\n",
-			path, heap->name);
+		       path, heap->name);
 	}
 
 	if (heap->shrinker.count_objects && heap->shrinker.scan_objects) {
@@ -1634,10 +1414,11 @@ void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)
 
 			path = dentry_path(dev->heaps_debug_root, buf, 256);
 			pr_err("Failed to create heap shrinker debugfs at %s/%s\n",
-				path, debug_name);
+			       path, debug_name);
 		}
 	}
 
+	dev->heap_cnt++;
 	up_write(&dev->lock);
 }
 EXPORT_SYMBOL(ion_device_add_heap);
@@ -1702,38 +1483,3 @@ void ion_device_destroy(struct ion_device *dev)
 	kfree(dev);
 }
 EXPORT_SYMBOL(ion_device_destroy);
-
-void __init ion_reserve(struct ion_platform_data *data)
-{
-	int i;
-
-	for (i = 0; i < data->nr; i++) {
-		if (data->heaps[i].size == 0)
-			continue;
-
-		if (data->heaps[i].base == 0) {
-			phys_addr_t paddr;
-
-			paddr = memblock_alloc_base(data->heaps[i].size,
-						    data->heaps[i].align,
-						    MEMBLOCK_ALLOC_ANYWHERE);
-			if (!paddr) {
-				pr_err("%s: error allocating memblock for heap %d\n",
-					__func__, i);
-				continue;
-			}
-			data->heaps[i].base = paddr;
-		} else {
-			int ret = memblock_reserve(data->heaps[i].base,
-					       data->heaps[i].size);
-			if (ret)
-				pr_err("memblock reserve of %zx@%lx failed\n",
-				       data->heaps[i].size,
-				       data->heaps[i].base);
-		}
-		pr_info("%s: %s reserved base %lx size %zu\n", __func__,
-			data->heaps[i].name,
-			data->heaps[i].base,
-			data->heaps[i].size);
-	}
-}
diff --git a/drivers/staging/android/ion/ion.h b/drivers/staging/android/ion/ion.h
index a1331fc169a1..93dafb4586e4 100644
--- a/drivers/staging/android/ion/ion.h
+++ b/drivers/staging/android/ion/ion.h
@@ -73,17 +73,6 @@ struct ion_platform_data {
 };
 
 /**
- * ion_reserve() - reserve memory for ion heaps if applicable
- * @data:	platform data specifying starting physical address and
- *		size
- *
- * Calls memblock reserve to set aside memory for heaps that are
- * located at specific memory addresses or of specific sizes not
- * managed by the kernel
- */
-void ion_reserve(struct ion_platform_data *data);
-
-/**
  * ion_client_create() -  allocate a client and returns it
  * @dev:		the global ion device
  * @name:		used for debugging
@@ -130,36 +119,6 @@ struct ion_handle *ion_alloc(struct ion_client *client, size_t len,
 void ion_free(struct ion_client *client, struct ion_handle *handle);
 
 /**
- * ion_phys - returns the physical address and len of a handle
- * @client:	the client
- * @handle:	the handle
- * @addr:	a pointer to put the address in
- * @len:	a pointer to put the length in
- *
- * This function queries the heap for a particular handle to get the
- * handle's physical address.  It't output is only correct if
- * a heap returns physically contiguous memory -- in other cases
- * this api should not be implemented -- ion_sg_table should be used
- * instead.  Returns -EINVAL if the handle is invalid.  This has
- * no implications on the reference counting of the handle --
- * the returned value may not be valid if the caller is not
- * holding a reference.
- */
-int ion_phys(struct ion_client *client, struct ion_handle *handle,
-	     ion_phys_addr_t *addr, size_t *len);
-
-/**
- * ion_map_dma - return an sg_table describing a handle
- * @client:	the client
- * @handle:	the handle
- *
- * This function returns the sg_table describing
- * a particular ion handle.
- */
-struct sg_table *ion_sg_table(struct ion_client *client,
-			      struct ion_handle *handle);
-
-/**
  * ion_map_kernel - create mapping for the given handle
  * @client:	the client
  * @handle:	handle to map
diff --git a/drivers/staging/android/ion/ion_carveout_heap.c b/drivers/staging/android/ion/ion_carveout_heap.c
index 1fb0d81556da..a8ea97391c40 100644
--- a/drivers/staging/android/ion/ion_carveout_heap.c
+++ b/drivers/staging/android/ion/ion_carveout_heap.c
@@ -25,15 +25,17 @@
 #include "ion.h"
 #include "ion_priv.h"
 
+#define ION_CARVEOUT_ALLOCATE_FAIL	-1
+
 struct ion_carveout_heap {
 	struct ion_heap heap;
 	struct gen_pool *pool;
 	ion_phys_addr_t base;
 };
 
-ion_phys_addr_t ion_carveout_allocate(struct ion_heap *heap,
-				      unsigned long size,
-				      unsigned long align)
+static ion_phys_addr_t ion_carveout_allocate(struct ion_heap *heap,
+					     unsigned long size,
+					     unsigned long align)
 {
 	struct ion_carveout_heap *carveout_heap =
 		container_of(heap, struct ion_carveout_heap, heap);
@@ -45,8 +47,8 @@ ion_phys_addr_t ion_carveout_allocate(struct ion_heap *heap,
 	return offset;
 }
 
-void ion_carveout_free(struct ion_heap *heap, ion_phys_addr_t addr,
-		       unsigned long size)
+static void ion_carveout_free(struct ion_heap *heap, ion_phys_addr_t addr,
+			      unsigned long size)
 {
 	struct ion_carveout_heap *carveout_heap =
 		container_of(heap, struct ion_carveout_heap, heap);
@@ -56,19 +58,6 @@ void ion_carveout_free(struct ion_heap *heap, ion_phys_addr_t addr,
 	gen_pool_free(carveout_heap->pool, addr, size);
 }
 
-static int ion_carveout_heap_phys(struct ion_heap *heap,
-				  struct ion_buffer *buffer,
-				  ion_phys_addr_t *addr, size_t *len)
-{
-	struct sg_table *table = buffer->priv_virt;
-	struct page *page = sg_page(table->sgl);
-	ion_phys_addr_t paddr = PFN_PHYS(page_to_pfn(page));
-
-	*addr = paddr;
-	*len = buffer->size;
-	return 0;
-}
-
 static int ion_carveout_heap_allocate(struct ion_heap *heap,
 				      struct ion_buffer *buffer,
 				      unsigned long size, unsigned long align,
@@ -95,7 +84,7 @@ static int ion_carveout_heap_allocate(struct ion_heap *heap,
 	}
 
 	sg_set_page(table->sgl, pfn_to_page(PFN_DOWN(paddr)), size, 0);
-	buffer->priv_virt = table;
+	buffer->sg_table = table;
 
 	return 0;
 
@@ -109,7 +98,7 @@ err_free:
 static void ion_carveout_heap_free(struct ion_buffer *buffer)
 {
 	struct ion_heap *heap = buffer->heap;
-	struct sg_table *table = buffer->priv_virt;
+	struct sg_table *table = buffer->sg_table;
 	struct page *page = sg_page(table->sgl);
 	ion_phys_addr_t paddr = PFN_PHYS(page_to_pfn(page));
 
@@ -124,23 +113,9 @@ static void ion_carveout_heap_free(struct ion_buffer *buffer)
 	kfree(table);
 }
 
-static struct sg_table *ion_carveout_heap_map_dma(struct ion_heap *heap,
-						  struct ion_buffer *buffer)
-{
-	return buffer->priv_virt;
-}
-
-static void ion_carveout_heap_unmap_dma(struct ion_heap *heap,
-					struct ion_buffer *buffer)
-{
-}
-
 static struct ion_heap_ops carveout_heap_ops = {
 	.allocate = ion_carveout_heap_allocate,
 	.free = ion_carveout_heap_free,
-	.phys = ion_carveout_heap_phys,
-	.map_dma = ion_carveout_heap_map_dma,
-	.unmap_dma = ion_carveout_heap_unmap_dma,
 	.map_user = ion_heap_map_user,
 	.map_kernel = ion_heap_map_kernel,
 	.unmap_kernel = ion_heap_unmap_kernel,
diff --git a/drivers/staging/android/ion/ion_chunk_heap.c b/drivers/staging/android/ion/ion_chunk_heap.c
index e0553fee9b8a..70495dc645ea 100644
--- a/drivers/staging/android/ion/ion_chunk_heap.c
+++ b/drivers/staging/android/ion/ion_chunk_heap.c
@@ -34,9 +34,9 @@ struct ion_chunk_heap {
 };
 
 static int ion_chunk_heap_allocate(struct ion_heap *heap,
-				      struct ion_buffer *buffer,
-				      unsigned long size, unsigned long align,
-				      unsigned long flags)
+				   struct ion_buffer *buffer,
+				   unsigned long size, unsigned long align,
+				   unsigned long flags)
 {
 	struct ion_chunk_heap *chunk_heap =
 		container_of(heap, struct ion_chunk_heap, heap);
@@ -71,11 +71,11 @@ static int ion_chunk_heap_allocate(struct ion_heap *heap,
 		if (!paddr)
 			goto err;
 		sg_set_page(sg, pfn_to_page(PFN_DOWN(paddr)),
-				chunk_heap->chunk_size, 0);
+			    chunk_heap->chunk_size, 0);
 		sg = sg_next(sg);
 	}
 
-	buffer->priv_virt = table;
+	buffer->sg_table = table;
 	chunk_heap->allocated += allocated_size;
 	return 0;
 err:
@@ -95,7 +95,7 @@ static void ion_chunk_heap_free(struct ion_buffer *buffer)
 	struct ion_heap *heap = buffer->heap;
 	struct ion_chunk_heap *chunk_heap =
 		container_of(heap, struct ion_chunk_heap, heap);
-	struct sg_table *table = buffer->priv_virt;
+	struct sg_table *table = buffer->sg_table;
 	struct scatterlist *sg;
 	int i;
 	unsigned long allocated_size;
@@ -106,7 +106,7 @@ static void ion_chunk_heap_free(struct ion_buffer *buffer)
 
 	if (ion_buffer_cached(buffer))
 		dma_sync_sg_for_device(NULL, table->sgl, table->nents,
-							DMA_BIDIRECTIONAL);
+				       DMA_BIDIRECTIONAL);
 
 	for_each_sg(table->sgl, sg, table->nents, i) {
 		gen_pool_free(chunk_heap->pool, page_to_phys(sg_page(sg)),
@@ -117,22 +117,9 @@ static void ion_chunk_heap_free(struct ion_buffer *buffer)
 	kfree(table);
 }
 
-static struct sg_table *ion_chunk_heap_map_dma(struct ion_heap *heap,
-					       struct ion_buffer *buffer)
-{
-	return buffer->priv_virt;
-}
-
-static void ion_chunk_heap_unmap_dma(struct ion_heap *heap,
-				     struct ion_buffer *buffer)
-{
-}
-
 static struct ion_heap_ops chunk_heap_ops = {
 	.allocate = ion_chunk_heap_allocate,
 	.free = ion_chunk_heap_free,
-	.map_dma = ion_chunk_heap_map_dma,
-	.unmap_dma = ion_chunk_heap_unmap_dma,
 	.map_user = ion_heap_map_user,
 	.map_kernel = ion_heap_map_kernel,
 	.unmap_kernel = ion_heap_unmap_kernel,
@@ -174,7 +161,7 @@ struct ion_heap *ion_chunk_heap_create(struct ion_platform_heap *heap_data)
 	chunk_heap->heap.type = ION_HEAP_TYPE_CHUNK;
 	chunk_heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE;
 	pr_debug("%s: base %lu size %zu align %ld\n", __func__,
-		chunk_heap->base, heap_data->size, heap_data->align);
+		 chunk_heap->base, heap_data->size, heap_data->align);
 
 	return &chunk_heap->heap;
 
diff --git a/drivers/staging/android/ion/ion_cma_heap.c b/drivers/staging/android/ion/ion_cma_heap.c
index a3446da4fdc2..6c7de74bc7ab 100644
--- a/drivers/staging/android/ion/ion_cma_heap.c
+++ b/drivers/staging/android/ion/ion_cma_heap.c
@@ -78,6 +78,7 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer,
 		goto free_table;
 	/* keep this for memory release */
 	buffer->priv_virt = info;
+	buffer->sg_table = info->table;
 	dev_dbg(dev, "Allocate buffer %p\n", buffer);
 	return 0;
 
@@ -105,36 +106,6 @@ static void ion_cma_free(struct ion_buffer *buffer)
 	kfree(info);
 }
 
-/* return physical address in addr */
-static int ion_cma_phys(struct ion_heap *heap, struct ion_buffer *buffer,
-			ion_phys_addr_t *addr, size_t *len)
-{
-	struct ion_cma_heap *cma_heap = to_cma_heap(buffer->heap);
-	struct device *dev = cma_heap->dev;
-	struct ion_cma_buffer_info *info = buffer->priv_virt;
-
-	dev_dbg(dev, "Return buffer %p physical address %pa\n", buffer,
-		&info->handle);
-
-	*addr = info->handle;
-	*len = buffer->size;
-
-	return 0;
-}
-
-static struct sg_table *ion_cma_heap_map_dma(struct ion_heap *heap,
-					     struct ion_buffer *buffer)
-{
-	struct ion_cma_buffer_info *info = buffer->priv_virt;
-
-	return info->table;
-}
-
-static void ion_cma_heap_unmap_dma(struct ion_heap *heap,
-				   struct ion_buffer *buffer)
-{
-}
-
 static int ion_cma_mmap(struct ion_heap *mapper, struct ion_buffer *buffer,
 			struct vm_area_struct *vma)
 {
@@ -155,16 +126,13 @@ static void *ion_cma_map_kernel(struct ion_heap *heap,
 }
 
 static void ion_cma_unmap_kernel(struct ion_heap *heap,
-					struct ion_buffer *buffer)
+				 struct ion_buffer *buffer)
 {
 }
 
 static struct ion_heap_ops ion_cma_ops = {
 	.allocate = ion_cma_allocate,
 	.free = ion_cma_free,
-	.map_dma = ion_cma_heap_map_dma,
-	.unmap_dma = ion_cma_heap_unmap_dma,
-	.phys = ion_cma_phys,
 	.map_user = ion_cma_mmap,
 	.map_kernel = ion_cma_map_kernel,
 	.unmap_kernel = ion_cma_unmap_kernel,
diff --git a/drivers/staging/android/ion/ion_dummy_driver.c b/drivers/staging/android/ion/ion_dummy_driver.c
index 814a3c92a56e..b23f2c76c753 100644
--- a/drivers/staging/android/ion/ion_dummy_driver.c
+++ b/drivers/staging/android/ion/ion_dummy_driver.c
@@ -99,7 +99,7 @@ static int __init ion_dummy_init(void)
 		struct ion_platform_heap *heap_data = &dummy_ion_pdata.heaps[i];
 
 		if (heap_data->type == ION_HEAP_TYPE_CARVEOUT &&
-							!heap_data->base)
+		    !heap_data->base)
 			continue;
 
 		if (heap_data->type == ION_HEAP_TYPE_CHUNK && !heap_data->base)
@@ -120,12 +120,12 @@ err:
 
 	if (carveout_ptr) {
 		free_pages_exact(carveout_ptr,
-				dummy_heaps[ION_HEAP_TYPE_CARVEOUT].size);
+				 dummy_heaps[ION_HEAP_TYPE_CARVEOUT].size);
 		carveout_ptr = NULL;
 	}
 	if (chunk_ptr) {
 		free_pages_exact(chunk_ptr,
-				dummy_heaps[ION_HEAP_TYPE_CHUNK].size);
+				 dummy_heaps[ION_HEAP_TYPE_CHUNK].size);
 		chunk_ptr = NULL;
 	}
 	return err;
@@ -144,12 +144,12 @@ static void __exit ion_dummy_exit(void)
 
 	if (carveout_ptr) {
 		free_pages_exact(carveout_ptr,
-				dummy_heaps[ION_HEAP_TYPE_CARVEOUT].size);
+				 dummy_heaps[ION_HEAP_TYPE_CARVEOUT].size);
 		carveout_ptr = NULL;
 	}
 	if (chunk_ptr) {
 		free_pages_exact(chunk_ptr,
-				dummy_heaps[ION_HEAP_TYPE_CHUNK].size);
+				 dummy_heaps[ION_HEAP_TYPE_CHUNK].size);
 		chunk_ptr = NULL;
 	}
 }
diff --git a/drivers/staging/android/ion/ion_heap.c b/drivers/staging/android/ion/ion_heap.c
index ca15a87f6fd3..4e5c0f17f579 100644
--- a/drivers/staging/android/ion/ion_heap.c
+++ b/drivers/staging/android/ion/ion_heap.c
@@ -93,7 +93,7 @@ int ion_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer,
 		}
 		len = min(len, remainder);
 		ret = remap_pfn_range(vma, addr, page_to_pfn(page), len,
-				vma->vm_page_prot);
+				      vma->vm_page_prot);
 		if (ret)
 			return ret;
 		addr += len;
@@ -116,7 +116,7 @@ static int ion_heap_clear_pages(struct page **pages, int num, pgprot_t pgprot)
 }
 
 static int ion_heap_sglist_zero(struct scatterlist *sgl, unsigned int nents,
-						pgprot_t pgprot)
+				pgprot_t pgprot)
 {
 	int p = 0;
 	int ret = 0;
@@ -181,7 +181,7 @@ size_t ion_heap_freelist_size(struct ion_heap *heap)
 }
 
 static size_t _ion_heap_freelist_drain(struct ion_heap *heap, size_t size,
-				bool skip_pools)
+				       bool skip_pools)
 {
 	struct ion_buffer *buffer;
 	size_t total_drained = 0;
@@ -266,7 +266,7 @@ int ion_heap_init_deferred_free(struct ion_heap *heap)
 }
 
 static unsigned long ion_heap_shrink_count(struct shrinker *shrinker,
-						struct shrink_control *sc)
+					   struct shrink_control *sc)
 {
 	struct ion_heap *heap = container_of(shrinker, struct ion_heap,
 					     shrinker);
@@ -279,7 +279,7 @@ static unsigned long ion_heap_shrink_count(struct shrinker *shrinker,
 }
 
 static unsigned long ion_heap_shrink_scan(struct shrinker *shrinker,
-						struct shrink_control *sc)
+					  struct shrink_control *sc)
 {
 	struct ion_heap *heap = container_of(shrinker, struct ion_heap,
 					     shrinker);
diff --git a/drivers/staging/android/ion/ion_of.c b/drivers/staging/android/ion/ion_of.c
new file mode 100644
index 000000000000..15bac92b7f04
--- /dev/null
+++ b/drivers/staging/android/ion/ion_of.c
@@ -0,0 +1,185 @@
+/*
+ * Based on work from:
+ *   Andrew Andrianov <andrew@ncrmnt.org>
+ *   Google
+ *   The Linux Foundation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/init.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/of_address.h>
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/cma.h>
+#include <linux/dma-contiguous.h>
+#include <linux/io.h>
+#include <linux/of_reserved_mem.h>
+#include "ion.h"
+#include "ion_priv.h"
+#include "ion_of.h"
+
+static int ion_parse_dt_heap_common(struct device_node *heap_node,
+				    struct ion_platform_heap *heap,
+				    struct ion_of_heap *compatible)
+{
+	int i;
+
+	for (i = 0; compatible[i].name; i++) {
+		if (of_device_is_compatible(heap_node, compatible[i].compat))
+			break;
+	}
+
+	if (!compatible[i].name)
+		return -ENODEV;
+
+	heap->id = compatible[i].heap_id;
+	heap->type = compatible[i].type;
+	heap->name = compatible[i].name;
+	heap->align = compatible[i].align;
+
+	/* Some kind of callback function pointer? */
+
+	pr_info("%s: id %d type %d name %s align %lx\n", __func__,
+		heap->id, heap->type, heap->name, heap->align);
+	return 0;
+}
+
+static int ion_setup_heap_common(struct platform_device *parent,
+				 struct device_node *heap_node,
+				 struct ion_platform_heap *heap)
+{
+	int ret = 0;
+
+	switch (heap->type) {
+	case ION_HEAP_TYPE_CARVEOUT:
+	case ION_HEAP_TYPE_CHUNK:
+		if (heap->base && heap->size)
+			return 0;
+
+		ret = of_reserved_mem_device_init(heap->priv);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+struct ion_platform_data *ion_parse_dt(struct platform_device *pdev,
+				       struct ion_of_heap *compatible)
+{
+	int num_heaps, ret;
+	const struct device_node *dt_node = pdev->dev.of_node;
+	struct device_node *node;
+	struct ion_platform_heap *heaps;
+	struct ion_platform_data *data;
+	int i = 0;
+
+	num_heaps = of_get_available_child_count(dt_node);
+
+	if (!num_heaps)
+		return ERR_PTR(-EINVAL);
+
+	heaps = devm_kzalloc(&pdev->dev,
+			     sizeof(struct ion_platform_heap) * num_heaps,
+			     GFP_KERNEL);
+	if (!heaps)
+		return ERR_PTR(-ENOMEM);
+
+	data = devm_kzalloc(&pdev->dev, sizeof(struct ion_platform_data),
+			    GFP_KERNEL);
+	if (!data)
+		return ERR_PTR(-ENOMEM);
+
+	for_each_available_child_of_node(dt_node, node) {
+		struct platform_device *heap_pdev;
+
+		ret = ion_parse_dt_heap_common(node, &heaps[i], compatible);
+		if (ret)
+			return ERR_PTR(ret);
+
+		heap_pdev = of_platform_device_create(node, heaps[i].name,
+						      &pdev->dev);
+		if (!pdev)
+			return ERR_PTR(-ENOMEM);
+		heap_pdev->dev.platform_data = &heaps[i];
+
+		heaps[i].priv = &heap_pdev->dev;
+
+		ret = ion_setup_heap_common(pdev, node, &heaps[i]);
+		if (ret)
+			goto out_err;
+		i++;
+	}
+
+	data->heaps = heaps;
+	data->nr = num_heaps;
+	return data;
+
+out_err:
+	for ( ; i >= 0; i--)
+		if (heaps[i].priv)
+			of_device_unregister(to_platform_device(heaps[i].priv));
+
+	return ERR_PTR(ret);
+}
+
+void ion_destroy_platform_data(struct ion_platform_data *data)
+{
+	int i;
+
+	for (i = 0; i < data->nr; i++)
+		if (data->heaps[i].priv)
+			of_device_unregister(to_platform_device(
+				data->heaps[i].priv));
+}
+
+#ifdef CONFIG_OF_RESERVED_MEM
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+#include <linux/of_reserved_mem.h>
+
+static int rmem_ion_device_init(struct reserved_mem *rmem, struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct ion_platform_heap *heap = pdev->dev.platform_data;
+
+	heap->base = rmem->base;
+	heap->base = rmem->size;
+	pr_debug("%s: heap %s base %pa size %pa dev %p\n", __func__,
+		 heap->name, &rmem->base, &rmem->size, dev);
+	return 0;
+}
+
+static void rmem_ion_device_release(struct reserved_mem *rmem,
+				    struct device *dev)
+{
+	return;
+}
+
+static const struct reserved_mem_ops rmem_dma_ops = {
+	.device_init	= rmem_ion_device_init,
+	.device_release	= rmem_ion_device_release,
+};
+
+static int __init rmem_ion_setup(struct reserved_mem *rmem)
+{
+	phys_addr_t size = rmem->size;
+
+	size = size / 1024;
+
+	pr_info("Ion memory setup at %pa size %pa MiB\n",
+		&rmem->base, &size);
+	rmem->ops = &rmem_dma_ops;
+	return 0;
+}
+
+RESERVEDMEM_OF_DECLARE(ion, "ion-region", rmem_ion_setup);
+#endif
diff --git a/drivers/staging/android/ion/ion_of.h b/drivers/staging/android/ion/ion_of.h
new file mode 100644
index 000000000000..8241a1770f0a
--- /dev/null
+++ b/drivers/staging/android/ion/ion_of.h
@@ -0,0 +1,37 @@
+/*
+ * Based on work from:
+ *   Andrew Andrianov <andrew@ncrmnt.org>
+ *   Google
+ *   The Linux Foundation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _ION_OF_H
+#define _ION_OF_H
+
+struct ion_of_heap {
+	const char *compat;
+	int heap_id;
+	int type;
+	const char *name;
+	int align;
+};
+
+#define PLATFORM_HEAP(_compat, _id, _type, _name) \
+{ \
+	.compat = _compat, \
+	.heap_id = _id, \
+	.type = _type, \
+	.name = _name, \
+	.align = PAGE_SIZE, \
+}
+
+struct ion_platform_data *ion_parse_dt(struct platform_device *pdev,
+					struct ion_of_heap *compatible);
+
+void ion_destroy_platform_data(struct ion_platform_data *data);
+
+#endif
diff --git a/drivers/staging/android/ion/ion_page_pool.c b/drivers/staging/android/ion/ion_page_pool.c
index 1fe80165a462..aea89c1ec345 100644
--- a/drivers/staging/android/ion/ion_page_pool.c
+++ b/drivers/staging/android/ion/ion_page_pool.c
@@ -30,8 +30,9 @@ static void *ion_page_pool_alloc_pages(struct ion_page_pool *pool)
 
 	if (!page)
 		return NULL;
-	ion_pages_sync_for_device(NULL, page, PAGE_SIZE << pool->order,
-						DMA_BIDIRECTIONAL);
+	if (!pool->cached)
+		ion_pages_sync_for_device(NULL, page, PAGE_SIZE << pool->order,
+					  DMA_BIDIRECTIONAL);
 	return page;
 }
 
@@ -114,7 +115,7 @@ static int ion_page_pool_total(struct ion_page_pool *pool, bool high)
 }
 
 int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
-				int nr_to_scan)
+			 int nr_to_scan)
 {
 	int freed = 0;
 	bool high;
@@ -147,7 +148,8 @@ int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
 	return freed;
 }
 
-struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order)
+struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order,
+					   bool cached)
 {
 	struct ion_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
 
@@ -161,6 +163,8 @@ struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order)
 	pool->order = order;
 	mutex_init(&pool->mutex);
 	plist_node_init(&pool->list, order);
+	if (cached)
+		pool->cached = true;
 
 	return pool;
 }
diff --git a/drivers/staging/android/ion/ion_priv.h b/drivers/staging/android/ion/ion_priv.h
index 0239883bffb7..3c3b3245275d 100644
--- a/drivers/staging/android/ion/ion_priv.h
+++ b/drivers/staging/android/ion/ion_priv.h
@@ -26,11 +26,10 @@
 #include <linux/sched.h>
 #include <linux/shrinker.h>
 #include <linux/types.h>
+#include <linux/miscdevice.h>
 
 #include "ion.h"
 
-struct ion_buffer *ion_handle_buffer(struct ion_handle *handle);
-
 /**
  * struct ion_buffer - metadata for a particular buffer
  * @ref:		reference count
@@ -42,8 +41,6 @@ struct ion_buffer *ion_handle_buffer(struct ion_handle *handle);
  * @size:		size of the buffer
  * @priv_virt:		private data to the buffer representable as
  *			a void *
- * @priv_phys:		private data to the buffer representable as
- *			an ion_phys_addr_t (and someday a phys_addr_t)
  * @lock:		protects the buffers cnt fields
  * @kmap_cnt:		number of times the buffer is mapped to the kernel
  * @vaddr:		the kernel mapping if kmap_cnt is not zero
@@ -69,10 +66,7 @@ struct ion_buffer {
 	unsigned long flags;
 	unsigned long private_flags;
 	size_t size;
-	union {
-		void *priv_virt;
-		ion_phys_addr_t priv_phys;
-	};
+	void *priv_virt;
 	struct mutex lock;
 	int kmap_cnt;
 	void *vaddr;
@@ -88,13 +82,84 @@ struct ion_buffer {
 void ion_buffer_destroy(struct ion_buffer *buffer);
 
 /**
+ * struct ion_device - the metadata of the ion device node
+ * @dev:		the actual misc device
+ * @buffers:		an rb tree of all the existing buffers
+ * @buffer_lock:	lock protecting the tree of buffers
+ * @lock:		rwsem protecting the tree of heaps and clients
+ * @heaps:		list of all the heaps in the system
+ * @user_clients:	list of all the clients created from userspace
+ */
+struct ion_device {
+	struct miscdevice dev;
+	struct rb_root buffers;
+	struct mutex buffer_lock;
+	struct rw_semaphore lock;
+	struct plist_head heaps;
+	long (*custom_ioctl)(struct ion_client *client, unsigned int cmd,
+			     unsigned long arg);
+	struct rb_root clients;
+	struct dentry *debug_root;
+	struct dentry *heaps_debug_root;
+	struct dentry *clients_debug_root;
+	int heap_cnt;
+};
+
+/**
+ * struct ion_client - a process/hw block local address space
+ * @node:		node in the tree of all clients
+ * @dev:		backpointer to ion device
+ * @handles:		an rb tree of all the handles in this client
+ * @idr:		an idr space for allocating handle ids
+ * @lock:		lock protecting the tree of handles
+ * @name:		used for debugging
+ * @display_name:	used for debugging (unique version of @name)
+ * @display_serial:	used for debugging (to make display_name unique)
+ * @task:		used for debugging
+ *
+ * A client represents a list of buffers this client may access.
+ * The mutex stored here is used to protect both handles tree
+ * as well as the handles themselves, and should be held while modifying either.
+ */
+struct ion_client {
+	struct rb_node node;
+	struct ion_device *dev;
+	struct rb_root handles;
+	struct idr idr;
+	struct mutex lock;
+	const char *name;
+	char *display_name;
+	int display_serial;
+	struct task_struct *task;
+	pid_t pid;
+	struct dentry *debug_root;
+};
+
+/**
+ * ion_handle - a client local reference to a buffer
+ * @ref:		reference count
+ * @client:		back pointer to the client the buffer resides in
+ * @buffer:		pointer to the buffer
+ * @node:		node in the client's handle rbtree
+ * @kmap_cnt:		count of times this client has mapped to kernel
+ * @id:			client-unique id allocated by client->idr
+ *
+ * Modifications to node, map_cnt or mapping should be protected by the
+ * lock in the client.  Other fields are never changed after initialization.
+ */
+struct ion_handle {
+	struct kref ref;
+	struct ion_client *client;
+	struct ion_buffer *buffer;
+	struct rb_node node;
+	unsigned int kmap_cnt;
+	int id;
+};
+
+/**
  * struct ion_heap_ops - ops to operate on a given heap
  * @allocate:		allocate memory
  * @free:		free memory
- * @phys		get physical address of a buffer (only define on
- *			physically contiguous heaps)
- * @map_dma		map the memory for dma to a scatterlist
- * @unmap_dma		unmap the memory for dma
  * @map_kernel		map memory to the kernel
  * @unmap_kernel	unmap memory to the kernel
  * @map_user		map memory to userspace
@@ -111,11 +176,6 @@ struct ion_heap_ops {
 			struct ion_buffer *buffer, unsigned long len,
 			unsigned long align, unsigned long flags);
 	void (*free)(struct ion_buffer *buffer);
-	int (*phys)(struct ion_heap *heap, struct ion_buffer *buffer,
-		    ion_phys_addr_t *addr, size_t *len);
-	struct sg_table * (*map_dma)(struct ion_heap *heap,
-				     struct ion_buffer *buffer);
-	void (*unmap_dma)(struct ion_heap *heap, struct ion_buffer *buffer);
 	void * (*map_kernel)(struct ion_heap *heap, struct ion_buffer *buffer);
 	void (*unmap_kernel)(struct ion_heap *heap, struct ion_buffer *buffer);
 	int (*map_user)(struct ion_heap *mapper, struct ion_buffer *buffer,
@@ -328,20 +388,6 @@ struct ion_heap *ion_cma_heap_create(struct ion_platform_heap *);
 void ion_cma_heap_destroy(struct ion_heap *);
 
 /**
- * kernel api to allocate/free from carveout -- used when carveout is
- * used to back an architecture specific custom heap
- */
-ion_phys_addr_t ion_carveout_allocate(struct ion_heap *heap, unsigned long size,
-				      unsigned long align);
-void ion_carveout_free(struct ion_heap *heap, ion_phys_addr_t addr,
-		       unsigned long size);
-/**
- * The carveout heap returns physical addresses, since 0 may be a valid
- * physical address, this is used to indicate allocation failed
- */
-#define ION_CARVEOUT_ALLOCATE_FAIL -1
-
-/**
  * functions for creating and destroying a heap pool -- allows you
  * to keep a pool of pre allocated memory to use from your heap.  Keeping
  * a pool of memory that is ready for dma, ie any cached mapping have been
@@ -360,6 +406,7 @@ void ion_carveout_free(struct ion_heap *heap, ion_phys_addr_t addr,
  * @gfp_mask:		gfp_mask to use from alloc
  * @order:		order of pages in the pool
  * @list:		plist node for list of pools
+ * @cached:		it's cached pool or not
  *
  * Allows you to keep a pool of pre allocated pages to use from your heap.
  * Keeping a pool of pages that is ready for dma, ie any cached mapping have
@@ -369,6 +416,7 @@ void ion_carveout_free(struct ion_heap *heap, ion_phys_addr_t addr,
 struct ion_page_pool {
 	int high_count;
 	int low_count;
+	bool cached;
 	struct list_head high_items;
 	struct list_head low_items;
 	struct mutex mutex;
@@ -377,7 +425,8 @@ struct ion_page_pool {
 	struct plist_node list;
 };
 
-struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order);
+struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order,
+					   bool cached);
 void ion_page_pool_destroy(struct ion_page_pool *);
 struct page *ion_page_pool_alloc(struct ion_page_pool *);
 void ion_page_pool_free(struct ion_page_pool *, struct page *);
@@ -403,4 +452,22 @@ int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
 void ion_pages_sync_for_device(struct device *dev, struct page *page,
 		size_t size, enum dma_data_direction dir);
 
+long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
+
+int ion_sync_for_device(struct ion_client *client, int fd);
+
+struct ion_handle *ion_handle_get_by_id_nolock(struct ion_client *client,
+						int id);
+
+void ion_free_nolock(struct ion_client *client, struct ion_handle *handle);
+
+int ion_handle_put_nolock(struct ion_handle *handle);
+
+struct ion_handle *ion_handle_get_by_id(struct ion_client *client,
+						int id);
+
+int ion_handle_put(struct ion_handle *handle);
+
+int ion_query_heaps(struct ion_client *client, struct ion_heap_query *query);
+
 #endif /* _ION_PRIV_H */
diff --git a/drivers/staging/android/ion/ion_system_heap.c b/drivers/staging/android/ion/ion_system_heap.c
index b69dfc706440..7e023d505af8 100644
--- a/drivers/staging/android/ion/ion_system_heap.c
+++ b/drivers/staging/android/ion/ion_system_heap.c
@@ -26,16 +26,18 @@
 #include "ion.h"
 #include "ion_priv.h"
 
+#define NUM_ORDERS ARRAY_SIZE(orders)
+
 static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |
 				     __GFP_NORETRY) & ~__GFP_RECLAIM;
-static gfp_t low_order_gfp_flags  = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN);
+static gfp_t low_order_gfp_flags  = (GFP_HIGHUSER | __GFP_ZERO);
 static const unsigned int orders[] = {8, 4, 0};
-static const int num_orders = ARRAY_SIZE(orders);
+
 static int order_to_index(unsigned int order)
 {
 	int i;
 
-	for (i = 0; i < num_orders; i++)
+	for (i = 0; i < NUM_ORDERS; i++)
 		if (order == orders[i])
 			return i;
 	BUG();
@@ -49,47 +51,55 @@ static inline unsigned int order_to_size(int order)
 
 struct ion_system_heap {
 	struct ion_heap heap;
-	struct ion_page_pool *pools[0];
+	struct ion_page_pool *uncached_pools[NUM_ORDERS];
+	struct ion_page_pool *cached_pools[NUM_ORDERS];
 };
 
+/**
+ * The page from page-pool are all zeroed before. We need do cache
+ * clean for cached buffer. The uncached buffer are always non-cached
+ * since it's allocated. So no need for non-cached pages.
+ */
 static struct page *alloc_buffer_page(struct ion_system_heap *heap,
 				      struct ion_buffer *buffer,
 				      unsigned long order)
 {
 	bool cached = ion_buffer_cached(buffer);
-	struct ion_page_pool *pool = heap->pools[order_to_index(order)];
+	struct ion_page_pool *pool;
 	struct page *page;
 
-	if (!cached) {
-		page = ion_page_pool_alloc(pool);
-	} else {
-		gfp_t gfp_flags = low_order_gfp_flags;
+	if (!cached)
+		pool = heap->uncached_pools[order_to_index(order)];
+	else
+		pool = heap->cached_pools[order_to_index(order)];
 
-		if (order > 4)
-			gfp_flags = high_order_gfp_flags;
-		page = alloc_pages(gfp_flags | __GFP_COMP, order);
-		if (!page)
-			return NULL;
-		ion_pages_sync_for_device(NULL, page, PAGE_SIZE << order,
-						DMA_BIDIRECTIONAL);
-	}
+	page = ion_page_pool_alloc(pool);
 
+	if (cached)
+		ion_pages_sync_for_device(NULL, page, PAGE_SIZE << order,
+					  DMA_BIDIRECTIONAL);
 	return page;
 }
 
 static void free_buffer_page(struct ion_system_heap *heap,
 			     struct ion_buffer *buffer, struct page *page)
 {
+	struct ion_page_pool *pool;
 	unsigned int order = compound_order(page);
 	bool cached = ion_buffer_cached(buffer);
 
-	if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) {
-		struct ion_page_pool *pool = heap->pools[order_to_index(order)];
-
-		ion_page_pool_free(pool, page);
-	} else {
+	/* go to system */
+	if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) {
 		__free_pages(page, order);
+		return;
 	}
+
+	if (!cached)
+		pool = heap->uncached_pools[order_to_index(order)];
+	else
+		pool = heap->cached_pools[order_to_index(order)];
+
+	ion_page_pool_free(pool, page);
 }
 
 
@@ -101,7 +111,7 @@ static struct page *alloc_largest_available(struct ion_system_heap *heap,
 	struct page *page;
 	int i;
 
-	for (i = 0; i < num_orders; i++) {
+	for (i = 0; i < NUM_ORDERS; i++) {
 		if (size < order_to_size(orders[i]))
 			continue;
 		if (max_order < orders[i])
@@ -118,9 +128,9 @@ static struct page *alloc_largest_available(struct ion_system_heap *heap,
 }
 
 static int ion_system_heap_allocate(struct ion_heap *heap,
-				     struct ion_buffer *buffer,
-				     unsigned long size, unsigned long align,
-				     unsigned long flags)
+				    struct ion_buffer *buffer,
+				    unsigned long size, unsigned long align,
+				    unsigned long flags)
 {
 	struct ion_system_heap *sys_heap = container_of(heap,
 							struct ion_system_heap,
@@ -142,7 +152,7 @@ static int ion_system_heap_allocate(struct ion_heap *heap,
 	INIT_LIST_HEAD(&pages);
 	while (size_remaining > 0) {
 		page = alloc_largest_available(sys_heap, buffer, size_remaining,
-						max_order);
+					       max_order);
 		if (!page)
 			goto free_pages;
 		list_add_tail(&page->lru, &pages);
@@ -164,7 +174,7 @@ static int ion_system_heap_allocate(struct ion_heap *heap,
 		list_del(&page->lru);
 	}
 
-	buffer->priv_virt = table;
+	buffer->sg_table = table;
 	return 0;
 
 free_table:
@@ -181,16 +191,11 @@ static void ion_system_heap_free(struct ion_buffer *buffer)
 							struct ion_system_heap,
 							heap);
 	struct sg_table *table = buffer->sg_table;
-	bool cached = ion_buffer_cached(buffer);
 	struct scatterlist *sg;
 	int i;
 
-	/*
-	 *  uncached pages come from the page pools, zero them before returning
-	 *  for security purposes (other allocations are zerod at
-	 *  alloc time
-	 */
-	if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE))
+	/* zero the buffer before goto page pool */
+	if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE))
 		ion_heap_buffer_zero(buffer);
 
 	for_each_sg(table->sgl, sg, table->nents, i)
@@ -199,20 +204,11 @@ static void ion_system_heap_free(struct ion_buffer *buffer)
 	kfree(table);
 }
 
-static struct sg_table *ion_system_heap_map_dma(struct ion_heap *heap,
-						struct ion_buffer *buffer)
-{
-	return buffer->priv_virt;
-}
-
-static void ion_system_heap_unmap_dma(struct ion_heap *heap,
-				      struct ion_buffer *buffer)
-{
-}
-
 static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask,
-					int nr_to_scan)
+				  int nr_to_scan)
 {
+	struct ion_page_pool *uncached_pool;
+	struct ion_page_pool *cached_pool;
 	struct ion_system_heap *sys_heap;
 	int nr_total = 0;
 	int i, nr_freed;
@@ -223,28 +219,41 @@ static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask,
 	if (!nr_to_scan)
 		only_scan = 1;
 
-	for (i = 0; i < num_orders; i++) {
-		struct ion_page_pool *pool = sys_heap->pools[i];
-
-		nr_freed = ion_page_pool_shrink(pool, gfp_mask, nr_to_scan);
-		nr_total += nr_freed;
-
-		if (!only_scan) {
+	for (i = 0; i < NUM_ORDERS; i++) {
+		uncached_pool = sys_heap->uncached_pools[i];
+		cached_pool = sys_heap->cached_pools[i];
+
+		if (only_scan) {
+			nr_total += ion_page_pool_shrink(uncached_pool,
+							 gfp_mask,
+							 nr_to_scan);
+
+			nr_total += ion_page_pool_shrink(cached_pool,
+							 gfp_mask,
+							 nr_to_scan);
+		} else {
+			nr_freed = ion_page_pool_shrink(uncached_pool,
+							gfp_mask,
+							nr_to_scan);
+			nr_to_scan -= nr_freed;
+			nr_total += nr_freed;
+			if (nr_to_scan <= 0)
+				break;
+			nr_freed = ion_page_pool_shrink(cached_pool,
+							gfp_mask,
+							nr_to_scan);
 			nr_to_scan -= nr_freed;
-			/* shrink completed */
+			nr_total += nr_freed;
 			if (nr_to_scan <= 0)
 				break;
 		}
 	}
-
 	return nr_total;
 }
 
 static struct ion_heap_ops system_heap_ops = {
 	.allocate = ion_system_heap_allocate,
 	.free = ion_system_heap_free,
-	.map_dma = ion_system_heap_map_dma,
-	.unmap_dma = ion_system_heap_unmap_dma,
 	.map_kernel = ion_heap_map_kernel,
 	.unmap_kernel = ion_heap_unmap_kernel,
 	.map_user = ion_heap_map_user,
@@ -259,52 +268,89 @@ static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s,
 							struct ion_system_heap,
 							heap);
 	int i;
+	struct ion_page_pool *pool;
+
+	for (i = 0; i < NUM_ORDERS; i++) {
+		pool = sys_heap->uncached_pools[i];
+
+		seq_printf(s, "%d order %u highmem pages uncached %lu total\n",
+			   pool->high_count, pool->order,
+			   (PAGE_SIZE << pool->order) * pool->high_count);
+		seq_printf(s, "%d order %u lowmem pages uncached %lu total\n",
+			   pool->low_count, pool->order,
+			   (PAGE_SIZE << pool->order) * pool->low_count);
+	}
 
-	for (i = 0; i < num_orders; i++) {
-		struct ion_page_pool *pool = sys_heap->pools[i];
+	for (i = 0; i < NUM_ORDERS; i++) {
+		pool = sys_heap->cached_pools[i];
 
-		seq_printf(s, "%d order %u highmem pages in pool = %lu total\n",
+		seq_printf(s, "%d order %u highmem pages cached %lu total\n",
 			   pool->high_count, pool->order,
 			   (PAGE_SIZE << pool->order) * pool->high_count);
-		seq_printf(s, "%d order %u lowmem pages in pool = %lu total\n",
+		seq_printf(s, "%d order %u lowmem pages cached %lu total\n",
 			   pool->low_count, pool->order,
 			   (PAGE_SIZE << pool->order) * pool->low_count);
 	}
 	return 0;
 }
 
+static void ion_system_heap_destroy_pools(struct ion_page_pool **pools)
+{
+	int i;
+
+	for (i = 0; i < NUM_ORDERS; i++)
+		if (pools[i])
+			ion_page_pool_destroy(pools[i]);
+}
+
+static int ion_system_heap_create_pools(struct ion_page_pool **pools,
+					bool cached)
+{
+	int i;
+	gfp_t gfp_flags = low_order_gfp_flags;
+
+	for (i = 0; i < NUM_ORDERS; i++) {
+		struct ion_page_pool *pool;
+
+		if (orders[i] > 4)
+			gfp_flags = high_order_gfp_flags;
+
+		pool = ion_page_pool_create(gfp_flags, orders[i], cached);
+		if (!pool)
+			goto err_create_pool;
+		pools[i] = pool;
+	}
+	return 0;
+
+err_create_pool:
+	ion_system_heap_destroy_pools(pools);
+	return -ENOMEM;
+}
+
 struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused)
 {
 	struct ion_system_heap *heap;
-	int i;
 
-	heap = kzalloc(sizeof(struct ion_system_heap) +
-			sizeof(struct ion_page_pool *) * num_orders,
-			GFP_KERNEL);
+	heap = kzalloc(sizeof(*heap), GFP_KERNEL);
 	if (!heap)
 		return ERR_PTR(-ENOMEM);
 	heap->heap.ops = &system_heap_ops;
 	heap->heap.type = ION_HEAP_TYPE_SYSTEM;
 	heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE;
 
-	for (i = 0; i < num_orders; i++) {
-		struct ion_page_pool *pool;
-		gfp_t gfp_flags = low_order_gfp_flags;
+	if (ion_system_heap_create_pools(heap->uncached_pools, false))
+		goto free_heap;
 
-		if (orders[i] > 4)
-			gfp_flags = high_order_gfp_flags;
-		pool = ion_page_pool_create(gfp_flags, orders[i]);
-		if (!pool)
-			goto destroy_pools;
-		heap->pools[i] = pool;
-	}
+	if (ion_system_heap_create_pools(heap->cached_pools, true))
+		goto destroy_uncached_pools;
 
 	heap->heap.debug_show = ion_system_heap_debug_show;
 	return &heap->heap;
 
-destroy_pools:
-	while (i--)
-		ion_page_pool_destroy(heap->pools[i]);
+destroy_uncached_pools:
+	ion_system_heap_destroy_pools(heap->uncached_pools);
+
+free_heap:
 	kfree(heap);
 	return ERR_PTR(-ENOMEM);
 }
@@ -316,8 +362,10 @@ void ion_system_heap_destroy(struct ion_heap *heap)
 							heap);
 	int i;
 
-	for (i = 0; i < num_orders; i++)
-		ion_page_pool_destroy(sys_heap->pools[i]);
+	for (i = 0; i < NUM_ORDERS; i++) {
+		ion_page_pool_destroy(sys_heap->uncached_pools[i]);
+		ion_page_pool_destroy(sys_heap->cached_pools[i]);
+	}
 	kfree(sys_heap);
 }
 
@@ -358,7 +406,7 @@ static int ion_system_contig_heap_allocate(struct ion_heap *heap,
 
 	sg_set_page(table->sgl, page, len, 0);
 
-	buffer->priv_virt = table;
+	buffer->sg_table = table;
 
 	ion_pages_sync_for_device(NULL, page, len, DMA_BIDIRECTIONAL);
 
@@ -375,7 +423,7 @@ free_pages:
 
 static void ion_system_contig_heap_free(struct ion_buffer *buffer)
 {
-	struct sg_table *table = buffer->priv_virt;
+	struct sg_table *table = buffer->sg_table;
 	struct page *page = sg_page(table->sgl);
 	unsigned long pages = PAGE_ALIGN(buffer->size) >> PAGE_SHIFT;
 	unsigned long i;
@@ -386,34 +434,9 @@ static void ion_system_contig_heap_free(struct ion_buffer *buffer)
 	kfree(table);
 }
 
-static int ion_system_contig_heap_phys(struct ion_heap *heap,
-				       struct ion_buffer *buffer,
-				       ion_phys_addr_t *addr, size_t *len)
-{
-	struct sg_table *table = buffer->priv_virt;
-	struct page *page = sg_page(table->sgl);
-	*addr = page_to_phys(page);
-	*len = buffer->size;
-	return 0;
-}
-
-static struct sg_table *ion_system_contig_heap_map_dma(struct ion_heap *heap,
-						struct ion_buffer *buffer)
-{
-	return buffer->priv_virt;
-}
-
-static void ion_system_contig_heap_unmap_dma(struct ion_heap *heap,
-					     struct ion_buffer *buffer)
-{
-}
-
 static struct ion_heap_ops kmalloc_ops = {
 	.allocate = ion_system_contig_heap_allocate,
 	.free = ion_system_contig_heap_free,
-	.phys = ion_system_contig_heap_phys,
-	.map_dma = ion_system_contig_heap_map_dma,
-	.unmap_dma = ion_system_contig_heap_unmap_dma,
 	.map_kernel = ion_heap_map_kernel,
 	.unmap_kernel = ion_heap_unmap_kernel,
 	.map_user = ion_heap_map_user,
diff --git a/drivers/staging/android/ion/ion_test.c b/drivers/staging/android/ion/ion_test.c
index 5a396a1a8238..5abf8320a96a 100644
--- a/drivers/staging/android/ion/ion_test.c
+++ b/drivers/staging/android/ion/ion_test.c
@@ -42,7 +42,8 @@ struct ion_test_data {
 };
 
 static int ion_handle_test_dma(struct device *dev, struct dma_buf *dma_buf,
-		void __user *ptr, size_t offset, size_t size, bool write)
+			       void __user *ptr, size_t offset, size_t size,
+			       bool write)
 {
 	int ret = 0;
 	struct dma_buf_attachment *attach;
@@ -98,7 +99,7 @@ err:
 }
 
 static int ion_handle_test_kernel(struct dma_buf *dma_buf, void __user *ptr,
-		size_t offset, size_t size, bool write)
+				  size_t offset, size_t size, bool write)
 {
 	int ret;
 	unsigned long page_offset = offset >> PAGE_SHIFT;
@@ -144,7 +145,7 @@ err:
 }
 
 static long ion_test_ioctl(struct file *filp, unsigned int cmd,
-						unsigned long arg)
+			   unsigned long arg)
 {
 	struct ion_test_data *test_data = filp->private_data;
 	int ret = 0;
@@ -179,17 +180,19 @@ static long ion_test_ioctl(struct file *filp, unsigned int cmd,
 	case ION_IOC_TEST_DMA_MAPPING:
 	{
 		ret = ion_handle_test_dma(test_data->dev, test_data->dma_buf,
-					u64_to_uptr(data.test_rw.ptr),
-					data.test_rw.offset, data.test_rw.size,
-					data.test_rw.write);
+					  u64_to_uptr(data.test_rw.ptr),
+					  data.test_rw.offset,
+					  data.test_rw.size,
+					  data.test_rw.write);
 		break;
 	}
 	case ION_IOC_TEST_KERNEL_MAPPING:
 	{
 		ret = ion_handle_test_kernel(test_data->dma_buf,
-					u64_to_uptr(data.test_rw.ptr),
-					data.test_rw.offset, data.test_rw.size,
-					data.test_rw.write);
+					     u64_to_uptr(data.test_rw.ptr),
+					     data.test_rw.offset,
+					     data.test_rw.size,
+					     data.test_rw.write);
 		break;
 	}
 	default:
@@ -242,7 +245,7 @@ static int __init ion_test_probe(struct platform_device *pdev)
 	struct ion_test_device *testdev;
 
 	testdev = devm_kzalloc(&pdev->dev, sizeof(struct ion_test_device),
-				GFP_KERNEL);
+			       GFP_KERNEL);
 	if (!testdev)
 		return -ENOMEM;
 
diff --git a/drivers/staging/android/lowmemorykiller.c b/drivers/staging/android/lowmemorykiller.c
index 45a1b4ec4ca3..ec3b66561412 100644
--- a/drivers/staging/android/lowmemorykiller.c
+++ b/drivers/staging/android/lowmemorykiller.c
@@ -92,8 +92,8 @@ static unsigned long lowmem_scan(struct shrinker *s, struct shrink_control *sc)
 	int array_size = ARRAY_SIZE(lowmem_adj);
 	int other_free = global_page_state(NR_FREE_PAGES) - totalreserve_pages;
 	int other_file = global_node_page_state(NR_FILE_PAGES) -
-						global_node_page_state(NR_SHMEM) -
-						total_swapcache_pages();
+				global_node_page_state(NR_SHMEM) -
+				total_swapcache_pages();
 
 	if (lowmem_adj_size < array_size)
 		array_size = lowmem_adj_size;
@@ -204,10 +204,9 @@ device_initcall(lowmem_init);
  * not really modular, but the easiest way to keep compat with existing
  * bootargs behaviour is to continue using module_param here.
  */
-module_param_named(cost, lowmem_shrinker.seeks, int, S_IRUGO | S_IWUSR);
-module_param_array_named(adj, lowmem_adj, short, &lowmem_adj_size,
-			 S_IRUGO | S_IWUSR);
+module_param_named(cost, lowmem_shrinker.seeks, int, 0644);
+module_param_array_named(adj, lowmem_adj, short, &lowmem_adj_size, 0644);
 module_param_array_named(minfree, lowmem_minfree, uint, &lowmem_minfree_size,
-			 S_IRUGO | S_IWUSR);
-module_param_named(debug_level, lowmem_debug_level, uint, S_IRUGO | S_IWUSR);
+			 0644);
+module_param_named(debug_level, lowmem_debug_level, uint, 0644);
 
diff --git a/drivers/staging/android/uapi/ion.h b/drivers/staging/android/uapi/ion.h
index 0a8e40f92cd7..14cd8738ecfc 100644
--- a/drivers/staging/android/uapi/ion.h
+++ b/drivers/staging/android/uapi/ion.h
@@ -44,32 +44,26 @@ enum ion_heap_type {
 			       * must be last so device specific heaps always
 			       * are at the end of this enum
 			       */
-	ION_NUM_HEAPS = 16,
 };
 
-#define ION_HEAP_SYSTEM_MASK		(1 << ION_HEAP_TYPE_SYSTEM)
-#define ION_HEAP_SYSTEM_CONTIG_MASK	(1 << ION_HEAP_TYPE_SYSTEM_CONTIG)
-#define ION_HEAP_CARVEOUT_MASK		(1 << ION_HEAP_TYPE_CARVEOUT)
-#define ION_HEAP_TYPE_DMA_MASK		(1 << ION_HEAP_TYPE_DMA)
-
 #define ION_NUM_HEAP_IDS		(sizeof(unsigned int) * 8)
 
 /**
  * allocation flags - the lower 16 bits are used by core ion, the upper 16
  * bits are reserved for use by the heaps themselves.
  */
-#define ION_FLAG_CACHED 1		/*
-					 * mappings of this buffer should be
-					 * cached, ion will do cache
-					 * maintenance when the buffer is
-					 * mapped for dma
-					*/
-#define ION_FLAG_CACHED_NEEDS_SYNC 2	/*
-					 * mappings of this buffer will created
-					 * at mmap time, if this is set
-					 * caches must be managed
-					 * manually
-					 */
+
+/*
+ * mappings of this buffer should be cached, ion will do cache maintenance
+ * when the buffer is mapped for dma
+ */
+#define ION_FLAG_CACHED 1
+
+/*
+ * mappings of this buffer will created at mmap time, if this is set
+ * caches must be managed manually
+ */
+#define ION_FLAG_CACHED_NEEDS_SYNC 2
 
 /**
  * DOC: Ion Userspace API
@@ -134,6 +128,36 @@ struct ion_custom_data {
 	unsigned long arg;
 };
 
+#define MAX_HEAP_NAME			32
+
+/**
+ * struct ion_heap_data - data about a heap
+ * @name - first 32 characters of the heap name
+ * @type - heap type
+ * @heap_id - heap id for the heap
+ */
+struct ion_heap_data {
+	char name[MAX_HEAP_NAME];
+	__u32 type;
+	__u32 heap_id;
+	__u32 reserved0;
+	__u32 reserved1;
+	__u32 reserved2;
+};
+
+/**
+ * struct ion_heap_query - collection of data about all heaps
+ * @cnt - total number of heaps to be copied
+ * @heaps - buffer to copy heap data
+ */
+struct ion_heap_query {
+	__u32 cnt; /* Total number of heaps to be copied */
+	__u32 reserved0; /* align to 64bits */
+	__u64 heaps; /* buffer to be populated */
+	__u32 reserved1;
+	__u32 reserved2;
+};
+
 #define ION_IOC_MAGIC		'I'
 
 /**
@@ -200,4 +224,13 @@ struct ion_custom_data {
  */
 #define ION_IOC_CUSTOM		_IOWR(ION_IOC_MAGIC, 6, struct ion_custom_data)
 
+/**
+ * DOC: ION_IOC_HEAP_QUERY - information about available heaps
+ *
+ * Takes an ion_heap_query structure and populates information about
+ * available Ion heaps.
+ */
+#define ION_IOC_HEAP_QUERY     _IOWR(ION_IOC_MAGIC, 8, \
+					struct ion_heap_query)
+
 #endif /* _UAPI_LINUX_ION_H */
diff --git a/drivers/staging/board/board.c b/drivers/staging/board/board.c
index 45807d8287d1..86dc41101610 100644
--- a/drivers/staging/board/board.c
+++ b/drivers/staging/board/board.c
@@ -140,7 +140,6 @@ static int board_staging_add_dev_domain(struct platform_device *pdev,
 					const char *domain)
 {
 	struct of_phandle_args pd_args;
-	struct generic_pm_domain *pd;
 	struct device_node *np;
 
 	np = of_find_node_by_path(domain);
@@ -151,14 +150,8 @@ static int board_staging_add_dev_domain(struct platform_device *pdev,
 
 	pd_args.np = np;
 	pd_args.args_count = 0;
-	pd = of_genpd_get_from_provider(&pd_args);
-	if (IS_ERR(pd)) {
-		pr_err("Cannot find genpd %s (%ld)\n", domain, PTR_ERR(pd));
-		return PTR_ERR(pd);
-	}
-	pr_debug("Found genpd %s for device %s\n", pd->name, pdev->name);
 
-	return pm_genpd_add_device(pd, &pdev->dev);
+	return of_genpd_add_device(&pd_args, &pdev->dev);
 }
 #else
 static inline int board_staging_add_dev_domain(struct platform_device *pdev,
diff --git a/drivers/staging/comedi/comedi_fops.c b/drivers/staging/comedi/comedi_fops.c
index 1999eed4f4c5..64b3966c5f1f 100644
--- a/drivers/staging/comedi/comedi_fops.c
+++ b/drivers/staging/comedi/comedi_fops.c
@@ -81,20 +81,20 @@ struct comedi_file {
 	(COMEDI_NUM_MINORS - COMEDI_NUM_BOARD_MINORS)
 
 static int comedi_num_legacy_minors;
-module_param(comedi_num_legacy_minors, int, S_IRUGO);
+module_param(comedi_num_legacy_minors, int, 0444);
 MODULE_PARM_DESC(comedi_num_legacy_minors,
 		 "number of comedi minor devices to reserve for non-auto-configured devices (default 0)"
 		);
 
 unsigned int comedi_default_buf_size_kb = CONFIG_COMEDI_DEFAULT_BUF_SIZE_KB;
-module_param(comedi_default_buf_size_kb, uint, S_IRUGO | S_IWUSR);
+module_param(comedi_default_buf_size_kb, uint, 0644);
 MODULE_PARM_DESC(comedi_default_buf_size_kb,
 		 "default asynchronous buffer size in KiB (default "
 		 __MODULE_STRING(CONFIG_COMEDI_DEFAULT_BUF_SIZE_KB) ")");
 
 unsigned int comedi_default_buf_maxsize_kb
 	= CONFIG_COMEDI_DEFAULT_BUF_MAXSIZE_KB;
-module_param(comedi_default_buf_maxsize_kb, uint, S_IRUGO | S_IWUSR);
+module_param(comedi_default_buf_maxsize_kb, uint, 0644);
 MODULE_PARM_DESC(comedi_default_buf_maxsize_kb,
 		 "default maximum size of asynchronous buffer in KiB (default "
 		 __MODULE_STRING(CONFIG_COMEDI_DEFAULT_BUF_MAXSIZE_KB) ")");
@@ -2233,7 +2233,7 @@ static int comedi_mmap(struct file *file, struct vm_area_struct *vma)
 		goto done;
 	}
 
-	n_pages = size >> PAGE_SHIFT;
+	n_pages = vma_pages(vma);
 
 	/* get reference to current buf map (if any) */
 	bm = comedi_buf_map_from_subdev_get(s);
diff --git a/drivers/staging/comedi/drivers.c b/drivers/staging/comedi/drivers.c
index 44511d729450..a5bf2cc165c0 100644
--- a/drivers/staging/comedi/drivers.c
+++ b/drivers/staging/comedi/drivers.c
@@ -15,7 +15,7 @@
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  *  GNU General Public License for more details.
-*/
+ */
 
 #include <linux/device.h>
 #include <linux/module.h>
diff --git a/drivers/staging/comedi/drivers/addi-data/hwdrv_apci3501.c b/drivers/staging/comedi/drivers/addi-data/hwdrv_apci3501.c
deleted file mode 100644
index 375707497896..000000000000
--- a/drivers/staging/comedi/drivers/addi-data/hwdrv_apci3501.c
+++ /dev/null
@@ -1,141 +0,0 @@
-/* Watchdog Related Defines */
-
-#define ADDIDATA_TIMER			0
-#define ADDIDATA_WATCHDOG		2
-
-/*
- * (*insn_config) for the timer subdevice
- *
- * Configures The Timer, Counter or Watchdog
- * Data Pointer contains configuration parameters as below
- *	data[0] : 0 Configure As Timer
- *		  1 Configure As Counter
- *		  2 Configure As Watchdog
- *	data[1] : 1 Enable  Interrupt
- *		  0 Disable Interrupt
- *	data[2] : Time Unit
- *	data[3] : Reload Value
- */
-static int apci3501_config_insn_timer(struct comedi_device *dev,
-				      struct comedi_subdevice *s,
-				      struct comedi_insn *insn,
-				      unsigned int *data)
-{
-	struct apci3501_private *devpriv = dev->private;
-	unsigned int ctrl;
-
-	if (data[0] != ADDIDATA_WATCHDOG &&
-	    data[0] != ADDIDATA_TIMER)
-		return -EINVAL;
-
-	devpriv->tsk_Current = current;
-
-	devpriv->timer_mode = data[0];
-
-	/* first, disable the watchdog or stop the timer */
-	if (devpriv->timer_mode == ADDIDATA_WATCHDOG) {
-		ctrl = 0;
-	} else {
-		ctrl = inl(devpriv->tcw + ADDI_TCW_CTRL_REG);
-		ctrl &= ~(ADDI_TCW_CTRL_GATE | ADDI_TCW_CTRL_TRIG |
-			  ADDI_TCW_CTRL_ENA);
-	}
-	outl(ctrl, devpriv->tcw + ADDI_TCW_CTRL_REG);
-
-	/* enable/disable the timer interrupt */
-	ctrl = (data[1] == 1) ? ADDI_TCW_CTRL_IRQ_ENA : 0;
-	outl(ctrl, devpriv->tcw + ADDI_TCW_CTRL_REG);
-
-	outl(data[2], devpriv->tcw + ADDI_TCW_TIMEBASE_REG);
-	outl(data[3], devpriv->tcw + ADDI_TCW_RELOAD_REG);
-
-	ctrl = inl(devpriv->tcw + ADDI_TCW_CTRL_REG);
-	if (devpriv->timer_mode == ADDIDATA_WATCHDOG) {
-		/* Set the mode (e2->e0) NOTE: this doesn't look correct */
-		ctrl |= ~(ADDI_TCW_CTRL_CNT_UP | ADDI_TCW_CTRL_EXT_CLK_MASK |
-			  ADDI_TCW_CTRL_MODE_MASK | ADDI_TCW_CTRL_GATE |
-			  ADDI_TCW_CTRL_TRIG | ADDI_TCW_CTRL_TIMER_ENA |
-			  ADDI_TCW_CTRL_RESET_ENA | ADDI_TCW_CTRL_WARN_ENA |
-			  ADDI_TCW_CTRL_IRQ_ENA | ADDI_TCW_CTRL_ENA);
-	} else {
-		/* mode 2 */
-		ctrl &= ~(ADDI_TCW_CTRL_CNTR_ENA | ADDI_TCW_CTRL_MODE_MASK |
-			  ADDI_TCW_CTRL_GATE | ADDI_TCW_CTRL_TRIG |
-			  ADDI_TCW_CTRL_TIMER_ENA | ADDI_TCW_CTRL_RESET_ENA |
-			  ADDI_TCW_CTRL_WARN_ENA | ADDI_TCW_CTRL_ENA);
-		ctrl |= ADDI_TCW_CTRL_MODE(2) | ADDI_TCW_CTRL_TIMER_ENA;
-	}
-	outl(ctrl, devpriv->tcw + ADDI_TCW_CTRL_REG);
-
-	return insn->n;
-}
-
-/*
- * (*insn_write) for the timer subdevice
- *
- * Start / Stop The Selected Timer , Counter or Watchdog
- * Data Pointer contains configuration parameters as below
- *	data[0] : 0 Timer
- *		  1 Counter
- *		  2 Watchdog
- *	data[1] : 1 Start
- *		  0 Stop
- *		  2 Trigger
- */
-static int apci3501_write_insn_timer(struct comedi_device *dev,
-				     struct comedi_subdevice *s,
-				     struct comedi_insn *insn,
-				     unsigned int *data)
-{
-	struct apci3501_private *devpriv = dev->private;
-	unsigned int ctrl;
-
-	if (devpriv->timer_mode == ADDIDATA_WATCHDOG ||
-	    devpriv->timer_mode == ADDIDATA_TIMER) {
-		ctrl = inl(devpriv->tcw + ADDI_TCW_CTRL_REG);
-		ctrl &= ~(ADDI_TCW_CTRL_GATE | ADDI_TCW_CTRL_TRIG);
-
-		if (data[1] == 1) {		/* enable */
-			ctrl |= ADDI_TCW_CTRL_ENA;
-		} else if (data[1] == 0) {	/* stop */
-			if (devpriv->timer_mode == ADDIDATA_WATCHDOG)
-				ctrl = 0;
-			else
-				ctrl &= ~ADDI_TCW_CTRL_ENA;
-		} else if (data[1] == 2) {	/* trigger */
-			ctrl |= ADDI_TCW_CTRL_TRIG;
-		}
-		outl(ctrl, devpriv->tcw + ADDI_TCW_CTRL_REG);
-	}
-
-	inl(devpriv->tcw + ADDI_TCW_STATUS_REG);
-	return insn->n;
-}
-
-/*
- * (*insn_read) for the timer subdevice
- *
- * Read The Selected Timer, Counter or Watchdog
- * Data Pointer contains configuration parameters as below
- *	data[0] : 0 Timer
- *		  1 Counter
- *		  2 Watchdog
- *	data[1] : Timer Counter Watchdog Number
- */
-static int apci3501_read_insn_timer(struct comedi_device *dev,
-				    struct comedi_subdevice *s,
-				    struct comedi_insn *insn,
-				    unsigned int *data)
-{
-	struct apci3501_private *devpriv = dev->private;
-
-	if (devpriv->timer_mode != ADDIDATA_TIMER &&
-	    devpriv->timer_mode != ADDIDATA_WATCHDOG)
-		return -EINVAL;
-
-	data[0] = inl(devpriv->tcw + ADDI_TCW_STATUS_REG) &
-		  ADDI_TCW_STATUS_OVERFLOW;
-	data[1] = inl(devpriv->tcw + ADDI_TCW_VAL_REG);
-
-	return insn->n;
-}
diff --git a/drivers/staging/comedi/drivers/addi_apci_3501.c b/drivers/staging/comedi/drivers/addi_apci_3501.c
index 40ff91411139..57f0f46de0be 100644
--- a/drivers/staging/comedi/drivers/addi_apci_3501.c
+++ b/drivers/staging/comedi/drivers/addi_apci_3501.c
@@ -22,12 +22,36 @@
  * more details.
  */
 
+/*
+ * Driver: addi_apci_3501
+ * Description: ADDI-DATA APCI-3501 Analog output board
+ * Devices: [ADDI-DATA] APCI-3501 (addi_apci_3501)
+ * Author: H Hartley Sweeten <hsweeten@visionengravers.com>
+ * Updated: Mon, 20 Jun 2016 10:57:01 -0700
+ * Status: untested
+ *
+ * Configuration Options: not applicable, uses comedi PCI auto config
+ *
+ * This board has the following features:
+ *   - 4 or 8 analog output channels
+ *   - 2 optically isolated digital inputs
+ *   - 2 optically isolated digital outputs
+ *   - 1 12-bit watchdog/timer
+ *
+ * There are 2 versions of the APCI-3501:
+ *   - APCI-3501-4  4 analog output channels
+ *   - APCI-3501-8  8 analog output channels
+ *
+ * These boards use the same PCI Vendor/Device IDs. The number of output
+ * channels used by this driver is determined by reading the EEPROM on
+ * the board.
+ *
+ * The watchdog/timer subdevice is not currently supported.
+ */
+
 #include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/sched.h>
 
 #include "../comedi_pci.h"
-#include "addi_tcw.h"
 #include "amcc_s5933.h"
 
 /*
@@ -67,8 +91,6 @@
 
 struct apci3501_private {
 	unsigned long amcc;
-	unsigned long tcw;
-	struct task_struct *tsk_Current;
 	unsigned char timer_mode;
 };
 
@@ -139,8 +161,6 @@ static int apci3501_ao_insn_write(struct comedi_device *dev,
 	return insn->n;
 }
 
-#include "addi-data/hwdrv_apci3501.c"
-
 static int apci3501_di_insn_bits(struct comedi_device *dev,
 				 struct comedi_subdevice *s,
 				 struct comedi_insn *insn,
@@ -253,37 +273,6 @@ static int apci3501_eeprom_insn_read(struct comedi_device *dev,
 	return insn->n;
 }
 
-static irqreturn_t apci3501_interrupt(int irq, void *d)
-{
-	struct comedi_device *dev = d;
-	struct apci3501_private *devpriv = dev->private;
-	unsigned int status;
-	unsigned int ctrl;
-
-	/*  Disable Interrupt */
-	ctrl = inl(devpriv->tcw + ADDI_TCW_CTRL_REG);
-	ctrl &= ~(ADDI_TCW_CTRL_GATE | ADDI_TCW_CTRL_TRIG |
-		  ADDI_TCW_CTRL_IRQ_ENA);
-	outl(ctrl, devpriv->tcw + ADDI_TCW_CTRL_REG);
-
-	status = inl(devpriv->tcw + ADDI_TCW_IRQ_REG);
-	if (!(status & ADDI_TCW_IRQ)) {
-		dev_err(dev->class_dev, "IRQ from unknown source\n");
-		return IRQ_NONE;
-	}
-
-	/* Enable Interrupt Send a signal to from kernel to user space */
-	send_sig(SIGIO, devpriv->tsk_Current, 0);
-	ctrl = inl(devpriv->tcw + ADDI_TCW_CTRL_REG);
-	ctrl &= ~(ADDI_TCW_CTRL_GATE | ADDI_TCW_CTRL_TRIG |
-		  ADDI_TCW_CTRL_IRQ_ENA);
-	ctrl |= ADDI_TCW_CTRL_IRQ_ENA;
-	outl(ctrl, devpriv->tcw + ADDI_TCW_CTRL_REG);
-	inl(devpriv->tcw + ADDI_TCW_STATUS_REG);
-
-	return IRQ_HANDLED;
-}
-
 static int apci3501_reset(struct comedi_device *dev)
 {
 	unsigned int val;
@@ -333,17 +322,9 @@ static int apci3501_auto_attach(struct comedi_device *dev,
 
 	devpriv->amcc = pci_resource_start(pcidev, 0);
 	dev->iobase = pci_resource_start(pcidev, 1);
-	devpriv->tcw = dev->iobase + APCI3501_TIMER_BASE;
 
 	ao_n_chan = apci3501_eeprom_get_ao_n_chan(dev);
 
-	if (pcidev->irq > 0) {
-		ret = request_irq(pcidev->irq, apci3501_interrupt, IRQF_SHARED,
-				  dev->board_name, dev);
-		if (ret == 0)
-			dev->irq = pcidev->irq;
-	}
-
 	ret = comedi_alloc_subdevices(dev, 5);
 	if (ret)
 		return ret;
@@ -383,17 +364,9 @@ static int apci3501_auto_attach(struct comedi_device *dev,
 	s->range_table	= &range_digital;
 	s->insn_bits	= apci3501_do_insn_bits;
 
-	/* Initialize the timer/watchdog subdevice */
+	/* Timer/Watchdog subdevice */
 	s = &dev->subdevices[3];
-	s->type = COMEDI_SUBD_TIMER;
-	s->subdev_flags = SDF_WRITABLE;
-	s->n_chan = 1;
-	s->maxdata = 0;
-	s->len_chanlist = 1;
-	s->range_table = &range_digital;
-	s->insn_write = apci3501_write_insn_timer;
-	s->insn_read = apci3501_read_insn_timer;
-	s->insn_config = apci3501_config_insn_timer;
+	s->type		= COMEDI_SUBD_UNUSED;
 
 	/* Initialize the eeprom subdevice */
 	s = &dev->subdevices[4];
diff --git a/drivers/staging/comedi/drivers/adl_pci9118.c b/drivers/staging/comedi/drivers/adl_pci9118.c
index be70bd333807..86450c08f291 100644
--- a/drivers/staging/comedi/drivers/adl_pci9118.c
+++ b/drivers/staging/comedi/drivers/adl_pci9118.c
@@ -1693,8 +1693,7 @@ static void pci9118_detach(struct comedi_device *dev)
 		pci9118_reset(dev);
 	comedi_pci_detach(dev);
 	pci9118_free_dma(dev);
-	if (pcidev)
-		pci_dev_put(pcidev);
+	pci_dev_put(pcidev);
 }
 
 static struct comedi_driver adl_pci9118_driver = {
diff --git a/drivers/staging/comedi/drivers/adv_pci1760.c b/drivers/staging/comedi/drivers/adv_pci1760.c
index d7dd1e55e347..9f525ff7290c 100644
--- a/drivers/staging/comedi/drivers/adv_pci1760.c
+++ b/drivers/staging/comedi/drivers/adv_pci1760.c
@@ -196,6 +196,7 @@ static int pci1760_pwm_ns_to_div(unsigned int flags, unsigned int ns)
 		break;
 	case CMDF_ROUND_DOWN:
 		divisor = ns / PCI1760_PWM_TIMEBASE;
+		break;
 	default:
 		return -EINVAL;
 	}
diff --git a/drivers/staging/comedi/drivers/cb_pcidas64.c b/drivers/staging/comedi/drivers/cb_pcidas64.c
index 1f9c08a845b6..cb9c2699277e 100644
--- a/drivers/staging/comedi/drivers/cb_pcidas64.c
+++ b/drivers/staging/comedi/drivers/cb_pcidas64.c
@@ -1,34 +1,34 @@
 /*
-    comedi/drivers/cb_pcidas64.c
-    This is a driver for the ComputerBoards/MeasurementComputing PCI-DAS
-    64xx, 60xx, and 4020 cards.
-
-    Author:  Frank Mori Hess <fmhess@users.sourceforge.net>
-    Copyright (C) 2001, 2002 Frank Mori Hess
-
-    Thanks also go to the following people:
-
-    Steve Rosenbluth, for providing the source code for
-    his pci-das6402 driver, and source code for working QNX pci-6402
-    drivers by Greg Laird and Mariusz Bogacz.  None of the code was
-    used directly here, but it was useful as an additional source of
-    documentation on how to program the boards.
-
-    John Sims, for much testing and feedback on pcidas-4020 support.
-
-    COMEDI - Linux Control and Measurement Device Interface
-    Copyright (C) 1997-8 David A. Schleef <ds@schleef.org>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-*/
+ * comedi/drivers/cb_pcidas64.c
+ * This is a driver for the ComputerBoards/MeasurementComputing PCI-DAS
+ * 64xx, 60xx, and 4020 cards.
+ *
+ * Author:  Frank Mori Hess <fmhess@users.sourceforge.net>
+ * Copyright (C) 2001, 2002 Frank Mori Hess
+ *
+ * Thanks also go to the following people:
+ *
+ * Steve Rosenbluth, for providing the source code for
+ * his pci-das6402 driver, and source code for working QNX pci-6402
+ * drivers by Greg Laird and Mariusz Bogacz.  None of the code was
+ * used directly here, but it was useful as an additional source of
+ * documentation on how to program the boards.
+ *
+ * John Sims, for much testing and feedback on pcidas-4020 support.
+ *
+ * COMEDI - Linux Control and Measurement Device Interface
+ * Copyright (C) 1997-8 David A. Schleef <ds@schleef.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
 
 /*
  * Driver: cb_pcidas64
@@ -66,19 +66,18 @@
  */
 
 /*
-
-TODO:
-	make it return error if user attempts an ai command that uses the
-	external queue, and an ao command simultaneously user counter subdevice
-	there are a number of boards this driver will support when they are
-	fully released, but does not yet since the pci device id numbers
-	are not yet available.
-
-	support prescaled 100khz clock for slow pacing (not available on 6000
-	series?)
-
-	make ao fifo size adjustable like ai fifo
-*/
+ * TODO:
+ * make it return error if user attempts an ai command that uses the
+ * external queue, and an ao command simultaneously user counter subdevice
+ * there are a number of boards this driver will support when they are
+ * fully released, but does not yet since the pci device id numbers
+ * are not yet available.
+ *
+ * support prescaled 100khz clock for slow pacing (not available on 6000
+ * series?)
+ *
+ * make ao fifo size adjustable like ai fifo
+ */
 
 #include <linux/module.h>
 #include <linux/delay.h>
@@ -90,53 +89,56 @@ TODO:
 #include "plx9080.h"
 
 #define TIMER_BASE 25		/*  40MHz master clock */
-/* 100kHz 'prescaled' clock for slow acquisition,
- * maybe I'll support this someday */
+/*
+ * 100kHz 'prescaled' clock for slow acquisition,
+ * maybe I'll support this someday
+ */
 #define PRESCALED_TIMER_BASE	10000
-#define DMA_BUFFER_SIZE 0x1000
+#define DMA_BUFFER_SIZE		0x1000
+#define DAC_FIFO_SIZE		0x2000
 
-/* maximum value that can be loaded into board's 24-bit counters*/
+/* maximum value that can be loaded into board's 24-bit counters */
 static const int max_counter_value = 0xffffff;
 
 /* PCI-DAS64xxx base addresses */
 
 /* devpriv->main_iobase registers */
 enum write_only_registers {
-	INTR_ENABLE_REG = 0x0,	/*  interrupt enable register */
-	HW_CONFIG_REG = 0x2,	/*  hardware config register */
+	INTR_ENABLE_REG = 0x0,		/* interrupt enable register */
+	HW_CONFIG_REG = 0x2,		/* hardware config register */
 	DAQ_SYNC_REG = 0xc,
 	DAQ_ATRIG_LOW_4020_REG = 0xc,
-	ADC_CONTROL0_REG = 0x10,	/*  adc control register 0 */
-	ADC_CONTROL1_REG = 0x12,	/*  adc control register 1 */
+	ADC_CONTROL0_REG = 0x10,	/* adc control register 0 */
+	ADC_CONTROL1_REG = 0x12,	/* adc control register 1 */
 	CALIBRATION_REG = 0x14,
-	/*  lower 16 bits of adc sample interval counter */
+	/* lower 16 bits of adc sample interval counter */
 	ADC_SAMPLE_INTERVAL_LOWER_REG = 0x16,
-	/*  upper 8 bits of adc sample interval counter */
+	/* upper 8 bits of adc sample interval counter */
 	ADC_SAMPLE_INTERVAL_UPPER_REG = 0x18,
-	/*  lower 16 bits of delay interval counter */
+	/* lower 16 bits of delay interval counter */
 	ADC_DELAY_INTERVAL_LOWER_REG = 0x1a,
-	/*  upper 8 bits of delay interval counter */
+	/* upper 8 bits of delay interval counter */
 	ADC_DELAY_INTERVAL_UPPER_REG = 0x1c,
-	/*  lower 16 bits of hardware conversion/scan counter */
+	/* lower 16 bits of hardware conversion/scan counter */
 	ADC_COUNT_LOWER_REG = 0x1e,
-	/*  upper 8 bits of hardware conversion/scan counter */
+	/* upper 8 bits of hardware conversion/scan counter */
 	ADC_COUNT_UPPER_REG = 0x20,
-	ADC_START_REG = 0x22,	/*  software trigger to start acquisition */
-	ADC_CONVERT_REG = 0x24,	/*  initiates single conversion */
-	ADC_QUEUE_CLEAR_REG = 0x26,	/*  clears adc queue */
-	ADC_QUEUE_LOAD_REG = 0x28,	/*  loads adc queue */
+	ADC_START_REG = 0x22,	/* software trigger to start acquisition */
+	ADC_CONVERT_REG = 0x24,	/* initiates single conversion */
+	ADC_QUEUE_CLEAR_REG = 0x26,	/* clears adc queue */
+	ADC_QUEUE_LOAD_REG = 0x28,	/* loads adc queue */
 	ADC_BUFFER_CLEAR_REG = 0x2a,
-	/*  high channel for internal queue, use adc_chan_bits() inline above */
+	/* high channel for internal queue, use adc_chan_bits() inline above */
 	ADC_QUEUE_HIGH_REG = 0x2c,
-	DAC_CONTROL0_REG = 0x50,	/*  dac control register 0 */
-	DAC_CONTROL1_REG = 0x52,	/*  dac control register 0 */
-	/*  lower 16 bits of dac sample interval counter */
+	DAC_CONTROL0_REG = 0x50,	/* dac control register 0 */
+	DAC_CONTROL1_REG = 0x52,	/* dac control register 0 */
+	/* lower 16 bits of dac sample interval counter */
 	DAC_SAMPLE_INTERVAL_LOWER_REG = 0x54,
-	/*  upper 8 bits of dac sample interval counter */
+	/* upper 8 bits of dac sample interval counter */
 	DAC_SAMPLE_INTERVAL_UPPER_REG = 0x56,
 	DAC_SELECT_REG = 0x60,
 	DAC_START_REG = 0x64,
-	DAC_BUFFER_CLEAR_REG = 0x66,	/*  clear dac buffer */
+	DAC_BUFFER_CLEAR_REG = 0x66,	/* clear dac buffer */
 };
 
 static inline unsigned int dac_convert_reg(unsigned int channel)
@@ -168,8 +170,8 @@ enum read_only_registers {
 };
 
 enum read_write_registers {
-	I8255_4020_REG = 0x48,	/*  8255 offset, for 4020 only */
-	/*  external channel/gain queue, uses same bits as ADC_QUEUE_LOAD_REG */
+	I8255_4020_REG = 0x48,	/* 8255 offset, for 4020 only */
+	/* external channel/gain queue, uses same bits as ADC_QUEUE_LOAD_REG */
 	ADC_QUEUE_FIFO_REG = 0x100,
 	ADC_FIFO_REG = 0x200,	/* adc data fifo */
 	/* dac data fifo, has weird interactions with external channel queue */
@@ -188,50 +190,51 @@ enum dio_counter_registers {
 /* bit definitions for write-only registers */
 
 enum intr_enable_contents {
-	ADC_INTR_SRC_MASK = 0x3,	/*  adc interrupt source mask */
-	ADC_INTR_QFULL_BITS = 0x0,	/*  interrupt fifo quarter full */
-	ADC_INTR_EOC_BITS = 0x1,	/*  interrupt end of conversion */
-	ADC_INTR_EOSCAN_BITS = 0x2,	/*  interrupt end of scan */
-	ADC_INTR_EOSEQ_BITS = 0x3,	/*  interrupt end of sequence mask */
-	EN_ADC_INTR_SRC_BIT = 0x4,	/*  enable adc interrupt source */
-	EN_ADC_DONE_INTR_BIT = 0x8,	/*  enable adc acquisition done intr */
+	ADC_INTR_SRC_MASK = 0x3,	/* adc interrupt source mask */
+	ADC_INTR_QFULL_BITS = 0x0,	/* interrupt fifo quarter full */
+	ADC_INTR_EOC_BITS = 0x1,	/* interrupt end of conversion */
+	ADC_INTR_EOSCAN_BITS = 0x2,	/* interrupt end of scan */
+	ADC_INTR_EOSEQ_BITS = 0x3,	/* interrupt end of sequence mask */
+	EN_ADC_INTR_SRC_BIT = 0x4,	/* enable adc interrupt source */
+	EN_ADC_DONE_INTR_BIT = 0x8,	/* enable adc acquisition done intr */
 	DAC_INTR_SRC_MASK = 0x30,
 	DAC_INTR_QEMPTY_BITS = 0x0,
 	DAC_INTR_HIGH_CHAN_BITS = 0x10,
-	EN_DAC_INTR_SRC_BIT = 0x40,	/*  enable dac interrupt source */
+	EN_DAC_INTR_SRC_BIT = 0x40,	/* enable dac interrupt source */
 	EN_DAC_DONE_INTR_BIT = 0x80,
-	EN_ADC_ACTIVE_INTR_BIT = 0x200,	/*  enable adc active interrupt */
-	EN_ADC_STOP_INTR_BIT = 0x400,	/*  enable adc stop trigger interrupt */
-	EN_DAC_ACTIVE_INTR_BIT = 0x800,	/*  enable dac active interrupt */
-	EN_DAC_UNDERRUN_BIT = 0x4000,	/*  enable dac underrun status bit */
-	EN_ADC_OVERRUN_BIT = 0x8000,	/*  enable adc overrun status bit */
+	EN_ADC_ACTIVE_INTR_BIT = 0x200,	/* enable adc active interrupt */
+	EN_ADC_STOP_INTR_BIT = 0x400,	/* enable adc stop trigger interrupt */
+	EN_DAC_ACTIVE_INTR_BIT = 0x800,	/* enable dac active interrupt */
+	EN_DAC_UNDERRUN_BIT = 0x4000,	/* enable dac underrun status bit */
+	EN_ADC_OVERRUN_BIT = 0x8000,	/* enable adc overrun status bit */
 };
 
 enum hw_config_contents {
-	MASTER_CLOCK_4020_MASK = 0x3,	/*  master clock source mask for 4020 */
-	INTERNAL_CLOCK_4020_BITS = 0x1,	/*  use 40 MHz internal master clock */
-	BNC_CLOCK_4020_BITS = 0x2,	/*  use BNC input for master clock */
-	EXT_CLOCK_4020_BITS = 0x3,	/*  use dio input for master clock */
-	EXT_QUEUE_BIT = 0x200,		/*  use external channel/gain queue */
-	/*  use 225 nanosec strobe when loading dac instead of 50 nanosec */
+	MASTER_CLOCK_4020_MASK = 0x3,	/* master clock source mask for 4020 */
+	INTERNAL_CLOCK_4020_BITS = 0x1,	/* use 40 MHz internal master clock */
+	BNC_CLOCK_4020_BITS = 0x2,	/* use BNC input for master clock */
+	EXT_CLOCK_4020_BITS = 0x3,	/* use dio input for master clock */
+	EXT_QUEUE_BIT = 0x200,		/* use external channel/gain queue */
+	/* use 225 nanosec strobe when loading dac instead of 50 nanosec */
 	SLOW_DAC_BIT = 0x400,
-	/*  bit with unknown function yet given as default value in pci-das64
-	 *  manual */
+	/*
+	 * bit with unknown function yet given as default value in pci-das64
+	 * manual
+	 */
 	HW_CONFIG_DUMMY_BITS = 0x2000,
-	/*  bit selects channels 1/0 for analog input/output, otherwise 0/1 */
+	/* bit selects channels 1/0 for analog input/output, otherwise 0/1 */
 	DMA_CH_SELECT_BIT = 0x8000,
-	FIFO_SIZE_REG = 0x4,		/*  allows adjustment of fifo sizes */
-	DAC_FIFO_SIZE_MASK = 0xff00,	/*  bits that set dac fifo size */
-	DAC_FIFO_BITS = 0xf800,		/*  8k sample ao fifo */
+	FIFO_SIZE_REG = 0x4,		/* allows adjustment of fifo sizes */
+	DAC_FIFO_SIZE_MASK = 0xff00,	/* bits that set dac fifo size */
+	DAC_FIFO_BITS = 0xf800,		/* 8k sample ao fifo */
 };
-#define DAC_FIFO_SIZE 0x2000
 
 enum daq_atrig_low_4020_contents {
-	/*  use trig/ext clk bnc input for analog gate signal */
+	/* use trig/ext clk bnc input for analog gate signal */
 	EXT_AGATE_BNC_BIT = 0x8000,
-	/*  use trig/ext clk bnc input for external stop trigger signal */
+	/* use trig/ext clk bnc input for external stop trigger signal */
 	EXT_STOP_TRIG_BNC_BIT = 0x4000,
-	/*  use trig/ext clk bnc input for external start trigger signal */
+	/* use trig/ext clk bnc input for external start trigger signal */
 	EXT_START_TRIG_BNC_BIT = 0x2000,
 };
 
@@ -241,38 +244,38 @@ static inline uint16_t analog_trig_low_threshold_bits(uint16_t threshold)
 }
 
 enum adc_control0_contents {
-	ADC_GATE_SRC_MASK = 0x3,	/*  bits that select gate */
-	ADC_SOFT_GATE_BITS = 0x1,	/*  software gate */
-	ADC_EXT_GATE_BITS = 0x2,	/*  external digital gate */
-	ADC_ANALOG_GATE_BITS = 0x3,	/*  analog level gate */
-	/*  level-sensitive gate (for digital) */
+	ADC_GATE_SRC_MASK = 0x3,	/* bits that select gate */
+	ADC_SOFT_GATE_BITS = 0x1,	/* software gate */
+	ADC_EXT_GATE_BITS = 0x2,	/* external digital gate */
+	ADC_ANALOG_GATE_BITS = 0x3,	/* analog level gate */
+	/* level-sensitive gate (for digital) */
 	ADC_GATE_LEVEL_BIT = 0x4,
-	ADC_GATE_POLARITY_BIT = 0x8,	/*  gate active low */
+	ADC_GATE_POLARITY_BIT = 0x8,	/* gate active low */
 	ADC_START_TRIG_SOFT_BITS = 0x10,
 	ADC_START_TRIG_EXT_BITS = 0x20,
 	ADC_START_TRIG_ANALOG_BITS = 0x30,
 	ADC_START_TRIG_MASK = 0x30,
-	ADC_START_TRIG_FALLING_BIT = 0x40,	/*  trig 1 uses falling edge */
-	/*  external pacing uses falling edge */
+	ADC_START_TRIG_FALLING_BIT = 0x40,	/* trig 1 uses falling edge */
+	/* external pacing uses falling edge */
 	ADC_EXT_CONV_FALLING_BIT = 0x800,
-	/*  enable hardware scan counter */
+	/* enable hardware scan counter */
 	ADC_SAMPLE_COUNTER_EN_BIT = 0x1000,
-	ADC_DMA_DISABLE_BIT = 0x4000,	/*  disables dma */
-	ADC_ENABLE_BIT = 0x8000,	/*  master adc enable */
+	ADC_DMA_DISABLE_BIT = 0x4000,	/* disables dma */
+	ADC_ENABLE_BIT = 0x8000,	/* master adc enable */
 };
 
 enum adc_control1_contents {
-	/*  should be set for boards with > 16 channels */
+	/* should be set for boards with > 16 channels */
 	ADC_QUEUE_CONFIG_BIT = 0x1,
 	CONVERT_POLARITY_BIT = 0x10,
 	EOC_POLARITY_BIT = 0x20,
-	ADC_SW_GATE_BIT = 0x40,	/*  software gate of adc */
-	ADC_DITHER_BIT = 0x200,	/*  turn on extra noise for dithering */
+	ADC_SW_GATE_BIT = 0x40,		/* software gate of adc */
+	ADC_DITHER_BIT = 0x200,		/* turn on extra noise for dithering */
 	RETRIGGER_BIT = 0x800,
 	ADC_LO_CHANNEL_4020_MASK = 0x300,
 	ADC_HI_CHANNEL_4020_MASK = 0xc00,
-	TWO_CHANNEL_4020_BITS = 0x1000,	/*  two channel mode for 4020 */
-	FOUR_CHANNEL_4020_BITS = 0x2000, /*  four channel mode for 4020 */
+	TWO_CHANNEL_4020_BITS = 0x1000,		/* two channel mode for 4020 */
+	FOUR_CHANNEL_4020_BITS = 0x2000,	/* four channel mode for 4020 */
 	CHANNEL_MODE_4020_MASK = 0x3000,
 	ADC_MODE_MASK = 0xf000,
 };
@@ -296,10 +299,10 @@ enum calibration_contents {
 	SELECT_8800_BIT = 0x1,
 	SELECT_8402_64XX_BIT = 0x2,
 	SELECT_1590_60XX_BIT = 0x2,
-	CAL_EN_64XX_BIT = 0x40,	/*  calibration enable for 64xx series */
+	CAL_EN_64XX_BIT = 0x40,		/* calibration enable for 64xx series */
 	SERIAL_DATA_IN_BIT = 0x80,
 	SERIAL_CLOCK_BIT = 0x100,
-	CAL_EN_60XX_BIT = 0x200, /*  calibration enable for 60xx series */
+	CAL_EN_60XX_BIT = 0x200,	/* calibration enable for 60xx series */
 	CAL_GAIN_BIT = 0x800,
 };
 
@@ -326,12 +329,12 @@ static inline uint16_t adc_convert_chan_4020_bits(unsigned int channel)
 };
 
 enum adc_queue_load_contents {
-	UNIP_BIT = 0x800,	/*  unipolar/bipolar bit */
-	ADC_SE_DIFF_BIT = 0x1000,	/*  single-ended/ differential bit */
-	/*  non-referenced single-ended (common-mode input) */
+	UNIP_BIT = 0x800,		/* unipolar/bipolar bit */
+	ADC_SE_DIFF_BIT = 0x1000,	/* single-ended/ differential bit */
+	/* non-referenced single-ended (common-mode input) */
 	ADC_COMMON_BIT = 0x2000,
-	QUEUE_EOSEQ_BIT = 0x4000,	/*  queue end of sequence */
-	QUEUE_EOSCAN_BIT = 0x8000,	/*  queue end of scan */
+	QUEUE_EOSEQ_BIT = 0x4000,	/* queue end of sequence */
+	QUEUE_EOSCAN_BIT = 0x8000,	/* queue end of scan */
 };
 
 static inline uint16_t adc_chan_bits(unsigned int channel)
@@ -340,7 +343,7 @@ static inline uint16_t adc_chan_bits(unsigned int channel)
 };
 
 enum dac_control0_contents {
-	DAC_ENABLE_BIT = 0x8000,	/*  dac controller enable bit */
+	DAC_ENABLE_BIT = 0x8000,	/* dac controller enable bit */
 	DAC_CYCLIC_STOP_BIT = 0x4000,
 	DAC_WAVEFORM_MODE_BIT = 0x100,
 	DAC_EXT_UPDATE_FALLING_BIT = 0x80,
@@ -360,7 +363,7 @@ enum dac_control1_contents {
 	DAC_WRITE_POLARITY_BIT = 0x800,	/* board-dependent setting */
 	DAC1_EXT_REF_BIT = 0x200,
 	DAC0_EXT_REF_BIT = 0x100,
-	DAC_OUTPUT_ENABLE_BIT = 0x80,	/*  dac output enable bit */
+	DAC_OUTPUT_ENABLE_BIT = 0x80,	/* dac output enable bit */
 	DAC_UPDATE_POLARITY_BIT = 0x40,	/* board-dependent setting */
 	DAC_SW_GATE_BIT = 0x20,
 	DAC1_UNIPOLAR_BIT = 0x8,
@@ -409,9 +412,9 @@ enum i2c_addresses {
 };
 
 enum range_cal_i2c_contents {
-	/*  bits that set what source the adc converter measures */
+	/* bits that set what source the adc converter measures */
 	ADC_SRC_4020_MASK = 0x70,
-	/*  make bnc trig/ext clock threshold 0V instead of 2.5V */
+	/* make bnc trig/ext clock threshold 0V instead of 2.5V */
 	BNC_TRIG_THRESHOLD_0V_BIT = 0x80,
 };
 
@@ -422,7 +425,7 @@ static inline uint8_t adc_src_4020_bits(unsigned int source)
 
 static inline uint8_t attenuate_bit(unsigned int channel)
 {
-	/*  attenuate channel (+-5V input range) */
+	/* attenuate channel (+-5V input range) */
 	return 1 << (channel & 0x3);
 };
 
@@ -627,18 +630,18 @@ enum pcidas64_boardid {
 
 struct pcidas64_board {
 	const char *name;
-	int ai_se_chans;	/*  number of ai inputs in single-ended mode */
-	int ai_bits;		/*  analog input resolution */
-	int ai_speed;		/*  fastest conversion period in ns */
+	int ai_se_chans;	/* number of ai inputs in single-ended mode */
+	int ai_bits;		/* analog input resolution */
+	int ai_speed;		/* fastest conversion period in ns */
 	const struct comedi_lrange *ai_range_table;
 	const uint8_t *ai_range_code;
-	int ao_nchan;		/*  number of analog out channels */
-	int ao_bits;		/*  analog output resolution */
-	int ao_scan_speed;	/*  analog output scan speed */
+	int ao_nchan;		/* number of analog out channels */
+	int ao_bits;		/* analog output resolution */
+	int ao_scan_speed;	/* analog output scan speed */
 	const struct comedi_lrange *ao_range_table;
 	const int *ao_range_code;
 	const struct hw_fifo_info *const ai_fifo;
-	/*  different board families have slightly different registers */
+	/* different board families have slightly different registers */
 	enum register_layout layout;
 	unsigned has_8255:1;
 };
@@ -699,7 +702,7 @@ static const struct pcidas64_board pcidas64_boards[] = {
 		.has_8255	= 1,
 	},
 	[BOARD_PCIDAS6402_12] = {
-		.name		= "pci-das6402/12",	/*  XXX check */
+		.name		= "pci-das6402/12",	/* XXX check */
 		.ai_se_chans	= 64,
 		.ai_bits	= 12,
 		.ai_speed	= 5000,
@@ -996,7 +999,7 @@ static const struct pcidas64_board pcidas64_boards[] = {
 		.ai_speed	= 50,
 		.ao_bits	= 12,
 		.ao_nchan	= 2,
-		.ao_scan_speed	= 0,	/*  no hardware pacing on ao */
+		.ao_scan_speed	= 0,	/* no hardware pacing on ao */
 		.layout		= LAYOUT_4020,
 		.ai_range_table	= &ai_ranges_4020,
 		.ao_range_table	= &ao_ranges_4020,
@@ -1005,9 +1008,7 @@ static const struct pcidas64_board pcidas64_boards[] = {
 		.has_8255	= 1,
 	},
 #if 0
-	/*
-	 * The device id for these boards is unknown
-	 */
+	/* The device id for these boards is unknown */
 
 	[BOARD_PCIDAS6402_16_JR] = {
 		.name		= "pci-das6402/16/jr",
@@ -1116,62 +1117,66 @@ static inline unsigned short se_diff_bit_6xxx(struct comedi_device *dev,
 }
 
 struct ext_clock_info {
-	/*  master clock divisor to use for scans with external master clock */
+	/* master clock divisor to use for scans with external master clock */
 	unsigned int divisor;
-	/*  chanspec for master clock input when used as scan begin src */
+	/* chanspec for master clock input when used as scan begin src */
 	unsigned int chanspec;
 };
 
 /* this structure is for data unique to this hardware driver. */
 struct pcidas64_private {
-	/*  base addresses (physical) */
+	/* base addresses (physical) */
 	resource_size_t main_phys_iobase;
 	resource_size_t dio_counter_phys_iobase;
-	/*  base addresses (ioremapped) */
+	/* base addresses (ioremapped) */
 	void __iomem *plx9080_iobase;
 	void __iomem *main_iobase;
-	/*  local address (used by dma controller) */
+	/* local address (used by dma controller) */
 	uint32_t local0_iobase;
 	uint32_t local1_iobase;
-	/*  dma buffers for analog input */
+	/* dma buffers for analog input */
 	uint16_t *ai_buffer[MAX_AI_DMA_RING_COUNT];
-	/*  physical addresses of ai dma buffers */
+	/* physical addresses of ai dma buffers */
 	dma_addr_t ai_buffer_bus_addr[MAX_AI_DMA_RING_COUNT];
-	/*  array of ai dma descriptors read by plx9080,
-	 *  allocated to get proper alignment */
+	/*
+	 * array of ai dma descriptors read by plx9080,
+	 * allocated to get proper alignment
+	 */
 	struct plx_dma_desc *ai_dma_desc;
-	/*  physical address of ai dma descriptor array */
+	/* physical address of ai dma descriptor array */
 	dma_addr_t ai_dma_desc_bus_addr;
-	/*  index of the ai dma descriptor/buffer
-	 *  that is currently being used */
+	/*
+	 * index of the ai dma descriptor/buffer
+	 * that is currently being used
+	 */
 	unsigned int ai_dma_index;
-	/*  dma buffers for analog output */
+	/* dma buffers for analog output */
 	uint16_t *ao_buffer[AO_DMA_RING_COUNT];
-	/*  physical addresses of ao dma buffers */
+	/* physical addresses of ao dma buffers */
 	dma_addr_t ao_buffer_bus_addr[AO_DMA_RING_COUNT];
 	struct plx_dma_desc *ao_dma_desc;
 	dma_addr_t ao_dma_desc_bus_addr;
-	/*  keeps track of buffer where the next ao sample should go */
+	/* keeps track of buffer where the next ao sample should go */
 	unsigned int ao_dma_index;
-	unsigned int hw_revision;	/*  stc chip hardware revision number */
-	/*  last bits sent to INTR_ENABLE_REG register */
+	unsigned int hw_revision;	/* stc chip hardware revision number */
+	/* last bits sent to INTR_ENABLE_REG register */
 	unsigned int intr_enable_bits;
-	/*  last bits sent to ADC_CONTROL1_REG register */
+	/* last bits sent to ADC_CONTROL1_REG register */
 	uint16_t adc_control1_bits;
-	/*  last bits sent to FIFO_SIZE_REG register */
+	/* last bits sent to FIFO_SIZE_REG register */
 	uint16_t fifo_size_bits;
-	/*  last bits sent to HW_CONFIG_REG register */
+	/* last bits sent to HW_CONFIG_REG register */
 	uint16_t hw_config_bits;
 	uint16_t dac_control1_bits;
-	/*  last bits written to plx9080 control register */
+	/* last bits written to plx9080 control register */
 	uint32_t plx_control_bits;
-	/*  last bits written to plx interrupt control and status register */
+	/* last bits written to plx interrupt control and status register */
 	uint32_t plx_intcsr_bits;
-	/*  index of calibration source readable through ai ch0 */
+	/* index of calibration source readable through ai ch0 */
 	int calibration_source;
-	/*  bits written to i2c calibration/range register */
+	/* bits written to i2c calibration/range register */
 	uint8_t i2c_cal_range_bits;
-	/*  configure digital triggers to trigger on falling edge */
+	/* configure digital triggers to trigger on falling edge */
 	unsigned int ext_trig_falling;
 	short ai_cmd_running;
 	unsigned int ai_fifo_segment_length;
@@ -1224,7 +1229,7 @@ static void abort_dma(struct comedi_device *dev, unsigned int channel)
 	struct pcidas64_private *devpriv = dev->private;
 	unsigned long flags;
 
-	/*  spinlock for plx dma control/status reg */
+	/* spinlock for plx dma control/status reg */
 	spin_lock_irqsave(&dev->spinlock, flags);
 
 	plx9080_abort_dma(devpriv->plx9080_iobase, channel);
@@ -1271,7 +1276,7 @@ static void enable_ai_interrupts(struct comedi_device *dev,
 	 * if CMDF_WAKE_EOS flag is set.
 	 */
 	if (cmd->flags & CMDF_WAKE_EOS) {
-		/*  4020 doesn't support pio transfers except for fifo dregs */
+		/* 4020 doesn't support pio transfers except for fifo dregs */
 		if (board->layout != LAYOUT_4020)
 			bits |= ADC_INTR_EOSCAN_BITS | EN_ADC_INTR_SRC_BIT;
 	}
@@ -1305,36 +1310,40 @@ static void init_plx9080(struct comedi_device *dev)
 	abort_dma(dev, 0);
 	abort_dma(dev, 1);
 
-	/*  configure dma0 mode */
+	/* configure dma0 mode */
 	bits = 0;
-	/*  enable ready input, not sure if this is necessary */
+	/* enable ready input, not sure if this is necessary */
 	bits |= PLX_DMAMODE_READYIEN;
-	/*  enable bterm, not sure if this is necessary */
+	/* enable bterm, not sure if this is necessary */
 	bits |= PLX_DMAMODE_BTERMIEN;
-	/*  enable dma chaining */
+	/* enable dma chaining */
 	bits |= PLX_DMAMODE_CHAINEN;
-	/*  enable interrupt on dma done
-	 *  (probably don't need this, since chain never finishes) */
+	/*
+	 * enable interrupt on dma done
+	 * (probably don't need this, since chain never finishes)
+	 */
 	bits |= PLX_DMAMODE_DONEIEN;
-	/*  don't increment local address during transfers
-	 *  (we are transferring from a fixed fifo register) */
+	/*
+	 * don't increment local address during transfers
+	 * (we are transferring from a fixed fifo register)
+	 */
 	bits |= PLX_DMAMODE_LACONST;
-	/*  route dma interrupt to pci bus */
+	/* route dma interrupt to pci bus */
 	bits |= PLX_DMAMODE_INTRPCI;
-	/*  enable demand mode */
+	/* enable demand mode */
 	bits |= PLX_DMAMODE_DEMAND;
-	/*  enable local burst mode */
+	/* enable local burst mode */
 	bits |= PLX_DMAMODE_BURSTEN;
-	/*  4020 uses 32 bit dma */
+	/* 4020 uses 32 bit dma */
 	if (board->layout == LAYOUT_4020)
-		bits |= PLX_DMAMODE_WIDTH32;
-	else		/*  localspace0 bus is 16 bits wide */
-		bits |= PLX_DMAMODE_WIDTH16;
+		bits |= PLX_DMAMODE_WIDTH_32;
+	else				/* localspace0 bus is 16 bits wide */
+		bits |= PLX_DMAMODE_WIDTH_16;
 	writel(bits, plx_iobase + PLX_REG_DMAMODE1);
 	if (ao_cmd_is_supported(board))
 		writel(bits, plx_iobase + PLX_REG_DMAMODE0);
 
-	/*  enable interrupts on plx 9080 */
+	/* enable interrupts on plx 9080 */
 	devpriv->plx_intcsr_bits |=
 	    PLX_INTCSR_LSEABORTEN | PLX_INTCSR_LSEPARITYEN | PLX_INTCSR_PIEN |
 	    PLX_INTCSR_PLIEN | PLX_INTCSR_PABORTIEN | PLX_INTCSR_LIOEN |
@@ -1376,7 +1385,7 @@ static int set_ai_fifo_segment_length(struct comedi_device *dev,
 	if (num_entries > fifo->max_segment_length)
 		num_entries = fifo->max_segment_length;
 
-	/*  1 == 256 entries, 2 == 512 entries, etc */
+	/* 1 == 256 entries, 2 == 512 entries, etc */
 	num_increments = DIV_ROUND_CLOSEST(num_entries, increment_size);
 
 	bits = (~(num_increments - 1)) & fifo->fifo_size_reg_mask;
@@ -1442,7 +1451,7 @@ static void init_stc_registers(struct comedi_device *dev)
 	writew(devpriv->adc_control1_bits,
 	       devpriv->main_iobase + ADC_CONTROL1_REG);
 
-	/*  6402/16 manual says this register must be initialized to 0xff? */
+	/* 6402/16 manual says this register must be initialized to 0xff? */
 	writew(0xff, devpriv->main_iobase + ADC_SAMPLE_INTERVAL_UPPER_REG);
 
 	bits = SLOW_DAC_BIT | DMA_CH_SELECT_BIT;
@@ -1457,7 +1466,7 @@ static void init_stc_registers(struct comedi_device *dev)
 
 	spin_unlock_irqrestore(&dev->spinlock, flags);
 
-	/*  set fifos to maximum size */
+	/* set fifos to maximum size */
 	devpriv->fifo_size_bits |= DAC_FIFO_BITS;
 	set_ai_fifo_segment_length(dev, board->ai_fifo->max_segment_length);
 
@@ -1478,7 +1487,7 @@ static int alloc_and_init_dma_members(struct comedi_device *dev)
 	struct pcidas64_private *devpriv = dev->private;
 	int i;
 
-	/*  allocate pci dma buffers */
+	/* allocate pci dma buffers */
 	for (i = 0; i < ai_dma_ring_count(board); i++) {
 		devpriv->ai_buffer[i] =
 			dma_alloc_coherent(&pcidev->dev, DMA_BUFFER_SIZE,
@@ -1499,7 +1508,7 @@ static int alloc_and_init_dma_members(struct comedi_device *dev)
 				return -ENOMEM;
 		}
 	}
-	/*  allocate dma descriptors */
+	/* allocate dma descriptors */
 	devpriv->ai_dma_desc =
 		dma_alloc_coherent(&pcidev->dev, sizeof(struct plx_dma_desc) *
 				   ai_dma_ring_count(board),
@@ -1517,7 +1526,7 @@ static int alloc_and_init_dma_members(struct comedi_device *dev)
 		if (!devpriv->ao_dma_desc)
 			return -ENOMEM;
 	}
-	/*  initialize dma descriptors */
+	/* initialize dma descriptors */
 	for (i = 0; i < ai_dma_ring_count(board); i++) {
 		devpriv->ai_dma_desc[i].pci_start_addr =
 			cpu_to_le32(devpriv->ai_buffer_bus_addr[i]);
@@ -1618,13 +1627,11 @@ static void i2c_set_sda(struct comedi_device *dev, int state)
 	void __iomem *plx_control_addr = devpriv->plx9080_iobase +
 					 PLX_REG_CNTRL;
 
-	if (state) {
-		/*  set data line high */
+	if (state) {				/* set data line high */
 		devpriv->plx_control_bits &= ~data_bit;
 		writel(devpriv->plx_control_bits, plx_control_addr);
 		udelay(i2c_high_udelay);
-	} else {		/*  set data line low */
-
+	} else {				/* set data line low */
 		devpriv->plx_control_bits |= data_bit;
 		writel(devpriv->plx_control_bits, plx_control_addr);
 		udelay(i2c_low_udelay);
@@ -1639,13 +1646,11 @@ static void i2c_set_scl(struct comedi_device *dev, int state)
 	void __iomem *plx_control_addr = devpriv->plx9080_iobase +
 					 PLX_REG_CNTRL;
 
-	if (state) {
-		/*  set clock line high */
+	if (state) {				/* set clock line high */
 		devpriv->plx_control_bits &= ~clock_bit;
 		writel(devpriv->plx_control_bits, plx_control_addr);
 		udelay(i2c_high_udelay);
-	} else {		/*  set clock line low */
-
+	} else {				/* set clock line low */
 		devpriv->plx_control_bits |= clock_bit;
 		writel(devpriv->plx_control_bits, plx_control_addr);
 		udelay(i2c_low_udelay);
@@ -1674,7 +1679,7 @@ static int i2c_read_ack(struct comedi_device *dev)
 	i2c_set_sda(dev, 1);
 	i2c_set_scl(dev, 1);
 
-	return 0;		/*  return fake acknowledge bit */
+	return 0;		/* return fake acknowledge bit */
 }
 
 /* send start bit */
@@ -1707,23 +1712,23 @@ static void i2c_write(struct comedi_device *dev, unsigned int address,
 	 * eeprom and i2c bus
 	 */
 
-	/*  make sure we dont send anything to eeprom */
+	/* make sure we dont send anything to eeprom */
 	devpriv->plx_control_bits &= ~PLX_CNTRL_EECS;
 
 	i2c_stop(dev);
 	i2c_start(dev);
 
-	/*  send address and write bit */
+	/* send address and write bit */
 	bitstream = (address << 1) & ~read_bit;
 	i2c_write_byte(dev, bitstream);
 
-	/*  get acknowledge */
+	/* get acknowledge */
 	if (i2c_read_ack(dev) != 0) {
 		dev_err(dev->class_dev, "failed: no acknowledge\n");
 		i2c_stop(dev);
 		return;
 	}
-	/*  write data bytes */
+	/* write data bytes */
 	for (i = 0; i < length; i++) {
 		i2c_write_byte(dev, data[i]);
 		if (i2c_read_ack(dev) != 0) {
@@ -1770,8 +1775,8 @@ static int ai_rinsn(struct comedi_device *dev, struct comedi_subdevice *s,
 	range = CR_RANGE(insn->chanspec);
 	aref = CR_AREF(insn->chanspec);
 
-	/*  disable card's analog input interrupt sources and pacing */
-	/*  4020 generates dac done interrupts even though they are disabled */
+	/* disable card's analog input interrupt sources and pacing */
+	/* 4020 generates dac done interrupts even though they are disabled */
 	disable_ai_pacing(dev);
 
 	spin_lock_irqsave(&dev->spinlock, flags);
@@ -1784,12 +1789,12 @@ static int ai_rinsn(struct comedi_device *dev, struct comedi_subdevice *s,
 	spin_unlock_irqrestore(&dev->spinlock, flags);
 
 	if (board->layout != LAYOUT_4020) {
-		/*  use internal queue */
+		/* use internal queue */
 		devpriv->hw_config_bits &= ~EXT_QUEUE_BIT;
 		writew(devpriv->hw_config_bits,
 		       devpriv->main_iobase + HW_CONFIG_REG);
 
-		/*  ALT_SOURCE is internal calibration reference */
+		/* ALT_SOURCE is internal calibration reference */
 		if (insn->chanspec & CR_ALT_SOURCE) {
 			unsigned int cal_en_bit;
 
@@ -1811,19 +1816,19 @@ static int ai_rinsn(struct comedi_device *dev, struct comedi_subdevice *s,
 			 */
 			writew(0, devpriv->main_iobase + CALIBRATION_REG);
 		}
-		/*  load internal queue */
+		/* load internal queue */
 		bits = 0;
-		/*  set gain */
+		/* set gain */
 		bits |= ai_range_bits_6xxx(dev, CR_RANGE(insn->chanspec));
-		/*  set single-ended / differential */
+		/* set single-ended / differential */
 		bits |= se_diff_bit_6xxx(dev, aref == AREF_DIFF);
 		if (aref == AREF_COMMON)
 			bits |= ADC_COMMON_BIT;
 		bits |= adc_chan_bits(channel);
-		/*  set stop channel */
+		/* set stop channel */
 		writew(adc_chan_bits(channel),
 		       devpriv->main_iobase + ADC_QUEUE_HIGH_REG);
-		/*  set start channel, and rest of settings */
+		/* set start channel, and rest of settings */
 		writew(bits, devpriv->main_iobase + ADC_QUEUE_LOAD_REG);
 	} else {
 		uint8_t old_cal_range_bits = devpriv->i2c_cal_range_bits;
@@ -1835,7 +1840,7 @@ static int ai_rinsn(struct comedi_device *dev, struct comedi_subdevice *s,
 		} else {	/* select BNC inputs */
 			devpriv->i2c_cal_range_bits |= adc_src_4020_bits(4);
 		}
-		/*  select range */
+		/* select range */
 		if (range == 0)
 			devpriv->i2c_cal_range_bits |= attenuate_bit(channel);
 		else
@@ -1862,14 +1867,14 @@ static int ai_rinsn(struct comedi_device *dev, struct comedi_subdevice *s,
 	}
 
 	for (n = 0; n < insn->n; n++) {
-		/*  clear adc buffer (inside loop for 4020 sake) */
+		/* clear adc buffer (inside loop for 4020 sake) */
 		writew(0, devpriv->main_iobase + ADC_BUFFER_CLEAR_REG);
 
 		/* trigger conversion, bits sent only matter for 4020 */
 		writew(adc_convert_chan_4020_bits(CR_CHAN(insn->chanspec)),
 		       devpriv->main_iobase + ADC_CONVERT_REG);
 
-		/*  wait for data */
+		/* wait for data */
 		ret = comedi_timeout(dev, s, insn, cb_pcidas64_ai_eoc, 0);
 		if (ret)
 			return ret;
@@ -2249,7 +2254,7 @@ static void setup_sample_counters(struct comedi_device *dev,
 {
 	struct pcidas64_private *devpriv = dev->private;
 
-	/*  load hardware conversion counter */
+	/* load hardware conversion counter */
 	if (use_hw_sample_counter(cmd)) {
 		writew(cmd->stop_arg & 0xffff,
 		       devpriv->main_iobase + ADC_COUNT_LOWER_REG);
@@ -2277,7 +2282,7 @@ static inline unsigned int dma_transfer_size(struct comedi_device *dev)
 static uint32_t ai_convert_counter_6xxx(const struct comedi_device *dev,
 					const struct comedi_cmd *cmd)
 {
-	/*  supposed to load counter with desired divisor minus 3 */
+	/* supposed to load counter with desired divisor minus 3 */
 	return cmd->convert_arg / TIMER_BASE - 3;
 }
 
@@ -2286,7 +2291,7 @@ static uint32_t ai_scan_counter_6xxx(struct comedi_device *dev,
 {
 	uint32_t count;
 
-	/*  figure out how long we need to delay at end of scan */
+	/* figure out how long we need to delay at end of scan */
 	switch (cmd->scan_begin_src) {
 	case TRIG_TIMER:
 		count = (cmd->scan_begin_arg -
@@ -2315,13 +2320,13 @@ static uint32_t ai_convert_counter_4020(struct comedi_device *dev,
 	case TRIG_OTHER:
 		divisor = devpriv->ext_clock.divisor;
 		break;
-	default:		/*  should never happen */
+	default:		/* should never happen */
 		dev_err(dev->class_dev, "bug! failed to set ai pacing!\n");
 		divisor = 1000;
 		break;
 	}
 
-	/*  supposed to load counter with desired divisor minus 2 for 4020 */
+	/* supposed to load counter with desired divisor minus 2 for 4020 */
 	return divisor - 2;
 }
 
@@ -2330,7 +2335,7 @@ static void select_master_clock_4020(struct comedi_device *dev,
 {
 	struct pcidas64_private *devpriv = dev->private;
 
-	/*  select internal/external master clock */
+	/* select internal/external master clock */
 	devpriv->hw_config_bits &= ~MASTER_CLOCK_4020_MASK;
 	if (cmd->scan_begin_src == TRIG_OTHER) {
 		int chanspec = devpriv->ext_clock.chanspec;
@@ -2366,7 +2371,7 @@ static inline void dma_start_sync(struct comedi_device *dev,
 	struct pcidas64_private *devpriv = dev->private;
 	unsigned long flags;
 
-	/*  spinlock for plx dma control/status reg */
+	/* spinlock for plx dma control/status reg */
 	spin_lock_irqsave(&dev->spinlock, flags);
 	writeb(PLX_DMACSR_ENABLE | PLX_DMACSR_START | PLX_DMACSR_CLEARINTR,
 	       devpriv->plx9080_iobase + PLX_REG_DMACSR(channel));
@@ -2390,16 +2395,16 @@ static void set_ai_pacing(struct comedi_device *dev, struct comedi_cmd *cmd)
 		scan_counter = ai_scan_counter_6xxx(dev, cmd);
 	}
 
-	/*  load lower 16 bits of convert interval */
+	/* load lower 16 bits of convert interval */
 	writew(convert_counter & 0xffff,
 	       devpriv->main_iobase + ADC_SAMPLE_INTERVAL_LOWER_REG);
-	/*  load upper 8 bits of convert interval */
+	/* load upper 8 bits of convert interval */
 	writew((convert_counter >> 16) & 0xff,
 	       devpriv->main_iobase + ADC_SAMPLE_INTERVAL_UPPER_REG);
-	/*  load lower 16 bits of scan delay */
+	/* load lower 16 bits of scan delay */
 	writew(scan_counter & 0xffff,
 	       devpriv->main_iobase + ADC_DELAY_INTERVAL_LOWER_REG);
-	/*  load upper 8 bits of scan delay */
+	/* load upper 8 bits of scan delay */
 	writew((scan_counter >> 16) & 0xff,
 	       devpriv->main_iobase + ADC_DELAY_INTERVAL_UPPER_REG);
 }
@@ -2435,26 +2440,26 @@ static int setup_channel_queue(struct comedi_device *dev,
 			writew(devpriv->hw_config_bits,
 			       devpriv->main_iobase + HW_CONFIG_REG);
 			bits = 0;
-			/*  set channel */
+			/* set channel */
 			bits |= adc_chan_bits(CR_CHAN(cmd->chanlist[0]));
-			/*  set gain */
+			/* set gain */
 			bits |= ai_range_bits_6xxx(dev,
 						   CR_RANGE(cmd->chanlist[0]));
-			/*  set single-ended / differential */
+			/* set single-ended / differential */
 			bits |= se_diff_bit_6xxx(dev,
 						 CR_AREF(cmd->chanlist[0]) ==
 						 AREF_DIFF);
 			if (CR_AREF(cmd->chanlist[0]) == AREF_COMMON)
 				bits |= ADC_COMMON_BIT;
-			/*  set stop channel */
+			/* set stop channel */
 			writew(adc_chan_bits
 			       (CR_CHAN(cmd->chanlist[cmd->chanlist_len - 1])),
 			       devpriv->main_iobase + ADC_QUEUE_HIGH_REG);
-			/*  set start channel, and rest of settings */
+			/* set start channel, and rest of settings */
 			writew(bits,
 			       devpriv->main_iobase + ADC_QUEUE_LOAD_REG);
 		} else {
-			/*  use external queue */
+			/* use external queue */
 			if (dev->write_subdev && dev->write_subdev->busy) {
 				warn_external_queue(dev);
 				return -EBUSY;
@@ -2462,30 +2467,30 @@ static int setup_channel_queue(struct comedi_device *dev,
 			devpriv->hw_config_bits |= EXT_QUEUE_BIT;
 			writew(devpriv->hw_config_bits,
 			       devpriv->main_iobase + HW_CONFIG_REG);
-			/*  clear DAC buffer to prevent weird interactions */
+			/* clear DAC buffer to prevent weird interactions */
 			writew(0,
 			       devpriv->main_iobase + DAC_BUFFER_CLEAR_REG);
-			/*  clear queue pointer */
+			/* clear queue pointer */
 			writew(0, devpriv->main_iobase + ADC_QUEUE_CLEAR_REG);
-			/*  load external queue */
+			/* load external queue */
 			for (i = 0; i < cmd->chanlist_len; i++) {
 				bits = 0;
-				/*  set channel */
+				/* set channel */
 				bits |= adc_chan_bits(CR_CHAN(cmd->
 							      chanlist[i]));
-				/*  set gain */
+				/* set gain */
 				bits |= ai_range_bits_6xxx(dev,
 							   CR_RANGE(cmd->
 								    chanlist
 								    [i]));
-				/*  set single-ended / differential */
+				/* set single-ended / differential */
 				bits |= se_diff_bit_6xxx(dev,
 							 CR_AREF(cmd->
 								 chanlist[i]) ==
 							 AREF_DIFF);
 				if (CR_AREF(cmd->chanlist[i]) == AREF_COMMON)
 					bits |= ADC_COMMON_BIT;
-				/*  mark end of queue */
+				/* mark end of queue */
 				if (i == cmd->chanlist_len - 1)
 					bits |= QUEUE_EOSCAN_BIT |
 						QUEUE_EOSEQ_BIT;
@@ -2498,7 +2503,7 @@ static int setup_channel_queue(struct comedi_device *dev,
 			 * but required for reliable operation
 			 */
 			writew(0, devpriv->main_iobase + ADC_QUEUE_CLEAR_REG);
-			/*  prime queue holding register */
+			/* prime queue holding register */
 			writew(0, devpriv->main_iobase + ADC_QUEUE_LOAD_REG);
 		}
 	} else {
@@ -2507,7 +2512,7 @@ static int setup_channel_queue(struct comedi_device *dev,
 		devpriv->i2c_cal_range_bits &= ~ADC_SRC_4020_MASK;
 		/* select BNC inputs */
 		devpriv->i2c_cal_range_bits |= adc_src_4020_bits(4);
-		/*  select ranges */
+		/* select ranges */
 		for (i = 0; i < cmd->chanlist_len; i++) {
 			unsigned int channel = CR_CHAN(cmd->chanlist[i]);
 			unsigned int range = CR_RANGE(cmd->chanlist[i]);
@@ -2579,7 +2584,7 @@ static int ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 	if (retval < 0)
 		return retval;
 
-	/*  make sure internal calibration source is turned off */
+	/* make sure internal calibration source is turned off */
 	writew(0, devpriv->main_iobase + CALIBRATION_REG);
 
 	set_ai_pacing(dev, cmd);
@@ -2595,10 +2600,10 @@ static int ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 	if (board->layout != LAYOUT_4020) {
 		devpriv->adc_control1_bits &= ~ADC_MODE_MASK;
 		if (cmd->convert_src == TRIG_EXT)
-			/*  good old mode 13 */
+			/* good old mode 13 */
 			devpriv->adc_control1_bits |= adc_mode_bits(13);
 		else
-			/*  mode 8.  What else could you need? */
+			/* mode 8.  What else could you need? */
 			devpriv->adc_control1_bits |= adc_mode_bits(8);
 	} else {
 		devpriv->adc_control1_bits &= ~CHANNEL_MODE_4020_MASK;
@@ -2618,20 +2623,20 @@ static int ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 	       devpriv->main_iobase + ADC_CONTROL1_REG);
 	spin_unlock_irqrestore(&dev->spinlock, flags);
 
-	/*  clear adc buffer */
+	/* clear adc buffer */
 	writew(0, devpriv->main_iobase + ADC_BUFFER_CLEAR_REG);
 
 	if ((cmd->flags & CMDF_WAKE_EOS) == 0 ||
 	    board->layout == LAYOUT_4020) {
 		devpriv->ai_dma_index = 0;
 
-		/*  set dma transfer size */
+		/* set dma transfer size */
 		for (i = 0; i < ai_dma_ring_count(board); i++)
 			devpriv->ai_dma_desc[i].transfer_size =
 				cpu_to_le32(dma_transfer_size(dev) *
 					    sizeof(uint16_t));
 
-		/*  give location of first dma descriptor */
+		/* give location of first dma descriptor */
 		load_first_dma_descriptor(dev, 1,
 					  devpriv->ai_dma_desc_bus_addr |
 					  PLX_DMADPR_DESCPCI |
@@ -2657,7 +2662,7 @@ static int ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 	bits = ADC_ENABLE_BIT | ADC_SOFT_GATE_BITS | ADC_GATE_LEVEL_BIT;
 	if (cmd->flags & CMDF_WAKE_EOS)
 		bits |= ADC_DMA_DISABLE_BIT;
-	/*  set start trigger */
+	/* set start trigger */
 	if (cmd->start_src == TRIG_EXT) {
 		bits |= ADC_START_TRIG_EXT_BITS;
 		if (cmd->start_arg & CR_INVERT)
@@ -2673,7 +2678,7 @@ static int ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 
 	spin_unlock_irqrestore(&dev->spinlock, flags);
 
-	/*  start acquisition */
+	/* start acquisition */
 	if (cmd->start_src == TRIG_NOW)
 		writew(0, devpriv->main_iobase + ADC_START_REG);
 
@@ -2691,7 +2696,7 @@ static void pio_drain_ai_fifo_16(struct comedi_device *dev)
 	int num_samples;
 
 	do {
-		/*  get least significant 15 bits */
+		/* get least significant 15 bits */
 		read_index = readw(devpriv->main_iobase + ADC_READ_PNTR_REG) &
 			     0x7fff;
 		write_index = readw(devpriv->main_iobase + ADC_WRITE_PNTR_REG) &
@@ -2796,14 +2801,14 @@ static void drain_dma_buffers(struct comedi_device *dev, unsigned int channel)
 
 	pci_addr_reg = devpriv->plx9080_iobase + PLX_REG_DMAPADR(channel);
 
-	/*  loop until we have read all the full buffers */
+	/* loop until we have read all the full buffers */
 	for (j = 0, next_transfer_addr = readl(pci_addr_reg);
 	     (next_transfer_addr <
 	      devpriv->ai_buffer_bus_addr[devpriv->ai_dma_index] ||
 	      next_transfer_addr >=
 	      devpriv->ai_buffer_bus_addr[devpriv->ai_dma_index] +
 	      DMA_BUFFER_SIZE) && j < ai_dma_ring_count(board); j++) {
-		/*  transfer data from dma buffer to comedi buffer */
+		/* transfer data from dma buffer to comedi buffer */
 		num_samples = comedi_nsamples_left(s, dma_transfer_size(dev));
 		comedi_buf_write_samples(s,
 				devpriv->ai_buffer[devpriv->ai_dma_index],
@@ -2829,15 +2834,15 @@ static void handle_ai_interrupt(struct comedi_device *dev,
 	uint8_t dma1_status;
 	unsigned long flags;
 
-	/*  check for fifo overrun */
+	/* check for fifo overrun */
 	if (status & ADC_OVERRUN_BIT) {
 		dev_err(dev->class_dev, "fifo overrun\n");
 		async->events |= COMEDI_CB_ERROR;
 	}
-	/*  spin lock makes sure no one else changes plx dma control reg */
+	/* spin lock makes sure no one else changes plx dma control reg */
 	spin_lock_irqsave(&dev->spinlock, flags);
 	dma1_status = readb(devpriv->plx9080_iobase + PLX_REG_DMACSR1);
-	if (plx_status & PLX_INTCSR_DMA1IA) {	/*  dma chan 1 interrupt */
+	if (plx_status & PLX_INTCSR_DMA1IA) {	/* dma chan 1 interrupt */
 		writeb((dma1_status & PLX_DMACSR_ENABLE) | PLX_DMACSR_CLEARINTR,
 		       devpriv->plx9080_iobase + PLX_REG_DMACSR1);
 
@@ -2846,7 +2851,7 @@ static void handle_ai_interrupt(struct comedi_device *dev,
 	}
 	spin_unlock_irqrestore(&dev->spinlock, flags);
 
-	/*  drain fifo with pio */
+	/* drain fifo with pio */
 	if ((status & ADC_DONE_BIT) ||
 	    ((cmd->flags & CMDF_WAKE_EOS) &&
 	     (status & ADC_INTR_PENDING_BIT) &&
@@ -2859,7 +2864,7 @@ static void handle_ai_interrupt(struct comedi_device *dev,
 			spin_unlock_irqrestore(&dev->spinlock, flags);
 		}
 	}
-	/*  if we are have all the data, then quit */
+	/* if we are have all the data, then quit */
 	if ((cmd->stop_src == TRIG_COUNT &&
 	     async->scans_done >= cmd->stop_arg) ||
 	    (cmd->stop_src == TRIG_EXT && (status & ADC_STOP_BIT)))
@@ -3012,7 +3017,7 @@ static void handle_ao_interrupt(struct comedi_device *dev,
 	async = s->async;
 	cmd = &async->cmd;
 
-	/*  spin lock makes sure no one else changes plx dma control reg */
+	/* spin lock makes sure no one else changes plx dma control reg */
 	spin_lock_irqsave(&dev->spinlock, flags);
 	dma0_status = readb(devpriv->plx9080_iobase + PLX_REG_DMACSR0);
 	if (plx_status & PLX_INTCSR_DMA0IA) {	/*  dma chan 0 interrupt */
@@ -3106,15 +3111,15 @@ static int ao_winsn(struct comedi_device *dev, struct comedi_subdevice *s,
 	int chan = CR_CHAN(insn->chanspec);
 	int range = CR_RANGE(insn->chanspec);
 
-	/*  do some initializing */
+	/* do some initializing */
 	writew(0, devpriv->main_iobase + DAC_CONTROL0_REG);
 
-	/*  set range */
+	/* set range */
 	set_dac_range_bits(dev, &devpriv->dac_control1_bits, chan, range);
 	writew(devpriv->dac_control1_bits,
 	       devpriv->main_iobase + DAC_CONTROL1_REG);
 
-	/*  write to channel */
+	/* write to channel */
 	if (board->layout == LAYOUT_4020) {
 		writew(data[0] & 0xff,
 		       devpriv->main_iobase + dac_lsb_4020_reg(chan));
@@ -3124,7 +3129,7 @@ static int ao_winsn(struct comedi_device *dev, struct comedi_subdevice *s,
 		writew(data[0], devpriv->main_iobase + dac_convert_reg(chan));
 	}
 
-	/*  remember output value */
+	/* remember output value */
 	s->readback[chan] = data[0];
 
 	return 1;
@@ -3556,7 +3561,7 @@ static int caldac_i2c_write(struct comedi_device *dev,
 	uint8_t serial_bytes[3];
 	uint8_t i2c_addr;
 	enum pointer_bits {
-		/*  manual has gain and offset bits switched */
+		/* manual has gain and offset bits switched */
 		OFFSET_0_2 = 0x1,
 		GAIN_0_2 = 0x2,
 		OFFSET_1_3 = 0x4,
@@ -3567,35 +3572,35 @@ static int caldac_i2c_write(struct comedi_device *dev,
 	};
 
 	switch (caldac_channel) {
-	case 0:		/*  chan 0 offset */
+	case 0:					/* chan 0 offset */
 		i2c_addr = CALDAC0_I2C_ADDR;
 		serial_bytes[0] = OFFSET_0_2;
 		break;
-	case 1:		/*  chan 1 offset */
+	case 1:					/* chan 1 offset */
 		i2c_addr = CALDAC0_I2C_ADDR;
 		serial_bytes[0] = OFFSET_1_3;
 		break;
-	case 2:		/*  chan 2 offset */
+	case 2:					/* chan 2 offset */
 		i2c_addr = CALDAC1_I2C_ADDR;
 		serial_bytes[0] = OFFSET_0_2;
 		break;
-	case 3:		/*  chan 3 offset */
+	case 3:					/* chan 3 offset */
 		i2c_addr = CALDAC1_I2C_ADDR;
 		serial_bytes[0] = OFFSET_1_3;
 		break;
-	case 4:		/*  chan 0 gain */
+	case 4:					/* chan 0 gain */
 		i2c_addr = CALDAC0_I2C_ADDR;
 		serial_bytes[0] = GAIN_0_2;
 		break;
-	case 5:		/*  chan 1 gain */
+	case 5:					/* chan 1 gain */
 		i2c_addr = CALDAC0_I2C_ADDR;
 		serial_bytes[0] = GAIN_1_3;
 		break;
-	case 6:		/*  chan 2 gain */
+	case 6:					/* chan 2 gain */
 		i2c_addr = CALDAC1_I2C_ADDR;
 		serial_bytes[0] = GAIN_0_2;
 		break;
-	case 7:		/*  chan 3 gain */
+	case 7:					/* chan 3 gain */
 		i2c_addr = CALDAC1_I2C_ADDR;
 		serial_bytes[0] = GAIN_1_3;
 		break;
@@ -3718,24 +3723,24 @@ static uint16_t read_eeprom(struct comedi_device *dev, uint8_t address)
 
 	udelay(eeprom_udelay);
 	devpriv->plx_control_bits &= ~PLX_CNTRL_EESK & ~PLX_CNTRL_EECS;
-	/*  make sure we don't send anything to the i2c bus on 4020 */
+	/* make sure we don't send anything to the i2c bus on 4020 */
 	devpriv->plx_control_bits |= PLX_CNTRL_USERO;
 	writel(devpriv->plx_control_bits, plx_control_addr);
-	/*  activate serial eeprom */
+	/* activate serial eeprom */
 	udelay(eeprom_udelay);
 	devpriv->plx_control_bits |= PLX_CNTRL_EECS;
 	writel(devpriv->plx_control_bits, plx_control_addr);
 
-	/*  write read command and desired memory address */
+	/* write read command and desired memory address */
 	for (bit = 1 << (bitstream_length - 1); bit; bit >>= 1) {
-		/*  set bit to be written */
+		/* set bit to be written */
 		udelay(eeprom_udelay);
 		if (bitstream & bit)
 			devpriv->plx_control_bits |= PLX_CNTRL_EEWB;
 		else
 			devpriv->plx_control_bits &= ~PLX_CNTRL_EEWB;
 		writel(devpriv->plx_control_bits, plx_control_addr);
-		/*  clock in bit */
+		/* clock in bit */
 		udelay(eeprom_udelay);
 		devpriv->plx_control_bits |= PLX_CNTRL_EESK;
 		writel(devpriv->plx_control_bits, plx_control_addr);
@@ -3743,10 +3748,10 @@ static uint16_t read_eeprom(struct comedi_device *dev, uint8_t address)
 		devpriv->plx_control_bits &= ~PLX_CNTRL_EESK;
 		writel(devpriv->plx_control_bits, plx_control_addr);
 	}
-	/*  read back value from eeprom memory location */
+	/* read back value from eeprom memory location */
 	value = 0;
 	for (bit = 1 << (value_length - 1); bit; bit >>= 1) {
-		/*  clock out bit */
+		/* clock out bit */
 		udelay(eeprom_udelay);
 		devpriv->plx_control_bits |= PLX_CNTRL_EESK;
 		writel(devpriv->plx_control_bits, plx_control_addr);
@@ -3758,7 +3763,7 @@ static uint16_t read_eeprom(struct comedi_device *dev, uint8_t address)
 			value |= bit;
 	}
 
-	/*  deactivate eeprom serial input */
+	/* deactivate eeprom serial input */
 	udelay(eeprom_udelay);
 	devpriv->plx_control_bits &= ~PLX_CNTRL_EECS;
 	writel(devpriv->plx_control_bits, plx_control_addr);
@@ -3775,9 +3780,7 @@ static int eeprom_read_insn(struct comedi_device *dev,
 	return 1;
 }
 
-/*
- * Allocate and initialize the subdevice structures.
- */
+/* Allocate and initialize the subdevice structures. */
 static int setup_subdevices(struct comedi_device *dev)
 {
 	const struct pcidas64_board *board = dev->board_ptr;
@@ -3816,7 +3819,7 @@ static int setup_subdevices(struct comedi_device *dev)
 		 * (not internal calibration sources)
 		 */
 		devpriv->i2c_cal_range_bits = adc_src_4020_bits(4);
-		/*  set channels to +-5 volt input ranges */
+		/* set channels to +-5 volt input ranges */
 		for (i = 0; i < s->n_chan; i++)
 			devpriv->i2c_cal_range_bits |= attenuate_bit(i);
 		data = devpriv->i2c_cal_range_bits;
@@ -3849,7 +3852,7 @@ static int setup_subdevices(struct comedi_device *dev)
 		s->type = COMEDI_SUBD_UNUSED;
 	}
 
-	/*  digital input */
+	/* digital input */
 	s = &dev->subdevices[2];
 	if (board->layout == LAYOUT_64XX) {
 		s->type = COMEDI_SUBD_DI;
@@ -3862,7 +3865,7 @@ static int setup_subdevices(struct comedi_device *dev)
 		s->type = COMEDI_SUBD_UNUSED;
 	}
 
-	/*  digital output */
+	/* digital output */
 	if (board->layout == LAYOUT_64XX) {
 		s = &dev->subdevices[3];
 		s->type = COMEDI_SUBD_DO;
@@ -3891,7 +3894,7 @@ static int setup_subdevices(struct comedi_device *dev)
 		s->type = COMEDI_SUBD_UNUSED;
 	}
 
-	/*  8 channel dio for 60xx */
+	/* 8 channel dio for 60xx */
 	s = &dev->subdevices[5];
 	if (board->layout == LAYOUT_60XX) {
 		s->type = COMEDI_SUBD_DIO;
@@ -3905,7 +3908,7 @@ static int setup_subdevices(struct comedi_device *dev)
 		s->type = COMEDI_SUBD_UNUSED;
 	}
 
-	/*  caldac */
+	/* caldac */
 	s = &dev->subdevices[6];
 	s->type = COMEDI_SUBD_CALIB;
 	s->subdev_flags = SDF_READABLE | SDF_WRITABLE | SDF_INTERNAL;
@@ -3925,7 +3928,7 @@ static int setup_subdevices(struct comedi_device *dev)
 		s->readback[i] = s->maxdata / 2;
 	}
 
-	/*  2 channel ad8402 potentiometer */
+	/* 2 channel ad8402 potentiometer */
 	s = &dev->subdevices[7];
 	if (board->layout == LAYOUT_64XX) {
 		s->type = COMEDI_SUBD_CALIB;
@@ -3959,7 +3962,7 @@ static int setup_subdevices(struct comedi_device *dev)
 		s->type = COMEDI_SUBD_UNUSED;
 	}
 
-	/*  user counter subd XXX */
+	/* user counter subd XXX */
 	s = &dev->subdevices[9];
 	s->type = COMEDI_SUBD_UNUSED;
 
@@ -4005,7 +4008,7 @@ static int auto_attach(struct comedi_device *dev,
 		return -ENOMEM;
 	}
 
-	/*  figure out what local addresses are */
+	/* figure out what local addresses are */
 	local_range = readl(devpriv->plx9080_iobase + PLX_REG_LAS0RR) &
 		      PLX_LASRR_MEM_MASK;
 	local_decode = readl(devpriv->plx9080_iobase + PLX_REG_LAS0BA) &
diff --git a/drivers/staging/comedi/drivers/comedi_test.c b/drivers/staging/comedi/drivers/comedi_test.c
index 4ab186669f0c..ec5b9a23494d 100644
--- a/drivers/staging/comedi/drivers/comedi_test.c
+++ b/drivers/staging/comedi/drivers/comedi_test.c
@@ -56,11 +56,6 @@
 
 #define N_CHANS 8
 
-enum waveform_state_bits {
-	WAVEFORM_AI_RUNNING,
-	WAVEFORM_AO_RUNNING
-};
-
 /* Data unique to this driver */
 struct waveform_private {
 	struct timer_list ai_timer;	/* timer for AI commands */
@@ -68,7 +63,6 @@ struct waveform_private {
 	unsigned int wf_amplitude;	/* waveform amplitude in microvolts */
 	unsigned int wf_period;		/* waveform period in microseconds */
 	unsigned int wf_current;	/* current time in waveform period */
-	unsigned long state_bits;
 	unsigned int ai_scan_period;	/* AI scan period in usec */
 	unsigned int ai_convert_period;	/* AI conversion period in usec */
 	struct timer_list ao_timer;	/* timer for AO commands */
@@ -191,10 +185,6 @@ static void waveform_ai_timer(unsigned long arg)
 	unsigned int nsamples;
 	unsigned int time_increment;
 
-	/* check command is still active */
-	if (!test_bit(WAVEFORM_AI_RUNNING, &devpriv->state_bits))
-		return;
-
 	now = ktime_to_us(ktime_get());
 	nsamples = comedi_nsamples_left(s, UINT_MAX);
 
@@ -386,11 +376,6 @@ static int waveform_ai_cmd(struct comedi_device *dev,
 	 */
 	devpriv->ai_timer.expires =
 		jiffies + usecs_to_jiffies(devpriv->ai_convert_period) + 1;
-
-	/* mark command as active */
-	smp_mb__before_atomic();
-	set_bit(WAVEFORM_AI_RUNNING, &devpriv->state_bits);
-	smp_mb__after_atomic();
 	add_timer(&devpriv->ai_timer);
 	return 0;
 }
@@ -400,11 +385,12 @@ static int waveform_ai_cancel(struct comedi_device *dev,
 {
 	struct waveform_private *devpriv = dev->private;
 
-	/* mark command as no longer active */
-	clear_bit(WAVEFORM_AI_RUNNING, &devpriv->state_bits);
-	smp_mb__after_atomic();
-	/* cannot call del_timer_sync() as may be called from timer routine */
-	del_timer(&devpriv->ai_timer);
+	if (in_softirq()) {
+		/* Assume we were called from the timer routine itself. */
+		del_timer(&devpriv->ai_timer);
+	} else {
+		del_timer_sync(&devpriv->ai_timer);
+	}
 	return 0;
 }
 
@@ -436,10 +422,6 @@ static void waveform_ao_timer(unsigned long arg)
 	u64 scans_since;
 	unsigned int scans_avail = 0;
 
-	/* check command is still active */
-	if (!test_bit(WAVEFORM_AO_RUNNING, &devpriv->state_bits))
-		return;
-
 	/* determine number of scan periods since last time */
 	now = ktime_to_us(ktime_get());
 	scans_since = now - devpriv->ao_last_scan_time;
@@ -518,11 +500,6 @@ static int waveform_ao_inttrig_start(struct comedi_device *dev,
 	devpriv->ao_last_scan_time = ktime_to_us(ktime_get());
 	devpriv->ao_timer.expires =
 		jiffies + usecs_to_jiffies(devpriv->ao_scan_period);
-
-	/* mark command as active */
-	smp_mb__before_atomic();
-	set_bit(WAVEFORM_AO_RUNNING, &devpriv->state_bits);
-	smp_mb__after_atomic();
 	add_timer(&devpriv->ao_timer);
 
 	return 1;
@@ -608,11 +585,12 @@ static int waveform_ao_cancel(struct comedi_device *dev,
 	struct waveform_private *devpriv = dev->private;
 
 	s->async->inttrig = NULL;
-	/* mark command as no longer active */
-	clear_bit(WAVEFORM_AO_RUNNING, &devpriv->state_bits);
-	smp_mb__after_atomic();
-	/* cannot call del_timer_sync() as may be called from timer routine */
-	del_timer(&devpriv->ao_timer);
+	if (in_softirq()) {
+		/* Assume we were called from the timer routine itself. */
+		del_timer(&devpriv->ao_timer);
+	} else {
+		del_timer_sync(&devpriv->ao_timer);
+	}
 	return 0;
 }
 
diff --git a/drivers/staging/comedi/drivers/daqboard2000.c b/drivers/staging/comedi/drivers/daqboard2000.c
index 65daef0c00d5..0f4eb954aa80 100644
--- a/drivers/staging/comedi/drivers/daqboard2000.c
+++ b/drivers/staging/comedi/drivers/daqboard2000.c
@@ -634,7 +634,7 @@ static const void *daqboard2000_find_boardinfo(struct comedi_device *dev,
 	const struct daq200_boardtype *board;
 	int i;
 
-	if (pcidev->subsystem_device != PCI_VENDOR_ID_IOTECH)
+	if (pcidev->subsystem_vendor != PCI_VENDOR_ID_IOTECH)
 		return NULL;
 
 	for (i = 0; i < ARRAY_SIZE(boardtypes); i++) {
diff --git a/drivers/staging/comedi/drivers/das08_cs.c b/drivers/staging/comedi/drivers/das08_cs.c
index 9c02b17a2834..317a9b5e4a3b 100644
--- a/drivers/staging/comedi/drivers/das08_cs.c
+++ b/drivers/staging/comedi/drivers/das08_cs.c
@@ -1,43 +1,42 @@
 /*
-    comedi/drivers/das08_cs.c
-    DAS08 driver
+ * Comedi driver for DAS008 PCMCIA boards
+ *
+ * COMEDI - Linux Control and Measurement Device Interface
+ * Copyright (C) 2000 David A. Schleef <ds@schleef.org>
+ * Copyright (C) 2001,2002,2003 Frank Mori Hess <fmhess@users.sourceforge.net>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * PCMCIA support code for this driver is adapted from the dummy_cs.c
+ * driver of the Linux PCMCIA Card Services package.
+ *
+ * The initial developer of the original code is David A. Hinds
+ * <dahinds@users.sourceforge.net>.  Portions created by David A. Hinds
+ * are Copyright (C) 1999 David A. Hinds.  All Rights Reserved.
+ */
 
-    COMEDI - Linux Control and Measurement Device Interface
-    Copyright (C) 2000 David A. Schleef <ds@schleef.org>
-    Copyright (C) 2001,2002,2003 Frank Mori Hess <fmhess@users.sourceforge.net>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    PCMCIA support code for this driver is adapted from the dummy_cs.c
-    driver of the Linux PCMCIA Card Services package.
-
-    The initial developer of the original code is David A. Hinds
-    <dahinds@users.sourceforge.net>.  Portions created by David A. Hinds
-    are Copyright (C) 1999 David A. Hinds.  All Rights Reserved.
-*/
 /*
-Driver: das08_cs
-Description: DAS-08 PCMCIA boards
-Author: Warren Jasper, ds, Frank Hess
-Devices: [ComputerBoards] PCM-DAS08 (pcm-das08)
-Status: works
-
-This is the PCMCIA-specific support split off from the
-das08 driver.
-
-Options (for pcm-das08):
-	NONE
-
-Command support does not exist, but could be added for this board.
-*/
+ * Driver: das08_cs
+ * Description: DAS-08 PCMCIA boards
+ * Author: Warren Jasper, ds, Frank Hess
+ * Devices: [ComputerBoards] PCM-DAS08 (pcm-das08)
+ * Status: works
+ *
+ * This is the PCMCIA-specific support split off from the
+ * das08 driver.
+ *
+ * Configuration Options: none, uses PCMCIA auto config
+ *
+ * Command support does not exist, but could be added for this board.
+ */
 
 #include <linux/module.h>
 
diff --git a/drivers/staging/comedi/drivers/dt2811.c b/drivers/staging/comedi/drivers/dt2811.c
index 904f637797b6..fcd85475e429 100644
--- a/drivers/staging/comedi/drivers/dt2811.c
+++ b/drivers/staging/comedi/drivers/dt2811.c
@@ -96,11 +96,11 @@
  *    6      6      100 kHz	 6   1000000
  *    7     12       50 kHz	 7   10000000
  */
-const unsigned int dt2811_clk_dividers[] = {
+static const unsigned int dt2811_clk_dividers[] = {
 	1, 10, 2, 3, 4, 5, 6, 12
 };
 
-const unsigned int dt2811_clk_multipliers[] = {
+static const unsigned int dt2811_clk_multipliers[] = {
 	1, 10, 100, 1000, 10000, 100000, 1000000, 10000000
 };
 
@@ -588,8 +588,8 @@ static int dt2811_attach(struct comedi_device *dev, struct comedi_devconfig *it)
 	s = &dev->subdevices[0];
 	s->type		= COMEDI_SUBD_AI;
 	s->subdev_flags	= SDF_READABLE |
-			  (it->options[2] == 1) ? SDF_DIFF :
-			  (it->options[2] == 2) ? SDF_COMMON : SDF_GROUND;
+			  ((it->options[2] == 1) ? SDF_DIFF :
+			   (it->options[2] == 2) ? SDF_COMMON : SDF_GROUND);
 	s->n_chan	= (it->options[2] == 1) ? 8 : 16;
 	s->maxdata	= 0x0fff;
 	s->range_table	= board->is_pgh ? &dt2811_pgh_ai_ranges
diff --git a/drivers/staging/comedi/drivers/dt9812.c b/drivers/staging/comedi/drivers/dt9812.c
index 3295bb4ac8c4..7ebca862ecaa 100644
--- a/drivers/staging/comedi/drivers/dt9812.c
+++ b/drivers/staging/comedi/drivers/dt9812.c
@@ -660,12 +660,12 @@ static int dt9812_find_endpoints(struct comedi_device *dev)
 		case 1:
 			dir = USB_DIR_OUT;
 			devpriv->cmd_wr.addr = ep->bEndpointAddress;
-			devpriv->cmd_wr.size = le16_to_cpu(ep->wMaxPacketSize);
+			devpriv->cmd_wr.size = usb_endpoint_maxp(ep);
 			break;
 		case 2:
 			dir = USB_DIR_IN;
 			devpriv->cmd_rd.addr = ep->bEndpointAddress;
-			devpriv->cmd_rd.size = le16_to_cpu(ep->wMaxPacketSize);
+			devpriv->cmd_rd.size = usb_endpoint_maxp(ep);
 			break;
 		case 3:
 			/* unused write stream */
diff --git a/drivers/staging/comedi/drivers/gsc_hpdi.c b/drivers/staging/comedi/drivers/gsc_hpdi.c
index af4b4175af4d..e5b948405fd9 100644
--- a/drivers/staging/comedi/drivers/gsc_hpdi.c
+++ b/drivers/staging/comedi/drivers/gsc_hpdi.c
@@ -582,7 +582,7 @@ static void gsc_hpdi_init_plx9080(struct comedi_device *dev)
 	bits |= PLX_DMAMODE_DEMAND;
 	/* enable local burst mode */
 	bits |= PLX_DMAMODE_BURSTEN;
-	bits |= PLX_DMAMODE_WIDTH32;
+	bits |= PLX_DMAMODE_WIDTH_32;
 	writel(bits, plx_iobase + PLX_REG_DMAMODE0);
 }
 
diff --git a/drivers/staging/comedi/drivers/jr3_pci.c b/drivers/staging/comedi/drivers/jr3_pci.c
index 6c4ff023717f..70390de66e0e 100644
--- a/drivers/staging/comedi/drivers/jr3_pci.c
+++ b/drivers/staging/comedi/drivers/jr3_pci.c
@@ -141,7 +141,7 @@ static void set_transforms(struct jr3_channel __iomem *channel,
 {
 	int i;
 
-	num &= 0x000f;		/*  Make sure that 0 <= num <= 15 */
+	num &= 0x000f;		/* Make sure that 0 <= num <= 15 */
 	for (i = 0; i < 8; i++) {
 		set_u16(&channel->transforms[num].link[i].link_type,
 			transf.link[i].link_type);
@@ -323,10 +323,10 @@ static int read_idm_word(const u8 *data, size_t size, int *pos,
 	int value;
 
 	if (pos && val) {
-		/*  Skip over non hex */
+		/* Skip over non hex */
 		for (; *pos < size && !isxdigit(data[*pos]); (*pos)++)
 			;
-		/*  Collect value */
+		/* Collect value */
 		*val = 0;
 		for (; *pos < size; (*pos)++) {
 			value = hex_to_bin(data[*pos]);
@@ -448,7 +448,8 @@ static int jr3_download_firmware(struct comedi_device *dev,
 	return 0;
 }
 
-static struct jr3_pci_poll_delay jr3_pci_poll_subdevice(struct comedi_subdevice *s)
+static struct jr3_pci_poll_delay
+jr3_pci_poll_subdevice(struct comedi_subdevice *s)
 {
 	struct jr3_pci_subdev_private *spriv = s->private;
 	struct jr3_pci_poll_delay result = poll_delay_min_max(1000, 2000);
@@ -733,13 +734,13 @@ static int jr3_pci_auto_attach(struct comedi_device *dev,
 		}
 	}
 
-	/*  Reset DSP card */
+	/* Reset DSP card */
 	writel(0, &devpriv->iobase->channel[0].reset);
 
 	ret = comedi_load_firmware(dev, &comedi_to_pci_dev(dev)->dev,
 				   "comedi/jr3pci.idm",
 				   jr3_download_firmware, 0);
-	dev_dbg(dev->class_dev, "Firmare load %d\n", ret);
+	dev_dbg(dev->class_dev, "Firmware load %d\n", ret);
 	if (ret < 0)
 		return ret;
 	/*
@@ -763,7 +764,7 @@ static int jr3_pci_auto_attach(struct comedi_device *dev,
 				data.copyright[i]) >> 8);
 	}
 
-	/*  Start card timer */
+	/* Start card timer */
 	for (i = 0; i < dev->n_subdevices; i++) {
 		s = &dev->subdevices[i];
 		spriv = s->private;
diff --git a/drivers/staging/comedi/drivers/jr3_pci.h b/drivers/staging/comedi/drivers/jr3_pci.h
index 356811defaf4..f10a84fb6c14 100644
--- a/drivers/staging/comedi/drivers/jr3_pci.h
+++ b/drivers/staging/comedi/drivers/jr3_pci.h
@@ -1,4 +1,5 @@
-/* Helper types to take care of the fact that the DSP card memory
+/*
+ * Helper types to take care of the fact that the DSP card memory
  * is 16 bits, but aligned on a 32 bit PCI boundary
  */
 
@@ -22,7 +23,8 @@ static inline void set_s16(s32 __iomem *p, s16 val)
 	writel(val, p);
 }
 
-/* The raw data is stored in a format which facilitates rapid
+/*
+ * The raw data is stored in a format which facilitates rapid
  * processing by the JR3 DSP chip. The raw_channel structure shows the
  * format for a single channel of data. Each channel takes four,
  * two-byte words.
@@ -47,7 +49,8 @@ struct raw_channel {
 	s32 reserved[2];
 };
 
-/* The force_array structure shows the layout for the decoupled and
+/*
+ * The force_array structure shows the layout for the decoupled and
  * filtered force data.
  */
 struct force_array {
@@ -61,7 +64,8 @@ struct force_array {
 	s32 v2;
 };
 
-/* The six_axis_array structure shows the layout for the offsets and
+/*
+ * The six_axis_array structure shows the layout for the offsets and
  * the full scales.
  */
 struct six_axis_array {
@@ -74,7 +78,8 @@ struct six_axis_array {
 };
 
 /* VECT_BITS */
-/* The vect_bits structure shows the layout for indicating
+/*
+ * The vect_bits structure shows the layout for indicating
  * which axes to use in computing the vectors. Each bit signifies
  * selection of a single axis. The V1x axis bit corresponds to a hex
  * value of 0x0001 and the V2z bit corresponds to a hex value of
@@ -100,12 +105,14 @@ enum {
 };
 
 /* WARNING_BITS */
-/* The warning_bits structure shows the bit pattern for the warning
+/*
+ * The warning_bits structure shows the bit pattern for the warning
  * word. The bit fields are shown from bit 0 (lsb) to bit 15 (msb).
  */
 
-/*  XX_NEAR_SET */
-/* The xx_near_sat bits signify that the indicated axis has reached or
+/* XX_NEAR_SET */
+/*
+ * The xx_near_sat bits signify that the indicated axis has reached or
  * exceeded the near saturation value.
  */
 
@@ -118,12 +125,13 @@ enum {
 	mz_near_sat = 0x0020
 };
 
-/*  ERROR_BITS */
-/*  XX_SAT */
-/*  MEMORY_ERROR */
-/*  SENSOR_CHANGE */
+/* ERROR_BITS */
+/* XX_SAT */
+/* MEMORY_ERROR */
+/* SENSOR_CHANGE */
 
-/* The error_bits structure shows the bit pattern for the error word.
+/*
+ * The error_bits structure shows the bit pattern for the error word.
  * The bit fields are shown from bit 0 (lsb) to bit 15 (msb). The
  * xx_sat bits signify that the indicated axis has reached or exceeded
  * the saturation value. The memory_error bit indicates that a problem
@@ -134,9 +142,10 @@ enum {
  *
  */
 
-/*  SYSTEM_BUSY */
+/* SYSTEM_BUSY */
 
-/* The system_busy bit indicates that the JR3 DSP is currently busy
+/*
+ * The system_busy bit indicates that the JR3 DSP is currently busy
  * and is not calculating force data. This occurs when a new
  * coordinate transformation, or new sensor full scale is set by the
  * user. A very fast system using the force data for feedback might
@@ -146,9 +155,10 @@ enum {
  * calibration CRC.
  */
 
-/*  CAL_CRC_BAD */
+/* CAL_CRC_BAD */
 
-/* The cal_crc_bad bit indicates that the calibration CRC has not
+/*
+ * The cal_crc_bad bit indicates that the calibration CRC has not
  * calculated to zero. CRC is short for cyclic redundancy code. It is
  * a method for determining the integrity of messages in data
  * communication. The calibration data stored inside the sensor is
@@ -168,7 +178,8 @@ enum {
 /* WATCH_DOG */
 /* WATCH_DOG2 */
 
-/* The watch_dog and watch_dog2 bits are sensor, not processor, watch
+/*
+ * The watch_dog and watch_dog2 bits are sensor, not processor, watch
  * dog bits. Watch_dog indicates that the sensor data line seems to be
  * acting correctly, while watch_dog2 indicates that sensor data and
  * clock are being received. It is possible for watch_dog2 to go off
@@ -192,9 +203,10 @@ enum error_bits_t {
 	watch_dog = 0x8000
 };
 
-/*  THRESH_STRUCT */
+/* THRESH_STRUCT */
 
-/* This structure shows the layout for a single threshold packet inside of a
+/*
+ * This structure shows the layout for a single threshold packet inside of a
  * load envelope. Each load envelope can contain several threshold structures.
  * 1. data_address contains the address of the data for that threshold. This
  *    includes filtered, unfiltered, raw, rate, counters, error and warning data
@@ -210,9 +222,10 @@ struct thresh_struct {
 	s32 bit_pattern;
 };
 
-/*  LE_STRUCT */
+/* LE_STRUCT */
 
-/* Layout of a load enveloped packet. Four thresholds are showed ... for more
+/*
+ * Layout of a load enveloped packet. Four thresholds are showed ... for more
  * see manual (pag.25)
  * 1. latch_bits is a bit pattern that show which bits the user wants to latch.
  *    The latched bits will not be reset once the threshold which set them is
@@ -228,8 +241,9 @@ struct le_struct {
 	s32 reserved;
 };
 
-/*  LINK_TYPES */
-/* Link types is an enumerated value showing the different possible transform
+/* LINK_TYPES */
+/*
+ * Link types is an enumerated value showing the different possible transform
  * link types.
  * 0 - end transform packet
  * 1 - translate along X axis (TX)
@@ -252,8 +266,8 @@ enum link_types {
 	neg
 };
 
-/*  TRANSFORM */
-/*  Structure used to describe a transform. */
+/* TRANSFORM */
+/* Structure used to describe a transform. */
 struct intern_transform {
 	struct {
 		u32 link_type;
@@ -261,23 +275,29 @@ struct intern_transform {
 	} link[8];
 };
 
-/*  JR3 force/torque sensor data definition. For more information see sensor
- *  and hardware manuals.
+/*
+ * JR3 force/torque sensor data definition. For more information see sensor
+ * and hardware manuals.
  */
 
 struct jr3_channel {
-	/*  Raw_channels is the area used to store the raw data coming from */
-	/*  the sensor. */
+	/*
+	 * Raw_channels is the area used to store the raw data coming from
+	 * the sensor.
+	 */
 
 	struct raw_channel raw_channels[16];	/* offset 0x0000 */
 
-	/*  Copyright is a null terminated ASCII string containing the JR3 */
-	/*  copyright notice. */
+	/*
+	 * Copyright is a null terminated ASCII string containing the JR3
+	 * copyright notice.
+	 */
 
 	u32 copyright[0x0018];	/* offset 0x0040 */
 	s32 reserved1[0x0008];	/* offset 0x0058 */
 
-	/* Shunts contains the sensor shunt readings. Some JR3 sensors have
+	/*
+	 * Shunts contains the sensor shunt readings. Some JR3 sensors have
 	 * the ability to have their gains adjusted. This allows the
 	 * hardware full scales to be adjusted to potentially allow
 	 * better resolution or dynamic range. For sensors that have
@@ -298,25 +318,29 @@ struct jr3_channel {
 	 * command (10) set new full scales (pg. 38).
 	 */
 
-	struct six_axis_array shunts;	/* offset 0x0060 */
-	s32 reserved2[2];	/* offset 0x0066 */
+	struct six_axis_array shunts;		/* offset 0x0060 */
+	s32 reserved2[2];			/* offset 0x0066 */
 
-	/* Default_FS contains the full scale that is used if the user does */
-	/* not set a full scale. */
+	/*
+	 * Default_FS contains the full scale that is used if the user does
+	 * not set a full scale.
+	 */
 
 	struct six_axis_array default_FS;	/* offset 0x0068 */
-	s32 reserved3;		/* offset 0x006e */
+	s32 reserved3;				/* offset 0x006e */
 
-	/* Load_envelope_num is the load envelope number that is currently
+	/*
+	 * Load_envelope_num is the load envelope number that is currently
 	 * in use. This value is set by the user after one of the load
 	 * envelopes has been initialized.
 	 */
 
-	s32 load_envelope_num;	/* offset 0x006f */
+	s32 load_envelope_num;			/* offset 0x006f */
 
 	/* Min_full_scale is the recommend minimum full scale. */
 
-	/* These values in conjunction with max_full_scale (pg. 9) helps
+	/*
+	 * These values in conjunction with max_full_scale (pg. 9) helps
 	 * determine the appropriate value for setting the full scales. The
 	 * software allows the user to set the sensor full scale to an
 	 * arbitrary value. But setting the full scales has some hazards. If
@@ -342,30 +366,35 @@ struct jr3_channel {
 	 */
 
 	struct six_axis_array min_full_scale;	/* offset 0x0070 */
-	s32 reserved4;		/* offset 0x0076 */
+	s32 reserved4;				/* offset 0x0076 */
 
-	/* Transform_num is the transform number that is currently in use.
+	/*
+	 * Transform_num is the transform number that is currently in use.
 	 * This value is set by the JR3 DSP after the user has used command
 	 * (5) use transform # (pg. 33).
 	 */
 
-	s32 transform_num;	/* offset 0x0077 */
+	s32 transform_num;			/* offset 0x0077 */
 
-	/*  Max_full_scale is the recommended maximum full scale. See */
-	/*  min_full_scale (pg. 9) for more details. */
+	/*
+	 * Max_full_scale is the recommended maximum full scale.
+	 * See min_full_scale (pg. 9) for more details.
+	 */
 
 	struct six_axis_array max_full_scale;	/* offset 0x0078 */
-	s32 reserved5;		/* offset 0x007e */
+	s32 reserved5;				/* offset 0x007e */
 
-	/* Peak_address is the address of the data which will be monitored
+	/*
+	 * Peak_address is the address of the data which will be monitored
 	 * by the peak routine. This value is set by the user. The peak
 	 * routine will monitor any 8 contiguous addresses for peak values.
 	 * (ex. to watch filter3 data for peaks, set this value to 0x00a8).
 	 */
 
-	s32 peak_address;	/* offset 0x007f */
+	s32 peak_address;			/* offset 0x007f */
 
-	/* Full_scale is the sensor full scales which are currently in use.
+	/*
+	 * Full_scale is the sensor full scales which are currently in use.
 	 * Decoupled and filtered data is scaled so that +/- 16384 is equal
 	 * to the full scales. The engineering units used are indicated by
 	 * the units value discussed on page 16. The full scales for Fx, Fy,
@@ -377,9 +406,10 @@ struct jr3_channel {
 	 * axes used for each vector respectively.
 	 */
 
-	struct force_array full_scale;	/* offset 0x0080 */
+	struct force_array full_scale;		/* offset 0x0080 */
 
-	/* Offsets contains the sensor offsets. These values are subtracted from
+	/*
+	 * Offsets contains the sensor offsets. These values are subtracted from
 	 * the sensor data to obtain the decoupled data. The offsets are set a
 	 * few seconds (< 10) after the calibration data has been received.
 	 * They are set so that the output data will be zero. These values
@@ -392,23 +422,26 @@ struct jr3_channel {
 	 * about Z by 90 degrees, FY would be 5 and all others would be zero.
 	 */
 
-	struct six_axis_array offsets;	/* offset 0x0088 */
+	struct six_axis_array offsets;		/* offset 0x0088 */
 
-	/* Offset_num is the number of the offset currently in use. This
+	/*
+	 * Offset_num is the number of the offset currently in use. This
 	 * value is set by the JR3 DSP after the user has executed the use
 	 * offset # command (pg. 34). It can vary between 0 and 15.
 	 */
 
-	s32 offset_num;		/* offset 0x008e */
+	s32 offset_num;				/* offset 0x008e */
 
-	/* Vect_axes is a bit map showing which of the axes are being used
+	/*
+	 * Vect_axes is a bit map showing which of the axes are being used
 	 * in the vector calculations. This value is set by the JR3 DSP
 	 * after the user has executed the set vector axes command (pg. 37).
 	 */
 
-	u32 vect_axes;		/* offset 0x008f */
+	u32 vect_axes;				/* offset 0x008f */
 
-	/* Filter0 is the decoupled, unfiltered data from the JR3 sensor.
+	/*
+	 * Filter0 is the decoupled, unfiltered data from the JR3 sensor.
 	 * This data has had the offsets removed.
 	 *
 	 * These force_arrays hold the filtered data. The decoupled data is
@@ -420,23 +453,27 @@ struct jr3_channel {
 	 * cutoff at 125 Hz, 31.25 Hz, 7.813 Hz, 1.953 Hz and 0.4883 Hz.
 	 */
 
-	struct force_array filter[7];	/* offset 0x0090,
-					   offset 0x0098,
-					   offset 0x00a0,
-					   offset 0x00a8,
-					   offset 0x00b0,
-					   offset 0x00b8 ,
-					   offset 0x00c0 */
-
-	/* Rate_data is the calculated rate data. It is a first derivative
+	struct force_array filter[7];		/*
+						 * offset 0x0090,
+						 * offset 0x0098,
+						 * offset 0x00a0,
+						 * offset 0x00a8,
+						 * offset 0x00b0,
+						 * offset 0x00b8,
+						 * offset 0x00c0
+						 */
+
+	/*
+	 * Rate_data is the calculated rate data. It is a first derivative
 	 * calculation. It is calculated at a frequency specified by the
 	 * variable rate_divisor (pg. 12). The data on which the rate is
 	 * calculated is specified by the variable rate_address (pg. 12).
 	 */
 
-	struct force_array rate_data;	/* offset 0x00c8 */
+	struct force_array rate_data;		/* offset 0x00c8 */
 
-	/* Minimum_data & maximum_data are the minimum and maximum (peak)
+	/*
+	 * Minimum_data & maximum_data are the minimum and maximum (peak)
 	 * data values. The JR3 DSP can monitor any 8 contiguous data items
 	 * for minimums and maximums at full sensor bandwidth. This area is
 	 * only updated at user request. This is done so that the user does
@@ -451,7 +488,8 @@ struct jr3_channel {
 	struct force_array minimum_data;	/* offset 0x00d0 */
 	struct force_array maximum_data;	/* offset 0x00d8 */
 
-	/* Near_sat_value & sat_value contain the value used to determine if
+	/*
+	 * Near_sat_value & sat_value contain the value used to determine if
 	 * the raw sensor is saturated. Because of decoupling and offset
 	 * removal, it is difficult to tell from the processed data if the
 	 * sensor is saturated. These values, in conjunction with the error
@@ -465,10 +503,11 @@ struct jr3_channel {
 	 *   sat_value = 32768 - 2^(16 - ADC bits)
 	 */
 
-	s32 near_sat_value;	/* offset 0x00e0 */
-	s32 sat_value;		/* offset 0x00e1 */
+	s32 near_sat_value;			/* offset 0x00e0 */
+	s32 sat_value;				/* offset 0x00e1 */
 
-	/* Rate_address, rate_divisor & rate_count contain the data used to
+	/*
+	 * Rate_address, rate_divisor & rate_count contain the data used to
 	 * control the calculations of the rates. Rate_address is the
 	 * address of the data used for the rate calculation. The JR3 DSP
 	 * will calculate rates for any 8 contiguous values (ex. to
@@ -485,11 +524,12 @@ struct jr3_channel {
 	 * will minimize the time necessary to start the rate calculations.
 	 */
 
-	s32 rate_address;	/* offset 0x00e2 */
-	u32 rate_divisor;	/* offset 0x00e3 */
-	u32 rate_count;		/* offset 0x00e4 */
+	s32 rate_address;			/* offset 0x00e2 */
+	u32 rate_divisor;			/* offset 0x00e3 */
+	u32 rate_count;				/* offset 0x00e4 */
 
-	/* Command_word2 through command_word0 are the locations used to
+	/*
+	 * Command_word2 through command_word0 are the locations used to
 	 * send commands to the JR3 DSP. Their usage varies with the command
 	 * and is detailed later in the Command Definitions section (pg.
 	 * 29). In general the user places values into various memory
@@ -502,11 +542,12 @@ struct jr3_channel {
 	 * command_word1).
 	 */
 
-	s32 command_word2;	/* offset 0x00e5 */
-	s32 command_word1;	/* offset 0x00e6 */
-	s32 command_word0;	/* offset 0x00e7 */
+	s32 command_word2;			/* offset 0x00e5 */
+	s32 command_word1;			/* offset 0x00e6 */
+	s32 command_word0;			/* offset 0x00e7 */
 
-	/* Count1 through count6 are unsigned counters which are incremented
+	/*
+	 * Count1 through count6 are unsigned counters which are incremented
 	 * every time the matching filters are calculated. Filter1 is
 	 * calculated at the sensor data bandwidth. So this counter would
 	 * increment at 8 kHz for a typical sensor. The rest of the counters
@@ -518,14 +559,15 @@ struct jr3_channel {
 	 * once.
 	 */
 
-	u32 count1;		/* offset 0x00e8 */
-	u32 count2;		/* offset 0x00e9 */
-	u32 count3;		/* offset 0x00ea */
-	u32 count4;		/* offset 0x00eb */
-	u32 count5;		/* offset 0x00ec */
-	u32 count6;		/* offset 0x00ed */
+	u32 count1;				/* offset 0x00e8 */
+	u32 count2;				/* offset 0x00e9 */
+	u32 count3;				/* offset 0x00ea */
+	u32 count4;				/* offset 0x00eb */
+	u32 count5;				/* offset 0x00ec */
+	u32 count6;				/* offset 0x00ed */
 
-	/* Error_count is a running count of data reception errors. If this
+	/*
+	 * Error_count is a running count of data reception errors. If this
 	 * counter is changing rapidly, it probably indicates a bad sensor
 	 * cable connection or other hardware problem. In most installations
 	 * error_count should not change at all. But it is possible in an
@@ -535,75 +577,84 @@ struct jr3_channel {
 	 * where this counter counts a bad sample, that sample is ignored.
 	 */
 
-	u32 error_count;	/* offset 0x00ee */
+	u32 error_count;			/* offset 0x00ee */
 
-	/* Count_x is a counter which is incremented every time the JR3 DSP
+	/*
+	 * Count_x is a counter which is incremented every time the JR3 DSP
 	 * searches its job queues and finds nothing to do. It indicates the
 	 * amount of idle time the JR3 DSP has available. It can also be
 	 * used to determine if the JR3 DSP is alive. See the Performance
 	 * Issues section on pg. 49 for more details.
 	 */
 
-	u32 count_x;		/* offset 0x00ef */
+	u32 count_x;				/* offset 0x00ef */
 
-	/* Warnings & errors contain the warning and error bits
+	/*
+	 * Warnings & errors contain the warning and error bits
 	 * respectively. The format of these two words is discussed on page
 	 * 21 under the headings warnings_bits and error_bits.
 	 */
 
-	u32 warnings;		/* offset 0x00f0 */
-	u32 errors;		/* offset 0x00f1 */
+	u32 warnings;				/* offset 0x00f0 */
+	u32 errors;				/* offset 0x00f1 */
 
-	/* Threshold_bits is a word containing the bits that are set by the
+	/*
+	 * Threshold_bits is a word containing the bits that are set by the
 	 * load envelopes. See load_envelopes (pg. 17) and thresh_struct
 	 * (pg. 23) for more details.
 	 */
 
-	s32 threshold_bits;	/* offset 0x00f2 */
+	s32 threshold_bits;			/* offset 0x00f2 */
 
-	/* Last_crc is the value that shows the actual calculated CRC. CRC
+	/*
+	 * Last_crc is the value that shows the actual calculated CRC. CRC
 	 * is short for cyclic redundancy code. It should be zero. See the
 	 * description for cal_crc_bad (pg. 21) for more information.
 	 */
 
-	s32 last_CRC;		/* offset 0x00f3 */
+	s32 last_CRC;				/* offset 0x00f3 */
 
-	/* EEProm_ver_no contains the version number of the sensor EEProm.
+	/*
+	 * EEProm_ver_no contains the version number of the sensor EEProm.
 	 * EEProm version numbers can vary between 0 and 255.
 	 * Software_ver_no contains the software version number. Version
 	 * 3.02 would be stored as 302.
 	 */
 
-	s32 eeprom_ver_no;	/* offset 0x00f4 */
-	s32 software_ver_no;	/* offset 0x00f5 */
+	s32 eeprom_ver_no;			/* offset 0x00f4 */
+	s32 software_ver_no;			/* offset 0x00f5 */
 
-	/* Software_day & software_year are the release date of the software
+	/*
+	 * Software_day & software_year are the release date of the software
 	 * the JR3 DSP is currently running. Day is the day of the year,
 	 * with January 1 being 1, and December 31, being 365 for non leap
 	 * years.
 	 */
 
-	s32 software_day;	/* offset 0x00f6 */
-	s32 software_year;	/* offset 0x00f7 */
+	s32 software_day;			/* offset 0x00f6 */
+	s32 software_year;			/* offset 0x00f7 */
 
-	/* Serial_no & model_no are the two values which uniquely identify a
+	/*
+	 * Serial_no & model_no are the two values which uniquely identify a
 	 * sensor. This model number does not directly correspond to the JR3
 	 * model number, but it will provide a unique identifier for
 	 * different sensor configurations.
 	 */
 
-	u32 serial_no;		/* offset 0x00f8 */
-	u32 model_no;		/* offset 0x00f9 */
+	u32 serial_no;				/* offset 0x00f8 */
+	u32 model_no;				/* offset 0x00f9 */
 
-	/* Cal_day & cal_year are the sensor calibration date. Day is the
+	/*
+	 * Cal_day & cal_year are the sensor calibration date. Day is the
 	 * day of the year, with January 1 being 1, and December 31, being
 	 * 366 for leap years.
 	 */
 
-	s32 cal_day;		/* offset 0x00fa */
-	s32 cal_year;		/* offset 0x00fb */
+	s32 cal_day;				/* offset 0x00fa */
+	s32 cal_year;				/* offset 0x00fb */
 
-	/* Units is an enumerated read only value defining the engineering
+	/*
+	 * Units is an enumerated read only value defining the engineering
 	 * units used in the sensor full scale. The meanings of particular
 	 * values are discussed in the section detailing the force_units
 	 * structure on page 22. The engineering units are setto customer
@@ -626,20 +677,22 @@ struct jr3_channel {
 	 * received.
 	 */
 
-	u32 units;		/* offset 0x00fc */
-	s32 bits;		/* offset 0x00fd */
-	s32 channels;		/* offset 0x00fe */
+	u32 units;				/* offset 0x00fc */
+	s32 bits;				/* offset 0x00fd */
+	s32 channels;				/* offset 0x00fe */
 
-	/* Thickness specifies the overall thickness of the sensor from
+	/*
+	 * Thickness specifies the overall thickness of the sensor from
 	 * flange to flange. The engineering units for this value are
 	 * contained in units (pg. 16). The sensor calibration is relative
 	 * to the center of the sensor. This value allows easy coordinate
 	 * transformation from the center of the sensor to either flange.
 	 */
 
-	s32 thickness;		/* offset 0x00ff */
+	s32 thickness;				/* offset 0x00ff */
 
-	/* Load_envelopes is a table containing the load envelope
+	/*
+	 * Load_envelopes is a table containing the load envelope
 	 * descriptions. There are 16 possible load envelope slots in the
 	 * table. The slots are on 16 word boundaries and are numbered 0-15.
 	 * Each load envelope needs to start at the beginning of a slot but
@@ -655,7 +708,8 @@ struct jr3_channel {
 
 	struct le_struct load_envelopes[0x10];	/* offset 0x0100 */
 
-	/* Transforms is a table containing the transform descriptions.
+	/*
+	 * Transforms is a table containing the transform descriptions.
 	 * There are 16 possible transform slots in the table. The slots are
 	 * on 16 word boundaries and are numbered 0-15. Each transform needs
 	 * to start at the beginning of a slot but need not be fully
diff --git a/drivers/staging/comedi/drivers/ni_670x.c b/drivers/staging/comedi/drivers/ni_670x.c
index 3e72718801a9..74911dbb2561 100644
--- a/drivers/staging/comedi/drivers/ni_670x.c
+++ b/drivers/staging/comedi/drivers/ni_670x.c
@@ -1,40 +1,34 @@
 /*
-    comedi/drivers/ni_670x.c
-    Hardware driver for NI 670x devices
-
-    COMEDI - Linux Control and Measurement Device Interface
-    Copyright (C) 1997-2001 David A. Schleef <ds@schleef.org>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-*/
-/*
-Driver: ni_670x
-Description: National Instruments 670x
-Author: Bart Joris <bjoris@advalvas.be>
-Updated: Wed, 11 Dec 2002 18:25:35 -0800
-Devices: [National Instruments] PCI-6703 (ni_670x), PCI-6704
-Status: unknown
-
-Commands are not supported.
-*/
+ * Comedi driver for NI 670x devices
+ *
+ * COMEDI - Linux Control and Measurement Device Interface
+ * Copyright (C) 1997-2001 David A. Schleef <ds@schleef.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
 
 /*
-	Bart Joris <bjoris@advalvas.be> Last updated on 20/08/2001
-
-	Manuals:
-
-	322110a.pdf	PCI/PXI-6704 User Manual
-	322110b.pdf	PCI/PXI-6703/6704 User Manual
-
-*/
+ * Driver: ni_670x
+ * Description: National Instruments 670x
+ * Author: Bart Joris <bjoris@advalvas.be>
+ * Updated: Wed, 11 Dec 2002 18:25:35 -0800
+ * Devices: [National Instruments] PCI-6703 (ni_670x), PCI-6704
+ * Status: unknown
+ *
+ * Commands are not supported.
+ *
+ * Manuals:
+ *   322110a.pdf	PCI/PXI-6704 User Manual
+ *   322110b.pdf	PCI/PXI-6703/6704 User Manual
+ */
 
 #include <linux/module.h>
 #include <linux/interrupt.h>
diff --git a/drivers/staging/comedi/drivers/ni_at_a2150.c b/drivers/staging/comedi/drivers/ni_at_a2150.c
index 9b444f8c4e33..5a4dcc6e61d8 100644
--- a/drivers/staging/comedi/drivers/ni_at_a2150.c
+++ b/drivers/staging/comedi/drivers/ni_at_a2150.c
@@ -1,62 +1,47 @@
 /*
-    comedi/drivers/ni_at_a2150.c
-    Driver for National Instruments AT-A2150 boards
-    Copyright (C) 2001, 2002 Frank Mori Hess <fmhess@users.sourceforge.net>
-
-    COMEDI - Linux Control and Measurement Device Interface
-    Copyright (C) 2000 David A. Schleef <ds@schleef.org>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-*/
-/*
-Driver: ni_at_a2150
-Description: National Instruments AT-A2150
-Author: Frank Mori Hess
-Status: works
-Devices: [National Instruments] AT-A2150C (at_a2150c), AT-2150S (at_a2150s)
-
-If you want to ac couple the board's inputs, use AREF_OTHER.
-
-Configuration options:
-  [0] - I/O port base address
-  [1] - IRQ (optional, required for timed conversions)
-  [2] - DMA (optional, required for timed conversions)
+ * Comedi driver for National Instruments AT-A2150 boards
+ * Copyright (C) 2001, 2002 Frank Mori Hess <fmhess@users.sourceforge.net>
+ *
+ * COMEDI - Linux Control and Measurement Device Interface
+ * Copyright (C) 2000 David A. Schleef <ds@schleef.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
 
-*/
 /*
-Yet another driver for obsolete hardware brought to you by Frank Hess.
-Testing and debugging help provided by Dave Andruczyk.
-
-This driver supports the boards:
-
-AT-A2150C
-AT-A2150S
-
-The only difference is their master clock frequencies.
-
-Options:
-	[0] - base io address
-	[1] - irq
-	[2] - dma channel
-
-References (from ftp://ftp.natinst.com/support/manuals):
-
-	   320360.pdf  AT-A2150 User Manual
-
-TODO:
-
-analog level triggering
-TRIG_WAKE_EOS
-
-*/
+ * Driver: ni_at_a2150
+ * Description: National Instruments AT-A2150
+ * Author: Frank Mori Hess
+ * Status: works
+ * Devices: [National Instruments] AT-A2150C (at_a2150c), AT-2150S (at_a2150s)
+ *
+ * Configuration options:
+ *   [0] - I/O port base address
+ *   [1] - IRQ (optional, required for timed conversions)
+ *   [2] - DMA (optional, required for timed conversions)
+ *
+ * Yet another driver for obsolete hardware brought to you by Frank Hess.
+ * Testing and debugging help provided by Dave Andruczyk.
+ *
+ * If you want to ac couple the board's inputs, use AREF_OTHER.
+ *
+ * The only difference in the boards is their master clock frequencies.
+ *
+ * References (from ftp://ftp.natinst.com/support/manuals):
+ *   320360.pdf  AT-A2150 User Manual
+ *
+ * TODO:
+ * - analog level triggering
+ * - TRIG_WAKE_EOS
+ */
 
 #include <linux/module.h>
 #include <linux/delay.h>
@@ -73,48 +58,52 @@ TRIG_WAKE_EOS
 
 /* Registers and bits */
 #define CONFIG_REG		0x0
-#define   CHANNEL_BITS(x)		((x) & 0x7)
+#define   CHANNEL_BITS(x)	((x) & 0x7)
 #define   CHANNEL_MASK		0x7
-#define   CLOCK_SELECT_BITS(x)		(((x) & 0x3) << 3)
-#define   CLOCK_DIVISOR_BITS(x)		(((x) & 0x3) << 5)
+#define   CLOCK_SELECT_BITS(x)	(((x) & 0x3) << 3)
+#define   CLOCK_DIVISOR_BITS(x)	(((x) & 0x3) << 5)
 #define   CLOCK_MASK		(0xf << 3)
-#define   ENABLE0_BIT		0x80	/*  enable (don't internally ground) channels 0 and 1 */
-#define   ENABLE1_BIT		0x100	/*  enable (don't internally ground) channels 2 and 3 */
-#define   AC0_BIT		0x200	/*  ac couple channels 0,1 */
-#define   AC1_BIT		0x400	/*  ac couple channels 2,3 */
-#define   APD_BIT		0x800	/*  analog power down */
-#define   DPD_BIT		0x1000	/*  digital power down */
-#define TRIGGER_REG		0x2	/*  trigger config register */
-#define   POST_TRIGGER_BITS		0x2
-#define   DELAY_TRIGGER_BITS		0x3
-#define   HW_TRIG_EN		0x10	/*  enable hardware trigger */
-#define FIFO_START_REG		0x6	/*  software start aquistion trigger */
-#define FIFO_RESET_REG		0x8	/*  clears fifo + fifo flags */
-#define FIFO_DATA_REG		0xa	/*  read data */
-#define DMA_TC_CLEAR_REG		0xe	/*  clear dma terminal count interrupt */
-#define STATUS_REG		0x12	/*  read only */
-#define   FNE_BIT		0x1	/*  fifo not empty */
-#define   OVFL_BIT		0x8	/*  fifo overflow */
-#define   EDAQ_BIT		0x10	/*  end of acquisition interrupt */
-#define   DCAL_BIT		0x20	/*  offset calibration in progress */
-#define   INTR_BIT		0x40	/*  interrupt has occurred */
-#define   DMA_TC_BIT		0x80	/*  dma terminal count interrupt has occurred */
-#define   ID_BITS(x)	(((x) >> 8) & 0x3)
-#define IRQ_DMA_CNTRL_REG		0x12	/*  write only */
-#define   DMA_CHAN_BITS(x)		((x) & 0x7)	/*  sets dma channel */
-#define   DMA_EN_BIT		0x8	/*  enables dma */
-#define   IRQ_LVL_BITS(x)		(((x) & 0xf) << 4)	/*  sets irq level */
-#define   FIFO_INTR_EN_BIT		0x100	/*  enable fifo interrupts */
-#define   FIFO_INTR_FHF_BIT		0x200	/*  interrupt fifo half full */
-#define   DMA_INTR_EN_BIT		0x800	/*  enable interrupt on dma terminal count */
-#define   DMA_DEM_EN_BIT	0x1000	/*  enables demand mode dma */
+/* enable (don't internally ground) channels 0 and 1 */
+#define   ENABLE0_BIT		0x80
+/* enable (don't internally ground) channels 2 and 3 */
+#define   ENABLE1_BIT		0x100
+#define   AC0_BIT		0x200	/* ac couple channels 0,1 */
+#define   AC1_BIT		0x400	/* ac couple channels 2,3 */
+#define   APD_BIT		0x800	/* analog power down */
+#define   DPD_BIT		0x1000	/* digital power down */
+#define TRIGGER_REG		0x2	/* trigger config register */
+#define   POST_TRIGGER_BITS	0x2
+#define   DELAY_TRIGGER_BITS	0x3
+#define   HW_TRIG_EN		0x10	/* enable hardware trigger */
+#define FIFO_START_REG		0x6	/* software start aquistion trigger */
+#define FIFO_RESET_REG		0x8	/* clears fifo + fifo flags */
+#define FIFO_DATA_REG		0xa	/* read data */
+#define DMA_TC_CLEAR_REG	0xe	/* clear dma terminal count interrupt */
+#define STATUS_REG		0x12	/* read only */
+#define   FNE_BIT		0x1	/* fifo not empty */
+#define   OVFL_BIT		0x8	/* fifo overflow */
+#define   EDAQ_BIT		0x10	/* end of acquisition interrupt */
+#define   DCAL_BIT		0x20	/* offset calibration in progress */
+#define   INTR_BIT		0x40	/* interrupt has occurred */
+/* dma terminal count interrupt has occurred */
+#define   DMA_TC_BIT		0x80
+#define   ID_BITS(x)		(((x) >> 8) & 0x3)
+#define IRQ_DMA_CNTRL_REG	0x12			/* write only */
+#define   DMA_CHAN_BITS(x)	((x) & 0x7)		/* sets dma channel */
+#define   DMA_EN_BIT		0x8			/* enables dma */
+#define   IRQ_LVL_BITS(x)	(((x) & 0xf) << 4)	/* sets irq level */
+#define   FIFO_INTR_EN_BIT	0x100	/* enable fifo interrupts */
+#define   FIFO_INTR_FHF_BIT	0x200	/* interrupt fifo half full */
+/* enable interrupt on dma terminal count */
+#define   DMA_INTR_EN_BIT	0x800
+#define   DMA_DEM_EN_BIT	0x1000	/* enables demand mode dma */
 #define I8253_BASE_REG		0x14
 
 struct a2150_board {
 	const char *name;
-	int clock[4];		/*  master clock periods, in nanoseconds */
-	int num_clocks;		/*  number of available master clock speeds */
-	int ai_speed;		/*  maximum conversion rate in nanoseconds */
+	int clock[4];		/* master clock periods, in nanoseconds */
+	int num_clocks;		/* number of available master clock speeds */
+	int ai_speed;		/* maximum conversion rate in nanoseconds */
 };
 
 /* analog input range */
@@ -144,8 +133,8 @@ static const struct a2150_board a2150_boards[] = {
 struct a2150_private {
 	struct comedi_isadma *dma;
 	unsigned int count;	/* number of data points left to be taken */
-	int irq_dma_bits;	/*  irq/dma register bits */
-	int config_bits;	/*  config register bits */
+	int irq_dma_bits;	/* irq/dma register bits */
+	int config_bits;	/* config register bits */
 };
 
 /* interrupt service routine */
@@ -189,13 +178,13 @@ static irqreturn_t a2150_interrupt(int irq, void *d)
 	 */
 	residue = comedi_isadma_disable(desc->chan);
 
-	/*  figure out how many points to read */
+	/* figure out how many points to read */
 	max_points = comedi_bytes_to_samples(s, desc->size);
 	num_points = max_points - comedi_bytes_to_samples(s, residue);
 	if (devpriv->count < num_points && cmd->stop_src == TRIG_COUNT)
 		num_points = devpriv->count;
 
-	/*  figure out how many points will be stored next time */
+	/* figure out how many points will be stored next time */
 	leftover = 0;
 	if (cmd->stop_src == TRIG_NONE) {
 		leftover = comedi_bytes_to_samples(s, desc->size);
@@ -204,7 +193,8 @@ static irqreturn_t a2150_interrupt(int irq, void *d)
 		if (leftover > max_points)
 			leftover = max_points;
 	}
-	/* there should only be a residue if collection was stopped by having
+	/*
+	 * There should only be a residue if collection was stopped by having
 	 * the stop_src set to an external trigger, in which case there
 	 * will be no more data
 	 */
@@ -214,7 +204,7 @@ static irqreturn_t a2150_interrupt(int irq, void *d)
 	for (i = 0; i < num_points; i++) {
 		/* write data point to comedi buffer */
 		dpnt = buf[i];
-		/*  convert from 2's complement to unsigned coding */
+		/* convert from 2's complement to unsigned coding */
 		dpnt ^= 0x8000;
 		comedi_buf_write_samples(s, &dpnt, 1);
 		if (cmd->stop_src == TRIG_COUNT) {
@@ -244,14 +234,14 @@ static int a2150_cancel(struct comedi_device *dev, struct comedi_subdevice *s)
 	struct comedi_isadma *dma = devpriv->dma;
 	struct comedi_isadma_desc *desc = &dma->desc[0];
 
-	/*  disable dma on card */
+	/* disable dma on card */
 	devpriv->irq_dma_bits &= ~DMA_INTR_EN_BIT & ~DMA_EN_BIT;
 	outw(devpriv->irq_dma_bits, dev->iobase + IRQ_DMA_CNTRL_REG);
 
-	/*  disable computer's dma */
+	/* disable computer's dma */
 	comedi_isadma_disable(desc->chan);
 
-	/*  clear fifo and reset triggering circuitry */
+	/* clear fifo and reset triggering circuitry */
 	outw(0, dev->iobase + FIFO_RESET_REG);
 
 	return 0;
@@ -270,7 +260,7 @@ static int a2150_get_timing(struct comedi_device *dev, unsigned int *period,
 	int lub_divisor_shift, lub_index, glb_divisor_shift, glb_index;
 	int i, j;
 
-	/*  initialize greatest lower and least upper bounds */
+	/* initialize greatest lower and least upper bounds */
 	lub_divisor_shift = 3;
 	lub_index = 0;
 	lub = board->clock[lub_index] * (1 << lub_divisor_shift);
@@ -278,19 +268,19 @@ static int a2150_get_timing(struct comedi_device *dev, unsigned int *period,
 	glb_index = board->num_clocks - 1;
 	glb = board->clock[glb_index] * (1 << glb_divisor_shift);
 
-	/*  make sure period is in available range */
+	/* make sure period is in available range */
 	if (*period < glb)
 		*period = glb;
 	if (*period > lub)
 		*period = lub;
 
-	/*  we can multiply period by 1, 2, 4, or 8, using (1 << i) */
+	/* we can multiply period by 1, 2, 4, or 8, using (1 << i) */
 	for (i = 0; i < 4; i++) {
-		/*  there are a maximum of 4 master clocks */
+		/* there are a maximum of 4 master clocks */
 		for (j = 0; j < board->num_clocks; j++) {
-			/*  temp is the period in nanosec we are evaluating */
+			/* temp is the period in nanosec we are evaluating */
 			temp = board->clock[j] * (1 << i);
-			/*  if it is the best match yet */
+			/* if it is the best match yet */
 			if (temp < lub && temp >= *period) {
 				lub_divisor_shift = i;
 				lub_index = j;
@@ -306,7 +296,7 @@ static int a2150_get_timing(struct comedi_device *dev, unsigned int *period,
 	switch (flags & CMDF_ROUND_MASK) {
 	case CMDF_ROUND_NEAREST:
 	default:
-		/*  if least upper bound is better approximation */
+		/* if least upper bound is better approximation */
 		if (lub - *period < *period - glb)
 			*period = lub;
 		else
@@ -320,7 +310,7 @@ static int a2150_get_timing(struct comedi_device *dev, unsigned int *period,
 		break;
 	}
 
-	/*  set clock bits for config register appropriately */
+	/* set clock bits for config register appropriately */
 	devpriv->config_bits &= ~CLOCK_MASK;
 	if (*period == lub) {
 		devpriv->config_bits |=
@@ -495,7 +485,7 @@ static int a2150_ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 			"dma incompatible with hard real-time interrupt (CMDF_PRIORITY), aborting\n");
 		return -1;
 	}
-	/*  clear fifo and reset triggering circuitry */
+	/* clear fifo and reset triggering circuitry */
 	outw(0, dev->iobase + FIFO_RESET_REG);
 
 	/* setup chanlist */
@@ -503,7 +493,7 @@ static int a2150_ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 			       cmd->chanlist_len) < 0)
 		return -1;
 
-	/*  setup ac/dc coupling */
+	/* setup ac/dc coupling */
 	if (CR_AREF(cmd->chanlist[0]) == AREF_OTHER)
 		devpriv->config_bits |= AC0_BIT;
 	else
@@ -513,18 +503,18 @@ static int a2150_ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 	else
 		devpriv->config_bits &= ~AC1_BIT;
 
-	/*  setup timing */
+	/* setup timing */
 	a2150_get_timing(dev, &cmd->scan_begin_arg, cmd->flags);
 
-	/*  send timing, channel, config bits */
+	/* send timing, channel, config bits */
 	outw(devpriv->config_bits, dev->iobase + CONFIG_REG);
 
-	/*  initialize number of samples remaining */
+	/* initialize number of samples remaining */
 	devpriv->count = cmd->stop_arg * cmd->chanlist_len;
 
 	comedi_isadma_disable(desc->chan);
 
-	/*  set size of transfer to fill in 1/3 second */
+	/* set size of transfer to fill in 1/3 second */
 #define ONE_THIRD_SECOND 333333333
 	desc->size = comedi_bytes_per_sample(s) * cmd->chanlist_len *
 		    ONE_THIRD_SECOND / cmd->scan_begin_arg;
@@ -536,40 +526,45 @@ static int a2150_ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 
 	comedi_isadma_program(desc);
 
-	/* clear dma interrupt before enabling it, to try and get rid of that
-	 * one spurious interrupt that has been happening */
+	/*
+	 * Clear dma interrupt before enabling it, to try and get rid of
+	 * that one spurious interrupt that has been happening.
+	 */
 	outw(0x00, dev->iobase + DMA_TC_CLEAR_REG);
 
-	/*  enable dma on card */
+	/* enable dma on card */
 	devpriv->irq_dma_bits |= DMA_INTR_EN_BIT | DMA_EN_BIT;
 	outw(devpriv->irq_dma_bits, dev->iobase + IRQ_DMA_CNTRL_REG);
 
-	/*  may need to wait 72 sampling periods if timing was changed */
+	/* may need to wait 72 sampling periods if timing was changed */
 	comedi_8254_load(dev->pacer, 2, 72, I8254_MODE0 | I8254_BINARY);
 
-	/*  setup start triggering */
+	/* setup start triggering */
 	trigger_bits = 0;
-	/*  decide if we need to wait 72 periods for valid data */
+	/* decide if we need to wait 72 periods for valid data */
 	if (cmd->start_src == TRIG_NOW &&
 	    (old_config_bits & CLOCK_MASK) !=
 	    (devpriv->config_bits & CLOCK_MASK)) {
-		/*  set trigger source to delay trigger */
+		/* set trigger source to delay trigger */
 		trigger_bits |= DELAY_TRIGGER_BITS;
 	} else {
-		/*  otherwise no delay */
+		/* otherwise no delay */
 		trigger_bits |= POST_TRIGGER_BITS;
 	}
-	/*  enable external hardware trigger */
+	/* enable external hardware trigger */
 	if (cmd->start_src == TRIG_EXT) {
 		trigger_bits |= HW_TRIG_EN;
 	} else if (cmd->start_src == TRIG_OTHER) {
-		/*  XXX add support for level/slope start trigger using TRIG_OTHER */
+		/*
+		 * XXX add support for level/slope start trigger
+		 * using TRIG_OTHER
+		 */
 		dev_err(dev->class_dev, "you shouldn't see this?\n");
 	}
-	/*  send trigger config bits */
+	/* send trigger config bits */
 	outw(trigger_bits, dev->iobase + TRIGGER_REG);
 
-	/*  start acquisition for soft trigger */
+	/* start acquisition for soft trigger */
 	if (cmd->start_src == TRIG_NOW)
 		outw(0, dev->iobase + FIFO_START_REG);
 
@@ -596,28 +591,28 @@ static int a2150_ai_rinsn(struct comedi_device *dev, struct comedi_subdevice *s,
 	unsigned int n;
 	int ret;
 
-	/*  clear fifo and reset triggering circuitry */
+	/* clear fifo and reset triggering circuitry */
 	outw(0, dev->iobase + FIFO_RESET_REG);
 
 	/* setup chanlist */
 	if (a2150_set_chanlist(dev, CR_CHAN(insn->chanspec), 1) < 0)
 		return -1;
 
-	/*  set dc coupling */
+	/* set dc coupling */
 	devpriv->config_bits &= ~AC0_BIT;
 	devpriv->config_bits &= ~AC1_BIT;
 
-	/*  send timing, channel, config bits */
+	/* send timing, channel, config bits */
 	outw(devpriv->config_bits, dev->iobase + CONFIG_REG);
 
-	/*  disable dma on card */
+	/* disable dma on card */
 	devpriv->irq_dma_bits &= ~DMA_INTR_EN_BIT & ~DMA_EN_BIT;
 	outw(devpriv->irq_dma_bits, dev->iobase + IRQ_DMA_CNTRL_REG);
 
-	/*  setup start triggering */
+	/* setup start triggering */
 	outw(0, dev->iobase + TRIGGER_REG);
 
-	/*  start acquisition for soft trigger */
+	/* start acquisition for soft trigger */
 	outw(0, dev->iobase + FIFO_START_REG);
 
 	/*
@@ -632,7 +627,7 @@ static int a2150_ai_rinsn(struct comedi_device *dev, struct comedi_subdevice *s,
 		inw(dev->iobase + FIFO_DATA_REG);
 	}
 
-	/*  read data */
+	/* read data */
 	for (n = 0; n < insn->n; n++) {
 		ret = comedi_timeout(dev, s, insn, a2150_ai_eoc, 0);
 		if (ret)
@@ -642,7 +637,7 @@ static int a2150_ai_rinsn(struct comedi_device *dev, struct comedi_subdevice *s,
 		data[n] ^= 0x8000;
 	}
 
-	/*  clear fifo and reset triggering circuitry */
+	/* clear fifo and reset triggering circuitry */
 	outw(0, dev->iobase + FIFO_RESET_REG);
 
 	return n;
@@ -749,16 +744,16 @@ static int a2150_attach(struct comedi_device *dev, struct comedi_devconfig *it)
 		s->cancel = a2150_cancel;
 	}
 
-	/*  set card's irq and dma levels */
+	/* set card's irq and dma levels */
 	outw(devpriv->irq_dma_bits, dev->iobase + IRQ_DMA_CNTRL_REG);
 
-	/*  reset and sync adc clock circuitry */
+	/* reset and sync adc clock circuitry */
 	outw_p(DPD_BIT | APD_BIT, dev->iobase + CONFIG_REG);
 	outw_p(DPD_BIT, dev->iobase + CONFIG_REG);
-	/*  initialize configuration register */
+	/* initialize configuration register */
 	devpriv->config_bits = 0;
 	outw(devpriv->config_bits, dev->iobase + CONFIG_REG);
-	/*  wait until offset calibration is done, then enable analog inputs */
+	/* wait until offset calibration is done, then enable analog inputs */
 	for (i = 0; i < timeout; i++) {
 		if ((DCAL_BIT & inw(dev->iobase + STATUS_REG)) == 0)
 			break;
diff --git a/drivers/staging/comedi/drivers/ni_atmio.c b/drivers/staging/comedi/drivers/ni_atmio.c
index 95435b81aa55..ffcf7afce684 100644
--- a/drivers/staging/comedi/drivers/ni_atmio.c
+++ b/drivers/staging/comedi/drivers/ni_atmio.c
@@ -1,93 +1,84 @@
 /*
-    comedi/drivers/ni_atmio.c
-    Hardware driver for NI AT-MIO E series cards
-
-    COMEDI - Linux Control and Measurement Device Interface
-    Copyright (C) 1997-2001 David A. Schleef <ds@schleef.org>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-*/
-/*
-Driver: ni_atmio
-Description: National Instruments AT-MIO-E series
-Author: ds
-Devices: [National Instruments] AT-MIO-16E-1 (ni_atmio),
-  AT-MIO-16E-2, AT-MIO-16E-10, AT-MIO-16DE-10, AT-MIO-64E-3,
-  AT-MIO-16XE-50, AT-MIO-16XE-10, AT-AI-16XE-10
-Status: works
-Updated: Thu May  1 20:03:02 CDT 2003
-
-The driver has 2.6 kernel isapnp support, and
-will automatically probe for a supported board if the
-I/O base is left unspecified with comedi_config.
-However, many of
-the isapnp id numbers are unknown.  If your board is not
-recognized, please send the output of 'cat /proc/isapnp'
-(you may need to modprobe the isa-pnp module for
-/proc/isapnp to exist) so the
-id numbers for your board can be added to the driver.
-
-Otherwise, you can use the isapnptools package to configure
-your board.  Use isapnp to
-configure the I/O base and IRQ for the board, and then pass
-the same values as
-parameters in comedi_config.  A sample isapnp.conf file is included
-in the etc/ directory of Comedilib.
-
-Comedilib includes a utility to autocalibrate these boards.  The
-boards seem to boot into a state where the all calibration DACs
-are at one extreme of their range, thus the default calibration
-is terrible.  Calibration at boot is strongly encouraged.
-
-To use the extended digital I/O on some of the boards, enable the
-8255 driver when configuring the Comedi source tree.
-
-External triggering is supported for some events.  The channel index
-(scan_begin_arg, etc.) maps to PFI0 - PFI9.
-
-Some of the more esoteric triggering possibilities of these boards
-are not supported.
-*/
-/*
-	The real guts of the driver is in ni_mio_common.c, which is included
-	both here and in ni_pcimio.c
-
-	Interrupt support added by Truxton Fulton <trux@truxton.com>
-
-	References for specifications:
-
-	   340747b.pdf  Register Level Programmer Manual (obsolete)
-	   340747c.pdf  Register Level Programmer Manual (new)
-	   DAQ-STC reference manual
-
-	Other possibly relevant info:
-
-	   320517c.pdf  User manual (obsolete)
-	   320517f.pdf  User manual (new)
-	   320889a.pdf  delete
-	   320906c.pdf  maximum signal ratings
-	   321066a.pdf  about 16x
-	   321791a.pdf  discontinuation of at-mio-16e-10 rev. c
-	   321808a.pdf  about at-mio-16e-10 rev P
-	   321837a.pdf  discontinuation of at-mio-16de-10 rev d
-	   321838a.pdf  about at-mio-16de-10 rev N
-
-	ISSUES:
-
-	need to deal with external reference for DAC, and other DAC
-	properties in board properties
+ * Comedi driver for NI AT-MIO E series cards
+ *
+ * COMEDI - Linux Control and Measurement Device Interface
+ * Copyright (C) 1997-2001 David A. Schleef <ds@schleef.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
 
-	deal with at-mio-16de-10 revision D to N changes, etc.
+/*
+ * Driver: ni_atmio
+ * Description: National Instruments AT-MIO-E series
+ * Author: ds
+ * Devices: [National Instruments] AT-MIO-16E-1 (ni_atmio),
+ *   AT-MIO-16E-2, AT-MIO-16E-10, AT-MIO-16DE-10, AT-MIO-64E-3,
+ *   AT-MIO-16XE-50, AT-MIO-16XE-10, AT-AI-16XE-10
+ * Status: works
+ * Updated: Thu May  1 20:03:02 CDT 2003
+ *
+ * The driver has 2.6 kernel isapnp support, and will automatically probe for
+ * a supported board if the I/O base is left unspecified with comedi_config.
+ * However, many of the isapnp id numbers are unknown. If your board is not
+ * recognized, please send the output of 'cat /proc/isapnp' (you may need to
+ * modprobe the isa-pnp module for /proc/isapnp to exist) so the id numbers
+ * for your board can be added to the driver.
+ *
+ * Otherwise, you can use the isapnptools package to configure your board.
+ * Use isapnp to configure the I/O base and IRQ for the board, and then pass
+ * the same values as parameters in comedi_config. A sample isapnp.conf file
+ * is included in the etc/ directory of Comedilib.
+ *
+ * Comedilib includes a utility to autocalibrate these boards. The boards
+ * seem to boot into a state where the all calibration DACs are at one
+ * extreme of their range, thus the default calibration is terrible.
+ * Calibration at boot is strongly encouraged.
+ *
+ * To use the extended digital I/O on some of the boards, enable the
+ * 8255 driver when configuring the Comedi source tree.
+ *
+ * External triggering is supported for some events. The channel index
+ * (scan_begin_arg, etc.) maps to PFI0 - PFI9.
+ *
+ * Some of the more esoteric triggering possibilities of these boards are
+ * not supported.
+ */
 
-*/
+/*
+ * The real guts of the driver is in ni_mio_common.c, which is included
+ * both here and in ni_pcimio.c
+ *
+ * Interrupt support added by Truxton Fulton <trux@truxton.com>
+ *
+ * References for specifications:
+ *	340747b.pdf  Register Level Programmer Manual (obsolete)
+ *	340747c.pdf  Register Level Programmer Manual (new)
+ *		     DAQ-STC reference manual
+ *
+ * Other possibly relevant info:
+ *	320517c.pdf  User manual (obsolete)
+ *	320517f.pdf  User manual (new)
+ *	320889a.pdf  delete
+ *	320906c.pdf  maximum signal ratings
+ *	321066a.pdf  about 16x
+ *	321791a.pdf  discontinuation of at-mio-16e-10 rev. c
+ *	321808a.pdf  about at-mio-16e-10 rev P
+ *	321837a.pdf  discontinuation of at-mio-16de-10 rev d
+ *	321838a.pdf  about at-mio-16de-10 rev N
+ *
+ * ISSUES:
+ * - need to deal with external reference for DAC, and other DAC
+ *   properties in board properties
+ * - deal with at-mio-16de-10 revision D to N changes, etc.
+ */
 
 #include <linux/module.h>
 #include <linux/interrupt.h>
@@ -98,10 +89,7 @@ are not supported.
 #include "ni_stc.h"
 #include "8255.h"
 
-/*
- *  AT specific setup
- */
-
+/* AT specific setup */
 static const struct ni_board_struct ni_boards[] = {
 	{
 		.name		= "at-mio-16e-1",
@@ -215,7 +203,7 @@ static const struct ni_board_struct ni_boards[] = {
 		.n_adchan	= 16,
 		.ai_maxdata	= 0xffff,
 		.ai_fifo_depth	= 512,
-		.alwaysdither	= 1,	/* unknown */
+		.alwaysdither	= 1,		/* unknown */
 		.gainlkup	= ai_gain_14,
 		.ai_speed	= 10000,
 		.caldac		= { dac8800, dac8043, ad8522 },
@@ -287,10 +275,10 @@ static const struct ni_board_struct *ni_atmio_probe(struct comedi_device *dev)
 	}
 	if (device_id == 255)
 		dev_err(dev->class_dev, "can't find board\n");
-	 else if (device_id == 0)
+	else if (device_id == 0)
 		dev_err(dev->class_dev,
 			"EEPROM read error (?) or device not found\n");
-	 else
+	else
 		dev_err(dev->class_dev,
 			"unknown device ID %d -- contact author\n", device_id);
 
diff --git a/drivers/staging/comedi/drivers/ni_atmio16d.c b/drivers/staging/comedi/drivers/ni_atmio16d.c
index c3eb54622bc3..fb59b0ffbba6 100644
--- a/drivers/staging/comedi/drivers/ni_atmio16d.c
+++ b/drivers/staging/comedi/drivers/ni_atmio16d.c
@@ -1,25 +1,41 @@
 /*
-   comedi/drivers/ni_atmio16d.c
-   Hardware driver for National Instruments AT-MIO16D board
-   Copyright (C) 2000 Chris R. Baugher <baugher@enteract.com>
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 2 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
+ * Comedi driver for National Instruments AT-MIO16D board
+ * Copyright (C) 2000 Chris R. Baugher <baugher@enteract.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
  */
+
 /*
-Driver: ni_atmio16d
-Description: National Instruments AT-MIO-16D
-Author: Chris R. Baugher <baugher@enteract.com>
-Status: unknown
-Devices: [National Instruments] AT-MIO-16 (atmio16), AT-MIO-16D (atmio16d)
-*/
+ * Driver: ni_atmio16d
+ * Description: National Instruments AT-MIO-16D
+ * Author: Chris R. Baugher <baugher@enteract.com>
+ * Status: unknown
+ * Devices: [National Instruments] AT-MIO-16 (atmio16), AT-MIO-16D (atmio16d)
+ *
+ * Configuration options:
+ *   [0] - I/O port
+ *   [1] - MIO irq (0 == no irq; or 3,4,5,6,7,9,10,11,12,14,15)
+ *   [2] - DIO irq (0 == no irq; or 3,4,5,6,7,9)
+ *   [3] - DMA1 channel (0 == no DMA; or 5,6,7)
+ *   [4] - DMA2 channel (0 == no DMA; or 5,6,7)
+ *   [5] - a/d mux (0=differential; 1=single)
+ *   [6] - a/d range (0=bipolar10; 1=bipolar5; 2=unipolar10)
+ *   [7] - dac0 range (0=bipolar; 1=unipolar)
+ *   [8] - dac0 reference (0=internal; 1=external)
+ *   [9] - dac0 coding (0=2's comp; 1=straight binary)
+ *   [10] - dac1 range (same as dac0 options)
+ *   [11] - dac1 reference (same as dac0 options)
+ *   [12] - dac1 coding (same as dac0 options)
+ */
+
 /*
  * I must give credit here to Michal Dobes <dobes@tesnet.cz> who
  * wrote the driver for Advantec's pcl812 boards. I used the interrupt
@@ -295,8 +311,10 @@ static int atmio16d_ai_cmd(struct comedi_device *dev,
 	unsigned int sample_count, tmp, chan, gain;
 	int i;
 
-	/* This is slowly becoming a working command interface. *
-	 * It is still uber-experimental */
+	/*
+	 * This is slowly becoming a working command interface.
+	 * It is still uber-experimental
+	 */
 
 	reset_counters(dev);
 
@@ -322,9 +340,10 @@ static int atmio16d_ai_cmd(struct comedi_device *dev,
 		outw(tmp, dev->iobase + MUX_GAIN_REG);
 	}
 
-	/* Now program the sample interval timer */
-	/* Figure out which clock to use then get an
-	 * appropriate timer value */
+	/*
+	 * Now program the sample interval timer.
+	 * Figure out which clock to use then get an appropriate timer value.
+	 */
 	if (cmd->convert_arg < 65536000) {
 		base_clock = CLOCK_1_MHZ;
 		timer = cmd->convert_arg / 1000;
@@ -386,9 +405,10 @@ static int atmio16d_ai_cmd(struct comedi_device *dev,
 		outw(devpriv->com_reg_1_state, dev->iobase + COM_REG_1);
 	}
 
-	/* Program the scan interval timer ONLY IF SCANNING IS ENABLED */
-	/* Figure out which clock to use then get an
-	 * appropriate timer value */
+	/*
+	 * Program the scan interval timer ONLY IF SCANNING IS ENABLED.
+	 * Figure out which clock to use then get an appropriate timer value.
+	 */
 	if (cmd->chanlist_len > 1) {
 		if (cmd->scan_begin_arg < 65536000) {
 			base_clock = CLOCK_1_MHZ;
@@ -566,38 +586,6 @@ static int atmio16d_dio_insn_config(struct comedi_device *dev,
 	return insn->n;
 }
 
-/*
-   options[0] - I/O port
-   options[1] - MIO irq
-		0 == no irq
-		N == irq N {3,4,5,6,7,9,10,11,12,14,15}
-   options[2] - DIO irq
-		0 == no irq
-		N == irq N {3,4,5,6,7,9}
-   options[3] - DMA1 channel
-		0 == no DMA
-		N == DMA N {5,6,7}
-   options[4] - DMA2 channel
-		0 == no DMA
-		N == DMA N {5,6,7}
-
-   options[5] - a/d mux
-	0=differential, 1=single
-   options[6] - a/d range
-	0=bipolar10, 1=bipolar5, 2=unipolar10
-
-   options[7] - dac0 range
-	0=bipolar, 1=unipolar
-   options[8] - dac0 reference
-	0=internal, 1=external
-   options[9] - dac0 coding
-	0=2's comp, 1=straight binary
-
-   options[10] - dac1 range
-   options[11] - dac1 reference
-   options[12] - dac1 coding
- */
-
 static int atmio16d_attach(struct comedi_device *dev,
 			   struct comedi_devconfig *it)
 {
diff --git a/drivers/staging/comedi/drivers/ni_daq_dio24.c b/drivers/staging/comedi/drivers/ni_daq_dio24.c
index d9de83ab0267..733d3fbafa4d 100644
--- a/drivers/staging/comedi/drivers/ni_daq_dio24.c
+++ b/drivers/staging/comedi/drivers/ni_daq_dio24.c
@@ -1,35 +1,35 @@
 /*
-    comedi/drivers/ni_daq_dio24.c
-    Driver for National Instruments PCMCIA DAQ-Card DIO-24
-    Copyright (C) 2002 Daniel Vecino Castel <dvecino@able.es>
+ * Comedi driver for National Instruments PCMCIA DAQ-Card DIO-24
+ * Copyright (C) 2002 Daniel Vecino Castel <dvecino@able.es>
+ *
+ * PCMCIA crap at end of file is adapted from dummy_cs.c 1.31
+ * 2001/08/24 12:13:13 from the pcmcia package.
+ * The initial developer of the pcmcia dummy_cs.c code is David A. Hinds
+ * <dahinds@users.sourceforge.net>.  Portions created by David A. Hinds
+ * are Copyright (C) 1999 David A. Hinds.  All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
 
-    PCMCIA crap at end of file is adapted from dummy_cs.c 1.31
-    2001/08/24 12:13:13 from the pcmcia package.
-    The initial developer of the pcmcia dummy_cs.c code is David A. Hinds
-    <dahinds@users.sourceforge.net>.  Portions created by David A. Hinds
-    are Copyright (C) 1999 David A. Hinds.  All Rights Reserved.
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-*/
 /*
-Driver: ni_daq_dio24
-Description: National Instruments PCMCIA DAQ-Card DIO-24
-Author: Daniel Vecino Castel <dvecino@able.es>
-Devices: [National Instruments] PCMCIA DAQ-Card DIO-24 (ni_daq_dio24)
-Status: ?
-Updated: Thu, 07 Nov 2002 21:53:06 -0800
-
-This is just a wrapper around the 8255.o driver to properly handle
-the PCMCIA interface.
-*/
+ * Driver: ni_daq_dio24
+ * Description: National Instruments PCMCIA DAQ-Card DIO-24
+ * Author: Daniel Vecino Castel <dvecino@able.es>
+ * Devices: [National Instruments] PCMCIA DAQ-Card DIO-24 (ni_daq_dio24)
+ * Status: ?
+ * Updated: Thu, 07 Nov 2002 21:53:06 -0800
+ *
+ * This is just a wrapper around the 8255.o driver to properly handle
+ * the PCMCIA interface.
+ */
 
 #include <linux/module.h>
 #include "../comedi_pcmcia.h"
diff --git a/drivers/staging/comedi/drivers/ni_mio_common.c b/drivers/staging/comedi/drivers/ni_mio_common.c
index 8dabb19519a5..0f97d7b611d7 100644
--- a/drivers/staging/comedi/drivers/ni_mio_common.c
+++ b/drivers/staging/comedi/drivers/ni_mio_common.c
@@ -2772,7 +2772,15 @@ static int ni_ao_inttrig(struct comedi_device *dev,
 	int i;
 	static const int timeout = 1000;
 
-	if (trig_num != cmd->start_arg)
+	/*
+	 * Require trig_num == cmd->start_arg when cmd->start_src == TRIG_INT.
+	 * For backwards compatibility, also allow trig_num == 0 when
+	 * cmd->start_src != TRIG_INT (i.e. when cmd->start_src == TRIG_EXT);
+	 * in that case, the internal trigger is being used as a pre-trigger
+	 * before the external trigger.
+	 */
+	if (!(trig_num == cmd->start_arg ||
+	      (trig_num == 0 && cmd->start_src != TRIG_INT)))
 		return -EINVAL;
 
 	/*
@@ -5480,7 +5488,7 @@ static int ni_E_init(struct comedi_device *dev,
 		s->maxdata	= (devpriv->is_m_series) ? 0xffffffff
 							 : 0x00ffffff;
 		s->insn_read	= ni_tio_insn_read;
-		s->insn_write	= ni_tio_insn_read;
+		s->insn_write	= ni_tio_insn_write;
 		s->insn_config	= ni_tio_insn_config;
 #ifdef PCIDMA
 		if (dev->irq && devpriv->mite) {
diff --git a/drivers/staging/comedi/drivers/ni_mio_cs.c b/drivers/staging/comedi/drivers/ni_mio_cs.c
index e3d821bf2d6a..21f823179356 100644
--- a/drivers/staging/comedi/drivers/ni_mio_cs.c
+++ b/drivers/staging/comedi/drivers/ni_mio_cs.c
@@ -1,40 +1,39 @@
 /*
-    comedi/drivers/ni_mio_cs.c
-    Hardware driver for NI PCMCIA MIO E series cards
-
-    COMEDI - Linux Control and Measurement Device Interface
-    Copyright (C) 1997-2000 David A. Schleef <ds@schleef.org>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-*/
-/*
-Driver: ni_mio_cs
-Description: National Instruments DAQCard E series
-Author: ds
-Status: works
-Devices: [National Instruments] DAQCard-AI-16XE-50 (ni_mio_cs),
-  DAQCard-AI-16E-4, DAQCard-6062E, DAQCard-6024E, DAQCard-6036E
-Updated: Thu Oct 23 19:43:17 CDT 2003
-
-See the notes in the ni_atmio.o driver.
-*/
-/*
-	The real guts of the driver is in ni_mio_common.c, which is
-	included by all the E series drivers.
-
-	References for specifications:
+ * Comedi driver for NI PCMCIA MIO E series cards
+ *
+ * COMEDI - Linux Control and Measurement Device Interface
+ * Copyright (C) 1997-2000 David A. Schleef <ds@schleef.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
 
-	   341080a.pdf  DAQCard E Series Register Level Programmer Manual
+/*
+ * Driver: ni_mio_cs
+ * Description: National Instruments DAQCard E series
+ * Author: ds
+ * Status: works
+ * Devices: [National Instruments] DAQCard-AI-16XE-50 (ni_mio_cs),
+ *   DAQCard-AI-16E-4, DAQCard-6062E, DAQCard-6024E, DAQCard-6036E
+ * Updated: Thu Oct 23 19:43:17 CDT 2003
+ *
+ * See the notes in the ni_atmio.o driver.
+ */
 
-*/
+/*
+ * The real guts of the driver is in ni_mio_common.c, which is
+ * included by all the E series drivers.
+ *
+ * References for specifications:
+ *	341080a.pdf  DAQCard E Series Register Level Programmer Manual
+ */
 
 #include <linux/module.h>
 #include <linux/delay.h>
diff --git a/drivers/staging/comedi/drivers/ni_pcidio.c b/drivers/staging/comedi/drivers/ni_pcidio.c
index 35ef1925703f..daeb4ad7a75f 100644
--- a/drivers/staging/comedi/drivers/ni_pcidio.c
+++ b/drivers/staging/comedi/drivers/ni_pcidio.c
@@ -1,50 +1,49 @@
 /*
-    comedi/drivers/ni_pcidio.c
-    driver for National Instruments PCI-DIO-32HS
-
-    COMEDI - Linux Control and Measurement Device Interface
-    Copyright (C) 1999,2002 David A. Schleef <ds@schleef.org>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-*/
+ * Comedi driver for National Instruments PCI-DIO-32HS
+ *
+ * COMEDI - Linux Control and Measurement Device Interface
+ * Copyright (C) 1999,2002 David A. Schleef <ds@schleef.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
 /*
-Driver: ni_pcidio
-Description: National Instruments PCI-DIO32HS, PCI-6533
-Author: ds
-Status: works
-Devices: [National Instruments] PCI-DIO-32HS (ni_pcidio)
-	 [National Instruments] PXI-6533, PCI-6533 (pxi-6533)
-	 [National Instruments] PCI-6534 (pci-6534)
-Updated: Mon, 09 Jan 2012 14:27:23 +0000
-
-The DIO32HS board appears as one subdevice, with 32 channels.
-Each channel is individually I/O configurable.  The channel order
-is 0=A0, 1=A1, 2=A2, ... 8=B0, 16=C0, 24=D0.  The driver only
-supports simple digital I/O; no handshaking is supported.
-
-DMA mostly works for the PCI-DIO32HS, but only in timed input mode.
-
-The PCI-DIO-32HS/PCI-6533 has a configurable external trigger. Setting
-scan_begin_arg to 0 or CR_EDGE triggers on the leading edge. Setting
-scan_begin_arg to CR_INVERT or (CR_EDGE | CR_INVERT) triggers on the
-trailing edge.
-
-This driver could be easily modified to support AT-MIO32HS and
-AT-MIO96.
-
-The PCI-6534 requires a firmware upload after power-up to work, the
-firmware data and instructions for loading it with comedi_config
-it are contained in the
-comedi_nonfree_firmware tarball available from http://www.comedi.org
-*/
+ * Driver: ni_pcidio
+ * Description: National Instruments PCI-DIO32HS, PCI-6533
+ * Author: ds
+ * Status: works
+ * Devices: [National Instruments] PCI-DIO-32HS (ni_pcidio)
+ *   [National Instruments] PXI-6533, PCI-6533 (pxi-6533)
+ *   [National Instruments] PCI-6534 (pci-6534)
+ * Updated: Mon, 09 Jan 2012 14:27:23 +0000
+ *
+ * The DIO32HS board appears as one subdevice, with 32 channels. Each
+ * channel is individually I/O configurable. The channel order is 0=A0,
+ * 1=A1, 2=A2, ... 8=B0, 16=C0, 24=D0. The driver only supports simple
+ * digital I/O; no handshaking is supported.
+ *
+ * DMA mostly works for the PCI-DIO32HS, but only in timed input mode.
+ *
+ * The PCI-DIO-32HS/PCI-6533 has a configurable external trigger. Setting
+ * scan_begin_arg to 0 or CR_EDGE triggers on the leading edge. Setting
+ * scan_begin_arg to CR_INVERT or (CR_EDGE | CR_INVERT) triggers on the
+ * trailing edge.
+ *
+ * This driver could be easily modified to support AT-MIO32HS and AT-MIO96.
+ *
+ * The PCI-6534 requires a firmware upload after power-up to work, the
+ * firmware data and instructions for loading it with comedi_config
+ * it are contained in the comedi_nonfree_firmware tarball available from
+ * http://www.comedi.org
+ */
 
 #define USE_DMA
 
@@ -61,36 +60,36 @@ comedi_nonfree_firmware tarball available from http://www.comedi.org
 
 #define Window_Address			4	/* W */
 #define Interrupt_And_Window_Status	4	/* R */
-#define IntStatus1				(1<<0)
-#define IntStatus2				(1<<1)
+#define IntStatus1				BIT(0)
+#define IntStatus2				BIT(1)
 #define WindowAddressStatus_mask		0x7c
 
 #define Master_DMA_And_Interrupt_Control 5	/* W */
 #define InterruptLine(x)			((x)&3)
-#define OpenInt				(1<<2)
+#define OpenInt				BIT(2)
 #define Group_Status			5	/* R */
-#define DataLeft				(1<<0)
-#define Req					(1<<2)
-#define StopTrig				(1<<3)
+#define DataLeft				BIT(0)
+#define Req					BIT(2)
+#define StopTrig				BIT(3)
 
 #define Group_1_Flags			6	/* R */
 #define Group_2_Flags			7	/* R */
-#define TransferReady				(1<<0)
-#define CountExpired				(1<<1)
-#define Waited				(1<<5)
-#define PrimaryTC				(1<<6)
-#define SecondaryTC				(1<<7)
+#define TransferReady				BIT(0)
+#define CountExpired				BIT(1)
+#define Waited					BIT(5)
+#define PrimaryTC				BIT(6)
+#define SecondaryTC				BIT(7)
   /* #define SerialRose */
   /* #define ReqRose */
   /* #define Paused */
 
 #define Group_1_First_Clear		6	/* W */
 #define Group_2_First_Clear		7	/* W */
-#define ClearWaited				(1<<3)
-#define ClearPrimaryTC			(1<<4)
-#define ClearSecondaryTC			(1<<5)
-#define DMAReset				(1<<6)
-#define FIFOReset				(1<<7)
+#define ClearWaited				BIT(3)
+#define ClearPrimaryTC				BIT(4)
+#define ClearSecondaryTC			BIT(5)
+#define DMAReset				BIT(6)
+#define FIFOReset				BIT(7)
 #define ClearAll				0xf8
 
 #define Group_1_FIFO			8	/* W */
@@ -111,27 +110,27 @@ comedi_nonfree_firmware tarball available from http://www.comedi.org
 
 #define Group_1_Second_Clear		46	/* W */
 #define Group_2_Second_Clear		47	/* W */
-#define ClearExpired				(1<<0)
+#define ClearExpired				BIT(0)
 
 #define Port_Pattern(x)			(48+(x))
 
 #define Data_Path			64
-#define FIFOEnableA		(1<<0)
-#define FIFOEnableB		(1<<1)
-#define FIFOEnableC		(1<<2)
-#define FIFOEnableD		(1<<3)
+#define FIFOEnableA		BIT(0)
+#define FIFOEnableB		BIT(1)
+#define FIFOEnableC		BIT(2)
+#define FIFOEnableD		BIT(3)
 #define Funneling(x)		(((x)&3)<<4)
-#define GroupDirection	(1<<7)
+#define GroupDirection		BIT(7)
 
 #define Protocol_Register_1		65
 #define OpMode				Protocol_Register_1
 #define RunMode(x)		((x)&7)
-#define Numbered		(1<<3)
+#define Numbered		BIT(3)
 
 #define Protocol_Register_2		66
 #define ClockReg			Protocol_Register_2
 #define ClockLine(x)		(((x)&3)<<5)
-#define InvertStopTrig	(1<<7)
+#define InvertStopTrig		BIT(7)
 #define DataLatching(x)       (((x)&3)<<5)
 
 #define Protocol_Register_3		67
@@ -152,17 +151,17 @@ comedi_nonfree_firmware tarball available from http://www.comedi.org
 
 #define Protocol_Register_6		73
 #define LinePolarities			Protocol_Register_6
-#define InvertAck		(1<<0)
-#define InvertReq		(1<<1)
-#define InvertClock		(1<<2)
-#define InvertSerial		(1<<3)
-#define OpenAck		(1<<4)
-#define OpenClock		(1<<5)
+#define InvertAck		BIT(0)
+#define InvertReq		BIT(1)
+#define InvertClock		BIT(2)
+#define InvertSerial		BIT(3)
+#define OpenAck		BIT(4)
+#define OpenClock		BIT(5)
 
 #define Protocol_Register_7		74
 #define AckSer				Protocol_Register_7
 #define AckLine(x)		(((x)&3)<<2)
-#define ExchangePins		(1<<7)
+#define ExchangePins		BIT(7)
 
 #define Interrupt_Control		75
   /* bits same as flags */
@@ -183,20 +182,20 @@ static inline unsigned int secondary_DMAChannel_bits(unsigned int channel)
 #define Transfer_Size_Control		77
 #define TransferWidth(x)	((x)&3)
 #define TransferLength(x)	(((x)&3)<<3)
-#define RequireRLevel		(1<<5)
+#define RequireRLevel		BIT(5)
 
 #define Protocol_Register_15		79
 #define DAQOptions			Protocol_Register_15
 #define StartSource(x)			((x)&0x3)
-#define InvertStart				(1<<2)
+#define InvertStart				BIT(2)
 #define StopSource(x)				(((x)&0x3)<<3)
-#define ReqStart				(1<<6)
-#define PreStart				(1<<7)
+#define ReqStart				BIT(6)
+#define PreStart				BIT(7)
 
 #define Pattern_Detection		81
-#define DetectionMethod			(1<<0)
-#define InvertMatch				(1<<1)
-#define IE_Pattern_Detection			(1<<2)
+#define DetectionMethod			BIT(0)
+#define InvertMatch				BIT(1)
+#define IE_Pattern_Detection			BIT(2)
 
 #define Protocol_Register_9		82
 #define ReqDelay			Protocol_Register_9
@@ -649,8 +648,10 @@ static int ni_pcidio_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 		writeb(1, dev->mmio + AckDelay);
 		writeb(0x0b, dev->mmio + AckNotDelay);
 		writeb(0x01, dev->mmio + Data1Delay);
-		/* manual, page 4-5: ClockSpeed comment is incorrectly listed
-		 * on DAQOptions */
+		/*
+		 * manual, page 4-5:
+		 * ClockSpeed comment is incorrectly listed on DAQOptions
+		 */
 		writew(0, dev->mmio + ClockSpeed);
 		writeb(0, dev->mmio + DAQOptions);
 	} else {
diff --git a/drivers/staging/comedi/drivers/ni_pcimio.c b/drivers/staging/comedi/drivers/ni_pcimio.c
index d8917392b9f9..f13a2f7360b3 100644
--- a/drivers/staging/comedi/drivers/ni_pcimio.c
+++ b/drivers/staging/comedi/drivers/ni_pcimio.c
@@ -1,111 +1,106 @@
 /*
-    comedi/drivers/ni_pcimio.c
-    Hardware driver for NI PCI-MIO E series cards
+ * Comedi driver for NI PCI-MIO E series cards
+ *
+ * COMEDI - Linux Control and Measurement Device Interface
+ * Copyright (C) 1997-8 David A. Schleef <ds@schleef.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
 
-    COMEDI - Linux Control and Measurement Device Interface
-    Copyright (C) 1997-8 David A. Schleef <ds@schleef.org>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-*/
 /*
-Driver: ni_pcimio
-Description: National Instruments PCI-MIO-E series and M series (all boards)
-Author: ds, John Hallen, Frank Mori Hess, Rolf Mueller, Herbert Peremans,
-  Herman Bruyninckx, Terry Barnaby
-Status: works
-Devices: [National Instruments] PCI-MIO-16XE-50 (ni_pcimio),
-  PCI-MIO-16XE-10, PXI-6030E, PCI-MIO-16E-1, PCI-MIO-16E-4, PCI-6014, PCI-6040E,
-  PXI-6040E, PCI-6030E, PCI-6031E, PCI-6032E, PCI-6033E, PCI-6071E, PCI-6023E,
-  PCI-6024E, PCI-6025E, PXI-6025E, PCI-6034E, PCI-6035E, PCI-6052E,
-  PCI-6110, PCI-6111, PCI-6220, PCI-6221, PCI-6224, PXI-6224,
-  PCI-6225, PXI-6225, PCI-6229, PCI-6250,
-  PCI-6251, PXI-6251, PCIe-6251, PXIe-6251,
-  PCI-6254, PCI-6259, PCIe-6259,
-  PCI-6280, PCI-6281, PXI-6281, PCI-6284, PCI-6289,
-  PCI-6711, PXI-6711, PCI-6713, PXI-6713,
-  PXI-6071E, PCI-6070E, PXI-6070E,
-  PXI-6052E, PCI-6036E, PCI-6731, PCI-6733, PXI-6733,
-  PCI-6143, PXI-6143
-Updated: Mon, 09 Jan 2012 14:52:48 +0000
-
-These boards are almost identical to the AT-MIO E series, except that
-they use the PCI bus instead of ISA (i.e., AT).  See the notes for
-the ni_atmio.o driver for additional information about these boards.
-
-Autocalibration is supported on many of the devices, using the
-comedi_calibrate (or comedi_soft_calibrate for m-series) utility.
-M-Series boards do analog input and analog output calibration entirely
-in software. The software calibration corrects
-the analog input for offset, gain and
-nonlinearity.  The analog outputs are corrected for offset and gain.
-See the comedilib documentation on comedi_get_softcal_converter() for
-more information.
-
-By default, the driver uses DMA to transfer analog input data to
-memory.  When DMA is enabled, not all triggering features are
-supported.
-
-Digital I/O may not work on 673x.
-
-Note that the PCI-6143 is a simultaineous sampling device with 8 convertors.
-With this board all of the convertors perform one simultaineous sample during
-a scan interval. The period for a scan is used for the convert time in a
-Comedi cmd. The convert trigger source is normally set to TRIG_NOW by default.
-
-The RTSI trigger bus is supported on these cards on
-subdevice 10. See the comedilib documentation for details.
-
-Information (number of channels, bits, etc.) for some devices may be
-incorrect.  Please check this and submit a bug if there are problems
-for your device.
-
-SCXI is probably broken for m-series boards.
-
-Bugs:
- - When DMA is enabled, COMEDI_EV_CONVERT does
-   not work correctly.
+ * Driver: ni_pcimio
+ * Description: National Instruments PCI-MIO-E series and M series (all boards)
+ * Author: ds, John Hallen, Frank Mori Hess, Rolf Mueller, Herbert Peremans,
+ *   Herman Bruyninckx, Terry Barnaby
+ * Status: works
+ * Devices: [National Instruments] PCI-MIO-16XE-50 (ni_pcimio),
+ *   PCI-MIO-16XE-10, PXI-6030E, PCI-MIO-16E-1, PCI-MIO-16E-4, PCI-6014,
+ *   PCI-6040E, PXI-6040E, PCI-6030E, PCI-6031E, PCI-6032E, PCI-6033E,
+ *   PCI-6071E, PCI-6023E, PCI-6024E, PCI-6025E, PXI-6025E, PCI-6034E,
+ *   PCI-6035E, PCI-6052E,
+ *   PCI-6110, PCI-6111, PCI-6220, PCI-6221, PCI-6224, PXI-6224,
+ *   PCI-6225, PXI-6225, PCI-6229, PCI-6250,
+ *   PCI-6251, PXI-6251, PCIe-6251, PXIe-6251,
+ *   PCI-6254, PCI-6259, PCIe-6259,
+ *   PCI-6280, PCI-6281, PXI-6281, PCI-6284, PCI-6289,
+ *   PCI-6711, PXI-6711, PCI-6713, PXI-6713,
+ *   PXI-6071E, PCI-6070E, PXI-6070E,
+ *   PXI-6052E, PCI-6036E, PCI-6731, PCI-6733, PXI-6733,
+ *   PCI-6143, PXI-6143
+ * Updated: Mon, 09 Jan 2012 14:52:48 +0000
+ *
+ * These boards are almost identical to the AT-MIO E series, except that
+ * they use the PCI bus instead of ISA (i.e., AT). See the notes for the
+ * ni_atmio.o driver for additional information about these boards.
+ *
+ * Autocalibration is supported on many of the devices, using the
+ * comedi_calibrate (or comedi_soft_calibrate for m-series) utility.
+ * M-Series boards do analog input and analog output calibration entirely
+ * in software. The software calibration corrects the analog input for
+ * offset, gain and nonlinearity. The analog outputs are corrected for
+ * offset and gain. See the comedilib documentation on
+ * comedi_get_softcal_converter() for more information.
+ *
+ * By default, the driver uses DMA to transfer analog input data to
+ * memory.  When DMA is enabled, not all triggering features are
+ * supported.
+ *
+ * Digital I/O may not work on 673x.
+ *
+ * Note that the PCI-6143 is a simultaineous sampling device with 8
+ * convertors. With this board all of the convertors perform one
+ * simultaineous sample during a scan interval. The period for a scan
+ * is used for the convert time in a Comedi cmd. The convert trigger
+ * source is normally set to TRIG_NOW by default.
+ *
+ * The RTSI trigger bus is supported on these cards on subdevice 10.
+ * See the comedilib documentation for details.
+ *
+ * Information (number of channels, bits, etc.) for some devices may be
+ * incorrect. Please check this and submit a bug if there are problems
+ * for your device.
+ *
+ * SCXI is probably broken for m-series boards.
+ *
+ * Bugs:
+ * - When DMA is enabled, COMEDI_EV_CONVERT does not work correctly.
+ */
 
-*/
 /*
-	The PCI-MIO E series driver was originally written by
-	Tomasz Motylewski <...>, and ported to comedi by ds.
-
-	References:
-
-	   341079b.pdf  PCI E Series Register-Level Programmer Manual
-	   340934b.pdf  DAQ-STC reference manual
-
-	   322080b.pdf  6711/6713/6715 User Manual
-
-	   320945c.pdf  PCI E Series User Manual
-	   322138a.pdf  PCI-6052E and DAQPad-6052E User Manual
-
-	ISSUES:
-
-	need to deal with external reference for DAC, and other DAC
-	properties in board properties
-
-	deal with at-mio-16de-10 revision D to N changes, etc.
-
-	need to add other CALDAC type
-
-	need to slow down DAC loading.  I don't trust NI's claim that
-	two writes to the PCI bus slows IO enough.  I would prefer to
-	use udelay().  Timing specs: (clock)
-		AD8522		30ns
-		DAC8043		120ns
-		DAC8800		60ns
-		MB88341		?
-
-*/
+ * The PCI-MIO E series driver was originally written by
+ * Tomasz Motylewski <...>, and ported to comedi by ds.
+ *
+ * References:
+ *	341079b.pdf  PCI E Series Register-Level Programmer Manual
+ *	340934b.pdf  DAQ-STC reference manual
+ *
+ *	322080b.pdf  6711/6713/6715 User Manual
+ *
+ *	320945c.pdf  PCI E Series User Manual
+ *	322138a.pdf  PCI-6052E and DAQPad-6052E User Manual
+ *
+ * ISSUES:
+ * - need to deal with external reference for DAC, and other DAC
+ *   properties in board properties
+ * - deal with at-mio-16de-10 revision D to N changes, etc.
+ * - need to add other CALDAC type
+ * - need to slow down DAC loading. I don't trust NI's claim that
+ *   two writes to the PCI bus slows IO enough. I would prefer to
+ *   use udelay().
+ *   Timing specs: (clock)
+ *	AD8522		30ns
+ *	DAC8043		120ns
+ *	DAC8800		60ns
+ *	MB88341		?
+ */
 
 #include <linux/module.h>
 #include <linux/delay.h>
@@ -119,13 +114,14 @@ Bugs:
 
 #define PCIDMA
 
-/* These are not all the possible ao ranges for 628x boards.
- They can do OFFSET +- REFERENCE where OFFSET can be
- 0V, 5V, APFI<0,1>, or AO<0...3> and RANGE can
- be 10V, 5V, 2V, 1V, APFI<0,1>, AO<0...3>.  That's
- 63 different possibilities.  An AO channel
- can not act as it's own OFFSET or REFERENCE.
-*/
+/*
+ * These are not all the possible ao ranges for 628x boards.
+ * They can do OFFSET +- REFERENCE where OFFSET can be
+ * 0V, 5V, APFI<0,1>, or AO<0...3> and RANGE can
+ * be 10V, 5V, 2V, 1V, APFI<0,1>, AO<0...3>.  That's
+ * 63 different possibilities.  An AO channel
+ * can not act as it's own OFFSET or REFERENCE.
+ */
 static const struct comedi_lrange range_ni_M_628x_ao = {
 	8, {
 		BIP_RANGE(10),
diff --git a/drivers/staging/comedi/drivers/ni_usb6501.c b/drivers/staging/comedi/drivers/ni_usb6501.c
index 95b537a8ecdb..5036eebb9162 100644
--- a/drivers/staging/comedi/drivers/ni_usb6501.c
+++ b/drivers/staging/comedi/drivers/ni_usb6501.c
@@ -465,12 +465,12 @@ static int ni6501_alloc_usb_buffers(struct comedi_device *dev)
 	struct ni6501_private *devpriv = dev->private;
 	size_t size;
 
-	size = le16_to_cpu(devpriv->ep_rx->wMaxPacketSize);
+	size = usb_endpoint_maxp(devpriv->ep_rx);
 	devpriv->usb_rx_buf = kzalloc(size, GFP_KERNEL);
 	if (!devpriv->usb_rx_buf)
 		return -ENOMEM;
 
-	size = le16_to_cpu(devpriv->ep_tx->wMaxPacketSize);
+	size = usb_endpoint_maxp(devpriv->ep_tx);
 	devpriv->usb_tx_buf = kzalloc(size, GFP_KERNEL);
 	if (!devpriv->usb_tx_buf) {
 		kfree(devpriv->usb_rx_buf);
diff --git a/drivers/staging/comedi/drivers/plx9080.h b/drivers/staging/comedi/drivers/plx9080.h
index 0e20cc5c9a69..e23e63a097b5 100644
--- a/drivers/staging/comedi/drivers/plx9080.h
+++ b/drivers/staging/comedi/drivers/plx9080.h
@@ -60,9 +60,9 @@ struct plx_dma_desc {
 #define PLX_REG_LAS1RR		0x00f0
 
 #define PLX_LASRR_IO		BIT(0)		/* Map to: 1=I/O, 0=Mem */
-#define PLX_LASRR_ANY32		(BIT(1) * 0)	/* Locate anywhere in 32 bit */
-#define PLX_LASRR_LT1MB		(BIT(1) * 1)	/* Locate in 1st meg */
-#define PLX_LASRR_ANY64		(BIT(1) * 2)	/* Locate anywhere in 64 bit */
+#define PLX_LASRR_MLOC_ANY32	(BIT(1) * 0)	/* Locate anywhere in 32 bit */
+#define PLX_LASRR_MLOC_LT1MB	(BIT(1) * 1)	/* Locate in 1st meg */
+#define PLX_LASRR_MLOC_ANY64	(BIT(1) * 2)	/* Locate anywhere in 64 bit */
 #define PLX_LASRR_MLOC_MASK	GENMASK(2, 1)	/* Memory location bits */
 #define PLX_LASRR_PREFETCH	BIT(3)		/* Memory is prefetchable */
 /* bits that specify range for memory space decode bits */
@@ -89,11 +89,11 @@ struct plx_dma_desc {
 /* Local Bus Latency Timer */
 #define PLX_MARBR_LT(x)		(BIT(0) * ((x) & 0xff))
 #define PLX_MARBR_LT_MASK	GENMASK(7, 0)
-#define PLX_MARBR_LT_SHIFT	0
+#define PLX_MARBR_TO_LT(r)	((r) & PLX_MARBR_LT_MASK)
 /* Local Bus Pause Timer */
 #define PLX_MARBR_PT(x)		(BIT(8) * ((x) & 0xff))
 #define PLX_MARBR_PT_MASK	GENMASK(15, 8)
-#define PLX_MARBR_PT_SHIFT	8
+#define PLX_MARBR_TO_PT(r)	(((r) & PLX_MARBR_PT_MASK) >> 8)
 /* Local Bus Latency Timer Enable */
 #define PLX_MARBR_LTEN		BIT(16)
 /* Local Bus Pause Timer Enable */
@@ -166,16 +166,15 @@ struct plx_dma_desc {
 #define PLX_REG_LBRD1		0x00f8
 
 /* Memory Space Local Bus Width */
-#define PLX_LBRD_MSWIDTH8	(BIT(0) * 0)	/* 8 bits wide */
-#define PLX_LBRD_MSWIDTH16	(BIT(0) * 1)	/* 16 bits wide */
-#define PLX_LBRD_MSWIDTH32	(BIT(0) * 2)	/* 32 bits wide */
-#define PLX_LBRD_MSWIDTH32A	(BIT(0) * 3)	/* 32 bits wide */
+#define PLX_LBRD_MSWIDTH_8	(BIT(0) * 0)	/* 8 bits wide */
+#define PLX_LBRD_MSWIDTH_16	(BIT(0) * 1)	/* 16 bits wide */
+#define PLX_LBRD_MSWIDTH_32	(BIT(0) * 2)	/* 32 bits wide */
+#define PLX_LBRD_MSWIDTH_32A	(BIT(0) * 3)	/* 32 bits wide */
 #define PLX_LBRD_MSWIDTH_MASK	GENMASK(1, 0)
-#define PLX_LBRD_MSWIDTH_SHIFT	0
 /* Memory Space Internal Wait States */
 #define PLX_LBRD_MSIWS(x)	(BIT(2) * ((x) & 0xf))
 #define PLX_LBRD_MSIWS_MASK	GENMASK(5, 2)
-#define PLX_LBRD_MSIWS_SHIFT	2
+#define PLX_LBRD_TO_MSIWS(r)	(((r) & PLS_LBRD_MSIWS_MASK) >> 2)
 /* Memory Space Ready Input Enable */
 #define PLX_LBRD_MSREADYIEN	BIT(6)
 /* Memory Space BTERM# Input Enable */
@@ -193,18 +192,17 @@ struct plx_dma_desc {
 /* Prefetch Counter */
 #define PLX_LBRD_PFCOUNT(x)	(BIT(11) * ((x) & 0xf))
 #define PLX_LBRD_PFCOUNT_MASK	GENMASK(14, 11)
-#define PLX_LBRD_PFCOUNT_SHIFT	11
+#define PLX_LBRD_TO_PFCOUNT(r)	(((r) & PLX_LBRD_PFCOUNT_MASK) >> 11)
 /* Expansion ROM Space Local Bus Width (LBRD0 only) */
-#define PLX_LBRD0_EROMWIDTH8	(BIT(16) * 0)	/* 8 bits wide */
-#define PLX_LBRD0_EROMWIDTH16	(BIT(16) * 1)	/* 16 bits wide */
-#define PLX_LBRD0_EROMWIDTH32	(BIT(16) * 2)	/* 32 bits wide */
-#define PLX_LBRD0_EROMWIDTH32A	(BIT(16) * 3)	/* 32 bits wide */
+#define PLX_LBRD0_EROMWIDTH_8	(BIT(16) * 0)	/* 8 bits wide */
+#define PLX_LBRD0_EROMWIDTH_16	(BIT(16) * 1)	/* 16 bits wide */
+#define PLX_LBRD0_EROMWIDTH_32	(BIT(16) * 2)	/* 32 bits wide */
+#define PLX_LBRD0_EROMWIDTH_32A	(BIT(16) * 3)	/* 32 bits wide */
 #define PLX_LBRD0_EROMWIDTH_MASK	GENMASK(17, 16)
-#define PLX_LBRD0_EROMWIDTH_SHIFT	16
 /* Expansion ROM Space Internal Wait States (LBRD0 only) */
 #define PLX_LBRD0_EROMIWS(x)	(BIT(18) * ((x) & 0xf))
 #define PLX_LBRD0_EROMIWS_MASK	GENMASK(21, 18)
-#define PLX_LBRD0_EROMIWS_SHIFT	18
+#define PLX_LBRD0_TO_EROMIWS(r)	(((r) & PLX_LBRD0_EROMIWS_MASK) >> 18)
 /* Expansion ROM Space Ready Input Enable (LBDR0 only) */
 #define PLX_LBRD0_EROMREADYIEN	BIT(22)
 /* Expansion ROM Space BTERM# Input Enable (LBRD0 only) */
@@ -220,7 +218,7 @@ struct plx_dma_desc {
 /* PCI Target Retry Delay Clocks / 8 (LBRD0 only) */
 #define PLX_LBRD0_TRDELAY(x)	(BIT(28) * ((x) & 0xF))
 #define PLX_LBRD0_TRDELAY_MASK	GENMASK(31, 28)
-#define PLX_LBRD0_TRDELAY_SHIFT	28
+#define PLX_LBRD0_TO_TRDELAY(r)	(((r) & PLX_LBRD0_TRDELAY_MASK) >> 28)
 
 /* Local Range Register for Direct Master to PCI */
 #define PLX_REG_DMRR		0x001c
@@ -241,10 +239,10 @@ struct plx_dma_desc {
 /* LLOCK# Input Enable */
 #define PLX_DMPBAM_LLOCKIEN	BIT(2)
 /* Direct Master Read Prefetch Size Control (bits 12, 3) */
-#define PLX_DMPBAM_RPSIZECONT	((BIT(12) * 0) | (BIT(3) * 0))
-#define PLX_DMPBAM_RPSIZE4	((BIT(12) * 0) | (BIT(3) * 1))
-#define PLX_DMPBAM_RPSIZE8	((BIT(12) * 1) | (BIT(3) * 0))
-#define PLX_DMPBAM_RPSIZE16	((BIT(12) * 1) | (BIT(3) * 1))
+#define PLX_DMPBAM_RPSIZE_CONT	((BIT(12) * 0) | (BIT(3) * 0))
+#define PLX_DMPBAM_RPSIZE_4	((BIT(12) * 0) | (BIT(3) * 1))
+#define PLX_DMPBAM_RPSIZE_8	((BIT(12) * 1) | (BIT(3) * 0))
+#define PLX_DMPBAM_RPSIZE_16	((BIT(12) * 1) | (BIT(3) * 1))
 #define PLX_DMPBAM_RPSIZE_MASK	(BIT(12) | BIT(3))
 /* Direct Master PCI Read Mode - deassert IRDY when FIFO full */
 #define PLX_DMPBAM_RMIRDY	BIT(4)
@@ -261,10 +259,10 @@ struct plx_dma_desc {
 /* I/O Remap Select */
 #define PLX_DMPBAM_IOREMAPSEL	BIT(13)
 /* Direct Master Write Delay */
-#define PLX_DMPBAM_WDELAYNONE	(BIT(14) * 0)
-#define PLX_DMPBAM_WDELAY4	(BIT(14) * 1)
-#define PLX_DMPBAM_WDELAY8	(BIT(14) * 2)
-#define PLX_DMPBAM_WDELAY16	(BIT(14) * 3)
+#define PLX_DMPBAM_WDELAY_NONE	(BIT(14) * 0)
+#define PLX_DMPBAM_WDELAY_4	(BIT(14) * 1)
+#define PLX_DMPBAM_WDELAY_8	(BIT(14) * 2)
+#define PLX_DMPBAM_WDELAY_16	(BIT(14) * 3)
 #define PLX_DMPBAM_WDELAY_MASK	GENMASK(15, 14)
 /* Remap of Local-to-PCI Space Into PCI Address Space */
 #define PLX_DMPBAM_REMAP_MASK	GENMASK(31, 16)
@@ -279,19 +277,19 @@ struct plx_dma_desc {
 /* Register Number */
 #define PLX_DMCFGA_REGNUM(x)	(BIT(2) * ((x) & 0x3f))
 #define PLX_DMCFGA_REGNUM_MASK	GENMASK(7, 2)
-#define PLX_DMCFGA_REGNUM_SHIFT	2
+#define PLX_DMCFGA_TO_REGNUM(r)	(((r) & PLX_DMCFGA_REGNUM_MASK) >> 2)
 /* Function Number */
 #define PLX_DMCFGA_FUNCNUM(x)	(BIT(8) * ((x) & 0x7))
 #define PLX_DMCFGA_FUNCNUM_MASK	GENMASK(10, 8)
-#define PLX_DMCFGA_FUNCNUM_SHIFT 8
+#define PLX_DMCFGA_TO_FUNCNUM(r) (((r) & PLX_DMCFGA_FUNCNUM_MASK) >> 8)
 /* Device Number */
 #define PLX_DMCFGA_DEVNUM(x)	(BIT(11) * ((x) & 0x1f))
 #define PLX_DMCFGA_DEVNUM_MASK	GENMASK(15, 11)
-#define PLX_DMCFGA_DEVNUM_SHIFT	11
+#define PLX_DMCFGA_TO_DEVNUM(r)	(((r) & PLX_DMCFGA_DEVNUM_MASK) >> 11)
 /* Bus Number */
 #define PLX_DMCFGA_BUSNUM(x)	(BIT(16) * ((x) & 0xff))
 #define PLX_DMCFGA_BUSNUM_MASK	GENMASK(23, 16)
-#define PLX_DMCFGA_BUSNUM_SHIFT	16
+#define PLX_DMCFGA_TO_BUSNUM(r)	(((r) & PLX_DMCFGA_BUSNUM_MASK) >> 16)
 /* Configuration Enable */
 #define PLX_DMCFGA_CONFIGEN	BIT(31)
 
@@ -402,22 +400,22 @@ struct plx_dma_desc {
 /* PCI Read Command Code For DMA */
 #define PLX_CNTRL_CCRDMA(x)	(BIT(0) * ((x) & 0xf))
 #define PLX_CNTRL_CCRDMA_MASK	GENMASK(3, 0)
-#define PLX_CNTRL_CCRDMA_SHIFT	0
+#define PLX_CNTRL_TO_CCRDMA(r)	((r) & PLX_CNTRL_CCRDMA_MASK)
 #define PLX_CNTRL_CCRDMA_NORMAL	PLX_CNTRL_CCRDMA(14)	/* value after reset */
 /* PCI Write Command Code For DMA 0 */
 #define PLX_CNTRL_CCWDMA(x)	(BIT(4) * ((x) & 0xf))
 #define PLX_CNTRL_CCWDMA_MASK	GENMASK(7, 4)
-#define PLX_CNTRL_CCWDMA_SHIFT	4
+#define PLX_CNTRL_TO_CCWDMA(r)	(((r) & PLX_CNTRL_CCWDMA_MASK) >> 4)
 #define PLX_CNTRL_CCWDMA_NORMAL	PLX_CNTRL_CCWDMA(7)	/* value after reset */
 /* PCI Memory Read Command Code For Direct Master */
 #define PLX_CNTRL_CCRDM(x)	(BIT(8) * ((x) & 0xf))
 #define PLX_CNTRL_CCRDM_MASK	GENMASK(11, 8)
-#define PLX_CNTRL_CCRDM_SHIFT	8
+#define PLX_CNTRL_TO_CCRDM(r)	(((r) & PLX_CNTRL_CCRDM_MASK) >> 8)
 #define PLX_CNTRL_CCRDM_NORMAL	PLX_CNTRL_CCRDM(6)	/* value after reset */
 /* PCI Memory Write Command Code For Direct Master */
 #define PLX_CNTRL_CCWDM(x)	(BIT(12) * ((x) & 0xf))
 #define PLX_CNTRL_CCWDM_MASK	GENMASK(15, 12)
-#define PLX_CNTRL_CCWDM_SHIFT	12
+#define PLX_CNTRL_TO_CCWDM(r)	(((r) & PLX_CNTRL_CCWDM_MASK) >> 12)
 #define PLX_CNTRL_CCWDM_NORMAL	PLX_CNTRL_CCWDM(7)	/* value after reset */
 /* General Purpose Output (USERO) */
 #define PLX_CNTRL_USERO		BIT(16)
@@ -464,16 +462,15 @@ struct plx_dma_desc {
 #define PLX_REG_DMAMODE1	0x0094
 
 /* Local Bus Width */
-#define PLX_DMAMODE_WIDTH8	(BIT(0) * 0)	/* 8 bits wide */
-#define PLX_DMAMODE_WIDTH16	(BIT(0) * 1)	/* 16 bits wide */
-#define PLX_DMAMODE_WIDTH32	(BIT(0) * 2)	/* 32 bits wide */
-#define PLX_DMAMODE_WIDTH32A	(BIT(0) * 3)	/* 32 bits wide */
+#define PLX_DMAMODE_WIDTH_8	(BIT(0) * 0)	/* 8 bits wide */
+#define PLX_DMAMODE_WIDTH_16	(BIT(0) * 1)	/* 16 bits wide */
+#define PLX_DMAMODE_WIDTH_32	(BIT(0) * 2)	/* 32 bits wide */
+#define PLX_DMAMODE_WIDTH_32A	(BIT(0) * 3)	/* 32 bits wide */
 #define PLX_DMAMODE_WIDTH_MASK	GENMASK(1, 0)
-#define PLX_DMAMODE_WIDTH_SHIFT	0
 /* Internal Wait States */
 #define PLX_DMAMODE_IWS(x)	(BIT(2) * ((x) & 0xf))
 #define PLX_DMAMODE_IWS_MASK	GENMASK(5, 2)
-#define PLX_DMAMODE_SHIFT	2
+#define PLX_DMAMODE_TO_IWS(r)	(((r) & PLX_DMAMODE_IWS_MASK) >> 2)
 /* Ready Input Enable */
 #define PLX_DMAMODE_READYIEN	BIT(6)
 /* BTERM# Input Enable */
@@ -560,35 +557,35 @@ struct plx_dma_desc {
 /* DMA Channel 0 PCI-to-Local Almost Full (divided by 2, minus 1) */
 #define PLX_DMATHR_C0PLAF(x)	(BIT(0) * ((x) & 0xf))
 #define PLX_DMATHR_C0PLAF_MASK	GENMASK(3, 0)
-#define PLX_DMATHR_C0PLAF_SHIFT	0
+#define PLX_DMATHR_TO_C0PLAF(r)	((r) & PLX_DMATHR_C0PLAF_MASK)
 /* DMA Channel 0 Local-to-PCI Almost Empty (divided by 2, minus 1) */
 #define PLX_DMATHR_C0LPAE(x)	(BIT(4) * ((x) & 0xf))
 #define PLX_DMATHR_C0LPAE_MASK	GENMASK(7, 4)
-#define PLX_DMATHR_C0LPAE_SHIFT	4
+#define PLX_DMATHR_TO_C0LPAE(r)	(((r) & PLX_DMATHR_C0LPAE_MASK) >> 4)
 /* DMA Channel 0 Local-to-PCI Almost Full (divided by 2, minus 1) */
 #define PLX_DMATHR_C0LPAF(x)	(BIT(8) * ((x) & 0xf))
 #define PLX_DMATHR_C0LPAF_MASK	GENMASK(11, 8)
-#define PLX_DMATHR_C0LPAF_SHIFT	8
+#define PLX_DMATHR_TO_C0LPAF(r)	(((r) & PLX_DMATHR_C0LPAF_MASK) >> 8)
 /* DMA Channel 0 PCI-to-Local Almost Empty (divided by 2, minus 1) */
 #define PLX_DMATHR_C0PLAE(x)	(BIT(12) * ((x) & 0xf))
 #define PLX_DMATHR_C0PLAE_MASK	GENMASK(15, 12)
-#define PLX_DMATHR_C0PLAE_SHIFT	12
+#define PLX_DMATHR_TO_C0PLAE(r)	(((r) & PLX_DMATHR_C0PLAE_MASK) >> 12)
 /* DMA Channel 1 PCI-to-Local Almost Full (divided by 2, minus 1) */
 #define PLX_DMATHR_C1PLAF(x)	(BIT(16) * ((x) & 0xf))
 #define PLX_DMATHR_C1PLAF_MASK	GENMASK(19, 16)
-#define PLX_DMATHR_C1PLAF_SHIFT	16
+#define PLX_DMATHR_TO_C1PLAF(r)	(((r) & PLX_DMATHR_C1PLAF_MASK) >> 16)
 /* DMA Channel 1 Local-to-PCI Almost Empty (divided by 2, minus 1) */
 #define PLX_DMATHR_C1LPAE(x)	(BIT(20) * ((x) & 0xf))
 #define PLX_DMATHR_C1LPAE_MASK	GENMASK(23, 20)
-#define PLX_DMATHR_C1LPAE_SHIFT	20
+#define PLX_DMATHR_TO_C1LPAE(r)	(((r) & PLX_DMATHR_C1LPAE_MASK) >> 20)
 /* DMA Channel 1 Local-to-PCI Almost Full (divided by 2, minus 1) */
 #define PLX_DMATHR_C1LPAF(x)	(BIT(24) * ((x) & 0xf))
 #define PLX_DMATHR_C1LPAF_MASK	GENMASK(27, 24)
-#define PLX_DMATHR_C1LPAF_SHIFT	24
+#define PLX_DMATHR_TO_C1LPAF(r)	(((r) & PLX_DMATHR_C1LPAF_MASK) >> 24)
 /* DMA Channel 1 PCI-to-Local Almost Empty (divided by 2, minus 1) */
 #define PLX_DMATHR_C1PLAE(x)	(BIT(28) * ((x) & 0xf))
 #define PLX_DMATHR_C1PLAE_MASK	GENMASK(31, 28)
-#define PLX_DMATHR_C1PLAE_SHIFT	28
+#define PLX_DMATHR_TO_C1PLAE(r)	(((r) & PLX_DMATHR_C1PLAE_MASK) >> 28)
 
 /*
  * Messaging Queue Registers OPLFIS, OPLFIM, IQP, OQP, MQCR, QBAR, IFHPR,
diff --git a/drivers/staging/comedi/drivers/s626.c b/drivers/staging/comedi/drivers/s626.c
index 4a87b4b52400..c14a02564432 100644
--- a/drivers/staging/comedi/drivers/s626.c
+++ b/drivers/staging/comedi/drivers/s626.c
@@ -697,134 +697,6 @@ static void s626_reset_cap_flags(struct comedi_device *dev,
 	s626_debi_replace(dev, S626_LP_CRB(chan), ~S626_CRBMSK_INTCTRL, set);
 }
 
-#ifdef unused
-/*
- * Return counter setup in a format (COUNTER_SETUP) that is consistent
- * for both A and B counters.
- */
-static uint16_t s626_get_mode_a(struct comedi_device *dev,
-				unsigned int chan)
-{
-	uint16_t cra;
-	uint16_t crb;
-	uint16_t setup;
-	unsigned int cntsrc, clkmult, clkpol, encmode;
-
-	/* Fetch CRA and CRB register images. */
-	cra = s626_debi_read(dev, S626_LP_CRA(chan));
-	crb = s626_debi_read(dev, S626_LP_CRB(chan));
-
-	/*
-	 * Populate the standardized counter setup bit fields.
-	 */
-	setup =
-		/* LoadSrc  = LoadSrcA. */
-		S626_SET_STD_LOADSRC(S626_GET_CRA_LOADSRC_A(cra)) |
-		/* LatchSrc = LatchSrcA. */
-		S626_SET_STD_LATCHSRC(S626_GET_CRB_LATCHSRC(crb)) |
-		/* IntSrc   = IntSrcA. */
-		S626_SET_STD_INTSRC(S626_GET_CRA_INTSRC_A(cra)) |
-		/* IndxSrc  = IndxSrcA. */
-		S626_SET_STD_INDXSRC(S626_GET_CRA_INDXSRC_A(cra)) |
-		/* IndxPol  = IndxPolA. */
-		S626_SET_STD_INDXPOL(S626_GET_CRA_INDXPOL_A(cra)) |
-		/* ClkEnab  = ClkEnabA. */
-		S626_SET_STD_CLKENAB(S626_GET_CRB_CLKENAB_A(crb));
-
-	/* Adjust mode-dependent parameters. */
-	cntsrc = S626_GET_CRA_CNTSRC_A(cra);
-	if (cntsrc & S626_CNTSRC_SYSCLK) {
-		/* Timer mode (CntSrcA<1> == 1): */
-		encmode = S626_ENCMODE_TIMER;
-		/* Set ClkPol to indicate count direction (CntSrcA<0>). */
-		clkpol = cntsrc & 1;
-		/* ClkMult must be 1x in Timer mode. */
-		clkmult = S626_CLKMULT_1X;
-	} else {
-		/* Counter mode (CntSrcA<1> == 0): */
-		encmode = S626_ENCMODE_COUNTER;
-		/* Pass through ClkPol. */
-		clkpol = S626_GET_CRA_CLKPOL_A(cra);
-		/* Force ClkMult to 1x if not legal, else pass through. */
-		clkmult = S626_GET_CRA_CLKMULT_A(cra);
-		if (clkmult == S626_CLKMULT_SPECIAL)
-			clkmult = S626_CLKMULT_1X;
-	}
-	setup |= S626_SET_STD_ENCMODE(encmode) | S626_SET_STD_CLKMULT(clkmult) |
-		 S626_SET_STD_CLKPOL(clkpol);
-
-	/* Return adjusted counter setup. */
-	return setup;
-}
-
-static uint16_t s626_get_mode_b(struct comedi_device *dev,
-				unsigned int chan)
-{
-	uint16_t cra;
-	uint16_t crb;
-	uint16_t setup;
-	unsigned int cntsrc, clkmult, clkpol, encmode;
-
-	/* Fetch CRA and CRB register images. */
-	cra = s626_debi_read(dev, S626_LP_CRA(chan));
-	crb = s626_debi_read(dev, S626_LP_CRB(chan));
-
-	/*
-	 * Populate the standardized counter setup bit fields.
-	 */
-	setup =
-		/* IntSrc   = IntSrcB. */
-		S626_SET_STD_INTSRC(S626_GET_CRB_INTSRC_B(crb)) |
-		/* LatchSrc = LatchSrcB. */
-		S626_SET_STD_LATCHSRC(S626_GET_CRB_LATCHSRC(crb)) |
-		/* LoadSrc  = LoadSrcB. */
-		S626_SET_STD_LOADSRC(S626_GET_CRB_LOADSRC_B(crb)) |
-		/* IndxPol  = IndxPolB. */
-		S626_SET_STD_INDXPOL(S626_GET_CRB_INDXPOL_B(crb)) |
-		/* ClkEnab  = ClkEnabB. */
-		S626_SET_STD_CLKENAB(S626_GET_CRB_CLKENAB_B(crb)) |
-		/* IndxSrc  = IndxSrcB. */
-		S626_SET_STD_INDXSRC(S626_GET_CRA_INDXSRC_B(cra));
-
-	/* Adjust mode-dependent parameters. */
-	cntsrc = S626_GET_CRA_CNTSRC_B(cra);
-	clkmult = S626_GET_CRB_CLKMULT_B(crb);
-	if (clkmult == S626_CLKMULT_SPECIAL) {
-		/* Extender mode (ClkMultB == S626_CLKMULT_SPECIAL): */
-		encmode = S626_ENCMODE_EXTENDER;
-		/* Indicate multiplier is 1x. */
-		clkmult = S626_CLKMULT_1X;
-		/* Set ClkPol equal to Timer count direction (CntSrcB<0>). */
-		clkpol = cntsrc & 1;
-	} else if (cntsrc & S626_CNTSRC_SYSCLK) {
-		/* Timer mode (CntSrcB<1> == 1): */
-		encmode = S626_ENCMODE_TIMER;
-		/* Indicate multiplier is 1x. */
-		clkmult = S626_CLKMULT_1X;
-		/* Set ClkPol equal to Timer count direction (CntSrcB<0>). */
-		clkpol = cntsrc & 1;
-	} else {
-		/* If Counter mode (CntSrcB<1> == 0): */
-		encmode = S626_ENCMODE_COUNTER;
-		/* Clock multiplier is passed through. */
-		/* Clock polarity is passed through. */
-		clkpol = S626_GET_CRB_CLKPOL_B(crb);
-	}
-	setup |= S626_SET_STD_ENCMODE(encmode) | S626_SET_STD_CLKMULT(clkmult) |
-		 S626_SET_STD_CLKPOL(clkpol);
-
-	/* Return adjusted counter setup. */
-	return setup;
-}
-
-static uint16_t s626_get_mode(struct comedi_device *dev,
-			      unsigned int chan)
-{
-	return (chan < 3) ? s626_get_mode_a(dev, chan)
-			  : s626_get_mode_b(dev, chan);
-}
-#endif
-
 /*
  * Set the operating mode for the specified counter.  The setup
  * parameter is treated as a COUNTER_SETUP data type.  The following
@@ -1023,25 +895,6 @@ static void s626_set_enable(struct comedi_device *dev,
 	s626_debi_replace(dev, S626_LP_CRB(chan), ~mask, set);
 }
 
-#ifdef unused
-static uint16_t s626_get_enable(struct comedi_device *dev,
-				unsigned int chan)
-{
-	uint16_t crb = s626_debi_read(dev, S626_LP_CRB(chan));
-
-	return (chan < 3) ? S626_GET_CRB_CLKENAB_A(crb)
-			  : S626_GET_CRB_CLKENAB_B(crb);
-}
-#endif
-
-#ifdef unused
-static uint16_t s626_get_latch_source(struct comedi_device *dev,
-				      unsigned int chan)
-{
-	return S626_GET_CRB_LATCHSRC(s626_debi_read(dev, S626_LP_CRB(chan)));
-}
-#endif
-
 /*
  * Return/set the event that will trigger transfer of the preload
  * register into the counter.  0=ThisCntr_Index, 1=ThisCntr_Overflow,
@@ -1066,19 +919,6 @@ static void s626_set_load_trig(struct comedi_device *dev,
 	s626_debi_replace(dev, reg, ~mask, set);
 }
 
-#ifdef unused
-static uint16_t s626_get_load_trig(struct comedi_device *dev,
-				   unsigned int chan)
-{
-	if (chan < 3)
-		return S626_GET_CRA_LOADSRC_A(s626_debi_read(dev,
-							S626_LP_CRA(chan)));
-	else
-		return S626_GET_CRB_LOADSRC_B(s626_debi_read(dev,
-							S626_LP_CRB(chan)));
-}
-#endif
-
 /*
  * Return/set counter interrupt source and clear any captured
  * index/overflow events.  int_source: 0=Disabled, 1=OverflowOnly,
@@ -1138,93 +978,6 @@ static void s626_set_int_src(struct comedi_device *dev,
 	}
 }
 
-#ifdef unused
-static uint16_t s626_get_int_src(struct comedi_device *dev,
-				 unsigned int chan)
-{
-	if (chan < 3)
-		return S626_GET_CRA_INTSRC_A(s626_debi_read(dev,
-							S626_LP_CRA(chan)));
-	else
-		return S626_GET_CRB_INTSRC_B(s626_debi_read(dev,
-							S626_LP_CRB(chan)));
-}
-#endif
-
-#ifdef unused
-/*
- * Return/set the clock multiplier.
- */
-static void s626_set_clk_mult(struct comedi_device *dev,
-			      unsigned int chan, uint16_t value)
-{
-	uint16_t mode;
-
-	mode = s626_get_mode(dev, chan);
-	mode &= ~S626_STDMSK_CLKMULT;
-	mode |= S626_SET_STD_CLKMULT(value);
-
-	s626_set_mode(dev, chan, mode, false);
-}
-
-/*
- * Return/set the clock polarity.
- */
-static void s626_set_clk_pol(struct comedi_device *dev,
-			     unsigned int chan, uint16_t value)
-{
-	uint16_t mode;
-
-	mode = s626_get_mode(dev, chan);
-	mode &= ~S626_STDMSK_CLKPOL;
-	mode |= S626_SET_STD_CLKPOL(value);
-
-	s626_set_mode(dev, chan, mode, false);
-}
-
-/*
- * Return/set the encoder mode.
- */
-static void s626_set_enc_mode(struct comedi_device *dev,
-			      unsigned int chan, uint16_t value)
-{
-	uint16_t mode;
-
-	mode = s626_get_mode(dev, chan);
-	mode &= ~S626_STDMSK_ENCMODE;
-	mode |= S626_SET_STD_ENCMODE(value);
-
-	s626_set_mode(dev, chan, mode, false);
-}
-
-static uint16_t s626_get_index_pol(struct comedi_device *dev,
-				   unsigned int chan)
-{
-	return S626_GET_STD_INDXPOL(s626_get_mode(dev, chan));
-}
-
-/*
- * Return/set the index source.
- */
-static void s626_set_index_src(struct comedi_device *dev,
-			       unsigned int chan, uint16_t value)
-{
-	uint16_t mode;
-
-	mode = s626_get_mode(dev, chan);
-	mode &= ~S626_STDMSK_INDXSRC;
-	mode |= S626_SET_STD_INDXSRC(value != 0);
-
-	s626_set_mode(dev, chan, mode, false);
-}
-
-static uint16_t s626_get_index_src(struct comedi_device *dev,
-				   unsigned int chan)
-{
-	return S626_GET_STD_INDXSRC(s626_get_mode(dev, chan));
-}
-#endif
-
 /*
  * Generate an index pulse.
  */
@@ -1717,43 +1470,6 @@ static void s626_reset_adc(struct comedi_device *dev, uint8_t *ppl)
 	/* End of RPS program build */
 }
 
-#ifdef unused_code
-static int s626_ai_rinsn(struct comedi_device *dev,
-			 struct comedi_subdevice *s,
-			 struct comedi_insn *insn,
-			 unsigned int *data)
-{
-	struct s626_private *devpriv = dev->private;
-	uint8_t i;
-	int32_t *readaddr;
-
-	/* Trigger ADC scan loop start */
-	s626_mc_enable(dev, S626_MC2_ADC_RPS, S626_P_MC2);
-
-	/* Wait until ADC scan loop is finished (RPS Signal 0 reset) */
-	while (s626_mc_test(dev, S626_MC2_ADC_RPS, S626_P_MC2))
-		;
-
-	/*
-	 * Init ptr to DMA buffer that holds new ADC data.  We skip the
-	 * first uint16_t in the buffer because it contains junk data from
-	 * the final ADC of the previous poll list scan.
-	 */
-	readaddr = (uint32_t *)devpriv->ana_buf.logical_base + 1;
-
-	/*
-	 * Convert ADC data to 16-bit integer values and
-	 * copy to application buffer.
-	 */
-	for (i = 0; i < devpriv->adc_items; i++) {
-		*data = s626_ai_reg_to_uint(*readaddr++);
-		data++;
-	}
-
-	return i;
-}
-#endif
-
 static int s626_ai_eoc(struct comedi_device *dev,
 		       struct comedi_subdevice *s,
 		       struct comedi_insn *insn,
@@ -2500,7 +2216,8 @@ static int s626_initialize(struct comedi_device *dev)
 	for (i = 0; i < 2; i++) {
 		writel(S626_I2C_CLKSEL, dev->mmio + S626_P_I2CSTAT);
 		s626_mc_enable(dev, S626_MC2_UPLD_IIC, S626_P_MC2);
-		ret = comedi_timeout(dev, NULL, NULL, s626_i2c_handshake_eoc, 0);
+		ret = comedi_timeout(dev, NULL,
+				     NULL, s626_i2c_handshake_eoc, 0);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/staging/comedi/drivers/s626.h b/drivers/staging/comedi/drivers/s626.h
index 6a00a64c6f3a..4cef45263267 100644
--- a/drivers/staging/comedi/drivers/s626.h
+++ b/drivers/staging/comedi/drivers/s626.h
@@ -79,7 +79,7 @@
 /* Address offsets, in DWORDS, from base of DMA buffer. */
 #define S626_DAC_WDMABUF_OS	S626_ADC_DMABUF_DWORDS
 
-/*  Interrupt enable bit in ISR and IER. */
+/* Interrupt enable bit in ISR and IER. */
 #define S626_IRQ_GPIO3		0x00000040	/* IRQ enable for GPIO3. */
 #define S626_IRQ_RPS1		0x10000000
 #define S626_ISR_AFOU		0x00000800
@@ -329,7 +329,7 @@
 						 * WS1-WS4 = CS* outputs.
 						 */
 
-#if S626_PLATFORM == S626_INTEL		/*
+#if (S626_PLATFORM == S626_INTEL)	/*
 					 * Base ACON1 config: always run
 					 * A1 based on TSL1.
 					 */
diff --git a/drivers/staging/comedi/drivers/usbduxfast.c b/drivers/staging/comedi/drivers/usbduxfast.c
index 10f94ec34536..608403c7586b 100644
--- a/drivers/staging/comedi/drivers/usbduxfast.c
+++ b/drivers/staging/comedi/drivers/usbduxfast.c
@@ -946,10 +946,8 @@ static int usbduxfast_auto_attach(struct comedi_device *dev,
 	}
 
 	devpriv->urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!devpriv->urb) {
-		dev_err(dev->class_dev, "Could not alloc. urb\n");
+	if (!devpriv->urb)
 		return -ENOMEM;
-	}
 
 	devpriv->inbuf = kmalloc(SIZEINBUF, GFP_KERNEL);
 	if (!devpriv->inbuf)
diff --git a/drivers/staging/comedi/drivers/vmk80xx.c b/drivers/staging/comedi/drivers/vmk80xx.c
index 8c7393ef762d..a004aed0147a 100644
--- a/drivers/staging/comedi/drivers/vmk80xx.c
+++ b/drivers/staging/comedi/drivers/vmk80xx.c
@@ -177,7 +177,7 @@ static void vmk80xx_do_bulk_msg(struct comedi_device *dev)
 	 * The max packet size attributes of the K8061
 	 * input/output endpoints are identical
 	 */
-	size = le16_to_cpu(devpriv->ep_tx->wMaxPacketSize);
+	size = usb_endpoint_maxp(devpriv->ep_tx);
 
 	usb_bulk_msg(usb, tx_pipe, devpriv->usb_tx_buf,
 		     size, NULL, devpriv->ep_tx->bInterval);
@@ -199,7 +199,7 @@ static int vmk80xx_read_packet(struct comedi_device *dev)
 	ep = devpriv->ep_rx;
 	pipe = usb_rcvintpipe(usb, ep->bEndpointAddress);
 	return usb_interrupt_msg(usb, pipe, devpriv->usb_rx_buf,
-				 le16_to_cpu(ep->wMaxPacketSize), NULL,
+				 usb_endpoint_maxp(ep), NULL,
 				 HZ * 10);
 }
 
@@ -220,7 +220,7 @@ static int vmk80xx_write_packet(struct comedi_device *dev, int cmd)
 	ep = devpriv->ep_tx;
 	pipe = usb_sndintpipe(usb, ep->bEndpointAddress);
 	return usb_interrupt_msg(usb, pipe, devpriv->usb_tx_buf,
-				 le16_to_cpu(ep->wMaxPacketSize), NULL,
+				 usb_endpoint_maxp(ep), NULL,
 				 HZ * 10);
 }
 
@@ -230,7 +230,7 @@ static int vmk80xx_reset_device(struct comedi_device *dev)
 	size_t size;
 	int retval;
 
-	size = le16_to_cpu(devpriv->ep_tx->wMaxPacketSize);
+	size = usb_endpoint_maxp(devpriv->ep_tx);
 	memset(devpriv->usb_tx_buf, 0, size);
 	retval = vmk80xx_write_packet(dev, VMK8055_CMD_RST);
 	if (retval)
@@ -684,12 +684,12 @@ static int vmk80xx_alloc_usb_buffers(struct comedi_device *dev)
 	struct vmk80xx_private *devpriv = dev->private;
 	size_t size;
 
-	size = le16_to_cpu(devpriv->ep_rx->wMaxPacketSize);
+	size = usb_endpoint_maxp(devpriv->ep_rx);
 	devpriv->usb_rx_buf = kzalloc(size, GFP_KERNEL);
 	if (!devpriv->usb_rx_buf)
 		return -ENOMEM;
 
-	size = le16_to_cpu(devpriv->ep_tx->wMaxPacketSize);
+	size = usb_endpoint_maxp(devpriv->ep_tx);
 	devpriv->usb_tx_buf = kzalloc(size, GFP_KERNEL);
 	if (!devpriv->usb_tx_buf) {
 		kfree(devpriv->usb_rx_buf);
diff --git a/drivers/staging/dgnc/dgnc_cls.c b/drivers/staging/dgnc/dgnc_cls.c
index 46c050cc7dbe..aedca66cbe41 100644
--- a/drivers/staging/dgnc/dgnc_cls.c
+++ b/drivers/staging/dgnc/dgnc_cls.c
@@ -26,56 +26,6 @@
 #include "dgnc_cls.h"
 #include "dgnc_tty.h"
 
-static inline void cls_parse_isr(struct dgnc_board *brd, uint port);
-static inline void cls_clear_break(struct channel_t *ch, int force);
-static inline void cls_set_cts_flow_control(struct channel_t *ch);
-static inline void cls_set_rts_flow_control(struct channel_t *ch);
-static inline void cls_set_ixon_flow_control(struct channel_t *ch);
-static inline void cls_set_ixoff_flow_control(struct channel_t *ch);
-static inline void cls_set_no_output_flow_control(struct channel_t *ch);
-static inline void cls_set_no_input_flow_control(struct channel_t *ch);
-static void cls_parse_modem(struct channel_t *ch, unsigned char signals);
-static void cls_tasklet(unsigned long data);
-static void cls_vpd(struct dgnc_board *brd);
-static void cls_uart_init(struct channel_t *ch);
-static void cls_uart_off(struct channel_t *ch);
-static int cls_drain(struct tty_struct *tty, uint seconds);
-static void cls_param(struct tty_struct *tty);
-static void cls_assert_modem_signals(struct channel_t *ch);
-static void cls_flush_uart_write(struct channel_t *ch);
-static void cls_flush_uart_read(struct channel_t *ch);
-static void cls_disable_receiver(struct channel_t *ch);
-static void cls_enable_receiver(struct channel_t *ch);
-static void cls_send_break(struct channel_t *ch, int msecs);
-static void cls_send_start_character(struct channel_t *ch);
-static void cls_send_stop_character(struct channel_t *ch);
-static void cls_copy_data_from_uart_to_queue(struct channel_t *ch);
-static void cls_copy_data_from_queue_to_uart(struct channel_t *ch);
-static uint cls_get_uart_bytes_left(struct channel_t *ch);
-static void cls_send_immediate_char(struct channel_t *ch, unsigned char);
-static irqreturn_t cls_intr(int irq, void *voidbrd);
-
-struct board_ops dgnc_cls_ops = {
-	.tasklet =			cls_tasklet,
-	.intr =				cls_intr,
-	.uart_init =			cls_uart_init,
-	.uart_off =			cls_uart_off,
-	.drain =			cls_drain,
-	.param =			cls_param,
-	.vpd =				cls_vpd,
-	.assert_modem_signals =		cls_assert_modem_signals,
-	.flush_uart_write =		cls_flush_uart_write,
-	.flush_uart_read =		cls_flush_uart_read,
-	.disable_receiver =		cls_disable_receiver,
-	.enable_receiver =		cls_enable_receiver,
-	.send_break =			cls_send_break,
-	.send_start_character =		cls_send_start_character,
-	.send_stop_character =		cls_send_stop_character,
-	.copy_data_from_queue_to_uart = cls_copy_data_from_queue_to_uart,
-	.get_uart_bytes_left =		cls_get_uart_bytes_left,
-	.send_immediate_char =		cls_send_immediate_char
-};
-
 static inline void cls_set_cts_flow_control(struct channel_t *ch)
 {
 	unsigned char lcrb = readb(&ch->ch_cls_uart->lcr);
@@ -357,6 +307,253 @@ static inline void cls_clear_break(struct channel_t *ch, int force)
 	spin_unlock_irqrestore(&ch->ch_lock, flags);
 }
 
+static void cls_copy_data_from_uart_to_queue(struct channel_t *ch)
+{
+	int qleft = 0;
+	unsigned char linestatus = 0;
+	unsigned char error_mask = 0;
+	ushort head;
+	ushort tail;
+	unsigned long flags;
+
+	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
+		return;
+
+	spin_lock_irqsave(&ch->ch_lock, flags);
+
+	/* cache head and tail of queue */
+	head = ch->ch_r_head;
+	tail = ch->ch_r_tail;
+
+	/* Store how much space we have left in the queue */
+	qleft = tail - head - 1;
+	if (qleft < 0)
+		qleft += RQUEUEMASK + 1;
+
+	/*
+	 * Create a mask to determine whether we should
+	 * insert the character (if any) into our queue.
+	 */
+	if (ch->ch_c_iflag & IGNBRK)
+		error_mask |= UART_LSR_BI;
+
+	while (1) {
+		linestatus = readb(&ch->ch_cls_uart->lsr);
+
+		if (!(linestatus & (UART_LSR_DR)))
+			break;
+
+		/*
+		 * Discard character if we are ignoring the error mask.
+		 */
+		if (linestatus & error_mask)  {
+			linestatus = 0;
+			readb(&ch->ch_cls_uart->txrx);
+			continue;
+		}
+
+		/*
+		 * If our queue is full, we have no choice but to drop some
+		 * data. The assumption is that HWFLOW or SWFLOW should have
+		 * stopped things way way before we got to this point.
+		 *
+		 * I decided that I wanted to ditch the oldest data first,
+		 * I hope thats okay with everyone? Yes? Good.
+		 */
+		while (qleft < 1) {
+			tail = (tail + 1) & RQUEUEMASK;
+			ch->ch_r_tail = tail;
+			ch->ch_err_overrun++;
+			qleft++;
+		}
+
+		ch->ch_equeue[head] = linestatus & (UART_LSR_BI | UART_LSR_PE
+								 | UART_LSR_FE);
+		ch->ch_rqueue[head] = readb(&ch->ch_cls_uart->txrx);
+
+		qleft--;
+
+		if (ch->ch_equeue[head] & UART_LSR_PE)
+			ch->ch_err_parity++;
+		if (ch->ch_equeue[head] & UART_LSR_BI)
+			ch->ch_err_break++;
+		if (ch->ch_equeue[head] & UART_LSR_FE)
+			ch->ch_err_frame++;
+
+		/* Add to, and flip head if needed */
+		head = (head + 1) & RQUEUEMASK;
+		ch->ch_rxcount++;
+	}
+
+	/*
+	 * Write new final heads to channel structure.
+	 */
+	ch->ch_r_head = head & RQUEUEMASK;
+	ch->ch_e_head = head & EQUEUEMASK;
+
+	spin_unlock_irqrestore(&ch->ch_lock, flags);
+}
+
+/* Make the UART raise any of the output signals we want up */
+static void cls_assert_modem_signals(struct channel_t *ch)
+{
+	unsigned char out;
+
+	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
+		return;
+
+	out = ch->ch_mostat;
+
+	if (ch->ch_flags & CH_LOOPBACK)
+		out |= UART_MCR_LOOP;
+
+	writeb(out, &ch->ch_cls_uart->mcr);
+
+	/* Give time for the UART to actually drop the signals */
+	udelay(10);
+}
+
+static void cls_copy_data_from_queue_to_uart(struct channel_t *ch)
+{
+	ushort head;
+	ushort tail;
+	int n;
+	int qlen;
+	uint len_written = 0;
+	unsigned long flags;
+
+	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
+		return;
+
+	spin_lock_irqsave(&ch->ch_lock, flags);
+
+	/* No data to write to the UART */
+	if (ch->ch_w_tail == ch->ch_w_head)
+		goto exit_unlock;
+
+	/* If port is "stopped", don't send any data to the UART */
+	if ((ch->ch_flags & CH_FORCED_STOP) ||
+	    (ch->ch_flags & CH_BREAK_SENDING))
+		goto exit_unlock;
+
+	if (!(ch->ch_flags & (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM)))
+		goto exit_unlock;
+
+	n = 32;
+
+	/* cache head and tail of queue */
+	head = ch->ch_w_head & WQUEUEMASK;
+	tail = ch->ch_w_tail & WQUEUEMASK;
+	qlen = (head - tail) & WQUEUEMASK;
+
+	/* Find minimum of the FIFO space, versus queue length */
+	n = min(n, qlen);
+
+	while (n > 0) {
+		/*
+		 * If RTS Toggle mode is on, turn on RTS now if not already set,
+		 * and make sure we get an event when the data transfer has
+		 * completed.
+		 */
+		if (ch->ch_digi.digi_flags & DIGI_RTS_TOGGLE) {
+			if (!(ch->ch_mostat & UART_MCR_RTS)) {
+				ch->ch_mostat |= (UART_MCR_RTS);
+				cls_assert_modem_signals(ch);
+			}
+			ch->ch_tun.un_flags |= (UN_EMPTY);
+		}
+
+		/*
+		 * If DTR Toggle mode is on, turn on DTR now if not already set,
+		 * and make sure we get an event when the data transfer has
+		 * completed.
+		 */
+		if (ch->ch_digi.digi_flags & DIGI_DTR_TOGGLE) {
+			if (!(ch->ch_mostat & UART_MCR_DTR)) {
+				ch->ch_mostat |= (UART_MCR_DTR);
+				cls_assert_modem_signals(ch);
+			}
+			ch->ch_tun.un_flags |= (UN_EMPTY);
+		}
+		writeb(ch->ch_wqueue[ch->ch_w_tail], &ch->ch_cls_uart->txrx);
+		ch->ch_w_tail++;
+		ch->ch_w_tail &= WQUEUEMASK;
+		ch->ch_txcount++;
+		len_written++;
+		n--;
+	}
+
+	if (len_written > 0)
+		ch->ch_flags &= ~(CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM);
+
+exit_unlock:
+	spin_unlock_irqrestore(&ch->ch_lock, flags);
+}
+
+static void cls_parse_modem(struct channel_t *ch, unsigned char signals)
+{
+	unsigned char msignals = signals;
+	unsigned long flags;
+
+	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
+		return;
+
+	/*
+	 * Do altpin switching. Altpin switches DCD and DSR.
+	 * This prolly breaks DSRPACE, so we should be more clever here.
+	 */
+	spin_lock_irqsave(&ch->ch_lock, flags);
+	if (ch->ch_digi.digi_flags & DIGI_ALTPIN) {
+		unsigned char mswap = signals;
+
+		if (mswap & UART_MSR_DDCD) {
+			msignals &= ~UART_MSR_DDCD;
+			msignals |= UART_MSR_DDSR;
+		}
+		if (mswap & UART_MSR_DDSR) {
+			msignals &= ~UART_MSR_DDSR;
+			msignals |= UART_MSR_DDCD;
+		}
+		if (mswap & UART_MSR_DCD) {
+			msignals &= ~UART_MSR_DCD;
+			msignals |= UART_MSR_DSR;
+		}
+		if (mswap & UART_MSR_DSR) {
+			msignals &= ~UART_MSR_DSR;
+			msignals |= UART_MSR_DCD;
+		}
+	}
+	spin_unlock_irqrestore(&ch->ch_lock, flags);
+
+	/*
+	 * Scrub off lower bits. They signify delta's, which I don't
+	 * care about
+	 */
+	signals &= 0xf0;
+
+	spin_lock_irqsave(&ch->ch_lock, flags);
+	if (msignals & UART_MSR_DCD)
+		ch->ch_mistat |= UART_MSR_DCD;
+	else
+		ch->ch_mistat &= ~UART_MSR_DCD;
+
+	if (msignals & UART_MSR_DSR)
+		ch->ch_mistat |= UART_MSR_DSR;
+	else
+		ch->ch_mistat &= ~UART_MSR_DSR;
+
+	if (msignals & UART_MSR_RI)
+		ch->ch_mistat |= UART_MSR_RI;
+	else
+		ch->ch_mistat &= ~UART_MSR_RI;
+
+	if (msignals & UART_MSR_CTS)
+		ch->ch_mistat |= UART_MSR_CTS;
+	else
+		ch->ch_mistat &= ~UART_MSR_CTS;
+	spin_unlock_irqrestore(&ch->ch_lock, flags);
+}
+
 /* Parse the ISR register for the specific port */
 static inline void cls_parse_isr(struct dgnc_board *brd, uint port)
 {
@@ -387,8 +584,6 @@ static inline void cls_parse_isr(struct dgnc_board *brd, uint port)
 		/* Receive Interrupt pending */
 		if (isr & (UART_IIR_RDI | UART_IIR_RDI_TIMEOUT)) {
 			/* Read data from uart -> queue */
-			brd->intr_rx++;
-			ch->ch_intr_rx++;
 			cls_copy_data_from_uart_to_queue(ch);
 			dgnc_check_queue_flow_control(ch);
 		}
@@ -398,27 +593,48 @@ static inline void cls_parse_isr(struct dgnc_board *brd, uint port)
 			/* Transfer data (if any) from Write Queue -> UART. */
 			spin_lock_irqsave(&ch->ch_lock, flags);
 			ch->ch_flags |= (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM);
-			brd->intr_tx++;
-			ch->ch_intr_tx++;
 			spin_unlock_irqrestore(&ch->ch_lock, flags);
 			cls_copy_data_from_queue_to_uart(ch);
 		}
 
-		/* CTS/RTS change of state */
-		if (isr & UART_IIR_CTSRTS) {
-			brd->intr_modem++;
-			ch->ch_intr_modem++;
-			/*
-			 * Don't need to do anything, the cls_parse_modem
-			 * below will grab the updated modem signals.
-			 */
-		}
-
 		/* Parse any modem signal changes */
 		cls_parse_modem(ch, readb(&ch->ch_cls_uart->msr));
 	}
 }
 
+/* Channel lock MUST be held before calling this function! */
+static void cls_flush_uart_write(struct channel_t *ch)
+{
+	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
+		return;
+
+	writeb((UART_FCR_ENABLE_FIFO | UART_FCR_CLEAR_XMIT),
+	       &ch->ch_cls_uart->isr_fcr);
+	usleep_range(10, 20);
+
+	ch->ch_flags |= (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM);
+}
+
+/* Channel lock MUST be held before calling this function! */
+static void cls_flush_uart_read(struct channel_t *ch)
+{
+	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
+		return;
+
+	/*
+	 * For complete POSIX compatibility, we should be purging the
+	 * read FIFO in the UART here.
+	 *
+	 * However, clearing the read FIFO (UART_FCR_CLEAR_RCVR) also
+	 * incorrectly flushes write data as well as just basically trashing the
+	 * FIFO.
+	 *
+	 * Presumably, this is a bug in this UART.
+	 */
+
+	udelay(10);
+}
+
 /*
  * cls_param()
  * Send any/all changes to the line to the UART.
@@ -760,8 +976,6 @@ static irqreturn_t cls_intr(int irq, void *voidbrd)
 
 	spin_lock_irqsave(&brd->bd_intr_lock, flags);
 
-	brd->intr_count++;
-
 	/*
 	 * Check the board's global interrupt offset to see if we
 	 * we actually do have an interrupt pending for us.
@@ -804,93 +1018,6 @@ static void cls_enable_receiver(struct channel_t *ch)
 	writeb(tmp, &ch->ch_cls_uart->ier);
 }
 
-static void cls_copy_data_from_uart_to_queue(struct channel_t *ch)
-{
-	int qleft = 0;
-	unsigned char linestatus = 0;
-	unsigned char error_mask = 0;
-	ushort head;
-	ushort tail;
-	unsigned long flags;
-
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return;
-
-	spin_lock_irqsave(&ch->ch_lock, flags);
-
-	/* cache head and tail of queue */
-	head = ch->ch_r_head;
-	tail = ch->ch_r_tail;
-
-	/* Store how much space we have left in the queue */
-	qleft = tail - head - 1;
-	if (qleft < 0)
-		qleft += RQUEUEMASK + 1;
-
-	/*
-	 * Create a mask to determine whether we should
-	 * insert the character (if any) into our queue.
-	 */
-	if (ch->ch_c_iflag & IGNBRK)
-		error_mask |= UART_LSR_BI;
-
-	while (1) {
-		linestatus = readb(&ch->ch_cls_uart->lsr);
-
-		if (!(linestatus & (UART_LSR_DR)))
-			break;
-
-		/*
-		 * Discard character if we are ignoring the error mask.
-		*/
-		if (linestatus & error_mask)  {
-			linestatus = 0;
-			readb(&ch->ch_cls_uart->txrx);
-			continue;
-		}
-
-		/*
-		 * If our queue is full, we have no choice but to drop some
-		 * data. The assumption is that HWFLOW or SWFLOW should have
-		 * stopped things way way before we got to this point.
-		 *
-		 * I decided that I wanted to ditch the oldest data first,
-		 * I hope thats okay with everyone? Yes? Good.
-		 */
-		while (qleft < 1) {
-			tail = (tail + 1) & RQUEUEMASK;
-			ch->ch_r_tail = tail;
-			ch->ch_err_overrun++;
-			qleft++;
-		}
-
-		ch->ch_equeue[head] = linestatus & (UART_LSR_BI | UART_LSR_PE
-								 | UART_LSR_FE);
-		ch->ch_rqueue[head] = readb(&ch->ch_cls_uart->txrx);
-
-		qleft--;
-
-		if (ch->ch_equeue[head] & UART_LSR_PE)
-			ch->ch_err_parity++;
-		if (ch->ch_equeue[head] & UART_LSR_BI)
-			ch->ch_err_break++;
-		if (ch->ch_equeue[head] & UART_LSR_FE)
-			ch->ch_err_frame++;
-
-		/* Add to, and flip head if needed */
-		head = (head + 1) & RQUEUEMASK;
-		ch->ch_rxcount++;
-	}
-
-	/*
-	 * Write new final heads to channel structure.
-	 */
-	ch->ch_r_head = head & RQUEUEMASK;
-	ch->ch_e_head = head & EQUEUEMASK;
-
-	spin_unlock_irqrestore(&ch->ch_lock, flags);
-}
-
 /*
  * This function basically goes to sleep for secs, or until
  * it gets signalled that the port has fully drained.
@@ -926,199 +1053,6 @@ static int cls_drain(struct tty_struct *tty, uint seconds)
 					 ((un->un_flags & UN_EMPTY) == 0));
 }
 
-/* Channel lock MUST be held before calling this function! */
-static void cls_flush_uart_write(struct channel_t *ch)
-{
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return;
-
-	writeb((UART_FCR_ENABLE_FIFO | UART_FCR_CLEAR_XMIT),
-	       &ch->ch_cls_uart->isr_fcr);
-	usleep_range(10, 20);
-
-	ch->ch_flags |= (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM);
-}
-
-/* Channel lock MUST be held before calling this function! */
-static void cls_flush_uart_read(struct channel_t *ch)
-{
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return;
-
-	/*
-	 * For complete POSIX compatibility, we should be purging the
-	 * read FIFO in the UART here.
-	 *
-	 * However, clearing the read FIFO (UART_FCR_CLEAR_RCVR) also
-	 * incorrectly flushes write data as well as just basically trashing the
-	 * FIFO.
-	 *
-	 * Presumably, this is a bug in this UART.
-	 */
-
-	udelay(10);
-}
-
-static void cls_copy_data_from_queue_to_uart(struct channel_t *ch)
-{
-	ushort head;
-	ushort tail;
-	int n;
-	int qlen;
-	uint len_written = 0;
-	unsigned long flags;
-
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return;
-
-	spin_lock_irqsave(&ch->ch_lock, flags);
-
-	/* No data to write to the UART */
-	if (ch->ch_w_tail == ch->ch_w_head)
-		goto exit_unlock;
-
-	/* If port is "stopped", don't send any data to the UART */
-	if ((ch->ch_flags & CH_FORCED_STOP) ||
-	    (ch->ch_flags & CH_BREAK_SENDING))
-		goto exit_unlock;
-
-	if (!(ch->ch_flags & (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM)))
-		goto exit_unlock;
-
-	n = 32;
-
-	/* cache head and tail of queue */
-	head = ch->ch_w_head & WQUEUEMASK;
-	tail = ch->ch_w_tail & WQUEUEMASK;
-	qlen = (head - tail) & WQUEUEMASK;
-
-	/* Find minimum of the FIFO space, versus queue length */
-	n = min(n, qlen);
-
-	while (n > 0) {
-		/*
-		 * If RTS Toggle mode is on, turn on RTS now if not already set,
-		 * and make sure we get an event when the data transfer has
-		 * completed.
-		 */
-		if (ch->ch_digi.digi_flags & DIGI_RTS_TOGGLE) {
-			if (!(ch->ch_mostat & UART_MCR_RTS)) {
-				ch->ch_mostat |= (UART_MCR_RTS);
-				cls_assert_modem_signals(ch);
-			}
-			ch->ch_tun.un_flags |= (UN_EMPTY);
-		}
-
-		/*
-		 * If DTR Toggle mode is on, turn on DTR now if not already set,
-		 * and make sure we get an event when the data transfer has
-		 * completed.
-		 */
-		if (ch->ch_digi.digi_flags & DIGI_DTR_TOGGLE) {
-			if (!(ch->ch_mostat & UART_MCR_DTR)) {
-				ch->ch_mostat |= (UART_MCR_DTR);
-				cls_assert_modem_signals(ch);
-			}
-			ch->ch_tun.un_flags |= (UN_EMPTY);
-		}
-		writeb(ch->ch_wqueue[ch->ch_w_tail], &ch->ch_cls_uart->txrx);
-		ch->ch_w_tail++;
-		ch->ch_w_tail &= WQUEUEMASK;
-		ch->ch_txcount++;
-		len_written++;
-		n--;
-	}
-
-	if (len_written > 0)
-		ch->ch_flags &= ~(CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM);
-
-exit_unlock:
-	spin_unlock_irqrestore(&ch->ch_lock, flags);
-}
-
-static void cls_parse_modem(struct channel_t *ch, unsigned char signals)
-{
-	unsigned char msignals = signals;
-	unsigned long flags;
-
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return;
-
-	/*
-	 * Do altpin switching. Altpin switches DCD and DSR.
-	 * This prolly breaks DSRPACE, so we should be more clever here.
-	 */
-	spin_lock_irqsave(&ch->ch_lock, flags);
-	if (ch->ch_digi.digi_flags & DIGI_ALTPIN) {
-		unsigned char mswap = signals;
-
-		if (mswap & UART_MSR_DDCD) {
-			msignals &= ~UART_MSR_DDCD;
-			msignals |= UART_MSR_DDSR;
-		}
-		if (mswap & UART_MSR_DDSR) {
-			msignals &= ~UART_MSR_DDSR;
-			msignals |= UART_MSR_DDCD;
-		}
-		if (mswap & UART_MSR_DCD) {
-			msignals &= ~UART_MSR_DCD;
-			msignals |= UART_MSR_DSR;
-		}
-		if (mswap & UART_MSR_DSR) {
-			msignals &= ~UART_MSR_DSR;
-			msignals |= UART_MSR_DCD;
-		}
-	}
-	spin_unlock_irqrestore(&ch->ch_lock, flags);
-
-	/*
-	 * Scrub off lower bits. They signify delta's, which I don't
-	 * care about
-	 */
-	signals &= 0xf0;
-
-	spin_lock_irqsave(&ch->ch_lock, flags);
-	if (msignals & UART_MSR_DCD)
-		ch->ch_mistat |= UART_MSR_DCD;
-	else
-		ch->ch_mistat &= ~UART_MSR_DCD;
-
-	if (msignals & UART_MSR_DSR)
-		ch->ch_mistat |= UART_MSR_DSR;
-	else
-		ch->ch_mistat &= ~UART_MSR_DSR;
-
-	if (msignals & UART_MSR_RI)
-		ch->ch_mistat |= UART_MSR_RI;
-	else
-		ch->ch_mistat &= ~UART_MSR_RI;
-
-	if (msignals & UART_MSR_CTS)
-		ch->ch_mistat |= UART_MSR_CTS;
-	else
-		ch->ch_mistat &= ~UART_MSR_CTS;
-	spin_unlock_irqrestore(&ch->ch_lock, flags);
-}
-
-/* Make the UART raise any of the output signals we want up */
-static void cls_assert_modem_signals(struct channel_t *ch)
-{
-	unsigned char out;
-
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return;
-
-	out = ch->ch_mostat;
-
-	if (ch->ch_flags & CH_LOOPBACK)
-		out |= UART_MCR_LOOP;
-
-	writeb(out, &ch->ch_cls_uart->mcr);
-
-	/* Give time for the UART to actually drop the signals */
-	udelay(10);
-}
-
 static void cls_send_start_character(struct channel_t *ch)
 {
 	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
@@ -1298,3 +1232,24 @@ static void cls_vpd(struct dgnc_board *brd)
 
 	iounmap(re_map_vpdbase);
 }
+
+struct board_ops dgnc_cls_ops = {
+	.tasklet =			cls_tasklet,
+	.intr =				cls_intr,
+	.uart_init =			cls_uart_init,
+	.uart_off =			cls_uart_off,
+	.drain =			cls_drain,
+	.param =			cls_param,
+	.vpd =				cls_vpd,
+	.assert_modem_signals =		cls_assert_modem_signals,
+	.flush_uart_write =		cls_flush_uart_write,
+	.flush_uart_read =		cls_flush_uart_read,
+	.disable_receiver =		cls_disable_receiver,
+	.enable_receiver =		cls_enable_receiver,
+	.send_break =			cls_send_break,
+	.send_start_character =		cls_send_start_character,
+	.send_stop_character =		cls_send_stop_character,
+	.copy_data_from_queue_to_uart = cls_copy_data_from_queue_to_uart,
+	.get_uart_bytes_left =		cls_get_uart_bytes_left,
+	.send_immediate_char =		cls_send_immediate_char
+};
diff --git a/drivers/staging/dgnc/dgnc_driver.c b/drivers/staging/dgnc/dgnc_driver.c
index af2e835efa1b..fd372d3afa46 100644
--- a/drivers/staging/dgnc/dgnc_driver.c
+++ b/drivers/staging/dgnc/dgnc_driver.c
@@ -37,13 +37,14 @@ MODULE_SUPPORTED_DEVICE("dgnc");
  *
  */
 static int		dgnc_start(void);
-static int		dgnc_finalize_board_init(struct dgnc_board *brd);
-static int		dgnc_found_board(struct pci_dev *pdev, int id);
+static int dgnc_request_irq(struct dgnc_board *brd);
+static void dgnc_free_irq(struct dgnc_board *brd);
+static struct dgnc_board *dgnc_found_board(struct pci_dev *pdev, int id);
 static void		dgnc_cleanup_board(struct dgnc_board *brd);
 static void		dgnc_poll_handler(ulong dummy);
 static int		dgnc_init_one(struct pci_dev *pdev,
 				      const struct pci_device_id *ent);
-static void		dgnc_do_remap(struct dgnc_board *brd);
+static int		dgnc_do_remap(struct dgnc_board *brd);
 
 /*
  * File operations permitted on Control/Management major.
@@ -146,7 +147,7 @@ static void cleanup(bool sysfiles)
 
 	for (i = 0; i < dgnc_num_boards; ++i) {
 		dgnc_remove_ports_sysfiles(dgnc_board[i]);
-		dgnc_tty_uninit(dgnc_board[i]);
+		dgnc_cleanup_tty(dgnc_board[i]);
 		dgnc_cleanup_board(dgnc_board[i]);
 	}
 
@@ -158,7 +159,7 @@ static void cleanup(bool sysfiles)
  *
  * Module unload.  This is where it all ends.
  */
-static void dgnc_cleanup_module(void)
+static void __exit dgnc_cleanup_module(void)
 {
 	cleanup(true);
 	pci_unregister_driver(&dgnc_driver);
@@ -274,6 +275,7 @@ failed_class:
 static int dgnc_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	int rc;
+	struct dgnc_board *brd;
 
 	/* wake up and enable device */
 	rc = pci_enable_device(pdev);
@@ -281,9 +283,48 @@ static int dgnc_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (rc)
 		return -EIO;
 
-	rc = dgnc_found_board(pdev, ent->driver_data);
-	if (rc == 0)
-		dgnc_num_boards++;
+	brd = dgnc_found_board(pdev, ent->driver_data);
+	if (IS_ERR(brd))
+		return PTR_ERR(brd);
+
+	/*
+	 * Do tty device initialization.
+	 */
+
+	rc = dgnc_tty_register(brd);
+	if (rc < 0) {
+		pr_err(DRVSTR ": Can't register tty devices (%d)\n", rc);
+		goto failed;
+	}
+
+	rc = dgnc_request_irq(brd);
+	if (rc < 0) {
+		pr_err(DRVSTR ": Can't finalize board init (%d)\n", rc);
+		goto unregister_tty;
+	}
+
+	rc = dgnc_tty_init(brd);
+	if (rc < 0) {
+		pr_err(DRVSTR ": Can't init tty devices (%d)\n", rc);
+		goto free_irq;
+	}
+
+	brd->state = BOARD_READY;
+	brd->dpastatus = BD_RUNNING;
+
+	dgnc_create_ports_sysfiles(brd);
+
+	dgnc_board[dgnc_num_boards++] = brd;
+
+	return 0;
+
+free_irq:
+	dgnc_free_irq(brd);
+unregister_tty:
+	dgnc_tty_unregister(brd);
+
+failed:
+	kfree(brd);
 
 	return rc;
 }
@@ -324,17 +365,6 @@ static void dgnc_cleanup_board(struct dgnc_board *brd)
 		brd->re_map_membase = NULL;
 	}
 
-	if (brd->msgbuf_head) {
-		unsigned long flags;
-
-		spin_lock_irqsave(&dgnc_global_lock, flags);
-		brd->msgbuf = NULL;
-		dev_dbg(&brd->pdev->dev, "%s\n", brd->msgbuf_head);
-		kfree(brd->msgbuf_head);
-		brd->msgbuf_head = NULL;
-		spin_unlock_irqrestore(&dgnc_global_lock, flags);
-	}
-
 	/* Free all allocated channels structs */
 	for (i = 0; i < MAXPORTS ; i++) {
 		if (brd->channels[i]) {
@@ -356,29 +386,17 @@ static void dgnc_cleanup_board(struct dgnc_board *brd)
  *
  * A board has been found, init it.
  */
-static int dgnc_found_board(struct pci_dev *pdev, int id)
+static struct dgnc_board *dgnc_found_board(struct pci_dev *pdev, int id)
 {
 	struct dgnc_board *brd;
 	unsigned int pci_irq;
 	int i = 0;
 	int rc = 0;
-	unsigned long flags;
 
 	/* get the board structure and prep it */
-	dgnc_board[dgnc_num_boards] = kzalloc(sizeof(*brd), GFP_KERNEL);
-	brd = dgnc_board[dgnc_num_boards];
-
+	brd = kzalloc(sizeof(*brd), GFP_KERNEL);
 	if (!brd)
-		return -ENOMEM;
-
-	/* make a temporary message buffer for the boot messages */
-	brd->msgbuf_head = kcalloc(8192, sizeof(u8), GFP_KERNEL);
-	brd->msgbuf = brd->msgbuf_head;
-
-	if (!brd->msgbuf) {
-		kfree(brd);
-		return -ENOMEM;
-	}
+		return ERR_PTR(-ENOMEM);
 
 	/* store the info for the board we've found */
 	brd->magic = DGNC_BOARD_MAGIC;
@@ -400,9 +418,6 @@ static int dgnc_found_board(struct pci_dev *pdev, int id)
 
 	brd->state		= BOARD_FOUND;
 
-	for (i = 0; i < MAXPORTS; i++)
-		brd->channels[i] = NULL;
-
 	/* store which card & revision we have */
 	pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &brd->subvendor);
 	pci_read_config_word(pdev, PCI_SUBSYSTEM_ID, &brd->subdevice);
@@ -435,7 +450,8 @@ static int dgnc_found_board(struct pci_dev *pdev, int id)
 		if (!brd->membase) {
 			dev_err(&brd->pdev->dev,
 				"Card has no PCI IO resources, failing.\n");
-			return -ENODEV;
+			rc = -ENODEV;
+			goto failed;
 		}
 
 		brd->membase_end = pci_resource_end(pdev, 4);
@@ -455,7 +471,10 @@ static int dgnc_found_board(struct pci_dev *pdev, int id)
 		brd->bd_uart_offset = 0x8;
 		brd->bd_dividend = 921600;
 
-		dgnc_do_remap(brd);
+		rc = dgnc_do_remap(brd);
+
+		if (rc < 0)
+			goto failed;
 
 		/* Get and store the board VPD, if it exists */
 		brd->bd_ops->vpd(brd);
@@ -507,81 +526,45 @@ static int dgnc_found_board(struct pci_dev *pdev, int id)
 		brd->bd_uart_offset = 0x200;
 		brd->bd_dividend = 921600;
 
-		dgnc_do_remap(brd);
+		rc = dgnc_do_remap(brd);
 
-		if (brd->re_map_membase) {
-			/* Read and store the dvid after remapping */
-			brd->dvid = readb(brd->re_map_membase + 0x8D);
+		if (rc < 0)
+			goto failed;
+
+		/* Read and store the dvid after remapping */
+		brd->dvid = readb(brd->re_map_membase + 0x8D);
+
+		/* Get and store the board VPD, if it exists */
+		brd->bd_ops->vpd(brd);
 
-			/* Get and store the board VPD, if it exists */
-			brd->bd_ops->vpd(brd);
-		}
 		break;
 
 	default:
 		dev_err(&brd->pdev->dev,
 			"Didn't find any compatible Neo/Classic PCI boards.\n");
-		return -ENXIO;
-	}
-
-	/*
-	 * Do tty device initialization.
-	 */
-
-	rc = dgnc_tty_register(brd);
-	if (rc < 0) {
-		pr_err(DRVSTR ": Can't register tty devices (%d)\n", rc);
-		goto failed;
-	}
-
-	rc = dgnc_finalize_board_init(brd);
-	if (rc < 0) {
-		pr_err(DRVSTR ": Can't finalize board init (%d)\n", rc);
-		goto failed;
-	}
-
-	rc = dgnc_tty_init(brd);
-	if (rc < 0) {
-		pr_err(DRVSTR ": Can't init tty devices (%d)\n", rc);
+		rc = -ENXIO;
 		goto failed;
 	}
 
-	brd->state = BOARD_READY;
-	brd->dpastatus = BD_RUNNING;
-
-	dgnc_create_ports_sysfiles(brd);
-
 	/* init our poll helper tasklet */
 	tasklet_init(&brd->helper_tasklet,
 		     brd->bd_ops->tasklet,
 		     (unsigned long)brd);
 
-	spin_lock_irqsave(&dgnc_global_lock, flags);
-	brd->msgbuf = NULL;
-	dev_dbg(&brd->pdev->dev, "%s\n", brd->msgbuf_head);
-	kfree(brd->msgbuf_head);
-	brd->msgbuf_head = NULL;
-	spin_unlock_irqrestore(&dgnc_global_lock, flags);
-
 	wake_up_interruptible(&brd->state_wait);
 
-	return 0;
+	return brd;
 
 failed:
-	dgnc_tty_uninit(brd);
-	brd->state = BOARD_FAILED;
-	brd->dpastatus = BD_NOFEP;
+	kfree(brd);
 
-	return -ENXIO;
+	return ERR_PTR(rc);
 }
 
-static int dgnc_finalize_board_init(struct dgnc_board *brd)
+static int dgnc_request_irq(struct dgnc_board *brd)
 {
 	int rc = 0;
 
-	if (!brd || brd->magic != DGNC_BOARD_MAGIC)
-		return -ENODEV;
-
 	if (brd->irq) {
 		rc = request_irq(brd->irq, brd->bd_ops->intr,
 				 IRQF_SHARED, "DGNC", brd);
@@ -597,42 +580,51 @@ static int dgnc_finalize_board_init(struct dgnc_board *brd)
 	return rc;
 }
 
+static void dgnc_free_irq(struct dgnc_board *brd)
+{
+	if (brd->irq)
+		free_irq(brd->irq, brd);
+}
+
 /*
  * Remap PCI memory.
  */
-static void dgnc_do_remap(struct dgnc_board *brd)
+static int dgnc_do_remap(struct dgnc_board *brd)
 {
-	if (!brd || brd->magic != DGNC_BOARD_MAGIC)
-		return;
+	int rc = 0;
 
 	brd->re_map_membase = ioremap(brd->membase, 0x1000);
+	if (!brd->re_map_membase)
+		rc = -ENOMEM;
+
+	return rc;
 }
 
-/*****************************************************************************
-*
-* Function:
-*
-*    dgnc_poll_handler
-*
-* Author:
-*
-*    Scott H Kilau
-*
-* Parameters:
-*
-*    dummy -- ignored
-*
-* Return Values:
-*
-*    none
-*
-* Description:
-*
-*    As each timer expires, it determines (a) whether the "transmit"
-*    waiter needs to be woken up, and (b) whether the poller needs to
-*    be rescheduled.
-*
-******************************************************************************/
+/*
+ *
+ * Function:
+ *
+ *    dgnc_poll_handler
+ *
+ * Author:
+ *
+ *    Scott H Kilau
+ *
+ * Parameters:
+ *
+ *    dummy -- ignored
+ *
+ * Return Values:
+ *
+ *    none
+ *
+ * Description:
+ *
+ *    As each timer expires, it determines (a) whether the "transmit"
+ *    waiter needs to be woken up, and (b) whether the poller needs to
+ *    be rescheduled.
+ *
+ */
 
 static void dgnc_poll_handler(ulong dummy)
 {
diff --git a/drivers/staging/dgnc/dgnc_driver.h b/drivers/staging/dgnc/dgnc_driver.h
index 95ec729fae38..879202663a98 100644
--- a/drivers/staging/dgnc/dgnc_driver.h
+++ b/drivers/staging/dgnc/dgnc_driver.h
@@ -183,10 +183,6 @@ struct dgnc_board {
 	uint		nasync;		/* Number of ports on card */
 
 	uint		irq;		/* Interrupt request number */
-	ulong		intr_count;	/* Count of interrupts */
-	ulong		intr_modem;	/* Count of interrupts */
-	ulong		intr_tx;	/* Count of interrupts */
-	ulong		intr_rx;	/* Count of interrupts */
 
 	ulong		membase;	/* Start of base memory of the card */
 	ulong		membase_end;	/* End of base memory of the card */
@@ -207,9 +203,6 @@ struct dgnc_board {
 	struct tty_driver *print_driver;
 	char		print_name[200];
 
-	bool		dgnc_major_serial_registered;
-	bool		dgnc_major_transparent_print_registered;
-
 	u16		dpatype;	/* The board "type",
 					 * as defined by DPA
 					 */
@@ -217,12 +210,6 @@ struct dgnc_board {
 					 * as defined by DPA
 					 */
 
-	/*
-	 *	Mgmt data.
-	 */
-	char		*msgbuf_head;
-	char		*msgbuf;
-
 	uint		bd_dividend;	/* Board/UARTs specific dividend */
 
 	struct board_ops *bd_ops;
@@ -381,10 +368,6 @@ struct channel_t {
 	ulong		ch_xon_sends;	/* Count of xons transmitted */
 	ulong		ch_xoff_sends;	/* Count of xoffs transmitted */
 
-	ulong		ch_intr_modem;	/* Count of interrupts */
-	ulong		ch_intr_tx;	/* Count of interrupts */
-	ulong		ch_intr_rx;	/* Count of interrupts */
-
 	/* /proc/<board>/<channel> entries */
 	struct proc_dir_entry *proc_entry_pointer;
 	struct dgnc_proc_entry *dgnc_channel_table;
@@ -398,7 +381,7 @@ extern uint		dgnc_major;		/* Our driver/mgmt major */
 extern int		dgnc_poll_tick;		/* Poll interval - 20 ms */
 extern spinlock_t	dgnc_global_lock;	/* Driver global spinlock */
 extern spinlock_t	dgnc_poll_lock;		/* Poll scheduling lock */
-extern uint		dgnc_num_boards;		/* Total number of boards */
+extern uint		dgnc_num_boards;	/* Total number of boards */
 extern struct dgnc_board	*dgnc_board[MAXBOARDS];	/* Array of board
 							 * structs
 							 */
diff --git a/drivers/staging/dgnc/dgnc_neo.c b/drivers/staging/dgnc/dgnc_neo.c
index ba57e9546f72..5becb3741b67 100644
--- a/drivers/staging/dgnc/dgnc_neo.c
+++ b/drivers/staging/dgnc/dgnc_neo.c
@@ -107,7 +107,9 @@ static inline void neo_set_cts_flow_control(struct channel_t *ch)
 	/* Turn off auto Xon flow control */
 	efr &= ~UART_17158_EFR_IXON;
 
-	/* Why? Becuz Exar's spec says we have to zero it out before setting it */
+	/* Why? Because Exar's spec says we have to zero it
+	 * out before setting it
+	 */
 	writeb(0, &ch->ch_neo_uart->efr);
 
 	/* Turn on UART enhanced bits */
@@ -143,7 +145,9 @@ static inline void neo_set_rts_flow_control(struct channel_t *ch)
 	ier &= ~UART_17158_IER_XOFF;
 	efr &= ~UART_17158_EFR_IXOFF;
 
-	/* Why? Becuz Exar's spec says we have to zero it out before setting it */
+	/* Why? Because Exar's spec says we have to zero it
+	 * out before setting it
+	 */
 	writeb(0, &ch->ch_neo_uart->efr);
 
 	/* Turn on UART enhanced bits */
@@ -181,7 +185,9 @@ static inline void neo_set_ixon_flow_control(struct channel_t *ch)
 	/* Turn on auto Xon flow control */
 	efr |= (UART_17158_EFR_ECB | UART_17158_EFR_IXON);
 
-	/* Why? Becuz Exar's spec says we have to zero it out before setting it */
+	/* Why? Because Exar's spec says we have to zero it
+	 * out before setting it
+	 */
 	writeb(0, &ch->ch_neo_uart->efr);
 
 	/* Turn on UART enhanced bits */
@@ -219,7 +225,9 @@ static inline void neo_set_ixoff_flow_control(struct channel_t *ch)
 	ier |= UART_17158_IER_XOFF;
 	efr |= (UART_17158_EFR_ECB | UART_17158_EFR_IXOFF);
 
-	/* Why? Becuz Exar's spec says we have to zero it out before setting it */
+	/* Why? Because Exar's spec says we have to zero it
+	 * out before setting it
+	 */
 	writeb(0, &ch->ch_neo_uart->efr);
 
 	/* Turn on UART enhanced bits */
@@ -260,7 +268,9 @@ static inline void neo_set_no_input_flow_control(struct channel_t *ch)
 	else
 		efr &= ~(UART_17158_EFR_ECB | UART_17158_EFR_IXOFF);
 
-	/* Why? Becuz Exar's spec says we have to zero it out before setting it */
+	/* Why? Because Exar's spec says we have to zero
+	 * it out before setting it
+	 */
 	writeb(0, &ch->ch_neo_uart->efr);
 
 	/* Turn on UART enhanced bits */
@@ -298,7 +308,9 @@ static inline void neo_set_no_output_flow_control(struct channel_t *ch)
 	else
 		efr &= ~(UART_17158_EFR_ECB | UART_17158_EFR_IXON);
 
-	/* Why? Becuz Exar's spec says we have to zero it out before setting it */
+	/* Why? Because Exar's spec says we have to zero it
+	 * out before setting it
+	 */
 	writeb(0, &ch->ch_neo_uart->efr);
 
 	/* Turn on UART enhanced bits */
@@ -399,19 +411,17 @@ static inline void neo_parse_isr(struct dgnc_board *brd, uint port)
 
 		if (isr & (UART_17158_IIR_RDI_TIMEOUT | UART_IIR_RDI)) {
 			/* Read data from uart -> queue */
-			brd->intr_rx++;
-			ch->ch_intr_rx++;
 			neo_copy_data_from_uart_to_queue(ch);
 
-			/* Call our tty layer to enforce queue flow control if needed. */
+			/* Call our tty layer to enforce queue
+			 * flow control if needed.
+			 */
 			spin_lock_irqsave(&ch->ch_lock, flags);
 			dgnc_check_queue_flow_control(ch);
 			spin_unlock_irqrestore(&ch->ch_lock, flags);
 		}
 
 		if (isr & UART_IIR_THRI) {
-			brd->intr_tx++;
-			ch->ch_intr_tx++;
 			/* Transfer data (if any) from Write Queue -> UART. */
 			spin_lock_irqsave(&ch->ch_lock, flags);
 			ch->ch_flags |= (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM);
@@ -428,7 +438,9 @@ static inline void neo_parse_isr(struct dgnc_board *brd, uint port)
 			 * one it was, so we can suspend or resume data flow.
 			 */
 			if (cause == UART_17158_XON_DETECT) {
-				/* Is output stopped right now, if so, resume it */
+				/* Is output stopped right now, if so,
+				 * resume it
+				 */
 				if (brd->channels[port]->ch_flags & CH_STOP) {
 					spin_lock_irqsave(&ch->ch_lock,
 							  flags);
@@ -437,7 +449,8 @@ static inline void neo_parse_isr(struct dgnc_board *brd, uint port)
 							       flags);
 				}
 			} else if (cause == UART_17158_XOFF_DETECT) {
-				if (!(brd->channels[port]->ch_flags & CH_STOP)) {
+				if (!(brd->channels[port]->ch_flags &
+				      CH_STOP)) {
 					spin_lock_irqsave(&ch->ch_lock,
 							  flags);
 					ch->ch_flags |= CH_STOP;
@@ -449,11 +462,10 @@ static inline void neo_parse_isr(struct dgnc_board *brd, uint port)
 
 		if (isr & UART_17158_IIR_HWFLOW_STATE_CHANGE) {
 			/*
-			 * If we get here, this means the hardware is doing auto flow control.
-			 * Check to see whether RTS/DTR or CTS/DSR caused this interrupt.
+			 * If we get here, this means the hardware is
+			 * doing auto flow control. Check to see whether
+			 * RTS/DTR or CTS/DSR caused this interrupt.
 			 */
-			brd->intr_modem++;
-			ch->ch_intr_modem++;
 			cause = readb(&ch->ch_neo_uart->mcr);
 			/* Which pin is doing auto flow? RTS or DTR? */
 			if ((cause & 0x4) == 0) {
@@ -517,8 +529,6 @@ static inline void neo_parse_lsr(struct dgnc_board *brd, uint port)
 	ch->ch_cached_lsr |= linestatus;
 
 	if (ch->ch_cached_lsr & UART_LSR_DR) {
-		brd->intr_rx++;
-		ch->ch_intr_rx++;
 		/* Read data from uart -> queue */
 		neo_copy_data_from_uart_to_queue(ch);
 		spin_lock_irqsave(&ch->ch_lock, flags);
@@ -545,14 +555,13 @@ static inline void neo_parse_lsr(struct dgnc_board *brd, uint port)
 		 * Rx Oruns. Exar says that an orun will NOT corrupt
 		 * the FIFO. It will just replace the holding register
 		 * with this new data byte. So basically just ignore this.
-		 * Probably we should eventually have an orun stat in our driver...
+		 * Probably we should eventually have an orun stat in our
+		 * driver...
 		 */
 		ch->ch_err_overrun++;
 	}
 
 	if (linestatus & UART_LSR_THRE) {
-		brd->intr_tx++;
-		ch->ch_intr_tx++;
 		spin_lock_irqsave(&ch->ch_lock, flags);
 		ch->ch_flags |= (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM);
 		spin_unlock_irqrestore(&ch->ch_lock, flags);
@@ -560,8 +569,6 @@ static inline void neo_parse_lsr(struct dgnc_board *brd, uint port)
 		/* Transfer data (if any) from Write Queue -> UART. */
 		neo_copy_data_from_queue_to_uart(ch);
 	} else if (linestatus & UART_17158_TX_AND_FIFO_CLR) {
-		brd->intr_tx++;
-		ch->ch_intr_tx++;
 		spin_lock_irqsave(&ch->ch_lock, flags);
 		ch->ch_flags |= (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM);
 		spin_unlock_irqrestore(&ch->ch_lock, flags);
@@ -665,7 +672,9 @@ static void neo_param(struct tty_struct *tty)
 				4800,   9600,   19200,  38400 }
 		};
 
-		/* Only use the TXPrint baud rate if the terminal unit is NOT open */
+		/* Only use the TXPrint baud rate if the terminal unit
+		 * is NOT open
+		 */
 		if (!(ch->ch_tun.un_flags & UN_ISOPEN) &&
 		    (un->un_type == DGNC_PRINT))
 			baud = C_BAUD(ch->ch_pun.un_tty) & 0xff;
@@ -788,7 +797,9 @@ static void neo_param(struct tty_struct *tty)
 	if (ch->ch_digi.digi_flags & CTSPACE || ch->ch_c_cflag & CRTSCTS) {
 		neo_set_cts_flow_control(ch);
 	} else if (ch->ch_c_iflag & IXON) {
-		/* If start/stop is set to disable, then we should disable flow control */
+		/* If start/stop is set to disable, then we should
+		 * disable flow control
+		 */
 		if ((ch->ch_startc == _POSIX_VDISABLE) ||
 		    (ch->ch_stopc == _POSIX_VDISABLE))
 			neo_set_no_output_flow_control(ch);
@@ -801,7 +812,9 @@ static void neo_param(struct tty_struct *tty)
 	if (ch->ch_digi.digi_flags & RTSPACE || ch->ch_c_cflag & CRTSCTS) {
 		neo_set_rts_flow_control(ch);
 	} else if (ch->ch_c_iflag & IXOFF) {
-		/* If start/stop is set to disable, then we should disable flow control */
+		/* If start/stop is set to disable, then we should
+		 * disable flow control
+		 */
 		if ((ch->ch_startc == _POSIX_VDISABLE) ||
 		    (ch->ch_stopc == _POSIX_VDISABLE))
 			neo_set_no_input_flow_control(ch);
@@ -926,8 +939,6 @@ static irqreturn_t neo_intr(int irq, void *voidbrd)
 	if (!brd || brd->magic != DGNC_BOARD_MAGIC)
 		return IRQ_NONE;
 
-	brd->intr_count++;
-
 	/* Lock out the slow poller from running on this board. */
 	spin_lock_irqsave(&brd->bd_intr_lock, flags);
 
@@ -940,14 +951,18 @@ static irqreturn_t neo_intr(int irq, void *voidbrd)
 
 	/*
 	 * If 0, no interrupts pending.
-	 * This can happen if the IRQ is shared among a couple Neo/Classic boards.
+	 * This can happen if the IRQ is shared among a couple Neo/Classic
+	 * boards.
 	 */
 	if (!uart_poll) {
 		spin_unlock_irqrestore(&brd->bd_intr_lock, flags);
 		return IRQ_NONE;
 	}
 
-	/* At this point, we have at least SOMETHING to service, dig further... */
+	/*
+	 * At this point, we have at least SOMETHING to service, dig
+	 * further...
+	 */
 
 	/* Loop on each port */
 	while ((uart_poll & 0xff) != 0) {
@@ -971,7 +986,10 @@ static irqreturn_t neo_intr(int irq, void *voidbrd)
 			ch = brd->channels[port];
 			neo_copy_data_from_uart_to_queue(ch);
 
-			/* Call our tty layer to enforce queue flow control if needed. */
+			/*
+			 * Call our tty layer to enforce queue flow control if
+			 * needed.
+			 */
 			spin_lock_irqsave(&ch->ch_lock, flags2);
 			dgnc_check_queue_flow_control(ch);
 			spin_unlock_irqrestore(&ch->ch_lock, flags2);
@@ -987,16 +1005,18 @@ static irqreturn_t neo_intr(int irq, void *voidbrd)
 
 		case UART_17158_TXRDY:
 			/*
-			 * TXRDY interrupt clears after reading ISR register for the UART channel.
+			 * TXRDY interrupt clears after reading ISR register
+			 * for the UART channel.
 			 */
 
 			/*
 			 * Yes, this is odd...
 			 * Why would I check EVERY possibility of type of
 			 * interrupt, when we know its TXRDY???
-			 * Becuz for some reason, even tho we got triggered for TXRDY,
-			 * it seems to be occasionally wrong. Instead of TX, which
-			 * it should be, I was getting things like RXDY too. Weird.
+			 * Becuz for some reason, even tho we got triggered for
+			 * TXRDY, it seems to be occasionally wrong. Instead of
+			 * TX, which it should be, I was getting things like
+			 * RXDY too. Weird.
 			 */
 			neo_parse_isr(brd, port);
 			break;
@@ -1011,8 +1031,8 @@ static irqreturn_t neo_intr(int irq, void *voidbrd)
 		default:
 			/*
 			 * The UART triggered us with a bogus interrupt type.
-			 * It appears the Exar chip, when REALLY bogged down, will throw
-			 * these once and awhile.
+			 * It appears the Exar chip, when REALLY bogged down,
+			 * will throw these once and awhile.
 			 * Its harmless, just ignore it and move on.
 			 */
 			break;
@@ -1230,7 +1250,8 @@ static void neo_copy_data_from_uart_to_queue(struct channel_t *ch)
 		}
 
 		/*
-		 * If our queue is full, we have no choice but to drop some data.
+		 * If our queue is full, we have no choice but to drop some
+		 * data.
 		 * The assumption is that HWFLOW or SWFLOW should have stopped
 		 * things way way before we got to this point.
 		 *
@@ -1323,7 +1344,10 @@ static void neo_flush_uart_write(struct channel_t *ch)
 	neo_pci_posting_flush(ch->ch_bd);
 
 	for (i = 0; i < 10; i++) {
-		/* Check to see if the UART feels it completely flushed the FIFO. */
+		/*
+		 * Check to see if the UART feels it completely flushed the
+		 * FIFO.
+		 */
 		tmp = readb(&ch->ch_neo_uart->isr_fcr);
 		if (tmp & 4)
 			udelay(10);
@@ -1352,7 +1376,10 @@ static void neo_flush_uart_read(struct channel_t *ch)
 	neo_pci_posting_flush(ch->ch_bd);
 
 	for (i = 0; i < 10; i++) {
-		/* Check to see if the UART feels it completely flushed the FIFO. */
+		/*
+		 * Check to see if the UART feels it completely flushed the
+		 * FIFO.
+		 */
 		tmp = readb(&ch->ch_neo_uart->isr_fcr);
 		if (tmp & 2)
 			udelay(10);
@@ -1397,8 +1424,9 @@ static void neo_copy_data_from_queue_to_uart(struct channel_t *ch)
 			ch->ch_cached_lsr &= ~(UART_LSR_THRE);
 
 			/*
-			 * If RTS Toggle mode is on, turn on RTS now if not already set,
-			 * and make sure we get an event when the data transfer has completed.
+			 * If RTS Toggle mode is on, turn on RTS now if not
+			 * already set, and make sure we get an event when the
+			 * data transfer has completed.
 			 */
 			if (ch->ch_digi.digi_flags & DIGI_RTS_TOGGLE) {
 				if (!(ch->ch_mostat & UART_MCR_RTS)) {
@@ -1408,8 +1436,9 @@ static void neo_copy_data_from_queue_to_uart(struct channel_t *ch)
 				ch->ch_tun.un_flags |= (UN_EMPTY);
 			}
 			/*
-			 * If DTR Toggle mode is on, turn on DTR now if not already set,
-			 * and make sure we get an event when the data transfer has completed.
+			 * If DTR Toggle mode is on, turn on DTR now if not
+			 * already set, and make sure we get an event when the
+			 * data transfer has completed.
 			 */
 			if (ch->ch_digi.digi_flags & DIGI_DTR_TOGGLE) {
 				if (!(ch->ch_mostat & UART_MCR_DTR)) {
@@ -1465,7 +1494,8 @@ static void neo_copy_data_from_queue_to_uart(struct channel_t *ch)
 
 		/*
 		 * If RTS Toggle mode is on, turn on RTS now if not already set,
-		 * and make sure we get an event when the data transfer has completed.
+		 * and make sure we get an event when the data transfer has
+		 * completed.
 		 */
 		if (ch->ch_digi.digi_flags & DIGI_RTS_TOGGLE) {
 			if (!(ch->ch_mostat & UART_MCR_RTS)) {
@@ -1477,7 +1507,8 @@ static void neo_copy_data_from_queue_to_uart(struct channel_t *ch)
 
 		/*
 		 * If DTR Toggle mode is on, turn on DTR now if not already set,
-		 * and make sure we get an event when the data transfer has completed.
+		 * and make sure we get an event when the data transfer has
+		 * completed.
 		 */
 		if (ch->ch_digi.digi_flags & DIGI_DTR_TOGGLE) {
 			if (!(ch->ch_mostat & UART_MCR_DTR)) {
@@ -1541,7 +1572,10 @@ static void neo_parse_modem(struct channel_t *ch, unsigned char signals)
 		}
 	}
 
-	/* Scrub off lower bits. They signify delta's, which I don't care about */
+	/*
+	 * Scrub off lower bits. They signify delta's, which I don't care
+	 * about
+	 */
 	msignals &= 0xf0;
 
 	if (msignals & UART_MSR_DCD)
diff --git a/drivers/staging/dgnc/dgnc_sysfs.c b/drivers/staging/dgnc/dgnc_sysfs.c
index b8d41c5617e2..290bf6e226ac 100644
--- a/drivers/staging/dgnc/dgnc_sysfs.c
+++ b/drivers/staging/dgnc/dgnc_sysfs.c
@@ -25,31 +25,31 @@
 #include "dgnc_driver.h"
 #include "dgnc_mgmt.h"
 
-static ssize_t dgnc_driver_version_show(struct device_driver *ddp, char *buf)
+static ssize_t version_show(struct device_driver *ddp, char *buf)
 {
 	return snprintf(buf, PAGE_SIZE, "%s\n", DG_PART);
 }
-static DRIVER_ATTR(version, S_IRUSR, dgnc_driver_version_show, NULL);
+static DRIVER_ATTR_RO(version);
 
-static ssize_t dgnc_driver_boards_show(struct device_driver *ddp, char *buf)
+static ssize_t boards_show(struct device_driver *ddp, char *buf)
 {
 	return snprintf(buf, PAGE_SIZE, "%d\n", dgnc_num_boards);
 }
-static DRIVER_ATTR(boards, S_IRUSR, dgnc_driver_boards_show, NULL);
+static DRIVER_ATTR_RO(boards);
 
-static ssize_t dgnc_driver_maxboards_show(struct device_driver *ddp, char *buf)
+static ssize_t maxboards_show(struct device_driver *ddp, char *buf)
 {
 	return snprintf(buf, PAGE_SIZE, "%d\n", MAXBOARDS);
 }
-static DRIVER_ATTR(maxboards, S_IRUSR, dgnc_driver_maxboards_show, NULL);
+static DRIVER_ATTR_RO(maxboards);
 
-static ssize_t dgnc_driver_pollrate_show(struct device_driver *ddp, char *buf)
+static ssize_t pollrate_show(struct device_driver *ddp, char *buf)
 {
 	return snprintf(buf, PAGE_SIZE, "%dms\n", dgnc_poll_tick);
 }
 
-static ssize_t dgnc_driver_pollrate_store(struct device_driver *ddp,
-					  const char *buf, size_t count)
+static ssize_t pollrate_store(struct device_driver *ddp,
+			      const char *buf, size_t count)
 {
 	unsigned long flags;
 	int tick;
@@ -65,8 +65,7 @@ static ssize_t dgnc_driver_pollrate_store(struct device_driver *ddp,
 
 	return count;
 }
-static DRIVER_ATTR(pollrate, (S_IRUSR | S_IWUSR), dgnc_driver_pollrate_show,
-		   dgnc_driver_pollrate_store);
+static DRIVER_ATTR_RW(pollrate);
 
 void dgnc_create_driver_sysfiles(struct pci_driver *dgnc_driver)
 {
@@ -103,8 +102,8 @@ void dgnc_remove_driver_sysfiles(struct pci_driver *dgnc_driver)
 			return 0;				\
 	} while (0)
 
-static ssize_t dgnc_vpd_show(struct device *p, struct device_attribute *attr,
-			     char *buf)
+static ssize_t vpd_show(struct device *p, struct device_attribute *attr,
+			char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -123,10 +122,10 @@ static ssize_t dgnc_vpd_show(struct device *p, struct device_attribute *attr,
 
 	return count;
 }
-static DEVICE_ATTR(vpd, S_IRUSR, dgnc_vpd_show, NULL);
+static DEVICE_ATTR_RO(vpd);
 
-static ssize_t dgnc_serial_number_show(struct device *p,
-				       struct device_attribute *attr, char *buf)
+static ssize_t serial_number_show(struct device *p,
+				  struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -140,10 +139,10 @@ static ssize_t dgnc_serial_number_show(struct device *p,
 
 	return count;
 }
-static DEVICE_ATTR(serial_number, S_IRUSR, dgnc_serial_number_show, NULL);
+static DEVICE_ATTR_RO(serial_number);
 
-static ssize_t dgnc_ports_state_show(struct device *p,
-				     struct device_attribute *attr, char *buf)
+static ssize_t ports_state_show(struct device *p,
+				struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -158,10 +157,10 @@ static ssize_t dgnc_ports_state_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_state, S_IRUSR, dgnc_ports_state_show, NULL);
+static DEVICE_ATTR_RO(ports_state);
 
-static ssize_t dgnc_ports_baud_show(struct device *p,
-				    struct device_attribute *attr, char *buf)
+static ssize_t ports_baud_show(struct device *p,
+			       struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -176,11 +175,10 @@ static ssize_t dgnc_ports_baud_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_baud, S_IRUSR, dgnc_ports_baud_show, NULL);
+static DEVICE_ATTR_RO(ports_baud);
 
-static ssize_t dgnc_ports_msignals_show(struct device *p,
-					struct device_attribute *attr,
-					char *buf)
+static ssize_t ports_msignals_show(struct device *p,
+				   struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -208,10 +206,10 @@ static ssize_t dgnc_ports_msignals_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_msignals, S_IRUSR, dgnc_ports_msignals_show, NULL);
+static DEVICE_ATTR_RO(ports_msignals);
 
-static ssize_t dgnc_ports_iflag_show(struct device *p,
-				     struct device_attribute *attr, char *buf)
+static ssize_t ports_iflag_show(struct device *p,
+				struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -226,10 +224,10 @@ static ssize_t dgnc_ports_iflag_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_iflag, S_IRUSR, dgnc_ports_iflag_show, NULL);
+static DEVICE_ATTR_RO(ports_iflag);
 
-static ssize_t dgnc_ports_cflag_show(struct device *p,
-				     struct device_attribute *attr, char *buf)
+static ssize_t ports_cflag_show(struct device *p,
+				struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -244,10 +242,10 @@ static ssize_t dgnc_ports_cflag_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_cflag, S_IRUSR, dgnc_ports_cflag_show, NULL);
+static DEVICE_ATTR_RO(ports_cflag);
 
-static ssize_t dgnc_ports_oflag_show(struct device *p,
-				     struct device_attribute *attr, char *buf)
+static ssize_t ports_oflag_show(struct device *p,
+				struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -262,10 +260,10 @@ static ssize_t dgnc_ports_oflag_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_oflag, S_IRUSR, dgnc_ports_oflag_show, NULL);
+static DEVICE_ATTR_RO(ports_oflag);
 
-static ssize_t dgnc_ports_lflag_show(struct device *p,
-				     struct device_attribute *attr, char *buf)
+static ssize_t ports_lflag_show(struct device *p,
+				struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -280,11 +278,10 @@ static ssize_t dgnc_ports_lflag_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_lflag, S_IRUSR, dgnc_ports_lflag_show, NULL);
+static DEVICE_ATTR_RO(ports_lflag);
 
-static ssize_t dgnc_ports_digi_flag_show(struct device *p,
-					 struct device_attribute *attr,
-					 char *buf)
+static ssize_t ports_digi_flag_show(struct device *p,
+				    struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -299,10 +296,10 @@ static ssize_t dgnc_ports_digi_flag_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_digi_flag, S_IRUSR, dgnc_ports_digi_flag_show, NULL);
+static DEVICE_ATTR_RO(ports_digi_flag);
 
-static ssize_t dgnc_ports_rxcount_show(struct device *p,
-				       struct device_attribute *attr, char *buf)
+static ssize_t ports_rxcount_show(struct device *p,
+				  struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -317,10 +314,10 @@ static ssize_t dgnc_ports_rxcount_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_rxcount, S_IRUSR, dgnc_ports_rxcount_show, NULL);
+static DEVICE_ATTR_RO(ports_rxcount);
 
-static ssize_t dgnc_ports_txcount_show(struct device *p,
-				       struct device_attribute *attr, char *buf)
+static ssize_t ports_txcount_show(struct device *p,
+				  struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	int count = 0;
@@ -335,7 +332,7 @@ static ssize_t dgnc_ports_txcount_show(struct device *p,
 	}
 	return count;
 }
-static DEVICE_ATTR(ports_txcount, S_IRUSR, dgnc_ports_txcount_show, NULL);
+static DEVICE_ATTR_RO(ports_txcount);
 
 /* this function creates the sys files that will export each signal status
  * to sysfs each value will be put in a separate filename
@@ -378,8 +375,8 @@ void dgnc_remove_ports_sysfiles(struct dgnc_board *bd)
 	device_remove_file(&bd->pdev->dev, &dev_attr_serial_number);
 }
 
-static ssize_t dgnc_tty_state_show(struct device *d,
-				   struct device_attribute *attr, char *buf)
+static ssize_t tty_state_show(struct device *d,
+			      struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -402,10 +399,10 @@ static ssize_t dgnc_tty_state_show(struct device *d,
 	return snprintf(buf, PAGE_SIZE, "%s",
 			un->un_open_count ? "Open" : "Closed");
 }
-static DEVICE_ATTR(state, S_IRUSR, dgnc_tty_state_show, NULL);
+static DEVICE_ATTR_RO(tty_state);
 
-static ssize_t dgnc_tty_baud_show(struct device *d,
-				  struct device_attribute *attr, char *buf)
+static ssize_t tty_baud_show(struct device *d,
+			     struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -427,10 +424,10 @@ static ssize_t dgnc_tty_baud_show(struct device *d,
 
 	return snprintf(buf, PAGE_SIZE, "%d\n", ch->ch_old_baud);
 }
-static DEVICE_ATTR(baud, S_IRUSR, dgnc_tty_baud_show, NULL);
+static DEVICE_ATTR_RO(tty_baud);
 
-static ssize_t dgnc_tty_msignals_show(struct device *d,
-				      struct device_attribute *attr, char *buf)
+static ssize_t tty_msignals_show(struct device *d,
+				 struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -461,10 +458,10 @@ static ssize_t dgnc_tty_msignals_show(struct device *d,
 	}
 	return 0;
 }
-static DEVICE_ATTR(msignals, S_IRUSR, dgnc_tty_msignals_show, NULL);
+static DEVICE_ATTR_RO(tty_msignals);
 
-static ssize_t dgnc_tty_iflag_show(struct device *d,
-				   struct device_attribute *attr, char *buf)
+static ssize_t tty_iflag_show(struct device *d,
+			      struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -486,10 +483,10 @@ static ssize_t dgnc_tty_iflag_show(struct device *d,
 
 	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_c_iflag);
 }
-static DEVICE_ATTR(iflag, S_IRUSR, dgnc_tty_iflag_show, NULL);
+static DEVICE_ATTR_RO(tty_iflag);
 
-static ssize_t dgnc_tty_cflag_show(struct device *d,
-				   struct device_attribute *attr, char *buf)
+static ssize_t tty_cflag_show(struct device *d,
+			      struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -511,10 +508,10 @@ static ssize_t dgnc_tty_cflag_show(struct device *d,
 
 	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_c_cflag);
 }
-static DEVICE_ATTR(cflag, S_IRUSR, dgnc_tty_cflag_show, NULL);
+static DEVICE_ATTR_RO(tty_cflag);
 
-static ssize_t dgnc_tty_oflag_show(struct device *d,
-				   struct device_attribute *attr, char *buf)
+static ssize_t tty_oflag_show(struct device *d,
+			      struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -536,10 +533,10 @@ static ssize_t dgnc_tty_oflag_show(struct device *d,
 
 	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_c_oflag);
 }
-static DEVICE_ATTR(oflag, S_IRUSR, dgnc_tty_oflag_show, NULL);
+static DEVICE_ATTR_RO(tty_oflag);
 
-static ssize_t dgnc_tty_lflag_show(struct device *d,
-				   struct device_attribute *attr, char *buf)
+static ssize_t tty_lflag_show(struct device *d,
+			      struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -561,10 +558,10 @@ static ssize_t dgnc_tty_lflag_show(struct device *d,
 
 	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_c_lflag);
 }
-static DEVICE_ATTR(lflag, S_IRUSR, dgnc_tty_lflag_show, NULL);
+static DEVICE_ATTR_RO(tty_lflag);
 
-static ssize_t dgnc_tty_digi_flag_show(struct device *d,
-				       struct device_attribute *attr, char *buf)
+static ssize_t tty_digi_flag_show(struct device *d,
+				  struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -586,10 +583,10 @@ static ssize_t dgnc_tty_digi_flag_show(struct device *d,
 
 	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_digi.digi_flags);
 }
-static DEVICE_ATTR(digi_flag, S_IRUSR, dgnc_tty_digi_flag_show, NULL);
+static DEVICE_ATTR_RO(tty_digi_flag);
 
-static ssize_t dgnc_tty_rxcount_show(struct device *d,
-				     struct device_attribute *attr, char *buf)
+static ssize_t tty_rxcount_show(struct device *d,
+				struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -611,10 +608,10 @@ static ssize_t dgnc_tty_rxcount_show(struct device *d,
 
 	return snprintf(buf, PAGE_SIZE, "%ld\n", ch->ch_rxcount);
 }
-static DEVICE_ATTR(rxcount, S_IRUSR, dgnc_tty_rxcount_show, NULL);
+static DEVICE_ATTR_RO(tty_rxcount);
 
-static ssize_t dgnc_tty_txcount_show(struct device *d,
-				     struct device_attribute *attr, char *buf)
+static ssize_t tty_txcount_show(struct device *d,
+				struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -636,10 +633,10 @@ static ssize_t dgnc_tty_txcount_show(struct device *d,
 
 	return snprintf(buf, PAGE_SIZE, "%ld\n", ch->ch_txcount);
 }
-static DEVICE_ATTR(txcount, S_IRUSR, dgnc_tty_txcount_show, NULL);
+static DEVICE_ATTR_RO(tty_txcount);
 
-static ssize_t dgnc_tty_name_show(struct device *d,
-				  struct device_attribute *attr, char *buf)
+static ssize_t tty_custom_name_show(struct device *d,
+				    struct device_attribute *attr, char *buf)
 {
 	struct dgnc_board *bd;
 	struct channel_t *ch;
@@ -663,24 +660,24 @@ static ssize_t dgnc_tty_name_show(struct device *d,
 		(un->un_type == DGNC_PRINT) ? "pr" : "tty",
 		bd->boardnum + 1, 'a' + ch->ch_portnum);
 }
-static DEVICE_ATTR(custom_name, S_IRUSR, dgnc_tty_name_show, NULL);
+static DEVICE_ATTR_RO(tty_custom_name);
 
 static struct attribute *dgnc_sysfs_tty_entries[] = {
-	&dev_attr_state.attr,
-	&dev_attr_baud.attr,
-	&dev_attr_msignals.attr,
-	&dev_attr_iflag.attr,
-	&dev_attr_cflag.attr,
-	&dev_attr_oflag.attr,
-	&dev_attr_lflag.attr,
-	&dev_attr_digi_flag.attr,
-	&dev_attr_rxcount.attr,
-	&dev_attr_txcount.attr,
-	&dev_attr_custom_name.attr,
+	&dev_attr_tty_state.attr,
+	&dev_attr_tty_baud.attr,
+	&dev_attr_tty_msignals.attr,
+	&dev_attr_tty_iflag.attr,
+	&dev_attr_tty_cflag.attr,
+	&dev_attr_tty_oflag.attr,
+	&dev_attr_tty_lflag.attr,
+	&dev_attr_tty_digi_flag.attr,
+	&dev_attr_tty_rxcount.attr,
+	&dev_attr_tty_txcount.attr,
+	&dev_attr_tty_custom_name.attr,
 	NULL
 };
 
-static struct attribute_group dgnc_tty_attribute_group = {
+static const struct attribute_group dgnc_tty_attribute_group = {
 	.name = NULL,
 	.attrs = dgnc_sysfs_tty_entries,
 };
diff --git a/drivers/staging/dgnc/dgnc_tty.c b/drivers/staging/dgnc/dgnc_tty.c
index 4eeecc992a02..953d9310fa74 100644
--- a/drivers/staging/dgnc/dgnc_tty.c
+++ b/drivers/staging/dgnc/dgnc_tty.c
@@ -45,7 +45,6 @@
 /*
  * internal variables
  */
-static struct dgnc_board	*dgnc_BoardsByMajor[256];
 static unsigned char		*dgnc_TmpWriteBuf;
 
 /*
@@ -100,7 +99,7 @@ static void dgnc_tty_unthrottle(struct tty_struct *tty);
 static void dgnc_tty_flush_chars(struct tty_struct *tty);
 static void dgnc_tty_flush_buffer(struct tty_struct *tty);
 static void dgnc_tty_hangup(struct tty_struct *tty);
-static int dgnc_set_modem_info(struct tty_struct *tty, unsigned int command,
+static int dgnc_set_modem_info(struct channel_t *ch, unsigned int command,
 			       unsigned int __user *value);
 static int dgnc_get_modem_info(struct channel_t *ch,
 			       unsigned int __user *value);
@@ -186,7 +185,8 @@ int dgnc_tty_register(struct dgnc_board *brd)
 	if (IS_ERR(brd->serial_driver))
 		return PTR_ERR(brd->serial_driver);
 
-	snprintf(brd->serial_name, MAXTTYNAMELEN, "tty_dgnc_%d_", brd->boardnum);
+	snprintf(brd->serial_name, MAXTTYNAMELEN, "tty_dgnc_%d_",
+		 brd->boardnum);
 
 	brd->serial_driver->name = brd->serial_name;
 	brd->serial_driver->name_base = 0;
@@ -203,15 +203,11 @@ int dgnc_tty_register(struct dgnc_board *brd)
 	 */
 	tty_set_operations(brd->serial_driver, &dgnc_tty_ops);
 
-	if (!brd->dgnc_major_serial_registered) {
-		/* Register tty devices */
-		rc = tty_register_driver(brd->serial_driver);
-		if (rc < 0) {
-			dev_dbg(&brd->pdev->dev,
-				"Can't register tty device (%d)\n", rc);
-			goto free_serial_driver;
-		}
-		brd->dgnc_major_serial_registered = true;
+	rc = tty_register_driver(brd->serial_driver);
+	if (rc < 0) {
+		dev_dbg(&brd->pdev->dev,
+			"Can't register tty device (%d)\n", rc);
+		goto free_serial_driver;
 	}
 
 	/*
@@ -246,20 +242,14 @@ int dgnc_tty_register(struct dgnc_board *brd)
 	 */
 	tty_set_operations(brd->print_driver, &dgnc_tty_ops);
 
-	if (!brd->dgnc_major_transparent_print_registered) {
-		/* Register Transparent Print devices */
-		rc = tty_register_driver(brd->print_driver);
-		if (rc < 0) {
-			dev_dbg(&brd->pdev->dev,
-				"Can't register Transparent Print device(%d)\n",
-				rc);
-			goto free_print_driver;
-		}
-		brd->dgnc_major_transparent_print_registered = true;
+	rc = tty_register_driver(brd->print_driver);
+	if (rc < 0) {
+		dev_dbg(&brd->pdev->dev,
+			"Can't register Transparent Print device(%d)\n",
+			rc);
+		goto free_print_driver;
 	}
 
-	dgnc_BoardsByMajor[brd->serial_driver->major] = brd;
-
 	return 0;
 
 free_print_driver:
@@ -272,6 +262,14 @@ free_serial_driver:
 	return rc;
 }
 
+void dgnc_tty_unregister(struct dgnc_board *brd)
+{
+	tty_unregister_driver(brd->print_driver);
+	tty_unregister_driver(brd->serial_driver);
+	put_tty_driver(brd->print_driver);
+	put_tty_driver(brd->serial_driver);
+}
+
 /*
  * dgnc_tty_init()
  *
@@ -378,38 +376,30 @@ void dgnc_tty_post_uninit(void)
 }
 
 /*
- * dgnc_tty_uninit()
+ * dgnc_cleanup_tty()
  *
  * Uninitialize the TTY portion of this driver.  Free all memory and
  * resources.
  */
-void dgnc_tty_uninit(struct dgnc_board *brd)
+void dgnc_cleanup_tty(struct dgnc_board *brd)
 {
 	int i = 0;
 
-	if (brd->dgnc_major_serial_registered) {
-		dgnc_BoardsByMajor[brd->serial_driver->major] = NULL;
-		for (i = 0; i < brd->nasync; i++) {
-			if (brd->channels[i])
-				dgnc_remove_tty_sysfs(brd->channels[i]->
-						      ch_tun.un_sysfs);
-			tty_unregister_device(brd->serial_driver, i);
-		}
-		tty_unregister_driver(brd->serial_driver);
-		brd->dgnc_major_serial_registered = false;
+	for (i = 0; i < brd->nasync; i++) {
+		if (brd->channels[i])
+			dgnc_remove_tty_sysfs(brd->channels[i]->
+					      ch_tun.un_sysfs);
+		tty_unregister_device(brd->serial_driver, i);
 	}
+	tty_unregister_driver(brd->serial_driver);
 
-	if (brd->dgnc_major_transparent_print_registered) {
-		dgnc_BoardsByMajor[brd->print_driver->major] = NULL;
-		for (i = 0; i < brd->nasync; i++) {
-			if (brd->channels[i])
-				dgnc_remove_tty_sysfs(brd->channels[i]->
-						      ch_pun.un_sysfs);
-			tty_unregister_device(brd->print_driver, i);
-		}
-		tty_unregister_driver(brd->print_driver);
-		brd->dgnc_major_transparent_print_registered = false;
+	for (i = 0; i < brd->nasync; i++) {
+		if (brd->channels[i])
+			dgnc_remove_tty_sysfs(brd->channels[i]->
+					      ch_pun.un_sysfs);
+		tty_unregister_device(brd->print_driver, i);
 	}
+	tty_unregister_driver(brd->print_driver);
 
 	put_tty_driver(brd->serial_driver);
 	put_tty_driver(brd->print_driver);
@@ -640,19 +630,12 @@ exit_unlock:
  ************************************************************************/
 void dgnc_carrier(struct channel_t *ch)
 {
-	struct dgnc_board *bd;
-
 	int virt_carrier = 0;
 	int phys_carrier = 0;
 
 	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
 		return;
 
-	bd = ch->ch_bd;
-
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return;
-
 	if (ch->ch_mistat & UART_MSR_DCD)
 		phys_carrier = 1;
 
@@ -947,6 +930,24 @@ void dgnc_wakeup_writes(struct channel_t *ch)
 	spin_unlock_irqrestore(&ch->ch_lock, flags);
 }
 
+struct dgnc_board *find_board_by_major(unsigned int major)
+{
+	int i;
+
+	for (i = 0; i < MAXBOARDS; i++) {
+		struct dgnc_board *brd = dgnc_board[i];
+
+		if (!brd)
+			return NULL;
+
+		if (major == brd->serial_driver->major ||
+		    major == brd->print_driver->major)
+			return brd;
+	}
+
+	return NULL;
+}
+
 /************************************************************************
  *
  * TTY Entry points and helper functions
@@ -976,7 +977,7 @@ static int dgnc_tty_open(struct tty_struct *tty, struct file *file)
 		return -ENXIO;
 
 	/* Get board pointer from our array of majors we have allocated */
-	brd = dgnc_BoardsByMajor[major];
+	brd = find_board_by_major(major);
 	if (!brd)
 		return -ENXIO;
 
@@ -1172,17 +1173,12 @@ static int dgnc_block_til_ready(struct tty_struct *tty,
 				struct channel_t *ch)
 {
 	int retval = 0;
-	struct un_t *un = NULL;
+	struct un_t *un = tty->driver_data;
 	unsigned long flags;
 	uint	old_flags = 0;
 	int	sleep_on_un_flags = 0;
 
-	if (!tty || tty->magic != TTY_MAGIC || !file || !ch ||
-	    ch->magic != DGNC_CHANNEL_MAGIC)
-		return -ENXIO;
-
-	un = tty->driver_data;
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
+	if (!file)
 		return -ENXIO;
 
 	spin_lock_irqsave(&ch->ch_lock, flags);
@@ -1301,15 +1297,9 @@ static int dgnc_block_til_ready(struct tty_struct *tty,
  */
 static void dgnc_tty_hangup(struct tty_struct *tty)
 {
-	struct un_t	*un;
-
 	if (!tty || tty->magic != TTY_MAGIC)
 		return;
 
-	un = tty->driver_data;
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return;
-
 	/* flush the transmit queues */
 	dgnc_tty_flush_buffer(tty);
 }
@@ -1510,18 +1500,8 @@ static int dgnc_tty_chars_in_buffer(struct tty_struct *tty)
  * returns the new bytes_available.  This only affects printer
  * output.
  */
-static int dgnc_maxcps_room(struct tty_struct *tty, int bytes_available)
+static int dgnc_maxcps_room(struct channel_t *ch, int bytes_available)
 {
-	struct un_t *un = tty->driver_data;
-	struct channel_t *ch = un->un_ch;
-
-	/*
-	 * If its not the Transparent print device, return
-	 * the full data amount.
-	 */
-	if (un->un_type != DGNC_PRINT)
-		return bytes_available;
-
 	if (ch->ch_digi.digi_maxcps > 0 && ch->ch_digi.digi_bufsize > 0) {
 		int cps_limit = 0;
 		unsigned long current_time = jiffies;
@@ -1585,7 +1565,8 @@ static int dgnc_tty_write_room(struct tty_struct *tty)
 		ret += WQUEUESIZE;
 
 	/* Limit printer to maxcps */
-	ret = dgnc_maxcps_room(tty, ret);
+	if (un->un_type != DGNC_PRINT)
+		ret = dgnc_maxcps_room(ch, ret);
 
 	/*
 	 * If we are printer device, leave space for
@@ -1677,7 +1658,8 @@ static int dgnc_tty_write(struct tty_struct *tty,
 	 * Limit printer output to maxcps overall, with bursts allowed
 	 * up to bufsize characters.
 	 */
-	bufcount = dgnc_maxcps_room(tty, bufcount);
+	if (un->un_type != DGNC_PRINT)
+		bufcount = dgnc_maxcps_room(ch, bufcount);
 
 	/*
 	 * Take minimum of what the user wants to send, and the
@@ -1984,7 +1966,7 @@ static void dgnc_tty_send_xchar(struct tty_struct *tty, char c)
 static inline int dgnc_get_mstat(struct channel_t *ch)
 {
 	unsigned char mstat;
-	int result = -EIO;
+	int result = 0;
 	unsigned long flags;
 
 	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
@@ -1996,8 +1978,6 @@ static inline int dgnc_get_mstat(struct channel_t *ch)
 
 	spin_unlock_irqrestore(&ch->ch_lock, flags);
 
-	result = 0;
-
 	if (mstat & UART_MCR_DTR)
 		result |= TIOCM_DTR;
 	if (mstat & UART_MCR_RTS)
@@ -2028,32 +2008,14 @@ static int dgnc_get_modem_info(struct channel_t *ch,
  *
  * Set modem signals, called by ld.
  */
-static int dgnc_set_modem_info(struct tty_struct *tty,
+static int dgnc_set_modem_info(struct channel_t *ch,
 			       unsigned int command,
 			       unsigned int __user *value)
 {
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
 	int ret = -ENXIO;
 	unsigned int arg = 0;
 	unsigned long flags;
 
-	if (!tty || tty->magic != TTY_MAGIC)
-		return ret;
-
-	un = tty->driver_data;
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return ret;
-
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return ret;
-
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return ret;
-
 	ret = get_user(arg, value);
 	if (ret)
 		return ret;
@@ -2593,9 +2555,8 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 
 		spin_unlock_irqrestore(&ch->ch_lock, flags);
 
-		rc = put_user(C_CLOCAL(tty) ? 1 : 0,
-			      (unsigned long __user *)arg);
-		return rc;
+		return put_user(C_CLOCAL(tty) ? 1 : 0,
+				(unsigned long __user *)arg);
 
 	case TIOCSSOFTCAR:
 
@@ -2620,7 +2581,7 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 	case TIOCMBIC:
 	case TIOCMSET:
 		spin_unlock_irqrestore(&ch->ch_lock, flags);
-		return dgnc_set_modem_info(tty, cmd, uarg);
+		return dgnc_set_modem_info(ch, cmd, uarg);
 
 		/*
 		 * Here are any additional ioctl's that we want to implement
@@ -2766,8 +2727,8 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 
 	case DIGI_GETCUSTOMBAUD:
 		spin_unlock_irqrestore(&ch->ch_lock, flags);
-		rc = put_user(ch->ch_custom_speed, (unsigned int __user *)arg);
-		return rc;
+		return put_user(ch->ch_custom_speed,
+				(unsigned int __user *)arg);
 
 	case DIGI_SETCUSTOMBAUD:
 	{
@@ -2853,8 +2814,7 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 			events |= (EV_IPU | EV_IPS);
 
 		spin_unlock_irqrestore(&ch->ch_lock, flags);
-		rc = put_user(events, (unsigned int __user *)arg);
-		return rc;
+		return put_user(events, (unsigned int __user *)arg);
 	}
 
 	/*
diff --git a/drivers/staging/dgnc/dgnc_tty.h b/drivers/staging/dgnc/dgnc_tty.h
index 21d3369b875c..24c9a412211e 100644
--- a/drivers/staging/dgnc/dgnc_tty.h
+++ b/drivers/staging/dgnc/dgnc_tty.h
@@ -19,12 +19,13 @@
 #include "dgnc_driver.h"
 
 int	dgnc_tty_register(struct dgnc_board *brd);
+void dgnc_tty_unregister(struct dgnc_board *brd);
 
 int	dgnc_tty_preinit(void);
 int     dgnc_tty_init(struct dgnc_board *);
 
 void	dgnc_tty_post_uninit(void);
-void	dgnc_tty_uninit(struct dgnc_board *);
+void	dgnc_cleanup_tty(struct dgnc_board *);
 
 void	dgnc_input(struct channel_t *ch);
 void	dgnc_carrier(struct channel_t *ch);
diff --git a/drivers/staging/emxx_udc/emxx_udc.c b/drivers/staging/emxx_udc/emxx_udc.c
index 3b56b2826263..c3e298843b43 100644
--- a/drivers/staging/emxx_udc/emxx_udc.c
+++ b/drivers/staging/emxx_udc/emxx_udc.c
@@ -131,7 +131,6 @@ static void _nbu2ss_dump_register(struct nbu2ss_udc *udc)
 		reg_data =  _nbu2ss_readl(
 			(u32 *)IO_ADDRESS(USB_BASE_ADDRESS + i + 12));
 		dev_dbg(&udc->dev, " %08x\n", (int)reg_data);
-
 	}
 
 	spin_lock(&udc->lock);
@@ -478,9 +477,9 @@ static void _nbu2ss_dma_map_single(
 )
 {
 	if (req->req.dma == DMA_ADDR_INVALID) {
-		if (req->unaligned)
+		if (req->unaligned) {
 			req->req.dma = ep->phys_buf;
-		else {
+		} else {
 			req->req.dma = dma_map_single(
 				udc->gadget.dev.parent,
 				req->req.buf,
@@ -1236,9 +1235,9 @@ static int _nbu2ss_start_transfer(
 	req->dma_flag = FALSE;
 	req->div_len = 0;
 
-	if (req->req.length == 0)
+	if (req->req.length == 0) {
 		req->zero = false;
-	else {
+	} else {
 		if ((req->req.length % ep->ep.maxpacket) == 0)
 			req->zero = req->req.zero;
 		else
@@ -1941,9 +1940,9 @@ static void _nbu2ss_ep_done(
 	if (likely(req->req.status == -EINPROGRESS))
 		req->req.status = status;
 
-	if (ep->stalled)
+	if (ep->stalled) {
 		_nbu2ss_epn_set_stall(udc, ep);
-	else {
+	} else {
 		if (!list_empty(&ep->queue))
 			_nbu2ss_restert_transfer(ep);
 	}
@@ -2264,9 +2263,7 @@ static int _nbu2ss_enable_controller(struct nbu2ss_udc *udc)
 	if (udc->udc_enabled)
 		return 0;
 
-	/*
-		Reset
-	*/
+	/* Reset */
 	_nbu2ss_bitset(&udc->p_regs->EPCTR, (DIRPD | EPC_RST));
 	udelay(EPC_RST_DISABLE_TIME);	/* 1us wait */
 
@@ -2476,8 +2473,9 @@ static irqreturn_t _nbu2ss_udc_irq(int irq, void *_udc)
 			_nbu2ss_writel(&preg->USB_INT_STA, ~USB_INT_STA_RW);
 			_nbu2ss_writel(&preg->USB_INT_ENA, 0);
 			status = 0;
-		} else
+		} else {
 			status = _nbu2ss_readl(&preg->USB_INT_STA);
+		}
 
 		if (status == 0)
 			break;
diff --git a/drivers/staging/emxx_udc/emxx_udc.h b/drivers/staging/emxx_udc/emxx_udc.h
index 39769e3a801c..789bfb97143c 100644
--- a/drivers/staging/emxx_udc/emxx_udc.h
+++ b/drivers/staging/emxx_udc/emxx_udc.h
@@ -586,7 +586,7 @@ struct nbu2ss_udc {
 	unsigned		remote_wakeup:1;
 	unsigned		udc_enabled:1;
 
-	unsigned		mA;
+	unsigned int		mA;
 
 	u32		curr_config;	/* Current Configuration Number */
 
diff --git a/drivers/staging/fbtft/fb_agm1264k-fl.c b/drivers/staging/fbtft/fb_agm1264k-fl.c
index 82b46cd27ca7..7561385761e9 100644
--- a/drivers/staging/fbtft/fb_agm1264k-fl.c
+++ b/drivers/staging/fbtft/fb_agm1264k-fl.c
@@ -350,8 +350,8 @@ static int write_vmem(struct fbtft_par *par, size_t offset, size_t len)
 				}
 		}
 
-	 /* 1 string = 2 pages */
-	 for (y = addr_win.ys_page; y <= addr_win.ye_page; ++y) {
+	/* 1 string = 2 pages */
+	for (y = addr_win.ys_page; y <= addr_win.ye_page; ++y) {
 		/* left half of display */
 		if (addr_win.xs < par->info->var.xres / 2) {
 			construct_line_bitmap(par, buf, convert_buf,
diff --git a/drivers/staging/fbtft/fb_ili9320.c b/drivers/staging/fbtft/fb_ili9320.c
index 6ff222d6d6d6..278e4c7e95e5 100644
--- a/drivers/staging/fbtft/fb_ili9320.c
+++ b/drivers/staging/fbtft/fb_ili9320.c
@@ -29,7 +29,7 @@
 #define DEFAULT_GAMMA	"07 07 6 0 0 0 5 5 4 0\n" \
 			"07 08 4 7 5 1 2 0 7 7"
 
-static unsigned read_devicecode(struct fbtft_par *par)
+static unsigned int read_devicecode(struct fbtft_par *par)
 {
 	int ret;
 	u8 rxbuf[8] = {0, };
@@ -41,7 +41,7 @@ static unsigned read_devicecode(struct fbtft_par *par)
 
 static int init_display(struct fbtft_par *par)
 {
-	unsigned devcode;
+	unsigned int devcode;
 
 	par->fbtftops.reset(par);
 
diff --git a/drivers/staging/fbtft/fb_ili9325.c b/drivers/staging/fbtft/fb_ili9325.c
index fdf98d37550e..c31e2e051d4a 100644
--- a/drivers/staging/fbtft/fb_ili9325.c
+++ b/drivers/staging/fbtft/fb_ili9325.c
@@ -32,26 +32,26 @@
 #define DEFAULT_GAMMA	"0F 00 7 2 0 0 6 5 4 1\n" \
 			"04 16 2 7 6 3 2 1 7 7"
 
-static unsigned bt = 6; /* VGL=Vci*4 , VGH=Vci*4 */
+static unsigned int bt = 6; /* VGL=Vci*4 , VGH=Vci*4 */
 module_param(bt, uint, 0);
 MODULE_PARM_DESC(bt, "Sets the factor used in the step-up circuits");
 
-static unsigned vc = 0x03; /* Vci1=Vci*0.80 */
+static unsigned int vc = 0x03; /* Vci1=Vci*0.80 */
 module_param(vc, uint, 0);
 MODULE_PARM_DESC(vc,
 "Sets the ratio factor of Vci to generate the reference voltages Vci1");
 
-static unsigned vrh = 0x0d; /* VREG1OUT=Vci*1.85 */
+static unsigned int vrh = 0x0d; /* VREG1OUT=Vci*1.85 */
 module_param(vrh, uint, 0);
 MODULE_PARM_DESC(vrh,
 "Set the amplifying rate (1.6 ~ 1.9) of Vci applied to output the VREG1OUT");
 
-static unsigned vdv = 0x12; /* VCOMH amplitude=VREG1OUT*0.98 */
+static unsigned int vdv = 0x12; /* VCOMH amplitude=VREG1OUT*0.98 */
 module_param(vdv, uint, 0);
 MODULE_PARM_DESC(vdv,
 "Select the factor of VREG1OUT to set the amplitude of Vcom");
 
-static unsigned vcm = 0x0a; /* VCOMH=VREG1OUT*0.735 */
+static unsigned int vcm = 0x0a; /* VCOMH=VREG1OUT*0.735 */
 module_param(vcm, uint, 0);
 MODULE_PARM_DESC(vcm, "Set the internal VcomH voltage");
 
diff --git a/drivers/staging/fbtft/fb_pcd8544.c b/drivers/staging/fbtft/fb_pcd8544.c
index a6b43323f29a..a4710dc067ef 100644
--- a/drivers/staging/fbtft/fb_pcd8544.c
+++ b/drivers/staging/fbtft/fb_pcd8544.c
@@ -32,11 +32,11 @@
 #define TXBUFLEN       (84 * 6)
 #define DEFAULT_GAMMA  "40" /* gamma controls the contrast in this driver */
 
-static unsigned tc;
+static unsigned int tc;
 module_param(tc, uint, 0);
 MODULE_PARM_DESC(tc, "TC[1:0] Temperature coefficient: 0-3 (default: 0)");
 
-static unsigned bs = 4;
+static unsigned int bs = 4;
 module_param(bs, uint, 0);
 MODULE_PARM_DESC(bs, "BS[2:0] Bias voltage level: 0-7 (default: 4)");
 
diff --git a/drivers/staging/fbtft/fb_s6d02a1.c b/drivers/staging/fbtft/fb_s6d02a1.c
index 3113355062fc..774b0ff69e6d 100644
--- a/drivers/staging/fbtft/fb_s6d02a1.c
+++ b/drivers/staging/fbtft/fb_s6d02a1.c
@@ -113,12 +113,14 @@ static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
 #define MV BIT(5)
 static int set_var(struct fbtft_par *par)
 {
-	/* Memory data access control (0x36h)
-	     RGB/BGR:
-		1. Mode selection pin SRGB
-			RGB H/W pin for color filter setting: 0=RGB, 1=BGR
-		2. MADCTL RGB bit
-			RGB-BGR ORDER color filter panel: 0=RGB, 1=BGR */
+	/*
+	 * Memory data access control (0x36h)
+	 * RGB/BGR:
+	 *	1. Mode selection pin SRGB
+	 *		RGB H/W pin for color filter setting: 0=RGB, 1=BGR
+	 *	2. MADCTL RGB bit
+	 *		RGB-BGR ORDER color filter panel: 0=RGB, 1=BGR
+	 */
 	switch (par->info->var.rotate) {
 	case 0:
 		write_reg(par, MIPI_DCS_SET_ADDRESS_MODE,
diff --git a/drivers/staging/fbtft/fb_s6d1121.c b/drivers/staging/fbtft/fb_s6d1121.c
index d6ae76b318ad..9b1d70b218df 100644
--- a/drivers/staging/fbtft/fb_s6d1121.c
+++ b/drivers/staging/fbtft/fb_s6d1121.c
@@ -125,10 +125,10 @@ static int set_var(struct fbtft_par *par)
 }
 
 /*
-  Gamma string format:
-    PKP0 PKP1 PKP2 PKP3 PKP4 PKP5 PKP6 PKP7 PKP8 PKP9 PKP10 PKP11 VRP0 VRP1
-    PKN0 PKN1 PKN2 PKN3 PKN4 PKN5 PKN6 PKN7 PRN8 PRN9 PRN10 PRN11 VRN0 VRN1
-*/
+ * Gamma string format:
+ * PKP0 PKP1 PKP2 PKP3 PKP4 PKP5 PKP6 PKP7 PKP8 PKP9 PKP10 PKP11 VRP0 VRP1
+ * PKN0 PKN1 PKN2 PKN3 PKN4 PKN5 PKN6 PKN7 PRN8 PRN9 PRN10 PRN11 VRN0 VRN1
+ */
 #define CURVE(num, idx)  curves[num * par->gamma.num_values + idx]
 static int set_gamma(struct fbtft_par *par, unsigned long *curves)
 {
diff --git a/drivers/staging/fbtft/fb_ssd1289.c b/drivers/staging/fbtft/fb_ssd1289.c
index 1162c08613fa..25f9fbe1e76f 100644
--- a/drivers/staging/fbtft/fb_ssd1289.c
+++ b/drivers/staging/fbtft/fb_ssd1289.c
@@ -29,7 +29,7 @@
 #define DEFAULT_GAMMA	"02 03 2 5 7 7 4 2 4 2\n" \
 			"02 03 2 5 7 5 4 2 4 2"
 
-static unsigned reg11 = 0x6040;
+static unsigned int reg11 = 0x6040;
 module_param(reg11, uint, 0);
 MODULE_PARM_DESC(reg11, "Register 11h value");
 
@@ -131,10 +131,10 @@ static int set_var(struct fbtft_par *par)
 }
 
 /*
-  Gamma string format:
-    VRP0 VRP1 PRP0 PRP1 PKP0 PKP1 PKP2 PKP3 PKP4 PKP5
-    VRN0 VRN1 PRN0 PRN1 PKN0 PKN1 PKN2 PKN3 PKN4 PKN5
-*/
+ * Gamma string format:
+ * VRP0 VRP1 PRP0 PRP1 PKP0 PKP1 PKP2 PKP3 PKP4 PKP5
+ * VRN0 VRN1 PRN0 PRN1 PKN0 PKN1 PKN2 PKN3 PKN4 PKN5
+ */
 #define CURVE(num, idx)  curves[num * par->gamma.num_values + idx]
 static int set_gamma(struct fbtft_par *par, unsigned long *curves)
 {
diff --git a/drivers/staging/fbtft/fb_ssd1306.c b/drivers/staging/fbtft/fb_ssd1306.c
index e0b34a42c9c6..80fc57029fee 100644
--- a/drivers/staging/fbtft/fb_ssd1306.c
+++ b/drivers/staging/fbtft/fb_ssd1306.c
@@ -27,15 +27,15 @@
 #define HEIGHT		64
 
 /*
-  write_reg() caveat:
-
-     This doesn't work because D/C has to be LOW for both values:
-       write_reg(par, val1, val2);
-
-     Do it like this:
-       write_reg(par, val1);
-       write_reg(par, val2);
-*/
+ * write_reg() caveat:
+ *
+ * This doesn't work because D/C has to be LOW for both values:
+ * write_reg(par, val1, val2);
+ *
+ * Do it like this:
+ * write_reg(par, val1);
+ * write_reg(par, val2);
+ */
 
 /* Init sequence taken from the Adafruit SSD1306 Arduino library */
 static int init_display(struct fbtft_par *par)
@@ -113,8 +113,9 @@ static int init_display(struct fbtft_par *par)
 	write_reg(par, 0xA4);
 
 	/* Set Normal Display
-	   0 in RAM: OFF in display panel
-	   1 in RAM: ON in display panel */
+	 * 0 in RAM: OFF in display panel
+	 * 1 in RAM: ON in display panel
+	 */
 	write_reg(par, 0xA6);
 
 	/* Set Display ON */
diff --git a/drivers/staging/fbtft/fb_ssd1331.c b/drivers/staging/fbtft/fb_ssd1331.c
index bd294f886c5f..1d74ac1343a8 100644
--- a/drivers/staging/fbtft/fb_ssd1331.c
+++ b/drivers/staging/fbtft/fb_ssd1331.c
@@ -102,26 +102,26 @@ static void write_reg8_bus8(struct fbtft_par *par, int len, ...)
 }
 
 /*
-	Grayscale Lookup Table
-	GS1 - GS63
-	The driver Gamma curve contains the relative values between the entries
-	in the Lookup table.
-
-	From datasheet:
-	8.8 Gray Scale Decoder
-
-		there are total 180 Gamma Settings (Setting 0 to Setting 180)
-		available for the Gray Scale table.
-
-		The gray scale is defined in incremental way, with reference
-		to the length of previous table entry:
-			Setting of GS1 has to be >= 0
-			Setting of GS2 has to be > Setting of GS1 +1
-			Setting of GS3 has to be > Setting of GS2 +1
-			:
-			Setting of GS63 has to be > Setting of GS62 +1
-
-*/
+ * Grayscale Lookup Table
+ * GS1 - GS63
+ * The driver Gamma curve contains the relative values between the entries
+ * in the Lookup table.
+ *
+ * From datasheet:
+ * 8.8 Gray Scale Decoder
+ *
+ * there are total 180 Gamma Settings (Setting 0 to Setting 180)
+ * available for the Gray Scale table.
+ *
+ * The gray scale is defined in incremental way, with reference
+ * to the length of previous table entry:
+ * Setting of GS1 has to be >= 0
+ * Setting of GS2 has to be > Setting of GS1 +1
+ * Setting of GS3 has to be > Setting of GS2 +1
+ * :
+ * Setting of GS63 has to be > Setting of GS62 +1
+ *
+ */
 static int set_gamma(struct fbtft_par *par, unsigned long *curves)
 {
 	unsigned long tmp[GAMMA_NUM * GAMMA_LEN];
diff --git a/drivers/staging/fbtft/fb_ssd1351.c b/drivers/staging/fbtft/fb_ssd1351.c
index cef33e439f46..200aa9ba98f9 100644
--- a/drivers/staging/fbtft/fb_ssd1351.c
+++ b/drivers/staging/fbtft/fb_ssd1351.c
@@ -25,8 +25,8 @@ static void register_onboard_backlight(struct fbtft_par *par);
 
 static int init_display(struct fbtft_par *par)
 {
-	if (par->pdata
-		&& par->pdata->display.backlight == FBTFT_ONBOARD_BACKLIGHT) {
+	if (par->pdata &&
+	    par->pdata->display.backlight == FBTFT_ONBOARD_BACKLIGHT) {
 		/* module uses onboard GPIO for panel power */
 		par->fbtftops.register_backlight = register_onboard_backlight;
 	}
@@ -66,7 +66,7 @@ static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
 
 static int set_var(struct fbtft_par *par)
 {
-	unsigned remap;
+	unsigned int remap;
 
 	if (par->fbtftops.init_display != init_display) {
 		/* don't risk messing up register A0h */
@@ -80,10 +80,10 @@ static int set_var(struct fbtft_par *par)
 
 	switch (par->info->var.rotate) {
 	case 0:
-		write_reg(par, 0xA0, remap | 0x00 | 1<<4);
+		write_reg(par, 0xA0, remap | 0x00 | 1 << 4);
 		break;
 	case 270:
-		write_reg(par, 0xA0, remap | 0x03 | 1<<4);
+		write_reg(par, 0xA0, remap | 0x03 | 1 << 4);
 		break;
 	case 180:
 		write_reg(par, 0xA0, remap | 0x02);
@@ -189,8 +189,8 @@ static int update_onboard_backlight(struct backlight_device *bd)
 		"%s: power=%d, fb_blank=%d\n",
 		__func__, bd->props.power, bd->props.fb_blank);
 
-	on = (bd->props.power == FB_BLANK_UNBLANK)
-		&& (bd->props.fb_blank == FB_BLANK_UNBLANK);
+	on = (bd->props.power == FB_BLANK_UNBLANK) &&
+	     (bd->props.fb_blank == FB_BLANK_UNBLANK);
 	/* Onboard backlight connected to GPIO0 on SSD1351, GPIO1 unused */
 	write_reg(par, 0xB5, on ? 0x03 : 0x02);
 
diff --git a/drivers/staging/fbtft/fb_st7735r.c b/drivers/staging/fbtft/fb_st7735r.c
index c5e51fe1aad5..6670f2bb62ec 100644
--- a/drivers/staging/fbtft/fb_st7735r.c
+++ b/drivers/staging/fbtft/fb_st7735r.c
@@ -33,35 +33,43 @@ static int default_init_sequence[] = {
 	-2, 500,                               /* delay */
 
 	/* FRMCTR1 - frame rate control: normal mode
-	     frame rate = fosc / (1 x 2 + 40) * (LINE + 2C + 2D) */
+	 * frame rate = fosc / (1 x 2 + 40) * (LINE + 2C + 2D)
+	 */
 	-1, 0xB1, 0x01, 0x2C, 0x2D,
 
 	/* FRMCTR2 - frame rate control: idle mode
-	     frame rate = fosc / (1 x 2 + 40) * (LINE + 2C + 2D) */
+	 * frame rate = fosc / (1 x 2 + 40) * (LINE + 2C + 2D)
+	 */
 	-1, 0xB2, 0x01, 0x2C, 0x2D,
 
 	/* FRMCTR3 - frame rate control - partial mode
-	     dot inversion mode, line inversion mode */
+	 * dot inversion mode, line inversion mode
+	 */
 	-1, 0xB3, 0x01, 0x2C, 0x2D, 0x01, 0x2C, 0x2D,
 
 	/* INVCTR - display inversion control
-	     no inversion */
+	 * no inversion
+	 */
 	-1, 0xB4, 0x07,
 
 	/* PWCTR1 - Power Control
-	     -4.6V, AUTO mode */
+	 * -4.6V, AUTO mode
+	 */
 	-1, 0xC0, 0xA2, 0x02, 0x84,
 
 	/* PWCTR2 - Power Control
-	     VGH25 = 2.4C VGSEL = -10 VGH = 3 * AVDD */
+	 * VGH25 = 2.4C VGSEL = -10 VGH = 3 * AVDD
+	 */
 	-1, 0xC1, 0xC5,
 
 	/* PWCTR3 - Power Control
-	     Opamp current small, Boost frequency */
+	 * Opamp current small, Boost frequency
+	 */
 	-1, 0xC2, 0x0A, 0x00,
 
 	/* PWCTR4 - Power Control
-	     BCLK/2, Opamp current small & Medium low */
+	 * BCLK/2, Opamp current small & Medium low
+	 */
 	-1, 0xC3, 0x8A, 0x2A,
 
 	/* PWCTR5 - Power Control */
@@ -101,11 +109,12 @@ static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
 static int set_var(struct fbtft_par *par)
 {
 	/* MADCTL - Memory data access control
-	     RGB/BGR:
-	     1. Mode selection pin SRGB
-		RGB H/W pin for color filter setting: 0=RGB, 1=BGR
-	     2. MADCTL RGB bit
-		RGB-BGR ORDER color filter panel: 0=RGB, 1=BGR */
+	 * RGB/BGR:
+	 * 1. Mode selection pin SRGB
+	 *    RGB H/W pin for color filter setting: 0=RGB, 1=BGR
+	 * 2. MADCTL RGB bit
+	 *    RGB-BGR ORDER color filter panel: 0=RGB, 1=BGR
+	 */
 	switch (par->info->var.rotate) {
 	case 0:
 		write_reg(par, MIPI_DCS_SET_ADDRESS_MODE,
@@ -129,10 +138,10 @@ static int set_var(struct fbtft_par *par)
 }
 
 /*
-  Gamma string format:
-    VRF0P VOS0P PK0P PK1P PK2P PK3P PK4P PK5P PK6P PK7P PK8P PK9P SELV0P SELV1P SELV62P SELV63P
-    VRF0N VOS0N PK0N PK1N PK2N PK3N PK4N PK5N PK6N PK7N PK8N PK9N SELV0N SELV1N SELV62N SELV63N
-*/
+ * Gamma string format:
+ * VRF0P VOS0P PK0P PK1P PK2P PK3P PK4P PK5P PK6P PK7P PK8P PK9P SELV0P SELV1P SELV62P SELV63P
+ * VRF0N VOS0N PK0N PK1N PK2N PK3N PK4N PK5N PK6N PK7N PK8N PK9N SELV0N SELV1N SELV62N SELV63N
+ */
 #define CURVE(num, idx)  curves[num * par->gamma.num_values + idx]
 static int set_gamma(struct fbtft_par *par, unsigned long *curves)
 {
diff --git a/drivers/staging/fbtft/fb_tls8204.c b/drivers/staging/fbtft/fb_tls8204.c
index 2183f98c8315..ea2ddacb9468 100644
--- a/drivers/staging/fbtft/fb_tls8204.c
+++ b/drivers/staging/fbtft/fb_tls8204.c
@@ -35,7 +35,7 @@
 /* gamma is used to control contrast in this driver */
 #define DEFAULT_GAMMA	"40"
 
-static unsigned bs = 4;
+static unsigned int bs = 4;
 module_param(bs, uint, 0);
 MODULE_PARM_DESC(bs, "BS[2:0] Bias voltage level: 0-7 (default: 4)");
 
@@ -44,21 +44,21 @@ static int init_display(struct fbtft_par *par)
 	par->fbtftops.reset(par);
 
 	/* Enter extended command mode */
-	write_reg(par, 0x21); /* 5:1  1
-				 2:0  PD - Powerdown control: chip is active
-				 1:0  V  - Entry mode: horizontal addressing
-				 0:1  H  - Extended instruction set control:
-						extended
-			      */
+	write_reg(par, 0x21);	/* 5:1  1
+				 * 2:0  PD - Powerdown control: chip is active
+				 * 1:0  V  - Entry mode: horizontal addressing
+				 * 0:1  H  - Extended instruction set control:
+				 *	     extended
+				 */
 
 	/* H=1 Bias system */
-	write_reg(par, 0x10 | (bs & 0x7)); /*
-				 4:1  1
-				 3:0  0
-				 2:x  BS2 - Bias System
-				 1:x  BS1
-				 0:x  BS0
-			      */
+	write_reg(par, 0x10 | (bs & 0x7));
+				/* 4:1  1
+				 * 3:0  0
+				 * 2:x  BS2 - Bias System
+				 * 1:x  BS1
+				 * 0:x  BS0
+				 */
 
 	/* Set the address of the first display line. */
 	write_reg(par, 0x04 | (64 >> 6));
@@ -68,12 +68,12 @@ static int init_display(struct fbtft_par *par)
 	write_reg(par, 0x20);
 
 	/* H=0 Display control */
-	write_reg(par, 0x08 | 4); /*
-				 3:1  1
-				 2:1  D  - DE: 10=normal mode
-				 1:0  0
-				 0:0  E
-			      */
+	write_reg(par, 0x08 | 4);
+				/* 3:1  1
+				 * 2:1  D - DE: 10=normal mode
+				 * 1:0  0
+				 * 0:0  E
+				 */
 
 	return 0;
 }
@@ -81,15 +81,15 @@ static int init_display(struct fbtft_par *par)
 static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye)
 {
 	/* H=0 Set X address of RAM */
-	write_reg(par, 0x80); /* 7:1  1
-				 6-0: X[6:0] - 0x00
-			      */
+	write_reg(par, 0x80);	/* 7:1  1
+				 * 6-0: X[6:0] - 0x00
+				 */
 
 	/* H=0 Set Y address of RAM */
-	write_reg(par, 0x40); /* 7:0  0
-				 6:1  1
-				 2-0: Y[2:0] - 0x0
-			      */
+	write_reg(par, 0x40);	/* 7:0  0
+				 * 6:1  1
+				 * 2-0: Y[2:0] - 0x0
+				 */
 }
 
 static int write_vmem(struct fbtft_par *par, size_t offset, size_t len)
@@ -100,8 +100,9 @@ static int write_vmem(struct fbtft_par *par, size_t offset, size_t len)
 
 	for (y = 0; y < HEIGHT / 8; y++) {
 		u8 *buf = par->txbuf.buf;
-		/* The display is 102x68 but the LCD is 84x48.  Set
-		   the write pointer at the start of each row. */
+		/* The display is 102x68 but the LCD is 84x48.
+		 * Set the write pointer at the start of each row.
+		 */
 		gpio_set_value(par->gpio.dc, 0);
 		write_reg(par, 0x80 | 0);
 		write_reg(par, 0x40 | y);
diff --git a/drivers/staging/fbtft/fb_uc1611.c b/drivers/staging/fbtft/fb_uc1611.c
index e87401aacfb3..b33b73f17da4 100644
--- a/drivers/staging/fbtft/fb_uc1611.c
+++ b/drivers/staging/fbtft/fb_uc1611.c
@@ -41,30 +41,30 @@
  */
 
 /* BR -> actual ratio: 0-3 -> 5, 10, 11, 13 */
-static unsigned ratio = 2;
+static unsigned int ratio = 2;
 module_param(ratio, uint, 0);
 MODULE_PARM_DESC(ratio, "BR[1:0] Bias voltage ratio: 0-3 (default: 2)");
 
-static unsigned gain = 3;
+static unsigned int gain = 3;
 module_param(gain, uint, 0);
 MODULE_PARM_DESC(gain, "GN[1:0] Bias voltage gain: 0-3 (default: 3)");
 
-static unsigned pot = 16;
+static unsigned int pot = 16;
 module_param(pot, uint, 0);
 MODULE_PARM_DESC(pot, "PM[6:0] Bias voltage pot.: 0-63 (default: 16)");
 
 /* TC -> % compensation per deg C: 0-3 -> -.05, -.10, -.015, -.20 */
-static unsigned temp;
+static unsigned int temp;
 module_param(temp, uint, 0);
 MODULE_PARM_DESC(temp, "TC[1:0] Temperature compensation: 0-3 (default: 0)");
 
 /* PC[1:0] -> LCD capacitance: 0-3 -> <20nF, 20-28 nF, 29-40 nF, 40-56 nF */
-static unsigned load = 1;
+static unsigned int load = 1;
 module_param(load, uint, 0);
 MODULE_PARM_DESC(load, "PC[1:0] Panel Loading: 0-3 (default: 1)");
 
 /* PC[3:2] -> V_LCD: 0, 1, 3 -> ext., int. with ratio = 5, int. standard */
-static unsigned pump = 3;
+static unsigned int pump = 3;
 module_param(pump, uint, 0);
 MODULE_PARM_DESC(pump, "PC[3:2] Pump control: 0,1,3 (default: 3)");
 
diff --git a/drivers/staging/fbtft/fb_watterott.c b/drivers/staging/fbtft/fb_watterott.c
index f8cb610a7b69..a52e28a48825 100644
--- a/drivers/staging/fbtft/fb_watterott.c
+++ b/drivers/staging/fbtft/fb_watterott.c
@@ -67,7 +67,7 @@ static void write_reg8_bus8(struct fbtft_par *par, int len, ...)
 
 static int write_vmem(struct fbtft_par *par, size_t offset, size_t len)
 {
-	unsigned start_line, end_line;
+	unsigned int start_line, end_line;
 	u16 *vmem16 = (u16 *)(par->info->screen_buffer + offset);
 	u16 *pos = par->txbuf.buf + 1;
 	u16 *buf16 = par->txbuf.buf + 10;
@@ -104,7 +104,7 @@ static int write_vmem(struct fbtft_par *par, size_t offset, size_t len)
 
 static int write_vmem_8bit(struct fbtft_par *par, size_t offset, size_t len)
 {
-	unsigned start_line, end_line;
+	unsigned int start_line, end_line;
 	u16 *vmem16 = (u16 *)(par->info->screen_buffer + offset);
 	u16 *pos = par->txbuf.buf + 1;
 	u8 *buf8 = par->txbuf.buf + 10;
@@ -137,7 +137,7 @@ static int write_vmem_8bit(struct fbtft_par *par, size_t offset, size_t len)
 	return 0;
 }
 
-static unsigned firmware_version(struct fbtft_par *par)
+static unsigned int firmware_version(struct fbtft_par *par)
 {
 	u8 rxbuf[4] = {0, };
 
@@ -152,7 +152,7 @@ static unsigned firmware_version(struct fbtft_par *par)
 static int init_display(struct fbtft_par *par)
 {
 	int ret;
-	unsigned version;
+	unsigned int version;
 	u8 save_mode;
 
 	/* enable SPI interface by having CS and MOSI low during reset */
diff --git a/drivers/staging/fbtft/fbtft-bus.c b/drivers/staging/fbtft/fbtft-bus.c
index 83505bce628a..ec45043c0830 100644
--- a/drivers/staging/fbtft/fbtft-bus.c
+++ b/drivers/staging/fbtft/fbtft-bus.c
@@ -92,7 +92,8 @@ void fbtft_write_reg8_bus9(struct fbtft_par *par, int len, ...)
 
 	if (par->spi && (par->spi->bits_per_word == 8)) {
 		/* we're emulating 9-bit, pad start of buffer with no-ops
-		   (assuming here that zero is a no-op) */
+		 * (assuming here that zero is a no-op)
+		 */
 		pad = (len % 4) ? 4 - (len % 4) : 0;
 		for (i = 0; i < pad; i++)
 			*buf++ = 0x000;
diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c
index 0c1a77cafe14..587f68aa466c 100644
--- a/drivers/staging/fbtft/fbtft-core.c
+++ b/drivers/staging/fbtft/fbtft-core.c
@@ -341,8 +341,8 @@ static void fbtft_reset(struct fbtft_par *par)
 	mdelay(120);
 }
 
-static void fbtft_update_display(struct fbtft_par *par, unsigned start_line,
-				 unsigned end_line)
+static void fbtft_update_display(struct fbtft_par *par, unsigned int start_line,
+				 unsigned int end_line)
 {
 	size_t offset, len;
 	ktime_t ts_start, ts_end;
@@ -391,11 +391,11 @@ static void fbtft_update_display(struct fbtft_par *par, unsigned start_line,
 
 	if (unlikely(timeit)) {
 		ts_end = ktime_get();
-		if (ktime_to_ns(par->update_time))
+		if (!ktime_to_ns(par->update_time))
 			par->update_time = ts_start;
 
-		par->update_time = ts_start;
 		fps = ktime_us_delta(ts_start, par->update_time);
+		par->update_time = ts_start;
 		fps = fps ? 1000000 / fps : 0;
 
 		throughput = ktime_us_delta(ts_end, ts_start);
@@ -435,10 +435,10 @@ static void fbtft_mkdirty(struct fb_info *info, int y, int height)
 static void fbtft_deferred_io(struct fb_info *info, struct list_head *pagelist)
 {
 	struct fbtft_par *par = info->par;
-	unsigned dirty_lines_start, dirty_lines_end;
+	unsigned int dirty_lines_start, dirty_lines_end;
 	struct page *page;
 	unsigned long index;
-	unsigned y_low = 0, y_high = 0;
+	unsigned int y_low = 0, y_high = 0;
 	int count = 0;
 
 	spin_lock(&par->dirty_lock);
@@ -526,18 +526,18 @@ static ssize_t fbtft_fb_write(struct fb_info *info, const char __user *buf,
 }
 
 /* from pxafb.c */
-static unsigned int chan_to_field(unsigned chan, struct fb_bitfield *bf)
+static unsigned int chan_to_field(unsigned int chan, struct fb_bitfield *bf)
 {
 	chan &= 0xffff;
 	chan >>= 16 - bf->length;
 	return chan << bf->offset;
 }
 
-static int fbtft_fb_setcolreg(unsigned regno, unsigned red, unsigned green,
-			      unsigned blue, unsigned transp,
+static int fbtft_fb_setcolreg(unsigned int regno, unsigned int red, unsigned int green,
+			      unsigned int blue, unsigned int transp,
 			      struct fb_info *info)
 {
-	unsigned val;
+	unsigned int val;
 	int ret = 1;
 
 	dev_dbg(info->dev,
@@ -654,11 +654,11 @@ struct fb_info *fbtft_framebuffer_alloc(struct fbtft_display *display,
 	u8 *vmem = NULL;
 	void *txbuf = NULL;
 	void *buf = NULL;
-	unsigned width;
-	unsigned height;
+	unsigned int width;
+	unsigned int height;
 	int txbuflen = display->txbuflen;
-	unsigned bpp = display->bpp;
-	unsigned fps = display->fps;
+	unsigned int bpp = display->bpp;
+	unsigned int fps = display->fps;
 	int vmem_size, i;
 	int *init_sequence = display->init_sequence;
 	char *gamma = display->gamma;
@@ -820,6 +820,8 @@ struct fb_info *fbtft_framebuffer_alloc(struct fbtft_display *display,
 	/* Transmit buffer */
 	if (txbuflen == -1)
 		txbuflen = vmem_size + 2; /* add in case startbyte is used */
+	if (txbuflen >= vmem_size + 2)
+		txbuflen = 0;
 
 #ifdef __LITTLE_ENDIAN
 	if ((!txbuflen) && (bpp > 8))
diff --git a/drivers/staging/fbtft/fbtft.h b/drivers/staging/fbtft/fbtft.h
index d3bc3943a983..89c4b5b76ce6 100644
--- a/drivers/staging/fbtft/fbtft.h
+++ b/drivers/staging/fbtft/fbtft.h
@@ -38,7 +38,7 @@
  */
 struct fbtft_gpio {
 	char name[FBTFT_GPIO_NAME_SIZE];
-	unsigned gpio;
+	unsigned int gpio;
 };
 
 struct fbtft_par;
@@ -79,7 +79,7 @@ struct fbtft_ops {
 	void (*reset)(struct fbtft_par *par);
 	void (*mkdirty)(struct fb_info *info, int from, int to);
 	void (*update_display)(struct fbtft_par *par,
-				unsigned start_line, unsigned end_line);
+				unsigned int start_line, unsigned int end_line);
 	int (*init_display)(struct fbtft_par *par);
 	int (*blank)(struct fbtft_par *par, bool on);
 
@@ -115,14 +115,14 @@ struct fbtft_ops {
  * This structure is not stored by FBTFT except for init_sequence.
  */
 struct fbtft_display {
-	unsigned width;
-	unsigned height;
-	unsigned regwidth;
-	unsigned buswidth;
-	unsigned backlight;
+	unsigned int width;
+	unsigned int height;
+	unsigned int regwidth;
+	unsigned int buswidth;
+	unsigned int backlight;
 	struct fbtft_ops fbtftops;
-	unsigned bpp;
-	unsigned fps;
+	unsigned int bpp;
+	unsigned int fps;
 	int txbuflen;
 	int *init_sequence;
 	char *gamma;
@@ -146,9 +146,9 @@ struct fbtft_display {
 struct fbtft_platform_data {
 	struct fbtft_display display;
 	const struct fbtft_gpio *gpios;
-	unsigned rotate;
+	unsigned int rotate;
 	bool bgr;
-	unsigned fps;
+	unsigned int fps;
 	int txbuflen;
 	u8 startbyte;
 	char *gamma;
@@ -216,8 +216,8 @@ struct fbtft_par {
 	u8 startbyte;
 	struct fbtft_ops fbtftops;
 	spinlock_t dirty_lock;
-	unsigned dirty_lines_start;
-	unsigned dirty_lines_end;
+	unsigned int dirty_lines_start;
+	unsigned int dirty_lines_end;
 	struct {
 		int reset;
 		int dc;
diff --git a/drivers/staging/fbtft/fbtft_device.c b/drivers/staging/fbtft/fbtft_device.c
index e4a355aefb25..e9211831b6a1 100644
--- a/drivers/staging/fbtft/fbtft_device.c
+++ b/drivers/staging/fbtft/fbtft_device.c
@@ -32,20 +32,20 @@ static char *name;
 module_param(name, charp, 0);
 MODULE_PARM_DESC(name, "Devicename (required). name=list => list all supported devices.");
 
-static unsigned rotate;
+static unsigned int rotate;
 module_param(rotate, uint, 0);
 MODULE_PARM_DESC(rotate,
 "Angle to rotate display counter clockwise: 0, 90, 180, 270");
 
-static unsigned busnum;
+static unsigned int busnum;
 module_param(busnum, uint, 0);
 MODULE_PARM_DESC(busnum, "SPI bus number (default=0)");
 
-static unsigned cs;
+static unsigned int cs;
 module_param(cs, uint, 0);
 MODULE_PARM_DESC(cs, "SPI chip select (default=0)");
 
-static unsigned speed;
+static unsigned int speed;
 module_param(speed, uint, 0);
 MODULE_PARM_DESC(speed, "SPI speed (override device default)");
 
@@ -58,7 +58,7 @@ module_param(gpios, charp, 0);
 MODULE_PARM_DESC(gpios,
 "List of gpios. Comma separated with the form: reset:23,dc:24 (when overriding the default, all gpios must be specified)");
 
-static unsigned fps;
+static unsigned int fps;
 module_param(fps, uint, 0);
 MODULE_PARM_DESC(fps, "Frames per second (override driver default)");
 
@@ -76,7 +76,7 @@ module_param(bgr, int, 0);
 MODULE_PARM_DESC(bgr,
 "BGR bit (supported by some drivers).");
 
-static unsigned startbyte;
+static unsigned int startbyte;
 module_param(startbyte, uint, 0);
 MODULE_PARM_DESC(startbyte, "Sets the Start byte used by some SPI displays.");
 
@@ -84,15 +84,15 @@ static bool custom;
 module_param(custom, bool, 0);
 MODULE_PARM_DESC(custom, "Add a custom display device. Use speed= argument to make it a SPI device, else platform_device");
 
-static unsigned width;
+static unsigned int width;
 module_param(width, uint, 0);
 MODULE_PARM_DESC(width, "Display width, used with the custom argument");
 
-static unsigned height;
+static unsigned int height;
 module_param(height, uint, 0);
 MODULE_PARM_DESC(height, "Display height, used with the custom argument");
 
-static unsigned buswidth = 8;
+static unsigned int buswidth = 8;
 module_param(buswidth, uint, 0);
 MODULE_PARM_DESC(buswidth, "Display bus width, used with the custom argument");
 
@@ -106,7 +106,7 @@ module_param(debug, ulong, 0);
 MODULE_PARM_DESC(debug,
 "level: 0-7 (the remaining 29 bits is for advanced usage)");
 
-static unsigned verbose = 3;
+static unsigned int verbose = 3;
 module_param(verbose, uint, 0);
 MODULE_PARM_DESC(verbose,
 "0 silent, >0 show gpios, >1 show devices, >2 show devices before (default=3)");
@@ -1215,7 +1215,8 @@ static struct fbtft_device_display displays[] = {
 		}
 	}, {
 		/* This should be the last item.
-		   Used with the custom argument */
+		 * Used with the custom argument
+		 */
 		.name = "",
 		.spi = &(struct spi_board_info) {
 			.modalias = "",
@@ -1306,8 +1307,9 @@ static struct fbtft_gpio fbtft_device_param_gpios[MAX_GPIOS + 1] = { };
 static void fbtft_device_pdev_release(struct device *dev)
 {
 /* Needed to silence this message:
-Device 'xxx' does not have a release() function, it is broken and must be fixed
-*/
+ * Device 'xxx' does not have a release() function,
+ * it is broken and must be fixed
+ */
 }
 
 static int spi_device_found(struct device *dev, void *data)
@@ -1346,7 +1348,7 @@ static void pr_p_devices(void)
 }
 
 #ifdef MODULE
-static void fbtft_device_spi_delete(struct spi_master *master, unsigned cs)
+static void fbtft_device_spi_delete(struct spi_master *master, unsigned int cs)
 {
 	struct device *dev;
 	char str[32];
@@ -1399,7 +1401,7 @@ static int __init fbtft_device_init(void)
 	long val;
 	int ret = 0;
 
-	if (name == NULL) {
+	if (!name) {
 #ifdef MODULE
 		pr_err("missing module parameter: 'name'\n");
 		return -EINVAL;
@@ -1416,14 +1418,14 @@ static int __init fbtft_device_init(void)
 
 	/* parse module parameter: gpios */
 	while ((p_gpio = strsep(&gpios, ","))) {
-		if (strchr(p_gpio, ':') == NULL) {
+		if (!strchr(p_gpio, ':')) {
 			pr_err("error: missing ':' in gpios parameter: %s\n",
 			       p_gpio);
 			return -EINVAL;
 		}
 		p_num = p_gpio;
 		p_name = strsep(&p_num, ":");
-		if (p_name == NULL || p_num == NULL) {
+		if (!p_name || !p_num) {
 			pr_err("something bad happened parsing gpios parameter: %s\n",
 			       p_gpio);
 			return -EINVAL;
diff --git a/drivers/staging/fsl-mc/bus/Makefile b/drivers/staging/fsl-mc/bus/Makefile
index e7315170b7a3..38716fd5cb58 100644
--- a/drivers/staging/fsl-mc/bus/Makefile
+++ b/drivers/staging/fsl-mc/bus/Makefile
@@ -7,13 +7,14 @@
 #
 obj-$(CONFIG_FSL_MC_BUS) += mc-bus-driver.o
 
-mc-bus-driver-objs := mc-bus.o \
+mc-bus-driver-objs := fsl-mc-bus.o \
 		      mc-sys.o \
+		      mc-io.o \
 		      dprc.o \
 		      dpmng.o \
 		      dprc-driver.o \
-		      mc-allocator.o \
-		      mc-msi.o \
+		      fsl-mc-allocator.o \
+		      fsl-mc-msi.o \
 		      irq-gic-v3-its-fsl-mc-msi.o \
 		      dpmcp.o \
 		      dpbp.o
diff --git a/drivers/staging/fsl-mc/bus/dpbp.c b/drivers/staging/fsl-mc/bus/dpbp.c
index fe271fbd629b..5d4cd812a400 100644
--- a/drivers/staging/fsl-mc/bus/dpbp.c
+++ b/drivers/staging/fsl-mc/bus/dpbp.c
@@ -1,34 +1,34 @@
 /* Copyright 2013-2016 Freescale Semiconductor Inc.
-*
-* Redistribution and use in source and binary forms, with or without
-* modification, are permitted provided that the following conditions are met:
-* * Redistributions of source code must retain the above copyright
-* notice, this list of conditions and the following disclaimer.
-* * Redistributions in binary form must reproduce the above copyright
-* notice, this list of conditions and the following disclaimer in the
-* documentation and/or other materials provided with the distribution.
-* * Neither the name of the above-listed copyright holders nor the
-* names of any contributors may be used to endorse or promote products
-* derived from this software without specific prior written permission.
-*
-*
-* ALTERNATIVELY, this software may be distributed under the terms of the
-* GNU General Public License ("GPL") as published by the Free Software
-* Foundation, either version 2 of that License or (at your option) any
-* later version.
-*
-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-* POSSIBILITY OF SUCH DAMAGE.
-*/
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the above-listed copyright holders nor the
+ * names of any contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
 #include "../include/mc-sys.h"
 #include "../include/mc-cmd.h"
 #include "../include/dpbp.h"
diff --git a/drivers/staging/fsl-mc/bus/dpmcp.c b/drivers/staging/fsl-mc/bus/dpmcp.c
index 06440176243a..55766f78a528 100644
--- a/drivers/staging/fsl-mc/bus/dpmcp.c
+++ b/drivers/staging/fsl-mc/bus/dpmcp.c
@@ -31,6 +31,7 @@
  */
 #include "../include/mc-sys.h"
 #include "../include/mc-cmd.h"
+
 #include "dpmcp.h"
 #include "dpmcp-cmd.h"
 
@@ -368,7 +369,6 @@ int dpmcp_set_irq_mask(struct fsl_mc_io *mc_io,
 	struct mc_command cmd = { 0 };
 	struct dpmcp_cmd_set_irq_mask *cmd_params;
 
-
 	/* prepare command */
 	cmd.header = mc_encode_cmd_header(DPMCP_CMDID_SET_IRQ_MASK,
 					  cmd_flags, token);
diff --git a/drivers/staging/fsl-mc/bus/dpmng-cmd.h b/drivers/staging/fsl-mc/bus/dpmng-cmd.h
index 779bf9c25bc0..a7b77d58c8cd 100644
--- a/drivers/staging/fsl-mc/bus/dpmng-cmd.h
+++ b/drivers/staging/fsl-mc/bus/dpmng-cmd.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2016 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -30,12 +31,12 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-/*************************************************************************//*
- dpmng-cmd.h
-
- defines portal commands
-
- *//**************************************************************************/
+/*
+ * dpmng-cmd.h
+ *
+ * defines portal commands
+ *
+ */
 
 #ifndef __FSL_DPMNG_CMD_H
 #define __FSL_DPMNG_CMD_H
diff --git a/drivers/staging/fsl-mc/bus/dpmng.c b/drivers/staging/fsl-mc/bus/dpmng.c
index 660bbe7ea899..96b1d67756fa 100644
--- a/drivers/staging/fsl-mc/bus/dpmng.c
+++ b/drivers/staging/fsl-mc/bus/dpmng.c
@@ -1,37 +1,38 @@
 /* Copyright 2013-2016 Freescale Semiconductor Inc.
-*
-* Redistribution and use in source and binary forms, with or without
-* modification, are permitted provided that the following conditions are met:
-* * Redistributions of source code must retain the above copyright
-* notice, this list of conditions and the following disclaimer.
-* * Redistributions in binary form must reproduce the above copyright
-* notice, this list of conditions and the following disclaimer in the
-* documentation and/or other materials provided with the distribution.
-* * Neither the name of the above-listed copyright holders nor the
-* names of any contributors may be used to endorse or promote products
-* derived from this software without specific prior written permission.
-*
-*
-* ALTERNATIVELY, this software may be distributed under the terms of the
-* GNU General Public License ("GPL") as published by the Free Software
-* Foundation, either version 2 of that License or (at your option) any
-* later version.
-*
-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-* POSSIBILITY OF SUCH DAMAGE.
-*/
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the above-listed copyright holders nor the
+ * names of any contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
 #include "../include/mc-sys.h"
 #include "../include/mc-cmd.h"
 #include "../include/dpmng.h"
+
 #include "dpmng-cmd.h"
 
 /**
diff --git a/drivers/staging/fsl-mc/bus/dprc-cmd.h b/drivers/staging/fsl-mc/bus/dprc-cmd.h
index bb127f4a3ae7..009d65673155 100644
--- a/drivers/staging/fsl-mc/bus/dprc-cmd.h
+++ b/drivers/staging/fsl-mc/bus/dprc-cmd.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2016 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -30,12 +31,12 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-/*************************************************************************//*
- dprc-cmd.h
-
- defines dprc portal commands
-
- *//**************************************************************************/
+/*
+ * dprc-cmd.h
+ *
+ * defines dprc portal commands
+ *
+ */
 
 #ifndef _FSL_DPRC_CMD_H
 #define _FSL_DPRC_CMD_H
diff --git a/drivers/staging/fsl-mc/bus/dprc-driver.c b/drivers/staging/fsl-mc/bus/dprc-driver.c
index d2a71f14bf72..c5ee4639682b 100644
--- a/drivers/staging/fsl-mc/bus/dprc-driver.c
+++ b/drivers/staging/fsl-mc/bus/dprc-driver.c
@@ -9,13 +9,21 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include "../include/mc-private.h"
-#include "../include/mc-sys.h"
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/msi.h>
+#include "../include/mc-bus.h"
+#include "../include/mc-sys.h"
+
 #include "dprc-cmd.h"
+#include "fsl-mc-private.h"
+
+#define FSL_MC_DPRC_DRIVER_NAME    "fsl_mc_dprc"
+
+#define FSL_MC_DEVICE_MATCH(_mc_dev, _obj_desc) \
+	(strcmp((_mc_dev)->obj_desc.type, (_obj_desc)->type) == 0 && \
+	 (_mc_dev)->obj_desc.id == (_obj_desc)->id)
 
 struct dprc_child_objs {
 	int child_count;
@@ -190,55 +198,6 @@ static void dprc_add_new_devices(struct fsl_mc_device *mc_bus_dev,
 	}
 }
 
-static void dprc_init_all_resource_pools(struct fsl_mc_device *mc_bus_dev)
-{
-	int pool_type;
-	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
-
-	for (pool_type = 0; pool_type < FSL_MC_NUM_POOL_TYPES; pool_type++) {
-		struct fsl_mc_resource_pool *res_pool =
-		    &mc_bus->resource_pools[pool_type];
-
-		res_pool->type = pool_type;
-		res_pool->max_count = 0;
-		res_pool->free_count = 0;
-		res_pool->mc_bus = mc_bus;
-		INIT_LIST_HEAD(&res_pool->free_list);
-		mutex_init(&res_pool->mutex);
-	}
-}
-
-static void dprc_cleanup_resource_pool(struct fsl_mc_device *mc_bus_dev,
-				       enum fsl_mc_pool_type pool_type)
-{
-	struct fsl_mc_resource *resource;
-	struct fsl_mc_resource *next;
-	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
-	struct fsl_mc_resource_pool *res_pool =
-					&mc_bus->resource_pools[pool_type];
-	int free_count = 0;
-
-	WARN_ON(res_pool->type != pool_type);
-	WARN_ON(res_pool->free_count != res_pool->max_count);
-
-	list_for_each_entry_safe(resource, next, &res_pool->free_list, node) {
-		free_count++;
-		WARN_ON(resource->type != res_pool->type);
-		WARN_ON(resource->parent_pool != res_pool);
-		devm_kfree(&mc_bus_dev->dev, resource);
-	}
-
-	WARN_ON(free_count != res_pool->free_count);
-}
-
-static void dprc_cleanup_all_resource_pools(struct fsl_mc_device *mc_bus_dev)
-{
-	int pool_type;
-
-	for (pool_type = 0; pool_type < FSL_MC_NUM_POOL_TYPES; pool_type++)
-		dprc_cleanup_resource_pool(mc_bus_dev, pool_type);
-}
-
 /**
  * dprc_scan_objects - Discover objects in a DPRC
  *
@@ -363,7 +322,7 @@ int dprc_scan_container(struct fsl_mc_device *mc_bus_dev)
 	unsigned int irq_count;
 	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
 
-	dprc_init_all_resource_pools(mc_bus_dev);
+	fsl_mc_init_all_resource_pools(mc_bus_dev);
 
 	/*
 	 * Discover objects in the DPRC:
@@ -390,7 +349,7 @@ int dprc_scan_container(struct fsl_mc_device *mc_bus_dev)
 
 	return 0;
 error:
-	dprc_cleanup_all_resource_pools(mc_bus_dev);
+	fsl_mc_cleanup_all_resource_pools(mc_bus_dev);
 	return error;
 }
 EXPORT_SYMBOL_GPL(dprc_scan_container);
@@ -649,7 +608,7 @@ static int dprc_probe(struct fsl_mc_device *mc_dev)
 		/*
 		 * This is a child DPRC:
 		 */
-		if (WARN_ON(parent_dev->bus != &fsl_mc_bus_type))
+		if (WARN_ON(!dev_is_fsl_mc(parent_dev)))
 			return -EINVAL;
 
 		if (WARN_ON(mc_dev->obj_desc.region_count == 0))
@@ -681,7 +640,7 @@ static int dprc_probe(struct fsl_mc_device *mc_dev)
 		 */
 		struct irq_domain *mc_msi_domain;
 
-		if (WARN_ON(parent_dev->bus == &fsl_mc_bus_type))
+		if (WARN_ON(dev_is_fsl_mc(parent_dev)))
 			return -EINVAL;
 
 		error = fsl_mc_find_msi_domain(parent_dev,
@@ -802,7 +761,7 @@ static int dprc_remove(struct fsl_mc_device *mc_dev)
 		dev_set_msi_domain(&mc_dev->dev, NULL);
 	}
 
-	dprc_cleanup_all_resource_pools(mc_dev);
+	fsl_mc_cleanup_all_resource_pools(mc_dev);
 
 	error = dprc_close(mc_dev->mc_io, 0, mc_dev->mc_handle);
 	if (error < 0)
diff --git a/drivers/staging/fsl-mc/bus/dprc.c b/drivers/staging/fsl-mc/bus/dprc.c
index c26054981333..9fea3def6041 100644
--- a/drivers/staging/fsl-mc/bus/dprc.c
+++ b/drivers/staging/fsl-mc/bus/dprc.c
@@ -1,37 +1,38 @@
 /* Copyright 2013-2016 Freescale Semiconductor Inc.
-*
-* Redistribution and use in source and binary forms, with or without
-* modification, are permitted provided that the following conditions are met:
-* * Redistributions of source code must retain the above copyright
-* notice, this list of conditions and the following disclaimer.
-* * Redistributions in binary form must reproduce the above copyright
-* notice, this list of conditions and the following disclaimer in the
-* documentation and/or other materials provided with the distribution.
-* * Neither the name of the above-listed copyright holders nor the
-* names of any contributors may be used to endorse or promote products
-* derived from this software without specific prior written permission.
-*
-*
-* ALTERNATIVELY, this software may be distributed under the terms of the
-* GNU General Public License ("GPL") as published by the Free Software
-* Foundation, either version 2 of that License or (at your option) any
-* later version.
-*
-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-* POSSIBILITY OF SUCH DAMAGE.
-*/
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the above-listed copyright holders nor the
+ * names of any contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
 #include "../include/mc-sys.h"
 #include "../include/mc-cmd.h"
 #include "../include/dprc.h"
+
 #include "dprc-cmd.h"
 
 /**
@@ -1334,20 +1335,20 @@ int dprc_disconnect(struct fsl_mc_io *mc_io,
 }
 
 /**
-* dprc_get_connection() - Get connected endpoint and link status if connection
-*			exists.
-* @mc_io:	Pointer to MC portal's I/O object
-* @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
-* @token:	Token of DPRC object
-* @endpoint1:	Endpoint 1 configuration parameters
-* @endpoint2:	Returned endpoint 2 configuration parameters
-* @state:	Returned link state:
-*		1 - link is up;
-*		0 - link is down;
-*		-1 - no connection (endpoint2 information is irrelevant)
-*
-* Return:     '0' on Success; -ENAVAIL if connection does not exist.
-*/
+ * dprc_get_connection() - Get connected endpoint and link status if connection
+ *			exists.
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPRC object
+ * @endpoint1:	Endpoint 1 configuration parameters
+ * @endpoint2:	Returned endpoint 2 configuration parameters
+ * @state:	Returned link state:
+ *		1 - link is up;
+ *		0 - link is down;
+ *		-1 - no connection (endpoint2 information is irrelevant)
+ *
+ * Return:     '0' on Success; -ENAVAIL if connection does not exist.
+ */
 int dprc_get_connection(struct fsl_mc_io *mc_io,
 			u32 cmd_flags,
 			u16 token,
diff --git a/drivers/staging/fsl-mc/bus/mc-allocator.c b/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c
index e59d85060c7b..e93ab53bae67 100644
--- a/drivers/staging/fsl-mc/bus/mc-allocator.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c
@@ -8,14 +8,19 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include "../include/mc-private.h"
-#include "../include/mc-sys.h"
 #include <linux/module.h>
+#include <linux/msi.h>
+#include "../include/mc-bus.h"
+#include "../include/mc-sys.h"
 #include "../include/dpbp-cmd.h"
 #include "../include/dpcon-cmd.h"
-#include "dpmcp-cmd.h"
-#include "dpmcp.h"
-#include <linux/msi.h>
+
+#include "fsl-mc-private.h"
+
+#define FSL_MC_IS_ALLOCATABLE(_obj_type) \
+	(strcmp(_obj_type, "dpbp") == 0 || \
+	 strcmp(_obj_type, "dpmcp") == 0 || \
+	 strcmp(_obj_type, "dpcon") == 0)
 
 /**
  * fsl_mc_resource_pool_add_device - add allocatable device to a resource
@@ -137,8 +142,7 @@ static int __must_check fsl_mc_resource_pool_remove_device(struct fsl_mc_device
 		goto out_unlock;
 	}
 
-	list_del(&resource->node);
-	INIT_LIST_HEAD(&resource->node);
+	list_del_init(&resource->node);
 	res_pool->free_count--;
 	res_pool->max_count--;
 
@@ -215,8 +219,7 @@ int __must_check fsl_mc_resource_allocate(struct fsl_mc_bus *mc_bus,
 		    res_pool->free_count > res_pool->max_count))
 		goto out_unlock;
 
-	list_del(&resource->node);
-	INIT_LIST_HEAD(&resource->node);
+	list_del_init(&resource->node);
 
 	res_pool->free_count--;
 	error = 0;
@@ -252,144 +255,6 @@ out_unlock:
 EXPORT_SYMBOL_GPL(fsl_mc_resource_free);
 
 /**
- * fsl_mc_portal_allocate - Allocates an MC portal
- *
- * @mc_dev: MC device for which the MC portal is to be allocated
- * @mc_io_flags: Flags for the fsl_mc_io object that wraps the allocated
- * MC portal.
- * @new_mc_io: Pointer to area where the pointer to the fsl_mc_io object
- * that wraps the allocated MC portal is to be returned
- *
- * This function allocates an MC portal from the device's parent DPRC,
- * from the corresponding MC bus' pool of MC portals and wraps
- * it in a new fsl_mc_io object. If 'mc_dev' is a DPRC itself, the
- * portal is allocated from its own MC bus.
- */
-int __must_check fsl_mc_portal_allocate(struct fsl_mc_device *mc_dev,
-					u16 mc_io_flags,
-					struct fsl_mc_io **new_mc_io)
-{
-	struct fsl_mc_device *mc_bus_dev;
-	struct fsl_mc_bus *mc_bus;
-	phys_addr_t mc_portal_phys_addr;
-	size_t mc_portal_size;
-	struct fsl_mc_device *dpmcp_dev;
-	int error = -EINVAL;
-	struct fsl_mc_resource *resource = NULL;
-	struct fsl_mc_io *mc_io = NULL;
-
-	if (mc_dev->flags & FSL_MC_IS_DPRC) {
-		mc_bus_dev = mc_dev;
-	} else {
-		if (WARN_ON(mc_dev->dev.parent->bus != &fsl_mc_bus_type))
-			return error;
-
-		mc_bus_dev = to_fsl_mc_device(mc_dev->dev.parent);
-	}
-
-	mc_bus = to_fsl_mc_bus(mc_bus_dev);
-	*new_mc_io = NULL;
-	error = fsl_mc_resource_allocate(mc_bus, FSL_MC_POOL_DPMCP, &resource);
-	if (error < 0)
-		return error;
-
-	error = -EINVAL;
-	dpmcp_dev = resource->data;
-	if (WARN_ON(!dpmcp_dev))
-		goto error_cleanup_resource;
-
-	if (dpmcp_dev->obj_desc.ver_major < DPMCP_MIN_VER_MAJOR ||
-	    (dpmcp_dev->obj_desc.ver_major == DPMCP_MIN_VER_MAJOR &&
-	     dpmcp_dev->obj_desc.ver_minor < DPMCP_MIN_VER_MINOR)) {
-		dev_err(&dpmcp_dev->dev,
-			"ERROR: Version %d.%d of DPMCP not supported.\n",
-			dpmcp_dev->obj_desc.ver_major,
-			dpmcp_dev->obj_desc.ver_minor);
-		error = -ENOTSUPP;
-		goto error_cleanup_resource;
-	}
-
-	if (WARN_ON(dpmcp_dev->obj_desc.region_count == 0))
-		goto error_cleanup_resource;
-
-	mc_portal_phys_addr = dpmcp_dev->regions[0].start;
-	mc_portal_size = dpmcp_dev->regions[0].end -
-			 dpmcp_dev->regions[0].start + 1;
-
-	if (WARN_ON(mc_portal_size != mc_bus_dev->mc_io->portal_size))
-		goto error_cleanup_resource;
-
-	error = fsl_create_mc_io(&mc_bus_dev->dev,
-				 mc_portal_phys_addr,
-				 mc_portal_size, dpmcp_dev,
-				 mc_io_flags, &mc_io);
-	if (error < 0)
-		goto error_cleanup_resource;
-
-	*new_mc_io = mc_io;
-	return 0;
-
-error_cleanup_resource:
-	fsl_mc_resource_free(resource);
-	return error;
-}
-EXPORT_SYMBOL_GPL(fsl_mc_portal_allocate);
-
-/**
- * fsl_mc_portal_free - Returns an MC portal to the pool of free MC portals
- * of a given MC bus
- *
- * @mc_io: Pointer to the fsl_mc_io object that wraps the MC portal to free
- */
-void fsl_mc_portal_free(struct fsl_mc_io *mc_io)
-{
-	struct fsl_mc_device *dpmcp_dev;
-	struct fsl_mc_resource *resource;
-
-	/*
-	 * Every mc_io obtained by calling fsl_mc_portal_allocate() is supposed
-	 * to have a DPMCP object associated with.
-	 */
-	dpmcp_dev = mc_io->dpmcp_dev;
-	if (WARN_ON(!dpmcp_dev))
-		return;
-
-	resource = dpmcp_dev->resource;
-	if (WARN_ON(!resource || resource->type != FSL_MC_POOL_DPMCP))
-		return;
-
-	if (WARN_ON(resource->data != dpmcp_dev))
-		return;
-
-	fsl_destroy_mc_io(mc_io);
-	fsl_mc_resource_free(resource);
-}
-EXPORT_SYMBOL_GPL(fsl_mc_portal_free);
-
-/**
- * fsl_mc_portal_reset - Resets the dpmcp object for a given fsl_mc_io object
- *
- * @mc_io: Pointer to the fsl_mc_io object that wraps the MC portal to free
- */
-int fsl_mc_portal_reset(struct fsl_mc_io *mc_io)
-{
-	int error;
-	struct fsl_mc_device *dpmcp_dev = mc_io->dpmcp_dev;
-
-	if (WARN_ON(!dpmcp_dev))
-		return -EINVAL;
-
-	error = dpmcp_reset(mc_io, 0, dpmcp_dev->mc_handle);
-	if (error < 0) {
-		dev_err(&dpmcp_dev->dev, "dpmcp_reset() failed: %d\n", error);
-		return error;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(fsl_mc_portal_reset);
-
-/**
  * fsl_mc_object_allocate - Allocates a MC object device of the given
  * pool type from a given MC bus
  *
@@ -420,7 +285,7 @@ int __must_check fsl_mc_object_allocate(struct fsl_mc_device *mc_dev,
 	if (WARN_ON(mc_dev->flags & FSL_MC_IS_DPRC))
 		goto error;
 
-	if (WARN_ON(mc_dev->dev.parent->bus != &fsl_mc_bus_type))
+	if (WARN_ON(!dev_is_fsl_mc(mc_dev->dev.parent)))
 		goto error;
 
 	if (WARN_ON(pool_type == FSL_MC_POOL_DPMCP))
@@ -663,6 +528,55 @@ void fsl_mc_free_irqs(struct fsl_mc_device *mc_dev)
 }
 EXPORT_SYMBOL_GPL(fsl_mc_free_irqs);
 
+void fsl_mc_init_all_resource_pools(struct fsl_mc_device *mc_bus_dev)
+{
+	int pool_type;
+	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
+
+	for (pool_type = 0; pool_type < FSL_MC_NUM_POOL_TYPES; pool_type++) {
+		struct fsl_mc_resource_pool *res_pool =
+		    &mc_bus->resource_pools[pool_type];
+
+		res_pool->type = pool_type;
+		res_pool->max_count = 0;
+		res_pool->free_count = 0;
+		res_pool->mc_bus = mc_bus;
+		INIT_LIST_HEAD(&res_pool->free_list);
+		mutex_init(&res_pool->mutex);
+	}
+}
+
+static void fsl_mc_cleanup_resource_pool(struct fsl_mc_device *mc_bus_dev,
+					 enum fsl_mc_pool_type pool_type)
+{
+	struct fsl_mc_resource *resource;
+	struct fsl_mc_resource *next;
+	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
+	struct fsl_mc_resource_pool *res_pool =
+					&mc_bus->resource_pools[pool_type];
+	int free_count = 0;
+
+	WARN_ON(res_pool->type != pool_type);
+	WARN_ON(res_pool->free_count != res_pool->max_count);
+
+	list_for_each_entry_safe(resource, next, &res_pool->free_list, node) {
+		free_count++;
+		WARN_ON(resource->type != res_pool->type);
+		WARN_ON(resource->parent_pool != res_pool);
+		devm_kfree(&mc_bus_dev->dev, resource);
+	}
+
+	WARN_ON(free_count != res_pool->free_count);
+}
+
+void fsl_mc_cleanup_all_resource_pools(struct fsl_mc_device *mc_bus_dev)
+{
+	int pool_type;
+
+	for (pool_type = 0; pool_type < FSL_MC_NUM_POOL_TYPES; pool_type++)
+		fsl_mc_cleanup_resource_pool(mc_bus_dev, pool_type);
+}
+
 /**
  * fsl_mc_allocator_probe - callback invoked when an allocatable device is
  * being added to the system
@@ -678,7 +592,7 @@ static int fsl_mc_allocator_probe(struct fsl_mc_device *mc_dev)
 		return -EINVAL;
 
 	mc_bus_dev = to_fsl_mc_device(mc_dev->dev.parent);
-	if (WARN_ON(mc_bus_dev->dev.bus != &fsl_mc_bus_type))
+	if (WARN_ON(!dev_is_fsl_mc(&mc_bus_dev->dev)))
 		return -EINVAL;
 
 	mc_bus = to_fsl_mc_bus(mc_bus_dev);
@@ -736,7 +650,6 @@ static const struct fsl_mc_device_id match_id_table[] = {
 static struct fsl_mc_driver fsl_mc_allocator_driver = {
 	.driver = {
 		   .name = "fsl_mc_allocator",
-		   .owner = THIS_MODULE,
 		   .pm = NULL,
 		   },
 	.match_id_table = match_id_table,
diff --git a/drivers/staging/fsl-mc/bus/mc-bus.c b/drivers/staging/fsl-mc/bus/fsl-mc-bus.c
index db3afdbdf4ae..44f64b6f0fc9 100644
--- a/drivers/staging/fsl-mc/bus/mc-bus.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-bus.c
@@ -9,7 +9,6 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include "../include/mc-private.h"
 #include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/of_address.h>
@@ -18,13 +17,50 @@
 #include <linux/limits.h>
 #include <linux/bitops.h>
 #include <linux/msi.h>
+#include <linux/dma-mapping.h>
+#include "../include/mc-bus.h"
 #include "../include/dpmng.h"
 #include "../include/mc-sys.h"
+
+#include "fsl-mc-private.h"
 #include "dprc-cmd.h"
 
 static struct kmem_cache *mc_dev_cache;
 
 /**
+ * Default DMA mask for devices on a fsl-mc bus
+ */
+#define FSL_MC_DEFAULT_DMA_MASK	(~0ULL)
+
+/**
+ * struct fsl_mc - Private data of a "fsl,qoriq-mc" platform device
+ * @root_mc_bus_dev: MC object device representing the root DPRC
+ * @num_translation_ranges: number of entries in addr_translation_ranges
+ * @translation_ranges: array of bus to system address translation ranges
+ */
+struct fsl_mc {
+	struct fsl_mc_device *root_mc_bus_dev;
+	u8 num_translation_ranges;
+	struct fsl_mc_addr_translation_range *translation_ranges;
+};
+
+/**
+ * struct fsl_mc_addr_translation_range - bus to system address translation
+ * range
+ * @mc_region_type: Type of MC region for the range being translated
+ * @start_mc_offset: Start MC offset of the range being translated
+ * @end_mc_offset: MC offset of the first byte after the range (last MC
+ * offset of the range is end_mc_offset - 1)
+ * @start_phys_addr: system physical address corresponding to start_mc_addr
+ */
+struct fsl_mc_addr_translation_range {
+	enum dprc_region_type mc_region_type;
+	u64 start_mc_offset;
+	u64 end_mc_offset;
+	phys_addr_t start_phys_addr;
+};
+
+/**
  * fsl_mc_bus_match - device to driver matching callback
  * @dev: the MC object device structure to match against
  * @drv: the device driver to search for matching MC object device id
@@ -101,14 +137,7 @@ static struct attribute *fsl_mc_dev_attrs[] = {
 	NULL,
 };
 
-static const struct attribute_group fsl_mc_dev_group = {
-	.attrs = fsl_mc_dev_attrs,
-};
-
-static const struct attribute_group *fsl_mc_dev_groups[] = {
-	&fsl_mc_dev_group,
-	NULL,
-};
+ATTRIBUTE_GROUPS(fsl_mc_dev);
 
 struct bus_type fsl_mc_bus_type = {
 	.name = "fsl-mc",
@@ -229,21 +258,22 @@ bool fsl_mc_bus_exists(void)
 EXPORT_SYMBOL_GPL(fsl_mc_bus_exists);
 
 /**
-* fsl_mc_get_root_dprc - function to traverse to the root dprc
-*/
-static void fsl_mc_get_root_dprc(struct device *dev,
-				 struct device **root_dprc_dev)
+ * fsl_mc_get_root_dprc - function to traverse to the root dprc
+ */
+void fsl_mc_get_root_dprc(struct device *dev,
+			  struct device **root_dprc_dev)
 {
 	if (WARN_ON(!dev)) {
 		*root_dprc_dev = NULL;
-	} else if (WARN_ON(dev->bus != &fsl_mc_bus_type)) {
+	} else if (WARN_ON(!dev_is_fsl_mc(dev))) {
 		*root_dprc_dev = NULL;
 	} else {
 		*root_dprc_dev = dev;
-		while ((*root_dprc_dev)->parent->bus == &fsl_mc_bus_type)
+		while (dev_is_fsl_mc((*root_dprc_dev)->parent))
 			*root_dprc_dev = (*root_dprc_dev)->parent;
 	}
 }
+EXPORT_SYMBOL_GPL(fsl_mc_get_root_dprc);
 
 static int get_dprc_attr(struct fsl_mc_io *mc_io,
 			 int container_id, struct dprc_attributes *attr)
@@ -434,7 +464,7 @@ int fsl_mc_device_add(struct dprc_obj_desc *obj_desc,
 	struct fsl_mc_bus *mc_bus = NULL;
 	struct fsl_mc_device *parent_mc_dev;
 
-	if (parent_dev->bus == &fsl_mc_bus_type)
+	if (dev_is_fsl_mc(parent_dev))
 		parent_mc_dev = to_fsl_mc_device(parent_dev);
 	else
 		parent_mc_dev = NULL;
@@ -887,25 +917,4 @@ error_cleanup_cache:
 	kmem_cache_destroy(mc_dev_cache);
 	return error;
 }
-
 postcore_initcall(fsl_mc_bus_driver_init);
-
-static void __exit fsl_mc_bus_driver_exit(void)
-{
-	if (WARN_ON(!mc_dev_cache))
-		return;
-
-	its_fsl_mc_msi_cleanup();
-	fsl_mc_allocator_driver_exit();
-	dprc_driver_exit();
-	platform_driver_unregister(&fsl_mc_bus_driver);
-	bus_unregister(&fsl_mc_bus_type);
-	kmem_cache_destroy(mc_dev_cache);
-	pr_info("MC bus unregistered\n");
-}
-
-module_exit(fsl_mc_bus_driver_exit);
-
-MODULE_AUTHOR("Freescale Semiconductor Inc.");
-MODULE_DESCRIPTION("Freescale Management Complex (MC) bus driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/fsl-mc/bus/mc-msi.c b/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
index c7be156ae5e0..3d46b1b1fa18 100644
--- a/drivers/staging/fsl-mc/bus/mc-msi.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
@@ -9,7 +9,6 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include "../include/mc-private.h"
 #include <linux/of_device.h>
 #include <linux/of_address.h>
 #include <linux/irqchip/arm-gic-v3.h>
@@ -17,8 +16,7 @@
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>
-#include "../include/mc-sys.h"
-#include "dprc-cmd.h"
+#include "../include/mc-bus.h"
 
 /*
  * Generate a unique ID identifying the interrupt (only used within the MSI
@@ -52,7 +50,7 @@ static void fsl_mc_msi_update_dom_ops(struct msi_domain_info *info)
 	/*
 	 * set_desc should not be set by the caller
 	 */
-	if (ops->set_desc == NULL)
+	if (!ops->set_desc)
 		ops->set_desc = fsl_mc_msi_set_desc;
 }
 
@@ -142,7 +140,7 @@ static void fsl_mc_msi_update_chip_ops(struct msi_domain_info *info)
 	/*
 	 * irq_write_msi_msg should not be set by the caller
 	 */
-	if (chip->irq_write_msi_msg == NULL)
+	if (!chip->irq_write_msi_msg)
 		chip->irq_write_msi_msg = fsl_mc_msi_write_msg;
 }
 
@@ -213,7 +211,7 @@ static int fsl_mc_msi_alloc_descs(struct device *dev, unsigned int irq_count)
 	struct msi_desc *msi_desc;
 
 	for (i = 0; i < irq_count; i++) {
-		msi_desc = alloc_msi_entry(dev);
+		msi_desc = alloc_msi_entry(dev, 1, NULL);
 		if (!msi_desc) {
 			dev_err(dev, "Failed to allocate msi entry\n");
 			error = -ENOMEM;
@@ -221,7 +219,6 @@ static int fsl_mc_msi_alloc_descs(struct device *dev, unsigned int irq_count)
 		}
 
 		msi_desc->fsl_mc.msi_index = i;
-		msi_desc->nvec_used = 1;
 		INIT_LIST_HEAD(&msi_desc->list);
 		list_add_tail(&msi_desc->list, dev_to_msi_list(dev));
 	}
diff --git a/drivers/staging/fsl-mc/bus/fsl-mc-private.h b/drivers/staging/fsl-mc/bus/fsl-mc-private.h
new file mode 100644
index 000000000000..d459c2673f39
--- /dev/null
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-private.h
@@ -0,0 +1,52 @@
+/*
+ * Freescale Management Complex (MC) bus private declarations
+ *
+ * Copyright (C) 2016 Freescale Semiconductor, Inc.
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+#ifndef _FSL_MC_PRIVATE_H_
+#define _FSL_MC_PRIVATE_H_
+
+int __must_check fsl_mc_device_add(struct dprc_obj_desc *obj_desc,
+				   struct fsl_mc_io *mc_io,
+				   struct device *parent_dev,
+				   struct fsl_mc_device **new_mc_dev);
+
+void fsl_mc_device_remove(struct fsl_mc_device *mc_dev);
+
+int __init dprc_driver_init(void);
+
+void dprc_driver_exit(void);
+
+int __init fsl_mc_allocator_driver_init(void);
+
+void fsl_mc_allocator_driver_exit(void);
+
+int __must_check fsl_mc_resource_allocate(struct fsl_mc_bus *mc_bus,
+					  enum fsl_mc_pool_type pool_type,
+					  struct fsl_mc_resource
+							  **new_resource);
+
+void fsl_mc_resource_free(struct fsl_mc_resource *resource);
+
+int fsl_mc_msi_domain_alloc_irqs(struct device *dev,
+				 unsigned int irq_count);
+
+void fsl_mc_msi_domain_free_irqs(struct device *dev);
+
+int __init its_fsl_mc_msi_init(void);
+
+void its_fsl_mc_msi_cleanup(void);
+
+int __must_check fsl_create_mc_io(struct device *dev,
+				  phys_addr_t mc_portal_phys_addr,
+				  u32 mc_portal_size,
+				  struct fsl_mc_device *dpmcp_dev,
+				  u32 flags, struct fsl_mc_io **new_mc_io);
+
+void fsl_destroy_mc_io(struct fsl_mc_io *mc_io);
+
+#endif /* _FSL_MC_PRIVATE_H_ */
diff --git a/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c b/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c
index 720e2b018d00..7a6ac640752f 100644
--- a/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c
+++ b/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c
@@ -9,7 +9,6 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include "../include/mc-private.h"
 #include <linux/of_device.h>
 #include <linux/of_address.h>
 #include <linux/irqchip/arm-gic-v3.h>
@@ -17,8 +16,7 @@
 #include <linux/msi.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
-#include "../include/mc-sys.h"
-#include "dprc-cmd.h"
+#include "../include/mc-bus.h"
 
 static struct irq_chip its_msi_irq_chip = {
 	.name = "fsl-mc-bus-msi",
@@ -35,7 +33,7 @@ static int its_fsl_mc_msi_prepare(struct irq_domain *msi_domain,
 	struct fsl_mc_device *mc_bus_dev;
 	struct msi_domain_info *msi_info;
 
-	if (WARN_ON(dev->bus != &fsl_mc_bus_type))
+	if (WARN_ON(!dev_is_fsl_mc(dev)))
 		return -EINVAL;
 
 	mc_bus_dev = to_fsl_mc_device(dev);
diff --git a/drivers/staging/fsl-mc/bus/mc-io.c b/drivers/staging/fsl-mc/bus/mc-io.c
new file mode 100644
index 000000000000..798c965fe203
--- /dev/null
+++ b/drivers/staging/fsl-mc/bus/mc-io.c
@@ -0,0 +1,320 @@
+/* Copyright 2013-2016 Freescale Semiconductor Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of the above-listed copyright holders nor the
+ *       names of any contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/io.h>
+#include "../include/mc-bus.h"
+#include "../include/mc-sys.h"
+
+#include "fsl-mc-private.h"
+#include "dpmcp.h"
+#include "dpmcp-cmd.h"
+
+static int fsl_mc_io_set_dpmcp(struct fsl_mc_io *mc_io,
+			       struct fsl_mc_device *dpmcp_dev)
+{
+	int error;
+
+	if (WARN_ON(!dpmcp_dev))
+		return -EINVAL;
+
+	if (WARN_ON(mc_io->dpmcp_dev))
+		return -EINVAL;
+
+	if (WARN_ON(dpmcp_dev->mc_io))
+		return -EINVAL;
+
+	error = dpmcp_open(mc_io,
+			   0,
+			   dpmcp_dev->obj_desc.id,
+			   &dpmcp_dev->mc_handle);
+	if (error < 0)
+		return error;
+
+	mc_io->dpmcp_dev = dpmcp_dev;
+	dpmcp_dev->mc_io = mc_io;
+	return 0;
+}
+
+static void fsl_mc_io_unset_dpmcp(struct fsl_mc_io *mc_io)
+{
+	int error;
+	struct fsl_mc_device *dpmcp_dev = mc_io->dpmcp_dev;
+
+	if (WARN_ON(!dpmcp_dev))
+		return;
+
+	if (WARN_ON(dpmcp_dev->mc_io != mc_io))
+		return;
+
+	error = dpmcp_close(mc_io,
+			    0,
+			    dpmcp_dev->mc_handle);
+	if (error < 0) {
+		dev_err(&dpmcp_dev->dev, "dpmcp_close() failed: %d\n",
+			error);
+	}
+
+	mc_io->dpmcp_dev = NULL;
+	dpmcp_dev->mc_io = NULL;
+}
+
+/**
+ * Creates an MC I/O object
+ *
+ * @dev: device to be associated with the MC I/O object
+ * @mc_portal_phys_addr: physical address of the MC portal to use
+ * @mc_portal_size: size in bytes of the MC portal
+ * @dpmcp-dev: Pointer to the DPMCP object associated with this MC I/O
+ * object or NULL if none.
+ * @flags: flags for the new MC I/O object
+ * @new_mc_io: Area to return pointer to newly created MC I/O object
+ *
+ * Returns '0' on Success; Error code otherwise.
+ */
+int __must_check fsl_create_mc_io(struct device *dev,
+				  phys_addr_t mc_portal_phys_addr,
+				  u32 mc_portal_size,
+				  struct fsl_mc_device *dpmcp_dev,
+				  u32 flags, struct fsl_mc_io **new_mc_io)
+{
+	int error;
+	struct fsl_mc_io *mc_io;
+	void __iomem *mc_portal_virt_addr;
+	struct resource *res;
+
+	mc_io = devm_kzalloc(dev, sizeof(*mc_io), GFP_KERNEL);
+	if (!mc_io)
+		return -ENOMEM;
+
+	mc_io->dev = dev;
+	mc_io->flags = flags;
+	mc_io->portal_phys_addr = mc_portal_phys_addr;
+	mc_io->portal_size = mc_portal_size;
+	if (flags & FSL_MC_IO_ATOMIC_CONTEXT_PORTAL)
+		spin_lock_init(&mc_io->spinlock);
+	else
+		mutex_init(&mc_io->mutex);
+
+	res = devm_request_mem_region(dev,
+				      mc_portal_phys_addr,
+				      mc_portal_size,
+				      "mc_portal");
+	if (!res) {
+		dev_err(dev,
+			"devm_request_mem_region failed for MC portal %#llx\n",
+			mc_portal_phys_addr);
+		return -EBUSY;
+	}
+
+	mc_portal_virt_addr = devm_ioremap_nocache(dev,
+						   mc_portal_phys_addr,
+						   mc_portal_size);
+	if (!mc_portal_virt_addr) {
+		dev_err(dev,
+			"devm_ioremap_nocache failed for MC portal %#llx\n",
+			mc_portal_phys_addr);
+		return -ENXIO;
+	}
+
+	mc_io->portal_virt_addr = mc_portal_virt_addr;
+	if (dpmcp_dev) {
+		error = fsl_mc_io_set_dpmcp(mc_io, dpmcp_dev);
+		if (error < 0)
+			goto error_destroy_mc_io;
+	}
+
+	*new_mc_io = mc_io;
+	return 0;
+
+error_destroy_mc_io:
+	fsl_destroy_mc_io(mc_io);
+	return error;
+}
+
+/**
+ * Destroys an MC I/O object
+ *
+ * @mc_io: MC I/O object to destroy
+ */
+void fsl_destroy_mc_io(struct fsl_mc_io *mc_io)
+{
+	struct fsl_mc_device *dpmcp_dev = mc_io->dpmcp_dev;
+
+	if (dpmcp_dev)
+		fsl_mc_io_unset_dpmcp(mc_io);
+
+	devm_iounmap(mc_io->dev, mc_io->portal_virt_addr);
+	devm_release_mem_region(mc_io->dev,
+				mc_io->portal_phys_addr,
+				mc_io->portal_size);
+
+	mc_io->portal_virt_addr = NULL;
+	devm_kfree(mc_io->dev, mc_io);
+}
+
+/**
+ * fsl_mc_portal_allocate - Allocates an MC portal
+ *
+ * @mc_dev: MC device for which the MC portal is to be allocated
+ * @mc_io_flags: Flags for the fsl_mc_io object that wraps the allocated
+ * MC portal.
+ * @new_mc_io: Pointer to area where the pointer to the fsl_mc_io object
+ * that wraps the allocated MC portal is to be returned
+ *
+ * This function allocates an MC portal from the device's parent DPRC,
+ * from the corresponding MC bus' pool of MC portals and wraps
+ * it in a new fsl_mc_io object. If 'mc_dev' is a DPRC itself, the
+ * portal is allocated from its own MC bus.
+ */
+int __must_check fsl_mc_portal_allocate(struct fsl_mc_device *mc_dev,
+					u16 mc_io_flags,
+					struct fsl_mc_io **new_mc_io)
+{
+	struct fsl_mc_device *mc_bus_dev;
+	struct fsl_mc_bus *mc_bus;
+	phys_addr_t mc_portal_phys_addr;
+	size_t mc_portal_size;
+	struct fsl_mc_device *dpmcp_dev;
+	int error = -EINVAL;
+	struct fsl_mc_resource *resource = NULL;
+	struct fsl_mc_io *mc_io = NULL;
+
+	if (mc_dev->flags & FSL_MC_IS_DPRC) {
+		mc_bus_dev = mc_dev;
+	} else {
+		if (WARN_ON(!dev_is_fsl_mc(mc_dev->dev.parent)))
+			return error;
+
+		mc_bus_dev = to_fsl_mc_device(mc_dev->dev.parent);
+	}
+
+	mc_bus = to_fsl_mc_bus(mc_bus_dev);
+	*new_mc_io = NULL;
+	error = fsl_mc_resource_allocate(mc_bus, FSL_MC_POOL_DPMCP, &resource);
+	if (error < 0)
+		return error;
+
+	error = -EINVAL;
+	dpmcp_dev = resource->data;
+	if (WARN_ON(!dpmcp_dev))
+		goto error_cleanup_resource;
+
+	if (dpmcp_dev->obj_desc.ver_major < DPMCP_MIN_VER_MAJOR ||
+	    (dpmcp_dev->obj_desc.ver_major == DPMCP_MIN_VER_MAJOR &&
+	     dpmcp_dev->obj_desc.ver_minor < DPMCP_MIN_VER_MINOR)) {
+		dev_err(&dpmcp_dev->dev,
+			"ERROR: Version %d.%d of DPMCP not supported.\n",
+			dpmcp_dev->obj_desc.ver_major,
+			dpmcp_dev->obj_desc.ver_minor);
+		error = -ENOTSUPP;
+		goto error_cleanup_resource;
+	}
+
+	if (WARN_ON(dpmcp_dev->obj_desc.region_count == 0))
+		goto error_cleanup_resource;
+
+	mc_portal_phys_addr = dpmcp_dev->regions[0].start;
+	mc_portal_size = dpmcp_dev->regions[0].end -
+			 dpmcp_dev->regions[0].start + 1;
+
+	if (WARN_ON(mc_portal_size != mc_bus_dev->mc_io->portal_size))
+		goto error_cleanup_resource;
+
+	error = fsl_create_mc_io(&mc_bus_dev->dev,
+				 mc_portal_phys_addr,
+				 mc_portal_size, dpmcp_dev,
+				 mc_io_flags, &mc_io);
+	if (error < 0)
+		goto error_cleanup_resource;
+
+	*new_mc_io = mc_io;
+	return 0;
+
+error_cleanup_resource:
+	fsl_mc_resource_free(resource);
+	return error;
+}
+EXPORT_SYMBOL_GPL(fsl_mc_portal_allocate);
+
+/**
+ * fsl_mc_portal_free - Returns an MC portal to the pool of free MC portals
+ * of a given MC bus
+ *
+ * @mc_io: Pointer to the fsl_mc_io object that wraps the MC portal to free
+ */
+void fsl_mc_portal_free(struct fsl_mc_io *mc_io)
+{
+	struct fsl_mc_device *dpmcp_dev;
+	struct fsl_mc_resource *resource;
+
+	/*
+	 * Every mc_io obtained by calling fsl_mc_portal_allocate() is supposed
+	 * to have a DPMCP object associated with.
+	 */
+	dpmcp_dev = mc_io->dpmcp_dev;
+	if (WARN_ON(!dpmcp_dev))
+		return;
+
+	resource = dpmcp_dev->resource;
+	if (WARN_ON(!resource || resource->type != FSL_MC_POOL_DPMCP))
+		return;
+
+	if (WARN_ON(resource->data != dpmcp_dev))
+		return;
+
+	fsl_destroy_mc_io(mc_io);
+	fsl_mc_resource_free(resource);
+}
+EXPORT_SYMBOL_GPL(fsl_mc_portal_free);
+
+/**
+ * fsl_mc_portal_reset - Resets the dpmcp object for a given fsl_mc_io object
+ *
+ * @mc_io: Pointer to the fsl_mc_io object that wraps the MC portal to free
+ */
+int fsl_mc_portal_reset(struct fsl_mc_io *mc_io)
+{
+	int error;
+	struct fsl_mc_device *dpmcp_dev = mc_io->dpmcp_dev;
+
+	if (WARN_ON(!dpmcp_dev))
+		return -EINVAL;
+
+	error = dpmcp_reset(mc_io, 0, dpmcp_dev->mc_handle);
+	if (error < 0) {
+		dev_err(&dpmcp_dev->dev, "dpmcp_reset() failed: %d\n", error);
+		return error;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(fsl_mc_portal_reset);
diff --git a/drivers/staging/fsl-mc/bus/mc-sys.c b/drivers/staging/fsl-mc/bus/mc-sys.c
index 0c185abe665e..285917c7c8e4 100644
--- a/drivers/staging/fsl-mc/bus/mc-sys.c
+++ b/drivers/staging/fsl-mc/bus/mc-sys.c
@@ -32,13 +32,15 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "../include/mc-sys.h"
-#include "../include/mc-cmd.h"
-#include "../include/mc.h"
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/ioport.h>
 #include <linux/device.h>
+#include <linux/io.h>
+#include "../include/mc-sys.h"
+#include "../include/mc-cmd.h"
+#include "../include/mc.h"
+
 #include "dpmcp.h"
 
 /**
@@ -68,153 +70,6 @@ static u16 mc_cmd_hdr_read_cmdid(struct mc_command *cmd)
 	return (cmd_id & MC_CMD_HDR_CMDID_MASK) >> MC_CMD_HDR_CMDID_SHIFT;
 }
 
-/**
- * Creates an MC I/O object
- *
- * @dev: device to be associated with the MC I/O object
- * @mc_portal_phys_addr: physical address of the MC portal to use
- * @mc_portal_size: size in bytes of the MC portal
- * @dpmcp-dev: Pointer to the DPMCP object associated with this MC I/O
- * object or NULL if none.
- * @flags: flags for the new MC I/O object
- * @new_mc_io: Area to return pointer to newly created MC I/O object
- *
- * Returns '0' on Success; Error code otherwise.
- */
-int __must_check fsl_create_mc_io(struct device *dev,
-				  phys_addr_t mc_portal_phys_addr,
-				  u32 mc_portal_size,
-				  struct fsl_mc_device *dpmcp_dev,
-				  u32 flags, struct fsl_mc_io **new_mc_io)
-{
-	int error;
-	struct fsl_mc_io *mc_io;
-	void __iomem *mc_portal_virt_addr;
-	struct resource *res;
-
-	mc_io = devm_kzalloc(dev, sizeof(*mc_io), GFP_KERNEL);
-	if (!mc_io)
-		return -ENOMEM;
-
-	mc_io->dev = dev;
-	mc_io->flags = flags;
-	mc_io->portal_phys_addr = mc_portal_phys_addr;
-	mc_io->portal_size = mc_portal_size;
-	if (flags & FSL_MC_IO_ATOMIC_CONTEXT_PORTAL)
-		spin_lock_init(&mc_io->spinlock);
-	else
-		mutex_init(&mc_io->mutex);
-
-	res = devm_request_mem_region(dev,
-				      mc_portal_phys_addr,
-				      mc_portal_size,
-				      "mc_portal");
-	if (!res) {
-		dev_err(dev,
-			"devm_request_mem_region failed for MC portal %#llx\n",
-			mc_portal_phys_addr);
-		return -EBUSY;
-	}
-
-	mc_portal_virt_addr = devm_ioremap_nocache(dev,
-						   mc_portal_phys_addr,
-						   mc_portal_size);
-	if (!mc_portal_virt_addr) {
-		dev_err(dev,
-			"devm_ioremap_nocache failed for MC portal %#llx\n",
-			mc_portal_phys_addr);
-		return -ENXIO;
-	}
-
-	mc_io->portal_virt_addr = mc_portal_virt_addr;
-	if (dpmcp_dev) {
-		error = fsl_mc_io_set_dpmcp(mc_io, dpmcp_dev);
-		if (error < 0)
-			goto error_destroy_mc_io;
-	}
-
-	*new_mc_io = mc_io;
-	return 0;
-
-error_destroy_mc_io:
-	fsl_destroy_mc_io(mc_io);
-	return error;
-}
-EXPORT_SYMBOL_GPL(fsl_create_mc_io);
-
-/**
- * Destroys an MC I/O object
- *
- * @mc_io: MC I/O object to destroy
- */
-void fsl_destroy_mc_io(struct fsl_mc_io *mc_io)
-{
-	struct fsl_mc_device *dpmcp_dev = mc_io->dpmcp_dev;
-
-	if (dpmcp_dev)
-		fsl_mc_io_unset_dpmcp(mc_io);
-
-	devm_iounmap(mc_io->dev, mc_io->portal_virt_addr);
-	devm_release_mem_region(mc_io->dev,
-				mc_io->portal_phys_addr,
-				mc_io->portal_size);
-
-	mc_io->portal_virt_addr = NULL;
-	devm_kfree(mc_io->dev, mc_io);
-}
-EXPORT_SYMBOL_GPL(fsl_destroy_mc_io);
-
-int fsl_mc_io_set_dpmcp(struct fsl_mc_io *mc_io,
-			struct fsl_mc_device *dpmcp_dev)
-{
-	int error;
-
-	if (WARN_ON(!dpmcp_dev))
-		return -EINVAL;
-
-	if (WARN_ON(mc_io->dpmcp_dev))
-		return -EINVAL;
-
-	if (WARN_ON(dpmcp_dev->mc_io))
-		return -EINVAL;
-
-	error = dpmcp_open(mc_io,
-			   0,
-			   dpmcp_dev->obj_desc.id,
-			   &dpmcp_dev->mc_handle);
-	if (error < 0)
-		return error;
-
-	mc_io->dpmcp_dev = dpmcp_dev;
-	dpmcp_dev->mc_io = mc_io;
-	return 0;
-}
-EXPORT_SYMBOL_GPL(fsl_mc_io_set_dpmcp);
-
-void fsl_mc_io_unset_dpmcp(struct fsl_mc_io *mc_io)
-{
-	int error;
-	struct fsl_mc_device *dpmcp_dev = mc_io->dpmcp_dev;
-
-	if (WARN_ON(!dpmcp_dev))
-		return;
-
-	if (WARN_ON(dpmcp_dev->mc_io != mc_io))
-		return;
-
-	error = dpmcp_close(mc_io,
-			    0,
-			    dpmcp_dev->mc_handle);
-	if (error < 0) {
-		dev_err(&dpmcp_dev->dev, "dpmcp_close() failed: %d\n",
-			error);
-	}
-
-	mc_io->dpmcp_dev = NULL;
-	dpmcp_dev->mc_io = NULL;
-}
-EXPORT_SYMBOL_GPL(fsl_mc_io_unset_dpmcp);
-
 static int mc_status_to_error(enum mc_cmd_status status)
 {
 	static const int mc_status_to_error_map[] = {
diff --git a/drivers/staging/fsl-mc/include/dpbp-cmd.h b/drivers/staging/fsl-mc/include/dpbp-cmd.h
index 4828ccd0cffd..2860411ddb51 100644
--- a/drivers/staging/fsl-mc/include/dpbp-cmd.h
+++ b/drivers/staging/fsl-mc/include/dpbp-cmd.h
@@ -1,34 +1,34 @@
 /* Copyright 2013-2016 Freescale Semiconductor Inc.
-*
-* Redistribution and use in source and binary forms, with or without
-* modification, are permitted provided that the following conditions are met:
-* * Redistributions of source code must retain the above copyright
-* notice, this list of conditions and the following disclaimer.
-* * Redistributions in binary form must reproduce the above copyright
-* notice, this list of conditions and the following disclaimer in the
-* documentation and/or other materials provided with the distribution.
-* * Neither the name of the above-listed copyright holders nor the
-* names of any contributors may be used to endorse or promote products
-* derived from this software without specific prior written permission.
-*
-*
-* ALTERNATIVELY, this software may be distributed under the terms of the
-* GNU General Public License ("GPL") as published by the Free Software
-* Foundation, either version 2 of that License or (at your option) any
-* later version.
-*
-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-* POSSIBILITY OF SUCH DAMAGE.
-*/
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the above-listed copyright holders nor the
+ * names of any contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
 #ifndef _FSL_DPBP_CMD_H
 #define _FSL_DPBP_CMD_H
 
diff --git a/drivers/staging/fsl-mc/include/mc-private.h b/drivers/staging/fsl-mc/include/mc-bus.h
index cab1ae90f09e..170684a57ca2 100644
--- a/drivers/staging/fsl-mc/include/mc-private.h
+++ b/drivers/staging/fsl-mc/include/mc-bus.h
@@ -1,5 +1,5 @@
 /*
- * Freescale Management Complex (MC) bus private declarations
+ * Freescale Management Complex (MC) bus declarations
  *
  * Copyright (C) 2014 Freescale Semiconductor, Inc.
  * Author: German Rivera <German.Rivera@freescale.com>
@@ -8,23 +8,11 @@
  * License version 2. This program is licensed "as is" without any
  * warranty of any kind, whether express or implied.
  */
-#ifndef _FSL_MC_PRIVATE_H_
-#define _FSL_MC_PRIVATE_H_
+#ifndef _FSL_MC_MCBUS_H_
+#define _FSL_MC_MCBUS_H_
 
 #include "../include/mc.h"
 #include <linux/mutex.h>
-#include <linux/stringify.h>
-
-#define FSL_MC_DPRC_DRIVER_NAME    "fsl_mc_dprc"
-
-#define FSL_MC_DEVICE_MATCH(_mc_dev, _obj_desc) \
-	(strcmp((_mc_dev)->obj_desc.type, (_obj_desc)->type) == 0 && \
-	 (_mc_dev)->obj_desc.id == (_obj_desc)->id)
-
-#define FSL_MC_IS_ALLOCATABLE(_obj_type) \
-	(strcmp(_obj_type, "dpbp") == 0 || \
-	 strcmp(_obj_type, "dpmcp") == 0 || \
-	 strcmp(_obj_type, "dpcon") == 0)
 
 struct irq_domain;
 struct msi_domain_info;
@@ -35,37 +23,12 @@ struct msi_domain_info;
  */
 #define FSL_MC_IRQ_POOL_MAX_TOTAL_IRQS	256
 
-struct device_node;
-struct irq_domain;
-struct msi_domain_info;
-
-/**
- * struct fsl_mc - Private data of a "fsl,qoriq-mc" platform device
- * @root_mc_bus_dev: MC object device representing the root DPRC
- * @num_translation_ranges: number of entries in addr_translation_ranges
- * @translation_ranges: array of bus to system address translation ranges
- */
-struct fsl_mc {
-	struct fsl_mc_device *root_mc_bus_dev;
-	u8 num_translation_ranges;
-	struct fsl_mc_addr_translation_range *translation_ranges;
-};
-
-/**
- * struct fsl_mc_addr_translation_range - bus to system address translation
- * range
- * @mc_region_type: Type of MC region for the range being translated
- * @start_mc_offset: Start MC offset of the range being translated
- * @end_mc_offset: MC offset of the first byte after the range (last MC
- * offset of the range is end_mc_offset - 1)
- * @start_phys_addr: system physical address corresponding to start_mc_addr
- */
-struct fsl_mc_addr_translation_range {
-	enum dprc_region_type mc_region_type;
-	u64 start_mc_offset;
-	u64 end_mc_offset;
-	phys_addr_t start_phys_addr;
-};
+#ifdef CONFIG_FSL_MC_BUS
+#define dev_is_fsl_mc(_dev) ((_dev)->bus == &fsl_mc_bus_type)
+#else
+/* If fsl-mc bus is not present device cannot belong to fsl-mc bus */
+#define dev_is_fsl_mc(_dev) (0)
+#endif
 
 /**
  * struct fsl_mc_resource_pool - Pool of MC resources of a given
@@ -107,13 +70,6 @@ struct fsl_mc_bus {
 #define to_fsl_mc_bus(_mc_dev) \
 	container_of(_mc_dev, struct fsl_mc_bus, mc_dev)
 
-int __must_check fsl_mc_device_add(struct dprc_obj_desc *obj_desc,
-				   struct fsl_mc_io *mc_io,
-				   struct device *parent_dev,
-				   struct fsl_mc_device **new_mc_dev);
-
-void fsl_mc_device_remove(struct fsl_mc_device *mc_dev);
-
 int dprc_scan_container(struct fsl_mc_device *mc_bus_dev);
 
 int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev,
@@ -127,13 +83,6 @@ int __init fsl_mc_allocator_driver_init(void);
 
 void fsl_mc_allocator_driver_exit(void);
 
-int __must_check fsl_mc_resource_allocate(struct fsl_mc_bus *mc_bus,
-					  enum fsl_mc_pool_type pool_type,
-					  struct fsl_mc_resource
-							  **new_resource);
-
-void fsl_mc_resource_free(struct fsl_mc_resource *resource);
-
 struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode,
 						struct msi_domain_info *info,
 						struct irq_domain *parent);
@@ -141,18 +90,22 @@ struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode,
 int fsl_mc_find_msi_domain(struct device *mc_platform_dev,
 			   struct irq_domain **mc_msi_domain);
 
-int fsl_mc_msi_domain_alloc_irqs(struct device *dev,
-				 unsigned int irq_count);
+int fsl_mc_populate_irq_pool(struct fsl_mc_bus *mc_bus,
+			     unsigned int irq_count);
+
+void fsl_mc_cleanup_irq_pool(struct fsl_mc_bus *mc_bus);
 
-void fsl_mc_msi_domain_free_irqs(struct device *dev);
+void fsl_mc_init_all_resource_pools(struct fsl_mc_device *mc_bus_dev);
 
-int __init its_fsl_mc_msi_init(void);
+void fsl_mc_cleanup_all_resource_pools(struct fsl_mc_device *mc_bus_dev);
 
-void its_fsl_mc_msi_cleanup(void);
+bool fsl_mc_bus_exists(void);
 
-int fsl_mc_populate_irq_pool(struct fsl_mc_bus *mc_bus,
-			     unsigned int irq_count);
+void fsl_mc_get_root_dprc(struct device *dev,
+			  struct device **root_dprc_dev);
 
-void fsl_mc_cleanup_irq_pool(struct fsl_mc_bus *mc_bus);
+bool fsl_mc_is_root_dprc(struct device *dev);
+
+extern struct bus_type fsl_mc_bus_type;
 
-#endif /* _FSL_MC_PRIVATE_H_ */
+#endif /* _FSL_MC_MCBUS_H_ */
diff --git a/drivers/staging/fsl-mc/include/mc-sys.h b/drivers/staging/fsl-mc/include/mc-sys.h
index c5038cc77240..89ad0cf54702 100644
--- a/drivers/staging/fsl-mc/include/mc-sys.h
+++ b/drivers/staging/fsl-mc/include/mc-sys.h
@@ -37,8 +37,6 @@
 
 #include <linux/types.h>
 #include <linux/errno.h>
-#include <linux/io.h>
-#include <linux/dma-mapping.h>
 #include <linux/mutex.h>
 #include <linux/spinlock.h>
 
@@ -95,19 +93,6 @@ struct fsl_mc_io {
 	};
 };
 
-int __must_check fsl_create_mc_io(struct device *dev,
-				  phys_addr_t mc_portal_phys_addr,
-				  u32 mc_portal_size,
-				  struct fsl_mc_device *dpmcp_dev,
-				  u32 flags, struct fsl_mc_io **new_mc_io);
-
-void fsl_destroy_mc_io(struct fsl_mc_io *mc_io);
-
-int fsl_mc_io_set_dpmcp(struct fsl_mc_io *mc_io,
-			struct fsl_mc_device *dpmcp_dev);
-
-void fsl_mc_io_unset_dpmcp(struct fsl_mc_io *mc_io);
-
 int mc_send_command(struct fsl_mc_io *mc_io, struct mc_command *cmd);
 
 #endif /* _FSL_MC_SYS_H */
diff --git a/drivers/staging/fsl-mc/include/mc.h b/drivers/staging/fsl-mc/include/mc.h
index 853cbf38a400..f6e720e84460 100644
--- a/drivers/staging/fsl-mc/include/mc.h
+++ b/drivers/staging/fsl-mc/include/mc.h
@@ -13,7 +13,6 @@
 
 #include <linux/device.h>
 #include <linux/mod_devicetable.h>
-#include <linux/list.h>
 #include <linux/interrupt.h>
 #include "../include/dprc.h"
 
@@ -21,7 +20,6 @@
 
 struct fsl_mc_device;
 struct fsl_mc_io;
-struct fsl_mc_bus;
 
 /**
  * struct fsl_mc_driver - MC object device driver object
@@ -112,11 +110,6 @@ struct fsl_mc_device_irq {
 #define FSL_MC_IS_DPRC	0x0001
 
 /**
- * Default DMA mask for devices on a fsl-mc bus
- */
-#define FSL_MC_DEFAULT_DMA_MASK	(~0ULL)
-
-/**
  * struct fsl_mc_device - MC object device object
  * @dev: Linux driver model device object
  * @dma_mask: Default DMA mask
@@ -187,8 +180,6 @@ int __must_check __fsl_mc_driver_register(struct fsl_mc_driver *fsl_mc_driver,
 
 void fsl_mc_driver_unregister(struct fsl_mc_driver *driver);
 
-bool fsl_mc_bus_exists(void);
-
 int __must_check fsl_mc_portal_allocate(struct fsl_mc_device *mc_dev,
 					u16 mc_io_flags,
 					struct fsl_mc_io **new_mc_io);
@@ -207,8 +198,4 @@ int __must_check fsl_mc_allocate_irqs(struct fsl_mc_device *mc_dev);
 
 void fsl_mc_free_irqs(struct fsl_mc_device *mc_dev);
 
-bool fsl_mc_is_root_dprc(struct device *dev);
-
-extern struct bus_type fsl_mc_bus_type;
-
 #endif /* _FSL_MC_H_ */
diff --git a/drivers/staging/fwserial/fwserial.c b/drivers/staging/fwserial/fwserial.c
index c241c0ae3f20..49c718b91e55 100644
--- a/drivers/staging/fwserial/fwserial.c
+++ b/drivers/staging/fwserial/fwserial.c
@@ -39,9 +39,9 @@ static int num_ttys = 4;	    /* # of std ttys to create per fw_card    */
 static bool auto_connect = true;    /* try to VIRT_CABLE to every peer        */
 static bool create_loop_dev = true; /* create a loopback device for each card */
 
-module_param_named(ttys, num_ttys, int, S_IRUGO | S_IWUSR);
-module_param_named(auto, auto_connect, bool, S_IRUGO | S_IWUSR);
-module_param_named(loop, create_loop_dev, bool, S_IRUGO | S_IWUSR);
+module_param_named(ttys, num_ttys, int, 0644);
+module_param_named(auto, auto_connect, bool, 0644);
+module_param_named(loop, create_loop_dev, bool, 0644);
 
 /*
  * Threshold below which the tty is woken for writing
diff --git a/drivers/staging/gdm724x/gdm_lte.c b/drivers/staging/gdm724x/gdm_lte.c
index bb552193e4ba..e72dfa9699f3 100644
--- a/drivers/staging/gdm724x/gdm_lte.c
+++ b/drivers/staging/gdm724x/gdm_lte.c
@@ -828,7 +828,7 @@ void start_rx_proc(struct phy_dev *phy_dev)
 				rx_complete, phy_dev, USB_COMPLETE);
 }
 
-static struct net_device_ops gdm_netdev_ops = {
+static const struct net_device_ops gdm_netdev_ops = {
 	.ndo_open			= gdm_lte_open,
 	.ndo_stop			= gdm_lte_close,
 	.ndo_set_config			= gdm_lte_set_config,
diff --git a/drivers/staging/gdm724x/gdm_mux.h b/drivers/staging/gdm724x/gdm_mux.h
index 3d50383c6ced..0871b8feec55 100644
--- a/drivers/staging/gdm724x/gdm_mux.h
+++ b/drivers/staging/gdm724x/gdm_mux.h
@@ -27,8 +27,8 @@
 
 #define START_FLAG 0xA512485A
 #define MUX_HEADER_SIZE 14
-#define MUX_TX_MAX_SIZE (1024*10)
-#define MUX_RX_MAX_SIZE (1024*30)
+#define MUX_TX_MAX_SIZE (1024 * 10)
+#define MUX_RX_MAX_SIZE (1024 * 30)
 #define AT_PKT_TYPE 0xF011
 #define DM_PKT_TYPE 0xF010
 
diff --git a/drivers/staging/gdm724x/gdm_tty.c b/drivers/staging/gdm724x/gdm_tty.c
index eb7e2523c354..ae396638f897 100644
--- a/drivers/staging/gdm724x/gdm_tty.c
+++ b/drivers/staging/gdm724x/gdm_tty.c
@@ -225,7 +225,6 @@ int register_lte_tty_device(struct tty_dev *tty_dev, struct device *device)
 	int j;
 
 	for (i = 0; i < TTY_MAX_COUNT; i++) {
-
 		gdm = kmalloc(sizeof(*gdm), GFP_KERNEL);
 		if (!gdm)
 			return -ENOMEM;
diff --git a/drivers/staging/gdm724x/gdm_usb.c b/drivers/staging/gdm724x/gdm_usb.c
index d650d772095b..15a7e81ec2d2 100644
--- a/drivers/staging/gdm724x/gdm_usb.c
+++ b/drivers/staging/gdm724x/gdm_usb.c
@@ -415,10 +415,10 @@ static void do_rx(struct work_struct *work)
 		switch (cmd_evt) {
 		case LTE_GET_INFORMATION_RESULT:
 			if (set_mac_address(hci->data, r->cb_data) == 0) {
-				ret = r->callback(r->cb_data,
-						  r->buf,
-						  r->urb->actual_length,
-						  KERNEL_THREAD);
+				r->callback(r->cb_data,
+					    r->buf,
+					    r->urb->actual_length,
+					    KERNEL_THREAD);
 			}
 			break;
 
diff --git a/drivers/staging/gdm724x/gdm_usb.h b/drivers/staging/gdm724x/gdm_usb.h
index e6486e71a428..ffb3d995097d 100644
--- a/drivers/staging/gdm724x/gdm_usb.h
+++ b/drivers/staging/gdm724x/gdm_usb.h
@@ -26,10 +26,10 @@
 #define PM_SUSPEND 1
 #define AUTO_SUSPEND_TIMER 5000 /* ms */
 
-#define RX_BUF_SIZE		(1024*32)
-#define TX_BUF_SIZE		(1024*32)
+#define RX_BUF_SIZE		(1024 * 32)
+#define TX_BUF_SIZE		(1024 * 32)
 #define SDU_BUF_SIZE	2048
-#define MAX_SDU_SIZE	(1024*30)
+#define MAX_SDU_SIZE	(1024 * 30)
 #define MAX_PACKET_IN_MULTI_SDU	256
 
 #define VID_GCT			0x1076
diff --git a/drivers/staging/gdm724x/hci_packet.h b/drivers/staging/gdm724x/hci_packet.h
index dbc4446cf78d..4644f84038c9 100644
--- a/drivers/staging/gdm724x/hci_packet.h
+++ b/drivers/staging/gdm724x/hci_packet.h
@@ -89,5 +89,4 @@ struct hci_connect_ind {
 	u32 connect;
 } __packed;
 
-
 #endif /* _HCI_PACKET_H_ */
diff --git a/drivers/staging/gdm724x/netlink_k.c b/drivers/staging/gdm724x/netlink_k.c
index a0232e8aec10..abe242505882 100644
--- a/drivers/staging/gdm724x/netlink_k.c
+++ b/drivers/staging/gdm724x/netlink_k.c
@@ -14,6 +14,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/export.h>
+#include <linux/mutex.h>
 #include <linux/etherdevice.h>
 #include <linux/netlink.h>
 #include <asm/byteorder.h>
@@ -21,13 +22,7 @@
 
 #include "netlink_k.h"
 
-#if defined(DEFINE_MUTEX)
 static DEFINE_MUTEX(netlink_mutex);
-#else
-static struct semaphore netlink_mutex;
-#define mutex_lock(x)		down(x)
-#define mutex_unlock(x)		up(x)
-#endif
 
 #define ND_MAX_GROUP		30
 #define ND_IFINDEX_LEN		sizeof(int)
@@ -96,10 +91,6 @@ struct sock *netlink_init(int unit,
 		.input  = netlink_rcv,
 	};
 
-#if !defined(DEFINE_MUTEX)
-	init_MUTEX(&netlink_mutex);
-#endif
-
 	sock = netlink_kernel_create(&init_net, unit, &cfg);
 
 	if (sock)
diff --git a/drivers/staging/greybus/Documentation/firmware/authenticate.c b/drivers/staging/greybus/Documentation/firmware/authenticate.c
new file mode 100644
index 000000000000..ab0688ad1e37
--- /dev/null
+++ b/drivers/staging/greybus/Documentation/firmware/authenticate.c
@@ -0,0 +1,139 @@
+/*
+ * Sample code to test CAP protocol
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details.
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name of Google Inc. or Linaro Ltd. nor the names of
+ *    its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written
+ *    permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GOOGLE INC. OR
+ * LINARO LTD. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "../../greybus_authentication.h"
+
+struct cap_ioc_get_endpoint_uid uid;
+struct cap_ioc_get_ims_certificate cert = {
+	.certificate_class = 0,
+	.certificate_id = 0,
+};
+
+struct cap_ioc_authenticate authenticate = {
+	.auth_type = 0,
+	.challenge = {0},
+};
+
+int main(int argc, char *argv[])
+{
+	unsigned int timeout = 10000;
+	char *capdev;
+	int fd, ret;
+
+	/* Make sure arguments are correct */
+	if (argc != 2) {
+		printf("\nUsage: ./firmware <Path of the gb-cap-X dev>\n");
+		return 0;
+	}
+
+	capdev = argv[1];
+
+	printf("Opening %s authentication device\n", capdev);
+
+	fd = open(capdev, O_RDWR);
+	if (fd < 0) {
+		printf("Failed to open: %s\n", capdev);
+		return -1;
+	}
+
+	/* Get UID */
+	printf("Get UID\n");
+
+	ret = ioctl(fd, CAP_IOC_GET_ENDPOINT_UID, &uid);
+	if (ret < 0) {
+		printf("Failed to get UID: %s (%d)\n", capdev, ret);
+		ret = -1;
+		goto close_fd;
+	}
+
+	printf("UID received: 0x%llx\n", *(long long unsigned int *)(uid.uid));
+
+	/* Get certificate */
+	printf("Get IMS certificate\n");
+
+	ret = ioctl(fd, CAP_IOC_GET_IMS_CERTIFICATE, &cert);
+	if (ret < 0) {
+		printf("Failed to get IMS certificate: %s (%d)\n", capdev, ret);
+		ret = -1;
+		goto close_fd;
+	}
+
+	printf("IMS Certificate size: %d\n", cert.cert_size);
+
+	/* Authenticate */
+	printf("Authenticate module\n");
+
+	memcpy(authenticate.uid, uid.uid, 8);
+
+	ret = ioctl(fd, CAP_IOC_AUTHENTICATE, &authenticate);
+	if (ret < 0) {
+		printf("Failed to authenticate module: %s (%d)\n", capdev, ret);
+		ret = -1;
+		goto close_fd;
+	}
+
+	printf("Authenticated, result (%02x), sig-size (%02x)\n",
+		authenticate.result_code, authenticate.signature_size);
+
+close_fd:
+	close(fd);
+
+	return ret;
+}
diff --git a/drivers/staging/greybus/Documentation/firmware/firmware-management b/drivers/staging/greybus/Documentation/firmware/firmware-management
new file mode 100644
index 000000000000..7918257e5b3b
--- /dev/null
+++ b/drivers/staging/greybus/Documentation/firmware/firmware-management
@@ -0,0 +1,333 @@
+
+Firmware Management
+-------------------
+ Copyright 2016 Google Inc.
+ Copyright 2016 Linaro Ltd.
+
+Interface-Manifest
+------------------
+
+All firmware packages on the Modules or Interfaces are managed by a special
+Firmware Management Protocol. To support Firmware Management by the AP, the
+Interface Manifest shall at least contain the Firmware Management Bundle and a
+Firmware Management Protocol CPort within it.
+
+The bundle may contain additional CPorts based on the extra functionality
+required to manage firmware packages.
+
+For example, this is how the Firmware Management part of the Interface Manifest
+may look like:
+
+	; Firmware Management Bundle (Bundle 1):
+	[bundle-descriptor 1]
+	class = 0x16
+
+	; (Mandatory) Firmware Management Protocol on CPort 1
+	[cport-descriptor 2]
+	bundle = 1
+	protocol = 0x18
+
+	; (Optional) Firmware Download Protocol on CPort 2
+	[cport-descriptor 1]
+	bundle = 1
+	protocol = 0x17
+
+	; (Optional) SPI protocol on CPort 3
+	[cport-descriptor 3]
+	bundle = 1
+	protocol = 0x0b
+
+	; (Optional) Component Authentication Protocol (CAP) on CPort 4
+	[cport-descriptor 4]
+	bundle = 1
+	protocol = 0x19
+
+
+Sysfs Interfaces - Firmware Management
+--------------------------------------
+
+The Firmware Management Protocol interacts with Userspace using the character
+device interface. The character device will be present in /dev/ directory
+and will be named gb-fw-mgmt-<N>. The number <N> is assigned at runtime.
+
+Identifying the Character Device
+================================
+
+There can be multiple devices present in /dev/ directory with name gb-fw-mgmt-N
+and user first needs to identify the character device used for
+firmware-management for a particular interface.
+
+The Firmware Management core creates a device of class 'gb_fw_mgmt', which shall
+be used by the user to identify the right character device for it. The class
+device is created within the Bundle directory for a particular Interface.
+
+For example this is how the class-device can be present:
+
+/sys/bus/greybus/devices/1-1/1-1.1/1-1.1.1/gb_fw_mgmt/gb-fw-mgmt-0
+
+The last name in this path: gb-fw-mgmt-0 is precisely the name of the char
+device and so the device in this case will be:
+
+/dev/gb-fw-mgmt-0.
+
+Operations on the Char device
+=============================
+
+The Character device (gb-fw-mgmt-0 in example) can be opened by the userspace
+application and it can perform various 'ioctl' operations on the device. The
+device doesn't support any read/write operations.
+
+Following are the IOCTLs and their data structures available to the user:
+
+/* IOCTL support */
+#define GB_FW_LOAD_METHOD_UNIPRO		0x01
+#define GB_FW_LOAD_METHOD_INTERNAL		0x02
+
+#define GB_FW_LOAD_STATUS_FAILED		0x00
+#define GB_FW_LOAD_STATUS_UNVALIDATED		0x01
+#define GB_FW_LOAD_STATUS_VALIDATED		0x02
+#define GB_FW_LOAD_STATUS_VALIDATION_FAILED	0x03
+
+#define GB_FW_BACKEND_FW_STATUS_SUCCESS		0x01
+#define GB_FW_BACKEND_FW_STATUS_FAIL_FIND	0x02
+#define GB_FW_BACKEND_FW_STATUS_FAIL_FETCH	0x03
+#define GB_FW_BACKEND_FW_STATUS_FAIL_WRITE	0x04
+#define GB_FW_BACKEND_FW_STATUS_INT		0x05
+#define GB_FW_BACKEND_FW_STATUS_RETRY		0x06
+#define GB_FW_BACKEND_FW_STATUS_NOT_SUPPORTED	0x07
+
+#define GB_FW_BACKEND_VERSION_STATUS_SUCCESS		0x01
+#define GB_FW_BACKEND_VERSION_STATUS_NOT_AVAILABLE	0x02
+#define GB_FW_BACKEND_VERSION_STATUS_NOT_SUPPORTED	0x03
+#define GB_FW_BACKEND_VERSION_STATUS_RETRY		0x04
+#define GB_FW_BACKEND_VERSION_STATUS_FAIL_INT		0x05
+
+
+struct fw_mgmt_ioc_get_intf_version {
+	__u8 firmware_tag[GB_FIRMWARE_U_TAG_MAX_SIZE];
+	__u16 major;
+	__u16 minor;
+} __attribute__ ((__packed__));
+
+struct fw_mgmt_ioc_get_backend_version {
+	__u8 firmware_tag[GB_FIRMWARE_U_TAG_MAX_SIZE];
+	__u16 major;
+	__u16 minor;
+	__u8 status;
+} __attribute__ ((__packed__));
+
+struct fw_mgmt_ioc_intf_load_and_validate {
+	__u8			firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE];
+	__u8			load_method;
+	__u8			status;
+	__u16			major;
+	__u16			minor;
+} __packed;
+
+struct fw_mgmt_ioc_backend_fw_update {
+	__u8			firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE];
+	__u8			status;
+} __packed;
+
+#define FW_MGMT_IOCTL_BASE			'S'
+#define FW_MGMT_IOC_GET_INTF_FW			_IOR(FW_MGMT_IOCTL_BASE, 0, struct fw_mgmt_ioc_get_intf_version)
+#define FW_MGMT_IOC_GET_BACKEND_FW		_IOWR(FW_MGMT_IOCTL_BASE, 1, struct fw_mgmt_ioc_get_backend_version)
+#define FW_MGMT_IOC_INTF_LOAD_AND_VALIDATE	_IOWR(FW_MGMT_IOCTL_BASE, 2, struct fw_mgmt_ioc_intf_load_and_validate)
+#define FW_MGMT_IOC_INTF_BACKEND_FW_UPDATE	_IOWR(FW_MGMT_IOCTL_BASE, 3, struct fw_mgmt_ioc_backend_fw_update)
+#define FW_MGMT_IOC_SET_TIMEOUT_MS		_IOW(FW_MGMT_IOCTL_BASE, 4, unsigned int)
+#define FW_MGMT_IOC_MODE_SWITCH			_IO(FW_MGMT_IOCTL_BASE, 5)
+
+1. FW_MGMT_IOC_GET_INTF_FW:
+
+   This ioctl shall be used by the user to get the version and firmware-tag of
+   the currently running Interface Firmware. All the fields of the 'struct
+   fw_mgmt_ioc_get_fw' are filled by the kernel.
+
+2. FW_MGMT_IOC_GET_BACKEND_FW:
+
+   This ioctl shall be used by the user to get the version of a currently
+   running Backend Interface Firmware identified by a firmware-tag. The user is
+   required to fill the 'firmware_tag' field of the 'struct fw_mgmt_ioc_get_fw'
+   in this case. The 'major' and 'minor' fields are set by the kernel in
+   response.
+
+3. FW_MGMT_IOC_INTF_LOAD_AND_VALIDATE:
+
+   This ioctl shall be used by the user to load an Interface Firmware package on
+   an Interface. The user needs to fill the 'firmware_tag' and 'load_method'
+   fields of the 'struct fw_mgmt_ioc_intf_load_and_validate'. The 'status',
+   'major' and 'minor' fields are set by the kernel in response.
+
+4. FW_MGMT_IOC_INTF_BACKEND_FW_UPDATE:
+
+   This ioctl shall be used by the user to request an Interface to update a
+   Backend Interface Firmware.  The user is required to fill the 'firmware_tag'
+   field of the 'struct fw_mgmt_ioc_get_fw' in this case. The 'status' field is
+   set by the kernel in response.
+
+5. FW_MGMT_IOC_SET_TIMEOUT_MS:
+
+   This ioctl shall be used by the user to increase the timeout interval within
+   which the firmware must get loaded by the Module. The default timeout is 1
+   second. The user needs to pass the timeout in milliseconds.
+
+6. FW_MGMT_IOC_MODE_SWITCH:
+
+   This ioctl shall be used by the user to mode-switch the module to the
+   previously loaded interface firmware. If the interface firmware isn't loaded
+   previously, or if another unsuccessful FW_MGMT_IOC_INTF_LOAD_AND_VALIDATE
+   operation is started after loading interface firmware, then the firmware core
+   wouldn't allow mode-switch.
+
+
+Sysfs Interfaces - Authentication
+---------------------------------
+
+The Component Authentication Protocol interacts with Userspace using the
+character device interface. The character device will be present in /dev/
+directory and will be named gb-authenticate-<N>. The number <N> is assigned at
+runtime.
+
+Identifying the Character Device
+================================
+
+There can be multiple devices present in /dev/ directory with name
+gb-authenticate-N and user first needs to identify the character device used for
+authentication a of particular interface.
+
+The Authentication core creates a device of class 'gb_authenticate', which shall
+be used by the user to identify the right character device for it. The class
+device is created within the Bundle directory for a particular Interface.
+
+For example this is how the class-device can be present:
+
+/sys/bus/greybus/devices/1-1/1-1.1/1-1.1.1/gb_authenticate/gb-authenticate-0
+
+The last name in this path: gb-authenticate-0 is precisely the name of the char
+device and so the device in this case will be:
+
+/dev/gb-authenticate-0.
+
+Operations on the Char device
+=============================
+
+The Character device (/dev/gb-authenticate-0 in above example) can be opened by
+the userspace application and it can perform various 'ioctl' operations on the
+device. The device doesn't support any read/write operations.
+
+Following are the IOCTLs and their data structures available to the user:
+
+#define CAP_CERTIFICATE_MAX_SIZE	1600
+#define CAP_SIGNATURE_MAX_SIZE		320
+
+/* Certificate class types */
+#define CAP_CERT_IMS_EAPC		0x00000001
+#define CAP_CERT_IMS_EASC		0x00000002
+#define CAP_CERT_IMS_EARC		0x00000003
+#define CAP_CERT_IMS_IAPC		0x00000004
+#define CAP_CERT_IMS_IASC		0x00000005
+#define CAP_CERT_IMS_IARC		0x00000006
+
+/* IMS Certificate response result codes */
+#define CAP_IMS_RESULT_CERT_FOUND	0x00
+#define CAP_IMS_RESULT_CERT_CLASS_INVAL	0x01
+#define CAP_IMS_RESULT_CERT_CORRUPT	0x02
+#define CAP_IMS_RESULT_CERT_NOT_FOUND	0x03
+
+/* Authentication types */
+#define CAP_AUTH_IMS_PRI		0x00000001
+#define CAP_AUTH_IMS_SEC		0x00000002
+#define CAP_AUTH_IMS_RSA		0x00000003
+
+/* Authenticate response result codes */
+#define CAP_AUTH_RESULT_CR_SUCCESS	0x00
+#define CAP_AUTH_RESULT_CR_BAD_TYPE	0x01
+#define CAP_AUTH_RESULT_CR_WRONG_EP	0x02
+#define CAP_AUTH_RESULT_CR_NO_KEY	0x03
+#define CAP_AUTH_RESULT_CR_SIG_FAIL	0x04
+
+
+/* IOCTL support */
+struct cap_ioc_get_endpoint_uid {
+	__u8			uid[8];
+} __attribute__ ((__packed__));
+
+struct cap_ioc_get_ims_certificate {
+	__u32			certificate_class;
+	__u32			certificate_id;
+
+	__u8			result_code;
+	__u32			cert_size;
+	__u8			certificate[CAP_CERTIFICATE_MAX_SIZE];
+} __attribute__ ((__packed__));
+
+struct cap_ioc_authenticate {
+	__u32			auth_type;
+	__u8			uid[8];
+	__u8			challenge[32];
+
+	__u8			result_code;
+	__u8			response[64];
+	__u32			signature_size;
+	__u8			signature[CAP_SIGNATURE_MAX_SIZE];
+} __attribute__ ((__packed__));
+
+#define CAP_IOCTL_BASE			'C'
+#define CAP_IOC_GET_ENDPOINT_UID	_IOR(CAP_IOCTL_BASE, 0, struct cap_ioc_get_endpoint_uid)
+#define CAP_IOC_GET_IMS_CERTIFICATE	_IOWR(CAP_IOCTL_BASE, 1, struct cap_ioc_get_ims_certificate)
+#define CAP_IOC_AUTHENTICATE		_IOWR(CAP_IOCTL_BASE, 2, struct cap_ioc_authenticate)
+
+
+1. CAP_IOC_GET_ENDPOINT_UID:
+
+   This ioctl shall be used by the user to get the endpoint UID associated with
+   the Interface.  All the fields of the 'struct cap_ioc_get_endpoint_uid' are
+   filled by the kernel.
+
+2. CAP_IOC_GET_IMS_CERTIFICATE:
+
+   This ioctl shall be used by the user to retrieve one of the available
+   cryptographic certificates held by the Interface for use in Component
+   Authentication. The user is required to fill the 'certificate_class' and
+   'certificate_id' field of the 'struct cap_ioc_get_ims_certificate' in this
+   case. The other fields will be set by the kernel in response. The first
+   'cert_size' bytes of the 'certificate' shall be read by the user and others
+   must be discarded.
+
+3. CAP_IOC_AUTHENTICATE:
+
+   This ioctl shall be used by the user to authenticate the Module attached to
+   an Interface.  The user needs to fill the 'auth_type', 'uid', and 'challenge'
+   fields of the 'struct cap_ioc_authenticate'. The other fields will be set by
+   the kernel in response.  The first 'signature_size' bytes of the 'signature'
+   shall be read by the user and others must be discarded.
+
+
+Sysfs Interfaces - Firmware Download
+------------------------------------
+
+The Firmware Download Protocol uses the existing Linux Kernel's Firmware class
+and the interface provided to userspace are described in:
+Documentation/firmware_class/.
+
+
+Sysfs Interfaces - SPI Flash
+----------------------------
+
+The SPI flash is exposed in userspace as a MTD device and is created
+within the Bundle directory. For example, this is how the path may look like:
+
+$ ls /sys/bus/greybus/devices/1-1/1-1.1/1-1.1.1/spi_master/spi32766/spi32766.0/mtd
+mtd0    mtd0ro
+
+
+Sample Applications
+-------------------
+
+The current directory also provides a firmware.c test application, which can be
+referenced while developing userspace application to talk to firmware-management
+protocol.
+
+The current directory also provides a authenticate.c test application, which can
+be referenced while developing userspace application to talk to
+component authentication protocol.
diff --git a/drivers/staging/greybus/Documentation/firmware/firmware.c b/drivers/staging/greybus/Documentation/firmware/firmware.c
new file mode 100644
index 000000000000..ff9382401030
--- /dev/null
+++ b/drivers/staging/greybus/Documentation/firmware/firmware.c
@@ -0,0 +1,262 @@
+/*
+ * Sample code to test firmware-management protocol
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details.
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name of Google Inc. or Linaro Ltd. nor the names of
+ *    its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written
+ *    permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GOOGLE INC. OR
+ * LINARO LTD. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "../../greybus_firmware.h"
+
+#define FW_DEV_DEFAULT		"/dev/gb-fw-mgmt-0"
+#define FW_TAG_INT_DEFAULT	"s3f"
+#define FW_TAG_BCND_DEFAULT	"bf_01"
+#define FW_UPDATE_TYPE_DEFAULT	0
+#define FW_TIMEOUT_DEFAULT	10000;
+
+static const char *firmware_tag;
+static const char *fwdev = FW_DEV_DEFAULT;
+static int fw_update_type = FW_UPDATE_TYPE_DEFAULT;
+static int fw_timeout = FW_TIMEOUT_DEFAULT;
+
+static struct fw_mgmt_ioc_get_intf_version intf_fw_info;
+static struct fw_mgmt_ioc_get_backend_version backend_fw_info;
+static struct fw_mgmt_ioc_intf_load_and_validate intf_load;
+static struct fw_mgmt_ioc_backend_fw_update backend_update;
+
+static void usage(void)
+{
+	printf("\nUsage: ./firmware <gb-fw-mgmt-X (default: gb-fw-mgmt-0)> <interface: 0, backend: 1 (default: 0)> <firmware-tag> (default: \"s3f\"/\"bf_01\") <timeout (default: 10000 ms)>\n");
+}
+
+static int update_intf_firmware(int fd)
+{
+	int ret;
+
+	/* Get Interface Firmware Version */
+	printf("Get Interface Firmware Version\n");
+
+	ret = ioctl(fd, FW_MGMT_IOC_GET_INTF_FW, &intf_fw_info);
+	if (ret < 0) {
+		printf("Failed to get interface firmware version: %s (%d)\n",
+			fwdev, ret);
+		return -1;
+	}
+
+	printf("Interface Firmware tag (%s), major (%d), minor (%d)\n",
+		intf_fw_info.firmware_tag, intf_fw_info.major,
+		intf_fw_info.minor);
+
+	/* Try Interface Firmware load over Unipro */
+	printf("Loading Interface Firmware\n");
+
+	intf_load.load_method = GB_FW_U_LOAD_METHOD_UNIPRO;
+	intf_load.status = 0;
+	intf_load.major = 0;
+	intf_load.minor = 0;
+
+	strncpy((char *)&intf_load.firmware_tag, firmware_tag,
+		GB_FIRMWARE_U_TAG_MAX_SIZE);
+
+	ret = ioctl(fd, FW_MGMT_IOC_INTF_LOAD_AND_VALIDATE, &intf_load);
+	if (ret < 0) {
+		printf("Failed to load interface firmware: %s (%d)\n", fwdev,
+			ret);
+		return -1;
+	}
+
+	if (intf_load.status != GB_FW_U_LOAD_STATUS_VALIDATED &&
+	    intf_load.status != GB_FW_U_LOAD_STATUS_UNVALIDATED) {
+		printf("Load status says loading failed: %d\n",
+			intf_load.status);
+		return -1;
+	}
+
+	printf("Interface Firmware (%s) Load done: major: %d, minor: %d, status: %d\n",
+		firmware_tag, intf_load.major, intf_load.minor,
+		intf_load.status);
+
+	/* Initiate Mode-switch to the newly loaded firmware */
+	printf("Initiate Mode switch\n");
+
+	ret = ioctl(fd, FW_MGMT_IOC_MODE_SWITCH);
+	if (ret < 0)
+		printf("Failed to initiate mode-switch (%d)\n", ret);
+
+	return ret;
+}
+
+static int update_backend_firmware(int fd)
+{
+	int ret;
+
+	/* Get Backend Firmware Version */
+	printf("Getting Backend Firmware Version\n");
+
+	strncpy((char *)&backend_fw_info.firmware_tag, firmware_tag,
+		GB_FIRMWARE_U_TAG_MAX_SIZE);
+
+retry_fw_version:
+	ret = ioctl(fd, FW_MGMT_IOC_GET_BACKEND_FW, &backend_fw_info);
+	if (ret < 0) {
+		printf("Failed to get backend firmware version: %s (%d)\n",
+			fwdev, ret);
+		return -1;
+	}
+
+	printf("Backend Firmware tag (%s), major (%d), minor (%d), status (%d)\n",
+		backend_fw_info.firmware_tag, backend_fw_info.major,
+		backend_fw_info.minor, backend_fw_info.status);
+
+	if (backend_fw_info.status == GB_FW_U_BACKEND_VERSION_STATUS_RETRY)
+		goto retry_fw_version;
+
+	if ((backend_fw_info.status != GB_FW_U_BACKEND_VERSION_STATUS_SUCCESS)
+	    && (backend_fw_info.status != GB_FW_U_BACKEND_VERSION_STATUS_NOT_AVAILABLE)) {
+		printf("Failed to get backend firmware version: %s (%d)\n",
+			fwdev, backend_fw_info.status);
+		return -1;
+	}
+
+	/* Try Backend Firmware Update over Unipro */
+	printf("Updating Backend Firmware\n");
+
+	strncpy((char *)&backend_update.firmware_tag, firmware_tag,
+		GB_FIRMWARE_U_TAG_MAX_SIZE);
+
+retry_fw_update:
+	backend_update.status = 0;
+
+	ret = ioctl(fd, FW_MGMT_IOC_INTF_BACKEND_FW_UPDATE, &backend_update);
+	if (ret < 0) {
+		printf("Failed to load backend firmware: %s (%d)\n", fwdev, ret);
+		return -1;
+	}
+
+	if (backend_update.status == GB_FW_U_BACKEND_FW_STATUS_RETRY) {
+		printf("Retrying firmware update: %d\n", backend_update.status);
+		goto retry_fw_update;
+	}
+
+	if (backend_update.status != GB_FW_U_BACKEND_FW_STATUS_SUCCESS) {
+		printf("Load status says loading failed: %d\n",
+			backend_update.status);
+	} else {
+		printf("Backend Firmware (%s) Load done: status: %d\n",
+				firmware_tag, backend_update.status);
+	}
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	int fd, ret;
+
+	if (argc > 1 &&
+	    (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help"))) {
+		usage();
+		return -1;
+	}
+
+	if (argc > 1)
+		fwdev = argv[1];
+
+	if (argc > 2)
+		sscanf(argv[2], "%u", &fw_update_type);
+
+	if (argc > 3) {
+		firmware_tag = argv[3];
+	} else if (!fw_update_type) {
+		firmware_tag = FW_TAG_INT_DEFAULT;
+	} else {
+		firmware_tag = FW_TAG_BCND_DEFAULT;
+	}
+
+	if (argc > 4)
+		sscanf(argv[4], "%u", &fw_timeout);
+
+	printf("Trying Firmware update: fwdev: %s, type: %s, tag: %s, timeout: %d\n",
+		fwdev, fw_update_type == 0 ? "interface" : "backend",
+		firmware_tag, fw_timeout);
+
+	printf("Opening %s firmware management device\n", fwdev);
+
+	fd = open(fwdev, O_RDWR);
+	if (fd < 0) {
+		printf("Failed to open: %s\n", fwdev);
+		return -1;
+	}
+
+	/* Set Timeout */
+	printf("Setting timeout to %u ms\n", fw_timeout);
+
+	ret = ioctl(fd, FW_MGMT_IOC_SET_TIMEOUT_MS, &fw_timeout);
+	if (ret < 0) {
+		printf("Failed to set timeout: %s (%d)\n", fwdev, ret);
+		ret = -1;
+		goto close_fd;
+	}
+
+	if (!fw_update_type)
+		ret = update_intf_firmware(fd);
+	else
+		ret = update_backend_firmware(fd);
+
+close_fd:
+	close(fd);
+
+	return ret;
+}
diff --git a/drivers/staging/greybus/Documentation/sysfs-bus-greybus b/drivers/staging/greybus/Documentation/sysfs-bus-greybus
new file mode 100644
index 000000000000..2e998966cbe1
--- /dev/null
+++ b/drivers/staging/greybus/Documentation/sysfs-bus-greybus
@@ -0,0 +1,275 @@
+What:		/sys/bus/greybus/devices/greybusN
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The "root" greybus device for the Greybus device tree, or bus,
+		where N is a dynamically assigned 1-based id.
+
+What:		/sys/bus/greybus/devices/greybusN/bus_id
+Date:		April 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The ID of the "root" greybus device, or bus.
+
+What:		/sys/bus/greybus/devices/N-M
+Date:		March 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		A Module M on the bus N, where M is the 1-byte interface
+		ID of the module's primary interface.
+
+What:		/sys/bus/greybus/devices/N-M/eject
+Date:		March 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Writing a non-zero argument to this attibute disables the
+		module's interfaces before physically ejecting it.
+
+What:		/sys/bus/greybus/devices/N-M/module_id
+Date:		March 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The ID of a Greybus module, corresponding to the ID of its
+		primary interface.
+
+What:		/sys/bus/greybus/devices/N-M/num_interfaces
+Date:		March 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The number of interfaces of a module.
+
+What:		/sys/bus/greybus/devices/N-M.I
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		An Interface I on the bus N and module N-M, where I is the
+		1-byte interface ID.
+
+What:		/sys/bus/greybus/devices/N-M.I/current_now
+Date:		March 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Current measurement of the interface in microamps (uA)
+
+What:		/sys/bus/greybus/devices/N-M.I/ddbl1_manufacturer_id
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Unipro Device Descriptor Block Level 1 manufacturer ID for the
+		greybus Interface.
+
+What:		/sys/bus/greybus/devices/N-M.I/ddbl1_product_id
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Unipro Device Descriptor Block Level 1 product ID for the
+		greybus Interface.
+
+What:		/sys/bus/greybus/devices/N-M.I/interface_id
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The ID of a Greybus interface.
+
+What:		/sys/bus/greybus/devices/N-M.I/interface_type
+Date:		June 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The type of a Greybus interface; "dummy", "unipro", "greybus",
+		or "unknown".
+
+What:		/sys/bus/greybus/devices/N-M.I/power_now
+Date:		March 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Power measurement of the interface in microwatts (uW)
+
+What:		/sys/bus/greybus/devices/N-M.I/power_state
+Date:		March 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		This file reflects the power state of a Greybus interface. If
+		the value read from it is "on", then power is currently
+		supplied to the interface. Otherwise it will read "off" and
+		power is currently not supplied to the interface.
+
+		If the value read is "off", then writing "on" (or '1', 'y',
+		'Y') to this file will enable power to the interface and an
+		attempt to boot and possibly enumerate it will be made. Note
+		that on errors, the interface will again be powered down.
+
+		If the value read is "on", then writing "off" (or '0', 'n',
+		'N') to this file will power down the interface.
+
+What:		/sys/bus/greybus/devices/N-M.I/product_id
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Product ID of a Greybus interface.
+
+What:		/sys/bus/greybus/devices/N-M.I/serial_number
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Serial Number of the Greybus interface, represented by a 64 bit
+		hexadecimal number.
+
+What:		/sys/bus/greybus/devices/N-M.I/vendor_id
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Vendor ID of a Greybus interface.
+
+What:		/sys/bus/greybus/devices/N-M.I/voltage_now
+Date:		March 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Voltage measurement of the interface in microvolts (uV)
+
+What:		/sys/bus/greybus/devices/N-M.I.ctrl
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Abstract control device for interface I that represents the
+		current mode of an enumerated Greybus interface.
+
+What:		/sys/bus/greybus/devices/N-M.I.ctrl/product_string
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Product ID string of a Greybus interface.
+
+What:		/sys/bus/greybus/devices/N-M.I.ctrl/vendor_string
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Vendor ID string of a Greybus interface.
+
+What:		/sys/bus/greybus/devices/N-M.I.B
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		A bundle B on the Interface I, B is replaced by a 1-byte
+		number representing the bundle.
+
+What:		/sys/bus/greybus/devices/N-M.I.B/bundle_class
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The greybus class of the bundle B.
+
+What:		/sys/bus/greybus/devices/N-M.I.B/bundle_id
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The interface-unique id of the bundle B.
+
+What:		/sys/bus/greybus/devices/N-M.I.B/gpbX
+Date:		April 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The General Purpose Bridged PHY device of the bundle B,
+		where X is a dynamically assigned 0-based id.
+
+What:		/sys/bus/greybus/devices/N-M.I.B/state
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		A bundle has a state that is managed by the userspace
+		Endo process.  This file allows that Endo to signal
+		other Android HALs that the state of the bundle has
+		changed to a specific value.  When written to, any
+		process watching the file will be woken up, and the new
+		value can be read. It's a "poor-man's IPC", yes, but
+		simplifies the Android userspace code immensely.
+
+What:		/sys/bus/greybus/devices/N-svc
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The singleton SVC device of bus N.
+
+What:		/sys/bus/greybus/devices/N-svc/ap_intf_id
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The AP interface ID, a 1-byte non-zero integer which
+		defines the position of the AP module on the frame.
+		The interface positions are defined in the GMP
+		Module Developer Kit.
+
+What:		/sys/bus/greybus/devices/N-svc/endo_id
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The Endo ID, which is a 2-byte hexadecimal value
+		defined by the Endo layout scheme, documented in
+		the GMP Module Developer Kit.
+
+What:		/sys/bus/greybus/devices/N-svc/intf_eject
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		Write the number of the interface that you wish to
+		forcibly eject from the system.
+
+What:		/sys/bus/greybus/devices/N-svc/version
+Date:		October 2015
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		The version number of the firmware in the SVC device.
+
+What:		/sys/bus/greybus/devices/N-svc/watchdog
+Date:		October 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		If the SVC watchdog is enabled or not.  Writing 0 to this
+		file will disable the watchdog, writing 1 will enable it.
+
+What:		/sys/bus/greybus/devices/N-svc/watchdog_action
+Date:		July 2016
+KernelVersion:	4.XX
+Contact:	Greg Kroah-Hartman <greg@kroah.com>
+Description:
+		This attribute indicates the action to be performed upon SVC
+		watchdog bite.
+
+		The action can be one of the "reset" or "panic". Writing either
+		one of the "reset" or "panic" will change the behavior of SVC
+		watchdog bite. Default value is "reset".
+
+		"reset" means the UniPro subsystem is to be reset.
+
+		"panic" means SVC watchdog bite will cause kernel to panic.
diff --git a/drivers/staging/greybus/Kconfig b/drivers/staging/greybus/Kconfig
new file mode 100644
index 000000000000..50de2d72dde0
--- /dev/null
+++ b/drivers/staging/greybus/Kconfig
@@ -0,0 +1,219 @@
+menuconfig GREYBUS
+	tristate "Greybus support"
+	depends on SYSFS
+	---help---
+	  This option enables the Greybus driver core.  Greybus is an
+	  hardware protocol that was designed to provide Unipro with a
+	  sane application layer.  It was originally designed for the
+	  ARA project, a module phone system, but has shown up in other
+	  phones, and can be tunneled over other busses in order to
+	  control hardware devices.
+
+	  Say Y here to enable support for these types of drivers.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called greybus.ko
+
+if GREYBUS
+
+config GREYBUS_ES2
+	tristate "Greybus ES3 USB host controller"
+	depends on USB
+	---help---
+	  Select this option if you have a Toshiba ES3 USB device that
+	  acts as a Greybus "host controller".  This device is a bridge
+	  from a USB device to a Unipro network.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-es2.ko
+
+config GREYBUS_AUDIO
+	tristate "Greybus Audio Class driver"
+	depends on SOUND
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus Audio Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-audio.ko
+
+config GREYBUS_BOOTROM
+	tristate "Greybus Bootrom Class driver"
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus Bootrom Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-bootrom.ko
+
+config GREYBUS_CAMERA
+	tristate "Greybus Camera Class driver"
+	depends on MEDIA_SUPPORT && LEDS_CLASS_FLASH && BROKEN
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus Camera Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-camera.ko
+
+config GREYBUS_FIRMWARE
+	tristate "Greybus Firmware Download Class driver"
+	depends on SPI
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus Firmware Download Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-firmware.ko
+
+config GREYBUS_HID
+	tristate "Greybus HID Class driver"
+	depends on HID && INPUT
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus HID Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-hid.ko
+
+config GREYBUS_LIGHT
+	tristate "Greybus LED Class driver"
+	depends on LEDS_CLASS
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus LED Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-light.ko
+
+config GREYBUS_LOG
+	tristate "Greybus Debug Log Class driver"
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus Debug Log Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-log.ko
+
+config GREYBUS_LOOPBACK
+	tristate "Greybus Loopback Class driver"
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus Debug Log Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-log.ko
+
+config GREYBUS_POWER
+	tristate "Greybus Powersupply Class driver"
+	depends on POWER_SUPPLY
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus Powersupply Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-power-supply.ko
+
+config GREYBUS_RAW
+	tristate "Greybus Raw Class driver"
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus Raw Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-raw.ko
+
+config GREYBUS_VIBRATOR
+	tristate "Greybus Vibrator Motor Class driver"
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus Vibrator Motor Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-vibrator.ko
+
+menuconfig GREYBUS_BRIDGED_PHY
+	tristate "Greybus Bridged PHY Class drivers"
+	---help---
+	  Select this option to pick from a variety of Greybus Bridged
+	  PHY class drivers.  These drivers emulate a number of
+	  different "traditional" busses by tunneling them over Greybus.
+	  Examples of this include serial, SPI, USB, and others.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-phy.ko
+
+if GREYBUS_BRIDGED_PHY
+
+config GREYBUS_GPIO
+	tristate "Greybus GPIO Bridged PHY driver"
+	depends on GPIOLIB
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus GPIO Bridged PHY Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-gpio.ko
+
+config GREYBUS_I2C
+	tristate "Greybus I2C Bridged PHY driver"
+	depends on I2C
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus I2C Bridged PHY Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-i2c.ko
+
+config GREYBUS_PWM
+	tristate "Greybus PWM Bridged PHY driver"
+	depends on PWM
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus PWM Bridged PHY Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-pwm.ko
+
+config GREYBUS_SDIO
+	tristate "Greybus SDIO Bridged PHY driver"
+	depends on MMC
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus SDIO Bridged PHY Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-sdio.ko
+
+config GREYBUS_SPI
+	tristate "Greybus SPI Bridged PHY driver"
+	depends on SPI
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus SPI Bridged PHY Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-spi.ko
+
+config GREYBUS_UART
+	tristate "Greybus UART Bridged PHY driver"
+	depends on TTY
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus UART Bridged PHY Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-uart.ko
+
+config GREYBUS_USB
+	tristate "Greybus USB Host Bridged PHY driver"
+	depends on USB
+	---help---
+	  Select this option if you have a device that follows the
+	  Greybus USB Host Bridged PHY Class specification.
+
+	  To compile this code as a module, chose M here: the module
+	  will be called gb-usb.ko
+
+endif	# GREYBUS_BRIDGED_PHY
+endif	# GREYBUS
diff --git a/drivers/staging/greybus/Makefile b/drivers/staging/greybus/Makefile
new file mode 100644
index 000000000000..f337b7b70782
--- /dev/null
+++ b/drivers/staging/greybus/Makefile
@@ -0,0 +1,96 @@
+# Greybus core
+greybus-y :=	core.o		\
+		debugfs.o	\
+		hd.o		\
+		manifest.o	\
+		module.o	\
+		interface.o	\
+		bundle.o	\
+		connection.o	\
+		control.o	\
+		svc.o		\
+		svc_watchdog.o	\
+		operation.o	\
+		timesync.o	\
+		timesync_platform.o
+
+obj-$(CONFIG_GREYBUS)		+= greybus.o
+
+# needed for trace events
+ccflags-y += -I$(src)
+
+
+# Greybus Host controller drivers
+gb-es2-y := es2.o
+
+obj-$(CONFIG_GREYBUS_ES2)	+= gb-es2.o
+
+# Greybus class drivers
+gb-bootrom-y		:= bootrom.o
+gb-camera-y		:= camera.o
+gb-firmware-y		:= fw-core.o fw-download.o fw-management.o authentication.o
+gb-spilib-y		:= spilib.o
+gb-hid-y		:= hid.o
+gb-light-y		:= light.o
+gb-log-y		:= log.o
+gb-loopback-y		:= loopback.o
+gb-power-supply-y	:= power_supply.o
+gb-raw-y		:= raw.o
+gb-vibrator-y		:= vibrator.o
+
+obj-$(CONFIG_GREYBUS_BOOTROM)	+= gb-bootrom.o
+obj-$(CONFIG_GREYBUS_CAMERA)	+= gb-camera.o
+obj-$(CONFIG_GREYBUS_FIRMWARE)	+= gb-firmware.o gb-spilib.o
+obj-$(CONFIG_GREYBUS_HID)	+= gb-hid.o
+obj-$(CONFIG_GREYBUS_LIGHT)	+= gb-light.o
+obj-$(CONFIG_GREYBUS_LOG)	+= gb-log.o
+obj-$(CONFIG_GREYBUS_LOOPBACK)	+= gb-loopback.o
+obj-$(CONFIG_GREYBUS_POWER)	+= gb-power-supply.o
+obj-$(CONFIG_GREYBUS_RAW)	+= gb-raw.o
+obj-$(CONFIG_GREYBUS_VIBRATOR)	+= gb-vibrator.o
+
+# Greybus Audio is a bunch of modules
+gb-audio-module-y	:= audio_module.o audio_topology.o
+gb-audio-codec-y	:= audio_codec.o
+gb-audio-gb-y		:= audio_gb.o
+gb-audio-apbridgea-y	:= audio_apbridgea.o
+gb-audio-manager-y	:= audio_manager.o audio_manager_module.o
+
+# Greybus Audio sysfs helpers can be useful when debugging
+#GB_AUDIO_MANAGER_SYSFS ?= true
+#ifeq ($(GB_AUDIO_MANAGER_SYSFS),true)
+#gb-audio-manager-y += audio_manager_sysfs.o
+#ccflags-y += -DGB_AUDIO_MANAGER_SYSFS
+#endif
+
+obj-$(CONFIG_GREYBUS_AUDIO_MSM8994)	+= gb-audio-codec.o
+obj-$(CONFIG_GREYBUS_AUDIO_MSM8994)	+= gb-audio-module.o
+obj-$(CONFIG_GREYBUS_AUDIO)		+= gb-audio-gb.o
+obj-$(CONFIG_GREYBUS_AUDIO)		+= gb-audio-apbridgea.o
+obj-$(CONFIG_GREYBUS_AUDIO)		+= gb-audio-manager.o
+
+
+# Greybus Bridged PHY drivers
+gb-gbphy-y := gbphy.o
+gb-gpio-y := gpio.o
+gb-i2c-y := i2c.o
+gb-pwm-y := pwm.o
+gb-sdio-y := sdio.o
+gb-spi-y := spi.o
+gb-uart-y := uart.o
+gb-usb-y := usb.o
+
+obj-$(CONFIG_GREYBUS_BRIDGED_PHY)	+= gb-gbphy.o
+obj-$(CONFIG_GREYBUS_GPIO)		+= gb-gpio.o
+obj-$(CONFIG_GREYBUS_I2C)		+= gb-i2c.o
+obj-$(CONFIG_GREYBUS_PWM)		+= gb-pwm.o
+obj-$(CONFIG_GREYBUS_SDIO)		+= gb-sdio.o
+obj-$(CONFIG_GREYBUS_SPI)		+= gb-spi.o gb-spilib.o
+obj-$(CONFIG_GREYBUS_UART)		+= gb-uart.o
+obj-$(CONFIG_GREYBUS_USB)		+= gb-usb.o
+
+
+# Greybus Platform driver
+gb-arche-y	:= arche-platform.o arche-apb-ctrl.o
+
+obj-$(CONFIG_USB_HSIC_USB3613)	+= gb-arche.o
diff --git a/drivers/staging/greybus/arche-apb-ctrl.c b/drivers/staging/greybus/arche-apb-ctrl.c
new file mode 100644
index 000000000000..70323aa11f24
--- /dev/null
+++ b/drivers/staging/greybus/arche-apb-ctrl.c
@@ -0,0 +1,522 @@
+/*
+ * Arche Platform driver to control APB.
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/interrupt.h>
+#include <linux/of_gpio.h>
+#include <linux/of_irq.h>
+#include <linux/module.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/regulator/consumer.h>
+#include <linux/spinlock.h>
+#include "arche_platform.h"
+
+
+struct arche_apb_ctrl_drvdata {
+	/* Control GPIO signals to and from AP <=> AP Bridges */
+	int resetn_gpio;
+	int boot_ret_gpio;
+	int pwroff_gpio;
+	int wake_in_gpio;
+	int wake_out_gpio;
+	int pwrdn_gpio;
+
+	enum arche_platform_state state;
+	bool init_disabled;
+
+	struct regulator *vcore;
+	struct regulator *vio;
+
+	int clk_en_gpio;
+	struct clk *clk;
+
+	struct pinctrl *pinctrl;
+	struct pinctrl_state *pin_default;
+
+	/* V2: SPI Bus control  */
+	int spi_en_gpio;
+	bool spi_en_polarity_high;
+};
+
+/*
+ * Note that these low level api's are active high
+ */
+static inline void deassert_reset(unsigned int gpio)
+{
+	gpio_set_value(gpio, 1);
+}
+
+static inline void assert_reset(unsigned int gpio)
+{
+	gpio_set_value(gpio, 0);
+}
+
+/*
+ * Note: Please do not modify the below sequence, as it is as per the spec
+ */
+static int coldboot_seq(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct arche_apb_ctrl_drvdata *apb = platform_get_drvdata(pdev);
+	int ret;
+
+	if (apb->init_disabled ||
+			apb->state == ARCHE_PLATFORM_STATE_ACTIVE)
+		return 0;
+
+	/* Hold APB in reset state */
+	assert_reset(apb->resetn_gpio);
+
+	if (apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING &&
+			gpio_is_valid(apb->spi_en_gpio))
+		devm_gpio_free(dev, apb->spi_en_gpio);
+
+	/* Enable power to APB */
+	if (!IS_ERR(apb->vcore)) {
+		ret = regulator_enable(apb->vcore);
+		if (ret) {
+			dev_err(dev, "failed to enable core regulator\n");
+			return ret;
+		}
+	}
+
+	if (!IS_ERR(apb->vio)) {
+		ret = regulator_enable(apb->vio);
+		if (ret) {
+			dev_err(dev, "failed to enable IO regulator\n");
+			return ret;
+		}
+	}
+
+	apb_bootret_deassert(dev);
+
+	/* On DB3 clock was not mandatory */
+	if (gpio_is_valid(apb->clk_en_gpio))
+		gpio_set_value(apb->clk_en_gpio, 1);
+
+	usleep_range(100, 200);
+
+	/* deassert reset to APB : Active-low signal */
+	deassert_reset(apb->resetn_gpio);
+
+	apb->state = ARCHE_PLATFORM_STATE_ACTIVE;
+
+	return 0;
+}
+
+static int fw_flashing_seq(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct arche_apb_ctrl_drvdata *apb = platform_get_drvdata(pdev);
+	int ret;
+
+	if (apb->init_disabled ||
+			apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING)
+		return 0;
+
+	ret = regulator_enable(apb->vcore);
+	if (ret) {
+		dev_err(dev, "failed to enable core regulator\n");
+		return ret;
+	}
+
+	ret = regulator_enable(apb->vio);
+	if (ret) {
+		dev_err(dev, "failed to enable IO regulator\n");
+		return ret;
+	}
+
+	if (gpio_is_valid(apb->spi_en_gpio)) {
+		unsigned long flags;
+
+		if (apb->spi_en_polarity_high)
+			flags = GPIOF_OUT_INIT_HIGH;
+		else
+			flags = GPIOF_OUT_INIT_LOW;
+
+		ret = devm_gpio_request_one(dev, apb->spi_en_gpio,
+				flags, "apb_spi_en");
+		if (ret) {
+			dev_err(dev, "Failed requesting SPI bus en gpio %d\n",
+				apb->spi_en_gpio);
+			return ret;
+		}
+	}
+
+	/* for flashing device should be in reset state */
+	assert_reset(apb->resetn_gpio);
+	apb->state = ARCHE_PLATFORM_STATE_FW_FLASHING;
+
+	return 0;
+}
+
+static int standby_boot_seq(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct arche_apb_ctrl_drvdata *apb = platform_get_drvdata(pdev);
+
+	if (apb->init_disabled)
+		return 0;
+
+	/* Even if it is in OFF state, then we do not want to change the state */
+	if (apb->state == ARCHE_PLATFORM_STATE_STANDBY ||
+			apb->state == ARCHE_PLATFORM_STATE_OFF)
+		return 0;
+
+	if (apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING &&
+			gpio_is_valid(apb->spi_en_gpio))
+		devm_gpio_free(dev, apb->spi_en_gpio);
+
+	/*
+	 * As per WDM spec, do nothing
+	 *
+	 * Pasted from WDM spec,
+	 *  - A falling edge on POWEROFF_L is detected (a)
+	 *  - WDM enters standby mode, but no output signals are changed
+	 * */
+
+	/* TODO: POWEROFF_L is input to WDM module  */
+	apb->state = ARCHE_PLATFORM_STATE_STANDBY;
+	return 0;
+}
+
+static void poweroff_seq(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct arche_apb_ctrl_drvdata *apb = platform_get_drvdata(pdev);
+
+	if (apb->init_disabled || apb->state == ARCHE_PLATFORM_STATE_OFF)
+		return;
+
+	if (apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING &&
+			gpio_is_valid(apb->spi_en_gpio))
+		devm_gpio_free(dev, apb->spi_en_gpio);
+
+	/* disable the clock */
+	if (gpio_is_valid(apb->clk_en_gpio))
+		gpio_set_value(apb->clk_en_gpio, 0);
+
+	if (!IS_ERR(apb->vcore) && regulator_is_enabled(apb->vcore) > 0)
+		regulator_disable(apb->vcore);
+
+	if (!IS_ERR(apb->vio) && regulator_is_enabled(apb->vio) > 0)
+		regulator_disable(apb->vio);
+
+	/* As part of exit, put APB back in reset state */
+	assert_reset(apb->resetn_gpio);
+	apb->state = ARCHE_PLATFORM_STATE_OFF;
+
+	/* TODO: May have to send an event to SVC about this exit */
+}
+
+void apb_bootret_assert(struct device *dev)
+{
+	struct arche_apb_ctrl_drvdata *apb = dev_get_drvdata(dev);
+
+	gpio_set_value(apb->boot_ret_gpio, 1);
+}
+
+void apb_bootret_deassert(struct device *dev)
+{
+	struct arche_apb_ctrl_drvdata *apb = dev_get_drvdata(dev);
+
+	gpio_set_value(apb->boot_ret_gpio, 0);
+}
+
+int apb_ctrl_coldboot(struct device *dev)
+{
+	return coldboot_seq(to_platform_device(dev));
+}
+
+int apb_ctrl_fw_flashing(struct device *dev)
+{
+	return fw_flashing_seq(to_platform_device(dev));
+}
+
+int apb_ctrl_standby_boot(struct device *dev)
+{
+	return standby_boot_seq(to_platform_device(dev));
+}
+
+void apb_ctrl_poweroff(struct device *dev)
+{
+	poweroff_seq(to_platform_device(dev));
+}
+
+static ssize_t state_store(struct device *dev,
+		struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct arche_apb_ctrl_drvdata *apb = platform_get_drvdata(pdev);
+	int ret = 0;
+	bool is_disabled;
+
+	if (sysfs_streq(buf, "off")) {
+		if (apb->state == ARCHE_PLATFORM_STATE_OFF)
+			return count;
+
+		poweroff_seq(pdev);
+	} else if (sysfs_streq(buf, "active")) {
+		if (apb->state == ARCHE_PLATFORM_STATE_ACTIVE)
+			return count;
+
+		poweroff_seq(pdev);
+		is_disabled = apb->init_disabled;
+		apb->init_disabled = false;
+		ret = coldboot_seq(pdev);
+		if (ret)
+			apb->init_disabled = is_disabled;
+	} else if (sysfs_streq(buf, "standby")) {
+		if (apb->state == ARCHE_PLATFORM_STATE_STANDBY)
+			return count;
+
+		ret = standby_boot_seq(pdev);
+	} else if (sysfs_streq(buf, "fw_flashing")) {
+		if (apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING)
+			return count;
+
+		/* First we want to make sure we power off everything
+		 * and then enter FW flashing state */
+		poweroff_seq(pdev);
+		ret = fw_flashing_seq(pdev);
+	} else {
+		dev_err(dev, "unknown state\n");
+		ret = -EINVAL;
+	}
+
+	return ret ? ret : count;
+}
+
+static ssize_t state_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct arche_apb_ctrl_drvdata *apb = dev_get_drvdata(dev);
+
+	switch (apb->state) {
+	case ARCHE_PLATFORM_STATE_OFF:
+		return sprintf(buf, "off%s\n",
+				apb->init_disabled ? ",disabled" : "");
+	case ARCHE_PLATFORM_STATE_ACTIVE:
+		return sprintf(buf, "active\n");
+	case ARCHE_PLATFORM_STATE_STANDBY:
+		return sprintf(buf, "standby\n");
+	case ARCHE_PLATFORM_STATE_FW_FLASHING:
+		return sprintf(buf, "fw_flashing\n");
+	default:
+		return sprintf(buf, "unknown state\n");
+	}
+}
+
+static DEVICE_ATTR_RW(state);
+
+static int apb_ctrl_get_devtree_data(struct platform_device *pdev,
+		struct arche_apb_ctrl_drvdata *apb)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	int ret;
+
+	apb->resetn_gpio = of_get_named_gpio(np, "reset-gpios", 0);
+	if (apb->resetn_gpio < 0) {
+		dev_err(dev, "failed to get reset gpio\n");
+		return apb->resetn_gpio;
+	}
+	ret = devm_gpio_request_one(dev, apb->resetn_gpio,
+			GPIOF_OUT_INIT_LOW, "apb-reset");
+	if (ret) {
+		dev_err(dev, "Failed requesting reset gpio %d\n",
+				apb->resetn_gpio);
+		return ret;
+	}
+
+	apb->boot_ret_gpio = of_get_named_gpio(np, "boot-ret-gpios", 0);
+	if (apb->boot_ret_gpio < 0) {
+		dev_err(dev, "failed to get boot retention gpio\n");
+		return apb->boot_ret_gpio;
+	}
+	ret = devm_gpio_request_one(dev, apb->boot_ret_gpio,
+			GPIOF_OUT_INIT_LOW, "boot retention");
+	if (ret) {
+		dev_err(dev, "Failed requesting bootret gpio %d\n",
+				apb->boot_ret_gpio);
+		return ret;
+	}
+
+	/* It's not mandatory to support power management interface */
+	apb->pwroff_gpio = of_get_named_gpio(np, "pwr-off-gpios", 0);
+	if (apb->pwroff_gpio < 0) {
+		dev_err(dev, "failed to get power off gpio\n");
+		return apb->pwroff_gpio;
+	}
+	ret = devm_gpio_request_one(dev, apb->pwroff_gpio,
+			GPIOF_IN, "pwroff_n");
+	if (ret) {
+		dev_err(dev, "Failed requesting pwroff_n gpio %d\n",
+				apb->pwroff_gpio);
+		return ret;
+	}
+
+	/* Do not make clock mandatory as of now (for DB3) */
+	apb->clk_en_gpio = of_get_named_gpio(np, "clock-en-gpio", 0);
+	if (apb->clk_en_gpio < 0) {
+		dev_warn(dev, "failed to get clock en gpio\n");
+	} else if (gpio_is_valid(apb->clk_en_gpio)) {
+		ret = devm_gpio_request_one(dev, apb->clk_en_gpio,
+				GPIOF_OUT_INIT_LOW, "apb_clk_en");
+		if (ret) {
+			dev_warn(dev, "Failed requesting APB clock en gpio %d\n",
+					apb->clk_en_gpio);
+			return ret;
+		}
+	}
+
+	apb->pwrdn_gpio = of_get_named_gpio(np, "pwr-down-gpios", 0);
+	if (apb->pwrdn_gpio < 0)
+		dev_warn(dev, "failed to get power down gpio\n");
+
+	/* Regulators are optional, as we may have fixed supply coming in */
+	apb->vcore = devm_regulator_get(dev, "vcore");
+	if (IS_ERR(apb->vcore))
+		dev_warn(dev, "no core regulator found\n");
+
+	apb->vio = devm_regulator_get(dev, "vio");
+	if (IS_ERR(apb->vio))
+		dev_warn(dev, "no IO regulator found\n");
+
+	apb->pinctrl = devm_pinctrl_get(&pdev->dev);
+	if (IS_ERR(apb->pinctrl)) {
+		dev_err(&pdev->dev, "could not get pinctrl handle\n");
+		return PTR_ERR(apb->pinctrl);
+	}
+	apb->pin_default = pinctrl_lookup_state(apb->pinctrl, "default");
+	if (IS_ERR(apb->pin_default)) {
+		dev_err(&pdev->dev, "could not get default pin state\n");
+		return PTR_ERR(apb->pin_default);
+	}
+
+	/* Only applicable for platform >= V2 */
+	apb->spi_en_gpio = of_get_named_gpio(np, "spi-en-gpio", 0);
+	if (apb->spi_en_gpio >= 0) {
+		if (of_property_read_bool(pdev->dev.of_node,
+					"spi-en-active-high"))
+			apb->spi_en_polarity_high = true;
+	}
+
+	return 0;
+}
+
+static int arche_apb_ctrl_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct arche_apb_ctrl_drvdata *apb;
+	struct device *dev = &pdev->dev;
+
+	apb = devm_kzalloc(&pdev->dev, sizeof(*apb), GFP_KERNEL);
+	if (!apb)
+		return -ENOMEM;
+
+	ret = apb_ctrl_get_devtree_data(pdev, apb);
+	if (ret) {
+		dev_err(dev, "failed to get apb devicetree data %d\n", ret);
+		return ret;
+	}
+
+	/* Initially set APB to OFF state */
+	apb->state = ARCHE_PLATFORM_STATE_OFF;
+	/* Check whether device needs to be enabled on boot */
+	if (of_property_read_bool(pdev->dev.of_node, "arche,init-disable"))
+		apb->init_disabled = true;
+
+	platform_set_drvdata(pdev, apb);
+
+	/* Create sysfs interface to allow user to change state dynamically */
+	ret = device_create_file(dev, &dev_attr_state);
+	if (ret) {
+		dev_err(dev, "failed to create state file in sysfs\n");
+		return ret;
+	}
+
+	dev_info(&pdev->dev, "Device registered successfully\n");
+	return 0;
+}
+
+static int arche_apb_ctrl_remove(struct platform_device *pdev)
+{
+	device_remove_file(&pdev->dev, &dev_attr_state);
+	poweroff_seq(pdev);
+	platform_set_drvdata(pdev, NULL);
+
+	return 0;
+}
+
+static int arche_apb_ctrl_suspend(struct device *dev)
+{
+	/*
+	 * If timing profile permits, we may shutdown bridge
+	 * completely
+	 *
+	 * TODO: sequence ??
+	 *
+	 * Also, need to make sure we meet precondition for unipro suspend
+	 * Precondition: Definition ???
+	 */
+	return 0;
+}
+
+static int arche_apb_ctrl_resume(struct device *dev)
+{
+	/*
+	 * Atleast for ES2 we have to meet the delay requirement between
+	 * unipro switch and AP bridge init, depending on whether bridge is in
+	 * OFF state or standby state.
+	 *
+	 * Based on whether bridge is in standby or OFF state we may have to
+	 * assert multiple signals. Please refer to WDM spec, for more info.
+	 *
+	 */
+	return 0;
+}
+
+static void arche_apb_ctrl_shutdown(struct platform_device *pdev)
+{
+	apb_ctrl_poweroff(&pdev->dev);
+}
+
+static SIMPLE_DEV_PM_OPS(arche_apb_ctrl_pm_ops, arche_apb_ctrl_suspend,
+			 arche_apb_ctrl_resume);
+
+static const struct of_device_id arche_apb_ctrl_of_match[] = {
+	{ .compatible = "usbffff,2", },
+	{ },
+};
+
+static struct platform_driver arche_apb_ctrl_device_driver = {
+	.probe		= arche_apb_ctrl_probe,
+	.remove		= arche_apb_ctrl_remove,
+	.shutdown	= arche_apb_ctrl_shutdown,
+	.driver		= {
+		.name	= "arche-apb-ctrl",
+		.pm	= &arche_apb_ctrl_pm_ops,
+		.of_match_table = arche_apb_ctrl_of_match,
+	}
+};
+
+int __init arche_apb_init(void)
+{
+	return platform_driver_register(&arche_apb_ctrl_device_driver);
+}
+
+void __exit arche_apb_exit(void)
+{
+	platform_driver_unregister(&arche_apb_ctrl_device_driver);
+}
diff --git a/drivers/staging/greybus/arche-platform.c b/drivers/staging/greybus/arche-platform.c
new file mode 100644
index 000000000000..e36ee984485b
--- /dev/null
+++ b/drivers/staging/greybus/arche-platform.c
@@ -0,0 +1,827 @@
+/*
+ * Arche Platform driver to enable Unipro link.
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of_gpio.h>
+#include <linux/of_platform.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/suspend.h>
+#include <linux/time.h>
+#include "arche_platform.h"
+#include "greybus.h"
+
+#include <linux/usb/usb3613.h>
+
+#define WD_COLDBOOT_PULSE_WIDTH_MS	30
+
+enum svc_wakedetect_state {
+	WD_STATE_IDLE,			/* Default state = pulled high/low */
+	WD_STATE_BOOT_INIT,		/* WD = falling edge (low) */
+	WD_STATE_COLDBOOT_TRIG,		/* WD = rising edge (high), > 30msec */
+	WD_STATE_STANDBYBOOT_TRIG,	/* As of now not used ?? */
+	WD_STATE_COLDBOOT_START,	/* Cold boot process started */
+	WD_STATE_STANDBYBOOT_START,	/* Not used */
+	WD_STATE_TIMESYNC,
+};
+
+struct arche_platform_drvdata {
+	/* Control GPIO signals to and from AP <=> SVC */
+	int svc_reset_gpio;
+	bool is_reset_act_hi;
+	int svc_sysboot_gpio;
+	int wake_detect_gpio; /* bi-dir,maps to WAKE_MOD & WAKE_FRAME signals */
+
+	enum arche_platform_state state;
+
+	int svc_refclk_req;
+	struct clk *svc_ref_clk;
+
+	struct pinctrl *pinctrl;
+	struct pinctrl_state *pin_default;
+
+	int num_apbs;
+
+	enum svc_wakedetect_state wake_detect_state;
+	int wake_detect_irq;
+	spinlock_t wake_lock;			/* Protect wake_detect_state */
+	struct mutex platform_state_mutex;	/* Protect state */
+	wait_queue_head_t wq;			/* WQ for arche_pdata->state */
+	unsigned long wake_detect_start;
+	struct notifier_block pm_notifier;
+
+	struct device *dev;
+	struct gb_timesync_svc *timesync_svc_pdata;
+};
+
+static int arche_apb_bootret_assert(struct device *dev, void *data)
+{
+	apb_bootret_assert(dev);
+	return 0;
+}
+
+static int arche_apb_bootret_deassert(struct device *dev, void *data)
+{
+	apb_bootret_deassert(dev);
+	return 0;
+}
+
+/* Requires calling context to hold arche_pdata->platform_state_mutex */
+static void arche_platform_set_state(struct arche_platform_drvdata *arche_pdata,
+				     enum arche_platform_state state)
+{
+	arche_pdata->state = state;
+}
+
+/*
+ * arche_platform_change_state: Change the operational state
+ *
+ * This exported function allows external drivers to change the state
+ * of the arche-platform driver.
+ * Note that this function only supports transitions between two states
+ * with limited functionality.
+ *
+ *  - ARCHE_PLATFORM_STATE_TIME_SYNC:
+ *    Once set, allows timesync operations between SVC <=> AP and makes
+ *    sure that arche-platform driver ignores any subsequent events/pulses
+ *    from SVC over wake/detect.
+ *
+ *  - ARCHE_PLATFORM_STATE_ACTIVE:
+ *    Puts back driver to active state, where any pulse from SVC on wake/detect
+ *    line would trigger either cold/standby boot.
+ *    Note: Transition request from this function does not trigger cold/standby
+ *          boot. It just puts back driver book keeping variable back to ACTIVE
+ *          state and restores the interrupt.
+ *
+ * Returns -ENODEV if device not found, -EAGAIN if the driver cannot currently
+ * satisfy the requested state-transition or -EINVAL for all other
+ * state-transition requests.
+ */
+int arche_platform_change_state(enum arche_platform_state state,
+				struct gb_timesync_svc *timesync_svc_pdata)
+{
+	struct arche_platform_drvdata *arche_pdata;
+	struct platform_device *pdev;
+	struct device_node *np;
+	int ret = -EAGAIN;
+	unsigned long flags;
+
+	np = of_find_compatible_node(NULL, NULL, "google,arche-platform");
+	if (!np) {
+		pr_err("google,arche-platform device node not found\n");
+		return -ENODEV;
+	}
+
+	pdev = of_find_device_by_node(np);
+	if (!pdev) {
+		pr_err("arche-platform device not found\n");
+		return -ENODEV;
+	}
+
+	arche_pdata = platform_get_drvdata(pdev);
+
+	mutex_lock(&arche_pdata->platform_state_mutex);
+	spin_lock_irqsave(&arche_pdata->wake_lock, flags);
+
+	if (arche_pdata->state == state) {
+		ret = 0;
+		goto exit;
+	}
+
+	switch (state) {
+	case ARCHE_PLATFORM_STATE_TIME_SYNC:
+		if (arche_pdata->state != ARCHE_PLATFORM_STATE_ACTIVE) {
+			ret = -EINVAL;
+			goto exit;
+		}
+		if (arche_pdata->wake_detect_state != WD_STATE_IDLE) {
+			dev_err(arche_pdata->dev,
+				"driver busy with wake/detect line ops\n");
+			goto  exit;
+		}
+		device_for_each_child(arche_pdata->dev, NULL,
+				      arche_apb_bootret_assert);
+		arche_pdata->wake_detect_state = WD_STATE_TIMESYNC;
+		break;
+	case ARCHE_PLATFORM_STATE_ACTIVE:
+		if (arche_pdata->state != ARCHE_PLATFORM_STATE_TIME_SYNC) {
+			ret = -EINVAL;
+			goto exit;
+		}
+		device_for_each_child(arche_pdata->dev, NULL,
+				      arche_apb_bootret_deassert);
+		arche_pdata->wake_detect_state = WD_STATE_IDLE;
+		break;
+	case ARCHE_PLATFORM_STATE_OFF:
+	case ARCHE_PLATFORM_STATE_STANDBY:
+	case ARCHE_PLATFORM_STATE_FW_FLASHING:
+		dev_err(arche_pdata->dev, "busy, request to retry later\n");
+		goto exit;
+	default:
+		ret = -EINVAL;
+		dev_err(arche_pdata->dev,
+			"invalid state transition request\n");
+		goto exit;
+	}
+	arche_pdata->timesync_svc_pdata = timesync_svc_pdata;
+	arche_platform_set_state(arche_pdata, state);
+	if (state == ARCHE_PLATFORM_STATE_ACTIVE)
+		wake_up(&arche_pdata->wq);
+
+	ret = 0;
+exit:
+	spin_unlock_irqrestore(&arche_pdata->wake_lock, flags);
+	mutex_unlock(&arche_pdata->platform_state_mutex);
+	of_node_put(np);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(arche_platform_change_state);
+
+/* Requires arche_pdata->wake_lock is held by calling context */
+static void arche_platform_set_wake_detect_state(
+				struct arche_platform_drvdata *arche_pdata,
+				enum svc_wakedetect_state state)
+{
+	arche_pdata->wake_detect_state = state;
+}
+
+static inline void svc_reset_onoff(unsigned int gpio, bool onoff)
+{
+	gpio_set_value(gpio, onoff);
+}
+
+static int apb_cold_boot(struct device *dev, void *data)
+{
+	int ret;
+
+	ret = apb_ctrl_coldboot(dev);
+	if (ret)
+		dev_warn(dev, "failed to coldboot\n");
+
+	/*Child nodes are independent, so do not exit coldboot operation */
+	return 0;
+}
+
+static int apb_poweroff(struct device *dev, void *data)
+{
+	apb_ctrl_poweroff(dev);
+
+	/* Enable HUB3613 into HUB mode. */
+	if (usb3613_hub_mode_ctrl(false))
+		dev_warn(dev, "failed to control hub device\n");
+
+	return 0;
+}
+
+static void arche_platform_wd_irq_en(struct arche_platform_drvdata *arche_pdata)
+{
+	/* Enable interrupt here, to read event back from SVC */
+	gpio_direction_input(arche_pdata->wake_detect_gpio);
+	enable_irq(arche_pdata->wake_detect_irq);
+}
+
+static irqreturn_t arche_platform_wd_irq_thread(int irq, void *devid)
+{
+	struct arche_platform_drvdata *arche_pdata = devid;
+	unsigned long flags;
+
+	spin_lock_irqsave(&arche_pdata->wake_lock, flags);
+	if (arche_pdata->wake_detect_state != WD_STATE_COLDBOOT_TRIG) {
+		/* Something is wrong */
+		spin_unlock_irqrestore(&arche_pdata->wake_lock, flags);
+		return IRQ_HANDLED;
+	}
+
+	arche_platform_set_wake_detect_state(arche_pdata,
+					     WD_STATE_COLDBOOT_START);
+	spin_unlock_irqrestore(&arche_pdata->wake_lock, flags);
+
+	/* It should complete power cycle, so first make sure it is poweroff */
+	device_for_each_child(arche_pdata->dev, NULL, apb_poweroff);
+
+	/* Bring APB out of reset: cold boot sequence */
+	device_for_each_child(arche_pdata->dev, NULL, apb_cold_boot);
+
+	/* Enable HUB3613 into HUB mode. */
+	if (usb3613_hub_mode_ctrl(true))
+		dev_warn(arche_pdata->dev, "failed to control hub device\n");
+
+	spin_lock_irqsave(&arche_pdata->wake_lock, flags);
+	arche_platform_set_wake_detect_state(arche_pdata, WD_STATE_IDLE);
+	spin_unlock_irqrestore(&arche_pdata->wake_lock, flags);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t arche_platform_wd_irq(int irq, void *devid)
+{
+	struct arche_platform_drvdata *arche_pdata = devid;
+	unsigned long flags;
+
+	spin_lock_irqsave(&arche_pdata->wake_lock, flags);
+
+	if (arche_pdata->wake_detect_state == WD_STATE_TIMESYNC) {
+		gb_timesync_irq(arche_pdata->timesync_svc_pdata);
+		goto exit;
+	}
+
+	if (gpio_get_value(arche_pdata->wake_detect_gpio)) {
+		/* wake/detect rising */
+
+		/*
+		 * If wake/detect line goes high after low, within less than
+		 * 30msec, then standby boot sequence is initiated, which is not
+		 * supported/implemented as of now. So ignore it.
+		 */
+		if (arche_pdata->wake_detect_state == WD_STATE_BOOT_INIT) {
+			if (time_before(jiffies,
+					arche_pdata->wake_detect_start +
+					msecs_to_jiffies(WD_COLDBOOT_PULSE_WIDTH_MS))) {
+				arche_platform_set_wake_detect_state(arche_pdata,
+								     WD_STATE_IDLE);
+			} else {
+				/* Check we are not in middle of irq thread already */
+				if (arche_pdata->wake_detect_state !=
+						WD_STATE_COLDBOOT_START) {
+					arche_platform_set_wake_detect_state(arche_pdata,
+									     WD_STATE_COLDBOOT_TRIG);
+					spin_unlock_irqrestore(
+						&arche_pdata->wake_lock,
+						flags);
+					return IRQ_WAKE_THREAD;
+				}
+			}
+		}
+	} else {
+		/* wake/detect falling */
+		if (arche_pdata->wake_detect_state == WD_STATE_IDLE) {
+			arche_pdata->wake_detect_start = jiffies;
+			/*
+			 * In the begining, when wake/detect goes low (first time), we assume
+			 * it is meant for coldboot and set the flag. If wake/detect line stays low
+			 * beyond 30msec, then it is coldboot else fallback to standby boot.
+			 */
+			arche_platform_set_wake_detect_state(arche_pdata,
+							     WD_STATE_BOOT_INIT);
+		}
+	}
+
+exit:
+	spin_unlock_irqrestore(&arche_pdata->wake_lock, flags);
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * Requires arche_pdata->platform_state_mutex to be held
+ */
+static int arche_platform_coldboot_seq(struct arche_platform_drvdata *arche_pdata)
+{
+	int ret;
+
+	if (arche_pdata->state == ARCHE_PLATFORM_STATE_ACTIVE)
+		return 0;
+
+	dev_info(arche_pdata->dev, "Booting from cold boot state\n");
+
+	svc_reset_onoff(arche_pdata->svc_reset_gpio,
+			arche_pdata->is_reset_act_hi);
+
+	gpio_set_value(arche_pdata->svc_sysboot_gpio, 0);
+	usleep_range(100, 200);
+
+	ret = clk_prepare_enable(arche_pdata->svc_ref_clk);
+	if (ret) {
+		dev_err(arche_pdata->dev, "failed to enable svc_ref_clk: %d\n",
+				ret);
+		return ret;
+	}
+
+	/* bring SVC out of reset */
+	svc_reset_onoff(arche_pdata->svc_reset_gpio,
+			!arche_pdata->is_reset_act_hi);
+
+	arche_platform_set_state(arche_pdata, ARCHE_PLATFORM_STATE_ACTIVE);
+
+	return 0;
+}
+
+/*
+ * Requires arche_pdata->platform_state_mutex to be held
+ */
+static int arche_platform_fw_flashing_seq(struct arche_platform_drvdata *arche_pdata)
+{
+	int ret;
+
+	if (arche_pdata->state == ARCHE_PLATFORM_STATE_FW_FLASHING)
+		return 0;
+
+	dev_info(arche_pdata->dev, "Switching to FW flashing state\n");
+
+	svc_reset_onoff(arche_pdata->svc_reset_gpio,
+			arche_pdata->is_reset_act_hi);
+
+	gpio_set_value(arche_pdata->svc_sysboot_gpio, 1);
+
+	usleep_range(100, 200);
+
+	ret = clk_prepare_enable(arche_pdata->svc_ref_clk);
+	if (ret) {
+		dev_err(arche_pdata->dev, "failed to enable svc_ref_clk: %d\n",
+				ret);
+		return ret;
+	}
+
+	svc_reset_onoff(arche_pdata->svc_reset_gpio,
+			!arche_pdata->is_reset_act_hi);
+
+	arche_platform_set_state(arche_pdata, ARCHE_PLATFORM_STATE_FW_FLASHING);
+
+	return 0;
+}
+
+/*
+ * Requires arche_pdata->platform_state_mutex to be held
+ */
+static void arche_platform_poweroff_seq(struct arche_platform_drvdata *arche_pdata)
+{
+	unsigned long flags;
+
+	if (arche_pdata->state == ARCHE_PLATFORM_STATE_OFF)
+		return;
+
+	/* If in fw_flashing mode, then no need to repeate things again */
+	if (arche_pdata->state != ARCHE_PLATFORM_STATE_FW_FLASHING) {
+		disable_irq(arche_pdata->wake_detect_irq);
+
+		spin_lock_irqsave(&arche_pdata->wake_lock, flags);
+		arche_platform_set_wake_detect_state(arche_pdata,
+						     WD_STATE_IDLE);
+		spin_unlock_irqrestore(&arche_pdata->wake_lock, flags);
+	}
+
+	clk_disable_unprepare(arche_pdata->svc_ref_clk);
+
+	/* As part of exit, put APB back in reset state */
+	svc_reset_onoff(arche_pdata->svc_reset_gpio,
+			arche_pdata->is_reset_act_hi);
+
+	arche_platform_set_state(arche_pdata, ARCHE_PLATFORM_STATE_OFF);
+}
+
+static ssize_t state_store(struct device *dev,
+		struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct arche_platform_drvdata *arche_pdata = platform_get_drvdata(pdev);
+	int ret = 0;
+
+retry:
+	mutex_lock(&arche_pdata->platform_state_mutex);
+	if (arche_pdata->state == ARCHE_PLATFORM_STATE_TIME_SYNC) {
+		mutex_unlock(&arche_pdata->platform_state_mutex);
+		ret = wait_event_interruptible(
+			arche_pdata->wq,
+			arche_pdata->state != ARCHE_PLATFORM_STATE_TIME_SYNC);
+		if (ret)
+			return ret;
+		goto retry;
+	}
+
+	if (sysfs_streq(buf, "off")) {
+		if (arche_pdata->state == ARCHE_PLATFORM_STATE_OFF)
+			goto exit;
+
+		/*  If SVC goes down, bring down APB's as well */
+		device_for_each_child(arche_pdata->dev, NULL, apb_poweroff);
+
+		arche_platform_poweroff_seq(arche_pdata);
+
+	} else if (sysfs_streq(buf, "active")) {
+		if (arche_pdata->state == ARCHE_PLATFORM_STATE_ACTIVE)
+			goto exit;
+
+		/* First we want to make sure we power off everything
+		 * and then activate back again */
+		device_for_each_child(arche_pdata->dev, NULL, apb_poweroff);
+		arche_platform_poweroff_seq(arche_pdata);
+
+		arche_platform_wd_irq_en(arche_pdata);
+		ret = arche_platform_coldboot_seq(arche_pdata);
+		if (ret)
+			goto exit;
+
+	} else if (sysfs_streq(buf, "standby")) {
+		if (arche_pdata->state == ARCHE_PLATFORM_STATE_STANDBY)
+			goto exit;
+
+		dev_warn(arche_pdata->dev, "standby state not supported\n");
+	} else if (sysfs_streq(buf, "fw_flashing")) {
+		if (arche_pdata->state == ARCHE_PLATFORM_STATE_FW_FLASHING)
+			goto exit;
+
+		/*
+		 * Here we only control SVC.
+		 *
+		 * In case of FW_FLASHING mode we do not want to control
+		 * APBs, as in case of V2, SPI bus is shared between both
+		 * the APBs. So let user chose which APB he wants to flash.
+		 */
+		arche_platform_poweroff_seq(arche_pdata);
+
+		ret = arche_platform_fw_flashing_seq(arche_pdata);
+		if (ret)
+			goto exit;
+	} else {
+		dev_err(arche_pdata->dev, "unknown state\n");
+		ret = -EINVAL;
+	}
+
+exit:
+	mutex_unlock(&arche_pdata->platform_state_mutex);
+	return ret ? ret : count;
+}
+
+static ssize_t state_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct arche_platform_drvdata *arche_pdata = dev_get_drvdata(dev);
+
+	switch (arche_pdata->state) {
+	case ARCHE_PLATFORM_STATE_OFF:
+		return sprintf(buf, "off\n");
+	case ARCHE_PLATFORM_STATE_ACTIVE:
+		return sprintf(buf, "active\n");
+	case ARCHE_PLATFORM_STATE_STANDBY:
+		return sprintf(buf, "standby\n");
+	case ARCHE_PLATFORM_STATE_FW_FLASHING:
+		return sprintf(buf, "fw_flashing\n");
+	case ARCHE_PLATFORM_STATE_TIME_SYNC:
+		return sprintf(buf, "time_sync\n");
+	default:
+		return sprintf(buf, "unknown state\n");
+	}
+}
+
+static DEVICE_ATTR_RW(state);
+
+static int arche_platform_pm_notifier(struct notifier_block *notifier,
+				      unsigned long pm_event, void *unused)
+{
+	struct arche_platform_drvdata *arche_pdata =
+		container_of(notifier, struct arche_platform_drvdata,
+			     pm_notifier);
+	int ret = NOTIFY_DONE;
+
+	mutex_lock(&arche_pdata->platform_state_mutex);
+	switch (pm_event) {
+	case PM_SUSPEND_PREPARE:
+		if (arche_pdata->state != ARCHE_PLATFORM_STATE_ACTIVE) {
+			ret = NOTIFY_STOP;
+			break;
+		}
+		device_for_each_child(arche_pdata->dev, NULL, apb_poweroff);
+		arche_platform_poweroff_seq(arche_pdata);
+		break;
+	case PM_POST_SUSPEND:
+		if (arche_pdata->state != ARCHE_PLATFORM_STATE_OFF)
+			break;
+
+		arche_platform_wd_irq_en(arche_pdata);
+		arche_platform_coldboot_seq(arche_pdata);
+		break;
+	default:
+		break;
+	}
+	mutex_unlock(&arche_pdata->platform_state_mutex);
+
+	return ret;
+}
+
+static int arche_platform_probe(struct platform_device *pdev)
+{
+	struct arche_platform_drvdata *arche_pdata;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	int ret;
+
+	arche_pdata = devm_kzalloc(&pdev->dev, sizeof(*arche_pdata), GFP_KERNEL);
+	if (!arche_pdata)
+		return -ENOMEM;
+
+	/* setup svc reset gpio */
+	arche_pdata->is_reset_act_hi = of_property_read_bool(np,
+					"svc,reset-active-high");
+	arche_pdata->svc_reset_gpio = of_get_named_gpio(np, "svc,reset-gpio", 0);
+	if (arche_pdata->svc_reset_gpio < 0) {
+		dev_err(dev, "failed to get reset-gpio\n");
+		return arche_pdata->svc_reset_gpio;
+	}
+	ret = devm_gpio_request(dev, arche_pdata->svc_reset_gpio, "svc-reset");
+	if (ret) {
+		dev_err(dev, "failed to request svc-reset gpio:%d\n", ret);
+		return ret;
+	}
+	ret = gpio_direction_output(arche_pdata->svc_reset_gpio,
+					arche_pdata->is_reset_act_hi);
+	if (ret) {
+		dev_err(dev, "failed to set svc-reset gpio dir:%d\n", ret);
+		return ret;
+	}
+	arche_platform_set_state(arche_pdata, ARCHE_PLATFORM_STATE_OFF);
+
+	arche_pdata->svc_sysboot_gpio = of_get_named_gpio(np,
+					"svc,sysboot-gpio", 0);
+	if (arche_pdata->svc_sysboot_gpio < 0) {
+		dev_err(dev, "failed to get sysboot gpio\n");
+		return arche_pdata->svc_sysboot_gpio;
+	}
+	ret = devm_gpio_request(dev, arche_pdata->svc_sysboot_gpio, "sysboot0");
+	if (ret) {
+		dev_err(dev, "failed to request sysboot0 gpio:%d\n", ret);
+		return ret;
+	}
+	ret = gpio_direction_output(arche_pdata->svc_sysboot_gpio, 0);
+	if (ret) {
+		dev_err(dev, "failed to set svc-reset gpio dir:%d\n", ret);
+		return ret;
+	}
+
+	/* setup the clock request gpio first */
+	arche_pdata->svc_refclk_req = of_get_named_gpio(np,
+					"svc,refclk-req-gpio", 0);
+	if (arche_pdata->svc_refclk_req < 0) {
+		dev_err(dev, "failed to get svc clock-req gpio\n");
+		return arche_pdata->svc_refclk_req;
+	}
+	ret = devm_gpio_request(dev, arche_pdata->svc_refclk_req, "svc-clk-req");
+	if (ret) {
+		dev_err(dev, "failed to request svc-clk-req gpio: %d\n", ret);
+		return ret;
+	}
+	ret = gpio_direction_input(arche_pdata->svc_refclk_req);
+	if (ret) {
+		dev_err(dev, "failed to set svc-clk-req gpio dir :%d\n", ret);
+		return ret;
+	}
+
+	/* setup refclk2 to follow the pin */
+	arche_pdata->svc_ref_clk = devm_clk_get(dev, "svc_ref_clk");
+	if (IS_ERR(arche_pdata->svc_ref_clk)) {
+		ret = PTR_ERR(arche_pdata->svc_ref_clk);
+		dev_err(dev, "failed to get svc_ref_clk: %d\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, arche_pdata);
+
+	arche_pdata->num_apbs = of_get_child_count(np);
+	dev_dbg(dev, "Number of APB's available - %d\n", arche_pdata->num_apbs);
+
+	arche_pdata->wake_detect_gpio = of_get_named_gpio(np, "svc,wake-detect-gpio", 0);
+	if (arche_pdata->wake_detect_gpio < 0) {
+		dev_err(dev, "failed to get wake detect gpio\n");
+		return arche_pdata->wake_detect_gpio;
+	}
+
+	ret = devm_gpio_request(dev, arche_pdata->wake_detect_gpio, "wake detect");
+	if (ret) {
+		dev_err(dev, "Failed requesting wake_detect gpio %d\n",
+				arche_pdata->wake_detect_gpio);
+		return ret;
+	}
+
+	arche_platform_set_wake_detect_state(arche_pdata, WD_STATE_IDLE);
+
+	arche_pdata->dev = &pdev->dev;
+
+	spin_lock_init(&arche_pdata->wake_lock);
+	mutex_init(&arche_pdata->platform_state_mutex);
+	init_waitqueue_head(&arche_pdata->wq);
+	arche_pdata->wake_detect_irq =
+		gpio_to_irq(arche_pdata->wake_detect_gpio);
+
+	ret = devm_request_threaded_irq(dev, arche_pdata->wake_detect_irq,
+			arche_platform_wd_irq,
+			arche_platform_wd_irq_thread,
+			IRQF_TRIGGER_FALLING | IRQF_TRIGGER_RISING | IRQF_ONESHOT,
+			dev_name(dev), arche_pdata);
+	if (ret) {
+		dev_err(dev, "failed to request wake detect IRQ %d\n", ret);
+		return ret;
+	}
+	disable_irq(arche_pdata->wake_detect_irq);
+
+	ret = device_create_file(dev, &dev_attr_state);
+	if (ret) {
+		dev_err(dev, "failed to create state file in sysfs\n");
+		return ret;
+	}
+
+	ret = of_platform_populate(np, NULL, NULL, dev);
+	if (ret) {
+		dev_err(dev, "failed to populate child nodes %d\n", ret);
+		goto err_device_remove;
+	}
+
+	arche_pdata->pm_notifier.notifier_call = arche_platform_pm_notifier;
+	ret = register_pm_notifier(&arche_pdata->pm_notifier);
+
+	if (ret) {
+		dev_err(dev, "failed to register pm notifier %d\n", ret);
+		goto err_device_remove;
+	}
+
+	/* Register callback pointer */
+	arche_platform_change_state_cb = arche_platform_change_state;
+
+	/* Explicitly power off if requested */
+	if (!of_property_read_bool(pdev->dev.of_node, "arche,init-off")) {
+		mutex_lock(&arche_pdata->platform_state_mutex);
+		ret = arche_platform_coldboot_seq(arche_pdata);
+		if (ret) {
+			dev_err(dev, "Failed to cold boot svc %d\n", ret);
+			goto err_coldboot;
+		}
+		arche_platform_wd_irq_en(arche_pdata);
+		mutex_unlock(&arche_pdata->platform_state_mutex);
+	}
+
+	dev_info(dev, "Device registered successfully\n");
+	return 0;
+
+err_coldboot:
+	mutex_unlock(&arche_pdata->platform_state_mutex);
+err_device_remove:
+	device_remove_file(&pdev->dev, &dev_attr_state);
+	return ret;
+}
+
+static int arche_remove_child(struct device *dev, void *unused)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	platform_device_unregister(pdev);
+
+	return 0;
+}
+
+static int arche_platform_remove(struct platform_device *pdev)
+{
+	struct arche_platform_drvdata *arche_pdata = platform_get_drvdata(pdev);
+
+	unregister_pm_notifier(&arche_pdata->pm_notifier);
+	device_remove_file(&pdev->dev, &dev_attr_state);
+	device_for_each_child(&pdev->dev, NULL, arche_remove_child);
+	arche_platform_poweroff_seq(arche_pdata);
+	platform_set_drvdata(pdev, NULL);
+
+	if (usb3613_hub_mode_ctrl(false))
+		dev_warn(arche_pdata->dev, "failed to control hub device\n");
+		/* TODO: Should we do anything more here ?? */
+	return 0;
+}
+
+static int arche_platform_suspend(struct device *dev)
+{
+	/*
+	 * If timing profile premits, we may shutdown bridge
+	 * completely
+	 *
+	 * TODO: sequence ??
+	 *
+	 * Also, need to make sure we meet precondition for unipro suspend
+	 * Precondition: Definition ???
+	 */
+	return 0;
+}
+
+static int arche_platform_resume(struct device *dev)
+{
+	/*
+	 * Atleast for ES2 we have to meet the delay requirement between
+	 * unipro switch and AP bridge init, depending on whether bridge is in
+	 * OFF state or standby state.
+	 *
+	 * Based on whether bridge is in standby or OFF state we may have to
+	 * assert multiple signals. Please refer to WDM spec, for more info.
+	 *
+	 */
+	return 0;
+}
+
+static void arche_platform_shutdown(struct platform_device *pdev)
+{
+	struct arche_platform_drvdata *arche_pdata = platform_get_drvdata(pdev);
+
+	arche_platform_poweroff_seq(arche_pdata);
+
+	usb3613_hub_mode_ctrl(false);
+}
+
+static SIMPLE_DEV_PM_OPS(arche_platform_pm_ops,
+			arche_platform_suspend,
+			arche_platform_resume);
+
+static const struct of_device_id arche_platform_of_match[] = {
+	{ .compatible = "google,arche-platform", }, /* Use PID/VID of SVC device */
+	{ },
+};
+
+static const struct of_device_id arche_combined_id[] = {
+	{ .compatible = "google,arche-platform", }, /* Use PID/VID of SVC device */
+	{ .compatible = "usbffff,2", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, arche_combined_id);
+
+static struct platform_driver arche_platform_device_driver = {
+	.probe		= arche_platform_probe,
+	.remove		= arche_platform_remove,
+	.shutdown	= arche_platform_shutdown,
+	.driver		= {
+		.name	= "arche-platform-ctrl",
+		.pm	= &arche_platform_pm_ops,
+		.of_match_table = arche_platform_of_match,
+	}
+};
+
+static int __init arche_init(void)
+{
+	int retval;
+
+	retval = platform_driver_register(&arche_platform_device_driver);
+	if (retval)
+		return retval;
+
+	retval = arche_apb_init();
+	if (retval)
+		platform_driver_unregister(&arche_platform_device_driver);
+
+	return retval;
+}
+module_init(arche_init);
+
+static void __exit arche_exit(void)
+{
+	arche_apb_exit();
+	platform_driver_unregister(&arche_platform_device_driver);
+}
+module_exit(arche_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Vaibhav Hiremath <vaibhav.hiremath@linaro.org>");
+MODULE_DESCRIPTION("Arche Platform Driver");
diff --git a/drivers/staging/greybus/arche_platform.h b/drivers/staging/greybus/arche_platform.h
new file mode 100644
index 000000000000..bd12345b82a2
--- /dev/null
+++ b/drivers/staging/greybus/arche_platform.h
@@ -0,0 +1,39 @@
+/*
+ * Arche Platform driver to enable Unipro link.
+ *
+ * Copyright 2015-2016 Google Inc.
+ * Copyright 2015-2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __ARCHE_PLATFORM_H
+#define __ARCHE_PLATFORM_H
+
+#include "timesync.h"
+
+enum arche_platform_state {
+	ARCHE_PLATFORM_STATE_OFF,
+	ARCHE_PLATFORM_STATE_ACTIVE,
+	ARCHE_PLATFORM_STATE_STANDBY,
+	ARCHE_PLATFORM_STATE_FW_FLASHING,
+	ARCHE_PLATFORM_STATE_TIME_SYNC,
+};
+
+int arche_platform_change_state(enum arche_platform_state state,
+				struct gb_timesync_svc *pdata);
+
+extern int (*arche_platform_change_state_cb)(enum arche_platform_state state,
+					     struct gb_timesync_svc *pdata);
+int __init arche_apb_init(void);
+void __exit arche_apb_exit(void);
+
+/* Operational states for the APB device */
+int apb_ctrl_coldboot(struct device *dev);
+int apb_ctrl_fw_flashing(struct device *dev);
+int apb_ctrl_standby_boot(struct device *dev);
+void apb_ctrl_poweroff(struct device *dev);
+void apb_bootret_assert(struct device *dev);
+void apb_bootret_deassert(struct device *dev);
+
+#endif	/* __ARCHE_PLATFORM_H */
diff --git a/drivers/staging/greybus/arpc.h b/drivers/staging/greybus/arpc.h
new file mode 100644
index 000000000000..7fbddfc40d83
--- /dev/null
+++ b/drivers/staging/greybus/arpc.h
@@ -0,0 +1,109 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details.
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name of Google Inc. or Linaro Ltd. nor the names of
+ *    its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written
+ *    permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GOOGLE INC. OR
+ * LINARO LTD. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __ARPC_H
+#define __ARPC_H
+
+/* APBridgeA RPC (ARPC) */
+
+enum arpc_result {
+	ARPC_SUCCESS		= 0x00,
+	ARPC_NO_MEMORY		= 0x01,
+	ARPC_INVALID		= 0x02,
+	ARPC_TIMEOUT		= 0x03,
+	ARPC_UNKNOWN_ERROR	= 0xff,
+};
+
+struct arpc_request_message {
+	__le16	id;		/* RPC unique id */
+	__le16	size;		/* Size in bytes of header + payload */
+	__u8	type;		/* RPC type */
+	__u8	data[0];	/* ARPC data */
+} __packed;
+
+struct arpc_response_message {
+	__le16	id;		/* RPC unique id */
+	__u8	result;		/* Result of RPC */
+} __packed;
+
+
+/* ARPC requests */
+#define ARPC_TYPE_CPORT_CONNECTED		0x01
+#define ARPC_TYPE_CPORT_QUIESCE			0x02
+#define ARPC_TYPE_CPORT_CLEAR			0x03
+#define ARPC_TYPE_CPORT_FLUSH			0x04
+#define ARPC_TYPE_CPORT_SHUTDOWN		0x05
+
+struct arpc_cport_connected_req {
+	__le16 cport_id;
+} __packed;
+
+struct arpc_cport_quiesce_req {
+	__le16 cport_id;
+	__le16 peer_space;
+	__le16 timeout;
+} __packed;
+
+struct arpc_cport_clear_req {
+	__le16 cport_id;
+} __packed;
+
+struct arpc_cport_flush_req {
+	__le16 cport_id;
+} __packed;
+
+struct arpc_cport_shutdown_req {
+	__le16 cport_id;
+	__le16 timeout;
+	__u8 phase;
+} __packed;
+
+#endif	/* __ARPC_H */
diff --git a/drivers/staging/greybus/audio_apbridgea.c b/drivers/staging/greybus/audio_apbridgea.c
new file mode 100644
index 000000000000..1b4252d5d255
--- /dev/null
+++ b/drivers/staging/greybus/audio_apbridgea.c
@@ -0,0 +1,207 @@
+/*
+ * Greybus Audio Device Class Protocol helpers
+ *
+ * Copyright 2015-2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include "greybus.h"
+#include "greybus_protocols.h"
+#include "audio_apbridgea.h"
+#include "audio_codec.h"
+
+int gb_audio_apbridgea_set_config(struct gb_connection *connection,
+				  __u16 i2s_port, __u32 format, __u32 rate,
+				  __u32 mclk_freq)
+{
+	struct audio_apbridgea_set_config_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_SET_CONFIG;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+	req.format = cpu_to_le32(format);
+	req.rate = cpu_to_le32(rate);
+	req.mclk_freq = cpu_to_le32(mclk_freq);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_set_config);
+
+int gb_audio_apbridgea_register_cport(struct gb_connection *connection,
+				      __u16 i2s_port, __u16 cportid,
+				      __u8 direction)
+{
+	struct audio_apbridgea_register_cport_request req;
+	int ret;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_REGISTER_CPORT;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+	req.cport = cpu_to_le16(cportid);
+	req.direction = direction;
+
+	ret = gb_pm_runtime_get_sync(connection->bundle);
+	if (ret)
+		return ret;
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_register_cport);
+
+int gb_audio_apbridgea_unregister_cport(struct gb_connection *connection,
+					__u16 i2s_port, __u16 cportid,
+					__u8 direction)
+{
+	struct audio_apbridgea_unregister_cport_request req;
+	int ret;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_UNREGISTER_CPORT;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+	req.cport = cpu_to_le16(cportid);
+	req.direction = direction;
+
+	ret = gb_hd_output(connection->hd, &req, sizeof(req),
+			   GB_APB_REQUEST_AUDIO_CONTROL, true);
+
+	gb_pm_runtime_put_autosuspend(connection->bundle);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_unregister_cport);
+
+int gb_audio_apbridgea_set_tx_data_size(struct gb_connection *connection,
+					__u16 i2s_port, __u16 size)
+{
+	struct audio_apbridgea_set_tx_data_size_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_SET_TX_DATA_SIZE;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+	req.size = cpu_to_le16(size);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_set_tx_data_size);
+
+int gb_audio_apbridgea_prepare_tx(struct gb_connection *connection,
+				  __u16 i2s_port)
+{
+	struct audio_apbridgea_prepare_tx_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_PREPARE_TX;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_prepare_tx);
+
+int gb_audio_apbridgea_start_tx(struct gb_connection *connection,
+				__u16 i2s_port, __u64 timestamp)
+{
+	struct audio_apbridgea_start_tx_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_START_TX;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+	req.timestamp = cpu_to_le64(timestamp);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_start_tx);
+
+int gb_audio_apbridgea_stop_tx(struct gb_connection *connection, __u16 i2s_port)
+{
+	struct audio_apbridgea_stop_tx_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_STOP_TX;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_stop_tx);
+
+int gb_audio_apbridgea_shutdown_tx(struct gb_connection *connection,
+				   __u16 i2s_port)
+{
+	struct audio_apbridgea_shutdown_tx_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_SHUTDOWN_TX;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_shutdown_tx);
+
+int gb_audio_apbridgea_set_rx_data_size(struct gb_connection *connection,
+					__u16 i2s_port, __u16 size)
+{
+	struct audio_apbridgea_set_rx_data_size_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_SET_RX_DATA_SIZE;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+	req.size = cpu_to_le16(size);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_set_rx_data_size);
+
+int gb_audio_apbridgea_prepare_rx(struct gb_connection *connection,
+				  __u16 i2s_port)
+{
+	struct audio_apbridgea_prepare_rx_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_PREPARE_RX;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_prepare_rx);
+
+int gb_audio_apbridgea_start_rx(struct gb_connection *connection,
+				__u16 i2s_port)
+{
+	struct audio_apbridgea_start_rx_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_START_RX;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_start_rx);
+
+int gb_audio_apbridgea_stop_rx(struct gb_connection *connection, __u16 i2s_port)
+{
+	struct audio_apbridgea_stop_rx_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_STOP_RX;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_stop_rx);
+
+int gb_audio_apbridgea_shutdown_rx(struct gb_connection *connection,
+				   __u16 i2s_port)
+{
+	struct audio_apbridgea_shutdown_rx_request req;
+
+	req.hdr.type = AUDIO_APBRIDGEA_TYPE_SHUTDOWN_RX;
+	req.hdr.i2s_port = cpu_to_le16(i2s_port);
+
+	return gb_hd_output(connection->hd, &req, sizeof(req),
+			    GB_APB_REQUEST_AUDIO_CONTROL, true);
+}
+EXPORT_SYMBOL_GPL(gb_audio_apbridgea_shutdown_rx);
+
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("greybus:audio-apbridgea");
+MODULE_DESCRIPTION("Greybus Special APBridgeA Audio Protocol library");
+MODULE_AUTHOR("Mark Greer <mgreer@animalcreek.com>");
diff --git a/drivers/staging/greybus/audio_apbridgea.h b/drivers/staging/greybus/audio_apbridgea.h
new file mode 100644
index 000000000000..b94cb05c89e4
--- /dev/null
+++ b/drivers/staging/greybus/audio_apbridgea.h
@@ -0,0 +1,156 @@
+/**
+ * Copyright (c) 2015-2016 Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * This is a special protocol for configuring communication over the
+ * I2S bus between the DSP on the MSM8994 and APBridgeA.  Therefore,
+ * we can predefine several low-level attributes of the communication
+ * because we know that they are supported.  In particular, the following
+ * assumptions are made:
+ *	- there are two channels (i.e., stereo)
+ *	- the low-level protocol is I2S as defined by Philips/NXP
+ *	- the DSP on the MSM8994 is the clock master for MCLK, BCLK, and WCLK
+ *	- WCLK changes on the falling edge of BCLK
+ *	- WCLK low for left channel; high for right channel
+ *	- TX data is sent on the falling edge of BCLK
+ *	- RX data is received/latched on the rising edge of BCLK
+ */
+
+#ifndef __AUDIO_APBRIDGEA_H
+#define __AUDIO_APBRIDGEA_H
+
+#define AUDIO_APBRIDGEA_TYPE_SET_CONFIG			0x01
+#define AUDIO_APBRIDGEA_TYPE_REGISTER_CPORT		0x02
+#define AUDIO_APBRIDGEA_TYPE_UNREGISTER_CPORT		0x03
+#define AUDIO_APBRIDGEA_TYPE_SET_TX_DATA_SIZE		0x04
+							/* 0x05 unused */
+#define AUDIO_APBRIDGEA_TYPE_PREPARE_TX			0x06
+#define AUDIO_APBRIDGEA_TYPE_START_TX			0x07
+#define AUDIO_APBRIDGEA_TYPE_STOP_TX			0x08
+#define AUDIO_APBRIDGEA_TYPE_SHUTDOWN_TX		0x09
+#define AUDIO_APBRIDGEA_TYPE_SET_RX_DATA_SIZE		0x0a
+							/* 0x0b unused */
+#define AUDIO_APBRIDGEA_TYPE_PREPARE_RX			0x0c
+#define AUDIO_APBRIDGEA_TYPE_START_RX			0x0d
+#define AUDIO_APBRIDGEA_TYPE_STOP_RX			0x0e
+#define AUDIO_APBRIDGEA_TYPE_SHUTDOWN_RX		0x0f
+
+#define AUDIO_APBRIDGEA_PCM_FMT_8			BIT(0)
+#define AUDIO_APBRIDGEA_PCM_FMT_16			BIT(1)
+#define AUDIO_APBRIDGEA_PCM_FMT_24			BIT(2)
+#define AUDIO_APBRIDGEA_PCM_FMT_32			BIT(3)
+#define AUDIO_APBRIDGEA_PCM_FMT_64			BIT(4)
+
+#define AUDIO_APBRIDGEA_PCM_RATE_5512			BIT(0)
+#define AUDIO_APBRIDGEA_PCM_RATE_8000			BIT(1)
+#define AUDIO_APBRIDGEA_PCM_RATE_11025			BIT(2)
+#define AUDIO_APBRIDGEA_PCM_RATE_16000			BIT(3)
+#define AUDIO_APBRIDGEA_PCM_RATE_22050			BIT(4)
+#define AUDIO_APBRIDGEA_PCM_RATE_32000			BIT(5)
+#define AUDIO_APBRIDGEA_PCM_RATE_44100			BIT(6)
+#define AUDIO_APBRIDGEA_PCM_RATE_48000			BIT(7)
+#define AUDIO_APBRIDGEA_PCM_RATE_64000			BIT(8)
+#define AUDIO_APBRIDGEA_PCM_RATE_88200			BIT(9)
+#define AUDIO_APBRIDGEA_PCM_RATE_96000			BIT(10)
+#define AUDIO_APBRIDGEA_PCM_RATE_176400			BIT(11)
+#define AUDIO_APBRIDGEA_PCM_RATE_192000			BIT(12)
+
+#define AUDIO_APBRIDGEA_DIRECTION_TX			BIT(0)
+#define AUDIO_APBRIDGEA_DIRECTION_RX			BIT(1)
+
+/* The I2S port is passed in the 'index' parameter of the USB request */
+/* The CPort is passed in the 'value' parameter of the USB request */
+
+struct audio_apbridgea_hdr {
+	__u8	type;
+	__le16	i2s_port;
+	__u8	data[0];
+} __packed;
+
+struct audio_apbridgea_set_config_request {
+	struct audio_apbridgea_hdr	hdr;
+	__le32				format;	/* AUDIO_APBRIDGEA_PCM_FMT_* */
+	__le32				rate;	/* AUDIO_APBRIDGEA_PCM_RATE_* */
+	__le32				mclk_freq; /* XXX Remove? */
+} __packed;
+
+struct audio_apbridgea_register_cport_request {
+	struct audio_apbridgea_hdr	hdr;
+	__le16				cport;
+	__u8				direction;
+} __packed;
+
+struct audio_apbridgea_unregister_cport_request {
+	struct audio_apbridgea_hdr	hdr;
+	__le16				cport;
+	__u8				direction;
+} __packed;
+
+struct audio_apbridgea_set_tx_data_size_request {
+	struct audio_apbridgea_hdr	hdr;
+	__le16				size;
+} __packed;
+
+struct audio_apbridgea_prepare_tx_request {
+	struct audio_apbridgea_hdr	hdr;
+} __packed;
+
+struct audio_apbridgea_start_tx_request {
+	struct audio_apbridgea_hdr	hdr;
+	__le64				timestamp;
+} __packed;
+
+struct audio_apbridgea_stop_tx_request {
+	struct audio_apbridgea_hdr	hdr;
+} __packed;
+
+struct audio_apbridgea_shutdown_tx_request {
+	struct audio_apbridgea_hdr	hdr;
+} __packed;
+
+struct audio_apbridgea_set_rx_data_size_request {
+	struct audio_apbridgea_hdr	hdr;
+	__le16				size;
+} __packed;
+
+struct audio_apbridgea_prepare_rx_request {
+	struct audio_apbridgea_hdr	hdr;
+} __packed;
+
+struct audio_apbridgea_start_rx_request {
+	struct audio_apbridgea_hdr	hdr;
+} __packed;
+
+struct audio_apbridgea_stop_rx_request {
+	struct audio_apbridgea_hdr	hdr;
+} __packed;
+
+struct audio_apbridgea_shutdown_rx_request {
+	struct audio_apbridgea_hdr	hdr;
+} __packed;
+
+#endif /*__AUDIO_APBRIDGEA_H */
diff --git a/drivers/staging/greybus/audio_codec.c b/drivers/staging/greybus/audio_codec.c
new file mode 100644
index 000000000000..8a0744b58a32
--- /dev/null
+++ b/drivers/staging/greybus/audio_codec.c
@@ -0,0 +1,1132 @@
+/*
+ * APBridge ALSA SoC dummy codec driver
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <sound/soc.h>
+#include <sound/pcm_params.h>
+#include <uapi/linux/input.h>
+
+#include "audio_codec.h"
+#include "audio_apbridgea.h"
+#include "audio_manager.h"
+
+static struct gbaudio_codec_info *gbcodec;
+
+static struct gbaudio_data_connection *
+find_data(struct gbaudio_module_info *module, int id)
+{
+	struct gbaudio_data_connection *data;
+
+	list_for_each_entry(data, &module->data_list, list) {
+		if (id == data->id)
+			return data;
+	}
+	return NULL;
+}
+
+static struct gbaudio_stream_params *
+find_dai_stream_params(struct gbaudio_codec_info *codec, int id, int stream)
+{
+	struct gbaudio_codec_dai *dai;
+
+	list_for_each_entry(dai, &codec->dai_list, list) {
+		if (dai->id == id)
+			return &dai->params[stream];
+	}
+	return NULL;
+}
+
+static int gbaudio_module_enable_tx(struct gbaudio_codec_info *codec,
+				    struct gbaudio_module_info *module, int id)
+{
+	int module_state, ret = 0;
+	u16 data_cport, i2s_port, cportid;
+	u8 sig_bits, channels;
+	uint32_t format, rate;
+	struct gbaudio_data_connection *data;
+	struct gbaudio_stream_params *params;
+
+	/* find the dai */
+	data = find_data(module, id);
+	if (!data) {
+		dev_err(module->dev, "%d:DATA connection missing\n", id);
+		return -ENODEV;
+	}
+	module_state = data->state[SNDRV_PCM_STREAM_PLAYBACK];
+
+	params = find_dai_stream_params(codec, id, SNDRV_PCM_STREAM_PLAYBACK);
+	if (!params) {
+		dev_err(codec->dev, "Failed to fetch dai_stream pointer\n");
+		return -EINVAL;
+	}
+
+	/* register cport */
+	if (module_state < GBAUDIO_CODEC_STARTUP) {
+		i2s_port = 0;	/* fixed for now */
+		cportid = data->connection->hd_cport_id;
+		ret = gb_audio_apbridgea_register_cport(data->connection,
+						i2s_port, cportid,
+						AUDIO_APBRIDGEA_DIRECTION_TX);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "reg_cport failed:%d\n", ret);
+			return ret;
+		}
+		data->state[SNDRV_PCM_STREAM_PLAYBACK] =
+			GBAUDIO_CODEC_STARTUP;
+		dev_dbg(module->dev, "Dynamic Register %d DAI\n", cportid);
+	}
+
+	/* hw_params */
+	if (module_state < GBAUDIO_CODEC_HWPARAMS) {
+		format = params->format;
+		channels = params->channels;
+		rate = params->rate;
+		sig_bits = params->sig_bits;
+		data_cport = data->connection->intf_cport_id;
+		ret = gb_audio_gb_set_pcm(module->mgmt_connection, data_cport,
+					  format, rate, channels, sig_bits);
+		if (ret) {
+			dev_err_ratelimited(module->dev, "set_pcm failed:%d\n",
+					    ret);
+			return ret;
+		}
+		data->state[SNDRV_PCM_STREAM_PLAYBACK] =
+			GBAUDIO_CODEC_HWPARAMS;
+		dev_dbg(module->dev, "Dynamic hw_params %d DAI\n", data_cport);
+	}
+
+	/* prepare */
+	if (module_state < GBAUDIO_CODEC_PREPARE) {
+		data_cport = data->connection->intf_cport_id;
+		ret = gb_audio_gb_set_tx_data_size(module->mgmt_connection,
+						   data_cport, 192);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "set_tx_data_size failed:%d\n",
+					    ret);
+			return ret;
+		}
+		ret = gb_audio_gb_activate_tx(module->mgmt_connection,
+					      data_cport);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "activate_tx failed:%d\n", ret);
+			return ret;
+		}
+		data->state[SNDRV_PCM_STREAM_PLAYBACK] =
+			GBAUDIO_CODEC_PREPARE;
+		dev_dbg(module->dev, "Dynamic prepare %d DAI\n", data_cport);
+	}
+
+	return 0;
+}
+
+static int gbaudio_module_disable_tx(struct gbaudio_module_info *module, int id)
+{
+	int ret;
+	u16 data_cport, cportid, i2s_port;
+	int module_state;
+	struct gbaudio_data_connection *data;
+
+	/* find the dai */
+	data = find_data(module, id);
+	if (!data) {
+		dev_err(module->dev, "%d:DATA connection missing\n", id);
+		return -ENODEV;
+	}
+	module_state = data->state[SNDRV_PCM_STREAM_PLAYBACK];
+
+	if (module_state > GBAUDIO_CODEC_HWPARAMS) {
+		data_cport = data->connection->intf_cport_id;
+		ret = gb_audio_gb_deactivate_tx(module->mgmt_connection,
+						data_cport);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "deactivate_tx failed:%d\n", ret);
+			return ret;
+		}
+		dev_dbg(module->dev, "Dynamic deactivate %d DAI\n", data_cport);
+		data->state[SNDRV_PCM_STREAM_PLAYBACK] =
+			GBAUDIO_CODEC_HWPARAMS;
+	}
+
+	if (module_state > GBAUDIO_CODEC_SHUTDOWN) {
+		i2s_port = 0;	/* fixed for now */
+		cportid = data->connection->hd_cport_id;
+		ret = gb_audio_apbridgea_unregister_cport(data->connection,
+						i2s_port, cportid,
+						AUDIO_APBRIDGEA_DIRECTION_TX);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "unregister_cport failed:%d\n",
+					    ret);
+			return ret;
+		}
+		dev_dbg(module->dev, "Dynamic Unregister %d DAI\n", cportid);
+		data->state[SNDRV_PCM_STREAM_PLAYBACK] =
+			GBAUDIO_CODEC_SHUTDOWN;
+	}
+
+	return 0;
+}
+
+static int gbaudio_module_enable_rx(struct gbaudio_codec_info *codec,
+				    struct gbaudio_module_info *module, int id)
+{
+	int module_state, ret = 0;
+	u16 data_cport, i2s_port, cportid;
+	u8 sig_bits, channels;
+	uint32_t format, rate;
+	struct gbaudio_data_connection *data;
+	struct gbaudio_stream_params *params;
+
+	/* find the dai */
+	data = find_data(module, id);
+	if (!data) {
+		dev_err(module->dev, "%d:DATA connection missing\n", id);
+		return -ENODEV;
+	}
+	module_state = data->state[SNDRV_PCM_STREAM_CAPTURE];
+
+	params = find_dai_stream_params(codec, id, SNDRV_PCM_STREAM_CAPTURE);
+	if (!params) {
+		dev_err(codec->dev, "Failed to fetch dai_stream pointer\n");
+		return -EINVAL;
+	}
+
+	/* register cport */
+	if (module_state < GBAUDIO_CODEC_STARTUP) {
+		i2s_port = 0;	/* fixed for now */
+		cportid = data->connection->hd_cport_id;
+		ret = gb_audio_apbridgea_register_cport(data->connection,
+						i2s_port, cportid,
+						AUDIO_APBRIDGEA_DIRECTION_RX);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "reg_cport failed:%d\n", ret);
+			return ret;
+		}
+		data->state[SNDRV_PCM_STREAM_CAPTURE] =
+			GBAUDIO_CODEC_STARTUP;
+		dev_dbg(module->dev, "Dynamic Register %d DAI\n", cportid);
+	}
+
+	/* hw_params */
+	if (module_state < GBAUDIO_CODEC_HWPARAMS) {
+		format = params->format;
+		channels = params->channels;
+		rate = params->rate;
+		sig_bits = params->sig_bits;
+		data_cport = data->connection->intf_cport_id;
+		ret = gb_audio_gb_set_pcm(module->mgmt_connection, data_cport,
+					  format, rate, channels, sig_bits);
+		if (ret) {
+			dev_err_ratelimited(module->dev, "set_pcm failed:%d\n",
+					    ret);
+			return ret;
+		}
+		data->state[SNDRV_PCM_STREAM_CAPTURE] =
+			GBAUDIO_CODEC_HWPARAMS;
+		dev_dbg(module->dev, "Dynamic hw_params %d DAI\n", data_cport);
+	}
+
+	/* prepare */
+	if (module_state < GBAUDIO_CODEC_PREPARE) {
+		data_cport = data->connection->intf_cport_id;
+		ret = gb_audio_gb_set_rx_data_size(module->mgmt_connection,
+						   data_cport, 192);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "set_rx_data_size failed:%d\n",
+					    ret);
+			return ret;
+		}
+		ret = gb_audio_gb_activate_rx(module->mgmt_connection,
+					      data_cport);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "activate_rx failed:%d\n", ret);
+			return ret;
+		}
+		data->state[SNDRV_PCM_STREAM_CAPTURE] =
+			GBAUDIO_CODEC_PREPARE;
+		dev_dbg(module->dev, "Dynamic prepare %d DAI\n", data_cport);
+	}
+
+	return 0;
+}
+
+static int gbaudio_module_disable_rx(struct gbaudio_module_info *module, int id)
+{
+	int ret;
+	u16 data_cport, cportid, i2s_port;
+	int module_state;
+	struct gbaudio_data_connection *data;
+
+	/* find the dai */
+	data = find_data(module, id);
+	if (!data) {
+		dev_err(module->dev, "%d:DATA connection missing\n", id);
+		return -ENODEV;
+	}
+	module_state = data->state[SNDRV_PCM_STREAM_CAPTURE];
+
+	if (module_state > GBAUDIO_CODEC_HWPARAMS) {
+		data_cport = data->connection->intf_cport_id;
+		ret = gb_audio_gb_deactivate_rx(module->mgmt_connection,
+						data_cport);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "deactivate_rx failed:%d\n", ret);
+			return ret;
+		}
+		dev_dbg(module->dev, "Dynamic deactivate %d DAI\n", data_cport);
+		data->state[SNDRV_PCM_STREAM_CAPTURE] =
+			GBAUDIO_CODEC_HWPARAMS;
+	}
+
+	if (module_state > GBAUDIO_CODEC_SHUTDOWN) {
+		i2s_port = 0;	/* fixed for now */
+		cportid = data->connection->hd_cport_id;
+		ret = gb_audio_apbridgea_unregister_cport(data->connection,
+						i2s_port, cportid,
+						AUDIO_APBRIDGEA_DIRECTION_RX);
+		if (ret) {
+			dev_err_ratelimited(module->dev,
+					    "unregister_cport failed:%d\n",
+					    ret);
+			return ret;
+		}
+		dev_dbg(module->dev, "Dynamic Unregister %d DAI\n", cportid);
+		data->state[SNDRV_PCM_STREAM_CAPTURE] =
+			GBAUDIO_CODEC_SHUTDOWN;
+	}
+
+	return 0;
+}
+
+int gbaudio_module_update(struct gbaudio_codec_info *codec,
+			  struct snd_soc_dapm_widget *w,
+			  struct gbaudio_module_info *module, int enable)
+{
+	int dai_id, ret;
+	char intf_name[NAME_SIZE], dir[NAME_SIZE];
+
+	dev_dbg(module->dev, "%s:Module update %s sequence\n", w->name,
+		enable ? "Enable":"Disable");
+
+	if ((w->id != snd_soc_dapm_aif_in) && (w->id != snd_soc_dapm_aif_out)) {
+		dev_dbg(codec->dev, "No action required for %s\n", w->name);
+		return 0;
+	}
+
+	/* parse dai_id from AIF widget's stream_name */
+	ret = sscanf(w->sname, "%s %d %s", intf_name, &dai_id, dir);
+	if (ret < 3) {
+		dev_err(codec->dev, "Error while parsing dai_id for %s\n",
+			w->name);
+		return -EINVAL;
+	}
+
+	mutex_lock(&codec->lock);
+	if (w->id == snd_soc_dapm_aif_in) {
+		if (enable)
+			ret = gbaudio_module_enable_tx(codec, module, dai_id);
+		else
+			ret = gbaudio_module_disable_tx(module, dai_id);
+	} else if (w->id == snd_soc_dapm_aif_out) {
+		if (enable)
+			ret = gbaudio_module_enable_rx(codec, module, dai_id);
+		else
+			ret = gbaudio_module_disable_rx(module, dai_id);
+	}
+
+	mutex_unlock(&codec->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(gbaudio_module_update);
+
+/*
+ * codec DAI ops
+ */
+static int gbcodec_startup(struct snd_pcm_substream *substream,
+			   struct snd_soc_dai *dai)
+{
+	struct gbaudio_codec_info *codec = dev_get_drvdata(dai->dev);
+	struct gbaudio_stream_params *params;
+
+	mutex_lock(&codec->lock);
+
+	if (list_empty(&codec->module_list)) {
+		dev_err(codec->dev, "No codec module available\n");
+		mutex_unlock(&codec->lock);
+		return -ENODEV;
+	}
+
+	params = find_dai_stream_params(codec, dai->id, substream->stream);
+	if (!params) {
+		dev_err(codec->dev, "Failed to fetch dai_stream pointer\n");
+		mutex_unlock(&codec->lock);
+		return -EINVAL;
+	}
+	params->state = GBAUDIO_CODEC_STARTUP;
+	mutex_unlock(&codec->lock);
+	/* to prevent suspend in case of active audio */
+	pm_stay_awake(dai->dev);
+
+	return 0;
+}
+
+static void gbcodec_shutdown(struct snd_pcm_substream *substream,
+			     struct snd_soc_dai *dai)
+{
+	struct gbaudio_codec_info *codec = dev_get_drvdata(dai->dev);
+	struct gbaudio_stream_params *params;
+
+	mutex_lock(&codec->lock);
+
+	if (list_empty(&codec->module_list))
+		dev_info(codec->dev, "No codec module available during shutdown\n");
+
+	params = find_dai_stream_params(codec, dai->id, substream->stream);
+	if (!params) {
+		dev_err(codec->dev, "Failed to fetch dai_stream pointer\n");
+		mutex_unlock(&codec->lock);
+		return;
+	}
+	params->state = GBAUDIO_CODEC_SHUTDOWN;
+	mutex_unlock(&codec->lock);
+	pm_relax(dai->dev);
+	return;
+}
+
+static int gbcodec_hw_params(struct snd_pcm_substream *substream,
+			     struct snd_pcm_hw_params *hwparams,
+			     struct snd_soc_dai *dai)
+{
+	int ret;
+	u8 sig_bits, channels;
+	uint32_t format, rate;
+	struct gbaudio_module_info *module;
+	struct gbaudio_data_connection *data;
+	struct gb_bundle *bundle;
+	struct gbaudio_codec_info *codec = dev_get_drvdata(dai->dev);
+	struct gbaudio_stream_params *params;
+
+	mutex_lock(&codec->lock);
+
+	if (list_empty(&codec->module_list)) {
+		dev_err(codec->dev, "No codec module available\n");
+		mutex_unlock(&codec->lock);
+		return -ENODEV;
+	}
+
+	/*
+	 * assuming, currently only 48000 Hz, 16BIT_LE, stereo
+	 * is supported, validate params before configuring codec
+	 */
+	if (params_channels(hwparams) != 2) {
+		dev_err(dai->dev, "Invalid channel count:%d\n",
+			params_channels(hwparams));
+		mutex_unlock(&codec->lock);
+		return -EINVAL;
+	}
+	channels = params_channels(hwparams);
+
+	if (params_rate(hwparams) != 48000) {
+		dev_err(dai->dev, "Invalid sampling rate:%d\n",
+			params_rate(hwparams));
+		mutex_unlock(&codec->lock);
+		return -EINVAL;
+	}
+	rate = GB_AUDIO_PCM_RATE_48000;
+
+	if (params_format(hwparams) != SNDRV_PCM_FORMAT_S16_LE) {
+		dev_err(dai->dev, "Invalid format:%d\n",
+			params_format(hwparams));
+		mutex_unlock(&codec->lock);
+		return -EINVAL;
+	}
+	format = GB_AUDIO_PCM_FMT_S16_LE;
+
+	/* find the data connection */
+	list_for_each_entry(module, &codec->module_list, list) {
+		data = find_data(module, dai->id);
+		if (data)
+			break;
+	}
+
+	if (!data) {
+		dev_err(dai->dev, "DATA connection missing\n");
+		mutex_unlock(&codec->lock);
+		return -EINVAL;
+	}
+
+	params = find_dai_stream_params(codec, dai->id, substream->stream);
+	if (!params) {
+		dev_err(codec->dev, "Failed to fetch dai_stream pointer\n");
+		mutex_unlock(&codec->lock);
+		return -EINVAL;
+	}
+
+	bundle = to_gb_bundle(module->dev);
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret) {
+		mutex_unlock(&codec->lock);
+		return ret;
+	}
+
+	ret = gb_audio_apbridgea_set_config(data->connection, 0,
+					    AUDIO_APBRIDGEA_PCM_FMT_16,
+					    AUDIO_APBRIDGEA_PCM_RATE_48000,
+					    6144000);
+	if (ret) {
+		dev_err_ratelimited(dai->dev, "%d: Error during set_config\n",
+				    ret);
+		mutex_unlock(&codec->lock);
+		return ret;
+	}
+
+	gb_pm_runtime_put_noidle(bundle);
+
+	params->state = GBAUDIO_CODEC_HWPARAMS;
+	params->format = format;
+	params->rate = rate;
+	params->channels = channels;
+	params->sig_bits = sig_bits;
+
+	mutex_unlock(&codec->lock);
+	return 0;
+}
+
+static int gbcodec_prepare(struct snd_pcm_substream *substream,
+			   struct snd_soc_dai *dai)
+{
+	int ret;
+	struct gbaudio_module_info *module;
+	struct gbaudio_data_connection *data;
+	struct gb_bundle *bundle;
+	struct gbaudio_codec_info *codec = dev_get_drvdata(dai->dev);
+	struct gbaudio_stream_params *params;
+
+	mutex_lock(&codec->lock);
+
+	if (list_empty(&codec->module_list)) {
+		dev_err(codec->dev, "No codec module available\n");
+		mutex_unlock(&codec->lock);
+		return -ENODEV;
+	}
+
+	list_for_each_entry(module, &codec->module_list, list) {
+		/* find the dai */
+		data = find_data(module, dai->id);
+		if (data)
+			break;
+	}
+	if (!data) {
+		dev_err(dai->dev, "DATA connection missing\n");
+		mutex_unlock(&codec->lock);
+		return -ENODEV;
+	}
+
+	params = find_dai_stream_params(codec, dai->id, substream->stream);
+	if (!params) {
+		dev_err(codec->dev, "Failed to fetch dai_stream pointer\n");
+		mutex_unlock(&codec->lock);
+		return -EINVAL;
+	}
+
+	bundle = to_gb_bundle(module->dev);
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret) {
+		mutex_unlock(&codec->lock);
+		return ret;
+	}
+
+	switch (substream->stream) {
+	case SNDRV_PCM_STREAM_PLAYBACK:
+		ret = gb_audio_apbridgea_set_tx_data_size(data->connection, 0,
+							  192);
+		break;
+	case SNDRV_PCM_STREAM_CAPTURE:
+		ret = gb_audio_apbridgea_set_rx_data_size(data->connection, 0,
+							  192);
+		break;
+	}
+	if (ret) {
+		mutex_unlock(&codec->lock);
+		dev_err_ratelimited(dai->dev, "set_data_size failed:%d\n",
+				     ret);
+		return ret;
+	}
+
+	gb_pm_runtime_put_noidle(bundle);
+
+	params->state = GBAUDIO_CODEC_PREPARE;
+	mutex_unlock(&codec->lock);
+	return 0;
+}
+
+static int gbcodec_mute_stream(struct snd_soc_dai *dai, int mute, int stream)
+{
+	int ret;
+	struct gbaudio_data_connection *data;
+	struct gbaudio_module_info *module;
+	struct gb_bundle *bundle;
+	struct gbaudio_codec_info *codec = dev_get_drvdata(dai->dev);
+	struct gbaudio_stream_params *params;
+
+
+	dev_dbg(dai->dev, "Mute:%d, Direction:%s\n", mute,
+		stream ? "CAPTURE":"PLAYBACK");
+
+	mutex_lock(&codec->lock);
+
+	params = find_dai_stream_params(codec, dai->id, stream);
+	if (!params) {
+		dev_err(codec->dev, "Failed to fetch dai_stream pointer\n");
+		mutex_unlock(&codec->lock);
+		return -EINVAL;
+	}
+
+	if (list_empty(&codec->module_list)) {
+		dev_err(codec->dev, "No codec module available\n");
+		if (mute) {
+			params->state = GBAUDIO_CODEC_STOP;
+			ret = 0;
+		} else {
+			ret = -ENODEV;
+		}
+		mutex_unlock(&codec->lock);
+		return ret;
+	}
+
+	list_for_each_entry(module, &codec->module_list, list) {
+		/* find the dai */
+		data = find_data(module, dai->id);
+		if (data)
+			break;
+	}
+	if (!data) {
+		dev_err(dai->dev, "%s:%s DATA connection missing\n",
+			dai->name, module->name);
+		mutex_unlock(&codec->lock);
+		return -ENODEV;
+	}
+
+	bundle = to_gb_bundle(module->dev);
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret) {
+		mutex_unlock(&codec->lock);
+		return ret;
+	}
+
+	if (!mute && !stream) {/* start playback */
+		ret = gb_audio_apbridgea_prepare_tx(data->connection,
+						    0);
+		if (!ret)
+			ret = gb_audio_apbridgea_start_tx(data->connection,
+							  0, 0);
+		params->state = GBAUDIO_CODEC_START;
+	} else if (!mute && stream) {/* start capture */
+		ret = gb_audio_apbridgea_prepare_rx(data->connection,
+						    0);
+		if (!ret)
+			ret = gb_audio_apbridgea_start_rx(data->connection,
+							  0);
+		params->state = GBAUDIO_CODEC_START;
+	} else if (mute && !stream) {/* stop playback */
+		ret = gb_audio_apbridgea_stop_tx(data->connection, 0);
+		if (!ret)
+			ret = gb_audio_apbridgea_shutdown_tx(data->connection,
+							     0);
+		params->state = GBAUDIO_CODEC_STOP;
+	} else if (mute && stream) {/* stop capture */
+		ret = gb_audio_apbridgea_stop_rx(data->connection, 0);
+		if (!ret)
+			ret = gb_audio_apbridgea_shutdown_rx(data->connection,
+							     0);
+		params->state = GBAUDIO_CODEC_STOP;
+	} else
+		ret = -EINVAL;
+	if (ret)
+		dev_err_ratelimited(dai->dev,
+				    "%s:Error during %s %s stream:%d\n",
+				    module->name, mute ? "Mute" : "Unmute",
+				    stream ? "Capture" : "Playback", ret);
+
+	gb_pm_runtime_put_noidle(bundle);
+	mutex_unlock(&codec->lock);
+	return ret;
+}
+
+static struct snd_soc_dai_ops gbcodec_dai_ops = {
+	.startup = gbcodec_startup,
+	.shutdown = gbcodec_shutdown,
+	.hw_params = gbcodec_hw_params,
+	.prepare = gbcodec_prepare,
+	.mute_stream = gbcodec_mute_stream,
+};
+
+static struct snd_soc_dai_driver gbaudio_dai[] = {
+	{
+		.name = "apb-i2s0",
+		.id = 0,
+		.playback = {
+			.stream_name = "I2S 0 Playback",
+			.rates = SNDRV_PCM_RATE_48000,
+			.formats = SNDRV_PCM_FORMAT_S16_LE,
+			.rate_max = 48000,
+			.rate_min = 48000,
+			.channels_min = 1,
+			.channels_max = 2,
+		},
+		.capture = {
+			.stream_name = "I2S 0 Capture",
+			.rates = SNDRV_PCM_RATE_48000,
+			.formats = SNDRV_PCM_FORMAT_S16_LE,
+			.rate_max = 48000,
+			.rate_min = 48000,
+			.channels_min = 1,
+			.channels_max = 2,
+		},
+		.ops = &gbcodec_dai_ops,
+	},
+};
+
+static int gbaudio_init_jack(struct gbaudio_module_info *module,
+			     struct snd_soc_codec *codec)
+{
+	int ret;
+
+	if (!module->jack_mask)
+		return 0;
+
+	snprintf(module->jack_name, NAME_SIZE, "GB %d Headset Jack",
+		 module->dev_id);
+	ret = snd_soc_jack_new(codec, module->jack_name, module->jack_mask,
+			       &module->headset_jack);
+	if (ret) {
+		dev_err(module->dev, "Failed to create new jack\n");
+		return ret;
+	}
+
+	if (!module->button_mask)
+		return 0;
+
+	snprintf(module->button_name, NAME_SIZE, "GB %d Button Jack",
+		 module->dev_id);
+	ret = snd_soc_jack_new(codec, module->button_name, module->button_mask,
+			       &module->button_jack);
+	if (ret) {
+		dev_err(module->dev, "Failed to create button jack\n");
+		return ret;
+	}
+
+	/*
+	 * Currently, max 4 buttons are supported with following key mapping
+	 * BTN_0 = KEY_MEDIA
+	 * BTN_1 = KEY_VOICECOMMAND
+	 * BTN_2 = KEY_VOLUMEUP
+	 * BTN_3 = KEY_VOLUMEDOWN
+	 */
+
+	if (module->button_mask & SND_JACK_BTN_0) {
+		ret = snd_jack_set_key(module->button_jack.jack, SND_JACK_BTN_0,
+				       KEY_MEDIA);
+		if (ret) {
+			dev_err(module->dev, "Failed to set BTN_0\n");
+			return ret;
+		}
+	}
+
+	if (module->button_mask & SND_JACK_BTN_1) {
+		ret = snd_jack_set_key(module->button_jack.jack, SND_JACK_BTN_1,
+				       KEY_VOICECOMMAND);
+		if (ret) {
+			dev_err(module->dev, "Failed to set BTN_1\n");
+			return ret;
+		}
+	}
+
+	if (module->button_mask & SND_JACK_BTN_2) {
+		ret = snd_jack_set_key(module->button_jack.jack, SND_JACK_BTN_2,
+				       KEY_VOLUMEUP);
+		if (ret) {
+			dev_err(module->dev, "Failed to set BTN_2\n");
+			return ret;
+		}
+	}
+
+	if (module->button_mask & SND_JACK_BTN_3) {
+		ret = snd_jack_set_key(module->button_jack.jack, SND_JACK_BTN_3,
+				       KEY_VOLUMEDOWN);
+		if (ret) {
+			dev_err(module->dev, "Failed to set BTN_0\n");
+			return ret;
+		}
+	}
+
+	/* FIXME
+	 * verify if this is really required
+	set_bit(INPUT_PROP_NO_DUMMY_RELEASE,
+		module->button_jack.jack->input_dev->propbit);
+	*/
+
+	return 0;
+}
+
+int gbaudio_register_module(struct gbaudio_module_info *module)
+{
+	int ret;
+	struct snd_soc_codec *codec;
+	struct snd_card *card;
+	struct snd_soc_jack *jack = NULL;
+
+	if (!gbcodec) {
+		dev_err(module->dev, "GB Codec not yet probed\n");
+		return -EAGAIN;
+	}
+
+	codec = gbcodec->codec;
+	card = codec->card->snd_card;
+
+	down_write(&card->controls_rwsem);
+
+	if (module->num_dais) {
+		dev_err(gbcodec->dev,
+			"%d:DAIs not supported via gbcodec driver\n",
+			module->num_dais);
+		up_write(&card->controls_rwsem);
+		return -EINVAL;
+	}
+
+	ret = gbaudio_init_jack(module, codec);
+	if (ret) {
+		up_write(&card->controls_rwsem);
+		return ret;
+	}
+
+	if (module->dapm_widgets)
+		snd_soc_dapm_new_controls(&codec->dapm, module->dapm_widgets,
+					  module->num_dapm_widgets);
+	if (module->controls)
+		snd_soc_add_codec_controls(codec, module->controls,
+				     module->num_controls);
+	if (module->dapm_routes)
+		snd_soc_dapm_add_routes(&codec->dapm, module->dapm_routes,
+					module->num_dapm_routes);
+
+	/* card already instantiated, create widgets here only */
+	if (codec->card->instantiated) {
+		snd_soc_dapm_link_component_dai_widgets(codec->card,
+							&codec->dapm);
+#ifdef CONFIG_SND_JACK
+		/* register jack devices for this module from codec->jack_list */
+		list_for_each_entry(jack, &codec->jack_list, list) {
+			if ((jack == &module->headset_jack)
+			    || (jack == &module->button_jack))
+				snd_device_register(codec->card->snd_card,
+						    jack->jack);
+		}
+#endif
+	}
+
+	mutex_lock(&gbcodec->lock);
+	list_add(&module->list, &gbcodec->module_list);
+	mutex_unlock(&gbcodec->lock);
+
+	if (codec->card->instantiated)
+		ret = snd_soc_dapm_new_widgets(&codec->dapm);
+	dev_dbg(codec->dev, "Registered %s module\n", module->name);
+
+	up_write(&card->controls_rwsem);
+	return ret;
+}
+EXPORT_SYMBOL(gbaudio_register_module);
+
+static void gbaudio_codec_clean_data_tx(struct gbaudio_data_connection *data)
+{
+	u16 i2s_port, cportid;
+	int ret;
+
+	if (list_is_singular(&gbcodec->module_list)) {
+		ret = gb_audio_apbridgea_stop_tx(data->connection, 0);
+		if (ret)
+			return;
+		ret = gb_audio_apbridgea_shutdown_tx(data->connection,
+						     0);
+		if (ret)
+			return;
+	}
+	i2s_port = 0;	/* fixed for now */
+	cportid = data->connection->hd_cport_id;
+	ret = gb_audio_apbridgea_unregister_cport(data->connection,
+						  i2s_port, cportid,
+						  AUDIO_APBRIDGEA_DIRECTION_TX);
+	data->state[0] = GBAUDIO_CODEC_SHUTDOWN;
+}
+
+static void gbaudio_codec_clean_data_rx(struct gbaudio_data_connection *data)
+{
+	u16 i2s_port, cportid;
+	int ret;
+
+	if (list_is_singular(&gbcodec->module_list)) {
+		ret = gb_audio_apbridgea_stop_rx(data->connection, 0);
+		if (ret)
+			return;
+		ret = gb_audio_apbridgea_shutdown_rx(data->connection,
+						     0);
+		if (ret)
+			return;
+	}
+	i2s_port = 0;	/* fixed for now */
+	cportid = data->connection->hd_cport_id;
+	ret = gb_audio_apbridgea_unregister_cport(data->connection,
+						  i2s_port, cportid,
+						  AUDIO_APBRIDGEA_DIRECTION_RX);
+	data->state[1] = GBAUDIO_CODEC_SHUTDOWN;
+}
+
+
+static void gbaudio_codec_cleanup(struct gbaudio_module_info *module)
+{
+	struct gbaudio_data_connection *data;
+	int pb_state, cap_state;
+
+	dev_dbg(gbcodec->dev, "%s: removed, cleanup APBridge\n", module->name);
+	list_for_each_entry(data, &module->data_list, list) {
+		pb_state = data->state[0];
+		cap_state = data->state[1];
+
+		if (pb_state > GBAUDIO_CODEC_SHUTDOWN)
+			gbaudio_codec_clean_data_tx(data);
+
+		if (cap_state > GBAUDIO_CODEC_SHUTDOWN)
+			gbaudio_codec_clean_data_rx(data);
+
+	}
+}
+
+void gbaudio_unregister_module(struct gbaudio_module_info *module)
+{
+	struct snd_soc_codec *codec = gbcodec->codec;
+	struct snd_card *card = codec->card->snd_card;
+	struct snd_soc_jack *jack, *next_j;
+	int mask;
+
+	dev_dbg(codec->dev, "Unregister %s module\n", module->name);
+
+	down_write(&card->controls_rwsem);
+	mutex_lock(&gbcodec->lock);
+	gbaudio_codec_cleanup(module);
+	list_del(&module->list);
+	dev_dbg(codec->dev, "Process Unregister %s module\n", module->name);
+	mutex_unlock(&gbcodec->lock);
+
+#ifdef CONFIG_SND_JACK
+	/* free jack devices for this module from codec->jack_list */
+	list_for_each_entry_safe(jack, next_j, &codec->jack_list, list) {
+		if (jack == &module->headset_jack)
+			mask = GBCODEC_JACK_MASK;
+		else if (jack == &module->button_jack)
+			mask = GBCODEC_JACK_BUTTON_MASK;
+		else
+			mask = 0;
+		if (mask) {
+			dev_dbg(module->dev, "Report %s removal\n",
+				jack->jack->id);
+			snd_soc_jack_report(jack, 0, mask);
+			snd_device_free(codec->card->snd_card, jack->jack);
+			list_del(&jack->list);
+		}
+	}
+#endif
+
+	if (module->dapm_routes) {
+		dev_dbg(codec->dev, "Removing %d routes\n",
+			module->num_dapm_routes);
+		snd_soc_dapm_del_routes(&codec->dapm, module->dapm_routes,
+					module->num_dapm_routes);
+	}
+	if (module->controls) {
+		dev_dbg(codec->dev, "Removing %d controls\n",
+			module->num_controls);
+		snd_soc_remove_codec_controls(codec, module->controls,
+					  module->num_controls);
+	}
+	if (module->dapm_widgets) {
+		dev_dbg(codec->dev, "Removing %d widgets\n",
+			module->num_dapm_widgets);
+		snd_soc_dapm_free_controls(&codec->dapm, module->dapm_widgets,
+					   module->num_dapm_widgets);
+	}
+
+	dev_dbg(codec->dev, "Unregistered %s module\n", module->name);
+
+	up_write(&card->controls_rwsem);
+}
+EXPORT_SYMBOL(gbaudio_unregister_module);
+
+/*
+ * codec driver ops
+ */
+static int gbcodec_probe(struct snd_soc_codec *codec)
+{
+	int i;
+	struct gbaudio_codec_info *info;
+	struct gbaudio_codec_dai *dai;
+
+	info = devm_kzalloc(codec->dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	info->dev = codec->dev;
+	INIT_LIST_HEAD(&info->module_list);
+	mutex_init(&info->lock);
+	INIT_LIST_HEAD(&info->dai_list);
+
+	/* init dai_list used to maintain runtime stream info */
+	for (i = 0; i < ARRAY_SIZE(gbaudio_dai); i++) {
+		dai = devm_kzalloc(codec->dev, sizeof(*dai), GFP_KERNEL);
+		if (!dai)
+			return -ENOMEM;
+		dai->id = gbaudio_dai[i].id;
+		list_add(&dai->list, &info->dai_list);
+	}
+
+	info->codec = codec;
+	snd_soc_codec_set_drvdata(codec, info);
+	gbcodec = info;
+
+	device_init_wakeup(codec->dev, 1);
+	return 0;
+}
+
+static int gbcodec_remove(struct snd_soc_codec *codec)
+{
+	/* Empty function for now */
+	return 0;
+}
+
+static u8 gbcodec_reg[GBCODEC_REG_COUNT] = {
+	[GBCODEC_CTL_REG] = GBCODEC_CTL_REG_DEFAULT,
+	[GBCODEC_MUTE_REG] = GBCODEC_MUTE_REG_DEFAULT,
+	[GBCODEC_PB_LVOL_REG] = GBCODEC_PB_VOL_REG_DEFAULT,
+	[GBCODEC_PB_RVOL_REG] = GBCODEC_PB_VOL_REG_DEFAULT,
+	[GBCODEC_CAP_LVOL_REG] = GBCODEC_CAP_VOL_REG_DEFAULT,
+	[GBCODEC_CAP_RVOL_REG] = GBCODEC_CAP_VOL_REG_DEFAULT,
+	[GBCODEC_APB1_MUX_REG] = GBCODEC_APB1_MUX_REG_DEFAULT,
+	[GBCODEC_APB2_MUX_REG] = GBCODEC_APB2_MUX_REG_DEFAULT,
+};
+
+static int gbcodec_write(struct snd_soc_codec *codec, unsigned int reg,
+			 unsigned int value)
+{
+	int ret = 0;
+
+	if (reg == SND_SOC_NOPM)
+		return 0;
+
+	BUG_ON(reg >= GBCODEC_REG_COUNT);
+
+	gbcodec_reg[reg] = value;
+	dev_dbg(codec->dev, "reg[%d] = 0x%x\n", reg, value);
+
+	return ret;
+}
+
+static unsigned int gbcodec_read(struct snd_soc_codec *codec,
+				 unsigned int reg)
+{
+	unsigned int val = 0;
+
+	if (reg == SND_SOC_NOPM)
+		return 0;
+
+	BUG_ON(reg >= GBCODEC_REG_COUNT);
+
+	val = gbcodec_reg[reg];
+	dev_dbg(codec->dev, "reg[%d] = 0x%x\n", reg, val);
+
+	return val;
+}
+
+static struct snd_soc_codec_driver soc_codec_dev_gbaudio = {
+	.probe	= gbcodec_probe,
+	.remove	= gbcodec_remove,
+
+	.read = gbcodec_read,
+	.write = gbcodec_write,
+
+	.reg_cache_size = GBCODEC_REG_COUNT,
+	.reg_cache_default = gbcodec_reg_defaults,
+	.reg_word_size = 1,
+
+	.idle_bias_off = true,
+	.ignore_pmdown_time = 1,
+};
+
+#ifdef CONFIG_PM
+static int gbaudio_codec_suspend(struct device *dev)
+{
+	dev_dbg(dev, "%s: suspend\n", __func__);
+	return 0;
+}
+
+static int gbaudio_codec_resume(struct device *dev)
+{
+	dev_dbg(dev, "%s: resume\n", __func__);
+	return 0;
+}
+
+static const struct dev_pm_ops gbaudio_codec_pm_ops = {
+	.suspend	= gbaudio_codec_suspend,
+	.resume		= gbaudio_codec_resume,
+};
+#endif
+
+static int gbaudio_codec_probe(struct platform_device *pdev)
+{
+	return snd_soc_register_codec(&pdev->dev, &soc_codec_dev_gbaudio,
+			gbaudio_dai, ARRAY_SIZE(gbaudio_dai));
+}
+
+static int gbaudio_codec_remove(struct platform_device *pdev)
+{
+	snd_soc_unregister_codec(&pdev->dev);
+	return 0;
+}
+
+static const struct of_device_id greybus_asoc_machine_of_match[]  = {
+	{ .compatible = "toshiba,apb-dummy-codec", },
+	{},
+};
+
+static struct platform_driver gbaudio_codec_driver = {
+	.driver = {
+		.name = "apb-dummy-codec",
+		.owner = THIS_MODULE,
+#ifdef CONFIG_PM
+		.pm = &gbaudio_codec_pm_ops,
+#endif
+		.of_match_table = greybus_asoc_machine_of_match,
+	},
+	.probe = gbaudio_codec_probe,
+	.remove = gbaudio_codec_remove,
+};
+module_platform_driver(gbaudio_codec_driver);
+
+MODULE_DESCRIPTION("APBridge ALSA SoC dummy codec driver");
+MODULE_AUTHOR("Vaibhav Agarwal <vaibhav.agarwal@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:apb-dummy-codec");
diff --git a/drivers/staging/greybus/audio_codec.h b/drivers/staging/greybus/audio_codec.h
new file mode 100644
index 000000000000..ca027bd99ad7
--- /dev/null
+++ b/drivers/staging/greybus/audio_codec.h
@@ -0,0 +1,283 @@
+/*
+ * Greybus audio driver
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __LINUX_GBAUDIO_CODEC_H
+#define __LINUX_GBAUDIO_CODEC_H
+
+#include <sound/soc.h>
+#include <sound/jack.h>
+
+#include "greybus.h"
+#include "greybus_protocols.h"
+
+#define NAME_SIZE	32
+#define MAX_DAIS	2	/* APB1, APB2 */
+
+enum {
+	APB1_PCM = 0,
+	APB2_PCM,
+	NUM_CODEC_DAIS,
+};
+
+enum gbcodec_reg_index {
+	GBCODEC_CTL_REG,
+	GBCODEC_MUTE_REG,
+	GBCODEC_PB_LVOL_REG,
+	GBCODEC_PB_RVOL_REG,
+	GBCODEC_CAP_LVOL_REG,
+	GBCODEC_CAP_RVOL_REG,
+	GBCODEC_APB1_MUX_REG,
+	GBCODEC_APB2_MUX_REG,
+	GBCODEC_REG_COUNT
+};
+
+/* device_type should be same as defined in audio.h (Android media layer) */
+enum {
+	GBAUDIO_DEVICE_NONE                     = 0x0,
+	/* reserved bits */
+	GBAUDIO_DEVICE_BIT_IN                   = 0x80000000,
+	GBAUDIO_DEVICE_BIT_DEFAULT              = 0x40000000,
+	/* output devices */
+	GBAUDIO_DEVICE_OUT_SPEAKER              = 0x2,
+	GBAUDIO_DEVICE_OUT_WIRED_HEADSET        = 0x4,
+	GBAUDIO_DEVICE_OUT_WIRED_HEADPHONE      = 0x8,
+	/* input devices */
+	GBAUDIO_DEVICE_IN_BUILTIN_MIC           = GBAUDIO_DEVICE_BIT_IN | 0x4,
+	GBAUDIO_DEVICE_IN_WIRED_HEADSET         = GBAUDIO_DEVICE_BIT_IN | 0x10,
+};
+
+/* bit 0-SPK, 1-HP, 2-DAC,
+ * 4-MIC, 5-HSMIC, 6-MIC2
+ */
+#define GBCODEC_CTL_REG_DEFAULT		0x00
+
+/* bit 0,1 - APB1-PB-L/R
+ * bit 2,3 - APB2-PB-L/R
+ * bit 4,5 - APB1-Cap-L/R
+ * bit 6,7 - APB2-Cap-L/R
+ */
+#define	GBCODEC_MUTE_REG_DEFAULT	0x00
+
+/* 0-127 steps */
+#define	GBCODEC_PB_VOL_REG_DEFAULT	0x00
+#define	GBCODEC_CAP_VOL_REG_DEFAULT	0x00
+
+/* bit 0,1,2 - PB stereo, left, right
+ * bit 8,9,10 - Cap stereo, left, right
+ */
+#define GBCODEC_APB1_MUX_REG_DEFAULT	0x00
+#define GBCODEC_APB2_MUX_REG_DEFAULT	0x00
+
+#define GBCODEC_JACK_MASK		0x0000FFFF
+#define GBCODEC_JACK_BUTTON_MASK	0xFFFF0000
+
+static const u8 gbcodec_reg_defaults[GBCODEC_REG_COUNT] = {
+	GBCODEC_CTL_REG_DEFAULT,
+	GBCODEC_MUTE_REG_DEFAULT,
+	GBCODEC_PB_VOL_REG_DEFAULT,
+	GBCODEC_PB_VOL_REG_DEFAULT,
+	GBCODEC_CAP_VOL_REG_DEFAULT,
+	GBCODEC_CAP_VOL_REG_DEFAULT,
+	GBCODEC_APB1_MUX_REG_DEFAULT,
+	GBCODEC_APB2_MUX_REG_DEFAULT,
+};
+
+enum gbaudio_codec_state {
+	GBAUDIO_CODEC_SHUTDOWN = 0,
+	GBAUDIO_CODEC_STARTUP,
+	GBAUDIO_CODEC_HWPARAMS,
+	GBAUDIO_CODEC_PREPARE,
+	GBAUDIO_CODEC_START,
+	GBAUDIO_CODEC_STOP,
+};
+
+struct gbaudio_stream_params {
+	int state;
+	u8 sig_bits, channels;
+	uint32_t format, rate;
+};
+
+struct gbaudio_codec_dai {
+	int id;
+	/* runtime params for playback/capture streams */
+	struct gbaudio_stream_params params[2];
+	struct list_head list;
+};
+
+struct gbaudio_codec_info {
+	struct device *dev;
+	struct snd_soc_codec *codec;
+	struct list_head module_list;
+	/* to maintain runtime stream params for each DAI */
+	struct list_head dai_list;
+	struct mutex lock;
+	u8 reg[GBCODEC_REG_COUNT];
+};
+
+struct gbaudio_widget {
+	__u8 id;
+	const char *name;
+	struct list_head list;
+};
+
+struct gbaudio_control {
+	__u8 id;
+	char *name;
+	char *wname;
+	const char * const *texts;
+	int items;
+	struct list_head list;
+};
+
+struct gbaudio_data_connection {
+	int id;
+	__le16 data_cport;
+	struct gb_connection *connection;
+	struct list_head list;
+	/* maintain runtime state for playback/capture stream */
+	int state[2];
+};
+
+/* stream direction */
+#define GB_PLAYBACK	BIT(0)
+#define GB_CAPTURE	BIT(1)
+
+enum gbaudio_module_state {
+	GBAUDIO_MODULE_OFF = 0,
+	GBAUDIO_MODULE_ON,
+};
+
+struct gbaudio_module_info {
+	/* module info */
+	struct device *dev;
+	int dev_id;	/* check if it should be bundle_id/hd_cport_id */
+	int vid;
+	int pid;
+	int slot;
+	int type;
+	int set_uevent;
+	char vstr[NAME_SIZE];
+	char pstr[NAME_SIZE];
+	struct list_head list;
+	/* need to share this info to above user space */
+	int manager_id;
+	char name[NAME_SIZE];
+	unsigned int ip_devices;
+	unsigned int op_devices;
+
+	/* jack related */
+	char jack_name[NAME_SIZE];
+	char button_name[NAME_SIZE];
+	int jack_type;
+	int jack_mask;
+	int button_mask;
+	int button_status;
+	struct snd_soc_jack headset_jack;
+	struct snd_soc_jack button_jack;
+
+	/* connection info */
+	struct gb_connection *mgmt_connection;
+	size_t num_data_connections;
+	struct list_head data_list;
+
+	/* topology related */
+	int num_dais;
+	int num_controls;
+	int num_dapm_widgets;
+	int num_dapm_routes;
+	unsigned long dai_offset;
+	unsigned long widget_offset;
+	unsigned long control_offset;
+	unsigned long route_offset;
+	struct snd_kcontrol_new *controls;
+	struct snd_soc_dapm_widget *dapm_widgets;
+	struct snd_soc_dapm_route *dapm_routes;
+	struct snd_soc_dai_driver *dais;
+
+	struct list_head widget_list;
+	struct list_head ctl_list;
+	struct list_head widget_ctl_list;
+
+	struct gb_audio_topology *topology;
+};
+
+int gbaudio_tplg_parse_data(struct gbaudio_module_info *module,
+			       struct gb_audio_topology *tplg_data);
+void gbaudio_tplg_release(struct gbaudio_module_info *module);
+
+int gbaudio_module_update(struct gbaudio_codec_info *codec,
+			  struct snd_soc_dapm_widget *w,
+			  struct gbaudio_module_info *module,
+			  int enable);
+int gbaudio_register_module(struct gbaudio_module_info *module);
+void gbaudio_unregister_module(struct gbaudio_module_info *module);
+
+/* protocol related */
+extern int gb_audio_gb_get_topology(struct gb_connection *connection,
+				    struct gb_audio_topology **topology);
+extern int gb_audio_gb_get_control(struct gb_connection *connection,
+				   u8 control_id, u8 index,
+				   struct gb_audio_ctl_elem_value *value);
+extern int gb_audio_gb_set_control(struct gb_connection *connection,
+				   u8 control_id, u8 index,
+				   struct gb_audio_ctl_elem_value *value);
+extern int gb_audio_gb_enable_widget(struct gb_connection *connection,
+				     u8 widget_id);
+extern int gb_audio_gb_disable_widget(struct gb_connection *connection,
+				      u8 widget_id);
+extern int gb_audio_gb_get_pcm(struct gb_connection *connection,
+			       u16 data_cport, uint32_t *format,
+			       uint32_t *rate, u8 *channels,
+			       u8 *sig_bits);
+extern int gb_audio_gb_set_pcm(struct gb_connection *connection,
+			       u16 data_cport, uint32_t format,
+			       uint32_t rate, u8 channels,
+			       u8 sig_bits);
+extern int gb_audio_gb_set_tx_data_size(struct gb_connection *connection,
+					u16 data_cport, u16 size);
+extern int gb_audio_gb_activate_tx(struct gb_connection *connection,
+				   u16 data_cport);
+extern int gb_audio_gb_deactivate_tx(struct gb_connection *connection,
+				     u16 data_cport);
+extern int gb_audio_gb_set_rx_data_size(struct gb_connection *connection,
+					u16 data_cport, u16 size);
+extern int gb_audio_gb_activate_rx(struct gb_connection *connection,
+				   u16 data_cport);
+extern int gb_audio_gb_deactivate_rx(struct gb_connection *connection,
+				     u16 data_cport);
+extern int gb_audio_apbridgea_set_config(struct gb_connection *connection,
+					 __u16 i2s_port, __u32 format,
+					 __u32 rate, __u32 mclk_freq);
+extern int gb_audio_apbridgea_register_cport(struct gb_connection *connection,
+					     __u16 i2s_port, __u16 cportid,
+					     __u8 direction);
+extern int gb_audio_apbridgea_unregister_cport(struct gb_connection *connection,
+					       __u16 i2s_port, __u16 cportid,
+					       __u8 direction);
+extern int gb_audio_apbridgea_set_tx_data_size(struct gb_connection *connection,
+					       __u16 i2s_port, __u16 size);
+extern int gb_audio_apbridgea_prepare_tx(struct gb_connection *connection,
+					 __u16 i2s_port);
+extern int gb_audio_apbridgea_start_tx(struct gb_connection *connection,
+				       __u16 i2s_port, __u64 timestamp);
+extern int gb_audio_apbridgea_stop_tx(struct gb_connection *connection,
+				      __u16 i2s_port);
+extern int gb_audio_apbridgea_shutdown_tx(struct gb_connection *connection,
+					  __u16 i2s_port);
+extern int gb_audio_apbridgea_set_rx_data_size(struct gb_connection *connection,
+					       __u16 i2s_port, __u16 size);
+extern int gb_audio_apbridgea_prepare_rx(struct gb_connection *connection,
+					 __u16 i2s_port);
+extern int gb_audio_apbridgea_start_rx(struct gb_connection *connection,
+				       __u16 i2s_port);
+extern int gb_audio_apbridgea_stop_rx(struct gb_connection *connection,
+				      __u16 i2s_port);
+extern int gb_audio_apbridgea_shutdown_rx(struct gb_connection *connection,
+					  __u16 i2s_port);
+
+#endif /* __LINUX_GBAUDIO_CODEC_H */
diff --git a/drivers/staging/greybus/audio_gb.c b/drivers/staging/greybus/audio_gb.c
new file mode 100644
index 000000000000..42f287dd7b84
--- /dev/null
+++ b/drivers/staging/greybus/audio_gb.c
@@ -0,0 +1,228 @@
+/*
+ * Greybus Audio Device Class Protocol helpers
+ *
+ * Copyright 2015-2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include "greybus.h"
+#include "greybus_protocols.h"
+#include "operation.h"
+#include "audio_codec.h"
+
+/* TODO: Split into separate calls */
+int gb_audio_gb_get_topology(struct gb_connection *connection,
+			     struct gb_audio_topology **topology)
+{
+	struct gb_audio_get_topology_size_response size_resp;
+	struct gb_audio_topology *topo;
+	u16 size;
+	int ret;
+
+	ret = gb_operation_sync(connection, GB_AUDIO_TYPE_GET_TOPOLOGY_SIZE,
+				NULL, 0, &size_resp, sizeof(size_resp));
+	if (ret)
+		return ret;
+
+	size = le16_to_cpu(size_resp.size);
+	if (size < sizeof(*topo))
+		return -ENODATA;
+
+	topo = kzalloc(size, GFP_KERNEL);
+	if (!topo)
+		return -ENOMEM;
+
+	ret = gb_operation_sync(connection, GB_AUDIO_TYPE_GET_TOPOLOGY, NULL, 0,
+				topo, size);
+	if (ret) {
+		kfree(topo);
+		return ret;
+	}
+
+	*topology = topo;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_get_topology);
+
+int gb_audio_gb_get_control(struct gb_connection *connection,
+			    u8 control_id, u8 index,
+			    struct gb_audio_ctl_elem_value *value)
+{
+	struct gb_audio_get_control_request req;
+	struct gb_audio_get_control_response resp;
+	int ret;
+
+	req.control_id = control_id;
+	req.index = index;
+
+	ret = gb_operation_sync(connection, GB_AUDIO_TYPE_GET_CONTROL,
+				&req, sizeof(req), &resp, sizeof(resp));
+	if (ret)
+		return ret;
+
+	memcpy(value, &resp.value, sizeof(*value));
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_get_control);
+
+int gb_audio_gb_set_control(struct gb_connection *connection,
+			    u8 control_id, u8 index,
+			    struct gb_audio_ctl_elem_value *value)
+{
+	struct gb_audio_set_control_request req;
+
+	req.control_id = control_id;
+	req.index = index;
+	memcpy(&req.value, value, sizeof(req.value));
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_SET_CONTROL,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_set_control);
+
+int gb_audio_gb_enable_widget(struct gb_connection *connection,
+			      u8 widget_id)
+{
+	struct gb_audio_enable_widget_request req;
+
+	req.widget_id = widget_id;
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_ENABLE_WIDGET,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_enable_widget);
+
+int gb_audio_gb_disable_widget(struct gb_connection *connection,
+			       u8 widget_id)
+{
+	struct gb_audio_disable_widget_request req;
+
+	req.widget_id = widget_id;
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_DISABLE_WIDGET,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_disable_widget);
+
+int gb_audio_gb_get_pcm(struct gb_connection *connection, u16 data_cport,
+			uint32_t *format, uint32_t *rate, u8 *channels,
+			u8 *sig_bits)
+{
+	struct gb_audio_get_pcm_request req;
+	struct gb_audio_get_pcm_response resp;
+	int ret;
+
+	req.data_cport = cpu_to_le16(data_cport);
+
+	ret = gb_operation_sync(connection, GB_AUDIO_TYPE_GET_PCM,
+				&req, sizeof(req), &resp, sizeof(resp));
+	if (ret)
+		return ret;
+
+	*format = le32_to_cpu(resp.format);
+	*rate = le32_to_cpu(resp.rate);
+	*channels = resp.channels;
+	*sig_bits = resp.sig_bits;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_get_pcm);
+
+int gb_audio_gb_set_pcm(struct gb_connection *connection, u16 data_cport,
+			uint32_t format, uint32_t rate, u8 channels,
+			u8 sig_bits)
+{
+	struct gb_audio_set_pcm_request req;
+
+	req.data_cport = cpu_to_le16(data_cport);
+	req.format = cpu_to_le32(format);
+	req.rate = cpu_to_le32(rate);
+	req.channels = channels;
+	req.sig_bits = sig_bits;
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_SET_PCM,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_set_pcm);
+
+int gb_audio_gb_set_tx_data_size(struct gb_connection *connection,
+				 u16 data_cport, u16 size)
+{
+	struct gb_audio_set_tx_data_size_request req;
+
+	req.data_cport = cpu_to_le16(data_cport);
+	req.size = cpu_to_le16(size);
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_SET_TX_DATA_SIZE,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_set_tx_data_size);
+
+int gb_audio_gb_activate_tx(struct gb_connection *connection,
+			    u16 data_cport)
+{
+	struct gb_audio_activate_tx_request req;
+
+	req.data_cport = cpu_to_le16(data_cport);
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_ACTIVATE_TX,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_activate_tx);
+
+int gb_audio_gb_deactivate_tx(struct gb_connection *connection,
+			      u16 data_cport)
+{
+	struct gb_audio_deactivate_tx_request req;
+
+	req.data_cport = cpu_to_le16(data_cport);
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_DEACTIVATE_TX,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_deactivate_tx);
+
+int gb_audio_gb_set_rx_data_size(struct gb_connection *connection,
+				 u16 data_cport, u16 size)
+{
+	struct gb_audio_set_rx_data_size_request req;
+
+	req.data_cport = cpu_to_le16(data_cport);
+	req.size = cpu_to_le16(size);
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_SET_RX_DATA_SIZE,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_set_rx_data_size);
+
+int gb_audio_gb_activate_rx(struct gb_connection *connection,
+			    u16 data_cport)
+{
+	struct gb_audio_activate_rx_request req;
+
+	req.data_cport = cpu_to_le16(data_cport);
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_ACTIVATE_RX,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_activate_rx);
+
+int gb_audio_gb_deactivate_rx(struct gb_connection *connection,
+			      u16 data_cport)
+{
+	struct gb_audio_deactivate_rx_request req;
+
+	req.data_cport = cpu_to_le16(data_cport);
+
+	return gb_operation_sync(connection, GB_AUDIO_TYPE_DEACTIVATE_RX,
+				 &req, sizeof(req), NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gb_audio_gb_deactivate_rx);
+
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("greybus:audio-gb");
+MODULE_DESCRIPTION("Greybus Audio Device Class Protocol library");
+MODULE_AUTHOR("Mark Greer <mgreer@animalcreek.com>");
diff --git a/drivers/staging/greybus/audio_manager.c b/drivers/staging/greybus/audio_manager.c
new file mode 100644
index 000000000000..aa6508b44fab
--- /dev/null
+++ b/drivers/staging/greybus/audio_manager.c
@@ -0,0 +1,184 @@
+/*
+ * Greybus operations
+ *
+ * Copyright 2015-2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/string.h>
+#include <linux/sysfs.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/rwlock.h>
+#include <linux/idr.h>
+
+#include "audio_manager.h"
+#include "audio_manager_private.h"
+
+static struct kset *manager_kset;
+
+static LIST_HEAD(modules_list);
+static DECLARE_RWSEM(modules_rwsem);
+static DEFINE_IDA(module_id);
+
+/* helpers */
+static struct gb_audio_manager_module *gb_audio_manager_get_locked(int id)
+{
+	struct gb_audio_manager_module *module;
+
+	if (id < 0)
+		return NULL;
+
+	list_for_each_entry(module, &modules_list, list) {
+		if (module->id == id)
+			return module;
+	}
+
+	return NULL;
+}
+
+/* public API */
+int gb_audio_manager_add(struct gb_audio_manager_module_descriptor *desc)
+{
+	struct gb_audio_manager_module *module;
+	int id;
+	int err;
+
+	id = ida_simple_get(&module_id, 0, 0, GFP_KERNEL);
+	err = gb_audio_manager_module_create(&module, manager_kset,
+					     id, desc);
+	if (err) {
+		ida_simple_remove(&module_id, id);
+		return err;
+	}
+
+	/* Add it to the list */
+	down_write(&modules_rwsem);
+	list_add_tail(&module->list, &modules_list);
+	up_write(&modules_rwsem);
+
+	return module->id;
+}
+EXPORT_SYMBOL_GPL(gb_audio_manager_add);
+
+int gb_audio_manager_remove(int id)
+{
+	struct gb_audio_manager_module *module;
+
+	down_write(&modules_rwsem);
+
+	module = gb_audio_manager_get_locked(id);
+	if (!module) {
+		up_write(&modules_rwsem);
+		return -EINVAL;
+	}
+	list_del(&module->list);
+	kobject_put(&module->kobj);
+	up_write(&modules_rwsem);
+	ida_simple_remove(&module_id, id);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gb_audio_manager_remove);
+
+void gb_audio_manager_remove_all(void)
+{
+	struct gb_audio_manager_module *module, *next;
+	int is_empty = 1;
+
+	down_write(&modules_rwsem);
+
+	list_for_each_entry_safe(module, next, &modules_list, list) {
+		list_del(&module->list);
+		kobject_put(&module->kobj);
+		ida_simple_remove(&module_id, module->id);
+	}
+
+	is_empty = list_empty(&modules_list);
+
+	up_write(&modules_rwsem);
+
+	if (!is_empty)
+		pr_warn("Not all nodes were deleted\n");
+}
+EXPORT_SYMBOL_GPL(gb_audio_manager_remove_all);
+
+struct gb_audio_manager_module *gb_audio_manager_get_module(int id)
+{
+	struct gb_audio_manager_module *module;
+
+	down_read(&modules_rwsem);
+	module = gb_audio_manager_get_locked(id);
+	kobject_get(&module->kobj);
+	up_read(&modules_rwsem);
+	return module;
+}
+EXPORT_SYMBOL_GPL(gb_audio_manager_get_module);
+
+void gb_audio_manager_put_module(struct gb_audio_manager_module *module)
+{
+	kobject_put(&module->kobj);
+}
+EXPORT_SYMBOL_GPL(gb_audio_manager_put_module);
+
+int gb_audio_manager_dump_module(int id)
+{
+	struct gb_audio_manager_module *module;
+
+	down_read(&modules_rwsem);
+	module = gb_audio_manager_get_locked(id);
+	up_read(&modules_rwsem);
+
+	if (!module)
+		return -EINVAL;
+
+	gb_audio_manager_module_dump(module);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gb_audio_manager_dump_module);
+
+void gb_audio_manager_dump_all(void)
+{
+	struct gb_audio_manager_module *module;
+	int count = 0;
+
+	down_read(&modules_rwsem);
+	list_for_each_entry(module, &modules_list, list) {
+		gb_audio_manager_module_dump(module);
+		count++;
+	}
+	up_read(&modules_rwsem);
+
+	pr_info("Number of connected modules: %d\n", count);
+}
+EXPORT_SYMBOL_GPL(gb_audio_manager_dump_all);
+
+/*
+ * module init/deinit
+ */
+static int __init manager_init(void)
+{
+	manager_kset = kset_create_and_add(GB_AUDIO_MANAGER_NAME, NULL,
+					   kernel_kobj);
+	if (!manager_kset)
+		return -ENOMEM;
+
+#ifdef GB_AUDIO_MANAGER_SYSFS
+	gb_audio_manager_sysfs_init(&manager_kset->kobj);
+#endif
+
+	return 0;
+}
+
+static void __exit manager_exit(void)
+{
+	gb_audio_manager_remove_all();
+	kset_unregister(manager_kset);
+	ida_destroy(&module_id);
+}
+
+module_init(manager_init);
+module_exit(manager_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Svetlin Ankov <ankov_svetlin@projectara.com>");
diff --git a/drivers/staging/greybus/audio_manager.h b/drivers/staging/greybus/audio_manager.h
new file mode 100644
index 000000000000..c4ca09754a6a
--- /dev/null
+++ b/drivers/staging/greybus/audio_manager.h
@@ -0,0 +1,83 @@
+/*
+ * Greybus operations
+ *
+ * Copyright 2015-2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef _GB_AUDIO_MANAGER_H_
+#define _GB_AUDIO_MANAGER_H_
+
+#include <linux/kobject.h>
+#include <linux/list.h>
+
+#define GB_AUDIO_MANAGER_NAME "gb_audio_manager"
+#define GB_AUDIO_MANAGER_MODULE_NAME_LEN 64
+#define GB_AUDIO_MANAGER_MODULE_NAME_LEN_SSCANF "63"
+
+struct gb_audio_manager_module_descriptor {
+	char name[GB_AUDIO_MANAGER_MODULE_NAME_LEN];
+	int slot;
+	int vid;
+	int pid;
+	int cport;
+	unsigned int ip_devices;
+	unsigned int op_devices;
+};
+
+struct gb_audio_manager_module {
+	struct kobject kobj;
+	struct list_head list;
+	int id;
+	struct gb_audio_manager_module_descriptor desc;
+};
+
+/*
+ * Creates a new gb_audio_manager_module_descriptor, using the specified
+ * descriptor.
+ *
+ * Returns a negative result on error, or the id of the newly created module.
+ *
+ */
+int gb_audio_manager_add(struct gb_audio_manager_module_descriptor *desc);
+
+/*
+ * Removes a connected gb_audio_manager_module_descriptor for the specified ID.
+ *
+ * Returns zero on success, or a negative value on error.
+ */
+int gb_audio_manager_remove(int id);
+
+/*
+ * Removes all connected gb_audio_modules
+ *
+ * Returns zero on success, or a negative value on error.
+ */
+void gb_audio_manager_remove_all(void);
+
+/*
+ * Retrieves a gb_audio_manager_module_descriptor for the specified id.
+ * Returns the gb_audio_manager_module_descriptor structure,
+ * or NULL if there is no module with the specified ID.
+ */
+struct gb_audio_manager_module *gb_audio_manager_get_module(int id);
+
+/*
+ * Decreases the refcount of the module, obtained by the get function.
+ * Modules are removed via gb_audio_manager_remove
+ */
+void gb_audio_manager_put_module(struct gb_audio_manager_module *module);
+
+/*
+ * Dumps the module for the specified id
+ * Return 0 on success
+ */
+int gb_audio_manager_dump_module(int id);
+
+/*
+ * Dumps all connected modules
+ */
+void gb_audio_manager_dump_all(void);
+
+#endif /* _GB_AUDIO_MANAGER_H_ */
diff --git a/drivers/staging/greybus/audio_manager_module.c b/drivers/staging/greybus/audio_manager_module.c
new file mode 100644
index 000000000000..a10e96ad79c1
--- /dev/null
+++ b/drivers/staging/greybus/audio_manager_module.c
@@ -0,0 +1,258 @@
+/*
+ * Greybus operations
+ *
+ * Copyright 2015-2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/slab.h>
+
+#include "audio_manager.h"
+#include "audio_manager_private.h"
+
+#define to_gb_audio_module_attr(x)	\
+		container_of(x, struct gb_audio_manager_module_attribute, attr)
+#define to_gb_audio_module(x)		\
+		container_of(x, struct gb_audio_manager_module, kobj)
+
+struct gb_audio_manager_module_attribute {
+	struct attribute attr;
+	ssize_t (*show)(struct gb_audio_manager_module *module,
+			struct gb_audio_manager_module_attribute *attr,
+			char *buf);
+	ssize_t (*store)(struct gb_audio_manager_module *module,
+			 struct gb_audio_manager_module_attribute *attr,
+			 const char *buf, size_t count);
+};
+
+static ssize_t gb_audio_module_attr_show(
+	struct kobject *kobj, struct attribute *attr, char *buf)
+{
+	struct gb_audio_manager_module_attribute *attribute;
+	struct gb_audio_manager_module *module;
+
+	attribute = to_gb_audio_module_attr(attr);
+	module = to_gb_audio_module(kobj);
+
+	if (!attribute->show)
+		return -EIO;
+
+	return attribute->show(module, attribute, buf);
+}
+
+static ssize_t gb_audio_module_attr_store(struct kobject *kobj,
+					  struct attribute *attr,
+					  const char *buf, size_t len)
+{
+	struct gb_audio_manager_module_attribute *attribute;
+	struct gb_audio_manager_module *module;
+
+	attribute = to_gb_audio_module_attr(attr);
+	module = to_gb_audio_module(kobj);
+
+	if (!attribute->store)
+		return -EIO;
+
+	return attribute->store(module, attribute, buf, len);
+}
+
+static const struct sysfs_ops gb_audio_module_sysfs_ops = {
+	.show = gb_audio_module_attr_show,
+	.store = gb_audio_module_attr_store,
+};
+
+static void gb_audio_module_release(struct kobject *kobj)
+{
+	struct gb_audio_manager_module *module = to_gb_audio_module(kobj);
+
+	pr_info("Destroying audio module #%d\n", module->id);
+	/* TODO -> delete from list */
+	kfree(module);
+}
+
+static ssize_t gb_audio_module_name_show(
+	struct gb_audio_manager_module *module,
+	struct gb_audio_manager_module_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%s", module->desc.name);
+}
+
+static struct gb_audio_manager_module_attribute gb_audio_module_name_attribute =
+	__ATTR(name, 0664, gb_audio_module_name_show, NULL);
+
+static ssize_t gb_audio_module_slot_show(
+	struct gb_audio_manager_module *module,
+	struct gb_audio_manager_module_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d", module->desc.slot);
+}
+
+static struct gb_audio_manager_module_attribute gb_audio_module_slot_attribute =
+	__ATTR(slot, 0664, gb_audio_module_slot_show, NULL);
+
+static ssize_t gb_audio_module_vid_show(
+	struct gb_audio_manager_module *module,
+	struct gb_audio_manager_module_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d", module->desc.vid);
+}
+
+static struct gb_audio_manager_module_attribute gb_audio_module_vid_attribute =
+	__ATTR(vid, 0664, gb_audio_module_vid_show, NULL);
+
+static ssize_t gb_audio_module_pid_show(
+	struct gb_audio_manager_module *module,
+	struct gb_audio_manager_module_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d", module->desc.pid);
+}
+
+static struct gb_audio_manager_module_attribute gb_audio_module_pid_attribute =
+	__ATTR(pid, 0664, gb_audio_module_pid_show, NULL);
+
+static ssize_t gb_audio_module_cport_show(
+	struct gb_audio_manager_module *module,
+	struct gb_audio_manager_module_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d", module->desc.cport);
+}
+
+static struct gb_audio_manager_module_attribute
+					gb_audio_module_cport_attribute =
+	__ATTR(cport, 0664, gb_audio_module_cport_show, NULL);
+
+static ssize_t gb_audio_module_ip_devices_show(
+	struct gb_audio_manager_module *module,
+	struct gb_audio_manager_module_attribute *attr, char *buf)
+{
+	return sprintf(buf, "0x%X", module->desc.ip_devices);
+}
+
+static struct gb_audio_manager_module_attribute
+					gb_audio_module_ip_devices_attribute =
+	__ATTR(ip_devices, 0664, gb_audio_module_ip_devices_show, NULL);
+
+static ssize_t gb_audio_module_op_devices_show(
+	struct gb_audio_manager_module *module,
+	struct gb_audio_manager_module_attribute *attr, char *buf)
+{
+	return sprintf(buf, "0x%X", module->desc.op_devices);
+}
+
+static struct gb_audio_manager_module_attribute
+					gb_audio_module_op_devices_attribute =
+	__ATTR(op_devices, 0664, gb_audio_module_op_devices_show, NULL);
+
+static struct attribute *gb_audio_module_default_attrs[] = {
+	&gb_audio_module_name_attribute.attr,
+	&gb_audio_module_slot_attribute.attr,
+	&gb_audio_module_vid_attribute.attr,
+	&gb_audio_module_pid_attribute.attr,
+	&gb_audio_module_cport_attribute.attr,
+	&gb_audio_module_ip_devices_attribute.attr,
+	&gb_audio_module_op_devices_attribute.attr,
+	NULL,   /* need to NULL terminate the list of attributes */
+};
+
+static struct kobj_type gb_audio_module_type = {
+	.sysfs_ops = &gb_audio_module_sysfs_ops,
+	.release = gb_audio_module_release,
+	.default_attrs = gb_audio_module_default_attrs,
+};
+
+static void send_add_uevent(struct gb_audio_manager_module *module)
+{
+	char name_string[128];
+	char slot_string[64];
+	char vid_string[64];
+	char pid_string[64];
+	char cport_string[64];
+	char ip_devices_string[64];
+	char op_devices_string[64];
+
+	char *envp[] = {
+		name_string,
+		slot_string,
+		vid_string,
+		pid_string,
+		cport_string,
+		ip_devices_string,
+		op_devices_string,
+		NULL
+	};
+
+	snprintf(name_string, 128, "NAME=%s", module->desc.name);
+	snprintf(slot_string, 64, "SLOT=%d", module->desc.slot);
+	snprintf(vid_string, 64, "VID=%d", module->desc.vid);
+	snprintf(pid_string, 64, "PID=%d", module->desc.pid);
+	snprintf(cport_string, 64, "CPORT=%d", module->desc.cport);
+	snprintf(ip_devices_string, 64, "I/P DEVICES=0x%X",
+		 module->desc.ip_devices);
+	snprintf(op_devices_string, 64, "O/P DEVICES=0x%X",
+		 module->desc.op_devices);
+
+	kobject_uevent_env(&module->kobj, KOBJ_ADD, envp);
+}
+
+int gb_audio_manager_module_create(
+	struct gb_audio_manager_module **module,
+	struct kset *manager_kset,
+	int id, struct gb_audio_manager_module_descriptor *desc)
+{
+	int err;
+	struct gb_audio_manager_module *m;
+
+	m = kzalloc(sizeof(*m), GFP_ATOMIC);
+	if (!m)
+		return -ENOMEM;
+
+	/* Initialize the node */
+	INIT_LIST_HEAD(&m->list);
+
+	/* Set the module id */
+	m->id = id;
+
+	/* Copy the provided descriptor */
+	memcpy(&m->desc, desc, sizeof(*desc));
+
+	/* set the kset */
+	m->kobj.kset = manager_kset;
+
+	/*
+	 * Initialize and add the kobject to the kernel.  All the default files
+	 * will be created here.  As we have already specified a kset for this
+	 * kobject, we don't have to set a parent for the kobject, the kobject
+	 * will be placed beneath that kset automatically.
+	 */
+	err = kobject_init_and_add(&m->kobj, &gb_audio_module_type, NULL, "%d",
+				   id);
+	if (err) {
+		pr_err("failed initializing kobject for audio module #%d\n",
+		       id);
+		kobject_put(&m->kobj);
+		return err;
+	}
+
+	/*
+	 * Notify the object was created
+	 */
+	send_add_uevent(m);
+
+	*module = m;
+	pr_info("Created audio module #%d\n", id);
+	return 0;
+}
+
+void gb_audio_manager_module_dump(struct gb_audio_manager_module *module)
+{
+	pr_info("audio module #%d name=%s slot=%d vid=%d pid=%d cport=%d i/p devices=0x%X o/p devices=0x%X\n",
+		module->id,
+		module->desc.name,
+		module->desc.slot,
+		module->desc.vid,
+		module->desc.pid,
+		module->desc.cport,
+		module->desc.ip_devices,
+		module->desc.op_devices);
+}
diff --git a/drivers/staging/greybus/audio_manager_private.h b/drivers/staging/greybus/audio_manager_private.h
new file mode 100644
index 000000000000..079ce953c256
--- /dev/null
+++ b/drivers/staging/greybus/audio_manager_private.h
@@ -0,0 +1,28 @@
+/*
+ * Greybus operations
+ *
+ * Copyright 2015-2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef _GB_AUDIO_MANAGER_PRIVATE_H_
+#define _GB_AUDIO_MANAGER_PRIVATE_H_
+
+#include <linux/kobject.h>
+
+#include "audio_manager.h"
+
+int gb_audio_manager_module_create(
+	struct gb_audio_manager_module **module,
+	struct kset *manager_kset,
+	int id, struct gb_audio_manager_module_descriptor *desc);
+
+/* module destroyed via kobject_put */
+
+void gb_audio_manager_module_dump(struct gb_audio_manager_module *module);
+
+/* sysfs control */
+void gb_audio_manager_sysfs_init(struct kobject *kobj);
+
+#endif /* _GB_AUDIO_MANAGER_PRIVATE_H_ */
diff --git a/drivers/staging/greybus/audio_manager_sysfs.c b/drivers/staging/greybus/audio_manager_sysfs.c
new file mode 100644
index 000000000000..d8bf8591ff9e
--- /dev/null
+++ b/drivers/staging/greybus/audio_manager_sysfs.c
@@ -0,0 +1,102 @@
+/*
+ * Greybus operations
+ *
+ * Copyright 2015-2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/string.h>
+#include <linux/sysfs.h>
+
+#include "audio_manager.h"
+#include "audio_manager_private.h"
+
+static ssize_t manager_sysfs_add_store(
+	struct kobject *kobj, struct kobj_attribute *attr,
+	const char *buf, size_t count)
+{
+	struct gb_audio_manager_module_descriptor desc = { {0} };
+
+	int num = sscanf(buf,
+			"name=%" GB_AUDIO_MANAGER_MODULE_NAME_LEN_SSCANF "s "
+			"slot=%d vid=%d pid=%d cport=%d i/p devices=0x%X"
+			"o/p devices=0x%X",
+			desc.name, &desc.slot, &desc.vid, &desc.pid,
+			&desc.cport, &desc.ip_devices, &desc.op_devices);
+
+	if (num != 7)
+		return -EINVAL;
+
+	num = gb_audio_manager_add(&desc);
+	if (num < 0)
+		return -EINVAL;
+
+	return count;
+}
+
+static struct kobj_attribute manager_add_attribute =
+	__ATTR(add, 0664, NULL, manager_sysfs_add_store);
+
+static ssize_t manager_sysfs_remove_store(
+	struct kobject *kobj, struct kobj_attribute *attr,
+	const char *buf, size_t count)
+{
+	int id;
+
+	int num = sscanf(buf, "%d", &id);
+
+	if (num != 1)
+		return -EINVAL;
+
+	num = gb_audio_manager_remove(id);
+	if (num)
+		return num;
+
+	return count;
+}
+
+static struct kobj_attribute manager_remove_attribute =
+	__ATTR(remove, 0664, NULL, manager_sysfs_remove_store);
+
+static ssize_t manager_sysfs_dump_store(
+	struct kobject *kobj, struct kobj_attribute *attr,
+	const char *buf, size_t count)
+{
+	int id;
+
+	int num = sscanf(buf, "%d", &id);
+
+	if (num == 1) {
+		num = gb_audio_manager_dump_module(id);
+		if (num)
+			return num;
+	} else if (!strncmp("all", buf, 3))
+		gb_audio_manager_dump_all();
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static struct kobj_attribute manager_dump_attribute =
+	__ATTR(dump, 0664, NULL, manager_sysfs_dump_store);
+
+static void manager_sysfs_init_attribute(
+		struct kobject *kobj, struct kobj_attribute *kattr)
+{
+	int err;
+
+	err = sysfs_create_file(kobj, &kattr->attr);
+	if (err) {
+		pr_warn("creating the sysfs entry for %s failed: %d\n",
+			kattr->attr.name, err);
+	}
+}
+
+void gb_audio_manager_sysfs_init(struct kobject *kobj)
+{
+	manager_sysfs_init_attribute(kobj, &manager_add_attribute);
+	manager_sysfs_init_attribute(kobj, &manager_remove_attribute);
+	manager_sysfs_init_attribute(kobj, &manager_dump_attribute);
+}
diff --git a/drivers/staging/greybus/audio_module.c b/drivers/staging/greybus/audio_module.c
new file mode 100644
index 000000000000..ae1c0fa85752
--- /dev/null
+++ b/drivers/staging/greybus/audio_module.c
@@ -0,0 +1,482 @@
+/*
+ * Greybus audio driver
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <sound/soc.h>
+#include <sound/pcm_params.h>
+
+#include "audio_codec.h"
+#include "audio_apbridgea.h"
+#include "audio_manager.h"
+
+/*
+ * gb_snd management functions
+ */
+
+static int gbaudio_request_jack(struct gbaudio_module_info *module,
+				  struct gb_audio_jack_event_request *req)
+{
+	int report;
+	struct snd_jack *jack = module->headset_jack.jack;
+	struct snd_jack *btn_jack = module->button_jack.jack;
+
+	if (!jack) {
+		dev_err_ratelimited(module->dev,
+			"Invalid jack event received:type: %u, event: %u\n",
+			req->jack_attribute, req->event);
+		return -EINVAL;
+	}
+
+	dev_warn_ratelimited(module->dev,
+			     "Jack Event received: type: %u, event: %u\n",
+			     req->jack_attribute, req->event);
+
+	if (req->event == GB_AUDIO_JACK_EVENT_REMOVAL) {
+		module->jack_type = 0;
+		if (btn_jack && module->button_status) {
+			snd_soc_jack_report(&module->button_jack, 0,
+					    module->button_mask);
+			module->button_status = 0;
+		}
+		snd_soc_jack_report(&module->headset_jack, 0,
+				    module->jack_mask);
+		return 0;
+	}
+
+	report = req->jack_attribute & module->jack_mask;
+	if (!report) {
+		dev_err_ratelimited(module->dev,
+			"Invalid jack event received:type: %u, event: %u\n",
+			req->jack_attribute, req->event);
+		return -EINVAL;
+	}
+
+	if (module->jack_type)
+		dev_warn_ratelimited(module->dev,
+				     "Modifying jack from %d to %d\n",
+				     module->jack_type, report);
+
+	module->jack_type = report;
+	snd_soc_jack_report(&module->headset_jack, report, module->jack_mask);
+
+	return 0;
+}
+
+static int gbaudio_request_button(struct gbaudio_module_info *module,
+				  struct gb_audio_button_event_request *req)
+{
+	int soc_button_id, report;
+	struct snd_jack *btn_jack = module->button_jack.jack;
+
+	if (!btn_jack) {
+		dev_err_ratelimited(module->dev,
+			"Invalid button event received:type: %u, event: %u\n",
+			req->button_id, req->event);
+		return -EINVAL;
+	}
+
+	dev_warn_ratelimited(module->dev,
+			     "Button Event received: id: %u, event: %u\n",
+			     req->button_id, req->event);
+
+	/* currently supports 4 buttons only */
+	if (!module->jack_type) {
+		dev_err_ratelimited(module->dev,
+				    "Jack not present. Bogus event!!\n");
+		return -EINVAL;
+	}
+
+	report = module->button_status & module->button_mask;
+	soc_button_id = 0;
+
+	switch (req->button_id) {
+	case 1:
+		soc_button_id = SND_JACK_BTN_0 & module->button_mask;
+		break;
+
+	case 2:
+		soc_button_id = SND_JACK_BTN_1 & module->button_mask;
+		break;
+
+	case 3:
+		soc_button_id = SND_JACK_BTN_2 & module->button_mask;
+		break;
+
+	case 4:
+		soc_button_id = SND_JACK_BTN_3 & module->button_mask;
+		break;
+	}
+
+	if (!soc_button_id) {
+		dev_err_ratelimited(module->dev,
+				    "Invalid button request received\n");
+		return -EINVAL;
+	}
+
+	if (req->event == GB_AUDIO_BUTTON_EVENT_PRESS)
+		report = report | soc_button_id;
+	else
+		report = report & ~soc_button_id;
+
+	module->button_status = report;
+
+	snd_soc_jack_report(&module->button_jack, report, module->button_mask);
+
+	return 0;
+}
+
+static int gbaudio_request_stream(struct gbaudio_module_info *module,
+				  struct gb_audio_streaming_event_request *req)
+{
+	dev_warn(module->dev, "Audio Event received: cport: %u, event: %u\n",
+		 req->data_cport, req->event);
+
+	return 0;
+}
+
+static int gbaudio_codec_request_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gbaudio_module_info *module =
+		greybus_get_drvdata(connection->bundle);
+	struct gb_operation_msg_hdr *header = op->request->header;
+	struct gb_audio_streaming_event_request *stream_req;
+	struct gb_audio_jack_event_request *jack_req;
+	struct gb_audio_button_event_request *button_req;
+	int ret;
+
+	switch (header->type) {
+	case GB_AUDIO_TYPE_STREAMING_EVENT:
+		stream_req = op->request->payload;
+		ret = gbaudio_request_stream(module, stream_req);
+		break;
+
+	case GB_AUDIO_TYPE_JACK_EVENT:
+		jack_req = op->request->payload;
+		ret = gbaudio_request_jack(module, jack_req);
+		break;
+
+	case GB_AUDIO_TYPE_BUTTON_EVENT:
+		button_req = op->request->payload;
+		ret = gbaudio_request_button(module, button_req);
+		break;
+
+	default:
+		dev_err_ratelimited(&connection->bundle->dev,
+				    "Invalid Audio Event received\n");
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int gb_audio_add_mgmt_connection(struct gbaudio_module_info *gbmodule,
+				struct greybus_descriptor_cport *cport_desc,
+				struct gb_bundle *bundle)
+{
+	struct gb_connection *connection;
+
+	/* Management Cport */
+	if (gbmodule->mgmt_connection) {
+		dev_err(&bundle->dev,
+			"Can't have multiple Management connections\n");
+		return -ENODEV;
+	}
+
+	connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
+					  gbaudio_codec_request_handler);
+	if (IS_ERR(connection))
+		return PTR_ERR(connection);
+
+	greybus_set_drvdata(bundle, gbmodule);
+	gbmodule->mgmt_connection = connection;
+
+	return 0;
+}
+
+static int gb_audio_add_data_connection(struct gbaudio_module_info *gbmodule,
+				struct greybus_descriptor_cport *cport_desc,
+				struct gb_bundle *bundle)
+{
+	struct gb_connection *connection;
+	struct gbaudio_data_connection *dai;
+
+	dai = devm_kzalloc(gbmodule->dev, sizeof(*dai), GFP_KERNEL);
+	if (!dai) {
+		dev_err(gbmodule->dev, "DAI Malloc failure\n");
+		return -ENOMEM;
+	}
+
+	connection = gb_connection_create_offloaded(bundle,
+					le16_to_cpu(cport_desc->id),
+					GB_CONNECTION_FLAG_CSD);
+	if (IS_ERR(connection)) {
+		devm_kfree(gbmodule->dev, dai);
+		return PTR_ERR(connection);
+	}
+
+	greybus_set_drvdata(bundle, gbmodule);
+	dai->id = 0;
+	dai->data_cport = connection->intf_cport_id;
+	dai->connection = connection;
+	list_add(&dai->list, &gbmodule->data_list);
+
+	return 0;
+}
+
+/*
+ * This is the basic hook get things initialized and registered w/ gb
+ */
+
+static int gb_audio_probe(struct gb_bundle *bundle,
+			  const struct greybus_bundle_id *id)
+{
+	struct device *dev = &bundle->dev;
+	struct gbaudio_module_info *gbmodule;
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_audio_manager_module_descriptor desc;
+	struct gbaudio_data_connection *dai, *_dai;
+	int ret, i;
+	struct gb_audio_topology *topology;
+
+	/* There should be at least one Management and one Data cport */
+	if (bundle->num_cports < 2)
+		return -ENODEV;
+
+	/*
+	 * There can be only one Management connection and any number of data
+	 * connections.
+	 */
+	gbmodule = devm_kzalloc(dev, sizeof(*gbmodule), GFP_KERNEL);
+	if (!gbmodule)
+		return -ENOMEM;
+
+	gbmodule->num_data_connections = bundle->num_cports - 1;
+	INIT_LIST_HEAD(&gbmodule->data_list);
+	INIT_LIST_HEAD(&gbmodule->widget_list);
+	INIT_LIST_HEAD(&gbmodule->ctl_list);
+	INIT_LIST_HEAD(&gbmodule->widget_ctl_list);
+	gbmodule->dev = dev;
+	snprintf(gbmodule->name, NAME_SIZE, "%s.%s", dev->driver->name,
+		 dev_name(dev));
+	greybus_set_drvdata(bundle, gbmodule);
+
+	/* Create all connections */
+	for (i = 0; i < bundle->num_cports; i++) {
+		cport_desc = &bundle->cport_desc[i];
+
+		switch (cport_desc->protocol_id) {
+		case GREYBUS_PROTOCOL_AUDIO_MGMT:
+			ret = gb_audio_add_mgmt_connection(gbmodule, cport_desc,
+							   bundle);
+			if (ret)
+				goto destroy_connections;
+			break;
+		case GREYBUS_PROTOCOL_AUDIO_DATA:
+			ret = gb_audio_add_data_connection(gbmodule, cport_desc,
+							   bundle);
+			if (ret)
+				goto destroy_connections;
+			break;
+		default:
+			dev_err(dev, "Unsupported protocol: 0x%02x\n",
+				cport_desc->protocol_id);
+			ret = -ENODEV;
+			goto destroy_connections;
+		}
+	}
+
+	/* There must be a management cport */
+	if (!gbmodule->mgmt_connection) {
+		ret = -EINVAL;
+		dev_err(dev, "Missing management connection\n");
+		goto destroy_connections;
+	}
+
+	/* Initialize management connection */
+	ret = gb_connection_enable(gbmodule->mgmt_connection);
+	if (ret) {
+		dev_err(dev, "%d: Error while enabling mgmt connection\n", ret);
+		goto destroy_connections;
+	}
+	gbmodule->dev_id = gbmodule->mgmt_connection->intf->interface_id;
+
+	/*
+	 * FIXME: malloc for topology happens via audio_gb driver
+	 * should be done within codec driver itself
+	 */
+	ret = gb_audio_gb_get_topology(gbmodule->mgmt_connection, &topology);
+	if (ret) {
+		dev_err(dev, "%d:Error while fetching topology\n", ret);
+		goto disable_connection;
+	}
+
+	/* process topology data */
+	ret = gbaudio_tplg_parse_data(gbmodule, topology);
+	if (ret) {
+		dev_err(dev, "%d:Error while parsing topology data\n",
+			  ret);
+		goto free_topology;
+	}
+	gbmodule->topology = topology;
+
+	/* Initialize data connections */
+	list_for_each_entry(dai, &gbmodule->data_list, list) {
+		ret = gb_connection_enable(dai->connection);
+		if (ret) {
+			dev_err(dev,
+				"%d:Error while enabling %d:data connection\n",
+				ret, dai->data_cport);
+			goto disable_data_connection;
+		}
+	}
+
+	/* register module with gbcodec */
+	ret = gbaudio_register_module(gbmodule);
+	if (ret)
+		goto disable_data_connection;
+
+	/* inform above layer for uevent */
+	dev_dbg(dev, "Inform set_event:%d to above layer\n", 1);
+	/* prepare for the audio manager */
+	strlcpy(desc.name, gbmodule->name, GB_AUDIO_MANAGER_MODULE_NAME_LEN);
+	desc.slot = 1; /* todo */
+	desc.vid = 2; /* todo */
+	desc.pid = 3; /* todo */
+	desc.cport = gbmodule->dev_id;
+	desc.op_devices = gbmodule->op_devices;
+	desc.ip_devices = gbmodule->ip_devices;
+	gbmodule->manager_id = gb_audio_manager_add(&desc);
+
+	dev_dbg(dev, "Add GB Audio device:%s\n", gbmodule->name);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return 0;
+
+disable_data_connection:
+	list_for_each_entry_safe(dai, _dai, &gbmodule->data_list, list)
+		gb_connection_disable(dai->connection);
+	gbaudio_tplg_release(gbmodule);
+	gbmodule->topology = NULL;
+
+free_topology:
+	kfree(topology);
+
+disable_connection:
+	gb_connection_disable(gbmodule->mgmt_connection);
+
+destroy_connections:
+	list_for_each_entry_safe(dai, _dai, &gbmodule->data_list, list) {
+		gb_connection_destroy(dai->connection);
+		list_del(&dai->list);
+		devm_kfree(dev, dai);
+	}
+
+	if (gbmodule->mgmt_connection)
+		gb_connection_destroy(gbmodule->mgmt_connection);
+
+	devm_kfree(dev, gbmodule);
+
+	return ret;
+}
+
+static void gb_audio_disconnect(struct gb_bundle *bundle)
+{
+	struct gbaudio_module_info *gbmodule = greybus_get_drvdata(bundle);
+	struct gbaudio_data_connection *dai, *_dai;
+
+	gb_pm_runtime_get_sync(bundle);
+
+	/* cleanup module related resources first */
+	gbaudio_unregister_module(gbmodule);
+
+	/* inform uevent to above layers */
+	gb_audio_manager_remove(gbmodule->manager_id);
+
+	gbaudio_tplg_release(gbmodule);
+	kfree(gbmodule->topology);
+	gbmodule->topology = NULL;
+	gb_connection_disable(gbmodule->mgmt_connection);
+	list_for_each_entry_safe(dai, _dai, &gbmodule->data_list, list) {
+		gb_connection_disable(dai->connection);
+		gb_connection_destroy(dai->connection);
+		list_del(&dai->list);
+		devm_kfree(gbmodule->dev, dai);
+	}
+	gb_connection_destroy(gbmodule->mgmt_connection);
+	gbmodule->mgmt_connection = NULL;
+
+	devm_kfree(&bundle->dev, gbmodule);
+}
+
+static const struct greybus_bundle_id gb_audio_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_AUDIO) },
+	{ }
+};
+MODULE_DEVICE_TABLE(greybus, gb_audio_id_table);
+
+#ifdef CONFIG_PM
+static int gb_audio_suspend(struct device *dev)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+	struct gbaudio_module_info *gbmodule = greybus_get_drvdata(bundle);
+	struct gbaudio_data_connection *dai;
+
+	list_for_each_entry(dai, &gbmodule->data_list, list)
+		gb_connection_disable(dai->connection);
+
+	gb_connection_disable(gbmodule->mgmt_connection);
+
+	return 0;
+}
+
+static int gb_audio_resume(struct device *dev)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+	struct gbaudio_module_info *gbmodule = greybus_get_drvdata(bundle);
+	struct gbaudio_data_connection *dai;
+	int ret;
+
+	ret = gb_connection_enable(gbmodule->mgmt_connection);
+	if (ret) {
+		dev_err(dev, "%d:Error while enabling mgmt connection\n", ret);
+		return ret;
+	}
+
+	list_for_each_entry(dai, &gbmodule->data_list, list) {
+		ret = gb_connection_enable(dai->connection);
+		if (ret) {
+			dev_err(dev,
+				"%d:Error while enabling %d:data connection\n",
+				ret, dai->data_cport);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops gb_audio_pm_ops = {
+	SET_RUNTIME_PM_OPS(gb_audio_suspend, gb_audio_resume, NULL)
+};
+
+static struct greybus_driver gb_audio_driver = {
+	.name		= "gb-audio",
+	.probe		= gb_audio_probe,
+	.disconnect	= gb_audio_disconnect,
+	.id_table	= gb_audio_id_table,
+	.driver.pm	= &gb_audio_pm_ops,
+};
+module_greybus_driver(gb_audio_driver);
+
+MODULE_DESCRIPTION("Greybus Audio module driver");
+MODULE_AUTHOR("Vaibhav Agarwal <vaibhav.agarwal@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:gbaudio-module");
diff --git a/drivers/staging/greybus/audio_topology.c b/drivers/staging/greybus/audio_topology.c
new file mode 100644
index 000000000000..b6251691a33d
--- /dev/null
+++ b/drivers/staging/greybus/audio_topology.c
@@ -0,0 +1,1443 @@
+/*
+ * Greybus audio driver
+ * Copyright 2015-2016 Google Inc.
+ * Copyright 2015-2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include "audio_codec.h"
+#include "greybus_protocols.h"
+
+#define GBAUDIO_INVALID_ID	0xFF
+
+/* mixer control */
+struct gb_mixer_control {
+	int min, max;
+	unsigned int reg, rreg, shift, rshift, invert;
+};
+
+struct gbaudio_ctl_pvt {
+	unsigned int ctl_id;
+	unsigned int data_cport;
+	unsigned int access;
+	unsigned int vcount;
+	struct gb_audio_ctl_elem_info *info;
+};
+
+static struct gbaudio_module_info *find_gb_module(
+					struct gbaudio_codec_info *codec,
+					char const *name)
+{
+	int dev_id, ret;
+	char begin[NAME_SIZE];
+	struct gbaudio_module_info *module;
+
+	if (!name)
+		return NULL;
+
+	ret = sscanf(name, "%s %d", begin, &dev_id);
+	dev_dbg(codec->dev, "%s:Find module#%d\n", __func__, dev_id);
+
+	mutex_lock(&codec->lock);
+	list_for_each_entry(module, &codec->module_list, list) {
+		if (module->dev_id == dev_id) {
+			mutex_unlock(&codec->lock);
+			return module;
+		}
+	}
+	mutex_unlock(&codec->lock);
+	dev_warn(codec->dev, "%s: module#%d missing in codec list\n", name,
+		 dev_id);
+	return NULL;
+}
+
+static const char *gbaudio_map_controlid(struct gbaudio_module_info *module,
+					 __u8 control_id, __u8 index)
+{
+	struct gbaudio_control *control;
+
+	if (control_id == GBAUDIO_INVALID_ID)
+		return NULL;
+
+	list_for_each_entry(control, &module->ctl_list, list) {
+		if (control->id == control_id) {
+			if (index == GBAUDIO_INVALID_ID)
+				return control->name;
+			if (index >= control->items)
+				return NULL;
+			return control->texts[index];
+		}
+	}
+	list_for_each_entry(control, &module->widget_ctl_list, list) {
+		if (control->id == control_id) {
+			if (index == GBAUDIO_INVALID_ID)
+				return control->name;
+			if (index >= control->items)
+				return NULL;
+			return control->texts[index];
+		}
+	}
+	return NULL;
+}
+
+static int gbaudio_map_controlname(struct gbaudio_module_info *module,
+				   const char *name)
+{
+	struct gbaudio_control *control;
+
+	list_for_each_entry(control, &module->ctl_list, list) {
+		if (!strncmp(control->name, name, NAME_SIZE))
+			return control->id;
+	}
+
+	dev_warn(module->dev, "%s: missing in modules controls list\n", name);
+
+	return -EINVAL;
+}
+
+static int gbaudio_map_wcontrolname(struct gbaudio_module_info *module,
+				    const char *name)
+{
+	struct gbaudio_control *control;
+
+	list_for_each_entry(control, &module->widget_ctl_list, list) {
+		if (!strncmp(control->wname, name, NAME_SIZE))
+			return control->id;
+	}
+	dev_warn(module->dev, "%s: missing in modules controls list\n", name);
+
+	return -EINVAL;
+}
+
+static int gbaudio_map_widgetname(struct gbaudio_module_info *module,
+				  const char *name)
+{
+	struct gbaudio_widget *widget;
+	list_for_each_entry(widget, &module->widget_list, list) {
+		if (!strncmp(widget->name, name, NAME_SIZE))
+			return widget->id;
+	}
+	dev_warn(module->dev, "%s: missing in modules widgets list\n", name);
+
+	return -EINVAL;
+}
+
+static const char *gbaudio_map_widgetid(struct gbaudio_module_info *module,
+					__u8 widget_id)
+{
+	struct gbaudio_widget *widget;
+
+	list_for_each_entry(widget, &module->widget_list, list) {
+		if (widget->id == widget_id)
+			return widget->name;
+	}
+	return NULL;
+}
+
+static const char **gb_generate_enum_strings(struct gbaudio_module_info *gb,
+					     struct gb_audio_enumerated *gbenum)
+{
+	const char **strings;
+	int i;
+	__u8 *data;
+
+	strings = devm_kzalloc(gb->dev, sizeof(char *) * gbenum->items,
+			       GFP_KERNEL);
+	data = gbenum->names;
+
+	for (i = 0; i < gbenum->items; i++) {
+		strings[i] = (const char *)data;
+		while (*data != '\0')
+			data++;
+		data++;
+	}
+
+	return strings;
+}
+
+static int gbcodec_mixer_ctl_info(struct snd_kcontrol *kcontrol,
+		     struct snd_ctl_elem_info *uinfo)
+{
+	unsigned int max;
+	const char *name;
+	struct gbaudio_ctl_pvt *data;
+	struct gb_audio_ctl_elem_info *info;
+	struct gbaudio_module_info *module;
+	struct snd_soc_codec *codec = snd_kcontrol_chip(kcontrol);
+	struct gbaudio_codec_info *gbcodec = snd_soc_codec_get_drvdata(codec);
+
+	dev_dbg(codec->dev, "Entered %s:%s\n", __func__, kcontrol->id.name);
+	data = (struct gbaudio_ctl_pvt *)kcontrol->private_value;
+	info = (struct gb_audio_ctl_elem_info *)data->info;
+
+	if (!info) {
+		dev_err(codec->dev, "NULL info for %s\n", uinfo->id.name);
+		return -EINVAL;
+	}
+
+	/* update uinfo */
+	uinfo->access = data->access;
+	uinfo->count = data->vcount;
+	uinfo->type = (snd_ctl_elem_type_t)info->type;
+
+	switch (info->type) {
+	case GB_AUDIO_CTL_ELEM_TYPE_BOOLEAN:
+	case GB_AUDIO_CTL_ELEM_TYPE_INTEGER:
+		uinfo->value.integer.min = info->value.integer.min;
+		uinfo->value.integer.max = info->value.integer.max;
+		break;
+	case GB_AUDIO_CTL_ELEM_TYPE_ENUMERATED:
+		max = info->value.enumerated.items;
+		uinfo->value.enumerated.items = max;
+		if (uinfo->value.enumerated.item > max - 1)
+			uinfo->value.enumerated.item = max - 1;
+		module = find_gb_module(gbcodec, kcontrol->id.name);
+		if (!module)
+			return -EINVAL;
+		name = gbaudio_map_controlid(module, data->ctl_id,
+					     uinfo->value.enumerated.item);
+		strlcpy(uinfo->value.enumerated.name, name, NAME_SIZE);
+		break;
+	default:
+		dev_err(codec->dev, "Invalid type: %d for %s:kcontrol\n",
+			info->type, kcontrol->id.name);
+		break;
+	}
+	return 0;
+}
+
+static int gbcodec_mixer_ctl_get(struct snd_kcontrol *kcontrol,
+	struct snd_ctl_elem_value *ucontrol)
+{
+	int ret;
+	struct gb_audio_ctl_elem_info *info;
+	struct gbaudio_ctl_pvt *data;
+	struct gb_audio_ctl_elem_value gbvalue;
+	struct gbaudio_module_info *module;
+	struct snd_soc_codec *codec = snd_kcontrol_chip(kcontrol);
+	struct gbaudio_codec_info *gb = snd_soc_codec_get_drvdata(codec);
+	struct gb_bundle *bundle;
+
+	dev_dbg(codec->dev, "Entered %s:%s\n", __func__, kcontrol->id.name);
+	module = find_gb_module(gb, kcontrol->id.name);
+	if (!module)
+		return -EINVAL;
+
+	data = (struct gbaudio_ctl_pvt *)kcontrol->private_value;
+	info = (struct gb_audio_ctl_elem_info *)data->info;
+	bundle = to_gb_bundle(module->dev);
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	ret = gb_audio_gb_get_control(module->mgmt_connection, data->ctl_id,
+				      GB_AUDIO_INVALID_INDEX, &gbvalue);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	if (ret) {
+		dev_err_ratelimited(codec->dev, "%d:Error in %s for %s\n", ret,
+				    __func__, kcontrol->id.name);
+		return ret;
+	}
+
+	/* update ucontrol */
+	switch (info->type) {
+	case GB_AUDIO_CTL_ELEM_TYPE_BOOLEAN:
+	case GB_AUDIO_CTL_ELEM_TYPE_INTEGER:
+		ucontrol->value.integer.value[0] =
+			gbvalue.value.integer_value[0];
+		if (data->vcount == 2)
+			ucontrol->value.integer.value[1] =
+				gbvalue.value.integer_value[1];
+		break;
+	case GB_AUDIO_CTL_ELEM_TYPE_ENUMERATED:
+		ucontrol->value.enumerated.item[0] =
+			gbvalue.value.enumerated_item[0];
+		if (data->vcount == 2)
+			ucontrol->value.enumerated.item[1] =
+				gbvalue.value.enumerated_item[1];
+		break;
+	default:
+		dev_err(codec->dev, "Invalid type: %d for %s:kcontrol\n",
+			info->type, kcontrol->id.name);
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static int gbcodec_mixer_ctl_put(struct snd_kcontrol *kcontrol,
+			      struct snd_ctl_elem_value *ucontrol)
+{
+	int ret = 0;
+	struct gb_audio_ctl_elem_info *info;
+	struct gbaudio_ctl_pvt *data;
+	struct gb_audio_ctl_elem_value gbvalue;
+	struct gbaudio_module_info *module;
+	struct snd_soc_codec *codec = snd_kcontrol_chip(kcontrol);
+	struct gbaudio_codec_info *gb = snd_soc_codec_get_drvdata(codec);
+	struct gb_bundle *bundle;
+
+	dev_dbg(codec->dev, "Entered %s:%s\n", __func__, kcontrol->id.name);
+	module = find_gb_module(gb, kcontrol->id.name);
+	if (!module)
+		return -EINVAL;
+
+	data = (struct gbaudio_ctl_pvt *)kcontrol->private_value;
+	info = (struct gb_audio_ctl_elem_info *)data->info;
+	bundle = to_gb_bundle(module->dev);
+
+	/* update ucontrol */
+	switch (info->type) {
+	case GB_AUDIO_CTL_ELEM_TYPE_BOOLEAN:
+	case GB_AUDIO_CTL_ELEM_TYPE_INTEGER:
+		gbvalue.value.integer_value[0] =
+			ucontrol->value.integer.value[0];
+		if (data->vcount == 2)
+			gbvalue.value.integer_value[1] =
+				ucontrol->value.integer.value[1];
+		break;
+	case GB_AUDIO_CTL_ELEM_TYPE_ENUMERATED:
+		gbvalue.value.enumerated_item[0] =
+			ucontrol->value.enumerated.item[0];
+		if (data->vcount == 2)
+			gbvalue.value.enumerated_item[1] =
+				ucontrol->value.enumerated.item[1];
+		break;
+	default:
+		dev_err(codec->dev, "Invalid type: %d for %s:kcontrol\n",
+			info->type, kcontrol->id.name);
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret)
+		return ret;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	ret = gb_audio_gb_set_control(module->mgmt_connection, data->ctl_id,
+				      GB_AUDIO_INVALID_INDEX, &gbvalue);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	if (ret) {
+		dev_err_ratelimited(codec->dev, "%d:Error in %s for %s\n", ret,
+				    __func__, kcontrol->id.name);
+	}
+
+	return ret;
+}
+
+#define SOC_MIXER_GB(xname, kcount, data) \
+{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
+	.count = kcount, .info = gbcodec_mixer_ctl_info, \
+	.get = gbcodec_mixer_ctl_get, .put = gbcodec_mixer_ctl_put, \
+	.private_value = (unsigned long)data }
+
+/*
+ * although below callback functions seems redundant to above functions.
+ * same are kept to allow provision for different handling in case
+ * of DAPM related sequencing, etc.
+ */
+static int gbcodec_mixer_dapm_ctl_info(struct snd_kcontrol *kcontrol,
+		     struct snd_ctl_elem_info *uinfo)
+{
+	int platform_max, platform_min;
+	struct gbaudio_ctl_pvt *data;
+	struct gb_audio_ctl_elem_info *info;
+	struct snd_soc_dapm_widget_list *wlist = snd_kcontrol_chip(kcontrol);
+	struct snd_soc_dapm_widget *widget = wlist->widgets[0];
+	struct snd_soc_codec *codec = widget->codec;
+
+	dev_dbg(codec->dev, "Entered %s:%s\n", __func__, kcontrol->id.name);
+	data = (struct gbaudio_ctl_pvt *)kcontrol->private_value;
+	info = (struct gb_audio_ctl_elem_info *)data->info;
+
+	/* update uinfo */
+	platform_max = info->value.integer.max;
+	platform_min = info->value.integer.min;
+
+	if (platform_max == 1 &&
+	    !strnstr(kcontrol->id.name, " Volume", NAME_SIZE))
+		uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN;
+	else
+		uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+
+	uinfo->count = data->vcount;
+	uinfo->value.integer.min = 0;
+	if (info->value.integer.min < 0 &&
+	    (uinfo->type == SNDRV_CTL_ELEM_TYPE_INTEGER))
+		uinfo->value.integer.max = platform_max - platform_min;
+	else
+		uinfo->value.integer.max = platform_max;
+
+	return 0;
+}
+
+static int gbcodec_mixer_dapm_ctl_get(struct snd_kcontrol *kcontrol,
+	struct snd_ctl_elem_value *ucontrol)
+{
+	int ret;
+	struct gb_audio_ctl_elem_info *info;
+	struct gbaudio_ctl_pvt *data;
+	struct gb_audio_ctl_elem_value gbvalue;
+	struct gbaudio_module_info *module;
+	struct snd_soc_dapm_widget_list *wlist = snd_kcontrol_chip(kcontrol);
+	struct snd_soc_dapm_widget *widget = wlist->widgets[0];
+	struct snd_soc_codec *codec = widget->codec;
+	struct gbaudio_codec_info *gb = snd_soc_codec_get_drvdata(codec);
+	struct gb_bundle *bundle;
+
+	dev_dbg(codec->dev, "Entered %s:%s\n", __func__, kcontrol->id.name);
+	module = find_gb_module(gb, kcontrol->id.name);
+	if (!module)
+		return -EINVAL;
+
+	data = (struct gbaudio_ctl_pvt *)kcontrol->private_value;
+	info = (struct gb_audio_ctl_elem_info *)data->info;
+	bundle = to_gb_bundle(module->dev);
+
+	if (data->vcount == 2)
+		dev_warn(widget->dapm->dev,
+			 "GB: Control '%s' is stereo, which is not supported\n",
+			 kcontrol->id.name);
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	ret = gb_audio_gb_get_control(module->mgmt_connection, data->ctl_id,
+				      GB_AUDIO_INVALID_INDEX, &gbvalue);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	if (ret) {
+		dev_err_ratelimited(codec->dev, "%d:Error in %s for %s\n", ret,
+				    __func__, kcontrol->id.name);
+		return ret;
+	}
+	/* update ucontrol */
+	ucontrol->value.integer.value[0] = gbvalue.value.integer_value[0];
+
+	return ret;
+}
+
+static int gbcodec_mixer_dapm_ctl_put(struct snd_kcontrol *kcontrol,
+			      struct snd_ctl_elem_value *ucontrol)
+{
+	int ret, wi, max, connect;
+	unsigned int mask, val;
+	struct gb_audio_ctl_elem_info *info;
+	struct gbaudio_ctl_pvt *data;
+	struct gb_audio_ctl_elem_value gbvalue;
+	struct gbaudio_module_info *module;
+	struct snd_soc_dapm_widget_list *wlist = snd_kcontrol_chip(kcontrol);
+	struct snd_soc_dapm_widget *widget = wlist->widgets[0];
+	struct snd_soc_codec *codec = widget->codec;
+	struct gbaudio_codec_info *gb = snd_soc_codec_get_drvdata(codec);
+	struct gb_bundle *bundle;
+
+	dev_dbg(codec->dev, "Entered %s:%s\n", __func__, kcontrol->id.name);
+	module = find_gb_module(gb, kcontrol->id.name);
+	if (!module)
+		return -EINVAL;
+
+	data = (struct gbaudio_ctl_pvt *)kcontrol->private_value;
+	info = (struct gb_audio_ctl_elem_info *)data->info;
+	bundle = to_gb_bundle(module->dev);
+
+	if (data->vcount == 2)
+		dev_warn(widget->dapm->dev,
+			 "GB: Control '%s' is stereo, which is not supported\n",
+			 kcontrol->id.name);
+
+	max = info->value.integer.max;
+	mask = (1 << fls(max)) - 1;
+	val = ucontrol->value.integer.value[0] & mask;
+	connect = !!val;
+
+	/* update ucontrol */
+	if (gbvalue.value.integer_value[0] != val) {
+		for (wi = 0; wi < wlist->num_widgets; wi++) {
+			widget = wlist->widgets[wi];
+
+			widget->value = val;
+			widget->dapm->update = NULL;
+			snd_soc_dapm_mixer_update_power(widget, kcontrol,
+							connect);
+		}
+		gbvalue.value.integer_value[0] =
+			ucontrol->value.integer.value[0];
+
+		ret = gb_pm_runtime_get_sync(bundle);
+		if (ret)
+			return ret;
+
+		ret = gb_audio_gb_set_control(module->mgmt_connection,
+					      data->ctl_id,
+					      GB_AUDIO_INVALID_INDEX, &gbvalue);
+
+		gb_pm_runtime_put_autosuspend(bundle);
+
+		if (ret) {
+			dev_err_ratelimited(codec->dev,
+					    "%d:Error in %s for %s\n", ret,
+					    __func__, kcontrol->id.name);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+#define SOC_DAPM_MIXER_GB(xname, kcount, data) \
+{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
+	.count = kcount, .info = gbcodec_mixer_dapm_ctl_info, \
+	.get = gbcodec_mixer_dapm_ctl_get, .put = gbcodec_mixer_dapm_ctl_put, \
+	.private_value = (unsigned long)data}
+
+static int gbcodec_event_spk(struct snd_soc_dapm_widget *w,
+					struct snd_kcontrol *k, int event)
+{
+	/* Ensure GB speaker is connected */
+
+	return 0;
+}
+
+static int gbcodec_event_hp(struct snd_soc_dapm_widget *w,
+					struct snd_kcontrol *k, int event)
+{
+	/* Ensure GB module supports jack slot */
+
+	return 0;
+}
+
+static int gbcodec_event_int_mic(struct snd_soc_dapm_widget *w,
+					struct snd_kcontrol *k, int event)
+{
+	/* Ensure GB module supports jack slot */
+
+	return 0;
+}
+
+static int gbaudio_validate_kcontrol_count(struct gb_audio_widget *w)
+{
+	int ret = 0;
+
+	switch (w->type) {
+	case snd_soc_dapm_spk:
+	case snd_soc_dapm_hp:
+	case snd_soc_dapm_mic:
+	case snd_soc_dapm_output:
+	case snd_soc_dapm_input:
+		if (w->ncontrols)
+			ret = -EINVAL;
+		break;
+	case snd_soc_dapm_switch:
+	case snd_soc_dapm_mux:
+		if (w->ncontrols != 1)
+			ret = -EINVAL;
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static int gbcodec_enum_ctl_get(struct snd_kcontrol *kcontrol,
+				struct snd_ctl_elem_value *ucontrol)
+{
+	int ret, ctl_id;
+	struct snd_soc_codec *codec = snd_kcontrol_chip(kcontrol);
+	struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+	struct gb_audio_ctl_elem_value gbvalue;
+	struct gbaudio_module_info *module;
+	struct gbaudio_codec_info *gb = snd_soc_codec_get_drvdata(codec);
+	struct gb_bundle *bundle;
+
+	module = find_gb_module(gb, kcontrol->id.name);
+	if (!module)
+		return -EINVAL;
+
+	ctl_id = gbaudio_map_controlname(module, kcontrol->id.name);
+	if (ctl_id < 0)
+		return -EINVAL;
+
+	bundle = to_gb_bundle(module->dev);
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	ret = gb_audio_gb_get_control(module->mgmt_connection, ctl_id,
+				      GB_AUDIO_INVALID_INDEX, &gbvalue);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	if (ret) {
+		dev_err_ratelimited(codec->dev, "%d:Error in %s for %s\n", ret,
+				    __func__, kcontrol->id.name);
+		return ret;
+	}
+
+	ucontrol->value.enumerated.item[0] = gbvalue.value.enumerated_item[0];
+	if (e->shift_l != e->shift_r)
+		ucontrol->value.enumerated.item[1] =
+			gbvalue.value.enumerated_item[1];
+
+	return 0;
+}
+
+static int gbcodec_enum_ctl_put(struct snd_kcontrol *kcontrol,
+				struct snd_ctl_elem_value *ucontrol)
+{
+	int ret, ctl_id;
+	struct snd_soc_codec *codec = snd_kcontrol_chip(kcontrol);
+	struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+	struct gb_audio_ctl_elem_value gbvalue;
+	struct gbaudio_module_info *module;
+	struct gbaudio_codec_info *gb = snd_soc_codec_get_drvdata(codec);
+	struct gb_bundle *bundle;
+
+	module = find_gb_module(gb, kcontrol->id.name);
+	if (!module)
+		return -EINVAL;
+
+	ctl_id = gbaudio_map_controlname(module, kcontrol->id.name);
+	if (ctl_id < 0)
+		return -EINVAL;
+
+	if (ucontrol->value.enumerated.item[0] > e->max - 1)
+		return -EINVAL;
+	gbvalue.value.enumerated_item[0] = ucontrol->value.enumerated.item[0];
+
+	if (e->shift_l != e->shift_r) {
+		if (ucontrol->value.enumerated.item[1] > e->max - 1)
+			return -EINVAL;
+		gbvalue.value.enumerated_item[1] =
+			ucontrol->value.enumerated.item[1];
+	}
+
+	bundle = to_gb_bundle(module->dev);
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	ret = gb_audio_gb_set_control(module->mgmt_connection, ctl_id,
+				      GB_AUDIO_INVALID_INDEX, &gbvalue);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	if (ret) {
+		dev_err_ratelimited(codec->dev, "%d:Error in %s for %s\n", ret,
+				    __func__, kcontrol->id.name);
+	}
+
+	return ret;
+}
+
+static int gbaudio_tplg_create_enum_kctl(struct gbaudio_module_info *gb,
+					 struct snd_kcontrol_new *kctl,
+					 struct gb_audio_control *ctl)
+{
+	struct soc_enum *gbe;
+	struct gb_audio_enumerated *gb_enum;
+	int i;
+
+	gbe = devm_kzalloc(gb->dev, sizeof(*gbe), GFP_KERNEL);
+	if (!gbe)
+		return -ENOMEM;
+
+	gb_enum = &ctl->info.value.enumerated;
+
+	/* since count=1, and reg is dummy */
+	gbe->max = gb_enum->items;
+	gbe->texts = gb_generate_enum_strings(gb, gb_enum);
+
+	/* debug enum info */
+	dev_dbg(gb->dev, "Max:%d, name_length:%d\n", gb_enum->items,
+		 gb_enum->names_length);
+	for (i = 0; i < gb_enum->items; i++)
+		dev_dbg(gb->dev, "src[%d]: %s\n", i, gbe->texts[i]);
+
+	*kctl = (struct snd_kcontrol_new)
+		SOC_ENUM_EXT(ctl->name, *gbe, gbcodec_enum_ctl_get,
+			     gbcodec_enum_ctl_put);
+	return 0;
+}
+
+static int gbaudio_tplg_create_kcontrol(struct gbaudio_module_info *gb,
+					struct snd_kcontrol_new *kctl,
+					struct gb_audio_control *ctl)
+{
+	int ret = 0;
+	struct gbaudio_ctl_pvt *ctldata;
+
+	switch (ctl->iface) {
+	case SNDRV_CTL_ELEM_IFACE_MIXER:
+		switch (ctl->info.type) {
+		case GB_AUDIO_CTL_ELEM_TYPE_ENUMERATED:
+			ret = gbaudio_tplg_create_enum_kctl(gb, kctl, ctl);
+			break;
+		default:
+			ctldata = devm_kzalloc(gb->dev,
+					       sizeof(struct gbaudio_ctl_pvt),
+					       GFP_KERNEL);
+			if (!ctldata)
+				return -ENOMEM;
+			ctldata->ctl_id = ctl->id;
+			ctldata->data_cport = ctl->data_cport;
+			ctldata->access = ctl->access;
+			ctldata->vcount = ctl->count_values;
+			ctldata->info = &ctl->info;
+			*kctl = (struct snd_kcontrol_new)
+				SOC_MIXER_GB(ctl->name, ctl->count, ctldata);
+			ctldata = NULL;
+			break;
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	dev_dbg(gb->dev, "%s:%d control created\n", ctl->name, ctl->id);
+	return ret;
+}
+
+static int gbcodec_enum_dapm_ctl_get(struct snd_kcontrol *kcontrol,
+				     struct snd_ctl_elem_value *ucontrol)
+{
+	int ret, ctl_id;
+	struct snd_soc_dapm_widget_list *wlist = snd_kcontrol_chip(kcontrol);
+	struct snd_soc_dapm_widget *widget = wlist->widgets[0];
+	struct gbaudio_module_info *module;
+	struct gb_audio_ctl_elem_value gbvalue;
+	struct snd_soc_codec *codec = widget->codec;
+	struct gbaudio_codec_info *gb = snd_soc_codec_get_drvdata(codec);
+	struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+	struct gb_bundle *bundle;
+
+	module = find_gb_module(gb, kcontrol->id.name);
+	if (!module)
+		return -EINVAL;
+
+	ctl_id = gbaudio_map_wcontrolname(module, kcontrol->id.name);
+	if (ctl_id < 0)
+		return -EINVAL;
+
+	bundle = to_gb_bundle(module->dev);
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	ret = gb_audio_gb_get_control(module->mgmt_connection, ctl_id,
+				      GB_AUDIO_INVALID_INDEX, &gbvalue);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	if (ret) {
+		dev_err_ratelimited(codec->dev, "%d:Error in %s for %s\n", ret,
+				    __func__, kcontrol->id.name);
+		return ret;
+	}
+
+	ucontrol->value.enumerated.item[0] = gbvalue.value.enumerated_item[0];
+	if (e->shift_l != e->shift_r)
+		ucontrol->value.enumerated.item[1] =
+			gbvalue.value.enumerated_item[1];
+
+	return 0;
+}
+
+static int gbcodec_enum_dapm_ctl_put(struct snd_kcontrol *kcontrol,
+				     struct snd_ctl_elem_value *ucontrol)
+{
+	int ret, wi, ctl_id;
+	unsigned int val, mux, change;
+	unsigned int mask;
+	struct snd_soc_dapm_widget_list *wlist = snd_kcontrol_chip(kcontrol);
+	struct snd_soc_dapm_widget *widget = wlist->widgets[0];
+	struct gb_audio_ctl_elem_value gbvalue;
+	struct gbaudio_module_info *module;
+	struct snd_soc_codec *codec = widget->codec;
+	struct gbaudio_codec_info *gb = snd_soc_codec_get_drvdata(codec);
+	struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+	struct gb_bundle *bundle;
+
+	if (ucontrol->value.enumerated.item[0] > e->max - 1)
+		return -EINVAL;
+
+	module = find_gb_module(gb, kcontrol->id.name);
+	if (!module)
+		return -EINVAL;
+
+	ctl_id = gbaudio_map_wcontrolname(module, kcontrol->id.name);
+	if (ctl_id < 0)
+		return -EINVAL;
+
+	change = 0;
+	bundle = to_gb_bundle(module->dev);
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	ret = gb_audio_gb_get_control(module->mgmt_connection, ctl_id,
+				      GB_AUDIO_INVALID_INDEX, &gbvalue);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	if (ret) {
+		dev_err_ratelimited(codec->dev, "%d:Error in %s for %s\n", ret,
+				    __func__, kcontrol->id.name);
+		return ret;
+	}
+
+	mux = ucontrol->value.enumerated.item[0];
+	val = mux << e->shift_l;
+	mask = e->mask << e->shift_l;
+
+	if (gbvalue.value.enumerated_item[0] !=
+	    ucontrol->value.enumerated.item[0]) {
+		change = 1;
+		gbvalue.value.enumerated_item[0] =
+			ucontrol->value.enumerated.item[0];
+	}
+
+	if (e->shift_l != e->shift_r) {
+		if (ucontrol->value.enumerated.item[1] > e->max - 1)
+			return -EINVAL;
+		val |= ucontrol->value.enumerated.item[1] << e->shift_r;
+		mask |= e->mask << e->shift_r;
+		if (gbvalue.value.enumerated_item[1] !=
+		    ucontrol->value.enumerated.item[1]) {
+			change = 1;
+			gbvalue.value.enumerated_item[1] =
+				ucontrol->value.enumerated.item[1];
+		}
+	}
+
+	if (change) {
+		ret = gb_pm_runtime_get_sync(bundle);
+		if (ret)
+			return ret;
+
+		ret = gb_audio_gb_set_control(module->mgmt_connection, ctl_id,
+					      GB_AUDIO_INVALID_INDEX, &gbvalue);
+
+		gb_pm_runtime_put_autosuspend(bundle);
+
+		if (ret) {
+			dev_err_ratelimited(codec->dev,
+					    "%d:Error in %s for %s\n", ret,
+					    __func__, kcontrol->id.name);
+		}
+		for (wi = 0; wi < wlist->num_widgets; wi++) {
+			widget = wlist->widgets[wi];
+
+			widget->value = val;
+			widget->dapm->update = NULL;
+			snd_soc_dapm_mux_update_power(widget, kcontrol, mux, e);
+		}
+	}
+
+	return change;
+}
+
+static int gbaudio_tplg_create_enum_ctl(struct gbaudio_module_info *gb,
+					struct snd_kcontrol_new *kctl,
+					struct gb_audio_control *ctl)
+{
+	struct soc_enum *gbe;
+	struct gb_audio_enumerated *gb_enum;
+	int i;
+
+	gbe = devm_kzalloc(gb->dev, sizeof(*gbe), GFP_KERNEL);
+	if (!gbe)
+		return -ENOMEM;
+
+	gb_enum = &ctl->info.value.enumerated;
+
+	/* since count=1, and reg is dummy */
+	gbe->max = gb_enum->items;
+	gbe->texts = gb_generate_enum_strings(gb, gb_enum);
+
+	/* debug enum info */
+	dev_dbg(gb->dev, "Max:%d, name_length:%d\n", gb_enum->items,
+		 gb_enum->names_length);
+	for (i = 0; i < gb_enum->items; i++)
+		dev_dbg(gb->dev, "src[%d]: %s\n", i, gbe->texts[i]);
+
+	*kctl = (struct snd_kcontrol_new)
+		SOC_DAPM_ENUM_EXT(ctl->name, *gbe, gbcodec_enum_dapm_ctl_get,
+				  gbcodec_enum_dapm_ctl_put);
+	return 0;
+}
+
+static int gbaudio_tplg_create_mixer_ctl(struct gbaudio_module_info *gb,
+					     struct snd_kcontrol_new *kctl,
+					     struct gb_audio_control *ctl)
+{
+	struct gbaudio_ctl_pvt *ctldata;
+
+	ctldata = devm_kzalloc(gb->dev, sizeof(struct gbaudio_ctl_pvt),
+			       GFP_KERNEL);
+	if (!ctldata)
+		return -ENOMEM;
+	ctldata->ctl_id = ctl->id;
+	ctldata->data_cport = ctl->data_cport;
+	ctldata->access = ctl->access;
+	ctldata->vcount = ctl->count_values;
+	ctldata->info = &ctl->info;
+	*kctl = (struct snd_kcontrol_new)
+		SOC_DAPM_MIXER_GB(ctl->name, ctl->count, ctldata);
+
+	return 0;
+}
+
+static int gbaudio_tplg_create_wcontrol(struct gbaudio_module_info *gb,
+					     struct snd_kcontrol_new *kctl,
+					     struct gb_audio_control *ctl)
+{
+	int ret;
+
+	switch (ctl->iface) {
+	case SNDRV_CTL_ELEM_IFACE_MIXER:
+		switch (ctl->info.type) {
+		case GB_AUDIO_CTL_ELEM_TYPE_ENUMERATED:
+			ret = gbaudio_tplg_create_enum_ctl(gb, kctl, ctl);
+			break;
+		default:
+			ret = gbaudio_tplg_create_mixer_ctl(gb, kctl, ctl);
+			break;
+		}
+		break;
+	default:
+		return -EINVAL;
+
+	}
+
+	dev_dbg(gb->dev, "%s:%d DAPM control created, ret:%d\n", ctl->name,
+		ctl->id, ret);
+	return ret;
+}
+
+static int gbaudio_widget_event(struct snd_soc_dapm_widget *w,
+				struct snd_kcontrol *kcontrol, int event)
+{
+	int wid;
+	int ret;
+	struct snd_soc_codec *codec = w->codec;
+	struct gbaudio_codec_info *gbcodec = snd_soc_codec_get_drvdata(codec);
+	struct gbaudio_module_info *module;
+	struct gb_bundle *bundle;
+
+	dev_dbg(codec->dev, "%s %s %d\n", __func__, w->name, event);
+
+	/* Find relevant module */
+	module = find_gb_module(gbcodec, w->name);
+	if (!module)
+		return -EINVAL;
+
+	/* map name to widget id */
+	wid = gbaudio_map_widgetname(module, w->name);
+	if (wid < 0) {
+		dev_err(codec->dev, "Invalid widget name:%s\n", w->name);
+		return -EINVAL;
+	}
+
+	bundle = to_gb_bundle(module->dev);
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		ret = gb_audio_gb_enable_widget(module->mgmt_connection, wid);
+		if (!ret)
+			ret = gbaudio_module_update(gbcodec, w, module, 1);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		ret = gb_audio_gb_disable_widget(module->mgmt_connection, wid);
+		if (!ret)
+			ret = gbaudio_module_update(gbcodec, w, module, 0);
+		break;
+	}
+	if (ret)
+		dev_err_ratelimited(codec->dev,
+				    "%d: widget, event:%d failed:%d\n", wid,
+				    event, ret);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return ret;
+}
+
+static int gbaudio_tplg_create_widget(struct gbaudio_module_info *module,
+				      struct snd_soc_dapm_widget *dw,
+				      struct gb_audio_widget *w, int *w_size)
+{
+	int i, ret, csize;
+	struct snd_kcontrol_new *widget_kctls;
+	struct gb_audio_control *curr;
+	struct gbaudio_control *control, *_control;
+	size_t size;
+	char temp_name[NAME_SIZE];
+
+	ret = gbaudio_validate_kcontrol_count(w);
+	if (ret) {
+		dev_err(module->dev, "Inavlid kcontrol count=%d for %s\n",
+			w->ncontrols, w->name);
+		return ret;
+	}
+
+	/* allocate memory for kcontrol */
+	if (w->ncontrols) {
+		size = sizeof(struct snd_kcontrol_new) * w->ncontrols;
+		widget_kctls = devm_kzalloc(module->dev, size, GFP_KERNEL);
+		if (!widget_kctls)
+			return -ENOMEM;
+	}
+
+	*w_size = sizeof(struct gb_audio_widget);
+
+	/* create relevant kcontrols */
+	curr = w->ctl;
+	for (i = 0; i < w->ncontrols; i++) {
+		ret = gbaudio_tplg_create_wcontrol(module, &widget_kctls[i],
+						   curr);
+		if (ret) {
+			dev_err(module->dev,
+				"%s:%d type widget_ctl not supported\n",
+				curr->name, curr->iface);
+			goto error;
+		}
+		control = devm_kzalloc(module->dev,
+				       sizeof(struct gbaudio_control),
+				       GFP_KERNEL);
+		if (!control) {
+			ret = -ENOMEM;
+			goto error;
+		}
+		control->id = curr->id;
+		control->name = curr->name;
+		control->wname = w->name;
+
+		if (curr->info.type == GB_AUDIO_CTL_ELEM_TYPE_ENUMERATED) {
+			struct gb_audio_enumerated *gbenum =
+				&curr->info.value.enumerated;
+
+			csize = offsetof(struct gb_audio_control, info);
+			csize += offsetof(struct gb_audio_ctl_elem_info, value);
+			csize += offsetof(struct gb_audio_enumerated, names);
+			csize += gbenum->names_length;
+			control->texts = (const char * const *)
+				gb_generate_enum_strings(module, gbenum);
+			control->items = gbenum->items;
+		} else
+			csize = sizeof(struct gb_audio_control);
+		*w_size += csize;
+		curr = (void *)curr + csize;
+		list_add(&control->list, &module->widget_ctl_list);
+		dev_dbg(module->dev, "%s: control of type %d created\n",
+			widget_kctls[i].name, widget_kctls[i].iface);
+	}
+
+	/* Prefix dev_id to widget control_name */
+	strlcpy(temp_name, w->name, NAME_SIZE);
+	snprintf(w->name, NAME_SIZE, "GB %d %s", module->dev_id, temp_name);
+
+	switch (w->type) {
+	case snd_soc_dapm_spk:
+		*dw = (struct snd_soc_dapm_widget)
+			SND_SOC_DAPM_SPK(w->name, gbcodec_event_spk);
+		module->op_devices |= GBAUDIO_DEVICE_OUT_SPEAKER;
+		break;
+	case snd_soc_dapm_hp:
+		*dw = (struct snd_soc_dapm_widget)
+			SND_SOC_DAPM_HP(w->name, gbcodec_event_hp);
+		module->op_devices |= (GBAUDIO_DEVICE_OUT_WIRED_HEADSET
+					| GBAUDIO_DEVICE_OUT_WIRED_HEADPHONE);
+		module->ip_devices |= GBAUDIO_DEVICE_IN_WIRED_HEADSET;
+		break;
+	case snd_soc_dapm_mic:
+		*dw = (struct snd_soc_dapm_widget)
+			SND_SOC_DAPM_MIC(w->name, gbcodec_event_int_mic);
+		module->ip_devices |= GBAUDIO_DEVICE_IN_BUILTIN_MIC;
+		break;
+	case snd_soc_dapm_output:
+		*dw = (struct snd_soc_dapm_widget)SND_SOC_DAPM_OUTPUT(w->name);
+		break;
+	case snd_soc_dapm_input:
+		*dw = (struct snd_soc_dapm_widget)SND_SOC_DAPM_INPUT(w->name);
+		break;
+	case snd_soc_dapm_switch:
+		*dw = (struct snd_soc_dapm_widget)
+			SND_SOC_DAPM_SWITCH_E(w->name, SND_SOC_NOPM, 0, 0,
+					    widget_kctls, gbaudio_widget_event,
+					    SND_SOC_DAPM_PRE_PMU |
+					    SND_SOC_DAPM_POST_PMD);
+		break;
+	case snd_soc_dapm_pga:
+		*dw = (struct snd_soc_dapm_widget)
+			SND_SOC_DAPM_PGA_E(w->name, SND_SOC_NOPM, 0, 0, NULL, 0,
+					   gbaudio_widget_event,
+					   SND_SOC_DAPM_PRE_PMU |
+					   SND_SOC_DAPM_POST_PMD);
+		break;
+	case snd_soc_dapm_mixer:
+		*dw = (struct snd_soc_dapm_widget)
+			SND_SOC_DAPM_MIXER_E(w->name, SND_SOC_NOPM, 0, 0, NULL,
+					   0, gbaudio_widget_event,
+					   SND_SOC_DAPM_PRE_PMU |
+					   SND_SOC_DAPM_POST_PMD);
+		break;
+	case snd_soc_dapm_mux:
+		*dw = (struct snd_soc_dapm_widget)
+			SND_SOC_DAPM_MUX_E(w->name, SND_SOC_NOPM, 0, 0,
+					 widget_kctls, gbaudio_widget_event,
+					 SND_SOC_DAPM_PRE_PMU |
+					 SND_SOC_DAPM_POST_PMD);
+		break;
+	case snd_soc_dapm_aif_in:
+		*dw = (struct snd_soc_dapm_widget)
+			SND_SOC_DAPM_AIF_IN_E(w->name, w->sname, 0,
+					      SND_SOC_NOPM,
+					      0, 0, gbaudio_widget_event,
+					      SND_SOC_DAPM_PRE_PMU |
+					      SND_SOC_DAPM_POST_PMD);
+		break;
+	case snd_soc_dapm_aif_out:
+		*dw = (struct snd_soc_dapm_widget)
+			SND_SOC_DAPM_AIF_OUT_E(w->name, w->sname, 0,
+					       SND_SOC_NOPM,
+					       0, 0, gbaudio_widget_event,
+					       SND_SOC_DAPM_PRE_PMU |
+					       SND_SOC_DAPM_POST_PMD);
+		break;
+	default:
+		ret = -EINVAL;
+		goto error;
+	}
+
+	dev_dbg(module->dev, "%s: widget of type %d created\n", dw->name,
+		dw->id);
+	return 0;
+error:
+	list_for_each_entry_safe(control, _control, &module->widget_ctl_list,
+				 list) {
+		list_del(&control->list);
+		devm_kfree(module->dev, control);
+	}
+	return ret;
+}
+
+static int gbaudio_tplg_process_kcontrols(struct gbaudio_module_info *module,
+				   struct gb_audio_control *controls)
+{
+	int i, csize, ret;
+	struct snd_kcontrol_new *dapm_kctls;
+	struct gb_audio_control *curr;
+	struct gbaudio_control *control, *_control;
+	size_t size;
+	char temp_name[NAME_SIZE];
+
+	size = sizeof(struct snd_kcontrol_new) * module->num_controls;
+	dapm_kctls = devm_kzalloc(module->dev, size, GFP_KERNEL);
+	if (!dapm_kctls)
+		return -ENOMEM;
+
+	curr = controls;
+	for (i = 0; i < module->num_controls; i++) {
+		ret = gbaudio_tplg_create_kcontrol(module, &dapm_kctls[i],
+						   curr);
+		if (ret) {
+			dev_err(module->dev, "%s:%d type not supported\n",
+				curr->name, curr->iface);
+			goto error;
+		}
+		control = devm_kzalloc(module->dev, sizeof(struct
+							   gbaudio_control),
+				      GFP_KERNEL);
+		if (!control) {
+			ret = -ENOMEM;
+			goto error;
+		}
+		control->id = curr->id;
+		/* Prefix dev_id to widget_name */
+		strlcpy(temp_name, curr->name, NAME_SIZE);
+		snprintf(curr->name, NAME_SIZE, "GB %d %s", module->dev_id,
+			 temp_name);
+		control->name = curr->name;
+		if (curr->info.type == GB_AUDIO_CTL_ELEM_TYPE_ENUMERATED) {
+			struct gb_audio_enumerated *gbenum =
+				&curr->info.value.enumerated;
+
+			csize = offsetof(struct gb_audio_control, info);
+			csize += offsetof(struct gb_audio_ctl_elem_info, value);
+			csize += offsetof(struct gb_audio_enumerated, names);
+			csize += gbenum->names_length;
+			control->texts = (const char * const *)
+				gb_generate_enum_strings(module, gbenum);
+			control->items = gbenum->items;
+		} else
+			csize = sizeof(struct gb_audio_control);
+
+		list_add(&control->list, &module->ctl_list);
+		dev_dbg(module->dev, "%d:%s created of type %d\n", curr->id,
+			curr->name, curr->info.type);
+		curr = (void *)curr + csize;
+	}
+	module->controls = dapm_kctls;
+
+	return 0;
+error:
+	list_for_each_entry_safe(control, _control, &module->ctl_list,
+				 list) {
+		list_del(&control->list);
+		devm_kfree(module->dev, control);
+	}
+	devm_kfree(module->dev, dapm_kctls);
+	return ret;
+}
+
+static int gbaudio_tplg_process_widgets(struct gbaudio_module_info *module,
+				   struct gb_audio_widget *widgets)
+{
+	int i, ret, w_size;
+	struct snd_soc_dapm_widget *dapm_widgets;
+	struct gb_audio_widget *curr;
+	struct gbaudio_widget *widget, *_widget;
+	size_t size;
+
+	size = sizeof(struct snd_soc_dapm_widget) * module->num_dapm_widgets;
+	dapm_widgets = devm_kzalloc(module->dev, size, GFP_KERNEL);
+	if (!dapm_widgets)
+		return -ENOMEM;
+
+	curr = widgets;
+	for (i = 0; i < module->num_dapm_widgets; i++) {
+		ret = gbaudio_tplg_create_widget(module, &dapm_widgets[i],
+						 curr, &w_size);
+		if (ret) {
+			dev_err(module->dev, "%s:%d type not supported\n",
+				curr->name, curr->type);
+			goto error;
+		}
+		widget = devm_kzalloc(module->dev, sizeof(struct
+							   gbaudio_widget),
+				      GFP_KERNEL);
+		if (!widget) {
+			ret = -ENOMEM;
+			goto error;
+		}
+		widget->id = curr->id;
+		widget->name = curr->name;
+		list_add(&widget->list, &module->widget_list);
+		curr = (void *)curr + w_size;
+	}
+	module->dapm_widgets = dapm_widgets;
+
+	return 0;
+
+error:
+	list_for_each_entry_safe(widget, _widget, &module->widget_list,
+				 list) {
+		list_del(&widget->list);
+		devm_kfree(module->dev, widget);
+	}
+	devm_kfree(module->dev, dapm_widgets);
+	return ret;
+}
+
+static int gbaudio_tplg_process_routes(struct gbaudio_module_info *module,
+				   struct gb_audio_route *routes)
+{
+	int i, ret;
+	struct snd_soc_dapm_route *dapm_routes;
+	struct gb_audio_route *curr;
+	size_t size;
+
+	size = sizeof(struct snd_soc_dapm_route) * module->num_dapm_routes;
+	dapm_routes = devm_kzalloc(module->dev, size, GFP_KERNEL);
+	if (!dapm_routes)
+		return -ENOMEM;
+
+	module->dapm_routes = dapm_routes;
+	curr = routes;
+
+	for (i = 0; i < module->num_dapm_routes; i++) {
+		dapm_routes->sink =
+			gbaudio_map_widgetid(module, curr->destination_id);
+		if (!dapm_routes->sink) {
+			dev_err(module->dev, "%d:%d:%d:%d - Invalid sink\n",
+				curr->source_id, curr->destination_id,
+				curr->control_id, curr->index);
+			ret = -EINVAL;
+			goto error;
+		}
+		dapm_routes->source =
+			gbaudio_map_widgetid(module, curr->source_id);
+		if (!dapm_routes->source) {
+			dev_err(module->dev, "%d:%d:%d:%d - Invalid source\n",
+				curr->source_id, curr->destination_id,
+				curr->control_id, curr->index);
+			ret = -EINVAL;
+			goto error;
+		}
+		dapm_routes->control =
+			gbaudio_map_controlid(module,
+						      curr->control_id,
+						      curr->index);
+		if ((curr->control_id !=  GBAUDIO_INVALID_ID) &&
+		    !dapm_routes->control) {
+			dev_err(module->dev, "%d:%d:%d:%d - Invalid control\n",
+				curr->source_id, curr->destination_id,
+				curr->control_id, curr->index);
+			ret = -EINVAL;
+			goto error;
+		}
+		dev_dbg(module->dev, "Route {%s, %s, %s}\n", dapm_routes->sink,
+			(dapm_routes->control) ? dapm_routes->control:"NULL",
+			dapm_routes->source);
+		dapm_routes++;
+		curr++;
+	}
+
+	return 0;
+
+error:
+	devm_kfree(module->dev, module->dapm_routes);
+	return ret;
+}
+
+static int gbaudio_tplg_process_header(struct gbaudio_module_info *module,
+				 struct gb_audio_topology *tplg_data)
+{
+	/* fetch no. of kcontrols, widgets & routes */
+	module->num_controls = tplg_data->num_controls;
+	module->num_dapm_widgets = tplg_data->num_widgets;
+	module->num_dapm_routes = tplg_data->num_routes;
+
+	/* update block offset */
+	module->dai_offset = (unsigned long)&tplg_data->data;
+	module->control_offset = module->dai_offset + tplg_data->size_dais;
+	module->widget_offset = module->control_offset +
+		tplg_data->size_controls;
+	module->route_offset = module->widget_offset +
+		tplg_data->size_widgets;
+
+	dev_dbg(module->dev, "DAI offset is 0x%lx\n", module->dai_offset);
+	dev_dbg(module->dev, "control offset is %lx\n",
+		module->control_offset);
+	dev_dbg(module->dev, "widget offset is %lx\n", module->widget_offset);
+	dev_dbg(module->dev, "route offset is %lx\n", module->route_offset);
+
+	return 0;
+}
+
+int gbaudio_tplg_parse_data(struct gbaudio_module_info *module,
+			       struct gb_audio_topology *tplg_data)
+{
+	int ret;
+	struct gb_audio_control *controls;
+	struct gb_audio_widget *widgets;
+	struct gb_audio_route *routes;
+
+	if (!tplg_data)
+		return -EINVAL;
+
+	ret = gbaudio_tplg_process_header(module, tplg_data);
+	if (ret) {
+		dev_err(module->dev, "%d: Error in parsing topology header\n",
+			ret);
+		return ret;
+	}
+
+	/* process control */
+	controls = (struct gb_audio_control *)module->control_offset;
+	ret = gbaudio_tplg_process_kcontrols(module, controls);
+	if (ret) {
+		dev_err(module->dev,
+			"%d: Error in parsing controls data\n", ret);
+		return ret;
+	}
+	dev_dbg(module->dev, "Control parsing finished\n");
+
+	/* process widgets */
+	widgets = (struct gb_audio_widget *)module->widget_offset;
+	ret = gbaudio_tplg_process_widgets(module, widgets);
+	if (ret) {
+		dev_err(module->dev,
+			"%d: Error in parsing widgets data\n", ret);
+		return ret;
+	}
+	dev_dbg(module->dev, "Widget parsing finished\n");
+
+	/* process route */
+	routes = (struct gb_audio_route *)module->route_offset;
+	ret = gbaudio_tplg_process_routes(module, routes);
+	if (ret) {
+		dev_err(module->dev,
+			"%d: Error in parsing routes data\n", ret);
+		return ret;
+	}
+	dev_dbg(module->dev, "Route parsing finished\n");
+
+	/* parse jack capabilities */
+	if (tplg_data->jack_type) {
+		module->jack_mask = tplg_data->jack_type & GBCODEC_JACK_MASK;
+		module->button_mask = tplg_data->jack_type &
+			GBCODEC_JACK_BUTTON_MASK;
+	}
+
+	return ret;
+}
+
+void gbaudio_tplg_release(struct gbaudio_module_info *module)
+{
+	struct gbaudio_control *control, *_control;
+	struct gbaudio_widget *widget, *_widget;
+
+	if (!module->topology)
+		return;
+
+	/* release kcontrols */
+	list_for_each_entry_safe(control, _control, &module->ctl_list,
+				 list) {
+		list_del(&control->list);
+		devm_kfree(module->dev, control);
+	}
+	if (module->controls)
+		devm_kfree(module->dev, module->controls);
+
+	/* release widget controls */
+	list_for_each_entry_safe(control, _control, &module->widget_ctl_list,
+				 list) {
+		list_del(&control->list);
+		devm_kfree(module->dev, control);
+	}
+
+	/* release widgets */
+	list_for_each_entry_safe(widget, _widget, &module->widget_list,
+				 list) {
+		list_del(&widget->list);
+		devm_kfree(module->dev, widget);
+	}
+	if (module->dapm_widgets)
+		devm_kfree(module->dev, module->dapm_widgets);
+
+	/* release routes */
+	if (module->dapm_routes)
+		devm_kfree(module->dev, module->dapm_routes);
+}
diff --git a/drivers/staging/greybus/authentication.c b/drivers/staging/greybus/authentication.c
new file mode 100644
index 000000000000..168626ba0c03
--- /dev/null
+++ b/drivers/staging/greybus/authentication.c
@@ -0,0 +1,429 @@
+/*
+ * Greybus Component Authentication Protocol (CAP) Driver.
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include "greybus.h"
+
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/ioctl.h>
+#include <linux/uaccess.h>
+
+#include "greybus_authentication.h"
+#include "firmware.h"
+#include "greybus.h"
+
+#define CAP_TIMEOUT_MS		1000
+
+/*
+ * Number of minor devices this driver supports.
+ * There will be exactly one required per Interface.
+ */
+#define NUM_MINORS		U8_MAX
+
+struct gb_cap {
+	struct device		*parent;
+	struct gb_connection	*connection;
+	struct kref		kref;
+	struct list_head	node;
+	bool			disabled; /* connection getting disabled */
+
+	struct mutex		mutex;
+	struct cdev		cdev;
+	struct device		*class_device;
+	dev_t			dev_num;
+};
+
+static struct class *cap_class;
+static dev_t cap_dev_num;
+static DEFINE_IDA(cap_minors_map);
+static LIST_HEAD(cap_list);
+static DEFINE_MUTEX(list_mutex);
+
+static void cap_kref_release(struct kref *kref)
+{
+	struct gb_cap *cap = container_of(kref, struct gb_cap, kref);
+
+	kfree(cap);
+}
+
+/*
+ * All users of cap take a reference (from within list_mutex lock), before
+ * they get a pointer to play with. And the structure will be freed only after
+ * the last user has put the reference to it.
+ */
+static void put_cap(struct gb_cap *cap)
+{
+	kref_put(&cap->kref, cap_kref_release);
+}
+
+/* Caller must call put_cap() after using struct gb_cap */
+static struct gb_cap *get_cap(struct cdev *cdev)
+{
+	struct gb_cap *cap;
+
+	mutex_lock(&list_mutex);
+
+	list_for_each_entry(cap, &cap_list, node) {
+		if (&cap->cdev == cdev) {
+			kref_get(&cap->kref);
+			goto unlock;
+		}
+	}
+
+	cap = NULL;
+
+unlock:
+	mutex_unlock(&list_mutex);
+
+	return cap;
+}
+
+static int cap_get_endpoint_uid(struct gb_cap *cap, u8 *euid)
+{
+	struct gb_connection *connection = cap->connection;
+	struct gb_cap_get_endpoint_uid_response response;
+	int ret;
+
+	ret = gb_operation_sync(connection, GB_CAP_TYPE_GET_ENDPOINT_UID, NULL,
+				0, &response, sizeof(response));
+	if (ret) {
+		dev_err(cap->parent, "failed to get endpoint uid (%d)\n", ret);
+		return ret;
+	}
+
+	memcpy(euid, response.uid, sizeof(response.uid));
+
+	return 0;
+}
+
+static int cap_get_ims_certificate(struct gb_cap *cap, u32 class, u32 id,
+				   u8 *certificate, u32 *size, u8 *result)
+{
+	struct gb_connection *connection = cap->connection;
+	struct gb_cap_get_ims_certificate_request *request;
+	struct gb_cap_get_ims_certificate_response *response;
+	size_t max_size = gb_operation_get_payload_size_max(connection);
+	struct gb_operation *op;
+	int ret;
+
+	op = gb_operation_create_flags(connection,
+				       GB_CAP_TYPE_GET_IMS_CERTIFICATE,
+				       sizeof(*request), max_size,
+				       GB_OPERATION_FLAG_SHORT_RESPONSE,
+				       GFP_KERNEL);
+	if (!op)
+		return -ENOMEM;
+
+	request = op->request->payload;
+	request->certificate_class = cpu_to_le32(class);
+	request->certificate_id = cpu_to_le32(id);
+
+	ret = gb_operation_request_send_sync(op);
+	if (ret) {
+		dev_err(cap->parent, "failed to get certificate (%d)\n", ret);
+		goto done;
+	}
+
+	response = op->response->payload;
+	*result = response->result_code;
+	*size = op->response->payload_size - sizeof(*response);
+	memcpy(certificate, response->certificate, *size);
+
+done:
+	gb_operation_put(op);
+	return ret;
+}
+
+static int cap_authenticate(struct gb_cap *cap, u32 auth_type, u8 *uid,
+			    u8 *challenge, u8 *result, u8 *auth_response,
+			    u32 *signature_size, u8 *signature)
+{
+	struct gb_connection *connection = cap->connection;
+	struct gb_cap_authenticate_request *request;
+	struct gb_cap_authenticate_response *response;
+	size_t max_size = gb_operation_get_payload_size_max(connection);
+	struct gb_operation *op;
+	int ret;
+
+	op = gb_operation_create_flags(connection, GB_CAP_TYPE_AUTHENTICATE,
+				       sizeof(*request), max_size,
+				       GB_OPERATION_FLAG_SHORT_RESPONSE,
+				       GFP_KERNEL);
+	if (!op)
+		return -ENOMEM;
+
+	request = op->request->payload;
+	request->auth_type = cpu_to_le32(auth_type);
+	memcpy(request->uid, uid, sizeof(request->uid));
+	memcpy(request->challenge, challenge, sizeof(request->challenge));
+
+	ret = gb_operation_request_send_sync(op);
+	if (ret) {
+		dev_err(cap->parent, "failed to authenticate (%d)\n", ret);
+		goto done;
+	}
+
+	response = op->response->payload;
+	*result = response->result_code;
+	*signature_size = op->response->payload_size - sizeof(*response);
+	memcpy(auth_response, response->response, sizeof(response->response));
+	memcpy(signature, response->signature, *signature_size);
+
+done:
+	gb_operation_put(op);
+	return ret;
+}
+
+/* Char device fops */
+
+static int cap_open(struct inode *inode, struct file *file)
+{
+	struct gb_cap *cap = get_cap(inode->i_cdev);
+
+	/* cap structure can't get freed until file descriptor is closed */
+	if (cap) {
+		file->private_data = cap;
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int cap_release(struct inode *inode, struct file *file)
+{
+	struct gb_cap *cap = file->private_data;
+
+	put_cap(cap);
+	return 0;
+}
+
+static int cap_ioctl(struct gb_cap *cap, unsigned int cmd,
+			 void __user *buf)
+{
+	struct cap_ioc_get_endpoint_uid endpoint_uid;
+	struct cap_ioc_get_ims_certificate *ims_cert;
+	struct cap_ioc_authenticate *authenticate;
+	size_t size;
+	int ret;
+
+	switch (cmd) {
+	case CAP_IOC_GET_ENDPOINT_UID:
+		ret = cap_get_endpoint_uid(cap, endpoint_uid.uid);
+		if (ret)
+			return ret;
+
+		if (copy_to_user(buf, &endpoint_uid, sizeof(endpoint_uid)))
+			return -EFAULT;
+
+		return 0;
+	case CAP_IOC_GET_IMS_CERTIFICATE:
+		size = sizeof(*ims_cert);
+		ims_cert = memdup_user(buf, size);
+		if (IS_ERR(ims_cert))
+			return PTR_ERR(ims_cert);
+
+		ret = cap_get_ims_certificate(cap, ims_cert->certificate_class,
+					      ims_cert->certificate_id,
+					      ims_cert->certificate,
+					      &ims_cert->cert_size,
+					      &ims_cert->result_code);
+		if (!ret && copy_to_user(buf, ims_cert, size))
+			ret = -EFAULT;
+		kfree(ims_cert);
+
+		return ret;
+	case CAP_IOC_AUTHENTICATE:
+		size = sizeof(*authenticate);
+		authenticate = memdup_user(buf, size);
+		if (IS_ERR(authenticate))
+			return PTR_ERR(authenticate);
+
+		ret = cap_authenticate(cap, authenticate->auth_type,
+				       authenticate->uid,
+				       authenticate->challenge,
+				       &authenticate->result_code,
+				       authenticate->response,
+				       &authenticate->signature_size,
+				       authenticate->signature);
+		if (!ret && copy_to_user(buf, authenticate, size))
+			ret = -EFAULT;
+		kfree(authenticate);
+
+		return ret;
+	default:
+		return -ENOTTY;
+	}
+}
+
+static long cap_ioctl_unlocked(struct file *file, unsigned int cmd,
+			       unsigned long arg)
+{
+	struct gb_cap *cap = file->private_data;
+	struct gb_bundle *bundle = cap->connection->bundle;
+	int ret = -ENODEV;
+
+	/*
+	 * Serialize ioctls.
+	 *
+	 * We don't want the user to do multiple authentication operations in
+	 * parallel.
+	 *
+	 * This is also used to protect ->disabled, which is used to check if
+	 * the connection is getting disconnected, so that we don't start any
+	 * new operations.
+	 */
+	mutex_lock(&cap->mutex);
+	if (!cap->disabled) {
+		ret = gb_pm_runtime_get_sync(bundle);
+		if (!ret) {
+			ret = cap_ioctl(cap, cmd, (void __user *)arg);
+			gb_pm_runtime_put_autosuspend(bundle);
+		}
+	}
+	mutex_unlock(&cap->mutex);
+
+	return ret;
+}
+
+static const struct file_operations cap_fops = {
+	.owner		= THIS_MODULE,
+	.open		= cap_open,
+	.release	= cap_release,
+	.unlocked_ioctl	= cap_ioctl_unlocked,
+};
+
+int gb_cap_connection_init(struct gb_connection *connection)
+{
+	struct gb_cap *cap;
+	int ret, minor;
+
+	if (!connection)
+		return 0;
+
+	cap = kzalloc(sizeof(*cap), GFP_KERNEL);
+	if (!cap)
+		return -ENOMEM;
+
+	cap->parent = &connection->bundle->dev;
+	cap->connection = connection;
+	mutex_init(&cap->mutex);
+	gb_connection_set_data(connection, cap);
+	kref_init(&cap->kref);
+
+	mutex_lock(&list_mutex);
+	list_add(&cap->node, &cap_list);
+	mutex_unlock(&list_mutex);
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto err_list_del;
+
+	minor = ida_simple_get(&cap_minors_map, 0, NUM_MINORS, GFP_KERNEL);
+	if (minor < 0) {
+		ret = minor;
+		goto err_connection_disable;
+	}
+
+	/* Add a char device to allow userspace to interact with cap */
+	cap->dev_num = MKDEV(MAJOR(cap_dev_num), minor);
+	cdev_init(&cap->cdev, &cap_fops);
+
+	ret = cdev_add(&cap->cdev, cap->dev_num, 1);
+	if (ret)
+		goto err_remove_ida;
+
+	/* Add a soft link to the previously added char-dev within the bundle */
+	cap->class_device = device_create(cap_class, cap->parent, cap->dev_num,
+					  NULL, "gb-authenticate-%d", minor);
+	if (IS_ERR(cap->class_device)) {
+		ret = PTR_ERR(cap->class_device);
+		goto err_del_cdev;
+	}
+
+	return 0;
+
+err_del_cdev:
+	cdev_del(&cap->cdev);
+err_remove_ida:
+	ida_simple_remove(&cap_minors_map, minor);
+err_connection_disable:
+	gb_connection_disable(connection);
+err_list_del:
+	mutex_lock(&list_mutex);
+	list_del(&cap->node);
+	mutex_unlock(&list_mutex);
+
+	put_cap(cap);
+
+	return ret;
+}
+
+void gb_cap_connection_exit(struct gb_connection *connection)
+{
+	struct gb_cap *cap;
+
+	if (!connection)
+		return;
+
+	cap = gb_connection_get_data(connection);
+
+	device_destroy(cap_class, cap->dev_num);
+	cdev_del(&cap->cdev);
+	ida_simple_remove(&cap_minors_map, MINOR(cap->dev_num));
+
+	/*
+	 * Disallow any new ioctl operations on the char device and wait for
+	 * existing ones to finish.
+	 */
+	mutex_lock(&cap->mutex);
+	cap->disabled = true;
+	mutex_unlock(&cap->mutex);
+
+	/* All pending greybus operations should have finished by now */
+	gb_connection_disable(cap->connection);
+
+	/* Disallow new users to get access to the cap structure */
+	mutex_lock(&list_mutex);
+	list_del(&cap->node);
+	mutex_unlock(&list_mutex);
+
+	/*
+	 * All current users of cap would have taken a reference to it by
+	 * now, we can drop our reference and wait the last user will get
+	 * cap freed.
+	 */
+	put_cap(cap);
+}
+
+int cap_init(void)
+{
+	int ret;
+
+	cap_class = class_create(THIS_MODULE, "gb_authenticate");
+	if (IS_ERR(cap_class))
+		return PTR_ERR(cap_class);
+
+	ret = alloc_chrdev_region(&cap_dev_num, 0, NUM_MINORS,
+				  "gb_authenticate");
+	if (ret)
+		goto err_remove_class;
+
+	return 0;
+
+err_remove_class:
+	class_destroy(cap_class);
+	return ret;
+}
+
+void cap_exit(void)
+{
+	unregister_chrdev_region(cap_dev_num, NUM_MINORS);
+	class_destroy(cap_class);
+	ida_destroy(&cap_minors_map);
+}
diff --git a/drivers/staging/greybus/bootrom.c b/drivers/staging/greybus/bootrom.c
new file mode 100644
index 000000000000..5f90721bcc51
--- /dev/null
+++ b/drivers/staging/greybus/bootrom.c
@@ -0,0 +1,524 @@
+/*
+ * BOOTROM Greybus driver.
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/firmware.h>
+#include <linux/jiffies.h>
+#include <linux/mutex.h>
+#include <linux/workqueue.h>
+
+#include "greybus.h"
+#include "firmware.h"
+
+/* Timeout, in jiffies, within which the next request must be received */
+#define NEXT_REQ_TIMEOUT_MS	1000
+
+/*
+ * FIXME: Reduce this timeout once svc core handles parallel processing of
+ * events from the SVC, which are handled sequentially today.
+ */
+#define MODE_SWITCH_TIMEOUT_MS	10000
+
+enum next_request_type {
+	NEXT_REQ_FIRMWARE_SIZE,
+	NEXT_REQ_GET_FIRMWARE,
+	NEXT_REQ_READY_TO_BOOT,
+	NEXT_REQ_MODE_SWITCH,
+};
+
+struct gb_bootrom {
+	struct gb_connection	*connection;
+	const struct firmware	*fw;
+	u8			protocol_major;
+	u8			protocol_minor;
+	enum next_request_type	next_request;
+	struct delayed_work	dwork;
+	struct mutex		mutex; /* Protects bootrom->fw */
+};
+
+static void free_firmware(struct gb_bootrom *bootrom)
+{
+	if (!bootrom->fw)
+		return;
+
+	release_firmware(bootrom->fw);
+	bootrom->fw = NULL;
+}
+
+static void gb_bootrom_timedout(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct gb_bootrom *bootrom = container_of(dwork, struct gb_bootrom, dwork);
+	struct device *dev = &bootrom->connection->bundle->dev;
+	const char *reason;
+
+	switch (bootrom->next_request) {
+	case NEXT_REQ_FIRMWARE_SIZE:
+		reason = "Firmware Size Request";
+		break;
+	case NEXT_REQ_GET_FIRMWARE:
+		reason = "Get Firmware Request";
+		break;
+	case NEXT_REQ_READY_TO_BOOT:
+		reason = "Ready to Boot Request";
+		break;
+	case NEXT_REQ_MODE_SWITCH:
+		reason = "Interface Mode Switch";
+		break;
+	default:
+		reason = NULL;
+		dev_err(dev, "Invalid next-request: %u", bootrom->next_request);
+		break;
+	}
+
+	dev_err(dev, "Timed out waiting for %s from the Module\n", reason);
+
+	mutex_lock(&bootrom->mutex);
+	free_firmware(bootrom);
+	mutex_unlock(&bootrom->mutex);
+
+	/* TODO: Power-off Module ? */
+}
+
+static void gb_bootrom_set_timeout(struct gb_bootrom *bootrom,
+			enum next_request_type next, unsigned long timeout)
+{
+	bootrom->next_request = next;
+	schedule_delayed_work(&bootrom->dwork, msecs_to_jiffies(timeout));
+}
+
+static void gb_bootrom_cancel_timeout(struct gb_bootrom *bootrom)
+{
+	cancel_delayed_work_sync(&bootrom->dwork);
+}
+
+/*
+ * The es2 chip doesn't have VID/PID programmed into the hardware and we need to
+ * hack that up to distinguish different modules and their firmware blobs.
+ *
+ * This fetches VID/PID (over bootrom protocol) for es2 chip only, when VID/PID
+ * already sent during hotplug are 0.
+ *
+ * Otherwise, we keep intf->vendor_id/product_id same as what's passed
+ * during hotplug.
+ */
+static void bootrom_es2_fixup_vid_pid(struct gb_bootrom *bootrom)
+{
+	struct gb_bootrom_get_vid_pid_response response;
+	struct gb_connection *connection = bootrom->connection;
+	struct gb_interface *intf = connection->bundle->intf;
+	int ret;
+
+	if (!(intf->quirks & GB_INTERFACE_QUIRK_NO_GMP_IDS))
+		return;
+
+	ret = gb_operation_sync(connection, GB_BOOTROM_TYPE_GET_VID_PID,
+				NULL, 0, &response, sizeof(response));
+	if (ret) {
+		dev_err(&connection->bundle->dev,
+			"Bootrom get vid/pid operation failed (%d)\n", ret);
+		return;
+	}
+
+	/*
+	 * NOTE: This is hacked, so that the same values of VID/PID can be used
+	 * by next firmware level as well. The uevent for bootrom will still
+	 * have VID/PID as 0, though after this point the sysfs files will start
+	 * showing the updated values. But yeah, that's a bit racy as the same
+	 * sysfs files would be showing 0 before this point.
+	 */
+	intf->vendor_id = le32_to_cpu(response.vendor_id);
+	intf->product_id = le32_to_cpu(response.product_id);
+
+	dev_dbg(&connection->bundle->dev, "Bootrom got vid (0x%x)/pid (0x%x)\n",
+		intf->vendor_id, intf->product_id);
+}
+
+/* This returns path of the firmware blob on the disk */
+static int find_firmware(struct gb_bootrom *bootrom, u8 stage)
+{
+	struct gb_connection *connection = bootrom->connection;
+	struct gb_interface *intf = connection->bundle->intf;
+	char firmware_name[49];
+	int rc;
+
+	/* Already have a firmware, free it */
+	free_firmware(bootrom);
+
+	/* Bootrom protocol is only supported for loading Stage 2 firmware */
+	if (stage != 2) {
+		dev_err(&connection->bundle->dev, "Invalid boot stage: %u\n",
+			stage);
+		return -EINVAL;
+	}
+
+	/*
+	 * Create firmware name
+	 *
+	 * XXX Name it properly..
+	 */
+	snprintf(firmware_name, sizeof(firmware_name),
+		 FW_NAME_PREFIX "%08x_%08x_%08x_%08x_s2l.tftf",
+		 intf->ddbl1_manufacturer_id, intf->ddbl1_product_id,
+		 intf->vendor_id, intf->product_id);
+
+	// FIXME:
+	// Turn to dev_dbg later after everyone has valid bootloaders with good
+	// ids, but leave this as dev_info for now to make it easier to track
+	// down "empty" vid/pid modules.
+	dev_info(&connection->bundle->dev, "Firmware file '%s' requested\n",
+		 firmware_name);
+
+	rc = request_firmware(&bootrom->fw, firmware_name,
+		&connection->bundle->dev);
+	if (rc) {
+		dev_err(&connection->bundle->dev,
+			"failed to find %s firmware (%d)\n", firmware_name, rc);
+	}
+
+	return rc;
+}
+
+static int gb_bootrom_firmware_size_request(struct gb_operation *op)
+{
+	struct gb_bootrom *bootrom = gb_connection_get_data(op->connection);
+	struct gb_bootrom_firmware_size_request *size_request = op->request->payload;
+	struct gb_bootrom_firmware_size_response *size_response;
+	struct device *dev = &op->connection->bundle->dev;
+	int ret;
+
+	/* Disable timeouts */
+	gb_bootrom_cancel_timeout(bootrom);
+
+	if (op->request->payload_size != sizeof(*size_request)) {
+		dev_err(dev, "%s: illegal size of firmware size request (%zu != %zu)\n",
+			__func__, op->request->payload_size,
+			sizeof(*size_request));
+		ret = -EINVAL;
+		goto queue_work;
+	}
+
+	mutex_lock(&bootrom->mutex);
+
+	ret = find_firmware(bootrom, size_request->stage);
+	if (ret)
+		goto unlock;
+
+	if (!gb_operation_response_alloc(op, sizeof(*size_response),
+					 GFP_KERNEL)) {
+		dev_err(dev, "%s: error allocating response\n", __func__);
+		free_firmware(bootrom);
+		ret = -ENOMEM;
+		goto unlock;
+	}
+
+	size_response = op->response->payload;
+	size_response->size = cpu_to_le32(bootrom->fw->size);
+
+	dev_dbg(dev, "%s: firmware size %d bytes\n", __func__, size_response->size);
+
+unlock:
+	mutex_unlock(&bootrom->mutex);
+
+queue_work:
+	if (!ret) {
+		/* Refresh timeout */
+		gb_bootrom_set_timeout(bootrom, NEXT_REQ_GET_FIRMWARE,
+				       NEXT_REQ_TIMEOUT_MS);
+	}
+
+	return ret;
+}
+
+static int gb_bootrom_get_firmware(struct gb_operation *op)
+{
+	struct gb_bootrom *bootrom = gb_connection_get_data(op->connection);
+	const struct firmware *fw;
+	struct gb_bootrom_get_firmware_request *firmware_request;
+	struct gb_bootrom_get_firmware_response *firmware_response;
+	struct device *dev = &op->connection->bundle->dev;
+	unsigned int offset, size;
+	enum next_request_type next_request;
+	int ret = 0;
+
+	/* Disable timeouts */
+	gb_bootrom_cancel_timeout(bootrom);
+
+	if (op->request->payload_size != sizeof(*firmware_request)) {
+		dev_err(dev, "%s: Illegal size of get firmware request (%zu %zu)\n",
+			__func__, op->request->payload_size,
+			sizeof(*firmware_request));
+		ret = -EINVAL;
+		goto queue_work;
+	}
+
+	mutex_lock(&bootrom->mutex);
+
+	fw = bootrom->fw;
+	if (!fw) {
+		dev_err(dev, "%s: firmware not available\n", __func__);
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	firmware_request = op->request->payload;
+	offset = le32_to_cpu(firmware_request->offset);
+	size = le32_to_cpu(firmware_request->size);
+
+	if (offset >= fw->size || size > fw->size - offset) {
+		dev_warn(dev, "bad firmware request (offs = %u, size = %u)\n",
+				offset, size);
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	if (!gb_operation_response_alloc(op, sizeof(*firmware_response) + size,
+					 GFP_KERNEL)) {
+		dev_err(dev, "%s: error allocating response\n", __func__);
+		ret = -ENOMEM;
+		goto unlock;
+	}
+
+	firmware_response = op->response->payload;
+	memcpy(firmware_response->data, fw->data + offset, size);
+
+	dev_dbg(dev, "responding with firmware (offs = %u, size = %u)\n", offset,
+		size);
+
+unlock:
+	mutex_unlock(&bootrom->mutex);
+
+queue_work:
+	/* Refresh timeout */
+	if (!ret && (offset + size == fw->size))
+		next_request = NEXT_REQ_READY_TO_BOOT;
+	else
+		next_request = NEXT_REQ_GET_FIRMWARE;
+
+	gb_bootrom_set_timeout(bootrom, next_request, NEXT_REQ_TIMEOUT_MS);
+
+	return ret;
+}
+
+static int gb_bootrom_ready_to_boot(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_bootrom *bootrom = gb_connection_get_data(connection);
+	struct gb_bootrom_ready_to_boot_request *rtb_request;
+	struct device *dev = &connection->bundle->dev;
+	u8 status;
+	int ret = 0;
+
+	/* Disable timeouts */
+	gb_bootrom_cancel_timeout(bootrom);
+
+	if (op->request->payload_size != sizeof(*rtb_request)) {
+		dev_err(dev, "%s: Illegal size of ready to boot request (%zu %zu)\n",
+			__func__, op->request->payload_size,
+			sizeof(*rtb_request));
+		ret = -EINVAL;
+		goto queue_work;
+	}
+
+	rtb_request = op->request->payload;
+	status = rtb_request->status;
+
+	/* Return error if the blob was invalid */
+	if (status == GB_BOOTROM_BOOT_STATUS_INVALID) {
+		ret = -EINVAL;
+		goto queue_work;
+	}
+
+	/*
+	 * XXX Should we return error for insecure firmware?
+	 */
+	dev_dbg(dev, "ready to boot: 0x%x, 0\n", status);
+
+queue_work:
+	/*
+	 * Refresh timeout, the Interface shall load the new personality and
+	 * send a new hotplug request, which shall get rid of the bootrom
+	 * connection. As that can take some time, increase the timeout a bit.
+	 */
+	gb_bootrom_set_timeout(bootrom, NEXT_REQ_MODE_SWITCH,
+			       MODE_SWITCH_TIMEOUT_MS);
+
+	return ret;
+}
+
+static int gb_bootrom_request_handler(struct gb_operation *op)
+{
+	u8 type = op->type;
+
+	switch (type) {
+	case GB_BOOTROM_TYPE_FIRMWARE_SIZE:
+		return gb_bootrom_firmware_size_request(op);
+	case GB_BOOTROM_TYPE_GET_FIRMWARE:
+		return gb_bootrom_get_firmware(op);
+	case GB_BOOTROM_TYPE_READY_TO_BOOT:
+		return gb_bootrom_ready_to_boot(op);
+	default:
+		dev_err(&op->connection->bundle->dev,
+			"unsupported request: %u\n", type);
+		return -EINVAL;
+	}
+}
+
+static int gb_bootrom_get_version(struct gb_bootrom *bootrom)
+{
+	struct gb_bundle *bundle = bootrom->connection->bundle;
+	struct gb_bootrom_version_request request;
+	struct gb_bootrom_version_response response;
+	int ret;
+
+	request.major = GB_BOOTROM_VERSION_MAJOR;
+	request.minor = GB_BOOTROM_VERSION_MINOR;
+
+	ret = gb_operation_sync(bootrom->connection,
+				GB_BOOTROM_TYPE_VERSION,
+				&request, sizeof(request), &response,
+				sizeof(response));
+	if (ret) {
+		dev_err(&bundle->dev,
+				"failed to get protocol version: %d\n",
+				ret);
+		return ret;
+	}
+
+	if (response.major > request.major) {
+		dev_err(&bundle->dev,
+				"unsupported major protocol version (%u > %u)\n",
+				response.major, request.major);
+		return -ENOTSUPP;
+	}
+
+	bootrom->protocol_major = response.major;
+	bootrom->protocol_minor = response.minor;
+
+	dev_dbg(&bundle->dev, "%s - %u.%u\n", __func__, response.major,
+			response.minor);
+
+	return 0;
+}
+
+static int gb_bootrom_probe(struct gb_bundle *bundle,
+					const struct greybus_bundle_id *id)
+{
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_connection *connection;
+	struct gb_bootrom *bootrom;
+	int ret;
+
+	if (bundle->num_cports != 1)
+		return -ENODEV;
+
+	cport_desc = &bundle->cport_desc[0];
+	if (cport_desc->protocol_id != GREYBUS_PROTOCOL_BOOTROM)
+		return -ENODEV;
+
+	bootrom = kzalloc(sizeof(*bootrom), GFP_KERNEL);
+	if (!bootrom)
+		return -ENOMEM;
+
+	connection = gb_connection_create(bundle,
+						le16_to_cpu(cport_desc->id),
+						gb_bootrom_request_handler);
+	if (IS_ERR(connection)) {
+		ret = PTR_ERR(connection);
+		goto err_free_bootrom;
+	}
+
+	gb_connection_set_data(connection, bootrom);
+
+	bootrom->connection = connection;
+
+	mutex_init(&bootrom->mutex);
+	INIT_DELAYED_WORK(&bootrom->dwork, gb_bootrom_timedout);
+	greybus_set_drvdata(bundle, bootrom);
+
+	ret = gb_connection_enable_tx(connection);
+	if (ret)
+		goto err_connection_destroy;
+
+	ret = gb_bootrom_get_version(bootrom);
+	if (ret)
+		goto err_connection_disable;
+
+	bootrom_es2_fixup_vid_pid(bootrom);
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto err_connection_disable;
+
+	/* Refresh timeout */
+	gb_bootrom_set_timeout(bootrom, NEXT_REQ_FIRMWARE_SIZE,
+			       NEXT_REQ_TIMEOUT_MS);
+
+	/* Tell bootrom we're ready. */
+	ret = gb_operation_sync(connection, GB_BOOTROM_TYPE_AP_READY, NULL, 0,
+				NULL, 0);
+	if (ret) {
+		dev_err(&connection->bundle->dev,
+				"failed to send AP READY: %d\n", ret);
+		goto err_cancel_timeout;
+	}
+
+	dev_dbg(&bundle->dev, "AP_READY sent\n");
+
+	return 0;
+
+err_cancel_timeout:
+	gb_bootrom_cancel_timeout(bootrom);
+err_connection_disable:
+	gb_connection_disable(connection);
+err_connection_destroy:
+	gb_connection_destroy(connection);
+err_free_bootrom:
+	kfree(bootrom);
+
+	return ret;
+}
+
+static void gb_bootrom_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_bootrom *bootrom = greybus_get_drvdata(bundle);
+
+	dev_dbg(&bundle->dev, "%s\n", __func__);
+
+	gb_connection_disable(bootrom->connection);
+
+	/* Disable timeouts */
+	gb_bootrom_cancel_timeout(bootrom);
+
+	/*
+	 * Release firmware:
+	 *
+	 * As the connection and the delayed work are already disabled, we don't
+	 * need to lock access to bootrom->fw here.
+	 */
+	free_firmware(bootrom);
+
+	gb_connection_destroy(bootrom->connection);
+	kfree(bootrom);
+}
+
+static const struct greybus_bundle_id gb_bootrom_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_BOOTROM) },
+	{ }
+};
+
+static struct greybus_driver gb_bootrom_driver = {
+	.name		= "bootrom",
+	.probe		= gb_bootrom_probe,
+	.disconnect	= gb_bootrom_disconnect,
+	.id_table	= gb_bootrom_id_table,
+};
+
+module_greybus_driver(gb_bootrom_driver);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/bundle.c b/drivers/staging/greybus/bundle.c
new file mode 100644
index 000000000000..d2ef57d090be
--- /dev/null
+++ b/drivers/staging/greybus/bundle.c
@@ -0,0 +1,253 @@
+/*
+ * Greybus bundles
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include "greybus.h"
+#include "greybus_trace.h"
+
+static ssize_t bundle_class_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+
+	return sprintf(buf, "0x%02x\n", bundle->class);
+}
+static DEVICE_ATTR_RO(bundle_class);
+
+static ssize_t bundle_id_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+
+	return sprintf(buf, "%u\n", bundle->id);
+}
+static DEVICE_ATTR_RO(bundle_id);
+
+static ssize_t state_show(struct device *dev, struct device_attribute *attr,
+			  char *buf)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+
+	if (bundle->state == NULL)
+		return sprintf(buf, "\n");
+
+	return sprintf(buf, "%s\n", bundle->state);
+}
+
+static ssize_t state_store(struct device *dev, struct device_attribute *attr,
+			   const char *buf, size_t size)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+
+	kfree(bundle->state);
+	bundle->state = kstrdup(buf, GFP_KERNEL);
+	if (!bundle->state)
+		return -ENOMEM;
+
+	/* Tell userspace that the file contents changed */
+	sysfs_notify(&bundle->dev.kobj, NULL, "state");
+
+	return size;
+}
+static DEVICE_ATTR_RW(state);
+
+static struct attribute *bundle_attrs[] = {
+	&dev_attr_bundle_class.attr,
+	&dev_attr_bundle_id.attr,
+	&dev_attr_state.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(bundle);
+
+static struct gb_bundle *gb_bundle_find(struct gb_interface *intf,
+							u8 bundle_id)
+{
+	struct gb_bundle *bundle;
+
+	list_for_each_entry(bundle, &intf->bundles, links) {
+		if (bundle->id == bundle_id)
+			return bundle;
+	}
+
+	return NULL;
+}
+
+static void gb_bundle_release(struct device *dev)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+
+	trace_gb_bundle_release(bundle);
+
+	kfree(bundle->state);
+	kfree(bundle->cport_desc);
+	kfree(bundle);
+}
+
+#ifdef CONFIG_PM
+static void gb_bundle_disable_all_connections(struct gb_bundle *bundle)
+{
+	struct gb_connection *connection;
+
+	list_for_each_entry(connection, &bundle->connections, bundle_links)
+		gb_connection_disable(connection);
+}
+
+static void gb_bundle_enable_all_connections(struct gb_bundle *bundle)
+{
+	struct gb_connection *connection;
+
+	list_for_each_entry(connection, &bundle->connections, bundle_links)
+		gb_connection_enable(connection);
+}
+
+static int gb_bundle_suspend(struct device *dev)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+	const struct dev_pm_ops *pm = dev->driver->pm;
+	int ret;
+
+	if (pm && pm->runtime_suspend) {
+		ret = pm->runtime_suspend(&bundle->dev);
+		if (ret)
+			return ret;
+	} else {
+		gb_bundle_disable_all_connections(bundle);
+	}
+
+	ret = gb_control_bundle_suspend(bundle->intf->control, bundle->id);
+	if (ret) {
+		if (pm && pm->runtime_resume)
+			ret = pm->runtime_resume(dev);
+		else
+			gb_bundle_enable_all_connections(bundle);
+
+		return ret;
+	}
+
+	return 0;
+}
+
+static int gb_bundle_resume(struct device *dev)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+	const struct dev_pm_ops *pm = dev->driver->pm;
+	int ret;
+
+	ret = gb_control_bundle_resume(bundle->intf->control, bundle->id);
+	if (ret)
+		return ret;
+
+	if (pm && pm->runtime_resume) {
+		ret = pm->runtime_resume(dev);
+		if (ret)
+			return ret;
+	} else {
+		gb_bundle_enable_all_connections(bundle);
+	}
+
+	return 0;
+}
+
+static int gb_bundle_idle(struct device *dev)
+{
+	pm_runtime_mark_last_busy(dev);
+	pm_request_autosuspend(dev);
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops gb_bundle_pm_ops = {
+	SET_RUNTIME_PM_OPS(gb_bundle_suspend, gb_bundle_resume, gb_bundle_idle)
+};
+
+struct device_type greybus_bundle_type = {
+	.name =		"greybus_bundle",
+	.release =	gb_bundle_release,
+	.pm =		&gb_bundle_pm_ops,
+};
+
+/*
+ * Create a gb_bundle structure to represent a discovered
+ * bundle.  Returns a pointer to the new bundle or a null
+ * pointer if a failure occurs due to memory exhaustion.
+ */
+struct gb_bundle *gb_bundle_create(struct gb_interface *intf, u8 bundle_id,
+				   u8 class)
+{
+	struct gb_bundle *bundle;
+
+	if (bundle_id == BUNDLE_ID_NONE) {
+		dev_err(&intf->dev, "can't use bundle id %u\n", bundle_id);
+		return NULL;
+	}
+
+	/*
+	 * Reject any attempt to reuse a bundle id.  We initialize
+	 * these serially, so there's no need to worry about keeping
+	 * the interface bundle list locked here.
+	 */
+	if (gb_bundle_find(intf, bundle_id)) {
+		dev_err(&intf->dev, "duplicate bundle id %u\n", bundle_id);
+		return NULL;
+	}
+
+	bundle = kzalloc(sizeof(*bundle), GFP_KERNEL);
+	if (!bundle)
+		return NULL;
+
+	bundle->intf = intf;
+	bundle->id = bundle_id;
+	bundle->class = class;
+	INIT_LIST_HEAD(&bundle->connections);
+
+	bundle->dev.parent = &intf->dev;
+	bundle->dev.bus = &greybus_bus_type;
+	bundle->dev.type = &greybus_bundle_type;
+	bundle->dev.groups = bundle_groups;
+	bundle->dev.dma_mask = intf->dev.dma_mask;
+	device_initialize(&bundle->dev);
+	dev_set_name(&bundle->dev, "%s.%d", dev_name(&intf->dev), bundle_id);
+
+	list_add(&bundle->links, &intf->bundles);
+
+	trace_gb_bundle_create(bundle);
+
+	return bundle;
+}
+
+int gb_bundle_add(struct gb_bundle *bundle)
+{
+	int ret;
+
+	ret = device_add(&bundle->dev);
+	if (ret) {
+		dev_err(&bundle->dev, "failed to register bundle: %d\n", ret);
+		return ret;
+	}
+
+	trace_gb_bundle_add(bundle);
+
+	return 0;
+}
+
+/*
+ * Tear down a previously set up bundle.
+ */
+void gb_bundle_destroy(struct gb_bundle *bundle)
+{
+	trace_gb_bundle_destroy(bundle);
+
+	if (device_is_registered(&bundle->dev))
+		device_del(&bundle->dev);
+
+	list_del(&bundle->links);
+
+	put_device(&bundle->dev);
+}
diff --git a/drivers/staging/greybus/bundle.h b/drivers/staging/greybus/bundle.h
new file mode 100644
index 000000000000..0c3491def96c
--- /dev/null
+++ b/drivers/staging/greybus/bundle.h
@@ -0,0 +1,90 @@
+/*
+ * Greybus bundles
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __BUNDLE_H
+#define __BUNDLE_H
+
+#include <linux/list.h>
+
+#define	BUNDLE_ID_NONE	U8_MAX
+
+/* Greybus "public" definitions" */
+struct gb_bundle {
+	struct device		dev;
+	struct gb_interface	*intf;
+
+	u8			id;
+	u8			class;
+	u8			class_major;
+	u8			class_minor;
+
+	size_t			num_cports;
+	struct greybus_descriptor_cport *cport_desc;
+
+	struct list_head	connections;
+	u8			*state;
+
+	struct list_head	links;	/* interface->bundles */
+};
+#define to_gb_bundle(d) container_of(d, struct gb_bundle, dev)
+
+/* Greybus "private" definitions" */
+struct gb_bundle *gb_bundle_create(struct gb_interface *intf, u8 bundle_id,
+				   u8 class);
+int gb_bundle_add(struct gb_bundle *bundle);
+void gb_bundle_destroy(struct gb_bundle *bundle);
+
+/* Bundle Runtime PM wrappers */
+#ifdef CONFIG_PM
+static inline int gb_pm_runtime_get_sync(struct gb_bundle *bundle)
+{
+	int retval;
+
+	retval = pm_runtime_get_sync(&bundle->dev);
+	if (retval < 0) {
+		dev_err(&bundle->dev,
+			"pm_runtime_get_sync failed: %d\n", retval);
+		pm_runtime_put_noidle(&bundle->dev);
+		return retval;
+	}
+
+	return 0;
+}
+
+static inline int gb_pm_runtime_put_autosuspend(struct gb_bundle *bundle)
+{
+	int retval;
+
+	pm_runtime_mark_last_busy(&bundle->dev);
+	retval = pm_runtime_put_autosuspend(&bundle->dev);
+
+	return retval;
+}
+
+static inline void gb_pm_runtime_get_noresume(struct gb_bundle *bundle)
+{
+	pm_runtime_get_noresume(&bundle->dev);
+}
+
+static inline void gb_pm_runtime_put_noidle(struct gb_bundle *bundle)
+{
+	pm_runtime_put_noidle(&bundle->dev);
+}
+
+#else
+static inline int gb_pm_runtime_get_sync(struct gb_bundle *bundle)
+{ return 0; }
+static inline int gb_pm_runtime_put_autosuspend(struct gb_bundle *bundle)
+{ return 0; }
+
+static inline void gb_pm_runtime_get_noresume(struct gb_bundle *bundle) {}
+static inline void gb_pm_runtime_put_noidle(struct gb_bundle *bundle) {}
+#endif
+
+#endif /* __BUNDLE_H */
diff --git a/drivers/staging/greybus/camera.c b/drivers/staging/greybus/camera.c
new file mode 100644
index 000000000000..491bdd720c0c
--- /dev/null
+++ b/drivers/staging/greybus/camera.c
@@ -0,0 +1,1400 @@
+/*
+ * Greybus Camera protocol driver.
+ *
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/uaccess.h>
+#include <linux/vmalloc.h>
+
+#include "gb-camera.h"
+#include "greybus.h"
+#include "greybus_protocols.h"
+
+enum gb_camera_debugs_buffer_id {
+	GB_CAMERA_DEBUGFS_BUFFER_CAPABILITIES,
+	GB_CAMERA_DEBUGFS_BUFFER_STREAMS,
+	GB_CAMERA_DEBUGFS_BUFFER_CAPTURE,
+	GB_CAMERA_DEBUGFS_BUFFER_FLUSH,
+	GB_CAMERA_DEBUGFS_BUFFER_MAX,
+};
+
+struct gb_camera_debugfs_buffer {
+	char data[PAGE_SIZE];
+	size_t length;
+};
+
+enum gb_camera_state {
+	GB_CAMERA_STATE_UNCONFIGURED,
+	GB_CAMERA_STATE_CONFIGURED,
+};
+
+/**
+ * struct gb_camera - A Greybus Camera Device
+ * @connection: the greybus connection for camera management
+ * @data_connection: the greybus connection for camera data
+ * @data_cport_id: the data CPort ID on the module side
+ * @mutex: protects the connection and state fields
+ * @state: the current module state
+ * @debugfs: debugfs entries for camera protocol operations testing
+ * @module: Greybus camera module registered to HOST processor.
+ */
+struct gb_camera {
+	struct gb_bundle *bundle;
+	struct gb_connection *connection;
+	struct gb_connection *data_connection;
+	u16 data_cport_id;
+
+	struct mutex mutex;
+	enum gb_camera_state state;
+
+	struct {
+		struct dentry *root;
+		struct gb_camera_debugfs_buffer *buffers;
+	} debugfs;
+
+	struct gb_camera_module module;
+};
+
+struct gb_camera_stream_config {
+	unsigned int width;
+	unsigned int height;
+	unsigned int format;
+	unsigned int vc;
+	unsigned int dt[2];
+	unsigned int max_size;
+};
+
+struct gb_camera_fmt_info {
+	enum v4l2_mbus_pixelcode mbus_code;
+	unsigned int gb_format;
+	unsigned int bpp;
+};
+
+/* GB format to media code map */
+static const struct gb_camera_fmt_info gb_fmt_info[] = {
+	{
+		.mbus_code = V4L2_MBUS_FMT_UYVY8_1X16,
+		.gb_format = 0x01,
+		.bpp	   = 16,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_NV12_1x8,
+		.gb_format = 0x12,
+		.bpp	   = 12,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_NV21_1x8,
+		.gb_format = 0x13,
+		.bpp	   = 12,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_YU12_1x8,
+		.gb_format = 0x16,
+		.bpp	   = 12,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_YV12_1x8,
+		.gb_format = 0x17,
+		.bpp	   = 12,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_JPEG_1X8,
+		.gb_format = 0x40,
+		.bpp	   = 0,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_GB_CAM_METADATA_1X8,
+		.gb_format = 0x41,
+		.bpp	   = 0,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_GB_CAM_DEBUG_DATA_1X8,
+		.gb_format = 0x42,
+		.bpp	   = 0,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_SBGGR10_1X10,
+		.gb_format = 0x80,
+		.bpp	   = 10,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_SGBRG10_1X10,
+		.gb_format = 0x81,
+		.bpp	   = 10,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_SGRBG10_1X10,
+		.gb_format = 0x82,
+		.bpp	   = 10,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_SRGGB10_1X10,
+		.gb_format = 0x83,
+		.bpp	   = 10,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_SBGGR12_1X12,
+		.gb_format = 0x84,
+		.bpp	   = 12,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_SGBRG12_1X12,
+		.gb_format = 0x85,
+		.bpp	   = 12,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_SGRBG12_1X12,
+		.gb_format = 0x86,
+		.bpp	   = 12,
+	},
+	{
+		.mbus_code = V4L2_MBUS_FMT_SRGGB12_1X12,
+		.gb_format = 0x87,
+		.bpp	   = 12,
+	},
+};
+
+static const struct gb_camera_fmt_info *gb_camera_get_format_info(u16 gb_fmt)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(gb_fmt_info); i++) {
+		if (gb_fmt_info[i].gb_format == gb_fmt)
+			return &gb_fmt_info[i];
+	}
+
+	return NULL;
+}
+
+#define ES2_APB_CDSI0_CPORT		16
+#define ES2_APB_CDSI1_CPORT		17
+
+#define GB_CAMERA_MAX_SETTINGS_SIZE	8192
+
+#define gcam_dbg(gcam, format...)	dev_dbg(&gcam->bundle->dev, format)
+#define gcam_info(gcam, format...)	dev_info(&gcam->bundle->dev, format)
+#define gcam_err(gcam, format...)	dev_err(&gcam->bundle->dev, format)
+
+static int gb_camera_operation_sync_flags(struct gb_connection *connection,
+					  int type, unsigned int flags,
+					  void *request, size_t request_size,
+					  void *response, size_t *response_size)
+{
+	struct gb_operation *operation;
+	int ret;
+
+	operation = gb_operation_create_flags(connection, type, request_size,
+					      *response_size, flags,
+					      GFP_KERNEL);
+	if (!operation)
+		return  -ENOMEM;
+
+	if (request_size)
+		memcpy(operation->request->payload, request, request_size);
+
+	ret = gb_operation_request_send_sync(operation);
+	if (ret) {
+		dev_err(&connection->hd->dev,
+			"%s: synchronous operation of type 0x%02x failed: %d\n",
+			connection->name, type, ret);
+	} else {
+		*response_size = operation->response->payload_size;
+
+		if (operation->response->payload_size)
+			memcpy(response, operation->response->payload,
+			       operation->response->payload_size);
+	}
+
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+static int gb_camera_get_max_pkt_size(struct gb_camera *gcam,
+		struct gb_camera_configure_streams_response *resp)
+{
+	unsigned int max_pkt_size = 0;
+	unsigned int i;
+
+	for (i = 0; i < resp->num_streams; i++) {
+		struct gb_camera_stream_config_response *cfg = &resp->config[i];
+		const struct gb_camera_fmt_info *fmt_info;
+		unsigned int pkt_size;
+
+		fmt_info = gb_camera_get_format_info(cfg->format);
+		if (!fmt_info) {
+			gcam_err(gcam, "unsupported greybus image format: %d\n",
+				 cfg->format);
+			return -EIO;
+		}
+
+		if (fmt_info->bpp == 0) {
+			pkt_size = le32_to_cpu(cfg->max_pkt_size);
+
+			if (pkt_size == 0) {
+				gcam_err(gcam,
+					 "Stream %u: invalid zero maximum packet size\n",
+					 i);
+				return -EIO;
+			}
+		} else {
+			pkt_size = le16_to_cpu(cfg->width) * fmt_info->bpp / 8;
+
+			if (pkt_size != le32_to_cpu(cfg->max_pkt_size)) {
+				gcam_err(gcam,
+					 "Stream %u: maximum packet size mismatch (%u/%u)\n",
+					 i, pkt_size, cfg->max_pkt_size);
+				return -EIO;
+			}
+		}
+
+		max_pkt_size = max(pkt_size, max_pkt_size);
+	}
+
+	return max_pkt_size;
+}
+
+/*
+ * Validate the stream configuration response verifying padding is correctly
+ * set and the returned number of streams is supported
+ */
+static const int gb_camera_configure_streams_validate_response(
+		struct gb_camera *gcam,
+		struct gb_camera_configure_streams_response *resp,
+		unsigned int nstreams)
+{
+	unsigned int i;
+
+	/* Validate the returned response structure */
+	if (resp->padding[0] || resp->padding[1]) {
+		gcam_err(gcam, "response padding != 0\n");
+		return -EIO;
+	}
+
+	if (resp->num_streams > nstreams) {
+		gcam_err(gcam, "got #streams %u > request %u\n",
+			 resp->num_streams, nstreams);
+		return -EIO;
+	}
+
+	for (i = 0; i < resp->num_streams; i++) {
+		struct gb_camera_stream_config_response *cfg = &resp->config[i];
+		if (cfg->padding) {
+			gcam_err(gcam, "stream #%u padding != 0\n", i);
+			return -EIO;
+		}
+	}
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * Hardware Configuration
+ */
+
+static int gb_camera_set_intf_power_mode(struct gb_camera *gcam, u8 intf_id,
+					 bool hs)
+{
+	struct gb_svc *svc = gcam->connection->hd->svc;
+	int ret;
+
+	if (hs)
+		ret = gb_svc_intf_set_power_mode(svc, intf_id,
+						 GB_SVC_UNIPRO_HS_SERIES_A,
+						 GB_SVC_UNIPRO_FAST_MODE, 2, 2,
+						 GB_SVC_SMALL_AMPLITUDE,
+						 GB_SVC_NO_DE_EMPHASIS,
+						 GB_SVC_UNIPRO_FAST_MODE, 2, 2,
+						 GB_SVC_PWRM_RXTERMINATION |
+						 GB_SVC_PWRM_TXTERMINATION, 0,
+						 NULL, NULL);
+	else
+		ret = gb_svc_intf_set_power_mode(svc, intf_id,
+						 GB_SVC_UNIPRO_HS_SERIES_A,
+						 GB_SVC_UNIPRO_SLOW_AUTO_MODE,
+						 2, 1,
+						 GB_SVC_SMALL_AMPLITUDE,
+						 GB_SVC_NO_DE_EMPHASIS,
+						 GB_SVC_UNIPRO_SLOW_AUTO_MODE,
+						 2, 1,
+						 0, 0,
+						 NULL, NULL);
+
+	return ret;
+}
+
+static int gb_camera_set_power_mode(struct gb_camera *gcam, bool hs)
+{
+	struct gb_interface *intf = gcam->connection->intf;
+	struct gb_svc *svc = gcam->connection->hd->svc;
+	int ret;
+
+	ret = gb_camera_set_intf_power_mode(gcam, intf->interface_id, hs);
+	if (ret < 0) {
+		gcam_err(gcam, "failed to set module interface to %s (%d)\n",
+			 hs ? "HS" : "PWM", ret);
+		return ret;
+	}
+
+	ret = gb_camera_set_intf_power_mode(gcam, svc->ap_intf_id, hs);
+	if (ret < 0) {
+		gb_camera_set_intf_power_mode(gcam, intf->interface_id, !hs);
+		gcam_err(gcam, "failed to set AP interface to %s (%d)\n",
+			 hs ? "HS" : "PWM", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+struct ap_csi_config_request {
+	__u8 csi_id;
+	__u8 flags;
+#define GB_CAMERA_CSI_FLAG_CLOCK_CONTINUOUS 0x01
+	__u8 num_lanes;
+	__u8 padding;
+	__le32 csi_clk_freq;
+	__le32 max_pkt_size;
+} __packed;
+
+/*
+ * TODO: Compute the number of lanes dynamically based on bandwidth
+ * requirements.
+ */
+#define GB_CAMERA_CSI_NUM_DATA_LANES		4
+
+#define GB_CAMERA_CSI_CLK_FREQ_MAX		999000000U
+#define GB_CAMERA_CSI_CLK_FREQ_MIN		100000000U
+#define GB_CAMERA_CSI_CLK_FREQ_MARGIN		150000000U
+
+static int gb_camera_setup_data_connection(struct gb_camera *gcam,
+		struct gb_camera_configure_streams_response *resp,
+		struct gb_camera_csi_params *csi_params)
+{
+	struct ap_csi_config_request csi_cfg;
+	struct gb_connection *conn;
+	unsigned int clk_freq;
+	int ret;
+
+	/*
+	 * Create the data connection between the camera module data CPort and
+	 * APB CDSI1. The CDSI1 CPort ID is hardcoded by the ES2 bridge.
+	 */
+	conn = gb_connection_create_offloaded(gcam->bundle, gcam->data_cport_id,
+					      GB_CONNECTION_FLAG_NO_FLOWCTRL |
+					      GB_CONNECTION_FLAG_CDSI1);
+	if (IS_ERR(conn))
+		return PTR_ERR(conn);
+
+	gcam->data_connection = conn;
+	gb_connection_set_data(conn, gcam);
+
+	ret = gb_connection_enable(conn);
+	if (ret)
+		goto error_conn_destroy;
+
+	/* Set the UniPro link to high speed mode. */
+	ret = gb_camera_set_power_mode(gcam, true);
+	if (ret < 0)
+		goto error_conn_disable;
+
+	/*
+	 * Configure the APB-A CSI-2 transmitter.
+	 *
+	 * Hardcode the number of lanes to 4 and compute the bus clock frequency
+	 * based on the module bandwidth requirements with a safety margin.
+	 */
+	memset(&csi_cfg, 0, sizeof(csi_cfg));
+	csi_cfg.csi_id = 1;
+	csi_cfg.flags = 0;
+	csi_cfg.num_lanes = GB_CAMERA_CSI_NUM_DATA_LANES;
+
+	clk_freq = resp->data_rate / 2 / GB_CAMERA_CSI_NUM_DATA_LANES;
+	clk_freq = clamp(clk_freq + GB_CAMERA_CSI_CLK_FREQ_MARGIN,
+			 GB_CAMERA_CSI_CLK_FREQ_MIN,
+			 GB_CAMERA_CSI_CLK_FREQ_MAX);
+	csi_cfg.csi_clk_freq = clk_freq;
+
+	ret = gb_camera_get_max_pkt_size(gcam, resp);
+	if (ret < 0) {
+		ret = -EIO;
+		goto error_power;
+	}
+	csi_cfg.max_pkt_size = ret;
+
+	ret = gb_hd_output(gcam->connection->hd, &csi_cfg,
+			   sizeof(csi_cfg),
+			   GB_APB_REQUEST_CSI_TX_CONTROL, false);
+	if (ret < 0) {
+		gcam_err(gcam, "failed to start the CSI transmitter\n");
+		goto error_power;
+	}
+
+	if (csi_params) {
+		csi_params->clk_freq = csi_cfg.csi_clk_freq;
+		csi_params->num_lanes = csi_cfg.num_lanes;
+	}
+
+	return 0;
+
+error_power:
+	gb_camera_set_power_mode(gcam, false);
+error_conn_disable:
+	gb_connection_disable(gcam->data_connection);
+error_conn_destroy:
+	gb_connection_destroy(gcam->data_connection);
+	gcam->data_connection = NULL;
+	return ret;
+}
+
+static void gb_camera_teardown_data_connection(struct gb_camera *gcam)
+{
+	struct ap_csi_config_request csi_cfg;
+	int ret;
+
+	/* Stop the APB1 CSI transmitter. */
+	memset(&csi_cfg, 0, sizeof(csi_cfg));
+	csi_cfg.csi_id = 1;
+
+	ret = gb_hd_output(gcam->connection->hd, &csi_cfg,
+			   sizeof(csi_cfg),
+			   GB_APB_REQUEST_CSI_TX_CONTROL, false);
+
+	if (ret < 0)
+		gcam_err(gcam, "failed to stop the CSI transmitter\n");
+
+	/* Set the UniPro link to low speed mode. */
+	gb_camera_set_power_mode(gcam, false);
+
+	/* Destroy the data connection. */
+	gb_connection_disable(gcam->data_connection);
+	gb_connection_destroy(gcam->data_connection);
+	gcam->data_connection = NULL;
+}
+
+/* -----------------------------------------------------------------------------
+ * Camera Protocol Operations
+ */
+
+static int gb_camera_capabilities(struct gb_camera *gcam,
+				  u8 *capabilities, size_t *size)
+{
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(gcam->bundle);
+	if (ret)
+		return ret;
+
+	mutex_lock(&gcam->mutex);
+
+	if (!gcam->connection) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = gb_camera_operation_sync_flags(gcam->connection,
+					     GB_CAMERA_TYPE_CAPABILITIES,
+					     GB_OPERATION_FLAG_SHORT_RESPONSE,
+					     NULL, 0,
+					     (void *)capabilities, size);
+	if (ret)
+		gcam_err(gcam, "failed to retrieve capabilities: %d\n", ret);
+
+done:
+	mutex_unlock(&gcam->mutex);
+
+	gb_pm_runtime_put_autosuspend(gcam->bundle);
+
+	return ret;
+}
+
+static int gb_camera_configure_streams(struct gb_camera *gcam,
+				       unsigned int *num_streams,
+				       unsigned int *flags,
+				       struct gb_camera_stream_config *streams,
+				       struct gb_camera_csi_params *csi_params)
+{
+	struct gb_camera_configure_streams_request *req;
+	struct gb_camera_configure_streams_response *resp;
+	unsigned int nstreams = *num_streams;
+	unsigned int i;
+	size_t req_size;
+	size_t resp_size;
+	int ret;
+
+	if (nstreams > GB_CAMERA_MAX_STREAMS)
+		return -EINVAL;
+
+	req_size = sizeof(*req) + nstreams * sizeof(req->config[0]);
+	resp_size = sizeof(*resp) + nstreams * sizeof(resp->config[0]);
+
+	req = kmalloc(req_size, GFP_KERNEL);
+	resp = kmalloc(resp_size, GFP_KERNEL);
+	if (!req || !resp) {
+		kfree(req);
+		kfree(resp);
+		return -ENOMEM;
+	}
+
+	req->num_streams = nstreams;
+	req->flags = *flags;
+	req->padding = 0;
+
+	for (i = 0; i < nstreams; ++i) {
+		struct gb_camera_stream_config_request *cfg = &req->config[i];
+
+		cfg->width = cpu_to_le16(streams[i].width);
+		cfg->height = cpu_to_le16(streams[i].height);
+		cfg->format = cpu_to_le16(streams[i].format);
+		cfg->padding = 0;
+	}
+
+	mutex_lock(&gcam->mutex);
+
+	ret = gb_pm_runtime_get_sync(gcam->bundle);
+	if (ret)
+		goto done_skip_pm_put;
+
+	if (!gcam->connection) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = gb_camera_operation_sync_flags(gcam->connection,
+					     GB_CAMERA_TYPE_CONFIGURE_STREAMS,
+					     GB_OPERATION_FLAG_SHORT_RESPONSE,
+					     req, req_size,
+					     resp, &resp_size);
+	if (ret < 0)
+		goto done;
+
+	ret = gb_camera_configure_streams_validate_response(gcam, resp,
+							    nstreams);
+	if (ret < 0)
+		goto done;
+
+	*flags = resp->flags;
+	*num_streams = resp->num_streams;
+
+	for (i = 0; i < resp->num_streams; ++i) {
+		struct gb_camera_stream_config_response *cfg = &resp->config[i];
+
+		streams[i].width = le16_to_cpu(cfg->width);
+		streams[i].height = le16_to_cpu(cfg->height);
+		streams[i].format = le16_to_cpu(cfg->format);
+		streams[i].vc = cfg->virtual_channel;
+		streams[i].dt[0] = cfg->data_type[0];
+		streams[i].dt[1] = cfg->data_type[1];
+		streams[i].max_size = le32_to_cpu(cfg->max_size);
+	}
+
+	if ((resp->flags & GB_CAMERA_CONFIGURE_STREAMS_ADJUSTED) ||
+	    (req->flags & GB_CAMERA_CONFIGURE_STREAMS_TEST_ONLY))
+		goto done;
+
+	if (gcam->state == GB_CAMERA_STATE_CONFIGURED) {
+		gb_camera_teardown_data_connection(gcam);
+		gcam->state = GB_CAMERA_STATE_UNCONFIGURED;
+
+		/*
+		 * When unconfiguring streams release the PM runtime reference
+		 * that was acquired when streams were configured. The bundle
+		 * won't be suspended until the PM runtime reference acquired at
+		 * the beginning of this function gets released right before
+		 * returning.
+		 */
+		gb_pm_runtime_put_noidle(gcam->bundle);
+	}
+
+	if (resp->num_streams == 0)
+		goto done;
+
+	/*
+	 * Make sure the bundle won't be suspended until streams get
+	 * unconfigured after the stream is configured successfully
+	 */
+	gb_pm_runtime_get_noresume(gcam->bundle);
+
+	/* Setup CSI-2 connection from APB-A to AP */
+	ret = gb_camera_setup_data_connection(gcam, resp, csi_params);
+	if (ret < 0) {
+		memset(req, 0, sizeof(*req));
+		gb_operation_sync(gcam->connection,
+				  GB_CAMERA_TYPE_CONFIGURE_STREAMS,
+				  req, sizeof(*req),
+				  resp, sizeof(*resp));
+		*flags = 0;
+		*num_streams = 0;
+		gb_pm_runtime_put_noidle(gcam->bundle);
+		goto done;
+	}
+
+	gcam->state = GB_CAMERA_STATE_CONFIGURED;
+
+done:
+	gb_pm_runtime_put_autosuspend(gcam->bundle);
+
+done_skip_pm_put:
+	mutex_unlock(&gcam->mutex);
+	kfree(req);
+	kfree(resp);
+	return ret;
+}
+
+static int gb_camera_capture(struct gb_camera *gcam, u32 request_id,
+			     unsigned int streams, unsigned int num_frames,
+			     size_t settings_size, const void *settings)
+{
+	struct gb_camera_capture_request *req;
+	size_t req_size;
+	int ret;
+
+	if (settings_size > GB_CAMERA_MAX_SETTINGS_SIZE)
+		return -EINVAL;
+
+	req_size = sizeof(*req) + settings_size;
+	req = kmalloc(req_size, GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	req->request_id = cpu_to_le32(request_id);
+	req->streams = streams;
+	req->padding = 0;
+	req->num_frames = cpu_to_le16(num_frames);
+	memcpy(req->settings, settings, settings_size);
+
+	mutex_lock(&gcam->mutex);
+
+	if (!gcam->connection) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = gb_operation_sync(gcam->connection, GB_CAMERA_TYPE_CAPTURE,
+				req, req_size, NULL, 0);
+done:
+	mutex_unlock(&gcam->mutex);
+
+	kfree(req);
+
+	return ret;
+}
+
+static int gb_camera_flush(struct gb_camera *gcam, u32 *request_id)
+{
+	struct gb_camera_flush_response resp;
+	int ret;
+
+	mutex_lock(&gcam->mutex);
+
+	if (!gcam->connection) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	ret = gb_operation_sync(gcam->connection, GB_CAMERA_TYPE_FLUSH, NULL, 0,
+				&resp, sizeof(resp));
+
+	if (ret < 0)
+		goto done;
+
+	if (request_id)
+		*request_id = le32_to_cpu(resp.request_id);
+
+done:
+	mutex_unlock(&gcam->mutex);
+
+	return ret;
+}
+
+static int gb_camera_request_handler(struct gb_operation *op)
+{
+	struct gb_camera *gcam = gb_connection_get_data(op->connection);
+	struct gb_camera_metadata_request *payload;
+	struct gb_message *request;
+
+	if (op->type != GB_CAMERA_TYPE_METADATA) {
+		gcam_err(gcam, "Unsupported unsolicited event: %u\n", op->type);
+		return -EINVAL;
+	}
+
+	request = op->request;
+
+	if (request->payload_size < sizeof(*payload)) {
+		gcam_err(gcam, "Wrong event size received (%zu < %zu)\n",
+			 request->payload_size, sizeof(*payload));
+		return -EINVAL;
+	}
+
+	payload = request->payload;
+
+	gcam_dbg(gcam, "received metadata for request %u, frame %u, stream %u\n",
+		 payload->request_id, payload->frame_number, payload->stream);
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * Interface with HOST gmp camera.
+ */
+static unsigned int gb_camera_mbus_to_gb(enum v4l2_mbus_pixelcode mbus_code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(gb_fmt_info); i++) {
+		if (gb_fmt_info[i].mbus_code == mbus_code)
+			return gb_fmt_info[i].gb_format;
+	}
+	return gb_fmt_info[0].gb_format;
+}
+
+static enum v4l2_mbus_pixelcode gb_camera_gb_to_mbus(u16 gb_fmt)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(gb_fmt_info); i++) {
+		if (gb_fmt_info[i].gb_format == gb_fmt)
+			return gb_fmt_info[i].mbus_code;
+	}
+	return gb_fmt_info[0].mbus_code;
+}
+
+static ssize_t gb_camera_op_capabilities(void *priv, char *data, size_t len)
+{
+	struct gb_camera *gcam = priv;
+	size_t capabilities_len = len;
+	int ret;
+
+	ret = gb_camera_capabilities(gcam, data, &capabilities_len);
+	if (ret)
+		return ret;
+
+	return capabilities_len;
+}
+
+static int gb_camera_op_configure_streams(void *priv, unsigned int *nstreams,
+		unsigned int *flags, struct gb_camera_stream *streams,
+		struct gb_camera_csi_params *csi_params)
+{
+	struct gb_camera *gcam = priv;
+	struct gb_camera_stream_config *gb_streams;
+	unsigned int gb_flags = 0;
+	unsigned int gb_nstreams = *nstreams;
+	unsigned int i;
+	int ret;
+
+	if (gb_nstreams > GB_CAMERA_MAX_STREAMS)
+		return -EINVAL;
+
+	gb_streams = kzalloc(gb_nstreams * sizeof(*gb_streams), GFP_KERNEL);
+	if (!gb_streams)
+		return -ENOMEM;
+
+	for (i = 0; i < gb_nstreams; i++) {
+		gb_streams[i].width = streams[i].width;
+		gb_streams[i].height = streams[i].height;
+		gb_streams[i].format =
+			gb_camera_mbus_to_gb(streams[i].pixel_code);
+	}
+
+	if (*flags & GB_CAMERA_IN_FLAG_TEST)
+		gb_flags |= GB_CAMERA_CONFIGURE_STREAMS_TEST_ONLY;
+
+	ret = gb_camera_configure_streams(gcam, &gb_nstreams,
+					  &gb_flags, gb_streams, csi_params);
+	if (ret < 0)
+		goto done;
+	if (gb_nstreams > *nstreams) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	*flags = 0;
+	if (gb_flags & GB_CAMERA_CONFIGURE_STREAMS_ADJUSTED)
+		*flags |= GB_CAMERA_OUT_FLAG_ADJUSTED;
+
+	for (i = 0; i < gb_nstreams; i++) {
+		streams[i].width = gb_streams[i].width;
+		streams[i].height = gb_streams[i].height;
+		streams[i].vc = gb_streams[i].vc;
+		streams[i].dt[0] = gb_streams[i].dt[0];
+		streams[i].dt[1] = gb_streams[i].dt[1];
+		streams[i].max_size = gb_streams[i].max_size;
+		streams[i].pixel_code =
+			gb_camera_gb_to_mbus(gb_streams[i].format);
+	}
+	*nstreams = gb_nstreams;
+
+done:
+	kfree(gb_streams);
+	return ret;
+}
+
+static int gb_camera_op_capture(void *priv, u32 request_id,
+		unsigned int streams, unsigned int num_frames,
+		size_t settings_size, const void *settings)
+{
+	struct gb_camera *gcam = priv;
+
+	return gb_camera_capture(gcam, request_id, streams, num_frames,
+				 settings_size, settings);
+}
+
+static int gb_camera_op_flush(void *priv, u32 *request_id)
+{
+	struct gb_camera *gcam = priv;
+
+	return gb_camera_flush(gcam, request_id);
+}
+
+static const struct gb_camera_ops gb_cam_ops = {
+	.capabilities = gb_camera_op_capabilities,
+	.configure_streams = gb_camera_op_configure_streams,
+	.capture = gb_camera_op_capture,
+	.flush = gb_camera_op_flush,
+};
+
+/* -----------------------------------------------------------------------------
+ * DebugFS
+ */
+
+static ssize_t gb_camera_debugfs_capabilities(struct gb_camera *gcam,
+		char *buf, size_t len)
+{
+	struct gb_camera_debugfs_buffer *buffer =
+		&gcam->debugfs.buffers[GB_CAMERA_DEBUGFS_BUFFER_CAPABILITIES];
+	size_t size = 1024;
+	unsigned int i;
+	u8 *caps;
+	int ret;
+
+	caps = kmalloc(size, GFP_KERNEL);
+	if (!caps)
+		return -ENOMEM;
+
+	ret = gb_camera_capabilities(gcam, caps, &size);
+	if (ret < 0)
+		goto done;
+
+	/*
+	 * hex_dump_to_buffer() doesn't return the number of bytes dumped prior
+	 * to v4.0, we need our own implementation :-(
+	 */
+	buffer->length = 0;
+
+	for (i = 0; i < size; i += 16) {
+		unsigned int nbytes = min_t(unsigned int, size - i, 16);
+
+		buffer->length += sprintf(buffer->data + buffer->length,
+					  "%*ph\n", nbytes, caps + i);
+	}
+
+done:
+	kfree(caps);
+	return ret;
+}
+
+static ssize_t gb_camera_debugfs_configure_streams(struct gb_camera *gcam,
+		char *buf, size_t len)
+{
+	struct gb_camera_debugfs_buffer *buffer =
+		&gcam->debugfs.buffers[GB_CAMERA_DEBUGFS_BUFFER_STREAMS];
+	struct gb_camera_stream_config *streams;
+	unsigned int nstreams;
+	unsigned int flags;
+	unsigned int i;
+	char *token;
+	int ret;
+
+	/* Retrieve number of streams to configure */
+	token = strsep(&buf, ";");
+	if (token == NULL)
+		return -EINVAL;
+
+	ret = kstrtouint(token, 10, &nstreams);
+	if (ret < 0)
+		return ret;
+
+	if (nstreams > GB_CAMERA_MAX_STREAMS)
+		return -EINVAL;
+
+	token = strsep(&buf, ";");
+	if (token == NULL)
+		return -EINVAL;
+
+	ret = kstrtouint(token, 10, &flags);
+	if (ret < 0)
+		return ret;
+
+	/* For each stream to configure parse width, height and format */
+	streams = kzalloc(nstreams * sizeof(*streams), GFP_KERNEL);
+	if (!streams)
+		return -ENOMEM;
+
+	for (i = 0; i < nstreams; ++i) {
+		struct gb_camera_stream_config *stream = &streams[i];
+
+		/* width */
+		token = strsep(&buf, ";");
+		if (token == NULL) {
+			ret = -EINVAL;
+			goto done;
+		}
+		ret = kstrtouint(token, 10, &stream->width);
+		if (ret < 0)
+			goto done;
+
+		/* height */
+		token = strsep(&buf, ";");
+		if (token == NULL)
+			goto done;
+
+		ret = kstrtouint(token, 10, &stream->height);
+		if (ret < 0)
+			goto done;
+
+		/* Image format code */
+		token = strsep(&buf, ";");
+		if (token == NULL)
+			goto done;
+
+		ret = kstrtouint(token, 16, &stream->format);
+		if (ret < 0)
+			goto done;
+	}
+
+	ret = gb_camera_configure_streams(gcam, &nstreams, &flags, streams,
+					  NULL);
+	if (ret < 0)
+		goto done;
+
+	buffer->length = sprintf(buffer->data, "%u;%u;", nstreams, flags);
+
+	for (i = 0; i < nstreams; ++i) {
+		struct gb_camera_stream_config *stream = &streams[i];
+
+		buffer->length += sprintf(buffer->data + buffer->length,
+					  "%u;%u;%u;%u;%u;%u;%u;",
+					  stream->width, stream->height,
+					  stream->format, stream->vc,
+					  stream->dt[0], stream->dt[1],
+					  stream->max_size);
+	}
+
+	ret = len;
+
+done:
+	kfree(streams);
+	return ret;
+};
+
+static ssize_t gb_camera_debugfs_capture(struct gb_camera *gcam,
+		char *buf, size_t len)
+{
+	unsigned int request_id;
+	unsigned int streams_mask;
+	unsigned int num_frames;
+	char *token;
+	int ret;
+
+	/* Request id */
+	token = strsep(&buf, ";");
+	if (token == NULL)
+		return -EINVAL;
+	ret = kstrtouint(token, 10, &request_id);
+	if (ret < 0)
+		return ret;
+
+	/* Stream mask */
+	token = strsep(&buf, ";");
+	if (token == NULL)
+		return -EINVAL;
+	ret = kstrtouint(token, 16, &streams_mask);
+	if (ret < 0)
+		return ret;
+
+	/* number of frames */
+	token = strsep(&buf, ";");
+	if (token == NULL)
+		return -EINVAL;
+	ret = kstrtouint(token, 10, &num_frames);
+	if (ret < 0)
+		return ret;
+
+	ret = gb_camera_capture(gcam, request_id, streams_mask, num_frames, 0,
+				NULL);
+	if (ret < 0)
+		return ret;
+
+	return len;
+}
+
+static ssize_t gb_camera_debugfs_flush(struct gb_camera *gcam,
+		char *buf, size_t len)
+{
+	struct gb_camera_debugfs_buffer *buffer =
+		&gcam->debugfs.buffers[GB_CAMERA_DEBUGFS_BUFFER_FLUSH];
+	unsigned int req_id;
+	int ret;
+
+	ret = gb_camera_flush(gcam, &req_id);
+	if (ret < 0)
+		return ret;
+
+	buffer->length = sprintf(buffer->data, "%u", req_id);
+
+	return len;
+}
+
+struct gb_camera_debugfs_entry {
+	const char *name;
+	unsigned int mask;
+	unsigned int buffer;
+	ssize_t (*execute)(struct gb_camera *gcam, char *buf, size_t len);
+};
+
+static const struct gb_camera_debugfs_entry gb_camera_debugfs_entries[] = {
+	{
+		.name = "capabilities",
+		.mask = S_IFREG | S_IRUGO,
+		.buffer = GB_CAMERA_DEBUGFS_BUFFER_CAPABILITIES,
+		.execute = gb_camera_debugfs_capabilities,
+	}, {
+		.name = "configure_streams",
+		.mask = S_IFREG | S_IRUGO | S_IWUGO,
+		.buffer = GB_CAMERA_DEBUGFS_BUFFER_STREAMS,
+		.execute = gb_camera_debugfs_configure_streams,
+	}, {
+		.name = "capture",
+		.mask = S_IFREG | S_IRUGO | S_IWUGO,
+		.buffer = GB_CAMERA_DEBUGFS_BUFFER_CAPTURE,
+		.execute = gb_camera_debugfs_capture,
+	}, {
+		.name = "flush",
+		.mask = S_IFREG | S_IRUGO | S_IWUGO,
+		.buffer = GB_CAMERA_DEBUGFS_BUFFER_FLUSH,
+		.execute = gb_camera_debugfs_flush,
+	},
+};
+
+static ssize_t gb_camera_debugfs_read(struct file *file, char __user *buf,
+				      size_t len, loff_t *offset)
+{
+	const struct gb_camera_debugfs_entry *op = file->private_data;
+	struct gb_camera *gcam = file->f_inode->i_private;
+	struct gb_camera_debugfs_buffer *buffer;
+	ssize_t ret;
+
+	/* For read-only entries the operation is triggered by a read. */
+	if (!(op->mask & S_IWUGO)) {
+		ret = op->execute(gcam, NULL, 0);
+		if (ret < 0)
+			return ret;
+	}
+
+	buffer = &gcam->debugfs.buffers[op->buffer];
+
+	return simple_read_from_buffer(buf, len, offset, buffer->data,
+				       buffer->length);
+}
+
+static ssize_t gb_camera_debugfs_write(struct file *file,
+				       const char __user *buf, size_t len,
+				       loff_t *offset)
+{
+	const struct gb_camera_debugfs_entry *op = file->private_data;
+	struct gb_camera *gcam = file->f_inode->i_private;
+	ssize_t ret;
+	char *kbuf;
+
+	if (len > 1024)
+	       return -EINVAL;
+
+	kbuf = kmalloc(len + 1, GFP_KERNEL);
+	if (!kbuf)
+		return -ENOMEM;
+
+	if (copy_from_user(kbuf, buf, len)) {
+		ret = -EFAULT;
+		goto done;
+	}
+
+	kbuf[len] = '\0';
+
+	ret = op->execute(gcam, kbuf, len);
+
+done:
+	kfree(kbuf);
+	return ret;
+}
+
+static int gb_camera_debugfs_open(struct inode *inode, struct file *file)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(gb_camera_debugfs_entries); ++i) {
+		const struct gb_camera_debugfs_entry *entry =
+			&gb_camera_debugfs_entries[i];
+
+		if (!strcmp(file->f_path.dentry->d_iname, entry->name)) {
+			file->private_data = (void *)entry;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static const struct file_operations gb_camera_debugfs_ops = {
+	.open = gb_camera_debugfs_open,
+	.read = gb_camera_debugfs_read,
+	.write = gb_camera_debugfs_write,
+};
+
+static int gb_camera_debugfs_init(struct gb_camera *gcam)
+{
+	struct gb_connection *connection = gcam->connection;
+	char dirname[27];
+	unsigned int i;
+
+	/*
+	 * Create root debugfs entry and a file entry for each camera operation.
+	 */
+	snprintf(dirname, 27, "camera-%u.%u", connection->intf->interface_id,
+		 gcam->bundle->id);
+
+	gcam->debugfs.root = debugfs_create_dir(dirname, gb_debugfs_get());
+	if (IS_ERR(gcam->debugfs.root)) {
+		gcam_err(gcam, "debugfs root create failed (%ld)\n",
+			 PTR_ERR(gcam->debugfs.root));
+		return PTR_ERR(gcam->debugfs.root);
+	}
+
+	gcam->debugfs.buffers = vmalloc(sizeof(*gcam->debugfs.buffers) *
+					GB_CAMERA_DEBUGFS_BUFFER_MAX);
+	if (!gcam->debugfs.buffers)
+		return -ENOMEM;
+
+	for (i = 0; i < ARRAY_SIZE(gb_camera_debugfs_entries); ++i) {
+		const struct gb_camera_debugfs_entry *entry =
+			&gb_camera_debugfs_entries[i];
+		struct dentry *dentry;
+
+		gcam->debugfs.buffers[i].length = 0;
+
+		dentry = debugfs_create_file(entry->name, entry->mask,
+					     gcam->debugfs.root, gcam,
+					     &gb_camera_debugfs_ops);
+		if (IS_ERR(dentry)) {
+			gcam_err(gcam,
+				 "debugfs operation %s create failed (%ld)\n",
+				 entry->name, PTR_ERR(dentry));
+			return PTR_ERR(dentry);
+		}
+	}
+
+	return 0;
+}
+
+static void gb_camera_debugfs_cleanup(struct gb_camera *gcam)
+{
+	debugfs_remove_recursive(gcam->debugfs.root);
+
+	vfree(gcam->debugfs.buffers);
+}
+
+/* -----------------------------------------------------------------------------
+ * Init & Cleanup
+ */
+
+static void gb_camera_cleanup(struct gb_camera *gcam)
+{
+	gb_camera_debugfs_cleanup(gcam);
+
+	mutex_lock(&gcam->mutex);
+	if (gcam->data_connection) {
+		gb_connection_disable(gcam->data_connection);
+		gb_connection_destroy(gcam->data_connection);
+		gcam->data_connection = NULL;
+	}
+
+	if (gcam->connection) {
+		gb_connection_disable(gcam->connection);
+		gb_connection_destroy(gcam->connection);
+		gcam->connection = NULL;
+	}
+	mutex_unlock(&gcam->mutex);
+}
+
+static void gb_camera_release_module(struct kref *ref)
+{
+	struct gb_camera_module *cam_mod =
+		container_of(ref, struct gb_camera_module, refcount);
+	kfree(cam_mod->priv);
+}
+
+static int gb_camera_probe(struct gb_bundle *bundle,
+			   const struct greybus_bundle_id *id)
+{
+	struct gb_connection *conn;
+	struct gb_camera *gcam;
+	u16 mgmt_cport_id = 0;
+	u16 data_cport_id = 0;
+	unsigned int i;
+	int ret;
+
+	/*
+	 * The camera bundle must contain exactly two CPorts, one for the
+	 * camera management protocol and one for the camera data protocol.
+	 */
+	if (bundle->num_cports != 2)
+		return -ENODEV;
+
+	for (i = 0; i < bundle->num_cports; ++i) {
+		struct greybus_descriptor_cport *desc = &bundle->cport_desc[i];
+
+		switch (desc->protocol_id) {
+		case GREYBUS_PROTOCOL_CAMERA_MGMT:
+			mgmt_cport_id = le16_to_cpu(desc->id);
+			break;
+		case GREYBUS_PROTOCOL_CAMERA_DATA:
+			data_cport_id = le16_to_cpu(desc->id);
+			break;
+		default:
+			return -ENODEV;
+		}
+	}
+
+	if (!mgmt_cport_id || !data_cport_id)
+		return -ENODEV;
+
+	gcam = kzalloc(sizeof(*gcam), GFP_KERNEL);
+	if (!gcam)
+		return -ENOMEM;
+
+	mutex_init(&gcam->mutex);
+
+	gcam->bundle = bundle;
+	gcam->state = GB_CAMERA_STATE_UNCONFIGURED;
+	gcam->data_cport_id = data_cport_id;
+
+	conn = gb_connection_create(bundle, mgmt_cport_id,
+				    gb_camera_request_handler);
+	if (IS_ERR(conn)) {
+		ret = PTR_ERR(conn);
+		goto error;
+	}
+
+	gcam->connection = conn;
+	gb_connection_set_data(conn, gcam);
+
+	ret = gb_connection_enable(conn);
+	if (ret)
+		goto error;
+
+	ret = gb_camera_debugfs_init(gcam);
+	if (ret < 0)
+		goto error;
+
+	gcam->module.priv = gcam;
+	gcam->module.ops = &gb_cam_ops;
+	gcam->module.interface_id = gcam->connection->intf->interface_id;
+	gcam->module.release = gb_camera_release_module;
+	ret = gb_camera_register(&gcam->module);
+	if (ret < 0)
+		goto error;
+
+	greybus_set_drvdata(bundle, gcam);
+
+	gb_pm_runtime_put_autosuspend(gcam->bundle);
+
+	return 0;
+
+error:
+	gb_camera_cleanup(gcam);
+	kfree(gcam);
+	return ret;
+}
+
+static void gb_camera_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_camera *gcam = greybus_get_drvdata(bundle);
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		gb_pm_runtime_get_noresume(bundle);
+
+	gb_camera_cleanup(gcam);
+	gb_camera_unregister(&gcam->module);
+}
+
+static const struct greybus_bundle_id gb_camera_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_CAMERA) },
+	{ },
+};
+
+#ifdef CONFIG_PM
+static int gb_camera_suspend(struct device *dev)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+	struct gb_camera *gcam = greybus_get_drvdata(bundle);
+
+	if (gcam->data_connection)
+		gb_connection_disable(gcam->data_connection);
+
+	gb_connection_disable(gcam->connection);
+
+	return 0;
+}
+
+static int gb_camera_resume(struct device *dev)
+{
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+	struct gb_camera *gcam = greybus_get_drvdata(bundle);
+	int ret;
+
+	ret = gb_connection_enable(gcam->connection);
+	if (ret) {
+		gcam_err(gcam, "failed to enable connection: %d\n", ret);
+		return ret;
+	}
+
+	if (gcam->data_connection) {
+		ret = gb_connection_enable(gcam->data_connection);
+		if (ret) {
+			gcam_err(gcam,
+				 "failed to enable data connection: %d\n", ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops gb_camera_pm_ops = {
+	SET_RUNTIME_PM_OPS(gb_camera_suspend, gb_camera_resume, NULL)
+};
+
+static struct greybus_driver gb_camera_driver = {
+	.name		= "camera",
+	.probe		= gb_camera_probe,
+	.disconnect	= gb_camera_disconnect,
+	.id_table	= gb_camera_id_table,
+	.driver.pm	= &gb_camera_pm_ops,
+};
+
+module_greybus_driver(gb_camera_driver);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/connection.c b/drivers/staging/greybus/connection.c
new file mode 100644
index 000000000000..557075147f2d
--- /dev/null
+++ b/drivers/staging/greybus/connection.c
@@ -0,0 +1,938 @@
+/*
+ * Greybus connections
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/workqueue.h>
+
+#include "greybus.h"
+#include "greybus_trace.h"
+
+
+#define GB_CONNECTION_CPORT_QUIESCE_TIMEOUT	1000
+
+
+static void gb_connection_kref_release(struct kref *kref);
+
+
+static DEFINE_SPINLOCK(gb_connections_lock);
+static DEFINE_MUTEX(gb_connection_mutex);
+
+
+/* Caller holds gb_connection_mutex. */
+static bool gb_connection_cport_in_use(struct gb_interface *intf, u16 cport_id)
+{
+	struct gb_host_device *hd = intf->hd;
+	struct gb_connection *connection;
+
+	list_for_each_entry(connection, &hd->connections, hd_links) {
+		if (connection->intf == intf &&
+				connection->intf_cport_id == cport_id)
+			return true;
+	}
+
+	return false;
+}
+
+static void gb_connection_get(struct gb_connection *connection)
+{
+	kref_get(&connection->kref);
+
+	trace_gb_connection_get(connection);
+}
+
+static void gb_connection_put(struct gb_connection *connection)
+{
+	trace_gb_connection_put(connection);
+
+	kref_put(&connection->kref, gb_connection_kref_release);
+}
+
+/*
+ * Returns a reference-counted pointer to the connection if found.
+ */
+static struct gb_connection *
+gb_connection_hd_find(struct gb_host_device *hd, u16 cport_id)
+{
+	struct gb_connection *connection;
+	unsigned long flags;
+
+	spin_lock_irqsave(&gb_connections_lock, flags);
+	list_for_each_entry(connection, &hd->connections, hd_links)
+		if (connection->hd_cport_id == cport_id) {
+			gb_connection_get(connection);
+			goto found;
+		}
+	connection = NULL;
+found:
+	spin_unlock_irqrestore(&gb_connections_lock, flags);
+
+	return connection;
+}
+
+/*
+ * Callback from the host driver to let us know that data has been
+ * received on the bundle.
+ */
+void greybus_data_rcvd(struct gb_host_device *hd, u16 cport_id,
+			u8 *data, size_t length)
+{
+	struct gb_connection *connection;
+
+	trace_gb_hd_in(hd);
+
+	connection = gb_connection_hd_find(hd, cport_id);
+	if (!connection) {
+		dev_err(&hd->dev,
+			"nonexistent connection (%zu bytes dropped)\n", length);
+		return;
+	}
+	gb_connection_recv(connection, data, length);
+	gb_connection_put(connection);
+}
+EXPORT_SYMBOL_GPL(greybus_data_rcvd);
+
+static void gb_connection_kref_release(struct kref *kref)
+{
+	struct gb_connection *connection;
+
+	connection = container_of(kref, struct gb_connection, kref);
+
+	trace_gb_connection_release(connection);
+
+	kfree(connection);
+}
+
+static void gb_connection_init_name(struct gb_connection *connection)
+{
+	u16 hd_cport_id = connection->hd_cport_id;
+	u16 cport_id = 0;
+	u8 intf_id = 0;
+
+	if (connection->intf) {
+		intf_id = connection->intf->interface_id;
+		cport_id = connection->intf_cport_id;
+	}
+
+	snprintf(connection->name, sizeof(connection->name),
+			"%u/%u:%u", hd_cport_id, intf_id, cport_id);
+}
+
+/*
+ * _gb_connection_create() - create a Greybus connection
+ * @hd:			host device of the connection
+ * @hd_cport_id:	host-device cport id, or -1 for dynamic allocation
+ * @intf:		remote interface, or NULL for static connections
+ * @bundle:		remote-interface bundle (may be NULL)
+ * @cport_id:		remote-interface cport id, or 0 for static connections
+ * @handler:		request handler (may be NULL)
+ * @flags:		connection flags
+ *
+ * Create a Greybus connection, representing the bidirectional link
+ * between a CPort on a (local) Greybus host device and a CPort on
+ * another Greybus interface.
+ *
+ * A connection also maintains the state of operations sent over the
+ * connection.
+ *
+ * Serialised against concurrent create and destroy using the
+ * gb_connection_mutex.
+ *
+ * Return: A pointer to the new connection if successful, or an ERR_PTR
+ * otherwise.
+ */
+static struct gb_connection *
+_gb_connection_create(struct gb_host_device *hd, int hd_cport_id,
+				struct gb_interface *intf,
+				struct gb_bundle *bundle, int cport_id,
+				gb_request_handler_t handler,
+				unsigned long flags)
+{
+	struct gb_connection *connection;
+	int ret;
+
+	mutex_lock(&gb_connection_mutex);
+
+	if (intf && gb_connection_cport_in_use(intf, cport_id)) {
+		dev_err(&intf->dev, "cport %u already in use\n", cport_id);
+		ret = -EBUSY;
+		goto err_unlock;
+	}
+
+	ret = gb_hd_cport_allocate(hd, hd_cport_id, flags);
+	if (ret < 0) {
+		dev_err(&hd->dev, "failed to allocate cport: %d\n", ret);
+		goto err_unlock;
+	}
+	hd_cport_id = ret;
+
+	connection = kzalloc(sizeof(*connection), GFP_KERNEL);
+	if (!connection) {
+		ret = -ENOMEM;
+		goto err_hd_cport_release;
+	}
+
+	connection->hd_cport_id = hd_cport_id;
+	connection->intf_cport_id = cport_id;
+	connection->hd = hd;
+	connection->intf = intf;
+	connection->bundle = bundle;
+	connection->handler = handler;
+	connection->flags = flags;
+	if (intf && (intf->quirks & GB_INTERFACE_QUIRK_NO_CPORT_FEATURES))
+		connection->flags |= GB_CONNECTION_FLAG_NO_FLOWCTRL;
+	connection->state = GB_CONNECTION_STATE_DISABLED;
+
+	atomic_set(&connection->op_cycle, 0);
+	mutex_init(&connection->mutex);
+	spin_lock_init(&connection->lock);
+	INIT_LIST_HEAD(&connection->operations);
+
+	connection->wq = alloc_workqueue("%s:%d", WQ_UNBOUND, 1,
+					 dev_name(&hd->dev), hd_cport_id);
+	if (!connection->wq) {
+		ret = -ENOMEM;
+		goto err_free_connection;
+	}
+
+	kref_init(&connection->kref);
+
+	gb_connection_init_name(connection);
+
+	spin_lock_irq(&gb_connections_lock);
+	list_add(&connection->hd_links, &hd->connections);
+
+	if (bundle)
+		list_add(&connection->bundle_links, &bundle->connections);
+	else
+		INIT_LIST_HEAD(&connection->bundle_links);
+
+	spin_unlock_irq(&gb_connections_lock);
+
+	mutex_unlock(&gb_connection_mutex);
+
+	trace_gb_connection_create(connection);
+
+	return connection;
+
+err_free_connection:
+	kfree(connection);
+err_hd_cport_release:
+	gb_hd_cport_release(hd, hd_cport_id);
+err_unlock:
+	mutex_unlock(&gb_connection_mutex);
+
+	return ERR_PTR(ret);
+}
+
+struct gb_connection *
+gb_connection_create_static(struct gb_host_device *hd, u16 hd_cport_id,
+					gb_request_handler_t handler)
+{
+	return _gb_connection_create(hd, hd_cport_id, NULL, NULL, 0, handler,
+					GB_CONNECTION_FLAG_HIGH_PRIO);
+}
+
+struct gb_connection *
+gb_connection_create_control(struct gb_interface *intf)
+{
+	return _gb_connection_create(intf->hd, -1, intf, NULL, 0, NULL,
+					GB_CONNECTION_FLAG_CONTROL |
+					GB_CONNECTION_FLAG_HIGH_PRIO);
+}
+
+struct gb_connection *
+gb_connection_create(struct gb_bundle *bundle, u16 cport_id,
+					gb_request_handler_t handler)
+{
+	struct gb_interface *intf = bundle->intf;
+
+	return _gb_connection_create(intf->hd, -1, intf, bundle, cport_id,
+					handler, 0);
+}
+EXPORT_SYMBOL_GPL(gb_connection_create);
+
+struct gb_connection *
+gb_connection_create_flags(struct gb_bundle *bundle, u16 cport_id,
+					gb_request_handler_t handler,
+					unsigned long flags)
+{
+	struct gb_interface *intf = bundle->intf;
+
+	if (WARN_ON_ONCE(flags & GB_CONNECTION_FLAG_CORE_MASK))
+		flags &= ~GB_CONNECTION_FLAG_CORE_MASK;
+
+	return _gb_connection_create(intf->hd, -1, intf, bundle, cport_id,
+					handler, flags);
+}
+EXPORT_SYMBOL_GPL(gb_connection_create_flags);
+
+struct gb_connection *
+gb_connection_create_offloaded(struct gb_bundle *bundle, u16 cport_id,
+					unsigned long flags)
+{
+	flags |= GB_CONNECTION_FLAG_OFFLOADED;
+
+	return gb_connection_create_flags(bundle, cport_id, NULL, flags);
+}
+EXPORT_SYMBOL_GPL(gb_connection_create_offloaded);
+
+static int gb_connection_hd_cport_enable(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+	int ret;
+
+	if (!hd->driver->cport_enable)
+		return 0;
+
+	ret = hd->driver->cport_enable(hd, connection->hd_cport_id,
+					connection->flags);
+	if (ret) {
+		dev_err(&hd->dev, "%s: failed to enable host cport: %d\n",
+				connection->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void gb_connection_hd_cport_disable(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+	int ret;
+
+	if (!hd->driver->cport_disable)
+		return;
+
+	ret = hd->driver->cport_disable(hd, connection->hd_cport_id);
+	if (ret) {
+		dev_err(&hd->dev, "%s: failed to disable host cport: %d\n",
+				connection->name, ret);
+	}
+}
+
+static int gb_connection_hd_cport_connected(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+	int ret;
+
+	if (!hd->driver->cport_connected)
+		return 0;
+
+	ret = hd->driver->cport_connected(hd, connection->hd_cport_id);
+	if (ret) {
+		dev_err(&hd->dev, "%s: failed to set connected state: %d\n",
+				connection->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int gb_connection_hd_cport_flush(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+	int ret;
+
+	if (!hd->driver->cport_flush)
+		return 0;
+
+	ret = hd->driver->cport_flush(hd, connection->hd_cport_id);
+	if (ret) {
+		dev_err(&hd->dev, "%s: failed to flush host cport: %d\n",
+				connection->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int gb_connection_hd_cport_quiesce(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+	size_t peer_space;
+	int ret;
+
+	peer_space = sizeof(struct gb_operation_msg_hdr) +
+			sizeof(struct gb_cport_shutdown_request);
+
+	if (connection->mode_switch)
+		peer_space += sizeof(struct gb_operation_msg_hdr);
+
+	ret = hd->driver->cport_quiesce(hd, connection->hd_cport_id,
+					peer_space,
+					GB_CONNECTION_CPORT_QUIESCE_TIMEOUT);
+	if (ret) {
+		dev_err(&hd->dev, "%s: failed to quiesce host cport: %d\n",
+				connection->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int gb_connection_hd_cport_clear(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+	int ret;
+
+	ret = hd->driver->cport_clear(hd, connection->hd_cport_id);
+	if (ret) {
+		dev_err(&hd->dev, "%s: failed to clear host cport: %d\n",
+				connection->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * Request the SVC to create a connection from AP's cport to interface's
+ * cport.
+ */
+static int
+gb_connection_svc_connection_create(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+	struct gb_interface *intf;
+	u8 cport_flags;
+	int ret;
+
+	if (gb_connection_is_static(connection))
+		return 0;
+
+	intf = connection->intf;
+
+	/*
+	 * Enable either E2EFC or CSD, unless no flow control is requested.
+	 */
+	cport_flags = GB_SVC_CPORT_FLAG_CSV_N;
+	if (gb_connection_flow_control_disabled(connection)) {
+		cport_flags |= GB_SVC_CPORT_FLAG_CSD_N;
+	} else if (gb_connection_e2efc_enabled(connection)) {
+		cport_flags |= GB_SVC_CPORT_FLAG_CSD_N |
+				GB_SVC_CPORT_FLAG_E2EFC;
+	}
+
+	ret = gb_svc_connection_create(hd->svc,
+			hd->svc->ap_intf_id,
+			connection->hd_cport_id,
+			intf->interface_id,
+			connection->intf_cport_id,
+			cport_flags);
+	if (ret) {
+		dev_err(&connection->hd->dev,
+			"%s: failed to create svc connection: %d\n",
+			connection->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void
+gb_connection_svc_connection_destroy(struct gb_connection *connection)
+{
+	if (gb_connection_is_static(connection))
+		return;
+
+	gb_svc_connection_destroy(connection->hd->svc,
+				  connection->hd->svc->ap_intf_id,
+				  connection->hd_cport_id,
+				  connection->intf->interface_id,
+				  connection->intf_cport_id);
+}
+
+/* Inform Interface about active CPorts */
+static int gb_connection_control_connected(struct gb_connection *connection)
+{
+	struct gb_control *control;
+	u16 cport_id = connection->intf_cport_id;
+	int ret;
+
+	if (gb_connection_is_static(connection))
+		return 0;
+
+	if (gb_connection_is_control(connection))
+		return 0;
+
+	control = connection->intf->control;
+
+	ret = gb_control_connected_operation(control, cport_id);
+	if (ret) {
+		dev_err(&connection->bundle->dev,
+			"failed to connect cport: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void
+gb_connection_control_disconnecting(struct gb_connection *connection)
+{
+	struct gb_control *control;
+	u16 cport_id = connection->intf_cport_id;
+	int ret;
+
+	if (gb_connection_is_static(connection))
+		return;
+
+	control = connection->intf->control;
+
+	ret = gb_control_disconnecting_operation(control, cport_id);
+	if (ret) {
+		dev_err(&connection->hd->dev,
+				"%s: failed to send disconnecting: %d\n",
+				connection->name, ret);
+	}
+}
+
+static void
+gb_connection_control_disconnected(struct gb_connection *connection)
+{
+	struct gb_control *control;
+	u16 cport_id = connection->intf_cport_id;
+	int ret;
+
+	if (gb_connection_is_static(connection))
+		return;
+
+	control = connection->intf->control;
+
+	if (gb_connection_is_control(connection)) {
+		if (connection->mode_switch) {
+			ret = gb_control_mode_switch_operation(control);
+			if (ret) {
+				/*
+				 * Allow mode switch to time out waiting for
+				 * mailbox event.
+				 */
+				return;
+			}
+		}
+
+		return;
+	}
+
+	ret = gb_control_disconnected_operation(control, cport_id);
+	if (ret) {
+		dev_warn(&connection->bundle->dev,
+			 "failed to disconnect cport: %d\n", ret);
+	}
+}
+
+static int gb_connection_shutdown_operation(struct gb_connection *connection,
+						u8 phase)
+{
+	struct gb_cport_shutdown_request *req;
+	struct gb_operation *operation;
+	int ret;
+
+	operation = gb_operation_create_core(connection,
+						GB_REQUEST_TYPE_CPORT_SHUTDOWN,
+						sizeof(*req), 0, 0,
+						GFP_KERNEL);
+	if (!operation)
+		return -ENOMEM;
+
+	req = operation->request->payload;
+	req->phase = phase;
+
+	ret = gb_operation_request_send_sync(operation);
+
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+static int gb_connection_cport_shutdown(struct gb_connection *connection,
+					u8 phase)
+{
+	struct gb_host_device *hd = connection->hd;
+	const struct gb_hd_driver *drv = hd->driver;
+	int ret;
+
+	if (gb_connection_is_static(connection))
+		return 0;
+
+	if (gb_connection_is_offloaded(connection)) {
+		if (!drv->cport_shutdown)
+			return 0;
+
+		ret = drv->cport_shutdown(hd, connection->hd_cport_id, phase,
+						GB_OPERATION_TIMEOUT_DEFAULT);
+	} else {
+		ret = gb_connection_shutdown_operation(connection, phase);
+	}
+
+	if (ret) {
+		dev_err(&hd->dev, "%s: failed to send cport shutdown (phase %d): %d\n",
+				connection->name, phase, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int
+gb_connection_cport_shutdown_phase_1(struct gb_connection *connection)
+{
+	return gb_connection_cport_shutdown(connection, 1);
+}
+
+static int
+gb_connection_cport_shutdown_phase_2(struct gb_connection *connection)
+{
+	return gb_connection_cport_shutdown(connection, 2);
+}
+
+/*
+ * Cancel all active operations on a connection.
+ *
+ * Locking: Called with connection lock held and state set to DISABLED or
+ * DISCONNECTING.
+ */
+static void gb_connection_cancel_operations(struct gb_connection *connection,
+						int errno)
+	__must_hold(&connection->lock)
+{
+	struct gb_operation *operation;
+
+	while (!list_empty(&connection->operations)) {
+		operation = list_last_entry(&connection->operations,
+						struct gb_operation, links);
+		gb_operation_get(operation);
+		spin_unlock_irq(&connection->lock);
+
+		if (gb_operation_is_incoming(operation))
+			gb_operation_cancel_incoming(operation, errno);
+		else
+			gb_operation_cancel(operation, errno);
+
+		gb_operation_put(operation);
+
+		spin_lock_irq(&connection->lock);
+	}
+}
+
+/*
+ * Cancel all active incoming operations on a connection.
+ *
+ * Locking: Called with connection lock held and state set to ENABLED_TX.
+ */
+static void
+gb_connection_flush_incoming_operations(struct gb_connection *connection,
+						int errno)
+	__must_hold(&connection->lock)
+{
+	struct gb_operation *operation;
+	bool incoming;
+
+	while (!list_empty(&connection->operations)) {
+		incoming = false;
+		list_for_each_entry(operation, &connection->operations,
+								links) {
+			if (gb_operation_is_incoming(operation)) {
+				gb_operation_get(operation);
+				incoming = true;
+				break;
+			}
+		}
+
+		if (!incoming)
+			break;
+
+		spin_unlock_irq(&connection->lock);
+
+		/* FIXME: flush, not cancel? */
+		gb_operation_cancel_incoming(operation, errno);
+		gb_operation_put(operation);
+
+		spin_lock_irq(&connection->lock);
+	}
+}
+
+/*
+ * _gb_connection_enable() - enable a connection
+ * @connection:		connection to enable
+ * @rx:			whether to enable incoming requests
+ *
+ * Connection-enable helper for DISABLED->ENABLED, DISABLED->ENABLED_TX, and
+ * ENABLED_TX->ENABLED state transitions.
+ *
+ * Locking: Caller holds connection->mutex.
+ */
+static int _gb_connection_enable(struct gb_connection *connection, bool rx)
+{
+	int ret;
+
+	/* Handle ENABLED_TX -> ENABLED transitions. */
+	if (connection->state == GB_CONNECTION_STATE_ENABLED_TX) {
+		if (!(connection->handler && rx))
+			return 0;
+
+		spin_lock_irq(&connection->lock);
+		connection->state = GB_CONNECTION_STATE_ENABLED;
+		spin_unlock_irq(&connection->lock);
+
+		return 0;
+	}
+
+	ret = gb_connection_hd_cport_enable(connection);
+	if (ret)
+		return ret;
+
+	ret = gb_connection_svc_connection_create(connection);
+	if (ret)
+		goto err_hd_cport_clear;
+
+	ret = gb_connection_hd_cport_connected(connection);
+	if (ret)
+		goto err_svc_connection_destroy;
+
+	spin_lock_irq(&connection->lock);
+	if (connection->handler && rx)
+		connection->state = GB_CONNECTION_STATE_ENABLED;
+	else
+		connection->state = GB_CONNECTION_STATE_ENABLED_TX;
+	spin_unlock_irq(&connection->lock);
+
+	ret = gb_connection_control_connected(connection);
+	if (ret)
+		goto err_control_disconnecting;
+
+	return 0;
+
+err_control_disconnecting:
+	spin_lock_irq(&connection->lock);
+	connection->state = GB_CONNECTION_STATE_DISCONNECTING;
+	gb_connection_cancel_operations(connection, -ESHUTDOWN);
+	spin_unlock_irq(&connection->lock);
+
+	/* Transmit queue should already be empty. */
+	gb_connection_hd_cport_flush(connection);
+
+	gb_connection_control_disconnecting(connection);
+	gb_connection_cport_shutdown_phase_1(connection);
+	gb_connection_hd_cport_quiesce(connection);
+	gb_connection_cport_shutdown_phase_2(connection);
+	gb_connection_control_disconnected(connection);
+	connection->state = GB_CONNECTION_STATE_DISABLED;
+err_svc_connection_destroy:
+	gb_connection_svc_connection_destroy(connection);
+err_hd_cport_clear:
+	gb_connection_hd_cport_clear(connection);
+
+	gb_connection_hd_cport_disable(connection);
+
+	return ret;
+}
+
+int gb_connection_enable(struct gb_connection *connection)
+{
+	int ret = 0;
+
+	mutex_lock(&connection->mutex);
+
+	if (connection->state == GB_CONNECTION_STATE_ENABLED)
+		goto out_unlock;
+
+	ret = _gb_connection_enable(connection, true);
+	if (!ret)
+		trace_gb_connection_enable(connection);
+
+out_unlock:
+	mutex_unlock(&connection->mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_connection_enable);
+
+int gb_connection_enable_tx(struct gb_connection *connection)
+{
+	int ret = 0;
+
+	mutex_lock(&connection->mutex);
+
+	if (connection->state == GB_CONNECTION_STATE_ENABLED) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	if (connection->state == GB_CONNECTION_STATE_ENABLED_TX)
+		goto out_unlock;
+
+	ret = _gb_connection_enable(connection, false);
+	if (!ret)
+		trace_gb_connection_enable(connection);
+
+out_unlock:
+	mutex_unlock(&connection->mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_connection_enable_tx);
+
+void gb_connection_disable_rx(struct gb_connection *connection)
+{
+	mutex_lock(&connection->mutex);
+
+	spin_lock_irq(&connection->lock);
+	if (connection->state != GB_CONNECTION_STATE_ENABLED) {
+		spin_unlock_irq(&connection->lock);
+		goto out_unlock;
+	}
+	connection->state = GB_CONNECTION_STATE_ENABLED_TX;
+	gb_connection_flush_incoming_operations(connection, -ESHUTDOWN);
+	spin_unlock_irq(&connection->lock);
+
+	trace_gb_connection_disable(connection);
+
+out_unlock:
+	mutex_unlock(&connection->mutex);
+}
+EXPORT_SYMBOL_GPL(gb_connection_disable_rx);
+
+void gb_connection_mode_switch_prepare(struct gb_connection *connection)
+{
+	connection->mode_switch = true;
+}
+
+void gb_connection_mode_switch_complete(struct gb_connection *connection)
+{
+	gb_connection_svc_connection_destroy(connection);
+	gb_connection_hd_cport_clear(connection);
+
+	gb_connection_hd_cport_disable(connection);
+
+	connection->mode_switch = false;
+}
+
+void gb_connection_disable(struct gb_connection *connection)
+{
+	mutex_lock(&connection->mutex);
+
+	if (connection->state == GB_CONNECTION_STATE_DISABLED)
+		goto out_unlock;
+
+	trace_gb_connection_disable(connection);
+
+	spin_lock_irq(&connection->lock);
+	connection->state = GB_CONNECTION_STATE_DISCONNECTING;
+	gb_connection_cancel_operations(connection, -ESHUTDOWN);
+	spin_unlock_irq(&connection->lock);
+
+	gb_connection_hd_cport_flush(connection);
+
+	gb_connection_control_disconnecting(connection);
+	gb_connection_cport_shutdown_phase_1(connection);
+	gb_connection_hd_cport_quiesce(connection);
+	gb_connection_cport_shutdown_phase_2(connection);
+	gb_connection_control_disconnected(connection);
+
+	connection->state = GB_CONNECTION_STATE_DISABLED;
+
+	/* control-connection tear down is deferred when mode switching */
+	if (!connection->mode_switch) {
+		gb_connection_svc_connection_destroy(connection);
+		gb_connection_hd_cport_clear(connection);
+
+		gb_connection_hd_cport_disable(connection);
+	}
+
+out_unlock:
+	mutex_unlock(&connection->mutex);
+}
+EXPORT_SYMBOL_GPL(gb_connection_disable);
+
+/* Disable a connection without communicating with the remote end. */
+void gb_connection_disable_forced(struct gb_connection *connection)
+{
+	mutex_lock(&connection->mutex);
+
+	if (connection->state == GB_CONNECTION_STATE_DISABLED)
+		goto out_unlock;
+
+	trace_gb_connection_disable(connection);
+
+	spin_lock_irq(&connection->lock);
+	connection->state = GB_CONNECTION_STATE_DISABLED;
+	gb_connection_cancel_operations(connection, -ESHUTDOWN);
+	spin_unlock_irq(&connection->lock);
+
+	gb_connection_hd_cport_flush(connection);
+
+	gb_connection_svc_connection_destroy(connection);
+	gb_connection_hd_cport_clear(connection);
+
+	gb_connection_hd_cport_disable(connection);
+out_unlock:
+	mutex_unlock(&connection->mutex);
+}
+EXPORT_SYMBOL_GPL(gb_connection_disable_forced);
+
+/* Caller must have disabled the connection before destroying it. */
+void gb_connection_destroy(struct gb_connection *connection)
+{
+	if (!connection)
+		return;
+
+	if (WARN_ON(connection->state != GB_CONNECTION_STATE_DISABLED))
+		gb_connection_disable(connection);
+
+	mutex_lock(&gb_connection_mutex);
+
+	spin_lock_irq(&gb_connections_lock);
+	list_del(&connection->bundle_links);
+	list_del(&connection->hd_links);
+	spin_unlock_irq(&gb_connections_lock);
+
+	destroy_workqueue(connection->wq);
+
+	gb_hd_cport_release(connection->hd, connection->hd_cport_id);
+	connection->hd_cport_id = CPORT_ID_BAD;
+
+	mutex_unlock(&gb_connection_mutex);
+
+	gb_connection_put(connection);
+}
+EXPORT_SYMBOL_GPL(gb_connection_destroy);
+
+void gb_connection_latency_tag_enable(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+	int ret;
+
+	if (!hd->driver->latency_tag_enable)
+		return;
+
+	ret = hd->driver->latency_tag_enable(hd, connection->hd_cport_id);
+	if (ret) {
+		dev_err(&connection->hd->dev,
+			"%s: failed to enable latency tag: %d\n",
+			connection->name, ret);
+	}
+}
+EXPORT_SYMBOL_GPL(gb_connection_latency_tag_enable);
+
+void gb_connection_latency_tag_disable(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+	int ret;
+
+	if (!hd->driver->latency_tag_disable)
+		return;
+
+	ret = hd->driver->latency_tag_disable(hd, connection->hd_cport_id);
+	if (ret) {
+		dev_err(&connection->hd->dev,
+			"%s: failed to disable latency tag: %d\n",
+			connection->name, ret);
+	}
+}
+EXPORT_SYMBOL_GPL(gb_connection_latency_tag_disable);
diff --git a/drivers/staging/greybus/connection.h b/drivers/staging/greybus/connection.h
new file mode 100644
index 000000000000..4d9f4c64176c
--- /dev/null
+++ b/drivers/staging/greybus/connection.h
@@ -0,0 +1,129 @@
+/*
+ * Greybus connections
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __CONNECTION_H
+#define __CONNECTION_H
+
+#include <linux/list.h>
+#include <linux/kfifo.h>
+
+#define GB_CONNECTION_FLAG_CSD		BIT(0)
+#define GB_CONNECTION_FLAG_NO_FLOWCTRL	BIT(1)
+#define GB_CONNECTION_FLAG_OFFLOADED	BIT(2)
+#define GB_CONNECTION_FLAG_CDSI1	BIT(3)
+#define GB_CONNECTION_FLAG_CONTROL	BIT(4)
+#define GB_CONNECTION_FLAG_HIGH_PRIO	BIT(5)
+
+#define GB_CONNECTION_FLAG_CORE_MASK	GB_CONNECTION_FLAG_CONTROL
+
+enum gb_connection_state {
+	GB_CONNECTION_STATE_DISABLED		= 0,
+	GB_CONNECTION_STATE_ENABLED_TX		= 1,
+	GB_CONNECTION_STATE_ENABLED		= 2,
+	GB_CONNECTION_STATE_DISCONNECTING	= 3,
+};
+
+struct gb_operation;
+
+typedef int (*gb_request_handler_t)(struct gb_operation *);
+
+struct gb_connection {
+	struct gb_host_device		*hd;
+	struct gb_interface		*intf;
+	struct gb_bundle		*bundle;
+	struct kref			kref;
+	u16				hd_cport_id;
+	u16				intf_cport_id;
+
+	struct list_head		hd_links;
+	struct list_head		bundle_links;
+
+	gb_request_handler_t		handler;
+	unsigned long			flags;
+
+	struct mutex			mutex;
+	spinlock_t			lock;
+	enum gb_connection_state	state;
+	struct list_head		operations;
+
+	char				name[16];
+	struct workqueue_struct		*wq;
+
+	atomic_t			op_cycle;
+
+	void				*private;
+
+	bool				mode_switch;
+};
+
+struct gb_connection *gb_connection_create_static(struct gb_host_device *hd,
+				u16 hd_cport_id, gb_request_handler_t handler);
+struct gb_connection *gb_connection_create_control(struct gb_interface *intf);
+struct gb_connection *gb_connection_create(struct gb_bundle *bundle,
+				u16 cport_id, gb_request_handler_t handler);
+struct gb_connection *gb_connection_create_flags(struct gb_bundle *bundle,
+				u16 cport_id, gb_request_handler_t handler,
+				unsigned long flags);
+struct gb_connection *gb_connection_create_offloaded(struct gb_bundle *bundle,
+				u16 cport_id, unsigned long flags);
+void gb_connection_destroy(struct gb_connection *connection);
+
+static inline bool gb_connection_is_static(struct gb_connection *connection)
+{
+	return !connection->intf;
+}
+
+int gb_connection_enable(struct gb_connection *connection);
+int gb_connection_enable_tx(struct gb_connection *connection);
+void gb_connection_disable_rx(struct gb_connection *connection);
+void gb_connection_disable(struct gb_connection *connection);
+void gb_connection_disable_forced(struct gb_connection *connection);
+
+void gb_connection_mode_switch_prepare(struct gb_connection *connection);
+void gb_connection_mode_switch_complete(struct gb_connection *connection);
+
+void greybus_data_rcvd(struct gb_host_device *hd, u16 cport_id,
+			u8 *data, size_t length);
+
+void gb_connection_latency_tag_enable(struct gb_connection *connection);
+void gb_connection_latency_tag_disable(struct gb_connection *connection);
+
+static inline bool gb_connection_e2efc_enabled(struct gb_connection *connection)
+{
+	return !(connection->flags & GB_CONNECTION_FLAG_CSD);
+}
+
+static inline bool
+gb_connection_flow_control_disabled(struct gb_connection *connection)
+{
+	return connection->flags & GB_CONNECTION_FLAG_NO_FLOWCTRL;
+}
+
+static inline bool gb_connection_is_offloaded(struct gb_connection *connection)
+{
+	return connection->flags & GB_CONNECTION_FLAG_OFFLOADED;
+}
+
+static inline bool gb_connection_is_control(struct gb_connection *connection)
+{
+	return connection->flags & GB_CONNECTION_FLAG_CONTROL;
+}
+
+static inline void *gb_connection_get_data(struct gb_connection *connection)
+{
+	return connection->private;
+}
+
+static inline void gb_connection_set_data(struct gb_connection *connection,
+					  void *data)
+{
+	connection->private = data;
+}
+
+#endif /* __CONNECTION_H */
diff --git a/drivers/staging/greybus/control.c b/drivers/staging/greybus/control.c
new file mode 100644
index 000000000000..4716190e740a
--- /dev/null
+++ b/drivers/staging/greybus/control.c
@@ -0,0 +1,635 @@
+/*
+ * Greybus CPort control protocol.
+ *
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include "greybus.h"
+
+/* Highest control-protocol version supported */
+#define GB_CONTROL_VERSION_MAJOR	0
+#define GB_CONTROL_VERSION_MINOR	1
+
+
+static int gb_control_get_version(struct gb_control *control)
+{
+	struct gb_interface *intf = control->connection->intf;
+	struct gb_control_version_request request;
+	struct gb_control_version_response response;
+	int ret;
+
+	request.major = GB_CONTROL_VERSION_MAJOR;
+	request.minor = GB_CONTROL_VERSION_MINOR;
+
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_VERSION,
+				&request, sizeof(request), &response,
+				sizeof(response));
+	if (ret) {
+		dev_err(&intf->dev,
+				"failed to get control-protocol version: %d\n",
+				ret);
+		return ret;
+	}
+
+	if (response.major > request.major) {
+		dev_err(&intf->dev,
+				"unsupported major control-protocol version (%u > %u)\n",
+				response.major, request.major);
+		return -ENOTSUPP;
+	}
+
+	control->protocol_major = response.major;
+	control->protocol_minor = response.minor;
+
+	dev_dbg(&intf->dev, "%s - %u.%u\n", __func__, response.major,
+			response.minor);
+
+	return 0;
+}
+
+static int gb_control_get_bundle_version(struct gb_control *control,
+						struct gb_bundle *bundle)
+{
+	struct gb_interface *intf = control->connection->intf;
+	struct gb_control_bundle_version_request request;
+	struct gb_control_bundle_version_response response;
+	int ret;
+
+	request.bundle_id = bundle->id;
+
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_BUNDLE_VERSION,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(&intf->dev,
+				"failed to get bundle %u class version: %d\n",
+				bundle->id, ret);
+		return ret;
+	}
+
+	bundle->class_major = response.major;
+	bundle->class_minor = response.minor;
+
+	dev_dbg(&intf->dev, "%s - %u: %u.%u\n", __func__, bundle->id,
+			response.major, response.minor);
+
+	return 0;
+}
+
+int gb_control_get_bundle_versions(struct gb_control *control)
+{
+	struct gb_interface *intf = control->connection->intf;
+	struct gb_bundle *bundle;
+	int ret;
+
+	if (!control->has_bundle_version)
+		return 0;
+
+	list_for_each_entry(bundle, &intf->bundles, links) {
+		ret = gb_control_get_bundle_version(control, bundle);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/* Get Manifest's size from the interface */
+int gb_control_get_manifest_size_operation(struct gb_interface *intf)
+{
+	struct gb_control_get_manifest_size_response response;
+	struct gb_connection *connection = intf->control->connection;
+	int ret;
+
+	ret = gb_operation_sync(connection, GB_CONTROL_TYPE_GET_MANIFEST_SIZE,
+				NULL, 0, &response, sizeof(response));
+	if (ret) {
+		dev_err(&connection->intf->dev,
+				"failed to get manifest size: %d\n", ret);
+		return ret;
+	}
+
+	return le16_to_cpu(response.size);
+}
+
+/* Reads Manifest from the interface */
+int gb_control_get_manifest_operation(struct gb_interface *intf, void *manifest,
+				      size_t size)
+{
+	struct gb_connection *connection = intf->control->connection;
+
+	return gb_operation_sync(connection, GB_CONTROL_TYPE_GET_MANIFEST,
+				NULL, 0, manifest, size);
+}
+
+int gb_control_connected_operation(struct gb_control *control, u16 cport_id)
+{
+	struct gb_control_connected_request request;
+
+	request.cport_id = cpu_to_le16(cport_id);
+	return gb_operation_sync(control->connection, GB_CONTROL_TYPE_CONNECTED,
+				 &request, sizeof(request), NULL, 0);
+}
+
+int gb_control_disconnected_operation(struct gb_control *control, u16 cport_id)
+{
+	struct gb_control_disconnected_request request;
+
+	request.cport_id = cpu_to_le16(cport_id);
+	return gb_operation_sync(control->connection,
+				 GB_CONTROL_TYPE_DISCONNECTED, &request,
+				 sizeof(request), NULL, 0);
+}
+
+int gb_control_disconnecting_operation(struct gb_control *control,
+					u16 cport_id)
+{
+	struct gb_control_disconnecting_request *request;
+	struct gb_operation *operation;
+	int ret;
+
+	operation = gb_operation_create_core(control->connection,
+					GB_CONTROL_TYPE_DISCONNECTING,
+					sizeof(*request), 0, 0,
+					GFP_KERNEL);
+	if (!operation)
+		return -ENOMEM;
+
+	request = operation->request->payload;
+	request->cport_id = cpu_to_le16(cport_id);
+
+	ret = gb_operation_request_send_sync(operation);
+	if (ret) {
+		dev_err(&control->dev, "failed to send disconnecting: %d\n",
+				ret);
+	}
+
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+int gb_control_mode_switch_operation(struct gb_control *control)
+{
+	struct gb_operation *operation;
+	int ret;
+
+	operation = gb_operation_create_core(control->connection,
+					GB_CONTROL_TYPE_MODE_SWITCH,
+					0, 0, GB_OPERATION_FLAG_UNIDIRECTIONAL,
+					GFP_KERNEL);
+	if (!operation)
+		return -ENOMEM;
+
+	ret = gb_operation_request_send_sync(operation);
+	if (ret)
+		dev_err(&control->dev, "failed to send mode switch: %d\n", ret);
+
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+int gb_control_timesync_enable(struct gb_control *control, u8 count,
+			       u64 frame_time, u32 strobe_delay, u32 refclk)
+{
+	struct gb_control_timesync_enable_request request;
+
+	request.count = count;
+	request.frame_time = cpu_to_le64(frame_time);
+	request.strobe_delay = cpu_to_le32(strobe_delay);
+	request.refclk = cpu_to_le32(refclk);
+	return gb_operation_sync(control->connection,
+				 GB_CONTROL_TYPE_TIMESYNC_ENABLE, &request,
+				 sizeof(request), NULL, 0);
+}
+
+int gb_control_timesync_disable(struct gb_control *control)
+{
+	return gb_operation_sync(control->connection,
+				 GB_CONTROL_TYPE_TIMESYNC_DISABLE, NULL, 0,
+				 NULL, 0);
+}
+
+int gb_control_timesync_get_last_event(struct gb_control *control,
+				       u64 *frame_time)
+{
+	struct gb_control_timesync_get_last_event_response response;
+	int ret;
+
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_TIMESYNC_GET_LAST_EVENT,
+				NULL, 0, &response, sizeof(response));
+	if (!ret)
+		*frame_time = le64_to_cpu(response.frame_time);
+	return ret;
+}
+
+int gb_control_timesync_authoritative(struct gb_control *control,
+				      u64 *frame_time)
+{
+	struct gb_control_timesync_authoritative_request request;
+	int i;
+
+	for (i = 0; i < GB_TIMESYNC_MAX_STROBES; i++)
+		request.frame_time[i] = cpu_to_le64(frame_time[i]);
+
+	return gb_operation_sync(control->connection,
+				 GB_CONTROL_TYPE_TIMESYNC_AUTHORITATIVE,
+				 &request, sizeof(request),
+				 NULL, 0);
+}
+
+static int gb_control_bundle_pm_status_map(u8 status)
+{
+	switch (status) {
+	case GB_CONTROL_BUNDLE_PM_INVAL:
+		return -EINVAL;
+	case GB_CONTROL_BUNDLE_PM_BUSY:
+		return -EBUSY;
+	case GB_CONTROL_BUNDLE_PM_NA:
+		return -ENOMSG;
+	case GB_CONTROL_BUNDLE_PM_FAIL:
+	default:
+		return -EREMOTEIO;
+	}
+}
+
+int gb_control_bundle_suspend(struct gb_control *control, u8 bundle_id)
+{
+	struct gb_control_bundle_pm_request request;
+	struct gb_control_bundle_pm_response response;
+	int ret;
+
+	request.bundle_id = bundle_id;
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_BUNDLE_SUSPEND, &request,
+				sizeof(request), &response, sizeof(response));
+	if (ret) {
+		dev_err(&control->dev, "failed to send bundle %u suspend: %d\n",
+			bundle_id, ret);
+		return ret;
+	}
+
+	if (response.status != GB_CONTROL_BUNDLE_PM_OK) {
+		dev_err(&control->dev, "failed to suspend bundle %u: %d\n",
+			bundle_id, response.status);
+		return gb_control_bundle_pm_status_map(response.status);
+	}
+
+	return 0;
+}
+
+int gb_control_bundle_resume(struct gb_control *control, u8 bundle_id)
+{
+	struct gb_control_bundle_pm_request request;
+	struct gb_control_bundle_pm_response response;
+	int ret;
+
+	request.bundle_id = bundle_id;
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_BUNDLE_RESUME, &request,
+				sizeof(request), &response, sizeof(response));
+	if (ret) {
+		dev_err(&control->dev, "failed to send bundle %u resume: %d\n",
+			bundle_id, ret);
+		return ret;
+	}
+
+	if (response.status != GB_CONTROL_BUNDLE_PM_OK) {
+		dev_err(&control->dev, "failed to resume bundle %u: %d\n",
+			bundle_id, response.status);
+		return gb_control_bundle_pm_status_map(response.status);
+	}
+
+	return 0;
+}
+
+int gb_control_bundle_deactivate(struct gb_control *control, u8 bundle_id)
+{
+	struct gb_control_bundle_pm_request request;
+	struct gb_control_bundle_pm_response response;
+	int ret;
+
+	request.bundle_id = bundle_id;
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_BUNDLE_DEACTIVATE, &request,
+				sizeof(request), &response, sizeof(response));
+	if (ret) {
+		dev_err(&control->dev,
+			"failed to send bundle %u deactivate: %d\n", bundle_id,
+			ret);
+		return ret;
+	}
+
+	if (response.status != GB_CONTROL_BUNDLE_PM_OK) {
+		dev_err(&control->dev, "failed to deactivate bundle %u: %d\n",
+			bundle_id, response.status);
+		return gb_control_bundle_pm_status_map(response.status);
+	}
+
+	return 0;
+}
+
+int gb_control_bundle_activate(struct gb_control *control, u8 bundle_id)
+{
+	struct gb_control_bundle_pm_request request;
+	struct gb_control_bundle_pm_response response;
+	int ret;
+
+	if (!control->has_bundle_activate)
+		return 0;
+
+	request.bundle_id = bundle_id;
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_BUNDLE_ACTIVATE, &request,
+				sizeof(request), &response, sizeof(response));
+	if (ret) {
+		dev_err(&control->dev,
+			"failed to send bundle %u activate: %d\n", bundle_id,
+			ret);
+		return ret;
+	}
+
+	if (response.status != GB_CONTROL_BUNDLE_PM_OK) {
+		dev_err(&control->dev, "failed to activate bundle %u: %d\n",
+			bundle_id, response.status);
+		return gb_control_bundle_pm_status_map(response.status);
+	}
+
+	return 0;
+}
+
+static int gb_control_interface_pm_status_map(u8 status)
+{
+	switch (status) {
+	case GB_CONTROL_INTF_PM_BUSY:
+		return -EBUSY;
+	case GB_CONTROL_INTF_PM_NA:
+		return -ENOMSG;
+	default:
+		return -EREMOTEIO;
+	}
+}
+
+int gb_control_interface_suspend_prepare(struct gb_control *control)
+{
+	struct gb_control_intf_pm_response response;
+	int ret;
+
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_INTF_SUSPEND_PREPARE, NULL, 0,
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(&control->dev,
+			"failed to send interface suspend prepare: %d\n", ret);
+		return ret;
+	}
+
+	if (response.status != GB_CONTROL_INTF_PM_OK) {
+		dev_err(&control->dev, "interface error while preparing suspend: %d\n",
+			response.status);
+		return gb_control_interface_pm_status_map(response.status);
+	}
+
+	return 0;
+}
+
+int gb_control_interface_deactivate_prepare(struct gb_control *control)
+{
+	struct gb_control_intf_pm_response response;
+	int ret;
+
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_INTF_DEACTIVATE_PREPARE, NULL,
+				0, &response, sizeof(response));
+	if (ret) {
+		dev_err(&control->dev, "failed to send interface deactivate prepare: %d\n",
+			ret);
+		return ret;
+	}
+
+	if (response.status != GB_CONTROL_INTF_PM_OK) {
+		dev_err(&control->dev, "interface error while preparing deactivate: %d\n",
+			response.status);
+		return gb_control_interface_pm_status_map(response.status);
+	}
+
+	return 0;
+}
+
+int gb_control_interface_hibernate_abort(struct gb_control *control)
+{
+	struct gb_control_intf_pm_response response;
+	int ret;
+
+	ret = gb_operation_sync(control->connection,
+				GB_CONTROL_TYPE_INTF_HIBERNATE_ABORT, NULL, 0,
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(&control->dev,
+			"failed to send interface aborting hibernate: %d\n",
+			ret);
+		return ret;
+	}
+
+	if (response.status != GB_CONTROL_INTF_PM_OK) {
+		dev_err(&control->dev, "interface error while aborting hibernate: %d\n",
+			response.status);
+		return gb_control_interface_pm_status_map(response.status);
+	}
+
+	return 0;
+}
+
+static ssize_t vendor_string_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	struct gb_control *control = to_gb_control(dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%s\n", control->vendor_string);
+}
+static DEVICE_ATTR_RO(vendor_string);
+
+static ssize_t product_string_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	struct gb_control *control = to_gb_control(dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%s\n", control->product_string);
+}
+static DEVICE_ATTR_RO(product_string);
+
+static struct attribute *control_attrs[] = {
+	&dev_attr_vendor_string.attr,
+	&dev_attr_product_string.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(control);
+
+static void gb_control_release(struct device *dev)
+{
+	struct gb_control *control = to_gb_control(dev);
+
+	gb_connection_destroy(control->connection);
+
+	kfree(control->vendor_string);
+	kfree(control->product_string);
+
+	kfree(control);
+}
+
+struct device_type greybus_control_type = {
+	.name =		"greybus_control",
+	.release =	gb_control_release,
+};
+
+struct gb_control *gb_control_create(struct gb_interface *intf)
+{
+	struct gb_connection *connection;
+	struct gb_control *control;
+
+	control = kzalloc(sizeof(*control), GFP_KERNEL);
+	if (!control)
+		return ERR_PTR(-ENOMEM);
+
+	control->intf = intf;
+
+	connection = gb_connection_create_control(intf);
+	if (IS_ERR(connection)) {
+		dev_err(&intf->dev,
+				"failed to create control connection: %ld\n",
+				PTR_ERR(connection));
+		kfree(control);
+		return ERR_CAST(connection);
+	}
+
+	control->connection = connection;
+
+	control->dev.parent = &intf->dev;
+	control->dev.bus = &greybus_bus_type;
+	control->dev.type = &greybus_control_type;
+	control->dev.groups = control_groups;
+	control->dev.dma_mask = intf->dev.dma_mask;
+	device_initialize(&control->dev);
+	dev_set_name(&control->dev, "%s.ctrl", dev_name(&intf->dev));
+
+	gb_connection_set_data(control->connection, control);
+
+	return control;
+}
+
+int gb_control_enable(struct gb_control *control)
+{
+	int ret;
+
+	dev_dbg(&control->connection->intf->dev, "%s\n", __func__);
+
+	ret = gb_connection_enable_tx(control->connection);
+	if (ret) {
+		dev_err(&control->connection->intf->dev,
+				"failed to enable control connection: %d\n",
+				ret);
+		return ret;
+	}
+
+	ret = gb_control_get_version(control);
+	if (ret)
+		goto err_disable_connection;
+
+	if (control->protocol_major > 0 || control->protocol_minor > 1)
+		control->has_bundle_version = true;
+
+	/* FIXME: use protocol version instead */
+	if (!(control->intf->quirks & GB_INTERFACE_QUIRK_NO_BUNDLE_ACTIVATE))
+		control->has_bundle_activate = true;
+
+	return 0;
+
+err_disable_connection:
+	gb_connection_disable(control->connection);
+
+	return ret;
+}
+
+void gb_control_disable(struct gb_control *control)
+{
+	dev_dbg(&control->connection->intf->dev, "%s\n", __func__);
+
+	if (control->intf->disconnected)
+		gb_connection_disable_forced(control->connection);
+	else
+		gb_connection_disable(control->connection);
+}
+
+int gb_control_suspend(struct gb_control *control)
+{
+	gb_connection_disable(control->connection);
+
+	return 0;
+}
+
+int gb_control_resume(struct gb_control *control)
+{
+	int ret;
+
+	ret = gb_connection_enable_tx(control->connection);
+	if (ret) {
+		dev_err(&control->connection->intf->dev,
+			"failed to enable control connection: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int gb_control_add(struct gb_control *control)
+{
+	int ret;
+
+	ret = device_add(&control->dev);
+	if (ret) {
+		dev_err(&control->dev,
+				"failed to register control device: %d\n",
+				ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+void gb_control_del(struct gb_control *control)
+{
+	if (device_is_registered(&control->dev))
+		device_del(&control->dev);
+}
+
+struct gb_control *gb_control_get(struct gb_control *control)
+{
+	get_device(&control->dev);
+
+	return control;
+}
+
+void gb_control_put(struct gb_control *control)
+{
+	put_device(&control->dev);
+}
+
+void gb_control_mode_switch_prepare(struct gb_control *control)
+{
+	gb_connection_mode_switch_prepare(control->connection);
+}
+
+void gb_control_mode_switch_complete(struct gb_control *control)
+{
+	gb_connection_mode_switch_complete(control->connection);
+}
diff --git a/drivers/staging/greybus/control.h b/drivers/staging/greybus/control.h
new file mode 100644
index 000000000000..f9a60daf9a72
--- /dev/null
+++ b/drivers/staging/greybus/control.h
@@ -0,0 +1,65 @@
+/*
+ * Greybus CPort control protocol
+ *
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __CONTROL_H
+#define __CONTROL_H
+
+struct gb_control {
+	struct device dev;
+	struct gb_interface *intf;
+
+	struct gb_connection *connection;
+
+	u8 protocol_major;
+	u8 protocol_minor;
+
+	bool has_bundle_activate;
+	bool has_bundle_version;
+
+	char *vendor_string;
+	char *product_string;
+};
+#define to_gb_control(d) container_of(d, struct gb_control, dev)
+
+struct gb_control *gb_control_create(struct gb_interface *intf);
+int gb_control_enable(struct gb_control *control);
+void gb_control_disable(struct gb_control *control);
+int gb_control_suspend(struct gb_control *control);
+int gb_control_resume(struct gb_control *control);
+int gb_control_add(struct gb_control *control);
+void gb_control_del(struct gb_control *control);
+struct gb_control *gb_control_get(struct gb_control *control);
+void gb_control_put(struct gb_control *control);
+
+int gb_control_get_bundle_versions(struct gb_control *control);
+int gb_control_connected_operation(struct gb_control *control, u16 cport_id);
+int gb_control_disconnected_operation(struct gb_control *control, u16 cport_id);
+int gb_control_disconnecting_operation(struct gb_control *control,
+					u16 cport_id);
+int gb_control_mode_switch_operation(struct gb_control *control);
+void gb_control_mode_switch_prepare(struct gb_control *control);
+void gb_control_mode_switch_complete(struct gb_control *control);
+int gb_control_get_manifest_size_operation(struct gb_interface *intf);
+int gb_control_get_manifest_operation(struct gb_interface *intf, void *manifest,
+				      size_t size);
+int gb_control_timesync_enable(struct gb_control *control, u8 count,
+			       u64 frame_time, u32 strobe_delay, u32 refclk);
+int gb_control_timesync_disable(struct gb_control *control);
+int gb_control_timesync_get_last_event(struct gb_control *control,
+				       u64 *frame_time);
+int gb_control_timesync_authoritative(struct gb_control *control,
+				      u64 *frame_time);
+int gb_control_bundle_suspend(struct gb_control *control, u8 bundle_id);
+int gb_control_bundle_resume(struct gb_control *control, u8 bundle_id);
+int gb_control_bundle_deactivate(struct gb_control *control, u8 bundle_id);
+int gb_control_bundle_activate(struct gb_control *control, u8 bundle_id);
+int gb_control_interface_suspend_prepare(struct gb_control *control);
+int gb_control_interface_deactivate_prepare(struct gb_control *control);
+int gb_control_interface_hibernate_abort(struct gb_control *control);
+#endif /* __CONTROL_H */
diff --git a/drivers/staging/greybus/core.c b/drivers/staging/greybus/core.c
new file mode 100644
index 000000000000..1049e9c0edb0
--- /dev/null
+++ b/drivers/staging/greybus/core.c
@@ -0,0 +1,361 @@
+/*
+ * Greybus "Core"
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#define CREATE_TRACE_POINTS
+#include "greybus.h"
+#include "greybus_trace.h"
+
+#define GB_BUNDLE_AUTOSUSPEND_MS	3000
+
+/* Allow greybus to be disabled at boot if needed */
+static bool nogreybus;
+#ifdef MODULE
+module_param(nogreybus, bool, 0444);
+#else
+core_param(nogreybus, nogreybus, bool, 0444);
+#endif
+int greybus_disabled(void)
+{
+	return nogreybus;
+}
+EXPORT_SYMBOL_GPL(greybus_disabled);
+
+static bool greybus_match_one_id(struct gb_bundle *bundle,
+				     const struct greybus_bundle_id *id)
+{
+	if ((id->match_flags & GREYBUS_ID_MATCH_VENDOR) &&
+	    (id->vendor != bundle->intf->vendor_id))
+		return false;
+
+	if ((id->match_flags & GREYBUS_ID_MATCH_PRODUCT) &&
+	    (id->product != bundle->intf->product_id))
+		return false;
+
+	if ((id->match_flags & GREYBUS_ID_MATCH_CLASS) &&
+	    (id->class != bundle->class))
+		return false;
+
+	return true;
+}
+
+static const struct greybus_bundle_id *
+greybus_match_id(struct gb_bundle *bundle, const struct greybus_bundle_id *id)
+{
+	if (id == NULL)
+		return NULL;
+
+	for (; id->vendor || id->product || id->class || id->driver_info;
+									id++) {
+		if (greybus_match_one_id(bundle, id))
+			return id;
+	}
+
+	return NULL;
+}
+
+static int greybus_match_device(struct device *dev, struct device_driver *drv)
+{
+	struct greybus_driver *driver = to_greybus_driver(drv);
+	struct gb_bundle *bundle;
+	const struct greybus_bundle_id *id;
+
+	if (!is_gb_bundle(dev))
+		return 0;
+
+	bundle = to_gb_bundle(dev);
+
+	id = greybus_match_id(bundle, driver->id_table);
+	if (id)
+		return 1;
+	/* FIXME - Dynamic ids? */
+	return 0;
+}
+
+static int greybus_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+	struct gb_host_device *hd;
+	struct gb_module *module = NULL;
+	struct gb_interface *intf = NULL;
+	struct gb_control *control = NULL;
+	struct gb_bundle *bundle = NULL;
+	struct gb_svc *svc = NULL;
+
+	if (is_gb_host_device(dev)) {
+		hd = to_gb_host_device(dev);
+	} else if (is_gb_module(dev)) {
+		module = to_gb_module(dev);
+		hd = module->hd;
+	} else if (is_gb_interface(dev)) {
+		intf = to_gb_interface(dev);
+		module = intf->module;
+		hd = intf->hd;
+	} else if (is_gb_control(dev)) {
+		control = to_gb_control(dev);
+		intf = control->intf;
+		module = intf->module;
+		hd = intf->hd;
+	} else if (is_gb_bundle(dev)) {
+		bundle = to_gb_bundle(dev);
+		intf = bundle->intf;
+		module = intf->module;
+		hd = intf->hd;
+	} else if (is_gb_svc(dev)) {
+		svc = to_gb_svc(dev);
+		hd = svc->hd;
+	} else {
+		dev_WARN(dev, "uevent for unknown greybus device \"type\"!\n");
+		return -EINVAL;
+	}
+
+	if (add_uevent_var(env, "BUS=%u", hd->bus_id))
+		return -ENOMEM;
+
+	if (module) {
+		if (add_uevent_var(env, "MODULE=%u", module->module_id))
+			return -ENOMEM;
+	}
+
+	if (intf) {
+		if (add_uevent_var(env, "INTERFACE=%u", intf->interface_id))
+			return -ENOMEM;
+		if (add_uevent_var(env, "GREYBUS_ID=%08x/%08x",
+				   intf->vendor_id, intf->product_id))
+			return -ENOMEM;
+	}
+
+	if (bundle) {
+		// FIXME
+		// add a uevent that can "load" a bundle type
+		// This is what we need to bind a driver to so use the info
+		// in gmod here as well
+
+		if (add_uevent_var(env, "BUNDLE=%u", bundle->id))
+			return -ENOMEM;
+		if (add_uevent_var(env, "BUNDLE_CLASS=%02x", bundle->class))
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void greybus_shutdown(struct device *dev)
+{
+	if (is_gb_host_device(dev)) {
+		struct gb_host_device *hd;
+
+		hd = to_gb_host_device(dev);
+		gb_hd_shutdown(hd);
+	}
+}
+
+struct bus_type greybus_bus_type = {
+	.name =		"greybus",
+	.match =	greybus_match_device,
+	.uevent =	greybus_uevent,
+	.shutdown =	greybus_shutdown,
+};
+
+static int greybus_probe(struct device *dev)
+{
+	struct greybus_driver *driver = to_greybus_driver(dev->driver);
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+	const struct greybus_bundle_id *id;
+	int retval;
+
+	/* match id */
+	id = greybus_match_id(bundle, driver->id_table);
+	if (!id)
+		return -ENODEV;
+
+	retval = pm_runtime_get_sync(&bundle->intf->dev);
+	if (retval < 0) {
+		pm_runtime_put_noidle(&bundle->intf->dev);
+		return retval;
+	}
+
+	retval = gb_control_bundle_activate(bundle->intf->control, bundle->id);
+	if (retval) {
+		pm_runtime_put(&bundle->intf->dev);
+		return retval;
+	}
+
+	/*
+	 * Unbound bundle devices are always deactivated. During probe, the
+	 * Runtime PM is set to enabled and active and the usage count is
+	 * incremented. If the driver supports runtime PM, it should call
+	 * pm_runtime_put() in its probe routine and pm_runtime_get_sync()
+	 * in remove routine.
+	 */
+	pm_runtime_set_autosuspend_delay(dev, GB_BUNDLE_AUTOSUSPEND_MS);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_get_noresume(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
+	retval = driver->probe(bundle, id);
+	if (retval) {
+		/*
+		 * Catch buggy drivers that fail to destroy their connections.
+		 */
+		WARN_ON(!list_empty(&bundle->connections));
+
+		gb_control_bundle_deactivate(bundle->intf->control, bundle->id);
+
+		pm_runtime_disable(dev);
+		pm_runtime_set_suspended(dev);
+		pm_runtime_put_noidle(dev);
+		pm_runtime_dont_use_autosuspend(dev);
+		pm_runtime_put(&bundle->intf->dev);
+
+		return retval;
+	}
+
+	gb_timesync_schedule_synchronous(bundle->intf);
+
+	pm_runtime_put(&bundle->intf->dev);
+
+	return 0;
+}
+
+static int greybus_remove(struct device *dev)
+{
+	struct greybus_driver *driver = to_greybus_driver(dev->driver);
+	struct gb_bundle *bundle = to_gb_bundle(dev);
+	struct gb_connection *connection;
+	int retval;
+
+	retval = pm_runtime_get_sync(dev);
+	if (retval < 0)
+		dev_err(dev, "failed to resume bundle: %d\n", retval);
+
+	/*
+	 * Disable (non-offloaded) connections early in case the interface is
+	 * already gone to avoid unceccessary operation timeouts during
+	 * driver disconnect. Otherwise, only disable incoming requests.
+	 */
+	list_for_each_entry(connection, &bundle->connections, bundle_links) {
+		if (gb_connection_is_offloaded(connection))
+			continue;
+
+		if (bundle->intf->disconnected)
+			gb_connection_disable_forced(connection);
+		else
+			gb_connection_disable_rx(connection);
+	}
+
+	driver->disconnect(bundle);
+
+	/* Catch buggy drivers that fail to destroy their connections. */
+	WARN_ON(!list_empty(&bundle->connections));
+
+	if (!bundle->intf->disconnected)
+		gb_control_bundle_deactivate(bundle->intf->control, bundle->id);
+
+	pm_runtime_put_noidle(dev);
+	pm_runtime_disable(dev);
+	pm_runtime_set_suspended(dev);
+	pm_runtime_dont_use_autosuspend(dev);
+	pm_runtime_put_noidle(dev);
+
+	return 0;
+}
+
+int greybus_register_driver(struct greybus_driver *driver, struct module *owner,
+		const char *mod_name)
+{
+	int retval;
+
+	if (greybus_disabled())
+		return -ENODEV;
+
+	driver->driver.bus = &greybus_bus_type;
+	driver->driver.name = driver->name;
+	driver->driver.probe = greybus_probe;
+	driver->driver.remove = greybus_remove;
+	driver->driver.owner = owner;
+	driver->driver.mod_name = mod_name;
+
+	retval = driver_register(&driver->driver);
+	if (retval)
+		return retval;
+
+	pr_info("registered new driver %s\n", driver->name);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(greybus_register_driver);
+
+void greybus_deregister_driver(struct greybus_driver *driver)
+{
+	driver_unregister(&driver->driver);
+}
+EXPORT_SYMBOL_GPL(greybus_deregister_driver);
+
+static int __init gb_init(void)
+{
+	int retval;
+
+	if (greybus_disabled())
+		return -ENODEV;
+
+	BUILD_BUG_ON(CPORT_ID_MAX >= (long)CPORT_ID_BAD);
+
+	gb_debugfs_init();
+
+	retval = bus_register(&greybus_bus_type);
+	if (retval) {
+		pr_err("bus_register failed (%d)\n", retval);
+		goto error_bus;
+	}
+
+	retval = gb_hd_init();
+	if (retval) {
+		pr_err("gb_hd_init failed (%d)\n", retval);
+		goto error_hd;
+	}
+
+	retval = gb_operation_init();
+	if (retval) {
+		pr_err("gb_operation_init failed (%d)\n", retval);
+		goto error_operation;
+	}
+
+	retval = gb_timesync_init();
+	if (retval) {
+		pr_err("gb_timesync_init failed\n");
+		goto error_timesync;
+	}
+	return 0;	/* Success */
+
+error_timesync:
+	gb_operation_exit();
+error_operation:
+	gb_hd_exit();
+error_hd:
+	bus_unregister(&greybus_bus_type);
+error_bus:
+	gb_debugfs_cleanup();
+
+	return retval;
+}
+module_init(gb_init);
+
+static void __exit gb_exit(void)
+{
+	gb_timesync_exit();
+	gb_operation_exit();
+	gb_hd_exit();
+	bus_unregister(&greybus_bus_type);
+	gb_debugfs_cleanup();
+	tracepoint_synchronize_unregister();
+}
+module_exit(gb_exit);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Greg Kroah-Hartman <gregkh@linuxfoundation.org>");
diff --git a/drivers/staging/greybus/debugfs.c b/drivers/staging/greybus/debugfs.c
new file mode 100644
index 000000000000..a9d4d3da99a0
--- /dev/null
+++ b/drivers/staging/greybus/debugfs.c
@@ -0,0 +1,31 @@
+/*
+ * Greybus debugfs code
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/debugfs.h>
+
+#include "greybus.h"
+
+static struct dentry *gb_debug_root;
+
+void __init gb_debugfs_init(void)
+{
+	gb_debug_root = debugfs_create_dir("greybus", NULL);
+}
+
+void gb_debugfs_cleanup(void)
+{
+	debugfs_remove_recursive(gb_debug_root);
+	gb_debug_root = NULL;
+}
+
+struct dentry *gb_debugfs_get(void)
+{
+	return gb_debug_root;
+}
+EXPORT_SYMBOL_GPL(gb_debugfs_get);
diff --git a/drivers/staging/greybus/es2.c b/drivers/staging/greybus/es2.c
new file mode 100644
index 000000000000..071bb1cfd3ae
--- /dev/null
+++ b/drivers/staging/greybus/es2.c
@@ -0,0 +1,1597 @@
+/*
+ * Greybus "AP" USB driver for "ES2" controller chips
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+#include <linux/kthread.h>
+#include <linux/sizes.h>
+#include <linux/usb.h>
+#include <linux/kfifo.h>
+#include <linux/debugfs.h>
+#include <linux/list.h>
+#include <asm/unaligned.h>
+
+#include "arpc.h"
+#include "greybus.h"
+#include "greybus_trace.h"
+#include "connection.h"
+
+
+/* Default timeout for USB vendor requests. */
+#define ES2_USB_CTRL_TIMEOUT	500
+
+/* Default timeout for ARPC CPort requests */
+#define ES2_ARPC_CPORT_TIMEOUT	500
+
+/* Fixed CPort numbers */
+#define ES2_CPORT_CDSI0		16
+#define ES2_CPORT_CDSI1		17
+
+/* Memory sizes for the buffers sent to/from the ES2 controller */
+#define ES2_GBUF_MSG_SIZE_MAX	2048
+
+/* Memory sizes for the ARPC buffers */
+#define ARPC_OUT_SIZE_MAX	U16_MAX
+#define ARPC_IN_SIZE_MAX	128
+
+static const struct usb_device_id id_table[] = {
+	{ USB_DEVICE(0x18d1, 0x1eaf) },
+	{ },
+};
+MODULE_DEVICE_TABLE(usb, id_table);
+
+#define APB1_LOG_SIZE		SZ_16K
+
+/*
+ * Number of CPort IN urbs in flight at any point in time.
+ * Adjust if we are having stalls in the USB buffer due to not enough urbs in
+ * flight.
+ */
+#define NUM_CPORT_IN_URB	4
+
+/* Number of CPort OUT urbs in flight at any point in time.
+ * Adjust if we get messages saying we are out of urbs in the system log.
+ */
+#define NUM_CPORT_OUT_URB	8
+
+/*
+ * Number of ARPC in urbs in flight at any point in time.
+ */
+#define NUM_ARPC_IN_URB		2
+
+/*
+ * @endpoint: bulk in endpoint for CPort data
+ * @urb: array of urbs for the CPort in messages
+ * @buffer: array of buffers for the @cport_in_urb urbs
+ */
+struct es2_cport_in {
+	__u8 endpoint;
+	struct urb *urb[NUM_CPORT_IN_URB];
+	u8 *buffer[NUM_CPORT_IN_URB];
+};
+
+/**
+ * es2_ap_dev - ES2 USB Bridge to AP structure
+ * @usb_dev: pointer to the USB device we are.
+ * @usb_intf: pointer to the USB interface we are bound to.
+ * @hd: pointer to our gb_host_device structure
+
+ * @cport_in: endpoint, urbs and buffer for cport in messages
+ * @cport_out_endpoint: endpoint for for cport out messages
+ * @cport_out_urb: array of urbs for the CPort out messages
+ * @cport_out_urb_busy: array of flags to see if the @cport_out_urb is busy or
+ *			not.
+ * @cport_out_urb_cancelled: array of flags indicating whether the
+ *			corresponding @cport_out_urb is being cancelled
+ * @cport_out_urb_lock: locks the @cport_out_urb_busy "list"
+ *
+ * @apb_log_task: task pointer for logging thread
+ * @apb_log_dentry: file system entry for the log file interface
+ * @apb_log_enable_dentry: file system entry for enabling logging
+ * @apb_log_fifo: kernel FIFO to carry logged data
+ * @arpc_urb: array of urbs for the ARPC in messages
+ * @arpc_buffer: array of buffers for the @arpc_urb urbs
+ * @arpc_endpoint_in: bulk in endpoint for APBridgeA RPC
+ * @arpc_id_cycle: gives an unique id to ARPC
+ * @arpc_lock: locks ARPC list
+ * @arpcs: list of in progress ARPCs
+ */
+struct es2_ap_dev {
+	struct usb_device *usb_dev;
+	struct usb_interface *usb_intf;
+	struct gb_host_device *hd;
+
+	struct es2_cport_in cport_in;
+	__u8 cport_out_endpoint;
+	struct urb *cport_out_urb[NUM_CPORT_OUT_URB];
+	bool cport_out_urb_busy[NUM_CPORT_OUT_URB];
+	bool cport_out_urb_cancelled[NUM_CPORT_OUT_URB];
+	spinlock_t cport_out_urb_lock;
+
+	bool cdsi1_in_use;
+
+	struct task_struct *apb_log_task;
+	struct dentry *apb_log_dentry;
+	struct dentry *apb_log_enable_dentry;
+	DECLARE_KFIFO(apb_log_fifo, char, APB1_LOG_SIZE);
+
+	__u8 arpc_endpoint_in;
+	struct urb *arpc_urb[NUM_ARPC_IN_URB];
+	u8 *arpc_buffer[NUM_ARPC_IN_URB];
+
+	int arpc_id_cycle;
+	spinlock_t arpc_lock;
+	struct list_head arpcs;
+};
+
+/**
+ * timesync_enable_request - Enable timesync in an APBridge
+ * @count: number of TimeSync Pulses to expect
+ * @frame_time: the initial FrameTime at the first TimeSync Pulse
+ * @strobe_delay: the expected delay in microseconds between each TimeSync Pulse
+ * @refclk: The AP mandated reference clock to run FrameTime at
+ */
+struct timesync_enable_request {
+	__u8	count;
+	__le64	frame_time;
+	__le32	strobe_delay;
+	__le32	refclk;
+} __packed;
+
+/**
+ * timesync_authoritative_request - Transmit authoritative FrameTime to APBridge
+ * @frame_time: An array of authoritative FrameTimes provided by the SVC
+ *              and relayed to the APBridge by the AP
+ */
+struct timesync_authoritative_request {
+	__le64	frame_time[GB_TIMESYNC_MAX_STROBES];
+} __packed;
+
+struct arpc {
+	struct list_head list;
+	struct arpc_request_message *req;
+	struct arpc_response_message *resp;
+	struct completion response_received;
+	bool active;
+};
+
+static inline struct es2_ap_dev *hd_to_es2(struct gb_host_device *hd)
+{
+	return (struct es2_ap_dev *)&hd->hd_priv;
+}
+
+static void cport_out_callback(struct urb *urb);
+static void usb_log_enable(struct es2_ap_dev *es2);
+static void usb_log_disable(struct es2_ap_dev *es2);
+static int arpc_sync(struct es2_ap_dev *es2, u8 type, void *payload,
+		     size_t size, int *result, unsigned int timeout);
+
+static int output_sync(struct es2_ap_dev *es2, void *req, u16 size, u8 cmd)
+{
+	struct usb_device *udev = es2->usb_dev;
+	u8 *data;
+	int retval;
+
+	data = kmalloc(size, GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+	memcpy(data, req, size);
+
+	retval = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+				 cmd,
+				 USB_DIR_OUT | USB_TYPE_VENDOR |
+				 USB_RECIP_INTERFACE,
+				 0, 0, data, size, ES2_USB_CTRL_TIMEOUT);
+	if (retval < 0)
+		dev_err(&udev->dev, "%s: return error %d\n", __func__, retval);
+	else
+		retval = 0;
+
+	kfree(data);
+	return retval;
+}
+
+static void ap_urb_complete(struct urb *urb)
+{
+	struct usb_ctrlrequest *dr = urb->context;
+
+	kfree(dr);
+	usb_free_urb(urb);
+}
+
+static int output_async(struct es2_ap_dev *es2, void *req, u16 size, u8 cmd)
+{
+	struct usb_device *udev = es2->usb_dev;
+	struct urb *urb;
+	struct usb_ctrlrequest *dr;
+	u8 *buf;
+	int retval;
+
+	urb = usb_alloc_urb(0, GFP_ATOMIC);
+	if (!urb)
+		return -ENOMEM;
+
+	dr = kmalloc(sizeof(*dr) + size, GFP_ATOMIC);
+	if (!dr) {
+		usb_free_urb(urb);
+		return -ENOMEM;
+	}
+
+	buf = (u8 *)dr + sizeof(*dr);
+	memcpy(buf, req, size);
+
+	dr->bRequest = cmd;
+	dr->bRequestType = USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE;
+	dr->wValue = 0;
+	dr->wIndex = 0;
+	dr->wLength = cpu_to_le16(size);
+
+	usb_fill_control_urb(urb, udev, usb_sndctrlpipe(udev, 0),
+			     (unsigned char *)dr, buf, size,
+			     ap_urb_complete, dr);
+	retval = usb_submit_urb(urb, GFP_ATOMIC);
+	if (retval) {
+		usb_free_urb(urb);
+		kfree(dr);
+	}
+	return retval;
+}
+
+static int output(struct gb_host_device *hd, void *req, u16 size, u8 cmd,
+		     bool async)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+
+	if (async)
+		return output_async(es2, req, size, cmd);
+
+	return output_sync(es2, req, size, cmd);
+}
+
+static int es2_cport_in_enable(struct es2_ap_dev *es2,
+				struct es2_cport_in *cport_in)
+{
+	struct urb *urb;
+	int ret;
+	int i;
+
+	for (i = 0; i < NUM_CPORT_IN_URB; ++i) {
+		urb = cport_in->urb[i];
+
+		ret = usb_submit_urb(urb, GFP_KERNEL);
+		if (ret) {
+			dev_err(&es2->usb_dev->dev,
+					"failed to submit in-urb: %d\n", ret);
+			goto err_kill_urbs;
+		}
+	}
+
+	return 0;
+
+err_kill_urbs:
+	for (--i; i >= 0; --i) {
+		urb = cport_in->urb[i];
+		usb_kill_urb(urb);
+	}
+
+	return ret;
+}
+
+static void es2_cport_in_disable(struct es2_ap_dev *es2,
+				struct es2_cport_in *cport_in)
+{
+	struct urb *urb;
+	int i;
+
+	for (i = 0; i < NUM_CPORT_IN_URB; ++i) {
+		urb = cport_in->urb[i];
+		usb_kill_urb(urb);
+	}
+}
+
+static int es2_arpc_in_enable(struct es2_ap_dev *es2)
+{
+	struct urb *urb;
+	int ret;
+	int i;
+
+	for (i = 0; i < NUM_ARPC_IN_URB; ++i) {
+		urb = es2->arpc_urb[i];
+
+		ret = usb_submit_urb(urb, GFP_KERNEL);
+		if (ret) {
+			dev_err(&es2->usb_dev->dev,
+				"failed to submit arpc in-urb: %d\n", ret);
+			goto err_kill_urbs;
+		}
+	}
+
+	return 0;
+
+err_kill_urbs:
+	for (--i; i >= 0; --i) {
+		urb = es2->arpc_urb[i];
+		usb_kill_urb(urb);
+	}
+
+	return ret;
+}
+
+static void es2_arpc_in_disable(struct es2_ap_dev *es2)
+{
+	struct urb *urb;
+	int i;
+
+	for (i = 0; i < NUM_ARPC_IN_URB; ++i) {
+		urb = es2->arpc_urb[i];
+		usb_kill_urb(urb);
+	}
+}
+
+static struct urb *next_free_urb(struct es2_ap_dev *es2, gfp_t gfp_mask)
+{
+	struct urb *urb = NULL;
+	unsigned long flags;
+	int i;
+
+	spin_lock_irqsave(&es2->cport_out_urb_lock, flags);
+
+	/* Look in our pool of allocated urbs first, as that's the "fastest" */
+	for (i = 0; i < NUM_CPORT_OUT_URB; ++i) {
+		if (es2->cport_out_urb_busy[i] == false &&
+				es2->cport_out_urb_cancelled[i] == false) {
+			es2->cport_out_urb_busy[i] = true;
+			urb = es2->cport_out_urb[i];
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&es2->cport_out_urb_lock, flags);
+	if (urb)
+		return urb;
+
+	/*
+	 * Crap, pool is empty, complain to the syslog and go allocate one
+	 * dynamically as we have to succeed.
+	 */
+	dev_dbg(&es2->usb_dev->dev,
+		"No free CPort OUT urbs, having to dynamically allocate one!\n");
+	return usb_alloc_urb(0, gfp_mask);
+}
+
+static void free_urb(struct es2_ap_dev *es2, struct urb *urb)
+{
+	unsigned long flags;
+	int i;
+	/*
+	 * See if this was an urb in our pool, if so mark it "free", otherwise
+	 * we need to free it ourselves.
+	 */
+	spin_lock_irqsave(&es2->cport_out_urb_lock, flags);
+	for (i = 0; i < NUM_CPORT_OUT_URB; ++i) {
+		if (urb == es2->cport_out_urb[i]) {
+			es2->cport_out_urb_busy[i] = false;
+			urb = NULL;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&es2->cport_out_urb_lock, flags);
+
+	/* If urb is not NULL, then we need to free this urb */
+	usb_free_urb(urb);
+}
+
+/*
+ * We (ab)use the operation-message header pad bytes to transfer the
+ * cport id in order to minimise overhead.
+ */
+static void
+gb_message_cport_pack(struct gb_operation_msg_hdr *header, u16 cport_id)
+{
+	header->pad[0] = cport_id;
+}
+
+/* Clear the pad bytes used for the CPort id */
+static void gb_message_cport_clear(struct gb_operation_msg_hdr *header)
+{
+	header->pad[0] = 0;
+}
+
+/* Extract the CPort id packed into the header, and clear it */
+static u16 gb_message_cport_unpack(struct gb_operation_msg_hdr *header)
+{
+	u16 cport_id = header->pad[0];
+
+	gb_message_cport_clear(header);
+
+	return cport_id;
+}
+
+/*
+ * Returns zero if the message was successfully queued, or a negative errno
+ * otherwise.
+ */
+static int message_send(struct gb_host_device *hd, u16 cport_id,
+			struct gb_message *message, gfp_t gfp_mask)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct usb_device *udev = es2->usb_dev;
+	size_t buffer_size;
+	int retval;
+	struct urb *urb;
+	unsigned long flags;
+
+	/*
+	 * The data actually transferred will include an indication
+	 * of where the data should be sent.  Do one last check of
+	 * the target CPort id before filling it in.
+	 */
+	if (!cport_id_valid(hd, cport_id)) {
+		dev_err(&udev->dev, "invalid cport %u\n", cport_id);
+		return -EINVAL;
+	}
+
+	/* Find a free urb */
+	urb = next_free_urb(es2, gfp_mask);
+	if (!urb)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&es2->cport_out_urb_lock, flags);
+	message->hcpriv = urb;
+	spin_unlock_irqrestore(&es2->cport_out_urb_lock, flags);
+
+	/* Pack the cport id into the message header */
+	gb_message_cport_pack(message->header, cport_id);
+
+	buffer_size = sizeof(*message->header) + message->payload_size;
+
+	usb_fill_bulk_urb(urb, udev,
+			  usb_sndbulkpipe(udev,
+					  es2->cport_out_endpoint),
+			  message->buffer, buffer_size,
+			  cport_out_callback, message);
+	urb->transfer_flags |= URB_ZERO_PACKET;
+
+	trace_gb_message_submit(message);
+
+	retval = usb_submit_urb(urb, gfp_mask);
+	if (retval) {
+		dev_err(&udev->dev, "failed to submit out-urb: %d\n", retval);
+
+		spin_lock_irqsave(&es2->cport_out_urb_lock, flags);
+		message->hcpriv = NULL;
+		spin_unlock_irqrestore(&es2->cport_out_urb_lock, flags);
+
+		free_urb(es2, urb);
+		gb_message_cport_clear(message->header);
+
+		return retval;
+	}
+
+	return 0;
+}
+
+/*
+ * Can not be called in atomic context.
+ */
+static void message_cancel(struct gb_message *message)
+{
+	struct gb_host_device *hd = message->operation->connection->hd;
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct urb *urb;
+	int i;
+
+	might_sleep();
+
+	spin_lock_irq(&es2->cport_out_urb_lock);
+	urb = message->hcpriv;
+
+	/* Prevent dynamically allocated urb from being deallocated. */
+	usb_get_urb(urb);
+
+	/* Prevent pre-allocated urb from being reused. */
+	for (i = 0; i < NUM_CPORT_OUT_URB; ++i) {
+		if (urb == es2->cport_out_urb[i]) {
+			es2->cport_out_urb_cancelled[i] = true;
+			break;
+		}
+	}
+	spin_unlock_irq(&es2->cport_out_urb_lock);
+
+	usb_kill_urb(urb);
+
+	if (i < NUM_CPORT_OUT_URB) {
+		spin_lock_irq(&es2->cport_out_urb_lock);
+		es2->cport_out_urb_cancelled[i] = false;
+		spin_unlock_irq(&es2->cport_out_urb_lock);
+	}
+
+	usb_free_urb(urb);
+}
+
+static int es2_cport_allocate(struct gb_host_device *hd, int cport_id,
+				unsigned long flags)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct ida *id_map = &hd->cport_id_map;
+	int ida_start, ida_end;
+
+	switch (cport_id) {
+	case ES2_CPORT_CDSI0:
+	case ES2_CPORT_CDSI1:
+		dev_err(&hd->dev, "cport %d not available\n", cport_id);
+		return -EBUSY;
+	}
+
+	if (flags & GB_CONNECTION_FLAG_OFFLOADED &&
+			flags & GB_CONNECTION_FLAG_CDSI1) {
+		if (es2->cdsi1_in_use) {
+			dev_err(&hd->dev, "CDSI1 already in use\n");
+			return -EBUSY;
+		}
+
+		es2->cdsi1_in_use = true;
+
+		return ES2_CPORT_CDSI1;
+	}
+
+	if (cport_id < 0) {
+		ida_start = 0;
+		ida_end = hd->num_cports;
+	} else if (cport_id < hd->num_cports) {
+		ida_start = cport_id;
+		ida_end = cport_id + 1;
+	} else {
+		dev_err(&hd->dev, "cport %d not available\n", cport_id);
+		return -EINVAL;
+	}
+
+	return ida_simple_get(id_map, ida_start, ida_end, GFP_KERNEL);
+}
+
+static void es2_cport_release(struct gb_host_device *hd, u16 cport_id)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+
+	switch (cport_id) {
+	case ES2_CPORT_CDSI1:
+		es2->cdsi1_in_use = false;
+		return;
+	}
+
+	ida_simple_remove(&hd->cport_id_map, cport_id);
+}
+
+static int cport_enable(struct gb_host_device *hd, u16 cport_id,
+			unsigned long flags)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct usb_device *udev = es2->usb_dev;
+	struct gb_apb_request_cport_flags *req;
+	u32 connection_flags;
+	int ret;
+
+	req = kzalloc(sizeof(*req), GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	connection_flags = 0;
+	if (flags & GB_CONNECTION_FLAG_CONTROL)
+		connection_flags |= GB_APB_CPORT_FLAG_CONTROL;
+	if (flags & GB_CONNECTION_FLAG_HIGH_PRIO)
+		connection_flags |= GB_APB_CPORT_FLAG_HIGH_PRIO;
+
+	req->flags = cpu_to_le32(connection_flags);
+
+	dev_dbg(&hd->dev, "%s - cport = %u, flags = %02x\n", __func__,
+			cport_id, connection_flags);
+
+	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+				GB_APB_REQUEST_CPORT_FLAGS,
+				USB_DIR_OUT | USB_TYPE_VENDOR |
+				USB_RECIP_INTERFACE, cport_id, 0,
+				req, sizeof(*req), ES2_USB_CTRL_TIMEOUT);
+	if (ret != sizeof(*req)) {
+		dev_err(&udev->dev, "failed to set cport flags for port %d\n",
+				cport_id);
+		if (ret >= 0)
+			ret = -EIO;
+
+		goto out;
+	}
+
+	ret = 0;
+out:
+	kfree(req);
+
+	return ret;
+}
+
+static int es2_cport_connected(struct gb_host_device *hd, u16 cport_id)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct device *dev = &es2->usb_dev->dev;
+	struct arpc_cport_connected_req req;
+	int ret;
+
+	req.cport_id = cpu_to_le16(cport_id);
+	ret = arpc_sync(es2, ARPC_TYPE_CPORT_CONNECTED, &req, sizeof(req),
+			NULL, ES2_ARPC_CPORT_TIMEOUT);
+	if (ret) {
+		dev_err(dev, "failed to set connected state for cport %u: %d\n",
+				cport_id, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int es2_cport_flush(struct gb_host_device *hd, u16 cport_id)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct device *dev = &es2->usb_dev->dev;
+	struct arpc_cport_flush_req req;
+	int ret;
+
+	req.cport_id = cpu_to_le16(cport_id);
+	ret = arpc_sync(es2, ARPC_TYPE_CPORT_FLUSH, &req, sizeof(req),
+			NULL, ES2_ARPC_CPORT_TIMEOUT);
+	if (ret) {
+		dev_err(dev, "failed to flush cport %u: %d\n", cport_id, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int es2_cport_shutdown(struct gb_host_device *hd, u16 cport_id,
+				u8 phase, unsigned int timeout)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct device *dev = &es2->usb_dev->dev;
+	struct arpc_cport_shutdown_req req;
+	int result;
+	int ret;
+
+	if (timeout > U16_MAX)
+		return -EINVAL;
+
+	req.cport_id = cpu_to_le16(cport_id);
+	req.timeout = cpu_to_le16(timeout);
+	req.phase = phase;
+	ret = arpc_sync(es2, ARPC_TYPE_CPORT_SHUTDOWN, &req, sizeof(req),
+			&result, ES2_ARPC_CPORT_TIMEOUT + timeout);
+	if (ret) {
+		dev_err(dev, "failed to send shutdown over cport %u: %d (%d)\n",
+				cport_id, ret, result);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int es2_cport_quiesce(struct gb_host_device *hd, u16 cport_id,
+				size_t peer_space, unsigned int timeout)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct device *dev = &es2->usb_dev->dev;
+	struct arpc_cport_quiesce_req req;
+	int result;
+	int ret;
+
+	if (peer_space > U16_MAX)
+		return -EINVAL;
+
+	if (timeout > U16_MAX)
+		return -EINVAL;
+
+	req.cport_id = cpu_to_le16(cport_id);
+	req.peer_space = cpu_to_le16(peer_space);
+	req.timeout = cpu_to_le16(timeout);
+	ret = arpc_sync(es2, ARPC_TYPE_CPORT_QUIESCE, &req, sizeof(req),
+			&result, ES2_ARPC_CPORT_TIMEOUT + timeout);
+	if (ret) {
+		dev_err(dev, "failed to quiesce cport %u: %d (%d)\n",
+				cport_id, ret, result);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int es2_cport_clear(struct gb_host_device *hd, u16 cport_id)
+{
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct device *dev = &es2->usb_dev->dev;
+	struct arpc_cport_clear_req req;
+	int ret;
+
+	req.cport_id = cpu_to_le16(cport_id);
+	ret = arpc_sync(es2, ARPC_TYPE_CPORT_CLEAR, &req, sizeof(req),
+			NULL, ES2_ARPC_CPORT_TIMEOUT);
+	if (ret) {
+		dev_err(dev, "failed to clear cport %u: %d\n", cport_id, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int latency_tag_enable(struct gb_host_device *hd, u16 cport_id)
+{
+	int retval;
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct usb_device *udev = es2->usb_dev;
+
+	retval = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+				 GB_APB_REQUEST_LATENCY_TAG_EN,
+				 USB_DIR_OUT | USB_TYPE_VENDOR |
+				 USB_RECIP_INTERFACE, cport_id, 0, NULL,
+				 0, ES2_USB_CTRL_TIMEOUT);
+
+	if (retval < 0)
+		dev_err(&udev->dev, "Cannot enable latency tag for cport %d\n",
+			cport_id);
+	return retval;
+}
+
+static int latency_tag_disable(struct gb_host_device *hd, u16 cport_id)
+{
+	int retval;
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct usb_device *udev = es2->usb_dev;
+
+	retval = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+				 GB_APB_REQUEST_LATENCY_TAG_DIS,
+				 USB_DIR_OUT | USB_TYPE_VENDOR |
+				 USB_RECIP_INTERFACE, cport_id, 0, NULL,
+				 0, ES2_USB_CTRL_TIMEOUT);
+
+	if (retval < 0)
+		dev_err(&udev->dev, "Cannot disable latency tag for cport %d\n",
+			cport_id);
+	return retval;
+}
+
+static int timesync_enable(struct gb_host_device *hd, u8 count,
+			   u64 frame_time, u32 strobe_delay, u32 refclk)
+{
+	int retval;
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct usb_device *udev = es2->usb_dev;
+	struct gb_control_timesync_enable_request *request;
+
+	request = kzalloc(sizeof(*request), GFP_KERNEL);
+	if (!request)
+		return -ENOMEM;
+
+	request->count = count;
+	request->frame_time = cpu_to_le64(frame_time);
+	request->strobe_delay = cpu_to_le32(strobe_delay);
+	request->refclk = cpu_to_le32(refclk);
+	retval = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+				 GB_APB_REQUEST_TIMESYNC_ENABLE,
+				 USB_DIR_OUT | USB_TYPE_VENDOR |
+				 USB_RECIP_INTERFACE, 0, 0, request,
+				 sizeof(*request), ES2_USB_CTRL_TIMEOUT);
+	if (retval < 0)
+		dev_err(&udev->dev, "Cannot enable timesync %d\n", retval);
+
+	kfree(request);
+	return retval;
+}
+
+static int timesync_disable(struct gb_host_device *hd)
+{
+	int retval;
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct usb_device *udev = es2->usb_dev;
+
+	retval = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+				 GB_APB_REQUEST_TIMESYNC_DISABLE,
+				 USB_DIR_OUT | USB_TYPE_VENDOR |
+				 USB_RECIP_INTERFACE, 0, 0, NULL,
+				 0, ES2_USB_CTRL_TIMEOUT);
+	if (retval < 0)
+		dev_err(&udev->dev, "Cannot disable timesync %d\n", retval);
+
+	return retval;
+}
+
+static int timesync_authoritative(struct gb_host_device *hd, u64 *frame_time)
+{
+	int retval, i;
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct usb_device *udev = es2->usb_dev;
+	struct timesync_authoritative_request *request;
+
+	request = kzalloc(sizeof(*request), GFP_KERNEL);
+	if (!request)
+		return -ENOMEM;
+
+	for (i = 0; i < GB_TIMESYNC_MAX_STROBES; i++)
+		request->frame_time[i] = cpu_to_le64(frame_time[i]);
+
+	retval = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+				 GB_APB_REQUEST_TIMESYNC_AUTHORITATIVE,
+				 USB_DIR_OUT | USB_TYPE_VENDOR |
+				 USB_RECIP_INTERFACE, 0, 0, request,
+				 sizeof(*request), ES2_USB_CTRL_TIMEOUT);
+	if (retval < 0)
+		dev_err(&udev->dev, "Cannot timesync authoritative out %d\n", retval);
+
+	kfree(request);
+	return retval;
+}
+
+static int timesync_get_last_event(struct gb_host_device *hd, u64 *frame_time)
+{
+	int retval;
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	struct usb_device *udev = es2->usb_dev;
+	__le64 *response_frame_time;
+
+	response_frame_time = kzalloc(sizeof(*response_frame_time), GFP_KERNEL);
+	if (!response_frame_time)
+		return -ENOMEM;
+
+	retval = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
+				 GB_APB_REQUEST_TIMESYNC_GET_LAST_EVENT,
+				 USB_DIR_IN | USB_TYPE_VENDOR |
+				 USB_RECIP_INTERFACE, 0, 0, response_frame_time,
+				 sizeof(*response_frame_time),
+				 ES2_USB_CTRL_TIMEOUT);
+
+	if (retval != sizeof(*response_frame_time)) {
+		dev_err(&udev->dev, "Cannot get last TimeSync event: %d\n",
+			retval);
+
+		if (retval >= 0)
+			retval = -EIO;
+
+		goto out;
+	}
+	*frame_time = le64_to_cpu(*response_frame_time);
+	retval = 0;
+out:
+	kfree(response_frame_time);
+	return retval;
+}
+
+static struct gb_hd_driver es2_driver = {
+	.hd_priv_size			= sizeof(struct es2_ap_dev),
+	.message_send			= message_send,
+	.message_cancel			= message_cancel,
+	.cport_allocate			= es2_cport_allocate,
+	.cport_release			= es2_cport_release,
+	.cport_enable			= cport_enable,
+	.cport_connected		= es2_cport_connected,
+	.cport_flush			= es2_cport_flush,
+	.cport_shutdown			= es2_cport_shutdown,
+	.cport_quiesce			= es2_cport_quiesce,
+	.cport_clear			= es2_cport_clear,
+	.latency_tag_enable		= latency_tag_enable,
+	.latency_tag_disable		= latency_tag_disable,
+	.output				= output,
+	.timesync_enable		= timesync_enable,
+	.timesync_disable		= timesync_disable,
+	.timesync_authoritative		= timesync_authoritative,
+	.timesync_get_last_event	= timesync_get_last_event,
+};
+
+/* Common function to report consistent warnings based on URB status */
+static int check_urb_status(struct urb *urb)
+{
+	struct device *dev = &urb->dev->dev;
+	int status = urb->status;
+
+	switch (status) {
+	case 0:
+		return 0;
+
+	case -EOVERFLOW:
+		dev_err(dev, "%s: overflow actual length is %d\n",
+			__func__, urb->actual_length);
+	case -ECONNRESET:
+	case -ENOENT:
+	case -ESHUTDOWN:
+	case -EILSEQ:
+	case -EPROTO:
+		/* device is gone, stop sending */
+		return status;
+	}
+	dev_err(dev, "%s: unknown status %d\n", __func__, status);
+
+	return -EAGAIN;
+}
+
+static void es2_destroy(struct es2_ap_dev *es2)
+{
+	struct usb_device *udev;
+	struct urb *urb;
+	int i;
+
+	debugfs_remove(es2->apb_log_enable_dentry);
+	usb_log_disable(es2);
+
+	/* Tear down everything! */
+	for (i = 0; i < NUM_CPORT_OUT_URB; ++i) {
+		urb = es2->cport_out_urb[i];
+		usb_kill_urb(urb);
+		usb_free_urb(urb);
+		es2->cport_out_urb[i] = NULL;
+		es2->cport_out_urb_busy[i] = false;	/* just to be anal */
+	}
+
+	for (i = 0; i < NUM_ARPC_IN_URB; ++i) {
+		usb_free_urb(es2->arpc_urb[i]);
+		kfree(es2->arpc_buffer[i]);
+		es2->arpc_buffer[i] = NULL;
+	}
+
+	for (i = 0; i < NUM_CPORT_IN_URB; ++i) {
+		usb_free_urb(es2->cport_in.urb[i]);
+		kfree(es2->cport_in.buffer[i]);
+		es2->cport_in.buffer[i] = NULL;
+	}
+
+	/* release reserved CDSI0 and CDSI1 cports */
+	gb_hd_cport_release_reserved(es2->hd, ES2_CPORT_CDSI1);
+	gb_hd_cport_release_reserved(es2->hd, ES2_CPORT_CDSI0);
+
+	udev = es2->usb_dev;
+	gb_hd_put(es2->hd);
+
+	usb_put_dev(udev);
+}
+
+static void cport_in_callback(struct urb *urb)
+{
+	struct gb_host_device *hd = urb->context;
+	struct device *dev = &urb->dev->dev;
+	struct gb_operation_msg_hdr *header;
+	int status = check_urb_status(urb);
+	int retval;
+	u16 cport_id;
+
+	if (status) {
+		if ((status == -EAGAIN) || (status == -EPROTO))
+			goto exit;
+
+		/* The urb is being unlinked */
+		if (status == -ENOENT || status == -ESHUTDOWN)
+			return;
+
+		dev_err(dev, "urb cport in error %d (dropped)\n", status);
+		return;
+	}
+
+	if (urb->actual_length < sizeof(*header)) {
+		dev_err(dev, "short message received\n");
+		goto exit;
+	}
+
+	/* Extract the CPort id, which is packed in the message header */
+	header = urb->transfer_buffer;
+	cport_id = gb_message_cport_unpack(header);
+
+	if (cport_id_valid(hd, cport_id)) {
+		greybus_data_rcvd(hd, cport_id, urb->transfer_buffer,
+							urb->actual_length);
+	} else {
+		dev_err(dev, "invalid cport id %u received\n", cport_id);
+	}
+exit:
+	/* put our urb back in the request pool */
+	retval = usb_submit_urb(urb, GFP_ATOMIC);
+	if (retval)
+		dev_err(dev, "failed to resubmit in-urb: %d\n", retval);
+}
+
+static void cport_out_callback(struct urb *urb)
+{
+	struct gb_message *message = urb->context;
+	struct gb_host_device *hd = message->operation->connection->hd;
+	struct es2_ap_dev *es2 = hd_to_es2(hd);
+	int status = check_urb_status(urb);
+	unsigned long flags;
+
+	gb_message_cport_clear(message->header);
+
+	spin_lock_irqsave(&es2->cport_out_urb_lock, flags);
+	message->hcpriv = NULL;
+	spin_unlock_irqrestore(&es2->cport_out_urb_lock, flags);
+
+	/*
+	 * Tell the submitter that the message send (attempt) is
+	 * complete, and report the status.
+	 */
+	greybus_message_sent(hd, message, status);
+
+	free_urb(es2, urb);
+}
+
+static struct arpc *arpc_alloc(void *payload, u16 size, u8 type)
+{
+	struct arpc *rpc;
+
+	if (size + sizeof(*rpc->req) > ARPC_OUT_SIZE_MAX)
+		return NULL;
+
+	rpc = kzalloc(sizeof(*rpc), GFP_KERNEL);
+	if (!rpc)
+		return NULL;
+
+	INIT_LIST_HEAD(&rpc->list);
+	rpc->req = kzalloc(sizeof(*rpc->req) + size, GFP_KERNEL);
+	if (!rpc->req)
+		goto err_free_rpc;
+
+	rpc->resp = kzalloc(sizeof(*rpc->resp), GFP_KERNEL);
+	if (!rpc->resp)
+		goto err_free_req;
+
+	rpc->req->type = type;
+	rpc->req->size = cpu_to_le16(sizeof(rpc->req) + size);
+	memcpy(rpc->req->data, payload, size);
+
+	init_completion(&rpc->response_received);
+
+	return rpc;
+
+err_free_req:
+	kfree(rpc->req);
+err_free_rpc:
+	kfree(rpc);
+
+	return NULL;
+}
+
+static void arpc_free(struct arpc *rpc)
+{
+	kfree(rpc->req);
+	kfree(rpc->resp);
+	kfree(rpc);
+}
+
+static struct arpc *arpc_find(struct es2_ap_dev *es2, __le16 id)
+{
+	struct arpc *rpc;
+
+	list_for_each_entry(rpc, &es2->arpcs, list) {
+		if (rpc->req->id == id)
+			return rpc;
+	}
+
+	return NULL;
+}
+
+static void arpc_add(struct es2_ap_dev *es2, struct arpc *rpc)
+{
+	rpc->active = true;
+	rpc->req->id = cpu_to_le16(es2->arpc_id_cycle++);
+	list_add_tail(&rpc->list, &es2->arpcs);
+}
+
+static void arpc_del(struct es2_ap_dev *es2, struct arpc *rpc)
+{
+	if (rpc->active) {
+		rpc->active = false;
+		list_del(&rpc->list);
+	}
+}
+
+static int arpc_send(struct es2_ap_dev *es2, struct arpc *rpc, int timeout)
+{
+	struct usb_device *udev = es2->usb_dev;
+	int retval;
+
+	retval = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+				 GB_APB_REQUEST_ARPC_RUN,
+				 USB_DIR_OUT | USB_TYPE_VENDOR |
+				 USB_RECIP_INTERFACE,
+				 0, 0,
+				 rpc->req, le16_to_cpu(rpc->req->size),
+				 ES2_USB_CTRL_TIMEOUT);
+	if (retval != le16_to_cpu(rpc->req->size)) {
+		dev_err(&udev->dev,
+			"failed to send ARPC request %d: %d\n",
+			rpc->req->type, retval);
+		if (retval > 0)
+			retval = -EIO;
+		return retval;
+	}
+
+	return 0;
+}
+
+static int arpc_sync(struct es2_ap_dev *es2, u8 type, void *payload,
+		     size_t size, int *result, unsigned int timeout)
+{
+	struct arpc *rpc;
+	unsigned long flags;
+	int retval;
+
+	if (result)
+		*result = 0;
+
+	rpc = arpc_alloc(payload, size, type);
+	if (!rpc)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&es2->arpc_lock, flags);
+	arpc_add(es2, rpc);
+	spin_unlock_irqrestore(&es2->arpc_lock, flags);
+
+	retval = arpc_send(es2, rpc, timeout);
+	if (retval)
+		goto out_arpc_del;
+
+	retval = wait_for_completion_interruptible_timeout(
+						&rpc->response_received,
+						msecs_to_jiffies(timeout));
+	if (retval <= 0) {
+		if (!retval)
+			retval = -ETIMEDOUT;
+		goto out_arpc_del;
+	}
+
+	if (rpc->resp->result) {
+		retval = -EREMOTEIO;
+		if (result)
+			*result = rpc->resp->result;
+	} else {
+		retval = 0;
+	}
+
+out_arpc_del:
+	spin_lock_irqsave(&es2->arpc_lock, flags);
+	arpc_del(es2, rpc);
+	spin_unlock_irqrestore(&es2->arpc_lock, flags);
+	arpc_free(rpc);
+
+	if (retval < 0 && retval != -EREMOTEIO) {
+		dev_err(&es2->usb_dev->dev,
+			"failed to execute ARPC: %d\n", retval);
+	}
+
+	return retval;
+}
+
+static void arpc_in_callback(struct urb *urb)
+{
+	struct es2_ap_dev *es2 = urb->context;
+	struct device *dev = &urb->dev->dev;
+	int status = check_urb_status(urb);
+	struct arpc *rpc;
+	struct arpc_response_message *resp;
+	unsigned long flags;
+	int retval;
+
+	if (status) {
+		if ((status == -EAGAIN) || (status == -EPROTO))
+			goto exit;
+
+		/* The urb is being unlinked */
+		if (status == -ENOENT || status == -ESHUTDOWN)
+			return;
+
+		dev_err(dev, "arpc in-urb error %d (dropped)\n", status);
+		return;
+	}
+
+	if (urb->actual_length < sizeof(*resp)) {
+		dev_err(dev, "short aprc response received\n");
+		goto exit;
+	}
+
+	resp = urb->transfer_buffer;
+	spin_lock_irqsave(&es2->arpc_lock, flags);
+	rpc = arpc_find(es2, resp->id);
+	if (!rpc) {
+		dev_err(dev, "invalid arpc response id received: %u\n",
+			le16_to_cpu(resp->id));
+		spin_unlock_irqrestore(&es2->arpc_lock, flags);
+		goto exit;
+	}
+
+	arpc_del(es2, rpc);
+	memcpy(rpc->resp, resp, sizeof(*resp));
+	complete(&rpc->response_received);
+	spin_unlock_irqrestore(&es2->arpc_lock, flags);
+
+exit:
+	/* put our urb back in the request pool */
+	retval = usb_submit_urb(urb, GFP_ATOMIC);
+	if (retval)
+		dev_err(dev, "failed to resubmit arpc in-urb: %d\n", retval);
+}
+
+#define APB1_LOG_MSG_SIZE	64
+static void apb_log_get(struct es2_ap_dev *es2, char *buf)
+{
+	int retval;
+
+	do {
+		retval = usb_control_msg(es2->usb_dev,
+					usb_rcvctrlpipe(es2->usb_dev, 0),
+					GB_APB_REQUEST_LOG,
+					USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+					0x00, 0x00,
+					buf,
+					APB1_LOG_MSG_SIZE,
+					ES2_USB_CTRL_TIMEOUT);
+		if (retval > 0)
+			kfifo_in(&es2->apb_log_fifo, buf, retval);
+	} while (retval > 0);
+}
+
+static int apb_log_poll(void *data)
+{
+	struct es2_ap_dev *es2 = data;
+	char *buf;
+
+	buf = kmalloc(APB1_LOG_MSG_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	while (!kthread_should_stop()) {
+		msleep(1000);
+		apb_log_get(es2, buf);
+	}
+
+	kfree(buf);
+
+	return 0;
+}
+
+static ssize_t apb_log_read(struct file *f, char __user *buf,
+				size_t count, loff_t *ppos)
+{
+	struct es2_ap_dev *es2 = f->f_inode->i_private;
+	ssize_t ret;
+	size_t copied;
+	char *tmp_buf;
+
+	if (count > APB1_LOG_SIZE)
+		count = APB1_LOG_SIZE;
+
+	tmp_buf = kmalloc(count, GFP_KERNEL);
+	if (!tmp_buf)
+		return -ENOMEM;
+
+	copied = kfifo_out(&es2->apb_log_fifo, tmp_buf, count);
+	ret = simple_read_from_buffer(buf, count, ppos, tmp_buf, copied);
+
+	kfree(tmp_buf);
+
+	return ret;
+}
+
+static const struct file_operations apb_log_fops = {
+	.read	= apb_log_read,
+};
+
+static void usb_log_enable(struct es2_ap_dev *es2)
+{
+	if (!IS_ERR_OR_NULL(es2->apb_log_task))
+		return;
+
+	/* get log from APB1 */
+	es2->apb_log_task = kthread_run(apb_log_poll, es2, "apb_log");
+	if (IS_ERR(es2->apb_log_task))
+		return;
+	/* XXX We will need to rename this per APB */
+	es2->apb_log_dentry = debugfs_create_file("apb_log", S_IRUGO,
+						gb_debugfs_get(), es2,
+						&apb_log_fops);
+}
+
+static void usb_log_disable(struct es2_ap_dev *es2)
+{
+	if (IS_ERR_OR_NULL(es2->apb_log_task))
+		return;
+
+	debugfs_remove(es2->apb_log_dentry);
+	es2->apb_log_dentry = NULL;
+
+	kthread_stop(es2->apb_log_task);
+	es2->apb_log_task = NULL;
+}
+
+static ssize_t apb_log_enable_read(struct file *f, char __user *buf,
+				size_t count, loff_t *ppos)
+{
+	struct es2_ap_dev *es2 = f->f_inode->i_private;
+	int enable = !IS_ERR_OR_NULL(es2->apb_log_task);
+	char tmp_buf[3];
+
+	sprintf(tmp_buf, "%d\n", enable);
+	return simple_read_from_buffer(buf, count, ppos, tmp_buf, 3);
+}
+
+static ssize_t apb_log_enable_write(struct file *f, const char __user *buf,
+				size_t count, loff_t *ppos)
+{
+	int enable;
+	ssize_t retval;
+	struct es2_ap_dev *es2 = f->f_inode->i_private;
+
+	retval = kstrtoint_from_user(buf, count, 10, &enable);
+	if (retval)
+		return retval;
+
+	if (enable)
+		usb_log_enable(es2);
+	else
+		usb_log_disable(es2);
+
+	return count;
+}
+
+static const struct file_operations apb_log_enable_fops = {
+	.read	= apb_log_enable_read,
+	.write	= apb_log_enable_write,
+};
+
+static int apb_get_cport_count(struct usb_device *udev)
+{
+	int retval;
+	__le16 *cport_count;
+
+	cport_count = kzalloc(sizeof(*cport_count), GFP_KERNEL);
+	if (!cport_count)
+		return -ENOMEM;
+
+	retval = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
+				 GB_APB_REQUEST_CPORT_COUNT,
+				 USB_DIR_IN | USB_TYPE_VENDOR |
+				 USB_RECIP_INTERFACE, 0, 0, cport_count,
+				 sizeof(*cport_count), ES2_USB_CTRL_TIMEOUT);
+	if (retval != sizeof(*cport_count)) {
+		dev_err(&udev->dev, "Cannot retrieve CPort count: %d\n",
+			retval);
+
+		if (retval >= 0)
+			retval = -EIO;
+
+		goto out;
+	}
+
+	retval = le16_to_cpu(*cport_count);
+
+	/* We need to fit a CPort ID in one byte of a message header */
+	if (retval > U8_MAX) {
+		retval = U8_MAX;
+		dev_warn(&udev->dev, "Limiting number of CPorts to U8_MAX\n");
+	}
+
+out:
+	kfree(cport_count);
+	return retval;
+}
+
+/*
+ * The ES2 USB Bridge device has 15 endpoints
+ * 1 Control - usual USB stuff + AP -> APBridgeA messages
+ * 7 Bulk IN - CPort data in
+ * 7 Bulk OUT - CPort data out
+ */
+static int ap_probe(struct usb_interface *interface,
+		    const struct usb_device_id *id)
+{
+	struct es2_ap_dev *es2;
+	struct gb_host_device *hd;
+	struct usb_device *udev;
+	struct usb_host_interface *iface_desc;
+	struct usb_endpoint_descriptor *endpoint;
+	__u8 ep_addr;
+	int retval;
+	int i;
+	int num_cports;
+	bool bulk_out_found = false;
+	bool bulk_in_found = false;
+	bool arpc_in_found = false;
+
+	udev = usb_get_dev(interface_to_usbdev(interface));
+
+	num_cports = apb_get_cport_count(udev);
+	if (num_cports < 0) {
+		usb_put_dev(udev);
+		dev_err(&udev->dev, "Cannot retrieve CPort count: %d\n",
+			num_cports);
+		return num_cports;
+	}
+
+	hd = gb_hd_create(&es2_driver, &udev->dev, ES2_GBUF_MSG_SIZE_MAX,
+				num_cports);
+	if (IS_ERR(hd)) {
+		usb_put_dev(udev);
+		return PTR_ERR(hd);
+	}
+
+	es2 = hd_to_es2(hd);
+	es2->hd = hd;
+	es2->usb_intf = interface;
+	es2->usb_dev = udev;
+	spin_lock_init(&es2->cport_out_urb_lock);
+	INIT_KFIFO(es2->apb_log_fifo);
+	usb_set_intfdata(interface, es2);
+
+	/*
+	 * Reserve the CDSI0 and CDSI1 CPorts so they won't be allocated
+	 * dynamically.
+	 */
+	retval = gb_hd_cport_reserve(hd, ES2_CPORT_CDSI0);
+	if (retval)
+		goto error;
+	retval = gb_hd_cport_reserve(hd, ES2_CPORT_CDSI1);
+	if (retval)
+		goto error;
+
+	/* find all bulk endpoints */
+	iface_desc = interface->cur_altsetting;
+	for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) {
+		endpoint = &iface_desc->endpoint[i].desc;
+		ep_addr = endpoint->bEndpointAddress;
+
+		if (usb_endpoint_is_bulk_in(endpoint)) {
+			if (!bulk_in_found) {
+				es2->cport_in.endpoint = ep_addr;
+				bulk_in_found = true;
+			} else if (!arpc_in_found) {
+				es2->arpc_endpoint_in = ep_addr;
+				arpc_in_found = true;
+			} else {
+				dev_warn(&udev->dev,
+					 "Unused bulk IN endpoint found: 0x%02x\n",
+					 ep_addr);
+			}
+			continue;
+		}
+		if (usb_endpoint_is_bulk_out(endpoint)) {
+			if (!bulk_out_found) {
+				es2->cport_out_endpoint = ep_addr;
+				bulk_out_found = true;
+			} else {
+				dev_warn(&udev->dev,
+					 "Unused bulk OUT endpoint found: 0x%02x\n",
+					 ep_addr);
+			}
+			continue;
+		}
+		dev_warn(&udev->dev,
+			 "Unknown endpoint type found, address 0x%02x\n",
+			 ep_addr);
+	}
+	if (!bulk_in_found || !arpc_in_found || !bulk_out_found) {
+		dev_err(&udev->dev, "Not enough endpoints found in device, aborting!\n");
+		retval = -ENODEV;
+		goto error;
+	}
+
+	/* Allocate buffers for our cport in messages */
+	for (i = 0; i < NUM_CPORT_IN_URB; ++i) {
+		struct urb *urb;
+		u8 *buffer;
+
+		urb = usb_alloc_urb(0, GFP_KERNEL);
+		if (!urb) {
+			retval = -ENOMEM;
+			goto error;
+		}
+		es2->cport_in.urb[i] = urb;
+
+		buffer = kmalloc(ES2_GBUF_MSG_SIZE_MAX, GFP_KERNEL);
+		if (!buffer) {
+			retval = -ENOMEM;
+			goto error;
+		}
+
+		usb_fill_bulk_urb(urb, udev,
+				  usb_rcvbulkpipe(udev, es2->cport_in.endpoint),
+				  buffer, ES2_GBUF_MSG_SIZE_MAX,
+				  cport_in_callback, hd);
+
+		es2->cport_in.buffer[i] = buffer;
+	}
+
+	/* Allocate buffers for ARPC in messages */
+	for (i = 0; i < NUM_ARPC_IN_URB; ++i) {
+		struct urb *urb;
+		u8 *buffer;
+
+		urb = usb_alloc_urb(0, GFP_KERNEL);
+		if (!urb) {
+			retval = -ENOMEM;
+			goto error;
+		}
+		es2->arpc_urb[i] = urb;
+
+		buffer = kmalloc(ARPC_IN_SIZE_MAX, GFP_KERNEL);
+		if (!buffer) {
+			retval = -ENOMEM;
+			goto error;
+		}
+
+		usb_fill_bulk_urb(urb, udev,
+				  usb_rcvbulkpipe(udev,
+						  es2->arpc_endpoint_in),
+				  buffer, ARPC_IN_SIZE_MAX,
+				  arpc_in_callback, es2);
+
+		es2->arpc_buffer[i] = buffer;
+	}
+
+	/* Allocate urbs for our CPort OUT messages */
+	for (i = 0; i < NUM_CPORT_OUT_URB; ++i) {
+		struct urb *urb;
+
+		urb = usb_alloc_urb(0, GFP_KERNEL);
+		if (!urb) {
+			retval = -ENOMEM;
+			goto error;
+		}
+
+		es2->cport_out_urb[i] = urb;
+		es2->cport_out_urb_busy[i] = false;	/* just to be anal */
+	}
+
+	/* XXX We will need to rename this per APB */
+	es2->apb_log_enable_dentry = debugfs_create_file("apb_log_enable",
+							(S_IWUSR | S_IRUGO),
+							gb_debugfs_get(), es2,
+							&apb_log_enable_fops);
+
+	INIT_LIST_HEAD(&es2->arpcs);
+	spin_lock_init(&es2->arpc_lock);
+
+	if (es2_arpc_in_enable(es2))
+		goto error;
+
+	retval = gb_hd_add(hd);
+	if (retval)
+		goto err_disable_arpc_in;
+
+	retval = es2_cport_in_enable(es2, &es2->cport_in);
+	if (retval)
+		goto err_hd_del;
+
+	return 0;
+
+err_hd_del:
+	gb_hd_del(hd);
+err_disable_arpc_in:
+	es2_arpc_in_disable(es2);
+error:
+	es2_destroy(es2);
+
+	return retval;
+}
+
+static void ap_disconnect(struct usb_interface *interface)
+{
+	struct es2_ap_dev *es2 = usb_get_intfdata(interface);
+
+	gb_hd_del(es2->hd);
+
+	es2_cport_in_disable(es2, &es2->cport_in);
+	es2_arpc_in_disable(es2);
+
+	es2_destroy(es2);
+}
+
+static struct usb_driver es2_ap_driver = {
+	.name =		"es2_ap_driver",
+	.probe =	ap_probe,
+	.disconnect =	ap_disconnect,
+	.id_table =	id_table,
+	.soft_unbind =	1,
+};
+
+module_usb_driver(es2_ap_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Greg Kroah-Hartman <gregkh@linuxfoundation.org>");
diff --git a/drivers/staging/greybus/firmware.h b/drivers/staging/greybus/firmware.h
new file mode 100644
index 000000000000..f4f0db1cefe8
--- /dev/null
+++ b/drivers/staging/greybus/firmware.h
@@ -0,0 +1,42 @@
+/*
+ * Greybus Firmware Management Header
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __FIRMWARE_H
+#define __FIRMWARE_H
+
+#include "greybus.h"
+
+#define FW_NAME_PREFIX	"gmp_"
+
+/*
+ * Length of the string in format: "FW_NAME_PREFIX""%08x_%08x_%08x_%08x_%s.tftf"
+ *                                  (3 + 1 + 4 * (8 + 1) + 10 + 1 + 4 + 1)
+ */
+#define FW_NAME_SIZE		56
+
+/* Firmware Management Protocol specific functions */
+int fw_mgmt_init(void);
+void fw_mgmt_exit(void);
+struct gb_connection *to_fw_mgmt_connection(struct device *dev);
+int gb_fw_mgmt_request_handler(struct gb_operation *op);
+int gb_fw_mgmt_connection_init(struct gb_connection *connection);
+void gb_fw_mgmt_connection_exit(struct gb_connection *connection);
+
+/* Firmware Download Protocol specific functions */
+int gb_fw_download_request_handler(struct gb_operation *op);
+int gb_fw_download_connection_init(struct gb_connection *connection);
+void gb_fw_download_connection_exit(struct gb_connection *connection);
+
+/* CAP Protocol specific functions */
+int cap_init(void);
+void cap_exit(void);
+int gb_cap_connection_init(struct gb_connection *connection);
+void gb_cap_connection_exit(struct gb_connection *connection);
+
+#endif /* __FIRMWARE_H */
diff --git a/drivers/staging/greybus/fw-core.c b/drivers/staging/greybus/fw-core.c
new file mode 100644
index 000000000000..454a98957ba5
--- /dev/null
+++ b/drivers/staging/greybus/fw-core.c
@@ -0,0 +1,312 @@
+/*
+ * Greybus Firmware Core Bundle Driver.
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/firmware.h>
+#include "firmware.h"
+#include "greybus.h"
+#include "spilib.h"
+
+struct gb_fw_core {
+	struct gb_connection	*download_connection;
+	struct gb_connection	*mgmt_connection;
+	struct gb_connection	*spi_connection;
+	struct gb_connection	*cap_connection;
+};
+
+static struct spilib_ops *spilib_ops;
+
+struct gb_connection *to_fw_mgmt_connection(struct device *dev)
+{
+	struct gb_fw_core *fw_core = dev_get_drvdata(dev);
+
+	return fw_core->mgmt_connection;
+}
+
+static int gb_fw_spi_connection_init(struct gb_connection *connection)
+{
+	int ret;
+
+	if (!connection)
+		return 0;
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		return ret;
+
+	ret = gb_spilib_master_init(connection, &connection->bundle->dev,
+				    spilib_ops);
+	if (ret) {
+		gb_connection_disable(connection);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void gb_fw_spi_connection_exit(struct gb_connection *connection)
+{
+	if (!connection)
+		return;
+
+	gb_spilib_master_exit(connection);
+	gb_connection_disable(connection);
+}
+
+static int gb_fw_core_probe(struct gb_bundle *bundle,
+			    const struct greybus_bundle_id *id)
+{
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_connection *connection;
+	struct gb_fw_core *fw_core;
+	int ret, i;
+	u16 cport_id;
+	u8 protocol_id;
+
+	fw_core = kzalloc(sizeof(*fw_core), GFP_KERNEL);
+	if (!fw_core)
+		return -ENOMEM;
+
+	/* Parse CPorts and create connections */
+	for (i = 0; i < bundle->num_cports; i++) {
+		cport_desc = &bundle->cport_desc[i];
+		cport_id = le16_to_cpu(cport_desc->id);
+		protocol_id = cport_desc->protocol_id;
+
+		switch (protocol_id) {
+		case GREYBUS_PROTOCOL_FW_MANAGEMENT:
+			/* Disallow multiple Firmware Management CPorts */
+			if (fw_core->mgmt_connection) {
+				dev_err(&bundle->dev,
+					"multiple management CPorts found\n");
+				ret = -EINVAL;
+				goto err_destroy_connections;
+			}
+
+			connection = gb_connection_create(bundle, cport_id,
+						gb_fw_mgmt_request_handler);
+			if (IS_ERR(connection)) {
+				ret = PTR_ERR(connection);
+				dev_err(&bundle->dev,
+					"failed to create management connection (%d)\n",
+					ret);
+				goto err_destroy_connections;
+			}
+
+			fw_core->mgmt_connection = connection;
+			break;
+		case GREYBUS_PROTOCOL_FW_DOWNLOAD:
+			/* Disallow multiple Firmware Download CPorts */
+			if (fw_core->download_connection) {
+				dev_err(&bundle->dev,
+					"multiple download CPorts found\n");
+				ret = -EINVAL;
+				goto err_destroy_connections;
+			}
+
+			connection = gb_connection_create(bundle, cport_id,
+						gb_fw_download_request_handler);
+			if (IS_ERR(connection)) {
+				dev_err(&bundle->dev, "failed to create download connection (%ld)\n",
+					PTR_ERR(connection));
+			} else {
+				fw_core->download_connection = connection;
+			}
+
+			break;
+		case GREYBUS_PROTOCOL_SPI:
+			/* Disallow multiple SPI CPorts */
+			if (fw_core->spi_connection) {
+				dev_err(&bundle->dev,
+					"multiple SPI CPorts found\n");
+				ret = -EINVAL;
+				goto err_destroy_connections;
+			}
+
+			connection = gb_connection_create(bundle, cport_id,
+							  NULL);
+			if (IS_ERR(connection)) {
+				dev_err(&bundle->dev, "failed to create SPI connection (%ld)\n",
+					PTR_ERR(connection));
+			} else {
+				fw_core->spi_connection = connection;
+			}
+
+			break;
+		case GREYBUS_PROTOCOL_AUTHENTICATION:
+			/* Disallow multiple CAP CPorts */
+			if (fw_core->cap_connection) {
+				dev_err(&bundle->dev, "multiple Authentication CPorts found\n");
+				ret = -EINVAL;
+				goto err_destroy_connections;
+			}
+
+			connection = gb_connection_create(bundle, cport_id,
+							  NULL);
+			if (IS_ERR(connection)) {
+				dev_err(&bundle->dev, "failed to create Authentication connection (%ld)\n",
+					PTR_ERR(connection));
+			} else {
+				fw_core->cap_connection = connection;
+			}
+
+			break;
+		default:
+			dev_err(&bundle->dev, "invalid protocol id (0x%02x)\n",
+				protocol_id);
+			ret = -EINVAL;
+			goto err_destroy_connections;
+		}
+	}
+
+	/* Firmware Management connection is mandatory */
+	if (!fw_core->mgmt_connection) {
+		dev_err(&bundle->dev, "missing management connection\n");
+		ret = -ENODEV;
+		goto err_destroy_connections;
+	}
+
+	ret = gb_fw_download_connection_init(fw_core->download_connection);
+	if (ret) {
+		/* We may still be able to work with the Interface */
+		dev_err(&bundle->dev, "failed to initialize firmware download connection, disable it (%d)\n",
+			ret);
+		gb_connection_destroy(fw_core->download_connection);
+		fw_core->download_connection = NULL;
+	}
+
+	ret = gb_fw_spi_connection_init(fw_core->spi_connection);
+	if (ret) {
+		/* We may still be able to work with the Interface */
+		dev_err(&bundle->dev, "failed to initialize SPI connection, disable it (%d)\n",
+			ret);
+		gb_connection_destroy(fw_core->spi_connection);
+		fw_core->spi_connection = NULL;
+	}
+
+	ret = gb_cap_connection_init(fw_core->cap_connection);
+	if (ret) {
+		/* We may still be able to work with the Interface */
+		dev_err(&bundle->dev, "failed to initialize CAP connection, disable it (%d)\n",
+			ret);
+		gb_connection_destroy(fw_core->cap_connection);
+		fw_core->cap_connection = NULL;
+	}
+
+	ret = gb_fw_mgmt_connection_init(fw_core->mgmt_connection);
+	if (ret) {
+		/* We may still be able to work with the Interface */
+		dev_err(&bundle->dev, "failed to initialize firmware management connection, disable it (%d)\n",
+			ret);
+		goto err_exit_connections;
+	}
+
+	greybus_set_drvdata(bundle, fw_core);
+
+	/* FIXME: Remove this after S2 Loader gets runtime PM support */
+	if (!(bundle->intf->quirks & GB_INTERFACE_QUIRK_NO_PM))
+		gb_pm_runtime_put_autosuspend(bundle);
+
+	return 0;
+
+err_exit_connections:
+	gb_cap_connection_exit(fw_core->cap_connection);
+	gb_fw_spi_connection_exit(fw_core->spi_connection);
+	gb_fw_download_connection_exit(fw_core->download_connection);
+err_destroy_connections:
+	gb_connection_destroy(fw_core->mgmt_connection);
+	gb_connection_destroy(fw_core->cap_connection);
+	gb_connection_destroy(fw_core->spi_connection);
+	gb_connection_destroy(fw_core->download_connection);
+	kfree(fw_core);
+
+	return ret;
+}
+
+static void gb_fw_core_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_fw_core *fw_core = greybus_get_drvdata(bundle);
+	int ret;
+
+	/* FIXME: Remove this after S2 Loader gets runtime PM support */
+	if (!(bundle->intf->quirks & GB_INTERFACE_QUIRK_NO_PM)) {
+		ret = gb_pm_runtime_get_sync(bundle);
+		if (ret)
+			gb_pm_runtime_get_noresume(bundle);
+	}
+
+	gb_fw_mgmt_connection_exit(fw_core->mgmt_connection);
+	gb_cap_connection_exit(fw_core->cap_connection);
+	gb_fw_spi_connection_exit(fw_core->spi_connection);
+	gb_fw_download_connection_exit(fw_core->download_connection);
+
+	gb_connection_destroy(fw_core->mgmt_connection);
+	gb_connection_destroy(fw_core->cap_connection);
+	gb_connection_destroy(fw_core->spi_connection);
+	gb_connection_destroy(fw_core->download_connection);
+
+	kfree(fw_core);
+}
+
+static const struct greybus_bundle_id gb_fw_core_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_FW_MANAGEMENT) },
+	{ }
+};
+
+static struct greybus_driver gb_fw_core_driver = {
+	.name		= "gb-firmware",
+	.probe		= gb_fw_core_probe,
+	.disconnect	= gb_fw_core_disconnect,
+	.id_table	= gb_fw_core_id_table,
+};
+
+static int fw_core_init(void)
+{
+	int ret;
+
+	ret = fw_mgmt_init();
+	if (ret) {
+		pr_err("Failed to initialize fw-mgmt core (%d)\n", ret);
+		return ret;
+	}
+
+	ret = cap_init();
+	if (ret) {
+		pr_err("Failed to initialize component authentication core (%d)\n",
+		       ret);
+		goto fw_mgmt_exit;
+	}
+
+	ret = greybus_register(&gb_fw_core_driver);
+	if (ret)
+		goto cap_exit;
+
+	return 0;
+
+cap_exit:
+	cap_exit();
+fw_mgmt_exit:
+	fw_mgmt_exit();
+
+	return ret;
+}
+module_init(fw_core_init);
+
+static void __exit fw_core_exit(void)
+{
+	greybus_deregister(&gb_fw_core_driver);
+	cap_exit();
+	fw_mgmt_exit();
+}
+module_exit(fw_core_exit);
+
+MODULE_ALIAS("greybus:firmware");
+MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>");
+MODULE_DESCRIPTION("Greybus Firmware Bundle Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/fw-download.c b/drivers/staging/greybus/fw-download.c
new file mode 100644
index 000000000000..2d7246887547
--- /dev/null
+++ b/drivers/staging/greybus/fw-download.c
@@ -0,0 +1,465 @@
+/*
+ * Greybus Firmware Download Protocol Driver.
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/firmware.h>
+#include <linux/jiffies.h>
+#include <linux/mutex.h>
+#include <linux/workqueue.h>
+#include "firmware.h"
+#include "greybus.h"
+
+/* Estimated minimum buffer size, actual size can be smaller than this */
+#define MIN_FETCH_SIZE		512
+/* Timeout, in jiffies, within which fetch or release firmware must be called */
+#define NEXT_REQ_TIMEOUT_J	msecs_to_jiffies(1000)
+
+struct fw_request {
+	u8			firmware_id;
+	bool			disabled;
+	bool			timedout;
+	char			name[FW_NAME_SIZE];
+	const struct firmware	*fw;
+	struct list_head	node;
+
+	struct delayed_work	dwork;
+	/* Timeout, in jiffies, within which the firmware shall download */
+	unsigned long		release_timeout_j;
+	struct kref		kref;
+	struct fw_download	*fw_download;
+};
+
+struct fw_download {
+	struct device		*parent;
+	struct gb_connection	*connection;
+	struct list_head	fw_requests;
+	struct ida		id_map;
+	struct mutex		mutex;
+};
+
+static void fw_req_release(struct kref *kref)
+{
+	struct fw_request *fw_req = container_of(kref, struct fw_request, kref);
+
+	dev_dbg(fw_req->fw_download->parent, "firmware %s released\n",
+		fw_req->name);
+
+	release_firmware(fw_req->fw);
+
+	/*
+	 * The request timed out and the module may send a fetch-fw or
+	 * release-fw request later. Lets block the id we allocated for this
+	 * request, so that the AP doesn't refer to a later fw-request (with
+	 * same firmware_id) for the old timedout fw-request.
+	 *
+	 * NOTE:
+	 *
+	 * This also means that after 255 timeouts we will fail to service new
+	 * firmware downloads. But what else can we do in that case anyway? Lets
+	 * just hope that it never happens.
+	 */
+	if (!fw_req->timedout)
+		ida_simple_remove(&fw_req->fw_download->id_map,
+				  fw_req->firmware_id);
+
+	kfree(fw_req);
+}
+
+/*
+ * Incoming requests are serialized for a connection, and the only race possible
+ * is between the timeout handler freeing this and an incoming request.
+ *
+ * The operations on the fw-request list are protected by the mutex and
+ * get_fw_req() increments the reference count before returning a fw_req pointer
+ * to the users.
+ *
+ * free_firmware() also takes the mutex while removing an entry from the list,
+ * it guarantees that every user of fw_req has taken a kref-reference by now and
+ * we wouldn't have any new users.
+ *
+ * Once the last user drops the reference, the fw_req structure is freed.
+ */
+static void put_fw_req(struct fw_request *fw_req)
+{
+	kref_put(&fw_req->kref, fw_req_release);
+}
+
+/* Caller must call put_fw_req() after using struct fw_request */
+static struct fw_request *get_fw_req(struct fw_download *fw_download,
+				     u8 firmware_id)
+{
+	struct fw_request *fw_req;
+
+	mutex_lock(&fw_download->mutex);
+
+	list_for_each_entry(fw_req, &fw_download->fw_requests, node) {
+		if (fw_req->firmware_id == firmware_id) {
+			kref_get(&fw_req->kref);
+			goto unlock;
+		}
+	}
+
+	fw_req = NULL;
+
+unlock:
+	mutex_unlock(&fw_download->mutex);
+
+	return fw_req;
+}
+
+static void free_firmware(struct fw_download *fw_download,
+			  struct fw_request *fw_req)
+{
+	/* Already disabled from timeout handlers */
+	if (fw_req->disabled)
+		return;
+
+	mutex_lock(&fw_download->mutex);
+	list_del(&fw_req->node);
+	mutex_unlock(&fw_download->mutex);
+
+	fw_req->disabled = true;
+	put_fw_req(fw_req);
+}
+
+static void fw_request_timedout(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct fw_request *fw_req = container_of(dwork, struct fw_request, dwork);
+	struct fw_download *fw_download = fw_req->fw_download;
+
+	dev_err(fw_download->parent,
+		"Timed out waiting for fetch / release firmware requests: %u\n",
+		fw_req->firmware_id);
+
+	fw_req->timedout = true;
+	free_firmware(fw_download, fw_req);
+}
+
+static int exceeds_release_timeout(struct fw_request *fw_req)
+{
+	struct fw_download *fw_download = fw_req->fw_download;
+
+	if (time_before(jiffies, fw_req->release_timeout_j))
+		return 0;
+
+	dev_err(fw_download->parent,
+		"Firmware download didn't finish in time, abort: %d\n",
+		fw_req->firmware_id);
+
+	fw_req->timedout = true;
+	free_firmware(fw_download, fw_req);
+
+	return -ETIMEDOUT;
+}
+
+/* This returns path of the firmware blob on the disk */
+static struct fw_request *find_firmware(struct fw_download *fw_download,
+					const char *tag)
+{
+	struct gb_interface *intf = fw_download->connection->bundle->intf;
+	struct fw_request *fw_req;
+	int ret, req_count;
+
+	fw_req = kzalloc(sizeof(*fw_req), GFP_KERNEL);
+	if (!fw_req)
+		return ERR_PTR(-ENOMEM);
+
+	/* Allocate ids from 1 to 255 (u8-max), 0 is an invalid id */
+	ret = ida_simple_get(&fw_download->id_map, 1, 256, GFP_KERNEL);
+	if (ret < 0) {
+		dev_err(fw_download->parent,
+			"failed to allocate firmware id (%d)\n", ret);
+		goto err_free_req;
+	}
+	fw_req->firmware_id = ret;
+
+	snprintf(fw_req->name, sizeof(fw_req->name),
+		 FW_NAME_PREFIX "%08x_%08x_%08x_%08x_%s.tftf",
+		 intf->ddbl1_manufacturer_id, intf->ddbl1_product_id,
+		 intf->vendor_id, intf->product_id, tag);
+
+	dev_info(fw_download->parent, "Requested firmware package '%s'\n",
+		 fw_req->name);
+
+	ret = request_firmware(&fw_req->fw, fw_req->name, fw_download->parent);
+	if (ret) {
+		dev_err(fw_download->parent,
+			"firmware request failed for %s (%d)\n", fw_req->name,
+			ret);
+		goto err_free_id;
+	}
+
+	fw_req->fw_download = fw_download;
+	kref_init(&fw_req->kref);
+
+	mutex_lock(&fw_download->mutex);
+	list_add(&fw_req->node, &fw_download->fw_requests);
+	mutex_unlock(&fw_download->mutex);
+
+	/* Timeout, in jiffies, within which firmware should get loaded */
+	req_count = DIV_ROUND_UP(fw_req->fw->size, MIN_FETCH_SIZE);
+	fw_req->release_timeout_j = jiffies + req_count * NEXT_REQ_TIMEOUT_J;
+
+	INIT_DELAYED_WORK(&fw_req->dwork, fw_request_timedout);
+	schedule_delayed_work(&fw_req->dwork, NEXT_REQ_TIMEOUT_J);
+
+	return fw_req;
+
+err_free_id:
+	ida_simple_remove(&fw_download->id_map, fw_req->firmware_id);
+err_free_req:
+	kfree(fw_req);
+
+	return ERR_PTR(ret);
+}
+
+static int fw_download_find_firmware(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct fw_download *fw_download = gb_connection_get_data(connection);
+	struct gb_fw_download_find_firmware_request *request;
+	struct gb_fw_download_find_firmware_response *response;
+	struct fw_request *fw_req;
+	const char *tag;
+
+	if (op->request->payload_size != sizeof(*request)) {
+		dev_err(fw_download->parent,
+			"illegal size of find firmware request (%zu != %zu)\n",
+			op->request->payload_size, sizeof(*request));
+		return -EINVAL;
+	}
+
+	request = op->request->payload;
+	tag = (const char *)request->firmware_tag;
+
+	/* firmware_tag must be null-terminated */
+	if (strnlen(tag, GB_FIRMWARE_TAG_MAX_SIZE) == GB_FIRMWARE_TAG_MAX_SIZE) {
+		dev_err(fw_download->parent,
+			"firmware-tag is not null-terminated\n");
+		return -EINVAL;
+	}
+
+	fw_req = find_firmware(fw_download, tag);
+	if (IS_ERR(fw_req))
+		return PTR_ERR(fw_req);
+
+	if (!gb_operation_response_alloc(op, sizeof(*response), GFP_KERNEL)) {
+		dev_err(fw_download->parent, "error allocating response\n");
+		free_firmware(fw_download, fw_req);
+		return -ENOMEM;
+	}
+
+	response = op->response->payload;
+	response->firmware_id = fw_req->firmware_id;
+	response->size = cpu_to_le32(fw_req->fw->size);
+
+	dev_dbg(fw_download->parent,
+		"firmware size is %zu bytes\n", fw_req->fw->size);
+
+	return 0;
+}
+
+static int fw_download_fetch_firmware(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct fw_download *fw_download = gb_connection_get_data(connection);
+	struct gb_fw_download_fetch_firmware_request *request;
+	struct gb_fw_download_fetch_firmware_response *response;
+	struct fw_request *fw_req;
+	const struct firmware *fw;
+	unsigned int offset, size;
+	u8 firmware_id;
+	int ret = 0;
+
+	if (op->request->payload_size != sizeof(*request)) {
+		dev_err(fw_download->parent,
+			"Illegal size of fetch firmware request (%zu %zu)\n",
+			op->request->payload_size, sizeof(*request));
+		return -EINVAL;
+	}
+
+	request = op->request->payload;
+	offset = le32_to_cpu(request->offset);
+	size = le32_to_cpu(request->size);
+	firmware_id = request->firmware_id;
+
+	fw_req = get_fw_req(fw_download, firmware_id);
+	if (!fw_req) {
+		dev_err(fw_download->parent,
+			"firmware not available for id: %02u\n", firmware_id);
+		return -EINVAL;
+	}
+
+	/* Make sure work handler isn't running in parallel */
+	cancel_delayed_work_sync(&fw_req->dwork);
+
+	/* We timed-out before reaching here ? */
+	if (fw_req->disabled) {
+		ret = -ETIMEDOUT;
+		goto put_fw;
+	}
+
+	/*
+	 * Firmware download must finish within a limited time interval. If it
+	 * doesn't, then we might have a buggy Module on the other side. Abort
+	 * download.
+	 */
+	ret = exceeds_release_timeout(fw_req);
+	if (ret)
+		goto put_fw;
+
+	fw = fw_req->fw;
+
+	if (offset >= fw->size || size > fw->size - offset) {
+		dev_err(fw_download->parent,
+			"bad fetch firmware request (offs = %u, size = %u)\n",
+			offset, size);
+		ret = -EINVAL;
+		goto put_fw;
+	}
+
+	if (!gb_operation_response_alloc(op, sizeof(*response) + size,
+					 GFP_KERNEL)) {
+		dev_err(fw_download->parent,
+			"error allocating fetch firmware response\n");
+		ret = -ENOMEM;
+		goto put_fw;
+	}
+
+	response = op->response->payload;
+	memcpy(response->data, fw->data + offset, size);
+
+	dev_dbg(fw_download->parent,
+		"responding with firmware (offs = %u, size = %u)\n", offset,
+		size);
+
+	/* Refresh timeout */
+	schedule_delayed_work(&fw_req->dwork, NEXT_REQ_TIMEOUT_J);
+
+put_fw:
+	put_fw_req(fw_req);
+
+	return ret;
+}
+
+static int fw_download_release_firmware(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct fw_download *fw_download = gb_connection_get_data(connection);
+	struct gb_fw_download_release_firmware_request *request;
+	struct fw_request *fw_req;
+	u8 firmware_id;
+
+	if (op->request->payload_size != sizeof(*request)) {
+		dev_err(fw_download->parent,
+			"Illegal size of release firmware request (%zu %zu)\n",
+			op->request->payload_size, sizeof(*request));
+		return -EINVAL;
+	}
+
+	request = op->request->payload;
+	firmware_id = request->firmware_id;
+
+	fw_req = get_fw_req(fw_download, firmware_id);
+	if (!fw_req) {
+		dev_err(fw_download->parent,
+			"firmware not available for id: %02u\n", firmware_id);
+		return -EINVAL;
+	}
+
+	cancel_delayed_work_sync(&fw_req->dwork);
+
+	free_firmware(fw_download, fw_req);
+	put_fw_req(fw_req);
+
+	dev_dbg(fw_download->parent, "release firmware\n");
+
+	return 0;
+}
+
+int gb_fw_download_request_handler(struct gb_operation *op)
+{
+	u8 type = op->type;
+
+	switch (type) {
+	case GB_FW_DOWNLOAD_TYPE_FIND_FIRMWARE:
+		return fw_download_find_firmware(op);
+	case GB_FW_DOWNLOAD_TYPE_FETCH_FIRMWARE:
+		return fw_download_fetch_firmware(op);
+	case GB_FW_DOWNLOAD_TYPE_RELEASE_FIRMWARE:
+		return fw_download_release_firmware(op);
+	default:
+		dev_err(&op->connection->bundle->dev,
+			"unsupported request: %u\n", type);
+		return -EINVAL;
+	}
+}
+
+int gb_fw_download_connection_init(struct gb_connection *connection)
+{
+	struct fw_download *fw_download;
+	int ret;
+
+	if (!connection)
+		return 0;
+
+	fw_download = kzalloc(sizeof(*fw_download), GFP_KERNEL);
+	if (!fw_download)
+		return -ENOMEM;
+
+	fw_download->parent = &connection->bundle->dev;
+	INIT_LIST_HEAD(&fw_download->fw_requests);
+	ida_init(&fw_download->id_map);
+	gb_connection_set_data(connection, fw_download);
+	fw_download->connection = connection;
+	mutex_init(&fw_download->mutex);
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto err_destroy_id_map;
+
+	return 0;
+
+err_destroy_id_map:
+	ida_destroy(&fw_download->id_map);
+	kfree(fw_download);
+
+	return ret;
+}
+
+void gb_fw_download_connection_exit(struct gb_connection *connection)
+{
+	struct fw_download *fw_download;
+	struct fw_request *fw_req, *tmp;
+
+	if (!connection)
+		return;
+
+	fw_download = gb_connection_get_data(connection);
+	gb_connection_disable(fw_download->connection);
+
+	/*
+	 * Make sure we have a reference to the pending requests, before they
+	 * are freed from the timeout handler.
+	 */
+	mutex_lock(&fw_download->mutex);
+	list_for_each_entry(fw_req, &fw_download->fw_requests, node)
+		kref_get(&fw_req->kref);
+	mutex_unlock(&fw_download->mutex);
+
+	/* Release pending firmware packages */
+	list_for_each_entry_safe(fw_req, tmp, &fw_download->fw_requests, node) {
+		cancel_delayed_work_sync(&fw_req->dwork);
+		free_firmware(fw_download, fw_req);
+		put_fw_req(fw_req);
+	}
+
+	ida_destroy(&fw_download->id_map);
+	kfree(fw_download);
+}
diff --git a/drivers/staging/greybus/fw-management.c b/drivers/staging/greybus/fw-management.c
new file mode 100644
index 000000000000..3cd6cf0a656b
--- /dev/null
+++ b/drivers/staging/greybus/fw-management.c
@@ -0,0 +1,721 @@
+/*
+ * Greybus Firmware Management Protocol Driver.
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/cdev.h>
+#include <linux/completion.h>
+#include <linux/firmware.h>
+#include <linux/fs.h>
+#include <linux/idr.h>
+#include <linux/ioctl.h>
+#include <linux/uaccess.h>
+
+#include "firmware.h"
+#include "greybus_firmware.h"
+#include "greybus.h"
+
+#define FW_MGMT_TIMEOUT_MS		1000
+
+struct fw_mgmt {
+	struct device		*parent;
+	struct gb_connection	*connection;
+	struct kref		kref;
+	struct list_head	node;
+
+	/* Common id-map for interface and backend firmware requests */
+	struct ida		id_map;
+	struct mutex		mutex;
+	struct completion	completion;
+	struct cdev		cdev;
+	struct device		*class_device;
+	dev_t			dev_num;
+	unsigned int		timeout_jiffies;
+	bool			disabled; /* connection getting disabled */
+
+	/* Interface Firmware specific fields */
+	bool			mode_switch_started;
+	bool			intf_fw_loaded;
+	u8			intf_fw_request_id;
+	u8			intf_fw_status;
+	u16			intf_fw_major;
+	u16			intf_fw_minor;
+
+	/* Backend Firmware specific fields */
+	u8			backend_fw_request_id;
+	u8			backend_fw_status;
+};
+
+/*
+ * Number of minor devices this driver supports.
+ * There will be exactly one required per Interface.
+ */
+#define NUM_MINORS		U8_MAX
+
+static struct class *fw_mgmt_class;
+static dev_t fw_mgmt_dev_num;
+static DEFINE_IDA(fw_mgmt_minors_map);
+static LIST_HEAD(fw_mgmt_list);
+static DEFINE_MUTEX(list_mutex);
+
+static void fw_mgmt_kref_release(struct kref *kref)
+{
+	struct fw_mgmt *fw_mgmt = container_of(kref, struct fw_mgmt, kref);
+
+	ida_destroy(&fw_mgmt->id_map);
+	kfree(fw_mgmt);
+}
+
+/*
+ * All users of fw_mgmt take a reference (from within list_mutex lock), before
+ * they get a pointer to play with. And the structure will be freed only after
+ * the last user has put the reference to it.
+ */
+static void put_fw_mgmt(struct fw_mgmt *fw_mgmt)
+{
+	kref_put(&fw_mgmt->kref, fw_mgmt_kref_release);
+}
+
+/* Caller must call put_fw_mgmt() after using struct fw_mgmt */
+static struct fw_mgmt *get_fw_mgmt(struct cdev *cdev)
+{
+	struct fw_mgmt *fw_mgmt;
+
+	mutex_lock(&list_mutex);
+
+	list_for_each_entry(fw_mgmt, &fw_mgmt_list, node) {
+		if (&fw_mgmt->cdev == cdev) {
+			kref_get(&fw_mgmt->kref);
+			goto unlock;
+		}
+	}
+
+	fw_mgmt = NULL;
+
+unlock:
+	mutex_unlock(&list_mutex);
+
+	return fw_mgmt;
+}
+
+static int fw_mgmt_interface_fw_version_operation(struct fw_mgmt *fw_mgmt,
+		struct fw_mgmt_ioc_get_intf_version *fw_info)
+{
+	struct gb_connection *connection = fw_mgmt->connection;
+	struct gb_fw_mgmt_interface_fw_version_response response;
+	int ret;
+
+	ret = gb_operation_sync(connection,
+				GB_FW_MGMT_TYPE_INTERFACE_FW_VERSION, NULL, 0,
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(fw_mgmt->parent,
+			"failed to get interface firmware version (%d)\n", ret);
+		return ret;
+	}
+
+	fw_info->major = le16_to_cpu(response.major);
+	fw_info->minor = le16_to_cpu(response.minor);
+
+	strncpy(fw_info->firmware_tag, response.firmware_tag,
+		GB_FIRMWARE_TAG_MAX_SIZE);
+
+	/*
+	 * The firmware-tag should be NULL terminated, otherwise throw error but
+	 * don't fail.
+	 */
+	if (fw_info->firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE - 1] != '\0') {
+		dev_err(fw_mgmt->parent,
+			"fw-version: firmware-tag is not NULL terminated\n");
+		fw_info->firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE - 1] = '\0';
+	}
+
+	return 0;
+}
+
+static int fw_mgmt_load_and_validate_operation(struct fw_mgmt *fw_mgmt,
+					       u8 load_method, const char *tag)
+{
+	struct gb_fw_mgmt_load_and_validate_fw_request request;
+	int ret;
+
+	if (load_method != GB_FW_LOAD_METHOD_UNIPRO &&
+	    load_method != GB_FW_LOAD_METHOD_INTERNAL) {
+		dev_err(fw_mgmt->parent,
+			"invalid load-method (%d)\n", load_method);
+		return -EINVAL;
+	}
+
+	request.load_method = load_method;
+	strncpy(request.firmware_tag, tag, GB_FIRMWARE_TAG_MAX_SIZE);
+
+	/*
+	 * The firmware-tag should be NULL terminated, otherwise throw error and
+	 * fail.
+	 */
+	if (request.firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE - 1] != '\0') {
+		dev_err(fw_mgmt->parent, "load-and-validate: firmware-tag is not NULL terminated\n");
+		return -EINVAL;
+	}
+
+	/* Allocate ids from 1 to 255 (u8-max), 0 is an invalid id */
+	ret = ida_simple_get(&fw_mgmt->id_map, 1, 256, GFP_KERNEL);
+	if (ret < 0) {
+		dev_err(fw_mgmt->parent, "failed to allocate request id (%d)\n",
+			ret);
+		return ret;
+	}
+
+	fw_mgmt->intf_fw_request_id = ret;
+	fw_mgmt->intf_fw_loaded = false;
+	request.request_id = ret;
+
+	ret = gb_operation_sync(fw_mgmt->connection,
+				GB_FW_MGMT_TYPE_LOAD_AND_VALIDATE_FW, &request,
+				sizeof(request), NULL, 0);
+	if (ret) {
+		ida_simple_remove(&fw_mgmt->id_map,
+				  fw_mgmt->intf_fw_request_id);
+		fw_mgmt->intf_fw_request_id = 0;
+		dev_err(fw_mgmt->parent,
+			"load and validate firmware request failed (%d)\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int fw_mgmt_interface_fw_loaded_operation(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct fw_mgmt *fw_mgmt = gb_connection_get_data(connection);
+	struct gb_fw_mgmt_loaded_fw_request *request;
+
+	/* No pending load and validate request ? */
+	if (!fw_mgmt->intf_fw_request_id) {
+		dev_err(fw_mgmt->parent,
+			"unexpected firmware loaded request received\n");
+		return -ENODEV;
+	}
+
+	if (op->request->payload_size != sizeof(*request)) {
+		dev_err(fw_mgmt->parent, "illegal size of firmware loaded request (%zu != %zu)\n",
+			op->request->payload_size, sizeof(*request));
+		return -EINVAL;
+	}
+
+	request = op->request->payload;
+
+	/* Invalid request-id ? */
+	if (request->request_id != fw_mgmt->intf_fw_request_id) {
+		dev_err(fw_mgmt->parent, "invalid request id for firmware loaded request (%02u != %02u)\n",
+			fw_mgmt->intf_fw_request_id, request->request_id);
+		return -ENODEV;
+	}
+
+	ida_simple_remove(&fw_mgmt->id_map, fw_mgmt->intf_fw_request_id);
+	fw_mgmt->intf_fw_request_id = 0;
+	fw_mgmt->intf_fw_status = request->status;
+	fw_mgmt->intf_fw_major = le16_to_cpu(request->major);
+	fw_mgmt->intf_fw_minor = le16_to_cpu(request->minor);
+
+	if (fw_mgmt->intf_fw_status == GB_FW_LOAD_STATUS_FAILED)
+		dev_err(fw_mgmt->parent,
+			"failed to load interface firmware, status:%02x\n",
+			fw_mgmt->intf_fw_status);
+	else if (fw_mgmt->intf_fw_status == GB_FW_LOAD_STATUS_VALIDATION_FAILED)
+		dev_err(fw_mgmt->parent,
+			"failed to validate interface firmware, status:%02x\n",
+			fw_mgmt->intf_fw_status);
+	else
+		fw_mgmt->intf_fw_loaded = true;
+
+	complete(&fw_mgmt->completion);
+
+	return 0;
+}
+
+static int fw_mgmt_backend_fw_version_operation(struct fw_mgmt *fw_mgmt,
+		struct fw_mgmt_ioc_get_backend_version *fw_info)
+{
+	struct gb_connection *connection = fw_mgmt->connection;
+	struct gb_fw_mgmt_backend_fw_version_request request;
+	struct gb_fw_mgmt_backend_fw_version_response response;
+	int ret;
+
+	strncpy(request.firmware_tag, fw_info->firmware_tag,
+		GB_FIRMWARE_TAG_MAX_SIZE);
+
+	/*
+	 * The firmware-tag should be NULL terminated, otherwise throw error and
+	 * fail.
+	 */
+	if (request.firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE - 1] != '\0') {
+		dev_err(fw_mgmt->parent, "backend-version: firmware-tag is not NULL terminated\n");
+		return -EINVAL;
+	}
+
+	ret = gb_operation_sync(connection,
+				GB_FW_MGMT_TYPE_BACKEND_FW_VERSION, &request,
+				sizeof(request), &response, sizeof(response));
+	if (ret) {
+		dev_err(fw_mgmt->parent, "failed to get version of %s backend firmware (%d)\n",
+			fw_info->firmware_tag, ret);
+		return ret;
+	}
+
+	fw_info->status = response.status;
+
+	/* Reset version as that should be non-zero only for success case */
+	fw_info->major = 0;
+	fw_info->minor = 0;
+
+	switch (fw_info->status) {
+	case GB_FW_BACKEND_VERSION_STATUS_SUCCESS:
+		fw_info->major = le16_to_cpu(response.major);
+		fw_info->minor = le16_to_cpu(response.minor);
+		break;
+	case GB_FW_BACKEND_VERSION_STATUS_NOT_AVAILABLE:
+	case GB_FW_BACKEND_VERSION_STATUS_RETRY:
+		break;
+	case GB_FW_BACKEND_VERSION_STATUS_NOT_SUPPORTED:
+		dev_err(fw_mgmt->parent,
+			"Firmware with tag %s is not supported by Interface\n",
+			fw_info->firmware_tag);
+		break;
+	default:
+		dev_err(fw_mgmt->parent, "Invalid status received: %u\n",
+			fw_info->status);
+	}
+
+	return 0;
+}
+
+static int fw_mgmt_backend_fw_update_operation(struct fw_mgmt *fw_mgmt,
+					       char *tag)
+{
+	struct gb_fw_mgmt_backend_fw_update_request request;
+	int ret;
+
+	strncpy(request.firmware_tag, tag, GB_FIRMWARE_TAG_MAX_SIZE);
+
+	/*
+	 * The firmware-tag should be NULL terminated, otherwise throw error and
+	 * fail.
+	 */
+	if (request.firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE - 1] != '\0') {
+		dev_err(fw_mgmt->parent, "backend-update: firmware-tag is not NULL terminated\n");
+		return -EINVAL;
+	}
+
+	/* Allocate ids from 1 to 255 (u8-max), 0 is an invalid id */
+	ret = ida_simple_get(&fw_mgmt->id_map, 1, 256, GFP_KERNEL);
+	if (ret < 0) {
+		dev_err(fw_mgmt->parent, "failed to allocate request id (%d)\n",
+			ret);
+		return ret;
+	}
+
+	fw_mgmt->backend_fw_request_id = ret;
+	request.request_id = ret;
+
+	ret = gb_operation_sync(fw_mgmt->connection,
+				GB_FW_MGMT_TYPE_BACKEND_FW_UPDATE, &request,
+				sizeof(request), NULL, 0);
+	if (ret) {
+		ida_simple_remove(&fw_mgmt->id_map,
+				  fw_mgmt->backend_fw_request_id);
+		fw_mgmt->backend_fw_request_id = 0;
+		dev_err(fw_mgmt->parent,
+			"backend %s firmware update request failed (%d)\n", tag,
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int fw_mgmt_backend_fw_updated_operation(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct fw_mgmt *fw_mgmt = gb_connection_get_data(connection);
+	struct gb_fw_mgmt_backend_fw_updated_request *request;
+
+	/* No pending load and validate request ? */
+	if (!fw_mgmt->backend_fw_request_id) {
+		dev_err(fw_mgmt->parent, "unexpected backend firmware updated request received\n");
+		return -ENODEV;
+	}
+
+	if (op->request->payload_size != sizeof(*request)) {
+		dev_err(fw_mgmt->parent, "illegal size of backend firmware updated request (%zu != %zu)\n",
+			op->request->payload_size, sizeof(*request));
+		return -EINVAL;
+	}
+
+	request = op->request->payload;
+
+	/* Invalid request-id ? */
+	if (request->request_id != fw_mgmt->backend_fw_request_id) {
+		dev_err(fw_mgmt->parent, "invalid request id for backend firmware updated request (%02u != %02u)\n",
+			fw_mgmt->backend_fw_request_id, request->request_id);
+		return -ENODEV;
+	}
+
+	ida_simple_remove(&fw_mgmt->id_map, fw_mgmt->backend_fw_request_id);
+	fw_mgmt->backend_fw_request_id = 0;
+	fw_mgmt->backend_fw_status = request->status;
+
+	if ((fw_mgmt->backend_fw_status != GB_FW_BACKEND_FW_STATUS_SUCCESS) &&
+	    (fw_mgmt->backend_fw_status != GB_FW_BACKEND_FW_STATUS_RETRY))
+		dev_err(fw_mgmt->parent,
+			"failed to load backend firmware: %02x\n",
+			fw_mgmt->backend_fw_status);
+
+	complete(&fw_mgmt->completion);
+
+	return 0;
+}
+
+/* Char device fops */
+
+static int fw_mgmt_open(struct inode *inode, struct file *file)
+{
+	struct fw_mgmt *fw_mgmt = get_fw_mgmt(inode->i_cdev);
+
+	/* fw_mgmt structure can't get freed until file descriptor is closed */
+	if (fw_mgmt) {
+		file->private_data = fw_mgmt;
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+static int fw_mgmt_release(struct inode *inode, struct file *file)
+{
+	struct fw_mgmt *fw_mgmt = file->private_data;
+
+	put_fw_mgmt(fw_mgmt);
+	return 0;
+}
+
+static int fw_mgmt_ioctl(struct fw_mgmt *fw_mgmt, unsigned int cmd,
+			 void __user *buf)
+{
+	struct fw_mgmt_ioc_get_intf_version intf_fw_info;
+	struct fw_mgmt_ioc_get_backend_version backend_fw_info;
+	struct fw_mgmt_ioc_intf_load_and_validate intf_load;
+	struct fw_mgmt_ioc_backend_fw_update backend_update;
+	unsigned int timeout;
+	int ret;
+
+	/* Reject any operations after mode-switch has started */
+	if (fw_mgmt->mode_switch_started)
+		return -EBUSY;
+
+	switch (cmd) {
+	case FW_MGMT_IOC_GET_INTF_FW:
+		ret = fw_mgmt_interface_fw_version_operation(fw_mgmt,
+							     &intf_fw_info);
+		if (ret)
+			return ret;
+
+		if (copy_to_user(buf, &intf_fw_info, sizeof(intf_fw_info)))
+			return -EFAULT;
+
+		return 0;
+	case FW_MGMT_IOC_GET_BACKEND_FW:
+		if (copy_from_user(&backend_fw_info, buf,
+				   sizeof(backend_fw_info)))
+			return -EFAULT;
+
+		ret = fw_mgmt_backend_fw_version_operation(fw_mgmt,
+							   &backend_fw_info);
+		if (ret)
+			return ret;
+
+		if (copy_to_user(buf, &backend_fw_info,
+				 sizeof(backend_fw_info)))
+			return -EFAULT;
+
+		return 0;
+	case FW_MGMT_IOC_INTF_LOAD_AND_VALIDATE:
+		if (copy_from_user(&intf_load, buf, sizeof(intf_load)))
+			return -EFAULT;
+
+		ret = fw_mgmt_load_and_validate_operation(fw_mgmt,
+				intf_load.load_method, intf_load.firmware_tag);
+		if (ret)
+			return ret;
+
+		if (!wait_for_completion_timeout(&fw_mgmt->completion,
+						 fw_mgmt->timeout_jiffies)) {
+			dev_err(fw_mgmt->parent, "timed out waiting for firmware load and validation to finish\n");
+			return -ETIMEDOUT;
+		}
+
+		intf_load.status = fw_mgmt->intf_fw_status;
+		intf_load.major = fw_mgmt->intf_fw_major;
+		intf_load.minor = fw_mgmt->intf_fw_minor;
+
+		if (copy_to_user(buf, &intf_load, sizeof(intf_load)))
+			return -EFAULT;
+
+		return 0;
+	case FW_MGMT_IOC_INTF_BACKEND_FW_UPDATE:
+		if (copy_from_user(&backend_update, buf,
+				   sizeof(backend_update)))
+			return -EFAULT;
+
+		ret = fw_mgmt_backend_fw_update_operation(fw_mgmt,
+				backend_update.firmware_tag);
+		if (ret)
+			return ret;
+
+		if (!wait_for_completion_timeout(&fw_mgmt->completion,
+						 fw_mgmt->timeout_jiffies)) {
+			dev_err(fw_mgmt->parent, "timed out waiting for backend firmware update to finish\n");
+			return -ETIMEDOUT;
+		}
+
+		backend_update.status = fw_mgmt->backend_fw_status;
+
+		if (copy_to_user(buf, &backend_update, sizeof(backend_update)))
+			return -EFAULT;
+
+		return 0;
+	case FW_MGMT_IOC_SET_TIMEOUT_MS:
+		if (get_user(timeout, (unsigned int __user *)buf))
+			return -EFAULT;
+
+		if (!timeout) {
+			dev_err(fw_mgmt->parent, "timeout can't be zero\n");
+			return -EINVAL;
+		}
+
+		fw_mgmt->timeout_jiffies = msecs_to_jiffies(timeout);
+
+		return 0;
+	case FW_MGMT_IOC_MODE_SWITCH:
+		if (!fw_mgmt->intf_fw_loaded) {
+			dev_err(fw_mgmt->parent,
+				"Firmware not loaded for mode-switch\n");
+			return -EPERM;
+		}
+
+		/*
+		 * Disallow new ioctls as the fw-core bundle driver is going to
+		 * get disconnected soon and the character device will get
+		 * removed.
+		 */
+		fw_mgmt->mode_switch_started = true;
+
+		ret = gb_interface_request_mode_switch(fw_mgmt->connection->intf);
+		if (ret) {
+			dev_err(fw_mgmt->parent, "Mode-switch failed: %d\n",
+				ret);
+			fw_mgmt->mode_switch_started = false;
+			return ret;
+		}
+
+		return 0;
+	default:
+		return -ENOTTY;
+	}
+}
+
+static long fw_mgmt_ioctl_unlocked(struct file *file, unsigned int cmd,
+				   unsigned long arg)
+{
+	struct fw_mgmt *fw_mgmt = file->private_data;
+	struct gb_bundle *bundle = fw_mgmt->connection->bundle;
+	int ret = -ENODEV;
+
+	/*
+	 * Serialize ioctls.
+	 *
+	 * We don't want the user to do few operations in parallel. For example,
+	 * updating Interface firmware in parallel for the same Interface. There
+	 * is no need to do things in parallel for speed and we can avoid having
+	 * complicated code for now.
+	 *
+	 * This is also used to protect ->disabled, which is used to check if
+	 * the connection is getting disconnected, so that we don't start any
+	 * new operations.
+	 */
+	mutex_lock(&fw_mgmt->mutex);
+	if (!fw_mgmt->disabled) {
+		ret = gb_pm_runtime_get_sync(bundle);
+		if (!ret) {
+			ret = fw_mgmt_ioctl(fw_mgmt, cmd, (void __user *)arg);
+			gb_pm_runtime_put_autosuspend(bundle);
+		}
+	}
+	mutex_unlock(&fw_mgmt->mutex);
+
+	return ret;
+}
+
+static const struct file_operations fw_mgmt_fops = {
+	.owner		= THIS_MODULE,
+	.open		= fw_mgmt_open,
+	.release	= fw_mgmt_release,
+	.unlocked_ioctl	= fw_mgmt_ioctl_unlocked,
+};
+
+int gb_fw_mgmt_request_handler(struct gb_operation *op)
+{
+	u8 type = op->type;
+
+	switch (type) {
+	case GB_FW_MGMT_TYPE_LOADED_FW:
+		return fw_mgmt_interface_fw_loaded_operation(op);
+	case GB_FW_MGMT_TYPE_BACKEND_FW_UPDATED:
+		return fw_mgmt_backend_fw_updated_operation(op);
+	default:
+		dev_err(&op->connection->bundle->dev,
+			"unsupported request: %u\n", type);
+		return -EINVAL;
+	}
+}
+
+int gb_fw_mgmt_connection_init(struct gb_connection *connection)
+{
+	struct fw_mgmt *fw_mgmt;
+	int ret, minor;
+
+	if (!connection)
+		return 0;
+
+	fw_mgmt = kzalloc(sizeof(*fw_mgmt), GFP_KERNEL);
+	if (!fw_mgmt)
+		return -ENOMEM;
+
+	fw_mgmt->parent = &connection->bundle->dev;
+	fw_mgmt->timeout_jiffies = msecs_to_jiffies(FW_MGMT_TIMEOUT_MS);
+	fw_mgmt->connection = connection;
+
+	gb_connection_set_data(connection, fw_mgmt);
+	init_completion(&fw_mgmt->completion);
+	ida_init(&fw_mgmt->id_map);
+	mutex_init(&fw_mgmt->mutex);
+	kref_init(&fw_mgmt->kref);
+
+	mutex_lock(&list_mutex);
+	list_add(&fw_mgmt->node, &fw_mgmt_list);
+	mutex_unlock(&list_mutex);
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto err_list_del;
+
+	minor = ida_simple_get(&fw_mgmt_minors_map, 0, NUM_MINORS, GFP_KERNEL);
+	if (minor < 0) {
+		ret = minor;
+		goto err_connection_disable;
+	}
+
+	/* Add a char device to allow userspace to interact with fw-mgmt */
+	fw_mgmt->dev_num = MKDEV(MAJOR(fw_mgmt_dev_num), minor);
+	cdev_init(&fw_mgmt->cdev, &fw_mgmt_fops);
+
+	ret = cdev_add(&fw_mgmt->cdev, fw_mgmt->dev_num, 1);
+	if (ret)
+		goto err_remove_ida;
+
+	/* Add a soft link to the previously added char-dev within the bundle */
+	fw_mgmt->class_device = device_create(fw_mgmt_class, fw_mgmt->parent,
+					      fw_mgmt->dev_num, NULL,
+					      "gb-fw-mgmt-%d", minor);
+	if (IS_ERR(fw_mgmt->class_device)) {
+		ret = PTR_ERR(fw_mgmt->class_device);
+		goto err_del_cdev;
+	}
+
+	return 0;
+
+err_del_cdev:
+	cdev_del(&fw_mgmt->cdev);
+err_remove_ida:
+	ida_simple_remove(&fw_mgmt_minors_map, minor);
+err_connection_disable:
+	gb_connection_disable(connection);
+err_list_del:
+	mutex_lock(&list_mutex);
+	list_del(&fw_mgmt->node);
+	mutex_unlock(&list_mutex);
+
+	put_fw_mgmt(fw_mgmt);
+
+	return ret;
+}
+
+void gb_fw_mgmt_connection_exit(struct gb_connection *connection)
+{
+	struct fw_mgmt *fw_mgmt;
+
+	if (!connection)
+		return;
+
+	fw_mgmt = gb_connection_get_data(connection);
+
+	device_destroy(fw_mgmt_class, fw_mgmt->dev_num);
+	cdev_del(&fw_mgmt->cdev);
+	ida_simple_remove(&fw_mgmt_minors_map, MINOR(fw_mgmt->dev_num));
+
+	/*
+	 * Disallow any new ioctl operations on the char device and wait for
+	 * existing ones to finish.
+	 */
+	mutex_lock(&fw_mgmt->mutex);
+	fw_mgmt->disabled = true;
+	mutex_unlock(&fw_mgmt->mutex);
+
+	/* All pending greybus operations should have finished by now */
+	gb_connection_disable(fw_mgmt->connection);
+
+	/* Disallow new users to get access to the fw_mgmt structure */
+	mutex_lock(&list_mutex);
+	list_del(&fw_mgmt->node);
+	mutex_unlock(&list_mutex);
+
+	/*
+	 * All current users of fw_mgmt would have taken a reference to it by
+	 * now, we can drop our reference and wait the last user will get
+	 * fw_mgmt freed.
+	 */
+	put_fw_mgmt(fw_mgmt);
+}
+
+int fw_mgmt_init(void)
+{
+	int ret;
+
+	fw_mgmt_class = class_create(THIS_MODULE, "gb_fw_mgmt");
+	if (IS_ERR(fw_mgmt_class))
+		return PTR_ERR(fw_mgmt_class);
+
+	ret = alloc_chrdev_region(&fw_mgmt_dev_num, 0, NUM_MINORS,
+				  "gb_fw_mgmt");
+	if (ret)
+		goto err_remove_class;
+
+	return 0;
+
+err_remove_class:
+	class_destroy(fw_mgmt_class);
+	return ret;
+}
+
+void fw_mgmt_exit(void)
+{
+	unregister_chrdev_region(fw_mgmt_dev_num, NUM_MINORS);
+	class_destroy(fw_mgmt_class);
+	ida_destroy(&fw_mgmt_minors_map);
+}
diff --git a/drivers/staging/greybus/gb-camera.h b/drivers/staging/greybus/gb-camera.h
new file mode 100644
index 000000000000..d45dabc5b367
--- /dev/null
+++ b/drivers/staging/greybus/gb-camera.h
@@ -0,0 +1,127 @@
+/*
+ * Greybus Camera protocol driver.
+ *
+ * Copyright 2015 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+#ifndef __GB_CAMERA_H
+#define __GB_CAMERA_H
+
+#include <linux/v4l2-mediabus.h>
+
+/* Input flags need to be set from the caller */
+#define GB_CAMERA_IN_FLAG_TEST		(1 << 0)
+/* Output flags returned */
+#define GB_CAMERA_OUT_FLAG_ADJUSTED	(1 << 0)
+
+/**
+ * struct gb_camera_stream - Represents greybus camera stream.
+ * @width: Stream width in pixels.
+ * @height: Stream height in pixels.
+ * @pixel_code: Media bus pixel code.
+ * @vc: MIPI CSI virtual channel.
+ * @dt: MIPI CSI data types. Most formats use a single data type, in which case
+ *      the second element will be ignored.
+ * @max_size: Maximum size of a frame in bytes. The camera module guarantees
+ *            that all data between the Frame Start and Frame End packet for
+ *            the associated virtual channel and data type(s) will not exceed
+ *            this size.
+ */
+struct gb_camera_stream {
+	unsigned int width;
+	unsigned int height;
+	enum v4l2_mbus_pixelcode pixel_code;
+	unsigned int vc;
+	unsigned int dt[2];
+	unsigned int max_size;
+};
+
+/**
+ * struct gb_camera_csi_params - CSI configuration parameters
+ * @num_lanes: number of CSI data lanes
+ * @clk_freq: CSI clock frequency in Hz
+ */
+struct gb_camera_csi_params {
+	unsigned int num_lanes;
+	unsigned int clk_freq;
+};
+
+/**
+ * struct gb_camera_ops - Greybus camera operations, used by the Greybus camera
+ *                        driver to expose operations to the host camera driver.
+ * @capabilities: Retrieve camera capabilities and store them in the buffer
+ *                'buf' capabilities. The buffer maximum size is specified by
+ *                the caller in the 'size' parameter, and the effective
+ *                capabilities size is returned from the function. If the buffer
+ *                size is too small to hold the capabilities an error is
+ *                returned and the buffer is left untouched.
+ *
+ * @configure_streams: Negotiate configuration and prepare the module for video
+ *                     capture. The caller specifies the number of streams it
+ *                     requests in the 'nstreams' argument and the associated
+ *                     streams configurations in the 'streams' argument. The
+ *                     GB_CAMERA_IN_FLAG_TEST 'flag' can be set to test a
+ *                     configuration without applying it, otherwise the
+ *                     configuration is applied by the module. The module can
+ *                     decide to modify the requested configuration, including
+ *                     using a different number of streams. In that case the
+ *                     modified configuration won't be applied, the
+ *                     GB_CAMERA_OUT_FLAG_ADJUSTED 'flag' will be set upon
+ *                     return, and the modified configuration and number of
+ *                     streams stored in 'streams' and 'array'. The module
+ *                     returns its CSI-2 bus parameters in the 'csi_params'
+ *                     structure in all cases.
+ *
+ * @capture: Submit a capture request. The supplied 'request_id' must be unique
+ *           and higher than the IDs of all the previously submitted requests.
+ *           The 'streams' argument specifies which streams are affected by the
+ *           request in the form of a bitmask, with bits corresponding to the
+ *           configured streams indexes. If the request contains settings, the
+ *           'settings' argument points to the settings buffer and its size is
+ *           specified by the 'settings_size' argument. Otherwise the 'settings'
+ *           argument should be set to NULL and 'settings_size' to 0.
+ *
+ * @flush: Flush the capture requests queue. Return the ID of the last request
+ *         that will processed by the device before it stops transmitting video
+ *         frames. All queued capture requests with IDs higher than the returned
+ *         ID will be dropped without being processed.
+ */
+struct gb_camera_ops {
+	ssize_t (*capabilities)(void *priv, char *buf, size_t len);
+	int (*configure_streams)(void *priv, unsigned int *nstreams,
+			unsigned int *flags, struct gb_camera_stream *streams,
+			struct gb_camera_csi_params *csi_params);
+	int (*capture)(void *priv, u32 request_id,
+			unsigned int streams, unsigned int num_frames,
+			size_t settings_size, const void *settings);
+	int (*flush)(void *priv, u32 *request_id);
+};
+
+/**
+ * struct gb_camera_module - Represents greybus camera module.
+ * @priv: Module private data, passed to all camera operations.
+ * @ops: Greybus camera operation callbacks.
+ * @interface_id: Interface id of the module.
+ * @refcount: Reference counting object.
+ * @release: Module release function.
+ * @list: List entry in the camera modules list.
+ */
+struct gb_camera_module {
+	void *priv;
+	const struct gb_camera_ops *ops;
+
+	unsigned int interface_id;
+	struct kref refcount;
+	void (*release)(struct kref *kref);
+	struct list_head list; /* Global list */
+};
+
+#define gb_camera_call(f, op, args...)      \
+	(!(f) ? -ENODEV : (((f)->ops->op) ?  \
+	(f)->ops->op((f)->priv, ##args) : -ENOIOCTLCMD))
+
+int gb_camera_register(struct gb_camera_module *module);
+int gb_camera_unregister(struct gb_camera_module *module);
+
+#endif /* __GB_CAMERA_H */
diff --git a/drivers/staging/greybus/gbphy.c b/drivers/staging/greybus/gbphy.c
new file mode 100644
index 000000000000..bcde7c9a0f17
--- /dev/null
+++ b/drivers/staging/greybus/gbphy.c
@@ -0,0 +1,360 @@
+/*
+ * Greybus Bridged-Phy Bus driver
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/device.h>
+
+#include "greybus.h"
+#include "gbphy.h"
+
+#define GB_GBPHY_AUTOSUSPEND_MS	3000
+
+struct gbphy_host {
+	struct gb_bundle *bundle;
+	struct list_head devices;
+};
+
+static DEFINE_IDA(gbphy_id);
+
+static ssize_t protocol_id_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct gbphy_device *gbphy_dev = to_gbphy_dev(dev);
+
+	return sprintf(buf, "0x%02x\n", gbphy_dev->cport_desc->protocol_id);
+}
+static DEVICE_ATTR_RO(protocol_id);
+
+static struct attribute *gbphy_dev_attrs[] = {
+	&dev_attr_protocol_id.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(gbphy_dev);
+
+static void gbphy_dev_release(struct device *dev)
+{
+	struct gbphy_device *gbphy_dev = to_gbphy_dev(dev);
+
+	ida_simple_remove(&gbphy_id, gbphy_dev->id);
+	kfree(gbphy_dev);
+}
+
+#ifdef CONFIG_PM
+static int gb_gbphy_idle(struct device *dev)
+{
+	pm_runtime_mark_last_busy(dev);
+	pm_request_autosuspend(dev);
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops gb_gbphy_pm_ops = {
+	SET_RUNTIME_PM_OPS(pm_generic_runtime_suspend,
+			   pm_generic_runtime_resume,
+			   gb_gbphy_idle)
+};
+
+static struct device_type greybus_gbphy_dev_type = {
+	.name	 =	"gbphy_device",
+	.release =	gbphy_dev_release,
+	.pm	=	&gb_gbphy_pm_ops,
+};
+
+static int gbphy_dev_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+	struct gbphy_device *gbphy_dev = to_gbphy_dev(dev);
+	struct greybus_descriptor_cport *cport_desc = gbphy_dev->cport_desc;
+	struct gb_bundle *bundle = gbphy_dev->bundle;
+	struct gb_interface *intf = bundle->intf;
+	struct gb_module *module = intf->module;
+	struct gb_host_device *hd = intf->hd;
+
+	if (add_uevent_var(env, "BUS=%u", hd->bus_id))
+		return -ENOMEM;
+	if (add_uevent_var(env, "MODULE=%u", module->module_id))
+		return -ENOMEM;
+	if (add_uevent_var(env, "INTERFACE=%u", intf->interface_id))
+		return -ENOMEM;
+	if (add_uevent_var(env, "GREYBUS_ID=%08x/%08x",
+			   intf->vendor_id, intf->product_id))
+		return -ENOMEM;
+	if (add_uevent_var(env, "BUNDLE=%u", gbphy_dev->bundle->id))
+		return -ENOMEM;
+	if (add_uevent_var(env, "BUNDLE_CLASS=%02x", bundle->class))
+		return -ENOMEM;
+	if (add_uevent_var(env, "GBPHY=%u", gbphy_dev->id))
+		return -ENOMEM;
+	if (add_uevent_var(env, "PROTOCOL_ID=%02x", cport_desc->protocol_id))
+		return -ENOMEM;
+
+	return 0;
+}
+
+static const struct gbphy_device_id *
+gbphy_dev_match_id(struct gbphy_device *gbphy_dev, struct gbphy_driver *gbphy_drv)
+{
+	const struct gbphy_device_id *id = gbphy_drv->id_table;
+
+	if (!id)
+		return NULL;
+
+	for (; id->protocol_id; id++)
+		if (id->protocol_id == gbphy_dev->cport_desc->protocol_id)
+			return id;
+
+	return NULL;
+}
+
+static int gbphy_dev_match(struct device *dev, struct device_driver *drv)
+{
+	struct gbphy_driver *gbphy_drv = to_gbphy_driver(drv);
+	struct gbphy_device *gbphy_dev = to_gbphy_dev(dev);
+	const struct gbphy_device_id *id;
+
+	id = gbphy_dev_match_id(gbphy_dev, gbphy_drv);
+	if (id)
+		return 1;
+
+	return 0;
+}
+
+static int gbphy_dev_probe(struct device *dev)
+{
+	struct gbphy_driver *gbphy_drv = to_gbphy_driver(dev->driver);
+	struct gbphy_device *gbphy_dev = to_gbphy_dev(dev);
+	const struct gbphy_device_id *id;
+	int ret;
+
+	id = gbphy_dev_match_id(gbphy_dev, gbphy_drv);
+	if (!id)
+		return -ENODEV;
+
+	/* for old kernels we need get_sync to resume parent devices */
+	ret = gb_pm_runtime_get_sync(gbphy_dev->bundle);
+	if (ret < 0)
+		return ret;
+
+	pm_runtime_set_autosuspend_delay(dev, GB_GBPHY_AUTOSUSPEND_MS);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_get_noresume(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
+	/*
+	 * Drivers should call put on the gbphy dev before returning
+	 * from probe if they support runtime pm.
+	 */
+	ret = gbphy_drv->probe(gbphy_dev, id);
+	if (ret) {
+		pm_runtime_disable(dev);
+		pm_runtime_set_suspended(dev);
+		pm_runtime_put_noidle(dev);
+		pm_runtime_dont_use_autosuspend(dev);
+	}
+
+	gb_pm_runtime_put_autosuspend(gbphy_dev->bundle);
+
+	return ret;
+}
+
+static int gbphy_dev_remove(struct device *dev)
+{
+	struct gbphy_driver *gbphy_drv = to_gbphy_driver(dev->driver);
+	struct gbphy_device *gbphy_dev = to_gbphy_dev(dev);
+
+	gbphy_drv->remove(gbphy_dev);
+
+	pm_runtime_disable(dev);
+	pm_runtime_set_suspended(dev);
+	pm_runtime_put_noidle(dev);
+	pm_runtime_dont_use_autosuspend(dev);
+
+	return 0;
+}
+
+static struct bus_type gbphy_bus_type = {
+	.name =		"gbphy",
+	.match =	gbphy_dev_match,
+	.probe =	gbphy_dev_probe,
+	.remove =	gbphy_dev_remove,
+	.uevent =	gbphy_dev_uevent,
+};
+
+int gb_gbphy_register_driver(struct gbphy_driver *driver,
+			     struct module *owner, const char *mod_name)
+{
+	int retval;
+
+	if (greybus_disabled())
+		return -ENODEV;
+
+	driver->driver.bus = &gbphy_bus_type;
+	driver->driver.name = driver->name;
+	driver->driver.owner = owner;
+	driver->driver.mod_name = mod_name;
+
+	retval = driver_register(&driver->driver);
+	if (retval)
+		return retval;
+
+	pr_info("registered new driver %s\n", driver->name);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gb_gbphy_register_driver);
+
+void gb_gbphy_deregister_driver(struct gbphy_driver *driver)
+{
+	driver_unregister(&driver->driver);
+}
+EXPORT_SYMBOL_GPL(gb_gbphy_deregister_driver);
+
+static struct gbphy_device *gb_gbphy_create_dev(struct gb_bundle *bundle,
+				struct greybus_descriptor_cport *cport_desc)
+{
+	struct gbphy_device *gbphy_dev;
+	int retval;
+	int id;
+
+	id = ida_simple_get(&gbphy_id, 1, 0, GFP_KERNEL);
+	if (id < 0)
+		return ERR_PTR(id);
+
+	gbphy_dev = kzalloc(sizeof(*gbphy_dev), GFP_KERNEL);
+	if (!gbphy_dev) {
+		ida_simple_remove(&gbphy_id, id);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	gbphy_dev->id = id;
+	gbphy_dev->bundle = bundle;
+	gbphy_dev->cport_desc = cport_desc;
+	gbphy_dev->dev.parent = &bundle->dev;
+	gbphy_dev->dev.bus = &gbphy_bus_type;
+	gbphy_dev->dev.type = &greybus_gbphy_dev_type;
+	gbphy_dev->dev.groups = gbphy_dev_groups;
+	gbphy_dev->dev.dma_mask = bundle->dev.dma_mask;
+	dev_set_name(&gbphy_dev->dev, "gbphy%d", id);
+
+	retval = device_register(&gbphy_dev->dev);
+	if (retval) {
+		put_device(&gbphy_dev->dev);
+		return ERR_PTR(retval);
+	}
+
+	return gbphy_dev;
+}
+
+static void gb_gbphy_disconnect(struct gb_bundle *bundle)
+{
+	struct gbphy_host *gbphy_host = greybus_get_drvdata(bundle);
+	struct gbphy_device *gbphy_dev, *temp;
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret < 0)
+		gb_pm_runtime_get_noresume(bundle);
+
+	list_for_each_entry_safe(gbphy_dev, temp, &gbphy_host->devices, list) {
+		list_del(&gbphy_dev->list);
+		device_unregister(&gbphy_dev->dev);
+	}
+
+	kfree(gbphy_host);
+}
+
+static int gb_gbphy_probe(struct gb_bundle *bundle,
+			  const struct greybus_bundle_id *id)
+{
+	struct gbphy_host *gbphy_host;
+	struct gbphy_device *gbphy_dev;
+	int i;
+
+	if (bundle->num_cports == 0)
+		return -ENODEV;
+
+	gbphy_host = kzalloc(sizeof(*gbphy_host), GFP_KERNEL);
+	if (!gbphy_host)
+		return -ENOMEM;
+
+	gbphy_host->bundle = bundle;
+	INIT_LIST_HEAD(&gbphy_host->devices);
+	greybus_set_drvdata(bundle, gbphy_host);
+
+	/*
+	 * Create a bunch of children devices, one per cport, and bind the
+	 * bridged phy drivers to them.
+	 */
+	for (i = 0; i < bundle->num_cports; ++i) {
+		gbphy_dev = gb_gbphy_create_dev(bundle, &bundle->cport_desc[i]);
+		if (IS_ERR(gbphy_dev)) {
+			gb_gbphy_disconnect(bundle);
+			return PTR_ERR(gbphy_dev);
+		}
+		list_add(&gbphy_dev->list, &gbphy_host->devices);
+	}
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return 0;
+}
+
+static const struct greybus_bundle_id gb_gbphy_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_BRIDGED_PHY) },
+	{ },
+};
+MODULE_DEVICE_TABLE(greybus, gb_gbphy_id_table);
+
+static struct greybus_driver gb_gbphy_driver = {
+	.name		= "gbphy",
+	.probe		= gb_gbphy_probe,
+	.disconnect	= gb_gbphy_disconnect,
+	.id_table	= gb_gbphy_id_table,
+};
+
+static int __init gbphy_init(void)
+{
+	int retval;
+
+	retval = bus_register(&gbphy_bus_type);
+	if (retval) {
+		pr_err("gbphy bus register failed (%d)\n", retval);
+		return retval;
+	}
+
+	retval = greybus_register(&gb_gbphy_driver);
+	if (retval) {
+		pr_err("error registering greybus driver\n");
+		goto error_gbphy;
+	}
+
+	return 0;
+
+error_gbphy:
+	bus_unregister(&gbphy_bus_type);
+	ida_destroy(&gbphy_id);
+	return retval;
+}
+module_init(gbphy_init);
+
+static void __exit gbphy_exit(void)
+{
+	greybus_deregister(&gb_gbphy_driver);
+	bus_unregister(&gbphy_bus_type);
+	ida_destroy(&gbphy_id);
+}
+module_exit(gbphy_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/gbphy.h b/drivers/staging/greybus/gbphy.h
new file mode 100644
index 000000000000..8ee68055ccc4
--- /dev/null
+++ b/drivers/staging/greybus/gbphy.h
@@ -0,0 +1,110 @@
+/*
+ * Greybus Bridged-Phy Bus driver
+ *
+ * Copyright 2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __GBPHY_H
+#define __GBPHY_H
+
+struct gbphy_device {
+	u32 id;
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_bundle *bundle;
+	struct list_head list;
+	struct device dev;
+};
+#define to_gbphy_dev(d) container_of(d, struct gbphy_device, dev)
+
+static inline void *gb_gbphy_get_data(struct gbphy_device *gdev)
+{
+	return dev_get_drvdata(&gdev->dev);
+}
+
+static inline void gb_gbphy_set_data(struct gbphy_device *gdev, void *data)
+{
+	dev_set_drvdata(&gdev->dev, data);
+}
+
+struct gbphy_device_id {
+	__u8 protocol_id;
+};
+
+#define GBPHY_PROTOCOL(p)		\
+	.protocol_id	= (p),
+
+struct gbphy_driver {
+	const char *name;
+	int (*probe)(struct gbphy_device *,
+		     const struct gbphy_device_id *id);
+	void (*remove)(struct gbphy_device *);
+	const struct gbphy_device_id *id_table;
+
+	struct device_driver driver;
+};
+#define to_gbphy_driver(d) container_of(d, struct gbphy_driver, driver)
+
+int gb_gbphy_register_driver(struct gbphy_driver *driver,
+			     struct module *owner, const char *mod_name);
+void gb_gbphy_deregister_driver(struct gbphy_driver *driver);
+
+#define gb_gbphy_register(driver) \
+	gb_gbphy_register_driver(driver, THIS_MODULE, KBUILD_MODNAME)
+#define gb_gbphy_deregister(driver) \
+	gb_gbphy_deregister_driver(driver)
+
+/**
+ * module_gbphy_driver() - Helper macro for registering a gbphy driver
+ * @__gbphy_driver: gbphy_driver structure
+ *
+ * Helper macro for gbphy drivers to set up proper module init / exit
+ * functions.  Replaces module_init() and module_exit() and keeps people from
+ * printing pointless things to the kernel log when their driver is loaded.
+ */
+#define module_gbphy_driver(__gbphy_driver)	\
+	module_driver(__gbphy_driver, gb_gbphy_register, gb_gbphy_deregister)
+
+#ifdef CONFIG_PM
+static inline int gbphy_runtime_get_sync(struct gbphy_device *gbphy_dev)
+{
+	struct device *dev = &gbphy_dev->dev;
+	int ret;
+
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "pm_runtime_get_sync failed: %d\n", ret);
+		pm_runtime_put_noidle(dev);
+		return ret;
+	}
+
+	return 0;
+}
+
+static inline void gbphy_runtime_put_autosuspend(struct gbphy_device *gbphy_dev)
+{
+	struct device *dev = &gbphy_dev->dev;
+
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+}
+
+static inline void gbphy_runtime_get_noresume(struct gbphy_device *gbphy_dev)
+{
+	pm_runtime_get_noresume(&gbphy_dev->dev);
+}
+
+static inline void gbphy_runtime_put_noidle(struct gbphy_device *gbphy_dev)
+{
+	pm_runtime_put_noidle(&gbphy_dev->dev);
+}
+#else
+static inline int gbphy_runtime_get_sync(struct gbphy_device *gbphy_dev) { return 0; }
+static inline void gbphy_runtime_put_autosuspend(struct gbphy_device *gbphy_dev) {}
+static inline void gbphy_runtime_get_noresume(struct gbphy_device *gbphy_dev) {}
+static inline void gbphy_runtime_put_noidle(struct gbphy_device *gbphy_dev) {}
+#endif
+
+#endif /* __GBPHY_H */
+
diff --git a/drivers/staging/greybus/gpio.c b/drivers/staging/greybus/gpio.c
new file mode 100644
index 000000000000..5e06e4229e42
--- /dev/null
+++ b/drivers/staging/greybus/gpio.c
@@ -0,0 +1,766 @@
+/*
+ * GPIO Greybus driver.
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/gpio.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/mutex.h>
+
+#include "greybus.h"
+#include "gbphy.h"
+
+struct gb_gpio_line {
+	/* The following has to be an array of line_max entries */
+	/* --> make them just a flags field */
+	u8			active:    1,
+				direction: 1,	/* 0 = output, 1 = input */
+				value:     1;	/* 0 = low, 1 = high */
+	u16			debounce_usec;
+
+	u8			irq_type;
+	bool			irq_type_pending;
+	bool			masked;
+	bool			masked_pending;
+};
+
+struct gb_gpio_controller {
+	struct gbphy_device	*gbphy_dev;
+	struct gb_connection	*connection;
+	u8			line_max;	/* max line number */
+	struct gb_gpio_line	*lines;
+
+	struct gpio_chip	chip;
+	struct irq_chip		irqc;
+	struct irq_chip		*irqchip;
+	struct irq_domain	*irqdomain;
+	unsigned int		irq_base;
+	irq_flow_handler_t	irq_handler;
+	unsigned int		irq_default_type;
+	struct mutex		irq_lock;
+};
+#define gpio_chip_to_gb_gpio_controller(chip) \
+	container_of(chip, struct gb_gpio_controller, chip)
+#define irq_data_to_gpio_chip(d) (d->domain->host_data)
+
+static int gb_gpio_line_count_operation(struct gb_gpio_controller *ggc)
+{
+	struct gb_gpio_line_count_response response;
+	int ret;
+
+	ret = gb_operation_sync(ggc->connection, GB_GPIO_TYPE_LINE_COUNT,
+				NULL, 0, &response, sizeof(response));
+	if (!ret)
+		ggc->line_max = response.count;
+	return ret;
+}
+
+static int gb_gpio_activate_operation(struct gb_gpio_controller *ggc, u8 which)
+{
+	struct gb_gpio_activate_request request;
+	struct gbphy_device *gbphy_dev = ggc->gbphy_dev;
+	int ret;
+
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		return ret;
+
+	request.which = which;
+	ret = gb_operation_sync(ggc->connection, GB_GPIO_TYPE_ACTIVATE,
+				 &request, sizeof(request), NULL, 0);
+	if (ret) {
+		gbphy_runtime_put_autosuspend(gbphy_dev);
+		return ret;
+	}
+
+	ggc->lines[which].active = true;
+
+	return 0;
+}
+
+static void gb_gpio_deactivate_operation(struct gb_gpio_controller *ggc,
+					u8 which)
+{
+	struct gbphy_device *gbphy_dev = ggc->gbphy_dev;
+	struct device *dev = &gbphy_dev->dev;
+	struct gb_gpio_deactivate_request request;
+	int ret;
+
+	request.which = which;
+	ret = gb_operation_sync(ggc->connection, GB_GPIO_TYPE_DEACTIVATE,
+				 &request, sizeof(request), NULL, 0);
+	if (ret) {
+		dev_err(dev, "failed to deactivate gpio %u\n", which);
+		goto out_pm_put;
+	}
+
+	ggc->lines[which].active = false;
+
+out_pm_put:
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+}
+
+static int gb_gpio_get_direction_operation(struct gb_gpio_controller *ggc,
+					u8 which)
+{
+	struct device *dev = &ggc->gbphy_dev->dev;
+	struct gb_gpio_get_direction_request request;
+	struct gb_gpio_get_direction_response response;
+	int ret;
+	u8 direction;
+
+	request.which = which;
+	ret = gb_operation_sync(ggc->connection, GB_GPIO_TYPE_GET_DIRECTION,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret)
+		return ret;
+
+	direction = response.direction;
+	if (direction && direction != 1) {
+		dev_warn(dev, "gpio %u direction was %u (should be 0 or 1)\n",
+			 which, direction);
+	}
+	ggc->lines[which].direction = direction ? 1 : 0;
+	return 0;
+}
+
+static int gb_gpio_direction_in_operation(struct gb_gpio_controller *ggc,
+					u8 which)
+{
+	struct gb_gpio_direction_in_request request;
+	int ret;
+
+	request.which = which;
+	ret = gb_operation_sync(ggc->connection, GB_GPIO_TYPE_DIRECTION_IN,
+				&request, sizeof(request), NULL, 0);
+	if (!ret)
+		ggc->lines[which].direction = 1;
+	return ret;
+}
+
+static int gb_gpio_direction_out_operation(struct gb_gpio_controller *ggc,
+					u8 which, bool value_high)
+{
+	struct gb_gpio_direction_out_request request;
+	int ret;
+
+	request.which = which;
+	request.value = value_high ? 1 : 0;
+	ret = gb_operation_sync(ggc->connection, GB_GPIO_TYPE_DIRECTION_OUT,
+				&request, sizeof(request), NULL, 0);
+	if (!ret)
+		ggc->lines[which].direction = 0;
+	return ret;
+}
+
+static int gb_gpio_get_value_operation(struct gb_gpio_controller *ggc,
+					u8 which)
+{
+	struct device *dev = &ggc->gbphy_dev->dev;
+	struct gb_gpio_get_value_request request;
+	struct gb_gpio_get_value_response response;
+	int ret;
+	u8 value;
+
+	request.which = which;
+	ret = gb_operation_sync(ggc->connection, GB_GPIO_TYPE_GET_VALUE,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(dev, "failed to get value of gpio %u\n", which);
+		return ret;
+	}
+
+	value = response.value;
+	if (value && value != 1) {
+		dev_warn(dev, "gpio %u value was %u (should be 0 or 1)\n",
+			 which, value);
+	}
+	ggc->lines[which].value = value ? 1 : 0;
+	return 0;
+}
+
+static void gb_gpio_set_value_operation(struct gb_gpio_controller *ggc,
+					u8 which, bool value_high)
+{
+	struct device *dev = &ggc->gbphy_dev->dev;
+	struct gb_gpio_set_value_request request;
+	int ret;
+
+	if (ggc->lines[which].direction == 1) {
+		dev_warn(dev, "refusing to set value of input gpio %u\n",
+			 which);
+		return;
+	}
+
+	request.which = which;
+	request.value = value_high ? 1 : 0;
+	ret = gb_operation_sync(ggc->connection, GB_GPIO_TYPE_SET_VALUE,
+				&request, sizeof(request), NULL, 0);
+	if (ret) {
+		dev_err(dev, "failed to set value of gpio %u\n", which);
+		return;
+	}
+
+	ggc->lines[which].value = request.value;
+}
+
+static int gb_gpio_set_debounce_operation(struct gb_gpio_controller *ggc,
+					u8 which, u16 debounce_usec)
+{
+	struct gb_gpio_set_debounce_request request;
+	int ret;
+
+	request.which = which;
+	request.usec = cpu_to_le16(debounce_usec);
+	ret = gb_operation_sync(ggc->connection, GB_GPIO_TYPE_SET_DEBOUNCE,
+				&request, sizeof(request), NULL, 0);
+	if (!ret)
+		ggc->lines[which].debounce_usec = debounce_usec;
+	return ret;
+}
+
+static void _gb_gpio_irq_mask(struct gb_gpio_controller *ggc, u8 hwirq)
+{
+	struct device *dev = &ggc->gbphy_dev->dev;
+	struct gb_gpio_irq_mask_request request;
+	int ret;
+
+	request.which = hwirq;
+	ret = gb_operation_sync(ggc->connection,
+				GB_GPIO_TYPE_IRQ_MASK,
+				&request, sizeof(request), NULL, 0);
+	if (ret)
+		dev_err(dev, "failed to mask irq: %d\n", ret);
+}
+
+static void _gb_gpio_irq_unmask(struct gb_gpio_controller *ggc, u8 hwirq)
+{
+	struct device *dev = &ggc->gbphy_dev->dev;
+	struct gb_gpio_irq_unmask_request request;
+	int ret;
+
+	request.which = hwirq;
+	ret = gb_operation_sync(ggc->connection,
+				GB_GPIO_TYPE_IRQ_UNMASK,
+				&request, sizeof(request), NULL, 0);
+	if (ret)
+		dev_err(dev, "failed to unmask irq: %d\n", ret);
+}
+
+static void _gb_gpio_irq_set_type(struct gb_gpio_controller *ggc,
+					u8 hwirq, u8 type)
+{
+	struct device *dev = &ggc->gbphy_dev->dev;
+	struct gb_gpio_irq_type_request request;
+	int ret;
+
+	request.which = hwirq;
+	request.type = type;
+
+	ret = gb_operation_sync(ggc->connection,
+				GB_GPIO_TYPE_IRQ_TYPE,
+				&request, sizeof(request), NULL, 0);
+	if (ret)
+		dev_err(dev, "failed to set irq type: %d\n", ret);
+}
+
+static void gb_gpio_irq_mask(struct irq_data *d)
+{
+	struct gpio_chip *chip = irq_data_to_gpio_chip(d);
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+	struct gb_gpio_line *line = &ggc->lines[d->hwirq];
+
+	line->masked = true;
+	line->masked_pending = true;
+}
+
+static void gb_gpio_irq_unmask(struct irq_data *d)
+{
+	struct gpio_chip *chip = irq_data_to_gpio_chip(d);
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+	struct gb_gpio_line *line = &ggc->lines[d->hwirq];
+
+	line->masked = false;
+	line->masked_pending = true;
+}
+
+static int gb_gpio_irq_set_type(struct irq_data *d, unsigned int type)
+{
+	struct gpio_chip *chip = irq_data_to_gpio_chip(d);
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+	struct gb_gpio_line *line = &ggc->lines[d->hwirq];
+	struct device *dev = &ggc->gbphy_dev->dev;
+	u8 irq_type;
+
+	switch (type) {
+	case IRQ_TYPE_NONE:
+		irq_type = GB_GPIO_IRQ_TYPE_NONE;
+		break;
+	case IRQ_TYPE_EDGE_RISING:
+		irq_type = GB_GPIO_IRQ_TYPE_EDGE_RISING;
+		break;
+	case IRQ_TYPE_EDGE_FALLING:
+		irq_type = GB_GPIO_IRQ_TYPE_EDGE_FALLING;
+		break;
+	case IRQ_TYPE_EDGE_BOTH:
+		irq_type = GB_GPIO_IRQ_TYPE_EDGE_BOTH;
+		break;
+	case IRQ_TYPE_LEVEL_LOW:
+		irq_type = GB_GPIO_IRQ_TYPE_LEVEL_LOW;
+		break;
+	case IRQ_TYPE_LEVEL_HIGH:
+		irq_type = GB_GPIO_IRQ_TYPE_LEVEL_HIGH;
+		break;
+	default:
+		dev_err(dev, "unsupported irq type: %u\n", type);
+		return -EINVAL;
+	}
+
+	line->irq_type = irq_type;
+	line->irq_type_pending = true;
+
+	return 0;
+}
+
+static void gb_gpio_irq_bus_lock(struct irq_data *d)
+{
+	struct gpio_chip *chip = irq_data_to_gpio_chip(d);
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+
+	mutex_lock(&ggc->irq_lock);
+}
+
+static void gb_gpio_irq_bus_sync_unlock(struct irq_data *d)
+{
+	struct gpio_chip *chip = irq_data_to_gpio_chip(d);
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+	struct gb_gpio_line *line = &ggc->lines[d->hwirq];
+
+	if (line->irq_type_pending) {
+		_gb_gpio_irq_set_type(ggc, d->hwirq, line->irq_type);
+		line->irq_type_pending = false;
+	}
+
+	if (line->masked_pending) {
+		if (line->masked)
+			_gb_gpio_irq_mask(ggc, d->hwirq);
+		else
+			_gb_gpio_irq_unmask(ggc, d->hwirq);
+		line->masked_pending = false;
+	}
+
+	mutex_unlock(&ggc->irq_lock);
+}
+
+static int gb_gpio_request_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_gpio_controller *ggc = gb_connection_get_data(connection);
+	struct device *dev = &ggc->gbphy_dev->dev;
+	struct gb_message *request;
+	struct gb_gpio_irq_event_request *event;
+	u8 type = op->type;
+	int irq;
+	struct irq_desc *desc;
+
+	if (type != GB_GPIO_TYPE_IRQ_EVENT) {
+		dev_err(dev, "unsupported unsolicited request: %u\n", type);
+		return -EINVAL;
+	}
+
+	request = op->request;
+
+	if (request->payload_size < sizeof(*event)) {
+		dev_err(dev, "short event received (%zu < %zu)\n",
+			request->payload_size, sizeof(*event));
+		return -EINVAL;
+	}
+
+	event = request->payload;
+	if (event->which > ggc->line_max) {
+		dev_err(dev, "invalid hw irq: %d\n", event->which);
+		return -EINVAL;
+	}
+
+	irq = irq_find_mapping(ggc->irqdomain, event->which);
+	if (!irq) {
+		dev_err(dev, "failed to find IRQ\n");
+		return -EINVAL;
+	}
+	desc = irq_to_desc(irq);
+	if (!desc) {
+		dev_err(dev, "failed to look up irq\n");
+		return -EINVAL;
+	}
+
+	local_irq_disable();
+	generic_handle_irq_desc(desc);
+	local_irq_enable();
+
+	return 0;
+}
+
+static int gb_gpio_request(struct gpio_chip *chip, unsigned offset)
+{
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+
+	return gb_gpio_activate_operation(ggc, (u8)offset);
+}
+
+static void gb_gpio_free(struct gpio_chip *chip, unsigned offset)
+{
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+
+	gb_gpio_deactivate_operation(ggc, (u8)offset);
+}
+
+static int gb_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
+{
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+	u8 which;
+	int ret;
+
+	which = (u8)offset;
+	ret = gb_gpio_get_direction_operation(ggc, which);
+	if (ret)
+		return ret;
+
+	return ggc->lines[which].direction ? 1 : 0;
+}
+
+static int gb_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
+{
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+
+	return gb_gpio_direction_in_operation(ggc, (u8)offset);
+}
+
+static int gb_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
+					int value)
+{
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+
+	return gb_gpio_direction_out_operation(ggc, (u8)offset, !!value);
+}
+
+static int gb_gpio_get(struct gpio_chip *chip, unsigned offset)
+{
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+	u8 which;
+	int ret;
+
+	which = (u8)offset;
+	ret = gb_gpio_get_value_operation(ggc, which);
+	if (ret)
+		return ret;
+
+	return ggc->lines[which].value;
+}
+
+static void gb_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
+{
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+
+	gb_gpio_set_value_operation(ggc, (u8)offset, !!value);
+}
+
+static int gb_gpio_set_debounce(struct gpio_chip *chip, unsigned offset,
+					unsigned debounce)
+{
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+	u16 usec;
+
+	if (debounce > U16_MAX)
+		return -EINVAL;
+	usec = (u16)debounce;
+
+	return gb_gpio_set_debounce_operation(ggc, (u8)offset, usec);
+}
+
+static int gb_gpio_controller_setup(struct gb_gpio_controller *ggc)
+{
+	int ret;
+
+	/* Now find out how many lines there are */
+	ret = gb_gpio_line_count_operation(ggc);
+	if (ret)
+		return ret;
+
+	ggc->lines = kcalloc(ggc->line_max + 1, sizeof(*ggc->lines),
+			     GFP_KERNEL);
+	if (!ggc->lines)
+		return -ENOMEM;
+
+	return ret;
+}
+
+/**
+ * gb_gpio_irq_map() - maps an IRQ into a GB gpio irqchip
+ * @d: the irqdomain used by this irqchip
+ * @irq: the global irq number used by this GB gpio irqchip irq
+ * @hwirq: the local IRQ/GPIO line offset on this GB gpio
+ *
+ * This function will set up the mapping for a certain IRQ line on a
+ * GB gpio by assigning the GB gpio as chip data, and using the irqchip
+ * stored inside the GB gpio.
+ */
+static int gb_gpio_irq_map(struct irq_domain *domain, unsigned int irq,
+			   irq_hw_number_t hwirq)
+{
+	struct gpio_chip *chip = domain->host_data;
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+
+	irq_set_chip_data(irq, ggc);
+	irq_set_chip_and_handler(irq, ggc->irqchip, ggc->irq_handler);
+	irq_set_noprobe(irq);
+	/*
+	 * No set-up of the hardware will happen if IRQ_TYPE_NONE
+	 * is passed as default type.
+	 */
+	if (ggc->irq_default_type != IRQ_TYPE_NONE)
+		irq_set_irq_type(irq, ggc->irq_default_type);
+
+	return 0;
+}
+
+static void gb_gpio_irq_unmap(struct irq_domain *d, unsigned int irq)
+{
+	irq_set_chip_and_handler(irq, NULL, NULL);
+	irq_set_chip_data(irq, NULL);
+}
+
+static const struct irq_domain_ops gb_gpio_domain_ops = {
+	.map	= gb_gpio_irq_map,
+	.unmap	= gb_gpio_irq_unmap,
+};
+
+/**
+ * gb_gpio_irqchip_remove() - removes an irqchip added to a gb_gpio_controller
+ * @ggc: the gb_gpio_controller to remove the irqchip from
+ *
+ * This is called only from gb_gpio_remove()
+ */
+static void gb_gpio_irqchip_remove(struct gb_gpio_controller *ggc)
+{
+	unsigned int offset;
+
+	/* Remove all IRQ mappings and delete the domain */
+	if (ggc->irqdomain) {
+		for (offset = 0; offset < (ggc->line_max + 1); offset++)
+			irq_dispose_mapping(irq_find_mapping(ggc->irqdomain, offset));
+		irq_domain_remove(ggc->irqdomain);
+	}
+
+	if (ggc->irqchip)
+		ggc->irqchip = NULL;
+}
+
+/**
+ * gb_gpio_irqchip_add() - adds an irqchip to a gpio chip
+ * @chip: the gpio chip to add the irqchip to
+ * @irqchip: the irqchip to add to the adapter
+ * @first_irq: if not dynamically assigned, the base (first) IRQ to
+ * allocate gpio irqs from
+ * @handler: the irq handler to use (often a predefined irq core function)
+ * @type: the default type for IRQs on this irqchip, pass IRQ_TYPE_NONE
+ * to have the core avoid setting up any default type in the hardware.
+ *
+ * This function closely associates a certain irqchip with a certain
+ * gpio chip, providing an irq domain to translate the local IRQs to
+ * global irqs, and making sure that the gpio chip
+ * is passed as chip data to all related functions. Driver callbacks
+ * need to use container_of() to get their local state containers back
+ * from the gpio chip passed as chip data. An irqdomain will be stored
+ * in the gpio chip that shall be used by the driver to handle IRQ number
+ * translation. The gpio chip will need to be initialized and registered
+ * before calling this function.
+ */
+static int gb_gpio_irqchip_add(struct gpio_chip *chip,
+			 struct irq_chip *irqchip,
+			 unsigned int first_irq,
+			 irq_flow_handler_t handler,
+			 unsigned int type)
+{
+	struct gb_gpio_controller *ggc;
+	unsigned int offset;
+	unsigned irq_base;
+
+	if (!chip || !irqchip)
+		return -EINVAL;
+
+	ggc = gpio_chip_to_gb_gpio_controller(chip);
+
+	ggc->irqchip = irqchip;
+	ggc->irq_handler = handler;
+	ggc->irq_default_type = type;
+	ggc->irqdomain = irq_domain_add_simple(NULL,
+					ggc->line_max + 1, first_irq,
+					&gb_gpio_domain_ops, chip);
+	if (!ggc->irqdomain) {
+		ggc->irqchip = NULL;
+		return -EINVAL;
+	}
+
+	/*
+	 * Prepare the mapping since the irqchip shall be orthogonal to
+	 * any gpio calls. If the first_irq was zero, this is
+	 * necessary to allocate descriptors for all IRQs.
+	 */
+	for (offset = 0; offset < (ggc->line_max + 1); offset++) {
+		irq_base = irq_create_mapping(ggc->irqdomain, offset);
+		if (offset == 0)
+			ggc->irq_base = irq_base;
+	}
+
+	return 0;
+}
+
+static int gb_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
+{
+	struct gb_gpio_controller *ggc = gpio_chip_to_gb_gpio_controller(chip);
+
+	return irq_find_mapping(ggc->irqdomain, offset);
+}
+
+static int gb_gpio_probe(struct gbphy_device *gbphy_dev,
+			 const struct gbphy_device_id *id)
+{
+	struct gb_connection *connection;
+	struct gb_gpio_controller *ggc;
+	struct gpio_chip *gpio;
+	struct irq_chip *irqc;
+	int ret;
+
+	ggc = kzalloc(sizeof(*ggc), GFP_KERNEL);
+	if (!ggc)
+		return -ENOMEM;
+
+	connection = gb_connection_create(gbphy_dev->bundle,
+					  le16_to_cpu(gbphy_dev->cport_desc->id),
+					  gb_gpio_request_handler);
+	if (IS_ERR(connection)) {
+		ret = PTR_ERR(connection);
+		goto exit_ggc_free;
+	}
+
+	ggc->connection = connection;
+	gb_connection_set_data(connection, ggc);
+	ggc->gbphy_dev = gbphy_dev;
+	gb_gbphy_set_data(gbphy_dev, ggc);
+
+	ret = gb_connection_enable_tx(connection);
+	if (ret)
+		goto exit_connection_destroy;
+
+	ret = gb_gpio_controller_setup(ggc);
+	if (ret)
+		goto exit_connection_disable;
+
+	irqc = &ggc->irqc;
+	irqc->irq_mask = gb_gpio_irq_mask;
+	irqc->irq_unmask = gb_gpio_irq_unmask;
+	irqc->irq_set_type = gb_gpio_irq_set_type;
+	irqc->irq_bus_lock = gb_gpio_irq_bus_lock;
+	irqc->irq_bus_sync_unlock = gb_gpio_irq_bus_sync_unlock;
+	irqc->name = "greybus_gpio";
+
+	mutex_init(&ggc->irq_lock);
+
+	gpio = &ggc->chip;
+
+	gpio->label = "greybus_gpio";
+	gpio->parent = &gbphy_dev->dev;
+	gpio->owner = THIS_MODULE;
+
+	gpio->request = gb_gpio_request;
+	gpio->free = gb_gpio_free;
+	gpio->get_direction = gb_gpio_get_direction;
+	gpio->direction_input = gb_gpio_direction_input;
+	gpio->direction_output = gb_gpio_direction_output;
+	gpio->get = gb_gpio_get;
+	gpio->set = gb_gpio_set;
+	gpio->set_debounce = gb_gpio_set_debounce;
+	gpio->to_irq = gb_gpio_to_irq;
+	gpio->base = -1;		/* Allocate base dynamically */
+	gpio->ngpio = ggc->line_max + 1;
+	gpio->can_sleep = true;
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto exit_line_free;
+
+	ret = gb_gpio_irqchip_add(gpio, irqc, 0,
+				   handle_level_irq, IRQ_TYPE_NONE);
+	if (ret) {
+		dev_err(&connection->bundle->dev,
+			"failed to add irq chip: %d\n", ret);
+		goto exit_line_free;
+	}
+
+	ret = gpiochip_add(gpio);
+	if (ret) {
+		dev_err(&connection->bundle->dev,
+			"failed to add gpio chip: %d\n", ret);
+		goto exit_gpio_irqchip_remove;
+	}
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+	return 0;
+
+exit_gpio_irqchip_remove:
+	gb_gpio_irqchip_remove(ggc);
+exit_line_free:
+	kfree(ggc->lines);
+exit_connection_disable:
+	gb_connection_disable(connection);
+exit_connection_destroy:
+	gb_connection_destroy(connection);
+exit_ggc_free:
+	kfree(ggc);
+	return ret;
+}
+
+static void gb_gpio_remove(struct gbphy_device *gbphy_dev)
+{
+	struct gb_gpio_controller *ggc = gb_gbphy_get_data(gbphy_dev);
+	struct gb_connection *connection = ggc->connection;
+	int ret;
+
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		gbphy_runtime_get_noresume(gbphy_dev);
+
+	gb_connection_disable_rx(connection);
+	gpiochip_remove(&ggc->chip);
+	gb_gpio_irqchip_remove(ggc);
+	gb_connection_disable(connection);
+	gb_connection_destroy(connection);
+	kfree(ggc->lines);
+	kfree(ggc);
+}
+
+static const struct gbphy_device_id gb_gpio_id_table[] = {
+	{ GBPHY_PROTOCOL(GREYBUS_PROTOCOL_GPIO) },
+	{ },
+};
+MODULE_DEVICE_TABLE(gbphy, gb_gpio_id_table);
+
+static struct gbphy_driver gpio_driver = {
+	.name		= "gpio",
+	.probe		= gb_gpio_probe,
+	.remove		= gb_gpio_remove,
+	.id_table	= gb_gpio_id_table,
+};
+
+module_gbphy_driver(gpio_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/greybus.h b/drivers/staging/greybus/greybus.h
new file mode 100644
index 000000000000..12526887ae2e
--- /dev/null
+++ b/drivers/staging/greybus/greybus.h
@@ -0,0 +1,154 @@
+/*
+ * Greybus driver and device API
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __LINUX_GREYBUS_H
+#define __LINUX_GREYBUS_H
+
+#ifdef __KERNEL__
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/idr.h>
+
+#include "greybus_id.h"
+#include "greybus_manifest.h"
+#include "greybus_protocols.h"
+#include "manifest.h"
+#include "hd.h"
+#include "svc.h"
+#include "control.h"
+#include "module.h"
+#include "interface.h"
+#include "bundle.h"
+#include "connection.h"
+#include "operation.h"
+#include "timesync.h"
+
+/* Matches up with the Greybus Protocol specification document */
+#define GREYBUS_VERSION_MAJOR	0x00
+#define GREYBUS_VERSION_MINOR	0x01
+
+#define GREYBUS_ID_MATCH_DEVICE \
+	(GREYBUS_ID_MATCH_VENDOR | GREYBUS_ID_MATCH_PRODUCT)
+
+#define GREYBUS_DEVICE(v, p)					\
+	.match_flags	= GREYBUS_ID_MATCH_DEVICE,		\
+	.vendor		= (v),					\
+	.product	= (p),
+
+#define GREYBUS_DEVICE_CLASS(c)					\
+	.match_flags	= GREYBUS_ID_MATCH_CLASS,		\
+	.class		= (c),
+
+/* Maximum number of CPorts */
+#define CPORT_ID_MAX	4095		/* UniPro max id is 4095 */
+#define CPORT_ID_BAD	U16_MAX
+
+struct greybus_driver {
+	const char *name;
+
+	int (*probe)(struct gb_bundle *bundle,
+		     const struct greybus_bundle_id *id);
+	void (*disconnect)(struct gb_bundle *bundle);
+
+	const struct greybus_bundle_id *id_table;
+
+	struct device_driver driver;
+};
+#define to_greybus_driver(d) container_of(d, struct greybus_driver, driver)
+
+static inline void greybus_set_drvdata(struct gb_bundle *bundle, void *data)
+{
+	dev_set_drvdata(&bundle->dev, data);
+}
+
+static inline void *greybus_get_drvdata(struct gb_bundle *bundle)
+{
+	return dev_get_drvdata(&bundle->dev);
+}
+
+/* Don't call these directly, use the module_greybus_driver() macro instead */
+int greybus_register_driver(struct greybus_driver *driver,
+			    struct module *module, const char *mod_name);
+void greybus_deregister_driver(struct greybus_driver *driver);
+
+/* define to get proper THIS_MODULE and KBUILD_MODNAME values */
+#define greybus_register(driver) \
+	greybus_register_driver(driver, THIS_MODULE, KBUILD_MODNAME)
+#define greybus_deregister(driver) \
+	greybus_deregister_driver(driver)
+
+/**
+ * module_greybus_driver() - Helper macro for registering a Greybus driver
+ * @__greybus_driver: greybus_driver structure
+ *
+ * Helper macro for Greybus drivers to set up proper module init / exit
+ * functions.  Replaces module_init() and module_exit() and keeps people from
+ * printing pointless things to the kernel log when their driver is loaded.
+ */
+#define module_greybus_driver(__greybus_driver)	\
+	module_driver(__greybus_driver, greybus_register, greybus_deregister)
+
+int greybus_disabled(void);
+
+void gb_debugfs_init(void);
+void gb_debugfs_cleanup(void);
+struct dentry *gb_debugfs_get(void);
+
+extern struct bus_type greybus_bus_type;
+
+extern struct device_type greybus_hd_type;
+extern struct device_type greybus_module_type;
+extern struct device_type greybus_interface_type;
+extern struct device_type greybus_control_type;
+extern struct device_type greybus_bundle_type;
+extern struct device_type greybus_svc_type;
+
+static inline int is_gb_host_device(const struct device *dev)
+{
+	return dev->type == &greybus_hd_type;
+}
+
+static inline int is_gb_module(const struct device *dev)
+{
+	return dev->type == &greybus_module_type;
+}
+
+static inline int is_gb_interface(const struct device *dev)
+{
+	return dev->type == &greybus_interface_type;
+}
+
+static inline int is_gb_control(const struct device *dev)
+{
+	return dev->type == &greybus_control_type;
+}
+
+static inline int is_gb_bundle(const struct device *dev)
+{
+	return dev->type == &greybus_bundle_type;
+}
+
+static inline int is_gb_svc(const struct device *dev)
+{
+	return dev->type == &greybus_svc_type;
+}
+
+static inline bool cport_id_valid(struct gb_host_device *hd, u16 cport_id)
+{
+	return cport_id != CPORT_ID_BAD && cport_id < hd->num_cports;
+}
+
+#endif /* __KERNEL__ */
+#endif /* __LINUX_GREYBUS_H */
diff --git a/drivers/staging/greybus/greybus_authentication.h b/drivers/staging/greybus/greybus_authentication.h
new file mode 100644
index 000000000000..4784ed98e8a3
--- /dev/null
+++ b/drivers/staging/greybus/greybus_authentication.h
@@ -0,0 +1,120 @@
+/*
+ * Greybus Component Authentication User Header
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details.
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name of Google Inc. or Linaro Ltd. nor the names of
+ *    its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written
+ *    permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GOOGLE INC. OR
+ * LINARO LTD. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __GREYBUS_AUTHENTICATION_USER_H
+#define __GREYBUS_AUTHENTICATION_USER_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define CAP_CERTIFICATE_MAX_SIZE	1600
+#define CAP_SIGNATURE_MAX_SIZE		320
+
+/* Certificate class types */
+#define CAP_CERT_IMS_EAPC		0x00000001
+#define CAP_CERT_IMS_EASC		0x00000002
+#define CAP_CERT_IMS_EARC		0x00000003
+#define CAP_CERT_IMS_IAPC		0x00000004
+#define CAP_CERT_IMS_IASC		0x00000005
+#define CAP_CERT_IMS_IARC		0x00000006
+
+/* IMS Certificate response result codes */
+#define CAP_IMS_RESULT_CERT_FOUND	0x00
+#define CAP_IMS_RESULT_CERT_CLASS_INVAL	0x01
+#define CAP_IMS_RESULT_CERT_CORRUPT	0x02
+#define CAP_IMS_RESULT_CERT_NOT_FOUND	0x03
+
+/* Authentication types */
+#define CAP_AUTH_IMS_PRI		0x00000001
+#define CAP_AUTH_IMS_SEC		0x00000002
+#define CAP_AUTH_IMS_RSA		0x00000003
+
+/* Authenticate response result codes */
+#define CAP_AUTH_RESULT_CR_SUCCESS	0x00
+#define CAP_AUTH_RESULT_CR_BAD_TYPE	0x01
+#define CAP_AUTH_RESULT_CR_WRONG_EP	0x02
+#define CAP_AUTH_RESULT_CR_NO_KEY	0x03
+#define CAP_AUTH_RESULT_CR_SIG_FAIL	0x04
+
+
+/* IOCTL support */
+struct cap_ioc_get_endpoint_uid {
+	__u8			uid[8];
+} __attribute__ ((__packed__));
+
+struct cap_ioc_get_ims_certificate {
+	__u32			certificate_class;
+	__u32			certificate_id;
+
+	__u8			result_code;
+	__u32			cert_size;
+	__u8			certificate[CAP_CERTIFICATE_MAX_SIZE];
+} __attribute__ ((__packed__));
+
+struct cap_ioc_authenticate {
+	__u32			auth_type;
+	__u8			uid[8];
+	__u8			challenge[32];
+
+	__u8			result_code;
+	__u8			response[64];
+	__u32			signature_size;
+	__u8			signature[CAP_SIGNATURE_MAX_SIZE];
+} __attribute__ ((__packed__));
+
+#define CAP_IOCTL_BASE			'C'
+#define CAP_IOC_GET_ENDPOINT_UID	_IOR(CAP_IOCTL_BASE, 0, struct cap_ioc_get_endpoint_uid)
+#define CAP_IOC_GET_IMS_CERTIFICATE	_IOWR(CAP_IOCTL_BASE, 1, struct cap_ioc_get_ims_certificate)
+#define CAP_IOC_AUTHENTICATE		_IOWR(CAP_IOCTL_BASE, 2, struct cap_ioc_authenticate)
+
+#endif /* __GREYBUS_AUTHENTICATION_USER_H */
diff --git a/drivers/staging/greybus/greybus_firmware.h b/drivers/staging/greybus/greybus_firmware.h
new file mode 100644
index 000000000000..277a2acce6fd
--- /dev/null
+++ b/drivers/staging/greybus/greybus_firmware.h
@@ -0,0 +1,120 @@
+/*
+ * Greybus Firmware Management User Header
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details.
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016 Google Inc. All rights reserved.
+ * Copyright(c) 2016 Linaro Ltd. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name of Google Inc. or Linaro Ltd. nor the names of
+ *    its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written
+ *    permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GOOGLE INC. OR
+ * LINARO LTD. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __GREYBUS_FIRMWARE_USER_H
+#define __GREYBUS_FIRMWARE_USER_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define GB_FIRMWARE_U_TAG_MAX_SIZE		10
+
+#define GB_FW_U_LOAD_METHOD_UNIPRO		0x01
+#define GB_FW_U_LOAD_METHOD_INTERNAL		0x02
+
+#define GB_FW_U_LOAD_STATUS_FAILED		0x00
+#define GB_FW_U_LOAD_STATUS_UNVALIDATED		0x01
+#define GB_FW_U_LOAD_STATUS_VALIDATED		0x02
+#define GB_FW_U_LOAD_STATUS_VALIDATION_FAILED	0x03
+
+#define GB_FW_U_BACKEND_FW_STATUS_SUCCESS	0x01
+#define GB_FW_U_BACKEND_FW_STATUS_FAIL_FIND	0x02
+#define GB_FW_U_BACKEND_FW_STATUS_FAIL_FETCH	0x03
+#define GB_FW_U_BACKEND_FW_STATUS_FAIL_WRITE	0x04
+#define GB_FW_U_BACKEND_FW_STATUS_INT		0x05
+#define GB_FW_U_BACKEND_FW_STATUS_RETRY		0x06
+#define GB_FW_U_BACKEND_FW_STATUS_NOT_SUPPORTED	0x07
+
+#define GB_FW_U_BACKEND_VERSION_STATUS_SUCCESS		0x01
+#define GB_FW_U_BACKEND_VERSION_STATUS_NOT_AVAILABLE	0x02
+#define GB_FW_U_BACKEND_VERSION_STATUS_NOT_SUPPORTED	0x03
+#define GB_FW_U_BACKEND_VERSION_STATUS_RETRY		0x04
+#define GB_FW_U_BACKEND_VERSION_STATUS_FAIL_INT		0x05
+
+/* IOCTL support */
+struct fw_mgmt_ioc_get_intf_version {
+	__u8 firmware_tag[GB_FIRMWARE_U_TAG_MAX_SIZE];
+	__u16 major;
+	__u16 minor;
+} __attribute__ ((__packed__));
+
+struct fw_mgmt_ioc_get_backend_version {
+	__u8 firmware_tag[GB_FIRMWARE_U_TAG_MAX_SIZE];
+	__u16 major;
+	__u16 minor;
+	__u8 status;
+} __attribute__ ((__packed__));
+
+struct fw_mgmt_ioc_intf_load_and_validate {
+	__u8 firmware_tag[GB_FIRMWARE_U_TAG_MAX_SIZE];
+	__u8 load_method;
+	__u8 status;
+	__u16 major;
+	__u16 minor;
+} __attribute__ ((__packed__));
+
+struct fw_mgmt_ioc_backend_fw_update {
+	__u8 firmware_tag[GB_FIRMWARE_U_TAG_MAX_SIZE];
+	__u8 status;
+} __attribute__ ((__packed__));
+
+#define FW_MGMT_IOCTL_BASE			'F'
+#define FW_MGMT_IOC_GET_INTF_FW			_IOR(FW_MGMT_IOCTL_BASE, 0, struct fw_mgmt_ioc_get_intf_version)
+#define FW_MGMT_IOC_GET_BACKEND_FW		_IOWR(FW_MGMT_IOCTL_BASE, 1, struct fw_mgmt_ioc_get_backend_version)
+#define FW_MGMT_IOC_INTF_LOAD_AND_VALIDATE	_IOWR(FW_MGMT_IOCTL_BASE, 2, struct fw_mgmt_ioc_intf_load_and_validate)
+#define FW_MGMT_IOC_INTF_BACKEND_FW_UPDATE	_IOWR(FW_MGMT_IOCTL_BASE, 3, struct fw_mgmt_ioc_backend_fw_update)
+#define FW_MGMT_IOC_SET_TIMEOUT_MS		_IOW(FW_MGMT_IOCTL_BASE, 4, unsigned int)
+#define FW_MGMT_IOC_MODE_SWITCH			_IO(FW_MGMT_IOCTL_BASE, 5)
+
+#endif /* __GREYBUS_FIRMWARE_USER_H */
+
diff --git a/drivers/staging/greybus/greybus_id.h b/drivers/staging/greybus/greybus_id.h
new file mode 100644
index 000000000000..4bb1fc1b811d
--- /dev/null
+++ b/drivers/staging/greybus/greybus_id.h
@@ -0,0 +1,26 @@
+/* FIXME
+ * move this to include/linux/mod_devicetable.h when merging
+ */
+
+#ifndef __LINUX_GREYBUS_ID_H
+#define __LINUX_GREYBUS_ID_H
+
+#include <linux/types.h>
+#include <linux/mod_devicetable.h>
+
+
+struct greybus_bundle_id {
+	__u16	match_flags;
+	__u32	vendor;
+	__u32	product;
+	__u8	class;
+
+	kernel_ulong_t	driver_info __aligned(sizeof(kernel_ulong_t));
+};
+
+/* Used to match the greybus_bundle_id */
+#define GREYBUS_ID_MATCH_VENDOR		BIT(0)
+#define GREYBUS_ID_MATCH_PRODUCT	BIT(1)
+#define GREYBUS_ID_MATCH_CLASS		BIT(2)
+
+#endif /* __LINUX_GREYBUS_ID_H */
diff --git a/drivers/staging/greybus/greybus_manifest.h b/drivers/staging/greybus/greybus_manifest.h
new file mode 100644
index 000000000000..d135945cefe1
--- /dev/null
+++ b/drivers/staging/greybus/greybus_manifest.h
@@ -0,0 +1,177 @@
+/*
+ * Greybus manifest definition
+ *
+ * See "Greybus Application Protocol" document (version 0.1) for
+ * details on these values and structures.
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 and BSD licenses.
+ */
+
+#ifndef __GREYBUS_MANIFEST_H
+#define __GREYBUS_MANIFEST_H
+
+enum greybus_descriptor_type {
+	GREYBUS_TYPE_INVALID		= 0x00,
+	GREYBUS_TYPE_INTERFACE		= 0x01,
+	GREYBUS_TYPE_STRING		= 0x02,
+	GREYBUS_TYPE_BUNDLE		= 0x03,
+	GREYBUS_TYPE_CPORT		= 0x04,
+};
+
+enum greybus_protocol {
+	GREYBUS_PROTOCOL_CONTROL	= 0x00,
+	/* 0x01 is unused */
+	GREYBUS_PROTOCOL_GPIO		= 0x02,
+	GREYBUS_PROTOCOL_I2C		= 0x03,
+	GREYBUS_PROTOCOL_UART		= 0x04,
+	GREYBUS_PROTOCOL_HID		= 0x05,
+	GREYBUS_PROTOCOL_USB		= 0x06,
+	GREYBUS_PROTOCOL_SDIO		= 0x07,
+	GREYBUS_PROTOCOL_POWER_SUPPLY	= 0x08,
+	GREYBUS_PROTOCOL_PWM		= 0x09,
+	/* 0x0a is unused */
+	GREYBUS_PROTOCOL_SPI		= 0x0b,
+	GREYBUS_PROTOCOL_DISPLAY	= 0x0c,
+	GREYBUS_PROTOCOL_CAMERA_MGMT	= 0x0d,
+	GREYBUS_PROTOCOL_SENSOR		= 0x0e,
+	GREYBUS_PROTOCOL_LIGHTS		= 0x0f,
+	GREYBUS_PROTOCOL_VIBRATOR	= 0x10,
+	GREYBUS_PROTOCOL_LOOPBACK	= 0x11,
+	GREYBUS_PROTOCOL_AUDIO_MGMT	= 0x12,
+	GREYBUS_PROTOCOL_AUDIO_DATA	= 0x13,
+	GREYBUS_PROTOCOL_SVC            = 0x14,
+	GREYBUS_PROTOCOL_BOOTROM	= 0x15,
+	GREYBUS_PROTOCOL_CAMERA_DATA	= 0x16,
+	GREYBUS_PROTOCOL_FW_DOWNLOAD	= 0x17,
+	GREYBUS_PROTOCOL_FW_MANAGEMENT	= 0x18,
+	GREYBUS_PROTOCOL_AUTHENTICATION	= 0x19,
+	GREYBUS_PROTOCOL_LOG		= 0x1a,
+		/* ... */
+	GREYBUS_PROTOCOL_RAW		= 0xfe,
+	GREYBUS_PROTOCOL_VENDOR		= 0xff,
+};
+
+enum greybus_class_type {
+	GREYBUS_CLASS_CONTROL		= 0x00,
+	/* 0x01 is unused */
+	/* 0x02 is unused */
+	/* 0x03 is unused */
+	/* 0x04 is unused */
+	GREYBUS_CLASS_HID		= 0x05,
+	/* 0x06 is unused */
+	/* 0x07 is unused */
+	GREYBUS_CLASS_POWER_SUPPLY	= 0x08,
+	/* 0x09 is unused */
+	GREYBUS_CLASS_BRIDGED_PHY	= 0x0a,
+	/* 0x0b is unused */
+	GREYBUS_CLASS_DISPLAY		= 0x0c,
+	GREYBUS_CLASS_CAMERA		= 0x0d,
+	GREYBUS_CLASS_SENSOR		= 0x0e,
+	GREYBUS_CLASS_LIGHTS		= 0x0f,
+	GREYBUS_CLASS_VIBRATOR		= 0x10,
+	GREYBUS_CLASS_LOOPBACK		= 0x11,
+	GREYBUS_CLASS_AUDIO		= 0x12,
+	/* 0x13 is unused */
+	/* 0x14 is unused */
+	GREYBUS_CLASS_BOOTROM		= 0x15,
+	GREYBUS_CLASS_FW_MANAGEMENT	= 0x16,
+	GREYBUS_CLASS_LOG		= 0x17,
+		/* ... */
+	GREYBUS_CLASS_RAW		= 0xfe,
+	GREYBUS_CLASS_VENDOR		= 0xff,
+};
+
+enum {
+	GREYBUS_INTERFACE_FEATURE_TIMESYNC = BIT(0),
+};
+
+/*
+ * The string in a string descriptor is not NUL-terminated.  The
+ * size of the descriptor will be rounded up to a multiple of 4
+ * bytes, by padding the string with 0x00 bytes if necessary.
+ */
+struct greybus_descriptor_string {
+	__u8	length;
+	__u8	id;
+	__u8	string[0];
+} __packed;
+
+/*
+ * An interface descriptor describes information about an interface as a whole,
+ * *not* the functions within it.
+ */
+struct greybus_descriptor_interface {
+	__u8	vendor_stringid;
+	__u8	product_stringid;
+	__u8	features;
+	__u8	pad;
+} __packed;
+
+/*
+ * An bundle descriptor defines an identification number and a class for
+ * each bundle.
+ *
+ * @id: Uniquely identifies a bundle within a interface, its sole purpose is to
+ * allow CPort descriptors to specify which bundle they are associated with.
+ * The first bundle will have id 0, second will have 1 and so on.
+ *
+ * The largest CPort id associated with an bundle (defined by a
+ * CPort descriptor in the manifest) is used to determine how to
+ * encode the device id and module number in UniPro packets
+ * that use the bundle.
+ *
+ * @class: It is used by kernel to know the functionality provided by the
+ * bundle and will be matched against drivers functinality while probing greybus
+ * driver. It should contain one of the values defined in
+ * 'enum greybus_class_type'.
+ *
+ */
+struct greybus_descriptor_bundle {
+	__u8	id;	/* interface-relative id (0..) */
+	__u8	class;
+	__u8	pad[2];
+} __packed;
+
+/*
+ * A CPort descriptor indicates the id of the bundle within the
+ * module it's associated with, along with the CPort id used to
+ * address the CPort.  The protocol id defines the format of messages
+ * exchanged using the CPort.
+ */
+struct greybus_descriptor_cport {
+	__le16	id;
+	__u8	bundle;
+	__u8	protocol_id;	/* enum greybus_protocol */
+} __packed;
+
+struct greybus_descriptor_header {
+	__le16	size;
+	__u8	type;		/* enum greybus_descriptor_type */
+	__u8	pad;
+} __packed;
+
+struct greybus_descriptor {
+	struct greybus_descriptor_header		header;
+	union {
+		struct greybus_descriptor_string	string;
+		struct greybus_descriptor_interface	interface;
+		struct greybus_descriptor_bundle	bundle;
+		struct greybus_descriptor_cport		cport;
+	};
+} __packed;
+
+struct greybus_manifest_header {
+	__le16	size;
+	__u8	version_major;
+	__u8	version_minor;
+} __packed;
+
+struct greybus_manifest {
+	struct greybus_manifest_header		header;
+	struct greybus_descriptor		descriptors[0];
+} __packed;
+
+#endif /* __GREYBUS_MANIFEST_H */
diff --git a/drivers/staging/greybus/greybus_protocols.h b/drivers/staging/greybus/greybus_protocols.h
new file mode 100644
index 000000000000..639578309c2a
--- /dev/null
+++ b/drivers/staging/greybus/greybus_protocols.h
@@ -0,0 +1,2268 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2014 - 2015 Google Inc. All rights reserved.
+ * Copyright(c) 2014 - 2015 Linaro Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details.
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2014 - 2015 Google Inc. All rights reserved.
+ * Copyright(c) 2014 - 2015 Linaro Ltd. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name of Google Inc. or Linaro Ltd. nor the names of
+ *    its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written
+ *    permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GOOGLE INC. OR
+ * LINARO LTD. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __GREYBUS_PROTOCOLS_H
+#define __GREYBUS_PROTOCOLS_H
+
+/* Fixed IDs for control/svc protocols */
+
+/* SVC switch-port device ids */
+#define GB_SVC_DEVICE_ID_SVC			0
+#define GB_SVC_DEVICE_ID_AP			1
+#define GB_SVC_DEVICE_ID_MIN			2
+#define GB_SVC_DEVICE_ID_MAX			31
+
+#define GB_SVC_CPORT_ID				0
+#define GB_CONTROL_BUNDLE_ID			0
+#define GB_CONTROL_CPORT_ID			0
+
+
+/*
+ * All operation messages (both requests and responses) begin with
+ * a header that encodes the size of the message (header included).
+ * This header also contains a unique identifier, that associates a
+ * response message with its operation.  The header contains an
+ * operation type field, whose interpretation is dependent on what
+ * type of protocol is used over the connection.  The high bit
+ * (0x80) of the operation type field is used to indicate whether
+ * the message is a request (clear) or a response (set).
+ *
+ * Response messages include an additional result byte, which
+ * communicates the result of the corresponding request.  A zero
+ * result value means the operation completed successfully.  Any
+ * other value indicates an error; in this case, the payload of the
+ * response message (if any) is ignored.  The result byte must be
+ * zero in the header for a request message.
+ *
+ * The wire format for all numeric fields in the header is little
+ * endian.  Any operation-specific data begins immediately after the
+ * header.
+ */
+struct gb_operation_msg_hdr {
+	__le16	size;		/* Size in bytes of header + payload */
+	__le16	operation_id;	/* Operation unique id */
+	__u8	type;		/* E.g GB_I2C_TYPE_* or GB_GPIO_TYPE_* */
+	__u8	result;		/* Result of request (in responses only) */
+	__u8	pad[2];		/* must be zero (ignore when read) */
+} __packed;
+
+
+/* Generic request types */
+#define GB_REQUEST_TYPE_CPORT_SHUTDOWN		0x00
+#define GB_REQUEST_TYPE_INVALID			0x7f
+
+struct gb_cport_shutdown_request {
+	__u8 phase;
+} __packed;
+
+
+/* Control Protocol */
+
+/* Greybus control request types */
+#define GB_CONTROL_TYPE_VERSION			0x01
+#define GB_CONTROL_TYPE_PROBE_AP		0x02
+#define GB_CONTROL_TYPE_GET_MANIFEST_SIZE	0x03
+#define GB_CONTROL_TYPE_GET_MANIFEST		0x04
+#define GB_CONTROL_TYPE_CONNECTED		0x05
+#define GB_CONTROL_TYPE_DISCONNECTED		0x06
+#define GB_CONTROL_TYPE_TIMESYNC_ENABLE		0x07
+#define GB_CONTROL_TYPE_TIMESYNC_DISABLE	0x08
+#define GB_CONTROL_TYPE_TIMESYNC_AUTHORITATIVE	0x09
+/*	Unused					0x0a */
+#define GB_CONTROL_TYPE_BUNDLE_VERSION		0x0b
+#define GB_CONTROL_TYPE_DISCONNECTING		0x0c
+#define GB_CONTROL_TYPE_TIMESYNC_GET_LAST_EVENT	0x0d
+#define GB_CONTROL_TYPE_MODE_SWITCH		0x0e
+#define GB_CONTROL_TYPE_BUNDLE_SUSPEND		0x0f
+#define GB_CONTROL_TYPE_BUNDLE_RESUME		0x10
+#define GB_CONTROL_TYPE_BUNDLE_DEACTIVATE	0x11
+#define GB_CONTROL_TYPE_BUNDLE_ACTIVATE		0x12
+#define GB_CONTROL_TYPE_INTF_SUSPEND_PREPARE		0x13
+#define GB_CONTROL_TYPE_INTF_DEACTIVATE_PREPARE	0x14
+#define GB_CONTROL_TYPE_INTF_HIBERNATE_ABORT	0x15
+
+struct gb_control_version_request {
+	__u8	major;
+	__u8	minor;
+} __packed;
+
+struct gb_control_version_response {
+	__u8	major;
+	__u8	minor;
+} __packed;
+
+struct gb_control_bundle_version_request {
+	__u8	bundle_id;
+} __packed;
+
+struct gb_control_bundle_version_response {
+	__u8	major;
+	__u8	minor;
+} __packed;
+
+/* Control protocol manifest get size request has no payload*/
+struct gb_control_get_manifest_size_response {
+	__le16			size;
+} __packed;
+
+/* Control protocol manifest get request has no payload */
+struct gb_control_get_manifest_response {
+	__u8			data[0];
+} __packed;
+
+/* Control protocol [dis]connected request */
+struct gb_control_connected_request {
+	__le16			cport_id;
+} __packed;
+
+struct gb_control_disconnecting_request {
+	__le16			cport_id;
+} __packed;
+/* disconnecting response has no payload */
+
+struct gb_control_disconnected_request {
+	__le16			cport_id;
+} __packed;
+/* Control protocol [dis]connected response has no payload */
+
+#define GB_TIMESYNC_MAX_STROBES			0x04
+
+struct gb_control_timesync_enable_request {
+	__u8	count;
+	__le64	frame_time;
+	__le32	strobe_delay;
+	__le32	refclk;
+} __packed;
+/* timesync enable response has no payload */
+
+struct gb_control_timesync_authoritative_request {
+	__le64	frame_time[GB_TIMESYNC_MAX_STROBES];
+} __packed;
+/* timesync authoritative response has no payload */
+
+/* timesync get_last_event_request has no payload */
+struct gb_control_timesync_get_last_event_response {
+	__le64	frame_time;
+} __packed;
+
+/*
+ * All Bundle power management operations use the same request and response
+ * layout and status codes.
+ */
+
+#define GB_CONTROL_BUNDLE_PM_OK		0x00
+#define GB_CONTROL_BUNDLE_PM_INVAL	0x01
+#define GB_CONTROL_BUNDLE_PM_BUSY	0x02
+#define GB_CONTROL_BUNDLE_PM_FAIL	0x03
+#define GB_CONTROL_BUNDLE_PM_NA		0x04
+
+struct gb_control_bundle_pm_request {
+	__u8	bundle_id;
+} __packed;
+
+struct gb_control_bundle_pm_response {
+	__u8	status;
+} __packed;
+
+/*
+ * Interface Suspend Prepare and Deactivate Prepare operations use the same
+ * response layout and error codes. Define a single response structure and reuse
+ * it. Both operations have no payload.
+ */
+
+#define GB_CONTROL_INTF_PM_OK		0x00
+#define GB_CONTROL_INTF_PM_BUSY		0x01
+#define GB_CONTROL_INTF_PM_NA		0x02
+
+struct gb_control_intf_pm_response {
+	__u8	status;
+} __packed;
+
+/* APBridge protocol */
+
+/* request APB1 log */
+#define GB_APB_REQUEST_LOG			0x02
+
+/* request to map a cport to bulk in and bulk out endpoints */
+#define GB_APB_REQUEST_EP_MAPPING		0x03
+
+/* request to get the number of cports available */
+#define GB_APB_REQUEST_CPORT_COUNT		0x04
+
+/* request to reset a cport state */
+#define GB_APB_REQUEST_RESET_CPORT		0x05
+
+/* request to time the latency of messages on a given cport */
+#define GB_APB_REQUEST_LATENCY_TAG_EN		0x06
+#define GB_APB_REQUEST_LATENCY_TAG_DIS		0x07
+
+/* request to control the CSI transmitter */
+#define GB_APB_REQUEST_CSI_TX_CONTROL		0x08
+
+/* request to control audio streaming */
+#define GB_APB_REQUEST_AUDIO_CONTROL		0x09
+
+/* TimeSync requests */
+#define GB_APB_REQUEST_TIMESYNC_ENABLE		0x0d
+#define GB_APB_REQUEST_TIMESYNC_DISABLE		0x0e
+#define GB_APB_REQUEST_TIMESYNC_AUTHORITATIVE	0x0f
+#define GB_APB_REQUEST_TIMESYNC_GET_LAST_EVENT	0x10
+
+/* requests to set Greybus CPort flags */
+#define GB_APB_REQUEST_CPORT_FLAGS		0x11
+
+/* ARPC request */
+#define GB_APB_REQUEST_ARPC_RUN			0x12
+
+struct gb_apb_request_cport_flags {
+	__le32	flags;
+#define GB_APB_CPORT_FLAG_CONTROL		0x01
+#define GB_APB_CPORT_FLAG_HIGH_PRIO		0x02
+} __packed;
+
+
+/* Firmware Download Protocol */
+
+/* Request Types */
+#define GB_FW_DOWNLOAD_TYPE_FIND_FIRMWARE	0x01
+#define GB_FW_DOWNLOAD_TYPE_FETCH_FIRMWARE	0x02
+#define GB_FW_DOWNLOAD_TYPE_RELEASE_FIRMWARE	0x03
+
+#define GB_FIRMWARE_TAG_MAX_SIZE		10
+
+/* firmware download find firmware request/response */
+struct gb_fw_download_find_firmware_request {
+	__u8			firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE];
+} __packed;
+
+struct gb_fw_download_find_firmware_response {
+	__u8			firmware_id;
+	__le32			size;
+} __packed;
+
+/* firmware download fetch firmware request/response */
+struct gb_fw_download_fetch_firmware_request {
+	__u8			firmware_id;
+	__le32			offset;
+	__le32			size;
+} __packed;
+
+struct gb_fw_download_fetch_firmware_response {
+	__u8			data[0];
+} __packed;
+
+/* firmware download release firmware request */
+struct gb_fw_download_release_firmware_request {
+	__u8			firmware_id;
+} __packed;
+/* firmware download release firmware response has no payload */
+
+
+/* Firmware Management Protocol */
+
+/* Request Types */
+#define GB_FW_MGMT_TYPE_INTERFACE_FW_VERSION	0x01
+#define GB_FW_MGMT_TYPE_LOAD_AND_VALIDATE_FW	0x02
+#define GB_FW_MGMT_TYPE_LOADED_FW		0x03
+#define GB_FW_MGMT_TYPE_BACKEND_FW_VERSION	0x04
+#define GB_FW_MGMT_TYPE_BACKEND_FW_UPDATE	0x05
+#define GB_FW_MGMT_TYPE_BACKEND_FW_UPDATED	0x06
+
+#define GB_FW_LOAD_METHOD_UNIPRO		0x01
+#define GB_FW_LOAD_METHOD_INTERNAL		0x02
+
+#define GB_FW_LOAD_STATUS_FAILED		0x00
+#define GB_FW_LOAD_STATUS_UNVALIDATED		0x01
+#define GB_FW_LOAD_STATUS_VALIDATED		0x02
+#define GB_FW_LOAD_STATUS_VALIDATION_FAILED	0x03
+
+#define GB_FW_BACKEND_FW_STATUS_SUCCESS		0x01
+#define GB_FW_BACKEND_FW_STATUS_FAIL_FIND	0x02
+#define GB_FW_BACKEND_FW_STATUS_FAIL_FETCH	0x03
+#define GB_FW_BACKEND_FW_STATUS_FAIL_WRITE	0x04
+#define GB_FW_BACKEND_FW_STATUS_INT		0x05
+#define GB_FW_BACKEND_FW_STATUS_RETRY		0x06
+#define GB_FW_BACKEND_FW_STATUS_NOT_SUPPORTED	0x07
+
+#define GB_FW_BACKEND_VERSION_STATUS_SUCCESS		0x01
+#define GB_FW_BACKEND_VERSION_STATUS_NOT_AVAILABLE	0x02
+#define GB_FW_BACKEND_VERSION_STATUS_NOT_SUPPORTED	0x03
+#define GB_FW_BACKEND_VERSION_STATUS_RETRY		0x04
+#define GB_FW_BACKEND_VERSION_STATUS_FAIL_INT		0x05
+
+/* firmware management interface firmware version request has no payload */
+struct gb_fw_mgmt_interface_fw_version_response {
+	__u8			firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE];
+	__le16			major;
+	__le16			minor;
+} __packed;
+
+/* firmware management load and validate firmware request/response */
+struct gb_fw_mgmt_load_and_validate_fw_request {
+	__u8			request_id;
+	__u8			load_method;
+	__u8			firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE];
+} __packed;
+/* firmware management load and validate firmware response has no payload*/
+
+/* firmware management loaded firmware request */
+struct gb_fw_mgmt_loaded_fw_request {
+	__u8			request_id;
+	__u8			status;
+	__le16			major;
+	__le16			minor;
+} __packed;
+/* firmware management loaded firmware response has no payload */
+
+/* firmware management backend firmware version request/response */
+struct gb_fw_mgmt_backend_fw_version_request {
+	__u8			firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE];
+} __packed;
+
+struct gb_fw_mgmt_backend_fw_version_response {
+	__le16			major;
+	__le16			minor;
+	__u8			status;
+} __packed;
+
+/* firmware management backend firmware update request */
+struct gb_fw_mgmt_backend_fw_update_request {
+	__u8			request_id;
+	__u8			firmware_tag[GB_FIRMWARE_TAG_MAX_SIZE];
+} __packed;
+/* firmware management backend firmware update response has no payload */
+
+/* firmware management backend firmware updated request */
+struct gb_fw_mgmt_backend_fw_updated_request {
+	__u8			request_id;
+	__u8			status;
+} __packed;
+/* firmware management backend firmware updated response has no payload */
+
+
+/* Component Authentication Protocol (CAP) */
+
+/* Request Types */
+#define GB_CAP_TYPE_GET_ENDPOINT_UID	0x01
+#define GB_CAP_TYPE_GET_IMS_CERTIFICATE	0x02
+#define GB_CAP_TYPE_AUTHENTICATE	0x03
+
+/* CAP get endpoint uid request has no payload */
+struct gb_cap_get_endpoint_uid_response {
+	__u8			uid[8];
+} __packed;
+
+/* CAP get endpoint ims certificate request/response */
+struct gb_cap_get_ims_certificate_request {
+	__le32			certificate_class;
+	__le32			certificate_id;
+} __packed;
+
+struct gb_cap_get_ims_certificate_response {
+	__u8			result_code;
+	__u8			certificate[0];
+} __packed;
+
+/* CAP authenticate request/response */
+struct gb_cap_authenticate_request {
+	__le32			auth_type;
+	__u8			uid[8];
+	__u8			challenge[32];
+} __packed;
+
+struct gb_cap_authenticate_response {
+	__u8			result_code;
+	__u8			response[64];
+	__u8			signature[0];
+} __packed;
+
+
+/* Bootrom Protocol */
+
+/* Version of the Greybus bootrom protocol we support */
+#define GB_BOOTROM_VERSION_MAJOR		0x00
+#define GB_BOOTROM_VERSION_MINOR		0x01
+
+/* Greybus bootrom request types */
+#define GB_BOOTROM_TYPE_VERSION			0x01
+#define GB_BOOTROM_TYPE_FIRMWARE_SIZE		0x02
+#define GB_BOOTROM_TYPE_GET_FIRMWARE		0x03
+#define GB_BOOTROM_TYPE_READY_TO_BOOT		0x04
+#define GB_BOOTROM_TYPE_AP_READY		0x05	/* Request with no-payload */
+#define GB_BOOTROM_TYPE_GET_VID_PID		0x06	/* Request with no-payload */
+
+/* Greybus bootrom boot stages */
+#define GB_BOOTROM_BOOT_STAGE_ONE		0x01 /* Reserved for the boot ROM */
+#define GB_BOOTROM_BOOT_STAGE_TWO		0x02 /* Bootrom package to be loaded by the boot ROM */
+#define GB_BOOTROM_BOOT_STAGE_THREE		0x03 /* Module personality package loaded by Stage 2 firmware */
+
+/* Greybus bootrom ready to boot status */
+#define GB_BOOTROM_BOOT_STATUS_INVALID		0x00 /* Firmware blob could not be validated */
+#define GB_BOOTROM_BOOT_STATUS_INSECURE		0x01 /* Firmware blob is valid but insecure */
+#define GB_BOOTROM_BOOT_STATUS_SECURE		0x02 /* Firmware blob is valid and secure */
+
+/* Max bootrom data fetch size in bytes */
+#define GB_BOOTROM_FETCH_MAX			2000
+
+struct gb_bootrom_version_request {
+	__u8	major;
+	__u8	minor;
+} __packed;
+
+struct gb_bootrom_version_response {
+	__u8	major;
+	__u8	minor;
+} __packed;
+
+/* Bootrom protocol firmware size request/response */
+struct gb_bootrom_firmware_size_request {
+	__u8			stage;
+} __packed;
+
+struct gb_bootrom_firmware_size_response {
+	__le32			size;
+} __packed;
+
+/* Bootrom protocol get firmware request/response */
+struct gb_bootrom_get_firmware_request {
+	__le32			offset;
+	__le32			size;
+} __packed;
+
+struct gb_bootrom_get_firmware_response {
+	__u8			data[0];
+} __packed;
+
+/* Bootrom protocol Ready to boot request */
+struct gb_bootrom_ready_to_boot_request {
+	__u8			status;
+} __packed;
+/* Bootrom protocol Ready to boot response has no payload */
+
+/* Bootrom protocol get VID/PID request has no payload */
+struct gb_bootrom_get_vid_pid_response {
+	__le32			vendor_id;
+	__le32			product_id;
+} __packed;
+
+
+/* Power Supply */
+
+/* Greybus power supply request types */
+#define GB_POWER_SUPPLY_TYPE_GET_SUPPLIES		0x02
+#define GB_POWER_SUPPLY_TYPE_GET_DESCRIPTION		0x03
+#define GB_POWER_SUPPLY_TYPE_GET_PROP_DESCRIPTORS	0x04
+#define GB_POWER_SUPPLY_TYPE_GET_PROPERTY		0x05
+#define GB_POWER_SUPPLY_TYPE_SET_PROPERTY		0x06
+#define GB_POWER_SUPPLY_TYPE_EVENT			0x07
+
+/* Greybus power supply battery technologies types */
+#define GB_POWER_SUPPLY_TECH_UNKNOWN			0x0000
+#define GB_POWER_SUPPLY_TECH_NiMH			0x0001
+#define GB_POWER_SUPPLY_TECH_LION			0x0002
+#define GB_POWER_SUPPLY_TECH_LIPO			0x0003
+#define GB_POWER_SUPPLY_TECH_LiFe			0x0004
+#define GB_POWER_SUPPLY_TECH_NiCd			0x0005
+#define GB_POWER_SUPPLY_TECH_LiMn			0x0006
+
+/* Greybus power supply types */
+#define GB_POWER_SUPPLY_UNKNOWN_TYPE			0x0000
+#define GB_POWER_SUPPLY_BATTERY_TYPE			0x0001
+#define GB_POWER_SUPPLY_UPS_TYPE			0x0002
+#define GB_POWER_SUPPLY_MAINS_TYPE			0x0003
+#define GB_POWER_SUPPLY_USB_TYPE			0x0004
+#define GB_POWER_SUPPLY_USB_DCP_TYPE			0x0005
+#define GB_POWER_SUPPLY_USB_CDP_TYPE			0x0006
+#define GB_POWER_SUPPLY_USB_ACA_TYPE			0x0007
+
+/* Greybus power supply health values */
+#define GB_POWER_SUPPLY_HEALTH_UNKNOWN			0x0000
+#define GB_POWER_SUPPLY_HEALTH_GOOD			0x0001
+#define GB_POWER_SUPPLY_HEALTH_OVERHEAT			0x0002
+#define GB_POWER_SUPPLY_HEALTH_DEAD			0x0003
+#define GB_POWER_SUPPLY_HEALTH_OVERVOLTAGE		0x0004
+#define GB_POWER_SUPPLY_HEALTH_UNSPEC_FAILURE		0x0005
+#define GB_POWER_SUPPLY_HEALTH_COLD			0x0006
+#define GB_POWER_SUPPLY_HEALTH_WATCHDOG_TIMER_EXPIRE	0x0007
+#define GB_POWER_SUPPLY_HEALTH_SAFETY_TIMER_EXPIRE	0x0008
+
+/* Greybus power supply status values */
+#define GB_POWER_SUPPLY_STATUS_UNKNOWN			0x0000
+#define GB_POWER_SUPPLY_STATUS_CHARGING			0x0001
+#define GB_POWER_SUPPLY_STATUS_DISCHARGING		0x0002
+#define GB_POWER_SUPPLY_STATUS_NOT_CHARGING		0x0003
+#define GB_POWER_SUPPLY_STATUS_FULL			0x0004
+
+/* Greybus power supply capacity level values */
+#define GB_POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN		0x0000
+#define GB_POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL		0x0001
+#define GB_POWER_SUPPLY_CAPACITY_LEVEL_LOW		0x0002
+#define GB_POWER_SUPPLY_CAPACITY_LEVEL_NORMAL		0x0003
+#define GB_POWER_SUPPLY_CAPACITY_LEVEL_HIGH		0x0004
+#define GB_POWER_SUPPLY_CAPACITY_LEVEL_FULL		0x0005
+
+/* Greybus power supply scope values */
+#define GB_POWER_SUPPLY_SCOPE_UNKNOWN			0x0000
+#define GB_POWER_SUPPLY_SCOPE_SYSTEM			0x0001
+#define GB_POWER_SUPPLY_SCOPE_DEVICE			0x0002
+
+struct gb_power_supply_get_supplies_response {
+	__u8	supplies_count;
+} __packed;
+
+struct gb_power_supply_get_description_request {
+	__u8	psy_id;
+} __packed;
+
+struct gb_power_supply_get_description_response {
+	__u8	manufacturer[32];
+	__u8	model[32];
+	__u8	serial_number[32];
+	__le16	type;
+	__u8	properties_count;
+} __packed;
+
+struct gb_power_supply_props_desc {
+	__u8	property;
+#define GB_POWER_SUPPLY_PROP_STATUS				0x00
+#define GB_POWER_SUPPLY_PROP_CHARGE_TYPE			0x01
+#define GB_POWER_SUPPLY_PROP_HEALTH				0x02
+#define GB_POWER_SUPPLY_PROP_PRESENT				0x03
+#define GB_POWER_SUPPLY_PROP_ONLINE				0x04
+#define GB_POWER_SUPPLY_PROP_AUTHENTIC				0x05
+#define GB_POWER_SUPPLY_PROP_TECHNOLOGY				0x06
+#define GB_POWER_SUPPLY_PROP_CYCLE_COUNT			0x07
+#define GB_POWER_SUPPLY_PROP_VOLTAGE_MAX			0x08
+#define GB_POWER_SUPPLY_PROP_VOLTAGE_MIN			0x09
+#define GB_POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN			0x0A
+#define GB_POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN			0x0B
+#define GB_POWER_SUPPLY_PROP_VOLTAGE_NOW			0x0C
+#define GB_POWER_SUPPLY_PROP_VOLTAGE_AVG			0x0D
+#define GB_POWER_SUPPLY_PROP_VOLTAGE_OCV			0x0E
+#define GB_POWER_SUPPLY_PROP_VOLTAGE_BOOT			0x0F
+#define GB_POWER_SUPPLY_PROP_CURRENT_MAX			0x10
+#define GB_POWER_SUPPLY_PROP_CURRENT_NOW			0x11
+#define GB_POWER_SUPPLY_PROP_CURRENT_AVG			0x12
+#define GB_POWER_SUPPLY_PROP_CURRENT_BOOT			0x13
+#define GB_POWER_SUPPLY_PROP_POWER_NOW				0x14
+#define GB_POWER_SUPPLY_PROP_POWER_AVG				0x15
+#define GB_POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN			0x16
+#define GB_POWER_SUPPLY_PROP_CHARGE_EMPTY_DESIGN		0x17
+#define GB_POWER_SUPPLY_PROP_CHARGE_FULL			0x18
+#define GB_POWER_SUPPLY_PROP_CHARGE_EMPTY			0x19
+#define GB_POWER_SUPPLY_PROP_CHARGE_NOW				0x1A
+#define GB_POWER_SUPPLY_PROP_CHARGE_AVG				0x1B
+#define GB_POWER_SUPPLY_PROP_CHARGE_COUNTER			0x1C
+#define GB_POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT		0x1D
+#define GB_POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX	0x1E
+#define GB_POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE		0x1F
+#define GB_POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX	0x20
+#define GB_POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT		0x21
+#define GB_POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT_MAX		0x22
+#define GB_POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT		0x23
+#define GB_POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN			0x24
+#define GB_POWER_SUPPLY_PROP_ENERGY_EMPTY_DESIGN		0x25
+#define GB_POWER_SUPPLY_PROP_ENERGY_FULL			0x26
+#define GB_POWER_SUPPLY_PROP_ENERGY_EMPTY			0x27
+#define GB_POWER_SUPPLY_PROP_ENERGY_NOW				0x28
+#define GB_POWER_SUPPLY_PROP_ENERGY_AVG				0x29
+#define GB_POWER_SUPPLY_PROP_CAPACITY				0x2A
+#define GB_POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN			0x2B
+#define GB_POWER_SUPPLY_PROP_CAPACITY_ALERT_MAX			0x2C
+#define GB_POWER_SUPPLY_PROP_CAPACITY_LEVEL			0x2D
+#define GB_POWER_SUPPLY_PROP_TEMP				0x2E
+#define GB_POWER_SUPPLY_PROP_TEMP_MAX				0x2F
+#define GB_POWER_SUPPLY_PROP_TEMP_MIN				0x30
+#define GB_POWER_SUPPLY_PROP_TEMP_ALERT_MIN			0x31
+#define GB_POWER_SUPPLY_PROP_TEMP_ALERT_MAX			0x32
+#define GB_POWER_SUPPLY_PROP_TEMP_AMBIENT			0x33
+#define GB_POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MIN		0x34
+#define GB_POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MAX		0x35
+#define GB_POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW			0x36
+#define GB_POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG			0x37
+#define GB_POWER_SUPPLY_PROP_TIME_TO_FULL_NOW			0x38
+#define GB_POWER_SUPPLY_PROP_TIME_TO_FULL_AVG			0x39
+#define GB_POWER_SUPPLY_PROP_TYPE				0x3A
+#define GB_POWER_SUPPLY_PROP_SCOPE				0x3B
+#define GB_POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT		0x3C
+#define GB_POWER_SUPPLY_PROP_CALIBRATE				0x3D
+	__u8	is_writeable;
+} __packed;
+
+struct gb_power_supply_get_property_descriptors_request {
+	__u8	psy_id;
+} __packed;
+
+struct gb_power_supply_get_property_descriptors_response {
+	__u8	properties_count;
+	struct gb_power_supply_props_desc props[];
+} __packed;
+
+struct gb_power_supply_get_property_request {
+	__u8	psy_id;
+	__u8	property;
+} __packed;
+
+struct gb_power_supply_get_property_response {
+	__le32	prop_val;
+};
+
+struct gb_power_supply_set_property_request {
+	__u8	psy_id;
+	__u8	property;
+	__le32	prop_val;
+} __packed;
+
+struct gb_power_supply_event_request {
+	__u8	psy_id;
+	__u8	event;
+#define GB_POWER_SUPPLY_UPDATE		0x01
+} __packed;
+
+
+/* HID */
+
+/* Greybus HID operation types */
+#define GB_HID_TYPE_GET_DESC		0x02
+#define GB_HID_TYPE_GET_REPORT_DESC	0x03
+#define GB_HID_TYPE_PWR_ON		0x04
+#define GB_HID_TYPE_PWR_OFF		0x05
+#define GB_HID_TYPE_GET_REPORT		0x06
+#define GB_HID_TYPE_SET_REPORT		0x07
+#define GB_HID_TYPE_IRQ_EVENT		0x08
+
+/* Report type */
+#define GB_HID_INPUT_REPORT		0
+#define GB_HID_OUTPUT_REPORT		1
+#define GB_HID_FEATURE_REPORT		2
+
+/* Different request/response structures */
+/* HID get descriptor response */
+struct gb_hid_desc_response {
+	__u8				bLength;
+	__le16				wReportDescLength;
+	__le16				bcdHID;
+	__le16				wProductID;
+	__le16				wVendorID;
+	__u8				bCountryCode;
+} __packed;
+
+/* HID get report request/response */
+struct gb_hid_get_report_request {
+	__u8				report_type;
+	__u8				report_id;
+} __packed;
+
+/* HID set report request */
+struct gb_hid_set_report_request {
+	__u8				report_type;
+	__u8				report_id;
+	__u8				report[0];
+} __packed;
+
+/* HID input report request, via interrupt pipe */
+struct gb_hid_input_report_request {
+	__u8				report[0];
+} __packed;
+
+
+/* I2C */
+
+/* Greybus i2c request types */
+#define GB_I2C_TYPE_FUNCTIONALITY	0x02
+#define GB_I2C_TYPE_TRANSFER		0x05
+
+/* functionality request has no payload */
+struct gb_i2c_functionality_response {
+	__le32	functionality;
+} __packed;
+
+/*
+ * Outgoing data immediately follows the op count and ops array.
+ * The data for each write (master -> slave) op in the array is sent
+ * in order, with no (e.g. pad) bytes separating them.
+ *
+ * Short reads cause the entire transfer request to fail So response
+ * payload consists only of bytes read, and the number of bytes is
+ * exactly what was specified in the corresponding op.  Like
+ * outgoing data, the incoming data is in order and contiguous.
+ */
+struct gb_i2c_transfer_op {
+	__le16	addr;
+	__le16	flags;
+	__le16	size;
+} __packed;
+
+struct gb_i2c_transfer_request {
+	__le16				op_count;
+	struct gb_i2c_transfer_op	ops[0];		/* op_count of these */
+} __packed;
+struct gb_i2c_transfer_response {
+	__u8				data[0];	/* inbound data */
+} __packed;
+
+
+/* GPIO */
+
+/* Greybus GPIO request types */
+#define GB_GPIO_TYPE_LINE_COUNT		0x02
+#define GB_GPIO_TYPE_ACTIVATE		0x03
+#define GB_GPIO_TYPE_DEACTIVATE		0x04
+#define GB_GPIO_TYPE_GET_DIRECTION	0x05
+#define GB_GPIO_TYPE_DIRECTION_IN	0x06
+#define GB_GPIO_TYPE_DIRECTION_OUT	0x07
+#define GB_GPIO_TYPE_GET_VALUE		0x08
+#define GB_GPIO_TYPE_SET_VALUE		0x09
+#define GB_GPIO_TYPE_SET_DEBOUNCE	0x0a
+#define GB_GPIO_TYPE_IRQ_TYPE		0x0b
+#define GB_GPIO_TYPE_IRQ_MASK		0x0c
+#define GB_GPIO_TYPE_IRQ_UNMASK		0x0d
+#define GB_GPIO_TYPE_IRQ_EVENT		0x0e
+
+#define GB_GPIO_IRQ_TYPE_NONE		0x00
+#define GB_GPIO_IRQ_TYPE_EDGE_RISING	0x01
+#define GB_GPIO_IRQ_TYPE_EDGE_FALLING	0x02
+#define GB_GPIO_IRQ_TYPE_EDGE_BOTH	0x03
+#define GB_GPIO_IRQ_TYPE_LEVEL_HIGH	0x04
+#define GB_GPIO_IRQ_TYPE_LEVEL_LOW	0x08
+
+/* line count request has no payload */
+struct gb_gpio_line_count_response {
+	__u8	count;
+} __packed;
+
+struct gb_gpio_activate_request {
+	__u8	which;
+} __packed;
+/* activate response has no payload */
+
+struct gb_gpio_deactivate_request {
+	__u8	which;
+} __packed;
+/* deactivate response has no payload */
+
+struct gb_gpio_get_direction_request {
+	__u8	which;
+} __packed;
+struct gb_gpio_get_direction_response {
+	__u8	direction;
+} __packed;
+
+struct gb_gpio_direction_in_request {
+	__u8	which;
+} __packed;
+/* direction in response has no payload */
+
+struct gb_gpio_direction_out_request {
+	__u8	which;
+	__u8	value;
+} __packed;
+/* direction out response has no payload */
+
+struct gb_gpio_get_value_request {
+	__u8	which;
+} __packed;
+struct gb_gpio_get_value_response {
+	__u8	value;
+} __packed;
+
+struct gb_gpio_set_value_request {
+	__u8	which;
+	__u8	value;
+} __packed;
+/* set value response has no payload */
+
+struct gb_gpio_set_debounce_request {
+	__u8	which;
+	__le16	usec;
+} __packed;
+/* debounce response has no payload */
+
+struct gb_gpio_irq_type_request {
+	__u8	which;
+	__u8	type;
+} __packed;
+/* irq type response has no payload */
+
+struct gb_gpio_irq_mask_request {
+	__u8	which;
+} __packed;
+/* irq mask response has no payload */
+
+struct gb_gpio_irq_unmask_request {
+	__u8	which;
+} __packed;
+/* irq unmask response has no payload */
+
+/* irq event requests originate on another module and are handled on the AP */
+struct gb_gpio_irq_event_request {
+	__u8	which;
+} __packed;
+/* irq event has no response */
+
+
+/* PWM */
+
+/* Greybus PWM operation types */
+#define GB_PWM_TYPE_PWM_COUNT		0x02
+#define GB_PWM_TYPE_ACTIVATE		0x03
+#define GB_PWM_TYPE_DEACTIVATE		0x04
+#define GB_PWM_TYPE_CONFIG		0x05
+#define GB_PWM_TYPE_POLARITY		0x06
+#define GB_PWM_TYPE_ENABLE		0x07
+#define GB_PWM_TYPE_DISABLE		0x08
+
+/* pwm count request has no payload */
+struct gb_pwm_count_response {
+	__u8	count;
+} __packed;
+
+struct gb_pwm_activate_request {
+	__u8	which;
+} __packed;
+
+struct gb_pwm_deactivate_request {
+	__u8	which;
+} __packed;
+
+struct gb_pwm_config_request {
+	__u8	which;
+	__le32	duty;
+	__le32	period;
+} __packed;
+
+struct gb_pwm_polarity_request {
+	__u8	which;
+	__u8	polarity;
+} __packed;
+
+struct gb_pwm_enable_request {
+	__u8	which;
+} __packed;
+
+struct gb_pwm_disable_request {
+	__u8	which;
+} __packed;
+
+/* SPI */
+
+/* Should match up with modes in linux/spi/spi.h */
+#define GB_SPI_MODE_CPHA		0x01		/* clock phase */
+#define GB_SPI_MODE_CPOL		0x02		/* clock polarity */
+#define GB_SPI_MODE_MODE_0		(0|0)		/* (original MicroWire) */
+#define GB_SPI_MODE_MODE_1		(0|GB_SPI_MODE_CPHA)
+#define GB_SPI_MODE_MODE_2		(GB_SPI_MODE_CPOL|0)
+#define GB_SPI_MODE_MODE_3		(GB_SPI_MODE_CPOL|GB_SPI_MODE_CPHA)
+#define GB_SPI_MODE_CS_HIGH		0x04		/* chipselect active high? */
+#define GB_SPI_MODE_LSB_FIRST		0x08		/* per-word bits-on-wire */
+#define GB_SPI_MODE_3WIRE		0x10		/* SI/SO signals shared */
+#define GB_SPI_MODE_LOOP		0x20		/* loopback mode */
+#define GB_SPI_MODE_NO_CS		0x40		/* 1 dev/bus, no chipselect */
+#define GB_SPI_MODE_READY		0x80		/* slave pulls low to pause */
+
+/* Should match up with flags in linux/spi/spi.h */
+#define GB_SPI_FLAG_HALF_DUPLEX		BIT(0)		/* can't do full duplex */
+#define GB_SPI_FLAG_NO_RX		BIT(1)		/* can't do buffer read */
+#define GB_SPI_FLAG_NO_TX		BIT(2)		/* can't do buffer write */
+
+/* Greybus spi operation types */
+#define GB_SPI_TYPE_MASTER_CONFIG	0x02
+#define GB_SPI_TYPE_DEVICE_CONFIG	0x03
+#define GB_SPI_TYPE_TRANSFER		0x04
+
+/* mode request has no payload */
+struct gb_spi_master_config_response {
+	__le32	bits_per_word_mask;
+	__le32	min_speed_hz;
+	__le32	max_speed_hz;
+	__le16	mode;
+	__le16	flags;
+	__u8	num_chipselect;
+} __packed;
+
+struct gb_spi_device_config_request {
+	__u8	chip_select;
+} __packed;
+
+struct gb_spi_device_config_response {
+	__le16	mode;
+	__u8	bits_per_word;
+	__le32	max_speed_hz;
+	__u8	device_type;
+#define GB_SPI_SPI_DEV		0x00
+#define GB_SPI_SPI_NOR		0x01
+#define GB_SPI_SPI_MODALIAS	0x02
+	__u8	name[32];
+} __packed;
+
+/**
+ * struct gb_spi_transfer - a read/write buffer pair
+ * @speed_hz: Select a speed other than the device default for this transfer. If
+ *	0 the default (from @spi_device) is used.
+ * @len: size of rx and tx buffers (in bytes)
+ * @delay_usecs: microseconds to delay after this transfer before (optionally)
+ * 	changing the chipselect status, then starting the next transfer or
+ * 	completing this spi_message.
+ * @cs_change: affects chipselect after this transfer completes
+ * @bits_per_word: select a bits_per_word other than the device default for this
+ *	transfer. If 0 the default (from @spi_device) is used.
+ */
+struct gb_spi_transfer {
+	__le32		speed_hz;
+	__le32		len;
+	__le16		delay_usecs;
+	__u8		cs_change;
+	__u8		bits_per_word;
+	__u8		xfer_flags;
+#define GB_SPI_XFER_READ	0x01
+#define GB_SPI_XFER_WRITE	0x02
+#define GB_SPI_XFER_INPROGRESS	0x04
+} __packed;
+
+struct gb_spi_transfer_request {
+	__u8			chip_select;	/* of the spi device */
+	__u8			mode;		/* of the spi device */
+	__le16			count;
+	struct gb_spi_transfer	transfers[0];	/* count of these */
+} __packed;
+
+struct gb_spi_transfer_response {
+	__u8			data[0];	/* inbound data */
+} __packed;
+
+/* Version of the Greybus SVC protocol we support */
+#define GB_SVC_VERSION_MAJOR		0x00
+#define GB_SVC_VERSION_MINOR		0x01
+
+/* Greybus SVC request types */
+#define GB_SVC_TYPE_PROTOCOL_VERSION		0x01
+#define GB_SVC_TYPE_SVC_HELLO			0x02
+#define GB_SVC_TYPE_INTF_DEVICE_ID		0x03
+#define GB_SVC_TYPE_INTF_RESET			0x06
+#define GB_SVC_TYPE_CONN_CREATE			0x07
+#define GB_SVC_TYPE_CONN_DESTROY		0x08
+#define GB_SVC_TYPE_DME_PEER_GET		0x09
+#define GB_SVC_TYPE_DME_PEER_SET		0x0a
+#define GB_SVC_TYPE_ROUTE_CREATE		0x0b
+#define GB_SVC_TYPE_ROUTE_DESTROY		0x0c
+#define GB_SVC_TYPE_TIMESYNC_ENABLE		0x0d
+#define GB_SVC_TYPE_TIMESYNC_DISABLE		0x0e
+#define GB_SVC_TYPE_TIMESYNC_AUTHORITATIVE	0x0f
+#define GB_SVC_TYPE_INTF_SET_PWRM		0x10
+#define GB_SVC_TYPE_INTF_EJECT			0x11
+#define GB_SVC_TYPE_PING			0x13
+#define GB_SVC_TYPE_PWRMON_RAIL_COUNT_GET	0x14
+#define GB_SVC_TYPE_PWRMON_RAIL_NAMES_GET	0x15
+#define GB_SVC_TYPE_PWRMON_SAMPLE_GET		0x16
+#define GB_SVC_TYPE_PWRMON_INTF_SAMPLE_GET	0x17
+#define GB_SVC_TYPE_TIMESYNC_WAKE_PINS_ACQUIRE	0x18
+#define GB_SVC_TYPE_TIMESYNC_WAKE_PINS_RELEASE	0x19
+#define GB_SVC_TYPE_TIMESYNC_PING		0x1a
+#define GB_SVC_TYPE_MODULE_INSERTED		0x1f
+#define GB_SVC_TYPE_MODULE_REMOVED		0x20
+#define GB_SVC_TYPE_INTF_VSYS_ENABLE		0x21
+#define GB_SVC_TYPE_INTF_VSYS_DISABLE		0x22
+#define GB_SVC_TYPE_INTF_REFCLK_ENABLE		0x23
+#define GB_SVC_TYPE_INTF_REFCLK_DISABLE		0x24
+#define GB_SVC_TYPE_INTF_UNIPRO_ENABLE		0x25
+#define GB_SVC_TYPE_INTF_UNIPRO_DISABLE		0x26
+#define GB_SVC_TYPE_INTF_ACTIVATE		0x27
+#define GB_SVC_TYPE_INTF_RESUME			0x28
+#define GB_SVC_TYPE_INTF_MAILBOX_EVENT		0x29
+#define GB_SVC_TYPE_INTF_OOPS			0x2a
+
+/* Greybus SVC protocol status values */
+#define GB_SVC_OP_SUCCESS			0x00
+#define GB_SVC_OP_UNKNOWN_ERROR			0x01
+#define GB_SVC_INTF_NOT_DETECTED		0x02
+#define GB_SVC_INTF_NO_UPRO_LINK		0x03
+#define GB_SVC_INTF_UPRO_NOT_DOWN		0x04
+#define GB_SVC_INTF_UPRO_NOT_HIBERNATED		0x05
+#define GB_SVC_INTF_NO_V_SYS			0x06
+#define GB_SVC_INTF_V_CHG			0x07
+#define GB_SVC_INTF_WAKE_BUSY			0x08
+#define GB_SVC_INTF_NO_REFCLK			0x09
+#define GB_SVC_INTF_RELEASING			0x0a
+#define GB_SVC_INTF_NO_ORDER			0x0b
+#define GB_SVC_INTF_MBOX_SET			0x0c
+#define GB_SVC_INTF_BAD_MBOX			0x0d
+#define GB_SVC_INTF_OP_TIMEOUT			0x0e
+#define GB_SVC_PWRMON_OP_NOT_PRESENT		0x0f
+
+struct gb_svc_version_request {
+	__u8	major;
+	__u8	minor;
+} __packed;
+
+struct gb_svc_version_response {
+	__u8	major;
+	__u8	minor;
+} __packed;
+
+/* SVC protocol hello request */
+struct gb_svc_hello_request {
+	__le16			endo_id;
+	__u8			interface_id;
+} __packed;
+/* hello response has no payload */
+
+struct gb_svc_intf_device_id_request {
+	__u8	intf_id;
+	__u8	device_id;
+} __packed;
+/* device id response has no payload */
+
+struct gb_svc_intf_reset_request {
+	__u8	intf_id;
+} __packed;
+/* interface reset response has no payload */
+
+struct gb_svc_intf_eject_request {
+	__u8	intf_id;
+} __packed;
+/* interface eject response has no payload */
+
+struct gb_svc_conn_create_request {
+	__u8	intf1_id;
+	__le16	cport1_id;
+	__u8	intf2_id;
+	__le16	cport2_id;
+	__u8	tc;
+	__u8	flags;
+} __packed;
+/* connection create response has no payload */
+
+struct gb_svc_conn_destroy_request {
+	__u8	intf1_id;
+	__le16	cport1_id;
+	__u8	intf2_id;
+	__le16	cport2_id;
+} __packed;
+/* connection destroy response has no payload */
+
+struct gb_svc_dme_peer_get_request {
+	__u8	intf_id;
+	__le16	attr;
+	__le16	selector;
+} __packed;
+
+struct gb_svc_dme_peer_get_response {
+	__le16	result_code;
+	__le32	attr_value;
+} __packed;
+
+struct gb_svc_dme_peer_set_request {
+	__u8	intf_id;
+	__le16	attr;
+	__le16	selector;
+	__le32	value;
+} __packed;
+
+struct gb_svc_dme_peer_set_response {
+	__le16	result_code;
+} __packed;
+
+/* Greybus init-status values, currently retrieved using DME peer gets. */
+#define GB_INIT_SPI_BOOT_STARTED			0x02
+#define GB_INIT_TRUSTED_SPI_BOOT_FINISHED		0x03
+#define GB_INIT_UNTRUSTED_SPI_BOOT_FINISHED		0x04
+#define GB_INIT_BOOTROM_UNIPRO_BOOT_STARTED		0x06
+#define GB_INIT_BOOTROM_FALLBACK_UNIPRO_BOOT_STARTED	0x09
+#define GB_INIT_S2_LOADER_BOOT_STARTED			0x0D
+
+struct gb_svc_route_create_request {
+	__u8	intf1_id;
+	__u8	dev1_id;
+	__u8	intf2_id;
+	__u8	dev2_id;
+} __packed;
+/* route create response has no payload */
+
+struct gb_svc_route_destroy_request {
+	__u8	intf1_id;
+	__u8	intf2_id;
+} __packed;
+/* route destroy response has no payload */
+
+/* used for svc_intf_vsys_{enable,disable} */
+struct gb_svc_intf_vsys_request {
+	__u8	intf_id;
+} __packed;
+
+struct gb_svc_intf_vsys_response {
+	__u8	result_code;
+#define GB_SVC_INTF_VSYS_OK				0x00
+	/* 0x01 is reserved */
+#define GB_SVC_INTF_VSYS_FAIL				0x02
+} __packed;
+
+/* used for svc_intf_refclk_{enable,disable} */
+struct gb_svc_intf_refclk_request {
+	__u8	intf_id;
+} __packed;
+
+struct gb_svc_intf_refclk_response {
+	__u8	result_code;
+#define GB_SVC_INTF_REFCLK_OK				0x00
+	/* 0x01 is reserved */
+#define GB_SVC_INTF_REFCLK_FAIL				0x02
+} __packed;
+
+/* used for svc_intf_unipro_{enable,disable} */
+struct gb_svc_intf_unipro_request {
+	__u8	intf_id;
+} __packed;
+
+struct gb_svc_intf_unipro_response {
+	__u8	result_code;
+#define GB_SVC_INTF_UNIPRO_OK				0x00
+	/* 0x01 is reserved */
+#define GB_SVC_INTF_UNIPRO_FAIL				0x02
+#define GB_SVC_INTF_UNIPRO_NOT_OFF			0x03
+} __packed;
+
+struct gb_svc_timesync_enable_request {
+	__u8	count;
+	__le64	frame_time;
+	__le32	strobe_delay;
+	__le32	refclk;
+} __packed;
+/* timesync enable response has no payload */
+
+/* timesync authoritative request has no payload */
+struct gb_svc_timesync_authoritative_response {
+	__le64	frame_time[GB_TIMESYNC_MAX_STROBES];
+};
+
+struct gb_svc_timesync_wake_pins_acquire_request {
+	__le32	strobe_mask;
+};
+
+/* timesync wake pins acquire response has no payload */
+
+/* timesync wake pins release request has no payload */
+/* timesync wake pins release response has no payload */
+
+/* timesync svc ping request has no payload */
+struct gb_svc_timesync_ping_response {
+	__le64	frame_time;
+} __packed;
+
+#define GB_SVC_UNIPRO_FAST_MODE			0x01
+#define GB_SVC_UNIPRO_SLOW_MODE			0x02
+#define GB_SVC_UNIPRO_FAST_AUTO_MODE		0x04
+#define GB_SVC_UNIPRO_SLOW_AUTO_MODE		0x05
+#define GB_SVC_UNIPRO_MODE_UNCHANGED		0x07
+#define GB_SVC_UNIPRO_HIBERNATE_MODE		0x11
+#define GB_SVC_UNIPRO_OFF_MODE			0x12
+
+#define GB_SVC_SMALL_AMPLITUDE          0x01
+#define GB_SVC_LARGE_AMPLITUDE          0x02
+
+#define GB_SVC_NO_DE_EMPHASIS           0x00
+#define GB_SVC_SMALL_DE_EMPHASIS        0x01
+#define GB_SVC_LARGE_DE_EMPHASIS        0x02
+
+#define GB_SVC_PWRM_RXTERMINATION		0x01
+#define GB_SVC_PWRM_TXTERMINATION		0x02
+#define GB_SVC_PWRM_LINE_RESET			0x04
+#define GB_SVC_PWRM_SCRAMBLING			0x20
+
+#define GB_SVC_PWRM_QUIRK_HSSER			0x00000001
+
+#define GB_SVC_UNIPRO_HS_SERIES_A		0x01
+#define GB_SVC_UNIPRO_HS_SERIES_B		0x02
+
+#define GB_SVC_SETPWRM_PWR_OK           0x00
+#define GB_SVC_SETPWRM_PWR_LOCAL        0x01
+#define GB_SVC_SETPWRM_PWR_REMOTE       0x02
+#define GB_SVC_SETPWRM_PWR_BUSY         0x03
+#define GB_SVC_SETPWRM_PWR_ERROR_CAP    0x04
+#define GB_SVC_SETPWRM_PWR_FATAL_ERROR  0x05
+
+struct gb_svc_l2_timer_cfg {
+	__le16 tsb_fc0_protection_timeout;
+	__le16 tsb_tc0_replay_timeout;
+	__le16 tsb_afc0_req_timeout;
+	__le16 tsb_fc1_protection_timeout;
+	__le16 tsb_tc1_replay_timeout;
+	__le16 tsb_afc1_req_timeout;
+	__le16 reserved_for_tc2[3];
+	__le16 reserved_for_tc3[3];
+} __packed;
+
+struct gb_svc_intf_set_pwrm_request {
+	__u8	intf_id;
+	__u8	hs_series;
+	__u8	tx_mode;
+	__u8	tx_gear;
+	__u8	tx_nlanes;
+	__u8	tx_amplitude;
+	__u8	tx_hs_equalizer;
+	__u8	rx_mode;
+	__u8	rx_gear;
+	__u8	rx_nlanes;
+	__u8	flags;
+	__le32	quirks;
+	struct gb_svc_l2_timer_cfg local_l2timerdata, remote_l2timerdata;
+} __packed;
+
+struct gb_svc_intf_set_pwrm_response {
+	__u8	result_code;
+} __packed;
+
+struct gb_svc_key_event_request {
+	__le16  key_code;
+#define GB_KEYCODE_ARA         0x00
+
+	__u8    key_event;
+#define GB_SVC_KEY_RELEASED    0x00
+#define GB_SVC_KEY_PRESSED     0x01
+} __packed;
+
+#define GB_SVC_PWRMON_MAX_RAIL_COUNT		254
+
+struct gb_svc_pwrmon_rail_count_get_response {
+	__u8	rail_count;
+} __packed;
+
+#define GB_SVC_PWRMON_RAIL_NAME_BUFSIZE		32
+
+struct gb_svc_pwrmon_rail_names_get_response {
+	__u8	status;
+	__u8	name[0][GB_SVC_PWRMON_RAIL_NAME_BUFSIZE];
+} __packed;
+
+#define GB_SVC_PWRMON_TYPE_CURR			0x01
+#define GB_SVC_PWRMON_TYPE_VOL			0x02
+#define GB_SVC_PWRMON_TYPE_PWR			0x03
+
+#define GB_SVC_PWRMON_GET_SAMPLE_OK		0x00
+#define GB_SVC_PWRMON_GET_SAMPLE_INVAL		0x01
+#define GB_SVC_PWRMON_GET_SAMPLE_NOSUPP		0x02
+#define GB_SVC_PWRMON_GET_SAMPLE_HWERR		0x03
+
+struct gb_svc_pwrmon_sample_get_request {
+	__u8	rail_id;
+	__u8	measurement_type;
+} __packed;
+
+struct gb_svc_pwrmon_sample_get_response {
+	__u8	result;
+	__le32	measurement;
+} __packed;
+
+struct gb_svc_pwrmon_intf_sample_get_request {
+	__u8	intf_id;
+	__u8	measurement_type;
+} __packed;
+
+struct gb_svc_pwrmon_intf_sample_get_response {
+	__u8	result;
+	__le32	measurement;
+} __packed;
+
+#define GB_SVC_MODULE_INSERTED_FLAG_NO_PRIMARY	0x0001
+
+struct gb_svc_module_inserted_request {
+	__u8	primary_intf_id;
+	__u8	intf_count;
+	__le16	flags;
+} __packed;
+/* module_inserted response has no payload */
+
+struct gb_svc_module_removed_request {
+	__u8	primary_intf_id;
+} __packed;
+/* module_removed response has no payload */
+
+struct gb_svc_intf_activate_request {
+	__u8	intf_id;
+} __packed;
+
+#define GB_SVC_INTF_TYPE_UNKNOWN		0x00
+#define GB_SVC_INTF_TYPE_DUMMY			0x01
+#define GB_SVC_INTF_TYPE_UNIPRO			0x02
+#define GB_SVC_INTF_TYPE_GREYBUS		0x03
+
+struct gb_svc_intf_activate_response {
+	__u8	status;
+	__u8	intf_type;
+} __packed;
+
+struct gb_svc_intf_resume_request {
+	__u8	intf_id;
+} __packed;
+
+struct gb_svc_intf_resume_response {
+	__u8	status;
+} __packed;
+
+#define GB_SVC_INTF_MAILBOX_NONE		0x00
+#define GB_SVC_INTF_MAILBOX_AP			0x01
+#define GB_SVC_INTF_MAILBOX_GREYBUS		0x02
+
+struct gb_svc_intf_mailbox_event_request {
+	__u8	intf_id;
+	__le16	result_code;
+	__le32	mailbox;
+} __packed;
+/* intf_mailbox_event response has no payload */
+
+struct gb_svc_intf_oops_request {
+	__u8	intf_id;
+	__u8	reason;
+} __packed;
+/* intf_oops response has no payload */
+
+
+/* RAW */
+
+/* Greybus raw request types */
+#define	GB_RAW_TYPE_SEND			0x02
+
+struct gb_raw_send_request {
+	__le32	len;
+	__u8	data[0];
+} __packed;
+
+
+/* UART */
+
+/* Greybus UART operation types */
+#define GB_UART_TYPE_SEND_DATA			0x02
+#define GB_UART_TYPE_RECEIVE_DATA		0x03	/* Unsolicited data */
+#define GB_UART_TYPE_SET_LINE_CODING		0x04
+#define GB_UART_TYPE_SET_CONTROL_LINE_STATE	0x05
+#define GB_UART_TYPE_SEND_BREAK			0x06
+#define GB_UART_TYPE_SERIAL_STATE		0x07	/* Unsolicited data */
+#define GB_UART_TYPE_RECEIVE_CREDITS		0x08
+#define GB_UART_TYPE_FLUSH_FIFOS		0x09
+
+/* Represents data from AP -> Module */
+struct gb_uart_send_data_request {
+	__le16	size;
+	__u8	data[0];
+} __packed;
+
+/* recv-data-request flags */
+#define GB_UART_RECV_FLAG_FRAMING		0x01	/* Framing error */
+#define GB_UART_RECV_FLAG_PARITY		0x02	/* Parity error */
+#define GB_UART_RECV_FLAG_OVERRUN		0x04	/* Overrun error */
+#define GB_UART_RECV_FLAG_BREAK			0x08	/* Break */
+
+/* Represents data from Module -> AP */
+struct gb_uart_recv_data_request {
+	__le16	size;
+	__u8	flags;
+	__u8	data[0];
+} __packed;
+
+struct gb_uart_receive_credits_request {
+	__le16  count;
+} __packed;
+
+struct gb_uart_set_line_coding_request {
+	__le32	rate;
+	__u8	format;
+#define GB_SERIAL_1_STOP_BITS			0
+#define GB_SERIAL_1_5_STOP_BITS			1
+#define GB_SERIAL_2_STOP_BITS			2
+
+	__u8	parity;
+#define GB_SERIAL_NO_PARITY			0
+#define GB_SERIAL_ODD_PARITY			1
+#define GB_SERIAL_EVEN_PARITY			2
+#define GB_SERIAL_MARK_PARITY			3
+#define GB_SERIAL_SPACE_PARITY			4
+
+	__u8	data_bits;
+
+	__u8	flow_control;
+#define GB_SERIAL_AUTO_RTSCTS_EN		0x1
+} __packed;
+
+/* output control lines */
+#define GB_UART_CTRL_DTR			0x01
+#define GB_UART_CTRL_RTS			0x02
+
+struct gb_uart_set_control_line_state_request {
+	__u8	control;
+} __packed;
+
+struct gb_uart_set_break_request {
+	__u8	state;
+} __packed;
+
+/* input control lines and line errors */
+#define GB_UART_CTRL_DCD			0x01
+#define GB_UART_CTRL_DSR			0x02
+#define GB_UART_CTRL_RI				0x04
+
+struct gb_uart_serial_state_request {
+	__u8	control;
+} __packed;
+
+struct gb_uart_serial_flush_request {
+	__u8    flags;
+#define GB_SERIAL_FLAG_FLUSH_TRANSMITTER	0x01
+#define GB_SERIAL_FLAG_FLUSH_RECEIVER		0x02
+} __packed;
+
+/* Loopback */
+
+/* Greybus loopback request types */
+#define GB_LOOPBACK_TYPE_PING			0x02
+#define GB_LOOPBACK_TYPE_TRANSFER		0x03
+#define GB_LOOPBACK_TYPE_SINK			0x04
+
+/*
+ * Loopback request/response header format should be identical
+ * to simplify bandwidth and data movement analysis.
+ */
+struct gb_loopback_transfer_request {
+	__le32	len;
+	__le32  reserved0;
+	__le32  reserved1;
+	__u8	data[0];
+} __packed;
+
+struct gb_loopback_transfer_response {
+	__le32	len;
+	__le32	reserved0;
+	__le32	reserved1;
+	__u8	data[0];
+} __packed;
+
+/* SDIO */
+/* Greybus SDIO operation types */
+#define GB_SDIO_TYPE_GET_CAPABILITIES		0x02
+#define GB_SDIO_TYPE_SET_IOS			0x03
+#define GB_SDIO_TYPE_COMMAND			0x04
+#define GB_SDIO_TYPE_TRANSFER			0x05
+#define GB_SDIO_TYPE_EVENT			0x06
+
+/* get caps response: request has no payload */
+struct gb_sdio_get_caps_response {
+	__le32	caps;
+#define GB_SDIO_CAP_NONREMOVABLE	0x00000001
+#define GB_SDIO_CAP_4_BIT_DATA		0x00000002
+#define GB_SDIO_CAP_8_BIT_DATA		0x00000004
+#define GB_SDIO_CAP_MMC_HS		0x00000008
+#define GB_SDIO_CAP_SD_HS		0x00000010
+#define GB_SDIO_CAP_ERASE		0x00000020
+#define GB_SDIO_CAP_1_2V_DDR		0x00000040
+#define GB_SDIO_CAP_1_8V_DDR		0x00000080
+#define GB_SDIO_CAP_POWER_OFF_CARD	0x00000100
+#define GB_SDIO_CAP_UHS_SDR12		0x00000200
+#define GB_SDIO_CAP_UHS_SDR25		0x00000400
+#define GB_SDIO_CAP_UHS_SDR50		0x00000800
+#define GB_SDIO_CAP_UHS_SDR104		0x00001000
+#define GB_SDIO_CAP_UHS_DDR50		0x00002000
+#define GB_SDIO_CAP_DRIVER_TYPE_A	0x00004000
+#define GB_SDIO_CAP_DRIVER_TYPE_C	0x00008000
+#define GB_SDIO_CAP_DRIVER_TYPE_D	0x00010000
+#define GB_SDIO_CAP_HS200_1_2V		0x00020000
+#define GB_SDIO_CAP_HS200_1_8V		0x00040000
+#define GB_SDIO_CAP_HS400_1_2V		0x00080000
+#define GB_SDIO_CAP_HS400_1_8V		0x00100000
+
+	/* see possible values below at vdd */
+	__le32 ocr;
+	__le32 f_min;
+	__le32 f_max;
+	__le16 max_blk_count;
+	__le16 max_blk_size;
+} __packed;
+
+/* set ios request: response has no payload */
+struct gb_sdio_set_ios_request {
+	__le32	clock;
+	__le32	vdd;
+#define GB_SDIO_VDD_165_195	0x00000001
+#define GB_SDIO_VDD_20_21	0x00000002
+#define GB_SDIO_VDD_21_22	0x00000004
+#define GB_SDIO_VDD_22_23	0x00000008
+#define GB_SDIO_VDD_23_24	0x00000010
+#define GB_SDIO_VDD_24_25	0x00000020
+#define GB_SDIO_VDD_25_26	0x00000040
+#define GB_SDIO_VDD_26_27	0x00000080
+#define GB_SDIO_VDD_27_28	0x00000100
+#define GB_SDIO_VDD_28_29	0x00000200
+#define GB_SDIO_VDD_29_30	0x00000400
+#define GB_SDIO_VDD_30_31	0x00000800
+#define GB_SDIO_VDD_31_32	0x00001000
+#define GB_SDIO_VDD_32_33	0x00002000
+#define GB_SDIO_VDD_33_34	0x00004000
+#define GB_SDIO_VDD_34_35	0x00008000
+#define GB_SDIO_VDD_35_36	0x00010000
+
+	__u8	bus_mode;
+#define GB_SDIO_BUSMODE_OPENDRAIN	0x00
+#define GB_SDIO_BUSMODE_PUSHPULL	0x01
+
+	__u8	power_mode;
+#define GB_SDIO_POWER_OFF	0x00
+#define GB_SDIO_POWER_UP	0x01
+#define GB_SDIO_POWER_ON	0x02
+#define GB_SDIO_POWER_UNDEFINED	0x03
+
+	__u8	bus_width;
+#define GB_SDIO_BUS_WIDTH_1	0x00
+#define GB_SDIO_BUS_WIDTH_4	0x02
+#define GB_SDIO_BUS_WIDTH_8	0x03
+
+	__u8	timing;
+#define GB_SDIO_TIMING_LEGACY		0x00
+#define GB_SDIO_TIMING_MMC_HS		0x01
+#define GB_SDIO_TIMING_SD_HS		0x02
+#define GB_SDIO_TIMING_UHS_SDR12	0x03
+#define GB_SDIO_TIMING_UHS_SDR25	0x04
+#define GB_SDIO_TIMING_UHS_SDR50	0x05
+#define GB_SDIO_TIMING_UHS_SDR104	0x06
+#define GB_SDIO_TIMING_UHS_DDR50	0x07
+#define GB_SDIO_TIMING_MMC_DDR52	0x08
+#define GB_SDIO_TIMING_MMC_HS200	0x09
+#define GB_SDIO_TIMING_MMC_HS400	0x0A
+
+	__u8	signal_voltage;
+#define GB_SDIO_SIGNAL_VOLTAGE_330	0x00
+#define GB_SDIO_SIGNAL_VOLTAGE_180	0x01
+#define GB_SDIO_SIGNAL_VOLTAGE_120	0x02
+
+	__u8	drv_type;
+#define GB_SDIO_SET_DRIVER_TYPE_B	0x00
+#define GB_SDIO_SET_DRIVER_TYPE_A	0x01
+#define GB_SDIO_SET_DRIVER_TYPE_C	0x02
+#define GB_SDIO_SET_DRIVER_TYPE_D	0x03
+} __packed;
+
+/* command request */
+struct gb_sdio_command_request {
+	__u8	cmd;
+	__u8	cmd_flags;
+#define GB_SDIO_RSP_NONE		0x00
+#define GB_SDIO_RSP_PRESENT		0x01
+#define GB_SDIO_RSP_136			0x02
+#define GB_SDIO_RSP_CRC			0x04
+#define GB_SDIO_RSP_BUSY		0x08
+#define GB_SDIO_RSP_OPCODE		0x10
+
+	__u8	cmd_type;
+#define GB_SDIO_CMD_AC		0x00
+#define GB_SDIO_CMD_ADTC	0x01
+#define GB_SDIO_CMD_BC		0x02
+#define GB_SDIO_CMD_BCR		0x03
+
+	__le32	cmd_arg;
+	__le16	data_blocks;
+	__le16	data_blksz;
+} __packed;
+
+struct gb_sdio_command_response {
+	__le32	resp[4];
+} __packed;
+
+/* transfer request */
+struct gb_sdio_transfer_request {
+	__u8	data_flags;
+#define GB_SDIO_DATA_WRITE	0x01
+#define GB_SDIO_DATA_READ	0x02
+#define GB_SDIO_DATA_STREAM	0x04
+
+	__le16	data_blocks;
+	__le16	data_blksz;
+	__u8	data[0];
+} __packed;
+
+struct gb_sdio_transfer_response {
+	__le16	data_blocks;
+	__le16	data_blksz;
+	__u8	data[0];
+} __packed;
+
+/* event request: generated by module and is defined as unidirectional */
+struct gb_sdio_event_request {
+	__u8	event;
+#define GB_SDIO_CARD_INSERTED	0x01
+#define GB_SDIO_CARD_REMOVED	0x02
+#define GB_SDIO_WP		0x04
+} __packed;
+
+/* Camera */
+
+/* Greybus Camera request types */
+#define GB_CAMERA_TYPE_CAPABILITIES		0x02
+#define GB_CAMERA_TYPE_CONFIGURE_STREAMS	0x03
+#define GB_CAMERA_TYPE_CAPTURE			0x04
+#define GB_CAMERA_TYPE_FLUSH			0x05
+#define GB_CAMERA_TYPE_METADATA			0x06
+
+#define GB_CAMERA_MAX_STREAMS			4
+#define GB_CAMERA_MAX_SETTINGS_SIZE		8192
+
+/* Greybus Camera Configure Streams request payload */
+struct gb_camera_stream_config_request {
+	__le16 width;
+	__le16 height;
+	__le16 format;
+	__le16 padding;
+} __packed;
+
+struct gb_camera_configure_streams_request {
+	__u8 num_streams;
+	__u8 flags;
+#define GB_CAMERA_CONFIGURE_STREAMS_TEST_ONLY	0x01
+	__le16 padding;
+	struct gb_camera_stream_config_request config[0];
+} __packed;
+
+/* Greybus Camera Configure Streams response payload */
+struct gb_camera_stream_config_response {
+	__le16 width;
+	__le16 height;
+	__le16 format;
+	__u8 virtual_channel;
+	__u8 data_type[2];
+	__le16 max_pkt_size;
+	__u8 padding;
+	__le32 max_size;
+} __packed;
+
+struct gb_camera_configure_streams_response {
+	__u8 num_streams;
+#define GB_CAMERA_CONFIGURE_STREAMS_ADJUSTED	0x01
+	__u8 flags;
+	__u8 padding[2];
+	__le32 data_rate;
+	struct gb_camera_stream_config_response config[0];
+};
+
+/* Greybus Camera Capture request payload - response has no payload */
+struct gb_camera_capture_request {
+	__le32 request_id;
+	__u8 streams;
+	__u8 padding;
+	__le16 num_frames;
+	__u8 settings[0];
+} __packed;
+
+/* Greybus Camera Flush response payload - request has no payload */
+struct gb_camera_flush_response {
+	__le32 request_id;
+} __packed;
+
+/* Greybus Camera Metadata request payload - operation has no response */
+struct gb_camera_metadata_request {
+	__le32 request_id;
+	__le16 frame_number;
+	__u8 stream;
+	__u8 padding;
+	__u8 metadata[0];
+} __packed;
+
+/* Lights */
+
+/* Greybus Lights request types */
+#define GB_LIGHTS_TYPE_GET_LIGHTS		0x02
+#define GB_LIGHTS_TYPE_GET_LIGHT_CONFIG		0x03
+#define GB_LIGHTS_TYPE_GET_CHANNEL_CONFIG	0x04
+#define GB_LIGHTS_TYPE_GET_CHANNEL_FLASH_CONFIG	0x05
+#define GB_LIGHTS_TYPE_SET_BRIGHTNESS		0x06
+#define GB_LIGHTS_TYPE_SET_BLINK		0x07
+#define GB_LIGHTS_TYPE_SET_COLOR		0x08
+#define GB_LIGHTS_TYPE_SET_FADE			0x09
+#define GB_LIGHTS_TYPE_EVENT			0x0A
+#define GB_LIGHTS_TYPE_SET_FLASH_INTENSITY	0x0B
+#define GB_LIGHTS_TYPE_SET_FLASH_STROBE		0x0C
+#define GB_LIGHTS_TYPE_SET_FLASH_TIMEOUT	0x0D
+#define GB_LIGHTS_TYPE_GET_FLASH_FAULT		0x0E
+
+/* Greybus Light modes */
+
+/*
+ * if you add any specific mode below, update also the
+ * GB_CHANNEL_MODE_DEFINED_RANGE value accordingly
+ */
+#define GB_CHANNEL_MODE_NONE		0x00000000
+#define GB_CHANNEL_MODE_BATTERY		0x00000001
+#define GB_CHANNEL_MODE_POWER		0x00000002
+#define GB_CHANNEL_MODE_WIRELESS	0x00000004
+#define GB_CHANNEL_MODE_BLUETOOTH	0x00000008
+#define GB_CHANNEL_MODE_KEYBOARD	0x00000010
+#define GB_CHANNEL_MODE_BUTTONS		0x00000020
+#define GB_CHANNEL_MODE_NOTIFICATION	0x00000040
+#define GB_CHANNEL_MODE_ATTENTION	0x00000080
+#define GB_CHANNEL_MODE_FLASH		0x00000100
+#define GB_CHANNEL_MODE_TORCH		0x00000200
+#define GB_CHANNEL_MODE_INDICATOR	0x00000400
+
+/* Lights Mode valid bit values */
+#define GB_CHANNEL_MODE_DEFINED_RANGE	0x000004FF
+#define GB_CHANNEL_MODE_VENDOR_RANGE	0x00F00000
+
+/* Greybus Light Channels Flags */
+#define GB_LIGHT_CHANNEL_MULTICOLOR	0x00000001
+#define GB_LIGHT_CHANNEL_FADER		0x00000002
+#define GB_LIGHT_CHANNEL_BLINK		0x00000004
+
+/* get count of lights in module */
+struct gb_lights_get_lights_response {
+	__u8	lights_count;
+} __packed;
+
+/* light config request payload */
+struct gb_lights_get_light_config_request {
+	__u8	id;
+} __packed;
+
+/* light config response payload */
+struct gb_lights_get_light_config_response {
+	__u8	channel_count;
+	__u8	name[32];
+} __packed;
+
+/* channel config request payload */
+struct gb_lights_get_channel_config_request {
+	__u8	light_id;
+	__u8	channel_id;
+} __packed;
+
+/* channel flash config request payload */
+struct gb_lights_get_channel_flash_config_request {
+	__u8	light_id;
+	__u8	channel_id;
+} __packed;
+
+/* channel config response payload */
+struct gb_lights_get_channel_config_response {
+	__u8	max_brightness;
+	__le32	flags;
+	__le32	color;
+	__u8	color_name[32];
+	__le32	mode;
+	__u8	mode_name[32];
+} __packed;
+
+/* channel flash config response payload */
+struct gb_lights_get_channel_flash_config_response {
+	__le32	intensity_min_uA;
+	__le32	intensity_max_uA;
+	__le32	intensity_step_uA;
+	__le32	timeout_min_us;
+	__le32	timeout_max_us;
+	__le32	timeout_step_us;
+} __packed;
+
+/* blink request payload: response have no payload */
+struct gb_lights_blink_request {
+	__u8	light_id;
+	__u8	channel_id;
+	__le16	time_on_ms;
+	__le16	time_off_ms;
+} __packed;
+
+/* set brightness request payload: response have no payload */
+struct gb_lights_set_brightness_request {
+	__u8	light_id;
+	__u8	channel_id;
+	__u8	brightness;
+} __packed;
+
+/* set color request payload: response have no payload */
+struct gb_lights_set_color_request {
+	__u8	light_id;
+	__u8	channel_id;
+	__le32	color;
+} __packed;
+
+/* set fade request payload: response have no payload */
+struct gb_lights_set_fade_request {
+	__u8	light_id;
+	__u8	channel_id;
+	__u8	fade_in;
+	__u8	fade_out;
+} __packed;
+
+/* event request: generated by module */
+struct gb_lights_event_request {
+	__u8	light_id;
+	__u8	event;
+#define GB_LIGHTS_LIGHT_CONFIG		0x01
+} __packed;
+
+/* set flash intensity request payload: response have no payload */
+struct gb_lights_set_flash_intensity_request {
+	__u8	light_id;
+	__u8	channel_id;
+	__le32	intensity_uA;
+} __packed;
+
+/* set flash strobe state request payload: response have no payload */
+struct gb_lights_set_flash_strobe_request {
+	__u8	light_id;
+	__u8	channel_id;
+	__u8	state;
+} __packed;
+
+/* set flash timeout request payload: response have no payload */
+struct gb_lights_set_flash_timeout_request {
+	__u8	light_id;
+	__u8	channel_id;
+	__le32	timeout_us;
+} __packed;
+
+/* get flash fault request payload */
+struct gb_lights_get_flash_fault_request {
+	__u8	light_id;
+	__u8	channel_id;
+} __packed;
+
+/* get flash fault response payload */
+struct gb_lights_get_flash_fault_response {
+	__le32	fault;
+#define GB_LIGHTS_FLASH_FAULT_OVER_VOLTAGE		0x00000000
+#define GB_LIGHTS_FLASH_FAULT_TIMEOUT			0x00000001
+#define GB_LIGHTS_FLASH_FAULT_OVER_TEMPERATURE		0x00000002
+#define GB_LIGHTS_FLASH_FAULT_SHORT_CIRCUIT		0x00000004
+#define GB_LIGHTS_FLASH_FAULT_OVER_CURRENT		0x00000008
+#define GB_LIGHTS_FLASH_FAULT_INDICATOR			0x00000010
+#define GB_LIGHTS_FLASH_FAULT_UNDER_VOLTAGE		0x00000020
+#define GB_LIGHTS_FLASH_FAULT_INPUT_VOLTAGE		0x00000040
+#define GB_LIGHTS_FLASH_FAULT_LED_OVER_TEMPERATURE	0x00000080
+} __packed;
+
+/* Audio */
+
+#define GB_AUDIO_TYPE_GET_TOPOLOGY_SIZE		0x02
+#define GB_AUDIO_TYPE_GET_TOPOLOGY		0x03
+#define GB_AUDIO_TYPE_GET_CONTROL		0x04
+#define GB_AUDIO_TYPE_SET_CONTROL		0x05
+#define GB_AUDIO_TYPE_ENABLE_WIDGET		0x06
+#define GB_AUDIO_TYPE_DISABLE_WIDGET		0x07
+#define GB_AUDIO_TYPE_GET_PCM			0x08
+#define GB_AUDIO_TYPE_SET_PCM			0x09
+#define GB_AUDIO_TYPE_SET_TX_DATA_SIZE		0x0a
+						/* 0x0b unused */
+#define GB_AUDIO_TYPE_ACTIVATE_TX		0x0c
+#define GB_AUDIO_TYPE_DEACTIVATE_TX		0x0d
+#define GB_AUDIO_TYPE_SET_RX_DATA_SIZE		0x0e
+						/* 0x0f unused */
+#define GB_AUDIO_TYPE_ACTIVATE_RX		0x10
+#define GB_AUDIO_TYPE_DEACTIVATE_RX		0x11
+#define GB_AUDIO_TYPE_JACK_EVENT		0x12
+#define GB_AUDIO_TYPE_BUTTON_EVENT		0x13
+#define GB_AUDIO_TYPE_STREAMING_EVENT		0x14
+#define GB_AUDIO_TYPE_SEND_DATA			0x15
+
+/* Module must be able to buffer 10ms of audio data, minimum */
+#define GB_AUDIO_SAMPLE_BUFFER_MIN_US		10000
+
+#define GB_AUDIO_PCM_NAME_MAX			32
+#define AUDIO_DAI_NAME_MAX			32
+#define AUDIO_CONTROL_NAME_MAX			32
+#define AUDIO_CTL_ELEM_NAME_MAX			44
+#define AUDIO_ENUM_NAME_MAX			64
+#define AUDIO_WIDGET_NAME_MAX			32
+
+/* See SNDRV_PCM_FMTBIT_* in Linux source */
+#define GB_AUDIO_PCM_FMT_S8			BIT(0)
+#define GB_AUDIO_PCM_FMT_U8			BIT(1)
+#define GB_AUDIO_PCM_FMT_S16_LE			BIT(2)
+#define GB_AUDIO_PCM_FMT_S16_BE			BIT(3)
+#define GB_AUDIO_PCM_FMT_U16_LE			BIT(4)
+#define GB_AUDIO_PCM_FMT_U16_BE			BIT(5)
+#define GB_AUDIO_PCM_FMT_S24_LE			BIT(6)
+#define GB_AUDIO_PCM_FMT_S24_BE			BIT(7)
+#define GB_AUDIO_PCM_FMT_U24_LE			BIT(8)
+#define GB_AUDIO_PCM_FMT_U24_BE			BIT(9)
+#define GB_AUDIO_PCM_FMT_S32_LE			BIT(10)
+#define GB_AUDIO_PCM_FMT_S32_BE			BIT(11)
+#define GB_AUDIO_PCM_FMT_U32_LE			BIT(12)
+#define GB_AUDIO_PCM_FMT_U32_BE			BIT(13)
+
+/* See SNDRV_PCM_RATE_* in Linux source */
+#define GB_AUDIO_PCM_RATE_5512			BIT(0)
+#define GB_AUDIO_PCM_RATE_8000			BIT(1)
+#define GB_AUDIO_PCM_RATE_11025			BIT(2)
+#define GB_AUDIO_PCM_RATE_16000			BIT(3)
+#define GB_AUDIO_PCM_RATE_22050			BIT(4)
+#define GB_AUDIO_PCM_RATE_32000			BIT(5)
+#define GB_AUDIO_PCM_RATE_44100			BIT(6)
+#define GB_AUDIO_PCM_RATE_48000			BIT(7)
+#define GB_AUDIO_PCM_RATE_64000			BIT(8)
+#define GB_AUDIO_PCM_RATE_88200			BIT(9)
+#define GB_AUDIO_PCM_RATE_96000			BIT(10)
+#define GB_AUDIO_PCM_RATE_176400		BIT(11)
+#define GB_AUDIO_PCM_RATE_192000		BIT(12)
+
+#define GB_AUDIO_STREAM_TYPE_CAPTURE		0x1
+#define GB_AUDIO_STREAM_TYPE_PLAYBACK		0x2
+
+#define GB_AUDIO_CTL_ELEM_ACCESS_READ		BIT(0)
+#define GB_AUDIO_CTL_ELEM_ACCESS_WRITE		BIT(1)
+
+/* See SNDRV_CTL_ELEM_TYPE_* in Linux source */
+#define GB_AUDIO_CTL_ELEM_TYPE_BOOLEAN		0x01
+#define GB_AUDIO_CTL_ELEM_TYPE_INTEGER		0x02
+#define GB_AUDIO_CTL_ELEM_TYPE_ENUMERATED	0x03
+#define GB_AUDIO_CTL_ELEM_TYPE_INTEGER64	0x06
+
+/* See SNDRV_CTL_ELEM_IFACE_* in Linux source */
+#define GB_AUDIO_CTL_ELEM_IFACE_CARD		0x00
+#define GB_AUDIO_CTL_ELEM_IFACE_HWDEP		0x01
+#define GB_AUDIO_CTL_ELEM_IFACE_MIXER		0x02
+#define GB_AUDIO_CTL_ELEM_IFACE_PCM		0x03
+#define GB_AUDIO_CTL_ELEM_IFACE_RAWMIDI		0x04
+#define GB_AUDIO_CTL_ELEM_IFACE_TIMER		0x05
+#define GB_AUDIO_CTL_ELEM_IFACE_SEQUENCER	0x06
+
+/* SNDRV_CTL_ELEM_ACCESS_* in Linux source */
+#define GB_AUDIO_ACCESS_READ			BIT(0)
+#define GB_AUDIO_ACCESS_WRITE			BIT(1)
+#define GB_AUDIO_ACCESS_VOLATILE		BIT(2)
+#define GB_AUDIO_ACCESS_TIMESTAMP		BIT(3)
+#define GB_AUDIO_ACCESS_TLV_READ		BIT(4)
+#define GB_AUDIO_ACCESS_TLV_WRITE		BIT(5)
+#define GB_AUDIO_ACCESS_TLV_COMMAND		BIT(6)
+#define GB_AUDIO_ACCESS_INACTIVE		BIT(7)
+#define GB_AUDIO_ACCESS_LOCK			BIT(8)
+#define GB_AUDIO_ACCESS_OWNER			BIT(9)
+
+/* enum snd_soc_dapm_type */
+#define GB_AUDIO_WIDGET_TYPE_INPUT		0x0
+#define GB_AUDIO_WIDGET_TYPE_OUTPUT		0x1
+#define GB_AUDIO_WIDGET_TYPE_MUX		0x2
+#define GB_AUDIO_WIDGET_TYPE_VIRT_MUX		0x3
+#define GB_AUDIO_WIDGET_TYPE_VALUE_MUX		0x4
+#define GB_AUDIO_WIDGET_TYPE_MIXER		0x5
+#define GB_AUDIO_WIDGET_TYPE_MIXER_NAMED_CTL	0x6
+#define GB_AUDIO_WIDGET_TYPE_PGA		0x7
+#define GB_AUDIO_WIDGET_TYPE_OUT_DRV		0x8
+#define GB_AUDIO_WIDGET_TYPE_ADC		0x9
+#define GB_AUDIO_WIDGET_TYPE_DAC		0xa
+#define GB_AUDIO_WIDGET_TYPE_MICBIAS		0xb
+#define GB_AUDIO_WIDGET_TYPE_MIC		0xc
+#define GB_AUDIO_WIDGET_TYPE_HP			0xd
+#define GB_AUDIO_WIDGET_TYPE_SPK		0xe
+#define GB_AUDIO_WIDGET_TYPE_LINE		0xf
+#define GB_AUDIO_WIDGET_TYPE_SWITCH		0x10
+#define GB_AUDIO_WIDGET_TYPE_VMID		0x11
+#define GB_AUDIO_WIDGET_TYPE_PRE		0x12
+#define GB_AUDIO_WIDGET_TYPE_POST		0x13
+#define GB_AUDIO_WIDGET_TYPE_SUPPLY		0x14
+#define GB_AUDIO_WIDGET_TYPE_REGULATOR_SUPPLY	0x15
+#define GB_AUDIO_WIDGET_TYPE_CLOCK_SUPPLY	0x16
+#define GB_AUDIO_WIDGET_TYPE_AIF_IN		0x17
+#define GB_AUDIO_WIDGET_TYPE_AIF_OUT		0x18
+#define GB_AUDIO_WIDGET_TYPE_SIGGEN		0x19
+#define GB_AUDIO_WIDGET_TYPE_DAI_IN		0x1a
+#define GB_AUDIO_WIDGET_TYPE_DAI_OUT		0x1b
+#define GB_AUDIO_WIDGET_TYPE_DAI_LINK		0x1c
+
+#define GB_AUDIO_WIDGET_STATE_DISABLED		0x01
+#define GB_AUDIO_WIDGET_STATE_ENAABLED		0x02
+
+#define GB_AUDIO_JACK_EVENT_INSERTION		0x1
+#define GB_AUDIO_JACK_EVENT_REMOVAL		0x2
+
+#define GB_AUDIO_BUTTON_EVENT_PRESS		0x1
+#define GB_AUDIO_BUTTON_EVENT_RELEASE		0x2
+
+#define GB_AUDIO_STREAMING_EVENT_UNSPECIFIED	0x1
+#define GB_AUDIO_STREAMING_EVENT_HALT		0x2
+#define GB_AUDIO_STREAMING_EVENT_INTERNAL_ERROR	0x3
+#define GB_AUDIO_STREAMING_EVENT_PROTOCOL_ERROR	0x4
+#define GB_AUDIO_STREAMING_EVENT_FAILURE	0x5
+#define GB_AUDIO_STREAMING_EVENT_UNDERRUN	0x6
+#define GB_AUDIO_STREAMING_EVENT_OVERRUN	0x7
+#define GB_AUDIO_STREAMING_EVENT_CLOCKING	0x8
+#define GB_AUDIO_STREAMING_EVENT_DATA_LEN	0x9
+
+#define GB_AUDIO_INVALID_INDEX			0xff
+
+/* enum snd_jack_types */
+#define GB_AUDIO_JACK_HEADPHONE			0x0000001
+#define GB_AUDIO_JACK_MICROPHONE		0x0000002
+#define GB_AUDIO_JACK_HEADSET			(GB_AUDIO_JACK_HEADPHONE | \
+						 GB_AUDIO_JACK_MICROPHONE)
+#define GB_AUDIO_JACK_LINEOUT			0x0000004
+#define GB_AUDIO_JACK_MECHANICAL		0x0000008
+#define GB_AUDIO_JACK_VIDEOOUT			0x0000010
+#define GB_AUDIO_JACK_AVOUT			(GB_AUDIO_JACK_LINEOUT | \
+						 GB_AUDIO_JACK_VIDEOOUT)
+#define GB_AUDIO_JACK_LINEIN			0x0000020
+#define GB_AUDIO_JACK_OC_HPHL			0x0000040
+#define GB_AUDIO_JACK_OC_HPHR			0x0000080
+#define GB_AUDIO_JACK_MICROPHONE2		0x0000200
+#define GB_AUDIO_JACK_ANC_HEADPHONE		(GB_AUDIO_JACK_HEADPHONE | \
+						 GB_AUDIO_JACK_MICROPHONE | \
+						 GB_AUDIO_JACK_MICROPHONE2)
+/* Kept separate from switches to facilitate implementation */
+#define GB_AUDIO_JACK_BTN_0			0x4000000
+#define GB_AUDIO_JACK_BTN_1			0x2000000
+#define GB_AUDIO_JACK_BTN_2			0x1000000
+#define GB_AUDIO_JACK_BTN_3			0x0800000
+
+struct gb_audio_pcm {
+	__u8	stream_name[GB_AUDIO_PCM_NAME_MAX];
+	__le32	formats;	/* GB_AUDIO_PCM_FMT_* */
+	__le32	rates;		/* GB_AUDIO_PCM_RATE_* */
+	__u8	chan_min;
+	__u8	chan_max;
+	__u8	sig_bits;	/* number of bits of content */
+} __packed;
+
+struct gb_audio_dai {
+	__u8			name[AUDIO_DAI_NAME_MAX];
+	__le16			data_cport;
+	struct gb_audio_pcm	capture;
+	struct gb_audio_pcm	playback;
+} __packed;
+
+struct gb_audio_integer {
+	__le32	min;
+	__le32	max;
+	__le32	step;
+} __packed;
+
+struct gb_audio_integer64 {
+	__le64	min;
+	__le64	max;
+	__le64	step;
+} __packed;
+
+struct gb_audio_enumerated {
+	__le32	items;
+	__le16	names_length;
+	__u8	names[0];
+} __packed;
+
+struct gb_audio_ctl_elem_info { /* See snd_ctl_elem_info in Linux source */
+	__u8		type;		/* GB_AUDIO_CTL_ELEM_TYPE_* */
+	__le16		dimen[4];
+	union {
+		struct gb_audio_integer		integer;
+		struct gb_audio_integer64	integer64;
+		struct gb_audio_enumerated	enumerated;
+	} value;
+} __packed;
+
+struct gb_audio_ctl_elem_value { /* See snd_ctl_elem_value in Linux source */
+	__le64				timestamp; /* XXX needed? */
+	union {
+		__le32	integer_value[2];	/* consider CTL_DOUBLE_xxx */
+		__le64	integer64_value[2];
+		__le32	enumerated_item[2];
+	} value;
+} __packed;
+
+struct gb_audio_control {
+	__u8	name[AUDIO_CONTROL_NAME_MAX];
+	__u8	id;		/* 0-63 */
+	__u8	iface;		/* GB_AUDIO_IFACE_* */
+	__le16	data_cport;
+	__le32	access;		/* GB_AUDIO_ACCESS_* */
+	__u8    count;		/* count of same elements */
+	__u8	count_values;	/* count of values, max=2 for CTL_DOUBLE_xxx */
+	struct gb_audio_ctl_elem_info	info;
+} __packed;
+
+struct gb_audio_widget {
+	__u8	name[AUDIO_WIDGET_NAME_MAX];
+	__u8	sname[AUDIO_WIDGET_NAME_MAX];
+	__u8	id;
+	__u8	type;		/* GB_AUDIO_WIDGET_TYPE_* */
+	__u8	state;		/* GB_AUDIO_WIDGET_STATE_* */
+	__u8	ncontrols;
+	struct gb_audio_control	ctl[0];	/* 'ncontrols' entries */
+} __packed;
+
+struct gb_audio_route {
+	__u8	source_id;	/* widget id */
+	__u8	destination_id;	/* widget id */
+	__u8	control_id;	/* 0-63 */
+	__u8	index;		/* Selection within the control */
+} __packed;
+
+struct gb_audio_topology {
+	__u8	num_dais;
+	__u8	num_controls;
+	__u8	num_widgets;
+	__u8	num_routes;
+	__le32	size_dais;
+	__le32	size_controls;
+	__le32	size_widgets;
+	__le32	size_routes;
+	__le32	jack_type;
+	/*
+	 * struct gb_audio_dai		dai[num_dais];
+	 * struct gb_audio_control	controls[num_controls];
+	 * struct gb_audio_widget	widgets[num_widgets];
+	 * struct gb_audio_route	routes[num_routes];
+	 */
+	__u8	data[0];
+} __packed;
+
+struct gb_audio_get_topology_size_response {
+	__le16	size;
+} __packed;
+
+struct gb_audio_get_topology_response {
+	struct gb_audio_topology	topology;
+} __packed;
+
+struct gb_audio_get_control_request {
+	__u8	control_id;
+	__u8	index;
+} __packed;
+
+struct gb_audio_get_control_response {
+	struct gb_audio_ctl_elem_value	value;
+} __packed;
+
+struct gb_audio_set_control_request {
+	__u8	control_id;
+	__u8	index;
+	struct gb_audio_ctl_elem_value	value;
+} __packed;
+
+struct gb_audio_enable_widget_request {
+	__u8	widget_id;
+} __packed;
+
+struct gb_audio_disable_widget_request {
+	__u8	widget_id;
+} __packed;
+
+struct gb_audio_get_pcm_request {
+	__le16	data_cport;
+} __packed;
+
+struct gb_audio_get_pcm_response {
+	__le32	format;
+	__le32	rate;
+	__u8	channels;
+	__u8	sig_bits;
+} __packed;
+
+struct gb_audio_set_pcm_request {
+	__le16	data_cport;
+	__le32	format;
+	__le32	rate;
+	__u8	channels;
+	__u8	sig_bits;
+} __packed;
+
+struct gb_audio_set_tx_data_size_request {
+	__le16	data_cport;
+	__le16	size;
+} __packed;
+
+struct gb_audio_activate_tx_request {
+	__le16	data_cport;
+} __packed;
+
+struct gb_audio_deactivate_tx_request {
+	__le16	data_cport;
+} __packed;
+
+struct gb_audio_set_rx_data_size_request {
+	__le16	data_cport;
+	__le16	size;
+} __packed;
+
+struct gb_audio_activate_rx_request {
+	__le16	data_cport;
+} __packed;
+
+struct gb_audio_deactivate_rx_request {
+	__le16	data_cport;
+} __packed;
+
+struct gb_audio_jack_event_request {
+	__u8	widget_id;
+	__u8	jack_attribute;
+	__u8	event;
+} __packed;
+
+struct gb_audio_button_event_request {
+	__u8	widget_id;
+	__u8	button_id;
+	__u8	event;
+} __packed;
+
+struct gb_audio_streaming_event_request {
+	__le16	data_cport;
+	__u8	event;
+} __packed;
+
+struct gb_audio_send_data_request {
+	__le64	timestamp;
+	__u8	data[0];
+} __packed;
+
+
+/* Log */
+
+/* operations */
+#define GB_LOG_TYPE_SEND_LOG	0x02
+
+/* length */
+#define GB_LOG_MAX_LEN		1024
+
+struct gb_log_send_log_request {
+	__le16	len;
+	__u8    msg[0];
+} __packed;
+
+#endif /* __GREYBUS_PROTOCOLS_H */
+
diff --git a/drivers/staging/greybus/greybus_trace.h b/drivers/staging/greybus/greybus_trace.h
new file mode 100644
index 000000000000..6f8692da9ec8
--- /dev/null
+++ b/drivers/staging/greybus/greybus_trace.h
@@ -0,0 +1,531 @@
+/*
+ * Greybus driver and device API
+ *
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM greybus
+
+#if !defined(_TRACE_GREYBUS_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_GREYBUS_H
+
+#include <linux/tracepoint.h>
+
+struct gb_message;
+struct gb_operation;
+struct gb_connection;
+struct gb_bundle;
+struct gb_host_device;
+
+DECLARE_EVENT_CLASS(gb_message,
+
+	TP_PROTO(struct gb_message *message),
+
+	TP_ARGS(message),
+
+	TP_STRUCT__entry(
+		__field(u16, size)
+		__field(u16, operation_id)
+		__field(u8, type)
+		__field(u8, result)
+	),
+
+	TP_fast_assign(
+		__entry->size = le16_to_cpu(message->header->size);
+		__entry->operation_id =
+			le16_to_cpu(message->header->operation_id);
+		__entry->type = message->header->type;
+		__entry->result = message->header->result;
+	),
+
+	TP_printk("size=%hu operation_id=0x%04x type=0x%02x result=0x%02x",
+		  __entry->size, __entry->operation_id,
+		  __entry->type, __entry->result)
+);
+
+#define DEFINE_MESSAGE_EVENT(name)					\
+		DEFINE_EVENT(gb_message, name,				\
+				TP_PROTO(struct gb_message *message),	\
+				TP_ARGS(message))
+
+/*
+ * Occurs immediately before calling a host device's message_send()
+ * method.
+ */
+DEFINE_MESSAGE_EVENT(gb_message_send);
+
+/*
+ * Occurs after an incoming request message has been received
+ */
+DEFINE_MESSAGE_EVENT(gb_message_recv_request);
+
+/*
+ * Occurs after an incoming response message has been received,
+ * after its matching request has been found.
+ */
+DEFINE_MESSAGE_EVENT(gb_message_recv_response);
+
+/*
+ * Occurs after an operation has been canceled, possibly before the
+ * cancellation is complete.
+ */
+DEFINE_MESSAGE_EVENT(gb_message_cancel_outgoing);
+
+/*
+ * Occurs when an incoming request is cancelled; if the response has
+ * been queued for sending, this occurs after it is sent.
+ */
+DEFINE_MESSAGE_EVENT(gb_message_cancel_incoming);
+
+/*
+ * Occurs in the host driver message_send() function just prior to
+ * handing off the data to be processed by hardware.
+ */
+DEFINE_MESSAGE_EVENT(gb_message_submit);
+
+#undef DEFINE_MESSAGE_EVENT
+
+DECLARE_EVENT_CLASS(gb_operation,
+
+	TP_PROTO(struct gb_operation *operation),
+
+	TP_ARGS(operation),
+
+	TP_STRUCT__entry(
+		__field(u16, cport_id)	/* CPort of HD side of connection */
+		__field(u16, id)	/* Operation ID */
+		__field(u8, type)
+		__field(unsigned long, flags)
+		__field(int, active)
+		__field(int, waiters)
+		__field(int, errno)
+	),
+
+	TP_fast_assign(
+		__entry->cport_id = operation->connection->hd_cport_id;
+		__entry->id = operation->id;
+		__entry->type = operation->type;
+		__entry->flags = operation->flags;
+		__entry->active = operation->active;
+		__entry->waiters = atomic_read(&operation->waiters);
+		__entry->errno = operation->errno;
+	),
+
+	TP_printk("id=%04x type=0x%02x cport_id=%04x flags=0x%lx active=%d waiters=%d errno=%d",
+		  __entry->id, __entry->cport_id, __entry->type, __entry->flags,
+		  __entry->active, __entry->waiters, __entry->errno)
+);
+
+#define DEFINE_OPERATION_EVENT(name)					\
+		DEFINE_EVENT(gb_operation, name,			\
+				TP_PROTO(struct gb_operation *operation), \
+				TP_ARGS(operation))
+
+/*
+ * Occurs after a new operation is created for an outgoing request
+ * has been successfully created.
+ */
+DEFINE_OPERATION_EVENT(gb_operation_create);
+
+/*
+ * Occurs after a new core operation has been created.
+ */
+DEFINE_OPERATION_EVENT(gb_operation_create_core);
+
+/*
+ * Occurs after a new operation has been created for an incoming
+ * request has been successfully created and initialized.
+ */
+DEFINE_OPERATION_EVENT(gb_operation_create_incoming);
+
+/*
+ * Occurs when the last reference to an operation has been dropped,
+ * prior to freeing resources.
+ */
+DEFINE_OPERATION_EVENT(gb_operation_destroy);
+
+/*
+ * Occurs when an operation has been marked active, after updating
+ * its active count.
+ */
+DEFINE_OPERATION_EVENT(gb_operation_get_active);
+
+/*
+ * Occurs when an operation has been marked active, before updating
+ * its active count.
+ */
+DEFINE_OPERATION_EVENT(gb_operation_put_active);
+
+#undef DEFINE_OPERATION_EVENT
+
+DECLARE_EVENT_CLASS(gb_connection,
+
+	TP_PROTO(struct gb_connection *connection),
+
+	TP_ARGS(connection),
+
+	TP_STRUCT__entry(
+		__field(int, hd_bus_id)
+		__field(u8, bundle_id)
+		/* name contains "hd_cport_id/intf_id:cport_id" */
+		__dynamic_array(char, name, sizeof(connection->name))
+		__field(enum gb_connection_state, state)
+		__field(unsigned long, flags)
+	),
+
+	TP_fast_assign(
+		__entry->hd_bus_id = connection->hd->bus_id;
+		__entry->bundle_id = connection->bundle ?
+				connection->bundle->id : BUNDLE_ID_NONE;
+		memcpy(__get_str(name), connection->name,
+					sizeof(connection->name));
+		__entry->state = connection->state;
+		__entry->flags = connection->flags;
+	),
+
+	TP_printk("hd_bus_id=%d bundle_id=0x%02x name=\"%s\" state=%u flags=0x%lx",
+		  __entry->hd_bus_id, __entry->bundle_id, __get_str(name),
+		  (unsigned int)__entry->state, __entry->flags)
+);
+
+#define DEFINE_CONNECTION_EVENT(name)					\
+		DEFINE_EVENT(gb_connection, name,			\
+				TP_PROTO(struct gb_connection *connection), \
+				TP_ARGS(connection))
+
+/*
+ * Occurs after a new connection is successfully created.
+ */
+DEFINE_CONNECTION_EVENT(gb_connection_create);
+
+/*
+ * Occurs when the last reference to a connection has been dropped,
+ * before its resources are freed.
+ */
+DEFINE_CONNECTION_EVENT(gb_connection_release);
+
+/*
+ * Occurs when a new reference to connection is added, currently
+ * only when a message over the connection is received.
+ */
+DEFINE_CONNECTION_EVENT(gb_connection_get);
+
+/*
+ * Occurs when a new reference to connection is dropped, after a
+ * a received message is handled, or when the connection is
+ * destroyed.
+ */
+DEFINE_CONNECTION_EVENT(gb_connection_put);
+
+/*
+ * Occurs when a request to enable a connection is made, either for
+ * transmit only, or for both transmit and receive.
+ */
+DEFINE_CONNECTION_EVENT(gb_connection_enable);
+
+/*
+ * Occurs when a request to disable a connection is made, either for
+ * receive only, or for both transmit and receive.  Also occurs when
+ * a request to forcefully disable a connection is made.
+ */
+DEFINE_CONNECTION_EVENT(gb_connection_disable);
+
+#undef DEFINE_CONNECTION_EVENT
+
+DECLARE_EVENT_CLASS(gb_bundle,
+
+	TP_PROTO(struct gb_bundle *bundle),
+
+	TP_ARGS(bundle),
+
+	TP_STRUCT__entry(
+		__field(u8, intf_id)
+		__field(u8, id)
+		__field(u8, class)
+		__field(size_t, num_cports)
+	),
+
+	TP_fast_assign(
+		__entry->intf_id = bundle->intf->interface_id;
+		__entry->id = bundle->id;
+		__entry->class = bundle->class;
+		__entry->num_cports = bundle->num_cports;
+	),
+
+	TP_printk("intf_id=0x%02x id=%02x class=0x%02x num_cports=%zu",
+		  __entry->intf_id, __entry->id, __entry->class,
+		  __entry->num_cports)
+);
+
+#define DEFINE_BUNDLE_EVENT(name)					\
+		DEFINE_EVENT(gb_bundle, name,			\
+				TP_PROTO(struct gb_bundle *bundle), \
+				TP_ARGS(bundle))
+
+/*
+ * Occurs after a new bundle is successfully created.
+ */
+DEFINE_BUNDLE_EVENT(gb_bundle_create);
+
+/*
+ * Occurs when the last reference to a bundle has been dropped,
+ * before its resources are freed.
+ */
+DEFINE_BUNDLE_EVENT(gb_bundle_release);
+
+/*
+ * Occurs when a bundle is added to an interface when the interface
+ * is enabled.
+ */
+DEFINE_BUNDLE_EVENT(gb_bundle_add);
+
+/*
+ * Occurs when a registered bundle gets destroyed, normally at the
+ * time an interface is disabled.
+ */
+DEFINE_BUNDLE_EVENT(gb_bundle_destroy);
+
+#undef DEFINE_BUNDLE_EVENT
+
+DECLARE_EVENT_CLASS(gb_interface,
+
+	TP_PROTO(struct gb_interface *intf),
+
+	TP_ARGS(intf),
+
+	TP_STRUCT__entry(
+		__field(u8, module_id)
+		__field(u8, id)		/* Interface id */
+		__field(u8, device_id)
+		__field(int, disconnected)	/* bool */
+		__field(int, ejected)		/* bool */
+		__field(int, active)		/* bool */
+		__field(int, enabled)		/* bool */
+		__field(int, mode_switch)	/* bool */
+	),
+
+	TP_fast_assign(
+		__entry->module_id = intf->module->module_id;
+		__entry->id = intf->interface_id;
+		__entry->device_id = intf->device_id;
+		__entry->disconnected = intf->disconnected;
+		__entry->ejected = intf->ejected;
+		__entry->active = intf->active;
+		__entry->enabled = intf->enabled;
+		__entry->mode_switch = intf->mode_switch;
+	),
+
+	TP_printk("intf_id=%hhu device_id=%hhu module_id=%hhu D=%d J=%d A=%d E=%d M=%d",
+		__entry->id, __entry->device_id, __entry->module_id,
+		__entry->disconnected, __entry->ejected, __entry->active,
+		__entry->enabled, __entry->mode_switch)
+);
+
+#define DEFINE_INTERFACE_EVENT(name)					\
+		DEFINE_EVENT(gb_interface, name,			\
+				TP_PROTO(struct gb_interface *intf),	\
+				TP_ARGS(intf))
+
+/*
+ * Occurs after a new interface is successfully created.
+ */
+DEFINE_INTERFACE_EVENT(gb_interface_create);
+
+/*
+ * Occurs after the last reference to an interface has been dropped.
+ */
+DEFINE_INTERFACE_EVENT(gb_interface_release);
+
+/*
+ * Occurs after an interface been registerd.
+ */
+DEFINE_INTERFACE_EVENT(gb_interface_add);
+
+/*
+ * Occurs when a registered interface gets deregisterd.
+ */
+DEFINE_INTERFACE_EVENT(gb_interface_del);
+
+/*
+ * Occurs when a registered interface has been successfully
+ * activated.
+ */
+DEFINE_INTERFACE_EVENT(gb_interface_activate);
+
+/*
+ * Occurs when an activated interface is being deactivated.
+ */
+DEFINE_INTERFACE_EVENT(gb_interface_deactivate);
+
+/*
+ * Occurs when an interface has been successfully enabled.
+ */
+DEFINE_INTERFACE_EVENT(gb_interface_enable);
+
+/*
+ * Occurs when an enabled interface is being disabled.
+ */
+DEFINE_INTERFACE_EVENT(gb_interface_disable);
+
+#undef DEFINE_INTERFACE_EVENT
+
+DECLARE_EVENT_CLASS(gb_module,
+
+	TP_PROTO(struct gb_module *module),
+
+	TP_ARGS(module),
+
+	TP_STRUCT__entry(
+		__field(int, hd_bus_id)
+		__field(u8, module_id)
+		__field(size_t, num_interfaces)
+		__field(int, disconnected)	/* bool */
+	),
+
+	TP_fast_assign(
+		__entry->hd_bus_id = module->hd->bus_id;
+		__entry->module_id = module->module_id;
+		__entry->num_interfaces = module->num_interfaces;
+		__entry->disconnected = module->disconnected;
+	),
+
+	TP_printk("hd_bus_id=%d module_id=%hhu num_interfaces=%zu disconnected=%d",
+		__entry->hd_bus_id, __entry->module_id,
+		__entry->num_interfaces, __entry->disconnected)
+);
+
+#define DEFINE_MODULE_EVENT(name)					\
+		DEFINE_EVENT(gb_module, name,				\
+				TP_PROTO(struct gb_module *module),	\
+				TP_ARGS(module))
+
+/*
+ * Occurs after a new module is successfully created, before
+ * creating any of its interfaces.
+ */
+DEFINE_MODULE_EVENT(gb_module_create);
+
+/*
+ * Occurs after the last reference to a module has been dropped.
+ */
+DEFINE_MODULE_EVENT(gb_module_release);
+
+/*
+ * Occurs after a module is successfully created, before registering
+ * any of its interfaces.
+ */
+DEFINE_MODULE_EVENT(gb_module_add);
+
+/*
+ * Occurs when a module is deleted, before deregistering its
+ * interfaces.
+ */
+DEFINE_MODULE_EVENT(gb_module_del);
+
+#undef DEFINE_MODULE_EVENT
+
+DECLARE_EVENT_CLASS(gb_host_device,
+
+	TP_PROTO(struct gb_host_device *hd),
+
+	TP_ARGS(hd),
+
+	TP_STRUCT__entry(
+		__field(int, bus_id)
+		__field(size_t, num_cports)
+		__field(size_t, buffer_size_max)
+	),
+
+	TP_fast_assign(
+		__entry->bus_id = hd->bus_id;
+		__entry->num_cports = hd->num_cports;
+		__entry->buffer_size_max = hd->buffer_size_max;
+	),
+
+	TP_printk("bus_id=%d num_cports=%zu mtu=%zu",
+		__entry->bus_id, __entry->num_cports,
+		__entry->buffer_size_max)
+);
+
+#define DEFINE_HD_EVENT(name)						\
+		DEFINE_EVENT(gb_host_device, name,			\
+				TP_PROTO(struct gb_host_device *hd),	\
+				TP_ARGS(hd))
+
+/*
+ * Occurs after a new host device is successfully created, before
+ * its SVC has been set up.
+ */
+DEFINE_HD_EVENT(gb_hd_create);
+
+/*
+ * Occurs after the last reference to a host device has been
+ * dropped.
+ */
+DEFINE_HD_EVENT(gb_hd_release);
+
+/*
+ * Occurs after a new host device has been added, after the
+ * connection to its SVC has been enabled.
+ */
+DEFINE_HD_EVENT(gb_hd_add);
+
+/*
+ * Occurs when a host device is being disconnected from the AP USB
+ * host controller.
+ */
+DEFINE_HD_EVENT(gb_hd_del);
+
+/*
+ * Occurs when a host device has passed received data to the Greybus
+ * core, after it has been determined it is destined for a valid
+ * CPort.
+ */
+DEFINE_HD_EVENT(gb_hd_in);
+
+#undef DEFINE_HD_EVENT
+
+/*
+ * Occurs on a TimeSync synchronization event or a TimeSync ping event.
+ */
+TRACE_EVENT(gb_timesync_irq,
+
+	TP_PROTO(u8 ping, u8 strobe, u8 count, u64 frame_time),
+
+	TP_ARGS(ping, strobe, count, frame_time),
+
+	TP_STRUCT__entry(
+		__field(u8, ping)
+		__field(u8, strobe)
+		__field(u8, count)
+		__field(u64, frame_time)
+	),
+
+	TP_fast_assign(
+		__entry->ping = ping;
+		__entry->strobe = strobe;
+		__entry->count = count;
+		__entry->frame_time = frame_time;
+	),
+
+	TP_printk("%s %d/%d frame-time %llu\n",
+		  __entry->ping ? "ping" : "strobe", __entry->strobe,
+		  __entry->count, __entry->frame_time)
+);
+
+#endif /* _TRACE_GREYBUS_H */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+
+/*
+ * TRACE_INCLUDE_FILE is not needed if the filename and TRACE_SYSTEM are equal
+ */
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE greybus_trace
+#include <trace/define_trace.h>
+
diff --git a/drivers/staging/greybus/hd.c b/drivers/staging/greybus/hd.c
new file mode 100644
index 000000000000..185ae3fa10fd
--- /dev/null
+++ b/drivers/staging/greybus/hd.c
@@ -0,0 +1,257 @@
+/*
+ * Greybus Host Device
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+
+#include "greybus.h"
+#include "greybus_trace.h"
+
+EXPORT_TRACEPOINT_SYMBOL_GPL(gb_hd_create);
+EXPORT_TRACEPOINT_SYMBOL_GPL(gb_hd_release);
+EXPORT_TRACEPOINT_SYMBOL_GPL(gb_hd_add);
+EXPORT_TRACEPOINT_SYMBOL_GPL(gb_hd_del);
+EXPORT_TRACEPOINT_SYMBOL_GPL(gb_hd_in);
+EXPORT_TRACEPOINT_SYMBOL_GPL(gb_message_submit);
+
+static struct ida gb_hd_bus_id_map;
+
+int gb_hd_output(struct gb_host_device *hd, void *req, u16 size, u8 cmd,
+		 bool async)
+{
+	if (!hd || !hd->driver || !hd->driver->output)
+		return -EINVAL;
+	return hd->driver->output(hd, req, size, cmd, async);
+}
+EXPORT_SYMBOL_GPL(gb_hd_output);
+
+static ssize_t bus_id_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct gb_host_device *hd = to_gb_host_device(dev);
+
+	return sprintf(buf, "%d\n", hd->bus_id);
+}
+static DEVICE_ATTR_RO(bus_id);
+
+static struct attribute *bus_attrs[] = {
+	&dev_attr_bus_id.attr,
+	NULL
+};
+ATTRIBUTE_GROUPS(bus);
+
+int gb_hd_cport_reserve(struct gb_host_device *hd, u16 cport_id)
+{
+	struct ida *id_map = &hd->cport_id_map;
+	int ret;
+
+	ret = ida_simple_get(id_map, cport_id, cport_id + 1, GFP_KERNEL);
+	if (ret < 0) {
+		dev_err(&hd->dev, "failed to reserve cport %u\n", cport_id);
+		return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gb_hd_cport_reserve);
+
+void gb_hd_cport_release_reserved(struct gb_host_device *hd, u16 cport_id)
+{
+	struct ida *id_map = &hd->cport_id_map;
+
+	ida_simple_remove(id_map, cport_id);
+}
+EXPORT_SYMBOL_GPL(gb_hd_cport_release_reserved);
+
+/* Locking: Caller guarantees serialisation */
+int gb_hd_cport_allocate(struct gb_host_device *hd, int cport_id,
+				unsigned long flags)
+{
+	struct ida *id_map = &hd->cport_id_map;
+	int ida_start, ida_end;
+
+	if (hd->driver->cport_allocate)
+		return hd->driver->cport_allocate(hd, cport_id, flags);
+
+	if (cport_id < 0) {
+		ida_start = 0;
+		ida_end = hd->num_cports;
+	} else if (cport_id < hd->num_cports) {
+		ida_start = cport_id;
+		ida_end = cport_id + 1;
+	} else {
+		dev_err(&hd->dev, "cport %d not available\n", cport_id);
+		return -EINVAL;
+	}
+
+	return ida_simple_get(id_map, ida_start, ida_end, GFP_KERNEL);
+}
+
+/* Locking: Caller guarantees serialisation */
+void gb_hd_cport_release(struct gb_host_device *hd, u16 cport_id)
+{
+	if (hd->driver->cport_release) {
+		hd->driver->cport_release(hd, cport_id);
+		return;
+	}
+
+	ida_simple_remove(&hd->cport_id_map, cport_id);
+}
+
+static void gb_hd_release(struct device *dev)
+{
+	struct gb_host_device *hd = to_gb_host_device(dev);
+
+	trace_gb_hd_release(hd);
+
+	if (hd->svc)
+		gb_svc_put(hd->svc);
+	ida_simple_remove(&gb_hd_bus_id_map, hd->bus_id);
+	ida_destroy(&hd->cport_id_map);
+	kfree(hd);
+}
+
+struct device_type greybus_hd_type = {
+	.name		= "greybus_host_device",
+	.release	= gb_hd_release,
+};
+
+struct gb_host_device *gb_hd_create(struct gb_hd_driver *driver,
+					struct device *parent,
+					size_t buffer_size_max,
+					size_t num_cports)
+{
+	struct gb_host_device *hd;
+	int ret;
+
+	/*
+	 * Validate that the driver implements all of the callbacks
+	 * so that we don't have to every time we make them.
+	 */
+	if ((!driver->message_send) || (!driver->message_cancel)) {
+		dev_err(parent, "mandatory hd-callbacks missing\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (buffer_size_max < GB_OPERATION_MESSAGE_SIZE_MIN) {
+		dev_err(parent, "greybus host-device buffers too small\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (num_cports == 0 || num_cports > CPORT_ID_MAX + 1) {
+		dev_err(parent, "Invalid number of CPorts: %zu\n", num_cports);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/*
+	 * Make sure to never allocate messages larger than what the Greybus
+	 * protocol supports.
+	 */
+	if (buffer_size_max > GB_OPERATION_MESSAGE_SIZE_MAX) {
+		dev_warn(parent, "limiting buffer size to %u\n",
+			 GB_OPERATION_MESSAGE_SIZE_MAX);
+		buffer_size_max = GB_OPERATION_MESSAGE_SIZE_MAX;
+	}
+
+	hd = kzalloc(sizeof(*hd) + driver->hd_priv_size, GFP_KERNEL);
+	if (!hd)
+		return ERR_PTR(-ENOMEM);
+
+	ret = ida_simple_get(&gb_hd_bus_id_map, 1, 0, GFP_KERNEL);
+	if (ret < 0) {
+		kfree(hd);
+		return ERR_PTR(ret);
+	}
+	hd->bus_id = ret;
+
+	hd->driver = driver;
+	INIT_LIST_HEAD(&hd->modules);
+	INIT_LIST_HEAD(&hd->connections);
+	ida_init(&hd->cport_id_map);
+	hd->buffer_size_max = buffer_size_max;
+	hd->num_cports = num_cports;
+
+	hd->dev.parent = parent;
+	hd->dev.bus = &greybus_bus_type;
+	hd->dev.type = &greybus_hd_type;
+	hd->dev.groups = bus_groups;
+	hd->dev.dma_mask = hd->dev.parent->dma_mask;
+	device_initialize(&hd->dev);
+	dev_set_name(&hd->dev, "greybus%d", hd->bus_id);
+
+	trace_gb_hd_create(hd);
+
+	hd->svc = gb_svc_create(hd);
+	if (!hd->svc) {
+		dev_err(&hd->dev, "failed to create svc\n");
+		put_device(&hd->dev);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return hd;
+}
+EXPORT_SYMBOL_GPL(gb_hd_create);
+
+int gb_hd_add(struct gb_host_device *hd)
+{
+	int ret;
+
+	ret = device_add(&hd->dev);
+	if (ret)
+		return ret;
+
+	ret = gb_svc_add(hd->svc);
+	if (ret) {
+		device_del(&hd->dev);
+		return ret;
+	}
+
+	trace_gb_hd_add(hd);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gb_hd_add);
+
+void gb_hd_del(struct gb_host_device *hd)
+{
+	trace_gb_hd_del(hd);
+
+	/*
+	 * Tear down the svc and flush any on-going hotplug processing before
+	 * removing the remaining interfaces.
+	 */
+	gb_svc_del(hd->svc);
+
+	device_del(&hd->dev);
+}
+EXPORT_SYMBOL_GPL(gb_hd_del);
+
+void gb_hd_shutdown(struct gb_host_device *hd)
+{
+	gb_svc_del(hd->svc);
+}
+EXPORT_SYMBOL_GPL(gb_hd_shutdown);
+
+void gb_hd_put(struct gb_host_device *hd)
+{
+	put_device(&hd->dev);
+}
+EXPORT_SYMBOL_GPL(gb_hd_put);
+
+int __init gb_hd_init(void)
+{
+	ida_init(&gb_hd_bus_id_map);
+
+	return 0;
+}
+
+void gb_hd_exit(void)
+{
+	ida_destroy(&gb_hd_bus_id_map);
+}
diff --git a/drivers/staging/greybus/hd.h b/drivers/staging/greybus/hd.h
new file mode 100644
index 000000000000..c4250cfe595f
--- /dev/null
+++ b/drivers/staging/greybus/hd.h
@@ -0,0 +1,90 @@
+/*
+ * Greybus Host Device
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __HD_H
+#define __HD_H
+
+struct gb_host_device;
+struct gb_message;
+
+struct gb_hd_driver {
+	size_t	hd_priv_size;
+
+	int (*cport_allocate)(struct gb_host_device *hd, int cport_id,
+				unsigned long flags);
+	void (*cport_release)(struct gb_host_device *hd, u16 cport_id);
+	int (*cport_enable)(struct gb_host_device *hd, u16 cport_id,
+				unsigned long flags);
+	int (*cport_disable)(struct gb_host_device *hd, u16 cport_id);
+	int (*cport_connected)(struct gb_host_device *hd, u16 cport_id);
+	int (*cport_flush)(struct gb_host_device *hd, u16 cport_id);
+	int (*cport_shutdown)(struct gb_host_device *hd, u16 cport_id,
+				u8 phase, unsigned int timeout);
+	int (*cport_quiesce)(struct gb_host_device *hd, u16 cport_id,
+				size_t peer_space, unsigned int timeout);
+	int (*cport_clear)(struct gb_host_device *hd, u16 cport_id);
+
+	int (*message_send)(struct gb_host_device *hd, u16 dest_cport_id,
+			struct gb_message *message, gfp_t gfp_mask);
+	void (*message_cancel)(struct gb_message *message);
+	int (*latency_tag_enable)(struct gb_host_device *hd, u16 cport_id);
+	int (*latency_tag_disable)(struct gb_host_device *hd, u16 cport_id);
+	int (*output)(struct gb_host_device *hd, void *req, u16 size, u8 cmd,
+		      bool async);
+	int (*timesync_enable)(struct gb_host_device *hd, u8 count,
+			       u64 frame_time, u32 strobe_delay, u32 refclk);
+	int (*timesync_disable)(struct gb_host_device *hd);
+	int (*timesync_authoritative)(struct gb_host_device *hd,
+				      u64 *frame_time);
+	int (*timesync_get_last_event)(struct gb_host_device *hd,
+				       u64 *frame_time);
+};
+
+struct gb_host_device {
+	struct device dev;
+	int bus_id;
+	const struct gb_hd_driver *driver;
+
+	struct list_head modules;
+	struct list_head connections;
+	struct ida cport_id_map;
+
+	/* Number of CPorts supported by the UniPro IP */
+	size_t num_cports;
+
+	/* Host device buffer constraints */
+	size_t buffer_size_max;
+
+	struct gb_svc *svc;
+	/* Private data for the host driver */
+	unsigned long hd_priv[0] __aligned(sizeof(s64));
+};
+#define to_gb_host_device(d) container_of(d, struct gb_host_device, dev)
+
+int gb_hd_cport_reserve(struct gb_host_device *hd, u16 cport_id);
+void gb_hd_cport_release_reserved(struct gb_host_device *hd, u16 cport_id);
+int gb_hd_cport_allocate(struct gb_host_device *hd, int cport_id,
+					unsigned long flags);
+void gb_hd_cport_release(struct gb_host_device *hd, u16 cport_id);
+
+struct gb_host_device *gb_hd_create(struct gb_hd_driver *driver,
+					struct device *parent,
+					size_t buffer_size_max,
+					size_t num_cports);
+int gb_hd_add(struct gb_host_device *hd);
+void gb_hd_del(struct gb_host_device *hd);
+void gb_hd_shutdown(struct gb_host_device *hd);
+void gb_hd_put(struct gb_host_device *hd);
+int gb_hd_output(struct gb_host_device *hd, void *req, u16 size, u8 cmd,
+		 bool in_irq);
+
+int gb_hd_init(void);
+void gb_hd_exit(void);
+
+#endif	/* __HD_H */
diff --git a/drivers/staging/greybus/hid.c b/drivers/staging/greybus/hid.c
new file mode 100644
index 000000000000..730d746fc4c2
--- /dev/null
+++ b/drivers/staging/greybus/hid.c
@@ -0,0 +1,536 @@
+/*
+ * HID class driver for the Greybus.
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/bitops.h>
+#include <linux/hid.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+
+#include "greybus.h"
+
+/* Greybus HID device's structure */
+struct gb_hid {
+	struct gb_bundle *bundle;
+	struct gb_connection		*connection;
+
+	struct hid_device		*hid;
+	struct gb_hid_desc_response	hdesc;
+
+	unsigned long			flags;
+#define GB_HID_STARTED			0x01
+#define GB_HID_READ_PENDING		0x04
+
+	unsigned int			bufsize;
+	char				*inbuf;
+};
+
+static DEFINE_MUTEX(gb_hid_open_mutex);
+
+/* Routines to get controller's information over greybus */
+
+/* Operations performed on greybus */
+static int gb_hid_get_desc(struct gb_hid *ghid)
+{
+	return gb_operation_sync(ghid->connection, GB_HID_TYPE_GET_DESC, NULL,
+				 0, &ghid->hdesc, sizeof(ghid->hdesc));
+}
+
+static int gb_hid_get_report_desc(struct gb_hid *ghid, char *rdesc)
+{
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(ghid->bundle);
+	if (ret)
+		return ret;
+
+	ret = gb_operation_sync(ghid->connection, GB_HID_TYPE_GET_REPORT_DESC,
+				 NULL, 0, rdesc,
+				 le16_to_cpu(ghid->hdesc.wReportDescLength));
+
+	gb_pm_runtime_put_autosuspend(ghid->bundle);
+
+	return ret;
+}
+
+static int gb_hid_set_power(struct gb_hid *ghid, int type)
+{
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(ghid->bundle);
+	if (ret)
+		return ret;
+
+	ret = gb_operation_sync(ghid->connection, type, NULL, 0, NULL, 0);
+
+	gb_pm_runtime_put_autosuspend(ghid->bundle);
+
+	return ret;
+}
+
+static int gb_hid_get_report(struct gb_hid *ghid, u8 report_type, u8 report_id,
+			     unsigned char *buf, int len)
+{
+	struct gb_hid_get_report_request request;
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(ghid->bundle);
+	if (ret)
+		return ret;
+
+	request.report_type = report_type;
+	request.report_id = report_id;
+
+	ret = gb_operation_sync(ghid->connection, GB_HID_TYPE_GET_REPORT,
+				 &request, sizeof(request), buf, len);
+
+	gb_pm_runtime_put_autosuspend(ghid->bundle);
+
+	return ret;
+}
+
+static int gb_hid_set_report(struct gb_hid *ghid, u8 report_type, u8 report_id,
+			     unsigned char *buf, int len)
+{
+	struct gb_hid_set_report_request *request;
+	struct gb_operation *operation;
+	int ret, size = sizeof(*request) + len - 1;
+
+	ret = gb_pm_runtime_get_sync(ghid->bundle);
+	if (ret)
+		return ret;
+
+	operation = gb_operation_create(ghid->connection,
+					GB_HID_TYPE_SET_REPORT, size, 0,
+					GFP_KERNEL);
+	if (!operation) {
+		gb_pm_runtime_put_autosuspend(ghid->bundle);
+		return -ENOMEM;
+	}
+
+	request = operation->request->payload;
+	request->report_type = report_type;
+	request->report_id = report_id;
+	memcpy(request->report, buf, len);
+
+	ret = gb_operation_request_send_sync(operation);
+	if (ret) {
+		dev_err(&operation->connection->bundle->dev,
+			"failed to set report: %d\n", ret);
+	} else {
+		ret = len;
+	}
+
+	gb_operation_put(operation);
+	gb_pm_runtime_put_autosuspend(ghid->bundle);
+
+	return ret;
+}
+
+static int gb_hid_request_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_hid *ghid = gb_connection_get_data(connection);
+	struct gb_hid_input_report_request *request = op->request->payload;
+
+	if (op->type != GB_HID_TYPE_IRQ_EVENT) {
+		dev_err(&connection->bundle->dev,
+			"unsupported unsolicited request\n");
+		return -EINVAL;
+	}
+
+	if (test_bit(GB_HID_STARTED, &ghid->flags))
+		hid_input_report(ghid->hid, HID_INPUT_REPORT,
+				 request->report, op->request->payload_size, 1);
+
+	return 0;
+}
+
+static int gb_hid_report_len(struct hid_report *report)
+{
+	return ((report->size - 1) >> 3) + 1 +
+		report->device->report_enum[report->type].numbered;
+}
+
+static void gb_hid_find_max_report(struct hid_device *hid, unsigned int type,
+				   unsigned int *max)
+{
+	struct hid_report *report;
+	unsigned int size;
+
+	list_for_each_entry(report, &hid->report_enum[type].report_list, list) {
+		size = gb_hid_report_len(report);
+		if (*max < size)
+			*max = size;
+	}
+}
+
+static void gb_hid_free_buffers(struct gb_hid *ghid)
+{
+	kfree(ghid->inbuf);
+	ghid->inbuf = NULL;
+	ghid->bufsize = 0;
+}
+
+static int gb_hid_alloc_buffers(struct gb_hid *ghid, size_t bufsize)
+{
+	ghid->inbuf = kzalloc(bufsize, GFP_KERNEL);
+	if (!ghid->inbuf)
+		return -ENOMEM;
+
+	ghid->bufsize = bufsize;
+
+	return 0;
+}
+
+/* Routines dealing with reports */
+static void gb_hid_init_report(struct gb_hid *ghid, struct hid_report *report)
+{
+	unsigned int size;
+
+	size = gb_hid_report_len(report);
+	if (gb_hid_get_report(ghid, report->type, report->id, ghid->inbuf,
+			      size))
+		return;
+
+	/*
+	 * hid->driver_lock is held as we are in probe function,
+	 * we just need to setup the input fields, so using
+	 * hid_report_raw_event is safe.
+	 */
+	hid_report_raw_event(ghid->hid, report->type, ghid->inbuf, size, 1);
+}
+
+static void gb_hid_init_reports(struct gb_hid *ghid)
+{
+	struct hid_device *hid = ghid->hid;
+	struct hid_report *report;
+
+	list_for_each_entry(report,
+		&hid->report_enum[HID_INPUT_REPORT].report_list, list)
+		gb_hid_init_report(ghid, report);
+
+	list_for_each_entry(report,
+		&hid->report_enum[HID_FEATURE_REPORT].report_list, list)
+		gb_hid_init_report(ghid, report);
+}
+
+static int __gb_hid_get_raw_report(struct hid_device *hid,
+		unsigned char report_number, __u8 *buf, size_t count,
+		unsigned char report_type)
+{
+	struct gb_hid *ghid = hid->driver_data;
+	int ret;
+
+	if (report_type == HID_OUTPUT_REPORT)
+		return -EINVAL;
+
+	ret = gb_hid_get_report(ghid, report_type, report_number, buf, count);
+	if (!ret)
+		ret = count;
+
+	return ret;
+}
+
+static int __gb_hid_output_raw_report(struct hid_device *hid, __u8 *buf,
+				      size_t len, unsigned char report_type)
+{
+	struct gb_hid *ghid = hid->driver_data;
+	int report_id = buf[0];
+	int ret;
+
+	if (report_type == HID_INPUT_REPORT)
+		return -EINVAL;
+
+	if (report_id) {
+		buf++;
+		len--;
+	}
+
+	ret = gb_hid_set_report(ghid, report_type, report_id, buf, len);
+	if (report_id && ret >= 0)
+		ret++; /* add report_id to the number of transfered bytes */
+
+	return 0;
+}
+
+static int gb_hid_raw_request(struct hid_device *hid, unsigned char reportnum,
+			       __u8 *buf, size_t len, unsigned char rtype,
+			       int reqtype)
+{
+	switch (reqtype) {
+	case HID_REQ_GET_REPORT:
+		return __gb_hid_get_raw_report(hid, reportnum, buf, len, rtype);
+	case HID_REQ_SET_REPORT:
+		if (buf[0] != reportnum)
+			return -EINVAL;
+		return __gb_hid_output_raw_report(hid, buf, len, rtype);
+	default:
+		return -EIO;
+	}
+}
+
+/* HID Callbacks */
+static int gb_hid_parse(struct hid_device *hid)
+{
+	struct gb_hid *ghid = hid->driver_data;
+	unsigned int rsize;
+	char *rdesc;
+	int ret;
+
+	rsize = le16_to_cpu(ghid->hdesc.wReportDescLength);
+	if (!rsize || rsize > HID_MAX_DESCRIPTOR_SIZE) {
+		dbg_hid("weird size of report descriptor (%u)\n", rsize);
+		return -EINVAL;
+	}
+
+	rdesc = kzalloc(rsize, GFP_KERNEL);
+	if (!rdesc) {
+		dbg_hid("couldn't allocate rdesc memory\n");
+		return -ENOMEM;
+	}
+
+	ret = gb_hid_get_report_desc(ghid, rdesc);
+	if (ret) {
+		hid_err(hid, "reading report descriptor failed\n");
+		goto free_rdesc;
+	}
+
+	ret = hid_parse_report(hid, rdesc, rsize);
+	if (ret)
+		dbg_hid("parsing report descriptor failed\n");
+
+free_rdesc:
+	kfree(rdesc);
+
+	return ret;
+}
+
+static int gb_hid_start(struct hid_device *hid)
+{
+	struct gb_hid *ghid = hid->driver_data;
+	unsigned int bufsize = HID_MIN_BUFFER_SIZE;
+	int ret;
+
+	gb_hid_find_max_report(hid, HID_INPUT_REPORT, &bufsize);
+	gb_hid_find_max_report(hid, HID_OUTPUT_REPORT, &bufsize);
+	gb_hid_find_max_report(hid, HID_FEATURE_REPORT, &bufsize);
+
+	if (bufsize > HID_MAX_BUFFER_SIZE)
+		bufsize = HID_MAX_BUFFER_SIZE;
+
+	ret = gb_hid_alloc_buffers(ghid, bufsize);
+	if (ret)
+		return ret;
+
+	if (!(hid->quirks & HID_QUIRK_NO_INIT_REPORTS))
+		gb_hid_init_reports(ghid);
+
+	return 0;
+}
+
+static void gb_hid_stop(struct hid_device *hid)
+{
+	struct gb_hid *ghid = hid->driver_data;
+
+	gb_hid_free_buffers(ghid);
+}
+
+static int gb_hid_open(struct hid_device *hid)
+{
+	struct gb_hid *ghid = hid->driver_data;
+	int ret = 0;
+
+	mutex_lock(&gb_hid_open_mutex);
+	if (!hid->open++) {
+		ret = gb_hid_set_power(ghid, GB_HID_TYPE_PWR_ON);
+		if (ret < 0)
+			hid->open--;
+		else
+			set_bit(GB_HID_STARTED, &ghid->flags);
+	}
+	mutex_unlock(&gb_hid_open_mutex);
+
+	return ret;
+}
+
+static void gb_hid_close(struct hid_device *hid)
+{
+	struct gb_hid *ghid = hid->driver_data;
+	int ret;
+
+	/*
+	 * Protecting hid->open to make sure we don't restart data acquistion
+	 * due to a resumption we no longer care about..
+	 */
+	mutex_lock(&gb_hid_open_mutex);
+	if (!--hid->open) {
+		clear_bit(GB_HID_STARTED, &ghid->flags);
+
+		/* Save some power */
+		ret = gb_hid_set_power(ghid, GB_HID_TYPE_PWR_OFF);
+		if (ret)
+			dev_err(&ghid->connection->bundle->dev,
+				"failed to power off (%d)\n", ret);
+	}
+	mutex_unlock(&gb_hid_open_mutex);
+}
+
+static int gb_hid_power(struct hid_device *hid, int lvl)
+{
+	struct gb_hid *ghid = hid->driver_data;
+
+	switch (lvl) {
+	case PM_HINT_FULLON:
+		return gb_hid_set_power(ghid, GB_HID_TYPE_PWR_ON);
+	case PM_HINT_NORMAL:
+		return gb_hid_set_power(ghid, GB_HID_TYPE_PWR_OFF);
+	}
+
+	return 0;
+}
+
+/* HID structure to pass callbacks */
+static struct hid_ll_driver gb_hid_ll_driver = {
+	.parse = gb_hid_parse,
+	.start = gb_hid_start,
+	.stop = gb_hid_stop,
+	.open = gb_hid_open,
+	.close = gb_hid_close,
+	.power = gb_hid_power,
+	.raw_request = gb_hid_raw_request,
+};
+
+static int gb_hid_init(struct gb_hid *ghid)
+{
+	struct hid_device *hid = ghid->hid;
+	int ret;
+
+	ret = gb_hid_get_desc(ghid);
+	if (ret)
+		return ret;
+
+	hid->version = le16_to_cpu(ghid->hdesc.bcdHID);
+	hid->vendor = le16_to_cpu(ghid->hdesc.wVendorID);
+	hid->product = le16_to_cpu(ghid->hdesc.wProductID);
+	hid->country = ghid->hdesc.bCountryCode;
+
+	hid->driver_data = ghid;
+	hid->ll_driver = &gb_hid_ll_driver;
+	hid->dev.parent = &ghid->connection->bundle->dev;
+//	hid->bus = BUS_GREYBUS; /* Need a bustype for GREYBUS in <linux/input.h> */
+
+	/* Set HID device's name */
+	snprintf(hid->name, sizeof(hid->name), "%s %04X:%04X",
+		 dev_name(&ghid->connection->bundle->dev),
+		 hid->vendor, hid->product);
+
+	return 0;
+}
+
+static int gb_hid_probe(struct gb_bundle *bundle,
+			const struct greybus_bundle_id *id)
+{
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_connection *connection;
+	struct hid_device *hid;
+	struct gb_hid *ghid;
+	int ret;
+
+	if (bundle->num_cports != 1)
+		return -ENODEV;
+
+	cport_desc = &bundle->cport_desc[0];
+	if (cport_desc->protocol_id != GREYBUS_PROTOCOL_HID)
+		return -ENODEV;
+
+	ghid = kzalloc(sizeof(*ghid), GFP_KERNEL);
+	if (!ghid)
+		return -ENOMEM;
+
+	connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
+						gb_hid_request_handler);
+	if (IS_ERR(connection)) {
+		ret = PTR_ERR(connection);
+		goto err_free_ghid;
+	}
+
+	gb_connection_set_data(connection, ghid);
+	ghid->connection = connection;
+
+	hid = hid_allocate_device();
+	if (IS_ERR(hid)) {
+		ret = PTR_ERR(hid);
+		goto err_connection_destroy;
+	}
+
+	ghid->hid = hid;
+	ghid->bundle = bundle;
+
+	greybus_set_drvdata(bundle, ghid);
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto err_destroy_hid;
+
+	ret = gb_hid_init(ghid);
+	if (ret)
+		goto err_connection_disable;
+
+	ret = hid_add_device(hid);
+	if (ret) {
+		hid_err(hid, "can't add hid device: %d\n", ret);
+		goto err_connection_disable;
+	}
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return 0;
+
+err_connection_disable:
+	gb_connection_disable(connection);
+err_destroy_hid:
+	hid_destroy_device(hid);
+err_connection_destroy:
+	gb_connection_destroy(connection);
+err_free_ghid:
+	kfree(ghid);
+
+	return ret;
+}
+
+static void gb_hid_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_hid *ghid = greybus_get_drvdata(bundle);
+
+	if (gb_pm_runtime_get_sync(bundle))
+		gb_pm_runtime_get_noresume(bundle);
+
+	hid_destroy_device(ghid->hid);
+	gb_connection_disable(ghid->connection);
+	gb_connection_destroy(ghid->connection);
+	kfree(ghid);
+}
+
+static const struct greybus_bundle_id gb_hid_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_HID) },
+	{ }
+};
+MODULE_DEVICE_TABLE(greybus, gb_hid_id_table);
+
+static struct greybus_driver gb_hid_driver = {
+	.name		= "hid",
+	.probe		= gb_hid_probe,
+	.disconnect	= gb_hid_disconnect,
+	.id_table	= gb_hid_id_table,
+};
+module_greybus_driver(gb_hid_driver);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/i2c.c b/drivers/staging/greybus/i2c.c
new file mode 100644
index 000000000000..c2a50087000c
--- /dev/null
+++ b/drivers/staging/greybus/i2c.c
@@ -0,0 +1,343 @@
+/*
+ * I2C bridge driver for the Greybus "generic" I2C module.
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/i2c.h>
+
+#include "greybus.h"
+#include "gbphy.h"
+
+struct gb_i2c_device {
+	struct gb_connection	*connection;
+	struct gbphy_device	*gbphy_dev;
+
+	u32			functionality;
+
+	struct i2c_adapter	adapter;
+};
+
+/*
+ * Map Greybus i2c functionality bits into Linux ones
+ */
+static u32 gb_i2c_functionality_map(u32 gb_i2c_functionality)
+{
+	return gb_i2c_functionality;	/* All bits the same for now */
+}
+
+static int gb_i2c_functionality_operation(struct gb_i2c_device *gb_i2c_dev)
+{
+	struct gb_i2c_functionality_response response;
+	u32 functionality;
+	int ret;
+
+	ret = gb_operation_sync(gb_i2c_dev->connection,
+				GB_I2C_TYPE_FUNCTIONALITY,
+				NULL, 0, &response, sizeof(response));
+	if (ret)
+		return ret;
+
+	functionality = le32_to_cpu(response.functionality);
+	gb_i2c_dev->functionality = gb_i2c_functionality_map(functionality);
+
+	return 0;
+}
+
+/*
+ * Map Linux i2c_msg flags into Greybus i2c transfer op flags.
+ */
+static u16 gb_i2c_transfer_op_flags_map(u16 flags)
+{
+	return flags;	/* All flags the same for now */
+}
+
+static void
+gb_i2c_fill_transfer_op(struct gb_i2c_transfer_op *op, struct i2c_msg *msg)
+{
+	u16 flags = gb_i2c_transfer_op_flags_map(msg->flags);
+
+	op->addr = cpu_to_le16(msg->addr);
+	op->flags = cpu_to_le16(flags);
+	op->size = cpu_to_le16(msg->len);
+}
+
+static struct gb_operation *
+gb_i2c_operation_create(struct gb_connection *connection,
+			struct i2c_msg *msgs, u32 msg_count)
+{
+	struct gb_i2c_device *gb_i2c_dev = gb_connection_get_data(connection);
+	struct gb_i2c_transfer_request *request;
+	struct gb_operation *operation;
+	struct gb_i2c_transfer_op *op;
+	struct i2c_msg *msg;
+	u32 data_out_size = 0;
+	u32 data_in_size = 0;
+	size_t request_size;
+	void *data;
+	u16 op_count;
+	u32 i;
+
+	if (msg_count > (u32)U16_MAX) {
+		dev_err(&gb_i2c_dev->gbphy_dev->dev, "msg_count (%u) too big\n",
+			msg_count);
+		return NULL;
+	}
+	op_count = (u16)msg_count;
+
+	/*
+	 * In addition to space for all message descriptors we need
+	 * to have enough to hold all outbound message data.
+	 */
+	msg = msgs;
+	for (i = 0; i < msg_count; i++, msg++)
+		if (msg->flags & I2C_M_RD)
+			data_in_size += (u32)msg->len;
+		else
+			data_out_size += (u32)msg->len;
+
+	request_size = sizeof(*request);
+	request_size += msg_count * sizeof(*op);
+	request_size += data_out_size;
+
+	/* Response consists only of incoming data */
+	operation = gb_operation_create(connection, GB_I2C_TYPE_TRANSFER,
+				request_size, data_in_size, GFP_KERNEL);
+	if (!operation)
+		return NULL;
+
+	request = operation->request->payload;
+	request->op_count = cpu_to_le16(op_count);
+	/* Fill in the ops array */
+	op = &request->ops[0];
+	msg = msgs;
+	for (i = 0; i < msg_count; i++)
+		gb_i2c_fill_transfer_op(op++, msg++);
+
+	if (!data_out_size)
+		return operation;
+
+	/* Copy over the outgoing data; it starts after the last op */
+	data = op;
+	msg = msgs;
+	for (i = 0; i < msg_count; i++) {
+		if (!(msg->flags & I2C_M_RD)) {
+			memcpy(data, msg->buf, msg->len);
+			data += msg->len;
+		}
+		msg++;
+	}
+
+	return operation;
+}
+
+static void gb_i2c_decode_response(struct i2c_msg *msgs, u32 msg_count,
+				struct gb_i2c_transfer_response *response)
+{
+	struct i2c_msg *msg = msgs;
+	u8 *data;
+	u32 i;
+
+	if (!response)
+		return;
+	data = response->data;
+	for (i = 0; i < msg_count; i++) {
+		if (msg->flags & I2C_M_RD) {
+			memcpy(msg->buf, data, msg->len);
+			data += msg->len;
+		}
+		msg++;
+	}
+}
+
+/*
+ * Some i2c transfer operations return results that are expected.
+ */
+static bool gb_i2c_expected_transfer_error(int errno)
+{
+	return errno == -EAGAIN || errno == -ENODEV;
+}
+
+static int gb_i2c_transfer_operation(struct gb_i2c_device *gb_i2c_dev,
+					struct i2c_msg *msgs, u32 msg_count)
+{
+	struct gb_connection *connection = gb_i2c_dev->connection;
+	struct device *dev = &gb_i2c_dev->gbphy_dev->dev;
+	struct gb_operation *operation;
+	int ret;
+
+	operation = gb_i2c_operation_create(connection, msgs, msg_count);
+	if (!operation)
+		return -ENOMEM;
+
+	ret = gbphy_runtime_get_sync(gb_i2c_dev->gbphy_dev);
+	if (ret)
+		goto exit_operation_put;
+
+	ret = gb_operation_request_send_sync(operation);
+	if (!ret) {
+		struct gb_i2c_transfer_response *response;
+
+		response = operation->response->payload;
+		gb_i2c_decode_response(msgs, msg_count, response);
+		ret = msg_count;
+	} else if (!gb_i2c_expected_transfer_error(ret)) {
+		dev_err(dev, "transfer operation failed (%d)\n", ret);
+	}
+
+	gbphy_runtime_put_autosuspend(gb_i2c_dev->gbphy_dev);
+
+exit_operation_put:
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+static int gb_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
+		int msg_count)
+{
+	struct gb_i2c_device *gb_i2c_dev;
+
+	gb_i2c_dev = i2c_get_adapdata(adap);
+
+	return gb_i2c_transfer_operation(gb_i2c_dev, msgs, msg_count);
+}
+
+#if 0
+/* Later */
+static int gb_i2c_smbus_xfer(struct i2c_adapter *adap,
+			u16 addr, unsigned short flags, char read_write,
+			u8 command, int size, union i2c_smbus_data *data)
+{
+	struct gb_i2c_device *gb_i2c_dev;
+
+	gb_i2c_dev = i2c_get_adapdata(adap);
+
+	return 0;
+}
+#endif
+
+static u32 gb_i2c_functionality(struct i2c_adapter *adap)
+{
+	struct gb_i2c_device *gb_i2c_dev = i2c_get_adapdata(adap);
+
+	return gb_i2c_dev->functionality;
+}
+
+static const struct i2c_algorithm gb_i2c_algorithm = {
+	.master_xfer	= gb_i2c_master_xfer,
+	/* .smbus_xfer	= gb_i2c_smbus_xfer, */
+	.functionality	= gb_i2c_functionality,
+};
+
+/*
+ * Do initial setup of the i2c device.  This includes verifying we
+ * can support it (based on the protocol version it advertises).
+ * If that's OK, we get and cached its functionality bits.
+ *
+ * Note: gb_i2c_dev->connection is assumed to have been valid.
+ */
+static int gb_i2c_device_setup(struct gb_i2c_device *gb_i2c_dev)
+{
+	/* Assume the functionality never changes, just get it once */
+	return gb_i2c_functionality_operation(gb_i2c_dev);
+}
+
+static int gb_i2c_probe(struct gbphy_device *gbphy_dev,
+			 const struct gbphy_device_id *id)
+{
+	struct gb_connection *connection;
+	struct gb_i2c_device *gb_i2c_dev;
+	struct i2c_adapter *adapter;
+	int ret;
+
+	gb_i2c_dev = kzalloc(sizeof(*gb_i2c_dev), GFP_KERNEL);
+	if (!gb_i2c_dev)
+		return -ENOMEM;
+
+	connection = gb_connection_create(gbphy_dev->bundle,
+					  le16_to_cpu(gbphy_dev->cport_desc->id),
+					  NULL);
+	if (IS_ERR(connection)) {
+		ret = PTR_ERR(connection);
+		goto exit_i2cdev_free;
+	}
+
+	gb_i2c_dev->connection = connection;
+	gb_connection_set_data(connection, gb_i2c_dev);
+	gb_i2c_dev->gbphy_dev = gbphy_dev;
+	gb_gbphy_set_data(gbphy_dev, gb_i2c_dev);
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto exit_connection_destroy;
+
+	ret = gb_i2c_device_setup(gb_i2c_dev);
+	if (ret)
+		goto exit_connection_disable;
+
+	/* Looks good; up our i2c adapter */
+	adapter = &gb_i2c_dev->adapter;
+	adapter->owner = THIS_MODULE;
+	adapter->class = I2C_CLASS_HWMON | I2C_CLASS_SPD;
+	adapter->algo = &gb_i2c_algorithm;
+	/* adapter->algo_data = what? */
+
+	adapter->dev.parent = &gbphy_dev->dev;
+	snprintf(adapter->name, sizeof(adapter->name), "Greybus i2c adapter");
+	i2c_set_adapdata(adapter, gb_i2c_dev);
+
+	ret = i2c_add_adapter(adapter);
+	if (ret)
+		goto exit_connection_disable;
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+	return 0;
+
+exit_connection_disable:
+	gb_connection_disable(connection);
+exit_connection_destroy:
+	gb_connection_destroy(connection);
+exit_i2cdev_free:
+	kfree(gb_i2c_dev);
+
+	return ret;
+}
+
+static void gb_i2c_remove(struct gbphy_device *gbphy_dev)
+{
+	struct gb_i2c_device *gb_i2c_dev = gb_gbphy_get_data(gbphy_dev);
+	struct gb_connection *connection = gb_i2c_dev->connection;
+	int ret;
+
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		gbphy_runtime_get_noresume(gbphy_dev);
+
+	i2c_del_adapter(&gb_i2c_dev->adapter);
+	gb_connection_disable(connection);
+	gb_connection_destroy(connection);
+	kfree(gb_i2c_dev);
+}
+
+static const struct gbphy_device_id gb_i2c_id_table[] = {
+	{ GBPHY_PROTOCOL(GREYBUS_PROTOCOL_I2C) },
+	{ },
+};
+MODULE_DEVICE_TABLE(gbphy, gb_i2c_id_table);
+
+static struct gbphy_driver i2c_driver = {
+	.name		= "i2c",
+	.probe		= gb_i2c_probe,
+	.remove		= gb_i2c_remove,
+	.id_table	= gb_i2c_id_table,
+};
+
+module_gbphy_driver(i2c_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/interface.c b/drivers/staging/greybus/interface.c
new file mode 100644
index 000000000000..546b090e2d51
--- /dev/null
+++ b/drivers/staging/greybus/interface.c
@@ -0,0 +1,1316 @@
+/*
+ * Greybus interface code
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/delay.h>
+
+#include "greybus.h"
+#include "greybus_trace.h"
+
+#define GB_INTERFACE_MODE_SWITCH_TIMEOUT	2000
+
+#define GB_INTERFACE_DEVICE_ID_BAD	0xff
+
+#define GB_INTERFACE_AUTOSUSPEND_MS			3000
+
+/* Time required for interface to enter standby before disabling REFCLK */
+#define GB_INTERFACE_SUSPEND_HIBERNATE_DELAY_MS			20
+
+/* Don't-care selector index */
+#define DME_SELECTOR_INDEX_NULL		0
+
+/* DME attributes */
+/* FIXME: remove ES2 support and DME_T_TST_SRC_INCREMENT */
+#define DME_T_TST_SRC_INCREMENT		0x4083
+
+#define DME_DDBL1_MANUFACTURERID	0x5003
+#define DME_DDBL1_PRODUCTID		0x5004
+
+#define DME_TOSHIBA_GMP_VID		0x6000
+#define DME_TOSHIBA_GMP_PID		0x6001
+#define DME_TOSHIBA_GMP_SN0		0x6002
+#define DME_TOSHIBA_GMP_SN1		0x6003
+#define DME_TOSHIBA_GMP_INIT_STATUS	0x6101
+
+/* DDBL1 Manufacturer and Product ids */
+#define TOSHIBA_DMID			0x0126
+#define TOSHIBA_ES2_BRIDGE_DPID		0x1000
+#define TOSHIBA_ES3_APBRIDGE_DPID	0x1001
+#define TOSHIBA_ES3_GBPHY_DPID	0x1002
+
+static int gb_interface_hibernate_link(struct gb_interface *intf);
+static int gb_interface_refclk_set(struct gb_interface *intf, bool enable);
+
+static int gb_interface_dme_attr_get(struct gb_interface *intf,
+							u16 attr, u32 *val)
+{
+	return gb_svc_dme_peer_get(intf->hd->svc, intf->interface_id,
+					attr, DME_SELECTOR_INDEX_NULL, val);
+}
+
+static int gb_interface_read_ara_dme(struct gb_interface *intf)
+{
+	u32 sn0, sn1;
+	int ret;
+
+	/*
+	 * Unless this is a Toshiba bridge, bail out until we have defined
+	 * standard GMP attributes.
+	 */
+	if (intf->ddbl1_manufacturer_id != TOSHIBA_DMID) {
+		dev_err(&intf->dev, "unknown manufacturer %08x\n",
+				intf->ddbl1_manufacturer_id);
+		return -ENODEV;
+	}
+
+	ret = gb_interface_dme_attr_get(intf, DME_TOSHIBA_GMP_VID,
+					&intf->vendor_id);
+	if (ret)
+		return ret;
+
+	ret = gb_interface_dme_attr_get(intf, DME_TOSHIBA_GMP_PID,
+					&intf->product_id);
+	if (ret)
+		return ret;
+
+	ret = gb_interface_dme_attr_get(intf, DME_TOSHIBA_GMP_SN0, &sn0);
+	if (ret)
+		return ret;
+
+	ret = gb_interface_dme_attr_get(intf, DME_TOSHIBA_GMP_SN1, &sn1);
+	if (ret)
+		return ret;
+
+	intf->serial_number = (u64)sn1 << 32 | sn0;
+
+	return 0;
+}
+
+static int gb_interface_read_dme(struct gb_interface *intf)
+{
+	int ret;
+
+	/* DME attributes have already been read */
+	if (intf->dme_read)
+		return 0;
+
+	ret = gb_interface_dme_attr_get(intf, DME_DDBL1_MANUFACTURERID,
+					&intf->ddbl1_manufacturer_id);
+	if (ret)
+		return ret;
+
+	ret = gb_interface_dme_attr_get(intf, DME_DDBL1_PRODUCTID,
+					&intf->ddbl1_product_id);
+	if (ret)
+		return ret;
+
+	if (intf->ddbl1_manufacturer_id == TOSHIBA_DMID &&
+			intf->ddbl1_product_id == TOSHIBA_ES2_BRIDGE_DPID) {
+		intf->quirks |= GB_INTERFACE_QUIRK_NO_GMP_IDS;
+		intf->quirks |= GB_INTERFACE_QUIRK_NO_INIT_STATUS;
+	}
+
+	ret = gb_interface_read_ara_dme(intf);
+	if (ret)
+		return ret;
+
+	intf->dme_read = true;
+
+	return 0;
+}
+
+static int gb_interface_route_create(struct gb_interface *intf)
+{
+	struct gb_svc *svc = intf->hd->svc;
+	u8 intf_id = intf->interface_id;
+	u8 device_id;
+	int ret;
+
+	/* Allocate an interface device id. */
+	ret = ida_simple_get(&svc->device_id_map,
+			     GB_SVC_DEVICE_ID_MIN, GB_SVC_DEVICE_ID_MAX + 1,
+			     GFP_KERNEL);
+	if (ret < 0) {
+		dev_err(&intf->dev, "failed to allocate device id: %d\n", ret);
+		return ret;
+	}
+	device_id = ret;
+
+	ret = gb_svc_intf_device_id(svc, intf_id, device_id);
+	if (ret) {
+		dev_err(&intf->dev, "failed to set device id %u: %d\n",
+				device_id, ret);
+		goto err_ida_remove;
+	}
+
+	/* FIXME: Hard-coded AP device id. */
+	ret = gb_svc_route_create(svc, svc->ap_intf_id, GB_SVC_DEVICE_ID_AP,
+				  intf_id, device_id);
+	if (ret) {
+		dev_err(&intf->dev, "failed to create route: %d\n", ret);
+		goto err_svc_id_free;
+	}
+
+	intf->device_id = device_id;
+
+	return 0;
+
+err_svc_id_free:
+	/*
+	 * XXX Should we tell SVC that this id doesn't belong to interface
+	 * XXX anymore.
+	 */
+err_ida_remove:
+	ida_simple_remove(&svc->device_id_map, device_id);
+
+	return ret;
+}
+
+static void gb_interface_route_destroy(struct gb_interface *intf)
+{
+	struct gb_svc *svc = intf->hd->svc;
+
+	if (intf->device_id == GB_INTERFACE_DEVICE_ID_BAD)
+		return;
+
+	gb_svc_route_destroy(svc, svc->ap_intf_id, intf->interface_id);
+	ida_simple_remove(&svc->device_id_map, intf->device_id);
+	intf->device_id = GB_INTERFACE_DEVICE_ID_BAD;
+}
+
+/* Locking: Caller holds the interface mutex. */
+static int gb_interface_legacy_mode_switch(struct gb_interface *intf)
+{
+	int ret;
+
+	dev_info(&intf->dev, "legacy mode switch detected\n");
+
+	/* Mark as disconnected to prevent I/O during disable. */
+	intf->disconnected = true;
+	gb_interface_disable(intf);
+	intf->disconnected = false;
+
+	ret = gb_interface_enable(intf);
+	if (ret) {
+		dev_err(&intf->dev, "failed to re-enable interface: %d\n", ret);
+		gb_interface_deactivate(intf);
+	}
+
+	return ret;
+}
+
+void gb_interface_mailbox_event(struct gb_interface *intf, u16 result,
+								u32 mailbox)
+{
+	mutex_lock(&intf->mutex);
+
+	if (result) {
+		dev_warn(&intf->dev,
+				"mailbox event with UniPro error: 0x%04x\n",
+				result);
+		goto err_disable;
+	}
+
+	if (mailbox != GB_SVC_INTF_MAILBOX_GREYBUS) {
+		dev_warn(&intf->dev,
+				"mailbox event with unexpected value: 0x%08x\n",
+				mailbox);
+		goto err_disable;
+	}
+
+	if (intf->quirks & GB_INTERFACE_QUIRK_LEGACY_MODE_SWITCH) {
+		gb_interface_legacy_mode_switch(intf);
+		goto out_unlock;
+	}
+
+	if (!intf->mode_switch) {
+		dev_warn(&intf->dev, "unexpected mailbox event: 0x%08x\n",
+				mailbox);
+		goto err_disable;
+	}
+
+	dev_info(&intf->dev, "mode switch detected\n");
+
+	complete(&intf->mode_switch_completion);
+
+out_unlock:
+	mutex_unlock(&intf->mutex);
+
+	return;
+
+err_disable:
+	gb_interface_disable(intf);
+	gb_interface_deactivate(intf);
+	mutex_unlock(&intf->mutex);
+}
+
+static void gb_interface_mode_switch_work(struct work_struct *work)
+{
+	struct gb_interface *intf;
+	struct gb_control *control;
+	unsigned long timeout;
+	int ret;
+
+	intf = container_of(work, struct gb_interface, mode_switch_work);
+
+	mutex_lock(&intf->mutex);
+	/* Make sure interface is still enabled. */
+	if (!intf->enabled) {
+		dev_dbg(&intf->dev, "mode switch aborted\n");
+		intf->mode_switch = false;
+		mutex_unlock(&intf->mutex);
+		goto out_interface_put;
+	}
+
+	/*
+	 * Prepare the control device for mode switch and make sure to get an
+	 * extra reference before it goes away during interface disable.
+	 */
+	control = gb_control_get(intf->control);
+	gb_control_mode_switch_prepare(control);
+	gb_interface_disable(intf);
+	mutex_unlock(&intf->mutex);
+
+	timeout = msecs_to_jiffies(GB_INTERFACE_MODE_SWITCH_TIMEOUT);
+	ret = wait_for_completion_interruptible_timeout(
+			&intf->mode_switch_completion, timeout);
+
+	/* Finalise control-connection mode switch. */
+	gb_control_mode_switch_complete(control);
+	gb_control_put(control);
+
+	if (ret < 0) {
+		dev_err(&intf->dev, "mode switch interrupted\n");
+		goto err_deactivate;
+	} else if (ret == 0) {
+		dev_err(&intf->dev, "mode switch timed out\n");
+		goto err_deactivate;
+	}
+
+	/* Re-enable (re-enumerate) interface if still active. */
+	mutex_lock(&intf->mutex);
+	intf->mode_switch = false;
+	if (intf->active) {
+		ret = gb_interface_enable(intf);
+		if (ret) {
+			dev_err(&intf->dev, "failed to re-enable interface: %d\n",
+					ret);
+			gb_interface_deactivate(intf);
+		}
+	}
+	mutex_unlock(&intf->mutex);
+
+out_interface_put:
+	gb_interface_put(intf);
+
+	return;
+
+err_deactivate:
+	mutex_lock(&intf->mutex);
+	intf->mode_switch = false;
+	gb_interface_deactivate(intf);
+	mutex_unlock(&intf->mutex);
+
+	gb_interface_put(intf);
+}
+
+int gb_interface_request_mode_switch(struct gb_interface *intf)
+{
+	int ret = 0;
+
+	mutex_lock(&intf->mutex);
+	if (intf->mode_switch) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
+	intf->mode_switch = true;
+	reinit_completion(&intf->mode_switch_completion);
+
+	/*
+	 * Get a reference to the interface device, which will be put once the
+	 * mode switch is complete.
+	 */
+	get_device(&intf->dev);
+
+	if (!queue_work(system_long_wq, &intf->mode_switch_work)) {
+		put_device(&intf->dev);
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
+out_unlock:
+	mutex_unlock(&intf->mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_interface_request_mode_switch);
+
+/*
+ * T_TstSrcIncrement is written by the module on ES2 as a stand-in for the
+ * init-status attribute DME_TOSHIBA_INIT_STATUS. The AP needs to read and
+ * clear it after reading a non-zero value from it.
+ *
+ * FIXME: This is module-hardware dependent and needs to be extended for every
+ * type of module we want to support.
+ */
+static int gb_interface_read_and_clear_init_status(struct gb_interface *intf)
+{
+	struct gb_host_device *hd = intf->hd;
+	unsigned long bootrom_quirks;
+	unsigned long s2l_quirks;
+	int ret;
+	u32 value;
+	u16 attr;
+	u8 init_status;
+
+	/*
+	 * ES2 bridges use T_TstSrcIncrement for the init status.
+	 *
+	 * FIXME: Remove ES2 support
+	 */
+	if (intf->quirks & GB_INTERFACE_QUIRK_NO_INIT_STATUS)
+		attr = DME_T_TST_SRC_INCREMENT;
+	else
+		attr = DME_TOSHIBA_GMP_INIT_STATUS;
+
+	ret = gb_svc_dme_peer_get(hd->svc, intf->interface_id, attr,
+				  DME_SELECTOR_INDEX_NULL, &value);
+	if (ret)
+		return ret;
+
+	/*
+	 * A nonzero init status indicates the module has finished
+	 * initializing.
+	 */
+	if (!value) {
+		dev_err(&intf->dev, "invalid init status\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * Extract the init status.
+	 *
+	 * For ES2: We need to check lowest 8 bits of 'value'.
+	 * For ES3: We need to check highest 8 bits out of 32 of 'value'.
+	 *
+	 * FIXME: Remove ES2 support
+	 */
+	if (intf->quirks & GB_INTERFACE_QUIRK_NO_INIT_STATUS)
+		init_status = value & 0xff;
+	else
+		init_status = value >> 24;
+
+	/*
+	 * Check if the interface is executing the quirky ES3 bootrom that,
+	 * for example, requires E2EFC, CSD and CSV to be disabled.
+	 */
+	bootrom_quirks = GB_INTERFACE_QUIRK_NO_CPORT_FEATURES |
+				GB_INTERFACE_QUIRK_FORCED_DISABLE |
+				GB_INTERFACE_QUIRK_LEGACY_MODE_SWITCH |
+				GB_INTERFACE_QUIRK_NO_BUNDLE_ACTIVATE;
+
+	s2l_quirks = GB_INTERFACE_QUIRK_NO_PM;
+
+	switch (init_status) {
+	case GB_INIT_BOOTROM_UNIPRO_BOOT_STARTED:
+	case GB_INIT_BOOTROM_FALLBACK_UNIPRO_BOOT_STARTED:
+		intf->quirks |= bootrom_quirks;
+		break;
+	case GB_INIT_S2_LOADER_BOOT_STARTED:
+		/* S2 Loader doesn't support runtime PM */
+		intf->quirks &= ~bootrom_quirks;
+		intf->quirks |= s2l_quirks;
+		break;
+	default:
+		intf->quirks &= ~bootrom_quirks;
+		intf->quirks &= ~s2l_quirks;
+	}
+
+	/* Clear the init status. */
+	return gb_svc_dme_peer_set(hd->svc, intf->interface_id, attr,
+				   DME_SELECTOR_INDEX_NULL, 0);
+}
+
+/* interface sysfs attributes */
+#define gb_interface_attr(field, type)					\
+static ssize_t field##_show(struct device *dev,				\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct gb_interface *intf = to_gb_interface(dev);		\
+	return scnprintf(buf, PAGE_SIZE, type"\n", intf->field);	\
+}									\
+static DEVICE_ATTR_RO(field)
+
+gb_interface_attr(ddbl1_manufacturer_id, "0x%08x");
+gb_interface_attr(ddbl1_product_id, "0x%08x");
+gb_interface_attr(interface_id, "%u");
+gb_interface_attr(vendor_id, "0x%08x");
+gb_interface_attr(product_id, "0x%08x");
+gb_interface_attr(serial_number, "0x%016llx");
+
+static ssize_t voltage_now_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct gb_interface *intf = to_gb_interface(dev);
+	int ret;
+	u32 measurement;
+
+	ret = gb_svc_pwrmon_intf_sample_get(intf->hd->svc, intf->interface_id,
+					    GB_SVC_PWRMON_TYPE_VOL,
+					    &measurement);
+	if (ret) {
+		dev_err(&intf->dev, "failed to get voltage sample (%d)\n", ret);
+		return ret;
+	}
+
+	return sprintf(buf, "%u\n", measurement);
+}
+static DEVICE_ATTR_RO(voltage_now);
+
+static ssize_t current_now_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct gb_interface *intf = to_gb_interface(dev);
+	int ret;
+	u32 measurement;
+
+	ret = gb_svc_pwrmon_intf_sample_get(intf->hd->svc, intf->interface_id,
+					    GB_SVC_PWRMON_TYPE_CURR,
+					    &measurement);
+	if (ret) {
+		dev_err(&intf->dev, "failed to get current sample (%d)\n", ret);
+		return ret;
+	}
+
+	return sprintf(buf, "%u\n", measurement);
+}
+static DEVICE_ATTR_RO(current_now);
+
+static ssize_t power_now_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct gb_interface *intf = to_gb_interface(dev);
+	int ret;
+	u32 measurement;
+
+	ret = gb_svc_pwrmon_intf_sample_get(intf->hd->svc, intf->interface_id,
+					    GB_SVC_PWRMON_TYPE_PWR,
+					    &measurement);
+	if (ret) {
+		dev_err(&intf->dev, "failed to get power sample (%d)\n", ret);
+		return ret;
+	}
+
+	return sprintf(buf, "%u\n", measurement);
+}
+static DEVICE_ATTR_RO(power_now);
+
+static ssize_t power_state_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct gb_interface *intf = to_gb_interface(dev);
+
+	if (intf->active)
+		return scnprintf(buf, PAGE_SIZE, "on\n");
+	else
+		return scnprintf(buf, PAGE_SIZE, "off\n");
+}
+
+static ssize_t power_state_store(struct device *dev,
+				 struct device_attribute *attr, const char *buf,
+				 size_t len)
+{
+	struct gb_interface *intf = to_gb_interface(dev);
+	bool activate;
+	int ret = 0;
+
+	if (kstrtobool(buf, &activate))
+		return -EINVAL;
+
+	mutex_lock(&intf->mutex);
+
+	if (activate == intf->active)
+		goto unlock;
+
+	if (activate) {
+		ret = gb_interface_activate(intf);
+		if (ret) {
+			dev_err(&intf->dev,
+				"failed to activate interface: %d\n", ret);
+			goto unlock;
+		}
+
+		ret = gb_interface_enable(intf);
+		if (ret) {
+			dev_err(&intf->dev,
+				"failed to enable interface: %d\n", ret);
+			gb_interface_deactivate(intf);
+			goto unlock;
+		}
+	} else {
+		gb_interface_disable(intf);
+		gb_interface_deactivate(intf);
+	}
+
+unlock:
+	mutex_unlock(&intf->mutex);
+
+	if (ret)
+		return ret;
+
+	return len;
+}
+static DEVICE_ATTR_RW(power_state);
+
+static const char *gb_interface_type_string(struct gb_interface *intf)
+{
+	static const char * const types[] = {
+		[GB_INTERFACE_TYPE_INVALID] = "invalid",
+		[GB_INTERFACE_TYPE_UNKNOWN] = "unknown",
+		[GB_INTERFACE_TYPE_DUMMY] = "dummy",
+		[GB_INTERFACE_TYPE_UNIPRO] = "unipro",
+		[GB_INTERFACE_TYPE_GREYBUS] = "greybus",
+	};
+
+	return types[intf->type];
+}
+
+static ssize_t interface_type_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	struct gb_interface *intf = to_gb_interface(dev);
+
+	return sprintf(buf, "%s\n", gb_interface_type_string(intf));
+}
+static DEVICE_ATTR_RO(interface_type);
+
+static struct attribute *interface_unipro_attrs[] = {
+	&dev_attr_ddbl1_manufacturer_id.attr,
+	&dev_attr_ddbl1_product_id.attr,
+	NULL
+};
+
+static struct attribute *interface_greybus_attrs[] = {
+	&dev_attr_vendor_id.attr,
+	&dev_attr_product_id.attr,
+	&dev_attr_serial_number.attr,
+	NULL
+};
+
+static struct attribute *interface_power_attrs[] = {
+	&dev_attr_voltage_now.attr,
+	&dev_attr_current_now.attr,
+	&dev_attr_power_now.attr,
+	&dev_attr_power_state.attr,
+	NULL
+};
+
+static struct attribute *interface_common_attrs[] = {
+	&dev_attr_interface_id.attr,
+	&dev_attr_interface_type.attr,
+	NULL
+};
+
+static umode_t interface_unipro_is_visible(struct kobject *kobj,
+						struct attribute *attr, int n)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct gb_interface *intf = to_gb_interface(dev);
+
+	switch (intf->type) {
+	case GB_INTERFACE_TYPE_UNIPRO:
+	case GB_INTERFACE_TYPE_GREYBUS:
+		return attr->mode;
+	default:
+		return 0;
+	}
+}
+
+static umode_t interface_greybus_is_visible(struct kobject *kobj,
+						struct attribute *attr, int n)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct gb_interface *intf = to_gb_interface(dev);
+
+	switch (intf->type) {
+	case GB_INTERFACE_TYPE_GREYBUS:
+		return attr->mode;
+	default:
+		return 0;
+	}
+}
+
+static umode_t interface_power_is_visible(struct kobject *kobj,
+						struct attribute *attr, int n)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct gb_interface *intf = to_gb_interface(dev);
+
+	switch (intf->type) {
+	case GB_INTERFACE_TYPE_UNIPRO:
+	case GB_INTERFACE_TYPE_GREYBUS:
+		return attr->mode;
+	default:
+		return 0;
+	}
+}
+
+static const struct attribute_group interface_unipro_group = {
+	.is_visible	= interface_unipro_is_visible,
+	.attrs		= interface_unipro_attrs,
+};
+
+static const struct attribute_group interface_greybus_group = {
+	.is_visible	= interface_greybus_is_visible,
+	.attrs		= interface_greybus_attrs,
+};
+
+static const struct attribute_group interface_power_group = {
+	.is_visible	= interface_power_is_visible,
+	.attrs		= interface_power_attrs,
+};
+
+static const struct attribute_group interface_common_group = {
+	.attrs		= interface_common_attrs,
+};
+
+static const struct attribute_group *interface_groups[] = {
+	&interface_unipro_group,
+	&interface_greybus_group,
+	&interface_power_group,
+	&interface_common_group,
+	NULL
+};
+
+static void gb_interface_release(struct device *dev)
+{
+	struct gb_interface *intf = to_gb_interface(dev);
+
+	trace_gb_interface_release(intf);
+
+	kfree(intf);
+}
+
+#ifdef CONFIG_PM
+static int gb_interface_suspend(struct device *dev)
+{
+	struct gb_interface *intf = to_gb_interface(dev);
+	int ret, timesync_ret;
+
+	ret = gb_control_interface_suspend_prepare(intf->control);
+	if (ret)
+		return ret;
+
+	gb_timesync_interface_remove(intf);
+
+	ret = gb_control_suspend(intf->control);
+	if (ret)
+		goto err_hibernate_abort;
+
+	ret = gb_interface_hibernate_link(intf);
+	if (ret)
+		return ret;
+
+	/* Delay to allow interface to enter standby before disabling refclk */
+	msleep(GB_INTERFACE_SUSPEND_HIBERNATE_DELAY_MS);
+
+	ret = gb_interface_refclk_set(intf, false);
+	if (ret)
+		return ret;
+
+	return 0;
+
+err_hibernate_abort:
+	gb_control_interface_hibernate_abort(intf->control);
+
+	timesync_ret = gb_timesync_interface_add(intf);
+	if (timesync_ret) {
+		dev_err(dev, "failed to add to timesync: %d\n", timesync_ret);
+		return timesync_ret;
+	}
+
+	return ret;
+}
+
+static int gb_interface_resume(struct device *dev)
+{
+	struct gb_interface *intf = to_gb_interface(dev);
+	struct gb_svc *svc = intf->hd->svc;
+	int ret;
+
+	ret = gb_interface_refclk_set(intf, true);
+	if (ret)
+		return ret;
+
+	ret = gb_svc_intf_resume(svc, intf->interface_id);
+	if (ret)
+		return ret;
+
+	ret = gb_control_resume(intf->control);
+	if (ret)
+		return ret;
+
+	ret = gb_timesync_interface_add(intf);
+	if (ret) {
+		dev_err(dev, "failed to add to timesync: %d\n", ret);
+		return ret;
+	}
+
+	ret = gb_timesync_schedule_synchronous(intf);
+	if (ret) {
+		dev_err(dev, "failed to synchronize FrameTime: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int gb_interface_runtime_idle(struct device *dev)
+{
+	pm_runtime_mark_last_busy(dev);
+	pm_request_autosuspend(dev);
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops gb_interface_pm_ops = {
+	SET_RUNTIME_PM_OPS(gb_interface_suspend, gb_interface_resume,
+			   gb_interface_runtime_idle)
+};
+
+struct device_type greybus_interface_type = {
+	.name =		"greybus_interface",
+	.release =	gb_interface_release,
+	.pm =		&gb_interface_pm_ops,
+};
+
+/*
+ * A Greybus module represents a user-replaceable component on a GMP
+ * phone.  An interface is the physical connection on that module.  A
+ * module may have more than one interface.
+ *
+ * Create a gb_interface structure to represent a discovered interface.
+ * The position of interface within the Endo is encoded in "interface_id"
+ * argument.
+ *
+ * Returns a pointer to the new interfce or a null pointer if a
+ * failure occurs due to memory exhaustion.
+ */
+struct gb_interface *gb_interface_create(struct gb_module *module,
+					 u8 interface_id)
+{
+	struct gb_host_device *hd = module->hd;
+	struct gb_interface *intf;
+
+	intf = kzalloc(sizeof(*intf), GFP_KERNEL);
+	if (!intf)
+		return NULL;
+
+	intf->hd = hd;		/* XXX refcount? */
+	intf->module = module;
+	intf->interface_id = interface_id;
+	INIT_LIST_HEAD(&intf->bundles);
+	INIT_LIST_HEAD(&intf->manifest_descs);
+	mutex_init(&intf->mutex);
+	INIT_WORK(&intf->mode_switch_work, gb_interface_mode_switch_work);
+	init_completion(&intf->mode_switch_completion);
+
+	/* Invalid device id to start with */
+	intf->device_id = GB_INTERFACE_DEVICE_ID_BAD;
+
+	intf->dev.parent = &module->dev;
+	intf->dev.bus = &greybus_bus_type;
+	intf->dev.type = &greybus_interface_type;
+	intf->dev.groups = interface_groups;
+	intf->dev.dma_mask = module->dev.dma_mask;
+	device_initialize(&intf->dev);
+	dev_set_name(&intf->dev, "%s.%u", dev_name(&module->dev),
+			interface_id);
+
+	pm_runtime_set_autosuspend_delay(&intf->dev,
+					 GB_INTERFACE_AUTOSUSPEND_MS);
+
+	trace_gb_interface_create(intf);
+
+	return intf;
+}
+
+static int gb_interface_vsys_set(struct gb_interface *intf, bool enable)
+{
+	struct gb_svc *svc = intf->hd->svc;
+	int ret;
+
+	dev_dbg(&intf->dev, "%s - %d\n", __func__, enable);
+
+	ret = gb_svc_intf_vsys_set(svc, intf->interface_id, enable);
+	if (ret) {
+		dev_err(&intf->dev, "failed to set v_sys: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int gb_interface_refclk_set(struct gb_interface *intf, bool enable)
+{
+	struct gb_svc *svc = intf->hd->svc;
+	int ret;
+
+	dev_dbg(&intf->dev, "%s - %d\n", __func__, enable);
+
+	ret = gb_svc_intf_refclk_set(svc, intf->interface_id, enable);
+	if (ret) {
+		dev_err(&intf->dev, "failed to set refclk: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int gb_interface_unipro_set(struct gb_interface *intf, bool enable)
+{
+	struct gb_svc *svc = intf->hd->svc;
+	int ret;
+
+	dev_dbg(&intf->dev, "%s - %d\n", __func__, enable);
+
+	ret = gb_svc_intf_unipro_set(svc, intf->interface_id, enable);
+	if (ret) {
+		dev_err(&intf->dev, "failed to set UniPro: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int gb_interface_activate_operation(struct gb_interface *intf,
+					   enum gb_interface_type *intf_type)
+{
+	struct gb_svc *svc = intf->hd->svc;
+	u8 type;
+	int ret;
+
+	dev_dbg(&intf->dev, "%s\n", __func__);
+
+	ret = gb_svc_intf_activate(svc, intf->interface_id, &type);
+	if (ret) {
+		dev_err(&intf->dev, "failed to activate: %d\n", ret);
+		return ret;
+	}
+
+	switch (type) {
+	case GB_SVC_INTF_TYPE_DUMMY:
+		*intf_type = GB_INTERFACE_TYPE_DUMMY;
+		/* FIXME: handle as an error for now */
+		return -ENODEV;
+	case GB_SVC_INTF_TYPE_UNIPRO:
+		*intf_type = GB_INTERFACE_TYPE_UNIPRO;
+		dev_err(&intf->dev, "interface type UniPro not supported\n");
+		/* FIXME: handle as an error for now */
+		return -ENODEV;
+	case GB_SVC_INTF_TYPE_GREYBUS:
+		*intf_type = GB_INTERFACE_TYPE_GREYBUS;
+		break;
+	default:
+		dev_err(&intf->dev, "unknown interface type: %u\n", type);
+		*intf_type = GB_INTERFACE_TYPE_UNKNOWN;
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int gb_interface_hibernate_link(struct gb_interface *intf)
+{
+	struct gb_svc *svc = intf->hd->svc;
+
+	return gb_svc_intf_set_power_mode_hibernate(svc, intf->interface_id);
+}
+
+static int _gb_interface_activate(struct gb_interface *intf,
+				  enum gb_interface_type *type)
+{
+	int ret;
+
+	*type = GB_INTERFACE_TYPE_UNKNOWN;
+
+	if (intf->ejected || intf->removed)
+		return -ENODEV;
+
+	ret = gb_interface_vsys_set(intf, true);
+	if (ret)
+		return ret;
+
+	ret = gb_interface_refclk_set(intf, true);
+	if (ret)
+		goto err_vsys_disable;
+
+	ret = gb_interface_unipro_set(intf, true);
+	if (ret)
+		goto err_refclk_disable;
+
+	ret = gb_interface_activate_operation(intf, type);
+	if (ret) {
+		switch (*type) {
+		case GB_INTERFACE_TYPE_UNIPRO:
+		case GB_INTERFACE_TYPE_GREYBUS:
+			goto err_hibernate_link;
+		default:
+			goto err_unipro_disable;
+		}
+	}
+
+	ret = gb_interface_read_dme(intf);
+	if (ret)
+		goto err_hibernate_link;
+
+	ret = gb_interface_route_create(intf);
+	if (ret)
+		goto err_hibernate_link;
+
+	intf->active = true;
+
+	trace_gb_interface_activate(intf);
+
+	return 0;
+
+err_hibernate_link:
+	gb_interface_hibernate_link(intf);
+err_unipro_disable:
+	gb_interface_unipro_set(intf, false);
+err_refclk_disable:
+	gb_interface_refclk_set(intf, false);
+err_vsys_disable:
+	gb_interface_vsys_set(intf, false);
+
+	return ret;
+}
+
+/*
+ * At present, we assume a UniPro-only module to be a Greybus module that
+ * failed to send its mailbox poke. There is some reason to believe that this
+ * is because of a bug in the ES3 bootrom.
+ *
+ * FIXME: Check if this is a Toshiba bridge before retrying?
+ */
+static int _gb_interface_activate_es3_hack(struct gb_interface *intf,
+					   enum gb_interface_type *type)
+{
+	int retries = 3;
+	int ret;
+
+	while (retries--) {
+		ret = _gb_interface_activate(intf, type);
+		if (ret == -ENODEV && *type == GB_INTERFACE_TYPE_UNIPRO)
+			continue;
+
+		break;
+	}
+
+	return ret;
+}
+
+/*
+ * Activate an interface.
+ *
+ * Locking: Caller holds the interface mutex.
+ */
+int gb_interface_activate(struct gb_interface *intf)
+{
+	enum gb_interface_type type;
+	int ret;
+
+	switch (intf->type) {
+	case GB_INTERFACE_TYPE_INVALID:
+	case GB_INTERFACE_TYPE_GREYBUS:
+		ret = _gb_interface_activate_es3_hack(intf, &type);
+		break;
+	default:
+		ret = _gb_interface_activate(intf, &type);
+	}
+
+	/* Make sure type is detected correctly during reactivation. */
+	if (intf->type != GB_INTERFACE_TYPE_INVALID) {
+		if (type != intf->type) {
+			dev_err(&intf->dev, "failed to detect interface type\n");
+
+			if (!ret)
+				gb_interface_deactivate(intf);
+
+			return -EIO;
+		}
+	} else {
+		intf->type = type;
+	}
+
+	return ret;
+}
+
+/*
+ * Deactivate an interface.
+ *
+ * Locking: Caller holds the interface mutex.
+ */
+void gb_interface_deactivate(struct gb_interface *intf)
+{
+	if (!intf->active)
+		return;
+
+	trace_gb_interface_deactivate(intf);
+
+	/* Abort any ongoing mode switch. */
+	if (intf->mode_switch)
+		complete(&intf->mode_switch_completion);
+
+	gb_interface_route_destroy(intf);
+	gb_interface_hibernate_link(intf);
+	gb_interface_unipro_set(intf, false);
+	gb_interface_refclk_set(intf, false);
+	gb_interface_vsys_set(intf, false);
+
+	intf->active = false;
+}
+
+/*
+ * Enable an interface by enabling its control connection, fetching the
+ * manifest and other information over it, and finally registering its child
+ * devices.
+ *
+ * Locking: Caller holds the interface mutex.
+ */
+int gb_interface_enable(struct gb_interface *intf)
+{
+	struct gb_control *control;
+	struct gb_bundle *bundle, *tmp;
+	int ret, size;
+	void *manifest;
+
+	ret = gb_interface_read_and_clear_init_status(intf);
+	if (ret) {
+		dev_err(&intf->dev, "failed to clear init status: %d\n", ret);
+		return ret;
+	}
+
+	/* Establish control connection */
+	control = gb_control_create(intf);
+	if (IS_ERR(control)) {
+		dev_err(&intf->dev, "failed to create control device: %ld\n",
+				PTR_ERR(control));
+		return PTR_ERR(control);
+	}
+	intf->control = control;
+
+	ret = gb_control_enable(intf->control);
+	if (ret)
+		goto err_put_control;
+
+	/* Get manifest size using control protocol on CPort */
+	size = gb_control_get_manifest_size_operation(intf);
+	if (size <= 0) {
+		dev_err(&intf->dev, "failed to get manifest size: %d\n", size);
+
+		if (size)
+			ret = size;
+		else
+			ret =  -EINVAL;
+
+		goto err_disable_control;
+	}
+
+	manifest = kmalloc(size, GFP_KERNEL);
+	if (!manifest) {
+		ret = -ENOMEM;
+		goto err_disable_control;
+	}
+
+	/* Get manifest using control protocol on CPort */
+	ret = gb_control_get_manifest_operation(intf, manifest, size);
+	if (ret) {
+		dev_err(&intf->dev, "failed to get manifest: %d\n", ret);
+		goto err_free_manifest;
+	}
+
+	/*
+	 * Parse the manifest and build up our data structures representing
+	 * what's in it.
+	 */
+	if (!gb_manifest_parse(intf, manifest, size)) {
+		dev_err(&intf->dev, "failed to parse manifest\n");
+		ret = -EINVAL;
+		goto err_destroy_bundles;
+	}
+
+	ret = gb_control_get_bundle_versions(intf->control);
+	if (ret)
+		goto err_destroy_bundles;
+
+	ret = gb_timesync_interface_add(intf);
+	if (ret) {
+		dev_err(&intf->dev, "failed to add to timesync: %d\n", ret);
+		goto err_destroy_bundles;
+	}
+
+	/* Register the control device and any bundles */
+	ret = gb_control_add(intf->control);
+	if (ret)
+		goto err_remove_timesync;
+
+	pm_runtime_use_autosuspend(&intf->dev);
+	pm_runtime_get_noresume(&intf->dev);
+	pm_runtime_set_active(&intf->dev);
+	pm_runtime_enable(&intf->dev);
+
+	list_for_each_entry_safe_reverse(bundle, tmp, &intf->bundles, links) {
+		ret = gb_bundle_add(bundle);
+		if (ret) {
+			gb_bundle_destroy(bundle);
+			continue;
+		}
+	}
+
+	kfree(manifest);
+
+	intf->enabled = true;
+
+	pm_runtime_put(&intf->dev);
+
+	trace_gb_interface_enable(intf);
+
+	return 0;
+
+err_remove_timesync:
+	gb_timesync_interface_remove(intf);
+err_destroy_bundles:
+	list_for_each_entry_safe(bundle, tmp, &intf->bundles, links)
+		gb_bundle_destroy(bundle);
+err_free_manifest:
+	kfree(manifest);
+err_disable_control:
+	gb_control_disable(intf->control);
+err_put_control:
+	gb_control_put(intf->control);
+	intf->control = NULL;
+
+	return ret;
+}
+
+/*
+ * Disable an interface and destroy its bundles.
+ *
+ * Locking: Caller holds the interface mutex.
+ */
+void gb_interface_disable(struct gb_interface *intf)
+{
+	struct gb_bundle *bundle;
+	struct gb_bundle *next;
+
+	if (!intf->enabled)
+		return;
+
+	trace_gb_interface_disable(intf);
+
+	pm_runtime_get_sync(&intf->dev);
+
+	/* Set disconnected flag to avoid I/O during connection tear down. */
+	if (intf->quirks & GB_INTERFACE_QUIRK_FORCED_DISABLE)
+		intf->disconnected = true;
+
+	list_for_each_entry_safe(bundle, next, &intf->bundles, links)
+		gb_bundle_destroy(bundle);
+
+	if (!intf->mode_switch && !intf->disconnected)
+		gb_control_interface_deactivate_prepare(intf->control);
+
+	gb_control_del(intf->control);
+	gb_timesync_interface_remove(intf);
+	gb_control_disable(intf->control);
+	gb_control_put(intf->control);
+	intf->control = NULL;
+
+	intf->enabled = false;
+
+	pm_runtime_disable(&intf->dev);
+	pm_runtime_set_suspended(&intf->dev);
+	pm_runtime_dont_use_autosuspend(&intf->dev);
+	pm_runtime_put_noidle(&intf->dev);
+}
+
+/* Enable TimeSync on an Interface control connection. */
+int gb_interface_timesync_enable(struct gb_interface *intf, u8 count,
+				 u64 frame_time, u32 strobe_delay, u32 refclk)
+{
+	return gb_control_timesync_enable(intf->control, count,
+					  frame_time, strobe_delay,
+					  refclk);
+}
+
+/* Disable TimeSync on an Interface control connection. */
+int gb_interface_timesync_disable(struct gb_interface *intf)
+{
+	return gb_control_timesync_disable(intf->control);
+}
+
+/* Transmit the Authoritative FrameTime via an Interface control connection. */
+int gb_interface_timesync_authoritative(struct gb_interface *intf,
+					u64 *frame_time)
+{
+	return gb_control_timesync_authoritative(intf->control,
+						frame_time);
+}
+
+/* Register an interface. */
+int gb_interface_add(struct gb_interface *intf)
+{
+	int ret;
+
+	ret = device_add(&intf->dev);
+	if (ret) {
+		dev_err(&intf->dev, "failed to register interface: %d\n", ret);
+		return ret;
+	}
+
+	trace_gb_interface_add(intf);
+
+	dev_info(&intf->dev, "Interface added (%s)\n",
+			gb_interface_type_string(intf));
+
+	switch (intf->type) {
+	case GB_INTERFACE_TYPE_GREYBUS:
+		dev_info(&intf->dev, "GMP VID=0x%08x, PID=0x%08x\n",
+				intf->vendor_id, intf->product_id);
+		/* fall-through */
+	case GB_INTERFACE_TYPE_UNIPRO:
+		dev_info(&intf->dev, "DDBL1 Manufacturer=0x%08x, Product=0x%08x\n",
+				intf->ddbl1_manufacturer_id,
+				intf->ddbl1_product_id);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+/* Deregister an interface. */
+void gb_interface_del(struct gb_interface *intf)
+{
+	if (device_is_registered(&intf->dev)) {
+		trace_gb_interface_del(intf);
+
+		device_del(&intf->dev);
+		dev_info(&intf->dev, "Interface removed\n");
+	}
+}
+
+void gb_interface_put(struct gb_interface *intf)
+{
+	put_device(&intf->dev);
+}
diff --git a/drivers/staging/greybus/interface.h b/drivers/staging/greybus/interface.h
new file mode 100644
index 000000000000..03299d2a8be5
--- /dev/null
+++ b/drivers/staging/greybus/interface.h
@@ -0,0 +1,88 @@
+/*
+ * Greybus Interface Block code
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __INTERFACE_H
+#define __INTERFACE_H
+
+enum gb_interface_type {
+	GB_INTERFACE_TYPE_INVALID = 0,
+	GB_INTERFACE_TYPE_UNKNOWN,
+	GB_INTERFACE_TYPE_DUMMY,
+	GB_INTERFACE_TYPE_UNIPRO,
+	GB_INTERFACE_TYPE_GREYBUS,
+};
+
+#define GB_INTERFACE_QUIRK_NO_CPORT_FEATURES		BIT(0)
+#define GB_INTERFACE_QUIRK_NO_INIT_STATUS		BIT(1)
+#define GB_INTERFACE_QUIRK_NO_GMP_IDS			BIT(2)
+#define GB_INTERFACE_QUIRK_FORCED_DISABLE		BIT(3)
+#define GB_INTERFACE_QUIRK_LEGACY_MODE_SWITCH		BIT(4)
+#define GB_INTERFACE_QUIRK_NO_BUNDLE_ACTIVATE		BIT(5)
+#define GB_INTERFACE_QUIRK_NO_PM			BIT(6)
+
+struct gb_interface {
+	struct device dev;
+	struct gb_control *control;
+
+	struct list_head bundles;
+	struct list_head module_node;
+	struct list_head manifest_descs;
+	u8 interface_id;	/* Physical location within the Endo */
+	u8 device_id;
+	u8 features;		/* Feature flags set in the manifest */
+
+	enum gb_interface_type type;
+
+	u32 ddbl1_manufacturer_id;
+	u32 ddbl1_product_id;
+	u32 vendor_id;
+	u32 product_id;
+	u64 serial_number;
+
+	struct gb_host_device *hd;
+	struct gb_module *module;
+
+	unsigned long quirks;
+
+	struct mutex mutex;
+
+	bool disconnected;
+
+	bool ejected;
+	bool removed;
+	bool active;
+	bool enabled;
+	bool mode_switch;
+	bool dme_read;
+
+	struct work_struct mode_switch_work;
+	struct completion mode_switch_completion;
+};
+#define to_gb_interface(d) container_of(d, struct gb_interface, dev)
+
+struct gb_interface *gb_interface_create(struct gb_module *module,
+					 u8 interface_id);
+int gb_interface_activate(struct gb_interface *intf);
+void gb_interface_deactivate(struct gb_interface *intf);
+int gb_interface_enable(struct gb_interface *intf);
+void gb_interface_disable(struct gb_interface *intf);
+int gb_interface_timesync_enable(struct gb_interface *intf, u8 count,
+				 u64 frame_time, u32 strobe_delay, u32 refclk);
+int gb_interface_timesync_authoritative(struct gb_interface *intf,
+					u64 *frame_time);
+int gb_interface_timesync_disable(struct gb_interface *intf);
+int gb_interface_add(struct gb_interface *intf);
+void gb_interface_del(struct gb_interface *intf);
+void gb_interface_put(struct gb_interface *intf);
+void gb_interface_mailbox_event(struct gb_interface *intf, u16 result,
+								u32 mailbox);
+
+int gb_interface_request_mode_switch(struct gb_interface *intf);
+
+#endif /* __INTERFACE_H */
diff --git a/drivers/staging/greybus/light.c b/drivers/staging/greybus/light.c
new file mode 100644
index 000000000000..8dffd8a7e762
--- /dev/null
+++ b/drivers/staging/greybus/light.c
@@ -0,0 +1,1361 @@
+/*
+ * Greybus Lights protocol driver.
+ *
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/leds.h>
+#include <linux/led-class-flash.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/version.h>
+#include <media/v4l2-flash-led-class.h>
+
+#include "greybus.h"
+#include "greybus_protocols.h"
+
+#define NAMES_MAX	32
+
+struct gb_channel {
+	u8				id;
+	u32				flags;
+	u32				color;
+	char				*color_name;
+	u8				fade_in;
+	u8				fade_out;
+	u32				mode;
+	char				*mode_name;
+	struct attribute		**attrs;
+	struct attribute_group		*attr_group;
+	const struct attribute_group	**attr_groups;
+	struct led_classdev		*led;
+#if IS_REACHABLE(CONFIG_LEDS_CLASS_FLASH)
+	struct led_classdev_flash	fled;
+	struct led_flash_setting	intensity_uA;
+	struct led_flash_setting	timeout_us;
+#else
+	struct led_classdev		cled;
+#endif
+	struct gb_light			*light;
+	bool				is_registered;
+	bool				releasing;
+	bool				strobe_state;
+	bool				active;
+	struct mutex			lock;
+};
+
+struct gb_light {
+	u8			id;
+	char			*name;
+	struct gb_lights	*glights;
+	u32			flags;
+	u8			channels_count;
+	struct gb_channel	*channels;
+	bool			has_flash;
+	bool			ready;
+#if IS_REACHABLE(CONFIG_V4L2_FLASH_LED_CLASS)
+	struct v4l2_flash	*v4l2_flash;
+#endif
+};
+
+struct gb_lights {
+	struct gb_connection	*connection;
+	u8			lights_count;
+	struct gb_light		*lights;
+	struct mutex		lights_lock;
+};
+
+static void gb_lights_channel_free(struct gb_channel *channel);
+
+static struct gb_connection *get_conn_from_channel(struct gb_channel *channel)
+{
+	return channel->light->glights->connection;
+}
+
+static struct gb_connection *get_conn_from_light(struct gb_light *light)
+{
+	return light->glights->connection;
+}
+
+static bool is_channel_flash(struct gb_channel *channel)
+{
+	return !!(channel->mode & (GB_CHANNEL_MODE_FLASH | GB_CHANNEL_MODE_TORCH
+				   | GB_CHANNEL_MODE_INDICATOR));
+}
+
+#if IS_REACHABLE(CONFIG_LEDS_CLASS_FLASH)
+static struct gb_channel *get_channel_from_cdev(struct led_classdev *cdev)
+{
+	struct led_classdev_flash *fled_cdev = lcdev_to_flcdev(cdev);
+
+	return container_of(fled_cdev, struct gb_channel, fled);
+}
+
+static struct led_classdev *get_channel_cdev(struct gb_channel *channel)
+{
+	return &channel->fled.led_cdev;
+}
+
+static struct gb_channel *get_channel_from_mode(struct gb_light *light,
+						u32 mode)
+{
+	struct gb_channel *channel = NULL;
+	int i;
+
+	for (i = 0; i < light->channels_count; i++) {
+		channel = &light->channels[i];
+		if (channel && channel->mode == mode)
+			break;
+	}
+	return channel;
+}
+
+static int __gb_lights_flash_intensity_set(struct gb_channel *channel,
+					   u32 intensity)
+{
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct gb_bundle *bundle = connection->bundle;
+	struct gb_lights_set_flash_intensity_request req;
+	int ret;
+
+	if (channel->releasing)
+		return -ESHUTDOWN;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret < 0)
+		return ret;
+
+	req.light_id = channel->light->id;
+	req.channel_id = channel->id;
+	req.intensity_uA = cpu_to_le32(intensity);
+
+	ret = gb_operation_sync(connection, GB_LIGHTS_TYPE_SET_FLASH_INTENSITY,
+				&req, sizeof(req), NULL, 0);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return ret;
+}
+
+static int __gb_lights_flash_brightness_set(struct gb_channel *channel)
+{
+	u32 intensity;
+
+	/* If the channel is flash we need to get the attached torch channel */
+	if (channel->mode & GB_CHANNEL_MODE_FLASH)
+		channel = get_channel_from_mode(channel->light,
+						GB_CHANNEL_MODE_TORCH);
+
+	/* For not flash we need to convert brightness to intensity */
+	intensity = channel->intensity_uA.min +
+			(channel->intensity_uA.step * channel->led->brightness);
+
+	return __gb_lights_flash_intensity_set(channel, intensity);
+}
+#else
+static struct gb_channel *get_channel_from_cdev(struct led_classdev *cdev)
+{
+	return container_of(cdev, struct gb_channel, cled);
+}
+
+static struct led_classdev *get_channel_cdev(struct gb_channel *channel)
+{
+	return &channel->cled;
+}
+
+static int __gb_lights_flash_brightness_set(struct gb_channel *channel)
+{
+	return 0;
+}
+#endif
+
+static int gb_lights_color_set(struct gb_channel *channel, u32 color);
+static int gb_lights_fade_set(struct gb_channel *channel);
+
+static void led_lock(struct led_classdev *cdev)
+{
+	mutex_lock(&cdev->led_access);
+}
+
+static void led_unlock(struct led_classdev *cdev)
+{
+	mutex_unlock(&cdev->led_access);
+}
+
+#define gb_lights_fade_attr(__dir)					\
+static ssize_t fade_##__dir##_show(struct device *dev,			\
+				   struct device_attribute *attr,	\
+				   char *buf)				\
+{									\
+	struct led_classdev *cdev = dev_get_drvdata(dev);		\
+	struct gb_channel *channel = get_channel_from_cdev(cdev);	\
+									\
+	return sprintf(buf, "%u\n", channel->fade_##__dir);		\
+}									\
+									\
+static ssize_t fade_##__dir##_store(struct device *dev,			\
+				    struct device_attribute *attr,	\
+				    const char *buf, size_t size)	\
+{									\
+	struct led_classdev *cdev = dev_get_drvdata(dev);		\
+	struct gb_channel *channel = get_channel_from_cdev(cdev);	\
+	u8 fade;							\
+	int ret;							\
+									\
+	led_lock(cdev);							\
+	if (led_sysfs_is_disabled(cdev)) {				\
+		ret = -EBUSY;						\
+		goto unlock;						\
+	}								\
+									\
+	ret = kstrtou8(buf, 0, &fade);					\
+	if (ret < 0) {							\
+		dev_err(dev, "could not parse fade value %d\n", ret);	\
+		goto unlock;						\
+	}								\
+	if (channel->fade_##__dir == fade)				\
+		goto unlock;						\
+	channel->fade_##__dir = fade;					\
+									\
+	ret = gb_lights_fade_set(channel);				\
+	if (ret < 0)							\
+		goto unlock;						\
+									\
+	ret = size;							\
+unlock:									\
+	led_unlock(cdev);						\
+	return ret;							\
+}									\
+static DEVICE_ATTR_RW(fade_##__dir)
+
+gb_lights_fade_attr(in);
+gb_lights_fade_attr(out);
+
+static ssize_t color_show(struct device *dev, struct device_attribute *attr,
+			  char *buf)
+{
+	struct led_classdev *cdev = dev_get_drvdata(dev);
+	struct gb_channel *channel = get_channel_from_cdev(cdev);
+
+	return sprintf(buf, "0x%08x\n", channel->color);
+}
+
+static ssize_t color_store(struct device *dev, struct device_attribute *attr,
+			   const char *buf, size_t size)
+{
+	struct led_classdev *cdev = dev_get_drvdata(dev);
+	struct gb_channel *channel = get_channel_from_cdev(cdev);
+	u32 color;
+	int ret;
+
+	led_lock(cdev);
+	if (led_sysfs_is_disabled(cdev)) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+	ret = kstrtou32(buf, 0, &color);
+	if (ret < 0) {
+		dev_err(dev, "could not parse color value %d\n", ret);
+		goto unlock;
+	}
+
+	ret = gb_lights_color_set(channel, color);
+	if (ret < 0)
+		goto unlock;
+
+	channel->color = color;
+	ret = size;
+unlock:
+	led_unlock(cdev);
+	return ret;
+}
+static DEVICE_ATTR_RW(color);
+
+static int channel_attr_groups_set(struct gb_channel *channel,
+				   struct led_classdev *cdev)
+{
+	int attr = 0;
+	int size = 0;
+
+	if (channel->flags & GB_LIGHT_CHANNEL_MULTICOLOR)
+		size++;
+	if (channel->flags & GB_LIGHT_CHANNEL_FADER)
+		size += 2;
+
+	if (!size)
+		return 0;
+
+	/* Set attributes based in the channel flags */
+	channel->attrs = kcalloc(size + 1, sizeof(*channel->attrs), GFP_KERNEL);
+	if (!channel->attrs)
+		return -ENOMEM;
+	channel->attr_group = kcalloc(1, sizeof(*channel->attr_group),
+				      GFP_KERNEL);
+	if (!channel->attr_group)
+		return -ENOMEM;
+	channel->attr_groups = kcalloc(2, sizeof(*channel->attr_groups),
+				       GFP_KERNEL);
+	if (!channel->attr_groups)
+		return -ENOMEM;
+
+	if (channel->flags & GB_LIGHT_CHANNEL_MULTICOLOR)
+		channel->attrs[attr++] = &dev_attr_color.attr;
+	if (channel->flags & GB_LIGHT_CHANNEL_FADER) {
+		channel->attrs[attr++] = &dev_attr_fade_in.attr;
+		channel->attrs[attr++] = &dev_attr_fade_out.attr;
+	}
+
+	channel->attr_group->attrs = channel->attrs;
+
+	channel->attr_groups[0] = channel->attr_group;
+
+	cdev->groups = channel->attr_groups;
+
+	return 0;
+}
+
+static int gb_lights_fade_set(struct gb_channel *channel)
+{
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct gb_bundle *bundle = connection->bundle;
+	struct gb_lights_set_fade_request req;
+	int ret;
+
+	if (channel->releasing)
+		return -ESHUTDOWN;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret < 0)
+		return ret;
+
+	req.light_id = channel->light->id;
+	req.channel_id = channel->id;
+	req.fade_in = channel->fade_in;
+	req.fade_out = channel->fade_out;
+	ret = gb_operation_sync(connection, GB_LIGHTS_TYPE_SET_FADE,
+				&req, sizeof(req), NULL, 0);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return ret;
+}
+
+static int gb_lights_color_set(struct gb_channel *channel, u32 color)
+{
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct gb_bundle *bundle = connection->bundle;
+	struct gb_lights_set_color_request req;
+	int ret;
+
+	if (channel->releasing)
+		return -ESHUTDOWN;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret < 0)
+		return ret;
+
+	req.light_id = channel->light->id;
+	req.channel_id = channel->id;
+	req.color = cpu_to_le32(color);
+	ret = gb_operation_sync(connection, GB_LIGHTS_TYPE_SET_COLOR,
+				&req, sizeof(req), NULL, 0);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return ret;
+}
+
+static int __gb_lights_led_brightness_set(struct gb_channel *channel)
+{
+	struct gb_lights_set_brightness_request req;
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct gb_bundle *bundle = connection->bundle;
+	bool old_active;
+	int ret;
+
+	mutex_lock(&channel->lock);
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret < 0)
+		goto out_unlock;
+
+	old_active = channel->active;
+
+	req.light_id = channel->light->id;
+	req.channel_id = channel->id;
+	req.brightness = (u8)channel->led->brightness;
+
+	ret = gb_operation_sync(connection, GB_LIGHTS_TYPE_SET_BRIGHTNESS,
+				&req, sizeof(req), NULL, 0);
+	if (ret < 0)
+		goto out_pm_put;
+
+	if (channel->led->brightness)
+		channel->active = true;
+	else
+		channel->active = false;
+
+	/* we need to keep module alive when turning to active state */
+	if (!old_active && channel->active)
+		goto out_unlock;
+
+	/*
+	 * on the other hand if going to inactive we still hold a reference and
+	 * need to put it, so we could go to suspend.
+	 */
+	if (old_active && !channel->active)
+		gb_pm_runtime_put_autosuspend(bundle);
+
+out_pm_put:
+	gb_pm_runtime_put_autosuspend(bundle);
+out_unlock:
+	mutex_unlock(&channel->lock);
+
+	return ret;
+}
+
+static int __gb_lights_brightness_set(struct gb_channel *channel)
+{
+	int ret;
+
+	if (channel->releasing)
+		return 0;
+
+	if (is_channel_flash(channel))
+		ret = __gb_lights_flash_brightness_set(channel);
+	else
+		ret = __gb_lights_led_brightness_set(channel);
+
+	return ret;
+}
+
+static int gb_brightness_set(struct led_classdev *cdev,
+			     enum led_brightness value)
+{
+	struct gb_channel *channel = get_channel_from_cdev(cdev);
+
+	channel->led->brightness = value;
+
+	return __gb_lights_brightness_set(channel);
+}
+
+static enum led_brightness gb_brightness_get(struct led_classdev *cdev)
+
+{
+	struct gb_channel *channel = get_channel_from_cdev(cdev);
+
+	return channel->led->brightness;
+}
+
+static int gb_blink_set(struct led_classdev *cdev, unsigned long *delay_on,
+			unsigned long *delay_off)
+{
+	struct gb_channel *channel = get_channel_from_cdev(cdev);
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct gb_bundle *bundle = connection->bundle;
+	struct gb_lights_blink_request req;
+	bool old_active;
+	int ret;
+
+	if (channel->releasing)
+		return -ESHUTDOWN;
+
+	if (!delay_on || !delay_off)
+		return -EINVAL;
+
+	mutex_lock(&channel->lock);
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret < 0)
+		goto out_unlock;
+
+	old_active = channel->active;
+
+	req.light_id = channel->light->id;
+	req.channel_id = channel->id;
+	req.time_on_ms = cpu_to_le16(*delay_on);
+	req.time_off_ms = cpu_to_le16(*delay_off);
+
+	ret = gb_operation_sync(connection, GB_LIGHTS_TYPE_SET_BLINK, &req,
+				sizeof(req), NULL, 0);
+	if (ret < 0)
+		goto out_pm_put;
+
+	if (*delay_on)
+		channel->active = true;
+	else
+		channel->active = false;
+
+	/* we need to keep module alive when turning to active state */
+	if (!old_active && channel->active)
+		goto out_unlock;
+
+	/*
+	 * on the other hand if going to inactive we still hold a reference and
+	 * need to put it, so we could go to suspend.
+	 */
+	if (old_active && !channel->active)
+		gb_pm_runtime_put_autosuspend(bundle);
+
+out_pm_put:
+	gb_pm_runtime_put_autosuspend(bundle);
+out_unlock:
+	mutex_unlock(&channel->lock);
+
+	return ret;
+}
+
+static void gb_lights_led_operations_set(struct gb_channel *channel,
+					 struct led_classdev *cdev)
+{
+	cdev->brightness_get = gb_brightness_get;
+	cdev->brightness_set_blocking = gb_brightness_set;
+
+	if (channel->flags & GB_LIGHT_CHANNEL_BLINK)
+		cdev->blink_set = gb_blink_set;
+}
+
+#if IS_REACHABLE(CONFIG_V4L2_FLASH_LED_CLASS)
+/* V4L2 specific helpers */
+static const struct v4l2_flash_ops v4l2_flash_ops;
+
+static void __gb_lights_channel_v4l2_config(struct led_flash_setting *channel_s,
+					    struct led_flash_setting *v4l2_s)
+{
+	v4l2_s->min = channel_s->min;
+	v4l2_s->max = channel_s->max;
+	v4l2_s->step = channel_s->step;
+	/* For v4l2 val is the default value */
+	v4l2_s->val = channel_s->max;
+}
+
+static int gb_lights_light_v4l2_register(struct gb_light *light)
+{
+	struct gb_connection *connection = get_conn_from_light(light);
+	struct device *dev = &connection->bundle->dev;
+	struct v4l2_flash_config *sd_cfg;
+	struct led_classdev_flash *fled;
+	struct led_classdev_flash *iled = NULL;
+	struct gb_channel *channel_torch, *channel_ind, *channel_flash;
+	int ret = 0;
+
+	sd_cfg = kcalloc(1, sizeof(*sd_cfg), GFP_KERNEL);
+	if (!sd_cfg)
+		return -ENOMEM;
+
+	channel_torch = get_channel_from_mode(light, GB_CHANNEL_MODE_TORCH);
+	if (channel_torch)
+		__gb_lights_channel_v4l2_config(&channel_torch->intensity_uA,
+						&sd_cfg->torch_intensity);
+
+	channel_ind = get_channel_from_mode(light, GB_CHANNEL_MODE_INDICATOR);
+	if (channel_ind) {
+		__gb_lights_channel_v4l2_config(&channel_ind->intensity_uA,
+						&sd_cfg->indicator_intensity);
+		iled = &channel_ind->fled;
+	}
+
+	channel_flash = get_channel_from_mode(light, GB_CHANNEL_MODE_FLASH);
+	WARN_ON(!channel_flash);
+
+	fled = &channel_flash->fled;
+
+	snprintf(sd_cfg->dev_name, sizeof(sd_cfg->dev_name), "%s", light->name);
+
+	/* Set the possible values to faults, in our case all faults */
+	sd_cfg->flash_faults = LED_FAULT_OVER_VOLTAGE | LED_FAULT_TIMEOUT |
+		LED_FAULT_OVER_TEMPERATURE | LED_FAULT_SHORT_CIRCUIT |
+		LED_FAULT_OVER_CURRENT | LED_FAULT_INDICATOR |
+		LED_FAULT_UNDER_VOLTAGE | LED_FAULT_INPUT_VOLTAGE |
+		LED_FAULT_LED_OVER_TEMPERATURE;
+
+	light->v4l2_flash = v4l2_flash_init(dev, NULL, fled, iled,
+					    &v4l2_flash_ops, sd_cfg);
+	if (IS_ERR_OR_NULL(light->v4l2_flash)) {
+		ret = PTR_ERR(light->v4l2_flash);
+		goto out_free;
+	}
+
+	return ret;
+
+out_free:
+	kfree(sd_cfg);
+	return ret;
+}
+
+static void gb_lights_light_v4l2_unregister(struct gb_light *light)
+{
+	v4l2_flash_release(light->v4l2_flash);
+}
+#else
+static int gb_lights_light_v4l2_register(struct gb_light *light)
+{
+	struct gb_connection *connection = get_conn_from_light(light);
+
+	dev_err(&connection->bundle->dev, "no support for v4l2 subdevices\n");
+	return 0;
+}
+
+static void gb_lights_light_v4l2_unregister(struct gb_light *light)
+{
+}
+#endif
+
+#if IS_REACHABLE(CONFIG_LEDS_CLASS_FLASH)
+/* Flash specific operations */
+static int gb_lights_flash_intensity_set(struct led_classdev_flash *fcdev,
+					 u32 brightness)
+{
+	struct gb_channel *channel = container_of(fcdev, struct gb_channel,
+						  fled);
+	int ret;
+
+	ret = __gb_lights_flash_intensity_set(channel, brightness);
+	if (ret < 0)
+		return ret;
+
+	fcdev->brightness.val = brightness;
+
+	return 0;
+}
+
+static int gb_lights_flash_intensity_get(struct led_classdev_flash *fcdev,
+					 u32 *brightness)
+{
+	*brightness = fcdev->brightness.val;
+
+	return 0;
+}
+
+static int gb_lights_flash_strobe_set(struct led_classdev_flash *fcdev,
+				      bool state)
+{
+	struct gb_channel *channel = container_of(fcdev, struct gb_channel,
+						  fled);
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct gb_bundle *bundle = connection->bundle;
+	struct gb_lights_set_flash_strobe_request req;
+	int ret;
+
+	if (channel->releasing)
+		return -ESHUTDOWN;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret < 0)
+		return ret;
+
+	req.light_id = channel->light->id;
+	req.channel_id = channel->id;
+	req.state = state ? 1 : 0;
+
+	ret = gb_operation_sync(connection, GB_LIGHTS_TYPE_SET_FLASH_STROBE,
+				&req, sizeof(req), NULL, 0);
+	if (!ret)
+		channel->strobe_state = state;
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return ret;
+}
+
+static int gb_lights_flash_strobe_get(struct led_classdev_flash *fcdev,
+				      bool *state)
+{
+	struct gb_channel *channel = container_of(fcdev, struct gb_channel,
+						  fled);
+
+	*state = channel->strobe_state;
+	return 0;
+}
+
+static int gb_lights_flash_timeout_set(struct led_classdev_flash *fcdev,
+				       u32 timeout)
+{
+	struct gb_channel *channel = container_of(fcdev, struct gb_channel,
+						  fled);
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct gb_bundle *bundle = connection->bundle;
+	struct gb_lights_set_flash_timeout_request req;
+	int ret;
+
+	if (channel->releasing)
+		return -ESHUTDOWN;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret < 0)
+		return ret;
+
+	req.light_id = channel->light->id;
+	req.channel_id = channel->id;
+	req.timeout_us = cpu_to_le32(timeout);
+
+	ret = gb_operation_sync(connection, GB_LIGHTS_TYPE_SET_FLASH_TIMEOUT,
+				&req, sizeof(req), NULL, 0);
+	if (!ret)
+		fcdev->timeout.val = timeout;
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return ret;
+}
+
+static int gb_lights_flash_fault_get(struct led_classdev_flash *fcdev,
+				     u32 *fault)
+{
+	struct gb_channel *channel = container_of(fcdev, struct gb_channel,
+						  fled);
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct gb_bundle *bundle = connection->bundle;
+	struct gb_lights_get_flash_fault_request req;
+	struct gb_lights_get_flash_fault_response resp;
+	int ret;
+
+	if (channel->releasing)
+		return -ESHUTDOWN;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret < 0)
+		return ret;
+
+	req.light_id = channel->light->id;
+	req.channel_id = channel->id;
+
+	ret = gb_operation_sync(connection, GB_LIGHTS_TYPE_GET_FLASH_FAULT,
+				&req, sizeof(req), &resp, sizeof(resp));
+	if (!ret)
+		*fault = le32_to_cpu(resp.fault);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return ret;
+}
+
+static const struct led_flash_ops gb_lights_flash_ops = {
+	.flash_brightness_set	= gb_lights_flash_intensity_set,
+	.flash_brightness_get	= gb_lights_flash_intensity_get,
+	.strobe_set		= gb_lights_flash_strobe_set,
+	.strobe_get		= gb_lights_flash_strobe_get,
+	.timeout_set		= gb_lights_flash_timeout_set,
+	.fault_get		= gb_lights_flash_fault_get,
+};
+
+static int __gb_lights_channel_torch_attach(struct gb_channel *channel,
+					    struct gb_channel *channel_torch)
+{
+	char *name;
+
+	/* we can only attach torch to a flash channel */
+	if (!(channel->mode & GB_CHANNEL_MODE_FLASH))
+		return 0;
+
+	/* Move torch brightness to the destination */
+	channel->led->max_brightness = channel_torch->led->max_brightness;
+
+	/* append mode name to flash name */
+	name = kasprintf(GFP_KERNEL, "%s_%s", channel->led->name,
+			 channel_torch->mode_name);
+	if (!name)
+		return -ENOMEM;
+	kfree(channel->led->name);
+	channel->led->name = name;
+
+	channel_torch->led = channel->led;
+
+	return 0;
+}
+
+static int __gb_lights_flash_led_register(struct gb_channel *channel)
+{
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct led_classdev_flash *fled = &channel->fled;
+	struct led_flash_setting *fset;
+	struct gb_channel *channel_torch;
+	int ret;
+
+	fled->ops = &gb_lights_flash_ops;
+
+	fled->led_cdev.flags |= LED_DEV_CAP_FLASH;
+
+	fset = &fled->brightness;
+	fset->min = channel->intensity_uA.min;
+	fset->max = channel->intensity_uA.max;
+	fset->step = channel->intensity_uA.step;
+	fset->val = channel->intensity_uA.max;
+
+	/* Only the flash mode have the timeout constraints settings */
+	if (channel->mode & GB_CHANNEL_MODE_FLASH) {
+		fset = &fled->timeout;
+		fset->min = channel->timeout_us.min;
+		fset->max = channel->timeout_us.max;
+		fset->step = channel->timeout_us.step;
+		fset->val = channel->timeout_us.max;
+	}
+
+	/*
+	 * If light have torch mode channel, this channel will be the led
+	 * classdev of the registered above flash classdev
+	 */
+	channel_torch = get_channel_from_mode(channel->light,
+					      GB_CHANNEL_MODE_TORCH);
+	if (channel_torch) {
+		ret = __gb_lights_channel_torch_attach(channel, channel_torch);
+		if (ret < 0)
+			goto fail;
+	}
+
+	ret = led_classdev_flash_register(&connection->bundle->dev, fled);
+	if (ret < 0)
+		goto fail;
+
+	channel->is_registered = true;
+	return 0;
+fail:
+	channel->led = NULL;
+	return ret;
+}
+
+static void __gb_lights_flash_led_unregister(struct gb_channel *channel)
+{
+	if (!channel->is_registered)
+		return;
+
+	led_classdev_flash_unregister(&channel->fled);
+}
+
+static int gb_lights_channel_flash_config(struct gb_channel *channel)
+{
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct gb_lights_get_channel_flash_config_request req;
+	struct gb_lights_get_channel_flash_config_response conf;
+	struct led_flash_setting *fset;
+	int ret;
+
+	req.light_id = channel->light->id;
+	req.channel_id = channel->id;
+
+	ret = gb_operation_sync(connection,
+				GB_LIGHTS_TYPE_GET_CHANNEL_FLASH_CONFIG,
+				&req, sizeof(req), &conf, sizeof(conf));
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Intensity constraints for flash related modes: flash, torch,
+	 * indicator.  They will be needed for v4l2 registration.
+	 */
+	fset = &channel->intensity_uA;
+	fset->min = le32_to_cpu(conf.intensity_min_uA);
+	fset->max = le32_to_cpu(conf.intensity_max_uA);
+	fset->step = le32_to_cpu(conf.intensity_step_uA);
+
+	/*
+	 * On flash type, max brightness is set as the number of intensity steps
+	 * available.
+	 */
+	channel->led->max_brightness = (fset->max - fset->min) / fset->step;
+
+	/* Only the flash mode have the timeout constraints settings */
+	if (channel->mode & GB_CHANNEL_MODE_FLASH) {
+		fset = &channel->timeout_us;
+		fset->min = le32_to_cpu(conf.timeout_min_us);
+		fset->max = le32_to_cpu(conf.timeout_max_us);
+		fset->step = le32_to_cpu(conf.timeout_step_us);
+	}
+
+	return 0;
+}
+#else
+static int gb_lights_channel_flash_config(struct gb_channel *channel)
+{
+	struct gb_connection *connection = get_conn_from_channel(channel);
+
+	dev_err(&connection->bundle->dev, "no support for flash devices\n");
+	return 0;
+}
+
+static int __gb_lights_flash_led_register(struct gb_channel *channel)
+{
+	return 0;
+}
+
+static void __gb_lights_flash_led_unregister(struct gb_channel *channel)
+{
+}
+
+#endif
+
+static int __gb_lights_led_register(struct gb_channel *channel)
+{
+	struct gb_connection *connection = get_conn_from_channel(channel);
+	struct led_classdev *cdev = get_channel_cdev(channel);
+	int ret;
+
+	ret = led_classdev_register(&connection->bundle->dev, cdev);
+	if (ret < 0)
+		channel->led = NULL;
+	else
+		channel->is_registered = true;
+	return ret;
+}
+
+static int gb_lights_channel_register(struct gb_channel *channel)
+{
+	/* Normal LED channel, just register in led classdev and we are done */
+	if (!is_channel_flash(channel))
+		return __gb_lights_led_register(channel);
+
+	/*
+	 * Flash Type need more work, register flash classdev, indicator as
+	 * flash classdev, torch will be led classdev of the flash classdev.
+	 */
+	if (!(channel->mode & GB_CHANNEL_MODE_TORCH))
+		return __gb_lights_flash_led_register(channel);
+
+	return 0;
+}
+
+static void __gb_lights_led_unregister(struct gb_channel *channel)
+{
+	struct led_classdev *cdev = get_channel_cdev(channel);
+
+	if (!channel->is_registered)
+		return;
+
+	led_classdev_unregister(cdev);
+	channel->led = NULL;
+}
+
+static void gb_lights_channel_unregister(struct gb_channel *channel)
+{
+	/* The same as register, handle channels differently */
+	if (!is_channel_flash(channel)) {
+		__gb_lights_led_unregister(channel);
+		return;
+	}
+
+	if (channel->mode & GB_CHANNEL_MODE_TORCH)
+		__gb_lights_led_unregister(channel);
+	else
+		__gb_lights_flash_led_unregister(channel);
+}
+
+static int gb_lights_channel_config(struct gb_light *light,
+				    struct gb_channel *channel)
+{
+	struct gb_lights_get_channel_config_response conf;
+	struct gb_lights_get_channel_config_request req;
+	struct gb_connection *connection = get_conn_from_light(light);
+	struct led_classdev *cdev = get_channel_cdev(channel);
+	char *name;
+	int ret;
+
+	req.light_id = light->id;
+	req.channel_id = channel->id;
+
+	ret = gb_operation_sync(connection, GB_LIGHTS_TYPE_GET_CHANNEL_CONFIG,
+				&req, sizeof(req), &conf, sizeof(conf));
+	if (ret < 0)
+		return ret;
+
+	channel->light = light;
+	channel->mode = le32_to_cpu(conf.mode);
+	channel->flags = le32_to_cpu(conf.flags);
+	channel->color = le32_to_cpu(conf.color);
+	channel->color_name = kstrndup(conf.color_name, NAMES_MAX, GFP_KERNEL);
+	if (!channel->color_name)
+		return -ENOMEM;
+	channel->mode_name = kstrndup(conf.mode_name, NAMES_MAX, GFP_KERNEL);
+	if (!channel->mode_name)
+		return -ENOMEM;
+
+	channel->led = cdev;
+
+	name = kasprintf(GFP_KERNEL, "%s:%s:%s", light->name,
+			 channel->color_name, channel->mode_name);
+	if (!name)
+		return -ENOMEM;
+
+	cdev->name = name;
+
+	cdev->max_brightness = conf.max_brightness;
+
+	ret = channel_attr_groups_set(channel, cdev);
+	if (ret < 0)
+		return ret;
+
+	gb_lights_led_operations_set(channel, cdev);
+
+	/*
+	 * If it is not a flash related channel (flash, torch or indicator) we
+	 * are done here. If not, continue and fetch flash related
+	 * configurations.
+	 */
+	if (!is_channel_flash(channel))
+		return ret;
+
+	light->has_flash = true;
+
+	ret = gb_lights_channel_flash_config(channel);
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+static int gb_lights_light_config(struct gb_lights *glights, u8 id)
+{
+	struct gb_light *light = &glights->lights[id];
+	struct gb_lights_get_light_config_request req;
+	struct gb_lights_get_light_config_response conf;
+	int ret;
+	int i;
+
+	light->glights = glights;
+	light->id = id;
+
+	req.id = id;
+
+	ret = gb_operation_sync(glights->connection,
+				GB_LIGHTS_TYPE_GET_LIGHT_CONFIG,
+				&req, sizeof(req), &conf, sizeof(conf));
+	if (ret < 0)
+		return ret;
+
+	if (!conf.channel_count)
+		return -EINVAL;
+	if (!strlen(conf.name))
+		return -EINVAL;
+
+	light->channels_count = conf.channel_count;
+	light->name = kstrndup(conf.name, NAMES_MAX, GFP_KERNEL);
+
+	light->channels = kzalloc(light->channels_count *
+				  sizeof(struct gb_channel), GFP_KERNEL);
+	if (!light->channels)
+		return -ENOMEM;
+
+	/* First we collect all the configurations for all channels */
+	for (i = 0; i < light->channels_count; i++) {
+		light->channels[i].id = i;
+		ret = gb_lights_channel_config(light, &light->channels[i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int gb_lights_light_register(struct gb_light *light)
+{
+	int ret;
+	int i;
+
+	/*
+	 * Then, if everything went ok in getting configurations, we register
+	 * the classdev, flash classdev and v4l2 subsystem, if a flash device is
+	 * found.
+	 */
+	for (i = 0; i < light->channels_count; i++) {
+		ret = gb_lights_channel_register(&light->channels[i]);
+		if (ret < 0)
+			return ret;
+
+		mutex_init(&light->channels[i].lock);
+	}
+
+	light->ready = true;
+
+	if (light->has_flash) {
+		ret = gb_lights_light_v4l2_register(light);
+		if (ret < 0) {
+			light->has_flash = false;
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void gb_lights_channel_free(struct gb_channel *channel)
+{
+	kfree(channel->attrs);
+	kfree(channel->attr_group);
+	kfree(channel->attr_groups);
+	kfree(channel->color_name);
+	kfree(channel->mode_name);
+	mutex_destroy(&channel->lock);
+}
+
+static void gb_lights_channel_release(struct gb_channel *channel)
+{
+	channel->releasing = true;
+
+	gb_lights_channel_unregister(channel);
+
+	gb_lights_channel_free(channel);
+}
+
+static void gb_lights_light_release(struct gb_light *light)
+{
+	int i;
+	int count;
+
+	light->ready = false;
+
+	count = light->channels_count;
+
+	if (light->has_flash)
+		gb_lights_light_v4l2_unregister(light);
+
+	for (i = 0; i < count; i++) {
+		gb_lights_channel_release(&light->channels[i]);
+		light->channels_count--;
+	}
+	kfree(light->channels);
+	kfree(light->name);
+}
+
+static void gb_lights_release(struct gb_lights *glights)
+{
+	int i;
+
+	if (!glights)
+		return;
+
+	mutex_lock(&glights->lights_lock);
+	if (!glights->lights)
+		goto free_glights;
+
+	for (i = 0; i < glights->lights_count; i++)
+		gb_lights_light_release(&glights->lights[i]);
+
+	kfree(glights->lights);
+
+free_glights:
+	mutex_unlock(&glights->lights_lock);
+	mutex_destroy(&glights->lights_lock);
+	kfree(glights);
+}
+
+static int gb_lights_get_count(struct gb_lights *glights)
+{
+	struct gb_lights_get_lights_response resp;
+	int ret;
+
+	ret = gb_operation_sync(glights->connection, GB_LIGHTS_TYPE_GET_LIGHTS,
+				NULL, 0, &resp, sizeof(resp));
+	if (ret < 0)
+		return ret;
+
+	if (!resp.lights_count)
+		return -EINVAL;
+
+	glights->lights_count = resp.lights_count;
+
+	return 0;
+}
+
+static int gb_lights_create_all(struct gb_lights *glights)
+{
+	struct gb_connection *connection = glights->connection;
+	int ret;
+	int i;
+
+	mutex_lock(&glights->lights_lock);
+	ret = gb_lights_get_count(glights);
+	if (ret < 0)
+		goto out;
+
+	glights->lights = kzalloc(glights->lights_count *
+				  sizeof(struct gb_light), GFP_KERNEL);
+	if (!glights->lights) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	for (i = 0; i < glights->lights_count; i++) {
+		ret = gb_lights_light_config(glights, i);
+		if (ret < 0) {
+			dev_err(&connection->bundle->dev,
+				"Fail to configure lights device\n");
+			goto out;
+		}
+	}
+
+out:
+	mutex_unlock(&glights->lights_lock);
+	return ret;
+}
+
+static int gb_lights_register_all(struct gb_lights *glights)
+{
+	struct gb_connection *connection = glights->connection;
+	int ret = 0;
+	int i;
+
+	mutex_lock(&glights->lights_lock);
+	for (i = 0; i < glights->lights_count; i++) {
+		ret = gb_lights_light_register(&glights->lights[i]);
+		if (ret < 0) {
+			dev_err(&connection->bundle->dev,
+				"Fail to enable lights device\n");
+			break;
+		}
+	}
+
+	mutex_unlock(&glights->lights_lock);
+	return ret;
+}
+
+static int gb_lights_request_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct device *dev = &connection->bundle->dev;
+	struct gb_lights *glights = gb_connection_get_data(connection);
+	struct gb_light *light;
+	struct gb_message *request;
+	struct gb_lights_event_request *payload;
+	int ret =  0;
+	u8 light_id;
+	u8 event;
+
+	if (op->type != GB_LIGHTS_TYPE_EVENT) {
+		dev_err(dev, "Unsupported unsolicited event: %u\n", op->type);
+		return -EINVAL;
+	}
+
+	request = op->request;
+
+	if (request->payload_size < sizeof(*payload)) {
+		dev_err(dev, "Wrong event size received (%zu < %zu)\n",
+			request->payload_size, sizeof(*payload));
+		return -EINVAL;
+	}
+
+	payload = request->payload;
+	light_id = payload->light_id;
+
+	if (light_id >= glights->lights_count ||
+	    !glights->lights[light_id].ready) {
+		dev_err(dev, "Event received for unconfigured light id: %d\n",
+			light_id);
+		return -EINVAL;
+	}
+
+	event = payload->event;
+
+	if (event & GB_LIGHTS_LIGHT_CONFIG) {
+		light = &glights->lights[light_id];
+
+		mutex_lock(&glights->lights_lock);
+		gb_lights_light_release(light);
+		ret = gb_lights_light_config(glights, light_id);
+		if (!ret)
+			ret = gb_lights_light_register(light);
+		if (ret < 0)
+			gb_lights_light_release(light);
+		mutex_unlock(&glights->lights_lock);
+	}
+
+	return ret;
+}
+
+static int gb_lights_probe(struct gb_bundle *bundle,
+			   const struct greybus_bundle_id *id)
+{
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_connection *connection;
+	struct gb_lights *glights;
+	int ret;
+
+	if (bundle->num_cports != 1)
+		return -ENODEV;
+
+	cport_desc = &bundle->cport_desc[0];
+	if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LIGHTS)
+		return -ENODEV;
+
+	glights = kzalloc(sizeof(*glights), GFP_KERNEL);
+	if (!glights)
+		return -ENOMEM;
+
+	mutex_init(&glights->lights_lock);
+
+	connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
+					  gb_lights_request_handler);
+	if (IS_ERR(connection)) {
+		ret = PTR_ERR(connection);
+		goto out;
+	}
+
+	glights->connection = connection;
+	gb_connection_set_data(connection, glights);
+
+	greybus_set_drvdata(bundle, glights);
+
+	/* We aren't ready to receive an incoming request yet */
+	ret = gb_connection_enable_tx(connection);
+	if (ret)
+		goto error_connection_destroy;
+
+	/*
+	 * Setup all the lights devices over this connection, if anything goes
+	 * wrong tear down all lights
+	 */
+	ret = gb_lights_create_all(glights);
+	if (ret < 0)
+		goto error_connection_disable;
+
+	/* We are ready to receive an incoming request now, enable RX as well */
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto error_connection_disable;
+
+	/* Enable & register lights */
+	ret = gb_lights_register_all(glights);
+	if (ret < 0)
+		goto error_connection_disable;
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return 0;
+
+error_connection_disable:
+	gb_connection_disable(connection);
+error_connection_destroy:
+	gb_connection_destroy(connection);
+out:
+	gb_lights_release(glights);
+	return ret;
+}
+
+static void gb_lights_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_lights *glights = greybus_get_drvdata(bundle);
+
+	if (gb_pm_runtime_get_sync(bundle))
+		gb_pm_runtime_get_noresume(bundle);
+
+	gb_connection_disable(glights->connection);
+	gb_connection_destroy(glights->connection);
+
+	gb_lights_release(glights);
+}
+
+static const struct greybus_bundle_id gb_lights_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LIGHTS) },
+	{ }
+};
+MODULE_DEVICE_TABLE(greybus, gb_lights_id_table);
+
+static struct greybus_driver gb_lights_driver = {
+	.name		= "lights",
+	.probe		= gb_lights_probe,
+	.disconnect	= gb_lights_disconnect,
+	.id_table	= gb_lights_id_table,
+};
+module_greybus_driver(gb_lights_driver);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/log.c b/drivers/staging/greybus/log.c
new file mode 100644
index 000000000000..70dd9e5a1cf2
--- /dev/null
+++ b/drivers/staging/greybus/log.c
@@ -0,0 +1,132 @@
+/*
+ * Greybus driver for the log protocol
+ *
+ * Copyright 2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/sizes.h>
+#include <linux/uaccess.h>
+
+#include "greybus.h"
+
+struct gb_log {
+	struct gb_connection *connection;
+};
+
+static int gb_log_request_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct device *dev = &connection->bundle->dev;
+	struct gb_log_send_log_request *receive;
+	u16 len;
+
+	if (op->type != GB_LOG_TYPE_SEND_LOG) {
+		dev_err(dev, "unknown request type 0x%02x\n", op->type);
+		return -EINVAL;
+	}
+
+	/* Verify size of payload */
+	if (op->request->payload_size < sizeof(*receive)) {
+		dev_err(dev, "log request too small (%zu < %zu)\n",
+				op->request->payload_size, sizeof(*receive));
+		return -EINVAL;
+	}
+	receive = op->request->payload;
+	len = le16_to_cpu(receive->len);
+	if (len != (int)(op->request->payload_size - sizeof(*receive))) {
+		dev_err(dev, "log request wrong size %d vs %d\n", len,
+				(int)(op->request->payload_size - sizeof(*receive)));
+		return -EINVAL;
+	}
+	if (len == 0) {
+		dev_err(dev, "log request of 0 bytes?\n");
+		return -EINVAL;
+	}
+
+	if (len > GB_LOG_MAX_LEN) {
+		dev_err(dev, "log request too big: %d\n", len);
+		return -EINVAL;
+	}
+
+	/* Ensure the buffer is 0 terminated */
+	receive->msg[len - 1] = '\0';
+
+	/* Print with dev_dbg() so that it can be easily turned off using
+	 * dynamic debugging (and prevent any DoS) */
+	dev_dbg(dev, "%s", receive->msg);
+
+	return 0;
+}
+
+static int gb_log_probe(struct gb_bundle *bundle,
+			const struct greybus_bundle_id *id)
+{
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_connection *connection;
+	struct gb_log *log;
+	int retval;
+
+	if (bundle->num_cports != 1)
+		return -ENODEV;
+
+	cport_desc = &bundle->cport_desc[0];
+	if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOG)
+		return -ENODEV;
+
+	log = kzalloc(sizeof(*log), GFP_KERNEL);
+	if (!log)
+		return -ENOMEM;
+
+	connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
+			gb_log_request_handler);
+	if (IS_ERR(connection)) {
+		retval = PTR_ERR(connection);
+		goto error_free;
+	}
+
+	log->connection = connection;
+	greybus_set_drvdata(bundle, log);
+
+	retval = gb_connection_enable(connection);
+	if (retval)
+		goto error_connection_destroy;
+
+	return 0;
+
+error_connection_destroy:
+	gb_connection_destroy(connection);
+error_free:
+	kfree(log);
+	return retval;
+}
+
+static void gb_log_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_log *log = greybus_get_drvdata(bundle);
+	struct gb_connection *connection = log->connection;
+
+	gb_connection_disable(connection);
+	gb_connection_destroy(connection);
+
+	kfree(log);
+}
+
+static const struct greybus_bundle_id gb_log_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOG) },
+	{ }
+};
+MODULE_DEVICE_TABLE(greybus, gb_log_id_table);
+
+static struct greybus_driver gb_log_driver = {
+	.name           = "log",
+	.probe          = gb_log_probe,
+	.disconnect     = gb_log_disconnect,
+	.id_table       = gb_log_id_table,
+};
+module_greybus_driver(gb_log_driver);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/loopback.c b/drivers/staging/greybus/loopback.c
new file mode 100644
index 000000000000..7882306adeca
--- /dev/null
+++ b/drivers/staging/greybus/loopback.c
@@ -0,0 +1,1364 @@
+/*
+ * Loopback bridge driver for the Greybus loopback module.
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/kthread.h>
+#include <linux/delay.h>
+#include <linux/random.h>
+#include <linux/sizes.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/kfifo.h>
+#include <linux/debugfs.h>
+#include <linux/list_sort.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/atomic.h>
+#include <linux/pm_runtime.h>
+
+#include <asm/div64.h>
+
+#include "greybus.h"
+#include "connection.h"
+
+#define NSEC_PER_DAY 86400000000000ULL
+
+struct gb_loopback_stats {
+	u32 min;
+	u32 max;
+	u64 sum;
+	u32 count;
+};
+
+struct gb_loopback_device {
+	struct dentry *root;
+	u32 count;
+	size_t size_max;
+
+	/* We need to take a lock in atomic context */
+	spinlock_t lock;
+	struct list_head list;
+	struct list_head list_op_async;
+	wait_queue_head_t wq;
+};
+
+static struct gb_loopback_device gb_dev;
+
+struct gb_loopback_async_operation {
+	struct gb_loopback *gb;
+	struct gb_operation *operation;
+	struct timeval ts;
+	struct timer_list timer;
+	struct list_head entry;
+	struct work_struct work;
+	struct kref kref;
+	bool pending;
+	int (*completion)(struct gb_loopback_async_operation *op_async);
+};
+
+struct gb_loopback {
+	struct gb_connection *connection;
+
+	struct dentry *file;
+	struct kfifo kfifo_lat;
+	struct kfifo kfifo_ts;
+	struct mutex mutex;
+	struct task_struct *task;
+	struct list_head entry;
+	struct device *dev;
+	wait_queue_head_t wq;
+	wait_queue_head_t wq_completion;
+	atomic_t outstanding_operations;
+
+	/* Per connection stats */
+	struct timeval ts;
+	struct gb_loopback_stats latency;
+	struct gb_loopback_stats throughput;
+	struct gb_loopback_stats requests_per_second;
+	struct gb_loopback_stats apbridge_unipro_latency;
+	struct gb_loopback_stats gbphy_firmware_latency;
+
+	int type;
+	int async;
+	int id;
+	u32 size;
+	u32 iteration_max;
+	u32 iteration_count;
+	int us_wait;
+	u32 error;
+	u32 requests_completed;
+	u32 requests_timedout;
+	u32 timeout;
+	u32 jiffy_timeout;
+	u32 timeout_min;
+	u32 timeout_max;
+	u32 outstanding_operations_max;
+	u32 lbid;
+	u64 elapsed_nsecs;
+	u32 apbridge_latency_ts;
+	u32 gbphy_latency_ts;
+
+	u32 send_count;
+};
+
+static struct class loopback_class = {
+	.name		= "gb_loopback",
+	.owner		= THIS_MODULE,
+};
+static DEFINE_IDA(loopback_ida);
+
+/* Min/max values in jiffies */
+#define GB_LOOPBACK_TIMEOUT_MIN				1
+#define GB_LOOPBACK_TIMEOUT_MAX				10000
+
+#define GB_LOOPBACK_FIFO_DEFAULT			8192
+
+static unsigned kfifo_depth = GB_LOOPBACK_FIFO_DEFAULT;
+module_param(kfifo_depth, uint, 0444);
+
+/* Maximum size of any one send data buffer we support */
+#define MAX_PACKET_SIZE (PAGE_SIZE * 2)
+
+#define GB_LOOPBACK_US_WAIT_MAX				1000000
+
+/* interface sysfs attributes */
+#define gb_loopback_ro_attr(field)				\
+static ssize_t field##_show(struct device *dev,			\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct gb_loopback *gb = dev_get_drvdata(dev);			\
+	return sprintf(buf, "%u\n", gb->field);			\
+}									\
+static DEVICE_ATTR_RO(field)
+
+#define gb_loopback_ro_stats_attr(name, field, type)		\
+static ssize_t name##_##field##_show(struct device *dev,	\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct gb_loopback *gb = dev_get_drvdata(dev);			\
+	/* Report 0 for min and max if no transfer successed */		\
+	if (!gb->requests_completed)					\
+		return sprintf(buf, "0\n");				\
+	return sprintf(buf, "%"#type"\n", gb->name.field);	\
+}									\
+static DEVICE_ATTR_RO(name##_##field)
+
+#define gb_loopback_ro_avg_attr(name)			\
+static ssize_t name##_avg_show(struct device *dev,		\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct gb_loopback_stats *stats;				\
+	struct gb_loopback *gb;						\
+	u64 avg, rem;							\
+	u32 count;							\
+	gb = dev_get_drvdata(dev);			\
+	stats = &gb->name;					\
+	count = stats->count ? stats->count : 1;			\
+	avg = stats->sum + count / 2000000; /* round closest */		\
+	rem = do_div(avg, count);					\
+	rem *= 1000000;							\
+	do_div(rem, count);						\
+	return sprintf(buf, "%llu.%06u\n", avg, (u32)rem);		\
+}									\
+static DEVICE_ATTR_RO(name##_avg)
+
+#define gb_loopback_stats_attrs(field)				\
+	gb_loopback_ro_stats_attr(field, min, u);		\
+	gb_loopback_ro_stats_attr(field, max, u);		\
+	gb_loopback_ro_avg_attr(field)
+
+#define gb_loopback_attr(field, type)					\
+static ssize_t field##_show(struct device *dev,				\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct gb_loopback *gb = dev_get_drvdata(dev);			\
+	return sprintf(buf, "%"#type"\n", gb->field);			\
+}									\
+static ssize_t field##_store(struct device *dev,			\
+			    struct device_attribute *attr,		\
+			    const char *buf,				\
+			    size_t len)					\
+{									\
+	int ret;							\
+	struct gb_loopback *gb = dev_get_drvdata(dev);			\
+	mutex_lock(&gb->mutex);						\
+	ret = sscanf(buf, "%"#type, &gb->field);			\
+	if (ret != 1)							\
+		len = -EINVAL;						\
+	else								\
+		gb_loopback_check_attr(gb, bundle);			\
+	mutex_unlock(&gb->mutex);					\
+	return len;							\
+}									\
+static DEVICE_ATTR_RW(field)
+
+#define gb_dev_loopback_ro_attr(field, conn)				\
+static ssize_t field##_show(struct device *dev,		\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct gb_loopback *gb = dev_get_drvdata(dev);			\
+	return sprintf(buf, "%u\n", gb->field);				\
+}									\
+static DEVICE_ATTR_RO(field)
+
+#define gb_dev_loopback_rw_attr(field, type)				\
+static ssize_t field##_show(struct device *dev,				\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct gb_loopback *gb = dev_get_drvdata(dev);			\
+	return sprintf(buf, "%"#type"\n", gb->field);			\
+}									\
+static ssize_t field##_store(struct device *dev,			\
+			    struct device_attribute *attr,		\
+			    const char *buf,				\
+			    size_t len)					\
+{									\
+	int ret;							\
+	struct gb_loopback *gb = dev_get_drvdata(dev);			\
+	mutex_lock(&gb->mutex);						\
+	ret = sscanf(buf, "%"#type, &gb->field);			\
+	if (ret != 1)							\
+		len = -EINVAL;						\
+	else								\
+		gb_loopback_check_attr(gb);		\
+	mutex_unlock(&gb->mutex);					\
+	return len;							\
+}									\
+static DEVICE_ATTR_RW(field)
+
+static void gb_loopback_reset_stats(struct gb_loopback *gb);
+static void gb_loopback_check_attr(struct gb_loopback *gb)
+{
+	if (gb->us_wait > GB_LOOPBACK_US_WAIT_MAX)
+		gb->us_wait = GB_LOOPBACK_US_WAIT_MAX;
+	if (gb->size > gb_dev.size_max)
+		gb->size = gb_dev.size_max;
+	gb->requests_timedout = 0;
+	gb->requests_completed = 0;
+	gb->iteration_count = 0;
+	gb->send_count = 0;
+	gb->error = 0;
+
+	if (kfifo_depth < gb->iteration_max) {
+		dev_warn(gb->dev,
+			 "cannot log bytes %u kfifo_depth %u\n",
+			 gb->iteration_max, kfifo_depth);
+	}
+	kfifo_reset_out(&gb->kfifo_lat);
+	kfifo_reset_out(&gb->kfifo_ts);
+
+	switch (gb->type) {
+	case GB_LOOPBACK_TYPE_PING:
+	case GB_LOOPBACK_TYPE_TRANSFER:
+	case GB_LOOPBACK_TYPE_SINK:
+		gb->jiffy_timeout = usecs_to_jiffies(gb->timeout);
+		if (!gb->jiffy_timeout)
+			gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MIN;
+		else if (gb->jiffy_timeout > GB_LOOPBACK_TIMEOUT_MAX)
+			gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MAX;
+		gb_loopback_reset_stats(gb);
+		wake_up(&gb->wq);
+		break;
+	default:
+		gb->type = 0;
+		break;
+	}
+}
+
+/* Time to send and receive one message */
+gb_loopback_stats_attrs(latency);
+/* Number of requests sent per second on this cport */
+gb_loopback_stats_attrs(requests_per_second);
+/* Quantity of data sent and received on this cport */
+gb_loopback_stats_attrs(throughput);
+/* Latency across the UniPro link from APBridge's perspective */
+gb_loopback_stats_attrs(apbridge_unipro_latency);
+/* Firmware induced overhead in the GPBridge */
+gb_loopback_stats_attrs(gbphy_firmware_latency);
+
+/* Number of errors encountered during loop */
+gb_loopback_ro_attr(error);
+/* Number of requests successfully completed async */
+gb_loopback_ro_attr(requests_completed);
+/* Number of requests timed out async */
+gb_loopback_ro_attr(requests_timedout);
+/* Timeout minimum in useconds */
+gb_loopback_ro_attr(timeout_min);
+/* Timeout minimum in useconds */
+gb_loopback_ro_attr(timeout_max);
+
+/*
+ * Type of loopback message to send based on protocol type definitions
+ * 0 => Don't send message
+ * 2 => Send ping message continuously (message without payload)
+ * 3 => Send transfer message continuously (message with payload,
+ *					   payload returned in response)
+ * 4 => Send a sink message (message with payload, no payload in response)
+ */
+gb_dev_loopback_rw_attr(type, d);
+/* Size of transfer message payload: 0-4096 bytes */
+gb_dev_loopback_rw_attr(size, u);
+/* Time to wait between two messages: 0-1000 ms */
+gb_dev_loopback_rw_attr(us_wait, d);
+/* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */
+gb_dev_loopback_rw_attr(iteration_max, u);
+/* The current index of the for (i = 0; i < iteration_max; i++) loop */
+gb_dev_loopback_ro_attr(iteration_count, false);
+/* A flag to indicate synchronous or asynchronous operations */
+gb_dev_loopback_rw_attr(async, u);
+/* Timeout of an individual asynchronous request */
+gb_dev_loopback_rw_attr(timeout, u);
+/* Maximum number of in-flight operations before back-off */
+gb_dev_loopback_rw_attr(outstanding_operations_max, u);
+
+static struct attribute *loopback_attrs[] = {
+	&dev_attr_latency_min.attr,
+	&dev_attr_latency_max.attr,
+	&dev_attr_latency_avg.attr,
+	&dev_attr_requests_per_second_min.attr,
+	&dev_attr_requests_per_second_max.attr,
+	&dev_attr_requests_per_second_avg.attr,
+	&dev_attr_throughput_min.attr,
+	&dev_attr_throughput_max.attr,
+	&dev_attr_throughput_avg.attr,
+	&dev_attr_apbridge_unipro_latency_min.attr,
+	&dev_attr_apbridge_unipro_latency_max.attr,
+	&dev_attr_apbridge_unipro_latency_avg.attr,
+	&dev_attr_gbphy_firmware_latency_min.attr,
+	&dev_attr_gbphy_firmware_latency_max.attr,
+	&dev_attr_gbphy_firmware_latency_avg.attr,
+	&dev_attr_type.attr,
+	&dev_attr_size.attr,
+	&dev_attr_us_wait.attr,
+	&dev_attr_iteration_count.attr,
+	&dev_attr_iteration_max.attr,
+	&dev_attr_async.attr,
+	&dev_attr_error.attr,
+	&dev_attr_requests_completed.attr,
+	&dev_attr_requests_timedout.attr,
+	&dev_attr_timeout.attr,
+	&dev_attr_outstanding_operations_max.attr,
+	&dev_attr_timeout_min.attr,
+	&dev_attr_timeout_max.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(loopback);
+
+static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error);
+
+static u32 gb_loopback_nsec_to_usec_latency(u64 elapsed_nsecs)
+{
+	u32 lat;
+
+	do_div(elapsed_nsecs, NSEC_PER_USEC);
+	lat = elapsed_nsecs;
+	return lat;
+}
+
+static u64 __gb_loopback_calc_latency(u64 t1, u64 t2)
+{
+	if (t2 > t1)
+		return t2 - t1;
+	else
+		return NSEC_PER_DAY - t2 + t1;
+}
+
+static u64 gb_loopback_calc_latency(struct timeval *ts, struct timeval *te)
+{
+	u64 t1, t2;
+
+	t1 = timeval_to_ns(ts);
+	t2 = timeval_to_ns(te);
+
+	return __gb_loopback_calc_latency(t1, t2);
+}
+
+static void gb_loopback_push_latency_ts(struct gb_loopback *gb,
+					struct timeval *ts, struct timeval *te)
+{
+	kfifo_in(&gb->kfifo_ts, (unsigned char *)ts, sizeof(*ts));
+	kfifo_in(&gb->kfifo_ts, (unsigned char *)te, sizeof(*te));
+}
+
+static int gb_loopback_operation_sync(struct gb_loopback *gb, int type,
+				      void *request, int request_size,
+				      void *response, int response_size)
+{
+	struct gb_operation *operation;
+	struct timeval ts, te;
+	int ret;
+
+	do_gettimeofday(&ts);
+	operation = gb_operation_create(gb->connection, type, request_size,
+					response_size, GFP_KERNEL);
+	if (!operation)
+		return -ENOMEM;
+
+	if (request_size)
+		memcpy(operation->request->payload, request, request_size);
+
+	ret = gb_operation_request_send_sync(operation);
+	if (ret) {
+		dev_err(&gb->connection->bundle->dev,
+			"synchronous operation failed: %d\n", ret);
+		goto out_put_operation;
+	} else {
+		if (response_size == operation->response->payload_size) {
+			memcpy(response, operation->response->payload,
+			       response_size);
+		} else {
+			dev_err(&gb->connection->bundle->dev,
+				"response size %zu expected %d\n",
+				operation->response->payload_size,
+				response_size);
+			ret = -EINVAL;
+			goto out_put_operation;
+		}
+	}
+
+	do_gettimeofday(&te);
+
+	/* Calculate the total time the message took */
+	gb_loopback_push_latency_ts(gb, &ts, &te);
+	gb->elapsed_nsecs = gb_loopback_calc_latency(&ts, &te);
+
+out_put_operation:
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+static void __gb_loopback_async_operation_destroy(struct kref *kref)
+{
+	struct gb_loopback_async_operation *op_async;
+
+	op_async = container_of(kref, struct gb_loopback_async_operation, kref);
+
+	list_del(&op_async->entry);
+	if (op_async->operation)
+		gb_operation_put(op_async->operation);
+	atomic_dec(&op_async->gb->outstanding_operations);
+	wake_up(&op_async->gb->wq_completion);
+	kfree(op_async);
+}
+
+static void gb_loopback_async_operation_get(struct gb_loopback_async_operation
+					    *op_async)
+{
+	kref_get(&op_async->kref);
+}
+
+static void gb_loopback_async_operation_put(struct gb_loopback_async_operation
+					    *op_async)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&gb_dev.lock, flags);
+	kref_put(&op_async->kref, __gb_loopback_async_operation_destroy);
+	spin_unlock_irqrestore(&gb_dev.lock, flags);
+}
+
+static struct gb_loopback_async_operation *
+	gb_loopback_operation_find(u16 id)
+{
+	struct gb_loopback_async_operation *op_async;
+	bool found = false;
+	unsigned long flags;
+
+	spin_lock_irqsave(&gb_dev.lock, flags);
+	list_for_each_entry(op_async, &gb_dev.list_op_async, entry) {
+		if (op_async->operation->id == id) {
+			gb_loopback_async_operation_get(op_async);
+			found = true;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&gb_dev.lock, flags);
+
+	return found ? op_async : NULL;
+}
+
+static void gb_loopback_async_wait_all(struct gb_loopback *gb)
+{
+	wait_event(gb->wq_completion,
+		   !atomic_read(&gb->outstanding_operations));
+}
+
+static void gb_loopback_async_operation_callback(struct gb_operation *operation)
+{
+	struct gb_loopback_async_operation *op_async;
+	struct gb_loopback *gb;
+	struct timeval te;
+	bool err = false;
+
+	do_gettimeofday(&te);
+	op_async = gb_loopback_operation_find(operation->id);
+	if (!op_async)
+		return;
+
+	gb = op_async->gb;
+	mutex_lock(&gb->mutex);
+
+	if (!op_async->pending || gb_operation_result(operation)) {
+		err = true;
+	} else {
+		if (op_async->completion)
+			if (op_async->completion(op_async))
+				err = true;
+	}
+
+	if (!err) {
+		gb_loopback_push_latency_ts(gb, &op_async->ts, &te);
+		gb->elapsed_nsecs = gb_loopback_calc_latency(&op_async->ts,
+							     &te);
+	}
+
+	if (op_async->pending) {
+		if (err)
+			gb->error++;
+		gb->iteration_count++;
+		op_async->pending = false;
+		del_timer_sync(&op_async->timer);
+		gb_loopback_async_operation_put(op_async);
+		gb_loopback_calculate_stats(gb, err);
+	}
+	mutex_unlock(&gb->mutex);
+
+	dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n",
+		operation->id);
+
+	gb_loopback_async_operation_put(op_async);
+}
+
+static void gb_loopback_async_operation_work(struct work_struct *work)
+{
+	struct gb_loopback *gb;
+	struct gb_operation *operation;
+	struct gb_loopback_async_operation *op_async;
+
+	op_async = container_of(work, struct gb_loopback_async_operation, work);
+	gb = op_async->gb;
+	operation = op_async->operation;
+
+	mutex_lock(&gb->mutex);
+	if (op_async->pending) {
+		gb->requests_timedout++;
+		gb->error++;
+		gb->iteration_count++;
+		op_async->pending = false;
+		gb_loopback_async_operation_put(op_async);
+		gb_loopback_calculate_stats(gb, true);
+	}
+	mutex_unlock(&gb->mutex);
+
+	dev_dbg(&gb->connection->bundle->dev, "timeout operation %d\n",
+		operation->id);
+
+	gb_operation_cancel(operation, -ETIMEDOUT);
+	gb_loopback_async_operation_put(op_async);
+}
+
+static void gb_loopback_async_operation_timeout(unsigned long data)
+{
+	struct gb_loopback_async_operation *op_async;
+	u16 id = data;
+
+	op_async = gb_loopback_operation_find(id);
+	if (!op_async) {
+		pr_err("operation %d not found - time out ?\n", id);
+		return;
+	}
+	schedule_work(&op_async->work);
+}
+
+static int gb_loopback_async_operation(struct gb_loopback *gb, int type,
+				       void *request, int request_size,
+				       int response_size,
+				       void *completion)
+{
+	struct gb_loopback_async_operation *op_async;
+	struct gb_operation *operation;
+	int ret;
+	unsigned long flags;
+
+	op_async = kzalloc(sizeof(*op_async), GFP_KERNEL);
+	if (!op_async)
+		return -ENOMEM;
+
+	INIT_WORK(&op_async->work, gb_loopback_async_operation_work);
+	kref_init(&op_async->kref);
+
+	operation = gb_operation_create(gb->connection, type, request_size,
+					response_size, GFP_KERNEL);
+	if (!operation) {
+		kfree(op_async);
+		return -ENOMEM;
+	}
+
+	if (request_size)
+		memcpy(operation->request->payload, request, request_size);
+
+	op_async->gb = gb;
+	op_async->operation = operation;
+	op_async->completion = completion;
+
+	spin_lock_irqsave(&gb_dev.lock, flags);
+	list_add_tail(&op_async->entry, &gb_dev.list_op_async);
+	spin_unlock_irqrestore(&gb_dev.lock, flags);
+
+	do_gettimeofday(&op_async->ts);
+	op_async->pending = true;
+	atomic_inc(&gb->outstanding_operations);
+	mutex_lock(&gb->mutex);
+	ret = gb_operation_request_send(operation,
+					gb_loopback_async_operation_callback,
+					GFP_KERNEL);
+	if (ret)
+		goto error;
+
+	setup_timer(&op_async->timer, gb_loopback_async_operation_timeout,
+			(unsigned long)operation->id);
+	op_async->timer.expires = jiffies + gb->jiffy_timeout;
+	add_timer(&op_async->timer);
+
+	goto done;
+error:
+	gb_loopback_async_operation_put(op_async);
+done:
+	mutex_unlock(&gb->mutex);
+	return ret;
+}
+
+static int gb_loopback_sync_sink(struct gb_loopback *gb, u32 len)
+{
+	struct gb_loopback_transfer_request *request;
+	int retval;
+
+	request = kmalloc(len + sizeof(*request), GFP_KERNEL);
+	if (!request)
+		return -ENOMEM;
+
+	request->len = cpu_to_le32(len);
+	retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_SINK,
+					    request, len + sizeof(*request),
+					    NULL, 0);
+	kfree(request);
+	return retval;
+}
+
+static int gb_loopback_sync_transfer(struct gb_loopback *gb, u32 len)
+{
+	struct gb_loopback_transfer_request *request;
+	struct gb_loopback_transfer_response *response;
+	int retval;
+
+	gb->apbridge_latency_ts = 0;
+	gb->gbphy_latency_ts = 0;
+
+	request = kmalloc(len + sizeof(*request), GFP_KERNEL);
+	if (!request)
+		return -ENOMEM;
+	response = kmalloc(len + sizeof(*response), GFP_KERNEL);
+	if (!response) {
+		kfree(request);
+		return -ENOMEM;
+	}
+
+	memset(request->data, 0x5A, len);
+
+	request->len = cpu_to_le32(len);
+	retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_TRANSFER,
+					    request, len + sizeof(*request),
+					    response, len + sizeof(*response));
+	if (retval)
+		goto gb_error;
+
+	if (memcmp(request->data, response->data, len)) {
+		dev_err(&gb->connection->bundle->dev,
+			"Loopback Data doesn't match\n");
+		retval = -EREMOTEIO;
+	}
+	gb->apbridge_latency_ts = (u32)__le32_to_cpu(response->reserved0);
+	gb->gbphy_latency_ts = (u32)__le32_to_cpu(response->reserved1);
+
+gb_error:
+	kfree(request);
+	kfree(response);
+
+	return retval;
+}
+
+static int gb_loopback_sync_ping(struct gb_loopback *gb)
+{
+	return gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_PING,
+					  NULL, 0, NULL, 0);
+}
+
+static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len)
+{
+	struct gb_loopback_transfer_request *request;
+	int retval;
+
+	request = kmalloc(len + sizeof(*request), GFP_KERNEL);
+	if (!request)
+		return -ENOMEM;
+
+	request->len = cpu_to_le32(len);
+	retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_SINK,
+					     request, len + sizeof(*request),
+					     0, NULL);
+	kfree(request);
+	return retval;
+}
+
+static int gb_loopback_async_transfer_complete(
+				struct gb_loopback_async_operation *op_async)
+{
+	struct gb_loopback *gb;
+	struct gb_operation *operation;
+	struct gb_loopback_transfer_request *request;
+	struct gb_loopback_transfer_response *response;
+	size_t len;
+	int retval = 0;
+
+	gb = op_async->gb;
+	operation = op_async->operation;
+	request = operation->request->payload;
+	response = operation->response->payload;
+	len = le32_to_cpu(request->len);
+
+	if (memcmp(request->data, response->data, len)) {
+		dev_err(&gb->connection->bundle->dev,
+			"Loopback Data doesn't match operation id %d\n",
+			operation->id);
+		retval = -EREMOTEIO;
+	} else {
+		gb->apbridge_latency_ts =
+			(u32)__le32_to_cpu(response->reserved0);
+		gb->gbphy_latency_ts =
+			(u32)__le32_to_cpu(response->reserved1);
+	}
+
+	return retval;
+}
+
+static int gb_loopback_async_transfer(struct gb_loopback *gb, u32 len)
+{
+	struct gb_loopback_transfer_request *request;
+	int retval, response_len;
+
+	request = kmalloc(len + sizeof(*request), GFP_KERNEL);
+	if (!request)
+		return -ENOMEM;
+
+	memset(request->data, 0x5A, len);
+
+	request->len = cpu_to_le32(len);
+	response_len = sizeof(struct gb_loopback_transfer_response);
+	retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_TRANSFER,
+					     request, len + sizeof(*request),
+					     len + response_len,
+					     gb_loopback_async_transfer_complete);
+	if (retval)
+		goto gb_error;
+
+gb_error:
+	kfree(request);
+	return retval;
+}
+
+static int gb_loopback_async_ping(struct gb_loopback *gb)
+{
+	return gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_PING,
+					   NULL, 0, 0, NULL);
+}
+
+static int gb_loopback_request_handler(struct gb_operation *operation)
+{
+	struct gb_connection *connection = operation->connection;
+	struct gb_loopback_transfer_request *request;
+	struct gb_loopback_transfer_response *response;
+	struct device *dev = &connection->bundle->dev;
+	size_t len;
+
+	/* By convention, the AP initiates the version operation */
+	switch (operation->type) {
+	case GB_LOOPBACK_TYPE_PING:
+	case GB_LOOPBACK_TYPE_SINK:
+		return 0;
+	case GB_LOOPBACK_TYPE_TRANSFER:
+		if (operation->request->payload_size < sizeof(*request)) {
+			dev_err(dev, "transfer request too small (%zu < %zu)\n",
+				operation->request->payload_size,
+				sizeof(*request));
+			return -EINVAL;	/* -EMSGSIZE */
+		}
+		request = operation->request->payload;
+		len = le32_to_cpu(request->len);
+		if (len > gb_dev.size_max) {
+			dev_err(dev, "transfer request too large (%zu > %zu)\n",
+				len, gb_dev.size_max);
+			return -EINVAL;
+		}
+
+		if (!gb_operation_response_alloc(operation,
+				len + sizeof(*response), GFP_KERNEL)) {
+			dev_err(dev, "error allocating response\n");
+			return -ENOMEM;
+		}
+		response = operation->response->payload;
+		response->len = cpu_to_le32(len);
+		if (len)
+			memcpy(response->data, request->data, len);
+
+		return 0;
+	default:
+		dev_err(dev, "unsupported request: %u\n", operation->type);
+		return -EINVAL;
+	}
+}
+
+static void gb_loopback_reset_stats(struct gb_loopback *gb)
+{
+	struct gb_loopback_stats reset = {
+		.min = U32_MAX,
+	};
+
+	/* Reset per-connection stats */
+	memcpy(&gb->latency, &reset,
+	       sizeof(struct gb_loopback_stats));
+	memcpy(&gb->throughput, &reset,
+	       sizeof(struct gb_loopback_stats));
+	memcpy(&gb->requests_per_second, &reset,
+	       sizeof(struct gb_loopback_stats));
+	memcpy(&gb->apbridge_unipro_latency, &reset,
+	       sizeof(struct gb_loopback_stats));
+	memcpy(&gb->gbphy_firmware_latency, &reset,
+	       sizeof(struct gb_loopback_stats));
+
+	/* Should be initialized at least once per transaction set */
+	gb->apbridge_latency_ts = 0;
+	gb->gbphy_latency_ts = 0;
+	memset(&gb->ts, 0, sizeof(struct timeval));
+}
+
+static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val)
+{
+	if (stats->min > val)
+		stats->min = val;
+	if (stats->max < val)
+		stats->max = val;
+	stats->sum += val;
+	stats->count++;
+}
+
+static void gb_loopback_update_stats_window(struct gb_loopback_stats *stats,
+					    u64 val, u32 count)
+{
+	stats->sum += val;
+	stats->count += count;
+
+	do_div(val, count);
+	if (stats->min > val)
+		stats->min = val;
+	if (stats->max < val)
+		stats->max = val;
+}
+
+static void gb_loopback_requests_update(struct gb_loopback *gb, u32 latency)
+{
+	u64 req = gb->requests_completed * USEC_PER_SEC;
+
+	gb_loopback_update_stats_window(&gb->requests_per_second, req, latency);
+}
+
+static void gb_loopback_throughput_update(struct gb_loopback *gb, u32 latency)
+{
+	u64 aggregate_size = sizeof(struct gb_operation_msg_hdr) * 2;
+
+	switch (gb->type) {
+	case GB_LOOPBACK_TYPE_PING:
+		break;
+	case GB_LOOPBACK_TYPE_SINK:
+		aggregate_size += sizeof(struct gb_loopback_transfer_request) +
+				  gb->size;
+		break;
+	case GB_LOOPBACK_TYPE_TRANSFER:
+		aggregate_size += sizeof(struct gb_loopback_transfer_request) +
+				  sizeof(struct gb_loopback_transfer_response) +
+				  gb->size * 2;
+		break;
+	default:
+		return;
+	}
+
+	aggregate_size *= gb->requests_completed;
+	aggregate_size *= USEC_PER_SEC;
+	gb_loopback_update_stats_window(&gb->throughput, aggregate_size,
+					latency);
+}
+
+static void gb_loopback_calculate_latency_stats(struct gb_loopback *gb)
+{
+	u32 lat;
+
+	/* Express latency in terms of microseconds */
+	lat = gb_loopback_nsec_to_usec_latency(gb->elapsed_nsecs);
+
+	/* Log latency stastic */
+	gb_loopback_update_stats(&gb->latency, lat);
+
+	/* Raw latency log on a per thread basis */
+	kfifo_in(&gb->kfifo_lat, (unsigned char *)&lat, sizeof(lat));
+
+	/* Log the firmware supplied latency values */
+	gb_loopback_update_stats(&gb->apbridge_unipro_latency,
+				 gb->apbridge_latency_ts);
+	gb_loopback_update_stats(&gb->gbphy_firmware_latency,
+				 gb->gbphy_latency_ts);
+}
+
+static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error)
+{
+	u64 nlat;
+	u32 lat;
+	struct timeval te;
+
+	if (!error) {
+		gb->requests_completed++;
+		gb_loopback_calculate_latency_stats(gb);
+	}
+
+	do_gettimeofday(&te);
+	nlat = gb_loopback_calc_latency(&gb->ts, &te);
+	if (nlat >= NSEC_PER_SEC || gb->iteration_count == gb->iteration_max) {
+		lat = gb_loopback_nsec_to_usec_latency(nlat);
+
+		gb_loopback_throughput_update(gb, lat);
+		gb_loopback_requests_update(gb, lat);
+
+		if (gb->iteration_count != gb->iteration_max) {
+			gb->ts = te;
+			gb->requests_completed = 0;
+		}
+	}
+}
+
+static void gb_loopback_async_wait_to_send(struct gb_loopback *gb)
+{
+	if (!(gb->async && gb->outstanding_operations_max))
+		return;
+	wait_event_interruptible(gb->wq_completion,
+				 (atomic_read(&gb->outstanding_operations) <
+				  gb->outstanding_operations_max) ||
+				  kthread_should_stop());
+}
+
+static int gb_loopback_fn(void *data)
+{
+	int error = 0;
+	int us_wait = 0;
+	int type;
+	int ret;
+	u32 size;
+
+	struct gb_loopback *gb = data;
+	struct gb_bundle *bundle = gb->connection->bundle;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	while (1) {
+		if (!gb->type) {
+			gb_pm_runtime_put_autosuspend(bundle);
+			wait_event_interruptible(gb->wq, gb->type ||
+						 kthread_should_stop());
+			ret = gb_pm_runtime_get_sync(bundle);
+			if (ret)
+				return ret;
+		}
+
+		if (kthread_should_stop())
+			break;
+
+		/* Limit the maximum number of in-flight async operations */
+		gb_loopback_async_wait_to_send(gb);
+		if (kthread_should_stop())
+			break;
+
+		mutex_lock(&gb->mutex);
+
+		/* Optionally terminate */
+		if (gb->send_count == gb->iteration_max) {
+			if (gb->iteration_count == gb->iteration_max) {
+				gb->type = 0;
+				gb->send_count = 0;
+				sysfs_notify(&gb->dev->kobj,  NULL,
+						"iteration_count");
+			}
+			mutex_unlock(&gb->mutex);
+			continue;
+		}
+		size = gb->size;
+		us_wait = gb->us_wait;
+		type = gb->type;
+		if (gb->ts.tv_usec == 0 && gb->ts.tv_sec == 0)
+			do_gettimeofday(&gb->ts);
+		mutex_unlock(&gb->mutex);
+
+		/* Else operations to perform */
+		if (gb->async) {
+			if (type == GB_LOOPBACK_TYPE_PING) {
+				error = gb_loopback_async_ping(gb);
+			} else if (type == GB_LOOPBACK_TYPE_TRANSFER) {
+				error = gb_loopback_async_transfer(gb, size);
+			} else if (type == GB_LOOPBACK_TYPE_SINK) {
+				error = gb_loopback_async_sink(gb, size);
+			}
+
+			if (error)
+				gb->error++;
+		} else {
+			/* We are effectively single threaded here */
+			if (type == GB_LOOPBACK_TYPE_PING)
+				error = gb_loopback_sync_ping(gb);
+			else if (type == GB_LOOPBACK_TYPE_TRANSFER)
+				error = gb_loopback_sync_transfer(gb, size);
+			else if (type == GB_LOOPBACK_TYPE_SINK)
+				error = gb_loopback_sync_sink(gb, size);
+
+			if (error)
+				gb->error++;
+			gb->iteration_count++;
+			gb_loopback_calculate_stats(gb, !!error);
+		}
+		gb->send_count++;
+		if (us_wait)
+			udelay(us_wait);
+	}
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return 0;
+}
+
+static int gb_loopback_dbgfs_latency_show_common(struct seq_file *s,
+						 struct kfifo *kfifo,
+						 struct mutex *mutex)
+{
+	u32 latency;
+	int retval;
+
+	if (kfifo_len(kfifo) == 0) {
+		retval = -EAGAIN;
+		goto done;
+	}
+
+	mutex_lock(mutex);
+	retval = kfifo_out(kfifo, &latency, sizeof(latency));
+	if (retval > 0) {
+		seq_printf(s, "%u", latency);
+		retval = 0;
+	}
+	mutex_unlock(mutex);
+done:
+	return retval;
+}
+
+static int gb_loopback_dbgfs_latency_show(struct seq_file *s, void *unused)
+{
+	struct gb_loopback *gb = s->private;
+
+	return gb_loopback_dbgfs_latency_show_common(s, &gb->kfifo_lat,
+						     &gb->mutex);
+}
+
+static int gb_loopback_latency_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, gb_loopback_dbgfs_latency_show,
+			   inode->i_private);
+}
+
+static const struct file_operations gb_loopback_debugfs_latency_ops = {
+	.open		= gb_loopback_latency_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int gb_loopback_bus_id_compare(void *priv, struct list_head *lha,
+				      struct list_head *lhb)
+{
+	struct gb_loopback *a = list_entry(lha, struct gb_loopback, entry);
+	struct gb_loopback *b = list_entry(lhb, struct gb_loopback, entry);
+	struct gb_connection *ca = a->connection;
+	struct gb_connection *cb = b->connection;
+
+	if (ca->bundle->intf->interface_id < cb->bundle->intf->interface_id)
+		return -1;
+	if (cb->bundle->intf->interface_id < ca->bundle->intf->interface_id)
+		return 1;
+	if (ca->bundle->id < cb->bundle->id)
+		return -1;
+	if (cb->bundle->id < ca->bundle->id)
+		return 1;
+	if (ca->intf_cport_id < cb->intf_cport_id)
+		return -1;
+	else if (cb->intf_cport_id < ca->intf_cport_id)
+		return 1;
+
+	return 0;
+}
+
+static void gb_loopback_insert_id(struct gb_loopback *gb)
+{
+	struct gb_loopback *gb_list;
+	u32 new_lbid = 0;
+
+	/* perform an insertion sort */
+	list_add_tail(&gb->entry, &gb_dev.list);
+	list_sort(NULL, &gb_dev.list, gb_loopback_bus_id_compare);
+	list_for_each_entry(gb_list, &gb_dev.list, entry) {
+		gb_list->lbid = 1 << new_lbid;
+		new_lbid++;
+	}
+}
+
+#define DEBUGFS_NAMELEN 32
+
+static int gb_loopback_probe(struct gb_bundle *bundle,
+			     const struct greybus_bundle_id *id)
+{
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_connection *connection;
+	struct gb_loopback *gb;
+	struct device *dev;
+	int retval;
+	char name[DEBUGFS_NAMELEN];
+	unsigned long flags;
+
+	if (bundle->num_cports != 1)
+		return -ENODEV;
+
+	cport_desc = &bundle->cport_desc[0];
+	if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOOPBACK)
+		return -ENODEV;
+
+	gb = kzalloc(sizeof(*gb), GFP_KERNEL);
+	if (!gb)
+		return -ENOMEM;
+
+	connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
+					  gb_loopback_request_handler);
+	if (IS_ERR(connection)) {
+		retval = PTR_ERR(connection);
+		goto out_kzalloc;
+	}
+
+	gb->connection = connection;
+	greybus_set_drvdata(bundle, gb);
+
+	init_waitqueue_head(&gb->wq);
+	init_waitqueue_head(&gb->wq_completion);
+	atomic_set(&gb->outstanding_operations, 0);
+	gb_loopback_reset_stats(gb);
+
+	/* Reported values to user-space for min/max timeouts */
+	gb->timeout_min = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MIN);
+	gb->timeout_max = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MAX);
+
+	if (!gb_dev.count) {
+		/* Calculate maximum payload */
+		gb_dev.size_max = gb_operation_get_payload_size_max(connection);
+		if (gb_dev.size_max <=
+			sizeof(struct gb_loopback_transfer_request)) {
+			retval = -EINVAL;
+			goto out_connection_destroy;
+		}
+		gb_dev.size_max -= sizeof(struct gb_loopback_transfer_request);
+	}
+
+	/* Create per-connection sysfs and debugfs data-points */
+	snprintf(name, sizeof(name), "raw_latency_%s",
+		 dev_name(&connection->bundle->dev));
+	gb->file = debugfs_create_file(name, S_IFREG | S_IRUGO, gb_dev.root, gb,
+				       &gb_loopback_debugfs_latency_ops);
+
+	gb->id = ida_simple_get(&loopback_ida, 0, 0, GFP_KERNEL);
+	if (gb->id < 0) {
+		retval = gb->id;
+		goto out_debugfs_remove;
+	}
+
+	retval = gb_connection_enable(connection);
+	if (retval)
+		goto out_ida_remove;
+
+	dev = device_create_with_groups(&loopback_class,
+					&connection->bundle->dev,
+					MKDEV(0, 0), gb, loopback_groups,
+					"gb_loopback%d", gb->id);
+	if (IS_ERR(dev)) {
+		retval = PTR_ERR(dev);
+		goto out_connection_disable;
+	}
+	gb->dev = dev;
+
+	/* Allocate kfifo */
+	if (kfifo_alloc(&gb->kfifo_lat, kfifo_depth * sizeof(u32),
+			  GFP_KERNEL)) {
+		retval = -ENOMEM;
+		goto out_conn;
+	}
+	if (kfifo_alloc(&gb->kfifo_ts, kfifo_depth * sizeof(struct timeval) * 2,
+			  GFP_KERNEL)) {
+		retval = -ENOMEM;
+		goto out_kfifo0;
+	}
+
+	/* Fork worker thread */
+	mutex_init(&gb->mutex);
+	gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback");
+	if (IS_ERR(gb->task)) {
+		retval = PTR_ERR(gb->task);
+		goto out_kfifo1;
+	}
+
+	spin_lock_irqsave(&gb_dev.lock, flags);
+	gb_loopback_insert_id(gb);
+	gb_dev.count++;
+	spin_unlock_irqrestore(&gb_dev.lock, flags);
+
+	gb_connection_latency_tag_enable(connection);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return 0;
+
+out_kfifo1:
+	kfifo_free(&gb->kfifo_ts);
+out_kfifo0:
+	kfifo_free(&gb->kfifo_lat);
+out_conn:
+	device_unregister(dev);
+out_connection_disable:
+	gb_connection_disable(connection);
+out_ida_remove:
+	ida_simple_remove(&loopback_ida, gb->id);
+out_debugfs_remove:
+	debugfs_remove(gb->file);
+out_connection_destroy:
+	gb_connection_destroy(connection);
+out_kzalloc:
+	kfree(gb);
+
+	return retval;
+}
+
+static void gb_loopback_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_loopback *gb = greybus_get_drvdata(bundle);
+	unsigned long flags;
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		gb_pm_runtime_get_noresume(bundle);
+
+	gb_connection_disable(gb->connection);
+
+	if (!IS_ERR_OR_NULL(gb->task))
+		kthread_stop(gb->task);
+
+	kfifo_free(&gb->kfifo_lat);
+	kfifo_free(&gb->kfifo_ts);
+	gb_connection_latency_tag_disable(gb->connection);
+	debugfs_remove(gb->file);
+
+	/*
+	 * FIXME: gb_loopback_async_wait_all() is redundant now, as connection
+	 * is disabled at the beginning and so we can't have any more
+	 * incoming/outgoing requests.
+	 */
+	gb_loopback_async_wait_all(gb);
+
+	spin_lock_irqsave(&gb_dev.lock, flags);
+	gb_dev.count--;
+	list_del(&gb->entry);
+	spin_unlock_irqrestore(&gb_dev.lock, flags);
+
+	device_unregister(gb->dev);
+	ida_simple_remove(&loopback_ida, gb->id);
+
+	gb_connection_destroy(gb->connection);
+	kfree(gb);
+}
+
+static const struct greybus_bundle_id gb_loopback_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOOPBACK) },
+	{ }
+};
+MODULE_DEVICE_TABLE(greybus, gb_loopback_id_table);
+
+static struct greybus_driver gb_loopback_driver = {
+	.name		= "loopback",
+	.probe		= gb_loopback_probe,
+	.disconnect	= gb_loopback_disconnect,
+	.id_table	= gb_loopback_id_table,
+};
+
+static int loopback_init(void)
+{
+	int retval;
+
+	INIT_LIST_HEAD(&gb_dev.list);
+	INIT_LIST_HEAD(&gb_dev.list_op_async);
+	spin_lock_init(&gb_dev.lock);
+	gb_dev.root = debugfs_create_dir("gb_loopback", NULL);
+
+	retval = class_register(&loopback_class);
+	if (retval)
+		goto err;
+
+	retval = greybus_register(&gb_loopback_driver);
+	if (retval)
+		goto err_unregister;
+
+	return 0;
+
+err_unregister:
+	class_unregister(&loopback_class);
+err:
+	debugfs_remove_recursive(gb_dev.root);
+	return retval;
+}
+module_init(loopback_init);
+
+static void __exit loopback_exit(void)
+{
+	debugfs_remove_recursive(gb_dev.root);
+	greybus_deregister(&gb_loopback_driver);
+	class_unregister(&loopback_class);
+	ida_destroy(&loopback_ida);
+}
+module_exit(loopback_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/manifest.c b/drivers/staging/greybus/manifest.c
new file mode 100644
index 000000000000..7b903770a684
--- /dev/null
+++ b/drivers/staging/greybus/manifest.c
@@ -0,0 +1,535 @@
+/*
+ * Greybus manifest parsing
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include "greybus.h"
+
+static const char *get_descriptor_type_string(u8 type)
+{
+	switch (type) {
+	case GREYBUS_TYPE_INVALID:
+		return "invalid";
+	case GREYBUS_TYPE_STRING:
+		return "string";
+	case GREYBUS_TYPE_INTERFACE:
+		return "interface";
+	case GREYBUS_TYPE_CPORT:
+		return "cport";
+	case GREYBUS_TYPE_BUNDLE:
+		return "bundle";
+	default:
+		WARN_ON(1);
+		return "unknown";
+	}
+}
+
+/*
+ * We scan the manifest once to identify where all the descriptors
+ * are.  The result is a list of these manifest_desc structures.  We
+ * then pick through them for what we're looking for (starting with
+ * the interface descriptor).  As each is processed we remove it from
+ * the list.  When we're done the list should (probably) be empty.
+ */
+struct manifest_desc {
+	struct list_head		links;
+
+	size_t				size;
+	void				*data;
+	enum greybus_descriptor_type	type;
+};
+
+static void release_manifest_descriptor(struct manifest_desc *descriptor)
+{
+	list_del(&descriptor->links);
+	kfree(descriptor);
+}
+
+static void release_manifest_descriptors(struct gb_interface *intf)
+{
+	struct manifest_desc *descriptor;
+	struct manifest_desc *next;
+
+	list_for_each_entry_safe(descriptor, next, &intf->manifest_descs, links)
+		release_manifest_descriptor(descriptor);
+}
+
+static void release_cport_descriptors(struct list_head *head, u8 bundle_id)
+{
+	struct manifest_desc *desc, *tmp;
+	struct greybus_descriptor_cport *desc_cport;
+
+	list_for_each_entry_safe(desc, tmp, head, links) {
+		desc_cport = desc->data;
+
+		if (desc->type != GREYBUS_TYPE_CPORT)
+			continue;
+
+		if (desc_cport->bundle == bundle_id)
+			release_manifest_descriptor(desc);
+	}
+}
+
+static struct manifest_desc *get_next_bundle_desc(struct gb_interface *intf)
+{
+	struct manifest_desc *descriptor;
+	struct manifest_desc *next;
+
+	list_for_each_entry_safe(descriptor, next, &intf->manifest_descs, links)
+		if (descriptor->type == GREYBUS_TYPE_BUNDLE)
+			return descriptor;
+
+	return NULL;
+}
+
+/*
+ * Validate the given descriptor.  Its reported size must fit within
+ * the number of bytes remaining, and it must have a recognized
+ * type.  Check that the reported size is at least as big as what
+ * we expect to see.  (It could be bigger, perhaps for a new version
+ * of the format.)
+ *
+ * Returns the (non-zero) number of bytes consumed by the descriptor,
+ * or a negative errno.
+ */
+static int identify_descriptor(struct gb_interface *intf,
+			       struct greybus_descriptor *desc, size_t size)
+{
+	struct greybus_descriptor_header *desc_header = &desc->header;
+	struct manifest_desc *descriptor;
+	size_t desc_size;
+	size_t expected_size;
+
+	if (size < sizeof(*desc_header)) {
+		dev_err(&intf->dev, "manifest too small (%zu < %zu)\n",
+				size, sizeof(*desc_header));
+		return -EINVAL;		/* Must at least have header */
+	}
+
+	desc_size = le16_to_cpu(desc_header->size);
+	if (desc_size > size) {
+		dev_err(&intf->dev, "descriptor too big (%zu > %zu)\n",
+				desc_size, size);
+		return -EINVAL;
+	}
+
+	/* Descriptor needs to at least have a header */
+	expected_size = sizeof(*desc_header);
+
+	switch (desc_header->type) {
+	case GREYBUS_TYPE_STRING:
+		expected_size += sizeof(struct greybus_descriptor_string);
+		expected_size += desc->string.length;
+
+		/* String descriptors are padded to 4 byte boundaries */
+		expected_size = ALIGN(expected_size, 4);
+		break;
+	case GREYBUS_TYPE_INTERFACE:
+		expected_size += sizeof(struct greybus_descriptor_interface);
+		break;
+	case GREYBUS_TYPE_BUNDLE:
+		expected_size += sizeof(struct greybus_descriptor_bundle);
+		break;
+	case GREYBUS_TYPE_CPORT:
+		expected_size += sizeof(struct greybus_descriptor_cport);
+		break;
+	case GREYBUS_TYPE_INVALID:
+	default:
+		dev_err(&intf->dev, "invalid descriptor type (%u)\n",
+				desc_header->type);
+		return -EINVAL;
+	}
+
+	if (desc_size < expected_size) {
+		dev_err(&intf->dev, "%s descriptor too small (%zu < %zu)\n",
+				get_descriptor_type_string(desc_header->type),
+				desc_size, expected_size);
+		return -EINVAL;
+	}
+
+	/* Descriptor bigger than what we expect */
+	if (desc_size > expected_size) {
+		dev_warn(&intf->dev, "%s descriptor size mismatch (want %zu got %zu)\n",
+				get_descriptor_type_string(desc_header->type),
+				expected_size, desc_size);
+	}
+
+	descriptor = kzalloc(sizeof(*descriptor), GFP_KERNEL);
+	if (!descriptor)
+		return -ENOMEM;
+
+	descriptor->size = desc_size;
+	descriptor->data = (char *)desc + sizeof(*desc_header);
+	descriptor->type = desc_header->type;
+	list_add_tail(&descriptor->links, &intf->manifest_descs);
+
+	/* desc_size is positive and is known to fit in a signed int */
+
+	return desc_size;
+}
+
+/*
+ * Find the string descriptor having the given id, validate it, and
+ * allocate a duplicate copy of it.  The duplicate has an extra byte
+ * which guarantees the returned string is NUL-terminated.
+ *
+ * String index 0 is valid (it represents "no string"), and for
+ * that a null pointer is returned.
+ *
+ * Otherwise returns a pointer to a newly-allocated copy of the
+ * descriptor string, or an error-coded pointer on failure.
+ */
+static char *gb_string_get(struct gb_interface *intf, u8 string_id)
+{
+	struct greybus_descriptor_string *desc_string;
+	struct manifest_desc *descriptor;
+	bool found = false;
+	char *string;
+
+	/* A zero string id means no string (but no error) */
+	if (!string_id)
+		return NULL;
+
+	list_for_each_entry(descriptor, &intf->manifest_descs, links) {
+		if (descriptor->type != GREYBUS_TYPE_STRING)
+			continue;
+
+		desc_string = descriptor->data;
+		if (desc_string->id == string_id) {
+			found = true;
+			break;
+		}
+	}
+	if (!found)
+		return ERR_PTR(-ENOENT);
+
+	/* Allocate an extra byte so we can guarantee it's NUL-terminated */
+	string = kmemdup(&desc_string->string, desc_string->length + 1,
+				GFP_KERNEL);
+	if (!string)
+		return ERR_PTR(-ENOMEM);
+	string[desc_string->length] = '\0';
+
+	/* Ok we've used this string, so we're done with it */
+	release_manifest_descriptor(descriptor);
+
+	return string;
+}
+
+/*
+ * Find cport descriptors in the manifest associated with the given
+ * bundle, and set up data structures for the functions that use
+ * them.  Returns the number of cports set up for the bundle, or 0
+ * if there is an error.
+ */
+static u32 gb_manifest_parse_cports(struct gb_bundle *bundle)
+{
+	struct gb_interface *intf = bundle->intf;
+	struct greybus_descriptor_cport *desc_cport;
+	struct manifest_desc *desc, *next, *tmp;
+	LIST_HEAD(list);
+	u8 bundle_id = bundle->id;
+	u16 cport_id;
+	u32 count = 0;
+	int i;
+
+	/* Set up all cport descriptors associated with this bundle */
+	list_for_each_entry_safe(desc, next, &intf->manifest_descs, links) {
+		if (desc->type != GREYBUS_TYPE_CPORT)
+			continue;
+
+		desc_cport = desc->data;
+		if (desc_cport->bundle != bundle_id)
+			continue;
+
+		cport_id = le16_to_cpu(desc_cport->id);
+		if (cport_id > CPORT_ID_MAX)
+			goto exit;
+
+		/* Nothing else should have its cport_id as control cport id */
+		if (cport_id == GB_CONTROL_CPORT_ID) {
+			dev_err(&bundle->dev, "invalid cport id found (%02u)\n",
+				cport_id);
+			goto exit;
+		}
+
+		/*
+		 * Found one, move it to our temporary list after checking for
+		 * duplicates.
+		 */
+		list_for_each_entry(tmp, &list, links) {
+			desc_cport = tmp->data;
+			if (cport_id == le16_to_cpu(desc_cport->id)) {
+				dev_err(&bundle->dev,
+						"duplicate CPort %u found\n",
+						cport_id);
+				goto exit;
+			}
+		}
+		list_move_tail(&desc->links, &list);
+		count++;
+	}
+
+	if (!count)
+		return 0;
+
+	bundle->cport_desc = kcalloc(count, sizeof(*bundle->cport_desc),
+					GFP_KERNEL);
+	if (!bundle->cport_desc)
+		goto exit;
+
+	bundle->num_cports = count;
+
+	i = 0;
+	list_for_each_entry_safe(desc, next, &list, links) {
+		desc_cport = desc->data;
+		memcpy(&bundle->cport_desc[i++], desc_cport,
+				sizeof(*desc_cport));
+
+		/* Release the cport descriptor */
+		release_manifest_descriptor(desc);
+	}
+
+	return count;
+exit:
+	release_cport_descriptors(&list, bundle_id);
+	/*
+	 * Free all cports for this bundle to avoid 'excess descriptors'
+	 * warnings.
+	 */
+	release_cport_descriptors(&intf->manifest_descs, bundle_id);
+
+	return 0;	/* Error; count should also be 0 */
+}
+
+/*
+ * Find bundle descriptors in the manifest and set up their data
+ * structures.  Returns the number of bundles set up for the
+ * given interface.
+ */
+static u32 gb_manifest_parse_bundles(struct gb_interface *intf)
+{
+	struct manifest_desc *desc;
+	struct gb_bundle *bundle;
+	struct gb_bundle *bundle_next;
+	u32 count = 0;
+	u8 bundle_id;
+	u8 class;
+
+	while ((desc = get_next_bundle_desc(intf))) {
+		struct greybus_descriptor_bundle *desc_bundle;
+
+		/* Found one.  Set up its bundle structure*/
+		desc_bundle = desc->data;
+		bundle_id = desc_bundle->id;
+		class = desc_bundle->class;
+
+		/* Done with this bundle descriptor */
+		release_manifest_descriptor(desc);
+
+		/* Ignore any legacy control bundles */
+		if (bundle_id == GB_CONTROL_BUNDLE_ID) {
+			dev_dbg(&intf->dev, "%s - ignoring control bundle\n",
+					__func__);
+			release_cport_descriptors(&intf->manifest_descs,
+								bundle_id);
+			continue;
+		}
+
+		/* Nothing else should have its class set to control class */
+		if (class == GREYBUS_CLASS_CONTROL) {
+			dev_err(&intf->dev,
+				"bundle %u cannot use control class\n",
+				bundle_id);
+			goto cleanup;
+		}
+
+		bundle = gb_bundle_create(intf, bundle_id, class);
+		if (!bundle)
+			goto cleanup;
+
+		/*
+		 * Now go set up this bundle's functions and cports.
+		 *
+		 * A 'bundle' represents a device in greybus. It may require
+		 * multiple cports for its functioning. If we fail to setup any
+		 * cport of a bundle, we better reject the complete bundle as
+		 * the device may not be able to function properly then.
+		 *
+		 * But, failing to setup a cport of bundle X doesn't mean that
+		 * the device corresponding to bundle Y will not work properly.
+		 * Bundles should be treated as separate independent devices.
+		 *
+		 * While parsing manifest for an interface, treat bundles as
+		 * separate entities and don't reject entire interface and its
+		 * bundles on failing to initialize a cport. But make sure the
+		 * bundle which needs the cport, gets destroyed properly.
+		 */
+		if (!gb_manifest_parse_cports(bundle)) {
+			gb_bundle_destroy(bundle);
+			continue;
+		}
+
+		count++;
+	}
+
+	return count;
+cleanup:
+	/* An error occurred; undo any changes we've made */
+	list_for_each_entry_safe(bundle, bundle_next, &intf->bundles, links) {
+		gb_bundle_destroy(bundle);
+		count--;
+	}
+	return 0;	/* Error; count should also be 0 */
+}
+
+static bool gb_manifest_parse_interface(struct gb_interface *intf,
+					struct manifest_desc *interface_desc)
+{
+	struct greybus_descriptor_interface *desc_intf = interface_desc->data;
+	struct gb_control *control = intf->control;
+	char *str;
+
+	/* Handle the strings first--they can fail */
+	str = gb_string_get(intf, desc_intf->vendor_stringid);
+	if (IS_ERR(str))
+		return false;
+	control->vendor_string = str;
+
+	str = gb_string_get(intf, desc_intf->product_stringid);
+	if (IS_ERR(str))
+		goto out_free_vendor_string;
+	control->product_string = str;
+
+	/* Assign feature flags communicated via manifest */
+	intf->features = desc_intf->features;
+
+	/* Release the interface descriptor, now that we're done with it */
+	release_manifest_descriptor(interface_desc);
+
+	/* An interface must have at least one bundle descriptor */
+	if (!gb_manifest_parse_bundles(intf)) {
+		dev_err(&intf->dev, "manifest bundle descriptors not valid\n");
+		goto out_err;
+	}
+
+	return true;
+out_err:
+	kfree(control->product_string);
+	control->product_string = NULL;
+out_free_vendor_string:
+	kfree(control->vendor_string);
+	control->vendor_string = NULL;
+
+	return false;
+}
+
+/*
+ * Parse a buffer containing an interface manifest.
+ *
+ * If we find anything wrong with the content/format of the buffer
+ * we reject it.
+ *
+ * The first requirement is that the manifest's version is
+ * one we can parse.
+ *
+ * We make an initial pass through the buffer and identify all of
+ * the descriptors it contains, keeping track for each its type
+ * and the location size of its data in the buffer.
+ *
+ * Next we scan the descriptors, looking for an interface descriptor;
+ * there must be exactly one of those.  When found, we record the
+ * information it contains, and then remove that descriptor (and any
+ * string descriptors it refers to) from further consideration.
+ *
+ * After that we look for the interface's bundles--there must be at
+ * least one of those.
+ *
+ * Returns true if parsing was successful, false otherwise.
+ */
+bool gb_manifest_parse(struct gb_interface *intf, void *data, size_t size)
+{
+	struct greybus_manifest *manifest;
+	struct greybus_manifest_header *header;
+	struct greybus_descriptor *desc;
+	struct manifest_desc *descriptor;
+	struct manifest_desc *interface_desc = NULL;
+	u16 manifest_size;
+	u32 found = 0;
+	bool result;
+
+	/* Manifest descriptor list should be empty here */
+	if (WARN_ON(!list_empty(&intf->manifest_descs)))
+		return false;
+
+	/* we have to have at _least_ the manifest header */
+	if (size < sizeof(*header)) {
+		dev_err(&intf->dev, "short manifest (%zu < %zu)\n",
+				size, sizeof(*header));
+		return false;
+	}
+
+	/* Make sure the size is right */
+	manifest = data;
+	header = &manifest->header;
+	manifest_size = le16_to_cpu(header->size);
+	if (manifest_size != size) {
+		dev_err(&intf->dev, "manifest size mismatch (%zu != %u)\n",
+				size, manifest_size);
+		return false;
+	}
+
+	/* Validate major/minor number */
+	if (header->version_major > GREYBUS_VERSION_MAJOR) {
+		dev_err(&intf->dev, "manifest version too new (%u.%u > %u.%u)\n",
+				header->version_major, header->version_minor,
+				GREYBUS_VERSION_MAJOR, GREYBUS_VERSION_MINOR);
+		return false;
+	}
+
+	/* OK, find all the descriptors */
+	desc = manifest->descriptors;
+	size -= sizeof(*header);
+	while (size) {
+		int desc_size;
+
+		desc_size = identify_descriptor(intf, desc, size);
+		if (desc_size < 0) {
+			result = false;
+			goto out;
+		}
+		desc = (struct greybus_descriptor *)((char *)desc + desc_size);
+		size -= desc_size;
+	}
+
+	/* There must be a single interface descriptor */
+	list_for_each_entry(descriptor, &intf->manifest_descs, links) {
+		if (descriptor->type == GREYBUS_TYPE_INTERFACE)
+			if (!found++)
+				interface_desc = descriptor;
+	}
+	if (found != 1) {
+		dev_err(&intf->dev, "manifest must have 1 interface descriptor (%u found)\n",
+				found);
+		result = false;
+		goto out;
+	}
+
+	/* Parse the manifest, starting with the interface descriptor */
+	result = gb_manifest_parse_interface(intf, interface_desc);
+
+	/*
+	 * We really should have no remaining descriptors, but we
+	 * don't know what newer format manifests might leave.
+	 */
+	if (result && !list_empty(&intf->manifest_descs))
+		dev_info(&intf->dev, "excess descriptors in interface manifest\n");
+out:
+	release_manifest_descriptors(intf);
+
+	return result;
+}
diff --git a/drivers/staging/greybus/manifest.h b/drivers/staging/greybus/manifest.h
new file mode 100644
index 000000000000..d96428407cd7
--- /dev/null
+++ b/drivers/staging/greybus/manifest.h
@@ -0,0 +1,16 @@
+/*
+ * Greybus manifest parsing
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __MANIFEST_H
+#define __MANIFEST_H
+
+struct gb_interface;
+bool gb_manifest_parse(struct gb_interface *intf, void *data, size_t size);
+
+#endif /* __MANIFEST_H */
diff --git a/drivers/staging/greybus/module.c b/drivers/staging/greybus/module.c
new file mode 100644
index 000000000000..69f67ddbd4a3
--- /dev/null
+++ b/drivers/staging/greybus/module.c
@@ -0,0 +1,238 @@
+/*
+ * Greybus Module code
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include "greybus.h"
+#include "greybus_trace.h"
+
+
+static ssize_t eject_store(struct device *dev,
+				struct device_attribute *attr,
+				const char *buf, size_t len)
+{
+	struct gb_module *module = to_gb_module(dev);
+	struct gb_interface *intf;
+	size_t i;
+	long val;
+	int ret;
+
+	ret = kstrtol(buf, 0, &val);
+	if (ret)
+		return ret;
+
+	if (!val)
+		return len;
+
+	for (i = 0; i < module->num_interfaces; ++i) {
+		intf = module->interfaces[i];
+
+		mutex_lock(&intf->mutex);
+		/* Set flag to prevent concurrent activation. */
+		intf->ejected = true;
+		gb_interface_disable(intf);
+		gb_interface_deactivate(intf);
+		mutex_unlock(&intf->mutex);
+	}
+
+	/* Tell the SVC to eject the primary interface. */
+	ret = gb_svc_intf_eject(module->hd->svc, module->module_id);
+	if (ret)
+		return ret;
+
+	return len;
+}
+static DEVICE_ATTR_WO(eject);
+
+static ssize_t module_id_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct gb_module *module = to_gb_module(dev);
+
+	return sprintf(buf, "%u\n", module->module_id);
+}
+static DEVICE_ATTR_RO(module_id);
+
+static ssize_t num_interfaces_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct gb_module *module = to_gb_module(dev);
+
+	return sprintf(buf, "%zu\n", module->num_interfaces);
+}
+static DEVICE_ATTR_RO(num_interfaces);
+
+static struct attribute *module_attrs[] = {
+	&dev_attr_eject.attr,
+	&dev_attr_module_id.attr,
+	&dev_attr_num_interfaces.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(module);
+
+static void gb_module_release(struct device *dev)
+{
+	struct gb_module *module = to_gb_module(dev);
+
+	trace_gb_module_release(module);
+
+	kfree(module);
+}
+
+struct device_type greybus_module_type = {
+	.name		= "greybus_module",
+	.release	= gb_module_release,
+};
+
+struct gb_module *gb_module_create(struct gb_host_device *hd, u8 module_id,
+					size_t num_interfaces)
+{
+	struct gb_interface *intf;
+	struct gb_module *module;
+	int i;
+
+	module = kzalloc(sizeof(*module) + num_interfaces * sizeof(intf),
+				GFP_KERNEL);
+	if (!module)
+		return NULL;
+
+	module->hd = hd;
+	module->module_id = module_id;
+	module->num_interfaces = num_interfaces;
+
+	module->dev.parent = &hd->dev;
+	module->dev.bus = &greybus_bus_type;
+	module->dev.type = &greybus_module_type;
+	module->dev.groups = module_groups;
+	module->dev.dma_mask = hd->dev.dma_mask;
+	device_initialize(&module->dev);
+	dev_set_name(&module->dev, "%d-%u", hd->bus_id, module_id);
+
+	trace_gb_module_create(module);
+
+	for (i = 0; i < num_interfaces; ++i) {
+		intf = gb_interface_create(module, module_id + i);
+		if (!intf) {
+			dev_err(&module->dev, "failed to create interface %u\n",
+					module_id + i);
+			goto err_put_interfaces;
+		}
+		module->interfaces[i] = intf;
+	}
+
+	return module;
+
+err_put_interfaces:
+	for (--i; i > 0; --i)
+		gb_interface_put(module->interfaces[i]);
+
+	put_device(&module->dev);
+
+	return NULL;
+}
+
+/*
+ * Register and enable an interface after first attempting to activate it.
+ */
+static void gb_module_register_interface(struct gb_interface *intf)
+{
+	struct gb_module *module = intf->module;
+	u8 intf_id = intf->interface_id;
+	int ret;
+
+	mutex_lock(&intf->mutex);
+
+	ret = gb_interface_activate(intf);
+	if (ret) {
+		if (intf->type != GB_INTERFACE_TYPE_DUMMY) {
+			dev_err(&module->dev,
+					"failed to activate interface %u: %d\n",
+					intf_id, ret);
+		}
+
+		gb_interface_add(intf);
+		goto err_unlock;
+	}
+
+	ret = gb_interface_add(intf);
+	if (ret)
+		goto err_interface_deactivate;
+
+	ret = gb_interface_enable(intf);
+	if (ret) {
+		dev_err(&module->dev, "failed to enable interface %u: %d\n",
+				intf_id, ret);
+		goto err_interface_deactivate;
+	}
+
+	mutex_unlock(&intf->mutex);
+
+	return;
+
+err_interface_deactivate:
+	gb_interface_deactivate(intf);
+err_unlock:
+	mutex_unlock(&intf->mutex);
+}
+
+static void gb_module_deregister_interface(struct gb_interface *intf)
+{
+	/* Mark as disconnected to prevent I/O during disable. */
+	if (intf->module->disconnected)
+		intf->disconnected = true;
+
+	mutex_lock(&intf->mutex);
+	intf->removed = true;
+	gb_interface_disable(intf);
+	gb_interface_deactivate(intf);
+	mutex_unlock(&intf->mutex);
+
+	gb_interface_del(intf);
+}
+
+/* Register a module and its interfaces. */
+int gb_module_add(struct gb_module *module)
+{
+	size_t i;
+	int ret;
+
+	ret = device_add(&module->dev);
+	if (ret) {
+		dev_err(&module->dev, "failed to register module: %d\n", ret);
+		return ret;
+	}
+
+	trace_gb_module_add(module);
+
+	for (i = 0; i < module->num_interfaces; ++i)
+		gb_module_register_interface(module->interfaces[i]);
+
+	return 0;
+}
+
+/* Deregister a module and its interfaces. */
+void gb_module_del(struct gb_module *module)
+{
+	size_t i;
+
+	for (i = 0; i < module->num_interfaces; ++i)
+		gb_module_deregister_interface(module->interfaces[i]);
+
+	trace_gb_module_del(module);
+
+	device_del(&module->dev);
+}
+
+void gb_module_put(struct gb_module *module)
+{
+	size_t i;
+
+	for (i = 0; i < module->num_interfaces; ++i)
+		gb_interface_put(module->interfaces[i]);
+
+	put_device(&module->dev);
+}
diff --git a/drivers/staging/greybus/module.h b/drivers/staging/greybus/module.h
new file mode 100644
index 000000000000..88a97ce04243
--- /dev/null
+++ b/drivers/staging/greybus/module.h
@@ -0,0 +1,34 @@
+/*
+ * Greybus Module code
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __MODULE_H
+#define __MODULE_H
+
+struct gb_module {
+	struct device dev;
+	struct gb_host_device *hd;
+
+	struct list_head hd_node;
+
+	u8 module_id;
+	size_t num_interfaces;
+
+	bool disconnected;
+
+	struct gb_interface *interfaces[0];
+};
+#define to_gb_module(d) container_of(d, struct gb_module, dev)
+
+struct gb_module *gb_module_create(struct gb_host_device *hd, u8 module_id,
+				   size_t num_interfaces);
+int gb_module_add(struct gb_module *module);
+void gb_module_del(struct gb_module *module);
+void gb_module_put(struct gb_module *module);
+
+#endif /* __MODULE_H */
diff --git a/drivers/staging/greybus/operation.c b/drivers/staging/greybus/operation.c
new file mode 100644
index 000000000000..0123109a1070
--- /dev/null
+++ b/drivers/staging/greybus/operation.c
@@ -0,0 +1,1239 @@
+/*
+ * Greybus operations
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+
+#include "greybus.h"
+#include "greybus_trace.h"
+
+static struct kmem_cache *gb_operation_cache;
+static struct kmem_cache *gb_message_cache;
+
+/* Workqueue to handle Greybus operation completions. */
+static struct workqueue_struct *gb_operation_completion_wq;
+
+/* Wait queue for synchronous cancellations. */
+static DECLARE_WAIT_QUEUE_HEAD(gb_operation_cancellation_queue);
+
+/*
+ * Protects updates to operation->errno.
+ */
+static DEFINE_SPINLOCK(gb_operations_lock);
+
+static int gb_operation_response_send(struct gb_operation *operation,
+					int errno);
+
+/*
+ * Increment operation active count and add to connection list unless the
+ * connection is going away.
+ *
+ * Caller holds operation reference.
+ */
+static int gb_operation_get_active(struct gb_operation *operation)
+{
+	struct gb_connection *connection = operation->connection;
+	unsigned long flags;
+
+	spin_lock_irqsave(&connection->lock, flags);
+	switch (connection->state) {
+	case GB_CONNECTION_STATE_ENABLED:
+		break;
+	case GB_CONNECTION_STATE_ENABLED_TX:
+		if (gb_operation_is_incoming(operation))
+			goto err_unlock;
+		break;
+	case GB_CONNECTION_STATE_DISCONNECTING:
+		if (!gb_operation_is_core(operation))
+			goto err_unlock;
+		break;
+	default:
+		goto err_unlock;
+	}
+
+	if (operation->active++ == 0)
+		list_add_tail(&operation->links, &connection->operations);
+
+	trace_gb_operation_get_active(operation);
+
+	spin_unlock_irqrestore(&connection->lock, flags);
+
+	return 0;
+
+err_unlock:
+	spin_unlock_irqrestore(&connection->lock, flags);
+
+	return -ENOTCONN;
+}
+
+/* Caller holds operation reference. */
+static void gb_operation_put_active(struct gb_operation *operation)
+{
+	struct gb_connection *connection = operation->connection;
+	unsigned long flags;
+
+	spin_lock_irqsave(&connection->lock, flags);
+
+	trace_gb_operation_put_active(operation);
+
+	if (--operation->active == 0) {
+		list_del(&operation->links);
+		if (atomic_read(&operation->waiters))
+			wake_up(&gb_operation_cancellation_queue);
+	}
+	spin_unlock_irqrestore(&connection->lock, flags);
+}
+
+static bool gb_operation_is_active(struct gb_operation *operation)
+{
+	struct gb_connection *connection = operation->connection;
+	unsigned long flags;
+	bool ret;
+
+	spin_lock_irqsave(&connection->lock, flags);
+	ret = operation->active;
+	spin_unlock_irqrestore(&connection->lock, flags);
+
+	return ret;
+}
+
+/*
+ * Set an operation's result.
+ *
+ * Initially an outgoing operation's errno value is -EBADR.
+ * If no error occurs before sending the request message the only
+ * valid value operation->errno can be set to is -EINPROGRESS,
+ * indicating the request has been (or rather is about to be) sent.
+ * At that point nobody should be looking at the result until the
+ * response arrives.
+ *
+ * The first time the result gets set after the request has been
+ * sent, that result "sticks."  That is, if two concurrent threads
+ * race to set the result, the first one wins.  The return value
+ * tells the caller whether its result was recorded; if not the
+ * caller has nothing more to do.
+ *
+ * The result value -EILSEQ is reserved to signal an implementation
+ * error; if it's ever observed, the code performing the request has
+ * done something fundamentally wrong.  It is an error to try to set
+ * the result to -EBADR, and attempts to do so result in a warning,
+ * and -EILSEQ is used instead.  Similarly, the only valid result
+ * value to set for an operation in initial state is -EINPROGRESS.
+ * Attempts to do otherwise will also record a (successful) -EILSEQ
+ * operation result.
+ */
+static bool gb_operation_result_set(struct gb_operation *operation, int result)
+{
+	unsigned long flags;
+	int prev;
+
+	if (result == -EINPROGRESS) {
+		/*
+		 * -EINPROGRESS is used to indicate the request is
+		 * in flight.  It should be the first result value
+		 * set after the initial -EBADR.  Issue a warning
+		 * and record an implementation error if it's
+		 * set at any other time.
+		 */
+		spin_lock_irqsave(&gb_operations_lock, flags);
+		prev = operation->errno;
+		if (prev == -EBADR)
+			operation->errno = result;
+		else
+			operation->errno = -EILSEQ;
+		spin_unlock_irqrestore(&gb_operations_lock, flags);
+		WARN_ON(prev != -EBADR);
+
+		return true;
+	}
+
+	/*
+	 * The first result value set after a request has been sent
+	 * will be the final result of the operation.  Subsequent
+	 * attempts to set the result are ignored.
+	 *
+	 * Note that -EBADR is a reserved "initial state" result
+	 * value.  Attempts to set this value result in a warning,
+	 * and the result code is set to -EILSEQ instead.
+	 */
+	if (WARN_ON(result == -EBADR))
+		result = -EILSEQ; /* Nobody should be setting -EBADR */
+
+	spin_lock_irqsave(&gb_operations_lock, flags);
+	prev = operation->errno;
+	if (prev == -EINPROGRESS)
+		operation->errno = result;	/* First and final result */
+	spin_unlock_irqrestore(&gb_operations_lock, flags);
+
+	return prev == -EINPROGRESS;
+}
+
+int gb_operation_result(struct gb_operation *operation)
+{
+	int result = operation->errno;
+
+	WARN_ON(result == -EBADR);
+	WARN_ON(result == -EINPROGRESS);
+
+	return result;
+}
+EXPORT_SYMBOL_GPL(gb_operation_result);
+
+/*
+ * Looks up an outgoing operation on a connection and returns a refcounted
+ * pointer if found, or NULL otherwise.
+ */
+static struct gb_operation *
+gb_operation_find_outgoing(struct gb_connection *connection, u16 operation_id)
+{
+	struct gb_operation *operation;
+	unsigned long flags;
+	bool found = false;
+
+	spin_lock_irqsave(&connection->lock, flags);
+	list_for_each_entry(operation, &connection->operations, links)
+		if (operation->id == operation_id &&
+				!gb_operation_is_incoming(operation)) {
+			gb_operation_get(operation);
+			found = true;
+			break;
+		}
+	spin_unlock_irqrestore(&connection->lock, flags);
+
+	return found ? operation : NULL;
+}
+
+static int gb_message_send(struct gb_message *message, gfp_t gfp)
+{
+	struct gb_connection *connection = message->operation->connection;
+
+	trace_gb_message_send(message);
+	return connection->hd->driver->message_send(connection->hd,
+					connection->hd_cport_id,
+					message,
+					gfp);
+}
+
+/*
+ * Cancel a message we have passed to the host device layer to be sent.
+ */
+static void gb_message_cancel(struct gb_message *message)
+{
+	struct gb_host_device *hd = message->operation->connection->hd;
+
+	hd->driver->message_cancel(message);
+}
+
+static void gb_operation_request_handle(struct gb_operation *operation)
+{
+	struct gb_connection *connection = operation->connection;
+	int status;
+	int ret;
+
+	if (connection->handler) {
+		status = connection->handler(operation);
+	} else {
+		dev_err(&connection->hd->dev,
+			"%s: unexpected incoming request of type 0x%02x\n",
+			connection->name, operation->type);
+
+		status = -EPROTONOSUPPORT;
+	}
+
+	ret = gb_operation_response_send(operation, status);
+	if (ret) {
+		dev_err(&connection->hd->dev,
+			"%s: failed to send response %d for type 0x%02x: %d\n",
+			connection->name, status, operation->type, ret);
+		return;
+	}
+}
+
+/*
+ * Process operation work.
+ *
+ * For incoming requests, call the protocol request handler. The operation
+ * result should be -EINPROGRESS at this point.
+ *
+ * For outgoing requests, the operation result value should have
+ * been set before queueing this.  The operation callback function
+ * allows the original requester to know the request has completed
+ * and its result is available.
+ */
+static void gb_operation_work(struct work_struct *work)
+{
+	struct gb_operation *operation;
+
+	operation = container_of(work, struct gb_operation, work);
+
+	if (gb_operation_is_incoming(operation))
+		gb_operation_request_handle(operation);
+	else
+		operation->callback(operation);
+
+	gb_operation_put_active(operation);
+	gb_operation_put(operation);
+}
+
+static void gb_operation_message_init(struct gb_host_device *hd,
+				struct gb_message *message, u16 operation_id,
+				size_t payload_size, u8 type)
+{
+	struct gb_operation_msg_hdr *header;
+
+	header = message->buffer;
+
+	message->header = header;
+	message->payload = payload_size ? header + 1 : NULL;
+	message->payload_size = payload_size;
+
+	/*
+	 * The type supplied for incoming message buffers will be
+	 * GB_REQUEST_TYPE_INVALID. Such buffers will be overwritten by
+	 * arriving data so there's no need to initialize the message header.
+	 */
+	if (type != GB_REQUEST_TYPE_INVALID) {
+		u16 message_size = (u16)(sizeof(*header) + payload_size);
+
+		/*
+		 * For a request, the operation id gets filled in
+		 * when the message is sent.  For a response, it
+		 * will be copied from the request by the caller.
+		 *
+		 * The result field in a request message must be
+		 * zero.  It will be set just prior to sending for
+		 * a response.
+		 */
+		header->size = cpu_to_le16(message_size);
+		header->operation_id = 0;
+		header->type = type;
+		header->result = 0;
+	}
+}
+
+/*
+ * Allocate a message to be used for an operation request or response.
+ * Both types of message contain a common header.  The request message
+ * for an outgoing operation is outbound, as is the response message
+ * for an incoming operation.  The message header for an outbound
+ * message is partially initialized here.
+ *
+ * The headers for inbound messages don't need to be initialized;
+ * they'll be filled in by arriving data.
+ *
+ * Our message buffers have the following layout:
+ *	message header  \_ these combined are
+ *	message payload /  the message size
+ */
+static struct gb_message *
+gb_operation_message_alloc(struct gb_host_device *hd, u8 type,
+				size_t payload_size, gfp_t gfp_flags)
+{
+	struct gb_message *message;
+	struct gb_operation_msg_hdr *header;
+	size_t message_size = payload_size + sizeof(*header);
+
+	if (message_size > hd->buffer_size_max) {
+		dev_warn(&hd->dev, "requested message size too big (%zu > %zu)\n",
+				message_size, hd->buffer_size_max);
+		return NULL;
+	}
+
+	/* Allocate the message structure and buffer. */
+	message = kmem_cache_zalloc(gb_message_cache, gfp_flags);
+	if (!message)
+		return NULL;
+
+	message->buffer = kzalloc(message_size, gfp_flags);
+	if (!message->buffer)
+		goto err_free_message;
+
+	/* Initialize the message.  Operation id is filled in later. */
+	gb_operation_message_init(hd, message, 0, payload_size, type);
+
+	return message;
+
+err_free_message:
+	kmem_cache_free(gb_message_cache, message);
+
+	return NULL;
+}
+
+static void gb_operation_message_free(struct gb_message *message)
+{
+	kfree(message->buffer);
+	kmem_cache_free(gb_message_cache, message);
+}
+
+/*
+ * Map an enum gb_operation_status value (which is represented in a
+ * message as a single byte) to an appropriate Linux negative errno.
+ */
+static int gb_operation_status_map(u8 status)
+{
+	switch (status) {
+	case GB_OP_SUCCESS:
+		return 0;
+	case GB_OP_INTERRUPTED:
+		return -EINTR;
+	case GB_OP_TIMEOUT:
+		return -ETIMEDOUT;
+	case GB_OP_NO_MEMORY:
+		return -ENOMEM;
+	case GB_OP_PROTOCOL_BAD:
+		return -EPROTONOSUPPORT;
+	case GB_OP_OVERFLOW:
+		return -EMSGSIZE;
+	case GB_OP_INVALID:
+		return -EINVAL;
+	case GB_OP_RETRY:
+		return -EAGAIN;
+	case GB_OP_NONEXISTENT:
+		return -ENODEV;
+	case GB_OP_MALFUNCTION:
+		return -EILSEQ;
+	case GB_OP_UNKNOWN_ERROR:
+	default:
+		return -EIO;
+	}
+}
+
+/*
+ * Map a Linux errno value (from operation->errno) into the value
+ * that should represent it in a response message status sent
+ * over the wire.  Returns an enum gb_operation_status value (which
+ * is represented in a message as a single byte).
+ */
+static u8 gb_operation_errno_map(int errno)
+{
+	switch (errno) {
+	case 0:
+		return GB_OP_SUCCESS;
+	case -EINTR:
+		return GB_OP_INTERRUPTED;
+	case -ETIMEDOUT:
+		return GB_OP_TIMEOUT;
+	case -ENOMEM:
+		return GB_OP_NO_MEMORY;
+	case -EPROTONOSUPPORT:
+		return GB_OP_PROTOCOL_BAD;
+	case -EMSGSIZE:
+		return GB_OP_OVERFLOW;	/* Could be underflow too */
+	case -EINVAL:
+		return GB_OP_INVALID;
+	case -EAGAIN:
+		return GB_OP_RETRY;
+	case -EILSEQ:
+		return GB_OP_MALFUNCTION;
+	case -ENODEV:
+		return GB_OP_NONEXISTENT;
+	case -EIO:
+	default:
+		return GB_OP_UNKNOWN_ERROR;
+	}
+}
+
+bool gb_operation_response_alloc(struct gb_operation *operation,
+					size_t response_size, gfp_t gfp)
+{
+	struct gb_host_device *hd = operation->connection->hd;
+	struct gb_operation_msg_hdr *request_header;
+	struct gb_message *response;
+	u8 type;
+
+	type = operation->type | GB_MESSAGE_TYPE_RESPONSE;
+	response = gb_operation_message_alloc(hd, type, response_size, gfp);
+	if (!response)
+		return false;
+	response->operation = operation;
+
+	/*
+	 * Size and type get initialized when the message is
+	 * allocated.  The errno will be set before sending.  All
+	 * that's left is the operation id, which we copy from the
+	 * request message header (as-is, in little-endian order).
+	 */
+	request_header = operation->request->header;
+	response->header->operation_id = request_header->operation_id;
+	operation->response = response;
+
+	return true;
+}
+EXPORT_SYMBOL_GPL(gb_operation_response_alloc);
+
+/*
+ * Create a Greybus operation to be sent over the given connection.
+ * The request buffer will be big enough for a payload of the given
+ * size.
+ *
+ * For outgoing requests, the request message's header will be
+ * initialized with the type of the request and the message size.
+ * Outgoing operations must also specify the response buffer size,
+ * which must be sufficient to hold all expected response data.  The
+ * response message header will eventually be overwritten, so there's
+ * no need to initialize it here.
+ *
+ * Request messages for incoming operations can arrive in interrupt
+ * context, so they must be allocated with GFP_ATOMIC.  In this case
+ * the request buffer will be immediately overwritten, so there is
+ * no need to initialize the message header.  Responsibility for
+ * allocating a response buffer lies with the incoming request
+ * handler for a protocol.  So we don't allocate that here.
+ *
+ * Returns a pointer to the new operation or a null pointer if an
+ * error occurs.
+ */
+static struct gb_operation *
+gb_operation_create_common(struct gb_connection *connection, u8 type,
+				size_t request_size, size_t response_size,
+				unsigned long op_flags, gfp_t gfp_flags)
+{
+	struct gb_host_device *hd = connection->hd;
+	struct gb_operation *operation;
+
+	operation = kmem_cache_zalloc(gb_operation_cache, gfp_flags);
+	if (!operation)
+		return NULL;
+	operation->connection = connection;
+
+	operation->request = gb_operation_message_alloc(hd, type, request_size,
+							gfp_flags);
+	if (!operation->request)
+		goto err_cache;
+	operation->request->operation = operation;
+
+	/* Allocate the response buffer for outgoing operations */
+	if (!(op_flags & GB_OPERATION_FLAG_INCOMING)) {
+		if (!gb_operation_response_alloc(operation, response_size,
+						 gfp_flags)) {
+			goto err_request;
+		}
+	}
+
+	operation->flags = op_flags;
+	operation->type = type;
+	operation->errno = -EBADR;  /* Initial value--means "never set" */
+
+	INIT_WORK(&operation->work, gb_operation_work);
+	init_completion(&operation->completion);
+	kref_init(&operation->kref);
+	atomic_set(&operation->waiters, 0);
+
+	return operation;
+
+err_request:
+	gb_operation_message_free(operation->request);
+err_cache:
+	kmem_cache_free(gb_operation_cache, operation);
+
+	return NULL;
+}
+
+/*
+ * Create a new operation associated with the given connection.  The
+ * request and response sizes provided are the number of bytes
+ * required to hold the request/response payload only.  Both of
+ * these are allowed to be 0.  Note that 0x00 is reserved as an
+ * invalid operation type for all protocols, and this is enforced
+ * here.
+ */
+struct gb_operation *
+gb_operation_create_flags(struct gb_connection *connection,
+				u8 type, size_t request_size,
+				size_t response_size, unsigned long flags,
+				gfp_t gfp)
+{
+	struct gb_operation *operation;
+
+	if (WARN_ON_ONCE(type == GB_REQUEST_TYPE_INVALID))
+		return NULL;
+	if (WARN_ON_ONCE(type & GB_MESSAGE_TYPE_RESPONSE))
+		type &= ~GB_MESSAGE_TYPE_RESPONSE;
+
+	if (WARN_ON_ONCE(flags & ~GB_OPERATION_FLAG_USER_MASK))
+		flags &= GB_OPERATION_FLAG_USER_MASK;
+
+	operation = gb_operation_create_common(connection, type,
+						request_size, response_size,
+						flags, gfp);
+	if (operation)
+		trace_gb_operation_create(operation);
+
+	return operation;
+}
+EXPORT_SYMBOL_GPL(gb_operation_create_flags);
+
+struct gb_operation *
+gb_operation_create_core(struct gb_connection *connection,
+				u8 type, size_t request_size,
+				size_t response_size, unsigned long flags,
+				gfp_t gfp)
+{
+	struct gb_operation *operation;
+
+	flags |= GB_OPERATION_FLAG_CORE;
+
+	operation = gb_operation_create_common(connection, type,
+						request_size, response_size,
+						flags, gfp);
+	if (operation)
+		trace_gb_operation_create_core(operation);
+
+	return operation;
+}
+/* Do not export this function. */
+
+size_t gb_operation_get_payload_size_max(struct gb_connection *connection)
+{
+	struct gb_host_device *hd = connection->hd;
+
+	return hd->buffer_size_max - sizeof(struct gb_operation_msg_hdr);
+}
+EXPORT_SYMBOL_GPL(gb_operation_get_payload_size_max);
+
+static struct gb_operation *
+gb_operation_create_incoming(struct gb_connection *connection, u16 id,
+				u8 type, void *data, size_t size)
+{
+	struct gb_operation *operation;
+	size_t request_size;
+	unsigned long flags = GB_OPERATION_FLAG_INCOMING;
+
+	/* Caller has made sure we at least have a message header. */
+	request_size = size - sizeof(struct gb_operation_msg_hdr);
+
+	if (!id)
+		flags |= GB_OPERATION_FLAG_UNIDIRECTIONAL;
+
+	operation = gb_operation_create_common(connection, type,
+						request_size,
+						GB_REQUEST_TYPE_INVALID,
+						flags, GFP_ATOMIC);
+	if (!operation)
+		return NULL;
+
+	operation->id = id;
+	memcpy(operation->request->header, data, size);
+	trace_gb_operation_create_incoming(operation);
+
+	return operation;
+}
+
+/*
+ * Get an additional reference on an operation.
+ */
+void gb_operation_get(struct gb_operation *operation)
+{
+	kref_get(&operation->kref);
+}
+EXPORT_SYMBOL_GPL(gb_operation_get);
+
+/*
+ * Destroy a previously created operation.
+ */
+static void _gb_operation_destroy(struct kref *kref)
+{
+	struct gb_operation *operation;
+
+	operation = container_of(kref, struct gb_operation, kref);
+
+	trace_gb_operation_destroy(operation);
+
+	if (operation->response)
+		gb_operation_message_free(operation->response);
+	gb_operation_message_free(operation->request);
+
+	kmem_cache_free(gb_operation_cache, operation);
+}
+
+/*
+ * Drop a reference on an operation, and destroy it when the last
+ * one is gone.
+ */
+void gb_operation_put(struct gb_operation *operation)
+{
+	if (WARN_ON(!operation))
+		return;
+
+	kref_put(&operation->kref, _gb_operation_destroy);
+}
+EXPORT_SYMBOL_GPL(gb_operation_put);
+
+/* Tell the requester we're done */
+static void gb_operation_sync_callback(struct gb_operation *operation)
+{
+	complete(&operation->completion);
+}
+
+/**
+ * gb_operation_request_send() - send an operation request message
+ * @operation:	the operation to initiate
+ * @callback:	the operation completion callback
+ * @gfp:	the memory flags to use for any allocations
+ *
+ * The caller has filled in any payload so the request message is ready to go.
+ * The callback function supplied will be called when the response message has
+ * arrived, a unidirectional request has been sent, or the operation is
+ * cancelled, indicating that the operation is complete. The callback function
+ * can fetch the result of the operation using gb_operation_result() if
+ * desired.
+ *
+ * Return: 0 if the request was successfully queued in the host-driver queues,
+ * or a negative errno.
+ */
+int gb_operation_request_send(struct gb_operation *operation,
+				gb_operation_callback callback,
+				gfp_t gfp)
+{
+	struct gb_connection *connection = operation->connection;
+	struct gb_operation_msg_hdr *header;
+	unsigned int cycle;
+	int ret;
+
+	if (gb_connection_is_offloaded(connection))
+		return -EBUSY;
+
+	if (!callback)
+		return -EINVAL;
+
+	/*
+	 * Record the callback function, which is executed in
+	 * non-atomic (workqueue) context when the final result
+	 * of an operation has been set.
+	 */
+	operation->callback = callback;
+
+	/*
+	 * Assign the operation's id, and store it in the request header.
+	 * Zero is a reserved operation id for unidirectional operations.
+	 */
+	if (gb_operation_is_unidirectional(operation)) {
+		operation->id = 0;
+	} else {
+		cycle = (unsigned int)atomic_inc_return(&connection->op_cycle);
+		operation->id = (u16)(cycle % U16_MAX + 1);
+	}
+
+	header = operation->request->header;
+	header->operation_id = cpu_to_le16(operation->id);
+
+	gb_operation_result_set(operation, -EINPROGRESS);
+
+	/*
+	 * Get an extra reference on the operation. It'll be dropped when the
+	 * operation completes.
+	 */
+	gb_operation_get(operation);
+	ret = gb_operation_get_active(operation);
+	if (ret)
+		goto err_put;
+
+	ret = gb_message_send(operation->request, gfp);
+	if (ret)
+		goto err_put_active;
+
+	return 0;
+
+err_put_active:
+	gb_operation_put_active(operation);
+err_put:
+	gb_operation_put(operation);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_operation_request_send);
+
+/*
+ * Send a synchronous operation.  This function is expected to
+ * block, returning only when the response has arrived, (or when an
+ * error is detected.  The return value is the result of the
+ * operation.
+ */
+int gb_operation_request_send_sync_timeout(struct gb_operation *operation,
+						unsigned int timeout)
+{
+	int ret;
+	unsigned long timeout_jiffies;
+
+	ret = gb_operation_request_send(operation, gb_operation_sync_callback,
+					GFP_KERNEL);
+	if (ret)
+		return ret;
+
+	if (timeout)
+		timeout_jiffies = msecs_to_jiffies(timeout);
+	else
+		timeout_jiffies = MAX_SCHEDULE_TIMEOUT;
+
+	ret = wait_for_completion_interruptible_timeout(&operation->completion,
+							timeout_jiffies);
+	if (ret < 0) {
+		/* Cancel the operation if interrupted */
+		gb_operation_cancel(operation, -ECANCELED);
+	} else if (ret == 0) {
+		/* Cancel the operation if op timed out */
+		gb_operation_cancel(operation, -ETIMEDOUT);
+	}
+
+	return gb_operation_result(operation);
+}
+EXPORT_SYMBOL_GPL(gb_operation_request_send_sync_timeout);
+
+/*
+ * Send a response for an incoming operation request.  A non-zero
+ * errno indicates a failed operation.
+ *
+ * If there is any response payload, the incoming request handler is
+ * responsible for allocating the response message.  Otherwise the
+ * it can simply supply the result errno; this function will
+ * allocate the response message if necessary.
+ */
+static int gb_operation_response_send(struct gb_operation *operation,
+					int errno)
+{
+	struct gb_connection *connection = operation->connection;
+	int ret;
+
+	if (!operation->response &&
+			!gb_operation_is_unidirectional(operation)) {
+		if (!gb_operation_response_alloc(operation, 0, GFP_KERNEL))
+			return -ENOMEM;
+	}
+
+	/* Record the result */
+	if (!gb_operation_result_set(operation, errno)) {
+		dev_err(&connection->hd->dev, "request result already set\n");
+		return -EIO;	/* Shouldn't happen */
+	}
+
+	/* Sender of request does not care about response. */
+	if (gb_operation_is_unidirectional(operation))
+		return 0;
+
+	/* Reference will be dropped when message has been sent. */
+	gb_operation_get(operation);
+	ret = gb_operation_get_active(operation);
+	if (ret)
+		goto err_put;
+
+	/* Fill in the response header and send it */
+	operation->response->header->result = gb_operation_errno_map(errno);
+
+	ret = gb_message_send(operation->response, GFP_KERNEL);
+	if (ret)
+		goto err_put_active;
+
+	return 0;
+
+err_put_active:
+	gb_operation_put_active(operation);
+err_put:
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+/*
+ * This function is called when a message send request has completed.
+ */
+void greybus_message_sent(struct gb_host_device *hd,
+					struct gb_message *message, int status)
+{
+	struct gb_operation *operation = message->operation;
+	struct gb_connection *connection = operation->connection;
+
+	/*
+	 * If the message was a response, we just need to drop our
+	 * reference to the operation.  If an error occurred, report
+	 * it.
+	 *
+	 * For requests, if there's no error and the operation in not
+	 * unidirectional, there's nothing more to do until the response
+	 * arrives. If an error occurred attempting to send it, or if the
+	 * operation is unidrectional, record the result of the operation and
+	 * schedule its completion.
+	 */
+	if (message == operation->response) {
+		if (status) {
+			dev_err(&connection->hd->dev,
+				"%s: error sending response 0x%02x: %d\n",
+				connection->name, operation->type, status);
+		}
+
+		gb_operation_put_active(operation);
+		gb_operation_put(operation);
+	} else if (status || gb_operation_is_unidirectional(operation)) {
+		if (gb_operation_result_set(operation, status)) {
+			queue_work(gb_operation_completion_wq,
+					&operation->work);
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(greybus_message_sent);
+
+/*
+ * We've received data on a connection, and it doesn't look like a
+ * response, so we assume it's a request.
+ *
+ * This is called in interrupt context, so just copy the incoming
+ * data into the request buffer and handle the rest via workqueue.
+ */
+static void gb_connection_recv_request(struct gb_connection *connection,
+				const struct gb_operation_msg_hdr *header,
+				void *data, size_t size)
+{
+	struct gb_operation *operation;
+	u16 operation_id;
+	u8 type;
+	int ret;
+
+	operation_id = le16_to_cpu(header->operation_id);
+	type = header->type;
+
+	operation = gb_operation_create_incoming(connection, operation_id,
+						type, data, size);
+	if (!operation) {
+		dev_err(&connection->hd->dev,
+			"%s: can't create incoming operation\n",
+			connection->name);
+		return;
+	}
+
+	ret = gb_operation_get_active(operation);
+	if (ret) {
+		gb_operation_put(operation);
+		return;
+	}
+	trace_gb_message_recv_request(operation->request);
+
+	/*
+	 * The initial reference to the operation will be dropped when the
+	 * request handler returns.
+	 */
+	if (gb_operation_result_set(operation, -EINPROGRESS))
+		queue_work(connection->wq, &operation->work);
+}
+
+/*
+ * We've received data that appears to be an operation response
+ * message.  Look up the operation, and record that we've received
+ * its response.
+ *
+ * This is called in interrupt context, so just copy the incoming
+ * data into the response buffer and handle the rest via workqueue.
+ */
+static void gb_connection_recv_response(struct gb_connection *connection,
+				const struct gb_operation_msg_hdr *header,
+				void *data, size_t size)
+{
+	struct gb_operation *operation;
+	struct gb_message *message;
+	size_t message_size;
+	u16 operation_id;
+	int errno;
+
+	operation_id = le16_to_cpu(header->operation_id);
+
+	if (!operation_id) {
+		dev_err_ratelimited(&connection->hd->dev,
+				"%s: invalid response id 0 received\n",
+				connection->name);
+		return;
+	}
+
+	operation = gb_operation_find_outgoing(connection, operation_id);
+	if (!operation) {
+		dev_err_ratelimited(&connection->hd->dev,
+				"%s: unexpected response id 0x%04x received\n",
+				connection->name, operation_id);
+		return;
+	}
+
+	errno = gb_operation_status_map(header->result);
+	message = operation->response;
+	message_size = sizeof(*header) + message->payload_size;
+	if (!errno && size > message_size) {
+		dev_err_ratelimited(&connection->hd->dev,
+				"%s: malformed response 0x%02x received (%zu > %zu)\n",
+				connection->name, header->type,
+				size, message_size);
+		errno = -EMSGSIZE;
+	} else if (!errno && size < message_size) {
+		if (gb_operation_short_response_allowed(operation)) {
+			message->payload_size = size - sizeof(*header);
+		} else {
+			dev_err_ratelimited(&connection->hd->dev,
+					"%s: short response 0x%02x received (%zu < %zu)\n",
+					connection->name, header->type,
+					size, message_size);
+			errno = -EMSGSIZE;
+		}
+	}
+
+	/* We must ignore the payload if a bad status is returned */
+	if (errno)
+		size = sizeof(*header);
+
+	/* The rest will be handled in work queue context */
+	if (gb_operation_result_set(operation, errno)) {
+		memcpy(message->buffer, data, size);
+
+		trace_gb_message_recv_response(message);
+
+		queue_work(gb_operation_completion_wq, &operation->work);
+	}
+
+	gb_operation_put(operation);
+}
+
+/*
+ * Handle data arriving on a connection.  As soon as we return the
+ * supplied data buffer will be reused (so unless we do something
+ * with, it's effectively dropped).
+ */
+void gb_connection_recv(struct gb_connection *connection,
+				void *data, size_t size)
+{
+	struct gb_operation_msg_hdr header;
+	struct device *dev = &connection->hd->dev;
+	size_t msg_size;
+
+	if (connection->state == GB_CONNECTION_STATE_DISABLED ||
+			gb_connection_is_offloaded(connection)) {
+		dev_warn_ratelimited(dev, "%s: dropping %zu received bytes\n",
+				connection->name, size);
+		return;
+	}
+
+	if (size < sizeof(header)) {
+		dev_err_ratelimited(dev, "%s: short message received\n",
+				connection->name);
+		return;
+	}
+
+	/* Use memcpy as data may be unaligned */
+	memcpy(&header, data, sizeof(header));
+	msg_size = le16_to_cpu(header.size);
+	if (size < msg_size) {
+		dev_err_ratelimited(dev,
+				"%s: incomplete message 0x%04x of type 0x%02x received (%zu < %zu)\n",
+				connection->name,
+				le16_to_cpu(header.operation_id),
+				header.type, size, msg_size);
+		return;		/* XXX Should still complete operation */
+	}
+
+	if (header.type & GB_MESSAGE_TYPE_RESPONSE) {
+		gb_connection_recv_response(connection,	&header, data,
+						msg_size);
+	} else {
+		gb_connection_recv_request(connection, &header, data,
+						msg_size);
+	}
+}
+
+/*
+ * Cancel an outgoing operation synchronously, and record the given error to
+ * indicate why.
+ */
+void gb_operation_cancel(struct gb_operation *operation, int errno)
+{
+	if (WARN_ON(gb_operation_is_incoming(operation)))
+		return;
+
+	if (gb_operation_result_set(operation, errno)) {
+		gb_message_cancel(operation->request);
+		queue_work(gb_operation_completion_wq, &operation->work);
+	}
+	trace_gb_message_cancel_outgoing(operation->request);
+
+	atomic_inc(&operation->waiters);
+	wait_event(gb_operation_cancellation_queue,
+			!gb_operation_is_active(operation));
+	atomic_dec(&operation->waiters);
+}
+EXPORT_SYMBOL_GPL(gb_operation_cancel);
+
+/*
+ * Cancel an incoming operation synchronously. Called during connection tear
+ * down.
+ */
+void gb_operation_cancel_incoming(struct gb_operation *operation, int errno)
+{
+	if (WARN_ON(!gb_operation_is_incoming(operation)))
+		return;
+
+	if (!gb_operation_is_unidirectional(operation)) {
+		/*
+		 * Make sure the request handler has submitted the response
+		 * before cancelling it.
+		 */
+		flush_work(&operation->work);
+		if (!gb_operation_result_set(operation, errno))
+			gb_message_cancel(operation->response);
+	}
+	trace_gb_message_cancel_incoming(operation->response);
+
+	atomic_inc(&operation->waiters);
+	wait_event(gb_operation_cancellation_queue,
+			!gb_operation_is_active(operation));
+	atomic_dec(&operation->waiters);
+}
+
+/**
+ * gb_operation_sync_timeout() - implement a "simple" synchronous operation
+ * @connection: the Greybus connection to send this to
+ * @type: the type of operation to send
+ * @request: pointer to a memory buffer to copy the request from
+ * @request_size: size of @request
+ * @response: pointer to a memory buffer to copy the response to
+ * @response_size: the size of @response.
+ * @timeout: operation timeout in milliseconds
+ *
+ * This function implements a simple synchronous Greybus operation.  It sends
+ * the provided operation request and waits (sleeps) until the corresponding
+ * operation response message has been successfully received, or an error
+ * occurs.  @request and @response are buffers to hold the request and response
+ * data respectively, and if they are not NULL, their size must be specified in
+ * @request_size and @response_size.
+ *
+ * If a response payload is to come back, and @response is not NULL,
+ * @response_size number of bytes will be copied into @response if the operation
+ * is successful.
+ *
+ * If there is an error, the response buffer is left alone.
+ */
+int gb_operation_sync_timeout(struct gb_connection *connection, int type,
+				void *request, int request_size,
+				void *response, int response_size,
+				unsigned int timeout)
+{
+	struct gb_operation *operation;
+	int ret;
+
+	if ((response_size && !response) ||
+	    (request_size && !request))
+		return -EINVAL;
+
+	operation = gb_operation_create(connection, type,
+					request_size, response_size,
+					GFP_KERNEL);
+	if (!operation)
+		return -ENOMEM;
+
+	if (request_size)
+		memcpy(operation->request->payload, request, request_size);
+
+	ret = gb_operation_request_send_sync_timeout(operation, timeout);
+	if (ret) {
+		dev_err(&connection->hd->dev,
+			"%s: synchronous operation id 0x%04x of type 0x%02x failed: %d\n",
+			connection->name, operation->id, type, ret);
+	} else {
+		if (response_size) {
+			memcpy(response, operation->response->payload,
+			       response_size);
+		}
+	}
+
+	gb_operation_put(operation);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_operation_sync_timeout);
+
+/**
+ * gb_operation_unidirectional_timeout() - initiate a unidirectional operation
+ * @connection:		connection to use
+ * @type:		type of operation to send
+ * @request:		memory buffer to copy the request from
+ * @request_size:	size of @request
+ * @timeout:		send timeout in milliseconds
+ *
+ * Initiate a unidirectional operation by sending a request message and
+ * waiting for it to be acknowledged as sent by the host device.
+ *
+ * Note that successful send of a unidirectional operation does not imply that
+ * the request as actually reached the remote end of the connection.
+ */
+int gb_operation_unidirectional_timeout(struct gb_connection *connection,
+				int type, void *request, int request_size,
+				unsigned int timeout)
+{
+	struct gb_operation *operation;
+	int ret;
+
+	if (request_size && !request)
+		return -EINVAL;
+
+	operation = gb_operation_create_flags(connection, type,
+					request_size, 0,
+					GB_OPERATION_FLAG_UNIDIRECTIONAL,
+					GFP_KERNEL);
+	if (!operation)
+		return -ENOMEM;
+
+	if (request_size)
+		memcpy(operation->request->payload, request, request_size);
+
+	ret = gb_operation_request_send_sync_timeout(operation, timeout);
+	if (ret) {
+		dev_err(&connection->hd->dev,
+			"%s: unidirectional operation of type 0x%02x failed: %d\n",
+			connection->name, type, ret);
+	}
+
+	gb_operation_put(operation);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_operation_unidirectional_timeout);
+
+int __init gb_operation_init(void)
+{
+	gb_message_cache = kmem_cache_create("gb_message_cache",
+				sizeof(struct gb_message), 0, 0, NULL);
+	if (!gb_message_cache)
+		return -ENOMEM;
+
+	gb_operation_cache = kmem_cache_create("gb_operation_cache",
+				sizeof(struct gb_operation), 0, 0, NULL);
+	if (!gb_operation_cache)
+		goto err_destroy_message_cache;
+
+	gb_operation_completion_wq = alloc_workqueue("greybus_completion",
+				0, 0);
+	if (!gb_operation_completion_wq)
+		goto err_destroy_operation_cache;
+
+	return 0;
+
+err_destroy_operation_cache:
+	kmem_cache_destroy(gb_operation_cache);
+	gb_operation_cache = NULL;
+err_destroy_message_cache:
+	kmem_cache_destroy(gb_message_cache);
+	gb_message_cache = NULL;
+
+	return -ENOMEM;
+}
+
+void gb_operation_exit(void)
+{
+	destroy_workqueue(gb_operation_completion_wq);
+	gb_operation_completion_wq = NULL;
+	kmem_cache_destroy(gb_operation_cache);
+	gb_operation_cache = NULL;
+	kmem_cache_destroy(gb_message_cache);
+	gb_message_cache = NULL;
+}
diff --git a/drivers/staging/greybus/operation.h b/drivers/staging/greybus/operation.h
new file mode 100644
index 000000000000..de09a2c7de54
--- /dev/null
+++ b/drivers/staging/greybus/operation.h
@@ -0,0 +1,210 @@
+/*
+ * Greybus operations
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __OPERATION_H
+#define __OPERATION_H
+
+#include <linux/completion.h>
+
+struct gb_operation;
+
+/* The default amount of time a request is given to complete */
+#define GB_OPERATION_TIMEOUT_DEFAULT	1000	/* milliseconds */
+
+/*
+ * The top bit of the type in an operation message header indicates
+ * whether the message is a request (bit clear) or response (bit set)
+ */
+#define GB_MESSAGE_TYPE_RESPONSE	((u8)0x80)
+
+enum gb_operation_result {
+	GB_OP_SUCCESS		= 0x00,
+	GB_OP_INTERRUPTED	= 0x01,
+	GB_OP_TIMEOUT		= 0x02,
+	GB_OP_NO_MEMORY		= 0x03,
+	GB_OP_PROTOCOL_BAD	= 0x04,
+	GB_OP_OVERFLOW		= 0x05,
+	GB_OP_INVALID		= 0x06,
+	GB_OP_RETRY		= 0x07,
+	GB_OP_NONEXISTENT	= 0x08,
+	GB_OP_UNKNOWN_ERROR	= 0xfe,
+	GB_OP_MALFUNCTION	= 0xff,
+};
+
+#define GB_OPERATION_MESSAGE_SIZE_MIN	sizeof(struct gb_operation_msg_hdr)
+#define GB_OPERATION_MESSAGE_SIZE_MAX	U16_MAX
+
+/*
+ * Protocol code should only examine the payload and payload_size fields, and
+ * host-controller drivers may use the hcpriv field. All other fields are
+ * intended to be private to the operations core code.
+ */
+struct gb_message {
+	struct gb_operation		*operation;
+	struct gb_operation_msg_hdr	*header;
+
+	void				*payload;
+	size_t				payload_size;
+
+	void				*buffer;
+
+	void				*hcpriv;
+};
+
+#define GB_OPERATION_FLAG_INCOMING		BIT(0)
+#define GB_OPERATION_FLAG_UNIDIRECTIONAL	BIT(1)
+#define GB_OPERATION_FLAG_SHORT_RESPONSE	BIT(2)
+#define GB_OPERATION_FLAG_CORE			BIT(3)
+
+#define GB_OPERATION_FLAG_USER_MASK	(GB_OPERATION_FLAG_SHORT_RESPONSE | \
+					 GB_OPERATION_FLAG_UNIDIRECTIONAL)
+
+/*
+ * A Greybus operation is a remote procedure call performed over a
+ * connection between two UniPro interfaces.
+ *
+ * Every operation consists of a request message sent to the other
+ * end of the connection coupled with a reply message returned to
+ * the sender.  Every operation has a type, whose interpretation is
+ * dependent on the protocol associated with the connection.
+ *
+ * Only four things in an operation structure are intended to be
+ * directly usable by protocol handlers:  the operation's connection
+ * pointer; the operation type; the request message payload (and
+ * size); and the response message payload (and size).  Note that a
+ * message with a 0-byte payload has a null message payload pointer.
+ *
+ * In addition, every operation has a result, which is an errno
+ * value.  Protocol handlers access the operation result using
+ * gb_operation_result().
+ */
+typedef void (*gb_operation_callback)(struct gb_operation *);
+struct gb_operation {
+	struct gb_connection	*connection;
+	struct gb_message	*request;
+	struct gb_message	*response;
+
+	unsigned long		flags;
+	u8			type;
+	u16			id;
+	int			errno;		/* Operation result */
+
+	struct work_struct	work;
+	gb_operation_callback	callback;
+	struct completion	completion;
+
+	struct kref		kref;
+	atomic_t		waiters;
+
+	int			active;
+	struct list_head	links;		/* connection->operations */
+};
+
+static inline bool
+gb_operation_is_incoming(struct gb_operation *operation)
+{
+	return operation->flags & GB_OPERATION_FLAG_INCOMING;
+}
+
+static inline bool
+gb_operation_is_unidirectional(struct gb_operation *operation)
+{
+	return operation->flags & GB_OPERATION_FLAG_UNIDIRECTIONAL;
+}
+
+static inline bool
+gb_operation_short_response_allowed(struct gb_operation *operation)
+{
+	return operation->flags & GB_OPERATION_FLAG_SHORT_RESPONSE;
+}
+
+static inline bool gb_operation_is_core(struct gb_operation *operation)
+{
+	return operation->flags & GB_OPERATION_FLAG_CORE;
+}
+
+void gb_connection_recv(struct gb_connection *connection,
+					void *data, size_t size);
+
+int gb_operation_result(struct gb_operation *operation);
+
+size_t gb_operation_get_payload_size_max(struct gb_connection *connection);
+struct gb_operation *
+gb_operation_create_flags(struct gb_connection *connection,
+				u8 type, size_t request_size,
+				size_t response_size, unsigned long flags,
+				gfp_t gfp);
+
+static inline struct gb_operation *
+gb_operation_create(struct gb_connection *connection,
+				u8 type, size_t request_size,
+				size_t response_size, gfp_t gfp)
+{
+	return gb_operation_create_flags(connection, type, request_size,
+						response_size, 0, gfp);
+}
+
+struct gb_operation *
+gb_operation_create_core(struct gb_connection *connection,
+				u8 type, size_t request_size,
+				size_t response_size, unsigned long flags,
+				gfp_t gfp);
+
+void gb_operation_get(struct gb_operation *operation);
+void gb_operation_put(struct gb_operation *operation);
+
+bool gb_operation_response_alloc(struct gb_operation *operation,
+					size_t response_size, gfp_t gfp);
+
+int gb_operation_request_send(struct gb_operation *operation,
+				gb_operation_callback callback,
+				gfp_t gfp);
+int gb_operation_request_send_sync_timeout(struct gb_operation *operation,
+						unsigned int timeout);
+static inline int
+gb_operation_request_send_sync(struct gb_operation *operation)
+{
+	return gb_operation_request_send_sync_timeout(operation,
+			GB_OPERATION_TIMEOUT_DEFAULT);
+}
+
+void gb_operation_cancel(struct gb_operation *operation, int errno);
+void gb_operation_cancel_incoming(struct gb_operation *operation, int errno);
+
+void greybus_message_sent(struct gb_host_device *hd,
+				struct gb_message *message, int status);
+
+int gb_operation_sync_timeout(struct gb_connection *connection, int type,
+				void *request, int request_size,
+				void *response, int response_size,
+				unsigned int timeout);
+int gb_operation_unidirectional_timeout(struct gb_connection *connection,
+				int type, void *request, int request_size,
+				unsigned int timeout);
+
+static inline int gb_operation_sync(struct gb_connection *connection, int type,
+		      void *request, int request_size,
+		      void *response, int response_size)
+{
+	return gb_operation_sync_timeout(connection, type,
+			request, request_size, response, response_size,
+			GB_OPERATION_TIMEOUT_DEFAULT);
+}
+
+static inline int gb_operation_unidirectional(struct gb_connection *connection,
+				int type, void *request, int request_size)
+{
+	return gb_operation_unidirectional_timeout(connection, type,
+			request, request_size, GB_OPERATION_TIMEOUT_DEFAULT);
+}
+
+int gb_operation_init(void);
+void gb_operation_exit(void);
+
+#endif /* !__OPERATION_H */
diff --git a/drivers/staging/greybus/power_supply.c b/drivers/staging/greybus/power_supply.c
new file mode 100644
index 000000000000..e85c988b7034
--- /dev/null
+++ b/drivers/staging/greybus/power_supply.c
@@ -0,0 +1,1141 @@
+/*
+ * Power Supply driver for a Greybus module.
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/power_supply.h>
+#include <linux/slab.h>
+
+#include "greybus.h"
+
+#define PROP_MAX 32
+
+struct gb_power_supply_prop {
+	enum power_supply_property	prop;
+	u8				gb_prop;
+	int				val;
+	int				previous_val;
+	bool				is_writeable;
+};
+
+struct gb_power_supply {
+	u8				id;
+	bool				registered;
+	struct power_supply		*psy;
+	struct power_supply_desc	desc;
+	char				name[64];
+	struct gb_power_supplies	*supplies;
+	struct delayed_work		work;
+	char				*manufacturer;
+	char				*model_name;
+	char				*serial_number;
+	u8				type;
+	u8				properties_count;
+	u8				properties_count_str;
+	unsigned long			last_update;
+	u8				cache_invalid;
+	unsigned int			update_interval;
+	bool				changed;
+	struct gb_power_supply_prop	*props;
+	enum power_supply_property	*props_raw;
+	bool				pm_acquired;
+	struct mutex			supply_lock;
+};
+
+struct gb_power_supplies {
+	struct gb_connection	*connection;
+	u8			supplies_count;
+	struct gb_power_supply	*supply;
+	struct mutex		supplies_lock;
+};
+
+#define to_gb_power_supply(x) power_supply_get_drvdata(x)
+
+/*
+ * General power supply properties that could be absent from various reasons,
+ * like kernel versions or vendor specific versions
+ */
+#ifndef POWER_SUPPLY_PROP_VOLTAGE_BOOT
+	#define POWER_SUPPLY_PROP_VOLTAGE_BOOT	-1
+#endif
+#ifndef POWER_SUPPLY_PROP_CURRENT_BOOT
+	#define POWER_SUPPLY_PROP_CURRENT_BOOT	-1
+#endif
+#ifndef POWER_SUPPLY_PROP_CALIBRATE
+	#define POWER_SUPPLY_PROP_CALIBRATE	-1
+#endif
+
+/* cache time in milliseconds, if cache_time is set to 0 cache is disable */
+static unsigned int cache_time = 1000;
+/*
+ * update interval initial and maximum value, between the two will
+ * back-off exponential
+ */
+static unsigned int update_interval_init = 1 * HZ;
+static unsigned int update_interval_max = 30 * HZ;
+
+struct gb_power_supply_changes {
+	enum power_supply_property	prop;
+	u32				tolerance_change;
+	void (*prop_changed)(struct gb_power_supply *gbpsy,
+			     struct gb_power_supply_prop *prop);
+};
+
+static void gb_power_supply_state_change(struct gb_power_supply *gbpsy,
+					 struct gb_power_supply_prop *prop);
+
+static const struct gb_power_supply_changes psy_props_changes[] = {
+	{	.prop			= GB_POWER_SUPPLY_PROP_STATUS,
+		.tolerance_change	= 0,
+		.prop_changed		= gb_power_supply_state_change,
+	},
+	{	.prop			= GB_POWER_SUPPLY_PROP_TEMP,
+		.tolerance_change	= 500,
+		.prop_changed		= NULL,
+	},
+	{	.prop			= GB_POWER_SUPPLY_PROP_ONLINE,
+		.tolerance_change	= 0,
+		.prop_changed		= NULL,
+	},
+};
+
+static int get_psp_from_gb_prop(int gb_prop, enum power_supply_property *psp)
+{
+	int prop;
+
+	switch (gb_prop) {
+	case GB_POWER_SUPPLY_PROP_STATUS:
+		prop = POWER_SUPPLY_PROP_STATUS;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_TYPE:
+		prop = POWER_SUPPLY_PROP_CHARGE_TYPE;
+		break;
+	case GB_POWER_SUPPLY_PROP_HEALTH:
+		prop = POWER_SUPPLY_PROP_HEALTH;
+		break;
+	case GB_POWER_SUPPLY_PROP_PRESENT:
+		prop = POWER_SUPPLY_PROP_PRESENT;
+		break;
+	case GB_POWER_SUPPLY_PROP_ONLINE:
+		prop = POWER_SUPPLY_PROP_ONLINE;
+		break;
+	case GB_POWER_SUPPLY_PROP_AUTHENTIC:
+		prop = POWER_SUPPLY_PROP_AUTHENTIC;
+		break;
+	case GB_POWER_SUPPLY_PROP_TECHNOLOGY:
+		prop = POWER_SUPPLY_PROP_TECHNOLOGY;
+		break;
+	case GB_POWER_SUPPLY_PROP_CYCLE_COUNT:
+		prop = POWER_SUPPLY_PROP_CYCLE_COUNT;
+		break;
+	case GB_POWER_SUPPLY_PROP_VOLTAGE_MAX:
+		prop = POWER_SUPPLY_PROP_VOLTAGE_MAX;
+		break;
+	case GB_POWER_SUPPLY_PROP_VOLTAGE_MIN:
+		prop = POWER_SUPPLY_PROP_VOLTAGE_MIN;
+		break;
+	case GB_POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN:
+		prop = POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN;
+		break;
+	case GB_POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN:
+		prop = POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN;
+		break;
+	case GB_POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		prop = POWER_SUPPLY_PROP_VOLTAGE_NOW;
+		break;
+	case GB_POWER_SUPPLY_PROP_VOLTAGE_AVG:
+		prop = POWER_SUPPLY_PROP_VOLTAGE_AVG;
+		break;
+	case GB_POWER_SUPPLY_PROP_VOLTAGE_OCV:
+		prop = POWER_SUPPLY_PROP_VOLTAGE_OCV;
+		break;
+	case GB_POWER_SUPPLY_PROP_VOLTAGE_BOOT:
+		prop = POWER_SUPPLY_PROP_VOLTAGE_BOOT;
+		break;
+	case GB_POWER_SUPPLY_PROP_CURRENT_MAX:
+		prop = POWER_SUPPLY_PROP_CURRENT_MAX;
+		break;
+	case GB_POWER_SUPPLY_PROP_CURRENT_NOW:
+		prop = POWER_SUPPLY_PROP_CURRENT_NOW;
+		break;
+	case GB_POWER_SUPPLY_PROP_CURRENT_AVG:
+		prop = POWER_SUPPLY_PROP_CURRENT_AVG;
+		break;
+	case GB_POWER_SUPPLY_PROP_CURRENT_BOOT:
+		prop = POWER_SUPPLY_PROP_CURRENT_BOOT;
+		break;
+	case GB_POWER_SUPPLY_PROP_POWER_NOW:
+		prop = POWER_SUPPLY_PROP_POWER_NOW;
+		break;
+	case GB_POWER_SUPPLY_PROP_POWER_AVG:
+		prop = POWER_SUPPLY_PROP_POWER_AVG;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN:
+		prop = POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_EMPTY_DESIGN:
+		prop = POWER_SUPPLY_PROP_CHARGE_EMPTY_DESIGN;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_FULL:
+		prop = POWER_SUPPLY_PROP_CHARGE_FULL;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_EMPTY:
+		prop = POWER_SUPPLY_PROP_CHARGE_EMPTY;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_NOW:
+		prop = POWER_SUPPLY_PROP_CHARGE_NOW;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_AVG:
+		prop = POWER_SUPPLY_PROP_CHARGE_AVG;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_COUNTER:
+		prop = POWER_SUPPLY_PROP_CHARGE_COUNTER;
+		break;
+	case GB_POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT:
+		prop = POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT;
+		break;
+	case GB_POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX:
+		prop = POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX;
+		break;
+	case GB_POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE:
+		prop = POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE;
+		break;
+	case GB_POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX:
+		prop = POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT:
+		prop = POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT_MAX:
+		prop = POWER_SUPPLY_PROP_CHARGE_CONTROL_LIMIT_MAX;
+		break;
+	case GB_POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
+		prop = POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT;
+		break;
+	case GB_POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN:
+		prop = POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN;
+		break;
+	case GB_POWER_SUPPLY_PROP_ENERGY_EMPTY_DESIGN:
+		prop = POWER_SUPPLY_PROP_ENERGY_EMPTY_DESIGN;
+		break;
+	case GB_POWER_SUPPLY_PROP_ENERGY_FULL:
+		prop = POWER_SUPPLY_PROP_ENERGY_FULL;
+		break;
+	case GB_POWER_SUPPLY_PROP_ENERGY_EMPTY:
+		prop = POWER_SUPPLY_PROP_ENERGY_EMPTY;
+		break;
+	case GB_POWER_SUPPLY_PROP_ENERGY_NOW:
+		prop = POWER_SUPPLY_PROP_ENERGY_NOW;
+		break;
+	case GB_POWER_SUPPLY_PROP_ENERGY_AVG:
+		prop = POWER_SUPPLY_PROP_ENERGY_AVG;
+		break;
+	case GB_POWER_SUPPLY_PROP_CAPACITY:
+		prop = POWER_SUPPLY_PROP_CAPACITY;
+		break;
+	case GB_POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN:
+		prop = POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN;
+		break;
+	case GB_POWER_SUPPLY_PROP_CAPACITY_ALERT_MAX:
+		prop = POWER_SUPPLY_PROP_CAPACITY_ALERT_MAX;
+		break;
+	case GB_POWER_SUPPLY_PROP_CAPACITY_LEVEL:
+		prop = POWER_SUPPLY_PROP_CAPACITY_LEVEL;
+		break;
+	case GB_POWER_SUPPLY_PROP_TEMP:
+		prop = POWER_SUPPLY_PROP_TEMP;
+		break;
+	case GB_POWER_SUPPLY_PROP_TEMP_MAX:
+		prop = POWER_SUPPLY_PROP_TEMP_MAX;
+		break;
+	case GB_POWER_SUPPLY_PROP_TEMP_MIN:
+		prop = POWER_SUPPLY_PROP_TEMP_MIN;
+		break;
+	case GB_POWER_SUPPLY_PROP_TEMP_ALERT_MIN:
+		prop = POWER_SUPPLY_PROP_TEMP_ALERT_MIN;
+		break;
+	case GB_POWER_SUPPLY_PROP_TEMP_ALERT_MAX:
+		prop = POWER_SUPPLY_PROP_TEMP_ALERT_MAX;
+		break;
+	case GB_POWER_SUPPLY_PROP_TEMP_AMBIENT:
+		prop = POWER_SUPPLY_PROP_TEMP_AMBIENT;
+		break;
+	case GB_POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MIN:
+		prop = POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MIN;
+		break;
+	case GB_POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MAX:
+		prop = POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MAX;
+		break;
+	case GB_POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW:
+		prop = POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW;
+		break;
+	case GB_POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG:
+		prop = POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG;
+		break;
+	case GB_POWER_SUPPLY_PROP_TIME_TO_FULL_NOW:
+		prop = POWER_SUPPLY_PROP_TIME_TO_FULL_NOW;
+		break;
+	case GB_POWER_SUPPLY_PROP_TIME_TO_FULL_AVG:
+		prop = POWER_SUPPLY_PROP_TIME_TO_FULL_AVG;
+		break;
+	case GB_POWER_SUPPLY_PROP_TYPE:
+		prop = POWER_SUPPLY_PROP_TYPE;
+		break;
+	case GB_POWER_SUPPLY_PROP_SCOPE:
+		prop = POWER_SUPPLY_PROP_SCOPE;
+		break;
+	case GB_POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
+		prop = POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT;
+		break;
+	case GB_POWER_SUPPLY_PROP_CALIBRATE:
+		prop = POWER_SUPPLY_PROP_CALIBRATE;
+		break;
+	default:
+		prop = -1;
+		break;
+	}
+
+	if (prop < 0)
+		return prop;
+
+	*psp = (enum power_supply_property)prop;
+
+	return 0;
+}
+
+static struct gb_connection *get_conn_from_psy(struct gb_power_supply *gbpsy)
+{
+	return gbpsy->supplies->connection;
+}
+
+static struct gb_power_supply_prop *get_psy_prop(struct gb_power_supply *gbpsy,
+						 enum power_supply_property psp)
+{
+	int i;
+
+	for (i = 0; i < gbpsy->properties_count; i++)
+		if (gbpsy->props[i].prop == psp)
+			return &gbpsy->props[i];
+	return NULL;
+}
+
+static int is_psy_prop_writeable(struct gb_power_supply *gbpsy,
+				     enum power_supply_property psp)
+{
+	struct gb_power_supply_prop *prop;
+
+	prop = get_psy_prop(gbpsy, psp);
+	if (!prop)
+		return -ENOENT;
+	return prop->is_writeable ? 1 : 0;
+}
+
+static int is_prop_valint(enum power_supply_property psp)
+{
+	return ((psp < POWER_SUPPLY_PROP_MODEL_NAME) ? 1 : 0);
+}
+
+static void next_interval(struct gb_power_supply *gbpsy)
+{
+	if (gbpsy->update_interval == update_interval_max)
+		return;
+
+	/* do some exponential back-off in the update interval */
+	gbpsy->update_interval *= 2;
+	if (gbpsy->update_interval > update_interval_max)
+		gbpsy->update_interval = update_interval_max;
+}
+
+static void __gb_power_supply_changed(struct gb_power_supply *gbpsy)
+{
+	power_supply_changed(gbpsy->psy);
+}
+
+static void gb_power_supply_state_change(struct gb_power_supply *gbpsy,
+					 struct gb_power_supply_prop *prop)
+{
+	struct gb_connection *connection = get_conn_from_psy(gbpsy);
+	int ret;
+
+	/*
+	 * Check gbpsy->pm_acquired to make sure only one pair of 'get_sync'
+	 * and 'put_autosuspend' runtime pm call for state property change.
+	 */
+	mutex_lock(&gbpsy->supply_lock);
+
+	if ((prop->val == GB_POWER_SUPPLY_STATUS_CHARGING) &&
+	    !gbpsy->pm_acquired) {
+		ret = gb_pm_runtime_get_sync(connection->bundle);
+		if (ret)
+			dev_err(&connection->bundle->dev,
+				"Fail to set wake lock for charging state\n");
+		else
+			gbpsy->pm_acquired = true;
+	} else {
+		if (gbpsy->pm_acquired) {
+			ret = gb_pm_runtime_put_autosuspend(connection->bundle);
+			if (ret)
+				dev_err(&connection->bundle->dev,
+					"Fail to set wake unlock for none charging\n");
+			else
+				gbpsy->pm_acquired = false;
+		}
+	}
+
+	mutex_unlock(&gbpsy->supply_lock);
+}
+
+static void check_changed(struct gb_power_supply *gbpsy,
+			  struct gb_power_supply_prop *prop)
+{
+	const struct gb_power_supply_changes *psyc;
+	int val = prop->val;
+	int prev_val = prop->previous_val;
+	bool changed = false;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(psy_props_changes); i++) {
+		psyc = &psy_props_changes[i];
+		if (prop->prop == psyc->prop) {
+			if (!psyc->tolerance_change)
+				changed = true;
+			else if (val < prev_val &&
+				 prev_val - val > psyc->tolerance_change)
+				changed = true;
+			else if (val > prev_val &&
+				 val - prev_val > psyc->tolerance_change)
+				changed = true;
+
+			if (changed && psyc->prop_changed)
+				psyc->prop_changed(gbpsy, prop);
+
+			if (changed)
+				gbpsy->changed = true;
+			break;
+		}
+	}
+}
+
+static int total_props(struct gb_power_supply *gbpsy)
+{
+	/* this return the intval plus the strval properties */
+	return (gbpsy->properties_count + gbpsy->properties_count_str);
+}
+
+static void prop_append(struct gb_power_supply *gbpsy,
+			enum power_supply_property prop)
+{
+	enum power_supply_property *new_props_raw;
+
+	gbpsy->properties_count_str++;
+	new_props_raw = krealloc(gbpsy->props_raw, total_props(gbpsy) *
+				 sizeof(enum power_supply_property),
+				 GFP_KERNEL);
+	if (!new_props_raw)
+		return;
+	gbpsy->props_raw = new_props_raw;
+	gbpsy->props_raw[total_props(gbpsy) - 1] = prop;
+}
+
+static int __gb_power_supply_set_name(char *init_name, char *name, size_t len)
+{
+	unsigned int i = 0;
+	int ret = 0;
+	struct power_supply *psy;
+
+	if (!strlen(init_name))
+		init_name = "gb_power_supply";
+	strlcpy(name, init_name, len);
+
+	while ((ret < len) && (psy = power_supply_get_by_name(name))) {
+		power_supply_put(psy);
+
+		ret = snprintf(name, len, "%s_%u", init_name, ++i);
+	}
+	if (ret >= len)
+		return -ENOMEM;
+	return i;
+}
+
+static void _gb_power_supply_append_props(struct gb_power_supply *gbpsy)
+{
+	if (strlen(gbpsy->manufacturer))
+		prop_append(gbpsy, POWER_SUPPLY_PROP_MANUFACTURER);
+	if (strlen(gbpsy->model_name))
+		prop_append(gbpsy, POWER_SUPPLY_PROP_MODEL_NAME);
+	if (strlen(gbpsy->serial_number))
+		prop_append(gbpsy, POWER_SUPPLY_PROP_SERIAL_NUMBER);
+}
+
+static int gb_power_supply_description_get(struct gb_power_supply *gbpsy)
+{
+	struct gb_connection *connection = get_conn_from_psy(gbpsy);
+	struct gb_power_supply_get_description_request req;
+	struct gb_power_supply_get_description_response resp;
+	int ret;
+
+	req.psy_id = gbpsy->id;
+
+	ret = gb_operation_sync(connection,
+				GB_POWER_SUPPLY_TYPE_GET_DESCRIPTION,
+				&req, sizeof(req), &resp, sizeof(resp));
+	if (ret < 0)
+		return ret;
+
+	gbpsy->manufacturer = kstrndup(resp.manufacturer, PROP_MAX, GFP_KERNEL);
+	if (!gbpsy->manufacturer)
+		return -ENOMEM;
+	gbpsy->model_name = kstrndup(resp.model, PROP_MAX, GFP_KERNEL);
+	if (!gbpsy->model_name)
+		return -ENOMEM;
+	gbpsy->serial_number = kstrndup(resp.serial_number, PROP_MAX,
+				       GFP_KERNEL);
+	if (!gbpsy->serial_number)
+		return -ENOMEM;
+
+	gbpsy->type = le16_to_cpu(resp.type);
+	gbpsy->properties_count = resp.properties_count;
+
+	return 0;
+}
+
+static int gb_power_supply_prop_descriptors_get(struct gb_power_supply *gbpsy)
+{
+	struct gb_connection *connection = get_conn_from_psy(gbpsy);
+	struct gb_power_supply_get_property_descriptors_request *req;
+	struct gb_power_supply_get_property_descriptors_response *resp;
+	struct gb_operation *op;
+	u8 props_count = gbpsy->properties_count;
+	enum power_supply_property psp;
+	int ret;
+	int i, r = 0;
+
+	if (props_count == 0)
+		return 0;
+
+	op = gb_operation_create(connection,
+				 GB_POWER_SUPPLY_TYPE_GET_PROP_DESCRIPTORS,
+				 sizeof(req), sizeof(*resp) + props_count *
+				 sizeof(struct gb_power_supply_props_desc),
+				 GFP_KERNEL);
+	if (!op)
+		return -ENOMEM;
+
+	req = op->request->payload;
+	req->psy_id = gbpsy->id;
+
+	ret = gb_operation_request_send_sync(op);
+	if (ret < 0)
+		goto out_put_operation;
+
+	resp = op->response->payload;
+
+	/* validate received properties */
+	for (i = 0; i < props_count; i++) {
+		ret = get_psp_from_gb_prop(resp->props[i].property, &psp);
+		if (ret < 0) {
+			dev_warn(&connection->bundle->dev,
+				 "greybus property %u it is not supported by this kernel, dropped\n",
+				 resp->props[i].property);
+			gbpsy->properties_count--;
+		}
+	}
+
+	gbpsy->props = kcalloc(gbpsy->properties_count, sizeof(*gbpsy->props),
+			      GFP_KERNEL);
+	if (!gbpsy->props) {
+		ret = -ENOMEM;
+		goto out_put_operation;
+	}
+
+	gbpsy->props_raw = kcalloc(gbpsy->properties_count,
+				   sizeof(*gbpsy->props_raw), GFP_KERNEL);
+	if (!gbpsy->props_raw) {
+		ret = -ENOMEM;
+		goto out_put_operation;
+	}
+
+	/* Store available properties, skip the ones we do not support */
+	for (i = 0; i < props_count; i++) {
+		ret = get_psp_from_gb_prop(resp->props[i].property, &psp);
+		if (ret < 0) {
+			r++;
+			continue;
+		}
+		gbpsy->props[i - r].prop = psp;
+		gbpsy->props[i - r].gb_prop = resp->props[i].property;
+		gbpsy->props_raw[i - r] = psp;
+		if (resp->props[i].is_writeable)
+			gbpsy->props[i - r].is_writeable = true;
+	}
+
+	/*
+	 * now append the properties that we already got information in the
+	 * get_description operation. (char * ones)
+	 */
+	_gb_power_supply_append_props(gbpsy);
+
+	ret = 0;
+out_put_operation:
+	gb_operation_put(op);
+
+	return ret;
+}
+
+static int __gb_power_supply_property_update(struct gb_power_supply *gbpsy,
+					     enum power_supply_property psp)
+{
+	struct gb_connection *connection = get_conn_from_psy(gbpsy);
+	struct gb_power_supply_prop *prop;
+	struct gb_power_supply_get_property_request req;
+	struct gb_power_supply_get_property_response resp;
+	int val;
+	int ret;
+
+	prop = get_psy_prop(gbpsy, psp);
+	if (!prop)
+		return -EINVAL;
+	req.psy_id = gbpsy->id;
+	req.property = prop->gb_prop;
+
+	ret = gb_operation_sync(connection, GB_POWER_SUPPLY_TYPE_GET_PROPERTY,
+				&req, sizeof(req), &resp, sizeof(resp));
+	if (ret < 0)
+		return ret;
+
+	val = le32_to_cpu(resp.prop_val);
+	if (val == prop->val)
+		return 0;
+
+	prop->previous_val = prop->val;
+	prop->val = val;
+
+	check_changed(gbpsy, prop);
+
+	return 0;
+}
+
+static int __gb_power_supply_property_get(struct gb_power_supply *gbpsy,
+					  enum power_supply_property psp,
+					  union power_supply_propval *val)
+{
+	struct gb_power_supply_prop *prop;
+
+	prop = get_psy_prop(gbpsy, psp);
+	if (!prop)
+		return -EINVAL;
+
+	val->intval = prop->val;
+	return 0;
+}
+
+static int __gb_power_supply_property_strval_get(struct gb_power_supply *gbpsy,
+						enum power_supply_property psp,
+						union power_supply_propval *val)
+{
+	switch (psp) {
+	case POWER_SUPPLY_PROP_MODEL_NAME:
+		val->strval = gbpsy->model_name;
+		break;
+	case POWER_SUPPLY_PROP_MANUFACTURER:
+		val->strval = gbpsy->manufacturer;
+		break;
+	case POWER_SUPPLY_PROP_SERIAL_NUMBER:
+		val->strval = gbpsy->serial_number;
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int _gb_power_supply_property_get(struct gb_power_supply *gbpsy,
+					 enum power_supply_property psp,
+					 union power_supply_propval *val)
+{
+	struct gb_connection *connection = get_conn_from_psy(gbpsy);
+	int ret;
+
+	/*
+	 * Properties of type const char *, were already fetched on
+	 * get_description operation and should be cached in gb
+	 */
+	if (is_prop_valint(psp))
+		ret = __gb_power_supply_property_get(gbpsy, psp, val);
+	else
+		ret = __gb_power_supply_property_strval_get(gbpsy, psp, val);
+
+	if (ret < 0)
+		dev_err(&connection->bundle->dev, "get property %u\n", psp);
+
+	return 0;
+}
+
+static int is_cache_valid(struct gb_power_supply *gbpsy)
+{
+	/* check if cache is good enough or it has expired */
+	if (gbpsy->cache_invalid) {
+		gbpsy->cache_invalid = 0;
+		return 0;
+	}
+
+	if (gbpsy->last_update &&
+	    time_is_after_jiffies(gbpsy->last_update +
+				  msecs_to_jiffies(cache_time)))
+		return 1;
+
+	return 0;
+}
+
+static int gb_power_supply_status_get(struct gb_power_supply *gbpsy)
+{
+	struct gb_connection *connection = get_conn_from_psy(gbpsy);
+	int ret = 0;
+	int i;
+
+	if (is_cache_valid(gbpsy))
+		return 0;
+
+	ret = gb_pm_runtime_get_sync(connection->bundle);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < gbpsy->properties_count; i++) {
+		ret = __gb_power_supply_property_update(gbpsy,
+							gbpsy->props[i].prop);
+		if (ret < 0)
+			break;
+	}
+
+	if (ret == 0)
+		gbpsy->last_update = jiffies;
+
+	gb_pm_runtime_put_autosuspend(connection->bundle);
+	return ret;
+}
+
+static void gb_power_supply_status_update(struct gb_power_supply *gbpsy)
+{
+	/* check if there a change that need to be reported */
+	gb_power_supply_status_get(gbpsy);
+
+	if (!gbpsy->changed)
+		return;
+
+	gbpsy->update_interval = update_interval_init;
+	__gb_power_supply_changed(gbpsy);
+	gbpsy->changed = false;
+}
+
+static void gb_power_supply_work(struct work_struct *work)
+{
+	struct gb_power_supply *gbpsy = container_of(work,
+						     struct gb_power_supply,
+						     work.work);
+
+	/*
+	 * if the poll interval is not set, disable polling, this is helpful
+	 * specially at unregister time.
+	 */
+	if (!gbpsy->update_interval)
+		return;
+
+	gb_power_supply_status_update(gbpsy);
+	next_interval(gbpsy);
+	schedule_delayed_work(&gbpsy->work, gbpsy->update_interval);
+}
+
+static int get_property(struct power_supply *b,
+			enum power_supply_property psp,
+			union power_supply_propval *val)
+{
+	struct gb_power_supply *gbpsy = to_gb_power_supply(b);
+
+	gb_power_supply_status_get(gbpsy);
+
+	return _gb_power_supply_property_get(gbpsy, psp, val);
+}
+
+static int gb_power_supply_property_set(struct gb_power_supply *gbpsy,
+					enum power_supply_property psp,
+					int val)
+{
+	struct gb_connection *connection = get_conn_from_psy(gbpsy);
+	struct gb_power_supply_prop *prop;
+	struct gb_power_supply_set_property_request req;
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(connection->bundle);
+	if (ret)
+		return ret;
+
+	prop = get_psy_prop(gbpsy, psp);
+	if (!prop) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	req.psy_id = gbpsy->id;
+	req.property = prop->gb_prop;
+	req.prop_val = cpu_to_le32((s32)val);
+
+	ret = gb_operation_sync(connection, GB_POWER_SUPPLY_TYPE_SET_PROPERTY,
+				&req, sizeof(req), NULL, 0);
+	if (ret < 0)
+		goto out;
+
+	/* cache immediately the new value */
+	prop->val = val;
+
+out:
+	gb_pm_runtime_put_autosuspend(connection->bundle);
+	return ret;
+}
+
+static int set_property(struct power_supply *b,
+			enum power_supply_property psp,
+			const union power_supply_propval *val)
+{
+	struct gb_power_supply *gbpsy = to_gb_power_supply(b);
+
+	return gb_power_supply_property_set(gbpsy, psp, val->intval);
+}
+
+static int property_is_writeable(struct power_supply *b,
+				 enum power_supply_property psp)
+{
+	struct gb_power_supply *gbpsy = to_gb_power_supply(b);
+
+	return is_psy_prop_writeable(gbpsy, psp);
+}
+
+static int gb_power_supply_register(struct gb_power_supply *gbpsy)
+{
+	struct gb_connection *connection = get_conn_from_psy(gbpsy);
+	struct power_supply_config cfg = {};
+
+	cfg.drv_data = gbpsy;
+
+	gbpsy->desc.name		= gbpsy->name;
+	gbpsy->desc.type		= gbpsy->type;
+	gbpsy->desc.properties		= gbpsy->props_raw;
+	gbpsy->desc.num_properties	= total_props(gbpsy);
+	gbpsy->desc.get_property	= get_property;
+	gbpsy->desc.set_property	= set_property;
+	gbpsy->desc.property_is_writeable = property_is_writeable;
+
+	gbpsy->psy = power_supply_register(&connection->bundle->dev,
+					   &gbpsy->desc, &cfg);
+	return PTR_ERR_OR_ZERO(gbpsy->psy);
+}
+
+static void _gb_power_supply_free(struct gb_power_supply *gbpsy)
+{
+	kfree(gbpsy->serial_number);
+	kfree(gbpsy->model_name);
+	kfree(gbpsy->manufacturer);
+	kfree(gbpsy->props_raw);
+	kfree(gbpsy->props);
+}
+
+static void _gb_power_supply_release(struct gb_power_supply *gbpsy)
+{
+	gbpsy->update_interval = 0;
+
+	cancel_delayed_work_sync(&gbpsy->work);
+
+	if (gbpsy->registered)
+		power_supply_unregister(gbpsy->psy);
+
+	_gb_power_supply_free(gbpsy);
+}
+
+static void _gb_power_supplies_release(struct gb_power_supplies *supplies)
+{
+	int i;
+
+	if (!supplies->supply)
+		return;
+
+	mutex_lock(&supplies->supplies_lock);
+	for (i = 0; i < supplies->supplies_count; i++)
+		_gb_power_supply_release(&supplies->supply[i]);
+	kfree(supplies->supply);
+	mutex_unlock(&supplies->supplies_lock);
+	kfree(supplies);
+}
+
+static int gb_power_supplies_get_count(struct gb_power_supplies *supplies)
+{
+	struct gb_power_supply_get_supplies_response resp;
+	int ret;
+
+	ret = gb_operation_sync(supplies->connection,
+				GB_POWER_SUPPLY_TYPE_GET_SUPPLIES,
+				NULL, 0, &resp, sizeof(resp));
+	if (ret < 0)
+		return ret;
+
+	if  (!resp.supplies_count)
+		return -EINVAL;
+
+	supplies->supplies_count = resp.supplies_count;
+
+	return ret;
+}
+
+static int gb_power_supply_config(struct gb_power_supplies *supplies, int id)
+{
+	struct gb_power_supply *gbpsy = &supplies->supply[id];
+	int ret;
+
+	gbpsy->supplies = supplies;
+	gbpsy->id = id;
+
+	ret = gb_power_supply_description_get(gbpsy);
+	if (ret < 0)
+		return ret;
+
+	return gb_power_supply_prop_descriptors_get(gbpsy);
+}
+
+static int gb_power_supply_enable(struct gb_power_supply *gbpsy)
+{
+	int ret;
+
+	/* guarantee that we have an unique name, before register */
+	ret =  __gb_power_supply_set_name(gbpsy->model_name, gbpsy->name,
+					  sizeof(gbpsy->name));
+	if (ret < 0)
+		return ret;
+
+	mutex_init(&gbpsy->supply_lock);
+
+	ret = gb_power_supply_register(gbpsy);
+	if (ret < 0)
+		return ret;
+
+	gbpsy->update_interval = update_interval_init;
+	INIT_DELAYED_WORK(&gbpsy->work, gb_power_supply_work);
+	schedule_delayed_work(&gbpsy->work, 0);
+
+	/* everything went fine, mark it for release code to know */
+	gbpsy->registered = true;
+
+	return 0;
+}
+
+static int gb_power_supplies_setup(struct gb_power_supplies *supplies)
+{
+	struct gb_connection *connection = supplies->connection;
+	int ret;
+	int i;
+
+	mutex_lock(&supplies->supplies_lock);
+
+	ret = gb_power_supplies_get_count(supplies);
+	if (ret < 0)
+		goto out;
+
+	supplies->supply = kzalloc(supplies->supplies_count *
+				     sizeof(struct gb_power_supply),
+				     GFP_KERNEL);
+
+	if (!supplies->supply) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	for (i = 0; i < supplies->supplies_count; i++) {
+		ret = gb_power_supply_config(supplies, i);
+		if (ret < 0) {
+			dev_err(&connection->bundle->dev,
+				"Fail to configure supplies devices\n");
+			goto out;
+		}
+	}
+out:
+	mutex_unlock(&supplies->supplies_lock);
+	return ret;
+}
+
+static int gb_power_supplies_register(struct gb_power_supplies *supplies)
+{
+	struct gb_connection *connection = supplies->connection;
+	int ret = 0;
+	int i;
+
+	mutex_lock(&supplies->supplies_lock);
+
+	for (i = 0; i < supplies->supplies_count; i++) {
+		ret = gb_power_supply_enable(&supplies->supply[i]);
+		if (ret < 0) {
+			dev_err(&connection->bundle->dev,
+				"Fail to enable supplies devices\n");
+			break;
+		}
+	}
+
+	mutex_unlock(&supplies->supplies_lock);
+	return ret;
+}
+
+static int gb_supplies_request_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_power_supplies *supplies = gb_connection_get_data(connection);
+	struct gb_power_supply *gbpsy;
+	struct gb_message *request;
+	struct gb_power_supply_event_request *payload;
+	u8 psy_id;
+	u8 event;
+	int ret = 0;
+
+	if (op->type != GB_POWER_SUPPLY_TYPE_EVENT) {
+		dev_err(&connection->bundle->dev,
+			"Unsupported unsolicited event: %u\n", op->type);
+		return -EINVAL;
+	}
+
+	request = op->request;
+
+	if (request->payload_size < sizeof(*payload)) {
+		dev_err(&connection->bundle->dev,
+			"Wrong event size received (%zu < %zu)\n",
+			request->payload_size, sizeof(*payload));
+		return -EINVAL;
+	}
+
+	payload = request->payload;
+	psy_id = payload->psy_id;
+	mutex_lock(&supplies->supplies_lock);
+	if (psy_id >= supplies->supplies_count ||
+	    !supplies->supply[psy_id].registered) {
+		dev_err(&connection->bundle->dev,
+			"Event received for unconfigured power_supply id: %d\n",
+			psy_id);
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	event = payload->event;
+	/*
+	 * we will only handle events after setup is done and before release is
+	 * running. For that just check update_interval.
+	 */
+	gbpsy = &supplies->supply[psy_id];
+	if (!gbpsy->update_interval) {
+		ret = -ESHUTDOWN;
+		goto out_unlock;
+	}
+
+	if (event & GB_POWER_SUPPLY_UPDATE) {
+		/*
+		 * we need to make sure we invalidate cache, if not no new
+		 * values for the properties will be fetch and the all propose
+		 * of this event is missed
+		 */
+		gbpsy->cache_invalid = 1;
+		gb_power_supply_status_update(gbpsy);
+	}
+
+out_unlock:
+	mutex_unlock(&supplies->supplies_lock);
+	return ret;
+}
+
+static int gb_power_supply_probe(struct gb_bundle *bundle,
+				 const struct greybus_bundle_id *id)
+{
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_connection *connection;
+	struct gb_power_supplies *supplies;
+	int ret;
+
+	if (bundle->num_cports != 1)
+		return -ENODEV;
+
+	cport_desc = &bundle->cport_desc[0];
+	if (cport_desc->protocol_id != GREYBUS_PROTOCOL_POWER_SUPPLY)
+		return -ENODEV;
+
+	supplies = kzalloc(sizeof(*supplies), GFP_KERNEL);
+	if (!supplies)
+		return -ENOMEM;
+
+	connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
+					  gb_supplies_request_handler);
+	if (IS_ERR(connection)) {
+		ret = PTR_ERR(connection);
+		goto out;
+	}
+
+	supplies->connection = connection;
+	gb_connection_set_data(connection, supplies);
+
+	mutex_init(&supplies->supplies_lock);
+
+	greybus_set_drvdata(bundle, supplies);
+
+	/* We aren't ready to receive an incoming request yet */
+	ret = gb_connection_enable_tx(connection);
+	if (ret)
+		goto error_connection_destroy;
+
+	ret = gb_power_supplies_setup(supplies);
+	if (ret < 0)
+		goto error_connection_disable;
+
+	/* We are ready to receive an incoming request now, enable RX as well */
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto error_connection_disable;
+
+	ret = gb_power_supplies_register(supplies);
+	if (ret < 0)
+		goto error_connection_disable;
+
+	gb_pm_runtime_put_autosuspend(bundle);
+	return 0;
+
+error_connection_disable:
+	gb_connection_disable(connection);
+error_connection_destroy:
+	gb_connection_destroy(connection);
+out:
+	_gb_power_supplies_release(supplies);
+	return ret;
+}
+
+static void gb_power_supply_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_power_supplies *supplies = greybus_get_drvdata(bundle);
+
+	gb_connection_disable(supplies->connection);
+	gb_connection_destroy(supplies->connection);
+
+	_gb_power_supplies_release(supplies);
+}
+
+static const struct greybus_bundle_id gb_power_supply_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_POWER_SUPPLY) },
+	{ }
+};
+MODULE_DEVICE_TABLE(greybus, gb_power_supply_id_table);
+
+static struct greybus_driver gb_power_supply_driver = {
+	.name		= "power_supply",
+	.probe		= gb_power_supply_probe,
+	.disconnect	= gb_power_supply_disconnect,
+	.id_table	= gb_power_supply_id_table,
+};
+module_greybus_driver(gb_power_supply_driver);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c
new file mode 100644
index 000000000000..c4bf3298ba07
--- /dev/null
+++ b/drivers/staging/greybus/pwm.c
@@ -0,0 +1,338 @@
+/*
+ * PWM Greybus driver.
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/pwm.h>
+
+#include "greybus.h"
+#include "gbphy.h"
+
+struct gb_pwm_chip {
+	struct gb_connection	*connection;
+	u8			pwm_max;	/* max pwm number */
+
+	struct pwm_chip		chip;
+	struct pwm_chip		*pwm;
+};
+#define pwm_chip_to_gb_pwm_chip(chip) \
+	container_of(chip, struct gb_pwm_chip, chip)
+
+
+static int gb_pwm_count_operation(struct gb_pwm_chip *pwmc)
+{
+	struct gb_pwm_count_response response;
+	int ret;
+
+	ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_PWM_COUNT,
+				NULL, 0, &response, sizeof(response));
+	if (ret)
+		return ret;
+	pwmc->pwm_max = response.count;
+	return 0;
+}
+
+static int gb_pwm_activate_operation(struct gb_pwm_chip *pwmc,
+				     u8 which)
+{
+	struct gb_pwm_activate_request request;
+	struct gbphy_device *gbphy_dev;
+	int ret;
+
+	if (which > pwmc->pwm_max)
+		return -EINVAL;
+
+	request.which = which;
+
+	gbphy_dev = to_gbphy_dev(pwmc->chip.dev);
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		return ret;
+
+	ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_ACTIVATE,
+				&request, sizeof(request), NULL, 0);
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+
+	return ret;
+}
+
+static int gb_pwm_deactivate_operation(struct gb_pwm_chip *pwmc,
+				       u8 which)
+{
+	struct gb_pwm_deactivate_request request;
+	struct gbphy_device *gbphy_dev;
+	int ret;
+
+	if (which > pwmc->pwm_max)
+		return -EINVAL;
+
+	request.which = which;
+
+	gbphy_dev = to_gbphy_dev(pwmc->chip.dev);
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		return ret;
+
+	ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_DEACTIVATE,
+				&request, sizeof(request), NULL, 0);
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+
+	return ret;
+}
+
+static int gb_pwm_config_operation(struct gb_pwm_chip *pwmc,
+				   u8 which, u32 duty, u32 period)
+{
+	struct gb_pwm_config_request request;
+	struct gbphy_device *gbphy_dev;
+	int ret;
+
+	if (which > pwmc->pwm_max)
+		return -EINVAL;
+
+	request.which = which;
+	request.duty = cpu_to_le32(duty);
+	request.period = cpu_to_le32(period);
+
+	gbphy_dev = to_gbphy_dev(pwmc->chip.dev);
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		return ret;
+
+	ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_CONFIG,
+				&request, sizeof(request), NULL, 0);
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+
+	return ret;
+}
+
+static int gb_pwm_set_polarity_operation(struct gb_pwm_chip *pwmc,
+					 u8 which, u8 polarity)
+{
+	struct gb_pwm_polarity_request request;
+	struct gbphy_device *gbphy_dev;
+	int ret;
+
+	if (which > pwmc->pwm_max)
+		return -EINVAL;
+
+	request.which = which;
+	request.polarity = polarity;
+
+	gbphy_dev = to_gbphy_dev(pwmc->chip.dev);
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		return ret;
+
+	ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_POLARITY,
+				&request, sizeof(request), NULL, 0);
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+
+	return ret;
+}
+
+static int gb_pwm_enable_operation(struct gb_pwm_chip *pwmc,
+				   u8 which)
+{
+	struct gb_pwm_enable_request request;
+	struct gbphy_device *gbphy_dev;
+	int ret;
+
+	if (which > pwmc->pwm_max)
+		return -EINVAL;
+
+	request.which = which;
+
+	gbphy_dev = to_gbphy_dev(pwmc->chip.dev);
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		return ret;
+
+	ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_ENABLE,
+				&request, sizeof(request), NULL, 0);
+	if (ret)
+		gbphy_runtime_put_autosuspend(gbphy_dev);
+
+	return ret;
+}
+
+static int gb_pwm_disable_operation(struct gb_pwm_chip *pwmc,
+				    u8 which)
+{
+	struct gb_pwm_disable_request request;
+	struct gbphy_device *gbphy_dev;
+	int ret;
+
+	if (which > pwmc->pwm_max)
+		return -EINVAL;
+
+	request.which = which;
+
+	ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_DISABLE,
+				&request, sizeof(request), NULL, 0);
+
+	gbphy_dev = to_gbphy_dev(pwmc->chip.dev);
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+
+	return ret;
+}
+
+static int gb_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip);
+
+	return gb_pwm_activate_operation(pwmc, pwm->hwpwm);
+};
+
+static void gb_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip);
+
+	if (pwm_is_enabled(pwm))
+		dev_warn(chip->dev, "freeing PWM device without disabling\n");
+
+	gb_pwm_deactivate_operation(pwmc, pwm->hwpwm);
+}
+
+static int gb_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
+			 int duty_ns, int period_ns)
+{
+	struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip);
+
+	return gb_pwm_config_operation(pwmc, pwm->hwpwm, duty_ns, period_ns);
+};
+
+static int gb_pwm_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm,
+			       enum pwm_polarity polarity)
+{
+	struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip);
+
+	return gb_pwm_set_polarity_operation(pwmc, pwm->hwpwm, polarity);
+};
+
+static int gb_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip);
+
+	return gb_pwm_enable_operation(pwmc, pwm->hwpwm);
+};
+
+static void gb_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip);
+
+	gb_pwm_disable_operation(pwmc, pwm->hwpwm);
+};
+
+static const struct pwm_ops gb_pwm_ops = {
+	.request = gb_pwm_request,
+	.free = gb_pwm_free,
+	.config = gb_pwm_config,
+	.set_polarity = gb_pwm_set_polarity,
+	.enable = gb_pwm_enable,
+	.disable = gb_pwm_disable,
+	.owner = THIS_MODULE,
+};
+
+static int gb_pwm_probe(struct gbphy_device *gbphy_dev,
+			const struct gbphy_device_id *id)
+{
+	struct gb_connection *connection;
+	struct gb_pwm_chip *pwmc;
+	struct pwm_chip *pwm;
+	int ret;
+
+	pwmc = kzalloc(sizeof(*pwmc), GFP_KERNEL);
+	if (!pwmc)
+		return -ENOMEM;
+
+	connection = gb_connection_create(gbphy_dev->bundle,
+					  le16_to_cpu(gbphy_dev->cport_desc->id),
+					  NULL);
+	if (IS_ERR(connection)) {
+		ret = PTR_ERR(connection);
+		goto exit_pwmc_free;
+	}
+
+	pwmc->connection = connection;
+	gb_connection_set_data(connection, pwmc);
+	gb_gbphy_set_data(gbphy_dev, pwmc);
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto exit_connection_destroy;
+
+	/* Query number of pwms present */
+	ret = gb_pwm_count_operation(pwmc);
+	if (ret)
+		goto exit_connection_disable;
+
+	pwm = &pwmc->chip;
+
+	pwm->dev = &gbphy_dev->dev;
+	pwm->ops = &gb_pwm_ops;
+	pwm->base = -1;			/* Allocate base dynamically */
+	pwm->npwm = pwmc->pwm_max + 1;
+	pwm->can_sleep = true;		/* FIXME */
+
+	ret = pwmchip_add(pwm);
+	if (ret) {
+		dev_err(&gbphy_dev->dev,
+			"failed to register PWM: %d\n", ret);
+		goto exit_connection_disable;
+	}
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+	return 0;
+
+exit_connection_disable:
+	gb_connection_disable(connection);
+exit_connection_destroy:
+	gb_connection_destroy(connection);
+exit_pwmc_free:
+	kfree(pwmc);
+	return ret;
+}
+
+static void gb_pwm_remove(struct gbphy_device *gbphy_dev)
+{
+	struct gb_pwm_chip *pwmc = gb_gbphy_get_data(gbphy_dev);
+	struct gb_connection *connection = pwmc->connection;
+	int ret;
+
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		gbphy_runtime_get_noresume(gbphy_dev);
+
+	pwmchip_remove(&pwmc->chip);
+	gb_connection_disable(connection);
+	gb_connection_destroy(connection);
+	kfree(pwmc);
+}
+
+static const struct gbphy_device_id gb_pwm_id_table[] = {
+	{ GBPHY_PROTOCOL(GREYBUS_PROTOCOL_PWM) },
+	{ },
+};
+MODULE_DEVICE_TABLE(gbphy, gb_pwm_id_table);
+
+static struct gbphy_driver pwm_driver = {
+	.name		= "pwm",
+	.probe		= gb_pwm_probe,
+	.remove		= gb_pwm_remove,
+	.id_table	= gb_pwm_id_table,
+};
+
+module_gbphy_driver(pwm_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/raw.c b/drivers/staging/greybus/raw.c
new file mode 100644
index 000000000000..729d25811568
--- /dev/null
+++ b/drivers/staging/greybus/raw.c
@@ -0,0 +1,381 @@
+/*
+ * Greybus driver for the Raw protocol
+ *
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/sizes.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/idr.h>
+#include <linux/uaccess.h>
+
+#include "greybus.h"
+
+struct gb_raw {
+	struct gb_connection *connection;
+
+	struct list_head list;
+	int list_data;
+	struct mutex list_lock;
+	dev_t dev;
+	struct cdev cdev;
+	struct device *device;
+};
+
+struct raw_data {
+	struct list_head entry;
+	u32 len;
+	u8 data[0];
+};
+
+static struct class *raw_class;
+static int raw_major;
+static const struct file_operations raw_fops;
+static DEFINE_IDA(minors);
+
+/* Number of minor devices this driver supports */
+#define NUM_MINORS	256
+
+/* Maximum size of any one send data buffer we support */
+#define MAX_PACKET_SIZE	(PAGE_SIZE * 2)
+
+/*
+ * Maximum size of the data in the receive buffer we allow before we start to
+ * drop messages on the floor
+ */
+#define MAX_DATA_SIZE	(MAX_PACKET_SIZE * 8)
+
+/*
+ * Add the raw data message to the list of received messages.
+ */
+static int receive_data(struct gb_raw *raw, u32 len, u8 *data)
+{
+	struct raw_data *raw_data;
+	struct device *dev = &raw->connection->bundle->dev;
+	int retval = 0;
+
+	if (len > MAX_PACKET_SIZE) {
+		dev_err(dev, "Too big of a data packet, rejected\n");
+		return -EINVAL;
+	}
+
+	mutex_lock(&raw->list_lock);
+	if ((raw->list_data + len) > MAX_DATA_SIZE) {
+		dev_err(dev, "Too much data in receive buffer, now dropping packets\n");
+		retval = -EINVAL;
+		goto exit;
+	}
+
+	raw_data = kmalloc(sizeof(*raw_data) + len, GFP_KERNEL);
+	if (!raw_data) {
+		retval = -ENOMEM;
+		goto exit;
+	}
+
+	raw->list_data += len;
+	raw_data->len = len;
+	memcpy(&raw_data->data[0], data, len);
+
+	list_add_tail(&raw_data->entry, &raw->list);
+exit:
+	mutex_unlock(&raw->list_lock);
+	return retval;
+}
+
+static int gb_raw_request_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct device *dev = &connection->bundle->dev;
+	struct gb_raw *raw = greybus_get_drvdata(connection->bundle);
+	struct gb_raw_send_request *receive;
+	u32 len;
+
+	if (op->type != GB_RAW_TYPE_SEND) {
+		dev_err(dev, "unknown request type 0x%02x\n", op->type);
+		return -EINVAL;
+	}
+
+	/* Verify size of payload */
+	if (op->request->payload_size < sizeof(*receive)) {
+		dev_err(dev, "raw receive request too small (%zu < %zu)\n",
+			op->request->payload_size, sizeof(*receive));
+		return -EINVAL;
+	}
+	receive = op->request->payload;
+	len = le32_to_cpu(receive->len);
+	if (len != (int)(op->request->payload_size - sizeof(__le32))) {
+		dev_err(dev, "raw receive request wrong size %d vs %d\n", len,
+			(int)(op->request->payload_size - sizeof(__le32)));
+		return -EINVAL;
+	}
+	if (len == 0) {
+		dev_err(dev, "raw receive request of 0 bytes?\n");
+		return -EINVAL;
+	}
+
+	return receive_data(raw, len, receive->data);
+}
+
+static int gb_raw_send(struct gb_raw *raw, u32 len, const char __user *data)
+{
+	struct gb_connection *connection = raw->connection;
+	struct gb_raw_send_request *request;
+	int retval;
+
+	request = kmalloc(len + sizeof(*request), GFP_KERNEL);
+	if (!request)
+		return -ENOMEM;
+
+	if (copy_from_user(&request->data[0], data, len)) {
+		kfree(request);
+		return -EFAULT;
+	}
+
+	request->len = cpu_to_le32(len);
+
+	retval = gb_operation_sync(connection, GB_RAW_TYPE_SEND,
+				   request, len + sizeof(*request),
+				   NULL, 0);
+
+	kfree(request);
+	return retval;
+}
+
+static int gb_raw_probe(struct gb_bundle *bundle,
+			const struct greybus_bundle_id *id)
+{
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_connection *connection;
+	struct gb_raw *raw;
+	int retval;
+	int minor;
+
+	if (bundle->num_cports != 1)
+		return -ENODEV;
+
+	cport_desc = &bundle->cport_desc[0];
+	if (cport_desc->protocol_id != GREYBUS_PROTOCOL_RAW)
+		return -ENODEV;
+
+	raw = kzalloc(sizeof(*raw), GFP_KERNEL);
+	if (!raw)
+		return -ENOMEM;
+
+	connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
+					  gb_raw_request_handler);
+	if (IS_ERR(connection)) {
+		retval = PTR_ERR(connection);
+		goto error_free;
+	}
+
+	INIT_LIST_HEAD(&raw->list);
+	mutex_init(&raw->list_lock);
+
+	raw->connection = connection;
+	greybus_set_drvdata(bundle, raw);
+
+	minor = ida_simple_get(&minors, 0, 0, GFP_KERNEL);
+	if (minor < 0) {
+		retval = minor;
+		goto error_connection_destroy;
+	}
+
+	raw->dev = MKDEV(raw_major, minor);
+	cdev_init(&raw->cdev, &raw_fops);
+
+	retval = gb_connection_enable(connection);
+	if (retval)
+		goto error_remove_ida;
+
+	retval = cdev_add(&raw->cdev, raw->dev, 1);
+	if (retval)
+		goto error_connection_disable;
+
+	raw->device = device_create(raw_class, &connection->bundle->dev,
+				    raw->dev, raw, "gb!raw%d", minor);
+	if (IS_ERR(raw->device)) {
+		retval = PTR_ERR(raw->device);
+		goto error_del_cdev;
+	}
+
+	return 0;
+
+error_del_cdev:
+	cdev_del(&raw->cdev);
+
+error_connection_disable:
+	gb_connection_disable(connection);
+
+error_remove_ida:
+	ida_simple_remove(&minors, minor);
+
+error_connection_destroy:
+	gb_connection_destroy(connection);
+
+error_free:
+	kfree(raw);
+	return retval;
+}
+
+static void gb_raw_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_raw *raw = greybus_get_drvdata(bundle);
+	struct gb_connection *connection = raw->connection;
+	struct raw_data *raw_data;
+	struct raw_data *temp;
+
+	// FIXME - handle removing a connection when the char device node is open.
+	device_destroy(raw_class, raw->dev);
+	cdev_del(&raw->cdev);
+	gb_connection_disable(connection);
+	ida_simple_remove(&minors, MINOR(raw->dev));
+	gb_connection_destroy(connection);
+
+	mutex_lock(&raw->list_lock);
+	list_for_each_entry_safe(raw_data, temp, &raw->list, entry) {
+		list_del(&raw_data->entry);
+		kfree(raw_data);
+	}
+	mutex_unlock(&raw->list_lock);
+
+	kfree(raw);
+}
+
+/*
+ * Character device node interfaces.
+ *
+ * Note, we are using read/write to only allow a single read/write per message.
+ * This means for read(), you have to provide a big enough buffer for the full
+ * message to be copied into.  If the buffer isn't big enough, the read() will
+ * fail with -ENOSPC.
+ */
+
+static int raw_open(struct inode *inode, struct file *file)
+{
+	struct cdev *cdev = inode->i_cdev;
+	struct gb_raw *raw = container_of(cdev, struct gb_raw, cdev);
+
+	file->private_data = raw;
+	return 0;
+}
+
+static ssize_t raw_write(struct file *file, const char __user *buf,
+			 size_t count, loff_t *ppos)
+{
+	struct gb_raw *raw = file->private_data;
+	int retval;
+
+	if (!count)
+		return 0;
+
+	if (count > MAX_PACKET_SIZE)
+		return -E2BIG;
+
+	retval = gb_raw_send(raw, count, buf);
+	if (retval)
+		return retval;
+
+	return count;
+}
+
+static ssize_t raw_read(struct file *file, char __user *buf, size_t count,
+			loff_t *ppos)
+{
+	struct gb_raw *raw = file->private_data;
+	int retval = 0;
+	struct raw_data *raw_data;
+
+	mutex_lock(&raw->list_lock);
+	if (list_empty(&raw->list))
+		goto exit;
+
+	raw_data = list_first_entry(&raw->list, struct raw_data, entry);
+	if (raw_data->len > count) {
+		retval = -ENOSPC;
+		goto exit;
+	}
+
+	if (copy_to_user(buf, &raw_data->data[0], raw_data->len)) {
+		retval = -EFAULT;
+		goto exit;
+	}
+
+	list_del(&raw_data->entry);
+	raw->list_data -= raw_data->len;
+	retval = raw_data->len;
+	kfree(raw_data);
+
+exit:
+	mutex_unlock(&raw->list_lock);
+	return retval;
+}
+
+static const struct file_operations raw_fops = {
+	.owner		= THIS_MODULE,
+	.write		= raw_write,
+	.read		= raw_read,
+	.open		= raw_open,
+	.llseek		= noop_llseek,
+};
+
+static const struct greybus_bundle_id gb_raw_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_RAW) },
+	{ }
+};
+MODULE_DEVICE_TABLE(greybus, gb_raw_id_table);
+
+static struct greybus_driver gb_raw_driver = {
+	.name		= "raw",
+	.probe		= gb_raw_probe,
+	.disconnect	= gb_raw_disconnect,
+	.id_table	= gb_raw_id_table,
+};
+
+static int raw_init(void)
+{
+	dev_t dev;
+	int retval;
+
+	raw_class = class_create(THIS_MODULE, "gb_raw");
+	if (IS_ERR(raw_class)) {
+		retval = PTR_ERR(raw_class);
+		goto error_class;
+	}
+
+	retval = alloc_chrdev_region(&dev, 0, NUM_MINORS, "gb_raw");
+	if (retval < 0)
+		goto error_chrdev;
+
+	raw_major = MAJOR(dev);
+
+	retval = greybus_register(&gb_raw_driver);
+	if (retval)
+		goto error_gb;
+
+	return 0;
+
+error_gb:
+	unregister_chrdev_region(dev, NUM_MINORS);
+error_chrdev:
+	class_destroy(raw_class);
+error_class:
+	return retval;
+}
+module_init(raw_init);
+
+static void __exit raw_exit(void)
+{
+	greybus_deregister(&gb_raw_driver);
+	unregister_chrdev_region(MKDEV(raw_major, 0), NUM_MINORS);
+	class_destroy(raw_class);
+	ida_destroy(&minors);
+}
+module_exit(raw_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/sdio.c b/drivers/staging/greybus/sdio.c
new file mode 100644
index 000000000000..5649ef1e379d
--- /dev/null
+++ b/drivers/staging/greybus/sdio.c
@@ -0,0 +1,884 @@
+/*
+ * SD/MMC Greybus driver.
+ *
+ * Copyright 2014-2015 Google Inc.
+ * Copyright 2014-2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/mmc/core.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/mmc.h>
+#include <linux/scatterlist.h>
+#include <linux/workqueue.h>
+
+#include "greybus.h"
+#include "gbphy.h"
+
+struct gb_sdio_host {
+	struct gb_connection	*connection;
+	struct gbphy_device	*gbphy_dev;
+	struct mmc_host		*mmc;
+	struct mmc_request	*mrq;
+	struct mutex		lock;	/* lock for this host */
+	size_t			data_max;
+	spinlock_t		xfer;	/* lock to cancel ongoing transfer */
+	bool			xfer_stop;
+	struct workqueue_struct	*mrq_workqueue;
+	struct work_struct	mrqwork;
+	u8			queued_events;
+	bool			removed;
+	bool			card_present;
+	bool			read_only;
+};
+
+
+#define GB_SDIO_RSP_R1_R5_R6_R7	(GB_SDIO_RSP_PRESENT | GB_SDIO_RSP_CRC | \
+				 GB_SDIO_RSP_OPCODE)
+#define GB_SDIO_RSP_R3_R4	(GB_SDIO_RSP_PRESENT)
+#define GB_SDIO_RSP_R2		(GB_SDIO_RSP_PRESENT | GB_SDIO_RSP_CRC | \
+				 GB_SDIO_RSP_136)
+#define GB_SDIO_RSP_R1B		(GB_SDIO_RSP_PRESENT | GB_SDIO_RSP_CRC | \
+				 GB_SDIO_RSP_OPCODE | GB_SDIO_RSP_BUSY)
+
+/* kernel vdd starts at 0x80 and we need to translate to greybus ones 0x01 */
+#define GB_SDIO_VDD_SHIFT	8
+
+#ifndef MMC_CAP2_CORE_RUNTIME_PM
+#define MMC_CAP2_CORE_RUNTIME_PM	0
+#endif
+
+static inline bool single_op(struct mmc_command *cmd)
+{
+	uint32_t opcode = cmd->opcode;
+
+	return opcode == MMC_WRITE_BLOCK ||
+	       opcode == MMC_READ_SINGLE_BLOCK;
+}
+
+static void _gb_sdio_set_host_caps(struct gb_sdio_host *host, u32 r)
+{
+	u32 caps = 0;
+	u32 caps2 = 0;
+
+	caps = ((r & GB_SDIO_CAP_NONREMOVABLE) ? MMC_CAP_NONREMOVABLE : 0) |
+		((r & GB_SDIO_CAP_4_BIT_DATA) ? MMC_CAP_4_BIT_DATA : 0) |
+		((r & GB_SDIO_CAP_8_BIT_DATA) ? MMC_CAP_8_BIT_DATA : 0) |
+		((r & GB_SDIO_CAP_MMC_HS) ? MMC_CAP_MMC_HIGHSPEED : 0) |
+		((r & GB_SDIO_CAP_SD_HS) ? MMC_CAP_SD_HIGHSPEED : 0) |
+		((r & GB_SDIO_CAP_ERASE) ? MMC_CAP_ERASE : 0) |
+		((r & GB_SDIO_CAP_1_2V_DDR) ? MMC_CAP_1_2V_DDR : 0) |
+		((r & GB_SDIO_CAP_1_8V_DDR) ? MMC_CAP_1_8V_DDR : 0) |
+		((r & GB_SDIO_CAP_POWER_OFF_CARD) ? MMC_CAP_POWER_OFF_CARD : 0) |
+		((r & GB_SDIO_CAP_UHS_SDR12) ? MMC_CAP_UHS_SDR12 : 0) |
+		((r & GB_SDIO_CAP_UHS_SDR25) ? MMC_CAP_UHS_SDR25 : 0) |
+		((r & GB_SDIO_CAP_UHS_SDR50) ? MMC_CAP_UHS_SDR50 : 0) |
+		((r & GB_SDIO_CAP_UHS_SDR104) ? MMC_CAP_UHS_SDR104 : 0) |
+		((r & GB_SDIO_CAP_UHS_DDR50) ? MMC_CAP_UHS_DDR50 : 0) |
+		((r & GB_SDIO_CAP_DRIVER_TYPE_A) ? MMC_CAP_DRIVER_TYPE_A : 0) |
+		((r & GB_SDIO_CAP_DRIVER_TYPE_C) ? MMC_CAP_DRIVER_TYPE_C : 0) |
+		((r & GB_SDIO_CAP_DRIVER_TYPE_D) ? MMC_CAP_DRIVER_TYPE_D : 0);
+
+	caps2 = ((r & GB_SDIO_CAP_HS200_1_2V) ? MMC_CAP2_HS200_1_2V_SDR : 0) |
+		((r & GB_SDIO_CAP_HS400_1_2V) ? MMC_CAP2_HS400_1_2V : 0) |
+		((r & GB_SDIO_CAP_HS400_1_8V) ? MMC_CAP2_HS400_1_8V : 0) |
+		((r & GB_SDIO_CAP_HS200_1_8V) ? MMC_CAP2_HS200_1_8V_SDR : 0);
+
+	host->mmc->caps = caps;
+	host->mmc->caps2 = caps2 | MMC_CAP2_CORE_RUNTIME_PM;
+
+	if (caps & MMC_CAP_NONREMOVABLE)
+		host->card_present = true;
+}
+
+static u32 _gb_sdio_get_host_ocr(u32 ocr)
+{
+	return (((ocr & GB_SDIO_VDD_165_195) ? MMC_VDD_165_195 : 0) |
+		((ocr & GB_SDIO_VDD_20_21) ? MMC_VDD_20_21 : 0) |
+		((ocr & GB_SDIO_VDD_21_22) ? MMC_VDD_21_22 : 0) |
+		((ocr & GB_SDIO_VDD_22_23) ? MMC_VDD_22_23 : 0) |
+		((ocr & GB_SDIO_VDD_23_24) ? MMC_VDD_23_24 : 0) |
+		((ocr & GB_SDIO_VDD_24_25) ? MMC_VDD_24_25 : 0) |
+		((ocr & GB_SDIO_VDD_25_26) ? MMC_VDD_25_26 : 0) |
+		((ocr & GB_SDIO_VDD_26_27) ? MMC_VDD_26_27 : 0) |
+		((ocr & GB_SDIO_VDD_27_28) ? MMC_VDD_27_28 : 0) |
+		((ocr & GB_SDIO_VDD_28_29) ? MMC_VDD_28_29 : 0) |
+		((ocr & GB_SDIO_VDD_29_30) ? MMC_VDD_29_30 : 0) |
+		((ocr & GB_SDIO_VDD_30_31) ? MMC_VDD_30_31 : 0) |
+		((ocr & GB_SDIO_VDD_31_32) ? MMC_VDD_31_32 : 0) |
+		((ocr & GB_SDIO_VDD_32_33) ? MMC_VDD_32_33 : 0) |
+		((ocr & GB_SDIO_VDD_33_34) ? MMC_VDD_33_34 : 0) |
+		((ocr & GB_SDIO_VDD_34_35) ? MMC_VDD_34_35 : 0) |
+		((ocr & GB_SDIO_VDD_35_36) ? MMC_VDD_35_36 : 0)
+		);
+}
+
+static int gb_sdio_get_caps(struct gb_sdio_host *host)
+{
+	struct gb_sdio_get_caps_response response;
+	struct mmc_host *mmc = host->mmc;
+	u16 data_max;
+	u32 blksz;
+	u32 ocr;
+	u32 r;
+	int ret;
+
+	ret = gb_operation_sync(host->connection, GB_SDIO_TYPE_GET_CAPABILITIES,
+				NULL, 0, &response, sizeof(response));
+	if (ret < 0)
+		return ret;
+	r = le32_to_cpu(response.caps);
+
+	_gb_sdio_set_host_caps(host, r);
+
+	/* get the max block size that could fit our payload */
+	data_max = gb_operation_get_payload_size_max(host->connection);
+	data_max = min(data_max - sizeof(struct gb_sdio_transfer_request),
+		       data_max - sizeof(struct gb_sdio_transfer_response));
+
+	blksz = min_t(u16, le16_to_cpu(response.max_blk_size), data_max);
+	blksz = max_t(u32, 512, blksz);
+
+	mmc->max_blk_size = rounddown_pow_of_two(blksz);
+	mmc->max_blk_count = le16_to_cpu(response.max_blk_count);
+	host->data_max = data_max;
+
+	/* get ocr supported values */
+	ocr = _gb_sdio_get_host_ocr(le32_to_cpu(response.ocr));
+	mmc->ocr_avail = ocr;
+	mmc->ocr_avail_sdio = mmc->ocr_avail;
+	mmc->ocr_avail_sd = mmc->ocr_avail;
+	mmc->ocr_avail_mmc = mmc->ocr_avail;
+
+	/* get frequency range values */
+	mmc->f_min = le32_to_cpu(response.f_min);
+	mmc->f_max = le32_to_cpu(response.f_max);
+
+	return 0;
+}
+
+static void _gb_queue_event(struct gb_sdio_host *host, u8 event)
+{
+	if (event & GB_SDIO_CARD_INSERTED)
+		host->queued_events &= ~GB_SDIO_CARD_REMOVED;
+	else if (event & GB_SDIO_CARD_REMOVED)
+		host->queued_events &= ~GB_SDIO_CARD_INSERTED;
+
+	host->queued_events |= event;
+}
+
+static int _gb_sdio_process_events(struct gb_sdio_host *host, u8 event)
+{
+	u8 state_changed = 0;
+
+	if (event & GB_SDIO_CARD_INSERTED) {
+		if (host->mmc->caps & MMC_CAP_NONREMOVABLE)
+			return 0;
+		if (host->card_present)
+			return 0;
+		host->card_present = true;
+		state_changed = 1;
+	}
+
+	if (event & GB_SDIO_CARD_REMOVED) {
+		if (host->mmc->caps & MMC_CAP_NONREMOVABLE)
+			return 0;
+		if (!(host->card_present))
+			return 0;
+		host->card_present = false;
+		state_changed = 1;
+	}
+
+	if (event & GB_SDIO_WP) {
+		host->read_only = true;
+	}
+
+	if (state_changed) {
+		dev_info(mmc_dev(host->mmc), "card %s now event\n",
+			 (host->card_present ?  "inserted" : "removed"));
+		mmc_detect_change(host->mmc, 0);
+	}
+
+	return 0;
+}
+
+static int gb_sdio_request_handler(struct gb_operation *op)
+{
+	struct gb_sdio_host *host = gb_connection_get_data(op->connection);
+	struct gb_message *request;
+	struct gb_sdio_event_request *payload;
+	u8 type = op->type;
+	int ret =  0;
+	u8 event;
+
+	if (type != GB_SDIO_TYPE_EVENT) {
+		dev_err(mmc_dev(host->mmc),
+			"unsupported unsolicited event: %u\n", type);
+		return -EINVAL;
+	}
+
+	request = op->request;
+
+	if (request->payload_size < sizeof(*payload)) {
+		dev_err(mmc_dev(host->mmc), "wrong event size received (%zu < %zu)\n",
+			request->payload_size, sizeof(*payload));
+		return -EINVAL;
+	}
+
+	payload = request->payload;
+	event = payload->event;
+
+	if (host->removed)
+		_gb_queue_event(host, event);
+	else
+		ret = _gb_sdio_process_events(host, event);
+
+	return ret;
+}
+
+static int gb_sdio_set_ios(struct gb_sdio_host *host,
+			   struct gb_sdio_set_ios_request *request)
+{
+	int ret;
+
+	ret = gbphy_runtime_get_sync(host->gbphy_dev);
+	if (ret)
+		return ret;
+
+	ret = gb_operation_sync(host->connection, GB_SDIO_TYPE_SET_IOS, request,
+				sizeof(*request), NULL, 0);
+
+	gbphy_runtime_put_autosuspend(host->gbphy_dev);
+
+	return ret;
+}
+
+static int _gb_sdio_send(struct gb_sdio_host *host, struct mmc_data *data,
+			 size_t len, u16 nblocks, off_t skip)
+{
+	struct gb_sdio_transfer_request *request;
+	struct gb_sdio_transfer_response *response;
+	struct gb_operation *operation;
+	struct scatterlist *sg = data->sg;
+	unsigned int sg_len = data->sg_len;
+	size_t copied;
+	u16 send_blksz;
+	u16 send_blocks;
+	int ret;
+
+	WARN_ON(len > host->data_max);
+
+	operation = gb_operation_create(host->connection, GB_SDIO_TYPE_TRANSFER,
+					len + sizeof(*request),
+					sizeof(*response), GFP_KERNEL);
+	if (!operation)
+		return -ENOMEM;
+
+	request = operation->request->payload;
+	request->data_flags = (data->flags >> 8);
+	request->data_blocks = cpu_to_le16(nblocks);
+	request->data_blksz = cpu_to_le16(data->blksz);
+
+	copied = sg_pcopy_to_buffer(sg, sg_len, &request->data[0], len, skip);
+
+	if (copied != len) {
+		ret = -EINVAL;
+		goto err_put_operation;
+	}
+
+	ret = gb_operation_request_send_sync(operation);
+	if (ret < 0)
+		goto err_put_operation;
+
+	response = operation->response->payload;
+
+	send_blocks = le16_to_cpu(response->data_blocks);
+	send_blksz = le16_to_cpu(response->data_blksz);
+
+	if (len != send_blksz * send_blocks) {
+		dev_err(mmc_dev(host->mmc), "send: size received: %zu != %d\n",
+			len, send_blksz * send_blocks);
+		ret = -EINVAL;
+	}
+
+err_put_operation:
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+static int _gb_sdio_recv(struct gb_sdio_host *host, struct mmc_data *data,
+			 size_t len, u16 nblocks, off_t skip)
+{
+	struct gb_sdio_transfer_request *request;
+	struct gb_sdio_transfer_response *response;
+	struct gb_operation *operation;
+	struct scatterlist *sg = data->sg;
+	unsigned int sg_len = data->sg_len;
+	size_t copied;
+	u16 recv_blksz;
+	u16 recv_blocks;
+	int ret;
+
+	WARN_ON(len > host->data_max);
+
+	operation = gb_operation_create(host->connection, GB_SDIO_TYPE_TRANSFER,
+					sizeof(*request),
+					len + sizeof(*response), GFP_KERNEL);
+	if (!operation)
+		return -ENOMEM;
+
+	request = operation->request->payload;
+	request->data_flags = (data->flags >> 8);
+	request->data_blocks = cpu_to_le16(nblocks);
+	request->data_blksz = cpu_to_le16(data->blksz);
+
+	ret = gb_operation_request_send_sync(operation);
+	if (ret < 0)
+		goto err_put_operation;
+
+	response = operation->response->payload;
+	recv_blocks = le16_to_cpu(response->data_blocks);
+	recv_blksz = le16_to_cpu(response->data_blksz);
+
+	if (len != recv_blksz * recv_blocks) {
+		dev_err(mmc_dev(host->mmc), "recv: size received: %d != %zu\n",
+			recv_blksz * recv_blocks, len);
+		ret = -EINVAL;
+		goto err_put_operation;
+	}
+
+	copied = sg_pcopy_from_buffer(sg, sg_len, &response->data[0], len,
+				      skip);
+	if (copied != len)
+		ret = -EINVAL;
+
+err_put_operation:
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+static int gb_sdio_transfer(struct gb_sdio_host *host, struct mmc_data *data)
+{
+	size_t left, len;
+	off_t skip = 0;
+	int ret = 0;
+	u16 nblocks;
+
+	if (single_op(data->mrq->cmd) && data->blocks > 1) {
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	left = data->blksz * data->blocks;
+
+	while (left) {
+		/* check is a stop transmission is pending */
+		spin_lock(&host->xfer);
+		if (host->xfer_stop) {
+			host->xfer_stop = false;
+			spin_unlock(&host->xfer);
+			ret = -EINTR;
+			goto out;
+		}
+		spin_unlock(&host->xfer);
+		len = min(left, host->data_max);
+		nblocks = len / data->blksz;
+		len = nblocks * data->blksz;
+
+		if (data->flags & MMC_DATA_READ) {
+			ret = _gb_sdio_recv(host, data, len, nblocks, skip);
+			if (ret < 0)
+				goto out;
+		} else {
+			ret = _gb_sdio_send(host, data, len, nblocks, skip);
+			if (ret < 0)
+				goto out;
+		}
+		data->bytes_xfered += len;
+		left -= len;
+		skip += len;
+	}
+
+out:
+	data->error = ret;
+	return ret;
+}
+
+static int gb_sdio_command(struct gb_sdio_host *host, struct mmc_command *cmd)
+{
+	struct gb_sdio_command_request request = {0};
+	struct gb_sdio_command_response response;
+	struct mmc_data *data = host->mrq->data;
+	u8 cmd_flags;
+	u8 cmd_type;
+	int i;
+	int ret;
+
+	switch (mmc_resp_type(cmd)) {
+	case MMC_RSP_NONE:
+		cmd_flags = GB_SDIO_RSP_NONE;
+		break;
+	case MMC_RSP_R1:
+		cmd_flags = GB_SDIO_RSP_R1_R5_R6_R7;
+		break;
+	case MMC_RSP_R1B:
+		cmd_flags = GB_SDIO_RSP_R1B;
+		break;
+	case MMC_RSP_R2:
+		cmd_flags = GB_SDIO_RSP_R2;
+		break;
+	case MMC_RSP_R3:
+		cmd_flags = GB_SDIO_RSP_R3_R4;
+		break;
+	default:
+		dev_err(mmc_dev(host->mmc), "cmd flag invalid 0x%04x\n",
+			mmc_resp_type(cmd));
+		ret = -EINVAL;
+		goto out;
+	}
+
+	switch (mmc_cmd_type(cmd)) {
+	case MMC_CMD_BC:
+		cmd_type = GB_SDIO_CMD_BC;
+		break;
+	case MMC_CMD_BCR:
+		cmd_type = GB_SDIO_CMD_BCR;
+		break;
+	case MMC_CMD_AC:
+		cmd_type = GB_SDIO_CMD_AC;
+		break;
+	case MMC_CMD_ADTC:
+		cmd_type = GB_SDIO_CMD_ADTC;
+		break;
+	default:
+		dev_err(mmc_dev(host->mmc), "cmd type invalid 0x%04x\n",
+			mmc_cmd_type(cmd));
+		ret = -EINVAL;
+		goto out;
+	}
+
+	request.cmd = cmd->opcode;
+	request.cmd_flags = cmd_flags;
+	request.cmd_type = cmd_type;
+	request.cmd_arg = cpu_to_le32(cmd->arg);
+	/* some controllers need to know at command time data details */
+	if (data) {
+		request.data_blocks = cpu_to_le16(data->blocks);
+		request.data_blksz = cpu_to_le16(data->blksz);
+	}
+
+	ret = gb_operation_sync(host->connection, GB_SDIO_TYPE_COMMAND,
+				&request, sizeof(request), &response,
+				sizeof(response));
+	if (ret < 0)
+		goto out;
+
+	/* no response expected */
+	if (cmd_flags == GB_SDIO_RSP_NONE)
+		goto out;
+
+	/* long response expected */
+	if (cmd_flags & GB_SDIO_RSP_R2)
+		for (i = 0; i < 4; i++)
+			cmd->resp[i] = le32_to_cpu(response.resp[i]);
+	else
+		cmd->resp[0] = le32_to_cpu(response.resp[0]);
+
+out:
+	cmd->error = ret;
+	return ret;
+}
+
+static void gb_sdio_mrq_work(struct work_struct *work)
+{
+	struct gb_sdio_host *host;
+	struct mmc_request *mrq;
+	int ret;
+
+	host = container_of(work, struct gb_sdio_host, mrqwork);
+
+	ret = gbphy_runtime_get_sync(host->gbphy_dev);
+	if (ret)
+		return;
+
+	mutex_lock(&host->lock);
+	mrq = host->mrq;
+	if (!mrq) {
+		mutex_unlock(&host->lock);
+		gbphy_runtime_put_autosuspend(host->gbphy_dev);
+		dev_err(mmc_dev(host->mmc), "mmc request is NULL");
+		return;
+	}
+
+	if (host->removed) {
+		mrq->cmd->error = -ESHUTDOWN;
+		goto done;
+	}
+
+	if (mrq->sbc) {
+		ret = gb_sdio_command(host, mrq->sbc);
+		if (ret < 0)
+			goto done;
+	}
+
+	ret = gb_sdio_command(host, mrq->cmd);
+	if (ret < 0)
+		goto done;
+
+	if (mrq->data) {
+		ret = gb_sdio_transfer(host, mrq->data);
+		if (ret < 0)
+			goto done;
+	}
+
+	if (mrq->stop) {
+		ret = gb_sdio_command(host, mrq->stop);
+		if (ret < 0)
+			goto done;
+	}
+
+done:
+	host->mrq = NULL;
+	mutex_unlock(&host->lock);
+	mmc_request_done(host->mmc, mrq);
+	gbphy_runtime_put_autosuspend(host->gbphy_dev);
+}
+
+static void gb_mmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
+{
+	struct gb_sdio_host *host = mmc_priv(mmc);
+	struct mmc_command *cmd = mrq->cmd;
+
+	/* Check if it is a cancel to ongoing transfer */
+	if (cmd->opcode == MMC_STOP_TRANSMISSION) {
+		spin_lock(&host->xfer);
+		host->xfer_stop = true;
+		spin_unlock(&host->xfer);
+	}
+
+	mutex_lock(&host->lock);
+
+	WARN_ON(host->mrq);
+	host->mrq = mrq;
+
+	if (host->removed) {
+		mrq->cmd->error = -ESHUTDOWN;
+		goto out;
+	}
+	if (!host->card_present) {
+		mrq->cmd->error = -ENOMEDIUM;
+		goto out;
+	}
+
+	queue_work(host->mrq_workqueue, &host->mrqwork);
+
+	mutex_unlock(&host->lock);
+	return;
+
+out:
+	host->mrq = NULL;
+	mutex_unlock(&host->lock);
+	mmc_request_done(mmc, mrq);
+}
+
+static void gb_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
+{
+	struct gb_sdio_host *host = mmc_priv(mmc);
+	struct gb_sdio_set_ios_request request;
+	int ret;
+	u8 power_mode;
+	u8 bus_width;
+	u8 timing;
+	u8 signal_voltage;
+	u8 drv_type;
+	u32 vdd = 0;
+
+	mutex_lock(&host->lock);
+	request.clock = cpu_to_le32(ios->clock);
+
+	if (ios->vdd)
+		vdd = 1 << (ios->vdd - GB_SDIO_VDD_SHIFT);
+	request.vdd = cpu_to_le32(vdd);
+
+	request.bus_mode = (ios->bus_mode == MMC_BUSMODE_OPENDRAIN ?
+			    GB_SDIO_BUSMODE_OPENDRAIN :
+			    GB_SDIO_BUSMODE_PUSHPULL);
+
+	switch (ios->power_mode) {
+	case MMC_POWER_OFF:
+	default:
+		power_mode = GB_SDIO_POWER_OFF;
+		break;
+	case MMC_POWER_UP:
+		power_mode = GB_SDIO_POWER_UP;
+		break;
+	case MMC_POWER_ON:
+		power_mode = GB_SDIO_POWER_ON;
+		break;
+	case MMC_POWER_UNDEFINED:
+		power_mode = GB_SDIO_POWER_UNDEFINED;
+		break;
+	}
+	request.power_mode = power_mode;
+
+	switch (ios->bus_width) {
+	case MMC_BUS_WIDTH_1:
+		bus_width = GB_SDIO_BUS_WIDTH_1;
+		break;
+	case MMC_BUS_WIDTH_4:
+	default:
+		bus_width = GB_SDIO_BUS_WIDTH_4;
+		break;
+	case MMC_BUS_WIDTH_8:
+		bus_width = GB_SDIO_BUS_WIDTH_8;
+		break;
+	}
+	request.bus_width = bus_width;
+
+	switch (ios->timing) {
+	case MMC_TIMING_LEGACY:
+	default:
+		timing = GB_SDIO_TIMING_LEGACY;
+		break;
+	case MMC_TIMING_MMC_HS:
+		timing = GB_SDIO_TIMING_MMC_HS;
+		break;
+	case MMC_TIMING_SD_HS:
+		timing = GB_SDIO_TIMING_SD_HS;
+		break;
+	case MMC_TIMING_UHS_SDR12:
+		timing = GB_SDIO_TIMING_UHS_SDR12;
+		break;
+	case MMC_TIMING_UHS_SDR25:
+		timing = GB_SDIO_TIMING_UHS_SDR25;
+		break;
+	case MMC_TIMING_UHS_SDR50:
+		timing = GB_SDIO_TIMING_UHS_SDR50;
+		break;
+	case MMC_TIMING_UHS_SDR104:
+		timing = GB_SDIO_TIMING_UHS_SDR104;
+		break;
+	case MMC_TIMING_UHS_DDR50:
+		timing = GB_SDIO_TIMING_UHS_DDR50;
+		break;
+	case MMC_TIMING_MMC_DDR52:
+		timing = GB_SDIO_TIMING_MMC_DDR52;
+		break;
+	case MMC_TIMING_MMC_HS200:
+		timing = GB_SDIO_TIMING_MMC_HS200;
+		break;
+	case MMC_TIMING_MMC_HS400:
+		timing = GB_SDIO_TIMING_MMC_HS400;
+		break;
+	}
+	request.timing = timing;
+
+	switch (ios->signal_voltage) {
+	case MMC_SIGNAL_VOLTAGE_330:
+		signal_voltage = GB_SDIO_SIGNAL_VOLTAGE_330;
+		break;
+	case MMC_SIGNAL_VOLTAGE_180:
+	default:
+		signal_voltage = GB_SDIO_SIGNAL_VOLTAGE_180;
+		break;
+	case MMC_SIGNAL_VOLTAGE_120:
+		signal_voltage = GB_SDIO_SIGNAL_VOLTAGE_120;
+		break;
+	}
+	request.signal_voltage = signal_voltage;
+
+	switch (ios->drv_type) {
+	case MMC_SET_DRIVER_TYPE_A:
+		drv_type = GB_SDIO_SET_DRIVER_TYPE_A;
+		break;
+	case MMC_SET_DRIVER_TYPE_C:
+		drv_type = GB_SDIO_SET_DRIVER_TYPE_C;
+		break;
+	case MMC_SET_DRIVER_TYPE_D:
+		drv_type = GB_SDIO_SET_DRIVER_TYPE_D;
+		break;
+	case MMC_SET_DRIVER_TYPE_B:
+	default:
+		drv_type = GB_SDIO_SET_DRIVER_TYPE_B;
+		break;
+	}
+	request.drv_type = drv_type;
+
+	ret = gb_sdio_set_ios(host, &request);
+	if (ret < 0)
+		goto out;
+
+	memcpy(&mmc->ios, ios, sizeof(mmc->ios));
+
+out:
+	mutex_unlock(&host->lock);
+}
+
+static int gb_mmc_get_ro(struct mmc_host *mmc)
+{
+	struct gb_sdio_host *host = mmc_priv(mmc);
+
+	mutex_lock(&host->lock);
+	if (host->removed) {
+		mutex_unlock(&host->lock);
+		return -ESHUTDOWN;
+	}
+	mutex_unlock(&host->lock);
+
+	return host->read_only;
+}
+
+static int gb_mmc_get_cd(struct mmc_host *mmc)
+{
+	struct gb_sdio_host *host = mmc_priv(mmc);
+
+	mutex_lock(&host->lock);
+	if (host->removed) {
+		mutex_unlock(&host->lock);
+		return -ESHUTDOWN;
+	}
+	mutex_unlock(&host->lock);
+
+	return host->card_present;
+}
+
+static int gb_mmc_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
+{
+	return 0;
+}
+
+static const struct mmc_host_ops gb_sdio_ops = {
+	.request	= gb_mmc_request,
+	.set_ios	= gb_mmc_set_ios,
+	.get_ro		= gb_mmc_get_ro,
+	.get_cd		= gb_mmc_get_cd,
+	.start_signal_voltage_switch	= gb_mmc_switch_voltage,
+};
+
+static int gb_sdio_probe(struct gbphy_device *gbphy_dev,
+			 const struct gbphy_device_id *id)
+{
+	struct gb_connection *connection;
+	struct mmc_host *mmc;
+	struct gb_sdio_host *host;
+	int ret = 0;
+
+	mmc = mmc_alloc_host(sizeof(*host), &gbphy_dev->dev);
+	if (!mmc)
+		return -ENOMEM;
+
+	connection = gb_connection_create(gbphy_dev->bundle,
+					  le16_to_cpu(gbphy_dev->cport_desc->id),
+					  gb_sdio_request_handler);
+	if (IS_ERR(connection)) {
+		ret = PTR_ERR(connection);
+		goto exit_mmc_free;
+	}
+
+	host = mmc_priv(mmc);
+	host->mmc = mmc;
+	host->removed = true;
+
+	host->connection = connection;
+	gb_connection_set_data(connection, host);
+	host->gbphy_dev = gbphy_dev;
+	gb_gbphy_set_data(gbphy_dev, host);
+
+	ret = gb_connection_enable_tx(connection);
+	if (ret)
+		goto exit_connection_destroy;
+
+	ret = gb_sdio_get_caps(host);
+	if (ret < 0)
+		goto exit_connection_disable;
+
+	mmc->ops = &gb_sdio_ops;
+
+	mmc->max_segs = host->mmc->max_blk_count;
+
+	/* for now we make a map 1:1 between max request and segment size */
+	mmc->max_req_size = mmc->max_blk_size * mmc->max_blk_count;
+	mmc->max_seg_size = mmc->max_req_size;
+
+	mutex_init(&host->lock);
+	spin_lock_init(&host->xfer);
+	host->mrq_workqueue = alloc_workqueue("mmc-%s", 0, 1,
+					      dev_name(&gbphy_dev->dev));
+	if (!host->mrq_workqueue) {
+		ret = -ENOMEM;
+		goto exit_connection_disable;
+	}
+	INIT_WORK(&host->mrqwork, gb_sdio_mrq_work);
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto exit_wq_destroy;
+
+	ret = mmc_add_host(mmc);
+	if (ret < 0)
+		goto exit_wq_destroy;
+	host->removed = false;
+	ret = _gb_sdio_process_events(host, host->queued_events);
+	host->queued_events = 0;
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+
+	return ret;
+
+exit_wq_destroy:
+	destroy_workqueue(host->mrq_workqueue);
+exit_connection_disable:
+	gb_connection_disable(connection);
+exit_connection_destroy:
+	gb_connection_destroy(connection);
+exit_mmc_free:
+	mmc_free_host(mmc);
+
+	return ret;
+}
+
+static void gb_sdio_remove(struct gbphy_device *gbphy_dev)
+{
+	struct gb_sdio_host *host = gb_gbphy_get_data(gbphy_dev);
+	struct gb_connection *connection = host->connection;
+	struct mmc_host *mmc;
+	int ret;
+
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		gbphy_runtime_get_noresume(gbphy_dev);
+
+	mutex_lock(&host->lock);
+	host->removed = true;
+	mmc = host->mmc;
+	gb_connection_set_data(connection, NULL);
+	mutex_unlock(&host->lock);
+
+	flush_workqueue(host->mrq_workqueue);
+	destroy_workqueue(host->mrq_workqueue);
+	gb_connection_disable_rx(connection);
+	mmc_remove_host(mmc);
+	gb_connection_disable(connection);
+	gb_connection_destroy(connection);
+	mmc_free_host(mmc);
+}
+
+static const struct gbphy_device_id gb_sdio_id_table[] = {
+	{ GBPHY_PROTOCOL(GREYBUS_PROTOCOL_SDIO) },
+	{ },
+};
+MODULE_DEVICE_TABLE(gbphy, gb_sdio_id_table);
+
+static struct gbphy_driver sdio_driver = {
+	.name		= "sdio",
+	.probe		= gb_sdio_probe,
+	.remove		= gb_sdio_remove,
+	.id_table	= gb_sdio_id_table,
+};
+
+module_gbphy_driver(sdio_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/spi.c b/drivers/staging/greybus/spi.c
new file mode 100644
index 000000000000..c893552b5c0b
--- /dev/null
+++ b/drivers/staging/greybus/spi.c
@@ -0,0 +1,79 @@
+/*
+ * SPI bridge PHY driver.
+ *
+ * Copyright 2014-2016 Google Inc.
+ * Copyright 2014-2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/module.h>
+
+#include "greybus.h"
+#include "gbphy.h"
+#include "spilib.h"
+
+static struct spilib_ops *spilib_ops;
+
+static int gb_spi_probe(struct gbphy_device *gbphy_dev,
+			const struct gbphy_device_id *id)
+{
+	struct gb_connection *connection;
+	int ret;
+
+	connection = gb_connection_create(gbphy_dev->bundle,
+					  le16_to_cpu(gbphy_dev->cport_desc->id),
+					  NULL);
+	if (IS_ERR(connection))
+		return PTR_ERR(connection);
+
+	ret = gb_connection_enable(connection);
+	if (ret)
+		goto exit_connection_destroy;
+
+	ret = gb_spilib_master_init(connection, &gbphy_dev->dev, spilib_ops);
+	if (ret)
+		goto exit_connection_disable;
+
+	gb_gbphy_set_data(gbphy_dev, connection);
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+	return 0;
+
+exit_connection_disable:
+	gb_connection_disable(connection);
+exit_connection_destroy:
+	gb_connection_destroy(connection);
+
+	return ret;
+}
+
+static void gb_spi_remove(struct gbphy_device *gbphy_dev)
+{
+	struct gb_connection *connection = gb_gbphy_get_data(gbphy_dev);
+	int ret;
+
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		gbphy_runtime_get_noresume(gbphy_dev);
+
+	gb_spilib_master_exit(connection);
+	gb_connection_disable(connection);
+	gb_connection_destroy(connection);
+}
+
+static const struct gbphy_device_id gb_spi_id_table[] = {
+	{ GBPHY_PROTOCOL(GREYBUS_PROTOCOL_SPI) },
+	{ },
+};
+MODULE_DEVICE_TABLE(gbphy, gb_spi_id_table);
+
+static struct gbphy_driver spi_driver = {
+	.name		= "spi",
+	.probe		= gb_spi_probe,
+	.remove		= gb_spi_remove,
+	.id_table	= gb_spi_id_table,
+};
+
+module_gbphy_driver(spi_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/spilib.c b/drivers/staging/greybus/spilib.c
new file mode 100644
index 000000000000..e97b19148497
--- /dev/null
+++ b/drivers/staging/greybus/spilib.c
@@ -0,0 +1,565 @@
+/*
+ * Greybus SPI library
+ *
+ * Copyright 2014-2016 Google Inc.
+ * Copyright 2014-2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/bitops.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/spi/spi.h>
+
+#include "greybus.h"
+#include "spilib.h"
+
+struct gb_spilib {
+	struct gb_connection	*connection;
+	struct device		*parent;
+	struct spi_transfer	*first_xfer;
+	struct spi_transfer	*last_xfer;
+	struct spilib_ops	*ops;
+	u32			rx_xfer_offset;
+	u32			tx_xfer_offset;
+	u32			last_xfer_size;
+	unsigned int		op_timeout;
+	u16			mode;
+	u16			flags;
+	u32			bits_per_word_mask;
+	u8			num_chipselect;
+	u32			min_speed_hz;
+	u32			max_speed_hz;
+};
+
+#define GB_SPI_STATE_MSG_DONE		((void *)0)
+#define GB_SPI_STATE_MSG_IDLE		((void *)1)
+#define GB_SPI_STATE_MSG_RUNNING	((void *)2)
+#define GB_SPI_STATE_OP_READY		((void *)3)
+#define GB_SPI_STATE_OP_DONE		((void *)4)
+#define GB_SPI_STATE_MSG_ERROR		((void *)-1)
+
+#define XFER_TIMEOUT_TOLERANCE		200
+
+static struct spi_master *get_master_from_spi(struct gb_spilib *spi)
+{
+	return gb_connection_get_data(spi->connection);
+}
+
+static int tx_header_fit_operation(u32 tx_size, u32 count, size_t data_max)
+{
+	size_t headers_size;
+
+	data_max -= sizeof(struct gb_spi_transfer_request);
+	headers_size = (count + 1) * sizeof(struct gb_spi_transfer);
+
+	return tx_size + headers_size > data_max ? 0 : 1;
+}
+
+static size_t calc_rx_xfer_size(u32 rx_size, u32 *tx_xfer_size, u32 len,
+				size_t data_max)
+{
+	size_t rx_xfer_size;
+
+	data_max -= sizeof(struct gb_spi_transfer_response);
+
+	if (rx_size + len > data_max)
+		rx_xfer_size = data_max - rx_size;
+	else
+		rx_xfer_size = len;
+
+	/* if this is a write_read, for symmetry read the same as write */
+	if (*tx_xfer_size && rx_xfer_size > *tx_xfer_size)
+		rx_xfer_size = *tx_xfer_size;
+	if (*tx_xfer_size && rx_xfer_size < *tx_xfer_size)
+		*tx_xfer_size = rx_xfer_size;
+
+	return rx_xfer_size;
+}
+
+static size_t calc_tx_xfer_size(u32 tx_size, u32 count, size_t len,
+				size_t data_max)
+{
+	size_t headers_size;
+
+	data_max -= sizeof(struct gb_spi_transfer_request);
+	headers_size = (count + 1) * sizeof(struct gb_spi_transfer);
+
+	if (tx_size + headers_size + len > data_max)
+		return data_max - (tx_size + sizeof(struct gb_spi_transfer));
+
+	return len;
+}
+
+static void clean_xfer_state(struct gb_spilib *spi)
+{
+	spi->first_xfer = NULL;
+	spi->last_xfer = NULL;
+	spi->rx_xfer_offset = 0;
+	spi->tx_xfer_offset = 0;
+	spi->last_xfer_size = 0;
+	spi->op_timeout = 0;
+}
+
+static bool is_last_xfer_done(struct gb_spilib *spi)
+{
+	struct spi_transfer *last_xfer = spi->last_xfer;
+
+	if ((spi->tx_xfer_offset + spi->last_xfer_size == last_xfer->len) ||
+	    (spi->rx_xfer_offset + spi->last_xfer_size == last_xfer->len))
+		return true;
+
+	return false;
+}
+
+static int setup_next_xfer(struct gb_spilib *spi, struct spi_message *msg)
+{
+	struct spi_transfer *last_xfer = spi->last_xfer;
+
+	if (msg->state != GB_SPI_STATE_OP_DONE)
+		return 0;
+
+	/*
+	 * if we transferred all content of the last transfer, reset values and
+	 * check if this was the last transfer in the message
+	 */
+	if (is_last_xfer_done(spi)) {
+		spi->tx_xfer_offset = 0;
+		spi->rx_xfer_offset = 0;
+		spi->op_timeout = 0;
+		if (last_xfer == list_last_entry(&msg->transfers,
+						 struct spi_transfer,
+						 transfer_list))
+			msg->state = GB_SPI_STATE_MSG_DONE;
+		else
+			spi->first_xfer = list_next_entry(last_xfer,
+							  transfer_list);
+		return 0;
+	}
+
+	spi->first_xfer = last_xfer;
+	if (last_xfer->tx_buf)
+		spi->tx_xfer_offset += spi->last_xfer_size;
+
+	if (last_xfer->rx_buf)
+		spi->rx_xfer_offset += spi->last_xfer_size;
+
+	return 0;
+}
+
+static struct spi_transfer *get_next_xfer(struct spi_transfer *xfer,
+					  struct spi_message *msg)
+{
+	if (xfer == list_last_entry(&msg->transfers, struct spi_transfer,
+				    transfer_list))
+		return NULL;
+
+	return list_next_entry(xfer, transfer_list);
+}
+
+/* Routines to transfer data */
+static struct gb_operation *gb_spi_operation_create(struct gb_spilib *spi,
+		struct gb_connection *connection, struct spi_message *msg)
+{
+	struct gb_spi_transfer_request *request;
+	struct spi_device *dev = msg->spi;
+	struct spi_transfer *xfer;
+	struct gb_spi_transfer *gb_xfer;
+	struct gb_operation *operation;
+	u32 tx_size = 0, rx_size = 0, count = 0, xfer_len = 0, request_size;
+	u32 tx_xfer_size = 0, rx_xfer_size = 0, len;
+	u32 total_len = 0;
+	unsigned int xfer_timeout;
+	size_t data_max;
+	void *tx_data;
+
+	data_max = gb_operation_get_payload_size_max(connection);
+	xfer = spi->first_xfer;
+
+	/* Find number of transfers queued and tx/rx length in the message */
+
+	while (msg->state != GB_SPI_STATE_OP_READY) {
+		msg->state = GB_SPI_STATE_MSG_RUNNING;
+		spi->last_xfer = xfer;
+
+		if (!xfer->tx_buf && !xfer->rx_buf) {
+			dev_err(spi->parent,
+				"bufferless transfer, length %u\n", xfer->len);
+			msg->state = GB_SPI_STATE_MSG_ERROR;
+			return NULL;
+		}
+
+		tx_xfer_size = 0;
+		rx_xfer_size = 0;
+
+		if (xfer->tx_buf) {
+			len = xfer->len - spi->tx_xfer_offset;
+			if (!tx_header_fit_operation(tx_size, count, data_max))
+				break;
+			tx_xfer_size = calc_tx_xfer_size(tx_size, count,
+							 len, data_max);
+			spi->last_xfer_size = tx_xfer_size;
+		}
+
+		if (xfer->rx_buf) {
+			len = xfer->len - spi->rx_xfer_offset;
+			rx_xfer_size = calc_rx_xfer_size(rx_size, &tx_xfer_size,
+							 len, data_max);
+			spi->last_xfer_size = rx_xfer_size;
+		}
+
+		tx_size += tx_xfer_size;
+		rx_size += rx_xfer_size;
+
+		total_len += spi->last_xfer_size;
+		count++;
+
+		xfer = get_next_xfer(xfer, msg);
+		if (!xfer || total_len >= data_max)
+			msg->state = GB_SPI_STATE_OP_READY;
+	}
+
+	/*
+	 * In addition to space for all message descriptors we need
+	 * to have enough to hold all tx data.
+	 */
+	request_size = sizeof(*request);
+	request_size += count * sizeof(*gb_xfer);
+	request_size += tx_size;
+
+	/* Response consists only of incoming data */
+	operation = gb_operation_create(connection, GB_SPI_TYPE_TRANSFER,
+					request_size, rx_size, GFP_KERNEL);
+	if (!operation)
+		return NULL;
+
+	request = operation->request->payload;
+	request->count = cpu_to_le16(count);
+	request->mode = dev->mode;
+	request->chip_select = dev->chip_select;
+
+	gb_xfer = &request->transfers[0];
+	tx_data = gb_xfer + count;	/* place tx data after last gb_xfer */
+
+	/* Fill in the transfers array */
+	xfer = spi->first_xfer;
+	while (msg->state != GB_SPI_STATE_OP_DONE) {
+		if (xfer == spi->last_xfer)
+			xfer_len = spi->last_xfer_size;
+		else
+			xfer_len = xfer->len;
+
+		/* make sure we do not timeout in a slow transfer */
+		xfer_timeout = xfer_len * 8 * MSEC_PER_SEC / xfer->speed_hz;
+		xfer_timeout += GB_OPERATION_TIMEOUT_DEFAULT;
+
+		if (xfer_timeout > spi->op_timeout)
+			spi->op_timeout = xfer_timeout;
+
+		gb_xfer->speed_hz = cpu_to_le32(xfer->speed_hz);
+		gb_xfer->len = cpu_to_le32(xfer_len);
+		gb_xfer->delay_usecs = cpu_to_le16(xfer->delay_usecs);
+		gb_xfer->cs_change = xfer->cs_change;
+		gb_xfer->bits_per_word = xfer->bits_per_word;
+
+		/* Copy tx data */
+		if (xfer->tx_buf) {
+			gb_xfer->xfer_flags |= GB_SPI_XFER_WRITE;
+			memcpy(tx_data, xfer->tx_buf + spi->tx_xfer_offset,
+			       xfer_len);
+			tx_data += xfer_len;
+		}
+
+		if (xfer->rx_buf)
+			gb_xfer->xfer_flags |= GB_SPI_XFER_READ;
+
+		if (xfer == spi->last_xfer) {
+			if (!is_last_xfer_done(spi))
+				gb_xfer->xfer_flags |= GB_SPI_XFER_INPROGRESS;
+			msg->state = GB_SPI_STATE_OP_DONE;
+			continue;
+		}
+
+		gb_xfer++;
+		xfer = get_next_xfer(xfer, msg);
+	}
+
+	msg->actual_length += total_len;
+
+	return operation;
+}
+
+static void gb_spi_decode_response(struct gb_spilib *spi,
+				   struct spi_message *msg,
+				   struct gb_spi_transfer_response *response)
+{
+	struct spi_transfer *xfer = spi->first_xfer;
+	void *rx_data = response->data;
+	u32 xfer_len;
+
+	while (xfer) {
+		/* Copy rx data */
+		if (xfer->rx_buf) {
+			if (xfer == spi->first_xfer)
+				xfer_len = xfer->len - spi->rx_xfer_offset;
+			else if (xfer == spi->last_xfer)
+				xfer_len = spi->last_xfer_size;
+			else
+				xfer_len = xfer->len;
+
+			memcpy(xfer->rx_buf + spi->rx_xfer_offset, rx_data,
+			       xfer_len);
+			rx_data += xfer_len;
+		}
+
+		if (xfer == spi->last_xfer)
+			break;
+
+		xfer = list_next_entry(xfer, transfer_list);
+	}
+}
+
+static int gb_spi_transfer_one_message(struct spi_master *master,
+				       struct spi_message *msg)
+{
+	struct gb_spilib *spi = spi_master_get_devdata(master);
+	struct gb_connection *connection = spi->connection;
+	struct gb_spi_transfer_response *response;
+	struct gb_operation *operation;
+	int ret = 0;
+
+	spi->first_xfer = list_first_entry_or_null(&msg->transfers,
+						   struct spi_transfer,
+						   transfer_list);
+	if (!spi->first_xfer) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	msg->state = GB_SPI_STATE_MSG_IDLE;
+
+	while (msg->state != GB_SPI_STATE_MSG_DONE &&
+	       msg->state != GB_SPI_STATE_MSG_ERROR) {
+		operation = gb_spi_operation_create(spi, connection, msg);
+		if (!operation) {
+			msg->state = GB_SPI_STATE_MSG_ERROR;
+			ret = -EINVAL;
+			continue;
+		}
+
+		ret = gb_operation_request_send_sync_timeout(operation,
+							     spi->op_timeout);
+		if (!ret) {
+			response = operation->response->payload;
+			if (response)
+				gb_spi_decode_response(spi, msg, response);
+		} else {
+			dev_err(spi->parent,
+				"transfer operation failed: %d\n", ret);
+			msg->state = GB_SPI_STATE_MSG_ERROR;
+		}
+
+		gb_operation_put(operation);
+		setup_next_xfer(spi, msg);
+	}
+
+out:
+	msg->status = ret;
+	clean_xfer_state(spi);
+	spi_finalize_current_message(master);
+
+	return ret;
+}
+
+static int gb_spi_prepare_transfer_hardware(struct spi_master *master)
+{
+	struct gb_spilib *spi = spi_master_get_devdata(master);
+
+	return spi->ops->prepare_transfer_hardware(spi->parent);
+}
+
+static int gb_spi_unprepare_transfer_hardware(struct spi_master *master)
+{
+	struct gb_spilib *spi = spi_master_get_devdata(master);
+
+	spi->ops->unprepare_transfer_hardware(spi->parent);
+
+	return 0;
+}
+
+static int gb_spi_setup(struct spi_device *spi)
+{
+	/* Nothing to do for now */
+	return 0;
+}
+
+static void gb_spi_cleanup(struct spi_device *spi)
+{
+	/* Nothing to do for now */
+}
+
+/* Routines to get controller information */
+
+/*
+ * Map Greybus spi mode bits/flags/bpw into Linux ones.
+ * All bits are same for now and so these macro's return same values.
+ */
+#define gb_spi_mode_map(mode) mode
+#define gb_spi_flags_map(flags) flags
+
+static int gb_spi_get_master_config(struct gb_spilib *spi)
+{
+	struct gb_spi_master_config_response response;
+	u16 mode, flags;
+	int ret;
+
+	ret = gb_operation_sync(spi->connection, GB_SPI_TYPE_MASTER_CONFIG,
+				NULL, 0, &response, sizeof(response));
+	if (ret < 0)
+		return ret;
+
+	mode = le16_to_cpu(response.mode);
+	spi->mode = gb_spi_mode_map(mode);
+
+	flags = le16_to_cpu(response.flags);
+	spi->flags = gb_spi_flags_map(flags);
+
+	spi->bits_per_word_mask = le32_to_cpu(response.bits_per_word_mask);
+	spi->num_chipselect = response.num_chipselect;
+
+	spi->min_speed_hz = le32_to_cpu(response.min_speed_hz);
+	spi->max_speed_hz = le32_to_cpu(response.max_speed_hz);
+
+	return 0;
+}
+
+static int gb_spi_setup_device(struct gb_spilib *spi, u8 cs)
+{
+	struct spi_master *master = get_master_from_spi(spi);
+	struct gb_spi_device_config_request request;
+	struct gb_spi_device_config_response response;
+	struct spi_board_info spi_board = { {0} };
+	struct spi_device *spidev;
+	int ret;
+	u8 dev_type;
+
+	request.chip_select = cs;
+
+	ret = gb_operation_sync(spi->connection, GB_SPI_TYPE_DEVICE_CONFIG,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret < 0)
+		return ret;
+
+	dev_type = response.device_type;
+
+	if (dev_type == GB_SPI_SPI_DEV)
+		strlcpy(spi_board.modalias, "spidev",
+			sizeof(spi_board.modalias));
+	else if (dev_type == GB_SPI_SPI_NOR)
+		strlcpy(spi_board.modalias, "spi-nor",
+			sizeof(spi_board.modalias));
+	else if (dev_type == GB_SPI_SPI_MODALIAS)
+		memcpy(spi_board.modalias, response.name,
+		       sizeof(spi_board.modalias));
+	else
+		return -EINVAL;
+
+	spi_board.mode		= le16_to_cpu(response.mode);
+	spi_board.bus_num	= master->bus_num;
+	spi_board.chip_select	= cs;
+	spi_board.max_speed_hz	= le32_to_cpu(response.max_speed_hz);
+
+	spidev = spi_new_device(master, &spi_board);
+	if (!spidev)
+		return -EINVAL;
+
+	return 0;
+}
+
+int gb_spilib_master_init(struct gb_connection *connection, struct device *dev,
+			  struct spilib_ops *ops)
+{
+	struct gb_spilib *spi;
+	struct spi_master *master;
+	int ret;
+	u8 i;
+
+	/* Allocate master with space for data */
+	master = spi_alloc_master(dev, sizeof(*spi));
+	if (!master) {
+		dev_err(dev, "cannot alloc SPI master\n");
+		return -ENOMEM;
+	}
+
+	spi = spi_master_get_devdata(master);
+	spi->connection = connection;
+	gb_connection_set_data(connection, master);
+	spi->parent = dev;
+	spi->ops = ops;
+
+	/* get master configuration */
+	ret = gb_spi_get_master_config(spi);
+	if (ret)
+		goto exit_spi_put;
+
+	master->bus_num = -1; /* Allow spi-core to allocate it dynamically */
+	master->num_chipselect = spi->num_chipselect;
+	master->mode_bits = spi->mode;
+	master->flags = spi->flags;
+	master->bits_per_word_mask = spi->bits_per_word_mask;
+
+	/* Attach methods */
+	master->cleanup = gb_spi_cleanup;
+	master->setup = gb_spi_setup;
+	master->transfer_one_message = gb_spi_transfer_one_message;
+
+	if (ops && ops->prepare_transfer_hardware) {
+		master->prepare_transfer_hardware =
+			gb_spi_prepare_transfer_hardware;
+	}
+
+	if (ops && ops->unprepare_transfer_hardware) {
+		master->unprepare_transfer_hardware =
+			gb_spi_unprepare_transfer_hardware;
+	}
+
+	master->auto_runtime_pm = true;
+
+	ret = spi_register_master(master);
+	if (ret < 0)
+		goto exit_spi_put;
+
+	/* now, fetch the devices configuration */
+	for (i = 0; i < spi->num_chipselect; i++) {
+		ret = gb_spi_setup_device(spi, i);
+		if (ret < 0) {
+			dev_err(dev, "failed to allocate spi device %d: %d\n",
+				i, ret);
+			goto exit_spi_unregister;
+		}
+	}
+
+	return 0;
+
+exit_spi_unregister:
+	spi_unregister_master(master);
+exit_spi_put:
+	spi_master_put(master);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_spilib_master_init);
+
+void gb_spilib_master_exit(struct gb_connection *connection)
+{
+	struct spi_master *master = gb_connection_get_data(connection);
+
+	spi_unregister_master(master);
+	spi_master_put(master);
+}
+EXPORT_SYMBOL_GPL(gb_spilib_master_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/spilib.h b/drivers/staging/greybus/spilib.h
new file mode 100644
index 000000000000..566d0dde7f79
--- /dev/null
+++ b/drivers/staging/greybus/spilib.h
@@ -0,0 +1,24 @@
+/*
+ * Greybus SPI library header
+ *
+ * copyright 2016 google inc.
+ * copyright 2016 linaro ltd.
+ *
+ * released under the gplv2 only.
+ */
+
+#ifndef __SPILIB_H
+#define __SPILIB_H
+
+struct device;
+struct gb_connection;
+
+struct spilib_ops {
+	int (*prepare_transfer_hardware)(struct device *dev);
+	void (*unprepare_transfer_hardware)(struct device *dev);
+};
+
+int gb_spilib_master_init(struct gb_connection *connection, struct device *dev, struct spilib_ops *ops);
+void gb_spilib_master_exit(struct gb_connection *connection);
+
+#endif /* __SPILIB_H */
diff --git a/drivers/staging/greybus/svc.c b/drivers/staging/greybus/svc.c
new file mode 100644
index 000000000000..550055ec27a5
--- /dev/null
+++ b/drivers/staging/greybus/svc.c
@@ -0,0 +1,1486 @@
+/*
+ * SVC Greybus driver.
+ *
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/debugfs.h>
+#include <linux/workqueue.h>
+
+#include "greybus.h"
+
+#define SVC_INTF_EJECT_TIMEOUT		9000
+#define SVC_INTF_ACTIVATE_TIMEOUT	6000
+#define SVC_INTF_RESUME_TIMEOUT		3000
+
+struct gb_svc_deferred_request {
+	struct work_struct work;
+	struct gb_operation *operation;
+};
+
+
+static int gb_svc_queue_deferred_request(struct gb_operation *operation);
+
+static ssize_t endo_id_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	struct gb_svc *svc = to_gb_svc(dev);
+
+	return sprintf(buf, "0x%04x\n", svc->endo_id);
+}
+static DEVICE_ATTR_RO(endo_id);
+
+static ssize_t ap_intf_id_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	struct gb_svc *svc = to_gb_svc(dev);
+
+	return sprintf(buf, "%u\n", svc->ap_intf_id);
+}
+static DEVICE_ATTR_RO(ap_intf_id);
+
+// FIXME
+// This is a hack, we need to do this "right" and clean the interface up
+// properly, not just forcibly yank the thing out of the system and hope for the
+// best.  But for now, people want their modules to come out without having to
+// throw the thing to the ground or get out a screwdriver.
+static ssize_t intf_eject_store(struct device *dev,
+				struct device_attribute *attr, const char *buf,
+				size_t len)
+{
+	struct gb_svc *svc = to_gb_svc(dev);
+	unsigned short intf_id;
+	int ret;
+
+	ret = kstrtou16(buf, 10, &intf_id);
+	if (ret < 0)
+		return ret;
+
+	dev_warn(dev, "Forcibly trying to eject interface %d\n", intf_id);
+
+	ret = gb_svc_intf_eject(svc, intf_id);
+	if (ret < 0)
+		return ret;
+
+	return len;
+}
+static DEVICE_ATTR_WO(intf_eject);
+
+static ssize_t watchdog_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	struct gb_svc *svc = to_gb_svc(dev);
+
+	return sprintf(buf, "%s\n",
+		       gb_svc_watchdog_enabled(svc) ? "enabled" : "disabled");
+}
+
+static ssize_t watchdog_store(struct device *dev,
+			      struct device_attribute *attr, const char *buf,
+			      size_t len)
+{
+	struct gb_svc *svc = to_gb_svc(dev);
+	int retval;
+	bool user_request;
+
+	retval = strtobool(buf, &user_request);
+	if (retval)
+		return retval;
+
+	if (user_request)
+		retval = gb_svc_watchdog_enable(svc);
+	else
+		retval = gb_svc_watchdog_disable(svc);
+	if (retval)
+		return retval;
+	return len;
+}
+static DEVICE_ATTR_RW(watchdog);
+
+static ssize_t watchdog_action_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct gb_svc *svc = to_gb_svc(dev);
+
+	if (svc->action == GB_SVC_WATCHDOG_BITE_PANIC_KERNEL)
+		return sprintf(buf, "panic\n");
+	else if (svc->action == GB_SVC_WATCHDOG_BITE_RESET_UNIPRO)
+		return sprintf(buf, "reset\n");
+
+	return -EINVAL;
+}
+
+static ssize_t watchdog_action_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t len)
+{
+	struct gb_svc *svc = to_gb_svc(dev);
+
+	if (sysfs_streq(buf, "panic"))
+		svc->action = GB_SVC_WATCHDOG_BITE_PANIC_KERNEL;
+	else if (sysfs_streq(buf, "reset"))
+		svc->action = GB_SVC_WATCHDOG_BITE_RESET_UNIPRO;
+	else
+		return -EINVAL;
+
+	return len;
+}
+static DEVICE_ATTR_RW(watchdog_action);
+
+static int gb_svc_pwrmon_rail_count_get(struct gb_svc *svc, u8 *value)
+{
+	struct gb_svc_pwrmon_rail_count_get_response response;
+	int ret;
+
+	ret = gb_operation_sync(svc->connection,
+				GB_SVC_TYPE_PWRMON_RAIL_COUNT_GET, NULL, 0,
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(&svc->dev, "failed to get rail count: %d\n", ret);
+		return ret;
+	}
+
+	*value = response.rail_count;
+
+	return 0;
+}
+
+static int gb_svc_pwrmon_rail_names_get(struct gb_svc *svc,
+		struct gb_svc_pwrmon_rail_names_get_response *response,
+		size_t bufsize)
+{
+	int ret;
+
+	ret = gb_operation_sync(svc->connection,
+				GB_SVC_TYPE_PWRMON_RAIL_NAMES_GET, NULL, 0,
+				response, bufsize);
+	if (ret) {
+		dev_err(&svc->dev, "failed to get rail names: %d\n", ret);
+		return ret;
+	}
+
+	if (response->status != GB_SVC_OP_SUCCESS) {
+		dev_err(&svc->dev,
+			"SVC error while getting rail names: %u\n",
+			response->status);
+		return -EREMOTEIO;
+	}
+
+	return 0;
+}
+
+static int gb_svc_pwrmon_sample_get(struct gb_svc *svc, u8 rail_id,
+				    u8 measurement_type, u32 *value)
+{
+	struct gb_svc_pwrmon_sample_get_request request;
+	struct gb_svc_pwrmon_sample_get_response response;
+	int ret;
+
+	request.rail_id = rail_id;
+	request.measurement_type = measurement_type;
+
+	ret = gb_operation_sync(svc->connection, GB_SVC_TYPE_PWRMON_SAMPLE_GET,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(&svc->dev, "failed to get rail sample: %d\n", ret);
+		return ret;
+	}
+
+	if (response.result) {
+		dev_err(&svc->dev,
+			"UniPro error while getting rail power sample (%d %d): %d\n",
+			rail_id, measurement_type, response.result);
+		switch (response.result) {
+		case GB_SVC_PWRMON_GET_SAMPLE_INVAL:
+			return -EINVAL;
+		case GB_SVC_PWRMON_GET_SAMPLE_NOSUPP:
+			return -ENOMSG;
+		default:
+			return -EREMOTEIO;
+		}
+	}
+
+	*value = le32_to_cpu(response.measurement);
+
+	return 0;
+}
+
+int gb_svc_pwrmon_intf_sample_get(struct gb_svc *svc, u8 intf_id,
+				  u8 measurement_type, u32 *value)
+{
+	struct gb_svc_pwrmon_intf_sample_get_request request;
+	struct gb_svc_pwrmon_intf_sample_get_response response;
+	int ret;
+
+	request.intf_id = intf_id;
+	request.measurement_type = measurement_type;
+
+	ret = gb_operation_sync(svc->connection,
+				GB_SVC_TYPE_PWRMON_INTF_SAMPLE_GET,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(&svc->dev, "failed to get intf sample: %d\n", ret);
+		return ret;
+	}
+
+	if (response.result) {
+		dev_err(&svc->dev,
+			"UniPro error while getting intf power sample (%d %d): %d\n",
+			intf_id, measurement_type, response.result);
+		switch (response.result) {
+		case GB_SVC_PWRMON_GET_SAMPLE_INVAL:
+			return -EINVAL;
+		case GB_SVC_PWRMON_GET_SAMPLE_NOSUPP:
+			return -ENOMSG;
+		default:
+			return -EREMOTEIO;
+		}
+	}
+
+	*value = le32_to_cpu(response.measurement);
+
+	return 0;
+}
+
+static struct attribute *svc_attrs[] = {
+	&dev_attr_endo_id.attr,
+	&dev_attr_ap_intf_id.attr,
+	&dev_attr_intf_eject.attr,
+	&dev_attr_watchdog.attr,
+	&dev_attr_watchdog_action.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(svc);
+
+int gb_svc_intf_device_id(struct gb_svc *svc, u8 intf_id, u8 device_id)
+{
+	struct gb_svc_intf_device_id_request request;
+
+	request.intf_id = intf_id;
+	request.device_id = device_id;
+
+	return gb_operation_sync(svc->connection, GB_SVC_TYPE_INTF_DEVICE_ID,
+				 &request, sizeof(request), NULL, 0);
+}
+
+int gb_svc_intf_eject(struct gb_svc *svc, u8 intf_id)
+{
+	struct gb_svc_intf_eject_request request;
+	int ret;
+
+	request.intf_id = intf_id;
+
+	/*
+	 * The pulse width for module release in svc is long so we need to
+	 * increase the timeout so the operation will not return to soon.
+	 */
+	ret = gb_operation_sync_timeout(svc->connection,
+					GB_SVC_TYPE_INTF_EJECT, &request,
+					sizeof(request), NULL, 0,
+					SVC_INTF_EJECT_TIMEOUT);
+	if (ret) {
+		dev_err(&svc->dev, "failed to eject interface %u\n", intf_id);
+		return ret;
+	}
+
+	return 0;
+}
+
+int gb_svc_intf_vsys_set(struct gb_svc *svc, u8 intf_id, bool enable)
+{
+	struct gb_svc_intf_vsys_request request;
+	struct gb_svc_intf_vsys_response response;
+	int type, ret;
+
+	request.intf_id = intf_id;
+
+	if (enable)
+		type = GB_SVC_TYPE_INTF_VSYS_ENABLE;
+	else
+		type = GB_SVC_TYPE_INTF_VSYS_DISABLE;
+
+	ret = gb_operation_sync(svc->connection, type,
+			&request, sizeof(request),
+			&response, sizeof(response));
+	if (ret < 0)
+		return ret;
+	if (response.result_code != GB_SVC_INTF_VSYS_OK)
+		return -EREMOTEIO;
+	return 0;
+}
+
+int gb_svc_intf_refclk_set(struct gb_svc *svc, u8 intf_id, bool enable)
+{
+	struct gb_svc_intf_refclk_request request;
+	struct gb_svc_intf_refclk_response response;
+	int type, ret;
+
+	request.intf_id = intf_id;
+
+	if (enable)
+		type = GB_SVC_TYPE_INTF_REFCLK_ENABLE;
+	else
+		type = GB_SVC_TYPE_INTF_REFCLK_DISABLE;
+
+	ret = gb_operation_sync(svc->connection, type,
+			&request, sizeof(request),
+			&response, sizeof(response));
+	if (ret < 0)
+		return ret;
+	if (response.result_code != GB_SVC_INTF_REFCLK_OK)
+		return -EREMOTEIO;
+	return 0;
+}
+
+int gb_svc_intf_unipro_set(struct gb_svc *svc, u8 intf_id, bool enable)
+{
+	struct gb_svc_intf_unipro_request request;
+	struct gb_svc_intf_unipro_response response;
+	int type, ret;
+
+	request.intf_id = intf_id;
+
+	if (enable)
+		type = GB_SVC_TYPE_INTF_UNIPRO_ENABLE;
+	else
+		type = GB_SVC_TYPE_INTF_UNIPRO_DISABLE;
+
+	ret = gb_operation_sync(svc->connection, type,
+			&request, sizeof(request),
+			&response, sizeof(response));
+	if (ret < 0)
+		return ret;
+	if (response.result_code != GB_SVC_INTF_UNIPRO_OK)
+		return -EREMOTEIO;
+	return 0;
+}
+
+int gb_svc_intf_activate(struct gb_svc *svc, u8 intf_id, u8 *intf_type)
+{
+	struct gb_svc_intf_activate_request request;
+	struct gb_svc_intf_activate_response response;
+	int ret;
+
+	request.intf_id = intf_id;
+
+	ret = gb_operation_sync_timeout(svc->connection,
+			GB_SVC_TYPE_INTF_ACTIVATE,
+			&request, sizeof(request),
+			&response, sizeof(response),
+			SVC_INTF_ACTIVATE_TIMEOUT);
+	if (ret < 0)
+		return ret;
+	if (response.status != GB_SVC_OP_SUCCESS) {
+		dev_err(&svc->dev, "failed to activate interface %u: %u\n",
+				intf_id, response.status);
+		return -EREMOTEIO;
+	}
+
+	*intf_type = response.intf_type;
+
+	return 0;
+}
+
+int gb_svc_intf_resume(struct gb_svc *svc, u8 intf_id)
+{
+	struct gb_svc_intf_resume_request request;
+	struct gb_svc_intf_resume_response response;
+	int ret;
+
+	request.intf_id = intf_id;
+
+	ret = gb_operation_sync_timeout(svc->connection,
+					GB_SVC_TYPE_INTF_RESUME,
+					&request, sizeof(request),
+					&response, sizeof(response),
+					SVC_INTF_RESUME_TIMEOUT);
+	if (ret < 0) {
+		dev_err(&svc->dev, "failed to send interface resume %u: %d\n",
+			intf_id, ret);
+		return ret;
+	}
+
+	if (response.status != GB_SVC_OP_SUCCESS) {
+		dev_err(&svc->dev, "failed to resume interface %u: %u\n",
+			intf_id, response.status);
+		return -EREMOTEIO;
+	}
+
+	return 0;
+}
+
+int gb_svc_dme_peer_get(struct gb_svc *svc, u8 intf_id, u16 attr, u16 selector,
+			u32 *value)
+{
+	struct gb_svc_dme_peer_get_request request;
+	struct gb_svc_dme_peer_get_response response;
+	u16 result;
+	int ret;
+
+	request.intf_id = intf_id;
+	request.attr = cpu_to_le16(attr);
+	request.selector = cpu_to_le16(selector);
+
+	ret = gb_operation_sync(svc->connection, GB_SVC_TYPE_DME_PEER_GET,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(&svc->dev, "failed to get DME attribute (%u 0x%04x %u): %d\n",
+				intf_id, attr, selector, ret);
+		return ret;
+	}
+
+	result = le16_to_cpu(response.result_code);
+	if (result) {
+		dev_err(&svc->dev, "UniPro error while getting DME attribute (%u 0x%04x %u): %u\n",
+				intf_id, attr, selector, result);
+		return -EREMOTEIO;
+	}
+
+	if (value)
+		*value = le32_to_cpu(response.attr_value);
+
+	return 0;
+}
+
+int gb_svc_dme_peer_set(struct gb_svc *svc, u8 intf_id, u16 attr, u16 selector,
+			u32 value)
+{
+	struct gb_svc_dme_peer_set_request request;
+	struct gb_svc_dme_peer_set_response response;
+	u16 result;
+	int ret;
+
+	request.intf_id = intf_id;
+	request.attr = cpu_to_le16(attr);
+	request.selector = cpu_to_le16(selector);
+	request.value = cpu_to_le32(value);
+
+	ret = gb_operation_sync(svc->connection, GB_SVC_TYPE_DME_PEER_SET,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret) {
+		dev_err(&svc->dev, "failed to set DME attribute (%u 0x%04x %u %u): %d\n",
+				intf_id, attr, selector, value, ret);
+		return ret;
+	}
+
+	result = le16_to_cpu(response.result_code);
+	if (result) {
+		dev_err(&svc->dev, "UniPro error while setting DME attribute (%u 0x%04x %u %u): %u\n",
+				intf_id, attr, selector, value, result);
+		return -EREMOTEIO;
+	}
+
+	return 0;
+}
+
+int gb_svc_connection_create(struct gb_svc *svc,
+				u8 intf1_id, u16 cport1_id,
+				u8 intf2_id, u16 cport2_id,
+				u8 cport_flags)
+{
+	struct gb_svc_conn_create_request request;
+
+	request.intf1_id = intf1_id;
+	request.cport1_id = cpu_to_le16(cport1_id);
+	request.intf2_id = intf2_id;
+	request.cport2_id = cpu_to_le16(cport2_id);
+	request.tc = 0;		/* TC0 */
+	request.flags = cport_flags;
+
+	return gb_operation_sync(svc->connection, GB_SVC_TYPE_CONN_CREATE,
+				 &request, sizeof(request), NULL, 0);
+}
+
+void gb_svc_connection_destroy(struct gb_svc *svc, u8 intf1_id, u16 cport1_id,
+			       u8 intf2_id, u16 cport2_id)
+{
+	struct gb_svc_conn_destroy_request request;
+	struct gb_connection *connection = svc->connection;
+	int ret;
+
+	request.intf1_id = intf1_id;
+	request.cport1_id = cpu_to_le16(cport1_id);
+	request.intf2_id = intf2_id;
+	request.cport2_id = cpu_to_le16(cport2_id);
+
+	ret = gb_operation_sync(connection, GB_SVC_TYPE_CONN_DESTROY,
+				&request, sizeof(request), NULL, 0);
+	if (ret) {
+		dev_err(&svc->dev, "failed to destroy connection (%u:%u %u:%u): %d\n",
+				intf1_id, cport1_id, intf2_id, cport2_id, ret);
+	}
+}
+
+int gb_svc_timesync_enable(struct gb_svc *svc, u8 count, u64 frame_time,
+			   u32 strobe_delay, u32 refclk)
+{
+	struct gb_connection *connection = svc->connection;
+	struct gb_svc_timesync_enable_request request;
+
+	request.count = count;
+	request.frame_time = cpu_to_le64(frame_time);
+	request.strobe_delay = cpu_to_le32(strobe_delay);
+	request.refclk = cpu_to_le32(refclk);
+	return gb_operation_sync(connection,
+				 GB_SVC_TYPE_TIMESYNC_ENABLE,
+				 &request, sizeof(request), NULL, 0);
+}
+
+int gb_svc_timesync_disable(struct gb_svc *svc)
+{
+	struct gb_connection *connection = svc->connection;
+
+	return gb_operation_sync(connection,
+				 GB_SVC_TYPE_TIMESYNC_DISABLE,
+				 NULL, 0, NULL, 0);
+}
+
+int gb_svc_timesync_authoritative(struct gb_svc *svc, u64 *frame_time)
+{
+	struct gb_connection *connection = svc->connection;
+	struct gb_svc_timesync_authoritative_response response;
+	int ret, i;
+
+	ret = gb_operation_sync(connection,
+				GB_SVC_TYPE_TIMESYNC_AUTHORITATIVE, NULL, 0,
+				&response, sizeof(response));
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < GB_TIMESYNC_MAX_STROBES; i++)
+		frame_time[i] = le64_to_cpu(response.frame_time[i]);
+	return 0;
+}
+
+int gb_svc_timesync_ping(struct gb_svc *svc, u64 *frame_time)
+{
+	struct gb_connection *connection = svc->connection;
+	struct gb_svc_timesync_ping_response response;
+	int ret;
+
+	ret = gb_operation_sync(connection,
+				GB_SVC_TYPE_TIMESYNC_PING,
+				NULL, 0,
+				&response, sizeof(response));
+	if (ret < 0)
+		return ret;
+
+	*frame_time = le64_to_cpu(response.frame_time);
+	return 0;
+}
+
+int gb_svc_timesync_wake_pins_acquire(struct gb_svc *svc, u32 strobe_mask)
+{
+	struct gb_connection *connection = svc->connection;
+	struct gb_svc_timesync_wake_pins_acquire_request request;
+
+	request.strobe_mask = cpu_to_le32(strobe_mask);
+	return gb_operation_sync(connection,
+				 GB_SVC_TYPE_TIMESYNC_WAKE_PINS_ACQUIRE,
+				 &request, sizeof(request),
+				 NULL, 0);
+}
+
+int gb_svc_timesync_wake_pins_release(struct gb_svc *svc)
+{
+	struct gb_connection *connection = svc->connection;
+
+	return gb_operation_sync(connection,
+				 GB_SVC_TYPE_TIMESYNC_WAKE_PINS_RELEASE,
+				 NULL, 0, NULL, 0);
+}
+
+/* Creates bi-directional routes between the devices */
+int gb_svc_route_create(struct gb_svc *svc, u8 intf1_id, u8 dev1_id,
+			       u8 intf2_id, u8 dev2_id)
+{
+	struct gb_svc_route_create_request request;
+
+	request.intf1_id = intf1_id;
+	request.dev1_id = dev1_id;
+	request.intf2_id = intf2_id;
+	request.dev2_id = dev2_id;
+
+	return gb_operation_sync(svc->connection, GB_SVC_TYPE_ROUTE_CREATE,
+				 &request, sizeof(request), NULL, 0);
+}
+
+/* Destroys bi-directional routes between the devices */
+void gb_svc_route_destroy(struct gb_svc *svc, u8 intf1_id, u8 intf2_id)
+{
+	struct gb_svc_route_destroy_request request;
+	int ret;
+
+	request.intf1_id = intf1_id;
+	request.intf2_id = intf2_id;
+
+	ret = gb_operation_sync(svc->connection, GB_SVC_TYPE_ROUTE_DESTROY,
+				&request, sizeof(request), NULL, 0);
+	if (ret) {
+		dev_err(&svc->dev, "failed to destroy route (%u %u): %d\n",
+				intf1_id, intf2_id, ret);
+	}
+}
+
+int gb_svc_intf_set_power_mode(struct gb_svc *svc, u8 intf_id, u8 hs_series,
+			       u8 tx_mode, u8 tx_gear, u8 tx_nlanes,
+			       u8 tx_amplitude, u8 tx_hs_equalizer,
+			       u8 rx_mode, u8 rx_gear, u8 rx_nlanes,
+			       u8 flags, u32 quirks,
+			       struct gb_svc_l2_timer_cfg *local,
+			       struct gb_svc_l2_timer_cfg *remote)
+{
+	struct gb_svc_intf_set_pwrm_request request;
+	struct gb_svc_intf_set_pwrm_response response;
+	int ret;
+	u16 result_code;
+
+	memset(&request, 0, sizeof(request));
+
+	request.intf_id = intf_id;
+	request.hs_series = hs_series;
+	request.tx_mode = tx_mode;
+	request.tx_gear = tx_gear;
+	request.tx_nlanes = tx_nlanes;
+	request.tx_amplitude = tx_amplitude;
+	request.tx_hs_equalizer = tx_hs_equalizer;
+	request.rx_mode = rx_mode;
+	request.rx_gear = rx_gear;
+	request.rx_nlanes = rx_nlanes;
+	request.flags = flags;
+	request.quirks = cpu_to_le32(quirks);
+	if (local)
+		request.local_l2timerdata = *local;
+	if (remote)
+		request.remote_l2timerdata = *remote;
+
+	ret = gb_operation_sync(svc->connection, GB_SVC_TYPE_INTF_SET_PWRM,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret < 0)
+		return ret;
+
+	result_code = response.result_code;
+	if (result_code != GB_SVC_SETPWRM_PWR_LOCAL) {
+		dev_err(&svc->dev, "set power mode = %d\n", result_code);
+		return -EIO;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gb_svc_intf_set_power_mode);
+
+int gb_svc_intf_set_power_mode_hibernate(struct gb_svc *svc, u8 intf_id)
+{
+	struct gb_svc_intf_set_pwrm_request request;
+	struct gb_svc_intf_set_pwrm_response response;
+	int ret;
+	u16 result_code;
+
+	memset(&request, 0, sizeof(request));
+
+	request.intf_id = intf_id;
+	request.hs_series = GB_SVC_UNIPRO_HS_SERIES_A;
+	request.tx_mode = GB_SVC_UNIPRO_HIBERNATE_MODE;
+	request.rx_mode = GB_SVC_UNIPRO_HIBERNATE_MODE;
+
+	ret = gb_operation_sync(svc->connection, GB_SVC_TYPE_INTF_SET_PWRM,
+				&request, sizeof(request),
+				&response, sizeof(response));
+	if (ret < 0) {
+		dev_err(&svc->dev,
+			"failed to send set power mode operation to interface %u: %d\n",
+			intf_id, ret);
+		return ret;
+	}
+
+	result_code = response.result_code;
+	if (result_code != GB_SVC_SETPWRM_PWR_OK) {
+		dev_err(&svc->dev,
+			"failed to hibernate the link for interface %u: %u\n",
+			intf_id, result_code);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+int gb_svc_ping(struct gb_svc *svc)
+{
+	return gb_operation_sync_timeout(svc->connection, GB_SVC_TYPE_PING,
+					 NULL, 0, NULL, 0,
+					 GB_OPERATION_TIMEOUT_DEFAULT * 2);
+}
+
+static int gb_svc_version_request(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_svc *svc = gb_connection_get_data(connection);
+	struct gb_svc_version_request *request;
+	struct gb_svc_version_response *response;
+
+	if (op->request->payload_size < sizeof(*request)) {
+		dev_err(&svc->dev, "short version request (%zu < %zu)\n",
+				op->request->payload_size,
+				sizeof(*request));
+		return -EINVAL;
+	}
+
+	request = op->request->payload;
+
+	if (request->major > GB_SVC_VERSION_MAJOR) {
+		dev_warn(&svc->dev, "unsupported major version (%u > %u)\n",
+				request->major, GB_SVC_VERSION_MAJOR);
+		return -ENOTSUPP;
+	}
+
+	svc->protocol_major = request->major;
+	svc->protocol_minor = request->minor;
+
+	if (!gb_operation_response_alloc(op, sizeof(*response), GFP_KERNEL))
+		return -ENOMEM;
+
+	response = op->response->payload;
+	response->major = svc->protocol_major;
+	response->minor = svc->protocol_minor;
+
+	return 0;
+}
+
+static ssize_t pwr_debugfs_voltage_read(struct file *file, char __user *buf,
+					size_t len, loff_t *offset)
+{
+	struct svc_debugfs_pwrmon_rail *pwrmon_rails = file->f_inode->i_private;
+	struct gb_svc *svc = pwrmon_rails->svc;
+	int ret, desc;
+	u32 value;
+	char buff[16];
+
+	ret = gb_svc_pwrmon_sample_get(svc, pwrmon_rails->id,
+				       GB_SVC_PWRMON_TYPE_VOL, &value);
+	if (ret) {
+		dev_err(&svc->dev,
+			"failed to get voltage sample %u: %d\n",
+			pwrmon_rails->id, ret);
+		return ret;
+	}
+
+	desc = scnprintf(buff, sizeof(buff), "%u\n", value);
+
+	return simple_read_from_buffer(buf, len, offset, buff, desc);
+}
+
+static ssize_t pwr_debugfs_current_read(struct file *file, char __user *buf,
+					size_t len, loff_t *offset)
+{
+	struct svc_debugfs_pwrmon_rail *pwrmon_rails = file->f_inode->i_private;
+	struct gb_svc *svc = pwrmon_rails->svc;
+	int ret, desc;
+	u32 value;
+	char buff[16];
+
+	ret = gb_svc_pwrmon_sample_get(svc, pwrmon_rails->id,
+				       GB_SVC_PWRMON_TYPE_CURR, &value);
+	if (ret) {
+		dev_err(&svc->dev,
+			"failed to get current sample %u: %d\n",
+			pwrmon_rails->id, ret);
+		return ret;
+	}
+
+	desc = scnprintf(buff, sizeof(buff), "%u\n", value);
+
+	return simple_read_from_buffer(buf, len, offset, buff, desc);
+}
+
+static ssize_t pwr_debugfs_power_read(struct file *file, char __user *buf,
+				      size_t len, loff_t *offset)
+{
+	struct svc_debugfs_pwrmon_rail *pwrmon_rails = file->f_inode->i_private;
+	struct gb_svc *svc = pwrmon_rails->svc;
+	int ret, desc;
+	u32 value;
+	char buff[16];
+
+	ret = gb_svc_pwrmon_sample_get(svc, pwrmon_rails->id,
+				       GB_SVC_PWRMON_TYPE_PWR, &value);
+	if (ret) {
+		dev_err(&svc->dev, "failed to get power sample %u: %d\n",
+			pwrmon_rails->id, ret);
+		return ret;
+	}
+
+	desc = scnprintf(buff, sizeof(buff), "%u\n", value);
+
+	return simple_read_from_buffer(buf, len, offset, buff, desc);
+}
+
+static const struct file_operations pwrmon_debugfs_voltage_fops = {
+	.read		= pwr_debugfs_voltage_read,
+};
+
+static const struct file_operations pwrmon_debugfs_current_fops = {
+	.read		= pwr_debugfs_current_read,
+};
+
+static const struct file_operations pwrmon_debugfs_power_fops = {
+	.read		= pwr_debugfs_power_read,
+};
+
+static void gb_svc_pwrmon_debugfs_init(struct gb_svc *svc)
+{
+	int i;
+	size_t bufsize;
+	struct dentry *dent;
+	struct gb_svc_pwrmon_rail_names_get_response *rail_names;
+	u8 rail_count;
+
+	dent = debugfs_create_dir("pwrmon", svc->debugfs_dentry);
+	if (IS_ERR_OR_NULL(dent))
+		return;
+
+	if (gb_svc_pwrmon_rail_count_get(svc, &rail_count))
+		goto err_pwrmon_debugfs;
+
+	if (!rail_count || rail_count > GB_SVC_PWRMON_MAX_RAIL_COUNT)
+		goto err_pwrmon_debugfs;
+
+	bufsize = sizeof(*rail_names) +
+		GB_SVC_PWRMON_RAIL_NAME_BUFSIZE * rail_count;
+
+	rail_names = kzalloc(bufsize, GFP_KERNEL);
+	if (!rail_names)
+		goto err_pwrmon_debugfs;
+
+	svc->pwrmon_rails = kcalloc(rail_count, sizeof(*svc->pwrmon_rails),
+				    GFP_KERNEL);
+	if (!svc->pwrmon_rails)
+		goto err_pwrmon_debugfs_free;
+
+	if (gb_svc_pwrmon_rail_names_get(svc, rail_names, bufsize))
+		goto err_pwrmon_debugfs_free;
+
+	for (i = 0; i < rail_count; i++) {
+		struct dentry *dir;
+		struct svc_debugfs_pwrmon_rail *rail = &svc->pwrmon_rails[i];
+		char fname[GB_SVC_PWRMON_RAIL_NAME_BUFSIZE];
+
+		snprintf(fname, sizeof(fname), "%s",
+			 (char *)&rail_names->name[i]);
+
+		rail->id = i;
+		rail->svc = svc;
+
+		dir = debugfs_create_dir(fname, dent);
+		debugfs_create_file("voltage_now", S_IRUGO, dir, rail,
+				    &pwrmon_debugfs_voltage_fops);
+		debugfs_create_file("current_now", S_IRUGO, dir, rail,
+				    &pwrmon_debugfs_current_fops);
+		debugfs_create_file("power_now", S_IRUGO, dir, rail,
+				    &pwrmon_debugfs_power_fops);
+	}
+
+	kfree(rail_names);
+	return;
+
+err_pwrmon_debugfs_free:
+	kfree(rail_names);
+	kfree(svc->pwrmon_rails);
+	svc->pwrmon_rails = NULL;
+
+err_pwrmon_debugfs:
+	debugfs_remove(dent);
+}
+
+static void gb_svc_debugfs_init(struct gb_svc *svc)
+{
+	svc->debugfs_dentry = debugfs_create_dir(dev_name(&svc->dev),
+						 gb_debugfs_get());
+	gb_svc_pwrmon_debugfs_init(svc);
+}
+
+static void gb_svc_debugfs_exit(struct gb_svc *svc)
+{
+	debugfs_remove_recursive(svc->debugfs_dentry);
+	kfree(svc->pwrmon_rails);
+	svc->pwrmon_rails = NULL;
+}
+
+static int gb_svc_hello(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_svc *svc = gb_connection_get_data(connection);
+	struct gb_svc_hello_request *hello_request;
+	int ret;
+
+	if (op->request->payload_size < sizeof(*hello_request)) {
+		dev_warn(&svc->dev, "short hello request (%zu < %zu)\n",
+				op->request->payload_size,
+				sizeof(*hello_request));
+		return -EINVAL;
+	}
+
+	hello_request = op->request->payload;
+	svc->endo_id = le16_to_cpu(hello_request->endo_id);
+	svc->ap_intf_id = hello_request->interface_id;
+
+	ret = device_add(&svc->dev);
+	if (ret) {
+		dev_err(&svc->dev, "failed to register svc device: %d\n", ret);
+		return ret;
+	}
+
+	ret = gb_svc_watchdog_create(svc);
+	if (ret) {
+		dev_err(&svc->dev, "failed to create watchdog: %d\n", ret);
+		goto err_unregister_device;
+	}
+
+	gb_svc_debugfs_init(svc);
+
+	ret = gb_timesync_svc_add(svc);
+	if (ret) {
+		dev_err(&svc->dev, "failed to add SVC to timesync: %d\n", ret);
+		gb_svc_debugfs_exit(svc);
+		goto err_unregister_device;
+	}
+
+	return gb_svc_queue_deferred_request(op);
+
+err_unregister_device:
+	gb_svc_watchdog_destroy(svc);
+	device_del(&svc->dev);
+	return ret;
+}
+
+static struct gb_interface *gb_svc_interface_lookup(struct gb_svc *svc,
+							u8 intf_id)
+{
+	struct gb_host_device *hd = svc->hd;
+	struct gb_module *module;
+	size_t num_interfaces;
+	u8 module_id;
+
+	list_for_each_entry(module, &hd->modules, hd_node) {
+		module_id = module->module_id;
+		num_interfaces = module->num_interfaces;
+
+		if (intf_id >= module_id &&
+				intf_id < module_id + num_interfaces) {
+			return module->interfaces[intf_id - module_id];
+		}
+	}
+
+	return NULL;
+}
+
+static struct gb_module *gb_svc_module_lookup(struct gb_svc *svc, u8 module_id)
+{
+	struct gb_host_device *hd = svc->hd;
+	struct gb_module *module;
+
+	list_for_each_entry(module, &hd->modules, hd_node) {
+		if (module->module_id == module_id)
+			return module;
+	}
+
+	return NULL;
+}
+
+static void gb_svc_process_hello_deferred(struct gb_operation *operation)
+{
+	struct gb_connection *connection = operation->connection;
+	struct gb_svc *svc = gb_connection_get_data(connection);
+	int ret;
+
+	/*
+	 * XXX This is a hack/work-around to reconfigure the APBridgeA-Switch
+	 * link to PWM G2, 1 Lane, Slow Auto, so that it has sufficient
+	 * bandwidth for 3 audio streams plus boot-over-UniPro of a hot-plugged
+	 * module.
+	 *
+	 * The code should be removed once SW-2217, Heuristic for UniPro
+	 * Power Mode Changes is resolved.
+	 */
+	ret = gb_svc_intf_set_power_mode(svc, svc->ap_intf_id,
+					GB_SVC_UNIPRO_HS_SERIES_A,
+					GB_SVC_UNIPRO_SLOW_AUTO_MODE,
+					2, 1,
+					GB_SVC_SMALL_AMPLITUDE, GB_SVC_NO_DE_EMPHASIS,
+					GB_SVC_UNIPRO_SLOW_AUTO_MODE,
+					2, 1,
+					0, 0,
+					NULL, NULL);
+
+	if (ret)
+		dev_warn(&svc->dev,
+			"power mode change failed on AP to switch link: %d\n",
+			ret);
+}
+
+static void gb_svc_process_module_inserted(struct gb_operation *operation)
+{
+	struct gb_svc_module_inserted_request *request;
+	struct gb_connection *connection = operation->connection;
+	struct gb_svc *svc = gb_connection_get_data(connection);
+	struct gb_host_device *hd = svc->hd;
+	struct gb_module *module;
+	size_t num_interfaces;
+	u8 module_id;
+	u16 flags;
+	int ret;
+
+	/* The request message size has already been verified. */
+	request = operation->request->payload;
+	module_id = request->primary_intf_id;
+	num_interfaces = request->intf_count;
+	flags = le16_to_cpu(request->flags);
+
+	dev_dbg(&svc->dev, "%s - id = %u, num_interfaces = %zu, flags = 0x%04x\n",
+			__func__, module_id, num_interfaces, flags);
+
+	if (flags & GB_SVC_MODULE_INSERTED_FLAG_NO_PRIMARY) {
+		dev_warn(&svc->dev, "no primary interface detected on module %u\n",
+				module_id);
+	}
+
+	module = gb_svc_module_lookup(svc, module_id);
+	if (module) {
+		dev_warn(&svc->dev, "unexpected module-inserted event %u\n",
+				module_id);
+		return;
+	}
+
+	module = gb_module_create(hd, module_id, num_interfaces);
+	if (!module) {
+		dev_err(&svc->dev, "failed to create module\n");
+		return;
+	}
+
+	ret = gb_module_add(module);
+	if (ret) {
+		gb_module_put(module);
+		return;
+	}
+
+	list_add(&module->hd_node, &hd->modules);
+}
+
+static void gb_svc_process_module_removed(struct gb_operation *operation)
+{
+	struct gb_svc_module_removed_request *request;
+	struct gb_connection *connection = operation->connection;
+	struct gb_svc *svc = gb_connection_get_data(connection);
+	struct gb_module *module;
+	u8 module_id;
+
+	/* The request message size has already been verified. */
+	request = operation->request->payload;
+	module_id = request->primary_intf_id;
+
+	dev_dbg(&svc->dev, "%s - id = %u\n", __func__, module_id);
+
+	module = gb_svc_module_lookup(svc, module_id);
+	if (!module) {
+		dev_warn(&svc->dev, "unexpected module-removed event %u\n",
+				module_id);
+		return;
+	}
+
+	module->disconnected = true;
+
+	gb_module_del(module);
+	list_del(&module->hd_node);
+	gb_module_put(module);
+}
+
+static void gb_svc_process_intf_oops(struct gb_operation *operation)
+{
+	struct gb_svc_intf_oops_request *request;
+	struct gb_connection *connection = operation->connection;
+	struct gb_svc *svc = gb_connection_get_data(connection);
+	struct gb_interface *intf;
+	u8 intf_id;
+	u8 reason;
+
+	/* The request message size has already been verified. */
+	request = operation->request->payload;
+	intf_id = request->intf_id;
+	reason = request->reason;
+
+	intf = gb_svc_interface_lookup(svc, intf_id);
+	if (!intf) {
+		dev_warn(&svc->dev, "unexpected interface-oops event %u\n",
+			 intf_id);
+		return;
+	}
+
+	dev_info(&svc->dev, "Deactivating interface %u, interface oops reason = %u\n",
+		 intf_id, reason);
+
+	mutex_lock(&intf->mutex);
+	intf->disconnected = true;
+	gb_interface_disable(intf);
+	gb_interface_deactivate(intf);
+	mutex_unlock(&intf->mutex);
+}
+
+static void gb_svc_process_intf_mailbox_event(struct gb_operation *operation)
+{
+	struct gb_svc_intf_mailbox_event_request *request;
+	struct gb_connection *connection = operation->connection;
+	struct gb_svc *svc = gb_connection_get_data(connection);
+	struct gb_interface *intf;
+	u8 intf_id;
+	u16 result_code;
+	u32 mailbox;
+
+	/* The request message size has already been verified. */
+	request = operation->request->payload;
+	intf_id = request->intf_id;
+	result_code = le16_to_cpu(request->result_code);
+	mailbox = le32_to_cpu(request->mailbox);
+
+	dev_dbg(&svc->dev, "%s - id = %u, result = 0x%04x, mailbox = 0x%08x\n",
+			__func__, intf_id, result_code, mailbox);
+
+	intf = gb_svc_interface_lookup(svc, intf_id);
+	if (!intf) {
+		dev_warn(&svc->dev, "unexpected mailbox event %u\n", intf_id);
+		return;
+	}
+
+	gb_interface_mailbox_event(intf, result_code, mailbox);
+}
+
+static void gb_svc_process_deferred_request(struct work_struct *work)
+{
+	struct gb_svc_deferred_request *dr;
+	struct gb_operation *operation;
+	struct gb_svc *svc;
+	u8 type;
+
+	dr = container_of(work, struct gb_svc_deferred_request, work);
+	operation = dr->operation;
+	svc = gb_connection_get_data(operation->connection);
+	type = operation->request->header->type;
+
+	switch (type) {
+	case GB_SVC_TYPE_SVC_HELLO:
+		gb_svc_process_hello_deferred(operation);
+		break;
+	case GB_SVC_TYPE_MODULE_INSERTED:
+		gb_svc_process_module_inserted(operation);
+		break;
+	case GB_SVC_TYPE_MODULE_REMOVED:
+		gb_svc_process_module_removed(operation);
+		break;
+	case GB_SVC_TYPE_INTF_MAILBOX_EVENT:
+		gb_svc_process_intf_mailbox_event(operation);
+		break;
+	case GB_SVC_TYPE_INTF_OOPS:
+		gb_svc_process_intf_oops(operation);
+		break;
+	default:
+		dev_err(&svc->dev, "bad deferred request type: 0x%02x\n", type);
+	}
+
+	gb_operation_put(operation);
+	kfree(dr);
+}
+
+static int gb_svc_queue_deferred_request(struct gb_operation *operation)
+{
+	struct gb_svc *svc = gb_connection_get_data(operation->connection);
+	struct gb_svc_deferred_request *dr;
+
+	dr = kmalloc(sizeof(*dr), GFP_KERNEL);
+	if (!dr)
+		return -ENOMEM;
+
+	gb_operation_get(operation);
+
+	dr->operation = operation;
+	INIT_WORK(&dr->work, gb_svc_process_deferred_request);
+
+	queue_work(svc->wq, &dr->work);
+
+	return 0;
+}
+
+static int gb_svc_intf_reset_recv(struct gb_operation *op)
+{
+	struct gb_svc *svc = gb_connection_get_data(op->connection);
+	struct gb_message *request = op->request;
+	struct gb_svc_intf_reset_request *reset;
+	u8 intf_id;
+
+	if (request->payload_size < sizeof(*reset)) {
+		dev_warn(&svc->dev, "short reset request received (%zu < %zu)\n",
+				request->payload_size, sizeof(*reset));
+		return -EINVAL;
+	}
+	reset = request->payload;
+
+	intf_id = reset->intf_id;
+
+	/* FIXME Reset the interface here */
+
+	return 0;
+}
+
+static int gb_svc_module_inserted_recv(struct gb_operation *op)
+{
+	struct gb_svc *svc = gb_connection_get_data(op->connection);
+	struct gb_svc_module_inserted_request *request;
+
+	if (op->request->payload_size < sizeof(*request)) {
+		dev_warn(&svc->dev, "short module-inserted request received (%zu < %zu)\n",
+				op->request->payload_size, sizeof(*request));
+		return -EINVAL;
+	}
+
+	request = op->request->payload;
+
+	dev_dbg(&svc->dev, "%s - id = %u\n", __func__,
+			request->primary_intf_id);
+
+	return gb_svc_queue_deferred_request(op);
+}
+
+static int gb_svc_module_removed_recv(struct gb_operation *op)
+{
+	struct gb_svc *svc = gb_connection_get_data(op->connection);
+	struct gb_svc_module_removed_request *request;
+
+	if (op->request->payload_size < sizeof(*request)) {
+		dev_warn(&svc->dev, "short module-removed request received (%zu < %zu)\n",
+				op->request->payload_size, sizeof(*request));
+		return -EINVAL;
+	}
+
+	request = op->request->payload;
+
+	dev_dbg(&svc->dev, "%s - id = %u\n", __func__,
+			request->primary_intf_id);
+
+	return gb_svc_queue_deferred_request(op);
+}
+
+static int gb_svc_intf_oops_recv(struct gb_operation *op)
+{
+	struct gb_svc *svc = gb_connection_get_data(op->connection);
+	struct gb_svc_intf_oops_request *request;
+
+	if (op->request->payload_size < sizeof(*request)) {
+		dev_warn(&svc->dev, "short intf-oops request received (%zu < %zu)\n",
+			 op->request->payload_size, sizeof(*request));
+		return -EINVAL;
+	}
+
+	return gb_svc_queue_deferred_request(op);
+}
+
+static int gb_svc_intf_mailbox_event_recv(struct gb_operation *op)
+{
+	struct gb_svc *svc = gb_connection_get_data(op->connection);
+	struct gb_svc_intf_mailbox_event_request *request;
+
+	if (op->request->payload_size < sizeof(*request)) {
+		dev_warn(&svc->dev, "short mailbox request received (%zu < %zu)\n",
+				op->request->payload_size, sizeof(*request));
+		return -EINVAL;
+	}
+
+	request = op->request->payload;
+
+	dev_dbg(&svc->dev, "%s - id = %u\n", __func__, request->intf_id);
+
+	return gb_svc_queue_deferred_request(op);
+}
+
+static int gb_svc_request_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_svc *svc = gb_connection_get_data(connection);
+	u8 type = op->type;
+	int ret = 0;
+
+	/*
+	 * SVC requests need to follow a specific order (at least initially) and
+	 * below code takes care of enforcing that. The expected order is:
+	 * - PROTOCOL_VERSION
+	 * - SVC_HELLO
+	 * - Any other request, but the earlier two.
+	 *
+	 * Incoming requests are guaranteed to be serialized and so we don't
+	 * need to protect 'state' for any races.
+	 */
+	switch (type) {
+	case GB_SVC_TYPE_PROTOCOL_VERSION:
+		if (svc->state != GB_SVC_STATE_RESET)
+			ret = -EINVAL;
+		break;
+	case GB_SVC_TYPE_SVC_HELLO:
+		if (svc->state != GB_SVC_STATE_PROTOCOL_VERSION)
+			ret = -EINVAL;
+		break;
+	default:
+		if (svc->state != GB_SVC_STATE_SVC_HELLO)
+			ret = -EINVAL;
+		break;
+	}
+
+	if (ret) {
+		dev_warn(&svc->dev, "unexpected request 0x%02x received (state %u)\n",
+				type, svc->state);
+		return ret;
+	}
+
+	switch (type) {
+	case GB_SVC_TYPE_PROTOCOL_VERSION:
+		ret = gb_svc_version_request(op);
+		if (!ret)
+			svc->state = GB_SVC_STATE_PROTOCOL_VERSION;
+		return ret;
+	case GB_SVC_TYPE_SVC_HELLO:
+		ret = gb_svc_hello(op);
+		if (!ret)
+			svc->state = GB_SVC_STATE_SVC_HELLO;
+		return ret;
+	case GB_SVC_TYPE_INTF_RESET:
+		return gb_svc_intf_reset_recv(op);
+	case GB_SVC_TYPE_MODULE_INSERTED:
+		return gb_svc_module_inserted_recv(op);
+	case GB_SVC_TYPE_MODULE_REMOVED:
+		return gb_svc_module_removed_recv(op);
+	case GB_SVC_TYPE_INTF_MAILBOX_EVENT:
+		return gb_svc_intf_mailbox_event_recv(op);
+	case GB_SVC_TYPE_INTF_OOPS:
+		return gb_svc_intf_oops_recv(op);
+	default:
+		dev_warn(&svc->dev, "unsupported request 0x%02x\n", type);
+		return -EINVAL;
+	}
+}
+
+static void gb_svc_release(struct device *dev)
+{
+	struct gb_svc *svc = to_gb_svc(dev);
+
+	if (svc->connection)
+		gb_connection_destroy(svc->connection);
+	ida_destroy(&svc->device_id_map);
+	destroy_workqueue(svc->wq);
+	kfree(svc);
+}
+
+struct device_type greybus_svc_type = {
+	.name		= "greybus_svc",
+	.release	= gb_svc_release,
+};
+
+struct gb_svc *gb_svc_create(struct gb_host_device *hd)
+{
+	struct gb_svc *svc;
+
+	svc = kzalloc(sizeof(*svc), GFP_KERNEL);
+	if (!svc)
+		return NULL;
+
+	svc->wq = alloc_workqueue("%s:svc", WQ_UNBOUND, 1, dev_name(&hd->dev));
+	if (!svc->wq) {
+		kfree(svc);
+		return NULL;
+	}
+
+	svc->dev.parent = &hd->dev;
+	svc->dev.bus = &greybus_bus_type;
+	svc->dev.type = &greybus_svc_type;
+	svc->dev.groups = svc_groups;
+	svc->dev.dma_mask = svc->dev.parent->dma_mask;
+	device_initialize(&svc->dev);
+
+	dev_set_name(&svc->dev, "%d-svc", hd->bus_id);
+
+	ida_init(&svc->device_id_map);
+	svc->state = GB_SVC_STATE_RESET;
+	svc->hd = hd;
+
+	svc->connection = gb_connection_create_static(hd, GB_SVC_CPORT_ID,
+						gb_svc_request_handler);
+	if (IS_ERR(svc->connection)) {
+		dev_err(&svc->dev, "failed to create connection: %ld\n",
+				PTR_ERR(svc->connection));
+		goto err_put_device;
+	}
+
+	gb_connection_set_data(svc->connection, svc);
+
+	return svc;
+
+err_put_device:
+	put_device(&svc->dev);
+	return NULL;
+}
+
+int gb_svc_add(struct gb_svc *svc)
+{
+	int ret;
+
+	/*
+	 * The SVC protocol is currently driven by the SVC, so the SVC device
+	 * is added from the connection request handler when enough
+	 * information has been received.
+	 */
+	ret = gb_connection_enable(svc->connection);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void gb_svc_remove_modules(struct gb_svc *svc)
+{
+	struct gb_host_device *hd = svc->hd;
+	struct gb_module *module, *tmp;
+
+	list_for_each_entry_safe(module, tmp, &hd->modules, hd_node) {
+		gb_module_del(module);
+		list_del(&module->hd_node);
+		gb_module_put(module);
+	}
+}
+
+void gb_svc_del(struct gb_svc *svc)
+{
+	gb_connection_disable_rx(svc->connection);
+
+	/*
+	 * The SVC device may have been registered from the request handler.
+	 */
+	if (device_is_registered(&svc->dev)) {
+		gb_timesync_svc_remove(svc);
+		gb_svc_debugfs_exit(svc);
+		gb_svc_watchdog_destroy(svc);
+		device_del(&svc->dev);
+	}
+
+	flush_workqueue(svc->wq);
+
+	gb_svc_remove_modules(svc);
+
+	gb_connection_disable(svc->connection);
+}
+
+void gb_svc_put(struct gb_svc *svc)
+{
+	put_device(&svc->dev);
+}
diff --git a/drivers/staging/greybus/svc.h b/drivers/staging/greybus/svc.h
new file mode 100644
index 000000000000..d1d7ef967385
--- /dev/null
+++ b/drivers/staging/greybus/svc.h
@@ -0,0 +1,109 @@
+/*
+ * Greybus SVC code
+ *
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __SVC_H
+#define __SVC_H
+
+#define GB_SVC_CPORT_FLAG_E2EFC		BIT(0)
+#define GB_SVC_CPORT_FLAG_CSD_N		BIT(1)
+#define GB_SVC_CPORT_FLAG_CSV_N		BIT(2)
+
+enum gb_svc_state {
+	GB_SVC_STATE_RESET,
+	GB_SVC_STATE_PROTOCOL_VERSION,
+	GB_SVC_STATE_SVC_HELLO,
+};
+
+enum gb_svc_watchdog_bite {
+	GB_SVC_WATCHDOG_BITE_RESET_UNIPRO = 0,
+	GB_SVC_WATCHDOG_BITE_PANIC_KERNEL,
+};
+
+struct gb_svc_watchdog;
+
+struct svc_debugfs_pwrmon_rail {
+	u8 id;
+	struct gb_svc *svc;
+};
+
+struct gb_svc {
+	struct device		dev;
+
+	struct gb_host_device	*hd;
+	struct gb_connection	*connection;
+	enum gb_svc_state	state;
+	struct ida		device_id_map;
+	struct workqueue_struct	*wq;
+
+	u16 endo_id;
+	u8 ap_intf_id;
+
+	u8 protocol_major;
+	u8 protocol_minor;
+
+	struct gb_svc_watchdog	*watchdog;
+	enum gb_svc_watchdog_bite action;
+
+	struct dentry *debugfs_dentry;
+	struct svc_debugfs_pwrmon_rail *pwrmon_rails;
+};
+#define to_gb_svc(d) container_of(d, struct gb_svc, dev)
+
+struct gb_svc *gb_svc_create(struct gb_host_device *hd);
+int gb_svc_add(struct gb_svc *svc);
+void gb_svc_del(struct gb_svc *svc);
+void gb_svc_put(struct gb_svc *svc);
+
+int gb_svc_pwrmon_intf_sample_get(struct gb_svc *svc, u8 intf_id,
+				  u8 measurement_type, u32 *value);
+int gb_svc_intf_device_id(struct gb_svc *svc, u8 intf_id, u8 device_id);
+int gb_svc_route_create(struct gb_svc *svc, u8 intf1_id, u8 dev1_id,
+			       u8 intf2_id, u8 dev2_id);
+void gb_svc_route_destroy(struct gb_svc *svc, u8 intf1_id, u8 intf2_id);
+int gb_svc_connection_create(struct gb_svc *svc, u8 intf1_id, u16 cport1_id,
+			     u8 intf2_id, u16 cport2_id, u8 cport_flags);
+void gb_svc_connection_destroy(struct gb_svc *svc, u8 intf1_id, u16 cport1_id,
+			       u8 intf2_id, u16 cport2_id);
+int gb_svc_intf_eject(struct gb_svc *svc, u8 intf_id);
+int gb_svc_intf_vsys_set(struct gb_svc *svc, u8 intf_id, bool enable);
+int gb_svc_intf_refclk_set(struct gb_svc *svc, u8 intf_id, bool enable);
+int gb_svc_intf_unipro_set(struct gb_svc *svc, u8 intf_id, bool enable);
+int gb_svc_intf_activate(struct gb_svc *svc, u8 intf_id, u8 *intf_type);
+int gb_svc_intf_resume(struct gb_svc *svc, u8 intf_id);
+
+int gb_svc_dme_peer_get(struct gb_svc *svc, u8 intf_id, u16 attr, u16 selector,
+			u32 *value);
+int gb_svc_dme_peer_set(struct gb_svc *svc, u8 intf_id, u16 attr, u16 selector,
+			u32 value);
+int gb_svc_intf_set_power_mode(struct gb_svc *svc, u8 intf_id, u8 hs_series,
+			       u8 tx_mode, u8 tx_gear, u8 tx_nlanes,
+			       u8 tx_amplitude, u8 tx_hs_equalizer,
+			       u8 rx_mode, u8 rx_gear, u8 rx_nlanes,
+			       u8 flags, u32 quirks,
+			       struct gb_svc_l2_timer_cfg *local,
+			       struct gb_svc_l2_timer_cfg *remote);
+int gb_svc_intf_set_power_mode_hibernate(struct gb_svc *svc, u8 intf_id);
+int gb_svc_ping(struct gb_svc *svc);
+int gb_svc_watchdog_create(struct gb_svc *svc);
+void gb_svc_watchdog_destroy(struct gb_svc *svc);
+bool gb_svc_watchdog_enabled(struct gb_svc *svc);
+int gb_svc_watchdog_enable(struct gb_svc *svc);
+int gb_svc_watchdog_disable(struct gb_svc *svc);
+int gb_svc_timesync_enable(struct gb_svc *svc, u8 count, u64 frame_time,
+			   u32 strobe_delay, u32 refclk);
+int gb_svc_timesync_disable(struct gb_svc *svc);
+int gb_svc_timesync_authoritative(struct gb_svc *svc, u64 *frame_time);
+int gb_svc_timesync_ping(struct gb_svc *svc, u64 *frame_time);
+int gb_svc_timesync_wake_pins_acquire(struct gb_svc *svc, u32 strobe_mask);
+int gb_svc_timesync_wake_pins_release(struct gb_svc *svc);
+
+int gb_svc_protocol_init(void);
+void gb_svc_protocol_exit(void);
+
+#endif /* __SVC_H */
diff --git a/drivers/staging/greybus/svc_watchdog.c b/drivers/staging/greybus/svc_watchdog.c
new file mode 100644
index 000000000000..3729460fb954
--- /dev/null
+++ b/drivers/staging/greybus/svc_watchdog.c
@@ -0,0 +1,198 @@
+/*
+ * SVC Greybus "watchdog" driver.
+ *
+ * Copyright 2016 Google Inc.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/delay.h>
+#include <linux/suspend.h>
+#include <linux/workqueue.h>
+#include "greybus.h"
+
+#define SVC_WATCHDOG_PERIOD	(2*HZ)
+
+struct gb_svc_watchdog {
+	struct delayed_work	work;
+	struct gb_svc		*svc;
+	bool			enabled;
+	struct notifier_block pm_notifier;
+};
+
+static struct delayed_work reset_work;
+
+static int svc_watchdog_pm_notifier(struct notifier_block *notifier,
+				    unsigned long pm_event, void *unused)
+{
+	struct gb_svc_watchdog *watchdog =
+		container_of(notifier, struct gb_svc_watchdog, pm_notifier);
+
+	switch (pm_event) {
+	case PM_SUSPEND_PREPARE:
+		gb_svc_watchdog_disable(watchdog->svc);
+		break;
+	case PM_POST_SUSPEND:
+		gb_svc_watchdog_enable(watchdog->svc);
+		break;
+	default:
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static void greybus_reset(struct work_struct *work)
+{
+	static char start_path[256] = "/system/bin/start";
+	static char *envp[] = {
+		"HOME=/",
+		"PATH=/sbin:/vendor/bin:/system/sbin:/system/bin:/system/xbin",
+		NULL,
+	};
+	static char *argv[] = {
+		start_path,
+		"unipro_reset",
+		NULL,
+	};
+
+	printk(KERN_ERR "svc_watchdog: calling \"%s %s\" to reset greybus network!\n",
+	       argv[0], argv[1]);
+	call_usermodehelper(start_path, argv, envp, UMH_WAIT_EXEC);
+}
+
+static void do_work(struct work_struct *work)
+{
+	struct gb_svc_watchdog *watchdog;
+	struct gb_svc *svc;
+	int retval;
+
+	watchdog = container_of(work, struct gb_svc_watchdog, work.work);
+	svc = watchdog->svc;
+
+	dev_dbg(&svc->dev, "%s: ping.\n", __func__);
+	retval = gb_svc_ping(svc);
+	if (retval) {
+		/*
+		 * Something went really wrong, let's warn userspace and then
+		 * pull the plug and reset the whole greybus network.
+		 * We need to do this outside of this workqueue as we will be
+		 * tearing down the svc device itself.  So queue up
+		 * yet-another-callback to do that.
+		 */
+		dev_err(&svc->dev,
+			"SVC ping has returned %d, something is wrong!!!\n",
+			retval);
+
+		if (svc->action == GB_SVC_WATCHDOG_BITE_PANIC_KERNEL) {
+			panic("SVC is not responding\n");
+		} else if (svc->action == GB_SVC_WATCHDOG_BITE_RESET_UNIPRO) {
+			dev_err(&svc->dev, "Resetting the greybus network, watch out!!!\n");
+
+			INIT_DELAYED_WORK(&reset_work, greybus_reset);
+			schedule_delayed_work(&reset_work, HZ / 2);
+
+			/*
+			 * Disable ourselves, we don't want to trip again unless
+			 * userspace wants us to.
+			 */
+			watchdog->enabled = false;
+		}
+	}
+
+	/* resubmit our work to happen again, if we are still "alive" */
+	if (watchdog->enabled)
+		schedule_delayed_work(&watchdog->work, SVC_WATCHDOG_PERIOD);
+}
+
+int gb_svc_watchdog_create(struct gb_svc *svc)
+{
+	struct gb_svc_watchdog *watchdog;
+	int retval;
+
+	if (svc->watchdog)
+		return 0;
+
+	watchdog = kmalloc(sizeof(*watchdog), GFP_KERNEL);
+	if (!watchdog)
+		return -ENOMEM;
+
+	watchdog->enabled = false;
+	watchdog->svc = svc;
+	INIT_DELAYED_WORK(&watchdog->work, do_work);
+	svc->watchdog = watchdog;
+
+	watchdog->pm_notifier.notifier_call = svc_watchdog_pm_notifier;
+	retval = register_pm_notifier(&watchdog->pm_notifier);
+	if (retval) {
+		dev_err(&svc->dev, "error registering pm notifier(%d)\n",
+			retval);
+		goto svc_watchdog_create_err;
+	}
+
+	retval = gb_svc_watchdog_enable(svc);
+	if (retval) {
+		dev_err(&svc->dev, "error enabling watchdog (%d)\n", retval);
+		unregister_pm_notifier(&watchdog->pm_notifier);
+		goto svc_watchdog_create_err;
+	}
+	return retval;
+
+svc_watchdog_create_err:
+	svc->watchdog = NULL;
+	kfree(watchdog);
+
+	return retval;
+}
+
+void gb_svc_watchdog_destroy(struct gb_svc *svc)
+{
+	struct gb_svc_watchdog *watchdog = svc->watchdog;
+
+	if (!watchdog)
+		return;
+
+	unregister_pm_notifier(&watchdog->pm_notifier);
+	gb_svc_watchdog_disable(svc);
+	svc->watchdog = NULL;
+	kfree(watchdog);
+}
+
+bool gb_svc_watchdog_enabled(struct gb_svc *svc)
+{
+	if (!svc || !svc->watchdog)
+		return false;
+	return svc->watchdog->enabled;
+}
+
+int gb_svc_watchdog_enable(struct gb_svc *svc)
+{
+	struct gb_svc_watchdog *watchdog;
+
+	if (!svc->watchdog)
+		return -ENODEV;
+
+	watchdog = svc->watchdog;
+	if (watchdog->enabled)
+		return 0;
+
+	watchdog->enabled = true;
+	schedule_delayed_work(&watchdog->work, SVC_WATCHDOG_PERIOD);
+	return 0;
+}
+
+int gb_svc_watchdog_disable(struct gb_svc *svc)
+{
+	struct gb_svc_watchdog *watchdog;
+
+	if (!svc->watchdog)
+		return -ENODEV;
+
+	watchdog = svc->watchdog;
+	if (!watchdog->enabled)
+		return 0;
+
+	watchdog->enabled = false;
+	cancel_delayed_work_sync(&watchdog->work);
+	return 0;
+}
diff --git a/drivers/staging/greybus/timesync.c b/drivers/staging/greybus/timesync.c
new file mode 100644
index 000000000000..2e68af7dea6d
--- /dev/null
+++ b/drivers/staging/greybus/timesync.c
@@ -0,0 +1,1357 @@
+/*
+ * TimeSync API driver.
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+#include <linux/debugfs.h>
+#include <linux/hrtimer.h>
+#include "greybus.h"
+#include "timesync.h"
+#include "greybus_trace.h"
+
+/*
+ * Minimum inter-strobe value of one millisecond is chosen because it
+ * just-about fits the common definition of a jiffy.
+ *
+ * Maximum value OTOH is constrained by the number of bits the SVC can fit
+ * into a 16 bit up-counter. The SVC configures the timer in microseconds
+ * so the maximum allowable value is 65535 microseconds. We clip that value
+ * to 10000 microseconds for the sake of using nice round base 10 numbers
+ * and since right-now there's no imaginable use-case requiring anything
+ * other than a one millisecond inter-strobe time, let alone something
+ * higher than ten milliseconds.
+ */
+#define GB_TIMESYNC_STROBE_DELAY_US		1000
+#define GB_TIMESYNC_DEFAULT_OFFSET_US		1000
+
+/* Work queue timers long, short and SVC strobe timeout */
+#define GB_TIMESYNC_DELAYED_WORK_LONG		msecs_to_jiffies(10)
+#define GB_TIMESYNC_DELAYED_WORK_SHORT		msecs_to_jiffies(1)
+#define GB_TIMESYNC_MAX_WAIT_SVC		msecs_to_jiffies(5000)
+#define GB_TIMESYNC_KTIME_UPDATE		msecs_to_jiffies(1000)
+#define GB_TIMESYNC_MAX_KTIME_CONVERSION	15
+
+/* Maximum number of times we'll retry a failed synchronous sync */
+#define GB_TIMESYNC_MAX_RETRIES			5
+
+/* Reported nanoseconds/femtoseconds per clock */
+static u64 gb_timesync_ns_per_clock;
+static u64 gb_timesync_fs_per_clock;
+
+/* Maximum difference we will accept converting FrameTime to ktime */
+static u32 gb_timesync_max_ktime_diff;
+
+/* Reported clock rate */
+static unsigned long gb_timesync_clock_rate;
+
+/* Workqueue */
+static void gb_timesync_worker(struct work_struct *work);
+
+/* List of SVCs with one FrameTime per SVC */
+static LIST_HEAD(gb_timesync_svc_list);
+
+/* Synchronize parallel contexts accessing a valid timesync_svc pointer */
+static DEFINE_MUTEX(gb_timesync_svc_list_mutex);
+
+/* Structure to convert from FrameTime to timespec/ktime */
+struct gb_timesync_frame_time_data {
+	u64 frame_time;
+	struct timespec ts;
+};
+
+struct gb_timesync_svc {
+	struct list_head list;
+	struct list_head interface_list;
+	struct gb_svc *svc;
+	struct gb_timesync_host_device *timesync_hd;
+
+	spinlock_t spinlock;	/* Per SVC spinlock to sync with ISR */
+	struct mutex mutex;	/* Per SVC mutex for regular synchronization */
+
+	struct dentry *frame_time_dentry;
+	struct dentry *frame_ktime_dentry;
+	struct workqueue_struct *work_queue;
+	wait_queue_head_t wait_queue;
+	struct delayed_work delayed_work;
+	struct timer_list ktime_timer;
+
+	/* The current local FrameTime */
+	u64 frame_time_offset;
+	struct gb_timesync_frame_time_data strobe_data[GB_TIMESYNC_MAX_STROBES];
+	struct gb_timesync_frame_time_data ktime_data;
+
+	/* The SVC FrameTime and relative AP FrameTime @ last TIMESYNC_PING */
+	u64 svc_ping_frame_time;
+	u64 ap_ping_frame_time;
+
+	/* Transitory settings */
+	u32 strobe_mask;
+	bool offset_down;
+	bool print_ping;
+	bool capture_ping;
+	int strobe;
+
+	/* Current state */
+	int state;
+};
+
+struct gb_timesync_host_device {
+	struct list_head list;
+	struct gb_host_device *hd;
+	u64 ping_frame_time;
+};
+
+struct gb_timesync_interface {
+	struct list_head list;
+	struct gb_interface *interface;
+	u64 ping_frame_time;
+};
+
+enum gb_timesync_state {
+	GB_TIMESYNC_STATE_INVALID		= 0,
+	GB_TIMESYNC_STATE_INACTIVE		= 1,
+	GB_TIMESYNC_STATE_INIT			= 2,
+	GB_TIMESYNC_STATE_WAIT_SVC		= 3,
+	GB_TIMESYNC_STATE_AUTHORITATIVE		= 4,
+	GB_TIMESYNC_STATE_PING			= 5,
+	GB_TIMESYNC_STATE_ACTIVE		= 6,
+};
+
+static void gb_timesync_ktime_timer_fn(unsigned long data);
+
+static u64 gb_timesync_adjust_count(struct gb_timesync_svc *timesync_svc,
+				    u64 counts)
+{
+	if (timesync_svc->offset_down)
+		return counts - timesync_svc->frame_time_offset;
+	else
+		return counts + timesync_svc->frame_time_offset;
+}
+
+/*
+ * This function provides the authoritative FrameTime to a calling function. It
+ * is designed to be lockless and should remain that way the caller is assumed
+ * to be state-aware.
+ */
+static u64 __gb_timesync_get_frame_time(struct gb_timesync_svc *timesync_svc)
+{
+	u64 clocks = gb_timesync_platform_get_counter();
+
+	return gb_timesync_adjust_count(timesync_svc, clocks);
+}
+
+static void gb_timesync_schedule_svc_timeout(struct gb_timesync_svc
+					     *timesync_svc)
+{
+	queue_delayed_work(timesync_svc->work_queue,
+			   &timesync_svc->delayed_work,
+			   GB_TIMESYNC_MAX_WAIT_SVC);
+}
+
+static void gb_timesync_set_state(struct gb_timesync_svc *timesync_svc,
+				  int state)
+{
+	switch (state) {
+	case GB_TIMESYNC_STATE_INVALID:
+		timesync_svc->state = state;
+		wake_up(&timesync_svc->wait_queue);
+		break;
+	case GB_TIMESYNC_STATE_INACTIVE:
+		timesync_svc->state = state;
+		wake_up(&timesync_svc->wait_queue);
+		break;
+	case GB_TIMESYNC_STATE_INIT:
+		if (timesync_svc->state != GB_TIMESYNC_STATE_INVALID) {
+			timesync_svc->strobe = 0;
+			timesync_svc->frame_time_offset = 0;
+			timesync_svc->state = state;
+			cancel_delayed_work(&timesync_svc->delayed_work);
+			queue_delayed_work(timesync_svc->work_queue,
+					   &timesync_svc->delayed_work,
+					   GB_TIMESYNC_DELAYED_WORK_LONG);
+		}
+		break;
+	case GB_TIMESYNC_STATE_WAIT_SVC:
+		if (timesync_svc->state == GB_TIMESYNC_STATE_INIT)
+			timesync_svc->state = state;
+		break;
+	case GB_TIMESYNC_STATE_AUTHORITATIVE:
+		if (timesync_svc->state == GB_TIMESYNC_STATE_WAIT_SVC) {
+			timesync_svc->state = state;
+			cancel_delayed_work(&timesync_svc->delayed_work);
+			queue_delayed_work(timesync_svc->work_queue,
+					   &timesync_svc->delayed_work, 0);
+		}
+		break;
+	case GB_TIMESYNC_STATE_PING:
+		if (timesync_svc->state == GB_TIMESYNC_STATE_ACTIVE) {
+			timesync_svc->state = state;
+			queue_delayed_work(timesync_svc->work_queue,
+					   &timesync_svc->delayed_work,
+					   GB_TIMESYNC_DELAYED_WORK_SHORT);
+		}
+		break;
+	case GB_TIMESYNC_STATE_ACTIVE:
+		if (timesync_svc->state == GB_TIMESYNC_STATE_AUTHORITATIVE ||
+		    timesync_svc->state == GB_TIMESYNC_STATE_PING) {
+			timesync_svc->state = state;
+			wake_up(&timesync_svc->wait_queue);
+		}
+		break;
+	}
+
+	if (WARN_ON(timesync_svc->state != state)) {
+		pr_err("Invalid state transition %d=>%d\n",
+		       timesync_svc->state, state);
+	}
+}
+
+static void gb_timesync_set_state_atomic(struct gb_timesync_svc *timesync_svc,
+					 int state)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&timesync_svc->spinlock, flags);
+	gb_timesync_set_state(timesync_svc, state);
+	spin_unlock_irqrestore(&timesync_svc->spinlock, flags);
+}
+
+static u64 gb_timesync_diff(u64 x, u64 y)
+{
+	if (x > y)
+		return x - y;
+	else
+		return y - x;
+}
+
+static void gb_timesync_adjust_to_svc(struct gb_timesync_svc *svc,
+				      u64 svc_frame_time, u64 ap_frame_time)
+{
+	if (svc_frame_time > ap_frame_time) {
+		svc->frame_time_offset = svc_frame_time - ap_frame_time;
+		svc->offset_down = false;
+	} else {
+		svc->frame_time_offset = ap_frame_time - svc_frame_time;
+		svc->offset_down = true;
+	}
+}
+
+/*
+ * Associate a FrameTime with a ktime timestamp represented as struct timespec
+ * Requires the calling context to hold timesync_svc->mutex
+ */
+static void gb_timesync_store_ktime(struct gb_timesync_svc *timesync_svc,
+				    struct timespec ts, u64 frame_time)
+{
+	timesync_svc->ktime_data.ts = ts;
+	timesync_svc->ktime_data.frame_time = frame_time;
+}
+
+/*
+ * Find the two pulses that best-match our expected inter-strobe gap and
+ * then calculate the difference between the SVC time at the second pulse
+ * to the local time at the second pulse.
+ */
+static void gb_timesync_collate_frame_time(struct gb_timesync_svc *timesync_svc,
+					   u64 *frame_time)
+{
+	int i = 0;
+	u64 delta, ap_frame_time;
+	u64 strobe_delay_ns = GB_TIMESYNC_STROBE_DELAY_US * NSEC_PER_USEC;
+	u64 least = 0;
+
+	for (i = 1; i < GB_TIMESYNC_MAX_STROBES; i++) {
+		delta = timesync_svc->strobe_data[i].frame_time -
+			timesync_svc->strobe_data[i - 1].frame_time;
+		delta *= gb_timesync_ns_per_clock;
+		delta = gb_timesync_diff(delta, strobe_delay_ns);
+
+		if (!least || delta < least) {
+			least = delta;
+			gb_timesync_adjust_to_svc(timesync_svc, frame_time[i],
+						  timesync_svc->strobe_data[i].frame_time);
+
+			ap_frame_time = timesync_svc->strobe_data[i].frame_time;
+			ap_frame_time = gb_timesync_adjust_count(timesync_svc,
+								 ap_frame_time);
+			gb_timesync_store_ktime(timesync_svc,
+						timesync_svc->strobe_data[i].ts,
+						ap_frame_time);
+
+			pr_debug("adjust %s local %llu svc %llu delta %llu\n",
+				 timesync_svc->offset_down ? "down" : "up",
+				 timesync_svc->strobe_data[i].frame_time,
+				 frame_time[i], delta);
+		}
+	}
+}
+
+static void gb_timesync_teardown(struct gb_timesync_svc *timesync_svc)
+{
+	struct gb_timesync_interface *timesync_interface;
+	struct gb_svc *svc = timesync_svc->svc;
+	struct gb_interface *interface;
+	struct gb_host_device *hd;
+	int ret;
+
+	list_for_each_entry(timesync_interface,
+			    &timesync_svc->interface_list, list) {
+		interface = timesync_interface->interface;
+		ret = gb_interface_timesync_disable(interface);
+		if (ret) {
+			dev_err(&interface->dev,
+				"interface timesync_disable %d\n", ret);
+		}
+	}
+
+	hd = timesync_svc->timesync_hd->hd;
+	ret = hd->driver->timesync_disable(hd);
+	if (ret < 0) {
+		dev_err(&hd->dev, "host timesync_disable %d\n",
+			ret);
+	}
+
+	gb_svc_timesync_wake_pins_release(svc);
+	gb_svc_timesync_disable(svc);
+	gb_timesync_platform_unlock_bus();
+
+	gb_timesync_set_state_atomic(timesync_svc, GB_TIMESYNC_STATE_INACTIVE);
+}
+
+static void gb_timesync_platform_lock_bus_fail(struct gb_timesync_svc
+						*timesync_svc, int ret)
+{
+	if (ret == -EAGAIN) {
+		gb_timesync_set_state(timesync_svc, timesync_svc->state);
+	} else {
+		pr_err("Failed to lock timesync bus %d\n", ret);
+		gb_timesync_set_state(timesync_svc, GB_TIMESYNC_STATE_INACTIVE);
+	}
+}
+
+static void gb_timesync_enable(struct gb_timesync_svc *timesync_svc)
+{
+	struct gb_svc *svc = timesync_svc->svc;
+	struct gb_host_device *hd;
+	struct gb_timesync_interface *timesync_interface;
+	struct gb_interface *interface;
+	u64 init_frame_time;
+	unsigned long clock_rate = gb_timesync_clock_rate;
+	int ret;
+
+	/*
+	 * Get access to the wake pins in the AP and SVC
+	 * Release these pins either in gb_timesync_teardown() or in
+	 * gb_timesync_authoritative()
+	 */
+	ret = gb_timesync_platform_lock_bus(timesync_svc);
+	if (ret < 0) {
+		gb_timesync_platform_lock_bus_fail(timesync_svc, ret);
+		return;
+	}
+	ret = gb_svc_timesync_wake_pins_acquire(svc, timesync_svc->strobe_mask);
+	if (ret) {
+		dev_err(&svc->dev,
+			"gb_svc_timesync_wake_pins_acquire %d\n", ret);
+		gb_timesync_teardown(timesync_svc);
+		return;
+	}
+
+	/* Choose an initial time in the future */
+	init_frame_time = __gb_timesync_get_frame_time(timesync_svc) + 100000UL;
+
+	/* Send enable command to all relevant participants */
+	list_for_each_entry(timesync_interface, &timesync_svc->interface_list,
+			    list) {
+		interface = timesync_interface->interface;
+		ret = gb_interface_timesync_enable(interface,
+						   GB_TIMESYNC_MAX_STROBES,
+						   init_frame_time,
+						   GB_TIMESYNC_STROBE_DELAY_US,
+						   clock_rate);
+		if (ret) {
+			dev_err(&interface->dev,
+				"interface timesync_enable %d\n", ret);
+		}
+	}
+
+	hd = timesync_svc->timesync_hd->hd;
+	ret = hd->driver->timesync_enable(hd, GB_TIMESYNC_MAX_STROBES,
+					  init_frame_time,
+					  GB_TIMESYNC_STROBE_DELAY_US,
+					  clock_rate);
+	if (ret < 0) {
+		dev_err(&hd->dev, "host timesync_enable %d\n",
+			ret);
+	}
+
+	gb_timesync_set_state_atomic(timesync_svc, GB_TIMESYNC_STATE_WAIT_SVC);
+	ret = gb_svc_timesync_enable(svc, GB_TIMESYNC_MAX_STROBES,
+				     init_frame_time,
+				     GB_TIMESYNC_STROBE_DELAY_US,
+				     clock_rate);
+	if (ret) {
+		dev_err(&svc->dev,
+			"gb_svc_timesync_enable %d\n", ret);
+		gb_timesync_teardown(timesync_svc);
+		return;
+	}
+
+	/* Schedule a timeout waiting for SVC to complete strobing */
+	gb_timesync_schedule_svc_timeout(timesync_svc);
+}
+
+static void gb_timesync_authoritative(struct gb_timesync_svc *timesync_svc)
+{
+	struct gb_svc *svc = timesync_svc->svc;
+	struct gb_host_device *hd;
+	struct gb_timesync_interface *timesync_interface;
+	struct gb_interface *interface;
+	u64 svc_frame_time[GB_TIMESYNC_MAX_STROBES];
+	int ret;
+
+	/* Get authoritative time from SVC and adjust local clock */
+	ret = gb_svc_timesync_authoritative(svc, svc_frame_time);
+	if (ret) {
+		dev_err(&svc->dev,
+			"gb_svc_timesync_authoritative %d\n", ret);
+		gb_timesync_teardown(timesync_svc);
+		return;
+	}
+	gb_timesync_collate_frame_time(timesync_svc, svc_frame_time);
+
+	/* Transmit authoritative time to downstream slaves */
+	hd = timesync_svc->timesync_hd->hd;
+	ret = hd->driver->timesync_authoritative(hd, svc_frame_time);
+	if (ret < 0)
+		dev_err(&hd->dev, "host timesync_authoritative %d\n", ret);
+
+	list_for_each_entry(timesync_interface,
+			    &timesync_svc->interface_list, list) {
+		interface = timesync_interface->interface;
+		ret = gb_interface_timesync_authoritative(
+						interface,
+						svc_frame_time);
+		if (ret) {
+			dev_err(&interface->dev,
+				"interface timesync_authoritative %d\n", ret);
+		}
+	}
+
+	/* Release wake pins */
+	gb_svc_timesync_wake_pins_release(svc);
+	gb_timesync_platform_unlock_bus();
+
+	/* Transition to state ACTIVE */
+	gb_timesync_set_state_atomic(timesync_svc, GB_TIMESYNC_STATE_ACTIVE);
+
+	/* Schedule a ping to verify the synchronized system time */
+	timesync_svc->print_ping = true;
+	gb_timesync_set_state_atomic(timesync_svc, GB_TIMESYNC_STATE_PING);
+}
+
+static int __gb_timesync_get_status(struct gb_timesync_svc *timesync_svc)
+{
+	int ret = -EINVAL;
+
+	switch (timesync_svc->state) {
+	case GB_TIMESYNC_STATE_INVALID:
+	case GB_TIMESYNC_STATE_INACTIVE:
+		ret = -ENODEV;
+		break;
+	case GB_TIMESYNC_STATE_INIT:
+	case GB_TIMESYNC_STATE_WAIT_SVC:
+	case GB_TIMESYNC_STATE_AUTHORITATIVE:
+		ret = -EAGAIN;
+		break;
+	case GB_TIMESYNC_STATE_PING:
+	case GB_TIMESYNC_STATE_ACTIVE:
+		ret = 0;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * This routine takes a FrameTime and derives the difference with-respect
+ * to a reference FrameTime/ktime pair. It then returns the calculated
+ * ktime based on the difference between the supplied FrameTime and
+ * the reference FrameTime.
+ *
+ * The time difference is calculated to six decimal places. Taking 19.2MHz
+ * as an example this means we have 52.083333~ nanoseconds per clock or
+ * 52083333~ femtoseconds per clock.
+ *
+ * Naively taking the count difference and converting to
+ * seconds/nanoseconds would quickly see the 0.0833 component produce
+ * noticeable errors. For example a time difference of one second would
+ * loose 19200000 * 0.08333x nanoseconds or 1.59 seconds.
+ *
+ * In contrast calculating in femtoseconds the same example of 19200000 *
+ * 0.000000083333x nanoseconds per count of error is just 1.59 nanoseconds!
+ *
+ * Continuing the example of 19.2 MHz we cap the maximum error difference
+ * at a worst-case 0.3 microseconds over a potential calculation window of
+ * abount 15 seconds, meaning you can convert a FrameTime that is <= 15
+ * seconds older/younger than the reference time with a maximum error of
+ * 0.2385 useconds. Note 19.2MHz is an example frequency not a requirement.
+ */
+static int gb_timesync_to_timespec(struct gb_timesync_svc *timesync_svc,
+				   u64 frame_time, struct timespec *ts)
+{
+	unsigned long flags;
+	u64 delta_fs, counts, sec, nsec;
+	bool add;
+	int ret = 0;
+
+	memset(ts, 0x00, sizeof(*ts));
+	mutex_lock(&timesync_svc->mutex);
+	spin_lock_irqsave(&timesync_svc->spinlock, flags);
+
+	ret = __gb_timesync_get_status(timesync_svc);
+	if (ret)
+		goto done;
+
+	/* Support calculating ktime upwards or downwards from the reference */
+	if (frame_time < timesync_svc->ktime_data.frame_time) {
+		add = false;
+		counts = timesync_svc->ktime_data.frame_time - frame_time;
+	} else {
+		add = true;
+		counts = frame_time - timesync_svc->ktime_data.frame_time;
+	}
+
+	/* Enforce the .23 of a usecond boundary @ 19.2MHz */
+	if (counts > gb_timesync_max_ktime_diff) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	/* Determine the time difference in femtoseconds */
+	delta_fs = counts * gb_timesync_fs_per_clock;
+
+	/* Convert to seconds */
+	sec = delta_fs;
+	do_div(sec, NSEC_PER_SEC);
+	do_div(sec, 1000000UL);
+
+	/* Get the nanosecond remainder */
+	nsec = do_div(delta_fs, sec);
+	do_div(nsec, 1000000UL);
+
+	if (add) {
+		/* Add the calculated offset - overflow nanoseconds upwards */
+		ts->tv_sec = timesync_svc->ktime_data.ts.tv_sec + sec;
+		ts->tv_nsec = timesync_svc->ktime_data.ts.tv_nsec + nsec;
+		if (ts->tv_nsec >= NSEC_PER_SEC) {
+			ts->tv_sec++;
+			ts->tv_nsec -= NSEC_PER_SEC;
+		}
+	} else {
+		/* Subtract the difference over/underflow as necessary */
+		if (nsec > timesync_svc->ktime_data.ts.tv_nsec) {
+			sec++;
+			nsec = nsec + timesync_svc->ktime_data.ts.tv_nsec;
+			nsec = do_div(nsec, NSEC_PER_SEC);
+		} else {
+			nsec = timesync_svc->ktime_data.ts.tv_nsec - nsec;
+		}
+		/* Cannot return a negative second value */
+		if (sec > timesync_svc->ktime_data.ts.tv_sec) {
+			ret = -EINVAL;
+			goto done;
+		}
+		ts->tv_sec = timesync_svc->ktime_data.ts.tv_sec - sec;
+		ts->tv_nsec = nsec;
+	}
+done:
+	spin_unlock_irqrestore(&timesync_svc->spinlock, flags);
+	mutex_unlock(&timesync_svc->mutex);
+	return ret;
+}
+
+static size_t gb_timesync_log_frame_time(struct gb_timesync_svc *timesync_svc,
+					 char *buf, size_t buflen)
+{
+	struct gb_svc *svc = timesync_svc->svc;
+	struct gb_host_device *hd;
+	struct gb_timesync_interface *timesync_interface;
+	struct gb_interface *interface;
+	unsigned int len;
+	size_t off;
+
+	/* AP/SVC */
+	off = snprintf(buf, buflen, "%s frametime: ap=%llu %s=%llu ",
+		       greybus_bus_type.name,
+		       timesync_svc->ap_ping_frame_time, dev_name(&svc->dev),
+		       timesync_svc->svc_ping_frame_time);
+	len = buflen - off;
+
+	/* APB/GPB */
+	if (len < buflen) {
+		hd = timesync_svc->timesync_hd->hd;
+		off += snprintf(&buf[off], len, "%s=%llu ", dev_name(&hd->dev),
+				timesync_svc->timesync_hd->ping_frame_time);
+		len = buflen - off;
+	}
+
+	list_for_each_entry(timesync_interface,
+			    &timesync_svc->interface_list, list) {
+		if (len < buflen) {
+			interface = timesync_interface->interface;
+			off += snprintf(&buf[off], len, "%s=%llu ",
+					dev_name(&interface->dev),
+					timesync_interface->ping_frame_time);
+			len = buflen - off;
+		}
+	}
+	if (len < buflen)
+		off += snprintf(&buf[off], len, "\n");
+	return off;
+}
+
+static size_t gb_timesync_log_frame_ktime(struct gb_timesync_svc *timesync_svc,
+					  char *buf, size_t buflen)
+{
+	struct gb_svc *svc = timesync_svc->svc;
+	struct gb_host_device *hd;
+	struct gb_timesync_interface *timesync_interface;
+	struct gb_interface *interface;
+	struct timespec ts;
+	unsigned int len;
+	size_t off;
+
+	/* AP */
+	gb_timesync_to_timespec(timesync_svc, timesync_svc->ap_ping_frame_time,
+				&ts);
+	off = snprintf(buf, buflen, "%s frametime: ap=%lu.%lu ",
+		       greybus_bus_type.name, ts.tv_sec, ts.tv_nsec);
+	len = buflen - off;
+	if (len >= buflen)
+		goto done;
+
+	/* SVC */
+	gb_timesync_to_timespec(timesync_svc, timesync_svc->svc_ping_frame_time,
+				&ts);
+	off += snprintf(&buf[off], len, "%s=%lu.%lu ", dev_name(&svc->dev),
+			ts.tv_sec, ts.tv_nsec);
+	len = buflen - off;
+	if (len >= buflen)
+		goto done;
+
+	/* APB/GPB */
+	hd = timesync_svc->timesync_hd->hd;
+	gb_timesync_to_timespec(timesync_svc,
+				timesync_svc->timesync_hd->ping_frame_time,
+				&ts);
+	off += snprintf(&buf[off], len, "%s=%lu.%lu ",
+			dev_name(&hd->dev),
+			ts.tv_sec, ts.tv_nsec);
+	len = buflen - off;
+	if (len >= buflen)
+		goto done;
+
+	list_for_each_entry(timesync_interface,
+			    &timesync_svc->interface_list, list) {
+		interface = timesync_interface->interface;
+		gb_timesync_to_timespec(timesync_svc,
+					timesync_interface->ping_frame_time,
+					&ts);
+		off += snprintf(&buf[off], len, "%s=%lu.%lu ",
+				dev_name(&interface->dev),
+				ts.tv_sec, ts.tv_nsec);
+		len = buflen - off;
+		if (len >= buflen)
+			goto done;
+	}
+	off += snprintf(&buf[off], len, "\n");
+done:
+	return off;
+}
+
+/*
+ * Send an SVC initiated wake 'ping' to each TimeSync participant.
+ * Get the FrameTime from each participant associated with the wake
+ * ping.
+ */
+static void gb_timesync_ping(struct gb_timesync_svc *timesync_svc)
+{
+	struct gb_svc *svc = timesync_svc->svc;
+	struct gb_host_device *hd;
+	struct gb_timesync_interface *timesync_interface;
+	struct gb_control *control;
+	u64 *ping_frame_time;
+	int ret;
+
+	/* Get access to the wake pins in the AP and SVC */
+	ret = gb_timesync_platform_lock_bus(timesync_svc);
+	if (ret < 0) {
+		gb_timesync_platform_lock_bus_fail(timesync_svc, ret);
+		return;
+	}
+	ret = gb_svc_timesync_wake_pins_acquire(svc, timesync_svc->strobe_mask);
+	if (ret) {
+		dev_err(&svc->dev,
+			"gb_svc_timesync_wake_pins_acquire %d\n", ret);
+		gb_timesync_teardown(timesync_svc);
+		return;
+	}
+
+	/* Have SVC generate a timesync ping */
+	timesync_svc->capture_ping = true;
+	timesync_svc->svc_ping_frame_time = 0;
+	ret = gb_svc_timesync_ping(svc, &timesync_svc->svc_ping_frame_time);
+	timesync_svc->capture_ping = false;
+	if (ret) {
+		dev_err(&svc->dev,
+			"gb_svc_timesync_ping %d\n", ret);
+		gb_timesync_teardown(timesync_svc);
+		return;
+	}
+
+	/* Get the ping FrameTime from each APB/GPB */
+	hd = timesync_svc->timesync_hd->hd;
+	timesync_svc->timesync_hd->ping_frame_time = 0;
+	ret = hd->driver->timesync_get_last_event(hd,
+		&timesync_svc->timesync_hd->ping_frame_time);
+	if (ret)
+		dev_err(&hd->dev, "host timesync_get_last_event %d\n", ret);
+
+	list_for_each_entry(timesync_interface,
+			    &timesync_svc->interface_list, list) {
+		control = timesync_interface->interface->control;
+		timesync_interface->ping_frame_time = 0;
+		ping_frame_time = &timesync_interface->ping_frame_time;
+		ret = gb_control_timesync_get_last_event(control,
+							 ping_frame_time);
+		if (ret) {
+			dev_err(&timesync_interface->interface->dev,
+				"gb_control_timesync_get_last_event %d\n", ret);
+		}
+	}
+
+	/* Ping success - move to timesync active */
+	gb_svc_timesync_wake_pins_release(svc);
+	gb_timesync_platform_unlock_bus();
+	gb_timesync_set_state_atomic(timesync_svc, GB_TIMESYNC_STATE_ACTIVE);
+}
+
+static void gb_timesync_log_ping_time(struct gb_timesync_svc *timesync_svc)
+{
+	char *buf;
+
+	if (!timesync_svc->print_ping)
+		return;
+
+	buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	if (buf) {
+		gb_timesync_log_frame_time(timesync_svc, buf, PAGE_SIZE);
+		dev_dbg(&timesync_svc->svc->dev, "%s", buf);
+		kfree(buf);
+	}
+}
+
+/*
+ * Perform the actual work of scheduled TimeSync logic.
+ */
+static void gb_timesync_worker(struct work_struct *work)
+{
+	struct delayed_work *delayed_work = to_delayed_work(work);
+	struct gb_timesync_svc *timesync_svc =
+		container_of(delayed_work, struct gb_timesync_svc, delayed_work);
+
+	mutex_lock(&timesync_svc->mutex);
+
+	switch (timesync_svc->state) {
+	case GB_TIMESYNC_STATE_INIT:
+		gb_timesync_enable(timesync_svc);
+		break;
+
+	case GB_TIMESYNC_STATE_WAIT_SVC:
+		dev_err(&timesync_svc->svc->dev,
+			"timeout SVC strobe completion %d/%d\n",
+			timesync_svc->strobe, GB_TIMESYNC_MAX_STROBES);
+		gb_timesync_teardown(timesync_svc);
+		break;
+
+	case GB_TIMESYNC_STATE_AUTHORITATIVE:
+		gb_timesync_authoritative(timesync_svc);
+		break;
+
+	case GB_TIMESYNC_STATE_PING:
+		gb_timesync_ping(timesync_svc);
+		gb_timesync_log_ping_time(timesync_svc);
+		break;
+
+	default:
+		pr_err("Invalid state %d for delayed work\n",
+		       timesync_svc->state);
+		break;
+	}
+
+	mutex_unlock(&timesync_svc->mutex);
+}
+
+/*
+ * Schedule a new TimeSync INIT or PING operation serialized w/r to
+ * gb_timesync_worker().
+ */
+static int gb_timesync_schedule(struct gb_timesync_svc *timesync_svc, int state)
+{
+	int ret = 0;
+
+	if (state != GB_TIMESYNC_STATE_INIT && state != GB_TIMESYNC_STATE_PING)
+		return -EINVAL;
+
+	mutex_lock(&timesync_svc->mutex);
+	if (timesync_svc->state !=  GB_TIMESYNC_STATE_INVALID) {
+		gb_timesync_set_state_atomic(timesync_svc, state);
+	} else {
+		ret = -ENODEV;
+	}
+	mutex_unlock(&timesync_svc->mutex);
+	return ret;
+}
+
+static int __gb_timesync_schedule_synchronous(
+	struct gb_timesync_svc *timesync_svc, int state)
+{
+	unsigned long flags;
+	int ret;
+
+	ret = gb_timesync_schedule(timesync_svc, state);
+	if (ret)
+		return ret;
+
+	ret = wait_event_interruptible(timesync_svc->wait_queue,
+			(timesync_svc->state == GB_TIMESYNC_STATE_ACTIVE ||
+			 timesync_svc->state == GB_TIMESYNC_STATE_INACTIVE ||
+			 timesync_svc->state == GB_TIMESYNC_STATE_INVALID));
+	if (ret)
+		return ret;
+
+	mutex_lock(&timesync_svc->mutex);
+	spin_lock_irqsave(&timesync_svc->spinlock, flags);
+
+	ret = __gb_timesync_get_status(timesync_svc);
+
+	spin_unlock_irqrestore(&timesync_svc->spinlock, flags);
+	mutex_unlock(&timesync_svc->mutex);
+
+	return ret;
+}
+
+static struct gb_timesync_svc *gb_timesync_find_timesync_svc(
+	struct gb_host_device *hd)
+{
+	struct gb_timesync_svc *timesync_svc;
+
+	list_for_each_entry(timesync_svc, &gb_timesync_svc_list, list) {
+		if (timesync_svc->svc == hd->svc)
+			return timesync_svc;
+	}
+	return NULL;
+}
+
+static struct gb_timesync_interface *gb_timesync_find_timesync_interface(
+	struct gb_timesync_svc *timesync_svc,
+	struct gb_interface *interface)
+{
+	struct gb_timesync_interface *timesync_interface;
+
+	list_for_each_entry(timesync_interface, &timesync_svc->interface_list, list) {
+		if (timesync_interface->interface == interface)
+			return timesync_interface;
+	}
+	return NULL;
+}
+
+int gb_timesync_schedule_synchronous(struct gb_interface *interface)
+{
+	int ret;
+	struct gb_timesync_svc *timesync_svc;
+	int retries;
+
+	if (!(interface->features & GREYBUS_INTERFACE_FEATURE_TIMESYNC))
+		return 0;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	for (retries = 0; retries < GB_TIMESYNC_MAX_RETRIES; retries++) {
+		timesync_svc = gb_timesync_find_timesync_svc(interface->hd);
+		if (!timesync_svc) {
+			ret = -ENODEV;
+			goto done;
+		}
+
+		ret = __gb_timesync_schedule_synchronous(timesync_svc,
+						 GB_TIMESYNC_STATE_INIT);
+		if (!ret)
+			break;
+	}
+	if (ret && retries == GB_TIMESYNC_MAX_RETRIES)
+		ret = -ETIMEDOUT;
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_timesync_schedule_synchronous);
+
+void gb_timesync_schedule_asynchronous(struct gb_interface *interface)
+{
+	struct gb_timesync_svc *timesync_svc;
+
+	if (!(interface->features & GREYBUS_INTERFACE_FEATURE_TIMESYNC))
+		return;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	timesync_svc = gb_timesync_find_timesync_svc(interface->hd);
+	if (!timesync_svc)
+		goto done;
+
+	gb_timesync_schedule(timesync_svc, GB_TIMESYNC_STATE_INIT);
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+	return;
+}
+EXPORT_SYMBOL_GPL(gb_timesync_schedule_asynchronous);
+
+static ssize_t gb_timesync_ping_read(struct file *file, char __user *ubuf,
+				     size_t len, loff_t *offset, bool ktime)
+{
+	struct gb_timesync_svc *timesync_svc = file->f_inode->i_private;
+	char *buf;
+	ssize_t ret = 0;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	mutex_lock(&timesync_svc->mutex);
+	if (list_empty(&timesync_svc->interface_list))
+		ret = -ENODEV;
+	timesync_svc->print_ping = false;
+	mutex_unlock(&timesync_svc->mutex);
+	if (ret)
+		goto done;
+
+	ret = __gb_timesync_schedule_synchronous(timesync_svc,
+						 GB_TIMESYNC_STATE_PING);
+	if (ret)
+		goto done;
+
+	buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	if (ktime)
+		ret = gb_timesync_log_frame_ktime(timesync_svc, buf, PAGE_SIZE);
+	else
+		ret = gb_timesync_log_frame_time(timesync_svc, buf, PAGE_SIZE);
+	if (ret > 0)
+		ret = simple_read_from_buffer(ubuf, len, offset, buf, ret);
+	kfree(buf);
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+	return ret;
+}
+
+static ssize_t gb_timesync_ping_read_frame_time(struct file *file,
+						char __user *buf,
+						size_t len, loff_t *offset)
+{
+	return gb_timesync_ping_read(file, buf, len, offset, false);
+}
+
+static ssize_t gb_timesync_ping_read_frame_ktime(struct file *file,
+						 char __user *buf,
+						 size_t len, loff_t *offset)
+{
+	return gb_timesync_ping_read(file, buf, len, offset, true);
+}
+
+static const struct file_operations gb_timesync_debugfs_frame_time_ops = {
+	.read		= gb_timesync_ping_read_frame_time,
+};
+
+static const struct file_operations gb_timesync_debugfs_frame_ktime_ops = {
+	.read		= gb_timesync_ping_read_frame_ktime,
+};
+
+static int gb_timesync_hd_add(struct gb_timesync_svc *timesync_svc,
+			      struct gb_host_device *hd)
+{
+	struct gb_timesync_host_device *timesync_hd;
+
+	timesync_hd = kzalloc(sizeof(*timesync_hd), GFP_KERNEL);
+	if (!timesync_hd)
+		return -ENOMEM;
+
+	WARN_ON(timesync_svc->timesync_hd);
+	timesync_hd->hd = hd;
+	timesync_svc->timesync_hd = timesync_hd;
+
+	return 0;
+}
+
+static void gb_timesync_hd_remove(struct gb_timesync_svc *timesync_svc,
+				  struct gb_host_device *hd)
+{
+	if (timesync_svc->timesync_hd->hd == hd) {
+		kfree(timesync_svc->timesync_hd);
+		timesync_svc->timesync_hd = NULL;
+		return;
+	}
+	WARN_ON(1);
+}
+
+int gb_timesync_svc_add(struct gb_svc *svc)
+{
+	struct gb_timesync_svc *timesync_svc;
+	int ret;
+
+	timesync_svc = kzalloc(sizeof(*timesync_svc), GFP_KERNEL);
+	if (!timesync_svc)
+		return -ENOMEM;
+
+	timesync_svc->work_queue =
+		create_singlethread_workqueue("gb-timesync-work_queue");
+
+	if (!timesync_svc->work_queue) {
+		kfree(timesync_svc);
+		return -ENOMEM;
+	}
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	INIT_LIST_HEAD(&timesync_svc->interface_list);
+	INIT_DELAYED_WORK(&timesync_svc->delayed_work, gb_timesync_worker);
+	mutex_init(&timesync_svc->mutex);
+	spin_lock_init(&timesync_svc->spinlock);
+	init_waitqueue_head(&timesync_svc->wait_queue);
+
+	timesync_svc->svc = svc;
+	timesync_svc->frame_time_offset = 0;
+	timesync_svc->capture_ping = false;
+	gb_timesync_set_state_atomic(timesync_svc, GB_TIMESYNC_STATE_INACTIVE);
+
+	timesync_svc->frame_time_dentry =
+		debugfs_create_file("frame-time", S_IRUGO, svc->debugfs_dentry,
+				    timesync_svc,
+				    &gb_timesync_debugfs_frame_time_ops);
+	timesync_svc->frame_ktime_dentry =
+		debugfs_create_file("frame-ktime", S_IRUGO, svc->debugfs_dentry,
+				    timesync_svc,
+				    &gb_timesync_debugfs_frame_ktime_ops);
+
+	list_add(&timesync_svc->list, &gb_timesync_svc_list);
+	ret = gb_timesync_hd_add(timesync_svc, svc->hd);
+	if (ret) {
+		list_del(&timesync_svc->list);
+		debugfs_remove(timesync_svc->frame_ktime_dentry);
+		debugfs_remove(timesync_svc->frame_time_dentry);
+		destroy_workqueue(timesync_svc->work_queue);
+		kfree(timesync_svc);
+		goto done;
+	}
+
+	init_timer(&timesync_svc->ktime_timer);
+	timesync_svc->ktime_timer.function = gb_timesync_ktime_timer_fn;
+	timesync_svc->ktime_timer.expires = jiffies + GB_TIMESYNC_KTIME_UPDATE;
+	timesync_svc->ktime_timer.data = (unsigned long)timesync_svc;
+	add_timer(&timesync_svc->ktime_timer);
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_timesync_svc_add);
+
+void gb_timesync_svc_remove(struct gb_svc *svc)
+{
+	struct gb_timesync_svc *timesync_svc;
+	struct gb_timesync_interface *timesync_interface;
+	struct gb_timesync_interface *next;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	timesync_svc = gb_timesync_find_timesync_svc(svc->hd);
+	if (!timesync_svc)
+		goto done;
+
+	cancel_delayed_work_sync(&timesync_svc->delayed_work);
+
+	mutex_lock(&timesync_svc->mutex);
+
+	gb_timesync_set_state_atomic(timesync_svc, GB_TIMESYNC_STATE_INVALID);
+	del_timer_sync(&timesync_svc->ktime_timer);
+	gb_timesync_teardown(timesync_svc);
+
+	gb_timesync_hd_remove(timesync_svc, svc->hd);
+	list_for_each_entry_safe(timesync_interface, next,
+				 &timesync_svc->interface_list, list) {
+		list_del(&timesync_interface->list);
+		kfree(timesync_interface);
+	}
+	debugfs_remove(timesync_svc->frame_ktime_dentry);
+	debugfs_remove(timesync_svc->frame_time_dentry);
+	destroy_workqueue(timesync_svc->work_queue);
+	list_del(&timesync_svc->list);
+
+	mutex_unlock(&timesync_svc->mutex);
+
+	kfree(timesync_svc);
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+}
+EXPORT_SYMBOL_GPL(gb_timesync_svc_remove);
+
+/*
+ * Add a Greybus Interface to the set of TimeSync Interfaces.
+ */
+int gb_timesync_interface_add(struct gb_interface *interface)
+{
+	struct gb_timesync_svc *timesync_svc;
+	struct gb_timesync_interface *timesync_interface;
+	int ret = 0;
+
+	if (!(interface->features & GREYBUS_INTERFACE_FEATURE_TIMESYNC))
+		return 0;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	timesync_svc = gb_timesync_find_timesync_svc(interface->hd);
+	if (!timesync_svc) {
+		ret = -ENODEV;
+		goto done;
+	}
+
+	timesync_interface = kzalloc(sizeof(*timesync_interface), GFP_KERNEL);
+	if (!timesync_interface) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	mutex_lock(&timesync_svc->mutex);
+	timesync_interface->interface = interface;
+	list_add(&timesync_interface->list, &timesync_svc->interface_list);
+	timesync_svc->strobe_mask |= 1 << interface->interface_id;
+	mutex_unlock(&timesync_svc->mutex);
+
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_timesync_interface_add);
+
+/*
+ * Remove a Greybus Interface from the set of TimeSync Interfaces.
+ */
+void gb_timesync_interface_remove(struct gb_interface *interface)
+{
+	struct gb_timesync_svc *timesync_svc;
+	struct gb_timesync_interface *timesync_interface;
+
+	if (!(interface->features & GREYBUS_INTERFACE_FEATURE_TIMESYNC))
+		return;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	timesync_svc = gb_timesync_find_timesync_svc(interface->hd);
+	if (!timesync_svc)
+		goto done;
+
+	timesync_interface = gb_timesync_find_timesync_interface(timesync_svc,
+								 interface);
+	if (!timesync_interface)
+		goto done;
+
+	mutex_lock(&timesync_svc->mutex);
+	timesync_svc->strobe_mask &= ~(1 << interface->interface_id);
+	list_del(&timesync_interface->list);
+	kfree(timesync_interface);
+	mutex_unlock(&timesync_svc->mutex);
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+}
+EXPORT_SYMBOL_GPL(gb_timesync_interface_remove);
+
+/*
+ * Give the authoritative FrameTime to the calling function. Returns zero if we
+ * are not in GB_TIMESYNC_STATE_ACTIVE.
+ */
+static u64 gb_timesync_get_frame_time(struct gb_timesync_svc *timesync_svc)
+{
+	unsigned long flags;
+	u64 ret;
+
+	spin_lock_irqsave(&timesync_svc->spinlock, flags);
+	if (timesync_svc->state == GB_TIMESYNC_STATE_ACTIVE)
+		ret = __gb_timesync_get_frame_time(timesync_svc);
+	else
+		ret = 0;
+	spin_unlock_irqrestore(&timesync_svc->spinlock, flags);
+	return ret;
+}
+
+u64 gb_timesync_get_frame_time_by_interface(struct gb_interface *interface)
+{
+	struct gb_timesync_svc *timesync_svc;
+	u64 ret = 0;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	timesync_svc = gb_timesync_find_timesync_svc(interface->hd);
+	if (!timesync_svc)
+		goto done;
+
+	ret = gb_timesync_get_frame_time(timesync_svc);
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_timesync_get_frame_time_by_interface);
+
+u64 gb_timesync_get_frame_time_by_svc(struct gb_svc *svc)
+{
+	struct gb_timesync_svc *timesync_svc;
+	u64 ret = 0;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	timesync_svc = gb_timesync_find_timesync_svc(svc->hd);
+	if (!timesync_svc)
+		goto done;
+
+	ret = gb_timesync_get_frame_time(timesync_svc);
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_timesync_get_frame_time_by_svc);
+
+/* Incrementally updates the conversion base from FrameTime to ktime */
+static void gb_timesync_ktime_timer_fn(unsigned long data)
+{
+	struct gb_timesync_svc *timesync_svc =
+		(struct gb_timesync_svc *)data;
+	unsigned long flags;
+	u64 frame_time;
+	struct timespec ts;
+
+	spin_lock_irqsave(&timesync_svc->spinlock, flags);
+
+	if (timesync_svc->state != GB_TIMESYNC_STATE_ACTIVE)
+		goto done;
+
+	ktime_get_ts(&ts);
+	frame_time = __gb_timesync_get_frame_time(timesync_svc);
+	gb_timesync_store_ktime(timesync_svc, ts, frame_time);
+
+done:
+	spin_unlock_irqrestore(&timesync_svc->spinlock, flags);
+	mod_timer(&timesync_svc->ktime_timer,
+		  jiffies + GB_TIMESYNC_KTIME_UPDATE);
+}
+
+int gb_timesync_to_timespec_by_svc(struct gb_svc *svc, u64 frame_time,
+				   struct timespec *ts)
+{
+	struct gb_timesync_svc *timesync_svc;
+	int ret = 0;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	timesync_svc = gb_timesync_find_timesync_svc(svc->hd);
+	if (!timesync_svc) {
+		ret = -ENODEV;
+		goto done;
+	}
+	ret = gb_timesync_to_timespec(timesync_svc, frame_time, ts);
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_timesync_to_timespec_by_svc);
+
+int gb_timesync_to_timespec_by_interface(struct gb_interface *interface,
+					 u64 frame_time, struct timespec *ts)
+{
+	struct gb_timesync_svc *timesync_svc;
+	int ret = 0;
+
+	mutex_lock(&gb_timesync_svc_list_mutex);
+	timesync_svc = gb_timesync_find_timesync_svc(interface->hd);
+	if (!timesync_svc) {
+		ret = -ENODEV;
+		goto done;
+	}
+
+	ret = gb_timesync_to_timespec(timesync_svc, frame_time, ts);
+done:
+	mutex_unlock(&gb_timesync_svc_list_mutex);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gb_timesync_to_timespec_by_interface);
+
+void gb_timesync_irq(struct gb_timesync_svc *timesync_svc)
+{
+	unsigned long flags;
+	u64 strobe_time;
+	bool strobe_is_ping = true;
+	struct timespec ts;
+
+	ktime_get_ts(&ts);
+	strobe_time = __gb_timesync_get_frame_time(timesync_svc);
+
+	spin_lock_irqsave(&timesync_svc->spinlock, flags);
+
+	if (timesync_svc->state == GB_TIMESYNC_STATE_PING) {
+		if (!timesync_svc->capture_ping)
+			goto done_nolog;
+		timesync_svc->ap_ping_frame_time = strobe_time;
+		goto done_log;
+	} else if (timesync_svc->state != GB_TIMESYNC_STATE_WAIT_SVC) {
+		goto done_nolog;
+	}
+
+	timesync_svc->strobe_data[timesync_svc->strobe].frame_time = strobe_time;
+	timesync_svc->strobe_data[timesync_svc->strobe].ts = ts;
+
+	if (++timesync_svc->strobe == GB_TIMESYNC_MAX_STROBES) {
+		gb_timesync_set_state(timesync_svc,
+				      GB_TIMESYNC_STATE_AUTHORITATIVE);
+	}
+	strobe_is_ping = false;
+done_log:
+	trace_gb_timesync_irq(strobe_is_ping, timesync_svc->strobe,
+			      GB_TIMESYNC_MAX_STROBES, strobe_time);
+done_nolog:
+	spin_unlock_irqrestore(&timesync_svc->spinlock, flags);
+}
+EXPORT_SYMBOL(gb_timesync_irq);
+
+int __init gb_timesync_init(void)
+{
+	int ret = 0;
+
+	ret = gb_timesync_platform_init();
+	if (ret) {
+		pr_err("timesync platform init fail!\n");
+		return ret;
+	}
+
+	gb_timesync_clock_rate = gb_timesync_platform_get_clock_rate();
+
+	/* Calculate nanoseconds and femtoseconds per clock */
+	gb_timesync_fs_per_clock = FSEC_PER_SEC;
+	do_div(gb_timesync_fs_per_clock, gb_timesync_clock_rate);
+	gb_timesync_ns_per_clock = NSEC_PER_SEC;
+	do_div(gb_timesync_ns_per_clock, gb_timesync_clock_rate);
+
+	/* Calculate the maximum number of clocks we will convert to ktime */
+	gb_timesync_max_ktime_diff =
+		GB_TIMESYNC_MAX_KTIME_CONVERSION * gb_timesync_clock_rate;
+
+	pr_info("Time-Sync @ %lu Hz max ktime conversion +/- %d seconds\n",
+		gb_timesync_clock_rate, GB_TIMESYNC_MAX_KTIME_CONVERSION);
+	return 0;
+}
+
+void gb_timesync_exit(void)
+{
+	gb_timesync_platform_exit();
+}
diff --git a/drivers/staging/greybus/timesync.h b/drivers/staging/greybus/timesync.h
new file mode 100644
index 000000000000..72fc9a35a002
--- /dev/null
+++ b/drivers/staging/greybus/timesync.h
@@ -0,0 +1,45 @@
+/*
+ * TimeSync API driver.
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#ifndef __TIMESYNC_H
+#define __TIMESYNC_H
+
+struct gb_svc;
+struct gb_interface;
+struct gb_timesync_svc;
+
+/* Platform */
+u64 gb_timesync_platform_get_counter(void);
+u32 gb_timesync_platform_get_clock_rate(void);
+int gb_timesync_platform_lock_bus(struct gb_timesync_svc *pdata);
+void gb_timesync_platform_unlock_bus(void);
+
+int gb_timesync_platform_init(void);
+void gb_timesync_platform_exit(void);
+
+/* Core API */
+int gb_timesync_interface_add(struct gb_interface *interface);
+void gb_timesync_interface_remove(struct gb_interface *interface);
+int gb_timesync_svc_add(struct gb_svc *svc);
+void gb_timesync_svc_remove(struct gb_svc *svc);
+
+u64 gb_timesync_get_frame_time_by_interface(struct gb_interface *interface);
+u64 gb_timesync_get_frame_time_by_svc(struct gb_svc *svc);
+int gb_timesync_to_timespec_by_svc(struct gb_svc *svc, u64 frame_time,
+				   struct timespec *ts);
+int gb_timesync_to_timespec_by_interface(struct gb_interface *interface,
+					 u64 frame_time, struct timespec *ts);
+
+int gb_timesync_schedule_synchronous(struct gb_interface *intf);
+void gb_timesync_schedule_asynchronous(struct gb_interface *intf);
+void gb_timesync_irq(struct gb_timesync_svc *timesync_svc);
+int gb_timesync_init(void);
+void gb_timesync_exit(void);
+
+#endif /* __TIMESYNC_H */
diff --git a/drivers/staging/greybus/timesync_platform.c b/drivers/staging/greybus/timesync_platform.c
new file mode 100644
index 000000000000..113f3d6c4b3a
--- /dev/null
+++ b/drivers/staging/greybus/timesync_platform.c
@@ -0,0 +1,82 @@
+/*
+ * TimeSync API driver.
+ *
+ * Copyright 2016 Google Inc.
+ * Copyright 2016 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ *
+ * This code reads directly from an ARMv7 memory-mapped timer that lives in
+ * MMIO space. Since this counter lives inside of MMIO space its shared between
+ * cores and that means we don't have to worry about issues like TSC on x86
+ * where each time-stamp-counter (TSC) is local to a particular core.
+ *
+ * Register-level access code is based on
+ * drivers/clocksource/arm_arch_timer.c
+ */
+#include <linux/cpufreq.h>
+#include <linux/of_platform.h>
+
+#include "greybus.h"
+#include "arche_platform.h"
+
+#define DEFAULT_FRAMETIME_CLOCK_HZ 19200000
+
+static u32 gb_timesync_clock_frequency;
+int (*arche_platform_change_state_cb)(enum arche_platform_state state,
+				      struct gb_timesync_svc *pdata);
+EXPORT_SYMBOL_GPL(arche_platform_change_state_cb);
+
+u64 gb_timesync_platform_get_counter(void)
+{
+	return (u64)get_cycles();
+}
+
+u32 gb_timesync_platform_get_clock_rate(void)
+{
+	if (unlikely(!gb_timesync_clock_frequency)) {
+		gb_timesync_clock_frequency = cpufreq_get(0);
+		if (!gb_timesync_clock_frequency)
+			gb_timesync_clock_frequency = DEFAULT_FRAMETIME_CLOCK_HZ;
+	}
+
+	return gb_timesync_clock_frequency;
+}
+
+int gb_timesync_platform_lock_bus(struct gb_timesync_svc *pdata)
+{
+	return arche_platform_change_state_cb(ARCHE_PLATFORM_STATE_TIME_SYNC,
+					      pdata);
+}
+
+void gb_timesync_platform_unlock_bus(void)
+{
+	arche_platform_change_state_cb(ARCHE_PLATFORM_STATE_ACTIVE, NULL);
+}
+
+static const struct of_device_id arch_timer_of_match[] = {
+	{ .compatible   = "google,greybus-frame-time-counter", },
+	{},
+};
+
+int __init gb_timesync_platform_init(void)
+{
+	struct device_node *np;
+
+	np = of_find_matching_node(NULL, arch_timer_of_match);
+	if (!np) {
+		/* Tolerate not finding to allow BBB etc to continue */
+		pr_warn("Unable to find a compatible ARMv7 timer\n");
+		return 0;
+	}
+
+	if (of_property_read_u32(np, "clock-frequency",
+				 &gb_timesync_clock_frequency)) {
+		pr_err("Unable to find timer clock-frequency\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+void gb_timesync_platform_exit(void) {}
diff --git a/drivers/staging/greybus/tools/.gitignore b/drivers/staging/greybus/tools/.gitignore
new file mode 100644
index 000000000000..023654c83068
--- /dev/null
+++ b/drivers/staging/greybus/tools/.gitignore
@@ -0,0 +1 @@
+loopback_test
diff --git a/drivers/staging/greybus/tools/Android.mk b/drivers/staging/greybus/tools/Android.mk
new file mode 100644
index 000000000000..fdadbf611757
--- /dev/null
+++ b/drivers/staging/greybus/tools/Android.mk
@@ -0,0 +1,10 @@
+LOCAL_PATH:= $(call my-dir)
+
+include $(CLEAR_VARS)
+
+LOCAL_SRC_FILES:= loopback_test.c
+LOCAL_MODULE_TAGS := optional
+LOCAL_MODULE := gb_loopback_test
+
+include $(BUILD_EXECUTABLE)
+
diff --git a/drivers/staging/greybus/tools/Makefile b/drivers/staging/greybus/tools/Makefile
new file mode 100644
index 000000000000..852b12b71149
--- /dev/null
+++ b/drivers/staging/greybus/tools/Makefile
@@ -0,0 +1,31 @@
+ifeq ($(strip $(V)), 1)
+  Q =
+else
+  Q = @
+endif
+
+CFLAGS	+= -std=gnu99 -Wall -Wextra -g \
+	    -D_GNU_SOURCE \
+	    -Wno-unused-parameter \
+	    -Wmaybe-uninitialized \
+	    -Wredundant-decls \
+	    -Wcast-align \
+	    -Wsign-compare \
+	    -Wno-missing-field-initializers
+
+CC	:= $(CROSS_COMPILE)gcc
+
+TOOLS = loopback_test
+
+all: $(TOOLS)
+
+%.o: %.c ../greybus_protocols.h
+	@echo '  TARGET_CC $@'
+	$(Q)$(CC) $(CFLAGS) -c $< -o $@
+
+loopback_%: loopback_%.o
+	@echo '  TARGET_LD $@'
+	$(Q)$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@
+
+clean::
+	rm -f *.o $(TOOLS)
diff --git a/drivers/staging/greybus/tools/README.loopback b/drivers/staging/greybus/tools/README.loopback
new file mode 100644
index 000000000000..845b08dc4696
--- /dev/null
+++ b/drivers/staging/greybus/tools/README.loopback
@@ -0,0 +1,198 @@
+
+
+        1 - LOOPBACK DRIVER
+
+The driver implements the main logic of the loopback test and provides
+sysfs files to configure the test and retrieve the results.
+A user could run a test without the need of the test application given
+that he understands the sysfs interface of the loopback driver.
+
+The loopback kernel driver needs to be loaded and at least one module
+with the loopback feature enabled must be present for the sysfs files to be
+created and for the loopback test application to be able to run.
+
+To load the module:
+# modprobe gb-loopback
+
+
+When the module is probed,  New files are available on the sysfs
+directory of the detected loopback device.
+(typically under "/sys/bus/graybus/devices").
+
+Here is a short summary of the sysfs interface files that should be visible:
+
+* Loopback Configuration Files:
+    async - Use asynchronous operations.
+    iteration_max - Number of tests iterations to perform.
+    size - payload size of the transfer.
+    timeout - The number of microseconds to give an individual
+              asynchronous request before timing out.
+    us_wait - Time to wait between 2 messages
+    type - By writing the test type to this file, the test starts.
+           Valid tests are:
+             0 stop the test
+             2 - ping
+             3 - transfer
+             4 - sink
+
+* Loopback feedback files:
+    error - number of errors that have occurred.
+    iteration_count - Number of iterations performed.
+    requests_completed - Number of requests successfully completed.
+    requests_timedout - Number of requests that have timed out.
+    timeout_max - Max allowed timeout
+    timeout_min - Min allowed timeout.
+
+* Loopback result files:
+    apbridge_unipro_latency_avg
+    apbridge_unipro_latency_max
+    apbridge_unipro_latency_min
+    gpbridge_firmware_latency_avg
+    gpbridge_firmware_latency_max
+    gpbridge_firmware_latency_min
+    requests_per_second_avg
+    requests_per_second_max
+    requests_per_second_min
+    latency_avg
+    latency_max
+    latency_min
+    throughput_avg
+    throughput_max
+    throughput_min
+
+
+
+            2 - LOOPBACK TEST APPLICATION
+
+The loopback test application manages and formats the results provided by
+the loopback kernel module. The purpose of this application
+is to:
+    - Start and manage multiple loopback device tests concurrently.
+    - Calculate the aggregate results for multiple devices.
+    - Gather and format test results (csv or human readable).
+
+The best way to get up to date usage information for the application is
+usually to pass the "-h" parameter.
+Here is the summary of the available options:
+
+ Mandatory arguments
+   -t     must be one of the test names - sink, transfer or ping
+   -i     iteration count - the number of iterations to run the test over
+ Optional arguments
+   -S     sysfs location - location for greybus 'endo' entires default /sys/bus/greybus/devices/
+   -D     debugfs location - location for loopback debugfs entries default /sys/kernel/debug/gb_loopback/
+   -s     size of data packet to send during test - defaults to zero
+   -m     mask - a bit mask of connections to include example: -m 8 = 4th connection -m 9 = 1st and 4th connection etc
+                 default is zero which means broadcast to all connections
+   -v     verbose output
+   -d     debug output
+   -r     raw data output - when specified the full list of latency values are included in the output CSV
+   -p     porcelain - when specified printout is in a user-friendly non-CSV format. This option suppresses writing to CSV file
+   -a     aggregate - show aggregation of all enabled devies
+   -l     list found loopback devices and exit.
+   -x     Async - Enable async transfers.
+   -o     Timeout - Timeout in microseconds for async operations.
+
+
+
+             3 - REAL WORLD EXAMPLE USAGES
+
+ 3.1 - Using the driver sysfs files to run a test on a single device:
+
+* Run a 1000 transfers of a 100 byte packet. Each transfer is started only
+after the previous one finished successfully:
+    echo 0 > /sys/bus/greybus/devices/1-2.17/type
+    echo 0 > /sys/bus/greybus/devices/1-2.17/async
+    echo 2000 > /sys/bus/greybus/devices/1-2.17/us_wait
+    echo 100 > /sys/bus/greybus/devices/1-2.17/size
+    echo 1000 > /sys/bus/greybus/devices/1-2.17/iteration_max
+    echo 0 > /sys/bus/greybus/devices/1-2.17/mask
+    echo 200000 > /sys/bus/greybus/devices/1-2.17/timeout
+    echo 3 > /sys/bus/greybus/devices/1-2.17/type
+
+* Run a 1000 transfers of a 100 byte packet. Transfers are started without
+waiting for the previous one to finish:
+    echo 0 > /sys/bus/greybus/devices/1-2.17/type
+    echo 3 > /sys/bus/greybus/devices/1-2.17/async
+    echo 0 > /sys/bus/greybus/devices/1-2.17/us_wait
+    echo 100 > /sys/bus/greybus/devices/1-2.17/size
+    echo 1000 > /sys/bus/greybus/devices/1-2.17/iteration_max
+    echo 0 > /sys/bus/greybus/devices/1-2.17/mask
+    echo 200000 > /sys/bus/greybus/devices/1-2.17/timeout
+    echo 3 > /sys/bus/greybus/devices/1-2.17/type
+
+* Read the results from sysfs:
+    cat /sys/bus/greybus/devices/1-2.17/requests_per_second_min
+    cat /sys/bus/greybus/devices/1-2.17/requests_per_second_max
+    cat /sys/bus/greybus/devices/1-2.17/requests_per_second_avg
+
+    cat /sys/bus/greybus/devices/1-2.17/latency_min
+    cat /sys/bus/greybus/devices/1-2.17/latency_max
+    cat /sys/bus/greybus/devices/1-2.17/latency_avg
+
+    cat /sys/bus/greybus/devices/1-2.17/apbridge_unipro_latency_min
+    cat /sys/bus/greybus/devices/1-2.17/apbridge_unipro_latency_max
+    cat /sys/bus/greybus/devices/1-2.17/apbridge_unipro_latency_avg
+
+    cat /sys/bus/greybus/devices/1-2.17/gpbridge_firmware_latency_min
+    cat /sys/bus/greybus/devices/1-2.17/gpbridge_firmware_latency_max
+    cat /sys/bus/greybus/devices/1-2.17/gpbridge_firmware_latency_avg
+
+    cat /sys/bus/greybus/devices/1-2.17/error
+    cat /sys/bus/greybus/devices/1-2.17/requests_completed
+    cat /sys/bus/greybus/devices/1-2.17/requests_timedout
+
+
+3.2 - using the test application:
+
+* Run a transfer test 10 iterations of size 100 bytes on all available devices
+    #/loopback_test -t transfer -i 10 -s 100
+    1970-1-1 0:10:7,transfer,1-4.17,100,10,0,443,509,471.700012,66,1963,2256,2124.600098,293,102776,118088,109318.898438,15312,1620,1998,1894.099976,378,56,57,56.799999,1
+    1970-1-1 0:10:7,transfer,1-5.17,100,10,0,399,542,463.399994,143,1845,2505,2175.800049,660,92568,125744,107393.296875,33176,1469,2305,1806.500000,836,56,57,56.799999,1
+
+
+* Show the aggregate results of both devices. ("-a")
+    #/loopback_test -t transfer -i 10 -s 100  -a
+    1970-1-1 0:10:35,transfer,1-4.17,100,10,0,448,580,494.100006,132,1722,2230,2039.400024,508,103936,134560,114515.703125,30624,1513,1980,1806.900024,467,56,57,57.299999,1
+    1970-1-1 0:10:35,transfer,1-5.17,100,10,0,383,558,478.600006,175,1791,2606,2115.199951,815,88856,129456,110919.703125,40600,1457,2246,1773.599976,789,56,57,57.099998,1
+    1970-1-1 0:10:35,transfer,aggregate,100,10,0,383,580,486.000000,197,1722,2606,2077.000000,884,88856,134560,112717.000000,45704,1457,2246,1789.000000,789,56,57,57.000000,1
+
+* Example usage of the mask option to select which devices will
+  run the test (1st, 2nd, or both devices):
+    # /loopback_test -t transfer -i 10 -s 100 -m 1
+    1970-1-1 0:11:56,transfer,1-4.17,100,10,0,514,558,544.900024,44,1791,1943,1836.599976,152,119248,129456,126301.296875,10208,1600,1001609,101613.601562,1000009,56,57,56.900002,1
+    # /loopback_test -t transfer -i 10 -s 100 -m 2
+    1970-1-1 0:12:0,transfer,1-5.17,100,10,0,468,554,539.000000,86,1804,2134,1859.500000,330,108576,128528,124932.500000,19952,1606,1626,1619.300049,20,56,57,57.400002,1
+    # /loopback_test -t transfer -i 10 -s 100 -m 3
+    1970-1-1 0:12:3,transfer,1-4.17,100,10,0,432,510,469.399994,78,1959,2313,2135.800049,354,100224,118320,108785.296875,18096,1610,2024,1893.500000,414,56,57,57.200001,1
+    1970-1-1 0:12:3,transfer,1-5.17,100,10,0,404,542,468.799988,138,1843,2472,2152.500000,629,93728,125744,108646.101562,32016,1504,2247,1853.099976,743,56,57,57.099998,1
+
+* Show output in human readable format ("-p")
+    # /loopback_test -t transfer -i 10 -s 100 -m 3 -p
+
+    1970-1-1 0:12:37
+    test:                       transfer
+    path:                       1-4.17
+    size:                       100
+    iterations:                 10
+    errors:                     0
+    async:                      Disabled
+    requests per-sec:           min=390, max=547, average=469.299988, jitter=157
+    ap-throughput B/s:          min=90480 max=126904 average=108762.101562 jitter=36424
+    ap-latency usec:            min=1826 max=2560 average=2146.000000 jitter=734
+    apbridge-latency usec:      min=1620 max=1982 average=1882.099976 jitter=362
+    gpbridge-latency usec:      min=56 max=57 average=57.099998 jitter=1
+
+
+    1970-1-1 0:12:37
+    test:                       transfer
+    path:                       1-5.17
+    size:                       100
+    iterations:                 10
+    errors:                     0
+    async:                      Disabled
+    requests per-sec:           min=397, max=538, average=461.700012, jitter=141
+    ap-throughput B/s:          min=92104 max=124816 average=106998.898438 jitter=32712
+    ap-latency usec:            min=1856 max=2514 average=2185.699951 jitter=658
+    apbridge-latency usec:      min=1460 max=2296 average=1828.599976 jitter=836
+    gpbridge-latency usec:      min=56 max=57 average=57.099998 jitter=1
diff --git a/drivers/staging/greybus/tools/lbtest b/drivers/staging/greybus/tools/lbtest
new file mode 100755
index 000000000000..d7353f1a2a6f
--- /dev/null
+++ b/drivers/staging/greybus/tools/lbtest
@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2015 Google, Inc.
+# Copyright (c) 2015 Linaro, Ltd.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+# contributors may be used to endorse or promote products derived from this
+# software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import print_function
+import csv
+import datetime
+import sys
+import time
+
+dict = {'ping': '2', 'transfer': '3', 'sink': '4'}
+verbose = 1
+
+def abort():
+	sys.exit(1)
+
+def usage():
+	print('Usage: looptest TEST SIZE ITERATIONS PATH\n\n'
+	'  Run TEST for a number of ITERATIONS with operation data SIZE bytes\n'
+	'  TEST may be \'ping\' \'transfer\' or \'sink\'\n'
+	'  SIZE indicates the size of transfer <= greybus max payload bytes\n'
+	'  ITERATIONS indicates the number of times to execute TEST at SIZE bytes\n'
+	'             Note if ITERATIONS is set to zero then this utility will\n'
+	'             initiate an infinite (non terminating) test and exit\n'
+	'             without logging any metrics data\n'
+	'  PATH indicates the sysfs path for the loopback greybus entries e.g.\n'
+	'             /sys/bus/greybus/devices/endo0:1:1:1:1/\n'
+	'Examples:\n'
+	'  looptest transfer 128 10000\n'
+	'  looptest ping 0 128\n'
+	'  looptest sink 2030 32768\n'
+	.format(sys.argv[0]), file=sys.stderr)
+
+	abort()
+
+def read_sysfs_int(path):
+	try:
+		f = open(path, "r");
+		val = f.read();
+		f.close()
+		return int(val)
+	except IOError as e:
+		print("I/O error({0}): {1}".format(e.errno, e.strerror))
+		print("Invalid path %s" % path)
+
+def write_sysfs_val(path, val):
+	try:
+		f = open(path, "r+")
+		f.write(val)
+		f.close()
+	except IOError as e:
+		print("I/O error({0}): {1}".format(e.errno, e.strerror))
+		print("Invalid path %s" % path)
+
+def log_csv(test_name, size, iteration_max, sys_pfx):
+	# file name will test_name_size_iteration_max.csv
+	# every time the same test with the same parameters is run we will then
+	# append to the same CSV with datestamp - representing each test dataset
+	fname = test_name + '_' + size + '_' + str(iteration_max) + '.csv'
+
+	try:
+		# gather data set
+		date = str(datetime.datetime.now())
+		error = read_sysfs_int(sys_pfx + 'error')
+		request_min = read_sysfs_int(sys_pfx + 'requests_per_second_min')
+		request_max = read_sysfs_int(sys_pfx + 'requests_per_second_max')
+		request_avg = read_sysfs_int(sys_pfx + 'requests_per_second_avg')
+		latency_min = read_sysfs_int(sys_pfx + 'latency_min')
+		latency_max = read_sysfs_int(sys_pfx + 'latency_max')
+		latency_avg = read_sysfs_int(sys_pfx + 'latency_avg')
+		throughput_min = read_sysfs_int(sys_pfx + 'throughput_min')
+		throughput_max = read_sysfs_int(sys_pfx + 'throughput_max')
+		throughput_avg = read_sysfs_int(sys_pfx + 'throughput_avg')
+
+		# derive jitter
+		request_jitter = request_max - request_min
+		latency_jitter = latency_max - latency_min
+		throughput_jitter = throughput_max - throughput_min
+
+		# append data set to file
+		with open(fname, 'a') as csvf:
+			row = csv.writer(csvf, delimiter=",", quotechar="'",
+					quoting=csv.QUOTE_MINIMAL)
+			row.writerow([date, test_name, size, iteration_max, error,
+					request_min, request_max, request_avg, request_jitter,
+					latency_min, latency_max, latency_avg, latency_jitter,
+					throughput_min, throughput_max, throughput_avg, throughput_jitter])
+	except IOError as e:
+		print("I/O error({0}): {1}".format(e.errno, e.strerror))
+
+def loopback_run(test_name, size, iteration_max, sys_pfx):
+	test_id = dict[test_name]
+	try:
+		# Terminate any currently running test
+		write_sysfs_val(sys_pfx + 'type', '0')
+		# Set parameter for no wait between messages
+		write_sysfs_val(sys_pfx + 'ms_wait', '0')
+		# Set operation size
+		write_sysfs_val(sys_pfx + 'size', size)
+		# Set iterations
+		write_sysfs_val(sys_pfx + 'iteration_max', str(iteration_max))
+		# Initiate by setting loopback operation type
+		write_sysfs_val(sys_pfx + 'type', test_id)
+		time.sleep(1)
+
+		if iteration_max == 0:
+			print ("Infinite test initiated CSV won't be logged\n")
+			return
+
+		previous = 0
+		err = 0
+		while True:
+			# get current count bail out if it hasn't changed
+			iteration_count = read_sysfs_int(sys_pfx + 'iteration_count')
+			if previous == iteration_count:
+				err = 1
+				break
+			elif iteration_count == iteration_max:
+				break
+			previous = iteration_count
+			if verbose:
+				print('%02d%% complete %d of %d ' %
+					(100 * iteration_count / iteration_max,
+					iteration_count, iteration_max))
+			time.sleep(1)
+		if err:
+			print ('\nError executing test\n')
+		else:
+			log_csv(test_name, size, iteration_max, sys_pfx)
+	except ValueError as ve:
+		print("Error: %s " % format(e.strerror), file=sys.stderr)
+		abort()
+
+def main():
+	if len(sys.argv) < 5:
+		usage()
+
+	if sys.argv[1] in dict.keys():
+		loopback_run(sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4])
+	else:
+		usage()
+if __name__ == '__main__':
+	main()
diff --git a/drivers/staging/greybus/tools/loopback_test.c b/drivers/staging/greybus/tools/loopback_test.c
new file mode 100644
index 000000000000..f7f4cd6fb55b
--- /dev/null
+++ b/drivers/staging/greybus/tools/loopback_test.c
@@ -0,0 +1,1000 @@
+/*
+ * Loopback test application
+ *
+ * Copyright 2015 Google Inc.
+ * Copyright 2015 Linaro Ltd.
+ *
+ * Provided under the three clause BSD license found in the LICENSE file.
+ */
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <poll.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <signal.h>
+
+#define MAX_NUM_DEVICES 10
+#define MAX_SYSFS_PATH	0x200
+#define CSV_MAX_LINE	0x1000
+#define SYSFS_MAX_INT	0x20
+#define MAX_STR_LEN	255
+#define DEFAULT_ASYNC_TIMEOUT 200000
+
+struct dict {
+	char *name;
+	int type;
+};
+
+static struct dict dict[] = {
+	{"ping", 2},
+	{"transfer", 3},
+	{"sink", 4},
+	{NULL,}		/* list termination */
+};
+
+struct loopback_results {
+	float latency_avg;
+	uint32_t latency_max;
+	uint32_t latency_min;
+	uint32_t latency_jitter;
+
+	float request_avg;
+	uint32_t request_max;
+	uint32_t request_min;
+	uint32_t request_jitter;
+
+	float throughput_avg;
+	uint32_t throughput_max;
+	uint32_t throughput_min;
+	uint32_t throughput_jitter;
+
+	float apbridge_unipro_latency_avg;
+	uint32_t apbridge_unipro_latency_max;
+	uint32_t apbridge_unipro_latency_min;
+	uint32_t apbridge_unipro_latency_jitter;
+
+	float gbphy_firmware_latency_avg;
+	uint32_t gbphy_firmware_latency_max;
+	uint32_t gbphy_firmware_latency_min;
+	uint32_t gbphy_firmware_latency_jitter;
+
+	uint32_t error;
+};
+
+struct loopback_device {
+	char name[MAX_SYSFS_PATH];
+	char sysfs_entry[MAX_SYSFS_PATH];
+	char debugfs_entry[MAX_SYSFS_PATH];
+	struct loopback_results results;
+};
+
+struct loopback_test {
+	int verbose;
+	int debug;
+	int raw_data_dump;
+	int porcelain;
+	int mask;
+	int size;
+	int iteration_max;
+	int aggregate_output;
+	int test_id;
+	int device_count;
+	int list_devices;
+	int use_async;
+	int async_timeout;
+	int async_outstanding_operations;
+	int us_wait;
+	int file_output;
+	int stop_all;
+	int poll_count;
+	char test_name[MAX_STR_LEN];
+	char sysfs_prefix[MAX_SYSFS_PATH];
+	char debugfs_prefix[MAX_SYSFS_PATH];
+	struct timespec poll_timeout;
+	struct loopback_device devices[MAX_NUM_DEVICES];
+	struct loopback_results aggregate_results;
+	struct pollfd fds[MAX_NUM_DEVICES];
+};
+
+struct loopback_test t;
+
+/* Helper macros to calculate the aggregate results for all devices */
+static inline int device_enabled(struct loopback_test *t, int dev_idx);
+
+#define GET_MAX(field)							\
+static int get_##field##_aggregate(struct loopback_test *t)		\
+{									\
+	uint32_t max = 0;						\
+	int i;								\
+	for (i = 0; i < t->device_count; i++) {				\
+		if (!device_enabled(t, i))				\
+			continue;					\
+		if (t->devices[i].results.field > max)			\
+			max = t->devices[i].results.field;		\
+	}								\
+	return max;							\
+}									\
+
+#define GET_MIN(field)							\
+static int get_##field##_aggregate(struct loopback_test *t)		\
+{									\
+	uint32_t min = ~0;						\
+	int i;								\
+	for (i = 0; i < t->device_count; i++) {				\
+		if (!device_enabled(t, i))				\
+			continue;					\
+		if (t->devices[i].results.field < min)			\
+			min = t->devices[i].results.field;		\
+	}								\
+	return min;							\
+}									\
+
+#define GET_AVG(field)							\
+static int get_##field##_aggregate(struct loopback_test *t)		\
+{									\
+	uint32_t val = 0;						\
+	uint32_t count = 0;						\
+	int i;								\
+	for (i = 0; i < t->device_count; i++) {				\
+		if (!device_enabled(t, i))				\
+			continue;					\
+		count++;						\
+		val += t->devices[i].results.field;			\
+	}								\
+	if (count)							\
+		val /= count;						\
+	return val;							\
+}									\
+
+GET_MAX(throughput_max);
+GET_MAX(request_max);
+GET_MAX(latency_max);
+GET_MAX(apbridge_unipro_latency_max);
+GET_MAX(gbphy_firmware_latency_max);
+GET_MIN(throughput_min);
+GET_MIN(request_min);
+GET_MIN(latency_min);
+GET_MIN(apbridge_unipro_latency_min);
+GET_MIN(gbphy_firmware_latency_min);
+GET_AVG(throughput_avg);
+GET_AVG(request_avg);
+GET_AVG(latency_avg);
+GET_AVG(apbridge_unipro_latency_avg);
+GET_AVG(gbphy_firmware_latency_avg);
+
+void abort()
+{
+	_exit(1);
+}
+
+void usage(void)
+{
+	fprintf(stderr, "Usage: loopback_test TEST [SIZE] ITERATIONS [SYSPATH] [DBGPATH]\n\n"
+	"  Run TEST for a number of ITERATIONS with operation data SIZE bytes\n"
+	"  TEST may be \'ping\' \'transfer\' or \'sink\'\n"
+	"  SIZE indicates the size of transfer <= greybus max payload bytes\n"
+	"  ITERATIONS indicates the number of times to execute TEST at SIZE bytes\n"
+	"             Note if ITERATIONS is set to zero then this utility will\n"
+	"             initiate an infinite (non terminating) test and exit\n"
+	"             without logging any metrics data\n"
+	"  SYSPATH indicates the sysfs path for the loopback greybus entries e.g.\n"
+	"          /sys/bus/greybus/devices\n"
+	"  DBGPATH indicates the debugfs path for the loopback greybus entries e.g.\n"
+	"          /sys/kernel/debug/gb_loopback/\n"
+	" Mandatory arguments\n"
+	"   -t     must be one of the test names - sink, transfer or ping\n"
+	"   -i     iteration count - the number of iterations to run the test over\n"
+	" Optional arguments\n"
+	"   -S     sysfs location - location for greybus 'endo' entires default /sys/bus/greybus/devices/\n"
+	"   -D     debugfs location - location for loopback debugfs entries default /sys/kernel/debug/gb_loopback/\n"
+	"   -s     size of data packet to send during test - defaults to zero\n"
+	"   -m     mask - a bit mask of connections to include example: -m 8 = 4th connection -m 9 = 1st and 4th connection etc\n"
+	"                 default is zero which means broadcast to all connections\n"
+	"   -v     verbose output\n"
+	"   -d     debug output\n"
+	"   -r     raw data output - when specified the full list of latency values are included in the output CSV\n"
+	"   -p     porcelain - when specified printout is in a user-friendly non-CSV format. This option suppresses writing to CSV file\n"
+	"   -a     aggregate - show aggregation of all enabled devices\n"
+	"   -l     list found loopback devices and exit\n"
+	"   -x     Async - Enable async transfers\n"
+	"   -o     Async Timeout - Timeout in uSec for async operations\n"
+	"   -O     Poll loop time out in seconds(max time a test is expected to last, default: 30sec)\n"
+	"   -c     Max number of outstanding operations for async operations\n"
+	"   -w     Wait in uSec between operations\n"
+	"   -z     Enable output to a CSV file (incompatible with -p)\n"
+	"   -f     When starting new loopback test, stop currently running tests on all devices\n"
+	"Examples:\n"
+	"  Send 10000 transfers with a packet size of 128 bytes to all active connections\n"
+	"  loopback_test -t transfer -s 128 -i 10000 -S /sys/bus/greybus/devices/ -D /sys/kernel/debug/gb_loopback/\n"
+	"  loopback_test -t transfer -s 128 -i 10000 -m 0\n"
+	"  Send 10000 transfers with a packet size of 128 bytes to connection 1 and 4\n"
+	"  loopback_test -t transfer -s 128 -i 10000 -m 9\n"
+	"  loopback_test -t ping -s 0 128 -i -S /sys/bus/greybus/devices/ -D /sys/kernel/debug/gb_loopback/\n"
+	"  loopback_test -t sink -s 2030 -i 32768 -S /sys/bus/greybus/devices/ -D /sys/kernel/debug/gb_loopback/\n");
+	abort();
+}
+
+static inline int device_enabled(struct loopback_test *t, int dev_idx)
+{
+	if (!t->mask || (t->mask & (1 << dev_idx)))
+		return 1;
+
+	return 0;
+}
+
+static void show_loopback_devices(struct loopback_test *t)
+{
+	int i;
+
+	if (t->device_count == 0) {
+		printf("No loopback devices.\n");
+		return;
+	}
+
+	for (i = 0; i < t->device_count; i++)
+		printf("device[%d] = %s\n", i, t->devices[i].name);
+
+}
+
+int open_sysfs(const char *sys_pfx, const char *node, int flags)
+{
+	int fd;
+	char path[MAX_SYSFS_PATH];
+
+	snprintf(path, sizeof(path), "%s%s", sys_pfx, node);
+	fd = open(path, flags);
+	if (fd < 0) {
+		fprintf(stderr, "unable to open %s\n", path);
+		abort();
+	}
+	return fd;
+}
+
+int read_sysfs_int_fd(int fd, const char *sys_pfx, const char *node)
+{
+	char buf[SYSFS_MAX_INT];
+
+	if (read(fd, buf, sizeof(buf)) < 0) {
+		fprintf(stderr, "unable to read from %s%s %s\n", sys_pfx, node,
+			strerror(errno));
+		close(fd);
+		abort();
+	}
+	return atoi(buf);
+}
+
+float read_sysfs_float_fd(int fd, const char *sys_pfx, const char *node)
+{
+	char buf[SYSFS_MAX_INT];
+
+	if (read(fd, buf, sizeof(buf)) < 0) {
+
+		fprintf(stderr, "unable to read from %s%s %s\n", sys_pfx, node,
+			strerror(errno));
+		close(fd);
+		abort();
+	}
+	return atof(buf);
+}
+
+int read_sysfs_int(const char *sys_pfx, const char *node)
+{
+	int fd, val;
+
+	fd = open_sysfs(sys_pfx, node, O_RDONLY);
+	val = read_sysfs_int_fd(fd, sys_pfx, node);
+	close(fd);
+	return val;
+}
+
+float read_sysfs_float(const char *sys_pfx, const char *node)
+{
+	int fd;
+	float val;
+
+	fd = open_sysfs(sys_pfx, node, O_RDONLY);
+	val = read_sysfs_float_fd(fd, sys_pfx, node);
+	close(fd);
+	return val;
+}
+
+void write_sysfs_val(const char *sys_pfx, const char *node, int val)
+{
+	int fd, len;
+	char buf[SYSFS_MAX_INT];
+
+	fd = open_sysfs(sys_pfx, node, O_RDWR);
+	len = snprintf(buf, sizeof(buf), "%d", val);
+	if (write(fd, buf, len) < 0) {
+		fprintf(stderr, "unable to write to %s%s %s\n", sys_pfx, node,
+			strerror(errno));
+		close(fd);
+		abort();
+	}
+	close(fd);
+}
+
+static int get_results(struct loopback_test *t)
+{
+	struct loopback_device *d;
+	struct loopback_results *r;
+	int i;
+
+	for (i = 0; i < t->device_count; i++) {
+		if (!device_enabled(t, i))
+			continue;
+
+		d = &t->devices[i];
+		r = &d->results;
+
+		r->error = read_sysfs_int(d->sysfs_entry, "error");
+		r->request_min = read_sysfs_int(d->sysfs_entry, "requests_per_second_min");
+		r->request_max = read_sysfs_int(d->sysfs_entry, "requests_per_second_max");
+		r->request_avg = read_sysfs_float(d->sysfs_entry, "requests_per_second_avg");
+
+		r->latency_min = read_sysfs_int(d->sysfs_entry, "latency_min");
+		r->latency_max = read_sysfs_int(d->sysfs_entry, "latency_max");
+		r->latency_avg = read_sysfs_float(d->sysfs_entry, "latency_avg");
+
+		r->throughput_min = read_sysfs_int(d->sysfs_entry, "throughput_min");
+		r->throughput_max = read_sysfs_int(d->sysfs_entry, "throughput_max");
+		r->throughput_avg = read_sysfs_float(d->sysfs_entry, "throughput_avg");
+
+		r->apbridge_unipro_latency_min =
+			read_sysfs_int(d->sysfs_entry, "apbridge_unipro_latency_min");
+		r->apbridge_unipro_latency_max =
+			read_sysfs_int(d->sysfs_entry, "apbridge_unipro_latency_max");
+		r->apbridge_unipro_latency_avg =
+			read_sysfs_float(d->sysfs_entry, "apbridge_unipro_latency_avg");
+
+		r->gbphy_firmware_latency_min =
+			read_sysfs_int(d->sysfs_entry, "gbphy_firmware_latency_min");
+		r->gbphy_firmware_latency_max =
+			read_sysfs_int(d->sysfs_entry, "gbphy_firmware_latency_max");
+		r->gbphy_firmware_latency_avg =
+			read_sysfs_float(d->sysfs_entry, "gbphy_firmware_latency_avg");
+
+		r->request_jitter = r->request_max - r->request_min;
+		r->latency_jitter = r->latency_max - r->latency_min;
+		r->throughput_jitter = r->throughput_max - r->throughput_min;
+		r->apbridge_unipro_latency_jitter =
+			r->apbridge_unipro_latency_max - r->apbridge_unipro_latency_min;
+		r->gbphy_firmware_latency_jitter =
+			r->gbphy_firmware_latency_max - r->gbphy_firmware_latency_min;
+
+	}
+
+	/*calculate the aggregate results of all enabled devices */
+	if (t->aggregate_output) {
+		r = &t->aggregate_results;
+
+		r->request_min = get_request_min_aggregate(t);
+		r->request_max = get_request_max_aggregate(t);
+		r->request_avg = get_request_avg_aggregate(t);
+
+		r->latency_min = get_latency_min_aggregate(t);
+		r->latency_max = get_latency_max_aggregate(t);
+		r->latency_avg = get_latency_avg_aggregate(t);
+
+		r->throughput_min = get_throughput_min_aggregate(t);
+		r->throughput_max = get_throughput_max_aggregate(t);
+		r->throughput_avg = get_throughput_avg_aggregate(t);
+
+		r->apbridge_unipro_latency_min =
+			get_apbridge_unipro_latency_min_aggregate(t);
+		r->apbridge_unipro_latency_max =
+			get_apbridge_unipro_latency_max_aggregate(t);
+		r->apbridge_unipro_latency_avg =
+			get_apbridge_unipro_latency_avg_aggregate(t);
+
+		r->gbphy_firmware_latency_min =
+			get_gbphy_firmware_latency_min_aggregate(t);
+		r->gbphy_firmware_latency_max =
+			get_gbphy_firmware_latency_max_aggregate(t);
+		r->gbphy_firmware_latency_avg =
+			get_gbphy_firmware_latency_avg_aggregate(t);
+
+		r->request_jitter = r->request_max - r->request_min;
+		r->latency_jitter = r->latency_max - r->latency_min;
+		r->throughput_jitter = r->throughput_max - r->throughput_min;
+		r->apbridge_unipro_latency_jitter =
+			r->apbridge_unipro_latency_max - r->apbridge_unipro_latency_min;
+		r->gbphy_firmware_latency_jitter =
+			r->gbphy_firmware_latency_max - r->gbphy_firmware_latency_min;
+
+	}
+
+	return 0;
+}
+
+void log_csv_error(int len, int err)
+{
+	fprintf(stderr, "unable to write %d bytes to csv %s\n", len,
+		strerror(err));
+}
+
+int format_output(struct loopback_test *t,
+			struct loopback_results *r,
+			const char *dev_name,
+			char *buf, int buf_len,
+			struct tm *tm)
+{
+	int len = 0;
+
+	memset(buf, 0x00, buf_len);
+	len = snprintf(buf, buf_len, "%u-%u-%u %u:%u:%u",
+		       tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday,
+		       tm->tm_hour, tm->tm_min, tm->tm_sec);
+
+	if (t->porcelain) {
+		len += snprintf(&buf[len], buf_len - len,
+			"\n test:\t\t\t%s\n path:\t\t\t%s\n size:\t\t\t%u\n iterations:\t\t%u\n errors:\t\t%u\n async:\t\t\t%s\n",
+			t->test_name,
+			dev_name,
+			t->size,
+			t->iteration_max,
+			r->error,
+			t->use_async ? "Enabled" : "Disabled");
+
+		len += snprintf(&buf[len], buf_len - len,
+			" requests per-sec:\tmin=%u, max=%u, average=%f, jitter=%u\n",
+			r->request_min,
+			r->request_max,
+			r->request_avg,
+			r->request_jitter);
+
+		len += snprintf(&buf[len], buf_len - len,
+			" ap-throughput B/s:\tmin=%u max=%u average=%f jitter=%u\n",
+			r->throughput_min,
+			r->throughput_max,
+			r->throughput_avg,
+			r->throughput_jitter);
+		len += snprintf(&buf[len], buf_len - len,
+			" ap-latency usec:\tmin=%u max=%u average=%f jitter=%u\n",
+			r->latency_min,
+			r->latency_max,
+			r->latency_avg,
+			r->latency_jitter);
+		len += snprintf(&buf[len], buf_len - len,
+			" apbridge-latency usec:\tmin=%u max=%u average=%f jitter=%u\n",
+			r->apbridge_unipro_latency_min,
+			r->apbridge_unipro_latency_max,
+			r->apbridge_unipro_latency_avg,
+			r->apbridge_unipro_latency_jitter);
+
+		len += snprintf(&buf[len], buf_len - len,
+			" gbphy-latency usec:\tmin=%u max=%u average=%f jitter=%u\n",
+			r->gbphy_firmware_latency_min,
+			r->gbphy_firmware_latency_max,
+			r->gbphy_firmware_latency_avg,
+			r->gbphy_firmware_latency_jitter);
+
+	} else {
+		len += snprintf(&buf[len], buf_len- len, ",%s,%s,%u,%u,%u",
+			t->test_name, dev_name, t->size, t->iteration_max,
+			r->error);
+
+		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
+			r->request_min,
+			r->request_max,
+			r->request_avg,
+			r->request_jitter);
+
+		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
+			r->latency_min,
+			r->latency_max,
+			r->latency_avg,
+			r->latency_jitter);
+
+		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
+			r->throughput_min,
+			r->throughput_max,
+			r->throughput_avg,
+			r->throughput_jitter);
+
+		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
+			r->apbridge_unipro_latency_min,
+			r->apbridge_unipro_latency_max,
+			r->apbridge_unipro_latency_avg,
+			r->apbridge_unipro_latency_jitter);
+
+		len += snprintf(&buf[len], buf_len - len, ",%u,%u,%f,%u",
+			r->gbphy_firmware_latency_min,
+			r->gbphy_firmware_latency_max,
+			r->gbphy_firmware_latency_avg,
+			r->gbphy_firmware_latency_jitter);
+	}
+
+	printf("\n%s\n", buf);
+
+	return len;
+}
+
+static int log_results(struct loopback_test *t)
+{
+	int fd, i, len, ret;
+	struct tm tm;
+	time_t local_time;
+	mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
+	char file_name[MAX_SYSFS_PATH];
+	char data[CSV_MAX_LINE];
+
+	local_time = time(NULL);
+	tm = *localtime(&local_time);
+
+	/*
+	* file name will test_name_size_iteration_max.csv
+	* every time the same test with the same parameters is run we will then
+	* append to the same CSV with datestamp - representing each test
+	* dataset.
+	*/
+	if (t->file_output && !t->porcelain) {
+		snprintf(file_name, sizeof(file_name), "%s_%d_%d.csv",
+			t->test_name, t->size, t->iteration_max);
+
+		fd = open(file_name, O_WRONLY | O_CREAT | O_APPEND, mode);
+		if (fd < 0) {
+			fprintf(stderr, "unable to open %s for appendation\n", file_name);
+			abort();
+		}
+
+	}
+	for (i = 0; i < t->device_count; i++) {
+		if (!device_enabled(t, i))
+			continue;
+
+		len = format_output(t, &t->devices[i].results,
+					t->devices[i].name,
+					data, sizeof(data), &tm);
+		if (t->file_output && !t->porcelain) {
+			ret = write(fd, data, len);
+			if (ret == -1)
+				fprintf(stderr, "unable to write %d bytes to csv.\n", len);
+		}
+
+	}
+
+
+	if (t->aggregate_output) {
+		len = format_output(t, &t->aggregate_results, "aggregate",
+					data, sizeof(data), &tm);
+		if (t->file_output && !t->porcelain) {
+			ret = write(fd, data, len);
+			if (ret == -1)
+				fprintf(stderr, "unable to write %d bytes to csv.\n", len);
+		}
+	}
+
+	if (t->file_output && !t->porcelain)
+		close(fd);
+
+	return 0;
+}
+
+int is_loopback_device(const char *path, const char *node)
+{
+	char file[MAX_SYSFS_PATH];
+
+	snprintf(file, MAX_SYSFS_PATH, "%s%s/iteration_count", path, node);
+	if (access(file, F_OK) == 0)
+		return 1;
+	return 0;
+}
+
+int find_loopback_devices(struct loopback_test *t)
+{
+	struct dirent **namelist;
+	int i, n, ret;
+	unsigned int dev_id;
+	struct loopback_device *d;
+
+	n = scandir(t->sysfs_prefix, &namelist, NULL, alphasort);
+	if (n < 0) {
+		perror("scandir");
+		ret = -ENODEV;
+		goto baddir;
+	}
+
+	/* Don't include '.' and '..' */
+	if (n <= 2) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	for (i = 0; i < n; i++) {
+		ret = sscanf(namelist[i]->d_name, "gb_loopback%u", &dev_id);
+		if (ret != 1)
+			continue;
+
+		if (!is_loopback_device(t->sysfs_prefix, namelist[i]->d_name))
+			continue;
+
+		if (t->device_count == MAX_NUM_DEVICES) {
+			fprintf(stderr, "max number of devices reached!\n");
+			break;
+		}
+
+		d = &t->devices[t->device_count++];
+		snprintf(d->name, MAX_STR_LEN, "gb_loopback%u", dev_id);
+
+		snprintf(d->sysfs_entry, MAX_SYSFS_PATH, "%s%s/",
+			t->sysfs_prefix, d->name);
+
+		snprintf(d->debugfs_entry, MAX_SYSFS_PATH, "%sraw_latency_%s",
+			t->debugfs_prefix, d->name);
+
+		if (t->debug)
+			printf("add %s %s\n", d->sysfs_entry,
+				d->debugfs_entry);
+	}
+
+	ret = 0;
+done:
+	for (i = 0; i < n; i++)
+		free(namelist[n]);
+	free(namelist);
+baddir:
+	return ret;
+}
+
+static int open_poll_files(struct loopback_test *t)
+{
+	struct loopback_device *dev;
+	char buf[MAX_STR_LEN];
+	char dummy;
+	int fds_idx = 0;
+	int i;
+
+	for (i = 0; i < t->device_count; i++) {
+		dev = &t->devices[i];
+
+		if (!device_enabled(t, i))
+			continue;
+
+		snprintf(buf, sizeof(buf), "%s%s", dev->sysfs_entry, "iteration_count");
+		t->fds[fds_idx].fd = open(buf, O_RDONLY);
+		if (t->fds[fds_idx].fd < 0) {
+			fprintf(stderr, "Error opening poll file!\n");
+			goto err;
+		}
+		read(t->fds[fds_idx].fd, &dummy, 1);
+		t->fds[fds_idx].events = POLLERR|POLLPRI;
+		t->fds[fds_idx].revents = 0;
+		fds_idx++;
+	}
+
+	t->poll_count = fds_idx;
+
+	return 0;
+
+err:
+	for (i = 0; i < fds_idx; i++)
+		close(t->fds[fds_idx].fd);
+
+	return -1;
+}
+
+static int close_poll_files(struct loopback_test *t)
+{
+	int i;
+	for (i = 0; i < t->poll_count; i++)
+		close(t->fds[i].fd);
+
+	return 0;
+}
+static int is_complete(struct loopback_test *t)
+{
+	int iteration_count;
+	int i;
+
+	for (i = 0; i < t->device_count; i++) {
+		if (!device_enabled(t, i))
+			continue;
+
+		iteration_count = read_sysfs_int(t->devices[i].sysfs_entry,
+						 "iteration_count");
+
+		/* at least one device did not finish yet */
+		if (iteration_count != t->iteration_max)
+			return 0;
+	}
+
+	return 1;
+}
+
+static void stop_tests(struct loopback_test *t)
+{
+	int i;
+
+	for (i = 0; i < t->device_count; i++) {
+		if (!device_enabled(t, i))
+			continue;
+		write_sysfs_val(t->devices[i].sysfs_entry, "type", 0);
+	}
+}
+
+static void handler(int sig) { /* do nothing */  }
+
+static int wait_for_complete(struct loopback_test *t)
+{
+	int number_of_events = 0;
+	char dummy;
+	int ret;
+	int i;
+	struct timespec *ts = NULL;
+	struct sigaction sa;
+	sigset_t mask_old, mask;
+
+	sigemptyset(&mask);
+	sigemptyset(&mask_old);
+	sigaddset(&mask, SIGINT);
+	sigprocmask(SIG_BLOCK, &mask, &mask_old);
+
+	sa.sa_handler = handler;
+	sa.sa_flags = 0;
+	sigemptyset(&sa.sa_mask);
+	if (sigaction(SIGINT, &sa, NULL) == -1) {
+		fprintf(stderr, "sigaction error\n");
+		return -1;
+	}
+
+	if (t->poll_timeout.tv_sec != 0)
+		ts = &t->poll_timeout;
+
+	while (1) {
+
+		ret = ppoll(t->fds, t->poll_count, ts, &mask_old);
+		if (ret <= 0) {
+			stop_tests(t);
+			fprintf(stderr, "Poll exit with errno %d\n", errno);
+			return -1;
+		}
+
+		for (i = 0; i < t->poll_count; i++) {
+			if (t->fds[i].revents & POLLPRI) {
+				/* Dummy read to clear the event */
+				read(t->fds[i].fd, &dummy, 1);
+				number_of_events++;
+			}
+		}
+
+		if (number_of_events == t->poll_count)
+			break;
+	}
+
+	if (!is_complete(t)) {
+		fprintf(stderr, "Iteration count did not finish!\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static void prepare_devices(struct loopback_test *t)
+{
+	int i;
+
+	/* Cancel any running tests on enabled devices. If
+	 * stop_all option is given, stop test on all devices.
+	 */
+	for (i = 0; i < t->device_count; i++)
+		if (t->stop_all || device_enabled(t, i))
+			write_sysfs_val(t->devices[i].sysfs_entry, "type", 0);
+
+
+	for (i = 0; i < t->device_count; i++) {
+		if (!device_enabled(t, i))
+			continue;
+
+		write_sysfs_val(t->devices[i].sysfs_entry, "us_wait",
+				t->us_wait);
+
+		/* Set operation size */
+		write_sysfs_val(t->devices[i].sysfs_entry, "size", t->size);
+
+		/* Set iterations */
+		write_sysfs_val(t->devices[i].sysfs_entry, "iteration_max",
+				t->iteration_max);
+
+		if (t->use_async) {
+			write_sysfs_val(t->devices[i].sysfs_entry,
+				"async", 1);
+			write_sysfs_val(t->devices[i].sysfs_entry,
+				"timeout", t->async_timeout);
+			write_sysfs_val(t->devices[i].sysfs_entry,
+				"outstanding_operations_max",
+				t->async_outstanding_operations);
+		} else
+			write_sysfs_val(t->devices[i].sysfs_entry,
+				"async", 0);
+	}
+}
+
+static int start(struct loopback_test *t)
+{
+	int i;
+
+	/* the test starts by writing test_id to the type file. */
+	for (i = 0; i < t->device_count; i++) {
+		if (!device_enabled(t, i))
+			continue;
+
+		write_sysfs_val(t->devices[i].sysfs_entry, "type", t->test_id);
+	}
+
+	return 0;
+}
+
+
+void loopback_run(struct loopback_test *t)
+{
+	int i;
+	int ret;
+
+	for (i = 0; dict[i].name != NULL; i++) {
+		if (strstr(dict[i].name, t->test_name))
+			t->test_id = dict[i].type;
+	}
+	if (!t->test_id) {
+		fprintf(stderr, "invalid test %s\n", t->test_name);
+		usage();
+		return;
+	}
+
+	prepare_devices(t);
+
+	ret = open_poll_files(t);
+	if (ret)
+		goto err;
+
+	start(t);
+
+	ret = wait_for_complete(t);
+	close_poll_files(t);
+	if (ret)
+		goto err;
+
+
+	get_results(t);
+
+	log_results(t);
+
+	return;
+
+err:
+	printf("Error running test\n");
+	return;
+}
+
+static int sanity_check(struct loopback_test *t)
+{
+	int i;
+
+	if (t->device_count == 0) {
+		fprintf(stderr, "No loopback devices found\n");
+		return -1;
+	}
+
+	for (i = 0; i < MAX_NUM_DEVICES; i++) {
+		if (!device_enabled(t, i))
+			continue;
+
+		if (t->mask && !strcmp(t->devices[i].name, "")) {
+			fprintf(stderr, "Bad device mask %x\n", (1 << i));
+			return -1;
+		}
+
+	}
+
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	int o, ret;
+	char *sysfs_prefix = "/sys/class/gb_loopback/";
+	char *debugfs_prefix = "/sys/kernel/debug/gb_loopback/";
+
+	memset(&t, 0, sizeof(t));
+
+	while ((o = getopt(argc, argv,
+			   "t:s:i:S:D:m:v::d::r::p::a::l::x::o:O:c:w:z::f::")) != -1) {
+		switch (o) {
+		case 't':
+			snprintf(t.test_name, MAX_STR_LEN, "%s", optarg);
+			break;
+		case 's':
+			t.size = atoi(optarg);
+			break;
+		case 'i':
+			t.iteration_max = atoi(optarg);
+			break;
+		case 'S':
+			snprintf(t.sysfs_prefix, MAX_SYSFS_PATH, "%s", optarg);
+			break;
+		case 'D':
+			snprintf(t.debugfs_prefix, MAX_SYSFS_PATH, "%s", optarg);
+			break;
+		case 'm':
+			t.mask = atol(optarg);
+			break;
+		case 'v':
+			t.verbose = 1;
+			break;
+		case 'd':
+			t.debug = 1;
+			break;
+		case 'r':
+			t.raw_data_dump = 1;
+			break;
+		case 'p':
+			t.porcelain = 1;
+			break;
+		case 'a':
+			t.aggregate_output = 1;
+			break;
+		case 'l':
+			t.list_devices = 1;
+			break;
+		case 'x':
+			t.use_async = 1;
+			break;
+		case 'o':
+			t.async_timeout = atoi(optarg);
+			break;
+		case 'O':
+			t.poll_timeout.tv_sec = atoi(optarg);
+			break;
+		case 'c':
+			t.async_outstanding_operations = atoi(optarg);
+			break;
+		case 'w':
+			t.us_wait = atoi(optarg);
+			break;
+		case 'z':
+			t.file_output = 1;
+			break;
+		case 'f':
+			t.stop_all = 1;
+			break;
+		default:
+			usage();
+			return -EINVAL;
+		}
+	}
+
+	if (!strcmp(t.sysfs_prefix, ""))
+		snprintf(t.sysfs_prefix, MAX_SYSFS_PATH, "%s", sysfs_prefix);
+
+	if (!strcmp(t.debugfs_prefix, ""))
+		snprintf(t.debugfs_prefix, MAX_SYSFS_PATH, "%s", debugfs_prefix);
+
+	ret = find_loopback_devices(&t);
+	if (ret)
+		return ret;
+	ret = sanity_check(&t);
+	if (ret)
+		return ret;
+
+	if (t.list_devices) {
+		show_loopback_devices(&t);
+		return 0;
+	}
+
+	if (t.test_name[0] == '\0' || t.iteration_max == 0)
+		usage();
+
+	if (t.async_timeout == 0)
+		t.async_timeout = DEFAULT_ASYNC_TIMEOUT;
+
+	loopback_run(&t);
+
+	return 0;
+}
diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c
new file mode 100644
index 000000000000..5ee7954bd9f9
--- /dev/null
+++ b/drivers/staging/greybus/uart.c
@@ -0,0 +1,1075 @@
+/*
+ * UART driver for the Greybus "generic" UART module.
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ *
+ * Heavily based on drivers/usb/class/cdc-acm.c and
+ * drivers/usb/serial/usb-serial.c.
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/mutex.h>
+#include <linux/tty.h>
+#include <linux/serial.h>
+#include <linux/tty_driver.h>
+#include <linux/tty_flip.h>
+#include <linux/serial.h>
+#include <linux/idr.h>
+#include <linux/fs.h>
+#include <linux/kdev_t.h>
+#include <linux/kfifo.h>
+#include <linux/workqueue.h>
+#include <linux/completion.h>
+
+#include "greybus.h"
+#include "gbphy.h"
+
+#define GB_NUM_MINORS	16	/* 16 is is more than enough */
+#define GB_NAME		"ttyGB"
+
+#define GB_UART_WRITE_FIFO_SIZE		PAGE_SIZE
+#define GB_UART_WRITE_ROOM_MARGIN	1	/* leave some space in fifo */
+#define GB_UART_FIRMWARE_CREDITS	4096
+#define GB_UART_CREDIT_WAIT_TIMEOUT_MSEC	10000
+
+struct gb_tty_line_coding {
+	__le32	rate;
+	__u8	format;
+	__u8	parity;
+	__u8	data_bits;
+	__u8	flow_control;
+};
+
+struct gb_tty {
+	struct gbphy_device *gbphy_dev;
+	struct tty_port port;
+	void *buffer;
+	size_t buffer_payload_max;
+	struct gb_connection *connection;
+	u16 cport_id;
+	unsigned int minor;
+	unsigned char clocal;
+	bool disconnected;
+	spinlock_t read_lock;
+	spinlock_t write_lock;
+	struct async_icount iocount;
+	struct async_icount oldcount;
+	wait_queue_head_t wioctl;
+	struct mutex mutex;
+	u8 ctrlin;	/* input control lines */
+	u8 ctrlout;	/* output control lines */
+	struct gb_tty_line_coding line_coding;
+	struct work_struct tx_work;
+	struct kfifo write_fifo;
+	bool close_pending;
+	unsigned int credits;
+	struct completion credits_complete;
+};
+
+static struct tty_driver *gb_tty_driver;
+static DEFINE_IDR(tty_minors);
+static DEFINE_MUTEX(table_lock);
+
+static int gb_uart_receive_data_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_tty *gb_tty = gb_connection_get_data(connection);
+	struct tty_port *port = &gb_tty->port;
+	struct gb_message *request = op->request;
+	struct gb_uart_recv_data_request *receive_data;
+	u16 recv_data_size;
+	int count;
+	unsigned long tty_flags = TTY_NORMAL;
+
+	if (request->payload_size < sizeof(*receive_data)) {
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"short receive-data request received (%zu < %zu)\n",
+			request->payload_size, sizeof(*receive_data));
+		return -EINVAL;
+	}
+
+	receive_data = op->request->payload;
+	recv_data_size = le16_to_cpu(receive_data->size);
+
+	if (recv_data_size != request->payload_size - sizeof(*receive_data)) {
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"malformed receive-data request received (%u != %zu)\n",
+			recv_data_size,
+			request->payload_size - sizeof(*receive_data));
+		return -EINVAL;
+	}
+
+	if (!recv_data_size)
+		return -EINVAL;
+
+	if (receive_data->flags) {
+		if (receive_data->flags & GB_UART_RECV_FLAG_BREAK)
+			tty_flags = TTY_BREAK;
+		else if (receive_data->flags & GB_UART_RECV_FLAG_PARITY)
+			tty_flags = TTY_PARITY;
+		else if (receive_data->flags & GB_UART_RECV_FLAG_FRAMING)
+			tty_flags = TTY_FRAME;
+
+		/* overrun is special, not associated with a char */
+		if (receive_data->flags & GB_UART_RECV_FLAG_OVERRUN)
+			tty_insert_flip_char(port, 0, TTY_OVERRUN);
+	}
+	count = tty_insert_flip_string_fixed_flag(port, receive_data->data,
+						  tty_flags, recv_data_size);
+	if (count != recv_data_size) {
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"UART: RX 0x%08x bytes only wrote 0x%08x\n",
+			recv_data_size, count);
+	}
+	if (count)
+		tty_flip_buffer_push(port);
+	return 0;
+}
+
+static int gb_uart_serial_state_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_tty *gb_tty = gb_connection_get_data(connection);
+	struct gb_message *request = op->request;
+	struct gb_uart_serial_state_request *serial_state;
+
+	if (request->payload_size < sizeof(*serial_state)) {
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"short serial-state event received (%zu < %zu)\n",
+			request->payload_size, sizeof(*serial_state));
+		return -EINVAL;
+	}
+
+	serial_state = request->payload;
+	gb_tty->ctrlin = serial_state->control;
+
+	return 0;
+}
+
+static int gb_uart_receive_credits_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_tty *gb_tty = gb_connection_get_data(connection);
+	struct gb_message *request = op->request;
+	struct gb_uart_receive_credits_request *credit_request;
+	unsigned long flags;
+	unsigned int incoming_credits;
+	int ret = 0;
+
+	if (request->payload_size < sizeof(*credit_request)) {
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"short receive_credits event received (%zu < %zu)\n",
+			request->payload_size,
+			sizeof(*credit_request));
+		return -EINVAL;
+	}
+
+	credit_request = request->payload;
+	incoming_credits = le16_to_cpu(credit_request->count);
+
+	spin_lock_irqsave(&gb_tty->write_lock, flags);
+	gb_tty->credits += incoming_credits;
+	if (gb_tty->credits > GB_UART_FIRMWARE_CREDITS) {
+		gb_tty->credits -= incoming_credits;
+		ret = -EINVAL;
+	}
+	spin_unlock_irqrestore(&gb_tty->write_lock, flags);
+
+	if (ret) {
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"invalid number of incoming credits: %d\n",
+			incoming_credits);
+		return ret;
+	}
+
+	if (!gb_tty->close_pending)
+		schedule_work(&gb_tty->tx_work);
+
+	/*
+	 * the port the tty layer may be waiting for credits
+	 */
+	tty_port_tty_wakeup(&gb_tty->port);
+
+	if (gb_tty->credits == GB_UART_FIRMWARE_CREDITS)
+		complete(&gb_tty->credits_complete);
+
+	return ret;
+}
+
+static int gb_uart_request_handler(struct gb_operation *op)
+{
+	struct gb_connection *connection = op->connection;
+	struct gb_tty *gb_tty = gb_connection_get_data(connection);
+	int type = op->type;
+	int ret;
+
+	switch (type) {
+	case GB_UART_TYPE_RECEIVE_DATA:
+		ret = gb_uart_receive_data_handler(op);
+		break;
+	case GB_UART_TYPE_SERIAL_STATE:
+		ret = gb_uart_serial_state_handler(op);
+		break;
+	case GB_UART_TYPE_RECEIVE_CREDITS:
+		ret = gb_uart_receive_credits_handler(op);
+		break;
+	default:
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"unsupported unsolicited request: 0x%02x\n", type);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static void  gb_uart_tx_write_work(struct work_struct *work)
+{
+	struct gb_uart_send_data_request *request;
+	struct gb_tty *gb_tty;
+	unsigned long flags;
+	unsigned int send_size;
+	int ret;
+
+	gb_tty = container_of(work, struct gb_tty, tx_work);
+	request = gb_tty->buffer;
+
+	while (1) {
+		if (gb_tty->close_pending)
+			break;
+
+		spin_lock_irqsave(&gb_tty->write_lock, flags);
+		send_size = gb_tty->buffer_payload_max;
+		if (send_size > gb_tty->credits)
+			send_size = gb_tty->credits;
+
+		send_size = kfifo_out_peek(&gb_tty->write_fifo,
+					   &request->data[0],
+					   send_size);
+		if (!send_size) {
+			spin_unlock_irqrestore(&gb_tty->write_lock, flags);
+			break;
+		}
+
+		gb_tty->credits -= send_size;
+		spin_unlock_irqrestore(&gb_tty->write_lock, flags);
+
+		request->size = cpu_to_le16(send_size);
+		ret = gb_operation_sync(gb_tty->connection,
+					GB_UART_TYPE_SEND_DATA,
+					request, sizeof(*request) + send_size,
+					NULL, 0);
+		if (ret) {
+			dev_err(&gb_tty->gbphy_dev->dev,
+				"send data error: %d\n", ret);
+			spin_lock_irqsave(&gb_tty->write_lock, flags);
+			gb_tty->credits += send_size;
+			spin_unlock_irqrestore(&gb_tty->write_lock, flags);
+			if (!gb_tty->close_pending)
+				schedule_work(work);
+			return;
+		}
+
+		spin_lock_irqsave(&gb_tty->write_lock, flags);
+		ret = kfifo_out(&gb_tty->write_fifo, &request->data[0],
+				send_size);
+		spin_unlock_irqrestore(&gb_tty->write_lock, flags);
+
+		tty_port_tty_wakeup(&gb_tty->port);
+	}
+}
+
+static int send_line_coding(struct gb_tty *tty)
+{
+	struct gb_uart_set_line_coding_request request;
+
+	memcpy(&request, &tty->line_coding,
+	       sizeof(tty->line_coding));
+	return gb_operation_sync(tty->connection, GB_UART_TYPE_SET_LINE_CODING,
+				 &request, sizeof(request), NULL, 0);
+}
+
+static int send_control(struct gb_tty *gb_tty, u8 control)
+{
+	struct gb_uart_set_control_line_state_request request;
+
+	request.control = control;
+	return gb_operation_sync(gb_tty->connection,
+				 GB_UART_TYPE_SET_CONTROL_LINE_STATE,
+				 &request, sizeof(request), NULL, 0);
+}
+
+static int send_break(struct gb_tty *gb_tty, u8 state)
+{
+	struct gb_uart_set_break_request request;
+
+	if ((state != 0) && (state != 1)) {
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"invalid break state of %d\n", state);
+		return -EINVAL;
+	}
+
+	request.state = state;
+	return gb_operation_sync(gb_tty->connection, GB_UART_TYPE_SEND_BREAK,
+				 &request, sizeof(request), NULL, 0);
+}
+
+static int gb_uart_wait_for_all_credits(struct gb_tty *gb_tty)
+{
+	int ret;
+
+	if (gb_tty->credits == GB_UART_FIRMWARE_CREDITS)
+		return 0;
+
+	ret = wait_for_completion_timeout(&gb_tty->credits_complete,
+			msecs_to_jiffies(GB_UART_CREDIT_WAIT_TIMEOUT_MSEC));
+	if (!ret) {
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"time out waiting for credits\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int gb_uart_flush(struct gb_tty *gb_tty, u8 flags)
+{
+	struct gb_uart_serial_flush_request request;
+
+	request.flags = flags;
+	return gb_operation_sync(gb_tty->connection, GB_UART_TYPE_FLUSH_FIFOS,
+				 &request, sizeof(request), NULL, 0);
+}
+
+static struct gb_tty *get_gb_by_minor(unsigned int minor)
+{
+	struct gb_tty *gb_tty;
+
+	mutex_lock(&table_lock);
+	gb_tty = idr_find(&tty_minors, minor);
+	if (gb_tty) {
+		mutex_lock(&gb_tty->mutex);
+		if (gb_tty->disconnected) {
+			mutex_unlock(&gb_tty->mutex);
+			gb_tty = NULL;
+		} else {
+			tty_port_get(&gb_tty->port);
+			mutex_unlock(&gb_tty->mutex);
+		}
+	}
+	mutex_unlock(&table_lock);
+	return gb_tty;
+}
+
+static int alloc_minor(struct gb_tty *gb_tty)
+{
+	int minor;
+
+	mutex_lock(&table_lock);
+	minor = idr_alloc(&tty_minors, gb_tty, 0, GB_NUM_MINORS, GFP_KERNEL);
+	mutex_unlock(&table_lock);
+	if (minor >= 0)
+		gb_tty->minor = minor;
+	return minor;
+}
+
+static void release_minor(struct gb_tty *gb_tty)
+{
+	int minor = gb_tty->minor;
+
+	gb_tty->minor = 0;	/* Maybe should use an invalid value instead */
+	mutex_lock(&table_lock);
+	idr_remove(&tty_minors, minor);
+	mutex_unlock(&table_lock);
+}
+
+static int gb_tty_install(struct tty_driver *driver, struct tty_struct *tty)
+{
+	struct gb_tty *gb_tty;
+	int retval;
+
+	gb_tty = get_gb_by_minor(tty->index);
+	if (!gb_tty)
+		return -ENODEV;
+
+	retval = tty_standard_install(driver, tty);
+	if (retval)
+		goto error;
+
+	tty->driver_data = gb_tty;
+	return 0;
+error:
+	tty_port_put(&gb_tty->port);
+	return retval;
+}
+
+static int gb_tty_open(struct tty_struct *tty, struct file *file)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+
+	return tty_port_open(&gb_tty->port, tty, file);
+}
+
+static void gb_tty_close(struct tty_struct *tty, struct file *file)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+
+	tty_port_close(&gb_tty->port, tty, file);
+}
+
+static void gb_tty_cleanup(struct tty_struct *tty)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+
+	tty_port_put(&gb_tty->port);
+}
+
+static void gb_tty_hangup(struct tty_struct *tty)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+
+	tty_port_hangup(&gb_tty->port);
+}
+
+static int gb_tty_write(struct tty_struct *tty, const unsigned char *buf,
+			int count)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+
+	count =  kfifo_in_spinlocked(&gb_tty->write_fifo, buf, count,
+				     &gb_tty->write_lock);
+	if (count && !gb_tty->close_pending)
+		schedule_work(&gb_tty->tx_work);
+
+	return count;
+}
+
+static int gb_tty_write_room(struct tty_struct *tty)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+	unsigned long flags;
+	int room;
+
+	spin_lock_irqsave(&gb_tty->write_lock, flags);
+	room = kfifo_avail(&gb_tty->write_fifo);
+	spin_unlock_irqrestore(&gb_tty->write_lock, flags);
+
+	room -= GB_UART_WRITE_ROOM_MARGIN;
+	if (room < 0)
+		return 0;
+
+	return room;
+}
+
+static int gb_tty_chars_in_buffer(struct tty_struct *tty)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+	unsigned long flags;
+	int chars;
+
+	spin_lock_irqsave(&gb_tty->write_lock, flags);
+	chars = kfifo_len(&gb_tty->write_fifo);
+	if (gb_tty->credits < GB_UART_FIRMWARE_CREDITS)
+		chars += GB_UART_FIRMWARE_CREDITS - gb_tty->credits;
+	spin_unlock_irqrestore(&gb_tty->write_lock, flags);
+
+	return chars;
+}
+
+static int gb_tty_break_ctl(struct tty_struct *tty, int state)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+
+	return send_break(gb_tty, state ? 1 : 0);
+}
+
+static void gb_tty_set_termios(struct tty_struct *tty,
+			       struct ktermios *termios_old)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+	struct ktermios *termios = &tty->termios;
+	struct gb_tty_line_coding newline;
+	u8 newctrl = gb_tty->ctrlout;
+
+	newline.rate = cpu_to_le32(tty_get_baud_rate(tty));
+	newline.format = termios->c_cflag & CSTOPB ?
+				GB_SERIAL_2_STOP_BITS : GB_SERIAL_1_STOP_BITS;
+	newline.parity = termios->c_cflag & PARENB ?
+				(termios->c_cflag & PARODD ? 1 : 2) +
+				(termios->c_cflag & CMSPAR ? 2 : 0) : 0;
+
+	switch (termios->c_cflag & CSIZE) {
+	case CS5:
+		newline.data_bits = 5;
+		break;
+	case CS6:
+		newline.data_bits = 6;
+		break;
+	case CS7:
+		newline.data_bits = 7;
+		break;
+	case CS8:
+	default:
+		newline.data_bits = 8;
+		break;
+	}
+
+	/* FIXME: needs to clear unsupported bits in the termios */
+	gb_tty->clocal = ((termios->c_cflag & CLOCAL) != 0);
+
+	if (C_BAUD(tty) == B0) {
+		newline.rate = gb_tty->line_coding.rate;
+		newctrl &= ~(GB_UART_CTRL_DTR | GB_UART_CTRL_RTS);
+	} else if (termios_old && (termios_old->c_cflag & CBAUD) == B0) {
+		newctrl |= (GB_UART_CTRL_DTR | GB_UART_CTRL_RTS);
+	}
+
+	if (newctrl != gb_tty->ctrlout) {
+		gb_tty->ctrlout = newctrl;
+		send_control(gb_tty, newctrl);
+	}
+
+	if (C_CRTSCTS(tty) && C_BAUD(tty) != B0)
+		newline.flow_control |= GB_SERIAL_AUTO_RTSCTS_EN;
+	else
+		newline.flow_control &= ~GB_SERIAL_AUTO_RTSCTS_EN;
+
+	if (memcmp(&gb_tty->line_coding, &newline, sizeof(newline))) {
+		memcpy(&gb_tty->line_coding, &newline, sizeof(newline));
+		send_line_coding(gb_tty);
+	}
+}
+
+static int gb_tty_tiocmget(struct tty_struct *tty)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+
+	return (gb_tty->ctrlout & GB_UART_CTRL_DTR ? TIOCM_DTR : 0) |
+	       (gb_tty->ctrlout & GB_UART_CTRL_RTS ? TIOCM_RTS : 0) |
+	       (gb_tty->ctrlin  & GB_UART_CTRL_DSR ? TIOCM_DSR : 0) |
+	       (gb_tty->ctrlin  & GB_UART_CTRL_RI  ? TIOCM_RI  : 0) |
+	       (gb_tty->ctrlin  & GB_UART_CTRL_DCD ? TIOCM_CD  : 0) |
+	       TIOCM_CTS;
+}
+
+static int gb_tty_tiocmset(struct tty_struct *tty, unsigned int set,
+			   unsigned int clear)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+	u8 newctrl = gb_tty->ctrlout;
+
+	set = (set & TIOCM_DTR ? GB_UART_CTRL_DTR : 0) |
+	      (set & TIOCM_RTS ? GB_UART_CTRL_RTS : 0);
+	clear = (clear & TIOCM_DTR ? GB_UART_CTRL_DTR : 0) |
+		(clear & TIOCM_RTS ? GB_UART_CTRL_RTS : 0);
+
+	newctrl = (newctrl & ~clear) | set;
+	if (gb_tty->ctrlout == newctrl)
+		return 0;
+
+	gb_tty->ctrlout = newctrl;
+	return send_control(gb_tty, newctrl);
+}
+
+static void gb_tty_throttle(struct tty_struct *tty)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+	unsigned char stop_char;
+	int retval;
+
+	if (I_IXOFF(tty)) {
+		stop_char = STOP_CHAR(tty);
+		retval = gb_tty_write(tty, &stop_char, 1);
+		if (retval <= 0)
+			return;
+	}
+
+	if (tty->termios.c_cflag & CRTSCTS) {
+		gb_tty->ctrlout &= ~GB_UART_CTRL_RTS;
+		retval = send_control(gb_tty, gb_tty->ctrlout);
+	}
+}
+
+static void gb_tty_unthrottle(struct tty_struct *tty)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+	unsigned char start_char;
+	int retval;
+
+	if (I_IXOFF(tty)) {
+		start_char = START_CHAR(tty);
+		retval = gb_tty_write(tty, &start_char, 1);
+		if (retval <= 0)
+			return;
+	}
+
+	if (tty->termios.c_cflag & CRTSCTS) {
+		gb_tty->ctrlout |= GB_UART_CTRL_RTS;
+		retval = send_control(gb_tty, gb_tty->ctrlout);
+	}
+}
+
+static int get_serial_info(struct gb_tty *gb_tty,
+			   struct serial_struct __user *info)
+{
+	struct serial_struct tmp;
+
+	if (!info)
+		return -EINVAL;
+
+	memset(&tmp, 0, sizeof(tmp));
+	tmp.flags = ASYNC_LOW_LATENCY | ASYNC_SKIP_TEST;
+	tmp.type = PORT_16550A;
+	tmp.line = gb_tty->minor;
+	tmp.xmit_fifo_size = 16;
+	tmp.baud_base = 9600;
+	tmp.close_delay = gb_tty->port.close_delay / 10;
+	tmp.closing_wait = gb_tty->port.closing_wait == ASYNC_CLOSING_WAIT_NONE ?
+				ASYNC_CLOSING_WAIT_NONE : gb_tty->port.closing_wait / 10;
+
+	if (copy_to_user(info, &tmp, sizeof(tmp)))
+		return -EFAULT;
+	return 0;
+}
+
+static int set_serial_info(struct gb_tty *gb_tty,
+			   struct serial_struct __user *newinfo)
+{
+	struct serial_struct new_serial;
+	unsigned int closing_wait;
+	unsigned int close_delay;
+	int retval = 0;
+
+	if (copy_from_user(&new_serial, newinfo, sizeof(new_serial)))
+		return -EFAULT;
+
+	close_delay = new_serial.close_delay * 10;
+	closing_wait = new_serial.closing_wait == ASYNC_CLOSING_WAIT_NONE ?
+			ASYNC_CLOSING_WAIT_NONE : new_serial.closing_wait * 10;
+
+	mutex_lock(&gb_tty->port.mutex);
+	if (!capable(CAP_SYS_ADMIN)) {
+		if ((close_delay != gb_tty->port.close_delay) ||
+		    (closing_wait != gb_tty->port.closing_wait))
+			retval = -EPERM;
+		else
+			retval = -EOPNOTSUPP;
+	} else {
+		gb_tty->port.close_delay = close_delay;
+		gb_tty->port.closing_wait = closing_wait;
+	}
+	mutex_unlock(&gb_tty->port.mutex);
+	return retval;
+}
+
+static int wait_serial_change(struct gb_tty *gb_tty, unsigned long arg)
+{
+	int retval = 0;
+	DECLARE_WAITQUEUE(wait, current);
+	struct async_icount old;
+	struct async_icount new;
+
+	if (!(arg & (TIOCM_DSR | TIOCM_RI | TIOCM_CD)))
+		return -EINVAL;
+
+	do {
+		spin_lock_irq(&gb_tty->read_lock);
+		old = gb_tty->oldcount;
+		new = gb_tty->iocount;
+		gb_tty->oldcount = new;
+		spin_unlock_irq(&gb_tty->read_lock);
+
+		if ((arg & TIOCM_DSR) && (old.dsr != new.dsr))
+			break;
+		if ((arg & TIOCM_CD) && (old.dcd != new.dcd))
+			break;
+		if ((arg & TIOCM_RI) && (old.rng != new.rng))
+			break;
+
+		add_wait_queue(&gb_tty->wioctl, &wait);
+		set_current_state(TASK_INTERRUPTIBLE);
+		schedule();
+		remove_wait_queue(&gb_tty->wioctl, &wait);
+		if (gb_tty->disconnected) {
+			if (arg & TIOCM_CD)
+				break;
+			retval = -ENODEV;
+		} else if (signal_pending(current)) {
+			retval = -ERESTARTSYS;
+		}
+	} while (!retval);
+
+	return retval;
+}
+
+static int get_serial_usage(struct gb_tty *gb_tty,
+			    struct serial_icounter_struct __user *count)
+{
+	struct serial_icounter_struct icount;
+	int retval = 0;
+
+	memset(&icount, 0, sizeof(icount));
+	icount.dsr = gb_tty->iocount.dsr;
+	icount.rng = gb_tty->iocount.rng;
+	icount.dcd = gb_tty->iocount.dcd;
+	icount.frame = gb_tty->iocount.frame;
+	icount.overrun = gb_tty->iocount.overrun;
+	icount.parity = gb_tty->iocount.parity;
+	icount.brk = gb_tty->iocount.brk;
+
+	if (copy_to_user(count, &icount, sizeof(icount)) > 0)
+		retval = -EFAULT;
+
+	return retval;
+}
+
+static int gb_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
+			unsigned long arg)
+{
+	struct gb_tty *gb_tty = tty->driver_data;
+
+	switch (cmd) {
+	case TIOCGSERIAL:
+		return get_serial_info(gb_tty,
+				       (struct serial_struct __user *)arg);
+	case TIOCSSERIAL:
+		return set_serial_info(gb_tty,
+				       (struct serial_struct __user *)arg);
+	case TIOCMIWAIT:
+		return wait_serial_change(gb_tty, arg);
+	case TIOCGICOUNT:
+		return get_serial_usage(gb_tty,
+					(struct serial_icounter_struct __user *)arg);
+	}
+
+	return -ENOIOCTLCMD;
+}
+
+static void gb_tty_dtr_rts(struct tty_port *port, int on)
+{
+	struct gb_tty *gb_tty;
+	u8 newctrl;
+
+	gb_tty = container_of(port, struct gb_tty, port);
+	newctrl = gb_tty->ctrlout;
+
+	if (on)
+		newctrl |= (GB_UART_CTRL_DTR | GB_UART_CTRL_RTS);
+	else
+		newctrl &= ~(GB_UART_CTRL_DTR | GB_UART_CTRL_RTS);
+
+	gb_tty->ctrlout = newctrl;
+	send_control(gb_tty, newctrl);
+}
+
+static int gb_tty_port_activate(struct tty_port *port,
+				struct tty_struct *tty)
+{
+	struct gb_tty *gb_tty;
+
+	gb_tty = container_of(port, struct gb_tty, port);
+
+	return gbphy_runtime_get_sync(gb_tty->gbphy_dev);
+}
+
+static void gb_tty_port_shutdown(struct tty_port *port)
+{
+	struct gb_tty *gb_tty;
+	unsigned long flags;
+	int ret;
+
+	gb_tty = container_of(port, struct gb_tty, port);
+
+	gb_tty->close_pending = true;
+
+	cancel_work_sync(&gb_tty->tx_work);
+
+	spin_lock_irqsave(&gb_tty->write_lock, flags);
+	kfifo_reset_out(&gb_tty->write_fifo);
+	spin_unlock_irqrestore(&gb_tty->write_lock, flags);
+
+	if (gb_tty->credits == GB_UART_FIRMWARE_CREDITS)
+		goto out;
+
+	ret = gb_uart_flush(gb_tty, GB_SERIAL_FLAG_FLUSH_TRANSMITTER);
+	if (ret) {
+		dev_err(&gb_tty->gbphy_dev->dev,
+			"error flushing transmitter: %d\n", ret);
+	}
+
+	gb_uart_wait_for_all_credits(gb_tty);
+
+out:
+	gb_tty->close_pending = false;
+
+	gbphy_runtime_put_autosuspend(gb_tty->gbphy_dev);
+}
+
+static const struct tty_operations gb_ops = {
+	.install =		gb_tty_install,
+	.open =			gb_tty_open,
+	.close =		gb_tty_close,
+	.cleanup =		gb_tty_cleanup,
+	.hangup =		gb_tty_hangup,
+	.write =		gb_tty_write,
+	.write_room =		gb_tty_write_room,
+	.ioctl =		gb_tty_ioctl,
+	.throttle =		gb_tty_throttle,
+	.unthrottle =		gb_tty_unthrottle,
+	.chars_in_buffer =	gb_tty_chars_in_buffer,
+	.break_ctl =		gb_tty_break_ctl,
+	.set_termios =		gb_tty_set_termios,
+	.tiocmget =		gb_tty_tiocmget,
+	.tiocmset =		gb_tty_tiocmset,
+};
+
+static struct tty_port_operations gb_port_ops = {
+	.dtr_rts =		gb_tty_dtr_rts,
+	.activate =		gb_tty_port_activate,
+	.shutdown =		gb_tty_port_shutdown,
+};
+
+static int gb_uart_probe(struct gbphy_device *gbphy_dev,
+			 const struct gbphy_device_id *id)
+{
+	struct gb_connection *connection;
+	size_t max_payload;
+	struct gb_tty *gb_tty;
+	struct device *tty_dev;
+	int retval;
+	int minor;
+
+	gb_tty = kzalloc(sizeof(*gb_tty), GFP_KERNEL);
+	if (!gb_tty)
+		return -ENOMEM;
+
+	connection = gb_connection_create(gbphy_dev->bundle,
+					  le16_to_cpu(gbphy_dev->cport_desc->id),
+					  gb_uart_request_handler);
+	if (IS_ERR(connection)) {
+		retval = PTR_ERR(connection);
+		goto exit_tty_free;
+	}
+
+	max_payload = gb_operation_get_payload_size_max(connection);
+	if (max_payload < sizeof(struct gb_uart_send_data_request)) {
+		retval = -EINVAL;
+		goto exit_connection_destroy;
+	}
+
+	gb_tty->buffer_payload_max = max_payload -
+			sizeof(struct gb_uart_send_data_request);
+
+	gb_tty->buffer = kzalloc(gb_tty->buffer_payload_max, GFP_KERNEL);
+	if (!gb_tty->buffer) {
+		retval = -ENOMEM;
+		goto exit_connection_destroy;
+	}
+
+	INIT_WORK(&gb_tty->tx_work, gb_uart_tx_write_work);
+
+	retval = kfifo_alloc(&gb_tty->write_fifo, GB_UART_WRITE_FIFO_SIZE,
+			     GFP_KERNEL);
+	if (retval)
+		goto exit_buf_free;
+
+	gb_tty->credits = GB_UART_FIRMWARE_CREDITS;
+	init_completion(&gb_tty->credits_complete);
+
+	minor = alloc_minor(gb_tty);
+	if (minor < 0) {
+		if (minor == -ENOSPC) {
+			dev_err(&connection->bundle->dev,
+				"no more free minor numbers\n");
+			retval = -ENODEV;
+		} else {
+			retval = minor;
+		}
+		goto exit_kfifo_free;
+	}
+
+	gb_tty->minor = minor;
+	spin_lock_init(&gb_tty->write_lock);
+	spin_lock_init(&gb_tty->read_lock);
+	init_waitqueue_head(&gb_tty->wioctl);
+	mutex_init(&gb_tty->mutex);
+
+	tty_port_init(&gb_tty->port);
+	gb_tty->port.ops = &gb_port_ops;
+
+	gb_tty->connection = connection;
+	gb_tty->gbphy_dev = gbphy_dev;
+	gb_connection_set_data(connection, gb_tty);
+	gb_gbphy_set_data(gbphy_dev, gb_tty);
+
+	retval = gb_connection_enable_tx(connection);
+	if (retval)
+		goto exit_release_minor;
+
+	send_control(gb_tty, gb_tty->ctrlout);
+
+	/* initialize the uart to be 9600n81 */
+	gb_tty->line_coding.rate = cpu_to_le32(9600);
+	gb_tty->line_coding.format = GB_SERIAL_1_STOP_BITS;
+	gb_tty->line_coding.parity = GB_SERIAL_NO_PARITY;
+	gb_tty->line_coding.data_bits = 8;
+	send_line_coding(gb_tty);
+
+	retval = gb_connection_enable(connection);
+	if (retval)
+		goto exit_connection_disable;
+
+	tty_dev = tty_port_register_device(&gb_tty->port, gb_tty_driver, minor,
+					   &gbphy_dev->dev);
+	if (IS_ERR(tty_dev)) {
+		retval = PTR_ERR(tty_dev);
+		goto exit_connection_disable;
+	}
+
+	gbphy_runtime_put_autosuspend(gbphy_dev);
+	return 0;
+
+exit_connection_disable:
+	gb_connection_disable(connection);
+exit_release_minor:
+	release_minor(gb_tty);
+exit_kfifo_free:
+	kfifo_free(&gb_tty->write_fifo);
+exit_buf_free:
+	kfree(gb_tty->buffer);
+exit_connection_destroy:
+	gb_connection_destroy(connection);
+exit_tty_free:
+	kfree(gb_tty);
+
+	return retval;
+}
+
+static void gb_uart_remove(struct gbphy_device *gbphy_dev)
+{
+	struct gb_tty *gb_tty = gb_gbphy_get_data(gbphy_dev);
+	struct gb_connection *connection = gb_tty->connection;
+	struct tty_struct *tty;
+	int ret;
+
+	ret = gbphy_runtime_get_sync(gbphy_dev);
+	if (ret)
+		gbphy_runtime_get_noresume(gbphy_dev);
+
+	mutex_lock(&gb_tty->mutex);
+	gb_tty->disconnected = true;
+
+	wake_up_all(&gb_tty->wioctl);
+	mutex_unlock(&gb_tty->mutex);
+
+	tty = tty_port_tty_get(&gb_tty->port);
+	if (tty) {
+		tty_vhangup(tty);
+		tty_kref_put(tty);
+	}
+
+	gb_connection_disable_rx(connection);
+	tty_unregister_device(gb_tty_driver, gb_tty->minor);
+
+	/* FIXME - free transmit / receive buffers */
+
+	gb_connection_disable(connection);
+	tty_port_destroy(&gb_tty->port);
+	gb_connection_destroy(connection);
+	release_minor(gb_tty);
+	kfifo_free(&gb_tty->write_fifo);
+	kfree(gb_tty->buffer);
+	kfree(gb_tty);
+}
+
+static int gb_tty_init(void)
+{
+	int retval = 0;
+
+	gb_tty_driver = tty_alloc_driver(GB_NUM_MINORS, 0);
+	if (IS_ERR(gb_tty_driver)) {
+		pr_err("Can not allocate tty driver\n");
+		retval = -ENOMEM;
+		goto fail_unregister_dev;
+	}
+
+	gb_tty_driver->driver_name = "gb";
+	gb_tty_driver->name = GB_NAME;
+	gb_tty_driver->major = 0;
+	gb_tty_driver->minor_start = 0;
+	gb_tty_driver->type = TTY_DRIVER_TYPE_SERIAL;
+	gb_tty_driver->subtype = SERIAL_TYPE_NORMAL;
+	gb_tty_driver->flags = TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV;
+	gb_tty_driver->init_termios = tty_std_termios;
+	gb_tty_driver->init_termios.c_cflag = B9600 | CS8 | CREAD | HUPCL | CLOCAL;
+	tty_set_operations(gb_tty_driver, &gb_ops);
+
+	retval = tty_register_driver(gb_tty_driver);
+	if (retval) {
+		pr_err("Can not register tty driver: %d\n", retval);
+		goto fail_put_gb_tty;
+	}
+
+	return 0;
+
+fail_put_gb_tty:
+	put_tty_driver(gb_tty_driver);
+fail_unregister_dev:
+	return retval;
+}
+
+static void gb_tty_exit(void)
+{
+	tty_unregister_driver(gb_tty_driver);
+	put_tty_driver(gb_tty_driver);
+	idr_destroy(&tty_minors);
+}
+
+static const struct gbphy_device_id gb_uart_id_table[] = {
+	{ GBPHY_PROTOCOL(GREYBUS_PROTOCOL_UART) },
+	{ },
+};
+MODULE_DEVICE_TABLE(gbphy, gb_uart_id_table);
+
+static struct gbphy_driver uart_driver = {
+	.name		= "uart",
+	.probe		= gb_uart_probe,
+	.remove		= gb_uart_remove,
+	.id_table	= gb_uart_id_table,
+};
+
+static int gb_uart_driver_init(void)
+{
+	int ret;
+
+	ret = gb_tty_init();
+	if (ret)
+		return ret;
+
+	ret = gb_gbphy_register(&uart_driver);
+	if (ret) {
+		gb_tty_exit();
+		return ret;
+	}
+
+	return 0;
+}
+module_init(gb_uart_driver_init);
+
+static void gb_uart_driver_exit(void)
+{
+	gb_gbphy_deregister(&uart_driver);
+	gb_tty_exit();
+}
+
+module_exit(gb_uart_driver_exit);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/usb.c b/drivers/staging/greybus/usb.c
new file mode 100644
index 000000000000..ccadda084b76
--- /dev/null
+++ b/drivers/staging/greybus/usb.c
@@ -0,0 +1,247 @@
+/*
+ * USB host driver for the Greybus "generic" USB module.
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ *
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/usb.h>
+#include <linux/usb/hcd.h>
+
+#include "greybus.h"
+#include "gbphy.h"
+
+/* Greybus USB request types */
+#define GB_USB_TYPE_HCD_START		0x02
+#define GB_USB_TYPE_HCD_STOP		0x03
+#define GB_USB_TYPE_HUB_CONTROL		0x04
+
+struct gb_usb_hub_control_request {
+	__le16 typeReq;
+	__le16 wValue;
+	__le16 wIndex;
+	__le16 wLength;
+};
+
+struct gb_usb_hub_control_response {
+	u8 buf[0];
+};
+
+struct gb_usb_device {
+	struct gb_connection *connection;
+	struct gbphy_device *gbphy_dev;
+};
+
+static inline struct gb_usb_device *to_gb_usb_device(struct usb_hcd *hcd)
+{
+	return (struct gb_usb_device *)hcd->hcd_priv;
+}
+
+static inline struct usb_hcd *gb_usb_device_to_hcd(struct gb_usb_device *dev)
+{
+	return container_of((void *)dev, struct usb_hcd, hcd_priv);
+}
+
+static void hcd_stop(struct usb_hcd *hcd)
+{
+	struct gb_usb_device *dev = to_gb_usb_device(hcd);
+	int ret;
+
+	ret = gb_operation_sync(dev->connection, GB_USB_TYPE_HCD_STOP,
+				NULL, 0, NULL, 0);
+	if (ret)
+		dev_err(&dev->gbphy_dev->dev, "HCD stop failed '%d'\n", ret);
+}
+
+static int hcd_start(struct usb_hcd *hcd)
+{
+	struct usb_bus *bus = hcd_to_bus(hcd);
+	struct gb_usb_device *dev = to_gb_usb_device(hcd);
+	int ret;
+
+	ret = gb_operation_sync(dev->connection, GB_USB_TYPE_HCD_START,
+				NULL, 0, NULL, 0);
+	if (ret) {
+		dev_err(&dev->gbphy_dev->dev, "HCD start failed '%d'\n", ret);
+		return ret;
+	}
+
+	hcd->state = HC_STATE_RUNNING;
+	if (bus->root_hub)
+		usb_hcd_resume_root_hub(hcd);
+	return 0;
+}
+
+static int urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
+{
+	return -ENXIO;
+}
+
+static int urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
+{
+	return -ENXIO;
+}
+
+static int get_frame_number(struct usb_hcd *hcd)
+{
+	return 0;
+}
+
+static int hub_status_data(struct usb_hcd *hcd, char *buf)
+{
+	return 0;
+}
+
+static int hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex,
+		       char *buf, u16 wLength)
+{
+	struct gb_usb_device *dev = to_gb_usb_device(hcd);
+	struct gb_operation *operation;
+	struct gb_usb_hub_control_request *request;
+	struct gb_usb_hub_control_response *response;
+	size_t response_size;
+	int ret;
+
+	/* FIXME: handle unspecified lengths */
+	response_size = sizeof(*response) + wLength;
+
+	operation = gb_operation_create(dev->connection,
+					GB_USB_TYPE_HUB_CONTROL,
+					sizeof(*request),
+					response_size,
+					GFP_KERNEL);
+	if (!operation)
+		return -ENOMEM;
+
+	request = operation->request->payload;
+	request->typeReq = cpu_to_le16(typeReq);
+	request->wValue = cpu_to_le16(wValue);
+	request->wIndex = cpu_to_le16(wIndex);
+	request->wLength = cpu_to_le16(wLength);
+
+	ret = gb_operation_request_send_sync(operation);
+	if (ret)
+		goto out;
+
+	if (wLength) {
+		/* Greybus core has verified response size */
+		response = operation->response->payload;
+		memcpy(buf, response->buf, wLength);
+	}
+out:
+	gb_operation_put(operation);
+
+	return ret;
+}
+
+static struct hc_driver usb_gb_hc_driver = {
+	.description = "greybus-hcd",
+	.product_desc = "Greybus USB Host Controller",
+	.hcd_priv_size = sizeof(struct gb_usb_device),
+
+	.flags = HCD_USB2,
+
+	.start = hcd_start,
+	.stop = hcd_stop,
+
+	.urb_enqueue = urb_enqueue,
+	.urb_dequeue = urb_dequeue,
+
+	.get_frame_number = get_frame_number,
+	.hub_status_data = hub_status_data,
+	.hub_control = hub_control,
+};
+
+static int gb_usb_probe(struct gbphy_device *gbphy_dev,
+			const struct gbphy_device_id *id)
+{
+	struct gb_connection *connection;
+	struct device *dev = &gbphy_dev->dev;
+	struct gb_usb_device *gb_usb_dev;
+	struct usb_hcd *hcd;
+	int retval;
+
+	hcd = usb_create_hcd(&usb_gb_hc_driver, dev, dev_name(dev));
+	if (!hcd)
+		return -ENOMEM;
+
+	connection = gb_connection_create(gbphy_dev->bundle,
+					  le16_to_cpu(gbphy_dev->cport_desc->id),
+					  NULL);
+	if (IS_ERR(connection)) {
+		retval = PTR_ERR(connection);
+		goto exit_usb_put;
+	}
+
+	gb_usb_dev = to_gb_usb_device(hcd);
+	gb_usb_dev->connection = connection;
+	gb_connection_set_data(connection, gb_usb_dev);
+	gb_usb_dev->gbphy_dev = gbphy_dev;
+	gb_gbphy_set_data(gbphy_dev, gb_usb_dev);
+
+	hcd->has_tt = 1;
+
+	retval = gb_connection_enable(connection);
+	if (retval)
+		goto exit_connection_destroy;
+
+	/*
+	 * FIXME: The USB bridged-PHY protocol driver depends on changes to
+	 *        USB core which are not yet upstream.
+	 *
+	 *        Disable for now.
+	 */
+	if (1) {
+		dev_warn(dev, "USB protocol disabled\n");
+		retval = -EPROTONOSUPPORT;
+		goto exit_connection_disable;
+	}
+
+	retval = usb_add_hcd(hcd, 0, 0);
+	if (retval)
+		goto exit_connection_disable;
+
+	return 0;
+
+exit_connection_disable:
+	gb_connection_disable(connection);
+exit_connection_destroy:
+	gb_connection_destroy(connection);
+exit_usb_put:
+	usb_put_hcd(hcd);
+
+	return retval;
+}
+
+static void gb_usb_remove(struct gbphy_device *gbphy_dev)
+{
+	struct gb_usb_device *gb_usb_dev = gb_gbphy_get_data(gbphy_dev);
+	struct gb_connection *connection = gb_usb_dev->connection;
+	struct usb_hcd *hcd = gb_usb_device_to_hcd(gb_usb_dev);
+
+	usb_remove_hcd(hcd);
+	gb_connection_disable(connection);
+	gb_connection_destroy(connection);
+	usb_put_hcd(hcd);
+}
+
+static const struct gbphy_device_id gb_usb_id_table[] = {
+	{ GBPHY_PROTOCOL(GREYBUS_PROTOCOL_USB) },
+	{ },
+};
+MODULE_DEVICE_TABLE(gbphy, gb_usb_id_table);
+
+static struct gbphy_driver usb_driver = {
+	.name		= "usb",
+	.probe		= gb_usb_probe,
+	.remove		= gb_usb_remove,
+	.id_table	= gb_usb_id_table,
+};
+
+module_gbphy_driver(usb_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/greybus/vibrator.c b/drivers/staging/greybus/vibrator.c
new file mode 100644
index 000000000000..4ba0e168930f
--- /dev/null
+++ b/drivers/staging/greybus/vibrator.c
@@ -0,0 +1,249 @@
+/*
+ * Greybus Vibrator protocol driver.
+ *
+ * Copyright 2014 Google Inc.
+ * Copyright 2014 Linaro Ltd.
+ *
+ * Released under the GPLv2 only.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/device.h>
+#include <linux/kdev_t.h>
+#include <linux/idr.h>
+#include <linux/pm_runtime.h>
+
+#include "greybus.h"
+
+struct gb_vibrator_device {
+	struct gb_connection	*connection;
+	struct device		*dev;
+	int			minor;		/* vibrator minor number */
+	struct delayed_work     delayed_work;
+};
+
+/* Greybus Vibrator operation types */
+#define	GB_VIBRATOR_TYPE_ON			0x02
+#define	GB_VIBRATOR_TYPE_OFF			0x03
+
+static int turn_off(struct gb_vibrator_device *vib)
+{
+	struct gb_bundle *bundle = vib->connection->bundle;
+	int ret;
+
+	ret = gb_operation_sync(vib->connection, GB_VIBRATOR_TYPE_OFF,
+			NULL, 0, NULL, 0);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return ret;
+}
+
+static int turn_on(struct gb_vibrator_device *vib, u16 timeout_ms)
+{
+	struct gb_bundle *bundle = vib->connection->bundle;
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		return ret;
+
+	/* Vibrator was switched ON earlier */
+	if (cancel_delayed_work_sync(&vib->delayed_work))
+		turn_off(vib);
+
+	ret = gb_operation_sync(vib->connection, GB_VIBRATOR_TYPE_ON,
+			NULL, 0, NULL, 0);
+	if (ret) {
+		gb_pm_runtime_put_autosuspend(bundle);
+		return ret;
+	}
+
+	schedule_delayed_work(&vib->delayed_work, msecs_to_jiffies(timeout_ms));
+
+	return 0;
+}
+
+static void gb_vibrator_worker(struct work_struct *work)
+{
+	struct delayed_work *delayed_work = to_delayed_work(work);
+	struct gb_vibrator_device *vib =
+		container_of(delayed_work, struct gb_vibrator_device, delayed_work);
+
+	turn_off(vib);
+}
+
+static ssize_t timeout_store(struct device *dev, struct device_attribute *attr,
+			     const char *buf, size_t count)
+{
+	struct gb_vibrator_device *vib = dev_get_drvdata(dev);
+	unsigned long val;
+	int retval;
+
+	retval = kstrtoul(buf, 10, &val);
+	if (retval < 0) {
+		dev_err(dev, "could not parse timeout value %d\n", retval);
+		return retval;
+	}
+
+	if (val)
+		retval = turn_on(vib, (u16)val);
+	else
+		retval = turn_off(vib);
+	if (retval)
+		return retval;
+
+	return count;
+}
+static DEVICE_ATTR_WO(timeout);
+
+static struct attribute *vibrator_attrs[] = {
+	&dev_attr_timeout.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(vibrator);
+
+static struct class vibrator_class = {
+	.name		= "vibrator",
+	.owner		= THIS_MODULE,
+	.dev_groups	= vibrator_groups,
+};
+
+static DEFINE_IDA(minors);
+
+static int gb_vibrator_probe(struct gb_bundle *bundle,
+					const struct greybus_bundle_id *id)
+{
+	struct greybus_descriptor_cport *cport_desc;
+	struct gb_connection *connection;
+	struct gb_vibrator_device *vib;
+	struct device *dev;
+	int retval;
+
+	if (bundle->num_cports != 1)
+		return -ENODEV;
+
+	cport_desc = &bundle->cport_desc[0];
+	if (cport_desc->protocol_id != GREYBUS_PROTOCOL_VIBRATOR)
+		return -ENODEV;
+
+	vib = kzalloc(sizeof(*vib), GFP_KERNEL);
+	if (!vib)
+		return -ENOMEM;
+
+	connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
+						NULL);
+	if (IS_ERR(connection)) {
+		retval = PTR_ERR(connection);
+		goto err_free_vib;
+	}
+	gb_connection_set_data(connection, vib);
+
+	vib->connection = connection;
+
+	greybus_set_drvdata(bundle, vib);
+
+	retval = gb_connection_enable(connection);
+	if (retval)
+		goto err_connection_destroy;
+
+	/*
+	 * For now we create a device in sysfs for the vibrator, but odds are
+	 * there is a "real" device somewhere in the kernel for this, but I
+	 * can't find it at the moment...
+	 */
+	vib->minor = ida_simple_get(&minors, 0, 0, GFP_KERNEL);
+	if (vib->minor < 0) {
+		retval = vib->minor;
+		goto err_connection_disable;
+	}
+	dev = device_create(&vibrator_class, &bundle->dev,
+			    MKDEV(0, 0), vib, "vibrator%d", vib->minor);
+	if (IS_ERR(dev)) {
+		retval = -EINVAL;
+		goto err_ida_remove;
+	}
+	vib->dev = dev;
+
+	INIT_DELAYED_WORK(&vib->delayed_work, gb_vibrator_worker);
+
+	gb_pm_runtime_put_autosuspend(bundle);
+
+	return 0;
+
+err_ida_remove:
+	ida_simple_remove(&minors, vib->minor);
+err_connection_disable:
+	gb_connection_disable(connection);
+err_connection_destroy:
+	gb_connection_destroy(connection);
+err_free_vib:
+	kfree(vib);
+
+	return retval;
+}
+
+static void gb_vibrator_disconnect(struct gb_bundle *bundle)
+{
+	struct gb_vibrator_device *vib = greybus_get_drvdata(bundle);
+	int ret;
+
+	ret = gb_pm_runtime_get_sync(bundle);
+	if (ret)
+		gb_pm_runtime_get_noresume(bundle);
+
+	if (cancel_delayed_work_sync(&vib->delayed_work))
+		turn_off(vib);
+
+	device_unregister(vib->dev);
+	ida_simple_remove(&minors, vib->minor);
+	gb_connection_disable(vib->connection);
+	gb_connection_destroy(vib->connection);
+	kfree(vib);
+}
+
+static const struct greybus_bundle_id gb_vibrator_id_table[] = {
+	{ GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_VIBRATOR) },
+	{ }
+};
+MODULE_DEVICE_TABLE(greybus, gb_vibrator_id_table);
+
+static struct greybus_driver gb_vibrator_driver = {
+	.name		= "vibrator",
+	.probe		= gb_vibrator_probe,
+	.disconnect	= gb_vibrator_disconnect,
+	.id_table	= gb_vibrator_id_table,
+};
+
+static __init int gb_vibrator_init(void)
+{
+	int retval;
+
+	retval = class_register(&vibrator_class);
+	if (retval)
+		return retval;
+
+	retval = greybus_register(&gb_vibrator_driver);
+	if (retval)
+		goto err_class_unregister;
+
+	return 0;
+
+err_class_unregister:
+	class_unregister(&vibrator_class);
+
+	return retval;
+}
+module_init(gb_vibrator_init);
+
+static __exit void gb_vibrator_exit(void)
+{
+	greybus_deregister(&gb_vibrator_driver);
+	class_unregister(&vibrator_class);
+	ida_destroy(&minors);
+}
+module_exit(gb_vibrator_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/gs_fpgaboot/gs_fpgaboot.c b/drivers/staging/gs_fpgaboot/gs_fpgaboot.c
index a221f261c3d3..8ed4d395be58 100644
--- a/drivers/staging/gs_fpgaboot/gs_fpgaboot.c
+++ b/drivers/staging/gs_fpgaboot/gs_fpgaboot.c
@@ -8,10 +8,6 @@
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/staging/gs_fpgaboot/gs_fpgaboot.h b/drivers/staging/gs_fpgaboot/gs_fpgaboot.h
index 8cc32555dbf3..7b8cc3a25214 100644
--- a/drivers/staging/gs_fpgaboot/gs_fpgaboot.h
+++ b/drivers/staging/gs_fpgaboot/gs_fpgaboot.h
@@ -7,12 +7,7 @@
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *
  * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
 #include <linux/firmware.h>
diff --git a/drivers/staging/gs_fpgaboot/io.h b/drivers/staging/gs_fpgaboot/io.h
index 7b46ac24b74e..5e839b1fc884 100644
--- a/drivers/staging/gs_fpgaboot/io.h
+++ b/drivers/staging/gs_fpgaboot/io.h
@@ -66,10 +66,8 @@ enum wbus {
 	bus_2byte	= 2,
 };
 
-
 #define MAX_WAIT_DONE	10000
 
-
 struct gpiobus {
 	int	ngpio;
 	void __iomem *r[4];
diff --git a/drivers/staging/i4l/act2000/act2000_isa.c b/drivers/staging/i4l/act2000/act2000_isa.c
index f0eb8441deed..ad7a0391369f 100644
--- a/drivers/staging/i4l/act2000/act2000_isa.c
+++ b/drivers/staging/i4l/act2000/act2000_isa.c
@@ -134,9 +134,9 @@ act2000_isa_config_irq(act2000_card *card, short irq)
 {
 	int old_irq;
 
-	if (card->flags & ACT2000_FLAGS_IVALID) {
+	if (card->flags & ACT2000_FLAGS_IVALID)
 		free_irq(card->irq, card);
-	}
+
 	card->flags &= ~ACT2000_FLAGS_IVALID;
 	outb(ISA_COR_IRQOFF, ISA_PORT_COR);
 	if (!irq)
@@ -166,7 +166,7 @@ act2000_isa_config_port(act2000_card *card, unsigned short portbase)
 		release_region(card->port, ISA_REGION);
 		card->flags &= ~ACT2000_FLAGS_PVALID;
 	}
-	if (request_region(portbase, ACT2000_PORTLEN, card->regname) == NULL)
+	if (!request_region(portbase, ACT2000_PORTLEN, card->regname))
 		return -EBUSY;
 	else {
 		card->port = portbase;
@@ -176,7 +176,7 @@ act2000_isa_config_port(act2000_card *card, unsigned short portbase)
 }
 
 /*
- * Release ressources, used by an adaptor.
+ * Release resources, used by an adaptor.
  */
 void
 act2000_isa_release(act2000_card *card)
@@ -244,7 +244,7 @@ act2000_isa_receive(act2000_card *card)
 				if (valid) {
 					card->idat.isa.rcvlen = ((actcapi_msghdr *)&card->idat.isa.rcvhdr)->len;
 					card->idat.isa.rcvskb = dev_alloc_skb(card->idat.isa.rcvlen);
-					if (card->idat.isa.rcvskb == NULL) {
+					if (!card->idat.isa.rcvskb) {
 						card->idat.isa.rcvignore = 1;
 						printk(KERN_WARNING
 						       "act2000_isa_receive: no memory\n");
@@ -399,7 +399,6 @@ act2000_isa_download(act2000_card *card, act2000_ddef __user *cb)
 	unsigned int length;
 	int l;
 	int c;
-	long timeout;
 	u_char *b;
 	u_char __user *p;
 	u_char *buf;
@@ -417,7 +416,6 @@ act2000_isa_download(act2000_card *card, act2000_ddef __user *cb)
 	buf = kmalloc(1024, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
-	timeout = 0;
 	while (length) {
 		l = (length > 1024) ? 1024 : length;
 		c = 0;
diff --git a/drivers/staging/i4l/act2000/capi.c b/drivers/staging/i4l/act2000/capi.c
index 3f66ca20b5e5..62f56294853c 100644
--- a/drivers/staging/i4l/act2000/capi.c
+++ b/drivers/staging/i4l/act2000/capi.c
@@ -113,7 +113,9 @@ actcapi_chkhdr(act2000_card *card, actcapi_msghdr *hdr)
 			m->hdr.cmd.cmd = c;			\
 			m->hdr.cmd.subcmd = s;			\
 			m->hdr.msgnum = actcapi_nextsmsg(card); \
-		} else m = NULL;				\
+		} else {					\
+			m = NULL;				\
+		}						\
 	}
 
 #define ACTCAPI_CHKSKB if (!skb) {					\
@@ -547,12 +549,11 @@ static int
 actcapi_data_b3_ind(act2000_card *card, struct sk_buff *skb) {
 	__u16 plci;
 	__u16 ncci;
-	__u16 controller;
 	__u8  blocknr;
 	int chan;
 	actcapi_msg *msg = (actcapi_msg *)skb->data;
 
-	EVAL_NCCI(msg->msg.data_b3_ind.fakencci, plci, controller, ncci);
+	EVAL_NCCI(msg->msg.data_b3_ind.fakencci, plci, ncci);
 	chan = find_ncci(card, ncci);
 	if (chan < 0)
 		return 0;
@@ -617,7 +618,7 @@ handle_ack(act2000_card *card, act2000_chan *chan, __u8 blocknr) {
 		spin_lock_irqsave(&card->lock, flags);
 		tmp = skb_peek((struct sk_buff_head *)tmp);
 		spin_unlock_irqrestore(&card->lock, flags);
-		if ((tmp == skb) || (tmp == NULL)) {
+		if ((tmp == skb) || !tmp) {
 			/* reached end of queue */
 			printk(KERN_WARNING "act2000: handle_ack nothing found!\n");
 			return 0;
@@ -990,7 +991,8 @@ actcapi_debug_dlpd(actcapi_dlpd *dlpd)
 }
 
 #ifdef DEBUG_DUMP_SKB
-static void dump_skb(struct sk_buff *skb) {
+static void dump_skb(struct sk_buff *skb)
+{
 	char tmp[80];
 	char *p = skb->data;
 	char *t = tmp;
diff --git a/drivers/staging/i4l/act2000/capi.h b/drivers/staging/i4l/act2000/capi.h
index 01ccdecd43f7..34884a5efee5 100644
--- a/drivers/staging/i4l/act2000/capi.h
+++ b/drivers/staging/i4l/act2000/capi.h
@@ -114,9 +114,8 @@ typedef struct actcapi_ncpd {
 #define MAKE_NCCI(plci, contr, ncci)					\
 	((plci & 0x1f) | ((contr & 0x7) << 5) | ((ncci & 0xff) << 8))
 
-#define EVAL_NCCI(fakencci, plci, contr, ncci) {	\
+#define EVAL_NCCI(fakencci, plci, ncci) {	\
 		plci  = fakencci & 0x1f;		\
-		contr = (fakencci >> 5) & 0x7;		\
 		ncci  = (fakencci >> 8) & 0xff;		\
 	}
 
@@ -128,13 +127,6 @@ typedef struct actcapi_ncpd {
  * Bit 5-7  = Controller
  * Bit 8-15 = reserved (must be 0)
  */
-#define MAKE_PLCI(plci, contr)			\
-	((plci & 0x1f) | ((contr & 0x7) << 5))
-
-#define EVAL_PLCI(fakeplci, plci, contr) {	\
-		plci  = fakeplci & 0x1f;	\
-		contr = (fakeplci >> 5) & 0x7;	\
-	}
 
 typedef struct actcapi_msg {
 	actcapi_msghdr hdr;
diff --git a/drivers/staging/i4l/act2000/module.c b/drivers/staging/i4l/act2000/module.c
index 68073d0da0e3..99c9c0a1c63e 100644
--- a/drivers/staging/i4l/act2000/module.c
+++ b/drivers/staging/i4l/act2000/module.c
@@ -28,7 +28,7 @@ static unsigned short act2000_isa_ports[] =
 static act2000_card *cards = (act2000_card *) NULL;
 
 /* Parameters to be set by insmod */
-static int   act_bus  =  0;
+static int   act_bus;
 static int   act_port = -1;  /* -1 = Autoprobe  */
 static int   act_irq  = -1;
 static char *act_id   = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
@@ -289,7 +289,8 @@ act2000_command(act2000_card *card, isdn_ctrl *c)
 			if (copy_from_user(tmp, arg,
 					   sizeof(tmp)))
 				return -EFAULT;
-			if ((ret = act2000_set_msn(card, tmp)))
+			ret = act2000_set_msn(card, tmp);
+			if (ret)
 				return ret;
 			if (card->flags & ACT2000_FLAGS_RUNNING)
 				return (actcapi_manufacturer_req_msn(card));
diff --git a/drivers/staging/i4l/icn/icn.c b/drivers/staging/i4l/icn/icn.c
index 46d957c34be1..514bfc2c5b53 100644
--- a/drivers/staging/i4l/icn/icn.c
+++ b/drivers/staging/i4l/icn/icn.c
@@ -62,7 +62,8 @@ icn_free_queue(icn_card *card, int channel)
 	skb_queue_purge(queue);
 	card->xlen[channel] = 0;
 	card->sndcount[channel] = 0;
-	if ((skb = card->xskb[channel])) {
+	skb = card->xskb[channel];
+	if (skb) {
 		card->xskb[channel] = NULL;
 		dev_kfree_skb(skb);
 	}
@@ -81,12 +82,11 @@ icn_shiftout(unsigned short port,
 	     int firstbit,
 	     int bitcount)
 {
-
 	register u_char s;
 	register u_char c;
 
 	for (s = firstbit, c = bitcount; c > 0; s--, c--)
-		OUTB_P((u_char) ((val >> s) & 1) ? 0xff : 0, port);
+		OUTB_P((u_char)((val >> s) & 1) ? 0xff : 0, port);
 }
 
 /*
@@ -272,8 +272,10 @@ icn_pollbchan_receive(int channel, icn_card *card)
 			rbnext;
 			icn_maprelease_channel(card, mch & 2);
 			if (!eflag) {
-				if ((cnt = card->rcvidx[channel])) {
-					if (!(skb = dev_alloc_skb(cnt))) {
+				cnt = card->rcvidx[channel];
+				if (cnt) {
+					skb = dev_alloc_skb(cnt);
+					if (!skb) {
 						printk(KERN_WARNING "icn: receive out of memory\n");
 						break;
 					}
@@ -382,7 +384,7 @@ icn_pollbchan_send(int channel, icn_card *card)
 static void
 icn_pollbchan(unsigned long data)
 {
-	icn_card *card = (icn_card *) data;
+	icn_card *card = (icn_card *)data;
 	unsigned long flags;
 
 	if (card->flags & ICN_FLAGS_B1ACTIVE) {
@@ -472,7 +474,6 @@ icn_parse_status(u_char *status, int channel, icn_card *card)
 
 		if (card->flags &
 		    ((channel) ? ICN_FLAGS_B2ACTIVE : ICN_FLAGS_B1ACTIVE)) {
-
 			isdn_ctrl ncmd;
 
 			card->flags &= ~((channel) ?
@@ -544,7 +545,7 @@ icn_parse_status(u_char *status, int channel, icn_card *card)
 		break;
 	case 6:
 		snprintf(cmd.parm.num, sizeof(cmd.parm.num), "%d",
-			 (int) simple_strtoul(status + 7, NULL, 16));
+			 (int)simple_strtoul(status + 7, NULL, 16));
 		break;
 	case 7:
 		status += 3;
@@ -604,7 +605,7 @@ icn_putmsg(icn_card *card, unsigned char c)
 static void
 icn_polldchan(unsigned long data)
 {
-	icn_card *card = (icn_card *) data;
+	icn_card *card = (icn_card *)data;
 	int mch = card->secondhalf ? 2 : 0;
 	int avail = 0;
 	int left;
@@ -656,9 +657,8 @@ icn_polldchan(unsigned long data)
 						*q = '\0';
 						strcat(vstr, "000");
 						vstr[3] = '\0';
-						card->fw_rev = (int) simple_strtoul(vstr, NULL, 10);
+						card->fw_rev = (int)simple_strtoul(vstr, NULL, 10);
 						continue;
-
 					}
 				}
 			} else {
@@ -683,7 +683,7 @@ icn_polldchan(unsigned long data)
 			card->flags |= ICN_FLAGS_RBTIMER;
 			del_timer(&card->rb_timer);
 			card->rb_timer.function = icn_pollbchan;
-			card->rb_timer.data = (unsigned long) card;
+			card->rb_timer.data = (unsigned long)card;
 			card->rb_timer.expires = jiffies + ICN_TIMER_BCREAD;
 			add_timer(&card->rb_timer);
 		}
@@ -805,17 +805,12 @@ icn_loadboot(u_char __user *buffer, icn_card *card)
 	unsigned long flags;
 
 #ifdef BOOT_DEBUG
-	printk(KERN_DEBUG "icn_loadboot called, buffaddr=%08lx\n", (ulong) buffer);
+	printk(KERN_DEBUG "icn_loadboot called, buffaddr=%08lx\n", (ulong)buffer);
 #endif
-	if (!(codebuf = kmalloc(ICN_CODE_STAGE1, GFP_KERNEL))) {
-		printk(KERN_WARNING "icn: Could not allocate code buffer\n");
-		ret = -ENOMEM;
-		goto out;
-	}
-	if (copy_from_user(codebuf, buffer, ICN_CODE_STAGE1)) {
-		ret = -EFAULT;
-		goto out_kfree;
-	}
+	codebuf = memdup_user(buffer, ICN_CODE_STAGE1);
+	if (IS_ERR(codebuf))
+		return PTR_ERR(codebuf);
+
 	if (!card->rvalid) {
 		if (!request_region(card->port, ICN_PORTLEN, card->regname)) {
 			printk(KERN_WARNING
@@ -878,9 +873,9 @@ icn_loadboot(u_char __user *buffer, icn_card *card)
 	}
 	SLEEP(1);
 	OUTB_P(0xff, ICN_RUN);  /* Start Boot-Code */
-	if ((ret = icn_check_loader(card->doubleS0 ? 2 : 1))) {
+	ret = icn_check_loader(card->doubleS0 ? 2 : 1);
+	if (ret)
 		goto out_kfree;
-	}
 	if (!card->doubleS0) {
 		ret = 0;
 		goto out_kfree;
@@ -898,7 +893,6 @@ icn_loadboot(u_char __user *buffer, icn_card *card)
 
 out_kfree:
 	kfree(codebuf);
-out:
 	return ret;
 }
 
@@ -980,18 +974,17 @@ icn_loadproto(u_char __user *buffer, icn_card *card)
 				       card->secondhalf);
 #endif
 				spin_lock_irqsave(&card->lock, flags);
-				init_timer(&card->st_timer);
-				card->st_timer.expires = jiffies + ICN_TIMER_DCREAD;
-				card->st_timer.function = icn_polldchan;
-				card->st_timer.data = (unsigned long) card;
-				add_timer(&card->st_timer);
+				setup_timer(&card->st_timer, icn_polldchan,
+					    (unsigned long)card);
+				mod_timer(&card->st_timer,
+					  jiffies + ICN_TIMER_DCREAD);
 				card->flags |= ICN_FLAGS_RUNNING;
 				if (card->doubleS0) {
-					init_timer(&card->other->st_timer);
-					card->other->st_timer.expires = jiffies + ICN_TIMER_DCREAD;
-					card->other->st_timer.function = icn_polldchan;
-					card->other->st_timer.data = (unsigned long) card->other;
-					add_timer(&card->other->st_timer);
+					setup_timer(&card->other->st_timer,
+						    icn_polldchan,
+						    (unsigned long)card->other);
+					mod_timer(&card->other->st_timer,
+						  jiffies + ICN_TIMER_DCREAD);
 					card->other->flags |= ICN_FLAGS_RUNNING;
 				}
 				spin_unlock_irqrestore(&card->lock, flags);
@@ -1022,7 +1015,8 @@ icn_readstatus(u_char __user *buf, int len, icn_card *card)
 
 /* Put command-strings into the command-queue of the Interface */
 static int
-icn_writecmd(const u_char *buf, int len, int user, icn_card *card)
+icn_writecmd(const u_char __user *ubuf, const u_char *kbuf, int len,
+	     int user, icn_card *card)
 {
 	int mch = card->secondhalf ? 2 : 0;
 	int pp;
@@ -1045,10 +1039,10 @@ icn_writecmd(const u_char *buf, int len, int user, icn_card *card)
 		if (count > len)
 			count = len;
 		if (user) {
-			if (copy_from_user(msg, buf, count))
+			if (copy_from_user(msg, ubuf, count))
 				return -EFAULT;
 		} else
-			memcpy(msg, buf, count);
+			memcpy(msg, kbuf, count);
 
 		spin_lock_irqsave(&dev.devlock, flags);
 		lastmap_card = dev.mcard;
@@ -1190,28 +1184,28 @@ icn_command(isdn_ctrl *c, icn_card *card)
 			}
 			break;
 		case ICN_IOCTL_GETMMIO:
-			return (long) dev.memaddr;
+			return (long)dev.memaddr;
 		case ICN_IOCTL_SETPORT:
 			if (a == 0x300 || a == 0x310 || a == 0x320 || a == 0x330
 			    || a == 0x340 || a == 0x350 || a == 0x360 ||
 			    a == 0x308 || a == 0x318 || a == 0x328 || a == 0x338
 			    || a == 0x348 || a == 0x358 || a == 0x368) {
-				if (card->port != (unsigned short) a) {
-					if (!request_region((unsigned short) a, ICN_PORTLEN, "icn-isdn")) {
+				if (card->port != (unsigned short)a) {
+					if (!request_region((unsigned short)a, ICN_PORTLEN, "icn-isdn")) {
 						printk(KERN_WARNING
 						       "icn: (%s) ports 0x%03x-0x%03x in use.\n",
-						       CID, (int) a, (int) a + ICN_PORTLEN);
+						       CID, (int)a, (int)a + ICN_PORTLEN);
 						return -EINVAL;
 					}
-					release_region((unsigned short) a, ICN_PORTLEN);
+					release_region((unsigned short)a, ICN_PORTLEN);
 					icn_stopcard(card);
 					spin_lock_irqsave(&card->lock, flags);
 					if (card->rvalid)
 						release_region(card->port, ICN_PORTLEN);
-					card->port = (unsigned short) a;
+					card->port = (unsigned short)a;
 					card->rvalid = 0;
 					if (card->doubleS0) {
-						card->other->port = (unsigned short) a;
+						card->other->port = (unsigned short)a;
 						card->other->rvalid = 0;
 					}
 					spin_unlock_irqrestore(&card->lock, flags);
@@ -1223,9 +1217,9 @@ icn_command(isdn_ctrl *c, icn_card *card)
 				return -EINVAL;
 			break;
 		case ICN_IOCTL_GETPORT:
-			return (int) card->port;
+			return (int)card->port;
 		case ICN_IOCTL_GETDOUBLE:
-			return (int) card->doubleS0;
+			return (int)card->doubleS0;
 		case ICN_IOCTL_DEBUGVAR:
 			if (copy_to_user(arg,
 					 &card,
@@ -1246,10 +1240,11 @@ icn_command(isdn_ctrl *c, icn_card *card)
 				dev.firstload = 0;
 			}
 			icn_stopcard(card);
-			return (icn_loadboot(arg, card));
+			return icn_loadboot(arg, card);
 		case ICN_IOCTL_LOADPROTO:
 			icn_stopcard(card);
-			if ((i = (icn_loadproto(arg, card))))
+			i = (icn_loadproto(arg, card));
+			if (i)
 				return i;
 			if (card->doubleS0)
 				i = icn_loadproto(arg + ICN_CODE_STAGE2, card->other);
@@ -1262,19 +1257,20 @@ icn_command(isdn_ctrl *c, icn_card *card)
 					   arg,
 					   sizeof(cdef)))
 				return -EFAULT;
-			return (icn_addcard(cdef.port, cdef.id1, cdef.id2));
+			return icn_addcard(cdef.port, cdef.id1, cdef.id2);
 			break;
 		case ICN_IOCTL_LEASEDCFG:
 			if (a) {
 				if (!card->leased) {
 					card->leased = 1;
-					while (card->ptype == ISDN_PTYPE_UNKNOWN) {
+					while (card->ptype == ISDN_PTYPE_UNKNOWN)
 						msleep_interruptible(ICN_BOOT_TIMEOUT1);
-					}
 					msleep_interruptible(ICN_BOOT_TIMEOUT1);
 					sprintf(cbuf, "00;FV2ON\n01;EAZ%c\n02;EAZ%c\n",
 						(a & 1) ? '1' : 'C', (a & 2) ? '2' : 'C');
-					i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+					i = icn_writecmd(NULL, cbuf,
+							 strlen(cbuf),
+							 0, card);
 					printk(KERN_INFO
 					       "icn: (%s) Leased-line mode enabled\n",
 					       CID);
@@ -1287,7 +1283,9 @@ icn_command(isdn_ctrl *c, icn_card *card)
 				if (card->leased) {
 					card->leased = 0;
 					sprintf(cbuf, "00;FV2OFF\n");
-					i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+					i = icn_writecmd(NULL, cbuf,
+							 strlen(cbuf),
+							 0, card);
 					printk(KERN_INFO
 					       "icn: (%s) Leased-line mode disabled\n",
 					       CID);
@@ -1321,10 +1319,10 @@ icn_command(isdn_ctrl *c, icn_card *card)
 				/* Normal Dial */
 				strcpy(dcode, "CAL");
 			snprintf(cbuf, sizeof(cbuf),
-				 "%02d;D%s_R%s,%02d,%02d,%s\n", (int) (a + 1),
+				 "%02d;D%s_R%s,%02d,%02d,%s\n", (int)(a + 1),
 				 dcode, p, c->parm.setup.si1,
 				 c->parm.setup.si2, c->parm.setup.eazmsn);
-			i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+			i = icn_writecmd(NULL, cbuf, strlen(cbuf), 0, card);
 		}
 		break;
 	case ISDN_CMD_ACCEPTD:
@@ -1335,16 +1333,18 @@ icn_command(isdn_ctrl *c, icn_card *card)
 			if (card->fw_rev >= 300) {
 				switch (card->l2_proto[a - 1]) {
 				case ISDN_PROTO_L2_X75I:
-					sprintf(cbuf, "%02d;BX75\n", (int) a);
+					sprintf(cbuf, "%02d;BX75\n", (int)a);
 					break;
 				case ISDN_PROTO_L2_HDLC:
-					sprintf(cbuf, "%02d;BTRA\n", (int) a);
+					sprintf(cbuf, "%02d;BTRA\n", (int)a);
 					break;
 				}
-				i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+				i = icn_writecmd(NULL, cbuf,
+						 strlen(cbuf), 0,
+						 card);
 			}
-			sprintf(cbuf, "%02d;DCON_R\n", (int) a);
-			i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+			sprintf(cbuf, "%02d;DCON_R\n", (int)a);
+			i = icn_writecmd(NULL, cbuf, strlen(cbuf), 0, card);
 		}
 		break;
 	case ISDN_CMD_ACCEPTB:
@@ -1355,14 +1355,14 @@ icn_command(isdn_ctrl *c, icn_card *card)
 			if (card->fw_rev >= 300)
 				switch (card->l2_proto[a - 1]) {
 				case ISDN_PROTO_L2_X75I:
-					sprintf(cbuf, "%02d;BCON_R,BX75\n", (int) a);
+					sprintf(cbuf, "%02d;BCON_R,BX75\n", (int)a);
 					break;
 				case ISDN_PROTO_L2_HDLC:
-					sprintf(cbuf, "%02d;BCON_R,BTRA\n", (int) a);
+					sprintf(cbuf, "%02d;BCON_R,BTRA\n", (int)a);
 					break;
 				} else
-				sprintf(cbuf, "%02d;BCON_R\n", (int) a);
-			i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+				sprintf(cbuf, "%02d;BCON_R\n", (int)a);
+			i = icn_writecmd(NULL, cbuf, strlen(cbuf), 0, card);
 		}
 		break;
 	case ISDN_CMD_HANGUP:
@@ -1370,8 +1370,8 @@ icn_command(isdn_ctrl *c, icn_card *card)
 			return -ENODEV;
 		if (c->arg < ICN_BCH) {
 			a = c->arg + 1;
-			sprintf(cbuf, "%02d;BDIS_R\n%02d;DDIS_R\n", (int) a, (int) a);
-			i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+			sprintf(cbuf, "%02d;BDIS_R\n%02d;DDIS_R\n", (int)a, (int)a);
+			i = icn_writecmd(NULL, cbuf, strlen(cbuf), 0, card);
 		}
 		break;
 	case ISDN_CMD_SETEAZ:
@@ -1382,12 +1382,12 @@ icn_command(isdn_ctrl *c, icn_card *card)
 		if (c->arg < ICN_BCH) {
 			a = c->arg + 1;
 			if (card->ptype == ISDN_PTYPE_EURO) {
-				sprintf(cbuf, "%02d;MS%s%s\n", (int) a,
+				sprintf(cbuf, "%02d;MS%s%s\n", (int)a,
 					c->parm.num[0] ? "N" : "ALL", c->parm.num);
 			} else
-				sprintf(cbuf, "%02d;EAZ%s\n", (int) a,
+				sprintf(cbuf, "%02d;EAZ%s\n", (int)a,
 					c->parm.num[0] ? (char *)(c->parm.num) : "0123456789");
-			i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+			i = icn_writecmd(NULL, cbuf, strlen(cbuf), 0, card);
 		}
 		break;
 	case ISDN_CMD_CLREAZ:
@@ -1398,10 +1398,10 @@ icn_command(isdn_ctrl *c, icn_card *card)
 		if (c->arg < ICN_BCH) {
 			a = c->arg + 1;
 			if (card->ptype == ISDN_PTYPE_EURO)
-				sprintf(cbuf, "%02d;MSNC\n", (int) a);
+				sprintf(cbuf, "%02d;MSNC\n", (int)a);
 			else
-				sprintf(cbuf, "%02d;EAZC\n", (int) a);
-			i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+				sprintf(cbuf, "%02d;EAZC\n", (int)a);
+			i = icn_writecmd(NULL, cbuf, strlen(cbuf), 0, card);
 		}
 		break;
 	case ISDN_CMD_SETL2:
@@ -1411,15 +1411,15 @@ icn_command(isdn_ctrl *c, icn_card *card)
 			a = c->arg;
 			switch (a >> 8) {
 			case ISDN_PROTO_L2_X75I:
-				sprintf(cbuf, "%02d;BX75\n", (int) (a & 255) + 1);
+				sprintf(cbuf, "%02d;BX75\n", (int)(a & 255) + 1);
 				break;
 			case ISDN_PROTO_L2_HDLC:
-				sprintf(cbuf, "%02d;BTRA\n", (int) (a & 255) + 1);
+				sprintf(cbuf, "%02d;BTRA\n", (int)(a & 255) + 1);
 				break;
 			default:
 				return -EINVAL;
 			}
-			i = icn_writecmd(cbuf, strlen(cbuf), 0, card);
+			i = icn_writecmd(NULL, cbuf, strlen(cbuf), 0, card);
 			card->l2_proto[a & 255] = (a >> 8);
 		}
 		break;
@@ -1446,7 +1446,7 @@ icn_findcard(int driverid)
 			return p;
 		p = p->next;
 	}
-	return (icn_card *) 0;
+	return (icn_card *)0;
 }
 
 /*
@@ -1458,7 +1458,7 @@ if_command(isdn_ctrl *c)
 	icn_card *card = icn_findcard(c->driver);
 
 	if (card)
-		return (icn_command(c, card));
+		return icn_command(c, card);
 	printk(KERN_ERR
 	       "icn: if_command %d called with invalid driverId %d!\n",
 	       c->command, c->driver);
@@ -1473,7 +1473,7 @@ if_writecmd(const u_char __user *buf, int len, int id, int channel)
 	if (card) {
 		if (!(card->flags & ICN_FLAGS_RUNNING))
 			return -ENODEV;
-		return (icn_writecmd(buf, len, 1, card));
+		return icn_writecmd(buf, NULL, len, 1, card);
 	}
 	printk(KERN_ERR
 	       "icn: if_writecmd called with invalid driverId!\n");
@@ -1488,7 +1488,7 @@ if_readstatus(u_char __user *buf, int len, int id, int channel)
 	if (card) {
 		if (!(card->flags & ICN_FLAGS_RUNNING))
 			return -ENODEV;
-		return (icn_readstatus(buf, len, card));
+		return icn_readstatus(buf, len, card);
 	}
 	printk(KERN_ERR
 	       "icn: if_readstatus called with invalid driverId!\n");
@@ -1503,7 +1503,7 @@ if_sendbuf(int id, int channel, int ack, struct sk_buff *skb)
 	if (card) {
 		if (!(card->flags & ICN_FLAGS_RUNNING))
 			return -ENODEV;
-		return (icn_sendbuf(channel, ack, skb, card));
+		return icn_sendbuf(channel, ack, skb, card);
 	}
 	printk(KERN_ERR
 	       "icn: if_sendbuf called with invalid driverId!\n");
@@ -1520,10 +1520,11 @@ icn_initcard(int port, char *id)
 	icn_card *card;
 	int i;
 
-	if (!(card = kzalloc(sizeof(icn_card), GFP_KERNEL))) {
+	card = kzalloc(sizeof(icn_card), GFP_KERNEL);
+	if (!card) {
 		printk(KERN_WARNING
 		       "icn: (%s) Could not allocate card-struct.\n", id);
-		return (icn_card *) 0;
+		return (icn_card *)0;
 	}
 	spin_lock_init(&card->lock);
 	card->port = port;
@@ -1555,7 +1556,7 @@ icn_initcard(int port, char *id)
 		printk(KERN_WARNING
 		       "icn: Unable to register %s\n", id);
 		kfree(card);
-		return (icn_card *) 0;
+		return (icn_card *)0;
 	}
 	card->myid = card->interface.channels;
 	sprintf(card->regname, "icn-isdn (%s)", card->interface.id);
@@ -1568,16 +1569,17 @@ icn_addcard(int port, char *id1, char *id2)
 	icn_card *card;
 	icn_card *card2;
 
-	if (!(card = icn_initcard(port, id1))) {
+	card = icn_initcard(port, id1);
+	if (!card)
 		return -EIO;
-	}
 	if (!strlen(id2)) {
 		printk(KERN_INFO
 		       "icn: (%s) ICN-2B, port 0x%x added\n",
 		       card->interface.id, port);
 		return 0;
 	}
-	if (!(card2 = icn_initcard(port, id2))) {
+	card2 = icn_initcard(port, id2);
+	if (!card2) {
 		printk(KERN_INFO
 		       "icn: (%s) half ICN-4B, port 0x%x added\n", id2, port);
 		return 0;
@@ -1611,13 +1613,14 @@ icn_setup(char *line)
 	if (str && *str) {
 		strlcpy(sid, str, sizeof(sid));
 		icn_id = sid;
-		if ((p = strchr(sid, ','))) {
+		p = strchr(sid, ',');
+		if (p) {
 			*p++ = 0;
 			strcpy(sid2, p);
 			icn_id2 = sid2;
 		}
 	}
-	return (1);
+	return 1;
 }
 __setup("icn=", icn_setup);
 #endif /* MODULE */
@@ -1634,7 +1637,8 @@ static int __init icn_init(void)
 	dev.firstload = 1;
 	spin_lock_init(&dev.devlock);
 
-	if ((p = strchr(revision, ':'))) {
+	p = strchr(revision, ':');
+	if (p) {
 		strncpy(rev, p + 1, 20);
 		rev[20] = '\0';
 		p = strchr(rev, '$');
@@ -1644,7 +1648,7 @@ static int __init icn_init(void)
 		strcpy(rev, " ??? ");
 	printk(KERN_NOTICE "ICN-ISDN-driver Rev%smem=0x%08lx\n", rev,
 	       dev.memaddr);
-	return (icn_addcard(portbase, icn_id, icn_id2));
+	return icn_addcard(portbase, icn_id, icn_id2);
 }
 
 static void __exit icn_exit(void)
diff --git a/drivers/staging/i4l/pcbit/capi.c b/drivers/staging/i4l/pcbit/capi.c
index 4e3cbf857d60..373f90feda5a 100644
--- a/drivers/staging/i4l/pcbit/capi.c
+++ b/drivers/staging/i4l/pcbit/capi.c
@@ -92,9 +92,7 @@ int capi_conn_req(const char *calledPN, struct sk_buff **skb, int proto)
 		*(skb_put(*skb, 1)) = 0x80;     /* Speech		*/
 		*(skb_put(*skb, 1)) = 0x10;     /* Circuit Mode		*/
 		*(skb_put(*skb, 1)) = 0x23;     /* A-law		*/
-	}
-	else
-	{
+	} else {
 		/* Bearer Capability - Mandatory*/
 		*(skb_put(*skb, 1)) = 2;        /* BC0.Length		*/
 		*(skb_put(*skb, 1)) = 0x88;     /* Digital Information	*/
diff --git a/drivers/staging/i4l/pcbit/drv.c b/drivers/staging/i4l/pcbit/drv.c
index c5270e229efb..d417df5efb5f 100644
--- a/drivers/staging/i4l/pcbit/drv.c
+++ b/drivers/staging/i4l/pcbit/drv.c
@@ -359,11 +359,9 @@ static int pcbit_xmit(int driver, int chnum, int ack, struct sk_buff *skb)
 		 */
 #ifdef BLOCK_TIMER
 		if (chan->block_timer.function == NULL) {
-			init_timer(&chan->block_timer);
-			chan->block_timer.function =  &pcbit_block_timer;
-			chan->block_timer.data = (long) chan;
-			chan->block_timer.expires = jiffies + 1 * HZ;
-			add_timer(&chan->block_timer);
+			setup_timer(&chan->block_timer, &pcbit_block_timer,
+				    (long)chan);
+			mod_timer(&chan->block_timer, jiffies + 1 * HZ);
 		}
 #endif
 		return 0;
@@ -804,11 +802,7 @@ static int set_protocol_running(struct pcbit_dev *dev)
 {
 	isdn_ctrl ctl;
 
-	init_timer(&dev->set_running_timer);
-
-	dev->set_running_timer.function = &set_running_timeout;
-	dev->set_running_timer.data = (ulong) dev;
-	dev->set_running_timer.expires = jiffies + SET_RUN_TIMEOUT;
+	setup_timer(&dev->set_running_timer, &set_running_timeout, (ulong)dev);
 
 	/* kick it */
 
@@ -817,7 +811,7 @@ static int set_protocol_running(struct pcbit_dev *dev)
 	writeb((0x80U | ((dev->rcv_seq & 0x07) << 3) | (dev->send_seq & 0x07)),
 	       dev->sh_mem + BANK4);
 
-	add_timer(&dev->set_running_timer);
+	mod_timer(&dev->set_running_timer, jiffies + SET_RUN_TIMEOUT);
 
 	wait_event(dev->set_running_wq, dev->l2_state == L2_RUNNING ||
 					dev->l2_state == L2_DOWN);
diff --git a/drivers/staging/i4l/pcbit/edss1.c b/drivers/staging/i4l/pcbit/edss1.c
index e72c16420712..6d291d548423 100644
--- a/drivers/staging/i4l/pcbit/edss1.c
+++ b/drivers/staging/i4l/pcbit/edss1.c
@@ -298,11 +298,8 @@ void pcbit_fsm_event(struct pcbit_dev *dev, struct pcbit_chan *chan,
 			break;
 
 	if (tentry->init != 0xff) {
-		init_timer(&chan->fsm_timer);
-		chan->fsm_timer.function = &pcbit_fsm_timer;
-		chan->fsm_timer.data = (ulong) chan;
-		chan->fsm_timer.expires = jiffies + tentry->timeout * HZ;
-		add_timer(&chan->fsm_timer);
+		setup_timer(&chan->fsm_timer, &pcbit_fsm_timer, (ulong)chan);
+		mod_timer(&chan->fsm_timer, jiffies + tentry->timeout * HZ);
 	}
 
 	spin_unlock_irqrestore(&dev->lock, flags);
diff --git a/drivers/staging/i4l/pcbit/layer2.c b/drivers/staging/i4l/pcbit/layer2.c
index 46e1240ae074..a136c72547e5 100644
--- a/drivers/staging/i4l/pcbit/layer2.c
+++ b/drivers/staging/i4l/pcbit/layer2.c
@@ -645,11 +645,9 @@ pcbit_l2_error(struct pcbit_dev *dev)
 
 		dev->l2_state = L2_DOWN;
 
-		init_timer(&dev->error_recover_timer);
-		dev->error_recover_timer.function = &pcbit_l2_err_recover;
-		dev->error_recover_timer.data = (ulong) dev;
-		dev->error_recover_timer.expires = jiffies + ERRTIME;
-		add_timer(&dev->error_recover_timer);
+		setup_timer(&dev->error_recover_timer, &pcbit_l2_err_recover,
+			    (ulong)dev);
+		mod_timer(&dev->error_recover_timer, jiffies + ERRTIME);
 	}
 }
 
diff --git a/drivers/staging/iio/accel/sca3000.h b/drivers/staging/iio/accel/sca3000.h
index 9c8a9587df7d..4dcc8575cbe3 100644
--- a/drivers/staging/iio/accel/sca3000.h
+++ b/drivers/staging/iio/accel/sca3000.h
@@ -113,6 +113,7 @@
 #define SCA3000_OUT_CTRL_BUF_X_EN		0x10
 #define SCA3000_OUT_CTRL_BUF_Y_EN		0x08
 #define SCA3000_OUT_CTRL_BUF_Z_EN		0x04
+#define SCA3000_OUT_CTRL_BUF_DIV_MASK		0x03
 #define SCA3000_OUT_CTRL_BUF_DIV_4		0x02
 #define SCA3000_OUT_CTRL_BUF_DIV_2		0x01
 
diff --git a/drivers/staging/iio/accel/sca3000_core.c b/drivers/staging/iio/accel/sca3000_core.c
index b5625f5d5e0e..d626125d7af9 100644
--- a/drivers/staging/iio/accel/sca3000_core.c
+++ b/drivers/staging/iio/accel/sca3000_core.c
@@ -402,6 +402,7 @@ static const struct iio_event_spec sca3000_event = {
 		.channel2 = mod,				\
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),\
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),\
 		.address = index,				\
 		.scan_index = index,				\
 		.scan_type = {					\
@@ -412,7 +413,7 @@ static const struct iio_event_spec sca3000_event = {
 		},						\
 		.event_spec = &sca3000_event,			\
 		.num_event_specs = 1,				\
-	 }
+	}
 
 static const struct iio_chan_spec sca3000_channels[] = {
 	SCA3000_CHAN(0, IIO_MOD_X),
@@ -443,6 +444,97 @@ static u8 sca3000_addresses[3][3] = {
 	       SCA3000_MD_CTRL_OR_Z},
 };
 
+/**
+ * __sca3000_get_base_freq() obtain mode specific base frequency
+ *
+ * lock must be held
+ **/
+static inline int __sca3000_get_base_freq(struct sca3000_state *st,
+					  const struct sca3000_chip_info *info,
+					  int *base_freq)
+{
+	int ret;
+
+	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
+	if (ret)
+		goto error_ret;
+	switch (0x03 & st->rx[0]) {
+	case SCA3000_MEAS_MODE_NORMAL:
+		*base_freq = info->measurement_mode_freq;
+		break;
+	case SCA3000_MEAS_MODE_OP_1:
+		*base_freq = info->option_mode_1_freq;
+		break;
+	case SCA3000_MEAS_MODE_OP_2:
+		*base_freq = info->option_mode_2_freq;
+		break;
+	}
+error_ret:
+	return ret;
+}
+
+/**
+ * read_raw handler for IIO_CHAN_INFO_SAMP_FREQ
+ *
+ * lock must be held
+ **/
+static int read_raw_samp_freq(struct sca3000_state *st, int *val)
+{
+	int ret;
+
+	ret = __sca3000_get_base_freq(st, st->info, val);
+	if (ret)
+		return ret;
+
+	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
+	if (ret < 0)
+		return ret;
+
+	if (*val > 0) {
+		ret &= SCA3000_OUT_CTRL_BUF_DIV_MASK;
+		switch (ret) {
+		case SCA3000_OUT_CTRL_BUF_DIV_2:
+			*val /= 2;
+			break;
+		case SCA3000_OUT_CTRL_BUF_DIV_4:
+			*val /= 4;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * write_raw handler for IIO_CHAN_INFO_SAMP_FREQ
+ *
+ * lock must be held
+ **/
+static int write_raw_samp_freq(struct sca3000_state *st, int val)
+{
+	int ret, base_freq, ctrlval;
+
+	ret = __sca3000_get_base_freq(st, st->info, &base_freq);
+	if (ret)
+		return ret;
+
+	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
+	if (ret < 0)
+		return ret;
+
+	ctrlval = ret & ~SCA3000_OUT_CTRL_BUF_DIV_MASK;
+
+	if (val == base_freq / 2)
+		ctrlval |= SCA3000_OUT_CTRL_BUF_DIV_2;
+	if (val == base_freq / 4)
+		ctrlval |= SCA3000_OUT_CTRL_BUF_DIV_4;
+	else if (val != base_freq)
+		return -EINVAL;
+
+	return sca3000_write_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL,
+				     ctrlval);
+}
+
 static int sca3000_read_raw(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *chan,
 			    int *val,
@@ -495,9 +587,36 @@ static int sca3000_read_raw(struct iio_dev *indio_dev,
 		*val = -214;
 		*val2 = 600000;
 		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		mutex_lock(&st->lock);
+		ret = read_raw_samp_freq(st, val);
+		mutex_unlock(&st->lock);
+		return ret ? ret : IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sca3000_write_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (val2)
+			return -EINVAL;
+		mutex_lock(&st->lock);
+		ret = write_raw_samp_freq(st, val);
+		mutex_unlock(&st->lock);
+		return ret;
 	default:
 		return -EINVAL;
 	}
+
+	return ret;
 }
 
 /**
@@ -548,133 +667,12 @@ error_ret:
 	return ret;
 }
 
-/**
- * __sca3000_get_base_freq() obtain mode specific base frequency
- *
- * lock must be held
- **/
-static inline int __sca3000_get_base_freq(struct sca3000_state *st,
-					  const struct sca3000_chip_info *info,
-					  int *base_freq)
-{
-	int ret;
-
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	if (ret)
-		goto error_ret;
-	switch (0x03 & st->rx[0]) {
-	case SCA3000_MEAS_MODE_NORMAL:
-		*base_freq = info->measurement_mode_freq;
-		break;
-	case SCA3000_MEAS_MODE_OP_1:
-		*base_freq = info->option_mode_1_freq;
-		break;
-	case SCA3000_MEAS_MODE_OP_2:
-		*base_freq = info->option_mode_2_freq;
-		break;
-	}
-error_ret:
-	return ret;
-}
-
-/**
- * sca3000_read_frequency() sysfs interface to get the current frequency
- **/
-static ssize_t sca3000_read_frequency(struct device *dev,
-				      struct device_attribute *attr,
-				      char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int ret, len = 0, base_freq = 0, val;
-
-	mutex_lock(&st->lock);
-	ret = __sca3000_get_base_freq(st, st->info, &base_freq);
-	if (ret)
-		goto error_ret_mut;
-	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
-	mutex_unlock(&st->lock);
-	if (ret < 0)
-		goto error_ret;
-	val = ret;
-	if (base_freq > 0)
-		switch (val & 0x03) {
-		case 0x00:
-		case 0x03:
-			len = sprintf(buf, "%d\n", base_freq);
-			break;
-		case 0x01:
-			len = sprintf(buf, "%d\n", base_freq / 2);
-			break;
-		case 0x02:
-			len = sprintf(buf, "%d\n", base_freq / 4);
-			break;
-	}
-
-	return len;
-error_ret_mut:
-	mutex_unlock(&st->lock);
-error_ret:
-	return ret;
-}
-
-/**
- * sca3000_set_frequency() sysfs interface to set the current frequency
- **/
-static ssize_t sca3000_set_frequency(struct device *dev,
-				     struct device_attribute *attr,
-				     const char *buf,
-				     size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int ret, base_freq = 0;
-	int ctrlval;
-	int val;
-
-	ret = kstrtoint(buf, 10, &val);
-	if (ret)
-		return ret;
-
-	mutex_lock(&st->lock);
-	/* What mode are we in? */
-	ret = __sca3000_get_base_freq(st, st->info, &base_freq);
-	if (ret)
-		goto error_free_lock;
-
-	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
-	if (ret < 0)
-		goto error_free_lock;
-	ctrlval = ret;
-	/* clear the bits */
-	ctrlval &= ~0x03;
-
-	if (val == base_freq / 2) {
-		ctrlval |= SCA3000_OUT_CTRL_BUF_DIV_2;
-	} else if (val == base_freq / 4) {
-		ctrlval |= SCA3000_OUT_CTRL_BUF_DIV_4;
-	} else if (val != base_freq) {
-		ret = -EINVAL;
-		goto error_free_lock;
-	}
-	ret = sca3000_write_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL,
-				     ctrlval);
-error_free_lock:
-	mutex_unlock(&st->lock);
-
-	return ret ? ret : len;
-}
-
 /*
  * Should only really be registered if ring buffer support is compiled in.
  * Does no harm however and doing it right would add a fair bit of complexity
  */
 static IIO_DEV_ATTR_SAMP_FREQ_AVAIL(sca3000_read_av_freq);
 
-static IIO_DEV_ATTR_SAMP_FREQ(S_IWUSR | S_IRUGO,
-			      sca3000_read_frequency,
-			      sca3000_set_frequency);
-
 /**
  * sca3000_read_thresh() - query of a threshold
  **/
@@ -751,7 +749,6 @@ static struct attribute *sca3000_attributes[] = {
 	&iio_dev_attr_measurement_mode_available.dev_attr.attr,
 	&iio_dev_attr_measurement_mode.dev_attr.attr,
 	&iio_dev_attr_sampling_frequency_available.dev_attr.attr,
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
 	NULL,
 };
 
@@ -1086,6 +1083,7 @@ error_ret:
 static const struct iio_info sca3000_info = {
 	.attrs = &sca3000_attribute_group,
 	.read_raw = &sca3000_read_raw,
+	.write_raw = &sca3000_write_raw,
 	.event_attrs = &sca3000_event_attribute_group,
 	.read_event_value = &sca3000_read_thresh,
 	.write_event_value = &sca3000_write_thresh,
diff --git a/drivers/staging/iio/adc/ad7280a.c b/drivers/staging/iio/adc/ad7280a.c
index 2177f1dd2b5d..b460dda7eb65 100644
--- a/drivers/staging/iio/adc/ad7280a.c
+++ b/drivers/staging/iio/adc/ad7280a.c
@@ -478,7 +478,7 @@ static ssize_t ad7280_store_balance_timer(struct device *dev,
 static struct attribute *ad7280_attributes[AD7280A_MAX_CHAIN *
 					   AD7280A_CELLS_PER_DEV * 2 + 1];
 
-static struct attribute_group ad7280_attrs_group = {
+static const struct attribute_group ad7280_attrs_group = {
 	.attrs = ad7280_attributes,
 };
 
diff --git a/drivers/staging/iio/impedance-analyzer/ad5933.c b/drivers/staging/iio/impedance-analyzer/ad5933.c
index 170ac980abcb..5eecf1cb1028 100644
--- a/drivers/staging/iio/impedance-analyzer/ad5933.c
+++ b/drivers/staging/iio/impedance-analyzer/ad5933.c
@@ -156,8 +156,7 @@ static const struct iio_chan_spec ad5933_channels[] = {
 	},
 };
 
-static int ad5933_i2c_write(struct i2c_client *client,
-			      u8 reg, u8 len, u8 *data)
+static int ad5933_i2c_write(struct i2c_client *client, u8 reg, u8 len, u8 *data)
 {
 	int ret;
 
@@ -171,8 +170,7 @@ static int ad5933_i2c_write(struct i2c_client *client,
 	return 0;
 }
 
-static int ad5933_i2c_read(struct i2c_client *client,
-			      u8 reg, u8 len, u8 *data)
+static int ad5933_i2c_read(struct i2c_client *client, u8 reg, u8 len, u8 *data)
 {
 	int ret;
 
@@ -269,7 +267,8 @@ static int ad5933_setup(struct ad5933_state *st)
 	dat = cpu_to_be16(st->settling_cycles);
 
 	ret = ad5933_i2c_write(st->client,
-			AD5933_REG_SETTLING_CYCLES, 2, (u8 *)&dat);
+			       AD5933_REG_SETTLING_CYCLES,
+			       2, (u8 *)&dat);
 	if (ret < 0)
 		return ret;
 
@@ -294,8 +293,8 @@ static void ad5933_calc_out_ranges(struct ad5933_state *st)
  */
 
 static ssize_t ad5933_show_frequency(struct device *dev,
-					struct device_attribute *attr,
-					char *buf)
+				     struct device_attribute *attr,
+				     char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad5933_state *st = iio_priv(indio_dev);
@@ -322,9 +321,9 @@ static ssize_t ad5933_show_frequency(struct device *dev,
 }
 
 static ssize_t ad5933_store_frequency(struct device *dev,
-					 struct device_attribute *attr,
-					 const char *buf,
-					 size_t len)
+				      struct device_attribute *attr,
+				      const char *buf,
+				      size_t len)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad5933_state *st = iio_priv(indio_dev);
@@ -357,8 +356,8 @@ static IIO_DEVICE_ATTR(out_voltage0_freq_increment, S_IRUGO | S_IWUSR,
 			AD5933_REG_FREQ_INC);
 
 static ssize_t ad5933_show(struct device *dev,
-					struct device_attribute *attr,
-					char *buf)
+			   struct device_attribute *attr,
+			   char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad5933_state *st = iio_priv(indio_dev);
@@ -399,9 +398,9 @@ static ssize_t ad5933_show(struct device *dev,
 }
 
 static ssize_t ad5933_store(struct device *dev,
-					 struct device_attribute *attr,
-					 const char *buf,
-					 size_t len)
+			    struct device_attribute *attr,
+			    const char *buf,
+			    size_t len)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad5933_state *st = iio_priv(indio_dev);
@@ -419,6 +418,7 @@ static ssize_t ad5933_store(struct device *dev,
 	mutex_lock(&indio_dev->mlock);
 	switch ((u32)this_attr->address) {
 	case AD5933_OUT_RANGE:
+		ret = -EINVAL;
 		for (i = 0; i < 4; i++)
 			if (val == st->range_avail[i]) {
 				st->ctrl_hb &= ~AD5933_CTRL_RANGE(0x3);
@@ -426,7 +426,6 @@ static ssize_t ad5933_store(struct device *dev,
 				ret = ad5933_cmd(st, 0);
 				break;
 			}
-		ret = -EINVAL;
 		break;
 	case AD5933_IN_PGA_GAIN:
 		if (sysfs_streq(buf, "1")) {
@@ -451,7 +450,8 @@ static ssize_t ad5933_store(struct device *dev,
 
 		dat = cpu_to_be16(val);
 		ret = ad5933_i2c_write(st->client,
-				AD5933_REG_SETTLING_CYCLES, 2, (u8 *)&dat);
+				       AD5933_REG_SETTLING_CYCLES,
+				       2, (u8 *)&dat);
 		break;
 	case AD5933_FREQ_POINTS:
 		val = clamp(val, (u16)0, (u16)511);
@@ -545,8 +545,8 @@ static int ad5933_read_raw(struct iio_dev *indio_dev,
 			goto out;
 
 		ret = ad5933_i2c_read(st->client,
-				AD5933_REG_TEMP_DATA, 2,
-				(u8 *)&dat);
+				      AD5933_REG_TEMP_DATA,
+				      2, (u8 *)&dat);
 		if (ret < 0)
 			goto out;
 		mutex_unlock(&indio_dev->mlock);
@@ -705,7 +705,7 @@ static void ad5933_work(struct work_struct *work)
 }
 
 static int ad5933_probe(struct i2c_client *client,
-				   const struct i2c_device_id *id)
+			const struct i2c_device_id *id)
 {
 	int ret, voltage_uv = 0;
 	struct ad5933_platform_data *pdata = dev_get_platdata(&client->dev);
diff --git a/drivers/staging/iio/light/isl29018.c b/drivers/staging/iio/light/isl29018.c
index 76d9f74e7dcb..a767a43c995c 100644
--- a/drivers/staging/iio/light/isl29018.c
+++ b/drivers/staging/iio/light/isl29018.c
@@ -15,10 +15,6 @@
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA	02110-1301, USA.
  */
 
 #include <linux/module.h>
@@ -32,25 +28,25 @@
 #include <linux/iio/sysfs.h>
 #include <linux/acpi.h>
 
-#define CONVERSION_TIME_MS		100
+#define ISL29018_CONV_TIME_MS		100
 
 #define ISL29018_REG_ADD_COMMAND1	0x00
-#define COMMMAND1_OPMODE_SHIFT		5
-#define COMMMAND1_OPMODE_MASK		(7 << COMMMAND1_OPMODE_SHIFT)
-#define COMMMAND1_OPMODE_POWER_DOWN	0
-#define COMMMAND1_OPMODE_ALS_ONCE	1
-#define COMMMAND1_OPMODE_IR_ONCE	2
-#define COMMMAND1_OPMODE_PROX_ONCE	3
+#define ISL29018_CMD1_OPMODE_SHIFT	5
+#define ISL29018_CMD1_OPMODE_MASK	(7 << ISL29018_CMD1_OPMODE_SHIFT)
+#define ISL29018_CMD1_OPMODE_POWER_DOWN	0
+#define ISL29018_CMD1_OPMODE_ALS_ONCE	1
+#define ISL29018_CMD1_OPMODE_IR_ONCE	2
+#define ISL29018_CMD1_OPMODE_PROX_ONCE	3
 
-#define ISL29018_REG_ADD_COMMANDII	0x01
-#define COMMANDII_RESOLUTION_SHIFT	2
-#define COMMANDII_RESOLUTION_MASK	(0x3 << COMMANDII_RESOLUTION_SHIFT)
+#define ISL29018_REG_ADD_COMMAND2	0x01
+#define ISL29018_CMD2_RESOLUTION_SHIFT	2
+#define ISL29018_CMD2_RESOLUTION_MASK	(0x3 << ISL29018_CMD2_RESOLUTION_SHIFT)
 
-#define COMMANDII_RANGE_SHIFT		0
-#define COMMANDII_RANGE_MASK		(0x3 << COMMANDII_RANGE_SHIFT)
+#define ISL29018_CMD2_RANGE_SHIFT	0
+#define ISL29018_CMD2_RANGE_MASK	(0x3 << ISL29018_CMD2_RANGE_SHIFT)
 
-#define COMMANDII_SCHEME_SHIFT		7
-#define COMMANDII_SCHEME_MASK		(0x1 << COMMANDII_SCHEME_SHIFT)
+#define ISL29018_CMD2_SCHEME_SHIFT	7
+#define ISL29018_CMD2_SCHEME_MASK	(0x1 << ISL29018_CMD2_SCHEME_SHIFT)
 
 #define ISL29018_REG_ADD_DATA_LSB	0x02
 #define ISL29018_REG_ADD_DATA_MSB	0x03
@@ -127,13 +123,13 @@ static int isl29018_set_integration_time(struct isl29018_chip *chip,
 	if (i >= ARRAY_SIZE(isl29018_int_utimes[chip->type]))
 		return -EINVAL;
 
-	ret = regmap_update_bits(chip->regmap, ISL29018_REG_ADD_COMMANDII,
-				 COMMANDII_RESOLUTION_MASK,
-				 i << COMMANDII_RESOLUTION_SHIFT);
+	ret = regmap_update_bits(chip->regmap, ISL29018_REG_ADD_COMMAND2,
+				 ISL29018_CMD2_RESOLUTION_MASK,
+				 i << ISL29018_CMD2_RESOLUTION_SHIFT);
 	if (ret < 0)
 		return ret;
 
-	/* keep the same range when integration time changes */
+	/* Keep the same range when integration time changes */
 	int_time = chip->int_time;
 	for (i = 0; i < ARRAY_SIZE(isl29018_scales[int_time]); ++i) {
 		if (chip->scale.scale == isl29018_scales[int_time][i].scale &&
@@ -163,9 +159,9 @@ static int isl29018_set_scale(struct isl29018_chip *chip, int scale, int uscale)
 	if (i >= ARRAY_SIZE(isl29018_scales[chip->int_time]))
 		return -EINVAL;
 
-	ret = regmap_update_bits(chip->regmap, ISL29018_REG_ADD_COMMANDII,
-				 COMMANDII_RANGE_MASK,
-				 i << COMMANDII_RANGE_SHIFT);
+	ret = regmap_update_bits(chip->regmap, ISL29018_REG_ADD_COMMAND2,
+				 ISL29018_CMD2_RANGE_MASK,
+				 i << ISL29018_CMD2_RANGE_SHIFT);
 	if (ret < 0)
 		return ret;
 
@@ -183,13 +179,13 @@ static int isl29018_read_sensor_input(struct isl29018_chip *chip, int mode)
 
 	/* Set mode */
 	status = regmap_write(chip->regmap, ISL29018_REG_ADD_COMMAND1,
-			      mode << COMMMAND1_OPMODE_SHIFT);
+			      mode << ISL29018_CMD1_OPMODE_SHIFT);
 	if (status) {
 		dev_err(dev,
 			"Error in setting operating mode err %d\n", status);
 		return status;
 	}
-	msleep(CONVERSION_TIME_MS);
+	msleep(ISL29018_CONV_TIME_MS);
 	status = regmap_read(chip->regmap, ISL29018_REG_ADD_DATA_LSB, &lsb);
 	if (status < 0) {
 		dev_err(dev,
@@ -213,8 +209,8 @@ static int isl29018_read_lux(struct isl29018_chip *chip, int *lux)
 	int lux_data;
 	unsigned int data_x_range;
 
-	lux_data = isl29018_read_sensor_input(chip, COMMMAND1_OPMODE_ALS_ONCE);
-
+	lux_data = isl29018_read_sensor_input(chip,
+					      ISL29018_CMD1_OPMODE_ALS_ONCE);
 	if (lux_data < 0)
 		return lux_data;
 
@@ -230,8 +226,8 @@ static int isl29018_read_ir(struct isl29018_chip *chip, int *ir)
 {
 	int ir_data;
 
-	ir_data = isl29018_read_sensor_input(chip, COMMMAND1_OPMODE_IR_ONCE);
-
+	ir_data = isl29018_read_sensor_input(chip,
+					     ISL29018_CMD1_OPMODE_IR_ONCE);
 	if (ir_data < 0)
 		return ir_data;
 
@@ -249,16 +245,16 @@ static int isl29018_read_proximity_ir(struct isl29018_chip *chip, int scheme,
 	struct device *dev = regmap_get_device(chip->regmap);
 
 	/* Do proximity sensing with required scheme */
-	status = regmap_update_bits(chip->regmap, ISL29018_REG_ADD_COMMANDII,
-				    COMMANDII_SCHEME_MASK,
-				    scheme << COMMANDII_SCHEME_SHIFT);
+	status = regmap_update_bits(chip->regmap, ISL29018_REG_ADD_COMMAND2,
+				    ISL29018_CMD2_SCHEME_MASK,
+				    scheme << ISL29018_CMD2_SCHEME_SHIFT);
 	if (status) {
 		dev_err(dev, "Error in setting operating mode\n");
 		return status;
 	}
 
 	prox_data = isl29018_read_sensor_input(chip,
-					       COMMMAND1_OPMODE_PROX_ONCE);
+					       ISL29018_CMD1_OPMODE_PROX_ONCE);
 	if (prox_data < 0)
 		return prox_data;
 
@@ -267,8 +263,8 @@ static int isl29018_read_proximity_ir(struct isl29018_chip *chip, int scheme,
 		return 0;
 	}
 
-	ir_data = isl29018_read_sensor_input(chip, COMMMAND1_OPMODE_IR_ONCE);
-
+	ir_data = isl29018_read_sensor_input(chip,
+					     ISL29018_CMD1_OPMODE_IR_ONCE);
 	if (ir_data < 0)
 		return ir_data;
 
@@ -280,7 +276,7 @@ static int isl29018_read_proximity_ir(struct isl29018_chip *chip, int scheme,
 	return 0;
 }
 
-static ssize_t show_scale_available(struct device *dev,
+static ssize_t isl29018_show_scale_available(struct device *dev,
 				    struct device_attribute *attr, char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
@@ -297,7 +293,7 @@ static ssize_t show_scale_available(struct device *dev,
 	return len;
 }
 
-static ssize_t show_int_time_available(struct device *dev,
+static ssize_t isl29018_show_int_time_available(struct device *dev,
 				       struct device_attribute *attr, char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
@@ -313,8 +309,7 @@ static ssize_t show_int_time_available(struct device *dev,
 	return len;
 }
 
-/* proximity scheme */
-static ssize_t show_prox_infrared_suppression(struct device *dev,
+static ssize_t isl29018_show_prox_infrared_suppression(struct device *dev,
 					      struct device_attribute *attr,
 					      char *buf)
 {
@@ -322,13 +317,13 @@ static ssize_t show_prox_infrared_suppression(struct device *dev,
 	struct isl29018_chip *chip = iio_priv(indio_dev);
 
 	/*
-	 * return the "proximity scheme" i.e. if the chip does on chip
+	 * Return the "proximity scheme" i.e. if the chip does on chip
 	 * infrared suppression (1 means perform on chip suppression)
 	 */
 	return sprintf(buf, "%d\n", chip->prox_scheme);
 }
 
-static ssize_t store_prox_infrared_suppression(struct device *dev,
+static ssize_t isl29018_store_prox_infrared_suppression(struct device *dev,
 					       struct device_attribute *attr,
 					       const char *buf, size_t count)
 {
@@ -338,13 +333,11 @@ static ssize_t store_prox_infrared_suppression(struct device *dev,
 
 	if (kstrtoint(buf, 10, &val))
 		return -EINVAL;
-	if (!(val == 0 || val == 1)) {
-		dev_err(dev, "The mode is not supported\n");
+	if (!(val == 0 || val == 1))
 		return -EINVAL;
-	}
 
 	/*
-	 * get the  "proximity scheme" i.e. if the chip does on chip
+	 * Get the "proximity scheme" i.e. if the chip does on chip
 	 * infrared suppression (1 means perform on chip suppression)
 	 */
 	mutex_lock(&chip->lock);
@@ -354,7 +347,6 @@ static ssize_t store_prox_infrared_suppression(struct device *dev,
 	return count;
 }
 
-/* Channel IO */
 static int isl29018_write_raw(struct iio_dev *indio_dev,
 			      struct iio_chan_spec const *chan,
 			      int val,
@@ -491,13 +483,13 @@ static const struct iio_chan_spec isl29023_channels[] = {
 };
 
 static IIO_DEVICE_ATTR(in_illuminance_integration_time_available, S_IRUGO,
-		       show_int_time_available, NULL, 0);
+		       isl29018_show_int_time_available, NULL, 0);
 static IIO_DEVICE_ATTR(in_illuminance_scale_available, S_IRUGO,
-		      show_scale_available, NULL, 0);
+		      isl29018_show_scale_available, NULL, 0);
 static IIO_DEVICE_ATTR(proximity_on_chip_ambient_infrared_suppression,
 					S_IRUGO | S_IWUSR,
-					show_prox_infrared_suppression,
-					store_prox_infrared_suppression, 0);
+					isl29018_show_prox_infrared_suppression,
+					isl29018_store_prox_infrared_suppression, 0);
 
 #define ISL29018_DEV_ATTR(name) (&iio_dev_attr_##name.dev_attr.attr)
 
@@ -541,7 +533,7 @@ static int isl29035_detect(struct isl29018_chip *chip)
 	if (id != ISL29035_DEVICE_ID)
 		return -ENODEV;
 
-	/* clear out brownout bit */
+	/* Clear brownout bit */
 	return regmap_update_bits(chip->regmap, ISL29035_REG_DEVICE_ID,
 				  ISL29035_BOUT_MASK, 0);
 }
@@ -574,7 +566,7 @@ static int isl29018_chip_init(struct isl29018_chip *chip)
 	 * conversions, clear the test registers, and then rewrite all
 	 * registers to the desired values.
 	 * ...
-	 * FOR ISL29011, ISL29018, ISL29021, ISL29023
+	 * For ISL29011, ISL29018, ISL29021, ISL29023
 	 * 1. Write 0x00 to register 0x08 (TEST)
 	 * 2. Write 0x00 to register 0x00 (CMD1)
 	 * 3. Rewrite all registers to the desired values
@@ -603,7 +595,7 @@ static int isl29018_chip_init(struct isl29018_chip *chip)
 
 	usleep_range(1000, 2000);	/* per data sheet, page 10 */
 
-	/* set defaults */
+	/* Set defaults */
 	status = isl29018_set_scale(chip, chip->scale.scale,
 				    chip->scale.uscale);
 	if (status < 0) {
@@ -635,7 +627,7 @@ static const struct iio_info isl29023_info = {
 	.write_raw = isl29018_write_raw,
 };
 
-static bool is_volatile_reg(struct device *dev, unsigned int reg)
+static bool isl29018_is_volatile_reg(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
 	case ISL29018_REG_ADD_DATA_LSB:
@@ -649,37 +641,32 @@ static bool is_volatile_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-/*
- * isl29018_regmap_config: regmap configuration.
- * Use RBTREE mechanism for caching.
- */
 static const struct regmap_config isl29018_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
-	.volatile_reg = is_volatile_reg,
+	.volatile_reg = isl29018_is_volatile_reg,
 	.max_register = ISL29018_REG_TEST,
 	.num_reg_defaults_raw = ISL29018_REG_TEST + 1,
 	.cache_type = REGCACHE_RBTREE,
 };
 
-/* isl29035_regmap_config: regmap configuration for ISL29035 */
 static const struct regmap_config isl29035_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
-	.volatile_reg = is_volatile_reg,
+	.volatile_reg = isl29018_is_volatile_reg,
 	.max_register = ISL29035_REG_DEVICE_ID,
 	.num_reg_defaults_raw = ISL29035_REG_DEVICE_ID + 1,
 	.cache_type = REGCACHE_RBTREE,
 };
 
-struct chip_info {
+struct isl29018_chip_info {
 	const struct iio_chan_spec *channels;
 	int num_channels;
 	const struct iio_info *indio_info;
 	const struct regmap_config *regmap_cfg;
 };
 
-static const struct chip_info chip_info_tbl[] = {
+static const struct isl29018_chip_info isl29018_chip_info_tbl[] = {
 	[isl29018] = {
 		.channels = isl29018_channels,
 		.num_channels = ARRAY_SIZE(isl29018_channels),
@@ -724,10 +711,8 @@ static int isl29018_probe(struct i2c_client *client,
 	int dev_id = 0;
 
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*chip));
-	if (!indio_dev) {
-		dev_err(&client->dev, "iio allocation fails\n");
+	if (!indio_dev)
 		return -ENOMEM;
-	}
 	chip = iio_priv(indio_dev);
 
 	i2c_set_clientdata(client, indio_dev);
@@ -750,7 +735,7 @@ static int isl29018_probe(struct i2c_client *client,
 	chip->suspended = false;
 
 	chip->regmap = devm_regmap_init_i2c(client,
-				chip_info_tbl[dev_id].regmap_cfg);
+				isl29018_chip_info_tbl[dev_id].regmap_cfg);
 	if (IS_ERR(chip->regmap)) {
 		err = PTR_ERR(chip->regmap);
 		dev_err(&client->dev, "regmap initialization fails: %d\n", err);
@@ -761,19 +746,13 @@ static int isl29018_probe(struct i2c_client *client,
 	if (err)
 		return err;
 
-	indio_dev->info = chip_info_tbl[dev_id].indio_info;
-	indio_dev->channels = chip_info_tbl[dev_id].channels;
-	indio_dev->num_channels = chip_info_tbl[dev_id].num_channels;
+	indio_dev->info = isl29018_chip_info_tbl[dev_id].indio_info;
+	indio_dev->channels = isl29018_chip_info_tbl[dev_id].channels;
+	indio_dev->num_channels = isl29018_chip_info_tbl[dev_id].num_channels;
 	indio_dev->name = name;
 	indio_dev->dev.parent = &client->dev;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	err = devm_iio_device_register(&client->dev, indio_dev);
-	if (err) {
-		dev_err(&client->dev, "iio registration fails\n");
-		return err;
-	}
-
-	return 0;
+	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -840,7 +819,6 @@ static const struct of_device_id isl29018_of_match[] = {
 MODULE_DEVICE_TABLE(of, isl29018_of_match);
 
 static struct i2c_driver isl29018_driver = {
-	.class	= I2C_CLASS_HWMON,
 	.driver	 = {
 			.name = "isl29018",
 			.acpi_match_table = ACPI_PTR(isl29018_acpi_match),
diff --git a/drivers/staging/iio/light/isl29028.c b/drivers/staging/iio/light/isl29028.c
index 2e3b1d64e32a..aa413e5878b9 100644
--- a/drivers/staging/iio/light/isl29028.c
+++ b/drivers/staging/iio/light/isl29028.c
@@ -27,29 +27,27 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
-#define CONVERSION_TIME_MS		100
+#define ISL29028_CONV_TIME_MS		100
 
 #define ISL29028_REG_CONFIGURE		0x01
 
-#define CONFIGURE_ALS_IR_MODE_ALS	0
-#define CONFIGURE_ALS_IR_MODE_IR	BIT(0)
-#define CONFIGURE_ALS_IR_MODE_MASK	BIT(0)
+#define ISL29028_CONF_ALS_IR_MODE_ALS	0
+#define ISL29028_CONF_ALS_IR_MODE_IR	BIT(0)
+#define ISL29028_CONF_ALS_IR_MODE_MASK	BIT(0)
 
-#define CONFIGURE_ALS_RANGE_LOW_LUX	0
-#define CONFIGURE_ALS_RANGE_HIGH_LUX	BIT(1)
-#define CONFIGURE_ALS_RANGE_MASK	BIT(1)
+#define ISL29028_CONF_ALS_RANGE_LOW_LUX	0
+#define ISL29028_CONF_ALS_RANGE_HIGH_LUX	BIT(1)
+#define ISL29028_CONF_ALS_RANGE_MASK	BIT(1)
 
-#define CONFIGURE_ALS_DIS		0
-#define CONFIGURE_ALS_EN		BIT(2)
-#define CONFIGURE_ALS_EN_MASK		BIT(2)
+#define ISL29028_CONF_ALS_DIS		0
+#define ISL29028_CONF_ALS_EN		BIT(2)
+#define ISL29028_CONF_ALS_EN_MASK	BIT(2)
 
-#define CONFIGURE_PROX_DRIVE		BIT(3)
+#define ISL29028_CONF_PROX_SLP_SH	4
+#define ISL29028_CONF_PROX_SLP_MASK	(7 << ISL29028_CONF_PROX_SLP_SH)
 
-#define CONFIGURE_PROX_SLP_SH		4
-#define CONFIGURE_PROX_SLP_MASK		(7 << CONFIGURE_PROX_SLP_SH)
-
-#define CONFIGURE_PROX_EN		BIT(7)
-#define CONFIGURE_PROX_EN_MASK		BIT(7)
+#define ISL29028_CONF_PROX_EN		BIT(7)
+#define ISL29028_CONF_PROX_EN_MASK	BIT(7)
 
 #define ISL29028_REG_INTERRUPT		0x02
 
@@ -62,10 +60,10 @@
 
 #define ISL29028_NUM_REGS		(ISL29028_REG_TEST2_MODE + 1)
 
-enum als_ir_mode {
-	MODE_NONE = 0,
-	MODE_ALS,
-	MODE_IR
+enum isl29028_als_ir_mode {
+	ISL29028_MODE_NONE = 0,
+	ISL29028_MODE_ALS,
+	ISL29028_MODE_IR,
 };
 
 struct isl29028_chip {
@@ -76,7 +74,7 @@ struct isl29028_chip {
 	bool			enable_prox;
 
 	int			lux_scale;
-	int			als_ir_mode;
+	enum isl29028_als_ir_mode	als_ir_mode;
 };
 
 static int isl29028_set_proxim_sampling(struct isl29028_chip *chip,
@@ -91,7 +89,8 @@ static int isl29028_set_proxim_sampling(struct isl29028_chip *chip,
 			break;
 	}
 	return regmap_update_bits(chip->regmap, ISL29028_REG_CONFIGURE,
-			CONFIGURE_PROX_SLP_MASK, sel << CONFIGURE_PROX_SLP_SH);
+				  ISL29028_CONF_PROX_SLP_MASK,
+				  sel << ISL29028_CONF_PROX_SLP_SH);
 }
 
 static int isl29028_enable_proximity(struct isl29028_chip *chip, bool enable)
@@ -100,9 +99,9 @@ static int isl29028_enable_proximity(struct isl29028_chip *chip, bool enable)
 	int val = 0;
 
 	if (enable)
-		val = CONFIGURE_PROX_EN;
+		val = ISL29028_CONF_PROX_EN;
 	ret = regmap_update_bits(chip->regmap, ISL29028_REG_CONFIGURE,
-				 CONFIGURE_PROX_EN_MASK, val);
+				 ISL29028_CONF_PROX_EN_MASK, val);
 	if (ret < 0)
 		return ret;
 
@@ -113,40 +112,40 @@ static int isl29028_enable_proximity(struct isl29028_chip *chip, bool enable)
 
 static int isl29028_set_als_scale(struct isl29028_chip *chip, int lux_scale)
 {
-	int val = (lux_scale == 2000) ? CONFIGURE_ALS_RANGE_HIGH_LUX :
-					CONFIGURE_ALS_RANGE_LOW_LUX;
+	int val = (lux_scale == 2000) ? ISL29028_CONF_ALS_RANGE_HIGH_LUX :
+					ISL29028_CONF_ALS_RANGE_LOW_LUX;
 
 	return regmap_update_bits(chip->regmap, ISL29028_REG_CONFIGURE,
-		CONFIGURE_ALS_RANGE_MASK, val);
+		ISL29028_CONF_ALS_RANGE_MASK, val);
 }
 
 static int isl29028_set_als_ir_mode(struct isl29028_chip *chip,
-				    enum als_ir_mode mode)
+				    enum isl29028_als_ir_mode mode)
 {
 	int ret = 0;
 
 	switch (mode) {
-	case MODE_ALS:
+	case ISL29028_MODE_ALS:
 		ret = regmap_update_bits(chip->regmap, ISL29028_REG_CONFIGURE,
-					 CONFIGURE_ALS_IR_MODE_MASK,
-					 CONFIGURE_ALS_IR_MODE_ALS);
+					 ISL29028_CONF_ALS_IR_MODE_MASK,
+					 ISL29028_CONF_ALS_IR_MODE_ALS);
 		if (ret < 0)
 			return ret;
 
 		ret = regmap_update_bits(chip->regmap, ISL29028_REG_CONFIGURE,
-					 CONFIGURE_ALS_RANGE_MASK,
-					 CONFIGURE_ALS_RANGE_HIGH_LUX);
+					 ISL29028_CONF_ALS_RANGE_MASK,
+					 ISL29028_CONF_ALS_RANGE_HIGH_LUX);
 		break;
 
-	case MODE_IR:
+	case ISL29028_MODE_IR:
 		ret = regmap_update_bits(chip->regmap, ISL29028_REG_CONFIGURE,
-					 CONFIGURE_ALS_IR_MODE_MASK,
-					 CONFIGURE_ALS_IR_MODE_IR);
+					 ISL29028_CONF_ALS_IR_MODE_MASK,
+					 ISL29028_CONF_ALS_IR_MODE_IR);
 		break;
 
-	case MODE_NONE:
+	case ISL29028_MODE_NONE:
 		return regmap_update_bits(chip->regmap, ISL29028_REG_CONFIGURE,
-			CONFIGURE_ALS_EN_MASK, CONFIGURE_ALS_DIS);
+			ISL29028_CONF_ALS_EN_MASK, ISL29028_CONF_ALS_DIS);
 	}
 
 	if (ret < 0)
@@ -154,12 +153,13 @@ static int isl29028_set_als_ir_mode(struct isl29028_chip *chip,
 
 	/* Enable the ALS/IR */
 	ret = regmap_update_bits(chip->regmap, ISL29028_REG_CONFIGURE,
-				 CONFIGURE_ALS_EN_MASK, CONFIGURE_ALS_EN);
+				 ISL29028_CONF_ALS_EN_MASK,
+				 ISL29028_CONF_ALS_EN);
 	if (ret < 0)
 		return ret;
 
 	/* Need to wait for conversion time if ALS/IR mode enabled */
-	mdelay(CONVERSION_TIME_MS);
+	mdelay(ISL29028_CONV_TIME_MS);
 	return 0;
 }
 
@@ -223,14 +223,14 @@ static int isl29028_als_get(struct isl29028_chip *chip, int *als_data)
 	int ret;
 	int als_ir_data;
 
-	if (chip->als_ir_mode != MODE_ALS) {
-		ret = isl29028_set_als_ir_mode(chip, MODE_ALS);
+	if (chip->als_ir_mode != ISL29028_MODE_ALS) {
+		ret = isl29028_set_als_ir_mode(chip, ISL29028_MODE_ALS);
 		if (ret < 0) {
 			dev_err(dev,
 				"Error in enabling ALS mode err %d\n", ret);
 			return ret;
 		}
-		chip->als_ir_mode = MODE_ALS;
+		chip->als_ir_mode = ISL29028_MODE_ALS;
 	}
 
 	ret = isl29028_read_als_ir(chip, &als_ir_data);
@@ -256,14 +256,14 @@ static int isl29028_ir_get(struct isl29028_chip *chip, int *ir_data)
 	struct device *dev = regmap_get_device(chip->regmap);
 	int ret;
 
-	if (chip->als_ir_mode != MODE_IR) {
-		ret = isl29028_set_als_ir_mode(chip, MODE_IR);
+	if (chip->als_ir_mode != ISL29028_MODE_IR) {
+		ret = isl29028_set_als_ir_mode(chip, ISL29028_MODE_IR);
 		if (ret < 0) {
 			dev_err(dev,
 				"Error in enabling IR mode err %d\n", ret);
 			return ret;
 		}
-		chip->als_ir_mode = MODE_IR;
+		chip->als_ir_mode = ISL29028_MODE_IR;
 	}
 	return isl29028_read_als_ir(chip, ir_data);
 }
@@ -383,8 +383,8 @@ static int isl29028_read_raw(struct iio_dev *indio_dev,
 }
 
 static IIO_CONST_ATTR(in_proximity_sampling_frequency_available,
-				"1, 3, 5, 10, 13, 20, 83, 100");
-static IIO_CONST_ATTR(in_illuminance_scale_available, "125, 2000");
+				"1 3 5 10 13 20 83 100");
+static IIO_CONST_ATTR(in_illuminance_scale_available, "125 2000");
 
 #define ISL29028_DEV_ATTR(name) (&iio_dev_attr_##name.dev_attr.attr)
 #define ISL29028_CONST_ATTR(name) (&iio_const_attr_##name.dev_attr.attr)
@@ -428,7 +428,7 @@ static int isl29028_chip_init(struct isl29028_chip *chip)
 	chip->enable_prox  = false;
 	chip->prox_sampling = 20;
 	chip->lux_scale = 2000;
-	chip->als_ir_mode = MODE_NONE;
+	chip->als_ir_mode = ISL29028_MODE_NONE;
 
 	ret = regmap_write(chip->regmap, ISL29028_REG_TEST1_MODE, 0x0);
 	if (ret < 0) {
@@ -462,7 +462,7 @@ static int isl29028_chip_init(struct isl29028_chip *chip)
 	return ret;
 }
 
-static bool is_volatile_reg(struct device *dev, unsigned int reg)
+static bool isl29028_is_volatile_reg(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
 	case ISL29028_REG_INTERRUPT:
@@ -478,7 +478,7 @@ static bool is_volatile_reg(struct device *dev, unsigned int reg)
 static const struct regmap_config isl29028_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
-	.volatile_reg = is_volatile_reg,
+	.volatile_reg = isl29028_is_volatile_reg,
 	.max_register = ISL29028_NUM_REGS - 1,
 	.num_reg_defaults_raw = ISL29028_NUM_REGS,
 	.cache_type = REGCACHE_RBTREE,
@@ -546,7 +546,6 @@ static const struct of_device_id isl29028_of_match[] = {
 MODULE_DEVICE_TABLE(of, isl29028_of_match);
 
 static struct i2c_driver isl29028_driver = {
-	.class	= I2C_CLASS_HWMON,
 	.driver  = {
 		.name = "isl29028",
 		.of_match_table = isl29028_of_match,
diff --git a/drivers/staging/iio/light/tsl2583.c b/drivers/staging/iio/light/tsl2583.c
index 05b4ad4e941c..08f1583ee34e 100644
--- a/drivers/staging/iio/light/tsl2583.c
+++ b/drivers/staging/iio/light/tsl2583.c
@@ -803,7 +803,7 @@ static struct attribute *sysfs_attrs_ctrl[] = {
 	NULL
 };
 
-static struct attribute_group tsl2583_attribute_group = {
+static const struct attribute_group tsl2583_attribute_group = {
 	.attrs = sysfs_attrs_ctrl,
 };
 
diff --git a/drivers/staging/iio/meter/ade7754.c b/drivers/staging/iio/meter/ade7754.c
index c46bef641613..17309591ca57 100644
--- a/drivers/staging/iio/meter/ade7754.c
+++ b/drivers/staging/iio/meter/ade7754.c
@@ -23,9 +23,7 @@
 #include "meter.h"
 #include "ade7754.h"
 
-static int ade7754_spi_write_reg_8(struct device *dev,
-		u8 reg_address,
-		u8 val)
+static int ade7754_spi_write_reg_8(struct device *dev, u8 reg_address, u8 val)
 {
 	int ret;
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
@@ -42,8 +40,7 @@ static int ade7754_spi_write_reg_8(struct device *dev,
 }
 
 static int ade7754_spi_write_reg_16(struct device *dev,
-		u8 reg_address,
-		u16 value)
+				    u8 reg_address, u16 value)
 {
 	int ret;
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
@@ -59,9 +56,7 @@ static int ade7754_spi_write_reg_16(struct device *dev,
 	return ret;
 }
 
-static int ade7754_spi_read_reg_8(struct device *dev,
-		u8 reg_address,
-		u8 *val)
+static int ade7754_spi_read_reg_8(struct device *dev, u8 reg_address, u8 *val)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7754_state *st = iio_priv(indio_dev);
@@ -70,7 +65,7 @@ static int ade7754_spi_read_reg_8(struct device *dev,
 	ret = spi_w8r8(st->us, ADE7754_READ_REG(reg_address));
 	if (ret < 0) {
 		dev_err(&st->us->dev, "problem when reading 8 bit register 0x%02X",
-				reg_address);
+			reg_address);
 		return ret;
 	}
 	*val = ret;
@@ -79,8 +74,7 @@ static int ade7754_spi_read_reg_8(struct device *dev,
 }
 
 static int ade7754_spi_read_reg_16(struct device *dev,
-		u8 reg_address,
-		u16 *val)
+				   u8 reg_address, u16 *val)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7754_state *st = iio_priv(indio_dev);
@@ -99,8 +93,7 @@ static int ade7754_spi_read_reg_16(struct device *dev,
 }
 
 static int ade7754_spi_read_reg_24(struct device *dev,
-		u8 reg_address,
-		u32 *val)
+				   u8 reg_address, u32 *val)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7754_state *st = iio_priv(indio_dev);
@@ -123,7 +116,7 @@ static int ade7754_spi_read_reg_24(struct device *dev,
 	ret = spi_sync_transfer(st->us, xfers, ARRAY_SIZE(xfers));
 	if (ret) {
 		dev_err(&st->us->dev, "problem when reading 24 bit register 0x%02X",
-				reg_address);
+			reg_address);
 		goto error_ret;
 	}
 	*val = (st->rx[1] << 16) | (st->rx[2] << 8) | st->rx[3];
@@ -134,8 +127,8 @@ error_ret:
 }
 
 static ssize_t ade7754_read_8bit(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				 struct device_attribute *attr,
+				 char *buf)
 {
 	int ret;
 	u8 val = 0;
@@ -149,8 +142,8 @@ static ssize_t ade7754_read_8bit(struct device *dev,
 }
 
 static ssize_t ade7754_read_16bit(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				  struct device_attribute *attr,
+				  char *buf)
 {
 	int ret;
 	u16 val = 0;
@@ -164,8 +157,8 @@ static ssize_t ade7754_read_16bit(struct device *dev,
 }
 
 static ssize_t ade7754_read_24bit(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				  struct device_attribute *attr,
+				  char *buf)
 {
 	int ret;
 	u32 val = 0;
@@ -179,9 +172,9 @@ static ssize_t ade7754_read_24bit(struct device *dev,
 }
 
 static ssize_t ade7754_write_8bit(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
+				  struct device_attribute *attr,
+				  const char *buf,
+				  size_t len)
 {
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 	int ret;
@@ -197,9 +190,9 @@ error_ret:
 }
 
 static ssize_t ade7754_write_16bit(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
+				   struct device_attribute *attr,
+				   const char *buf,
+				   size_t len)
 {
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 	int ret;
@@ -403,16 +396,14 @@ err_ret:
 }
 
 static ssize_t ade7754_read_frequency(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				      struct device_attribute *attr,
+				      char *buf)
 {
 	int ret;
 	u8 t;
 	int sps;
 
-	ret = ade7754_spi_read_reg_8(dev,
-			ADE7754_WAVMODE,
-			&t);
+	ret = ade7754_spi_read_reg_8(dev, ADE7754_WAVMODE, &t);
 	if (ret)
 		return ret;
 
@@ -423,9 +414,9 @@ static ssize_t ade7754_read_frequency(struct device *dev,
 }
 
 static ssize_t ade7754_write_frequency(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
+				       struct device_attribute *attr,
+				       const char *buf,
+				       size_t len)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ade7754_state *st = iio_priv(indio_dev);
diff --git a/drivers/staging/iio/meter/ade7758_ring.c b/drivers/staging/iio/meter/ade7758_ring.c
index a6b76d4b1c80..57c213dfadcc 100644
--- a/drivers/staging/iio/meter/ade7758_ring.c
+++ b/drivers/staging/iio/meter/ade7758_ring.c
@@ -38,18 +38,14 @@ static int ade7758_write_waveform_type(struct device *dev, unsigned int type)
 	int ret;
 	u8 reg;
 
-	ret = ade7758_spi_read_reg_8(dev,
-			ADE7758_WAVMODE,
-			&reg);
+	ret = ade7758_spi_read_reg_8(dev, ADE7758_WAVMODE, &reg);
 	if (ret)
 		goto out;
 
 	reg &= ~0x1F;
 	reg |= type & 0x1F;
 
-	ret = ade7758_spi_write_reg_8(dev,
-			ADE7758_WAVMODE,
-			reg);
+	ret = ade7758_spi_write_reg_8(dev, ADE7758_WAVMODE, reg);
 out:
 	return ret;
 }
@@ -94,7 +90,7 @@ static int ade7758_ring_preenable(struct iio_dev *indio_dev)
 				 indio_dev->masklength);
 
 	ade7758_write_waveform_type(&indio_dev->dev,
-		indio_dev->channels[channel].address);
+				    indio_dev->channels[channel].address);
 
 	return 0;
 }
diff --git a/drivers/staging/iio/meter/ade7854.c b/drivers/staging/iio/meter/ade7854.c
index 75e8685e6df2..24edbc39ab4e 100644
--- a/drivers/staging/iio/meter/ade7854.c
+++ b/drivers/staging/iio/meter/ade7854.c
@@ -23,8 +23,8 @@
 #include "ade7854.h"
 
 static ssize_t ade7854_read_8bit(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				 struct device_attribute *attr,
+				 char *buf)
 {
 	int ret;
 	u8 val = 0;
@@ -40,8 +40,8 @@ static ssize_t ade7854_read_8bit(struct device *dev,
 }
 
 static ssize_t ade7854_read_16bit(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				  struct device_attribute *attr,
+				  char *buf)
 {
 	int ret;
 	u16 val = 0;
@@ -57,8 +57,8 @@ static ssize_t ade7854_read_16bit(struct device *dev,
 }
 
 static ssize_t ade7854_read_24bit(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				  struct device_attribute *attr,
+				  char *buf)
 {
 	int ret;
 	u32 val;
@@ -74,8 +74,8 @@ static ssize_t ade7854_read_24bit(struct device *dev,
 }
 
 static ssize_t ade7854_read_32bit(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				  struct device_attribute *attr,
+				  char *buf)
 {
 	int ret;
 	u32 val = 0;
@@ -91,9 +91,9 @@ static ssize_t ade7854_read_32bit(struct device *dev,
 }
 
 static ssize_t ade7854_write_8bit(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
+				  struct device_attribute *attr,
+				  const char *buf,
+				  size_t len)
 {
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
@@ -112,9 +112,9 @@ error_ret:
 }
 
 static ssize_t ade7854_write_16bit(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
+				   struct device_attribute *attr,
+				   const char *buf,
+				   size_t len)
 {
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
@@ -133,9 +133,9 @@ error_ret:
 }
 
 static ssize_t ade7854_write_24bit(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
+				   struct device_attribute *attr,
+				   const char *buf,
+				   size_t len)
 {
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
@@ -154,9 +154,9 @@ error_ret:
 }
 
 static ssize_t ade7854_write_32bit(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
+				   struct device_attribute *attr,
+				   const char *buf,
+				   size_t len)
 {
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
diff --git a/drivers/staging/ks7010/eap_packet.h b/drivers/staging/ks7010/eap_packet.h
index 16a392abdaec..df7f760e4110 100644
--- a/drivers/staging/ks7010/eap_packet.h
+++ b/drivers/staging/ks7010/eap_packet.h
@@ -1,6 +1,8 @@
 #ifndef EAP_PACKET_H
 #define EAP_PACKET_H
 
+#include <linux/compiler.h>
+
 #define WBIT(n) (1 << (n))
 
 #ifndef ETH_ALEN
@@ -19,14 +21,14 @@ struct ether_hdr {
 #define ETHER_PROTOCOL_TYPE_IP		0x0800
 #define ETHER_PROTOCOL_TYPE_ARP		0x0806
 	/* followed by length octets of data */
-} __attribute__ ((packed));
+} __packed;
 
 struct ieee802_1x_hdr {
 	unsigned char version;
 	unsigned char type;
 	unsigned short length;
 	/* followed by length octets of data */
-} __attribute__ ((packed));
+} __packed;
 
 #define EAPOL_VERSION 2
 
@@ -51,25 +53,33 @@ enum { EAPOL_KEY_TYPE_RC4 = 1, EAPOL_KEY_TYPE_RSN = 2,
 struct ieee802_1x_eapol_key {
 	unsigned char type;
 	unsigned short key_length;
-	/* does not repeat within the life of the keying material used to
-	 * encrypt the Key field; 64-bit NTP timestamp MAY be used here */
+	/*
+	 * does not repeat within the life of the keying material used to
+	 * encrypt the Key field; 64-bit NTP timestamp MAY be used here
+	 */
 	unsigned char replay_counter[IEEE8021X_REPLAY_COUNTER_LEN];
 	unsigned char key_iv[IEEE8021X_KEY_IV_LEN];	/* cryptographically random number */
-	unsigned char key_index;	/* key flag in the most significant bit:
+	unsigned char key_index;	/*
+					 * key flag in the most significant bit:
 					 * 0 = broadcast (default key),
 					 * 1 = unicast (key mapping key); key index is in the
-					 * 7 least significant bits */
-	/* HMAC-MD5 message integrity check computed with MS-MPPE-Send-Key as
-	 * the key */
+					 * 7 least significant bits
+					 */
+	/*
+	 * HMAC-MD5 message integrity check computed with MS-MPPE-Send-Key as
+	 * the key
+	 */
 	unsigned char key_signature[IEEE8021X_KEY_SIGN_LEN];
 
-	/* followed by key: if packet body length = 44 + key length, then the
+	/*
+	 * followed by key: if packet body length = 44 + key length, then the
 	 * key field (of key_length bytes) contains the key in encrypted form;
 	 * if packet body length = 44, key field is absent and key_length
 	 * represents the number of least significant octets from
 	 * MS-MPPE-Send-Key attribute to be used as the keying material;
-	 * RC4 key used in encryption = Key-IV + MS-MPPE-Recv-Key */
-} __attribute__ ((packed));
+	 * RC4 key used in encryption = Key-IV + MS-MPPE-Recv-Key
+	 */
+} __packed;
 
 #define WPA_NONCE_LEN 32
 #define WPA_REPLAY_COUNTER_LEN 8
@@ -86,7 +96,7 @@ struct wpa_eapol_key {
 	unsigned char key_mic[16];
 	unsigned short key_data_length;
 	/* followed by key_data_length bytes of key_data */
-} __attribute__ ((packed));
+} __packed;
 
 #define WPA_KEY_INFO_TYPE_MASK (WBIT(0) | WBIT(1) | WBIT(2))
 #define WPA_KEY_INFO_TYPE_HMAC_MD5_RC4 WBIT(0)
diff --git a/drivers/staging/ks7010/ks7010_sdio.c b/drivers/staging/ks7010/ks7010_sdio.c
index b7337fd813d5..81c46f4d0935 100644
--- a/drivers/staging/ks7010/ks7010_sdio.c
+++ b/drivers/staging/ks7010/ks7010_sdio.c
@@ -14,7 +14,7 @@
 #include <linux/mmc/card.h>
 #include <linux/mmc/sdio_func.h>
 #include <linux/workqueue.h>
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 
 #include "ks_wlan.h"
 #include "ks_wlan_ioctl.h"
@@ -35,18 +35,18 @@ MODULE_DEVICE_TABLE(sdio, ks7010_sdio_ids);
 /* macro */
 
 #define inc_txqhead(priv) \
-        ( priv->tx_dev.qhead = (priv->tx_dev.qhead + 1) % TX_DEVICE_BUFF_SIZE )
+        (priv->tx_dev.qhead = (priv->tx_dev.qhead + 1) % TX_DEVICE_BUFF_SIZE)
 #define inc_txqtail(priv) \
-        ( priv->tx_dev.qtail = (priv->tx_dev.qtail + 1) % TX_DEVICE_BUFF_SIZE )
+        (priv->tx_dev.qtail = (priv->tx_dev.qtail + 1) % TX_DEVICE_BUFF_SIZE)
 #define cnt_txqbody(priv) \
-        (((priv->tx_dev.qtail + TX_DEVICE_BUFF_SIZE) - (priv->tx_dev.qhead)) % TX_DEVICE_BUFF_SIZE )
+        (((priv->tx_dev.qtail + TX_DEVICE_BUFF_SIZE) - (priv->tx_dev.qhead)) % TX_DEVICE_BUFF_SIZE)
 
 #define inc_rxqhead(priv) \
-        ( priv->rx_dev.qhead = (priv->rx_dev.qhead + 1) % RX_DEVICE_BUFF_SIZE )
+        (priv->rx_dev.qhead = (priv->rx_dev.qhead + 1) % RX_DEVICE_BUFF_SIZE)
 #define inc_rxqtail(priv) \
-        ( priv->rx_dev.qtail = (priv->rx_dev.qtail + 1) % RX_DEVICE_BUFF_SIZE )
+        (priv->rx_dev.qtail = (priv->rx_dev.qtail + 1) % RX_DEVICE_BUFF_SIZE)
 #define cnt_rxqbody(priv) \
-        (((priv->rx_dev.qtail + RX_DEVICE_BUFF_SIZE) - (priv->rx_dev.qhead)) % RX_DEVICE_BUFF_SIZE )
+        (((priv->rx_dev.qtail + RX_DEVICE_BUFF_SIZE) - (priv->rx_dev.qhead)) % RX_DEVICE_BUFF_SIZE)
 
 static int ks7010_sdio_read(struct ks_wlan_private *priv, unsigned int address,
 			    unsigned char *buffer, int length)
@@ -87,7 +87,7 @@ static int ks7010_sdio_write(struct ks_wlan_private *priv, unsigned int address,
 	return rc;
 }
 
-void ks_wlan_hw_sleep_doze_request(struct ks_wlan_private *priv)
+static void ks_wlan_hw_sleep_doze_request(struct ks_wlan_private *priv)
 {
 	unsigned char rw_data;
 	int retval;
@@ -115,10 +115,9 @@ void ks_wlan_hw_sleep_doze_request(struct ks_wlan_private *priv)
 
  out:
 	priv->sleep_mode = atomic_read(&priv->sleepstatus.status);
-	return;
 }
 
-void ks_wlan_hw_sleep_wakeup_request(struct ks_wlan_private *priv)
+static void ks_wlan_hw_sleep_wakeup_request(struct ks_wlan_private *priv)
 {
 	unsigned char rw_data;
 	int retval;
@@ -146,7 +145,6 @@ void ks_wlan_hw_sleep_wakeup_request(struct ks_wlan_private *priv)
 
  out:
 	priv->sleep_mode = atomic_read(&priv->sleepstatus.status);
-	return;
 }
 
 void ks_wlan_hw_wakeup_request(struct ks_wlan_private *priv)
@@ -159,9 +157,9 @@ void ks_wlan_hw_wakeup_request(struct ks_wlan_private *priv)
 		rw_data = WAKEUP_REQ;
 		retval =
 		    ks7010_sdio_write(priv, WAKEUP, &rw_data, sizeof(rw_data));
-		if (retval) {
+		if (retval)
 			DPRINTK(1, " error : WAKEUP=%02X\n", rw_data);
-		}
+
 		DPRINTK(4, "wake up : WAKEUP=%02X\n", rw_data);
 		priv->last_wakeup = jiffies;
 		++priv->wakeup_count;
@@ -171,19 +169,16 @@ void ks_wlan_hw_wakeup_request(struct ks_wlan_private *priv)
 	}
 }
 
-int _ks_wlan_hw_power_save(struct ks_wlan_private *priv)
+static int _ks_wlan_hw_power_save(struct ks_wlan_private *priv)
 {
-	int rc = 0;
 	unsigned char rw_data;
 	int retval;
 
 	if (priv->reg.powermgt == POWMGT_ACTIVE_MODE)
-		return rc;
+		return 0;
 
 	if (priv->reg.operation_mode == MODE_INFRASTRUCTURE &&
 	    (priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
-
-		//DPRINTK(1,"psstatus.status=%d\n",atomic_read(&priv->psstatus.status));
 		if (priv->dev_state == DEVICE_STATE_SLEEP) {
 			switch (atomic_read(&priv->psstatus.status)) {
 			case PS_SNOOZE:	/* 4 */
@@ -246,10 +241,9 @@ int _ks_wlan_hw_power_save(struct ks_wlan_private *priv)
 				break;
 			}
 		}
-
 	}
 
-	return rc;
+	return 0;
 }
 
 int ks_wlan_hw_power_save(struct ks_wlan_private *priv)
@@ -268,7 +262,7 @@ static int enqueue_txdev(struct ks_wlan_private *priv, unsigned char *p,
 
 	if (priv->dev_state < DEVICE_STATE_BOOT) {
 		kfree(p);
-		if (complete_handler != NULL)
+		if (complete_handler)
 			(*complete_handler) (arg1, arg2);
 		return 1;
 	}
@@ -277,7 +271,7 @@ static int enqueue_txdev(struct ks_wlan_private *priv, unsigned char *p,
 		/* in case of buffer overflow */
 		DPRINTK(1, "tx buffer overflow\n");
 		kfree(p);
-		if (complete_handler != NULL)
+		if (complete_handler)
 			(*complete_handler) (arg1, arg2);
 		return 1;
 	}
@@ -297,11 +291,10 @@ static int enqueue_txdev(struct ks_wlan_private *priv, unsigned char *p,
 static int write_to_device(struct ks_wlan_private *priv, unsigned char *buffer,
 			   unsigned long size)
 {
-	int rc, retval;
+	int retval;
 	unsigned char rw_data;
 	struct hostif_hdr *hdr;
 	hdr = (struct hostif_hdr *)buffer;
-	rc = 0;
 
 	DPRINTK(4, "size=%d\n", hdr->size);
 	if (hdr->event < HIF_DATA_REQ || HIF_REQ_MAX < hdr->event) {
@@ -346,10 +339,9 @@ static void tx_device_task(void *dev)
 						   &priv->ks_wlan_hw.rw_wq, 1);
 				return;
 			}
-
 		}
 		kfree(sp->sendp);	/* allocated memory free */
-		if (sp->complete_handler != NULL)	/* TX Complete */
+		if (sp->complete_handler)	/* TX Complete */
 			(*sp->complete_handler) (sp->arg1, sp->arg2);
 		inc_txqhead(priv);
 
@@ -358,7 +350,6 @@ static void tx_device_task(void *dev)
 					   &priv->ks_wlan_hw.rw_wq, 0);
 		}
 	}
-	return;
 }
 
 int ks_wlan_hw_tx(struct ks_wlan_private *priv, void *p, unsigned long size,
@@ -402,12 +393,9 @@ static void rx_event_task(unsigned long dev)
 		hostif_receive(priv, rp->data, rp->size);
 		inc_rxqhead(priv);
 
-		if (cnt_rxqbody(priv) > 0) {
+		if (cnt_rxqbody(priv) > 0)
 			tasklet_schedule(&priv->ks_wlan_hw.rx_bh_task);
-		}
 	}
-
-	return;
 }
 
 static void ks_wlan_hw_rx(void *dev, uint16_t size)
@@ -432,9 +420,8 @@ static void ks_wlan_hw_rx(void *dev, uint16_t size)
 	retval =
 	    ks7010_sdio_read(priv, DATA_WINDOW, &rx_buffer->data[0],
 			     hif_align_size(size));
-	if (retval) {
+	if (retval)
 		goto error_out;
-	}
 
 	/* length check */
 	if (size > 2046 || size == 0) {
@@ -449,9 +436,9 @@ static void ks_wlan_hw_rx(void *dev, uint16_t size)
 		retval =
 		    ks7010_sdio_write(priv, READ_STATUS, &read_status,
 				      sizeof(read_status));
-		if (retval) {
+		if (retval)
 			DPRINTK(1, " error : READ_STATUS=%02X\n", read_status);
-		}
+
 		goto error_out;
 	}
 
@@ -465,9 +452,9 @@ static void ks_wlan_hw_rx(void *dev, uint16_t size)
 	retval =
 	    ks7010_sdio_write(priv, READ_STATUS, &read_status,
 			      sizeof(read_status));
-	if (retval) {
+	if (retval)
 		DPRINTK(1, " error : READ_STATUS=%02X\n", read_status);
-	}
+
 	DPRINTK(4, "READ_STATUS=%02X\n", read_status);
 
 	if (atomic_read(&priv->psstatus.confirm_wait)) {
@@ -498,7 +485,7 @@ static void ks7010_rw_function(struct work_struct *work)
 
 	/* wiat after DOZE */
 	if (time_after(priv->last_doze + ((30 * HZ) / 1000), jiffies)) {
-		DPRINTK(4, "wait after DOZE \n");
+		DPRINTK(4, "wait after DOZE\n");
 		queue_delayed_work(priv->ks_wlan_hw.ks7010sdio_wq,
 				   &priv->ks_wlan_hw.rw_wq, 1);
 		return;
@@ -506,11 +493,13 @@ static void ks7010_rw_function(struct work_struct *work)
 
 	/* wiat after WAKEUP */
 	while (time_after(priv->last_wakeup + ((30 * HZ) / 1000), jiffies)) {
-		DPRINTK(4, "wait after WAKEUP \n");
+		DPRINTK(4, "wait after WAKEUP\n");
 /*		queue_delayed_work(priv->ks_wlan_hw.ks7010sdio_wq,&priv->ks_wlan_hw.rw_wq,
 		(priv->last_wakeup + ((30*HZ)/1000) - jiffies));*/
-		printk("wake: %lu %lu\n", priv->last_wakeup + (30 * HZ) / 1000,
-		       jiffies);
+		dev_info(&priv->ks_wlan_hw.sdio_card->func->dev,
+			 "wake: %lu %lu\n",
+			 priv->last_wakeup + (30 * HZ) / 1000,
+				jiffies);
 		msleep(30);
 	}
 
@@ -549,17 +538,15 @@ static void ks7010_rw_function(struct work_struct *work)
 
 	if (rw_data & RSIZE_MASK) {	/* Read schedule */
 		ks_wlan_hw_rx((void *)priv,
-			      (uint16_t) (((rw_data & RSIZE_MASK) << 4)));
+			      (uint16_t)((rw_data & RSIZE_MASK) << 4));
 	}
-	if ((rw_data & WSTATUS_MASK)) {
+	if ((rw_data & WSTATUS_MASK))
 		tx_device_task((void *)priv);
-	}
+
 	_ks_wlan_hw_power_save(priv);
 
  err_out:
 	sdio_release_host(priv->ks_wlan_hw.sdio_card->func);
-
-	return;
 }
 
 static void ks_sdio_interrupt(struct sdio_func *func)
@@ -607,7 +594,6 @@ static void ks_sdio_interrupt(struct sdio_func *func)
 				}
 				complete(&priv->psstatus.wakeup_wait);
 			}
-
 		}
 
 		do {
@@ -624,7 +610,7 @@ static void ks_sdio_interrupt(struct sdio_func *func)
 			rsize = rw_data & RSIZE_MASK;
 			if (rsize) {	/* Read schedule */
 				ks_wlan_hw_rx((void *)priv,
-					      (uint16_t) (((rsize) << 4)));
+					      (uint16_t)(rsize << 4));
 			}
 			if (rw_data & WSTATUS_MASK) {
 #if 0
@@ -667,7 +653,6 @@ static void ks_sdio_interrupt(struct sdio_func *func)
  intr_out:
 	queue_delayed_work(priv->ks_wlan_hw.ks7010sdio_wq,
 			   &priv->ks_wlan_hw.rw_wq, 0);
-	return;
 }
 
 static int trx_device_init(struct ks_wlan_private *priv)
@@ -696,14 +681,12 @@ static void trx_device_exit(struct ks_wlan_private *priv)
 	while (cnt_txqbody(priv) > 0) {
 		sp = &priv->tx_dev.tx_dev_buff[priv->tx_dev.qhead];
 		kfree(sp->sendp);	/* allocated memory free */
-		if (sp->complete_handler != NULL)	/* TX Complete */
+		if (sp->complete_handler)	/* TX Complete */
 			(*sp->complete_handler) (sp->arg1, sp->arg2);
 		inc_txqhead(priv);
 	}
 
 	tasklet_kill(&priv->ks_wlan_hw.rx_bh_task);
-
-	return;
 }
 
 static int ks7010_sdio_update_index(struct ks_wlan_private *priv, u32 index)
@@ -711,7 +694,6 @@ static int ks7010_sdio_update_index(struct ks_wlan_private *priv, u32 index)
 	int rc = 0;
 	int retval;
 	unsigned char *data_buf;
-	data_buf = NULL;
 
 	data_buf = kmalloc(sizeof(u32), GFP_KERNEL);
 	if (!data_buf) {
@@ -732,8 +714,7 @@ static int ks7010_sdio_update_index(struct ks_wlan_private *priv, u32 index)
 		goto error_out;
 	}
  error_out:
-	if (data_buf)
-		kfree(data_buf);
+	kfree(data_buf);
 	return rc;
 }
 
@@ -744,7 +725,7 @@ static int ks7010_sdio_data_compare(struct ks_wlan_private *priv, u32 address,
 	int rc = 0;
 	int retval;
 	unsigned char *read_buf;
-	read_buf = NULL;
+
 	read_buf = kmalloc(ROM_BUFF_SIZE, GFP_KERNEL);
 	if (!read_buf) {
 		rc = 1;
@@ -758,13 +739,12 @@ static int ks7010_sdio_data_compare(struct ks_wlan_private *priv, u32 address,
 	retval = memcmp(data, read_buf, size);
 
 	if (retval) {
-		DPRINTK(0, "data compare error (%d) \n", retval);
+		DPRINTK(0, "data compare error (%d)\n", retval);
 		rc = 3;
 		goto error_out;
 	}
  error_out:
-	if (read_buf)
-		kfree(read_buf);
+	kfree(read_buf);
 	return rc;
 }
 
@@ -778,14 +758,10 @@ static int ks7010_upload_firmware(struct ks_wlan_private *priv,
 	int length;
 	const struct firmware *fw_entry = NULL;
 
-	rom_buf = NULL;
-
 	/* buffer allocate */
 	rom_buf = kmalloc(ROM_BUFF_SIZE, GFP_KERNEL);
-	if (!rom_buf) {
-		rc = 3;
-		goto error_out0;
-	}
+	if (!rom_buf)
+		return 3;
 
 	sdio_claim_host(card->func);
 
@@ -799,7 +775,7 @@ static int ks7010_upload_firmware(struct ks_wlan_private *priv,
 
 	retval = request_firmware(&fw_entry, ROM_FILE, &priv->ks_wlan_hw.sdio_card->func->dev);
 	if (retval)
-		return retval;
+		goto error_out0;
 
 	length = fw_entry->size;
 
@@ -879,8 +855,7 @@ static int ks7010_upload_firmware(struct ks_wlan_private *priv,
 	release_firmware(fw_entry);
  error_out0:
 	sdio_release_host(card->func);
-	if (rom_buf)
-		kfree(rom_buf);
+	kfree(rom_buf);
 	return rc;
 }
 
@@ -903,9 +878,9 @@ static void ks7010_card_init(struct ks_wlan_private *priv)
 		DPRINTK(1, "wait time out!! SME_START\n");
 	}
 
-	if (priv->mac_address_valid && priv->version_size) {
+	if (priv->mac_address_valid && priv->version_size)
 		priv->dev_state = DEVICE_STATE_PREINIT;
-	}
+
 
 	hostif_sme_enqueue(priv, SME_GET_EEPROM_CKSUM);
 
@@ -981,7 +956,7 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	netdev = NULL;
 
 	/* initilize ks_sdio_card */
-	card = kzalloc(sizeof(struct ks_sdio_card), GFP_KERNEL);
+	card = kzalloc(sizeof(*card), GFP_KERNEL);
 	if (!card)
 		return -ENOMEM;
 
@@ -1029,12 +1004,13 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 
 	/* private memory allocate */
 	netdev = alloc_etherdev(sizeof(*priv));
-	if (netdev == NULL) {
-		printk(KERN_ERR "ks7010 : Unable to alloc new net device\n");
+	if (!netdev) {
+		dev_err(&card->func->dev, "ks7010 : Unable to alloc new net device\n");
 		goto error_release_irq;
 	}
 	if (dev_alloc_name(netdev, "wlan%d") < 0) {
-		printk(KERN_ERR "ks7010 :  Couldn't get name!\n");
+		dev_err(&card->func->dev,
+			"ks7010 :  Couldn't get name!\n");
 		goto error_free_netdev;
 	}
 
@@ -1048,9 +1024,9 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	init_completion(&priv->ks_wlan_hw.ks7010_sdio_wait);
 	priv->ks_wlan_hw.read_buf = NULL;
 	priv->ks_wlan_hw.read_buf = kmalloc(RX_DATA_SIZE, GFP_KERNEL);
-	if (!priv->ks_wlan_hw.read_buf) {
+	if (!priv->ks_wlan_hw.read_buf)
 		goto error_free_netdev;
-	}
+
 	priv->dev_state = DEVICE_STATE_PREBOOT;
 	priv->net_dev = netdev;
 	priv->firmware_version[0] = '\0';
@@ -1074,9 +1050,9 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	/* Upload firmware */
 	ret = ks7010_upload_firmware(priv, card);	/* firmware load */
 	if (ret) {
-		printk(KERN_ERR
-		       "ks7010: firmware load failed !! retern code = %d\n",
-		       ret);
+		dev_err(&card->func->dev,
+			"ks7010: firmware load failed !! return code = %d\n",
+			 ret);
 		goto error_free_read_buf;
 	}
 
@@ -1086,9 +1062,9 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	sdio_claim_host(func);
 	ret = ks7010_sdio_write(priv, INT_PENDING, &rw_data, sizeof(rw_data));
 	sdio_release_host(func);
-	if (ret) {
+	if (ret)
 		DPRINTK(1, " error : INT_PENDING=%02X\n", rw_data);
-	}
+
 	DPRINTK(4, " clear Interrupt : INT_PENDING=%02X\n", rw_data);
 
 	/* enable ks7010sdio interrupt (INT_GCR_B|INT_READ_STATUS|INT_WRITE_STATUS) */
@@ -1096,9 +1072,9 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	sdio_claim_host(func);
 	ret = ks7010_sdio_write(priv, INT_ENABLE, &rw_data, sizeof(rw_data));
 	sdio_release_host(func);
-	if (ret) {
+	if (ret)
 		DPRINTK(1, " error : INT_ENABLE=%02X\n", rw_data);
-	}
+
 	DPRINTK(4, " enable Interrupt : INT_ENABLE=%02X\n", rw_data);
 	priv->dev_state = DEVICE_STATE_BOOT;
 
@@ -1141,18 +1117,18 @@ static void ks7010_sdio_remove(struct sdio_func *func)
 	int ret;
 	struct ks_sdio_card *card;
 	struct ks_wlan_private *priv;
-	struct net_device *netdev;
 	DPRINTK(1, "ks7010_sdio_remove()\n");
 
 	card = sdio_get_drvdata(func);
 
-	if (card == NULL)
+	if (!card)
 		return;
 
 	DPRINTK(1, "priv = card->priv\n");
 	priv = card->priv;
-	netdev = priv->net_dev;
 	if (priv) {
+		struct net_device *netdev = priv->net_dev;
+
 		ks_wlan_net_stop(netdev);
 		DPRINTK(1, "ks_wlan_net_stop\n");
 
@@ -1166,9 +1142,8 @@ static void ks7010_sdio_remove(struct sdio_func *func)
 		/* send stop request to MAC */
 		{
 			struct hostif_stop_request_t *pp;
-			pp = (struct hostif_stop_request_t *)
-			    kzalloc(hif_align_size(sizeof(*pp)), GFP_KERNEL);
-			if (pp == NULL) {
+			pp = kzalloc(hif_align_size(sizeof(*pp)), GFP_KERNEL);
+			if (!pp) {
 				DPRINTK(3, "allocate memory failed..\n");
 				return;	/* to do goto ni suru */
 			}
@@ -1176,7 +1151,7 @@ static void ks7010_sdio_remove(struct sdio_func *func)
 			    cpu_to_le16((uint16_t)
 					(sizeof(*pp) -
 					 sizeof(pp->header.size)));
-			pp->header.event = cpu_to_le16((uint16_t) HIF_STOP_REQ);
+			pp->header.event = cpu_to_le16((uint16_t)HIF_STOP_REQ);
 
 			sdio_claim_host(func);
 			write_to_device(priv, (unsigned char *)pp,
@@ -1199,9 +1174,7 @@ static void ks7010_sdio_remove(struct sdio_func *func)
 		unregister_netdev(netdev);
 
 		trx_device_exit(priv);
-		if (priv->ks_wlan_hw.read_buf) {
-			kfree(priv->ks_wlan_hw.read_buf);
-		}
+		kfree(priv->ks_wlan_hw.read_buf);
 		free_netdev(priv->net_dev);
 		card->priv = NULL;
 	}
@@ -1219,7 +1192,6 @@ static void ks7010_sdio_remove(struct sdio_func *func)
 	DPRINTK(1, "kfree()\n");
 
 	DPRINTK(5, " Bye !!\n");
-	return;
 }
 
 static struct sdio_driver ks7010_sdio_driver = {
diff --git a/drivers/staging/ks7010/ks_hostif.c b/drivers/staging/ks7010/ks_hostif.c
index a8822fe2bd60..c57ca581550a 100644
--- a/drivers/staging/ks7010/ks_hostif.c
+++ b/drivers/staging/ks7010/ks_hostif.c
@@ -14,21 +14,13 @@
 #include "eap_packet.h"
 #include "michael_mic.h"
 
+#include <linux/etherdevice.h>
 #include <linux/if_ether.h>
 #include <linux/if_arp.h>
 
 /* Include Wireless Extension definition and check version */
 #include <net/iw_handler.h>	/* New driver API */
 
-extern int ks_wlan_hw_tx(struct ks_wlan_private *priv, void *p,
-			 unsigned long size,
-			 void (*complete_handler) (void *arg1, void *arg2),
-			 void *arg1, void *arg2);
-extern void send_packet_complete(void *, void *);
-
-extern void ks_wlan_hw_wakeup_request(struct ks_wlan_private *priv);
-extern int ks_wlan_hw_power_save(struct ks_wlan_private *priv);
-
 /* macro */
 #define inc_smeqhead(priv) \
         ( priv->sme_i.qhead = (priv->sme_i.qhead + 1) % SME_EVENT_BUFF_SIZE )
@@ -43,6 +35,7 @@ static
 inline u8 get_BYTE(struct ks_wlan_private *priv)
 {
 	u8 data;
+
 	data = *(priv->rxp)++;
 	/* length check in advance ! */
 	--(priv->rx_size);
@@ -53,6 +46,7 @@ static
 inline u16 get_WORD(struct ks_wlan_private *priv)
 {
 	u16 data;
+
 	data = (get_BYTE(priv) & 0xff);
 	data |= ((get_BYTE(priv) << 8) & 0xff00);
 	return data;
@@ -62,6 +56,7 @@ static
 inline u32 get_DWORD(struct ks_wlan_private *priv)
 {
 	u32 data;
+
 	data = (get_BYTE(priv) & 0xff);
 	data |= ((get_BYTE(priv) << 8) & 0x0000ff00);
 	data |= ((get_BYTE(priv) << 16) & 0x00ff0000);
@@ -69,16 +64,20 @@ inline u32 get_DWORD(struct ks_wlan_private *priv)
 	return data;
 }
 
-void ks_wlan_hw_wakeup_task(struct work_struct *work)
+static void ks_wlan_hw_wakeup_task(struct work_struct *work)
 {
 	struct ks_wlan_private *priv =
 	    container_of(work, struct ks_wlan_private, ks_wlan_wakeup_task);
 	int ps_status = atomic_read(&priv->psstatus.status);
+	long time_left;
 
 	if (ps_status == PS_SNOOZE) {
 		ks_wlan_hw_wakeup_request(priv);
-		if (!wait_for_completion_interruptible_timeout(&priv->psstatus.wakeup_wait, HZ / 50)) {	/* 20ms timeout */
-			DPRINTK(1, "wake up timeout !!!\n");
+		time_left = wait_for_completion_interruptible_timeout(
+				&priv->psstatus.wakeup_wait,
+				msecs_to_jiffies(20));
+		if (time_left <= 0) {
+			DPRINTK(1, "wake up timeout or interrupted !!!\n");
 			schedule_work(&priv->ks_wlan_wakeup_task);
 			return;
 		}
@@ -96,8 +95,6 @@ void ks_wlan_hw_wakeup_task(struct work_struct *work)
 static
 int ks_wlan_do_power_save(struct ks_wlan_private *priv)
 {
-	int rc = 0;
-
 	DPRINTK(4, "psstatus.status=%d\n", atomic_read(&priv->psstatus.status));
 
 	if ((priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
@@ -105,7 +102,7 @@ int ks_wlan_do_power_save(struct ks_wlan_private *priv)
 	} else {
 		priv->dev_state = DEVICE_STATE_READY;
 	}
-	return rc;
+	return 0;
 }
 
 static
@@ -217,7 +214,6 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 {
 	unsigned char *bp;
 	int bsize, offset;
-	int rc = 0;
 
 	DPRINTK(3, "\n");
 	memset(ap, 0, sizeof(struct local_ap_t));
@@ -240,13 +236,13 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 	offset = 0;
 
 	while (bsize > offset) {
-		/* DPRINTK(4, "Element ID=%d \n",*bp); */
+		/* DPRINTK(4, "Element ID=%d\n",*bp); */
 		switch (*bp) {
 		case 0:	/* ssid */
 			if (*(bp + 1) <= SSID_MAX_SIZE) {
 				ap->ssid.size = *(bp + 1);
 			} else {
-				DPRINTK(1, "size over :: ssid size=%d \n",
+				DPRINTK(1, "size over :: ssid size=%d\n",
 					*(bp + 1));
 				ap->ssid.size = SSID_MAX_SIZE;
 			}
@@ -260,7 +256,7 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 				       bp + 2, *(bp + 1));
 				ap->rate_set.size += *(bp + 1);
 			} else {
-				DPRINTK(1, "size over :: rate size=%d \n",
+				DPRINTK(1, "size over :: rate size=%d\n",
 					(*(bp + 1) + ap->rate_set.size));
 				memcpy(&(ap->rate_set.body[ap->rate_set.size]),
 				       bp + 2,
@@ -276,7 +272,7 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 			if (*(bp + 1) <= RSN_IE_BODY_MAX) {
 				ap->rsn_ie.size = *(bp + 1);
 			} else {
-				DPRINTK(1, "size over :: rsn size=%d \n",
+				DPRINTK(1, "size over :: rsn size=%d\n",
 					*(bp + 1));
 				ap->rsn_ie.size = RSN_IE_BODY_MAX;
 			}
@@ -289,7 +285,7 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 					ap->wpa_ie.size = *(bp + 1);
 				} else {
 					DPRINTK(1,
-						"size over :: wpa size=%d \n",
+						"size over :: wpa size=%d\n",
 						*(bp + 1));
 					ap->wpa_ie.size = RSN_IE_BODY_MAX;
 				}
@@ -307,7 +303,7 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 		case 47:	/* Reserve ID 47 Broadcom AP */
 			break;
 		default:
-			DPRINTK(4, "unknown Element ID=%d \n", *bp);
+			DPRINTK(4, "unknown Element ID=%d\n", *bp);
 			break;
 		}
 		offset += 2;	/* id & size field */
@@ -315,7 +311,7 @@ int get_ap_information(struct ks_wlan_private *priv, struct ap_info_t *ap_info,
 		bp += (*(bp + 1) + 2);	/* pointer update */
 	}
 
-	return rc;
+	return 0;
 }
 
 static
@@ -404,7 +400,7 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 					    HZ >= 60) {
 						mic_failure->failure = 0;
 					}
-					DPRINTK(4, "MIC FAILURE \n");
+					DPRINTK(4, "MIC FAILURE\n");
 					if (mic_failure->failure == 0) {
 						mic_failure->failure = 1;
 						mic_failure->counter = 0;
@@ -481,8 +477,7 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 			netif_rx(skb);
 		} else {
 			printk(KERN_WARNING
-			       "%s: Memory squeeze, dropping packet.\n",
-			       skb->dev->name);
+			       "ks_wlan: Memory squeeze, dropping packet.\n");
 			priv->nstats.rx_dropped++;
 		}
 		break;
@@ -517,8 +512,7 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 			netif_rx(skb);
 		} else {
 			printk(KERN_WARNING
-			       "%s: Memory squeeze, dropping packet.\n",
-			       skb->dev->name);
+			       "ks_wlan: Memory squeeze, dropping packet.\n");
 			priv->nstats.rx_dropped++;
 		}
 		break;
@@ -778,7 +772,7 @@ void hostif_start_confirm(struct ks_wlan_private *priv)
 	wrqu.data.flags = 0;
 	wrqu.ap_addr.sa_family = ARPHRD_ETHER;
 	if ((priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
-		memset(wrqu.ap_addr.sa_data, '\0', ETH_ALEN);
+		eth_zero_addr(wrqu.ap_addr.sa_data);
 		DPRINTK(3, "IWEVENT: disconnect\n");
 		wireless_send_event(priv->net_dev, SIOCGIWAP, &wrqu, NULL);
 	}
@@ -836,7 +830,7 @@ void hostif_connect_indication(struct ks_wlan_private *priv)
 	wrqu0.ap_addr.sa_family = ARPHRD_ETHER;
 	if ((priv->connect_status & CONNECT_STATUS_MASK) == DISCONNECT_STATUS &&
 	    (old_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
-		memset(wrqu0.ap_addr.sa_data, '\0', ETH_ALEN);
+		eth_zero_addr(wrqu0.ap_addr.sa_data);
 		DPRINTK(3, "IWEVENT: disconnect\n");
 		DPRINTK(3, "disconnect :: scan_ind_count=%d\n",
 			priv->scan_ind_count);
@@ -908,7 +902,7 @@ void hostif_stop_confirm(struct ks_wlan_private *priv)
 		if ((priv->connect_status & CONNECT_STATUS_MASK) ==
 		    DISCONNECT_STATUS
 		    && (old_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
-			memset(wrqu0.ap_addr.sa_data, '\0', ETH_ALEN);
+			eth_zero_addr(wrqu0.ap_addr.sa_data);
 			DPRINTK(3, "IWEVENT: disconnect\n");
 			printk("IWEVENT: disconnect\n");
 			DPRINTK(3, "disconnect :: scan_ind_count=%d\n",
@@ -1148,7 +1142,7 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *packet)
 
 	packet_len = packet->len;
 	if (packet_len > ETH_FRAME_LEN) {
-		DPRINTK(1, "bad length packet_len=%d \n", packet_len);
+		DPRINTK(1, "bad length packet_len=%d\n", packet_len);
 		dev_kfree_skb(packet);
 		return -1;
 	}
@@ -1172,11 +1166,10 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *packet)
 	}
 
 	DPRINTK(4, "skb_buff length=%d\n", packet_len);
-	pp = (struct hostif_data_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp) + 6 + packet_len + 8),
-		    KS_WLAN_MEM_FLAG);
+	pp = kmalloc(hif_align_size(sizeof(*pp) + 6 + packet_len + 8),
+		     KS_WLAN_MEM_FLAG);
 
-	if (pp == NULL) {
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		dev_kfree_skb(packet);
 		return -2;
@@ -1194,6 +1187,8 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *packet)
 		DPRINTK(1, "ethernet->h_source=%02X:%02X:%02X:%02X:%02X:%02X\n",
 			eth->h_source[0], eth->h_source[1], eth->h_source[2],
 			eth->h_source[3], eth->h_source[4], eth->h_source[5]);
+		dev_kfree_skb(packet);
+		kfree(pp);
 		return -3;
 	}
 
@@ -1231,7 +1226,7 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *packet)
 	eth_hdr = (struct ether_hdr *)&pp->data[0];
 	eth_proto = ntohs(eth_hdr->h_proto);
 
-	/* for MIC FAILUER REPORT check */
+	/* for MIC FAILURE REPORT check */
 	if (eth_proto == ETHER_PROTOCOL_TYPE_EAP
 	    && priv->wpa.mic_failure.failure > 0) {
 		aa1x_hdr = (struct ieee802_1x_hdr *)(eth_hdr + 1);
@@ -1284,7 +1279,7 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *packet)
 			  (void *)send_packet_complete, (void *)priv,
 			  (void *)packet);
 
-	/* MIC FAILUER REPORT check */
+	/* MIC FAILURE REPORT check */
 	if (eth_proto == ETHER_PROTOCOL_TYPE_EAP
 	    && priv->wpa.mic_failure.failure > 0) {
 		if (keyinfo & WPA_KEY_INFO_ERROR
@@ -1313,9 +1308,8 @@ void hostif_mib_get_request(struct ks_wlan_private *priv,
 	DPRINTK(3, "\n");
 
 	/* make primitive */
-	pp = (struct hostif_mib_get_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1344,9 +1338,8 @@ void hostif_mib_set_request(struct ks_wlan_private *priv,
 	}
 
 	/* make primitive */
-	pp = (struct hostif_mib_set_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp) + size), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp) + size), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1374,9 +1367,8 @@ void hostif_start_request(struct ks_wlan_private *priv, unsigned char mode)
 	DPRINTK(3, "\n");
 
 	/* make primitive */
-	pp = (struct hostif_start_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1402,9 +1394,8 @@ void hostif_ps_adhoc_set_request(struct ks_wlan_private *priv)
 	DPRINTK(3, "\n");
 
 	/* make primitive */
-	pp = (struct hostif_ps_adhoc_set_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1443,12 +1434,11 @@ void hostif_infrastructure_set_request(struct ks_wlan_private *priv)
 	struct hostif_infrastructure_set_request_t *pp;
 	uint16_t capability;
 
-	DPRINTK(3, "ssid.size=%d \n", priv->reg.ssid.size);
+	DPRINTK(3, "ssid.size=%d\n", priv->reg.ssid.size);
 
 	/* make primitive */
-	pp = (struct hostif_infrastructure_set_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1505,17 +1495,16 @@ void hostif_infrastructure_set_request(struct ks_wlan_private *priv)
 	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL, NULL);
 }
 
-void hostif_infrastructure_set2_request(struct ks_wlan_private *priv)
+static void hostif_infrastructure_set2_request(struct ks_wlan_private *priv)
 {
 	struct hostif_infrastructure_set2_request_t *pp;
 	uint16_t capability;
 
-	DPRINTK(2, "ssid.size=%d \n", priv->reg.ssid.size);
+	DPRINTK(2, "ssid.size=%d\n", priv->reg.ssid.size);
 
 	/* make primitive */
-	pp = (struct hostif_infrastructure_set2_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1583,9 +1572,8 @@ void hostif_adhoc_set_request(struct ks_wlan_private *priv)
 	DPRINTK(3, "\n");
 
 	/* make primitive */
-	pp = (struct hostif_adhoc_set_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1629,9 +1617,8 @@ void hostif_adhoc_set2_request(struct ks_wlan_private *priv)
 	DPRINTK(3, "\n");
 
 	/* make primitive */
-	pp = (struct hostif_adhoc_set2_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1677,9 +1664,8 @@ void hostif_stop_request(struct ks_wlan_private *priv)
 	DPRINTK(3, "\n");
 
 	/* make primitive */
-	pp = (struct hostif_stop_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1700,9 +1686,8 @@ void hostif_phy_information_request(struct ks_wlan_private *priv)
 	DPRINTK(3, "\n");
 
 	/* make primitive */
-	pp = (struct hostif_phy_information_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1732,9 +1717,8 @@ void hostif_power_mngmt_request(struct ks_wlan_private *priv,
 	DPRINTK(3, "mode=%lu wake_up=%lu receiveDTIMs=%lu\n", mode, wake_up,
 		receiveDTIMs);
 	/* make primitive */
-	pp = (struct hostif_power_mngmt_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1755,13 +1739,12 @@ void hostif_sleep_request(struct ks_wlan_private *priv, unsigned long mode)
 {
 	struct hostif_sleep_request_t *pp;
 
-	DPRINTK(3, "mode=%lu \n", mode);
+	DPRINTK(3, "mode=%lu\n", mode);
 
 	if (mode == SLP_SLEEP) {
 		/* make primitive */
-		pp = (struct hostif_sleep_request_t *)
-		    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-		if (pp == NULL) {
+		pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+		if (!pp) {
 			DPRINTK(3, "allocate memory failed..\n");
 			return;
 		}
@@ -1779,23 +1762,22 @@ void hostif_sleep_request(struct ks_wlan_private *priv, unsigned long mode)
 		queue_delayed_work(priv->ks_wlan_hw.ks7010sdio_wq,
 				   &priv->ks_wlan_hw.rw_wq, 1);
 	} else {
-		DPRINTK(3, "invalid mode %ld \n", mode);
+		DPRINTK(3, "invalid mode %ld\n", mode);
 		return;
 	}
 }
 
 static
 void hostif_bss_scan_request(struct ks_wlan_private *priv,
-			     unsigned long scan_type, uint8_t * scan_ssid,
+			     unsigned long scan_type, uint8_t *scan_ssid,
 			     uint8_t scan_ssid_len)
 {
 	struct hostif_bss_scan_request_t *pp;
 
 	DPRINTK(2, "\n");
 	/* make primitive */
-	pp = (struct hostif_bss_scan_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1850,9 +1832,8 @@ void hostif_mic_failure_request(struct ks_wlan_private *priv,
 
 	DPRINTK(3, "count=%d :: timer=%d\n", failure_count, timer);
 	/* make primitive */
-	pp = (struct hostif_mic_failure_request_t *)
-	    kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
-	if (pp == NULL) {
+	pp = kmalloc(hif_align_size(sizeof(*pp)), KS_WLAN_MEM_FLAG);
+	if (!pp) {
 		DPRINTK(3, "allocate memory failed..\n");
 		return;
 	}
@@ -1867,7 +1848,7 @@ void hostif_mic_failure_request(struct ks_wlan_private *priv,
 	ks_wlan_hw_tx(priv, pp, hif_align_size(sizeof(*pp)), NULL, NULL, NULL);
 }
 
-/* Device I/O Recieve indicate */
+/* Device I/O Receive indicate */
 static void devio_rec_ind(struct ks_wlan_private *priv, unsigned char *p,
 			  unsigned int size)
 {
@@ -2700,7 +2681,6 @@ void hostif_sme_enqueue(struct ks_wlan_private *priv, unsigned short event)
 
 int hostif_init(struct ks_wlan_private *priv)
 {
-	int rc = 0;
 	int i;
 
 	DPRINTK(3, "\n");
@@ -2750,7 +2730,7 @@ int hostif_init(struct ks_wlan_private *priv)
 
 	tasklet_init(&priv->sme_task, hostif_sme_task, (unsigned long)priv);
 
-	return rc;
+	return 0;
 }
 
 void hostif_exit(struct ks_wlan_private *priv)
diff --git a/drivers/staging/ks7010/ks_hostif.h b/drivers/staging/ks7010/ks_hostif.h
index dc806b5b47be..743f31ead56e 100644
--- a/drivers/staging/ks7010/ks_hostif.h
+++ b/drivers/staging/ks7010/ks_hostif.h
@@ -11,6 +11,9 @@
 
 #ifndef _KS_HOSTIF_H_
 #define _KS_HOSTIF_H_
+
+#include <linux/compiler.h>
+
 /*
  * HOST-MAC I/F events
  */
@@ -61,7 +64,7 @@
 struct hostif_hdr {
 	uint16_t size;
 	uint16_t event;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_data_request_t {
 	struct hostif_hdr header;
@@ -70,7 +73,7 @@ struct hostif_data_request_t {
 #define TYPE_AUTH 0x0001
 	uint16_t reserved;
 	uint8_t data[0];
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_data_indication_t {
 	struct hostif_hdr header;
@@ -81,14 +84,14 @@ struct hostif_data_indication_t {
 #define TYPE_GMK2 0x0003
 	uint16_t reserved;
 	uint8_t data[0];
-} __attribute__ ((packed));
+} __packed;
 
 #define CHANNEL_LIST_MAX_SIZE 14
 struct channel_list_t {
 	uint8_t size;
 	uint8_t body[CHANNEL_LIST_MAX_SIZE];
 	uint8_t pad;
-} __attribute__ ((packed));
+} __packed;
 
 /* MIB Attribute */
 #define DOT11_MAC_ADDRESS                 0x21010100	/* MAC Address (R) */
@@ -141,7 +144,7 @@ struct channel_list_t {
 struct hostif_mib_get_request_t {
 	struct hostif_hdr header;
 	uint32_t mib_attribute;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_mib_value_t {
 	uint16_t size;
@@ -152,7 +155,7 @@ struct hostif_mib_value_t {
 #define MIB_VALUE_TYPE_COUNT32  3
 #define MIB_VALUE_TYPE_OSTRING  4
 	uint8_t body[0];
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_mib_get_confirm_t {
 	struct hostif_hdr header;
@@ -163,19 +166,19 @@ struct hostif_mib_get_confirm_t {
 #define MIB_WRITE_ONLY 3
 	uint32_t mib_attribute;
 	struct hostif_mib_value_t mib_value;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_mib_set_request_t {
 	struct hostif_hdr header;
 	uint32_t mib_attribute;
 	struct hostif_mib_value_t mib_value;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_mib_set_confirm_t {
 	struct hostif_hdr header;
 	uint32_t mib_status;
 	uint32_t mib_attribute;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_power_mngmt_request_t {
 	struct hostif_hdr header;
@@ -188,7 +191,7 @@ struct hostif_power_mngmt_request_t {
 	uint32_t receiveDTIMs;
 #define DTIM_FALSE 0
 #define DTIM_TRUE  1
-} __attribute__ ((packed));
+} __packed;
 
 /* power management mode */
 enum {
@@ -206,7 +209,7 @@ enum {
 struct hostif_power_mngmt_confirm_t {
 	struct hostif_hdr header;
 	uint16_t result_code;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_start_request_t {
 	struct hostif_hdr header;
@@ -215,64 +218,64 @@ struct hostif_start_request_t {
 #define MODE_INFRASTRUCTURE 1
 #define MODE_AP             2	/* not used */
 #define MODE_ADHOC          3
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_start_confirm_t {
 	struct hostif_hdr header;
 	uint16_t result_code;
-} __attribute__ ((packed));
+} __packed;
 
 #define SSID_MAX_SIZE 32
 struct ssid_t {
 	uint8_t size;
 	uint8_t body[SSID_MAX_SIZE];
 	uint8_t ssid_pad;
-} __attribute__ ((packed));
+} __packed;
 
 #define RATE_SET_MAX_SIZE 16
 struct rate_set8_t {
 	uint8_t size;
 	uint8_t body[8];
 	uint8_t rate_pad;
-} __attribute__ ((packed));
+} __packed;
 
 struct FhParms_t {
 	uint16_t dwellTime;
 	uint8_t hopSet;
 	uint8_t hopPattern;
 	uint8_t hopIndex;
-} __attribute__ ((packed));
+} __packed;
 
 struct DsParms_t {
 	uint8_t channel;
-} __attribute__ ((packed));
+} __packed;
 
 struct CfParms_t {
 	uint8_t count;
 	uint8_t period;
 	uint16_t maxDuration;
 	uint16_t durRemaining;
-} __attribute__ ((packed));
+} __packed;
 
 struct IbssParms_t {
 	uint16_t atimWindow;
-} __attribute__ ((packed));
+} __packed;
 
 struct rsn_t {
 	uint8_t size;
 #define RSN_BODY_SIZE 64
 	uint8_t body[RSN_BODY_SIZE];
-} __attribute__ ((packed));
+} __packed;
 
 struct ErpParams_t {
 	uint8_t erp_info;
-} __attribute__ ((packed));
+} __packed;
 
 struct rate_set16_t {
 	uint8_t size;
 	uint8_t body[16];
 	uint8_t rate_pad;
-} __attribute__ ((packed));
+} __packed;
 
 struct ap_info_t {
 	uint8_t bssid[6];	/* +00 */
@@ -299,7 +302,7 @@ struct ap_info_t {
 	uint16_t body_size;	/* +16 */
 	uint8_t body[1024];	/* +18 */
 	/* +1032 */
-} __attribute__ ((packed));
+} __packed;
 
 struct link_ap_info_t {
 	uint8_t bssid[6];	/* +00 */
@@ -325,8 +328,8 @@ struct link_ap_info_t {
 	struct {
 		uint8_t size;	/* +52 */
 		uint8_t body[128];	/* +53 */
-	} __attribute__ ((packed)) rsn;
-} __attribute__ ((packed));
+	} __packed rsn;
+} __packed;
 
 struct hostif_connect_indication_t {
 	struct hostif_hdr header;
@@ -334,16 +337,16 @@ struct hostif_connect_indication_t {
 #define RESULT_CONNECT    0
 #define RESULT_DISCONNECT 1
 	struct link_ap_info_t link_ap_info;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_stop_request_t {
 	struct hostif_hdr header;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_stop_confirm_t {
 	struct hostif_hdr header;
 	uint16_t result_code;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_ps_adhoc_set_request_t {
 	struct hostif_hdr header;
@@ -360,12 +363,12 @@ struct hostif_ps_adhoc_set_request_t {
 	uint16_t capability;	/* bit5:preamble bit6:pbcc pbcc not supported always 0 
 				 * bit10:ShortSlotTime bit13:DSSS-OFDM DSSS-OFDM not supported always 0 */
 	uint16_t scan_type;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_ps_adhoc_set_confirm_t {
 	struct hostif_hdr header;
 	uint16_t result_code;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_infrastructure_set_request_t {
 	struct hostif_hdr header;
@@ -381,7 +384,7 @@ struct hostif_infrastructure_set_request_t {
 #define AUTH_TYPE_SHARED_KEY  1
 	struct channel_list_t channel_list;
 	uint16_t scan_type;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_infrastructure_set2_request_t {
 	struct hostif_hdr header;
@@ -398,12 +401,12 @@ struct hostif_infrastructure_set2_request_t {
 	struct channel_list_t channel_list;
 	uint16_t scan_type;
 	uint8_t bssid[ETH_ALEN];
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_infrastructure_set_confirm_t {
 	struct hostif_hdr header;
 	uint16_t result_code;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_adhoc_set_request_t {
 	struct hostif_hdr header;
@@ -415,7 +418,7 @@ struct hostif_adhoc_set_request_t {
 	uint16_t capability;	/* bit5:preamble bit6:pbcc pbcc not supported always 0 
 				 * bit10:ShortSlotTime bit13:DSSS-OFDM DSSS-OFDM not supported always 0 */
 	uint16_t scan_type;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_adhoc_set2_request_t {
 	struct hostif_hdr header;
@@ -429,17 +432,17 @@ struct hostif_adhoc_set2_request_t {
 	uint16_t scan_type;
 	struct channel_list_t channel_list;
 	uint8_t bssid[ETH_ALEN];
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_adhoc_set_confirm_t {
 	struct hostif_hdr header;
 	uint16_t result_code;
-} __attribute__ ((packed));
+} __packed;
 
 struct last_associate_t {
 	uint8_t type;
 	uint8_t status;
-} __attribute__ ((packed));
+} __packed;
 
 struct association_request_t {
 	uint8_t type;
@@ -450,7 +453,7 @@ struct association_request_t {
 	uint16_t listen_interval;
 	uint8_t ap_address[6];
 	uint16_t reqIEs_size;
-} __attribute__ ((packed));
+} __packed;
 
 struct association_response_t {
 	uint8_t type;
@@ -461,7 +464,7 @@ struct association_response_t {
 	uint16_t status;
 	uint16_t association_id;
 	uint16_t respIEs_size;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_associate_indication_t {
 	struct hostif_hdr header;
@@ -469,7 +472,7 @@ struct hostif_associate_indication_t {
 	struct association_response_t assoc_resp;
 	/* followed by (reqIEs_size + respIEs_size) octets of data */
 	/* reqIEs data *//* respIEs data */
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_bss_scan_request_t {
 	struct hostif_hdr header;
@@ -481,13 +484,13 @@ struct hostif_bss_scan_request_t {
 	uint32_t ch_time_max;
 	struct channel_list_t channel_list;
 	struct ssid_t ssid;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_bss_scan_confirm_t {
 	struct hostif_hdr header;
 	uint16_t result_code;
 	uint16_t reserved;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_phy_information_request_t {
 	struct hostif_hdr header;
@@ -495,7 +498,7 @@ struct hostif_phy_information_request_t {
 #define NORMAL_TYPE	0
 #define TIME_TYPE	1
 	uint16_t time;	/* unit 100ms */
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_phy_information_confirm_t {
 	struct hostif_hdr header;
@@ -507,30 +510,30 @@ struct hostif_phy_information_confirm_t {
 	uint32_t rx_frame;
 	uint32_t tx_error;
 	uint32_t rx_error;
-} __attribute__ ((packed));
+} __packed;
 
 /* sleep mode */
 #define SLP_ACTIVE  0
 #define SLP_SLEEP   1
 struct hostif_sleep_request_t {
 	struct hostif_hdr header;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_sleep_confirm_t {
 	struct hostif_hdr header;
 	uint16_t result_code;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_mic_failure_request_t {
 	struct hostif_hdr header;
 	uint16_t failure_count;
 	uint16_t timer;
-} __attribute__ ((packed));
+} __packed;
 
 struct hostif_mic_failure_confirm_t {
 	struct hostif_hdr header;
 	uint16_t result_code;
-} __attribute__ ((packed));
+} __packed;
 
 #define BASIC_RATE	0x80
 #define RATE_MASK	0x7F
@@ -616,13 +619,21 @@ enum {
 #include "ks_wlan.h"
 
 /* function prototype */
-extern int hostif_data_request(struct ks_wlan_private *priv,
-			       struct sk_buff *packet);
-extern void hostif_receive(struct ks_wlan_private *priv, unsigned char *p,
-			   unsigned int size);
-extern void hostif_sme_enqueue(struct ks_wlan_private *priv, uint16_t event);
-extern int hostif_init(struct ks_wlan_private *priv);
-extern void hostif_exit(struct ks_wlan_private *priv);
+int hostif_data_request(struct ks_wlan_private *priv,
+			 struct sk_buff *packet);
+void hostif_receive(struct ks_wlan_private *priv, unsigned char *p,
+	             unsigned int size);
+void hostif_sme_enqueue(struct ks_wlan_private *priv, uint16_t event);
+int hostif_init(struct ks_wlan_private *priv);
+void hostif_exit(struct ks_wlan_private *priv);
+int ks_wlan_hw_tx(struct ks_wlan_private *priv, void *p,
+		   unsigned long size,
+		   void (*complete_handler) (void *arg1, void *arg2),
+		   void *arg1, void *arg2);
+void send_packet_complete(void *, void *);
+
+void ks_wlan_hw_wakeup_request(struct ks_wlan_private *priv);
+int ks_wlan_hw_power_save(struct ks_wlan_private *priv);
 
 static
 inline int hif_align_size(int size)
diff --git a/drivers/staging/ks7010/ks_wlan.h b/drivers/staging/ks7010/ks_wlan.h
index f05dc0122fcb..c2cc288ae899 100644
--- a/drivers/staging/ks7010/ks_wlan.h
+++ b/drivers/staging/ks7010/ks_wlan.h
@@ -219,7 +219,7 @@ struct rsn_ie_t {
 	uint8_t id;	/* 0xdd = WPA or 0x30 = RSN */
 	uint8_t size;	/* max ? 255 ? */
 	uint8_t body[RSN_IE_BODY_MAX];
-} __attribute__ ((packed));
+} __packed;
 
 #ifdef WPS
 #define WPS_IE_BODY_MAX 255
@@ -227,7 +227,7 @@ struct wps_ie_t {
 	uint8_t id;	/* 221 'dd <len> 00 50 F2 04' */
 	uint8_t size;	/* max ? 255 ? */
 	uint8_t body[WPS_IE_BODY_MAX];
-} __attribute__ ((packed));
+} __packed;
 #endif /* WPS */
 
 struct local_ap_t {
@@ -499,7 +499,7 @@ struct ks_wlan_private {
 	uint wakeup_count;	/* for detect wakeup loop */
 };
 
-extern int ks_wlan_net_start(struct net_device *dev);
-extern int ks_wlan_net_stop(struct net_device *dev);
+int ks_wlan_net_start(struct net_device *dev);
+int ks_wlan_net_stop(struct net_device *dev);
 
 #endif /* _KS_WLAN_H */
diff --git a/drivers/staging/ks7010/ks_wlan_ioctl.h b/drivers/staging/ks7010/ks_wlan_ioctl.h
index 49369e497808..84554b6bb239 100644
--- a/drivers/staging/ks7010/ks_wlan_ioctl.h
+++ b/drivers/staging/ks7010/ks_wlan_ioctl.h
@@ -58,9 +58,9 @@
 #include "ks_wlan.h"
 #include <linux/netdevice.h>
 
-extern int ks_wlan_read_config_file(struct ks_wlan_private *priv);
-extern int ks_wlan_setup_parameter(struct ks_wlan_private *priv,
-				   unsigned int commit_flag);
+int ks_wlan_read_config_file(struct ks_wlan_private *priv);
+int ks_wlan_setup_parameter(struct ks_wlan_private *priv,
+		             unsigned int commit_flag);
 
 #endif /* __KERNEL__ */
 
diff --git a/drivers/staging/ks7010/ks_wlan_net.c b/drivers/staging/ks7010/ks_wlan_net.c
index 1e21eb1c4667..b2b4fa4c3834 100644
--- a/drivers/staging/ks7010/ks_wlan_net.c
+++ b/drivers/staging/ks7010/ks_wlan_net.c
@@ -9,7 +9,6 @@
  *   published by the Free Software Foundation.
  */
 
-#include <linux/version.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/compiler.h>
@@ -70,10 +69,6 @@ static const struct iw_handler_def ks_wlan_handler_def;
 /*
  *	function prototypes
  */
-extern int ks_wlan_hw_tx(struct ks_wlan_private *priv, void *p,
-			 unsigned long size,
-			 void (*complete_handler) (void *arg1, void *arg2),
-			 void *arg1, void *arg2);
 static int ks_wlan_open(struct net_device *dev);
 static void ks_wlan_tx_timeout(struct net_device *dev);
 static int ks_wlan_start_xmit(struct sk_buff *skb, struct net_device *dev);
@@ -238,9 +233,9 @@ static int ks_wlan_set_freq(struct net_device *dev,
 		/* We should do a better check than that,
 		 * based on the card capability !!! */
 		if ((channel < 1) || (channel > 14)) {
-			printk(KERN_DEBUG
-			       "%s: New channel value of %d is invalid!\n",
-			       dev->name, fwrq->m);
+			netdev_dbg(dev,
+				   "%s: New channel value of %d is invalid!\n",
+				   dev->name, fwrq->m);
 			rc = -EINVAL;
 		} else {
 			/* Yes ! We can set it !!! */
@@ -402,7 +397,7 @@ static int ks_wlan_set_wap(struct net_device *dev, struct iw_request_info *info,
 			priv->need_commit |= SME_MODE_SET;
 		}
 	} else {
-		memset(priv->reg.bssid, 0x0, ETH_ALEN);
+		eth_zero_addr(priv->reg.bssid);
 		return -EOPNOTSUPP;
 	}
 
@@ -433,7 +428,7 @@ static int ks_wlan_get_wap(struct net_device *dev, struct iw_request_info *info,
 	if ((priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
 		memcpy(awrq->sa_data, &(priv->current_ap.bssid[0]), ETH_ALEN);
 	} else {
-		memset(awrq->sa_data, 0, ETH_ALEN);
+		eth_zero_addr(awrq->sa_data);
 	}
 
 	awrq->sa_family = ARPHRD_ETHER;
@@ -2092,7 +2087,7 @@ static int ks_wlan_set_pmksa(struct net_device *dev,
 			list_for_each(ptr, &priv->pmklist.head) {
 				pmk = list_entry(ptr, struct pmk_t, list);
 				if (!memcmp(pmksa->bssid.sa_data, pmk->bssid, ETH_ALEN)) {	/* match address! list del. */
-					memset(pmk->bssid, 0, ETH_ALEN);
+					eth_zero_addr(pmk->bssid);
 					memset(pmk->pmkid, 0, IW_PMKID_LEN);
 					list_del_init(&pmk->list);
 					break;
@@ -2676,17 +2671,17 @@ static int ks_wlan_set_sleep_mode(struct net_device *dev,
 
 	if (*uwrq == SLP_SLEEP) {
 		priv->sleep_mode = *uwrq;
-		printk("SET_SLEEP_MODE %d\n", priv->sleep_mode);
+		netdev_info(dev, "SET_SLEEP_MODE %d\n", priv->sleep_mode);
 
 		hostif_sme_enqueue(priv, SME_STOP_REQUEST);
 		hostif_sme_enqueue(priv, SME_SLEEP_REQUEST);
 
 	} else if (*uwrq == SLP_ACTIVE) {
 		priv->sleep_mode = *uwrq;
-		printk("SET_SLEEP_MODE %d\n", priv->sleep_mode);
+		netdev_info(dev, "SET_SLEEP_MODE %d\n", priv->sleep_mode);
 		hostif_sme_enqueue(priv, SME_SLEEP_REQUEST);
 	} else {
-		printk("SET_SLEEP_MODE %d errror\n", *uwrq);
+		netdev_err(dev, "SET_SLEEP_MODE %d errror\n", *uwrq);
 		return -EINVAL;
 	}
 
@@ -2788,7 +2783,7 @@ static int ks_wlan_get_wps_enable(struct net_device *dev,
 	}
 	/* for SLEEP MODE */
 	*uwrq = priv->wps.wps_enabled;
-	printk("return=%d\n", *uwrq);
+	netdev_info(dev, "return=%d\n", *uwrq);
 
 	return 0;
 }
@@ -2978,117 +2973,117 @@ static int ks_wlan_get_eeprom_cksum(struct net_device *dev,
 	return 0;
 }
 
-static void print_hif_event(int event)
+static void print_hif_event(struct net_device *dev, int event)
 {
 
 	switch (event) {
 	case HIF_DATA_REQ:
-		printk("HIF_DATA_REQ\n");
+		netdev_info(dev, "HIF_DATA_REQ\n");
 		break;
 	case HIF_DATA_IND:
-		printk("HIF_DATA_IND\n");
+		netdev_info(dev, "HIF_DATA_IND\n");
 		break;
 	case HIF_MIB_GET_REQ:
-		printk("HIF_MIB_GET_REQ\n");
+		netdev_info(dev, "HIF_MIB_GET_REQ\n");
 		break;
 	case HIF_MIB_GET_CONF:
-		printk("HIF_MIB_GET_CONF\n");
+		netdev_info(dev, "HIF_MIB_GET_CONF\n");
 		break;
 	case HIF_MIB_SET_REQ:
-		printk("HIF_MIB_SET_REQ\n");
+		netdev_info(dev, "HIF_MIB_SET_REQ\n");
 		break;
 	case HIF_MIB_SET_CONF:
-		printk("HIF_MIB_SET_CONF\n");
+		netdev_info(dev, "HIF_MIB_SET_CONF\n");
 		break;
 	case HIF_POWERMGT_REQ:
-		printk("HIF_POWERMGT_REQ\n");
+		netdev_info(dev, "HIF_POWERMGT_REQ\n");
 		break;
 	case HIF_POWERMGT_CONF:
-		printk("HIF_POWERMGT_CONF\n");
+		netdev_info(dev, "HIF_POWERMGT_CONF\n");
 		break;
 	case HIF_START_REQ:
-		printk("HIF_START_REQ\n");
+		netdev_info(dev, "HIF_START_REQ\n");
 		break;
 	case HIF_START_CONF:
-		printk("HIF_START_CONF\n");
+		netdev_info(dev, "HIF_START_CONF\n");
 		break;
 	case HIF_CONNECT_IND:
-		printk("HIF_CONNECT_IND\n");
+		netdev_info(dev, "HIF_CONNECT_IND\n");
 		break;
 	case HIF_STOP_REQ:
-		printk("HIF_STOP_REQ\n");
+		netdev_info(dev, "HIF_STOP_REQ\n");
 		break;
 	case HIF_STOP_CONF:
-		printk("HIF_STOP_CONF\n");
+		netdev_info(dev, "HIF_STOP_CONF\n");
 		break;
 	case HIF_PS_ADH_SET_REQ:
-		printk("HIF_PS_ADH_SET_REQ\n");
+		netdev_info(dev, "HIF_PS_ADH_SET_REQ\n");
 		break;
 	case HIF_PS_ADH_SET_CONF:
-		printk("HIF_PS_ADH_SET_CONF\n");
+		netdev_info(dev, "HIF_PS_ADH_SET_CONF\n");
 		break;
 	case HIF_INFRA_SET_REQ:
-		printk("HIF_INFRA_SET_REQ\n");
+		netdev_info(dev, "HIF_INFRA_SET_REQ\n");
 		break;
 	case HIF_INFRA_SET_CONF:
-		printk("HIF_INFRA_SET_CONF\n");
+		netdev_info(dev, "HIF_INFRA_SET_CONF\n");
 		break;
 	case HIF_ADH_SET_REQ:
-		printk("HIF_ADH_SET_REQ\n");
+		netdev_info(dev, "HIF_ADH_SET_REQ\n");
 		break;
 	case HIF_ADH_SET_CONF:
-		printk("HIF_ADH_SET_CONF\n");
+		netdev_info(dev, "HIF_ADH_SET_CONF\n");
 		break;
 	case HIF_AP_SET_REQ:
-		printk("HIF_AP_SET_REQ\n");
+		netdev_info(dev, "HIF_AP_SET_REQ\n");
 		break;
 	case HIF_AP_SET_CONF:
-		printk("HIF_AP_SET_CONF\n");
+		netdev_info(dev, "HIF_AP_SET_CONF\n");
 		break;
 	case HIF_ASSOC_INFO_IND:
-		printk("HIF_ASSOC_INFO_IND\n");
+		netdev_info(dev, "HIF_ASSOC_INFO_IND\n");
 		break;
 	case HIF_MIC_FAILURE_REQ:
-		printk("HIF_MIC_FAILURE_REQ\n");
+		netdev_info(dev, "HIF_MIC_FAILURE_REQ\n");
 		break;
 	case HIF_MIC_FAILURE_CONF:
-		printk("HIF_MIC_FAILURE_CONF\n");
+		netdev_info(dev, "HIF_MIC_FAILURE_CONF\n");
 		break;
 	case HIF_SCAN_REQ:
-		printk("HIF_SCAN_REQ\n");
+		netdev_info(dev, "HIF_SCAN_REQ\n");
 		break;
 	case HIF_SCAN_CONF:
-		printk("HIF_SCAN_CONF\n");
+		netdev_info(dev, "HIF_SCAN_CONF\n");
 		break;
 	case HIF_PHY_INFO_REQ:
-		printk("HIF_PHY_INFO_REQ\n");
+		netdev_info(dev, "HIF_PHY_INFO_REQ\n");
 		break;
 	case HIF_PHY_INFO_CONF:
-		printk("HIF_PHY_INFO_CONF\n");
+		netdev_info(dev, "HIF_PHY_INFO_CONF\n");
 		break;
 	case HIF_SLEEP_REQ:
-		printk("HIF_SLEEP_REQ\n");
+		netdev_info(dev, "HIF_SLEEP_REQ\n");
 		break;
 	case HIF_SLEEP_CONF:
-		printk("HIF_SLEEP_CONF\n");
+		netdev_info(dev, "HIF_SLEEP_CONF\n");
 		break;
 	case HIF_PHY_INFO_IND:
-		printk("HIF_PHY_INFO_IND\n");
+		netdev_info(dev, "HIF_PHY_INFO_IND\n");
 		break;
 	case HIF_SCAN_IND:
-		printk("HIF_SCAN_IND\n");
+		netdev_info(dev, "HIF_SCAN_IND\n");
 		break;
 	case HIF_INFRA_SET2_REQ:
-		printk("HIF_INFRA_SET2_REQ\n");
+		netdev_info(dev, "HIF_INFRA_SET2_REQ\n");
 		break;
 	case HIF_INFRA_SET2_CONF:
-		printk("HIF_INFRA_SET2_CONF\n");
+		netdev_info(dev, "HIF_INFRA_SET2_CONF\n");
 		break;
 	case HIF_ADH_SET2_REQ:
-		printk("HIF_ADH_SET2_REQ\n");
+		netdev_info(dev, "HIF_ADH_SET2_REQ\n");
 		break;
 	case HIF_ADH_SET2_CONF:
-		printk("HIF_ADH_SET2_CONF\n");
+		netdev_info(dev, "HIF_ADH_SET2_CONF\n");
 	}
 }
 
@@ -3105,7 +3100,7 @@ static int ks_wlan_hostt(struct net_device *dev, struct iw_request_info *info,
 		event =
 		    priv->hostt.buff[(priv->hostt.qtail - 1 - i) %
 				     SME_EVENT_BUFF_SIZE];
-		print_hif_event(event);
+		print_hif_event(dev, event);
 	}
 	return 0;
 }
@@ -3335,7 +3330,7 @@ int ks_wlan_set_mac_address(struct net_device *dev, void *addr)
 
 	priv->mac_address_valid = 0;
 	hostif_sme_enqueue(priv, SME_MACADDRESS_SET_REQUEST);
-	printk(KERN_INFO
+	netdev_info(dev,
 	       "ks_wlan: MAC ADDRESS = %02x:%02x:%02x:%02x:%02x:%02x\n",
 	       priv->eth_addr[0], priv->eth_addr[1], priv->eth_addr[2],
 	       priv->eth_addr[3], priv->eth_addr[4], priv->eth_addr[5]);
@@ -3354,8 +3349,6 @@ void ks_wlan_tx_timeout(struct net_device *dev)
 	}
 	priv->nstats.tx_errors++;
 	netif_wake_queue(dev);
-
-	return;
 }
 
 static
@@ -3366,8 +3359,8 @@ int ks_wlan_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	DPRINTK(3, "in_interrupt()=%ld\n", in_interrupt());
 
-	if (skb == NULL) {
-		printk(KERN_ERR "ks_wlan:  skb == NULL!!!\n");
+	if (!skb) {
+		netdev_err(dev, "ks_wlan:  skb == NULL!!!\n");
 		return 0;
 	}
 	if (priv->dev_state < DEVICE_STATE_READY) {
@@ -3396,13 +3389,13 @@ void send_packet_complete(void *arg1, void *arg2)
 
 	DPRINTK(3, "\n");
 
-	priv->nstats.tx_bytes += packet->len;
 	priv->nstats.tx_packets++;
 
 	if (netif_queue_stopped(priv->net_dev))
 		netif_wake_queue(priv->net_dev);
 
 	if (packet) {
+		priv->nstats.tx_bytes += packet->len;
 		dev_kfree_skb(packet);
 		packet = NULL;
 	}
@@ -3421,8 +3414,6 @@ void ks_wlan_set_multicast_list(struct net_device *dev)
 		return;	/* not finished initialize */
 	}
 	hostif_sme_enqueue(priv, SME_MULTICAST_REQUEST);
-
-	return;
 }
 
 static
@@ -3433,7 +3424,7 @@ int ks_wlan_open(struct net_device *dev)
 	priv->cur_rx = 0;
 
 	if (!priv->mac_address_valid) {
-		printk(KERN_ERR "ks_wlan : %s Not READY !!\n", dev->name);
+		netdev_err(dev, "ks_wlan : %s Not READY !!\n", dev->name);
 		return -EBUSY;
 	} else
 		netif_start_queue(dev);
@@ -3512,17 +3503,11 @@ int ks_wlan_net_stop(struct net_device *dev)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
-	int ret = 0;
 	priv->device_open_status = 0;
 	del_timer_sync(&update_phyinfo_timer);
 
 	if (netif_running(dev))
 		netif_stop_queue(dev);
 
-	return ret;
-}
-
-int ks_wlan_reset(struct net_device *dev)
-{
 	return 0;
 }
diff --git a/drivers/staging/ks7010/michael_mic.c b/drivers/staging/ks7010/michael_mic.c
index e14c109b3cab..78ae2b8fb7f3 100644
--- a/drivers/staging/ks7010/michael_mic.c
+++ b/drivers/staging/ks7010/michael_mic.c
@@ -20,18 +20,24 @@
 #define getUInt32( A, B ) 	(uint32_t)(A[B+0] << 0) + (A[B+1] << 8) + (A[B+2] << 16) + (A[B+3] << 24)
 
 // Convert from UInt32 to Byte[] in a portable way
-#define putUInt32( A, B, C ) 	A[B+0] = (uint8_t) (C & 0xff);		\
-				A[B+1] = (uint8_t) ((C>>8) & 0xff);	\
-				A[B+2] = (uint8_t) ((C>>16) & 0xff);	\
-				A[B+3] = (uint8_t) ((C>>24) & 0xff)
+#define putUInt32(A, B, C)					\
+do {								\
+	A[B + 0] = (uint8_t)(C & 0xff);				\
+	A[B + 1] = (uint8_t)((C >> 8) & 0xff);			\
+	A[B + 2] = (uint8_t)((C >> 16) & 0xff);			\
+	A[B + 3] = (uint8_t)((C >> 24) & 0xff);			\
+} while (0)
 
 // Reset the state to the empty message.
-#define MichaelClear( A ) 	A->L = A->K0; \
-				A->R = A->K1; \
-				A->nBytesInM = 0;
+#define MichaelClear(A)			\
+do {					\
+	A->L = A->K0;			\
+	A->R = A->K1;			\
+	A->nBytesInM = 0;		\
+} while (0)
 
 static
-void MichaelInitializeFunction(struct michel_mic_t *Mic, uint8_t * key)
+void MichaelInitializeFunction(struct michel_mic_t *Mic, uint8_t *key)
 {
 	// Set the key
 	Mic->K0 = getUInt32(key, 0);
@@ -54,7 +60,7 @@ do{								\
 }while(0)
 
 static
-void MichaelAppend(struct michel_mic_t *Mic, uint8_t * src, int nBytes)
+void MichaelAppend(struct michel_mic_t *Mic, uint8_t *src, int nBytes)
 {
 	int addlen;
 	if (Mic->nBytesInM) {
@@ -88,7 +94,7 @@ void MichaelAppend(struct michel_mic_t *Mic, uint8_t * src, int nBytes)
 }
 
 static
-void MichaelGetMIC(struct michel_mic_t *Mic, uint8_t * dst)
+void MichaelGetMIC(struct michel_mic_t *Mic, uint8_t *dst)
 {
 	uint8_t *data = Mic->M;
 	switch (Mic->nBytesInM) {
@@ -116,9 +122,9 @@ void MichaelGetMIC(struct michel_mic_t *Mic, uint8_t * dst)
 	MichaelClear(Mic);
 }
 
-void MichaelMICFunction(struct michel_mic_t *Mic, uint8_t * Key,
-			uint8_t * Data, int Len, uint8_t priority,
-			uint8_t * Result)
+void MichaelMICFunction(struct michel_mic_t *Mic, uint8_t *Key,
+			uint8_t *Data, int Len, uint8_t priority,
+			uint8_t *Result)
 {
 	uint8_t pad_data[4] = { priority, 0, 0, 0 };
 	// Compute the MIC value
diff --git a/drivers/staging/ks7010/michael_mic.h b/drivers/staging/ks7010/michael_mic.h
index c7e4eb280961..efaa21788fc7 100644
--- a/drivers/staging/ks7010/michael_mic.h
+++ b/drivers/staging/ks7010/michael_mic.h
@@ -20,7 +20,6 @@ struct michel_mic_t {
 	uint8_t Result[8];
 };
 
-extern
-void MichaelMICFunction(struct michel_mic_t *Mic, uint8_t * Key,
-			uint8_t * Data, int Len, uint8_t priority,
-			uint8_t * Result);
+void MichaelMICFunction(struct michel_mic_t *Mic, uint8_t *Key,
+			uint8_t *Data, int Len, uint8_t priority,
+			uint8_t *Result);
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs.h b/drivers/staging/lustre/include/linux/libcfs/libcfs.h
index 3f6447c65042..3b92d38d37e2 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs.h
@@ -138,8 +138,8 @@ struct lnet_debugfs_symlink_def {
 void lustre_insert_debugfs(struct ctl_table *table,
 			   const struct lnet_debugfs_symlink_def *symlinks);
 int lprocfs_call_handler(void *data, int write, loff_t *ppos,
-			  void __user *buffer, size_t *lenp,
-			  int (*handler)(void *data, int write,
-			  loff_t pos, void __user *buffer, int len));
+			 void __user *buffer, size_t *lenp,
+			 int (*handler)(void *data, int write, loff_t pos,
+					void __user *buffer, int len));
 
 #endif /* _LIBCFS_H */
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h
index 25adab19fd86..b7bd6e8ab33f 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h
@@ -247,19 +247,19 @@ do {									\
 #define LCONSOLE_EMERG(format, ...) CDEBUG(D_CONSOLE | D_EMERG, format, ## __VA_ARGS__)
 
 int libcfs_debug_msg(struct libcfs_debug_msg_data *msgdata,
-			    const char *format1, ...)
+		     const char *format1, ...)
 	__printf(2, 3);
 
 int libcfs_debug_vmsg2(struct libcfs_debug_msg_data *msgdata,
-			      const char *format1,
-			      va_list args, const char *format2, ...)
+		       const char *format1,
+		       va_list args, const char *format2, ...)
 	__printf(4, 5);
 
 /* other external symbols that tracefile provides: */
 int cfs_trace_copyin_string(char *knl_buffer, int knl_buffer_nob,
-		const char __user *usr_buffer, int usr_buffer_nob);
+			    const char __user *usr_buffer, int usr_buffer_nob);
 int cfs_trace_copyout_string(char __user *usr_buffer, int usr_buffer_nob,
-		const char *knl_buffer, char *append);
+			     const char *knl_buffer, char *append);
 
 #define LIBCFS_DEBUG_FILE_PATH_DEFAULT "/tmp/lustre-log"
 
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h
index d3f9a6020ee3..bdbbe934584c 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h
@@ -143,6 +143,9 @@ static inline int cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
 #define CFS_FAIL_TIMEOUT_ORSET(id, value, secs) \
 	cfs_fail_timeout_set(id, value, secs * 1000, CFS_FAIL_LOC_ORSET)
 
+#define CFS_FAIL_TIMEOUT_RESET(id, value, secs) \
+	cfs_fail_timeout_set(id, value, secs * 1000, CFS_FAIL_LOC_RESET)
+
 #define CFS_FAIL_TIMEOUT_MS_ORSET(id, value, ms) \
 	cfs_fail_timeout_set(id, value, ms, CFS_FAIL_LOC_ORSET)
 
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
index 4daa3823f60a..e0e1a5d0949d 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
@@ -310,13 +310,13 @@ do {							    \
 
 #define MKSTR(ptr) ((ptr)) ? (ptr) : ""
 
-static inline int cfs_size_round4(int val)
+static inline size_t cfs_size_round4(int val)
 {
 	return (val + 3) & (~0x3);
 }
 
 #ifndef HAVE_CFS_SIZE_ROUND
-static inline int cfs_size_round(int val)
+static inline size_t cfs_size_round(int val)
 {
 	return (val + 7) & (~0x7);
 }
@@ -324,17 +324,17 @@ static inline int cfs_size_round(int val)
 #define HAVE_CFS_SIZE_ROUND
 #endif
 
-static inline int cfs_size_round16(int val)
+static inline size_t cfs_size_round16(int val)
 {
 	return (val + 0xf) & (~0xf);
 }
 
-static inline int cfs_size_round32(int val)
+static inline size_t cfs_size_round32(int val)
 {
 	return (val + 0x1f) & (~0x1f);
 }
 
-static inline int cfs_size_round0(int val)
+static inline size_t cfs_size_round0(int val)
 {
 	if (!val)
 		return 0;
@@ -343,7 +343,7 @@ static inline int cfs_size_round0(int val)
 
 static inline size_t cfs_round_strlen(char *fset)
 {
-	return (size_t)cfs_size_round((int)strlen(fset) + 1);
+	return cfs_size_round((int)strlen(fset) + 1);
 }
 
 #define LOGL(var, len, ptr)				       \
@@ -360,13 +360,4 @@ do {							    \
 	ptr += cfs_size_round(len);			     \
 } while (0)
 
-#define LOGL0(var, len, ptr)			      \
-do {						    \
-	if (!len)				       \
-		break;				  \
-	memcpy((char *)ptr, (const char *)var, len);    \
-	*((char *)(ptr) + len) = 0;		     \
-	ptr += cfs_size_round(len + 1);		 \
-} while (0)
-
 #endif
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
index 513a8225f888..a59c5e99cbd3 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
@@ -605,73 +605,20 @@ void lnet_counters_reset(void);
 
 unsigned int lnet_iov_nob(unsigned int niov, struct kvec *iov);
 int lnet_extract_iov(int dst_niov, struct kvec *dst,
-		     int src_niov, struct kvec *src,
+		     int src_niov, const struct kvec *src,
 		      unsigned int offset, unsigned int len);
 
 unsigned int lnet_kiov_nob(unsigned int niov, lnet_kiov_t *iov);
 int lnet_extract_kiov(int dst_niov, lnet_kiov_t *dst,
-		      int src_niov, lnet_kiov_t *src,
+		      int src_niov, const lnet_kiov_t *src,
 		      unsigned int offset, unsigned int len);
 
-void lnet_copy_iov2iov(unsigned int ndiov, struct kvec *diov,
-		       unsigned int doffset,
-			unsigned int nsiov, struct kvec *siov,
+void lnet_copy_iov2iter(struct iov_iter *to,
+			unsigned int nsiov, const struct kvec *siov,
 			unsigned int soffset, unsigned int nob);
-void lnet_copy_kiov2iov(unsigned int niov, struct kvec *iov,
-			unsigned int iovoffset,
-			 unsigned int nkiov, lnet_kiov_t *kiov,
+void lnet_copy_kiov2iter(struct iov_iter *to,
+			 unsigned int nkiov, const lnet_kiov_t *kiov,
 			 unsigned int kiovoffset, unsigned int nob);
-void lnet_copy_iov2kiov(unsigned int nkiov, lnet_kiov_t *kiov,
-			unsigned int kiovoffset,
-			 unsigned int niov, struct kvec *iov,
-			 unsigned int iovoffset, unsigned int nob);
-void lnet_copy_kiov2kiov(unsigned int ndkiov, lnet_kiov_t *dkiov,
-			 unsigned int doffset,
-			  unsigned int nskiov, lnet_kiov_t *skiov,
-			  unsigned int soffset, unsigned int nob);
-
-static inline void
-lnet_copy_iov2flat(int dlen, void *dest, unsigned int doffset,
-		   unsigned int nsiov, struct kvec *siov, unsigned int soffset,
-		   unsigned int nob)
-{
-	struct kvec diov = {/*.iov_base = */ dest, /*.iov_len = */ dlen};
-
-	lnet_copy_iov2iov(1, &diov, doffset,
-			  nsiov, siov, soffset, nob);
-}
-
-static inline void
-lnet_copy_kiov2flat(int dlen, void *dest, unsigned int doffset,
-		    unsigned int nsiov, lnet_kiov_t *skiov,
-		    unsigned int soffset, unsigned int nob)
-{
-	struct kvec diov = {/* .iov_base = */ dest, /* .iov_len = */ dlen};
-
-	lnet_copy_kiov2iov(1, &diov, doffset,
-			   nsiov, skiov, soffset, nob);
-}
-
-static inline void
-lnet_copy_flat2iov(unsigned int ndiov, struct kvec *diov, unsigned int doffset,
-		   int slen, void *src, unsigned int soffset, unsigned int nob)
-{
-	struct kvec siov = {/*.iov_base = */ src, /*.iov_len = */slen};
-
-	lnet_copy_iov2iov(ndiov, diov, doffset,
-			  1, &siov, soffset, nob);
-}
-
-static inline void
-lnet_copy_flat2kiov(unsigned int ndiov, lnet_kiov_t *dkiov,
-		    unsigned int doffset, int slen, void *src,
-		    unsigned int soffset, unsigned int nob)
-{
-	struct kvec siov = {/* .iov_base = */ src, /* .iov_len = */ slen};
-
-	lnet_copy_iov2kiov(ndiov, dkiov, doffset,
-			   1, &siov, soffset, nob);
-}
 
 void lnet_me_unlink(lnet_me_t *me);
 
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
index 7967b013cbae..b84a5bb9186c 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
@@ -220,10 +220,7 @@ typedef struct lnet_lnd {
 	 * credit if the LND does flow control.
 	 */
 	int (*lnd_recv)(struct lnet_ni *ni, void *private, lnet_msg_t *msg,
-			int delayed, unsigned int niov,
-			struct kvec *iov, lnet_kiov_t *kiov,
-			unsigned int offset, unsigned int mlen,
-			unsigned int rlen);
+			int delayed, struct iov_iter *to, unsigned int rlen);
 
 	/*
 	 * lnet_parse() has had to delay processing of this message
@@ -278,6 +275,8 @@ typedef struct lnet_ni {
 	struct lnet_ioctl_config_lnd_tunables *ni_lnd_tunables;
 	/* equivalent interfaces to use */
 	char			 *ni_interfaces[LNET_MAX_INTERFACES];
+	/* original net namespace */
+	struct net		 *ni_net_ns;
 } lnet_ni_t;
 
 #define LNET_PROTO_PING_MATCHBITS	0x8000000000000000LL
diff --git a/drivers/staging/lustre/include/linux/lnet/types.h b/drivers/staging/lustre/include/linux/lnet/types.h
index e098b6c086e1..f8be0e2f7bf7 100644
--- a/drivers/staging/lustre/include/linux/lnet/types.h
+++ b/drivers/staging/lustre/include/linux/lnet/types.h
@@ -503,21 +503,7 @@ typedef struct {
 /* NB lustre portals uses struct iovec internally! */
 typedef struct iovec lnet_md_iovec_t;
 
-/**
- * A page-based fragment of a MD.
- */
-typedef struct {
-	/** Pointer to the page where the fragment resides */
-	struct page	*kiov_page;
-	/** Length in bytes of the fragment */
-	unsigned int	 kiov_len;
-	/**
-	 * Starting offset of the fragment within the page. Note that the
-	 * end of the fragment must not pass the end of the page; i.e.,
-	 * kiov_len + kiov_offset <= PAGE_SIZE.
-	 */
-	unsigned int	 kiov_offset;
-} lnet_kiov_t;
+typedef struct bio_vec lnet_kiov_t;
 /** @} lnet_md */
 
 /** \addtogroup lnet_eq
diff --git a/drivers/staging/lustre/lnet/Kconfig b/drivers/staging/lustre/lnet/Kconfig
index 2b5930150cda..13b43278a38d 100644
--- a/drivers/staging/lustre/lnet/Kconfig
+++ b/drivers/staging/lustre/lnet/Kconfig
@@ -35,6 +35,7 @@ config LNET_SELFTEST
 config LNET_XPRT_IB
 	tristate "LNET infiniband support"
 	depends on LNET && INFINIBAND && INFINIBAND_ADDR_TRANS
+	depends on BROKEN
 	default LNET && INFINIBAND
 	help
 	  This option allows the LNET users to use infiniband as an
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
index 4f5978b3767b..9e8802181452 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
@@ -128,6 +128,7 @@ static int kiblnd_msgtype2size(int type)
 static int kiblnd_unpack_rd(struct kib_msg *msg, int flip)
 {
 	struct kib_rdma_desc *rd;
+	int msg_size;
 	int nob;
 	int n;
 	int i;
@@ -146,12 +147,6 @@ static int kiblnd_unpack_rd(struct kib_msg *msg, int flip)
 
 	n = rd->rd_nfrags;
 
-	if (n <= 0 || n > IBLND_MAX_RDMA_FRAGS) {
-		CERROR("Bad nfrags: %d, should be 0 < n <= %d\n",
-		       n, IBLND_MAX_RDMA_FRAGS);
-		return 1;
-	}
-
 	nob = offsetof(struct kib_msg, ibm_u) +
 	      kiblnd_rd_msg_size(rd, msg->ibm_type, n);
 
@@ -161,6 +156,13 @@ static int kiblnd_unpack_rd(struct kib_msg *msg, int flip)
 		return 1;
 	}
 
+	msg_size = kiblnd_rd_size(rd);
+	if (msg_size <= 0 || msg_size > LNET_MAX_PAYLOAD) {
+		CERROR("Bad msg_size: %d, should be 0 < n <= %d\n",
+		       msg_size, LNET_MAX_PAYLOAD);
+		return 1;
+	}
+
 	if (!flip)
 		return 0;
 
@@ -618,7 +620,7 @@ static int kiblnd_get_completion_vector(struct kib_conn *conn, int cpt)
 }
 
 struct kib_conn *kiblnd_create_conn(struct kib_peer *peer, struct rdma_cm_id *cmid,
-			       int state, int version)
+				    int state, int version)
 {
 	/*
 	 * CAVEAT EMPTOR:
@@ -2465,7 +2467,7 @@ int kiblnd_dev_failover(struct kib_dev *dev)
 	hdev->ibh_cmid  = cmid;
 	hdev->ibh_ibdev = cmid->device;
 
-	pd = ib_alloc_pd(cmid->device);
+	pd = ib_alloc_pd(cmid->device, 0);
 	if (IS_ERR(pd)) {
 		rc = PTR_ERR(pd);
 		CERROR("Can't allocate PD: %d\n", rc);
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index 078a0c3e8845..14576977200f 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -113,8 +113,9 @@ extern struct kib_tunables  kiblnd_tunables;
 #define IBLND_OOB_CAPABLE(v)       ((v) != IBLND_MSG_VERSION_1)
 #define IBLND_OOB_MSGS(v)	   (IBLND_OOB_CAPABLE(v) ? 2 : 0)
 
-#define IBLND_MSG_SIZE		(4 << 10)	 /* max size of queued messages (inc hdr) */
-#define IBLND_MAX_RDMA_FRAGS	 LNET_MAX_IOV	   /* max # of fragments supported */
+#define IBLND_FRAG_SHIFT	(PAGE_SHIFT - 12)	/* frag size on wire is in 4K units */
+#define IBLND_MSG_SIZE		(4 << 10)		/* max size of queued messages (inc hdr) */
+#define IBLND_MAX_RDMA_FRAGS	(LNET_MAX_PAYLOAD >> 12)/* max # of fragments supported in 4K size */
 
 /************************/
 /* derived constants... */
@@ -133,8 +134,8 @@ extern struct kib_tunables  kiblnd_tunables;
 /* WRs and CQEs (per connection) */
 #define IBLND_RECV_WRS(c)	IBLND_RX_MSGS(c)
 #define IBLND_SEND_WRS(c)	\
-	((c->ibc_max_frags + 1) * kiblnd_concurrent_sends(c->ibc_version, \
-							  c->ibc_peer->ibp_ni))
+	(((c->ibc_max_frags + 1) << IBLND_FRAG_SHIFT) * \
+	  kiblnd_concurrent_sends(c->ibc_version, c->ibc_peer->ibp_ni))
 #define IBLND_CQ_ENTRIES(c)	(IBLND_RECV_WRS(c) + IBLND_SEND_WRS(c))
 
 struct kib_hca_dev;
@@ -582,6 +583,8 @@ struct kib_peer {
 	unsigned short		ibp_connecting;
 	/* reconnect this peer later */
 	unsigned short		ibp_reconnecting:1;
+	/* counter of how many times we triggered a conn race */
+	unsigned char		ibp_races;
 	/* # consecutive reconnection attempts to this peer */
 	unsigned int		ibp_reconnected;
 	/* errno on closing this peer */
@@ -607,14 +610,14 @@ kiblnd_cfg_rdma_frags(struct lnet_ni *ni)
 
 	tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
 	mod = tunables->lnd_map_on_demand;
-	return mod ? mod : IBLND_MAX_RDMA_FRAGS;
+	return mod ? mod : IBLND_MAX_RDMA_FRAGS >> IBLND_FRAG_SHIFT;
 }
 
 static inline int
 kiblnd_rdma_frags(int version, struct lnet_ni *ni)
 {
 	return version == IBLND_MSG_VERSION_1 ?
-			  IBLND_MAX_RDMA_FRAGS :
+			  (IBLND_MAX_RDMA_FRAGS >> IBLND_FRAG_SHIFT) :
 			  kiblnd_cfg_rdma_frags(ni);
 }
 
@@ -1034,5 +1037,4 @@ int  kiblnd_post_rx(struct kib_rx *rx, int credit);
 
 int  kiblnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg);
 int  kiblnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, int delayed,
-		 unsigned int niov, struct kvec *iov, lnet_kiov_t *kiov,
-		 unsigned int offset, unsigned int mlen, unsigned int rlen);
+		 struct iov_iter *to, unsigned int rlen);
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 596a697b9d39..b27de8888149 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -36,16 +36,19 @@
 
 #include "o2iblnd.h"
 
+#define MAX_CONN_RACES_BEFORE_ABORT 20
+
 static void kiblnd_peer_alive(struct kib_peer *peer);
 static void kiblnd_peer_connect_failed(struct kib_peer *peer, int active, int error);
-static void kiblnd_check_sends(struct kib_conn *conn);
 static void kiblnd_init_tx_msg(lnet_ni_t *ni, struct kib_tx *tx,
-				int type, int body_nob);
+			       int type, int body_nob);
 static int kiblnd_init_rdma(struct kib_conn *conn, struct kib_tx *tx, int type,
-			     int resid, struct kib_rdma_desc *dstrd, __u64 dstcookie);
+			    int resid, struct kib_rdma_desc *dstrd,
+			    __u64 dstcookie);
 static void kiblnd_queue_tx_locked(struct kib_tx *tx, struct kib_conn *conn);
 static void kiblnd_queue_tx(struct kib_tx *tx, struct kib_conn *conn);
 static void kiblnd_unmap_tx(lnet_ni_t *ni, struct kib_tx *tx);
+static void kiblnd_check_sends_locked(struct kib_conn *conn);
 
 static void
 kiblnd_tx_done(lnet_ni_t *ni, struct kib_tx *tx)
@@ -211,9 +214,9 @@ kiblnd_post_rx(struct kib_rx *rx, int credit)
 		conn->ibc_outstanding_credits++;
 	else
 		conn->ibc_reserved_credits++;
+	kiblnd_check_sends_locked(conn);
 	spin_unlock(&conn->ibc_lock);
 
-	kiblnd_check_sends(conn);
 out:
 	kiblnd_conn_decref(conn);
 	return rc;
@@ -344,8 +347,8 @@ kiblnd_handle_rx(struct kib_rx *rx)
 		    !IBLND_OOB_CAPABLE(conn->ibc_version)) /* v1 only */
 			conn->ibc_outstanding_credits++;
 
+		kiblnd_check_sends_locked(conn);
 		spin_unlock(&conn->ibc_lock);
-		kiblnd_check_sends(conn);
 	}
 
 	switch (msg->ibm_type) {
@@ -648,7 +651,7 @@ static int kiblnd_map_tx(lnet_ni_t *ni, struct kib_tx *tx, struct kib_rdma_desc
 
 static int
 kiblnd_setup_rd_iov(lnet_ni_t *ni, struct kib_tx *tx, struct kib_rdma_desc *rd,
-		    unsigned int niov, struct kvec *iov, int offset, int nob)
+		    unsigned int niov, const struct kvec *iov, int offset, int nob)
 {
 	struct kib_net *net = ni->ni_data;
 	struct page *page;
@@ -705,7 +708,7 @@ kiblnd_setup_rd_iov(lnet_ni_t *ni, struct kib_tx *tx, struct kib_rdma_desc *rd,
 
 static int
 kiblnd_setup_rd_kiov(lnet_ni_t *ni, struct kib_tx *tx, struct kib_rdma_desc *rd,
-		     int nkiov, lnet_kiov_t *kiov, int offset, int nob)
+		     int nkiov, const lnet_kiov_t *kiov, int offset, int nob)
 {
 	struct kib_net *net = ni->ni_data;
 	struct scatterlist *sg;
@@ -717,8 +720,8 @@ kiblnd_setup_rd_kiov(lnet_ni_t *ni, struct kib_tx *tx, struct kib_rdma_desc *rd,
 	LASSERT(nkiov > 0);
 	LASSERT(net);
 
-	while (offset >= kiov->kiov_len) {
-		offset -= kiov->kiov_len;
+	while (offset >= kiov->bv_len) {
+		offset -= kiov->bv_len;
 		nkiov--;
 		kiov++;
 		LASSERT(nkiov > 0);
@@ -728,10 +731,10 @@ kiblnd_setup_rd_kiov(lnet_ni_t *ni, struct kib_tx *tx, struct kib_rdma_desc *rd,
 	do {
 		LASSERT(nkiov > 0);
 
-		fragnob = min((int)(kiov->kiov_len - offset), nob);
+		fragnob = min((int)(kiov->bv_len - offset), nob);
 
-		sg_set_page(sg, kiov->kiov_page, fragnob,
-			    kiov->kiov_offset + offset);
+		sg_set_page(sg, kiov->bv_page, fragnob,
+			    kiov->bv_offset + offset);
 		sg = sg_next(sg);
 		if (!sg) {
 			CERROR("lacking enough sg entries to map tx\n");
@@ -761,7 +764,6 @@ kiblnd_post_tx_locked(struct kib_conn *conn, struct kib_tx *tx, int credit)
 	LASSERT(tx->tx_queued);
 	/* We rely on this for QP sizing */
 	LASSERT(tx->tx_nwrq > 0);
-	LASSERT(tx->tx_nwrq <= 1 + conn->ibc_max_frags);
 
 	LASSERT(!credit || credit == 1);
 	LASSERT(conn->ibc_outstanding_credits >= 0);
@@ -800,7 +802,7 @@ kiblnd_post_tx_locked(struct kib_conn *conn, struct kib_tx *tx, int credit)
 	      conn->ibc_noops_posted == IBLND_OOB_MSGS(ver)))) {
 		/*
 		 * OK to drop when posted enough NOOPs, since
-		 * kiblnd_check_sends will queue NOOP again when
+		 * kiblnd_check_sends_locked will queue NOOP again when
 		 * posted NOOPs complete
 		 */
 		spin_unlock(&conn->ibc_lock);
@@ -905,7 +907,7 @@ kiblnd_post_tx_locked(struct kib_conn *conn, struct kib_tx *tx, int credit)
 }
 
 static void
-kiblnd_check_sends(struct kib_conn *conn)
+kiblnd_check_sends_locked(struct kib_conn *conn)
 {
 	int ver = conn->ibc_version;
 	lnet_ni_t *ni = conn->ibc_peer->ibp_ni;
@@ -918,8 +920,6 @@ kiblnd_check_sends(struct kib_conn *conn)
 		return;
 	}
 
-	spin_lock(&conn->ibc_lock);
-
 	LASSERT(conn->ibc_nsends_posted <= kiblnd_concurrent_sends(ver, ni));
 	LASSERT(!IBLND_OOB_CAPABLE(ver) ||
 		conn->ibc_noops_posted <= IBLND_OOB_MSGS(ver));
@@ -969,8 +969,6 @@ kiblnd_check_sends(struct kib_conn *conn)
 		if (kiblnd_post_tx_locked(conn, tx, credit))
 			break;
 	}
-
-	spin_unlock(&conn->ibc_lock);
 }
 
 static void
@@ -1016,16 +1014,11 @@ kiblnd_tx_complete(struct kib_tx *tx, int status)
 	if (idle)
 		list_del(&tx->tx_list);
 
-	kiblnd_conn_addref(conn);	       /* 1 ref for me.... */
-
+	kiblnd_check_sends_locked(conn);
 	spin_unlock(&conn->ibc_lock);
 
 	if (idle)
 		kiblnd_tx_done(conn->ibc_peer->ibp_ni, tx);
-
-	kiblnd_check_sends(conn);
-
-	kiblnd_conn_decref(conn);	       /* ...until here */
 }
 
 static void
@@ -1078,6 +1071,15 @@ kiblnd_init_rdma(struct kib_conn *conn, struct kib_tx *tx, int type,
 	LASSERT(type == IBLND_MSG_GET_DONE ||
 		type == IBLND_MSG_PUT_DONE);
 
+	if (kiblnd_rd_size(srcrd) > conn->ibc_max_frags << PAGE_SHIFT) {
+		CERROR("RDMA is too large for peer %s (%d), src size: %d dst size: %d\n",
+		       libcfs_nid2str(conn->ibc_peer->ibp_nid),
+		       conn->ibc_max_frags << PAGE_SHIFT,
+		       kiblnd_rd_size(srcrd), kiblnd_rd_size(dstrd));
+		rc = -EMSGSIZE;
+		goto too_big;
+	}
+
 	while (resid > 0) {
 		if (srcidx >= srcrd->rd_nfrags) {
 			CERROR("Src buffer exhausted: %d frags\n", srcidx);
@@ -1091,10 +1093,10 @@ kiblnd_init_rdma(struct kib_conn *conn, struct kib_tx *tx, int type,
 			break;
 		}
 
-		if (tx->tx_nwrq >= conn->ibc_max_frags) {
+		if (tx->tx_nwrq >= IBLND_MAX_RDMA_FRAGS) {
 			CERROR("RDMA has too many fragments for peer %s (%d), src idx/frags: %d/%d dst idx/frags: %d/%d\n",
 			       libcfs_nid2str(conn->ibc_peer->ibp_nid),
-			       conn->ibc_max_frags,
+			       IBLND_MAX_RDMA_FRAGS,
 			       srcidx, srcrd->rd_nfrags,
 			       dstidx, dstrd->rd_nfrags);
 			rc = -EMSGSIZE;
@@ -1132,7 +1134,7 @@ kiblnd_init_rdma(struct kib_conn *conn, struct kib_tx *tx, int type,
 		wrq++;
 		sge++;
 	}
-
+too_big:
 	if (rc < 0)			     /* no RDMA if completing with failure */
 		tx->tx_nwrq = 0;
 
@@ -1204,9 +1206,8 @@ kiblnd_queue_tx(struct kib_tx *tx, struct kib_conn *conn)
 {
 	spin_lock(&conn->ibc_lock);
 	kiblnd_queue_tx_locked(tx, conn);
+	kiblnd_check_sends_locked(conn);
 	spin_unlock(&conn->ibc_lock);
-
-	kiblnd_check_sends(conn);
 }
 
 static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
@@ -1499,6 +1500,7 @@ kiblnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg)
 	lnet_kiov_t *payload_kiov = lntmsg->msg_kiov;
 	unsigned int payload_offset = lntmsg->msg_offset;
 	unsigned int payload_nob = lntmsg->msg_len;
+	struct iov_iter from;
 	struct kib_msg *ibmsg;
 	struct kib_rdma_desc  *rd;
 	struct kib_tx *tx;
@@ -1518,6 +1520,17 @@ kiblnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg)
 	/* payload is either all vaddrs or all pages */
 	LASSERT(!(payload_kiov && payload_iov));
 
+	if (payload_kiov)
+		iov_iter_bvec(&from, ITER_BVEC | WRITE,
+			      payload_kiov, payload_niov,
+			      payload_nob + payload_offset);
+	else
+		iov_iter_kvec(&from, ITER_KVEC | WRITE,
+			      payload_iov, payload_niov,
+			      payload_nob + payload_offset);
+
+	iov_iter_advance(&from, payload_offset);
+
 	switch (type) {
 	default:
 		LBUG();
@@ -1637,17 +1650,8 @@ kiblnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg)
 	ibmsg = tx->tx_msg;
 	ibmsg->ibm_u.immediate.ibim_hdr = *hdr;
 
-	if (payload_kiov)
-		lnet_copy_kiov2flat(IBLND_MSG_SIZE, ibmsg,
-				    offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),
-				    payload_niov, payload_kiov,
-				    payload_offset, payload_nob);
-	else
-		lnet_copy_iov2flat(IBLND_MSG_SIZE, ibmsg,
-				   offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),
-				   payload_niov, payload_iov,
-				   payload_offset, payload_nob);
-
+	copy_from_iter(&ibmsg->ibm_u.immediate.ibim_payload, IBLND_MSG_SIZE,
+		       &from);
 	nob = offsetof(struct kib_immediate_msg, ibim_payload[payload_nob]);
 	kiblnd_init_tx_msg(ni, tx, IBLND_MSG_IMMEDIATE, nob);
 
@@ -1719,8 +1723,7 @@ kiblnd_reply(lnet_ni_t *ni, struct kib_rx *rx, lnet_msg_t *lntmsg)
 
 int
 kiblnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, int delayed,
-	    unsigned int niov, struct kvec *iov, lnet_kiov_t *kiov,
-	    unsigned int offset, unsigned int mlen, unsigned int rlen)
+	    struct iov_iter *to, unsigned int rlen)
 {
 	struct kib_rx *rx = private;
 	struct kib_msg *rxmsg = rx->rx_msg;
@@ -1730,10 +1733,9 @@ kiblnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, int delayed,
 	int post_credit = IBLND_POSTRX_PEER_CREDIT;
 	int rc = 0;
 
-	LASSERT(mlen <= rlen);
+	LASSERT(iov_iter_count(to) <= rlen);
 	LASSERT(!in_interrupt());
 	/* Either all pages or all vaddrs */
-	LASSERT(!(kiov && iov));
 
 	switch (rxmsg->ibm_type) {
 	default:
@@ -1749,16 +1751,8 @@ kiblnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, int delayed,
 			break;
 		}
 
-		if (kiov)
-			lnet_copy_flat2kiov(niov, kiov, offset,
-					    IBLND_MSG_SIZE, rxmsg,
-					    offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),
-					    mlen);
-		else
-			lnet_copy_flat2iov(niov, iov, offset,
-					   IBLND_MSG_SIZE, rxmsg,
-					   offsetof(struct kib_msg, ibm_u.immediate.ibim_payload),
-					   mlen);
+		copy_to_iter(&rxmsg->ibm_u.immediate.ibim_payload,
+			     IBLND_MSG_SIZE, to);
 		lnet_finalize(ni, lntmsg, 0);
 		break;
 
@@ -1766,7 +1760,7 @@ kiblnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, int delayed,
 		struct kib_msg	*txmsg;
 		struct kib_rdma_desc *rd;
 
-		if (!mlen) {
+		if (!iov_iter_count(to)) {
 			lnet_finalize(ni, lntmsg, 0);
 			kiblnd_send_completion(rx->rx_conn, IBLND_MSG_PUT_NAK, 0,
 					       rxmsg->ibm_u.putreq.ibprm_cookie);
@@ -1784,12 +1778,16 @@ kiblnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, int delayed,
 
 		txmsg = tx->tx_msg;
 		rd = &txmsg->ibm_u.putack.ibpam_rd;
-		if (!kiov)
+		if (!(to->type & ITER_BVEC))
 			rc = kiblnd_setup_rd_iov(ni, tx, rd,
-						 niov, iov, offset, mlen);
+						 to->nr_segs, to->kvec,
+						 to->iov_offset,
+						 iov_iter_count(to));
 		else
 			rc = kiblnd_setup_rd_kiov(ni, tx, rd,
-						  niov, kiov, offset, mlen);
+						  to->nr_segs, to->bvec,
+						  to->iov_offset,
+						  iov_iter_count(to));
 		if (rc) {
 			CERROR("Can't setup PUT sink for %s: %d\n",
 			       libcfs_nid2str(conn->ibc_peer->ibp_nid), rc);
@@ -2183,14 +2181,11 @@ kiblnd_connreq_done(struct kib_conn *conn, int status)
 		return;
 	}
 
-	/**
-	 * refcount taken by cmid is not reliable after I released the glock
-	 * because this connection is visible to other threads now, another
-	 * thread can find and close this connection right after I released
-	 * the glock, if kiblnd_cm_callback for RDMA_CM_EVENT_DISCONNECTED is
-	 * called, it can release the connection refcount taken by cmid.
-	 * It means the connection could be destroyed before I finish my
-	 * operations on it.
+	/*
+	 * +1 ref for myself, this connection is visible to other threads
+	 * now, refcount of peer:ibp_conns can be released by connection
+	 * close from either a different thread, or the calling of
+	 * kiblnd_check_sends_locked() below. See bz21911 for details.
 	 */
 	kiblnd_conn_addref(conn);
 	write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
@@ -2202,10 +2197,9 @@ kiblnd_connreq_done(struct kib_conn *conn, int status)
 
 		kiblnd_queue_tx_locked(tx, conn);
 	}
+	kiblnd_check_sends_locked(conn);
 	spin_unlock(&conn->ibc_lock);
 
-	kiblnd_check_sends(conn);
-
 	/* schedule blocked rxs */
 	kiblnd_handle_early_rxs(conn);
 
@@ -2240,6 +2234,7 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
 	struct kib_rej rej;
 	int version = IBLND_MSG_VERSION;
 	unsigned long flags;
+	int max_frags;
 	int rc;
 	struct sockaddr_in *peer_addr;
 
@@ -2346,22 +2341,20 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
 		goto failed;
 	}
 
-	if (reqmsg->ibm_u.connparams.ibcp_max_frags >
-	    kiblnd_rdma_frags(version, ni)) {
-		CWARN("Can't accept conn from %s (version %x): max_frags %d too large (%d wanted)\n",
-		      libcfs_nid2str(nid), version,
-		      reqmsg->ibm_u.connparams.ibcp_max_frags,
+	max_frags = reqmsg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT;
+	if (max_frags > kiblnd_rdma_frags(version, ni)) {
+		CWARN("Can't accept conn from %s (version %x): max message size %d is too large (%d wanted)\n",
+		      libcfs_nid2str(nid), version, max_frags,
 		      kiblnd_rdma_frags(version, ni));
 
 		if (version >= IBLND_MSG_VERSION)
 			rej.ibr_why = IBLND_REJECT_RDMA_FRAGS;
 
 		goto failed;
-	} else if (reqmsg->ibm_u.connparams.ibcp_max_frags <
-		   kiblnd_rdma_frags(version, ni) && !net->ibn_fmr_ps) {
-		CWARN("Can't accept conn from %s (version %x): max_frags %d incompatible without FMR pool (%d wanted)\n",
-		      libcfs_nid2str(nid), version,
-		      reqmsg->ibm_u.connparams.ibcp_max_frags,
+	} else if (max_frags < kiblnd_rdma_frags(version, ni) &&
+		   !net->ibn_fmr_ps) {
+		CWARN("Can't accept conn from %s (version %x): max message size %d incompatible without FMR pool (%d wanted)\n",
+		      libcfs_nid2str(nid), version, max_frags,
 		      kiblnd_rdma_frags(version, ni));
 
 		if (version == IBLND_MSG_VERSION)
@@ -2387,7 +2380,7 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
 	}
 
 	/* We have validated the peer's parameters so use those */
-	peer->ibp_max_frags = reqmsg->ibm_u.connparams.ibcp_max_frags;
+	peer->ibp_max_frags = max_frags;
 	peer->ibp_queue_depth = reqmsg->ibm_u.connparams.ibcp_queue_depth;
 
 	write_lock_irqsave(g_lock, flags);
@@ -2419,23 +2412,37 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
 			goto failed;
 		}
 
-		/* tie-break connection race in favour of the higher NID */
+		/*
+		 * Tie-break connection race in favour of the higher NID.
+		 * If we keep running into a race condition multiple times,
+		 * we have to assume that the connection attempt with the
+		 * higher NID is stuck in a connecting state and will never
+		 * recover.  As such, we pass through this if-block and let
+		 * the lower NID connection win so we can move forward.
+		 */
 		if (peer2->ibp_connecting &&
-		    nid < ni->ni_nid) {
+		    nid < ni->ni_nid && peer2->ibp_races <
+		    MAX_CONN_RACES_BEFORE_ABORT) {
+			peer2->ibp_races++;
 			write_unlock_irqrestore(g_lock, flags);
 
-			CWARN("Conn race %s\n", libcfs_nid2str(peer2->ibp_nid));
+			CDEBUG(D_NET, "Conn race %s\n",
+			       libcfs_nid2str(peer2->ibp_nid));
 
 			kiblnd_peer_decref(peer);
 			rej.ibr_why = IBLND_REJECT_CONN_RACE;
 			goto failed;
 		}
-
+		if (peer2->ibp_races >= MAX_CONN_RACES_BEFORE_ABORT)
+			CNETERR("Conn race %s: unresolved after %d attempts, letting lower NID win\n",
+				libcfs_nid2str(peer2->ibp_nid),
+				MAX_CONN_RACES_BEFORE_ABORT);
 		/**
 		 * passive connection is allowed even this peer is waiting for
 		 * reconnection.
 		 */
 		peer2->ibp_reconnecting = 0;
+		peer2->ibp_races = 0;
 		peer2->ibp_accepting++;
 		kiblnd_peer_addref(peer2);
 
@@ -2494,7 +2501,7 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
 	kiblnd_init_msg(ackmsg, IBLND_MSG_CONNACK,
 			sizeof(ackmsg->ibm_u.connparams));
 	ackmsg->ibm_u.connparams.ibcp_queue_depth = conn->ibc_queue_depth;
-	ackmsg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags;
+	ackmsg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags << IBLND_FRAG_SHIFT;
 	ackmsg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE;
 
 	kiblnd_pack_msg(ni, ackmsg, version, 0, nid, reqmsg->ibm_srcstamp);
@@ -2526,9 +2533,9 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
 
  failed:
 	if (ni) {
-		lnet_ni_decref(ni);
 		rej.ibr_cp.ibcp_queue_depth = kiblnd_msg_queue_size(version, ni);
 		rej.ibr_cp.ibcp_max_frags = kiblnd_rdma_frags(version, ni);
+		lnet_ni_decref(ni);
 	}
 
 	rej.ibr_version             = version;
@@ -2556,7 +2563,7 @@ kiblnd_check_reconnect(struct kib_conn *conn, int version,
 
 	if (cp) {
 		msg_size = cp->ibcp_max_msg_size;
-		frag_num = cp->ibcp_max_frags;
+		frag_num	= cp->ibcp_max_frags << IBLND_FRAG_SHIFT;
 		queue_dep = cp->ibcp_queue_depth;
 	}
 
@@ -2821,11 +2828,11 @@ kiblnd_check_connreply(struct kib_conn *conn, void *priv, int priv_nob)
 		goto failed;
 	}
 
-	if (msg->ibm_u.connparams.ibcp_max_frags >
+	if ((msg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT) >
 	    conn->ibc_max_frags) {
 		CERROR("%s has incompatible max_frags %d (<=%d wanted)\n",
 		       libcfs_nid2str(peer->ibp_nid),
-		       msg->ibm_u.connparams.ibcp_max_frags,
+		       msg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT,
 		       conn->ibc_max_frags);
 		rc = -EPROTO;
 		goto failed;
@@ -2859,7 +2866,7 @@ kiblnd_check_connreply(struct kib_conn *conn, void *priv, int priv_nob)
 	conn->ibc_credits = msg->ibm_u.connparams.ibcp_queue_depth;
 	conn->ibc_reserved_credits = msg->ibm_u.connparams.ibcp_queue_depth;
 	conn->ibc_queue_depth = msg->ibm_u.connparams.ibcp_queue_depth;
-	conn->ibc_max_frags = msg->ibm_u.connparams.ibcp_max_frags;
+	conn->ibc_max_frags = msg->ibm_u.connparams.ibcp_max_frags >> IBLND_FRAG_SHIFT;
 	LASSERT(conn->ibc_credits + conn->ibc_reserved_credits +
 		IBLND_OOB_MSGS(ver) <= IBLND_RX_MSGS(conn));
 
@@ -2916,7 +2923,7 @@ kiblnd_active_connect(struct rdma_cm_id *cmid)
 	memset(msg, 0, sizeof(*msg));
 	kiblnd_init_msg(msg, IBLND_MSG_CONNREQ, sizeof(msg->ibm_u.connparams));
 	msg->ibm_u.connparams.ibcp_queue_depth = conn->ibc_queue_depth;
-	msg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags;
+	msg->ibm_u.connparams.ibcp_max_frags = conn->ibc_max_frags << IBLND_FRAG_SHIFT;
 	msg->ibm_u.connparams.ibcp_max_msg_size = IBLND_MSG_SIZE;
 
 	kiblnd_pack_msg(peer->ibp_ni, msg, version,
@@ -3233,7 +3240,11 @@ kiblnd_check_conns(int idx)
 	 */
 	list_for_each_entry_safe(conn, temp, &checksends, ibc_connd_list) {
 		list_del(&conn->ibc_connd_list);
-		kiblnd_check_sends(conn);
+
+		spin_lock(&conn->ibc_lock);
+		kiblnd_check_sends_locked(conn);
+		spin_unlock(&conn->ibc_lock);
+
 		kiblnd_conn_decref(conn);
 	}
 }
@@ -3419,6 +3430,12 @@ kiblnd_qp_event(struct ib_event *event, void *arg)
 	case IB_EVENT_COMM_EST:
 		CDEBUG(D_NET, "%s established\n",
 		       libcfs_nid2str(conn->ibc_peer->ibp_nid));
+		/*
+		 * We received a packet but connection isn't established
+		 * probably handshake packet was lost, so free to
+		 * force make connection established
+		 */
+		rdma_notify(conn->ibc_cmid, IB_EVENT_COMM_EST);
 		return;
 
 	default:
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c
index 07ec540946cd..cbc9a9c5385f 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c
@@ -1468,11 +1468,6 @@ ksocknal_close_conn_locked(struct ksock_conn *conn, int error)
 
 		conn->ksnc_route = NULL;
 
-#if 0	   /* irrelevant with only eager routes */
-		/* make route least favourite */
-		list_del(&route->ksnr_list);
-		list_add_tail(&route->ksnr_list, &peer->ksnp_routes);
-#endif
 		ksocknal_route_decref(route);     /* drop conn's ref on route */
 	}
 
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
index a56632b4ee37..e6ca0cf52691 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
@@ -86,8 +86,6 @@ struct ksock_sched {				/* per scheduler state */
 	int                     kss_nconns;     /* # connections assigned to
 						 * this scheduler */
 	struct ksock_sched_info *kss_info;	/* owner of it */
-	struct page             *kss_rx_scratch_pgs[LNET_MAX_IOV];
-	struct kvec             kss_scratch_iov[LNET_MAX_IOV];
 };
 
 struct ksock_sched_info {
@@ -616,9 +614,7 @@ void ksocknal_shutdown(lnet_ni_t *ni);
 int ksocknal_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg);
 int ksocknal_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg);
 int ksocknal_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg,
-		  int delayed, unsigned int niov,
-		  struct kvec *iov, lnet_kiov_t *kiov,
-		  unsigned int offset, unsigned int mlen, unsigned int rlen);
+		  int delayed, struct iov_iter *to, unsigned int rlen);
 int ksocknal_accept(lnet_ni_t *ni, struct socket *sock);
 
 int ksocknal_add_peer(lnet_ni_t *ni, lnet_process_id_t id, __u32 ip, int port);
@@ -635,7 +631,7 @@ int  ksocknal_close_peer_conns_locked(struct ksock_peer *peer,
 int ksocknal_close_conn_and_siblings(struct ksock_conn *conn, int why);
 int ksocknal_close_matching_conns(lnet_process_id_t id, __u32 ipaddr);
 struct ksock_conn *ksocknal_find_conn_locked(struct ksock_peer *peer,
-					struct ksock_tx *tx, int nonblk);
+					     struct ksock_tx *tx, int nonblk);
 
 int  ksocknal_launch_packet(lnet_ni_t *ni, struct ksock_tx *tx,
 			    lnet_process_id_t id);
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c
index 303576d815c6..c1c6f604e6ad 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c
@@ -35,8 +35,8 @@ ksocknal_alloc_tx(int type, int size)
 		spin_lock(&ksocknal_data.ksnd_tx_lock);
 
 		if (!list_empty(&ksocknal_data.ksnd_idle_noop_txs)) {
-			tx = list_entry(ksocknal_data.ksnd_idle_noop_txs. \
-					    next, struct ksock_tx, tx_list);
+			tx = list_entry(ksocknal_data.ksnd_idle_noop_txs.next,
+					struct ksock_tx, tx_list);
 			LASSERT(tx->tx_desc_size == size);
 			list_del(&tx->tx_list);
 		}
@@ -164,13 +164,13 @@ ksocknal_send_kiov(struct ksock_conn *conn, struct ksock_tx *tx)
 	do {
 		LASSERT(tx->tx_nkiov > 0);
 
-		if (nob < (int)kiov->kiov_len) {
-			kiov->kiov_offset += nob;
-			kiov->kiov_len -= nob;
+		if (nob < (int)kiov->bv_len) {
+			kiov->bv_offset += nob;
+			kiov->bv_len -= nob;
 			return rc;
 		}
 
-		nob -= (int)kiov->kiov_len;
+		nob -= (int)kiov->bv_len;
 		tx->tx_kiov = ++kiov;
 		tx->tx_nkiov--;
 	} while (nob);
@@ -326,13 +326,13 @@ ksocknal_recv_kiov(struct ksock_conn *conn)
 	do {
 		LASSERT(conn->ksnc_rx_nkiov > 0);
 
-		if (nob < (int)kiov->kiov_len) {
-			kiov->kiov_offset += nob;
-			kiov->kiov_len -= nob;
+		if (nob < (int)kiov->bv_len) {
+			kiov->bv_offset += nob;
+			kiov->bv_len -= nob;
 			return -EAGAIN;
 		}
 
-		nob -= kiov->kiov_len;
+		nob -= kiov->bv_len;
 		conn->ksnc_rx_kiov = ++kiov;
 		conn->ksnc_rx_nkiov--;
 	} while (nob);
@@ -1325,39 +1325,36 @@ ksocknal_process_receive(struct ksock_conn *conn)
 
 int
 ksocknal_recv(lnet_ni_t *ni, void *private, lnet_msg_t *msg, int delayed,
-	      unsigned int niov, struct kvec *iov, lnet_kiov_t *kiov,
-	      unsigned int offset, unsigned int mlen, unsigned int rlen)
+	      struct iov_iter *to, unsigned int rlen)
 {
 	struct ksock_conn *conn = private;
 	struct ksock_sched *sched = conn->ksnc_scheduler;
 
-	LASSERT(mlen <= rlen);
-	LASSERT(niov <= LNET_MAX_IOV);
+	LASSERT(iov_iter_count(to) <= rlen);
+	LASSERT(to->nr_segs <= LNET_MAX_IOV);
 
 	conn->ksnc_cookie = msg;
-	conn->ksnc_rx_nob_wanted = mlen;
+	conn->ksnc_rx_nob_wanted = iov_iter_count(to);
 	conn->ksnc_rx_nob_left = rlen;
 
-	if (!mlen || iov) {
+	if (to->type & ITER_KVEC) {
 		conn->ksnc_rx_nkiov = 0;
 		conn->ksnc_rx_kiov = NULL;
 		conn->ksnc_rx_iov = conn->ksnc_rx_iov_space.iov;
 		conn->ksnc_rx_niov =
 			lnet_extract_iov(LNET_MAX_IOV, conn->ksnc_rx_iov,
-					 niov, iov, offset, mlen);
+					 to->nr_segs, to->kvec,
+					 to->iov_offset, iov_iter_count(to));
 	} else {
 		conn->ksnc_rx_niov = 0;
 		conn->ksnc_rx_iov = NULL;
 		conn->ksnc_rx_kiov = conn->ksnc_rx_iov_space.kiov;
 		conn->ksnc_rx_nkiov =
 			lnet_extract_kiov(LNET_MAX_IOV, conn->ksnc_rx_kiov,
-					  niov, kiov, offset, mlen);
+					 to->nr_segs, to->bvec,
+					 to->iov_offset, iov_iter_count(to));
 	}
 
-	LASSERT(mlen ==
-		lnet_iov_nob(conn->ksnc_rx_niov, conn->ksnc_rx_iov) +
-		lnet_kiov_nob(conn->ksnc_rx_nkiov, conn->ksnc_rx_kiov));
-
 	LASSERT(conn->ksnc_rx_scheduled);
 
 	spin_lock_bh(&sched->kss_lock);
@@ -2008,13 +2005,6 @@ ksocknal_connect(struct ksock_route *route)
 		list_splice_init(&peer->ksnp_tx_queue, &zombies);
 	}
 
-#if 0	   /* irrelevant with only eager routes */
-	if (!route->ksnr_deleted) {
-		/* make this route least-favourite for re-selection */
-		list_del(&route->ksnr_list);
-		list_add_tail(&route->ksnr_list, &peer->ksnp_routes);
-	}
-#endif
 	write_unlock_bh(&ksocknal_data.ksnd_global_lock);
 
 	ksocknal_peer_failed(peer);
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
index 6a17757fce1e..6c95e989ca12 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
@@ -73,9 +73,9 @@ ksocknal_lib_zc_capable(struct ksock_conn *conn)
 int
 ksocknal_lib_send_iov(struct ksock_conn *conn, struct ksock_tx *tx)
 {
+	struct msghdr msg = {.msg_flags = MSG_DONTWAIT};
 	struct socket *sock = conn->ksnc_sock;
-	int nob;
-	int rc;
+	int nob, i;
 
 	if (*ksocknal_tunables.ksnd_enable_csum	&& /* checksum enabled */
 	    conn->ksnc_proto == &ksocknal_protocol_v2x && /* V2.x connection  */
@@ -83,34 +83,16 @@ ksocknal_lib_send_iov(struct ksock_conn *conn, struct ksock_tx *tx)
 	    !tx->tx_msg.ksm_csum)		     /* not checksummed  */
 		ksocknal_lib_csum_tx(tx);
 
-	/*
-	 * NB we can't trust socket ops to either consume our iovs
-	 * or leave them alone.
-	 */
-	{
-#if SOCKNAL_SINGLE_FRAG_TX
-		struct kvec scratch;
-		struct kvec *scratchiov = &scratch;
-		unsigned int niov = 1;
-#else
-		struct kvec *scratchiov = conn->ksnc_scheduler->kss_scratch_iov;
-		unsigned int niov = tx->tx_niov;
-#endif
-		struct msghdr msg = {.msg_flags = MSG_DONTWAIT};
-		int i;
+	for (nob = i = 0; i < tx->tx_niov; i++)
+		nob += tx->tx_iov[i].iov_len;
 
-		for (nob = i = 0; i < niov; i++) {
-			scratchiov[i] = tx->tx_iov[i];
-			nob += scratchiov[i].iov_len;
-		}
+	if (!list_empty(&conn->ksnc_tx_queue) ||
+	    nob < tx->tx_resid)
+		msg.msg_flags |= MSG_MORE;
 
-		if (!list_empty(&conn->ksnc_tx_queue) ||
-		    nob < tx->tx_resid)
-			msg.msg_flags |= MSG_MORE;
-
-		rc = kernel_sendmsg(sock, &msg, scratchiov, niov, nob);
-	}
-	return rc;
+	iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC,
+		      tx->tx_iov, tx->tx_niov, nob);
+	return sock_sendmsg(sock, &msg);
 }
 
 int
@@ -124,20 +106,16 @@ ksocknal_lib_send_kiov(struct ksock_conn *conn, struct ksock_tx *tx)
 	/* Not NOOP message */
 	LASSERT(tx->tx_lnetmsg);
 
-	/*
-	 * NB we can't trust socket ops to either consume our iovs
-	 * or leave them alone.
-	 */
 	if (tx->tx_msg.ksm_zc_cookies[0]) {
 		/* Zero copy is enabled */
 		struct sock *sk = sock->sk;
-		struct page *page = kiov->kiov_page;
-		int offset = kiov->kiov_offset;
-		int fragsize = kiov->kiov_len;
+		struct page *page = kiov->bv_page;
+		int offset = kiov->bv_offset;
+		int fragsize = kiov->bv_len;
 		int msgflg = MSG_DONTWAIT;
 
 		CDEBUG(D_NET, "page %p + offset %x for %d\n",
-		       page, offset, kiov->kiov_len);
+		       page, offset, kiov->bv_len);
 
 		if (!list_empty(&conn->ksnc_tx_queue) ||
 		    fragsize < tx->tx_resid)
@@ -150,34 +128,19 @@ ksocknal_lib_send_kiov(struct ksock_conn *conn, struct ksock_tx *tx)
 			rc = tcp_sendpage(sk, page, offset, fragsize, msgflg);
 		}
 	} else {
-#if SOCKNAL_SINGLE_FRAG_TX || !SOCKNAL_RISK_KMAP_DEADLOCK
-		struct kvec scratch;
-		struct kvec *scratchiov = &scratch;
-		unsigned int niov = 1;
-#else
-#ifdef CONFIG_HIGHMEM
-#warning "XXX risk of kmap deadlock on multiple frags..."
-#endif
-		struct kvec *scratchiov = conn->ksnc_scheduler->kss_scratch_iov;
-		unsigned int niov = tx->tx_nkiov;
-#endif
 		struct msghdr msg = {.msg_flags = MSG_DONTWAIT};
 		int i;
 
-		for (nob = i = 0; i < niov; i++) {
-			scratchiov[i].iov_base = kmap(kiov[i].kiov_page) +
-						 kiov[i].kiov_offset;
-			nob += scratchiov[i].iov_len = kiov[i].kiov_len;
-		}
+		for (nob = i = 0; i < tx->tx_nkiov; i++)
+			nob += kiov[i].bv_len;
 
 		if (!list_empty(&conn->ksnc_tx_queue) ||
 		    nob < tx->tx_resid)
 			msg.msg_flags |= MSG_MORE;
 
-		rc = kernel_sendmsg(sock, &msg, (struct kvec *)scratchiov, niov, nob);
-
-		for (i = 0; i < niov; i++)
-			kunmap(kiov[i].kiov_page);
+		iov_iter_bvec(&msg.msg_iter, WRITE | ITER_BVEC,
+			      kiov, tx->tx_nkiov, nob);
+		rc = sock_sendmsg(sock, &msg);
 	}
 	return rc;
 }
@@ -201,14 +164,7 @@ ksocknal_lib_eager_ack(struct ksock_conn *conn)
 int
 ksocknal_lib_recv_iov(struct ksock_conn *conn)
 {
-#if SOCKNAL_SINGLE_FRAG_RX
-	struct kvec scratch;
-	struct kvec *scratchiov = &scratch;
-	unsigned int niov = 1;
-#else
-	struct kvec *scratchiov = conn->ksnc_scheduler->kss_scratch_iov;
 	unsigned int niov = conn->ksnc_rx_niov;
-#endif
 	struct kvec *iov = conn->ksnc_rx_iov;
 	struct msghdr msg = {
 		.msg_flags = 0
@@ -220,20 +176,15 @@ ksocknal_lib_recv_iov(struct ksock_conn *conn)
 	int sum;
 	__u32 saved_csum;
 
-	/*
-	 * NB we can't trust socket ops to either consume our iovs
-	 * or leave them alone.
-	 */
 	LASSERT(niov > 0);
 
-	for (nob = i = 0; i < niov; i++) {
-		scratchiov[i] = iov[i];
-		nob += scratchiov[i].iov_len;
-	}
+	for (nob = i = 0; i < niov; i++)
+		nob += iov[i].iov_len;
+
 	LASSERT(nob <= conn->ksnc_rx_nob_wanted);
 
-	rc = kernel_recvmsg(conn->ksnc_sock, &msg, scratchiov, niov, nob,
-			    MSG_DONTWAIT);
+	iov_iter_kvec(&msg.msg_iter, READ | ITER_KVEC, iov, niov, nob);
+	rc = sock_recvmsg(conn->ksnc_sock, &msg, MSG_DONTWAIT);
 
 	saved_csum = 0;
 	if (conn->ksnc_proto == &ksocknal_protocol_v2x) {
@@ -259,67 +210,10 @@ ksocknal_lib_recv_iov(struct ksock_conn *conn)
 	return rc;
 }
 
-static void
-ksocknal_lib_kiov_vunmap(void *addr)
-{
-	if (!addr)
-		return;
-
-	vunmap(addr);
-}
-
-static void *
-ksocknal_lib_kiov_vmap(lnet_kiov_t *kiov, int niov,
-		       struct kvec *iov, struct page **pages)
-{
-	void *addr;
-	int nob;
-	int i;
-
-	if (!*ksocknal_tunables.ksnd_zc_recv || !pages)
-		return NULL;
-
-	LASSERT(niov <= LNET_MAX_IOV);
-
-	if (niov < 2 ||
-	    niov < *ksocknal_tunables.ksnd_zc_recv_min_nfrags)
-		return NULL;
-
-	for (nob = i = 0; i < niov; i++) {
-		if ((kiov[i].kiov_offset && i > 0) ||
-		    (kiov[i].kiov_offset + kiov[i].kiov_len != PAGE_SIZE && i < niov - 1))
-			return NULL;
-
-		pages[i] = kiov[i].kiov_page;
-		nob += kiov[i].kiov_len;
-	}
-
-	addr = vmap(pages, niov, VM_MAP, PAGE_KERNEL);
-	if (!addr)
-		return NULL;
-
-	iov->iov_base = addr + kiov[0].kiov_offset;
-	iov->iov_len = nob;
-
-	return addr;
-}
-
 int
 ksocknal_lib_recv_kiov(struct ksock_conn *conn)
 {
-#if SOCKNAL_SINGLE_FRAG_RX || !SOCKNAL_RISK_KMAP_DEADLOCK
-	struct kvec scratch;
-	struct kvec *scratchiov = &scratch;
-	struct page **pages = NULL;
-	unsigned int niov = 1;
-#else
-#ifdef CONFIG_HIGHMEM
-#warning "XXX risk of kmap deadlock on multiple frags..."
-#endif
-	struct kvec *scratchiov = conn->ksnc_scheduler->kss_scratch_iov;
-	struct page **pages = conn->ksnc_scheduler->kss_rx_scratch_pgs;
 	unsigned int niov = conn->ksnc_rx_nkiov;
-#endif
 	lnet_kiov_t   *kiov = conn->ksnc_rx_kiov;
 	struct msghdr msg = {
 		.msg_flags = 0
@@ -328,63 +222,32 @@ ksocknal_lib_recv_kiov(struct ksock_conn *conn)
 	int i;
 	int rc;
 	void *base;
-	void *addr;
 	int sum;
 	int fragnob;
-	int n;
-
-	/*
-	 * NB we can't trust socket ops to either consume our iovs
-	 * or leave them alone.
-	 */
-	addr = ksocknal_lib_kiov_vmap(kiov, niov, scratchiov, pages);
-	if (addr) {
-		nob = scratchiov[0].iov_len;
-		n = 1;
 
-	} else {
-		for (nob = i = 0; i < niov; i++) {
-			nob += scratchiov[i].iov_len = kiov[i].kiov_len;
-			scratchiov[i].iov_base = kmap(kiov[i].kiov_page) +
-						 kiov[i].kiov_offset;
-		}
-		n = niov;
-	}
+	for (nob = i = 0; i < niov; i++)
+		nob += kiov[i].bv_len;
 
 	LASSERT(nob <= conn->ksnc_rx_nob_wanted);
 
-	rc = kernel_recvmsg(conn->ksnc_sock, &msg, (struct kvec *)scratchiov,
-			    n, nob, MSG_DONTWAIT);
+	iov_iter_bvec(&msg.msg_iter, READ | ITER_BVEC, kiov, niov, nob);
+	rc = sock_recvmsg(conn->ksnc_sock, &msg, MSG_DONTWAIT);
 
 	if (conn->ksnc_msg.ksm_csum) {
 		for (i = 0, sum = rc; sum > 0; i++, sum -= fragnob) {
 			LASSERT(i < niov);
 
-			/*
-			 * Dang! have to kmap again because I have nowhere to
-			 * stash the mapped address.  But by doing it while the
-			 * page is still mapped, the kernel just bumps the map
-			 * count and returns me the address it stashed.
-			 */
-			base = kmap(kiov[i].kiov_page) + kiov[i].kiov_offset;
-			fragnob = kiov[i].kiov_len;
+			base = kmap(kiov[i].bv_page) + kiov[i].bv_offset;
+			fragnob = kiov[i].bv_len;
 			if (fragnob > sum)
 				fragnob = sum;
 
 			conn->ksnc_rx_csum = ksocknal_csum(conn->ksnc_rx_csum,
 							   base, fragnob);
 
-			kunmap(kiov[i].kiov_page);
+			kunmap(kiov[i].bv_page);
 		}
 	}
-
-	if (addr) {
-		ksocknal_lib_kiov_vunmap(addr);
-	} else {
-		for (i = 0; i < niov; i++)
-			kunmap(kiov[i].kiov_page);
-	}
-
 	return rc;
 }
 
@@ -406,12 +269,12 @@ ksocknal_lib_csum_tx(struct ksock_tx *tx)
 
 	if (tx->tx_kiov) {
 		for (i = 0; i < tx->tx_nkiov; i++) {
-			base = kmap(tx->tx_kiov[i].kiov_page) +
-			       tx->tx_kiov[i].kiov_offset;
+			base = kmap(tx->tx_kiov[i].bv_page) +
+			       tx->tx_kiov[i].bv_offset;
 
-			csum = ksocknal_csum(csum, base, tx->tx_kiov[i].kiov_len);
+			csum = ksocknal_csum(csum, base, tx->tx_kiov[i].bv_len);
 
-			kunmap(tx->tx_kiov[i].kiov_page);
+			kunmap(tx->tx_kiov[i].bv_page);
 		}
 	} else {
 		for (i = 1; i < tx->tx_niov; i++)
diff --git a/drivers/staging/lustre/lnet/libcfs/debug.c b/drivers/staging/lustre/lnet/libcfs/debug.c
index 42b15a769183..23b36b890964 100644
--- a/drivers/staging/lustre/lnet/libcfs/debug.c
+++ b/drivers/staging/lustre/lnet/libcfs/debug.c
@@ -328,15 +328,20 @@ libcfs_debug_str2mask(int *mask, const char *str, int is_subsys)
  */
 void libcfs_debug_dumplog_internal(void *arg)
 {
+	static time64_t last_dump_time;
+	time64_t current_time;
 	void *journal_info;
 
 	journal_info = current->journal_info;
 	current->journal_info = NULL;
+	current_time = ktime_get_real_seconds();
 
-	if (strncmp(libcfs_debug_file_path_arr, "NONE", 4) != 0) {
+	if (strncmp(libcfs_debug_file_path_arr, "NONE", 4) &&
+	    current_time > last_dump_time) {
+		last_dump_time = current_time;
 		snprintf(debug_file_name, sizeof(debug_file_name) - 1,
 			 "%s.%lld.%ld", libcfs_debug_file_path_arr,
-			 (s64)ktime_get_real_seconds(), (long_ptr_t)arg);
+			 (s64)current_time, (long_ptr_t)arg);
 		pr_alert("LustreError: dumping log to %s\n", debug_file_name);
 		cfs_tracefile_dump_all_pages(debug_file_name);
 		libcfs_run_debug_log_upcall(debug_file_name);
diff --git a/drivers/staging/lustre/lnet/libcfs/fail.c b/drivers/staging/lustre/lnet/libcfs/fail.c
index 9288ee08d1f7..e4b1a0a86eae 100644
--- a/drivers/staging/lustre/lnet/libcfs/fail.c
+++ b/drivers/staging/lustre/lnet/libcfs/fail.c
@@ -90,8 +90,10 @@ int __cfs_fail_check_set(__u32 id, __u32 value, int set)
 		}
 	}
 
-	if ((set == CFS_FAIL_LOC_ORSET || set == CFS_FAIL_LOC_RESET) &&
-	    (value & CFS_FAIL_ONCE))
+	/* Take into account the current call for FAIL_ONCE for ORSET only,
+	 * as RESET is a new fail_loc, it does not change the current call
+	 */
+	if ((set == CFS_FAIL_LOC_ORSET) && (value & CFS_FAIL_ONCE))
 		set_bit(CFS_FAIL_ONCE_BIT, &cfs_fail_loc);
 	/* Lost race to set CFS_FAILED_BIT. */
 	if (test_and_set_bit(CFS_FAILED_BIT, &cfs_fail_loc)) {
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_string.c b/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
index fc697cdfcdaf..56a614d7713b 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
@@ -229,8 +229,6 @@ cfs_str2num_check(char *str, int nob, unsigned *num,
 	char *endp, cache;
 	int rc;
 
-	str = cfs_trimwhite(str);
-
 	/**
 	 * kstrouint can only handle strings composed
 	 * of only numbers. We need to scan the string
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
index b52518c54efe..e8b1a61420de 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
@@ -74,6 +74,17 @@ struct cfs_cpt_data {
 
 static struct cfs_cpt_data	cpt_data;
 
+static void
+cfs_node_to_cpumask(int node, cpumask_t *mask)
+{
+	const cpumask_t *tmp = cpumask_of_node(node);
+
+	if (tmp)
+		cpumask_copy(mask, tmp);
+	else
+		cpumask_clear(mask);
+}
+
 void
 cfs_cpt_table_free(struct cfs_cpt_table *cptab)
 {
@@ -403,7 +414,7 @@ cfs_cpt_set_node(struct cfs_cpt_table *cptab, int cpt, int node)
 	mutex_lock(&cpt_data.cpt_mutex);
 
 	mask = cpt_data.cpt_cpumask;
-	cpumask_copy(mask, cpumask_of_node(node));
+	cfs_node_to_cpumask(node, mask);
 
 	rc = cfs_cpt_set_cpumask(cptab, cpt, mask);
 
@@ -427,7 +438,7 @@ cfs_cpt_unset_node(struct cfs_cpt_table *cptab, int cpt, int node)
 	mutex_lock(&cpt_data.cpt_mutex);
 
 	mask = cpt_data.cpt_cpumask;
-	cpumask_copy(mask, cpumask_of_node(node));
+	cfs_node_to_cpumask(node, mask);
 
 	cfs_cpt_unset_cpumask(cptab, cpt, mask);
 
@@ -749,7 +760,7 @@ cfs_cpt_table_create(int ncpt)
 	}
 
 	for_each_online_node(i) {
-		cpumask_copy(mask, cpumask_of_node(i));
+		cfs_node_to_cpumask(i, mask);
 
 		while (!cpumask_empty(mask)) {
 			struct cfs_cpu_partition *part;
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
index 5c0116ade909..7f56d2c9dd00 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
@@ -95,8 +95,8 @@ static int cfs_crypto_hash_alloc(enum cfs_crypto_hash_alg hash_alg,
 		err = crypto_ahash_setkey(tfm, key, key_len);
 	else if ((*type)->cht_key != 0)
 		err = crypto_ahash_setkey(tfm,
-					 (unsigned char *)&((*type)->cht_key),
-					 (*type)->cht_size);
+					  (unsigned char *)&((*type)->cht_key),
+					  (*type)->cht_size);
 
 	if (err != 0) {
 		ahash_request_free(*req);
diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index 346db892f275..4daf828198c3 100644
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -1286,6 +1286,25 @@ lnet_startup_lndni(struct lnet_ni *ni, struct lnet_ioctl_config_data *conf)
 		       sizeof(*ni->ni_lnd_tunables));
 	}
 
+	/*
+	 * If given some LND tunable parameters, parse those now to
+	 * override the values in the NI structure.
+	 */
+	if (conf) {
+		if (conf->cfg_config_u.cfg_net.net_peer_rtr_credits >= 0)
+			ni->ni_peerrtrcredits =
+				conf->cfg_config_u.cfg_net.net_peer_rtr_credits;
+		if (conf->cfg_config_u.cfg_net.net_peer_timeout >= 0)
+			ni->ni_peertimeout =
+				conf->cfg_config_u.cfg_net.net_peer_timeout;
+		if (conf->cfg_config_u.cfg_net.net_peer_tx_credits != -1)
+			ni->ni_peertxcredits =
+				conf->cfg_config_u.cfg_net.net_peer_tx_credits;
+		if (conf->cfg_config_u.cfg_net.net_max_tx_credits >= 0)
+			ni->ni_maxtxcredits =
+				conf->cfg_config_u.cfg_net.net_max_tx_credits;
+	}
+
 	rc = lnd->lnd_startup(ni);
 
 	mutex_unlock(&the_lnet.ln_lnd_mutex);
@@ -1299,33 +1318,6 @@ lnet_startup_lndni(struct lnet_ni *ni, struct lnet_ioctl_config_data *conf)
 		goto failed0;
 	}
 
-	/*
-	 * If given some LND tunable parameters, parse those now to
-	 * override the values in the NI structure.
-	 */
-	if (conf && conf->cfg_config_u.cfg_net.net_peer_rtr_credits >= 0) {
-		ni->ni_peerrtrcredits =
-			conf->cfg_config_u.cfg_net.net_peer_rtr_credits;
-	}
-	if (conf && conf->cfg_config_u.cfg_net.net_peer_timeout >= 0) {
-		ni->ni_peertimeout =
-			conf->cfg_config_u.cfg_net.net_peer_timeout;
-	}
-	/*
-	 * TODO
-	 * Note: For now, don't allow the user to change
-	 * peertxcredits as this number is used in the
-	 * IB LND to control queue depth.
-	 *
-	 * if (conf && conf->cfg_config_u.cfg_net.net_peer_tx_credits != -1)
-	 *	ni->ni_peertxcredits =
-	 *		conf->cfg_config_u.cfg_net.net_peer_tx_credits;
-	 */
-	if (conf && conf->cfg_config_u.cfg_net.net_max_tx_credits >= 0) {
-		ni->ni_maxtxcredits =
-			conf->cfg_config_u.cfg_net.net_max_tx_credits;
-	}
-
 	LASSERT(ni->ni_peertimeout <= 0 || lnd->lnd_query);
 
 	lnet_net_lock(LNET_LOCK_EX);
diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index a72afdf68bb2..9e2183ff847e 100644
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -31,6 +31,8 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LNET
+#include <linux/nsproxy.h>
+#include <net/net_namespace.h>
 #include "../../include/linux/lnet/lib-lnet.h"
 
 struct lnet_text_buf {	    /* tmp struct for parsing routes */
@@ -110,6 +112,11 @@ lnet_ni_free(struct lnet_ni *ni)
 		LIBCFS_FREE(ni->ni_interfaces[i],
 			    strlen(ni->ni_interfaces[i]) + 1);
 	}
+
+	/* release reference to net namespace */
+	if (ni->ni_net_ns)
+		put_net(ni->ni_net_ns);
+
 	LIBCFS_FREE(ni, sizeof(*ni));
 }
 
@@ -171,6 +178,13 @@ lnet_ni_alloc(__u32 net, struct cfs_expr_list *el, struct list_head *nilist)
 
 	/* LND will fill in the address part of the NID */
 	ni->ni_nid = LNET_MKNID(net, 0);
+
+	/* Store net namespace in which current ni is being created */
+	if (current->nsproxy->net_ns)
+		ni->ni_net_ns = get_net(current->nsproxy->net_ns);
+	else
+		ni->ni_net_ns = NULL;
+
 	ni->ni_last_alive = ktime_get_real_seconds();
 	list_add_tail(&ni->ni_list, nilist);
 	return ni;
diff --git a/drivers/staging/lustre/lnet/lnet/lib-md.c b/drivers/staging/lustre/lnet/lnet/lib-md.c
index 1834bf7a27ef..eab53cd57296 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-md.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-md.c
@@ -134,11 +134,11 @@ lnet_md_build(lnet_libmd_t *lmd, lnet_md_t *umd, int unlink)
 
 		for (i = 0; i < (int)niov; i++) {
 			/* We take the page pointer on trust */
-			if (lmd->md_iov.kiov[i].kiov_offset +
-			    lmd->md_iov.kiov[i].kiov_len > PAGE_SIZE)
+			if (lmd->md_iov.kiov[i].bv_offset +
+			    lmd->md_iov.kiov[i].bv_len > PAGE_SIZE)
 				return -EINVAL; /* invalid length */
 
-			total_length += lmd->md_iov.kiov[i].kiov_len;
+			total_length += lmd->md_iov.kiov[i].bv_len;
 		}
 
 		lmd->md_length = total_length;
@@ -292,11 +292,12 @@ LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd,
 		return -ENOMEM;
 
 	rc = lnet_md_build(md, &umd, unlink);
+	if (rc)
+		goto out_free;
+
 	cpt = lnet_cpt_of_cookie(meh.cookie);
 
 	lnet_res_lock(cpt);
-	if (rc)
-		goto failed;
 
 	me = lnet_handle2me(&meh);
 	if (!me)
@@ -307,7 +308,7 @@ LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd,
 		rc = lnet_md_link(md, umd.eq_handle, cpt);
 
 	if (rc)
-		goto failed;
+		goto out_unlock;
 
 	/*
 	 * attach this MD to portal of ME and check if it matches any
@@ -324,10 +325,10 @@ LNetMDAttach(lnet_handle_me_t meh, lnet_md_t umd,
 
 	return 0;
 
- failed:
-	lnet_md_free(md);
-
+out_unlock:
 	lnet_res_unlock(cpt);
+out_free:
+	lnet_md_free(md);
 	return rc;
 }
 EXPORT_SYMBOL(LNetMDAttach);
@@ -370,24 +371,25 @@ LNetMDBind(lnet_md_t umd, lnet_unlink_t unlink, lnet_handle_md_t *handle)
 		return -ENOMEM;
 
 	rc = lnet_md_build(md, &umd, unlink);
+	if (rc)
+		goto out_free;
 
 	cpt = lnet_res_lock_current();
-	if (rc)
-		goto failed;
 
 	rc = lnet_md_link(md, umd.eq_handle, cpt);
 	if (rc)
-		goto failed;
+		goto out_unlock;
 
 	lnet_md2handle(handle, md);
 
 	lnet_res_unlock(cpt);
 	return 0;
 
- failed:
+out_unlock:
+	lnet_res_unlock(cpt);
+out_free:
 	lnet_md_free(md);
 
-	lnet_res_unlock(cpt);
 	return rc;
 }
 EXPORT_SYMBOL(LNetMDBind);
diff --git a/drivers/staging/lustre/lnet/lnet/lib-move.c b/drivers/staging/lustre/lnet/lnet/lib-move.c
index e6d3b801d87d..48e6f8f2392f 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-move.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-move.c
@@ -37,6 +37,8 @@
 #define DEBUG_SUBSYSTEM S_LNET
 
 #include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/nsproxy.h>
+#include <net/net_namespace.h>
 
 static int local_nid_dist_zero = 1;
 module_param(local_nid_dist_zero, int, 0444);
@@ -166,25 +168,17 @@ lnet_iov_nob(unsigned int niov, struct kvec *iov)
 EXPORT_SYMBOL(lnet_iov_nob);
 
 void
-lnet_copy_iov2iov(unsigned int ndiov, struct kvec *diov, unsigned int doffset,
-		  unsigned int nsiov, struct kvec *siov, unsigned int soffset,
-		  unsigned int nob)
+lnet_copy_iov2iter(struct iov_iter *to,
+		   unsigned int nsiov, const struct kvec *siov,
+		   unsigned int soffset, unsigned int nob)
 {
 	/* NB diov, siov are READ-ONLY */
-	unsigned int this_nob;
+	const char *s;
+	size_t left;
 
 	if (!nob)
 		return;
 
-	/* skip complete frags before 'doffset' */
-	LASSERT(ndiov > 0);
-	while (doffset >= diov->iov_len) {
-		doffset -= diov->iov_len;
-		diov++;
-		ndiov--;
-		LASSERT(ndiov > 0);
-	}
-
 	/* skip complete frags before 'soffset' */
 	LASSERT(nsiov > 0);
 	while (soffset >= siov->iov_len) {
@@ -194,39 +188,68 @@ lnet_copy_iov2iov(unsigned int ndiov, struct kvec *diov, unsigned int doffset,
 		LASSERT(nsiov > 0);
 	}
 
+	s = (char *)siov->iov_base + soffset;
+	left = siov->iov_len - soffset;
 	do {
-		LASSERT(ndiov > 0);
+		size_t n, copy = left;
 		LASSERT(nsiov > 0);
-		this_nob = min(diov->iov_len - doffset,
-			       siov->iov_len - soffset);
-		this_nob = min(this_nob, nob);
 
-		memcpy((char *)diov->iov_base + doffset,
-		       (char *)siov->iov_base + soffset, this_nob);
-		nob -= this_nob;
+		if (copy > nob)
+			copy = nob;
+		n = copy_to_iter(s, copy, to);
+		if (n != copy)
+			return;
+		nob -= n;
 
-		if (diov->iov_len > doffset + this_nob) {
-			doffset += this_nob;
-		} else {
-			diov++;
-			ndiov--;
-			doffset = 0;
-		}
+		siov++;
+		s = (char *)siov->iov_base;
+		left = siov->iov_len;
+		nsiov--;
+	} while (nob > 0);
+}
+EXPORT_SYMBOL(lnet_copy_iov2iter);
 
-		if (siov->iov_len > soffset + this_nob) {
-			soffset += this_nob;
-		} else {
-			siov++;
-			nsiov--;
-			soffset = 0;
-		}
+void
+lnet_copy_kiov2iter(struct iov_iter *to,
+		    unsigned int nsiov, const lnet_kiov_t *siov,
+		    unsigned int soffset, unsigned int nob)
+{
+	if (!nob)
+		return;
+
+	LASSERT(!in_interrupt());
+
+	LASSERT(nsiov > 0);
+	while (soffset >= siov->bv_len) {
+		soffset -= siov->bv_len;
+		siov++;
+		nsiov--;
+		LASSERT(nsiov > 0);
+	}
+
+	do {
+		size_t copy = siov->bv_len - soffset, n;
+
+		LASSERT(nsiov > 0);
+
+		if (copy > nob)
+			copy = nob;
+		n = copy_page_to_iter(siov->bv_page,
+				      siov->bv_offset + soffset,
+				      copy, to);
+		if (n != copy)
+			return;
+		nob -= n;
+		siov++;
+		nsiov--;
+		soffset = 0;
 	} while (nob > 0);
 }
-EXPORT_SYMBOL(lnet_copy_iov2iov);
+EXPORT_SYMBOL(lnet_copy_kiov2iter);
 
 int
 lnet_extract_iov(int dst_niov, struct kvec *dst,
-		 int src_niov, struct kvec *src,
+		 int src_niov, const struct kvec *src,
 		 unsigned int offset, unsigned int len)
 {
 	/*
@@ -280,238 +303,15 @@ lnet_kiov_nob(unsigned int niov, lnet_kiov_t *kiov)
 
 	LASSERT(!niov || kiov);
 	while (niov-- > 0)
-		nob += (kiov++)->kiov_len;
+		nob += (kiov++)->bv_len;
 
 	return nob;
 }
 EXPORT_SYMBOL(lnet_kiov_nob);
 
-void
-lnet_copy_kiov2kiov(unsigned int ndiov, lnet_kiov_t *diov, unsigned int doffset,
-		    unsigned int nsiov, lnet_kiov_t *siov, unsigned int soffset,
-		    unsigned int nob)
-{
-	/* NB diov, siov are READ-ONLY */
-	unsigned int this_nob;
-	char *daddr = NULL;
-	char *saddr = NULL;
-
-	if (!nob)
-		return;
-
-	LASSERT(!in_interrupt());
-
-	LASSERT(ndiov > 0);
-	while (doffset >= diov->kiov_len) {
-		doffset -= diov->kiov_len;
-		diov++;
-		ndiov--;
-		LASSERT(ndiov > 0);
-	}
-
-	LASSERT(nsiov > 0);
-	while (soffset >= siov->kiov_len) {
-		soffset -= siov->kiov_len;
-		siov++;
-		nsiov--;
-		LASSERT(nsiov > 0);
-	}
-
-	do {
-		LASSERT(ndiov > 0);
-		LASSERT(nsiov > 0);
-		this_nob = min(diov->kiov_len - doffset,
-			       siov->kiov_len - soffset);
-		this_nob = min(this_nob, nob);
-
-		if (!daddr)
-			daddr = ((char *)kmap(diov->kiov_page)) +
-				diov->kiov_offset + doffset;
-		if (!saddr)
-			saddr = ((char *)kmap(siov->kiov_page)) +
-				siov->kiov_offset + soffset;
-
-		/*
-		 * Vanishing risk of kmap deadlock when mapping 2 pages.
-		 * However in practice at least one of the kiovs will be mapped
-		 * kernel pages and the map/unmap will be NOOPs
-		 */
-		memcpy(daddr, saddr, this_nob);
-		nob -= this_nob;
-
-		if (diov->kiov_len > doffset + this_nob) {
-			daddr += this_nob;
-			doffset += this_nob;
-		} else {
-			kunmap(diov->kiov_page);
-			daddr = NULL;
-			diov++;
-			ndiov--;
-			doffset = 0;
-		}
-
-		if (siov->kiov_len > soffset + this_nob) {
-			saddr += this_nob;
-			soffset += this_nob;
-		} else {
-			kunmap(siov->kiov_page);
-			saddr = NULL;
-			siov++;
-			nsiov--;
-			soffset = 0;
-		}
-	} while (nob > 0);
-
-	if (daddr)
-		kunmap(diov->kiov_page);
-	if (saddr)
-		kunmap(siov->kiov_page);
-}
-EXPORT_SYMBOL(lnet_copy_kiov2kiov);
-
-void
-lnet_copy_kiov2iov(unsigned int niov, struct kvec *iov, unsigned int iovoffset,
-		   unsigned int nkiov, lnet_kiov_t *kiov,
-		   unsigned int kiovoffset, unsigned int nob)
-{
-	/* NB iov, kiov are READ-ONLY */
-	unsigned int this_nob;
-	char *addr = NULL;
-
-	if (!nob)
-		return;
-
-	LASSERT(!in_interrupt());
-
-	LASSERT(niov > 0);
-	while (iovoffset >= iov->iov_len) {
-		iovoffset -= iov->iov_len;
-		iov++;
-		niov--;
-		LASSERT(niov > 0);
-	}
-
-	LASSERT(nkiov > 0);
-	while (kiovoffset >= kiov->kiov_len) {
-		kiovoffset -= kiov->kiov_len;
-		kiov++;
-		nkiov--;
-		LASSERT(nkiov > 0);
-	}
-
-	do {
-		LASSERT(niov > 0);
-		LASSERT(nkiov > 0);
-		this_nob = min(iov->iov_len - iovoffset,
-			       (__kernel_size_t)kiov->kiov_len - kiovoffset);
-		this_nob = min(this_nob, nob);
-
-		if (!addr)
-			addr = ((char *)kmap(kiov->kiov_page)) +
-				kiov->kiov_offset + kiovoffset;
-
-		memcpy((char *)iov->iov_base + iovoffset, addr, this_nob);
-		nob -= this_nob;
-
-		if (iov->iov_len > iovoffset + this_nob) {
-			iovoffset += this_nob;
-		} else {
-			iov++;
-			niov--;
-			iovoffset = 0;
-		}
-
-		if (kiov->kiov_len > kiovoffset + this_nob) {
-			addr += this_nob;
-			kiovoffset += this_nob;
-		} else {
-			kunmap(kiov->kiov_page);
-			addr = NULL;
-			kiov++;
-			nkiov--;
-			kiovoffset = 0;
-		}
-
-	} while (nob > 0);
-
-	if (addr)
-		kunmap(kiov->kiov_page);
-}
-EXPORT_SYMBOL(lnet_copy_kiov2iov);
-
-void
-lnet_copy_iov2kiov(unsigned int nkiov, lnet_kiov_t *kiov,
-		   unsigned int kiovoffset, unsigned int niov,
-		   struct kvec *iov, unsigned int iovoffset,
-		   unsigned int nob)
-{
-	/* NB kiov, iov are READ-ONLY */
-	unsigned int this_nob;
-	char *addr = NULL;
-
-	if (!nob)
-		return;
-
-	LASSERT(!in_interrupt());
-
-	LASSERT(nkiov > 0);
-	while (kiovoffset >= kiov->kiov_len) {
-		kiovoffset -= kiov->kiov_len;
-		kiov++;
-		nkiov--;
-		LASSERT(nkiov > 0);
-	}
-
-	LASSERT(niov > 0);
-	while (iovoffset >= iov->iov_len) {
-		iovoffset -= iov->iov_len;
-		iov++;
-		niov--;
-		LASSERT(niov > 0);
-	}
-
-	do {
-		LASSERT(nkiov > 0);
-		LASSERT(niov > 0);
-		this_nob = min((__kernel_size_t)kiov->kiov_len - kiovoffset,
-			       iov->iov_len - iovoffset);
-		this_nob = min(this_nob, nob);
-
-		if (!addr)
-			addr = ((char *)kmap(kiov->kiov_page)) +
-				kiov->kiov_offset + kiovoffset;
-
-		memcpy(addr, (char *)iov->iov_base + iovoffset, this_nob);
-		nob -= this_nob;
-
-		if (kiov->kiov_len > kiovoffset + this_nob) {
-			addr += this_nob;
-			kiovoffset += this_nob;
-		} else {
-			kunmap(kiov->kiov_page);
-			addr = NULL;
-			kiov++;
-			nkiov--;
-			kiovoffset = 0;
-		}
-
-		if (iov->iov_len > iovoffset + this_nob) {
-			iovoffset += this_nob;
-		} else {
-			iov++;
-			niov--;
-			iovoffset = 0;
-		}
-	} while (nob > 0);
-
-	if (addr)
-		kunmap(kiov->kiov_page);
-}
-EXPORT_SYMBOL(lnet_copy_iov2kiov);
-
 int
 lnet_extract_kiov(int dst_niov, lnet_kiov_t *dst,
-		  int src_niov, lnet_kiov_t *src,
+		  int src_niov, const lnet_kiov_t *src,
 		  unsigned int offset, unsigned int len)
 {
 	/*
@@ -526,8 +326,8 @@ lnet_extract_kiov(int dst_niov, lnet_kiov_t *dst,
 		return 0;		     /* no frags */
 
 	LASSERT(src_niov > 0);
-	while (offset >= src->kiov_len) {      /* skip initial frags */
-		offset -= src->kiov_len;
+	while (offset >= src->bv_len) {      /* skip initial frags */
+		offset -= src->bv_len;
 		src_niov--;
 		src++;
 		LASSERT(src_niov > 0);
@@ -538,19 +338,19 @@ lnet_extract_kiov(int dst_niov, lnet_kiov_t *dst,
 		LASSERT(src_niov > 0);
 		LASSERT((int)niov <= dst_niov);
 
-		frag_len = src->kiov_len - offset;
-		dst->kiov_page = src->kiov_page;
-		dst->kiov_offset = src->kiov_offset + offset;
+		frag_len = src->bv_len - offset;
+		dst->bv_page = src->bv_page;
+		dst->bv_offset = src->bv_offset + offset;
 
 		if (len <= frag_len) {
-			dst->kiov_len = len;
-			LASSERT(dst->kiov_offset + dst->kiov_len
+			dst->bv_len = len;
+			LASSERT(dst->bv_offset + dst->bv_len
 					<= PAGE_SIZE);
 			return niov;
 		}
 
-		dst->kiov_len = frag_len;
-		LASSERT(dst->kiov_offset + dst->kiov_len <= PAGE_SIZE);
+		dst->bv_len = frag_len;
+		LASSERT(dst->bv_offset + dst->bv_len <= PAGE_SIZE);
 
 		len -= frag_len;
 		dst++;
@@ -569,6 +369,7 @@ lnet_ni_recv(lnet_ni_t *ni, void *private, lnet_msg_t *msg, int delayed,
 	unsigned int niov = 0;
 	struct kvec *iov = NULL;
 	lnet_kiov_t *kiov = NULL;
+	struct iov_iter to;
 	int rc;
 
 	LASSERT(!in_interrupt());
@@ -594,8 +395,14 @@ lnet_ni_recv(lnet_ni_t *ni, void *private, lnet_msg_t *msg, int delayed,
 		}
 	}
 
-	rc = ni->ni_lnd->lnd_recv(ni, private, msg, delayed,
-				  niov, iov, kiov, offset, mlen, rlen);
+	if (iov) {
+		iov_iter_kvec(&to, ITER_KVEC | READ, iov, niov, mlen + offset);
+		iov_iter_advance(&to, offset);
+	} else {
+		iov_iter_bvec(&to, ITER_BVEC | READ, kiov, niov, mlen + offset);
+		iov_iter_advance(&to, offset);
+	}
+	rc = ni->ni_lnd->lnd_recv(ni, private, msg, delayed, &to, rlen);
 	if (rc < 0)
 		lnet_finalize(ni, msg, rc);
 }
@@ -2002,6 +1809,9 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid,
 		       libcfs_nid2str(from_nid), libcfs_nid2str(src_nid),
 		       lnet_msgtyp2str(type), rc);
 		lnet_msg_free(msg);
+		if (rc == -ESHUTDOWN)
+			/* We are shutting down. Don't do anything more */
+			return 0;
 		goto drop;
 	}
 
@@ -2512,6 +2322,15 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp)
 		}
 
 		if (LNET_NIDNET(ni->ni_nid) == dstnet) {
+			/*
+			 * Check if ni was originally created in
+			 * current net namespace.
+			 * If not, assign order above 0xffff0000,
+			 * to make this ni not a priority.
+			 */
+			if (!net_eq(ni->ni_net_ns, current->nsproxy->net_ns))
+				order += 0xffff0000;
+
 			if (srcnidp)
 				*srcnidp = ni->ni_nid;
 			if (orderp)
diff --git a/drivers/staging/lustre/lnet/lnet/lib-msg.c b/drivers/staging/lustre/lnet/lnet/lib-msg.c
index 910e106e221d..0897e588bd54 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-msg.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-msg.c
@@ -449,23 +449,7 @@ lnet_finalize(lnet_ni_t *ni, lnet_msg_t *msg, int status)
 
 	if (!msg)
 		return;
-#if 0
-	CDEBUG(D_WARNING, "%s msg->%s Flags:%s%s%s%s%s%s%s%s%s%s%s txp %s rxp %s\n",
-	       lnet_msgtyp2str(msg->msg_type), libcfs_id2str(msg->msg_target),
-	       msg->msg_target_is_router ? "t" : "",
-	       msg->msg_routing ? "X" : "",
-	       msg->msg_ack ? "A" : "",
-	       msg->msg_sending ? "S" : "",
-	       msg->msg_receiving ? "R" : "",
-	       msg->msg_delayed ? "d" : "",
-	       msg->msg_txcredit ? "C" : "",
-	       msg->msg_peertxcredit ? "c" : "",
-	       msg->msg_rtrcredit ? "F" : "",
-	       msg->msg_peerrtrcredit ? "f" : "",
-	       msg->msg_onactivelist ? "!" : "",
-	       !msg->msg_txpeer ? "<none>" : libcfs_nid2str(msg->msg_txpeer->lp_nid),
-	       !msg->msg_rxpeer ? "<none>" : libcfs_nid2str(msg->msg_rxpeer->lp_nid));
-#endif
+
 	msg->msg_ev.status = status;
 
 	if (msg->msg_md) {
diff --git a/drivers/staging/lustre/lnet/lnet/lib-socket.c b/drivers/staging/lustre/lnet/lnet/lib-socket.c
index 891fd59401d7..4e6dd5149b4f 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-socket.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-socket.c
@@ -265,21 +265,17 @@ lnet_sock_write(struct socket *sock, void *buffer, int nob, int timeout)
 	long jiffies_left = timeout * msecs_to_jiffies(MSEC_PER_SEC);
 	unsigned long then;
 	struct timeval tv;
+	struct kvec  iov = { .iov_base = buffer, .iov_len  = nob };
+	struct msghdr msg = {NULL,};
 
 	LASSERT(nob > 0);
 	/*
 	 * Caller may pass a zero timeout if she thinks the socket buffer is
 	 * empty enough to take the whole message immediately
 	 */
+	iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC, &iov, 1, nob);
 	for (;;) {
-		struct kvec  iov = {
-			.iov_base = buffer,
-			.iov_len  = nob
-		};
-		struct msghdr msg = {
-			.msg_flags      = !timeout ? MSG_DONTWAIT : 0
-		};
-
+		msg.msg_flags = !timeout ? MSG_DONTWAIT : 0;
 		if (timeout) {
 			/* Set send timeout to remaining time */
 			jiffies_to_timeval(jiffies_left, &tv);
@@ -296,9 +292,6 @@ lnet_sock_write(struct socket *sock, void *buffer, int nob, int timeout)
 		rc = kernel_sendmsg(sock, &msg, &iov, 1, nob);
 		jiffies_left -= jiffies - then;
 
-		if (rc == nob)
-			return 0;
-
 		if (rc < 0)
 			return rc;
 
@@ -307,11 +300,11 @@ lnet_sock_write(struct socket *sock, void *buffer, int nob, int timeout)
 			return -ECONNABORTED;
 		}
 
+		if (!msg_data_left(&msg))
+			break;
+
 		if (jiffies_left <= 0)
 			return -EAGAIN;
-
-		buffer = ((char *)buffer) + rc;
-		nob -= rc;
 	}
 	return 0;
 }
diff --git a/drivers/staging/lustre/lnet/lnet/lo.c b/drivers/staging/lustre/lnet/lnet/lo.c
index 08402712a452..cb213b8f51cf 100644
--- a/drivers/staging/lustre/lnet/lnet/lo.c
+++ b/drivers/staging/lustre/lnet/lnet/lo.c
@@ -42,36 +42,23 @@ lolnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg)
 
 static int
 lolnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg,
-	   int delayed, unsigned int niov,
-	   struct kvec *iov, lnet_kiov_t *kiov,
-	   unsigned int offset, unsigned int mlen, unsigned int rlen)
+	   int delayed, struct iov_iter *to, unsigned int rlen)
 {
 	lnet_msg_t *sendmsg = private;
 
 	if (lntmsg) {		   /* not discarding */
-		if (sendmsg->msg_iov) {
-			if (iov)
-				lnet_copy_iov2iov(niov, iov, offset,
-						  sendmsg->msg_niov,
-						  sendmsg->msg_iov,
-						  sendmsg->msg_offset, mlen);
-			else
-				lnet_copy_iov2kiov(niov, kiov, offset,
-						   sendmsg->msg_niov,
-						   sendmsg->msg_iov,
-						   sendmsg->msg_offset, mlen);
-		} else {
-			if (iov)
-				lnet_copy_kiov2iov(niov, iov, offset,
-						   sendmsg->msg_niov,
-						   sendmsg->msg_kiov,
-						   sendmsg->msg_offset, mlen);
-			else
-				lnet_copy_kiov2kiov(niov, kiov, offset,
-						    sendmsg->msg_niov,
-						    sendmsg->msg_kiov,
-						    sendmsg->msg_offset, mlen);
-		}
+		if (sendmsg->msg_iov)
+			lnet_copy_iov2iter(to,
+					   sendmsg->msg_niov,
+					   sendmsg->msg_iov,
+					   sendmsg->msg_offset,
+					   iov_iter_count(to));
+		else
+			lnet_copy_kiov2iter(to,
+					    sendmsg->msg_niov,
+					    sendmsg->msg_kiov,
+					    sendmsg->msg_offset,
+					    iov_iter_count(to));
 
 		lnet_finalize(ni, lntmsg, 0);
 	}
diff --git a/drivers/staging/lustre/lnet/lnet/router.c b/drivers/staging/lustre/lnet/lnet/router.c
index 063543233035..063ad55ec950 100644
--- a/drivers/staging/lustre/lnet/lnet/router.c
+++ b/drivers/staging/lustre/lnet/lnet/router.c
@@ -1307,7 +1307,7 @@ lnet_destroy_rtrbuf(lnet_rtrbuf_t *rb, int npages)
 	int sz = offsetof(lnet_rtrbuf_t, rb_kiov[npages]);
 
 	while (--npages >= 0)
-		__free_page(rb->rb_kiov[npages].kiov_page);
+		__free_page(rb->rb_kiov[npages].bv_page);
 
 	LIBCFS_FREE(rb, sz);
 }
@@ -1333,15 +1333,15 @@ lnet_new_rtrbuf(lnet_rtrbufpool_t *rbp, int cpt)
 				GFP_KERNEL | __GFP_ZERO, 0);
 		if (!page) {
 			while (--i >= 0)
-				__free_page(rb->rb_kiov[i].kiov_page);
+				__free_page(rb->rb_kiov[i].bv_page);
 
 			LIBCFS_FREE(rb, sz);
 			return NULL;
 		}
 
-		rb->rb_kiov[i].kiov_len = PAGE_SIZE;
-		rb->rb_kiov[i].kiov_offset = 0;
-		rb->rb_kiov[i].kiov_page = page;
+		rb->rb_kiov[i].bv_len = PAGE_SIZE;
+		rb->rb_kiov[i].bv_offset = 0;
+		rb->rb_kiov[i].bv_page = page;
 	}
 
 	return rb;
@@ -1693,7 +1693,7 @@ lnet_rtrpools_adjust(int tiny, int small, int large)
 int
 lnet_rtrpools_enable(void)
 {
-	int rc;
+	int rc = 0;
 
 	if (the_lnet.ln_routing)
 		return 0;
@@ -1706,9 +1706,9 @@ lnet_rtrpools_enable(void)
 		 * if we are just configuring this for the first
 		 * time.
 		 */
-		return lnet_rtrpools_alloc(1);
-
-	rc = lnet_rtrpools_adjust_helper(0, 0, 0);
+		rc = lnet_rtrpools_alloc(1);
+	else
+		rc = lnet_rtrpools_adjust_helper(0, 0, 0);
 	if (rc)
 		return rc;
 
@@ -1718,7 +1718,7 @@ lnet_rtrpools_enable(void)
 	the_lnet.ln_ping_info->pi_features &= ~LNET_PING_FEAT_RTE_DISABLED;
 	lnet_net_unlock(LNET_LOCK_EX);
 
-	return 0;
+	return rc;
 }
 
 void
diff --git a/drivers/staging/lustre/lnet/selftest/brw_test.c b/drivers/staging/lustre/lnet/selftest/brw_test.c
index 13d0454e7fcb..b20c5d394e3b 100644
--- a/drivers/staging/lustre/lnet/selftest/brw_test.c
+++ b/drivers/staging/lustre/lnet/selftest/brw_test.c
@@ -226,7 +226,7 @@ brw_fill_bulk(struct srpc_bulk *bk, int pattern, __u64 magic)
 	struct page *pg;
 
 	for (i = 0; i < bk->bk_niov; i++) {
-		pg = bk->bk_iovs[i].kiov_page;
+		pg = bk->bk_iovs[i].bv_page;
 		brw_fill_page(pg, pattern, magic);
 	}
 }
@@ -238,7 +238,7 @@ brw_check_bulk(struct srpc_bulk *bk, int pattern, __u64 magic)
 	struct page *pg;
 
 	for (i = 0; i < bk->bk_niov; i++) {
-		pg = bk->bk_iovs[i].kiov_page;
+		pg = bk->bk_iovs[i].bv_page;
 		if (brw_check_page(pg, pattern, magic)) {
 			CERROR("Bulk page %p (%d/%d) is corrupted!\n",
 			       pg, i, bk->bk_niov);
diff --git a/drivers/staging/lustre/lnet/selftest/conrpc.c b/drivers/staging/lustre/lnet/selftest/conrpc.c
index 1be3cad727ae..55afb53b0743 100644
--- a/drivers/staging/lustre/lnet/selftest/conrpc.c
+++ b/drivers/staging/lustre/lnet/selftest/conrpc.c
@@ -152,10 +152,10 @@ lstcon_rpc_put(struct lstcon_rpc *crpc)
 	LASSERT(list_empty(&crpc->crp_link));
 
 	for (i = 0; i < bulk->bk_niov; i++) {
-		if (!bulk->bk_iovs[i].kiov_page)
+		if (!bulk->bk_iovs[i].bv_page)
 			continue;
 
-		__free_page(bulk->bk_iovs[i].kiov_page);
+		__free_page(bulk->bk_iovs[i].bv_page);
 	}
 
 	srpc_client_rpc_decref(crpc->crp_rpc);
@@ -705,7 +705,7 @@ lstcon_next_id(int idx, int nkiov, lnet_kiov_t *kiov)
 
 	LASSERT(i < nkiov);
 
-	pid = (lnet_process_id_packed_t *)page_address(kiov[i].kiov_page);
+	pid = (lnet_process_id_packed_t *)page_address(kiov[i].bv_page);
 
 	return &pid[idx % SFW_ID_PER_PAGE];
 }
@@ -849,12 +849,11 @@ lstcon_testrpc_prep(struct lstcon_node *nd, int transop, unsigned feats,
 			      min_t(int, nob, PAGE_SIZE);
 			nob -= len;
 
-			bulk->bk_iovs[i].kiov_offset = 0;
-			bulk->bk_iovs[i].kiov_len = len;
-			bulk->bk_iovs[i].kiov_page =
-				alloc_page(GFP_KERNEL);
+			bulk->bk_iovs[i].bv_offset = 0;
+			bulk->bk_iovs[i].bv_len = len;
+			bulk->bk_iovs[i].bv_page = alloc_page(GFP_KERNEL);
 
-			if (!bulk->bk_iovs[i].kiov_page) {
+			if (!bulk->bk_iovs[i].bv_page) {
 				lstcon_rpc_put(*crpc);
 				return -ENOMEM;
 			}
diff --git a/drivers/staging/lustre/lnet/selftest/console.c b/drivers/staging/lustre/lnet/selftest/console.c
index 4c33621f06da..a0fcbf3bcc95 100644
--- a/drivers/staging/lustre/lnet/selftest/console.c
+++ b/drivers/staging/lustre/lnet/selftest/console.c
@@ -1993,8 +1993,6 @@ static void lstcon_init_acceptor_service(void)
 	lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
 }
 
-extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
-
 static DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
 
 /* initialize console */
diff --git a/drivers/staging/lustre/lnet/selftest/console.h b/drivers/staging/lustre/lnet/selftest/console.h
index 78b147732615..78388a611c22 100644
--- a/drivers/staging/lustre/lnet/selftest/console.h
+++ b/drivers/staging/lustre/lnet/selftest/console.h
@@ -184,6 +184,7 @@ lstcon_id2hash(lnet_process_id_t id, struct list_head *hash)
 	return &hash[idx];
 }
 
+int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
 int lstcon_console_init(void);
 int lstcon_console_fini(void);
 int lstcon_session_match(lst_sid_t sid);
diff --git a/drivers/staging/lustre/lnet/selftest/framework.c b/drivers/staging/lustre/lnet/selftest/framework.c
index c2f121f44d33..abbd6287b4bd 100644
--- a/drivers/staging/lustre/lnet/selftest/framework.c
+++ b/drivers/staging/lustre/lnet/selftest/framework.c
@@ -784,8 +784,8 @@ sfw_add_test_instance(struct sfw_batch *tsb, struct srpc_server_rpc *rpc)
 		lnet_process_id_packed_t id;
 		int j;
 
-		dests = page_address(bk->bk_iovs[i / SFW_ID_PER_PAGE].kiov_page);
-		LASSERT(dests);		/* my pages are within KVM always */
+		dests = page_address(bk->bk_iovs[i / SFW_ID_PER_PAGE].bv_page);
+		LASSERT(dests);  /* my pages are within KVM always */
 		id = dests[i % SFW_ID_PER_PAGE];
 		if (msg->msg_magic != SRPC_MSG_MAGIC)
 			sfw_unpack_id(id);
diff --git a/drivers/staging/lustre/lnet/selftest/rpc.c b/drivers/staging/lustre/lnet/selftest/rpc.c
index 3b26d6eb4240..f5619d8744ef 100644
--- a/drivers/staging/lustre/lnet/selftest/rpc.c
+++ b/drivers/staging/lustre/lnet/selftest/rpc.c
@@ -91,9 +91,9 @@ srpc_add_bulk_page(struct srpc_bulk *bk, struct page *pg, int i, int nob)
 	LASSERT(nob > 0);
 	LASSERT(i >= 0 && i < bk->bk_niov);
 
-	bk->bk_iovs[i].kiov_offset = 0;
-	bk->bk_iovs[i].kiov_page = pg;
-	bk->bk_iovs[i].kiov_len = nob;
+	bk->bk_iovs[i].bv_offset = 0;
+	bk->bk_iovs[i].bv_page = pg;
+	bk->bk_iovs[i].bv_len = nob;
 	return nob;
 }
 
@@ -106,7 +106,7 @@ srpc_free_bulk(struct srpc_bulk *bk)
 	LASSERT(bk);
 
 	for (i = 0; i < bk->bk_niov; i++) {
-		pg = bk->bk_iovs[i].kiov_page;
+		pg = bk->bk_iovs[i].bv_page;
 		if (!pg)
 			break;
 
diff --git a/drivers/staging/lustre/lustre/fid/fid_lib.c b/drivers/staging/lustre/lustre/fid/fid_lib.c
index 99ae7eb6720e..4e49cb356d64 100644
--- a/drivers/staging/lustre/lustre/fid/fid_lib.c
+++ b/drivers/staging/lustre/lustre/fid/fid_lib.c
@@ -63,14 +63,12 @@ const struct lu_seq_range LUSTRE_SEQ_SPACE_RANGE = {
 	FID_SEQ_NORMAL,
 	(__u64)~0ULL
 };
-EXPORT_SYMBOL(LUSTRE_SEQ_SPACE_RANGE);
 
 /* Zero range, used for init and other purposes. */
 const struct lu_seq_range LUSTRE_SEQ_ZERO_RANGE = {
 	0,
 	0
 };
-EXPORT_SYMBOL(LUSTRE_SEQ_ZERO_RANGE);
 
 /* Lustre Big Fs Lock fid. */
 const struct lu_fid LUSTRE_BFL_FID = { .f_seq = FID_SEQ_SPECIAL,
diff --git a/drivers/staging/lustre/lustre/fid/fid_request.c b/drivers/staging/lustre/lustre/fid/fid_request.c
index 454744d25956..edd72b926f81 100644
--- a/drivers/staging/lustre/lustre/fid/fid_request.c
+++ b/drivers/staging/lustre/lustre/fid/fid_request.c
@@ -125,19 +125,19 @@ static int seq_client_rpc(struct lu_client_seq *seq,
 
 	if (!range_is_sane(output)) {
 		CERROR("%s: Invalid range received from server: "
-		       DRANGE"\n", seq->lcs_name, PRANGE(output));
+		       DRANGE "\n", seq->lcs_name, PRANGE(output));
 		rc = -EINVAL;
 		goto out_req;
 	}
 
 	if (range_is_exhausted(output)) {
 		CERROR("%s: Range received from server is exhausted: "
-		       DRANGE"]\n", seq->lcs_name, PRANGE(output));
+		       DRANGE "]\n", seq->lcs_name, PRANGE(output));
 		rc = -EINVAL;
 		goto out_req;
 	}
 
-	CDEBUG_LIMIT(debug_mask, "%s: Allocated %s-sequence "DRANGE"]\n",
+	CDEBUG_LIMIT(debug_mask, "%s: Allocated %s-sequence " DRANGE "]\n",
 		     seq->lcs_name, opcname, PRANGE(output));
 
 out_req:
@@ -179,7 +179,7 @@ static int seq_client_alloc_seq(const struct lu_env *env,
 			       seq->lcs_name, rc);
 			return rc;
 		}
-		CDEBUG(D_INFO, "%s: New range - "DRANGE"\n",
+		CDEBUG(D_INFO, "%s: New range - " DRANGE "\n",
 		       seq->lcs_name, PRANGE(&seq->lcs_space));
 	} else {
 		rc = 0;
diff --git a/drivers/staging/lustre/lustre/fid/lproc_fid.c b/drivers/staging/lustre/lustre/fid/lproc_fid.c
index 81b7ca9ea2fd..3ed32d77f38b 100644
--- a/drivers/staging/lustre/lustre/fid/lproc_fid.c
+++ b/drivers/staging/lustre/lustre/fid/lproc_fid.c
@@ -105,7 +105,7 @@ ldebugfs_fid_space_seq_write(struct file *file,
 	rc = ldebugfs_fid_write_common(buffer, count, &seq->lcs_space);
 
 	if (rc == 0) {
-		CDEBUG(D_INFO, "%s: Space: "DRANGE"\n",
+		CDEBUG(D_INFO, "%s: Space: " DRANGE "\n",
 		       seq->lcs_name, PRANGE(&seq->lcs_space));
 	}
 
diff --git a/drivers/staging/lustre/lustre/fld/fld_internal.h b/drivers/staging/lustre/lustre/fld/fld_internal.h
index f0efe5b9fbec..08eaec735d6f 100644
--- a/drivers/staging/lustre/lustre/fld/fld_internal.h
+++ b/drivers/staging/lustre/lustre/fld/fld_internal.h
@@ -31,6 +31,25 @@
  *
  * lustre/fld/fld_internal.h
  *
+ * Subsystem Description:
+ * FLD is FID Location Database, which stores where (IE, on which MDT)
+ * FIDs are located.
+ * The database is basically a record file, each record consists of a FID
+ * sequence range, MDT/OST index, and flags. The FLD for the whole FS
+ * is only stored on the sequence controller(MDT0) right now, but each target
+ * also has its local FLD, which only stores the local sequence.
+ *
+ * The FLD subsystem usually has two tasks:
+ * 1. maintain the database, i.e. when the sequence controller allocates
+ * new sequence ranges to some nodes, it will call the FLD API to insert the
+ * location information <sequence_range, node_index> in FLDB.
+ *
+ * 2. Handle requests from other nodes, i.e. if client needs to know where
+ * the FID is located, if it can not find the information in the local cache,
+ * it will send a FLD lookup RPC to the FLD service, and the FLD service will
+ * look up the FLDB entry and return the location information to client.
+ *
+ *
  * Author: Yury Umanets <umka@clusterfs.com>
  * Author: Tom WangDi <wangdi@clusterfs.com>
  */
diff --git a/drivers/staging/lustre/lustre/fld/fld_request.c b/drivers/staging/lustre/lustre/fld/fld_request.c
index e59d626a1548..0de72b717ce5 100644
--- a/drivers/staging/lustre/lustre/fld/fld_request.c
+++ b/drivers/staging/lustre/lustre/fld/fld_request.c
@@ -53,57 +53,6 @@
 #include "../include/lustre_mdc.h"
 #include "fld_internal.h"
 
-/* TODO: these 3 functions are copies of flow-control code from mdc_lib.c
- * It should be common thing. The same about mdc RPC lock
- */
-static int fld_req_avail(struct client_obd *cli, struct mdc_cache_waiter *mcw)
-{
-	int rc;
-
-	spin_lock(&cli->cl_loi_list_lock);
-	rc = list_empty(&mcw->mcw_entry);
-	spin_unlock(&cli->cl_loi_list_lock);
-	return rc;
-};
-
-static void fld_enter_request(struct client_obd *cli)
-{
-	struct mdc_cache_waiter mcw;
-	struct l_wait_info lwi = { 0 };
-
-	spin_lock(&cli->cl_loi_list_lock);
-	if (cli->cl_r_in_flight >= cli->cl_max_rpcs_in_flight) {
-		list_add_tail(&mcw.mcw_entry, &cli->cl_cache_waiters);
-		init_waitqueue_head(&mcw.mcw_waitq);
-		spin_unlock(&cli->cl_loi_list_lock);
-		l_wait_event(mcw.mcw_waitq, fld_req_avail(cli, &mcw), &lwi);
-	} else {
-		cli->cl_r_in_flight++;
-		spin_unlock(&cli->cl_loi_list_lock);
-	}
-}
-
-static void fld_exit_request(struct client_obd *cli)
-{
-	struct list_head *l, *tmp;
-	struct mdc_cache_waiter *mcw;
-
-	spin_lock(&cli->cl_loi_list_lock);
-	cli->cl_r_in_flight--;
-	list_for_each_safe(l, tmp, &cli->cl_cache_waiters) {
-		if (cli->cl_r_in_flight >= cli->cl_max_rpcs_in_flight) {
-			/* No free request slots anymore */
-			break;
-		}
-
-		mcw = list_entry(l, struct mdc_cache_waiter, mcw_entry);
-		list_del_init(&mcw->mcw_entry);
-		cli->cl_r_in_flight++;
-		wake_up(&mcw->mcw_waitq);
-	}
-	spin_unlock(&cli->cl_loi_list_lock);
-}
-
 static int fld_rrb_hash(struct lu_client_fld *fld, u64 seq)
 {
 	LASSERT(fld->lcf_count > 0);
@@ -270,7 +219,6 @@ int fld_client_del_target(struct lu_client_fld *fld, __u64 idx)
 	spin_unlock(&fld->lcf_lock);
 	return -ENOENT;
 }
-EXPORT_SYMBOL(fld_client_del_target);
 
 static struct dentry *fld_debugfs_dir;
 
@@ -439,9 +387,9 @@ int fld_client_rpc(struct obd_export *exp,
 	req->rq_reply_portal = MDC_REPLY_PORTAL;
 	ptlrpc_at_set_req_timeout(req);
 
-	fld_enter_request(&exp->exp_obd->u.cli);
+	obd_get_request_slot(&exp->exp_obd->u.cli);
 	rc = ptlrpc_queue_wait(req);
-	fld_exit_request(&exp->exp_obd->u.cli);
+	obd_put_request_slot(&exp->exp_obd->u.cli);
 	if (rc)
 		goto out_req;
 
@@ -505,7 +453,6 @@ void fld_client_flush(struct lu_client_fld *fld)
 {
 	fld_cache_flush(fld->lcf_cache);
 }
-EXPORT_SYMBOL(fld_client_flush);
 
 static int __init fld_init(void)
 {
diff --git a/drivers/staging/lustre/lustre/include/cl_object.h b/drivers/staging/lustre/lustre/include/cl_object.h
index 3cd4a2577d90..89292c93dcd5 100644
--- a/drivers/staging/lustre/lustre/include/cl_object.h
+++ b/drivers/staging/lustre/lustre/include/cl_object.h
@@ -93,8 +93,8 @@
  * super-class definitions.
  */
 #include "lu_object.h"
+#include "lustre_compat.h"
 #include <linux/atomic.h>
-#include "linux/lustre_compat25.h"
 #include <linux/mutex.h>
 #include <linux/radix-tree.h>
 #include <linux/spinlock.h>
@@ -191,6 +191,9 @@ struct cl_attr {
 	 * Group identifier for quota purposes.
 	 */
 	gid_t  cat_gid;
+
+	/* nlink of the directory */
+	__u64  cat_nlink;
 };
 
 /**
@@ -320,7 +323,7 @@ struct cl_object_operations {
 	 *	 to be used instead of newly created.
 	 */
 	int  (*coo_page_init)(const struct lu_env *env, struct cl_object *obj,
-				struct cl_page *page, pgoff_t index);
+			      struct cl_page *page, pgoff_t index);
 	/**
 	 * Initialize lock slice for this layer. Called top-to-bottom through
 	 * every object layer when a new cl_lock is instantiated. Layer
@@ -366,8 +369,8 @@ struct cl_object_operations {
 	 * \return the same convention as for
 	 * cl_object_operations::coo_attr_get() is used.
 	 */
-	int (*coo_attr_set)(const struct lu_env *env, struct cl_object *obj,
-			    const struct cl_attr *attr, unsigned valid);
+	int (*coo_attr_update)(const struct lu_env *env, struct cl_object *obj,
+			       const struct cl_attr *attr, unsigned int valid);
 	/**
 	 * Update object configuration. Called top-to-bottom to modify object
 	 * configuration.
@@ -392,6 +395,11 @@ struct cl_object_operations {
 	 * mainly pages and locks.
 	 */
 	int (*coo_prune)(const struct lu_env *env, struct cl_object *obj);
+	/**
+	 * Object getstripe method.
+	 */
+	int (*coo_getstripe)(const struct lu_env *env, struct cl_object *obj,
+			     struct lov_user_md __user *lum);
 };
 
 /**
@@ -687,17 +695,6 @@ enum cl_page_type {
 };
 
 /**
- * Flags maintained for every cl_page.
- */
-enum cl_page_flags {
-	/**
-	 * Set when pagein completes. Used for debugging (read completes at
-	 * most once for a page).
-	 */
-	CPF_READ_COMPLETED = 1 << 0
-};
-
-/**
  * Fields are protected by the lock on struct page, except for atomics and
  * immutables.
  *
@@ -711,24 +708,19 @@ struct cl_page {
 	atomic_t	     cp_ref;
 	/** An object this page is a part of. Immutable after creation. */
 	struct cl_object	*cp_obj;
-	/** List of slices. Immutable after creation. */
-	struct list_head	       cp_layers;
 	/** vmpage */
 	struct page		*cp_vmpage;
+	/** Linkage of pages within group. Pages must be owned */
+	struct list_head	 cp_batch;
+	/** List of slices. Immutable after creation. */
+	struct list_head	 cp_layers;
+	/** Linkage of pages within cl_req. */
+	struct list_head         cp_flight;
 	/**
 	 * Page state. This field is const to avoid accidental update, it is
 	 * modified only internally within cl_page.c. Protected by a VM lock.
 	 */
 	const enum cl_page_state cp_state;
-	/** Linkage of pages within group. Protected by cl_page::cp_mutex. */
-	struct list_head		cp_batch;
-	/** Mutex serializing membership of a page in a batch. */
-	struct mutex		cp_mutex;
-	/** Linkage of pages within cl_req. */
-	struct list_head	       cp_flight;
-	/** Transfer error. */
-	int		      cp_error;
-
 	/**
 	 * Page type. Only CPT_TRANSIENT is used so far. Immutable after
 	 * creation.
@@ -741,10 +733,6 @@ struct cl_page {
 	 */
 	struct cl_io	    *cp_owner;
 	/**
-	 * Debug information, the task is owning the page.
-	 */
-	struct task_struct	*cp_task;
-	/**
 	 * Owning IO request in cl_page_state::CPS_PAGEOUT and
 	 * cl_page_state::CPS_PAGEIN states. This field is maintained only in
 	 * the top-level pages. Protected by a VM lock.
@@ -756,8 +744,6 @@ struct cl_page {
 	struct lu_ref_link       cp_obj_ref;
 	/** Link to a queue, for debugging. */
 	struct lu_ref_link       cp_queue_ref;
-	/** Per-page flags from enum cl_page_flags. Protected by a VM lock. */
-	unsigned                 cp_flags;
 	/** Assigned if doing a sync_io */
 	struct cl_sync_io       *cp_sync_io;
 };
@@ -1056,23 +1042,32 @@ do {									  \
 	}								     \
 } while (0)
 
-static inline int __page_in_use(const struct cl_page *page, int refc)
-{
-	if (page->cp_type == CPT_CACHEABLE)
-		++refc;
-	LASSERT(atomic_read(&page->cp_ref) > 0);
-	return (atomic_read(&page->cp_ref) > refc);
-}
-
-#define cl_page_in_use(pg)       __page_in_use(pg, 1)
-#define cl_page_in_use_noref(pg) __page_in_use(pg, 0)
-
 static inline struct page *cl_page_vmpage(struct cl_page *page)
 {
 	LASSERT(page->cp_vmpage);
 	return page->cp_vmpage;
 }
 
+/**
+ * Check if a cl_page is in use.
+ *
+ * Client cache holds a refcount, this refcount will be dropped when
+ * the page is taken out of cache, see vvp_page_delete().
+ */
+static inline bool __page_in_use(const struct cl_page *page, int refc)
+{
+	return (atomic_read(&page->cp_ref) > refc + 1);
+}
+
+/**
+ * Caller itself holds a refcount of cl_page.
+ */
+#define cl_page_in_use(pg)	 __page_in_use(pg, 1)
+/**
+ * Caller doesn't hold a refcount.
+ */
+#define cl_page_in_use_noref(pg) __page_in_use(pg, 0)
+
 /** @} cl_page */
 
 /** \addtogroup cl_lock cl_lock
@@ -1771,12 +1766,14 @@ struct cl_io {
 		struct cl_setattr_io {
 			struct ost_lvb   sa_attr;
 			unsigned int     sa_valid;
+			int		sa_stripe_index;
+			struct lu_fid  *sa_parent_fid;
 		} ci_setattr;
 		struct cl_fault_io {
 			/** page index within file. */
 			pgoff_t	 ft_index;
 			/** bytes valid byte on a faulted page. */
-			int	     ft_nob;
+			size_t	     ft_nob;
 			/** writable page? for nopage() only */
 			int	     ft_writable;
 			/** page of an executable? */
@@ -1909,7 +1906,7 @@ struct cl_req_attr {
 	/** Generic attributes for the server consumption. */
 	struct obdo	*cra_oa;
 	/** Jobid */
-	char		 cra_jobid[JOBSTATS_JOBID_SIZE];
+	char		 cra_jobid[LUSTRE_JOBID_SIZE];
 };
 
 /**
@@ -2176,14 +2173,16 @@ void cl_object_attr_lock(struct cl_object *o);
 void cl_object_attr_unlock(struct cl_object *o);
 int  cl_object_attr_get(const struct lu_env *env, struct cl_object *obj,
 			struct cl_attr *attr);
-int  cl_object_attr_set(const struct lu_env *env, struct cl_object *obj,
-			const struct cl_attr *attr, unsigned valid);
+int  cl_object_attr_update(const struct lu_env *env, struct cl_object *obj,
+			   const struct cl_attr *attr, unsigned int valid);
 int  cl_object_glimpse(const struct lu_env *env, struct cl_object *obj,
 		       struct ost_lvb *lvb);
 int  cl_conf_set(const struct lu_env *env, struct cl_object *obj,
 		 const struct cl_object_conf *conf);
 int cl_object_prune(const struct lu_env *env, struct cl_object *obj);
 void cl_object_kill(const struct lu_env *env, struct cl_object *obj);
+int  cl_object_getstripe(const struct lu_env *env, struct cl_object *obj,
+			 struct lov_user_md __user *lum);
 
 /**
  * Returns true, iff \a o0 and \a o1 are slices of the same object.
@@ -2197,6 +2196,7 @@ static inline void cl_object_page_init(struct cl_object *clob, int size)
 {
 	clob->co_slice_off = cl_object_header(clob)->coh_page_bufsize;
 	cl_object_header(clob)->coh_page_bufsize += cfs_size_round(size);
+	WARN_ON(cl_object_header(clob)->coh_page_bufsize > 512);
 }
 
 static inline void *cl_object_page_slice(struct cl_object *clob,
@@ -2263,6 +2263,8 @@ void cl_page_unassume(const struct lu_env *env,
 		      struct cl_io *io, struct cl_page *pg);
 void cl_page_disown(const struct lu_env *env,
 		    struct cl_io *io, struct cl_page *page);
+void cl_page_disown0(const struct lu_env *env,
+		     struct cl_io *io, struct cl_page *pg);
 int cl_page_is_owned(const struct cl_page *pg, const struct cl_io *io);
 
 /** @} ownership */
@@ -2304,7 +2306,7 @@ int cl_page_is_under_lock(const struct lu_env *env, struct cl_io *io,
 			  struct cl_page *page, pgoff_t *max_index);
 loff_t cl_offset(const struct cl_object *obj, pgoff_t idx);
 pgoff_t cl_index(const struct cl_object *obj, loff_t offset);
-int cl_page_size(const struct cl_object *obj);
+size_t cl_page_size(const struct cl_object *obj);
 int cl_pages_prune(const struct lu_env *env, struct cl_object *obj);
 
 void cl_lock_print(const struct lu_env *env, void *cookie,
@@ -2333,7 +2335,7 @@ struct cl_client_cache {
 	/**
 	 * # of LRU entries available
 	 */
-	atomic_t		ccc_lru_left;
+	atomic_long_t		ccc_lru_left;
 	/**
 	 * List of entities(OSCs) for this LRU cache
 	 */
@@ -2347,14 +2349,18 @@ struct cl_client_cache {
 	 */
 	spinlock_t		ccc_lru_lock;
 	/**
+	 * Set if unstable check is enabled
+	 */
+	unsigned int		ccc_unstable_check:1;
+	/**
 	 * # of unstable pages for this mount point
 	 */
-	atomic_t		ccc_unstable_nr;
+	atomic_long_t		ccc_unstable_nr;
 	/**
 	 * Waitq for awaiting unstable pages to reach zero.
 	 * Used at umounting time and signaled on BRW commit
 	 */
-        wait_queue_head_t	ccc_unstable_waitq;
+	wait_queue_head_t	ccc_unstable_waitq;
 
 };
 
diff --git a/drivers/staging/lustre/lustre/include/interval_tree.h b/drivers/staging/lustre/lustre/include/interval_tree.h
index 4a15228b5570..5d387d372547 100644
--- a/drivers/staging/lustre/lustre/include/interval_tree.h
+++ b/drivers/staging/lustre/lustre/include/interval_tree.h
@@ -63,6 +63,11 @@ static inline int interval_is_intree(struct interval_node *node)
 	return node->in_intree == 1;
 }
 
+static inline __u64 interval_low(struct interval_node *node)
+{
+	return node->in_extent.start;
+}
+
 static inline __u64 interval_high(struct interval_node *node)
 {
 	return node->in_extent.end;
@@ -77,8 +82,29 @@ static inline void interval_set(struct interval_node *node,
 	node->in_max_high = end;
 }
 
+/*
+ * Rules to write an interval callback.
+ *  - the callback returns INTERVAL_ITER_STOP when it thinks the iteration
+ *    should be stopped. It will then cause the iteration function to return
+ *    immediately with return value INTERVAL_ITER_STOP.
+ *  - callbacks for interval_iterate and interval_iterate_reverse: Every
+ *    nodes in the tree will be set to @node before the callback being called
+ *  - callback for interval_search: Only overlapped node will be set to @node
+ *    before the callback being called.
+ */
+typedef enum interval_iter (*interval_callback_t)(struct interval_node *node,
+						  void *args);
+
 struct interval_node *interval_insert(struct interval_node *node,
 				      struct interval_node **root);
 void interval_erase(struct interval_node *node, struct interval_node **root);
 
+/*
+ * Search the extents in the tree and call @func for each overlapped
+ * extents.
+ */
+enum interval_iter interval_search(struct interval_node *root,
+				   struct interval_node_extent *ex,
+				   interval_callback_t func, void *data);
+
 #endif
diff --git a/drivers/staging/lustre/lustre/include/linux/lustre_lite.h b/drivers/staging/lustre/lustre/include/linux/lustre_lite.h
deleted file mode 100644
index d18e8a76bb25..000000000000
--- a/drivers/staging/lustre/lustre/include/linux/lustre_lite.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- */
-
-#ifndef _LINUX_LL_H
-#define _LINUX_LL_H
-
-#ifndef _LL_H
-#error Do not #include this file directly. #include <lustre_lite.h> instead
-#endif
-
-#include <linux/statfs.h>
-
-#include <linux/fs.h>
-#include <linux/dcache.h>
-
-#include "../obd_class.h"
-#include "../lustre_net.h"
-#include "../lustre_ha.h"
-
-#include <linux/rbtree.h>
-#include "../../include/linux/lustre_compat25.h"
-#include <linux/pagemap.h>
-
-/* lprocfs.c */
-enum {
-	 LPROC_LL_DIRTY_HITS = 0,
-	 LPROC_LL_DIRTY_MISSES,
-	 LPROC_LL_READ_BYTES,
-	 LPROC_LL_WRITE_BYTES,
-	 LPROC_LL_BRW_READ,
-	 LPROC_LL_BRW_WRITE,
-	 LPROC_LL_OSC_READ,
-	 LPROC_LL_OSC_WRITE,
-	 LPROC_LL_IOCTL,
-	 LPROC_LL_OPEN,
-	 LPROC_LL_RELEASE,
-	 LPROC_LL_MAP,
-	 LPROC_LL_LLSEEK,
-	 LPROC_LL_FSYNC,
-	 LPROC_LL_READDIR,
-	 LPROC_LL_SETATTR,
-	 LPROC_LL_TRUNC,
-	 LPROC_LL_FLOCK,
-	 LPROC_LL_GETATTR,
-	 LPROC_LL_CREATE,
-	 LPROC_LL_LINK,
-	 LPROC_LL_UNLINK,
-	 LPROC_LL_SYMLINK,
-	 LPROC_LL_MKDIR,
-	 LPROC_LL_RMDIR,
-	 LPROC_LL_MKNOD,
-	 LPROC_LL_RENAME,
-	 LPROC_LL_STAFS,
-	 LPROC_LL_ALLOC_INODE,
-	 LPROC_LL_SETXATTR,
-	 LPROC_LL_GETXATTR,
-	 LPROC_LL_GETXATTR_HITS,
-	 LPROC_LL_LISTXATTR,
-	 LPROC_LL_REMOVEXATTR,
-	 LPROC_LL_INODE_PERM,
-	 LPROC_LL_FILE_OPCODES
-};
-
-#endif
diff --git a/drivers/staging/lustre/lustre/include/linux/lustre_user.h b/drivers/staging/lustre/lustre/include/linux/lustre_user.h
deleted file mode 100644
index e967950e8536..000000000000
--- a/drivers/staging/lustre/lustre/include/linux/lustre_user.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * lustre/include/linux/lustre_user.h
- *
- * Lustre public user-space interface definitions.
- */
-
-#ifndef _LINUX_LUSTRE_USER_H
-#define _LINUX_LUSTRE_USER_H
-
-# include <linux/quota.h>
-
-/*
- * asm-x86_64/processor.h on some SLES 9 distros seems to use
- * kernel-only typedefs.  fortunately skipping it altogether is ok
- * (for now).
- */
-#define __ASM_X86_64_PROCESSOR_H
-
-#include <linux/string.h>
-
-/*
- * We need to always use 64bit version because the structure
- * is shared across entire cluster where 32bit and 64bit machines
- * are co-existing.
- */
-#if __BITS_PER_LONG != 64 || defined(__ARCH_WANT_STAT64)
-typedef struct stat64   lstat_t;
-#define lstat_f	 lstat64
-#else
-typedef struct stat     lstat_t;
-#define lstat_f	 lstat
-#endif
-
-#define HAVE_LOV_USER_MDS_DATA
-
-#endif /* _LUSTRE_USER_H */
diff --git a/drivers/staging/lustre/lustre/include/lprocfs_status.h b/drivers/staging/lustre/lustre/include/lprocfs_status.h
index d68e60e7fef7..cc0713ef8ae5 100644
--- a/drivers/staging/lustre/lustre/include/lprocfs_status.h
+++ b/drivers/staging/lustre/lustre/include/lprocfs_status.h
@@ -165,8 +165,10 @@ struct lprocfs_percpu {
 	struct lprocfs_counter lp_cntr[0];
 };
 
-#define LPROCFS_GET_NUM_CPU 0x0001
-#define LPROCFS_GET_SMP_ID  0x0002
+enum lprocfs_stats_lock_ops {
+	LPROCFS_GET_NUM_CPU	= 0x0001, /* number allocated per-CPU stats */
+	LPROCFS_GET_SMP_ID	= 0x0002, /* current stat to be updated */
+};
 
 enum lprocfs_stats_flags {
 	LPROCFS_STATS_FLAG_NONE     = 0x0000, /* per cpu counter */
@@ -363,82 +365,99 @@ static inline void s2dhms(struct dhms *ts, time64_t secs64)
 #define JOBSTATS_PROCNAME_UID		"procname_uid"
 #define JOBSTATS_NODELOCAL		"nodelocal"
 
+/* obd_config.c */
+void lustre_register_client_process_config(int (*cpc)(struct lustre_cfg *lcfg));
+
 int lprocfs_write_frac_helper(const char __user *buffer,
 			      unsigned long count, int *val, int mult);
 int lprocfs_read_frac_helper(char *buffer, unsigned long count,
 			     long val, int mult);
 int lprocfs_stats_alloc_one(struct lprocfs_stats *stats, unsigned int cpuid);
-/*
- * \return value
- *      < 0     : on error (only possible for opc as LPROCFS_GET_SMP_ID)
+
+/**
+ * Lock statistics structure for access, possibly only on this CPU.
+ *
+ * The statistics struct may be allocated with per-CPU structures for
+ * efficient concurrent update (usually only on server-wide stats), or
+ * as a single global struct (e.g. for per-client or per-job statistics),
+ * so the required locking depends on the type of structure allocated.
+ *
+ * For per-CPU statistics, pin the thread to the current cpuid so that
+ * will only access the statistics for that CPU.  If the stats structure
+ * for the current CPU has not been allocated (or previously freed),
+ * allocate it now.  The per-CPU statistics do not need locking since
+ * the thread is pinned to the CPU during update.
+ *
+ * For global statistics, lock the stats structure to prevent concurrent update.
+ *
+ * \param[in] stats	statistics structure to lock
+ * \param[in] opc	type of operation:
+ *			LPROCFS_GET_SMP_ID: "lock" and return current CPU index
+ *				for incrementing statistics for that CPU
+ *			LPROCFS_GET_NUM_CPU: "lock" and return number of used
+ *				CPU indices to iterate over all indices
+ * \param[out] flags	CPU interrupt saved state for IRQ-safe locking
+ *
+ * \retval cpuid of current thread or number of allocated structs
+ * \retval negative on error (only for opc LPROCFS_GET_SMP_ID + per-CPU stats)
  */
-static inline int lprocfs_stats_lock(struct lprocfs_stats *stats, int opc,
+static inline int lprocfs_stats_lock(struct lprocfs_stats *stats,
+				     enum lprocfs_stats_lock_ops opc,
 				     unsigned long *flags)
 {
-	int		rc = 0;
+	if (stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU) {
+		if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
+			spin_lock_irqsave(&stats->ls_lock, *flags);
+		else
+			spin_lock(&stats->ls_lock);
+		return opc == LPROCFS_GET_NUM_CPU ? 1 : 0;
+	}
 
 	switch (opc) {
-	default:
-		LBUG();
+	case LPROCFS_GET_SMP_ID: {
+		unsigned int cpuid = get_cpu();
 
-	case LPROCFS_GET_SMP_ID:
-		if (stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU) {
-			if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
-				spin_lock_irqsave(&stats->ls_lock, *flags);
-			else
-				spin_lock(&stats->ls_lock);
-			return 0;
-		} else {
-			unsigned int cpuid = get_cpu();
-
-			if (unlikely(!stats->ls_percpu[cpuid])) {
-				rc = lprocfs_stats_alloc_one(stats, cpuid);
-				if (rc < 0) {
-					put_cpu();
-					return rc;
-				}
+		if (unlikely(!stats->ls_percpu[cpuid])) {
+			int rc = lprocfs_stats_alloc_one(stats, cpuid);
+
+			if (rc < 0) {
+				put_cpu();
+				return rc;
 			}
-			return cpuid;
 		}
-
+		return cpuid;
+	}
 	case LPROCFS_GET_NUM_CPU:
-		if (stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU) {
-			if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
-				spin_lock_irqsave(&stats->ls_lock, *flags);
-			else
-				spin_lock(&stats->ls_lock);
-			return 1;
-		}
 		return stats->ls_biggest_alloc_num;
+	default:
+		LBUG();
 	}
 }
 
-static inline void lprocfs_stats_unlock(struct lprocfs_stats *stats, int opc,
+/**
+ * Unlock statistics structure after access.
+ *
+ * Unlock the lock acquired via lprocfs_stats_lock() for global statistics,
+ * or unpin this thread from the current cpuid for per-CPU statistics.
+ *
+ * This function must be called using the same arguments as used when calling
+ * lprocfs_stats_lock() so that the correct operation can be performed.
+ *
+ * \param[in] stats	statistics structure to unlock
+ * \param[in] opc	type of operation (current cpuid or number of structs)
+ * \param[in] flags	CPU interrupt saved state for IRQ-safe locking
+ */
+static inline void lprocfs_stats_unlock(struct lprocfs_stats *stats,
+					enum lprocfs_stats_lock_ops opc,
 					unsigned long *flags)
 {
-	switch (opc) {
-	default:
-		LBUG();
-
-	case LPROCFS_GET_SMP_ID:
-		if (stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU) {
-			if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
-				spin_unlock_irqrestore(&stats->ls_lock, *flags);
-			else
-				spin_unlock(&stats->ls_lock);
-		} else {
-			put_cpu();
-		}
-		return;
-
-	case LPROCFS_GET_NUM_CPU:
-		if (stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU) {
-			if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
-				spin_unlock_irqrestore(&stats->ls_lock, *flags);
-			else
-				spin_unlock(&stats->ls_lock);
-		}
-		return;
+	if (stats->ls_flags & LPROCFS_STATS_FLAG_NOPERCPU) {
+		if (stats->ls_flags & LPROCFS_STATS_FLAG_IRQ_SAFE)
+			spin_unlock_irqrestore(&stats->ls_lock, *flags);
+		else
+			spin_unlock(&stats->ls_lock);
+	} else if (opc == LPROCFS_GET_SMP_ID) {
+		put_cpu();
 	}
 }
 
@@ -496,7 +515,7 @@ static inline __u64 lprocfs_stats_collector(struct lprocfs_stats *stats,
 					    int idx,
 					    enum lprocfs_fields_flags field)
 {
-	int	      i;
+	unsigned int i;
 	unsigned int  num_cpu;
 	unsigned long flags	= 0;
 	__u64	      ret	= 0;
@@ -681,6 +700,12 @@ static struct lustre_attr lustre_attr_##name = __ATTR(name, mode, show, store)
 
 extern const struct sysfs_ops lustre_sysfs_ops;
 
+struct root_squash_info;
+int lprocfs_wr_root_squash(const char *buffer, unsigned long count,
+			   struct root_squash_info *squash, char *name);
+int lprocfs_wr_nosquash_nids(const char *buffer, unsigned long count,
+			     struct root_squash_info *squash, char *name);
+
 /* all quota proc functions */
 int lprocfs_quota_rd_bunit(char *page, char **start,
 			   loff_t off, int count,
diff --git a/drivers/staging/lustre/lustre/include/lu_object.h b/drivers/staging/lustre/lustre/include/lu_object.h
index 6e25c1bb6aa3..260643ee0d48 100644
--- a/drivers/staging/lustre/lustre/include/lu_object.h
+++ b/drivers/staging/lustre/lustre/include/lu_object.h
@@ -327,7 +327,7 @@ struct lu_device_type {
 	/**
 	 * Number of existing device type instances.
 	 */
-	unsigned				ldt_device_nr;
+	atomic_t				ldt_device_nr;
 	/**
 	 * Linkage into a global list of all device types.
 	 *
@@ -602,7 +602,7 @@ struct lu_site {
 	/**
 	 * index of bucket on hash table while purging
 	 */
-	int		       ls_purge_start;
+	unsigned int		ls_purge_start;
 	/**
 	 * Top-level device for this stack.
 	 */
@@ -623,6 +623,11 @@ struct lu_site {
 	spinlock_t		ls_ld_lock;
 
 	/**
+	 * Lock to serialize site purge.
+	 */
+	struct mutex		ls_purge_mutex;
+
+	/**
 	 * lu_site stats
 	 */
 	struct lprocfs_stats	*ls_stats;
@@ -673,7 +678,6 @@ void lu_object_add(struct lu_object *before, struct lu_object *o);
 
 int  lu_device_type_init(struct lu_device_type *ldt);
 void lu_device_type_fini(struct lu_device_type *ldt);
-void lu_types_stop(void);
 
 /** @} ctors */
 
@@ -1025,7 +1029,8 @@ enum lu_context_tag {
 	/**
 	 * Contexts usable in cache shrinker thread.
 	 */
-	LCT_SHRINKER  = LCT_MD_THREAD|LCT_DT_THREAD|LCT_CL_THREAD|LCT_NOREF
+	LCT_SHRINKER  = LCT_MD_THREAD | LCT_DT_THREAD | LCT_CL_THREAD |
+			LCT_NOREF
 };
 
 /**
@@ -1264,12 +1269,28 @@ struct lu_name {
 };
 
 /**
+ * Validate names (path components)
+ *
+ * To be valid \a name must be non-empty, '\0' terminated of length \a
+ * name_len, and not contain '/'. The maximum length of a name (before
+ * say -ENAMETOOLONG will be returned) is really controlled by llite
+ * and the server. We only check for something insane coming from bad
+ * integer handling here.
+ */
+static inline bool lu_name_is_valid_2(const char *name, size_t name_len)
+{
+	return name && name_len > 0 && name_len < INT_MAX &&
+	       name[name_len] == '\0' && strlen(name) == name_len &&
+	       !memchr(name, '/', name_len);
+}
+
+/**
  * Common buffer structure to be passed around for various xattr_{s,g}et()
  * methods.
  */
 struct lu_buf {
 	void   *lb_buf;
-	ssize_t lb_len;
+	size_t	lb_len;
 };
 
 #define DLUBUF "(%p %zu)"
@@ -1298,5 +1319,12 @@ struct lu_kmem_descr {
 int  lu_kmem_init(struct lu_kmem_descr *caches);
 void lu_kmem_fini(struct lu_kmem_descr *caches);
 
+void lu_buf_free(struct lu_buf *buf);
+void lu_buf_alloc(struct lu_buf *buf, size_t size);
+void lu_buf_realloc(struct lu_buf *buf, size_t size);
+
+int lu_buf_check_and_grow(struct lu_buf *buf, size_t len);
+struct lu_buf *lu_buf_check_and_alloc(struct lu_buf *buf, size_t len);
+
 /** @} lu */
 #endif /* __LUSTRE_LU_OBJECT_H */
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h b/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h
index 051864c23b5b..72eaee95c6b8 100644
--- a/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h
+++ b/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h
@@ -93,6 +93,7 @@
 /* Defn's shared with user-space. */
 #include "lustre_user.h"
 #include "lustre_errno.h"
+#include "../lustre_ver.h"
 
 /*
  *  GENERAL STUFF
@@ -196,12 +197,12 @@ static inline unsigned fld_range_type(const struct lu_seq_range *range)
 	return range->lsr_flags & LU_SEQ_RANGE_MASK;
 }
 
-static inline int fld_range_is_ost(const struct lu_seq_range *range)
+static inline bool fld_range_is_ost(const struct lu_seq_range *range)
 {
 	return fld_range_type(range) == LU_SEQ_RANGE_OST;
 }
 
-static inline int fld_range_is_mdt(const struct lu_seq_range *range)
+static inline bool fld_range_is_mdt(const struct lu_seq_range *range)
 {
 	return fld_range_type(range) == LU_SEQ_RANGE_MDT;
 }
@@ -260,23 +261,23 @@ static inline void range_init(struct lu_seq_range *range)
  * check if given seq id \a s is within given range \a r
  */
 
-static inline int range_within(const struct lu_seq_range *range,
-			       __u64 s)
+static inline bool range_within(const struct lu_seq_range *range,
+				__u64 s)
 {
 	return s >= range->lsr_start && s < range->lsr_end;
 }
 
-static inline int range_is_sane(const struct lu_seq_range *range)
+static inline bool range_is_sane(const struct lu_seq_range *range)
 {
 	return (range->lsr_end >= range->lsr_start);
 }
 
-static inline int range_is_zero(const struct lu_seq_range *range)
+static inline bool range_is_zero(const struct lu_seq_range *range)
 {
 	return (range->lsr_start == 0 && range->lsr_end == 0);
 }
 
-static inline int range_is_exhausted(const struct lu_seq_range *range)
+static inline bool range_is_exhausted(const struct lu_seq_range *range)
 
 {
 	return range_space(range) == 0;
@@ -437,69 +438,69 @@ enum dot_lustre_oid {
 	FID_OID_DOT_LUSTRE_OBF = 2UL,
 };
 
-static inline int fid_seq_is_mdt0(__u64 seq)
+static inline bool fid_seq_is_mdt0(__u64 seq)
 {
 	return (seq == FID_SEQ_OST_MDT0);
 }
 
-static inline int fid_seq_is_mdt(const __u64 seq)
+static inline bool fid_seq_is_mdt(__u64 seq)
 {
 	return seq == FID_SEQ_OST_MDT0 || seq >= FID_SEQ_NORMAL;
 };
 
-static inline int fid_seq_is_echo(__u64 seq)
+static inline bool fid_seq_is_echo(__u64 seq)
 {
 	return (seq == FID_SEQ_ECHO);
 }
 
-static inline int fid_is_echo(const struct lu_fid *fid)
+static inline bool fid_is_echo(const struct lu_fid *fid)
 {
 	return fid_seq_is_echo(fid_seq(fid));
 }
 
-static inline int fid_seq_is_llog(__u64 seq)
+static inline bool fid_seq_is_llog(__u64 seq)
 {
 	return (seq == FID_SEQ_LLOG);
 }
 
-static inline int fid_is_llog(const struct lu_fid *fid)
+static inline bool fid_is_llog(const struct lu_fid *fid)
 {
 	/* file with OID == 0 is not llog but contains last oid */
 	return fid_seq_is_llog(fid_seq(fid)) && fid_oid(fid) > 0;
 }
 
-static inline int fid_seq_is_rsvd(const __u64 seq)
+static inline bool fid_seq_is_rsvd(__u64 seq)
 {
 	return (seq > FID_SEQ_OST_MDT0 && seq <= FID_SEQ_RSVD);
 };
 
-static inline int fid_seq_is_special(const __u64 seq)
+static inline bool fid_seq_is_special(__u64 seq)
 {
 	return seq == FID_SEQ_SPECIAL;
 };
 
-static inline int fid_seq_is_local_file(const __u64 seq)
+static inline bool fid_seq_is_local_file(__u64 seq)
 {
 	return seq == FID_SEQ_LOCAL_FILE ||
 	       seq == FID_SEQ_LOCAL_NAME;
 };
 
-static inline int fid_seq_is_root(const __u64 seq)
+static inline bool fid_seq_is_root(__u64 seq)
 {
 	return seq == FID_SEQ_ROOT;
 }
 
-static inline int fid_seq_is_dot(const __u64 seq)
+static inline bool fid_seq_is_dot(__u64 seq)
 {
 	return seq == FID_SEQ_DOT_LUSTRE;
 }
 
-static inline int fid_seq_is_default(const __u64 seq)
+static inline bool fid_seq_is_default(__u64 seq)
 {
 	return seq == FID_SEQ_LOV_DEFAULT;
 }
 
-static inline int fid_is_mdt0(const struct lu_fid *fid)
+static inline bool fid_is_mdt0(const struct lu_fid *fid)
 {
 	return fid_seq_is_mdt0(fid_seq(fid));
 }
@@ -516,12 +517,12 @@ static inline void lu_root_fid(struct lu_fid *fid)
  * \param fid the fid to be tested.
  * \return true if the fid is a igif; otherwise false.
  */
-static inline int fid_seq_is_igif(const __u64 seq)
+static inline bool fid_seq_is_igif(__u64 seq)
 {
 	return seq >= FID_SEQ_IGIF && seq <= FID_SEQ_IGIF_MAX;
 }
 
-static inline int fid_is_igif(const struct lu_fid *fid)
+static inline bool fid_is_igif(const struct lu_fid *fid)
 {
 	return fid_seq_is_igif(fid_seq(fid));
 }
@@ -531,27 +532,27 @@ static inline int fid_is_igif(const struct lu_fid *fid)
  * \param fid the fid to be tested.
  * \return true if the fid is a idif; otherwise false.
  */
-static inline int fid_seq_is_idif(const __u64 seq)
+static inline bool fid_seq_is_idif(__u64 seq)
 {
 	return seq >= FID_SEQ_IDIF && seq <= FID_SEQ_IDIF_MAX;
 }
 
-static inline int fid_is_idif(const struct lu_fid *fid)
+static inline bool fid_is_idif(const struct lu_fid *fid)
 {
 	return fid_seq_is_idif(fid_seq(fid));
 }
 
-static inline int fid_is_local_file(const struct lu_fid *fid)
+static inline bool fid_is_local_file(const struct lu_fid *fid)
 {
 	return fid_seq_is_local_file(fid_seq(fid));
 }
 
-static inline int fid_seq_is_norm(const __u64 seq)
+static inline bool fid_seq_is_norm(__u64 seq)
 {
 	return (seq >= FID_SEQ_NORMAL);
 }
 
-static inline int fid_is_norm(const struct lu_fid *fid)
+static inline bool fid_is_norm(const struct lu_fid *fid)
 {
 	return fid_seq_is_norm(fid_seq(fid));
 }
@@ -658,7 +659,7 @@ static inline void ostid_set_id(struct ost_id *oi, __u64 oid)
 		oi->oi_fid.f_oid = oid;
 		oi->oi_fid.f_ver = oid >> 48;
 	} else {
-		if (oid > OBIF_MAX_OID) {
+		if (oid >= OBIF_MAX_OID) {
 			CERROR("Bad %llu to set " DOSTID "\n", oid, POSTID(oi));
 			return;
 		}
@@ -683,7 +684,7 @@ static inline int fid_set_id(struct lu_fid *fid, __u64 oid)
 		fid->f_oid = oid;
 		fid->f_ver = oid >> 48;
 	} else {
-		if (oid > OBIF_MAX_OID) {
+		if (oid >= OBIF_MAX_OID) {
 			CERROR("Too large OID %#llx to set REG "DFID"\n",
 			       (unsigned long long)oid, PFID(fid));
 			return -EBADF;
@@ -769,7 +770,7 @@ static inline int fid_to_ostid(const struct lu_fid *fid, struct ost_id *ostid)
 }
 
 /* Check whether the fid is for LAST_ID */
-static inline int fid_is_last_id(const struct lu_fid *fid)
+static inline bool fid_is_last_id(const struct lu_fid *fid)
 {
 	return (fid_oid(fid) == 0);
 }
@@ -838,7 +839,7 @@ static inline void fid_be_to_cpu(struct lu_fid *dst, const struct lu_fid *src)
 	dst->f_ver = be32_to_cpu(fid_ver(src));
 }
 
-static inline int fid_is_sane(const struct lu_fid *fid)
+static inline bool fid_is_sane(const struct lu_fid *fid)
 {
 	return fid &&
 	       ((fid_seq(fid) >= FID_SEQ_START && fid_ver(fid) == 0) ||
@@ -846,15 +847,10 @@ static inline int fid_is_sane(const struct lu_fid *fid)
 		fid_seq_is_rsvd(fid_seq(fid)));
 }
 
-static inline int fid_is_zero(const struct lu_fid *fid)
-{
-	return fid_seq(fid) == 0 && fid_oid(fid) == 0;
-}
-
 void lustre_swab_lu_fid(struct lu_fid *fid);
 void lustre_swab_lu_seq_range(struct lu_seq_range *range);
 
-static inline int lu_fid_eq(const struct lu_fid *f0, const struct lu_fid *f1)
+static inline bool lu_fid_eq(const struct lu_fid *f0, const struct lu_fid *f1)
 {
 	return memcmp(f0, f1, sizeof(*f0)) == 0;
 }
@@ -1017,12 +1013,12 @@ static inline struct lu_dirent *lu_dirent_next(struct lu_dirent *ent)
 	return next;
 }
 
-static inline int lu_dirent_calc_size(int namelen, __u16 attr)
+static inline size_t lu_dirent_calc_size(size_t namelen, __u16 attr)
 {
-	int size;
+	size_t size;
 
 	if (attr & LUDA_TYPE) {
-		const unsigned align = sizeof(struct luda_type) - 1;
+		const size_t align = sizeof(struct luda_type) - 1;
 
 		size = (sizeof(struct lu_dirent) + namelen + align) & ~align;
 		size += sizeof(struct luda_type);
@@ -1033,15 +1029,6 @@ static inline int lu_dirent_calc_size(int namelen, __u16 attr)
 	return (size + 7) & ~7;
 }
 
-static inline int lu_dirent_size(struct lu_dirent *ent)
-{
-	if (le16_to_cpu(ent->lde_reclen) == 0) {
-		return lu_dirent_calc_size(le16_to_cpu(ent->lde_namelen),
-					   le32_to_cpu(ent->lde_attrs));
-	}
-	return le16_to_cpu(ent->lde_reclen);
-}
-
 #define MDS_DIR_END_OFF 0xfffffffffffffffeULL
 
 /**
@@ -1067,19 +1054,19 @@ struct lustre_handle {
 
 #define DEAD_HANDLE_MAGIC 0xdeadbeefcafebabeULL
 
-static inline int lustre_handle_is_used(struct lustre_handle *lh)
+static inline bool lustre_handle_is_used(const struct lustre_handle *lh)
 {
 	return lh->cookie != 0ull;
 }
 
-static inline int lustre_handle_equal(const struct lustre_handle *lh1,
-				      const struct lustre_handle *lh2)
+static inline bool lustre_handle_equal(const struct lustre_handle *lh1,
+				       const struct lustre_handle *lh2)
 {
 	return lh1->cookie == lh2->cookie;
 }
 
 static inline void lustre_handle_copy(struct lustre_handle *tgt,
-				      struct lustre_handle *src)
+				      const struct lustre_handle *src)
 {
 	tgt->cookie = src->cookie;
 }
@@ -1105,7 +1092,7 @@ struct lustre_msg_v2 {
 
 /* without gss, ptlrpc_body is put at the first buffer. */
 #define PTLRPC_NUM_VERSIONS     4
-#define JOBSTATS_JOBID_SIZE     32  /* 32 bytes string */
+
 struct ptlrpc_body_v3 {
 	struct lustre_handle pb_handle;
 	__u32 pb_type;
@@ -1127,7 +1114,7 @@ struct ptlrpc_body_v3 {
 	__u64 pb_pre_versions[PTLRPC_NUM_VERSIONS];
 	/* padding for future needs */
 	__u64 pb_padding[4];
-	char  pb_jobid[JOBSTATS_JOBID_SIZE];
+	char  pb_jobid[LUSTRE_JOBID_SIZE];
 };
 
 #define ptlrpc_body     ptlrpc_body_v3
@@ -1293,6 +1280,9 @@ void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb);
 #define OBD_CONNECT_OPEN_BY_FID	0x20000000000000ULL	/* open by fid won't pack
 							 * name in request
 							 */
+#define OBD_CONNECT_LFSCK	0x40000000000000ULL/* support online LFSCK */
+#define OBD_CONNECT_UNLINK_CLOSE 0x100000000000000ULL/* close file in unlink */
+#define OBD_CONNECT_DIR_STRIPE	 0x400000000000000ULL/* striped DNE dir */
 
 /* XXX README XXX:
  * Please DO NOT add flag values here before first ensuring that this same
@@ -1318,14 +1308,6 @@ void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb);
 #define CLIENT_CONNECT_MDT_REQD (OBD_CONNECT_IBITS | OBD_CONNECT_FID | \
 				 OBD_CONNECT_FULL20)
 
-#define OBD_OCD_VERSION(major, minor, patch, fix) (((major)<<24) + \
-						  ((minor)<<16) + \
-						  ((patch)<<8) + (fix))
-#define OBD_OCD_VERSION_MAJOR(version) ((int)((version)>>24)&255)
-#define OBD_OCD_VERSION_MINOR(version) ((int)((version)>>16)&255)
-#define OBD_OCD_VERSION_PATCH(version) ((int)((version)>>8)&255)
-#define OBD_OCD_VERSION_FIX(version)   ((int)(version)&255)
-
 /* This structure is used for both request and reply.
  *
  * If we eventually have separate connect data for different types, which we
@@ -1478,10 +1460,23 @@ enum obdo_flags {
 	OBD_FL_LOCAL_MASK   = 0xF0000000,
 };
 
-#define LOV_MAGIC_V1      0x0BD10BD0
-#define LOV_MAGIC	 LOV_MAGIC_V1
-#define LOV_MAGIC_JOIN_V1 0x0BD20BD0
-#define LOV_MAGIC_V3      0x0BD30BD0
+/*
+ * All LOV EA magics should have the same postfix, if some new version
+ * Lustre instroduces new LOV EA magic, then when down-grade to an old
+ * Lustre, even though the old version system does not recognizes such
+ * new magic, it still can distinguish the corrupted cases by checking
+ * the magic's postfix.
+ */
+#define LOV_MAGIC_MAGIC 0x0BD0
+#define LOV_MAGIC_MASK  0xFFFF
+
+#define LOV_MAGIC_V1		(0x0BD10000 | LOV_MAGIC_MAGIC)
+#define LOV_MAGIC_JOIN_V1	(0x0BD20000 | LOV_MAGIC_MAGIC)
+#define LOV_MAGIC_V3		(0x0BD30000 | LOV_MAGIC_MAGIC)
+#define LOV_MAGIC_MIGRATE	(0x0BD40000 | LOV_MAGIC_MAGIC)
+/* reserved for specifying OSTs */
+#define LOV_MAGIC_SPECIFIC	(0x0BD50000 | LOV_MAGIC_MAGIC)
+#define LOV_MAGIC		LOV_MAGIC_V1
 
 /*
  * magic for fully defined striping
@@ -1498,14 +1493,6 @@ enum obdo_flags {
 #define LOV_MAGIC_V1_DEF  0x0CD10BD0
 #define LOV_MAGIC_V3_DEF  0x0CD30BD0
 
-#define LOV_PATTERN_RAID0	0x001   /* stripes are used round-robin */
-#define LOV_PATTERN_RAID1	0x002   /* stripes are mirrors of each other */
-#define LOV_PATTERN_FIRST	0x100   /* first stripe is not in round-robin */
-#define LOV_PATTERN_CMOBD	0x200
-
-#define LOV_PATTERN_F_MASK	0xffff0000
-#define LOV_PATTERN_F_RELEASED	0x80000000 /* HSM released file */
-
 #define lov_pattern(pattern)		(pattern & ~LOV_PATTERN_F_MASK)
 #define lov_pattern_flags(pattern)	(pattern & LOV_PATTERN_F_MASK)
 
@@ -1569,25 +1556,25 @@ static inline void lmm_oi_set_id(struct ost_id *oi, __u64 oid)
 	oi->oi.oi_id = oid;
 }
 
-static inline __u64 lmm_oi_id(struct ost_id *oi)
+static inline __u64 lmm_oi_id(const struct ost_id *oi)
 {
 	return oi->oi.oi_id;
 }
 
-static inline __u64 lmm_oi_seq(struct ost_id *oi)
+static inline __u64 lmm_oi_seq(const struct ost_id *oi)
 {
 	return oi->oi.oi_seq;
 }
 
 static inline void lmm_oi_le_to_cpu(struct ost_id *dst_oi,
-				    struct ost_id *src_oi)
+				    const struct ost_id *src_oi)
 {
 	dst_oi->oi.oi_id = le64_to_cpu(src_oi->oi.oi_id);
 	dst_oi->oi.oi_seq = le64_to_cpu(src_oi->oi.oi_seq);
 }
 
 static inline void lmm_oi_cpu_to_le(struct ost_id *dst_oi,
-				    struct ost_id *src_oi)
+				    const struct ost_id *src_oi)
 {
 	dst_oi->oi.oi_id = cpu_to_le64(src_oi->oi.oi_id);
 	dst_oi->oi.oi_seq = cpu_to_le64(src_oi->oi.oi_seq);
@@ -1610,6 +1597,7 @@ static inline void lmm_oi_cpu_to_le(struct ost_id *dst_oi,
 #define XATTR_NAME_LOV	  "trusted.lov"
 #define XATTR_NAME_LMA	  "trusted.lma"
 #define XATTR_NAME_LMV	  "trusted.lmv"
+#define XATTR_NAME_DEFAULT_LMV	"trusted.dmv"
 #define XATTR_NAME_LINK	 "trusted.link"
 #define XATTR_NAME_FID	  "trusted.fid"
 #define XATTR_NAME_VERSION      "trusted.version"
@@ -1625,7 +1613,7 @@ struct lov_mds_md_v3 {	    /* LOV EA mds/wire data (little-endian) */
 	/* lmm_stripe_count used to be __u32 */
 	__u16 lmm_stripe_count;   /* num stripes in use for this object */
 	__u16 lmm_layout_gen;     /* layout generation number */
-	char  lmm_pool_name[LOV_MAXPOOLNAME]; /* must be 32bit aligned */
+	char  lmm_pool_name[LOV_MAXPOOLNAME + 1]; /* must be 32bit aligned */
 	struct lov_ost_data_v1 lmm_objects[0]; /* per-stripe data */
 };
 
@@ -1727,6 +1715,8 @@ lov_mds_md_max_stripe_count(size_t buf_size, __u32 lmm_magic)
 #define OBD_MD_FLDATAVERSION (0x0010000000000000ULL) /* iversion sum */
 #define OBD_MD_FLRELEASED    (0x0020000000000000ULL) /* file released */
 
+#define OBD_MD_DEFAULT_MEA   (0x0040000000000000ULL) /* default MEA */
+
 #define OBD_MD_FLGETATTR (OBD_MD_FLID    | OBD_MD_FLATIME | OBD_MD_FLMTIME | \
 			  OBD_MD_FLCTIME | OBD_MD_FLSIZE  | OBD_MD_FLBLKSZ | \
 			  OBD_MD_FLMODE  | OBD_MD_FLTYPE  | OBD_MD_FLUID   | \
@@ -1782,7 +1772,7 @@ void lustre_swab_obd_statfs(struct obd_statfs *os);
 				      * it to sync quickly
 				      */
 
-#define OBD_OBJECT_EOF 0xffffffffffffffffULL
+#define OBD_OBJECT_EOF	LUSTRE_EOF
 
 #define OST_MIN_PRECREATE 32
 #define OST_MAX_PRECREATE 20000
@@ -1806,9 +1796,9 @@ void lustre_swab_obd_ioobj(struct obd_ioobj *ioo);
 
 /* multiple of 8 bytes => can array */
 struct niobuf_remote {
-	__u64 offset;
-	__u32 len;
-	__u32 flags;
+	__u64	rnb_offset;
+	__u32	rnb_len;
+	__u32	rnb_flags;
 };
 
 void lustre_swab_niobuf_remote(struct niobuf_remote *nbr);
@@ -1878,12 +1868,6 @@ struct obd_quotactl {
 
 void lustre_swab_obd_quotactl(struct obd_quotactl *q);
 
-#define Q_QUOTACHECK	0x800100 /* deprecated as of 2.4 */
-#define Q_INITQUOTA	0x800101 /* deprecated as of 2.4  */
-#define Q_GETOINFO	0x800102 /* get obd quota info */
-#define Q_GETOQUOTA	0x800103 /* get obd quotas */
-#define Q_FINVALIDATE	0x800104 /* deprecated as of 2.4 */
-
 #define Q_COPY(out, in, member) (out)->member = (in)->member
 
 #define QCTL_COPY(out, in)		\
@@ -1946,8 +1930,8 @@ enum mds_cmd {
 	MDS_DISCONNECT		= 39,
 	MDS_GETSTATUS		= 40,
 	MDS_STATFS		= 41,
-	MDS_PIN			= 42,
-	MDS_UNPIN		= 43,
+	MDS_PIN			= 42, /* obsolete, never used in a release */
+	MDS_UNPIN		= 43, /* obsolete, never used in a release */
 	MDS_SYNC		= 44,
 	MDS_DONE_WRITING	= 45,
 	MDS_SET_INFO		= 46,
@@ -1956,7 +1940,7 @@ enum mds_cmd {
 	MDS_GETXATTR		= 49,
 	MDS_SETXATTR		= 50, /* obsolete, now it's MDS_REINT op */
 	MDS_WRITEPAGE		= 51,
-	MDS_IS_SUBDIR		= 52,
+	MDS_IS_SUBDIR		= 52, /* obsolete, never used in a release */
 	MDS_GET_INFO		= 53,
 	MDS_HSM_STATE_GET	= 54,
 	MDS_HSM_STATE_SET	= 55,
@@ -1984,7 +1968,7 @@ enum mdt_reint_cmd {
 	REINT_OPEN     = 6,
 	REINT_SETXATTR = 7,
 	REINT_RMENTRY  = 8,
-/*      REINT_WRITE    = 9, */
+	REINT_MIGRATE  = 9,
 	REINT_MAX
 };
 
@@ -2003,6 +1987,7 @@ void lustre_swab_generic_32s(__u32 *val);
 #define DISP_OPEN_LOCK       0x02000000
 #define DISP_OPEN_LEASE      0x04000000
 #define DISP_OPEN_STRIPE     0x08000000
+#define DISP_OPEN_DENY		0x10000000
 
 /* INODE LOCK PARTS */
 #define MDS_INODELOCK_LOOKUP 0x000001	/* For namespace, dentry etc, and also
@@ -2028,7 +2013,7 @@ void lustre_swab_generic_32s(__u32 *val);
 
 #define MDS_INODELOCK_MAXSHIFT 5
 /* This FULL lock is useful to take on unlink sort of operations */
-#define MDS_INODELOCK_FULL ((1<<(MDS_INODELOCK_MAXSHIFT+1))-1)
+#define MDS_INODELOCK_FULL ((1 << (MDS_INODELOCK_MAXSHIFT + 1)) - 1)
 
 /* NOTE: until Lustre 1.8.7/2.1.1 the fid_ver() was packed into name[2],
  * but was moved into name[1] along with the OID to avoid consuming the
@@ -2108,43 +2093,43 @@ enum md_transient_state {
 };
 
 struct mdt_body {
-	struct lu_fid  fid1;
-	struct lu_fid  fid2;
-	struct lustre_handle handle;
-	__u64	  valid;
-	__u64	  size;   /* Offset, in the case of MDS_READPAGE */
-	__s64	  mtime;
-	__s64	  atime;
-	__s64	  ctime;
-	__u64	  blocks; /* XID, in the case of MDS_READPAGE */
-	__u64	  ioepoch;
-	__u64	  t_state; /* transient file state defined in
-			    * enum md_transient_state
-			    * was "ino" until 2.4.0
-			    */
-	__u32	  fsuid;
-	__u32	  fsgid;
-	__u32	  capability;
-	__u32	  mode;
-	__u32	  uid;
-	__u32	  gid;
-	__u32	  flags; /* from vfs for pin/unpin, LUSTRE_BFLAG close */
-	__u32	  rdev;
-	__u32	  nlink; /* #bytes to read in the case of MDS_READPAGE */
-	__u32	  unused2; /* was "generation" until 2.4.0 */
-	__u32	  suppgid;
-	__u32	  eadatasize;
-	__u32	  aclsize;
-	__u32	  max_mdsize;
-	__u32	  max_cookiesize;
-	__u32	  uid_h; /* high 32-bits of uid, for FUID */
-	__u32	  gid_h; /* high 32-bits of gid, for FUID */
-	__u32	  padding_5; /* also fix lustre_swab_mdt_body */
-	__u64	  padding_6;
-	__u64	  padding_7;
-	__u64	  padding_8;
-	__u64	  padding_9;
-	__u64	  padding_10;
+	struct lu_fid mbo_fid1;
+	struct lu_fid mbo_fid2;
+	struct lustre_handle mbo_handle;
+	__u64	mbo_valid;
+	__u64	mbo_size;	/* Offset, in the case of MDS_READPAGE */
+	__s64	mbo_mtime;
+	__s64	mbo_atime;
+	__s64	mbo_ctime;
+	__u64	mbo_blocks;	/* XID, in the case of MDS_READPAGE */
+	__u64	mbo_ioepoch;
+	__u64	mbo_t_state;	/* transient file state defined in
+				 * enum md_transient_state
+				 * was "ino" until 2.4.0
+				 */
+	__u32	mbo_fsuid;
+	__u32	mbo_fsgid;
+	__u32	mbo_capability;
+	__u32	mbo_mode;
+	__u32	mbo_uid;
+	__u32	mbo_gid;
+	__u32	mbo_flags;
+	__u32	mbo_rdev;
+	__u32	mbo_nlink;	/* #bytes to read in the case of MDS_READPAGE */
+	__u32	mbo_unused2;	/* was "generation" until 2.4.0 */
+	__u32	mbo_suppgid;
+	__u32	mbo_eadatasize;
+	__u32	mbo_aclsize;
+	__u32	mbo_max_mdsize;
+	__u32	mbo_max_cookiesize;
+	__u32	mbo_uid_h;	/* high 32-bits of uid, for FUID */
+	__u32	mbo_gid_h;	/* high 32-bits of gid, for FUID */
+	__u32	mbo_padding_5;	/* also fix lustre_swab_mdt_body */
+	__u64	mbo_padding_6;
+	__u64	mbo_padding_7;
+	__u64	mbo_padding_8;
+	__u64	mbo_padding_9;
+	__u64	mbo_padding_10;
 }; /* 216 */
 
 void lustre_swab_mdt_body(struct mdt_body *b);
@@ -2263,6 +2248,11 @@ void lustre_swab_mdt_rec_setattr(struct mdt_rec_setattr *sa);
 					      */
 #define MDS_OPEN_RELEASE   02000000000000ULL /* Open the file for HSM release */
 
+#define MDS_OPEN_FL_INTERNAL (MDS_OPEN_HAS_EA | MDS_OPEN_HAS_OBJS |	\
+			      MDS_OPEN_OWNEROVERRIDE | MDS_OPEN_LOCK |	\
+			      MDS_OPEN_BY_FID | MDS_OPEN_LEASE |	\
+			      MDS_OPEN_RELEASE)
+
 enum mds_op_bias {
 	MDS_CHECK_SPLIT		= 1 << 0,
 	MDS_CROSS_REF		= 1 << 1,
@@ -2277,6 +2267,7 @@ enum mds_op_bias {
 	MDS_CREATE_VOLATILE	= 1 << 10,
 	MDS_OWNEROVERRIDE	= 1 << 11,
 	MDS_HSM_RELEASE		= 1 << 12,
+	MDS_RENAME_MIGRATE	= BIT(13),
 };
 
 /* instance of mdt_reint_rec */
@@ -2472,7 +2463,7 @@ struct lmv_desc {
 	__u32 ld_tgt_count;		/* how many MDS's */
 	__u32 ld_active_tgt_count;	 /* how many active */
 	__u32 ld_default_stripe_count;     /* how many objects are used */
-	__u32 ld_pattern;		  /* default MEA_MAGIC_* */
+	__u32 ld_pattern;		  /* default hash pattern */
 	__u64 ld_default_hash_size;
 	__u64 ld_padding_1;		/* also fix lustre_swab_lmv_desc */
 	__u32 ld_padding_2;		/* also fix lustre_swab_lmv_desc */
@@ -2482,23 +2473,129 @@ struct lmv_desc {
 	struct obd_uuid ld_uuid;
 };
 
-/* TODO: lmv_stripe_md should contain mds capabilities for all slave fids */
-struct lmv_stripe_md {
-	__u32	 mea_magic;
-	__u32	 mea_count;
-	__u32	 mea_master;
-	__u32	 mea_padding;
-	char	  mea_pool_name[LOV_MAXPOOLNAME];
-	struct lu_fid mea_ids[0];
+/* LMV layout EA, and it will be stored both in master and slave object */
+struct lmv_mds_md_v1 {
+	__u32 lmv_magic;
+	__u32 lmv_stripe_count;
+	__u32 lmv_master_mdt_index;	/* On master object, it is master
+					 * MDT index, on slave object, it
+					 * is stripe index of the slave obj
+					 */
+	__u32 lmv_hash_type;		/* dir stripe policy, i.e. indicate
+					 * which hash function to be used,
+					 * Note: only lower 16 bits is being
+					 * used for now. Higher 16 bits will
+					 * be used to mark the object status,
+					 * for example migrating or dead.
+					 */
+	__u32 lmv_layout_version;	/* Used for directory restriping */
+	__u32 lmv_padding1;
+	__u64 lmv_padding2;
+	__u64 lmv_padding3;
+	char lmv_pool_name[LOV_MAXPOOLNAME + 1];/* pool name */
+	struct lu_fid lmv_stripe_fids[0];	/* FIDs for each stripe */
 };
 
-#define MEA_MAGIC_LAST_CHAR      0xb2221ca1
-#define MEA_MAGIC_ALL_CHARS      0xb222a11c
-#define MEA_MAGIC_HASH_SEGMENT   0xb222a11b
+#define LMV_MAGIC_V1	 0x0CD20CD0	/* normal stripe lmv magic */
+#define LMV_MAGIC	 LMV_MAGIC_V1
+
+/* #define LMV_USER_MAGIC 0x0CD30CD0 */
+#define LMV_MAGIC_STRIPE 0x0CD40CD0	/* magic for dir sub_stripe */
+
+/*
+ *Right now only the lower part(0-16bits) of lmv_hash_type is being used,
+ * and the higher part will be the flag to indicate the status of object,
+ * for example the object is being migrated. And the hash function
+ * might be interpreted differently with different flags.
+ */
+#define LMV_HASH_TYPE_MASK		0x0000ffff
+
+#define LMV_HASH_FLAG_MIGRATION		0x80000000
+#define LMV_HASH_FLAG_DEAD		0x40000000
 
-#define MAX_HASH_SIZE_32	 0x7fffffffUL
-#define MAX_HASH_SIZE	    0x7fffffffffffffffULL
-#define MAX_HASH_HIGHEST_BIT     0x1000000000000000ULL
+/**
+ * The FNV-1a hash algorithm is as follows:
+ *     hash = FNV_offset_basis
+ *     for each octet_of_data to be hashed
+ *             hash = hash XOR octet_of_data
+ *             hash = hash × FNV_prime
+ *     return hash
+ * http://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash
+ *
+ * http://www.isthe.com/chongo/tech/comp/fnv/index.html#FNV-reference-source
+ * FNV_prime is 2^40 + 2^8 + 0xb3 = 0x100000001b3ULL
+ **/
+#define LUSTRE_FNV_1A_64_PRIME		0x100000001b3ULL
+#define LUSTRE_FNV_1A_64_OFFSET_BIAS	0xcbf29ce484222325ULL
+static inline __u64 lustre_hash_fnv_1a_64(const void *buf, size_t size)
+{
+	__u64 hash = LUSTRE_FNV_1A_64_OFFSET_BIAS;
+	const unsigned char *p = buf;
+	size_t i;
+
+	for (i = 0; i < size; i++) {
+		hash ^= p[i];
+		hash *= LUSTRE_FNV_1A_64_PRIME;
+	}
+
+	return hash;
+}
+
+union lmv_mds_md {
+	__u32			lmv_magic;
+	struct lmv_mds_md_v1	lmv_md_v1;
+	struct lmv_user_md	lmv_user_md;
+};
+
+void lustre_swab_lmv_mds_md(union lmv_mds_md *lmm);
+
+static inline ssize_t lmv_mds_md_size(int stripe_count, unsigned int lmm_magic)
+{
+	ssize_t len = -EINVAL;
+
+	switch (lmm_magic) {
+	case LMV_MAGIC_V1: {
+		struct lmv_mds_md_v1 *lmm1;
+
+		len = sizeof(*lmm1);
+		len += stripe_count * sizeof(lmm1->lmv_stripe_fids[0]);
+		break; }
+	default:
+		break;
+	}
+	return len;
+}
+
+static inline int lmv_mds_md_stripe_count_get(const union lmv_mds_md *lmm)
+{
+	switch (le32_to_cpu(lmm->lmv_magic)) {
+	case LMV_MAGIC_V1:
+		return le32_to_cpu(lmm->lmv_md_v1.lmv_stripe_count);
+	case LMV_USER_MAGIC:
+		return le32_to_cpu(lmm->lmv_user_md.lum_stripe_count);
+	default:
+		return -EINVAL;
+	}
+}
+
+static inline int lmv_mds_md_stripe_count_set(union lmv_mds_md *lmm,
+					      unsigned int stripe_count)
+{
+	int rc = 0;
+
+	switch (le32_to_cpu(lmm->lmv_magic)) {
+	case LMV_MAGIC_V1:
+		lmm->lmv_md_v1.lmv_stripe_count = cpu_to_le32(stripe_count);
+		break;
+	case LMV_USER_MAGIC:
+		lmm->lmv_user_md.lum_stripe_count = cpu_to_le32(stripe_count);
+		break;
+	default:
+		rc = -EINVAL;
+		break;
+	}
+	return rc;
+}
 
 enum fld_rpc_opc {
 	FLD_QUERY	= 900,
@@ -2582,8 +2679,8 @@ struct ldlm_res_id {
 #define PLDLMRES(res)	(res)->lr_name.name[0], (res)->lr_name.name[1], \
 			(res)->lr_name.name[2], (res)->lr_name.name[3]
 
-static inline int ldlm_res_eq(const struct ldlm_res_id *res0,
-			      const struct ldlm_res_id *res1)
+static inline bool ldlm_res_eq(const struct ldlm_res_id *res0,
+			       const struct ldlm_res_id *res1)
 {
 	return !memcmp(res0, res1, sizeof(*res0));
 }
@@ -2620,17 +2717,15 @@ struct ldlm_extent {
 	__u64 gid;
 };
 
-#define LDLM_GID_ANY ((__u64)-1)
-
-static inline int ldlm_extent_overlap(struct ldlm_extent *ex1,
-				      struct ldlm_extent *ex2)
+static inline int ldlm_extent_overlap(const struct ldlm_extent *ex1,
+				      const struct ldlm_extent *ex2)
 {
 	return (ex1->start <= ex2->end) && (ex2->start <= ex1->end);
 }
 
 /* check if @ex1 contains @ex2 */
-static inline int ldlm_extent_contain(struct ldlm_extent *ex1,
-				      struct ldlm_extent *ex2)
+static inline int ldlm_extent_contain(const struct ldlm_extent *ex1,
+				      const struct ldlm_extent *ex2)
 {
 	return (ex1->start <= ex2->start) && (ex1->end >= ex2->end);
 }
@@ -2833,7 +2928,29 @@ enum obd_cmd {
 };
 #define OBD_FIRST_OPC OBD_PING
 
-/* catalog of log objects */
+/**
+ * llog contexts indices.
+ *
+ * There is compatibility problem with indexes below, they are not
+ * continuous and must keep their numbers for compatibility needs.
+ * See LU-5218 for details.
+ */
+enum llog_ctxt_id {
+	LLOG_CONFIG_ORIG_CTXT  =  0,
+	LLOG_CONFIG_REPL_CTXT = 1,
+	LLOG_MDS_OST_ORIG_CTXT = 2,
+	LLOG_MDS_OST_REPL_CTXT = 3, /* kept just to avoid re-assignment */
+	LLOG_SIZE_ORIG_CTXT = 4,
+	LLOG_SIZE_REPL_CTXT = 5,
+	LLOG_TEST_ORIG_CTXT = 8,
+	LLOG_TEST_REPL_CTXT = 9, /* kept just to avoid re-assignment */
+	LLOG_CHANGELOG_ORIG_CTXT = 12, /**< changelog generation on mdd */
+	LLOG_CHANGELOG_REPL_CTXT = 13, /**< changelog access on clients */
+	/* for multiple changelog consumers */
+	LLOG_CHANGELOG_USER_ORIG_CTXT = 14,
+	LLOG_AGENT_ORIG_CTXT = 15, /**< agent requests generation on cdt */
+	LLOG_MAX_CTXTS
+};
 
 /** Identifier for a single log object */
 struct llog_logid {
@@ -2939,7 +3056,7 @@ struct llog_setattr64_rec {
 	__u32			lsr_uid_h;
 	__u32			lsr_gid;
 	__u32			lsr_gid_h;
-	__u64			lsr_padding;
+	__u64			lsr_valid;
 	struct llog_rec_tail    lsr_tail;
 } __packed;
 
@@ -2963,15 +3080,9 @@ struct changelog_setinfo {
 
 /** changelog record */
 struct llog_changelog_rec {
-	struct llog_rec_hdr  cr_hdr;
-	struct changelog_rec cr;
-	struct llog_rec_tail cr_tail; /**< for_sizezof_only */
-} __packed;
-
-struct llog_changelog_ext_rec {
-	struct llog_rec_hdr      cr_hdr;
-	struct changelog_ext_rec cr;
-	struct llog_rec_tail     cr_tail; /**< for_sizezof_only */
+	struct llog_rec_hdr	cr_hdr;
+	struct changelog_rec	cr;		/**< Variable length field */
+	struct llog_rec_tail	cr_do_not_use;	/**< for_sizezof_only */
 } __packed;
 
 struct llog_changelog_user_rec {
@@ -2990,7 +3101,7 @@ enum agent_req_status {
 	ARS_SUCCEED,
 };
 
-static inline char *agent_req_status2name(enum agent_req_status ars)
+static inline const char *agent_req_status2name(const enum agent_req_status ars)
 {
 	switch (ars) {
 	case ARS_WAITING:
@@ -3056,6 +3167,9 @@ enum llog_flag {
 	LLOG_F_ZAP_WHEN_EMPTY	= 0x1,
 	LLOG_F_IS_CAT		= 0x2,
 	LLOG_F_IS_PLAIN		= 0x4,
+	LLOG_F_EXT_JOBID        = BIT(3),
+
+	LLOG_F_EXT_MASK = LLOG_F_EXT_JOBID,
 };
 
 struct llog_log_hdr {
@@ -3068,8 +3182,8 @@ struct llog_log_hdr {
 	__u32		   llh_cat_idx;
 	/* for a catalog the first plain slot is next to it */
 	struct obd_uuid	 llh_tgtuuid;
-	__u32		   llh_reserved[LLOG_HEADER_SIZE/sizeof(__u32) - 23];
-	__u32		   llh_bitmap[LLOG_BITMAP_BYTES/sizeof(__u32)];
+	__u32		   llh_reserved[LLOG_HEADER_SIZE / sizeof(__u32) - 23];
+	__u32		   llh_bitmap[LLOG_BITMAP_BYTES / sizeof(__u32)];
 	struct llog_rec_tail    llh_tail;
 } __packed;
 
@@ -3166,7 +3280,7 @@ struct obdo {
 #define o_cksum   o_nlink
 #define o_grant_used o_data_version
 
-static inline void lustre_set_wire_obdo(struct obd_connect_data *ocd,
+static inline void lustre_set_wire_obdo(const struct obd_connect_data *ocd,
 					struct obdo *wobdo,
 					const struct obdo *lobdo)
 {
@@ -3185,7 +3299,7 @@ static inline void lustre_set_wire_obdo(struct obd_connect_data *ocd,
 	}
 }
 
-static inline void lustre_get_wire_obdo(struct obd_connect_data *ocd,
+static inline void lustre_get_wire_obdo(const struct obd_connect_data *ocd,
 					struct obdo *lobdo,
 					const struct obdo *wobdo)
 {
@@ -3284,17 +3398,17 @@ void lustre_swab_lustre_capa(struct lustre_capa *c);
 
 /** lustre_capa::lc_opc */
 enum {
-	CAPA_OPC_BODY_WRITE   = 1<<0,  /**< write object data */
-	CAPA_OPC_BODY_READ    = 1<<1,  /**< read object data */
-	CAPA_OPC_INDEX_LOOKUP = 1<<2,  /**< lookup object fid */
-	CAPA_OPC_INDEX_INSERT = 1<<3,  /**< insert object fid */
-	CAPA_OPC_INDEX_DELETE = 1<<4,  /**< delete object fid */
-	CAPA_OPC_OSS_WRITE    = 1<<5,  /**< write oss object data */
-	CAPA_OPC_OSS_READ     = 1<<6,  /**< read oss object data */
-	CAPA_OPC_OSS_TRUNC    = 1<<7,  /**< truncate oss object */
-	CAPA_OPC_OSS_DESTROY  = 1<<8,  /**< destroy oss object */
-	CAPA_OPC_META_WRITE   = 1<<9,  /**< write object meta data */
-	CAPA_OPC_META_READ    = 1<<10, /**< read object meta data */
+	CAPA_OPC_BODY_WRITE   = 1 << 0,  /**< write object data */
+	CAPA_OPC_BODY_READ    = 1 << 1,  /**< read object data */
+	CAPA_OPC_INDEX_LOOKUP = 1 << 2,  /**< lookup object fid */
+	CAPA_OPC_INDEX_INSERT = 1 << 3,  /**< insert object fid */
+	CAPA_OPC_INDEX_DELETE = 1 << 4,  /**< delete object fid */
+	CAPA_OPC_OSS_WRITE    = 1 << 5,  /**< write oss object data */
+	CAPA_OPC_OSS_READ     = 1 << 6,  /**< read oss object data */
+	CAPA_OPC_OSS_TRUNC    = 1 << 7,  /**< truncate oss object */
+	CAPA_OPC_OSS_DESTROY  = 1 << 8,  /**< destroy oss object */
+	CAPA_OPC_META_WRITE   = 1 << 9,  /**< write object meta data */
+	CAPA_OPC_META_READ    = 1 << 10, /**< read object meta data */
 };
 
 #define CAPA_OPC_OSS_RW (CAPA_OPC_OSS_READ | CAPA_OPC_OSS_WRITE)
@@ -3346,6 +3460,14 @@ struct getinfo_fid2path {
 
 void lustre_swab_fid2path(struct getinfo_fid2path *gf);
 
+/** path2parent request/reply structures */
+struct getparent {
+	struct lu_fid	gp_fid;		/**< parent FID */
+	__u32		gp_linkno;	/**< hardlink number */
+	__u32		gp_name_size;	/**< size of the name field */
+	char		gp_name[0];	/**< zero-terminated link name */
+} __packed;
+
 enum {
 	LAYOUT_INTENT_ACCESS    = 0,
 	LAYOUT_INTENT_READ      = 1,
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_ioctl.h b/drivers/staging/lustre/lustre/include/lustre/lustre_ioctl.h
new file mode 100644
index 000000000000..f3d7c94c3b50
--- /dev/null
+++ b/drivers/staging/lustre/lustre/include/lustre/lustre_ioctl.h
@@ -0,0 +1,412 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2011, 2015, Intel Corporation.
+ */
+#ifndef LUSTRE_IOCTL_H_
+#define LUSTRE_IOCTL_H_
+
+#include <linux/types.h>
+#include "../../../include/linux/libcfs/libcfs.h"
+#include "lustre_idl.h"
+
+#ifdef __KERNEL__
+# include <linux/ioctl.h>
+# include <linux/string.h>
+# include "../obd_support.h"
+#else /* __KERNEL__ */
+# include <malloc.h>
+# include <string.h>
+#include <libcfs/util/ioctl.h>
+#endif /* !__KERNEL__ */
+
+#if !defined(__KERNEL__) && !defined(LUSTRE_UTILS)
+# error This file is for Lustre internal use only.
+#endif
+
+enum md_echo_cmd {
+	ECHO_MD_CREATE		= 1, /* Open/Create file on MDT */
+	ECHO_MD_MKDIR		= 2, /* Mkdir on MDT */
+	ECHO_MD_DESTROY		= 3, /* Unlink file on MDT */
+	ECHO_MD_RMDIR		= 4, /* Rmdir on MDT */
+	ECHO_MD_LOOKUP		= 5, /* Lookup on MDT */
+	ECHO_MD_GETATTR		= 6, /* Getattr on MDT */
+	ECHO_MD_SETATTR		= 7, /* Setattr on MDT */
+	ECHO_MD_ALLOC_FID	= 8, /* Get FIDs from MDT */
+};
+
+#define OBD_DEV_ID 1
+#define OBD_DEV_NAME "obd"
+#define OBD_DEV_PATH "/dev/" OBD_DEV_NAME
+#define OBD_DEV_MAJOR 10
+#define OBD_DEV_MINOR 241
+
+#define OBD_IOCTL_VERSION	0x00010004
+#define OBD_DEV_BY_DEVNAME	0xffffd0de
+#define OBD_MAX_IOCTL_BUFFER	CONFIG_LUSTRE_OBD_MAX_IOCTL_BUFFER
+
+struct obd_ioctl_data {
+	__u32		ioc_len;
+	__u32		ioc_version;
+
+	union {
+		__u64	ioc_cookie;
+		__u64	ioc_u64_1;
+	};
+	union {
+		__u32	ioc_conn1;
+		__u32	ioc_u32_1;
+	};
+	union {
+		__u32	ioc_conn2;
+		__u32	ioc_u32_2;
+	};
+
+	struct obdo	ioc_obdo1;
+	struct obdo	ioc_obdo2;
+
+	__u64		ioc_count;
+	__u64		ioc_offset;
+	__u32		ioc_dev;
+	__u32		ioc_command;
+
+	__u64		ioc_nid;
+	__u32		ioc_nal;
+	__u32		ioc_type;
+
+	/* buffers the kernel will treat as user pointers */
+	__u32		ioc_plen1;
+	char __user    *ioc_pbuf1;
+	__u32		ioc_plen2;
+	char __user    *ioc_pbuf2;
+
+	/* inline buffers for various arguments */
+	__u32		ioc_inllen1;
+	char	       *ioc_inlbuf1;
+	__u32		ioc_inllen2;
+	char	       *ioc_inlbuf2;
+	__u32		ioc_inllen3;
+	char	       *ioc_inlbuf3;
+	__u32		ioc_inllen4;
+	char	       *ioc_inlbuf4;
+
+	char		ioc_bulk[0];
+};
+
+struct obd_ioctl_hdr {
+	__u32		ioc_len;
+	__u32		ioc_version;
+};
+
+static inline __u32 obd_ioctl_packlen(struct obd_ioctl_data *data)
+{
+	__u32 len = cfs_size_round(sizeof(*data));
+
+	len += cfs_size_round(data->ioc_inllen1);
+	len += cfs_size_round(data->ioc_inllen2);
+	len += cfs_size_round(data->ioc_inllen3);
+	len += cfs_size_round(data->ioc_inllen4);
+
+	return len;
+}
+
+static inline int obd_ioctl_is_invalid(struct obd_ioctl_data *data)
+{
+	if (data->ioc_len > (1 << 30)) {
+		CERROR("OBD ioctl: ioc_len larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inllen1 > (1 << 30)) {
+		CERROR("OBD ioctl: ioc_inllen1 larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inllen2 > (1 << 30)) {
+		CERROR("OBD ioctl: ioc_inllen2 larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inllen3 > (1 << 30)) {
+		CERROR("OBD ioctl: ioc_inllen3 larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inllen4 > (1 << 30)) {
+		CERROR("OBD ioctl: ioc_inllen4 larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inlbuf1 && !data->ioc_inllen1) {
+		CERROR("OBD ioctl: inlbuf1 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_inlbuf2 && !data->ioc_inllen2) {
+		CERROR("OBD ioctl: inlbuf2 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_inlbuf3 && !data->ioc_inllen3) {
+		CERROR("OBD ioctl: inlbuf3 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_inlbuf4 && !data->ioc_inllen4) {
+		CERROR("OBD ioctl: inlbuf4 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_pbuf1 && !data->ioc_plen1) {
+		CERROR("OBD ioctl: pbuf1 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_pbuf2 && !data->ioc_plen2) {
+		CERROR("OBD ioctl: pbuf2 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (!data->ioc_pbuf1 && data->ioc_plen1) {
+		CERROR("OBD ioctl: plen1 set but NULL pointer\n");
+		return 1;
+	}
+
+	if (!data->ioc_pbuf2 && data->ioc_plen2) {
+		CERROR("OBD ioctl: plen2 set but NULL pointer\n");
+		return 1;
+	}
+
+	if (obd_ioctl_packlen(data) > data->ioc_len) {
+		CERROR("OBD ioctl: packlen exceeds ioc_len (%d > %d)\n",
+		       obd_ioctl_packlen(data), data->ioc_len);
+		return 1;
+	}
+
+	return 0;
+}
+
+#ifdef __KERNEL__
+
+int obd_ioctl_getdata(char **buf, int *len, void __user *arg);
+int obd_ioctl_popdata(void __user *arg, void *data, int len);
+
+static inline void obd_ioctl_freedata(char *buf, size_t len)
+{
+	kvfree(buf);
+}
+
+#else /* __KERNEL__ */
+
+static inline int obd_ioctl_pack(struct obd_ioctl_data *data, char **pbuf,
+				 int max_len)
+{
+	char *ptr;
+	struct obd_ioctl_data *overlay;
+
+	data->ioc_len = obd_ioctl_packlen(data);
+	data->ioc_version = OBD_IOCTL_VERSION;
+
+	if (*pbuf && data->ioc_len > max_len) {
+		fprintf(stderr, "pbuf = %p, ioc_len = %u, max_len = %d\n",
+			*pbuf, data->ioc_len, max_len);
+		return -EINVAL;
+	}
+
+	if (!*pbuf)
+		*pbuf = malloc(data->ioc_len);
+
+	if (!*pbuf)
+		return -ENOMEM;
+
+	overlay = (struct obd_ioctl_data *)*pbuf;
+	memcpy(*pbuf, data, sizeof(*data));
+
+	ptr = overlay->ioc_bulk;
+	if (data->ioc_inlbuf1)
+		LOGL(data->ioc_inlbuf1, data->ioc_inllen1, ptr);
+
+	if (data->ioc_inlbuf2)
+		LOGL(data->ioc_inlbuf2, data->ioc_inllen2, ptr);
+
+	if (data->ioc_inlbuf3)
+		LOGL(data->ioc_inlbuf3, data->ioc_inllen3, ptr);
+
+	if (data->ioc_inlbuf4)
+		LOGL(data->ioc_inlbuf4, data->ioc_inllen4, ptr);
+
+	if (obd_ioctl_is_invalid(overlay)) {
+		fprintf(stderr, "invalid ioctl data: ioc_len = %u, max_len = %d\n",
+			data->ioc_len, max_len);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static inline int
+obd_ioctl_unpack(struct obd_ioctl_data *data, char *pbuf, int max_len)
+{
+	char *ptr;
+	struct obd_ioctl_data *overlay;
+
+	if (!pbuf)
+		return 1;
+
+	overlay = (struct obd_ioctl_data *)pbuf;
+
+	/* Preserve the caller's buffer pointers */
+	overlay->ioc_inlbuf1 = data->ioc_inlbuf1;
+	overlay->ioc_inlbuf2 = data->ioc_inlbuf2;
+	overlay->ioc_inlbuf3 = data->ioc_inlbuf3;
+	overlay->ioc_inlbuf4 = data->ioc_inlbuf4;
+
+	memcpy(data, pbuf, sizeof(*data));
+
+	ptr = overlay->ioc_bulk;
+	if (data->ioc_inlbuf1)
+		LOGU(data->ioc_inlbuf1, data->ioc_inllen1, ptr);
+
+	if (data->ioc_inlbuf2)
+		LOGU(data->ioc_inlbuf2, data->ioc_inllen2, ptr);
+
+	if (data->ioc_inlbuf3)
+		LOGU(data->ioc_inlbuf3, data->ioc_inllen3, ptr);
+
+	if (data->ioc_inlbuf4)
+		LOGU(data->ioc_inlbuf4, data->ioc_inllen4, ptr);
+
+	return 0;
+}
+
+#endif /* !__KERNEL__ */
+
+/*
+ * OBD_IOC_DATA_TYPE is only for compatibility reasons with older
+ * Linux Lustre user tools. New ioctls should NOT use this macro as
+ * the ioctl "size". Instead the ioctl should get a "size" argument
+ * which is the actual data type used by the ioctl, to ensure the
+ * ioctl interface is versioned correctly.
+ */
+#define OBD_IOC_DATA_TYPE	long
+
+/*	IOC_LDLM_TEST		_IOWR('f', 40, long) */
+/*	IOC_LDLM_DUMP		_IOWR('f', 41, long) */
+/*	IOC_LDLM_REGRESS_START	_IOWR('f', 42, long) */
+/*	IOC_LDLM_REGRESS_STOP	_IOWR('f', 43, long) */
+
+#define OBD_IOC_CREATE		_IOWR('f', 101, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_DESTROY		_IOW('f', 104, OBD_IOC_DATA_TYPE)
+/*	OBD_IOC_PREALLOCATE	_IOWR('f', 105, OBD_IOC_DATA_TYPE) */
+
+#define OBD_IOC_SETATTR		_IOW('f', 107, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_GETATTR		_IOWR('f', 108, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_READ		_IOWR('f', 109, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_WRITE		_IOWR('f', 110, OBD_IOC_DATA_TYPE)
+
+#define OBD_IOC_STATFS		_IOWR('f', 113, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_SYNC		_IOW('f', 114, OBD_IOC_DATA_TYPE)
+/*	OBD_IOC_READ2		_IOWR('f', 115, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_FORMAT		_IOWR('f', 116, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_PARTITION	_IOWR('f', 117, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_COPY		_IOWR('f', 120, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_MIGR		_IOWR('f', 121, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_PUNCH		_IOWR('f', 122, OBD_IOC_DATA_TYPE) */
+
+/*	OBD_IOC_MODULE_DEBUG	_IOWR('f', 124, OBD_IOC_DATA_TYPE) */
+#define OBD_IOC_BRW_READ	_IOWR('f', 125, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_BRW_WRITE	_IOWR('f', 126, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_NAME2DEV	_IOWR('f', 127, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_UUID2DEV	_IOWR('f', 130, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_GETNAME		_IOWR('f', 131, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_GETMDNAME	_IOR('f', 131, char[MAX_OBD_NAME])
+#define OBD_IOC_GETDTNAME	OBD_IOC_GETNAME
+#define OBD_IOC_LOV_GET_CONFIG	_IOWR('f', 132, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_CLIENT_RECOVER	_IOW('f', 133, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_PING_TARGET	_IOW('f', 136, OBD_IOC_DATA_TYPE)
+
+/*	OBD_IOC_DEC_FS_USE_COUNT _IO('f', 139) */
+#define OBD_IOC_NO_TRANSNO	_IOW('f', 140, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_SET_READONLY	_IOW('f', 141, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_ABORT_RECOVERY	_IOR('f', 142, OBD_IOC_DATA_TYPE)
+/*	OBD_IOC_ROOT_SQUASH	_IOWR('f', 143, OBD_IOC_DATA_TYPE) */
+#define OBD_GET_VERSION		_IOWR('f', 144, OBD_IOC_DATA_TYPE)
+/*	OBD_IOC_GSS_SUPPORT	_IOWR('f', 145, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_CLOSE_UUID	_IOWR('f', 147, OBD_IOC_DATA_TYPE) */
+#define OBD_IOC_CHANGELOG_SEND	_IOW('f', 148, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_GETDEVICE	_IOWR('f', 149, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_FID2PATH	_IOWR('f', 150, OBD_IOC_DATA_TYPE)
+/*	lustre/lustre_user.h	151-153 */
+/*	OBD_IOC_LOV_SETSTRIPE	154 LL_IOC_LOV_SETSTRIPE */
+/*	OBD_IOC_LOV_GETSTRIPE	155 LL_IOC_LOV_GETSTRIPE */
+/*	OBD_IOC_LOV_SETEA	156 LL_IOC_LOV_SETEA */
+/*	lustre/lustre_user.h	157-159 */
+#define	OBD_IOC_QUOTACHECK	_IOW('f', 160, int)
+#define	OBD_IOC_POLL_QUOTACHECK	_IOR('f', 161, struct if_quotacheck *)
+#define OBD_IOC_QUOTACTL	_IOWR('f', 162, struct if_quotactl)
+/*	lustre/lustre_user.h	163-176 */
+#define OBD_IOC_CHANGELOG_REG	_IOW('f', 177, struct obd_ioctl_data)
+#define OBD_IOC_CHANGELOG_DEREG	_IOW('f', 178, struct obd_ioctl_data)
+#define OBD_IOC_CHANGELOG_CLEAR	_IOW('f', 179, struct obd_ioctl_data)
+/*	OBD_IOC_RECORD		_IOWR('f', 180, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_ENDRECORD	_IOWR('f', 181, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_PARSE		_IOWR('f', 182, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_DORECORD	_IOWR('f', 183, OBD_IOC_DATA_TYPE) */
+#define OBD_IOC_PROCESS_CFG	_IOWR('f', 184, OBD_IOC_DATA_TYPE)
+/*	OBD_IOC_DUMP_LOG	_IOWR('f', 185, OBD_IOC_DATA_TYPE) */
+/*	OBD_IOC_CLEAR_LOG	_IOWR('f', 186, OBD_IOC_DATA_TYPE) */
+#define OBD_IOC_PARAM		_IOW('f', 187, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_POOL		_IOWR('f', 188, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_REPLACE_NIDS	_IOWR('f', 189, OBD_IOC_DATA_TYPE)
+
+#define OBD_IOC_CATLOGLIST	_IOWR('f', 190, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_LLOG_INFO	_IOWR('f', 191, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_LLOG_PRINT	_IOWR('f', 192, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_LLOG_CANCEL	_IOWR('f', 193, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_LLOG_REMOVE	_IOWR('f', 194, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_LLOG_CHECK	_IOWR('f', 195, OBD_IOC_DATA_TYPE)
+/*	OBD_IOC_LLOG_CATINFO	_IOWR('f', 196, OBD_IOC_DATA_TYPE) */
+#define OBD_IOC_NODEMAP		_IOWR('f', 197, OBD_IOC_DATA_TYPE)
+
+/*	ECHO_IOC_GET_STRIPE	_IOWR('f', 200, OBD_IOC_DATA_TYPE) */
+/*	ECHO_IOC_SET_STRIPE	_IOWR('f', 201, OBD_IOC_DATA_TYPE) */
+/*	ECHO_IOC_ENQUEUE	_IOWR('f', 202, OBD_IOC_DATA_TYPE) */
+/*	ECHO_IOC_CANCEL		_IOWR('f', 203, OBD_IOC_DATA_TYPE) */
+
+#define OBD_IOC_GET_OBJ_VERSION	_IOR('f', 210, OBD_IOC_DATA_TYPE)
+
+/*	lustre/lustre_user.h	212-217 */
+#define OBD_IOC_GET_MNTOPT	_IOW('f', 220, mntopt_t)
+#define OBD_IOC_ECHO_MD		_IOR('f', 221, struct obd_ioctl_data)
+#define OBD_IOC_ECHO_ALLOC_SEQ	_IOWR('f', 222, struct obd_ioctl_data)
+#define OBD_IOC_START_LFSCK	_IOWR('f', 230, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_STOP_LFSCK	_IOW('f', 231, OBD_IOC_DATA_TYPE)
+#define OBD_IOC_QUERY_LFSCK	_IOR('f', 232, struct obd_ioctl_data)
+/*	lustre/lustre_user.h	240-249 */
+/*	LIBCFS_IOC_DEBUG_MASK	250 */
+
+#define IOC_OSC_SET_ACTIVE	_IOWR('h', 21, void *)
+
+#endif /* LUSTRE_IOCTL_H_ */
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_user.h b/drivers/staging/lustre/lustre/include/lustre/lustre_user.h
index ef6f38ff359e..6fc985571cba 100644
--- a/drivers/staging/lustre/lustre/include/lustre/lustre_user.h
+++ b/drivers/staging/lustre/lustre/include/lustre/lustre_user.h
@@ -42,8 +42,35 @@
  * @{
  */
 
+#ifdef __KERNEL__
+# include <linux/quota.h>
+# include <linux/string.h> /* snprintf() */
+# include <linux/version.h>
+#else /* !__KERNEL__ */
+# define NEED_QUOTA_DEFS
+# include <stdio.h> /* snprintf() */
+# include <string.h>
+# include <sys/quota.h>
+# include <sys/stat.h>
+#endif /* __KERNEL__ */
 #include "ll_fiemap.h"
-#include "../linux/lustre_user.h"
+
+/*
+ * We need to always use 64bit version because the structure
+ * is shared across entire cluster where 32bit and 64bit machines
+ * are co-existing.
+ */
+#if __BITS_PER_LONG != 64 || defined(__ARCH_WANT_STAT64)
+typedef struct stat64   lstat_t;
+#define lstat_f  lstat64
+#else
+typedef struct stat     lstat_t;
+#define lstat_f  lstat
+#endif
+
+#define HAVE_LOV_USER_MDS_DATA
+
+#define LUSTRE_EOF 0xffffffffffffffffULL
 
 /* for statfs() */
 #define LL_SUPER_MAGIC 0x0BD00BD0
@@ -117,6 +144,11 @@ struct lu_fid {
 	__u32 f_ver;
 };
 
+static inline bool fid_is_zero(const struct lu_fid *fid)
+{
+	return !fid->f_seq && !fid->f_oid;
+}
+
 struct filter_fid {
 	struct lu_fid	ff_parent;  /* ff_parent.f_ver == file stripe number */
 };
@@ -167,7 +199,7 @@ struct lustre_mdt_attrs {
  */
 struct ost_id {
 	union {
-		struct ostid {
+		struct {
 			__u64	oi_id;
 			__u64	oi_seq;
 		} oi;
@@ -188,26 +220,20 @@ struct ost_id {
  * *STRIPE* - set/get lov_user_md
  * *INFO    - set/get lov_user_mds_data
  */
-/* see <lustre_lib.h> for ioctl numberss 101-150 */
+/*	lustre_ioctl.h			101-150 */
 #define LL_IOC_GETFLAGS		 _IOR('f', 151, long)
 #define LL_IOC_SETFLAGS		 _IOW('f', 152, long)
 #define LL_IOC_CLRFLAGS		 _IOW('f', 153, long)
-/* LL_IOC_LOV_SETSTRIPE: See also OBD_IOC_LOV_SETSTRIPE */
 #define LL_IOC_LOV_SETSTRIPE	    _IOW('f', 154, long)
-/* LL_IOC_LOV_GETSTRIPE: See also OBD_IOC_LOV_GETSTRIPE */
 #define LL_IOC_LOV_GETSTRIPE	    _IOW('f', 155, long)
-/* LL_IOC_LOV_SETEA: See also OBD_IOC_LOV_SETEA */
 #define LL_IOC_LOV_SETEA		_IOW('f', 156, long)
-#define LL_IOC_RECREATE_OBJ	     _IOW('f', 157, long)
-#define LL_IOC_RECREATE_FID	     _IOW('f', 157, struct lu_fid)
+/*	LL_IOC_RECREATE_OBJ		157 obsolete */
+/*	LL_IOC_RECREATE_FID		158 obsolete */
 #define LL_IOC_GROUP_LOCK	       _IOW('f', 158, long)
 #define LL_IOC_GROUP_UNLOCK	     _IOW('f', 159, long)
-/* LL_IOC_QUOTACHECK: See also OBD_IOC_QUOTACHECK */
-#define LL_IOC_QUOTACHECK	       _IOW('f', 160, int)
-/* LL_IOC_POLL_QUOTACHECK: See also OBD_IOC_POLL_QUOTACHECK */
-#define LL_IOC_POLL_QUOTACHECK	  _IOR('f', 161, struct if_quotacheck *)
-/* LL_IOC_QUOTACTL: See also OBD_IOC_QUOTACTL */
-#define LL_IOC_QUOTACTL		 _IOWR('f', 162, struct if_quotactl)
+/* #define LL_IOC_QUOTACHECK		160 OBD_IOC_QUOTACHECK */
+/* #define LL_IOC_POLL_QUOTACHECK	161 OBD_IOC_POLL_QUOTACHECK */
+/* #define LL_IOC_QUOTACTL		162 OBD_IOC_QUOTACTL */
 #define IOC_OBD_STATFS		  _IOWR('f', 164, struct obd_statfs *)
 #define IOC_LOV_GETINFO		 _IOWR('f', 165, struct lov_user_mds_data *)
 #define LL_IOC_FLUSHCTX		 _IOW('f', 166, long)
@@ -221,8 +247,7 @@ struct ost_id {
 #define LL_IOC_GET_CONNECT_FLAGS	_IOWR('f', 174, __u64 *)
 #define LL_IOC_GET_MDTIDX	       _IOR('f', 175, int)
 
-/* see <lustre_lib.h> for ioctl numbers 177-210 */
-
+/*	lustre_ioctl.h			177-210 */
 #define LL_IOC_HSM_STATE_GET		_IOR('f', 211, struct hsm_user_state)
 #define LL_IOC_HSM_STATE_SET		_IOW('f', 212, struct hsm_state_set)
 #define LL_IOC_HSM_CT_START		_IOW('f', 213, struct lustre_kernelcomm)
@@ -242,6 +267,17 @@ struct ost_id {
 #define LL_IOC_SET_LEASE		_IOWR('f', 243, long)
 #define LL_IOC_GET_LEASE		_IO('f', 244)
 #define LL_IOC_HSM_IMPORT		_IOWR('f', 245, struct hsm_user_import)
+#define LL_IOC_LMV_SET_DEFAULT_STRIPE	_IOWR('f', 246, struct lmv_user_md)
+#define LL_IOC_MIGRATE			_IOR('f', 247, int)
+#define LL_IOC_FID2MDTIDX		_IOWR('f', 248, struct lu_fid)
+#define LL_IOC_GETPARENT		_IOWR('f', 249, struct getparent)
+
+/* Lease types for use as arg and return of LL_IOC_{GET,SET}_LEASE ioctl. */
+enum ll_lease_type {
+	LL_LEASE_RDLCK	= 0x1,
+	LL_LEASE_WRLCK	= 0x2,
+	LL_LEASE_UNLCK	= 0x4,
+};
 
 #define LL_STATFS_LMV	   1
 #define LL_STATFS_LOV	   2
@@ -253,10 +289,6 @@ struct ost_id {
 #define IOC_MDC_GETFILEINFO     _IOWR(IOC_MDC_TYPE, 22, struct lov_user_mds_data *)
 #define LL_IOC_MDC_GETINFO      _IOWR(IOC_MDC_TYPE, 23, struct lov_user_mds_data *)
 
-/* Keep these for backward compartability. */
-#define LL_IOC_OBD_STATFS       IOC_OBD_STATFS
-#define IOC_MDC_GETSTRIPE       IOC_MDC_GETFILESTRIPE
-
 #define MAX_OBD_NAME 128 /* If this changes, a NEW ioctl must be added */
 
 /* Define O_LOV_DELAY_CREATE to be a mask that is not useful for regular
@@ -273,20 +305,26 @@ struct ost_id {
 #define LL_FILE_LOCKLESS_IO     0x00000010 /* server-side locks with cio */
 #define LL_FILE_RMTACL	  0x00000020
 
-#define LOV_USER_MAGIC_V1 0x0BD10BD0
-#define LOV_USER_MAGIC    LOV_USER_MAGIC_V1
-#define LOV_USER_MAGIC_JOIN_V1 0x0BD20BD0
-#define LOV_USER_MAGIC_V3 0x0BD30BD0
+#define LOV_USER_MAGIC_V1	0x0BD10BD0
+#define LOV_USER_MAGIC		LOV_USER_MAGIC_V1
+#define LOV_USER_MAGIC_JOIN_V1	0x0BD20BD0
+#define LOV_USER_MAGIC_V3	0x0BD30BD0
+/* 0x0BD40BD0 is occupied by LOV_MAGIC_MIGRATE */
+#define LOV_USER_MAGIC_SPECIFIC	0x0BD50BD0	/* for specific OSTs */
+
+#define LMV_USER_MAGIC    0x0CD30CD0    /*default lmv magic*/
 
-#define LMV_MAGIC_V1      0x0CD10CD0    /*normal stripe lmv magic */
-#define LMV_USER_MAGIC    0x0CD20CD0    /*default lmv magic*/
+#define LOV_PATTERN_RAID0	0x001
+#define LOV_PATTERN_RAID1	0x002
+#define LOV_PATTERN_FIRST	0x100
+#define LOV_PATTERN_CMOBD	0x200
 
-#define LOV_PATTERN_RAID0 0x001
-#define LOV_PATTERN_RAID1 0x002
-#define LOV_PATTERN_FIRST 0x100
+#define LOV_PATTERN_F_MASK	0xffff0000
+#define LOV_PATTERN_F_HOLE	0x40000000 /* there is hole in LOV EA */
+#define LOV_PATTERN_F_RELEASED	0x80000000 /* HSM released file */
 
-#define LOV_MAXPOOLNAME 16
-#define LOV_POOLNAMEF "%.16s"
+#define LOV_MAXPOOLNAME 15
+#define LOV_POOLNAMEF "%.15s"
 
 #define LOV_MIN_STRIPE_BITS 16   /* maximum PAGE_SIZE (ia64), power of 2 */
 #define LOV_MIN_STRIPE_SIZE (1 << LOV_MIN_STRIPE_BITS)
@@ -344,18 +382,17 @@ struct lov_user_md_v3 {	   /* LOV EA user data (host-endian) */
 					   * used when reading
 					   */
 	};
-	char  lmm_pool_name[LOV_MAXPOOLNAME]; /* pool name */
+	char  lmm_pool_name[LOV_MAXPOOLNAME + 1];   /* pool name */
 	struct lov_user_ost_data_v1 lmm_objects[0]; /* per-stripe data */
 } __packed;
 
 static inline __u32 lov_user_md_size(__u16 stripes, __u32 lmm_magic)
 {
-	if (lmm_magic == LOV_USER_MAGIC_V3)
-		return sizeof(struct lov_user_md_v3) +
-				stripes * sizeof(struct lov_user_ost_data_v1);
-	else
+	if (lmm_magic == LOV_USER_MAGIC_V1)
 		return sizeof(struct lov_user_md_v1) +
 				stripes * sizeof(struct lov_user_ost_data_v1);
+	return sizeof(struct lov_user_md_v3) +
+	       stripes * sizeof(struct lov_user_ost_data_v1);
 }
 
 /* Compile with -D_LARGEFILE64_SOURCE or -D_GNU_SOURCE (or #define) to
@@ -374,19 +411,26 @@ struct lov_user_mds_data_v3 {
 } __packed;
 #endif
 
-/* keep this to be the same size as lov_user_ost_data_v1 */
 struct lmv_user_mds_data {
 	struct lu_fid	lum_fid;
 	__u32		lum_padding;
 	__u32		lum_mds;
 };
 
-/* lum_type */
-enum {
-	LMV_STRIPE_TYPE = 0,
-	LMV_DEFAULT_TYPE = 1,
+enum lmv_hash_type {
+	LMV_HASH_TYPE_UNKNOWN	= 0,	/* 0 is reserved for testing purpose */
+	LMV_HASH_TYPE_ALL_CHARS = 1,
+	LMV_HASH_TYPE_FNV_1A_64 = 2,
 };
 
+#define LMV_HASH_NAME_ALL_CHARS		"all_char"
+#define LMV_HASH_NAME_FNV_1A_64		"fnv_1a_64"
+
+/*
+ * Got this according to how get LOV_MAX_STRIPE_COUNT, see above,
+ * (max buffer size - lmv+rpc header) / sizeof(struct lmv_user_mds_data)
+ */
+#define LMV_MAX_STRIPE_COUNT 2000  /* ((12 * 4096 - 256) / 24) */
 #define lmv_user_md lmv_user_md_v1
 struct lmv_user_md_v1 {
 	__u32	lum_magic;	 /* must be the first field */
@@ -397,9 +441,9 @@ struct lmv_user_md_v1 {
 	__u32	lum_padding1;
 	__u32	lum_padding2;
 	__u32	lum_padding3;
-	char	lum_pool_name[LOV_MAXPOOLNAME];
+	char	lum_pool_name[LOV_MAXPOOLNAME + 1];
 	struct	lmv_user_mds_data  lum_objects[0];
-};
+} __packed;
 
 static inline int lmv_user_md_size(int stripes, int lmm_magic)
 {
@@ -407,6 +451,8 @@ static inline int lmv_user_md_size(int stripes, int lmm_magic)
 		      stripes * sizeof(struct lmv_user_mds_data);
 }
 
+void lustre_swab_lmv_user_md(struct lmv_user_md *lum);
+
 struct ll_recreate_obj {
 	__u64 lrc_id;
 	__u32 lrc_ost_idx;
@@ -498,6 +544,12 @@ static inline void obd_uuid2fsname(char *buf, char *uuid, int buflen)
 
 /********* Quotas **********/
 
+#define Q_QUOTACHECK   0x800100 /* deprecated as of 2.4 */
+#define Q_INITQUOTA    0x800101 /* deprecated as of 2.4  */
+#define Q_GETOINFO     0x800102 /* get obd quota info */
+#define Q_GETOQUOTA    0x800103 /* get obd quotas */
+#define Q_FINVALIDATE  0x800104 /* deprecated as of 2.4 */
+
 /* these must be explicitly translated into linux Q_* in ll_dir_ioctl */
 #define LUSTRE_Q_QUOTAON    0x800002     /* turn quotas on */
 #define LUSTRE_Q_QUOTAOFF   0x800003     /* turn quotas off */
@@ -648,11 +700,16 @@ static inline const char *changelog_type2str(int type)
 }
 
 /* per-record flags */
-#define CLF_VERSION     0x1000
-#define CLF_EXT_VERSION 0x2000
 #define CLF_FLAGSHIFT   12
 #define CLF_FLAGMASK    ((1U << CLF_FLAGSHIFT) - 1)
 #define CLF_VERMASK     (~CLF_FLAGMASK)
+enum changelog_rec_flags {
+	CLF_VERSION	= 0x1000,
+	CLF_RENAME	= 0x2000,
+	CLF_JOBID	= 0x4000,
+	CLF_SUPPORTED	= CLF_VERSION | CLF_RENAME | CLF_JOBID
+};
+
 /* Anything under the flagmask may be per-type (if desired) */
 /* Flags for unlink */
 #define CLF_UNLINK_LAST       0x0001 /* Unlink of last hardlink */
@@ -736,12 +793,35 @@ static inline void hsm_set_cl_error(int *flags, int error)
 	*flags |= (error << CLF_HSM_ERR_L);
 }
 
-#define CR_MAXSIZE cfs_size_round(2*NAME_MAX + 1 + \
-				  sizeof(struct changelog_ext_rec))
+enum changelog_send_flag {
+	/* Not yet implemented */
+	CHANGELOG_FLAG_FOLLOW	= BIT(0),
+	/*
+	 * Blocking IO makes sense in case of slow user parsing of the records,
+	 * but it also prevents us from cleaning up if the records are not
+	 * consumed.
+	 */
+	CHANGELOG_FLAG_BLOCK	= BIT(1),
+	/* Pack jobid into the changelog records if available. */
+	CHANGELOG_FLAG_JOBID	= BIT(2),
+};
+
+#define CR_MAXSIZE cfs_size_round(2 * NAME_MAX + 2 + \
+				  changelog_rec_offset(CLF_SUPPORTED))
+
+/* 31 usable bytes string + null terminator. */
+#define LUSTRE_JOBID_SIZE	32
 
+/*
+ * This is the minimal changelog record. It can contain extensions
+ * such as rename fields or process jobid. Its exact content is described
+ * by the cr_flags.
+ *
+ * Extensions are packed in the same order as their corresponding flags.
+ */
 struct changelog_rec {
 	__u16		 cr_namelen;
-	__u16		 cr_flags; /**< (flags&CLF_FLAGMASK)|CLF_VERSION */
+	__u16		 cr_flags; /**< \a changelog_rec_flags */
 	__u32		 cr_type;  /**< \a changelog_rec_type */
 	__u64		 cr_index; /**< changelog record number */
 	__u64		 cr_prev;  /**< last index for this target fid */
@@ -751,55 +831,138 @@ struct changelog_rec {
 		__u32	 cr_markerflags; /**< CL_MARK flags */
 	};
 	struct lu_fid	    cr_pfid;	/**< parent fid */
-	char		  cr_name[0];     /**< last element */
 } __packed;
 
-/* changelog_ext_rec is 2*sizeof(lu_fid) bigger than changelog_rec, to save
- * space, only rename uses changelog_ext_rec, while others use changelog_rec to
- * store records.
- */
-struct changelog_ext_rec {
-	__u16			cr_namelen;
-	__u16			cr_flags; /**< (flags & CLF_FLAGMASK) |
-					   *	CLF_EXT_VERSION
-					   */
-	__u32			cr_type;  /**< \a changelog_rec_type */
-	__u64			cr_index; /**< changelog record number */
-	__u64			cr_prev;  /**< last index for this target fid */
-	__u64			cr_time;
-	union {
-		struct lu_fid	cr_tfid;	/**< target fid */
-		__u32		cr_markerflags; /**< CL_MARK flags */
-	};
-	struct lu_fid		cr_pfid;	/**< target parent fid */
-	struct lu_fid		cr_sfid;	/**< source fid, or zero */
-	struct lu_fid		cr_spfid;     /**< source parent fid, or zero */
-	char			cr_name[0];     /**< last element */
-} __packed;
+/* Changelog extension for RENAME. */
+struct changelog_ext_rename {
+	struct lu_fid	cr_sfid;	/**< source fid, or zero */
+	struct lu_fid	cr_spfid;	/**< source parent fid, or zero */
+};
 
-#define CHANGELOG_REC_EXTENDED(rec) \
-	(((rec)->cr_flags & CLF_VERMASK) == CLF_EXT_VERSION)
+/* Changelog extension to include JOBID. */
+struct changelog_ext_jobid {
+	char	cr_jobid[LUSTRE_JOBID_SIZE];	/**< zero-terminated string. */
+};
+
+static inline size_t changelog_rec_offset(enum changelog_rec_flags crf)
+{
+	size_t size = sizeof(struct changelog_rec);
+
+	if (crf & CLF_RENAME)
+		size += sizeof(struct changelog_ext_rename);
+
+	if (crf & CLF_JOBID)
+		size += sizeof(struct changelog_ext_jobid);
 
-static inline int changelog_rec_size(struct changelog_rec *rec)
+	return size;
+}
+
+static inline size_t changelog_rec_size(struct changelog_rec *rec)
 {
-	return CHANGELOG_REC_EXTENDED(rec) ? sizeof(struct changelog_ext_rec) :
-					     sizeof(*rec);
+	return changelog_rec_offset(rec->cr_flags);
+}
+
+static inline size_t changelog_rec_varsize(struct changelog_rec *rec)
+{
+	return changelog_rec_size(rec) - sizeof(*rec) + rec->cr_namelen;
+}
+
+static inline
+struct changelog_ext_rename *changelog_rec_rename(struct changelog_rec *rec)
+{
+	enum changelog_rec_flags crf = rec->cr_flags & CLF_VERSION;
+
+	return (struct changelog_ext_rename *)((char *)rec +
+					       changelog_rec_offset(crf));
+}
+
+/* The jobid follows the rename extension, if present */
+static inline
+struct changelog_ext_jobid *changelog_rec_jobid(struct changelog_rec *rec)
+{
+	enum changelog_rec_flags crf = rec->cr_flags &
+				       (CLF_VERSION | CLF_RENAME);
+
+	return (struct changelog_ext_jobid *)((char *)rec +
+					      changelog_rec_offset(crf));
 }
 
+/* The name follows the rename and jobid extensions, if present */
 static inline char *changelog_rec_name(struct changelog_rec *rec)
 {
-	return CHANGELOG_REC_EXTENDED(rec) ?
-		((struct changelog_ext_rec *)rec)->cr_name : rec->cr_name;
+	return (char *)rec + changelog_rec_offset(rec->cr_flags &
+						  CLF_SUPPORTED);
 }
 
-static inline int changelog_rec_snamelen(struct changelog_ext_rec *rec)
+static inline size_t changelog_rec_snamelen(struct changelog_rec *rec)
 {
-	return rec->cr_namelen - strlen(rec->cr_name) - 1;
+	return rec->cr_namelen - strlen(changelog_rec_name(rec)) - 1;
 }
 
-static inline char *changelog_rec_sname(struct changelog_ext_rec *rec)
+static inline char *changelog_rec_sname(struct changelog_rec *rec)
 {
-	return rec->cr_name + strlen(rec->cr_name) + 1;
+	char *cr_name = changelog_rec_name(rec);
+
+	return cr_name + strlen(cr_name) + 1;
+}
+
+/**
+ * Remap a record to the desired format as specified by the crf flags.
+ * The record must be big enough to contain the final remapped version.
+ * Superfluous extension fields are removed and missing ones are added
+ * and zeroed. The flags of the record are updated accordingly.
+ *
+ * The jobid and rename extensions can be added to a record, to match the
+ * format an application expects, typically. In this case, the newly added
+ * fields will be zeroed.
+ * The Jobid field can be removed, to guarantee compatibility with older
+ * clients that don't expect this field in the records they process.
+ *
+ * The following assumptions are being made:
+ *	- CLF_RENAME will not be removed
+ *	- CLF_JOBID will not be added without CLF_RENAME being added too
+ *
+ * @param[in,out]  rec		The record to remap.
+ * @param[in]	   crf_wanted	Flags describing the desired extensions.
+ */
+static inline void changelog_remap_rec(struct changelog_rec *rec,
+				       enum changelog_rec_flags crf_wanted)
+{
+	char *jid_mov, *rnm_mov;
+
+	crf_wanted &= CLF_SUPPORTED;
+
+	if ((rec->cr_flags & CLF_SUPPORTED) == crf_wanted)
+		return;
+
+	/* First move the variable-length name field */
+	memmove((char *)rec + changelog_rec_offset(crf_wanted),
+		changelog_rec_name(rec), rec->cr_namelen);
+
+	/* Locations of jobid and rename extensions in the remapped record */
+	jid_mov = (char *)rec +
+		  changelog_rec_offset(crf_wanted & ~CLF_JOBID);
+	rnm_mov = (char *)rec +
+		  changelog_rec_offset(crf_wanted & ~(CLF_JOBID | CLF_RENAME));
+
+	/* Move the extension fields to the desired positions */
+	if ((crf_wanted & CLF_JOBID) && (rec->cr_flags & CLF_JOBID))
+		memmove(jid_mov, changelog_rec_jobid(rec),
+			sizeof(struct changelog_ext_jobid));
+
+	if ((crf_wanted & CLF_RENAME) && (rec->cr_flags & CLF_RENAME))
+		memmove(rnm_mov, changelog_rec_rename(rec),
+			sizeof(struct changelog_ext_rename));
+
+	/* Clear newly added fields */
+	if ((crf_wanted & CLF_JOBID) && !(rec->cr_flags & CLF_JOBID))
+		memset(jid_mov, 0, sizeof(struct changelog_ext_jobid));
+
+	if ((crf_wanted & CLF_RENAME) && !(rec->cr_flags & CLF_RENAME))
+		memset(rnm_mov, 0, sizeof(struct changelog_ext_rename));
+
+	/* Update the record's flags accordingly */
+	rec->cr_flags = (rec->cr_flags & CLF_FLAGMASK) | crf_wanted;
 }
 
 struct ioc_changelog {
@@ -978,7 +1141,7 @@ struct hsm_user_request {
 /** Return pointer to data field in a hsm user request */
 static inline void *hur_data(struct hsm_user_request *hur)
 {
-	return &(hur->hur_user_item[hur->hur_request.hr_itemcount]);
+	return &hur->hur_user_item[hur->hur_request.hr_itemcount];
 }
 
 /**
diff --git a/drivers/staging/lustre/lustre/include/lustre_acl.h b/drivers/staging/lustre/lustre/include/lustre_acl.h
index fecabe139b1f..9786f6caaade 100644
--- a/drivers/staging/lustre/lustre/include/lustre_acl.h
+++ b/drivers/staging/lustre/lustre/include/lustre_acl.h
@@ -38,8 +38,8 @@
 #include <linux/posix_acl_xattr.h>
 
 #define LUSTRE_POSIX_ACL_MAX_ENTRIES	32
-#define LUSTRE_POSIX_ACL_MAX_SIZE					\
-	(sizeof(posix_acl_xattr_header) +				\
-	 LUSTRE_POSIX_ACL_MAX_ENTRIES * sizeof(posix_acl_xattr_entry))
+#define LUSTRE_POSIX_ACL_MAX_SIZE						\
+	(sizeof(struct posix_acl_xattr_header) +				\
+	 LUSTRE_POSIX_ACL_MAX_ENTRIES * sizeof(struct posix_acl_xattr_entry))
 
 #endif
diff --git a/drivers/staging/lustre/lustre/include/lustre_cfg.h b/drivers/staging/lustre/lustre/include/lustre_cfg.h
index 95a0be13c0fb..8eb394e64b25 100644
--- a/drivers/staging/lustre/lustre/include/lustre_cfg.h
+++ b/drivers/staging/lustre/lustre/include/lustre_cfg.h
@@ -151,13 +151,11 @@ static inline void lustre_cfg_bufs_reset(struct lustre_cfg_bufs *bufs, char *nam
 		lustre_cfg_bufs_set_string(bufs, 0, name);
 }
 
-static inline void *lustre_cfg_buf(struct lustre_cfg *lcfg, int index)
+static inline void *lustre_cfg_buf(struct lustre_cfg *lcfg, __u32 index)
 {
-	int i;
-	int offset;
-	int bufcount;
-
-	LASSERT(index >= 0);
+	__u32 i;
+	size_t offset;
+	__u32 bufcount;
 
 	bufcount = lcfg->lcfg_bufcount;
 	if (index >= bufcount)
@@ -172,7 +170,7 @@ static inline void *lustre_cfg_buf(struct lustre_cfg *lcfg, int index)
 static inline void lustre_cfg_bufs_init(struct lustre_cfg_bufs *bufs,
 					struct lustre_cfg *lcfg)
 {
-	int i;
+	__u32 i;
 
 	bufs->lcfg_bufcount = lcfg->lcfg_bufcount;
 	for (i = 0; i < bufs->lcfg_bufcount; i++) {
@@ -181,7 +179,7 @@ static inline void lustre_cfg_bufs_init(struct lustre_cfg_bufs *bufs,
 	}
 }
 
-static inline char *lustre_cfg_string(struct lustre_cfg *lcfg, int index)
+static inline char *lustre_cfg_string(struct lustre_cfg *lcfg, __u32 index)
 {
 	char *s;
 
@@ -197,8 +195,8 @@ static inline char *lustre_cfg_string(struct lustre_cfg *lcfg, int index)
 	 * of data.  Try to use the padding first though.
 	 */
 	if (s[lcfg->lcfg_buflens[index] - 1] != '\0') {
-		int last = min((int)lcfg->lcfg_buflens[index],
-			       cfs_size_round(lcfg->lcfg_buflens[index]) - 1);
+		size_t last = min((size_t)lcfg->lcfg_buflens[index],
+				  cfs_size_round(lcfg->lcfg_buflens[index]) - 1);
 		char lost = s[last];
 
 		s[last] = '\0';
@@ -210,10 +208,10 @@ static inline char *lustre_cfg_string(struct lustre_cfg *lcfg, int index)
 	return s;
 }
 
-static inline int lustre_cfg_len(__u32 bufcount, __u32 *buflens)
+static inline __u32 lustre_cfg_len(__u32 bufcount, __u32 *buflens)
 {
-	int i;
-	int len;
+	__u32 i;
+	__u32 len;
 
 	len = LCFG_HDR_SIZE(bufcount);
 	for (i = 0; i < bufcount; i++)
@@ -254,7 +252,7 @@ static inline void lustre_cfg_free(struct lustre_cfg *lcfg)
 	return;
 }
 
-static inline int lustre_cfg_sanity_check(void *buf, int len)
+static inline int lustre_cfg_sanity_check(void *buf, size_t len)
 {
 	struct lustre_cfg *lcfg = (struct lustre_cfg *)buf;
 
diff --git a/drivers/staging/lustre/lustre/include/linux/lustre_compat25.h b/drivers/staging/lustre/lustre/include/lustre_compat.h
index 1eb64ec4bed4..567c438e93cb 100644
--- a/drivers/staging/lustre/lustre/include/linux/lustre_compat25.h
+++ b/drivers/staging/lustre/lustre/include/lustre_compat.h
@@ -30,8 +30,8 @@
  * Lustre is a trademark of Sun Microsystems, Inc.
  */
 
-#ifndef _LINUX_COMPAT25_H
-#define _LINUX_COMPAT25_H
+#ifndef _LUSTRE_COMPAT_H
+#define _LUSTRE_COMPAT_H
 
 #include <linux/fs_struct.h>
 #include <linux/namei.h>
@@ -74,4 +74,4 @@
 # define ext2_find_next_zero_bit  find_next_zero_bit_le
 #endif
 
-#endif /* _COMPAT25_H */
+#endif /* _LUSTRE_COMPAT_H */
diff --git a/drivers/staging/lustre/lustre/include/lustre_dlm.h b/drivers/staging/lustre/lustre/include/lustre_dlm.h
index 60051a5cfe20..d03534432624 100644
--- a/drivers/staging/lustre/lustre/include/lustre_dlm.h
+++ b/drivers/staging/lustre/lustre/include/lustre_dlm.h
@@ -573,6 +573,11 @@ enum lvb_type {
 };
 
 /**
+ * LDLM_GID_ANY is used to match any group id in ldlm_lock_match().
+ */
+#define LDLM_GID_ANY	((__u64)-1)
+
+/**
  * LDLM lock structure
  *
  * Represents a single LDLM lock and its state in memory. Each lock is
@@ -968,6 +973,7 @@ struct ldlm_enqueue_info {
 	void *ei_cb_cp;  /** lock completion callback */
 	void *ei_cb_gl;  /** lock glimpse callback */
 	void *ei_cbdata; /** Data to be passed into callbacks. */
+	unsigned int ei_enq_slave:1; /* whether enqueue slave stripes */
 };
 
 extern struct obd_ops ldlm_obd_ops;
@@ -1281,16 +1287,6 @@ int ldlm_cli_cancel_list(struct list_head *head, int count,
 int intent_disposition(struct ldlm_reply *rep, int flag);
 void intent_set_disposition(struct ldlm_reply *rep, int flag);
 
-/* ioctls for trying requests */
-#define IOC_LDLM_TYPE		   'f'
-#define IOC_LDLM_MIN_NR		 40
-
-#define IOC_LDLM_TEST		   _IOWR('f', 40, long)
-#define IOC_LDLM_DUMP		   _IOWR('f', 41, long)
-#define IOC_LDLM_REGRESS_START	  _IOWR('f', 42, long)
-#define IOC_LDLM_REGRESS_STOP	   _IOWR('f', 43, long)
-#define IOC_LDLM_MAX_NR		 43
-
 /**
  * "Modes" of acquiring lock_res, necessary to tell lockdep that taking more
  * than one lock_res is dead-lock safe.
diff --git a/drivers/staging/lustre/lustre/include/lustre_dlm_flags.h b/drivers/staging/lustre/lustre/include/lustre_dlm_flags.h
index e7e0c21a9b40..a0f064d237c9 100644
--- a/drivers/staging/lustre/lustre/include/lustre_dlm_flags.h
+++ b/drivers/staging/lustre/lustre/include/lustre_dlm_flags.h
@@ -28,21 +28,6 @@
 /** l_flags bits marked as "all_flags" bits */
 #define LDLM_FL_ALL_FLAGS_MASK          0x00FFFFFFC08F932FULL
 
-/** l_flags bits marked as "ast" bits */
-#define LDLM_FL_AST_MASK                0x0000000080008000ULL
-
-/** l_flags bits marked as "blocked" bits */
-#define LDLM_FL_BLOCKED_MASK            0x000000000000000EULL
-
-/** l_flags bits marked as "gone" bits */
-#define LDLM_FL_GONE_MASK               0x0006004000000000ULL
-
-/** l_flags bits marked as "inherit" bits */
-#define LDLM_FL_INHERIT_MASK            0x0000000000800000ULL
-
-/** l_flags bits marked as "off_wire" bits */
-#define LDLM_FL_OFF_WIRE_MASK           0x00FFFFFF00000000ULL
-
 /** extent, mode, or resource changed */
 #define LDLM_FL_LOCK_CHANGED            0x0000000000000001ULL /* bit 0 */
 #define ldlm_is_lock_changed(_l)        LDLM_TEST_FLAG((_l), 1ULL <<  0)
@@ -372,6 +357,27 @@
 #define ldlm_set_excl(_l)               LDLM_SET_FLAG((_l), 1ULL << 55)
 #define ldlm_clear_excl(_l)             LDLM_CLEAR_FLAG((_l), 1ULL << 55)
 
+/** l_flags bits marked as "ast" bits */
+#define LDLM_FL_AST_MASK		(LDLM_FL_FLOCK_DEADLOCK		|\
+					 LDLM_FL_AST_DISCARD_DATA)
+
+/** l_flags bits marked as "blocked" bits */
+#define LDLM_FL_BLOCKED_MASK		(LDLM_FL_BLOCK_GRANTED		|\
+					 LDLM_FL_BLOCK_CONV		|\
+					 LDLM_FL_BLOCK_WAIT)
+
+/** l_flags bits marked as "gone" bits */
+#define LDLM_FL_GONE_MASK		(LDLM_FL_DESTROYED		|\
+					 LDLM_FL_FAILED)
+
+/** l_flags bits marked as "inherit" bits */
+/* Flags inherited from wire on enqueue/reply between client/server. */
+/* NO_TIMEOUT flag to force ldlm_lock_match() to wait with no timeout. */
+/* TEST_LOCK flag to not let TEST lock to be granted. */
+#define LDLM_FL_INHERIT_MASK		(LDLM_FL_CANCEL_ON_BLOCK	|\
+					 LDLM_FL_NO_TIMEOUT		|\
+					 LDLM_FL_TEST_LOCK)
+
 /** test for ldlm_lock flag bit set */
 #define LDLM_TEST_FLAG(_l, _b)        (((_l)->l_flags & (_b)) != 0)
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_eacl.h b/drivers/staging/lustre/lustre/include/lustre_eacl.h
index d1039e1ff70d..1e71a8638186 100644
--- a/drivers/staging/lustre/lustre/include/lustre_eacl.h
+++ b/drivers/staging/lustre/lustre/include/lustre_eacl.h
@@ -46,6 +46,7 @@
 
 #ifdef CONFIG_FS_POSIX_ACL
 
+#include <linux/fs.h>
 #include <linux/posix_acl_xattr.h>
 
 typedef struct {
diff --git a/drivers/staging/lustre/lustre/include/lustre_fid.h b/drivers/staging/lustre/lustre/include/lustre_fid.h
index 743671a547ef..316780693193 100644
--- a/drivers/staging/lustre/lustre/include/lustre_fid.h
+++ b/drivers/staging/lustre/lustre/include/lustre_fid.h
@@ -229,6 +229,7 @@ enum local_oid {
 	MDD_LOV_OBJ_OSEQ	= 4121UL,
 	LFSCK_NAMESPACE_OID     = 4122UL,
 	REMOTE_PARENT_DIR_OID	= 4123UL,
+	SLAVE_LLOG_CATALOGS_OID	= 4124UL,
 };
 
 static inline void lu_local_obj_fid(struct lu_fid *fid, __u32 oid)
@@ -392,21 +393,19 @@ struct ldlm_namespace;
  * but was moved into name[1] along with the OID to avoid consuming the
  * renaming name[2,3] fields that need to be used for the quota identifier.
  */
-static inline struct ldlm_res_id *
+static inline void
 fid_build_reg_res_name(const struct lu_fid *fid, struct ldlm_res_id *res)
 {
 	memset(res, 0, sizeof(*res));
 	res->name[LUSTRE_RES_ID_SEQ_OFF] = fid_seq(fid);
 	res->name[LUSTRE_RES_ID_VER_OID_OFF] = fid_ver_oid(fid);
-
-	return res;
 }
 
 /*
  * Return true if resource is for object identified by FID.
  */
-static inline int fid_res_name_eq(const struct lu_fid *fid,
-				  const struct ldlm_res_id *res)
+static inline bool fid_res_name_eq(const struct lu_fid *fid,
+				   const struct ldlm_res_id *res)
 {
 	return res->name[LUSTRE_RES_ID_SEQ_OFF] == fid_seq(fid) &&
 	       res->name[LUSTRE_RES_ID_VER_OID_OFF] == fid_ver_oid(fid);
@@ -415,29 +414,25 @@ static inline int fid_res_name_eq(const struct lu_fid *fid,
 /*
  * Extract FID from LDLM resource. Reverse of fid_build_reg_res_name().
  */
-static inline struct lu_fid *
+static inline void
 fid_extract_from_res_name(struct lu_fid *fid, const struct ldlm_res_id *res)
 {
 	fid->f_seq = res->name[LUSTRE_RES_ID_SEQ_OFF];
 	fid->f_oid = (__u32)(res->name[LUSTRE_RES_ID_VER_OID_OFF]);
 	fid->f_ver = (__u32)(res->name[LUSTRE_RES_ID_VER_OID_OFF] >> 32);
 	LASSERT(fid_res_name_eq(fid, res));
-
-	return fid;
 }
 
 /*
  * Build (DLM) resource identifier from global quota FID and quota ID.
  */
-static inline struct ldlm_res_id *
+static inline void
 fid_build_quota_res_name(const struct lu_fid *glb_fid, union lquota_id *qid,
 			 struct ldlm_res_id *res)
 {
 	fid_build_reg_res_name(glb_fid, res);
 	res->name[LUSTRE_RES_ID_QUOTA_SEQ_OFF] = fid_seq(&qid->qid_fid);
 	res->name[LUSTRE_RES_ID_QUOTA_VER_OID_OFF] = fid_ver_oid(&qid->qid_fid);
-
-	return res;
 }
 
 /*
@@ -454,14 +449,12 @@ static inline void fid_extract_from_quota_res(struct lu_fid *glb_fid,
 		(__u32)(res->name[LUSTRE_RES_ID_QUOTA_VER_OID_OFF] >> 32);
 }
 
-static inline struct ldlm_res_id *
+static inline void
 fid_build_pdo_res_name(const struct lu_fid *fid, unsigned int hash,
 		       struct ldlm_res_id *res)
 {
 	fid_build_reg_res_name(fid, res);
 	res->name[LUSTRE_RES_ID_HSH_OFF] = hash;
-
-	return res;
 }
 
 /**
@@ -482,7 +475,7 @@ fid_build_pdo_res_name(const struct lu_fid *fid, unsigned int hash,
  *    res will be built from normal FID directly, i.e. res[0] = f_seq,
  *    res[1] = f_oid + f_ver.
  */
-static inline void ostid_build_res_name(struct ost_id *oi,
+static inline void ostid_build_res_name(const struct ost_id *oi,
 					struct ldlm_res_id *name)
 {
 	memset(name, 0, sizeof(*name));
@@ -497,8 +490,8 @@ static inline void ostid_build_res_name(struct ost_id *oi,
 /**
  * Return true if the resource is for the object identified by this id & group.
  */
-static inline int ostid_res_name_eq(struct ost_id *oi,
-				    struct ldlm_res_id *name)
+static inline int ostid_res_name_eq(const struct ost_id *oi,
+				    const struct ldlm_res_id *name)
 {
 	/* Note: it is just a trick here to save some effort, probably the
 	 * correct way would be turn them into the FID and compare
@@ -603,13 +596,14 @@ static inline __u32 fid_flatten32(const struct lu_fid *fid)
 	 * (from OID), or up to 128M inodes without collisions for new files.
 	 */
 	ino = ((seq & 0x000fffffULL) << 12) + ((seq >> 8) & 0xfffff000) +
-	       (seq >> (64 - (40-8)) & 0xffffff00) +
+	       (seq >> (64 - (40 - 8)) & 0xffffff00) +
 	       (fid_oid(fid) & 0xff000fff) + ((fid_oid(fid) & 0x00fff000) << 8);
 
 	return ino ? ino : fid_oid(fid);
 }
 
-static inline int lu_fid_diff(struct lu_fid *fid1, struct lu_fid *fid2)
+static inline int lu_fid_diff(const struct lu_fid *fid1,
+			      const struct lu_fid *fid2)
 {
 	LASSERTF(fid_seq(fid1) == fid_seq(fid2), "fid1:"DFID", fid2:"DFID"\n",
 		 PFID(fid1), PFID(fid2));
diff --git a/drivers/staging/lustre/lustre/include/lustre_handles.h b/drivers/staging/lustre/lustre/include/lustre_handles.h
index 1a63a6b9e116..e071bac9df57 100644
--- a/drivers/staging/lustre/lustre/include/lustre_handles.h
+++ b/drivers/staging/lustre/lustre/include/lustre_handles.h
@@ -66,6 +66,7 @@ struct portals_handle_ops {
 struct portals_handle {
 	struct list_head			h_link;
 	__u64				h_cookie;
+	const void			*h_owner;
 	struct portals_handle_ops	*h_ops;
 
 	/* newly added fields to handle the RCU issue. -jxiong */
@@ -75,15 +76,13 @@ struct portals_handle {
 	unsigned int			h_in:1;
 };
 
-#define RCU2HANDLE(rcu)    container_of(rcu, struct portals_handle, h_rcu)
-
 /* handles.c */
 
 /* Add a handle to the hash table */
 void class_handle_hash(struct portals_handle *,
 		       struct portals_handle_ops *ops);
 void class_handle_unhash(struct portals_handle *);
-void *class_handle2object(__u64 cookie);
+void *class_handle2object(__u64 cookie, const void *owner);
 void class_handle_free_cb(struct rcu_head *rcu);
 int class_handle_init(void);
 void class_handle_cleanup(void);
diff --git a/drivers/staging/lustre/lustre/include/lustre_import.h b/drivers/staging/lustre/lustre/include/lustre_import.h
index 4445be7a59dd..5461ba33d90c 100644
--- a/drivers/staging/lustre/lustre/include/lustre_import.h
+++ b/drivers/staging/lustre/lustre/include/lustre_import.h
@@ -285,8 +285,10 @@ struct obd_import {
 				  imp_resend_replay:1,
 				  /* disable normal recovery, for test only. */
 				  imp_no_pinger_recover:1,
+#if OBD_OCD_VERSION(3, 0, 53, 0) > LUSTRE_VERSION_CODE
 				  /* need IR MNE swab */
 				  imp_need_mne_swab:1,
+#endif
 				  /* import must be reconnected instead of
 				   * chosing new connection
 				   */
@@ -305,28 +307,6 @@ struct obd_import {
 	time64_t	     imp_last_reply_time;    /* for health check */
 };
 
-typedef void (*obd_import_callback)(struct obd_import *imp, void *closure,
-				    int event, void *event_arg, void *cb_data);
-
-/**
- * Structure for import observer.
- * It is possible to register "observer" on an import and every time
- * something happens to an import (like connect/evict/disconnect)
- * obderver will get its callback called with event type
- */
-struct obd_import_observer {
-	struct list_head	   oio_chain;
-	obd_import_callback  oio_cb;
-	void		*oio_cb_data;
-};
-
-void class_observe_import(struct obd_import *imp, obd_import_callback cb,
-			  void *cb_data);
-void class_unobserve_import(struct obd_import *imp, obd_import_callback cb,
-			    void *cb_data);
-void class_notify_import_observers(struct obd_import *imp, int event,
-				   void *event_arg);
-
 /* import.c */
 static inline unsigned int at_est2timeout(unsigned int val)
 {
diff --git a/drivers/staging/lustre/lustre/include/lustre_lib.h b/drivers/staging/lustre/lustre/include/lustre_lib.h
index 06958f217fc8..6b231913ba2e 100644
--- a/drivers/staging/lustre/lustre/include/lustre_lib.h
+++ b/drivers/staging/lustre/lustre/include/lustre_lib.h
@@ -51,7 +51,6 @@
 #include "lustre_cfg.h"
 
 /* target.c */
-struct kstatfs;
 struct ptlrpc_request;
 struct obd_export;
 struct lu_target;
@@ -74,325 +73,8 @@ int do_set_info_async(struct obd_import *imp,
 		      u32 vallen, void *val,
 		      struct ptlrpc_request_set *set);
 
-#define OBD_RECOVERY_MAX_TIME (obd_timeout * 18) /* b13079 */
-#define OBD_MAX_IOCTL_BUFFER CONFIG_LUSTRE_OBD_MAX_IOCTL_BUFFER
-
 void target_send_reply(struct ptlrpc_request *req, int rc, int fail_id);
 
-/* client.c */
-
-int client_sanobd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg);
-struct client_obd *client_conn2cli(struct lustre_handle *conn);
-
-struct md_open_data;
-struct obd_client_handle {
-	struct lustre_handle	 och_fh;
-	struct lu_fid		 och_fid;
-	struct md_open_data	*och_mod;
-	struct lustre_handle	 och_lease_handle; /* open lock for lease */
-	__u32			 och_magic;
-	fmode_t			 och_flags;
-};
-
-#define OBD_CLIENT_HANDLE_MAGIC 0xd15ea5ed
-
-/* statfs_pack.c */
-void statfs_unpack(struct kstatfs *sfs, struct obd_statfs *osfs);
-
-/*
- * For md echo client
- */
-enum md_echo_cmd {
-	ECHO_MD_CREATE       = 1, /* Open/Create file on MDT */
-	ECHO_MD_MKDIR	= 2, /* Mkdir on MDT */
-	ECHO_MD_DESTROY      = 3, /* Unlink file on MDT */
-	ECHO_MD_RMDIR	= 4, /* Rmdir on MDT */
-	ECHO_MD_LOOKUP       = 5, /* Lookup on MDT */
-	ECHO_MD_GETATTR      = 6, /* Getattr on MDT */
-	ECHO_MD_SETATTR      = 7, /* Setattr on MDT */
-	ECHO_MD_ALLOC_FID    = 8, /* Get FIDs from MDT */
-};
-
-/*
- *   OBD IOCTLS
- */
-#define OBD_IOCTL_VERSION 0x00010004
-
-struct obd_ioctl_data {
-	__u32 ioc_len;
-	__u32 ioc_version;
-
-	union {
-		__u64 ioc_cookie;
-		__u64 ioc_u64_1;
-	};
-	union {
-		__u32 ioc_conn1;
-		__u32 ioc_u32_1;
-	};
-	union {
-		__u32 ioc_conn2;
-		__u32 ioc_u32_2;
-	};
-
-	struct obdo ioc_obdo1;
-	struct obdo ioc_obdo2;
-
-	u64	 ioc_count;
-	u64	 ioc_offset;
-	__u32    ioc_dev;
-	__u32    ioc_command;
-
-	__u64 ioc_nid;
-	__u32 ioc_nal;
-	__u32 ioc_type;
-
-	/* buffers the kernel will treat as user pointers */
-	__u32  ioc_plen1;
-	void __user *ioc_pbuf1;
-	__u32  ioc_plen2;
-	void __user *ioc_pbuf2;
-
-	/* inline buffers for various arguments */
-	__u32  ioc_inllen1;
-	char  *ioc_inlbuf1;
-	__u32  ioc_inllen2;
-	char  *ioc_inlbuf2;
-	__u32  ioc_inllen3;
-	char  *ioc_inlbuf3;
-	__u32  ioc_inllen4;
-	char  *ioc_inlbuf4;
-
-	char    ioc_bulk[0];
-};
-
-struct obd_ioctl_hdr {
-	__u32 ioc_len;
-	__u32 ioc_version;
-};
-
-static inline int obd_ioctl_packlen(struct obd_ioctl_data *data)
-{
-	int len = cfs_size_round(sizeof(struct obd_ioctl_data));
-
-	len += cfs_size_round(data->ioc_inllen1);
-	len += cfs_size_round(data->ioc_inllen2);
-	len += cfs_size_round(data->ioc_inllen3);
-	len += cfs_size_round(data->ioc_inllen4);
-	return len;
-}
-
-static inline int obd_ioctl_is_invalid(struct obd_ioctl_data *data)
-{
-	if (data->ioc_len > OBD_MAX_IOCTL_BUFFER) {
-		CERROR("OBD ioctl: ioc_len larger than %d\n",
-		       OBD_MAX_IOCTL_BUFFER);
-		return 1;
-	}
-	if (data->ioc_inllen1 > OBD_MAX_IOCTL_BUFFER) {
-		CERROR("OBD ioctl: ioc_inllen1 larger than ioc_len\n");
-		return 1;
-	}
-	if (data->ioc_inllen2 > OBD_MAX_IOCTL_BUFFER) {
-		CERROR("OBD ioctl: ioc_inllen2 larger than ioc_len\n");
-		return 1;
-	}
-	if (data->ioc_inllen3 > OBD_MAX_IOCTL_BUFFER) {
-		CERROR("OBD ioctl: ioc_inllen3 larger than ioc_len\n");
-		return 1;
-	}
-	if (data->ioc_inllen4 > OBD_MAX_IOCTL_BUFFER) {
-		CERROR("OBD ioctl: ioc_inllen4 larger than ioc_len\n");
-		return 1;
-	}
-	if (data->ioc_inlbuf1 && !data->ioc_inllen1) {
-		CERROR("OBD ioctl: inlbuf1 pointer but 0 length\n");
-		return 1;
-	}
-	if (data->ioc_inlbuf2 && !data->ioc_inllen2) {
-		CERROR("OBD ioctl: inlbuf2 pointer but 0 length\n");
-		return 1;
-	}
-	if (data->ioc_inlbuf3 && !data->ioc_inllen3) {
-		CERROR("OBD ioctl: inlbuf3 pointer but 0 length\n");
-		return 1;
-	}
-	if (data->ioc_inlbuf4 && !data->ioc_inllen4) {
-		CERROR("OBD ioctl: inlbuf4 pointer but 0 length\n");
-		return 1;
-	}
-	if (data->ioc_pbuf1 && !data->ioc_plen1) {
-		CERROR("OBD ioctl: pbuf1 pointer but 0 length\n");
-		return 1;
-	}
-	if (data->ioc_pbuf2 && !data->ioc_plen2) {
-		CERROR("OBD ioctl: pbuf2 pointer but 0 length\n");
-		return 1;
-	}
-	if (data->ioc_plen1 && !data->ioc_pbuf1) {
-		CERROR("OBD ioctl: plen1 set but NULL pointer\n");
-		return 1;
-	}
-	if (data->ioc_plen2 && !data->ioc_pbuf2) {
-		CERROR("OBD ioctl: plen2 set but NULL pointer\n");
-		return 1;
-	}
-	if (obd_ioctl_packlen(data) > data->ioc_len) {
-		CERROR("OBD ioctl: packlen exceeds ioc_len (%d > %d)\n",
-		       obd_ioctl_packlen(data), data->ioc_len);
-		return 1;
-	}
-	return 0;
-}
-
-#include "obd_support.h"
-
-/* function defined in lustre/obdclass/<platform>/<platform>-module.c */
-int obd_ioctl_getdata(char **buf, int *len, void __user *arg);
-int obd_ioctl_popdata(void __user *arg, void *data, int len);
-
-static inline void obd_ioctl_freedata(char *buf, int len)
-{
-	kvfree(buf);
-	return;
-}
-
-/*
- * BSD ioctl description:
- * #define IOC_V1       _IOR(g, n1, long)
- * #define IOC_V2       _IOW(g, n2, long)
- *
- * ioctl(f, IOC_V1, arg);
- * arg will be treated as a long value,
- *
- * ioctl(f, IOC_V2, arg)
- * arg will be treated as a pointer, bsd will call
- * copyin(buf, arg, sizeof(long))
- *
- * To make BSD ioctl handles argument correctly and simplely,
- * we change _IOR to _IOWR so BSD will copyin obd_ioctl_data
- * for us. Does this change affect Linux?  (XXX Liang)
- */
-#define OBD_IOC_DATA_TYPE long
-
-#define OBD_IOC_CREATE		 _IOWR('f', 101, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_DESTROY		_IOW('f', 104, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_PREALLOCATE	    _IOWR('f', 105, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_SETATTR		_IOW('f', 107, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_GETATTR		_IOWR ('f', 108, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_READ		   _IOWR('f', 109, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_WRITE		  _IOWR('f', 110, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_STATFS		 _IOWR('f', 113, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_SYNC		   _IOW('f', 114, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_READ2		  _IOWR('f', 115, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_FORMAT		 _IOWR('f', 116, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_PARTITION	      _IOWR('f', 117, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_COPY		   _IOWR('f', 120, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_MIGR		   _IOWR('f', 121, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_PUNCH		  _IOWR('f', 122, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_MODULE_DEBUG	   _IOWR('f', 124, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_BRW_READ	       _IOWR('f', 125, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_BRW_WRITE	      _IOWR('f', 126, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_NAME2DEV	       _IOWR('f', 127, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_UUID2DEV	       _IOWR('f', 130, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_GETNAME		_IOWR('f', 131, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_GETMDNAME	      _IOR('f', 131, char[MAX_OBD_NAME])
-#define OBD_IOC_GETDTNAME	       OBD_IOC_GETNAME
-
-#define OBD_IOC_LOV_GET_CONFIG	 _IOWR('f', 132, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_CLIENT_RECOVER	 _IOW('f', 133, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_PING_TARGET	    _IOW('f', 136, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_DEC_FS_USE_COUNT       _IO  ('f', 139)
-#define OBD_IOC_NO_TRANSNO	     _IOW('f', 140, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_SET_READONLY	   _IOW('f', 141, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_ABORT_RECOVERY	 _IOR('f', 142, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_ROOT_SQUASH	    _IOWR('f', 143, OBD_IOC_DATA_TYPE)
-
-#define OBD_GET_VERSION		_IOWR ('f', 144, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_GSS_SUPPORT	    _IOWR('f', 145, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_CLOSE_UUID	     _IOWR ('f', 147, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_CHANGELOG_SEND	 _IOW('f', 148, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_GETDEVICE	      _IOWR ('f', 149, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_FID2PATH	       _IOWR ('f', 150, OBD_IOC_DATA_TYPE)
-/* see also <lustre/lustre_user.h> for ioctls 151-153 */
-/* OBD_IOC_LOV_SETSTRIPE: See also LL_IOC_LOV_SETSTRIPE */
-#define OBD_IOC_LOV_SETSTRIPE	  _IOW('f', 154, OBD_IOC_DATA_TYPE)
-/* OBD_IOC_LOV_GETSTRIPE: See also LL_IOC_LOV_GETSTRIPE */
-#define OBD_IOC_LOV_GETSTRIPE	  _IOW('f', 155, OBD_IOC_DATA_TYPE)
-/* OBD_IOC_LOV_SETEA: See also LL_IOC_LOV_SETEA */
-#define OBD_IOC_LOV_SETEA	      _IOW('f', 156, OBD_IOC_DATA_TYPE)
-/* see <lustre/lustre_user.h> for ioctls 157-159 */
-/* OBD_IOC_QUOTACHECK: See also LL_IOC_QUOTACHECK */
-#define OBD_IOC_QUOTACHECK	     _IOW('f', 160, int)
-/* OBD_IOC_POLL_QUOTACHECK: See also LL_IOC_POLL_QUOTACHECK */
-#define OBD_IOC_POLL_QUOTACHECK	_IOR('f', 161, struct if_quotacheck *)
-/* OBD_IOC_QUOTACTL: See also LL_IOC_QUOTACTL */
-#define OBD_IOC_QUOTACTL	       _IOWR('f', 162, struct if_quotactl)
-/* see  also <lustre/lustre_user.h> for ioctls 163-176 */
-#define OBD_IOC_CHANGELOG_REG	  _IOW('f', 177, struct obd_ioctl_data)
-#define OBD_IOC_CHANGELOG_DEREG	_IOW('f', 178, struct obd_ioctl_data)
-#define OBD_IOC_CHANGELOG_CLEAR	_IOW('f', 179, struct obd_ioctl_data)
-#define OBD_IOC_RECORD		 _IOWR('f', 180, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_ENDRECORD	      _IOWR('f', 181, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_PARSE		  _IOWR('f', 182, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_DORECORD	       _IOWR('f', 183, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_PROCESS_CFG	    _IOWR('f', 184, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_DUMP_LOG	       _IOWR('f', 185, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_CLEAR_LOG	      _IOWR('f', 186, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_PARAM		  _IOW('f', 187, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_POOL		   _IOWR('f', 188, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_REPLACE_NIDS	   _IOWR('f', 189, OBD_IOC_DATA_TYPE)
-
-#define OBD_IOC_CATLOGLIST	     _IOWR('f', 190, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_LLOG_INFO	      _IOWR('f', 191, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_LLOG_PRINT	     _IOWR('f', 192, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_LLOG_CANCEL	    _IOWR('f', 193, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_LLOG_REMOVE	    _IOWR('f', 194, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_LLOG_CHECK	     _IOWR('f', 195, OBD_IOC_DATA_TYPE)
-/* OBD_IOC_LLOG_CATINFO is deprecated */
-#define OBD_IOC_LLOG_CATINFO	   _IOWR('f', 196, OBD_IOC_DATA_TYPE)
-
-/*	#define ECHO_IOC_GET_STRIPE    _IOWR('f', 200, OBD_IOC_DATA_TYPE) */
-/*	#define ECHO_IOC_SET_STRIPE    _IOWR('f', 201, OBD_IOC_DATA_TYPE) */
-/*	#define ECHO_IOC_ENQUEUE       _IOWR('f', 202, OBD_IOC_DATA_TYPE) */
-/*	#define ECHO_IOC_CANCEL        _IOWR('f', 203, OBD_IOC_DATA_TYPE) */
-
-#define OBD_IOC_GET_OBJ_VERSION	_IOR('f', 210, OBD_IOC_DATA_TYPE)
-
-/* <lustre/lustre_user.h> defines ioctl number 218-219 */
-#define OBD_IOC_GET_MNTOPT	     _IOW('f', 220, mntopt_t)
-
-#define OBD_IOC_ECHO_MD		_IOR('f', 221, struct obd_ioctl_data)
-#define OBD_IOC_ECHO_ALLOC_SEQ	 _IOWR('f', 222, struct obd_ioctl_data)
-
-#define OBD_IOC_START_LFSCK	       _IOWR('f', 230, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_STOP_LFSCK	       _IOW('f', 231, OBD_IOC_DATA_TYPE)
-#define OBD_IOC_PAUSE_LFSCK	       _IOW('f', 232, OBD_IOC_DATA_TYPE)
-
-/* XXX _IOWR('f', 250, long) has been defined in
- * libcfs/include/libcfs/libcfs_private.h for debug, don't use it
- */
-
-/* Until such time as we get_info the per-stripe maximum from the OST,
- * we define this to be 2T - 4k, which is the ext3 maxbytes.
- */
-#define LUSTRE_STRIPE_MAXBYTES 0x1fffffff000ULL
-
-/* Special values for remove LOV EA from disk */
-#define LOVEA_DELETE_VALUES(size, count, offset) (size == 0 && count == 0 && \
-						 offset == (typeof(offset))(-1))
-
-/* #define POISON_BULK 0 */
-
 /*
  * l_wait_event is a flexible sleeping function, permitting simple caller
  * configuration of interrupt and timeout sensitivity along with actions to
diff --git a/drivers/staging/lustre/lustre/include/lustre_linkea.h b/drivers/staging/lustre/lustre/include/lustre_linkea.h
new file mode 100644
index 000000000000..249e8bf4fa22
--- /dev/null
+++ b/drivers/staging/lustre/lustre/include/lustre_linkea.h
@@ -0,0 +1,79 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2013, 2014, Intel Corporation.
+ * Use is subject to license terms.
+ *
+ * Author: di wang <di.wang@intel.com>
+ */
+
+#define DEFAULT_LINKEA_SIZE	4096
+
+struct linkea_data {
+	/**
+	 * Buffer to keep link EA body.
+	 */
+	struct lu_buf		*ld_buf;
+	/**
+	 * The matched header, entry and its length in the EA
+	 */
+	struct link_ea_header	*ld_leh;
+	struct link_ea_entry	*ld_lee;
+	int			ld_reclen;
+};
+
+int linkea_data_new(struct linkea_data *ldata, struct lu_buf *buf);
+int linkea_init(struct linkea_data *ldata);
+void linkea_entry_unpack(const struct link_ea_entry *lee, int *reclen,
+			 struct lu_name *lname, struct lu_fid *pfid);
+int linkea_entry_pack(struct link_ea_entry *lee, const struct lu_name *lname,
+		      const struct lu_fid *pfid);
+int linkea_add_buf(struct linkea_data *ldata, const struct lu_name *lname,
+		   const struct lu_fid *pfid);
+void linkea_del_buf(struct linkea_data *ldata, const struct lu_name *lname);
+int linkea_links_find(struct linkea_data *ldata, const struct lu_name *lname,
+		      const struct lu_fid  *pfid);
+
+static inline void linkea_first_entry(struct linkea_data *ldata)
+{
+	LASSERT(ldata);
+	LASSERT(ldata->ld_leh);
+
+	if (ldata->ld_leh->leh_reccount == 0)
+		ldata->ld_lee = NULL;
+	else
+		ldata->ld_lee = (struct link_ea_entry *)(ldata->ld_leh + 1);
+}
+
+static inline void linkea_next_entry(struct linkea_data *ldata)
+{
+	LASSERT(ldata);
+	LASSERT(ldata->ld_leh);
+
+	if (ldata->ld_lee) {
+		ldata->ld_lee = (struct link_ea_entry *)((char *)ldata->ld_lee +
+							 ldata->ld_reclen);
+		if ((char *)ldata->ld_lee >= ((char *)ldata->ld_leh +
+					      ldata->ld_leh->leh_len))
+			ldata->ld_lee = NULL;
+	}
+}
diff --git a/drivers/staging/lustre/lustre/include/lustre_lite.h b/drivers/staging/lustre/lustre/include/lustre_lite.h
deleted file mode 100644
index b16897702559..000000000000
--- a/drivers/staging/lustre/lustre/include/lustre_lite.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2011, 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- */
-
-#ifndef _LL_H
-#define _LL_H
-
-/** \defgroup lite lite
- *
- * @{
- */
-
-#include "linux/lustre_lite.h"
-
-#include "obd_class.h"
-#include "lustre_net.h"
-#include "lustre_mds.h"
-#include "lustre_ha.h"
-
-/* 4UL * 1024 * 1024 */
-#define LL_MAX_BLKSIZE_BITS     (22)
-#define LL_MAX_BLKSIZE	  (1UL<<LL_MAX_BLKSIZE_BITS)
-
-/*
- * This is embedded into llite super-blocks to keep track of
- * connect flags (capabilities) supported by all imports given mount is
- * connected to.
- */
-struct lustre_client_ocd {
-	/*
-	 * This is conjunction of connect_flags across all imports (LOVs) this
-	 * mount is connected to. This field is updated by cl_ocd_update()
-	 * under ->lco_lock.
-	 */
-	__u64	      lco_flags;
-	struct mutex	   lco_lock;
-	struct obd_export *lco_md_exp;
-	struct obd_export *lco_dt_exp;
-};
-
-/*
- * Chain of hash overflow pages.
- */
-struct ll_dir_chain {
-	/* XXX something. Later */
-};
-
-static inline void ll_dir_chain_init(struct ll_dir_chain *chain)
-{
-}
-
-static inline void ll_dir_chain_fini(struct ll_dir_chain *chain)
-{
-}
-
-static inline unsigned long hash_x_index(__u64 hash, int hash64)
-{
-	if (BITS_PER_LONG == 32 && hash64)
-		hash >>= 32;
-	/* save hash 0 as index 0 because otherwise we'll save it at
-	 * page index end (~0UL) and it causes truncate_inode_pages_range()
-	 * to loop forever.
-	 */
-	return ~0UL - (hash + !hash);
-}
-
-/** @} lite */
-
-#endif
diff --git a/drivers/staging/lustre/lustre/include/lustre_lmv.h b/drivers/staging/lustre/lustre/include/lustre_lmv.h
new file mode 100644
index 000000000000..d7f7afa8dfa7
--- /dev/null
+++ b/drivers/staging/lustre/lustre/include/lustre_lmv.h
@@ -0,0 +1,184 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License version 2 for more details.  A copy is
+ * included in the COPYING file that accompanied this code.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2013, Intel Corporation.
+ */
+/*
+ * lustre/include/lustre_lmv.h
+ *
+ * Lustre LMV structures and functions.
+ *
+ * Author: Di Wang <di.wang@intel.com>
+ */
+
+#ifndef _LUSTRE_LMV_H
+#define _LUSTRE_LMV_H
+#include "lustre/lustre_idl.h"
+
+struct lmv_oinfo {
+	struct lu_fid	lmo_fid;
+	u32		lmo_mds;
+	struct inode	*lmo_root;
+};
+
+struct lmv_stripe_md {
+	__u32	lsm_md_magic;
+	__u32	lsm_md_stripe_count;
+	__u32	lsm_md_master_mdt_index;
+	__u32	lsm_md_hash_type;
+	__u32	lsm_md_layout_version;
+	__u32	lsm_md_default_count;
+	__u32	lsm_md_default_index;
+	char	lsm_md_pool_name[LOV_MAXPOOLNAME + 1];
+	struct lmv_oinfo lsm_md_oinfo[0];
+};
+
+static inline bool
+lsm_md_eq(const struct lmv_stripe_md *lsm1, const struct lmv_stripe_md *lsm2)
+{
+	__u32 idx;
+
+	if (lsm1->lsm_md_magic != lsm2->lsm_md_magic ||
+	    lsm1->lsm_md_stripe_count != lsm2->lsm_md_stripe_count ||
+	    lsm1->lsm_md_master_mdt_index != lsm2->lsm_md_master_mdt_index ||
+	    lsm1->lsm_md_hash_type != lsm2->lsm_md_hash_type ||
+	    lsm1->lsm_md_layout_version != lsm2->lsm_md_layout_version ||
+	    !strcmp(lsm1->lsm_md_pool_name, lsm2->lsm_md_pool_name))
+		return false;
+
+	for (idx = 0; idx < lsm1->lsm_md_stripe_count; idx++) {
+		if (!lu_fid_eq(&lsm1->lsm_md_oinfo[idx].lmo_fid,
+			       &lsm2->lsm_md_oinfo[idx].lmo_fid))
+			return false;
+	}
+
+	return true;
+}
+
+union lmv_mds_md;
+
+int lmv_unpack_md(struct obd_export *exp, struct lmv_stripe_md **lsmp,
+		  const union lmv_mds_md *lmm, int stripe_count);
+
+static inline int lmv_alloc_memmd(struct lmv_stripe_md **lsmp, int stripe_count)
+{
+	return lmv_unpack_md(NULL, lsmp, NULL, stripe_count);
+}
+
+static inline void lmv_free_memmd(struct lmv_stripe_md *lsm)
+{
+	lmv_unpack_md(NULL, &lsm, NULL, 0);
+}
+
+static inline void lmv1_le_to_cpu(struct lmv_mds_md_v1 *lmv_dst,
+				  const struct lmv_mds_md_v1 *lmv_src)
+{
+	__u32 i;
+
+	lmv_dst->lmv_magic = le32_to_cpu(lmv_src->lmv_magic);
+	lmv_dst->lmv_stripe_count = le32_to_cpu(lmv_src->lmv_stripe_count);
+	lmv_dst->lmv_master_mdt_index =
+		le32_to_cpu(lmv_src->lmv_master_mdt_index);
+	lmv_dst->lmv_hash_type = le32_to_cpu(lmv_src->lmv_hash_type);
+	lmv_dst->lmv_layout_version = le32_to_cpu(lmv_src->lmv_layout_version);
+
+	for (i = 0; i < lmv_src->lmv_stripe_count; i++)
+		fid_le_to_cpu(&lmv_dst->lmv_stripe_fids[i],
+			      &lmv_src->lmv_stripe_fids[i]);
+}
+
+static inline void lmv_le_to_cpu(union lmv_mds_md *lmv_dst,
+				 const union lmv_mds_md *lmv_src)
+{
+	switch (le32_to_cpu(lmv_src->lmv_magic)) {
+	case LMV_MAGIC_V1:
+		lmv1_le_to_cpu(&lmv_dst->lmv_md_v1, &lmv_src->lmv_md_v1);
+		break;
+	default:
+		break;
+	}
+}
+
+/* This hash is only for testing purpose */
+static inline unsigned int
+lmv_hash_all_chars(unsigned int count, const char *name, int namelen)
+{
+	const unsigned char *p = (const unsigned char *)name;
+	unsigned int c = 0;
+
+	while (--namelen >= 0)
+		c += p[namelen];
+
+	c = c % count;
+
+	return c;
+}
+
+static inline unsigned int
+lmv_hash_fnv1a(unsigned int count, const char *name, int namelen)
+{
+	__u64 hash;
+
+	hash = lustre_hash_fnv_1a_64(name, namelen);
+
+	return do_div(hash, count);
+}
+
+static inline int lmv_name_to_stripe_index(__u32 lmv_hash_type,
+					   unsigned int stripe_count,
+					   const char *name, int namelen)
+{
+	__u32 hash_type = lmv_hash_type & LMV_HASH_TYPE_MASK;
+	int idx;
+
+	LASSERT(namelen > 0);
+	if (stripe_count <= 1)
+		return 0;
+
+	/* for migrating object, always start from 0 stripe */
+	if (lmv_hash_type & LMV_HASH_FLAG_MIGRATION)
+		return 0;
+
+	switch (hash_type) {
+	case LMV_HASH_TYPE_ALL_CHARS:
+		idx = lmv_hash_all_chars(stripe_count, name, namelen);
+		break;
+	case LMV_HASH_TYPE_FNV_1A_64:
+		idx = lmv_hash_fnv1a(stripe_count, name, namelen);
+		break;
+	default:
+		idx = -EBADFD;
+		break;
+	}
+	CDEBUG(D_INFO, "name %.*s hash_type %d idx %d\n", namelen, name,
+	       hash_type, idx);
+
+	return idx;
+}
+
+static inline bool lmv_is_known_hash_type(__u32 type)
+{
+	return (type & LMV_HASH_TYPE_MASK) == LMV_HASH_TYPE_FNV_1A_64 ||
+	       (type & LMV_HASH_TYPE_MASK) == LMV_HASH_TYPE_ALL_CHARS;
+}
+
+#endif
diff --git a/drivers/staging/lustre/lustre/include/lustre_log.h b/drivers/staging/lustre/lustre/include/lustre_log.h
index b96e02317bfc..995b266932e3 100644
--- a/drivers/staging/lustre/lustre/include/lustre_log.h
+++ b/drivers/staging/lustre/lustre/include/lustre_log.h
@@ -277,12 +277,11 @@ static inline void llog_ctxt_put(struct llog_ctxt *ctxt)
 	__llog_ctxt_put(NULL, ctxt);
 }
 
-static inline void llog_group_init(struct obd_llog_group *olg, int group)
+static inline void llog_group_init(struct obd_llog_group *olg)
 {
 	init_waitqueue_head(&olg->olg_waitq);
 	spin_lock_init(&olg->olg_lock);
 	mutex_init(&olg->olg_cat_processing);
-	olg->olg_seq = group;
 }
 
 static inline int llog_group_set_ctxt(struct obd_llog_group *olg,
diff --git a/drivers/staging/lustre/lustre/include/lustre_mdc.h b/drivers/staging/lustre/lustre/include/lustre_mdc.h
index fa62b95d351f..8fc2d3f2dfd6 100644
--- a/drivers/staging/lustre/lustre/include/lustre_mdc.h
+++ b/drivers/staging/lustre/lustre/include/lustre_mdc.h
@@ -96,7 +96,7 @@ static inline void mdc_get_rpc_lock(struct mdc_rpc_lock *lck,
 				    struct lookup_intent *it)
 {
 	if (it && (it->it_op == IT_GETATTR || it->it_op == IT_LOOKUP ||
-		   it->it_op == IT_LAYOUT))
+		   it->it_op == IT_LAYOUT || it->it_op == IT_READDIR))
 		return;
 
 	/* This would normally block until the existing request finishes.
@@ -136,7 +136,7 @@ static inline void mdc_put_rpc_lock(struct mdc_rpc_lock *lck,
 				    struct lookup_intent *it)
 {
 	if (it && (it->it_op == IT_GETATTR || it->it_op == IT_LOOKUP ||
-		   it->it_op == IT_LAYOUT))
+		   it->it_op == IT_LAYOUT || it->it_op == IT_READDIR))
 		return;
 
 	if (lck->rpcl_it == MDC_FAKE_RPCL_IT) { /* OBD_FAIL_MDC_RPCS_SEM */
@@ -156,34 +156,44 @@ static inline void mdc_put_rpc_lock(struct mdc_rpc_lock *lck,
 	mutex_unlock(&lck->rpcl_mutex);
 }
 
-/* Update the maximum observed easize and cookiesize.  The default easize
- * and cookiesize is initialized to the minimum value but allowed to grow
- * up to a single page in size if required to handle the common case.
+/**
+ * Update the maximum possible easize and cookiesize.
+ *
+ * The values are learned from ptlrpc replies sent by the MDT.  The
+ * default easize and cookiesize is initialized to the minimum value but
+ * allowed to grow up to a single page in size if required to handle the
+ * common case.
+ *
+ * \see client_obd::cl_default_mds_easize and
+ * client_obd::cl_default_mds_cookiesize
+ *
+ * \param[in] exp	export for MDC device
+ * \param[in] body	body of ptlrpc reply from MDT
+ *
  */
 static inline void mdc_update_max_ea_from_body(struct obd_export *exp,
 					       struct mdt_body *body)
 {
-	if (body->valid & OBD_MD_FLMODEASIZE) {
+	if (body->mbo_valid & OBD_MD_FLMODEASIZE) {
 		struct client_obd *cli = &exp->exp_obd->u.cli;
+		u32 def_cookiesize, def_easize;
 
-		if (cli->cl_max_mds_easize < body->max_mdsize) {
-			cli->cl_max_mds_easize = body->max_mdsize;
-			cli->cl_default_mds_easize =
-			    min_t(__u32, body->max_mdsize, PAGE_SIZE);
-		}
-		if (cli->cl_max_mds_cookiesize < body->max_cookiesize) {
-			cli->cl_max_mds_cookiesize = body->max_cookiesize;
-			cli->cl_default_mds_cookiesize =
-			    min_t(__u32, body->max_cookiesize, PAGE_SIZE);
-		}
+		if (cli->cl_max_mds_easize < body->mbo_max_mdsize)
+			cli->cl_max_mds_easize = body->mbo_max_mdsize;
+
+		def_easize = min_t(__u32, body->mbo_max_mdsize,
+				   OBD_MAX_DEFAULT_EA_SIZE);
+		cli->cl_default_mds_easize = def_easize;
+
+		if (cli->cl_max_mds_cookiesize < body->mbo_max_cookiesize)
+			cli->cl_max_mds_cookiesize = body->mbo_max_cookiesize;
+
+		def_cookiesize = min_t(__u32, body->mbo_max_cookiesize,
+				       OBD_MAX_DEFAULT_COOKIE_SIZE);
+		cli->cl_default_mds_cookiesize = def_cookiesize;
 	}
 }
 
-struct mdc_cache_waiter {
-	struct list_head	      mcw_entry;
-	wait_queue_head_t	     mcw_waitq;
-};
-
 /* mdc/mdc_locks.c */
 int it_open_error(int phase, struct lookup_intent *it);
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_mds.h b/drivers/staging/lustre/lustre/include/lustre_mds.h
index 4104bd9bd5c4..23a7e4f78e9a 100644
--- a/drivers/staging/lustre/lustre/include/lustre_mds.h
+++ b/drivers/staging/lustre/lustre/include/lustre_mds.h
@@ -58,9 +58,6 @@ struct mds_group_info {
 #define MDD_OBD_NAME     "mdd_obd"
 #define MDD_OBD_UUID     "mdd_obd_uuid"
 
-/* these are local flags, used only on the client, private */
-#define M_CHECK_STALE	   0200000000
-
 /** @} mds */
 
 #endif
diff --git a/drivers/staging/lustre/lustre/include/lustre_net.h b/drivers/staging/lustre/lustre/include/lustre_net.h
index d5debd615fdf..e9aba99ee52a 100644
--- a/drivers/staging/lustre/lustre/include/lustre_net.h
+++ b/drivers/staging/lustre/lustre/include/lustre_net.h
@@ -261,7 +261,10 @@
 
 #define MDS_MAXREQSIZE		(5 * 1024)	/* >= 4736 */
 
-#define OST_MAXREQSIZE		(5 * 1024)
+/**
+ * FIEMAP request can be 4K+ for now
+ */
+#define OST_MAXREQSIZE		(16 * 1024)
 
 /* Macro to hide a typecast. */
 #define ptlrpc_req_async_args(req) ((void *)&req->rq_async_args)
@@ -570,13 +573,13 @@ struct ptlrpc_nrs_pol_ops {
 	 *
 	 * \param[in,out] policy The policy being initialized
 	 */
-	int	(*op_policy_init) (struct ptlrpc_nrs_policy *policy);
+	int	(*op_policy_init)(struct ptlrpc_nrs_policy *policy);
 	/**
 	 * Called during policy unregistration; this operation is optional.
 	 *
 	 * \param[in,out] policy The policy being unregistered/finalized
 	 */
-	void	(*op_policy_fini) (struct ptlrpc_nrs_policy *policy);
+	void	(*op_policy_fini)(struct ptlrpc_nrs_policy *policy);
 	/**
 	 * Called when activating a policy via lprocfs; policies allocate and
 	 * initialize their resources here; this operation is optional.
@@ -585,7 +588,7 @@ struct ptlrpc_nrs_pol_ops {
 	 *
 	 * \see nrs_policy_start_locked()
 	 */
-	int	(*op_policy_start) (struct ptlrpc_nrs_policy *policy);
+	int	(*op_policy_start)(struct ptlrpc_nrs_policy *policy);
 	/**
 	 * Called when deactivating a policy via lprocfs; policies deallocate
 	 * their resources here; this operation is optional
@@ -594,7 +597,7 @@ struct ptlrpc_nrs_pol_ops {
 	 *
 	 * \see nrs_policy_stop0()
 	 */
-	void	(*op_policy_stop) (struct ptlrpc_nrs_policy *policy);
+	void	(*op_policy_stop)(struct ptlrpc_nrs_policy *policy);
 	/**
 	 * Used for policy-specific operations; i.e. not generic ones like
 	 * \e PTLRPC_NRS_CTL_START and \e PTLRPC_NRS_CTL_GET_INFO; analogous
@@ -610,8 +613,8 @@ struct ptlrpc_nrs_pol_ops {
 	 *
 	 * \see ptlrpc_nrs_policy_control()
 	 */
-	int	(*op_policy_ctl) (struct ptlrpc_nrs_policy *policy,
-				  enum ptlrpc_nrs_ctl opc, void *arg);
+	int	(*op_policy_ctl)(struct ptlrpc_nrs_policy *policy,
+				 enum ptlrpc_nrs_ctl opc, void *arg);
 
 	/**
 	 * Called when obtaining references to the resources of the resource
@@ -648,11 +651,11 @@ struct ptlrpc_nrs_pol_ops {
 	 * \see ptlrpc_nrs_req_initialize()
 	 * \see ptlrpc_nrs_hpreq_add_nolock()
 	 */
-	int	(*op_res_get) (struct ptlrpc_nrs_policy *policy,
-			       struct ptlrpc_nrs_request *nrq,
-			       const struct ptlrpc_nrs_resource *parent,
-			       struct ptlrpc_nrs_resource **resp,
-			       bool moving_req);
+	int	(*op_res_get)(struct ptlrpc_nrs_policy *policy,
+			      struct ptlrpc_nrs_request *nrq,
+			      const struct ptlrpc_nrs_resource *parent,
+			      struct ptlrpc_nrs_resource **resp,
+			      bool moving_req);
 	/**
 	 * Called when releasing references taken for resources in the resource
 	 * hierarchy for the request; this operation is optional.
@@ -663,8 +666,8 @@ struct ptlrpc_nrs_pol_ops {
 	 * \see ptlrpc_nrs_req_finalize()
 	 * \see ptlrpc_nrs_hpreq_add_nolock()
 	 */
-	void	(*op_res_put) (struct ptlrpc_nrs_policy *policy,
-			       const struct ptlrpc_nrs_resource *res);
+	void	(*op_res_put)(struct ptlrpc_nrs_policy *policy,
+			      const struct ptlrpc_nrs_resource *res);
 
 	/**
 	 * Obtains a request for handling from the policy, and optionally
@@ -683,8 +686,8 @@ struct ptlrpc_nrs_pol_ops {
 	 * \see ptlrpc_nrs_req_get_nolock()
 	 */
 	struct ptlrpc_nrs_request *
-		(*op_req_get) (struct ptlrpc_nrs_policy *policy, bool peek,
-			       bool force);
+		(*op_req_get)(struct ptlrpc_nrs_policy *policy, bool peek,
+			      bool force);
 	/**
 	 * Called when attempting to add a request to a policy for later
 	 * handling; this operation is mandatory.
@@ -697,8 +700,8 @@ struct ptlrpc_nrs_pol_ops {
 	 *
 	 * \see ptlrpc_nrs_req_add_nolock()
 	 */
-	int	(*op_req_enqueue) (struct ptlrpc_nrs_policy *policy,
-				   struct ptlrpc_nrs_request *nrq);
+	int	(*op_req_enqueue)(struct ptlrpc_nrs_policy *policy,
+				  struct ptlrpc_nrs_request *nrq);
 	/**
 	 * Removes a request from the policy's set of pending requests. Normally
 	 * called after a request has been polled successfully from the policy
@@ -707,8 +710,8 @@ struct ptlrpc_nrs_pol_ops {
 	 * \param[in,out] policy The policy the request \a nrq belongs to
 	 * \param[in,out] nrq    The request to dequeue
 	 */
-	void	(*op_req_dequeue) (struct ptlrpc_nrs_policy *policy,
-				   struct ptlrpc_nrs_request *nrq);
+	void	(*op_req_dequeue)(struct ptlrpc_nrs_policy *policy,
+				  struct ptlrpc_nrs_request *nrq);
 	/**
 	 * Called after the request being carried out. Could be used for
 	 * job/resource control; this operation is optional.
@@ -721,8 +724,8 @@ struct ptlrpc_nrs_pol_ops {
 	 *
 	 * \see ptlrpc_nrs_req_stop_nolock()
 	 */
-	void	(*op_req_stop) (struct ptlrpc_nrs_policy *policy,
-				struct ptlrpc_nrs_request *nrq);
+	void	(*op_req_stop)(struct ptlrpc_nrs_policy *policy,
+			       struct ptlrpc_nrs_request *nrq);
 	/**
 	 * Registers the policy's lprocfs interface with a PTLRPC service.
 	 *
@@ -731,7 +734,7 @@ struct ptlrpc_nrs_pol_ops {
 	 * \retval 0	success
 	 * \retval != 0	error
 	 */
-	int	(*op_lprocfs_init) (struct ptlrpc_service *svc);
+	int	(*op_lprocfs_init)(struct ptlrpc_service *svc);
 	/**
 	 * Unegisters the policy's lprocfs interface with a PTLRPC service.
 	 *
@@ -743,7 +746,7 @@ struct ptlrpc_nrs_pol_ops {
 	 *
 	 * \param[in] svc The service
 	 */
-	void	(*op_lprocfs_fini) (struct ptlrpc_service *svc);
+	void	(*op_lprocfs_fini)(struct ptlrpc_service *svc);
 };
 
 /**
@@ -1628,7 +1631,7 @@ static inline bool ptlrpc_nrs_req_can_move(struct ptlrpc_request *req)
 /**
  * Returns 1 if request buffer at offset \a index was already swabbed
  */
-static inline int lustre_req_swabbed(struct ptlrpc_request *req, int index)
+static inline int lustre_req_swabbed(struct ptlrpc_request *req, size_t index)
 {
 	LASSERT(index < sizeof(req->rq_req_swab_mask) * 8);
 	return req->rq_req_swab_mask & (1 << index);
@@ -1637,7 +1640,7 @@ static inline int lustre_req_swabbed(struct ptlrpc_request *req, int index)
 /**
  * Returns 1 if request reply buffer at offset \a index was already swabbed
  */
-static inline int lustre_rep_swabbed(struct ptlrpc_request *req, int index)
+static inline int lustre_rep_swabbed(struct ptlrpc_request *req, size_t index)
 {
 	LASSERT(index < sizeof(req->rq_rep_swab_mask) * 8);
 	return req->rq_rep_swab_mask & (1 << index);
@@ -1662,7 +1665,8 @@ static inline int ptlrpc_rep_need_swab(struct ptlrpc_request *req)
 /**
  * Mark request buffer at offset \a index that it was already swabbed
  */
-static inline void lustre_set_req_swabbed(struct ptlrpc_request *req, int index)
+static inline void lustre_set_req_swabbed(struct ptlrpc_request *req,
+					  size_t index)
 {
 	LASSERT(index < sizeof(req->rq_req_swab_mask) * 8);
 	LASSERT((req->rq_req_swab_mask & (1 << index)) == 0);
@@ -1672,7 +1676,8 @@ static inline void lustre_set_req_swabbed(struct ptlrpc_request *req, int index)
 /**
  * Mark request reply buffer at offset \a index that it was already swabbed
  */
-static inline void lustre_set_rep_swabbed(struct ptlrpc_request *req, int index)
+static inline void lustre_set_rep_swabbed(struct ptlrpc_request *req,
+					  size_t index)
 {
 	LASSERT(index < sizeof(req->rq_rep_swab_mask) * 8);
 	LASSERT((req->rq_rep_swab_mask & (1 << index)) == 0);
@@ -2403,7 +2408,6 @@ int ptlrpc_send_reply(struct ptlrpc_request *req, int flags);
 int ptlrpc_reply(struct ptlrpc_request *req);
 int ptlrpc_send_error(struct ptlrpc_request *req, int difficult);
 int ptlrpc_error(struct ptlrpc_request *req);
-void ptlrpc_resend_req(struct ptlrpc_request *request);
 int ptlrpc_at_get_net_latency(struct ptlrpc_request *req);
 int ptl_send_rpc(struct ptlrpc_request *request, int noreply);
 int ptlrpc_register_rqbd(struct ptlrpc_request_buffer_desc *rqbd);
@@ -2423,23 +2427,17 @@ struct ptlrpc_connection *ptlrpc_uuid_to_connection(struct obd_uuid *uuid);
 
 int ptlrpc_queue_wait(struct ptlrpc_request *req);
 int ptlrpc_replay_req(struct ptlrpc_request *req);
-int ptlrpc_unregister_reply(struct ptlrpc_request *req, int async);
 void ptlrpc_abort_inflight(struct obd_import *imp);
 void ptlrpc_abort_set(struct ptlrpc_request_set *set);
 
 struct ptlrpc_request_set *ptlrpc_prep_set(void);
 struct ptlrpc_request_set *ptlrpc_prep_fcset(int max, set_producer_func func,
 					     void *arg);
-int ptlrpc_set_next_timeout(struct ptlrpc_request_set *);
 int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set);
 int ptlrpc_set_wait(struct ptlrpc_request_set *);
-int ptlrpc_expired_set(void *data);
-void ptlrpc_interrupted_set(void *data);
 void ptlrpc_mark_interrupted(struct ptlrpc_request *req);
 void ptlrpc_set_destroy(struct ptlrpc_request_set *);
 void ptlrpc_set_add_req(struct ptlrpc_request_set *, struct ptlrpc_request *);
-void ptlrpc_set_add_new_req(struct ptlrpcd_ctl *pc,
-			    struct ptlrpc_request *req);
 
 void ptlrpc_free_rq_pool(struct ptlrpc_request_pool *pool);
 int ptlrpc_add_rqs_to_pool(struct ptlrpc_request_pool *pool, int num_rq);
@@ -2611,9 +2609,9 @@ int ptlrpc_reconnect_import(struct obd_import *imp);
  * @{
  */
 int ptlrpc_buf_need_swab(struct ptlrpc_request *req, const int inout,
-			 int index);
+			 u32 index);
 void ptlrpc_buf_set_swabbed(struct ptlrpc_request *req, const int inout,
-			    int index);
+			    u32 index);
 int ptlrpc_unpack_rep_msg(struct ptlrpc_request *req, int len);
 int ptlrpc_unpack_req_msg(struct ptlrpc_request *req, int len);
 
@@ -2632,27 +2630,27 @@ int lustre_shrink_msg(struct lustre_msg *msg, int segment,
 		      unsigned int newlen, int move_data);
 void lustre_free_reply_state(struct ptlrpc_reply_state *rs);
 int __lustre_unpack_msg(struct lustre_msg *m, int len);
-int lustre_msg_hdr_size(__u32 magic, int count);
-int lustre_msg_size(__u32 magic, int count, __u32 *lengths);
-int lustre_msg_size_v2(int count, __u32 *lengths);
-int lustre_packed_msg_size(struct lustre_msg *msg);
-int lustre_msg_early_size(void);
-void *lustre_msg_buf_v2(struct lustre_msg_v2 *m, int n, int min_size);
-void *lustre_msg_buf(struct lustre_msg *m, int n, int minlen);
-int lustre_msg_buflen(struct lustre_msg *m, int n);
-int lustre_msg_bufcount(struct lustre_msg *m);
-char *lustre_msg_string(struct lustre_msg *m, int n, int max_len);
+u32 lustre_msg_hdr_size(__u32 magic, u32 count);
+u32 lustre_msg_size(__u32 magic, int count, __u32 *lengths);
+u32 lustre_msg_size_v2(int count, __u32 *lengths);
+u32 lustre_packed_msg_size(struct lustre_msg *msg);
+u32 lustre_msg_early_size(void);
+void *lustre_msg_buf_v2(struct lustre_msg_v2 *m, u32 n, u32 min_size);
+void *lustre_msg_buf(struct lustre_msg *m, u32 n, u32 minlen);
+u32 lustre_msg_buflen(struct lustre_msg *m, u32 n);
+u32 lustre_msg_bufcount(struct lustre_msg *m);
+char *lustre_msg_string(struct lustre_msg *m, u32 n, u32 max_len);
 __u32 lustre_msghdr_get_flags(struct lustre_msg *msg);
 void lustre_msghdr_set_flags(struct lustre_msg *msg, __u32 flags);
 __u32 lustre_msg_get_flags(struct lustre_msg *msg);
-void lustre_msg_add_flags(struct lustre_msg *msg, int flags);
-void lustre_msg_set_flags(struct lustre_msg *msg, int flags);
-void lustre_msg_clear_flags(struct lustre_msg *msg, int flags);
+void lustre_msg_add_flags(struct lustre_msg *msg, u32 flags);
+void lustre_msg_set_flags(struct lustre_msg *msg, u32 flags);
+void lustre_msg_clear_flags(struct lustre_msg *msg, u32 flags);
 __u32 lustre_msg_get_op_flags(struct lustre_msg *msg);
-void lustre_msg_add_op_flags(struct lustre_msg *msg, int flags);
+void lustre_msg_add_op_flags(struct lustre_msg *msg, u32 flags);
 struct lustre_handle *lustre_msg_get_handle(struct lustre_msg *msg);
 __u32 lustre_msg_get_type(struct lustre_msg *msg);
-void lustre_msg_add_version(struct lustre_msg *msg, int version);
+void lustre_msg_add_version(struct lustre_msg *msg, u32 version);
 __u32 lustre_msg_get_opc(struct lustre_msg *msg);
 __u64 lustre_msg_get_last_committed(struct lustre_msg *msg);
 __u64 *lustre_msg_get_versions(struct lustre_msg *msg);
diff --git a/drivers/staging/lustre/lustre/include/lustre_param.h b/drivers/staging/lustre/lustre/include/lustre_param.h
index 82aadd32c2b8..8061a04ee806 100644
--- a/drivers/staging/lustre/lustre/include/lustre_param.h
+++ b/drivers/staging/lustre/lustre/include/lustre_param.h
@@ -39,6 +39,9 @@
 #ifndef _LUSTRE_PARAM_H
 #define _LUSTRE_PARAM_H
 
+#include "../../include/linux/libcfs/libcfs.h"
+#include "../../include/linux/lnet/types.h"
+
 /** \defgroup param param
  *
  * @{
diff --git a/drivers/staging/lustre/lustre/include/linux/lustre_patchless_compat.h b/drivers/staging/lustre/lustre/include/lustre_patchless_compat.h
index 5842cb18b49e..5842cb18b49e 100644
--- a/drivers/staging/lustre/lustre/include/linux/lustre_patchless_compat.h
+++ b/drivers/staging/lustre/lustre/include/lustre_patchless_compat.h
diff --git a/drivers/staging/lustre/lustre/include/lustre_req_layout.h b/drivers/staging/lustre/lustre/include/lustre_req_layout.h
index 544a43c862b9..a13558e53274 100644
--- a/drivers/staging/lustre/lustre/include/lustre_req_layout.h
+++ b/drivers/staging/lustre/lustre/include/lustre_req_layout.h
@@ -76,7 +76,8 @@ void req_capsule_init(struct req_capsule *pill, struct ptlrpc_request *req,
 void req_capsule_fini(struct req_capsule *pill);
 
 void req_capsule_set(struct req_capsule *pill, const struct req_format *fmt);
-int req_capsule_filled_sizes(struct req_capsule *pill, enum req_location loc);
+size_t req_capsule_filled_sizes(struct req_capsule *pill,
+				enum req_location loc);
 int  req_capsule_server_pack(struct req_capsule *pill);
 
 void *req_capsule_client_get(struct req_capsule *pill,
@@ -86,27 +87,27 @@ void *req_capsule_client_swab_get(struct req_capsule *pill,
 				  void *swabber);
 void *req_capsule_client_sized_get(struct req_capsule *pill,
 				   const struct req_msg_field *field,
-				   int len);
+				   u32 len);
 void *req_capsule_server_get(struct req_capsule *pill,
 			     const struct req_msg_field *field);
 void *req_capsule_server_sized_get(struct req_capsule *pill,
 				   const struct req_msg_field *field,
-				   int len);
+				   u32 len);
 void *req_capsule_server_swab_get(struct req_capsule *pill,
 				  const struct req_msg_field *field,
 				  void *swabber);
 void *req_capsule_server_sized_swab_get(struct req_capsule *pill,
 					const struct req_msg_field *field,
-					int len, void *swabber);
+					u32 len, void *swabber);
 
 void req_capsule_set_size(struct req_capsule *pill,
 			  const struct req_msg_field *field,
-			  enum req_location loc, int size);
-int req_capsule_get_size(const struct req_capsule *pill,
+			  enum req_location loc, u32 size);
+u32 req_capsule_get_size(const struct req_capsule *pill,
 			 const struct req_msg_field *field,
 			 enum req_location loc);
-int req_capsule_msg_size(struct req_capsule *pill, enum req_location loc);
-int req_capsule_fmt_size(__u32 magic, const struct req_format *fmt,
+u32 req_capsule_msg_size(struct req_capsule *pill, enum req_location loc);
+u32 req_capsule_fmt_size(__u32 magic, const struct req_format *fmt,
 			 enum req_location loc);
 void req_capsule_extend(struct req_capsule *pill, const struct req_format *fmt);
 
@@ -115,8 +116,7 @@ int req_capsule_has_field(const struct req_capsule *pill,
 			  enum req_location loc);
 void req_capsule_shrink(struct req_capsule *pill,
 			const struct req_msg_field *field,
-			unsigned int newlen,
-			enum req_location loc);
+			u32 newlen, enum req_location loc);
 int  req_layout_init(void);
 void req_layout_fini(void);
 
@@ -149,14 +149,11 @@ extern struct req_format RQF_MDS_GETATTR;
 extern struct req_format RQF_MDS_GETATTR_NAME;
 extern struct req_format RQF_MDS_CLOSE;
 extern struct req_format RQF_MDS_RELEASE_CLOSE;
-extern struct req_format RQF_MDS_PIN;
-extern struct req_format RQF_MDS_UNPIN;
 extern struct req_format RQF_MDS_CONNECT;
 extern struct req_format RQF_MDS_DISCONNECT;
 extern struct req_format RQF_MDS_GET_INFO;
 extern struct req_format RQF_MDS_READPAGE;
 extern struct req_format RQF_MDS_WRITEPAGE;
-extern struct req_format RQF_MDS_IS_SUBDIR;
 extern struct req_format RQF_MDS_DONE_WRITING;
 extern struct req_format RQF_MDS_REINT;
 extern struct req_format RQF_MDS_REINT_CREATE;
diff --git a/drivers/staging/lustre/lustre/include/lustre_ver.h b/drivers/staging/lustre/lustre/include/lustre_ver.h
index 64559a16f4de..19c9135e2273 100644
--- a/drivers/staging/lustre/lustre/include/lustre_ver.h
+++ b/drivers/staging/lustre/lustre/include/lustre_ver.h
@@ -2,14 +2,21 @@
 #define _LUSTRE_VER_H_
 
 #define LUSTRE_MAJOR 2
-#define LUSTRE_MINOR 4
-#define LUSTRE_PATCH 60
+#define LUSTRE_MINOR 6
+#define LUSTRE_PATCH 99
 #define LUSTRE_FIX 0
-#define LUSTRE_VERSION_STRING "2.4.60"
+#define LUSTRE_VERSION_STRING "2.6.99"
 
-#define LUSTRE_VERSION_CODE OBD_OCD_VERSION(LUSTRE_MAJOR, \
-					    LUSTRE_MINOR, LUSTRE_PATCH, \
-					    LUSTRE_FIX)
+#define OBD_OCD_VERSION(major, minor, patch, fix)			\
+	(((major) << 24) + ((minor) << 16) + ((patch) << 8) + (fix))
+
+#define OBD_OCD_VERSION_MAJOR(version)	((int)((version) >> 24) & 255)
+#define OBD_OCD_VERSION_MINOR(version)	((int)((version) >> 16) & 255)
+#define OBD_OCD_VERSION_PATCH(version)	((int)((version) >>  8) & 255)
+#define OBD_OCD_VERSION_FIX(version)	((int)((version) >>  0) & 255)
+
+#define LUSTRE_VERSION_CODE						\
+	OBD_OCD_VERSION(LUSTRE_MAJOR, LUSTRE_MINOR, LUSTRE_PATCH, LUSTRE_FIX)
 
 /*
  * If lustre version of client and servers it connects to differs by more
diff --git a/drivers/staging/lustre/lustre/include/obd.h b/drivers/staging/lustre/lustre/include/obd.h
index a1bc2c478ff9..f6fc4dd05bd6 100644
--- a/drivers/staging/lustre/lustre/include/obd.h
+++ b/drivers/staging/lustre/lustre/include/obd.h
@@ -35,21 +35,13 @@
 
 #include <linux/spinlock.h>
 
-#define IOC_OSC_TYPE	 'h'
-#define IOC_OSC_MIN_NR       20
-#define IOC_OSC_SET_ACTIVE   _IOWR(IOC_OSC_TYPE, 21, struct obd_device *)
-#define IOC_OSC_MAX_NR       50
-
-#define IOC_MDC_TYPE	 'i'
-#define IOC_MDC_MIN_NR       20
-#define IOC_MDC_MAX_NR       50
-
 #include "lustre/lustre_idl.h"
 #include "lustre_lib.h"
 #include "lu_ref.h"
 #include "lustre_export.h"
 #include "lustre_fid.h"
 #include "lustre_fld.h"
+#include "lustre_handles.h"
 #include "lustre_intent.h"
 
 #define MAX_OBD_DEVICES 8192
@@ -81,6 +73,13 @@ static inline void loi_init(struct lov_oinfo *loi)
 {
 }
 
+/*
+ * If we are unable to get the maximum object size from the OST in
+ * ocd_maxbytes using OBD_CONNECT_MAXBYTES, then we fall back to using
+ * the old maximum object size from ext3.
+ */
+#define LUSTRE_EXT3_STRIPE_MAXBYTES 0x1fffffff000ULL
+
 struct lov_stripe_md {
 	atomic_t     lsm_refc;
 	spinlock_t	lsm_lock;
@@ -89,31 +88,17 @@ struct lov_stripe_md {
 	/* maximum possible file size, might change as OSTs status changes,
 	 * e.g. disconnected, deactivated
 	 */
-	__u64	    lsm_maxbytes;
-	struct {
-		/* Public members. */
-		struct ost_id lw_object_oi; /* lov object id/seq */
-
-		/* LOV-private members start here -- only for use in lov/. */
-		__u32 lw_magic;
-		__u32 lw_stripe_size;      /* size of the stripe */
-		__u32 lw_pattern;	  /* striping pattern (RAID0, RAID1) */
-		__u16 lw_stripe_count;  /* number of objects being striped over */
-		__u16 lw_layout_gen;       /* generation of the layout */
-		char  lw_pool_name[LOV_MAXPOOLNAME]; /* pool name */
-	} lsm_wire;
-
+	__u64		lsm_maxbytes;
+	struct ost_id	lsm_oi;
+	__u32		lsm_magic;
+	__u32		lsm_stripe_size;
+	__u32		lsm_pattern;	/* striping pattern (RAID0, RAID1) */
+	__u16		lsm_stripe_count;
+	__u16		lsm_layout_gen;
+	char		lsm_pool_name[LOV_MAXPOOLNAME + 1];
 	struct lov_oinfo *lsm_oinfo[0];
 };
 
-#define lsm_oi		 lsm_wire.lw_object_oi
-#define lsm_magic	lsm_wire.lw_magic
-#define lsm_layout_gen   lsm_wire.lw_layout_gen
-#define lsm_stripe_size  lsm_wire.lw_stripe_size
-#define lsm_pattern      lsm_wire.lw_pattern
-#define lsm_stripe_count lsm_wire.lw_stripe_count
-#define lsm_pool_name    lsm_wire.lw_pool_name
-
 static inline bool lsm_is_released(struct lov_stripe_md *lsm)
 {
 	return !!(lsm->lsm_pattern & LOV_PATTERN_F_RELEASED);
@@ -177,31 +162,10 @@ struct obd_type {
 struct brw_page {
 	u64 off;
 	struct page *pg;
-	int count;
+	unsigned int count;
 	u32 flag;
 };
 
-/* llog contexts */
-enum llog_ctxt_id {
-	LLOG_CONFIG_ORIG_CTXT  =  0,
-	LLOG_CONFIG_REPL_CTXT,
-	LLOG_MDS_OST_ORIG_CTXT,
-	LLOG_MDS_OST_REPL_CTXT,
-	LLOG_SIZE_ORIG_CTXT,
-	LLOG_SIZE_REPL_CTXT,
-	LLOG_RD1_ORIG_CTXT,
-	LLOG_RD1_REPL_CTXT,
-	LLOG_TEST_ORIG_CTXT,
-	LLOG_TEST_REPL_CTXT,
-	LLOG_LOVEA_ORIG_CTXT,
-	LLOG_LOVEA_REPL_CTXT,
-	LLOG_CHANGELOG_ORIG_CTXT,	/**< changelog generation on mdd */
-	LLOG_CHANGELOG_REPL_CTXT,	/**< changelog access on clients */
-	LLOG_CHANGELOG_USER_ORIG_CTXT,	/**< for multiple changelog consumers */
-	LLOG_AGENT_ORIG_CTXT,		/**< agent requests generation on cdt */
-	LLOG_MAX_CTXTS
-};
-
 struct timeout_item {
 	enum timeout_event ti_event;
 	unsigned long	 ti_timeout;
@@ -211,11 +175,12 @@ struct timeout_item {
 	struct list_head	 ti_chain;
 };
 
-#define OSC_MAX_RIF_DEFAULT       8
-#define OSC_MAX_RIF_MAX	 256
-#define OSC_MAX_DIRTY_DEFAULT  (OSC_MAX_RIF_DEFAULT * 4)
-#define OSC_MAX_DIRTY_MB_MAX   2048     /* arbitrary, but < MAX_LONG bytes */
-#define OSC_DEFAULT_RESENDS      10
+#define OBD_MAX_RIF_DEFAULT	8
+#define OBD_MAX_RIF_MAX		512
+#define OSC_MAX_RIF_MAX		256
+#define OSC_MAX_DIRTY_DEFAULT	(OBD_MAX_RIF_DEFAULT * 4)
+#define OSC_MAX_DIRTY_MB_MAX	2048	/* arbitrary, but < MAX_LONG bytes */
+#define OSC_DEFAULT_RESENDS	10
 
 /* possible values for fo_sync_lock_cancel */
 enum {
@@ -225,40 +190,74 @@ enum {
 	NUM_SYNC_ON_CANCEL_STATES
 };
 
-#define MDC_MAX_RIF_DEFAULT       8
-#define MDC_MAX_RIF_MAX	 512
-
 enum obd_cl_sem_lock_class {
 	OBD_CLI_SEM_NORMAL,
 	OBD_CLI_SEM_MGC,
 	OBD_CLI_SEM_MDCOSC,
 };
 
+/*
+ * Limit reply buffer size for striping data to one x86_64 page. This
+ * value is chosen to fit the striping data for common use cases while
+ * staying well below the limit at which the buffer must be backed by
+ * vmalloc(). Excessive use of vmalloc() may cause spinlock contention
+ * on the MDS.
+ */
+#define OBD_MAX_DEFAULT_EA_SIZE		4096
+#define OBD_MAX_DEFAULT_COOKIE_SIZE	4096
+
 struct mdc_rpc_lock;
 struct obd_import;
 struct client_obd {
 	struct rw_semaphore  cl_sem;
 	struct obd_uuid	  cl_target_uuid;
 	struct obd_import       *cl_import; /* ptlrpc connection state */
-	int		      cl_conn_count;
-	/* max_mds_easize is purely a performance thing so we don't have to
-	 * call obd_size_diskmd() all the time.
+	size_t			 cl_conn_count;
+	/*
+	 * Cache maximum and default values for easize and cookiesize. This is
+	 * strictly a performance optimization to minimize calls to
+	 * obd_size_diskmd(). The default values are used to calculate the
+	 * initial size of a request buffer. The ptlrpc layer will resize the
+	 * buffer as needed to accommodate a larger reply from the
+	 * server. The default values should be small enough to avoid wasted
+	 * memory and excessive use of vmalloc(), yet large enough to avoid
+	 * reallocating the buffer in the common use case.
 	 */
-	int			 cl_default_mds_easize;
-	int			 cl_max_mds_easize;
-	int			 cl_default_mds_cookiesize;
-	int			 cl_max_mds_cookiesize;
+	/*
+	 * Default EA size for striping attributes. It is initialized at
+	 * mount-time based on the default stripe width of the filesystem,
+	 * then it tracks the largest observed EA size advertised by
+	 * the MDT, up to a maximum value of OBD_MAX_DEFAULT_EA_SIZE.
+	 */
+	u32			 cl_default_mds_easize;
+	/* Maximum possible EA size computed at mount-time based on
+	 * the number of OSTs in the filesystem. May be increased at
+	 * run-time if a larger observed size is advertised by the MDT.
+	 */
+	u32			 cl_max_mds_easize;
+	/* Default cookie size for llog cookies (see struct llog_cookie). It is
+	 * initialized to zero at mount-time, then it tracks the largest
+	 * observed cookie size advertised by the MDT, up to a maximum value of
+	 * OBD_MAX_DEFAULT_COOKIE_SIZE. Note that llog_cookies are not
+	 * used by clients communicating with MDS versions 2.4.0 and later.
+	 */
+	u32			 cl_default_mds_cookiesize;
+	/* Maximum possible cookie size computed at mount-time based on
+	 * the number of OSTs in the filesystem. May be increased at
+	 * run-time if a larger observed size is advertised by the MDT.
+	 */
+	u32			 cl_max_mds_cookiesize;
 
 	enum lustre_sec_part     cl_sp_me;
 	enum lustre_sec_part     cl_sp_to;
 	struct sptlrpc_flavor    cl_flvr_mgc;   /* fixed flavor of mgc->mgs */
 
 	/* the grant values are protected by loi_list_lock below */
-	long		     cl_dirty;	 /* all _dirty_ in bytes */
-	long		     cl_dirty_max;     /* allowed w/o rpc */
-	long		     cl_dirty_transit; /* dirty synchronous */
-	long		     cl_avail_grant;   /* bytes of credit for ost */
-	long		     cl_lost_grant;    /* lost credits (trunc) */
+	unsigned long		 cl_dirty_pages;	/* all _dirty_ in pahges */
+	unsigned long		 cl_dirty_max_pages;	/* allowed w/o rpc */
+	unsigned long		 cl_dirty_transit;	/* dirty synchronous */
+	unsigned long		 cl_avail_grant;	/* bytes of credit for ost */
+	unsigned long		 cl_lost_grant;		/* lost credits (trunc) */
 
 	/* since we allocate grant by blocks, we don't know how many grant will
 	 * be used to add a page into cache. As a solution, we reserve maximum
@@ -275,8 +274,7 @@ struct client_obd {
 	 * the extent size. A chunk is max(PAGE_SIZE, OST block size)
 	 */
 	int		  cl_chunkbits;
-	int		  cl_chunk;
-	int		  cl_extent_tax; /* extent overhead, by bytes */
+	unsigned int	  cl_extent_tax; /* extent overhead, by bytes */
 
 	/* keep track of objects that have lois that contain pages which
 	 * have been queued for async brw.  this lock also protects the
@@ -301,13 +299,13 @@ struct client_obd {
 	struct list_head	       cl_loi_hp_ready_list;
 	struct list_head	       cl_loi_write_list;
 	struct list_head	       cl_loi_read_list;
-	int		      cl_r_in_flight;
-	int		      cl_w_in_flight;
+	__u32			 cl_r_in_flight;
+	__u32			 cl_w_in_flight;
 	/* just a sum of the loi/lop pending numbers to be exported by sysfs */
 	atomic_t	     cl_pending_w_pages;
 	atomic_t	     cl_pending_r_pages;
 	__u32			 cl_max_pages_per_rpc;
-	int		      cl_max_rpcs_in_flight;
+	__u32			 cl_max_rpcs_in_flight;
 	struct obd_histogram     cl_read_rpc_hist;
 	struct obd_histogram     cl_write_rpc_hist;
 	struct obd_histogram     cl_read_page_hist;
@@ -318,13 +316,13 @@ struct client_obd {
 	/* lru for osc caching pages */
 	struct cl_client_cache	*cl_cache;
 	struct list_head	 cl_lru_osc; /* member of cl_cache->ccc_lru */
-	atomic_t		*cl_lru_left;
-	atomic_t		 cl_lru_busy;
+	atomic_long_t		*cl_lru_left;
+	atomic_long_t		 cl_lru_busy;
+	atomic_long_t		 cl_lru_in_list;
 	atomic_t		 cl_lru_shrinkers;
-	atomic_t		 cl_lru_in_list;
 	struct list_head	 cl_lru_list; /* lru page list */
 	spinlock_t		 cl_lru_list_lock; /* page list protector */
-	atomic_t		 cl_unstable_count;
+	atomic_long_t		 cl_unstable_count;
 
 	/* number of in flight destroy rpcs is limited to max_rpcs_in_flight */
 	atomic_t	     cl_destroy_in_flight;
@@ -350,7 +348,7 @@ struct client_obd {
 	/* used by quotacheck when the servers are older than 2.4 */
 	int		      cl_qchk_stat; /* quotacheck stat of the peer */
 #define CL_NOT_QUOTACHECKED 1   /* client->cl_qchk_stat init value */
-#if LUSTRE_VERSION_CODE >= OBD_OCD_VERSION(2, 7, 50, 0)
+#if OBD_OCD_VERSION(2, 7, 53, 0) < LUSTRE_VERSION_CODE
 #warning "please consider removing quotacheck compatibility code"
 #endif
 
@@ -431,7 +429,7 @@ struct lov_obd {
 struct lmv_tgt_desc {
 	struct obd_uuid		ltd_uuid;
 	struct obd_export	*ltd_exp;
-	int			ltd_idx;
+	u32			ltd_idx;
 	struct mutex		ltd_fid_mutex;
 	unsigned long		ltd_active:1; /* target up for requests */
 };
@@ -458,9 +456,8 @@ struct lmv_obd {
 	int			max_def_easize;
 	int			max_cookiesize;
 	int			max_def_cookiesize;
-	int			server_timeout;
 
-	int			tgts_size; /* size of tgts array */
+	u32			tgts_size; /* size of tgts array */
 	struct lmv_tgt_desc	**tgts;
 
 	struct obd_connect_data	conn_data;
@@ -470,12 +467,11 @@ struct lmv_obd {
 struct niobuf_local {
 	__u64		lnb_file_offset;
 	__u32		lnb_page_offset;
-	__u32		len;
-	__u32		flags;
-	struct page	*page;
-	struct dentry	*dentry;
-	int		lnb_grant_used;
-	int		rc;
+	__u32		lnb_len;
+	__u32		lnb_flags;
+	struct page	*lnb_page;
+	void		*lnb_data;
+	int		lnb_rc;
 };
 
 #define LUSTRE_FLD_NAME	 "fld"
@@ -517,7 +513,6 @@ struct niobuf_local {
 #define N_LOCAL_TEMP_PAGE 0x10000000
 
 struct obd_trans_info {
-	__u64		    oti_transno;
 	__u64		    oti_xid;
 	/* Only used on the server side for tracking acks. */
 	struct oti_req_ack_lock {
@@ -527,50 +522,11 @@ struct obd_trans_info {
 	void		    *oti_handle;
 	struct llog_cookie       oti_onecookie;
 	struct llog_cookie      *oti_logcookies;
-	int		      oti_numcookies;
-	/** synchronous write is needed */
-	unsigned long		 oti_sync_write:1;
 
-	/* initial thread handling transaction */
-	struct ptlrpc_thread *oti_thread;
-	__u32		    oti_conn_cnt;
 	/** VBR: versions */
 	__u64		    oti_pre_version;
-	/** JobID */
-	char		    *oti_jobid;
-
-	struct obd_uuid	 *oti_ost_uuid;
 };
 
-static inline void oti_alloc_cookies(struct obd_trans_info *oti,
-				     int num_cookies)
-{
-	if (!oti)
-		return;
-
-	if (num_cookies == 1)
-		oti->oti_logcookies = &oti->oti_onecookie;
-	else
-		oti->oti_logcookies = libcfs_kvzalloc(num_cookies * sizeof(oti->oti_onecookie),
-						      GFP_NOFS);
-
-	oti->oti_numcookies = num_cookies;
-}
-
-static inline void oti_free_cookies(struct obd_trans_info *oti)
-{
-	if (!oti || !oti->oti_logcookies)
-		return;
-
-	if (oti->oti_logcookies == &oti->oti_onecookie)
-		LASSERT(oti->oti_numcookies == 1);
-	else
-		kvfree(oti->oti_logcookies);
-
-	oti->oti_logcookies = NULL;
-	oti->oti_numcookies = 0;
-}
-
 /*
  * Events signalled through obd_notify() upcall-chain.
  */
@@ -616,7 +572,6 @@ struct target_recovery_data {
 };
 
 struct obd_llog_group {
-	int		olg_seq;
 	struct llog_ctxt  *olg_ctxts[LLOG_MAX_CTXTS];
 	wait_queue_head_t	olg_waitq;
 	spinlock_t	   olg_lock;
@@ -625,7 +580,6 @@ struct obd_llog_group {
 
 /* corresponds to one of the obd's */
 #define OBD_DEVICE_MAGIC	0XAB5CD6EF
-#define OBD_DEV_BY_DEVNAME      0xffffd0de
 
 struct lvfs_run_ctxt {
 	struct dt_device *dt;
@@ -653,7 +607,6 @@ struct obd_device {
 		      obd_starting:1,      /* started setup */
 		      obd_force:1,	 /* cleanup with > 0 obd refcount */
 		      obd_fail:1,	 /* cleanup with failover */
-		      obd_async_recov:1, /* allow asynchronous orphan cleanup */
 		      obd_no_conn:1,       /* deny new connections */
 		      obd_inactive:1,      /* device active/inactive
 					    * (for sysfs status only!!)
@@ -728,9 +681,6 @@ struct obd_device {
 	struct completion	obd_kobj_unregister;
 };
 
-#define OBD_LLOG_FL_SENDNOW     0x0001
-#define OBD_LLOG_FL_EXIT	0x0002
-
 enum obd_cleanup_stage {
 /* Special case hack for MDS LOVs */
 	OBD_CLEANUP_EARLY,
@@ -740,8 +690,6 @@ enum obd_cleanup_stage {
 
 /* get/set_info keys */
 #define KEY_ASYNC	       "async"
-#define KEY_BLOCKSIZE_BITS      "blocksize_bits"
-#define KEY_BLOCKSIZE	   "blocksize"
 #define KEY_CHANGELOG_CLEAR     "changelog_clear"
 #define KEY_FID2PATH	    "fid2path"
 #define KEY_CHECKSUM	    "checksum"
@@ -753,30 +701,22 @@ enum obd_cleanup_stage {
 #define KEY_GRANT_SHRINK	"grant_shrink"
 #define KEY_HSM_COPYTOOL_SEND   "hsm_send"
 #define KEY_INIT_RECOV_BACKUP   "init_recov_bk"
-#define KEY_INIT_RECOV	  "initial_recov"
 #define KEY_INTERMDS	    "inter_mds"
 #define KEY_LAST_ID	     "last_id"
 #define KEY_LAST_FID		"last_fid"
-#define KEY_LOCK_TO_STRIPE      "lock_to_stripe"
 #define KEY_LOVDESC	     "lovdesc"
-#define KEY_LOV_IDX	     "lov_idx"
 #define KEY_MAX_EASIZE		"max_easize"
 #define KEY_DEFAULT_EASIZE	"default_easize"
-#define KEY_MDS_CONN	    "mds_conn"
 #define KEY_MGSSEC	      "mgssec"
-#define KEY_NEXT_ID	     "next_id"
 #define KEY_READ_ONLY	   "read-only"
 #define KEY_REGISTER_TARGET     "register_target"
 #define KEY_SET_FS	      "set_fs"
 #define KEY_TGT_COUNT	   "tgt_count"
 /*      KEY_SET_INFO in lustre_idl.h */
 #define KEY_SPTLRPC_CONF	"sptlrpc_conf"
-#define KEY_CONNECT_FLAG	"connect_flags"
-#define KEY_SYNC_LOCK_CANCEL    "sync_lock_cancel"
 
 #define KEY_CACHE_SET		"cache_set"
 #define KEY_CACHE_LRU_SHRINK	"cache_lru_shrink"
-#define KEY_CHANGELOG_INDEX	"changelog_index"
 
 struct lu_context;
 
@@ -801,9 +741,11 @@ static inline int it_to_lock_mode(struct lookup_intent *it)
 	/* CREAT needs to be tested before open (both could be set) */
 	if (it->it_op & IT_CREAT)
 		return LCK_CW;
-	else if (it->it_op & (IT_READDIR | IT_GETATTR | IT_OPEN | IT_LOOKUP |
+	else if (it->it_op & (IT_GETATTR | IT_OPEN | IT_LOOKUP |
 			      IT_LAYOUT))
 		return LCK_CR;
+	else if (it->it_op & IT_READDIR)
+		return LCK_PR;
 	else if (it->it_op &  IT_GETXATTR)
 		return LCK_PR;
 	else if (it->it_op &  IT_SETXATTR)
@@ -813,6 +755,14 @@ static inline int it_to_lock_mode(struct lookup_intent *it)
 	return -EINVAL;
 }
 
+enum md_cli_flags {
+	CLI_SET_MEA	= BIT(0),
+	CLI_RM_ENTRY	= BIT(1),
+	CLI_HASH64	= BIT(2),
+	CLI_API32	= BIT(3),
+	CLI_MIGRATE	= BIT(4),
+};
+
 struct md_op_data {
 	struct lu_fid	   op_fid1; /* operation fid1 (usually parent) */
 	struct lu_fid	   op_fid2; /* operation fid2 (usually child) */
@@ -822,7 +772,7 @@ struct md_op_data {
 	struct lustre_handle    op_handle;
 	s64			op_mod_time;
 	const char	     *op_name;
-	int		     op_namelen;
+	size_t			op_namelen;
 	__u32		   op_mode;
 	struct lmv_stripe_md   *op_mea1;
 	struct lmv_stripe_md   *op_mea2;
@@ -831,6 +781,7 @@ struct md_op_data {
 	__u32		   op_fsgid;
 	cfs_cap_t	       op_cap;
 	void		   *op_data;
+	size_t			op_data_size;
 
 	/* iattr fields and blocks. */
 	struct iattr	    op_attr;
@@ -845,28 +796,29 @@ struct md_op_data {
 	/* Various operation flags. */
 	enum mds_op_bias        op_bias;
 
-	/* Operation type */
-	__u32		   op_opc;
-
 	/* Used by readdir */
 	__u64		   op_offset;
 
 	/* Used by readdir */
-	__u32		   op_npages;
+	__u32			op_max_pages;
 
 	/* used to transfer info between the stacks of MD client
 	 * see enum op_cli_flags
 	 */
-	__u32			op_cli_flags;
+	enum md_cli_flags	op_cli_flags;
 
 	/* File object data version for HSM release, on client */
 	__u64			op_data_version;
 	struct lustre_handle	op_lease_handle;
+
+	/* default stripe offset */
+	__u32			op_default_stripe_offset;
 };
 
-enum op_cli_flags {
-	CLI_SET_MEA	= 1 << 0,
-	CLI_RM_ENTRY	= 1 << 1,
+struct md_callback {
+	int (*md_blocking_ast)(struct ldlm_lock *lock,
+			       struct ldlm_lock_desc *desc,
+			       void *data, int flag);
 };
 
 struct md_enqueue_info;
@@ -879,8 +831,7 @@ struct md_enqueue_info {
 	struct inode	   *mi_dir;
 	int (*mi_cb)(struct ptlrpc_request *req,
 		     struct md_enqueue_info *minfo, int rc);
-	__u64		   mi_cbdata;
-	unsigned int	    mi_generation;
+	void			*mi_cbdata;
 };
 
 struct obd_ops {
@@ -894,8 +845,6 @@ struct obd_ops {
 			      __u32 keylen, void *key,
 			      __u32 vallen, void *val,
 			      struct ptlrpc_request_set *set);
-	int (*attach)(struct obd_device *dev, u32 len, void *data);
-	int (*detach)(struct obd_device *dev);
 	int (*setup)(struct obd_device *dev, struct lustre_cfg *cfg);
 	int (*precleanup)(struct obd_device *dev,
 			  enum obd_cleanup_stage cleanup_stage);
@@ -927,8 +876,8 @@ struct obd_ops {
 	int (*fid_fini)(struct obd_device *obd);
 
 	/* Allocate new fid according to passed @hint. */
-	int (*fid_alloc)(struct obd_export *exp, struct lu_fid *fid,
-			 struct md_op_data *op_data);
+	int (*fid_alloc)(const struct lu_env *env, struct obd_export *exp,
+			 struct lu_fid *fid, struct md_op_data *op_data);
 
 	/*
 	 * Object with @fid is getting deleted, we may want to do something
@@ -943,13 +892,10 @@ struct obd_ops {
 	int (*unpackmd)(struct obd_export *exp,
 			struct lov_stripe_md **mem_tgt,
 			struct lov_mds_md *disk_src, int disk_len);
-	int (*preallocate)(struct lustre_handle *, u32 *req, u64 *ids);
 	int (*create)(const struct lu_env *env, struct obd_export *exp,
-		      struct obdo *oa, struct lov_stripe_md **ea,
-		      struct obd_trans_info *oti);
+		      struct obdo *oa, struct obd_trans_info *oti);
 	int (*destroy)(const struct lu_env *env, struct obd_export *exp,
-		       struct obdo *oa, struct lov_stripe_md *ea,
-		       struct obd_trans_info *oti, struct obd_export *md_exp);
+		       struct obdo *oa, struct obd_trans_info *oti);
 	int (*setattr)(const struct lu_env *, struct obd_export *exp,
 		       struct obd_info *oinfo, struct obd_trans_info *oti);
 	int (*setattr_async)(struct obd_export *exp, struct obd_info *oinfo,
@@ -959,8 +905,6 @@ struct obd_ops {
 		       struct obd_info *oinfo);
 	int (*getattr_async)(struct obd_export *exp, struct obd_info *oinfo,
 			     struct ptlrpc_request_set *set);
-	int (*adjust_kms)(struct obd_export *exp, struct lov_stripe_md *lsm,
-			  u64 size, int shrink);
 	int (*preprw)(const struct lu_env *env, int cmd,
 		      struct obd_export *exp, struct obdo *oa, int objcount,
 		      struct obd_ioobj *obj, struct niobuf_remote *remote,
@@ -972,8 +916,6 @@ struct obd_ops {
 			struct niobuf_remote *remote, int pages,
 			struct niobuf_local *local,
 			struct obd_trans_info *oti, int rc);
-	int (*find_cbdata)(struct obd_export *, struct lov_stripe_md *,
-			   ldlm_iterator_t it, void *data);
 	int (*init_export)(struct obd_export *exp);
 	int (*destroy_export)(struct obd_export *exp);
 
@@ -1009,27 +951,11 @@ struct obd_ops {
 	 */
 };
 
-enum {
-	LUSTRE_OPC_MKDIR    = (1 << 0),
-	LUSTRE_OPC_SYMLINK  = (1 << 1),
-	LUSTRE_OPC_MKNOD    = (1 << 2),
-	LUSTRE_OPC_CREATE   = (1 << 3),
-	LUSTRE_OPC_ANY      = (1 << 4)
-};
-
 /* lmv structures */
-#define MEA_MAGIC_LAST_CHAR      0xb2221ca1
-#define MEA_MAGIC_ALL_CHARS      0xb222a11c
-#define MEA_MAGIC_HASH_SEGMENT   0xb222a11b
-
-#define MAX_HASH_SIZE_32	 0x7fffffffUL
-#define MAX_HASH_SIZE	    0x7fffffffffffffffULL
-#define MAX_HASH_HIGHEST_BIT     0x1000000000000000ULL
-
 struct lustre_md {
 	struct mdt_body	 *body;
 	struct lov_stripe_md    *lsm;
-	struct lmv_stripe_md    *mea;
+	struct lmv_stripe_md    *lmv;
 #ifdef CONFIG_FS_POSIX_ACL
 	struct posix_acl	*posix_acl;
 #endif
@@ -1044,48 +970,55 @@ struct md_open_data {
 	bool			  mod_is_create;
 };
 
+struct obd_client_handle {
+	struct lustre_handle	 och_fh;
+	struct lu_fid		 och_fid;
+	struct md_open_data	*och_mod;
+	struct lustre_handle	 och_lease_handle; /* open lock for lease */
+	__u32			 och_magic;
+	int			 och_flags;
+};
+
+#define OBD_CLIENT_HANDLE_MAGIC 0xd15ea5ed
+
 struct lookup_intent;
+struct cl_attr;
 
 struct md_ops {
 	int (*getstatus)(struct obd_export *, struct lu_fid *);
 	int (*null_inode)(struct obd_export *, const struct lu_fid *);
-	int (*find_cbdata)(struct obd_export *, const struct lu_fid *,
-			   ldlm_iterator_t, void *);
 	int (*close)(struct obd_export *, struct md_op_data *,
 		     struct md_open_data *, struct ptlrpc_request **);
 	int (*create)(struct obd_export *, struct md_op_data *,
-		      const void *, int, int, __u32, __u32, cfs_cap_t,
-		      __u64, struct ptlrpc_request **);
+		      const void *, size_t, umode_t, uid_t, gid_t,
+		      cfs_cap_t, __u64, struct ptlrpc_request **);
 	int (*done_writing)(struct obd_export *, struct md_op_data  *,
 			    struct md_open_data *);
 	int (*enqueue)(struct obd_export *, struct ldlm_enqueue_info *,
+		       const ldlm_policy_data_t *,
 		       struct lookup_intent *, struct md_op_data *,
-		       struct lustre_handle *, void *, int,
-		       struct ptlrpc_request **, __u64);
+		       struct lustre_handle *, __u64);
 	int (*getattr)(struct obd_export *, struct md_op_data *,
 		       struct ptlrpc_request **);
 	int (*getattr_name)(struct obd_export *, struct md_op_data *,
 			    struct ptlrpc_request **);
 	int (*intent_lock)(struct obd_export *, struct md_op_data *,
-			   void *, int, struct lookup_intent *, int,
+			   struct lookup_intent *,
 			   struct ptlrpc_request **,
 			   ldlm_blocking_callback, __u64);
 	int (*link)(struct obd_export *, struct md_op_data *,
 		    struct ptlrpc_request **);
 	int (*rename)(struct obd_export *, struct md_op_data *,
-		      const char *, int, const char *, int,
+		      const char *, size_t, const char *, size_t,
 		      struct ptlrpc_request **);
-	int (*is_subdir)(struct obd_export *, const struct lu_fid *,
-			 const struct lu_fid *,
-			   struct ptlrpc_request **);
 	int (*setattr)(struct obd_export *, struct md_op_data *, void *,
-		       int, void *, int, struct ptlrpc_request **,
+		       size_t, void *, size_t, struct ptlrpc_request **,
 			 struct md_open_data **mod);
 	int (*sync)(struct obd_export *, const struct lu_fid *,
 		    struct ptlrpc_request **);
-	int (*readpage)(struct obd_export *, struct md_op_data *,
-			struct page **, struct ptlrpc_request **);
-
+	int (*read_page)(struct obd_export *, struct md_op_data *,
+			 struct md_callback *cb_op, __u64 hash_offset,
+			 struct page **ppage);
 	int (*unlink)(struct obd_export *, struct md_op_data *,
 		      struct ptlrpc_request **);
 
@@ -1097,7 +1030,7 @@ struct md_ops {
 			u64, const char *, const char *, int, int, int,
 			struct ptlrpc_request **);
 
-	int (*init_ea_size)(struct obd_export *, int, int, int, int);
+	int (*init_ea_size)(struct obd_export *, u32, u32, u32, u32);
 
 	int (*get_lustre_md)(struct obd_export *, struct ptlrpc_request *,
 			     struct obd_export *, struct obd_export *,
@@ -1105,12 +1038,17 @@ struct md_ops {
 
 	int (*free_lustre_md)(struct obd_export *, struct lustre_md *);
 
+	int (*merge_attr)(struct obd_export *,
+			  const struct lmv_stripe_md *lsm,
+			  struct cl_attr *attr, ldlm_blocking_callback);
+
 	int (*set_open_replay_data)(struct obd_export *,
 				    struct obd_client_handle *,
 				    struct lookup_intent *);
 	int (*clear_open_replay_data)(struct obd_export *,
 				      struct obd_client_handle *);
-	int (*set_lock_data)(struct obd_export *, __u64 *, void *, __u64 *);
+	int (*set_lock_data)(struct obd_export *, const struct lustre_handle *,
+			     void *, __u64 *);
 
 	enum ldlm_mode (*lock_match)(struct obd_export *, __u64,
 				     const struct lu_fid *, enum ldlm_type,
@@ -1121,6 +1059,11 @@ struct md_ops {
 			     ldlm_policy_data_t *, enum ldlm_mode,
 			     enum ldlm_cancel_flags flags, void *opaque);
 
+	int (*get_fid_from_lsm)(struct obd_export *,
+				const struct lmv_stripe_md *,
+				const char *name, int namelen,
+				struct lu_fid *fid);
+
 	int (*intent_getattr_async)(struct obd_export *,
 				    struct md_enqueue_info *,
 				    struct ldlm_enqueue_info *);
@@ -1137,8 +1080,6 @@ struct md_ops {
 
 struct lsm_operations {
 	void (*lsm_free)(struct lov_stripe_md *);
-	int (*lsm_destroy)(struct lov_stripe_md *, struct obdo *oa,
-			   struct obd_export *md_exp);
 	void (*lsm_stripe_by_index)(struct lov_stripe_md *, int *, u64 *,
 				    u64 *);
 	void (*lsm_stripe_by_offset)(struct lov_stripe_md *, int *, u64 *,
@@ -1164,10 +1105,6 @@ static inline const struct lsm_operations *lsm_op_find(int magic)
 	}
 }
 
-/* Requests for obd_extent_calc() */
-#define OBD_CALC_STRIPE_START   1
-#define OBD_CALC_STRIPE_END     2
-
 static inline struct md_open_data *obd_mod_alloc(void)
 {
 	struct md_open_data *mod;
@@ -1211,7 +1148,8 @@ static inline const char *lu_dev_name(const struct lu_device *lu_dev)
 	return lu_dev->ld_obd->obd_name;
 }
 
-static inline bool filename_is_volatile(const char *name, int namelen, int *idx)
+static inline bool filename_is_volatile(const char *name, size_t namelen,
+					int *idx)
 {
 	const char	*start;
 	char		*end;
@@ -1259,4 +1197,28 @@ static inline int cli_brw_size(struct obd_device *obd)
 	return obd->u.cli.cl_max_pages_per_rpc << PAGE_SHIFT;
 }
 
+/*
+ * when RPC size or the max RPCs in flight is increased, the max dirty pages
+ * of the client should be increased accordingly to avoid sending fragmented
+ * RPCs over the network when the client runs out of the maximum dirty space
+ * when so many RPCs are being generated.
+ */
+static inline void client_adjust_max_dirty(struct client_obd *cli)
+{
+	/* initializing */
+	if (cli->cl_dirty_max_pages <= 0)
+		cli->cl_dirty_max_pages =
+			(OSC_MAX_DIRTY_DEFAULT * 1024 * 1024) >> PAGE_SHIFT;
+	else {
+		unsigned long dirty_max = cli->cl_max_rpcs_in_flight *
+					  cli->cl_max_pages_per_rpc;
+
+		if (dirty_max > cli->cl_dirty_max_pages)
+			cli->cl_dirty_max_pages = dirty_max;
+	}
+
+	if (cli->cl_dirty_max_pages > totalram_pages / 8)
+		cli->cl_dirty_max_pages = totalram_pages / 8;
+}
+
 #endif /* __OBD_H */
diff --git a/drivers/staging/lustre/lustre/include/obd_class.h b/drivers/staging/lustre/lustre/include/obd_class.h
index 6482a937000b..16094dbec08b 100644
--- a/drivers/staging/lustre/lustre/include/obd_class.h
+++ b/drivers/staging/lustre/lustre/include/obd_class.h
@@ -56,7 +56,6 @@
 #define OBD_STATFS_FOR_MDT0	0x0008	/* The statfs is only for retrieving
 					 * information from MDT0.
 					 */
-#define OBD_FL_PUNCH	    0x00000001  /* To indicate it is punch operation */
 
 /* OBD Device Declarations */
 extern struct obd_device *obd_devs[MAX_OBD_DEVICES];
@@ -97,6 +96,11 @@ int obd_zombie_impexp_init(void);
 void obd_zombie_impexp_stop(void);
 void obd_zombie_barrier(void);
 
+int obd_get_request_slot(struct client_obd *cli);
+void obd_put_request_slot(struct client_obd *cli);
+__u32 obd_get_max_rpcs_in_flight(struct client_obd *cli);
+int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max);
+
 struct llog_handle;
 struct llog_rec_hdr;
 typedef int (*llog_cb_t)(const struct lu_env *, struct llog_handle *,
@@ -265,10 +269,10 @@ static inline int lprocfs_climp_check(struct obd_device *obd)
 struct inode;
 struct lu_attr;
 struct obdo;
-void obdo_refresh_inode(struct inode *dst, struct obdo *src, u32 valid);
+void obdo_refresh_inode(struct inode *dst, const struct obdo *src, u32 valid);
 
-void obdo_to_ioobj(struct obdo *oa, struct obd_ioobj *ioobj);
-void md_from_obdo(struct md_op_data *op_data, struct obdo *oa, u32 valid);
+void obdo_to_ioobj(const struct obdo *oa, struct obd_ioobj *ioobj);
+void md_from_obdo(struct md_op_data *op_data, const struct obdo *oa, u32 valid);
 
 #define OBT(dev)	(dev)->obd_type
 #define OBP(dev, op)    (dev)->obd_type->typ_dt_ops->op
@@ -673,15 +677,6 @@ static inline int obd_unpackmd(struct obd_export *exp,
 	return rc;
 }
 
-/* helper functions */
-static inline int obd_alloc_memmd(struct obd_export *exp,
-				  struct lov_stripe_md **mem_tgt)
-{
-	LASSERT(mem_tgt);
-	LASSERT(!*mem_tgt);
-	return obd_unpackmd(exp, mem_tgt, NULL, 0);
-}
-
 static inline int obd_free_memmd(struct obd_export *exp,
 				 struct lov_stripe_md **mem_tgt)
 {
@@ -695,29 +690,26 @@ static inline int obd_free_memmd(struct obd_export *exp,
 }
 
 static inline int obd_create(const struct lu_env *env, struct obd_export *exp,
-			     struct obdo *obdo, struct lov_stripe_md **ea,
-			     struct obd_trans_info *oti)
+			     struct obdo *obdo, struct obd_trans_info *oti)
 {
 	int rc;
 
 	EXP_CHECK_DT_OP(exp, create);
 	EXP_COUNTER_INCREMENT(exp, create);
 
-	rc = OBP(exp->exp_obd, create)(env, exp, obdo, ea, oti);
+	rc = OBP(exp->exp_obd, create)(env, exp, obdo, oti);
 	return rc;
 }
 
 static inline int obd_destroy(const struct lu_env *env, struct obd_export *exp,
-			      struct obdo *obdo, struct lov_stripe_md *ea,
-			      struct obd_trans_info *oti,
-			      struct obd_export *md_exp)
+			      struct obdo *obdo, struct obd_trans_info *oti)
 {
 	int rc;
 
 	EXP_CHECK_DT_OP(exp, destroy);
 	EXP_COUNTER_INCREMENT(exp, destroy);
 
-	rc = OBP(exp->exp_obd, destroy)(env, exp, obdo, ea, oti, md_exp);
+	rc = OBP(exp->exp_obd, destroy)(env, exp, obdo, oti);
 	return rc;
 }
 
@@ -925,7 +917,8 @@ static inline int obd_fid_fini(struct obd_device *obd)
 	return rc;
 }
 
-static inline int obd_fid_alloc(struct obd_export *exp,
+static inline int obd_fid_alloc(const struct lu_env *env,
+				struct obd_export *exp,
 				struct lu_fid *fid,
 				struct md_op_data *op_data)
 {
@@ -934,7 +927,7 @@ static inline int obd_fid_alloc(struct obd_export *exp,
 	EXP_CHECK_DT_OP(exp, fid_alloc);
 	EXP_COUNTER_INCREMENT(exp, fid_alloc);
 
-	rc = OBP(exp->exp_obd, fid_alloc)(exp, fid, op_data);
+	rc = OBP(exp->exp_obd, fid_alloc)(env, exp, fid, op_data);
 	return rc;
 }
 
@@ -1147,19 +1140,6 @@ static inline int obd_commitrw(const struct lu_env *env, int cmd,
 	return rc;
 }
 
-static inline int obd_adjust_kms(struct obd_export *exp,
-				 struct lov_stripe_md *lsm, u64 size,
-				 int shrink)
-{
-	int rc;
-
-	EXP_CHECK_DT_OP(exp, adjust_kms);
-	EXP_COUNTER_INCREMENT(exp, adjust_kms);
-
-	rc = OBP(exp->exp_obd, adjust_kms)(exp, lsm, size, shrink);
-	return rc;
-}
-
 static inline int obd_iocontrol(unsigned int cmd, struct obd_export *exp,
 				int len, void *karg, void __user *uarg)
 {
@@ -1172,19 +1152,6 @@ static inline int obd_iocontrol(unsigned int cmd, struct obd_export *exp,
 	return rc;
 }
 
-static inline int obd_find_cbdata(struct obd_export *exp,
-				  struct lov_stripe_md *lsm,
-				  ldlm_iterator_t it, void *data)
-{
-	int rc;
-
-	EXP_CHECK_DT_OP(exp, find_cbdata);
-	EXP_COUNTER_INCREMENT(exp, find_cbdata);
-
-	rc = OBP(exp->exp_obd, find_cbdata)(exp, lsm, it, data);
-	return rc;
-}
-
 static inline void obd_import_event(struct obd_device *obd,
 				    struct obd_import *imp,
 				    enum obd_import_event event)
@@ -1210,12 +1177,7 @@ static inline int obd_notify(struct obd_device *obd,
 	if (rc)
 		return rc;
 
-	/* the check for async_recov is a complete hack - I'm hereby
-	 * overloading the meaning to also mean "this was called from
-	 * mds_postsetup".  I know that my mds is able to handle notifies
-	 * by this point, and it needs to get them to execute mds_postrecov.
-	 */
-	if (!obd->obd_set_up && !obd->obd_async_recov) {
+	if (!obd->obd_set_up) {
 		CDEBUG(D_HA, "obd %s not set up\n", obd->obd_name);
 		return -EINVAL;
 	}
@@ -1358,18 +1320,6 @@ static inline int md_null_inode(struct obd_export *exp,
 	return rc;
 }
 
-static inline int md_find_cbdata(struct obd_export *exp,
-				 const struct lu_fid *fid,
-				 ldlm_iterator_t it, void *data)
-{
-	int rc;
-
-	EXP_CHECK_MD_OP(exp, find_cbdata);
-	EXP_MD_COUNTER_INCREMENT(exp, find_cbdata);
-	rc = MDP(exp->exp_obd, find_cbdata)(exp, fid, it, data);
-	return rc;
-}
-
 static inline int md_close(struct obd_export *exp, struct md_op_data *op_data,
 			   struct md_open_data *mod,
 			   struct ptlrpc_request **request)
@@ -1383,9 +1333,9 @@ static inline int md_close(struct obd_export *exp, struct md_op_data *op_data,
 }
 
 static inline int md_create(struct obd_export *exp, struct md_op_data *op_data,
-			    const void *data, int datalen, int mode, __u32 uid,
-			    __u32 gid, cfs_cap_t cap_effective, __u64 rdev,
-			    struct ptlrpc_request **request)
+			    const void *data, size_t datalen, umode_t mode,
+			    uid_t uid, gid_t gid, cfs_cap_t cap_effective,
+			    __u64 rdev, struct ptlrpc_request **request)
 {
 	int rc;
 
@@ -1410,19 +1360,18 @@ static inline int md_done_writing(struct obd_export *exp,
 
 static inline int md_enqueue(struct obd_export *exp,
 			     struct ldlm_enqueue_info *einfo,
+			     const ldlm_policy_data_t *policy,
 			     struct lookup_intent *it,
 			     struct md_op_data *op_data,
 			     struct lustre_handle *lockh,
-			     void *lmm, int lmmsize,
-			     struct ptlrpc_request **req,
 			     __u64 extra_lock_flags)
 {
 	int rc;
 
 	EXP_CHECK_MD_OP(exp, enqueue);
 	EXP_MD_COUNTER_INCREMENT(exp, enqueue);
-	rc = MDP(exp->exp_obd, enqueue)(exp, einfo, it, op_data, lockh,
-					lmm, lmmsize, req, extra_lock_flags);
+	rc = MDP(exp->exp_obd, enqueue)(exp, einfo, policy, it, op_data, lockh,
+					extra_lock_flags);
 	return rc;
 }
 
@@ -1439,9 +1388,9 @@ static inline int md_getattr_name(struct obd_export *exp,
 }
 
 static inline int md_intent_lock(struct obd_export *exp,
-				 struct md_op_data *op_data, void *lmm,
-				 int lmmsize, struct lookup_intent *it,
-				 int lookup_flags, struct ptlrpc_request **reqp,
+				 struct md_op_data *op_data,
+				 struct lookup_intent *it,
+				 struct ptlrpc_request **reqp,
 				 ldlm_blocking_callback cb_blocking,
 				 __u64 extra_lock_flags)
 {
@@ -1449,9 +1398,8 @@ static inline int md_intent_lock(struct obd_export *exp,
 
 	EXP_CHECK_MD_OP(exp, intent_lock);
 	EXP_MD_COUNTER_INCREMENT(exp, intent_lock);
-	rc = MDP(exp->exp_obd, intent_lock)(exp, op_data, lmm, lmmsize,
-					    it, lookup_flags, reqp, cb_blocking,
-					    extra_lock_flags);
+	rc = MDP(exp->exp_obd, intent_lock)(exp, op_data, it, reqp,
+					    cb_blocking, extra_lock_flags);
 	return rc;
 }
 
@@ -1467,8 +1415,8 @@ static inline int md_link(struct obd_export *exp, struct md_op_data *op_data,
 }
 
 static inline int md_rename(struct obd_export *exp, struct md_op_data *op_data,
-			    const char *old, int oldlen, const char *new,
-			    int newlen, struct ptlrpc_request **request)
+			    const char *old, size_t oldlen, const char *new,
+			    size_t newlen, struct ptlrpc_request **request)
 {
 	int rc;
 
@@ -1479,21 +1427,8 @@ static inline int md_rename(struct obd_export *exp, struct md_op_data *op_data,
 	return rc;
 }
 
-static inline int md_is_subdir(struct obd_export *exp,
-			       const struct lu_fid *pfid,
-			       const struct lu_fid *cfid,
-			       struct ptlrpc_request **request)
-{
-	int rc;
-
-	EXP_CHECK_MD_OP(exp, is_subdir);
-	EXP_MD_COUNTER_INCREMENT(exp, is_subdir);
-	rc = MDP(exp->exp_obd, is_subdir)(exp, pfid, cfid, request);
-	return rc;
-}
-
 static inline int md_setattr(struct obd_export *exp, struct md_op_data *op_data,
-			     void *ea, int ealen, void *ea2, int ea2len,
+			     void *ea, size_t ealen, void *ea2, size_t ea2len,
 			     struct ptlrpc_request **request,
 			     struct md_open_data **mod)
 {
@@ -1517,15 +1452,18 @@ static inline int md_sync(struct obd_export *exp, const struct lu_fid *fid,
 	return rc;
 }
 
-static inline int md_readpage(struct obd_export *exp, struct md_op_data *opdata,
-			      struct page **pages,
-			      struct ptlrpc_request **request)
+static inline int md_read_page(struct obd_export *exp,
+			       struct md_op_data *op_data,
+			       struct md_callback *cb_op,
+			       __u64  hash_offset,
+			       struct page **ppage)
 {
 	int rc;
 
-	EXP_CHECK_MD_OP(exp, readpage);
-	EXP_MD_COUNTER_INCREMENT(exp, readpage);
-	rc = MDP(exp->exp_obd, readpage)(exp, opdata, pages, request);
+	EXP_CHECK_MD_OP(exp, read_page);
+	EXP_MD_COUNTER_INCREMENT(exp, read_page);
+	rc = MDP(exp->exp_obd, read_page)(exp, op_data, cb_op, hash_offset,
+					  ppage);
 	return rc;
 }
 
@@ -1559,6 +1497,16 @@ static inline int md_free_lustre_md(struct obd_export *exp,
 	return MDP(exp->exp_obd, free_lustre_md)(exp, md);
 }
 
+static inline int md_merge_attr(struct obd_export *exp,
+				const struct lmv_stripe_md *lsm,
+				struct cl_attr *attr,
+				ldlm_blocking_callback cb)
+{
+	EXP_CHECK_MD_OP(exp, merge_attr);
+	EXP_MD_COUNTER_INCREMENT(exp, merge_attr);
+	return MDP(exp->exp_obd, merge_attr)(exp, lsm, attr, cb);
+}
+
 static inline int md_setxattr(struct obd_export *exp, const struct lu_fid *fid,
 			      u64 valid, const char *name,
 			      const char *input, int input_size,
@@ -1603,7 +1551,8 @@ static inline int md_clear_open_replay_data(struct obd_export *exp,
 }
 
 static inline int md_set_lock_data(struct obd_export *exp,
-				   __u64 *lockh, void *data, __u64 *bits)
+				   const struct lustre_handle *lockh,
+				   void *data, __u64 *bits)
 {
 	EXP_CHECK_MD_OP(exp, set_lock_data);
 	EXP_MD_COUNTER_INCREMENT(exp, set_lock_data);
@@ -1674,6 +1623,19 @@ static inline int md_revalidate_lock(struct obd_export *exp,
 	return rc;
 }
 
+static inline int md_get_fid_from_lsm(struct obd_export *exp,
+				      const struct lmv_stripe_md *lsm,
+				      const char *name, int namelen,
+				      struct lu_fid *fid)
+{
+	int rc;
+
+	EXP_CHECK_MD_OP(exp, get_fid_from_lsm);
+	EXP_MD_COUNTER_INCREMENT(exp, get_fid_from_lsm);
+	rc = MDP(exp->exp_obd, get_fid_from_lsm)(exp, lsm, name, namelen, fid);
+	return rc;
+}
+
 /* OBD Metadata Support */
 
 int obd_init_caches(void);
@@ -1682,16 +1644,6 @@ void obd_cleanup_caches(void);
 /* support routines */
 extern struct kmem_cache *obdo_cachep;
 
-static inline void obdo2fid(struct obdo *oa, struct lu_fid *fid)
-{
-	/* something here */
-}
-
-static inline void fid2obdo(struct lu_fid *fid, struct obdo *oa)
-{
-	/* something here */
-}
-
 typedef int (*register_lwp_cb)(void *data);
 
 struct lwp_register_item {
@@ -1710,6 +1662,9 @@ struct lwp_register_item {
 extern int (*ptlrpc_put_connection_superhack)(struct ptlrpc_connection *c);
 
 /* obd_mount.c */
+int lustre_unregister_fs(void);
+int lustre_register_fs(void);
+int lustre_check_exclusion(struct super_block *sb, char *svname);
 
 /* sysctl.c */
 int obd_sysctl_init(void);
@@ -1730,8 +1685,24 @@ void class_exit_uuidlist(void);
 extern char obd_jobid_node[];
 extern struct miscdevice obd_psdev;
 extern spinlock_t obd_types_lock;
+int class_procfs_init(void);
+int class_procfs_clean(void);
 
 /* prng.c */
 #define ll_generate_random_uuid(uuid_out) cfs_get_random_bytes(uuid_out, sizeof(class_uuid_t))
 
+/* statfs_pack.c */
+struct kstatfs;
+void statfs_pack(struct obd_statfs *osfs, struct kstatfs *sfs);
+void statfs_unpack(struct kstatfs *sfs, struct obd_statfs *osfs);
+
+/* root squash info */
+struct rw_semaphore;
+struct root_squash_info {
+	uid_t			rsi_uid;
+	gid_t			rsi_gid;
+	struct list_head	rsi_nosquash_nids;
+	struct rw_semaphore	rsi_sem;
+};
+
 #endif /* __LINUX_OBD_CLASS_H */
diff --git a/drivers/staging/lustre/lustre/include/obd_support.h b/drivers/staging/lustre/lustre/include/obd_support.h
index 845e64a56c21..b346a7f10aa4 100644
--- a/drivers/staging/lustre/lustre/include/obd_support.h
+++ b/drivers/staging/lustre/lustre/include/obd_support.h
@@ -35,7 +35,7 @@
 
 #include <linux/slab.h>
 #include "../../include/linux/libcfs/libcfs.h"
-#include "linux/lustre_compat25.h"
+#include "lustre_compat.h"
 #include "lprocfs_status.h"
 
 /* global variables */
@@ -52,11 +52,9 @@ extern unsigned int at_max;
 extern unsigned int at_history;
 extern int at_early_margin;
 extern int at_extra;
-extern unsigned int obd_sync_filter;
-extern unsigned int obd_max_dirty_pages;
-extern atomic_t obd_unstable_pages;
-extern atomic_t obd_dirty_pages;
-extern atomic_t obd_dirty_transit_pages;
+extern unsigned long obd_max_dirty_pages;
+extern atomic_long_t obd_dirty_pages;
+extern atomic_long_t obd_dirty_transit_pages;
 extern char obd_jobid_var[];
 
 /* Some hash init argument constants */
@@ -117,17 +115,17 @@ extern char obd_jobid_var[];
  * running on a backup server. (If it's too low, import_select_connection
  * will increase the timeout anyhow.)
  */
-#define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN, obd_timeout/20)
+#define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN, obd_timeout / 20)
 /* The max delay between connects is SWITCH_MAX + SWITCH_INC + INITIAL */
 #define RECONNECT_DELAY_MAX (CONNECTION_SWITCH_MAX + CONNECTION_SWITCH_INC + \
 			     INITIAL_CONNECT_TIMEOUT)
 /* The min time a target should wait for clients to reconnect in recovery */
-#define OBD_RECOVERY_TIME_MIN    (2*RECONNECT_DELAY_MAX)
+#define OBD_RECOVERY_TIME_MIN    (2 * RECONNECT_DELAY_MAX)
 #define OBD_IR_FACTOR_MIN	 1
 #define OBD_IR_FACTOR_MAX	 10
-#define OBD_IR_FACTOR_DEFAULT    (OBD_IR_FACTOR_MAX/2)
+#define OBD_IR_FACTOR_DEFAULT    (OBD_IR_FACTOR_MAX / 2)
 /* default timeout for the MGS to become IR_FULL */
-#define OBD_IR_MGS_TIMEOUT       (4*obd_timeout)
+#define OBD_IR_MGS_TIMEOUT       (4 * obd_timeout)
 #define LONG_UNLINK 300	  /* Unlink should happen before now */
 
 /**
@@ -318,6 +316,10 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_LDLM_AGL_NOLOCK	 0x31b
 #define OBD_FAIL_LDLM_OST_LVB		 0x31c
 #define OBD_FAIL_LDLM_ENQUEUE_HANG	 0x31d
+#define OBD_FAIL_LDLM_CP_CB_WAIT2	 0x320
+#define OBD_FAIL_LDLM_CP_CB_WAIT3	 0x321
+#define OBD_FAIL_LDLM_CP_CB_WAIT4	 0x322
+#define OBD_FAIL_LDLM_CP_CB_WAIT5	 0x323
 
 /* LOCKLESS IO */
 #define OBD_FAIL_LDLM_SET_CONTENTION     0x385
@@ -400,6 +402,7 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_MDC_GETATTR_ENQUEUE     0x803
 #define OBD_FAIL_MDC_RPCS_SEM		 0x804
 #define OBD_FAIL_MDC_LIGHTWEIGHT	 0x805
+#define OBD_FAIL_MDC_CLOSE		 0x806
 
 #define OBD_FAIL_MGS		     0x900
 #define OBD_FAIL_MGS_ALL_REQUEST_NET     0x901
@@ -455,6 +458,7 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_LOV_INIT			    0x1403
 #define OBD_FAIL_GLIMPSE_DELAY			    0x1404
 #define OBD_FAIL_LLITE_XATTR_ENOMEM		    0x1405
+#define OBD_FAIL_GETATTR_DELAY			    0x1409
 
 #define OBD_FAIL_FID_INDIR	0x1501
 #define OBD_FAIL_FID_INLMA	0x1502
@@ -474,11 +478,16 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_LFSCK_CRASH		0x160a
 #define OBD_FAIL_LFSCK_NO_AUTO		0x160b
 #define OBD_FAIL_LFSCK_NO_DOUBLESCAN	0x160c
+#define OBD_FAIL_LFSCK_INVALID_PFID	0x1619
+#define OBD_FAIL_LFSCK_BAD_NAME_HASH	0x1628
 
 /* UPDATE */
 #define OBD_FAIL_UPDATE_OBJ_NET			0x1700
 #define OBD_FAIL_UPDATE_OBJ_NET_REP		0x1701
 
+/* LMV */
+#define OBD_FAIL_UNKNOWN_LMV_STRIPE		0x1901
+
 /* Assign references to moved code to reduce code changes */
 #define OBD_FAIL_PRECHECK(id)		   CFS_FAIL_PRECHECK(id)
 #define OBD_FAIL_CHECK(id)		      CFS_FAIL_CHECK(id)
@@ -520,7 +529,8 @@ do {									      \
 	POISON_PTR(ptr);						      \
 } while (0)
 
-#define KEY_IS(str) \
-	(keylen >= (sizeof(str)-1) && memcmp(key, str, (sizeof(str)-1)) == 0)
+#define KEY_IS(str)					\
+	(keylen >= (sizeof(str) - 1) &&			\
+	memcmp(key, str, (sizeof(str) - 1)) == 0)
 
 #endif
diff --git a/drivers/staging/lustre/lustre/ldlm/interval_tree.c b/drivers/staging/lustre/lustre/ldlm/interval_tree.c
index f4a70ebddeaf..e134ecd21bb2 100644
--- a/drivers/staging/lustre/lustre/ldlm/interval_tree.c
+++ b/drivers/staging/lustre/lustre/ldlm/interval_tree.c
@@ -90,6 +90,17 @@ static inline int extent_equal(struct interval_node_extent *e1,
 	return (e1->start == e2->start) && (e1->end == e2->end);
 }
 
+static inline int extent_overlapped(struct interval_node_extent *e1,
+				    struct interval_node_extent *e2)
+{
+	return (e1->start <= e2->end) && (e2->start <= e1->end);
+}
+
+static inline int node_equal(struct interval_node *n1, struct interval_node *n2)
+{
+	return extent_equal(&n1->in_extent, &n2->in_extent);
+}
+
 static inline __u64 max_u64(__u64 x, __u64 y)
 {
 	return x > y ? x : y;
@@ -262,7 +273,7 @@ struct interval_node *interval_insert(struct interval_node *node,
 	p = root;
 	while (*p) {
 		parent = *p;
-		if (extent_equal(&parent->in_extent, &node->in_extent))
+		if (node_equal(parent, node))
 			return parent;
 
 		/* max_high field must be updated after each iteration */
@@ -463,3 +474,90 @@ color:
 		interval_erase_color(child, parent, root);
 }
 EXPORT_SYMBOL(interval_erase);
+
+static inline int interval_may_overlap(struct interval_node *node,
+				       struct interval_node_extent *ext)
+{
+	return (ext->start <= node->in_max_high &&
+		ext->end >= interval_low(node));
+}
+
+/*
+ * This function finds all intervals that overlap interval ext,
+ * and calls func to handle resulted intervals one by one.
+ * in lustre, this function will find all conflicting locks in
+ * the granted queue and add these locks to the ast work list.
+ *
+ * {
+ *	if (!node)
+ *		return 0;
+ *	if (ext->end < interval_low(node)) {
+ *		interval_search(node->in_left, ext, func, data);
+ *	} else if (interval_may_overlap(node, ext)) {
+ *		if (extent_overlapped(ext, &node->in_extent))
+ *			func(node, data);
+ *		interval_search(node->in_left, ext, func, data);
+ *		interval_search(node->in_right, ext, func, data);
+ *	}
+ *	return 0;
+ * }
+ *
+ */
+enum interval_iter interval_search(struct interval_node *node,
+				   struct interval_node_extent *ext,
+				   interval_callback_t func,
+				   void *data)
+{
+	enum interval_iter rc = INTERVAL_ITER_CONT;
+	struct interval_node *parent;
+
+	LASSERT(ext);
+	LASSERT(func);
+
+	while (node) {
+		if (ext->end < interval_low(node)) {
+			if (node->in_left) {
+				node = node->in_left;
+				continue;
+			}
+		} else if (interval_may_overlap(node, ext)) {
+			if (extent_overlapped(ext, &node->in_extent)) {
+				rc = func(node, data);
+				if (rc == INTERVAL_ITER_STOP)
+					break;
+			}
+
+			if (node->in_left) {
+				node = node->in_left;
+				continue;
+			}
+			if (node->in_right) {
+				node = node->in_right;
+				continue;
+			}
+		}
+
+		parent = node->in_parent;
+		while (parent) {
+			if (node_is_left_child(node) &&
+			    parent->in_right) {
+				/*
+				 * If we ever got the left, it means that the
+				 * parent met ext->end<interval_low(parent), or
+				 * may_overlap(parent). If the former is true,
+				 * we needn't go back. So stop early and check
+				 * may_overlap(parent) after this loop.
+				 */
+				node = parent->in_right;
+				break;
+			}
+			node = parent;
+			parent = parent->in_parent;
+		}
+		if (!parent || !interval_may_overlap(parent, ext))
+			break;
+	}
+
+	return rc;
+}
+EXPORT_SYMBOL(interval_search);
diff --git a/drivers/staging/lustre/lustre/ldlm/l_lock.c b/drivers/staging/lustre/lustre/ldlm/l_lock.c
index ea8840cb9056..3845f386f1db 100644
--- a/drivers/staging/lustre/lustre/ldlm/l_lock.c
+++ b/drivers/staging/lustre/lustre/ldlm/l_lock.c
@@ -45,6 +45,8 @@
  * being an atomic operation.
  */
 struct ldlm_resource *lock_res_and_lock(struct ldlm_lock *lock)
+				__acquires(&lock->l_lock)
+				__acquires(&lock->l_resource->lr_lock)
 {
 	spin_lock(&lock->l_lock);
 
@@ -59,6 +61,8 @@ EXPORT_SYMBOL(lock_res_and_lock);
  * Unlock a lock and its resource previously locked with lock_res_and_lock
  */
 void unlock_res_and_lock(struct ldlm_lock *lock)
+		__releases(&lock->l_resource->lr_lock)
+		__releases(&lock->l_lock)
 {
 	/* on server-side resource of lock doesn't change */
 	ldlm_clear_res_locked(lock);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c b/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
index f5023d9b78f5..ecf472e4813d 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
@@ -221,7 +221,7 @@ void ldlm_extent_unlink_lock(struct ldlm_lock *lock)
 }
 
 void ldlm_extent_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				     ldlm_policy_data_t *lpolicy)
+				      ldlm_policy_data_t *lpolicy)
 {
 	memset(lpolicy, 0, sizeof(*lpolicy));
 	lpolicy->l_extent.start = wpolicy->l_extent.start;
@@ -230,7 +230,7 @@ void ldlm_extent_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
 }
 
 void ldlm_extent_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				     ldlm_wire_policy_data_t *wpolicy)
+				      ldlm_wire_policy_data_t *wpolicy)
 {
 	memset(wpolicy, 0, sizeof(*wpolicy));
 	wpolicy->l_extent.start = lpolicy->l_extent.start;
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c b/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
index d6b61bc39135..861f36f039b5 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
@@ -97,7 +97,7 @@ ldlm_flock_destroy(struct ldlm_lock *lock, enum ldlm_mode mode, __u64 flags)
 	LASSERT(hlist_unhashed(&lock->l_exp_flock_hash));
 
 	list_del_init(&lock->l_res_link);
-	if (flags == LDLM_FL_WAIT_NOREPROC && !ldlm_is_failed(lock)) {
+	if (flags == LDLM_FL_WAIT_NOREPROC) {
 		/* client side - set a flag to prevent sending a CANCEL */
 		lock->l_flags |= LDLM_FL_LOCAL_ONLY | LDLM_FL_CBPENDING;
 
@@ -166,7 +166,7 @@ reprocess:
 		 */
 		list_for_each(tmp, &res->lr_granted) {
 			lock = list_entry(tmp, struct ldlm_lock,
-					      l_res_link);
+					  l_res_link);
 			if (ldlm_same_flock_owner(lock, req)) {
 				ownlocks = tmp;
 				break;
@@ -182,7 +182,7 @@ reprocess:
 		 */
 		list_for_each(tmp, &res->lr_granted) {
 			lock = list_entry(tmp, struct ldlm_lock,
-					      l_res_link);
+					  l_res_link);
 
 			if (ldlm_same_flock_owner(lock, req)) {
 				if (!ownlocks)
@@ -339,10 +339,10 @@ reprocess:
 						lock->l_granted_mode, &null_cbs,
 						NULL, 0, LVB_T_NONE);
 			lock_res_and_lock(req);
-			if (!new2) {
+			if (IS_ERR(new2)) {
 				ldlm_flock_destroy(req, lock->l_granted_mode,
 						   *flags);
-				*err = -ENOLCK;
+				*err = PTR_ERR(new2);
 				return LDLM_ITER_STOP;
 			}
 			goto reprocess;
@@ -455,29 +455,22 @@ ldlm_flock_completion_ast(struct ldlm_lock *lock, __u64 flags, void *data)
 	enum ldlm_error		    err;
 	int			     rc = 0;
 
+	OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_CP_CB_WAIT2, 4);
+	if (OBD_FAIL_PRECHECK(OBD_FAIL_LDLM_CP_CB_WAIT3)) {
+		lock_res_and_lock(lock);
+		lock->l_flags |= LDLM_FL_FAIL_LOC;
+		unlock_res_and_lock(lock);
+		OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_CP_CB_WAIT3, 4);
+	}
 	CDEBUG(D_DLMTRACE, "flags: 0x%llx data: %p getlk: %p\n",
 	       flags, data, getlk);
 
-	/* Import invalidation. We need to actually release the lock
-	 * references being held, so that it can go away. No point in
-	 * holding the lock even if app still believes it has it, since
-	 * server already dropped it anyway. Only for granted locks too.
-	 */
-	if ((lock->l_flags & (LDLM_FL_FAILED|LDLM_FL_LOCAL_ONLY)) ==
-	    (LDLM_FL_FAILED|LDLM_FL_LOCAL_ONLY)) {
-		if (lock->l_req_mode == lock->l_granted_mode &&
-		    lock->l_granted_mode != LCK_NL && !data)
-			ldlm_lock_decref_internal(lock, lock->l_req_mode);
-
-		/* Need to wake up the waiter if we were evicted */
-		wake_up(&lock->l_waitq);
-		return 0;
-	}
-
 	LASSERT(flags != LDLM_FL_WAIT_NOREPROC);
 
-	if (!(flags & (LDLM_FL_BLOCK_WAIT | LDLM_FL_BLOCK_GRANTED |
-		       LDLM_FL_BLOCK_CONV))) {
+	if (flags & LDLM_FL_FAILED)
+		goto granted;
+
+	if (!(flags & LDLM_FL_BLOCKED_MASK)) {
 		if (!data)
 			/* mds granted the lock in the reply */
 			goto granted;
@@ -514,12 +507,21 @@ ldlm_flock_completion_ast(struct ldlm_lock *lock, __u64 flags, void *data)
 granted:
 	OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_CP_CB_WAIT, 10);
 
-	if (ldlm_is_failed(lock)) {
-		LDLM_DEBUG(lock, "client-side enqueue waking up: failed");
-		return -EIO;
+	if (OBD_FAIL_PRECHECK(OBD_FAIL_LDLM_CP_CB_WAIT4)) {
+		lock_res_and_lock(lock);
+		/* DEADLOCK is always set with CBPENDING */
+		lock->l_flags |= LDLM_FL_FLOCK_DEADLOCK | LDLM_FL_CBPENDING;
+		unlock_res_and_lock(lock);
+		OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_CP_CB_WAIT4, 4);
+	}
+	if (OBD_FAIL_PRECHECK(OBD_FAIL_LDLM_CP_CB_WAIT5)) {
+		lock_res_and_lock(lock);
+		/* DEADLOCK is always set with CBPENDING */
+		lock->l_flags |= LDLM_FL_FAIL_LOC |
+				 LDLM_FL_FLOCK_DEADLOCK | LDLM_FL_CBPENDING;
+		unlock_res_and_lock(lock);
+		OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_CP_CB_WAIT5, 4);
 	}
-
-	LDLM_DEBUG(lock, "client-side enqueue granted");
 
 	lock_res_and_lock(lock);
 
@@ -530,20 +532,59 @@ granted:
 	if (ldlm_is_destroyed(lock)) {
 		unlock_res_and_lock(lock);
 		LDLM_DEBUG(lock, "client-side enqueue waking up: destroyed");
-		return 0;
+		/*
+		 * An error is still to be returned, to propagate it up to
+		 * ldlm_cli_enqueue_fini() caller.
+		 */
+		return -EIO;
 	}
 
 	/* ldlm_lock_enqueue() has already placed lock on the granted list. */
-	list_del_init(&lock->l_res_link);
+	ldlm_resource_unlink_lock(lock);
+
+	/*
+	 * Import invalidation. We need to actually release the lock
+	 * references being held, so that it can go away. No point in
+	 * holding the lock even if app still believes it has it, since
+	 * server already dropped it anyway. Only for granted locks too.
+	 */
+	/* Do the same for DEADLOCK'ed locks. */
+	if (ldlm_is_failed(lock) || ldlm_is_flock_deadlock(lock)) {
+		int mode;
+
+		if (flags & LDLM_FL_TEST_LOCK)
+			LASSERT(ldlm_is_test_lock(lock));
+
+		if (ldlm_is_test_lock(lock) || ldlm_is_flock_deadlock(lock))
+			mode = getlk->fl_type;
+		else
+			mode = lock->l_granted_mode;
+
+		if (ldlm_is_flock_deadlock(lock)) {
+			LDLM_DEBUG(lock, "client-side enqueue deadlock received");
+			rc = -EDEADLK;
+		}
+		ldlm_flock_destroy(lock, mode, LDLM_FL_WAIT_NOREPROC);
+		unlock_res_and_lock(lock);
+
+		/* Need to wake up the waiter if we were evicted */
+		wake_up(&lock->l_waitq);
+
+		/*
+		 * An error is still to be returned, to propagate it up to
+		 * ldlm_cli_enqueue_fini() caller.
+		 */
+		return rc ? : -EIO;
+	}
+
+	LDLM_DEBUG(lock, "client-side enqueue granted");
 
-	if (ldlm_is_flock_deadlock(lock)) {
-		LDLM_DEBUG(lock, "client-side enqueue deadlock received");
-		rc = -EDEADLK;
-	} else if (flags & LDLM_FL_TEST_LOCK) {
+	if (flags & LDLM_FL_TEST_LOCK) {
 		/* fcntl(F_GETLK) request */
 		/* The old mode was saved in getlk->fl_type so that if the mode
 		 * in the lock changes we can decref the appropriate refcount.
 		 */
+		LASSERT(ldlm_is_test_lock(lock));
 		ldlm_flock_destroy(lock, getlk->fl_type, LDLM_FL_WAIT_NOREPROC);
 		switch (lock->l_granted_mode) {
 		case LCK_PR:
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h b/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h
index e4cf65d2d3b1..5e82cfc245b2 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h
@@ -100,9 +100,10 @@ enum {
 int ldlm_cancel_lru(struct ldlm_namespace *ns, int nr,
 		    enum ldlm_cancel_flags sync, int flags);
 int ldlm_cancel_lru_local(struct ldlm_namespace *ns,
-			 struct list_head *cancels, int count, int max,
-			 enum ldlm_cancel_flags cancel_flags, int flags);
-extern int ldlm_enqueue_min;
+			  struct list_head *cancels, int count, int max,
+			  enum ldlm_cancel_flags cancel_flags, int flags);
+extern unsigned int ldlm_enqueue_min;
+extern unsigned int ldlm_cancel_unused_locks_before_replay;
 
 /* ldlm_resource.c */
 int ldlm_resource_putref_locked(struct ldlm_resource *res);
@@ -200,8 +201,7 @@ ldlm_interval_extent(struct ldlm_interval *node)
 
 	LASSERT(!list_empty(&node->li_group));
 
-	lock = list_entry(node->li_group.next, struct ldlm_lock,
-			      l_sl_policy);
+	lock = list_entry(node->li_group.next, struct ldlm_lock, l_sl_policy);
 	return &lock->l_policy_data.l_extent;
 }
 
@@ -302,7 +302,7 @@ static inline int is_granted_or_cancelled(struct ldlm_lock *lock)
 
 	lock_res_and_lock(lock);
 	if ((lock->l_req_mode == lock->l_granted_mode) &&
-	     !ldlm_is_cp_reqd(lock))
+	    !ldlm_is_cp_reqd(lock))
 		ret = 1;
 	else if (ldlm_is_failed(lock) || ldlm_is_cancel(lock))
 		ret = 1;
@@ -326,13 +326,13 @@ void ldlm_ibits_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
 void ldlm_ibits_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
 				     ldlm_wire_policy_data_t *wpolicy);
 void ldlm_extent_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				     ldlm_policy_data_t *lpolicy);
+				      ldlm_policy_data_t *lpolicy);
 void ldlm_extent_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				     ldlm_wire_policy_data_t *wpolicy);
+				      ldlm_wire_policy_data_t *wpolicy);
 void ldlm_flock_policy_wire18_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				     ldlm_policy_data_t *lpolicy);
+				       ldlm_policy_data_t *lpolicy);
 void ldlm_flock_policy_wire21_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				     ldlm_policy_data_t *lpolicy);
+				       ldlm_policy_data_t *lpolicy);
 
 void ldlm_flock_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
 				     ldlm_wire_policy_data_t *wpolicy);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
index 7c832aae7d5e..153e990c494e 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
@@ -82,7 +82,7 @@ static int import_set_conn(struct obd_import *imp, struct obd_uuid *uuid,
 			if (priority) {
 				list_del(&item->oic_item);
 				list_add(&item->oic_item,
-					     &imp->imp_conn_list);
+					 &imp->imp_conn_list);
 				item->oic_last_attempt = 0;
 			}
 			CDEBUG(D_HA, "imp %p@%s: found existing conn %s%s\n",
@@ -102,7 +102,7 @@ static int import_set_conn(struct obd_import *imp, struct obd_uuid *uuid,
 			list_add(&imp_conn->oic_item, &imp->imp_conn_list);
 		else
 			list_add_tail(&imp_conn->oic_item,
-					  &imp->imp_conn_list);
+				      &imp->imp_conn_list);
 		CDEBUG(D_HA, "imp %p@%s: add connection %s at %s\n",
 		       imp, imp->imp_obd->obd_name, uuid->uuid,
 		       (priority ? "head" : "tail"));
@@ -299,12 +299,14 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
 	       min_t(unsigned int, LUSTRE_CFG_BUFLEN(lcfg, 2),
 		     sizeof(server_uuid)));
 
-	cli->cl_dirty = 0;
+	cli->cl_dirty_pages = 0;
 	cli->cl_avail_grant = 0;
-	/* FIXME: Should limit this for the sum of all cl_dirty_max. */
-	cli->cl_dirty_max = OSC_MAX_DIRTY_DEFAULT * 1024 * 1024;
-	if (cli->cl_dirty_max >> PAGE_SHIFT > totalram_pages / 8)
-		cli->cl_dirty_max = totalram_pages << (PAGE_SHIFT - 3);
+	/* FIXME: Should limit this for the sum of all cl_dirty_max_pages. */
+	/*
+	 * cl_dirty_max_pages may be changed at connect time in
+	 * ptlrpc_connect_interpret().
+	 */
+	client_adjust_max_dirty(cli);
 	INIT_LIST_HEAD(&cli->cl_cache_waiters);
 	INIT_LIST_HEAD(&cli->cl_loi_ready_list);
 	INIT_LIST_HEAD(&cli->cl_loi_hp_ready_list);
@@ -326,11 +328,11 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
 	/* lru for osc. */
 	INIT_LIST_HEAD(&cli->cl_lru_osc);
 	atomic_set(&cli->cl_lru_shrinkers, 0);
-	atomic_set(&cli->cl_lru_busy, 0);
-	atomic_set(&cli->cl_lru_in_list, 0);
+	atomic_long_set(&cli->cl_lru_busy, 0);
+	atomic_long_set(&cli->cl_lru_in_list, 0);
 	INIT_LIST_HEAD(&cli->cl_lru_list);
 	spin_lock_init(&cli->cl_lru_list_lock);
-	atomic_set(&cli->cl_unstable_count, 0);
+	atomic_long_set(&cli->cl_unstable_count, 0);
 
 	init_waitqueue_head(&cli->cl_destroy_waitq);
 	atomic_set(&cli->cl_destroy_in_flight, 0);
@@ -360,7 +362,7 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
 	cli->cl_chunkbits = PAGE_SHIFT;
 
 	if (!strcmp(name, LUSTRE_MDC_NAME)) {
-		cli->cl_max_rpcs_in_flight = MDC_MAX_RIF_DEFAULT;
+		cli->cl_max_rpcs_in_flight = OBD_MAX_RIF_DEFAULT;
 	} else if (totalram_pages >> (20 - PAGE_SHIFT) <= 128 /* MB */) {
 		cli->cl_max_rpcs_in_flight = 2;
 	} else if (totalram_pages >> (20 - PAGE_SHIFT) <= 256 /* MB */) {
@@ -368,7 +370,7 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
 	} else if (totalram_pages >> (20 - PAGE_SHIFT) <= 512 /* MB */) {
 		cli->cl_max_rpcs_in_flight = 4;
 	} else {
-		cli->cl_max_rpcs_in_flight = OSC_MAX_RIF_DEFAULT;
+		cli->cl_max_rpcs_in_flight = OBD_MAX_RIF_DEFAULT;
 	}
 	rc = ldlm_get_ref();
 	if (rc) {
@@ -534,7 +536,7 @@ int client_disconnect_export(struct obd_export *exp)
 	imp = cli->cl_import;
 
 	down_write(&cli->cl_sem);
-	CDEBUG(D_INFO, "disconnect %s - %d\n", obd->obd_name,
+	CDEBUG(D_INFO, "disconnect %s - %zu\n", obd->obd_name,
 	       cli->cl_conn_count);
 
 	if (!cli->cl_conn_count) {
@@ -690,7 +692,7 @@ void target_send_reply(struct ptlrpc_request *req, int rc, int fail_id)
 	if (rs->rs_transno > exp->exp_last_committed) {
 		/* not committed already */
 		list_add_tail(&rs->rs_obd_list,
-				  &exp->exp_uncommitted_replies);
+			      &exp->exp_uncommitted_replies);
 	}
 	spin_unlock(&exp->exp_uncommitted_replies_lock);
 
@@ -795,7 +797,7 @@ void ldlm_dump_export_locks(struct obd_export *exp)
 		CERROR("dumping locks for export %p,ignore if the unmount doesn't hang\n",
 		       exp);
 		list_for_each_entry(lock, &exp->exp_locks_list,
-					l_exp_refs_link)
+				    l_exp_refs_link)
 			LDLM_ERROR(lock, "lock:");
 	}
 	spin_unlock(&exp->exp_locks_list_guard);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
index a5993f745ebe..3c48b4fb96f1 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
@@ -481,8 +481,8 @@ int ldlm_lock_change_resource(struct ldlm_namespace *ns, struct ldlm_lock *lock,
 	unlock_res_and_lock(lock);
 
 	newres = ldlm_resource_get(ns, NULL, new_resid, type, 1);
-	if (!newres)
-		return -ENOMEM;
+	if (IS_ERR(newres))
+		return PTR_ERR(newres);
 
 	lu_ref_add(&newres->lr_reference, "lock", lock);
 	/*
@@ -542,7 +542,7 @@ struct ldlm_lock *__ldlm_handle2lock(const struct lustre_handle *handle,
 
 	LASSERT(handle);
 
-	lock = class_handle2object(handle->cookie);
+	lock = class_handle2object(handle->cookie, NULL);
 	if (!lock)
 		return NULL;
 
@@ -937,7 +937,7 @@ static void search_granted_lock(struct list_head *queue,
 				/* go to next policy group within mode group */
 				tmp = policy_end->l_res_link.next;
 				lock = list_entry(tmp, struct ldlm_lock,
-						      l_res_link);
+						  l_res_link);
 			}  /* loop over policy groups within the mode group */
 
 			/* insert point is last lock of the mode group,
@@ -1028,15 +1028,28 @@ void ldlm_grant_lock(struct ldlm_lock *lock, struct list_head *work_list)
 	check_res_locked(res);
 
 	lock->l_granted_mode = lock->l_req_mode;
+
+	if (work_list && lock->l_completion_ast)
+		ldlm_add_ast_work_item(lock, NULL, work_list);
+
 	if (res->lr_type == LDLM_PLAIN || res->lr_type == LDLM_IBITS)
 		ldlm_grant_lock_with_skiplist(lock);
 	else if (res->lr_type == LDLM_EXTENT)
 		ldlm_extent_add_lock(res, lock);
-	else
+	else if (res->lr_type == LDLM_FLOCK) {
+		/*
+		 * We should not add locks to granted list in the following cases:
+		 * - this is an UNLOCK but not a real lock;
+		 * - this is a TEST lock;
+		 * - this is a F_CANCELLK lock (async flock has req_mode == 0)
+		 * - this is a deadlock (flock cannot be granted)
+		 */
+		if (!lock->l_req_mode || lock->l_req_mode == LCK_NL ||
+		    ldlm_is_test_lock(lock) || ldlm_is_flock_deadlock(lock))
+			return;
 		ldlm_resource_add_lock(res, &res->lr_granted, lock);
-
-	if (work_list && lock->l_completion_ast)
-		ldlm_add_ast_work_item(lock, NULL, work_list);
+	} else
+		LBUG();
 
 	ldlm_pool_add(&ldlm_res_to_ns(res)->ns_pool, lock);
 }
@@ -1103,7 +1116,7 @@ static struct ldlm_lock *search_queue(struct list_head *queue,
 		 * of bits.
 		 */
 		if (lock->l_resource->lr_type == LDLM_IBITS &&
-		     ((lock->l_policy_data.l_inodebits.bits &
+		    ((lock->l_policy_data.l_inodebits.bits &
 		      policy->l_inodebits.bits) !=
 		      policy->l_inodebits.bits))
 			continue;
@@ -1214,7 +1227,7 @@ enum ldlm_mode ldlm_lock_match(struct ldlm_namespace *ns, __u64 flags,
 	}
 
 	res = ldlm_resource_get(ns, NULL, res_id, type, 0);
-	if (!res) {
+	if (IS_ERR(res)) {
 		LASSERT(!old_lock);
 		return 0;
 	}
@@ -1363,12 +1376,12 @@ int ldlm_fill_lvb(struct ldlm_lock *lock, struct req_capsule *pill,
 		if (size == sizeof(struct ost_lvb)) {
 			if (loc == RCL_CLIENT)
 				lvb = req_capsule_client_swab_get(pill,
-						&RMF_DLM_LVB,
-						lustre_swab_ost_lvb);
+								  &RMF_DLM_LVB,
+							lustre_swab_ost_lvb);
 			else
 				lvb = req_capsule_server_swab_get(pill,
-						&RMF_DLM_LVB,
-						lustre_swab_ost_lvb);
+								  &RMF_DLM_LVB,
+							lustre_swab_ost_lvb);
 			if (unlikely(!lvb)) {
 				LDLM_ERROR(lock, "no LVB");
 				return -EPROTO;
@@ -1380,8 +1393,8 @@ int ldlm_fill_lvb(struct ldlm_lock *lock, struct req_capsule *pill,
 
 			if (loc == RCL_CLIENT)
 				lvb = req_capsule_client_swab_get(pill,
-						&RMF_DLM_LVB,
-						lustre_swab_ost_lvb_v1);
+								  &RMF_DLM_LVB,
+							lustre_swab_ost_lvb_v1);
 			else
 				lvb = req_capsule_server_sized_swab_get(pill,
 						&RMF_DLM_LVB, size,
@@ -1405,12 +1418,12 @@ int ldlm_fill_lvb(struct ldlm_lock *lock, struct req_capsule *pill,
 		if (size == sizeof(struct lquota_lvb)) {
 			if (loc == RCL_CLIENT)
 				lvb = req_capsule_client_swab_get(pill,
-						&RMF_DLM_LVB,
-						lustre_swab_lquota_lvb);
+								  &RMF_DLM_LVB,
+							lustre_swab_lquota_lvb);
 			else
 				lvb = req_capsule_server_swab_get(pill,
-						&RMF_DLM_LVB,
-						lustre_swab_lquota_lvb);
+								  &RMF_DLM_LVB,
+							lustre_swab_lquota_lvb);
 			if (unlikely(!lvb)) {
 				LDLM_ERROR(lock, "no LVB");
 				return -EPROTO;
@@ -1462,15 +1475,15 @@ struct ldlm_lock *ldlm_lock_create(struct ldlm_namespace *ns,
 {
 	struct ldlm_lock *lock;
 	struct ldlm_resource *res;
+	int rc;
 
 	res = ldlm_resource_get(ns, NULL, res_id, type, 1);
-	if (!res)
-		return NULL;
+	if (IS_ERR(res))
+		return ERR_CAST(res);
 
 	lock = ldlm_lock_new(res);
-
 	if (!lock)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	lock->l_req_mode = mode;
 	lock->l_ast_data = data;
@@ -1484,27 +1497,33 @@ struct ldlm_lock *ldlm_lock_create(struct ldlm_namespace *ns,
 	lock->l_tree_node = NULL;
 	/* if this is the extent lock, allocate the interval tree node */
 	if (type == LDLM_EXTENT) {
-		if (!ldlm_interval_alloc(lock))
+		if (!ldlm_interval_alloc(lock)) {
+			rc = -ENOMEM;
 			goto out;
+		}
 	}
 
 	if (lvb_len) {
 		lock->l_lvb_len = lvb_len;
 		lock->l_lvb_data = kzalloc(lvb_len, GFP_NOFS);
-		if (!lock->l_lvb_data)
+		if (!lock->l_lvb_data) {
+			rc = -ENOMEM;
 			goto out;
+		}
 	}
 
 	lock->l_lvb_type = lvb_type;
-	if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_NEW_LOCK))
+	if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_NEW_LOCK)) {
+		rc = -ENOENT;
 		goto out;
+	}
 
 	return lock;
 
 out:
 	ldlm_lock_destroy(lock);
 	LDLM_LOCK_RELEASE(lock);
-	return NULL;
+	return ERR_PTR(rc);
 }
 
 /**
@@ -1522,16 +1541,13 @@ enum ldlm_error ldlm_lock_enqueue(struct ldlm_namespace *ns,
 	struct ldlm_lock *lock = *lockp;
 	struct ldlm_resource *res = lock->l_resource;
 
-	lock->l_last_activity = ktime_get_real_seconds();
-
 	lock_res_and_lock(lock);
 	if (lock->l_req_mode == lock->l_granted_mode) {
 		/* The server returned a blocked lock, but it was granted
 		 * before we got a chance to actually enqueue it.  We don't
 		 * need to do anything else.
 		 */
-		*flags &= ~(LDLM_FL_BLOCK_GRANTED |
-			    LDLM_FL_BLOCK_CONV | LDLM_FL_BLOCK_WAIT);
+		*flags &= ~LDLM_FL_BLOCKED_MASK;
 		goto out;
 	}
 
@@ -1546,6 +1562,8 @@ enum ldlm_error ldlm_lock_enqueue(struct ldlm_namespace *ns,
 	 */
 	if (*flags & LDLM_FL_AST_DISCARD_DATA)
 		ldlm_set_ast_discard_data(lock);
+	if (*flags & LDLM_FL_TEST_LOCK)
+		ldlm_set_test_lock(lock);
 
 	/*
 	 * This distinction between local lock trees is very important; a client
@@ -1688,7 +1706,7 @@ static int ldlm_work_gl_ast_lock(struct ptlrpc_request_set *rqset, void *opaq)
 		return -ENOENT;
 
 	gl_work = list_entry(arg->list->next, struct ldlm_glimpse_work,
-				 gl_list);
+			     gl_list);
 	list_del_init(&gl_work->gl_list);
 
 	lock = gl_work->gl_lock;
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
index 821939ff2e6b..fde697ebaadc 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
@@ -559,8 +559,11 @@ static int ldlm_callback_handler(struct ptlrpc_request *req)
 
 	switch (lustre_msg_get_opc(req->rq_reqmsg)) {
 	case LDLM_BL_CALLBACK:
-		if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_BL_CALLBACK_NET))
+		if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_BL_CALLBACK_NET)) {
+			if (cfs_fail_err)
+				ldlm_callback_reply(req, -(int)cfs_fail_err);
 			return 0;
+		}
 		break;
 	case LDLM_CP_CALLBACK:
 		if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_CP_CALLBACK_NET))
@@ -706,12 +709,12 @@ static struct ldlm_bl_work_item *ldlm_bl_get_work(struct ldlm_bl_pool *blp)
 	if (!list_empty(&blp->blp_list) &&
 	    (list_empty(&blp->blp_prio_list) || num_bl == 0))
 		blwi = list_entry(blp->blp_list.next,
-				      struct ldlm_bl_work_item, blwi_entry);
+				  struct ldlm_bl_work_item, blwi_entry);
 	else
 		if (!list_empty(&blp->blp_prio_list))
 			blwi = list_entry(blp->blp_prio_list.next,
-					      struct ldlm_bl_work_item,
-					      blwi_entry);
+					  struct ldlm_bl_work_item,
+					  blwi_entry);
 
 	if (blwi) {
 		if (++num_bl >= atomic_read(&blp->blp_num_threads))
@@ -741,7 +744,7 @@ static int ldlm_bl_thread_start(struct ldlm_bl_pool *blp)
 	init_completion(&bltd.bltd_comp);
 	bltd.bltd_num = atomic_read(&blp->blp_num_threads);
 	snprintf(bltd.bltd_name, sizeof(bltd.bltd_name),
-		"ldlm_bl_%02d", bltd.bltd_num);
+		 "ldlm_bl_%02d", bltd.bltd_num);
 	task = kthread_run(ldlm_bl_thread_main, &bltd, "%s", bltd.bltd_name);
 	if (IS_ERR(task)) {
 		CERROR("cannot start LDLM thread ldlm_bl_%02d: rc %ld\n",
@@ -786,8 +789,8 @@ static int ldlm_bl_thread_main(void *arg)
 		if (!blwi) {
 			atomic_dec(&blp->blp_busy_threads);
 			l_wait_event_exclusive(blp->blp_waitq,
-					 (blwi = ldlm_bl_get_work(blp)),
-					 &lwi);
+					       (blwi = ldlm_bl_get_work(blp)),
+					       &lwi);
 			busy = atomic_inc_return(&blp->blp_busy_threads);
 		} else {
 			busy = atomic_read(&blp->blp_busy_threads);
@@ -874,8 +877,6 @@ void ldlm_put_ref(void)
 }
 EXPORT_SYMBOL(ldlm_put_ref);
 
-extern unsigned int ldlm_cancel_unused_locks_before_replay;
-
 static ssize_t cancel_unused_locks_before_replay_show(struct kobject *kobj,
 						      struct attribute *attr,
 						      char *buf)
@@ -1094,16 +1095,17 @@ int ldlm_init(void)
 		return -ENOMEM;
 
 	ldlm_lock_slab = kmem_cache_create("ldlm_locks",
-			      sizeof(struct ldlm_lock), 0,
-			      SLAB_HWCACHE_ALIGN | SLAB_DESTROY_BY_RCU, NULL);
+					   sizeof(struct ldlm_lock), 0,
+					   SLAB_HWCACHE_ALIGN |
+					   SLAB_DESTROY_BY_RCU, NULL);
 	if (!ldlm_lock_slab) {
 		kmem_cache_destroy(ldlm_resource_slab);
 		return -ENOMEM;
 	}
 
 	ldlm_interval_slab = kmem_cache_create("interval_node",
-					sizeof(struct ldlm_interval),
-					0, SLAB_HWCACHE_ALIGN, NULL);
+					       sizeof(struct ldlm_interval),
+					       0, SLAB_HWCACHE_ALIGN, NULL);
 	if (!ldlm_interval_slab) {
 		kmem_cache_destroy(ldlm_resource_slab);
 		kmem_cache_destroy(ldlm_lock_slab);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c b/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c
index 657ed4012776..9a1136e32dfc 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c
@@ -357,38 +357,40 @@ static int ldlm_pool_recalc(struct ldlm_pool *pl)
 	int count;
 
 	recalc_interval_sec = ktime_get_seconds() - pl->pl_recalc_time;
-	if (recalc_interval_sec <= 0)
-		goto recalc;
-
-	spin_lock(&pl->pl_lock);
 	if (recalc_interval_sec > 0) {
-		/*
-		 * Update pool statistics every 1s.
-		 */
-		ldlm_pool_recalc_stats(pl);
-
-		/*
-		 * Zero out all rates and speed for the last period.
-		 */
-		atomic_set(&pl->pl_grant_rate, 0);
-		atomic_set(&pl->pl_cancel_rate, 0);
+		spin_lock(&pl->pl_lock);
+		recalc_interval_sec = ktime_get_seconds() - pl->pl_recalc_time;
+
+		if (recalc_interval_sec > 0) {
+			/*
+			 * Update pool statistics every 1s.
+			 */
+			ldlm_pool_recalc_stats(pl);
+
+			/*
+			 * Zero out all rates and speed for the last period.
+			 */
+			atomic_set(&pl->pl_grant_rate, 0);
+			atomic_set(&pl->pl_cancel_rate, 0);
+		}
+		spin_unlock(&pl->pl_lock);
 	}
-	spin_unlock(&pl->pl_lock);
 
- recalc:
 	if (pl->pl_ops->po_recalc) {
 		count = pl->pl_ops->po_recalc(pl);
 		lprocfs_counter_add(pl->pl_stats, LDLM_POOL_RECALC_STAT,
 				    count);
 	}
+
 	recalc_interval_sec = pl->pl_recalc_time - ktime_get_seconds() +
 			      pl->pl_recalc_period;
 	if (recalc_interval_sec <= 0) {
+		/* DEBUG: should be re-removed after LU-4536 is fixed */
+		CDEBUG(D_DLMTRACE, "%s: Negative interval(%ld), too short period(%ld)\n",
+		       pl->pl_name, (long)recalc_interval_sec,
+		       (long)pl->pl_recalc_period);
+
 		/* Prevent too frequent recalculation. */
-		CDEBUG(D_DLMTRACE,
-		       "Negative interval(%d), too short period(%lld)",
-		       recalc_interval_sec,
-		       (s64)pl->pl_recalc_period);
 		recalc_interval_sec = 1;
 	}
 
@@ -792,7 +794,8 @@ static struct completion ldlm_pools_comp;
  */
 static unsigned long ldlm_pools_count(ldlm_side_t client, gfp_t gfp_mask)
 {
-	int total = 0, nr_ns;
+	unsigned long total = 0;
+	int nr_ns;
 	struct ldlm_namespace *ns;
 	struct ldlm_namespace *ns_old = NULL; /* loop detection */
 	void *cookie;
@@ -995,7 +998,7 @@ static int ldlm_pools_thread_main(void *arg)
 	wake_up(&thread->t_ctl_waitq);
 
 	CDEBUG(D_DLMTRACE, "%s: pool thread starting, process %d\n",
-		"ldlm_poold", current_pid());
+	       "ldlm_poold", current_pid());
 
 	while (1) {
 		struct l_wait_info lwi;
@@ -1025,7 +1028,7 @@ static int ldlm_pools_thread_main(void *arg)
 	wake_up(&thread->t_ctl_waitq);
 
 	CDEBUG(D_DLMTRACE, "%s: pool thread exiting, process %d\n",
-		"ldlm_poold", current_pid());
+	       "ldlm_poold", current_pid());
 
 	complete_and_exit(&ldlm_pools_comp, 0);
 }
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_request.c b/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
index af487f9937f4..35ba6f14d95f 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
@@ -63,8 +63,8 @@
 
 #include "ldlm_internal.h"
 
-int ldlm_enqueue_min = OBD_TIMEOUT_DEFAULT;
-module_param(ldlm_enqueue_min, int, 0644);
+unsigned int ldlm_enqueue_min = OBD_TIMEOUT_DEFAULT;
+module_param(ldlm_enqueue_min, uint, 0644);
 MODULE_PARM_DESC(ldlm_enqueue_min, "lock enqueue timeout minimum");
 
 /* in client side, whether the cached locks will be canceled before replay */
@@ -123,44 +123,56 @@ static int ldlm_expired_completion_wait(void *data)
 	return 0;
 }
 
+/**
+ * Calculate the Completion timeout (covering enqueue, BL AST, data flush,
+ * lock cancel, and their replies). Used for lock completion timeout on the
+ * client side.
+ *
+ * \param[in] lock	lock which is waiting the completion callback
+ *
+ * \retval		timeout in seconds to wait for the server reply
+ */
 /* We use the same basis for both server side and client side functions
  * from a single node.
  */
-static int ldlm_get_enq_timeout(struct ldlm_lock *lock)
+static unsigned int ldlm_cp_timeout(struct ldlm_lock *lock)
 {
-	int timeout = at_get(ldlm_lock_to_ns_at(lock));
+	unsigned int timeout;
 
 	if (AT_OFF)
-		return obd_timeout / 2;
-	/* Since these are non-updating timeouts, we should be conservative.
-	 * It would be nice to have some kind of "early reply" mechanism for
-	 * lock callbacks too...
+		return obd_timeout;
+
+	/*
+	 * Wait a long time for enqueue - server may have to callback a
+	 * lock from another client.  Server will evict the other client if it
+	 * doesn't respond reasonably, and then give us the lock.
 	 */
-	timeout = min_t(int, at_max, timeout + (timeout >> 1)); /* 150% */
-	return max(timeout, ldlm_enqueue_min);
+	timeout = at_get(ldlm_lock_to_ns_at(lock));
+	return max(3 * timeout, ldlm_enqueue_min);
 }
 
 /**
  * Helper function for ldlm_completion_ast(), updating timings when lock is
  * actually granted.
  */
-static int ldlm_completion_tail(struct ldlm_lock *lock)
+static int ldlm_completion_tail(struct ldlm_lock *lock, void *data)
 {
 	long delay;
-	int  result;
+	int result = 0;
 
 	if (ldlm_is_destroyed(lock) || ldlm_is_failed(lock)) {
 		LDLM_DEBUG(lock, "client-side enqueue: destroyed");
 		result = -EIO;
+	} else if (!data) {
+		LDLM_DEBUG(lock, "client-side enqueue: granted");
 	} else {
+		/* Take into AT only CP RPC, not immediately granted locks */
 		delay = ktime_get_real_seconds() - lock->l_last_activity;
 		LDLM_DEBUG(lock, "client-side enqueue: granted after %lds",
 			   delay);
 
 		/* Update our time estimate */
-		at_measured(ldlm_lock_to_ns_at(lock),
-			    delay);
-		result = 0;
+		at_measured(ldlm_lock_to_ns_at(lock), delay);
 	}
 	return result;
 }
@@ -177,10 +189,9 @@ int ldlm_completion_ast_async(struct ldlm_lock *lock, __u64 flags, void *data)
 		return 0;
 	}
 
-	if (!(flags & (LDLM_FL_BLOCK_WAIT | LDLM_FL_BLOCK_GRANTED |
-		       LDLM_FL_BLOCK_CONV))) {
+	if (!(flags & LDLM_FL_BLOCKED_MASK)) {
 		wake_up(&lock->l_waitq);
-		return ldlm_completion_tail(lock);
+		return ldlm_completion_tail(lock, data);
 	}
 
 	LDLM_DEBUG(lock, "client-side enqueue returned a blocked lock, going forward");
@@ -224,8 +235,7 @@ int ldlm_completion_ast(struct ldlm_lock *lock, __u64 flags, void *data)
 		goto noreproc;
 	}
 
-	if (!(flags & (LDLM_FL_BLOCK_WAIT | LDLM_FL_BLOCK_GRANTED |
-		       LDLM_FL_BLOCK_CONV))) {
+	if (!(flags & LDLM_FL_BLOCKED_MASK)) {
 		wake_up(&lock->l_waitq);
 		return 0;
 	}
@@ -240,13 +250,10 @@ noreproc:
 	if (obd)
 		imp = obd->u.cli.cl_import;
 
-	/* Wait a long time for enqueue - server may have to callback a
-	 * lock from another client.  Server will evict the other client if it
-	 * doesn't respond reasonably, and then give us the lock.
-	 */
-	timeout = ldlm_get_enq_timeout(lock) * 2;
+	timeout = ldlm_cp_timeout(lock);
 
 	lwd.lwd_lock = lock;
+	lock->l_last_activity = ktime_get_real_seconds();
 
 	if (ldlm_is_no_timeout(lock)) {
 		LDLM_DEBUG(lock, "waiting indefinitely because of NO_TIMEOUT");
@@ -279,7 +286,7 @@ noreproc:
 		return rc;
 	}
 
-	return ldlm_completion_tail(lock);
+	return ldlm_completion_tail(lock, data);
 }
 EXPORT_SYMBOL(ldlm_completion_ast);
 
@@ -309,8 +316,6 @@ static void failed_lock_cleanup(struct ldlm_namespace *ns,
 	else
 		LDLM_DEBUG(lock, "lock was granted or failed in race");
 
-	ldlm_lock_decref_internal(lock, mode);
-
 	/* XXX - HACK because we shouldn't call ldlm_lock_destroy()
 	 *       from llite/file.c/ll_file_flock().
 	 */
@@ -321,9 +326,14 @@ static void failed_lock_cleanup(struct ldlm_namespace *ns,
 	 */
 	if (lock->l_resource->lr_type == LDLM_FLOCK) {
 		lock_res_and_lock(lock);
-		ldlm_resource_unlink_lock(lock);
-		ldlm_lock_destroy_nolock(lock);
+		if (!ldlm_is_destroyed(lock)) {
+			ldlm_resource_unlink_lock(lock);
+			ldlm_lock_decref_internal_nolock(lock, mode);
+			ldlm_lock_destroy_nolock(lock);
+		}
 		unlock_res_and_lock(lock);
+	} else {
+		ldlm_lock_decref_internal(lock, mode);
 	}
 }
 
@@ -418,11 +428,6 @@ int ldlm_cli_enqueue_fini(struct obd_export *exp, struct ptlrpc_request *req,
 	*flags = ldlm_flags_from_wire(reply->lock_flags);
 	lock->l_flags |= ldlm_flags_from_wire(reply->lock_flags &
 					      LDLM_FL_INHERIT_MASK);
-	/* move NO_TIMEOUT flag to the lock to force ldlm_lock_match()
-	 * to wait with no timeout as well
-	 */
-	lock->l_flags |= ldlm_flags_from_wire(reply->lock_flags &
-					      LDLM_FL_NO_TIMEOUT);
 	unlock_res_and_lock(lock);
 
 	CDEBUG(D_INFO, "local: %p, remote cookie: %#llx, flags: 0x%llx\n",
@@ -556,7 +561,7 @@ static inline int ldlm_capsule_handles_avail(struct req_capsule *pill,
 					     enum req_location loc,
 					     int off)
 {
-	int size = req_capsule_msg_size(pill, loc);
+	u32 size = req_capsule_msg_size(pill, loc);
 
 	return ldlm_req_handles_avail(size, off);
 }
@@ -565,7 +570,7 @@ static inline int ldlm_format_handles_avail(struct obd_import *imp,
 					    const struct req_format *fmt,
 					    enum req_location loc, int off)
 {
-	int size = req_capsule_fmt_size(imp->imp_msg_magic, fmt, loc);
+	u32 size = req_capsule_fmt_size(imp->imp_msg_magic, fmt, loc);
 
 	return ldlm_req_handles_avail(size, off);
 }
@@ -696,8 +701,8 @@ int ldlm_cli_enqueue(struct obd_export *exp, struct ptlrpc_request **reqp,
 		lock = ldlm_lock_create(ns, res_id, einfo->ei_type,
 					einfo->ei_mode, &cbs, einfo->ei_cbdata,
 					lvb_len, lvb_type);
-		if (!lock)
-			return -ENOMEM;
+		if (IS_ERR(lock))
+			return PTR_ERR(lock);
 		/* for the local lock, add the reference */
 		ldlm_lock_addref_internal(lock, einfo->ei_mode);
 		ldlm_lock2handle(lock, lockh);
@@ -719,6 +724,7 @@ int ldlm_cli_enqueue(struct obd_export *exp, struct ptlrpc_request **reqp,
 	lock->l_export = NULL;
 	lock->l_blocking_ast = einfo->ei_cb_bl;
 	lock->l_flags |= (*flags & (LDLM_FL_NO_LRU | LDLM_FL_EXCL));
+	lock->l_last_activity = ktime_get_real_seconds();
 
 	/* lock not sent to server yet */
 
@@ -819,7 +825,7 @@ static __u64 ldlm_cli_cancel_local(struct ldlm_lock *lock)
 		lock_res_and_lock(lock);
 		ldlm_set_cbpending(lock);
 		local_only = !!(lock->l_flags &
-				(LDLM_FL_LOCAL_ONLY|LDLM_FL_CANCEL_ON_BLOCK));
+				(LDLM_FL_LOCAL_ONLY | LDLM_FL_CANCEL_ON_BLOCK));
 		ldlm_cancel_callback(lock);
 		rc = ldlm_is_bl_ast(lock) ? LDLM_FL_BL_AST : LDLM_FL_CANCELING;
 		unlock_res_and_lock(lock);
@@ -1180,8 +1186,7 @@ static enum ldlm_policy_res ldlm_cancel_lrur_policy(struct ldlm_namespace *ns,
 
 	slv = ldlm_pool_get_slv(pl);
 	lvf = ldlm_pool_get_lvf(pl);
-	la = cfs_duration_sec(cfs_time_sub(cur,
-			      lock->l_last_used));
+	la = cfs_duration_sec(cfs_time_sub(cur, lock->l_last_used));
 	lv = lvf * la * unused;
 
 	/* Inform pool about current CLV to see it via debugfs. */
@@ -1193,9 +1198,6 @@ static enum ldlm_policy_res ldlm_cancel_lrur_policy(struct ldlm_namespace *ns,
 	if (slv == 0 || lv < slv)
 		return LDLM_POLICY_KEEP_LOCK;
 
-	if (ns->ns_cancel && ns->ns_cancel(lock) == 0)
-		return LDLM_POLICY_KEEP_LOCK;
-
 	return LDLM_POLICY_CANCEL_LOCK;
 }
 
@@ -1239,9 +1241,6 @@ static enum ldlm_policy_res ldlm_cancel_aged_policy(struct ldlm_namespace *ns,
 			cfs_time_add(lock->l_last_used, ns->ns_max_age)))
 		return LDLM_POLICY_KEEP_LOCK;
 
-	if (ns->ns_cancel && ns->ns_cancel(lock) == 0)
-		return LDLM_POLICY_KEEP_LOCK;
-
 	return LDLM_POLICY_CANCEL_LOCK;
 }
 
@@ -1374,7 +1373,7 @@ static int ldlm_prepare_lru_list(struct ldlm_namespace *ns,
 			break;
 
 		list_for_each_entry_safe(lock, next, &ns->ns_unused_list,
-					     l_lru) {
+					 l_lru) {
 			/* No locks which got blocking requests. */
 			LASSERT(!ldlm_is_bl_ast(lock));
 
@@ -1413,7 +1412,8 @@ static int ldlm_prepare_lru_list(struct ldlm_namespace *ns,
 		 * That is, for shrinker policy we drop only
 		 * old locks, but additionally choose them by
 		 * their weight. Big extent locks will stay in
-		 * the cache. */
+		 * the cache.
+		 */
 		result = pf(ns, lock, unused, added, count);
 		if (result == LDLM_POLICY_KEEP_LOCK) {
 			lu_ref_del(&lock->l_reference,
@@ -1610,8 +1610,7 @@ int ldlm_cli_cancel_list(struct list_head *cancels, int count,
 	 */
 	while (count > 0) {
 		LASSERT(!list_empty(cancels));
-		lock = list_entry(cancels->next, struct ldlm_lock,
-				      l_bl_ast);
+		lock = list_entry(cancels->next, struct ldlm_lock, l_bl_ast);
 		LASSERT(lock->l_conn_export);
 
 		if (exp_connect_cancelset(lock->l_conn_export)) {
@@ -1660,7 +1659,7 @@ int ldlm_cli_cancel_unused_resource(struct ldlm_namespace *ns,
 	int rc;
 
 	res = ldlm_resource_get(ns, NULL, res_id, 0, 0);
-	if (!res) {
+	if (IS_ERR(res)) {
 		/* This is not a problem. */
 		CDEBUG(D_INFO, "No resource %llu\n", res_id->name[0]);
 		return 0;
@@ -1704,7 +1703,8 @@ static int ldlm_cli_hash_cancel_unused(struct cfs_hash *hs,
  * that have 0 readers/writers.
  *
  * If flags & LCF_LOCAL, throw the locks away without trying
- * to notify the server. */
+ * to notify the server.
+ */
 int ldlm_cli_cancel_unused(struct ldlm_namespace *ns,
 			   const struct ldlm_res_id *res_id,
 			   enum ldlm_cancel_flags flags, void *opaque)
@@ -1811,13 +1811,10 @@ int ldlm_resource_iterate(struct ldlm_namespace *ns,
 	struct ldlm_resource *res;
 	int rc;
 
-	if (!ns) {
-		CERROR("must pass in namespace\n");
-		LBUG();
-	}
+	LASSERTF(ns, "must pass in namespace\n");
 
 	res = ldlm_resource_get(ns, NULL, res_id, 0, 0);
-	if (!res)
+	if (IS_ERR(res))
 		return 0;
 
 	LDLM_RESOURCE_ADDREF(res);
@@ -1843,7 +1840,7 @@ static int ldlm_chain_lock_for_replay(struct ldlm_lock *lock, void *closure)
 	 * bug 17614: locks being actively cancelled. Get a reference
 	 * on a lock so that it does not disappear under us (e.g. due to cancel)
 	 */
-	if (!(lock->l_flags & (LDLM_FL_FAILED|LDLM_FL_CANCELING))) {
+	if (!(lock->l_flags & (LDLM_FL_FAILED | LDLM_FL_CANCELING))) {
 		list_add(&lock->l_pending_chain, list);
 		LDLM_LOCK_GET(lock);
 	}
@@ -2013,7 +2010,7 @@ static void ldlm_cancel_unused_locks_for_replay(struct ldlm_namespace *ns)
 					 LCF_LOCAL, LDLM_CANCEL_NO_WAIT);
 
 	CDEBUG(D_DLMTRACE, "Canceled %d unused locks from namespace %s\n",
-			   canceled, ldlm_ns_name(ns));
+	       canceled, ldlm_ns_name(ns));
 }
 
 int ldlm_replay_locks(struct obd_import *imp)
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c b/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
index 51a28d96af39..a09c25aea698 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
@@ -449,8 +449,8 @@ static unsigned ldlm_res_hop_hash(struct cfs_hash *hs,
 				  const void *key, unsigned mask)
 {
 	const struct ldlm_res_id     *id  = key;
-	unsigned		val = 0;
-	unsigned		i;
+	unsigned int		val = 0;
+	unsigned int		i;
 
 	for (i = 0; i < RES_NAME_SIZE; i++)
 		val += id->name[i];
@@ -561,9 +561,9 @@ static struct cfs_hash_ops ldlm_ns_fid_hash_ops = {
 struct ldlm_ns_hash_def {
 	enum ldlm_ns_type nsd_type;
 	/** hash bucket bits */
-	unsigned	nsd_bkt_bits;
+	unsigned int	nsd_bkt_bits;
 	/** hash bits */
-	unsigned	nsd_all_bits;
+	unsigned int	nsd_all_bits;
 	/** hash operations */
 	struct cfs_hash_ops *nsd_hops;
 };
@@ -758,8 +758,7 @@ static void cleanup_resource(struct ldlm_resource *res, struct list_head *q,
 		 */
 		lock_res(res);
 		list_for_each(tmp, q) {
-			lock = list_entry(tmp, struct ldlm_lock,
-					      l_res_link);
+			lock = list_entry(tmp, struct ldlm_lock, l_res_link);
 			if (ldlm_is_cleaned(lock)) {
 				lock = NULL;
 				continue;
@@ -793,8 +792,14 @@ static void cleanup_resource(struct ldlm_resource *res, struct list_head *q,
 			 */
 			unlock_res(res);
 			LDLM_DEBUG(lock, "setting FL_LOCAL_ONLY");
+			if (lock->l_flags & LDLM_FL_FAIL_LOC) {
+				set_current_state(TASK_UNINTERRUPTIBLE);
+				schedule_timeout(cfs_time_seconds(4));
+				set_current_state(TASK_RUNNING);
+			}
 			if (lock->l_completion_ast)
-				lock->l_completion_ast(lock, 0, NULL);
+				lock->l_completion_ast(lock, LDLM_FL_FAILED,
+						       NULL);
 			LDLM_LOCK_RELEASE(lock);
 			continue;
 		}
@@ -875,7 +880,8 @@ static int __ldlm_namespace_free(struct ldlm_namespace *ns, int force)
 		       ldlm_ns_name(ns), atomic_read(&ns->ns_bref));
 force_wait:
 		if (force)
-			lwi = LWI_TIMEOUT(obd_timeout * HZ / 4, NULL, NULL);
+			lwi = LWI_TIMEOUT(msecs_to_jiffies(obd_timeout *
+					  MSEC_PER_SEC) / 4, NULL, NULL);
 
 		rc = l_wait_event(ns->ns_waitq,
 				  atomic_read(&ns->ns_bref) == 0, &lwi);
@@ -1082,10 +1088,11 @@ ldlm_resource_get(struct ldlm_namespace *ns, struct ldlm_resource *parent,
 		  int create)
 {
 	struct hlist_node     *hnode;
-	struct ldlm_resource *res;
+	struct ldlm_resource *res = NULL;
 	struct cfs_hash_bd	 bd;
 	__u64		 version;
 	int		      ns_refcount = 0;
+	int rc;
 
 	LASSERT(!parent);
 	LASSERT(ns->ns_rs_hash);
@@ -1095,31 +1102,20 @@ ldlm_resource_get(struct ldlm_namespace *ns, struct ldlm_resource *parent,
 	hnode = cfs_hash_bd_lookup_locked(ns->ns_rs_hash, &bd, (void *)name);
 	if (hnode) {
 		cfs_hash_bd_unlock(ns->ns_rs_hash, &bd, 0);
-		res = hlist_entry(hnode, struct ldlm_resource, lr_hash);
-		/* Synchronize with regard to resource creation. */
-		if (ns->ns_lvbo && ns->ns_lvbo->lvbo_init) {
-			mutex_lock(&res->lr_lvb_mutex);
-			mutex_unlock(&res->lr_lvb_mutex);
-		}
-
-		if (unlikely(res->lr_lvb_len < 0)) {
-			ldlm_resource_putref(res);
-			res = NULL;
-		}
-		return res;
+		goto lvbo_init;
 	}
 
 	version = cfs_hash_bd_version_get(&bd);
 	cfs_hash_bd_unlock(ns->ns_rs_hash, &bd, 0);
 
 	if (create == 0)
-		return NULL;
+		return ERR_PTR(-ENOENT);
 
 	LASSERTF(type >= LDLM_MIN_TYPE && type < LDLM_MAX_TYPE,
 		 "type: %d\n", type);
 	res = ldlm_resource_new();
 	if (!res)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	res->lr_ns_bucket  = cfs_hash_bd_extra_get(ns->ns_rs_hash, &bd);
 	res->lr_name       = *name;
@@ -1137,7 +1133,7 @@ ldlm_resource_get(struct ldlm_namespace *ns, struct ldlm_resource *parent,
 		/* We have taken lr_lvb_mutex. Drop it. */
 		mutex_unlock(&res->lr_lvb_mutex);
 		kmem_cache_free(ldlm_resource_slab, res);
-
+lvbo_init:
 		res = hlist_entry(hnode, struct ldlm_resource, lr_hash);
 		/* Synchronize with regard to resource creation. */
 		if (ns->ns_lvbo && ns->ns_lvbo->lvbo_init) {
@@ -1146,8 +1142,9 @@ ldlm_resource_get(struct ldlm_namespace *ns, struct ldlm_resource *parent,
 		}
 
 		if (unlikely(res->lr_lvb_len < 0)) {
+			rc = res->lr_lvb_len;
 			ldlm_resource_putref(res);
-			res = NULL;
+			res = ERR_PTR(rc);
 		}
 		return res;
 	}
@@ -1158,8 +1155,6 @@ ldlm_resource_get(struct ldlm_namespace *ns, struct ldlm_resource *parent,
 
 	cfs_hash_bd_unlock(ns->ns_rs_hash, &bd, 1);
 	if (ns->ns_lvbo && ns->ns_lvbo->lvbo_init) {
-		int rc;
-
 		OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_CREATE_RESOURCE, 2);
 		rc = ns->ns_lvbo->lvbo_init(res);
 		if (rc < 0) {
@@ -1169,7 +1164,7 @@ ldlm_resource_get(struct ldlm_namespace *ns, struct ldlm_resource *parent,
 			res->lr_lvb_len = rc;
 			mutex_unlock(&res->lr_lvb_mutex);
 			ldlm_resource_putref(res);
-			return NULL;
+			return ERR_PTR(rc);
 		}
 	}
 
@@ -1386,7 +1381,7 @@ void ldlm_resource_dump(int level, struct ldlm_resource *res)
 	if (!list_empty(&res->lr_granted)) {
 		CDEBUG(level, "Granted locks (in reverse order):\n");
 		list_for_each_entry_reverse(lock, &res->lr_granted,
-						l_res_link) {
+					    l_res_link) {
 			LDLM_DEBUG_LIMIT(level, lock, "###");
 			if (!(level & D_CANTMASK) &&
 			    ++granted > ldlm_dump_granted_max) {
diff --git a/drivers/staging/lustre/lustre/llite/Makefile b/drivers/staging/lustre/lustre/llite/Makefile
index 2cbb1b80bd41..1ac0940bd8df 100644
--- a/drivers/staging/lustre/lustre/llite/Makefile
+++ b/drivers/staging/lustre/lustre/llite/Makefile
@@ -1,6 +1,6 @@
 obj-$(CONFIG_LUSTRE_FS) += lustre.o
 lustre-y := dcache.o dir.o file.o llite_close.o llite_lib.o llite_nfs.o \
-	    rw.o namei.o symlink.o llite_mmap.o \
+	    rw.o namei.o symlink.o llite_mmap.o range_lock.o \
 	    xattr.o xattr_cache.o rw26.o super25.o statahead.o \
 	    glimpse.o lcommon_cl.o lcommon_misc.o \
 	    vvp_dev.o vvp_page.o vvp_lock.o vvp_io.o vvp_object.o vvp_req.o \
diff --git a/drivers/staging/lustre/lustre/llite/dcache.c b/drivers/staging/lustre/lustre/llite/dcache.c
index 463b1a360733..0e45d8fc4d7c 100644
--- a/drivers/staging/lustre/lustre/llite/dcache.c
+++ b/drivers/staging/lustre/lustre/llite/dcache.c
@@ -37,7 +37,6 @@
 #define DEBUG_SUBSYSTEM S_LLITE
 
 #include "../include/obd_support.h"
-#include "../include/lustre_lite.h"
 #include "../include/lustre/lustre_idl.h"
 #include "../include/lustre_dlm.h"
 
@@ -102,39 +101,6 @@ static int ll_dcompare(const struct dentry *dentry,
 	return 0;
 }
 
-static inline int return_if_equal(struct ldlm_lock *lock, void *data)
-{
-	return (ldlm_is_canceling(lock) && ldlm_is_discard_data(lock)) ?
-		LDLM_ITER_CONTINUE : LDLM_ITER_STOP;
-}
-
-/* find any ldlm lock of the inode in mdc and lov
- * return 0    not find
- *	1    find one
- *      < 0    error
- */
-static int find_cbdata(struct inode *inode)
-{
-	struct ll_sb_info *sbi = ll_i2sbi(inode);
-	struct lov_stripe_md *lsm;
-	int rc = 0;
-
-	LASSERT(inode);
-	rc = md_find_cbdata(sbi->ll_md_exp, ll_inode2fid(inode),
-			    return_if_equal, NULL);
-	if (rc != 0)
-		return rc;
-
-	lsm = ccc_inode_lsm_get(inode);
-	if (!lsm)
-		return rc;
-
-	rc = obd_find_cbdata(sbi->ll_dt_exp, lsm, return_if_equal, NULL);
-	ccc_inode_lsm_put(inode, lsm);
-
-	return rc;
-}
-
 /**
  * Called when last reference to a dentry is dropped and dcache wants to know
  * whether or not it should cache it:
@@ -155,19 +121,6 @@ static int ll_ddelete(const struct dentry *de)
 	/* kernel >= 2.6.38 last refcount is decreased after this function. */
 	LASSERT(d_count(de) == 1);
 
-	/* Disable this piece of code temporarily because this is called
-	 * inside dcache_lock so it's not appropriate to do lots of work
-	 * here. ATTENTION: Before this piece of code enabling, LU-2487 must be
-	 * resolved.
-	 */
-#if 0
-	/* if not ldlm lock for this inode, set i_nlink to 0 so that
-	 * this inode can be recycled later b=20433
-	 */
-	if (d_really_is_positive(de) && !find_cbdata(d_inode(de)))
-		clear_nlink(d_inode(de));
-#endif
-
 	if (d_lustre_invalid((struct dentry *)de))
 		return 1;
 	return 0;
@@ -325,14 +278,13 @@ static int ll_revalidate_dentry(struct dentry *dentry,
 	if (lookup_flags & (LOOKUP_PARENT | LOOKUP_OPEN | LOOKUP_CREATE))
 		return 1;
 
-	if (d_need_statahead(dir, dentry) <= 0)
+	if (!dentry_may_statahead(dir, dentry))
 		return 1;
 
 	if (lookup_flags & LOOKUP_RCU)
 		return -ECHILD;
 
-	do_statahead_enter(dir, &dentry, !d_inode(dentry));
-	ll_statahead_mark(dir, dentry);
+	ll_statahead(dir, &dentry, !d_inode(dentry));
 	return 1;
 }
 
@@ -347,18 +299,9 @@ static int ll_revalidate_nd(struct dentry *dentry, unsigned int flags)
 	return ll_revalidate_dentry(dentry, flags);
 }
 
-static void ll_d_iput(struct dentry *de, struct inode *inode)
-{
-	LASSERT(inode);
-	if (!find_cbdata(inode))
-		clear_nlink(inode);
-	iput(inode);
-}
-
 const struct dentry_operations ll_d_ops = {
 	.d_revalidate = ll_revalidate_nd,
 	.d_release = ll_release,
 	.d_delete  = ll_ddelete,
-	.d_iput    = ll_d_iput,
 	.d_compare = ll_dcompare,
 };
diff --git a/drivers/staging/lustre/lustre/llite/dir.c b/drivers/staging/lustre/lustre/llite/dir.c
index 5b381779c827..7f32a539d260 100644
--- a/drivers/staging/lustre/lustre/llite/dir.c
+++ b/drivers/staging/lustre/lustre/llite/dir.c
@@ -46,9 +46,8 @@
 
 #include "../include/obd_support.h"
 #include "../include/obd_class.h"
+#include "../include/lustre/lustre_ioctl.h"
 #include "../include/lustre_lib.h"
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre_lite.h"
 #include "../include/lustre_dlm.h"
 #include "../include/lustre_fid.h"
 #include "../include/lustre_kernelcomm.h"
@@ -134,111 +133,35 @@
  * for this integrated page will be adjusted. See lmv_adjust_dirpages().
  *
  */
-
-/* returns the page unlocked, but with a reference */
-static int ll_dir_filler(void *_hash, struct page *page0)
+struct page *ll_get_dir_page(struct inode *dir, struct md_op_data *op_data,
+			     __u64 offset)
 {
-	struct inode *inode = page0->mapping->host;
-	int hash64 = ll_i2sbi(inode)->ll_flags & LL_SBI_64BIT_HASH;
-	struct obd_export *exp = ll_i2sbi(inode)->ll_md_exp;
-	struct ptlrpc_request *request;
-	struct mdt_body *body;
-	struct md_op_data *op_data;
-	__u64 hash = *((__u64 *)_hash);
-	struct page **page_pool;
+	struct md_callback cb_op;
 	struct page *page;
-	struct lu_dirpage *dp;
-	int max_pages = ll_i2sbi(inode)->ll_md_brw_size >> PAGE_SHIFT;
-	int nrdpgs = 0; /* number of pages read actually */
-	int npages;
-	int i;
 	int rc;
 
-	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p) hash %llu\n",
-	       PFID(ll_inode2fid(inode)), inode, hash);
-
-	LASSERT(max_pages > 0 && max_pages <= MD_MAX_BRW_PAGES);
-
-	op_data = ll_prep_md_op_data(NULL, inode, NULL, NULL, 0, 0,
-				     LUSTRE_OPC_ANY, NULL);
-	if (IS_ERR(op_data))
-		return PTR_ERR(op_data);
-
-	page_pool = kcalloc(max_pages, sizeof(page), GFP_NOFS);
-	if (page_pool) {
-		page_pool[0] = page0;
-	} else {
-		page_pool = &page0;
-		max_pages = 1;
-	}
-	for (npages = 1; npages < max_pages; npages++) {
-		page = page_cache_alloc_cold(inode->i_mapping);
-		if (!page)
-			break;
-		page_pool[npages] = page;
-	}
-
-	op_data->op_npages = npages;
-	op_data->op_offset = hash;
-	rc = md_readpage(exp, op_data, page_pool, &request);
-	ll_finish_md_op_data(op_data);
-	if (rc < 0) {
-		/* page0 is special, which was added into page cache early */
-		delete_from_page_cache(page0);
-	} else if (rc == 0) {
-		body = req_capsule_server_get(&request->rq_pill, &RMF_MDT_BODY);
-		/* Checked by mdc_readpage() */
-		if (body->valid & OBD_MD_FLSIZE)
-			i_size_write(inode, body->size);
-
-		nrdpgs = (request->rq_bulk->bd_nob_transferred+PAGE_SIZE-1)
-			 >> PAGE_SHIFT;
-		SetPageUptodate(page0);
-	}
-	unlock_page(page0);
-	ptlrpc_req_finished(request);
-
-	CDEBUG(D_VFSTRACE, "read %d/%d pages\n", nrdpgs, npages);
-
-	for (i = 1; i < npages; i++) {
-		unsigned long offset;
-		int ret;
-
-		page = page_pool[i];
-
-		if (rc < 0 || i >= nrdpgs) {
-			put_page(page);
-			continue;
-		}
-
-		SetPageUptodate(page);
-
-		dp = kmap(page);
-		hash = le64_to_cpu(dp->ldp_hash_start);
-		kunmap(page);
-
-		offset = hash_x_index(hash, hash64);
-
-		prefetchw(&page->flags);
-		ret = add_to_page_cache_lru(page, inode->i_mapping, offset,
-					    GFP_NOFS);
-		if (ret == 0) {
-			unlock_page(page);
-		} else {
-			CDEBUG(D_VFSTRACE, "page %lu add to page cache failed: %d\n",
-			       offset, ret);
-		}
-		put_page(page);
-	}
+	cb_op.md_blocking_ast = ll_md_blocking_ast;
+	rc = md_read_page(ll_i2mdexp(dir), op_data, &cb_op, offset, &page);
+	if (rc)
+		return ERR_PTR(rc);
 
-	if (page_pool != &page0)
-		kfree(page_pool);
-	return rc;
+	return page;
 }
 
-void ll_release_page(struct page *page, int remove)
+void ll_release_page(struct inode *inode, struct page *page, bool remove)
 {
 	kunmap(page);
+
+	/*
+	 * Always remove the page for striped dir, because the page is
+	 * built from temporarily in LMV layer
+	 */
+	if (inode && S_ISDIR(inode->i_mode) &&
+	    ll_i2info(inode)->lli_lsm_md) {
+		__free_page(page);
+		return;
+	}
+
 	if (remove) {
 		lock_page(page);
 		if (likely(page->mapping))
@@ -248,225 +171,6 @@ void ll_release_page(struct page *page, int remove)
 	put_page(page);
 }
 
-/*
- * Find, kmap and return page that contains given hash.
- */
-static struct page *ll_dir_page_locate(struct inode *dir, __u64 *hash,
-				       __u64 *start, __u64 *end)
-{
-	int hash64 = ll_i2sbi(dir)->ll_flags & LL_SBI_64BIT_HASH;
-	struct address_space *mapping = dir->i_mapping;
-	/*
-	 * Complement of hash is used as an index so that
-	 * radix_tree_gang_lookup() can be used to find a page with starting
-	 * hash _smaller_ than one we are looking for.
-	 */
-	unsigned long offset = hash_x_index(*hash, hash64);
-	struct page *page;
-	int found;
-
-	spin_lock_irq(&mapping->tree_lock);
-	found = radix_tree_gang_lookup(&mapping->page_tree,
-				       (void **)&page, offset, 1);
-	if (found > 0 && !radix_tree_exceptional_entry(page)) {
-		struct lu_dirpage *dp;
-
-		get_page(page);
-		spin_unlock_irq(&mapping->tree_lock);
-		/*
-		 * In contrast to find_lock_page() we are sure that directory
-		 * page cannot be truncated (while DLM lock is held) and,
-		 * hence, can avoid restart.
-		 *
-		 * In fact, page cannot be locked here at all, because
-		 * ll_dir_filler() does synchronous io.
-		 */
-		wait_on_page_locked(page);
-		if (PageUptodate(page)) {
-			dp = kmap(page);
-			if (BITS_PER_LONG == 32 && hash64) {
-				*start = le64_to_cpu(dp->ldp_hash_start) >> 32;
-				*end   = le64_to_cpu(dp->ldp_hash_end) >> 32;
-				*hash  = *hash >> 32;
-			} else {
-				*start = le64_to_cpu(dp->ldp_hash_start);
-				*end   = le64_to_cpu(dp->ldp_hash_end);
-			}
-			LASSERTF(*start <= *hash, "start = %#llx,end = %#llx,hash = %#llx\n",
-				 *start, *end, *hash);
-			CDEBUG(D_VFSTRACE, "page %lu [%llu %llu], hash %llu\n",
-			       offset, *start, *end, *hash);
-			if (*hash > *end) {
-				ll_release_page(page, 0);
-				page = NULL;
-			} else if (*end != *start && *hash == *end) {
-				/*
-				 * upon hash collision, remove this page,
-				 * otherwise put page reference, and
-				 * ll_get_dir_page() will issue RPC to fetch
-				 * the page we want.
-				 */
-				ll_release_page(page,
-				    le32_to_cpu(dp->ldp_flags) & LDF_COLLIDE);
-				page = NULL;
-			}
-		} else {
-			put_page(page);
-			page = ERR_PTR(-EIO);
-		}
-
-	} else {
-		spin_unlock_irq(&mapping->tree_lock);
-		page = NULL;
-	}
-	return page;
-}
-
-struct page *ll_get_dir_page(struct inode *dir, __u64 hash,
-			     struct ll_dir_chain *chain)
-{
-	ldlm_policy_data_t policy = {.l_inodebits = {MDS_INODELOCK_UPDATE} };
-	struct address_space *mapping = dir->i_mapping;
-	struct lustre_handle lockh;
-	struct lu_dirpage *dp;
-	struct page *page;
-	enum ldlm_mode mode;
-	int rc;
-	__u64 start = 0;
-	__u64 end = 0;
-	__u64 lhash = hash;
-	struct ll_inode_info *lli = ll_i2info(dir);
-	int hash64 = ll_i2sbi(dir)->ll_flags & LL_SBI_64BIT_HASH;
-
-	mode = LCK_PR;
-	rc = md_lock_match(ll_i2sbi(dir)->ll_md_exp, LDLM_FL_BLOCK_GRANTED,
-			   ll_inode2fid(dir), LDLM_IBITS, &policy, mode, &lockh);
-	if (!rc) {
-		struct ldlm_enqueue_info einfo = {
-			.ei_type = LDLM_IBITS,
-			.ei_mode = mode,
-			.ei_cb_bl = ll_md_blocking_ast,
-			.ei_cb_cp = ldlm_completion_ast,
-		};
-		struct lookup_intent it = { .it_op = IT_READDIR };
-		struct ptlrpc_request *request;
-		struct md_op_data *op_data;
-
-		op_data = ll_prep_md_op_data(NULL, dir, dir, NULL, 0, 0,
-					     LUSTRE_OPC_ANY, NULL);
-		if (IS_ERR(op_data))
-			return (void *)op_data;
-
-		rc = md_enqueue(ll_i2sbi(dir)->ll_md_exp, &einfo, &it,
-				op_data, &lockh, NULL, 0, NULL, 0);
-
-		ll_finish_md_op_data(op_data);
-
-		request = (struct ptlrpc_request *)it.it_request;
-		if (request)
-			ptlrpc_req_finished(request);
-		if (rc < 0) {
-			CERROR("lock enqueue: " DFID " at %llu: rc %d\n",
-			       PFID(ll_inode2fid(dir)), hash, rc);
-			return ERR_PTR(rc);
-		}
-
-		CDEBUG(D_INODE, "setting lr_lvb_inode to inode "DFID"(%p)\n",
-		       PFID(ll_inode2fid(dir)), dir);
-		md_set_lock_data(ll_i2sbi(dir)->ll_md_exp,
-				 &it.it_lock_handle, dir, NULL);
-	} else {
-		/* for cross-ref object, l_ast_data of the lock may not be set,
-		 * we reset it here
-		 */
-		md_set_lock_data(ll_i2sbi(dir)->ll_md_exp, &lockh.cookie,
-				 dir, NULL);
-	}
-	ldlm_lock_dump_handle(D_OTHER, &lockh);
-
-	mutex_lock(&lli->lli_readdir_mutex);
-	page = ll_dir_page_locate(dir, &lhash, &start, &end);
-	if (IS_ERR(page)) {
-		CERROR("dir page locate: "DFID" at %llu: rc %ld\n",
-		       PFID(ll_inode2fid(dir)), lhash, PTR_ERR(page));
-		goto out_unlock;
-	} else if (page) {
-		/*
-		 * XXX nikita: not entirely correct handling of a corner case:
-		 * suppose hash chain of entries with hash value HASH crosses
-		 * border between pages P0 and P1. First both P0 and P1 are
-		 * cached, seekdir() is called for some entry from the P0 part
-		 * of the chain. Later P0 goes out of cache. telldir(HASH)
-		 * happens and finds P1, as it starts with matching hash
-		 * value. Remaining entries from P0 part of the chain are
-		 * skipped. (Is that really a bug?)
-		 *
-		 * Possible solutions: 0. don't cache P1 is such case, handle
-		 * it as an "overflow" page. 1. invalidate all pages at
-		 * once. 2. use HASH|1 as an index for P1.
-		 */
-		goto hash_collision;
-	}
-
-	page = read_cache_page(mapping, hash_x_index(hash, hash64),
-			       ll_dir_filler, &lhash);
-	if (IS_ERR(page)) {
-		CERROR("read cache page: "DFID" at %llu: rc %ld\n",
-		       PFID(ll_inode2fid(dir)), hash, PTR_ERR(page));
-		goto out_unlock;
-	}
-
-	wait_on_page_locked(page);
-	(void)kmap(page);
-	if (!PageUptodate(page)) {
-		CERROR("page not updated: "DFID" at %llu: rc %d\n",
-		       PFID(ll_inode2fid(dir)), hash, -5);
-		goto fail;
-	}
-	if (!PageChecked(page))
-		/* XXX: check page format later */
-		SetPageChecked(page);
-	if (PageError(page)) {
-		CERROR("page error: "DFID" at %llu: rc %d\n",
-		       PFID(ll_inode2fid(dir)), hash, -5);
-		goto fail;
-	}
-hash_collision:
-	dp = page_address(page);
-	if (BITS_PER_LONG == 32 && hash64) {
-		start = le64_to_cpu(dp->ldp_hash_start) >> 32;
-		end   = le64_to_cpu(dp->ldp_hash_end) >> 32;
-		lhash = hash >> 32;
-	} else {
-		start = le64_to_cpu(dp->ldp_hash_start);
-		end   = le64_to_cpu(dp->ldp_hash_end);
-		lhash = hash;
-	}
-	if (end == start) {
-		LASSERT(start == lhash);
-		CWARN("Page-wide hash collision: %llu\n", end);
-		if (BITS_PER_LONG == 32 && hash64)
-			CWARN("Real page-wide hash collision at [%llu %llu] with hash %llu\n",
-			      le64_to_cpu(dp->ldp_hash_start),
-			      le64_to_cpu(dp->ldp_hash_end), hash);
-		/*
-		 * Fetch whole overflow chain...
-		 *
-		 * XXX not yet.
-		 */
-		goto fail;
-	}
-out_unlock:
-	mutex_unlock(&lli->lli_readdir_mutex);
-	ldlm_lock_decref(&lockh, mode);
-	return page;
-
-fail:
-	ll_release_page(page, 1);
-	page = ERR_PTR(-EIO);
-	goto out_unlock;
-}
-
 /**
  * return IF_* type for given lu_dirent entry.
  * IF_* flag shld be converted to particular OS file type in
@@ -489,119 +193,100 @@ static __u16 ll_dirent_type_get(struct lu_dirent *ent)
 	return type;
 }
 
-int ll_dir_read(struct inode *inode, struct dir_context *ctx)
+int ll_dir_read(struct inode *inode, __u64 *ppos, struct md_op_data *op_data,
+		struct dir_context *ctx)
 {
-	struct ll_inode_info *info       = ll_i2info(inode);
 	struct ll_sb_info    *sbi	= ll_i2sbi(inode);
-	__u64		   pos		= ctx->pos;
-	int		   api32      = ll_need_32bit_api(sbi);
-	int		   hash64     = sbi->ll_flags & LL_SBI_64BIT_HASH;
+	__u64		   pos		= *ppos;
+	int		   is_api32 = ll_need_32bit_api(sbi);
+	int		   is_hash64 = sbi->ll_flags & LL_SBI_64BIT_HASH;
 	struct page	  *page;
-	struct ll_dir_chain   chain;
-	int		   done = 0;
+	bool		   done = false;
 	int		   rc = 0;
 
-	ll_dir_chain_init(&chain);
-
-	page = ll_get_dir_page(inode, pos, &chain);
+	page = ll_get_dir_page(inode, op_data, pos);
 
 	while (rc == 0 && !done) {
 		struct lu_dirpage *dp;
 		struct lu_dirent  *ent;
+		__u64 hash;
+		__u64 next;
 
-		if (!IS_ERR(page)) {
-			/*
-			 * If page is empty (end of directory is reached),
-			 * use this value.
-			 */
-			__u64 hash = MDS_DIR_END_OFF;
-			__u64 next;
-
-			dp = page_address(page);
-			for (ent = lu_dirent_start(dp); ent && !done;
-			     ent = lu_dirent_next(ent)) {
-				__u16	  type;
-				int	    namelen;
-				struct lu_fid  fid;
-				__u64	  lhash;
-				__u64	  ino;
+		if (IS_ERR(page)) {
+			rc = PTR_ERR(page);
+			break;
+		}
 
+		hash = MDS_DIR_END_OFF;
+		dp = page_address(page);
+		for (ent = lu_dirent_start(dp); ent && !done;
+		     ent = lu_dirent_next(ent)) {
+			__u16	  type;
+			int	    namelen;
+			struct lu_fid  fid;
+			__u64	  lhash;
+			__u64	  ino;
+
+			hash = le64_to_cpu(ent->lde_hash);
+			if (hash < pos)
 				/*
-				 * XXX: implement correct swabbing here.
+				 * Skip until we find target hash
+				 * value.
 				 */
+				continue;
 
-				hash = le64_to_cpu(ent->lde_hash);
-				if (hash < pos)
-					/*
-					 * Skip until we find target hash
-					 * value.
-					 */
-					continue;
-
-				namelen = le16_to_cpu(ent->lde_namelen);
-				if (namelen == 0)
-					/*
-					 * Skip dummy record.
-					 */
-					continue;
-
-				if (api32 && hash64)
-					lhash = hash >> 32;
-				else
-					lhash = hash;
-				fid_le_to_cpu(&fid, &ent->lde_fid);
-				ino = cl_fid_build_ino(&fid, api32);
-				type = ll_dirent_type_get(ent);
-				ctx->pos = lhash;
-				/* For 'll_nfs_get_name_filldir()', it will try
-				 * to access the 'ent' through its 'lde_name',
-				 * so the parameter 'name' for 'ctx->actor()'
-				 * must be part of the 'ent'.
+			namelen = le16_to_cpu(ent->lde_namelen);
+			if (namelen == 0)
+				/*
+				 * Skip dummy record.
 				 */
-				done = !dir_emit(ctx, ent->lde_name,
-						 namelen, ino, type);
-			}
-			next = le64_to_cpu(dp->ldp_hash_end);
-			if (!done) {
-				pos = next;
-				if (pos == MDS_DIR_END_OFF) {
-					/*
-					 * End of directory reached.
-					 */
-					done = 1;
-					ll_release_page(page, 0);
-				} else if (1 /* chain is exhausted*/) {
-					/*
-					 * Normal case: continue to the next
-					 * page.
-					 */
-					ll_release_page(page,
-					    le32_to_cpu(dp->ldp_flags) &
-							LDF_COLLIDE);
-					next = pos;
-					page = ll_get_dir_page(inode, pos,
-							       &chain);
-				} else {
-					/*
-					 * go into overflow page.
-					 */
-					LASSERT(le32_to_cpu(dp->ldp_flags) &
-						LDF_COLLIDE);
-					ll_release_page(page, 1);
-				}
-			} else {
-				pos = hash;
-				ll_release_page(page, 0);
-			}
+				continue;
+
+			if (is_api32 && is_hash64)
+				lhash = hash >> 32;
+			else
+				lhash = hash;
+			fid_le_to_cpu(&fid, &ent->lde_fid);
+			ino = cl_fid_build_ino(&fid, is_api32);
+			type = ll_dirent_type_get(ent);
+			ctx->pos = lhash;
+			/* For 'll_nfs_get_name_filldir()', it will try
+			 * to access the 'ent' through its 'lde_name',
+			 * so the parameter 'name' for 'ctx->actor()'
+			 * must be part of the 'ent'.
+			 */
+			done = !dir_emit(ctx, ent->lde_name,
+					 namelen, ino, type);
+		}
+
+		if (done) {
+			pos = hash;
+			ll_release_page(inode, page, false);
+			break;
+		}
+
+		next = le64_to_cpu(dp->ldp_hash_end);
+		pos = next;
+		if (pos == MDS_DIR_END_OFF) {
+			/*
+			 * End of directory reached.
+			 */
+			done = 1;
+			ll_release_page(inode, page, false);
 		} else {
-			rc = PTR_ERR(page);
-			CERROR("error reading dir "DFID" at %lu: rc %d\n",
-			       PFID(&info->lli_fid), (unsigned long)pos, rc);
+			/*
+			 * Normal case: continue to the next
+			 * page.
+			 */
+			ll_release_page(inode, page,
+					le32_to_cpu(dp->ldp_flags) &
+					LDF_COLLIDE);
+			next = pos;
+			page = ll_get_dir_page(inode, op_data, pos);
 		}
 	}
 
 	ctx->pos = pos;
-	ll_dir_chain_fini(&chain);
 	return rc;
 }
 
@@ -613,9 +298,10 @@ static int ll_readdir(struct file *filp, struct dir_context *ctx)
 	__u64 pos = lfd ? lfd->lfd_pos : 0;
 	int			hash64	= sbi->ll_flags & LL_SBI_64BIT_HASH;
 	int			api32	= ll_need_32bit_api(sbi);
+	struct md_op_data *op_data;
 	int			rc;
 
-	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p) pos %lu/%llu 32bit_api %d\n",
+	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p) pos/size %lu/%llu 32bit_api %d\n",
 	       PFID(ll_inode2fid(inode)), inode, (unsigned long)pos,
 	       i_size_read(inode), api32);
 
@@ -627,19 +313,58 @@ static int ll_readdir(struct file *filp, struct dir_context *ctx)
 		goto out;
 	}
 
+	op_data = ll_prep_md_op_data(NULL, inode, inode, NULL, 0, 0,
+				     LUSTRE_OPC_ANY, inode);
+	if (IS_ERR(op_data)) {
+		rc = PTR_ERR(op_data);
+		goto out;
+	}
+
+	if (unlikely(op_data->op_mea1)) {
+		/*
+		 * This is only needed for striped dir to fill ..,
+		 * see lmv_read_page
+		 */
+		if (file_dentry(filp)->d_parent &&
+		    file_dentry(filp)->d_parent->d_inode) {
+			__u64 ibits = MDS_INODELOCK_UPDATE;
+			struct inode *parent;
+
+			parent = file_dentry(filp)->d_parent->d_inode;
+			if (ll_have_md_lock(parent, &ibits, LCK_MINMODE))
+				op_data->op_fid3 = *ll_inode2fid(parent);
+		}
+
+		/*
+		 * If it can not find in cache, do lookup .. on the master
+		 * object
+		 */
+		if (fid_is_zero(&op_data->op_fid3)) {
+			rc = ll_dir_get_parent_fid(inode, &op_data->op_fid3);
+			if (rc) {
+				ll_finish_md_op_data(op_data);
+				return rc;
+			}
+		}
+	}
+	op_data->op_max_pages = sbi->ll_md_brw_pages;
 	ctx->pos = pos;
-	rc = ll_dir_read(inode, ctx);
+	rc = ll_dir_read(inode, &pos, op_data, ctx);
+	pos = ctx->pos;
 	if (lfd)
-		lfd->lfd_pos = ctx->pos;
-	if (ctx->pos == MDS_DIR_END_OFF) {
+		lfd->lfd_pos = pos;
+
+	if (pos == MDS_DIR_END_OFF) {
 		if (api32)
-			ctx->pos = LL_DIR_END_OFF_32BIT;
+			pos = LL_DIR_END_OFF_32BIT;
 		else
-			ctx->pos = LL_DIR_END_OFF;
+			pos = LL_DIR_END_OFF;
 	} else {
 		if (api32 && hash64)
-			ctx->pos >>= 32;
+			pos >>= 32;
 	}
+	ctx->pos = pos;
+	ll_finish_md_op_data(op_data);
 	filp->f_version = inode->i_version;
 
 out:
@@ -668,18 +393,40 @@ static int ll_send_mgc_param(struct obd_export *mgc, char *string)
 	return rc;
 }
 
-static int ll_dir_setdirstripe(struct inode *dir, struct lmv_user_md *lump,
-			       char *filename)
+/**
+ * Create striped directory with specified stripe(@lump)
+ *
+ * param[in] parent	the parent of the directory.
+ * param[in] lump	the specified stripes.
+ * param[in] dirname	the name of the directory.
+ * param[in] mode	the specified mode of the directory.
+ *
+ * retval		=0 if striped directory is being created successfully.
+ *			<0 if the creation is failed.
+ */
+static int ll_dir_setdirstripe(struct inode *parent, struct lmv_user_md *lump,
+			       const char *dirname, umode_t mode)
 {
 	struct ptlrpc_request *request = NULL;
 	struct md_op_data *op_data;
-	struct ll_sb_info *sbi = ll_i2sbi(dir);
-	int mode;
+	struct ll_sb_info *sbi = ll_i2sbi(parent);
 	int err;
 
-	mode = (~current_umask() & 0755) | S_IFDIR;
-	op_data = ll_prep_md_op_data(NULL, dir, NULL, filename,
-				     strlen(filename), mode, LUSTRE_OPC_MKDIR,
+	if (unlikely(lump->lum_magic != LMV_USER_MAGIC))
+		return -EINVAL;
+
+	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p) name %s stripe_offset %d, stripe_count: %u\n",
+	       PFID(ll_inode2fid(parent)), parent, dirname,
+	       (int)lump->lum_stripe_offset, lump->lum_stripe_count);
+
+	if (lump->lum_magic != cpu_to_le32(LMV_USER_MAGIC))
+		lustre_swab_lmv_user_md(lump);
+
+	if (!IS_POSIXACL(parent) || !exp_connect_umask(ll_i2mdexp(parent)))
+		mode &= ~current_umask();
+	mode = (mode & (S_IRWXUGO | S_ISVTX)) | S_IFDIR;
+	op_data = ll_prep_md_op_data(NULL, parent, NULL, dirname,
+				     strlen(dirname), mode, LUSTRE_OPC_MKDIR,
 				     lump);
 	if (IS_ERR(op_data)) {
 		err = PTR_ERR(op_data);
@@ -730,6 +477,13 @@ int ll_dir_setstripe(struct inode *inode, struct lov_user_md *lump,
 			lum_size = sizeof(struct lov_user_md_v3);
 			break;
 		}
+		case LMV_USER_MAGIC: {
+			if (lump->lmm_magic != cpu_to_le32(LMV_USER_MAGIC))
+				lustre_swab_lmv_user_md(
+					(struct lmv_user_md *)lump);
+			lum_size = sizeof(struct lmv_user_md);
+			break;
+		}
 		default: {
 			CDEBUG(D_IOCTL, "bad userland LOV MAGIC: %#08x != %#08x nor %#08x\n",
 			       lump->lmm_magic, LOV_USER_MAGIC_V1,
@@ -746,9 +500,6 @@ int ll_dir_setstripe(struct inode *inode, struct lov_user_md *lump,
 	if (IS_ERR(op_data))
 		return PTR_ERR(op_data);
 
-	if (lump && lump->lmm_magic == cpu_to_le32(LMV_USER_MAGIC))
-		op_data->op_cli_flags |= CLI_SET_MEA;
-
 	/* swabbing is done in lov_setstripe() on server side */
 	rc = md_setattr(sbi->ll_md_exp, op_data, lump, lum_size,
 			NULL, 0, &req, NULL);
@@ -803,8 +554,16 @@ end:
 	return rc;
 }
 
-int ll_dir_getstripe(struct inode *inode, struct lov_mds_md **lmmp,
-		     int *lmm_size, struct ptlrpc_request **request)
+/**
+ * This function will be used to get default LOV/LMV/Default LMV
+ * @valid will be used to indicate which stripe it will retrieve
+ *	OBD_MD_MEA		LMV stripe EA
+ *	OBD_MD_DEFAULT_MEA	Default LMV stripe EA
+ *	otherwise		Default LOV EA.
+ * Each time, it can only retrieve 1 stripe EA
+ **/
+int ll_dir_getstripe(struct inode *inode, void **plmm, int *plmm_size,
+		     struct ptlrpc_request **request, u64 valid)
 {
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 	struct mdt_body   *body;
@@ -813,7 +572,7 @@ int ll_dir_getstripe(struct inode *inode, struct lov_mds_md **lmmp,
 	int rc, lmmsize;
 	struct md_op_data *op_data;
 
-	rc = ll_get_default_mdsize(sbi, &lmmsize);
+	rc = ll_get_max_mdsize(sbi, &lmmsize);
 	if (rc)
 		return rc;
 
@@ -834,9 +593,9 @@ int ll_dir_getstripe(struct inode *inode, struct lov_mds_md **lmmp,
 
 	body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
 
-	lmmsize = body->eadatasize;
+	lmmsize = body->mbo_eadatasize;
 
-	if (!(body->valid & (OBD_MD_FLEASIZE | OBD_MD_FLDIREA)) ||
+	if (!(body->mbo_valid & (OBD_MD_FLEASIZE | OBD_MD_FLDIREA)) ||
 	    lmmsize == 0) {
 		rc = -ENODATA;
 		goto out;
@@ -844,6 +603,7 @@ int ll_dir_getstripe(struct inode *inode, struct lov_mds_md **lmmp,
 
 	lmm = req_capsule_server_sized_get(&req->rq_pill,
 					   &RMF_MDT_MD, lmmsize);
+	LASSERT(lmm);
 
 	/*
 	 * This is coming from the MDS, so is probably in
@@ -860,40 +620,51 @@ int ll_dir_getstripe(struct inode *inode, struct lov_mds_md **lmmp,
 		if (cpu_to_le32(LOV_MAGIC) != LOV_MAGIC)
 			lustre_swab_lov_user_md_v3((struct lov_user_md_v3 *)lmm);
 		break;
+	case LMV_MAGIC_V1:
+		if (cpu_to_le32(LMV_MAGIC) != LMV_MAGIC)
+			lustre_swab_lmv_mds_md((union lmv_mds_md *)lmm);
+		break;
+	case LMV_USER_MAGIC:
+		if (cpu_to_le32(LMV_USER_MAGIC) != LMV_USER_MAGIC)
+			lustre_swab_lmv_user_md((struct lmv_user_md *)lmm);
+		break;
 	default:
 		CERROR("unknown magic: %lX\n", (unsigned long)lmm->lmm_magic);
 		rc = -EPROTO;
 	}
 out:
-	*lmmp = lmm;
-	*lmm_size = lmmsize;
+	*plmm = lmm;
+	*plmm_size = lmmsize;
 	*request = req;
 	return rc;
 }
 
-/*
- *  Get MDT index for the inode.
- */
-int ll_get_mdt_idx(struct inode *inode)
+int ll_get_mdt_idx_by_fid(struct ll_sb_info *sbi, const struct lu_fid *fid)
 {
-	struct ll_sb_info *sbi = ll_i2sbi(inode);
 	struct md_op_data *op_data;
-	int rc, mdtidx;
+	int mdt_index, rc;
 
-	op_data = ll_prep_md_op_data(NULL, inode, NULL, NULL, 0,
-				     0, LUSTRE_OPC_ANY, NULL);
-	if (IS_ERR(op_data))
-		return PTR_ERR(op_data);
+	op_data = kzalloc(sizeof(*op_data), GFP_NOFS);
+	if (!op_data)
+		return -ENOMEM;
 
 	op_data->op_flags |= MF_GET_MDT_IDX;
+	op_data->op_fid1 = *fid;
 	rc = md_getattr(sbi->ll_md_exp, op_data, NULL);
-	mdtidx = op_data->op_mds;
-	ll_finish_md_op_data(op_data);
-	if (rc < 0) {
-		CDEBUG(D_INFO, "md_getattr_name: %d\n", rc);
+	mdt_index = op_data->op_mds;
+	kvfree(op_data);
+	if (rc < 0)
 		return rc;
-	}
-	return mdtidx;
+
+	return mdt_index;
+}
+
+/*
+ *  Get MDT index for the inode.
+ */
+int ll_get_mdt_idx(struct inode *inode)
+{
+	return ll_get_mdt_idx_by_fid(ll_i2sbi(inode), ll_inode2fid(inode));
 }
 
 /**
@@ -1288,11 +1059,9 @@ static long ll_dir_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		return 0;
 	}
 	case IOC_MDC_LOOKUP: {
-		struct ptlrpc_request *request = NULL;
 		int namelen, len = 0;
 		char *buf = NULL;
 		char *filename;
-		struct md_op_data *op_data;
 
 		rc = obd_ioctl_getdata(&buf, &len, (void __user *)arg);
 		if (rc)
@@ -1308,21 +1077,13 @@ static long ll_dir_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			goto out_free;
 		}
 
-		op_data = ll_prep_md_op_data(NULL, inode, NULL, filename, namelen,
-					     0, LUSTRE_OPC_ANY, NULL);
-		if (IS_ERR(op_data)) {
-			rc = PTR_ERR(op_data);
-			goto out_free;
-		}
-
-		op_data->op_valid = OBD_MD_FLID;
-		rc = md_getattr_name(sbi->ll_md_exp, op_data, &request);
-		ll_finish_md_op_data(op_data);
+		rc = ll_get_fid_by_name(inode, filename, namelen, NULL);
 		if (rc < 0) {
-			CDEBUG(D_INFO, "md_getattr_name: %d\n", rc);
+			CERROR("%s: lookup %.*s failed: rc = %d\n",
+			       ll_get_fsname(inode->i_sb, NULL, 0), namelen,
+			       filename, rc);
 			goto out_free;
 		}
-		ptlrpc_req_finished(request);
 out_free:
 		obd_ioctl_freedata(buf, len);
 		return rc;
@@ -1333,6 +1094,7 @@ out_free:
 		char		*filename;
 		int		 namelen = 0;
 		int		 lumlen = 0;
+		umode_t mode;
 		int		 len;
 		int		 rc;
 
@@ -1366,15 +1128,32 @@ out_free:
 			goto lmv_out_free;
 		}
 
-		/**
-		 * ll_dir_setdirstripe will be used to set dir stripe
-		 *  mdc_create--->mdt_reint_create (with dirstripe)
-		 */
-		rc = ll_dir_setdirstripe(inode, lum, filename);
+#if OBD_OCD_VERSION(2, 9, 50, 0) > LUSTRE_VERSION_CODE
+		mode = data->ioc_type != 0 ? data->ioc_type : S_IRWXUGO;
+#else
+		mode = data->ioc_type;
+#endif
+		rc = ll_dir_setdirstripe(inode, lum, filename, mode);
 lmv_out_free:
 		obd_ioctl_freedata(buf, len);
 		return rc;
 	}
+	case LL_IOC_LMV_SET_DEFAULT_STRIPE: {
+		struct lmv_user_md __user *ulump;
+		struct lmv_user_md lum;
+		int rc;
+
+		ulump = (struct lmv_user_md __user *)arg;
+		if (copy_from_user(&lum, ulump, sizeof(lum)))
+			return -EFAULT;
+
+		if (lum.lum_magic != LMV_USER_MAGIC)
+			return -EINVAL;
+
+		rc = ll_dir_setstripe(inode, (struct lov_user_md *)&lum, 0);
+
+		return rc;
+	}
 	case LL_IOC_LOV_SETSTRIPE: {
 		struct lov_user_md_v3 lumv3;
 		struct lov_user_md_v1 *lumv1 = (struct lov_user_md_v1 *)&lumv3;
@@ -1404,50 +1183,100 @@ lmv_out_free:
 		return rc;
 	}
 	case LL_IOC_LMV_GETSTRIPE: {
-		struct lmv_user_md __user *lump = (void __user *)arg;
+		struct lmv_user_md __user *ulmv;
 		struct lmv_user_md lum;
-		struct lmv_user_md *tmp;
+		struct ptlrpc_request *request = NULL;
+		struct lmv_user_md *tmp = NULL;
+		union lmv_mds_md *lmm = NULL;
+		u64 valid = 0;
+		int stripe_count;
+		int mdt_index;
 		int lum_size;
-		int rc = 0;
-		int mdtindex;
+		int lmmsize;
+		int rc;
+		int i;
 
-		if (copy_from_user(&lum, lump, sizeof(struct lmv_user_md)))
+		ulmv = (struct lmv_user_md __user *)arg;
+		if (copy_from_user(&lum, ulmv, sizeof(*ulmv)))
 			return -EFAULT;
 
-		if (lum.lum_magic != LMV_MAGIC_V1)
+		/*
+		 * lum_magic will indicate which stripe the ioctl will like
+		 * to get, LMV_MAGIC_V1 is for normal LMV stripe, LMV_USER_MAGIC
+		 * is for default LMV stripe
+		 */
+		if (lum.lum_magic == LMV_MAGIC_V1)
+			valid |= OBD_MD_MEA;
+		else if (lum.lum_magic == LMV_USER_MAGIC)
+			valid |= OBD_MD_DEFAULT_MEA;
+		else
 			return -EINVAL;
 
-		lum_size = lmv_user_md_size(1, LMV_MAGIC_V1);
+		rc = ll_dir_getstripe(inode, (void **)&lmm, &lmmsize, &request,
+				      valid);
+		if (rc)
+			goto finish_req;
+
+		/* Get default LMV EA */
+		if (lum.lum_magic == LMV_USER_MAGIC) {
+			if (rc)
+				goto finish_req;
+
+			if (lmmsize > sizeof(*ulmv)) {
+				rc = -EINVAL;
+				goto finish_req;
+			}
+
+			if (copy_to_user(ulmv, lmm, lmmsize))
+				rc = -EFAULT;
+
+			goto finish_req;
+		}
+
+		stripe_count = lmv_mds_md_stripe_count_get(lmm);
+		lum_size = lmv_user_md_size(stripe_count, LMV_MAGIC_V1);
 		tmp = kzalloc(lum_size, GFP_NOFS);
 		if (!tmp) {
 			rc = -ENOMEM;
-			goto free_lmv;
+			goto finish_req;
 		}
 
-		*tmp = lum;
-		tmp->lum_type = LMV_STRIPE_TYPE;
-		tmp->lum_stripe_count = 1;
-		mdtindex = ll_get_mdt_idx(inode);
-		if (mdtindex < 0) {
+		mdt_index = ll_get_mdt_idx(inode);
+		if (mdt_index < 0) {
 			rc = -ENOMEM;
-			goto free_lmv;
+			goto out_tmp;
+		}
+		tmp->lum_magic = LMV_MAGIC_V1;
+		tmp->lum_stripe_count = 0;
+		tmp->lum_stripe_offset = mdt_index;
+		for (i = 0; i < stripe_count; i++) {
+			struct lu_fid fid;
+
+			fid_le_to_cpu(&fid, &lmm->lmv_md_v1.lmv_stripe_fids[i]);
+			mdt_index = ll_get_mdt_idx_by_fid(sbi, &fid);
+			if (mdt_index < 0) {
+				rc = mdt_index;
+				goto out_tmp;
+			}
+			tmp->lum_objects[i].lum_mds = mdt_index;
+			tmp->lum_objects[i].lum_fid = fid;
+			tmp->lum_stripe_count++;
 		}
 
-		tmp->lum_stripe_offset = mdtindex;
-		tmp->lum_objects[0].lum_mds = mdtindex;
-		memcpy(&tmp->lum_objects[0].lum_fid, ll_inode2fid(inode),
-		       sizeof(struct lu_fid));
-		if (copy_to_user((void __user *)arg, tmp, lum_size)) {
+		if (copy_to_user(ulmv, tmp, lum_size)) {
 			rc = -EFAULT;
-			goto free_lmv;
+			goto out_tmp;
 		}
-free_lmv:
+out_tmp:
 		kfree(tmp);
+finish_req:
+		ptlrpc_req_finished(request);
 		return rc;
 	}
+
 	case LL_IOC_LOV_SWAP_LAYOUTS:
 		return -EPERM;
-	case LL_IOC_OBD_STATFS:
+	case IOC_OBD_STATFS:
 		return ll_obd_statfs(inode, (void __user *)arg);
 	case LL_IOC_LOV_GETSTRIPE:
 	case LL_IOC_MDC_GETINFO:
@@ -1469,7 +1298,8 @@ free_lmv:
 			rc = ll_lov_getstripe_ea_info(inode, filename, &lmm,
 						      &lmmsize, &request);
 		} else {
-			rc = ll_dir_getstripe(inode, &lmm, &lmmsize, &request);
+			rc = ll_dir_getstripe(inode, (void **)&lmm, &lmmsize,
+					      &request, 0);
 		}
 
 		if (request) {
@@ -1512,18 +1342,18 @@ skip_lmm:
 			lstat_t st = { 0 };
 
 			st.st_dev     = inode->i_sb->s_dev;
-			st.st_mode    = body->mode;
-			st.st_nlink   = body->nlink;
-			st.st_uid     = body->uid;
-			st.st_gid     = body->gid;
-			st.st_rdev    = body->rdev;
-			st.st_size    = body->size;
+			st.st_mode    = body->mbo_mode;
+			st.st_nlink   = body->mbo_nlink;
+			st.st_uid     = body->mbo_uid;
+			st.st_gid     = body->mbo_gid;
+			st.st_rdev    = body->mbo_rdev;
+			st.st_size    = body->mbo_size;
 			st.st_blksize = PAGE_SIZE;
-			st.st_blocks  = body->blocks;
-			st.st_atime   = body->atime;
-			st.st_mtime   = body->mtime;
-			st.st_ctime   = body->ctime;
-			st.st_ino     = cl_fid_build_ino(&body->fid1,
+			st.st_blocks  = body->mbo_blocks;
+			st.st_atime   = body->mbo_atime;
+			st.st_mtime   = body->mbo_mtime;
+			st.st_ctime   = body->mbo_ctime;
+			st.st_ino     = cl_fid_build_ino(&body->mbo_fid1,
 							 sbi->ll_flags &
 							 LL_SBI_32BIT_API);
 
@@ -1611,9 +1441,6 @@ free_lmm:
 		kvfree(lmm);
 		return rc;
 	}
-	case OBD_IOC_LLOG_CATINFO: {
-		return -EOPNOTSUPP;
-	}
 	case OBD_IOC_QUOTACHECK: {
 		struct obd_quotactl *oqctl;
 		int error = 0;
@@ -1671,7 +1498,7 @@ out_poll:
 		kfree(check);
 		return rc;
 	}
-	case LL_IOC_QUOTACTL: {
+	case OBD_IOC_QUOTACTL: {
 		struct if_quotactl *qctl;
 
 		qctl = kzalloc(sizeof(*qctl), GFP_NOFS);
@@ -1739,6 +1566,25 @@ out_quotactl:
 		return rc;
 	case OBD_IOC_FID2PATH:
 		return ll_fid2path(inode, (void __user *)arg);
+	case LL_IOC_GETPARENT:
+		return ll_getparent(file, (void __user *)arg);
+	case LL_IOC_FID2MDTIDX: {
+		struct obd_export *exp = ll_i2mdexp(inode);
+		struct lu_fid fid;
+		__u32 index;
+
+		if (copy_from_user(&fid, (const struct lu_fid __user *)arg,
+				   sizeof(fid)))
+			return -EFAULT;
+
+		/* Call mdc_iocontrol */
+		rc = obd_iocontrol(LL_IOC_FID2MDTIDX, exp, sizeof(fid), &fid,
+				   &index);
+		if (rc)
+			return rc;
+
+		return index;
+	}
 	case LL_IOC_HSM_REQUEST: {
 		struct hsm_user_request	*hur;
 		ssize_t			 totalsize;
@@ -1853,6 +1699,45 @@ out_quotactl:
 		kfree(copy);
 		return rc;
 	}
+	case LL_IOC_MIGRATE: {
+		char *buf = NULL;
+		const char *filename;
+		int namelen = 0;
+		int len;
+		int rc;
+		int mdtidx;
+
+		rc = obd_ioctl_getdata(&buf, &len, (void __user *)arg);
+		if (rc < 0)
+			return rc;
+
+		data = (struct obd_ioctl_data *)buf;
+		if (!data->ioc_inlbuf1 || !data->ioc_inlbuf2 ||
+		    !data->ioc_inllen1 || !data->ioc_inllen2) {
+			rc = -EINVAL;
+			goto migrate_free;
+		}
+
+		filename = data->ioc_inlbuf1;
+		namelen = data->ioc_inllen1;
+		if (namelen < 1 || namelen != strlen(filename) + 1) {
+			rc = -EINVAL;
+			goto migrate_free;
+		}
+
+		if (data->ioc_inllen2 != sizeof(mdtidx)) {
+			rc = -EINVAL;
+			goto migrate_free;
+		}
+		mdtidx = *(int *)data->ioc_inlbuf2;
+
+		rc = ll_migrate(inode, file, mdtidx, filename, namelen - 1);
+migrate_free:
+		obd_ioctl_freedata(buf, len);
+
+		return rc;
+	}
+
 	default:
 		return obd_iocontrol(cmd, sbi->ll_dt_exp, 0, NULL,
 				     (void __user *)arg);
diff --git a/drivers/staging/lustre/lustre/llite/file.c b/drivers/staging/lustre/lustre/llite/file.c
index 57281b9e31ff..e1d784bae064 100644
--- a/drivers/staging/lustre/lustre/llite/file.c
+++ b/drivers/staging/lustre/lustre/llite/file.c
@@ -38,14 +38,15 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 #include "../include/lustre_dlm.h"
-#include "../include/lustre_lite.h"
 #include <linux/pagemap.h>
 #include <linux/file.h>
+#include <linux/sched.h>
 #include <linux/mount.h>
-#include "llite_internal.h"
 #include "../include/lustre/ll_fiemap.h"
+#include "../include/lustre/lustre_ioctl.h"
 
 #include "../include/cl_object.h"
+#include "llite_internal.h"
 
 static int
 ll_put_grouplock(struct inode *inode, struct file *file, unsigned long arg);
@@ -188,17 +189,11 @@ static int ll_close_inode_openhandle(struct obd_export *md_exp,
 		spin_unlock(&lli->lli_lock);
 	}
 
-	if (rc == 0) {
-		rc = ll_objects_destroy(req, inode);
-		if (rc)
-			CERROR("inode %lu ll_objects destroy: rc = %d\n",
-			       inode->i_ino, rc);
-	}
 	if (rc == 0 && op_data->op_bias & MDS_HSM_RELEASE) {
 		struct mdt_body *body;
 
 		body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
-		if (!(body->valid & OBD_MD_FLRELEASED))
+		if (!(body->mbo_valid & OBD_MD_FLRELEASED))
 			rc = -EBUSY;
 	}
 
@@ -349,13 +344,11 @@ int ll_file_release(struct inode *inode, struct file *file)
 	fd = LUSTRE_FPRIVATE(file);
 	LASSERT(fd);
 
-	/* The last ref on @file, maybe not be the owner pid of statahead.
-	 * Different processes can open the same dir, "ll_opendir_key" means:
-	 * it is me that should stop the statahead thread.
+	/* The last ref on @file, maybe not be the owner pid of statahead,
+	 * because parent and child process can share the same file handle.
 	 */
-	if (S_ISDIR(inode->i_mode) && lli->lli_opendir_key == fd &&
-	    lli->lli_opendir_pid != 0)
-		ll_stop_statahead(inode, lli->lli_opendir_key);
+	if (S_ISDIR(inode->i_mode) && lli->lli_opendir_key == fd)
+		ll_deauthorize_statahead(inode, fd);
 
 	if (is_root_inode(inode)) {
 		LUSTRE_FPRIVATE(file) = NULL;
@@ -364,7 +357,8 @@ int ll_file_release(struct inode *inode, struct file *file)
 	}
 
 	if (!S_ISDIR(inode->i_mode)) {
-		lov_read_and_clear_async_rc(lli->lli_clob);
+		if (lli->lli_clob)
+			lov_read_and_clear_async_rc(lli->lli_clob);
 		lli->lli_async_rc = 0;
 	}
 
@@ -376,55 +370,39 @@ int ll_file_release(struct inode *inode, struct file *file)
 	return rc;
 }
 
-static int ll_intent_file_open(struct dentry *dentry, void *lmm,
-			       int lmmsize, struct lookup_intent *itp)
+static int ll_intent_file_open(struct dentry *de, void *lmm, int lmmsize,
+			       struct lookup_intent *itp)
 {
-	struct inode *inode = d_inode(dentry);
+	struct inode *inode = d_inode(de);
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
-	struct dentry *parent = dentry->d_parent;
-	const char *name = dentry->d_name.name;
-	const int len = dentry->d_name.len;
+	struct dentry *parent = de->d_parent;
+	const char *name = NULL;
 	struct md_op_data *op_data;
-	struct ptlrpc_request *req;
-	__u32 opc = LUSTRE_OPC_ANY;
-	int rc;
+	struct ptlrpc_request *req = NULL;
+	int len = 0, rc;
 
-	/* Usually we come here only for NFSD, and we want open lock. */
-	/* We can also get here if there was cached open handle in revalidate_it
-	 * but it disappeared while we were getting from there to ll_file_open.
-	 * But this means this file was closed and immediately opened which
-	 * makes a good candidate for using OPEN lock
-	 */
-	/* If lmmsize & lmm are not 0, we are just setting stripe info
-	 * parameters. No need for the open lock
+	LASSERT(parent);
+	LASSERT(itp->it_flags & MDS_OPEN_BY_FID);
+
+	/*
+	 * if server supports open-by-fid, or file name is invalid, don't pack
+	 * name in open request
 	 */
-	if (!lmm && lmmsize == 0) {
-		struct ll_dentry_data *ldd = ll_d2d(dentry);
-		/*
-		 * If we came via ll_iget_for_nfs, then we need to request
-		 * struct ll_dentry_data *ldd = ll_d2d(file->f_dentry);
-		 *
-		 * NB: when ldd is NULL, it must have come via normal
-		 * lookup path only, since ll_iget_for_nfs always calls
-		 * ll_d_init().
-		 */
-		if (ldd && ldd->lld_nfs_dentry) {
-			ldd->lld_nfs_dentry = 0;
-			itp->it_flags |= MDS_OPEN_LOCK;
-		}
-		if (itp->it_flags & FMODE_WRITE)
-			opc = LUSTRE_OPC_CREATE;
+	if (!(exp_connect_flags(sbi->ll_md_exp) & OBD_CONNECT_OPEN_BY_FID) &&
+	    lu_name_is_valid_2(de->d_name.name, de->d_name.len)) {
+		name = de->d_name.name;
+		len = de->d_name.len;
 	}
 
-	op_data  = ll_prep_md_op_data(NULL, d_inode(parent),
-				      inode, name, len,
-				      O_RDWR, opc, NULL);
+	op_data  = ll_prep_md_op_data(NULL, d_inode(parent), inode, name, len,
+				      O_RDWR, LUSTRE_OPC_ANY, NULL);
 	if (IS_ERR(op_data))
 		return PTR_ERR(op_data);
+	op_data->op_data = lmm;
+	op_data->op_data_size = lmmsize;
 
-	itp->it_flags |= MDS_OPEN_BY_FID;
-	rc = md_intent_lock(sbi->ll_md_exp, op_data, lmm, lmmsize, itp,
-			    0 /*unused */, &req, ll_md_blocking_ast, 0);
+	rc = md_intent_lock(sbi->ll_md_exp, op_data, itp, &req,
+			    &ll_md_blocking_ast, 0);
 	ll_finish_md_op_data(op_data);
 	if (rc == -ESTALE) {
 		/* reason for keep own exit path - don`t flood log
@@ -479,8 +457,8 @@ static int ll_och_fill(struct obd_export *md_exp, struct lookup_intent *it,
 	struct mdt_body *body;
 
 	body = req_capsule_server_get(&it->it_request->rq_pill, &RMF_MDT_BODY);
-	och->och_fh = body->handle;
-	och->och_fid = body->fid1;
+	och->och_fh = body->mbo_handle;
+	och->och_fid = body->mbo_fid1;
 	och->och_lease_handle.cookie = it->it_lock_handle;
 	och->och_magic = OBD_CLIENT_HANDLE_MAGIC;
 	och->och_flags = it->it_flags;
@@ -508,7 +486,7 @@ static int ll_local_open(struct file *file, struct lookup_intent *it,
 
 		body = req_capsule_server_get(&it->it_request->rq_pill,
 					      &RMF_MDT_BODY);
-		ll_ioepoch_open(lli, body->ioepoch);
+		ll_ioepoch_open(lli, body->mbo_ioepoch);
 	}
 
 	LUSTRE_FPRIVATE(file) = fd;
@@ -543,7 +521,7 @@ int ll_file_open(struct inode *inode, struct file *file)
 	struct obd_client_handle **och_p = NULL;
 	__u64 *och_usecount = NULL;
 	struct ll_file_data *fd;
-	int rc = 0, opendir_set = 0;
+	int rc = 0;
 
 	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p), flags %o\n",
 	       PFID(ll_inode2fid(inode)), inode, file->f_flags);
@@ -558,16 +536,8 @@ int ll_file_open(struct inode *inode, struct file *file)
 	}
 
 	fd->fd_file = file;
-	if (S_ISDIR(inode->i_mode)) {
-		spin_lock(&lli->lli_sa_lock);
-		if (!lli->lli_opendir_key && !lli->lli_sai &&
-		    lli->lli_opendir_pid == 0) {
-			lli->lli_opendir_key = fd;
-			lli->lli_opendir_pid = current_pid();
-			opendir_set = 1;
-		}
-		spin_unlock(&lli->lli_sa_lock);
-	}
+	if (S_ISDIR(inode->i_mode))
+		ll_authorize_statahead(inode, fd);
 
 	if (is_root_inode(inode)) {
 		LUSTRE_FPRIVATE(file) = fd;
@@ -615,7 +585,7 @@ restart:
 	} else if (it->it_flags & FMODE_EXEC) {
 		och_p = &lli->lli_mds_exec_och;
 		och_usecount = &lli->lli_open_fd_exec_count;
-	 } else {
+	} else {
 		och_p = &lli->lli_mds_read_och;
 		och_usecount = &lli->lli_open_fd_read_count;
 	}
@@ -652,9 +622,19 @@ restart:
 			 * result in a deadlock
 			 */
 			mutex_unlock(&lli->lli_och_mutex);
-			it->it_create_mode |= M_CHECK_STALE;
+			/*
+			 * Normally called under two situations:
+			 * 1. NFS export.
+			 * 2. revalidate with IT_OPEN (revalidate doesn't
+			 *    execute this intent any more).
+			 *
+			 * Always fetch MDS_OPEN_LOCK if this is not setstripe.
+			 *
+			 * Always specify MDS_OPEN_BY_FID because we don't want
+			 * to get file with different fid.
+			 */
+			it->it_flags |= MDS_OPEN_LOCK | MDS_OPEN_BY_FID;
 			rc = ll_intent_file_open(file->f_path.dentry, NULL, 0, it);
-			it->it_create_mode &= ~M_CHECK_STALE;
 			if (rc)
 				goto out_openerr;
 
@@ -716,9 +696,10 @@ out_och_free:
 		mutex_unlock(&lli->lli_och_mutex);
 
 out_openerr:
-		if (opendir_set != 0)
-			ll_stop_statahead(inode, lli->lli_opendir_key);
-		ll_file_data_put(fd);
+		if (lli->lli_opendir_key == fd)
+			ll_deauthorize_statahead(inode, fd);
+		if (fd)
+			ll_file_data_put(fd);
 	} else {
 		ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_OPEN, 1);
 	}
@@ -764,7 +745,7 @@ ll_lease_open(struct inode *inode, struct file *file, fmode_t fmode,
 	struct lookup_intent it = { .it_op = IT_OPEN };
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 	struct md_op_data *op_data;
-	struct ptlrpc_request *req;
+	struct ptlrpc_request *req = NULL;
 	struct lustre_handle old_handle = { 0 };
 	struct obd_client_handle *och = NULL;
 	int rc;
@@ -831,8 +812,8 @@ ll_lease_open(struct inode *inode, struct file *file, fmode_t fmode,
 
 	it.it_flags = fmode | open_flags;
 	it.it_flags |= MDS_OPEN_LOCK | MDS_OPEN_BY_FID | MDS_OPEN_LEASE;
-	rc = md_intent_lock(sbi->ll_md_exp, op_data, NULL, 0, &it, 0, &req,
-			    ll_md_blocking_lease_ast,
+	rc = md_intent_lock(sbi->ll_md_exp, op_data, &it, &req,
+			    &ll_md_blocking_lease_ast,
 	/* LDLM_FL_NO_LRU: To not put the lease lock into LRU list, otherwise
 	 * it can be cancelled which may mislead applications that the lease is
 	 * broken;
@@ -840,7 +821,7 @@ ll_lease_open(struct inode *inode, struct file *file, fmode_t fmode,
 	 * open in ll_md_blocking_ast(). Otherwise as ll_md_blocking_lease_ast
 	 * doesn't deal with openhandle, so normal openhandle will be leaked.
 	 */
-				LDLM_FL_NO_LRU | LDLM_FL_EXCL);
+			    LDLM_FL_NO_LRU | LDLM_FL_EXCL);
 	ll_finish_md_op_data(op_data);
 	ptlrpc_req_finished(req);
 	if (rc < 0)
@@ -908,7 +889,6 @@ static int ll_lease_close(struct obd_client_handle *och, struct inode *inode,
 {
 	struct ldlm_lock *lock;
 	bool cancelled = true;
-	int rc;
 
 	lock = ldlm_handle2lock(&och->och_lease_handle);
 	if (lock) {
@@ -926,9 +906,8 @@ static int ll_lease_close(struct obd_client_handle *och, struct inode *inode,
 	if (lease_broken)
 		*lease_broken = cancelled;
 
-	rc = ll_close_inode_openhandle(ll_i2sbi(inode)->ll_md_exp, inode, och,
-				       NULL);
-	return rc;
+	return ll_close_inode_openhandle(ll_i2sbi(inode)->ll_md_exp,
+					 inode, och, NULL);
 }
 
 /* Fills the obdo with the attributes for the lsm */
@@ -1138,11 +1117,12 @@ ll_file_io_generic(const struct lu_env *env, struct vvp_io_args *args,
 {
 	struct ll_inode_info *lli = ll_i2info(file_inode(file));
 	struct ll_file_data  *fd  = LUSTRE_FPRIVATE(file);
+	struct range_lock range;
 	struct cl_io	 *io;
 	ssize_t	       result;
 
-	CDEBUG(D_VFSTRACE, "file: %s, type: %d ppos: %llu, count: %zd\n",
-	       file->f_path.dentry->d_name.name, iot, *ppos, count);
+	CDEBUG(D_VFSTRACE, "file: %pD, type: %d ppos: %llu, count: %zu\n",
+	       file, iot, *ppos, count);
 
 restart:
 	io = vvp_env_thread_io(env);
@@ -1150,41 +1130,45 @@ restart:
 
 	if (cl_io_rw_init(env, io, iot, *ppos, count) == 0) {
 		struct vvp_io *vio = vvp_env_io(env);
-		int write_mutex_locked = 0;
+		bool range_locked = false;
+
+		if (file->f_flags & O_APPEND)
+			range_lock_init(&range, 0, LUSTRE_EOF);
+		else
+			range_lock_init(&range, *ppos, *ppos + count - 1);
 
 		vio->vui_fd  = LUSTRE_FPRIVATE(file);
-		vio->vui_io_subtype = args->via_io_subtype;
-
-		switch (vio->vui_io_subtype) {
-		case IO_NORMAL:
-			vio->vui_iter = args->u.normal.via_iter;
-			vio->vui_iocb = args->u.normal.via_iocb;
-			if ((iot == CIT_WRITE) &&
-			    !(vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED)) {
-				if (mutex_lock_interruptible(&lli->
-							       lli_write_mutex)) {
-					result = -ERESTARTSYS;
-					goto out;
-				}
-				write_mutex_locked = 1;
-			}
-			down_read(&lli->lli_trunc_sem);
-			break;
-		case IO_SPLICE:
-			vio->u.splice.vui_pipe = args->u.splice.via_pipe;
-			vio->u.splice.vui_flags = args->u.splice.via_flags;
-			break;
-		default:
-			CERROR("Unknown IO type - %u\n", vio->vui_io_subtype);
-			LBUG();
+		vio->vui_iter = args->u.normal.via_iter;
+		vio->vui_iocb = args->u.normal.via_iocb;
+		/*
+		 * Direct IO reads must also take range lock,
+		 * or multiple reads will try to work on the same pages
+		 * See LU-6227 for details.
+		 */
+		if (((iot == CIT_WRITE) ||
+		     (iot == CIT_READ && (file->f_flags & O_DIRECT))) &&
+		    !(vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED)) {
+			CDEBUG(D_VFSTRACE, "Range lock [%llu, %llu]\n",
+			       range.rl_node.in_extent.start,
+			       range.rl_node.in_extent.end);
+			result = range_lock(&lli->lli_write_tree,
+					    &range);
+			if (result < 0)
+				goto out;
+
+			range_locked = true;
 		}
+		down_read(&lli->lli_trunc_sem);
 		ll_cl_add(file, env, io);
 		result = cl_io_loop(env, io);
 		ll_cl_remove(file, env);
-		if (args->via_io_subtype == IO_NORMAL)
-			up_read(&lli->lli_trunc_sem);
-		if (write_mutex_locked)
-			mutex_unlock(&lli->lli_write_mutex);
+		up_read(&lli->lli_trunc_sem);
+		if (range_locked) {
+			CDEBUG(D_VFSTRACE, "Range unlock [%llu, %llu]\n",
+			       range.rl_node.in_extent.start,
+			       range.rl_node.in_extent.end);
+			range_unlock(&lli->lli_write_tree, &range);
+		}
 	} else {
 		/* cl_io_rw_init() handled IO */
 		result = io->ci_result;
@@ -1201,7 +1185,7 @@ out:
 	 * short read/write instead of restart io.
 	 */
 	if ((result == 0 || result == -ENODATA) && io->ci_need_restart) {
-		CDEBUG(D_VFSTRACE, "Restart %s on %pD from %lld, count:%zd\n",
+		CDEBUG(D_VFSTRACE, "Restart %s on %pD from %lld, count:%zu\n",
 		       iot == CIT_READ ? "read" : "write",
 		       file, *ppos, count);
 		LASSERTF(io->ci_nob == 0, "%zd\n", io->ci_nob);
@@ -1237,7 +1221,7 @@ static ssize_t ll_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	if (IS_ERR(env))
 		return PTR_ERR(env);
 
-	args = ll_env_args(env, IO_NORMAL);
+	args = ll_env_args(env);
 	args->u.normal.via_iter = to;
 	args->u.normal.via_iocb = iocb;
 
@@ -1261,7 +1245,7 @@ static ssize_t ll_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	if (IS_ERR(env))
 		return PTR_ERR(env);
 
-	args = ll_env_args(env, IO_NORMAL);
+	args = ll_env_args(env);
 	args->u.normal.via_iter = from;
 	args->u.normal.via_iocb = iocb;
 
@@ -1271,119 +1255,15 @@ static ssize_t ll_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	return result;
 }
 
-/*
- * Send file content (through pagecache) somewhere with helper
- */
-static ssize_t ll_file_splice_read(struct file *in_file, loff_t *ppos,
-				   struct pipe_inode_info *pipe, size_t count,
-				   unsigned int flags)
-{
-	struct lu_env      *env;
-	struct vvp_io_args *args;
-	ssize_t	     result;
-	int		 refcheck;
-
-	env = cl_env_get(&refcheck);
-	if (IS_ERR(env))
-		return PTR_ERR(env);
-
-	args = ll_env_args(env, IO_SPLICE);
-	args->u.splice.via_pipe = pipe;
-	args->u.splice.via_flags = flags;
-
-	result = ll_file_io_generic(env, args, in_file, CIT_READ, ppos, count);
-	cl_env_put(env, &refcheck);
-	return result;
-}
-
-static int ll_lov_recreate(struct inode *inode, struct ost_id *oi, u32 ost_idx)
-{
-	struct obd_export *exp = ll_i2dtexp(inode);
-	struct obd_trans_info oti = { 0 };
-	struct obdo *oa = NULL;
-	int lsm_size;
-	int rc = 0;
-	struct lov_stripe_md *lsm = NULL, *lsm2;
-
-	oa = kmem_cache_zalloc(obdo_cachep, GFP_NOFS);
-	if (!oa)
-		return -ENOMEM;
-
-	lsm = ccc_inode_lsm_get(inode);
-	if (!lsm_has_objects(lsm)) {
-		rc = -ENOENT;
-		goto out;
-	}
-
-	lsm_size = sizeof(*lsm) + (sizeof(struct lov_oinfo) *
-		   (lsm->lsm_stripe_count));
-
-	lsm2 = libcfs_kvzalloc(lsm_size, GFP_NOFS);
-	if (!lsm2) {
-		rc = -ENOMEM;
-		goto out;
-	}
-
-	oa->o_oi = *oi;
-	oa->o_nlink = ost_idx;
-	oa->o_flags |= OBD_FL_RECREATE_OBJS;
-	oa->o_valid = OBD_MD_FLID | OBD_MD_FLFLAGS | OBD_MD_FLGROUP;
-	obdo_from_inode(oa, inode, OBD_MD_FLTYPE | OBD_MD_FLATIME |
-				   OBD_MD_FLMTIME | OBD_MD_FLCTIME);
-	obdo_set_parent_fid(oa, &ll_i2info(inode)->lli_fid);
-	memcpy(lsm2, lsm, lsm_size);
-	ll_inode_size_lock(inode);
-	rc = obd_create(NULL, exp, oa, &lsm2, &oti);
-	ll_inode_size_unlock(inode);
-
-	kvfree(lsm2);
-	goto out;
-out:
-	ccc_inode_lsm_put(inode, lsm);
-	kmem_cache_free(obdo_cachep, oa);
-	return rc;
-}
-
-static int ll_lov_recreate_obj(struct inode *inode, unsigned long arg)
-{
-	struct ll_recreate_obj ucreat;
-	struct ost_id		oi;
-
-	if (!capable(CFS_CAP_SYS_ADMIN))
-		return -EPERM;
-
-	if (copy_from_user(&ucreat, (struct ll_recreate_obj __user *)arg,
-			   sizeof(ucreat)))
-		return -EFAULT;
-
-	ostid_set_seq_mdt0(&oi);
-	ostid_set_id(&oi, ucreat.lrc_id);
-	return ll_lov_recreate(inode, &oi, ucreat.lrc_ost_idx);
-}
-
-static int ll_lov_recreate_fid(struct inode *inode, unsigned long arg)
-{
-	struct lu_fid	fid;
-	struct ost_id	oi;
-	u32		ost_idx;
-
-	if (!capable(CFS_CAP_SYS_ADMIN))
-		return -EPERM;
-
-	if (copy_from_user(&fid, (struct lu_fid __user *)arg, sizeof(fid)))
-		return -EFAULT;
-
-	fid_to_ostid(&fid, &oi);
-	ost_idx = (fid_seq(&fid) >> 16) & 0xffff;
-	return ll_lov_recreate(inode, &oi, ost_idx);
-}
-
 int ll_lov_setstripe_ea_info(struct inode *inode, struct dentry *dentry,
 			     __u64 flags, struct lov_user_md *lum,
 			     int lum_size)
 {
 	struct lov_stripe_md *lsm = NULL;
-	struct lookup_intent oit = {.it_op = IT_OPEN, .it_flags = flags};
+	struct lookup_intent oit = {
+		.it_op = IT_OPEN,
+		.it_flags = flags | MDS_OPEN_BY_FID,
+	};
 	int rc = 0;
 
 	lsm = ccc_inode_lsm_get(inode);
@@ -1397,11 +1277,11 @@ int ll_lov_setstripe_ea_info(struct inode *inode, struct dentry *dentry,
 
 	ll_inode_size_lock(inode);
 	rc = ll_intent_file_open(dentry, lum, lum_size, &oit);
-	if (rc)
+	if (rc < 0)
 		goto out_unlock;
 	rc = oit.it_status;
 	if (rc < 0)
-		goto out_req_free;
+		goto out_unlock;
 
 	ll_release_openhandle(inode, &oit);
 
@@ -1411,9 +1291,6 @@ out_unlock:
 	ccc_inode_lsm_put(inode, lsm);
 out:
 	return rc;
-out_req_free:
-	ptlrpc_req_finished((struct ptlrpc_request *)oit.it_request);
-	goto out;
 }
 
 int ll_lov_getstripe_ea_info(struct inode *inode, const char *filename,
@@ -1448,9 +1325,9 @@ int ll_lov_getstripe_ea_info(struct inode *inode, const char *filename,
 
 	body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
 
-	lmmsize = body->eadatasize;
+	lmmsize = body->mbo_eadatasize;
 
-	if (!(body->valid & (OBD_MD_FLEASIZE | OBD_MD_FLDIREA)) ||
+	if (!(body->mbo_valid & (OBD_MD_FLEASIZE | OBD_MD_FLDIREA)) ||
 	    lmmsize == 0) {
 		rc = -ENODATA;
 		goto out;
@@ -1481,13 +1358,13 @@ int ll_lov_getstripe_ea_info(struct inode *inode, const char *filename,
 		 */
 		if (lmm->lmm_magic == cpu_to_le32(LOV_MAGIC_V1)) {
 			lustre_swab_lov_user_md_v1((struct lov_user_md_v1 *)lmm);
-			if (S_ISREG(body->mode))
+			if (S_ISREG(body->mbo_mode))
 				lustre_swab_lov_user_md_objects(
 				 ((struct lov_user_md_v1 *)lmm)->lmm_objects,
 				 stripe_count);
 		} else if (lmm->lmm_magic == cpu_to_le32(LOV_MAGIC_V3)) {
 			lustre_swab_lov_user_md_v3((struct lov_user_md_v3 *)lmm);
-			if (S_ISREG(body->mode))
+			if (S_ISREG(body->mbo_mode))
 				lustre_swab_lov_user_md_objects(
 				 ((struct lov_user_md_v3 *)lmm)->lmm_objects,
 				 stripe_count);
@@ -1530,55 +1407,48 @@ static int ll_lov_setea(struct inode *inode, struct file *file,
 	return rc;
 }
 
+static int ll_file_getstripe(struct inode *inode,
+			     struct lov_user_md __user *lum)
+{
+	struct lu_env *env;
+	int refcheck;
+	int rc;
+
+	env = cl_env_get(&refcheck);
+	if (IS_ERR(env))
+		return PTR_ERR(env);
+
+	rc = cl_object_getstripe(env, ll_i2info(inode)->lli_clob, lum);
+	cl_env_put(env, &refcheck);
+	return rc;
+}
+
 static int ll_lov_setstripe(struct inode *inode, struct file *file,
 			    unsigned long arg)
 {
-	struct lov_user_md_v3 lumv3;
-	struct lov_user_md_v1 *lumv1 = (struct lov_user_md_v1 *)&lumv3;
-	struct lov_user_md_v1 __user *lumv1p = (void __user *)arg;
-	struct lov_user_md_v3 __user *lumv3p = (void __user *)arg;
+	struct lov_user_md __user *lum = (struct lov_user_md __user *)arg;
+	struct lov_user_md *klum;
 	int lum_size, rc;
 	__u64 flags = FMODE_WRITE;
 
-	/* first try with v1 which is smaller than v3 */
-	lum_size = sizeof(struct lov_user_md_v1);
-	if (copy_from_user(lumv1, lumv1p, lum_size))
-		return -EFAULT;
-
-	if (lumv1->lmm_magic == LOV_USER_MAGIC_V3) {
-		lum_size = sizeof(struct lov_user_md_v3);
-		if (copy_from_user(&lumv3, lumv3p, lum_size))
-			return -EFAULT;
-	}
+	rc = ll_copy_user_md(lum, &klum);
+	if (rc < 0)
+		return rc;
 
-	rc = ll_lov_setstripe_ea_info(inode, file->f_path.dentry, flags, lumv1,
+	lum_size = rc;
+	rc = ll_lov_setstripe_ea_info(inode, file->f_path.dentry, flags, klum,
 				      lum_size);
 	cl_lov_delay_create_clear(&file->f_flags);
 	if (rc == 0) {
-		struct lov_stripe_md *lsm;
 		__u32 gen;
 
-		put_user(0, &lumv1p->lmm_stripe_count);
+		put_user(0, &lum->lmm_stripe_count);
 
 		ll_layout_refresh(inode, &gen);
-		lsm = ccc_inode_lsm_get(inode);
-		rc = obd_iocontrol(LL_IOC_LOV_GETSTRIPE, ll_i2dtexp(inode),
-				   0, lsm, (void __user *)arg);
-		ccc_inode_lsm_put(inode, lsm);
+		rc = ll_file_getstripe(inode, (struct lov_user_md __user *)arg);
 	}
-	return rc;
-}
 
-static int ll_lov_getstripe(struct inode *inode, unsigned long arg)
-{
-	struct lov_stripe_md *lsm;
-	int rc = -ENODATA;
-
-	lsm = ccc_inode_lsm_get(inode);
-	if (lsm)
-		rc = obd_iocontrol(LL_IOC_LOV_GETSTRIPE, ll_i2dtexp(inode), 0,
-				   lsm, (void __user *)arg);
-	ccc_inode_lsm_put(inode, lsm);
+	kfree(klum);
 	return rc;
 }
 
@@ -2247,6 +2117,12 @@ free_hss:
 	return rc;
 }
 
+static inline long ll_lease_type_from_fmode(fmode_t fmode)
+{
+	return ((fmode & FMODE_READ) ? LL_LEASE_RDLCK : 0) |
+	       ((fmode & FMODE_WRITE) ? LL_LEASE_WRLCK : 0);
+}
+
 static long
 ll_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
@@ -2314,11 +2190,8 @@ ll_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		return rc;
 	}
 	case LL_IOC_LOV_GETSTRIPE:
-		return ll_lov_getstripe(inode, arg);
-	case LL_IOC_RECREATE_OBJ:
-		return ll_lov_recreate_obj(inode, arg);
-	case LL_IOC_RECREATE_FID:
-		return ll_lov_recreate_fid(inode, arg);
+		return ll_file_getstripe(inode,
+					 (struct lov_user_md __user *)arg);
 	case FSFILT_IOC_FIEMAP:
 		return ll_ioctl_fiemap(inode, arg);
 	case FSFILT_IOC_GETFLAGS:
@@ -2349,6 +2222,8 @@ ll_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 
 		return 0;
 	}
+	case LL_IOC_GETPARENT:
+		return ll_getparent(file, (struct getparent __user *)arg);
 	case OBD_IOC_FID2PATH:
 		return ll_fid2path(inode, (void __user *)arg);
 	case LL_IOC_DATA_VERSION: {
@@ -2451,20 +2326,20 @@ ll_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		struct ll_inode_info *lli = ll_i2info(inode);
 		struct obd_client_handle *och = NULL;
 		bool lease_broken;
-		fmode_t mode = 0;
+		fmode_t fmode;
 
 		switch (arg) {
-		case F_WRLCK:
+		case LL_LEASE_WRLCK:
 			if (!(file->f_mode & FMODE_WRITE))
 				return -EPERM;
-			mode = FMODE_WRITE;
+			fmode = FMODE_WRITE;
 			break;
-		case F_RDLCK:
+		case LL_LEASE_RDLCK:
 			if (!(file->f_mode & FMODE_READ))
 				return -EPERM;
-			mode = FMODE_READ;
+			fmode = FMODE_READ;
 			break;
-		case F_UNLCK:
+		case LL_LEASE_UNLCK:
 			mutex_lock(&lli->lli_och_mutex);
 			if (fd->fd_lease_och) {
 				och = fd->fd_lease_och;
@@ -2472,26 +2347,26 @@ ll_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			}
 			mutex_unlock(&lli->lli_och_mutex);
 
-			if (och) {
-				mode = och->och_flags &
-				       (FMODE_READ|FMODE_WRITE);
-				rc = ll_lease_close(och, inode, &lease_broken);
-				if (rc == 0 && lease_broken)
-					mode = 0;
-			} else {
-				rc = -ENOLCK;
-			}
+			if (!och)
+				return -ENOLCK;
+
+			fmode = och->och_flags;
+			rc = ll_lease_close(och, inode, &lease_broken);
+			if (rc < 0)
+				return rc;
+
+			if (lease_broken)
+				fmode = 0;
 
-			/* return the type of lease or error */
-			return rc < 0 ? rc : (int)mode;
+			return ll_lease_type_from_fmode(fmode);
 		default:
 			return -EINVAL;
 		}
 
-		CDEBUG(D_INODE, "Set lease with mode %d\n", mode);
+		CDEBUG(D_INODE, "Set lease with mode %u\n", fmode);
 
 		/* apply for lease */
-		och = ll_lease_open(inode, file, mode, 0);
+		och = ll_lease_open(inode, file, fmode, 0);
 		if (IS_ERR(och))
 			return PTR_ERR(och);
 
@@ -2512,8 +2387,8 @@ ll_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 	case LL_IOC_GET_LEASE: {
 		struct ll_inode_info *lli = ll_i2info(inode);
 		struct ldlm_lock *lock = NULL;
+		fmode_t fmode = 0;
 
-		rc = 0;
 		mutex_lock(&lli->lli_och_mutex);
 		if (fd->fd_lease_och) {
 			struct obd_client_handle *och = fd->fd_lease_och;
@@ -2522,14 +2397,13 @@ ll_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			if (lock) {
 				lock_res_and_lock(lock);
 				if (!ldlm_is_cancel(lock))
-					rc = och->och_flags &
-						(FMODE_READ | FMODE_WRITE);
+					fmode = och->och_flags;
 				unlock_res_and_lock(lock);
 				LDLM_LOCK_PUT(lock);
 			}
 		}
 		mutex_unlock(&lli->lli_och_mutex);
-		return rc;
+		return ll_lease_type_from_fmode(fmode);
 	}
 	case LL_IOC_HSM_IMPORT: {
 		struct hsm_user_import *hui;
@@ -2574,9 +2448,8 @@ static loff_t ll_file_seek(struct file *file, loff_t offset, int origin)
 		eof = i_size_read(inode);
 	}
 
-	retval = generic_file_llseek_size(file, offset, origin,
-					  ll_file_maxbytes(inode), eof);
-	return retval;
+	return generic_file_llseek_size(file, offset, origin,
+					ll_file_maxbytes(inode), eof);
 }
 
 static int ll_flush(struct file *file, fl_owner_t id)
@@ -2593,9 +2466,11 @@ static int ll_flush(struct file *file, fl_owner_t id)
 	 */
 	rc = lli->lli_async_rc;
 	lli->lli_async_rc = 0;
-	err = lov_read_and_clear_async_rc(lli->lli_clob);
-	if (rc == 0)
-		rc = err;
+	if (lli->lli_clob) {
+		err = lov_read_and_clear_async_rc(lli->lli_clob);
+		if (!rc)
+			rc = err;
+	}
 
 	/* The application has been told about write failure already.
 	 * Do not report failure again.
@@ -2714,6 +2589,7 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 	struct md_op_data *op_data;
 	struct lustre_handle lockh = {0};
 	ldlm_policy_data_t flock = { {0} };
+	int fl_type = file_lock->fl_type;
 	__u64 flags = 0;
 	int rc;
 	int rc2 = 0;
@@ -2744,7 +2620,7 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 	if (file_lock->fl_lmops && file_lock->fl_lmops->lm_compare_owner)
 		flock.l_flock.owner = (unsigned long)file_lock->fl_pid;
 
-	switch (file_lock->fl_type) {
+	switch (fl_type) {
 	case F_RDLCK:
 		einfo.ei_mode = LCK_PR;
 		break;
@@ -2764,8 +2640,7 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 		einfo.ei_mode = LCK_PW;
 		break;
 	default:
-		CDEBUG(D_INFO, "Unknown fcntl lock type: %d\n",
-		       file_lock->fl_type);
+		CDEBUG(D_INFO, "Unknown fcntl lock type: %d\n", fl_type);
 		return -ENOTSUPP;
 	}
 
@@ -2787,16 +2662,18 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 	case F_GETLK64:
 #endif
 		flags = LDLM_FL_TEST_LOCK;
-		/* Save the old mode so that if the mode in the lock changes we
-		 * can decrement the appropriate reader or writer refcount.
-		 */
-		file_lock->fl_type = einfo.ei_mode;
 		break;
 	default:
 		CERROR("unknown fcntl lock command: %d\n", cmd);
 		return -EINVAL;
 	}
 
+	/*
+	 * Save the old mode so that if the mode in the lock changes we
+	 * can decrement the appropriate reader or writer refcount.
+	 */
+	file_lock->fl_type = einfo.ei_mode;
+
 	op_data = ll_prep_md_op_data(NULL, inode, NULL, NULL, 0, 0,
 				     LUSTRE_OPC_ANY, NULL);
 	if (IS_ERR(op_data))
@@ -2806,8 +2683,12 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 	       PFID(ll_inode2fid(inode)), flock.l_flock.pid, flags,
 	       einfo.ei_mode, flock.l_flock.start, flock.l_flock.end);
 
-	rc = md_enqueue(sbi->ll_md_exp, &einfo, NULL,
-			op_data, &lockh, &flock, 0, NULL /* req */, flags);
+	rc = md_enqueue(sbi->ll_md_exp, &einfo, &flock, NULL, op_data, &lockh,
+			flags);
+
+	/* Restore the file lock type if not TEST lock. */
+	if (!(flags & LDLM_FL_TEST_LOCK))
+		file_lock->fl_type = fl_type;
 
 	if ((rc == 0 || file_lock->fl_type == F_UNLCK) &&
 	    !(flags & LDLM_FL_TEST_LOCK))
@@ -2815,8 +2696,8 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 
 	if (rc2 && file_lock->fl_type != F_UNLCK) {
 		einfo.ei_mode = LCK_NL;
-		md_enqueue(sbi->ll_md_exp, &einfo, NULL,
-			   op_data, &lockh, &flock, 0, NULL /* req */, flags);
+		md_enqueue(sbi->ll_md_exp, &einfo, &flock, NULL, op_data,
+			   &lockh, flags);
 		rc = rc2;
 	}
 
@@ -2825,6 +2706,117 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 	return rc;
 }
 
+int ll_get_fid_by_name(struct inode *parent, const char *name,
+		       int namelen, struct lu_fid *fid)
+{
+	struct md_op_data *op_data = NULL;
+	struct ptlrpc_request *req;
+	struct mdt_body *body;
+	int rc;
+
+	op_data = ll_prep_md_op_data(NULL, parent, NULL, name, namelen, 0,
+				     LUSTRE_OPC_ANY, NULL);
+	if (IS_ERR(op_data))
+		return PTR_ERR(op_data);
+
+	op_data->op_valid = OBD_MD_FLID;
+	rc = md_getattr_name(ll_i2sbi(parent)->ll_md_exp, op_data, &req);
+	ll_finish_md_op_data(op_data);
+	if (rc < 0)
+		return rc;
+
+	body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
+	if (!body) {
+		rc = -EFAULT;
+		goto out_req;
+	}
+	if (fid)
+		*fid = body->mbo_fid1;
+out_req:
+	ptlrpc_req_finished(req);
+	return rc;
+}
+
+int ll_migrate(struct inode *parent, struct file *file, int mdtidx,
+	       const char *name, int namelen)
+{
+	struct ptlrpc_request *request = NULL;
+	struct inode *child_inode = NULL;
+	struct dentry *dchild = NULL;
+	struct md_op_data *op_data;
+	struct qstr qstr;
+	int rc;
+
+	CDEBUG(D_VFSTRACE, "migrate %s under "DFID" to MDT%d\n",
+	       name, PFID(ll_inode2fid(parent)), mdtidx);
+
+	op_data = ll_prep_md_op_data(NULL, parent, NULL, name, namelen,
+				     0, LUSTRE_OPC_ANY, NULL);
+	if (IS_ERR(op_data))
+		return PTR_ERR(op_data);
+
+	/* Get child FID first */
+	qstr.hash = full_name_hash(parent, name, namelen);
+	qstr.name = name;
+	qstr.len = namelen;
+	dchild = d_lookup(file_dentry(file), &qstr);
+	if (dchild) {
+		op_data->op_fid3 = *ll_inode2fid(dchild->d_inode);
+		if (dchild->d_inode) {
+			child_inode = igrab(dchild->d_inode);
+			if (child_inode) {
+				inode_lock(child_inode);
+				op_data->op_fid3 = *ll_inode2fid(child_inode);
+				ll_invalidate_aliases(child_inode);
+			}
+		}
+		dput(dchild);
+	} else {
+		rc = ll_get_fid_by_name(parent, name, namelen,
+					&op_data->op_fid3);
+		if (rc)
+			goto out_free;
+	}
+
+	if (!fid_is_sane(&op_data->op_fid3)) {
+		CERROR("%s: migrate %s, but fid "DFID" is insane\n",
+		       ll_get_fsname(parent->i_sb, NULL, 0), name,
+		       PFID(&op_data->op_fid3));
+		rc = -EINVAL;
+		goto out_free;
+	}
+
+	rc = ll_get_mdt_idx_by_fid(ll_i2sbi(parent), &op_data->op_fid3);
+	if (rc < 0)
+		goto out_free;
+
+	if (rc == mdtidx) {
+		CDEBUG(D_INFO, "%s:"DFID" is already on MDT%d.\n", name,
+		       PFID(&op_data->op_fid3), mdtidx);
+		rc = 0;
+		goto out_free;
+	}
+
+	op_data->op_mds = mdtidx;
+	op_data->op_cli_flags = CLI_MIGRATE;
+	rc = md_rename(ll_i2sbi(parent)->ll_md_exp, op_data, name,
+		       namelen, name, namelen, &request);
+	if (!rc)
+		ll_update_times(request, parent);
+
+	ptlrpc_req_finished(request);
+
+out_free:
+	if (child_inode) {
+		clear_nlink(child_inode);
+		inode_unlock(child_inode);
+		iput(child_inode);
+	}
+
+	ll_finish_md_op_data(op_data);
+	return rc;
+}
+
 static int
 ll_file_noflock(struct file *file, int cmd, struct file_lock *file_lock)
 {
@@ -2847,7 +2839,7 @@ int ll_have_md_lock(struct inode *inode, __u64 *bits,
 	struct lustre_handle lockh;
 	ldlm_policy_data_t policy;
 	enum ldlm_mode mode = (l_req_mode == LCK_MINMODE) ?
-				(LCK_CR|LCK_CW|LCK_PR|LCK_PW) : l_req_mode;
+			      (LCK_CR | LCK_CW | LCK_PR | LCK_PW) : l_req_mode;
 	struct lu_fid *fid;
 	__u64 flags;
 	int i;
@@ -2888,15 +2880,12 @@ enum ldlm_mode ll_take_md_lock(struct inode *inode, __u64 bits,
 {
 	ldlm_policy_data_t policy = { .l_inodebits = {bits} };
 	struct lu_fid *fid;
-	enum ldlm_mode rc;
 
 	fid = &ll_i2info(inode)->lli_fid;
 	CDEBUG(D_INFO, "trying to match res "DFID"\n", PFID(fid));
 
-	rc = md_lock_match(ll_i2mdexp(inode), flags | LDLM_FL_BLOCK_GRANTED,
-			   fid, LDLM_IBITS, &policy, mode, lockh);
-
-	return rc;
+	return md_lock_match(ll_i2mdexp(inode), flags | LDLM_FL_BLOCK_GRANTED,
+			     fid, LDLM_IBITS, &policy, mode, lockh);
 }
 
 static int ll_inode_revalidate_fini(struct inode *inode, int rc)
@@ -2949,15 +2938,9 @@ static int __ll_inode_revalidate(struct dentry *dentry, __u64 ibits)
 		if (IS_ERR(op_data))
 			return PTR_ERR(op_data);
 
-		oit.it_create_mode |= M_CHECK_STALE;
-		rc = md_intent_lock(exp, op_data, NULL, 0,
-				    /* we are not interested in name
-				     * based lookup
-				     */
-				    &oit, 0, &req,
-				    ll_md_blocking_ast, 0);
+		rc = md_intent_lock(exp, op_data, &oit, &req,
+				    &ll_md_blocking_ast, 0);
 		ll_finish_md_op_data(op_data);
-		oit.it_create_mode &= ~M_CHECK_STALE;
 		if (rc < 0) {
 			rc = ll_inode_revalidate_fini(inode, rc);
 			goto out;
@@ -3003,10 +2986,8 @@ static int __ll_inode_revalidate(struct dentry *dentry, __u64 ibits)
 		op_data->op_valid = valid;
 		rc = md_getattr(sbi->ll_md_exp, op_data, &req);
 		ll_finish_md_op_data(op_data);
-		if (rc) {
-			rc = ll_inode_revalidate_fini(inode, rc);
-			return rc;
-		}
+		if (rc)
+			return ll_inode_revalidate_fini(inode, rc);
 
 		rc = ll_prep_inode(&inode, req, NULL, NULL);
 	}
@@ -3015,6 +2996,28 @@ out:
 	return rc;
 }
 
+static int ll_merge_md_attr(struct inode *inode)
+{
+	struct cl_attr attr = { 0 };
+	int rc;
+
+	LASSERT(ll_i2info(inode)->lli_lsm_md);
+	rc = md_merge_attr(ll_i2mdexp(inode), ll_i2info(inode)->lli_lsm_md,
+			   &attr, ll_md_blocking_ast);
+	if (rc)
+		return rc;
+
+	set_nlink(inode, attr.cat_nlink);
+	inode->i_blocks = attr.cat_blocks;
+	i_size_write(inode, attr.cat_size);
+
+	ll_i2info(inode)->lli_atime = attr.cat_atime;
+	ll_i2info(inode)->lli_mtime = attr.cat_mtime;
+	ll_i2info(inode)->lli_ctime = attr.cat_ctime;
+
+	return 0;
+}
+
 static int ll_inode_revalidate(struct dentry *dentry, __u64 ibits)
 {
 	struct inode *inode = d_inode(dentry);
@@ -3026,6 +3029,13 @@ static int ll_inode_revalidate(struct dentry *dentry, __u64 ibits)
 
 	/* if object isn't regular file, don't validate size */
 	if (!S_ISREG(inode->i_mode)) {
+		if (S_ISDIR(inode->i_mode) &&
+		    ll_i2info(inode)->lli_lsm_md) {
+			rc = ll_merge_md_attr(inode);
+			if (rc)
+				return rc;
+		}
+
 		LTIME_S(inode->i_atime) = ll_i2info(inode)->lli_atime;
 		LTIME_S(inode->i_mtime) = ll_i2info(inode)->lli_mtime;
 		LTIME_S(inode->i_ctime) = ll_i2info(inode)->lli_ctime;
@@ -3057,13 +3067,14 @@ int ll_getattr(struct vfsmount *mnt, struct dentry *de, struct kstat *stat)
 	if (res)
 		return res;
 
+	OBD_FAIL_TIMEOUT(OBD_FAIL_GETATTR_DELAY, 30);
+
 	stat->dev = inode->i_sb->s_dev;
 	if (ll_need_32bit_api(sbi))
 		stat->ino = cl_fid_build_ino(&lli->lli_fid, 1);
 	else
 		stat->ino = inode->i_ino;
 	stat->mode = inode->i_mode;
-	stat->nlink = inode->i_nlink;
 	stat->uid = inode->i_uid;
 	stat->gid = inode->i_gid;
 	stat->rdev = inode->i_rdev;
@@ -3072,6 +3083,7 @@ int ll_getattr(struct vfsmount *mnt, struct dentry *de, struct kstat *stat)
 	stat->ctime = inode->i_ctime;
 	stat->blksize = 1 << inode->i_blkbits;
 
+	stat->nlink = inode->i_nlink;
 	stat->size = i_size_read(inode);
 	stat->blocks = inode->i_blocks;
 
@@ -3139,6 +3151,12 @@ struct posix_acl *ll_get_acl(struct inode *inode, int type)
 
 int ll_inode_permission(struct inode *inode, int mask)
 {
+	struct ll_sb_info *sbi;
+	struct root_squash_info *squash;
+	const struct cred *old_cred = NULL;
+	struct cred *cred = NULL;
+	bool squash_id = false;
+	cfs_cap_t cap;
 	int rc = 0;
 
 	if (mask & MAY_NOT_BLOCK)
@@ -3158,9 +3176,46 @@ int ll_inode_permission(struct inode *inode, int mask)
 	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p), inode mode %x mask %o\n",
 	       PFID(ll_inode2fid(inode)), inode, inode->i_mode, mask);
 
+	/* squash fsuid/fsgid if needed */
+	sbi = ll_i2sbi(inode);
+	squash = &sbi->ll_squash;
+	if (unlikely(squash->rsi_uid &&
+		     uid_eq(current_fsuid(), GLOBAL_ROOT_UID) &&
+		     !(sbi->ll_flags & LL_SBI_NOROOTSQUASH))) {
+		squash_id = true;
+	}
+
+	if (squash_id) {
+		CDEBUG(D_OTHER, "squash creds (%d:%d)=>(%d:%d)\n",
+		       __kuid_val(current_fsuid()), __kgid_val(current_fsgid()),
+		       squash->rsi_uid, squash->rsi_gid);
+
+		/*
+		 * update current process's credentials
+		 * and FS capability
+		 */
+		cred = prepare_creds();
+		if (!cred)
+			return -ENOMEM;
+
+		cred->fsuid = make_kuid(&init_user_ns, squash->rsi_uid);
+		cred->fsgid = make_kgid(&init_user_ns, squash->rsi_gid);
+		for (cap = 0; cap < sizeof(cfs_cap_t) * 8; cap++) {
+			if ((1 << cap) & CFS_CAP_FS_MASK)
+				cap_lower(cred->cap_effective, cap);
+		}
+		old_cred = override_creds(cred);
+	}
+
 	ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_INODE_PERM, 1);
 	rc = generic_permission(inode, mask);
 
+	/* restore current process's credentials and FS capability */
+	if (squash_id) {
+		revert_creds(old_cred);
+		put_cred(cred);
+	}
+
 	return rc;
 }
 
@@ -3173,7 +3228,7 @@ struct file_operations ll_file_operations = {
 	.release	= ll_file_release,
 	.mmap	   = ll_file_mmap,
 	.llseek	 = ll_file_seek,
-	.splice_read    = ll_file_splice_read,
+	.splice_read    = generic_file_splice_read,
 	.fsync	  = ll_fsync,
 	.flush	  = ll_flush
 };
@@ -3186,7 +3241,7 @@ struct file_operations ll_file_operations_flock = {
 	.release	= ll_file_release,
 	.mmap	   = ll_file_mmap,
 	.llseek	 = ll_file_seek,
-	.splice_read    = ll_file_splice_read,
+	.splice_read    = generic_file_splice_read,
 	.fsync	  = ll_fsync,
 	.flush	  = ll_flush,
 	.flock	  = ll_file_flock,
@@ -3202,7 +3257,7 @@ struct file_operations ll_file_operations_noflock = {
 	.release	= ll_file_release,
 	.mmap	   = ll_file_mmap,
 	.llseek	 = ll_file_seek,
-	.splice_read    = ll_file_splice_read,
+	.splice_read    = generic_file_splice_read,
 	.fsync	  = ll_fsync,
 	.flush	  = ll_flush,
 	.flock	  = ll_file_noflock,
@@ -3213,10 +3268,7 @@ const struct inode_operations ll_file_inode_operations = {
 	.setattr	= ll_setattr,
 	.getattr	= ll_getattr,
 	.permission	= ll_inode_permission,
-	.setxattr	= ll_setxattr,
-	.getxattr	= ll_getxattr,
 	.listxattr	= ll_listxattr,
-	.removexattr	= ll_removexattr,
 	.fiemap		= ll_fiemap,
 	.get_acl	= ll_get_acl,
 };
@@ -3251,7 +3303,6 @@ void *ll_iocontrol_register(llioc_callback_t cb, int count, unsigned int *cmd)
 	if (!in_data)
 		return NULL;
 
-	memset(in_data, 0, sizeof(*in_data));
 	in_data->iocd_size = size;
 	in_data->iocd_cb = cb;
 	in_data->iocd_count = count;
@@ -3389,7 +3440,7 @@ static int ll_layout_fetch(struct inode *inode, struct ldlm_lock *lock)
 		goto out;
 	}
 
-	lmmsize = body->eadatasize;
+	lmmsize = body->mbo_eadatasize;
 	if (lmmsize == 0) /* empty layout */ {
 		rc = 0;
 		goto out;
@@ -3447,7 +3498,7 @@ static int ll_layout_lock_set(struct lustre_handle *lockh, enum ldlm_mode mode,
 		   PFID(&lli->lli_fid), inode, reconf);
 
 	/* in case this is a caching lock and reinstate with new inode */
-	md_set_lock_data(sbi->ll_md_exp, &lockh->cookie, inode, NULL);
+	md_set_lock_data(sbi->ll_md_exp, lockh, inode, NULL);
 
 	lock_res_and_lock(lock);
 	lvb_ready = ldlm_is_lvb_ready(lock);
@@ -3557,8 +3608,8 @@ int ll_layout_refresh(struct inode *inode, __u32 *gen)
 	struct ldlm_enqueue_info einfo = {
 		.ei_type = LDLM_IBITS,
 		.ei_mode = LCK_CR,
-		.ei_cb_bl = ll_md_blocking_ast,
-		.ei_cb_cp = ldlm_completion_ast,
+		.ei_cb_bl = &ll_md_blocking_ast,
+		.ei_cb_cp = &ldlm_completion_ast,
 	};
 	int rc;
 
@@ -3604,8 +3655,7 @@ again:
 			  ll_get_fsname(inode->i_sb, NULL, 0),
 			  PFID(&lli->lli_fid), inode);
 
-	rc = md_enqueue(sbi->ll_md_exp, &einfo, &it, op_data, &lockh,
-			NULL, 0, NULL, 0);
+	rc = md_enqueue(sbi->ll_md_exp, &einfo, NULL, &it, op_data, &lockh, 0);
 	ptlrpc_req_finished(it.it_request);
 	it.it_request = NULL;
 
diff --git a/drivers/staging/lustre/lustre/llite/glimpse.c b/drivers/staging/lustre/lustre/llite/glimpse.c
index 92004a05f9ee..22507b9c6d69 100644
--- a/drivers/staging/lustre/lustre/llite/glimpse.c
+++ b/drivers/staging/lustre/lustre/llite/glimpse.c
@@ -42,7 +42,6 @@
 #include "../include/obd.h"
 
 #include "../include/lustre_dlm.h"
-#include "../include/lustre_lite.h"
 #include "../include/lustre_mdc.h"
 #include <linux/pagemap.h>
 #include <linux/file.h>
diff --git a/drivers/staging/lustre/lustre/llite/lcommon_cl.c b/drivers/staging/lustre/lustre/llite/lcommon_cl.c
index 396e4e4f0715..084330d08f7a 100644
--- a/drivers/staging/lustre/lustre/llite/lcommon_cl.c
+++ b/drivers/staging/lustre/lustre/llite/lcommon_cl.c
@@ -49,7 +49,6 @@
 #include "../include/obd.h"
 #include "../include/obd_support.h"
 #include "../include/lustre_fid.h"
-#include "../include/lustre_lite.h"
 #include "../include/lustre_dlm.h"
 #include "../include/lustre_ver.h"
 #include "../include/lustre_mdc.h"
@@ -100,6 +99,7 @@ int cl_setattr_ost(struct inode *inode, const struct iattr *attr)
 	io->u.ci_setattr.sa_attr.lvb_ctime = LTIME_S(attr->ia_ctime);
 	io->u.ci_setattr.sa_attr.lvb_size = attr->ia_size;
 	io->u.ci_setattr.sa_valid = attr->ia_valid;
+	io->u.ci_setattr.sa_parent_fid = ll_inode2fid(inode);
 
 again:
 	if (cl_io_init(env, io, CIT_SETATTR, io->ci_obj) == 0) {
@@ -154,7 +154,7 @@ int cl_file_inode_init(struct inode *inode, struct lustre_md *md)
 	int result = 0;
 	int refcheck;
 
-	LASSERT(md->body->valid & OBD_MD_FLID);
+	LASSERT(md->body->mbo_valid & OBD_MD_FLID);
 	LASSERT(S_ISREG(inode->i_mode));
 
 	env = cl_env_get(&refcheck);
diff --git a/drivers/staging/lustre/lustre/llite/lcommon_misc.c b/drivers/staging/lustre/lustre/llite/lcommon_misc.c
index f6be105eeef7..fb346c12dad2 100644
--- a/drivers/staging/lustre/lustre/llite/lcommon_misc.c
+++ b/drivers/staging/lustre/lustre/llite/lcommon_misc.c
@@ -38,7 +38,6 @@
 #include "../include/obd.h"
 #include "../include/cl_object.h"
 
-#include "../include/lustre_lite.h"
 #include "llite_internal.h"
 
 /* Initialize the default and maximum LOV EA and cookie sizes.  This allows
diff --git a/drivers/staging/lustre/lustre/llite/llite_close.c b/drivers/staging/lustre/lustre/llite/llite_close.c
index 2326b40a0870..8644631bf2ba 100644
--- a/drivers/staging/lustre/lustre/llite/llite_close.c
+++ b/drivers/staging/lustre/lustre/llite/llite_close.c
@@ -38,7 +38,6 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/lustre_lite.h"
 #include "llite_internal.h"
 
 /** records that a write is in flight */
diff --git a/drivers/staging/lustre/lustre/llite/llite_internal.h b/drivers/staging/lustre/lustre/llite/llite_internal.h
index 4d6d589a1677..4bc551279aa4 100644
--- a/drivers/staging/lustre/lustre/llite/llite_internal.h
+++ b/drivers/staging/lustre/lustre/llite/llite_internal.h
@@ -36,14 +36,21 @@
 #include "../include/lustre_ver.h"
 #include "../include/lustre_disk.h"	/* for s2sbi */
 #include "../include/lustre_eacl.h"
+#include "../include/lustre_linkea.h"
 
 /* for struct cl_lock_descr and struct cl_io */
+#include "../include/lustre_patchless_compat.h"
+#include "../include/lustre_compat.h"
 #include "../include/cl_object.h"
+#include "../include/lustre_lmv.h"
 #include "../include/lustre_mdc.h"
 #include "../include/lustre_intent.h"
 #include <linux/compat.h>
+#include <linux/namei.h>
+#include <linux/xattr.h>
 #include <linux/posix_acl_xattr.h>
 #include "vvp_internal.h"
+#include "range_lock.h"
 
 #ifndef FMODE_EXEC
 #define FMODE_EXEC 0
@@ -57,6 +64,9 @@
 #define LL_DIR_END_OFF	  0x7fffffffffffffffULL
 #define LL_DIR_END_OFF_32BIT    0x7fffffffUL
 
+/* 4UL * 1024 * 1024 */
+#define LL_MAX_BLKSIZE_BITS 22
+
 #define LL_IT2STR(it) ((it) ? ldlm_it2str((it)->it_op) : "0")
 #define LUSTRE_FPRIVATE(file) ((file)->private_data)
 
@@ -116,9 +126,7 @@ struct ll_inode_info {
 
 	/* identifying fields for both metadata and data stacks. */
 	struct lu_fid		   lli_fid;
-	/* Parent fid for accessing default stripe data on parent directory
-	 * for allocating OST objects after a mknod() and later open-by-FID.
-	 */
+	/* master inode fid for stripe directory */
 	struct lu_fid		   lli_pfid;
 
 	struct list_head	      lli_close_list;
@@ -156,7 +164,7 @@ struct ll_inode_info {
 		/* for directory */
 		struct {
 			/* serialize normal readdir and statahead-readdir. */
-			struct mutex			d_readdir_mutex;
+			struct mutex			lli_readdir_mutex;
 
 			/* metadata statahead */
 			/* since parent-child threads can share the same @file
@@ -164,27 +172,39 @@ struct ll_inode_info {
 			 * case of parent exit before child -- it is me should
 			 * cleanup the dir readahead.
 			 */
-			void			   *d_opendir_key;
-			struct ll_statahead_info       *d_sai;
+			void			       *lli_opendir_key;
+			struct ll_statahead_info       *lli_sai;
 			/* protect statahead stuff. */
-			spinlock_t			d_sa_lock;
+			spinlock_t			lli_sa_lock;
 			/* "opendir_pid" is the token when lookup/revalidate
 			 * -- I am the owner of dir statahead.
 			 */
-			pid_t			   d_opendir_pid;
-		} d;
-
-#define lli_readdir_mutex       u.d.d_readdir_mutex
-#define lli_opendir_key	 u.d.d_opendir_key
-#define lli_sai		 u.d.d_sai
-#define lli_sa_lock	     u.d.d_sa_lock
-#define lli_opendir_pid	 u.d.d_opendir_pid
+			pid_t				lli_opendir_pid;
+			/* stat will try to access statahead entries or start
+			 * statahead if this flag is set, and this flag will be
+			 * set upon dir open, and cleared when dir is closed,
+			 * statahead hit ratio is too low, or start statahead
+			 * thread failed.
+			 */
+			unsigned int			lli_sa_enabled:1;
+			/* generation for statahead */
+			unsigned int			lli_sa_generation;
+			/* directory stripe information */
+			struct lmv_stripe_md	       *lli_lsm_md;
+			/* default directory stripe offset.  This is extracted
+			 * from the "dmv" xattr in order to decide which MDT to
+			 * create a subdirectory on.  The MDS itself fetches
+			 * "dmv" and gets the rest of the default layout itself
+			 * (count, hash, etc).
+			 */
+			__u32				lli_def_stripe_offset;
+		};
 
 		/* for non-directory */
 		struct {
-			struct mutex			f_size_mutex;
-			char				*f_symlink_name;
-			__u64				f_maxbytes;
+			struct mutex			lli_size_mutex;
+			char			       *lli_symlink_name;
+			__u64				lli_maxbytes;
 			/*
 			 * struct rw_semaphore {
 			 *    signed long	count;     // align d.d_def_acl
@@ -192,16 +212,16 @@ struct ll_inode_info {
 			 *    struct list_head wait_list;
 			 * }
 			 */
-			struct rw_semaphore		f_trunc_sem;
-			struct mutex			f_write_mutex;
+			struct rw_semaphore		lli_trunc_sem;
+			struct range_lock_tree		lli_write_tree;
 
-			struct rw_semaphore		f_glimpse_sem;
-			unsigned long			f_glimpse_time;
-			struct list_head			f_agl_list;
-			__u64				f_agl_index;
+			struct rw_semaphore		lli_glimpse_sem;
+			unsigned long			lli_glimpse_time;
+			struct list_head		lli_agl_list;
+			__u64				lli_agl_index;
 
 			/* for writepage() only to communicate to fsync */
-			int				f_async_rc;
+			int				lli_async_rc;
 
 			/*
 			 * whenever a process try to read/write the file, the
@@ -211,22 +231,9 @@ struct ll_inode_info {
 			 * so the read/write statistics for jobid will not be
 			 * accurate if the file is shared by different jobs.
 			 */
-			char		     f_jobid[JOBSTATS_JOBID_SIZE];
-		} f;
-
-#define lli_size_mutex          u.f.f_size_mutex
-#define lli_symlink_name	u.f.f_symlink_name
-#define lli_maxbytes	    u.f.f_maxbytes
-#define lli_trunc_sem	   u.f.f_trunc_sem
-#define lli_write_mutex	 u.f.f_write_mutex
-#define lli_glimpse_sem		u.f.f_glimpse_sem
-#define lli_glimpse_time	u.f.f_glimpse_time
-#define lli_agl_list		u.f.f_agl_list
-#define lli_agl_index		u.f.f_agl_index
-#define lli_async_rc		u.f.f_async_rc
-#define lli_jobid		u.f.f_jobid
-
-	} u;
+			char				lli_jobid[LUSTRE_JOBID_SIZE];
+		};
+	};
 
 	/* XXX: For following frequent used members, although they maybe special
 	 *      used for non-directory object, it is some time-wasting to check
@@ -401,12 +408,13 @@ enum stats_track_type {
 #define LL_SBI_LAYOUT_LOCK    0x20000 /* layout lock support */
 #define LL_SBI_USER_FID2PATH  0x40000 /* allow fid2path by unprivileged users */
 #define LL_SBI_XATTR_CACHE    0x80000 /* support for xattr cache */
+#define LL_SBI_NOROOTSQUASH	0x100000 /* do not apply root squash */
 
 #define LL_SBI_FLAGS {	\
 	"nolck",	\
 	"checksum",	\
 	"flock",	\
-	"xattr",	\
+	"user_xattr",	\
 	"acl",		\
 	"???",		\
 	"???",		\
@@ -422,9 +430,27 @@ enum stats_track_type {
 	"verbose",	\
 	"layout",	\
 	"user_fid2path",\
-	"xattr",	\
+	"xattr_cache",	\
+	"norootsquash",	\
 }
 
+/*
+ * This is embedded into llite super-blocks to keep track of connect
+ * flags (capabilities) supported by all imports given mount is
+ * connected to.
+ */
+struct lustre_client_ocd {
+	/*
+	 * This is conjunction of connect_flags across all imports
+	 * (LOVs) this mount is connected to. This field is updated by
+	 * cl_ocd_update() under ->lco_lock.
+	 */
+	__u64			 lco_flags;
+	struct mutex		 lco_lock;
+	struct obd_export	*lco_md_exp;
+	struct obd_export	*lco_dt_exp;
+};
+
 struct ll_sb_info {
 	/* this protects pglist and ra_info.  It isn't safe to
 	 * grab from interrupt contexts
@@ -461,7 +487,7 @@ struct ll_sb_info {
 	unsigned int	      ll_namelen;
 	struct file_operations   *ll_fop;
 
-	unsigned int	      ll_md_brw_size; /* used by readdir */
+	unsigned int		  ll_md_brw_pages; /* readdir pages per RPC */
 
 	struct lu_site	   *ll_site;
 	struct cl_device	 *ll_cl;
@@ -484,11 +510,17 @@ struct ll_sb_info {
 	atomic_t		  ll_sa_wrong;   /* statahead thread stopped for
 						  * low hit ratio
 						  */
+	atomic_t		ll_sa_running;	/* running statahead thread
+						 * count
+						 */
 	atomic_t		  ll_agl_total;  /* AGL thread started count */
 
 	dev_t			  ll_sdev_orig; /* save s_dev before assign for
 						 * clustered nfs
 						 */
+	/* root squash */
+	struct root_squash_info	  ll_squash;
+
 	__kernel_fsid_t		  ll_fsid;
 	struct kobject		 ll_kobj; /* sysfs object */
 	struct super_block	*ll_sb; /* struct super_block (for sysfs code)*/
@@ -643,25 +675,66 @@ void ll_rw_stats_tally(struct ll_sb_info *sbi, pid_t pid,
 		       struct ll_file_data *file, loff_t pos,
 		       size_t count, int rw);
 
+enum {
+	LPROC_LL_DIRTY_HITS,
+	LPROC_LL_DIRTY_MISSES,
+	LPROC_LL_READ_BYTES,
+	LPROC_LL_WRITE_BYTES,
+	LPROC_LL_BRW_READ,
+	LPROC_LL_BRW_WRITE,
+	LPROC_LL_OSC_READ,
+	LPROC_LL_OSC_WRITE,
+	LPROC_LL_IOCTL,
+	LPROC_LL_OPEN,
+	LPROC_LL_RELEASE,
+	LPROC_LL_MAP,
+	LPROC_LL_LLSEEK,
+	LPROC_LL_FSYNC,
+	LPROC_LL_READDIR,
+	LPROC_LL_SETATTR,
+	LPROC_LL_TRUNC,
+	LPROC_LL_FLOCK,
+	LPROC_LL_GETATTR,
+	LPROC_LL_CREATE,
+	LPROC_LL_LINK,
+	LPROC_LL_UNLINK,
+	LPROC_LL_SYMLINK,
+	LPROC_LL_MKDIR,
+	LPROC_LL_RMDIR,
+	LPROC_LL_MKNOD,
+	LPROC_LL_RENAME,
+	LPROC_LL_STAFS,
+	LPROC_LL_ALLOC_INODE,
+	LPROC_LL_SETXATTR,
+	LPROC_LL_GETXATTR,
+	LPROC_LL_GETXATTR_HITS,
+	LPROC_LL_LISTXATTR,
+	LPROC_LL_REMOVEXATTR,
+	LPROC_LL_INODE_PERM,
+	LPROC_LL_FILE_OPCODES
+};
+
 /* llite/dir.c */
-void ll_release_page(struct page *page, int remove);
 extern const struct file_operations ll_dir_operations;
 extern const struct inode_operations ll_dir_inode_operations;
-struct page *ll_get_dir_page(struct inode *dir, __u64 hash,
-			     struct ll_dir_chain *chain);
-int ll_dir_read(struct inode *inode, struct dir_context *ctx);
-
+int ll_dir_read(struct inode *inode, __u64 *ppos, struct md_op_data *op_data,
+		struct dir_context *ctx);
 int ll_get_mdt_idx(struct inode *inode);
+int ll_get_mdt_idx_by_fid(struct ll_sb_info *sbi, const struct lu_fid *fid);
+struct page *ll_get_dir_page(struct inode *dir, struct md_op_data *op_data,
+			     __u64 offset);
+void ll_release_page(struct inode *inode, struct page *page, bool remove);
+
 /* llite/namei.c */
 extern const struct inode_operations ll_special_inode_operations;
 
-int ll_objects_destroy(struct ptlrpc_request *request,
-		       struct inode *dir);
 struct inode *ll_iget(struct super_block *sb, ino_t hash,
 		      struct lustre_md *lic);
+int ll_test_inode_by_fid(struct inode *inode, void *opaque);
 int ll_md_blocking_ast(struct ldlm_lock *, struct ldlm_lock_desc *,
 		       void *data, int flag);
 struct dentry *ll_splice_alias(struct inode *inode, struct dentry *de);
+void ll_update_times(struct ptlrpc_request *request, struct inode *inode);
 
 /* llite/rw.c */
 int ll_writepage(struct page *page, struct writeback_control *wbc);
@@ -704,7 +777,10 @@ void ll_pack_inode2opdata(struct inode *inode, struct md_op_data *op_data,
 			  struct lustre_handle *fh);
 int ll_getattr(struct vfsmount *mnt, struct dentry *de, struct kstat *stat);
 struct posix_acl *ll_get_acl(struct inode *inode, int type);
-
+int ll_migrate(struct inode *parent, struct file *file, int mdtidx,
+	       const char *name, int namelen);
+int ll_get_fid_by_name(struct inode *parent, const char *name,
+		       int namelen, struct lu_fid *fid);
 int ll_inode_permission(struct inode *inode, int mask);
 
 int ll_lov_setstripe_ea_info(struct inode *inode, struct dentry *dentry,
@@ -715,8 +791,8 @@ int ll_lov_getstripe_ea_info(struct inode *inode, const char *filename,
 			     struct ptlrpc_request **request);
 int ll_dir_setstripe(struct inode *inode, struct lov_user_md *lump,
 		     int set_default);
-int ll_dir_getstripe(struct inode *inode, struct lov_mds_md **lmmp,
-		     int *lmm_size, struct ptlrpc_request **request);
+int ll_dir_getstripe(struct inode *inode, void **lmmp, int *lmm_size,
+		     struct ptlrpc_request **request, u64 valid);
 int ll_fsync(struct file *file, loff_t start, loff_t end, int data);
 int ll_merge_attr(const struct lu_env *env, struct inode *inode);
 int ll_fid2path(struct inode *inode, void __user *arg);
@@ -748,8 +824,8 @@ int ll_setattr(struct dentry *de, struct iattr *attr);
 int ll_statfs(struct dentry *de, struct kstatfs *sfs);
 int ll_statfs_internal(struct super_block *sb, struct obd_statfs *osfs,
 		       __u64 max_age, __u32 flags);
-void ll_update_inode(struct inode *inode, struct lustre_md *md);
-void ll_read_inode2(struct inode *inode, void *opaque);
+int ll_update_inode(struct inode *inode, struct lustre_md *md);
+int ll_read_inode2(struct inode *inode, void *opaque);
 void ll_delete_inode(struct inode *inode);
 int ll_iocontrol(struct inode *inode, struct file *file,
 		 unsigned int cmd, unsigned long arg);
@@ -763,15 +839,46 @@ int ll_prep_inode(struct inode **inode, struct ptlrpc_request *req,
 int ll_obd_statfs(struct inode *inode, void __user *arg);
 int ll_get_max_mdsize(struct ll_sb_info *sbi, int *max_mdsize);
 int ll_get_default_mdsize(struct ll_sb_info *sbi, int *default_mdsize);
+int ll_set_default_mdsize(struct ll_sb_info *sbi, int default_mdsize);
 int ll_process_config(struct lustre_cfg *lcfg);
+
+enum {
+	LUSTRE_OPC_MKDIR	= 0,
+	LUSTRE_OPC_SYMLINK	= 1,
+	LUSTRE_OPC_MKNOD	= 2,
+	LUSTRE_OPC_CREATE	= 3,
+	LUSTRE_OPC_ANY		= 5,
+};
+
 struct md_op_data *ll_prep_md_op_data(struct md_op_data *op_data,
 				      struct inode *i1, struct inode *i2,
-				      const char *name, int namelen,
-				      int mode, __u32 opc, void *data);
+				      const char *name, size_t namelen,
+				      u32 mode, __u32 opc, void *data);
 void ll_finish_md_op_data(struct md_op_data *op_data);
 int ll_get_obd_name(struct inode *inode, unsigned int cmd, unsigned long arg);
 char *ll_get_fsname(struct super_block *sb, char *buf, int buflen);
+void ll_compute_rootsquash_state(struct ll_sb_info *sbi);
 void ll_open_cleanup(struct super_block *sb, struct ptlrpc_request *open_req);
+ssize_t ll_copy_user_md(const struct lov_user_md __user *md,
+			struct lov_user_md **kbuf);
+
+/* Compute expected user md size when passing in a md from user space */
+static inline ssize_t ll_lov_user_md_size(const struct lov_user_md *lum)
+{
+	switch (lum->lmm_magic) {
+	case LOV_USER_MAGIC_V1:
+		return sizeof(struct lov_user_md_v1);
+	case LOV_USER_MAGIC_V3:
+		return sizeof(struct lov_user_md_v3);
+	case LOV_USER_MAGIC_SPECIFIC:
+		if (lum->lmm_stripe_count > LOV_MAX_STRIPE_COUNT)
+			return -EINVAL;
+
+		return lov_user_md_size(lum->lmm_stripe_count,
+					LOV_USER_MAGIC_SPECIFIC);
+	}
+	return -EINVAL;
+}
 
 /* llite/llite_nfs.c */
 extern const struct export_operations lustre_export_operations;
@@ -779,6 +886,7 @@ __u32 get_uuid2int(const char *name, int len);
 void get_uuid2fsid(const char *name, int len, __kernel_fsid_t *fsid);
 struct inode *search_inode_for_lustre(struct super_block *sb,
 				      const struct lu_fid *fid);
+int ll_dir_get_parent_fid(struct inode *dir, struct lu_fid *parent_fid);
 
 /* llite/symlink.c */
 extern const struct inode_operations ll_fast_symlink_inode_operations;
@@ -800,17 +908,11 @@ void vvp_write_complete(struct vvp_object *club, struct vvp_page *page);
  */
 struct vvp_io_args {
 	/** normal/splice */
-	enum vvp_io_subtype via_io_subtype;
-
 	union {
 		struct {
 			struct kiocb      *via_iocb;
 			struct iov_iter   *via_iter;
 		} normal;
-		struct {
-			struct pipe_inode_info  *via_pipe;
-			unsigned int       via_flags;
-		} splice;
 	} u;
 };
 
@@ -838,14 +940,9 @@ static inline struct ll_thread_info *ll_env_info(const struct lu_env *env)
 	return lti;
 }
 
-static inline struct vvp_io_args *ll_env_args(const struct lu_env *env,
-					      enum vvp_io_subtype type)
+static inline struct vvp_io_args *ll_env_args(const struct lu_env *env)
 {
-	struct vvp_io_args *via = &ll_env_info(env)->lti_args;
-
-	via->via_io_subtype = type;
-
-	return via;
+	return &ll_env_info(env)->lti_args;
 }
 
 void ll_queue_done_writing(struct inode *inode, unsigned long flags);
@@ -933,12 +1030,19 @@ static inline __u64 ll_file_maxbytes(struct inode *inode)
 }
 
 /* llite/xattr.c */
-int ll_setxattr(struct dentry *dentry, struct inode *inode,
-		const char *name, const void *value, size_t size, int flags);
-ssize_t ll_getxattr(struct dentry *dentry, struct inode *inode,
-		    const char *name, void *buffer, size_t size);
+extern const struct xattr_handler *ll_xattr_handlers[];
+
+#define XATTR_USER_T		1
+#define XATTR_TRUSTED_T		2
+#define XATTR_SECURITY_T	3
+#define XATTR_ACL_ACCESS_T	4
+#define XATTR_ACL_DEFAULT_T	5
+#define XATTR_LUSTRE_T		6
+#define XATTR_OTHER_T		7
+
 ssize_t ll_listxattr(struct dentry *dentry, char *buffer, size_t size);
-int ll_removexattr(struct dentry *dentry, const char *name);
+int ll_xattr_list(struct inode *inode, const char *name, int type,
+		  void *buffer, size_t size, __u64 valid);
 
 /**
  * Common IO arguments for various VFS I/O interfaces.
@@ -964,11 +1068,10 @@ void ll_ra_stats_inc(struct inode *inode, enum ra_stat which);
 
 /* per inode struct, for dir only */
 struct ll_statahead_info {
-	struct inode	   *sai_inode;
+	struct dentry	   *sai_dentry;
 	atomic_t	    sai_refcount;   /* when access this struct, hold
 					     * refcount
 					     */
-	unsigned int	    sai_generation; /* generation for statahead */
 	unsigned int	    sai_max;	/* max ahead of lookup */
 	__u64		   sai_sent;       /* stat requests sent count */
 	__u64		   sai_replied;    /* stat requests which received
@@ -995,22 +1098,25 @@ struct ll_statahead_info {
 	unsigned int	    sai_ls_all:1,   /* "ls -al", do stat-ahead for
 					     * hidden entries
 					     */
-				sai_agl_valid:1;/* AGL is valid for the dir */
+				sai_agl_valid:1,/* AGL is valid for the dir */
+				sai_in_readpage:1;/* statahead is in readdir() */
 	wait_queue_head_t	sai_waitq;      /* stat-ahead wait queue */
 	struct ptlrpc_thread    sai_thread;     /* stat-ahead thread */
 	struct ptlrpc_thread    sai_agl_thread; /* AGL thread */
-	struct list_head	sai_entries;    /* entry list */
-	struct list_head	sai_entries_received; /* entries returned */
-	struct list_head	sai_entries_stated;   /* entries stated */
-	struct list_head	sai_entries_agl; /* AGL entries to be sent */
+	struct list_head	sai_interim_entries; /* entries which got async
+						      * stat reply, but not
+						      * instantiated
+						      */
+	struct list_head	sai_entries;	/* completed entries */
+	struct list_head	sai_agls;	/* AGLs to be sent */
 	struct list_head	sai_cache[LL_SA_CACHE_SIZE];
 	spinlock_t		sai_cache_lock[LL_SA_CACHE_SIZE];
 	atomic_t		sai_cache_count; /* entry count in cache */
 };
 
-int do_statahead_enter(struct inode *dir, struct dentry **dentry,
-		       int only_unplug);
-void ll_stop_statahead(struct inode *dir, void *key);
+int ll_statahead(struct inode *dir, struct dentry **dentry, bool unplug);
+void ll_authorize_statahead(struct inode *dir, void *key);
+void ll_deauthorize_statahead(struct inode *dir, void *key);
 
 blkcnt_t dirty_cnt(struct inode *inode);
 
@@ -1040,73 +1146,53 @@ static inline int ll_glimpse_size(struct inode *inode)
 	return rc;
 }
 
-static inline void
-ll_statahead_mark(struct inode *dir, struct dentry *dentry)
-{
-	struct ll_inode_info     *lli = ll_i2info(dir);
-	struct ll_statahead_info *sai = lli->lli_sai;
-	struct ll_dentry_data    *ldd = ll_d2d(dentry);
-
-	/* not the same process, don't mark */
-	if (lli->lli_opendir_pid != current_pid())
-		return;
-
-	LASSERT(ldd);
-	if (sai)
-		ldd->lld_sa_generation = sai->sai_generation;
-}
-
-static inline int
-d_need_statahead(struct inode *dir, struct dentry *dentryp)
+/*
+ * dentry may statahead when statahead is enabled and current process has opened
+ * parent directory, and this dentry hasn't accessed statahead cache before
+ */
+static inline bool
+dentry_may_statahead(struct inode *dir, struct dentry *dentry)
 {
 	struct ll_inode_info  *lli;
 	struct ll_dentry_data *ldd;
 
 	if (ll_i2sbi(dir)->ll_sa_max == 0)
-		return -EAGAIN;
+		return false;
 
 	lli = ll_i2info(dir);
+
+	/*
+	 * statahead is not allowed for this dir, there may be three causes:
+	 * 1. dir is not opened.
+	 * 2. statahead hit ratio is too low.
+	 * 3. previous stat started statahead thread failed.
+	 */
+	if (!lli->lli_sa_enabled)
+		return false;
+
 	/* not the same process, don't statahead */
 	if (lli->lli_opendir_pid != current_pid())
-		return -EAGAIN;
-
-	/* statahead has been stopped */
-	if (!lli->lli_opendir_key)
-		return -EAGAIN;
+		return false;
 
-	ldd = ll_d2d(dentryp);
 	/*
-	 * When stats a dentry, the system trigger more than once "revalidate"
-	 * or "lookup", for "getattr", for "getxattr", and maybe for others.
-	 * Under patchless client mode, the operation intent is not accurate,
-	 * which maybe misguide the statahead thread. For example:
-	 * The "revalidate" call for "getattr" and "getxattr" of a dentry maybe
-	 * have the same operation intent -- "IT_GETATTR".
-	 * In fact, one dentry should has only one chance to interact with the
-	 * statahead thread, otherwise the statahead windows will be confused.
+	 * When stating a dentry, kernel may trigger 'revalidate' or 'lookup'
+	 * multiple times, eg. for 'getattr', 'getxattr' and etc.
+	 * For patchless client, lookup intent is not accurate, which may
+	 * misguide statahead. For example:
+	 * The 'revalidate' call for 'getattr' and 'getxattr' of a dentry will
+	 * have the same intent -- IT_GETATTR, while one dentry should access
+	 * statahead cache once, otherwise statahead windows is messed up.
 	 * The solution is as following:
-	 * Assign "lld_sa_generation" with "sai_generation" when a dentry
-	 * "IT_GETATTR" for the first time, and the subsequent "IT_GETATTR"
-	 * will bypass interacting with statahead thread for checking:
-	 * "lld_sa_generation == lli_sai->sai_generation"
+	 * Assign 'lld_sa_generation' with 'lli_sa_generation' when a dentry
+	 * IT_GETATTR for the first time, and subsequent IT_GETATTR will
+	 * bypass interacting with statahead cache by checking
+	 * 'lld_sa_generation == lli->lli_sa_generation'.
 	 */
-	if (ldd && lli->lli_sai &&
-	    ldd->lld_sa_generation == lli->lli_sai->sai_generation)
-		return -EAGAIN;
+	ldd = ll_d2d(dentry);
+	if (ldd && ldd->lld_sa_generation == lli->lli_sa_generation)
+		return false;
 
-	return 1;
-}
-
-static inline int
-ll_statahead_enter(struct inode *dir, struct dentry **dentryp, int only_unplug)
-{
-	int ret;
-
-	ret = d_need_statahead(dir, *dentryp);
-	if (ret <= 0)
-		return ret;
-
-	return do_statahead_enter(dir, dentryp, only_unplug);
+	return true;
 }
 
 /* llite ioctl register support routine */
@@ -1213,7 +1299,7 @@ static inline void ll_set_lock_data(struct obd_export *exp, struct inode *inode,
 			CDEBUG(D_DLMTRACE, "setting l_data to inode "DFID"%p for remote lock %#llx\n",
 			       PFID(ll_inode2fid(inode)), inode,
 			       handle.cookie);
-			md_set_lock_data(exp, &handle.cookie, inode, NULL);
+			md_set_lock_data(exp, &handle, inode, NULL);
 		}
 
 		handle.cookie = it->it_lock_handle;
@@ -1221,8 +1307,7 @@ static inline void ll_set_lock_data(struct obd_export *exp, struct inode *inode,
 		CDEBUG(D_DLMTRACE, "setting l_data to inode "DFID"%p for lock %#llx\n",
 		       PFID(ll_inode2fid(inode)), inode, handle.cookie);
 
-		md_set_lock_data(exp, &handle.cookie, inode,
-				 &it->it_lock_bits);
+		md_set_lock_data(exp, &handle, inode, &it->it_lock_bits);
 		it->it_lock_set = 1;
 	}
 
@@ -1295,6 +1380,8 @@ void ll_xattr_fini(void);
 int ll_page_sync_io(const struct lu_env *env, struct cl_io *io,
 		    struct cl_page *page, enum cl_req_type crt);
 
+int ll_getparent(struct file *file, struct getparent __user *arg);
+
 /* lcommon_cl.c */
 int cl_setattr_ost(struct inode *inode, const struct iattr *attr);
 
diff --git a/drivers/staging/lustre/lustre/llite/llite_lib.c b/drivers/staging/lustre/lustre/llite/llite_lib.c
index 546063e728db..e5c62f4ce3d8 100644
--- a/drivers/staging/lustre/lustre/llite/llite_lib.c
+++ b/drivers/staging/lustre/lustre/llite/llite_lib.c
@@ -41,7 +41,7 @@
 #include <linux/types.h>
 #include <linux/mm.h>
 
-#include "../include/lustre_lite.h"
+#include "../include/lustre/lustre_ioctl.h"
 #include "../include/lustre_ha.h"
 #include "../include/lustre_dlm.h"
 #include "../include/lprocfs_status.h"
@@ -115,9 +115,16 @@ static struct ll_sb_info *ll_init_sbi(struct super_block *sb)
 	sbi->ll_sa_max = LL_SA_RPC_DEF;
 	atomic_set(&sbi->ll_sa_total, 0);
 	atomic_set(&sbi->ll_sa_wrong, 0);
+	atomic_set(&sbi->ll_sa_running, 0);
 	atomic_set(&sbi->ll_agl_total, 0);
 	sbi->ll_flags |= LL_SBI_AGL_ENABLED;
 
+	/* root squash */
+	sbi->ll_squash.rsi_uid = 0;
+	sbi->ll_squash.rsi_gid = 0;
+	INIT_LIST_HEAD(&sbi->ll_squash.rsi_nosquash_nids);
+	init_rwsem(&sbi->ll_squash.rsi_sem);
+
 	sbi->ll_sb = sb;
 
 	return sbi;
@@ -128,6 +135,8 @@ static void ll_free_sbi(struct super_block *sb)
 	struct ll_sb_info *sbi = ll_s2sbi(sb);
 
 	if (sbi->ll_cache) {
+		if (!list_empty(&sbi->ll_squash.rsi_nosquash_nids))
+			cfs_free_nidlist(&sbi->ll_squash.rsi_nosquash_nids);
 		cl_cache_decref(sbi->ll_cache);
 		sbi->ll_cache = NULL;
 	}
@@ -180,7 +189,9 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 				  OBD_CONNECT_PINGLESS |
 				  OBD_CONNECT_MAX_EASIZE |
 				  OBD_CONNECT_FLOCK_DEAD |
-				  OBD_CONNECT_DISP_STRIPE;
+				  OBD_CONNECT_DISP_STRIPE | OBD_CONNECT_LFSCK |
+				  OBD_CONNECT_OPEN_BY_FID |
+				  OBD_CONNECT_DIR_STRIPE;
 
 	if (sbi->ll_flags & LL_SBI_SOM_PREVIEW)
 		data->ocd_connect_flags |= OBD_CONNECT_SOM;
@@ -310,9 +321,9 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 		sbi->ll_flags |= LL_SBI_64BIT_HASH;
 
 	if (data->ocd_connect_flags & OBD_CONNECT_BRW_SIZE)
-		sbi->ll_md_brw_size = data->ocd_brw_size;
+		sbi->ll_md_brw_pages = data->ocd_brw_size >> PAGE_SHIFT;
 	else
-		sbi->ll_md_brw_size = PAGE_SIZE;
+		sbi->ll_md_brw_pages = 1;
 
 	if (data->ocd_connect_flags & OBD_CONNECT_LAYOUTLOCK)
 		sbi->ll_flags |= LL_SBI_LAYOUT_LOCK;
@@ -418,6 +429,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 	CDEBUG(D_SUPER, "rootfid "DFID"\n", PFID(&sbi->ll_root_fid));
 
 	sb->s_op = &lustre_super_operations;
+	sb->s_xattr = ll_xattr_handlers;
 #if THREAD_SIZE >= 8192 /*b=17630*/
 	sb->s_export_op = &lustre_export_operations;
 #endif
@@ -462,7 +474,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 	md_free_lustre_md(sbi->ll_md_exp, &lmd);
 	ptlrpc_req_finished(request);
 
-	if (!(root)) {
+	if (IS_ERR(root)) {
 		if (lmd.lsm)
 			obd_free_memmd(sbi->ll_dt_exp, &lmd.lsm);
 #ifdef CONFIG_FS_POSIX_ACL
@@ -486,11 +498,21 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 	err = obd_set_info_async(NULL, sbi->ll_dt_exp, sizeof(KEY_CHECKSUM),
 				 KEY_CHECKSUM, sizeof(checksum), &checksum,
 				 NULL);
+	if (err) {
+		CERROR("%s: Set checksum failed: rc = %d\n",
+		       sbi->ll_dt_exp->exp_obd->obd_name, err);
+		goto out_root;
+	}
 	cl_sb_init(sb);
 
 	err = obd_set_info_async(NULL, sbi->ll_dt_exp, sizeof(KEY_CACHE_SET),
 				 KEY_CACHE_SET, sizeof(*sbi->ll_cache),
 				 sbi->ll_cache, NULL);
+	if (err) {
+		CERROR("%s: Set cache_set failed: rc = %d\n",
+		       sbi->ll_dt_exp->exp_obd->obd_name, err);
+		goto out_root;
+	}
 
 	sb->s_root = d_make_root(root);
 	if (!sb->s_root) {
@@ -560,6 +582,17 @@ int ll_get_max_mdsize(struct ll_sb_info *sbi, int *lmmsize)
 	return rc;
 }
 
+/**
+ * Get the value of the default_easize parameter.
+ *
+ * \see client_obd::cl_default_mds_easize
+ *
+ * \param[in]  sbi	superblock info for this filesystem
+ * \param[out] lmmsize	pointer to storage location for value
+ *
+ * \retval 0		on success
+ * \retval negative	negated errno on failure
+ */
 int ll_get_default_mdsize(struct ll_sb_info *sbi, int *lmmsize)
 {
 	int size, rc;
@@ -573,6 +606,29 @@ int ll_get_default_mdsize(struct ll_sb_info *sbi, int *lmmsize)
 	return rc;
 }
 
+/**
+ * Set the default_easize parameter to the given value.
+ *
+ * \see client_obd::cl_default_mds_easize
+ *
+ * \param[in] sbi	superblock info for this filesystem
+ * \param[in] lmmsize	the size to set
+ *
+ * \retval 0		on success
+ * \retval negative	negated errno on failure
+ */
+int ll_set_default_mdsize(struct ll_sb_info *sbi, int lmmsize)
+{
+	if (lmmsize < sizeof(struct lov_mds_md) ||
+	    lmmsize > OBD_MAX_DEFAULT_EA_SIZE)
+		return -EINVAL;
+
+	return obd_set_info_async(NULL, sbi->ll_md_exp,
+				  sizeof(KEY_DEFAULT_EASIZE),
+				  KEY_DEFAULT_EASIZE,
+				  sizeof(int), &lmmsize, NULL);
+}
+
 static void client_common_put_super(struct super_block *sb)
 {
 	struct ll_sb_info *sbi = ll_s2sbi(sb);
@@ -608,6 +664,12 @@ void ll_kill_super(struct super_block *sb)
 	if (sbi) {
 		sb->s_dev = sbi->ll_sdev_orig;
 		sbi->ll_umounting = 1;
+
+		/* wait running statahead threads to quit */
+		while (atomic_read(&sbi->ll_sa_running) > 0) {
+			set_current_state(TASK_UNINTERRUPTIBLE);
+			schedule_timeout(msecs_to_jiffies(MSEC_PER_SEC >> 3));
+		}
 	}
 }
 
@@ -647,7 +709,8 @@ static int ll_options(char *options, int *flags)
 			*flags |= tmp;
 			goto next;
 		}
-		tmp = ll_set_opt("noflock", s1, LL_SBI_FLOCK|LL_SBI_LOCALFLOCK);
+		tmp = ll_set_opt("noflock", s1,
+				 LL_SBI_FLOCK | LL_SBI_LOCALFLOCK);
 		if (tmp) {
 			*flags &= ~tmp;
 			goto next;
@@ -772,11 +835,13 @@ void ll_lli_init(struct ll_inode_info *lli)
 		lli->lli_sai = NULL;
 		spin_lock_init(&lli->lli_sa_lock);
 		lli->lli_opendir_pid = 0;
+		lli->lli_sa_enabled = 0;
+		lli->lli_def_stripe_offset = -1;
 	} else {
 		mutex_init(&lli->lli_size_mutex);
 		lli->lli_symlink_name = NULL;
 		init_rwsem(&lli->lli_trunc_sem);
-		mutex_init(&lli->lli_write_mutex);
+		range_lock_tree_init(&lli->lli_write_tree);
 		init_rwsem(&lli->lli_glimpse_sem);
 		lli->lli_glimpse_time = 0;
 		INIT_LIST_HEAD(&lli->lli_agl_list);
@@ -896,7 +961,8 @@ void ll_put_super(struct super_block *sb)
 	struct lustre_sb_info *lsi = s2lsi(sb);
 	struct ll_sb_info *sbi = ll_s2sbi(sb);
 	char *profilenm = get_profile_name(sb);
-	int ccc_count, next, force = 1, rc = 0;
+	int next, force = 1, rc = 0;
+	long ccc_count;
 
 	CDEBUG(D_VFSTRACE, "VFS Op: sb %p - %s\n", sb, profilenm);
 
@@ -917,13 +983,13 @@ void ll_put_super(struct super_block *sb)
 		struct l_wait_info lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP, NULL);
 
 		rc = l_wait_event(sbi->ll_cache->ccc_unstable_waitq,
-				  !atomic_read(&sbi->ll_cache->ccc_unstable_nr),
+				  !atomic_long_read(&sbi->ll_cache->ccc_unstable_nr),
 				  &lwi);
 	}
 
-	ccc_count = atomic_read(&sbi->ll_cache->ccc_unstable_nr);
+	ccc_count = atomic_long_read(&sbi->ll_cache->ccc_unstable_nr);
 	if (!force && rc != -EINTR)
-		LASSERTF(!ccc_count, "count: %i\n", ccc_count);
+		LASSERTF(!ccc_count, "count: %li\n", ccc_count);
 
 	/* We need to set force before the lov_disconnect in
 	 * lustre_common_put_super, since l_d cleans up osc's as well.
@@ -991,6 +1057,206 @@ struct inode *ll_inode_from_resource_lock(struct ldlm_lock *lock)
 	return inode;
 }
 
+static void ll_dir_clear_lsm_md(struct inode *inode)
+{
+	struct ll_inode_info *lli = ll_i2info(inode);
+
+	LASSERT(S_ISDIR(inode->i_mode));
+
+	if (lli->lli_lsm_md) {
+		lmv_free_memmd(lli->lli_lsm_md);
+		lli->lli_lsm_md = NULL;
+	}
+}
+
+static struct inode *ll_iget_anon_dir(struct super_block *sb,
+				      const struct lu_fid *fid,
+				      struct lustre_md *md)
+{
+	struct ll_sb_info *sbi = ll_s2sbi(sb);
+	struct mdt_body *body = md->body;
+	struct inode *inode;
+	ino_t ino;
+
+	ino = cl_fid_build_ino(fid, sbi->ll_flags & LL_SBI_32BIT_API);
+	inode = iget_locked(sb, ino);
+	if (!inode) {
+		CERROR("%s: failed get simple inode "DFID": rc = -ENOENT\n",
+		       ll_get_fsname(sb, NULL, 0), PFID(fid));
+		return ERR_PTR(-ENOENT);
+	}
+
+	if (inode->i_state & I_NEW) {
+		struct ll_inode_info *lli = ll_i2info(inode);
+		struct lmv_stripe_md *lsm = md->lmv;
+
+		inode->i_mode = (inode->i_mode & ~S_IFMT) |
+				(body->mbo_mode & S_IFMT);
+		LASSERTF(S_ISDIR(inode->i_mode), "Not slave inode "DFID"\n",
+			 PFID(fid));
+
+		LTIME_S(inode->i_mtime) = 0;
+		LTIME_S(inode->i_atime) = 0;
+		LTIME_S(inode->i_ctime) = 0;
+		inode->i_rdev = 0;
+
+		inode->i_op = &ll_dir_inode_operations;
+		inode->i_fop = &ll_dir_operations;
+		lli->lli_fid = *fid;
+		ll_lli_init(lli);
+
+		LASSERT(lsm);
+		/* master object FID */
+		lli->lli_pfid = body->mbo_fid1;
+		CDEBUG(D_INODE, "lli %p slave "DFID" master "DFID"\n",
+		       lli, PFID(fid), PFID(&lli->lli_pfid));
+		unlock_new_inode(inode);
+	}
+
+	return inode;
+}
+
+static int ll_init_lsm_md(struct inode *inode, struct lustre_md *md)
+{
+	struct lmv_stripe_md *lsm = md->lmv;
+	struct lu_fid *fid;
+	int i;
+
+	LASSERT(lsm);
+	/*
+	 * XXX sigh, this lsm_root initialization should be in
+	 * LMV layer, but it needs ll_iget right now, so we
+	 * put this here right now.
+	 */
+	for (i = 0; i < lsm->lsm_md_stripe_count; i++) {
+		fid = &lsm->lsm_md_oinfo[i].lmo_fid;
+		LASSERT(!lsm->lsm_md_oinfo[i].lmo_root);
+		/* Unfortunately ll_iget will call ll_update_inode,
+		 * where the initialization of slave inode is slightly
+		 * different, so it reset lsm_md to NULL to avoid
+		 * initializing lsm for slave inode.
+		 */
+		/* For migrating inode, master stripe and master object will
+		 * be same, so we only need assign this inode
+		 */
+		if (lsm->lsm_md_hash_type & LMV_HASH_FLAG_MIGRATION && !i)
+			lsm->lsm_md_oinfo[i].lmo_root = inode;
+		else
+			lsm->lsm_md_oinfo[i].lmo_root =
+				ll_iget_anon_dir(inode->i_sb, fid, md);
+		if (IS_ERR(lsm->lsm_md_oinfo[i].lmo_root)) {
+			int rc = PTR_ERR(lsm->lsm_md_oinfo[i].lmo_root);
+
+			lsm->lsm_md_oinfo[i].lmo_root = NULL;
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+static inline int lli_lsm_md_eq(const struct lmv_stripe_md *lsm_md1,
+				const struct lmv_stripe_md *lsm_md2)
+{
+	return lsm_md1->lsm_md_magic == lsm_md2->lsm_md_magic &&
+	       lsm_md1->lsm_md_stripe_count == lsm_md2->lsm_md_stripe_count &&
+	       lsm_md1->lsm_md_master_mdt_index ==
+			lsm_md2->lsm_md_master_mdt_index &&
+	       lsm_md1->lsm_md_hash_type == lsm_md2->lsm_md_hash_type &&
+	       lsm_md1->lsm_md_layout_version ==
+			lsm_md2->lsm_md_layout_version &&
+	       !strcmp(lsm_md1->lsm_md_pool_name,
+		       lsm_md2->lsm_md_pool_name);
+}
+
+static int ll_update_lsm_md(struct inode *inode, struct lustre_md *md)
+{
+	struct ll_inode_info *lli = ll_i2info(inode);
+	struct lmv_stripe_md *lsm = md->lmv;
+	int rc;
+
+	LASSERT(S_ISDIR(inode->i_mode));
+	CDEBUG(D_INODE, "update lsm %p of "DFID"\n", lli->lli_lsm_md,
+	       PFID(ll_inode2fid(inode)));
+
+	/* no striped information from request. */
+	if (!lsm) {
+		if (!lli->lli_lsm_md) {
+			return 0;
+		} else if (lli->lli_lsm_md->lsm_md_hash_type &
+			   LMV_HASH_FLAG_MIGRATION) {
+			/*
+			 * migration is done, the temporay MIGRATE layout has
+			 * been removed
+			 */
+			CDEBUG(D_INODE, DFID" finish migration.\n",
+			       PFID(ll_inode2fid(inode)));
+			lmv_free_memmd(lli->lli_lsm_md);
+			lli->lli_lsm_md = NULL;
+			return 0;
+		} else {
+			/*
+			 * The lustre_md from req does not include stripeEA,
+			 * see ll_md_setattr
+			 */
+			return 0;
+		}
+	}
+
+	/* set the directory layout */
+	if (!lli->lli_lsm_md) {
+		rc = ll_init_lsm_md(inode, md);
+		if (rc)
+			return rc;
+
+		lli->lli_lsm_md = lsm;
+		/*
+		 * set lsm_md to NULL, so the following free lustre_md
+		 * will not free this lsm
+		 */
+		md->lmv = NULL;
+		CDEBUG(D_INODE, "Set lsm %p magic %x to "DFID"\n", lsm,
+		       lsm->lsm_md_magic, PFID(ll_inode2fid(inode)));
+		return 0;
+	}
+
+	/* Compare the old and new stripe information */
+	if (!lsm_md_eq(lli->lli_lsm_md, lsm)) {
+		struct lmv_stripe_md *old_lsm = lli->lli_lsm_md;
+		int idx;
+
+		CERROR("%s: inode "DFID"(%p)'s lmv layout mismatch (%p)/(%p) magic:0x%x/0x%x stripe count: %d/%d master_mdt: %d/%d hash_type:0x%x/0x%x layout: 0x%x/0x%x pool:%s/%s\n",
+		       ll_get_fsname(inode->i_sb, NULL, 0), PFID(&lli->lli_fid),
+		       inode, lsm, old_lsm,
+		       lsm->lsm_md_magic, old_lsm->lsm_md_magic,
+		       lsm->lsm_md_stripe_count,
+		       old_lsm->lsm_md_stripe_count,
+		       lsm->lsm_md_master_mdt_index,
+		       old_lsm->lsm_md_master_mdt_index,
+		       lsm->lsm_md_hash_type, old_lsm->lsm_md_hash_type,
+		       lsm->lsm_md_layout_version,
+		       old_lsm->lsm_md_layout_version,
+		       lsm->lsm_md_pool_name,
+		       old_lsm->lsm_md_pool_name);
+
+		for (idx = 0; idx < old_lsm->lsm_md_stripe_count; idx++) {
+			CERROR("%s: sub FIDs in old lsm idx %d, old: "DFID"\n",
+			       ll_get_fsname(inode->i_sb, NULL, 0), idx,
+			       PFID(&old_lsm->lsm_md_oinfo[idx].lmo_fid));
+		}
+
+		for (idx = 0; idx < lsm->lsm_md_stripe_count; idx++) {
+			CERROR("%s: sub FIDs in new lsm idx %d, new: "DFID"\n",
+			       ll_get_fsname(inode->i_sb, NULL, 0), idx,
+			       PFID(&lsm->lsm_md_oinfo[idx].lmo_fid));
+		}
+
+		return -EIO;
+	}
+
+	return 0;
+}
+
 void ll_clear_inode(struct inode *inode)
 {
 	struct ll_inode_info *lli = ll_i2info(inode);
@@ -1031,14 +1297,15 @@ void ll_clear_inode(struct inode *inode)
 
 #ifdef CONFIG_FS_POSIX_ACL
 	if (lli->lli_posix_acl) {
-		LASSERT(atomic_read(&lli->lli_posix_acl->a_refcount) == 1);
 		posix_acl_release(lli->lli_posix_acl);
 		lli->lli_posix_acl = NULL;
 	}
 #endif
 	lli->lli_inode_magic = LLI_INODE_DEAD;
 
-	if (!S_ISDIR(inode->i_mode))
+	if (S_ISDIR(inode->i_mode))
+		ll_dir_clear_lsm_md(inode);
+	if (S_ISREG(inode->i_mode) && !is_bad_inode(inode))
 		LASSERT(list_empty(&lli->lli_agl_list));
 
 	/*
@@ -1103,10 +1370,10 @@ static int ll_md_setattr(struct dentry *dentry, struct md_op_data *op_data,
 	op_data->op_attr.ia_valid = ia_valid;
 
 	/* Extract epoch data if obtained. */
-	op_data->op_handle = md.body->handle;
-	op_data->op_ioepoch = md.body->ioepoch;
+	op_data->op_handle = md.body->mbo_handle;
+	op_data->op_ioepoch = md.body->mbo_ioepoch;
 
-	ll_update_inode(inode, &md);
+	rc = ll_update_inode(inode, &md);
 	ptlrpc_req_finished(request);
 
 	return rc;
@@ -1138,8 +1405,8 @@ static int ll_setattr_done_writing(struct inode *inode,
 		rc = ll_som_update(inode, op_data);
 	else if (rc) {
 		CERROR("%s: inode "DFID" mdc truncate failed: rc = %d\n",
-		      ll_i2sbi(inode)->ll_md_exp->exp_obd->obd_name,
-		      PFID(ll_inode2fid(inode)), rc);
+		       ll_i2sbi(inode)->ll_md_exp->exp_obd->obd_name,
+		       PFID(ll_inode2fid(inode)), rc);
 	}
 	return rc;
 }
@@ -1192,7 +1459,7 @@ int ll_setattr_raw(struct dentry *dentry, struct iattr *attr, bool hsm_import)
 		attr->ia_valid |= ATTR_MTIME | ATTR_CTIME;
 	}
 
-	/* POSIX: check before ATTR_*TIME_SET set (from inode_change_ok) */
+	/* POSIX: check before ATTR_*TIME_SET set (from setattr_prepare) */
 	if (attr->ia_valid & TIMES_SET_FLAGS) {
 		if ((!uid_eq(current_fsuid(), inode->i_uid)) &&
 		    !capable(CFS_CAP_FOWNER))
@@ -1331,14 +1598,14 @@ int ll_setattr(struct dentry *de, struct iattr *attr)
 {
 	int mode = d_inode(de)->i_mode;
 
-	if ((attr->ia_valid & (ATTR_CTIME|ATTR_SIZE|ATTR_MODE)) ==
-			      (ATTR_CTIME|ATTR_SIZE|ATTR_MODE))
+	if ((attr->ia_valid & (ATTR_CTIME | ATTR_SIZE | ATTR_MODE)) ==
+			      (ATTR_CTIME | ATTR_SIZE | ATTR_MODE))
 		attr->ia_valid |= MDS_OPEN_OWNEROVERRIDE;
 
-	if (((attr->ia_valid & (ATTR_MODE|ATTR_FORCE|ATTR_SIZE)) ==
-			       (ATTR_SIZE|ATTR_MODE)) &&
+	if (((attr->ia_valid & (ATTR_MODE | ATTR_FORCE | ATTR_SIZE)) ==
+			       (ATTR_SIZE | ATTR_MODE)) &&
 	    (((mode & S_ISUID) && !(attr->ia_mode & S_ISUID)) ||
-	     (((mode & (S_ISGID|S_IXGRP)) == (S_ISGID|S_IXGRP)) &&
+	     (((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) &&
 	      !(attr->ia_mode & S_ISGID))))
 		attr->ia_valid |= ATTR_FORCE;
 
@@ -1349,7 +1616,7 @@ int ll_setattr(struct dentry *de, struct iattr *attr)
 		attr->ia_valid |= ATTR_KILL_SUID;
 
 	if ((attr->ia_valid & ATTR_MODE) &&
-	    ((mode & (S_ISGID|S_IXGRP)) == (S_ISGID|S_IXGRP)) &&
+	    ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) &&
 	    !(attr->ia_mode & S_ISGID) &&
 	    !(attr->ia_valid & ATTR_KILL_SGID))
 		attr->ia_valid |= ATTR_KILL_SGID;
@@ -1465,14 +1732,14 @@ void ll_inode_size_unlock(struct inode *inode)
 	mutex_unlock(&lli->lli_size_mutex);
 }
 
-void ll_update_inode(struct inode *inode, struct lustre_md *md)
+int ll_update_inode(struct inode *inode, struct lustre_md *md)
 {
 	struct ll_inode_info *lli = ll_i2info(inode);
 	struct mdt_body *body = md->body;
 	struct lov_stripe_md *lsm = md->lsm;
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 
-	LASSERT((lsm != NULL) == ((body->valid & OBD_MD_FLEASIZE) != 0));
+	LASSERT((lsm != NULL) == ((body->mbo_valid & OBD_MD_FLEASIZE) != 0));
 	if (lsm) {
 		if (!lli->lli_has_smd &&
 		    !(sbi->ll_flags & LL_SBI_LAYOUT_LOCK))
@@ -1483,8 +1750,16 @@ void ll_update_inode(struct inode *inode, struct lustre_md *md)
 			lli->lli_maxbytes = MAX_LFS_FILESIZE;
 	}
 
+	if (S_ISDIR(inode->i_mode)) {
+		int rc;
+
+		rc = ll_update_lsm_md(inode, md);
+		if (rc)
+			return rc;
+	}
+
 #ifdef CONFIG_FS_POSIX_ACL
-	if (body->valid & OBD_MD_FLACL) {
+	if (body->mbo_valid & OBD_MD_FLACL) {
 		spin_lock(&lli->lli_lock);
 		if (lli->lli_posix_acl)
 			posix_acl_release(lli->lli_posix_acl);
@@ -1492,65 +1767,67 @@ void ll_update_inode(struct inode *inode, struct lustre_md *md)
 		spin_unlock(&lli->lli_lock);
 	}
 #endif
-	inode->i_ino = cl_fid_build_ino(&body->fid1,
+	inode->i_ino = cl_fid_build_ino(&body->mbo_fid1,
 					sbi->ll_flags & LL_SBI_32BIT_API);
-	inode->i_generation = cl_fid_build_gen(&body->fid1);
+	inode->i_generation = cl_fid_build_gen(&body->mbo_fid1);
 
-	if (body->valid & OBD_MD_FLATIME) {
-		if (body->atime > LTIME_S(inode->i_atime))
-			LTIME_S(inode->i_atime) = body->atime;
-		lli->lli_atime = body->atime;
+	if (body->mbo_valid & OBD_MD_FLATIME) {
+		if (body->mbo_atime > LTIME_S(inode->i_atime))
+			LTIME_S(inode->i_atime) = body->mbo_atime;
+		lli->lli_atime = body->mbo_atime;
 	}
-	if (body->valid & OBD_MD_FLMTIME) {
-		if (body->mtime > LTIME_S(inode->i_mtime)) {
+	if (body->mbo_valid & OBD_MD_FLMTIME) {
+		if (body->mbo_mtime > LTIME_S(inode->i_mtime)) {
 			CDEBUG(D_INODE, "setting ino %lu mtime from %lu to %llu\n",
 			       inode->i_ino, LTIME_S(inode->i_mtime),
-			       body->mtime);
-			LTIME_S(inode->i_mtime) = body->mtime;
+			       body->mbo_mtime);
+			LTIME_S(inode->i_mtime) = body->mbo_mtime;
 		}
-		lli->lli_mtime = body->mtime;
-	}
-	if (body->valid & OBD_MD_FLCTIME) {
-		if (body->ctime > LTIME_S(inode->i_ctime))
-			LTIME_S(inode->i_ctime) = body->ctime;
-		lli->lli_ctime = body->ctime;
-	}
-	if (body->valid & OBD_MD_FLMODE)
-		inode->i_mode = (inode->i_mode & S_IFMT)|(body->mode & ~S_IFMT);
-	if (body->valid & OBD_MD_FLTYPE)
-		inode->i_mode = (inode->i_mode & ~S_IFMT)|(body->mode & S_IFMT);
+		lli->lli_mtime = body->mbo_mtime;
+	}
+	if (body->mbo_valid & OBD_MD_FLCTIME) {
+		if (body->mbo_ctime > LTIME_S(inode->i_ctime))
+			LTIME_S(inode->i_ctime) = body->mbo_ctime;
+		lli->lli_ctime = body->mbo_ctime;
+	}
+	if (body->mbo_valid & OBD_MD_FLMODE)
+		inode->i_mode = (inode->i_mode & S_IFMT) |
+				(body->mbo_mode & ~S_IFMT);
+	if (body->mbo_valid & OBD_MD_FLTYPE)
+		inode->i_mode = (inode->i_mode & ~S_IFMT) |
+				(body->mbo_mode & S_IFMT);
 	LASSERT(inode->i_mode != 0);
 	if (S_ISREG(inode->i_mode))
 		inode->i_blkbits = min(PTLRPC_MAX_BRW_BITS + 1,
 				       LL_MAX_BLKSIZE_BITS);
 	else
 		inode->i_blkbits = inode->i_sb->s_blocksize_bits;
-	if (body->valid & OBD_MD_FLUID)
-		inode->i_uid = make_kuid(&init_user_ns, body->uid);
-	if (body->valid & OBD_MD_FLGID)
-		inode->i_gid = make_kgid(&init_user_ns, body->gid);
-	if (body->valid & OBD_MD_FLFLAGS)
-		inode->i_flags = ll_ext_to_inode_flags(body->flags);
-	if (body->valid & OBD_MD_FLNLINK)
-		set_nlink(inode, body->nlink);
-	if (body->valid & OBD_MD_FLRDEV)
-		inode->i_rdev = old_decode_dev(body->rdev);
-
-	if (body->valid & OBD_MD_FLID) {
+	if (body->mbo_valid & OBD_MD_FLUID)
+		inode->i_uid = make_kuid(&init_user_ns, body->mbo_uid);
+	if (body->mbo_valid & OBD_MD_FLGID)
+		inode->i_gid = make_kgid(&init_user_ns, body->mbo_gid);
+	if (body->mbo_valid & OBD_MD_FLFLAGS)
+		inode->i_flags = ll_ext_to_inode_flags(body->mbo_flags);
+	if (body->mbo_valid & OBD_MD_FLNLINK)
+		set_nlink(inode, body->mbo_nlink);
+	if (body->mbo_valid & OBD_MD_FLRDEV)
+		inode->i_rdev = old_decode_dev(body->mbo_rdev);
+
+	if (body->mbo_valid & OBD_MD_FLID) {
 		/* FID shouldn't be changed! */
 		if (fid_is_sane(&lli->lli_fid)) {
-			LASSERTF(lu_fid_eq(&lli->lli_fid, &body->fid1),
+			LASSERTF(lu_fid_eq(&lli->lli_fid, &body->mbo_fid1),
 				 "Trying to change FID "DFID" to the "DFID", inode "DFID"(%p)\n",
-				 PFID(&lli->lli_fid), PFID(&body->fid1),
+				 PFID(&lli->lli_fid), PFID(&body->mbo_fid1),
 				 PFID(ll_inode2fid(inode)), inode);
 		} else {
-			lli->lli_fid = body->fid1;
+			lli->lli_fid = body->mbo_fid1;
 		}
 	}
 
 	LASSERT(fid_seq(&lli->lli_fid) != 0);
 
-	if (body->valid & OBD_MD_FLSIZE) {
+	if (body->mbo_valid & OBD_MD_FLSIZE) {
 		if (exp_connect_som(ll_i2mdexp(inode)) &&
 		    S_ISREG(inode->i_mode)) {
 			struct lustre_handle lockh;
@@ -1577,7 +1854,7 @@ void ll_update_inode(struct inode *inode, struct lustre_md *md)
 					/* Use old size assignment to avoid
 					 * deadlock bz14138 & bz14326
 					 */
-					i_size_write(inode, body->size);
+					i_size_write(inode, body->mbo_size);
 					spin_lock(&lli->lli_lock);
 					lli->lli_flags |= LLIF_MDS_SIZE_LOCK;
 					spin_unlock(&lli->lli_lock);
@@ -1588,26 +1865,29 @@ void ll_update_inode(struct inode *inode, struct lustre_md *md)
 			/* Use old size assignment to avoid
 			 * deadlock bz14138 & bz14326
 			 */
-			i_size_write(inode, body->size);
+			i_size_write(inode, body->mbo_size);
 
 			CDEBUG(D_VFSTRACE, "inode=%lu, updating i_size %llu\n",
-			       inode->i_ino, (unsigned long long)body->size);
+			       inode->i_ino, (unsigned long long)body->mbo_size);
 		}
 
-		if (body->valid & OBD_MD_FLBLOCKS)
-			inode->i_blocks = body->blocks;
+		if (body->mbo_valid & OBD_MD_FLBLOCKS)
+			inode->i_blocks = body->mbo_blocks;
 	}
 
-	if (body->valid & OBD_MD_TSTATE) {
-		if (body->t_state & MS_RESTORE)
+	if (body->mbo_valid & OBD_MD_TSTATE) {
+		if (body->mbo_t_state & MS_RESTORE)
 			lli->lli_flags |= LLIF_FILE_RESTORING;
 	}
+
+	return 0;
 }
 
-void ll_read_inode2(struct inode *inode, void *opaque)
+int ll_read_inode2(struct inode *inode, void *opaque)
 {
 	struct lustre_md *md = opaque;
 	struct ll_inode_info *lli = ll_i2info(inode);
+	int rc;
 
 	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p)\n",
 	       PFID(&lli->lli_fid), inode);
@@ -1623,7 +1903,9 @@ void ll_read_inode2(struct inode *inode, void *opaque)
 	LTIME_S(inode->i_atime) = 0;
 	LTIME_S(inode->i_ctime) = 0;
 	inode->i_rdev = 0;
-	ll_update_inode(inode, md);
+	rc = ll_update_inode(inode, md);
+	if (rc)
+		return rc;
 
 	/* OIDEBUG(inode); */
 
@@ -1644,6 +1926,8 @@ void ll_read_inode2(struct inode *inode, void *opaque)
 		init_special_inode(inode, inode->i_mode,
 				   inode->i_rdev);
 	}
+
+	return 0;
 }
 
 void ll_delete_inode(struct inode *inode)
@@ -1655,20 +1939,13 @@ void ll_delete_inode(struct inode *inode)
 		 * osc_extent implementation at LU-1030.
 		 */
 		cl_sync_file_range(inode, 0, OBD_OBJECT_EOF,
-				   CL_FSYNC_DISCARD, 1);
+				   CL_FSYNC_LOCAL, 1);
 
 	truncate_inode_pages_final(&inode->i_data);
 
-	/* Workaround for LU-118 */
-	if (inode->i_data.nrpages) {
-		spin_lock_irq(&inode->i_data.tree_lock);
-		spin_unlock_irq(&inode->i_data.tree_lock);
-		LASSERTF(inode->i_data.nrpages == 0,
-			 "inode="DFID"(%p) nrpages=%lu, see http://jira.whamcloud.com/browse/LU-118\n",
-			 PFID(ll_inode2fid(inode)), inode,
-			 inode->i_data.nrpages);
-	}
-	/* Workaround end */
+	LASSERTF(!inode->i_data.nrpages,
+		 "inode=" DFID "(%p) nrpages=%lu, see http://jira.whamcloud.com/browse/LU-118\n",
+		 PFID(ll_inode2fid(inode)), inode, inode->i_data.nrpages);
 
 	ll_clear_inode(inode);
 	clear_inode(inode);
@@ -1704,7 +1981,7 @@ int ll_iocontrol(struct inode *inode, struct file *file,
 
 		body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
 
-		flags = body->flags;
+		flags = body->mbo_flags;
 
 		ptlrpc_req_finished(req);
 
@@ -1886,9 +2163,9 @@ void ll_open_cleanup(struct super_block *sb, struct ptlrpc_request *open_req)
 	if (!op_data)
 		return;
 
-	op_data->op_fid1 = body->fid1;
-	op_data->op_ioepoch = body->ioepoch;
-	op_data->op_handle = body->handle;
+	op_data->op_fid1 = body->mbo_fid1;
+	op_data->op_ioepoch = body->mbo_ioepoch;
+	op_data->op_handle = body->mbo_handle;
 	op_data->op_mod_time = get_seconds();
 	md_close(exp, op_data, NULL, &close_req);
 	ptlrpc_req_finished(close_req);
@@ -1910,7 +2187,9 @@ int ll_prep_inode(struct inode **inode, struct ptlrpc_request *req,
 		goto cleanup;
 
 	if (*inode) {
-		ll_update_inode(*inode, &md);
+		rc = ll_update_inode(*inode, &md);
+		if (rc)
+			goto out;
 	} else {
 		LASSERT(sb);
 
@@ -1918,18 +2197,18 @@ int ll_prep_inode(struct inode **inode, struct ptlrpc_request *req,
 		 * At this point server returns to client's same fid as client
 		 * generated for creating. So using ->fid1 is okay here.
 		 */
-		if (!fid_is_sane(&md.body->fid1)) {
+		if (!fid_is_sane(&md.body->mbo_fid1)) {
 			CERROR("%s: Fid is insane " DFID "\n",
 			       ll_get_fsname(sb, NULL, 0),
-			       PFID(&md.body->fid1));
+			       PFID(&md.body->mbo_fid1));
 			rc = -EINVAL;
 			goto out;
 		}
 
-		*inode = ll_iget(sb, cl_fid_build_ino(&md.body->fid1,
+		*inode = ll_iget(sb, cl_fid_build_ino(&md.body->mbo_fid1,
 					     sbi->ll_flags & LL_SBI_32BIT_API),
 				 &md);
-		if (!*inode) {
+		if (IS_ERR(*inode)) {
 #ifdef CONFIG_FS_POSIX_ACL
 			if (md.posix_acl) {
 				posix_acl_release(md.posix_acl);
@@ -2075,11 +2354,20 @@ int ll_process_config(struct lustre_cfg *lcfg)
 /* this function prepares md_op_data hint for passing ot down to MD stack. */
 struct md_op_data *ll_prep_md_op_data(struct md_op_data *op_data,
 				      struct inode *i1, struct inode *i2,
-				      const char *name, int namelen,
-				      int mode, __u32 opc, void *data)
+				      const char *name, size_t namelen,
+				      u32 mode, __u32 opc, void *data)
 {
-	if (namelen > ll_i2sbi(i1)->ll_namelen)
-		return ERR_PTR(-ENAMETOOLONG);
+	if (!name) {
+		/* Do not reuse namelen for something else. */
+		if (namelen)
+			return ERR_PTR(-EINVAL);
+	} else {
+		if (namelen > ll_i2sbi(i1)->ll_namelen)
+			return ERR_PTR(-ENAMETOOLONG);
+
+		if (!lu_name_is_valid_2(name, namelen))
+			return ERR_PTR(-EINVAL);
+	}
 
 	if (!op_data)
 		op_data = kzalloc(sizeof(*op_data), GFP_NOFS);
@@ -2089,11 +2377,26 @@ struct md_op_data *ll_prep_md_op_data(struct md_op_data *op_data,
 
 	ll_i2gids(op_data->op_suppgids, i1, i2);
 	op_data->op_fid1 = *ll_inode2fid(i1);
+	op_data->op_default_stripe_offset = -1;
+	if (S_ISDIR(i1->i_mode)) {
+		op_data->op_mea1 = ll_i2info(i1)->lli_lsm_md;
+		op_data->op_default_stripe_offset =
+			ll_i2info(i1)->lli_def_stripe_offset;
+	}
 
-	if (i2)
+	if (i2) {
 		op_data->op_fid2 = *ll_inode2fid(i2);
-	else
+		if (S_ISDIR(i2->i_mode))
+			op_data->op_mea2 = ll_i2info(i2)->lli_lsm_md;
+	} else {
 		fid_zero(&op_data->op_fid2);
+	}
+
+	if (ll_i2sbi(i1)->ll_flags & LL_SBI_64BIT_HASH)
+		op_data->op_cli_flags |= CLI_HASH64;
+
+	if (ll_need_32bit_api(ll_i2sbi(i1)))
+		op_data->op_cli_flags |= CLI_API32;
 
 	op_data->op_name = name;
 	op_data->op_namelen = namelen;
@@ -2105,26 +2408,12 @@ struct md_op_data *ll_prep_md_op_data(struct md_op_data *op_data,
 	op_data->op_bias = 0;
 	op_data->op_cli_flags = 0;
 	if ((opc == LUSTRE_OPC_CREATE) && name &&
-	    filename_is_volatile(name, namelen, NULL))
+	    filename_is_volatile(name, namelen, &op_data->op_mds))
 		op_data->op_bias |= MDS_CREATE_VOLATILE;
-	op_data->op_opc = opc;
-	op_data->op_mds = 0;
+	else
+		op_data->op_mds = 0;
 	op_data->op_data = data;
 
-	/* If the file is being opened after mknod() (normally due to NFS)
-	 * try to use the default stripe data from parent directory for
-	 * allocating OST objects.  Try to pass the parent FID to MDS.
-	 */
-	if (opc == LUSTRE_OPC_CREATE && i1 == i2 && S_ISREG(i2->i_mode) &&
-	    !ll_i2info(i2)->lli_has_smd) {
-		struct ll_inode_info *lli = ll_i2info(i2);
-
-		spin_lock(&lli->lli_lock);
-		if (likely(!lli->lli_has_smd && !fid_is_zero(&lli->lli_pfid)))
-			op_data->op_fid1 = lli->lli_pfid;
-		spin_unlock(&lli->lli_lock);
-	}
-
 	/* When called by ll_setattr_raw, file is i1. */
 	if (ll_i2info(i1)->lli_flags & LLIF_DATA_MODIFIED)
 		op_data->op_bias |= MDS_DATA_MODIFIED;
@@ -2251,3 +2540,197 @@ void ll_dirty_page_discard_warn(struct page *page, int ioret)
 	if (buf)
 		free_page((unsigned long)buf);
 }
+
+ssize_t ll_copy_user_md(const struct lov_user_md __user *md,
+			struct lov_user_md **kbuf)
+{
+	struct lov_user_md lum;
+	ssize_t lum_size;
+
+	if (copy_from_user(&lum, md, sizeof(lum))) {
+		lum_size = -EFAULT;
+		goto no_kbuf;
+	}
+
+	lum_size = ll_lov_user_md_size(&lum);
+	if (lum_size < 0)
+		goto no_kbuf;
+
+	*kbuf = kzalloc(lum_size, GFP_NOFS);
+	if (!*kbuf) {
+		lum_size = -ENOMEM;
+		goto no_kbuf;
+	}
+
+	if (copy_from_user(*kbuf, md, lum_size) != 0) {
+		kfree(*kbuf);
+		*kbuf = NULL;
+		lum_size = -EFAULT;
+	}
+no_kbuf:
+	return lum_size;
+}
+
+/*
+ * Compute llite root squash state after a change of root squash
+ * configuration setting or add/remove of a lnet nid
+ */
+void ll_compute_rootsquash_state(struct ll_sb_info *sbi)
+{
+	struct root_squash_info *squash = &sbi->ll_squash;
+	lnet_process_id_t id;
+	bool matched;
+	int i;
+
+	/* Update norootsquash flag */
+	down_write(&squash->rsi_sem);
+	if (list_empty(&squash->rsi_nosquash_nids)) {
+		sbi->ll_flags &= ~LL_SBI_NOROOTSQUASH;
+	} else {
+		/*
+		 * Do not apply root squash as soon as one of our NIDs is
+		 * in the nosquash_nids list
+		 */
+		matched = false;
+		i = 0;
+
+		while (LNetGetId(i++, &id) != -ENOENT) {
+			if (LNET_NETTYP(LNET_NIDNET(id.nid)) == LOLND)
+				continue;
+			if (cfs_match_nid(id.nid, &squash->rsi_nosquash_nids)) {
+				matched = true;
+				break;
+			}
+		}
+		if (matched)
+			sbi->ll_flags |= LL_SBI_NOROOTSQUASH;
+		else
+			sbi->ll_flags &= ~LL_SBI_NOROOTSQUASH;
+	}
+	up_write(&squash->rsi_sem);
+}
+
+/**
+ * Parse linkea content to extract information about a given hardlink
+ *
+ * \param[in]	ldata		- Initialized linkea data
+ * \param[in]	linkno		- Link identifier
+ * \param[out]	parent_fid	- The entry's parent FID
+ * \param[in]	size		- Entry name destination buffer
+ *
+ * \retval 0 on success
+ * \retval Appropriate negative error code on failure
+ */
+static int ll_linkea_decode(struct linkea_data *ldata, unsigned int linkno,
+			    struct lu_fid *parent_fid, struct lu_name *ln)
+{
+	unsigned int idx;
+	int rc;
+
+	rc = linkea_init(ldata);
+	if (rc < 0)
+		return rc;
+
+	if (linkno >= ldata->ld_leh->leh_reccount)
+		/* beyond last link */
+		return -ENODATA;
+
+	linkea_first_entry(ldata);
+	for (idx = 0; ldata->ld_lee; idx++) {
+		linkea_entry_unpack(ldata->ld_lee, &ldata->ld_reclen, ln,
+				    parent_fid);
+		if (idx == linkno)
+			break;
+
+		linkea_next_entry(ldata);
+	}
+
+	if (idx < linkno)
+		return -ENODATA;
+
+	return 0;
+}
+
+/**
+ * Get parent FID and name of an identified link. Operation is performed for
+ * a given link number, letting the caller iterate over linkno to list one or
+ * all links of an entry.
+ *
+ * \param[in]	  file	- File descriptor against which to perform the operation
+ * \param[in,out] arg	- User-filled structure containing the linkno to operate
+ *			  on and the available size. It is eventually filled with
+ *			  the requested information or left untouched on error
+ *
+ * \retval - 0 on success
+ * \retval - Appropriate negative error code on failure
+ */
+int ll_getparent(struct file *file, struct getparent __user *arg)
+{
+	struct inode *inode = file_inode(file);
+	struct linkea_data *ldata;
+	struct lu_fid parent_fid;
+	struct lu_buf buf = {
+		.lb_buf = NULL,
+		.lb_len = 0
+	};
+	struct lu_name ln;
+	u32 name_size;
+	u32 linkno;
+	int rc;
+
+	if (!capable(CFS_CAP_DAC_READ_SEARCH) &&
+	    !(ll_i2sbi(inode)->ll_flags & LL_SBI_USER_FID2PATH))
+		return -EPERM;
+
+	if (get_user(name_size, &arg->gp_name_size))
+		return -EFAULT;
+
+	if (get_user(linkno, &arg->gp_linkno))
+		return -EFAULT;
+
+	if (name_size > PATH_MAX)
+		return -EINVAL;
+
+	ldata = kzalloc(sizeof(*ldata), GFP_NOFS);
+	if (!ldata)
+		return -ENOMEM;
+
+	rc = linkea_data_new(ldata, &buf);
+	if (rc < 0)
+		goto ldata_free;
+
+	rc = ll_xattr_list(inode, XATTR_NAME_LINK, XATTR_TRUSTED_T, buf.lb_buf,
+			   buf.lb_len, OBD_MD_FLXATTR);
+	if (rc < 0)
+		goto lb_free;
+
+	rc = ll_linkea_decode(ldata, linkno, &parent_fid, &ln);
+	if (rc < 0)
+		goto lb_free;
+
+	if (ln.ln_namelen >= name_size) {
+		rc = -EOVERFLOW;
+		goto lb_free;
+	}
+
+	if (copy_to_user(&arg->gp_fid, &parent_fid, sizeof(arg->gp_fid))) {
+		rc = -EFAULT;
+		goto lb_free;
+	}
+
+	if (copy_to_user(&arg->gp_name, ln.ln_name, ln.ln_namelen)) {
+		rc = -EFAULT;
+		goto lb_free;
+	}
+
+	if (put_user('\0', arg->gp_name + ln.ln_namelen)) {
+		rc = -EFAULT;
+		goto lb_free;
+	}
+
+lb_free:
+	lu_buf_free(&buf);
+ldata_free:
+	kfree(ldata);
+	return rc;
+}
diff --git a/drivers/staging/lustre/lustre/llite/llite_mmap.c b/drivers/staging/lustre/lustre/llite/llite_mmap.c
index 66ee5db5fce8..436691814a5e 100644
--- a/drivers/staging/lustre/lustre/llite/llite_mmap.c
+++ b/drivers/staging/lustre/lustre/llite/llite_mmap.c
@@ -43,9 +43,7 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/lustre_lite.h"
 #include "llite_internal.h"
-#include "../include/linux/lustre_compat25.h"
 
 static const struct vm_operations_struct ll_file_vm_ops;
 
@@ -126,7 +124,7 @@ restart:
 
 	fio = &io->u.ci_fault;
 	fio->ft_index      = index;
-	fio->ft_executable = vma->vm_flags&VM_EXEC;
+	fio->ft_executable = vma->vm_flags & VM_EXEC;
 
 	/*
 	 * disable VM_SEQ_READ and use VM_RAND_READ to make sure that
@@ -134,7 +132,7 @@ restart:
 	 * filemap_nopage. we do our readahead in ll_readpage.
 	 */
 	if (ra_flags)
-		*ra_flags = vma->vm_flags & (VM_RAND_READ|VM_SEQ_READ);
+		*ra_flags = vma->vm_flags & (VM_RAND_READ | VM_SEQ_READ);
 	vma->vm_flags &= ~VM_SEQ_READ;
 	vma->vm_flags |= VM_RAND_READ;
 
@@ -429,7 +427,6 @@ static void ll_vm_open(struct vm_area_struct *vma)
 	struct inode *inode    = file_inode(vma->vm_file);
 	struct vvp_object *vob = cl_inode2vvp(inode);
 
-	LASSERT(vma->vm_file);
 	LASSERT(atomic_read(&vob->vob_mmap_cnt) >= 0);
 	atomic_inc(&vob->vob_mmap_cnt);
 }
@@ -442,7 +439,6 @@ static void ll_vm_close(struct vm_area_struct *vma)
 	struct inode      *inode = file_inode(vma->vm_file);
 	struct vvp_object *vob   = cl_inode2vvp(inode);
 
-	LASSERT(vma->vm_file);
 	atomic_dec(&vob->vob_mmap_cnt);
 	LASSERT(atomic_read(&vob->vob_mmap_cnt) >= 0);
 }
diff --git a/drivers/staging/lustre/lustre/llite/llite_nfs.c b/drivers/staging/lustre/lustre/llite/llite_nfs.c
index 65972c892731..709230571b4b 100644
--- a/drivers/staging/lustre/lustre/llite/llite_nfs.c
+++ b/drivers/staging/lustre/lustre/llite/llite_nfs.c
@@ -38,7 +38,6 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LLITE
-#include "../include/lustre_lite.h"
 #include "llite_internal.h"
 #include <linux/exportfs.h>
 
@@ -73,11 +72,6 @@ void get_uuid2fsid(const char *name, int len, __kernel_fsid_t *fsid)
 	fsid->val[1] = key >> 32;
 }
 
-static int ll_nfs_test_inode(struct inode *inode, void *opaque)
-{
-	return lu_fid_eq(&ll_i2info(inode)->lli_fid, opaque);
-}
-
 struct inode *search_inode_for_lustre(struct super_block *sb,
 				      const struct lu_fid *fid)
 {
@@ -92,7 +86,7 @@ struct inode *search_inode_for_lustre(struct super_block *sb,
 
 	CDEBUG(D_INFO, "searching inode for:(%lu,"DFID")\n", hash, PFID(fid));
 
-	inode = ilookup5(sb, hash, ll_nfs_test_inode, (void *)fid);
+	inode = ilookup5(sb, hash, ll_test_inode_by_fid, (void *)fid);
 	if (inode)
 		return inode;
 
@@ -153,12 +147,18 @@ ll_iget_for_nfs(struct super_block *sb, struct lu_fid *fid, struct lu_fid *paren
 		return ERR_PTR(-ESTALE);
 	}
 
+	result = d_obtain_alias(inode);
+	if (IS_ERR(result)) {
+		iput(inode);
+		return result;
+	}
+
 	/**
-	 * It is an anonymous dentry without OST objects created yet.
-	 * We have to find the parent to tell MDS how to init lov objects.
+	 * In case d_obtain_alias() found a disconnected dentry, always update
+	 * lli_pfid to allow later operation (normally open) have parent fid,
+	 * which may be used by MDS to create data.
 	 */
-	if (S_ISREG(inode->i_mode) && !ll_i2info(inode)->lli_has_smd &&
-	    parent && !fid_is_zero(parent)) {
+	if (parent) {
 		struct ll_inode_info *lli = ll_i2info(inode);
 
 		spin_lock(&lli->lli_lock);
@@ -255,6 +255,8 @@ static int ll_get_name(struct dentry *dentry, char *name,
 		.lgd_fid = ll_i2info(d_inode(child))->lli_fid,
 		.ctx.actor = ll_nfs_get_name_filldir,
 	};
+	struct md_op_data *op_data;
+	__u64 pos = 0;
 
 	if (!dir || !S_ISDIR(dir->i_mode)) {
 		rc = -ENOTDIR;
@@ -266,9 +268,18 @@ static int ll_get_name(struct dentry *dentry, char *name,
 		goto out;
 	}
 
+	op_data = ll_prep_md_op_data(NULL, dir, dir, NULL, 0, 0,
+				     LUSTRE_OPC_ANY, dir);
+	if (IS_ERR(op_data)) {
+		rc = PTR_ERR(op_data);
+		goto out;
+	}
+
+	op_data->op_max_pages = ll_i2sbi(dir)->ll_md_brw_pages;
 	inode_lock(dir);
-	rc = ll_dir_read(dir, &lgd.ctx);
+	rc = ll_dir_read(dir, &pos, op_data, &lgd.ctx);
 	inode_unlock(dir);
+	ll_finish_md_op_data(op_data);
 	if (!rc && !lgd.lgd_found)
 		rc = -ENOENT;
 out:
@@ -297,14 +308,12 @@ static struct dentry *ll_fh_to_parent(struct super_block *sb, struct fid *fid,
 	return ll_iget_for_nfs(sb, &nfs_fid->lnf_parent, NULL);
 }
 
-static struct dentry *ll_get_parent(struct dentry *dchild)
+int ll_dir_get_parent_fid(struct inode *dir, struct lu_fid *parent_fid)
 {
 	struct ptlrpc_request *req = NULL;
-	struct inode	  *dir = d_inode(dchild);
 	struct ll_sb_info     *sbi;
-	struct dentry	 *result = NULL;
 	struct mdt_body       *body;
-	static char	   dotdot[] = "..";
+	static const char dotdot[] = "..";
 	struct md_op_data     *op_data;
 	int		   rc;
 	int		      lmmsize;
@@ -319,13 +328,13 @@ static struct dentry *ll_get_parent(struct dentry *dchild)
 
 	rc = ll_get_default_mdsize(sbi, &lmmsize);
 	if (rc != 0)
-		return ERR_PTR(rc);
+		return rc;
 
 	op_data = ll_prep_md_op_data(NULL, dir, NULL, dotdot,
 				     strlen(dotdot), lmmsize,
 				     LUSTRE_OPC_ANY, NULL);
 	if (IS_ERR(op_data))
-		return (void *)op_data;
+		return PTR_ERR(op_data);
 
 	rc = md_getattr_name(sbi->ll_md_exp, op_data, &req);
 	ll_finish_md_op_data(op_data);
@@ -333,21 +342,36 @@ static struct dentry *ll_get_parent(struct dentry *dchild)
 		CERROR("%s: failure inode "DFID" get parent: rc = %d\n",
 		       ll_get_fsname(dir->i_sb, NULL, 0),
 		       PFID(ll_inode2fid(dir)), rc);
-		return ERR_PTR(rc);
+		return rc;
 	}
 	body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
 	/*
 	 * LU-3952: MDT may lost the FID of its parent, we should not crash
 	 * the NFS server, ll_iget_for_nfs() will handle the error.
 	 */
-	if (body->valid & OBD_MD_FLID) {
+	if (body->mbo_valid & OBD_MD_FLID) {
 		CDEBUG(D_INFO, "parent for " DFID " is " DFID "\n",
-		       PFID(ll_inode2fid(dir)), PFID(&body->fid1));
+		       PFID(ll_inode2fid(dir)), PFID(&body->mbo_fid1));
+		*parent_fid = body->mbo_fid1;
 	}
-	result = ll_iget_for_nfs(dir->i_sb, &body->fid1, NULL);
 
 	ptlrpc_req_finished(req);
-	return result;
+	return 0;
+}
+
+static struct dentry *ll_get_parent(struct dentry *dchild)
+{
+	struct lu_fid parent_fid = { 0 };
+	struct dentry *dentry;
+	int rc;
+
+	rc = ll_dir_get_parent_fid(dchild->d_inode, &parent_fid);
+	if (rc)
+		return ERR_PTR(rc);
+
+	dentry = ll_iget_for_nfs(dchild->d_inode->i_sb, &parent_fid, NULL);
+
+	return dentry;
 }
 
 const struct export_operations lustre_export_operations = {
diff --git a/drivers/staging/lustre/lustre/llite/lproc_llite.c b/drivers/staging/lustre/lustre/llite/lproc_llite.c
index e86bf3c53be3..6eae60595905 100644
--- a/drivers/staging/lustre/lustre/llite/lproc_llite.c
+++ b/drivers/staging/lustre/lustre/llite/lproc_llite.c
@@ -31,7 +31,6 @@
  */
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/lustre_lite.h"
 #include "../include/lprocfs_status.h"
 #include <linux/seq_file.h>
 #include "../include/obd_support.h"
@@ -358,16 +357,16 @@ static int ll_max_cached_mb_seq_show(struct seq_file *m, void *v)
 	struct ll_sb_info      *sbi   = ll_s2sbi(sb);
 	struct cl_client_cache *cache = sbi->ll_cache;
 	int shift = 20 - PAGE_SHIFT;
-	int max_cached_mb;
-	int unused_mb;
+	long max_cached_mb;
+	long unused_mb;
 
 	max_cached_mb = cache->ccc_lru_max >> shift;
-	unused_mb = atomic_read(&cache->ccc_lru_left) >> shift;
+	unused_mb = atomic_long_read(&cache->ccc_lru_left) >> shift;
 	seq_printf(m,
 		   "users: %d\n"
-		   "max_cached_mb: %d\n"
-		   "used_mb: %d\n"
-		   "unused_mb: %d\n"
+		   "max_cached_mb: %ld\n"
+		   "used_mb: %ld\n"
+		   "unused_mb: %ld\n"
 		   "reclaim_count: %u\n",
 		   atomic_read(&cache->ccc_users),
 		   max_cached_mb,
@@ -385,10 +384,13 @@ static ssize_t ll_max_cached_mb_seq_write(struct file *file,
 	struct ll_sb_info *sbi = ll_s2sbi(sb);
 	struct cl_client_cache *cache = sbi->ll_cache;
 	struct lu_env *env;
+	long diff = 0;
+	long nrpages = 0;
 	int refcheck;
-	int mult, rc, pages_number;
-	int diff = 0;
-	int nrpages = 0;
+	long pages_number;
+	int mult;
+	long rc;
+	u64 val;
 	char kernbuf[128];
 
 	if (count >= sizeof(kernbuf))
@@ -401,10 +403,14 @@ static ssize_t ll_max_cached_mb_seq_write(struct file *file,
 	mult = 1 << (20 - PAGE_SHIFT);
 	buffer += lprocfs_find_named_value(kernbuf, "max_cached_mb:", &count) -
 		  kernbuf;
-	rc = lprocfs_write_frac_helper(buffer, count, &pages_number, mult);
+	rc = lprocfs_write_frac_u64_helper(buffer, count, &val, mult);
 	if (rc)
 		return rc;
 
+	if (val > LONG_MAX)
+		return -ERANGE;
+	pages_number = (long)val;
+
 	if (pages_number < 0 || pages_number > totalram_pages) {
 		CERROR("%s: can't set max cache more than %lu MB\n",
 		       ll_get_fsname(sb, NULL, 0),
@@ -418,7 +424,7 @@ static ssize_t ll_max_cached_mb_seq_write(struct file *file,
 
 	/* easy - add more LRU slots. */
 	if (diff >= 0) {
-		atomic_add(diff, &cache->ccc_lru_left);
+		atomic_long_add(diff, &cache->ccc_lru_left);
 		rc = 0;
 		goto out;
 	}
@@ -429,18 +435,18 @@ static ssize_t ll_max_cached_mb_seq_write(struct file *file,
 
 	diff = -diff;
 	while (diff > 0) {
-		int tmp;
+		long tmp;
 
 		/* reduce LRU budget from free slots. */
 		do {
-			int ov, nv;
+			long ov, nv;
 
-			ov = atomic_read(&cache->ccc_lru_left);
+			ov = atomic_long_read(&cache->ccc_lru_left);
 			if (ov == 0)
 				break;
 
 			nv = ov > diff ? ov - diff : 0;
-			rc = atomic_cmpxchg(&cache->ccc_lru_left, ov, nv);
+			rc = atomic_long_cmpxchg(&cache->ccc_lru_left, ov, nv);
 			if (likely(ov == rc)) {
 				diff -= ov - nv;
 				nrpages += ov - nv;
@@ -474,7 +480,7 @@ out:
 		spin_unlock(&sbi->ll_lock);
 		rc = count;
 	} else {
-		atomic_add(nrpages, &cache->ccc_lru_left);
+		atomic_long_add(nrpages, &cache->ccc_lru_left);
 	}
 	return rc;
 }
@@ -738,6 +744,18 @@ static ssize_t max_easize_show(struct kobject *kobj,
 }
 LUSTRE_RO_ATTR(max_easize);
 
+/**
+ * Get default_easize.
+ *
+ * \see client_obd::cl_default_mds_easize
+ *
+ * \param[in] kobj	kernel object for sysfs tree
+ * \param[in] attr	attribute of this kernel object
+ * \param[in] buf	buffer to write data into
+ *
+ * \retval positive	\a count on success
+ * \retval negative	negated errno on failure
+ */
 static ssize_t default_easize_show(struct kobject *kobj,
 				   struct attribute *attr,
 				   char *buf)
@@ -753,7 +771,44 @@ static ssize_t default_easize_show(struct kobject *kobj,
 
 	return sprintf(buf, "%u\n", ealen);
 }
-LUSTRE_RO_ATTR(default_easize);
+
+/**
+ * Set default_easize.
+ *
+ * Range checking on the passed value is handled by
+ * ll_set_default_mdsize().
+ *
+ * \see client_obd::cl_default_mds_easize
+ *
+ * \param[in] kobj	kernel object for sysfs tree
+ * \param[in] attr	attribute of this kernel object
+ * \param[in] buffer	string passed from user space
+ * \param[in] count	\a buffer length
+ *
+ * \retval positive	\a count on success
+ * \retval negative	negated errno on failure
+ */
+static ssize_t default_easize_store(struct kobject *kobj,
+				    struct attribute *attr,
+				    const char *buffer,
+				    size_t count)
+{
+	struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info,
+					      ll_kobj);
+	unsigned long val;
+	int rc;
+
+	rc = kstrtoul(buffer, 10, &val);
+	if (rc)
+		return rc;
+
+	rc = ll_set_default_mdsize(sbi, val);
+	if (rc)
+		return rc;
+
+	return count;
+}
+LUSTRE_RW_ATTR(default_easize);
 
 static int ll_sbi_flags_seq_show(struct seq_file *m, void *v)
 {
@@ -774,7 +829,7 @@ static int ll_sbi_flags_seq_show(struct seq_file *m, void *v)
 		flags >>= 1;
 		++i;
 	}
-	seq_printf(m, "\b\n");
+	seq_puts(m, "\b\n");
 	return 0;
 }
 
@@ -823,15 +878,116 @@ static ssize_t unstable_stats_show(struct kobject *kobj,
 	struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info,
 					      ll_kobj);
 	struct cl_client_cache *cache = sbi->ll_cache;
-	int pages, mb;
+	long pages;
+	int mb;
 
-	pages = atomic_read(&cache->ccc_unstable_nr);
+	pages = atomic_long_read(&cache->ccc_unstable_nr);
 	mb = (pages * PAGE_SIZE) >> 20;
 
-	return sprintf(buf, "unstable_pages: %8d\n"
-			    "unstable_mb:    %8d\n", pages, mb);
+	return sprintf(buf, "unstable_check:     %8d\n"
+			    "unstable_pages: %12ld\n"
+			    "unstable_mb:        %8d\n",
+			    cache->ccc_unstable_check, pages, mb);
 }
-LUSTRE_RO_ATTR(unstable_stats);
+
+static ssize_t unstable_stats_store(struct kobject *kobj,
+				    struct attribute *attr,
+				    const char *buffer,
+				    size_t count)
+{
+	struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info,
+					      ll_kobj);
+	char kernbuf[128];
+	int val, rc;
+
+	if (!count)
+		return 0;
+	if (count >= sizeof(kernbuf))
+		return -EINVAL;
+
+	if (copy_from_user(kernbuf, buffer, count))
+		return -EFAULT;
+	kernbuf[count] = 0;
+
+	buffer += lprocfs_find_named_value(kernbuf, "unstable_check:", &count) -
+		  kernbuf;
+	rc = lprocfs_write_helper(buffer, count, &val);
+	if (rc < 0)
+		return rc;
+
+	/* borrow lru lock to set the value */
+	spin_lock(&sbi->ll_cache->ccc_lru_lock);
+	sbi->ll_cache->ccc_unstable_check = !!val;
+	spin_unlock(&sbi->ll_cache->ccc_lru_lock);
+
+	return count;
+}
+LUSTRE_RW_ATTR(unstable_stats);
+
+static ssize_t root_squash_show(struct kobject *kobj, struct attribute *attr,
+				char *buf)
+{
+	struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info,
+					      ll_kobj);
+	struct root_squash_info *squash = &sbi->ll_squash;
+
+	return sprintf(buf, "%u:%u\n", squash->rsi_uid, squash->rsi_gid);
+}
+
+static ssize_t root_squash_store(struct kobject *kobj, struct attribute *attr,
+				 const char *buffer, size_t count)
+{
+	struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info,
+					      ll_kobj);
+	struct root_squash_info *squash = &sbi->ll_squash;
+
+	return lprocfs_wr_root_squash(buffer, count, squash,
+				      ll_get_fsname(sbi->ll_sb, NULL, 0));
+}
+LUSTRE_RW_ATTR(root_squash);
+
+static int ll_nosquash_nids_seq_show(struct seq_file *m, void *v)
+{
+	struct super_block *sb = m->private;
+	struct ll_sb_info *sbi = ll_s2sbi(sb);
+	struct root_squash_info *squash = &sbi->ll_squash;
+	int len;
+
+	down_read(&squash->rsi_sem);
+	if (!list_empty(&squash->rsi_nosquash_nids)) {
+		len = cfs_print_nidlist(m->buf + m->count, m->size - m->count,
+					&squash->rsi_nosquash_nids);
+		m->count += len;
+		seq_puts(m, "\n");
+	} else {
+		seq_puts(m, "NONE\n");
+	}
+	up_read(&squash->rsi_sem);
+
+	return 0;
+}
+
+static ssize_t ll_nosquash_nids_seq_write(struct file *file,
+					  const char __user *buffer,
+					  size_t count, loff_t *off)
+{
+	struct seq_file *m = file->private_data;
+	struct super_block *sb = m->private;
+	struct ll_sb_info *sbi = ll_s2sbi(sb);
+	struct root_squash_info *squash = &sbi->ll_squash;
+	int rc;
+
+	rc = lprocfs_wr_nosquash_nids(buffer, count, squash,
+				      ll_get_fsname(sb, NULL, 0));
+	if (rc < 0)
+		return rc;
+
+	ll_compute_rootsquash_state(sbi);
+
+	return rc;
+}
+
+LPROC_SEQ_FOPS(ll_nosquash_nids);
 
 static struct lprocfs_vars lprocfs_llite_obd_vars[] = {
 	/* { "mntpt_path",   ll_rd_path,	     0, 0 }, */
@@ -840,6 +996,8 @@ static struct lprocfs_vars lprocfs_llite_obd_vars[] = {
 	{ "max_cached_mb",    &ll_max_cached_mb_fops, NULL },
 	{ "statahead_stats",  &ll_statahead_stats_fops, NULL, 0 },
 	{ "sbi_flags",	      &ll_sbi_flags_fops, NULL, 0 },
+	{ .name =		"nosquash_nids",
+	  .fops =		&ll_nosquash_nids_fops		},
 	{ NULL }
 };
 
@@ -869,6 +1027,7 @@ static struct attribute *llite_attrs[] = {
 	&lustre_attr_default_easize.attr,
 	&lustre_attr_xattr_cache.attr,
 	&lustre_attr_unstable_stats.attr,
+	&lustre_attr_root_squash.attr,
 	NULL,
 };
 
@@ -893,17 +1052,17 @@ static const struct llite_file_opcode {
 	/* file operation */
 	{ LPROC_LL_DIRTY_HITS,     LPROCFS_TYPE_REGS, "dirty_pages_hits" },
 	{ LPROC_LL_DIRTY_MISSES,   LPROCFS_TYPE_REGS, "dirty_pages_misses" },
-	{ LPROC_LL_READ_BYTES,     LPROCFS_CNTR_AVGMINMAX|LPROCFS_TYPE_BYTES,
+	{ LPROC_LL_READ_BYTES,     LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_BYTES,
 				   "read_bytes" },
-	{ LPROC_LL_WRITE_BYTES,    LPROCFS_CNTR_AVGMINMAX|LPROCFS_TYPE_BYTES,
+	{ LPROC_LL_WRITE_BYTES,    LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_BYTES,
 				   "write_bytes" },
-	{ LPROC_LL_BRW_READ,       LPROCFS_CNTR_AVGMINMAX|LPROCFS_TYPE_PAGES,
+	{ LPROC_LL_BRW_READ,       LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_PAGES,
 				   "brw_read" },
-	{ LPROC_LL_BRW_WRITE,      LPROCFS_CNTR_AVGMINMAX|LPROCFS_TYPE_PAGES,
+	{ LPROC_LL_BRW_WRITE,      LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_PAGES,
 				   "brw_write" },
-	{ LPROC_LL_OSC_READ,       LPROCFS_CNTR_AVGMINMAX|LPROCFS_TYPE_BYTES,
+	{ LPROC_LL_OSC_READ,       LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_BYTES,
 				   "osc_read" },
-	{ LPROC_LL_OSC_WRITE,      LPROCFS_CNTR_AVGMINMAX|LPROCFS_TYPE_BYTES,
+	{ LPROC_LL_OSC_WRITE,      LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_BYTES,
 				   "osc_write" },
 	{ LPROC_LL_IOCTL,	  LPROCFS_TYPE_REGS, "ioctl" },
 	{ LPROC_LL_OPEN,	   LPROCFS_TYPE_REGS, "open" },
@@ -1150,7 +1309,7 @@ static void ll_display_extents_info(struct ll_rw_extents_info *io_extents,
 			   r, pct(r, read_tot), pct(read_cum, read_tot),
 			   w, pct(w, write_tot), pct(write_cum, write_tot));
 		start = end;
-		if (start == 1<<10) {
+		if (start == 1 << 10) {
 			start = 1;
 			units += 10;
 			unitp++;
diff --git a/drivers/staging/lustre/lustre/llite/namei.c b/drivers/staging/lustre/lustre/llite/namei.c
index 3664bfd0178b..180f35e3afd9 100644
--- a/drivers/staging/lustre/lustre/llite/namei.c
+++ b/drivers/staging/lustre/lustre/llite/namei.c
@@ -42,13 +42,12 @@
 
 #include "../include/obd_support.h"
 #include "../include/lustre_fid.h"
-#include "../include/lustre_lite.h"
 #include "../include/lustre_dlm.h"
 #include "../include/lustre_ver.h"
 #include "llite_internal.h"
 
-static int ll_create_it(struct inode *, struct dentry *,
-			int, struct lookup_intent *);
+static int ll_create_it(struct inode *dir, struct dentry *dentry,
+			struct lookup_intent *it);
 
 /* called from iget5_locked->find_inode() under inode_hash_lock spinlock */
 static int ll_test_inode(struct inode *inode, void *opaque)
@@ -56,12 +55,12 @@ static int ll_test_inode(struct inode *inode, void *opaque)
 	struct ll_inode_info *lli = ll_i2info(inode);
 	struct lustre_md     *md = opaque;
 
-	if (unlikely(!(md->body->valid & OBD_MD_FLID))) {
+	if (unlikely(!(md->body->mbo_valid & OBD_MD_FLID))) {
 		CERROR("MDS body missing FID\n");
 		return 0;
 	}
 
-	if (!lu_fid_eq(&lli->lli_fid, &md->body->fid1))
+	if (!lu_fid_eq(&lli->lli_fid, &md->body->mbo_fid1))
 		return 0;
 
 	return 1;
@@ -72,20 +71,20 @@ static int ll_set_inode(struct inode *inode, void *opaque)
 	struct ll_inode_info *lli = ll_i2info(inode);
 	struct mdt_body *body = ((struct lustre_md *)opaque)->body;
 
-	if (unlikely(!(body->valid & OBD_MD_FLID))) {
+	if (unlikely(!(body->mbo_valid & OBD_MD_FLID))) {
 		CERROR("MDS body missing FID\n");
 		return -EINVAL;
 	}
 
-	lli->lli_fid = body->fid1;
-	if (unlikely(!(body->valid & OBD_MD_FLTYPE))) {
+	lli->lli_fid = body->mbo_fid1;
+	if (unlikely(!(body->mbo_valid & OBD_MD_FLTYPE))) {
 		CERROR("Can not initialize inode " DFID
 		       " without object type: valid = %#llx\n",
-		       PFID(&lli->lli_fid), body->valid);
+		       PFID(&lli->lli_fid), body->mbo_valid);
 		return -EINVAL;
 	}
 
-	inode->i_mode = (inode->i_mode & ~S_IFMT) | (body->mode & S_IFMT);
+	inode->i_mode = (inode->i_mode & ~S_IFMT) | (body->mbo_mode & S_IFMT);
 	if (unlikely(inode->i_mode == 0)) {
 		CERROR("Invalid inode "DFID" type\n", PFID(&lli->lli_fid));
 		return -EINVAL;
@@ -96,41 +95,45 @@ static int ll_set_inode(struct inode *inode, void *opaque)
 	return 0;
 }
 
-/*
- * Get an inode by inode number (already instantiated by the intent lookup).
- * Returns inode or NULL
+/**
+ * Get an inode by inode number(@hash), which is already instantiated by
+ * the intent lookup).
  */
 struct inode *ll_iget(struct super_block *sb, ino_t hash,
 		      struct lustre_md *md)
 {
 	struct inode	 *inode;
+	int rc = 0;
 
 	LASSERT(hash != 0);
 	inode = iget5_locked(sb, hash, ll_test_inode, ll_set_inode, md);
-
-	if (inode) {
-		if (inode->i_state & I_NEW) {
-			int rc = 0;
-
-			ll_read_inode2(inode, md);
-			if (S_ISREG(inode->i_mode) &&
-			    !ll_i2info(inode)->lli_clob) {
-				CDEBUG(D_INODE,
-				       "%s: apply lsm %p to inode " DFID ".\n",
-				       ll_get_fsname(sb, NULL, 0), md->lsm,
-				       PFID(ll_inode2fid(inode)));
-				rc = cl_file_inode_init(inode, md);
-			}
-			if (rc != 0) {
-				iget_failed(inode);
-				inode = NULL;
-			} else {
-				unlock_new_inode(inode);
-			}
-		} else if (!(inode->i_state & (I_FREEING | I_CLEAR))) {
-			ll_update_inode(inode, md);
-			CDEBUG(D_VFSTRACE, "got inode: "DFID"(%p)\n",
-			       PFID(&md->body->fid1), inode);
+	if (!inode)
+		return ERR_PTR(-ENOMEM);
+
+	if (inode->i_state & I_NEW) {
+		rc = ll_read_inode2(inode, md);
+		if (!rc && S_ISREG(inode->i_mode) &&
+		    !ll_i2info(inode)->lli_clob) {
+			CDEBUG(D_INODE, "%s: apply lsm %p to inode "DFID"\n",
+			       ll_get_fsname(sb, NULL, 0), md->lsm,
+			       PFID(ll_inode2fid(inode)));
+			rc = cl_file_inode_init(inode, md);
+		}
+		if (rc) {
+			make_bad_inode(inode);
+			unlock_new_inode(inode);
+			iput(inode);
+			inode = ERR_PTR(rc);
+		} else {
+			unlock_new_inode(inode);
+		}
+	} else if (!(inode->i_state & (I_FREEING | I_CLEAR))) {
+		rc = ll_update_inode(inode, md);
+		CDEBUG(D_VFSTRACE, "got inode: "DFID"(%p): rc = %d\n",
+		       PFID(&md->body->mbo_fid1), inode, rc);
+		if (rc) {
+			iput(inode);
+			inode = ERR_PTR(rc);
 		}
 	}
 	return inode;
@@ -158,6 +161,11 @@ static void ll_invalidate_negative_children(struct inode *dir)
 	spin_unlock(&dir->i_lock);
 }
 
+int ll_test_inode_by_fid(struct inode *inode, void *opaque)
+{
+	return lu_fid_eq(&ll_i2info(inode)->lli_fid, opaque);
+}
+
 int ll_md_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
 		       void *data, int flag)
 {
@@ -196,6 +204,8 @@ int ll_md_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
 		}
 
 		if (bits & MDS_INODELOCK_XATTR) {
+			if (S_ISDIR(inode->i_mode))
+				ll_i2info(inode)->lli_def_stripe_offset = -1;
 			ll_xattr_cache_destroy(inode);
 			bits &= ~MDS_INODELOCK_XATTR;
 		}
@@ -253,10 +263,41 @@ int ll_md_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
 		}
 
 		if ((bits & MDS_INODELOCK_UPDATE) && S_ISDIR(inode->i_mode)) {
-			CDEBUG(D_INODE, "invalidating inode "DFID"\n",
-			       PFID(ll_inode2fid(inode)));
+			struct ll_inode_info *lli = ll_i2info(inode);
+
+			CDEBUG(D_INODE, "invalidating inode "DFID" lli = %p, pfid  = "DFID"\n",
+			       PFID(ll_inode2fid(inode)), lli,
+			       PFID(&lli->lli_pfid));
+
 			truncate_inode_pages(inode->i_mapping, 0);
-			ll_invalidate_negative_children(inode);
+
+			if (unlikely(!fid_is_zero(&lli->lli_pfid))) {
+				struct inode *master_inode = NULL;
+				unsigned long hash;
+
+				/*
+				 * This is slave inode, since all of the child
+				 * dentry is connected on the master inode, so
+				 * we have to invalidate the negative children
+				 * on master inode
+				 */
+				CDEBUG(D_INODE, "Invalidate s"DFID" m"DFID"\n",
+				       PFID(ll_inode2fid(inode)),
+				       PFID(&lli->lli_pfid));
+
+				hash = cl_fid_build_ino(&lli->lli_pfid,
+							ll_need_32bit_api(ll_i2sbi(inode)));
+
+				master_inode = ilookup5(inode->i_sb, hash,
+							ll_test_inode_by_fid,
+							(void *)&lli->lli_pfid);
+				if (master_inode && !IS_ERR(master_inode)) {
+					ll_invalidate_negative_children(master_inode);
+					iput(master_inode);
+				}
+			} else {
+				ll_invalidate_negative_children(inode);
+			}
 		}
 
 		if ((bits & (MDS_INODELOCK_LOOKUP | MDS_INODELOCK_PERM)) &&
@@ -322,7 +363,8 @@ static struct dentry *ll_find_alias(struct inode *inode, struct dentry *dentry)
 		LASSERT(alias != dentry);
 
 		spin_lock(&alias->d_lock);
-		if (alias->d_flags & DCACHE_DISCONNECTED)
+		if ((alias->d_flags & DCACHE_DISCONNECTED) &&
+		    S_ISDIR(inode->i_mode))
 			/* LASSERT(last_discon == NULL); LU-405, bz 20055 */
 			discon_alias = alias;
 		else if (alias->d_parent == dentry->d_parent	     &&
@@ -388,6 +430,7 @@ static int ll_lookup_it_finish(struct ptlrpc_request *request,
 	struct inode *inode = NULL;
 	__u64 bits = 0;
 	int rc = 0;
+	struct dentry *alias;
 
 	/* NB 1 request reference will be taken away by ll_intent_lock()
 	 * when I return
@@ -412,26 +455,12 @@ static int ll_lookup_it_finish(struct ptlrpc_request *request,
 		 */
 	}
 
-	/* Only hash *de if it is unhashed (new dentry).
-	 * Atoimc_open may passing hashed dentries for open.
-	 */
-	if (d_unhashed(*de)) {
-		struct dentry *alias;
-
-		alias = ll_splice_alias(inode, *de);
-		if (IS_ERR(alias)) {
-			rc = PTR_ERR(alias);
-			goto out;
-		}
-		*de = alias;
-	} else if (!it_disposition(it, DISP_LOOKUP_NEG)  &&
-		   !it_disposition(it, DISP_OPEN_CREATE)) {
-		/* With DISP_OPEN_CREATE dentry will be
-		 * instantiated in ll_create_it.
-		 */
-		LASSERT(!d_inode(*de));
-		d_instantiate(*de, inode);
+	alias = ll_splice_alias(inode, *de);
+	if (IS_ERR(alias)) {
+		rc = PTR_ERR(alias);
+		goto out;
 	}
+	*de = alias;
 
 	if (!it_disposition(it, DISP_LOOKUP_NEG)) {
 		/* we have lookup look - unhide dentry */
@@ -446,9 +475,20 @@ static int ll_lookup_it_finish(struct ptlrpc_request *request,
 		struct lookup_intent parent_it = {
 					.it_op = IT_GETATTR,
 					.it_lock_handle = 0 };
+		struct lu_fid fid = ll_i2info(parent)->lli_fid;
+
+		/* If it is striped directory, get the real stripe parent */
+		if (unlikely(ll_i2info(parent)->lli_lsm_md)) {
+			rc = md_get_fid_from_lsm(ll_i2mdexp(parent),
+						 ll_i2info(parent)->lli_lsm_md,
+						 (*de)->d_name.name,
+						 (*de)->d_name.len, &fid);
+			if (rc)
+				return rc;
+		}
 
-		if (md_revalidate_lock(ll_i2mdexp(parent), &parent_it,
-				       &ll_i2info(parent)->lli_fid, NULL)) {
+		if (md_revalidate_lock(ll_i2mdexp(parent), &parent_it, &fid,
+				       NULL)) {
 			d_lustre_revalidate(*de);
 			ll_intent_release(&parent_it);
 		}
@@ -462,13 +502,13 @@ out:
 }
 
 static struct dentry *ll_lookup_it(struct inode *parent, struct dentry *dentry,
-				   struct lookup_intent *it, int lookup_flags)
+				   struct lookup_intent *it)
 {
 	struct lookup_intent lookup_it = { .it_op = IT_LOOKUP };
 	struct dentry *save = dentry, *retval;
 	struct ptlrpc_request *req = NULL;
+	struct md_op_data *op_data = NULL;
 	struct inode *inode;
-	struct md_op_data *op_data;
 	__u32 opc;
 	int rc;
 
@@ -484,8 +524,8 @@ static struct dentry *ll_lookup_it(struct inode *parent, struct dentry *dentry,
 	if (!it || it->it_op == IT_GETXATTR)
 		it = &lookup_it;
 
-	if (it->it_op == IT_GETATTR) {
-		rc = ll_statahead_enter(parent, &dentry, 0);
+	if (it->it_op == IT_GETATTR && dentry_may_statahead(parent, dentry)) {
+		rc = ll_statahead(parent, &dentry, 0);
 		if (rc == 1) {
 			if (dentry == save)
 				retval = NULL;
@@ -501,8 +541,7 @@ static struct dentry *ll_lookup_it(struct inode *parent, struct dentry *dentry,
 		opc = LUSTRE_OPC_ANY;
 
 	op_data = ll_prep_md_op_data(NULL, parent, NULL, dentry->d_name.name,
-				     dentry->d_name.len, lookup_flags, opc,
-				     NULL);
+				     dentry->d_name.len, 0, opc, NULL);
 	if (IS_ERR(op_data))
 		return (void *)op_data;
 
@@ -510,9 +549,38 @@ static struct dentry *ll_lookup_it(struct inode *parent, struct dentry *dentry,
 	if (!IS_POSIXACL(parent) || !exp_connect_umask(ll_i2mdexp(parent)))
 		it->it_create_mode &= ~current_umask();
 
-	rc = md_intent_lock(ll_i2mdexp(parent), op_data, NULL, 0, it,
-			    lookup_flags, &req, ll_md_blocking_ast, 0);
-	ll_finish_md_op_data(op_data);
+	rc = md_intent_lock(ll_i2mdexp(parent), op_data, it, &req,
+			    &ll_md_blocking_ast, 0);
+	/*
+	 * If the MDS allows the client to chgrp (CFS_SETGRP_PERM), but the
+	 * client does not know which suppgid should be sent to the MDS, or
+	 * some other(s) changed the target file's GID after this RPC sent
+	 * to the MDS with the suppgid as the original GID, then we should
+	 * try again with right suppgid.
+	 */
+	if (rc == -EACCES && it->it_op & IT_OPEN &&
+	    it_disposition(it, DISP_OPEN_DENY)) {
+		struct mdt_body *body;
+
+		LASSERT(req);
+
+		body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
+		if (op_data->op_suppgids[0] == body->mbo_gid ||
+		    op_data->op_suppgids[1] == body->mbo_gid ||
+		    !in_group_p(make_kgid(&init_user_ns, body->mbo_gid))) {
+			retval = ERR_PTR(-EACCES);
+			goto out;
+		}
+
+		fid_zero(&op_data->op_fid2);
+		op_data->op_suppgids[1] = body->mbo_gid;
+		ptlrpc_req_finished(req);
+		req = NULL;
+		ll_intent_release(it);
+		rc = md_intent_lock(ll_i2mdexp(parent), op_data, it, &req,
+				    ll_md_blocking_ast, 0);
+	}
+
 	if (rc < 0) {
 		retval = ERR_PTR(rc);
 		goto out;
@@ -537,11 +605,11 @@ static struct dentry *ll_lookup_it(struct inode *parent, struct dentry *dentry,
 		retval = NULL;
 	else
 		retval = dentry;
- out:
-	if (req)
-		ptlrpc_req_finished(req);
-	if (it->it_op == IT_GETATTR && (!retval || retval == dentry))
-		ll_statahead_mark(parent, dentry);
+out:
+	if (op_data && !IS_ERR(op_data))
+		ll_finish_md_op_data(op_data);
+
+	ptlrpc_req_finished(req);
 	return retval;
 }
 
@@ -554,15 +622,19 @@ static struct dentry *ll_lookup_nd(struct inode *parent, struct dentry *dentry,
 	CDEBUG(D_VFSTRACE, "VFS Op:name=%pd, dir="DFID"(%p),flags=%u\n",
 	       dentry, PFID(ll_inode2fid(parent)), parent, flags);
 
-	/* Optimize away (CREATE && !OPEN). Let .create handle the race. */
-	if ((flags & LOOKUP_CREATE) && !(flags & LOOKUP_OPEN))
+	/* Optimize away (CREATE && !OPEN). Let .create handle the race.
+	 * but only if we have write permissions there, otherwise we need
+	 * to proceed with lookup. LU-4185
+	 */
+	if ((flags & LOOKUP_CREATE) && !(flags & LOOKUP_OPEN) &&
+	    (inode_permission(parent, MAY_WRITE | MAY_EXEC) == 0))
 		return NULL;
 
-	if (flags & (LOOKUP_PARENT|LOOKUP_OPEN|LOOKUP_CREATE))
+	if (flags & (LOOKUP_PARENT | LOOKUP_OPEN | LOOKUP_CREATE))
 		itp = NULL;
 	else
 		itp = &it;
-	de = ll_lookup_it(parent, dentry, itp, 0);
+	de = ll_lookup_it(parent, dentry, itp);
 
 	if (itp)
 		ll_intent_release(itp);
@@ -580,27 +652,43 @@ static int ll_atomic_open(struct inode *dir, struct dentry *dentry,
 {
 	struct lookup_intent *it;
 	struct dentry *de;
-	long long lookup_flags = LOOKUP_OPEN;
 	int rc = 0;
 
 	CDEBUG(D_VFSTRACE, "VFS Op:name=%pd, dir="DFID"(%p),file %p,open_flags %x,mode %x opened %d\n",
 	       dentry, PFID(ll_inode2fid(dir)), dir, file, open_flags, mode,
 	       *opened);
 
+	/* Only negative dentries enter here */
+	LASSERT(!d_inode(dentry));
+
+	if (!d_in_lookup(dentry)) {
+		/* A valid negative dentry that just passed revalidation,
+		 * there's little point to try and open it server-side,
+		 * even though there's a minuscle chance it might succeed.
+		 * Either way it's a valid race to just return -ENOENT here.
+		 */
+		if (!(open_flags & O_CREAT))
+			return -ENOENT;
+
+		/* Otherwise we just unhash it to be rehashed afresh via
+		 * lookup if necessary
+		 */
+		d_drop(dentry);
+	}
+
 	it = kzalloc(sizeof(*it), GFP_NOFS);
 	if (!it)
 		return -ENOMEM;
 
 	it->it_op = IT_OPEN;
-	if (open_flags & O_CREAT) {
+	if (open_flags & O_CREAT)
 		it->it_op |= IT_CREAT;
-		lookup_flags |= LOOKUP_CREATE;
-	}
 	it->it_create_mode = (mode & S_IALLUGO) | S_IFREG;
 	it->it_flags = (open_flags & ~O_ACCMODE) | OPEN_FMODE(open_flags);
+	it->it_flags &= ~MDS_OPEN_FL_INTERNAL;
 
 	/* Dentry added to dcache tree in ll_lookup_it */
-	de = ll_lookup_it(dir, dentry, it, lookup_flags);
+	de = ll_lookup_it(dir, dentry, it);
 	if (IS_ERR(de))
 		rc = PTR_ERR(de);
 	else if (de)
@@ -609,7 +697,7 @@ static int ll_atomic_open(struct inode *dir, struct dentry *dentry,
 	if (!rc) {
 		if (it_disposition(it, DISP_OPEN_CREATE)) {
 			/* Dentry instantiated in ll_create_it. */
-			rc = ll_create_it(dir, dentry, mode, it);
+			rc = ll_create_it(dir, dentry, it);
 			if (rc) {
 				/* We dget in ll_splice_alias. */
 				if (de)
@@ -695,7 +783,7 @@ static struct inode *ll_create_node(struct inode *dir, struct lookup_intent *it)
  * If the create succeeds, we fill in the inode information
  * with d_instantiate().
  */
-static int ll_create_it(struct inode *dir, struct dentry *dentry, int mode,
+static int ll_create_it(struct inode *dir, struct dentry *dentry,
 			struct lookup_intent *it)
 {
 	struct inode *inode;
@@ -716,27 +804,26 @@ static int ll_create_it(struct inode *dir, struct dentry *dentry, int mode,
 	return 0;
 }
 
-static void ll_update_times(struct ptlrpc_request *request,
-			    struct inode *inode)
+void ll_update_times(struct ptlrpc_request *request, struct inode *inode)
 {
 	struct mdt_body *body = req_capsule_server_get(&request->rq_pill,
 						       &RMF_MDT_BODY);
 
 	LASSERT(body);
-	if (body->valid & OBD_MD_FLMTIME &&
-	    body->mtime > LTIME_S(inode->i_mtime)) {
+	if (body->mbo_valid & OBD_MD_FLMTIME &&
+	    body->mbo_mtime > LTIME_S(inode->i_mtime)) {
 		CDEBUG(D_INODE, "setting fid "DFID" mtime from %lu to %llu\n",
 		       PFID(ll_inode2fid(inode)), LTIME_S(inode->i_mtime),
-		       body->mtime);
-		LTIME_S(inode->i_mtime) = body->mtime;
+		       body->mbo_mtime);
+		LTIME_S(inode->i_mtime) = body->mbo_mtime;
 	}
-	if (body->valid & OBD_MD_FLCTIME &&
-	    body->ctime > LTIME_S(inode->i_ctime))
-		LTIME_S(inode->i_ctime) = body->ctime;
+	if (body->mbo_valid & OBD_MD_FLCTIME &&
+	    body->mbo_ctime > LTIME_S(inode->i_ctime))
+		LTIME_S(inode->i_ctime) = body->mbo_ctime;
 }
 
 static int ll_new_node(struct inode *dir, struct dentry *dentry,
-		       const char *tgt, int mode, int rdev,
+		       const char *tgt, umode_t mode, int rdev,
 		       __u32 opc)
 {
 	struct ptlrpc_request *request = NULL;
@@ -748,7 +835,7 @@ static int ll_new_node(struct inode *dir, struct dentry *dentry,
 
 	if (unlikely(tgt))
 		tgt_len = strlen(tgt) + 1;
-
+again:
 	op_data = ll_prep_md_op_data(NULL, dir, NULL,
 				     dentry->d_name.name,
 				     dentry->d_name.len,
@@ -763,9 +850,45 @@ static int ll_new_node(struct inode *dir, struct dentry *dentry,
 			from_kgid(&init_user_ns, current_fsgid()),
 			cfs_curproc_cap_pack(), rdev, &request);
 	ll_finish_md_op_data(op_data);
-	if (err)
+	if (err < 0 && err != -EREMOTE)
 		goto err_exit;
 
+	/*
+	 * If the client doesn't know where to create a subdirectory (or
+	 * in case of a race that sends the RPC to the wrong MDS), the
+	 * MDS will return -EREMOTE and the client will fetch the layout
+	 * of the directory, then create the directory on the right MDT.
+	 */
+	if (unlikely(err == -EREMOTE)) {
+		struct ll_inode_info *lli = ll_i2info(dir);
+		struct lmv_user_md *lum;
+		int lumsize, err2;
+
+		ptlrpc_req_finished(request);
+		request = NULL;
+
+		err2 = ll_dir_getstripe(dir, (void **)&lum, &lumsize, &request,
+					OBD_MD_DEFAULT_MEA);
+		if (!err2) {
+			/* Update stripe_offset and retry */
+			lli->lli_def_stripe_offset = lum->lum_stripe_offset;
+		} else if (err2 == -ENODATA &&
+			   lli->lli_def_stripe_offset != -1) {
+			/*
+			 * If there are no default stripe EA on the MDT, but the
+			 * client has default stripe, then it probably means
+			 * default stripe EA has just been deleted.
+			 */
+			lli->lli_def_stripe_offset = -1;
+		} else {
+			goto err_exit;
+		}
+
+		ptlrpc_req_finished(request);
+		request = NULL;
+		goto again;
+	}
+
 	ll_update_times(request, dir);
 
 	err = ll_prep_inode(&inode, request, dir->i_sb, NULL);
@@ -774,7 +897,8 @@ static int ll_new_node(struct inode *dir, struct dentry *dentry,
 
 	d_instantiate(dentry, inode);
 err_exit:
-	ptlrpc_req_finished(request);
+	if (request)
+		ptlrpc_req_finished(request);
 
 	return err;
 }
@@ -837,77 +961,6 @@ static int ll_create_nd(struct inode *dir, struct dentry *dentry,
 	return rc;
 }
 
-int ll_objects_destroy(struct ptlrpc_request *request, struct inode *dir)
-{
-	struct mdt_body *body;
-	struct lov_mds_md *eadata;
-	struct lov_stripe_md *lsm = NULL;
-	struct obd_trans_info oti = { 0 };
-	struct obdo *oa;
-	int rc;
-
-	/* req is swabbed so this is safe */
-	body = req_capsule_server_get(&request->rq_pill, &RMF_MDT_BODY);
-	if (!(body->valid & OBD_MD_FLEASIZE))
-		return 0;
-
-	if (body->eadatasize == 0) {
-		CERROR("OBD_MD_FLEASIZE set but eadatasize zero\n");
-		rc = -EPROTO;
-		goto out;
-	}
-
-	/* The MDS sent back the EA because we unlinked the last reference
-	 * to this file. Use this EA to unlink the objects on the OST.
-	 * It's opaque so we don't swab here; we leave it to obd_unpackmd() to
-	 * check it is complete and sensible.
-	 */
-	eadata = req_capsule_server_sized_get(&request->rq_pill, &RMF_MDT_MD,
-					      body->eadatasize);
-	LASSERT(eadata);
-
-	rc = obd_unpackmd(ll_i2dtexp(dir), &lsm, eadata, body->eadatasize);
-	if (rc < 0) {
-		CERROR("obd_unpackmd: %d\n", rc);
-		goto out;
-	}
-	LASSERT(rc >= sizeof(*lsm));
-
-	oa = kmem_cache_zalloc(obdo_cachep, GFP_NOFS);
-	if (!oa) {
-		rc = -ENOMEM;
-		goto out_free_memmd;
-	}
-
-	oa->o_oi = lsm->lsm_oi;
-	oa->o_mode = body->mode & S_IFMT;
-	oa->o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLGROUP;
-
-	if (body->valid & OBD_MD_FLCOOKIE) {
-		oa->o_valid |= OBD_MD_FLCOOKIE;
-		oti.oti_logcookies =
-			req_capsule_server_sized_get(&request->rq_pill,
-						     &RMF_LOGCOOKIES,
-						   sizeof(struct llog_cookie) *
-						     lsm->lsm_stripe_count);
-		if (!oti.oti_logcookies) {
-			oa->o_valid &= ~OBD_MD_FLCOOKIE;
-			body->valid &= ~OBD_MD_FLCOOKIE;
-		}
-	}
-
-	rc = obd_destroy(NULL, ll_i2dtexp(dir), oa, lsm, &oti,
-			 ll_i2mdexp(dir));
-	if (rc)
-		CERROR("obd destroy objid "DOSTID" error %d\n",
-		       POSTID(&lsm->lsm_oi), rc);
-out_free_memmd:
-	obd_free_memmd(ll_i2dtexp(dir), &lsm);
-	kmem_cache_free(obdo_cachep, oa);
-out:
-	return rc;
-}
-
 /* ll_unlink() doesn't update the inode with the new link count.
  * Instead, ll_ddelete() and ll_d_iput() will update it based upon if there
  * is any lock existing. They will recycle dentries and inodes based upon locks
@@ -929,7 +982,7 @@ static int ll_unlink(struct inode *dir, struct dentry *dchild)
 	if (IS_ERR(op_data))
 		return PTR_ERR(op_data);
 
-	if (dchild && dchild->d_inode)
+	if (dchild->d_inode)
 		op_data->op_fid3 = *ll_inode2fid(dchild->d_inode);
 
 	op_data->op_fid2 = op_data->op_fid3;
@@ -941,7 +994,6 @@ static int ll_unlink(struct inode *dir, struct dentry *dchild)
 	ll_update_times(request, dir);
 	ll_stats_ops_tally(ll_i2sbi(dir), LPROC_LL_UNLINK, 1);
 
-	rc = ll_objects_destroy(request, dir);
  out:
 	ptlrpc_req_finished(request);
 	return rc;
@@ -956,9 +1008,9 @@ static int ll_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
 
 	if (!IS_POSIXACL(dir) || !exp_connect_umask(ll_i2mdexp(dir)))
 		mode &= ~current_umask();
-	mode = (mode & (S_IRWXUGO|S_ISVTX)) | S_IFDIR;
-	err = ll_new_node(dir, dentry, NULL, mode, 0, LUSTRE_OPC_MKDIR);
+	mode = (mode & (S_IRWXUGO | S_ISVTX)) | S_IFDIR;
 
+	err = ll_new_node(dir, dentry, NULL, mode, 0, LUSTRE_OPC_MKDIR);
 	if (!err)
 		ll_stats_ops_tally(ll_i2sbi(dir), LPROC_LL_MKDIR, 1);
 
@@ -981,7 +1033,7 @@ static int ll_rmdir(struct inode *dir, struct dentry *dchild)
 	if (IS_ERR(op_data))
 		return PTR_ERR(op_data);
 
-	if (dchild && dchild->d_inode)
+	if (dchild->d_inode)
 		op_data->op_fid3 = *ll_inode2fid(dchild->d_inode);
 
 	op_data->op_fid2 = op_data->op_fid3;
@@ -1045,13 +1097,17 @@ out:
 }
 
 static int ll_rename(struct inode *src, struct dentry *src_dchild,
-		     struct inode *tgt, struct dentry *tgt_dchild)
+		     struct inode *tgt, struct dentry *tgt_dchild,
+		     unsigned int flags)
 {
 	struct ptlrpc_request *request = NULL;
 	struct ll_sb_info *sbi = ll_i2sbi(src);
 	struct md_op_data *op_data;
 	int err;
 
+	if (flags)
+		return -EINVAL;
+
 	CDEBUG(D_VFSTRACE,
 	       "VFS Op:oldname=%pd, src_dir="DFID"(%p), newname=%pd, tgt_dir="DFID"(%p)\n",
 	       src_dchild, PFID(ll_inode2fid(src)), src,
@@ -1062,9 +1118,9 @@ static int ll_rename(struct inode *src, struct dentry *src_dchild,
 	if (IS_ERR(op_data))
 		return PTR_ERR(op_data);
 
-	if (src_dchild && src_dchild->d_inode)
+	if (src_dchild->d_inode)
 		op_data->op_fid3 = *ll_inode2fid(src_dchild->d_inode);
-	if (tgt_dchild && tgt_dchild->d_inode)
+	if (tgt_dchild->d_inode)
 		op_data->op_fid4 = *ll_inode2fid(tgt_dchild->d_inode);
 
 	err = md_rename(sbi->ll_md_exp, op_data,
@@ -1077,7 +1133,6 @@ static int ll_rename(struct inode *src, struct dentry *src_dchild,
 		ll_update_times(request, src);
 		ll_update_times(request, tgt);
 		ll_stats_ops_tally(sbi, LPROC_LL_RENAME, 1);
-		err = ll_objects_destroy(request, src);
 	}
 
 	ptlrpc_req_finished(request);
@@ -1097,14 +1152,11 @@ const struct inode_operations ll_dir_inode_operations = {
 	.rmdir	      = ll_rmdir,
 	.symlink	    = ll_symlink,
 	.link	       = ll_link,
-	.rename	     = ll_rename,
+	.rename		= ll_rename,
 	.setattr	    = ll_setattr,
 	.getattr	    = ll_getattr,
 	.permission	 = ll_inode_permission,
-	.setxattr	   = ll_setxattr,
-	.getxattr	   = ll_getxattr,
 	.listxattr	  = ll_listxattr,
-	.removexattr	= ll_removexattr,
 	.get_acl	    = ll_get_acl,
 };
 
@@ -1112,9 +1164,6 @@ const struct inode_operations ll_special_inode_operations = {
 	.setattr	= ll_setattr,
 	.getattr	= ll_getattr,
 	.permission     = ll_inode_permission,
-	.setxattr       = ll_setxattr,
-	.getxattr       = ll_getxattr,
 	.listxattr      = ll_listxattr,
-	.removexattr    = ll_removexattr,
 	.get_acl	    = ll_get_acl,
 };
diff --git a/drivers/staging/lustre/lustre/llite/range_lock.c b/drivers/staging/lustre/lustre/llite/range_lock.c
new file mode 100644
index 000000000000..94c818f1478b
--- /dev/null
+++ b/drivers/staging/lustre/lustre/llite/range_lock.c
@@ -0,0 +1,233 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Range lock is used to allow multiple threads writing a single shared
+ * file given each thread is writing to a non-overlapping portion of the
+ * file.
+ *
+ * Refer to the possible upstream kernel version of range lock by
+ * Jan Kara <jack@suse.cz>: https://lkml.org/lkml/2013/1/31/480
+ *
+ * This file could later replaced by the upstream kernel version.
+ */
+/*
+ * Author: Prakash Surya <surya1@llnl.gov>
+ * Author: Bobi Jam <bobijam.xu@intel.com>
+ */
+#include "range_lock.h"
+#include "../include/lustre/lustre_user.h"
+
+/**
+ * Initialize a range lock tree
+ *
+ * \param tree [in]	an empty range lock tree
+ *
+ * Pre:  Caller should have allocated the range lock tree.
+ * Post: The range lock tree is ready to function.
+ */
+void range_lock_tree_init(struct range_lock_tree *tree)
+{
+	tree->rlt_root = NULL;
+	tree->rlt_sequence = 0;
+	spin_lock_init(&tree->rlt_lock);
+}
+
+/**
+ * Initialize a range lock node
+ *
+ * \param lock  [in]	an empty range lock node
+ * \param start [in]	start of the covering region
+ * \param end   [in]	end of the covering region
+ *
+ * Pre:  Caller should have allocated the range lock node.
+ * Post: The range lock node is meant to cover [start, end] region
+ */
+void range_lock_init(struct range_lock *lock, __u64 start, __u64 end)
+{
+	memset(&lock->rl_node, 0, sizeof(lock->rl_node));
+	if (end != LUSTRE_EOF)
+		end >>= PAGE_SHIFT;
+	interval_set(&lock->rl_node, start >> PAGE_SHIFT, end);
+	INIT_LIST_HEAD(&lock->rl_next_lock);
+	lock->rl_task = NULL;
+	lock->rl_lock_count = 0;
+	lock->rl_blocking_ranges = 0;
+	lock->rl_sequence = 0;
+}
+
+static inline struct range_lock *next_lock(struct range_lock *lock)
+{
+	return list_entry(lock->rl_next_lock.next, typeof(*lock), rl_next_lock);
+}
+
+/**
+ * Helper function of range_unlock()
+ *
+ * \param node [in]	a range lock found overlapped during interval node
+ *			search
+ * \param arg [in]	the range lock to be tested
+ *
+ * \retval INTERVAL_ITER_CONT	indicate to continue the search for next
+ *				overlapping range node
+ * \retval INTERVAL_ITER_STOP	indicate to stop the search
+ */
+static enum interval_iter range_unlock_cb(struct interval_node *node, void *arg)
+{
+	struct range_lock *lock = arg;
+	struct range_lock *overlap = node2rangelock(node);
+	struct range_lock *iter;
+
+	list_for_each_entry(iter, &overlap->rl_next_lock, rl_next_lock) {
+		if (iter->rl_sequence > lock->rl_sequence) {
+			--iter->rl_blocking_ranges;
+			LASSERT(iter->rl_blocking_ranges > 0);
+		}
+	}
+	if (overlap->rl_sequence > lock->rl_sequence) {
+		--overlap->rl_blocking_ranges;
+		if (overlap->rl_blocking_ranges == 0)
+			wake_up_process(overlap->rl_task);
+	}
+	return INTERVAL_ITER_CONT;
+}
+
+/**
+ * Unlock a range lock, wake up locks blocked by this lock.
+ *
+ * \param tree [in]	range lock tree
+ * \param lock [in]	range lock to be deleted
+ *
+ * If this lock has been granted, relase it; if not, just delete it from
+ * the tree or the same region lock list. Wake up those locks only blocked
+ * by this lock through range_unlock_cb().
+ */
+void range_unlock(struct range_lock_tree *tree, struct range_lock *lock)
+{
+	spin_lock(&tree->rlt_lock);
+	if (!list_empty(&lock->rl_next_lock)) {
+		struct range_lock *next;
+
+		if (interval_is_intree(&lock->rl_node)) { /* first lock */
+			/* Insert the next same range lock into the tree */
+			next = next_lock(lock);
+			next->rl_lock_count = lock->rl_lock_count - 1;
+			interval_erase(&lock->rl_node, &tree->rlt_root);
+			interval_insert(&next->rl_node, &tree->rlt_root);
+		} else {
+			/* find the first lock in tree */
+			list_for_each_entry(next, &lock->rl_next_lock,
+					    rl_next_lock) {
+				if (!interval_is_intree(&next->rl_node))
+					continue;
+
+				LASSERT(next->rl_lock_count > 0);
+				next->rl_lock_count--;
+				break;
+			}
+		}
+		list_del_init(&lock->rl_next_lock);
+	} else {
+		LASSERT(interval_is_intree(&lock->rl_node));
+		interval_erase(&lock->rl_node, &tree->rlt_root);
+	}
+
+	interval_search(tree->rlt_root, &lock->rl_node.in_extent,
+			range_unlock_cb, lock);
+	spin_unlock(&tree->rlt_lock);
+}
+
+/**
+ * Helper function of range_lock()
+ *
+ * \param node [in]	a range lock found overlapped during interval node
+ *			search
+ * \param arg [in]	the range lock to be tested
+ *
+ * \retval INTERVAL_ITER_CONT	indicate to continue the search for next
+ *				overlapping range node
+ * \retval INTERVAL_ITER_STOP	indicate to stop the search
+ */
+static enum interval_iter range_lock_cb(struct interval_node *node, void *arg)
+{
+	struct range_lock *lock = (struct range_lock *)arg;
+	struct range_lock *overlap = node2rangelock(node);
+
+	lock->rl_blocking_ranges += overlap->rl_lock_count + 1;
+	return INTERVAL_ITER_CONT;
+}
+
+/**
+ * Lock a region
+ *
+ * \param tree [in]	range lock tree
+ * \param lock [in]	range lock node containing the region span
+ *
+ * \retval 0	get the range lock
+ * \retval <0	error code while not getting the range lock
+ *
+ * If there exists overlapping range lock, the new lock will wait and
+ * retry, if later it find that it is not the chosen one to wake up,
+ * it wait again.
+ */
+int range_lock(struct range_lock_tree *tree, struct range_lock *lock)
+{
+	struct interval_node *node;
+	int rc = 0;
+
+	spin_lock(&tree->rlt_lock);
+	/*
+	 * We need to check for all conflicting intervals
+	 * already in the tree.
+	 */
+	interval_search(tree->rlt_root, &lock->rl_node.in_extent,
+			range_lock_cb, lock);
+	/*
+	 * Insert to the tree if I am unique, otherwise I've been linked to
+	 * the rl_next_lock of another lock which has the same range as mine
+	 * in range_lock_cb().
+	 */
+	node = interval_insert(&lock->rl_node, &tree->rlt_root);
+	if (node) {
+		struct range_lock *tmp = node2rangelock(node);
+
+		list_add_tail(&lock->rl_next_lock, &tmp->rl_next_lock);
+		tmp->rl_lock_count++;
+	}
+	lock->rl_sequence = ++tree->rlt_sequence;
+
+	while (lock->rl_blocking_ranges > 0) {
+		lock->rl_task = current;
+		__set_current_state(TASK_INTERRUPTIBLE);
+		spin_unlock(&tree->rlt_lock);
+		schedule();
+
+		if (signal_pending(current)) {
+			range_unlock(tree, lock);
+			rc = -EINTR;
+			goto out;
+		}
+		spin_lock(&tree->rlt_lock);
+	}
+	spin_unlock(&tree->rlt_lock);
+out:
+	return rc;
+}
diff --git a/drivers/staging/lustre/lustre/llite/range_lock.h b/drivers/staging/lustre/lustre/llite/range_lock.h
new file mode 100644
index 000000000000..c6d04a6f99fd
--- /dev/null
+++ b/drivers/staging/lustre/lustre/llite/range_lock.h
@@ -0,0 +1,82 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Range lock is used to allow multiple threads writing a single shared
+ * file given each thread is writing to a non-overlapping portion of the
+ * file.
+ *
+ * Refer to the possible upstream kernel version of range lock by
+ * Jan Kara <jack@suse.cz>: https://lkml.org/lkml/2013/1/31/480
+ *
+ * This file could later replaced by the upstream kernel version.
+ */
+/*
+ * Author: Prakash Surya <surya1@llnl.gov>
+ * Author: Bobi Jam <bobijam.xu@intel.com>
+ */
+#ifndef _RANGE_LOCK_H
+#define _RANGE_LOCK_H
+
+#include "../../include/linux/libcfs/libcfs.h"
+#include "../include/interval_tree.h"
+
+struct range_lock {
+	struct interval_node	rl_node;
+	/**
+	 * Process to enqueue this lock.
+	 */
+	struct task_struct	*rl_task;
+	/**
+	 * List of locks with the same range.
+	 */
+	struct list_head	rl_next_lock;
+	/**
+	 * Number of locks in the list rl_next_lock
+	 */
+	unsigned int		rl_lock_count;
+	/**
+	 * Number of ranges which are blocking acquisition of the lock
+	 */
+	unsigned int		rl_blocking_ranges;
+	/**
+	 * Sequence number of range lock. This number is used to get to know
+	 * the order the locks are queued; this is required for range_cancel().
+	 */
+	__u64			rl_sequence;
+};
+
+static inline struct range_lock *node2rangelock(const struct interval_node *n)
+{
+	return container_of(n, struct range_lock, rl_node);
+}
+
+struct range_lock_tree {
+	struct interval_node	*rlt_root;
+	spinlock_t		 rlt_lock;	/* protect range lock tree */
+	__u64			 rlt_sequence;
+};
+
+void range_lock_tree_init(struct range_lock_tree *tree);
+void range_lock_init(struct range_lock *lock, __u64 start, __u64 end);
+int  range_lock(struct range_lock_tree *tree, struct range_lock *lock);
+void range_unlock(struct range_lock_tree *tree, struct range_lock *lock);
+#endif
diff --git a/drivers/staging/lustre/lustre/llite/rw.c b/drivers/staging/lustre/lustre/llite/rw.c
index 87393c4bd51e..50c0152ba022 100644
--- a/drivers/staging/lustre/lustre/llite/rw.c
+++ b/drivers/staging/lustre/lustre/llite/rw.c
@@ -50,10 +50,8 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/lustre_lite.h"
 #include "../include/obd_cksum.h"
 #include "llite_internal.h"
-#include "../include/linux/lustre_compat25.h"
 
 static void ll_ra_stats_inc_sbi(struct ll_sb_info *sbi, enum ra_stat which);
 
@@ -413,7 +411,7 @@ static int ll_read_ahead_pages(const struct lu_env *env,
 			 * forward read-ahead, it will be fixed when backward
 			 * read-ahead is implemented
 			 */
-			LASSERTF(page_idx > ria->ria_stoff, "Invalid page_idx %lu rs %lu re %lu ro %lu rl %lu rp %lu\n",
+			LASSERTF(page_idx >= ria->ria_stoff, "Invalid page_idx %lu rs %lu re %lu ro %lu rl %lu rp %lu\n",
 				 page_idx,
 				 ria->ria_start, ria->ria_end, ria->ria_stoff,
 				 ria->ria_length, ria->ria_pages);
@@ -474,10 +472,22 @@ int ll_readahead(const struct lu_env *env, struct cl_io *io,
 	}
 
 	/* Reserve a part of the read-ahead window that we'll be issuing */
-	if (ras->ras_window_len) {
-		start = ras->ras_next_readahead;
+	if (ras->ras_window_len > 0) {
+		/*
+		 * Note: other thread might rollback the ras_next_readahead,
+		 * if it can not get the full size of prepared pages, see the
+		 * end of this function. For stride read ahead, it needs to
+		 * make sure the offset is no less than ras_stride_offset,
+		 * so that stride read ahead can work correctly.
+		 */
+		if (stride_io_mode(ras))
+			start = max(ras->ras_next_readahead,
+				    ras->ras_stride_offset);
+		else
+			start = ras->ras_next_readahead;
 		end = ras->ras_window_start + ras->ras_window_len - 1;
 	}
+
 	if (end != 0) {
 		unsigned long rpc_boundary;
 		/*
@@ -648,10 +658,11 @@ static void ras_update_stride_detector(struct ll_readahead_state *ras,
 {
 	unsigned long stride_gap = index - ras->ras_last_readpage - 1;
 
-	if (!stride_io_mode(ras) && (stride_gap != 0 ||
-	    ras->ras_consecutive_stride_requests == 0)) {
+	if ((stride_gap != 0 || ras->ras_consecutive_stride_requests == 0) &&
+	    !stride_io_mode(ras)) {
 		ras->ras_stride_pages = ras->ras_consecutive_pages;
-		ras->ras_stride_length = stride_gap+ras->ras_consecutive_pages;
+		ras->ras_stride_length = ras->ras_consecutive_pages +
+					 stride_gap;
 	}
 	LASSERT(ras->ras_request_index == 0);
 	LASSERT(ras->ras_consecutive_stride_requests == 0);
@@ -663,10 +674,9 @@ static void ras_update_stride_detector(struct ll_readahead_state *ras,
 	}
 
 	ras->ras_stride_pages = ras->ras_consecutive_pages;
-	ras->ras_stride_length = stride_gap+ras->ras_consecutive_pages;
+	ras->ras_stride_length = stride_gap + ras->ras_consecutive_pages;
 
 	RAS_CDEBUG(ras);
-	return;
 }
 
 /* Stride Read-ahead window will be increased inc_len according to
@@ -882,7 +892,6 @@ out_unlock:
 	RAS_CDEBUG(ras);
 	ras->ras_request_index++;
 	spin_unlock(&ras->ras_lock);
-	return;
 }
 
 int ll_writepage(struct page *vmpage, struct writeback_control *wbc)
@@ -1015,11 +1024,15 @@ int ll_writepages(struct address_space *mapping, struct writeback_control *wbc)
 		 * is called later on.
 		 */
 		ignore_layout = 1;
+
+	if (!ll_i2info(inode)->lli_clob)
+		return 0;
+
 	result = cl_sync_file_range(inode, start, end, mode, ignore_layout);
 	if (result > 0) {
 		wbc->nr_to_write -= result;
 		result = 0;
-	 }
+	}
 
 	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) {
 		if (end == OBD_OBJECT_EOF)
diff --git a/drivers/staging/lustre/lustre/llite/rw26.c b/drivers/staging/lustre/lustre/llite/rw26.c
index d98c7acc0832..26f3a37873a7 100644
--- a/drivers/staging/lustre/lustre/llite/rw26.c
+++ b/drivers/staging/lustre/lustre/llite/rw26.c
@@ -51,9 +51,7 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/lustre_lite.h"
 #include "llite_internal.h"
-#include "../include/linux/lustre_compat25.h"
 
 /**
  * Implements Linux VM address_space::invalidatepage() method. This method is
@@ -161,7 +159,7 @@ static int ll_releasepage(struct page *vmpage, gfp_t gfp_mask)
 	return result;
 }
 
-#define MAX_DIRECTIO_SIZE (2*1024*1024*1024UL)
+#define MAX_DIRECTIO_SIZE (2 * 1024 * 1024 * 1024UL)
 
 static inline int ll_get_user_pages(int rw, unsigned long user_addr,
 				    size_t size, struct page ***pages,
@@ -214,10 +212,10 @@ ssize_t ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io,
 	int i;
 	ssize_t rc = 0;
 	loff_t file_offset  = pv->ldp_start_offset;
-	long size	   = pv->ldp_size;
+	size_t size = pv->ldp_size;
 	int page_count      = pv->ldp_nr;
 	struct page **pages = pv->ldp_pages;
-	long page_size      = cl_page_size(obj);
+	size_t page_size = cl_page_size(obj);
 	bool do_io;
 	int  io_pages       = 0;
 
@@ -346,7 +344,6 @@ static ssize_t ll_direct_IO_26(struct kiocb *iocb, struct iov_iter *iter)
 	struct cl_io *io;
 	struct file *file = iocb->ki_filp;
 	struct inode *inode = file->f_mapping->host;
-	struct vvp_object *obj = cl_inode2vvp(inode);
 	loff_t file_offset = iocb->ki_pos;
 	ssize_t count = iov_iter_count(iter);
 	ssize_t tot_bytes = 0, result = 0;
@@ -375,14 +372,6 @@ static ssize_t ll_direct_IO_26(struct kiocb *iocb, struct iov_iter *iter)
 	io = vvp_env_io(env)->vui_cl.cis_io;
 	LASSERT(io);
 
-	/* 0. Need locking between buffered and direct access. and race with
-	 *    size changing by concurrent truncates and writes.
-	 * 1. Need inode mutex to operate transient pages.
-	 */
-	if (iov_iter_rw(iter) == READ)
-		inode_lock(inode);
-
-	LASSERT(obj->vob_transient_pages == 0);
 	while (iov_iter_count(iter)) {
 		struct page **pages;
 		size_t offs;
@@ -430,10 +419,6 @@ static ssize_t ll_direct_IO_26(struct kiocb *iocb, struct iov_iter *iter)
 		file_offset += result;
 	}
 out:
-	LASSERT(obj->vob_transient_pages == 0);
-	if (iov_iter_rw(iter) == READ)
-		inode_unlock(inode);
-
 	if (tot_bytes > 0) {
 		struct vvp_io *vio = vvp_env_io(env);
 
@@ -616,6 +601,13 @@ static int ll_write_end(struct file *file, struct address_space *mapping,
 			LASSERT(from == 0);
 		vio->u.write.vui_to = from + copied;
 
+		/*
+		 * To address the deadlock in balance_dirty_pages() where
+		 * this dirty page may be written back in the same thread.
+		 */
+		if (PageDirty(vmpage))
+			unplug = true;
+
 		/* We may have one full RPC, commit it soon */
 		if (plist->pl_nr >= PTLRPC_MAX_BRW_PAGES)
 			unplug = true;
diff --git a/drivers/staging/lustre/lustre/llite/statahead.c b/drivers/staging/lustre/lustre/llite/statahead.c
index c1cb6b19e724..0677513476ec 100644
--- a/drivers/staging/lustre/lustre/llite/statahead.c
+++ b/drivers/staging/lustre/lustre/llite/statahead.c
@@ -39,7 +39,6 @@
 #define DEBUG_SUBSYSTEM S_LLITE
 
 #include "../include/obd_support.h"
-#include "../include/lustre_lite.h"
 #include "../include/lustre_dlm.h"
 #include "llite_internal.h"
 
@@ -50,24 +49,26 @@ enum se_stat {
 	SA_ENTRY_INIT = 0,      /** init entry */
 	SA_ENTRY_SUCC = 1,      /** stat succeed */
 	SA_ENTRY_INVA = 2,      /** invalid entry */
-	SA_ENTRY_DEST = 3,      /** entry to be destroyed */
 };
 
-struct ll_sa_entry {
-	/* link into sai->sai_entries */
-	struct list_head	      se_link;
-	/* link into sai->sai_entries_{received,stated} */
+/*
+ * sa_entry is not refcounted: statahead thread allocates it and do async stat,
+ * and in async stat callback ll_statahead_interpret() will add it into
+ * sai_interim_entries, later statahead thread will call sa_handle_callback() to
+ * instantiate entry and move it into sai_entries, and then only scanner process
+ * can access and free it.
+ */
+struct sa_entry {
+	/* link into sai_interim_entries or sai_entries */
 	struct list_head	      se_list;
 	/* link into sai hash table locally */
 	struct list_head	      se_hash;
-	/* entry reference count */
-	atomic_t	    se_refcount;
 	/* entry index in the sai */
 	__u64		   se_index;
 	/* low layer ldlm lock handle */
 	__u64		   se_handle;
 	/* entry status */
-	enum se_stat	   se_stat;
+	enum se_stat		se_state;
 	/* entry size, contains name */
 	int		     se_size;
 	/* pointer to async getattr enqueue info */
@@ -83,27 +84,24 @@ struct ll_sa_entry {
 static unsigned int sai_generation;
 static DEFINE_SPINLOCK(sai_generation_lock);
 
-/*
- * The entry only can be released by the caller, it is necessary to hold lock.
- */
-static inline int ll_sa_entry_stated(struct ll_sa_entry *entry)
+/* sa_entry is ready to use */
+static inline int sa_ready(struct sa_entry *entry)
 {
 	smp_rmb();
-	return (entry->se_stat != SA_ENTRY_INIT);
+	return (entry->se_state != SA_ENTRY_INIT);
 }
 
-static inline int ll_sa_entry_hash(int val)
+/* hash value to put in sai_cache */
+static inline int sa_hash(int val)
 {
 	return val & LL_SA_CACHE_MASK;
 }
 
-/*
- * Insert entry to hash SA table.
- */
+/* hash entry into sai_cache */
 static inline void
-ll_sa_entry_enhash(struct ll_statahead_info *sai, struct ll_sa_entry *entry)
+sa_rehash(struct ll_statahead_info *sai, struct sa_entry *entry)
 {
-	int i = ll_sa_entry_hash(entry->se_qstr.hash);
+	int i = sa_hash(entry->se_qstr.hash);
 
 	spin_lock(&sai->sai_cache_lock[i]);
 	list_add_tail(&entry->se_hash, &sai->sai_cache[i]);
@@ -114,9 +112,9 @@ ll_sa_entry_enhash(struct ll_statahead_info *sai, struct ll_sa_entry *entry)
  * Remove entry from SA table.
  */
 static inline void
-ll_sa_entry_unhash(struct ll_statahead_info *sai, struct ll_sa_entry *entry)
+sa_unhash(struct ll_statahead_info *sai, struct sa_entry *entry)
 {
-	int i = ll_sa_entry_hash(entry->se_qstr.hash);
+	int i = sa_hash(entry->se_qstr.hash);
 
 	spin_lock(&sai->sai_cache_lock[i]);
 	list_del_init(&entry->se_hash);
@@ -129,19 +127,21 @@ static inline int agl_should_run(struct ll_statahead_info *sai,
 	return (inode && S_ISREG(inode->i_mode) && sai->sai_agl_valid);
 }
 
+/* statahead window is full */
 static inline int sa_sent_full(struct ll_statahead_info *sai)
 {
 	return atomic_read(&sai->sai_cache_count) >= sai->sai_max;
 }
 
-static inline int sa_received_empty(struct ll_statahead_info *sai)
+/* got async stat replies */
+static inline int sa_has_callback(struct ll_statahead_info *sai)
 {
-	return list_empty(&sai->sai_entries_received);
+	return !list_empty(&sai->sai_interim_entries);
 }
 
 static inline int agl_list_empty(struct ll_statahead_info *sai)
 {
-	return list_empty(&sai->sai_entries_agl);
+	return list_empty(&sai->sai_agls);
 }
 
 /**
@@ -157,7 +157,7 @@ static inline int sa_low_hit(struct ll_statahead_info *sai)
 }
 
 /*
- * If the given index is behind of statahead window more than
+ * if the given index is behind of statahead window more than
  * SA_OMITTED_ENTRY_MAX, then it is old.
  */
 static inline int is_omitted_entry(struct ll_statahead_info *sai, __u64 index)
@@ -166,20 +166,17 @@ static inline int is_omitted_entry(struct ll_statahead_info *sai, __u64 index)
 		 sai->sai_index);
 }
 
-/*
- * Insert it into sai_entries tail when init.
- */
-static struct ll_sa_entry *
-ll_sa_entry_alloc(struct dentry *parent,
-		  struct ll_statahead_info *sai, __u64 index,
-		  const char *name, int len)
+/* allocate sa_entry and hash it to allow scanner process to find it */
+static struct sa_entry *
+sa_alloc(struct dentry *parent, struct ll_statahead_info *sai, __u64 index,
+	 const char *name, int len)
 {
 	struct ll_inode_info *lli;
-	struct ll_sa_entry   *entry;
+	struct sa_entry   *entry;
 	int		   entry_size;
 	char		 *dname;
 
-	entry_size = sizeof(struct ll_sa_entry) + (len & ~3) + 4;
+	entry_size = sizeof(struct sa_entry) + (len & ~3) + 4;
 	entry = kzalloc(entry_size, GFP_NOFS);
 	if (unlikely(!entry))
 		return ERR_PTR(-ENOMEM);
@@ -188,34 +185,9 @@ ll_sa_entry_alloc(struct dentry *parent,
 	       len, name, entry, index);
 
 	entry->se_index = index;
-
-	/*
-	 * Statahead entry reference rules:
-	 *
-	 * 1) When statahead entry is initialized, its reference is set as 2.
-	 *    One reference is used by the directory scanner. When the scanner
-	 *    searches the statahead cache for the given name, it can perform
-	 *    lockless hash lookup (only the scanner can remove entry from hash
-	 *    list), and once found, it needn't to call "atomic_inc()" for the
-	 *    entry reference. So the performance is improved. After using the
-	 *    statahead entry, the scanner will call "atomic_dec()" to drop the
-	 *    reference held when initialization. If it is the last reference,
-	 *    the statahead entry will be freed.
-	 *
-	 * 2) All other threads, including statahead thread and ptlrpcd thread,
-	 *    when they process the statahead entry, the reference for target
-	 *    should be held to guarantee the entry will not be released by the
-	 *    directory scanner. After processing the entry, these threads will
-	 *    drop the entry reference. If it is the last reference, the entry
-	 *    will be freed.
-	 *
-	 *    The second reference when initializes the statahead entry is used
-	 *    by the statahead thread, following the rule 2).
-	 */
-	atomic_set(&entry->se_refcount, 2);
-	entry->se_stat = SA_ENTRY_INIT;
+	entry->se_state = SA_ENTRY_INIT;
 	entry->se_size = entry_size;
-	dname = (char *)entry + sizeof(struct ll_sa_entry);
+	dname = (char *)entry + sizeof(struct sa_entry);
 	memcpy(dname, name, len);
 	dname[len] = 0;
 
@@ -223,11 +195,10 @@ ll_sa_entry_alloc(struct dentry *parent,
 	entry->se_qstr.len = len;
 	entry->se_qstr.name = dname;
 
-	lli = ll_i2info(sai->sai_inode);
+	lli = ll_i2info(sai->sai_dentry->d_inode);
 	spin_lock(&lli->lli_sa_lock);
-	list_add_tail(&entry->se_link, &sai->sai_entries);
 	INIT_LIST_HEAD(&entry->se_list);
-	ll_sa_entry_enhash(sai, entry);
+	sa_rehash(sai, entry);
 	spin_unlock(&lli->lli_sa_lock);
 
 	atomic_inc(&sai->sai_cache_count);
@@ -235,18 +206,29 @@ ll_sa_entry_alloc(struct dentry *parent,
 	return entry;
 }
 
+/* free sa_entry, which should have been unhashed and not in any list */
+static void sa_free(struct ll_statahead_info *sai, struct sa_entry *entry)
+{
+	CDEBUG(D_READA, "free sa entry %.*s(%p) index %llu\n",
+	       entry->se_qstr.len, entry->se_qstr.name, entry,
+	       entry->se_index);
+
+	LASSERT(list_empty(&entry->se_list));
+	LASSERT(list_empty(&entry->se_hash));
+
+	kfree(entry);
+	atomic_dec(&sai->sai_cache_count);
+}
+
 /*
- * Used by the directory scanner to search entry with name.
- *
- * Only the caller can remove the entry from hash, so it is unnecessary to hold
- * hash lock. It is caller's duty to release the init refcount on the entry, so
- * it is also unnecessary to increase refcount on the entry.
+ * find sa_entry by name, used by directory scanner, lock is not needed because
+ * only scanner can remove the entry from cache.
  */
-static struct ll_sa_entry *
-ll_sa_entry_get_byname(struct ll_statahead_info *sai, const struct qstr *qstr)
+static struct sa_entry *
+sa_get(struct ll_statahead_info *sai, const struct qstr *qstr)
 {
-	struct ll_sa_entry *entry;
-	int i = ll_sa_entry_hash(qstr->hash);
+	struct sa_entry *entry;
+	int i = sa_hash(qstr->hash);
 
 	list_for_each_entry(entry, &sai->sai_cache[i], se_hash) {
 		if (entry->se_qstr.hash == qstr->hash &&
@@ -257,164 +239,126 @@ ll_sa_entry_get_byname(struct ll_statahead_info *sai, const struct qstr *qstr)
 	return NULL;
 }
 
-/*
- * Used by the async getattr request callback to find entry with index.
- *
- * Inside lli_sa_lock to prevent others to change the list during the search.
- * It needs to increase entry refcount before returning to guarantee that the
- * entry cannot be freed by others.
- */
-static struct ll_sa_entry *
-ll_sa_entry_get_byindex(struct ll_statahead_info *sai, __u64 index)
-{
-	struct ll_sa_entry *entry;
-
-	list_for_each_entry(entry, &sai->sai_entries, se_link) {
-		if (entry->se_index == index) {
-			LASSERT(atomic_read(&entry->se_refcount) > 0);
-			atomic_inc(&entry->se_refcount);
-			return entry;
-		}
-		if (entry->se_index > index)
-			break;
-	}
-	return NULL;
-}
-
-static void ll_sa_entry_cleanup(struct ll_statahead_info *sai,
-				struct ll_sa_entry *entry)
-{
-	struct md_enqueue_info *minfo = entry->se_minfo;
-	struct ptlrpc_request  *req   = entry->se_req;
-
-	if (minfo) {
-		entry->se_minfo = NULL;
-		ll_intent_release(&minfo->mi_it);
-		iput(minfo->mi_dir);
-		kfree(minfo);
-	}
-
-	if (req) {
-		entry->se_req = NULL;
-		ptlrpc_req_finished(req);
-	}
-}
-
-static void ll_sa_entry_put(struct ll_statahead_info *sai,
-			    struct ll_sa_entry *entry)
-{
-	if (atomic_dec_and_test(&entry->se_refcount)) {
-		CDEBUG(D_READA, "free sa entry %.*s(%p) index %llu\n",
-		       entry->se_qstr.len, entry->se_qstr.name, entry,
-		       entry->se_index);
-
-		LASSERT(list_empty(&entry->se_link));
-		LASSERT(list_empty(&entry->se_list));
-		LASSERT(list_empty(&entry->se_hash));
-
-		ll_sa_entry_cleanup(sai, entry);
-		iput(entry->se_inode);
-
-		kfree(entry);
-		atomic_dec(&sai->sai_cache_count);
-	}
-}
-
+/* unhash and unlink sa_entry, and then free it */
 static inline void
-do_sa_entry_fini(struct ll_statahead_info *sai, struct ll_sa_entry *entry)
+sa_kill(struct ll_statahead_info *sai, struct sa_entry *entry)
 {
-	struct ll_inode_info *lli = ll_i2info(sai->sai_inode);
+	struct ll_inode_info *lli = ll_i2info(sai->sai_dentry->d_inode);
 
 	LASSERT(!list_empty(&entry->se_hash));
-	LASSERT(!list_empty(&entry->se_link));
+	LASSERT(!list_empty(&entry->se_list));
+	LASSERT(sa_ready(entry));
 
-	ll_sa_entry_unhash(sai, entry);
+	sa_unhash(sai, entry);
 
 	spin_lock(&lli->lli_sa_lock);
-	entry->se_stat = SA_ENTRY_DEST;
-	list_del_init(&entry->se_link);
-	if (likely(!list_empty(&entry->se_list)))
-		list_del_init(&entry->se_list);
+	list_del_init(&entry->se_list);
 	spin_unlock(&lli->lli_sa_lock);
 
-	ll_sa_entry_put(sai, entry);
+	if (entry->se_inode)
+		iput(entry->se_inode);
+
+	sa_free(sai, entry);
 }
 
-/*
- * Delete it from sai_entries_stated list when fini.
- */
+/* called by scanner after use, sa_entry will be killed */
 static void
-ll_sa_entry_fini(struct ll_statahead_info *sai, struct ll_sa_entry *entry)
+sa_put(struct ll_statahead_info *sai, struct sa_entry *entry)
 {
-	struct ll_sa_entry *pos, *next;
+	struct sa_entry *tmp, *next;
+
+	if (entry && entry->se_state == SA_ENTRY_SUCC) {
+		struct ll_sb_info *sbi = ll_i2sbi(sai->sai_dentry->d_inode);
+
+		sai->sai_hit++;
+		sai->sai_consecutive_miss = 0;
+		sai->sai_max = min(2 * sai->sai_max, sbi->ll_sa_max);
+	} else {
+		sai->sai_miss++;
+		sai->sai_consecutive_miss++;
+	}
 
 	if (entry)
-		do_sa_entry_fini(sai, entry);
+		sa_kill(sai, entry);
 
-	/* drop old entry, only 'scanner' process does this, no need to lock */
-	list_for_each_entry_safe(pos, next, &sai->sai_entries, se_link) {
-		if (!is_omitted_entry(sai, pos->se_index))
+	/*
+	 * kill old completed entries, only scanner process does this, no need
+	 * to lock
+	 */
+	list_for_each_entry_safe(tmp, next, &sai->sai_entries, se_list) {
+		if (!is_omitted_entry(sai, tmp->se_index))
 			break;
-		do_sa_entry_fini(sai, pos);
+		sa_kill(sai, tmp);
 	}
+
+	wake_up(&sai->sai_thread.t_ctl_waitq);
 }
 
 /*
- * Inside lli_sa_lock.
+ * update state and sort add entry to sai_entries by index, return true if
+ * scanner is waiting on this entry.
  */
-static void
-do_sa_entry_to_stated(struct ll_statahead_info *sai,
-		      struct ll_sa_entry *entry, enum se_stat stat)
+static bool
+__sa_make_ready(struct ll_statahead_info *sai, struct sa_entry *entry, int ret)
 {
-	struct ll_sa_entry *se;
-	struct list_head	 *pos = &sai->sai_entries_stated;
+	struct list_head *pos = &sai->sai_entries;
+	__u64 index = entry->se_index;
+	struct sa_entry *se;
 
-	if (!list_empty(&entry->se_list))
-		list_del_init(&entry->se_list);
+	LASSERT(!sa_ready(entry));
+	LASSERT(list_empty(&entry->se_list));
 
-	list_for_each_entry_reverse(se, &sai->sai_entries_stated, se_list) {
+	list_for_each_entry_reverse(se, &sai->sai_entries, se_list) {
 		if (se->se_index < entry->se_index) {
 			pos = &se->se_list;
 			break;
 		}
 	}
-
 	list_add(&entry->se_list, pos);
-	entry->se_stat = stat;
+	entry->se_state = ret < 0 ? SA_ENTRY_INVA : SA_ENTRY_SUCC;
+
+	return (index == sai->sai_index_wait);
 }
 
 /*
- * Move entry to sai_entries_stated and sort with the index.
- * \retval 1    -- entry to be destroyed.
- * \retval 0    -- entry is inserted into stated list.
+ * release resources used in async stat RPC, update entry state and wakeup if
+ * scanner process it waiting on this entry.
  */
-static int
-ll_sa_entry_to_stated(struct ll_statahead_info *sai,
-		      struct ll_sa_entry *entry, enum se_stat stat)
+static void
+sa_make_ready(struct ll_statahead_info *sai, struct sa_entry *entry, int ret)
 {
-	struct ll_inode_info *lli = ll_i2info(sai->sai_inode);
-	int		   ret = 1;
+	struct ll_inode_info *lli = ll_i2info(sai->sai_dentry->d_inode);
+	struct md_enqueue_info *minfo = entry->se_minfo;
+	struct ptlrpc_request *req = entry->se_req;
+	bool wakeup;
+
+	/* release resources used in RPC */
+	if (minfo) {
+		entry->se_minfo = NULL;
+		ll_intent_release(&minfo->mi_it);
+		iput(minfo->mi_dir);
+		kfree(minfo);
+	}
 
-	ll_sa_entry_cleanup(sai, entry);
+	if (req) {
+		entry->se_req = NULL;
+		ptlrpc_req_finished(req);
+	}
 
 	spin_lock(&lli->lli_sa_lock);
-	if (likely(entry->se_stat != SA_ENTRY_DEST)) {
-		do_sa_entry_to_stated(sai, entry, stat);
-		ret = 0;
-	}
+	wakeup = __sa_make_ready(sai, entry, ret);
 	spin_unlock(&lli->lli_sa_lock);
 
-	return ret;
+	if (wakeup)
+		wake_up(&sai->sai_waitq);
 }
 
-/*
- * Insert inode into the list of sai_entries_agl.
- */
+/* Insert inode into the list of sai_agls. */
 static void ll_agl_add(struct ll_statahead_info *sai,
 		       struct inode *inode, int index)
 {
 	struct ll_inode_info *child  = ll_i2info(inode);
-	struct ll_inode_info *parent = ll_i2info(sai->sai_inode);
+	struct ll_inode_info *parent = ll_i2info(sai->sai_dentry->d_inode);
 	int		   added  = 0;
 
 	spin_lock(&child->lli_agl_lock);
@@ -426,9 +370,9 @@ static void ll_agl_add(struct ll_statahead_info *sai,
 
 		igrab(inode);
 		spin_lock(&parent->lli_agl_lock);
-		if (list_empty(&sai->sai_entries_agl))
+		if (list_empty(&sai->sai_agls))
 			added = 1;
-		list_add_tail(&child->lli_agl_list, &sai->sai_entries_agl);
+		list_add_tail(&child->lli_agl_list, &sai->sai_agls);
 		spin_unlock(&parent->lli_agl_lock);
 	} else {
 		spin_unlock(&child->lli_agl_lock);
@@ -438,8 +382,10 @@ static void ll_agl_add(struct ll_statahead_info *sai,
 		wake_up(&sai->sai_agl_thread.t_ctl_waitq);
 }
 
-static struct ll_statahead_info *ll_sai_alloc(void)
+/* allocate sai */
+static struct ll_statahead_info *ll_sai_alloc(struct dentry *dentry)
 {
+	struct ll_inode_info *lli = ll_i2info(dentry->d_inode);
 	struct ll_statahead_info *sai;
 	int		       i;
 
@@ -447,24 +393,18 @@ static struct ll_statahead_info *ll_sai_alloc(void)
 	if (!sai)
 		return NULL;
 
+	sai->sai_dentry = dget(dentry);
 	atomic_set(&sai->sai_refcount, 1);
 
-	spin_lock(&sai_generation_lock);
-	sai->sai_generation = ++sai_generation;
-	if (unlikely(sai_generation == 0))
-		sai->sai_generation = ++sai_generation;
-	spin_unlock(&sai_generation_lock);
-
 	sai->sai_max = LL_SA_RPC_MIN;
 	sai->sai_index = 1;
 	init_waitqueue_head(&sai->sai_waitq);
 	init_waitqueue_head(&sai->sai_thread.t_ctl_waitq);
 	init_waitqueue_head(&sai->sai_agl_thread.t_ctl_waitq);
 
+	INIT_LIST_HEAD(&sai->sai_interim_entries);
 	INIT_LIST_HEAD(&sai->sai_entries);
-	INIT_LIST_HEAD(&sai->sai_entries_received);
-	INIT_LIST_HEAD(&sai->sai_entries_stated);
-	INIT_LIST_HEAD(&sai->sai_entries_agl);
+	INIT_LIST_HEAD(&sai->sai_agls);
 
 	for (i = 0; i < LL_SA_CACHE_SIZE; i++) {
 		INIT_LIST_HEAD(&sai->sai_cache[i]);
@@ -472,63 +412,74 @@ static struct ll_statahead_info *ll_sai_alloc(void)
 	}
 	atomic_set(&sai->sai_cache_count, 0);
 
+	spin_lock(&sai_generation_lock);
+	lli->lli_sa_generation = ++sai_generation;
+	if (unlikely(!sai_generation))
+		lli->lli_sa_generation = ++sai_generation;
+	spin_unlock(&sai_generation_lock);
+
 	return sai;
 }
 
-static inline struct ll_statahead_info *
-ll_sai_get(struct ll_statahead_info *sai)
+/* free sai */
+static inline void ll_sai_free(struct ll_statahead_info *sai)
 {
-	atomic_inc(&sai->sai_refcount);
+	LASSERT(sai->sai_dentry);
+	dput(sai->sai_dentry);
+	kfree(sai);
+}
+
+/*
+ * take refcount of sai if sai for @dir exists, which means statahead is on for
+ * this directory.
+ */
+static inline struct ll_statahead_info *ll_sai_get(struct inode *dir)
+{
+	struct ll_inode_info *lli = ll_i2info(dir);
+	struct ll_statahead_info *sai = NULL;
+
+	spin_lock(&lli->lli_sa_lock);
+	sai = lli->lli_sai;
+	if (sai)
+		atomic_inc(&sai->sai_refcount);
+	spin_unlock(&lli->lli_sa_lock);
+
 	return sai;
 }
 
+/*
+ * put sai refcount after use, if refcount reaches zero, free sai and sa_entries
+ * attached to it.
+ */
 static void ll_sai_put(struct ll_statahead_info *sai)
 {
-	struct inode	 *inode = sai->sai_inode;
-	struct ll_inode_info *lli   = ll_i2info(inode);
+	struct ll_inode_info *lli = ll_i2info(sai->sai_dentry->d_inode);
 
 	if (atomic_dec_and_lock(&sai->sai_refcount, &lli->lli_sa_lock)) {
-		struct ll_sa_entry *entry, *next;
-
-		if (unlikely(atomic_read(&sai->sai_refcount) > 0)) {
-			/* It is race case, the interpret callback just hold
-			 * a reference count
-			 */
-			spin_unlock(&lli->lli_sa_lock);
-			return;
-		}
-
-		LASSERT(!lli->lli_opendir_key);
-		LASSERT(thread_is_stopped(&sai->sai_thread));
-		LASSERT(thread_is_stopped(&sai->sai_agl_thread));
+		struct ll_sb_info *sbi = ll_i2sbi(sai->sai_dentry->d_inode);
+		struct sa_entry *entry, *next;
 
 		lli->lli_sai = NULL;
-		lli->lli_opendir_pid = 0;
 		spin_unlock(&lli->lli_sa_lock);
 
-		if (sai->sai_sent > sai->sai_replied)
-			CDEBUG(D_READA, "statahead for dir "DFID
-			      " does not finish: [sent:%llu] [replied:%llu]\n",
-			      PFID(&lli->lli_fid),
-			      sai->sai_sent, sai->sai_replied);
+		LASSERT(thread_is_stopped(&sai->sai_thread));
+		LASSERT(thread_is_stopped(&sai->sai_agl_thread));
+		LASSERT(sai->sai_sent == sai->sai_replied);
+		LASSERT(!sa_has_callback(sai));
 
 		list_for_each_entry_safe(entry, next, &sai->sai_entries,
-					 se_link)
-			do_sa_entry_fini(sai, entry);
-
-		LASSERT(list_empty(&sai->sai_entries));
-		LASSERT(list_empty(&sai->sai_entries_received));
-		LASSERT(list_empty(&sai->sai_entries_stated));
+					 se_list)
+			sa_kill(sai, entry);
 
 		LASSERT(atomic_read(&sai->sai_cache_count) == 0);
-		LASSERT(list_empty(&sai->sai_entries_agl));
+		LASSERT(list_empty(&sai->sai_agls));
 
-		iput(inode);
-		kfree(sai);
+		ll_sai_free(sai);
+		atomic_dec(&sbi->ll_sa_running);
 	}
 }
 
-/* Do NOT forget to drop inode refcount when into sai_entries_agl. */
+/* Do NOT forget to drop inode refcount when into sai_agls. */
 static void ll_agl_trigger(struct inode *inode, struct ll_statahead_info *sai)
 {
 	struct ll_inode_info *lli   = ll_i2info(inode);
@@ -588,29 +539,21 @@ static void ll_agl_trigger(struct inode *inode, struct ll_statahead_info *sai)
 	iput(inode);
 }
 
-static void ll_post_statahead(struct ll_statahead_info *sai)
+/*
+ * prepare inode for sa entry, add it into agl list, now sa_entry is ready
+ * to be used by scanner process.
+ */
+static void sa_instantiate(struct ll_statahead_info *sai,
+			   struct sa_entry *entry)
 {
-	struct inode	   *dir   = sai->sai_inode;
+	struct inode *dir = sai->sai_dentry->d_inode;
 	struct inode	   *child;
-	struct ll_inode_info   *lli   = ll_i2info(dir);
-	struct ll_sa_entry     *entry;
 	struct md_enqueue_info *minfo;
 	struct lookup_intent   *it;
 	struct ptlrpc_request  *req;
 	struct mdt_body	*body;
 	int		     rc    = 0;
 
-	spin_lock(&lli->lli_sa_lock);
-	if (unlikely(list_empty(&sai->sai_entries_received))) {
-		spin_unlock(&lli->lli_sa_lock);
-		return;
-	}
-	entry = list_entry(sai->sai_entries_received.next,
-			   struct ll_sa_entry, se_list);
-	atomic_inc(&entry->se_refcount);
-	list_del_init(&entry->se_list);
-	spin_unlock(&lli->lli_sa_lock);
-
 	LASSERT(entry->se_handle != 0);
 
 	minfo = entry->se_minfo;
@@ -632,7 +575,7 @@ static void ll_post_statahead(struct ll_statahead_info *sai)
 		/* XXX: No fid in reply, this is probably cross-ref case.
 		 * SA can't handle it yet.
 		 */
-		if (body->valid & OBD_MD_MDS) {
+		if (body->mbo_valid & OBD_MD_MDS) {
 			rc = -EAGAIN;
 			goto out;
 		}
@@ -641,7 +584,7 @@ static void ll_post_statahead(struct ll_statahead_info *sai)
 		 * revalidate.
 		 */
 		/* unlinked and re-created with the same name */
-		if (unlikely(!lu_fid_eq(&minfo->mi_data.op_fid2, &body->fid1))) {
+		if (unlikely(!lu_fid_eq(&minfo->mi_data.op_fid2, &body->mbo_fid1))) {
 			entry->se_inode = NULL;
 			iput(child);
 			child = NULL;
@@ -659,8 +602,9 @@ static void ll_post_statahead(struct ll_statahead_info *sai)
 	if (rc)
 		goto out;
 
-	CDEBUG(D_DLMTRACE, "%s: setting l_data to inode "DFID"%p\n",
+	CDEBUG(D_READA, "%s: setting %.*s" DFID " l_data to inode %p\n",
 	       ll_get_fsname(child->i_sb, NULL, 0),
+	       entry->se_qstr.len, entry->se_qstr.name,
 	       PFID(ll_inode2fid(child)), child);
 	ll_set_lock_data(ll_i2sbi(dir)->ll_md_exp, child, it, NULL);
 
@@ -670,34 +614,75 @@ static void ll_post_statahead(struct ll_statahead_info *sai)
 		ll_agl_add(sai, child, entry->se_index);
 
 out:
-	/* The "ll_sa_entry_to_stated()" will drop related ldlm ibits lock
-	 * reference count by calling "ll_intent_drop_lock()" in spite of the
-	 * above operations failed or not. Do not worry about calling
-	 * "ll_intent_drop_lock()" more than once.
+	/*
+	 * sa_make_ready() will drop ldlm ibits lock refcount by calling
+	 * ll_intent_drop_lock() in spite of failures. Do not worry about
+	 * calling ll_intent_drop_lock() more than once.
 	 */
-	rc = ll_sa_entry_to_stated(sai, entry,
-				   rc < 0 ? SA_ENTRY_INVA : SA_ENTRY_SUCC);
-	if (rc == 0 && entry->se_index == sai->sai_index_wait)
-		wake_up(&sai->sai_waitq);
-	ll_sa_entry_put(sai, entry);
+	sa_make_ready(sai, entry, rc);
 }
 
+/* once there are async stat replies, instantiate sa_entry from replies */
+static void sa_handle_callback(struct ll_statahead_info *sai)
+{
+	struct ll_inode_info *lli;
+
+	lli = ll_i2info(sai->sai_dentry->d_inode);
+
+	while (sa_has_callback(sai)) {
+		struct sa_entry *entry;
+
+		spin_lock(&lli->lli_sa_lock);
+		if (unlikely(!sa_has_callback(sai))) {
+			spin_unlock(&lli->lli_sa_lock);
+			break;
+		}
+		entry = list_entry(sai->sai_interim_entries.next,
+				   struct sa_entry, se_list);
+		list_del_init(&entry->se_list);
+		spin_unlock(&lli->lli_sa_lock);
+
+		sa_instantiate(sai, entry);
+	}
+}
+
+/*
+ * callback for async stat, because this is called in ptlrpcd context, we only
+ * put sa_entry in sai_cb_entries list, and let sa_handle_callback() to really
+ * prepare inode and instantiate sa_entry later.
+ */
 static int ll_statahead_interpret(struct ptlrpc_request *req,
 				  struct md_enqueue_info *minfo, int rc)
 {
 	struct lookup_intent     *it  = &minfo->mi_it;
 	struct inode	     *dir = minfo->mi_dir;
 	struct ll_inode_info     *lli = ll_i2info(dir);
-	struct ll_statahead_info *sai = NULL;
-	struct ll_sa_entry       *entry;
-	__u64			  handle = 0;
-	int		       wakeup;
+	struct ll_statahead_info *sai = lli->lli_sai;
+	struct sa_entry *entry = (struct sa_entry *)minfo->mi_cbdata;
+	__u64 handle = 0;
+	bool wakeup;
 
 	if (it_disposition(it, DISP_LOOKUP_NEG))
 		rc = -ENOENT;
 
-	if (rc == 0) {
-		/* release ibits lock ASAP to avoid deadlock when statahead
+	/*
+	 * because statahead thread will wait for all inflight RPC to finish,
+	 * sai should be always valid, no need to refcount
+	 */
+	LASSERT(sai);
+	LASSERT(!thread_is_stopped(&sai->sai_thread));
+	LASSERT(entry);
+
+	CDEBUG(D_READA, "sa_entry %.*s rc %d\n",
+	       entry->se_qstr.len, entry->se_qstr.name, rc);
+
+	if (rc) {
+		ll_intent_release(it);
+		iput(dir);
+		kfree(minfo);
+	} else {
+		/*
+		 * release ibits lock ASAP to avoid deadlock when statahead
 		 * thread enqueues lock on parent in readdir and another
 		 * process enqueues lock on child with parent lock held, eg.
 		 * unlink.
@@ -707,65 +692,32 @@ static int ll_statahead_interpret(struct ptlrpc_request *req,
 	}
 
 	spin_lock(&lli->lli_sa_lock);
-	/* stale entry */
-	if (unlikely(!lli->lli_sai ||
-		     lli->lli_sai->sai_generation != minfo->mi_generation)) {
-		spin_unlock(&lli->lli_sa_lock);
-		rc = -ESTALE;
-		goto out;
+	if (rc) {
+		wakeup = __sa_make_ready(sai, entry, rc);
 	} else {
-		sai = ll_sai_get(lli->lli_sai);
-		if (unlikely(!thread_is_running(&sai->sai_thread))) {
-			sai->sai_replied++;
-			spin_unlock(&lli->lli_sa_lock);
-			rc = -EBADFD;
-			goto out;
-		}
-
-		entry = ll_sa_entry_get_byindex(sai, minfo->mi_cbdata);
-		if (!entry) {
-			sai->sai_replied++;
-			spin_unlock(&lli->lli_sa_lock);
-			rc = -EIDRM;
-			goto out;
-		}
-
-		if (rc != 0) {
-			do_sa_entry_to_stated(sai, entry, SA_ENTRY_INVA);
-			wakeup = (entry->se_index == sai->sai_index_wait);
-		} else {
-			entry->se_minfo = minfo;
-			entry->se_req = ptlrpc_request_addref(req);
-			/* Release the async ibits lock ASAP to avoid deadlock
-			 * when statahead thread tries to enqueue lock on parent
-			 * for readpage and other tries to enqueue lock on child
-			 * with parent's lock held, for example: unlink.
-			 */
-			entry->se_handle = handle;
-			wakeup = list_empty(&sai->sai_entries_received);
-			list_add_tail(&entry->se_list,
-				      &sai->sai_entries_received);
-		}
-		sai->sai_replied++;
-		spin_unlock(&lli->lli_sa_lock);
-
-		ll_sa_entry_put(sai, entry);
-		if (wakeup)
-			wake_up(&sai->sai_thread.t_ctl_waitq);
+		entry->se_minfo = minfo;
+		entry->se_req = ptlrpc_request_addref(req);
+		/*
+		 * Release the async ibits lock ASAP to avoid deadlock
+		 * when statahead thread tries to enqueue lock on parent
+		 * for readpage and other tries to enqueue lock on child
+		 * with parent's lock held, for example: unlink.
+		 */
+		entry->se_handle = handle;
+		wakeup = !sa_has_callback(sai);
+		list_add_tail(&entry->se_list, &sai->sai_interim_entries);
 	}
+	sai->sai_replied++;
+
+	if (wakeup)
+		wake_up(&sai->sai_thread.t_ctl_waitq);
+	spin_unlock(&lli->lli_sa_lock);
 
-out:
-	if (rc != 0) {
-		ll_intent_release(it);
-		iput(dir);
-		kfree(minfo);
-	}
-	if (sai)
-		ll_sai_put(sai);
 	return rc;
 }
 
-static void sa_args_fini(struct md_enqueue_info *minfo,
+/* finish async stat RPC arguments */
+static void sa_fini_data(struct md_enqueue_info *minfo,
 			 struct ldlm_enqueue_info *einfo)
 {
 	LASSERT(minfo && einfo);
@@ -777,12 +729,11 @@ static void sa_args_fini(struct md_enqueue_info *minfo,
 /**
  * prepare arguments for async stat RPC.
  */
-static int sa_args_init(struct inode *dir, struct inode *child,
-			struct ll_sa_entry *entry, struct md_enqueue_info **pmi,
+static int sa_prep_data(struct inode *dir, struct inode *child,
+			struct sa_entry *entry, struct md_enqueue_info **pmi,
 			struct ldlm_enqueue_info **pei)
 {
 	const struct qstr      *qstr = &entry->se_qstr;
-	struct ll_inode_info     *lli  = ll_i2info(dir);
 	struct md_enqueue_info   *minfo;
 	struct ldlm_enqueue_info *einfo;
 	struct md_op_data	*op_data;
@@ -808,8 +759,7 @@ static int sa_args_init(struct inode *dir, struct inode *child,
 	minfo->mi_it.it_op = IT_GETATTR;
 	minfo->mi_dir = igrab(dir);
 	minfo->mi_cb = ll_statahead_interpret;
-	minfo->mi_generation = lli->lli_sai->sai_generation;
-	minfo->mi_cbdata = entry->se_index;
+	minfo->mi_cbdata = entry;
 
 	einfo->ei_type   = LDLM_IBITS;
 	einfo->ei_mode   = it_to_lock_mode(&minfo->mi_it);
@@ -824,31 +774,33 @@ static int sa_args_init(struct inode *dir, struct inode *child,
 	return 0;
 }
 
-static int do_sa_lookup(struct inode *dir, struct ll_sa_entry *entry)
+/* async stat for file not found in dcache */
+static int sa_lookup(struct inode *dir, struct sa_entry *entry)
 {
 	struct md_enqueue_info   *minfo;
 	struct ldlm_enqueue_info *einfo;
 	int		       rc;
 
-	rc = sa_args_init(dir, NULL, entry, &minfo, &einfo);
+	rc = sa_prep_data(dir, NULL, entry, &minfo, &einfo);
 	if (rc)
 		return rc;
 
 	rc = md_intent_getattr_async(ll_i2mdexp(dir), minfo, einfo);
-	if (rc < 0)
-		sa_args_fini(minfo, einfo);
+	if (rc)
+		sa_fini_data(minfo, einfo);
 
 	return rc;
 }
 
 /**
- * similar to ll_revalidate_it().
- * \retval      1 -- dentry valid
- * \retval      0 -- will send stat-ahead request
- * \retval others -- prepare stat-ahead request failed
+ * async stat for file found in dcache, similar to .revalidate
+ *
+ * \retval	1 dentry valid, no RPC sent
+ * \retval	0 dentry invalid, will send async stat RPC
+ * \retval	negative number upon error
  */
-static int do_sa_revalidate(struct inode *dir, struct ll_sa_entry *entry,
-			    struct dentry *dentry)
+static int sa_revalidate(struct inode *dir, struct sa_entry *entry,
+			 struct dentry *dentry)
 {
 	struct inode	     *inode = d_inode(dentry);
 	struct lookup_intent      it = { .it_op = IT_GETATTR,
@@ -872,7 +824,7 @@ static int do_sa_revalidate(struct inode *dir, struct ll_sa_entry *entry,
 		return 1;
 	}
 
-	rc = sa_args_init(dir, inode, entry, &minfo, &einfo);
+	rc = sa_prep_data(dir, inode, entry, &minfo, &einfo);
 	if (rc) {
 		entry->se_inode = NULL;
 		iput(inode);
@@ -880,56 +832,50 @@ static int do_sa_revalidate(struct inode *dir, struct ll_sa_entry *entry,
 	}
 
 	rc = md_intent_getattr_async(ll_i2mdexp(dir), minfo, einfo);
-	if (rc < 0) {
+	if (rc) {
 		entry->se_inode = NULL;
 		iput(inode);
-		sa_args_fini(minfo, einfo);
+		sa_fini_data(minfo, einfo);
 	}
 
 	return rc;
 }
 
-static void ll_statahead_one(struct dentry *parent, const char *entry_name,
-			     int entry_name_len)
+/* async stat for file with @name */
+static void sa_statahead(struct dentry *parent, const char *name, int len)
 {
 	struct inode	     *dir    = d_inode(parent);
 	struct ll_inode_info     *lli    = ll_i2info(dir);
 	struct ll_statahead_info *sai    = lli->lli_sai;
 	struct dentry	    *dentry = NULL;
-	struct ll_sa_entry       *entry;
+	struct sa_entry *entry;
 	int		       rc;
-	int		       rc1;
 
-	entry = ll_sa_entry_alloc(parent, sai, sai->sai_index, entry_name,
-				  entry_name_len);
+	entry = sa_alloc(parent, sai, sai->sai_index, name, len);
 	if (IS_ERR(entry))
 		return;
 
 	dentry = d_lookup(parent, &entry->se_qstr);
 	if (!dentry) {
-		rc = do_sa_lookup(dir, entry);
+		rc = sa_lookup(dir, entry);
 	} else {
-		rc = do_sa_revalidate(dir, entry, dentry);
+		rc = sa_revalidate(dir, entry, dentry);
 		if (rc == 1 && agl_should_run(sai, d_inode(dentry)))
 			ll_agl_add(sai, d_inode(dentry), entry->se_index);
+	}
 
+	if (dentry)
 		dput(dentry);
-	}
 
-	if (rc) {
-		rc1 = ll_sa_entry_to_stated(sai, entry,
-					rc < 0 ? SA_ENTRY_INVA : SA_ENTRY_SUCC);
-		if (rc1 == 0 && entry->se_index == sai->sai_index_wait)
-			wake_up(&sai->sai_waitq);
-	} else {
+	if (rc)
+		sa_make_ready(sai, entry, rc);
+	else
 		sai->sai_sent++;
-	}
 
 	sai->sai_index++;
-	/* drop one refcount on entry by ll_sa_entry_alloc */
-	ll_sa_entry_put(sai, entry);
 }
 
+/* async glimpse (agl) thread main function */
 static int ll_agl_thread(void *arg)
 {
 	struct dentry	    *parent = arg;
@@ -937,10 +883,12 @@ static int ll_agl_thread(void *arg)
 	struct ll_inode_info     *plli   = ll_i2info(dir);
 	struct ll_inode_info     *clli;
 	struct ll_sb_info	*sbi    = ll_i2sbi(dir);
-	struct ll_statahead_info *sai    = ll_sai_get(plli->lli_sai);
-	struct ptlrpc_thread     *thread = &sai->sai_agl_thread;
+	struct ll_statahead_info *sai;
+	struct ptlrpc_thread *thread;
 	struct l_wait_info	lwi    = { 0 };
 
+	sai = ll_sai_get(dir);
+	thread = &sai->sai_agl_thread;
 	thread->t_pid = current_pid();
 	CDEBUG(D_READA, "agl thread started: sai %p, parent %pd\n",
 	       sai, parent);
@@ -959,7 +907,7 @@ static int ll_agl_thread(void *arg)
 
 	while (1) {
 		l_wait_event(thread->t_ctl_waitq,
-			     !list_empty(&sai->sai_entries_agl) ||
+			     !list_empty(&sai->sai_agls) ||
 			     !thread_is_running(thread),
 			     &lwi);
 
@@ -970,8 +918,8 @@ static int ll_agl_thread(void *arg)
 		/* The statahead thread maybe help to process AGL entries,
 		 * so check whether list empty again.
 		 */
-		if (!list_empty(&sai->sai_entries_agl)) {
-			clli = list_entry(sai->sai_entries_agl.next,
+		if (!list_empty(&sai->sai_agls)) {
+			clli = list_entry(sai->sai_agls.next,
 					  struct ll_inode_info, lli_agl_list);
 			list_del_init(&clli->lli_agl_list);
 			spin_unlock(&plli->lli_agl_lock);
@@ -983,8 +931,8 @@ static int ll_agl_thread(void *arg)
 
 	spin_lock(&plli->lli_agl_lock);
 	sai->sai_agl_valid = 0;
-	while (!list_empty(&sai->sai_entries_agl)) {
-		clli = list_entry(sai->sai_entries_agl.next,
+	while (!list_empty(&sai->sai_agls)) {
+		clli = list_entry(sai->sai_agls.next,
 				  struct ll_inode_info, lli_agl_list);
 		list_del_init(&clli->lli_agl_list);
 		spin_unlock(&plli->lli_agl_lock);
@@ -1001,6 +949,7 @@ static int ll_agl_thread(void *arg)
 	return 0;
 }
 
+/* start agl thread */
 static void ll_start_agl(struct dentry *parent, struct ll_statahead_info *sai)
 {
 	struct ptlrpc_thread *thread = &sai->sai_agl_thread;
@@ -1025,58 +974,71 @@ static void ll_start_agl(struct dentry *parent, struct ll_statahead_info *sai)
 		     &lwi);
 }
 
+/* statahead thread main function */
 static int ll_statahead_thread(void *arg)
 {
 	struct dentry	    *parent = arg;
 	struct inode	     *dir    = d_inode(parent);
-	struct ll_inode_info     *plli   = ll_i2info(dir);
-	struct ll_inode_info     *clli;
+	struct ll_inode_info     *lli   = ll_i2info(dir);
 	struct ll_sb_info	*sbi    = ll_i2sbi(dir);
-	struct ll_statahead_info *sai    = ll_sai_get(plli->lli_sai);
-	struct ptlrpc_thread     *thread = &sai->sai_thread;
-	struct ptlrpc_thread *agl_thread = &sai->sai_agl_thread;
-	struct page	      *page;
+	struct ll_statahead_info *sai;
+	struct ptlrpc_thread *sa_thread;
+	struct ptlrpc_thread *agl_thread;
+	struct page	      *page = NULL;
 	__u64		     pos    = 0;
 	int		       first  = 0;
 	int		       rc     = 0;
-	struct ll_dir_chain       chain;
+	struct md_op_data *op_data;
 	struct l_wait_info	lwi    = { 0 };
 
-	thread->t_pid = current_pid();
+	sai = ll_sai_get(dir);
+	sa_thread = &sai->sai_thread;
+	agl_thread = &sai->sai_agl_thread;
+	sa_thread->t_pid = current_pid();
 	CDEBUG(D_READA, "statahead thread starting: sai %p, parent %pd\n",
 	       sai, parent);
 
+	op_data = ll_prep_md_op_data(NULL, dir, dir, NULL, 0, 0,
+				     LUSTRE_OPC_ANY, dir);
+	if (IS_ERR(op_data)) {
+		rc = PTR_ERR(op_data);
+		goto out;
+	}
+
+	op_data->op_max_pages = ll_i2sbi(dir)->ll_md_brw_pages;
+
 	if (sbi->ll_flags & LL_SBI_AGL_ENABLED)
 		ll_start_agl(parent, sai);
 
 	atomic_inc(&sbi->ll_sa_total);
-	spin_lock(&plli->lli_sa_lock);
-	if (thread_is_init(thread))
+	spin_lock(&lli->lli_sa_lock);
+	if (thread_is_init(sa_thread))
 		/* If someone else has changed the thread state
 		 * (e.g. already changed to SVC_STOPPING), we can't just
 		 * blindly overwrite that setting.
 		 */
-		thread_set_flags(thread, SVC_RUNNING);
-	spin_unlock(&plli->lli_sa_lock);
-	wake_up(&thread->t_ctl_waitq);
-
-	ll_dir_chain_init(&chain);
-	page = ll_get_dir_page(dir, pos, &chain);
+		thread_set_flags(sa_thread, SVC_RUNNING);
+	spin_unlock(&lli->lli_sa_lock);
+	wake_up(&sa_thread->t_ctl_waitq);
 
-	while (1) {
+	while (pos != MDS_DIR_END_OFF && thread_is_running(sa_thread)) {
 		struct lu_dirpage *dp;
 		struct lu_dirent  *ent;
 
+		sai->sai_in_readpage = 1;
+		page = ll_get_dir_page(dir, op_data, pos);
+		sai->sai_in_readpage = 0;
 		if (IS_ERR(page)) {
 			rc = PTR_ERR(page);
-			CDEBUG(D_READA, "error reading dir "DFID" at %llu/%llu: [rc %d] [parent %u]\n",
+			CDEBUG(D_READA, "error reading dir "DFID" at %llu/%llu: opendir_pid = %u: rc = %d\n",
 			       PFID(ll_inode2fid(dir)), pos, sai->sai_index,
-			       rc, plli->lli_opendir_pid);
-			goto out;
+			       lli->lli_opendir_pid, rc);
+			break;
 		}
 
 		dp = page_address(page);
-		for (ent = lu_dirent_start(dp); ent;
+		for (ent = lu_dirent_start(dp);
+		     ent && thread_is_running(sa_thread) && !sa_low_hit(sai);
 		     ent = lu_dirent_next(ent)) {
 			__u64 hash;
 			int namelen;
@@ -1123,123 +1085,79 @@ static int ll_statahead_thread(void *arg)
 			if (unlikely(++first == 1))
 				continue;
 
-keep_it:
-			l_wait_event(thread->t_ctl_waitq,
-				     !sa_sent_full(sai) ||
-				     !list_empty(&sai->sai_entries_received) ||
-				     !list_empty(&sai->sai_entries_agl) ||
-				     !thread_is_running(thread),
-				     &lwi);
-
-interpret_it:
-			while (!list_empty(&sai->sai_entries_received))
-				ll_post_statahead(sai);
-
-			if (unlikely(!thread_is_running(thread))) {
-				ll_release_page(page, 0);
-				rc = 0;
-				goto out;
-			}
+			/* wait for spare statahead window */
+			do {
+				l_wait_event(sa_thread->t_ctl_waitq,
+					     !sa_sent_full(sai) ||
+					     sa_has_callback(sai) ||
+					     !list_empty(&sai->sai_agls) ||
+					     !thread_is_running(sa_thread),
+					     &lwi);
+				sa_handle_callback(sai);
 
-			/* If no window for metadata statahead, but there are
-			 * some AGL entries to be triggered, then try to help
-			 * to process the AGL entries.
-			 */
-			if (sa_sent_full(sai)) {
-				spin_lock(&plli->lli_agl_lock);
-				while (!list_empty(&sai->sai_entries_agl)) {
-					clli = list_entry(sai->sai_entries_agl.next,
+				spin_lock(&lli->lli_agl_lock);
+				while (sa_sent_full(sai) &&
+				       !agl_list_empty(sai)) {
+					struct ll_inode_info *clli;
+
+					clli = list_entry(sai->sai_agls.next,
 							  struct ll_inode_info, lli_agl_list);
 					list_del_init(&clli->lli_agl_list);
-					spin_unlock(&plli->lli_agl_lock);
+					spin_unlock(&lli->lli_agl_lock);
+
 					ll_agl_trigger(&clli->lli_vfs_inode,
 						       sai);
 
-					if (!list_empty(&sai->sai_entries_received))
-						goto interpret_it;
-
-					if (unlikely(
-						!thread_is_running(thread))) {
-						ll_release_page(page, 0);
-						rc = 0;
-						goto out;
-					}
-
-					if (!sa_sent_full(sai))
-						goto do_it;
-
-					spin_lock(&plli->lli_agl_lock);
+					spin_lock(&lli->lli_agl_lock);
 				}
-				spin_unlock(&plli->lli_agl_lock);
+				spin_unlock(&lli->lli_agl_lock);
+			} while (sa_sent_full(sai) &&
+				 thread_is_running(sa_thread));
 
-				goto keep_it;
-			}
-
-do_it:
-			ll_statahead_one(parent, name, namelen);
+			sa_statahead(parent, name, namelen);
 		}
-		pos = le64_to_cpu(dp->ldp_hash_end);
-		if (pos == MDS_DIR_END_OFF) {
-			/*
-			 * End of directory reached.
-			 */
-			ll_release_page(page, 0);
-			while (1) {
-				l_wait_event(thread->t_ctl_waitq,
-					     !list_empty(&sai->sai_entries_received) ||
-					     sai->sai_sent == sai->sai_replied ||
-					     !thread_is_running(thread),
-					     &lwi);
 
-				while (!list_empty(&sai->sai_entries_received))
-					ll_post_statahead(sai);
+		pos = le64_to_cpu(dp->ldp_hash_end);
+		ll_release_page(dir, page,
+				le32_to_cpu(dp->ldp_flags) & LDF_COLLIDE);
 
-				if (unlikely(!thread_is_running(thread))) {
-					rc = 0;
-					goto out;
-				}
+		if (sa_low_hit(sai)) {
+			rc = -EFAULT;
+			atomic_inc(&sbi->ll_sa_wrong);
+			CDEBUG(D_READA, "Statahead for dir "DFID" hit ratio too low: hit/miss %llu/%llu, sent/replied %llu/%llu, stopping statahead thread: pid %d\n",
+			       PFID(&lli->lli_fid), sai->sai_hit,
+			       sai->sai_miss, sai->sai_sent,
+			       sai->sai_replied, current_pid());
+			break;
+		}
+	}
+	ll_finish_md_op_data(op_data);
 
-				if (sai->sai_sent == sai->sai_replied &&
-				    list_empty(&sai->sai_entries_received))
-					break;
-			}
+	if (rc < 0) {
+		spin_lock(&lli->lli_sa_lock);
+		thread_set_flags(sa_thread, SVC_STOPPING);
+		lli->lli_sa_enabled = 0;
+		spin_unlock(&lli->lli_sa_lock);
+	}
 
-			spin_lock(&plli->lli_agl_lock);
-			while (!list_empty(&sai->sai_entries_agl) &&
-			       thread_is_running(thread)) {
-				clli = list_entry(sai->sai_entries_agl.next,
-						  struct ll_inode_info, lli_agl_list);
-				list_del_init(&clli->lli_agl_list);
-				spin_unlock(&plli->lli_agl_lock);
-				ll_agl_trigger(&clli->lli_vfs_inode, sai);
-				spin_lock(&plli->lli_agl_lock);
-			}
-			spin_unlock(&plli->lli_agl_lock);
+	/*
+	 * statahead is finished, but statahead entries need to be cached, wait
+	 * for file release to stop me.
+	 */
+	while (thread_is_running(sa_thread)) {
+		l_wait_event(sa_thread->t_ctl_waitq,
+			     sa_has_callback(sai) ||
+			     !agl_list_empty(sai) ||
+			     !thread_is_running(sa_thread),
+			     &lwi);
 
-			rc = 0;
-			goto out;
-		} else if (1) {
-			/*
-			 * chain is exhausted.
-			 * Normal case: continue to the next page.
-			 */
-			ll_release_page(page, le32_to_cpu(dp->ldp_flags) &
-					      LDF_COLLIDE);
-			page = ll_get_dir_page(dir, pos, &chain);
-		} else {
-			LASSERT(le32_to_cpu(dp->ldp_flags) & LDF_COLLIDE);
-			ll_release_page(page, 1);
-			/*
-			 * go into overflow page.
-			 */
-		}
+		sa_handle_callback(sai);
 	}
-
 out:
 	if (sai->sai_agl_valid) {
-		spin_lock(&plli->lli_agl_lock);
+		spin_lock(&lli->lli_agl_lock);
 		thread_set_flags(agl_thread, SVC_STOPPING);
-		spin_unlock(&plli->lli_agl_lock);
+		spin_unlock(&lli->lli_agl_lock);
 		wake_up(&agl_thread->t_ctl_waitq);
 
 		CDEBUG(D_READA, "stop agl thread: sai %p pid %u\n",
@@ -1249,84 +1167,93 @@ out:
 			     &lwi);
 	} else {
 		/* Set agl_thread flags anyway. */
-		thread_set_flags(&sai->sai_agl_thread, SVC_STOPPED);
+		thread_set_flags(agl_thread, SVC_STOPPED);
 	}
-	ll_dir_chain_fini(&chain);
-	spin_lock(&plli->lli_sa_lock);
-	if (!list_empty(&sai->sai_entries_received)) {
-		thread_set_flags(thread, SVC_STOPPING);
-		spin_unlock(&plli->lli_sa_lock);
-
-		/* To release the resources held by received entries. */
-		while (!list_empty(&sai->sai_entries_received))
-			ll_post_statahead(sai);
 
-		spin_lock(&plli->lli_sa_lock);
+	/*
+	 * wait for inflight statahead RPCs to finish, and then we can free sai
+	 * safely because statahead RPC will access sai data
+	 */
+	while (sai->sai_sent != sai->sai_replied) {
+		/* in case we're not woken up, timeout wait */
+		lwi = LWI_TIMEOUT(msecs_to_jiffies(MSEC_PER_SEC >> 3),
+				  NULL, NULL);
+		l_wait_event(sa_thread->t_ctl_waitq,
+			     sai->sai_sent == sai->sai_replied, &lwi);
 	}
-	thread_set_flags(thread, SVC_STOPPED);
-	spin_unlock(&plli->lli_sa_lock);
-	wake_up(&sai->sai_waitq);
-	wake_up(&thread->t_ctl_waitq);
-	ll_sai_put(sai);
-	dput(parent);
+
+	/* release resources held by statahead RPCs */
+	sa_handle_callback(sai);
+
+	spin_lock(&lli->lli_sa_lock);
+	thread_set_flags(sa_thread, SVC_STOPPED);
+	spin_unlock(&lli->lli_sa_lock);
+
 	CDEBUG(D_READA, "statahead thread stopped: sai %p, parent %pd\n",
 	       sai, parent);
+
+	wake_up(&sai->sai_waitq);
+	wake_up(&sa_thread->t_ctl_waitq);
+	ll_sai_put(sai);
+
 	return rc;
 }
 
-/**
- * called in ll_file_release().
- */
-void ll_stop_statahead(struct inode *dir, void *key)
+/* authorize opened dir handle @key to statahead */
+void ll_authorize_statahead(struct inode *dir, void *key)
 {
 	struct ll_inode_info *lli = ll_i2info(dir);
 
-	if (unlikely(!key))
-		return;
-
 	spin_lock(&lli->lli_sa_lock);
-	if (lli->lli_opendir_key != key || lli->lli_opendir_pid == 0) {
-		spin_unlock(&lli->lli_sa_lock);
-		return;
+	if (!lli->lli_opendir_key && !lli->lli_sai) {
+		/*
+		 * if lli_sai is not NULL, it means previous statahead is not
+		 * finished yet, we'd better not start a new statahead for now.
+		 */
+		LASSERT(!lli->lli_opendir_pid);
+		lli->lli_opendir_key = key;
+		lli->lli_opendir_pid = current_pid();
+		lli->lli_sa_enabled = 1;
 	}
+	spin_unlock(&lli->lli_sa_lock);
+}
 
-	lli->lli_opendir_key = NULL;
-
-	if (lli->lli_sai) {
-		struct l_wait_info lwi = { 0 };
-		struct ptlrpc_thread *thread = &lli->lli_sai->sai_thread;
+/*
+ * deauthorize opened dir handle @key to statahead, but statahead thread may
+ * still be running, notify it to quit.
+ */
+void ll_deauthorize_statahead(struct inode *dir, void *key)
+{
+	struct ll_inode_info *lli = ll_i2info(dir);
+	struct ll_statahead_info *sai;
 
-		if (!thread_is_stopped(thread)) {
-			thread_set_flags(thread, SVC_STOPPING);
-			spin_unlock(&lli->lli_sa_lock);
-			wake_up(&thread->t_ctl_waitq);
+	LASSERT(lli->lli_opendir_key == key);
+	LASSERT(lli->lli_opendir_pid);
 
-			CDEBUG(D_READA, "stop statahead thread: sai %p pid %u\n",
-			       lli->lli_sai, (unsigned int)thread->t_pid);
-			l_wait_event(thread->t_ctl_waitq,
-				     thread_is_stopped(thread),
-				     &lwi);
-		} else {
-			spin_unlock(&lli->lli_sa_lock);
-		}
+	CDEBUG(D_READA, "deauthorize statahead for "DFID"\n",
+	       PFID(&lli->lli_fid));
 
+	spin_lock(&lli->lli_sa_lock);
+	lli->lli_opendir_key = NULL;
+	lli->lli_opendir_pid = 0;
+	lli->lli_sa_enabled = 0;
+	sai = lli->lli_sai;
+	if (sai && thread_is_running(&sai->sai_thread)) {
 		/*
-		 * Put the ref which was held when first statahead_enter.
-		 * It maybe not the last ref for some statahead requests
-		 * maybe inflight.
+		 * statahead thread may not quit yet because it needs to cache
+		 * entries, now it's time to tell it to quit.
 		 */
-		ll_sai_put(lli->lli_sai);
-	} else {
-		lli->lli_opendir_pid = 0;
-		spin_unlock(&lli->lli_sa_lock);
+		thread_set_flags(&sai->sai_thread, SVC_STOPPING);
+		wake_up(&sai->sai_thread.t_ctl_waitq);
 	}
+	spin_unlock(&lli->lli_sa_lock);
 }
 
 enum {
 	/**
 	 * not first dirent, or is "."
 	 */
-	LS_NONE_FIRST_DE = 0,
+	LS_NOT_FIRST_DE = 0,
 	/**
 	 * the first non-hidden dirent
 	 */
@@ -1337,17 +1264,26 @@ enum {
 	LS_FIRST_DOT_DE
 };
 
+/* file is first dirent under @dir */
 static int is_first_dirent(struct inode *dir, struct dentry *dentry)
 {
-	struct ll_dir_chain   chain;
 	const struct qstr  *target = &dentry->d_name;
+	struct md_op_data *op_data;
 	struct page	  *page;
 	__u64		 pos    = 0;
 	int		   dot_de;
-	int		   rc     = LS_NONE_FIRST_DE;
+	int rc = LS_NOT_FIRST_DE;
 
-	ll_dir_chain_init(&chain);
-	page = ll_get_dir_page(dir, pos, &chain);
+	op_data = ll_prep_md_op_data(NULL, dir, dir, NULL, 0, 0,
+				     LUSTRE_OPC_ANY, dir);
+	if (IS_ERR(op_data))
+		return PTR_ERR(op_data);
+	/**
+	 * FIXME choose the start offset of the readdir
+	 */
+	op_data->op_max_pages = ll_i2sbi(dir)->ll_md_brw_pages;
+
+	page = ll_get_dir_page(dir, op_data, pos);
 
 	while (1) {
 		struct lu_dirpage *dp;
@@ -1357,9 +1293,10 @@ static int is_first_dirent(struct inode *dir, struct dentry *dentry)
 			struct ll_inode_info *lli = ll_i2info(dir);
 
 			rc = PTR_ERR(page);
-			CERROR("error reading dir "DFID" at %llu: [rc %d] [parent %u]\n",
+			CERROR("%s: error reading dir "DFID" at %llu: opendir_pid = %u : rc = %d\n",
+			       ll_get_fsname(dir->i_sb, NULL, 0),
 			       PFID(ll_inode2fid(dir)), pos,
-			       rc, lli->lli_opendir_pid);
+			       lli->lli_opendir_pid, rc);
 			break;
 		}
 
@@ -1411,13 +1348,13 @@ static int is_first_dirent(struct inode *dir, struct dentry *dentry)
 
 			if (target->len != namelen ||
 			    memcmp(target->name, name, namelen) != 0)
-				rc = LS_NONE_FIRST_DE;
+				rc = LS_NOT_FIRST_DE;
 			else if (!dot_de)
 				rc = LS_FIRST_DE;
 			else
 				rc = LS_FIRST_DOT_DE;
 
-			ll_release_page(page, 0);
+			ll_release_page(dir, page, false);
 			goto out;
 		}
 		pos = le64_to_cpu(dp->ldp_hash_end);
@@ -1425,261 +1362,228 @@ static int is_first_dirent(struct inode *dir, struct dentry *dentry)
 			/*
 			 * End of directory reached.
 			 */
-			ll_release_page(page, 0);
-			break;
-		} else if (1) {
+			ll_release_page(dir, page, false);
+			goto out;
+		} else {
 			/*
 			 * chain is exhausted
 			 * Normal case: continue to the next page.
 			 */
-			ll_release_page(page, le32_to_cpu(dp->ldp_flags) &
-					      LDF_COLLIDE);
-			page = ll_get_dir_page(dir, pos, &chain);
-		} else {
-			/*
-			 * go into overflow page.
-			 */
-			LASSERT(le32_to_cpu(dp->ldp_flags) & LDF_COLLIDE);
-			ll_release_page(page, 1);
+			ll_release_page(dir, page,
+					le32_to_cpu(dp->ldp_flags) &
+					LDF_COLLIDE);
+			page = ll_get_dir_page(dir, op_data, pos);
 		}
 	}
-
 out:
-	ll_dir_chain_fini(&chain);
+	ll_finish_md_op_data(op_data);
 	return rc;
 }
 
-static void
-ll_sai_unplug(struct ll_statahead_info *sai, struct ll_sa_entry *entry)
-{
-	struct ptlrpc_thread *thread = &sai->sai_thread;
-	struct ll_sb_info    *sbi    = ll_i2sbi(sai->sai_inode);
-	int		   hit;
-
-	if (entry && entry->se_stat == SA_ENTRY_SUCC)
-		hit = 1;
-	else
-		hit = 0;
-
-	ll_sa_entry_fini(sai, entry);
-	if (hit) {
-		sai->sai_hit++;
-		sai->sai_consecutive_miss = 0;
-		sai->sai_max = min(2 * sai->sai_max, sbi->ll_sa_max);
-	} else {
-		struct ll_inode_info *lli = ll_i2info(sai->sai_inode);
-
-		sai->sai_miss++;
-		sai->sai_consecutive_miss++;
-		if (sa_low_hit(sai) && thread_is_running(thread)) {
-			atomic_inc(&sbi->ll_sa_wrong);
-			CDEBUG(D_READA, "Statahead for dir " DFID " hit ratio too low: hit/miss %llu/%llu, sent/replied %llu/%llu, stopping statahead thread\n",
-			       PFID(&lli->lli_fid), sai->sai_hit,
-			       sai->sai_miss, sai->sai_sent,
-			       sai->sai_replied);
-			spin_lock(&lli->lli_sa_lock);
-			if (!thread_is_stopped(thread))
-				thread_set_flags(thread, SVC_STOPPING);
-			spin_unlock(&lli->lli_sa_lock);
-		}
-	}
-
-	if (!thread_is_stopped(thread))
-		wake_up(&thread->t_ctl_waitq);
-}
-
 /**
- * Start statahead thread if this is the first dir entry.
- * Otherwise if a thread is started already, wait it until it is ahead of me.
- * \retval 1       -- find entry with lock in cache, the caller needs to do
- *		    nothing.
- * \retval 0       -- find entry in cache, but without lock, the caller needs
- *		    refresh from MDS.
- * \retval others  -- the caller need to process as non-statahead.
+ * revalidate @dentryp from statahead cache
+ *
+ * \param[in]  dir	parent directory
+ * \param[in]  sai	sai structure
+ * \param[out] dentryp	pointer to dentry which will be revalidated
+ * \param[in]  unplug	unplug statahead window only (normally for negative
+ *			dentry)
+ * \retval		1 on success, dentry is saved in @dentryp
+ * \retval		0 if revalidation failed (no proper lock on client)
+ * \retval		negative number upon error
  */
-int do_statahead_enter(struct inode *dir, struct dentry **dentryp,
-		       int only_unplug)
+static int revalidate_statahead_dentry(struct inode *dir,
+				       struct ll_statahead_info *sai,
+				       struct dentry **dentryp,
+				       bool unplug)
 {
-	struct ll_inode_info     *lli   = ll_i2info(dir);
-	struct ll_statahead_info *sai   = lli->lli_sai;
-	struct dentry	    *parent;
-	struct ll_sa_entry       *entry;
-	struct ptlrpc_thread     *thread;
-	struct l_wait_info	lwi   = { 0 };
-	struct task_struct *task;
-	int		       rc    = 0;
-	struct ll_inode_info     *plli;
+	struct sa_entry *entry = NULL;
+	struct l_wait_info lwi = { 0 };
+	struct ll_dentry_data *ldd;
+	struct ll_inode_info *lli;
+	int rc = 0;
 
-	LASSERT(lli->lli_opendir_pid == current_pid());
+	if ((*dentryp)->d_name.name[0] == '.') {
+		if (sai->sai_ls_all ||
+		    sai->sai_miss_hidden >= sai->sai_skip_hidden) {
+			/*
+			 * Hidden dentry is the first one, or statahead
+			 * thread does not skip so many hidden dentries
+			 * before "sai_ls_all" enabled as below.
+			 */
+		} else {
+			if (!sai->sai_ls_all)
+				/*
+				 * It maybe because hidden dentry is not
+				 * the first one, "sai_ls_all" was not
+				 * set, then "ls -al" missed. Enable
+				 * "sai_ls_all" for such case.
+				 */
+				sai->sai_ls_all = 1;
 
-	if (sai) {
-		thread = &sai->sai_thread;
-		if (unlikely(thread_is_stopped(thread) &&
-			     list_empty(&sai->sai_entries_stated))) {
-			/* to release resource */
-			ll_stop_statahead(dir, lli->lli_opendir_key);
+			/*
+			 * Such "getattr" has been skipped before
+			 * "sai_ls_all" enabled as above.
+			 */
+			sai->sai_miss_hidden++;
 			return -EAGAIN;
 		}
+	}
 
-		if ((*dentryp)->d_name.name[0] == '.') {
-			if (sai->sai_ls_all ||
-			    sai->sai_miss_hidden >= sai->sai_skip_hidden) {
-				/*
-				 * Hidden dentry is the first one, or statahead
-				 * thread does not skip so many hidden dentries
-				 * before "sai_ls_all" enabled as below.
-				 */
-			} else {
-				if (!sai->sai_ls_all)
-					/*
-					 * It maybe because hidden dentry is not
-					 * the first one, "sai_ls_all" was not
-					 * set, then "ls -al" missed. Enable
-					 * "sai_ls_all" for such case.
-					 */
-					sai->sai_ls_all = 1;
+	if (unplug) {
+		rc = 1;
+		goto out_unplug;
+	}
 
-				/*
-				 * Such "getattr" has been skipped before
-				 * "sai_ls_all" enabled as above.
-				 */
-				sai->sai_miss_hidden++;
-				return -EAGAIN;
-			}
-		}
+	entry = sa_get(sai, &(*dentryp)->d_name);
+	if (!entry) {
+		rc = -EAGAIN;
+		goto out_unplug;
+	}
 
-		entry = ll_sa_entry_get_byname(sai, &(*dentryp)->d_name);
-		if (!entry || only_unplug) {
-			ll_sai_unplug(sai, entry);
-			return entry ? 1 : -EAGAIN;
-		}
+	/* if statahead is busy in readdir, help it do post-work */
+	if (!sa_ready(entry) && sai->sai_in_readpage)
+		sa_handle_callback(sai);
 
-		if (!ll_sa_entry_stated(entry)) {
-			sai->sai_index_wait = entry->se_index;
-			lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(30), NULL,
-					       LWI_ON_SIGNAL_NOOP, NULL);
-			rc = l_wait_event(sai->sai_waitq,
-					  ll_sa_entry_stated(entry) ||
-					  thread_is_stopped(thread),
-					  &lwi);
-			if (rc < 0) {
-				ll_sai_unplug(sai, entry);
-				return -EAGAIN;
-			}
+	if (!sa_ready(entry)) {
+		sai->sai_index_wait = entry->se_index;
+		lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(30), NULL,
+				       LWI_ON_SIGNAL_NOOP, NULL);
+		rc = l_wait_event(sai->sai_waitq, sa_ready(entry), &lwi);
+		if (rc < 0) {
+			/*
+			 * entry may not be ready, so it may be used by inflight
+			 * statahead RPC, don't free it.
+			 */
+			entry = NULL;
+			rc = -EAGAIN;
+			goto out_unplug;
 		}
+	}
 
-		if (entry->se_stat == SA_ENTRY_SUCC && entry->se_inode) {
-			struct inode *inode = entry->se_inode;
-			struct lookup_intent it = { .it_op = IT_GETATTR,
-						    .it_lock_handle =
-						     entry->se_handle };
-			__u64 bits;
-
-			rc = md_revalidate_lock(ll_i2mdexp(dir), &it,
-						ll_inode2fid(inode), &bits);
-			if (rc == 1) {
-				if (!d_inode(*dentryp)) {
-					struct dentry *alias;
-
-					alias = ll_splice_alias(inode,
-								*dentryp);
-					if (IS_ERR(alias)) {
-						ll_sai_unplug(sai, entry);
-						return PTR_ERR(alias);
-					}
-					*dentryp = alias;
-				} else if (d_inode(*dentryp) != inode) {
-					/* revalidate, but inode is recreated */
-					CDEBUG(D_READA, "%s: stale dentry %pd inode "DFID", statahead inode "DFID"\n",
-					       ll_get_fsname(d_inode(*dentryp)->i_sb, NULL, 0),
-					       *dentryp,
-					       PFID(ll_inode2fid(d_inode(*dentryp))),
-					       PFID(ll_inode2fid(inode)));
-					ll_sai_unplug(sai, entry);
-					return -ESTALE;
-				} else {
-					iput(inode);
+	if (entry->se_state == SA_ENTRY_SUCC && entry->se_inode) {
+		struct inode *inode = entry->se_inode;
+		struct lookup_intent it = { .it_op = IT_GETATTR,
+					    .it_lock_handle = entry->se_handle };
+		__u64 bits;
+
+		rc = md_revalidate_lock(ll_i2mdexp(dir), &it,
+					ll_inode2fid(inode), &bits);
+		if (rc == 1) {
+			if (!(*dentryp)->d_inode) {
+				struct dentry *alias;
+
+				alias = ll_splice_alias(inode, *dentryp);
+				if (IS_ERR(alias)) {
+					rc = PTR_ERR(alias);
+					goto out_unplug;
 				}
+				*dentryp = alias;
+				/**
+				 * statahead prepared this inode, transfer inode
+				 * refcount from sa_entry to dentry
+				 */
 				entry->se_inode = NULL;
-
-				if ((bits & MDS_INODELOCK_LOOKUP) &&
-				    d_lustre_invalid(*dentryp))
-					d_lustre_revalidate(*dentryp);
-				ll_intent_release(&it);
+			} else if ((*dentryp)->d_inode != inode) {
+				/* revalidate, but inode is recreated */
+				CDEBUG(D_READA,
+				       "%s: stale dentry %pd inode "DFID", statahead inode "DFID"\n",
+				       ll_get_fsname((*dentryp)->d_inode->i_sb,
+						     NULL, 0),
+				       *dentryp,
+				       PFID(ll_inode2fid((*dentryp)->d_inode)),
+				       PFID(ll_inode2fid(inode)));
+				rc = -ESTALE;
+				goto out_unplug;
 			}
-		}
 
-		ll_sai_unplug(sai, entry);
-		return rc;
+			if ((bits & MDS_INODELOCK_LOOKUP) &&
+			    d_lustre_invalid(*dentryp))
+				d_lustre_revalidate(*dentryp);
+			ll_intent_release(&it);
+		}
 	}
+out_unplug:
+	/*
+	 * statahead cached sa_entry can be used only once, and will be killed
+	 * right after use, so if lookup/revalidate accessed statahead cache,
+	 * set dentry ldd_sa_generation to parent lli_sa_generation, later if we
+	 * stat this file again, we know we've done statahead before, see
+	 * dentry_may_statahead().
+	 */
+	ldd = ll_d2d(*dentryp);
+	lli = ll_i2info(dir);
+	/* ldd can be NULL if llite lookup failed. */
+	if (ldd)
+		ldd->lld_sa_generation = lli->lli_sa_generation;
+	sa_put(sai, entry);
+	return rc;
+}
+
+/**
+ * start statahead thread
+ *
+ * \param[in] dir	parent directory
+ * \param[in] dentry	dentry that triggers statahead, normally the first
+ *			dirent under @dir
+ * \retval		-EAGAIN on success, because when this function is
+ *			called, it's already in lookup call, so client should
+ *			do it itself instead of waiting for statahead thread
+ *			to do it asynchronously.
+ * \retval		negative number upon error
+ */
+static int start_statahead_thread(struct inode *dir, struct dentry *dentry)
+{
+	struct ll_inode_info *lli = ll_i2info(dir);
+	struct ll_statahead_info *sai = NULL;
+	struct l_wait_info lwi = { 0 };
+	struct ptlrpc_thread *thread;
+	struct task_struct *task;
+	struct dentry *parent = dentry->d_parent;
+	int rc;
 
 	/* I am the "lli_opendir_pid" owner, only me can set "lli_sai". */
-	rc = is_first_dirent(dir, *dentryp);
-	if (rc == LS_NONE_FIRST_DE) {
+	rc = is_first_dirent(dir, dentry);
+	if (rc == LS_NOT_FIRST_DE) {
 		/* It is not "ls -{a}l" operation, no need statahead for it. */
-		rc = -EAGAIN;
+		rc = -EFAULT;
 		goto out;
 	}
 
-	sai = ll_sai_alloc();
+	sai = ll_sai_alloc(parent);
 	if (!sai) {
 		rc = -ENOMEM;
 		goto out;
 	}
 
 	sai->sai_ls_all = (rc == LS_FIRST_DOT_DE);
-	sai->sai_inode = igrab(dir);
-	if (unlikely(!sai->sai_inode)) {
-		CWARN("Do not start stat ahead on dying inode "DFID"\n",
-		      PFID(&lli->lli_fid));
-		rc = -ESTALE;
-		goto out;
-	}
-
-	/* get parent reference count here, and put it in ll_statahead_thread */
-	parent = dget((*dentryp)->d_parent);
-	if (unlikely(sai->sai_inode != d_inode(parent))) {
-		struct ll_inode_info *nlli = ll_i2info(d_inode(parent));
-
-		CWARN("Race condition, someone changed %pd just now: old parent "DFID", new parent "DFID"\n",
-		      *dentryp,
-		      PFID(&lli->lli_fid), PFID(&nlli->lli_fid));
-		dput(parent);
-		iput(sai->sai_inode);
-		rc = -EAGAIN;
+	/*
+	 * if current lli_opendir_key was deauthorized, or dir re-opened by
+	 * another process, don't start statahead, otherwise the newly spawned
+	 * statahead thread won't be notified to quit.
+	 */
+	spin_lock(&lli->lli_sa_lock);
+	if (unlikely(lli->lli_sai || lli->lli_opendir_key ||
+		     lli->lli_opendir_pid != current->pid)) {
+		spin_unlock(&lli->lli_sa_lock);
+		rc = -EPERM;
 		goto out;
 	}
+	lli->lli_sai = sai;
+	spin_unlock(&lli->lli_sa_lock);
 
-	CDEBUG(D_READA, "start statahead thread: sai %p, parent %pd\n",
-	       sai, parent);
+	atomic_inc(&ll_i2sbi(parent->d_inode)->ll_sa_running);
 
-	/* The sai buffer already has one reference taken at allocation time,
-	 * but as soon as we expose the sai by attaching it to the lli that
-	 * default reference can be dropped by another thread calling
-	 * ll_stop_statahead. We need to take a local reference to protect
-	 * the sai buffer while we intend to access it.
-	 */
-	ll_sai_get(sai);
-	lli->lli_sai = sai;
+	CDEBUG(D_READA, "start statahead thread: [pid %d] [parent %pd]\n",
+	       current_pid(), parent);
 
-	plli = ll_i2info(d_inode(parent));
 	task = kthread_run(ll_statahead_thread, parent, "ll_sa_%u",
-			   plli->lli_opendir_pid);
+			   lli->lli_opendir_pid);
 	thread = &sai->sai_thread;
 	if (IS_ERR(task)) {
 		rc = PTR_ERR(task);
-		CERROR("can't start ll_sa thread, rc: %d\n", rc);
-		dput(parent);
-		lli->lli_opendir_key = NULL;
-		thread_set_flags(thread, SVC_STOPPED);
-		thread_set_flags(&sai->sai_agl_thread, SVC_STOPPED);
-		/* Drop both our own local reference and the default
-		 * reference from allocation time.
-		 */
-		ll_sai_put(sai);
-		ll_sai_put(sai);
-		LASSERT(!lli->lli_sai);
-		return -EAGAIN;
+		CERROR("can't start ll_sa thread, rc : %d\n", rc);
+		goto out;
 	}
 
 	l_wait_event(thread->t_ctl_waitq,
@@ -1694,10 +1598,47 @@ int do_statahead_enter(struct inode *dir, struct dentry **dentryp,
 	return -EAGAIN;
 
 out:
-	kfree(sai);
+	/*
+	 * once we start statahead thread failed, disable statahead so
+	 * that subsequent stat won't waste time to try it.
+	 */
 	spin_lock(&lli->lli_sa_lock);
-	lli->lli_opendir_key = NULL;
-	lli->lli_opendir_pid = 0;
+	lli->lli_sa_enabled = 0;
+	lli->lli_sai = NULL;
 	spin_unlock(&lli->lli_sa_lock);
+	if (sai)
+		ll_sai_free(sai);
 	return rc;
 }
+
+/**
+ * statahead entry function, this is called when client getattr on a file, it
+ * will start statahead thread if this is the first dir entry, else revalidate
+ * dentry from statahead cache.
+ *
+ * \param[in]  dir	parent directory
+ * \param[out] dentryp	dentry to getattr
+ * \param[in]  unplug	unplug statahead window only (normally for negative
+ *			dentry)
+ * \retval		1 on success
+ * \retval		0 revalidation from statahead cache failed, caller needs
+ *			to getattr from server directly
+ * \retval		negative number on error, caller often ignores this and
+ *			then getattr from server
+ */
+int ll_statahead(struct inode *dir, struct dentry **dentryp, bool unplug)
+{
+	struct ll_statahead_info *sai;
+
+	sai = ll_sai_get(dir);
+	if (sai) {
+		int rc;
+
+		rc = revalidate_statahead_dentry(dir, sai, dentryp, unplug);
+		CDEBUG(D_READA, "revalidate statahead %pd: %d.\n",
+		       *dentryp, rc);
+		ll_sai_put(sai);
+		return rc;
+	}
+	return start_statahead_thread(dir, *dentryp);
+}
diff --git a/drivers/staging/lustre/lustre/llite/super25.c b/drivers/staging/lustre/lustre/llite/super25.c
index 3dd7e0eb0b54..106cd00910a7 100644
--- a/drivers/staging/lustre/lustre/llite/super25.c
+++ b/drivers/staging/lustre/lustre/llite/super25.c
@@ -34,7 +34,6 @@
 
 #include <linux/module.h>
 #include <linux/types.h>
-#include "../include/lustre_lite.h"
 #include "../include/lustre_ha.h"
 #include "../include/lustre_dlm.h"
 #include <linux/init.h>
@@ -83,8 +82,6 @@ struct super_operations lustre_super_operations = {
 };
 MODULE_ALIAS_FS("lustre");
 
-void lustre_register_client_process_config(int (*cpc)(struct lustre_cfg *lcfg));
-
 static int __init lustre_init(void)
 {
 	lnet_process_id_t lnet_id;
@@ -102,8 +99,8 @@ static int __init lustre_init(void)
 
 	rc = -ENOMEM;
 	ll_inode_cachep = kmem_cache_create("lustre_inode_cache",
-					    sizeof(struct ll_inode_info),
-					    0, SLAB_HWCACHE_ALIGN|SLAB_ACCOUNT,
+					    sizeof(struct ll_inode_info), 0,
+					    SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT,
 					    NULL);
 	if (!ll_inode_cachep)
 		goto out_cache;
diff --git a/drivers/staging/lustre/lustre/llite/symlink.c b/drivers/staging/lustre/lustre/llite/symlink.c
index 8c8bdfe1ad71..82c7c48aa619 100644
--- a/drivers/staging/lustre/lustre/llite/symlink.c
+++ b/drivers/staging/lustre/lustre/llite/symlink.c
@@ -35,7 +35,6 @@
 #include <linux/stat.h>
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/lustre_lite.h"
 #include "llite_internal.h"
 
 static int ll_readlink_internal(struct inode *inode,
@@ -80,17 +79,17 @@ static int ll_readlink_internal(struct inode *inode,
 	}
 
 	body = req_capsule_server_get(&(*request)->rq_pill, &RMF_MDT_BODY);
-	if ((body->valid & OBD_MD_LINKNAME) == 0) {
+	if ((body->mbo_valid & OBD_MD_LINKNAME) == 0) {
 		CERROR("OBD_MD_LINKNAME not set on reply\n");
 		rc = -EPROTO;
 		goto failed;
 	}
 
 	LASSERT(symlen != 0);
-	if (body->eadatasize != symlen) {
+	if (body->mbo_eadatasize != symlen) {
 		CERROR("%s: inode "DFID": symlink length %d not expected %d\n",
 		       ll_get_fsname(inode->i_sb, NULL, 0),
-		       PFID(ll_inode2fid(inode)), body->eadatasize - 1,
+		       PFID(ll_inode2fid(inode)), body->mbo_eadatasize - 1,
 		       symlen - 1);
 		rc = -EPROTO;
 		goto failed;
@@ -155,8 +154,5 @@ const struct inode_operations ll_fast_symlink_inode_operations = {
 	.get_link	= ll_get_link,
 	.getattr	= ll_getattr,
 	.permission	= ll_inode_permission,
-	.setxattr	= ll_setxattr,
-	.getxattr	= ll_getxattr,
 	.listxattr	= ll_listxattr,
-	.removexattr	= ll_removexattr,
 };
diff --git a/drivers/staging/lustre/lustre/llite/vvp_dev.c b/drivers/staging/lustre/lustre/llite/vvp_dev.c
index e623216e962d..8aa8ecc09a48 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_dev.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_dev.c
@@ -38,7 +38,6 @@
 #define DEBUG_SUBSYSTEM S_LLITE
 
 #include "../include/obd.h"
-#include "../include/lustre_lite.h"
 #include "llite_internal.h"
 #include "vvp_internal.h"
 
@@ -368,12 +367,6 @@ int cl_sb_fini(struct super_block *sb)
 		CERROR("Cannot cleanup cl-stack due to memory shortage.\n");
 		result = PTR_ERR(env);
 	}
-	/*
-	 * If mount failed (sbi->ll_cl == NULL), and this there are no other
-	 * mounts, stop device types manually (this usually happens
-	 * automatically when last device is destroyed).
-	 */
-	lu_types_stop();
 	return result;
 }
 
diff --git a/drivers/staging/lustre/lustre/llite/vvp_internal.h b/drivers/staging/lustre/lustre/llite/vvp_internal.h
index 79fc428461ed..4464ad258387 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_internal.h
+++ b/drivers/staging/lustre/lustre/llite/vvp_internal.h
@@ -49,14 +49,6 @@ struct obd_device;
 struct obd_export;
 struct page;
 
-/* specific architecture can implement only part of this list */
-enum vvp_io_subtype {
-	/** normal IO */
-	IO_NORMAL,
-	/** io started from splice_{read|write} */
-	IO_SPLICE
-};
-
 /**
  * IO state private to IO state private to VVP layer.
  */
@@ -99,10 +91,6 @@ struct vvp_io {
 			bool		ft_flags_valid;
 		} fault;
 		struct {
-			struct pipe_inode_info	*vui_pipe;
-			unsigned int		 vui_flags;
-		} splice;
-		struct {
 			struct cl_page_list vui_queue;
 			unsigned long vui_written;
 			int vui_from;
@@ -110,8 +98,6 @@ struct vvp_io {
 		} write;
 	} u;
 
-	enum vvp_io_subtype	vui_io_subtype;
-
 	/**
 	 * Layout version when this IO is initialized
 	 */
@@ -217,11 +203,12 @@ struct vvp_object {
 	struct list_head	vob_pending_list;
 
 	/**
-	 * Access this counter is protected by inode->i_sem. Now that
-	 * the lifetime of transient pages must be covered by inode sem,
-	 * we don't need to hold any lock..
+	 * Number of transient pages.  This is no longer protected by i_sem,
+	 * and needs to be atomic.  This is not actually used for anything,
+	 * and can probably be removed.
 	 */
-	int			vob_transient_pages;
+	atomic_t		vob_transient_pages;
+
 	/**
 	 * Number of outstanding mmaps on this file.
 	 *
@@ -247,9 +234,9 @@ struct vvp_object {
  */
 struct vvp_page {
 	struct cl_page_slice vpg_cl;
-	int		  vpg_defer_uptodate;
-	int		  vpg_ra_used;
-	int		  vpg_write_queued;
+	unsigned int	vpg_defer_uptodate:1,
+			vpg_ra_used:1,
+			vpg_write_queued:1;
 	/**
 	 * Non-empty iff this page is already counted in
 	 * vvp_object::vob_pending_list. This list is only used as a flag,
diff --git a/drivers/staging/lustre/lustre/llite/vvp_io.c b/drivers/staging/lustre/lustre/llite/vvp_io.c
index 94916dcc6caa..2b7f182a15e2 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_io.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_io.c
@@ -38,7 +38,6 @@
 #define DEBUG_SUBSYSTEM S_LLITE
 
 #include "../include/obd.h"
-#include "../include/lustre_lite.h"
 
 #include "llite_internal.h"
 #include "vvp_internal.h"
@@ -55,18 +54,6 @@ static struct vvp_io *cl2vvp_io(const struct lu_env *env,
 }
 
 /**
- * True, if \a io is a normal io, False for splice_{read,write}
- */
-static int cl_is_normalio(const struct lu_env *env, const struct cl_io *io)
-{
-	struct vvp_io *vio = vvp_env_io(env);
-
-	LASSERT(io->ci_type == CIT_READ || io->ci_type == CIT_WRITE);
-
-	return vio->vui_io_subtype == IO_NORMAL;
-}
-
-/**
  * For swapping layout. The file's layout may have changed.
  * To avoid populating pages to a wrong stripe, we have to verify the
  * correctness of layout. It works because swapping layout processes
@@ -391,9 +378,6 @@ static int vvp_mmap_locks(const struct lu_env *env,
 
 	LASSERT(io->ci_type == CIT_READ || io->ci_type == CIT_WRITE);
 
-	if (!cl_is_normalio(env, io))
-		return 0;
-
 	if (!vio->vui_iter) /* nfs or loop back device write */
 		return 0;
 
@@ -462,15 +446,10 @@ static void vvp_io_advance(const struct lu_env *env,
 			   const struct cl_io_slice *ios,
 			   size_t nob)
 {
-	struct vvp_io    *vio = cl2vvp_io(env, ios);
-	struct cl_io     *io  = ios->cis_io;
 	struct cl_object *obj = ios->cis_io->ci_obj;
-
+	struct vvp_io	 *vio = cl2vvp_io(env, ios);
 	CLOBINVRNT(env, obj, vvp_object_invariant(obj));
 
-	if (!cl_is_normalio(env, io))
-		return;
-
 	iov_iter_reexpand(vio->vui_iter, vio->vui_tot_count  -= nob);
 }
 
@@ -479,7 +458,7 @@ static void vvp_io_update_iov(const struct lu_env *env,
 {
 	size_t size = io->u.ci_rw.crw_count;
 
-	if (!cl_is_normalio(env, io) || !vio->vui_iter)
+	if (!vio->vui_iter)
 		return;
 
 	iov_iter_truncate(vio->vui_iter, size);
@@ -628,7 +607,7 @@ static int vvp_io_setattr_time(const struct lu_env *env,
 		attr->cat_mtime = io->u.ci_setattr.sa_attr.lvb_mtime;
 		valid |= CAT_MTIME;
 	}
-	result = cl_object_attr_set(env, obj, attr, valid);
+	result = cl_object_attr_update(env, obj, attr, valid);
 	cl_object_attr_unlock(obj);
 
 	return result;
@@ -716,25 +695,8 @@ static int vvp_io_read_start(const struct lu_env *env,
 
 	/* BUG: 5972 */
 	file_accessed(file);
-	switch (vio->vui_io_subtype) {
-	case IO_NORMAL:
-		LASSERT(vio->vui_iocb->ki_pos == pos);
-		result = generic_file_read_iter(vio->vui_iocb, vio->vui_iter);
-		break;
-	case IO_SPLICE:
-		result = generic_file_splice_read(file, &pos,
-						  vio->u.splice.vui_pipe, cnt,
-						  vio->u.splice.vui_flags);
-		/* LU-1109: do splice read stripe by stripe otherwise if it
-		 * may make nfsd stuck if this read occupied all internal pipe
-		 * buffers.
-		 */
-		io->ci_continue = 0;
-		break;
-	default:
-		CERROR("Wrong IO type %u\n", vio->vui_io_subtype);
-		LBUG();
-	}
+	LASSERT(vio->vui_iocb->ki_pos == pos);
+	result = generic_file_read_iter(vio->vui_iocb, vio->vui_iter);
 
 out:
 	if (result >= 0) {
@@ -821,7 +783,7 @@ static void write_commit_callback(const struct lu_env *env, struct cl_io *io,
 	cl_page_disown(env, io, page);
 
 	/* held in ll_cl_init() */
-	lu_ref_del(&page->cp_reference, "cl_io", io);
+	lu_ref_del(&page->cp_reference, "cl_io", cl_io_top(io));
 	cl_page_put(env, page);
 }
 
@@ -959,10 +921,30 @@ static int vvp_io_write_start(const struct lu_env *env,
 
 	CDEBUG(D_VFSTRACE, "write: [%lli, %lli)\n", pos, pos + (long long)cnt);
 
-	if (!vio->vui_iter) /* from a temp io in ll_cl_init(). */
+	if (!vio->vui_iter) {
+		/* from a temp io in ll_cl_init(). */
 		result = 0;
-	else
-		result = generic_file_write_iter(vio->vui_iocb, vio->vui_iter);
+	} else {
+		/*
+		 * When using the locked AIO function (generic_file_aio_write())
+		 * testing has shown the inode mutex to be a limiting factor
+		 * with multi-threaded single shared file performance. To get
+		 * around this, we now use the lockless version. To maintain
+		 * consistency, proper locking to protect against writes,
+		 * trucates, etc. is handled in the higher layers of lustre.
+		 */
+		bool lock_node = !IS_NOSEC(inode);
+
+		if (lock_node)
+			inode_lock(inode);
+		result = __generic_file_write_iter(vio->vui_iocb,
+						   vio->vui_iter);
+		if (lock_node)
+			inode_unlock(inode);
+
+		if (result > 0 || result == -EIOCBQUEUED)
+			result = generic_write_sync(vio->vui_iocb, result);
+	}
 
 	if (result > 0) {
 		result = vvp_io_write_commit(env, io);
diff --git a/drivers/staging/lustre/lustre/llite/vvp_lock.c b/drivers/staging/lustre/lustre/llite/vvp_lock.c
index 64be0c9df35b..07eb26cc43f5 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_lock.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_lock.c
@@ -37,7 +37,6 @@
 #define DEBUG_SUBSYSTEM S_LLITE
 
 #include "../include/obd_support.h"
-#include "../include/lustre_lite.h"
 
 #include "vvp_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/llite/vvp_object.c b/drivers/staging/lustre/lustre/llite/vvp_object.c
index 2c520b0bf6ca..b57195d15674 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_object.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_object.c
@@ -39,7 +39,6 @@
 #include "../../include/linux/libcfs/libcfs.h"
 
 #include "../include/obd.h"
-#include "../include/lustre_lite.h"
 
 #include "llite_internal.h"
 #include "vvp_internal.h"
@@ -68,8 +67,8 @@ static int vvp_object_print(const struct lu_env *env, void *cookie,
 
 	(*p)(env, cookie, "(%s %d %d) inode: %p ",
 	     list_empty(&obj->vob_pending_list) ? "-" : "+",
-	     obj->vob_transient_pages, atomic_read(&obj->vob_mmap_cnt),
-	     inode);
+	     atomic_read(&obj->vob_transient_pages),
+	     atomic_read(&obj->vob_mmap_cnt), inode);
 	if (inode) {
 		lli = ll_i2info(inode);
 		(*p)(env, cookie, "%lu/%u %o %u %d %p "DFID,
@@ -102,8 +101,8 @@ static int vvp_attr_get(const struct lu_env *env, struct cl_object *obj,
 	return 0; /* layers below have to fill in the rest */
 }
 
-static int vvp_attr_set(const struct lu_env *env, struct cl_object *obj,
-			const struct cl_attr *attr, unsigned valid)
+static int vvp_attr_update(const struct lu_env *env, struct cl_object *obj,
+			   const struct cl_attr *attr, unsigned int valid)
 {
 	struct inode *inode = vvp_object_inode(obj);
 
@@ -120,7 +119,7 @@ static int vvp_attr_set(const struct lu_env *env, struct cl_object *obj,
 	if (0 && valid & CAT_SIZE)
 		i_size_write(inode, attr->cat_size);
 	/* not currently necessary */
-	if (0 && valid & (CAT_UID|CAT_GID|CAT_SIZE))
+	if (0 && valid & (CAT_UID | CAT_GID | CAT_SIZE))
 		mark_inode_dirty(inode);
 	return 0;
 }
@@ -210,7 +209,7 @@ static const struct cl_object_operations vvp_ops = {
 	.coo_lock_init = vvp_lock_init,
 	.coo_io_init   = vvp_io_init,
 	.coo_attr_get  = vvp_attr_get,
-	.coo_attr_set  = vvp_attr_set,
+	.coo_attr_update = vvp_attr_update,
 	.coo_conf_set  = vvp_conf_set,
 	.coo_prune     = vvp_prune,
 	.coo_glimpse   = vvp_object_glimpse
@@ -221,7 +220,7 @@ static int vvp_object_init0(const struct lu_env *env,
 			    const struct cl_object_conf *conf)
 {
 	vob->vob_inode = conf->coc_inode;
-	vob->vob_transient_pages = 0;
+	atomic_set(&vob->vob_transient_pages, 0);
 	cl_object_page_init(&vob->vob_cl, sizeof(struct vvp_page));
 	return 0;
 }
diff --git a/drivers/staging/lustre/lustre/llite/vvp_page.c b/drivers/staging/lustre/lustre/llite/vvp_page.c
index 2e566d90bb94..046e84d7a158 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_page.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_page.c
@@ -44,8 +44,6 @@
 #include <linux/page-flags.h>
 #include <linux/pagemap.h>
 
-#include "../include/lustre_lite.h"
-
 #include "llite_internal.h"
 #include "vvp_internal.h"
 
@@ -243,13 +241,10 @@ static void vvp_vmpage_error(struct inode *inode, struct page *vmpage, int ioret
 		obj->vob_discard_page_warned = 0;
 	} else {
 		SetPageError(vmpage);
-		if (ioret == -ENOSPC)
-			set_bit(AS_ENOSPC, &inode->i_mapping->flags);
-		else
-			set_bit(AS_EIO, &inode->i_mapping->flags);
+		mapping_set_error(inode->i_mapping, ioret);
 
 		if ((ioret == -ESHUTDOWN || ioret == -EINTR) &&
-		     obj->vob_discard_page_warned == 0) {
+		    obj->vob_discard_page_warned == 0) {
 			obj->vob_discard_page_warned = 1;
 			ll_dirty_page_discard_warn(vmpage, ioret);
 		}
@@ -444,18 +439,10 @@ static int vvp_transient_page_prep(const struct lu_env *env,
 	return 0;
 }
 
-static void vvp_transient_page_verify(const struct cl_page *page)
-{
-	struct inode *inode = vvp_object_inode(page->cp_obj);
-
-	LASSERT(!inode_trylock(inode));
-}
-
 static int vvp_transient_page_own(const struct lu_env *env,
 				  const struct cl_page_slice *slice,
 				  struct cl_io *unused, int nonblock)
 {
-	vvp_transient_page_verify(slice->cpl_page);
 	return 0;
 }
 
@@ -463,21 +450,18 @@ static void vvp_transient_page_assume(const struct lu_env *env,
 				      const struct cl_page_slice *slice,
 				      struct cl_io *unused)
 {
-	vvp_transient_page_verify(slice->cpl_page);
 }
 
 static void vvp_transient_page_unassume(const struct lu_env *env,
 					const struct cl_page_slice *slice,
 					struct cl_io *unused)
 {
-	vvp_transient_page_verify(slice->cpl_page);
 }
 
 static void vvp_transient_page_disown(const struct lu_env *env,
 				      const struct cl_page_slice *slice,
 				      struct cl_io *unused)
 {
-	vvp_transient_page_verify(slice->cpl_page);
 }
 
 static void vvp_transient_page_discard(const struct lu_env *env,
@@ -486,8 +470,6 @@ static void vvp_transient_page_discard(const struct lu_env *env,
 {
 	struct cl_page *page = slice->cpl_page;
 
-	vvp_transient_page_verify(slice->cpl_page);
-
 	/*
 	 * For transient pages, remove it from the radix tree.
 	 */
@@ -511,7 +493,6 @@ vvp_transient_page_completion(const struct lu_env *env,
 			      const struct cl_page_slice *slice,
 			      int ioret)
 {
-	vvp_transient_page_verify(slice->cpl_page);
 }
 
 static void vvp_transient_page_fini(const struct lu_env *env,
@@ -522,8 +503,7 @@ static void vvp_transient_page_fini(const struct lu_env *env,
 	struct vvp_object *clobj = cl2vvp(clp->cp_obj);
 
 	vvp_page_fini_common(vpg);
-	LASSERT(!inode_trylock(clobj->vob_inode));
-	clobj->vob_transient_pages--;
+	atomic_dec(&clobj->vob_transient_pages);
 }
 
 static const struct cl_page_operations vvp_transient_page_ops = {
@@ -549,7 +529,7 @@ static const struct cl_page_operations vvp_transient_page_ops = {
 };
 
 int vvp_page_init(const struct lu_env *env, struct cl_object *obj,
-		struct cl_page *page, pgoff_t index)
+		  struct cl_page *page, pgoff_t index)
 {
 	struct vvp_page *vpg = cl_object_page_slice(obj, page);
 	struct page     *vmpage = page->cp_vmpage;
@@ -570,10 +550,9 @@ int vvp_page_init(const struct lu_env *env, struct cl_object *obj,
 	} else {
 		struct vvp_object *clobj = cl2vvp(obj);
 
-		LASSERT(!inode_trylock(clobj->vob_inode));
 		cl_page_slice_add(page, &vpg->vpg_cl, obj, index,
 				  &vvp_transient_page_ops);
-		clobj->vob_transient_pages++;
+		atomic_inc(&clobj->vob_transient_pages);
 	}
 	return 0;
 }
diff --git a/drivers/staging/lustre/lustre/llite/vvp_req.c b/drivers/staging/lustre/lustre/llite/vvp_req.c
index 9fe9d6c0a7d1..e3f4c790d646 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_req.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_req.c
@@ -32,7 +32,6 @@
 #include "../include/cl_object.h"
 #include "../include/obd.h"
 #include "../include/obd_support.h"
-#include "../include/lustre_lite.h"
 #include "llite_internal.h"
 #include "vvp_internal.h"
 
@@ -83,8 +82,10 @@ static void vvp_req_attr_set(const struct lu_env *env,
 	}
 	obdo_from_inode(oa, inode, valid_flags & flags);
 	obdo_set_parent_fid(oa, &ll_i2info(inode)->lli_fid);
+	if (OBD_FAIL_CHECK(OBD_FAIL_LFSCK_INVALID_PFID))
+		oa->o_parent_oid++;
 	memcpy(attr->cra_jobid, ll_i2info(inode)->lli_jobid,
-	       JOBSTATS_JOBID_SIZE);
+	       LUSTRE_JOBID_SIZE);
 }
 
 static void vvp_req_completion(const struct lu_env *env,
diff --git a/drivers/staging/lustre/lustre/llite/xattr.c b/drivers/staging/lustre/lustre/llite/xattr.c
index 98303cf85815..e070adb7a3cc 100644
--- a/drivers/staging/lustre/lustre/llite/xattr.c
+++ b/drivers/staging/lustre/lustre/llite/xattr.c
@@ -38,21 +38,12 @@
 #define DEBUG_SUBSYSTEM S_LLITE
 
 #include "../include/obd_support.h"
-#include "../include/lustre_lite.h"
 #include "../include/lustre_dlm.h"
 #include "../include/lustre_ver.h"
 #include "../include/lustre_eacl.h"
 
 #include "llite_internal.h"
 
-#define XATTR_USER_T	    (1)
-#define XATTR_TRUSTED_T	 (2)
-#define XATTR_SECURITY_T	(3)
-#define XATTR_ACL_ACCESS_T      (4)
-#define XATTR_ACL_DEFAULT_T     (5)
-#define XATTR_LUSTRE_T	  (6)
-#define XATTR_OTHER_T	   (7)
-
 static
 int get_xattr_type(const char *name)
 {
@@ -99,46 +90,57 @@ int xattr_type_filter(struct ll_sb_info *sbi, int xattr_type)
 	return 0;
 }
 
-static
-int ll_setxattr_common(struct inode *inode, const char *name,
-		       const void *value, size_t size,
-		       int flags, __u64 valid)
+static int
+ll_xattr_set_common(const struct xattr_handler *handler,
+		    struct dentry *dentry, struct inode *inode,
+		    const char *name, const void *value, size_t size,
+		    int flags)
 {
+	char fullname[strlen(handler->prefix) + strlen(name) + 1];
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 	struct ptlrpc_request *req = NULL;
-	int xattr_type, rc;
 	const char *pv = value;
+	__u64 valid;
+	int rc;
+
+	if (flags == XATTR_REPLACE) {
+		ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_REMOVEXATTR, 1);
+		valid = OBD_MD_FLXATTRRM;
+	} else {
+		ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_SETXATTR, 1);
+		valid = OBD_MD_FLXATTR;
+	}
 
-	xattr_type = get_xattr_type(name);
-	rc = xattr_type_filter(sbi, xattr_type);
+	rc = xattr_type_filter(sbi, handler->flags);
 	if (rc)
 		return rc;
 
-	if ((xattr_type == XATTR_ACL_ACCESS_T ||
-	     xattr_type == XATTR_ACL_DEFAULT_T) &&
+	if ((handler->flags == XATTR_ACL_ACCESS_T ||
+	     handler->flags == XATTR_ACL_DEFAULT_T) &&
 	    !inode_owner_or_capable(inode))
 		return -EPERM;
 
 	/* b10667: ignore lustre special xattr for now */
-	if ((xattr_type == XATTR_TRUSTED_T && strcmp(name, "trusted.lov") == 0) ||
-	    (xattr_type == XATTR_LUSTRE_T && strcmp(name, "lustre.lov") == 0))
+	if ((handler->flags == XATTR_TRUSTED_T && !strcmp(name, "lov")) ||
+	    (handler->flags == XATTR_LUSTRE_T && !strcmp(name, "lov")))
 		return 0;
 
 	/* b15587: ignore security.capability xattr for now */
-	if ((xattr_type == XATTR_SECURITY_T &&
-	     strcmp(name, "security.capability") == 0))
+	if ((handler->flags == XATTR_SECURITY_T &&
+	     !strcmp(name, "capability")))
 		return 0;
 
 	/* LU-549:  Disable security.selinux when selinux is disabled */
-	if (xattr_type == XATTR_SECURITY_T && !selinux_is_enabled() &&
-	    strcmp(name, "security.selinux") == 0)
+	if (handler->flags == XATTR_SECURITY_T && !selinux_is_enabled() &&
+	    strcmp(name, "selinux") == 0)
 		return -EOPNOTSUPP;
 
+	sprintf(fullname, "%s%s\n", handler->prefix, name);
 	rc = md_setxattr(sbi->ll_md_exp, ll_inode2fid(inode),
-			 valid, name, pv, size, 0, flags,
+			 valid, fullname, pv, size, 0, flags,
 			 ll_i2suppgid(inode), &req);
 	if (rc) {
-		if (rc == -EOPNOTSUPP && xattr_type == XATTR_USER_T) {
+		if (rc == -EOPNOTSUPP && handler->flags == XATTR_USER_T) {
 			LCONSOLE_INFO("Disabling user_xattr feature because it is not supported on the server\n");
 			sbi->ll_flags &= ~LL_SBI_USER_XATTR;
 		}
@@ -149,8 +151,10 @@ int ll_setxattr_common(struct inode *inode, const char *name,
 	return 0;
 }
 
-int ll_setxattr(struct dentry *dentry, struct inode *inode,
-		const char *name, const void *value, size_t size, int flags)
+static int ll_xattr_set(const struct xattr_handler *handler,
+			struct dentry *dentry, struct inode *inode,
+			const char *name, const void *value, size_t size,
+			int flags)
 {
 	LASSERT(inode);
 	LASSERT(name);
@@ -158,20 +162,24 @@ int ll_setxattr(struct dentry *dentry, struct inode *inode,
 	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p), xattr %s\n",
 	       PFID(ll_inode2fid(inode)), inode, name);
 
-	ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_SETXATTR, 1);
-
-	if ((strncmp(name, XATTR_TRUSTED_PREFIX,
-		     sizeof(XATTR_TRUSTED_PREFIX) - 1) == 0 &&
-	     strcmp(name + sizeof(XATTR_TRUSTED_PREFIX) - 1, "lov") == 0) ||
-	    (strncmp(name, XATTR_LUSTRE_PREFIX,
-		     sizeof(XATTR_LUSTRE_PREFIX) - 1) == 0 &&
-	     strcmp(name + sizeof(XATTR_LUSTRE_PREFIX) - 1, "lov") == 0)) {
+	if (!strcmp(name, "lov")) {
 		struct lov_user_md *lump = (struct lov_user_md *)value;
+		int op_type = flags == XATTR_REPLACE ? LPROC_LL_REMOVEXATTR :
+						       LPROC_LL_SETXATTR;
 		int rc = 0;
 
+		ll_stats_ops_tally(ll_i2sbi(inode), op_type, 1);
+
 		if (size != 0 && size < sizeof(struct lov_user_md))
 			return -EINVAL;
 
+		/*
+		 * It is possible to set an xattr to a "" value of zero size.
+		 * For this case we are going to treat it as a removal.
+		 */
+		if (!size && lump)
+			lump = NULL;
+
 		/* Attributes that are saved via getxattr will always have
 		 * the stripe_offset as 0.  Instead, the MDS should be
 		 * allowed to pick the starting OST index.   b=17846
@@ -181,12 +189,15 @@ int ll_setxattr(struct dentry *dentry, struct inode *inode,
 
 		if (lump && S_ISREG(inode->i_mode)) {
 			__u64 it_flags = FMODE_WRITE;
-			int lum_size = (lump->lmm_magic == LOV_USER_MAGIC_V1) ?
-				sizeof(*lump) : sizeof(struct lov_user_md_v3);
+			int lum_size;
+
+			lum_size = ll_lov_user_md_size(lump);
+			if (lum_size < 0 || size < lum_size)
+				return 0; /* b=10667: ignore error */
 
 			rc = ll_lov_setstripe_ea_info(inode, dentry, it_flags,
 						      lump, lum_size);
-			/* b10667: rc always be 0 here for now */
+			/* b=10667: rc always be 0 here for now */
 			rc = 0;
 		} else if (S_ISDIR(inode->i_mode)) {
 			rc = ll_dir_setstripe(inode, lump, 0);
@@ -194,92 +205,27 @@ int ll_setxattr(struct dentry *dentry, struct inode *inode,
 
 		return rc;
 
-	} else if (strcmp(name, XATTR_NAME_LMA) == 0 ||
-		   strcmp(name, XATTR_NAME_LINK) == 0)
+	} else if (!strcmp(name, "lma") || !strcmp(name, "link")) {
+		ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_SETXATTR, 1);
 		return 0;
+	}
 
-	return ll_setxattr_common(inode, name, value, size, flags,
-				  OBD_MD_FLXATTR);
-}
-
-int ll_removexattr(struct dentry *dentry, const char *name)
-{
-	struct inode *inode = d_inode(dentry);
-
-	LASSERT(inode);
-	LASSERT(name);
-
-	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p), xattr %s\n",
-	       PFID(ll_inode2fid(inode)), inode, name);
-
-	ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_REMOVEXATTR, 1);
-	return ll_setxattr_common(inode, name, NULL, 0, 0,
-				  OBD_MD_FLXATTRRM);
+	return ll_xattr_set_common(handler, dentry, inode, name, value, size,
+				   flags);
 }
 
-static
-int ll_getxattr_common(struct inode *inode, const char *name,
-		       void *buffer, size_t size, __u64 valid)
+int
+ll_xattr_list(struct inode *inode, const char *name, int type, void *buffer,
+	      size_t size, __u64 valid)
 {
+	struct ll_inode_info *lli = ll_i2info(inode);
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 	struct ptlrpc_request *req = NULL;
 	struct mdt_body *body;
-	int xattr_type, rc;
 	void *xdata;
-	struct ll_inode_info *lli = ll_i2info(inode);
-
-	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p)\n",
-	       PFID(ll_inode2fid(inode)), inode);
-
-	/* listxattr have slightly different behavior from of ext3:
-	 * without 'user_xattr' ext3 will list all xattr names but
-	 * filtered out "^user..*"; we list them all for simplicity.
-	 */
-	if (!name) {
-		xattr_type = XATTR_OTHER_T;
-		goto do_getxattr;
-	}
+	int rc;
 
-	xattr_type = get_xattr_type(name);
-	rc = xattr_type_filter(sbi, xattr_type);
-	if (rc)
-		return rc;
-
-	/* b15587: ignore security.capability xattr for now */
-	if ((xattr_type == XATTR_SECURITY_T &&
-	     strcmp(name, "security.capability") == 0))
-		return -ENODATA;
-
-	/* LU-549:  Disable security.selinux when selinux is disabled */
-	if (xattr_type == XATTR_SECURITY_T && !selinux_is_enabled() &&
-	    strcmp(name, "security.selinux") == 0)
-		return -EOPNOTSUPP;
-
-#ifdef CONFIG_FS_POSIX_ACL
-	/* posix acl is under protection of LOOKUP lock. when calling to this,
-	 * we just have path resolution to the target inode, so we have great
-	 * chance that cached ACL is uptodate.
-	 */
-	if (xattr_type == XATTR_ACL_ACCESS_T) {
-		struct posix_acl *acl;
-
-		spin_lock(&lli->lli_lock);
-		acl = posix_acl_dup(lli->lli_posix_acl);
-		spin_unlock(&lli->lli_lock);
-
-		if (!acl)
-			return -ENODATA;
-
-		rc = posix_acl_to_xattr(&init_user_ns, acl, buffer, size);
-		posix_acl_release(acl);
-		return rc;
-	}
-	if (xattr_type == XATTR_ACL_DEFAULT_T && !S_ISDIR(inode->i_mode))
-		return -ENODATA;
-#endif
-
-do_getxattr:
-	if (sbi->ll_xattr_cache_enabled && xattr_type != XATTR_ACL_ACCESS_T) {
+	if (sbi->ll_xattr_cache_enabled && type != XATTR_ACL_ACCESS_T) {
 		rc = ll_xattr_cache_get(inode, name, buffer, size, valid);
 		if (rc == -EAGAIN)
 			goto getxattr_nocache;
@@ -311,36 +257,36 @@ getxattr_nocache:
 
 		/* only detect the xattr size */
 		if (size == 0) {
-			rc = body->eadatasize;
+			rc = body->mbo_eadatasize;
 			goto out;
 		}
 
-		if (size < body->eadatasize) {
+		if (size < body->mbo_eadatasize) {
 			CERROR("server bug: replied size %u > %u\n",
-			       body->eadatasize, (int)size);
+			       body->mbo_eadatasize, (int)size);
 			rc = -ERANGE;
 			goto out;
 		}
 
-		if (body->eadatasize == 0) {
+		if (body->mbo_eadatasize == 0) {
 			rc = -ENODATA;
 			goto out;
 		}
 
 		/* do not need swab xattr data */
 		xdata = req_capsule_server_sized_get(&req->rq_pill, &RMF_EADATA,
-						     body->eadatasize);
+						     body->mbo_eadatasize);
 		if (!xdata) {
 			rc = -EFAULT;
 			goto out;
 		}
 
-		memcpy(buffer, xdata, body->eadatasize);
-		rc = body->eadatasize;
+		memcpy(buffer, xdata, body->mbo_eadatasize);
+		rc = body->mbo_eadatasize;
 	}
 
 out_xattr:
-	if (rc == -EOPNOTSUPP && xattr_type == XATTR_USER_T) {
+	if (rc == -EOPNOTSUPP && type == XATTR_USER_T) {
 		LCONSOLE_INFO(
 			"%s: disabling user_xattr feature because it is not supported on the server: rc = %d\n",
 			ll_get_fsname(inode->i_sb, NULL, 0), rc);
@@ -351,8 +297,65 @@ out:
 	return rc;
 }
 
-ssize_t ll_getxattr(struct dentry *dentry, struct inode *inode,
-		    const char *name, void *buffer, size_t size)
+static int ll_xattr_get_common(const struct xattr_handler *handler,
+			       struct dentry *dentry, struct inode *inode,
+			       const char *name, void *buffer, size_t size)
+{
+	char fullname[strlen(handler->prefix) + strlen(name) + 1];
+	struct ll_sb_info *sbi = ll_i2sbi(inode);
+#ifdef CONFIG_FS_POSIX_ACL
+	struct ll_inode_info *lli = ll_i2info(inode);
+#endif
+	int rc;
+
+	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p)\n",
+	       PFID(ll_inode2fid(inode)), inode);
+
+	ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_GETXATTR, 1);
+
+	rc = xattr_type_filter(sbi, handler->flags);
+	if (rc)
+		return rc;
+
+	/* b15587: ignore security.capability xattr for now */
+	if ((handler->flags == XATTR_SECURITY_T && !strcmp(name, "capability")))
+		return -ENODATA;
+
+	/* LU-549:  Disable security.selinux when selinux is disabled */
+	if (handler->flags == XATTR_SECURITY_T && !selinux_is_enabled() &&
+	    !strcmp(name, "selinux"))
+		return -EOPNOTSUPP;
+
+#ifdef CONFIG_FS_POSIX_ACL
+	/* posix acl is under protection of LOOKUP lock. when calling to this,
+	 * we just have path resolution to the target inode, so we have great
+	 * chance that cached ACL is uptodate.
+	 */
+	if (handler->flags == XATTR_ACL_ACCESS_T) {
+		struct posix_acl *acl;
+
+		spin_lock(&lli->lli_lock);
+		acl = posix_acl_dup(lli->lli_posix_acl);
+		spin_unlock(&lli->lli_lock);
+
+		if (!acl)
+			return -ENODATA;
+
+		rc = posix_acl_to_xattr(&init_user_ns, acl, buffer, size);
+		posix_acl_release(acl);
+		return rc;
+	}
+	if (handler->flags == XATTR_ACL_DEFAULT_T && !S_ISDIR(inode->i_mode))
+		return -ENODATA;
+#endif
+	sprintf(fullname, "%s%s\n", handler->prefix, name);
+	return ll_xattr_list(inode, fullname, handler->flags, buffer, size,
+			     OBD_MD_FLXATTR);
+}
+
+static int ll_xattr_get(const struct xattr_handler *handler,
+			struct dentry *dentry, struct inode *inode,
+			const char *name, void *buffer, size_t size)
 {
 	LASSERT(inode);
 	LASSERT(name);
@@ -360,36 +363,23 @@ ssize_t ll_getxattr(struct dentry *dentry, struct inode *inode,
 	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p), xattr %s\n",
 	       PFID(ll_inode2fid(inode)), inode, name);
 
-	ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_GETXATTR, 1);
-
-	if ((strncmp(name, XATTR_TRUSTED_PREFIX,
-		     sizeof(XATTR_TRUSTED_PREFIX) - 1) == 0 &&
-	     strcmp(name + sizeof(XATTR_TRUSTED_PREFIX) - 1, "lov") == 0) ||
-	    (strncmp(name, XATTR_LUSTRE_PREFIX,
-		     sizeof(XATTR_LUSTRE_PREFIX) - 1) == 0 &&
-	     strcmp(name + sizeof(XATTR_LUSTRE_PREFIX) - 1, "lov") == 0)) {
+	if (!strcmp(name, "lov")) {
 		struct lov_stripe_md *lsm;
 		struct lov_user_md *lump;
 		struct lov_mds_md *lmm = NULL;
 		struct ptlrpc_request *request = NULL;
 		int rc = 0, lmmsize = 0;
 
+		ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_GETXATTR, 1);
+
 		if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))
 			return -ENODATA;
 
-		if (size == 0 && S_ISDIR(inode->i_mode)) {
-			/* XXX directory EA is fix for now, optimize to save
-			 * RPC transfer
-			 */
-			rc = sizeof(struct lov_user_md);
-			goto out;
-		}
-
 		lsm = ccc_inode_lsm_get(inode);
 		if (!lsm) {
 			if (S_ISDIR(inode->i_mode)) {
-				rc = ll_dir_getstripe(inode, &lmm,
-						      &lmmsize, &request);
+				rc = ll_dir_getstripe(inode, (void **)&lmm,
+						      &lmmsize, &request, 0);
 			} else {
 				rc = -ENODATA;
 			}
@@ -439,7 +429,7 @@ out:
 		return rc;
 	}
 
-	return ll_getxattr_common(inode, name, buffer, size, OBD_MD_FLXATTR);
+	return ll_xattr_get_common(handler, dentry, inode, name, buffer, size);
 }
 
 ssize_t ll_listxattr(struct dentry *dentry, char *buffer, size_t size)
@@ -457,7 +447,8 @@ ssize_t ll_listxattr(struct dentry *dentry, char *buffer, size_t size)
 
 	ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_LISTXATTR, 1);
 
-	rc = ll_getxattr_common(inode, NULL, buffer, size, OBD_MD_FLXATTRLS);
+	rc = ll_xattr_list(inode, NULL, XATTR_OTHER_T, buffer, size,
+			   OBD_MD_FLXATTRLS);
 	if (rc < 0)
 		goto out;
 
@@ -488,7 +479,8 @@ ssize_t ll_listxattr(struct dentry *dentry, char *buffer, size_t size)
 		if (!ll_i2info(inode)->lli_has_smd)
 			rc2 = -1;
 	} else if (S_ISDIR(inode->i_mode)) {
-		rc2 = ll_dir_getstripe(inode, &lmm, &lmmsize, &request);
+		rc2 = ll_dir_getstripe(inode, (void **)&lmm, &lmmsize,
+				       &request, 0);
 	}
 
 	if (rc2 < 0) {
@@ -518,3 +510,57 @@ out:
 
 	return rc;
 }
+
+static const struct xattr_handler ll_user_xattr_handler = {
+	.prefix = XATTR_USER_PREFIX,
+	.flags = XATTR_USER_T,
+	.get = ll_xattr_get_common,
+	.set = ll_xattr_set_common,
+};
+
+static const struct xattr_handler ll_trusted_xattr_handler = {
+	.prefix = XATTR_TRUSTED_PREFIX,
+	.flags = XATTR_TRUSTED_T,
+	.get = ll_xattr_get,
+	.set = ll_xattr_set,
+};
+
+static const struct xattr_handler ll_security_xattr_handler = {
+	.prefix = XATTR_SECURITY_PREFIX,
+	.flags = XATTR_SECURITY_T,
+	.get = ll_xattr_get_common,
+	.set = ll_xattr_set_common,
+};
+
+static const struct xattr_handler ll_acl_access_xattr_handler = {
+	.prefix = XATTR_NAME_POSIX_ACL_ACCESS,
+	.flags = XATTR_ACL_ACCESS_T,
+	.get = ll_xattr_get_common,
+	.set = ll_xattr_set_common,
+};
+
+static const struct xattr_handler ll_acl_default_xattr_handler = {
+	.prefix = XATTR_NAME_POSIX_ACL_DEFAULT,
+	.flags = XATTR_ACL_DEFAULT_T,
+	.get = ll_xattr_get_common,
+	.set = ll_xattr_set_common,
+};
+
+static const struct xattr_handler ll_lustre_xattr_handler = {
+	.prefix = XATTR_LUSTRE_PREFIX,
+	.flags = XATTR_LUSTRE_T,
+	.get = ll_xattr_get,
+	.set = ll_xattr_set,
+};
+
+const struct xattr_handler *ll_xattr_handlers[] = {
+	&ll_user_xattr_handler,
+	&ll_trusted_xattr_handler,
+	&ll_security_xattr_handler,
+#ifdef CONFIG_FS_POSIX_ACL
+	&ll_acl_access_xattr_handler,
+	&ll_acl_default_xattr_handler,
+#endif
+	&ll_lustre_xattr_handler,
+	NULL,
+};
diff --git a/drivers/staging/lustre/lustre/llite/xattr_cache.c b/drivers/staging/lustre/lustre/llite/xattr_cache.c
index 8089da8143d9..50a19a40bd4e 100644
--- a/drivers/staging/lustre/lustre/llite/xattr_cache.c
+++ b/drivers/staging/lustre/lustre/llite/xattr_cache.c
@@ -13,7 +13,6 @@
 #include <linux/sched.h>
 #include <linux/mm.h>
 #include "../include/obd_support.h"
-#include "../include/lustre_lite.h"
 #include "../include/lustre_dlm.h"
 #include "../include/lustre_ver.h"
 #include "llite_internal.h"
@@ -270,10 +269,12 @@ static int ll_xattr_find_get_lock(struct inode *inode,
 	struct lustre_handle lockh = { 0 };
 	struct md_op_data *op_data;
 	struct ll_inode_info *lli = ll_i2info(inode);
-	struct ldlm_enqueue_info einfo = { .ei_type = LDLM_IBITS,
-					   .ei_mode = it_to_lock_mode(oit),
-					   .ei_cb_bl = ll_md_blocking_ast,
-					   .ei_cb_cp = ldlm_completion_ast };
+	struct ldlm_enqueue_info einfo = {
+		.ei_type = LDLM_IBITS,
+		.ei_mode = it_to_lock_mode(oit),
+		.ei_cb_bl = &ll_md_blocking_ast,
+		.ei_cb_cp = &ldlm_completion_ast,
+	};
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 	struct obd_export *exp = sbi->ll_md_exp;
 	int rc;
@@ -304,7 +305,7 @@ static int ll_xattr_find_get_lock(struct inode *inode,
 
 	op_data->op_valid = OBD_MD_FLXATTR | OBD_MD_FLXATTRLS;
 
-	rc = md_enqueue(exp, &einfo, oit, op_data, &lockh, NULL, 0, NULL, 0);
+	rc = md_enqueue(exp, &einfo, NULL, oit, op_data, &lockh, 0);
 	ll_finish_md_op_data(op_data);
 
 	if (rc < 0) {
@@ -380,25 +381,25 @@ static int ll_xattr_cache_refill(struct inode *inode, struct lookup_intent *oit)
 	}
 	/* do not need swab xattr data */
 	xdata = req_capsule_server_sized_get(&req->rq_pill, &RMF_EADATA,
-					     body->eadatasize);
+					     body->mbo_eadatasize);
 	xval = req_capsule_server_sized_get(&req->rq_pill, &RMF_EAVALS,
-					    body->aclsize);
+					    body->mbo_aclsize);
 	xsizes = req_capsule_server_sized_get(&req->rq_pill, &RMF_EAVALS_LENS,
-					      body->max_mdsize * sizeof(__u32));
+					      body->mbo_max_mdsize * sizeof(__u32));
 	if (!xdata || !xval || !xsizes) {
 		CERROR("wrong setxattr reply\n");
 		rc = -EPROTO;
 		goto out_destroy;
 	}
 
-	xtail = xdata + body->eadatasize;
-	xvtail = xval + body->aclsize;
+	xtail = xdata + body->mbo_eadatasize;
+	xvtail = xval + body->mbo_aclsize;
 
 	CDEBUG(D_CACHE, "caching: xdata=%p xtail=%p\n", xdata, xtail);
 
 	ll_xattr_cache_init(lli);
 
-	for (i = 0; i < body->max_mdsize; i++) {
+	for (i = 0; i < body->mbo_max_mdsize; i++) {
 		CDEBUG(D_CACHE, "caching [%s]=%.*s\n", xdata, *xsizes, xval);
 		/* Perform consistency checks: attr names and vals in pill */
 		if (!memchr(xdata, 0, xtail - xdata)) {
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_fld.c b/drivers/staging/lustre/lustre/lmv/lmv_fld.c
index a3d170aa6fd2..a5265f9b5797 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_fld.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_fld.c
@@ -47,18 +47,20 @@
 #include "../include/lprocfs_status.h"
 #include "lmv_internal.h"
 
-int lmv_fld_lookup(struct lmv_obd *lmv,
-		   const struct lu_fid *fid,
-		   u32 *mds)
+int lmv_fld_lookup(struct lmv_obd *lmv, const struct lu_fid *fid, u32 *mds)
 {
+	struct obd_device *obd = lmv2obd_dev(lmv);
 	int rc;
 
-	/* FIXME: Currently ZFS still use local seq for ROOT unfortunately, and
+	/*
+	 * FIXME: Currently ZFS still use local seq for ROOT unfortunately, and
 	 * this fid_is_local check should be removed once LU-2240 is fixed
 	 */
-	LASSERTF((fid_seq_in_fldb(fid_seq(fid)) ||
-		  fid_seq_is_local_file(fid_seq(fid))) &&
-		 fid_is_sane(fid), DFID" is insane!\n", PFID(fid));
+	if (!fid_is_sane(fid) || !(fid_seq_in_fldb(fid_seq(fid)) ||
+				   fid_seq_is_local_file(fid_seq(fid)))) {
+		CERROR("%s: invalid FID " DFID "\n", obd->obd_name, PFID(fid));
+		return -EINVAL;
+	}
 
 	rc = fld_client_lookup(&lmv->lmv_fld, fid_seq(fid), mds,
 			       LU_SEQ_RANGE_MDT, NULL);
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_intent.c b/drivers/staging/lustre/lustre/lmv/lmv_intent.c
index 2f58fdab8d1e..9f4e826bb0af 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_intent.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_intent.c
@@ -43,13 +43,13 @@
 #include "../include/lustre_lib.h"
 #include "../include/lustre_net.h"
 #include "../include/lustre_dlm.h"
+#include "../include/lustre_mdc.h"
 #include "../include/obd_class.h"
 #include "../include/lprocfs_status.h"
 #include "lmv_internal.h"
 
-static int lmv_intent_remote(struct obd_export *exp, void *lmm,
-			     int lmmsize, struct lookup_intent *it,
-			     const struct lu_fid *parent_fid, int flags,
+static int lmv_intent_remote(struct obd_export *exp, struct lookup_intent *it,
+			     const struct lu_fid *parent_fid,
 			     struct ptlrpc_request **reqp,
 			     ldlm_blocking_callback cb_blocking,
 			     __u64 extra_lock_flags)
@@ -68,7 +68,7 @@ static int lmv_intent_remote(struct obd_export *exp, void *lmm,
 	if (!body)
 		return -EPROTO;
 
-	LASSERT((body->valid & OBD_MD_MDS));
+	LASSERT((body->mbo_valid & OBD_MD_MDS));
 
 	/*
 	 * Unfortunately, we have to lie to MDC/MDS to retrieve
@@ -87,9 +87,9 @@ static int lmv_intent_remote(struct obd_export *exp, void *lmm,
 		it->it_request = NULL;
 	}
 
-	LASSERT(fid_is_sane(&body->fid1));
+	LASSERT(fid_is_sane(&body->mbo_fid1));
 
-	tgt = lmv_find_target(lmv, &body->fid1);
+	tgt = lmv_find_target(lmv, &body->mbo_fid1);
 	if (IS_ERR(tgt)) {
 		rc = PTR_ERR(tgt);
 		goto out;
@@ -101,7 +101,7 @@ static int lmv_intent_remote(struct obd_export *exp, void *lmm,
 		goto out;
 	}
 
-	op_data->op_fid1 = body->fid1;
+	op_data->op_fid1 = body->mbo_fid1;
 	/* Sent the parent FID to the remote MDT */
 	if (parent_fid) {
 		/* The parent fid is only for remote open to
@@ -110,18 +110,14 @@ static int lmv_intent_remote(struct obd_export *exp, void *lmm,
 		 */
 		LASSERT(it->it_op & IT_OPEN);
 		op_data->op_fid2 = *parent_fid;
-		/* Add object FID to op_fid3, in case it needs to check stale
-		 * (M_CHECK_STALE), see mdc_finish_intent_lock
-		 */
-		op_data->op_fid3 = body->fid1;
 	}
 
 	op_data->op_bias = MDS_CROSS_REF;
-	CDEBUG(D_INODE, "REMOTE_INTENT with fid="DFID" -> mds #%d\n",
-	       PFID(&body->fid1), tgt->ltd_idx);
+	CDEBUG(D_INODE, "REMOTE_INTENT with fid=" DFID " -> mds #%u\n",
+	       PFID(&body->mbo_fid1), tgt->ltd_idx);
 
-	rc = md_intent_lock(tgt->ltd_exp, op_data, lmm, lmmsize, it,
-			    flags, &req, cb_blocking, extra_lock_flags);
+	rc = md_intent_lock(tgt->ltd_exp, op_data, it, &req, cb_blocking,
+			    extra_lock_flags);
 	if (rc)
 		goto out_free_op_data;
 
@@ -136,8 +132,10 @@ static int lmv_intent_remote(struct obd_export *exp, void *lmm,
 		it->it_remote_lock_mode = it->it_lock_mode;
 	}
 
-	it->it_lock_handle = plock.cookie;
-	it->it_lock_mode = pmode;
+	if (pmode) {
+		it->it_lock_handle = plock.cookie;
+		it->it_lock_mode = pmode;
+	}
 
 out_free_op_data:
 	kfree(op_data);
@@ -150,13 +148,126 @@ out:
 	return rc;
 }
 
+int lmv_revalidate_slaves(struct obd_export *exp,
+			  const struct lmv_stripe_md *lsm,
+			  ldlm_blocking_callback cb_blocking,
+			  int extra_lock_flags)
+{
+	struct obd_device *obd = exp->exp_obd;
+	struct lmv_obd *lmv = &obd->u.lmv;
+	struct ptlrpc_request *req = NULL;
+	struct mdt_body *body;
+	struct md_op_data *op_data;
+	int rc = 0, i;
+
+	/**
+	 * revalidate slaves has some problems, temporarily return,
+	 * we may not need that
+	 */
+	op_data = kzalloc(sizeof(*op_data), GFP_NOFS);
+	if (!op_data)
+		return -ENOMEM;
+
+	/**
+	 * Loop over the stripe information, check validity and update them
+	 * from MDS if needed.
+	 */
+	for (i = 0; i < lsm->lsm_md_stripe_count; i++) {
+		struct lookup_intent it = { .it_op = IT_GETATTR };
+		struct lustre_handle *lockh = NULL;
+		struct lmv_tgt_desc *tgt = NULL;
+		struct inode *inode;
+		struct lu_fid fid;
+
+		fid = lsm->lsm_md_oinfo[i].lmo_fid;
+		inode = lsm->lsm_md_oinfo[i].lmo_root;
+
+		/*
+		 * Prepare op_data for revalidating. Note that @fid2 shluld be
+		 * defined otherwise it will go to server and take new lock
+		 * which is not needed here.
+		 */
+		memset(op_data, 0, sizeof(*op_data));
+		op_data->op_fid1 = fid;
+		op_data->op_fid2 = fid;
+
+		tgt = lmv_locate_mds(lmv, op_data, &fid);
+		if (IS_ERR(tgt)) {
+			rc = PTR_ERR(tgt);
+			goto cleanup;
+		}
+
+		CDEBUG(D_INODE, "Revalidate slave " DFID " -> mds #%u\n",
+		       PFID(&fid), tgt->ltd_idx);
+
+		if (req) {
+			ptlrpc_req_finished(req);
+			req = NULL;
+		}
+
+		rc = md_intent_lock(tgt->ltd_exp, op_data, &it, &req,
+				    cb_blocking, extra_lock_flags);
+		if (rc < 0)
+			goto cleanup;
+
+		lockh = (struct lustre_handle *)&it.it_lock_handle;
+		if (rc > 0 && !req) {
+			/* slave inode is still valid */
+			CDEBUG(D_INODE, "slave "DFID" is still valid.\n",
+			       PFID(&fid));
+			rc = 0;
+		} else {
+			/* refresh slave from server */
+			body = req_capsule_server_get(&req->rq_pill,
+						      &RMF_MDT_BODY);
+			LASSERT(body);
+
+			if (unlikely(body->mbo_nlink < 2)) {
+				CERROR("%s: nlink %d < 2 corrupt stripe %d "DFID":" DFID"\n",
+				       obd->obd_name, body->mbo_nlink, i,
+				       PFID(&lsm->lsm_md_oinfo[i].lmo_fid),
+				       PFID(&lsm->lsm_md_oinfo[0].lmo_fid));
+
+				if (it.it_lock_mode && lockh) {
+					ldlm_lock_decref(lockh, it.it_lock_mode);
+					it.it_lock_mode = 0;
+				}
+
+				rc = -EIO;
+				goto cleanup;
+			}
+
+			i_size_write(inode, body->mbo_size);
+			inode->i_blocks = body->mbo_blocks;
+			set_nlink(inode, body->mbo_nlink);
+			LTIME_S(inode->i_atime) = body->mbo_atime;
+			LTIME_S(inode->i_ctime) = body->mbo_ctime;
+			LTIME_S(inode->i_mtime) = body->mbo_mtime;
+		}
+
+		md_set_lock_data(tgt->ltd_exp, lockh, inode, NULL);
+
+		if (it.it_lock_mode && lockh) {
+			ldlm_lock_decref(lockh, it.it_lock_mode);
+			it.it_lock_mode = 0;
+		}
+	}
+
+cleanup:
+	if (req)
+		ptlrpc_req_finished(req);
+
+	kfree(op_data);
+	return rc;
+}
+
 /*
  * IT_OPEN is intended to open (and create, possible) an object. Parent (pid)
  * may be split dir.
  */
 static int lmv_intent_open(struct obd_export *exp, struct md_op_data *op_data,
-			   void *lmm, int lmmsize, struct lookup_intent *it,
-			   int flags, struct ptlrpc_request **reqp,
+			   struct lookup_intent *it,
+			   struct ptlrpc_request **reqp,
 			   ldlm_blocking_callback cb_blocking,
 			   __u64 extra_lock_flags)
 {
@@ -166,35 +277,55 @@ static int lmv_intent_open(struct obd_export *exp, struct md_op_data *op_data,
 	struct mdt_body		*body;
 	int			rc;
 
-	tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid1);
-	if (IS_ERR(tgt))
-		return PTR_ERR(tgt);
+	if (it->it_flags & MDS_OPEN_BY_FID) {
+		LASSERT(fid_is_sane(&op_data->op_fid2));
+
+		/*
+		 * for striped directory, we can't know parent stripe fid
+		 * without name, but we can set it to child fid, and MDT
+		 * will obtain it from linkea in open in such case.
+		 */
+		if (op_data->op_mea1)
+			op_data->op_fid1 = op_data->op_fid2;
+
+		tgt = lmv_find_target(lmv, &op_data->op_fid2);
+		if (IS_ERR(tgt))
+			return PTR_ERR(tgt);
+
+		op_data->op_mds = tgt->ltd_idx;
+	} else {
+		LASSERT(fid_is_sane(&op_data->op_fid1));
+		LASSERT(fid_is_zero(&op_data->op_fid2));
+		LASSERT(op_data->op_name);
+
+		tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid1);
+		if (IS_ERR(tgt))
+			return PTR_ERR(tgt);
+	}
 
 	/* If it is ready to open the file by FID, do not need
 	 * allocate FID at all, otherwise it will confuse MDT
 	 */
-	if ((it->it_op & IT_CREAT) &&
-	    !(it->it_flags & MDS_OPEN_BY_FID)) {
+	if ((it->it_op & IT_CREAT) && !(it->it_flags & MDS_OPEN_BY_FID)) {
 		/*
-		 * For open with IT_CREATE and for IT_CREATE cases allocate new
-		 * fid and setup FLD for it.
+		 * For lookup(IT_CREATE) cases allocate new fid and setup FLD
+		 * for it.
 		 */
-		op_data->op_fid3 = op_data->op_fid2;
-		rc = lmv_fid_alloc(exp, &op_data->op_fid2, op_data);
+		rc = lmv_fid_alloc(NULL, exp, &op_data->op_fid2, op_data);
 		if (rc != 0)
 			return rc;
 	}
 
-	CDEBUG(D_INODE, "OPEN_INTENT with fid1=" DFID ", fid2=" DFID ", name='%s' -> mds #%d\n",
+	CDEBUG(D_INODE, "OPEN_INTENT with fid1=" DFID ", fid2=" DFID ", name='%s' -> mds #%u\n",
 	       PFID(&op_data->op_fid1),
 	       PFID(&op_data->op_fid2), op_data->op_name, tgt->ltd_idx);
 
-	rc = md_intent_lock(tgt->ltd_exp, op_data, lmm, lmmsize, it, flags,
-			    reqp, cb_blocking, extra_lock_flags);
+	rc = md_intent_lock(tgt->ltd_exp, op_data, it, reqp, cb_blocking,
+			    extra_lock_flags);
 	if (rc != 0)
 		return rc;
 	/*
-	 * Nothing is found, do not access body->fid1 as it is zero and thus
+	 * Nothing is found, do not access body->mbo_fid1 as it is zero and thus
 	 * pointless.
 	 */
 	if ((it->it_disposition & DISP_LOOKUP_NEG) &&
@@ -205,31 +336,17 @@ static int lmv_intent_open(struct obd_export *exp, struct md_op_data *op_data,
 	body = req_capsule_server_get(&(*reqp)->rq_pill, &RMF_MDT_BODY);
 	if (!body)
 		return -EPROTO;
-	/*
-	 * Not cross-ref case, just get out of here.
-	 */
-	if (likely(!(body->valid & OBD_MD_MDS)))
-		return 0;
 
-	/*
-	 * Okay, MDS has returned success. Probably name has been resolved in
-	 * remote inode.
-	 */
-	rc = lmv_intent_remote(exp, lmm, lmmsize, it, &op_data->op_fid1, flags,
-			       reqp, cb_blocking, extra_lock_flags);
-	if (rc != 0) {
-		LASSERT(rc < 0);
-		/*
-		 * This is possible, that some userspace application will try to
-		 * open file as directory and we will have -ENOTDIR here. As
-		 * this is normal situation, we should not print error here,
-		 * only debug info.
-		 */
-		CDEBUG(D_INODE, "Can't handle remote %s: dir " DFID "(" DFID "):%*s: %d\n",
-		       LL_IT2STR(it), PFID(&op_data->op_fid2),
-		       PFID(&op_data->op_fid1), op_data->op_namelen,
-		       op_data->op_name, rc);
-		return rc;
+	/* Not cross-ref case, just get out of here. */
+	if (unlikely((body->mbo_valid & OBD_MD_MDS))) {
+		rc = lmv_intent_remote(exp, it, &op_data->op_fid1, reqp,
+				       cb_blocking, extra_lock_flags);
+		if (rc != 0)
+			return rc;
+
+		body = req_capsule_server_get(&(*reqp)->rq_pill, &RMF_MDT_BODY);
+		if (!body)
+			return -EPROTO;
 	}
 
 	return rc;
@@ -240,37 +357,102 @@ static int lmv_intent_open(struct obd_export *exp, struct md_op_data *op_data,
  */
 static int lmv_intent_lookup(struct obd_export *exp,
 			     struct md_op_data *op_data,
-			     void *lmm, int lmmsize, struct lookup_intent *it,
-			     int flags, struct ptlrpc_request **reqp,
+			     struct lookup_intent *it,
+			     struct ptlrpc_request **reqp,
 			     ldlm_blocking_callback cb_blocking,
 			     __u64 extra_lock_flags)
 {
+	struct lmv_stripe_md *lsm = op_data->op_mea1;
 	struct obd_device      *obd = exp->exp_obd;
 	struct lmv_obd	 *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc    *tgt = NULL;
 	struct mdt_body	*body;
 	int		     rc = 0;
 
+	/*
+	 * If it returns ERR_PTR(-EBADFD) then it is an unknown hash type
+	 * it will try all stripes to locate the object
+	 */
 	tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid1);
-	if (IS_ERR(tgt))
+	if (IS_ERR(tgt) && (PTR_ERR(tgt) != -EBADFD))
 		return PTR_ERR(tgt);
 
+	/*
+	 * Both migrating dir and unknown hash dir need to try
+	 * all of sub-stripes
+	 */
+	if (lsm && !lmv_is_known_hash_type(lsm->lsm_md_hash_type)) {
+		struct lmv_oinfo *oinfo = &lsm->lsm_md_oinfo[0];
+
+		op_data->op_fid1 = oinfo->lmo_fid;
+		op_data->op_mds = oinfo->lmo_mds;
+		tgt = lmv_get_target(lmv, oinfo->lmo_mds, NULL);
+		if (IS_ERR(tgt))
+			return PTR_ERR(tgt);
+	}
+
 	if (!fid_is_sane(&op_data->op_fid2))
 		fid_zero(&op_data->op_fid2);
 
-	CDEBUG(D_INODE, "LOOKUP_INTENT with fid1="DFID", fid2="DFID
-	       ", name='%s' -> mds #%d\n", PFID(&op_data->op_fid1),
-	       PFID(&op_data->op_fid2),
+	CDEBUG(D_INODE, "LOOKUP_INTENT with fid1=" DFID ", fid2=" DFID ", name='%s' -> mds #%u lsm=%p lsm_magic=%x\n",
+	       PFID(&op_data->op_fid1), PFID(&op_data->op_fid2),
 	       op_data->op_name ? op_data->op_name : "<NULL>",
-	       tgt->ltd_idx);
+	       tgt->ltd_idx, lsm, !lsm ? -1 : lsm->lsm_md_magic);
 
 	op_data->op_bias &= ~MDS_CROSS_REF;
 
-	rc = md_intent_lock(tgt->ltd_exp, op_data, lmm, lmmsize, it,
-			    flags, reqp, cb_blocking, extra_lock_flags);
+	rc = md_intent_lock(tgt->ltd_exp, op_data, it, reqp, cb_blocking,
+			    extra_lock_flags);
+	if (rc < 0)
+		return rc;
 
-	if (rc < 0 || !*reqp)
+	if (!*reqp) {
+		/*
+		 * If RPC happens, lsm information will be revalidated
+		 * during update_inode process (see ll_update_lsm_md)
+		 */
+		if (op_data->op_mea2) {
+			rc = lmv_revalidate_slaves(exp, op_data->op_mea2,
+						   cb_blocking,
+						   extra_lock_flags);
+			if (rc != 0)
+				return rc;
+		}
 		return rc;
+	} else if (it_disposition(it, DISP_LOOKUP_NEG) && lsm &&
+		   lmv_need_try_all_stripes(lsm)) {
+		/*
+		 * For migrating and unknown hash type directory, it will
+		 * try to target the entry on other stripes
+		 */
+		int stripe_index;
+
+		for (stripe_index = 1;
+		     stripe_index < lsm->lsm_md_stripe_count &&
+		     it_disposition(it, DISP_LOOKUP_NEG); stripe_index++) {
+			struct lmv_oinfo *oinfo;
+
+			/* release the previous request */
+			ptlrpc_req_finished(*reqp);
+			it->it_request = NULL;
+			*reqp = NULL;
+
+			oinfo = &lsm->lsm_md_oinfo[stripe_index];
+			tgt = lmv_find_target(lmv, &oinfo->lmo_fid);
+			if (IS_ERR(tgt))
+				return PTR_ERR(tgt);
+
+			CDEBUG(D_INODE, "Try other stripes " DFID"\n",
+			       PFID(&oinfo->lmo_fid));
+
+			op_data->op_fid1 = oinfo->lmo_fid;
+			it->it_disposition &= ~DISP_ENQ_COMPLETE;
+			rc = md_intent_lock(tgt->ltd_exp, op_data, it, reqp,
+					    cb_blocking, extra_lock_flags);
+			if (rc)
+				return rc;
+		}
+	}
 
 	/*
 	 * MDS has returned success. Probably name has been resolved in
@@ -279,19 +461,23 @@ static int lmv_intent_lookup(struct obd_export *exp,
 	body = req_capsule_server_get(&(*reqp)->rq_pill, &RMF_MDT_BODY);
 	if (!body)
 		return -EPROTO;
-	/* Not cross-ref case, just get out of here. */
-	if (likely(!(body->valid & OBD_MD_MDS)))
-		return 0;
 
-	rc = lmv_intent_remote(exp, lmm, lmmsize, it, NULL, flags, reqp,
-			       cb_blocking, extra_lock_flags);
+	/* Not cross-ref case, just get out of here. */
+	if (unlikely((body->mbo_valid & OBD_MD_MDS))) {
+		rc = lmv_intent_remote(exp, it, NULL, reqp, cb_blocking,
+				       extra_lock_flags);
+		if (rc != 0)
+			return rc;
+		body = req_capsule_server_get(&(*reqp)->rq_pill, &RMF_MDT_BODY);
+		if (!body)
+			return -EPROTO;
+	}
 
 	return rc;
 }
 
 int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
-		    void *lmm, int lmmsize, struct lookup_intent *it,
-		    int flags, struct ptlrpc_request **reqp,
+		    struct lookup_intent *it, struct ptlrpc_request **reqp,
 		    ldlm_blocking_callback cb_blocking,
 		    __u64 extra_lock_flags)
 {
@@ -300,8 +486,9 @@ int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 
 	LASSERT(fid_is_sane(&op_data->op_fid1));
 
-	CDEBUG(D_INODE, "INTENT LOCK '%s' for '%*s' on "DFID"\n",
-	       LL_IT2STR(it), op_data->op_namelen, op_data->op_name,
+	CDEBUG(D_INODE, "INTENT LOCK '%s' for "DFID" '%*s' on "DFID"\n",
+	       LL_IT2STR(it), PFID(&op_data->op_fid2),
+	       (int)op_data->op_namelen, op_data->op_name,
 	       PFID(&op_data->op_fid1));
 
 	rc = lmv_check_connect(obd);
@@ -309,14 +496,34 @@ int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 		return rc;
 
 	if (it->it_op & (IT_LOOKUP | IT_GETATTR | IT_LAYOUT))
-		rc = lmv_intent_lookup(exp, op_data, lmm, lmmsize, it,
-				       flags, reqp, cb_blocking,
+		rc = lmv_intent_lookup(exp, op_data, it, reqp, cb_blocking,
 				       extra_lock_flags);
 	else if (it->it_op & IT_OPEN)
-		rc = lmv_intent_open(exp, op_data, lmm, lmmsize, it,
-				     flags, reqp, cb_blocking,
+		rc = lmv_intent_open(exp, op_data, it, reqp, cb_blocking,
 				     extra_lock_flags);
 	else
 		LBUG();
+
+	if (rc < 0) {
+		struct lustre_handle lock_handle;
+
+		if (it->it_lock_mode) {
+			lock_handle.cookie = it->it_lock_handle;
+			ldlm_lock_decref(&lock_handle, it->it_lock_mode);
+		}
+
+		it->it_lock_handle = 0;
+		it->it_lock_mode = 0;
+
+		if (it->it_remote_lock_mode) {
+			lock_handle.cookie = it->it_remote_lock_handle;
+			ldlm_lock_decref(&lock_handle,
+					 it->it_remote_lock_mode);
+		}
+
+		it->it_remote_lock_handle = 0;
+		it->it_remote_lock_mode = 0;
+	}
+
 	return rc;
 }
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_internal.h b/drivers/staging/lustre/lustre/lmv/lmv_internal.h
index 0beafc49b8d2..52b03745ac19 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_internal.h
+++ b/drivers/staging/lustre/lustre/lmv/lmv_internal.h
@@ -35,6 +35,7 @@
 
 #include "../include/lustre/lustre_idl.h"
 #include "../include/obd.h"
+#include "../include/lustre_lmv.h"
 
 #define LMV_MAX_TGT_COUNT 128
 
@@ -44,77 +45,116 @@
 int lmv_check_connect(struct obd_device *obd);
 
 int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
-		    void *lmm, int lmmsize, struct lookup_intent *it,
-		    int flags, struct ptlrpc_request **reqp,
+		    struct lookup_intent *it, struct ptlrpc_request **reqp,
 		    ldlm_blocking_callback cb_blocking,
 		    __u64 extra_lock_flags);
 
 int lmv_fld_lookup(struct lmv_obd *lmv, const struct lu_fid *fid, u32 *mds);
 int __lmv_fid_alloc(struct lmv_obd *lmv, struct lu_fid *fid, u32 mds);
-int lmv_fid_alloc(struct obd_export *exp, struct lu_fid *fid,
-		  struct md_op_data *op_data);
+int lmv_fid_alloc(const struct lu_env *env, struct obd_export *exp,
+		  struct lu_fid *fid, struct md_op_data *op_data);
 
-static inline struct lmv_stripe_md *lmv_get_mea(struct ptlrpc_request *req)
-{
-	struct mdt_body	 *body;
-	struct lmv_stripe_md    *mea;
-
-	LASSERT(req);
-
-	body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
-
-	if (!body || !S_ISDIR(body->mode) || !body->eadatasize)
-		return NULL;
+int lmv_unpack_md(struct obd_export *exp, struct lmv_stripe_md **lsmp,
+		  const union lmv_mds_md *lmm, int stripe_count);
 
-	mea = req_capsule_server_sized_get(&req->rq_pill, &RMF_MDT_MD,
-					   body->eadatasize);
-	if (mea->mea_count == 0)
-		return NULL;
-	if (mea->mea_magic != MEA_MAGIC_LAST_CHAR &&
-	    mea->mea_magic != MEA_MAGIC_ALL_CHARS &&
-	    mea->mea_magic != MEA_MAGIC_HASH_SEGMENT)
-		return NULL;
+int lmv_revalidate_slaves(struct obd_export *exp,
+			  const struct lmv_stripe_md *lsm,
+			  ldlm_blocking_callback cb_blocking,
+			  int extra_lock_flags);
 
-	return mea;
-}
-
-static inline int lmv_get_easize(struct lmv_obd *lmv)
+static inline struct obd_device *lmv2obd_dev(struct lmv_obd *lmv)
 {
-	return sizeof(struct lmv_stripe_md) +
-		lmv->desc.ld_tgt_count *
-		sizeof(struct lu_fid);
+	return container_of0(lmv, struct obd_device, u.lmv);
 }
 
 static inline struct lmv_tgt_desc *
-lmv_get_target(struct lmv_obd *lmv, u32 mds)
+lmv_get_target(struct lmv_obd *lmv, u32 mdt_idx, int *index)
 {
-	int count = lmv->desc.ld_tgt_count;
 	int i;
 
-	for (i = 0; i < count; i++) {
+	for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
 		if (!lmv->tgts[i])
 			continue;
 
-		if (lmv->tgts[i]->ltd_idx == mds)
+		if (lmv->tgts[i]->ltd_idx == mdt_idx) {
+			if (index)
+				*index = i;
 			return lmv->tgts[i];
+		}
 	}
 
 	return ERR_PTR(-ENODEV);
 }
 
-static inline struct lmv_tgt_desc *
-lmv_find_target(struct lmv_obd *lmv, const struct lu_fid *fid)
+static inline int
+lmv_find_target_index(struct lmv_obd *lmv, const struct lu_fid *fid)
 {
-	u32 mds = 0;
-	int rc;
+	struct lmv_tgt_desc *ltd;
+	u32 mdt_idx = 0;
+	int index = 0;
 
 	if (lmv->desc.ld_tgt_count > 1) {
-		rc = lmv_fld_lookup(lmv, fid, &mds);
-		if (rc)
-			return ERR_PTR(rc);
+		int rc;
+
+		rc = lmv_fld_lookup(lmv, fid, &mdt_idx);
+		if (rc < 0)
+			return rc;
 	}
 
-	return lmv_get_target(lmv, mds);
+	ltd = lmv_get_target(lmv, mdt_idx, &index);
+	if (IS_ERR(ltd))
+		return PTR_ERR(ltd);
+
+	return index;
+}
+
+static inline struct lmv_tgt_desc *
+lmv_find_target(struct lmv_obd *lmv, const struct lu_fid *fid)
+{
+	int index;
+
+	index = lmv_find_target_index(lmv, fid);
+	if (index < 0)
+		return ERR_PTR(index);
+
+	return lmv->tgts[index];
+}
+
+static inline int lmv_stripe_md_size(int stripe_count)
+{
+	struct lmv_stripe_md *lsm;
+
+	return sizeof(*lsm) + stripe_count * sizeof(lsm->lsm_md_oinfo[0]);
+}
+
+int lmv_name_to_stripe_index(enum lmv_hash_type hashtype,
+			     unsigned int max_mdt_index,
+			     const char *name, int namelen);
+
+static inline const struct lmv_oinfo *
+lsm_name_to_stripe_info(const struct lmv_stripe_md *lsm, const char *name,
+			int namelen)
+{
+	int stripe_index;
+
+	stripe_index = lmv_name_to_stripe_index(lsm->lsm_md_hash_type,
+						lsm->lsm_md_stripe_count,
+						name, namelen);
+	if (stripe_index < 0)
+		return ERR_PTR(stripe_index);
+
+	LASSERTF(stripe_index < lsm->lsm_md_stripe_count,
+		 "stripe_index = %d, stripe_count = %d hash_type = %x name = %.*s\n",
+		 stripe_index, lsm->lsm_md_stripe_count,
+		 lsm->lsm_md_hash_type, namelen, name);
+
+	return &lsm->lsm_md_oinfo[stripe_index];
+}
+
+static inline bool lmv_need_try_all_stripes(const struct lmv_stripe_md *lsm)
+{
+	return !lmv_is_known_hash_type(lsm->lsm_md_hash_type) ||
+	       lsm->lsm_md_hash_type & LMV_HASH_FLAG_MIGRATION;
 }
 
 struct lmv_tgt_desc
@@ -123,6 +163,6 @@ struct lmv_tgt_desc
 /* lproc_lmv.c */
 void lprocfs_lmv_init_vars(struct lprocfs_static_vars *lvars);
 
-extern struct file_operations lmv_proc_target_fops;
+extern const struct file_operations lmv_proc_target_fops;
 
 #endif
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_obd.c b/drivers/staging/lustre/lustre/lmv/lmv_obd.c
index 0e1588a43187..7dbb2b946acf 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_obd.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_obd.c
@@ -43,12 +43,13 @@
 
 #include "../include/lustre/lustre_idl.h"
 #include "../include/obd_support.h"
-#include "../include/lustre_lib.h"
 #include "../include/lustre_net.h"
 #include "../include/obd_class.h"
+#include "../include/lustre_lmv.h"
 #include "../include/lprocfs_status.h"
-#include "../include/lustre_lite.h"
+#include "../include/cl_object.h"
 #include "../include/lustre_fid.h"
+#include "../include/lustre/lustre_ioctl.h"
 #include "../include/lustre_kernelcomm.h"
 #include "lmv_internal.h"
 
@@ -70,12 +71,12 @@ static void lmv_activate_target(struct lmv_obd *lmv,
  *  -ENOTCONN: The UUID is found, but the target connection is bad (!)
  *  -EBADF   : The UUID is found, but the OBD of the wrong type (!)
  */
-static int lmv_set_mdc_active(struct lmv_obd *lmv, struct obd_uuid *uuid,
+static int lmv_set_mdc_active(struct lmv_obd *lmv, const struct obd_uuid *uuid,
 			      int activate)
 {
 	struct lmv_tgt_desc    *uninitialized_var(tgt);
 	struct obd_device      *obd;
-	int		     i;
+	u32		     i;
 	int		     rc = 0;
 
 	CDEBUG(D_INFO, "Searching in lmv %p for uuid %s (activate=%d)\n",
@@ -244,36 +245,12 @@ static int lmv_connect(const struct lu_env *env,
 	return rc;
 }
 
-static void lmv_set_timeouts(struct obd_device *obd)
-{
-	struct lmv_obd	*lmv;
-	int		    i;
-
-	lmv = &obd->u.lmv;
-	if (lmv->server_timeout == 0)
-		return;
-
-	if (lmv->connected == 0)
-		return;
-
-	for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
-		struct lmv_tgt_desc *tgt = lmv->tgts[i];
-
-		tgt = lmv->tgts[i];
-		if (!tgt || !tgt->ltd_exp || !tgt->ltd_active)
-			continue;
-
-		obd_set_info_async(NULL, tgt->ltd_exp, sizeof(KEY_INTERMDS),
-				   KEY_INTERMDS, 0, NULL, NULL);
-	}
-}
-
-static int lmv_init_ea_size(struct obd_export *exp, int easize,
-			    int def_easize, int cookiesize, int def_cookiesize)
+static int lmv_init_ea_size(struct obd_export *exp, u32 easize, u32 def_easize,
+			    u32 cookiesize, u32 def_cookiesize)
 {
 	struct obd_device   *obd = exp->exp_obd;
 	struct lmv_obd      *lmv = &obd->u.lmv;
-	int		  i;
+	u32 i;
 	int		  rc = 0;
 	int		  change = 0;
 
@@ -420,6 +397,7 @@ static int lmv_add_target(struct obd_device *obd, struct obd_uuid *uuidp,
 {
 	struct lmv_obd      *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc *tgt;
+	int orig_tgt_count = 0;
 	int		  rc = 0;
 
 	CDEBUG(D_CONFIG, "Target uuid: %s. index %d\n", uuidp->uuid, index);
@@ -489,14 +467,17 @@ static int lmv_add_target(struct obd_device *obd, struct obd_uuid *uuidp,
 	tgt->ltd_uuid = *uuidp;
 	tgt->ltd_active = 0;
 	lmv->tgts[index] = tgt;
-	if (index >= lmv->desc.ld_tgt_count)
+	if (index >= lmv->desc.ld_tgt_count) {
+		orig_tgt_count = lmv->desc.ld_tgt_count;
 		lmv->desc.ld_tgt_count = index + 1;
+	}
 
 	if (lmv->connected) {
 		rc = lmv_connect_mdc(obd, tgt);
 		if (rc) {
 			spin_lock(&lmv->lmv_lock);
-			lmv->desc.ld_tgt_count--;
+			if (lmv->desc.ld_tgt_count == index + 1)
+				lmv->desc.ld_tgt_count = orig_tgt_count;
 			memset(tgt, 0, sizeof(*tgt));
 			spin_unlock(&lmv->lmv_lock);
 		} else {
@@ -514,7 +495,7 @@ int lmv_check_connect(struct obd_device *obd)
 {
 	struct lmv_obd       *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc  *tgt;
-	int		   i;
+	u32 i;
 	int		   rc;
 	int		   easize;
 
@@ -554,10 +535,9 @@ int lmv_check_connect(struct obd_device *obd)
 			goto out_disc;
 	}
 
-	lmv_set_timeouts(obd);
 	class_export_put(lmv->exp);
 	lmv->connected = 1;
-	easize = lmv_get_easize(lmv);
+	easize = lmv_mds_md_size(lmv->desc.ld_tgt_count, LMV_MAGIC);
 	lmv_init_ea_size(obd->obd_self_export, easize, 0, 0, 0);
 	mutex_unlock(&lmv->lmv_init_mutex);
 	return 0;
@@ -629,7 +609,7 @@ static int lmv_disconnect(struct obd_export *exp)
 	struct obd_device     *obd = class_exp2obd(exp);
 	struct lmv_obd	*lmv = &obd->u.lmv;
 	int		    rc;
-	int		    i;
+	u32 i;
 
 	if (!lmv->tgts)
 		goto out_local;
@@ -758,7 +738,7 @@ static int lmv_hsm_req_count(struct lmv_obd *lmv,
 			     const struct hsm_user_request *hur,
 			     const struct lmv_tgt_desc *tgt_mds)
 {
-	int			i, nr = 0;
+	u32 i, nr = 0;
 	struct lmv_tgt_desc    *curr_tgt;
 
 	/* count how many requests must be sent to the given target */
@@ -885,10 +865,8 @@ static int lmv_hsm_ct_register(struct lmv_obd *lmv, unsigned int cmd, int len,
 
 	rc = libcfs_kkuc_group_add(filp, lk->lk_uid, lk->lk_group,
 				   &kcd, sizeof(kcd));
-	if (rc) {
-		if (filp)
-			fput(filp);
-	}
+	if (rc)
+		fput(filp);
 
 	return rc;
 }
@@ -899,10 +877,10 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp,
 	struct obd_device    *obddev = class_exp2obd(exp);
 	struct lmv_obd       *lmv = &obddev->u.lmv;
 	struct lmv_tgt_desc *tgt = NULL;
-	int		   i = 0;
+	u32 i = 0;
 	int		   rc = 0;
 	int		   set = 0;
-	int		   count = lmv->desc.ld_tgt_count;
+	u32 count = lmv->desc.ld_tgt_count;
 
 	if (count == 0)
 		return -ENOTTY;
@@ -1011,6 +989,21 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp,
 		rc = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg);
 		break;
 	}
+	case LL_IOC_FID2MDTIDX: {
+		struct lu_fid *fid = karg;
+		int mdt_index;
+
+		rc = lmv_fld_lookup(lmv, fid, &mdt_index);
+		if (rc)
+			return rc;
+
+		/*
+		 * Note: this is from llite(see ll_dir_ioctl()), @uarg does not
+		 * point to user space memory for FID2MDTIDX.
+		 */
+		*(__u32 *)uarg = mdt_index;
+		break;
+	}
 	case OBD_IOC_FID2PATH: {
 		rc = lmv_fid2path(exp, len, karg, uarg);
 		break;
@@ -1169,32 +1162,37 @@ static int lmv_placement_policy(struct obd_device *obd,
 		return 0;
 	}
 
+	if (op_data->op_default_stripe_offset != -1) {
+		*mds = op_data->op_default_stripe_offset;
+		return 0;
+	}
+
 	/**
 	 * If stripe_offset is provided during setdirstripe
 	 * (setdirstripe -i xx), xx MDS will be chosen.
 	 */
-	if (op_data->op_cli_flags & CLI_SET_MEA) {
+	if (op_data->op_cli_flags & CLI_SET_MEA && op_data->op_data) {
 		struct lmv_user_md *lum;
 
-		lum = (struct lmv_user_md *)op_data->op_data;
-		if (lum->lum_type == LMV_STRIPE_TYPE &&
-		    lum->lum_stripe_offset != -1) {
-			if (lum->lum_stripe_offset >= lmv->desc.ld_tgt_count) {
-				CERROR("%s: Stripe_offset %d > MDT count %d: rc = %d\n",
-				       obd->obd_name,
-				       lum->lum_stripe_offset,
-				       lmv->desc.ld_tgt_count, -ERANGE);
-				return -ERANGE;
-			}
-			*mds = lum->lum_stripe_offset;
-			return 0;
+		lum = op_data->op_data;
+		if (le32_to_cpu(lum->lum_stripe_offset) != (__u32)-1) {
+			*mds = le32_to_cpu(lum->lum_stripe_offset);
+		} else {
+			/*
+			 * -1 means default, which will be in the same MDT with
+			 * the stripe
+			 */
+			*mds = op_data->op_mds;
+			lum->lum_stripe_offset = cpu_to_le32(op_data->op_mds);
 		}
+	} else {
+		/*
+		 * Allocate new fid on target according to operation type and
+		 * parent home mds.
+		 */
+		*mds = op_data->op_mds;
 	}
 
-	/* Allocate new fid on target according to operation type and parent
-	 * home mds.
-	 */
-	*mds = op_data->op_mds;
 	return 0;
 }
 
@@ -1203,7 +1201,7 @@ int __lmv_fid_alloc(struct lmv_obd *lmv, struct lu_fid *fid, u32 mds)
 	struct lmv_tgt_desc	*tgt;
 	int			 rc;
 
-	tgt = lmv_get_target(lmv, mds);
+	tgt = lmv_get_target(lmv, mds, NULL);
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
 
@@ -1221,7 +1219,7 @@ int __lmv_fid_alloc(struct lmv_obd *lmv, struct lu_fid *fid, u32 mds)
 	/*
 	 * Asking underlaying tgt layer to allocate new fid.
 	 */
-	rc = obd_fid_alloc(tgt->ltd_exp, fid, NULL);
+	rc = obd_fid_alloc(NULL, tgt->ltd_exp, fid, NULL);
 	if (rc > 0) {
 		LASSERT(fid_is_sane(fid));
 		rc = 0;
@@ -1232,8 +1230,8 @@ out:
 	return rc;
 }
 
-int lmv_fid_alloc(struct obd_export *exp, struct lu_fid *fid,
-		  struct md_op_data *op_data)
+int lmv_fid_alloc(const struct lu_env *env, struct obd_export *exp,
+		  struct lu_fid *fid, struct md_op_data *op_data)
 {
 	struct obd_device     *obd = class_exp2obd(exp);
 	struct lmv_obd	*lmv = &obd->u.lmv;
@@ -1278,10 +1276,10 @@ static int lmv_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
 		return -EINVAL;
 	}
 
-	lmv->tgts = kcalloc(32, sizeof(*lmv->tgts), GFP_NOFS);
+	lmv->tgts_size = 32U;
+	lmv->tgts = kcalloc(lmv->tgts_size, sizeof(*lmv->tgts), GFP_NOFS);
 	if (!lmv->tgts)
 		return -ENOMEM;
-	lmv->tgts_size = 32;
 
 	obd_str2uuid(&lmv->desc.ld_uuid, desc->ld_uuid.uuid);
 	lmv->desc.ld_tgt_count = 0;
@@ -1354,7 +1352,7 @@ static int lmv_process_config(struct obd_device *obd, u32 len, void *buf)
 
 		obd_str2uuid(&obd_uuid,  lustre_cfg_buf(lcfg, 1));
 
-		if (sscanf(lustre_cfg_buf(lcfg, 2), "%d", &index) != 1) {
+		if (sscanf(lustre_cfg_buf(lcfg, 2), "%u", &index) != 1) {
 			rc = -EINVAL;
 			goto out;
 		}
@@ -1380,7 +1378,7 @@ static int lmv_statfs(const struct lu_env *env, struct obd_export *exp,
 	struct lmv_obd	*lmv = &obd->u.lmv;
 	struct obd_statfs     *temp;
 	int		    rc = 0;
-	int		    i;
+	u32 i;
 
 	rc = lmv_check_connect(obd);
 	if (rc)
@@ -1522,7 +1520,7 @@ static int lmv_null_inode(struct obd_export *exp, const struct lu_fid *fid)
 {
 	struct obd_device   *obd = exp->exp_obd;
 	struct lmv_obd      *lmv = &obd->u.lmv;
-	int		  i;
+	u32 i;
 	int		  rc;
 
 	rc = lmv_check_connect(obd);
@@ -1545,36 +1543,6 @@ static int lmv_null_inode(struct obd_export *exp, const struct lu_fid *fid)
 	return 0;
 }
 
-static int lmv_find_cbdata(struct obd_export *exp, const struct lu_fid *fid,
-			   ldlm_iterator_t it, void *data)
-{
-	struct obd_device   *obd = exp->exp_obd;
-	struct lmv_obd      *lmv = &obd->u.lmv;
-	int		  i;
-	int		  rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
-	CDEBUG(D_INODE, "CBDATA for "DFID"\n", PFID(fid));
-
-	/*
-	 * With DNE every object can have two locks in different namespaces:
-	 * lookup lock in space of MDT storing direntry and update/open lock in
-	 * space of MDT storing inode.
-	 */
-	for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
-		if (!lmv->tgts[i] || !lmv->tgts[i]->ltd_exp)
-			continue;
-		rc = md_find_cbdata(lmv->tgts[i]->ltd_exp, fid, it, data);
-		if (rc)
-			return rc;
-	}
-
-	return rc;
-}
-
 static int lmv_close(struct obd_export *exp, struct md_op_data *op_data,
 		     struct md_open_data *mod, struct ptlrpc_request **request)
 {
@@ -1596,25 +1564,116 @@ static int lmv_close(struct obd_export *exp, struct md_op_data *op_data,
 	return rc;
 }
 
-struct lmv_tgt_desc
-*lmv_locate_mds(struct lmv_obd *lmv, struct md_op_data *op_data,
-		struct lu_fid *fid)
+/**
+ * Choosing the MDT by name or FID in @op_data.
+ * For non-striped directory, it will locate MDT by fid.
+ * For striped-directory, it will locate MDT by name. And also
+ * it will reset op_fid1 with the FID of the chosen stripe.
+ **/
+static struct lmv_tgt_desc *
+lmv_locate_target_for_name(struct lmv_obd *lmv, struct lmv_stripe_md *lsm,
+			   const char *name, int namelen, struct lu_fid *fid,
+			   u32 *mds)
+{
+	const struct lmv_oinfo *oinfo;
+	struct lmv_tgt_desc *tgt;
+
+	if (OBD_FAIL_CHECK(OBD_FAIL_LFSCK_BAD_NAME_HASH)) {
+		if (cfs_fail_val >= lsm->lsm_md_stripe_count)
+			return ERR_PTR(-EBADF);
+		oinfo = &lsm->lsm_md_oinfo[cfs_fail_val];
+	} else {
+		oinfo = lsm_name_to_stripe_info(lsm, name, namelen);
+		if (IS_ERR(oinfo))
+			return ERR_CAST(oinfo);
+	}
+
+	if (fid)
+		*fid = oinfo->lmo_fid;
+	if (mds)
+		*mds = oinfo->lmo_mds;
+
+	tgt = lmv_get_target(lmv, oinfo->lmo_mds, NULL);
+
+	CDEBUG(D_INFO, "locate on mds %u " DFID "\n", oinfo->lmo_mds,
+	       PFID(&oinfo->lmo_fid));
+	return tgt;
+}
+
+/**
+ * Locate mds by fid or name
+ *
+ * For striped directory (lsm != NULL), it will locate the stripe
+ * by name hash (see lsm_name_to_stripe_info()). Note: if the hash_type
+ * is unknown, it will return -EBADFD, and lmv_intent_lookup might need
+ * walk through all of stripes to locate the entry.
+ *
+ * For normal direcotry, it will locate MDS by FID directly.
+ * \param[in] lmv	LMV device
+ * \param[in] op_data	client MD stack parameters, name, namelen
+ *			mds_num etc.
+ * \param[in] fid	object FID used to locate MDS.
+ *
+ * retval		pointer to the lmv_tgt_desc if succeed.
+ *			ERR_PTR(errno) if failed.
+ */
+struct lmv_tgt_desc*
+lmv_locate_mds(struct lmv_obd *lmv, struct md_op_data *op_data,
+	       struct lu_fid *fid)
 {
+	struct lmv_stripe_md *lsm = op_data->op_mea1;
 	struct lmv_tgt_desc *tgt;
 
-	tgt = lmv_find_target(lmv, fid);
-	if (IS_ERR(tgt))
+	/*
+	 * During creating VOLATILE file, it should honor the mdt
+	 * index if the file under striped dir is being restored, see
+	 * ct_restore().
+	 */
+	if (op_data->op_bias & MDS_CREATE_VOLATILE &&
+	    (int)op_data->op_mds != -1 && lsm) {
+		int i;
+
+		tgt = lmv_get_target(lmv, op_data->op_mds, NULL);
+		if (IS_ERR(tgt))
+			return tgt;
+
+		/* refill the right parent fid */
+		for (i = 0; i < lsm->lsm_md_stripe_count; i++) {
+			struct lmv_oinfo *oinfo;
+
+			oinfo = &lsm->lsm_md_oinfo[i];
+			if (oinfo->lmo_mds == op_data->op_mds) {
+				*fid = oinfo->lmo_fid;
+				break;
+			}
+		}
+
+		/* Hmm, can not find the stripe by mdt_index(op_mds) */
+		if (i == lsm->lsm_md_stripe_count)
+			tgt = ERR_PTR(-EINVAL);
+
 		return tgt;
+	}
 
-	op_data->op_mds = tgt->ltd_idx;
+	if (!lsm || !op_data->op_namelen) {
+		tgt = lmv_find_target(lmv, fid);
+		if (IS_ERR(tgt))
+			return tgt;
 
-	return tgt;
+		op_data->op_mds = tgt->ltd_idx;
+
+		return tgt;
+	}
+
+	return lmv_locate_target_for_name(lmv, lsm, op_data->op_name,
+					  op_data->op_namelen, fid,
+					  &op_data->op_mds);
 }
 
 static int lmv_create(struct obd_export *exp, struct md_op_data *op_data,
-		      const void *data, int datalen, int mode, __u32 uid,
-		      __u32 gid, cfs_cap_t cap_effective, __u64 rdev,
-		      struct ptlrpc_request **request)
+		      const void *data, size_t datalen, umode_t mode,
+		      uid_t uid, gid_t gid, cfs_cap_t cap_effective,
+		      __u64 rdev, struct ptlrpc_request **request)
 {
 	struct obd_device       *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
@@ -1632,13 +1691,30 @@ static int lmv_create(struct obd_export *exp, struct md_op_data *op_data,
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
 
-	rc = lmv_fid_alloc(exp, &op_data->op_fid2, op_data);
+	CDEBUG(D_INODE, "CREATE name '%.*s' on "DFID" -> mds #%x\n",
+	       (int)op_data->op_namelen, op_data->op_name,
+	       PFID(&op_data->op_fid1), op_data->op_mds);
+
+	rc = lmv_fid_alloc(NULL, exp, &op_data->op_fid2, op_data);
 	if (rc)
 		return rc;
 
-	CDEBUG(D_INODE, "CREATE '%*s' on "DFID" -> mds #%x\n",
-	       op_data->op_namelen, op_data->op_name, PFID(&op_data->op_fid1),
-	       op_data->op_mds);
+	if (exp_connect_flags(exp) & OBD_CONNECT_DIR_STRIPE) {
+		/*
+		 * Send the create request to the MDT where the object
+		 * will be located
+		 */
+		tgt = lmv_find_target(lmv, &op_data->op_fid2);
+		if (IS_ERR(tgt))
+			return PTR_ERR(tgt);
+
+		op_data->op_mds = tgt->ltd_idx;
+	} else {
+		CDEBUG(D_CONFIG, "Server doesn't support striped dirs\n");
+	}
+
+	CDEBUG(D_INODE, "CREATE obj "DFID" -> mds #%x\n",
+	       PFID(&op_data->op_fid1), op_data->op_mds);
 
 	op_data->op_flags |= MF_MDC_CANCEL_FID1;
 	rc = md_create(tgt->ltd_exp, op_data, data, datalen, mode, uid, gid,
@@ -1674,70 +1750,10 @@ static int lmv_done_writing(struct obd_export *exp,
 }
 
 static int
-lmv_enqueue_remote(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
-		   struct lookup_intent *it, struct md_op_data *op_data,
-		   struct lustre_handle *lockh, void *lmm, int lmmsize,
-		   __u64 extra_lock_flags)
-{
-	struct ptlrpc_request      *req = it->it_request;
-	struct obd_device	  *obd = exp->exp_obd;
-	struct lmv_obd	     *lmv = &obd->u.lmv;
-	struct lustre_handle	plock;
-	struct lmv_tgt_desc	*tgt;
-	struct md_op_data	  *rdata;
-	struct lu_fid	       fid1;
-	struct mdt_body	    *body;
-	int			 rc = 0;
-	int			 pmode;
-
-	body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
-
-	if (!(body->valid & OBD_MD_MDS))
-		return 0;
-
-	CDEBUG(D_INODE, "REMOTE_ENQUEUE '%s' on "DFID" -> "DFID"\n",
-	       LL_IT2STR(it), PFID(&op_data->op_fid1), PFID(&body->fid1));
-
-	/*
-	 * We got LOOKUP lock, but we really need attrs.
-	 */
-	pmode = it->it_lock_mode;
-	LASSERT(pmode != 0);
-	memcpy(&plock, lockh, sizeof(plock));
-	it->it_lock_mode = 0;
-	it->it_request = NULL;
-	fid1 = body->fid1;
-
-	ptlrpc_req_finished(req);
-
-	tgt = lmv_find_target(lmv, &fid1);
-	if (IS_ERR(tgt)) {
-		rc = PTR_ERR(tgt);
-		goto out;
-	}
-
-	rdata = kzalloc(sizeof(*rdata), GFP_NOFS);
-	if (!rdata) {
-		rc = -ENOMEM;
-		goto out;
-	}
-
-	rdata->op_fid1 = fid1;
-	rdata->op_bias = MDS_CROSS_REF;
-
-	rc = md_enqueue(tgt->ltd_exp, einfo, it, rdata, lockh,
-			lmm, lmmsize, NULL, extra_lock_flags);
-	kfree(rdata);
-out:
-	ldlm_lock_decref(&plock, pmode);
-	return rc;
-}
-
-static int
 lmv_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
+	    const ldlm_policy_data_t *policy,
 	    struct lookup_intent *it, struct md_op_data *op_data,
-	    struct lustre_handle *lockh, void *lmm, int lmmsize,
-	    struct ptlrpc_request **req, __u64 extra_lock_flags)
+	    struct lustre_handle *lockh, __u64 extra_lock_flags)
 {
 	struct obd_device	*obd = exp->exp_obd;
 	struct lmv_obd	   *lmv = &obd->u.lmv;
@@ -1755,22 +1771,18 @@ lmv_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
 
-	CDEBUG(D_INODE, "ENQUEUE '%s' on "DFID" -> mds #%d\n",
+	CDEBUG(D_INODE, "ENQUEUE '%s' on " DFID " -> mds #%u\n",
 	       LL_IT2STR(it), PFID(&op_data->op_fid1), tgt->ltd_idx);
 
-	rc = md_enqueue(tgt->ltd_exp, einfo, it, op_data, lockh,
-			lmm, lmmsize, req, extra_lock_flags);
+	rc = md_enqueue(tgt->ltd_exp, einfo, policy, it, op_data, lockh,
+			extra_lock_flags);
 
-	if (rc == 0 && it && it->it_op == IT_OPEN) {
-		rc = lmv_enqueue_remote(exp, einfo, it, op_data, lockh,
-					lmm, lmmsize, extra_lock_flags);
-	}
 	return rc;
 }
 
 static int
 lmv_getattr_name(struct obd_export *exp, struct md_op_data *op_data,
-		 struct ptlrpc_request **request)
+		 struct ptlrpc_request **preq)
 {
 	struct ptlrpc_request   *req = NULL;
 	struct obd_device       *obd = exp->exp_obd;
@@ -1787,26 +1799,25 @@ lmv_getattr_name(struct obd_export *exp, struct md_op_data *op_data,
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
 
-	CDEBUG(D_INODE, "GETATTR_NAME for %*s on "DFID" -> mds #%d\n",
-	       op_data->op_namelen, op_data->op_name, PFID(&op_data->op_fid1),
-	       tgt->ltd_idx);
+	CDEBUG(D_INODE, "GETATTR_NAME for %*s on " DFID " -> mds #%u\n",
+	       (int)op_data->op_namelen, op_data->op_name,
+	       PFID(&op_data->op_fid1), tgt->ltd_idx);
 
-	rc = md_getattr_name(tgt->ltd_exp, op_data, request);
+	rc = md_getattr_name(tgt->ltd_exp, op_data, preq);
 	if (rc != 0)
 		return rc;
 
-	body = req_capsule_server_get(&(*request)->rq_pill,
-				      &RMF_MDT_BODY);
-
-	if (body->valid & OBD_MD_MDS) {
-		struct lu_fid rid = body->fid1;
+	body = req_capsule_server_get(&(*preq)->rq_pill, &RMF_MDT_BODY);
+	if (body->mbo_valid & OBD_MD_MDS) {
+		struct lu_fid rid = body->mbo_fid1;
 
 		CDEBUG(D_INODE, "Request attrs for "DFID"\n",
 		       PFID(&rid));
 
 		tgt = lmv_find_target(lmv, &rid);
 		if (IS_ERR(tgt)) {
-			ptlrpc_req_finished(*request);
+			ptlrpc_req_finished(*preq);
+			*preq = NULL;
 			return PTR_ERR(tgt);
 		}
 
@@ -1815,8 +1826,8 @@ lmv_getattr_name(struct obd_export *exp, struct md_op_data *op_data,
 		op_data->op_namelen = 0;
 		op_data->op_name = NULL;
 		rc = md_getattr_name(tgt->ltd_exp, op_data, &req);
-		ptlrpc_req_finished(*request);
-		*request = req;
+		ptlrpc_req_finished(*preq);
+		*preq = req;
 	}
 
 	return rc;
@@ -1829,23 +1840,24 @@ lmv_getattr_name(struct obd_export *exp, struct md_op_data *op_data,
 	 fl == MF_MDC_CANCEL_FID4 ? &op_data->op_fid4 : \
 	 NULL)
 
-static int lmv_early_cancel(struct obd_export *exp, struct md_op_data *op_data,
-			    int op_tgt, enum ldlm_mode mode, int bits,
-			    int flag)
+static int lmv_early_cancel(struct obd_export *exp, struct lmv_tgt_desc *tgt,
+			    struct md_op_data *op_data, int op_tgt,
+			    enum ldlm_mode mode, int bits, int flag)
 {
 	struct lu_fid	  *fid = md_op_data_fid(op_data, flag);
 	struct obd_device      *obd = exp->exp_obd;
 	struct lmv_obd	 *lmv = &obd->u.lmv;
-	struct lmv_tgt_desc    *tgt;
 	ldlm_policy_data_t      policy = { {0} };
 	int		     rc = 0;
 
 	if (!fid_is_sane(fid))
 		return 0;
 
-	tgt = lmv_find_target(lmv, fid);
-	if (IS_ERR(tgt))
-		return PTR_ERR(tgt);
+	if (!tgt) {
+		tgt = lmv_find_target(lmv, fid);
+		if (IS_ERR(tgt))
+			return PTR_ERR(tgt);
+	}
 
 	if (tgt->ltd_idx != op_tgt) {
 		CDEBUG(D_INODE, "EARLY_CANCEL on "DFID"\n", PFID(fid));
@@ -1882,12 +1894,24 @@ static int lmv_link(struct obd_export *exp, struct md_op_data *op_data,
 	LASSERT(op_data->op_namelen != 0);
 
 	CDEBUG(D_INODE, "LINK "DFID":%*s to "DFID"\n",
-	       PFID(&op_data->op_fid2), op_data->op_namelen,
+	       PFID(&op_data->op_fid2), (int)op_data->op_namelen,
 	       op_data->op_name, PFID(&op_data->op_fid1));
 
 	op_data->op_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	op_data->op_fsgid = from_kgid(&init_user_ns, current_fsgid());
 	op_data->op_cap = cfs_curproc_cap_pack();
+	if (op_data->op_mea2) {
+		struct lmv_stripe_md *lsm = op_data->op_mea2;
+		const struct lmv_oinfo *oinfo;
+
+		oinfo = lsm_name_to_stripe_info(lsm, op_data->op_name,
+						op_data->op_namelen);
+		if (IS_ERR(oinfo))
+			return PTR_ERR(oinfo);
+
+		op_data->op_fid2 = oinfo->lmo_fid;
+	}
+
 	tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid2);
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
@@ -1896,7 +1920,7 @@ static int lmv_link(struct obd_export *exp, struct md_op_data *op_data,
 	 * Cancel UPDATE lock on child (fid1).
 	 */
 	op_data->op_flags |= MF_MDC_CANCEL_FID2;
-	rc = lmv_early_cancel(exp, op_data, tgt->ltd_idx, LCK_EX,
+	rc = lmv_early_cancel(exp, NULL, op_data, tgt->ltd_idx, LCK_EX,
 			      MDS_INODELOCK_UPDATE, MF_MDC_CANCEL_FID1);
 	if (rc != 0)
 		return rc;
@@ -1907,20 +1931,22 @@ static int lmv_link(struct obd_export *exp, struct md_op_data *op_data,
 }
 
 static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data,
-		      const char *old, int oldlen, const char *new, int newlen,
+		      const char *old, size_t oldlen,
+		      const char *new, size_t newlen,
 		      struct ptlrpc_request **request)
 {
 	struct obd_device       *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc     *src_tgt;
-	struct lmv_tgt_desc     *tgt_tgt;
 	int			rc;
 
 	LASSERT(oldlen != 0);
 
-	CDEBUG(D_INODE, "RENAME %*s in "DFID" to %*s in "DFID"\n",
-	       oldlen, old, PFID(&op_data->op_fid1),
-	       newlen, new, PFID(&op_data->op_fid2));
+	CDEBUG(D_INODE, "RENAME %.*s in "DFID":%d to %.*s in "DFID":%d\n",
+	       (int)oldlen, old, PFID(&op_data->op_fid1),
+	       op_data->op_mea1 ? op_data->op_mea1->lsm_md_stripe_count : 0,
+	       (int)newlen, new, PFID(&op_data->op_fid2),
+	       op_data->op_mea2 ? op_data->op_mea2->lsm_md_stripe_count : 0);
 
 	rc = lmv_check_connect(obd);
 	if (rc)
@@ -1929,13 +1955,60 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data,
 	op_data->op_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	op_data->op_fsgid = from_kgid(&init_user_ns, current_fsgid());
 	op_data->op_cap = cfs_curproc_cap_pack();
-	src_tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid1);
+
+	if (op_data->op_cli_flags & CLI_MIGRATE) {
+		LASSERTF(fid_is_sane(&op_data->op_fid3), "invalid FID "DFID"\n",
+			 PFID(&op_data->op_fid3));
+
+		if (op_data->op_mea1) {
+			struct lmv_stripe_md *lsm = op_data->op_mea1;
+			struct lmv_tgt_desc *tmp;
+
+			/* Fix the parent fid for striped dir */
+			tmp = lmv_locate_target_for_name(lmv, lsm, old,
+							 oldlen,
+							 &op_data->op_fid1,
+							 NULL);
+			if (IS_ERR(tmp))
+				return PTR_ERR(tmp);
+		}
+
+		rc = lmv_fid_alloc(NULL, exp, &op_data->op_fid2, op_data);
+		if (rc)
+			return rc;
+		src_tgt = lmv_find_target(lmv, &op_data->op_fid3);
+	} else {
+		if (op_data->op_mea1) {
+			struct lmv_stripe_md *lsm = op_data->op_mea1;
+
+			src_tgt = lmv_locate_target_for_name(lmv, lsm, old,
+							     oldlen,
+							     &op_data->op_fid1,
+							     &op_data->op_mds);
+			if (IS_ERR(src_tgt))
+				return PTR_ERR(src_tgt);
+		} else {
+			src_tgt = lmv_find_target(lmv, &op_data->op_fid1);
+			if (IS_ERR(src_tgt))
+				return PTR_ERR(src_tgt);
+
+			op_data->op_mds = src_tgt->ltd_idx;
+		}
+
+		if (op_data->op_mea2) {
+			struct lmv_stripe_md *lsm = op_data->op_mea2;
+			const struct lmv_oinfo *oinfo;
+
+			oinfo = lsm_name_to_stripe_info(lsm, new, newlen);
+			if (IS_ERR(oinfo))
+				return PTR_ERR(oinfo);
+
+			op_data->op_fid2 = oinfo->lmo_fid;
+		}
+	}
 	if (IS_ERR(src_tgt))
 		return PTR_ERR(src_tgt);
 
-	tgt_tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid2);
-	if (IS_ERR(tgt_tgt))
-		return PTR_ERR(tgt_tgt);
 	/*
 	 * LOOKUP lock on src child (fid3) should also be cancelled for
 	 * src_tgt in mdc_rename.
@@ -1946,35 +2019,53 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data,
 	 * Cancel UPDATE locks on tgt parent (fid2), tgt_tgt is its
 	 * own target.
 	 */
-	rc = lmv_early_cancel(exp, op_data, src_tgt->ltd_idx,
+	rc = lmv_early_cancel(exp, NULL, op_data, src_tgt->ltd_idx,
 			      LCK_EX, MDS_INODELOCK_UPDATE,
 			      MF_MDC_CANCEL_FID2);
-
+	if (rc)
+		return rc;
 	/*
-	 * Cancel LOOKUP locks on tgt child (fid4) for parent tgt_tgt.
+	 * Cancel LOOKUP locks on source child (fid3) for parent tgt_tgt.
 	 */
-	if (rc == 0) {
-		rc = lmv_early_cancel(exp, op_data, src_tgt->ltd_idx,
+	if (fid_is_sane(&op_data->op_fid3)) {
+		struct lmv_tgt_desc *tgt;
+
+		tgt = lmv_find_target(lmv, &op_data->op_fid1);
+		if (IS_ERR(tgt))
+			return PTR_ERR(tgt);
+
+		/* Cancel LOOKUP lock on its parent */
+		rc = lmv_early_cancel(exp, tgt, op_data, src_tgt->ltd_idx,
 				      LCK_EX, MDS_INODELOCK_LOOKUP,
-				      MF_MDC_CANCEL_FID4);
+				      MF_MDC_CANCEL_FID3);
+		if (rc)
+			return rc;
+
+		rc = lmv_early_cancel(exp, NULL, op_data, src_tgt->ltd_idx,
+				      LCK_EX, MDS_INODELOCK_FULL,
+				      MF_MDC_CANCEL_FID3);
+		if (rc)
+			return rc;
 	}
 
 	/*
 	 * Cancel all the locks on tgt child (fid4).
 	 */
-	if (rc == 0)
-		rc = lmv_early_cancel(exp, op_data, src_tgt->ltd_idx,
+	if (fid_is_sane(&op_data->op_fid4))
+		rc = lmv_early_cancel(exp, NULL, op_data, src_tgt->ltd_idx,
 				      LCK_EX, MDS_INODELOCK_FULL,
 				      MF_MDC_CANCEL_FID4);
 
-	if (rc == 0)
-		rc = md_rename(src_tgt->ltd_exp, op_data, old, oldlen,
-			       new, newlen, request);
+	CDEBUG(D_INODE, DFID":m%d to "DFID"\n", PFID(&op_data->op_fid1),
+	       op_data->op_mds, PFID(&op_data->op_fid2));
+
+	rc = md_rename(src_tgt->ltd_exp, op_data, old, oldlen,
+		       new, newlen, request);
 	return rc;
 }
 
 static int lmv_setattr(struct obd_export *exp, struct md_op_data *op_data,
-		       void *ea, int ealen, void *ea2, int ea2len,
+		       void *ea, size_t ealen, void *ea2, size_t ea2len,
 		       struct ptlrpc_request **request,
 		       struct md_open_data **mod)
 {
@@ -2021,169 +2112,419 @@ static int lmv_sync(struct obd_export *exp, const struct lu_fid *fid,
 	return rc;
 }
 
-/*
- * Adjust a set of pages, each page containing an array of lu_dirpages,
- * so that each page can be used as a single logical lu_dirpage.
+/**
+ * Get current minimum entry from striped directory
  *
- * A lu_dirpage is laid out as follows, where s = ldp_hash_start,
- * e = ldp_hash_end, f = ldp_flags, p = padding, and each "ent" is a
- * struct lu_dirent.  It has size up to LU_PAGE_SIZE. The ldp_hash_end
- * value is used as a cookie to request the next lu_dirpage in a
- * directory listing that spans multiple pages (two in this example):
- *   ________
- *  |	|
- * .|--------v-------   -----.
- * |s|e|f|p|ent|ent| ... |ent|
- * '--|--------------   -----'   Each CFS_PAGE contains a single
- *    '------.		   lu_dirpage.
- * .---------v-------   -----.
- * |s|e|f|p|ent| 0 | ... | 0 |
- * '-----------------   -----'
+ * This function will search the dir entry, whose hash value is the
+ * closest(>=) to @hash_offset, from all of sub-stripes, and it is
+ * only being called for striped directory.
  *
- * However, on hosts where the native VM page size (PAGE_SIZE) is
- * larger than LU_PAGE_SIZE, a single host page may contain multiple
- * lu_dirpages. After reading the lu_dirpages from the MDS, the
- * ldp_hash_end of the first lu_dirpage refers to the one immediately
- * after it in the same CFS_PAGE (arrows simplified for brevity, but
- * in general e0==s1, e1==s2, etc.):
+ * \param[in] exp		export of LMV
+ * \param[in] op_data		parameters transferred beween client MD stack
+ *				stripe_information will be included in this
+ *				parameter
+ * \param[in] cb_op		ldlm callback being used in enqueue in
+ *				mdc_read_page
+ * \param[in] hash_offset	the hash value, which is used to locate
+ *				minum(closet) dir entry
+ * \param[in|out] stripe_offset	the caller use this to indicate the stripe
+ *				index of last entry, so to avoid hash conflict
+ *				between stripes. It will also be used to
+ *				return the stripe index of current dir entry.
+ * \param[in|out] entp		the minum entry and it also is being used
+ *				to input the last dir entry to resolve the
+ *				hash conflict
  *
- * .--------------------   -----.
- * |s0|e0|f0|p|ent|ent| ... |ent|
- * |---v----------------   -----|
- * |s1|e1|f1|p|ent|ent| ... |ent|
- * |---v----------------   -----|  Here, each CFS_PAGE contains
- *	     ...		 multiple lu_dirpages.
- * |---v----------------   -----|
- * |s'|e'|f'|p|ent|ent| ... |ent|
- * '---|----------------   -----'
- *     v
- * .----------------------------.
- * |	next CFS_PAGE       |
+ * \param[out] ppage		the page which holds the minum entry
  *
- * This structure is transformed into a single logical lu_dirpage as follows:
+ * \retval			= 0 get the entry successfully
+ *				negative errno (< 0) does not get the entry
+ */
+static int lmv_get_min_striped_entry(struct obd_export *exp,
+				     struct md_op_data *op_data,
+				     struct md_callback *cb_op,
+				     __u64 hash_offset, int *stripe_offset,
+				     struct lu_dirent **entp,
+				     struct page **ppage)
+{
+	struct lmv_stripe_md *lsm = op_data->op_mea1;
+	struct obd_device *obd = exp->exp_obd;
+	struct lmv_obd *lmv = &obd->u.lmv;
+	struct lu_dirent *min_ent = NULL;
+	struct page *min_page = NULL;
+	struct lmv_tgt_desc *tgt;
+	int stripe_count;
+	int min_idx = 0;
+	int rc = 0;
+	int i;
+
+	stripe_count = lsm->lsm_md_stripe_count;
+	for (i = 0; i < stripe_count; i++) {
+		__u64 stripe_hash = hash_offset;
+		struct lu_dirent *ent = NULL;
+		struct page *page = NULL;
+		struct lu_dirpage *dp;
+
+		tgt = lmv_get_target(lmv, lsm->lsm_md_oinfo[i].lmo_mds, NULL);
+		if (IS_ERR(tgt)) {
+			rc = PTR_ERR(tgt);
+			goto out;
+		}
+
+		/*
+		 * op_data will be shared by each stripe, so we need
+		 * reset these value for each stripe
+		 */
+		op_data->op_fid1 = lsm->lsm_md_oinfo[i].lmo_fid;
+		op_data->op_fid2 = lsm->lsm_md_oinfo[i].lmo_fid;
+		op_data->op_data = lsm->lsm_md_oinfo[i].lmo_root;
+next:
+		rc = md_read_page(tgt->ltd_exp, op_data, cb_op, stripe_hash,
+				  &page);
+		if (rc)
+			goto out;
+
+		dp = page_address(page);
+		for (ent = lu_dirent_start(dp); ent;
+		     ent = lu_dirent_next(ent)) {
+			/* Skip dummy entry */
+			if (!le16_to_cpu(ent->lde_namelen))
+				continue;
+
+			if (le64_to_cpu(ent->lde_hash) < hash_offset)
+				continue;
+
+			if (le64_to_cpu(ent->lde_hash) == hash_offset &&
+			    (*entp == ent || i < *stripe_offset))
+				continue;
+
+			/* skip . and .. for other stripes */
+			if (i && (!strncmp(ent->lde_name, ".",
+					   le16_to_cpu(ent->lde_namelen)) ||
+				  !strncmp(ent->lde_name, "..",
+					   le16_to_cpu(ent->lde_namelen))))
+				continue;
+			break;
+		}
+
+		if (!ent) {
+			stripe_hash = le64_to_cpu(dp->ldp_hash_end);
+
+			kunmap(page);
+			put_page(page);
+			page = NULL;
+
+			/*
+			 * reach the end of current stripe, go to next stripe
+			 */
+			if (stripe_hash == MDS_DIR_END_OFF)
+				continue;
+			else
+				goto next;
+		}
+
+		if (min_ent) {
+			if (le64_to_cpu(min_ent->lde_hash) >
+			    le64_to_cpu(ent->lde_hash)) {
+				min_ent = ent;
+				kunmap(min_page);
+				put_page(min_page);
+				min_idx = i;
+				min_page = page;
+			} else {
+				kunmap(page);
+				put_page(page);
+				page = NULL;
+			}
+		} else {
+			min_ent = ent;
+			min_page = page;
+			min_idx = i;
+		}
+	}
+
+out:
+	if (*ppage) {
+		kunmap(*ppage);
+		put_page(*ppage);
+	}
+	*stripe_offset = min_idx;
+	*entp = min_ent;
+	*ppage = min_page;
+	return rc;
+}
+
+/**
+ * Build dir entry page from a striped directory
  *
- * - Replace e0 with e' so the request for the next lu_dirpage gets the page
- *   labeled 'next CFS_PAGE'.
+ * This function gets one entry by @offset from a striped directory. It will
+ * read entries from all of stripes, and choose one closest to the required
+ * offset(&offset). A few notes
+ * 1. skip . and .. for non-zero stripes, because there can only have one .
+ * and .. in a directory.
+ * 2. op_data will be shared by all of stripes, instead of allocating new
+ * one, so need to restore before reusing.
+ * 3. release the entry page if that is not being chosen.
  *
- * - Copy the LDF_COLLIDE flag from f' to f0 to correctly reflect whether
- *   a hash collision with the next page exists.
+ * \param[in] exp	obd export refer to LMV
+ * \param[in] op_data	hold those MD parameters of read_entry
+ * \param[in] cb_op	ldlm callback being used in enqueue in mdc_read_entry
+ * \param[out] ldp	the entry being read
+ * \param[out] ppage	the page holding the entry. Note: because the entry
+ *			will be accessed in upper layer, so we need hold the
+ *			page until the usages of entry is finished, see
+ *			ll_dir_entry_next.
  *
- * - Adjust the lde_reclen of the ending entry of each lu_dirpage to span
- *   to the first entry of the next lu_dirpage.
+ * retval		=0 if get entry successfully
+ *			<0 cannot get entry
  */
-#if PAGE_SIZE > LU_PAGE_SIZE
-static void lmv_adjust_dirpages(struct page **pages, int ncfspgs, int nlupgs)
-{
-	int i;
+static int lmv_read_striped_page(struct obd_export *exp,
+				 struct md_op_data *op_data,
+				 struct md_callback *cb_op,
+				 __u64 offset, struct page **ppage)
+{
+	struct inode *master_inode = op_data->op_data;
+	struct lu_fid master_fid = op_data->op_fid1;
+	struct obd_device *obd = exp->exp_obd;
+	__u64 hash_offset = offset;
+	struct page *min_ent_page = NULL;
+	struct page *ent_page = NULL;
+	struct lu_dirent *min_ent = NULL;
+	struct lu_dirent *last_ent;
+	struct lu_dirent *ent;
+	struct lu_dirpage *dp;
+	size_t left_bytes;
+	int ent_idx = 0;
+	void *area;
+	int rc;
 
-	for (i = 0; i < ncfspgs; i++) {
-		struct lu_dirpage	*dp = kmap(pages[i]);
-		struct lu_dirpage	*first = dp;
-		struct lu_dirent	*end_dirent = NULL;
-		struct lu_dirent	*ent;
-		__u64			hash_end = dp->ldp_hash_end;
-		__u32			flags = dp->ldp_flags;
-
-		while (--nlupgs > 0) {
-			ent = lu_dirent_start(dp);
-			for (end_dirent = ent; ent;
-			     end_dirent = ent, ent = lu_dirent_next(ent))
-				;
-
-			/* Advance dp to next lu_dirpage. */
-			dp = (struct lu_dirpage *)((char *)dp + LU_PAGE_SIZE);
-
-			/* Check if we've reached the end of the CFS_PAGE. */
-			if (!((unsigned long)dp & ~PAGE_MASK))
-				break;
+	rc = lmv_check_connect(obd);
+	if (rc)
+		return rc;
 
-			/* Save the hash and flags of this lu_dirpage. */
-			hash_end = dp->ldp_hash_end;
-			flags = dp->ldp_flags;
+	/*
+	 * Allocate a page and read entries from all of stripes and fill
+	 * the page by hash order
+	 */
+	ent_page = alloc_page(GFP_KERNEL);
+	if (!ent_page)
+		return -ENOMEM;
 
-			/* Check if lu_dirpage contains no entries. */
-			if (!end_dirent)
-				break;
+	/* Initialize the entry page */
+	dp = kmap(ent_page);
+	memset(dp, 0, sizeof(*dp));
+	dp->ldp_hash_start = cpu_to_le64(offset);
+	dp->ldp_flags |= LDF_COLLIDE;
+
+	area = dp + 1;
+	left_bytes = PAGE_SIZE - sizeof(*dp);
+	ent = area;
+	last_ent = ent;
+	do {
+		__u16 ent_size;
+
+		/* Find the minum entry from all sub-stripes */
+		rc = lmv_get_min_striped_entry(exp, op_data, cb_op, hash_offset,
+					       &ent_idx, &min_ent,
+					       &min_ent_page);
+		if (rc)
+			goto out;
 
-			/* Enlarge the end entry lde_reclen from 0 to
-			 * first entry of next lu_dirpage.
-			 */
-			LASSERT(le16_to_cpu(end_dirent->lde_reclen) == 0);
-			end_dirent->lde_reclen =
-				cpu_to_le16((char *)(dp->ldp_entries) -
-					    (char *)end_dirent);
+		/*
+		 * If it can not get minum entry, it means it already reaches
+		 * the end of this directory
+		 */
+		if (!min_ent) {
+			last_ent->lde_reclen = 0;
+			hash_offset = MDS_DIR_END_OFF;
+			goto out;
+		}
+
+		ent_size = le16_to_cpu(min_ent->lde_reclen);
+
+		/*
+		 * the last entry lde_reclen is 0, but it might not
+		 * the end of this entry of this temporay entry
+		 */
+		if (!ent_size)
+			ent_size = lu_dirent_calc_size(
+					le16_to_cpu(min_ent->lde_namelen),
+					le32_to_cpu(min_ent->lde_attrs));
+		if (ent_size > left_bytes) {
+			last_ent->lde_reclen = cpu_to_le16(0);
+			hash_offset = le64_to_cpu(min_ent->lde_hash);
+			goto out;
 		}
 
-		first->ldp_hash_end = hash_end;
-		first->ldp_flags &= ~cpu_to_le32(LDF_COLLIDE);
-		first->ldp_flags |= flags & cpu_to_le32(LDF_COLLIDE);
+		memcpy(ent, min_ent, ent_size);
+
+		/*
+		 * Replace . with master FID and Replace .. with the parent FID
+		 * of master object
+		 */
+		if (!strncmp(ent->lde_name, ".",
+			     le16_to_cpu(ent->lde_namelen)) &&
+		    le16_to_cpu(ent->lde_namelen) == 1)
+			fid_cpu_to_le(&ent->lde_fid, &master_fid);
+		else if (!strncmp(ent->lde_name, "..",
+				  le16_to_cpu(ent->lde_namelen)) &&
+			 le16_to_cpu(ent->lde_namelen) == 2)
+			fid_cpu_to_le(&ent->lde_fid, &op_data->op_fid3);
+
+		left_bytes -= ent_size;
+		ent->lde_reclen = cpu_to_le16(ent_size);
+		last_ent = ent;
+		ent = (void *)ent + ent_size;
+		hash_offset = le64_to_cpu(min_ent->lde_hash);
+		if (hash_offset == MDS_DIR_END_OFF) {
+			last_ent->lde_reclen = 0;
+			break;
+		}
+	} while (1);
+out:
+	if (min_ent_page) {
+		kunmap(min_ent_page);
+		put_page(min_ent_page);
+	}
 
-		kunmap(pages[i]);
+	if (unlikely(rc)) {
+		__free_page(ent_page);
+		ent_page = NULL;
+	} else {
+		if (ent == area)
+			dp->ldp_flags |= LDF_EMPTY;
+		dp->ldp_flags = cpu_to_le32(dp->ldp_flags);
+		dp->ldp_hash_end = cpu_to_le64(hash_offset);
 	}
-	LASSERTF(nlupgs == 0, "left = %d", nlupgs);
+
+	/*
+	 * We do not want to allocate md_op_data during each
+	 * dir entry reading, so op_data will be shared by every stripe,
+	 * then we need to restore it back to original value before
+	 * return to the upper layer
+	 */
+	op_data->op_fid1 = master_fid;
+	op_data->op_fid2 = master_fid;
+	op_data->op_data = master_inode;
+
+	*ppage = ent_page;
+
+	return rc;
 }
-#else
-#define lmv_adjust_dirpages(pages, ncfspgs, nlupgs) do {} while (0)
-#endif	/* PAGE_SIZE > LU_PAGE_SIZE */
 
-static int lmv_readpage(struct obd_export *exp, struct md_op_data *op_data,
-			struct page **pages, struct ptlrpc_request **request)
+static int lmv_read_page(struct obd_export *exp, struct md_op_data *op_data,
+			 struct md_callback *cb_op, __u64 offset,
+			 struct page **ppage)
 {
-	struct obd_device	*obd = exp->exp_obd;
-	struct lmv_obd		*lmv = &obd->u.lmv;
-	__u64			offset = op_data->op_offset;
-	int			rc;
-	int			ncfspgs; /* pages read in PAGE_SIZE */
-	int			nlupgs; /* pages read in LU_PAGE_SIZE */
-	struct lmv_tgt_desc	*tgt;
+	struct lmv_stripe_md *lsm = op_data->op_mea1;
+	struct obd_device *obd = exp->exp_obd;
+	struct lmv_obd *lmv = &obd->u.lmv;
+	struct lmv_tgt_desc *tgt;
+	int rc;
 
 	rc = lmv_check_connect(obd);
 	if (rc)
 		return rc;
 
-	CDEBUG(D_INODE, "READPAGE at %#llx from "DFID"\n",
-	       offset, PFID(&op_data->op_fid1));
+	if (unlikely(lsm)) {
+		rc = lmv_read_striped_page(exp, op_data, cb_op, offset, ppage);
+		return rc;
+	}
 
 	tgt = lmv_find_target(lmv, &op_data->op_fid1);
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
 
-	rc = md_readpage(tgt->ltd_exp, op_data, pages, request);
-	if (rc != 0)
-		return rc;
-
-	ncfspgs = ((*request)->rq_bulk->bd_nob_transferred + PAGE_SIZE - 1)
-		 >> PAGE_SHIFT;
-	nlupgs = (*request)->rq_bulk->bd_nob_transferred >> LU_PAGE_SHIFT;
-	LASSERT(!((*request)->rq_bulk->bd_nob_transferred & ~LU_PAGE_MASK));
-	LASSERT(ncfspgs > 0 && ncfspgs <= op_data->op_npages);
-
-	CDEBUG(D_INODE, "read %d(%d)/%d pages\n", ncfspgs, nlupgs,
-	       op_data->op_npages);
-
-	lmv_adjust_dirpages(pages, ncfspgs, nlupgs);
+	rc = md_read_page(tgt->ltd_exp, op_data, cb_op, offset, ppage);
 
 	return rc;
 }
 
+/**
+ * Unlink a file/directory
+ *
+ * Unlink a file or directory under the parent dir. The unlink request
+ * usually will be sent to the MDT where the child is located, but if
+ * the client does not have the child FID then request will be sent to the
+ * MDT where the parent is located.
+ *
+ * If the parent is a striped directory then it also needs to locate which
+ * stripe the name of the child is located, and replace the parent FID
+ * (@op->op_fid1) with the stripe FID. Note: if the stripe is unknown,
+ * it will walk through all of sub-stripes until the child is being
+ * unlinked finally.
+ *
+ * \param[in] exp	export refer to LMV
+ * \param[in] op_data	different parameters transferred beween client
+ *			MD stacks, name, namelen, FIDs etc.
+ *			op_fid1 is the parent FID, op_fid2 is the child
+ *			FID.
+ * \param[out] request point to the request of unlink.
+ *
+ * retval		0 if succeed
+ *			negative errno if failed.
+ */
 static int lmv_unlink(struct obd_export *exp, struct md_op_data *op_data,
 		      struct ptlrpc_request **request)
 {
-	struct obd_device       *obd = exp->exp_obd;
+	struct lmv_stripe_md *lsm = op_data->op_mea1;
+	struct obd_device    *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
+	struct lmv_tgt_desc *parent_tgt = NULL;
 	struct lmv_tgt_desc     *tgt = NULL;
 	struct mdt_body		*body;
+	int stripe_index = 0;
 	int		     rc;
 
 	rc = lmv_check_connect(obd);
 	if (rc)
 		return rc;
-retry:
+retry_unlink:
+	/* For striped dir, we need to locate the parent as well */
+	if (lsm) {
+		struct lmv_tgt_desc *tmp;
+
+		LASSERT(op_data->op_name && op_data->op_namelen);
+
+		tmp = lmv_locate_target_for_name(lmv, lsm,
+						 op_data->op_name,
+						 op_data->op_namelen,
+						 &op_data->op_fid1,
+						 &op_data->op_mds);
+
+		/*
+		 * return -EBADFD means unknown hash type, might
+		 * need try all sub-stripe here
+		 */
+		if (IS_ERR(tmp) && PTR_ERR(tmp) != -EBADFD)
+			return PTR_ERR(tmp);
+
+		/*
+		 * Note: both migrating dir and unknown hash dir need to
+		 * try all of sub-stripes, so we need start search the
+		 * name from stripe 0, but migrating dir is already handled
+		 * inside lmv_locate_target_for_name(), so we only check
+		 * unknown hash type directory here
+		 */
+		if (!lmv_is_known_hash_type(lsm->lsm_md_hash_type)) {
+			struct lmv_oinfo *oinfo;
+
+			oinfo = &lsm->lsm_md_oinfo[stripe_index];
+
+			op_data->op_fid1 = oinfo->lmo_fid;
+			op_data->op_mds = oinfo->lmo_mds;
+		}
+	}
+
+try_next_stripe:
 	/* Send unlink requests to the MDT where the child is located */
 	if (likely(!fid_is_zero(&op_data->op_fid2)))
-		tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid2);
+		tgt = lmv_find_target(lmv, &op_data->op_fid2);
+	else if (lsm)
+		tgt = lmv_get_target(lmv, op_data->op_mds, NULL);
 	else
 		tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid1);
+
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
 
@@ -2203,29 +2544,57 @@ retry:
 	/*
 	 * Cancel FULL locks on child (fid3).
 	 */
-	rc = lmv_early_cancel(exp, op_data, tgt->ltd_idx, LCK_EX,
-			      MDS_INODELOCK_FULL, MF_MDC_CANCEL_FID3);
+	parent_tgt = lmv_find_target(lmv, &op_data->op_fid1);
+	if (IS_ERR(parent_tgt))
+		return PTR_ERR(parent_tgt);
 
+	if (parent_tgt != tgt) {
+		rc = lmv_early_cancel(exp, parent_tgt, op_data, tgt->ltd_idx,
+				      LCK_EX, MDS_INODELOCK_LOOKUP,
+				      MF_MDC_CANCEL_FID3);
+	}
+
+	rc = lmv_early_cancel(exp, NULL, op_data, tgt->ltd_idx, LCK_EX,
+			      MDS_INODELOCK_FULL, MF_MDC_CANCEL_FID3);
 	if (rc != 0)
 		return rc;
 
-	CDEBUG(D_INODE, "unlink with fid="DFID"/"DFID" -> mds #%d\n",
+	CDEBUG(D_INODE, "unlink with fid=" DFID "/" DFID " -> mds #%u\n",
 	       PFID(&op_data->op_fid1), PFID(&op_data->op_fid2), tgt->ltd_idx);
 
 	rc = md_unlink(tgt->ltd_exp, op_data, request);
-	if (rc != 0 && rc != -EREMOTE)
+	if (rc != 0 && rc != -EREMOTE  && rc != -ENOENT)
 		return rc;
 
+	/* Try next stripe if it is needed. */
+	if (rc == -ENOENT && lsm && lmv_need_try_all_stripes(lsm)) {
+		struct lmv_oinfo *oinfo;
+
+		stripe_index++;
+		if (stripe_index >= lsm->lsm_md_stripe_count)
+			return rc;
+
+		oinfo = &lsm->lsm_md_oinfo[stripe_index];
+
+		op_data->op_fid1 = oinfo->lmo_fid;
+		op_data->op_mds = oinfo->lmo_mds;
+
+		ptlrpc_req_finished(*request);
+		*request = NULL;
+
+		goto try_next_stripe;
+	}
+
 	body = req_capsule_server_get(&(*request)->rq_pill, &RMF_MDT_BODY);
 	if (!body)
 		return -EPROTO;
 
 	/* Not cross-ref case, just get out of here. */
-	if (likely(!(body->valid & OBD_MD_MDS)))
-		return 0;
+	if (likely(!(body->mbo_valid & OBD_MD_MDS)))
+		return rc;
 
 	CDEBUG(D_INODE, "%s: try unlink to another MDT for "DFID"\n",
-	       exp->exp_obd->obd_name, PFID(&body->fid1));
+	       exp->exp_obd->obd_name, PFID(&body->mbo_fid1));
 
 	/* This is a remote object, try remote MDT, Note: it may
 	 * try more than 1 time here, Considering following case
@@ -2247,11 +2616,11 @@ retry:
 	 * In theory, it might try unlimited time here, but it should
 	 * be very rare case.
 	 */
-	op_data->op_fid2 = body->fid1;
+	op_data->op_fid2 = body->mbo_fid1;
 	ptlrpc_req_finished(*request);
 	*request = NULL;
 
-	goto retry;
+	goto retry_unlink;
 }
 
 static int lmv_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
@@ -2274,6 +2643,22 @@ static int lmv_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
 	return 0;
 }
 
+/**
+ * Get by key a value associated with a LMV device.
+ *
+ * Dispatch request to lower-layer devices as needed.
+ *
+ * \param[in]  env	execution environment for this thread
+ * \param[in]  exp	export for the LMV device
+ * \param[in]  keylen	length of key identifier
+ * \param[in]  key	identifier of key to get value for
+ * \param[in]  vallen	size of \a val
+ * \param[out] val	pointer to storage location for value
+ * \param[in]  lsm	optional striping metadata of object
+ *
+ * \retval 0		on success
+ * \retval negative	negated errno on failure
+ */
 static int lmv_get_info(const struct lu_env *env, struct obd_export *exp,
 			__u32 keylen, void *key, __u32 *vallen, void *val,
 			struct lov_stripe_md *lsm)
@@ -2337,6 +2722,22 @@ static int lmv_get_info(const struct lu_env *env, struct obd_export *exp,
 	return -EINVAL;
 }
 
+/**
+ * Asynchronously set by key a value associated with a LMV device.
+ *
+ * Dispatch request to lower-layer devices as needed.
+ *
+ * \param[in] env	execution environment for this thread
+ * \param[in] exp	export for the LMV device
+ * \param[in] keylen	length of key identifier
+ * \param[in] key	identifier of key to store value for
+ * \param[in] vallen	size of value to store
+ * \param[in] val	pointer to data to be stored
+ * \param[in] set	optional list of related ptlrpc requests
+ *
+ * \retval 0		on success
+ * \retval negative	negated errno on failure
+ */
 static int lmv_set_info_async(const struct lu_env *env, struct obd_export *exp,
 			      u32 keylen, void *key, u32 vallen,
 			      void *val, struct ptlrpc_request_set *set)
@@ -2354,7 +2755,8 @@ static int lmv_set_info_async(const struct lu_env *env, struct obd_export *exp,
 	}
 	lmv = &obd->u.lmv;
 
-	if (KEY_IS(KEY_READ_ONLY) || KEY_IS(KEY_FLUSH_CTX)) {
+	if (KEY_IS(KEY_READ_ONLY) || KEY_IS(KEY_FLUSH_CTX) ||
+	    KEY_IS(KEY_DEFAULT_EASIZE)) {
 		int i, err = 0;
 
 		for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
@@ -2375,105 +2777,247 @@ static int lmv_set_info_async(const struct lu_env *env, struct obd_export *exp,
 	return -EINVAL;
 }
 
-static int lmv_packmd(struct obd_export *exp, struct lov_mds_md **lmmp,
-		      struct lov_stripe_md *lsm)
+static int lmv_pack_md_v1(const struct lmv_stripe_md *lsm,
+			  struct lmv_mds_md_v1 *lmm1)
 {
-	struct obd_device	 *obd = class_exp2obd(exp);
-	struct lmv_obd	    *lmv = &obd->u.lmv;
-	struct lmv_stripe_md      *meap;
-	struct lmv_stripe_md      *lsmp;
-	int			mea_size;
-	int			i;
+	int cplen;
+	int i;
+
+	lmm1->lmv_magic = cpu_to_le32(lsm->lsm_md_magic);
+	lmm1->lmv_stripe_count = cpu_to_le32(lsm->lsm_md_stripe_count);
+	lmm1->lmv_master_mdt_index = cpu_to_le32(lsm->lsm_md_master_mdt_index);
+	lmm1->lmv_hash_type = cpu_to_le32(lsm->lsm_md_hash_type);
+	cplen = strlcpy(lmm1->lmv_pool_name, lsm->lsm_md_pool_name,
+			sizeof(lmm1->lmv_pool_name));
+	if (cplen >= sizeof(lmm1->lmv_pool_name))
+		return -E2BIG;
+
+	for (i = 0; i < lsm->lsm_md_stripe_count; i++)
+		fid_cpu_to_le(&lmm1->lmv_stripe_fids[i],
+			      &lsm->lsm_md_oinfo[i].lmo_fid);
+	return 0;
+}
 
-	mea_size = lmv_get_easize(lmv);
-	if (!lmmp)
-		return mea_size;
+static int
+lmv_pack_md(union lmv_mds_md **lmmp, const struct lmv_stripe_md *lsm,
+	    int stripe_count)
+{
+	int lmm_size = 0, rc = 0;
+	bool allocated = false;
 
+	LASSERT(lmmp);
+
+	/* Free lmm */
 	if (*lmmp && !lsm) {
+		int stripe_cnt;
+
+		stripe_cnt = lmv_mds_md_stripe_count_get(*lmmp);
+		lmm_size = lmv_mds_md_size(stripe_cnt,
+					   le32_to_cpu((*lmmp)->lmv_magic));
+		if (!lmm_size)
+			return -EINVAL;
 		kvfree(*lmmp);
 		*lmmp = NULL;
 		return 0;
 	}
 
+	/* Alloc lmm */
+	if (!*lmmp && !lsm) {
+		lmm_size = lmv_mds_md_size(stripe_count, LMV_MAGIC);
+		LASSERT(lmm_size > 0);
+		*lmmp = libcfs_kvzalloc(lmm_size, GFP_NOFS);
+		if (!*lmmp)
+			return -ENOMEM;
+		lmv_mds_md_stripe_count_set(*lmmp, stripe_count);
+		(*lmmp)->lmv_magic = cpu_to_le32(LMV_MAGIC);
+		return lmm_size;
+	}
+
+	/* pack lmm */
+	LASSERT(lsm);
+	lmm_size = lmv_mds_md_size(lsm->lsm_md_stripe_count,
+				   lsm->lsm_md_magic);
 	if (!*lmmp) {
-		*lmmp = libcfs_kvzalloc(mea_size, GFP_NOFS);
+		*lmmp = libcfs_kvzalloc(lmm_size, GFP_NOFS);
 		if (!*lmmp)
 			return -ENOMEM;
+		allocated = true;
 	}
 
-	if (!lsm)
-		return mea_size;
+	switch (lsm->lsm_md_magic) {
+	case LMV_MAGIC_V1:
+		rc = lmv_pack_md_v1(lsm, &(*lmmp)->lmv_md_v1);
+		break;
+	default:
+		rc = -EINVAL;
+		break;
+	}
 
-	lsmp = (struct lmv_stripe_md *)lsm;
-	meap = (struct lmv_stripe_md *)*lmmp;
+	if (rc && allocated) {
+		kvfree(*lmmp);
+		*lmmp = NULL;
+	}
 
-	if (lsmp->mea_magic != MEA_MAGIC_LAST_CHAR &&
-	    lsmp->mea_magic != MEA_MAGIC_ALL_CHARS)
-		return -EINVAL;
+	return lmm_size;
+}
 
-	meap->mea_magic = cpu_to_le32(lsmp->mea_magic);
-	meap->mea_count = cpu_to_le32(lsmp->mea_count);
-	meap->mea_master = cpu_to_le32(lsmp->mea_master);
+static int lmv_unpack_md_v1(struct obd_export *exp, struct lmv_stripe_md *lsm,
+			    const struct lmv_mds_md_v1 *lmm1)
+{
+	struct lmv_obd *lmv = &exp->exp_obd->u.lmv;
+	int stripe_count;
+	int rc = 0;
+	int cplen;
+	int i;
 
-	for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
-		meap->mea_ids[i] = lsmp->mea_ids[i];
-		fid_cpu_to_le(&meap->mea_ids[i], &lsmp->mea_ids[i]);
+	lsm->lsm_md_magic = le32_to_cpu(lmm1->lmv_magic);
+	lsm->lsm_md_stripe_count = le32_to_cpu(lmm1->lmv_stripe_count);
+	lsm->lsm_md_master_mdt_index = le32_to_cpu(lmm1->lmv_master_mdt_index);
+	if (OBD_FAIL_CHECK(OBD_FAIL_UNKNOWN_LMV_STRIPE))
+		lsm->lsm_md_hash_type = LMV_HASH_TYPE_UNKNOWN;
+	else
+		lsm->lsm_md_hash_type = le32_to_cpu(lmm1->lmv_hash_type);
+	lsm->lsm_md_layout_version = le32_to_cpu(lmm1->lmv_layout_version);
+	cplen = strlcpy(lsm->lsm_md_pool_name, lmm1->lmv_pool_name,
+			sizeof(lsm->lsm_md_pool_name));
+
+	if (cplen >= sizeof(lsm->lsm_md_pool_name))
+		return -E2BIG;
+
+	CDEBUG(D_INFO, "unpack lsm count %d, master %d hash_type %d layout_version %d\n",
+	       lsm->lsm_md_stripe_count, lsm->lsm_md_master_mdt_index,
+	       lsm->lsm_md_hash_type, lsm->lsm_md_layout_version);
+
+	stripe_count = le32_to_cpu(lmm1->lmv_stripe_count);
+	for (i = 0; i < stripe_count; i++) {
+		fid_le_to_cpu(&lsm->lsm_md_oinfo[i].lmo_fid,
+			      &lmm1->lmv_stripe_fids[i]);
+		rc = lmv_fld_lookup(lmv, &lsm->lsm_md_oinfo[i].lmo_fid,
+				    &lsm->lsm_md_oinfo[i].lmo_mds);
+		if (rc)
+			return rc;
+		CDEBUG(D_INFO, "unpack fid #%d "DFID"\n", i,
+		       PFID(&lsm->lsm_md_oinfo[i].lmo_fid));
 	}
 
-	return mea_size;
+	return rc;
 }
 
-static int lmv_unpackmd(struct obd_export *exp, struct lov_stripe_md **lsmp,
-			struct lov_mds_md *lmm, int lmm_size)
+int lmv_unpack_md(struct obd_export *exp, struct lmv_stripe_md **lsmp,
+		  const union lmv_mds_md *lmm, int stripe_count)
 {
-	struct obd_device	  *obd = class_exp2obd(exp);
-	struct lmv_stripe_md      **tmea = (struct lmv_stripe_md **)lsmp;
-	struct lmv_stripe_md       *mea = (struct lmv_stripe_md *)lmm;
-	struct lmv_obd	     *lmv = &obd->u.lmv;
-	int			 mea_size;
-	int			 i;
-	__u32		       magic;
+	struct lmv_stripe_md *lsm;
+	bool allocated = false;
+	int lsm_size, rc;
 
-	mea_size = lmv_get_easize(lmv);
-	if (!lsmp)
-		return mea_size;
+	LASSERT(lsmp);
 
-	if (*lsmp && !lmm) {
-		kvfree(*tmea);
+	lsm = *lsmp;
+	/* Free memmd */
+	if (lsm && !lmm) {
+		int i;
+
+		for (i = 1; i < lsm->lsm_md_stripe_count; i++) {
+			/*
+			 * For migrating inode, the master stripe and master
+			 * object will be the same, so do not need iput, see
+			 * ll_update_lsm_md
+			 */
+			if (!(lsm->lsm_md_hash_type & LMV_HASH_FLAG_MIGRATION &&
+			      !i) && lsm->lsm_md_oinfo[i].lmo_root)
+				iput(lsm->lsm_md_oinfo[i].lmo_root);
+		}
+
+		kvfree(lsm);
 		*lsmp = NULL;
 		return 0;
 	}
 
-	LASSERT(mea_size == lmm_size);
+	/* Alloc memmd */
+	if (!lsm && !lmm) {
+		lsm_size = lmv_stripe_md_size(stripe_count);
+		lsm = libcfs_kvzalloc(lsm_size, GFP_NOFS);
+		if (!lsm)
+			return -ENOMEM;
+		lsm->lsm_md_stripe_count = stripe_count;
+		*lsmp = lsm;
+		return 0;
+	}
 
-	*tmea = libcfs_kvzalloc(mea_size, GFP_NOFS);
-	if (!*tmea)
-		return -ENOMEM;
+	if (le32_to_cpu(lmm->lmv_magic) == LMV_MAGIC_STRIPE)
+		return -EPERM;
 
-	if (!lmm)
-		return mea_size;
+	/* Unpack memmd */
+	if (le32_to_cpu(lmm->lmv_magic) != LMV_MAGIC_V1 &&
+	    le32_to_cpu(lmm->lmv_magic) != LMV_USER_MAGIC) {
+		CERROR("%s: invalid lmv magic %x: rc = %d\n",
+		       exp->exp_obd->obd_name, le32_to_cpu(lmm->lmv_magic),
+		       -EIO);
+		return -EIO;
+	}
 
-	if (mea->mea_magic == MEA_MAGIC_LAST_CHAR ||
-	    mea->mea_magic == MEA_MAGIC_ALL_CHARS ||
-	    mea->mea_magic == MEA_MAGIC_HASH_SEGMENT) {
-		magic = le32_to_cpu(mea->mea_magic);
-	} else {
-		/*
-		 * Old mea is not handled here.
+	if (le32_to_cpu(lmm->lmv_magic) == LMV_MAGIC_V1)
+		lsm_size = lmv_stripe_md_size(lmv_mds_md_stripe_count_get(lmm));
+	else
+		/**
+		 * Unpack default dirstripe(lmv_user_md) to lmv_stripe_md,
+		 * stripecount should be 0 then.
 		 */
-		CERROR("Old not supportable EA is found\n");
-		LBUG();
+		lsm_size = lmv_stripe_md_size(0);
+
+	if (!lsm) {
+		lsm = libcfs_kvzalloc(lsm_size, GFP_NOFS);
+		if (!lsm)
+			return -ENOMEM;
+		allocated = true;
+		*lsmp = lsm;
+	}
+
+	switch (le32_to_cpu(lmm->lmv_magic)) {
+	case LMV_MAGIC_V1:
+		rc = lmv_unpack_md_v1(exp, lsm, &lmm->lmv_md_v1);
+		break;
+	default:
+		CERROR("%s: unrecognized magic %x\n", exp->exp_obd->obd_name,
+		       le32_to_cpu(lmm->lmv_magic));
+		rc = -EINVAL;
+		break;
 	}
 
-	(*tmea)->mea_magic = magic;
-	(*tmea)->mea_count = le32_to_cpu(mea->mea_count);
-	(*tmea)->mea_master = le32_to_cpu(mea->mea_master);
+	if (rc && allocated) {
+		kvfree(lsm);
+		*lsmp = NULL;
+		lsm_size = rc;
+	}
+	return lsm_size;
+}
+EXPORT_SYMBOL(lmv_unpack_md);
+
+static int lmv_unpackmd(struct obd_export *exp, struct lov_stripe_md **lsmp,
+			struct lov_mds_md *lmm, int disk_len)
+{
+	return lmv_unpack_md(exp, (struct lmv_stripe_md **)lsmp,
+			     (union lmv_mds_md *)lmm, disk_len);
+}
+
+static int lmv_packmd(struct obd_export *exp, struct lov_mds_md **lmmp,
+		      struct lov_stripe_md *lsm)
+{
+	const struct lmv_stripe_md *lmv = (struct lmv_stripe_md *)lsm;
+	struct obd_device *obd = exp->exp_obd;
+	struct lmv_obd *lmv_obd = &obd->u.lmv;
+	int stripe_count;
 
-	for (i = 0; i < (*tmea)->mea_count; i++) {
-		(*tmea)->mea_ids[i] = mea->mea_ids[i];
-		fid_le_to_cpu(&(*tmea)->mea_ids[i], &(*tmea)->mea_ids[i]);
+	if (!lmmp) {
+		if (lsm)
+			stripe_count = lmv->lsm_md_stripe_count;
+		else
+			stripe_count = lmv_obd->desc.ld_tgt_count;
+
+		return lmv_mds_md_size(stripe_count, LMV_MAGIC_V1);
 	}
-	return mea_size;
+
+	return lmv_pack_md((union lmv_mds_md **)lmmp, lmv, 0);
 }
 
 static int lmv_cancel_unused(struct obd_export *exp, const struct lu_fid *fid,
@@ -2484,7 +3028,7 @@ static int lmv_cancel_unused(struct obd_export *exp, const struct lu_fid *fid,
 	struct lmv_obd	  *lmv = &obd->u.lmv;
 	int		      rc = 0;
 	int		      err;
-	int		      i;
+	u32 i;
 
 	LASSERT(fid);
 
@@ -2502,8 +3046,9 @@ static int lmv_cancel_unused(struct obd_export *exp, const struct lu_fid *fid,
 	return rc;
 }
 
-static int lmv_set_lock_data(struct obd_export *exp, __u64 *lockh, void *data,
-			     __u64 *bits)
+static int lmv_set_lock_data(struct obd_export *exp,
+			     const struct lustre_handle *lockh,
+			     void *data, __u64 *bits)
 {
 	struct lmv_obd	  *lmv = &exp->exp_obd->u.lmv;
 	struct lmv_tgt_desc *tgt = lmv->tgts[0];
@@ -2526,24 +3071,32 @@ static enum ldlm_mode lmv_lock_match(struct obd_export *exp, __u64 flags,
 	struct obd_device       *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
 	enum ldlm_mode	      rc;
-	int		      i;
+	int tgt;
+	u32 i;
 
 	CDEBUG(D_INODE, "Lock match for "DFID"\n", PFID(fid));
 
 	/*
-	 * With CMD every object can have two locks in different namespaces:
-	 * lookup lock in space of mds storing direntry and update/open lock in
-	 * space of mds storing inode. Thus we check all targets, not only that
-	 * one fid was created in.
+	 * With DNE every object can have two locks in different namespaces:
+	 * lookup lock in space of MDT storing direntry and update/open lock in
+	 * space of MDT storing inode.  Try the MDT that the FID maps to first,
+	 * since this can be easily found, and only try others if that fails.
 	 */
-	for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
-		struct lmv_tgt_desc *tgt = lmv->tgts[i];
+	for (i = 0, tgt = lmv_find_target_index(lmv, fid);
+	     i < lmv->desc.ld_tgt_count;
+	     i++, tgt = (tgt + 1) % lmv->desc.ld_tgt_count) {
+		if (tgt < 0) {
+			CDEBUG(D_HA, "%s: "DFID" is inaccessible: rc = %d\n",
+			       obd->obd_name, PFID(fid), tgt);
+			tgt = 0;
+		}
 
-		if (!tgt || !tgt->ltd_exp || !tgt->ltd_active)
+		if (!lmv->tgts[tgt] || !lmv->tgts[tgt]->ltd_exp ||
+		    !lmv->tgts[tgt]->ltd_active)
 			continue;
 
-		rc = md_lock_match(tgt->ltd_exp, flags, fid, type, policy, mode,
-				   lockh);
+		rc = md_lock_match(lmv->tgts[tgt]->ltd_exp, flags, fid,
+				   type, policy, mode, lockh);
 		if (rc)
 			return rc;
 	}
@@ -2571,8 +3124,10 @@ static int lmv_free_lustre_md(struct obd_export *exp, struct lustre_md *md)
 	struct lmv_obd	  *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc *tgt = lmv->tgts[0];
 
-	if (md->mea)
-		obd_free_memmd(exp, (void *)&md->mea);
+	if (md->lmv) {
+		lmv_free_memmd(md->lmv);
+		md->lmv = NULL;
+	}
 	if (!tgt || !tgt->ltd_exp)
 		return -EINVAL;
 	return md_free_lustre_md(tgt->ltd_exp, md);
@@ -2621,7 +3176,7 @@ static int lmv_intent_getattr_async(struct obd_export *exp,
 	if (rc)
 		return rc;
 
-	tgt = lmv_find_target(lmv, &op_data->op_fid1);
+	tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid1);
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
 
@@ -2649,6 +3204,23 @@ static int lmv_revalidate_lock(struct obd_export *exp, struct lookup_intent *it,
 	return rc;
 }
 
+static int
+lmv_get_fid_from_lsm(struct obd_export *exp,
+		     const struct lmv_stripe_md *lsm,
+		     const char *name, int namelen, struct lu_fid *fid)
+{
+	const struct lmv_oinfo *oinfo;
+
+	LASSERT(lsm);
+	oinfo = lsm_name_to_stripe_info(lsm, name, namelen);
+	if (IS_ERR(oinfo))
+		return PTR_ERR(oinfo);
+
+	*fid = oinfo->lmo_fid;
+
+	return 0;
+}
+
 /**
  * For lmv, only need to send request to master MDT, and the master MDT will
  * process with other slave MDTs. The only exception is Q_GETOQUOTA for which
@@ -2660,8 +3232,9 @@ static int lmv_quotactl(struct obd_device *unused, struct obd_export *exp,
 	struct obd_device   *obd = class_exp2obd(exp);
 	struct lmv_obd      *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc *tgt = lmv->tgts[0];
-	int		  rc = 0, i;
+	int rc = 0;
 	__u64 curspace = 0, curinodes = 0;
+	u32 i;
 
 	if (!tgt || !tgt->ltd_exp || !tgt->ltd_active ||
 	    !lmv->desc.ld_tgt_count) {
@@ -2704,7 +3277,8 @@ static int lmv_quotacheck(struct obd_device *unused, struct obd_export *exp,
 	struct obd_device   *obd = class_exp2obd(exp);
 	struct lmv_obd      *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc *tgt;
-	int		  i, rc = 0;
+	int rc = 0;
+	u32 i;
 
 	for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
 		int err;
@@ -2723,6 +3297,47 @@ static int lmv_quotacheck(struct obd_device *unused, struct obd_export *exp,
 	return rc;
 }
 
+static int lmv_merge_attr(struct obd_export *exp,
+			  const struct lmv_stripe_md *lsm,
+			  struct cl_attr *attr,
+			  ldlm_blocking_callback cb_blocking)
+{
+	int rc, i;
+
+	rc = lmv_revalidate_slaves(exp, lsm, cb_blocking, 0);
+	if (rc < 0)
+		return rc;
+
+	for (i = 0; i < lsm->lsm_md_stripe_count; i++) {
+		struct inode *inode = lsm->lsm_md_oinfo[i].lmo_root;
+
+		CDEBUG(D_INFO, ""DFID" size %llu, blocks %llu nlink %u, atime %lu ctime %lu, mtime %lu.\n",
+		       PFID(&lsm->lsm_md_oinfo[i].lmo_fid),
+		       i_size_read(inode), (unsigned long long)inode->i_blocks,
+		       inode->i_nlink, LTIME_S(inode->i_atime),
+		       LTIME_S(inode->i_ctime), LTIME_S(inode->i_mtime));
+
+		/* for slave stripe, it needs to subtract nlink for . and .. */
+		if (i)
+			attr->cat_nlink += inode->i_nlink - 2;
+		else
+			attr->cat_nlink = inode->i_nlink;
+
+		attr->cat_size += i_size_read(inode);
+		attr->cat_blocks += inode->i_blocks;
+
+		if (attr->cat_atime < LTIME_S(inode->i_atime))
+			attr->cat_atime = LTIME_S(inode->i_atime);
+
+		if (attr->cat_ctime < LTIME_S(inode->i_ctime))
+			attr->cat_ctime = LTIME_S(inode->i_ctime);
+
+		if (attr->cat_mtime < LTIME_S(inode->i_mtime))
+			attr->cat_mtime = LTIME_S(inode->i_mtime);
+	}
+	return 0;
+}
+
 static struct obd_ops lmv_obd_ops = {
 	.owner		= THIS_MODULE,
 	.setup		= lmv_setup,
@@ -2746,7 +3361,6 @@ static struct obd_ops lmv_obd_ops = {
 static struct md_ops lmv_md_ops = {
 	.getstatus		= lmv_getstatus,
 	.null_inode		= lmv_null_inode,
-	.find_cbdata		= lmv_find_cbdata,
 	.close			= lmv_close,
 	.create			= lmv_create,
 	.done_writing		= lmv_done_writing,
@@ -2760,7 +3374,7 @@ static struct md_ops lmv_md_ops = {
 	.setattr		= lmv_setattr,
 	.setxattr		= lmv_setxattr,
 	.sync			= lmv_sync,
-	.readpage		= lmv_readpage,
+	.read_page		= lmv_read_page,
 	.unlink			= lmv_unlink,
 	.init_ea_size		= lmv_init_ea_size,
 	.cancel_unused		= lmv_cancel_unused,
@@ -2768,10 +3382,12 @@ static struct md_ops lmv_md_ops = {
 	.lock_match		= lmv_lock_match,
 	.get_lustre_md		= lmv_get_lustre_md,
 	.free_lustre_md		= lmv_free_lustre_md,
+	.merge_attr		= lmv_merge_attr,
 	.set_open_replay_data	= lmv_set_open_replay_data,
 	.clear_open_replay_data	= lmv_clear_open_replay_data,
 	.intent_getattr_async	= lmv_intent_getattr_async,
-	.revalidate_lock	= lmv_revalidate_lock
+	.revalidate_lock	= lmv_revalidate_lock,
+	.get_fid_from_lsm	= lmv_get_fid_from_lsm,
 };
 
 static int __init lmv_init(void)
diff --git a/drivers/staging/lustre/lustre/lmv/lproc_lmv.c b/drivers/staging/lustre/lustre/lmv/lproc_lmv.c
index c29c361eb0cc..20bbdfc21d15 100644
--- a/drivers/staging/lustre/lustre/lmv/lproc_lmv.c
+++ b/drivers/staging/lustre/lustre/lmv/lproc_lmv.c
@@ -169,7 +169,7 @@ static int lmv_tgt_seq_show(struct seq_file *p, void *v)
 
 	if (!tgt)
 		return 0;
-	seq_printf(p, "%d: %s %sACTIVE\n",
+	seq_printf(p, "%u: %s %sACTIVE\n",
 		   tgt->ltd_idx, tgt->ltd_uuid.uuid,
 		   tgt->ltd_active ? "" : "IN");
 	return 0;
@@ -202,7 +202,7 @@ static struct lprocfs_vars lprocfs_lmv_obd_vars[] = {
 	{ NULL }
 };
 
-struct file_operations lmv_proc_target_fops = {
+const struct file_operations lmv_proc_target_fops = {
 	.owner		= THIS_MODULE,
 	.open		 = lmv_target_seq_open,
 	.read		 = seq_read,
diff --git a/drivers/staging/lustre/lustre/lov/lov_cl_internal.h b/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
index 9740568d9521..4d2b7d303fea 100644
--- a/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
+++ b/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
@@ -289,8 +289,8 @@ struct lov_lock {
 };
 
 struct lov_page {
-	struct cl_page_slice lps_cl;
-	int		  lps_invalid;
+	struct cl_page_slice	lps_cl;
+	unsigned int		lps_stripe; /* stripe index */
 };
 
 /*
@@ -556,6 +556,8 @@ struct lov_lock_link *lov_lock_link_find(const struct lu_env *env,
 					 struct lovsub_lock *sub);
 struct lov_io_sub *lov_page_subio(const struct lu_env *env, struct lov_io *lio,
 				  const struct cl_page_slice *slice);
+
+struct lov_stripe_md *lov_lsm_addref(struct lov_object *lov);
 int lov_page_stripe(const struct cl_page *page);
 
 #define lov_foreach_target(lov, var)		    \
@@ -742,11 +744,15 @@ static inline struct lov_thread_info *lov_env_info(const struct lu_env *env)
 static inline struct lov_layout_raid0 *lov_r0(struct lov_object *lov)
 {
 	LASSERT(lov->lo_type == LLT_RAID0);
-	LASSERT(lov->lo_lsm->lsm_wire.lw_magic == LOV_MAGIC ||
-		lov->lo_lsm->lsm_wire.lw_magic == LOV_MAGIC_V3);
+	LASSERT(lov->lo_lsm->lsm_magic == LOV_MAGIC ||
+		lov->lo_lsm->lsm_magic == LOV_MAGIC_V3);
 	return &lov->u.raid0;
 }
 
+/* lov_pack.c */
+int lov_getstripe(struct lov_object *obj, struct lov_stripe_md *lsm,
+		  struct lov_user_md __user *lump);
+
 /** @} lov */
 
 #endif
diff --git a/drivers/staging/lustre/lustre/lov/lov_dev.c b/drivers/staging/lustre/lustre/lov/lov_dev.c
index b1f260d43bc7..056ae2ed88e8 100644
--- a/drivers/staging/lustre/lustre/lov/lov_dev.c
+++ b/drivers/staging/lustre/lustre/lov/lov_dev.c
@@ -516,6 +516,5 @@ struct lu_device_type lov_device_type = {
 	.ldt_ops      = &lov_device_type_ops,
 	.ldt_ctx_tags = LCT_CL_THREAD
 };
-EXPORT_SYMBOL(lov_device_type);
 
 /** @} lov */
diff --git a/drivers/staging/lustre/lustre/lov/lov_ea.c b/drivers/staging/lustre/lustre/lov/lov_ea.c
index 5053dead17bb..214c561767e0 100644
--- a/drivers/staging/lustre/lustre/lov/lov_ea.c
+++ b/drivers/staging/lustre/lustre/lov/lov_ea.c
@@ -66,7 +66,8 @@ static int lsm_lmm_verify_common(struct lov_mds_md *lmm, int lmm_bytes,
 	}
 
 	if (lmm->lmm_stripe_size == 0 ||
-	    (le32_to_cpu(lmm->lmm_stripe_size)&(LOV_MIN_STRIPE_SIZE-1)) != 0) {
+	    (le32_to_cpu(lmm->lmm_stripe_size) &
+	     (LOV_MIN_STRIPE_SIZE - 1)) != 0) {
 		CERROR("bad stripe size %u\n",
 		       le32_to_cpu(lmm->lmm_stripe_size));
 		lov_dump_lmm_common(D_WARNING, lmm);
@@ -146,21 +147,15 @@ lsm_stripe_by_offset_plain(struct lov_stripe_md *lsm, int *stripeno,
 		*swidth = (u64)lsm->lsm_stripe_size * lsm->lsm_stripe_count;
 }
 
-static int lsm_destroy_plain(struct lov_stripe_md *lsm, struct obdo *oa,
-			     struct obd_export *md_exp)
-{
-	return 0;
-}
-
 /* Find minimum stripe maxbytes value.  For inactive or
- * reconnecting targets use LUSTRE_STRIPE_MAXBYTES.
+ * reconnecting targets use LUSTRE_EXT3_STRIPE_MAXBYTES.
  */
 static void lov_tgt_maxbytes(struct lov_tgt_desc *tgt, __u64 *stripe_maxbytes)
 {
 	struct obd_import *imp = tgt->ltd_obd->u.cli.cl_import;
 
 	if (!imp || !tgt->ltd_active) {
-		*stripe_maxbytes = LUSTRE_STRIPE_MAXBYTES;
+		*stripe_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
 		return;
 	}
 
@@ -171,7 +166,7 @@ static void lov_tgt_maxbytes(struct lov_tgt_desc *tgt, __u64 *stripe_maxbytes)
 		if (*stripe_maxbytes > imp->imp_connect_data.ocd_maxbytes)
 			*stripe_maxbytes = imp->imp_connect_data.ocd_maxbytes;
 	} else {
-		*stripe_maxbytes = LUSTRE_STRIPE_MAXBYTES;
+		*stripe_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
 	}
 	spin_unlock(&imp->imp_lock);
 }
@@ -245,7 +240,6 @@ static int lsm_unpackmd_v1(struct lov_obd *lov, struct lov_stripe_md *lsm,
 
 const struct lsm_operations lsm_v1_ops = {
 	.lsm_free	    = lsm_free_plain,
-	.lsm_destroy	 = lsm_destroy_plain,
 	.lsm_stripe_by_index    = lsm_stripe_by_index_plain,
 	.lsm_stripe_by_offset   = lsm_stripe_by_offset_plain,
 	.lsm_lmm_verify	 = lsm_lmm_verify_v1,
@@ -335,7 +329,6 @@ static int lsm_unpackmd_v3(struct lov_obd *lov, struct lov_stripe_md *lsm,
 
 const struct lsm_operations lsm_v3_ops = {
 	.lsm_free	    = lsm_free_plain,
-	.lsm_destroy	 = lsm_destroy_plain,
 	.lsm_stripe_by_index    = lsm_stripe_by_index_plain,
 	.lsm_stripe_by_offset   = lsm_stripe_by_offset_plain,
 	.lsm_lmm_verify	 = lsm_lmm_verify_v3,
diff --git a/drivers/staging/lustre/lustre/lov/lov_internal.h b/drivers/staging/lustre/lustre/lov/lov_internal.h
index 12bd511e8988..07e5ede3e952 100644
--- a/drivers/staging/lustre/lustre/lov/lov_internal.h
+++ b/drivers/staging/lustre/lustre/lov/lov_internal.h
@@ -134,8 +134,6 @@ static inline void lov_put_reqset(struct lov_request_set *set)
 /* lov_merge.c */
 void lov_merge_attrs(struct obdo *tgt, struct obdo *src, u64 valid,
 		     struct lov_stripe_md *lsm, int stripeno, int *set);
-int lov_adjust_kms(struct obd_export *exp, struct lov_stripe_md *lsm,
-		   u64 size, int shrink);
 int lov_merge_lvb_kms(struct lov_stripe_md *lsm,
 		      struct ost_lvb *lvb, __u64 *kms_place);
 
@@ -157,11 +155,6 @@ int lov_update_common_set(struct lov_request_set *set,
 int lov_prep_getattr_set(struct obd_export *exp, struct obd_info *oinfo,
 			 struct lov_request_set **reqset);
 int lov_fini_getattr_set(struct lov_request_set *set);
-int lov_prep_destroy_set(struct obd_export *exp, struct obd_info *oinfo,
-			 struct obdo *src_oa, struct lov_stripe_md *lsm,
-			 struct obd_trans_info *oti,
-			 struct lov_request_set **reqset);
-int lov_fini_destroy_set(struct lov_request_set *set);
 int lov_prep_setattr_set(struct obd_export *exp, struct obd_info *oinfo,
 			 struct obd_trans_info *oti,
 			 struct lov_request_set **reqset);
@@ -197,8 +190,6 @@ int lov_packmd(struct obd_export *exp, struct lov_mds_md **lmm,
 	       struct lov_stripe_md *lsm);
 int lov_unpackmd(struct obd_export *exp, struct lov_stripe_md **lsmp,
 		 struct lov_mds_md *lmm, int lmm_bytes);
-int lov_getstripe(struct obd_export *exp,
-		  struct lov_stripe_md *lsm, struct lov_user_md __user *lump);
 int lov_alloc_memmd(struct lov_stripe_md **lsmp, __u16 stripe_count,
 		    int pattern, int magic);
 int lov_free_memmd(struct lov_stripe_md **lsmp);
diff --git a/drivers/staging/lustre/lustre/lov/lov_io.c b/drivers/staging/lustre/lustre/lov/lov_io.c
index 84032a510254..d10157985ed9 100644
--- a/drivers/staging/lustre/lustre/lov/lov_io.c
+++ b/drivers/staging/lustre/lustre/lov/lov_io.c
@@ -87,6 +87,9 @@ static void lov_io_sub_inherit(struct cl_io *io, struct lov_io *lio,
 	case CIT_SETATTR: {
 		io->u.ci_setattr.sa_attr = parent->u.ci_setattr.sa_attr;
 		io->u.ci_setattr.sa_valid = parent->u.ci_setattr.sa_valid;
+		io->u.ci_setattr.sa_stripe_index = stripe;
+		io->u.ci_setattr.sa_parent_fid =
+					parent->u.ci_setattr.sa_parent_fid;
 		if (cl_io_is_trunc(io)) {
 			loff_t new_size = parent->u.ci_setattr.sa_attr.lvb_size;
 
@@ -244,14 +247,12 @@ void lov_sub_put(struct lov_io_sub *sub)
 
 int lov_page_stripe(const struct cl_page *page)
 {
-	struct lovsub_object *subobj;
 	const struct cl_page_slice *slice;
 
-	slice = cl_page_at(page, &lovsub_device_type);
+	slice = cl_page_at(page, &lov_device_type);
 	LASSERT(slice->cpl_obj);
 
-	subobj = cl2lovsub(slice->cpl_obj);
-	return subobj->lso_index;
+	return cl2lov_page(slice)->lps_stripe;
 }
 
 struct lov_io_sub *lov_page_subio(const struct lu_env *env, struct lov_io *lio,
@@ -298,8 +299,8 @@ static int lov_io_subio_init(const struct lu_env *env, struct lov_io *lio,
 	return result;
 }
 
-static void lov_io_slice_init(struct lov_io *lio,
-			      struct lov_object *obj, struct cl_io *io)
+static int lov_io_slice_init(struct lov_io *lio, struct lov_object *obj,
+			     struct cl_io *io)
 {
 	io->ci_result = 0;
 	lio->lis_object = obj;
@@ -314,6 +315,15 @@ static void lov_io_slice_init(struct lov_io *lio,
 		lio->lis_io_endpos = lio->lis_endpos;
 		if (cl_io_is_append(io)) {
 			LASSERT(io->ci_type == CIT_WRITE);
+
+			/*
+			 * If there is LOV EA hole, then we may cannot locate
+			 * the current file-tail exactly.
+			 */
+			if (unlikely(obj->lo_lsm->lsm_pattern &
+				     LOV_PATTERN_F_HOLE))
+				return -EIO;
+
 			lio->lis_pos = 0;
 			lio->lis_endpos = OBD_OBJECT_EOF;
 		}
@@ -349,6 +359,7 @@ static void lov_io_slice_init(struct lov_io *lio,
 	default:
 		LBUG();
 	}
+	return 0;
 }
 
 static void lov_io_fini(const struct lu_env *env, const struct cl_io_slice *ios)
@@ -870,7 +881,7 @@ int lov_io_init_raid0(const struct lu_env *env, struct cl_object *obj,
 	struct lov_object   *lov = cl2lov(obj);
 
 	INIT_LIST_HEAD(&lio->lis_active);
-	lov_io_slice_init(lio, lov, io);
+	io->ci_result = lov_io_slice_init(lio, lov, io);
 	if (io->ci_result == 0) {
 		io->ci_result = lov_io_subio_init(env, lio, io);
 		if (io->ci_result == 0) {
diff --git a/drivers/staging/lustre/lustre/lov/lov_merge.c b/drivers/staging/lustre/lustre/lov/lov_merge.c
index b9c90865fdfc..674af106b50b 100644
--- a/drivers/staging/lustre/lustre/lov/lov_merge.c
+++ b/drivers/staging/lustre/lustre/lov/lov_merge.c
@@ -105,45 +105,6 @@ int lov_merge_lvb_kms(struct lov_stripe_md *lsm,
 	return rc;
 }
 
-/* Must be called under the lov_stripe_lock() */
-int lov_adjust_kms(struct obd_export *exp, struct lov_stripe_md *lsm,
-		   u64 size, int shrink)
-{
-	struct lov_oinfo *loi;
-	int stripe = 0;
-	__u64 kms;
-
-	assert_spin_locked(&lsm->lsm_lock);
-	LASSERT(lsm->lsm_lock_owner == current_pid());
-
-	if (shrink) {
-		for (; stripe < lsm->lsm_stripe_count; stripe++) {
-			struct lov_oinfo *loi = lsm->lsm_oinfo[stripe];
-
-			kms = lov_size_to_stripe(lsm, size, stripe);
-			CDEBUG(D_INODE,
-			       "stripe %d KMS %sing %llu->%llu\n",
-			       stripe, kms > loi->loi_kms ? "increase":"shrink",
-			       loi->loi_kms, kms);
-			loi->loi_lvb.lvb_size = kms;
-			loi_kms_set(loi, loi->loi_lvb.lvb_size);
-		}
-		return 0;
-	}
-
-	if (size > 0)
-		stripe = lov_stripe_number(lsm, size - 1);
-	kms = lov_size_to_stripe(lsm, size, stripe);
-	loi = lsm->lsm_oinfo[stripe];
-
-	CDEBUG(D_INODE, "stripe %d KMS %sincreasing %llu->%llu\n",
-	       stripe, kms > loi->loi_kms ? "" : "not ", loi->loi_kms, kms);
-	if (kms > loi->loi_kms)
-		loi_kms_set(loi, kms);
-
-	return 0;
-}
-
 void lov_merge_attrs(struct obdo *tgt, struct obdo *src, u64 valid,
 		     struct lov_stripe_md *lsm, int stripeno, int *set)
 {
diff --git a/drivers/staging/lustre/lustre/lov/lov_obd.c b/drivers/staging/lustre/lustre/lov/lov_obd.c
index 9b92d5522edb..b23016f7ec26 100644
--- a/drivers/staging/lustre/lustre/lov/lov_obd.c
+++ b/drivers/staging/lustre/lustre/lov/lov_obd.c
@@ -41,6 +41,7 @@
 #include "../../include/linux/libcfs/libcfs.h"
 
 #include "../include/obd_support.h"
+#include "../include/lustre/lustre_ioctl.h"
 #include "../include/lustre_lib.h"
 #include "../include/lustre_net.h"
 #include "../include/lustre/lustre_idl.h"
@@ -940,7 +941,7 @@ int lov_process_config_base(struct obd_device *obd, struct lustre_cfg *lcfg,
 	}
 	case LCFG_PARAM: {
 		struct lprocfs_static_vars lvars = { NULL };
-		struct lov_desc *desc = &(obd->u.lov.desc);
+		struct lov_desc *desc = &obd->u.lov.desc;
 
 		if (!desc) {
 			rc = -EINVAL;
@@ -971,92 +972,6 @@ out:
 	return rc;
 }
 
-static int lov_recreate(struct obd_export *exp, struct obdo *src_oa,
-			struct lov_stripe_md **ea, struct obd_trans_info *oti)
-{
-	struct lov_stripe_md *obj_mdp, *lsm;
-	struct lov_obd *lov = &exp->exp_obd->u.lov;
-	unsigned ost_idx;
-	int rc, i;
-
-	LASSERT(src_oa->o_valid & OBD_MD_FLFLAGS &&
-		src_oa->o_flags & OBD_FL_RECREATE_OBJS);
-
-	obj_mdp = kzalloc(sizeof(*obj_mdp), GFP_NOFS);
-	if (!obj_mdp)
-		return -ENOMEM;
-
-	ost_idx = src_oa->o_nlink;
-	lsm = *ea;
-	if (!lsm) {
-		rc = -EINVAL;
-		goto out;
-	}
-	if (ost_idx >= lov->desc.ld_tgt_count ||
-	    !lov->lov_tgts[ost_idx]) {
-		rc = -EINVAL;
-		goto out;
-	}
-
-	for (i = 0; i < lsm->lsm_stripe_count; i++) {
-		struct lov_oinfo *loi = lsm->lsm_oinfo[i];
-
-		if (lov_oinfo_is_dummy(loi))
-			continue;
-
-		if (loi->loi_ost_idx == ost_idx) {
-			if (ostid_id(&loi->loi_oi) != ostid_id(&src_oa->o_oi)) {
-				rc = -EINVAL;
-				goto out;
-			}
-			break;
-		}
-	}
-	if (i == lsm->lsm_stripe_count) {
-		rc = -EINVAL;
-		goto out;
-	}
-
-	rc = obd_create(NULL, lov->lov_tgts[ost_idx]->ltd_exp,
-			src_oa, &obj_mdp, oti);
-out:
-	kfree(obj_mdp);
-	return rc;
-}
-
-/* the LOV expects oa->o_id to be set to the LOV object id */
-static int lov_create(const struct lu_env *env, struct obd_export *exp,
-		      struct obdo *src_oa, struct lov_stripe_md **ea,
-		      struct obd_trans_info *oti)
-{
-	struct lov_obd *lov;
-	int rc = 0;
-
-	LASSERT(ea);
-	if (!exp)
-		return -EINVAL;
-
-	if ((src_oa->o_valid & OBD_MD_FLFLAGS) &&
-	    src_oa->o_flags == OBD_FL_DELORPHAN) {
-		/* should be used with LOV anymore */
-		LBUG();
-	}
-
-	lov = &exp->exp_obd->u.lov;
-	if (!lov->desc.ld_active_tgt_count)
-		return -EIO;
-
-	obd_getref(exp->exp_obd);
-	/* Recreate a specific object id at the given OST index */
-	if ((src_oa->o_valid & OBD_MD_FLFLAGS) &&
-	    (src_oa->o_flags & OBD_FL_RECREATE_OBJS)) {
-		rc = lov_recreate(exp, src_oa, ea, oti);
-	}
-
-	obd_putref(exp->exp_obd);
-	return rc;
-}
-
 #define ASSERT_LSM_MAGIC(lsmp)						  \
 do {									    \
 	LASSERT((lsmp));						\
@@ -1065,59 +980,6 @@ do {									    \
 		 "%p->lsm_magic=%x\n", (lsmp), (lsmp)->lsm_magic);	      \
 } while (0)
 
-static int lov_destroy(const struct lu_env *env, struct obd_export *exp,
-		       struct obdo *oa, struct lov_stripe_md *lsm,
-		       struct obd_trans_info *oti, struct obd_export *md_exp)
-{
-	struct lov_request_set *set;
-	struct obd_info oinfo;
-	struct lov_request *req;
-	struct lov_obd *lov;
-	int rc = 0, err = 0;
-
-	ASSERT_LSM_MAGIC(lsm);
-
-	if (!exp || !exp->exp_obd)
-		return -ENODEV;
-
-	if (oa->o_valid & OBD_MD_FLCOOKIE) {
-		LASSERT(oti);
-		LASSERT(oti->oti_logcookies);
-	}
-
-	lov = &exp->exp_obd->u.lov;
-	obd_getref(exp->exp_obd);
-	rc = lov_prep_destroy_set(exp, &oinfo, oa, lsm, oti, &set);
-	if (rc)
-		goto out;
-
-	list_for_each_entry(req, &set->set_list, rq_link) {
-		if (oa->o_valid & OBD_MD_FLCOOKIE)
-			oti->oti_logcookies = set->set_cookies + req->rq_stripe;
-
-		err = obd_destroy(env, lov->lov_tgts[req->rq_idx]->ltd_exp,
-				  req->rq_oi.oi_oa, NULL, oti, NULL);
-		err = lov_update_common_set(set, req, err);
-		if (err) {
-			CERROR("%s: destroying objid "DOSTID" subobj "
-			       DOSTID" on OST idx %d: rc = %d\n",
-			       exp->exp_obd->obd_name, POSTID(&oa->o_oi),
-			       POSTID(&req->rq_oi.oi_oa->o_oi),
-			       req->rq_idx, err);
-			if (!rc)
-				rc = err;
-		}
-	}
-
-	if (rc == 0)
-		rc = lsm_op_find(lsm->lsm_magic)->lsm_destroy(lsm, oa, md_exp);
-
-	err = lov_fini_destroy_set(set);
-out:
-	obd_putref(exp->exp_obd);
-	return rc ? rc : err;
-}
-
 static int lov_getattr_interpret(struct ptlrpc_request_set *rqset,
 				 void *data, int rc)
 {
@@ -1267,46 +1129,6 @@ static int lov_setattr_async(struct obd_export *exp, struct obd_info *oinfo,
 	return 0;
 }
 
-/* find any ldlm lock of the inode in lov
- * return 0    not find
- *	1    find one
- *      < 0    error
- */
-static int lov_find_cbdata(struct obd_export *exp,
-			   struct lov_stripe_md *lsm, ldlm_iterator_t it,
-			   void *data)
-{
-	struct lov_obd *lov;
-	int rc = 0, i;
-
-	ASSERT_LSM_MAGIC(lsm);
-
-	if (!exp || !exp->exp_obd)
-		return -ENODEV;
-
-	lov = &exp->exp_obd->u.lov;
-	for (i = 0; i < lsm->lsm_stripe_count; i++) {
-		struct lov_stripe_md submd;
-		struct lov_oinfo *loi = lsm->lsm_oinfo[i];
-
-		if (lov_oinfo_is_dummy(loi))
-			continue;
-
-		if (!lov->lov_tgts[loi->loi_ost_idx]) {
-			CDEBUG(D_HA, "lov idx %d NULL\n", loi->loi_ost_idx);
-			continue;
-		}
-
-		submd.lsm_oi = loi->loi_oi;
-		submd.lsm_stripe_count = 0;
-		rc = obd_find_cbdata(lov->lov_tgts[loi->loi_ost_idx]->ltd_exp,
-				     &submd, it, data);
-		if (rc != 0)
-			return rc;
-	}
-	return rc;
-}
-
 int lov_statfs_interpret(struct ptlrpc_request_set *rqset, void *data, int rc)
 {
 	struct lov_request_set *lovset = (struct lov_request_set *)data;
@@ -1460,7 +1282,7 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 		}
 
 		desc = (struct lov_desc *)data->ioc_inlbuf1;
-		memcpy(desc, &(lov->desc), sizeof(*desc));
+		memcpy(desc, &lov->desc, sizeof(*desc));
 
 		uuidp = (struct obd_uuid *)data->ioc_inlbuf2;
 		genp = (__u32 *)data->ioc_inlbuf3;
@@ -1477,9 +1299,6 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 		obd_ioctl_freedata(buf, len);
 		break;
 	}
-	case LL_IOC_LOV_GETSTRIPE:
-		rc = lov_getstripe(exp, karg, uarg);
-		break;
 	case OBD_IOC_QUOTACTL: {
 		struct if_quotactl *qctl = karg;
 		struct lov_tgt_desc *tgt = NULL;
@@ -1726,6 +1545,8 @@ static int lov_fiemap(struct lov_obd *lov, __u32 keylen, void *key,
 	u64 fm_start, fm_end, fm_length, fm_end_offset;
 	u64 curr_loc;
 	int current_extent = 0, rc = 0, i;
+	/* Whether have we collected enough extents */
+	bool enough = false;
 	int ost_eof = 0; /* EOF for object */
 	int ost_done = 0; /* done with required mapping for this OST? */
 	int last_stripe;
@@ -1860,7 +1681,7 @@ static int lov_fiemap(struct lov_obd *lov, __u32 keylen, void *key,
 			lun_start += len_mapped_single_call;
 			fm_local->fm_length = req_fm_len - len_mapped_single_call;
 			req_fm_len = fm_local->fm_length;
-			fm_local->fm_extent_count = count_local;
+			fm_local->fm_extent_count = enough ? 1 : count_local;
 			fm_local->fm_mapped_extents = 0;
 			fm_local->fm_flags = fiemap->fm_flags;
 
@@ -1908,6 +1729,12 @@ inactive_tgt:
 					goto finish;
 				}
 				break;
+			} else if (enough) {
+				/*
+				 * We've collected enough extents and there are
+				 * more extents after it.
+				 */
+				goto finish;
 			}
 
 			/* If we just need num of extents then go to next device */
@@ -1916,8 +1743,9 @@ inactive_tgt:
 				break;
 			}
 
-			len_mapped_single_call = lcl_fm_ext[ext_count-1].fe_logical -
-				  lun_start + lcl_fm_ext[ext_count - 1].fe_length;
+			len_mapped_single_call =
+				lcl_fm_ext[ext_count - 1].fe_logical -
+				lun_start + lcl_fm_ext[ext_count - 1].fe_length;
 
 			/* Have we finished mapping on this device? */
 			if (req_fm_len <= len_mapped_single_call)
@@ -1926,14 +1754,15 @@ inactive_tgt:
 			/* Clear the EXTENT_LAST flag which can be present on
 			 * last extent
 			 */
-			if (lcl_fm_ext[ext_count-1].fe_flags & FIEMAP_EXTENT_LAST)
+			if (lcl_fm_ext[ext_count - 1].fe_flags &
+			    FIEMAP_EXTENT_LAST)
 				lcl_fm_ext[ext_count - 1].fe_flags &=
 							    ~FIEMAP_EXTENT_LAST;
 
 			curr_loc = lov_stripe_size(lsm,
-					   lcl_fm_ext[ext_count - 1].fe_logical+
-					   lcl_fm_ext[ext_count - 1].fe_length,
-					   cur_stripe);
+					lcl_fm_ext[ext_count - 1].fe_logical +
+					lcl_fm_ext[ext_count - 1].fe_length,
+					cur_stripe);
 			if (curr_loc >= fm_key->oa.o_size)
 				ost_eof = 1;
 
@@ -1945,7 +1774,7 @@ inactive_tgt:
 
 			/* Ran out of available extents? */
 			if (current_extent >= fiemap->fm_extent_count)
-				goto finish;
+				enough = true;
 		} while (ost_done == 0 && ost_eof == 0);
 
 		if (cur_stripe_wrap == last_stripe)
@@ -1985,73 +1814,14 @@ static int lov_get_info(const struct lu_env *env, struct obd_export *exp,
 {
 	struct obd_device *obddev = class_exp2obd(exp);
 	struct lov_obd *lov = &obddev->u.lov;
-	int i, rc;
+	int rc;
 
 	if (!vallen || !val)
 		return -EFAULT;
 
 	obd_getref(obddev);
 
-	if (KEY_IS(KEY_LOCK_TO_STRIPE)) {
-		struct {
-			char name[16];
-			struct ldlm_lock *lock;
-		} *data = key;
-		struct ldlm_res_id *res_id = &data->lock->l_resource->lr_name;
-		struct lov_oinfo *loi;
-		__u32 *stripe = val;
-
-		if (*vallen < sizeof(*stripe)) {
-			rc = -EFAULT;
-			goto out;
-		}
-		*vallen = sizeof(*stripe);
-
-		/* XXX This is another one of those bits that will need to
-		 * change if we ever actually support nested LOVs.  It uses
-		 * the lock's export to find out which stripe it is.
-		 */
-		/* XXX - it's assumed all the locks for deleted OSTs have
-		 * been cancelled. Also, the export for deleted OSTs will
-		 * be NULL and won't match the lock's export.
-		 */
-		for (i = 0; i < lsm->lsm_stripe_count; i++) {
-			loi = lsm->lsm_oinfo[i];
-			if (lov_oinfo_is_dummy(loi))
-				continue;
-
-			if (!lov->lov_tgts[loi->loi_ost_idx])
-				continue;
-			if (lov->lov_tgts[loi->loi_ost_idx]->ltd_exp ==
-			    data->lock->l_conn_export &&
-			    ostid_res_name_eq(&loi->loi_oi, res_id)) {
-				*stripe = i;
-				rc = 0;
-				goto out;
-			}
-		}
-		LDLM_ERROR(data->lock, "lock on inode without such object");
-		dump_lsm(D_ERROR, lsm);
-		rc = -ENXIO;
-		goto out;
-	} else if (KEY_IS(KEY_LAST_ID)) {
-		struct obd_id_info *info = val;
-		__u32 size = sizeof(u64);
-		struct lov_tgt_desc *tgt;
-
-		LASSERT(*vallen == sizeof(struct obd_id_info));
-		tgt = lov->lov_tgts[info->idx];
-
-		if (!tgt || !tgt->ltd_active) {
-			rc = -ESRCH;
-			goto out;
-		}
-
-		rc = obd_get_info(env, tgt->ltd_exp, keylen, key,
-				  &size, info->data, NULL);
-		rc = 0;
-		goto out;
-	} else if (KEY_IS(KEY_LOVDESC)) {
+	if (KEY_IS(KEY_LOVDESC)) {
 		struct lov_desc *desc_ret = val;
 		*desc_ret = lov->desc;
 
@@ -2060,22 +1830,6 @@ static int lov_get_info(const struct lu_env *env, struct obd_export *exp,
 	} else if (KEY_IS(KEY_FIEMAP)) {
 		rc = lov_fiemap(lov, keylen, key, vallen, val, lsm);
 		goto out;
-	} else if (KEY_IS(KEY_CONNECT_FLAG)) {
-		struct lov_tgt_desc *tgt;
-		__u64 ost_idx = *((__u64 *)val);
-
-		LASSERT(*vallen == sizeof(__u64));
-		LASSERT(ost_idx < lov->desc.ld_tgt_count);
-		tgt = lov->lov_tgts[ost_idx];
-
-		if (!tgt || !tgt->ltd_exp) {
-			rc = -ESRCH;
-			goto out;
-		}
-
-		*((__u64 *)val) = exp_connect_flags(tgt->ltd_exp);
-		rc = 0;
-		goto out;
 	} else if (KEY_IS(KEY_TGT_COUNT)) {
 		*((int *)val) = lov->desc.ld_tgt_count;
 		rc = 0;
@@ -2098,8 +1852,7 @@ static int lov_set_info_async(const struct lu_env *env, struct obd_export *exp,
 	u32 count;
 	int i, rc = 0, err;
 	struct lov_tgt_desc *tgt;
-	unsigned int incr = 0, check_uuid = 0, do_inactive = 0, no_set = 0;
-	unsigned int next_id = 0, mds_con = 0;
+	int do_inactive = 0, no_set = 0;
 
 	if (!set) {
 		no_set = 1;
@@ -2111,18 +1864,8 @@ static int lov_set_info_async(const struct lu_env *env, struct obd_export *exp,
 	obd_getref(obddev);
 	count = lov->desc.ld_tgt_count;
 
-	if (KEY_IS(KEY_NEXT_ID)) {
-		count = vallen / sizeof(struct obd_id_info);
-		vallen = sizeof(u64);
-		incr = sizeof(struct obd_id_info);
-		do_inactive = 1;
-		next_id = 1;
-	} else if (KEY_IS(KEY_CHECKSUM)) {
+	if (KEY_IS(KEY_CHECKSUM)) {
 		do_inactive = 1;
-	} else if (KEY_IS(KEY_EVICT_BY_NID)) {
-		/* use defaults:  do_inactive = incr = 0; */
-	} else if (KEY_IS(KEY_MDS_CONN)) {
-		mds_con = 1;
 	} else if (KEY_IS(KEY_CACHE_SET)) {
 		LASSERT(!lov->lov_cache);
 		lov->lov_cache = val;
@@ -2130,11 +1873,9 @@ static int lov_set_info_async(const struct lu_env *env, struct obd_export *exp,
 		cl_cache_incref(lov->lov_cache);
 	}
 
-	for (i = 0; i < count; i++, val = (char *)val + incr) {
-		if (next_id)
-			tgt = lov->lov_tgts[((struct obd_id_info *)val)->idx];
-		else
-			tgt = lov->lov_tgts[i];
+	for (i = 0; i < count; i++) {
+		tgt = lov->lov_tgts[i];
+
 		/* OST was disconnected */
 		if (!tgt || !tgt->ltd_exp)
 			continue;
@@ -2143,34 +1884,8 @@ static int lov_set_info_async(const struct lu_env *env, struct obd_export *exp,
 		if (!tgt->ltd_active && !do_inactive)
 			continue;
 
-		if (mds_con) {
-			struct mds_group_info *mgi;
-
-			LASSERT(vallen == sizeof(*mgi));
-			mgi = (struct mds_group_info *)val;
-
-			/* Only want a specific OSC */
-			if (mgi->uuid && !obd_uuid_equals(mgi->uuid,
-							  &tgt->ltd_uuid))
-				continue;
-
-			err = obd_set_info_async(env, tgt->ltd_exp,
-						 keylen, key, sizeof(int),
-						 &mgi->group, set);
-		} else if (next_id) {
-			err = obd_set_info_async(env, tgt->ltd_exp,
-					 keylen, key, vallen,
-					 ((struct obd_id_info *)val)->data, set);
-		} else {
-			/* Only want a specific OSC */
-			if (check_uuid &&
-			    !obd_uuid_equals(val, &tgt->ltd_uuid))
-				continue;
-
-			err = obd_set_info_async(env, tgt->ltd_exp,
-						 keylen, key, vallen, val, set);
-		}
-
+		err = obd_set_info_async(env, tgt->ltd_exp, keylen, key,
+					 vallen, val, set);
 		if (!rc)
 			rc = err;
 	}
@@ -2318,12 +2033,8 @@ static struct obd_ops lov_obd_ops = {
 	.statfs_async   = lov_statfs_async,
 	.packmd         = lov_packmd,
 	.unpackmd       = lov_unpackmd,
-	.create         = lov_create,
-	.destroy        = lov_destroy,
 	.getattr_async  = lov_getattr_async,
 	.setattr_async  = lov_setattr_async,
-	.adjust_kms     = lov_adjust_kms,
-	.find_cbdata    = lov_find_cbdata,
 	.iocontrol      = lov_iocontrol,
 	.get_info       = lov_get_info,
 	.set_info_async = lov_set_info_async,
diff --git a/drivers/staging/lustre/lustre/lov/lov_object.c b/drivers/staging/lustre/lustre/lov/lov_object.c
index f9621b0fd469..52f736338887 100644
--- a/drivers/staging/lustre/lustre/lov/lov_object.c
+++ b/drivers/staging/lustre/lustre/lov/lov_object.c
@@ -75,6 +75,13 @@ struct lov_layout_operations {
 
 static int lov_layout_wait(const struct lu_env *env, struct lov_object *lov);
 
+void lov_lsm_put(struct cl_object *unused, struct lov_stripe_md *lsm)
+{
+	if (lsm)
+		lov_free_memmd(&lsm);
+}
+EXPORT_SYMBOL(lov_lsm_put);
+
 /*****************************************************************************
  *
  * Lov object layout operations.
@@ -195,6 +202,10 @@ static int lov_page_slice_fixup(struct lov_object *lov,
 	struct cl_object_header *hdr = cl_object_header(&lov->lo_cl);
 	struct cl_object *o;
 
+	if (!stripe)
+		return hdr->coh_page_bufsize - lov->lo_cl.co_slice_off -
+		       cfs_size_round(sizeof(struct lov_page));
+
 	cl_object_for_each(o, stripe)
 		o->co_slice_off += hdr->coh_page_bufsize;
 
@@ -224,6 +235,7 @@ static int lov_init_raid0(const struct lu_env *env,
 
 	LASSERT(!lov->lo_lsm);
 	lov->lo_lsm = lsm_addref(lsm);
+	lov->lo_layout_invalid = true;
 	r0->lo_nr  = lsm->lsm_stripe_count;
 	LASSERT(r0->lo_nr <= lov_targets_nr(dev));
 
@@ -719,6 +731,10 @@ static int lov_layout_change(const struct lu_env *unused,
 		LASSERT(atomic_read(&lov->lo_active_ios) == 0);
 
 		lov->lo_type = LLT_EMPTY;
+		/* page bufsize fixup */
+		cl_object_header(&lov->lo_cl)->coh_page_bufsize -=
+			lov_page_slice_fixup(lov, NULL);
+
 		result = new_ops->llo_init(env,
 					lu2lov_dev(lov->lo_cl.co_lu.lo_dev),
 					lov, conf, state);
@@ -878,8 +894,8 @@ static int lov_attr_get(const struct lu_env *env, struct cl_object *obj,
 	return LOV_2DISPATCH_NOLOCK(cl2lov(obj), llo_getattr, env, obj, attr);
 }
 
-static int lov_attr_set(const struct lu_env *env, struct cl_object *obj,
-			const struct cl_attr *attr, unsigned valid)
+static int lov_attr_update(const struct lu_env *env, struct cl_object *obj,
+			   const struct cl_attr *attr, unsigned int valid)
 {
 	/*
 	 * No dispatch is required here, as no layout implements this.
@@ -895,13 +911,30 @@ int lov_lock_init(const struct lu_env *env, struct cl_object *obj,
 				    io);
 }
 
+static int lov_object_getstripe(const struct lu_env *env, struct cl_object *obj,
+				struct lov_user_md __user *lum)
+{
+	struct lov_object *lov = cl2lov(obj);
+	struct lov_stripe_md *lsm;
+	int rc = 0;
+
+	lsm = lov_lsm_addref(lov);
+	if (!lsm)
+		return -ENODATA;
+
+	rc = lov_getstripe(cl2lov(obj), lsm, lum);
+	lov_lsm_put(obj, lsm);
+	return rc;
+}
+
 static const struct cl_object_operations lov_ops = {
 	.coo_page_init = lov_page_init,
 	.coo_lock_init = lov_lock_init,
 	.coo_io_init   = lov_io_init,
 	.coo_attr_get  = lov_attr_get,
-	.coo_attr_set  = lov_attr_set,
-	.coo_conf_set  = lov_conf_set
+	.coo_attr_update = lov_attr_update,
+	.coo_conf_set  = lov_conf_set,
+	.coo_getstripe = lov_object_getstripe
 };
 
 static const struct lu_object_operations lov_lu_obj_ops = {
@@ -938,7 +971,7 @@ struct lu_object *lov_object_alloc(const struct lu_env *env,
 	return obj;
 }
 
-static struct lov_stripe_md *lov_lsm_addref(struct lov_object *lov)
+struct lov_stripe_md *lov_lsm_addref(struct lov_object *lov)
 {
 	struct lov_stripe_md *lsm = NULL;
 
@@ -969,13 +1002,6 @@ struct lov_stripe_md *lov_lsm_get(struct cl_object *clobj)
 }
 EXPORT_SYMBOL(lov_lsm_get);
 
-void lov_lsm_put(struct cl_object *unused, struct lov_stripe_md *lsm)
-{
-	if (lsm)
-		lov_free_memmd(&lsm);
-}
-EXPORT_SYMBOL(lov_lsm_put);
-
 int lov_read_and_clear_async_rc(struct cl_object *clob)
 {
 	struct lu_object *luobj;
diff --git a/drivers/staging/lustre/lustre/lov/lov_pack.c b/drivers/staging/lustre/lustre/lov/lov_pack.c
index 869ef41b13ca..be6e9857ce2a 100644
--- a/drivers/staging/lustre/lustre/lov/lov_pack.c
+++ b/drivers/staging/lustre/lustre/lov/lov_pack.c
@@ -45,6 +45,7 @@
 #include "../include/lustre/lustre_user.h"
 
 #include "lov_internal.h"
+#include "lov_cl_internal.h"
 
 void lov_dump_lmm_common(int level, void *lmmp)
 {
@@ -104,11 +105,9 @@ void lov_dump_lmm_v3(int level, struct lov_mds_md_v3 *lmm)
  *     LOVs properly.  For now lov_mds_md_size() just assumes one u64
  *     per stripe.
  */
-int lov_packmd(struct obd_export *exp, struct lov_mds_md **lmmp,
-	       struct lov_stripe_md *lsm)
+int lov_obd_packmd(struct lov_obd *lov, struct lov_mds_md **lmmp,
+		   struct lov_stripe_md *lsm)
 {
-	struct obd_device *obd = class_exp2obd(exp);
-	struct lov_obd *lov = &obd->u.lov;
 	struct lov_mds_md_v1 *lmmv1;
 	struct lov_mds_md_v3 *lmmv3;
 	__u16 stripe_count;
@@ -148,16 +147,11 @@ int lov_packmd(struct obd_export *exp, struct lov_mds_md **lmmp,
 			stripe_count = 0;
 		}
 	} else {
-		/* No need to allocate more than maximum supported stripes.
-		 * Anyway, this is pretty inaccurate since ld_tgt_count now
-		 * represents max index and we should rely on the actual number
-		 * of OSTs instead
+		/*
+		 * To calculate maximum easize by active targets at present,
+		 * which is exactly the maximum easize to be seen by LOV
 		 */
-		stripe_count = lov_mds_md_max_stripe_count(
-			lov->lov_ocd.ocd_max_easize, lmm_magic);
-
-		if (stripe_count > lov->desc.ld_tgt_count)
-			stripe_count = lov->desc.ld_tgt_count;
+		stripe_count = lov->desc.ld_active_tgt_count;
 	}
 
 	/* XXX LOV STACKING call into osc for sizes */
@@ -225,6 +219,15 @@ int lov_packmd(struct obd_export *exp, struct lov_mds_md **lmmp,
 	return lmm_size;
 }
 
+int lov_packmd(struct obd_export *exp, struct lov_mds_md **lmmp,
+	       struct lov_stripe_md *lsm)
+{
+	struct obd_device *obd = class_exp2obd(exp);
+	struct lov_obd *lov = &obd->u.lov;
+
+	return lov_obd_packmd(lov, lmmp, lsm);
+}
+
 /* Find the max stripecount we should use */
 __u16 lov_get_stripecnt(struct lov_obd *lov, __u32 magic, __u16 stripe_count)
 {
@@ -284,7 +287,7 @@ int lov_alloc_memmd(struct lov_stripe_md **lsmp, __u16 stripe_count,
 	spin_lock_init(&(*lsmp)->lsm_lock);
 	(*lsmp)->lsm_magic = magic;
 	(*lsmp)->lsm_stripe_count = stripe_count;
-	(*lsmp)->lsm_maxbytes = LUSTRE_STRIPE_MAXBYTES * stripe_count;
+	(*lsmp)->lsm_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES * stripe_count;
 	(*lsmp)->lsm_pattern = pattern;
 	(*lsmp)->lsm_pool_name[0] = '\0';
 	(*lsmp)->lsm_layout_gen = 0;
@@ -372,16 +375,17 @@ int lov_unpackmd(struct obd_export *exp,  struct lov_stripe_md **lsmp,
  * the maximum number of OST indices which will fit in the user buffer.
  * lmm_magic must be LOV_USER_MAGIC.
  */
-int lov_getstripe(struct obd_export *exp, struct lov_stripe_md *lsm,
+int lov_getstripe(struct lov_object *obj, struct lov_stripe_md *lsm,
 		  struct lov_user_md __user *lump)
 {
 	/*
 	 * XXX huge struct allocated on stack.
 	 */
 	/* we use lov_user_md_v3 because it is larger than lov_user_md_v1 */
+	struct lov_obd *lov;
 	struct lov_user_md_v3 lum;
 	struct lov_mds_md *lmmk = NULL;
-	int rc, lmm_size;
+	int rc, lmmk_size, lmm_size;
 	int lum_size;
 	mm_segment_t seg;
 
@@ -401,12 +405,13 @@ int lov_getstripe(struct obd_export *exp, struct lov_stripe_md *lsm,
 	lum_size = sizeof(struct lov_user_md_v1);
 	if (copy_from_user(&lum, lump, lum_size)) {
 		rc = -EFAULT;
-		goto out_set;
+		goto out;
 	}
-	if ((lum.lmm_magic != LOV_USER_MAGIC) &&
-	    (lum.lmm_magic != LOV_USER_MAGIC_V3)) {
+	if (lum.lmm_magic != LOV_USER_MAGIC_V1 &&
+	    lum.lmm_magic != LOV_USER_MAGIC_V3 &&
+	    lum.lmm_magic != LOV_USER_MAGIC_SPECIFIC) {
 		rc = -EINVAL;
-		goto out_set;
+		goto out;
 	}
 
 	if (lum.lmm_stripe_count &&
@@ -415,11 +420,13 @@ int lov_getstripe(struct obd_export *exp, struct lov_stripe_md *lsm,
 		lum.lmm_stripe_count = lsm->lsm_stripe_count;
 		rc = copy_to_user(lump, &lum, lum_size);
 		rc = -EOVERFLOW;
-		goto out_set;
+		goto out;
 	}
-	rc = lov_packmd(exp, &lmmk, lsm);
+	lov = lu2lov_dev(obj->lo_cl.co_lu.lo_dev)->ld_lov;
+	rc = lov_obd_packmd(lov, &lmmk, lsm);
 	if (rc < 0)
-		goto out_set;
+		goto out;
+	lmmk_size = rc;
 	lmm_size = rc;
 	rc = 0;
 
@@ -455,7 +462,7 @@ int lov_getstripe(struct obd_export *exp, struct lov_stripe_md *lsm,
 		lmm_size = lum_size;
 	} else if (lum.lmm_stripe_count < lmmk->lmm_stripe_count) {
 		rc = -EOVERFLOW;
-		goto out_set;
+		goto out_free;
 	}
 	/*
 	 * Have a difference between lov_mds_md & lov_user_md.
@@ -468,8 +475,9 @@ int lov_getstripe(struct obd_export *exp, struct lov_stripe_md *lsm,
 	if (copy_to_user(lump, lmmk, lmm_size))
 		rc = -EFAULT;
 
-	obd_free_diskmd(exp, &lmmk);
-out_set:
+out_free:
+	kfree(lmmk);
+out:
 	set_fs(seg);
 	return rc;
 }
diff --git a/drivers/staging/lustre/lustre/lov/lov_page.c b/drivers/staging/lustre/lustre/lov/lov_page.c
index c17026f14896..00bfabad78eb 100644
--- a/drivers/staging/lustre/lustre/lov/lov_page.c
+++ b/drivers/staging/lustre/lustre/lov/lov_page.c
@@ -65,7 +65,9 @@ static int lov_raid0_page_is_under_lock(const struct lu_env *env,
 	pgoff_t index = *max_index;
 	unsigned int pps; /* pages per stripe */
 
-	CDEBUG(D_READA, "*max_index = %lu, nr = %d\n", index, r0->lo_nr);
+	CDEBUG(D_READA, DFID "*max_index = %lu, nr = %d\n",
+	       PFID(lu_object_fid(lov2lu(loo))), index, r0->lo_nr);
+
 	if (index == 0) /* the page is not covered by any lock */
 		return 0;
 
@@ -80,7 +82,12 @@ static int lov_raid0_page_is_under_lock(const struct lu_env *env,
 
 	/* calculate the end of current stripe */
 	pps = loo->lo_lsm->lsm_stripe_size >> PAGE_SHIFT;
-	index = ((slice->cpl_index + pps) & ~(pps - 1)) - 1;
+	index = slice->cpl_index + pps - slice->cpl_index % pps - 1;
+
+	CDEBUG(D_READA, DFID "*max_index = %lu, index = %lu, pps = %u, stripe_size = %u, stripe no = %u, page index = %lu\n",
+	       PFID(lu_object_fid(lov2lu(loo))), *max_index, index, pps,
+	       loo->lo_lsm->lsm_stripe_size, lov_page_stripe(slice->cpl_page),
+	       slice->cpl_index);
 
 	/* never exceed the end of the stripe */
 	*max_index = min_t(pgoff_t, *max_index, index);
@@ -122,6 +129,7 @@ int lov_page_init_raid0(const struct lu_env *env, struct cl_object *obj,
 	rc = lov_stripe_offset(loo->lo_lsm, offset, stripe, &suboff);
 	LASSERT(rc == 0);
 
+	lpg->lps_stripe = stripe;
 	cl_page_slice_add(page, &lpg->lps_cl, obj, index, &lov_raid0_page_ops);
 
 	sub = lov_sub_get(env, lio, stripe);
diff --git a/drivers/staging/lustre/lustre/lov/lov_pool.c b/drivers/staging/lustre/lustre/lov/lov_pool.c
index 4c2d21729589..f8c8a361ef79 100644
--- a/drivers/staging/lustre/lustre/lov/lov_pool.c
+++ b/drivers/staging/lustre/lustre/lov/lov_pool.c
@@ -61,7 +61,7 @@ void lov_pool_putref(struct pool_desc *pool)
 		LASSERT(hlist_unhashed(&pool->pool_hash));
 		LASSERT(list_empty(&pool->pool_list));
 		LASSERT(!pool->pool_debugfs_entry);
-		lov_ost_pool_free(&(pool->pool_obds));
+		lov_ost_pool_free(&pool->pool_obds);
 		kfree(pool);
 	}
 }
@@ -92,7 +92,7 @@ static __u32 pool_hashfn(struct cfs_hash *hash_body, const void *key, unsigned m
 	for (i = 0; i < LOV_MAXPOOLNAME; i++) {
 		if (poolname[i] == '\0')
 			break;
-		result = (result << 4)^(result >> 28) ^  poolname[i];
+		result = (result << 4) ^ (result >> 28) ^  poolname[i];
 	}
 	return (result % mask);
 }
@@ -260,7 +260,7 @@ static int pool_proc_show(struct seq_file *s, void *v)
 	tgt = pool_tgt(iter->pool, iter->idx);
 	up_read(&pool_tgt_rw_sem(iter->pool));
 	if (tgt)
-		seq_printf(s, "%s\n", obd_uuid2str(&(tgt->ltd_uuid)));
+		seq_printf(s, "%s\n", obd_uuid2str(&tgt->ltd_uuid));
 
 	return 0;
 }
@@ -400,7 +400,7 @@ int lov_pool_new(struct obd_device *obd, char *poolname)
 	struct pool_desc *new_pool;
 	int rc;
 
-	lov = &(obd->u.lov);
+	lov = &obd->u.lov;
 
 	if (strlen(poolname) > LOV_MAXPOOLNAME)
 		return -ENAMETOOLONG;
@@ -471,7 +471,7 @@ int lov_pool_del(struct obd_device *obd, char *poolname)
 	struct lov_obd *lov;
 	struct pool_desc *pool;
 
-	lov = &(obd->u.lov);
+	lov = &obd->u.lov;
 
 	/* lookup and kill hash reference */
 	pool = cfs_hash_del_key(lov->lov_pools_hash_body, poolname);
@@ -503,7 +503,7 @@ int lov_pool_add(struct obd_device *obd, char *poolname, char *ostname)
 	unsigned int lov_idx;
 	int rc;
 
-	lov = &(obd->u.lov);
+	lov = &obd->u.lov;
 
 	pool = cfs_hash_lookup(lov->lov_pools_hash_body, poolname);
 	if (!pool)
@@ -517,7 +517,7 @@ int lov_pool_add(struct obd_device *obd, char *poolname, char *ostname)
 		if (!lov->lov_tgts[lov_idx])
 			continue;
 		if (obd_uuid_equals(&ost_uuid,
-				    &(lov->lov_tgts[lov_idx]->ltd_uuid)))
+				    &lov->lov_tgts[lov_idx]->ltd_uuid))
 			break;
 	}
 	/* test if ost found in lov */
@@ -547,7 +547,7 @@ int lov_pool_remove(struct obd_device *obd, char *poolname, char *ostname)
 	unsigned int lov_idx;
 	int rc = 0;
 
-	lov = &(obd->u.lov);
+	lov = &obd->u.lov;
 
 	pool = cfs_hash_lookup(lov->lov_pools_hash_body, poolname);
 	if (!pool)
@@ -562,7 +562,7 @@ int lov_pool_remove(struct obd_device *obd, char *poolname, char *ostname)
 			continue;
 
 		if (obd_uuid_equals(&ost_uuid,
-				    &(lov->lov_tgts[lov_idx]->ltd_uuid)))
+				    &lov->lov_tgts[lov_idx]->ltd_uuid))
 			break;
 	}
 
diff --git a/drivers/staging/lustre/lustre/lov/lov_request.c b/drivers/staging/lustre/lustre/lov/lov_request.c
index 4099b51f826e..09dcaf484c89 100644
--- a/drivers/staging/lustre/lustre/lov/lov_request.c
+++ b/drivers/staging/lustre/lustre/lov/lov_request.c
@@ -325,84 +325,6 @@ out_set:
 	return rc;
 }
 
-int lov_fini_destroy_set(struct lov_request_set *set)
-{
-	if (!set)
-		return 0;
-	LASSERT(set->set_exp);
-	if (atomic_read(&set->set_completes)) {
-		/* FIXME update qos data here */
-	}
-
-	lov_put_reqset(set);
-
-	return 0;
-}
-
-int lov_prep_destroy_set(struct obd_export *exp, struct obd_info *oinfo,
-			 struct obdo *src_oa, struct lov_stripe_md *lsm,
-			 struct obd_trans_info *oti,
-			 struct lov_request_set **reqset)
-{
-	struct lov_request_set *set;
-	struct lov_obd *lov = &exp->exp_obd->u.lov;
-	int rc = 0, i;
-
-	set = kzalloc(sizeof(*set), GFP_NOFS);
-	if (!set)
-		return -ENOMEM;
-	lov_init_set(set);
-
-	set->set_exp = exp;
-	set->set_oi = oinfo;
-	set->set_oi->oi_md = lsm;
-	set->set_oi->oi_oa = src_oa;
-	if (oti && src_oa->o_valid & OBD_MD_FLCOOKIE)
-		set->set_cookies = oti->oti_logcookies;
-
-	for (i = 0; i < lsm->lsm_stripe_count; i++) {
-		struct lov_oinfo *loi;
-		struct lov_request *req;
-
-		loi = lsm->lsm_oinfo[i];
-		if (lov_oinfo_is_dummy(loi))
-			continue;
-
-		if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
-			CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-			continue;
-		}
-
-		req = kzalloc(sizeof(*req), GFP_NOFS);
-		if (!req) {
-			rc = -ENOMEM;
-			goto out_set;
-		}
-
-		req->rq_stripe = i;
-		req->rq_idx = loi->loi_ost_idx;
-
-		req->rq_oi.oi_oa = kmem_cache_zalloc(obdo_cachep, GFP_NOFS);
-		if (!req->rq_oi.oi_oa) {
-			kfree(req);
-			rc = -ENOMEM;
-			goto out_set;
-		}
-		memcpy(req->rq_oi.oi_oa, src_oa, sizeof(*req->rq_oi.oi_oa));
-		req->rq_oi.oi_oa->o_oi = loi->loi_oi;
-		lov_set_add_req(req, set);
-	}
-	if (!set->set_count) {
-		rc = -EIO;
-		goto out_set;
-	}
-	*reqset = set;
-	return rc;
-out_set:
-	lov_fini_destroy_set(set);
-	return rc;
-}
-
 int lov_fini_setattr_set(struct lov_request_set *set)
 {
 	int rc = 0;
diff --git a/drivers/staging/lustre/lustre/lov/lovsub_object.c b/drivers/staging/lustre/lustre/lov/lovsub_object.c
index fb2f2660b3e9..a2bac7a3b71b 100644
--- a/drivers/staging/lustre/lustre/lov/lovsub_object.c
+++ b/drivers/staging/lustre/lustre/lov/lovsub_object.c
@@ -98,8 +98,8 @@ static int lovsub_object_print(const struct lu_env *env, void *cookie,
 	return (*p)(env, cookie, "[%d]", los->lso_index);
 }
 
-static int lovsub_attr_set(const struct lu_env *env, struct cl_object *obj,
-			   const struct cl_attr *attr, unsigned valid)
+static int lovsub_attr_update(const struct lu_env *env, struct cl_object *obj,
+			      const struct cl_attr *attr, unsigned int valid)
 {
 	struct lov_object *lov = cl2lovsub(obj)->lso_super;
 
@@ -119,7 +119,7 @@ static int lovsub_object_glimpse(const struct lu_env *env,
 static const struct cl_object_operations lovsub_ops = {
 	.coo_page_init = lovsub_page_init,
 	.coo_lock_init = lovsub_lock_init,
-	.coo_attr_set  = lovsub_attr_set,
+	.coo_attr_update = lovsub_attr_update,
 	.coo_glimpse   = lovsub_object_glimpse
 };
 
diff --git a/drivers/staging/lustre/lustre/mdc/lproc_mdc.c b/drivers/staging/lustre/lustre/mdc/lproc_mdc.c
index 98d15fb247bc..fca9450de57c 100644
--- a/drivers/staging/lustre/lustre/mdc/lproc_mdc.c
+++ b/drivers/staging/lustre/lustre/mdc/lproc_mdc.c
@@ -43,11 +43,10 @@ static ssize_t max_rpcs_in_flight_show(struct kobject *kobj,
 	int len;
 	struct obd_device *dev = container_of(kobj, struct obd_device,
 					      obd_kobj);
-	struct client_obd *cli = &dev->u.cli;
+	__u32 max;
 
-	spin_lock(&cli->cl_loi_list_lock);
-	len = sprintf(buf, "%u\n", cli->cl_max_rpcs_in_flight);
-	spin_unlock(&cli->cl_loi_list_lock);
+	max = obd_get_max_rpcs_in_flight(&dev->u.cli);
+	len = sprintf(buf, "%u\n", max);
 
 	return len;
 }
@@ -59,7 +58,6 @@ static ssize_t max_rpcs_in_flight_store(struct kobject *kobj,
 {
 	struct obd_device *dev = container_of(kobj, struct obd_device,
 					      obd_kobj);
-	struct client_obd *cli = &dev->u.cli;
 	int rc;
 	unsigned long val;
 
@@ -67,12 +65,9 @@ static ssize_t max_rpcs_in_flight_store(struct kobject *kobj,
 	if (rc)
 		return rc;
 
-	if (val < 1 || val > MDC_MAX_RIF_MAX)
-		return -ERANGE;
-
-	spin_lock(&cli->cl_loi_list_lock);
-	cli->cl_max_rpcs_in_flight = val;
-	spin_unlock(&cli->cl_loi_list_lock);
+	rc = obd_set_max_rpcs_in_flight(&dev->u.cli, val);
+	if (rc)
+		count = rc;
 
 	return count;
 }
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_internal.h b/drivers/staging/lustre/lustre/mdc/mdc_internal.h
index 58f2841cabe4..f446c1c2584b 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_internal.h
+++ b/drivers/staging/lustre/lustre/mdc/mdc_internal.h
@@ -34,63 +34,57 @@
 #define _MDC_INTERNAL_H
 
 #include "../include/lustre_mdc.h"
-#include "../include/lustre_mds.h"
 
 void lprocfs_mdc_init_vars(struct lprocfs_static_vars *lvars);
 
 void mdc_pack_body(struct ptlrpc_request *req, const struct lu_fid *fid,
-		   __u64 valid, int ea_size, __u32 suppgid, int flags);
-void mdc_is_subdir_pack(struct ptlrpc_request *req, const struct lu_fid *pfid,
-			const struct lu_fid *cfid, int flags);
+		   __u64 valid, size_t ea_size, __u32 suppgid, u32 flags);
 void mdc_swap_layouts_pack(struct ptlrpc_request *req,
 			   struct md_op_data *op_data);
-void mdc_readdir_pack(struct ptlrpc_request *req, __u64 pgoff, __u32 size,
+void mdc_readdir_pack(struct ptlrpc_request *req, __u64 pgoff, size_t size,
 		      const struct lu_fid *fid);
-void mdc_getattr_pack(struct ptlrpc_request *req, __u64 valid, int flags,
-		      struct md_op_data *data, int ea_size);
+void mdc_getattr_pack(struct ptlrpc_request *req, __u64 valid, u32 flags,
+		      struct md_op_data *data, size_t ea_size);
 void mdc_setattr_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		      void *ea, int ealen, void *ea2, int ea2len);
+		      void *ea, size_t ealen, void *ea2, size_t ea2len);
 void mdc_create_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		     const void *data, int datalen, __u32 mode, __u32 uid,
-		     __u32 gid, cfs_cap_t capability, __u64 rdev);
+		     const void *data, size_t datalen, umode_t mode, uid_t uid,
+		     gid_t gid, cfs_cap_t capability, __u64 rdev);
 void mdc_open_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		   __u32 mode, __u64 rdev, __u64 flags, const void *data,
-		   int datalen);
+		   umode_t mode, __u64 rdev, __u64 flags, const void *data,
+		   size_t datalen);
 void mdc_unlink_pack(struct ptlrpc_request *req, struct md_op_data *op_data);
 void mdc_link_pack(struct ptlrpc_request *req, struct md_op_data *op_data);
 void mdc_rename_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		     const char *old, int oldlen, const char *new, int newlen);
+		     const char *old, size_t oldlen,
+		     const char *new, size_t newlen);
 void mdc_close_pack(struct ptlrpc_request *req, struct md_op_data *op_data);
-int mdc_enter_request(struct client_obd *cli);
-void mdc_exit_request(struct client_obd *cli);
 
 /* mdc/mdc_locks.c */
 int mdc_set_lock_data(struct obd_export *exp,
-		      __u64 *lockh, void *data, __u64 *bits);
+		      const struct lustre_handle *lockh,
+		      void *data, __u64 *bits);
 
 int mdc_null_inode(struct obd_export *exp, const struct lu_fid *fid);
 
-int mdc_find_cbdata(struct obd_export *exp, const struct lu_fid *fid,
-		    ldlm_iterator_t it, void *data);
-
 int mdc_intent_lock(struct obd_export *exp,
-		    struct md_op_data *,
-		    void *lmm, int lmmsize,
-		    struct lookup_intent *, int,
+		    struct md_op_data *op_data,
+		    struct lookup_intent *it,
 		    struct ptlrpc_request **reqp,
 		    ldlm_blocking_callback cb_blocking,
 		    __u64 extra_lock_flags);
+
 int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
+		const ldlm_policy_data_t *policy,
 		struct lookup_intent *it, struct md_op_data *op_data,
-		struct lustre_handle *lockh, void *lmm, int lmmsize,
-		struct ptlrpc_request **req, __u64 extra_lock_flags);
+		struct lustre_handle *lockh, __u64 extra_lock_flags);
 
 int mdc_resource_get_unused(struct obd_export *exp, const struct lu_fid *fid,
 			    struct list_head *cancels, enum ldlm_mode  mode,
 			    __u64 bits);
 /* mdc/mdc_request.c */
-int mdc_fid_alloc(struct obd_export *exp, struct lu_fid *fid,
-		  struct md_op_data *op_data);
+int mdc_fid_alloc(const struct lu_env *env, struct obd_export *exp,
+		  struct lu_fid *fid, struct md_op_data *op_data);
 struct obd_client_handle;
 
 int mdc_set_open_replay_data(struct obd_export *exp,
@@ -101,16 +95,17 @@ void mdc_commit_open(struct ptlrpc_request *req);
 void mdc_replay_open(struct ptlrpc_request *req);
 
 int mdc_create(struct obd_export *exp, struct md_op_data *op_data,
-	       const void *data, int datalen, int mode, __u32 uid, __u32 gid,
-	       cfs_cap_t capability, __u64 rdev,
+	       const void *data, size_t datalen, umode_t mode, uid_t uid,
+	       gid_t gid, cfs_cap_t capability, __u64 rdev,
 	       struct ptlrpc_request **request);
 int mdc_link(struct obd_export *exp, struct md_op_data *op_data,
 	     struct ptlrpc_request **request);
 int mdc_rename(struct obd_export *exp, struct md_op_data *op_data,
-	       const char *old, int oldlen, const char *new, int newlen,
+	       const char *old, size_t oldlen,
+	       const char *new, size_t newlen,
 	       struct ptlrpc_request **request);
 int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
-		void *ea, int ealen, void *ea2, int ea2len,
+		void *ea, size_t ealen, void *ea2, size_t ea2len,
 		struct ptlrpc_request **request, struct md_open_data **mod);
 int mdc_unlink(struct obd_export *exp, struct md_op_data *op_data,
 	       struct ptlrpc_request **request);
@@ -138,4 +133,12 @@ static inline int mdc_prep_elc_req(struct obd_export *exp,
 				 count);
 }
 
+static inline unsigned long hash_x_index(__u64 hash, int hash64)
+{
+	if (BITS_PER_LONG == 32 && hash64)
+		hash >>= 32;
+	/* save hash 0 with hash 1 */
+	return ~0UL - (hash + !hash);
+}
+
 #endif
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_lib.c b/drivers/staging/lustre/lustre/mdc/mdc_lib.c
index 143bd7628572..aac7e04873e2 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_lib.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_lib.c
@@ -37,27 +37,12 @@
 
 static void __mdc_pack_body(struct mdt_body *b, __u32 suppgid)
 {
-	b->suppgid = suppgid;
-	b->uid = from_kuid(&init_user_ns, current_uid());
-	b->gid = from_kgid(&init_user_ns, current_gid());
-	b->fsuid = from_kuid(&init_user_ns, current_fsuid());
-	b->fsgid = from_kgid(&init_user_ns, current_fsgid());
-	b->capability = cfs_curproc_cap_pack();
-}
-
-void mdc_is_subdir_pack(struct ptlrpc_request *req, const struct lu_fid *pfid,
-			const struct lu_fid *cfid, int flags)
-{
-	struct mdt_body *b = req_capsule_client_get(&req->rq_pill,
-						    &RMF_MDT_BODY);
-
-	if (pfid) {
-		b->fid1 = *pfid;
-		b->valid = OBD_MD_FLID;
-	}
-	if (cfid)
-		b->fid2 = *cfid;
-	b->flags = flags;
+	b->mbo_suppgid = suppgid;
+	b->mbo_uid = from_kuid(&init_user_ns, current_uid());
+	b->mbo_gid = from_kgid(&init_user_ns, current_gid());
+	b->mbo_fsuid = from_kuid(&init_user_ns, current_fsuid());
+	b->mbo_fsgid = from_kgid(&init_user_ns, current_fsgid());
+	b->mbo_capability = cfs_curproc_cap_pack();
 }
 
 void mdc_swap_layouts_pack(struct ptlrpc_request *req,
@@ -67,43 +52,74 @@ void mdc_swap_layouts_pack(struct ptlrpc_request *req,
 						    &RMF_MDT_BODY);
 
 	__mdc_pack_body(b, op_data->op_suppgids[0]);
-	b->fid1 = op_data->op_fid1;
-	b->fid2 = op_data->op_fid2;
-	b->valid |= OBD_MD_FLID;
+	b->mbo_fid1 = op_data->op_fid1;
+	b->mbo_fid2 = op_data->op_fid2;
+	b->mbo_valid |= OBD_MD_FLID;
 }
 
 void mdc_pack_body(struct ptlrpc_request *req, const struct lu_fid *fid,
-		   __u64 valid, int ea_size, __u32 suppgid, int flags)
+		   __u64 valid, size_t ea_size, __u32 suppgid, u32 flags)
 {
 	struct mdt_body *b = req_capsule_client_get(&req->rq_pill,
 						    &RMF_MDT_BODY);
-	b->valid = valid;
-	b->eadatasize = ea_size;
-	b->flags = flags;
+	b->mbo_valid = valid;
+	b->mbo_eadatasize = ea_size;
+	b->mbo_flags = flags;
 	__mdc_pack_body(b, suppgid);
 	if (fid) {
-		b->fid1 = *fid;
-		b->valid |= OBD_MD_FLID;
+		b->mbo_fid1 = *fid;
+		b->mbo_valid |= OBD_MD_FLID;
 	}
 }
 
-void mdc_readdir_pack(struct ptlrpc_request *req, __u64 pgoff,
-		      __u32 size, const struct lu_fid *fid)
+/**
+ * Pack a name (path component) into a request
+ *
+ * \param[in] req	request
+ * \param[in] field	request field (usually RMF_NAME)
+ * \param[in] name	path component
+ * \param[in] name_len	length of path component
+ *
+ * \a field must be present in \a req and of size \a name_len + 1.
+ *
+ * \a name must be '\0' terminated of length \a name_len and represent
+ * a single path component (not contain '/').
+ */
+static void mdc_pack_name(struct ptlrpc_request *req,
+			  const struct req_msg_field *field,
+			  const char *name, size_t name_len)
+{
+	size_t buf_size;
+	size_t cpy_len;
+	char *buf;
+
+	buf = req_capsule_client_get(&req->rq_pill, field);
+	buf_size = req_capsule_get_size(&req->rq_pill, field, RCL_CLIENT);
+
+	LASSERT(name && name_len && buf && buf_size == name_len + 1);
+
+	cpy_len = strlcpy(buf, name, buf_size);
+
+	LASSERT(cpy_len == name_len && lu_name_is_valid_2(buf, cpy_len));
+}
+
+void mdc_readdir_pack(struct ptlrpc_request *req, __u64 pgoff, size_t size,
+		      const struct lu_fid *fid)
 {
 	struct mdt_body *b = req_capsule_client_get(&req->rq_pill,
 						    &RMF_MDT_BODY);
-	b->fid1 = *fid;
-	b->valid |= OBD_MD_FLID;
-	b->size = pgoff;		       /* !! */
-	b->nlink = size;			/* !! */
+	b->mbo_fid1 = *fid;
+	b->mbo_valid |= OBD_MD_FLID;
+	b->mbo_size = pgoff;		       /* !! */
+	b->mbo_nlink = size;			/* !! */
 	__mdc_pack_body(b, -1);
-	b->mode = LUDA_FID | LUDA_TYPE;
+	b->mbo_mode = LUDA_FID | LUDA_TYPE;
 }
 
 /* packing of MDS records */
 void mdc_create_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		     const void *data, int datalen, __u32 mode,
-		     __u32 uid, __u32 gid, cfs_cap_t cap_effective, __u64 rdev)
+		     const void *data, size_t datalen, umode_t mode,
+		     uid_t uid, gid_t gid, cfs_cap_t cap_effective, __u64 rdev)
 {
 	struct mdt_rec_create	*rec;
 	char			*tmp;
@@ -130,22 +146,17 @@ void mdc_create_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	rec->cr_bias     = op_data->op_bias;
 	rec->cr_umask    = current_umask();
 
-	tmp = req_capsule_client_get(&req->rq_pill, &RMF_NAME);
-	LOGL0(op_data->op_name, op_data->op_namelen, tmp);
-
+	mdc_pack_name(req, &RMF_NAME, op_data->op_name, op_data->op_namelen);
 	if (data) {
 		tmp = req_capsule_client_get(&req->rq_pill, &RMF_EADATA);
 		memcpy(tmp, data, datalen);
 	}
 }
 
-static __u64 mds_pack_open_flags(__u64 flags, __u32 mode)
+static inline __u64 mds_pack_open_flags(__u64 flags)
 {
 	__u64 cr_flags = (flags & (FMODE_READ | FMODE_WRITE |
-				   MDS_OPEN_HAS_EA | MDS_OPEN_HAS_OBJS |
-				   MDS_OPEN_OWNEROVERRIDE | MDS_OPEN_LOCK |
-				   MDS_OPEN_BY_FID | MDS_OPEN_LEASE |
-				   MDS_OPEN_RELEASE));
+				   MDS_OPEN_FL_INTERNAL));
 	if (flags & O_CREAT)
 		cr_flags |= MDS_OPEN_CREAT;
 	if (flags & O_EXCL)
@@ -171,8 +182,8 @@ static __u64 mds_pack_open_flags(__u64 flags, __u32 mode)
 
 /* packing of MDS records */
 void mdc_open_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		   __u32 mode, __u64 rdev, __u64 flags, const void *lmm,
-		   int lmmlen)
+		   umode_t mode, __u64 rdev, __u64 flags, const void *lmm,
+		   size_t lmmlen)
 {
 	struct mdt_rec_create *rec;
 	char *tmp;
@@ -190,7 +201,7 @@ void mdc_open_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	rec->cr_fid2 = op_data->op_fid2;
 
 	rec->cr_mode     = mode;
-	cr_flags = mds_pack_open_flags(flags, mode);
+	cr_flags = mds_pack_open_flags(flags);
 	rec->cr_rdev     = rdev;
 	rec->cr_time     = op_data->op_mod_time;
 	rec->cr_suppgid1 = op_data->op_suppgids[0];
@@ -200,8 +211,9 @@ void mdc_open_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	rec->cr_old_handle = op_data->op_handle;
 
 	if (op_data->op_name) {
-		tmp = req_capsule_client_get(&req->rq_pill, &RMF_NAME);
-		LOGL0(op_data->op_name, op_data->op_namelen, tmp);
+		mdc_pack_name(req, &RMF_NAME, op_data->op_name,
+			      op_data->op_namelen);
+
 		if (op_data->op_bias & MDS_CREATE_VOLATILE)
 			cr_flags |= MDS_OPEN_VOLATILE;
 	}
@@ -295,7 +307,7 @@ static void mdc_ioepoch_pack(struct mdt_ioepoch *epoch,
 }
 
 void mdc_setattr_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		      void *ea, int ealen, void *ea2, int ea2len)
+		      void *ea, size_t ealen, void *ea2, size_t ea2len)
 {
 	struct mdt_rec_setattr *rec;
 	struct mdt_ioepoch *epoch;
@@ -316,7 +328,7 @@ void mdc_setattr_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 
 	lum = req_capsule_client_get(&req->rq_pill, &RMF_EADATA);
 	if (!ea) { /* Remove LOV EA */
-		lum->lmm_magic = LOV_USER_MAGIC_V1;
+		lum->lmm_magic = cpu_to_le32(LOV_USER_MAGIC_V1);
 		lum->lmm_stripe_size = 0;
 		lum->lmm_stripe_count = 0;
 		lum->lmm_stripe_offset = (typeof(lum->lmm_stripe_offset))(-1);
@@ -334,7 +346,6 @@ void mdc_setattr_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 void mdc_unlink_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 {
 	struct mdt_rec_unlink *rec;
-	char *tmp;
 
 	CLASSERT(sizeof(struct mdt_rec_reint) == sizeof(struct mdt_rec_unlink));
 	rec = req_capsule_client_get(&req->rq_pill, &RMF_REC_REINT);
@@ -352,15 +363,12 @@ void mdc_unlink_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 	rec->ul_time     = op_data->op_mod_time;
 	rec->ul_bias     = op_data->op_bias;
 
-	tmp = req_capsule_client_get(&req->rq_pill, &RMF_NAME);
-	LASSERT(tmp);
-	LOGL0(op_data->op_name, op_data->op_namelen, tmp);
+	mdc_pack_name(req, &RMF_NAME, op_data->op_name, op_data->op_namelen);
 }
 
 void mdc_link_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 {
 	struct mdt_rec_link *rec;
-	char *tmp;
 
 	CLASSERT(sizeof(struct mdt_rec_reint) == sizeof(struct mdt_rec_link));
 	rec = req_capsule_client_get(&req->rq_pill, &RMF_REC_REINT);
@@ -376,20 +384,21 @@ void mdc_link_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 	rec->lk_time     = op_data->op_mod_time;
 	rec->lk_bias     = op_data->op_bias;
 
-	tmp = req_capsule_client_get(&req->rq_pill, &RMF_NAME);
-	LOGL0(op_data->op_name, op_data->op_namelen, tmp);
+	mdc_pack_name(req, &RMF_NAME, op_data->op_name, op_data->op_namelen);
 }
 
 void mdc_rename_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		     const char *old, int oldlen, const char *new, int newlen)
+		     const char *old, size_t oldlen,
+		     const char *new, size_t newlen)
 {
 	struct mdt_rec_rename *rec;
-	char *tmp;
 
 	CLASSERT(sizeof(struct mdt_rec_reint) == sizeof(struct mdt_rec_rename));
 	rec = req_capsule_client_get(&req->rq_pill, &RMF_REC_REINT);
 
 	/* XXX do something about time, uid, gid */
+	rec->rn_opcode	 = op_data->op_cli_flags & CLI_MIGRATE ?
+				REINT_MIGRATE : REINT_RENAME;
 	rec->rn_opcode   = REINT_RENAME;
 	rec->rn_fsuid    = op_data->op_fsuid;
 	rec->rn_fsgid    = op_data->op_fsgid;
@@ -402,39 +411,34 @@ void mdc_rename_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	rec->rn_mode     = op_data->op_mode;
 	rec->rn_bias     = op_data->op_bias;
 
-	tmp = req_capsule_client_get(&req->rq_pill, &RMF_NAME);
-	LOGL0(old, oldlen, tmp);
+	mdc_pack_name(req, &RMF_NAME, old, oldlen);
 
-	if (new) {
-		tmp = req_capsule_client_get(&req->rq_pill, &RMF_SYMTGT);
-		LOGL0(new, newlen, tmp);
-	}
+	if (new)
+		mdc_pack_name(req, &RMF_SYMTGT, new, newlen);
 }
 
-void mdc_getattr_pack(struct ptlrpc_request *req, __u64 valid, int flags,
-		      struct md_op_data *op_data, int ea_size)
+void mdc_getattr_pack(struct ptlrpc_request *req, __u64 valid, u32 flags,
+		      struct md_op_data *op_data, size_t ea_size)
 {
 	struct mdt_body *b = req_capsule_client_get(&req->rq_pill,
 						    &RMF_MDT_BODY);
 
-	b->valid = valid;
+	b->mbo_valid = valid;
 	if (op_data->op_bias & MDS_CHECK_SPLIT)
-		b->valid |= OBD_MD_FLCKSPLIT;
+		b->mbo_valid |= OBD_MD_FLCKSPLIT;
 	if (op_data->op_bias & MDS_CROSS_REF)
-		b->valid |= OBD_MD_FLCROSSREF;
-	b->eadatasize = ea_size;
-	b->flags = flags;
+		b->mbo_valid |= OBD_MD_FLCROSSREF;
+	b->mbo_eadatasize = ea_size;
+	b->mbo_flags = flags;
 	__mdc_pack_body(b, op_data->op_suppgids[0]);
 
-	b->fid1 = op_data->op_fid1;
-	b->fid2 = op_data->op_fid2;
-	b->valid |= OBD_MD_FLID;
-
-	if (op_data->op_name) {
-		char *tmp = req_capsule_client_get(&req->rq_pill, &RMF_NAME);
+	b->mbo_fid1 = op_data->op_fid1;
+	b->mbo_fid2 = op_data->op_fid2;
+	b->mbo_valid |= OBD_MD_FLID;
 
-		LOGL0(op_data->op_name, op_data->op_namelen, tmp);
-	}
+	if (op_data->op_name)
+		mdc_pack_name(req, &RMF_NAME, op_data->op_name,
+			      op_data->op_namelen);
 }
 
 static void mdc_hsm_release_pack(struct ptlrpc_request *req,
@@ -482,67 +486,3 @@ void mdc_close_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 	mdc_ioepoch_pack(epoch, op_data);
 	mdc_hsm_release_pack(req, op_data);
 }
-
-static int mdc_req_avail(struct client_obd *cli, struct mdc_cache_waiter *mcw)
-{
-	int rc;
-
-	spin_lock(&cli->cl_loi_list_lock);
-	rc = list_empty(&mcw->mcw_entry);
-	spin_unlock(&cli->cl_loi_list_lock);
-	return rc;
-};
-
-/* We record requests in flight in cli->cl_r_in_flight here.
- * There is only one write rpc possible in mdc anyway. If this to change
- * in the future - the code may need to be revisited.
- */
-int mdc_enter_request(struct client_obd *cli)
-{
-	int rc = 0;
-	struct mdc_cache_waiter mcw;
-	struct l_wait_info lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP, NULL);
-
-	spin_lock(&cli->cl_loi_list_lock);
-	if (cli->cl_r_in_flight >= cli->cl_max_rpcs_in_flight) {
-		list_add_tail(&mcw.mcw_entry, &cli->cl_cache_waiters);
-		init_waitqueue_head(&mcw.mcw_waitq);
-		spin_unlock(&cli->cl_loi_list_lock);
-		rc = l_wait_event(mcw.mcw_waitq, mdc_req_avail(cli, &mcw),
-				  &lwi);
-		if (rc) {
-			spin_lock(&cli->cl_loi_list_lock);
-			if (list_empty(&mcw.mcw_entry))
-				cli->cl_r_in_flight--;
-			list_del_init(&mcw.mcw_entry);
-			spin_unlock(&cli->cl_loi_list_lock);
-		}
-	} else {
-		cli->cl_r_in_flight++;
-		spin_unlock(&cli->cl_loi_list_lock);
-	}
-	return rc;
-}
-
-void mdc_exit_request(struct client_obd *cli)
-{
-	struct list_head *l, *tmp;
-	struct mdc_cache_waiter *mcw;
-
-	spin_lock(&cli->cl_loi_list_lock);
-	cli->cl_r_in_flight--;
-	list_for_each_safe(l, tmp, &cli->cl_cache_waiters) {
-		if (cli->cl_r_in_flight >= cli->cl_max_rpcs_in_flight) {
-			/* No free request slots anymore */
-			break;
-		}
-
-		mcw = list_entry(l, struct mdc_cache_waiter, mcw_entry);
-		list_del_init(&mcw->mcw_entry);
-		cli->cl_r_in_flight++;
-		wake_up(&mcw->mcw_waitq);
-	}
-	/* Empty waiting list? Decrease reqs in-flight number */
-
-	spin_unlock(&cli->cl_loi_list_lock);
-}
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_locks.c b/drivers/staging/lustre/lustre/mdc/mdc_locks.c
index f48b58423307..f1f6c082fa42 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_locks.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_locks.c
@@ -93,8 +93,8 @@ int it_open_error(int phase, struct lookup_intent *it)
 EXPORT_SYMBOL(it_open_error);
 
 /* this must be called on a lockh that is known to have a referenced lock */
-int mdc_set_lock_data(struct obd_export *exp, __u64 *lockh, void *data,
-		      __u64 *bits)
+int mdc_set_lock_data(struct obd_export *exp, const struct lustre_handle *lockh,
+		      void *data, __u64 *bits)
 {
 	struct ldlm_lock *lock;
 	struct inode *new_inode = data;
@@ -102,10 +102,10 @@ int mdc_set_lock_data(struct obd_export *exp, __u64 *lockh, void *data,
 	if (bits)
 		*bits = 0;
 
-	if (!*lockh)
+	if (!lustre_handle_is_used(lockh))
 		return 0;
 
-	lock = ldlm_handle2lock((struct lustre_handle *)lockh);
+	lock = ldlm_handle2lock(lockh);
 
 	LASSERT(lock);
 	lock_res_and_lock(lock);
@@ -174,7 +174,7 @@ int mdc_null_inode(struct obd_export *exp,
 	fid_build_reg_res_name(fid, &res_id);
 
 	res = ldlm_resource_get(ns, NULL, &res_id, 0, 0);
-	if (!res)
+	if (IS_ERR(res))
 		return 0;
 
 	lock_res(res);
@@ -185,28 +185,6 @@ int mdc_null_inode(struct obd_export *exp,
 	return 0;
 }
 
-/* find any ldlm lock of the inode in mdc
- * return 0    not find
- *	1    find one
- *      < 0    error
- */
-int mdc_find_cbdata(struct obd_export *exp,
-		    const struct lu_fid *fid,
-		    ldlm_iterator_t it, void *data)
-{
-	struct ldlm_res_id res_id;
-	int rc = 0;
-
-	fid_build_reg_res_name((struct lu_fid *)fid, &res_id);
-	rc = ldlm_resource_iterate(class_exp2obd(exp)->obd_namespace, &res_id,
-				   it, data);
-	if (rc == LDLM_ITER_STOP)
-		return 1;
-	else if (rc == LDLM_ITER_CONTINUE)
-		return 0;
-	return rc;
-}
-
 static inline void mdc_clear_replay_flag(struct ptlrpc_request *req, int rc)
 {
 	/* Don't hold error requests for replay. */
@@ -240,24 +218,24 @@ static void mdc_realloc_openmsg(struct ptlrpc_request *req,
 
 	/* FIXME: remove this explicit offset. */
 	rc = sptlrpc_cli_enlarge_reqbuf(req, DLM_INTENT_REC_OFF + 4,
-					body->eadatasize);
+					body->mbo_eadatasize);
 	if (rc) {
 		CERROR("Can't enlarge segment %d size to %d\n",
-		       DLM_INTENT_REC_OFF + 4, body->eadatasize);
-		body->valid &= ~OBD_MD_FLEASIZE;
-		body->eadatasize = 0;
+		       DLM_INTENT_REC_OFF + 4, body->mbo_eadatasize);
+		body->mbo_valid &= ~OBD_MD_FLEASIZE;
+		body->mbo_eadatasize = 0;
 	}
 }
 
-static struct ptlrpc_request *mdc_intent_open_pack(struct obd_export *exp,
-						   struct lookup_intent *it,
-						   struct md_op_data *op_data,
-						   void *lmm, int lmmsize,
-						   void *cb_data)
+static struct ptlrpc_request *
+mdc_intent_open_pack(struct obd_export *exp, struct lookup_intent *it,
+		     struct md_op_data *op_data)
 {
 	struct ptlrpc_request *req;
 	struct obd_device     *obddev = class_exp2obd(exp);
 	struct ldlm_intent    *lit;
+	const void *lmm = op_data->op_data;
+	u32 lmmsize = op_data->op_data_size;
 	LIST_HEAD(cancels);
 	int		    count = 0;
 	int		    mode;
@@ -274,7 +252,7 @@ static struct ptlrpc_request *mdc_intent_open_pack(struct obd_export *exp,
 			else
 				mode = LCK_PR;
 		} else {
-			if (it->it_flags & (FMODE_WRITE|MDS_OPEN_TRUNC))
+			if (it->it_flags & (FMODE_WRITE | MDS_OPEN_TRUNC))
 				mode = LCK_CW;
 			else if (it->it_flags & __FMODE_EXEC)
 				mode = LCK_PR;
@@ -325,6 +303,9 @@ static struct ptlrpc_request *mdc_intent_open_pack(struct obd_export *exp,
 	mdc_open_pack(req, op_data, it->it_create_mode, 0, it->it_flags, lmm,
 		      lmmsize);
 
+	req_capsule_set_size(&req->rq_pill, &RMF_MDT_MD, RCL_SERVER,
+			     obddev->u.cli.cl_max_mds_easize);
+
 	ptlrpc_request_set_replen(req);
 	return req;
 }
@@ -336,7 +317,8 @@ mdc_intent_getxattr_pack(struct obd_export *exp,
 {
 	struct ptlrpc_request	*req;
 	struct ldlm_intent	*lit;
-	int			rc, count = 0, maxdata;
+	int rc, count = 0;
+	u32 maxdata;
 	LIST_HEAD(cancels);
 
 	req = ptlrpc_request_alloc(class_exp2cliimp(exp),
@@ -421,7 +403,7 @@ static struct ptlrpc_request *mdc_intent_getattr_pack(struct obd_export *exp,
 				       OBD_MD_MEA | OBD_MD_FLACL;
 	struct ldlm_intent    *lit;
 	int		    rc;
-	int		    easize;
+	u32 easize;
 
 	req = ptlrpc_request_alloc(class_exp2cliimp(exp),
 				   &RQF_LDLM_INTENT_GETATTR);
@@ -526,7 +508,7 @@ static int mdc_finish_enqueue(struct obd_export *exp,
 	struct ldlm_reply   *lockrep;
 	struct ldlm_lock    *lock;
 	void		*lvb_data = NULL;
-	int		  lvb_len = 0;
+	u32 lvb_len = 0;
 
 	LASSERT(rc >= 0);
 	/* Similarly, if we're going to replay this request, we don't want to
@@ -605,7 +587,7 @@ static int mdc_finish_enqueue(struct obd_export *exp,
 			mdc_set_open_replay_data(NULL, NULL, it);
 		}
 
-		if ((body->valid & (OBD_MD_FLDIREA | OBD_MD_FLEASIZE)) != 0) {
+		if ((body->mbo_valid & (OBD_MD_FLDIREA | OBD_MD_FLEASIZE)) != 0) {
 			void *eadata;
 
 			mdc_update_max_ea_from_body(exp, body);
@@ -615,7 +597,7 @@ static int mdc_finish_enqueue(struct obd_export *exp,
 			 * Eventually, obd_unpackmd() will check the contents.
 			 */
 			eadata = req_capsule_server_sized_get(pill, &RMF_MDT_MD,
-							      body->eadatasize);
+							      body->mbo_eadatasize);
 			if (!eadata)
 				return -EPROTO;
 
@@ -623,7 +605,7 @@ static int mdc_finish_enqueue(struct obd_export *exp,
 			 * lock
 			 */
 			lvb_data = eadata;
-			lvb_len = body->eadatasize;
+			lvb_len = body->mbo_eadatasize;
 
 			/*
 			 * We save the reply LOV EA in case we have to replay a
@@ -639,20 +621,20 @@ static int mdc_finish_enqueue(struct obd_export *exp,
 
 				if (req_capsule_get_size(pill, &RMF_EADATA,
 							 RCL_CLIENT) <
-				    body->eadatasize)
+				    body->mbo_eadatasize)
 					mdc_realloc_openmsg(req, body);
 				else
 					req_capsule_shrink(pill, &RMF_EADATA,
-							   body->eadatasize,
+							   body->mbo_eadatasize,
 							   RCL_CLIENT);
 
 				req_capsule_set_size(pill, &RMF_EADATA,
 						     RCL_CLIENT,
-						     body->eadatasize);
+						     body->mbo_eadatasize);
 
 				lmm = req_capsule_client_get(pill, &RMF_EADATA);
 				if (lmm)
-					memcpy(lmm, eadata, body->eadatasize);
+					memcpy(lmm, eadata, body->mbo_eadatasize);
 			}
 		}
 	} else if (it->it_op & IT_LAYOUT) {
@@ -662,7 +644,8 @@ static int mdc_finish_enqueue(struct obd_export *exp,
 		lvb_len = req_capsule_get_size(pill, &RMF_DLM_LVB, RCL_SERVER);
 		if (lvb_len > 0) {
 			lvb_data = req_capsule_server_sized_get(pill,
-							&RMF_DLM_LVB, lvb_len);
+								&RMF_DLM_LVB,
+								lvb_len);
 			if (!lvb_data)
 				return -EPROTO;
 		}
@@ -705,9 +688,9 @@ static int mdc_finish_enqueue(struct obd_export *exp,
  * we don't know in advance the file type.
  */
 int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
+		const ldlm_policy_data_t *policy,
 		struct lookup_intent *it, struct md_op_data *op_data,
-		struct lustre_handle *lockh, void *lmm, int lmmsize,
-		struct ptlrpc_request **reqp, u64 extra_lock_flags)
+		struct lustre_handle *lockh, u64 extra_lock_flags)
 {
 	static const ldlm_policy_data_t lookup_policy = {
 		.l_inodebits = { MDS_INODELOCK_LOOKUP }
@@ -721,9 +704,8 @@ int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
 	static const ldlm_policy_data_t getxattr_policy = {
 		.l_inodebits = { MDS_INODELOCK_XATTR }
 	};
-	ldlm_policy_data_t const *policy = &lookup_policy;
 	struct obd_device *obddev = class_exp2obd(exp);
-	struct ptlrpc_request *req;
+	struct ptlrpc_request *req = NULL;
 	u64 flags, saved_flags = extra_lock_flags;
 	struct ldlm_res_id res_id;
 	int generation, resends = 0;
@@ -733,40 +715,32 @@ int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
 
 	LASSERTF(!it || einfo->ei_type == LDLM_IBITS, "lock type %d\n",
 		 einfo->ei_type);
-
 	fid_build_reg_res_name(&op_data->op_fid1, &res_id);
 
 	if (it) {
+		LASSERT(!policy);
+
 		saved_flags |= LDLM_FL_HAS_INTENT;
-		if (it->it_op & (IT_UNLINK | IT_GETATTR | IT_READDIR))
+		if (it->it_op & (IT_OPEN | IT_UNLINK | IT_GETATTR | IT_READDIR))
 			policy = &update_policy;
 		else if (it->it_op & IT_LAYOUT)
 			policy = &layout_policy;
 		else if (it->it_op & (IT_GETXATTR | IT_SETXATTR))
 			policy = &getxattr_policy;
+		else
+			policy = &lookup_policy;
 	}
 
-	LASSERT(!reqp);
-
 	generation = obddev->u.cli.cl_import->imp_generation;
 resend:
 	flags = saved_flags;
 	if (!it) {
-		/* The only way right now is FLOCK, in this case we hide flock
-		 * policy as lmm, but lmmsize is 0
-		 */
-		LASSERT(lmm && lmmsize == 0);
+		/* The only way right now is FLOCK. */
 		LASSERTF(einfo->ei_type == LDLM_FLOCK, "lock type %d\n",
 			 einfo->ei_type);
-		policy = lmm;
 		res_id.name[3] = LDLM_FLOCK;
-		req = NULL;
 	} else if (it->it_op & IT_OPEN) {
-		req = mdc_intent_open_pack(exp, it, op_data, lmm, lmmsize,
-					   einfo->ei_cbdata);
-		policy = &update_policy;
-		einfo->ei_cbdata = NULL;
-		lmm = NULL;
+		req = mdc_intent_open_pack(exp, it, op_data);
 	} else if (it->it_op & IT_UNLINK) {
 		req = mdc_intent_unlink_pack(exp, it, op_data);
 	} else if (it->it_op & (IT_GETATTR | IT_LOOKUP)) {
@@ -806,7 +780,7 @@ resend:
 	 */
 	if (it) {
 		mdc_get_rpc_lock(obddev->u.cli.cl_rpc_lock, it);
-		rc = mdc_enter_request(&obddev->u.cli);
+		rc = obd_get_request_slot(&obddev->u.cli);
 		if (rc != 0) {
 			mdc_put_rpc_lock(obddev->u.cli.cl_rpc_lock, it);
 			mdc_clear_replay_flag(req, 0);
@@ -834,13 +808,12 @@ resend:
 		return rc;
 	}
 
-	mdc_exit_request(&obddev->u.cli);
+	obd_put_request_slot(&obddev->u.cli);
 	mdc_put_rpc_lock(obddev->u.cli.cl_rpc_lock, it);
 
 	if (rc < 0) {
-		CDEBUG_LIMIT((rc == -EACCES || rc == -EIDRM) ? D_INFO : D_ERROR,
-			     "%s: ldlm_cli_enqueue failed: rc = %d\n",
-			     obddev->obd_name, rc);
+		CDEBUG(D_INFO, "%s: ldlm_cli_enqueue failed: rc = %d\n",
+		       obddev->obd_name, rc);
 
 		mdc_clear_replay_flag(req, rc);
 		ptlrpc_req_finished(req);
@@ -903,6 +876,9 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 	LASSERT(request != LP_POISON);
 	LASSERT(request->rq_repmsg != LP_POISON);
 
+	if (it->it_op & IT_READDIR)
+		return 0;
+
 	if (!it_disposition(it, DISP_IT_EXECD)) {
 		/* The server failed before it even started executing the
 		 * intent, i.e. because it couldn't unpack the request.
@@ -917,27 +893,6 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 	mdt_body = req_capsule_server_get(&request->rq_pill, &RMF_MDT_BODY);
 	LASSERT(mdt_body);      /* mdc_enqueue checked */
 
-	/* If we were revalidating a fid/name pair, mark the intent in
-	 * case we fail and get called again from lookup
-	 */
-	if (fid_is_sane(&op_data->op_fid2) &&
-	    it->it_create_mode & M_CHECK_STALE &&
-	    it->it_op != IT_GETATTR) {
-		/* Also: did we find the same inode? */
-		/* sever can return one of two fids:
-		 * op_fid2 - new allocated fid - if file is created.
-		 * op_fid3 - existent fid - if file only open.
-		 * op_fid3 is saved in lmv_intent_open
-		 */
-		if ((!lu_fid_eq(&op_data->op_fid2, &mdt_body->fid1)) &&
-		    (!lu_fid_eq(&op_data->op_fid3, &mdt_body->fid1))) {
-			CDEBUG(D_DENTRY, "Found stale data "DFID"("DFID")/"DFID
-			       "\n", PFID(&op_data->op_fid2),
-			       PFID(&op_data->op_fid2), PFID(&mdt_body->fid1));
-			return -ESTALE;
-		}
-	}
-
 	rc = it_open_error(DISP_LOOKUP_EXECD, it);
 	if (rc)
 		return rc;
@@ -980,10 +935,10 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 
 		LDLM_DEBUG(lock, "matching against this");
 
-		LASSERTF(fid_res_name_eq(&mdt_body->fid1,
+		LASSERTF(fid_res_name_eq(&mdt_body->mbo_fid1,
 					 &lock->l_resource->lr_name),
 			 "Lock res_id: "DLDLMRES", fid: "DFID"\n",
-			 PLDLMRES(lock->l_resource), PFID(&mdt_body->fid1));
+			 PLDLMRES(lock->l_resource), PFID(&mdt_body->mbo_fid1));
 		LDLM_LOCK_PUT(lock);
 
 		memcpy(&old_lock, lockh, sizeof(*lockh));
@@ -998,8 +953,8 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 	}
 	CDEBUG(D_DENTRY,
 	       "D_IT dentry %.*s intent: %s status %d disp %x rc %d\n",
-	       op_data->op_namelen, op_data->op_name, ldlm_it2str(it->it_op),
-	       it->it_status, it->it_disposition, rc);
+	       (int)op_data->op_namelen, op_data->op_name,
+	       ldlm_it2str(it->it_op), it->it_status, it->it_disposition, rc);
 	return rc;
 }
 
@@ -1042,6 +997,9 @@ int mdc_revalidate_lock(struct obd_export *exp, struct lookup_intent *it,
 						  MDS_INODELOCK_LOOKUP |
 						  MDS_INODELOCK_PERM;
 			break;
+		case IT_READDIR:
+			policy.l_inodebits.bits = MDS_INODELOCK_UPDATE;
+			break;
 		case IT_LAYOUT:
 			policy.l_inodebits.bits = MDS_INODELOCK_LAYOUT;
 			break;
@@ -1095,10 +1053,8 @@ int mdc_revalidate_lock(struct obd_export *exp, struct lookup_intent *it,
  * child lookup.
  */
 int mdc_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
-		    void *lmm, int lmmsize, struct lookup_intent *it,
-		    int lookup_flags, struct ptlrpc_request **reqp,
-		    ldlm_blocking_callback cb_blocking,
-		    __u64 extra_lock_flags)
+		    struct lookup_intent *it, struct ptlrpc_request **reqp,
+		    ldlm_blocking_callback cb_blocking, __u64 extra_lock_flags)
 {
 	struct ldlm_enqueue_info einfo = {
 		.ei_type	= LDLM_IBITS,
@@ -1112,14 +1068,14 @@ int mdc_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 	LASSERT(it);
 
 	CDEBUG(D_DLMTRACE, "(name: %.*s,"DFID") in obj "DFID
-		", intent: %s flags %#Lo\n", op_data->op_namelen,
+		", intent: %s flags %#Lo\n", (int)op_data->op_namelen,
 		op_data->op_name, PFID(&op_data->op_fid2),
 		PFID(&op_data->op_fid1), ldlm_it2str(it->it_op),
 		it->it_flags);
 
 	lockh.cookie = 0;
 	if (fid_is_sane(&op_data->op_fid2) &&
-	    (it->it_op & (IT_LOOKUP | IT_GETATTR))) {
+	    (it->it_op & (IT_LOOKUP | IT_GETATTR | IT_READDIR))) {
 		/* We could just return 1 immediately, but since we should only
 		 * be called in revalidate_it if we already have a lock, let's
 		 * verify that.
@@ -1135,13 +1091,13 @@ int mdc_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 
 	/* For case if upper layer did not alloc fid, do it now. */
 	if (!fid_is_sane(&op_data->op_fid2) && it->it_op & IT_CREAT) {
-		rc = mdc_fid_alloc(exp, &op_data->op_fid2, op_data);
+		rc = mdc_fid_alloc(NULL, exp, &op_data->op_fid2, op_data);
 		if (rc < 0) {
 			CERROR("Can't alloc new fid, rc %d\n", rc);
 			return rc;
 		}
 	}
-	rc = mdc_enqueue(exp, &einfo, it, op_data, &lockh, lmm, lmmsize, NULL,
+	rc = mdc_enqueue(exp, &einfo, NULL, it, op_data, &lockh,
 			 extra_lock_flags);
 	if (rc < 0)
 		return rc;
@@ -1170,7 +1126,7 @@ static int mdc_intent_getattr_async_interpret(const struct lu_env *env,
 
 	obddev = class_exp2obd(exp);
 
-	mdc_exit_request(&obddev->u.cli);
+	obd_put_request_slot(&obddev->u.cli);
 	if (OBD_FAIL_CHECK(OBD_FAIL_MDC_GETATTR_ENQUEUE))
 		rc = -ETIMEDOUT;
 
@@ -1222,15 +1178,15 @@ int mdc_intent_getattr_async(struct obd_export *exp,
 
 	CDEBUG(D_DLMTRACE,
 	       "name: %.*s in inode " DFID ", intent: %s flags %#Lo\n",
-	       op_data->op_namelen, op_data->op_name, PFID(&op_data->op_fid1),
-	       ldlm_it2str(it->it_op), it->it_flags);
+	       (int)op_data->op_namelen, op_data->op_name,
+	       PFID(&op_data->op_fid1), ldlm_it2str(it->it_op), it->it_flags);
 
 	fid_build_reg_res_name(&op_data->op_fid1, &res_id);
 	req = mdc_intent_getattr_pack(exp, it, op_data);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
-	rc = mdc_enter_request(&obddev->u.cli);
+	rc = obd_get_request_slot(&obddev->u.cli);
 	if (rc != 0) {
 		ptlrpc_req_finished(req);
 		return rc;
@@ -1239,7 +1195,7 @@ int mdc_intent_getattr_async(struct obd_export *exp,
 	rc = ldlm_cli_enqueue(exp, &req, einfo, &res_id, &policy, &flags, NULL,
 			      0, LVB_T_NONE, &minfo->mi_lockh, 1);
 	if (rc < 0) {
-		mdc_exit_request(&obddev->u.cli);
+		obd_put_request_slot(&obddev->u.cli);
 		ptlrpc_req_finished(req);
 		return rc;
 	}
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_reint.c b/drivers/staging/lustre/lustre/mdc/mdc_reint.c
index 5dba2c813857..c921e471fa27 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_reint.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_reint.c
@@ -86,7 +86,7 @@ int mdc_resource_get_unused(struct obd_export *exp, const struct lu_fid *fid,
 	fid_build_reg_res_name(fid, &res_id);
 	res = ldlm_resource_get(exp->exp_obd->obd_namespace,
 				NULL, &res_id, 0, 0);
-	if (!res)
+	if (IS_ERR(res))
 		return 0;
 	LDLM_RESOURCE_ADDREF(res);
 	/* Initialize ibits lock policy. */
@@ -99,7 +99,7 @@ int mdc_resource_get_unused(struct obd_export *exp, const struct lu_fid *fid,
 }
 
 int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
-		void *ea, int ealen, void *ea2, int ea2len,
+		void *ea, size_t ealen, void *ea2, size_t ea2len,
 		struct ptlrpc_request **request, struct md_open_data **mod)
 {
 	LIST_HEAD(cancels);
@@ -110,11 +110,10 @@ int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
 	__u64 bits;
 
 	bits = MDS_INODELOCK_UPDATE;
-	if (op_data->op_attr.ia_valid & (ATTR_MODE|ATTR_UID|ATTR_GID))
+	if (op_data->op_attr.ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID))
 		bits |= MDS_INODELOCK_LOOKUP;
 	if ((op_data->op_flags & MF_MDC_CANCEL_FID1) &&
-	    (fid_is_sane(&op_data->op_fid1)) &&
-	    !OBD_FAIL_CHECK(OBD_FAIL_LDLM_BL_CALLBACK_NET))
+	    (fid_is_sane(&op_data->op_fid1)))
 		count = mdc_resource_get_unused(exp, &op_data->op_fid1,
 						&cancels, LCK_EX, bits);
 	req = ptlrpc_request_alloc(class_exp2cliimp(exp),
@@ -177,8 +176,8 @@ int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
 
 		epoch = req_capsule_client_get(&req->rq_pill, &RMF_MDT_EPOCH);
 		body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
-		epoch->handle = body->handle;
-		epoch->ioepoch = body->ioepoch;
+		epoch->handle = body->mbo_handle;
+		epoch->ioepoch = body->mbo_ioepoch;
 		req->rq_replay_cb = mdc_replay_open;
 	/** bug 3633, open may be committed and estale answer is not error */
 	} else if (rc == -ESTALE && (op_data->op_flags & MF_SOM_CHANGE)) {
@@ -197,9 +196,9 @@ int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
 }
 
 int mdc_create(struct obd_export *exp, struct md_op_data *op_data,
-	       const void *data, int datalen, int mode, __u32 uid, __u32 gid,
-	       cfs_cap_t cap_effective, __u64 rdev,
-	       struct ptlrpc_request **request)
+	       const void *data, size_t datalen, umode_t mode,
+	       uid_t uid, gid_t gid, cfs_cap_t cap_effective,
+	       __u64 rdev, struct ptlrpc_request **request)
 {
 	struct ptlrpc_request *req;
 	int level, rc;
@@ -214,11 +213,9 @@ int mdc_create(struct obd_export *exp, struct md_op_data *op_data,
 		 * mdc_fid_alloc() may return errno 1 in case of switch to new
 		 * sequence, handle this.
 		 */
-		rc = mdc_fid_alloc(exp, &op_data->op_fid2, op_data);
-		if (rc < 0) {
-			CERROR("Can't alloc new fid, rc %d\n", rc);
+		rc = mdc_fid_alloc(NULL, exp, &op_data->op_fid2, op_data);
+		if (rc < 0)
 			return rc;
-		}
 	}
 
 rebuild:
@@ -307,14 +304,12 @@ int mdc_unlink(struct obd_export *exp, struct md_op_data *op_data,
 	LASSERT(!req);
 
 	if ((op_data->op_flags & MF_MDC_CANCEL_FID1) &&
-	    (fid_is_sane(&op_data->op_fid1)) &&
-	    !OBD_FAIL_CHECK(OBD_FAIL_LDLM_BL_CALLBACK_NET))
+	    (fid_is_sane(&op_data->op_fid1)))
 		count = mdc_resource_get_unused(exp, &op_data->op_fid1,
 						&cancels, LCK_EX,
 						MDS_INODELOCK_UPDATE);
 	if ((op_data->op_flags & MF_MDC_CANCEL_FID3) &&
-	    (fid_is_sane(&op_data->op_fid3)) &&
-	    !OBD_FAIL_CHECK(OBD_FAIL_LDLM_BL_CALLBACK_NET))
+	    (fid_is_sane(&op_data->op_fid3)))
 		count += mdc_resource_get_unused(exp, &op_data->op_fid3,
 						 &cancels, LCK_EX,
 						 MDS_INODELOCK_FULL);
@@ -394,7 +389,7 @@ int mdc_link(struct obd_export *exp, struct md_op_data *op_data,
 }
 
 int mdc_rename(struct obd_export *exp, struct md_op_data *op_data,
-	       const char *old, int oldlen, const char *new, int newlen,
+	       const char *old, size_t oldlen, const char *new, size_t newlen,
 	       struct ptlrpc_request **request)
 {
 	LIST_HEAD(cancels);
@@ -431,7 +426,8 @@ int mdc_rename(struct obd_export *exp, struct md_op_data *op_data,
 	}
 
 	req_capsule_set_size(&req->rq_pill, &RMF_NAME, RCL_CLIENT, oldlen + 1);
-	req_capsule_set_size(&req->rq_pill, &RMF_SYMTGT, RCL_CLIENT, newlen+1);
+	req_capsule_set_size(&req->rq_pill, &RMF_SYMTGT, RCL_CLIENT,
+			     newlen + 1);
 
 	rc = mdc_prep_elc_req(exp, req, MDS_REINT, &cancels, count);
 	if (rc) {
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_request.c b/drivers/staging/lustre/lustre/mdc/mdc_request.c
index 542801f04b0d..f56ea643f9bf 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_request.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_request.c
@@ -39,7 +39,9 @@
 # include <linux/utsname.h>
 
 #include "../include/lustre_acl.h"
+#include "../include/lustre/lustre_ioctl.h"
 #include "../include/obd_class.h"
+#include "../include/lustre_lmv.h"
 #include "../include/lustre_fid.h"
 #include "../include/lprocfs_status.h"
 #include "../include/lustre_param.h"
@@ -57,16 +59,16 @@ static inline int mdc_queue_wait(struct ptlrpc_request *req)
 	struct client_obd *cli = &req->rq_import->imp_obd->u.cli;
 	int rc;
 
-	/* mdc_enter_request() ensures that this client has no more
+	/* obd_get_request_slot() ensures that this client has no more
 	 * than cl_max_rpcs_in_flight RPCs simultaneously inf light
 	 * against an MDT.
 	 */
-	rc = mdc_enter_request(cli);
+	rc = obd_get_request_slot(cli);
 	if (rc != 0)
 		return rc;
 
 	rc = ptlrpc_queue_wait(req);
-	mdc_exit_request(cli);
+	obd_put_request_slot(cli);
 
 	return rc;
 }
@@ -98,7 +100,7 @@ static int mdc_getstatus(struct obd_export *exp, struct lu_fid *rootfid)
 		goto out;
 	}
 
-	*rootfid = body->fid1;
+	*rootfid = body->mbo_fid1;
 	CDEBUG(D_NET,
 	       "root fid="DFID", last_committed=%llu\n",
 	       PFID(rootfid),
@@ -136,12 +138,12 @@ static int mdc_getattr_common(struct obd_export *exp,
 	if (!body)
 		return -EPROTO;
 
-	CDEBUG(D_NET, "mode: %o\n", body->mode);
+	CDEBUG(D_NET, "mode: %o\n", body->mbo_mode);
 
 	mdc_update_max_ea_from_body(exp, body);
-	if (body->eadatasize != 0) {
+	if (body->mbo_eadatasize != 0) {
 		eadata = req_capsule_server_sized_get(pill, &RMF_MDT_MD,
-						      body->eadatasize);
+						      body->mbo_eadatasize);
 		if (!eadata)
 			return -EPROTO;
 	}
@@ -230,32 +232,6 @@ static int mdc_getattr_name(struct obd_export *exp, struct md_op_data *op_data,
 	return rc;
 }
 
-static int mdc_is_subdir(struct obd_export *exp,
-			 const struct lu_fid *pfid,
-			 const struct lu_fid *cfid,
-			 struct ptlrpc_request **request)
-{
-	struct ptlrpc_request  *req;
-	int		     rc;
-
-	*request = NULL;
-	req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp),
-					&RQF_MDS_IS_SUBDIR, LUSTRE_MDS_VERSION,
-					MDS_IS_SUBDIR);
-	if (!req)
-		return -ENOMEM;
-
-	mdc_is_subdir_pack(req, pfid, cfid, 0);
-	ptlrpc_request_set_replen(req);
-
-	rc = ptlrpc_queue_wait(req);
-	if (rc && rc != -EREMOTE)
-		ptlrpc_req_finished(req);
-	else
-		*request = req;
-	return rc;
-}
-
 static int mdc_xattr_common(struct obd_export *exp,
 			    const struct req_format *fmt,
 			    const struct lu_fid *fid,
@@ -397,15 +373,15 @@ static int mdc_unpack_acl(struct ptlrpc_request *req, struct lustre_md *md)
 	void		   *buf;
 	int		     rc;
 
-	if (!body->aclsize)
+	if (!body->mbo_aclsize)
 		return 0;
 
-	buf = req_capsule_server_sized_get(pill, &RMF_ACL, body->aclsize);
+	buf = req_capsule_server_sized_get(pill, &RMF_ACL, body->mbo_aclsize);
 
 	if (!buf)
 		return -EPROTO;
 
-	acl = posix_acl_from_xattr(&init_user_ns, buf, body->aclsize);
+	acl = posix_acl_from_xattr(&init_user_ns, buf, body->mbo_aclsize);
 	if (!acl)
 		return 0;
 
@@ -443,24 +419,24 @@ static int mdc_get_lustre_md(struct obd_export *exp,
 
 	md->body = req_capsule_server_get(pill, &RMF_MDT_BODY);
 
-	if (md->body->valid & OBD_MD_FLEASIZE) {
+	if (md->body->mbo_valid & OBD_MD_FLEASIZE) {
 		int lmmsize;
 		struct lov_mds_md *lmm;
 
-		if (!S_ISREG(md->body->mode)) {
+		if (!S_ISREG(md->body->mbo_mode)) {
 			CDEBUG(D_INFO,
 			       "OBD_MD_FLEASIZE set, should be a regular file, but is not\n");
 			rc = -EPROTO;
 			goto out;
 		}
 
-		if (md->body->eadatasize == 0) {
+		if (md->body->mbo_eadatasize == 0) {
 			CDEBUG(D_INFO,
 			       "OBD_MD_FLEASIZE set, but eadatasize 0\n");
 			rc = -EPROTO;
 			goto out;
 		}
-		lmmsize = md->body->eadatasize;
+		lmmsize = md->body->mbo_eadatasize;
 		lmm = req_capsule_server_sized_get(pill, &RMF_MDT_MD, lmmsize);
 		if (!lmm) {
 			rc = -EPROTO;
@@ -471,7 +447,7 @@ static int mdc_get_lustre_md(struct obd_export *exp,
 		if (rc < 0)
 			goto out;
 
-		if (rc < sizeof(*md->lsm)) {
+		if (rc < (typeof(rc))sizeof(*md->lsm)) {
 			CDEBUG(D_INFO,
 			       "lsm size too small: rc < sizeof (*md->lsm) (%d < %d)\n",
 			       rc, (int)sizeof(*md->lsm));
@@ -479,24 +455,24 @@ static int mdc_get_lustre_md(struct obd_export *exp,
 			goto out;
 		}
 
-	} else if (md->body->valid & OBD_MD_FLDIREA) {
+	} else if (md->body->mbo_valid & OBD_MD_FLDIREA) {
 		int lmvsize;
 		struct lov_mds_md *lmv;
 
-		if (!S_ISDIR(md->body->mode)) {
+		if (!S_ISDIR(md->body->mbo_mode)) {
 			CDEBUG(D_INFO,
 			       "OBD_MD_FLDIREA set, should be a directory, but is not\n");
 			rc = -EPROTO;
 			goto out;
 		}
 
-		if (md->body->eadatasize == 0) {
+		if (md->body->mbo_eadatasize == 0) {
 			CDEBUG(D_INFO,
 			       "OBD_MD_FLDIREA is set, but eadatasize 0\n");
 			return -EPROTO;
 		}
-		if (md->body->valid & OBD_MD_MEA) {
-			lmvsize = md->body->eadatasize;
+		if (md->body->mbo_valid & OBD_MD_MEA) {
+			lmvsize = md->body->mbo_eadatasize;
 			lmv = req_capsule_server_sized_get(pill, &RMF_MDT_MD,
 							   lmvsize);
 			if (!lmv) {
@@ -504,15 +480,15 @@ static int mdc_get_lustre_md(struct obd_export *exp,
 				goto out;
 			}
 
-			rc = obd_unpackmd(md_exp, (void *)&md->mea, lmv,
+			rc = obd_unpackmd(md_exp, (void *)&md->lmv, lmv,
 					  lmvsize);
 			if (rc < 0)
 				goto out;
 
-			if (rc < sizeof(*md->mea)) {
+			if (rc < (typeof(rc))sizeof(*md->lmv)) {
 				CDEBUG(D_INFO,
-				       "size too small: rc < sizeof(*md->mea) (%d < %d)\n",
-					rc, (int)sizeof(*md->mea));
+				       "size too small: rc < sizeof(*md->lmv) (%d < %d)\n",
+					rc, (int)sizeof(*md->lmv));
 				rc = -EPROTO;
 				goto out;
 			}
@@ -520,12 +496,12 @@ static int mdc_get_lustre_md(struct obd_export *exp,
 	}
 	rc = 0;
 
-	if (md->body->valid & OBD_MD_FLACL) {
+	if (md->body->mbo_valid & OBD_MD_FLACL) {
 		/* for ACL, it's possible that FLACL is set but aclsize is zero.
 		 * only when aclsize != 0 there's an actual segment for ACL
 		 * in reply buffer.
 		 */
-		if (md->body->aclsize) {
+		if (md->body->mbo_aclsize) {
 			rc = mdc_unpack_acl(req, md);
 			if (rc)
 				goto out;
@@ -580,9 +556,9 @@ void mdc_replay_open(struct ptlrpc_request *req)
 
 		file_fh = &och->och_fh;
 		CDEBUG(D_HA, "updating handle from %#llx to %#llx\n",
-		       file_fh->cookie, body->handle.cookie);
+		       file_fh->cookie, body->mbo_handle.cookie);
 		old = *file_fh;
-		*file_fh = body->handle;
+		*file_fh = body->mbo_handle;
 	}
 	close_req = mod->mod_close_req;
 	if (close_req) {
@@ -597,7 +573,7 @@ void mdc_replay_open(struct ptlrpc_request *req)
 		if (och)
 			LASSERT(!memcmp(&old, &epoch->handle, sizeof(old)));
 		DEBUG_REQ(D_HA, close_req, "updating close body with new fh");
-		epoch->handle = body->handle;
+		epoch->handle = body->mbo_handle;
 	}
 }
 
@@ -679,11 +655,11 @@ int mdc_set_open_replay_data(struct obd_export *exp,
 		spin_unlock(&open_req->rq_lock);
 	}
 
-	rec->cr_fid2 = body->fid1;
-	rec->cr_ioepoch = body->ioepoch;
-	rec->cr_old_handle.cookie = body->handle.cookie;
+	rec->cr_fid2 = body->mbo_fid1;
+	rec->cr_ioepoch = body->mbo_ioepoch;
+	rec->cr_old_handle.cookie = body->mbo_handle.cookie;
 	open_req->rq_replay_cb = mdc_replay_open;
-	if (!fid_is_sane(&body->fid1)) {
+	if (!fid_is_sane(&body->mbo_fid1)) {
 		DEBUG_REQ(D_ERROR, open_req,
 			  "Saving replay request with insane fid");
 		LBUG();
@@ -701,9 +677,15 @@ static void mdc_free_open(struct md_open_data *mod)
 	    imp_connect_disp_stripe(mod->mod_open_req->rq_import))
 		committed = 1;
 
-	LASSERT(mod->mod_open_req->rq_replay == 0);
-
-	DEBUG_REQ(D_RPCTRACE, mod->mod_open_req, "free open request\n");
+	/*
+	 * No reason to asssert here if the open request has
+	 * rq_replay == 1. It means that mdc_close failed, and
+	 * close request wasn`t sent. It is not fatal to client.
+	 * The worst thing is eviction if the client gets open lock
+	 */
+	DEBUG_REQ(D_RPCTRACE, mod->mod_open_req,
+		  "free open request rq_replay = %d\n",
+		   mod->mod_open_req->rq_replay);
 
 	ptlrpc_request_committed(mod->mod_open_req, committed);
 	if (mod->mod_close_req)
@@ -744,7 +726,7 @@ static void mdc_close_handle_reply(struct ptlrpc_request *req,
 		epoch = req_capsule_client_get(&req->rq_pill, &RMF_MDT_EPOCH);
 
 		epoch->flags |= MF_SOM_AU;
-		if (repbody->valid & OBD_MD_FLGETATTRLOCK)
+		if (repbody->mbo_valid & OBD_MD_FLGETATTRLOCK)
 			op_data->op_flags |= MF_GETATTR_LOCK;
 	}
 }
@@ -763,7 +745,7 @@ static int mdc_close(struct obd_export *exp, struct md_op_data *op_data,
 		req_fmt = &RQF_MDS_RELEASE_CLOSE;
 
 		/* allocate a FID for volatile file */
-		rc = mdc_fid_alloc(exp, &op_data->op_fid2, op_data);
+		rc = mdc_fid_alloc(NULL, exp, &op_data->op_fid2, op_data);
 		if (rc < 0) {
 			CERROR("%s: "DFID" failed to allocate FID: %d\n",
 			       obd->obd_name, PFID(&op_data->op_fid1), rc);
@@ -773,22 +755,10 @@ static int mdc_close(struct obd_export *exp, struct md_op_data *op_data,
 	}
 
 	*request = NULL;
-	req = ptlrpc_request_alloc(class_exp2cliimp(exp), req_fmt);
-	if (!req)
-		return -ENOMEM;
-
-	rc = ptlrpc_request_pack(req, LUSTRE_MDS_VERSION, MDS_CLOSE);
-	if (rc) {
-		ptlrpc_request_free(req);
-		return rc;
-	}
-
-	/* To avoid a livelock (bug 7034), we need to send CLOSE RPCs to a
-	 * portal whose threads are not taking any DLM locks and are therefore
-	 * always progressing
-	 */
-	req->rq_request_portal = MDS_READPAGE_PORTAL;
-	ptlrpc_at_set_req_timeout(req);
+	if (OBD_FAIL_CHECK(OBD_FAIL_MDC_CLOSE))
+		req = NULL;
+	else
+		req = ptlrpc_request_alloc(class_exp2cliimp(exp), req_fmt);
 
 	/* Ensure that this close's handle is fixed up during replay. */
 	if (likely(mod)) {
@@ -809,6 +779,29 @@ static int mdc_close(struct obd_export *exp, struct md_op_data *op_data,
 		 CDEBUG(D_HA,
 			"couldn't find open req; expecting close error\n");
 	}
+	if (!req) {
+		/*
+		 * TODO: repeat close after errors
+		 */
+		CWARN("%s: close of FID "DFID" failed, file reference will be dropped when this client unmounts or is evicted\n",
+		      obd->obd_name, PFID(&op_data->op_fid1));
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	rc = ptlrpc_request_pack(req, LUSTRE_MDS_VERSION, MDS_CLOSE);
+	if (rc) {
+		ptlrpc_request_free(req);
+		goto out;
+	}
+
+	/*
+	 * To avoid a livelock (bug 7034), we need to send CLOSE RPCs to a
+	 * portal whose threads are not taking any DLM locks and are therefore
+	 * always progressing
+	 */
+	req->rq_request_portal = MDS_READPAGE_PORTAL;
+	ptlrpc_at_set_req_timeout(req);
 
 	mdc_close_pack(req, op_data);
 
@@ -854,6 +847,7 @@ static int mdc_close(struct obd_export *exp, struct md_op_data *op_data,
 		}
 	}
 
+out:
 	if (mod) {
 		if (rc != 0)
 			mod->mod_close_req = NULL;
@@ -936,16 +930,17 @@ static int mdc_done_writing(struct obd_export *exp, struct md_op_data *op_data,
 	return rc;
 }
 
-static int mdc_readpage(struct obd_export *exp, struct md_op_data *op_data,
-			struct page **pages, struct ptlrpc_request **request)
+static int mdc_getpage(struct obd_export *exp, const struct lu_fid *fid,
+		       u64 offset, struct page **pages, int npages,
+		       struct ptlrpc_request **request)
 {
-	struct ptlrpc_request   *req;
 	struct ptlrpc_bulk_desc *desc;
-	int		      i;
-	wait_queue_head_t	      waitq;
-	int		      resends = 0;
-	struct l_wait_info       lwi;
-	int		      rc;
+	struct ptlrpc_request *req;
+	wait_queue_head_t waitq;
+	struct l_wait_info lwi;
+	int resends = 0;
+	int rc;
+	int i;
 
 	*request = NULL;
 	init_waitqueue_head(&waitq);
@@ -964,7 +959,7 @@ restart_bulk:
 	req->rq_request_portal = MDS_READPAGE_PORTAL;
 	ptlrpc_at_set_req_timeout(req);
 
-	desc = ptlrpc_prep_bulk_imp(req, op_data->op_npages, 1, BULK_PUT_SINK,
+	desc = ptlrpc_prep_bulk_imp(req, npages, 1, BULK_PUT_SINK,
 				    MDS_BULK_PORTAL);
 	if (!desc) {
 		ptlrpc_request_free(req);
@@ -972,12 +967,10 @@ restart_bulk:
 	}
 
 	/* NB req now owns desc and will free it when it gets freed */
-	for (i = 0; i < op_data->op_npages; i++)
+	for (i = 0; i < npages; i++)
 		ptlrpc_prep_bulk_page_pin(desc, pages[i], 0, PAGE_SIZE);
 
-	mdc_readdir_pack(req, op_data->op_offset,
-			 PAGE_SIZE * op_data->op_npages,
-			 &op_data->op_fid1);
+	mdc_readdir_pack(req, offset, PAGE_SIZE * npages, fid);
 
 	ptlrpc_request_set_replen(req);
 	rc = ptlrpc_queue_wait(req);
@@ -988,11 +981,12 @@ restart_bulk:
 
 		resends++;
 		if (!client_should_resend(resends, &exp->exp_obd->u.cli)) {
-			CERROR("too many resend retries, returning error\n");
+			CERROR("%s: too many resend retries: rc = %d\n",
+			       exp->exp_obd->obd_name, -EIO);
 			return -EIO;
 		}
-		lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(resends),
-				       NULL, NULL, NULL);
+		lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(resends), NULL, NULL,
+				       NULL);
 		l_wait_event(waitq, 0, &lwi);
 
 		goto restart_bulk;
@@ -1006,9 +1000,9 @@ restart_bulk:
 	}
 
 	if (req->rq_bulk->bd_nob_transferred & ~LU_PAGE_MASK) {
-		CERROR("Unexpected # bytes transferred: %d (%ld expected)\n",
-		       req->rq_bulk->bd_nob_transferred,
-		       PAGE_SIZE * op_data->op_npages);
+		CERROR("%s: unexpected bytes transferred: %d (%ld expected)\n",
+		       exp->exp_obd->obd_name, req->rq_bulk->bd_nob_transferred,
+		       PAGE_SIZE * npages);
 		ptlrpc_req_finished(req);
 		return -EPROTO;
 	}
@@ -1017,6 +1011,453 @@ restart_bulk:
 	return 0;
 }
 
+static void mdc_release_page(struct page *page, int remove)
+{
+	if (remove) {
+		lock_page(page);
+		if (likely(page->mapping))
+			truncate_complete_page(page->mapping, page);
+		unlock_page(page);
+	}
+	put_page(page);
+}
+
+static struct page *mdc_page_locate(struct address_space *mapping, __u64 *hash,
+				    __u64 *start, __u64 *end, int hash64)
+{
+	/*
+	 * Complement of hash is used as an index so that
+	 * radix_tree_gang_lookup() can be used to find a page with starting
+	 * hash _smaller_ than one we are looking for.
+	 */
+	unsigned long offset = hash_x_index(*hash, hash64);
+	struct page *page;
+	int found;
+
+	spin_lock_irq(&mapping->tree_lock);
+	found = radix_tree_gang_lookup(&mapping->page_tree,
+				       (void **)&page, offset, 1);
+	if (found > 0 && !radix_tree_exceptional_entry(page)) {
+		struct lu_dirpage *dp;
+
+		get_page(page);
+		spin_unlock_irq(&mapping->tree_lock);
+		/*
+		 * In contrast to find_lock_page() we are sure that directory
+		 * page cannot be truncated (while DLM lock is held) and,
+		 * hence, can avoid restart.
+		 *
+		 * In fact, page cannot be locked here at all, because
+		 * mdc_read_page_remote does synchronous io.
+		 */
+		wait_on_page_locked(page);
+		if (PageUptodate(page)) {
+			dp = kmap(page);
+			if (BITS_PER_LONG == 32 && hash64) {
+				*start = le64_to_cpu(dp->ldp_hash_start) >> 32;
+				*end   = le64_to_cpu(dp->ldp_hash_end) >> 32;
+				*hash  = *hash >> 32;
+			} else {
+				*start = le64_to_cpu(dp->ldp_hash_start);
+				*end   = le64_to_cpu(dp->ldp_hash_end);
+			}
+			if (unlikely(*start == 1 && *hash == 0))
+				*hash = *start;
+			else
+				LASSERTF(*start <= *hash, "start = %#llx,end = %#llx,hash = %#llx\n",
+					 *start, *end, *hash);
+			CDEBUG(D_VFSTRACE, "offset %lx [%#llx %#llx], hash %#llx\n",
+			       offset, *start, *end, *hash);
+			if (*hash > *end) {
+				kunmap(page);
+				mdc_release_page(page, 0);
+				page = NULL;
+			} else if (*end != *start && *hash == *end) {
+				/*
+				 * upon hash collision, remove this page,
+				 * otherwise put page reference, and
+				 * mdc_read_page_remote() will issue RPC to
+				 * fetch the page we want.
+				 */
+				kunmap(page);
+				mdc_release_page(page,
+						 le32_to_cpu(dp->ldp_flags) & LDF_COLLIDE);
+				page = NULL;
+			}
+		} else {
+			put_page(page);
+			page = ERR_PTR(-EIO);
+		}
+	} else {
+		spin_unlock_irq(&mapping->tree_lock);
+		page = NULL;
+	}
+	return page;
+}
+
+/*
+ * Adjust a set of pages, each page containing an array of lu_dirpages,
+ * so that each page can be used as a single logical lu_dirpage.
+ *
+ * A lu_dirpage is laid out as follows, where s = ldp_hash_start,
+ * e = ldp_hash_end, f = ldp_flags, p = padding, and each "ent" is a
+ * struct lu_dirent.  It has size up to LU_PAGE_SIZE. The ldp_hash_end
+ * value is used as a cookie to request the next lu_dirpage in a
+ * directory listing that spans multiple pages (two in this example):
+ *   ________
+ *  |        |
+ * .|--------v-------   -----.
+ * |s|e|f|p|ent|ent| ... |ent|
+ * '--|--------------   -----'   Each PAGE contains a single
+ *    '------.                   lu_dirpage.
+ * .---------v-------   -----.
+ * |s|e|f|p|ent| 0 | ... | 0 |
+ * '-----------------   -----'
+ *
+ * However, on hosts where the native VM page size (PAGE_SIZE) is
+ * larger than LU_PAGE_SIZE, a single host page may contain multiple
+ * lu_dirpages. After reading the lu_dirpages from the MDS, the
+ * ldp_hash_end of the first lu_dirpage refers to the one immediately
+ * after it in the same PAGE (arrows simplified for brevity, but
+ * in general e0==s1, e1==s2, etc.):
+ *
+ * .--------------------   -----.
+ * |s0|e0|f0|p|ent|ent| ... |ent|
+ * |---v----------------   -----|
+ * |s1|e1|f1|p|ent|ent| ... |ent|
+ * |---v----------------   -----|  Here, each PAGE contains
+ *             ...                 multiple lu_dirpages.
+ * |---v----------------   -----|
+ * |s'|e'|f'|p|ent|ent| ... |ent|
+ * '---|----------------   -----'
+ *     v
+ * .----------------------------.
+ * |        next PAGE           |
+ *
+ * This structure is transformed into a single logical lu_dirpage as follows:
+ *
+ * - Replace e0 with e' so the request for the next lu_dirpage gets the page
+ *   labeled 'next PAGE'.
+ *
+ * - Copy the LDF_COLLIDE flag from f' to f0 to correctly reflect whether
+ *   a hash collision with the next page exists.
+ *
+ * - Adjust the lde_reclen of the ending entry of each lu_dirpage to span
+ *   to the first entry of the next lu_dirpage.
+ */
+#if PAGE_SIZE > LU_PAGE_SIZE
+static void mdc_adjust_dirpages(struct page **pages, int cfs_pgs, int lu_pgs)
+{
+	int i;
+
+	for (i = 0; i < cfs_pgs; i++) {
+		struct lu_dirpage *dp = kmap(pages[i]);
+		__u64 hash_end = le64_to_cpu(dp->ldp_hash_end);
+		__u32 flags = le32_to_cpu(dp->ldp_flags);
+		struct lu_dirpage *first = dp;
+		struct lu_dirent *end_dirent = NULL;
+		struct lu_dirent *ent;
+
+		while (--lu_pgs > 0) {
+			ent = lu_dirent_start(dp);
+			for (end_dirent = ent; ent;
+			     end_dirent = ent, ent = lu_dirent_next(ent));
+
+			/* Advance dp to next lu_dirpage. */
+			dp = (struct lu_dirpage *)((char *)dp + LU_PAGE_SIZE);
+
+			/* Check if we've reached the end of the CFS_PAGE. */
+			if (!((unsigned long)dp & ~PAGE_MASK))
+				break;
+
+			/* Save the hash and flags of this lu_dirpage. */
+			hash_end = le64_to_cpu(dp->ldp_hash_end);
+			flags = le32_to_cpu(dp->ldp_flags);
+
+			/* Check if lu_dirpage contains no entries. */
+			if (!end_dirent)
+				break;
+
+			/*
+			 * Enlarge the end entry lde_reclen from 0 to
+			 * first entry of next lu_dirpage.
+			 */
+			LASSERT(!le16_to_cpu(end_dirent->lde_reclen));
+			end_dirent->lde_reclen =
+				cpu_to_le16((char *)(dp->ldp_entries) -
+					    (char *)end_dirent);
+		}
+
+		first->ldp_hash_end = hash_end;
+		first->ldp_flags &= ~cpu_to_le32(LDF_COLLIDE);
+		first->ldp_flags |= flags & cpu_to_le32(LDF_COLLIDE);
+
+		kunmap(pages[i]);
+	}
+	LASSERTF(lu_pgs == 0, "left = %d", lu_pgs);
+}
+#else
+#define mdc_adjust_dirpages(pages, cfs_pgs, lu_pgs) do {} while (0)
+#endif  /* PAGE_SIZE > LU_PAGE_SIZE */
+
+/* parameters for readdir page */
+struct readpage_param {
+	struct md_op_data	*rp_mod;
+	__u64			rp_off;
+	int			rp_hash64;
+	struct obd_export	*rp_exp;
+	struct md_callback	*rp_cb;
+};
+
+/**
+ * Read pages from server.
+ *
+ * Page in MDS_READPAGE RPC is packed in LU_PAGE_SIZE, and each page contains
+ * a header lu_dirpage which describes the start/end hash, and whether this
+ * page is empty (contains no dir entry) or hash collide with next page.
+ * After client receives reply, several pages will be integrated into dir page
+ * in PAGE_SIZE (if PAGE_SIZE greater than LU_PAGE_SIZE), and the
+ * lu_dirpage for this integrated page will be adjusted.
+ **/
+static int mdc_read_page_remote(void *data, struct page *page0)
+{
+	struct readpage_param *rp = data;
+	struct page **page_pool;
+	struct page *page;
+	struct lu_dirpage *dp;
+	int rd_pgs = 0; /* number of pages read actually */
+	int npages;
+	struct md_op_data *op_data = rp->rp_mod;
+	struct ptlrpc_request *req;
+	int max_pages = op_data->op_max_pages;
+	struct inode *inode;
+	struct lu_fid *fid;
+	int i;
+	int rc;
+
+	LASSERT(max_pages > 0 && max_pages <= PTLRPC_MAX_BRW_PAGES);
+	inode = op_data->op_data;
+	fid = &op_data->op_fid1;
+	LASSERT(inode);
+
+	page_pool = kcalloc(max_pages, sizeof(page), GFP_NOFS);
+	if (page_pool) {
+		page_pool[0] = page0;
+	} else {
+		page_pool = &page0;
+		max_pages = 1;
+	}
+
+	for (npages = 1; npages < max_pages; npages++) {
+		page = page_cache_alloc_cold(inode->i_mapping);
+		if (!page)
+			break;
+		page_pool[npages] = page;
+	}
+
+	rc = mdc_getpage(rp->rp_exp, fid, rp->rp_off, page_pool, npages, &req);
+	if (!rc) {
+		int lu_pgs = req->rq_bulk->bd_nob_transferred;
+
+		rd_pgs = (req->rq_bulk->bd_nob_transferred +
+			  PAGE_SIZE - 1) >> PAGE_SHIFT;
+		lu_pgs >>= LU_PAGE_SHIFT;
+		LASSERT(!(req->rq_bulk->bd_nob_transferred & ~LU_PAGE_MASK));
+
+		CDEBUG(D_INODE, "read %d(%d) pages\n", rd_pgs, lu_pgs);
+
+		mdc_adjust_dirpages(page_pool, rd_pgs, lu_pgs);
+
+		SetPageUptodate(page0);
+	}
+
+	unlock_page(page0);
+	ptlrpc_req_finished(req);
+	CDEBUG(D_CACHE, "read %d/%d pages\n", rd_pgs, npages);
+	for (i = 1; i < npages; i++) {
+		unsigned long offset;
+		__u64 hash;
+		int ret;
+
+		page = page_pool[i];
+
+		if (rc < 0 || i >= rd_pgs) {
+			put_page(page);
+			continue;
+		}
+
+		SetPageUptodate(page);
+
+		dp = kmap(page);
+		hash = le64_to_cpu(dp->ldp_hash_start);
+		kunmap(page);
+
+		offset = hash_x_index(hash, rp->rp_hash64);
+
+		prefetchw(&page->flags);
+		ret = add_to_page_cache_lru(page, inode->i_mapping, offset,
+					    GFP_KERNEL);
+		if (!ret)
+			unlock_page(page);
+		else
+			CDEBUG(D_VFSTRACE, "page %lu add to page cache failed: rc = %d\n",
+			       offset, ret);
+		put_page(page);
+	}
+
+	if (page_pool != &page0)
+		kfree(page_pool);
+
+	return rc;
+}
+
+/**
+ * Read dir page from cache first, if it can not find it, read it from
+ * server and add into the cache.
+ *
+ * \param[in] exp	MDC export
+ * \param[in] op_data	client MD stack parameters, transferring parameters
+ *			between different layers on client MD stack.
+ * \param[in] cb_op	callback required for ldlm lock enqueue during
+ *			read page
+ * \param[in] hash_offset the hash offset of the page to be read
+ * \param[in] ppage	the page to be read
+ *
+ * retval		= 0 get the page successfully
+ *			errno(<0) get the page failed
+ */
+static int mdc_read_page(struct obd_export *exp, struct md_op_data *op_data,
+			 struct md_callback *cb_op, __u64 hash_offset,
+			 struct page **ppage)
+{
+	struct lookup_intent it = { .it_op = IT_READDIR };
+	struct page *page;
+	struct inode *dir = op_data->op_data;
+	struct address_space *mapping;
+	struct lu_dirpage *dp;
+	__u64 start = 0;
+	__u64 end = 0;
+	struct lustre_handle lockh;
+	struct ptlrpc_request *enq_req = NULL;
+	struct readpage_param rp_param;
+	int rc;
+
+	*ppage = NULL;
+
+	LASSERT(dir);
+	mapping = dir->i_mapping;
+
+	rc = mdc_intent_lock(exp, op_data, &it, &enq_req,
+			     cb_op->md_blocking_ast, 0);
+	if (enq_req)
+		ptlrpc_req_finished(enq_req);
+
+	if (rc < 0) {
+		CERROR("%s: "DFID" lock enqueue fails: rc = %d\n",
+		       exp->exp_obd->obd_name, PFID(&op_data->op_fid1), rc);
+		return rc;
+	}
+
+	rc = 0;
+	lockh.cookie = it.it_lock_handle;
+	mdc_set_lock_data(exp, &lockh, dir, NULL);
+
+	rp_param.rp_off = hash_offset;
+	rp_param.rp_hash64 = op_data->op_cli_flags & CLI_HASH64;
+	page = mdc_page_locate(mapping, &rp_param.rp_off, &start, &end,
+			       rp_param.rp_hash64);
+	if (IS_ERR(page)) {
+		CDEBUG(D_INFO, "%s: dir page locate: " DFID " at %llu: rc %ld\n",
+		       exp->exp_obd->obd_name, PFID(&op_data->op_fid1),
+		       rp_param.rp_off, PTR_ERR(page));
+		rc = PTR_ERR(page);
+		goto out_unlock;
+	} else if (page) {
+		/*
+		 * XXX nikita: not entirely correct handling of a corner case:
+		 * suppose hash chain of entries with hash value HASH crosses
+		 * border between pages P0 and P1. First both P0 and P1 are
+		 * cached, seekdir() is called for some entry from the P0 part
+		 * of the chain. Later P0 goes out of cache. telldir(HASH)
+		 * happens and finds P1, as it starts with matching hash
+		 * value. Remaining entries from P0 part of the chain are
+		 * skipped. (Is that really a bug?)
+		 *
+		 * Possible solutions: 0. don't cache P1 is such case, handle
+		 * it as an "overflow" page. 1. invalidate all pages at
+		 * once. 2. use HASH|1 as an index for P1.
+		 */
+		goto hash_collision;
+	}
+
+	rp_param.rp_exp = exp;
+	rp_param.rp_mod = op_data;
+	page = read_cache_page(mapping,
+			       hash_x_index(rp_param.rp_off,
+					    rp_param.rp_hash64),
+			       mdc_read_page_remote, &rp_param);
+	if (IS_ERR(page)) {
+		CERROR("%s: read cache page: "DFID" at %llu: rc %ld\n",
+		       exp->exp_obd->obd_name, PFID(&op_data->op_fid1),
+		       rp_param.rp_off, PTR_ERR(page));
+		rc = PTR_ERR(page);
+		goto out_unlock;
+	}
+
+	wait_on_page_locked(page);
+	(void)kmap(page);
+	if (!PageUptodate(page)) {
+		CERROR("%s: page not updated: "DFID" at %llu: rc %d\n",
+		       exp->exp_obd->obd_name, PFID(&op_data->op_fid1),
+		       rp_param.rp_off, -5);
+		goto fail;
+	}
+	if (!PageChecked(page))
+		SetPageChecked(page);
+	if (PageError(page)) {
+		CERROR("%s: page error: "DFID" at %llu: rc %d\n",
+		       exp->exp_obd->obd_name, PFID(&op_data->op_fid1),
+		       rp_param.rp_off, -5);
+		goto fail;
+	}
+
+hash_collision:
+	dp = page_address(page);
+	if (BITS_PER_LONG == 32 && rp_param.rp_hash64) {
+		start = le64_to_cpu(dp->ldp_hash_start) >> 32;
+		end = le64_to_cpu(dp->ldp_hash_end) >> 32;
+		rp_param.rp_off = hash_offset >> 32;
+	} else {
+		start = le64_to_cpu(dp->ldp_hash_start);
+		end = le64_to_cpu(dp->ldp_hash_end);
+		rp_param.rp_off = hash_offset;
+	}
+	if (end == start) {
+		LASSERT(start == rp_param.rp_off);
+		CWARN("Page-wide hash collision: %#lx\n", (unsigned long)end);
+#if BITS_PER_LONG == 32
+		CWARN("Real page-wide hash collision at [%llu %llu] with hash %llu\n",
+		      le64_to_cpu(dp->ldp_hash_start),
+		      le64_to_cpu(dp->ldp_hash_end), hash_offset);
+#endif
+		/*
+		 * Fetch whole overflow chain...
+		 *
+		 * XXX not yet.
+		 */
+		goto fail;
+	}
+	*ppage = page;
+out_unlock:
+	ldlm_lock_decref(&lockh, it.it_lock_mode);
+	return rc;
+fail:
+	kunmap(page);
+	mdc_release_page(page, 1);
+	rc = -EIO;
+	goto out_unlock;
+}
+
 static int mdc_statfs(const struct lu_env *env,
 		      struct obd_export *exp, struct obd_statfs *osfs,
 		      __u64 max_age, __u32 flags)
@@ -1401,7 +1842,7 @@ out:
 	return rc;
 }
 
-static struct kuc_hdr *changelog_kuc_hdr(char *buf, int len, int flags)
+static struct kuc_hdr *changelog_kuc_hdr(char *buf, size_t len, u32 flags)
 {
 	struct kuc_hdr *lh = (struct kuc_hdr *)buf;
 
@@ -1415,40 +1856,44 @@ static struct kuc_hdr *changelog_kuc_hdr(char *buf, int len, int flags)
 	return lh;
 }
 
-#define D_CHANGELOG 0
-
 struct changelog_show {
 	__u64		cs_startrec;
-	__u32		cs_flags;
+	enum changelog_send_flag	cs_flags;
 	struct file	*cs_fp;
 	char		*cs_buf;
 	struct obd_device *cs_obd;
 };
 
+static inline char *cs_obd_name(struct changelog_show *cs)
+{
+	return cs->cs_obd->obd_name;
+}
+
 static int changelog_kkuc_cb(const struct lu_env *env, struct llog_handle *llh,
 			     struct llog_rec_hdr *hdr, void *data)
 {
 	struct changelog_show *cs = data;
 	struct llog_changelog_rec *rec = (struct llog_changelog_rec *)hdr;
 	struct kuc_hdr *lh;
-	int len, rc;
+	size_t len;
+	int rc;
 
 	if (rec->cr_hdr.lrh_type != CHANGELOG_REC) {
 		rc = -EINVAL;
 		CERROR("%s: not a changelog rec %x/%d: rc = %d\n",
-		       cs->cs_obd->obd_name, rec->cr_hdr.lrh_type,
+		       cs_obd_name(cs), rec->cr_hdr.lrh_type,
 		       rec->cr.cr_type, rc);
 		return rc;
 	}
 
 	if (rec->cr.cr_index < cs->cs_startrec) {
 		/* Skip entries earlier than what we are interested in */
-		CDEBUG(D_CHANGELOG, "rec=%llu start=%llu\n",
+		CDEBUG(D_HSM, "rec=%llu start=%llu\n",
 		       rec->cr.cr_index, cs->cs_startrec);
 		return 0;
 	}
 
-	CDEBUG(D_CHANGELOG, "%llu %02d%-5s %llu 0x%x t="DFID" p="DFID
+	CDEBUG(D_HSM, "%llu %02d%-5s %llu 0x%x t=" DFID " p=" DFID
 		" %.*s\n", rec->cr.cr_index, rec->cr.cr_type,
 		changelog_type2str(rec->cr.cr_type), rec->cr.cr_time,
 		rec->cr.cr_flags & CLF_FLAGMASK,
@@ -1462,20 +1907,21 @@ static int changelog_kkuc_cb(const struct lu_env *env, struct llog_handle *llh,
 	memcpy(lh + 1, &rec->cr, len - sizeof(*lh));
 
 	rc = libcfs_kkuc_msg_put(cs->cs_fp, lh);
-	CDEBUG(D_CHANGELOG, "kucmsg fp %p len %d rc %d\n", cs->cs_fp, len, rc);
+	CDEBUG(D_HSM, "kucmsg fp %p len %zu rc %d\n", cs->cs_fp, len, rc);
 
 	return rc;
 }
 
 static int mdc_changelog_send_thread(void *csdata)
 {
+	enum llog_flag flags = LLOG_F_IS_CAT;
 	struct changelog_show *cs = csdata;
 	struct llog_ctxt *ctxt = NULL;
 	struct llog_handle *llh = NULL;
 	struct kuc_hdr *kuch;
 	int rc;
 
-	CDEBUG(D_CHANGELOG, "changelog to fp=%p start %llu\n",
+	CDEBUG(D_HSM, "changelog to fp=%p start %llu\n",
 	       cs->cs_fp, cs->cs_startrec);
 
 	cs->cs_buf = kzalloc(KUC_CHANGELOG_MSG_MAXSIZE, GFP_NOFS);
@@ -1494,10 +1940,14 @@ static int mdc_changelog_send_thread(void *csdata)
 		       LLOG_OPEN_EXISTS);
 	if (rc) {
 		CERROR("%s: fail to open changelog catalog: rc = %d\n",
-		       cs->cs_obd->obd_name, rc);
+		       cs_obd_name(cs), rc);
 		goto out;
 	}
-	rc = llog_init_handle(NULL, llh, LLOG_F_IS_CAT, NULL);
+
+	if (cs->cs_flags & CHANGELOG_FLAG_JOBID)
+		flags |= LLOG_F_EXT_JOBID;
+
+	rc = llog_init_handle(NULL, llh, flags, NULL);
 	if (rc) {
 		CERROR("llog_init_handle failed %d\n", rc);
 		goto out;
@@ -1550,12 +2000,12 @@ static int mdc_ioc_changelog_send(struct obd_device *obd,
 	if (IS_ERR(task)) {
 		rc = PTR_ERR(task);
 		CERROR("%s: can't start changelog thread: rc = %d\n",
-		       obd->obd_name, rc);
+		       cs_obd_name(cs), rc);
 		kfree(cs);
 	} else {
 		rc = 0;
-		CDEBUG(D_CHANGELOG, "%s: started changelog thread\n",
-		       obd->obd_name);
+		CDEBUG(D_HSM, "%s: started changelog thread\n",
+		       cs_obd_name(cs));
 	}
 
 	CERROR("Failed to start changelog thread: %d\n", rc);
@@ -1669,9 +2119,11 @@ static int mdc_ioc_swap_layouts(struct obd_export *exp,
 	 * with the request RPC to avoid extra RPC round trips
 	 */
 	count = mdc_resource_get_unused(exp, &op_data->op_fid1, &cancels,
-					LCK_CR, MDS_INODELOCK_LAYOUT);
+					LCK_CR, MDS_INODELOCK_LAYOUT |
+					MDS_INODELOCK_XATTR);
 	count += mdc_resource_get_unused(exp, &op_data->op_fid2, &cancels,
-					 LCK_CR, MDS_INODELOCK_LAYOUT);
+					 LCK_CR, MDS_INODELOCK_LAYOUT |
+					 MDS_INODELOCK_XATTR);
 
 	req = ptlrpc_request_alloc(class_exp2cliimp(exp),
 				   &RQF_MDS_SWAP_LAYOUTS);
@@ -1917,7 +2369,7 @@ static void lustre_swab_hai(struct hsm_action_item *h)
 static void lustre_swab_hal(struct hsm_action_list *h)
 {
 	struct hsm_action_item	*hai;
-	int			 i;
+	u32 i;
 
 	__swab32s(&h->hal_version);
 	__swab32s(&h->hal_count);
@@ -1966,14 +2418,14 @@ static int mdc_ioc_hsm_ct_start(struct obd_export *exp,
  * @param val KUC message (kuc_hdr + hsm_action_list)
  * @param len total length of message
  */
-static int mdc_hsm_copytool_send(int len, void *val)
+static int mdc_hsm_copytool_send(size_t len, void *val)
 {
 	struct kuc_hdr		*lh = (struct kuc_hdr *)val;
 	struct hsm_action_list	*hal = (struct hsm_action_list *)(lh + 1);
 
 	if (len < sizeof(*lh) + sizeof(*hal)) {
-		CERROR("Short HSM message %d < %d\n", len,
-		       (int)(sizeof(*lh) + sizeof(*hal)));
+		CERROR("Short HSM message %zu < %zu\n", len,
+		       sizeof(*lh) + sizeof(*hal));
 		return -EPROTO;
 	}
 	if (lh->kuc_magic == __swab16(KUC_MAGIC)) {
@@ -2044,9 +2496,8 @@ static int mdc_set_info_async(const struct lu_env *env,
 		}
 		spin_unlock(&imp->imp_lock);
 
-		rc = do_set_info_async(imp, MDS_SET_INFO, LUSTRE_MDS_VERSION,
-				       keylen, key, vallen, val, set);
-		return rc;
+		return do_set_info_async(imp, MDS_SET_INFO, LUSTRE_MDS_VERSION,
+					 keylen, key, vallen, val, set);
 	}
 	if (KEY_IS(KEY_SPTLRPC_CONF)) {
 		sptlrpc_conf_client_adapt(exp->exp_obd);
@@ -2065,6 +2516,12 @@ static int mdc_set_info_async(const struct lu_env *env,
 		rc = mdc_hsm_copytool_send(vallen, val);
 		return rc;
 	}
+	if (KEY_IS(KEY_DEFAULT_EASIZE)) {
+		u32 *default_easize = val;
+
+		exp->exp_obd->u.cli.cl_default_mds_easize = *default_easize;
+		return 0;
+	}
 
 	CERROR("Unknown key %s\n", (char *)key);
 	return -EINVAL;
@@ -2077,18 +2534,18 @@ static int mdc_get_info(const struct lu_env *env, struct obd_export *exp,
 	int rc = -EINVAL;
 
 	if (KEY_IS(KEY_MAX_EASIZE)) {
-		int mdsize, *max_easize;
+		u32 mdsize, *max_easize;
 
 		if (*vallen != sizeof(int))
 			return -EINVAL;
-		mdsize = *(int *)val;
+		mdsize = *(u32 *)val;
 		if (mdsize > exp->exp_obd->u.cli.cl_max_mds_easize)
 			exp->exp_obd->u.cli.cl_max_mds_easize = mdsize;
 		max_easize = val;
 		*max_easize = exp->exp_obd->u.cli.cl_max_mds_easize;
 		return 0;
 	} else if (KEY_IS(KEY_DEFAULT_EASIZE)) {
-		int *default_easize;
+		u32 *default_easize;
 
 		if (*vallen != sizeof(int))
 			return -EINVAL;
@@ -2105,7 +2562,7 @@ static int mdc_get_info(const struct lu_env *env, struct obd_export *exp,
 		*data = imp->imp_connect_data;
 		return 0;
 	} else if (KEY_IS(KEY_TGT_COUNT)) {
-		*((int *)val) = 1;
+		*((u32 *)val) = 1;
 		return 0;
 	}
 
@@ -2199,13 +2656,13 @@ static int mdc_import_event(struct obd_device *obd, struct obd_import *imp,
 	return rc;
 }
 
-int mdc_fid_alloc(struct obd_export *exp, struct lu_fid *fid,
-		  struct md_op_data *op_data)
+int mdc_fid_alloc(const struct lu_env *env, struct obd_export *exp,
+		  struct lu_fid *fid, struct md_op_data *op_data)
 {
 	struct client_obd *cli = &exp->exp_obd->u.cli;
 	struct lu_client_seq *seq = cli->cl_seq;
 
-	return seq_client_alloc_fid(NULL, seq, fid);
+	return seq_client_alloc_fid(env, seq, fid);
 }
 
 static struct obd_uuid *mdc_get_uuid(struct obd_export *exp)
@@ -2333,8 +2790,8 @@ err_rpc_lock:
  * a large number of stripes is possible.  If a larger reply buffer is
  * required it will be reallocated in the ptlrpc layer due to overflow.
  */
-static int mdc_init_ea_size(struct obd_export *exp, int easize,
-			    int def_easize, int cookiesize, int def_cookiesize)
+static int mdc_init_ea_size(struct obd_export *exp, u32 easize, u32 def_easize,
+			    u32 cookiesize, u32 def_cookiesize)
 {
 	struct obd_device *obd = exp->exp_obd;
 	struct client_obd *cli = &obd->u.cli;
@@ -2430,7 +2887,6 @@ static struct obd_ops mdc_obd_ops = {
 static struct md_ops mdc_md_ops = {
 	.getstatus		= mdc_getstatus,
 	.null_inode		= mdc_null_inode,
-	.find_cbdata		= mdc_find_cbdata,
 	.close			= mdc_close,
 	.create			= mdc_create,
 	.done_writing		= mdc_done_writing,
@@ -2439,13 +2895,12 @@ static struct md_ops mdc_md_ops = {
 	.getattr_name		= mdc_getattr_name,
 	.intent_lock		= mdc_intent_lock,
 	.link			= mdc_link,
-	.is_subdir		= mdc_is_subdir,
 	.rename			= mdc_rename,
 	.setattr		= mdc_setattr,
 	.setxattr		= mdc_setxattr,
 	.getxattr		= mdc_getxattr,
 	.sync			= mdc_sync,
-	.readpage		= mdc_readpage,
+	.read_page		= mdc_read_page,
 	.unlink			= mdc_unlink,
 	.cancel_unused		= mdc_cancel_unused,
 	.init_ea_size		= mdc_init_ea_size,
diff --git a/drivers/staging/lustre/lustre/mgc/mgc_request.c b/drivers/staging/lustre/lustre/mgc/mgc_request.c
index 9d0bd4745865..23374cae5133 100644
--- a/drivers/staging/lustre/lustre/mgc/mgc_request.c
+++ b/drivers/staging/lustre/lustre/mgc/mgc_request.c
@@ -549,8 +549,9 @@ static int mgc_requeue_thread(void *data)
 		 * caused the lock revocation to finish its setup, plus some
 		 * random so everyone doesn't try to reconnect at once.
 		 */
-		to = MGC_TIMEOUT_MIN_SECONDS * HZ;
-		to += rand * HZ / 100; /* rand is centi-seconds */
+		to = msecs_to_jiffies(MGC_TIMEOUT_MIN_SECONDS * MSEC_PER_SEC);
+		/* rand is centi-seconds */
+		to += msecs_to_jiffies(rand * MSEC_PER_SEC / 100);
 		lwi = LWI_TIMEOUT(to, NULL, NULL);
 		l_wait_event(rq_waitq, rq_state & (RQ_STOP | RQ_PRECLEANUP),
 			     &lwi);
@@ -1158,7 +1159,7 @@ static int mgc_apply_recover_logs(struct obd_device *mgc,
 
 	while (datalen > 0) {
 		int   entry_len = sizeof(*entry);
-		int   is_ost;
+		int is_ost, i;
 		struct obd_device *obd;
 		char *obdname;
 		char *cname;
@@ -1264,11 +1265,17 @@ static int mgc_apply_recover_logs(struct obd_device *mgc,
 			continue;
 		}
 
-		/* TODO: iterate all nids to find one */
+		/* iterate all nids to find one */
 		/* find uuid by nid */
-		rc = client_import_find_conn(obd->u.cli.cl_import,
-					     entry->u.nids[0],
-					     (struct obd_uuid *)uuid);
+		rc = -ENOENT;
+		for (i = 0; i < entry->mne_nid_count; i++) {
+			rc = client_import_find_conn(obd->u.cli.cl_import,
+						     entry->u.nids[0],
+						     (struct obd_uuid *)uuid);
+			if (!rc)
+				break;
+		}
+
 		up_read(&obd->u.cli.cl_sem);
 		if (rc < 0) {
 			CERROR("mgc: cannot find uuid by nid %s\n",
@@ -1428,14 +1435,12 @@ again:
 	}
 
 	mne_swab = !!ptlrpc_rep_need_swab(req);
-#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 2, 50, 0)
+#if OBD_OCD_VERSION(3, 0, 53, 0) > LUSTRE_VERSION_CODE
 	/* This import flag means the server did an extra swab of IR MNE
 	 * records (fixed in LU-1252), reverse it here if needed. LU-1644
 	 */
 	if (unlikely(req->rq_import->imp_need_mne_swab))
 		mne_swab = !mne_swab;
-#else
-#warning "LU-1644: Remove old OBD_CONNECT_MNE_SWAB fixup and imp_need_mne_swab"
 #endif
 
 	for (i = 0; i < nrpages && ealen > 0; i++) {
@@ -1740,8 +1745,6 @@ static struct obd_ops mgc_obd_ops = {
 	.del_conn       = client_import_del_conn,
 	.connect        = client_connect_import,
 	.disconnect     = client_disconnect_export,
-	/* .enqueue     = mgc_enqueue, */
-	/* .iocontrol   = mgc_iocontrol, */
 	.set_info_async = mgc_set_info_async,
 	.get_info       = mgc_get_info,
 	.import_event   = mgc_import_event,
diff --git a/drivers/staging/lustre/lustre/obdclass/Makefile b/drivers/staging/lustre/lustre/obdclass/Makefile
index df7e47f35a66..b42e109b30e0 100644
--- a/drivers/staging/lustre/lustre/obdclass/Makefile
+++ b/drivers/staging/lustre/lustre/obdclass/Makefile
@@ -3,6 +3,6 @@ obj-$(CONFIG_LUSTRE_FS) += obdclass.o
 obdclass-y := linux/linux-module.o linux/linux-obdo.o linux/linux-sysctl.o \
 	      llog.o llog_cat.o llog_obd.o llog_swab.o class_obd.o debug.o \
 	      genops.o uuid.o lprocfs_status.o lprocfs_counters.o \
-	      lustre_handles.o lustre_peer.o statfs_pack.o \
+	      lustre_handles.o lustre_peer.o statfs_pack.o linkea.o \
 	      obdo.o obd_config.o obd_mount.o lu_object.o lu_ref.o \
 	      cl_object.o cl_page.o cl_lock.o cl_io.o kernelcomm.o
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_io.c b/drivers/staging/lustre/lustre/obdclass/cl_io.c
index e72f1fc00a13..bc4b7b6b9a20 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_io.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_io.c
@@ -73,7 +73,6 @@ int cl_io_is_going(const struct lu_env *env)
 {
 	return cl_env_info(env)->clt_current_io != NULL;
 }
-EXPORT_SYMBOL(cl_io_is_going);
 
 /**
  * cl_io invariant that holds at all times when exported cl_io_*() functions
@@ -859,9 +858,6 @@ void cl_page_list_add(struct cl_page_list *plist, struct cl_page *page)
 	LASSERT(page->cp_owner);
 	LINVRNT(plist->pl_owner == current);
 
-	lockdep_off();
-	mutex_lock(&page->cp_mutex);
-	lockdep_on();
 	LASSERT(list_empty(&page->cp_batch));
 	list_add_tail(&page->cp_batch, &plist->pl_pages);
 	++plist->pl_nr;
@@ -877,12 +873,10 @@ void cl_page_list_del(const struct lu_env *env, struct cl_page_list *plist,
 		      struct cl_page *page)
 {
 	LASSERT(plist->pl_nr > 0);
+	LASSERT(cl_page_is_vmlocked(env, page));
 	LINVRNT(plist->pl_owner == current);
 
 	list_del_init(&page->cp_batch);
-	lockdep_off();
-	mutex_unlock(&page->cp_mutex);
-	lockdep_on();
 	--plist->pl_nr;
 	lu_ref_del_at(&page->cp_reference, &page->cp_queue_ref, "queue", plist);
 	cl_page_put(env, page);
@@ -941,8 +935,6 @@ void cl_page_list_splice(struct cl_page_list *list, struct cl_page_list *head)
 }
 EXPORT_SYMBOL(cl_page_list_splice);
 
-void cl_page_disown0(const struct lu_env *env,
-		     struct cl_io *io, struct cl_page *pg);
 
 /**
  * Disowns pages in a queue.
@@ -959,9 +951,6 @@ void cl_page_list_disown(const struct lu_env *env,
 		LASSERT(plist->pl_nr > 0);
 
 		list_del_init(&page->cp_batch);
-		lockdep_off();
-		mutex_unlock(&page->cp_mutex);
-		lockdep_on();
 		--plist->pl_nr;
 		/*
 		 * cl_page_disown0 rather than usual cl_page_disown() is used,
@@ -1221,7 +1210,7 @@ void cl_req_page_add(const struct lu_env *env,
 {
 	struct cl_object  *obj;
 	struct cl_req_obj *rqo;
-	int i;
+	unsigned int i;
 
 	LASSERT(list_empty(&page->cp_flight));
 	LASSERT(!page->cp_req);
@@ -1268,7 +1257,7 @@ EXPORT_SYMBOL(cl_req_page_done);
  */
 int cl_req_prep(const struct lu_env *env, struct cl_req *req)
 {
-	int i;
+	unsigned int i;
 	int result;
 	const struct cl_req_slice *slice;
 
@@ -1301,7 +1290,7 @@ void cl_req_attr_set(const struct lu_env *env, struct cl_req *req,
 {
 	const struct cl_req_slice *slice;
 	struct cl_page	    *page;
-	int i;
+	unsigned int i;
 
 	LASSERT(!list_empty(&req->crq_pages));
 
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_object.c b/drivers/staging/lustre/lustre/obdclass/cl_object.c
index 91a5806d0239..3199dd4a3b72 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_object.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_object.c
@@ -163,7 +163,7 @@ static spinlock_t *cl_object_attr_guard(struct cl_object *o)
  *
  * Prevents data-attributes from changing, until lock is released by
  * cl_object_attr_unlock(). This has to be called before calls to
- * cl_object_attr_get(), cl_object_attr_set().
+ * cl_object_attr_get(), cl_object_attr_update().
  */
 void cl_object_attr_lock(struct cl_object *o)
 	__acquires(cl_object_attr_guard(o))
@@ -217,11 +217,11 @@ EXPORT_SYMBOL(cl_object_attr_get);
  * Updates data-attributes of an object \a obj.
  *
  * Only attributes, mentioned in a validness bit-mask \a v are
- * updated. Calls cl_object_operations::coo_attr_set() on every layer, bottom
- * to top.
+ * updated. Calls cl_object_operations::coo_attr_update() on every layer,
+ * bottom to top.
  */
-int cl_object_attr_set(const struct lu_env *env, struct cl_object *obj,
-		       const struct cl_attr *attr, unsigned v)
+int cl_object_attr_update(const struct lu_env *env, struct cl_object *obj,
+			  const struct cl_attr *attr, unsigned int v)
 {
 	struct lu_object_header *top;
 	int result;
@@ -231,8 +231,9 @@ int cl_object_attr_set(const struct lu_env *env, struct cl_object *obj,
 	top = obj->co_lu.lo_header;
 	result = 0;
 	list_for_each_entry_reverse(obj, &top->loh_layers, co_lu.lo_linkage) {
-		if (obj->co_ops->coo_attr_set) {
-			result = obj->co_ops->coo_attr_set(env, obj, attr, v);
+		if (obj->co_ops->coo_attr_update) {
+			result = obj->co_ops->coo_attr_update(env, obj, attr,
+							      v);
 			if (result != 0) {
 				if (result > 0)
 					result = 0;
@@ -242,7 +243,7 @@ int cl_object_attr_set(const struct lu_env *env, struct cl_object *obj,
 	}
 	return result;
 }
-EXPORT_SYMBOL(cl_object_attr_set);
+EXPORT_SYMBOL(cl_object_attr_update);
 
 /**
  * Notifies layers (bottom-to-top) that glimpse AST was received.
@@ -321,6 +322,27 @@ int cl_object_prune(const struct lu_env *env, struct cl_object *obj)
 EXPORT_SYMBOL(cl_object_prune);
 
 /**
+ * Get stripe information of this object.
+ */
+int cl_object_getstripe(const struct lu_env *env, struct cl_object *obj,
+			struct lov_user_md __user *uarg)
+{
+	struct lu_object_header *top;
+	int result = 0;
+
+	top = obj->co_lu.lo_header;
+	list_for_each_entry(obj, &top->loh_layers, co_lu.lo_linkage) {
+		if (obj->co_ops->coo_getstripe) {
+			result = obj->co_ops->coo_getstripe(env, obj, uarg);
+			if (result)
+			break;
+		}
+	}
+	return result;
+}
+EXPORT_SYMBOL(cl_object_getstripe);
+
+/**
  * Helper function removing all object locks, and marking object for
  * deletion. All object pages must have been deleted at this point.
  *
@@ -377,7 +399,7 @@ static void cl_env_percpu_refill(void);
  */
 int cl_site_init(struct cl_site *s, struct cl_device *d)
 {
-	int i;
+	size_t i;
 	int result;
 
 	result = lu_site_init(&s->cs_lu, &d->cd_lu_dev);
@@ -411,7 +433,7 @@ static struct cache_stats cl_env_stats = {
  */
 int cl_site_stats_print(const struct cl_site *site, struct seq_file *m)
 {
-	int i;
+	size_t i;
 	static const char *pstate[] = {
 		[CPS_CACHED]  = "c",
 		[CPS_OWNED]   = "o",
@@ -1000,7 +1022,7 @@ static int cl_env_percpu_init(void)
 		 * thus we must uninitialize up to i, the rest are undefined.
 		 */
 		for (j = 0; j < i; j++) {
-			cle = &cl_env_percpu[i];
+			cle = &cl_env_percpu[j];
 			lu_context_exit(&cle->ce_ses);
 			lu_context_fini(&cle->ce_ses);
 			lu_env_fini(&cle->ce_lu);
@@ -1126,7 +1148,7 @@ static void *cl_key_init(const struct lu_context *ctx,
 
 	info = cl0_key_init(ctx, key);
 	if (!IS_ERR(info)) {
-		int i;
+		size_t i;
 
 		for (i = 0; i < ARRAY_SIZE(info->clt_counters); ++i)
 			lu_ref_init(&info->clt_counters[i].ctc_locks_locked);
@@ -1138,7 +1160,7 @@ static void cl_key_fini(const struct lu_context *ctx,
 			struct lu_context_key *key, void *data)
 {
 	struct cl_thread_info *info;
-	int i;
+	size_t i;
 
 	info = data;
 	for (i = 0; i < ARRAY_SIZE(info->clt_counters); ++i)
@@ -1150,7 +1172,7 @@ static void cl_key_exit(const struct lu_context *ctx,
 			struct lu_context_key *key, void *data)
 {
 	struct cl_thread_info *info = data;
-	int i;
+	size_t i;
 
 	for (i = 0; i < ARRAY_SIZE(info->clt_counters); ++i) {
 		LASSERT(info->clt_counters[i].ctc_nr_held == 0);
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_page.c b/drivers/staging/lustre/lustre/obdclass/cl_page.c
index db2dc6b39073..63973ba096da 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_page.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_page.c
@@ -151,7 +151,6 @@ struct cl_page *cl_page_alloc(const struct lu_env *env,
 		INIT_LIST_HEAD(&page->cp_layers);
 		INIT_LIST_HEAD(&page->cp_batch);
 		INIT_LIST_HEAD(&page->cp_flight);
-		mutex_init(&page->cp_mutex);
 		lu_ref_init(&page->cp_reference);
 		head = o->co_lu.lo_header;
 		list_for_each_entry(o, &head->loh_layers, co_lu.lo_linkage) {
@@ -171,7 +170,6 @@ struct cl_page *cl_page_alloc(const struct lu_env *env,
 	}
 	return page;
 }
-EXPORT_SYMBOL(cl_page_alloc);
 
 /**
  * Returns a cl_page with index \a idx at the object \a o, and associated with
@@ -229,11 +227,6 @@ EXPORT_SYMBOL(cl_page_find);
 
 static inline int cl_page_invariant(const struct cl_page *pg)
 {
-	/*
-	 * Page invariant is protected by a VM lock.
-	 */
-	LINVRNT(cl_page_is_vmlocked(NULL, pg));
-
 	return cl_page_in_use_noref(pg);
 }
 
@@ -478,7 +471,6 @@ static void cl_page_owner_clear(struct cl_page *page)
 		LASSERT(page->cp_owner->ci_owned_nr > 0);
 		page->cp_owner->ci_owned_nr--;
 		page->cp_owner = NULL;
-		page->cp_task = NULL;
 	}
 }
 
@@ -562,7 +554,6 @@ static int cl_page_own0(const struct lu_env *env, struct cl_io *io,
 			PASSERT(env, pg, !pg->cp_owner);
 			PASSERT(env, pg, !pg->cp_req);
 			pg->cp_owner = cl_io_top(io);
-			pg->cp_task  = current;
 			cl_page_owner_set(pg);
 			if (pg->cp_state != CPS_FREEING) {
 				cl_page_state_set(env, pg, CPS_OWNED);
@@ -619,7 +610,6 @@ void cl_page_assume(const struct lu_env *env,
 	cl_page_invoid(env, io, pg, CL_PAGE_OP(cpo_assume));
 	PASSERT(env, pg, !pg->cp_owner);
 	pg->cp_owner = cl_io_top(io);
-	pg->cp_task = current;
 	cl_page_owner_set(pg);
 	cl_page_state_set(env, pg, CPS_OWNED);
 }
@@ -860,10 +850,6 @@ void cl_page_completion(const struct lu_env *env,
 	PASSERT(env, pg, pg->cp_state == cl_req_type_state(crt));
 
 	CL_PAGE_HEADER(D_TRACE, env, pg, "%d %d\n", crt, ioret);
-	if (crt == CRT_READ && ioret == 0) {
-		PASSERT(env, pg, !(pg->cp_flags & CPF_READ_COMPLETED));
-		pg->cp_flags |= CPF_READ_COMPLETED;
-	}
 
 	cl_page_state_set(env, pg, CPS_CACHED);
 	if (crt >= CRT_NR)
@@ -872,7 +858,6 @@ void cl_page_completion(const struct lu_env *env,
 			       (const struct lu_env *,
 				const struct cl_page_slice *, int), ioret);
 	if (anchor) {
-		LASSERT(cl_page_is_vmlocked(env, pg));
 		LASSERT(pg->cp_sync_io == anchor);
 		pg->cp_sync_io = NULL;
 	}
@@ -989,10 +974,10 @@ void cl_page_header_print(const struct lu_env *env, void *cookie,
 			  lu_printer_t printer, const struct cl_page *pg)
 {
 	(*printer)(env, cookie,
-		   "page@%p[%d %p %d %d %d %p %p %#x]\n",
+		   "page@%p[%d %p %d %d %p %p]\n",
 		   pg, atomic_read(&pg->cp_ref), pg->cp_obj,
-		   pg->cp_state, pg->cp_error, pg->cp_type,
-		   pg->cp_owner, pg->cp_req, pg->cp_flags);
+		   pg->cp_state, pg->cp_type,
+		   pg->cp_owner, pg->cp_req);
 }
 EXPORT_SYMBOL(cl_page_header_print);
 
@@ -1020,7 +1005,6 @@ int cl_page_cancel(const struct lu_env *env, struct cl_page *page)
 			      (const struct lu_env *,
 			       const struct cl_page_slice *));
 }
-EXPORT_SYMBOL(cl_page_cancel);
 
 /**
  * Converts a byte offset within object \a obj into a page index.
@@ -1046,9 +1030,9 @@ pgoff_t cl_index(const struct cl_object *obj, loff_t offset)
 }
 EXPORT_SYMBOL(cl_index);
 
-int cl_page_size(const struct cl_object *obj)
+size_t cl_page_size(const struct cl_object *obj)
 {
-	return 1 << PAGE_SHIFT;
+	return 1UL << PAGE_SHIFT;
 }
 EXPORT_SYMBOL(cl_page_size);
 
@@ -1087,11 +1071,11 @@ struct cl_client_cache *cl_cache_init(unsigned long lru_page_max)
 	/* Initialize cache data */
 	atomic_set(&cache->ccc_users, 1);
 	cache->ccc_lru_max = lru_page_max;
-	atomic_set(&cache->ccc_lru_left, lru_page_max);
+	atomic_long_set(&cache->ccc_lru_left, lru_page_max);
 	spin_lock_init(&cache->ccc_lru_lock);
 	INIT_LIST_HEAD(&cache->ccc_lru);
 
-	atomic_set(&cache->ccc_unstable_nr, 0);
+	atomic_long_set(&cache->ccc_unstable_nr, 0);
 	init_waitqueue_head(&cache->ccc_unstable_waitq);
 
 	return cache;
diff --git a/drivers/staging/lustre/lustre/obdclass/class_obd.c b/drivers/staging/lustre/lustre/obdclass/class_obd.c
index d9d2a1952b8b..76e1ee83a723 100644
--- a/drivers/staging/lustre/lustre/obdclass/class_obd.c
+++ b/drivers/staging/lustre/lustre/obdclass/class_obd.c
@@ -40,10 +40,10 @@
 #include "../include/lprocfs_status.h"
 #include <linux/list.h>
 #include "../include/cl_object.h"
+#include "../include/lustre/lustre_ioctl.h"
 #include "llog_internal.h"
 
 struct obd_device *obd_devs[MAX_OBD_DEVICES];
-EXPORT_SYMBOL(obd_devs);
 struct list_head obd_types;
 DEFINE_RWLOCK(obd_dev_lock);
 
@@ -54,11 +54,9 @@ unsigned int obd_dump_on_timeout;
 EXPORT_SYMBOL(obd_dump_on_timeout);
 unsigned int obd_dump_on_eviction;
 EXPORT_SYMBOL(obd_dump_on_eviction);
-unsigned int obd_max_dirty_pages = 256;
+unsigned long obd_max_dirty_pages;
 EXPORT_SYMBOL(obd_max_dirty_pages);
-atomic_t obd_unstable_pages;
-EXPORT_SYMBOL(obd_unstable_pages);
-atomic_t obd_dirty_pages;
+atomic_long_t obd_dirty_pages;
 EXPORT_SYMBOL(obd_dirty_pages);
 unsigned int obd_timeout = OBD_TIMEOUT_DEFAULT;   /* seconds */
 EXPORT_SYMBOL(obd_timeout);
@@ -76,13 +74,11 @@ EXPORT_SYMBOL(at_early_margin);
 int at_extra = 30;
 EXPORT_SYMBOL(at_extra);
 
-atomic_t obd_dirty_transit_pages;
+atomic_long_t obd_dirty_transit_pages;
 EXPORT_SYMBOL(obd_dirty_transit_pages);
 
 char obd_jobid_var[JOBSTATS_JOBID_VAR_MAX_LEN + 1] = JOBSTATS_DISABLE;
-EXPORT_SYMBOL(obd_jobid_var);
-
-char obd_jobid_node[JOBSTATS_JOBID_SIZE + 1];
+char obd_jobid_node[LUSTRE_JOBID_SIZE + 1];
 
 /* Get jobid of current process from stored variable or calculate
  * it from pid and user_id.
@@ -93,14 +89,14 @@ char obd_jobid_node[JOBSTATS_JOBID_SIZE + 1];
  */
 int lustre_get_jobid(char *jobid)
 {
-	memset(jobid, 0, JOBSTATS_JOBID_SIZE);
+	memset(jobid, 0, LUSTRE_JOBID_SIZE);
 	/* Jobstats isn't enabled */
 	if (strcmp(obd_jobid_var, JOBSTATS_DISABLE) == 0)
 		return 0;
 
 	/* Use process name + fsuid as jobid */
 	if (strcmp(obd_jobid_var, JOBSTATS_PROCNAME_UID) == 0) {
-		snprintf(jobid, JOBSTATS_JOBID_SIZE, "%s.%u",
+		snprintf(jobid, LUSTRE_JOBID_SIZE, "%s.%u",
 			 current_comm(),
 			 from_kuid(&init_user_ns, current_fsuid()));
 		return 0;
@@ -116,19 +112,6 @@ int lustre_get_jobid(char *jobid)
 }
 EXPORT_SYMBOL(lustre_get_jobid);
 
-static inline void obd_data2conn(struct lustre_handle *conn,
-				 struct obd_ioctl_data *data)
-{
-	memset(conn, 0, sizeof(*conn));
-	conn->cookie = data->ioc_cookie;
-}
-
-static inline void obd_conn2data(struct obd_ioctl_data *data,
-				 struct lustre_handle *conn)
-{
-	data->ioc_cookie = conn->cookie;
-}
-
 static int class_resolve_dev_name(__u32 len, const char *name)
 {
 	int rc;
@@ -287,13 +270,6 @@ int class_handle_ioctl(unsigned int cmd, unsigned long arg)
 		goto out;
 	}
 
-	case OBD_IOC_CLOSE_UUID: {
-		CDEBUG(D_IOCTL, "closing all connections to uuid %s (NOOP)\n",
-		       data->ioc_inlbuf1);
-		err = 0;
-		goto out;
-	}
-
 	case OBD_IOC_GETDEVICE: {
 		int     index = data->ioc_count;
 		char    *status, *str;
@@ -467,15 +443,10 @@ static int obd_init_checks(void)
 	return ret;
 }
 
-extern int class_procfs_init(void);
-extern int class_procfs_clean(void);
-
 static int __init obdclass_init(void)
 {
 	int i, err;
 
-	int lustre_register_fs(void);
-
 	LCONSOLE_INFO("Lustre: Build Version: " LUSTRE_VERSION_STRING "\n");
 
 	spin_lock_init(&obd_types_lock);
@@ -542,23 +513,9 @@ static int __init obdclass_init(void)
 
 static void obdclass_exit(void)
 {
-	int i;
-
-	int lustre_unregister_fs(void);
-
 	lustre_unregister_fs();
 
 	misc_deregister(&obd_psdev);
-	for (i = 0; i < class_devno_max(); i++) {
-		struct obd_device *obd = class_num2obd(i);
-
-		if (obd && obd->obd_set_up &&
-		    OBT(obd) && OBP(obd, detach)) {
-			/* XXX should this call generic detach otherwise? */
-			LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC);
-			OBP(obd, detach)(obd);
-		}
-	}
 	llog_info_fini();
 	cl_global_fini();
 	lu_global_fini();
diff --git a/drivers/staging/lustre/lustre/obdclass/debug.c b/drivers/staging/lustre/lustre/obdclass/debug.c
index 8acf67239fa8..0bd4ad20aba7 100644
--- a/drivers/staging/lustre/lustre/obdclass/debug.c
+++ b/drivers/staging/lustre/lustre/obdclass/debug.c
@@ -48,10 +48,10 @@ int block_debug_setup(void *addr, int len, __u64 off, __u64 id)
 	LASSERT(addr);
 
 	put_unaligned_le64(off, addr);
-	put_unaligned_le64(id, addr+LPDS);
+	put_unaligned_le64(id, addr + LPDS);
 	addr += len - LPDS - LPDS;
 	put_unaligned_le64(off, addr);
-	put_unaligned_le64(id, addr+LPDS);
+	put_unaligned_le64(id, addr + LPDS);
 
 	return 0;
 }
diff --git a/drivers/staging/lustre/lustre/obdclass/genops.c b/drivers/staging/lustre/lustre/obdclass/genops.c
index 99c2da632b51..cf8bb2a2f40b 100644
--- a/drivers/staging/lustre/lustre/obdclass/genops.c
+++ b/drivers/staging/lustre/lustre/obdclass/genops.c
@@ -133,7 +133,6 @@ void class_put_type(struct obd_type *type)
 	module_put(type->typ_dt_ops->owner);
 	spin_unlock(&type->obd_type_lock);
 }
-EXPORT_SYMBOL(class_put_type);
 
 #define CLASS_MAX_NAME 1024
 
@@ -166,10 +165,10 @@ int class_register_type(struct obd_ops *dt_ops, struct md_ops *md_ops,
 	    !type->typ_name)
 		goto failed;
 
-	*(type->typ_dt_ops) = *dt_ops;
+	*type->typ_dt_ops = *dt_ops;
 	/* md_ops is optional */
 	if (md_ops)
-		*(type->typ_md_ops) = *md_ops;
+		*type->typ_md_ops = *md_ops;
 	strcpy(type->typ_name, name);
 	spin_lock_init(&type->obd_type_lock);
 
@@ -391,7 +390,6 @@ int class_name2dev(const char *name)
 
 	return -1;
 }
-EXPORT_SYMBOL(class_name2dev);
 
 struct obd_device *class_name2obd(const char *name)
 {
@@ -421,7 +419,6 @@ int class_uuid2dev(struct obd_uuid *uuid)
 
 	return -1;
 }
-EXPORT_SYMBOL(class_uuid2dev);
 
 /**
  * Get obd device from ::obd_devs[]
@@ -450,7 +447,6 @@ struct obd_device *class_num2obd(int num)
 
 	return obd;
 }
-EXPORT_SYMBOL(class_num2obd);
 
 /* Search for a client OBD connected to tgt_uuid.  If grp_uuid is
  * specified, then only the client with that uuid is returned,
@@ -509,7 +505,7 @@ struct obd_device *class_devices_in_group(struct obd_uuid *grp_uuid, int *next)
 			continue;
 		if (obd_uuid_equals(grp_uuid, &obd->obd_uuid)) {
 			if (next)
-				*next = i+1;
+				*next = i + 1;
 			read_unlock(&obd_dev_lock);
 			return obd;
 		}
@@ -618,7 +614,7 @@ struct obd_export *class_conn2export(struct lustre_handle *conn)
 	}
 
 	CDEBUG(D_INFO, "looking for export cookie %#llx\n", conn->cookie);
-	export = class_handle2object(conn->cookie);
+	export = class_handle2object(conn->cookie, NULL);
 	return export;
 }
 EXPORT_SYMBOL(class_conn2export);
@@ -817,7 +813,6 @@ void class_unlink_export(struct obd_export *exp)
 	spin_unlock(&exp->exp_obd->obd_dev_lock);
 	class_export_put(exp);
 }
-EXPORT_SYMBOL(class_unlink_export);
 
 /* Import management functions */
 static void class_import_destroy(struct obd_import *imp)
@@ -973,7 +968,6 @@ void __class_export_add_lock_ref(struct obd_export *exp, struct ldlm_lock *lock)
 	       lock, exp, lock->l_exp_refs_nr);
 	spin_unlock(&exp->exp_locks_list_guard);
 }
-EXPORT_SYMBOL(__class_export_add_lock_ref);
 
 void __class_export_del_lock_ref(struct obd_export *exp, struct ldlm_lock *lock)
 {
@@ -991,7 +985,6 @@ void __class_export_del_lock_ref(struct obd_export *exp, struct ldlm_lock *lock)
 	       lock, exp, lock->l_exp_refs_nr);
 	spin_unlock(&exp->exp_locks_list_guard);
 }
-EXPORT_SYMBOL(__class_export_del_lock_ref);
 #endif
 
 /* A connection defines an export context in which preallocation can
@@ -1100,7 +1093,6 @@ EXPORT_SYMBOL(class_fail_export);
 
 #if LUSTRE_TRACKS_LOCK_EXP_REFS
 void (*class_export_dump_hook)(struct obd_export *) = NULL;
-EXPORT_SYMBOL(class_export_dump_hook);
 #endif
 
 /* Total amount of zombies to be destroyed */
@@ -1312,3 +1304,135 @@ void obd_zombie_impexp_stop(void)
 	obd_zombie_impexp_notify();
 	wait_for_completion(&obd_zombie_stop);
 }
+
+struct obd_request_slot_waiter {
+	struct list_head	orsw_entry;
+	wait_queue_head_t	orsw_waitq;
+	bool			orsw_signaled;
+};
+
+static bool obd_request_slot_avail(struct client_obd *cli,
+				   struct obd_request_slot_waiter *orsw)
+{
+	bool avail;
+
+	spin_lock(&cli->cl_loi_list_lock);
+	avail = !!list_empty(&orsw->orsw_entry);
+	spin_unlock(&cli->cl_loi_list_lock);
+
+	return avail;
+};
+
+/*
+ * For network flow control, the RPC sponsor needs to acquire a credit
+ * before sending the RPC. The credits count for a connection is defined
+ * by the "cl_max_rpcs_in_flight". If all the credits are occpuied, then
+ * the subsequent RPC sponsors need to wait until others released their
+ * credits, or the administrator increased the "cl_max_rpcs_in_flight".
+ */
+int obd_get_request_slot(struct client_obd *cli)
+{
+	struct obd_request_slot_waiter orsw;
+	struct l_wait_info lwi;
+	int rc;
+
+	spin_lock(&cli->cl_loi_list_lock);
+	if (cli->cl_r_in_flight < cli->cl_max_rpcs_in_flight) {
+		cli->cl_r_in_flight++;
+		spin_unlock(&cli->cl_loi_list_lock);
+		return 0;
+	}
+
+	init_waitqueue_head(&orsw.orsw_waitq);
+	list_add_tail(&orsw.orsw_entry, &cli->cl_loi_read_list);
+	orsw.orsw_signaled = false;
+	spin_unlock(&cli->cl_loi_list_lock);
+
+	lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP, NULL);
+	rc = l_wait_event(orsw.orsw_waitq,
+			  obd_request_slot_avail(cli, &orsw) ||
+			  orsw.orsw_signaled,
+			  &lwi);
+
+	/*
+	 * Here, we must take the lock to avoid the on-stack 'orsw' to be
+	 * freed but other (such as obd_put_request_slot) is using it.
+	 */
+	spin_lock(&cli->cl_loi_list_lock);
+	if (rc) {
+		if (!orsw.orsw_signaled) {
+			if (list_empty(&orsw.orsw_entry))
+				cli->cl_r_in_flight--;
+			else
+				list_del(&orsw.orsw_entry);
+		}
+	}
+
+	if (orsw.orsw_signaled) {
+		LASSERT(list_empty(&orsw.orsw_entry));
+
+		rc = -EINTR;
+	}
+	spin_unlock(&cli->cl_loi_list_lock);
+
+	return rc;
+}
+EXPORT_SYMBOL(obd_get_request_slot);
+
+void obd_put_request_slot(struct client_obd *cli)
+{
+	struct obd_request_slot_waiter *orsw;
+
+	spin_lock(&cli->cl_loi_list_lock);
+	cli->cl_r_in_flight--;
+
+	/* If there is free slot, wakeup the first waiter. */
+	if (!list_empty(&cli->cl_loi_read_list) &&
+	    likely(cli->cl_r_in_flight < cli->cl_max_rpcs_in_flight)) {
+		orsw = list_entry(cli->cl_loi_read_list.next,
+				  struct obd_request_slot_waiter, orsw_entry);
+		list_del_init(&orsw->orsw_entry);
+		cli->cl_r_in_flight++;
+		wake_up(&orsw->orsw_waitq);
+	}
+	spin_unlock(&cli->cl_loi_list_lock);
+}
+EXPORT_SYMBOL(obd_put_request_slot);
+
+__u32 obd_get_max_rpcs_in_flight(struct client_obd *cli)
+{
+	return cli->cl_max_rpcs_in_flight;
+}
+EXPORT_SYMBOL(obd_get_max_rpcs_in_flight);
+
+int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max)
+{
+	struct obd_request_slot_waiter *orsw;
+	__u32 old;
+	int diff;
+	int i;
+
+	if (max > OBD_MAX_RIF_MAX || max < 1)
+		return -ERANGE;
+
+	spin_lock(&cli->cl_loi_list_lock);
+	old = cli->cl_max_rpcs_in_flight;
+	cli->cl_max_rpcs_in_flight = max;
+	diff = max - old;
+
+	/* We increase the max_rpcs_in_flight, then wakeup some waiters. */
+	for (i = 0; i < diff; i++) {
+		if (list_empty(&cli->cl_loi_read_list))
+			break;
+
+		orsw = list_entry(cli->cl_loi_read_list.next,
+				  struct obd_request_slot_waiter, orsw_entry);
+		list_del_init(&orsw->orsw_entry);
+		cli->cl_r_in_flight++;
+		wake_up(&orsw->orsw_waitq);
+	}
+	spin_unlock(&cli->cl_loi_list_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL(obd_set_max_rpcs_in_flight);
diff --git a/drivers/staging/lustre/lustre/obdclass/linkea.c b/drivers/staging/lustre/lustre/obdclass/linkea.c
new file mode 100644
index 000000000000..0b1d2f0a422c
--- /dev/null
+++ b/drivers/staging/lustre/lustre/obdclass/linkea.c
@@ -0,0 +1,201 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2013, 2014, Intel Corporation.
+ * Use is subject to license terms.
+ *
+ * Author: Di Wang <di.wang@intel.com>
+ */
+
+#include "../include/lustre/lustre_idl.h"
+#include "../include/obd.h"
+#include "../include/lustre_linkea.h"
+
+int linkea_data_new(struct linkea_data *ldata, struct lu_buf *buf)
+{
+	ldata->ld_buf = lu_buf_check_and_alloc(buf, PAGE_SIZE);
+	if (!ldata->ld_buf->lb_buf)
+		return -ENOMEM;
+	ldata->ld_leh = ldata->ld_buf->lb_buf;
+	ldata->ld_leh->leh_magic = LINK_EA_MAGIC;
+	ldata->ld_leh->leh_len = sizeof(struct link_ea_header);
+	ldata->ld_leh->leh_reccount = 0;
+	return 0;
+}
+EXPORT_SYMBOL(linkea_data_new);
+
+int linkea_init(struct linkea_data *ldata)
+{
+	struct link_ea_header *leh;
+
+	LASSERT(ldata->ld_buf);
+	leh = ldata->ld_buf->lb_buf;
+	if (leh->leh_magic == __swab32(LINK_EA_MAGIC)) {
+		leh->leh_magic = LINK_EA_MAGIC;
+		leh->leh_reccount = __swab32(leh->leh_reccount);
+		leh->leh_len = __swab64(leh->leh_len);
+		/* entries are swabbed by linkea_entry_unpack */
+	}
+	if (leh->leh_magic != LINK_EA_MAGIC)
+		return -EINVAL;
+	if (leh->leh_reccount == 0)
+		return -ENODATA;
+
+	ldata->ld_leh = leh;
+	return 0;
+}
+EXPORT_SYMBOL(linkea_init);
+
+/**
+ * Pack a link_ea_entry.
+ * All elements are stored as chars to avoid alignment issues.
+ * Numbers are always big-endian
+ * \retval record length
+ */
+int linkea_entry_pack(struct link_ea_entry *lee, const struct lu_name *lname,
+		      const struct lu_fid *pfid)
+{
+	struct lu_fid   tmpfid;
+	int             reclen;
+
+	tmpfid = *pfid;
+	if (OBD_FAIL_CHECK(OBD_FAIL_LFSCK_LINKEA_CRASH))
+		tmpfid.f_ver = ~0;
+	fid_cpu_to_be(&tmpfid, &tmpfid);
+	memcpy(&lee->lee_parent_fid, &tmpfid, sizeof(tmpfid));
+	memcpy(lee->lee_name, lname->ln_name, lname->ln_namelen);
+	reclen = sizeof(struct link_ea_entry) + lname->ln_namelen;
+
+	lee->lee_reclen[0] = (reclen >> 8) & 0xff;
+	lee->lee_reclen[1] = reclen & 0xff;
+	return reclen;
+}
+EXPORT_SYMBOL(linkea_entry_pack);
+
+void linkea_entry_unpack(const struct link_ea_entry *lee, int *reclen,
+			 struct lu_name *lname, struct lu_fid *pfid)
+{
+	*reclen = (lee->lee_reclen[0] << 8) | lee->lee_reclen[1];
+	memcpy(pfid, &lee->lee_parent_fid, sizeof(*pfid));
+	fid_be_to_cpu(pfid, pfid);
+	if (lname) {
+		lname->ln_name = lee->lee_name;
+		lname->ln_namelen = *reclen - sizeof(struct link_ea_entry);
+	}
+}
+EXPORT_SYMBOL(linkea_entry_unpack);
+
+/**
+ * Add a record to the end of link ea buf
+ **/
+int linkea_add_buf(struct linkea_data *ldata, const struct lu_name *lname,
+		   const struct lu_fid *pfid)
+{
+	LASSERT(ldata->ld_leh);
+
+	if (!lname || !pfid)
+		return -EINVAL;
+
+	ldata->ld_reclen = lname->ln_namelen + sizeof(struct link_ea_entry);
+	if (ldata->ld_leh->leh_len + ldata->ld_reclen >
+	    ldata->ld_buf->lb_len) {
+		if (lu_buf_check_and_grow(ldata->ld_buf,
+					  ldata->ld_leh->leh_len +
+					  ldata->ld_reclen) < 0)
+			return -ENOMEM;
+	}
+
+	ldata->ld_leh = ldata->ld_buf->lb_buf;
+	ldata->ld_lee = ldata->ld_buf->lb_buf + ldata->ld_leh->leh_len;
+	ldata->ld_reclen = linkea_entry_pack(ldata->ld_lee, lname, pfid);
+	ldata->ld_leh->leh_len += ldata->ld_reclen;
+	ldata->ld_leh->leh_reccount++;
+	CDEBUG(D_INODE, "New link_ea name '" DFID ":%.*s' is added\n",
+	       PFID(pfid), lname->ln_namelen, lname->ln_name);
+	return 0;
+}
+EXPORT_SYMBOL(linkea_add_buf);
+
+/** Del the current record from the link ea buf */
+void linkea_del_buf(struct linkea_data *ldata, const struct lu_name *lname)
+{
+	LASSERT(ldata->ld_leh && ldata->ld_lee);
+
+	ldata->ld_leh->leh_reccount--;
+	ldata->ld_leh->leh_len -= ldata->ld_reclen;
+	memmove(ldata->ld_lee, (char *)ldata->ld_lee + ldata->ld_reclen,
+		(char *)ldata->ld_leh + ldata->ld_leh->leh_len -
+		(char *)ldata->ld_lee);
+	CDEBUG(D_INODE, "Old link_ea name '%.*s' is removed\n",
+	       lname->ln_namelen, lname->ln_name);
+
+	if ((char *)ldata->ld_lee >= ((char *)ldata->ld_leh +
+				      ldata->ld_leh->leh_len))
+		ldata->ld_lee = NULL;
+}
+EXPORT_SYMBOL(linkea_del_buf);
+
+/**
+ * Check if such a link exists in linkEA.
+ *
+ * \param ldata link data the search to be done on
+ * \param lname name in the parent's directory entry pointing to this object
+ * \param pfid parent fid the link to be found for
+ *
+ * \retval   0 success
+ * \retval -ENOENT link does not exist
+ * \retval -ve on error
+ */
+int linkea_links_find(struct linkea_data *ldata, const struct lu_name *lname,
+		      const struct lu_fid  *pfid)
+{
+	struct lu_name tmpname;
+	struct lu_fid  tmpfid;
+	int count;
+
+	LASSERT(ldata->ld_leh);
+
+	/* link #0 */
+	ldata->ld_lee = (struct link_ea_entry *)(ldata->ld_leh + 1);
+
+	for (count = 0; count < ldata->ld_leh->leh_reccount; count++) {
+		linkea_entry_unpack(ldata->ld_lee, &ldata->ld_reclen,
+				    &tmpname, &tmpfid);
+		if (tmpname.ln_namelen == lname->ln_namelen &&
+		    lu_fid_eq(&tmpfid, pfid) &&
+		    (strncmp(tmpname.ln_name, lname->ln_name,
+			     tmpname.ln_namelen) == 0))
+			break;
+		ldata->ld_lee = (struct link_ea_entry *)((char *)ldata->ld_lee +
+							 ldata->ld_reclen);
+	}
+
+	if (count == ldata->ld_leh->leh_reccount) {
+		CDEBUG(D_INODE, "Old link_ea name '%.*s' not found\n",
+		       lname->ln_namelen, lname->ln_name);
+		ldata->ld_lee = NULL;
+		ldata->ld_reclen = 0;
+		return -ENOENT;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(linkea_links_find);
diff --git a/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c b/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
index 33342bfcc90e..be09e04b042f 100644
--- a/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
+++ b/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
@@ -65,6 +65,7 @@
 #include "../../include/obd_support.h"
 #include "../../include/obd_class.h"
 #include "../../include/lprocfs_status.h"
+#include "../../include/lustre/lustre_ioctl.h"
 #include "../../include/lustre_ver.h"
 
 /* buffer MUST be at least the size of obd_ioctl_hdr */
@@ -157,7 +158,6 @@ int obd_ioctl_popdata(void __user *arg, void *data, int len)
 	err = copy_to_user(arg, data, len) ? -EFAULT : 0;
 	return err;
 }
-EXPORT_SYMBOL(obd_ioctl_popdata);
 
 /*  opening /dev/obd */
 static int obd_class_open(struct inode *inode, struct file *file)
@@ -191,7 +191,7 @@ static long obd_class_ioctl(struct file *filp, unsigned int cmd,
 }
 
 /* declare character device */
-static struct file_operations obd_psdev_fops = {
+static const struct file_operations obd_psdev_fops = {
 	.owner	  = THIS_MODULE,
 	.unlocked_ioctl = obd_class_ioctl, /* unlocked_ioctl */
 	.open	   = obd_class_open,      /* open */
@@ -291,7 +291,7 @@ static ssize_t jobid_name_store(struct kobject *kobj, struct attribute *attr,
 				const char *buffer,
 				size_t count)
 {
-	if (!count || count > JOBSTATS_JOBID_SIZE)
+	if (!count || count > LUSTRE_JOBID_SIZE)
 		return -EINVAL;
 
 	memcpy(obd_jobid_node, buffer, count);
diff --git a/drivers/staging/lustre/lustre/obdclass/linux/linux-obdo.c b/drivers/staging/lustre/lustre/obdclass/linux/linux-obdo.c
index c6cc6a7666e3..41b77a30feb3 100644
--- a/drivers/staging/lustre/lustre/obdclass/linux/linux-obdo.c
+++ b/drivers/staging/lustre/lustre/obdclass/linux/linux-obdo.c
@@ -44,7 +44,7 @@
 
 #include <linux/fs.h>
 
-void obdo_refresh_inode(struct inode *dst, struct obdo *src, u32 valid)
+void obdo_refresh_inode(struct inode *dst, const struct obdo *src, u32 valid)
 {
 	valid &= src->o_valid;
 
diff --git a/drivers/staging/lustre/lustre/obdclass/linux/linux-sysctl.c b/drivers/staging/lustre/lustre/obdclass/linux/linux-sysctl.c
index 8f70dd2686f9..e6c785afceba 100644
--- a/drivers/staging/lustre/lustre/obdclass/linux/linux-sysctl.c
+++ b/drivers/staging/lustre/lustre/obdclass/linux/linux-sysctl.c
@@ -45,6 +45,7 @@
 
 #include "../../include/obd_support.h"
 #include "../../include/lprocfs_status.h"
+#include "../../include/obd_class.h"
 
 struct static_lustre_uintvalue_attr {
 	struct {
@@ -95,8 +96,8 @@ LUSTRE_STATIC_UINT_ATTR(timeout, &obd_timeout);
 static ssize_t max_dirty_mb_show(struct kobject *kobj, struct attribute *attr,
 				 char *buf)
 {
-	return sprintf(buf, "%ul\n",
-			obd_max_dirty_pages / (1 << (20 - PAGE_SHIFT)));
+	return sprintf(buf, "%lu\n",
+		       obd_max_dirty_pages / (1 << (20 - PAGE_SHIFT)));
 }
 
 static ssize_t max_dirty_mb_store(struct kobject *kobj, struct attribute *attr,
diff --git a/drivers/staging/lustre/lustre/obdclass/llog.c b/drivers/staging/lustre/lustre/obdclass/llog.c
index 1784ca063428..43797f106745 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog.c
@@ -80,7 +80,7 @@ static void llog_free_handle(struct llog_handle *loghandle)
 		LASSERT(list_empty(&loghandle->u.phd.phd_entry));
 	else if (loghandle->lgh_hdr->llh_flags & LLOG_F_IS_CAT)
 		LASSERT(list_empty(&loghandle->u.chd.chd_head));
-	LASSERT(sizeof(*(loghandle->lgh_hdr)) == LLOG_CHUNK_SIZE);
+	LASSERT(sizeof(*loghandle->lgh_hdr) == LLOG_CHUNK_SIZE);
 	kfree(loghandle->lgh_hdr);
 out:
 	kfree(loghandle);
@@ -137,6 +137,7 @@ static int llog_read_header(const struct lu_env *env,
 int llog_init_handle(const struct lu_env *env, struct llog_handle *handle,
 		     int flags, struct obd_uuid *uuid)
 {
+	enum llog_flag fmt = flags & LLOG_F_EXT_MASK;
 	struct llog_log_hdr	*llh;
 	int			 rc;
 
@@ -194,6 +195,7 @@ int llog_init_handle(const struct lu_env *env, struct llog_handle *handle,
 		       flags, LLOG_F_IS_CAT, LLOG_F_IS_PLAIN);
 		rc = -EINVAL;
 	}
+	llh->llh_flags |= fmt;
 out:
 	if (rc) {
 		kfree(llh);
@@ -233,6 +235,10 @@ static int llog_process_thread(void *arg)
 	else
 		last_index = LLOG_BITMAP_BYTES * 8 - 1;
 
+	/* Record is not in this buffer. */
+	if (index > last_index)
+		goto out;
+
 	while (rc == 0) {
 		struct llog_rec_hdr *rec;
 
@@ -262,7 +268,7 @@ repeat:
 		 */
 		for (rec = (struct llog_rec_hdr *)buf;
 		     (char *)rec < buf + LLOG_CHUNK_SIZE;
-		     rec = (struct llog_rec_hdr *)((char *)rec + rec->lrh_len)) {
+		     rec = llog_rec_hdr_next(rec)) {
 			CDEBUG(D_OTHER, "processing rec 0x%p type %#x\n",
 			       rec, rec->lrh_type);
 
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_cat.c b/drivers/staging/lustre/lustre/obdclass/llog_cat.c
index a82a2950295a..ce8e2f6f002a 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_cat.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog_cat.c
@@ -63,11 +63,13 @@ static int llog_cat_id2handle(const struct lu_env *env,
 			      struct llog_logid *logid)
 {
 	struct llog_handle	*loghandle;
+	enum llog_flag fmt;
 	int			 rc = 0;
 
 	if (!cathandle)
 		return -EBADF;
 
+	fmt = cathandle->lgh_hdr->llh_flags & LLOG_F_EXT_MASK;
 	down_write(&cathandle->lgh_lock);
 	list_for_each_entry(loghandle, &cathandle->u.chd.chd_head,
 			    u.phd.phd_entry) {
@@ -99,7 +101,7 @@ static int llog_cat_id2handle(const struct lu_env *env,
 		return rc;
 	}
 
-	rc = llog_init_handle(env, loghandle, LLOG_F_IS_PLAIN, NULL);
+	rc = llog_init_handle(env, loghandle, fmt | LLOG_F_IS_PLAIN, NULL);
 	if (rc < 0) {
 		llog_close(env, loghandle);
 		loghandle = NULL;
@@ -107,7 +109,7 @@ static int llog_cat_id2handle(const struct lu_env *env,
 	}
 
 	down_write(&cathandle->lgh_lock);
-	list_add(&loghandle->u.phd.phd_entry, &cathandle->u.chd.chd_head);
+	list_add_tail(&loghandle->u.phd.phd_entry, &cathandle->u.chd.chd_head);
 	up_write(&cathandle->lgh_lock);
 
 	loghandle->u.phd.phd_cat_handle = cathandle;
@@ -123,7 +125,6 @@ out:
 int llog_cat_close(const struct lu_env *env, struct llog_handle *cathandle)
 {
 	struct llog_handle	*loghandle, *n;
-	int			 rc;
 
 	list_for_each_entry_safe(loghandle, n, &cathandle->u.chd.chd_head,
 				 u.phd.phd_entry) {
@@ -134,8 +135,7 @@ int llog_cat_close(const struct lu_env *env, struct llog_handle *cathandle)
 	/* if handle was stored in ctxt, remove it too */
 	if (cathandle->lgh_ctxt->loc_handle == cathandle)
 		cathandle->lgh_ctxt->loc_handle = NULL;
-	rc = llog_close(env, cathandle);
-	return rc;
+	return llog_close(env, cathandle);
 }
 EXPORT_SYMBOL(llog_cat_close);
 
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_internal.h b/drivers/staging/lustre/lustre/obdclass/llog_internal.h
index f7949525d952..21a93c73756a 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_internal.h
+++ b/drivers/staging/lustre/lustre/obdclass/llog_internal.h
@@ -70,4 +70,9 @@ int llog_process_or_fork(const struct lu_env *env,
 			 llog_cb_t cb, void *data, void *catdata, bool fork);
 int llog_cat_cleanup(const struct lu_env *env, struct llog_handle *cathandle,
 		     struct llog_handle *loghandle, int index);
+
+static inline struct llog_rec_hdr *llog_rec_hdr_next(struct llog_rec_hdr *rec)
+{
+	return (struct llog_rec_hdr *)((char *)rec + rec->lrh_len);
+}
 #endif
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_obd.c b/drivers/staging/lustre/lustre/obdclass/llog_obd.c
index 6ace7e097859..a4277d684614 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_obd.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog_obd.c
@@ -210,7 +210,6 @@ LU_KEY_INIT_FINI(llog, struct llog_thread_info);
 /* context key: llog_thread_key */
 LU_CONTEXT_KEY_DEFINE(llog, LCT_MD_THREAD | LCT_MG_THREAD | LCT_LOCAL);
 LU_KEY_INIT_GENERIC(llog);
-EXPORT_SYMBOL(llog_thread_key);
 
 int llog_info_init(void)
 {
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_swab.c b/drivers/staging/lustre/lustre/obdclass/llog_swab.c
index f7b9b190350c..8c4c1b3f1b45 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_swab.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog_swab.c
@@ -172,20 +172,23 @@ void lustre_swab_llog_rec(struct llog_rec_hdr *rec)
 		__swab64s(&cr->cr.cr_time);
 		lustre_swab_lu_fid(&cr->cr.cr_tfid);
 		lustre_swab_lu_fid(&cr->cr.cr_pfid);
-		if (CHANGELOG_REC_EXTENDED(&cr->cr)) {
-			struct llog_changelog_ext_rec *ext =
-				(struct llog_changelog_ext_rec *)rec;
-
-			lustre_swab_lu_fid(&ext->cr.cr_sfid);
-			lustre_swab_lu_fid(&ext->cr.cr_spfid);
-			tail = &ext->cr_tail;
-		} else {
-			tail = &cr->cr_tail;
+		if (cr->cr.cr_flags & CLF_RENAME) {
+			struct changelog_ext_rename *rnm =
+				changelog_rec_rename(&cr->cr);
+
+			lustre_swab_lu_fid(&rnm->cr_sfid);
+			lustre_swab_lu_fid(&rnm->cr_spfid);
 		}
-		tail = (struct llog_rec_tail *)((char *)tail +
+		/*
+		 * Because the tail follows a variable-length structure we need
+		 * to compute its location at runtime
+		 */
+		tail = (struct llog_rec_tail *)((char *)&cr->cr +
+						changelog_rec_size(&cr->cr) +
 						cr->cr.cr_namelen);
 		break;
 	}
+
 	case CHANGELOG_USER_REC:
 	{
 		struct llog_changelog_user_rec *cur =
@@ -224,6 +227,7 @@ void lustre_swab_llog_rec(struct llog_rec_hdr *rec)
 		__swab32s(&lsr->lsr_uid_h);
 		__swab32s(&lsr->lsr_gid);
 		__swab32s(&lsr->lsr_gid_h);
+		__swab64s(&lsr->lsr_valid);
 		tail = &lsr->lsr_tail;
 		break;
 	}
@@ -343,7 +347,6 @@ void lustre_swab_lustre_cfg(struct lustre_cfg *lcfg)
 
 	print_lustre_cfg(lcfg);
 }
-EXPORT_SYMBOL(lustre_swab_lustre_cfg);
 
 /* used only for compatibility with old on-disk cfg_marker data */
 struct cfg_marker32 {
@@ -403,4 +406,3 @@ void lustre_swab_cfg_marker(struct cfg_marker *marker, int swab, int size)
 		__swab64s(&marker->cm_canceltime);
 	}
 }
-EXPORT_SYMBOL(lustre_swab_cfg_marker);
diff --git a/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c b/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
index 279b625f1afe..852a5acfefab 100644
--- a/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
+++ b/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
@@ -96,6 +96,12 @@ static const char * const obd_connect_names[] = {
 	"pingless",
 	"flock_deadlock",
 	"disp_stripe",
+	"open_by_fid",
+	"lfsck",
+	"unknown",
+	"unlink_close",
+	"unknown",
+	"dir_stripe",
 	"unknown",
 	NULL
 };
@@ -309,7 +315,7 @@ struct dentry *ldebugfs_add_simple(struct dentry *root,
 }
 EXPORT_SYMBOL_GPL(ldebugfs_add_simple);
 
-static struct file_operations lprocfs_generic_fops = { };
+static const struct file_operations lprocfs_generic_fops = { };
 
 int ldebugfs_add_vars(struct dentry *parent,
 		      struct lprocfs_vars *list,
@@ -615,7 +621,6 @@ void lprocfs_stats_collect(struct lprocfs_stats *stats, int idx,
 
 	lprocfs_stats_unlock(stats, LPROCFS_GET_NUM_CPU, &flags);
 }
-EXPORT_SYMBOL(lprocfs_stats_collect);
 
 /**
  * Append a space separated list of current set flags to str.
@@ -1043,7 +1048,6 @@ int lprocfs_stats_alloc_one(struct lprocfs_stats *stats, unsigned int cpuid)
 	}
 	return rc;
 }
-EXPORT_SYMBOL(lprocfs_stats_alloc_one);
 
 struct lprocfs_stats *lprocfs_alloc_stats(unsigned int num,
 					  enum lprocfs_stats_flags flags)
@@ -1547,6 +1551,146 @@ void lprocfs_oh_clear(struct obd_histogram *oh)
 }
 EXPORT_SYMBOL(lprocfs_oh_clear);
 
+int lprocfs_wr_root_squash(const char __user *buffer, unsigned long count,
+			   struct root_squash_info *squash, char *name)
+{
+	char kernbuf[64], *tmp, *errmsg;
+	unsigned long uid, gid;
+	int rc;
+
+	if (count >= sizeof(kernbuf)) {
+		errmsg = "string too long";
+		rc = -EINVAL;
+		goto failed_noprint;
+	}
+	if (copy_from_user(kernbuf, buffer, count)) {
+		errmsg = "bad address";
+		rc = -EFAULT;
+		goto failed_noprint;
+	}
+	kernbuf[count] = '\0';
+
+	/* look for uid gid separator */
+	tmp = strchr(kernbuf, ':');
+	if (!tmp) {
+		errmsg = "needs uid:gid format";
+		rc = -EINVAL;
+		goto failed;
+	}
+	*tmp = '\0';
+	tmp++;
+
+	/* parse uid */
+	if (kstrtoul(kernbuf, 0, &uid) != 0) {
+		errmsg = "bad uid";
+		rc = -EINVAL;
+		goto failed;
+	}
+	/* parse gid */
+	if (kstrtoul(tmp, 0, &gid) != 0) {
+		errmsg = "bad gid";
+		rc = -EINVAL;
+		goto failed;
+	}
+
+	squash->rsi_uid = uid;
+	squash->rsi_gid = gid;
+
+	LCONSOLE_INFO("%s: root_squash is set to %u:%u\n",
+		      name, squash->rsi_uid, squash->rsi_gid);
+	return count;
+
+failed:
+	if (tmp) {
+		tmp--;
+		*tmp = ':';
+	}
+	CWARN("%s: failed to set root_squash to \"%s\", %s, rc = %d\n",
+	      name, kernbuf, errmsg, rc);
+	return rc;
+failed_noprint:
+	CWARN("%s: failed to set root_squash due to %s, rc = %d\n",
+	      name, errmsg, rc);
+	return rc;
+}
+EXPORT_SYMBOL(lprocfs_wr_root_squash);
+
+int lprocfs_wr_nosquash_nids(const char __user *buffer, unsigned long count,
+			     struct root_squash_info *squash, char *name)
+{
+	char *kernbuf = NULL, *errmsg;
+	struct list_head tmp;
+	int len = count;
+	int rc;
+
+	if (count > 4096) {
+		errmsg = "string too long";
+		rc = -EINVAL;
+		goto failed;
+	}
+
+	kernbuf = kzalloc(count + 1, GFP_NOFS);
+	if (!kernbuf) {
+		errmsg = "no memory";
+		rc = -ENOMEM;
+		goto failed;
+	}
+
+	if (copy_from_user(kernbuf, buffer, count)) {
+		errmsg = "bad address";
+		rc = -EFAULT;
+		goto failed;
+	}
+	kernbuf[count] = '\0';
+
+	if (count > 0 && kernbuf[count - 1] == '\n')
+		len = count - 1;
+
+	if ((len == 4 && !strncmp(kernbuf, "NONE", len)) ||
+	    (len == 5 && !strncmp(kernbuf, "clear", len))) {
+		/* empty string is special case */
+		down_write(&squash->rsi_sem);
+		if (!list_empty(&squash->rsi_nosquash_nids))
+			cfs_free_nidlist(&squash->rsi_nosquash_nids);
+		up_write(&squash->rsi_sem);
+		LCONSOLE_INFO("%s: nosquash_nids is cleared\n", name);
+		kfree(kernbuf);
+		return count;
+	}
+
+	INIT_LIST_HEAD(&tmp);
+	if (cfs_parse_nidlist(kernbuf, count, &tmp) <= 0) {
+		errmsg = "can't parse";
+		rc = -EINVAL;
+		goto failed;
+	}
+	LCONSOLE_INFO("%s: nosquash_nids set to %s\n",
+		      name, kernbuf);
+	kfree(kernbuf);
+	kernbuf = NULL;
+
+	down_write(&squash->rsi_sem);
+	if (!list_empty(&squash->rsi_nosquash_nids))
+		cfs_free_nidlist(&squash->rsi_nosquash_nids);
+	list_splice(&tmp, &squash->rsi_nosquash_nids);
+	up_write(&squash->rsi_sem);
+
+	return count;
+
+failed:
+	if (kernbuf) {
+		CWARN("%s: failed to set nosquash_nids to \"%s\", %s rc = %d\n",
+		      name, kernbuf, errmsg, rc);
+		kfree(kernbuf);
+		kernbuf = NULL;
+	} else {
+		CWARN("%s: failed to set nosquash_nids due to %s rc = %d\n",
+		      name, errmsg, rc);
+	}
+	return rc;
+}
+EXPORT_SYMBOL(lprocfs_wr_nosquash_nids);
+
 static ssize_t lustre_attr_show(struct kobject *kobj,
 				struct attribute *attr, char *buf)
 {
diff --git a/drivers/staging/lustre/lustre/obdclass/lu_object.c b/drivers/staging/lustre/lustre/obdclass/lu_object.c
index 9b03059f34d6..054e567e6c8d 100644
--- a/drivers/staging/lustre/lustre/obdclass/lu_object.c
+++ b/drivers/staging/lustre/lustre/obdclass/lu_object.c
@@ -55,6 +55,34 @@
 #include "../include/lu_ref.h"
 #include <linux/list.h>
 
+enum {
+	LU_CACHE_PERCENT_MAX	 = 50,
+	LU_CACHE_PERCENT_DEFAULT = 20
+};
+
+#define LU_CACHE_NR_MAX_ADJUST		128
+#define LU_CACHE_NR_UNLIMITED		-1
+#define LU_CACHE_NR_DEFAULT		LU_CACHE_NR_UNLIMITED
+#define LU_CACHE_NR_LDISKFS_LIMIT	LU_CACHE_NR_UNLIMITED
+#define LU_CACHE_NR_ZFS_LIMIT		256
+
+#define LU_SITE_BITS_MIN	12
+#define LU_SITE_BITS_MAX	24
+/**
+ * total 256 buckets, we don't want too many buckets because:
+ * - consume too much memory
+ * - avoid unbalanced LRU list
+ */
+#define LU_SITE_BKT_BITS	8
+
+static unsigned int lu_cache_percent = LU_CACHE_PERCENT_DEFAULT;
+module_param(lu_cache_percent, int, 0644);
+MODULE_PARM_DESC(lu_cache_percent, "Percentage of memory to be used as lu_object cache");
+
+static long lu_cache_nr = LU_CACHE_NR_DEFAULT;
+module_param(lu_cache_nr, long, 0644);
+MODULE_PARM_DESC(lu_cache_nr, "Maximum number of objects in lu_object cache");
+
 static void lu_object_free(const struct lu_env *env, struct lu_object *o);
 static __u32 ls_stats_read(struct lprocfs_stats *stats, int idx);
 
@@ -310,10 +338,10 @@ int lu_site_purge(const struct lu_env *env, struct lu_site *s, int nr)
 	struct cfs_hash_bd	    bd2;
 	struct list_head	       dispose;
 	int		      did_sth;
-	int		      start;
+	unsigned int start;
 	int		      count;
 	int		      bnr;
-	int		      i;
+	unsigned int i;
 
 	if (OBD_FAIL_CHECK(OBD_FAIL_OBD_NO_LRU))
 		return 0;
@@ -324,8 +352,13 @@ int lu_site_purge(const struct lu_env *env, struct lu_site *s, int nr)
 	 * the dispose list, removing them from LRU and hash table.
 	 */
 	start = s->ls_purge_start;
-	bnr = (nr == ~0) ? -1 : nr / CFS_HASH_NBKT(s->ls_obj_hash) + 1;
+	bnr = (nr == ~0) ? -1 : nr / (int)CFS_HASH_NBKT(s->ls_obj_hash) + 1;
  again:
+	/*
+	 * It doesn't make any sense to make purge threads parallel, that can
+	 * only bring troubles to us. See LU-5331.
+	 */
+	mutex_lock(&s->ls_purge_mutex);
 	did_sth = 0;
 	cfs_hash_for_each_bucket(s->ls_obj_hash, &bd, i) {
 		if (i < start)
@@ -371,6 +404,7 @@ int lu_site_purge(const struct lu_env *env, struct lu_site *s, int nr)
 		if (nr == 0)
 			break;
 	}
+	mutex_unlock(&s->ls_purge_mutex);
 
 	if (nr != 0 && did_sth && start != 0) {
 		start = 0; /* restart from the first bucket */
@@ -573,6 +607,27 @@ static struct lu_object *lu_object_find(const struct lu_env *env,
 	return lu_object_find_at(env, dev->ld_site->ls_top_dev, f, conf);
 }
 
+/*
+ * Limit the lu_object cache to a maximum of lu_cache_nr objects.  Because
+ * the calculation for the number of objects to reclaim is not covered by
+ * a lock the maximum number of objects is capped by LU_CACHE_MAX_ADJUST.
+ * This ensures that many concurrent threads will not accidentally purge
+ * the entire cache.
+ */
+static void lu_object_limit(const struct lu_env *env, struct lu_device *dev)
+{
+	__u64 size, nr;
+
+	if (lu_cache_nr == LU_CACHE_NR_UNLIMITED)
+		return;
+
+	size = cfs_hash_size_get(dev->ld_site->ls_obj_hash);
+	nr = (__u64)lu_cache_nr;
+	if (size > nr)
+		lu_site_purge(env, dev->ld_site,
+			      min_t(__u64, size - nr, LU_CACHE_NR_MAX_ADJUST));
+}
+
 static struct lu_object *lu_object_new(const struct lu_env *env,
 				       struct lu_device *dev,
 				       const struct lu_fid *f,
@@ -590,6 +645,9 @@ static struct lu_object *lu_object_new(const struct lu_env *env,
 	cfs_hash_bd_get_and_lock(hs, (void *)f, &bd, 1);
 	cfs_hash_bd_add_locked(hs, &bd, &o->lo_header->loh_hash);
 	cfs_hash_bd_unlock(hs, &bd, 1);
+
+	lu_object_limit(env, dev);
+
 	return o;
 }
 
@@ -656,6 +714,9 @@ static struct lu_object *lu_object_find_try(const struct lu_env *env,
 	if (likely(PTR_ERR(shadow) == -ENOENT)) {
 		cfs_hash_bd_add_locked(hs, &bd, &o->lo_header->loh_hash);
 		cfs_hash_bd_unlock(hs, &bd, 1);
+
+		lu_object_limit(env, dev);
+
 		return o;
 	}
 
@@ -706,13 +767,15 @@ struct lu_object *lu_object_find_slice(const struct lu_env *env,
 	struct lu_object *obj;
 
 	top = lu_object_find(env, dev, f, conf);
-	if (!IS_ERR(top)) {
-		obj = lu_object_locate(top->lo_header, dev->ld_type);
-		if (!obj)
-			lu_object_put(env, top);
-	} else {
-		obj = top;
+	if (IS_ERR(top))
+		return top;
+
+	obj = lu_object_locate(top->lo_header, dev->ld_type);
+	if (unlikely(!obj)) {
+		lu_object_put(env, top);
+		obj = ERR_PTR(-ENOENT);
 	}
+
 	return obj;
 }
 EXPORT_SYMBOL(lu_object_find_slice);
@@ -726,34 +789,31 @@ int lu_device_type_init(struct lu_device_type *ldt)
 {
 	int result = 0;
 
+	atomic_set(&ldt->ldt_device_nr, 0);
 	INIT_LIST_HEAD(&ldt->ldt_linkage);
 	if (ldt->ldt_ops->ldto_init)
 		result = ldt->ldt_ops->ldto_init(ldt);
-	if (result == 0)
+
+	if (!result) {
+		spin_lock(&obd_types_lock);
 		list_add(&ldt->ldt_linkage, &lu_device_types);
+		spin_unlock(&obd_types_lock);
+	}
+
 	return result;
 }
 EXPORT_SYMBOL(lu_device_type_init);
 
 void lu_device_type_fini(struct lu_device_type *ldt)
 {
+	spin_lock(&obd_types_lock);
 	list_del_init(&ldt->ldt_linkage);
+	spin_unlock(&obd_types_lock);
 	if (ldt->ldt_ops->ldto_fini)
 		ldt->ldt_ops->ldto_fini(ldt);
 }
 EXPORT_SYMBOL(lu_device_type_fini);
 
-void lu_types_stop(void)
-{
-	struct lu_device_type *ldt;
-
-	list_for_each_entry(ldt, &lu_device_types, ldt_linkage) {
-		if (ldt->ldt_device_nr == 0 && ldt->ldt_ops->ldto_stop)
-			ldt->ldt_ops->ldto_stop(ldt);
-	}
-}
-EXPORT_SYMBOL(lu_types_stop);
-
 /**
  * Global list of all sites on this node
  */
@@ -808,22 +868,14 @@ void lu_site_print(const struct lu_env *env, struct lu_site *s, void *cookie,
 }
 EXPORT_SYMBOL(lu_site_print);
 
-enum {
-	LU_CACHE_PERCENT_MAX     = 50,
-	LU_CACHE_PERCENT_DEFAULT = 20
-};
-
-static unsigned int lu_cache_percent = LU_CACHE_PERCENT_DEFAULT;
-module_param(lu_cache_percent, int, 0644);
-MODULE_PARM_DESC(lu_cache_percent, "Percentage of memory to be used as lu_object cache");
-
 /**
  * Return desired hash table order.
  */
-static int lu_htable_order(void)
+static unsigned long lu_htable_order(struct lu_device *top)
 {
+	unsigned long bits_max = LU_SITE_BITS_MAX;
 	unsigned long cache_size;
-	int bits;
+	unsigned long bits;
 
 	/*
 	 * Calculate hash table size, assuming that we want reasonable
@@ -854,7 +906,7 @@ static int lu_htable_order(void)
 	for (bits = 1; (1 << bits) < cache_size; ++bits) {
 		;
 	}
-	return bits;
+	return clamp_t(typeof(bits), bits, LU_SITE_BITS_MIN, bits_max);
 }
 
 static unsigned lu_obj_hop_hash(struct cfs_hash *hs,
@@ -930,28 +982,18 @@ static void lu_dev_add_linkage(struct lu_site *s, struct lu_device *d)
 /**
  * Initialize site \a s, with \a d as the top level device.
  */
-#define LU_SITE_BITS_MIN    12
-#define LU_SITE_BITS_MAX    19
-/**
- * total 256 buckets, we don't want too many buckets because:
- * - consume too much memory
- * - avoid unbalanced LRU list
- */
-#define LU_SITE_BKT_BITS    8
-
 int lu_site_init(struct lu_site *s, struct lu_device *top)
 {
 	struct lu_site_bkt_data *bkt;
 	struct cfs_hash_bd bd;
+	unsigned long bits;
+	unsigned long i;
 	char name[16];
-	int bits;
-	int i;
 
 	memset(s, 0, sizeof(*s));
-	bits = lu_htable_order();
-	snprintf(name, 16, "lu_site_%s", top->ld_type->ldt_name);
-	for (bits = min(max(LU_SITE_BITS_MIN, bits), LU_SITE_BITS_MAX);
-	     bits >= LU_SITE_BITS_MIN; bits--) {
+	mutex_init(&s->ls_purge_mutex);
+	snprintf(name, sizeof(name), "lu_site_%s", top->ld_type->ldt_name);
+	for (bits = lu_htable_order(top); bits >= LU_SITE_BITS_MIN; bits--) {
 		s->ls_obj_hash = cfs_hash_create(name, bits, bits,
 						 bits - LU_SITE_BKT_BITS,
 						 sizeof(*bkt), 0, 0,
@@ -959,13 +1001,14 @@ int lu_site_init(struct lu_site *s, struct lu_device *top)
 						 CFS_HASH_SPIN_BKTLOCK |
 						 CFS_HASH_NO_ITEMREF |
 						 CFS_HASH_DEPTH |
-						 CFS_HASH_ASSERT_EMPTY);
+						 CFS_HASH_ASSERT_EMPTY |
+						 CFS_HASH_COUNTER);
 		if (s->ls_obj_hash)
 			break;
 	}
 
 	if (!s->ls_obj_hash) {
-		CERROR("failed to create lu_site hash with bits: %d\n", bits);
+		CERROR("failed to create lu_site hash with bits: %lu\n", bits);
 		return -ENOMEM;
 	}
 
@@ -1082,8 +1125,10 @@ EXPORT_SYMBOL(lu_device_put);
  */
 int lu_device_init(struct lu_device *d, struct lu_device_type *t)
 {
-	if (t->ldt_device_nr++ == 0 && t->ldt_ops->ldto_start)
+	if (atomic_inc_return(&t->ldt_device_nr) == 1 &&
+	    t->ldt_ops->ldto_start)
 		t->ldt_ops->ldto_start(t);
+
 	memset(d, 0, sizeof(*d));
 	atomic_set(&d->ld_ref, 0);
 	d->ld_type = t;
@@ -1098,9 +1143,8 @@ EXPORT_SYMBOL(lu_device_init);
  */
 void lu_device_fini(struct lu_device *d)
 {
-	struct lu_device_type *t;
+	struct lu_device_type *t = d->ld_type;
 
-	t = d->ld_type;
 	if (d->ld_obd) {
 		d->ld_obd->obd_lu_dev = NULL;
 		d->ld_obd = NULL;
@@ -1109,8 +1153,10 @@ void lu_device_fini(struct lu_device *d)
 	lu_ref_fini(&d->ld_reference);
 	LASSERTF(atomic_read(&d->ld_ref) == 0,
 		 "Refcount is %u\n", atomic_read(&d->ld_ref));
-	LASSERT(t->ldt_device_nr > 0);
-	if (--t->ldt_device_nr == 0 && t->ldt_ops->ldto_stop)
+	LASSERT(atomic_read(&t->ldt_device_nr) > 0);
+
+	if (atomic_dec_and_test(&t->ldt_device_nr) &&
+	    t->ldt_ops->ldto_stop)
 		t->ldt_ops->ldto_stop(t);
 }
 EXPORT_SYMBOL(lu_device_fini);
@@ -1254,7 +1300,6 @@ void lu_stack_fini(const struct lu_env *env, struct lu_device *top)
 		}
 	}
 }
-EXPORT_SYMBOL(lu_stack_fini);
 
 enum {
 	/**
@@ -1281,7 +1326,7 @@ static unsigned key_set_version;
 int lu_context_key_register(struct lu_context_key *key)
 {
 	int result;
-	int i;
+	unsigned int i;
 
 	LASSERT(key->lct_init);
 	LASSERT(key->lct_fini);
@@ -1476,18 +1521,16 @@ void lu_context_key_quiesce(struct lu_context_key *key)
 		++key_set_version;
 	}
 }
-EXPORT_SYMBOL(lu_context_key_quiesce);
 
 void lu_context_key_revive(struct lu_context_key *key)
 {
 	key->lct_tags &= ~LCT_QUIESCENT;
 	++key_set_version;
 }
-EXPORT_SYMBOL(lu_context_key_revive);
 
 static void keys_fini(struct lu_context *ctx)
 {
-	int	i;
+	unsigned int i;
 
 	if (!ctx->lc_value)
 		return;
@@ -1501,7 +1544,7 @@ static void keys_fini(struct lu_context *ctx)
 
 static int keys_fill(struct lu_context *ctx)
 {
-	int i;
+	unsigned int i;
 
 	LINVRNT(ctx->lc_value);
 	for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
@@ -1614,7 +1657,7 @@ EXPORT_SYMBOL(lu_context_enter);
  */
 void lu_context_exit(struct lu_context *ctx)
 {
-	int i;
+	unsigned int i;
 
 	LINVRNT(ctx->lc_state == LCS_ENTERED);
 	ctx->lc_state = LCS_LEFT;
@@ -1642,7 +1685,6 @@ int lu_context_refill(struct lu_context *ctx)
 {
 	return likely(ctx->lc_version == key_set_version) ? 0 : keys_fill(ctx);
 }
-EXPORT_SYMBOL(lu_context_refill);
 
 /**
  * lu_ctx_tags/lu_ses_tags will be updated if there are new types of
@@ -1696,7 +1738,7 @@ static void lu_site_stats_get(struct cfs_hash *hs,
 			      struct lu_site_stats *stats, int populated)
 {
 	struct cfs_hash_bd bd;
-	int	   i;
+	unsigned int i;
 
 	cfs_hash_for_each_bucket(hs, &bd, i) {
 		struct lu_site_bkt_data *bkt = cfs_hash_bd_extra_get(hs, &bd);
@@ -1940,3 +1982,73 @@ void lu_kmem_fini(struct lu_kmem_descr *caches)
 	}
 }
 EXPORT_SYMBOL(lu_kmem_fini);
+
+void lu_buf_free(struct lu_buf *buf)
+{
+	LASSERT(buf);
+	if (buf->lb_buf) {
+		LASSERT(buf->lb_len > 0);
+		kvfree(buf->lb_buf);
+		buf->lb_buf = NULL;
+		buf->lb_len = 0;
+	}
+}
+EXPORT_SYMBOL(lu_buf_free);
+
+void lu_buf_alloc(struct lu_buf *buf, size_t size)
+{
+	LASSERT(buf);
+	LASSERT(!buf->lb_buf);
+	LASSERT(!buf->lb_len);
+	buf->lb_buf = libcfs_kvzalloc(size, GFP_NOFS);
+	if (likely(buf->lb_buf))
+		buf->lb_len = size;
+}
+EXPORT_SYMBOL(lu_buf_alloc);
+
+void lu_buf_realloc(struct lu_buf *buf, size_t size)
+{
+	lu_buf_free(buf);
+	lu_buf_alloc(buf, size);
+}
+EXPORT_SYMBOL(lu_buf_realloc);
+
+struct lu_buf *lu_buf_check_and_alloc(struct lu_buf *buf, size_t len)
+{
+	if (!buf->lb_buf && !buf->lb_len)
+		lu_buf_alloc(buf, len);
+
+	if ((len > buf->lb_len) && buf->lb_buf)
+		lu_buf_realloc(buf, len);
+
+	return buf;
+}
+EXPORT_SYMBOL(lu_buf_check_and_alloc);
+
+/**
+ * Increase the size of the \a buf.
+ * preserves old data in buffer
+ * old buffer remains unchanged on error
+ * \retval 0 or -ENOMEM
+ */
+int lu_buf_check_and_grow(struct lu_buf *buf, size_t len)
+{
+	char *ptr;
+
+	if (len <= buf->lb_len)
+		return 0;
+
+	ptr = libcfs_kvzalloc(len, GFP_NOFS);
+	if (!ptr)
+		return -ENOMEM;
+
+	/* Free the old buf */
+	if (buf->lb_buf) {
+		memcpy(ptr, buf->lb_buf, buf->lb_len);
+		kvfree(buf->lb_buf);
+	}
+
+	buf->lb_buf = ptr;
+	buf->lb_len = len;
+	return 0;
+}
diff --git a/drivers/staging/lustre/lustre/obdclass/lustre_handles.c b/drivers/staging/lustre/lustre/obdclass/lustre_handles.c
index 082f530c527c..c9445e5ec271 100644
--- a/drivers/staging/lustre/lustre/obdclass/lustre_handles.c
+++ b/drivers/staging/lustre/lustre/obdclass/lustre_handles.c
@@ -130,7 +130,7 @@ void class_handle_unhash(struct portals_handle *h)
 }
 EXPORT_SYMBOL(class_handle_unhash);
 
-void *class_handle2object(__u64 cookie)
+void *class_handle2object(__u64 cookie, const void *owner)
 {
 	struct handle_bucket *bucket;
 	struct portals_handle *h;
@@ -145,7 +145,7 @@ void *class_handle2object(__u64 cookie)
 
 	rcu_read_lock();
 	list_for_each_entry_rcu(h, &bucket->head, h_link) {
-		if (h->h_cookie != cookie)
+		if (h->h_cookie != cookie || h->h_owner != owner)
 			continue;
 
 		spin_lock(&h->h_lock);
@@ -164,8 +164,11 @@ EXPORT_SYMBOL(class_handle2object);
 
 void class_handle_free_cb(struct rcu_head *rcu)
 {
-	struct portals_handle *h = RCU2HANDLE(rcu);
-	void *ptr = (void *)(unsigned long)h->h_cookie;
+	struct portals_handle *h;
+	void *ptr;
+
+	h = container_of(rcu, struct portals_handle, h_rcu);
+	ptr = (void *)(unsigned long)h->h_cookie;
 
 	if (h->h_ops->hop_free)
 		h->h_ops->hop_free(ptr, h->h_size);
@@ -214,7 +217,7 @@ static int cleanup_all_handles(void)
 		struct portals_handle *h;
 
 		spin_lock(&handle_hash[i].lock);
-		list_for_each_entry_rcu(h, &(handle_hash[i].head), h_link) {
+		list_for_each_entry_rcu(h, &handle_hash[i].head, h_link) {
 			CERROR("force clean handle %#llx addr %p ops %p\n",
 			       h->h_cookie, h, h->h_ops);
 
diff --git a/drivers/staging/lustre/lustre/obdclass/lustre_peer.c b/drivers/staging/lustre/lustre/obdclass/lustre_peer.c
index 5974a9bf77c0..ffa740aa861c 100644
--- a/drivers/staging/lustre/lustre/obdclass/lustre_peer.c
+++ b/drivers/staging/lustre/lustre/obdclass/lustre_peer.c
@@ -139,7 +139,6 @@ int class_add_uuid(const char *uuid, __u64 nid)
 	}
 	return 0;
 }
-EXPORT_SYMBOL(class_add_uuid);
 
 /* Delete the nids for one uuid if specified, otherwise delete all */
 int class_del_uuid(const char *uuid)
diff --git a/drivers/staging/lustre/lustre/obdclass/obd_config.c b/drivers/staging/lustre/lustre/obdclass/obd_config.c
index 0eab1236501b..bbed1b72d52e 100644
--- a/drivers/staging/lustre/lustre/obdclass/obd_config.c
+++ b/drivers/staging/lustre/lustre/obdclass/obd_config.c
@@ -37,6 +37,7 @@
 #define DEBUG_SUBSYSTEM S_CLASS
 #include "../include/obd_class.h"
 #include <linux/string.h>
+#include "../include/lustre/lustre_ioctl.h"
 #include "../include/lustre_log.h"
 #include "../include/lprocfs_status.h"
 #include "../include/lustre_param.h"
@@ -237,7 +238,7 @@ static int class_attach(struct lustre_cfg *lcfg)
 	/* recovery data */
 	init_waitqueue_head(&obd->obd_evict_inprogress_waitq);
 
-	llog_group_init(&obd->obd_olg, FID_SEQ_LLOG);
+	llog_group_init(&obd->obd_olg);
 
 	obd->obd_conn_inprogress = 0;
 
@@ -250,15 +251,6 @@ static int class_attach(struct lustre_cfg *lcfg)
 	}
 	memcpy(obd->obd_uuid.uuid, uuid, len);
 
-	/* do the attach */
-	if (OBP(obd, attach)) {
-		rc = OBP(obd, attach)(obd, sizeof(*lcfg), lcfg);
-		if (rc) {
-			rc = -EINVAL;
-			goto out;
-		}
-	}
-
 	/* Detach drops this */
 	spin_lock(&obd->obd_dev_lock);
 	atomic_set(&obd->obd_refcount, 1);
@@ -422,17 +414,12 @@ static int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
 	}
 	/* Leave this on forever */
 	obd->obd_stopping = 1;
-
-	/* wait for already-arrived-connections to finish. */
-	while (obd->obd_conn_inprogress > 0) {
-		spin_unlock(&obd->obd_dev_lock);
-
-		cond_resched();
-
-		spin_lock(&obd->obd_dev_lock);
-	}
 	spin_unlock(&obd->obd_dev_lock);
 
+	while (obd->obd_conn_inprogress > 0)
+		yield();
+	smp_rmb();
+
 	if (lcfg->lcfg_bufcount >= 2 && LUSTRE_CFG_BUFLEN(lcfg, 1) > 0) {
 		for (flag = lustre_cfg_string(lcfg, 1); *flag != 0; flag++)
 			switch (*flag) {
@@ -526,11 +513,6 @@ void class_decref(struct obd_device *obd, const char *scope, const void *source)
 				CERROR("Cleanup %s returned %d\n",
 				       obd->obd_name, err);
 		}
-		if (OBP(obd, detach)) {
-			err = OBP(obd, detach)(obd);
-			if (err)
-				CERROR("Detach returned %d\n", err);
-		}
 		class_release_dev(obd);
 	}
 }
@@ -756,7 +738,7 @@ static int process_param2_config(struct lustre_cfg *lcfg)
 	}
 
 	start = ktime_get();
-	rc = call_usermodehelper(argv[0], argv, NULL, 1);
+	rc = call_usermodehelper(argv[0], argv, NULL, UMH_WAIT_PROC);
 	end = ktime_get();
 
 	if (rc < 0) {
@@ -1026,7 +1008,7 @@ int class_process_proc_param(char *prefix, struct lprocfs_vars *lvars,
 
 					oldfs = get_fs();
 					set_fs(KERNEL_DS);
-					rc = (var->fops->write)(&fakefile, sval,
+					rc = var->fops->write(&fakefile, sval,
 								vallen, NULL);
 					set_fs(oldfs);
 				}
@@ -1060,8 +1042,6 @@ int class_process_proc_param(char *prefix, struct lprocfs_vars *lvars,
 }
 EXPORT_SYMBOL(class_process_proc_param);
 
-extern int lustre_check_exclusion(struct super_block *sb, char *svname);
-
 /** Parse a configuration llog, doing various manipulations on them
  * for various reasons, (modifications for compatibility, skip obsolete
  * records, change uuids, etc), then class_process_config() resulting
@@ -1317,33 +1297,33 @@ static int class_config_parse_rec(struct llog_rec_hdr *rec, char *buf,
 	if (rc < 0)
 		return rc;
 
-	ptr += snprintf(ptr, end-ptr, "cmd=%05x ", lcfg->lcfg_command);
+	ptr += snprintf(ptr, end - ptr, "cmd=%05x ", lcfg->lcfg_command);
 	if (lcfg->lcfg_flags)
-		ptr += snprintf(ptr, end-ptr, "flags=%#08x ",
+		ptr += snprintf(ptr, end - ptr, "flags=%#08x ",
 				lcfg->lcfg_flags);
 
 	if (lcfg->lcfg_num)
-		ptr += snprintf(ptr, end-ptr, "num=%#08x ", lcfg->lcfg_num);
+		ptr += snprintf(ptr, end - ptr, "num=%#08x ", lcfg->lcfg_num);
 
 	if (lcfg->lcfg_nid) {
 		char nidstr[LNET_NIDSTR_SIZE];
 
 		libcfs_nid2str_r(lcfg->lcfg_nid, nidstr, sizeof(nidstr));
-		ptr += snprintf(ptr, end-ptr, "nid=%s(%#llx)\n     ",
+		ptr += snprintf(ptr, end - ptr, "nid=%s(%#llx)\n     ",
 				nidstr, lcfg->lcfg_nid);
 	}
 
 	if (lcfg->lcfg_command == LCFG_MARKER) {
 		struct cfg_marker *marker = lustre_cfg_buf(lcfg, 1);
 
-		ptr += snprintf(ptr, end-ptr, "marker=%d(%#x)%s '%s'",
+		ptr += snprintf(ptr, end - ptr, "marker=%d(%#x)%s '%s'",
 				marker->cm_step, marker->cm_flags,
 				marker->cm_tgtname, marker->cm_comment);
 	} else {
 		int i;
 
 		for (i = 0; i <  lcfg->lcfg_bufcount; i++) {
-			ptr += snprintf(ptr, end-ptr, "%d:%s  ", i,
+			ptr += snprintf(ptr, end - ptr, "%d:%s  ", i,
 					lustre_cfg_string(lcfg, i));
 		}
 	}
diff --git a/drivers/staging/lustre/lustre/obdclass/obd_mount.c b/drivers/staging/lustre/lustre/obdclass/obd_mount.c
index aa84a50e9904..0d3a3b05a637 100644
--- a/drivers/staging/lustre/lustre/obdclass/obd_mount.c
+++ b/drivers/staging/lustre/lustre/obdclass/obd_mount.c
@@ -37,11 +37,11 @@
  */
 
 #define DEBUG_SUBSYSTEM S_CLASS
-#define D_MOUNT (D_SUPER|D_CONFIG/*|D_WARNING */)
+#define D_MOUNT (D_SUPER | D_CONFIG/*|D_WARNING */)
 #define PRINT_CMD CDEBUG
 
 #include "../include/obd.h"
-#include "../include/linux/lustre_compat25.h"
+#include "../include/lustre_compat.h"
 #include "../include/obd_class.h"
 #include "../include/lustre/lustre_user.h"
 #include "../include/lustre_log.h"
@@ -68,7 +68,7 @@ static void (*kill_super_cb)(struct super_block *sb);
  *   this log, and is added to the mgc's list of logs to follow.
  */
 int lustre_process_log(struct super_block *sb, char *logname,
-		      struct config_llog_instance *cfg)
+		       struct config_llog_instance *cfg)
 {
 	struct lustre_cfg *lcfg;
 	struct lustre_cfg_bufs *bufs;
@@ -384,17 +384,15 @@ int lustre_start_mgc(struct super_block *sb)
 				  OBD_CONNECT_FULL20 | OBD_CONNECT_IMP_RECOV |
 				  OBD_CONNECT_LVB_TYPE;
 
-#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 2, 50, 0)
+#if OBD_OCD_VERSION(3, 0, 53, 0) > LUSTRE_VERSION_CODE
 	data->ocd_connect_flags |= OBD_CONNECT_MNE_SWAB;
-#else
-#warning "LU-1644: Remove old OBD_CONNECT_MNE_SWAB fixup and imp_need_mne_swab"
 #endif
 
 	if (lmd_is_client(lsi->lsi_lmd) &&
 	    lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)
 		data->ocd_connect_flags &= ~OBD_CONNECT_IMP_RECOV;
 	data->ocd_version = LUSTRE_VERSION_CODE;
-	rc = obd_connect(NULL, &exp, obd, &(obd->obd_uuid), data, NULL);
+	rc = obd_connect(NULL, &exp, obd, &obd->obd_uuid, data, NULL);
 	if (rc) {
 		CERROR("connect failed %d\n", rc);
 		goto out;
@@ -670,7 +668,6 @@ int lustre_common_put_super(struct super_block *sb)
 	}
 	/* Drop a ref to the mounted disk */
 	lustre_put_lsi(sb);
-	lu_types_stop();
 	return rc;
 }
 EXPORT_SYMBOL(lustre_common_put_super);
@@ -731,7 +728,7 @@ int lustre_check_exclusion(struct super_block *sb, char *svname)
 static int lmd_make_exclusion(struct lustre_mount_data *lmd, const char *ptr)
 {
 	const char *s1 = ptr, *s2;
-	__u32 index, *exclude_list;
+	__u32 index = 0, *exclude_list;
 	int rc = 0, devmax;
 
 	/* The shortest an ost name can be is 8 chars: -OST0000.
@@ -758,7 +755,7 @@ static int lmd_make_exclusion(struct lustre_mount_data *lmd, const char *ptr)
 			exclude_list[lmd->lmd_exclude_count++] = index;
 		else
 			CDEBUG(D_MOUNT, "ignoring exclude %.*s: type = %#x\n",
-			       (uint)(s2-s1), s1, rc);
+			       (uint)(s2 - s1), s1, rc);
 		s1 = s2;
 		/* now we are pointing at ':' (next exclude)
 		 * or ',' (end of excludes)
@@ -880,7 +877,7 @@ static int lmd_parse_mgs(struct lustre_mount_data *lmd, char **ptr)
  */
 static int lmd_parse(char *options, struct lustre_mount_data *lmd)
 {
-	char *s1, *s2, *devname = NULL;
+	char *s1, *s2, *s3, *devname = NULL;
 	struct lustre_mount_data *raw = (struct lustre_mount_data *)options;
 	int rc = 0;
 
@@ -913,6 +910,7 @@ static int lmd_parse(char *options, struct lustre_mount_data *lmd)
 		/* Skip whitespace and extra commas */
 		while (*s1 == ' ' || *s1 == ',')
 			s1++;
+		s3 = s1;
 
 		/* Client options are parsed in ll_options: eg. flock,
 		 * user_xattr, acl
@@ -970,6 +968,7 @@ static int lmd_parse(char *options, struct lustre_mount_data *lmd)
 			rc = lmd_parse_mgssec(lmd, s1 + 7);
 			if (rc)
 				goto invalid;
+			s3 = s2;
 			clear++;
 		/* ost exclusion list */
 		} else if (strncmp(s1, "exclude=", 8) == 0) {
@@ -990,10 +989,19 @@ static int lmd_parse(char *options, struct lustre_mount_data *lmd)
 			size_t length, params_length;
 			char *tail = strchr(s1 + 6, ',');
 
-			if (!tail)
+			if (!tail) {
 				length = strlen(s1);
-			else
-				length = tail - s1;
+			} else {
+				lnet_nid_t nid;
+				char *param_str = tail + 1;
+				int supplementary = 1;
+
+				while (!class_parse_nid_quiet(param_str, &nid,
+							      &param_str)) {
+					supplementary = 0;
+				}
+				length = param_str - s1 - supplementary;
+			}
 			length -= 6;
 			params_length = strlen(lmd->lmd_params);
 			if (params_length + length + 1 >= LMD_PARAMS_MAXLEN)
@@ -1001,6 +1009,7 @@ static int lmd_parse(char *options, struct lustre_mount_data *lmd)
 			strncat(lmd->lmd_params, s1 + 6, length);
 			lmd->lmd_params[params_length + length] = '\0';
 			strlcat(lmd->lmd_params, " ", LMD_PARAMS_MAXLEN);
+			s3 = s1 + 6 + length;
 			clear++;
 		} else if (strncmp(s1, "osd=", 4) == 0) {
 			rc = lmd_parse_string(&lmd->lmd_osd_type, s1 + 4);
@@ -1097,7 +1106,7 @@ static int lustre_fill_super(struct super_block *sb, void *data, int silent)
 	struct lustre_sb_info *lsi;
 	int rc;
 
-	CDEBUG(D_MOUNT|D_VFSTRACE, "VFS Op: sb %p\n", sb);
+	CDEBUG(D_MOUNT | D_VFSTRACE, "VFS Op: sb %p\n", sb);
 
 	lsi = lustre_init_lsi(sb);
 	if (!lsi)
@@ -1133,7 +1142,7 @@ static int lustre_fill_super(struct super_block *sb, void *data, int silent)
 		} else {
 			rc = lustre_start_mgc(sb);
 			if (rc) {
-				lustre_put_lsi(sb);
+				lustre_common_put_super(sb);
 				goto out;
 			}
 			/* Connect and start */
diff --git a/drivers/staging/lustre/lustre/obdclass/obdo.c b/drivers/staging/lustre/lustre/obdclass/obdo.c
index 8583a4a8c206..79104a66da96 100644
--- a/drivers/staging/lustre/lustre/obdclass/obdo.c
+++ b/drivers/staging/lustre/lustre/obdclass/obdo.c
@@ -112,7 +112,7 @@ void obdo_from_inode(struct obdo *dst, struct inode *src, u32 valid)
 }
 EXPORT_SYMBOL(obdo_from_inode);
 
-void obdo_to_ioobj(struct obdo *oa, struct obd_ioobj *ioobj)
+void obdo_to_ioobj(const struct obdo *oa, struct obd_ioobj *ioobj)
 {
 	ioobj->ioo_oid = oa->o_oi;
 	if (unlikely(!(oa->o_valid & OBD_MD_FLGROUP)))
@@ -125,7 +125,8 @@ void obdo_to_ioobj(struct obdo *oa, struct obd_ioobj *ioobj)
 }
 EXPORT_SYMBOL(obdo_to_ioobj);
 
-static void iattr_from_obdo(struct iattr *attr, struct obdo *oa, u32 valid)
+static void iattr_from_obdo(struct iattr *attr, const struct obdo *oa,
+			    u32 valid)
 {
 	valid &= oa->o_valid;
 
@@ -152,12 +153,14 @@ static void iattr_from_obdo(struct iattr *attr, struct obdo *oa, u32 valid)
 	}
 #if 0   /* you shouldn't be able to change a file's type with setattr */
 	if (valid & OBD_MD_FLTYPE) {
-		attr->ia_mode = (attr->ia_mode & ~S_IFMT)|(oa->o_mode & S_IFMT);
+		attr->ia_mode = (attr->ia_mode & ~S_IFMT) |
+				(oa->o_mode & S_IFMT);
 		attr->ia_valid |= ATTR_MODE;
 	}
 #endif
 	if (valid & OBD_MD_FLMODE) {
-		attr->ia_mode = (attr->ia_mode & S_IFMT)|(oa->o_mode & ~S_IFMT);
+		attr->ia_mode = (attr->ia_mode & S_IFMT) |
+				(oa->o_mode & ~S_IFMT);
 		attr->ia_valid |= ATTR_MODE;
 		if (!in_group_p(make_kgid(&init_user_ns, oa->o_gid)) &&
 		    !capable(CFS_CAP_FSETID))
@@ -173,7 +176,7 @@ static void iattr_from_obdo(struct iattr *attr, struct obdo *oa, u32 valid)
 	}
 }
 
-void md_from_obdo(struct md_op_data *op_data, struct obdo *oa, u32 valid)
+void md_from_obdo(struct md_op_data *op_data, const struct obdo *oa, u32 valid)
 {
 	iattr_from_obdo(&op_data->op_attr, oa, valid);
 	if (valid & OBD_MD_FLBLOCKS) {
diff --git a/drivers/staging/lustre/lustre/obdecho/echo_client.c b/drivers/staging/lustre/lustre/obdecho/echo_client.c
index 5b29c4a44fe5..505582ff4d1e 100644
--- a/drivers/staging/lustre/lustre/obdecho/echo_client.c
+++ b/drivers/staging/lustre/lustre/obdecho/echo_client.c
@@ -41,6 +41,7 @@
 #include "../include/cl_object.h"
 #include "../include/lustre_fid.h"
 #include "../include/lustre_acl.h"
+#include "../include/lustre/lustre_ioctl.h"
 #include "../include/lustre_net.h"
 
 #include "echo_internal.h"
@@ -64,14 +65,14 @@ struct echo_object {
 
 	struct echo_device     *eo_dev;
 	struct list_head	      eo_obj_chain;
-	struct lov_stripe_md   *eo_lsm;
+	struct lov_oinfo       *eo_oinfo;
 	atomic_t	    eo_npages;
 	int		     eo_deleted;
 };
 
 struct echo_object_conf {
 	struct cl_object_conf  eoc_cl;
-	struct lov_stripe_md **eoc_md;
+	struct lov_oinfo      **eoc_oinfo;
 };
 
 struct echo_page {
@@ -152,9 +153,6 @@ struct echo_object_conf *cl2echo_conf(const struct cl_object_conf *c)
 }
 
 /** @} echo_helpers */
-
-static struct echo_object *cl_echo_object_find(struct echo_device *d,
-					       struct lov_stripe_md **lsm);
 static int cl_echo_object_put(struct echo_object *eco);
 static int cl_echo_object_brw(struct echo_object *eco, int rw, u64 offset,
 			      struct page **pages, int npages, int async);
@@ -413,10 +411,13 @@ static int echo_object_init(const struct lu_env *env, struct lu_object *obj,
 	cconf = lu2cl_conf(conf);
 	econf = cl2echo_conf(cconf);
 
-	LASSERT(econf->eoc_md);
-	eco->eo_lsm = *econf->eoc_md;
-	/* clear the lsm pointer so that it won't get freed. */
-	*econf->eoc_md = NULL;
+	LASSERT(econf->eoc_oinfo);
+	/*
+	 * Transfer the oinfo pointer to eco that it won't be
+	 * freed.
+	 */
+	eco->eo_oinfo = *econf->eoc_oinfo;
+	*econf->eoc_oinfo = NULL;
 
 	eco->eo_dev = ed;
 	atomic_set(&eco->eo_npages, 0);
@@ -429,52 +430,6 @@ static int echo_object_init(const struct lu_env *env, struct lu_object *obj,
 	return 0;
 }
 
-/* taken from osc_unpackmd() */
-static int echo_alloc_memmd(struct echo_device *ed,
-			    struct lov_stripe_md **lsmp)
-{
-	int lsm_size;
-
-	/* If export is lov/osc then use their obd method */
-	if (ed->ed_next)
-		return obd_alloc_memmd(ed->ed_ec->ec_exp, lsmp);
-	/* OFD has no unpackmd method, do everything here */
-	lsm_size = lov_stripe_md_size(1);
-
-	LASSERT(!*lsmp);
-	*lsmp = kzalloc(lsm_size, GFP_NOFS);
-	if (!*lsmp)
-		return -ENOMEM;
-
-	(*lsmp)->lsm_oinfo[0] = kzalloc(sizeof(struct lov_oinfo), GFP_NOFS);
-	if (!(*lsmp)->lsm_oinfo[0]) {
-		kfree(*lsmp);
-		return -ENOMEM;
-	}
-
-	loi_init((*lsmp)->lsm_oinfo[0]);
-	(*lsmp)->lsm_maxbytes = LUSTRE_STRIPE_MAXBYTES;
-	ostid_set_seq_echo(&(*lsmp)->lsm_oi);
-
-	return lsm_size;
-}
-
-static int echo_free_memmd(struct echo_device *ed, struct lov_stripe_md **lsmp)
-{
-	int lsm_size;
-
-	/* If export is lov/osc then use their obd method */
-	if (ed->ed_next)
-		return obd_free_memmd(ed->ed_ec->ec_exp, lsmp);
-	/* OFD has no unpackmd method, do everything here */
-	lsm_size = lov_stripe_md_size(1);
-
-	kfree((*lsmp)->lsm_oinfo[0]);
-	kfree(*lsmp);
-	*lsmp = NULL;
-	return 0;
-}
-
 static void echo_object_free(const struct lu_env *env, struct lu_object *obj)
 {
 	struct echo_object *eco    = cl2echo_obj(lu2cl(obj));
@@ -489,8 +444,7 @@ static void echo_object_free(const struct lu_env *env, struct lu_object *obj)
 	lu_object_fini(obj);
 	lu_object_header_fini(obj->lo_header);
 
-	if (eco->eo_lsm)
-		echo_free_memmd(eco->eo_dev, &eco->eo_lsm);
+	kfree(eco->eo_oinfo);
 	kmem_cache_free(echo_object_kmem, eco);
 }
 
@@ -864,25 +818,21 @@ static struct lu_device_type echo_device_type = {
  */
 
 /* Interfaces to echo client obd device */
-static struct echo_object *cl_echo_object_find(struct echo_device *d,
-					       struct lov_stripe_md **lsmp)
+static struct echo_object *
+cl_echo_object_find(struct echo_device *d, const struct ost_id *oi)
 {
 	struct lu_env *env;
 	struct echo_thread_info *info;
 	struct echo_object_conf *conf;
-	struct lov_stripe_md    *lsm;
+	struct lov_oinfo *oinfo = NULL;
 	struct echo_object *eco;
 	struct cl_object   *obj;
 	struct lu_fid *fid;
 	int refcheck;
 	int rc;
 
-	LASSERT(lsmp);
-	lsm = *lsmp;
-	LASSERT(lsm);
-	LASSERTF(ostid_id(&lsm->lsm_oi) != 0, DOSTID"\n", POSTID(&lsm->lsm_oi));
-	LASSERTF(ostid_seq(&lsm->lsm_oi) == FID_SEQ_ECHO, DOSTID"\n",
-		 POSTID(&lsm->lsm_oi));
+	LASSERTF(ostid_id(oi), DOSTID "\n", POSTID(oi));
+	LASSERTF(ostid_seq(oi) == FID_SEQ_ECHO, DOSTID "\n", POSTID(oi));
 
 	/* Never return an object if the obd is to be freed. */
 	if (echo_dev2cl(d)->cd_lu_dev.ld_obd->obd_stopping)
@@ -895,16 +845,24 @@ static struct echo_object *cl_echo_object_find(struct echo_device *d,
 	info = echo_env_info(env);
 	conf = &info->eti_conf;
 	if (d->ed_next) {
-		struct lov_oinfo *oinfo = lsm->lsm_oinfo[0];
+		oinfo = kzalloc(sizeof(*oinfo), GFP_NOFS);
+		if (!oinfo) {
+			eco = ERR_PTR(-ENOMEM);
+			goto out;
+		}
 
-		LASSERT(oinfo);
-		oinfo->loi_oi = lsm->lsm_oi;
+		oinfo->loi_oi = *oi;
 		conf->eoc_cl.u.coc_oinfo = oinfo;
 	}
-	conf->eoc_md = lsmp;
+
+	/*
+	 * If echo_object_init() is successful then ownership of oinfo
+	 * is transferred to the object.
+	 */
+	conf->eoc_oinfo = &oinfo;
 
 	fid  = &info->eti_fid;
-	rc = ostid_to_fid(fid, &lsm->lsm_oi, 0);
+	rc = ostid_to_fid(fid, (struct ost_id *)oi, 0);
 	if (rc != 0) {
 		eco = ERR_PTR(rc);
 		goto out;
@@ -927,6 +885,7 @@ static struct echo_object *cl_echo_object_find(struct echo_device *d,
 	}
 
 out:
+	kfree(oinfo);
 	cl_env_put(env, &refcheck);
 	return eco;
 }
@@ -1051,7 +1010,7 @@ static int cl_echo_object_brw(struct echo_object *eco, int rw, u64 offset,
 	struct cl_io	    *io;
 	struct cl_page	  *clp;
 	struct lustre_handle    lh = { 0 };
-	int page_size = cl_page_size(obj);
+	size_t page_size = cl_page_size(obj);
 	int refcheck;
 	int rc;
 	int i;
@@ -1145,7 +1104,6 @@ static int echo_create_object(const struct lu_env *env, struct echo_device *ed,
 {
 	struct echo_object     *eco;
 	struct echo_client_obd *ec = ed->ed_ec;
-	struct lov_stripe_md   *lsm = NULL;
 	int		     rc;
 	int		     created = 0;
 
@@ -1156,30 +1114,19 @@ static int echo_create_object(const struct lu_env *env, struct echo_device *ed,
 		return -EINVAL;
 	}
 
-	rc = echo_alloc_memmd(ed, &lsm);
-	if (rc < 0) {
-		CERROR("Cannot allocate md: rc = %d\n", rc);
-		goto failed;
-	}
-
-	/* setup object ID here */
-	lsm->lsm_oi = oa->o_oi;
+	if (!ostid_id(&oa->o_oi))
+		ostid_set_id(&oa->o_oi, ++last_object_id);
 
-	if (ostid_id(&lsm->lsm_oi) == 0)
-		ostid_set_id(&lsm->lsm_oi, ++last_object_id);
-
-	rc = obd_create(env, ec->ec_exp, oa, &lsm, oti);
+	rc = obd_create(env, ec->ec_exp, oa, oti);
 	if (rc != 0) {
 		CERROR("Cannot create objects: rc = %d\n", rc);
 		goto failed;
 	}
 	created = 1;
 
-	/* See what object ID we were given */
-	oa->o_oi = lsm->lsm_oi;
 	oa->o_valid |= OBD_MD_FLID;
 
-	eco = cl_echo_object_find(ed, &lsm);
+	eco = cl_echo_object_find(ed, &oa->o_oi);
 	if (IS_ERR(eco)) {
 		rc = PTR_ERR(eco);
 		goto failed;
@@ -1190,9 +1137,7 @@ static int echo_create_object(const struct lu_env *env, struct echo_device *ed,
 
  failed:
 	if (created && rc)
-		obd_destroy(env, ec->ec_exp, oa, lsm, oti, NULL);
-	if (lsm)
-		echo_free_memmd(ed, &lsm);
+		obd_destroy(env, ec->ec_exp, oa, oti);
 	if (rc)
 		CERROR("create object failed with: rc = %d\n", rc);
 	return rc;
@@ -1201,32 +1146,21 @@ static int echo_create_object(const struct lu_env *env, struct echo_device *ed,
 static int echo_get_object(struct echo_object **ecop, struct echo_device *ed,
 			   struct obdo *oa)
 {
-	struct lov_stripe_md   *lsm = NULL;
 	struct echo_object     *eco;
 	int		     rc;
 
-	if ((oa->o_valid & OBD_MD_FLID) == 0 || ostid_id(&oa->o_oi) == 0) {
-		/* disallow use of object id 0 */
-		CERROR("No valid oid\n");
+	if (!(oa->o_valid & OBD_MD_FLID) || !(oa->o_valid & OBD_MD_FLGROUP) ||
+	    !ostid_id(&oa->o_oi)) {
+		CERROR("invalid oid " DOSTID "\n", POSTID(&oa->o_oi));
 		return -EINVAL;
 	}
 
-	rc = echo_alloc_memmd(ed, &lsm);
-	if (rc < 0)
-		return rc;
-
-	lsm->lsm_oi = oa->o_oi;
-	if (!(oa->o_valid & OBD_MD_FLGROUP))
-		ostid_set_seq_echo(&lsm->lsm_oi);
-
 	rc = 0;
-	eco = cl_echo_object_find(ed, &lsm);
+	eco = cl_echo_object_find(ed, &oa->o_oi);
 	if (!IS_ERR(eco))
 		*ecop = eco;
 	else
 		rc = PTR_ERR(eco);
-	if (lsm)
-		echo_free_memmd(ed, &lsm);
 	return rc;
 }
 
@@ -1436,13 +1370,12 @@ static int echo_client_prep_commit(const struct lu_env *env,
 			npages = tot_pages;
 
 		for (i = 0; i < npages; i++, off += PAGE_SIZE) {
-			rnb[i].offset = off;
-			rnb[i].len = PAGE_SIZE;
-			rnb[i].flags = brw_flags;
+			rnb[i].rnb_offset = off;
+			rnb[i].rnb_len = PAGE_SIZE;
+			rnb[i].rnb_flags = brw_flags;
 		}
 
 		ioo.ioo_bufcnt = npages;
-		oti->oti_transno = 0;
 
 		lpages = npages;
 		ret = obd_preprw(env, rw, exp, oa, 1, &ioo, rnb, &lpages,
@@ -1452,14 +1385,14 @@ static int echo_client_prep_commit(const struct lu_env *env,
 		LASSERT(lpages == npages);
 
 		for (i = 0; i < lpages; i++) {
-			struct page *page = lnb[i].page;
+			struct page *page = lnb[i].lnb_page;
 
 			/* read past eof? */
-			if (!page  && lnb[i].rc == 0)
+			if (!page  && lnb[i].lnb_rc == 0)
 				continue;
 
 			if (async)
-				lnb[i].flags |= OBD_BRW_ASYNC;
+				lnb[i].lnb_flags |= OBD_BRW_ASYNC;
 
 			if (ostid_id(&oa->o_oi) == ECHO_PERSISTENT_OBJID ||
 			    (oa->o_valid & OBD_MD_FLFLAGS) == 0 ||
@@ -1469,13 +1402,13 @@ static int echo_client_prep_commit(const struct lu_env *env,
 			if (rw == OBD_BRW_WRITE)
 				echo_client_page_debug_setup(page, rw,
 							    ostid_id(&oa->o_oi),
-							     rnb[i].offset,
-							     rnb[i].len);
+							     rnb[i].rnb_offset,
+							     rnb[i].rnb_len);
 			else
 				echo_client_page_debug_check(page,
 							    ostid_id(&oa->o_oi),
-							     rnb[i].offset,
-							     rnb[i].len);
+							     rnb[i].rnb_offset,
+							     rnb[i].rnb_len);
 		}
 
 		ret = obd_commitrw(env, rw, exp, oa, 1, &ioo,
@@ -1613,8 +1546,7 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 
 		rc = echo_get_object(&eco, ed, oa);
 		if (rc == 0) {
-			rc = obd_destroy(env, ec->ec_exp, oa, NULL,
-					 &dummy_oti, NULL);
+			rc = obd_destroy(env, ec->ec_exp, oa, &dummy_oti);
 			if (rc == 0)
 				eco->eo_deleted = 1;
 			echo_put_object(eco);
diff --git a/drivers/staging/lustre/lustre/obdecho/echo_internal.h b/drivers/staging/lustre/lustre/obdecho/echo_internal.h
index f5034a253f6d..966414fd5424 100644
--- a/drivers/staging/lustre/lustre/obdecho/echo_internal.h
+++ b/drivers/staging/lustre/lustre/obdecho/echo_internal.h
@@ -33,9 +33,9 @@
 
 /* The persistent object (i.e. actually stores stuff!) */
 #define ECHO_PERSISTENT_OBJID    1ULL
-#define ECHO_PERSISTENT_SIZE     ((__u64)(1<<20))
+#define ECHO_PERSISTENT_SIZE     ((__u64)(1 << 20))
 
 /* block size to use for data verification */
-#define OBD_ECHO_BLOCK_SIZE	(4<<10)
+#define OBD_ECHO_BLOCK_SIZE	(4 << 10)
 
 #endif
diff --git a/drivers/staging/lustre/lustre/osc/lproc_osc.c b/drivers/staging/lustre/lustre/osc/lproc_osc.c
index 7e83d395b998..f0062d44ee03 100644
--- a/drivers/staging/lustre/lustre/osc/lproc_osc.c
+++ b/drivers/staging/lustre/lustre/osc/lproc_osc.c
@@ -119,6 +119,7 @@ static ssize_t max_rpcs_in_flight_store(struct kobject *kobj,
 
 	spin_lock(&cli->cl_loi_list_lock);
 	cli->cl_max_rpcs_in_flight = val;
+	client_adjust_max_dirty(cli);
 	spin_unlock(&cli->cl_loi_list_lock);
 
 	return count;
@@ -136,10 +137,10 @@ static ssize_t max_dirty_mb_show(struct kobject *kobj,
 	int mult;
 
 	spin_lock(&cli->cl_loi_list_lock);
-	val = cli->cl_dirty_max;
+	val = cli->cl_dirty_max_pages;
 	spin_unlock(&cli->cl_loi_list_lock);
 
-	mult = 1 << 20;
+	mult = 1 << (20 - PAGE_SHIFT);
 	return lprocfs_read_frac_helper(buf, PAGE_SIZE, val, mult);
 }
 
@@ -166,7 +167,7 @@ static ssize_t max_dirty_mb_store(struct kobject *kobj,
 		return -ERANGE;
 
 	spin_lock(&cli->cl_loi_list_lock);
-	cli->cl_dirty_max = (u32)(pages_number << PAGE_SHIFT);
+	cli->cl_dirty_max_pages = pages_number;
 	osc_wake_cache_waiters(cli);
 	spin_unlock(&cli->cl_loi_list_lock);
 
@@ -181,11 +182,11 @@ static int osc_cached_mb_seq_show(struct seq_file *m, void *v)
 	int shift = 20 - PAGE_SHIFT;
 
 	seq_printf(m,
-		   "used_mb: %d\n"
-		   "busy_cnt: %d\n",
-		   (atomic_read(&cli->cl_lru_in_list) +
-		    atomic_read(&cli->cl_lru_busy)) >> shift,
-		   atomic_read(&cli->cl_lru_busy));
+		   "used_mb: %ld\n"
+		   "busy_cnt: %ld\n",
+		   (atomic_long_read(&cli->cl_lru_in_list) +
+		    atomic_long_read(&cli->cl_lru_busy)) >> shift,
+		   atomic_long_read(&cli->cl_lru_busy));
 
 	return 0;
 }
@@ -197,8 +198,10 @@ static ssize_t osc_cached_mb_seq_write(struct file *file,
 {
 	struct obd_device *dev = ((struct seq_file *)file->private_data)->private;
 	struct client_obd *cli = &dev->u.cli;
-	int pages_number, mult, rc;
+	long pages_number, rc;
 	char kernbuf[128];
+	int mult;
+	u64 val;
 
 	if (count >= sizeof(kernbuf))
 		return -EINVAL;
@@ -210,14 +213,18 @@ static ssize_t osc_cached_mb_seq_write(struct file *file,
 	mult = 1 << (20 - PAGE_SHIFT);
 	buffer += lprocfs_find_named_value(kernbuf, "used_mb:", &count) -
 		  kernbuf;
-	rc = lprocfs_write_frac_helper(buffer, count, &pages_number, mult);
+	rc = lprocfs_write_frac_u64_helper(buffer, count, &val, mult);
 	if (rc)
 		return rc;
 
+	if (val > LONG_MAX)
+		return -ERANGE;
+	pages_number = (long)val;
+
 	if (pages_number < 0)
 		return -ERANGE;
 
-	rc = atomic_read(&cli->cl_lru_in_list) - pages_number;
+	rc = atomic_long_read(&cli->cl_lru_in_list) - pages_number;
 	if (rc > 0) {
 		struct lu_env *env;
 		int refcheck;
@@ -244,7 +251,7 @@ static ssize_t cur_dirty_bytes_show(struct kobject *kobj,
 	int len;
 
 	spin_lock(&cli->cl_loi_list_lock);
-	len = sprintf(buf, "%lu\n", cli->cl_dirty);
+	len = sprintf(buf, "%lu\n", cli->cl_dirty_pages << PAGE_SHIFT);
 	spin_unlock(&cli->cl_loi_list_lock);
 
 	return len;
@@ -583,6 +590,7 @@ static ssize_t max_pages_per_rpc_store(struct kobject *kobj,
 	}
 	spin_lock(&cli->cl_loi_list_lock);
 	cli->cl_max_pages_per_rpc = val;
+	client_adjust_max_dirty(cli);
 	spin_unlock(&cli->cl_loi_list_lock);
 
 	return count;
@@ -596,13 +604,14 @@ static ssize_t unstable_stats_show(struct kobject *kobj,
 	struct obd_device *dev = container_of(kobj, struct obd_device,
 					      obd_kobj);
 	struct client_obd *cli = &dev->u.cli;
-	int pages, mb;
+	long pages;
+	int mb;
 
-	pages = atomic_read(&cli->cl_unstable_count);
+	pages = atomic_long_read(&cli->cl_unstable_count);
 	mb = (pages * PAGE_SIZE) >> 20;
 
-	return sprintf(buf, "unstable_pages: %8d\n"
-		       "unstable_mb:    %8d\n", pages, mb);
+	return sprintf(buf, "unstable_pages: %20ld\n"
+		       "unstable_mb:              %10d\n", pages, mb);
 }
 LUSTRE_RO_ATTR(unstable_stats);
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_cache.c b/drivers/staging/lustre/lustre/osc/osc_cache.c
index d011135802d5..4bbe219add98 100644
--- a/drivers/staging/lustre/lustre/osc/osc_cache.c
+++ b/drivers/staging/lustre/lustre/osc/osc_cache.c
@@ -44,7 +44,7 @@ static int extent_debug; /* set it to be true for more debug */
 
 static void osc_update_pending(struct osc_object *obj, int cmd, int delta);
 static int osc_extent_wait(const struct lu_env *env, struct osc_extent *ext,
-			   int state);
+			   enum osc_extent_state state);
 static void osc_ap_completion(const struct lu_env *env, struct client_obd *cli,
 			      struct osc_async_page *oap, int sent, int rc);
 static int osc_make_ready(const struct lu_env *env, struct osc_async_page *oap,
@@ -177,7 +177,7 @@ static int osc_extent_sanity_check0(struct osc_extent *ext,
 {
 	struct osc_object *obj = ext->oe_obj;
 	struct osc_async_page *oap;
-	int page_count;
+	size_t page_count;
 	int rc = 0;
 
 	if (!osc_object_is_locked(obj)) {
@@ -632,7 +632,7 @@ static inline int overlapped(struct osc_extent *ex1, struct osc_extent *ex2)
  */
 static struct osc_extent *osc_extent_find(const struct lu_env *env,
 					  struct osc_object *obj, pgoff_t index,
-					  int *grants)
+					  unsigned int *grants)
 {
 	struct client_obd *cli = osc_cli(obj);
 	struct osc_lock   *olck;
@@ -643,10 +643,10 @@ static struct osc_extent *osc_extent_find(const struct lu_env *env,
 	struct osc_extent *found = NULL;
 	pgoff_t chunk;
 	pgoff_t max_end;
-	int max_pages; /* max_pages_per_rpc */
-	int chunksize;
+	unsigned int max_pages; /* max_pages_per_rpc */
+	unsigned int chunksize;
 	int ppc_bits; /* pages per chunk bits */
-	int chunk_mask;
+	pgoff_t chunk_mask;
 	int rc;
 
 	cur = osc_extent_alloc(obj);
@@ -700,8 +700,8 @@ restart:
 	if (!ext)
 		ext = first_extent(obj);
 	while (ext) {
-		loff_t ext_chk_start = ext->oe_start >> ppc_bits;
-		loff_t ext_chk_end = ext->oe_end >> ppc_bits;
+		pgoff_t ext_chk_start = ext->oe_start >> ppc_bits;
+		pgoff_t ext_chk_end = ext->oe_end >> ppc_bits;
 
 		LASSERT(sanity_check_nolock(ext) == 0);
 		if (chunk > ext_chk_end + 1)
@@ -913,7 +913,7 @@ int osc_extent_finish(const struct lu_env *env, struct osc_extent *ext,
 	return 0;
 }
 
-static int extent_wait_cb(struct osc_extent *ext, int state)
+static int extent_wait_cb(struct osc_extent *ext, enum osc_extent_state state)
 {
 	int ret;
 
@@ -928,7 +928,7 @@ static int extent_wait_cb(struct osc_extent *ext, int state)
  * Wait for the extent's state to become @state.
  */
 static int osc_extent_wait(const struct lu_env *env, struct osc_extent *ext,
-			   int state)
+			   enum osc_extent_state state)
 {
 	struct osc_object *obj = ext->oe_obj;
 	struct l_wait_info lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(600), NULL,
@@ -958,8 +958,8 @@ static int osc_extent_wait(const struct lu_env *env, struct osc_extent *ext,
 	rc = l_wait_event(ext->oe_waitq, extent_wait_cb(ext, state), &lwi);
 	if (rc == -ETIMEDOUT) {
 		OSC_EXTENT_DUMP(D_ERROR, ext,
-			"%s: wait ext to %d timedout, recovery in progress?\n",
-			osc_export(obj)->exp_obd->obd_name, state);
+				"%s: wait ext to %u timedout, recovery in progress?\n",
+				osc_export(obj)->exp_obd->obd_name, state);
 
 		lwi = LWI_INTR(NULL, NULL);
 		rc = l_wait_event(ext->oe_waitq, extent_wait_cb(ext, state),
@@ -1099,7 +1099,7 @@ static int osc_extent_make_ready(const struct lu_env *env,
 	struct osc_async_page *oap;
 	struct osc_async_page *last = NULL;
 	struct osc_object *obj = ext->oe_obj;
-	int page_count = 0;
+	unsigned int page_count = 0;
 	int rc;
 
 	/* we're going to grab page lock, so object lock must not be taken. */
@@ -1140,9 +1140,11 @@ static int osc_extent_make_ready(const struct lu_env *env,
 	 * the size of file.
 	 */
 	if (!(last->oap_async_flags & ASYNC_COUNT_STABLE)) {
-		last->oap_count = osc_refresh_count(env, last, OBD_BRW_WRITE);
-		LASSERT(last->oap_count > 0);
-		LASSERT(last->oap_page_off + last->oap_count <= PAGE_SIZE);
+		int last_oap_count = osc_refresh_count(env, last, OBD_BRW_WRITE);
+
+		LASSERT(last_oap_count > 0);
+		LASSERT(last->oap_page_off + last_oap_count <= PAGE_SIZE);
+		last->oap_count = last_oap_count;
 		spin_lock(&last->oap_lock);
 		last->oap_async_flags |= ASYNC_COUNT_STABLE;
 		spin_unlock(&last->oap_lock);
@@ -1174,7 +1176,8 @@ static int osc_extent_make_ready(const struct lu_env *env,
  * called to expand the extent for the same IO. To expand the extent, the
  * page index must be in the same or next chunk of ext->oe_end.
  */
-static int osc_extent_expand(struct osc_extent *ext, pgoff_t index, int *grants)
+static int osc_extent_expand(struct osc_extent *ext, pgoff_t index,
+			     unsigned int *grants)
 {
 	struct osc_object *obj = ext->oe_obj;
 	struct client_obd *cli = osc_cli(obj);
@@ -1183,7 +1186,7 @@ static int osc_extent_expand(struct osc_extent *ext, pgoff_t index, int *grants)
 	pgoff_t chunk = index >> ppc_bits;
 	pgoff_t end_chunk;
 	pgoff_t end_index;
-	int chunksize = 1 << cli->cl_chunkbits;
+	unsigned int chunksize = 1 << cli->cl_chunkbits;
 	int rc = 0;
 
 	LASSERT(ext->oe_max_end >= index && ext->oe_start <= index);
@@ -1361,7 +1364,7 @@ static int osc_completion(const struct lu_env *env, struct osc_async_page *oap,
 	if (rc == 0 && srvlock) {
 		struct lu_device *ld = opg->ops_cl.cpl_obj->co_lu.lo_dev;
 		struct osc_stats *stats = &lu2osc_dev(ld)->od_stats;
-		int bytes = oap->oap_count;
+		size_t bytes = oap->oap_count;
 
 		if (crt == CRT_READ)
 			stats->os_lockless_reads += bytes;
@@ -1383,18 +1386,16 @@ static int osc_completion(const struct lu_env *env, struct osc_async_page *oap,
 
 #define OSC_DUMP_GRANT(lvl, cli, fmt, args...) do {			      \
 	struct client_obd *__tmp = (cli);				      \
-	CDEBUG(lvl, "%s: grant { dirty: %ld/%ld dirty_pages: %d/%d "	      \
-	       "unstable_pages: %d/%d dropped: %ld avail: %ld, "	      \
-	       "reserved: %ld, flight: %d } lru {in list: %d, "		      \
-	       "left: %d, waiters: %d }" fmt,                                 \
+	CDEBUG(lvl, "%s: grant { dirty: %ld/%ld dirty_pages: %ld/%lu "	      \
+	       "dropped: %ld avail: %ld, reserved: %ld, flight: %d }"	      \
+	       "lru {in list: %ld, left: %ld, waiters: %d }" fmt "\n",	      \
 	       __tmp->cl_import->imp_obd->obd_name,			      \
-	       __tmp->cl_dirty, __tmp->cl_dirty_max,			      \
-	       atomic_read(&obd_dirty_pages), obd_max_dirty_pages,	      \
-	       atomic_read(&obd_unstable_pages), obd_max_dirty_pages,	      \
+	       __tmp->cl_dirty_pages, __tmp->cl_dirty_max_pages,	      \
+	       atomic_long_read(&obd_dirty_pages), obd_max_dirty_pages,	      \
 	       __tmp->cl_lost_grant, __tmp->cl_avail_grant,		      \
 	       __tmp->cl_reserved_grant, __tmp->cl_w_in_flight,		      \
-	       atomic_read(&__tmp->cl_lru_in_list),			      \
-	       atomic_read(&__tmp->cl_lru_busy),			      \
+	       atomic_long_read(&__tmp->cl_lru_in_list),		      \
+	       atomic_long_read(&__tmp->cl_lru_busy),			      \
 	       atomic_read(&__tmp->cl_lru_shrinkers), ##args);		      \
 } while (0)
 
@@ -1404,8 +1405,8 @@ static void osc_consume_write_grant(struct client_obd *cli,
 {
 	assert_spin_locked(&cli->cl_loi_list_lock);
 	LASSERT(!(pga->flag & OBD_BRW_FROM_GRANT));
-	atomic_inc(&obd_dirty_pages);
-	cli->cl_dirty += PAGE_SIZE;
+	atomic_long_inc(&obd_dirty_pages);
+	cli->cl_dirty_pages++;
 	pga->flag |= OBD_BRW_FROM_GRANT;
 	CDEBUG(D_CACHE, "using %lu grant credits for brw %p page %p\n",
 	       PAGE_SIZE, pga, pga->pg);
@@ -1424,12 +1425,12 @@ static void osc_release_write_grant(struct client_obd *cli,
 	}
 
 	pga->flag &= ~OBD_BRW_FROM_GRANT;
-	atomic_dec(&obd_dirty_pages);
-	cli->cl_dirty -= PAGE_SIZE;
+	atomic_long_dec(&obd_dirty_pages);
+	cli->cl_dirty_pages--;
 	if (pga->flag & OBD_BRW_NOCACHE) {
 		pga->flag &= ~OBD_BRW_NOCACHE;
-		atomic_dec(&obd_dirty_transit_pages);
-		cli->cl_dirty_transit -= PAGE_SIZE;
+		atomic_long_dec(&obd_dirty_transit_pages);
+		cli->cl_dirty_transit--;
 	}
 }
 
@@ -1494,11 +1495,11 @@ static void osc_unreserve_grant(struct client_obd *cli,
 static void osc_free_grant(struct client_obd *cli, unsigned int nr_pages,
 			   unsigned int lost_grant)
 {
-	int grant = (1 << cli->cl_chunkbits) + cli->cl_extent_tax;
+	unsigned long grant = (1 << cli->cl_chunkbits) + cli->cl_extent_tax;
 
 	spin_lock(&cli->cl_loi_list_lock);
-	atomic_sub(nr_pages, &obd_dirty_pages);
-	cli->cl_dirty -= nr_pages << PAGE_SHIFT;
+	atomic_long_sub(nr_pages, &obd_dirty_pages);
+	cli->cl_dirty_pages -= nr_pages;
 	cli->cl_lost_grant += lost_grant;
 	if (cli->cl_avail_grant < grant && cli->cl_lost_grant >= grant) {
 		/* borrow some grant from truncate to avoid the case that
@@ -1511,7 +1512,7 @@ static void osc_free_grant(struct client_obd *cli, unsigned int nr_pages,
 	spin_unlock(&cli->cl_loi_list_lock);
 	CDEBUG(D_CACHE, "lost %u grant: %lu avail: %lu dirty: %lu\n",
 	       lost_grant, cli->cl_lost_grant,
-	       cli->cl_avail_grant, cli->cl_dirty);
+	       cli->cl_avail_grant, cli->cl_dirty_pages << PAGE_SHIFT);
 }
 
 /**
@@ -1535,19 +1536,18 @@ static int osc_enter_cache_try(struct client_obd *cli,
 {
 	int rc;
 
-	OSC_DUMP_GRANT(D_CACHE, cli, "need:%d.\n", bytes);
+	OSC_DUMP_GRANT(D_CACHE, cli, "need:%d\n", bytes);
 
 	rc = osc_reserve_grant(cli, bytes);
 	if (rc < 0)
 		return 0;
 
-	if (cli->cl_dirty + PAGE_SIZE <= cli->cl_dirty_max &&
-	    atomic_read(&obd_unstable_pages) + 1 +
-	    atomic_read(&obd_dirty_pages) <= obd_max_dirty_pages) {
+	if (cli->cl_dirty_pages <= cli->cl_dirty_max_pages &&
+	    atomic_long_read(&obd_dirty_pages) + 1 <= obd_max_dirty_pages) {
 		osc_consume_write_grant(cli, &oap->oap_brw_page);
 		if (transient) {
-			cli->cl_dirty_transit += PAGE_SIZE;
-			atomic_inc(&obd_dirty_transit_pages);
+			cli->cl_dirty_transit++;
+			atomic_long_inc(&obd_dirty_transit_pages);
 			oap->oap_brw_flags |= OBD_BRW_NOCACHE;
 		}
 		rc = 1;
@@ -1581,11 +1581,13 @@ static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
 	struct osc_object *osc = oap->oap_obj;
 	struct lov_oinfo *loi = osc->oo_oinfo;
 	struct osc_cache_waiter ocw;
-	struct l_wait_info lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(600), NULL,
-						  LWI_ON_SIGNAL_NOOP, NULL);
+	struct l_wait_info lwi;
 	int rc = -EDQUOT;
 
-	OSC_DUMP_GRANT(D_CACHE, cli, "need:%d.\n", bytes);
+	lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(AT_OFF ? obd_timeout : at_max),
+			       NULL, LWI_ON_SIGNAL_NOOP, NULL);
+
+	OSC_DUMP_GRANT(D_CACHE, cli, "need:%d\n", bytes);
 
 	spin_lock(&cli->cl_loi_list_lock);
 
@@ -1593,14 +1595,16 @@ static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
 	 * of queued writes and create a discontiguous rpc stream
 	 */
 	if (OBD_FAIL_CHECK(OBD_FAIL_OSC_NO_GRANT) ||
-	    cli->cl_dirty_max < PAGE_SIZE     ||
-	    cli->cl_ar.ar_force_sync || loi->loi_ar.ar_force_sync) {
+	    !cli->cl_dirty_max_pages || cli->cl_ar.ar_force_sync ||
+	    loi->loi_ar.ar_force_sync) {
+		OSC_DUMP_GRANT(D_CACHE, cli, "forced sync i/o\n");
 		rc = -EDQUOT;
 		goto out;
 	}
 
 	/* Hopefully normal case - cache space and write credits available */
 	if (osc_enter_cache_try(cli, oap, bytes, 0)) {
+		OSC_DUMP_GRANT(D_CACHE, cli, "granted from cache\n");
 		rc = 0;
 		goto out;
 	}
@@ -1615,7 +1619,7 @@ static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
 	init_waitqueue_head(&ocw.ocw_waitq);
 	ocw.ocw_oap   = oap;
 	ocw.ocw_grant = bytes;
-	while (cli->cl_dirty > 0 || cli->cl_w_in_flight > 0) {
+	while (cli->cl_dirty_pages > 0 || cli->cl_w_in_flight > 0) {
 		list_add_tail(&ocw.ocw_entry, &cli->cl_cache_waiters);
 		ocw.ocw_rc = 0;
 		spin_unlock(&cli->cl_loi_list_lock);
@@ -1629,32 +1633,49 @@ static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
 
 		spin_lock(&cli->cl_loi_list_lock);
 
-		/* l_wait_event is interrupted by signal, or timed out */
 		if (rc < 0) {
-			if (rc == -ETIMEDOUT) {
-				OSC_DUMP_GRANT(D_ERROR, cli,
-					       "try to reserve %d.\n", bytes);
-				osc_extent_tree_dump(D_ERROR, osc);
-				rc = -EDQUOT;
-			}
-
+			/* l_wait_event is interrupted by signal, or timed out */
 			list_del_init(&ocw.ocw_entry);
-			goto out;
+			break;
 		}
-
 		LASSERT(list_empty(&ocw.ocw_entry));
 		rc = ocw.ocw_rc;
 
 		if (rc != -EDQUOT)
-			goto out;
+			break;
 		if (osc_enter_cache_try(cli, oap, bytes, 0)) {
 			rc = 0;
-			goto out;
+			break;
 		}
 	}
+
+	switch (rc) {
+	case 0:
+		OSC_DUMP_GRANT(D_CACHE, cli, "finally got grant space\n");
+		break;
+	case -ETIMEDOUT:
+		OSC_DUMP_GRANT(D_CACHE, cli,
+			       "timeout, fall back to sync i/o\n");
+		osc_extent_tree_dump(D_CACHE, osc);
+		/* fall back to synchronous I/O */
+		rc = -EDQUOT;
+		break;
+	case -EINTR:
+		/* Ensures restartability - LU-3581 */
+		OSC_DUMP_GRANT(D_CACHE, cli, "interrupted\n");
+		rc = -ERESTARTSYS;
+		break;
+	case -EDQUOT:
+		OSC_DUMP_GRANT(D_CACHE, cli,
+			       "no grant space, fall back to sync i/o\n");
+		break;
+	default:
+		CDEBUG(D_CACHE, "%s: event for cache space @ %p never arrived due to %d, fall back to sync i/o\n",
+		       cli->cl_import->imp_obd->obd_name, &ocw, rc);
+		break;
+	}
 out:
 	spin_unlock(&cli->cl_loi_list_lock);
-	OSC_DUMP_GRANT(D_CACHE, cli, "returned %d.\n", rc);
 	return rc;
 }
 
@@ -1670,19 +1691,17 @@ void osc_wake_cache_waiters(struct client_obd *cli)
 
 		ocw->ocw_rc = -EDQUOT;
 		/* we can't dirty more */
-		if ((cli->cl_dirty + PAGE_SIZE > cli->cl_dirty_max) ||
-		    (atomic_read(&obd_unstable_pages) + 1 +
-		     atomic_read(&obd_dirty_pages) > obd_max_dirty_pages)) {
-			CDEBUG(D_CACHE, "no dirty room: dirty: %ld osc max %ld, sys max %d\n",
-			       cli->cl_dirty,
-			       cli->cl_dirty_max, obd_max_dirty_pages);
+		if ((cli->cl_dirty_pages > cli->cl_dirty_max_pages) ||
+		    (atomic_long_read(&obd_dirty_pages) + 1 >
+		     obd_max_dirty_pages)) {
+			CDEBUG(D_CACHE, "no dirty room: dirty: %ld osc max %ld, sys max %ld\n",
+			       cli->cl_dirty_pages, cli->cl_dirty_max_pages,
+			       obd_max_dirty_pages);
 			goto wakeup;
 		}
 
-		ocw->ocw_rc = 0;
-		if (!osc_enter_cache_try(cli, ocw->ocw_oap, ocw->ocw_grant, 0))
-			ocw->ocw_rc = -EDQUOT;
-
+		if (osc_enter_cache_try(cli, ocw->ocw_oap, ocw->ocw_grant, 0))
+			ocw->ocw_rc = 0;
 wakeup:
 		CDEBUG(D_CACHE, "wake up %p for oap %p, avail grant %ld, %d\n",
 		       ocw, ocw->ocw_oap, cli->cl_avail_grant, ocw->ocw_rc);
@@ -1843,97 +1862,6 @@ static void osc_process_ar(struct osc_async_rc *ar, __u64 xid,
 		ar->ar_force_sync = 0;
 }
 
-/**
- * Performs "unstable" page accounting. This function balances the
- * increment operations performed in osc_inc_unstable_pages. It is
- * registered as the RPC request callback, and is executed when the
- * bulk RPC is committed on the server. Thus at this point, the pages
- * involved in the bulk transfer are no longer considered unstable.
- */
-void osc_dec_unstable_pages(struct ptlrpc_request *req)
-{
-	struct client_obd *cli = &req->rq_import->imp_obd->u.cli;
-	struct ptlrpc_bulk_desc *desc = req->rq_bulk;
-	int page_count = desc->bd_iov_count;
-	int i;
-
-	/* No unstable page tracking */
-	if (!cli->cl_cache)
-		return;
-
-	LASSERT(page_count >= 0);
-
-	for (i = 0; i < page_count; i++)
-		dec_node_page_state(desc->bd_iov[i].kiov_page,
-							NR_UNSTABLE_NFS);
-
-	atomic_sub(page_count, &cli->cl_cache->ccc_unstable_nr);
-	LASSERT(atomic_read(&cli->cl_cache->ccc_unstable_nr) >= 0);
-
-	atomic_sub(page_count, &cli->cl_unstable_count);
-	LASSERT(atomic_read(&cli->cl_unstable_count) >= 0);
-
-	atomic_sub(page_count, &obd_unstable_pages);
-	LASSERT(atomic_read(&obd_unstable_pages) >= 0);
-
-	spin_lock(&req->rq_lock);
-	req->rq_committed = 1;
-	req->rq_unstable  = 0;
-	spin_unlock(&req->rq_lock);
-
-	wake_up_all(&cli->cl_cache->ccc_unstable_waitq);
-}
-
-/* "unstable" page accounting. See: osc_dec_unstable_pages. */
-void osc_inc_unstable_pages(struct ptlrpc_request *req)
-{
-	struct client_obd *cli = &req->rq_import->imp_obd->u.cli;
-	struct ptlrpc_bulk_desc *desc = req->rq_bulk;
-	long page_count = desc->bd_iov_count;
-	int i;
-
-	/* No unstable page tracking */
-	if (!cli->cl_cache)
-		return;
-
-	LASSERT(page_count >= 0);
-
-	for (i = 0; i < page_count; i++)
-		inc_node_page_state(desc->bd_iov[i].kiov_page,
-							NR_UNSTABLE_NFS);
-
-	LASSERT(atomic_read(&cli->cl_cache->ccc_unstable_nr) >= 0);
-	atomic_add(page_count, &cli->cl_cache->ccc_unstable_nr);
-
-	LASSERT(atomic_read(&cli->cl_unstable_count) >= 0);
-	atomic_add(page_count, &cli->cl_unstable_count);
-
-	LASSERT(atomic_read(&obd_unstable_pages) >= 0);
-	atomic_add(page_count, &obd_unstable_pages);
-
-	spin_lock(&req->rq_lock);
-
-	/*
-	 * If the request has already been committed (i.e. brw_commit
-	 * called via rq_commit_cb), we need to undo the unstable page
-	 * increments we just performed because rq_commit_cb wont be
-	 * called again. Otherwise, just set the commit callback so the
-	 * unstable page accounting is properly updated when the request
-	 * is committed
-	 */
-	if (req->rq_committed) {
-		/* Drop lock before calling osc_dec_unstable_pages */
-		spin_unlock(&req->rq_lock);
-		osc_dec_unstable_pages(req);
-		spin_lock(&req->rq_lock);
-	} else {
-		req->rq_unstable = 1;
-		req->rq_commit_cb = osc_dec_unstable_pages;
-	}
-
-	spin_unlock(&req->rq_lock);
-}
-
 /* this must be called holding the loi list lock to give coverage to exit_cache,
  * async_flag maintenance, and oap_request
  */
@@ -1945,9 +1873,6 @@ static void osc_ap_completion(const struct lu_env *env, struct client_obd *cli,
 	__u64 xid = 0;
 
 	if (oap->oap_request) {
-		if (!rc)
-			osc_inc_unstable_pages(oap->oap_request);
-
 		xid = ptlrpc_req_xid(oap->oap_request);
 		ptlrpc_req_finished(oap->oap_request);
 		oap->oap_request = NULL;
@@ -1979,7 +1904,7 @@ static void osc_ap_completion(const struct lu_env *env, struct client_obd *cli,
  */
 static int try_to_add_extent_for_io(struct client_obd *cli,
 				    struct osc_extent *ext, struct list_head *rpclist,
-				    int *pc, unsigned int *max_pages)
+				    unsigned int *pc, unsigned int *max_pages)
 {
 	struct osc_extent *tmp;
 	struct osc_async_page *oap = list_first_entry(&ext->oe_pages,
@@ -2032,12 +1957,13 @@ static int try_to_add_extent_for_io(struct client_obd *cli,
  * 5. Traverse the extent tree from the 1st extent;
  * 6. Above steps exit if there is no space in this RPC.
  */
-static int get_write_extents(struct osc_object *obj, struct list_head *rpclist)
+static unsigned int get_write_extents(struct osc_object *obj,
+				      struct list_head *rpclist)
 {
 	struct client_obd *cli = osc_cli(obj);
 	struct osc_extent *ext;
 	struct osc_extent *temp;
-	int page_count = 0;
+	unsigned int page_count = 0;
 	unsigned int max_pages = cli->cl_max_pages_per_rpc;
 
 	LASSERT(osc_object_is_locked(obj));
@@ -2175,7 +2101,7 @@ osc_send_read_rpc(const struct lu_env *env, struct client_obd *cli,
 	struct osc_extent *ext;
 	struct osc_extent *next;
 	LIST_HEAD(rpclist);
-	int page_count = 0;
+	unsigned int page_count = 0;
 	unsigned int max_pages = cli->cl_max_pages_per_rpc;
 	int rc = 0;
 
@@ -2390,7 +2316,7 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
 	struct client_obd *cli = oap->oap_cli;
 	struct osc_object *osc = oap->oap_obj;
 	pgoff_t index;
-	int grants = 0;
+	unsigned int grants = 0, tmp;
 	int brw_flags = OBD_BRW_ASYNC;
 	int cmd = OBD_BRW_WRITE;
 	int need_release = 0;
@@ -2434,9 +2360,6 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
 			return rc;
 	}
 
-	if (osc_over_unstable_soft_limit(cli))
-		brw_flags |= OBD_BRW_SOFT_SYNC;
-
 	oap->oap_cmd = cmd;
 	oap->oap_page_off = ops->ops_from;
 	oap->oap_count = ops->ops_to - ops->ops_from;
@@ -2476,7 +2399,7 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
 			grants = 0;
 			need_release = 1;
 		} else if (ext->oe_end < index) {
-			int tmp = grants;
+			tmp = grants;
 			/* try to expand this extent */
 			rc = osc_extent_expand(ext, index, &tmp);
 			if (rc < 0) {
@@ -2501,7 +2424,7 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
 	}
 
 	if (!ext) {
-		int tmp = (1 << cli->cl_chunkbits) + cli->cl_extent_tax;
+		tmp = (1 << cli->cl_chunkbits) + cli->cl_extent_tax;
 
 		/* try to find new extent to cover this page */
 		LASSERT(!oio->oi_active);
@@ -2645,7 +2568,7 @@ int osc_flush_async_page(const struct lu_env *env, struct cl_io *io,
 		goto out;
 
 	spin_lock(&oap->oap_lock);
-	oap->oap_async_flags |= ASYNC_READY|ASYNC_URGENT;
+	oap->oap_async_flags |= ASYNC_READY | ASYNC_URGENT;
 	spin_unlock(&oap->oap_lock);
 
 	if (memory_pressure_get())
diff --git a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
index c8c3f1ca77be..9c8de15c309c 100644
--- a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
+++ b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
@@ -64,7 +64,7 @@ struct osc_io {
 	/** true if this io is lockless. */
 	unsigned int		oi_lockless;
 	/** how many LRU pages are reserved for this IO */
-	int oi_lru_reserved;
+	unsigned long		oi_lru_reserved;
 
 	/** active extents, we know how many bytes is going to be written,
 	 * so having an active extent will prevent it from being fragmented
@@ -389,7 +389,7 @@ extern struct lu_device_type osc_device_type;
 extern struct lu_context_key osc_key;
 extern struct lu_context_key osc_session_key;
 
-#define OSC_FLAGS (ASYNC_URGENT|ASYNC_READY)
+#define OSC_FLAGS (ASYNC_URGENT | ASYNC_READY)
 
 int osc_lock_init(const struct lu_env *env,
 		  struct cl_object *obj, struct cl_lock *lock,
@@ -608,7 +608,7 @@ struct osc_extent {
 	/** link list of osc_object's oo_{hp|urgent|locking}_exts. */
 	struct list_head	 oe_link;
 	/** state of this extent */
-	unsigned int       oe_state;
+	enum osc_extent_state	oe_state;
 	/** flags for this extent. */
 	unsigned int       oe_intree:1,
 	/** 0 is write, 1 is read */
diff --git a/drivers/staging/lustre/lustre/osc/osc_internal.h b/drivers/staging/lustre/lustre/osc/osc_internal.h
index 7a27f0961955..67fe0a254991 100644
--- a/drivers/staging/lustre/lustre/osc/osc_internal.h
+++ b/drivers/staging/lustre/lustre/osc/osc_internal.h
@@ -71,7 +71,6 @@ struct osc_async_page {
 	struct client_obd       *oap_cli;
 	struct osc_object       *oap_obj;
 
-	struct ldlm_lock	*oap_ldlm_lock;
 	spinlock_t		 oap_lock;
 };
 
@@ -134,9 +133,9 @@ int osc_sync_base(struct obd_export *exp, struct obd_info *oinfo,
 int osc_process_config_base(struct obd_device *obd, struct lustre_cfg *cfg);
 int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 		  struct list_head *ext_list, int cmd);
-int osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
-		   int target, bool force);
-int osc_lru_reclaim(struct client_obd *cli);
+long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
+		    long target, bool force);
+long osc_lru_reclaim(struct client_obd *cli);
 
 unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock);
 
@@ -198,7 +197,7 @@ int osc_quotacheck(struct obd_device *unused, struct obd_export *exp,
 int osc_quota_poll_check(struct obd_export *exp, struct if_quotacheck *qchk);
 void osc_inc_unstable_pages(struct ptlrpc_request *req);
 void osc_dec_unstable_pages(struct ptlrpc_request *req);
-int  osc_over_unstable_soft_limit(struct client_obd *cli);
+bool osc_over_unstable_soft_limit(struct client_obd *cli);
 
 struct ldlm_lock *osc_dlmlock_at_pgoff(const struct lu_env *env,
 				       struct osc_object *obj, pgoff_t index,
diff --git a/drivers/staging/lustre/lustre/osc/osc_io.c b/drivers/staging/lustre/lustre/osc/osc_io.c
index 6e3dcd38913f..8a559cbcdd0c 100644
--- a/drivers/staging/lustre/lustre/osc/osc_io.c
+++ b/drivers/staging/lustre/lustre/osc/osc_io.c
@@ -109,11 +109,11 @@ static int osc_io_submit(const struct lu_env *env,
 
 	struct cl_page_list *qin = &queue->c2_qin;
 	struct cl_page_list *qout = &queue->c2_qout;
-	int queued = 0;
+	unsigned int queued = 0;
 	int result = 0;
 	int cmd;
 	int brw_flags;
-	int max_pages;
+	unsigned int max_pages;
 
 	LASSERT(qin->pl_nr > 0);
 
@@ -163,14 +163,19 @@ static int osc_io_submit(const struct lu_env *env,
 			continue;
 		}
 
-		cl_page_list_move(qout, qin, page);
 		spin_lock(&oap->oap_lock);
-		oap->oap_async_flags = ASYNC_URGENT|ASYNC_READY;
+		oap->oap_async_flags = ASYNC_URGENT | ASYNC_READY;
 		oap->oap_async_flags |= ASYNC_COUNT_STABLE;
 		spin_unlock(&oap->oap_lock);
 
 		osc_page_submit(env, opg, crt, brw_flags);
 		list_add_tail(&oap->oap_pending_item, &list);
+
+		if (page->cp_sync_io)
+			cl_page_list_move(qout, qin, page);
+		else /* async IO */
+			cl_page_list_del(env, qin, page);
+
 		if (++queued == max_pages) {
 			queued = 0;
 			result = osc_queue_sync_pages(env, osc, &list, cmd,
@@ -195,7 +200,7 @@ static int osc_io_submit(const struct lu_env *env,
  * Expand stripe KMS if necessary.
  */
 static void osc_page_touch_at(const struct lu_env *env,
-			      struct cl_object *obj, pgoff_t idx, unsigned to)
+			      struct cl_object *obj, pgoff_t idx, size_t to)
 {
 	struct lov_oinfo *loi = cl2osc(obj)->oo_oinfo;
 	struct cl_attr *attr = &osc_env_info(env)->oti_attr;
@@ -228,7 +233,7 @@ static void osc_page_touch_at(const struct lu_env *env,
 		attr->cat_size = kms;
 		valid |= CAT_SIZE;
 	}
-	cl_object_attr_set(env, obj, attr, valid);
+	cl_object_attr_update(env, obj, attr, valid);
 	cl_object_attr_unlock(obj);
 }
 
@@ -314,8 +319,8 @@ static int osc_io_rw_iter_init(const struct lu_env *env,
 	struct osc_object *osc = cl2osc(ios->cis_obj);
 	struct client_obd *cli = osc_cli(osc);
 	unsigned long c;
-	unsigned int npages;
-	unsigned int max_pages;
+	unsigned long npages;
+	unsigned long max_pages;
 
 	if (cl_io_is_append(io))
 		return 0;
@@ -328,15 +333,15 @@ static int osc_io_rw_iter_init(const struct lu_env *env,
 	if (npages > max_pages)
 		npages = max_pages;
 
-	c = atomic_read(cli->cl_lru_left);
+	c = atomic_long_read(cli->cl_lru_left);
 	if (c < npages && osc_lru_reclaim(cli) > 0)
-		c = atomic_read(cli->cl_lru_left);
+		c = atomic_long_read(cli->cl_lru_left);
 	while (c >= npages) {
-		if (c == atomic_cmpxchg(cli->cl_lru_left, c, c - npages)) {
+		if (c == atomic_long_cmpxchg(cli->cl_lru_left, c, c - npages)) {
 			oio->oi_lru_reserved = npages;
 			break;
 		}
-		c = atomic_read(cli->cl_lru_left);
+		c = atomic_long_read(cli->cl_lru_left);
 	}
 
 	return 0;
@@ -350,7 +355,7 @@ static void osc_io_rw_iter_fini(const struct lu_env *env,
 	struct client_obd *cli = osc_cli(osc);
 
 	if (oio->oi_lru_reserved > 0) {
-		atomic_add(oio->oi_lru_reserved, cli->cl_lru_left);
+		atomic_long_add(oio->oi_lru_reserved, cli->cl_lru_left);
 		oio->oi_lru_reserved = 0;
 	}
 	oio->oi_write_osclock = NULL;
@@ -364,7 +369,7 @@ static int osc_io_fault_start(const struct lu_env *env,
 
 	io = ios->cis_io;
 	fio = &io->u.ci_fault;
-	CDEBUG(D_INFO, "%lu %d %d\n",
+	CDEBUG(D_INFO, "%lu %d %zu\n",
 	       fio->ft_index, fio->ft_writable, fio->ft_nob);
 	/*
 	 * If mapping is writeable, adjust kms to cover this page,
@@ -471,18 +476,21 @@ static int osc_io_setattr_start(const struct lu_env *env,
 				attr->cat_ctime = lvb->lvb_ctime;
 				cl_valid |= CAT_CTIME;
 			}
-			result = cl_object_attr_set(env, obj, attr, cl_valid);
+			result = cl_object_attr_update(env, obj, attr,
+						       cl_valid);
 		}
 		cl_object_attr_unlock(obj);
 	}
 	memset(oa, 0, sizeof(*oa));
 	if (result == 0) {
 		oa->o_oi = loi->loi_oi;
+		obdo_set_parent_fid(oa, io->u.ci_setattr.sa_parent_fid);
+		oa->o_stripe_idx = io->u.ci_setattr.sa_stripe_index;
 		oa->o_mtime = attr->cat_mtime;
 		oa->o_atime = attr->cat_atime;
 		oa->o_ctime = attr->cat_ctime;
-		oa->o_valid = OBD_MD_FLID | OBD_MD_FLGROUP | OBD_MD_FLATIME |
-			OBD_MD_FLCTIME | OBD_MD_FLMTIME;
+		oa->o_valid |= OBD_MD_FLID | OBD_MD_FLGROUP | OBD_MD_FLATIME |
+			       OBD_MD_FLCTIME | OBD_MD_FLMTIME;
 		if (ia_valid & ATTR_SIZE) {
 			oa->o_size = size;
 			oa->o_blocks = OBD_OBJECT_EOF;
@@ -559,7 +567,7 @@ static int osc_io_read_start(const struct lu_env *env,
 	if (!slice->cis_io->ci_noatime) {
 		cl_object_attr_lock(obj);
 		attr->cat_atime = ktime_get_real_seconds();
-		rc = cl_object_attr_set(env, obj, attr, CAT_ATIME);
+		rc = cl_object_attr_update(env, obj, attr, CAT_ATIME);
 		cl_object_attr_unlock(obj);
 	}
 	return rc;
@@ -576,7 +584,7 @@ static int osc_io_write_start(const struct lu_env *env,
 	cl_object_attr_lock(obj);
 	attr->cat_ctime = ktime_get_real_seconds();
 	attr->cat_mtime = attr->cat_ctime;
-	rc = cl_object_attr_set(env, obj, attr, CAT_MTIME | CAT_CTIME);
+	rc = cl_object_attr_update(env, obj, attr, CAT_MTIME | CAT_CTIME);
 	cl_object_attr_unlock(obj);
 
 	return rc;
diff --git a/drivers/staging/lustre/lustre/osc/osc_lock.c b/drivers/staging/lustre/lustre/osc/osc_lock.c
index 717d3ffb6789..39a8a5851603 100644
--- a/drivers/staging/lustre/lustre/osc/osc_lock.c
+++ b/drivers/staging/lustre/lustre/osc/osc_lock.c
@@ -222,7 +222,7 @@ static void osc_lock_lvb_update(const struct lu_env *env,
 		ldlm_lock_allow_match_locked(dlmlock);
 	}
 
-	cl_object_attr_set(env, obj, attr, valid);
+	cl_object_attr_update(env, obj, attr, valid);
 	cl_object_attr_unlock(obj);
 }
 
@@ -467,7 +467,7 @@ static int osc_dlm_blocking_ast0(const struct lu_env *env,
 		 */
 		attr->cat_kms = ldlm_extent_shift_kms(dlmlock, old_kms);
 
-		cl_object_attr_set(env, obj, attr, CAT_KMS);
+		cl_object_attr_update(env, obj, attr, CAT_KMS);
 		cl_object_attr_unlock(obj);
 		unlock_res_and_lock(dlmlock);
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_object.c b/drivers/staging/lustre/lustre/osc/osc_object.c
index d211d1905e83..aae3a2d4243f 100644
--- a/drivers/staging/lustre/lustre/osc/osc_object.c
+++ b/drivers/staging/lustre/lustre/osc/osc_object.c
@@ -159,8 +159,8 @@ static int osc_attr_get(const struct lu_env *env, struct cl_object *obj,
 	return 0;
 }
 
-static int osc_attr_set(const struct lu_env *env, struct cl_object *obj,
-			const struct cl_attr *attr, unsigned valid)
+static int osc_attr_update(const struct lu_env *env, struct cl_object *obj,
+			   const struct cl_attr *attr, unsigned int valid)
 {
 	struct lov_oinfo *oinfo = cl2osc(obj)->oo_oinfo;
 	struct ost_lvb *lvb = &oinfo->loi_lvb;
@@ -195,7 +195,6 @@ static int osc_object_glimpse(const struct lu_env *env,
 
 static int osc_object_ast_clear(struct ldlm_lock *lock, void *data)
 {
-	LASSERT(lock->l_granted_mode == lock->l_req_mode);
 	if (lock->l_ast_data == data)
 		lock->l_ast_data = NULL;
 	return LDLM_ITER_CONTINUE;
@@ -262,7 +261,7 @@ static const struct cl_object_operations osc_ops = {
 	.coo_lock_init = osc_lock_init,
 	.coo_io_init   = osc_io_init,
 	.coo_attr_get  = osc_attr_get,
-	.coo_attr_set  = osc_attr_set,
+	.coo_attr_update = osc_attr_update,
 	.coo_glimpse   = osc_object_glimpse,
 	.coo_prune     = osc_object_prune
 };
diff --git a/drivers/staging/lustre/lustre/osc/osc_page.c b/drivers/staging/lustre/lustre/osc/osc_page.c
index 355f496a2093..2a7a70aa9e80 100644
--- a/drivers/staging/lustre/lustre/osc/osc_page.c
+++ b/drivers/staging/lustre/lustre/osc/osc_page.c
@@ -323,32 +323,6 @@ int osc_page_init(const struct lu_env *env, struct cl_object *obj,
 	return result;
 }
 
-int osc_over_unstable_soft_limit(struct client_obd *cli)
-{
-	long obd_upages, obd_dpages, osc_upages;
-
-	/* Can't check cli->cl_unstable_count, therefore, no soft limit */
-	if (!cli)
-		return 0;
-
-	obd_upages = atomic_read(&obd_unstable_pages);
-	obd_dpages = atomic_read(&obd_dirty_pages);
-
-	osc_upages = atomic_read(&cli->cl_unstable_count);
-
-	/*
-	 * obd_max_dirty_pages is the max number of (dirty + unstable)
-	 * pages allowed at any given time. To simulate an unstable page
-	 * only limit, we subtract the current number of dirty pages
-	 * from this max. This difference is roughly the amount of pages
-	 * currently available for unstable pages. Thus, the soft limit
-	 * is half of that difference. Check osc_upages to ensure we don't
-	 * set SOFT_SYNC for OSCs without any outstanding unstable pages.
-	 */
-	return osc_upages &&
-	       obd_upages >= (obd_max_dirty_pages - obd_dpages) / 2;
-}
-
 /**
  * Helper function called by osc_io_submit() for every page in an immediate
  * transfer (i.e., transferred synchronously).
@@ -368,9 +342,6 @@ void osc_page_submit(const struct lu_env *env, struct osc_page *opg,
 	oap->oap_count = opg->ops_to - opg->ops_from;
 	oap->oap_brw_flags = brw_flags | OBD_BRW_SYNC;
 
-	if (osc_over_unstable_soft_limit(oap->oap_cli))
-		oap->oap_brw_flags |= OBD_BRW_SOFT_SYNC;
-
 	if (capable(CFS_CAP_SYS_RESOURCE)) {
 		oap->oap_brw_flags |= OBD_BRW_NOQUOTA;
 		oap->oap_cmd |= OBD_BRW_NOQUOTA;
@@ -409,7 +380,7 @@ static const int lru_shrink_max = 8 << (20 - PAGE_SHIFT); /* 8M */
 static int osc_cache_too_much(struct client_obd *cli)
 {
 	struct cl_client_cache *cache = cli->cl_cache;
-	int pages = atomic_read(&cli->cl_lru_in_list);
+	long pages = atomic_long_read(&cli->cl_lru_in_list);
 	unsigned long budget;
 
 	budget = cache->ccc_lru_max / (atomic_read(&cache->ccc_users) - 2);
@@ -417,7 +388,7 @@ static int osc_cache_too_much(struct client_obd *cli)
 	/* if it's going to run out LRU slots, we should free some, but not
 	 * too much to maintain fairness among OSCs.
 	 */
-	if (atomic_read(cli->cl_lru_left) < cache->ccc_lru_max >> 4) {
+	if (atomic_long_read(cli->cl_lru_left) < cache->ccc_lru_max >> 4) {
 		if (pages >= budget)
 			return lru_shrink_max;
 		else if (pages >= budget / 2)
@@ -444,7 +415,7 @@ void osc_lru_add_batch(struct client_obd *cli, struct list_head *plist)
 {
 	LIST_HEAD(lru);
 	struct osc_async_page *oap;
-	int npages = 0;
+	long npages = 0;
 
 	list_for_each_entry(oap, plist, oap_pending_item) {
 		struct osc_page *opg = oap2osc_page(oap);
@@ -460,8 +431,8 @@ void osc_lru_add_batch(struct client_obd *cli, struct list_head *plist)
 	if (npages > 0) {
 		spin_lock(&cli->cl_lru_list_lock);
 		list_splice_tail(&lru, &cli->cl_lru_list);
-		atomic_sub(npages, &cli->cl_lru_busy);
-		atomic_add(npages, &cli->cl_lru_in_list);
+		atomic_long_sub(npages, &cli->cl_lru_busy);
+		atomic_long_add(npages, &cli->cl_lru_in_list);
 		spin_unlock(&cli->cl_lru_list_lock);
 
 		/* XXX: May set force to be true for better performance */
@@ -472,9 +443,9 @@ void osc_lru_add_batch(struct client_obd *cli, struct list_head *plist)
 
 static void __osc_lru_del(struct client_obd *cli, struct osc_page *opg)
 {
-	LASSERT(atomic_read(&cli->cl_lru_in_list) > 0);
+	LASSERT(atomic_long_read(&cli->cl_lru_in_list) > 0);
 	list_del_init(&opg->ops_lru);
-	atomic_dec(&cli->cl_lru_in_list);
+	atomic_long_dec(&cli->cl_lru_in_list);
 }
 
 /**
@@ -488,12 +459,12 @@ static void osc_lru_del(struct client_obd *cli, struct osc_page *opg)
 		if (!list_empty(&opg->ops_lru)) {
 			__osc_lru_del(cli, opg);
 		} else {
-			LASSERT(atomic_read(&cli->cl_lru_busy) > 0);
-			atomic_dec(&cli->cl_lru_busy);
+			LASSERT(atomic_long_read(&cli->cl_lru_busy) > 0);
+			atomic_long_dec(&cli->cl_lru_busy);
 		}
 		spin_unlock(&cli->cl_lru_list_lock);
 
-		atomic_inc(cli->cl_lru_left);
+		atomic_long_inc(cli->cl_lru_left);
 		/* this is a great place to release more LRU pages if
 		 * this osc occupies too many LRU pages and kernel is
 		 * stealing one of them.
@@ -518,7 +489,7 @@ static void osc_lru_use(struct client_obd *cli, struct osc_page *opg)
 		spin_lock(&cli->cl_lru_list_lock);
 		__osc_lru_del(cli, opg);
 		spin_unlock(&cli->cl_lru_list_lock);
-		atomic_inc(&cli->cl_lru_busy);
+		atomic_long_inc(&cli->cl_lru_busy);
 	}
 }
 
@@ -540,10 +511,32 @@ static void discard_pagevec(const struct lu_env *env, struct cl_io *io,
 }
 
 /**
+ * Check if a cl_page can be released, i.e, it's not being used.
+ *
+ * If unstable account is turned on, bulk transfer may hold one refcount
+ * for recovery so we need to check vmpage refcount as well; otherwise,
+ * even we can destroy cl_page but the corresponding vmpage can't be reused.
+ */
+static inline bool lru_page_busy(struct client_obd *cli, struct cl_page *page)
+{
+	if (cl_page_in_use_noref(page))
+		return true;
+
+	if (cli->cl_cache->ccc_unstable_check) {
+		struct page *vmpage = cl_page_vmpage(page);
+
+		/* vmpage have two known users: cl_page and VM page cache */
+		if (page_count(vmpage) - page_mapcount(vmpage) > 2)
+			return true;
+	}
+	return false;
+}
+
+/**
  * Drop @target of pages from LRU at most.
  */
-int osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
-		   int target, bool force)
+long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
+		    long target, bool force)
 {
 	struct cl_io *io;
 	struct cl_object *clobj = NULL;
@@ -551,12 +544,12 @@ int osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 	struct osc_page *opg;
 	struct osc_page *temp;
 	int maxscan = 0;
-	int count = 0;
+	long count = 0;
 	int index = 0;
 	int rc = 0;
 
-	LASSERT(atomic_read(&cli->cl_lru_in_list) >= 0);
-	if (atomic_read(&cli->cl_lru_in_list) == 0 || target <= 0)
+	LASSERT(atomic_long_read(&cli->cl_lru_in_list) >= 0);
+	if (atomic_long_read(&cli->cl_lru_in_list) == 0 || target <= 0)
 		return 0;
 
 	if (!force) {
@@ -575,7 +568,7 @@ int osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 	io = &osc_env_info(env)->oti_io;
 
 	spin_lock(&cli->cl_lru_list_lock);
-	maxscan = min(target << 1, atomic_read(&cli->cl_lru_in_list));
+	maxscan = min(target << 1, atomic_long_read(&cli->cl_lru_in_list));
 	list_for_each_entry_safe(opg, temp, &cli->cl_lru_list, ops_lru) {
 		struct cl_page *page;
 		bool will_free = false;
@@ -584,7 +577,7 @@ int osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 			break;
 
 		page = opg->ops_cl.cpl_page;
-		if (cl_page_in_use_noref(page)) {
+		if (lru_page_busy(cli, page)) {
 			list_move_tail(&opg->ops_lru, &cli->cl_lru_list);
 			continue;
 		}
@@ -620,7 +613,7 @@ int osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 		}
 
 		if (cl_page_own_try(env, io, page) == 0) {
-			if (!cl_page_in_use_noref(page)) {
+			if (!lru_page_busy(cli, page)) {
 				/* remove it from lru list earlier to avoid
 				 * lock contention
 				 */
@@ -663,24 +656,19 @@ int osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 
 	atomic_dec(&cli->cl_lru_shrinkers);
 	if (count > 0) {
-		atomic_add(count, cli->cl_lru_left);
+		atomic_long_add(count, cli->cl_lru_left);
 		wake_up_all(&osc_lru_waitq);
 	}
 	return count > 0 ? count : rc;
 }
 
-static inline int max_to_shrink(struct client_obd *cli)
-{
-	return min(atomic_read(&cli->cl_lru_in_list) >> 1, lru_shrink_max);
-}
-
-int osc_lru_reclaim(struct client_obd *cli)
+long osc_lru_reclaim(struct client_obd *cli)
 {
 	struct cl_env_nest nest;
 	struct lu_env *env;
 	struct cl_client_cache *cache = cli->cl_cache;
 	int max_scans;
-	int rc = 0;
+	long rc = 0;
 
 	LASSERT(cache);
 
@@ -693,15 +681,15 @@ int osc_lru_reclaim(struct client_obd *cli)
 		if (rc == -EBUSY)
 			rc = 0;
 
-		CDEBUG(D_CACHE, "%s: Free %d pages from own LRU: %p.\n",
+		CDEBUG(D_CACHE, "%s: Free %ld pages from own LRU: %p.\n",
 		       cli->cl_import->imp_obd->obd_name, rc, cli);
 		goto out;
 	}
 
-	CDEBUG(D_CACHE, "%s: cli %p no free slots, pages: %d, busy: %d.\n",
+	CDEBUG(D_CACHE, "%s: cli %p no free slots, pages: %ld, busy: %ld.\n",
 	       cli->cl_import->imp_obd->obd_name, cli,
-	       atomic_read(&cli->cl_lru_in_list),
-	       atomic_read(&cli->cl_lru_busy));
+	       atomic_long_read(&cli->cl_lru_in_list),
+	       atomic_long_read(&cli->cl_lru_busy));
 
 	/* Reclaim LRU slots from other client_obd as it can't free enough
 	 * from its own. This should rarely happen.
@@ -717,10 +705,10 @@ int osc_lru_reclaim(struct client_obd *cli)
 		cli = list_entry(cache->ccc_lru.next, struct client_obd,
 				 cl_lru_osc);
 
-		CDEBUG(D_CACHE, "%s: cli %p LRU pages: %d, busy: %d.\n",
+		CDEBUG(D_CACHE, "%s: cli %p LRU pages: %ld, busy: %ld.\n",
 		       cli->cl_import->imp_obd->obd_name, cli,
-		       atomic_read(&cli->cl_lru_in_list),
-		       atomic_read(&cli->cl_lru_busy));
+		       atomic_long_read(&cli->cl_lru_in_list),
+		       atomic_long_read(&cli->cl_lru_busy));
 
 		list_move_tail(&cli->cl_lru_osc, &cache->ccc_lru);
 		if (osc_cache_too_much(cli) > 0) {
@@ -737,11 +725,18 @@ int osc_lru_reclaim(struct client_obd *cli)
 
 out:
 	cl_env_nested_put(&nest, env);
-	CDEBUG(D_CACHE, "%s: cli %p freed %d pages.\n",
+	CDEBUG(D_CACHE, "%s: cli %p freed %ld pages.\n",
 	       cli->cl_import->imp_obd->obd_name, cli, rc);
 	return rc;
 }
 
+/**
+ * osc_lru_reserve() is called to reserve an LRU slot for a cl_page.
+ *
+ * Usually the LRU slots are reserved in osc_io_iter_rw_init().
+ * Only in the case that the LRU slots are in extreme shortage, it should
+ * have reserved enough slots for an IO.
+ */
 static int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj,
 			   struct osc_page *opg)
 {
@@ -758,8 +753,8 @@ static int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj,
 		goto out;
 	}
 
-	LASSERT(atomic_read(cli->cl_lru_left) >= 0);
-	while (!atomic_add_unless(cli->cl_lru_left, -1, 0)) {
+	LASSERT(atomic_long_read(cli->cl_lru_left) >= 0);
+	while (!atomic_long_add_unless(cli->cl_lru_left, -1, 0)) {
 		/* run out of LRU spaces, try to drop some by itself */
 		rc = osc_lru_reclaim(cli);
 		if (rc < 0)
@@ -770,7 +765,7 @@ static int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj,
 		cond_resched();
 
 		rc = l_wait_event(osc_lru_waitq,
-				  atomic_read(cli->cl_lru_left) > 0,
+				  atomic_long_read(cli->cl_lru_left) > 0,
 				  &lwi);
 
 		if (rc < 0)
@@ -779,7 +774,7 @@ static int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj,
 
 out:
 	if (rc >= 0) {
-		atomic_inc(&cli->cl_lru_busy);
+		atomic_long_inc(&cli->cl_lru_busy);
 		opg->ops_in_lru = 1;
 		rc = 0;
 	}
@@ -787,4 +782,151 @@ out:
 	return rc;
 }
 
+/**
+ * Atomic operations are expensive. We accumulate the accounting for the
+ * same page pgdat to get better performance.
+ * In practice this can work pretty good because the pages in the same RPC
+ * are likely from the same page zone.
+ */
+static inline void unstable_page_accounting(struct ptlrpc_bulk_desc *desc,
+					    int factor)
+{
+	int page_count = desc->bd_iov_count;
+	pg_data_t *last = NULL;
+	int count = 0;
+	int i;
+
+	for (i = 0; i < page_count; i++) {
+		pg_data_t *pgdat = page_pgdat(desc->bd_iov[i].bv_page);
+
+		if (likely(pgdat == last)) {
+			++count;
+			continue;
+		}
+
+		if (count > 0) {
+			mod_node_page_state(pgdat, NR_UNSTABLE_NFS,
+					    factor * count);
+			count = 0;
+		}
+		last = pgdat;
+		++count;
+	}
+	if (count > 0)
+		mod_node_page_state(last, NR_UNSTABLE_NFS, factor * count);
+}
+
+static inline void add_unstable_page_accounting(struct ptlrpc_bulk_desc *desc)
+{
+	unstable_page_accounting(desc, 1);
+}
+
+static inline void dec_unstable_page_accounting(struct ptlrpc_bulk_desc *desc)
+{
+	unstable_page_accounting(desc, -1);
+}
+
+/**
+ * Performs "unstable" page accounting. This function balances the
+ * increment operations performed in osc_inc_unstable_pages. It is
+ * registered as the RPC request callback, and is executed when the
+ * bulk RPC is committed on the server. Thus at this point, the pages
+ * involved in the bulk transfer are no longer considered unstable.
+ *
+ * If this function is called, the request should have been committed
+ * or req:rq_unstable must have been set; it implies that the unstable
+ * statistic have been added.
+ */
+void osc_dec_unstable_pages(struct ptlrpc_request *req)
+{
+	struct client_obd *cli = &req->rq_import->imp_obd->u.cli;
+	struct ptlrpc_bulk_desc *desc = req->rq_bulk;
+	int page_count = desc->bd_iov_count;
+	long unstable_count;
+
+	LASSERT(page_count >= 0);
+	dec_unstable_page_accounting(desc);
+
+	unstable_count = atomic_long_sub_return(page_count,
+						&cli->cl_unstable_count);
+	LASSERT(unstable_count >= 0);
+
+	unstable_count = atomic_long_sub_return(page_count,
+						&cli->cl_cache->ccc_unstable_nr);
+	LASSERT(unstable_count >= 0);
+	if (!unstable_count)
+		wake_up_all(&cli->cl_cache->ccc_unstable_waitq);
+
+	if (osc_cache_too_much(cli))
+		(void)ptlrpcd_queue_work(cli->cl_lru_work);
+}
+
+/**
+ * "unstable" page accounting. See: osc_dec_unstable_pages.
+ */
+void osc_inc_unstable_pages(struct ptlrpc_request *req)
+{
+	struct client_obd *cli  = &req->rq_import->imp_obd->u.cli;
+	struct ptlrpc_bulk_desc *desc = req->rq_bulk;
+	long page_count = desc->bd_iov_count;
+
+	/* No unstable page tracking */
+	if (!cli->cl_cache || !cli->cl_cache->ccc_unstable_check)
+		return;
+
+	add_unstable_page_accounting(desc);
+	atomic_long_add(page_count, &cli->cl_unstable_count);
+	atomic_long_add(page_count, &cli->cl_cache->ccc_unstable_nr);
+
+	/*
+	 * If the request has already been committed (i.e. brw_commit
+	 * called via rq_commit_cb), we need to undo the unstable page
+	 * increments we just performed because rq_commit_cb wont be
+	 * called again.
+	 */
+	spin_lock(&req->rq_lock);
+	if (unlikely(req->rq_committed)) {
+		spin_unlock(&req->rq_lock);
+
+		osc_dec_unstable_pages(req);
+	} else {
+		req->rq_unstable = 1;
+		spin_unlock(&req->rq_lock);
+	}
+}
+
+/**
+ * Check if it piggybacks SOFT_SYNC flag to OST from this OSC.
+ * This function will be called by every BRW RPC so it's critical
+ * to make this function fast.
+ */
+bool osc_over_unstable_soft_limit(struct client_obd *cli)
+{
+	long unstable_nr, osc_unstable_count;
+
+	/* Can't check cli->cl_unstable_count, therefore, no soft limit */
+	if (!cli->cl_cache || !cli->cl_cache->ccc_unstable_check)
+		return false;
+
+	osc_unstable_count = atomic_long_read(&cli->cl_unstable_count);
+	unstable_nr = atomic_long_read(&cli->cl_cache->ccc_unstable_nr);
+
+	CDEBUG(D_CACHE,
+	       "%s: cli: %p unstable pages: %lu, osc unstable pages: %lu\n",
+	       cli->cl_import->imp_obd->obd_name, cli,
+	       unstable_nr, osc_unstable_count);
+
+	/*
+	 * If the LRU slots are in shortage - 25% remaining AND this OSC
+	 * has one full RPC window of unstable pages, it's a good chance
+	 * to piggyback a SOFT_SYNC flag.
+	 * Please notice that the OST won't take immediate response for the
+	 * SOFT_SYNC request so active OSCs will have more chance to carry
+	 * the flag, this is reasonable.
+	 */
+	return unstable_nr > cli->cl_cache->ccc_lru_max >> 2 &&
+	       osc_unstable_count > cli->cl_max_pages_per_rpc *
+				    cli->cl_max_rpcs_in_flight;
+}
+
 /** @} osc */
diff --git a/drivers/staging/lustre/lustre/osc/osc_request.c b/drivers/staging/lustre/lustre/osc/osc_request.c
index 536b868ff776..749781f022e2 100644
--- a/drivers/staging/lustre/lustre/osc/osc_request.c
+++ b/drivers/staging/lustre/lustre/osc/osc_request.c
@@ -41,6 +41,7 @@
 
 #include "../include/lustre_ha.h"
 #include "../include/lprocfs_status.h"
+#include "../include/lustre/lustre_ioctl.h"
 #include "../include/lustre_debug.h"
 #include "../include/lustre_param.h"
 #include "../include/lustre_fid.h"
@@ -102,36 +103,6 @@ static void osc_release_ppga(struct brw_page **ppga, u32 count);
 static int brw_interpret(const struct lu_env *env,
 			 struct ptlrpc_request *req, void *data, int rc);
 
-/* Pack OSC object metadata for disk storage (LE byte order). */
-static int osc_packmd(struct obd_export *exp, struct lov_mds_md **lmmp,
-		      struct lov_stripe_md *lsm)
-{
-	int lmm_size;
-
-	lmm_size = sizeof(**lmmp);
-	if (!lmmp)
-		return lmm_size;
-
-	if (*lmmp && !lsm) {
-		kfree(*lmmp);
-		*lmmp = NULL;
-		return 0;
-	} else if (unlikely(lsm && ostid_id(&lsm->lsm_oi) == 0)) {
-		return -EBADF;
-	}
-
-	if (!*lmmp) {
-		*lmmp = kzalloc(lmm_size, GFP_NOFS);
-		if (!*lmmp)
-			return -ENOMEM;
-	}
-
-	if (lsm)
-		ostid_cpu_to_le(&lsm->lsm_oi, &(*lmmp)->lmm_oi);
-
-	return lmm_size;
-}
-
 /* Unpack OSC object metadata from disk storage (LE byte order). */
 static int osc_unpackmd(struct obd_export *exp, struct lov_stripe_md **lsmp,
 			struct lov_mds_md *lmm, int lmm_bytes)
@@ -189,7 +160,7 @@ static int osc_unpackmd(struct obd_export *exp, struct lov_stripe_md **lsmp,
 	    (imp->imp_connect_data.ocd_connect_flags & OBD_CONNECT_MAXBYTES))
 		(*lsmp)->lsm_maxbytes = imp->imp_connect_data.ocd_maxbytes;
 	else
-		(*lsmp)->lsm_maxbytes = LUSTRE_STRIPE_MAXBYTES;
+		(*lsmp)->lsm_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
 
 	return lsm_size;
 }
@@ -427,24 +398,16 @@ static int osc_setattr_async(struct obd_export *exp, struct obd_info *oinfo,
 				      oinfo->oi_cb_up, oinfo, rqset);
 }
 
-static int osc_real_create(struct obd_export *exp, struct obdo *oa,
-			   struct lov_stripe_md **ea,
-			   struct obd_trans_info *oti)
+static int osc_create(const struct lu_env *env, struct obd_export *exp,
+		      struct obdo *oa, struct obd_trans_info *oti)
 {
 	struct ptlrpc_request *req;
 	struct ost_body *body;
-	struct lov_stripe_md *lsm;
 	int rc;
 
 	LASSERT(oa);
-	LASSERT(ea);
-
-	lsm = *ea;
-	if (!lsm) {
-		rc = obd_alloc_memmd(exp, &lsm);
-		if (rc < 0)
-			return rc;
-	}
+	LASSERT(oa->o_valid & OBD_MD_FLGROUP);
+	LASSERT(fid_seq_is_echo(ostid_seq(&oa->o_oi)));
 
 	req = ptlrpc_request_alloc(class_exp2cliimp(exp), &RQF_OST_CREATE);
 	if (!req) {
@@ -490,21 +453,10 @@ static int osc_real_create(struct obd_export *exp, struct obdo *oa,
 	oa->o_blksize = cli_brw_size(exp->exp_obd);
 	oa->o_valid |= OBD_MD_FLBLKSZ;
 
-	/* XXX LOV STACKING: the lsm that is passed to us from LOV does not
-	 * have valid lsm_oinfo data structs, so don't go touching that.
-	 * This needs to be fixed in a big way.
-	 */
-	lsm->lsm_oi = oa->o_oi;
-	*ea = lsm;
-
-	if (oti) {
-		oti->oti_transno = lustre_msg_get_transno(req->rq_repmsg);
-
-		if (oa->o_valid & OBD_MD_FLCOOKIE) {
-			if (!oti->oti_logcookies)
-				oti_alloc_cookies(oti, 1);
-			*oti->oti_logcookies = oa->o_lcookie;
-		}
+	if (oti && oa->o_valid & OBD_MD_FLCOOKIE) {
+		if (!oti->oti_logcookies)
+			oti->oti_logcookies = &oti->oti_onecookie;
+		*oti->oti_logcookies = oa->o_lcookie;
 	}
 
 	CDEBUG(D_HA, "transno: %lld\n",
@@ -512,8 +464,6 @@ static int osc_real_create(struct obd_export *exp, struct obdo *oa,
 out_req:
 	ptlrpc_req_finished(req);
 out:
-	if (rc && !*ea)
-		obd_free_memmd(exp, &lsm);
 	return rc;
 }
 
@@ -649,7 +599,7 @@ static int osc_resource_get_unused(struct obd_export *exp, struct obdo *oa,
 
 	ostid_build_res_name(&oa->o_oi, &res_id);
 	res = ldlm_resource_get(ns, NULL, &res_id, 0, 0);
-	if (!res)
+	if (IS_ERR(res))
 		return 0;
 
 	LDLM_RESOURCE_ADDREF(res);
@@ -689,30 +639,6 @@ static int osc_can_send_destroy(struct client_obd *cli)
 	return 0;
 }
 
-static int osc_create(const struct lu_env *env, struct obd_export *exp,
-		      struct obdo *oa, struct lov_stripe_md **ea,
-		      struct obd_trans_info *oti)
-{
-	int rc = 0;
-
-	LASSERT(oa);
-	LASSERT(ea);
-	LASSERT(oa->o_valid & OBD_MD_FLGROUP);
-
-	if ((oa->o_valid & OBD_MD_FLFLAGS) &&
-	    oa->o_flags == OBD_FL_RECREATE_OBJS) {
-		return osc_real_create(exp, oa, ea, oti);
-	}
-
-	if (!fid_seq_is_mdt(ostid_seq(&oa->o_oi)))
-		return osc_real_create(exp, oa, ea, oti);
-
-	/* we should not get here anymore */
-	LBUG();
-
-	return rc;
-}
-
 /* Destroy requests can be async always on the client, and we don't even really
  * care about the return code since the client cannot do anything at all about
  * a destroy failure.
@@ -725,8 +651,7 @@ static int osc_create(const struct lu_env *env, struct obd_export *exp,
  * cookies to the MDS after committing destroy transactions.
  */
 static int osc_destroy(const struct lu_env *env, struct obd_export *exp,
-		       struct obdo *oa, struct lov_stripe_md *ea,
-		       struct obd_trans_info *oti, struct obd_export *md_export)
+		       struct obdo *oa, struct obd_trans_info *oti)
 {
 	struct client_obd *cli = &exp->exp_obd->u.cli;
 	struct ptlrpc_request *req;
@@ -794,42 +719,44 @@ static int osc_destroy(const struct lu_env *env, struct obd_export *exp,
 static void osc_announce_cached(struct client_obd *cli, struct obdo *oa,
 				long writing_bytes)
 {
-	u32 bits = OBD_MD_FLBLOCKS|OBD_MD_FLGRANT;
+	u32 bits = OBD_MD_FLBLOCKS | OBD_MD_FLGRANT;
 
 	LASSERT(!(oa->o_valid & bits));
 
 	oa->o_valid |= bits;
 	spin_lock(&cli->cl_loi_list_lock);
-	oa->o_dirty = cli->cl_dirty;
-	if (unlikely(cli->cl_dirty - cli->cl_dirty_transit >
-		     cli->cl_dirty_max)) {
+	oa->o_dirty = cli->cl_dirty_pages << PAGE_SHIFT;
+	if (unlikely(cli->cl_dirty_pages - cli->cl_dirty_transit >
+		     cli->cl_dirty_max_pages)) {
 		CERROR("dirty %lu - %lu > dirty_max %lu\n",
-		       cli->cl_dirty, cli->cl_dirty_transit, cli->cl_dirty_max);
+		       cli->cl_dirty_pages, cli->cl_dirty_transit,
+		       cli->cl_dirty_max_pages);
 		oa->o_undirty = 0;
-	} else if (unlikely(atomic_read(&obd_unstable_pages) +
-			    atomic_read(&obd_dirty_pages) -
-			    atomic_read(&obd_dirty_transit_pages) >
-			    (long)(obd_max_dirty_pages + 1))) {
+	} else if (unlikely(atomic_long_read(&obd_dirty_pages) -
+			    atomic_long_read(&obd_dirty_transit_pages) >
+			    (obd_max_dirty_pages + 1))) {
 		/* The atomic_read() allowing the atomic_inc() are
 		 * not covered by a lock thus they may safely race and trip
 		 * this CERROR() unless we add in a small fudge factor (+1).
 		 */
-		CERROR("%s: dirty %d + %d - %d > system dirty_max %d\n",
+		CERROR("%s: dirty %ld + %ld > system dirty_max %lu\n",
 		       cli->cl_import->imp_obd->obd_name,
-		       atomic_read(&obd_unstable_pages),
-		       atomic_read(&obd_dirty_pages),
-		       atomic_read(&obd_dirty_transit_pages),
+		       atomic_long_read(&obd_dirty_pages),
+		       atomic_long_read(&obd_dirty_transit_pages),
 		       obd_max_dirty_pages);
 		oa->o_undirty = 0;
-	} else if (unlikely(cli->cl_dirty_max - cli->cl_dirty > 0x7fffffff)) {
+	} else if (unlikely(cli->cl_dirty_max_pages - cli->cl_dirty_pages >
+		   0x7fffffff)) {
 		CERROR("dirty %lu - dirty_max %lu too big???\n",
-		       cli->cl_dirty, cli->cl_dirty_max);
+		       cli->cl_dirty_pages, cli->cl_dirty_max_pages);
 		oa->o_undirty = 0;
 	} else {
-		long max_in_flight = (cli->cl_max_pages_per_rpc <<
-				      PAGE_SHIFT)*
-				     (cli->cl_max_rpcs_in_flight + 1);
-		oa->o_undirty = max(cli->cl_dirty_max, max_in_flight);
+		unsigned long max_in_flight;
+
+		max_in_flight = (cli->cl_max_pages_per_rpc << PAGE_SHIFT) *
+				(cli->cl_max_rpcs_in_flight + 1);
+		oa->o_undirty = max(cli->cl_dirty_max_pages << PAGE_SHIFT,
+				    max_in_flight);
 	}
 	oa->o_grant = cli->cl_avail_grant + cli->cl_reserved_grant;
 	oa->o_dropped = cli->cl_lost_grant;
@@ -1029,22 +956,24 @@ static void osc_init_grant(struct client_obd *cli, struct obd_connect_data *ocd)
 {
 	/*
 	 * ocd_grant is the total grant amount we're expect to hold: if we've
-	 * been evicted, it's the new avail_grant amount, cl_dirty will drop
-	 * to 0 as inflight RPCs fail out; otherwise, it's avail_grant + dirty.
+	 * been evicted, it's the new avail_grant amount, cl_dirty_pages will
+	 * drop to 0 as inflight RPCs fail out; otherwise, it's avail_grant +
+	 * dirty.
 	 *
 	 * race is tolerable here: if we're evicted, but imp_state already
-	 * left EVICTED state, then cl_dirty must be 0 already.
+	 * left EVICTED state, then cl_dirty_pages must be 0 already.
 	 */
 	spin_lock(&cli->cl_loi_list_lock);
 	if (cli->cl_import->imp_state == LUSTRE_IMP_EVICTED)
 		cli->cl_avail_grant = ocd->ocd_grant;
 	else
-		cli->cl_avail_grant = ocd->ocd_grant - cli->cl_dirty;
+		cli->cl_avail_grant = ocd->ocd_grant -
+				      (cli->cl_dirty_pages << PAGE_SHIFT);
 
 	if (cli->cl_avail_grant < 0) {
 		CWARN("%s: available grant < 0: avail/ocd/dirty %ld/%u/%ld\n",
 		      cli->cl_import->imp_obd->obd_name, cli->cl_avail_grant,
-		      ocd->ocd_grant, cli->cl_dirty);
+		      ocd->ocd_grant, cli->cl_dirty_pages << PAGE_SHIFT);
 		/* workaround for servers which do not have the patch from
 		 * LU-2679
 		 */
@@ -1181,7 +1110,7 @@ static u32 osc_checksum_bulk(int nob, u32 pg_count,
 	}
 
 	while (nob > 0 && pg_count > 0) {
-		int count = pga[i]->count > nob ? nob : pga[i]->count;
+		unsigned int count = pga[i]->count > nob ? nob : pga[i]->count;
 
 		/* corrupt the data before we compute the checksum, to
 		 * simulate an OST->client data error
@@ -1191,7 +1120,7 @@ static u32 osc_checksum_bulk(int nob, u32 pg_count,
 			unsigned char *ptr = kmap(pga[i]->pg);
 			int off = pga[i]->off & ~PAGE_MASK;
 
-			memcpy(ptr + off, "bad1", min(4, nob));
+			memcpy(ptr + off, "bad1", min_t(typeof(nob), 4, nob));
 			kunmap(pga[i]->pg);
 		}
 		cfs_crypto_hash_update_page(hdesc, pga[i]->pg,
@@ -1335,11 +1264,11 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 
 		if (i > 0 && can_merge_pages(pg_prev, pg)) {
 			niobuf--;
-			niobuf->len += pg->count;
+			niobuf->rnb_len += pg->count;
 		} else {
-			niobuf->offset = pg->off;
-			niobuf->len = pg->count;
-			niobuf->flags = pg->flag;
+			niobuf->rnb_offset = pg->off;
+			niobuf->rnb_len = pg->count;
+			niobuf->rnb_flags = pg->flag;
 		}
 		pg_prev = pg;
 	}
@@ -1418,6 +1347,11 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 	INIT_LIST_HEAD(&aa->aa_oaps);
 
 	*reqp = req;
+	niobuf = req_capsule_client_get(pill, &RMF_NIOBUF_REMOTE);
+	CDEBUG(D_RPCTRACE, "brw rpc %p - object " DOSTID " offset %lld<>%lld\n",
+	       req, POSTID(&oa->o_oi), niobuf[0].rnb_offset,
+	       niobuf[niocount - 1].rnb_offset + niobuf[niocount - 1].rnb_len);
+
 	return 0;
 
  out:
@@ -1463,7 +1397,8 @@ static int check_write_checksum(struct obdo *oa, const lnet_process_id_t *peer,
 			   oa->o_valid & OBD_MD_FLFID ? oa->o_parent_oid : 0,
 			   oa->o_valid & OBD_MD_FLFID ? oa->o_parent_ver : 0,
 			   POSTID(&oa->o_oi), pga[0]->off,
-			   pga[page_count-1]->off + pga[page_count-1]->count - 1);
+			   pga[page_count - 1]->off +
+			   pga[page_count - 1]->count - 1);
 	CERROR("original client csum %x (type %x), server csum %x (type %x), client csum now %x\n",
 	       client_cksum, client_cksum_type,
 	       server_cksum, cksum_type, new_cksum);
@@ -1565,7 +1500,8 @@ static int osc_brw_fini_request(struct ptlrpc_request *req, int rc)
 		char *router = "";
 		enum cksum_type cksum_type;
 
-		cksum_type = cksum_type_unpack(body->oa.o_valid&OBD_MD_FLFLAGS ?
+		cksum_type = cksum_type_unpack(body->oa.o_valid &
+					       OBD_MD_FLFLAGS ?
 					       body->oa.o_flags : 0);
 		client_cksum = osc_checksum_bulk(rc, aa->aa_page_count,
 						 aa->aa_ppga, OST_READ,
@@ -1794,7 +1730,8 @@ static int brw_interpret(const struct lu_env *env,
 
 		if (lustre_msg_get_opc(req->rq_reqmsg) == OST_WRITE) {
 			struct lov_oinfo *loi = cl2osc(obj)->oo_oinfo;
-			loff_t last_off = last->oap_count + last->oap_obj_off;
+			loff_t last_off = last->oap_count + last->oap_obj_off +
+					  last->oap_page_off;
 
 			/* Change file size if this is an out of quota or
 			 * direct IO write and it extends the file size
@@ -1812,11 +1749,14 @@ static int brw_interpret(const struct lu_env *env,
 		}
 
 		if (valid != 0)
-			cl_object_attr_set(env, obj, attr, valid);
+			cl_object_attr_update(env, obj, attr, valid);
 		cl_object_attr_unlock(obj);
 	}
 	kmem_cache_free(obdo_cachep, aa->aa_oa);
 
+	if (lustre_msg_get_opc(req->rq_reqmsg) == OST_WRITE && rc == 0)
+		osc_inc_unstable_pages(req);
+
 	list_for_each_entry_safe(ext, tmp, &aa->aa_exts, oe_link) {
 		list_del_init(&ext->oe_link);
 		osc_extent_finish(env, ext, 1, rc);
@@ -1847,21 +1787,21 @@ static int brw_interpret(const struct lu_env *env,
 
 static void brw_commit(struct ptlrpc_request *req)
 {
-	spin_lock(&req->rq_lock);
 	/*
 	 * If osc_inc_unstable_pages (via osc_extent_finish) races with
 	 * this called via the rq_commit_cb, I need to ensure
 	 * osc_dec_unstable_pages is still called. Otherwise unstable
 	 * pages may be leaked.
 	 */
-	if (req->rq_unstable) {
+	spin_lock(&req->rq_lock);
+	if (unlikely(req->rq_unstable)) {
+		req->rq_unstable = 0;
 		spin_unlock(&req->rq_lock);
 		osc_dec_unstable_pages(req);
-		spin_lock(&req->rq_lock);
 	} else {
 		req->rq_committed = 1;
+		spin_unlock(&req->rq_lock);
 	}
-	spin_unlock(&req->rq_lock);
 }
 
 /**
@@ -1881,13 +1821,13 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	struct osc_async_page *tmp;
 	struct cl_req *clerq = NULL;
 	enum cl_req_type crt = (cmd & OBD_BRW_WRITE) ? CRT_WRITE : CRT_READ;
-	struct ldlm_lock *lock = NULL;
 	struct cl_req_attr *crattr = NULL;
 	u64 starting_offset = OBD_OBJECT_EOF;
 	u64 ending_offset = 0;
 	int mpflag = 0;
 	int mem_tight = 0;
 	int page_count = 0;
+	bool soft_sync = false;
 	int i;
 	int rc;
 	struct ost_body *body;
@@ -1915,6 +1855,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 		}
 	}
 
+	soft_sync = osc_over_unstable_soft_limit(cli);
 	if (mem_tight)
 		mpflag = cfs_memory_pressure_get_and_set();
 
@@ -1947,10 +1888,11 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 				rc = PTR_ERR(clerq);
 				goto out;
 			}
-			lock = oap->oap_ldlm_lock;
 		}
 		if (mem_tight)
 			oap->oap_brw_flags |= OBD_BRW_MEMALLOC;
+		if (soft_sync)
+			oap->oap_brw_flags |= OBD_BRW_SOFT_SYNC;
 		pga[i] = &oap->oap_brw_page;
 		pga[i]->off = oap->oap_obj_off + oap->oap_page_off;
 		CDEBUG(0, "put page %p index %lu oap %p flg %x to pga\n",
@@ -1964,10 +1906,6 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	LASSERT(clerq);
 	crattr->cra_oa = oa;
 	cl_req_attr_set(env, clerq, crattr, ~0ULL);
-	if (lock) {
-		oa->o_handle = lock->l_remote_handle;
-		oa->o_valid |= OBD_MD_FLHANDLE;
-	}
 
 	rc = cl_req_prep(env, clerq);
 	if (rc != 0) {
@@ -1998,7 +1936,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	body = req_capsule_client_get(&req->rq_pill, &RMF_OST_BODY);
 	crattr->cra_oa = &body->oa;
 	cl_req_attr_set(env, clerq, crattr,
-			OBD_MD_FLMTIME|OBD_MD_FLCTIME|OBD_MD_FLATIME);
+			OBD_MD_FLMTIME | OBD_MD_FLCTIME | OBD_MD_FLATIME);
 
 	lustre_msg_set_jobid(req->rq_reqmsg, crattr->cra_jobid);
 
@@ -2044,7 +1982,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	}
 	spin_unlock(&cli->cl_loi_list_lock);
 
-	DEBUG_REQ(D_INODE, req, "%d pages, aa %p. now %dr/%dw in flight",
+	DEBUG_REQ(D_INODE, req, "%d pages, aa %p. now %ur/%dw in flight",
 		  page_count, aa, cli->cl_r_in_flight,
 		  cli->cl_w_in_flight);
 
@@ -2116,27 +2054,6 @@ static int osc_set_data_with_check(struct lustre_handle *lockh,
 	return set;
 }
 
-/* find any ldlm lock of the inode in osc
- * return 0    not find
- *	1    find one
- *      < 0    error
- */
-static int osc_find_cbdata(struct obd_export *exp, struct lov_stripe_md *lsm,
-			   ldlm_iterator_t replace, void *data)
-{
-	struct ldlm_res_id res_id;
-	struct obd_device *obd = class_exp2obd(exp);
-	int rc = 0;
-
-	ostid_build_res_name(&lsm->lsm_oi, &res_id);
-	rc = ldlm_resource_iterate(obd->obd_namespace, &res_id, replace, data);
-	if (rc == LDLM_ITER_STOP)
-		return 1;
-	if (rc == LDLM_ITER_CONTINUE)
-		return 0;
-	return rc;
-}
-
 static int osc_enqueue_fini(struct ptlrpc_request *req,
 			    osc_enqueue_upcall_f upcall, void *cookie,
 			    struct lustre_handle *lockh, enum ldlm_mode mode,
@@ -2586,71 +2503,6 @@ static int osc_statfs(const struct lu_env *env, struct obd_export *exp,
 	return rc;
 }
 
-/* Retrieve object striping information.
- *
- * @lmmu is a pointer to an in-core struct with lmm_ost_count indicating
- * the maximum number of OST indices which will fit in the user buffer.
- * lmm_magic must be LOV_MAGIC (we only use 1 slot here).
- */
-static int osc_getstripe(struct lov_stripe_md *lsm,
-			 struct lov_user_md __user *lump)
-{
-	/* we use lov_user_md_v3 because it is larger than lov_user_md_v1 */
-	struct lov_user_md_v3 lum, *lumk;
-	struct lov_user_ost_data_v1 *lmm_objects;
-	int rc = 0, lum_size;
-
-	if (!lsm)
-		return -ENODATA;
-
-	/* we only need the header part from user space to get lmm_magic and
-	 * lmm_stripe_count, (the header part is common to v1 and v3)
-	 */
-	lum_size = sizeof(struct lov_user_md_v1);
-	if (copy_from_user(&lum, lump, lum_size))
-		return -EFAULT;
-
-	if ((lum.lmm_magic != LOV_USER_MAGIC_V1) &&
-	    (lum.lmm_magic != LOV_USER_MAGIC_V3))
-		return -EINVAL;
-
-	/* lov_user_md_vX and lov_mds_md_vX must have the same size */
-	LASSERT(sizeof(struct lov_user_md_v1) == sizeof(struct lov_mds_md_v1));
-	LASSERT(sizeof(struct lov_user_md_v3) == sizeof(struct lov_mds_md_v3));
-	LASSERT(sizeof(lum.lmm_objects[0]) == sizeof(lumk->lmm_objects[0]));
-
-	/* we can use lov_mds_md_size() to compute lum_size
-	 * because lov_user_md_vX and lov_mds_md_vX have the same size
-	 */
-	if (lum.lmm_stripe_count > 0) {
-		lum_size = lov_mds_md_size(lum.lmm_stripe_count, lum.lmm_magic);
-		lumk = kzalloc(lum_size, GFP_NOFS);
-		if (!lumk)
-			return -ENOMEM;
-
-		if (lum.lmm_magic == LOV_USER_MAGIC_V1)
-			lmm_objects =
-			    &(((struct lov_user_md_v1 *)lumk)->lmm_objects[0]);
-		else
-			lmm_objects = &(lumk->lmm_objects[0]);
-		lmm_objects->l_ost_oi = lsm->lsm_oi;
-	} else {
-		lum_size = lov_mds_md_size(0, lum.lmm_magic);
-		lumk = &lum;
-	}
-
-	lumk->lmm_oi = lsm->lsm_oi;
-	lumk->lmm_stripe_count = 1;
-
-	if (copy_to_user(lump, lumk, lum_size))
-		rc = -EFAULT;
-
-	if (lumk != &lum)
-		kfree(lumk);
-
-	return rc;
-}
-
 static int osc_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 			 void *karg, void __user *uarg)
 {
@@ -2664,57 +2516,6 @@ static int osc_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 		return -EINVAL;
 	}
 	switch (cmd) {
-	case OBD_IOC_LOV_GET_CONFIG: {
-		char *buf;
-		struct lov_desc *desc;
-		struct obd_uuid uuid;
-
-		buf = NULL;
-		len = 0;
-		if (obd_ioctl_getdata(&buf, &len, uarg)) {
-			err = -EINVAL;
-			goto out;
-		}
-
-		data = (struct obd_ioctl_data *)buf;
-
-		if (sizeof(*desc) > data->ioc_inllen1) {
-			obd_ioctl_freedata(buf, len);
-			err = -EINVAL;
-			goto out;
-		}
-
-		if (data->ioc_inllen2 < sizeof(uuid)) {
-			obd_ioctl_freedata(buf, len);
-			err = -EINVAL;
-			goto out;
-		}
-
-		desc = (struct lov_desc *)data->ioc_inlbuf1;
-		desc->ld_tgt_count = 1;
-		desc->ld_active_tgt_count = 1;
-		desc->ld_default_stripe_count = 1;
-		desc->ld_default_stripe_size = 0;
-		desc->ld_default_stripe_offset = 0;
-		desc->ld_pattern = 0;
-		memcpy(&desc->ld_uuid, &obd->obd_uuid, sizeof(uuid));
-
-		memcpy(data->ioc_inlbuf2, &obd->obd_uuid, sizeof(uuid));
-
-		err = copy_to_user(uarg, buf, len);
-		if (err)
-			err = -EFAULT;
-		obd_ioctl_freedata(buf, len);
-		goto out;
-	}
-	case LL_IOC_LOV_SETSTRIPE:
-		err = obd_alloc_memmd(exp, karg);
-		if (err > 0)
-			err = 0;
-		goto out;
-	case LL_IOC_LOV_GETSTRIPE:
-		err = osc_getstripe(karg, uarg);
-		goto out;
 	case OBD_IOC_CLIENT_RECOVER:
 		err = ptlrpc_recover_import(obd->u.cli.cl_import,
 					    data->ioc_inlbuf1, 0);
@@ -2749,51 +2550,7 @@ static int osc_get_info(const struct lu_env *env, struct obd_export *exp,
 	if (!vallen || !val)
 		return -EFAULT;
 
-	if (KEY_IS(KEY_LOCK_TO_STRIPE)) {
-		__u32 *stripe = val;
-		*vallen = sizeof(*stripe);
-		*stripe = 0;
-		return 0;
-	} else if (KEY_IS(KEY_LAST_ID)) {
-		struct ptlrpc_request *req;
-		u64 *reply;
-		char *tmp;
-		int rc;
-
-		req = ptlrpc_request_alloc(class_exp2cliimp(exp),
-					   &RQF_OST_GET_INFO_LAST_ID);
-		if (!req)
-			return -ENOMEM;
-
-		req_capsule_set_size(&req->rq_pill, &RMF_SETINFO_KEY,
-				     RCL_CLIENT, keylen);
-		rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_GET_INFO);
-		if (rc) {
-			ptlrpc_request_free(req);
-			return rc;
-		}
-
-		tmp = req_capsule_client_get(&req->rq_pill, &RMF_SETINFO_KEY);
-		memcpy(tmp, key, keylen);
-
-		req->rq_no_delay = 1;
-		req->rq_no_resend = 1;
-		ptlrpc_request_set_replen(req);
-		rc = ptlrpc_queue_wait(req);
-		if (rc)
-			goto out;
-
-		reply = req_capsule_server_get(&req->rq_pill, &RMF_OBD_ID);
-		if (!reply) {
-			rc = -EPROTO;
-			goto out;
-		}
-
-		*((u64 *)val) = *reply;
-out:
-		ptlrpc_req_finished(req);
-		return rc;
-	} else if (KEY_IS(KEY_FIEMAP)) {
+	if (KEY_IS(KEY_FIEMAP)) {
 		struct ll_fiemap_info_key *fm_key = key;
 		struct ldlm_res_id res_id;
 		ldlm_policy_data_t policy;
@@ -2931,11 +2688,11 @@ static int osc_set_info_async(const struct lu_env *env, struct obd_export *exp,
 
 	if (KEY_IS(KEY_CACHE_LRU_SHRINK)) {
 		struct client_obd *cli = &obd->u.cli;
-		int nr = atomic_read(&cli->cl_lru_in_list) >> 1;
-		int target = *(int *)val;
+		long nr = atomic_long_read(&cli->cl_lru_in_list) >> 1;
+		long target = *(long *)val;
 
 		nr = osc_lru_shrink(env, cli, min(nr, target), true);
-		*(int *)val -= nr;
+		*(long *)val -= nr;
 		return 0;
 	}
 
@@ -3014,8 +2771,9 @@ static int osc_reconnect(const struct lu_env *env,
 		long lost_grant;
 
 		spin_lock(&cli->cl_loi_list_lock);
-		data->ocd_grant = (cli->cl_avail_grant + cli->cl_dirty) ?:
-				2 * cli_brw_size(obd);
+		data->ocd_grant = (cli->cl_avail_grant +
+				   (cli->cl_dirty_pages << PAGE_SHIFT)) ?:
+				   2 * cli_brw_size(obd);
 		lost_grant = cli->cl_lost_grant;
 		cli->cl_lost_grant = 0;
 		spin_unlock(&cli->cl_loi_list_lock);
@@ -3346,7 +3104,6 @@ static struct obd_ops osc_obd_ops = {
 	.disconnect     = osc_disconnect,
 	.statfs         = osc_statfs,
 	.statfs_async   = osc_statfs_async,
-	.packmd         = osc_packmd,
 	.unpackmd       = osc_unpackmd,
 	.create         = osc_create,
 	.destroy        = osc_destroy,
@@ -3354,7 +3111,6 @@ static struct obd_ops osc_obd_ops = {
 	.getattr_async  = osc_getattr_async,
 	.setattr        = osc_setattr,
 	.setattr_async  = osc_setattr_async,
-	.find_cbdata    = osc_find_cbdata,
 	.iocontrol      = osc_iocontrol,
 	.get_info       = osc_get_info,
 	.set_info_async = osc_set_info_async,
diff --git a/drivers/staging/lustre/lustre/ptlrpc/client.c b/drivers/staging/lustre/lustre/ptlrpc/client.c
index d4463d7c81d2..8c51d51a678b 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/client.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/client.c
@@ -45,6 +45,7 @@
 
 static int ptlrpc_send_new_req(struct ptlrpc_request *req);
 static int ptlrpcd_check_work(struct ptlrpc_request *req);
+static int ptlrpc_unregister_reply(struct ptlrpc_request *request, int async);
 
 /**
  * Initialize passed in client structure \a cl.
@@ -89,7 +90,6 @@ struct ptlrpc_connection *ptlrpc_uuid_to_connection(struct obd_uuid *uuid)
 
 	return c;
 }
-EXPORT_SYMBOL(ptlrpc_uuid_to_connection);
 
 /**
  * Allocate and initialize new bulk descriptor on the sender.
@@ -202,7 +202,7 @@ void __ptlrpc_free_bulk(struct ptlrpc_bulk_desc *desc, int unpin)
 
 	if (unpin) {
 		for (i = 0; i < desc->bd_iov_count; i++)
-			put_page(desc->bd_iov[i].kiov_page);
+			put_page(desc->bd_iov[i].bv_page);
 	}
 
 	kfree(desc);
@@ -283,8 +283,8 @@ int ptlrpc_at_get_net_latency(struct ptlrpc_request *req)
 }
 
 /* Adjust expected network latency */
-static void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req,
-				      unsigned int service_time)
+void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req,
+			       unsigned int service_time)
 {
 	unsigned int nl, oldnl;
 	struct imp_at *at;
@@ -364,31 +364,37 @@ static int ptlrpc_at_recv_early_reply(struct ptlrpc_request *req)
 	}
 
 	rc = unpack_reply(early_req);
-	if (rc == 0) {
-		/* Expecting to increase the service time estimate here */
-		ptlrpc_at_adj_service(req,
-			lustre_msg_get_timeout(early_req->rq_repmsg));
-		ptlrpc_at_adj_net_latency(req,
-			lustre_msg_get_service_time(early_req->rq_repmsg));
-	}
-
-	sptlrpc_cli_finish_early_reply(early_req);
-
-	if (rc != 0) {
+	if (rc) {
+		sptlrpc_cli_finish_early_reply(early_req);
 		spin_lock(&req->rq_lock);
 		return rc;
 	}
 
-	/* Adjust the local timeout for this req */
-	ptlrpc_at_set_req_timeout(req);
+	/*
+	 * Use new timeout value just to adjust the local value for this
+	 * request, don't include it into at_history. It is unclear yet why
+	 * service time increased and should it be counted or skipped, e.g.
+	 * that can be recovery case or some error or server, the real reply
+	 * will add all new data if it is worth to add.
+	 */
+	req->rq_timeout = lustre_msg_get_timeout(early_req->rq_repmsg);
+	lustre_msg_set_timeout(req->rq_reqmsg, req->rq_timeout);
+
+	/* Network latency can be adjusted, it is pure network delays */
+	ptlrpc_at_adj_net_latency(req,
+				  lustre_msg_get_service_time(early_req->rq_repmsg));
+
+	sptlrpc_cli_finish_early_reply(early_req);
 
 	spin_lock(&req->rq_lock);
 	olddl = req->rq_deadline;
 	/*
-	 * server assumes it now has rq_timeout from when it sent the
-	 * early reply, so client should give it at least that long.
+	 * server assumes it now has rq_timeout from when the request
+	 * arrived, so the client should give it at least that long.
+	 * since we don't know the arrival time we'll use the original
+	 * sent time
 	 */
-	req->rq_deadline = ktime_get_real_seconds() + req->rq_timeout +
+	req->rq_deadline = req->rq_sent + req->rq_timeout +
 			   ptlrpc_at_get_net_latency(req);
 
 	DEBUG_REQ(D_ADAPTTO, req,
@@ -884,7 +890,6 @@ struct ptlrpc_request_set *ptlrpc_prep_fcset(int max, set_producer_func func,
 
 	return set;
 }
-EXPORT_SYMBOL(ptlrpc_prep_fcset);
 
 /**
  * Wind down and free request set structure previously allocated with
@@ -1004,7 +1009,6 @@ void ptlrpc_set_add_new_req(struct ptlrpcd_ctl *pc,
 			wake_up(&pc->pc_partners[i]->pc_set->set_waitq);
 	}
 }
-EXPORT_SYMBOL(ptlrpc_set_add_new_req);
 
 /**
  * Based on the current state of the import, determine if the request
@@ -1035,8 +1039,8 @@ static int ptlrpc_import_delay_req(struct obd_import *imp,
 		*status = -EIO;
 	} else if (ptlrpc_send_limit_expired(req)) {
 		/* probably doesn't need to be a D_ERROR after initial testing */
-		DEBUG_REQ(D_ERROR, req, "send limit expired ");
-		*status = -EIO;
+		DEBUG_REQ(D_HA, req, "send limit expired ");
+		*status = -ETIMEDOUT;
 	} else if (req->rq_send_state == LUSTRE_IMP_CONNECTING &&
 		   imp->imp_state == LUSTRE_IMP_CONNECTING) {
 		/* allow CONNECT even if import is invalid */
@@ -1073,36 +1077,42 @@ static int ptlrpc_import_delay_req(struct obd_import *imp,
 }
 
 /**
- * Decide if the error message regarding provided request \a req
- * should be printed to the console or not.
- * Makes it's decision on request status and other properties.
- * Returns 1 to print error on the system console or 0 if not.
+ * Decide if the error message should be printed to the console or not.
+ * Makes its decision based on request type, status, and failure frequency.
+ *
+ * \param[in] req  request that failed and may need a console message
+ *
+ * \retval false if no message should be printed
+ * \retval true  if console message should be printed
  */
-static int ptlrpc_console_allow(struct ptlrpc_request *req)
+static bool ptlrpc_console_allow(struct ptlrpc_request *req)
 {
 	__u32 opc;
-	int err;
 
 	LASSERT(req->rq_reqmsg);
 	opc = lustre_msg_get_opc(req->rq_reqmsg);
 
-	/*
-	 * Suppress particular reconnect errors which are to be expected.  No
-	 * errors are suppressed for the initial connection on an import
-	 */
-	if ((lustre_handle_is_used(&req->rq_import->imp_remote_handle)) &&
-	    (opc == OST_CONNECT || opc == MDS_CONNECT || opc == MGS_CONNECT)) {
+	/* Suppress particular reconnect errors which are to be expected. */
+	if (opc == OST_CONNECT || opc == MDS_CONNECT || opc == MGS_CONNECT) {
+		int err;
+
 		/* Suppress timed out reconnect requests */
-		if (req->rq_timedout)
-			return 0;
+		if (lustre_handle_is_used(&req->rq_import->imp_remote_handle) ||
+		    req->rq_timedout)
+			return false;
 
-		/* Suppress unavailable/again reconnect requests */
+		/*
+		 * Suppress most unavailable/again reconnect requests, but
+		 * print occasionally so it is clear client is trying to
+		 * connect to a server where no target is running.
+		 */
 		err = lustre_msg_get_status(req->rq_repmsg);
-		if (err == -ENODEV || err == -EAGAIN)
-			return 0;
+		if ((err == -ENODEV || err == -EAGAIN) &&
+		    req->rq_import->imp_conn_cnt % 30 != 20)
+			return false;
 	}
 
-	return 1;
+	return true;
 }
 
 /**
@@ -1116,14 +1126,14 @@ static int ptlrpc_check_status(struct ptlrpc_request *req)
 	err = lustre_msg_get_status(req->rq_repmsg);
 	if (lustre_msg_get_type(req->rq_repmsg) == PTL_RPC_MSG_ERR) {
 		struct obd_import *imp = req->rq_import;
+		lnet_nid_t nid = imp->imp_connection->c_peer.nid;
 		__u32 opc = lustre_msg_get_opc(req->rq_reqmsg);
 
 		if (ptlrpc_console_allow(req))
-			LCONSOLE_ERROR_MSG(0x011, "%s: Communicating with %s, operation %s failed with %d.\n",
+			LCONSOLE_ERROR_MSG(0x011, "%s: operation %s to node %s failed: rc = %d\n",
 					   imp->imp_obd->obd_name,
-					   libcfs_nid2str(
-						   imp->imp_connection->c_peer.nid),
-					   ll_opcode2str(opc), err);
+					   ll_opcode2str(opc),
+					   libcfs_nid2str(nid), err);
 		return err < 0 ? err : -EINVAL;
 	}
 
@@ -1280,7 +1290,7 @@ static int after_reply(struct ptlrpc_request *req)
 		 * some reason. Try to reconnect, and if that fails, punt to
 		 * the upcall.
 		 */
-		if (ll_rpc_recoverable_error(rc)) {
+		if (ptlrpc_recoverable_error(rc)) {
 			if (req->rq_send_state != LUSTRE_IMP_FULL ||
 			    imp->imp_obd->obd_no_recov || imp->imp_dlm_fake) {
 				return rc;
@@ -1628,8 +1638,10 @@ int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set)
 			    req->rq_waiting || req->rq_wait_ctx) {
 				int status;
 
-				if (!ptlrpc_unregister_reply(req, 1))
+				if (!ptlrpc_unregister_reply(req, 1)) {
+					ptlrpc_unregister_bulk(req, 1);
 					continue;
+				}
 
 				spin_lock(&imp->imp_lock);
 				if (ptlrpc_import_delay_req(imp, req,
@@ -1995,7 +2007,6 @@ int ptlrpc_expired_set(void *data)
 	 */
 	return 1;
 }
-EXPORT_SYMBOL(ptlrpc_expired_set);
 
 /**
  * Sets rq_intr flag in \a req under spinlock.
@@ -2012,7 +2023,7 @@ EXPORT_SYMBOL(ptlrpc_mark_interrupted);
  * Interrupts (sets interrupted flag) all uncompleted requests in
  * a set \a data. Callback for l_wait_event for interruptible waits.
  */
-void ptlrpc_interrupted_set(void *data)
+static void ptlrpc_interrupted_set(void *data)
 {
 	struct ptlrpc_request_set *set = data;
 	struct list_head *tmp;
@@ -2030,7 +2041,6 @@ void ptlrpc_interrupted_set(void *data)
 		ptlrpc_mark_interrupted(req);
 	}
 }
-EXPORT_SYMBOL(ptlrpc_interrupted_set);
 
 /**
  * Get the smallest timeout in the set; this does NOT set a timeout.
@@ -2074,7 +2084,6 @@ int ptlrpc_set_next_timeout(struct ptlrpc_request_set *set)
 	}
 	return timeout;
 }
-EXPORT_SYMBOL(ptlrpc_set_next_timeout);
 
 /**
  * Send all unset request from the set and then wait until all
@@ -2325,7 +2334,7 @@ EXPORT_SYMBOL(ptlrpc_req_xid);
  * The request owner (i.e. the thread doing the I/O) must call...
  * Returns 0 on success or 1 if unregistering cannot be made.
  */
-int ptlrpc_unregister_reply(struct ptlrpc_request *request, int async)
+static int ptlrpc_unregister_reply(struct ptlrpc_request *request, int async)
 {
 	int rc;
 	wait_queue_head_t *wq;
@@ -2390,7 +2399,6 @@ int ptlrpc_unregister_reply(struct ptlrpc_request *request, int async)
 	}
 	return 0;
 }
-EXPORT_SYMBOL(ptlrpc_unregister_reply);
 
 static void ptlrpc_free_request(struct ptlrpc_request *req)
 {
@@ -2451,7 +2459,8 @@ void ptlrpc_free_committed(struct obd_import *imp)
 	       imp->imp_obd->obd_name, imp->imp_peer_committed_transno,
 	       imp->imp_generation);
 
-	if (imp->imp_generation != imp->imp_last_generation_checked)
+	if (imp->imp_generation != imp->imp_last_generation_checked ||
+	    !imp->imp_last_transno_checked)
 		skip_committed_list = false;
 
 	imp->imp_last_transno_checked = imp->imp_peer_committed_transno;
@@ -2499,6 +2508,9 @@ free_req:
 		if (req->rq_import_generation < imp->imp_generation) {
 			DEBUG_REQ(D_RPCTRACE, req, "free stale open request");
 			ptlrpc_free_request(req);
+		} else if (!req->rq_replay) {
+			DEBUG_REQ(D_RPCTRACE, req, "free closed open request");
+			ptlrpc_free_request(req);
 		}
 	}
 }
@@ -2541,7 +2553,6 @@ void ptlrpc_resend_req(struct ptlrpc_request *req)
 	ptlrpc_client_wake_req(req);
 	spin_unlock(&req->rq_lock);
 }
-EXPORT_SYMBOL(ptlrpc_resend_req);
 
 /**
  * Grab additional reference on a request \a req
@@ -2610,7 +2621,6 @@ void ptlrpc_retain_replayable_request(struct ptlrpc_request *req,
 
 	list_add(&req->rq_replay_list, &imp->imp_replay_list);
 }
-EXPORT_SYMBOL(ptlrpc_retain_replayable_request);
 
 /**
  * Send request and wait until it completes.
@@ -2783,7 +2793,6 @@ int ptlrpc_replay_req(struct ptlrpc_request *req)
 	ptlrpcd_add_req(req);
 	return 0;
 }
-EXPORT_SYMBOL(ptlrpc_replay_req);
 
 /**
  * Aborts all in-flight request on import \a imp sending and delayed lists
@@ -2843,7 +2852,6 @@ void ptlrpc_abort_inflight(struct obd_import *imp)
 
 	spin_unlock(&imp->imp_lock);
 }
-EXPORT_SYMBOL(ptlrpc_abort_inflight);
 
 /**
  * Abort all uncompleted requests in request set \a set
@@ -2929,7 +2937,6 @@ __u64 ptlrpc_next_xid(void)
 
 	return next;
 }
-EXPORT_SYMBOL(ptlrpc_next_xid);
 
 /**
  * Get a glimpse at what next xid value might have been.
diff --git a/drivers/staging/lustre/lustre/ptlrpc/connection.c b/drivers/staging/lustre/lustre/ptlrpc/connection.c
index 177a379da9fa..7b020d60c9e5 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/connection.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/connection.c
@@ -82,7 +82,6 @@ out:
 	       libcfs_nid2str(conn->c_peer.nid));
 	return conn;
 }
-EXPORT_SYMBOL(ptlrpc_connection_get);
 
 int ptlrpc_connection_put(struct ptlrpc_connection *conn)
 {
@@ -118,7 +117,6 @@ int ptlrpc_connection_put(struct ptlrpc_connection *conn)
 
 	return rc;
 }
-EXPORT_SYMBOL(ptlrpc_connection_put);
 
 struct ptlrpc_connection *
 ptlrpc_connection_addref(struct ptlrpc_connection *conn)
@@ -130,7 +128,6 @@ ptlrpc_connection_addref(struct ptlrpc_connection *conn)
 
 	return conn;
 }
-EXPORT_SYMBOL(ptlrpc_connection_addref);
 
 int ptlrpc_connection_init(void)
 {
@@ -146,13 +143,11 @@ int ptlrpc_connection_init(void)
 
 	return 0;
 }
-EXPORT_SYMBOL(ptlrpc_connection_init);
 
 void ptlrpc_connection_fini(void)
 {
 	cfs_hash_putref(conn_hash);
 }
-EXPORT_SYMBOL(ptlrpc_connection_fini);
 
 /*
  * Hash operations for net_peer<->connection
diff --git a/drivers/staging/lustre/lustre/ptlrpc/events.c b/drivers/staging/lustre/lustre/ptlrpc/events.c
index b1ce72511509..283dfb296d35 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/events.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/events.c
@@ -543,7 +543,7 @@ static int ptlrpc_ni_init(void)
 	rc = LNetNIInit(pid);
 	if (rc < 0) {
 		CDEBUG(D_NET, "Can't init network interface: %d\n", rc);
-		return -ENOENT;
+		return rc;
 	}
 
 	/* CAVEAT EMPTOR: how we process portals events is _radically_
@@ -561,7 +561,7 @@ static int ptlrpc_ni_init(void)
 	CERROR("Failed to allocate event queue: %d\n", rc);
 	LNetNIFini();
 
-	return -ENOMEM;
+	return rc;
 }
 
 int ptlrpc_init_portals(void)
@@ -570,7 +570,7 @@ int ptlrpc_init_portals(void)
 
 	if (rc != 0) {
 		CERROR("network initialisation failed\n");
-		return -EIO;
+		return rc;
 	}
 	rc = ptlrpcd_addref();
 	if (rc == 0)
diff --git a/drivers/staging/lustre/lustre/ptlrpc/import.c b/drivers/staging/lustre/lustre/ptlrpc/import.c
index 3292e6ea0102..a23d0a05b574 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/import.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/import.c
@@ -307,7 +307,8 @@ void ptlrpc_invalidate_import(struct obd_import *imp)
 		 */
 		lwi = LWI_TIMEOUT_INTERVAL(
 			cfs_timeout_cap(cfs_time_seconds(timeout)),
-			(timeout > 1)?cfs_time_seconds(1):cfs_time_seconds(1)/2,
+			(timeout > 1) ? cfs_time_seconds(1) :
+			cfs_time_seconds(1) / 2,
 			NULL, NULL);
 		rc = l_wait_event(imp->imp_recovery_waitq,
 				  (atomic_read(&imp->imp_inflight) == 0),
@@ -424,7 +425,6 @@ void ptlrpc_fail_import(struct obd_import *imp, __u32 conn_cnt)
 		ptlrpc_pinger_force(imp);
 	}
 }
-EXPORT_SYMBOL(ptlrpc_fail_import);
 
 int ptlrpc_reconnect_import(struct obd_import *imp)
 {
@@ -698,7 +698,8 @@ int ptlrpc_connect_import(struct obd_import *imp)
 	request->rq_send_state = LUSTRE_IMP_CONNECTING;
 	/* Allow a slightly larger reply for future growth compatibility */
 	req_capsule_set_size(&request->rq_pill, &RMF_CONNECT_DATA, RCL_SERVER,
-			     sizeof(struct obd_connect_data)+16*sizeof(__u64));
+			     sizeof(struct obd_connect_data) +
+			     16 * sizeof(__u64));
 	ptlrpc_request_set_replen(request);
 	request->rq_interpret_reply = ptlrpc_connect_interpret;
 
@@ -750,6 +751,153 @@ static int ptlrpc_busy_reconnect(int rc)
 	return (rc == -EBUSY) || (rc == -EAGAIN);
 }
 
+static int ptlrpc_connect_set_flags(struct obd_import *imp,
+				    struct obd_connect_data *ocd,
+				    u64 old_connect_flags,
+				    struct obd_export *exp, int init_connect)
+{
+	struct client_obd *cli = &imp->imp_obd->u.cli;
+	static bool warned;
+
+	if ((imp->imp_connect_flags_orig & OBD_CONNECT_IBITS) &&
+	    !(ocd->ocd_connect_flags & OBD_CONNECT_IBITS)) {
+		LCONSOLE_WARN("%s: MDS %s does not support ibits lock, either very old or invalid: requested %#llx, replied %#llx\n",
+			      imp->imp_obd->obd_name,
+			      imp->imp_connection->c_remote_uuid.uuid,
+			      imp->imp_connect_flags_orig,
+			      ocd->ocd_connect_flags);
+		return -EPROTO;
+	}
+
+	spin_lock(&imp->imp_lock);
+	list_del(&imp->imp_conn_current->oic_item);
+	list_add(&imp->imp_conn_current->oic_item, &imp->imp_conn_list);
+	imp->imp_last_success_conn = imp->imp_conn_current->oic_last_attempt;
+
+	spin_unlock(&imp->imp_lock);
+
+	if (!warned && (ocd->ocd_connect_flags & OBD_CONNECT_VERSION) &&
+	    (ocd->ocd_version > LUSTRE_VERSION_CODE +
+				LUSTRE_VERSION_OFFSET_WARN ||
+	     ocd->ocd_version < LUSTRE_VERSION_CODE -
+				LUSTRE_VERSION_OFFSET_WARN)) {
+		/*
+		 * Sigh, some compilers do not like #ifdef in the middle
+		 * of macro arguments
+		 */
+		const char *older = "older than client. Consider upgrading server";
+		const char *newer = "newer than client. Consider recompiling application";
+
+		LCONSOLE_WARN("Server %s version (%d.%d.%d.%d) is much %s (%s)\n",
+			      obd2cli_tgt(imp->imp_obd),
+			      OBD_OCD_VERSION_MAJOR(ocd->ocd_version),
+			      OBD_OCD_VERSION_MINOR(ocd->ocd_version),
+			      OBD_OCD_VERSION_PATCH(ocd->ocd_version),
+			      OBD_OCD_VERSION_FIX(ocd->ocd_version),
+			      ocd->ocd_version > LUSTRE_VERSION_CODE ?
+			      newer : older, LUSTRE_VERSION_STRING);
+		warned = true;
+	}
+
+#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0)
+	/*
+	 * Check if server has LU-1252 fix applied to not always swab
+	 * the IR MNE entries. Do this only once per connection.  This
+	 * fixup is version-limited, because we don't want to carry the
+	 * OBD_CONNECT_MNE_SWAB flag around forever, just so long as we
+	 * need interop with unpatched 2.2 servers.  For newer servers,
+	 * the client will do MNE swabbing only as needed.  LU-1644
+	 */
+	if (unlikely((ocd->ocd_connect_flags & OBD_CONNECT_VERSION) &&
+		     !(ocd->ocd_connect_flags & OBD_CONNECT_MNE_SWAB) &&
+		     OBD_OCD_VERSION_MAJOR(ocd->ocd_version) == 2 &&
+		     OBD_OCD_VERSION_MINOR(ocd->ocd_version) == 2 &&
+		     OBD_OCD_VERSION_PATCH(ocd->ocd_version) < 55 &&
+		     !strcmp(imp->imp_obd->obd_type->typ_name,
+			     LUSTRE_MGC_NAME)))
+		imp->imp_need_mne_swab = 1;
+	else /* clear if server was upgraded since last connect */
+		imp->imp_need_mne_swab = 0;
+#endif
+
+	if (ocd->ocd_connect_flags & OBD_CONNECT_CKSUM) {
+		/*
+		 * We sent to the server ocd_cksum_types with bits set
+		 * for algorithms we understand. The server masked off
+		 * the checksum types it doesn't support
+		 */
+		if (!(ocd->ocd_cksum_types & cksum_types_supported_client())) {
+			LCONSOLE_WARN("The negotiation of the checksum algorithm to use with server %s failed (%x/%x), disabling checksums\n",
+				      obd2cli_tgt(imp->imp_obd),
+				      ocd->ocd_cksum_types,
+				      cksum_types_supported_client());
+			cli->cl_checksum = 0;
+			cli->cl_supp_cksum_types = OBD_CKSUM_ADLER;
+		} else {
+			cli->cl_supp_cksum_types = ocd->ocd_cksum_types;
+		}
+	} else {
+		/*
+		 * The server does not support OBD_CONNECT_CKSUM.
+		 * Enforce ADLER for backward compatibility
+		 */
+		cli->cl_supp_cksum_types = OBD_CKSUM_ADLER;
+	}
+	cli->cl_cksum_type = cksum_type_select(cli->cl_supp_cksum_types);
+
+	if (ocd->ocd_connect_flags & OBD_CONNECT_BRW_SIZE)
+		cli->cl_max_pages_per_rpc =
+			min(ocd->ocd_brw_size >> PAGE_SHIFT,
+			    cli->cl_max_pages_per_rpc);
+	else if (imp->imp_connect_op == MDS_CONNECT ||
+		 imp->imp_connect_op == MGS_CONNECT)
+		cli->cl_max_pages_per_rpc = 1;
+
+	LASSERT((cli->cl_max_pages_per_rpc <= PTLRPC_MAX_BRW_PAGES) &&
+		(cli->cl_max_pages_per_rpc > 0));
+
+	client_adjust_max_dirty(cli);
+
+	/*
+	 * Reset ns_connect_flags only for initial connect. It might be
+	 * changed in while using FS and if we reset it in reconnect
+	 * this leads to losing user settings done before such as
+	 * disable lru_resize, etc.
+	 */
+	if (old_connect_flags != exp_connect_flags(exp) || init_connect) {
+		CDEBUG(D_HA, "%s: Resetting ns_connect_flags to server flags: %#llx\n",
+		       imp->imp_obd->obd_name, ocd->ocd_connect_flags);
+		imp->imp_obd->obd_namespace->ns_connect_flags =
+			ocd->ocd_connect_flags;
+		imp->imp_obd->obd_namespace->ns_orig_connect_flags =
+			ocd->ocd_connect_flags;
+	}
+
+	if ((ocd->ocd_connect_flags & OBD_CONNECT_AT) &&
+	    (imp->imp_msg_magic == LUSTRE_MSG_MAGIC_V2))
+		/*
+		 * We need a per-message support flag, because
+		 * a. we don't know if the incoming connect reply
+		 *    supports AT or not (in reply_in_callback)
+		 *    until we unpack it.
+		 * b. failovered server means export and flags are gone
+		 *    (in ptlrpc_send_reply).
+		 *    Can only be set when we know AT is supported at
+		 *    both ends
+		 */
+		imp->imp_msghdr_flags |= MSGHDR_AT_SUPPORT;
+	else
+		imp->imp_msghdr_flags &= ~MSGHDR_AT_SUPPORT;
+
+	if ((ocd->ocd_connect_flags & OBD_CONNECT_FULL20) &&
+	    (imp->imp_msg_magic == LUSTRE_MSG_MAGIC_V2))
+		imp->imp_msghdr_flags |= MSGHDR_CKSUM_INCOMPAT18;
+	else
+		imp->imp_msghdr_flags &= ~MSGHDR_CKSUM_INCOMPAT18;
+
+	return 0;
+}
+
 /**
  * interpret_reply callback for connect RPCs.
  * Looks into returned status of connect operation and decides
@@ -762,7 +910,6 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 {
 	struct ptlrpc_connect_async_args *aa = data;
 	struct obd_import *imp = request->rq_import;
-	struct client_obd *cli = &imp->imp_obd->u.cli;
 	struct lustre_handle old_hdl;
 	__u64 old_connect_flags;
 	int msg_flags;
@@ -842,7 +989,21 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 	old_connect_flags = exp_connect_flags(exp);
 	exp->exp_connect_data = *ocd;
 	imp->imp_obd->obd_self_export->exp_connect_data = *ocd;
+
+	/*
+	 * The net statistics after (re-)connect is not valid anymore,
+	 * because may reflect other routing, etc.
+	 */
+	at_init(&imp->imp_at.iat_net_latency, 0, 0);
+	ptlrpc_at_adj_net_latency(request,
+				  lustre_msg_get_service_time(request->rq_repmsg));
+
+	/* Import flags should be updated before waking import at FULL state */
+	rc = ptlrpc_connect_set_flags(imp, ocd, old_connect_flags, exp,
+				      aa->pcaa_initial_connect);
 	class_export_put(exp);
+	if (rc)
+		goto out;
 
 	obd_import_event(imp->imp_obd, imp, IMP_EVENT_OCD);
 
@@ -987,151 +1148,13 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 
 finish:
 	rc = ptlrpc_import_recovery_state_machine(imp);
-	if (rc != 0) {
-		if (rc == -ENOTCONN) {
-			CDEBUG(D_HA, "evicted/aborted by %s@%s during recovery; invalidating and reconnecting\n",
-			       obd2cli_tgt(imp->imp_obd),
-			       imp->imp_connection->c_remote_uuid.uuid);
-			ptlrpc_connect_import(imp);
-			imp->imp_connect_tried = 1;
-			return 0;
-		}
-	} else {
-		static bool warned;
-
-		spin_lock(&imp->imp_lock);
-		list_del(&imp->imp_conn_current->oic_item);
-		list_add(&imp->imp_conn_current->oic_item, &imp->imp_conn_list);
-		imp->imp_last_success_conn =
-			imp->imp_conn_current->oic_last_attempt;
-
-		spin_unlock(&imp->imp_lock);
-
-		if ((imp->imp_connect_flags_orig & OBD_CONNECT_IBITS) &&
-		    !(ocd->ocd_connect_flags & OBD_CONNECT_IBITS)) {
-			LCONSOLE_WARN("%s: MDS %s does not support ibits lock, either very old or invalid: requested %llx, replied %llx\n",
-				      imp->imp_obd->obd_name,
-				      imp->imp_connection->c_remote_uuid.uuid,
-				      imp->imp_connect_flags_orig,
-				      ocd->ocd_connect_flags);
-			rc = -EPROTO;
-			goto out;
-		}
-
-		if (!warned && (ocd->ocd_connect_flags & OBD_CONNECT_VERSION) &&
-		    (ocd->ocd_version > LUSTRE_VERSION_CODE +
-					LUSTRE_VERSION_OFFSET_WARN ||
-		     ocd->ocd_version < LUSTRE_VERSION_CODE -
-					LUSTRE_VERSION_OFFSET_WARN)) {
-			/* Sigh, some compilers do not like #ifdef in the middle
-			 * of macro arguments
-			 */
-			const char *older = "older than client. Consider upgrading server";
-			const char *newer = "newer than client. Consider recompiling application";
-
-			LCONSOLE_WARN("Server %s version (%d.%d.%d.%d) is much %s (%s)\n",
-				      obd2cli_tgt(imp->imp_obd),
-				      OBD_OCD_VERSION_MAJOR(ocd->ocd_version),
-				      OBD_OCD_VERSION_MINOR(ocd->ocd_version),
-				      OBD_OCD_VERSION_PATCH(ocd->ocd_version),
-				      OBD_OCD_VERSION_FIX(ocd->ocd_version),
-				      ocd->ocd_version > LUSTRE_VERSION_CODE ?
-				      newer : older, LUSTRE_VERSION_STRING);
-			warned = true;
-		}
-
-#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 2, 50, 0)
-		/* Check if server has LU-1252 fix applied to not always swab
-		 * the IR MNE entries. Do this only once per connection.  This
-		 * fixup is version-limited, because we don't want to carry the
-		 * OBD_CONNECT_MNE_SWAB flag around forever, just so long as we
-		 * need interop with unpatched 2.2 servers.  For newer servers,
-		 * the client will do MNE swabbing only as needed.  LU-1644
-		 */
-		if (unlikely((ocd->ocd_connect_flags & OBD_CONNECT_VERSION) &&
-			     !(ocd->ocd_connect_flags & OBD_CONNECT_MNE_SWAB) &&
-			     OBD_OCD_VERSION_MAJOR(ocd->ocd_version) == 2 &&
-			     OBD_OCD_VERSION_MINOR(ocd->ocd_version) == 2 &&
-			     OBD_OCD_VERSION_PATCH(ocd->ocd_version) < 55 &&
-			     strcmp(imp->imp_obd->obd_type->typ_name,
-				    LUSTRE_MGC_NAME) == 0))
-			imp->imp_need_mne_swab = 1;
-		else /* clear if server was upgraded since last connect */
-			imp->imp_need_mne_swab = 0;
-#else
-#warning "LU-1644: Remove old OBD_CONNECT_MNE_SWAB fixup and imp_need_mne_swab"
-#endif
-
-		if (ocd->ocd_connect_flags & OBD_CONNECT_CKSUM) {
-			/* We sent to the server ocd_cksum_types with bits set
-			 * for algorithms we understand. The server masked off
-			 * the checksum types it doesn't support
-			 */
-			if ((ocd->ocd_cksum_types &
-			     cksum_types_supported_client()) == 0) {
-				LCONSOLE_WARN("The negotiation of the checksum algorithm to use with server %s failed (%x/%x), disabling checksums\n",
-					      obd2cli_tgt(imp->imp_obd),
-					      ocd->ocd_cksum_types,
-					      cksum_types_supported_client());
-				cli->cl_checksum = 0;
-				cli->cl_supp_cksum_types = OBD_CKSUM_ADLER;
-			} else {
-				cli->cl_supp_cksum_types = ocd->ocd_cksum_types;
-			}
-		} else {
-			/* The server does not support OBD_CONNECT_CKSUM.
-			 * Enforce ADLER for backward compatibility
-			 */
-			cli->cl_supp_cksum_types = OBD_CKSUM_ADLER;
-		}
-		cli->cl_cksum_type = cksum_type_select(cli->cl_supp_cksum_types);
-
-		if (ocd->ocd_connect_flags & OBD_CONNECT_BRW_SIZE)
-			cli->cl_max_pages_per_rpc =
-				min(ocd->ocd_brw_size >> PAGE_SHIFT,
-				    cli->cl_max_pages_per_rpc);
-		else if (imp->imp_connect_op == MDS_CONNECT ||
-			 imp->imp_connect_op == MGS_CONNECT)
-			cli->cl_max_pages_per_rpc = 1;
-
-		/* Reset ns_connect_flags only for initial connect. It might be
-		 * changed in while using FS and if we reset it in reconnect
-		 * this leads to losing user settings done before such as
-		 * disable lru_resize, etc.
-		 */
-		if (old_connect_flags != exp_connect_flags(exp) ||
-		    aa->pcaa_initial_connect) {
-			CDEBUG(D_HA, "%s: Resetting ns_connect_flags to server flags: %#llx\n",
-			       imp->imp_obd->obd_name, ocd->ocd_connect_flags);
-			imp->imp_obd->obd_namespace->ns_connect_flags =
-				ocd->ocd_connect_flags;
-			imp->imp_obd->obd_namespace->ns_orig_connect_flags =
-				ocd->ocd_connect_flags;
-		}
-
-		if ((ocd->ocd_connect_flags & OBD_CONNECT_AT) &&
-		    (imp->imp_msg_magic == LUSTRE_MSG_MAGIC_V2))
-			/* We need a per-message support flag, because
-			 * a. we don't know if the incoming connect reply
-			 *    supports AT or not (in reply_in_callback)
-			 *    until we unpack it.
-			 * b. failovered server means export and flags are gone
-			 *    (in ptlrpc_send_reply).
-			 * Can only be set when we know AT is supported at
-			 * both ends
-			 */
-			imp->imp_msghdr_flags |= MSGHDR_AT_SUPPORT;
-		else
-			imp->imp_msghdr_flags &= ~MSGHDR_AT_SUPPORT;
-
-		if ((ocd->ocd_connect_flags & OBD_CONNECT_FULL20) &&
-		    (imp->imp_msg_magic == LUSTRE_MSG_MAGIC_V2))
-			imp->imp_msghdr_flags |= MSGHDR_CKSUM_INCOMPAT18;
-		else
-			imp->imp_msghdr_flags &= ~MSGHDR_CKSUM_INCOMPAT18;
-
-		LASSERT((cli->cl_max_pages_per_rpc <= PTLRPC_MAX_BRW_PAGES) &&
-			(cli->cl_max_pages_per_rpc > 0));
+	if (rc == -ENOTCONN) {
+		CDEBUG(D_HA, "evicted/aborted by %s@%s during recovery; invalidating and reconnecting\n",
+		       obd2cli_tgt(imp->imp_obd),
+		       imp->imp_connection->c_remote_uuid.uuid);
+		ptlrpc_connect_import(imp);
+		imp->imp_connect_tried = 1;
+		return 0;
 	}
 
 out:
@@ -1497,10 +1520,13 @@ EXPORT_SYMBOL(ptlrpc_disconnect_import);
 /* Adaptive Timeout utils */
 extern unsigned int at_min, at_max, at_history;
 
-/* Bin into timeslices using AT_BINS bins.
- * This gives us a max of the last binlimit*AT_BINS secs without the storage,
- * but still smoothing out a return to normalcy from a slow response.
- * (E.g. remember the maximum latency in each minute of the last 4 minutes.)
+/*
+ *Update at_current with the specified value (bounded by at_min and at_max),
+ * as well as the AT history "bins".
+ *  - Bin into timeslices using AT_BINS bins.
+ *  - This gives us a max of the last at_history seconds without the storage,
+ *    but still smoothing out a return to normalcy from a slow response.
+ *  - (E.g. remember the maximum latency in each minute of the last 4 minutes.)
  */
 int at_measured(struct adaptive_timeout *at, unsigned int val)
 {
diff --git a/drivers/staging/lustre/lustre/ptlrpc/layout.c b/drivers/staging/lustre/lustre/ptlrpc/layout.c
index ab5d85174245..839ef3e80c1a 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/layout.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/layout.c
@@ -667,11 +667,8 @@ static struct req_format *req_formats[] = {
 	&RQF_MDS_SYNC,
 	&RQF_MDS_CLOSE,
 	&RQF_MDS_RELEASE_CLOSE,
-	&RQF_MDS_PIN,
-	&RQF_MDS_UNPIN,
 	&RQF_MDS_READPAGE,
 	&RQF_MDS_WRITEPAGE,
-	&RQF_MDS_IS_SUBDIR,
 	&RQF_MDS_DONE_WRITING,
 	&RQF_MDS_REINT,
 	&RQF_MDS_REINT_CREATE,
@@ -1116,9 +1113,9 @@ EXPORT_SYMBOL(RMF_SWAP_LAYOUTS);
 
 struct req_format {
 	const char *rf_name;
-	int rf_idx;
+	size_t rf_idx;
 	struct {
-		int nr;
+		size_t nr;
 		const struct req_msg_field **d;
 	} rf_fields[RCL_NR];
 };
@@ -1389,15 +1386,6 @@ struct req_format RQF_MDS_RELEASE_CLOSE =
 			mdt_release_close_client, mds_last_unlink_server);
 EXPORT_SYMBOL(RQF_MDS_RELEASE_CLOSE);
 
-struct req_format RQF_MDS_PIN =
-	DEFINE_REQ_FMT0("MDS_PIN",
-			mdt_body_capa, mdt_body_only);
-EXPORT_SYMBOL(RQF_MDS_PIN);
-
-struct req_format RQF_MDS_UNPIN =
-	DEFINE_REQ_FMT0("MDS_UNPIN", mdt_body_only, empty);
-EXPORT_SYMBOL(RQF_MDS_UNPIN);
-
 struct req_format RQF_MDS_DONE_WRITING =
 	DEFINE_REQ_FMT0("MDS_DONE_WRITING",
 			mdt_close_client, mdt_body_only);
@@ -1448,11 +1436,6 @@ struct req_format RQF_MDS_WRITEPAGE =
 			mdt_body_capa, mdt_body_only);
 EXPORT_SYMBOL(RQF_MDS_WRITEPAGE);
 
-struct req_format RQF_MDS_IS_SUBDIR =
-	DEFINE_REQ_FMT0("MDS_IS_SUBDIR",
-			mdt_body_only, mdt_body_only);
-EXPORT_SYMBOL(RQF_MDS_IS_SUBDIR);
-
 struct req_format RQF_LLOG_ORIGIN_HANDLE_CREATE =
 	DEFINE_REQ_FMT0("LLOG_ORIGIN_HANDLE_CREATE",
 			llog_origin_handle_create_client, llogd_body_only);
@@ -1572,9 +1555,9 @@ EXPORT_SYMBOL(RQF_OST_GET_INFO_FIEMAP);
  */
 int req_layout_init(void)
 {
-	int i;
-	int j;
-	int k;
+	size_t i;
+	size_t j;
+	size_t k;
 	struct req_format *rf = NULL;
 
 	for (i = 0; i < ARRAY_SIZE(req_formats); ++i) {
@@ -1616,7 +1599,7 @@ EXPORT_SYMBOL(req_layout_fini);
  */
 static void req_capsule_init_area(struct req_capsule *pill)
 {
-	int i;
+	size_t i;
 
 	for (i = 0; i < ARRAY_SIZE(pill->rc_area[RCL_CLIENT]); i++) {
 		pill->rc_area[RCL_CLIENT][i] = -1;
@@ -1667,8 +1650,7 @@ EXPORT_SYMBOL(req_capsule_fini);
 
 static int __req_format_is_sane(const struct req_format *fmt)
 {
-	return
-		0 <= fmt->rf_idx && fmt->rf_idx < ARRAY_SIZE(req_formats) &&
+	return fmt->rf_idx < ARRAY_SIZE(req_formats) &&
 		req_formats[fmt->rf_idx] == fmt;
 }
 
@@ -1702,11 +1684,11 @@ EXPORT_SYMBOL(req_capsule_set);
  * variable-sized fields.  The field sizes come from the declared \a rmf_size
  * field of a \a pill's \a rc_fmt's RMF's.
  */
-int req_capsule_filled_sizes(struct req_capsule *pill,
-			     enum req_location loc)
+size_t req_capsule_filled_sizes(struct req_capsule *pill,
+				enum req_location loc)
 {
 	const struct req_format *fmt = pill->rc_fmt;
-	int i;
+	size_t i;
 
 	for (i = 0; i < fmt->rf_fields[loc].nr; ++i) {
 		if (pill->rc_area[loc][i] == -1) {
@@ -1761,11 +1743,11 @@ EXPORT_SYMBOL(req_capsule_server_pack);
  * Returns the PTLRPC request or reply (\a loc) buffer offset of a \a pill
  * corresponding to the given RMF (\a field).
  */
-static int __req_capsule_offset(const struct req_capsule *pill,
+static u32 __req_capsule_offset(const struct req_capsule *pill,
 				const struct req_msg_field *field,
 				enum req_location loc)
 {
-	int offset;
+	u32 offset;
 
 	offset = field->rmf_offset[pill->rc_fmt->rf_idx][loc];
 	LASSERTF(offset > 0, "%s:%s, off=%d, loc=%d\n", pill->rc_fmt->rf_name,
@@ -1869,10 +1851,10 @@ static void *__req_capsule_get(struct req_capsule *pill,
 	const struct req_format *fmt;
 	struct lustre_msg *msg;
 	void *value;
-	int len;
-	int offset;
+	u32 len;
+	u32 offset;
 
-	void *(*getter)(struct lustre_msg *m, int n, int minlen);
+	void *(*getter)(struct lustre_msg *m, u32 n, u32 minlen);
 
 	static const char *rcl_names[RCL_NR] = {
 		[RCL_CLIENT] = "client",
@@ -1899,20 +1881,20 @@ static void *__req_capsule_get(struct req_capsule *pill,
 		 */
 		len = lustre_msg_buflen(msg, offset);
 		if ((len % field->rmf_size) != 0) {
-			CERROR("%s: array field size mismatch %d modulo %d != 0 (%d)\n",
+			CERROR("%s: array field size mismatch %d modulo %u != 0 (%d)\n",
 			       field->rmf_name, len, field->rmf_size, loc);
 			return NULL;
 		}
 	} else if (pill->rc_area[loc][offset] != -1) {
 		len = pill->rc_area[loc][offset];
 	} else {
-		len = max(field->rmf_size, 0);
+		len = max_t(typeof(field->rmf_size), field->rmf_size, 0);
 	}
 	value = getter(msg, offset, len);
 
 	if (!value) {
 		DEBUG_REQ(D_ERROR, pill->rc_req,
-			  "Wrong buffer for field `%s' (%d of %d) in format `%s': %d vs. %d (%s)\n",
+			  "Wrong buffer for field `%s' (%u of %u) in format `%s': %u vs. %u (%s)\n",
 			  field->rmf_name, offset, lustre_msg_bufcount(msg),
 			  fmt->rf_name, lustre_msg_buflen(msg, offset), len,
 			  rcl_names[loc]);
@@ -1958,7 +1940,7 @@ EXPORT_SYMBOL(req_capsule_client_swab_get);
  */
 void *req_capsule_client_sized_get(struct req_capsule *pill,
 				   const struct req_msg_field *field,
-				   int len)
+				   u32 len)
 {
 	req_capsule_set_size(pill, field, RCL_CLIENT, len);
 	return __req_capsule_get(pill, field, RCL_CLIENT, NULL, 0);
@@ -1999,7 +1981,7 @@ EXPORT_SYMBOL(req_capsule_server_swab_get);
  */
 void *req_capsule_server_sized_get(struct req_capsule *pill,
 				   const struct req_msg_field *field,
-				   int len)
+				   u32 len)
 {
 	req_capsule_set_size(pill, field, RCL_SERVER, len);
 	return __req_capsule_get(pill, field, RCL_SERVER, NULL, 0);
@@ -2008,7 +1990,7 @@ EXPORT_SYMBOL(req_capsule_server_sized_get);
 
 void *req_capsule_server_sized_swab_get(struct req_capsule *pill,
 					const struct req_msg_field *field,
-					int len, void *swabber)
+					u32 len, void *swabber)
 {
 	req_capsule_set_size(pill, field, RCL_SERVER, len);
 	return __req_capsule_get(pill, field, RCL_SERVER, swabber, 0);
@@ -2024,23 +2006,25 @@ EXPORT_SYMBOL(req_capsule_server_sized_swab_get);
  */
 void req_capsule_set_size(struct req_capsule *pill,
 			  const struct req_msg_field *field,
-			  enum req_location loc, int size)
+			  enum req_location loc, u32 size)
 {
 	LASSERT(loc == RCL_SERVER || loc == RCL_CLIENT);
 
-	if ((size != field->rmf_size) &&
+	if ((size != (u32)field->rmf_size) &&
 	    (field->rmf_size != -1) &&
 	    !(field->rmf_flags & RMF_F_NO_SIZE_CHECK) &&
 	    (size > 0)) {
+		u32 rmf_size = (u32)field->rmf_size;
+
 		if ((field->rmf_flags & RMF_F_STRUCT_ARRAY) &&
-		    (size % field->rmf_size != 0)) {
-			CERROR("%s: array field size mismatch %d %% %d != 0 (%d)\n",
-			       field->rmf_name, size, field->rmf_size, loc);
+		    (size % rmf_size != 0)) {
+			CERROR("%s: array field size mismatch %u %% %u != 0 (%d)\n",
+			       field->rmf_name, size, rmf_size, loc);
 			LBUG();
 		} else if (!(field->rmf_flags & RMF_F_STRUCT_ARRAY) &&
-		    size < field->rmf_size) {
-			CERROR("%s: field size mismatch %d != %d (%d)\n",
-			       field->rmf_name, size, field->rmf_size, loc);
+			   size < rmf_size) {
+			CERROR("%s: field size mismatch %u != %u (%d)\n",
+			       field->rmf_name, size, rmf_size, loc);
 			LBUG();
 		}
 	}
@@ -2057,7 +2041,7 @@ EXPORT_SYMBOL(req_capsule_set_size);
  * actually sets the size in pill.rc_area[loc][offset], but this function
  * returns the message buflen[offset], maybe we should use another name.
  */
-int req_capsule_get_size(const struct req_capsule *pill,
+u32 req_capsule_get_size(const struct req_capsule *pill,
 			 const struct req_msg_field *field,
 			 enum req_location loc)
 {
@@ -2075,7 +2059,7 @@ EXPORT_SYMBOL(req_capsule_get_size);
  *
  * See also req_capsule_set_size().
  */
-int req_capsule_msg_size(struct req_capsule *pill, enum req_location loc)
+u32 req_capsule_msg_size(struct req_capsule *pill, enum req_location loc)
 {
 	return lustre_msg_size(pill->rc_req->rq_import->imp_msg_magic,
 			       pill->rc_fmt->rf_fields[loc].nr,
@@ -2090,10 +2074,11 @@ int req_capsule_msg_size(struct req_capsule *pill, enum req_location loc)
  * This function should not be used for formats which contain variable size
  * fields.
  */
-int req_capsule_fmt_size(__u32 magic, const struct req_format *fmt,
+u32 req_capsule_fmt_size(__u32 magic, const struct req_format *fmt,
 			 enum req_location loc)
 {
-	int size, i = 0;
+	size_t i = 0;
+	u32 size;
 
 	/*
 	 * This function should probably LASSERT() that fmt has no fields with
@@ -2103,7 +2088,7 @@ int req_capsule_fmt_size(__u32 magic, const struct req_format *fmt,
 	 * we do.
 	 */
 	size = lustre_msg_hdr_size(magic, fmt->rf_fields[loc].nr);
-	if (size < 0)
+	if (!size)
 		return size;
 
 	for (; i < fmt->rf_fields[loc].nr; ++i)
@@ -2135,7 +2120,7 @@ int req_capsule_fmt_size(__u32 magic, const struct req_format *fmt,
 void req_capsule_extend(struct req_capsule *pill, const struct req_format *fmt)
 {
 	int i;
-	int j;
+	size_t j;
 
 	const struct req_format *old;
 
@@ -2193,7 +2178,7 @@ static int req_capsule_field_present(const struct req_capsule *pill,
 				     const struct req_msg_field *field,
 				     enum req_location loc)
 {
-	int offset;
+	u32 offset;
 
 	LASSERT(loc == RCL_SERVER || loc == RCL_CLIENT);
 	LASSERT(req_capsule_has_field(pill, field, loc));
@@ -2210,12 +2195,11 @@ static int req_capsule_field_present(const struct req_capsule *pill,
  */
 void req_capsule_shrink(struct req_capsule *pill,
 			const struct req_msg_field *field,
-			unsigned int newlen,
-			enum req_location loc)
+			u32 newlen, enum req_location loc)
 {
 	const struct req_format *fmt;
 	struct lustre_msg *msg;
-	int len;
+	u32 len;
 	int offset;
 
 	fmt = pill->rc_fmt;
@@ -2228,7 +2212,7 @@ void req_capsule_shrink(struct req_capsule *pill,
 
 	msg = __req_msg(pill, loc);
 	len = lustre_msg_buflen(msg, offset);
-	LASSERTF(newlen <= len, "%s:%s, oldlen=%d, newlen=%d\n",
+	LASSERTF(newlen <= len, "%s:%s, oldlen=%u, newlen=%u\n",
 		 fmt->rf_name, field->rmf_name, len, newlen);
 
 	if (loc == RCL_CLIENT)
diff --git a/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c b/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c
index bc93b75744e1..9bad57d65db4 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c
@@ -191,7 +191,7 @@ ptlrpc_ldebugfs_register(struct dentry *root, char *dir,
 	LASSERT(!*debugfs_root_ret);
 	LASSERT(!*stats_ret);
 
-	svc_stats = lprocfs_alloc_stats(EXTRA_MAX_OPCODES+LUSTRE_MAX_OPCODES,
+	svc_stats = lprocfs_alloc_stats(EXTRA_MAX_OPCODES + LUSTRE_MAX_OPCODES,
 					0);
 	if (!svc_stats)
 		return;
@@ -937,7 +937,7 @@ static int ptlrpc_lprocfs_svc_req_history_show(struct seq_file *s, void *iter)
 static int
 ptlrpc_lprocfs_svc_req_history_open(struct inode *inode, struct file *file)
 {
-	static struct seq_operations sops = {
+	static const struct seq_operations sops = {
 		.start = ptlrpc_lprocfs_svc_req_history_start,
 		.stop  = ptlrpc_lprocfs_svc_req_history_stop,
 		.next  = ptlrpc_lprocfs_svc_req_history_next,
diff --git a/drivers/staging/lustre/lustre/ptlrpc/niobuf.c b/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
index 11ec82545347..9c937398a085 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
@@ -295,7 +295,6 @@ int ptlrpc_unregister_bulk(struct ptlrpc_request *req, int async)
 	}
 	return 0;
 }
-EXPORT_SYMBOL(ptlrpc_unregister_bulk);
 
 static void ptlrpc_at_set_reply(struct ptlrpc_request *req, int flags)
 {
@@ -398,7 +397,8 @@ int ptlrpc_send_reply(struct ptlrpc_request *req, int flags)
 	lustre_msg_set_status(req->rq_repmsg,
 			      ptlrpc_status_hton(req->rq_status));
 	lustre_msg_set_opc(req->rq_repmsg,
-		req->rq_reqmsg ? lustre_msg_get_opc(req->rq_reqmsg) : 0);
+			   req->rq_reqmsg ?
+			   lustre_msg_get_opc(req->rq_reqmsg) : 0);
 
 	target_pack_pool_reply(req);
 
@@ -433,7 +433,6 @@ out:
 	ptlrpc_connection_put(conn);
 	return rc;
 }
-EXPORT_SYMBOL(ptlrpc_send_reply);
 
 int ptlrpc_reply(struct ptlrpc_request *req)
 {
@@ -441,7 +440,6 @@ int ptlrpc_reply(struct ptlrpc_request *req)
 		return 0;
 	return ptlrpc_send_reply(req, 0);
 }
-EXPORT_SYMBOL(ptlrpc_reply);
 
 /**
  * For request \a req send an error reply back. Create empty
@@ -468,13 +466,11 @@ int ptlrpc_send_error(struct ptlrpc_request *req, int may_be_difficult)
 	rc = ptlrpc_send_reply(req, may_be_difficult);
 	return rc;
 }
-EXPORT_SYMBOL(ptlrpc_send_error);
 
 int ptlrpc_error(struct ptlrpc_request *req)
 {
 	return ptlrpc_send_error(req, 0);
 }
-EXPORT_SYMBOL(ptlrpc_error);
 
 /**
  * Send request \a request.
@@ -490,7 +486,8 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 	struct ptlrpc_connection *connection;
 	lnet_handle_me_t reply_me_h;
 	lnet_md_t reply_md;
-	struct obd_device *obd = request->rq_import->imp_obd;
+	struct obd_import *imp = request->rq_import;
+	struct obd_device *obd = imp->imp_obd;
 
 	if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_DROP_RPC))
 		return 0;
@@ -503,7 +500,7 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 	 */
 	LASSERT(!request->rq_receiving_reply);
 	LASSERT(!((lustre_msg_get_flags(request->rq_reqmsg) & MSG_REPLAY) &&
-		  (request->rq_import->imp_state == LUSTRE_IMP_FULL)));
+		  (imp->imp_state == LUSTRE_IMP_FULL)));
 
 	if (unlikely(obd && obd->obd_fail)) {
 		CDEBUG(D_HA, "muting rpc for failed imp obd %s\n",
@@ -516,15 +513,22 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 		return -ENODEV;
 	}
 
-	connection = request->rq_import->imp_connection;
+	connection = imp->imp_connection;
 
 	lustre_msg_set_handle(request->rq_reqmsg,
-			      &request->rq_import->imp_remote_handle);
+			      &imp->imp_remote_handle);
 	lustre_msg_set_type(request->rq_reqmsg, PTL_RPC_MSG_REQUEST);
-	lustre_msg_set_conn_cnt(request->rq_reqmsg,
-				request->rq_import->imp_conn_cnt);
-	lustre_msghdr_set_flags(request->rq_reqmsg,
-				request->rq_import->imp_msghdr_flags);
+	lustre_msg_set_conn_cnt(request->rq_reqmsg, imp->imp_conn_cnt);
+	lustre_msghdr_set_flags(request->rq_reqmsg, imp->imp_msghdr_flags);
+
+	/**
+	 * For enabled AT all request should have AT_SUPPORT in the
+	 * FULL import state when OBD_CONNECT_AT is set
+	 */
+	LASSERT(AT_OFF || imp->imp_state != LUSTRE_IMP_FULL ||
+		(imp->imp_msghdr_flags & MSGHDR_AT_SUPPORT) ||
+		!(imp->imp_connect_data.ocd_connect_flags &
+		OBD_CONNECT_AT));
 
 	if (request->rq_resend)
 		lustre_msg_add_flags(request->rq_reqmsg, MSG_RESENT);
@@ -628,7 +632,7 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 	ptlrpc_request_addref(request);
 	if (obd && obd->obd_svc_stats)
 		lprocfs_counter_add(obd->obd_svc_stats, PTLRPC_REQACTIVE_CNTR,
-			atomic_read(&request->rq_import->imp_inflight));
+			atomic_read(&imp->imp_inflight));
 
 	OBD_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_DELAY_SEND, request->rq_timeout + 5);
 
@@ -640,7 +644,7 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 	request->rq_deadline = request->rq_sent + request->rq_timeout +
 		ptlrpc_at_get_net_latency(request);
 
-	ptlrpc_pinger_sending_on_import(request->rq_import);
+	ptlrpc_pinger_sending_on_import(imp);
 
 	DEBUG_REQ(D_INFO, request, "send flg=%x",
 		  lustre_msg_get_flags(request->rq_reqmsg));
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c b/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
index b514f18fae50..871768511e8c 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
@@ -50,36 +50,34 @@
 
 #include "ptlrpc_internal.h"
 
-static inline int lustre_msg_hdr_size_v2(int count)
+static inline u32 lustre_msg_hdr_size_v2(u32 count)
 {
 	return cfs_size_round(offsetof(struct lustre_msg_v2,
 				       lm_buflens[count]));
 }
 
-int lustre_msg_hdr_size(__u32 magic, int count)
+u32 lustre_msg_hdr_size(__u32 magic, u32 count)
 {
 	switch (magic) {
 	case LUSTRE_MSG_MAGIC_V2:
 		return lustre_msg_hdr_size_v2(count);
 	default:
 		LASSERTF(0, "incorrect message magic: %08x\n", magic);
-		return -EINVAL;
+		return 0;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_hdr_size);
 
 void ptlrpc_buf_set_swabbed(struct ptlrpc_request *req, const int inout,
-			    int index)
+			    u32 index)
 {
 	if (inout)
 		lustre_set_req_swabbed(req, index);
 	else
 		lustre_set_rep_swabbed(req, index);
 }
-EXPORT_SYMBOL(ptlrpc_buf_set_swabbed);
 
 int ptlrpc_buf_need_swab(struct ptlrpc_request *req, const int inout,
-			 int index)
+			 u32 index)
 {
 	if (inout)
 		return (ptlrpc_req_need_swab(req) &&
@@ -88,12 +86,11 @@ int ptlrpc_buf_need_swab(struct ptlrpc_request *req, const int inout,
 		return (ptlrpc_rep_need_swab(req) &&
 			!lustre_rep_swabbed(req, index));
 }
-EXPORT_SYMBOL(ptlrpc_buf_need_swab);
 
 /* early reply size */
-int lustre_msg_early_size(void)
+u32 lustre_msg_early_size(void)
 {
-	static int size;
+	static u32 size;
 
 	if (!size) {
 		/* Always reply old ptlrpc_body_v2 to keep interoperability
@@ -111,9 +108,9 @@ int lustre_msg_early_size(void)
 }
 EXPORT_SYMBOL(lustre_msg_early_size);
 
-int lustre_msg_size_v2(int count, __u32 *lengths)
+u32 lustre_msg_size_v2(int count, __u32 *lengths)
 {
-	int size;
+	u32 size;
 	int i;
 
 	size = lustre_msg_hdr_size_v2(count);
@@ -131,7 +128,7 @@ EXPORT_SYMBOL(lustre_msg_size_v2);
  *       target then the first buffer will be stripped because the ptlrpc
  *       data is part of the lustre_msg_v1 header. b=14043
  */
-int lustre_msg_size(__u32 magic, int count, __u32 *lens)
+u32 lustre_msg_size(__u32 magic, int count, __u32 *lens)
 {
 	__u32 size[] = { sizeof(struct ptlrpc_body) };
 
@@ -148,15 +145,14 @@ int lustre_msg_size(__u32 magic, int count, __u32 *lens)
 		return lustre_msg_size_v2(count, lens);
 	default:
 		LASSERTF(0, "incorrect message magic: %08x\n", magic);
-		return -EINVAL;
+		return 0;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_size);
 
 /* This is used to determine the size of a buffer that was already packed
  * and will correctly handle the different message formats.
  */
-int lustre_packed_msg_size(struct lustre_msg *msg)
+u32 lustre_packed_msg_size(struct lustre_msg *msg)
 {
 	switch (msg->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2:
@@ -166,7 +162,6 @@ int lustre_packed_msg_size(struct lustre_msg *msg)
 		return 0;
 	}
 }
-EXPORT_SYMBOL(lustre_packed_msg_size);
 
 void lustre_init_msg_v2(struct lustre_msg_v2 *msg, int count, __u32 *lens,
 			char **bufs)
@@ -227,7 +222,6 @@ int lustre_pack_request(struct ptlrpc_request *req, __u32 magic, int count,
 	/* only use new format, we don't need to be compatible with 1.4 */
 	return lustre_pack_request_v2(req, count, lens, bufs);
 }
-EXPORT_SYMBOL(lustre_pack_request);
 
 #if RS_DEBUG
 LIST_HEAD(ptlrpc_rs_debug_lru);
@@ -369,7 +363,6 @@ int lustre_pack_reply_flags(struct ptlrpc_request *req, int count, __u32 *lens,
 		       lustre_msg_size(req->rq_reqmsg->lm_magic, count, lens));
 	return rc;
 }
-EXPORT_SYMBOL(lustre_pack_reply_flags);
 
 int lustre_pack_reply(struct ptlrpc_request *req, int count, __u32 *lens,
 		      char **bufs)
@@ -378,11 +371,9 @@ int lustre_pack_reply(struct ptlrpc_request *req, int count, __u32 *lens,
 }
 EXPORT_SYMBOL(lustre_pack_reply);
 
-void *lustre_msg_buf_v2(struct lustre_msg_v2 *m, int n, int min_size)
+void *lustre_msg_buf_v2(struct lustre_msg_v2 *m, u32 n, u32 min_size)
 {
-	int i, offset, buflen, bufcount;
-
-	LASSERT(n >= 0);
+	u32 i, offset, buflen, bufcount;
 
 	bufcount = m->lm_bufcount;
 	if (unlikely(n >= bufcount)) {
@@ -406,7 +397,7 @@ void *lustre_msg_buf_v2(struct lustre_msg_v2 *m, int n, int min_size)
 	return (char *)m + offset;
 }
 
-void *lustre_msg_buf(struct lustre_msg *m, int n, int min_size)
+void *lustre_msg_buf(struct lustre_msg *m, u32 n, u32 min_size)
 {
 	switch (m->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2:
@@ -419,7 +410,7 @@ void *lustre_msg_buf(struct lustre_msg *m, int n, int min_size)
 }
 EXPORT_SYMBOL(lustre_msg_buf);
 
-static int lustre_shrink_msg_v2(struct lustre_msg_v2 *msg, int segment,
+static int lustre_shrink_msg_v2(struct lustre_msg_v2 *msg, u32 segment,
 				unsigned int newlen, int move_data)
 {
 	char *tail = NULL, *newpos;
@@ -493,7 +484,6 @@ void lustre_free_reply_state(struct ptlrpc_reply_state *rs)
 
 	sptlrpc_svc_free_rs(rs);
 }
-EXPORT_SYMBOL(lustre_free_reply_state);
 
 static int lustre_unpack_msg_v2(struct lustre_msg_v2 *m, int len)
 {
@@ -581,7 +571,6 @@ int ptlrpc_unpack_req_msg(struct ptlrpc_request *req, int len)
 	}
 	return rc;
 }
-EXPORT_SYMBOL(ptlrpc_unpack_req_msg);
 
 int ptlrpc_unpack_rep_msg(struct ptlrpc_request *req, int len)
 {
@@ -594,7 +583,6 @@ int ptlrpc_unpack_rep_msg(struct ptlrpc_request *req, int len)
 	}
 	return rc;
 }
-EXPORT_SYMBOL(ptlrpc_unpack_rep_msg);
 
 static inline int lustre_unpack_ptlrpc_body_v2(struct ptlrpc_request *req,
 					       const int inout, int offset)
@@ -647,7 +635,7 @@ int lustre_unpack_rep_ptlrpc_body(struct ptlrpc_request *req, int offset)
 	}
 }
 
-static inline int lustre_msg_buflen_v2(struct lustre_msg_v2 *m, int n)
+static inline u32 lustre_msg_buflen_v2(struct lustre_msg_v2 *m, u32 n)
 {
 	if (n >= m->lm_bufcount)
 		return 0;
@@ -662,14 +650,14 @@ static inline int lustre_msg_buflen_v2(struct lustre_msg_v2 *m, int n)
  *
  * returns zero for non-existent message indices
  */
-int lustre_msg_buflen(struct lustre_msg *m, int n)
+u32 lustre_msg_buflen(struct lustre_msg *m, u32 n)
 {
 	switch (m->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2:
 		return lustre_msg_buflen_v2(m, n);
 	default:
 		CERROR("incorrect message magic: %08x\n", m->lm_magic);
-		return -EINVAL;
+		return 0;
 	}
 }
 EXPORT_SYMBOL(lustre_msg_buflen);
@@ -677,23 +665,22 @@ EXPORT_SYMBOL(lustre_msg_buflen);
 /* NB return the bufcount for lustre_msg_v2 format, so if message is packed
  * in V1 format, the result is one bigger. (add struct ptlrpc_body).
  */
-int lustre_msg_bufcount(struct lustre_msg *m)
+u32 lustre_msg_bufcount(struct lustre_msg *m)
 {
 	switch (m->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2:
 		return m->lm_bufcount;
 	default:
 		CERROR("incorrect message magic: %08x\n", m->lm_magic);
-		return -EINVAL;
+		return 0;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_bufcount);
 
-char *lustre_msg_string(struct lustre_msg *m, int index, int max_len)
+char *lustre_msg_string(struct lustre_msg *m, u32 index, u32 max_len)
 {
 	/* max_len == 0 means the string should fill the buffer */
 	char *str;
-	int slen, blen;
+	u32 slen, blen;
 
 	switch (m->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2:
@@ -731,11 +718,10 @@ char *lustre_msg_string(struct lustre_msg *m, int index, int max_len)
 
 	return str;
 }
-EXPORT_SYMBOL(lustre_msg_string);
 
 /* Wrap up the normal fixed length cases */
-static inline void *__lustre_swab_buf(struct lustre_msg *msg, int index,
-				      int min_size, void *swabber)
+static inline void *__lustre_swab_buf(struct lustre_msg *msg, u32 index,
+				      u32 min_size, void *swabber)
 {
 	void *ptr = NULL;
 
@@ -804,7 +790,7 @@ __u32 lustre_msg_get_flags(struct lustre_msg *msg)
 }
 EXPORT_SYMBOL(lustre_msg_get_flags);
 
-void lustre_msg_add_flags(struct lustre_msg *msg, int flags)
+void lustre_msg_add_flags(struct lustre_msg *msg, u32 flags)
 {
 	switch (msg->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2: {
@@ -820,7 +806,7 @@ void lustre_msg_add_flags(struct lustre_msg *msg, int flags)
 }
 EXPORT_SYMBOL(lustre_msg_add_flags);
 
-void lustre_msg_set_flags(struct lustre_msg *msg, int flags)
+void lustre_msg_set_flags(struct lustre_msg *msg, u32 flags)
 {
 	switch (msg->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2: {
@@ -834,9 +820,8 @@ void lustre_msg_set_flags(struct lustre_msg *msg, int flags)
 		LASSERTF(0, "incorrect message magic: %08x\n", msg->lm_magic);
 	}
 }
-EXPORT_SYMBOL(lustre_msg_set_flags);
 
-void lustre_msg_clear_flags(struct lustre_msg *msg, int flags)
+void lustre_msg_clear_flags(struct lustre_msg *msg, u32 flags)
 {
 	switch (msg->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2: {
@@ -868,9 +853,8 @@ __u32 lustre_msg_get_op_flags(struct lustre_msg *msg)
 		return 0;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_get_op_flags);
 
-void lustre_msg_add_op_flags(struct lustre_msg *msg, int flags)
+void lustre_msg_add_op_flags(struct lustre_msg *msg, u32 flags)
 {
 	switch (msg->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2: {
@@ -903,7 +887,6 @@ struct lustre_handle *lustre_msg_get_handle(struct lustre_msg *msg)
 		return NULL;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_get_handle);
 
 __u32 lustre_msg_get_type(struct lustre_msg *msg)
 {
@@ -924,7 +907,7 @@ __u32 lustre_msg_get_type(struct lustre_msg *msg)
 }
 EXPORT_SYMBOL(lustre_msg_get_type);
 
-void lustre_msg_add_version(struct lustre_msg *msg, int version)
+void lustre_msg_add_version(struct lustre_msg *msg, u32 version)
 {
 	switch (msg->lm_magic) {
 	case LUSTRE_MSG_MAGIC_V2: {
@@ -938,7 +921,6 @@ void lustre_msg_add_version(struct lustre_msg *msg, int version)
 		LASSERTF(0, "incorrect message magic: %08x\n", msg->lm_magic);
 	}
 }
-EXPORT_SYMBOL(lustre_msg_add_version);
 
 __u32 lustre_msg_get_opc(struct lustre_msg *msg)
 {
@@ -1055,7 +1037,6 @@ __u64 lustre_msg_get_slv(struct lustre_msg *msg)
 		return -EINVAL;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_get_slv);
 
 void lustre_msg_set_slv(struct lustre_msg *msg, __u64 slv)
 {
@@ -1075,7 +1056,6 @@ void lustre_msg_set_slv(struct lustre_msg *msg, __u64 slv)
 		return;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_set_slv);
 
 __u32 lustre_msg_get_limit(struct lustre_msg *msg)
 {
@@ -1094,7 +1074,6 @@ __u32 lustre_msg_get_limit(struct lustre_msg *msg)
 		return -EINVAL;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_get_limit);
 
 void lustre_msg_set_limit(struct lustre_msg *msg, __u64 limit)
 {
@@ -1114,7 +1093,6 @@ void lustre_msg_set_limit(struct lustre_msg *msg, __u64 limit)
 		return;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_set_limit);
 
 __u32 lustre_msg_get_conn_cnt(struct lustre_msg *msg)
 {
@@ -1145,7 +1123,6 @@ __u32 lustre_msg_get_magic(struct lustre_msg *msg)
 		return 0;
 	}
 }
-EXPORT_SYMBOL(lustre_msg_get_magic);
 
 __u32 lustre_msg_get_timeout(struct lustre_msg *msg)
 {
@@ -1203,8 +1180,9 @@ __u32 lustre_msg_calc_cksum(struct lustre_msg *msg)
 		unsigned int hsize = 4;
 
 		cfs_crypto_hash_digest(CFS_HASH_ALG_CRC32, (unsigned char *)pb,
-				   lustre_msg_buflen(msg, MSG_PTLRPC_BODY_OFF),
-				   NULL, 0, (unsigned char *)&crc, &hsize);
+				       lustre_msg_buflen(msg,
+							 MSG_PTLRPC_BODY_OFF),
+				       NULL, 0, (unsigned char *)&crc, &hsize);
 		return crc;
 	}
 	default:
@@ -1227,7 +1205,6 @@ void lustre_msg_set_handle(struct lustre_msg *msg, struct lustre_handle *handle)
 		LASSERTF(0, "incorrect message magic: %08x\n", msg->lm_magic);
 	}
 }
-EXPORT_SYMBOL(lustre_msg_set_handle);
 
 void lustre_msg_set_type(struct lustre_msg *msg, __u32 type)
 {
@@ -1243,7 +1220,6 @@ void lustre_msg_set_type(struct lustre_msg *msg, __u32 type)
 		LASSERTF(0, "incorrect message magic: %08x\n", msg->lm_magic);
 	}
 }
-EXPORT_SYMBOL(lustre_msg_set_type);
 
 void lustre_msg_set_opc(struct lustre_msg *msg, __u32 opc)
 {
@@ -1259,7 +1235,6 @@ void lustre_msg_set_opc(struct lustre_msg *msg, __u32 opc)
 		LASSERTF(0, "incorrect message magic: %08x\n", msg->lm_magic);
 	}
 }
-EXPORT_SYMBOL(lustre_msg_set_opc);
 
 void lustre_msg_set_versions(struct lustre_msg *msg, __u64 *versions)
 {
@@ -1326,7 +1301,6 @@ void lustre_msg_set_conn_cnt(struct lustre_msg *msg, __u32 conn_cnt)
 		LASSERTF(0, "incorrect message magic: %08x\n", msg->lm_magic);
 	}
 }
-EXPORT_SYMBOL(lustre_msg_set_conn_cnt);
 
 void lustre_msg_set_timeout(struct lustre_msg *msg, __u32 timeout)
 {
@@ -1377,7 +1351,7 @@ void lustre_msg_set_jobid(struct lustre_msg *msg, char *jobid)
 		LASSERTF(pb, "invalid msg %p: no ptlrpc body!\n", msg);
 
 		if (jobid)
-			memcpy(pb->pb_jobid, jobid, JOBSTATS_JOBID_SIZE);
+			memcpy(pb->pb_jobid, jobid, LUSTRE_JOBID_SIZE);
 		else if (pb->pb_jobid[0] == '\0')
 			lustre_get_jobid(pb->pb_jobid);
 		return;
@@ -1491,7 +1465,6 @@ void lustre_swab_ptlrpc_body(struct ptlrpc_body *b)
 	 */
 	CLASSERT(offsetof(typeof(*b), pb_jobid) != 0);
 }
-EXPORT_SYMBOL(lustre_swab_ptlrpc_body);
 
 void lustre_swab_connect(struct obd_connect_data *ocd)
 {
@@ -1591,7 +1564,6 @@ void lustre_swab_obd_statfs(struct obd_statfs *os)
 	CLASSERT(offsetof(typeof(*os), os_spare8) != 0);
 	CLASSERT(offsetof(typeof(*os), os_spare9) != 0);
 }
-EXPORT_SYMBOL(lustre_swab_obd_statfs);
 
 void lustre_swab_obd_ioobj(struct obd_ioobj *ioo)
 {
@@ -1599,33 +1571,28 @@ void lustre_swab_obd_ioobj(struct obd_ioobj *ioo)
 	__swab32s(&ioo->ioo_max_brw);
 	__swab32s(&ioo->ioo_bufcnt);
 }
-EXPORT_SYMBOL(lustre_swab_obd_ioobj);
 
 void lustre_swab_niobuf_remote(struct niobuf_remote *nbr)
 {
-	__swab64s(&nbr->offset);
-	__swab32s(&nbr->len);
-	__swab32s(&nbr->flags);
+	__swab64s(&nbr->rnb_offset);
+	__swab32s(&nbr->rnb_len);
+	__swab32s(&nbr->rnb_flags);
 }
-EXPORT_SYMBOL(lustre_swab_niobuf_remote);
 
 void lustre_swab_ost_body(struct ost_body *b)
 {
 	lustre_swab_obdo(&b->oa);
 }
-EXPORT_SYMBOL(lustre_swab_ost_body);
 
 void lustre_swab_ost_last_id(u64 *id)
 {
 	__swab64s(id);
 }
-EXPORT_SYMBOL(lustre_swab_ost_last_id);
 
 void lustre_swab_generic_32s(__u32 *val)
 {
 	__swab32s(val);
 }
-EXPORT_SYMBOL(lustre_swab_generic_32s);
 
 void lustre_swab_gl_desc(union ldlm_gl_desc *desc)
 {
@@ -1674,37 +1641,36 @@ EXPORT_SYMBOL(lustre_swab_lquota_lvb);
 
 void lustre_swab_mdt_body(struct mdt_body *b)
 {
-	lustre_swab_lu_fid(&b->fid1);
-	lustre_swab_lu_fid(&b->fid2);
+	lustre_swab_lu_fid(&b->mbo_fid1);
+	lustre_swab_lu_fid(&b->mbo_fid2);
 	/* handle is opaque */
-	__swab64s(&b->valid);
-	__swab64s(&b->size);
-	__swab64s(&b->mtime);
-	__swab64s(&b->atime);
-	__swab64s(&b->ctime);
-	__swab64s(&b->blocks);
-	__swab64s(&b->ioepoch);
-	__swab64s(&b->t_state);
-	__swab32s(&b->fsuid);
-	__swab32s(&b->fsgid);
-	__swab32s(&b->capability);
-	__swab32s(&b->mode);
-	__swab32s(&b->uid);
-	__swab32s(&b->gid);
-	__swab32s(&b->flags);
-	__swab32s(&b->rdev);
-	__swab32s(&b->nlink);
-	CLASSERT(offsetof(typeof(*b), unused2) != 0);
-	__swab32s(&b->suppgid);
-	__swab32s(&b->eadatasize);
-	__swab32s(&b->aclsize);
-	__swab32s(&b->max_mdsize);
-	__swab32s(&b->max_cookiesize);
-	__swab32s(&b->uid_h);
-	__swab32s(&b->gid_h);
-	CLASSERT(offsetof(typeof(*b), padding_5) != 0);
-}
-EXPORT_SYMBOL(lustre_swab_mdt_body);
+	__swab64s(&b->mbo_valid);
+	__swab64s(&b->mbo_size);
+	__swab64s(&b->mbo_mtime);
+	__swab64s(&b->mbo_atime);
+	__swab64s(&b->mbo_ctime);
+	__swab64s(&b->mbo_blocks);
+	__swab64s(&b->mbo_ioepoch);
+	__swab64s(&b->mbo_t_state);
+	__swab32s(&b->mbo_fsuid);
+	__swab32s(&b->mbo_fsgid);
+	__swab32s(&b->mbo_capability);
+	__swab32s(&b->mbo_mode);
+	__swab32s(&b->mbo_uid);
+	__swab32s(&b->mbo_gid);
+	__swab32s(&b->mbo_flags);
+	__swab32s(&b->mbo_rdev);
+	__swab32s(&b->mbo_nlink);
+	CLASSERT(offsetof(typeof(*b), mbo_unused2) != 0);
+	__swab32s(&b->mbo_suppgid);
+	__swab32s(&b->mbo_eadatasize);
+	__swab32s(&b->mbo_aclsize);
+	__swab32s(&b->mbo_max_mdsize);
+	__swab32s(&b->mbo_max_cookiesize);
+	__swab32s(&b->mbo_uid_h);
+	__swab32s(&b->mbo_gid_h);
+	CLASSERT(offsetof(typeof(*b), mbo_padding_5) != 0);
+}
 
 void lustre_swab_mdt_ioepoch(struct mdt_ioepoch *b)
 {
@@ -1713,7 +1679,6 @@ void lustre_swab_mdt_ioepoch(struct mdt_ioepoch *b)
 	 __swab32s(&b->flags);
 	 CLASSERT(offsetof(typeof(*b), padding) != 0);
 }
-EXPORT_SYMBOL(lustre_swab_mdt_ioepoch);
 
 void lustre_swab_mgs_target_info(struct mgs_target_info *mti)
 {
@@ -1729,11 +1694,10 @@ void lustre_swab_mgs_target_info(struct mgs_target_info *mti)
 	for (i = 0; i < MTI_NIDS_MAX; i++)
 		__swab64s(&mti->mti_nids[i]);
 }
-EXPORT_SYMBOL(lustre_swab_mgs_target_info);
 
 void lustre_swab_mgs_nidtbl_entry(struct mgs_nidtbl_entry *entry)
 {
-	int i;
+	__u8 i;
 
 	__swab64s(&entry->mne_version);
 	__swab32s(&entry->mne_instance);
@@ -1760,14 +1724,12 @@ void lustre_swab_mgs_config_body(struct mgs_config_body *body)
 	__swab32s(&body->mcb_units);
 	__swab16s(&body->mcb_type);
 }
-EXPORT_SYMBOL(lustre_swab_mgs_config_body);
 
 void lustre_swab_mgs_config_res(struct mgs_config_res *body)
 {
 	__swab64s(&body->mcr_offset);
 	__swab64s(&body->mcr_size);
 }
-EXPORT_SYMBOL(lustre_swab_mgs_config_res);
 
 static void lustre_swab_obd_dqinfo(struct obd_dqinfo *i)
 {
@@ -1800,7 +1762,6 @@ void lustre_swab_obd_quotactl(struct obd_quotactl *q)
 	lustre_swab_obd_dqinfo(&q->qc_dqinfo);
 	lustre_swab_obd_dqblk(&q->qc_dqblk);
 }
-EXPORT_SYMBOL(lustre_swab_obd_quotactl);
 
 void lustre_swab_fid2path(struct getinfo_fid2path *gf)
 {
@@ -1822,7 +1783,7 @@ static void lustre_swab_fiemap_extent(struct ll_fiemap_extent *fm_extent)
 
 void lustre_swab_fiemap(struct ll_user_fiemap *fiemap)
 {
-	int i;
+	__u32 i;
 
 	__swab64s(&fiemap->fm_start);
 	__swab64s(&fiemap->fm_length);
@@ -1834,7 +1795,6 @@ void lustre_swab_fiemap(struct ll_user_fiemap *fiemap)
 	for (i = 0; i < fiemap->fm_mapped_extents; i++)
 		lustre_swab_fiemap_extent(&fiemap->fm_extents[i]);
 }
-EXPORT_SYMBOL(lustre_swab_fiemap);
 
 void lustre_swab_mdt_rec_reint (struct mdt_rec_reint *rr)
 {
@@ -1863,7 +1823,6 @@ void lustre_swab_mdt_rec_reint (struct mdt_rec_reint *rr)
 
 	CLASSERT(offsetof(typeof(*rr), rr_padding_4) != 0);
 };
-EXPORT_SYMBOL(lustre_swab_mdt_rec_reint);
 
 void lustre_swab_lov_desc(struct lov_desc *ld)
 {
@@ -1878,18 +1837,42 @@ void lustre_swab_lov_desc(struct lov_desc *ld)
 }
 EXPORT_SYMBOL(lustre_swab_lov_desc);
 
-static void print_lum(struct lov_user_md *lum)
+/* This structure is always in little-endian */
+static void lustre_swab_lmv_mds_md_v1(struct lmv_mds_md_v1 *lmm1)
+{
+	int i;
+
+	__swab32s(&lmm1->lmv_magic);
+	__swab32s(&lmm1->lmv_stripe_count);
+	__swab32s(&lmm1->lmv_master_mdt_index);
+	__swab32s(&lmm1->lmv_hash_type);
+	__swab32s(&lmm1->lmv_layout_version);
+	for (i = 0; i < lmm1->lmv_stripe_count; i++)
+		lustre_swab_lu_fid(&lmm1->lmv_stripe_fids[i]);
+}
+
+void lustre_swab_lmv_mds_md(union lmv_mds_md *lmm)
+{
+	switch (lmm->lmv_magic) {
+	case LMV_MAGIC_V1:
+		lustre_swab_lmv_mds_md_v1(&lmm->lmv_md_v1);
+		break;
+	default:
+		break;
+	}
+}
+EXPORT_SYMBOL(lustre_swab_lmv_mds_md);
+
+void lustre_swab_lmv_user_md(struct lmv_user_md *lum)
 {
-	CDEBUG(D_OTHER, "lov_user_md %p:\n", lum);
-	CDEBUG(D_OTHER, "\tlmm_magic: %#x\n", lum->lmm_magic);
-	CDEBUG(D_OTHER, "\tlmm_pattern: %#x\n", lum->lmm_pattern);
-	CDEBUG(D_OTHER, "\tlmm_object_id: %llu\n", lmm_oi_id(&lum->lmm_oi));
-	CDEBUG(D_OTHER, "\tlmm_object_gr: %llu\n", lmm_oi_seq(&lum->lmm_oi));
-	CDEBUG(D_OTHER, "\tlmm_stripe_size: %#x\n", lum->lmm_stripe_size);
-	CDEBUG(D_OTHER, "\tlmm_stripe_count: %#x\n", lum->lmm_stripe_count);
-	CDEBUG(D_OTHER, "\tlmm_stripe_offset/lmm_layout_gen: %#x\n",
-	       lum->lmm_stripe_offset);
+	__swab32s(&lum->lum_magic);
+	__swab32s(&lum->lum_stripe_count);
+	__swab32s(&lum->lum_stripe_offset);
+	__swab32s(&lum->lum_hash_type);
+	__swab32s(&lum->lum_type);
+	CLASSERT(offsetof(typeof(*lum), lum_padding1));
 }
+EXPORT_SYMBOL(lustre_swab_lmv_user_md);
 
 static void lustre_swab_lmm_oi(struct ost_id *oi)
 {
@@ -1905,7 +1888,6 @@ static void lustre_swab_lov_user_md_common(struct lov_user_md_v1 *lum)
 	__swab32s(&lum->lmm_stripe_size);
 	__swab16s(&lum->lmm_stripe_count);
 	__swab16s(&lum->lmm_stripe_offset);
-	print_lum(lum);
 }
 
 void lustre_swab_lov_user_md_v1(struct lov_user_md_v1 *lum)
@@ -1941,9 +1923,9 @@ void lustre_swab_lov_user_md_objects(struct lov_user_ost_data *lod,
 	int i;
 
 	for (i = 0; i < stripe_count; i++) {
-		lustre_swab_ost_id(&(lod[i].l_ost_oi));
-		__swab32s(&(lod[i].l_ost_gen));
-		__swab32s(&(lod[i].l_ost_idx));
+		lustre_swab_ost_id(&lod[i].l_ost_oi);
+		__swab32s(&lod[i].l_ost_gen);
+		__swab32s(&lod[i].l_ost_idx);
 	}
 }
 EXPORT_SYMBOL(lustre_swab_lov_user_md_objects);
@@ -1973,7 +1955,6 @@ void lustre_swab_ldlm_intent(struct ldlm_intent *i)
 {
 	__swab64s(&i->opc);
 }
-EXPORT_SYMBOL(lustre_swab_ldlm_intent);
 
 static void lustre_swab_ldlm_resource_desc(struct ldlm_resource_desc *r)
 {
@@ -1997,7 +1978,6 @@ void lustre_swab_ldlm_request(struct ldlm_request *rq)
 	__swab32s(&rq->lock_count);
 	/* lock_handle[] opaque */
 }
-EXPORT_SYMBOL(lustre_swab_ldlm_request);
 
 void lustre_swab_ldlm_reply(struct ldlm_reply *r)
 {
@@ -2008,7 +1988,6 @@ void lustre_swab_ldlm_reply(struct ldlm_reply *r)
 	__swab64s(&r->lock_policy_res1);
 	__swab64s(&r->lock_policy_res2);
 }
-EXPORT_SYMBOL(lustre_swab_ldlm_reply);
 
 /* Dump functions */
 void dump_ioo(struct obd_ioobj *ioo)
@@ -2018,14 +1997,12 @@ void dump_ioo(struct obd_ioobj *ioo)
 	       POSTID(&ioo->ioo_oid), ioo->ioo_max_brw,
 	       ioo->ioo_bufcnt);
 }
-EXPORT_SYMBOL(dump_ioo);
 
 void dump_rniobuf(struct niobuf_remote *nb)
 {
 	CDEBUG(D_RPCTRACE, "niobuf_remote: offset=%llu, len=%d, flags=%x\n",
-	       nb->offset, nb->len, nb->flags);
+	       nb->rnb_offset, nb->rnb_len, nb->rnb_flags);
 }
-EXPORT_SYMBOL(dump_rniobuf);
 
 static void dump_obdo(struct obdo *oa)
 {
@@ -2093,13 +2070,11 @@ void dump_ost_body(struct ost_body *ob)
 {
 	dump_obdo(&ob->oa);
 }
-EXPORT_SYMBOL(dump_ost_body);
 
 void dump_rcs(__u32 *rc)
 {
 	CDEBUG(D_RPCTRACE, "rmf_rcs: %d\n", *rc);
 }
-EXPORT_SYMBOL(dump_rcs);
 
 static inline int req_ptlrpc_body_swabbed(struct ptlrpc_request *req)
 {
@@ -2184,14 +2159,12 @@ void lustre_swab_lustre_capa(struct lustre_capa *c)
 	__swab32s(&c->lc_timeout);
 	__swab32s(&c->lc_expiry);
 }
-EXPORT_SYMBOL(lustre_swab_lustre_capa);
 
 void lustre_swab_hsm_user_state(struct hsm_user_state *state)
 {
 	__swab32s(&state->hus_states);
 	__swab32s(&state->hus_archive_id);
 }
-EXPORT_SYMBOL(lustre_swab_hsm_user_state);
 
 void lustre_swab_hsm_state_set(struct hsm_state_set *hss)
 {
@@ -2214,14 +2187,12 @@ void lustre_swab_hsm_current_action(struct hsm_current_action *action)
 	__swab32s(&action->hca_action);
 	lustre_swab_hsm_extent(&action->hca_location);
 }
-EXPORT_SYMBOL(lustre_swab_hsm_current_action);
 
 void lustre_swab_hsm_user_item(struct hsm_user_item *hui)
 {
 	lustre_swab_lu_fid(&hui->hui_fid);
 	lustre_swab_hsm_extent(&hui->hui_extent);
 }
-EXPORT_SYMBOL(lustre_swab_hsm_user_item);
 
 void lustre_swab_layout_intent(struct layout_intent *li)
 {
@@ -2230,7 +2201,6 @@ void lustre_swab_layout_intent(struct layout_intent *li)
 	__swab64s(&li->li_start);
 	__swab64s(&li->li_end);
 }
-EXPORT_SYMBOL(lustre_swab_layout_intent);
 
 void lustre_swab_hsm_progress_kernel(struct hsm_progress_kernel *hpk)
 {
@@ -2241,7 +2211,6 @@ void lustre_swab_hsm_progress_kernel(struct hsm_progress_kernel *hpk)
 	__swab16s(&hpk->hpk_flags);
 	__swab16s(&hpk->hpk_errval);
 }
-EXPORT_SYMBOL(lustre_swab_hsm_progress_kernel);
 
 void lustre_swab_hsm_request(struct hsm_request *hr)
 {
@@ -2251,7 +2220,6 @@ void lustre_swab_hsm_request(struct hsm_request *hr)
 	__swab32s(&hr->hr_itemcount);
 	__swab32s(&hr->hr_data_len);
 }
-EXPORT_SYMBOL(lustre_swab_hsm_request);
 
 void lustre_swab_swap_layouts(struct mdc_swap_layouts *msl)
 {
@@ -2264,4 +2232,3 @@ void lustre_swab_close_data(struct close_data *cd)
 	lustre_swab_lu_fid(&cd->cd_fid);
 	__swab64s(&cd->cd_data_version);
 }
-EXPORT_SYMBOL(lustre_swab_close_data);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pers.c b/drivers/staging/lustre/lustre/ptlrpc/pers.c
index 6c820e944171..5b9fb11c0b6b 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pers.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pers.c
@@ -64,9 +64,9 @@ void ptlrpc_add_bulk_page(struct ptlrpc_bulk_desc *desc, struct page *page,
 {
 	lnet_kiov_t *kiov = &desc->bd_iov[desc->bd_iov_count];
 
-	kiov->kiov_page = page;
-	kiov->kiov_offset = pageoffset;
-	kiov->kiov_len = len;
+	kiov->bv_page = page;
+	kiov->bv_offset = pageoffset;
+	kiov->bv_len = len;
 
 	desc->bd_iov_count++;
 }
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pinger.c b/drivers/staging/lustre/lustre/ptlrpc/pinger.c
index c0529d808d81..5504fc2363ac 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pinger.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pinger.c
@@ -340,7 +340,6 @@ void ptlrpc_pinger_sending_on_import(struct obd_import *imp)
 {
 	ptlrpc_update_next_ping(imp, 0);
 }
-EXPORT_SYMBOL(ptlrpc_pinger_sending_on_import);
 
 void ptlrpc_pinger_commit_expected(struct obd_import *imp)
 {
diff --git a/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_internal.h b/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_internal.h
index a9831fab80f3..f14d193287da 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_internal.h
+++ b/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_internal.h
@@ -53,6 +53,8 @@ int ptlrpc_start_thread(struct ptlrpc_service_part *svcpt, int wait);
 int ptlrpcd_start(struct ptlrpcd_ctl *pc);
 
 /* client.c */
+void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req,
+			       unsigned int service_time);
 struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned npages, unsigned max_brw,
 					 unsigned type, unsigned portal);
 int ptlrpc_request_cache_init(void);
@@ -60,6 +62,11 @@ void ptlrpc_request_cache_fini(void);
 struct ptlrpc_request *ptlrpc_request_cache_alloc(gfp_t flags);
 void ptlrpc_request_cache_free(struct ptlrpc_request *req);
 void ptlrpc_init_xid(void);
+void ptlrpc_set_add_new_req(struct ptlrpcd_ctl *pc,
+			    struct ptlrpc_request *req);
+int ptlrpc_expired_set(void *data);
+int ptlrpc_set_next_timeout(struct ptlrpc_request_set *);
+void ptlrpc_resend_req(struct ptlrpc_request *request);
 
 /* events.c */
 int ptlrpc_init_portals(void);
@@ -268,7 +275,7 @@ void sptlrpc_conf_fini(void);
 int  sptlrpc_init(void);
 void sptlrpc_fini(void);
 
-static inline int ll_rpc_recoverable_error(int rc)
+static inline bool ptlrpc_recoverable_error(int rc)
 {
 	return (rc == -ENOTCONN || rc == -ENODEV);
 }
diff --git a/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c b/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c
index 0a374b6c2f71..1f55d642aa75 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c
@@ -412,7 +412,7 @@ static int ptlrpcd(void *arg)
 	 * an argument, describing its "scope".
 	 */
 	rc = lu_context_init(&env.le_ctx,
-			     LCT_CL_THREAD|LCT_REMEMBER|LCT_NOREF);
+			     LCT_CL_THREAD | LCT_REMEMBER | LCT_NOREF);
 	if (rc == 0) {
 		rc = lu_context_init(env.le_ses,
 				     LCT_SESSION | LCT_REMEMBER | LCT_NOREF);
@@ -567,7 +567,7 @@ int ptlrpcd_start(struct ptlrpcd_ctl *pc)
 	 * ptlrpcd thread (or a thread-set) has to be given an argument,
 	 * describing its "scope".
 	 */
-	rc = lu_context_init(&pc->pc_env.le_ctx, LCT_CL_THREAD|LCT_REMEMBER);
+	rc = lu_context_init(&pc->pc_env.le_ctx, LCT_CL_THREAD | LCT_REMEMBER);
 	if (rc != 0)
 		goto out;
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/recover.c b/drivers/staging/lustre/lustre/ptlrpc/recover.c
index 718b3a8d61c6..405faf0dc9fc 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/recover.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/recover.c
@@ -201,7 +201,6 @@ int ptlrpc_resend(struct obd_import *imp)
 
 	return 0;
 }
-EXPORT_SYMBOL(ptlrpc_resend);
 
 /**
  * Go through all requests in delayed list and wake their threads
@@ -221,7 +220,6 @@ void ptlrpc_wake_delayed(struct obd_import *imp)
 	}
 	spin_unlock(&imp->imp_lock);
 }
-EXPORT_SYMBOL(ptlrpc_wake_delayed);
 
 void ptlrpc_request_handle_notconn(struct ptlrpc_request *failed_req)
 {
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec.c b/drivers/staging/lustre/lustre/ptlrpc/sec.c
index dbd819fa6b75..a7416cd9ac71 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec.c
@@ -311,6 +311,19 @@ static int import_sec_check_expire(struct obd_import *imp)
 	return sptlrpc_import_sec_adapt(imp, NULL, NULL);
 }
 
+/**
+ * Get and validate the client side ptlrpc security facilities from
+ * \a imp. There is a race condition on client reconnect when the import is
+ * being destroyed while there are outstanding client bound requests. In
+ * this case do not output any error messages if import secuity is not
+ * found.
+ *
+ * \param[in] imp obd import associated with client
+ * \param[out] sec client side ptlrpc security
+ *
+ * \retval 0 if security retrieved successfully
+ * \retval -ve errno if there was a problem
+ */
 static int import_sec_validate_get(struct obd_import *imp,
 				   struct ptlrpc_sec **sec)
 {
@@ -323,9 +336,11 @@ static int import_sec_validate_get(struct obd_import *imp,
 	}
 
 	*sec = sptlrpc_import_sec_ref(imp);
+	/* Only output an error when the import is still active */
 	if (!*sec) {
-		CERROR("import %p (%s) with no sec\n",
-		       imp, ptlrpc_import_state_name(imp->imp_state));
+		if (list_empty(&imp->imp_zombie_chain))
+			CERROR("import %p (%s) with no sec\n",
+			       imp, ptlrpc_import_state_name(imp->imp_state));
 		return -EACCES;
 	}
 
@@ -499,7 +514,7 @@ static int sptlrpc_req_replace_dead_ctx(struct ptlrpc_request *req)
 		       newctx, newctx->cc_flags);
 
 		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(HZ);
+		schedule_timeout(msecs_to_jiffies(MSEC_PER_SEC));
 	} else {
 		/*
 		 * it's possible newctx == oldctx if we're switching
@@ -718,8 +733,9 @@ again:
 	req->rq_restart = 0;
 	spin_unlock(&req->rq_lock);
 
-	lwi = LWI_TIMEOUT_INTR(timeout * HZ, ctx_refresh_timeout,
-			       ctx_refresh_interrupt, req);
+	lwi = LWI_TIMEOUT_INTR(msecs_to_jiffies(timeout * MSEC_PER_SEC),
+			       ctx_refresh_timeout, ctx_refresh_interrupt,
+			       req);
 	rc = l_wait_event(req->rq_reply_waitq, ctx_check_refresh(ctx), &lwi);
 
 	/*
@@ -2204,7 +2220,7 @@ int sptlrpc_pack_user_desc(struct lustre_msg *msg, int offset)
 	task_lock(current);
 	if (pud->pud_ngroups > current_ngroups)
 		pud->pud_ngroups = current_ngroups;
-	memcpy(pud->pud_groups, current_cred()->group_info->blocks[0],
+	memcpy(pud->pud_groups, current_cred()->group_info->gid,
 	       pud->pud_ngroups * sizeof(__u32));
 	task_unlock(current);
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c b/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c
index 5f4d79718589..b2cc5ea6cb93 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c
@@ -139,7 +139,7 @@ int sptlrpc_proc_enc_pool_seq_show(struct seq_file *m, void *v)
 		   "cache missing:	   %lu\n"
 		   "low free mark:	   %lu\n"
 		   "max waitqueue depth:     %u\n"
-		   "max wait time:	   %ld/%u\n",
+		   "max wait time:	   %ld/%lu\n",
 		   totalram_pages,
 		   PAGES_PER_POOL,
 		   page_pools.epp_max_pages,
@@ -158,7 +158,7 @@ int sptlrpc_proc_enc_pool_seq_show(struct seq_file *m, void *v)
 		   page_pools.epp_st_lowfree,
 		   page_pools.epp_st_max_wqlen,
 		   page_pools.epp_st_max_wait,
-		   HZ);
+		   msecs_to_jiffies(MSEC_PER_SEC));
 
 	spin_unlock(&page_pools.epp_lock);
 
@@ -326,12 +326,12 @@ void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
 	LASSERT(page_pools.epp_pools[p_idx]);
 
 	for (i = 0; i < desc->bd_iov_count; i++) {
-		LASSERT(desc->bd_enc_iov[i].kiov_page);
+		LASSERT(desc->bd_enc_iov[i].bv_page);
 		LASSERT(g_idx != 0 || page_pools.epp_pools[p_idx]);
 		LASSERT(!page_pools.epp_pools[p_idx][g_idx]);
 
 		page_pools.epp_pools[p_idx][g_idx] =
-					desc->bd_enc_iov[i].kiov_page;
+					desc->bd_enc_iov[i].bv_page;
 
 		if (++g_idx == PAGES_PER_POOL) {
 			p_idx++;
@@ -348,7 +348,6 @@ void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
 	kfree(desc->bd_enc_iov);
 	desc->bd_enc_iov = NULL;
 }
-EXPORT_SYMBOL(sptlrpc_enc_pool_put_pages);
 
 static inline void enc_pools_alloc(void)
 {
@@ -432,12 +431,13 @@ void sptlrpc_enc_pool_fini(void)
 
 	if (page_pools.epp_st_access > 0) {
 		CDEBUG(D_SEC,
-		       "max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%d\n",
+		       "max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%ld\n",
 		       page_pools.epp_st_max_pages, page_pools.epp_st_grows,
 		       page_pools.epp_st_grow_fails,
 		       page_pools.epp_st_shrinks, page_pools.epp_st_access,
 		       page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
-		       page_pools.epp_st_max_wait, HZ);
+		       page_pools.epp_st_max_wait,
+		       msecs_to_jiffies(MSEC_PER_SEC));
 	}
 }
 
@@ -456,13 +456,11 @@ const char *sptlrpc_get_hash_name(__u8 hash_alg)
 {
 	return cfs_crypto_hash_name(cfs_hash_alg_id[hash_alg]);
 }
-EXPORT_SYMBOL(sptlrpc_get_hash_name);
 
 __u8 sptlrpc_get_hash_alg(const char *algname)
 {
 	return cfs_crypto_hash_alg(algname);
 }
-EXPORT_SYMBOL(sptlrpc_get_hash_alg);
 
 int bulk_sec_desc_unpack(struct lustre_msg *msg, int offset, int swabbed)
 {
@@ -522,9 +520,10 @@ int sptlrpc_get_bulk_checksum(struct ptlrpc_bulk_desc *desc, __u8 alg,
 	hashsize = cfs_crypto_hash_digestsize(cfs_hash_alg_id[alg]);
 
 	for (i = 0; i < desc->bd_iov_count; i++) {
-		cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].kiov_page,
-				  desc->bd_iov[i].kiov_offset & ~PAGE_MASK,
-				  desc->bd_iov[i].kiov_len);
+		cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].bv_page,
+					    desc->bd_iov[i].bv_offset &
+					    ~PAGE_MASK,
+					    desc->bd_iov[i].bv_len);
 	}
 
 	if (hashsize > buflen) {
@@ -542,4 +541,3 @@ int sptlrpc_get_bulk_checksum(struct ptlrpc_bulk_desc *desc, __u8 alg,
 
 	return err;
 }
-EXPORT_SYMBOL(sptlrpc_get_bulk_checksum);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_config.c b/drivers/staging/lustre/lustre/ptlrpc/sec_config.c
index c14035479c5f..2181a85efd49 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_config.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_config.c
@@ -58,7 +58,6 @@ enum lustre_sec_part sptlrpc_target_sec_part(struct obd_device *obd)
 	CERROR("unknown target %p(%s)\n", obd, type);
 	return LUSTRE_SP_ANY;
 }
-EXPORT_SYMBOL(sptlrpc_target_sec_part);
 
 /****************************************
  * user supplied flavor string parsing  *
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_gc.c b/drivers/staging/lustre/lustre/ptlrpc/sec_gc.c
index 9b9801ece582..8ffd000eafac 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_gc.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_gc.c
@@ -71,7 +71,6 @@ void sptlrpc_gc_add_sec(struct ptlrpc_sec *sec)
 
 	CDEBUG(D_SEC, "added sec %p(%s)\n", sec, sec->ps_policy->sp_name);
 }
-EXPORT_SYMBOL(sptlrpc_gc_add_sec);
 
 void sptlrpc_gc_del_sec(struct ptlrpc_sec *sec)
 {
@@ -95,7 +94,6 @@ void sptlrpc_gc_del_sec(struct ptlrpc_sec *sec)
 
 	CDEBUG(D_SEC, "del sec %p(%s)\n", sec, sec->ps_policy->sp_name);
 }
-EXPORT_SYMBOL(sptlrpc_gc_del_sec);
 
 static void sec_process_ctx_list(void)
 {
@@ -182,7 +180,8 @@ again:
 		/* check ctx list again before sleep */
 		sec_process_ctx_list();
 
-		lwi = LWI_TIMEOUT(SEC_GC_INTERVAL * HZ, NULL, NULL);
+		lwi = LWI_TIMEOUT(msecs_to_jiffies(SEC_GC_INTERVAL * MSEC_PER_SEC),
+				  NULL, NULL);
 		l_wait_event(thread->t_ctl_waitq,
 			     thread_is_stopping(thread) ||
 			     thread_is_signal(thread),
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c b/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c
index 5c4590b0c521..cd305bcb334a 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c
@@ -154,13 +154,13 @@ static void corrupt_bulk_data(struct ptlrpc_bulk_desc *desc)
 	unsigned int off, i;
 
 	for (i = 0; i < desc->bd_iov_count; i++) {
-		if (desc->bd_iov[i].kiov_len == 0)
+		if (desc->bd_iov[i].bv_len == 0)
 			continue;
 
-		ptr = kmap(desc->bd_iov[i].kiov_page);
-		off = desc->bd_iov[i].kiov_offset & ~PAGE_MASK;
+		ptr = kmap(desc->bd_iov[i].bv_page);
+		off = desc->bd_iov[i].bv_offset & ~PAGE_MASK;
 		ptr[off] ^= 0x1;
-		kunmap(desc->bd_iov[i].kiov_page);
+		kunmap(desc->bd_iov[i].bv_page);
 		return;
 	}
 }
@@ -249,9 +249,12 @@ int plain_ctx_verify(struct ptlrpc_cli_ctx *ctx, struct ptlrpc_request *req)
 		unsigned int hsize = 4;
 
 		cfs_crypto_hash_digest(CFS_HASH_ALG_CRC32,
-				lustre_msg_buf(msg, PLAIN_PACK_MSG_OFF, 0),
-				lustre_msg_buflen(msg, PLAIN_PACK_MSG_OFF),
-				NULL, 0, (unsigned char *)&cksum, &hsize);
+				       lustre_msg_buf(msg, PLAIN_PACK_MSG_OFF,
+						      0),
+				       lustre_msg_buflen(msg,
+							 PLAIN_PACK_MSG_OFF),
+				       NULL, 0, (unsigned char *)&cksum,
+				       &hsize);
 		if (cksum != msg->lm_cksum) {
 			CDEBUG(D_SEC,
 			       "early reply checksum mismatch: %08x != %08x\n",
@@ -349,11 +352,11 @@ int plain_cli_unwrap_bulk(struct ptlrpc_cli_ctx *ctx,
 
 	/* fix the actual data size */
 	for (i = 0, nob = 0; i < desc->bd_iov_count; i++) {
-		if (desc->bd_iov[i].kiov_len + nob > desc->bd_nob_transferred) {
-			desc->bd_iov[i].kiov_len =
+		if (desc->bd_iov[i].bv_len + nob > desc->bd_nob_transferred) {
+			desc->bd_iov[i].bv_len =
 				desc->bd_nob_transferred - nob;
 		}
-		nob += desc->bd_iov[i].kiov_len;
+		nob += desc->bd_iov[i].bv_len;
 	}
 
 	rc = plain_verify_bulk_csum(desc, req->rq_flvr.u_bulk.hash.hash_alg,
@@ -869,9 +872,12 @@ int plain_authorize(struct ptlrpc_request *req)
 		unsigned int hsize = 4;
 
 		cfs_crypto_hash_digest(CFS_HASH_ALG_CRC32,
-			lustre_msg_buf(msg, PLAIN_PACK_MSG_OFF, 0),
-			lustre_msg_buflen(msg, PLAIN_PACK_MSG_OFF),
-			NULL, 0, (unsigned char *)&msg->lm_cksum, &hsize);
+				       lustre_msg_buf(msg, PLAIN_PACK_MSG_OFF,
+						      0),
+				       lustre_msg_buflen(msg,
+							 PLAIN_PACK_MSG_OFF),
+				       NULL, 0, (unsigned char *)&msg->lm_cksum,
+				       &hsize);
 		req->rq_reply_off = 0;
 	}
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/service.c b/drivers/staging/lustre/lustre/ptlrpc/service.c
index 4788c4940c2a..72f39308eebb 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/service.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/service.c
@@ -1005,6 +1005,10 @@ ptlrpc_at_remove_timed(struct ptlrpc_request *req)
 	array->paa_count--;
 }
 
+/*
+ * Attempt to extend the request deadline by sending an early reply to the
+ * client.
+ */
 static int ptlrpc_at_send_early_reply(struct ptlrpc_request *req)
 {
 	struct ptlrpc_service_part *svcpt = req->rq_rqbd->rqbd_svcpt;
@@ -1039,24 +1043,26 @@ static int ptlrpc_at_send_early_reply(struct ptlrpc_request *req)
 		return -ENOSYS;
 	}
 
-	/* Fake our processing time into the future to ask the clients
-	 * for some extra amount of time
+	/*
+	 * We want to extend the request deadline by at_extra seconds,
+	 * so we set our service estimate to reflect how much time has
+	 * passed since this request arrived plus an additional
+	 * at_extra seconds. The client will calculate the new deadline
+	 * based on this service estimate (plus some additional time to
+	 * account for network latency). See ptlrpc_at_recv_early_reply
 	 */
 	at_measured(&svcpt->scp_at_estimate, at_extra +
 		    ktime_get_real_seconds() - req->rq_arrival_time.tv_sec);
+	newdl = req->rq_arrival_time.tv_sec + at_get(&svcpt->scp_at_estimate);
 
 	/* Check to see if we've actually increased the deadline -
 	 * we may be past adaptive_max
 	 */
-	if (req->rq_deadline >= req->rq_arrival_time.tv_sec +
-	    at_get(&svcpt->scp_at_estimate)) {
+	if (req->rq_deadline >= newdl) {
 		DEBUG_REQ(D_WARNING, req, "Couldn't add any time (%ld/%lld), not sending early reply\n",
-			  olddl, req->rq_arrival_time.tv_sec +
-			  at_get(&svcpt->scp_at_estimate) -
-			  ktime_get_real_seconds());
+			  olddl, newdl - ktime_get_real_seconds());
 		return -ETIMEDOUT;
 	}
-	newdl = ktime_get_real_seconds() + at_get(&svcpt->scp_at_estimate);
 
 	reqcopy = ptlrpc_request_cache_alloc(GFP_NOFS);
 	if (!reqcopy)
@@ -1982,11 +1988,12 @@ ptlrpc_wait_event(struct ptlrpc_service_part *svcpt,
 	cond_resched();
 
 	l_wait_event_exclusive_head(svcpt->scp_waitq,
-				ptlrpc_thread_stopping(thread) ||
-				ptlrpc_server_request_incoming(svcpt) ||
-				ptlrpc_server_request_pending(svcpt, false) ||
-				ptlrpc_rqbd_pending(svcpt) ||
-				ptlrpc_at_check(svcpt), &lwi);
+				    ptlrpc_thread_stopping(thread) ||
+				    ptlrpc_server_request_incoming(svcpt) ||
+				    ptlrpc_server_request_pending(svcpt,
+								  false) ||
+				    ptlrpc_rqbd_pending(svcpt) ||
+				    ptlrpc_at_check(svcpt), &lwi);
 
 	if (ptlrpc_thread_stopping(thread))
 		return -EINTR;
@@ -2049,7 +2056,7 @@ static int ptlrpc_main(void *arg)
 	}
 
 	rc = lu_context_init(&env->le_ctx,
-			     svc->srv_ctx_tags|LCT_REMEMBER|LCT_NOREF);
+			     svc->srv_ctx_tags | LCT_REMEMBER | LCT_NOREF);
 	if (rc)
 		goto out_srv_fini;
 
@@ -2349,7 +2356,7 @@ static void ptlrpc_svcpt_stop_threads(struct ptlrpc_service_part *svcpt)
 
 	while (!list_empty(&zombie)) {
 		thread = list_entry(zombie.next,
-					struct ptlrpc_thread, t_link);
+				    struct ptlrpc_thread, t_link);
 		list_del(&thread->t_link);
 		kfree(thread);
 	}
@@ -2398,7 +2405,6 @@ int ptlrpc_start_threads(struct ptlrpc_service *svc)
 	ptlrpc_stop_all_threads(svc);
 	return rc;
 }
-EXPORT_SYMBOL(ptlrpc_start_threads);
 
 int ptlrpc_start_thread(struct ptlrpc_service_part *svcpt, int wait)
 {
@@ -2539,8 +2545,8 @@ int ptlrpc_hr_init(void)
 		LASSERT(hrp->hrp_nthrs > 0);
 		hrp->hrp_thrs =
 			kzalloc_node(hrp->hrp_nthrs * sizeof(*hrt), GFP_NOFS,
-				cfs_cpt_spread_node(ptlrpc_hr.hr_cpt_table,
-						    i));
+				     cfs_cpt_spread_node(ptlrpc_hr.hr_cpt_table,
+							 i));
 		if (!hrp->hrp_thrs) {
 			rc = -ENOMEM;
 			goto out;
@@ -2593,7 +2599,8 @@ static void ptlrpc_wait_replies(struct ptlrpc_service_part *svcpt)
 						     NULL, NULL);
 
 		rc = l_wait_event(svcpt->scp_waitq,
-		     atomic_read(&svcpt->scp_nreps_difficult) == 0, &lwi);
+				  atomic_read(&svcpt->scp_nreps_difficult) == 0,
+				  &lwi);
 		if (rc == 0)
 			break;
 		CWARN("Unexpectedly long timeout %s %p\n",
@@ -2639,7 +2646,7 @@ ptlrpc_service_unlink_rqbd(struct ptlrpc_service *svc)
 		 * event with its 'unlink' flag set for each posted rqbd
 		 */
 		list_for_each_entry(rqbd, &svcpt->scp_rqbd_posted,
-					rqbd_list) {
+				    rqbd_list) {
 			rc = LNetMDUnlink(rqbd->rqbd_md_h);
 			LASSERT(rc == 0 || rc == -ENOENT);
 		}
diff --git a/drivers/staging/lustre/lustre/ptlrpc/wiretest.c b/drivers/staging/lustre/lustre/ptlrpc/wiretest.c
index 6cc2b2edf3fc..b05b1f935e4c 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/wiretest.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/wiretest.c
@@ -190,28 +190,30 @@ void lustre_assert_wire_constants(void)
 		 (long long)REINT_SETXATTR);
 	LASSERTF(REINT_RMENTRY == 8, "found %lld\n",
 		 (long long)REINT_RMENTRY);
-	LASSERTF(REINT_MAX == 9, "found %lld\n",
+	LASSERTF(REINT_MIGRATE == 9, "found %lld\n",
+		 (long long)REINT_MIGRATE);
+	LASSERTF(REINT_MAX == 10, "found %lld\n",
 		 (long long)REINT_MAX);
 	LASSERTF(DISP_IT_EXECD == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_IT_EXECD);
+		 (unsigned)DISP_IT_EXECD);
 	LASSERTF(DISP_LOOKUP_EXECD == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_LOOKUP_EXECD);
+		 (unsigned)DISP_LOOKUP_EXECD);
 	LASSERTF(DISP_LOOKUP_NEG == 0x00000004UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_LOOKUP_NEG);
+		 (unsigned)DISP_LOOKUP_NEG);
 	LASSERTF(DISP_LOOKUP_POS == 0x00000008UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_LOOKUP_POS);
+		 (unsigned)DISP_LOOKUP_POS);
 	LASSERTF(DISP_OPEN_CREATE == 0x00000010UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_OPEN_CREATE);
+		 (unsigned)DISP_OPEN_CREATE);
 	LASSERTF(DISP_OPEN_OPEN == 0x00000020UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_OPEN_OPEN);
+		 (unsigned)DISP_OPEN_OPEN);
 	LASSERTF(DISP_ENQ_COMPLETE == 0x00400000UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_ENQ_COMPLETE);
+		 (unsigned)DISP_ENQ_COMPLETE);
 	LASSERTF(DISP_ENQ_OPEN_REF == 0x00800000UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_ENQ_OPEN_REF);
+		 (unsigned)DISP_ENQ_OPEN_REF);
 	LASSERTF(DISP_ENQ_CREATE_REF == 0x01000000UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_ENQ_CREATE_REF);
+		 (unsigned)DISP_ENQ_CREATE_REF);
 	LASSERTF(DISP_OPEN_LOCK == 0x02000000UL, "found 0x%.8xUL\n",
-		(unsigned)DISP_OPEN_LOCK);
+		 (unsigned)DISP_OPEN_LOCK);
 	LASSERTF(MDS_STATUS_CONN == 1, "found %lld\n",
 		 (long long)MDS_STATUS_CONN);
 	LASSERTF(MDS_STATUS_LOV == 2, "found %lld\n",
@@ -219,55 +221,55 @@ void lustre_assert_wire_constants(void)
 	LASSERTF(LUSTRE_BFLAG_UNCOMMITTED_WRITES == 1, "found %lld\n",
 		 (long long)LUSTRE_BFLAG_UNCOMMITTED_WRITES);
 	LASSERTF(MF_SOM_CHANGE == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)MF_SOM_CHANGE);
+		 (unsigned)MF_SOM_CHANGE);
 	LASSERTF(MF_EPOCH_OPEN == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)MF_EPOCH_OPEN);
+		 (unsigned)MF_EPOCH_OPEN);
 	LASSERTF(MF_EPOCH_CLOSE == 0x00000004UL, "found 0x%.8xUL\n",
-		(unsigned)MF_EPOCH_CLOSE);
+		 (unsigned)MF_EPOCH_CLOSE);
 	LASSERTF(MF_MDC_CANCEL_FID1 == 0x00000008UL, "found 0x%.8xUL\n",
-		(unsigned)MF_MDC_CANCEL_FID1);
+		 (unsigned)MF_MDC_CANCEL_FID1);
 	LASSERTF(MF_MDC_CANCEL_FID2 == 0x00000010UL, "found 0x%.8xUL\n",
-		(unsigned)MF_MDC_CANCEL_FID2);
+		 (unsigned)MF_MDC_CANCEL_FID2);
 	LASSERTF(MF_MDC_CANCEL_FID3 == 0x00000020UL, "found 0x%.8xUL\n",
-		(unsigned)MF_MDC_CANCEL_FID3);
+		 (unsigned)MF_MDC_CANCEL_FID3);
 	LASSERTF(MF_MDC_CANCEL_FID4 == 0x00000040UL, "found 0x%.8xUL\n",
-		(unsigned)MF_MDC_CANCEL_FID4);
+		 (unsigned)MF_MDC_CANCEL_FID4);
 	LASSERTF(MF_SOM_AU == 0x00000080UL, "found 0x%.8xUL\n",
-		(unsigned)MF_SOM_AU);
+		 (unsigned)MF_SOM_AU);
 	LASSERTF(MF_GETATTR_LOCK == 0x00000100UL, "found 0x%.8xUL\n",
-		(unsigned)MF_GETATTR_LOCK);
+		 (unsigned)MF_GETATTR_LOCK);
 	LASSERTF(MDS_ATTR_MODE == 0x0000000000000001ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_MODE);
+		 (long long)MDS_ATTR_MODE);
 	LASSERTF(MDS_ATTR_UID == 0x0000000000000002ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_UID);
+		 (long long)MDS_ATTR_UID);
 	LASSERTF(MDS_ATTR_GID == 0x0000000000000004ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_GID);
+		 (long long)MDS_ATTR_GID);
 	LASSERTF(MDS_ATTR_SIZE == 0x0000000000000008ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_SIZE);
+		 (long long)MDS_ATTR_SIZE);
 	LASSERTF(MDS_ATTR_ATIME == 0x0000000000000010ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_ATIME);
+		 (long long)MDS_ATTR_ATIME);
 	LASSERTF(MDS_ATTR_MTIME == 0x0000000000000020ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_MTIME);
+		 (long long)MDS_ATTR_MTIME);
 	LASSERTF(MDS_ATTR_CTIME == 0x0000000000000040ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_CTIME);
+		 (long long)MDS_ATTR_CTIME);
 	LASSERTF(MDS_ATTR_ATIME_SET == 0x0000000000000080ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_ATIME_SET);
+		 (long long)MDS_ATTR_ATIME_SET);
 	LASSERTF(MDS_ATTR_MTIME_SET == 0x0000000000000100ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_MTIME_SET);
+		 (long long)MDS_ATTR_MTIME_SET);
 	LASSERTF(MDS_ATTR_FORCE == 0x0000000000000200ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_FORCE);
+		 (long long)MDS_ATTR_FORCE);
 	LASSERTF(MDS_ATTR_ATTR_FLAG == 0x0000000000000400ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_ATTR_FLAG);
+		 (long long)MDS_ATTR_ATTR_FLAG);
 	LASSERTF(MDS_ATTR_KILL_SUID == 0x0000000000000800ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_KILL_SUID);
+		 (long long)MDS_ATTR_KILL_SUID);
 	LASSERTF(MDS_ATTR_KILL_SGID == 0x0000000000001000ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_KILL_SGID);
+		 (long long)MDS_ATTR_KILL_SGID);
 	LASSERTF(MDS_ATTR_CTIME_SET == 0x0000000000002000ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_CTIME_SET);
+		 (long long)MDS_ATTR_CTIME_SET);
 	LASSERTF(MDS_ATTR_FROM_OPEN == 0x0000000000004000ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_FROM_OPEN);
+		 (long long)MDS_ATTR_FROM_OPEN);
 	LASSERTF(MDS_ATTR_BLOCKS == 0x0000000000008000ULL, "found 0x%.16llxULL\n",
-			(long long)MDS_ATTR_BLOCKS);
+		 (long long)MDS_ATTR_BLOCKS);
 	LASSERTF(FLD_QUERY == 900, "found %lld\n",
 		 (long long)FLD_QUERY);
 	LASSERTF(FLD_FIRST_OPC == 900, "found %lld\n",
@@ -418,15 +420,15 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct lustre_mdt_attrs *)0)->lma_self_fid) == 16, "found %lld\n",
 		 (long long)(int)sizeof(((struct lustre_mdt_attrs *)0)->lma_self_fid));
 	LASSERTF(LMAI_RELEASED == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)LMAI_RELEASED);
+		 (unsigned)LMAI_RELEASED);
 	LASSERTF(LMAC_HSM == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)LMAC_HSM);
+		 (unsigned)LMAC_HSM);
 	LASSERTF(LMAC_SOM == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)LMAC_SOM);
+		 (unsigned)LMAC_SOM);
 	LASSERTF(LMAC_NOT_IN_OI == 0x00000004UL, "found 0x%.8xUL\n",
-		(unsigned)LMAC_NOT_IN_OI);
+		 (unsigned)LMAC_NOT_IN_OI);
 	LASSERTF(LMAC_FID_ON_OST == 0x00000008UL, "found 0x%.8xUL\n",
-		(unsigned)LMAC_FID_ON_OST);
+		 (unsigned)LMAC_FID_ON_OST);
 
 	/* Checks for struct ost_id */
 	LASSERTF((int)sizeof(struct ost_id) == 16, "found %lld\n",
@@ -452,35 +454,35 @@ void lustre_assert_wire_constants(void)
 	LASSERTF(FID_SEQ_IGIF == 12, "found %lld\n",
 		 (long long)FID_SEQ_IGIF);
 	LASSERTF(FID_SEQ_IGIF_MAX == 0x00000000ffffffffULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_IGIF_MAX);
+		 (long long)FID_SEQ_IGIF_MAX);
 	LASSERTF(FID_SEQ_IDIF == 0x0000000100000000ULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_IDIF);
+		 (long long)FID_SEQ_IDIF);
 	LASSERTF(FID_SEQ_IDIF_MAX == 0x00000001ffffffffULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_IDIF_MAX);
+		 (long long)FID_SEQ_IDIF_MAX);
 	LASSERTF(FID_SEQ_START == 0x0000000200000000ULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_START);
+		 (long long)FID_SEQ_START);
 	LASSERTF(FID_SEQ_LOCAL_FILE == 0x0000000200000001ULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_LOCAL_FILE);
+		 (long long)FID_SEQ_LOCAL_FILE);
 	LASSERTF(FID_SEQ_DOT_LUSTRE == 0x0000000200000002ULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_DOT_LUSTRE);
+		 (long long)FID_SEQ_DOT_LUSTRE);
 	LASSERTF(FID_SEQ_SPECIAL == 0x0000000200000004ULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_SPECIAL);
+		 (long long)FID_SEQ_SPECIAL);
 	LASSERTF(FID_SEQ_QUOTA == 0x0000000200000005ULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_QUOTA);
+		 (long long)FID_SEQ_QUOTA);
 	LASSERTF(FID_SEQ_QUOTA_GLB == 0x0000000200000006ULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_QUOTA_GLB);
+		 (long long)FID_SEQ_QUOTA_GLB);
 	LASSERTF(FID_SEQ_ROOT == 0x0000000200000007ULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_ROOT);
+		 (long long)FID_SEQ_ROOT);
 	LASSERTF(FID_SEQ_NORMAL == 0x0000000200000400ULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_NORMAL);
+		 (long long)FID_SEQ_NORMAL);
 	LASSERTF(FID_SEQ_LOV_DEFAULT == 0xffffffffffffffffULL, "found 0x%.16llxULL\n",
-			(long long)FID_SEQ_LOV_DEFAULT);
+		 (long long)FID_SEQ_LOV_DEFAULT);
 	LASSERTF(FID_OID_SPECIAL_BFL == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)FID_OID_SPECIAL_BFL);
+		 (unsigned)FID_OID_SPECIAL_BFL);
 	LASSERTF(FID_OID_DOT_LUSTRE == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)FID_OID_DOT_LUSTRE);
+		 (unsigned)FID_OID_DOT_LUSTRE);
 	LASSERTF(FID_OID_DOT_LUSTRE_OBF == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)FID_OID_DOT_LUSTRE_OBF);
+		 (unsigned)FID_OID_DOT_LUSTRE_OBF);
 
 	/* Checks for struct lu_dirent */
 	LASSERTF((int)sizeof(struct lu_dirent) == 32, "found %lld\n",
@@ -510,11 +512,11 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct lu_dirent *)0)->lde_name[0]) == 1, "found %lld\n",
 		 (long long)(int)sizeof(((struct lu_dirent *)0)->lde_name[0]));
 	LASSERTF(LUDA_FID == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)LUDA_FID);
+		 (unsigned)LUDA_FID);
 	LASSERTF(LUDA_TYPE == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)LUDA_TYPE);
+		 (unsigned)LUDA_TYPE);
 	LASSERTF(LUDA_64BITHASH == 0x00000004UL, "found 0x%.8xUL\n",
-		(unsigned)LUDA_64BITHASH);
+		 (unsigned)LUDA_64BITHASH);
 
 	/* Checks for struct luda_type */
 	LASSERTF((int)sizeof(struct luda_type) == 2, "found %lld\n",
@@ -602,9 +604,9 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct lustre_msg_v2 *)0)->lm_buflens[0]) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct lustre_msg_v2 *)0)->lm_buflens[0]));
 	LASSERTF(LUSTRE_MSG_MAGIC_V2 == 0x0BD00BD3, "found 0x%.8x\n",
-		LUSTRE_MSG_MAGIC_V2);
+		 LUSTRE_MSG_MAGIC_V2);
 	LASSERTF(LUSTRE_MSG_MAGIC_V2_SWABBED == 0xD30BD00B, "found 0x%.8x\n",
-		LUSTRE_MSG_MAGIC_V2_SWABBED);
+		 LUSTRE_MSG_MAGIC_V2_SWABBED);
 
 	/* Checks for struct ptlrpc_body */
 	LASSERTF((int)sizeof(struct ptlrpc_body_v3) == 184, "found %lld\n",
@@ -682,7 +684,7 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_padding));
 	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding) == 32, "found %lld\n",
 		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding));
-	CLASSERT(JOBSTATS_JOBID_SIZE == 32);
+	CLASSERT(LUSTRE_JOBID_SIZE == 32);
 	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_jobid) == 152, "found %lld\n",
 		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_jobid));
 	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_jobid) == 32, "found %lld\n",
@@ -780,61 +782,61 @@ void lustre_assert_wire_constants(void)
 	LASSERTF(MSG_PTLRPC_HEADER_OFF == 31, "found %lld\n",
 		 (long long)MSG_PTLRPC_HEADER_OFF);
 	LASSERTF(PTLRPC_MSG_VERSION == 0x00000003, "found 0x%.8x\n",
-		PTLRPC_MSG_VERSION);
+		 PTLRPC_MSG_VERSION);
 	LASSERTF(LUSTRE_VERSION_MASK == 0xffff0000, "found 0x%.8x\n",
-		LUSTRE_VERSION_MASK);
+		 LUSTRE_VERSION_MASK);
 	LASSERTF(LUSTRE_OBD_VERSION == 0x00010000, "found 0x%.8x\n",
-		LUSTRE_OBD_VERSION);
+		 LUSTRE_OBD_VERSION);
 	LASSERTF(LUSTRE_MDS_VERSION == 0x00020000, "found 0x%.8x\n",
-		LUSTRE_MDS_VERSION);
+		 LUSTRE_MDS_VERSION);
 	LASSERTF(LUSTRE_OST_VERSION == 0x00030000, "found 0x%.8x\n",
-		LUSTRE_OST_VERSION);
+		 LUSTRE_OST_VERSION);
 	LASSERTF(LUSTRE_DLM_VERSION == 0x00040000, "found 0x%.8x\n",
-		LUSTRE_DLM_VERSION);
+		 LUSTRE_DLM_VERSION);
 	LASSERTF(LUSTRE_LOG_VERSION == 0x00050000, "found 0x%.8x\n",
-		LUSTRE_LOG_VERSION);
+		 LUSTRE_LOG_VERSION);
 	LASSERTF(LUSTRE_MGS_VERSION == 0x00060000, "found 0x%.8x\n",
-		LUSTRE_MGS_VERSION);
+		 LUSTRE_MGS_VERSION);
 	LASSERTF(MSGHDR_AT_SUPPORT == 1, "found %lld\n",
 		 (long long)MSGHDR_AT_SUPPORT);
 	LASSERTF(MSGHDR_CKSUM_INCOMPAT18 == 2, "found %lld\n",
 		 (long long)MSGHDR_CKSUM_INCOMPAT18);
 	LASSERTF(MSG_OP_FLAG_MASK == 0xffff0000UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_OP_FLAG_MASK);
+		 (unsigned)MSG_OP_FLAG_MASK);
 	LASSERTF(MSG_OP_FLAG_SHIFT == 16, "found %lld\n",
 		 (long long)MSG_OP_FLAG_SHIFT);
 	LASSERTF(MSG_GEN_FLAG_MASK == 0x0000ffffUL, "found 0x%.8xUL\n",
-		(unsigned)MSG_GEN_FLAG_MASK);
+		 (unsigned)MSG_GEN_FLAG_MASK);
 	LASSERTF(MSG_LAST_REPLAY == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_LAST_REPLAY);
+		 (unsigned)MSG_LAST_REPLAY);
 	LASSERTF(MSG_RESENT == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_RESENT);
+		 (unsigned)MSG_RESENT);
 	LASSERTF(MSG_REPLAY == 0x00000004UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_REPLAY);
+		 (unsigned)MSG_REPLAY);
 	LASSERTF(MSG_DELAY_REPLAY == 0x00000010UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_DELAY_REPLAY);
+		 (unsigned)MSG_DELAY_REPLAY);
 	LASSERTF(MSG_VERSION_REPLAY == 0x00000020UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_VERSION_REPLAY);
+		 (unsigned)MSG_VERSION_REPLAY);
 	LASSERTF(MSG_REQ_REPLAY_DONE == 0x00000040UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_REQ_REPLAY_DONE);
+		 (unsigned)MSG_REQ_REPLAY_DONE);
 	LASSERTF(MSG_LOCK_REPLAY_DONE == 0x00000080UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_LOCK_REPLAY_DONE);
+		 (unsigned)MSG_LOCK_REPLAY_DONE);
 	LASSERTF(MSG_CONNECT_RECOVERING == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_CONNECT_RECOVERING);
+		 (unsigned)MSG_CONNECT_RECOVERING);
 	LASSERTF(MSG_CONNECT_RECONNECT == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_CONNECT_RECONNECT);
+		 (unsigned)MSG_CONNECT_RECONNECT);
 	LASSERTF(MSG_CONNECT_REPLAYABLE == 0x00000004UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_CONNECT_REPLAYABLE);
+		 (unsigned)MSG_CONNECT_REPLAYABLE);
 	LASSERTF(MSG_CONNECT_LIBCLIENT == 0x00000010UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_CONNECT_LIBCLIENT);
+		 (unsigned)MSG_CONNECT_LIBCLIENT);
 	LASSERTF(MSG_CONNECT_INITIAL == 0x00000020UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_CONNECT_INITIAL);
+		 (unsigned)MSG_CONNECT_INITIAL);
 	LASSERTF(MSG_CONNECT_ASYNC == 0x00000040UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_CONNECT_ASYNC);
+		 (unsigned)MSG_CONNECT_ASYNC);
 	LASSERTF(MSG_CONNECT_NEXT_VER == 0x00000080UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_CONNECT_NEXT_VER);
+		 (unsigned)MSG_CONNECT_NEXT_VER);
 	LASSERTF(MSG_CONNECT_TRANSNO == 0x00000100UL, "found 0x%.8xUL\n",
-		(unsigned)MSG_CONNECT_TRANSNO);
+		 (unsigned)MSG_CONNECT_TRANSNO);
 
 	/* Checks for struct obd_connect_data */
 	LASSERTF((int)sizeof(struct obd_connect_data) == 192, "found %lld\n",
@@ -1069,12 +1071,18 @@ void lustre_assert_wire_constants(void)
 		 "found 0x%.16llxULL\n", OBD_CONNECT_FLOCK_DEAD);
 	LASSERTF(OBD_CONNECT_OPEN_BY_FID == 0x20000000000000ULL,
 		 "found 0x%.16llxULL\n", OBD_CONNECT_OPEN_BY_FID);
+	LASSERTF(OBD_CONNECT_LFSCK == 0x40000000000000ULL, "found 0x%.16llxULL\n",
+		 OBD_CONNECT_LFSCK);
+	LASSERTF(OBD_CONNECT_UNLINK_CLOSE == 0x100000000000000ULL, "found 0x%.16llxULL\n",
+		 OBD_CONNECT_UNLINK_CLOSE);
+	LASSERTF(OBD_CONNECT_DIR_STRIPE == 0x400000000000000ULL, "found 0x%.16llxULL\n",
+		 OBD_CONNECT_DIR_STRIPE);
 	LASSERTF(OBD_CKSUM_CRC32 == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)OBD_CKSUM_CRC32);
+		 (unsigned)OBD_CKSUM_CRC32);
 	LASSERTF(OBD_CKSUM_ADLER == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)OBD_CKSUM_ADLER);
+		 (unsigned)OBD_CKSUM_ADLER);
 	LASSERTF(OBD_CKSUM_CRC32C == 0x00000004UL, "found 0x%.8xUL\n",
-		(unsigned)OBD_CKSUM_CRC32C);
+		 (unsigned)OBD_CKSUM_CRC32C);
 
 	/* Checks for struct obdo */
 	LASSERTF((int)sizeof(struct obdo) == 208, "found %lld\n",
@@ -1346,7 +1354,7 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct lov_mds_md_v1, lmm_objects[0]));
 	LASSERTF((int)sizeof(((struct lov_mds_md_v1 *)0)->lmm_objects[0]) == 24, "found %lld\n",
 		 (long long)(int)sizeof(((struct lov_mds_md_v1 *)0)->lmm_objects[0]));
-	CLASSERT(LOV_MAGIC_V1 == 0x0BD10BD0);
+	CLASSERT(LOV_MAGIC_V1 == (0x0BD10000 | 0x0BD0));
 
 	/* Checks for struct lov_mds_md_v3 */
 	LASSERTF((int)sizeof(struct lov_mds_md_v3) == 48, "found %lld\n",
@@ -1375,7 +1383,7 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct lov_mds_md_v3, lmm_layout_gen));
 	LASSERTF((int)sizeof(((struct lov_mds_md_v3 *)0)->lmm_layout_gen) == 2, "found %lld\n",
 		 (long long)(int)sizeof(((struct lov_mds_md_v3 *)0)->lmm_layout_gen));
-	CLASSERT(LOV_MAXPOOLNAME == 16);
+	CLASSERT(LOV_MAXPOOLNAME == 15);
 	LASSERTF((int)offsetof(struct lov_mds_md_v3, lmm_pool_name[16]) == 48, "found %lld\n",
 		 (long long)(int)offsetof(struct lov_mds_md_v3, lmm_pool_name[16]));
 	LASSERTF((int)sizeof(((struct lov_mds_md_v3 *)0)->lmm_pool_name[16]) == 1, "found %lld\n",
@@ -1384,15 +1392,64 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct lov_mds_md_v3, lmm_objects[0]));
 	LASSERTF((int)sizeof(((struct lov_mds_md_v3 *)0)->lmm_objects[0]) == 24, "found %lld\n",
 		 (long long)(int)sizeof(((struct lov_mds_md_v3 *)0)->lmm_objects[0]));
-	CLASSERT(LOV_MAGIC_V3 == 0x0BD30BD0);
+	CLASSERT(LOV_MAGIC_V3 == (0x0BD30000 | 0x0BD0));
 	LASSERTF(LOV_PATTERN_RAID0 == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)LOV_PATTERN_RAID0);
+		 (unsigned)LOV_PATTERN_RAID0);
 	LASSERTF(LOV_PATTERN_RAID1 == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)LOV_PATTERN_RAID1);
+		 (unsigned)LOV_PATTERN_RAID1);
 	LASSERTF(LOV_PATTERN_FIRST == 0x00000100UL, "found 0x%.8xUL\n",
-		(unsigned)LOV_PATTERN_FIRST);
+		 (unsigned)LOV_PATTERN_FIRST);
 	LASSERTF(LOV_PATTERN_CMOBD == 0x00000200UL, "found 0x%.8xUL\n",
-		(unsigned)LOV_PATTERN_CMOBD);
+		 (unsigned)LOV_PATTERN_CMOBD);
+
+	/* Checks for struct lmv_mds_md_v1 */
+	LASSERTF((int)sizeof(struct lmv_mds_md_v1) == 56, "found %lld\n",
+		 (long long)(int)sizeof(struct lmv_mds_md_v1));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_magic) == 0, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_magic));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_magic) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_magic));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_stripe_count) == 4, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_stripe_count));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_stripe_count) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_stripe_count));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_master_mdt_index) == 8, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_master_mdt_index));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_master_mdt_index) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_master_mdt_index));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_hash_type) == 12, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_hash_type));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_hash_type) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_hash_type));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_layout_version) == 16, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_layout_version));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_layout_version) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_layout_version));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_padding1) == 20, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_padding1));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_padding1) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_padding1));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_padding2) == 24, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_padding2));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_padding2) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_padding2));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_padding3) == 32, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_padding3));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_padding3) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_padding3));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_pool_name[16]) == 56, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_pool_name[16]));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_pool_name[16]) == 1, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_pool_name[16]));
+	LASSERTF((int)offsetof(struct lmv_mds_md_v1, lmv_stripe_fids[0]) == 56, "found %lld\n",
+		 (long long)(int)offsetof(struct lmv_mds_md_v1, lmv_stripe_fids[0]));
+	LASSERTF((int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_stripe_fids[0]) == 16, "found %lld\n",
+		 (long long)(int)sizeof(((struct lmv_mds_md_v1 *)0)->lmv_stripe_fids[0]));
+	CLASSERT(LMV_MAGIC_V1 == 0x0CD20CD0);
+	CLASSERT(LMV_MAGIC_STRIPE == 0x0CD40CD0);
+	CLASSERT(LMV_HASH_TYPE_MASK == 0x0000ffff);
+	CLASSERT(LMV_HASH_FLAG_MIGRATION == 0x80000000);
+	CLASSERT(LMV_HASH_FLAG_DEAD == 0x40000000);
 
 	/* Checks for struct obd_statfs */
 	LASSERTF((int)sizeof(struct obd_statfs) == 144, "found %lld\n",
@@ -1582,53 +1639,53 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct obd_dqblk *)0)->dqb_padding) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct obd_dqblk *)0)->dqb_padding));
 	LASSERTF(Q_QUOTACHECK == 0x800100, "found 0x%.8x\n",
-		Q_QUOTACHECK);
+		 Q_QUOTACHECK);
 	LASSERTF(Q_INITQUOTA == 0x800101, "found 0x%.8x\n",
-		Q_INITQUOTA);
+		 Q_INITQUOTA);
 	LASSERTF(Q_GETOINFO == 0x800102, "found 0x%.8x\n",
-		Q_GETOINFO);
+		 Q_GETOINFO);
 	LASSERTF(Q_GETOQUOTA == 0x800103, "found 0x%.8x\n",
-		Q_GETOQUOTA);
+		 Q_GETOQUOTA);
 	LASSERTF(Q_FINVALIDATE == 0x800104, "found 0x%.8x\n",
-		Q_FINVALIDATE);
+		 Q_FINVALIDATE);
 
 	/* Checks for struct niobuf_remote */
 	LASSERTF((int)sizeof(struct niobuf_remote) == 16, "found %lld\n",
 		 (long long)(int)sizeof(struct niobuf_remote));
-	LASSERTF((int)offsetof(struct niobuf_remote, offset) == 0, "found %lld\n",
-		 (long long)(int)offsetof(struct niobuf_remote, offset));
-	LASSERTF((int)sizeof(((struct niobuf_remote *)0)->offset) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct niobuf_remote *)0)->offset));
-	LASSERTF((int)offsetof(struct niobuf_remote, len) == 8, "found %lld\n",
-		 (long long)(int)offsetof(struct niobuf_remote, len));
-	LASSERTF((int)sizeof(((struct niobuf_remote *)0)->len) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct niobuf_remote *)0)->len));
-	LASSERTF((int)offsetof(struct niobuf_remote, flags) == 12, "found %lld\n",
-		 (long long)(int)offsetof(struct niobuf_remote, flags));
-	LASSERTF((int)sizeof(((struct niobuf_remote *)0)->flags) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct niobuf_remote *)0)->flags));
+	LASSERTF((int)offsetof(struct niobuf_remote, rnb_offset) == 0, "found %lld\n",
+		 (long long)(int)offsetof(struct niobuf_remote, rnb_offset));
+	LASSERTF((int)sizeof(((struct niobuf_remote *)0)->rnb_offset) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct niobuf_remote *)0)->rnb_offset));
+	LASSERTF((int)offsetof(struct niobuf_remote, rnb_len) == 8, "found %lld\n",
+		 (long long)(int)offsetof(struct niobuf_remote, rnb_len));
+	LASSERTF((int)sizeof(((struct niobuf_remote *)0)->rnb_len) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct niobuf_remote *)0)->rnb_len));
+	LASSERTF((int)offsetof(struct niobuf_remote, rnb_flags) == 12, "found %lld\n",
+		 (long long)(int)offsetof(struct niobuf_remote, rnb_flags));
+	LASSERTF((int)sizeof(((struct niobuf_remote *)0)->rnb_flags) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct niobuf_remote *)0)->rnb_flags));
 	LASSERTF(OBD_BRW_READ == 0x01, "found 0x%.8x\n",
-		OBD_BRW_READ);
+		 OBD_BRW_READ);
 	LASSERTF(OBD_BRW_WRITE == 0x02, "found 0x%.8x\n",
-		OBD_BRW_WRITE);
+		 OBD_BRW_WRITE);
 	LASSERTF(OBD_BRW_SYNC == 0x08, "found 0x%.8x\n",
-		OBD_BRW_SYNC);
+		 OBD_BRW_SYNC);
 	LASSERTF(OBD_BRW_CHECK == 0x10, "found 0x%.8x\n",
-		OBD_BRW_CHECK);
+		 OBD_BRW_CHECK);
 	LASSERTF(OBD_BRW_FROM_GRANT == 0x20, "found 0x%.8x\n",
-		OBD_BRW_FROM_GRANT);
+		 OBD_BRW_FROM_GRANT);
 	LASSERTF(OBD_BRW_GRANTED == 0x40, "found 0x%.8x\n",
-		OBD_BRW_GRANTED);
+		 OBD_BRW_GRANTED);
 	LASSERTF(OBD_BRW_NOCACHE == 0x80, "found 0x%.8x\n",
-		OBD_BRW_NOCACHE);
+		 OBD_BRW_NOCACHE);
 	LASSERTF(OBD_BRW_NOQUOTA == 0x100, "found 0x%.8x\n",
-		OBD_BRW_NOQUOTA);
+		 OBD_BRW_NOQUOTA);
 	LASSERTF(OBD_BRW_SRVLOCK == 0x200, "found 0x%.8x\n",
-		OBD_BRW_SRVLOCK);
+		 OBD_BRW_SRVLOCK);
 	LASSERTF(OBD_BRW_ASYNC == 0x400, "found 0x%.8x\n",
-		OBD_BRW_ASYNC);
+		 OBD_BRW_ASYNC);
 	LASSERTF(OBD_BRW_MEMALLOC == 0x800, "found 0x%.8x\n",
-		OBD_BRW_MEMALLOC);
+		 OBD_BRW_MEMALLOC);
 	LASSERTF(OBD_BRW_OVER_USRQUOTA == 0x1000, "found 0x%.8x\n",
 		 OBD_BRW_OVER_USRQUOTA);
 	LASSERTF(OBD_BRW_OVER_GRPQUOTA == 0x2000, "found 0x%.8x\n",
@@ -1663,203 +1720,203 @@ void lustre_assert_wire_constants(void)
 	/* Checks for struct mdt_body */
 	LASSERTF((int)sizeof(struct mdt_body) == 216, "found %lld\n",
 		 (long long)(int)sizeof(struct mdt_body));
-	LASSERTF((int)offsetof(struct mdt_body, fid1) == 0, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, fid1));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->fid1) == 16, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->fid1));
-	LASSERTF((int)offsetof(struct mdt_body, fid2) == 16, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, fid2));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->fid2) == 16, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->fid2));
-	LASSERTF((int)offsetof(struct mdt_body, handle) == 32, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, handle));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->handle) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->handle));
-	LASSERTF((int)offsetof(struct mdt_body, valid) == 40, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, valid));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->valid) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->valid));
-	LASSERTF((int)offsetof(struct mdt_body, size) == 48, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, size));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->size) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->size));
-	LASSERTF((int)offsetof(struct mdt_body, mtime) == 56, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, mtime));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->mtime) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->mtime));
-	LASSERTF((int)offsetof(struct mdt_body, atime) == 64, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, atime));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->atime) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->atime));
-	LASSERTF((int)offsetof(struct mdt_body, ctime) == 72, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, ctime));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->ctime) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->ctime));
-	LASSERTF((int)offsetof(struct mdt_body, blocks) == 80, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, blocks));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->blocks) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->blocks));
-	LASSERTF((int)offsetof(struct mdt_body, t_state) == 96, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, t_state));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->t_state) == 8,
+	LASSERTF((int)offsetof(struct mdt_body, mbo_fid1) == 0, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_fid1));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_fid1) == 16, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_fid1));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_fid2) == 16, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_fid2));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_fid2) == 16, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_fid2));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_handle) == 32, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_handle));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_handle) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_handle));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_valid) == 40, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_valid));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_valid) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_valid));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_size) == 48, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_size));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_size) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_size));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_mtime) == 56, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_mtime));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_mtime) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_mtime));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_atime) == 64, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_atime));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_atime) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_atime));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_ctime) == 72, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_ctime));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_ctime) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_ctime));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_blocks) == 80, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_blocks));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_blocks) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_blocks));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_t_state) == 96, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_t_state));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_t_state) == 8,
 		 "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->t_state));
-	LASSERTF((int)offsetof(struct mdt_body, fsuid) == 104, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, fsuid));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->fsuid) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->fsuid));
-	LASSERTF((int)offsetof(struct mdt_body, fsgid) == 108, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, fsgid));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->fsgid) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->fsgid));
-	LASSERTF((int)offsetof(struct mdt_body, capability) == 112, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, capability));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->capability) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->capability));
-	LASSERTF((int)offsetof(struct mdt_body, mode) == 116, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, mode));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->mode) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->mode));
-	LASSERTF((int)offsetof(struct mdt_body, uid) == 120, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, uid));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->uid) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->uid));
-	LASSERTF((int)offsetof(struct mdt_body, gid) == 124, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, gid));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->gid) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->gid));
-	LASSERTF((int)offsetof(struct mdt_body, flags) == 128, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, flags));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->flags) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->flags));
-	LASSERTF((int)offsetof(struct mdt_body, rdev) == 132, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, rdev));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->rdev) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->rdev));
-	LASSERTF((int)offsetof(struct mdt_body, nlink) == 136, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, nlink));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->nlink) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->nlink));
-	LASSERTF((int)offsetof(struct mdt_body, unused2) == 140, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, unused2));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->unused2) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->unused2));
-	LASSERTF((int)offsetof(struct mdt_body, suppgid) == 144, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, suppgid));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->suppgid) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->suppgid));
-	LASSERTF((int)offsetof(struct mdt_body, eadatasize) == 148, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, eadatasize));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->eadatasize) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->eadatasize));
-	LASSERTF((int)offsetof(struct mdt_body, aclsize) == 152, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, aclsize));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->aclsize) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->aclsize));
-	LASSERTF((int)offsetof(struct mdt_body, max_mdsize) == 156, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, max_mdsize));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->max_mdsize) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->max_mdsize));
-	LASSERTF((int)offsetof(struct mdt_body, max_cookiesize) == 160, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, max_cookiesize));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->max_cookiesize) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->max_cookiesize));
-	LASSERTF((int)offsetof(struct mdt_body, uid_h) == 164, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, uid_h));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->uid_h) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->uid_h));
-	LASSERTF((int)offsetof(struct mdt_body, gid_h) == 168, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, gid_h));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->gid_h) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->gid_h));
-	LASSERTF((int)offsetof(struct mdt_body, padding_5) == 172, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, padding_5));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->padding_5) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->padding_5));
-	LASSERTF((int)offsetof(struct mdt_body, padding_6) == 176, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, padding_6));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->padding_6) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->padding_6));
-	LASSERTF((int)offsetof(struct mdt_body, padding_7) == 184, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, padding_7));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->padding_7) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->padding_7));
-	LASSERTF((int)offsetof(struct mdt_body, padding_8) == 192, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, padding_8));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->padding_8) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->padding_8));
-	LASSERTF((int)offsetof(struct mdt_body, padding_9) == 200, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, padding_9));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->padding_9) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->padding_9));
-	LASSERTF((int)offsetof(struct mdt_body, padding_10) == 208, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, padding_10));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->padding_10) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->padding_10));
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_t_state));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_fsuid) == 104, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_fsuid));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_fsuid) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_fsuid));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_fsgid) == 108, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_fsgid));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_fsgid) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_fsgid));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_capability) == 112, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_capability));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_capability) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_capability));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_mode) == 116, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_mode));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_mode) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_mode));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_uid) == 120, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_uid));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_uid) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_uid));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_gid) == 124, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_gid));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_gid) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_gid));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_flags) == 128, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_flags));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_flags) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_flags));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_rdev) == 132, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_rdev));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_rdev) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_rdev));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_nlink) == 136, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_nlink));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_nlink) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_nlink));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_unused2) == 140, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_unused2));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_unused2) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_unused2));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_suppgid) == 144, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_suppgid));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_suppgid) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_suppgid));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_eadatasize) == 148, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_eadatasize));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_eadatasize) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_eadatasize));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_aclsize) == 152, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_aclsize));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_aclsize) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_aclsize));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_max_mdsize) == 156, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_max_mdsize));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_max_mdsize) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_max_mdsize));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_max_cookiesize) == 160, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_max_cookiesize));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_max_cookiesize) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_max_cookiesize));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_uid_h) == 164, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_uid_h));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_uid_h) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_uid_h));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_gid_h) == 168, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_gid_h));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_gid_h) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_gid_h));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_padding_5) == 172, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_padding_5));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_padding_5) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_padding_5));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_padding_6) == 176, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_padding_6));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_padding_6) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_padding_6));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_padding_7) == 184, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_padding_7));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_padding_7) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_padding_7));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_padding_8) == 192, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_padding_8));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_padding_8) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_padding_8));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_padding_9) == 200, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_padding_9));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_padding_9) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_padding_9));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_padding_10) == 208, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_padding_10));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_padding_10) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_padding_10));
 	LASSERTF(MDS_FMODE_CLOSED == 000000000000UL, "found 0%.11oUL\n",
-		MDS_FMODE_CLOSED);
+		 MDS_FMODE_CLOSED);
 	LASSERTF(MDS_FMODE_EXEC == 000000000004UL, "found 0%.11oUL\n",
-		MDS_FMODE_EXEC);
+		 MDS_FMODE_EXEC);
 	LASSERTF(MDS_FMODE_EPOCH == 000001000000UL, "found 0%.11oUL\n",
-		MDS_FMODE_EPOCH);
+		 MDS_FMODE_EPOCH);
 	LASSERTF(MDS_FMODE_TRUNC == 000002000000UL, "found 0%.11oUL\n",
-		MDS_FMODE_TRUNC);
+		 MDS_FMODE_TRUNC);
 	LASSERTF(MDS_FMODE_SOM == 000004000000UL, "found 0%.11oUL\n",
-		MDS_FMODE_SOM);
+		 MDS_FMODE_SOM);
 	LASSERTF(MDS_OPEN_CREATED == 000000000010UL, "found 0%.11oUL\n",
-		MDS_OPEN_CREATED);
+		 MDS_OPEN_CREATED);
 	LASSERTF(MDS_OPEN_CROSS == 000000000020UL, "found 0%.11oUL\n",
-		MDS_OPEN_CROSS);
+		 MDS_OPEN_CROSS);
 	LASSERTF(MDS_OPEN_CREAT == 000000000100UL, "found 0%.11oUL\n",
-		MDS_OPEN_CREAT);
+		 MDS_OPEN_CREAT);
 	LASSERTF(MDS_OPEN_EXCL == 000000000200UL, "found 0%.11oUL\n",
-		MDS_OPEN_EXCL);
+		 MDS_OPEN_EXCL);
 	LASSERTF(MDS_OPEN_TRUNC == 000000001000UL, "found 0%.11oUL\n",
-		MDS_OPEN_TRUNC);
+		 MDS_OPEN_TRUNC);
 	LASSERTF(MDS_OPEN_APPEND == 000000002000UL, "found 0%.11oUL\n",
-		MDS_OPEN_APPEND);
+		 MDS_OPEN_APPEND);
 	LASSERTF(MDS_OPEN_SYNC == 000000010000UL, "found 0%.11oUL\n",
-		MDS_OPEN_SYNC);
+		 MDS_OPEN_SYNC);
 	LASSERTF(MDS_OPEN_DIRECTORY == 000000200000UL, "found 0%.11oUL\n",
-		MDS_OPEN_DIRECTORY);
+		 MDS_OPEN_DIRECTORY);
 	LASSERTF(MDS_OPEN_BY_FID == 000040000000UL, "found 0%.11oUL\n",
-		MDS_OPEN_BY_FID);
+		 MDS_OPEN_BY_FID);
 	LASSERTF(MDS_OPEN_DELAY_CREATE == 000100000000UL, "found 0%.11oUL\n",
-		MDS_OPEN_DELAY_CREATE);
+		 MDS_OPEN_DELAY_CREATE);
 	LASSERTF(MDS_OPEN_OWNEROVERRIDE == 000200000000UL, "found 0%.11oUL\n",
-		MDS_OPEN_OWNEROVERRIDE);
+		 MDS_OPEN_OWNEROVERRIDE);
 	LASSERTF(MDS_OPEN_JOIN_FILE == 000400000000UL, "found 0%.11oUL\n",
-		MDS_OPEN_JOIN_FILE);
+		 MDS_OPEN_JOIN_FILE);
 	LASSERTF(MDS_OPEN_LOCK == 004000000000UL, "found 0%.11oUL\n",
-		MDS_OPEN_LOCK);
+		 MDS_OPEN_LOCK);
 	LASSERTF(MDS_OPEN_HAS_EA == 010000000000UL, "found 0%.11oUL\n",
-		MDS_OPEN_HAS_EA);
+		 MDS_OPEN_HAS_EA);
 	LASSERTF(MDS_OPEN_HAS_OBJS == 020000000000UL, "found 0%.11oUL\n",
-		MDS_OPEN_HAS_OBJS);
+		 MDS_OPEN_HAS_OBJS);
 	LASSERTF(MDS_OPEN_NORESTORE == 00000000000100000000000ULL, "found 0%.22lloULL\n",
-			(long long)MDS_OPEN_NORESTORE);
+		 (long long)MDS_OPEN_NORESTORE);
 	LASSERTF(MDS_OPEN_NEWSTRIPE == 00000000000200000000000ULL, "found 0%.22lloULL\n",
-			(long long)MDS_OPEN_NEWSTRIPE);
+		 (long long)MDS_OPEN_NEWSTRIPE);
 	LASSERTF(MDS_OPEN_VOLATILE == 00000000000400000000000ULL, "found 0%.22lloULL\n",
-			(long long)MDS_OPEN_VOLATILE);
+		 (long long)MDS_OPEN_VOLATILE);
 	LASSERTF(LUSTRE_SYNC_FL == 0x00000008, "found 0x%.8x\n",
-		LUSTRE_SYNC_FL);
+		 LUSTRE_SYNC_FL);
 	LASSERTF(LUSTRE_IMMUTABLE_FL == 0x00000010, "found 0x%.8x\n",
-		LUSTRE_IMMUTABLE_FL);
+		 LUSTRE_IMMUTABLE_FL);
 	LASSERTF(LUSTRE_APPEND_FL == 0x00000020, "found 0x%.8x\n",
-		LUSTRE_APPEND_FL);
+		 LUSTRE_APPEND_FL);
 	LASSERTF(LUSTRE_NOATIME_FL == 0x00000080, "found 0x%.8x\n",
-		LUSTRE_NOATIME_FL);
+		 LUSTRE_NOATIME_FL);
 	LASSERTF(LUSTRE_DIRSYNC_FL == 0x00010000, "found 0x%.8x\n",
-		LUSTRE_DIRSYNC_FL);
+		 LUSTRE_DIRSYNC_FL);
 	LASSERTF(MDS_INODELOCK_LOOKUP == 0x000001, "found 0x%.8x\n",
-		MDS_INODELOCK_LOOKUP);
+		 MDS_INODELOCK_LOOKUP);
 	LASSERTF(MDS_INODELOCK_UPDATE == 0x000002, "found 0x%.8x\n",
-		MDS_INODELOCK_UPDATE);
+		 MDS_INODELOCK_UPDATE);
 	LASSERTF(MDS_INODELOCK_OPEN == 0x000004, "found 0x%.8x\n",
-		MDS_INODELOCK_OPEN);
+		 MDS_INODELOCK_OPEN);
 	LASSERTF(MDS_INODELOCK_LAYOUT == 0x000008, "found 0x%.8x\n",
-		MDS_INODELOCK_LAYOUT);
+		 MDS_INODELOCK_LAYOUT);
 
 	/* Checks for struct mdt_ioepoch */
 	LASSERTF((int)sizeof(struct mdt_ioepoch) == 24, "found %lld\n",
@@ -2617,35 +2674,6 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct lmv_desc *)0)->ld_uuid) == 40, "found %lld\n",
 		 (long long)(int)sizeof(((struct lmv_desc *)0)->ld_uuid));
 
-	/* Checks for struct lmv_stripe_md */
-	LASSERTF((int)sizeof(struct lmv_stripe_md) == 32, "found %lld\n",
-		 (long long)(int)sizeof(struct lmv_stripe_md));
-	LASSERTF((int)offsetof(struct lmv_stripe_md, mea_magic) == 0, "found %lld\n",
-		 (long long)(int)offsetof(struct lmv_stripe_md, mea_magic));
-	LASSERTF((int)sizeof(((struct lmv_stripe_md *)0)->mea_magic) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct lmv_stripe_md *)0)->mea_magic));
-	LASSERTF((int)offsetof(struct lmv_stripe_md, mea_count) == 4, "found %lld\n",
-		 (long long)(int)offsetof(struct lmv_stripe_md, mea_count));
-	LASSERTF((int)sizeof(((struct lmv_stripe_md *)0)->mea_count) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct lmv_stripe_md *)0)->mea_count));
-	LASSERTF((int)offsetof(struct lmv_stripe_md, mea_master) == 8, "found %lld\n",
-		 (long long)(int)offsetof(struct lmv_stripe_md, mea_master));
-	LASSERTF((int)sizeof(((struct lmv_stripe_md *)0)->mea_master) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct lmv_stripe_md *)0)->mea_master));
-	LASSERTF((int)offsetof(struct lmv_stripe_md, mea_padding) == 12, "found %lld\n",
-		 (long long)(int)offsetof(struct lmv_stripe_md, mea_padding));
-	LASSERTF((int)sizeof(((struct lmv_stripe_md *)0)->mea_padding) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct lmv_stripe_md *)0)->mea_padding));
-	CLASSERT(LOV_MAXPOOLNAME == 16);
-	LASSERTF((int)offsetof(struct lmv_stripe_md, mea_pool_name[16]) == 32, "found %lld\n",
-		 (long long)(int)offsetof(struct lmv_stripe_md, mea_pool_name[16]));
-	LASSERTF((int)sizeof(((struct lmv_stripe_md *)0)->mea_pool_name[16]) == 1, "found %lld\n",
-		 (long long)(int)sizeof(((struct lmv_stripe_md *)0)->mea_pool_name[16]));
-	LASSERTF((int)offsetof(struct lmv_stripe_md, mea_ids[0]) == 32, "found %lld\n",
-		 (long long)(int)offsetof(struct lmv_stripe_md, mea_ids[0]));
-	LASSERTF((int)sizeof(((struct lmv_stripe_md *)0)->mea_ids[0]) == 16, "found %lld\n",
-		 (long long)(int)sizeof(((struct lmv_stripe_md *)0)->mea_ids[0]));
-
 	/* Checks for struct lov_desc */
 	LASSERTF((int)sizeof(struct lov_desc) == 88, "found %lld\n",
 		 (long long)(int)sizeof(struct lov_desc));
@@ -3195,10 +3223,10 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct llog_setattr64_rec, lsr_gid_h));
 	LASSERTF((int)sizeof(((struct llog_setattr64_rec *)0)->lsr_gid_h) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct llog_setattr64_rec *)0)->lsr_gid_h));
-	LASSERTF((int)offsetof(struct llog_setattr64_rec, lsr_padding) == 48, "found %lld\n",
-		 (long long)(int)offsetof(struct llog_setattr64_rec, lsr_padding));
-	LASSERTF((int)sizeof(((struct llog_setattr64_rec *)0)->lsr_padding) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct llog_setattr64_rec *)0)->lsr_padding));
+	LASSERTF((int)offsetof(struct llog_setattr64_rec, lsr_valid) == 48, "found %lld\n",
+		 (long long)(int)offsetof(struct llog_setattr64_rec, lsr_valid));
+	LASSERTF((int)sizeof(((struct llog_setattr64_rec *)0)->lsr_valid) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct llog_setattr64_rec *)0)->lsr_valid));
 	LASSERTF((int)offsetof(struct llog_setattr64_rec, lsr_tail) == 56, "found %lld\n",
 		 (long long)(int)offsetof(struct llog_setattr64_rec, lsr_tail));
 	LASSERTF((int)sizeof(((struct llog_setattr64_rec *)0)->lsr_tail) == 8, "found %lld\n",
@@ -3272,50 +3300,6 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct changelog_rec *)0)->cr_pfid) == 16, "found %lld\n",
 		 (long long)(int)sizeof(((struct changelog_rec *)0)->cr_pfid));
 
-	/* Checks for struct changelog_ext_rec */
-	LASSERTF((int)sizeof(struct changelog_ext_rec) == 96, "found %lld\n",
-		 (long long)(int)sizeof(struct changelog_ext_rec));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_namelen) == 0, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_namelen));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_namelen) == 2, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_namelen));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_flags) == 2, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_flags));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_flags) == 2, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_flags));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_type) == 4, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_type));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_type) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_type));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_index) == 8, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_index));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_index) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_index));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_prev) == 16, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_prev));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_prev) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_prev));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_time) == 24, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_time));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_time) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_time));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_tfid) == 32, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_tfid));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_tfid) == 16, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_tfid));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_pfid) == 48, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_pfid));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_pfid) == 16, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_pfid));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_sfid) == 64, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_sfid));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_sfid) == 16, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_sfid));
-	LASSERTF((int)offsetof(struct changelog_ext_rec, cr_spfid) == 80, "found %lld\n",
-		 (long long)(int)offsetof(struct changelog_ext_rec, cr_spfid));
-	LASSERTF((int)sizeof(((struct changelog_ext_rec *)0)->cr_spfid) == 16, "found %lld\n",
-		 (long long)(int)sizeof(((struct changelog_ext_rec *)0)->cr_spfid));
-
 	/* Checks for struct changelog_setinfo */
 	LASSERTF((int)sizeof(struct changelog_setinfo) == 12, "found %lld\n",
 		 (long long)(int)sizeof(struct changelog_setinfo));
@@ -3339,10 +3323,10 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct llog_changelog_rec, cr));
 	LASSERTF((int)sizeof(((struct llog_changelog_rec *)0)->cr) == 64, "found %lld\n",
 		 (long long)(int)sizeof(((struct llog_changelog_rec *)0)->cr));
-	LASSERTF((int)offsetof(struct llog_changelog_rec, cr_tail) == 80, "found %lld\n",
-		 (long long)(int)offsetof(struct llog_changelog_rec, cr_tail));
-	LASSERTF((int)sizeof(((struct llog_changelog_rec *)0)->cr_tail) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct llog_changelog_rec *)0)->cr_tail));
+	LASSERTF((int)offsetof(struct llog_changelog_rec, cr_do_not_use) == 80, "found %lld\n",
+		 (long long)(int)offsetof(struct llog_changelog_rec, cr_do_not_use));
+	LASSERTF((int)sizeof(((struct llog_changelog_rec *)0)->cr_do_not_use) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct llog_changelog_rec *)0)->cr_do_not_use));
 
 	/* Checks for struct llog_changelog_user_rec */
 	LASSERTF((int)sizeof(struct llog_changelog_user_rec) == 40, "found %lld\n",
@@ -3506,6 +3490,19 @@ void lustre_assert_wire_constants(void)
 	CLASSERT(LLOG_ORIGIN_HANDLE_DESTROY == 509);
 	CLASSERT(LLOG_FIRST_OPC == 501);
 	CLASSERT(LLOG_LAST_OPC == 510);
+	CLASSERT(LLOG_CONFIG_ORIG_CTXT == 0);
+	CLASSERT(LLOG_CONFIG_REPL_CTXT == 1);
+	CLASSERT(LLOG_MDS_OST_ORIG_CTXT == 2);
+	CLASSERT(LLOG_MDS_OST_REPL_CTXT == 3);
+	CLASSERT(LLOG_SIZE_ORIG_CTXT == 4);
+	CLASSERT(LLOG_SIZE_REPL_CTXT == 5);
+	CLASSERT(LLOG_TEST_ORIG_CTXT == 8);
+	CLASSERT(LLOG_TEST_REPL_CTXT == 9);
+	CLASSERT(LLOG_CHANGELOG_ORIG_CTXT == 12);
+	CLASSERT(LLOG_CHANGELOG_REPL_CTXT == 13);
+	CLASSERT(LLOG_CHANGELOG_USER_ORIG_CTXT == 14);
+	CLASSERT(LLOG_AGENT_ORIG_CTXT == 15);
+	CLASSERT(LLOG_MAX_CTXTS == 16);
 
 	/* Checks for struct llogd_conn_body */
 	LASSERTF((int)sizeof(struct llogd_conn_body) == 40, "found %lld\n",
@@ -3745,32 +3742,28 @@ void lustre_assert_wire_constants(void)
 	CLASSERT(FIEMAP_EXTENT_NET == 0x80000000);
 
 	/* Checks for type posix_acl_xattr_entry */
-	LASSERTF((int)sizeof(posix_acl_xattr_entry) == 8, "found %lld\n",
-		 (long long)(int)sizeof(posix_acl_xattr_entry));
-	LASSERTF((int)offsetof(posix_acl_xattr_entry, e_tag) == 0, "found %lld\n",
-		 (long long)(int)offsetof(posix_acl_xattr_entry, e_tag));
-	LASSERTF((int)sizeof(((posix_acl_xattr_entry *)0)->e_tag) == 2, "found %lld\n",
-		 (long long)(int)sizeof(((posix_acl_xattr_entry *)0)->e_tag));
-	LASSERTF((int)offsetof(posix_acl_xattr_entry, e_perm) == 2, "found %lld\n",
-		 (long long)(int)offsetof(posix_acl_xattr_entry, e_perm));
-	LASSERTF((int)sizeof(((posix_acl_xattr_entry *)0)->e_perm) == 2, "found %lld\n",
-		 (long long)(int)sizeof(((posix_acl_xattr_entry *)0)->e_perm));
-	LASSERTF((int)offsetof(posix_acl_xattr_entry, e_id) == 4, "found %lld\n",
-		 (long long)(int)offsetof(posix_acl_xattr_entry, e_id));
-	LASSERTF((int)sizeof(((posix_acl_xattr_entry *)0)->e_id) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((posix_acl_xattr_entry *)0)->e_id));
+	LASSERTF((int)sizeof(struct posix_acl_xattr_entry) == 8, "found %lld\n",
+		 (long long)(int)sizeof(struct posix_acl_xattr_entry));
+	LASSERTF((int)offsetof(struct posix_acl_xattr_entry, e_tag) == 0, "found %lld\n",
+		 (long long)(int)offsetof(struct posix_acl_xattr_entry, e_tag));
+	LASSERTF((int)sizeof(((struct posix_acl_xattr_entry *)0)->e_tag) == 2, "found %lld\n",
+		 (long long)(int)sizeof(((struct posix_acl_xattr_entry *)0)->e_tag));
+	LASSERTF((int)offsetof(struct posix_acl_xattr_entry, e_perm) == 2, "found %lld\n",
+		 (long long)(int)offsetof(struct posix_acl_xattr_entry, e_perm));
+	LASSERTF((int)sizeof(((struct posix_acl_xattr_entry *)0)->e_perm) == 2, "found %lld\n",
+		 (long long)(int)sizeof(((struct posix_acl_xattr_entry *)0)->e_perm));
+	LASSERTF((int)offsetof(struct posix_acl_xattr_entry, e_id) == 4, "found %lld\n",
+		 (long long)(int)offsetof(struct posix_acl_xattr_entry, e_id));
+	LASSERTF((int)sizeof(((struct posix_acl_xattr_entry *)0)->e_id) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct posix_acl_xattr_entry *)0)->e_id));
 
 	/* Checks for type posix_acl_xattr_header */
-	LASSERTF((int)sizeof(posix_acl_xattr_header) == 4, "found %lld\n",
-		 (long long)(int)sizeof(posix_acl_xattr_header));
-	LASSERTF((int)offsetof(posix_acl_xattr_header, a_version) == 0, "found %lld\n",
-		 (long long)(int)offsetof(posix_acl_xattr_header, a_version));
-	LASSERTF((int)sizeof(((posix_acl_xattr_header *)0)->a_version) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((posix_acl_xattr_header *)0)->a_version));
-	LASSERTF((int)offsetof(posix_acl_xattr_header, a_entries) == 4, "found %lld\n",
-		 (long long)(int)offsetof(posix_acl_xattr_header, a_entries));
-	LASSERTF((int)sizeof(((posix_acl_xattr_header *)0)->a_entries) == 0, "found %lld\n",
-		 (long long)(int)sizeof(((posix_acl_xattr_header *)0)->a_entries));
+	LASSERTF((int)sizeof(struct posix_acl_xattr_header) == 4, "found %lld\n",
+		 (long long)(int)sizeof(struct posix_acl_xattr_header));
+	LASSERTF((int)offsetof(struct posix_acl_xattr_header, a_version) == 0, "found %lld\n",
+		 (long long)(int)offsetof(struct posix_acl_xattr_header, a_version));
+	LASSERTF((int)sizeof(((struct posix_acl_xattr_header *)0)->a_version) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct posix_acl_xattr_header *)0)->a_version));
 
 	/* Checks for struct link_ea_header */
 	LASSERTF((int)sizeof(struct link_ea_header) == 24, "found %lld\n",
@@ -3943,9 +3936,9 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct hsm_progress *)0)->padding) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct hsm_progress *)0)->padding));
 	LASSERTF(HP_FLAG_COMPLETED == 0x01, "found 0x%.8x\n",
-		HP_FLAG_COMPLETED);
+		 HP_FLAG_COMPLETED);
 	LASSERTF(HP_FLAG_RETRY == 0x02, "found 0x%.8x\n",
-		HP_FLAG_RETRY);
+		 HP_FLAG_RETRY);
 
 	LASSERTF((int)offsetof(struct hsm_copy, hc_data_version) == 0, "found %lld\n",
 		 (long long)(int)offsetof(struct hsm_copy, hc_data_version));
@@ -4100,9 +4093,9 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct hsm_request *)0)->hr_data_len) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct hsm_request *)0)->hr_data_len));
 	LASSERTF(HSM_FORCE_ACTION == 0x00000001UL, "found 0x%.8xUL\n",
-		(unsigned)HSM_FORCE_ACTION);
+		 (unsigned)HSM_FORCE_ACTION);
 	LASSERTF(HSM_GHOST_COPY == 0x00000002UL, "found 0x%.8xUL\n",
-		(unsigned)HSM_GHOST_COPY);
+		 (unsigned)HSM_GHOST_COPY);
 
 	/* Checks for struct hsm_user_request */
 	LASSERTF((int)sizeof(struct hsm_user_request) == 24, "found %lld\n",
diff --git a/drivers/staging/media/Kconfig b/drivers/staging/media/Kconfig
index 7292f23954df..6620d96ee44d 100644
--- a/drivers/staging/media/Kconfig
+++ b/drivers/staging/media/Kconfig
@@ -31,11 +31,11 @@ source "drivers/staging/media/omap4iss/Kconfig"
 
 source "drivers/staging/media/pulse8-cec/Kconfig"
 
-source "drivers/staging/media/tw686x-kh/Kconfig"
-
 source "drivers/staging/media/s5p-cec/Kconfig"
 
 # Keep LIRC at the end, as it has sub-menus
 source "drivers/staging/media/lirc/Kconfig"
 
+source "drivers/staging/media/st-cec/Kconfig"
+
 endif
diff --git a/drivers/staging/media/Makefile b/drivers/staging/media/Makefile
index 87ce8ad1e22a..906257e94dda 100644
--- a/drivers/staging/media/Makefile
+++ b/drivers/staging/media/Makefile
@@ -6,4 +6,4 @@ obj-$(CONFIG_LIRC_STAGING)	+= lirc/
 obj-$(CONFIG_VIDEO_DM365_VPFE)	+= davinci_vpfe/
 obj-$(CONFIG_VIDEO_OMAP4)	+= omap4iss/
 obj-$(CONFIG_USB_PULSE8_CEC)    += pulse8-cec/
-obj-$(CONFIG_VIDEO_TW686X_KH)	+= tw686x-kh/
+obj-$(CONFIG_VIDEO_STI_HDMI_CEC) += st-cec/
diff --git a/drivers/staging/media/bcm2048/radio-bcm2048.c b/drivers/staging/media/bcm2048/radio-bcm2048.c
index 8dade197f053..ea15cc638097 100644
--- a/drivers/staging/media/bcm2048/radio-bcm2048.c
+++ b/drivers/staging/media/bcm2048/radio-bcm2048.c
@@ -483,10 +483,8 @@ static int bcm2048_set_rds_no_lock(struct bcm2048_device *bdev, u8 rds_on)
 		memset(&bdev->rds_info, 0, sizeof(bdev->rds_info));
 	}
 
-	err = bcm2048_send_command(bdev, BCM2048_I2C_FM_RDS_SYSTEM,
-				   bdev->cache_fm_rds_system);
-
-	return err;
+	return bcm2048_send_command(bdev, BCM2048_I2C_FM_RDS_SYSTEM,
+				    bdev->cache_fm_rds_system);
 }
 
 static int bcm2048_get_rds_no_lock(struct bcm2048_device *bdev)
@@ -1834,9 +1832,7 @@ static int bcm2048_deinit(struct bcm2048_device *bdev)
 	if (err < 0)
 		return err;
 
-	err = bcm2048_set_power_state(bdev, BCM2048_POWER_OFF);
-
-	return err;
+	return bcm2048_set_power_state(bdev, BCM2048_POWER_OFF);
 }
 
 /*
@@ -1995,9 +1991,7 @@ static ssize_t bcm2048_##prop##_read(struct device *dev,		\
 									\
 	value = bcm2048_get_##prop(bdev);				\
 									\
-	value = sprintf(buf, mask "\n", value);				\
-									\
-	return value;							\
+	return sprintf(buf, mask "\n", value);				\
 }
 
 #define DEFINE_SYSFS_PROPERTY(prop, signal, size, mask, check)		\
diff --git a/drivers/staging/media/cec/Kconfig b/drivers/staging/media/cec/Kconfig
index 21457a1f6c9f..6e12d41b1f86 100644
--- a/drivers/staging/media/cec/Kconfig
+++ b/drivers/staging/media/cec/Kconfig
@@ -5,9 +5,6 @@ config MEDIA_CEC
 	---help---
 	  Enable the CEC API.
 
-	  To compile this driver as a module, choose M here: the
-	  module will be called cec.
-
 config MEDIA_CEC_DEBUG
 	bool "CEC debugfs interface (EXPERIMENTAL)"
 	depends on MEDIA_CEC && DEBUG_FS
diff --git a/drivers/staging/media/cec/TODO b/drivers/staging/media/cec/TODO
index a10d4f82b954..13224694a8ae 100644
--- a/drivers/staging/media/cec/TODO
+++ b/drivers/staging/media/cec/TODO
@@ -12,6 +12,7 @@ Hopefully this will happen later in 2016.
 
 Other TODOs:
 
+- There are two possible replies to CEC_MSG_INITIATE_ARC. How to handle that?
 - Add a flag to inhibit passing CEC RC messages to the rc subsystem.
   Applications should be able to choose this when calling S_LOG_ADDRS.
 - If the reply field of cec_msg is set then when the reply arrives it
diff --git a/drivers/staging/media/cec/cec-adap.c b/drivers/staging/media/cec/cec-adap.c
index b2393bbacb26..611e07b78bfe 100644
--- a/drivers/staging/media/cec/cec-adap.c
+++ b/drivers/staging/media/cec/cec-adap.c
@@ -124,10 +124,10 @@ static void cec_queue_event(struct cec_adapter *adap,
 	u64 ts = ktime_get_ns();
 	struct cec_fh *fh;
 
-	mutex_lock(&adap->devnode.fhs_lock);
+	mutex_lock(&adap->devnode.lock);
 	list_for_each_entry(fh, &adap->devnode.fhs, list)
 		cec_queue_event_fh(fh, ev, ts);
-	mutex_unlock(&adap->devnode.fhs_lock);
+	mutex_unlock(&adap->devnode.lock);
 }
 
 /*
@@ -191,12 +191,12 @@ static void cec_queue_msg_monitor(struct cec_adapter *adap,
 	u32 monitor_mode = valid_la ? CEC_MODE_MONITOR :
 				      CEC_MODE_MONITOR_ALL;
 
-	mutex_lock(&adap->devnode.fhs_lock);
+	mutex_lock(&adap->devnode.lock);
 	list_for_each_entry(fh, &adap->devnode.fhs, list) {
 		if (fh->mode_follower >= monitor_mode)
 			cec_queue_msg_fh(fh, msg);
 	}
-	mutex_unlock(&adap->devnode.fhs_lock);
+	mutex_unlock(&adap->devnode.lock);
 }
 
 /*
@@ -207,12 +207,12 @@ static void cec_queue_msg_followers(struct cec_adapter *adap,
 {
 	struct cec_fh *fh;
 
-	mutex_lock(&adap->devnode.fhs_lock);
+	mutex_lock(&adap->devnode.lock);
 	list_for_each_entry(fh, &adap->devnode.fhs, list) {
 		if (fh->mode_follower == CEC_MODE_FOLLOWER)
 			cec_queue_msg_fh(fh, msg);
 	}
-	mutex_unlock(&adap->devnode.fhs_lock);
+	mutex_unlock(&adap->devnode.lock);
 }
 
 /* Notify userspace of an adapter state change. */
@@ -851,6 +851,9 @@ void cec_received_msg(struct cec_adapter *adap, struct cec_msg *msg)
 	if (!valid_la || msg->len <= 1)
 		return;
 
+	if (adap->log_addrs.log_addr_mask == 0)
+		return;
+
 	/*
 	 * Process the message on the protocol level. If is_reply is true,
 	 * then cec_receive_notify() won't pass on the reply to the listener(s)
@@ -1047,11 +1050,17 @@ static int cec_config_thread_func(void *arg)
 			dprintk(1, "could not claim LA %d\n", i);
 	}
 
+	if (adap->log_addrs.log_addr_mask == 0 &&
+	    !(las->flags & CEC_LOG_ADDRS_FL_ALLOW_UNREG_FALLBACK))
+		goto unconfigure;
+
 configured:
 	if (adap->log_addrs.log_addr_mask == 0) {
 		/* Fall back to unregistered */
 		las->log_addr[0] = CEC_LOG_ADDR_UNREGISTERED;
 		las->log_addr_mask = 1 << las->log_addr[0];
+		for (i = 1; i < las->num_log_addrs; i++)
+			las->log_addr[i] = CEC_LOG_ADDR_INVALID;
 	}
 	adap->is_configured = true;
 	adap->is_configuring = false;
@@ -1070,6 +1079,8 @@ configured:
 			cec_report_features(adap, i);
 		cec_report_phys_addr(adap, i);
 	}
+	for (i = las->num_log_addrs; i < CEC_MAX_LOG_ADDRS; i++)
+		las->log_addr[i] = CEC_LOG_ADDR_INVALID;
 	mutex_lock(&adap->lock);
 	adap->kthread_config = NULL;
 	mutex_unlock(&adap->lock);
@@ -1153,8 +1164,6 @@ void cec_s_phys_addr(struct cec_adapter *adap, u16 phys_addr, bool block)
 	if (IS_ERR_OR_NULL(adap))
 		return;
 
-	if (WARN_ON(adap->capabilities & CEC_CAP_PHYS_ADDR))
-		return;
 	mutex_lock(&adap->lock);
 	__cec_s_phys_addr(adap, phys_addr, block);
 	mutex_unlock(&adap->lock);
@@ -1295,8 +1304,6 @@ int cec_s_log_addrs(struct cec_adapter *adap,
 {
 	int err;
 
-	if (WARN_ON(adap->capabilities & CEC_CAP_LOG_ADDRS))
-		return -EINVAL;
 	mutex_lock(&adap->lock);
 	err = __cec_s_log_addrs(adap, log_addrs, block);
 	mutex_unlock(&adap->lock);
@@ -1398,7 +1405,6 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 	u8 init_laddr = cec_msg_initiator(msg);
 	u8 devtype = cec_log_addr2dev(adap, dest_laddr);
 	int la_idx = cec_log_addr2idx(adap, dest_laddr);
-	bool is_directed = la_idx >= 0;
 	bool from_unregistered = init_laddr == 0xf;
 	struct cec_msg tx_cec_msg = { };
 
@@ -1560,7 +1566,7 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 		 * Unprocessed messages are aborted if userspace isn't doing
 		 * any processing either.
 		 */
-		if (is_directed && !is_reply && !adap->follower_cnt &&
+		if (!is_broadcast && !is_reply && !adap->follower_cnt &&
 		    !adap->cec_follower && msg->msg[1] != CEC_MSG_FEATURE_ABORT)
 			return cec_feature_abort(adap, msg);
 		break;
diff --git a/drivers/staging/media/cec/cec-api.c b/drivers/staging/media/cec/cec-api.c
index 7be7615a0fdf..e274e2f22398 100644
--- a/drivers/staging/media/cec/cec-api.c
+++ b/drivers/staging/media/cec/cec-api.c
@@ -162,7 +162,7 @@ static long cec_adap_s_log_addrs(struct cec_adapter *adap, struct cec_fh *fh,
 		return -ENOTTY;
 	if (copy_from_user(&log_addrs, parg, sizeof(log_addrs)))
 		return -EFAULT;
-	log_addrs.flags = 0;
+	log_addrs.flags &= CEC_LOG_ADDRS_FL_ALLOW_UNREG_FALLBACK;
 	mutex_lock(&adap->lock);
 	if (!adap->is_configuring &&
 	    (!log_addrs.num_log_addrs || !adap->is_configured) &&
@@ -435,7 +435,7 @@ static long cec_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	void __user *parg = (void __user *)arg;
 
 	if (!devnode->registered)
-		return -EIO;
+		return -ENODEV;
 
 	switch (cmd) {
 	case CEC_ADAP_G_CAPS:
@@ -508,14 +508,14 @@ static int cec_open(struct inode *inode, struct file *filp)
 
 	filp->private_data = fh;
 
-	mutex_lock(&devnode->fhs_lock);
+	mutex_lock(&devnode->lock);
 	/* Queue up initial state events */
 	ev_state.state_change.phys_addr = adap->phys_addr;
 	ev_state.state_change.log_addr_mask = adap->log_addrs.log_addr_mask;
 	cec_queue_event_fh(fh, &ev_state, 0);
 
 	list_add(&fh->list, &devnode->fhs);
-	mutex_unlock(&devnode->fhs_lock);
+	mutex_unlock(&devnode->lock);
 
 	return 0;
 }
@@ -540,9 +540,9 @@ static int cec_release(struct inode *inode, struct file *filp)
 		cec_monitor_all_cnt_dec(adap);
 	mutex_unlock(&adap->lock);
 
-	mutex_lock(&devnode->fhs_lock);
+	mutex_lock(&devnode->lock);
 	list_del(&fh->list);
-	mutex_unlock(&devnode->fhs_lock);
+	mutex_unlock(&devnode->lock);
 
 	/* Unhook pending transmits from this filehandle. */
 	mutex_lock(&adap->lock);
diff --git a/drivers/staging/media/cec/cec-core.c b/drivers/staging/media/cec/cec-core.c
index 112a5fae12f5..b0137e247dc9 100644
--- a/drivers/staging/media/cec/cec-core.c
+++ b/drivers/staging/media/cec/cec-core.c
@@ -51,31 +51,29 @@ int cec_get_device(struct cec_devnode *devnode)
 {
 	/*
 	 * Check if the cec device is available. This needs to be done with
-	 * the cec_devnode_lock held to prevent an open/unregister race:
+	 * the devnode->lock held to prevent an open/unregister race:
 	 * without the lock, the device could be unregistered and freed between
 	 * the devnode->registered check and get_device() calls, leading to
 	 * a crash.
 	 */
-	mutex_lock(&cec_devnode_lock);
+	mutex_lock(&devnode->lock);
 	/*
 	 * return ENXIO if the cec device has been removed
 	 * already or if it is not registered anymore.
 	 */
 	if (!devnode->registered) {
-		mutex_unlock(&cec_devnode_lock);
+		mutex_unlock(&devnode->lock);
 		return -ENXIO;
 	}
 	/* and increase the device refcount */
 	get_device(&devnode->dev);
-	mutex_unlock(&cec_devnode_lock);
+	mutex_unlock(&devnode->lock);
 	return 0;
 }
 
 void cec_put_device(struct cec_devnode *devnode)
 {
-	mutex_lock(&cec_devnode_lock);
 	put_device(&devnode->dev);
-	mutex_unlock(&cec_devnode_lock);
 }
 
 /* Called when the last user of the cec device exits. */
@@ -84,11 +82,10 @@ static void cec_devnode_release(struct device *cd)
 	struct cec_devnode *devnode = to_cec_devnode(cd);
 
 	mutex_lock(&cec_devnode_lock);
-
 	/* Mark device node number as free */
 	clear_bit(devnode->minor, cec_devnode_nums);
-
 	mutex_unlock(&cec_devnode_lock);
+
 	cec_delete_adapter(to_cec_adapter(devnode));
 }
 
@@ -117,7 +114,7 @@ static int __must_check cec_devnode_register(struct cec_devnode *devnode,
 
 	/* Initialization */
 	INIT_LIST_HEAD(&devnode->fhs);
-	mutex_init(&devnode->fhs_lock);
+	mutex_init(&devnode->lock);
 
 	/* Part 1: Find a free minor number */
 	mutex_lock(&cec_devnode_lock);
@@ -160,7 +157,9 @@ static int __must_check cec_devnode_register(struct cec_devnode *devnode,
 cdev_del:
 	cdev_del(&devnode->cdev);
 clr_bit:
+	mutex_lock(&cec_devnode_lock);
 	clear_bit(devnode->minor, cec_devnode_nums);
+	mutex_unlock(&cec_devnode_lock);
 	return ret;
 }
 
@@ -177,17 +176,21 @@ static void cec_devnode_unregister(struct cec_devnode *devnode)
 {
 	struct cec_fh *fh;
 
+	mutex_lock(&devnode->lock);
+
 	/* Check if devnode was never registered or already unregistered */
-	if (!devnode->registered || devnode->unregistered)
+	if (!devnode->registered || devnode->unregistered) {
+		mutex_unlock(&devnode->lock);
 		return;
+	}
 
-	mutex_lock(&devnode->fhs_lock);
 	list_for_each_entry(fh, &devnode->fhs, list)
 		wake_up_interruptible(&fh->wait);
-	mutex_unlock(&devnode->fhs_lock);
 
 	devnode->registered = false;
 	devnode->unregistered = true;
+	mutex_unlock(&devnode->lock);
+
 	device_del(&devnode->dev);
 	cdev_del(&devnode->cdev);
 	put_device(&devnode->dev);
@@ -354,8 +357,7 @@ void cec_delete_adapter(struct cec_adapter *adap)
 	if (adap->kthread_config)
 		kthread_stop(adap->kthread_config);
 #if IS_REACHABLE(CONFIG_RC_CORE)
-	if (adap->rc)
-		rc_free_device(adap->rc);
+	rc_free_device(adap->rc);
 #endif
 	kfree(adap);
 }
diff --git a/drivers/staging/media/cxd2099/cxd2099.c b/drivers/staging/media/cxd2099/cxd2099.c
index 692ba3e63e14..fedeb3c3549e 100644
--- a/drivers/staging/media/cxd2099/cxd2099.c
+++ b/drivers/staging/media/cxd2099/cxd2099.c
@@ -660,7 +660,7 @@ static int write_data(struct dvb_ca_en50221 *ca, int slot, u8 *ebuf, int ecount)
 	struct cxd *ci = ca->data;
 
 	mutex_lock(&ci->lock);
-	printk(kern_INFO "write_data %d\n", ecount);
+	dev_info(&ci->i2c->dev, "write_data %d\n", ecount);
 	write_reg(ci, 0x0d, ecount>>8);
 	write_reg(ci, 0x0e, ecount&0xff);
 	write_block(ci, 0x11, ebuf, ecount);
diff --git a/drivers/staging/media/davinci_vpfe/davinci_vpfe_user.h b/drivers/staging/media/davinci_vpfe/davinci_vpfe_user.h
index f4f35c9ad1ab..d3f34f9bf712 100644
--- a/drivers/staging/media/davinci_vpfe/davinci_vpfe_user.h
+++ b/drivers/staging/media/davinci_vpfe/davinci_vpfe_user.h
@@ -544,41 +544,41 @@ struct vpfe_isif_raw_config {
 /* IPIPE module configurations */
 
 /* IPIPE input configuration */
-#define VPFE_IPIPE_INPUT_CONFIG		(1 << 0)
+#define VPFE_IPIPE_INPUT_CONFIG		BIT(0)
 /* LUT based Defect Pixel Correction */
-#define VPFE_IPIPE_LUTDPC		(1 << 1)
+#define VPFE_IPIPE_LUTDPC		BIT(1)
 /* On the fly (OTF) Defect Pixel Correction */
-#define VPFE_IPIPE_OTFDPC		(1 << 2)
+#define VPFE_IPIPE_OTFDPC		BIT(2)
 /* Noise Filter - 1 */
-#define VPFE_IPIPE_NF1			(1 << 3)
+#define VPFE_IPIPE_NF1			BIT(3)
 /* Noise Filter - 2 */
-#define VPFE_IPIPE_NF2			(1 << 4)
+#define VPFE_IPIPE_NF2			BIT(4)
 /* White Balance.  Also a control ID */
-#define VPFE_IPIPE_WB			(1 << 5)
+#define VPFE_IPIPE_WB			BIT(5)
 /* 1st RGB to RBG Blend module */
-#define VPFE_IPIPE_RGB2RGB_1		(1 << 6)
+#define VPFE_IPIPE_RGB2RGB_1		BIT(6)
 /* 2nd RGB to RBG Blend module */
-#define VPFE_IPIPE_RGB2RGB_2		(1 << 7)
+#define VPFE_IPIPE_RGB2RGB_2		BIT(7)
 /* Gamma Correction */
-#define VPFE_IPIPE_GAMMA		(1 << 8)
+#define VPFE_IPIPE_GAMMA		BIT(8)
 /* 3D LUT color conversion */
-#define VPFE_IPIPE_3D_LUT		(1 << 9)
+#define VPFE_IPIPE_3D_LUT		BIT(9)
 /* RGB to YCbCr module */
-#define VPFE_IPIPE_RGB2YUV		(1 << 10)
+#define VPFE_IPIPE_RGB2YUV		BIT(10)
 /* YUV 422 conversion module */
-#define VPFE_IPIPE_YUV422_CONV		(1 << 11)
+#define VPFE_IPIPE_YUV422_CONV		BIT(11)
 /* Edge Enhancement */
-#define VPFE_IPIPE_YEE			(1 << 12)
+#define VPFE_IPIPE_YEE			BIT(12)
 /* Green Imbalance Correction */
-#define VPFE_IPIPE_GIC			(1 << 13)
+#define VPFE_IPIPE_GIC			BIT(13)
 /* CFA Interpolation */
-#define VPFE_IPIPE_CFA			(1 << 14)
+#define VPFE_IPIPE_CFA			BIT(14)
 /* Chroma Artifact Reduction */
-#define VPFE_IPIPE_CAR			(1 << 15)
+#define VPFE_IPIPE_CAR			BIT(15)
 /* Chroma Gain Suppression */
-#define VPFE_IPIPE_CGS			(1 << 16)
+#define VPFE_IPIPE_CGS			BIT(16)
 /* Global brightness and contrast control */
-#define VPFE_IPIPE_GBCE			(1 << 17)
+#define VPFE_IPIPE_GBCE			BIT(17)
 
 #define VPFE_IPIPE_MAX_MODULES		18
 
diff --git a/drivers/staging/media/davinci_vpfe/dm365_isif.c b/drivers/staging/media/davinci_vpfe/dm365_isif.c
index ae9202ded59f..569bcdc9ce2f 100644
--- a/drivers/staging/media/davinci_vpfe/dm365_isif.c
+++ b/drivers/staging/media/davinci_vpfe/dm365_isif.c
@@ -146,9 +146,8 @@ enum v4l2_field vpfe_isif_get_fid(struct vpfe_device *vpfe_dev)
 	u32 field_status;
 
 	field_status = isif_read(isif->isif_cfg.base_addr, MODESET);
-	field_status = (field_status >> DM365_ISIF_MDFS_OFFSET) &
-			DM365_ISIF_MDFS_MASK;
-	return field_status;
+	return (field_status >> DM365_ISIF_MDFS_OFFSET) &
+		DM365_ISIF_MDFS_MASK;
 }
 
 static int
@@ -594,8 +593,7 @@ isif_validate_raw_params(struct vpfe_isif_raw_config *params)
 	ret = isif_validate_dfc_params(&params->dfc);
 	if (ret)
 		return ret;
-	ret = isif_validate_bclamp_params(&params->bclamp);
-	return ret;
+	return isif_validate_bclamp_params(&params->bclamp);
 }
 
 static int isif_set_params(struct v4l2_subdev *sd, void *params)
diff --git a/drivers/staging/media/davinci_vpfe/dm365_resizer.c b/drivers/staging/media/davinci_vpfe/dm365_resizer.c
index 3cd56cc132c7..128662623ea8 100644
--- a/drivers/staging/media/davinci_vpfe/dm365_resizer.c
+++ b/drivers/staging/media/davinci_vpfe/dm365_resizer.c
@@ -128,7 +128,7 @@ resizer_configure_passthru(struct vpfe_resizer_device *resizer, int bypass)
 static void
 configure_resizer_out_params(struct vpfe_resizer_device *resizer, int index,
 			     void *output_spec, unsigned char partial,
-			     unsigned flag)
+			     unsigned int flag)
 {
 	struct resizer_params *param = &resizer->config;
 	struct v4l2_mbus_framefmt *outformat;
diff --git a/drivers/staging/media/davinci_vpfe/vpfe_video.c b/drivers/staging/media/davinci_vpfe/vpfe_video.c
index 3319fb8f7d01..8be9f854510f 100644
--- a/drivers/staging/media/davinci_vpfe/vpfe_video.c
+++ b/drivers/staging/media/davinci_vpfe/vpfe_video.c
@@ -37,7 +37,7 @@ static struct media_entity *vpfe_get_input_entity
 	struct media_pad *remote;
 
 	remote = media_entity_remote_pad(&vpfe_dev->vpfe_isif.pads[0]);
-	if (remote == NULL) {
+	if (!remote) {
 		pr_err("Invalid media connection to isif/ccdc\n");
 		return NULL;
 	}
@@ -54,7 +54,7 @@ static int vpfe_update_current_ext_subdev(struct vpfe_video_device *video)
 	int i;
 
 	remote = media_entity_remote_pad(&vpfe_dev->vpfe_isif.pads[0]);
-	if (remote == NULL) {
+	if (!remote) {
 		pr_err("Invalid media connection to isif/ccdc\n");
 		return -EINVAL;
 	}
@@ -107,7 +107,7 @@ __vpfe_video_get_format(struct vpfe_video_device *video,
 	int ret;
 
 	subdev = vpfe_video_remote_subdev(video, &pad);
-	if (subdev == NULL)
+	if (!subdev)
 		return -EINVAL;
 
 	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
@@ -236,7 +236,7 @@ static int vpfe_video_validate_pipeline(struct vpfe_pipeline *pipe)
 	 * format of the connected pad.
 	 */
 	subdev = vpfe_video_remote_subdev(pipe->outputs[0], NULL);
-	if (subdev == NULL)
+	if (!subdev)
 		return -EPIPE;
 
 	while (1) {
@@ -413,7 +413,7 @@ static int vpfe_open(struct file *file)
 	/* Allocate memory for the file handle object */
 	handle = kzalloc(sizeof(struct vpfe_fh), GFP_KERNEL);
 
-	if (handle == NULL)
+	if (!handle)
 		return -ENOMEM;
 
 	v4l2_fh_init(&handle->vfh, &video->video_dev);
@@ -683,14 +683,14 @@ static int vpfe_enum_fmt(struct file *file, void  *priv,
 	}
 	/* get the remote pad */
 	remote = media_entity_remote_pad(&video->pad);
-	if (remote == NULL) {
+	if (!remote) {
 		v4l2_err(&vpfe_dev->v4l2_dev,
 			 "invalid remote pad for video node\n");
 		return -EINVAL;
 	}
 	/* get the remote subdev */
 	subdev = vpfe_video_remote_subdev(video, NULL);
-	if (subdev == NULL) {
+	if (!subdev) {
 		v4l2_err(&vpfe_dev->v4l2_dev,
 			 "invalid remote subdev for video node\n");
 		return -EINVAL;
diff --git a/drivers/staging/media/lirc/lirc_bt829.c b/drivers/staging/media/lirc/lirc_bt829.c
index 44f565547179..04d881b391c7 100644
--- a/drivers/staging/media/lirc/lirc_bt829.c
+++ b/drivers/staging/media/lirc/lirc_bt829.c
@@ -120,7 +120,7 @@ int init_module(void)
 	int rc;
 
 	pdev = do_pci_probe();
-	if (pdev == NULL)
+	if (!pdev)
 		return -ENODEV;
 
 	rc = pci_enable_device(pdev);
@@ -163,7 +163,6 @@ err_put_dev:
 	return rc;
 }
 
-
 void cleanup_module(void)
 {
 	struct pci_dev *pdev = to_pci_dev(atir_driver.dev);
@@ -174,7 +173,6 @@ void cleanup_module(void)
 	pci_dev_put(pdev);
 }
 
-
 static int atir_init_start(void)
 {
 	pci_addr_lin = ioremap(pci_addr_phys + DATA_PCI_OFF, 0x400);
@@ -187,10 +185,9 @@ static int atir_init_start(void)
 
 static void cycle_delay(int cycle)
 {
-	udelay(WAIT_CYCLE*cycle);
+	udelay(WAIT_CYCLE * cycle);
 }
 
-
 static int poll_main(void)
 {
 	unsigned char status_high, status_low;
diff --git a/drivers/staging/media/lirc/lirc_imon.c b/drivers/staging/media/lirc/lirc_imon.c
index ff1926ca1f96..198a8057f2f1 100644
--- a/drivers/staging/media/lirc/lirc_imon.c
+++ b/drivers/staging/media/lirc/lirc_imon.c
@@ -32,7 +32,6 @@
 #include <media/lirc.h>
 #include <media/lirc_dev.h>
 
-
 #define MOD_AUTHOR	"Venky Raju <dev@venky.ws>"
 #define MOD_DESC	"Driver for SoundGraph iMON MultiMedia IR/Display"
 #define MOD_NAME	"lirc_imon"
@@ -212,7 +211,6 @@ static void deregister_from_lirc(struct imon_context *context)
 	else
 		dev_info(&context->usbdev->dev,
 			 "Deregistered iMON driver (minor:%d)\n", minor);
-
 }
 
 /**
@@ -429,7 +427,7 @@ static ssize_t vfd_write(struct file *file, const char __user *buf,
 
 	do {
 		memcpy(context->usb_tx_buf, context->tx.data_buf + offset, 7);
-		context->usb_tx_buf[7] = (unsigned char) seq;
+		context->usb_tx_buf[7] = (unsigned char)seq;
 
 		retval = send_packet(context);
 		if (retval) {
@@ -447,7 +445,7 @@ static ssize_t vfd_write(struct file *file, const char __user *buf,
 	if (context->vfd_proto_6p) {
 		/* Send packet #6 */
 		memcpy(context->usb_tx_buf, &vfd_packet6, sizeof(vfd_packet6));
-		context->usb_tx_buf[7] = (unsigned char) seq;
+		context->usb_tx_buf[7] = (unsigned char)seq;
 		retval = send_packet(context);
 		if (retval)
 			dev_err(&context->usbdev->dev,
@@ -563,7 +561,7 @@ static void submit_data(struct imon_context *context)
 		value |= PULSE_BIT;
 
 	for (i = 0; i < 4; ++i)
-		buf[i] = value>>(i*8);
+		buf[i] = value >> (i * 8);
 
 	lirc_buffer_write(context->driver->rbuf, buf);
 	wake_up(&context->driver->rbuf->wait_poll);
@@ -589,7 +587,7 @@ static void imon_incoming_packet(struct imon_context *context,
 
 	if (len != 8) {
 		dev_warn(dev, "imon %s: invalid incoming packet size (len = %d, intf%d)\n",
-			__func__, len, intf);
+			 __func__, len, intf);
 		return;
 	}
 
@@ -704,7 +702,7 @@ static int imon_probe(struct usb_interface *interface,
 	/* prevent races probing devices w/multiple interfaces */
 	mutex_lock(&driver_lock);
 
-	context = kzalloc(sizeof(struct imon_context), GFP_KERNEL);
+	context = kzalloc(sizeof(*context), GFP_KERNEL);
 	if (!context)
 		goto driver_unlock;
 
@@ -784,11 +782,11 @@ static int imon_probe(struct usb_interface *interface,
 			__func__, vfd_proto_6p);
 	}
 
-	driver = kzalloc(sizeof(struct lirc_driver), GFP_KERNEL);
+	driver = kzalloc(sizeof(*driver), GFP_KERNEL);
 	if (!driver)
 		goto free_context;
 
-	rbuf = kmalloc(sizeof(struct lirc_buffer), GFP_KERNEL);
+	rbuf = kmalloc(sizeof(*rbuf), GFP_KERNEL);
 	if (!rbuf)
 		goto free_driver;
 
@@ -797,16 +795,11 @@ static int imon_probe(struct usb_interface *interface,
 		goto free_rbuf;
 	}
 	rx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!rx_urb) {
-		dev_err(dev, "%s: usb_alloc_urb failed for IR urb\n", __func__);
+	if (!rx_urb)
 		goto free_lirc_buf;
-	}
 	tx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!tx_urb) {
-		dev_err(dev, "%s: usb_alloc_urb failed for display urb\n",
-		    __func__);
+	if (!tx_urb)
 		goto free_rx_urb;
-	}
 
 	mutex_init(&context->ctx_lock);
 	context->vfd_proto_6p = vfd_proto_6p;
@@ -835,7 +828,7 @@ static int imon_probe(struct usb_interface *interface,
 	}
 
 	dev_info(dev, "Registered iMON driver (lirc minor: %d)\n",
-			lirc_minor);
+		 lirc_minor);
 
 	/* Needed while unregistering! */
 	driver->minor = lirc_minor;
@@ -856,8 +849,8 @@ static int imon_probe(struct usb_interface *interface,
 		context->display = 1;
 
 	usb_fill_int_urb(context->rx_urb, context->usbdev,
-		usb_rcvintpipe(context->usbdev,
-			context->rx_endpoint->bEndpointAddress),
+			 usb_rcvintpipe(context->usbdev,
+			 context->rx_endpoint->bEndpointAddress),
 		context->usb_rx_buf, sizeof(context->usb_rx_buf),
 		usb_rx_callback, context,
 		context->rx_endpoint->bInterval);
@@ -882,7 +875,7 @@ static int imon_probe(struct usb_interface *interface,
 	}
 
 	dev_info(dev, "iMON device (%04x:%04x, intf%d) on usb<%d:%d> initialized\n",
-		vendor, product, ifnum, usbdev->bus->busnum, usbdev->devnum);
+		 vendor, product, ifnum, usbdev->bus->busnum, usbdev->devnum);
 
 	/* Everything went fine. Just unlock and return retval (with is 0) */
 	mutex_unlock(&context->ctx_lock);
@@ -973,8 +966,8 @@ static int imon_resume(struct usb_interface *intf)
 	struct imon_context *context = usb_get_intfdata(intf);
 
 	usb_fill_int_urb(context->rx_urb, context->usbdev,
-		usb_rcvintpipe(context->usbdev,
-			context->rx_endpoint->bEndpointAddress),
+			 usb_rcvintpipe(context->usbdev,
+			 context->rx_endpoint->bEndpointAddress),
 		context->usb_rx_buf, sizeof(context->usb_rx_buf),
 		usb_rx_callback, context,
 		context->rx_endpoint->bInterval);
diff --git a/drivers/staging/media/lirc/lirc_parallel.c b/drivers/staging/media/lirc/lirc_parallel.c
index 3906ac6e686d..bfb76a45bfbf 100644
--- a/drivers/staging/media/lirc/lirc_parallel.c
+++ b/drivers/staging/media/lirc/lirc_parallel.c
@@ -163,12 +163,12 @@ static unsigned int init_lirc_timer(void)
 	if (count >= 1000 && timeelapsed > 0) {
 		if (default_timer == 0) {
 			/* autodetect timer */
-			newtimer = (1000000*count)/timeelapsed;
+			newtimer = (1000000 * count) / timeelapsed;
 			pr_info("%u Hz timer detected\n", newtimer);
 			return newtimer;
 		}
-		newtimer = (1000000*count)/timeelapsed;
-		if (abs(newtimer - default_timer) > default_timer/10) {
+		newtimer = (1000000 * count) / timeelapsed;
+		if (abs(newtimer - default_timer) > default_timer / 10) {
 			/* bad timer */
 			pr_notice("bad timer: %u Hz\n", newtimer);
 			pr_notice("using default timer: %u Hz\n",
@@ -650,7 +650,7 @@ static int __init lirc_parallel_init(void)
 	if (!pport) {
 		pr_notice("no port at %x found\n", io);
 		result = -ENXIO;
-		goto exit_device_put;
+		goto exit_device_del;
 	}
 	ppdevice = parport_register_device(pport, LIRC_DRIVER_NAME,
 					   pf, kf, lirc_lirc_irq_handler, 0,
@@ -659,7 +659,7 @@ static int __init lirc_parallel_init(void)
 	if (!ppdevice) {
 		pr_notice("parport_register_device() failed\n");
 		result = -ENXIO;
-		goto exit_device_put;
+		goto exit_device_del;
 	}
 	if (parport_claim(ppdevice) != 0)
 		goto skip_init;
@@ -678,7 +678,7 @@ static int __init lirc_parallel_init(void)
 		parport_release(pport);
 		parport_unregister_device(ppdevice);
 		result = -EIO;
-		goto exit_device_put;
+		goto exit_device_del;
 	}
 
 #endif
@@ -695,11 +695,13 @@ static int __init lirc_parallel_init(void)
 		pr_notice("register_chrdev() failed\n");
 		parport_unregister_device(ppdevice);
 		result = -EIO;
-		goto exit_device_put;
+		goto exit_device_del;
 	}
 	pr_info("installed using port 0x%04x irq %d\n", io, irq);
 	return 0;
 
+exit_device_del:
+	platform_device_del(lirc_parallel_dev);
 exit_device_put:
 	platform_device_put(lirc_parallel_dev);
 exit_driver_unregister:
diff --git a/drivers/staging/media/lirc/lirc_sasem.c b/drivers/staging/media/lirc/lirc_sasem.c
index 2218d0042030..4678ae10b030 100644
--- a/drivers/staging/media/lirc/lirc_sasem.c
+++ b/drivers/staging/media/lirc/lirc_sasem.c
@@ -47,7 +47,6 @@
 #include <media/lirc.h>
 #include <media/lirc_dev.h>
 
-
 #define MOD_AUTHOR	"Oliver Stabel <oliver.stabel@gmx.de>, " \
 			"Tim Davies <tim@opensystems.net.au>"
 #define MOD_DESC	"USB Driver for Sasem Remote Controller V1.1"
@@ -73,7 +72,7 @@ static void usb_tx_callback(struct urb *urb);
 
 /* VFD file_operations function prototypes */
 static int vfd_open(struct inode *inode, struct file *file);
-static long vfd_ioctl(struct file *file, unsigned cmd, unsigned long arg);
+static long vfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
 static int vfd_close(struct inode *inode, struct file *file);
 static ssize_t vfd_write(struct file *file, const char __user *buf,
 				size_t n_bytes, loff_t *pos);
@@ -86,7 +85,6 @@ static void ir_close(void *data);
 #define SASEM_DATA_BUF_SZ	32
 
 struct sasem_context {
-
 	struct usb_device *dev;
 	int vfd_isopen;			/* VFD port has been opened */
 	unsigned int vfd_contrast;	/* VFD contrast */
@@ -156,7 +154,6 @@ static int debug;
 
 
 /*** M O D U L E   C O D E ***/
-
 MODULE_AUTHOR(MOD_AUTHOR);
 MODULE_DESCRIPTION(MOD_DESC);
 MODULE_LICENSE("GPL");
@@ -186,7 +183,6 @@ static void deregister_from_lirc(struct sasem_context *context)
 	else
 		dev_info(&context->dev->dev,
 			 "Deregistered Sasem driver (minor:%d)\n", minor);
-
 }
 
 /**
@@ -243,7 +239,7 @@ exit:
  * Called when the VFD device (e.g. /dev/usb/lcd)
  * is closed by the application.
  */
-static long vfd_ioctl(struct file *file, unsigned cmd, unsigned long arg)
+static long vfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	struct sasem_context *context;
 
@@ -297,7 +293,6 @@ static int vfd_close(struct inode *inode, struct file *file)
 		context->vfd_isopen = 0;
 		dev_info(&context->dev->dev, "VFD port closed\n");
 		if (!context->dev_present && !context->ir_isopen) {
-
 			/* Device disconnected before close and IR port is
 			 * not open. If IR port is open, context will be
 			 * deleted by ir_close. */
@@ -546,9 +541,7 @@ static void ir_close(void *data)
 		 * at disconnect time, so do it now.
 		 */
 		deregister_from_lirc(context);
-
 		if (!context->vfd_isopen) {
-
 			mutex_unlock(&context->ctx_lock);
 			delete_context(context);
 			return;
@@ -633,7 +626,6 @@ static void usb_rx_callback(struct urb *urb)
 		return;
 
 	switch (urb->status) {
-
 	case -ENOENT:		/* usbcore unlink successful! */
 		return;
 
@@ -651,8 +643,6 @@ static void usb_rx_callback(struct urb *urb)
 	usb_submit_urb(context->rx_urb, GFP_ATOMIC);
 }
 
-
-
 /**
  * Callback function for USB core API: Probe
  */
@@ -709,7 +699,6 @@ static int sasem_probe(struct usb_interface *interface,
 
 		} else if (!vfd_ep_found &&
 			usb_endpoint_is_int_out(ep)) {
-
 			tx_endpoint = ep;
 			vfd_ep_found = 1;
 			if (debug)
@@ -735,17 +724,17 @@ static int sasem_probe(struct usb_interface *interface,
 	/* Allocate memory */
 	alloc_status = 0;
 
-	context = kzalloc(sizeof(struct sasem_context), GFP_KERNEL);
+	context = kzalloc(sizeof(*context), GFP_KERNEL);
 	if (!context) {
 		alloc_status = 1;
 		goto alloc_status_switch;
 	}
-	driver = kzalloc(sizeof(struct lirc_driver), GFP_KERNEL);
+	driver = kzalloc(sizeof(*driver), GFP_KERNEL);
 	if (!driver) {
 		alloc_status = 2;
 		goto alloc_status_switch;
 	}
-	rbuf = kmalloc(sizeof(struct lirc_buffer), GFP_KERNEL);
+	rbuf = kmalloc(sizeof(*rbuf), GFP_KERNEL);
 	if (!rbuf) {
 		alloc_status = 3;
 		goto alloc_status_switch;
@@ -758,17 +747,12 @@ static int sasem_probe(struct usb_interface *interface,
 	}
 	rx_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!rx_urb) {
-		dev_err(&interface->dev,
-			"%s: usb_alloc_urb failed for IR urb\n", __func__);
 		alloc_status = 5;
 		goto alloc_status_switch;
 	}
 	if (vfd_ep_found) {
 		tx_urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!tx_urb) {
-			dev_err(&interface->dev,
-				"%s: usb_alloc_urb failed for VFD urb",
-				__func__);
 			alloc_status = 6;
 			goto alloc_status_switch;
 		}
diff --git a/drivers/staging/media/omap4iss/iss.c b/drivers/staging/media/omap4iss/iss.c
index 6ceb4eb00493..c26c99fd4a24 100644
--- a/drivers/staging/media/omap4iss/iss.c
+++ b/drivers/staging/media/omap4iss/iss.c
@@ -61,7 +61,7 @@ static void iss_print_status(struct iss_device *iss)
  * See this link for reference:
  *   http://www.mail-archive.com/linux-omap@vger.kernel.org/msg08149.html
  */
-void omap4iss_flush(struct iss_device *iss)
+static void omap4iss_flush(struct iss_device *iss)
 {
 	iss_reg_write(iss, OMAP4_ISS_MEM_TOP, ISS_HL_REVISION, 0);
 	iss_reg_read(iss, OMAP4_ISS_MEM_TOP, ISS_HL_REVISION);
@@ -362,6 +362,10 @@ static irqreturn_t iss_isr(int irq, void *_iss)
 	return IRQ_HANDLED;
 }
 
+static const struct media_device_ops iss_media_ops = {
+	.link_notify = v4l2_pipeline_link_notify,
+};
+
 /* -----------------------------------------------------------------------------
  * Pipeline stream management
  */
@@ -988,7 +992,7 @@ static int iss_register_entities(struct iss_device *iss)
 	strlcpy(iss->media_dev.model, "TI OMAP4 ISS",
 		sizeof(iss->media_dev.model));
 	iss->media_dev.hw_revision = iss->revision;
-	iss->media_dev.link_notify = v4l2_pipeline_link_notify;
+	iss->media_dev.ops = &iss_media_ops;
 	ret = media_device_register(&iss->media_dev);
 	if (ret < 0) {
 		dev_err(iss->dev, "Media device registration failed (%d)\n",
diff --git a/drivers/staging/media/omap4iss/iss_video.c b/drivers/staging/media/omap4iss/iss_video.c
index 90b7ff56722d..c16927ac8eb0 100644
--- a/drivers/staging/media/omap4iss/iss_video.c
+++ b/drivers/staging/media/omap4iss/iss_video.c
@@ -646,6 +646,103 @@ iss_video_try_format(struct file *file, void *fh, struct v4l2_format *format)
 }
 
 static int
+iss_video_get_selection(struct file *file, void *fh, struct v4l2_selection *sel)
+{
+	struct iss_video *video = video_drvdata(file);
+	struct v4l2_subdev_format format;
+	struct v4l2_subdev *subdev;
+	struct v4l2_subdev_selection sdsel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = sel->target,
+	};
+	u32 pad;
+	int ret;
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		if (video->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_COMPOSE:
+	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+	case V4L2_SEL_TGT_COMPOSE_DEFAULT:
+		if (video->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+	subdev = iss_video_remote_subdev(video, &pad);
+	if (subdev == NULL)
+		return -EINVAL;
+
+	/* Try the get selection operation first and fallback to get format if not
+	 * implemented.
+	 */
+	sdsel.pad = pad;
+	ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel);
+	if (!ret)
+		sel->r = sdsel.r;
+	if (ret != -ENOIOCTLCMD)
+		return ret;
+
+	format.pad = pad;
+	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &format);
+	if (ret < 0)
+		return ret == -ENOIOCTLCMD ? -ENOTTY : ret;
+
+	sel->r.left = 0;
+	sel->r.top = 0;
+	sel->r.width = format.format.width;
+	sel->r.height = format.format.height;
+
+	return 0;
+}
+
+static int
+iss_video_set_selection(struct file *file, void *fh, struct v4l2_selection *sel)
+{
+	struct iss_video *video = video_drvdata(file);
+	struct v4l2_subdev *subdev;
+	struct v4l2_subdev_selection sdsel = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = sel->target,
+		.flags = sel->flags,
+		.r = sel->r,
+	};
+	u32 pad;
+	int ret;
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		if (video->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+			return -EINVAL;
+		break;
+	case V4L2_SEL_TGT_COMPOSE:
+		if (video->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+			return -EINVAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+	subdev = iss_video_remote_subdev(video, &pad);
+	if (subdev == NULL)
+		return -EINVAL;
+
+	sdsel.pad = pad;
+	mutex_lock(&video->mutex);
+	ret = v4l2_subdev_call(subdev, pad, set_selection, NULL, &sdsel);
+	mutex_unlock(&video->mutex);
+	if (!ret)
+		sel->r = sdsel.r;
+
+	return ret == -ENOIOCTLCMD ? -ENOTTY : ret;
+}
+
+static int
 iss_video_get_param(struct file *file, void *fh, struct v4l2_streamparm *a)
 {
 	struct iss_video_fh *vfh = to_iss_video_fh(fh);
@@ -971,6 +1068,8 @@ static const struct v4l2_ioctl_ops iss_video_ioctl_ops = {
 	.vidioc_g_fmt_vid_out		= iss_video_get_format,
 	.vidioc_s_fmt_vid_out		= iss_video_set_format,
 	.vidioc_try_fmt_vid_out		= iss_video_try_format,
+	.vidioc_g_selection		= iss_video_get_selection,
+	.vidioc_s_selection		= iss_video_set_selection,
 	.vidioc_g_parm			= iss_video_get_param,
 	.vidioc_s_parm			= iss_video_set_param,
 	.vidioc_reqbufs			= iss_video_reqbufs,
diff --git a/drivers/staging/media/pulse8-cec/pulse8-cec.c b/drivers/staging/media/pulse8-cec/pulse8-cec.c
index 94f8590492dc..1732c3857b8e 100644
--- a/drivers/staging/media/pulse8-cec/pulse8-cec.c
+++ b/drivers/staging/media/pulse8-cec/pulse8-cec.c
@@ -10,6 +10,29 @@
  * this archive for more details.
  */
 
+/*
+ * Notes:
+ *
+ * - Devices with firmware version < 2 do not store their configuration in
+ *   EEPROM.
+ *
+ * - In autonomous mode, only messages from a TV will be acknowledged, even
+ *   polling messages. Upon receiving a message from a TV, the dongle will
+ *   respond to messages from any logical address.
+ *
+ * - In autonomous mode, the dongle will by default reply Feature Abort
+ *   [Unrecognized Opcode] when it receives Give Device Vendor ID. It will
+ *   however observe vendor ID's reported by other devices and possibly
+ *   alter this behavior. When TV's (and TV's only) report that their vendor ID
+ *   is LG (0x00e091), the dongle will itself reply that it has the same vendor
+ *   ID, and it will respond to at least one vendor specific command.
+ *
+ * - In autonomous mode, the dongle is known to attempt wakeup if it receives
+ *   <User Control Pressed> ["Power On"], ["Power] or ["Power Toggle"], or if it
+ *   receives <Set Stream Path> with its own physical address. It also does this
+ *   if it receives <Vendor Specific Command> [0x03 0x00] from an LG TV.
+ */
+
 #include <linux/completion.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -28,8 +51,11 @@ MODULE_DESCRIPTION("Pulse Eight HDMI CEC driver");
 MODULE_LICENSE("GPL");
 
 static int debug;
+static int persistent_config = 1;
 module_param(debug, int, 0644);
+module_param(persistent_config, int, 0644);
 MODULE_PARM_DESC(debug, "debug level (0-1)");
+MODULE_PARM_DESC(persistent_config, "read config from persistent memory (0-1)");
 
 enum pulse8_msgcodes {
 	MSGCODE_NOTHING = 0,
@@ -86,12 +112,16 @@ enum pulse8_msgcodes {
 
 #define DATA_SIZE 256
 
+#define PING_PERIOD	(15 * HZ)
+
 struct pulse8 {
 	struct device *dev;
 	struct serio *serio;
 	struct cec_adapter *adap;
+	unsigned int vers;
 	struct completion cmd_done;
 	struct work_struct work;
+	struct delayed_work ping_eeprom_work;
 	struct cec_msg rx_msg;
 	u8 data[DATA_SIZE];
 	unsigned int len;
@@ -99,8 +129,15 @@ struct pulse8 {
 	unsigned int idx;
 	bool escape;
 	bool started;
+	struct mutex config_lock;
+	struct mutex write_lock;
+	bool config_pending;
+	bool restoring_config;
+	bool autonomous;
 };
 
+static void pulse8_ping_eeprom_work_handler(struct work_struct *work);
+
 static void pulse8_irq_work_handler(struct work_struct *work)
 {
 	struct pulse8 *pulse8 =
@@ -114,14 +151,11 @@ static void pulse8_irq_work_handler(struct work_struct *work)
 		cec_transmit_done(pulse8->adap, CEC_TX_STATUS_OK,
 				  0, 0, 0, 0);
 		break;
-	case MSGCODE_TRANSMIT_FAILED_LINE:
-		cec_transmit_done(pulse8->adap, CEC_TX_STATUS_ARB_LOST,
-				  1, 0, 0, 0);
-		break;
 	case MSGCODE_TRANSMIT_FAILED_ACK:
 		cec_transmit_done(pulse8->adap, CEC_TX_STATUS_NACK,
 				  0, 1, 0, 0);
 		break;
+	case MSGCODE_TRANSMIT_FAILED_LINE:
 	case MSGCODE_TRANSMIT_FAILED_TIMEOUT_DATA:
 	case MSGCODE_TRANSMIT_FAILED_TIMEOUT_LINE:
 		cec_transmit_done(pulse8->adap, CEC_TX_STATUS_ERROR,
@@ -170,6 +204,9 @@ static irqreturn_t pulse8_interrupt(struct serio *serio, unsigned char data,
 		case MSGCODE_TRANSMIT_FAILED_TIMEOUT_LINE:
 			schedule_work(&pulse8->work);
 			break;
+		case MSGCODE_HIGH_ERROR:
+		case MSGCODE_LOW_ERROR:
+		case MSGCODE_RECEIVE_FAILED:
 		case MSGCODE_TIMEOUT_ERROR:
 			break;
 		case MSGCODE_COMMAND_ACCEPTED:
@@ -205,6 +242,7 @@ static void pulse8_disconnect(struct serio *serio)
 	struct pulse8 *pulse8 = serio_get_drvdata(serio);
 
 	cec_unregister_adapter(pulse8->adap);
+	cancel_delayed_work_sync(&pulse8->ping_eeprom_work);
 	dev_info(&serio->dev, "disconnected\n");
 	serio_close(serio);
 	serio_set_drvdata(serio, NULL);
@@ -228,13 +266,14 @@ static int pulse8_send(struct serio *serio, const u8 *command, u8 cmd_len)
 		}
 	}
 	if (!err)
-		err = serio_write(serio, 0xfe);
+		err = serio_write(serio, MSGEND);
 
 	return err;
 }
 
-static int pulse8_send_and_wait(struct pulse8 *pulse8,
-				const u8 *cmd, u8 cmd_len, u8 response, u8 size)
+static int pulse8_send_and_wait_once(struct pulse8 *pulse8,
+				     const u8 *cmd, u8 cmd_len,
+				     u8 response, u8 size)
 {
 	int err;
 
@@ -250,24 +289,8 @@ static int pulse8_send_and_wait(struct pulse8 *pulse8,
 	if ((pulse8->data[0] & 0x3f) == MSGCODE_COMMAND_REJECTED &&
 	    cmd[0] != MSGCODE_SET_CONTROLLED &&
 	    cmd[0] != MSGCODE_SET_AUTO_ENABLED &&
-	    cmd[0] != MSGCODE_GET_BUILDDATE) {
-		u8 cmd_sc[2];
-
-		cmd_sc[0] = MSGCODE_SET_CONTROLLED;
-		cmd_sc[1] = 1;
-		err = pulse8_send_and_wait(pulse8, cmd_sc, 2,
-					   MSGCODE_COMMAND_ACCEPTED, 1);
-		if (err)
-			return err;
-		init_completion(&pulse8->cmd_done);
-
-		err = pulse8_send(pulse8->serio, cmd, cmd_len);
-		if (err)
-			return err;
-
-		if (!wait_for_completion_timeout(&pulse8->cmd_done, HZ))
-			return -ETIMEDOUT;
-	}
+	    cmd[0] != MSGCODE_GET_BUILDDATE)
+		return -ENOTTY;
 	if (response &&
 	    ((pulse8->data[0] & 0x3f) != response || pulse8->len < size + 1)) {
 		dev_info(pulse8->dev, "transmit: failed %02x\n",
@@ -277,74 +300,155 @@ static int pulse8_send_and_wait(struct pulse8 *pulse8,
 	return 0;
 }
 
-static int pulse8_setup(struct pulse8 *pulse8, struct serio *serio)
+static int pulse8_send_and_wait(struct pulse8 *pulse8,
+				const u8 *cmd, u8 cmd_len, u8 response, u8 size)
+{
+	u8 cmd_sc[2];
+	int err;
+
+	mutex_lock(&pulse8->write_lock);
+	err = pulse8_send_and_wait_once(pulse8, cmd, cmd_len, response, size);
+
+	if (err == -ENOTTY) {
+		cmd_sc[0] = MSGCODE_SET_CONTROLLED;
+		cmd_sc[1] = 1;
+		err = pulse8_send_and_wait_once(pulse8, cmd_sc, 2,
+						MSGCODE_COMMAND_ACCEPTED, 1);
+		if (err)
+			goto unlock;
+		err = pulse8_send_and_wait_once(pulse8, cmd, cmd_len,
+						response, size);
+	}
+
+unlock:
+	mutex_unlock(&pulse8->write_lock);
+	return err == -ENOTTY ? -EIO : err;
+}
+
+static int pulse8_setup(struct pulse8 *pulse8, struct serio *serio,
+			struct cec_log_addrs *log_addrs, u16 *pa)
 {
 	u8 *data = pulse8->data + 1;
-	unsigned int count = 0;
-	unsigned int vers = 0;
 	u8 cmd[2];
 	int err;
+	struct tm tm;
+	time_t date;
+
+	pulse8->vers = 0;
 
-	cmd[0] = MSGCODE_PING;
-	err = pulse8_send_and_wait(pulse8, cmd, 1,
-				   MSGCODE_COMMAND_ACCEPTED, 0);
 	cmd[0] = MSGCODE_FIRMWARE_VERSION;
-	if (!err)
-		err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 2);
+	err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 2);
 	if (err)
 		return err;
-
-	vers = (data[0] << 8) | data[1];
-
-	dev_info(pulse8->dev, "Firmware version %04x\n", vers);
-	if (vers < 2)
+	pulse8->vers = (data[0] << 8) | data[1];
+	dev_info(pulse8->dev, "Firmware version %04x\n", pulse8->vers);
+	if (pulse8->vers < 2) {
+		*pa = CEC_PHYS_ADDR_INVALID;
 		return 0;
+	}
 
 	cmd[0] = MSGCODE_GET_BUILDDATE;
-	if (!err)
-		err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 4);
-	if (!err) {
-		time_t date = (data[0] << 24) | (data[1] << 16) |
-			(data[2] << 8) | data[3];
-		struct tm tm;
-
-		time_to_tm(date, 0, &tm);
+	err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 4);
+	if (err)
+		return err;
+	date = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3];
+	time_to_tm(date, 0, &tm);
+	dev_info(pulse8->dev, "Firmware build date %04ld.%02d.%02d %02d:%02d:%02d\n",
+		 tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+		 tm.tm_hour, tm.tm_min, tm.tm_sec);
+
+	dev_dbg(pulse8->dev, "Persistent config:\n");
+	cmd[0] = MSGCODE_GET_AUTO_ENABLED;
+	err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 1);
+	if (err)
+		return err;
+	pulse8->autonomous = data[0];
+	dev_dbg(pulse8->dev, "Autonomous mode: %s",
+		data[0] ? "on" : "off");
 
-		dev_info(pulse8->dev, "Firmware build date %04ld.%02d.%02d %02d:%02d:%02d\n",
-			 tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
-			 tm.tm_hour, tm.tm_min, tm.tm_sec);
+	cmd[0] = MSGCODE_GET_DEVICE_TYPE;
+	err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 1);
+	if (err)
+		return err;
+	log_addrs->primary_device_type[0] = data[0];
+	dev_dbg(pulse8->dev, "Primary device type: %d\n", data[0]);
+	switch (log_addrs->primary_device_type[0]) {
+	case CEC_OP_PRIM_DEVTYPE_TV:
+		log_addrs->log_addr_type[0] = CEC_LOG_ADDR_TYPE_TV;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_RECORD:
+		log_addrs->log_addr_type[0] = CEC_LOG_ADDR_TYPE_RECORD;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_TUNER:
+		log_addrs->log_addr_type[0] = CEC_LOG_ADDR_TYPE_TUNER;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_PLAYBACK:
+		log_addrs->log_addr_type[0] = CEC_LOG_ADDR_TYPE_PLAYBACK;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_AUDIOSYSTEM:
+		log_addrs->log_addr_type[0] = CEC_LOG_ADDR_TYPE_PLAYBACK;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_SWITCH:
+		log_addrs->log_addr_type[0] = CEC_LOG_ADDR_TYPE_UNREGISTERED;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_PROCESSOR:
+		log_addrs->log_addr_type[0] = CEC_LOG_ADDR_TYPE_SPECIFIC;
+		break;
+	default:
+		log_addrs->log_addr_type[0] = CEC_LOG_ADDR_TYPE_UNREGISTERED;
+		dev_info(pulse8->dev, "Unknown Primary Device Type: %d\n",
+			 log_addrs->primary_device_type[0]);
+		break;
 	}
 
-	do {
-		if (count)
-			msleep(500);
-		cmd[0] = MSGCODE_SET_AUTO_ENABLED;
-		cmd[1] = 0;
-		err = pulse8_send_and_wait(pulse8, cmd, 2,
-					   MSGCODE_COMMAND_ACCEPTED, 1);
-		if (err && count == 0) {
-			dev_info(pulse8->dev, "No Auto Enabled supported\n");
-			return 0;
-		}
+	cmd[0] = MSGCODE_GET_LOGICAL_ADDRESS_MASK;
+	err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 2);
+	if (err)
+		return err;
+	log_addrs->log_addr_mask = (data[0] << 8) | data[1];
+	dev_dbg(pulse8->dev, "Logical address ACK mask: %x\n",
+		log_addrs->log_addr_mask);
+	if (log_addrs->log_addr_mask)
+		log_addrs->num_log_addrs = 1;
+
+	cmd[0] = MSGCODE_GET_PHYSICAL_ADDRESS;
+	err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 1);
+	if (err)
+		return err;
+	*pa = (data[0] << 8) | data[1];
+	dev_dbg(pulse8->dev, "Physical address: %x.%x.%x.%x\n",
+		cec_phys_addr_exp(*pa));
 
-		cmd[0] = MSGCODE_GET_AUTO_ENABLED;
-		if (!err)
-			err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 1);
-		if (!err && !data[0]) {
-			cmd[0] = MSGCODE_WRITE_EEPROM;
-			err = pulse8_send_and_wait(pulse8, cmd, 1,
-						   MSGCODE_COMMAND_ACCEPTED, 1);
-			cmd[0] = MSGCODE_GET_AUTO_ENABLED;
-			if (!err)
-				err = pulse8_send_and_wait(pulse8, cmd, 1,
-							   cmd[0], 1);
-		}
-	} while (!err && data[0] && count++ < 5);
+	cmd[0] = MSGCODE_GET_HDMI_VERSION;
+	err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 1);
+	if (err)
+		return err;
+	log_addrs->cec_version = data[0];
+	dev_dbg(pulse8->dev, "CEC version: %d\n", log_addrs->cec_version);
 
-	if (!err && data[0])
-		err = -EIO;
+	cmd[0] = MSGCODE_GET_OSD_NAME;
+	err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 0);
+	if (err)
+		return err;
+	strncpy(log_addrs->osd_name, data, 13);
+	dev_dbg(pulse8->dev, "OSD name: %s\n", log_addrs->osd_name);
 
-	return err;
+	return 0;
+}
+
+static int pulse8_apply_persistent_config(struct pulse8 *pulse8,
+					  struct cec_log_addrs *log_addrs,
+					  u16 pa)
+{
+	int err;
+
+	err = cec_s_log_addrs(pulse8->adap, log_addrs, false);
+	if (err)
+		return err;
+
+	cec_s_phys_addr(pulse8->adap, pa, false);
+
+	return 0;
 }
 
 static int pulse8_cec_adap_enable(struct cec_adapter *adap, bool enable)
@@ -364,9 +468,11 @@ static int pulse8_cec_adap_log_addr(struct cec_adapter *adap, u8 log_addr)
 {
 	struct pulse8 *pulse8 = adap->priv;
 	u16 mask = 0;
-	u8 cmd[3];
-	int err;
+	u16 pa = adap->phys_addr;
+	u8 cmd[16];
+	int err = 0;
 
+	mutex_lock(&pulse8->config_lock);
 	if (log_addr != CEC_LOG_ADDR_INVALID)
 		mask = 1 << log_addr;
 	cmd[0] = MSGCODE_SET_ACK_MASK;
@@ -374,8 +480,106 @@ static int pulse8_cec_adap_log_addr(struct cec_adapter *adap, u8 log_addr)
 	cmd[2] = mask & 0xff;
 	err = pulse8_send_and_wait(pulse8, cmd, 3,
 				   MSGCODE_COMMAND_ACCEPTED, 0);
-	if (mask == 0)
-		return 0;
+	if ((err && mask != 0) || pulse8->restoring_config)
+		goto unlock;
+
+	cmd[0] = MSGCODE_SET_AUTO_ENABLED;
+	cmd[1] = log_addr == CEC_LOG_ADDR_INVALID ? 0 : 1;
+	err = pulse8_send_and_wait(pulse8, cmd, 2,
+				   MSGCODE_COMMAND_ACCEPTED, 0);
+	if (err)
+		goto unlock;
+	pulse8->autonomous = cmd[1];
+	if (log_addr == CEC_LOG_ADDR_INVALID)
+		goto unlock;
+
+	cmd[0] = MSGCODE_SET_DEVICE_TYPE;
+	cmd[1] = adap->log_addrs.primary_device_type[0];
+	err = pulse8_send_and_wait(pulse8, cmd, 2,
+				   MSGCODE_COMMAND_ACCEPTED, 0);
+	if (err)
+		goto unlock;
+
+	switch (adap->log_addrs.primary_device_type[0]) {
+	case CEC_OP_PRIM_DEVTYPE_TV:
+		mask = CEC_LOG_ADDR_MASK_TV;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_RECORD:
+		mask = CEC_LOG_ADDR_MASK_RECORD;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_TUNER:
+		mask = CEC_LOG_ADDR_MASK_TUNER;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_PLAYBACK:
+		mask = CEC_LOG_ADDR_MASK_PLAYBACK;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_AUDIOSYSTEM:
+		mask = CEC_LOG_ADDR_MASK_AUDIOSYSTEM;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_SWITCH:
+		mask = CEC_LOG_ADDR_MASK_UNREGISTERED;
+		break;
+	case CEC_OP_PRIM_DEVTYPE_PROCESSOR:
+		mask = CEC_LOG_ADDR_MASK_SPECIFIC;
+		break;
+	default:
+		mask = 0;
+		break;
+	}
+	cmd[0] = MSGCODE_SET_LOGICAL_ADDRESS_MASK;
+	cmd[1] = mask >> 8;
+	cmd[2] = mask & 0xff;
+	err = pulse8_send_and_wait(pulse8, cmd, 3,
+				   MSGCODE_COMMAND_ACCEPTED, 0);
+	if (err)
+		goto unlock;
+
+	cmd[0] = MSGCODE_SET_DEFAULT_LOGICAL_ADDRESS;
+	cmd[1] = log_addr;
+	err = pulse8_send_and_wait(pulse8, cmd, 2,
+				   MSGCODE_COMMAND_ACCEPTED, 0);
+	if (err)
+		goto unlock;
+
+	cmd[0] = MSGCODE_SET_PHYSICAL_ADDRESS;
+	cmd[1] = pa >> 8;
+	cmd[2] = pa & 0xff;
+	err = pulse8_send_and_wait(pulse8, cmd, 3,
+				   MSGCODE_COMMAND_ACCEPTED, 0);
+	if (err)
+		goto unlock;
+
+	cmd[0] = MSGCODE_SET_HDMI_VERSION;
+	cmd[1] = adap->log_addrs.cec_version;
+	err = pulse8_send_and_wait(pulse8, cmd, 2,
+				   MSGCODE_COMMAND_ACCEPTED, 0);
+	if (err)
+		goto unlock;
+
+	if (adap->log_addrs.osd_name[0]) {
+		size_t osd_len = strlen(adap->log_addrs.osd_name);
+		char *osd_str = cmd + 1;
+
+		cmd[0] = MSGCODE_SET_OSD_NAME;
+		strncpy(cmd + 1, adap->log_addrs.osd_name, 13);
+		if (osd_len < 4) {
+			memset(osd_str + osd_len, ' ', 4 - osd_len);
+			osd_len = 4;
+			osd_str[osd_len] = '\0';
+			strcpy(adap->log_addrs.osd_name, osd_str);
+		}
+		err = pulse8_send_and_wait(pulse8, cmd, 1 + osd_len,
+					   MSGCODE_COMMAND_ACCEPTED, 0);
+		if (err)
+			goto unlock;
+	}
+
+unlock:
+	if (pulse8->restoring_config)
+		pulse8->restoring_config = false;
+	else
+		pulse8->config_pending = true;
+	mutex_unlock(&pulse8->config_lock);
 	return err;
 }
 
@@ -388,7 +592,7 @@ static int pulse8_cec_adap_transmit(struct cec_adapter *adap, u8 attempts,
 	int err;
 
 	cmd[0] = MSGCODE_TRANSMIT_IDLETIME;
-	cmd[1] = 3;
+	cmd[1] = signal_free_time;
 	err = pulse8_send_and_wait(pulse8, cmd, 2,
 				   MSGCODE_COMMAND_ACCEPTED, 1);
 	cmd[0] = MSGCODE_TRANSMIT_ACK_POLARITY;
@@ -437,6 +641,8 @@ static int pulse8_connect(struct serio *serio, struct serio_driver *drv)
 		CEC_CAP_PASSTHROUGH | CEC_CAP_RC | CEC_CAP_MONITOR_ALL;
 	struct pulse8 *pulse8;
 	int err = -ENOMEM;
+	struct cec_log_addrs log_addrs = {};
+	u16 pa = CEC_PHYS_ADDR_INVALID;
 
 	pulse8 = kzalloc(sizeof(*pulse8), GFP_KERNEL);
 
@@ -453,12 +659,15 @@ static int pulse8_connect(struct serio *serio, struct serio_driver *drv)
 	pulse8->dev = &serio->dev;
 	serio_set_drvdata(serio, pulse8);
 	INIT_WORK(&pulse8->work, pulse8_irq_work_handler);
+	mutex_init(&pulse8->write_lock);
+	mutex_init(&pulse8->config_lock);
+	pulse8->config_pending = false;
 
 	err = serio_open(serio, drv);
 	if (err)
 		goto delete_adap;
 
-	err = pulse8_setup(pulse8, serio);
+	err = pulse8_setup(pulse8, serio, &log_addrs, &pa);
 	if (err)
 		goto close_serio;
 
@@ -467,6 +676,18 @@ static int pulse8_connect(struct serio *serio, struct serio_driver *drv)
 		goto close_serio;
 
 	pulse8->dev = &pulse8->adap->devnode.dev;
+
+	if (persistent_config && pulse8->autonomous) {
+		err = pulse8_apply_persistent_config(pulse8, &log_addrs, pa);
+		if (err)
+			goto close_serio;
+		pulse8->restoring_config = true;
+	}
+
+	INIT_DELAYED_WORK(&pulse8->ping_eeprom_work,
+			  pulse8_ping_eeprom_work_handler);
+	schedule_delayed_work(&pulse8->ping_eeprom_work, PING_PERIOD);
+
 	return 0;
 
 close_serio:
@@ -479,6 +700,33 @@ free_device:
 	return err;
 }
 
+static void pulse8_ping_eeprom_work_handler(struct work_struct *work)
+{
+	struct pulse8 *pulse8 =
+		container_of(work, struct pulse8, ping_eeprom_work.work);
+	u8 cmd;
+
+	schedule_delayed_work(&pulse8->ping_eeprom_work, PING_PERIOD);
+	cmd = MSGCODE_PING;
+	pulse8_send_and_wait(pulse8, &cmd, 1,
+			     MSGCODE_COMMAND_ACCEPTED, 0);
+
+	if (pulse8->vers < 2)
+		return;
+
+	mutex_lock(&pulse8->config_lock);
+	if (pulse8->config_pending && persistent_config) {
+		dev_dbg(pulse8->dev, "writing pending config to EEPROM\n");
+		cmd = MSGCODE_WRITE_EEPROM;
+		if (pulse8_send_and_wait(pulse8, &cmd, 1,
+					 MSGCODE_COMMAND_ACCEPTED, 0))
+			dev_info(pulse8->dev, "failed to write pending config to EEPROM\n");
+		else
+			pulse8->config_pending = false;
+	}
+	mutex_unlock(&pulse8->config_lock);
+}
+
 static struct serio_device_id pulse8_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
diff --git a/drivers/staging/media/s5p-cec/s5p_cec.c b/drivers/staging/media/s5p-cec/s5p_cec.c
index 78333273c4e5..1780a08b73c9 100644
--- a/drivers/staging/media/s5p-cec/s5p_cec.c
+++ b/drivers/staging/media/s5p-cec/s5p_cec.c
@@ -173,7 +173,7 @@ static int s5p_cec_probe(struct platform_device *pdev)
 	int ret;
 
 	cec = devm_kzalloc(&pdev->dev, sizeof(*cec), GFP_KERNEL);
-	if (!dev)
+	if (!cec)
 		return -ENOMEM;
 
 	cec->dev = dev;
@@ -250,22 +250,9 @@ static int s5p_cec_runtime_resume(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused s5p_cec_suspend(struct device *dev)
-{
-	if (pm_runtime_suspended(dev))
-		return 0;
-	return s5p_cec_runtime_suspend(dev);
-}
-
-static int __maybe_unused s5p_cec_resume(struct device *dev)
-{
-	if (pm_runtime_suspended(dev))
-		return 0;
-	return s5p_cec_runtime_resume(dev);
-}
-
 static const struct dev_pm_ops s5p_cec_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(s5p_cec_suspend, s5p_cec_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 	SET_RUNTIME_PM_OPS(s5p_cec_runtime_suspend, s5p_cec_runtime_resume,
 			   NULL)
 };
diff --git a/drivers/staging/media/st-cec/Kconfig b/drivers/staging/media/st-cec/Kconfig
new file mode 100644
index 000000000000..784d2c600aca
--- /dev/null
+++ b/drivers/staging/media/st-cec/Kconfig
@@ -0,0 +1,8 @@
+config VIDEO_STI_HDMI_CEC
+       tristate "STMicroelectronics STiH4xx HDMI CEC driver"
+       depends on VIDEO_DEV && MEDIA_CEC && (ARCH_STI || COMPILE_TEST)
+       ---help---
+         This is a driver for STIH4xx HDMI CEC interface. It uses the
+         generic CEC framework interface.
+         CEC bus is present in the HDMI connector and enables communication
+         between compatible devices.
diff --git a/drivers/staging/media/st-cec/Makefile b/drivers/staging/media/st-cec/Makefile
new file mode 100644
index 000000000000..f07905e1448a
--- /dev/null
+++ b/drivers/staging/media/st-cec/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_VIDEO_STI_HDMI_CEC) += stih-cec.o
diff --git a/drivers/staging/media/st-cec/stih-cec.c b/drivers/staging/media/st-cec/stih-cec.c
new file mode 100644
index 000000000000..214344866a6b
--- /dev/null
+++ b/drivers/staging/media/st-cec/stih-cec.c
@@ -0,0 +1,380 @@
+/*
+ * drivers/staging/media/st-cec/stih-cec.c
+ *
+ * STIH4xx CEC driver
+ * Copyright (C) STMicroelectronic SA 2016
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/version.h>
+
+#include <media/cec.h>
+
+#define CEC_NAME	"stih-cec"
+
+/* CEC registers  */
+#define CEC_CLK_DIV           0x0
+#define CEC_CTRL              0x4
+#define CEC_IRQ_CTRL          0x8
+#define CEC_STATUS            0xC
+#define CEC_EXT_STATUS        0x10
+#define CEC_TX_CTRL           0x14
+#define CEC_FREE_TIME_THRESH  0x18
+#define CEC_BIT_TOUT_THRESH   0x1C
+#define CEC_BIT_PULSE_THRESH  0x20
+#define CEC_DATA              0x24
+#define CEC_TX_ARRAY_CTRL     0x28
+#define CEC_CTRL2             0x2C
+#define CEC_TX_ERROR_STS      0x30
+#define CEC_ADDR_TABLE        0x34
+#define CEC_DATA_ARRAY_CTRL   0x38
+#define CEC_DATA_ARRAY_STATUS 0x3C
+#define CEC_TX_DATA_BASE      0x40
+#define CEC_TX_DATA_TOP       0x50
+#define CEC_TX_DATA_SIZE      0x1
+#define CEC_RX_DATA_BASE      0x54
+#define CEC_RX_DATA_TOP       0x64
+#define CEC_RX_DATA_SIZE      0x1
+
+/* CEC_CTRL2 */
+#define CEC_LINE_INACTIVE_EN   BIT(0)
+#define CEC_AUTO_BUS_ERR_EN    BIT(1)
+#define CEC_STOP_ON_ARB_ERR_EN BIT(2)
+#define CEC_TX_REQ_WAIT_EN     BIT(3)
+
+/* CEC_DATA_ARRAY_CTRL */
+#define CEC_TX_ARRAY_EN          BIT(0)
+#define CEC_RX_ARRAY_EN          BIT(1)
+#define CEC_TX_ARRAY_RESET       BIT(2)
+#define CEC_RX_ARRAY_RESET       BIT(3)
+#define CEC_TX_N_OF_BYTES_IRQ_EN BIT(4)
+#define CEC_TX_STOP_ON_NACK      BIT(7)
+
+/* CEC_TX_ARRAY_CTRL */
+#define CEC_TX_N_OF_BYTES  0x1F
+#define CEC_TX_START       BIT(5)
+#define CEC_TX_AUTO_SOM_EN BIT(6)
+#define CEC_TX_AUTO_EOM_EN BIT(7)
+
+/* CEC_IRQ_CTRL */
+#define CEC_TX_DONE_IRQ_EN   BIT(0)
+#define CEC_ERROR_IRQ_EN     BIT(2)
+#define CEC_RX_DONE_IRQ_EN   BIT(3)
+#define CEC_RX_SOM_IRQ_EN    BIT(4)
+#define CEC_RX_EOM_IRQ_EN    BIT(5)
+#define CEC_FREE_TIME_IRQ_EN BIT(6)
+#define CEC_PIN_STS_IRQ_EN   BIT(7)
+
+/* CEC_CTRL */
+#define CEC_IN_FILTER_EN    BIT(0)
+#define CEC_PWR_SAVE_EN     BIT(1)
+#define CEC_EN              BIT(4)
+#define CEC_ACK_CTRL        BIT(5)
+#define CEC_RX_RESET_EN     BIT(6)
+#define CEC_IGNORE_RX_ERROR BIT(7)
+
+/* CEC_STATUS */
+#define CEC_TX_DONE_STS       BIT(0)
+#define CEC_TX_ACK_GET_STS    BIT(1)
+#define CEC_ERROR_STS         BIT(2)
+#define CEC_RX_DONE_STS       BIT(3)
+#define CEC_RX_SOM_STS        BIT(4)
+#define CEC_RX_EOM_STS        BIT(5)
+#define CEC_FREE_TIME_IRQ_STS BIT(6)
+#define CEC_PIN_STS           BIT(7)
+#define CEC_SBIT_TOUT_STS     BIT(8)
+#define CEC_DBIT_TOUT_STS     BIT(9)
+#define CEC_LPULSE_ERROR_STS  BIT(10)
+#define CEC_HPULSE_ERROR_STS  BIT(11)
+#define CEC_TX_ERROR          BIT(12)
+#define CEC_TX_ARB_ERROR      BIT(13)
+#define CEC_RX_ERROR_MIN      BIT(14)
+#define CEC_RX_ERROR_MAX      BIT(15)
+
+/* Signal free time in bit periods (2.4ms) */
+#define CEC_PRESENT_INIT_SFT 7
+#define CEC_NEW_INIT_SFT     5
+#define CEC_RETRANSMIT_SFT   3
+
+/* Constants for CEC_BIT_TOUT_THRESH register */
+#define CEC_SBIT_TOUT_47MS BIT(1)
+#define CEC_SBIT_TOUT_48MS BIT(0) | BIT(1)
+#define CEC_SBIT_TOUT_50MS BIT(2)
+#define CEC_DBIT_TOUT_27MS BIT(0)
+#define CEC_DBIT_TOUT_28MS BIT(1)
+#define CEC_DBIT_TOUT_29MS BIT(0) | BIT(1)
+
+/* Constants for CEC_BIT_PULSE_THRESH register */
+#define CEC_BIT_LPULSE_03MS BIT(1)
+#define CEC_BIT_HPULSE_03MS BIT(3)
+
+/* Constants for CEC_DATA_ARRAY_STATUS register */
+#define CEC_RX_N_OF_BYTES                     0x1F
+#define CEC_TX_N_OF_BYTES_SENT                BIT(5)
+#define CEC_RX_OVERRUN                        BIT(6)
+
+struct stih_cec {
+	struct cec_adapter	*adap;
+	struct device		*dev;
+	struct clk		*clk;
+	void __iomem		*regs;
+	int			irq;
+	u32			irq_status;
+};
+
+static int stih_cec_adap_enable(struct cec_adapter *adap, bool enable)
+{
+	struct stih_cec *cec = adap->priv;
+
+	if (enable) {
+		/* The doc says (input TCLK_PERIOD * CEC_CLK_DIV) = 0.1ms */
+		unsigned long clk_freq = clk_get_rate(cec->clk);
+		u32 cec_clk_div = clk_freq / 10000;
+
+		writel(cec_clk_div, cec->regs + CEC_CLK_DIV);
+
+		/* Configuration of the durations activating a timeout */
+		writel(CEC_SBIT_TOUT_47MS | (CEC_DBIT_TOUT_28MS << 4),
+		       cec->regs + CEC_BIT_TOUT_THRESH);
+
+		/* Configuration of the smallest allowed duration for pulses */
+		writel(CEC_BIT_LPULSE_03MS | CEC_BIT_HPULSE_03MS,
+		       cec->regs + CEC_BIT_PULSE_THRESH);
+
+		/* Minimum received bit period threshold */
+		writel(BIT(5) | BIT(7), cec->regs + CEC_TX_CTRL);
+
+		/* Configuration of transceiver data arrays */
+		writel(CEC_TX_ARRAY_EN | CEC_RX_ARRAY_EN | CEC_TX_STOP_ON_NACK,
+		       cec->regs + CEC_DATA_ARRAY_CTRL);
+
+		/* Configuration of the control bits for CEC Transceiver */
+		writel(CEC_IN_FILTER_EN | CEC_EN | CEC_RX_RESET_EN,
+		       cec->regs + CEC_CTRL);
+
+		/* Clear logical addresses */
+		writel(0, cec->regs + CEC_ADDR_TABLE);
+
+		/* Clear the status register */
+		writel(0x0, cec->regs + CEC_STATUS);
+
+		/* Enable the interrupts */
+		writel(CEC_TX_DONE_IRQ_EN | CEC_RX_DONE_IRQ_EN |
+		       CEC_RX_SOM_IRQ_EN | CEC_RX_EOM_IRQ_EN |
+		       CEC_ERROR_IRQ_EN,
+		       cec->regs + CEC_IRQ_CTRL);
+
+	} else {
+		/* Clear logical addresses */
+		writel(0, cec->regs + CEC_ADDR_TABLE);
+
+		/* Clear the status register */
+		writel(0x0, cec->regs + CEC_STATUS);
+
+		/* Disable the interrupts */
+		writel(0, cec->regs + CEC_IRQ_CTRL);
+	}
+
+	return 0;
+}
+
+static int stih_cec_adap_log_addr(struct cec_adapter *adap, u8 logical_addr)
+{
+	struct stih_cec *cec = adap->priv;
+	u32 reg = readl(cec->regs + CEC_ADDR_TABLE);
+
+	reg |= 1 << logical_addr;
+
+	if (logical_addr == CEC_LOG_ADDR_INVALID)
+		reg = 0;
+
+	writel(reg, cec->regs + CEC_ADDR_TABLE);
+
+	return 0;
+}
+
+static int stih_cec_adap_transmit(struct cec_adapter *adap, u8 attempts,
+				  u32 signal_free_time, struct cec_msg *msg)
+{
+	struct stih_cec *cec = adap->priv;
+	int i;
+
+	/* Copy message into registers */
+	for (i = 0; i < msg->len; i++)
+		writeb(msg->msg[i], cec->regs + CEC_TX_DATA_BASE + i);
+
+	/* Start transmission, configure hardware to add start and stop bits
+	 * Signal free time is handled by the hardware
+	 */
+	writel(CEC_TX_AUTO_SOM_EN | CEC_TX_AUTO_EOM_EN | CEC_TX_START |
+	       msg->len, cec->regs + CEC_TX_ARRAY_CTRL);
+
+	return 0;
+}
+
+static void stih_tx_done(struct stih_cec *cec, u32 status)
+{
+	if (status & CEC_TX_ERROR) {
+		cec_transmit_done(cec->adap, CEC_TX_STATUS_ERROR, 0, 0, 0, 1);
+		return;
+	}
+
+	if (status & CEC_TX_ARB_ERROR) {
+		cec_transmit_done(cec->adap,
+				  CEC_TX_STATUS_ARB_LOST, 1, 0, 0, 0);
+		return;
+	}
+
+	if (!(status & CEC_TX_ACK_GET_STS)) {
+		cec_transmit_done(cec->adap, CEC_TX_STATUS_NACK, 0, 1, 0, 0);
+		return;
+	}
+
+	cec_transmit_done(cec->adap, CEC_TX_STATUS_OK, 0, 0, 0, 0);
+}
+
+static void stih_rx_done(struct stih_cec *cec, u32 status)
+{
+	struct cec_msg msg = {};
+	u8 i;
+
+	if (status & CEC_RX_ERROR_MIN)
+		return;
+
+	if (status & CEC_RX_ERROR_MAX)
+		return;
+
+	msg.len = readl(cec->regs + CEC_DATA_ARRAY_STATUS) & 0x1f;
+
+	if (!msg.len)
+		return;
+
+	if (msg.len > 16)
+		msg.len = 16;
+
+	for (i = 0; i < msg.len; i++)
+		msg.msg[i] = readl(cec->regs + CEC_RX_DATA_BASE + i);
+
+	cec_received_msg(cec->adap, &msg);
+}
+
+static irqreturn_t stih_cec_irq_handler_thread(int irq, void *priv)
+{
+	struct stih_cec *cec = priv;
+
+	if (cec->irq_status & CEC_TX_DONE_STS)
+		stih_tx_done(cec, cec->irq_status);
+
+	if (cec->irq_status & CEC_RX_DONE_STS)
+		stih_rx_done(cec, cec->irq_status);
+
+	cec->irq_status = 0;
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t stih_cec_irq_handler(int irq, void *priv)
+{
+	struct stih_cec *cec = priv;
+
+	cec->irq_status = readl(cec->regs + CEC_STATUS);
+	writel(cec->irq_status, cec->regs + CEC_STATUS);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static const struct cec_adap_ops sti_cec_adap_ops = {
+	.adap_enable = stih_cec_adap_enable,
+	.adap_log_addr = stih_cec_adap_log_addr,
+	.adap_transmit = stih_cec_adap_transmit,
+};
+
+static int stih_cec_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	struct stih_cec *cec;
+	int ret;
+
+	cec = devm_kzalloc(dev, sizeof(*cec), GFP_KERNEL);
+	if (!cec)
+		return -ENOMEM;
+
+	cec->dev = dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	cec->regs = devm_ioremap_resource(dev, res);
+	if (IS_ERR(cec->regs))
+		return PTR_ERR(cec->regs);
+
+	cec->irq = platform_get_irq(pdev, 0);
+	if (cec->irq < 0)
+		return cec->irq;
+
+	ret = devm_request_threaded_irq(dev, cec->irq, stih_cec_irq_handler,
+					stih_cec_irq_handler_thread, 0,
+					pdev->name, cec);
+	if (ret)
+		return ret;
+
+	cec->clk = devm_clk_get(dev, "cec-clk");
+	if (IS_ERR(cec->clk)) {
+		dev_err(dev, "Cannot get cec clock\n");
+		return PTR_ERR(cec->clk);
+	}
+
+	cec->adap = cec_allocate_adapter(&sti_cec_adap_ops, cec,
+			CEC_NAME,
+			CEC_CAP_LOG_ADDRS | CEC_CAP_PASSTHROUGH |
+			CEC_CAP_PHYS_ADDR | CEC_CAP_TRANSMIT,
+			1, &pdev->dev);
+	ret = PTR_ERR_OR_ZERO(cec->adap);
+	if (ret)
+		return ret;
+
+	ret = cec_register_adapter(cec->adap);
+	if (ret) {
+		cec_delete_adapter(cec->adap);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, cec);
+	return 0;
+}
+
+static int stih_cec_remove(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static const struct of_device_id stih_cec_match[] = {
+	{
+		.compatible	= "st,stih-cec",
+	},
+	{},
+};
+
+static struct platform_driver stih_cec_pdrv = {
+	.probe	= stih_cec_probe,
+	.remove = stih_cec_remove,
+	.driver = {
+		.name		= CEC_NAME,
+		.of_match_table	= stih_cec_match,
+	},
+};
+
+module_platform_driver(stih_cec_pdrv);
+
+MODULE_AUTHOR("Benjamin Gaignard <benjamin.gaignard@linaro.org>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("STIH4xx CEC driver");
diff --git a/drivers/staging/media/tw686x-kh/Kconfig b/drivers/staging/media/tw686x-kh/Kconfig
deleted file mode 100644
index 6264d30edf5a..000000000000
--- a/drivers/staging/media/tw686x-kh/Kconfig
+++ /dev/null
@@ -1,17 +0,0 @@
-config VIDEO_TW686X_KH
-	tristate "Intersil/Techwell TW686x Video For Linux"
-	depends on VIDEO_DEV && PCI && VIDEO_V4L2
-	depends on !(VIDEO_TW686X=y || VIDEO_TW686X=m) || COMPILE_TEST
-	select VIDEOBUF2_DMA_SG
-	help
-	  Support for Intersil/Techwell TW686x-based frame grabber cards.
-
-	  Currently supported chips:
-	  - TW6864 (4 video channels),
-	  - TW6865 (4 video channels, not tested, second generation chip),
-	  - TW6868 (8 video channels but only 4 first channels using
-	    built-in video decoder are supported, not tested),
-	  - TW6869 (8 video channels, second generation chip).
-
-	  To compile this driver as a module, choose M here: the module
-	  will be named tw686x-kh.
diff --git a/drivers/staging/media/tw686x-kh/Makefile b/drivers/staging/media/tw686x-kh/Makefile
deleted file mode 100644
index 2a36a38cf30e..000000000000
--- a/drivers/staging/media/tw686x-kh/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-tw686x-kh-objs := tw686x-kh-core.o tw686x-kh-video.o
-
-obj-$(CONFIG_VIDEO_TW686X_KH) += tw686x-kh.o
diff --git a/drivers/staging/media/tw686x-kh/TODO b/drivers/staging/media/tw686x-kh/TODO
deleted file mode 100644
index 480a495b11fb..000000000000
--- a/drivers/staging/media/tw686x-kh/TODO
+++ /dev/null
@@ -1,6 +0,0 @@
-TODO:
-
-- implement V4L2_FIELD_INTERLACED* mode(s).
-- add audio support
-
-Please Cc: patches to Krzysztof Halasa <khalasa@piap.pl>.
diff --git a/drivers/staging/media/tw686x-kh/tw686x-kh-core.c b/drivers/staging/media/tw686x-kh/tw686x-kh-core.c
deleted file mode 100644
index 03b3b62c59c4..000000000000
--- a/drivers/staging/media/tw686x-kh/tw686x-kh-core.c
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Copyright (C) 2015 Industrial Research Institute for Automation
- * and Measurements PIAP
- *
- * Written by Krzysztof Ha?asa.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License
- * as published by the Free Software Foundation.
- */
-
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include "tw686x-kh.h"
-#include "tw686x-kh-regs.h"
-
-static irqreturn_t tw686x_irq(int irq, void *dev_id)
-{
-	struct tw686x_dev *dev = (struct tw686x_dev *)dev_id;
-	u32 int_status = reg_read(dev, INT_STATUS); /* cleared on read */
-	unsigned long flags;
-	unsigned int handled = 0;
-
-	if (int_status) {
-		spin_lock_irqsave(&dev->irq_lock, flags);
-		dev->dma_requests |= int_status;
-		spin_unlock_irqrestore(&dev->irq_lock, flags);
-
-		if (int_status & 0xFF0000FF)
-			handled = tw686x_kh_video_irq(dev);
-	}
-
-	return IRQ_RETVAL(handled);
-}
-
-static int tw686x_probe(struct pci_dev *pci_dev,
-			const struct pci_device_id *pci_id)
-{
-	struct tw686x_dev *dev;
-	int err;
-
-	dev = devm_kzalloc(&pci_dev->dev, sizeof(*dev) +
-			   (pci_id->driver_data & TYPE_MAX_CHANNELS) *
-			   sizeof(dev->video_channels[0]), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
-
-	sprintf(dev->name, "TW%04X", pci_dev->device);
-	dev->type = pci_id->driver_data;
-
-	pr_info("%s: PCI %s, IRQ %d, MMIO 0x%lx\n", dev->name,
-		pci_name(pci_dev), pci_dev->irq,
-		(unsigned long)pci_resource_start(pci_dev, 0));
-
-	dev->pci_dev = pci_dev;
-	if (pcim_enable_device(pci_dev))
-		return -EIO;
-
-	pci_set_master(pci_dev);
-
-	if (pci_set_dma_mask(pci_dev, DMA_BIT_MASK(32))) {
-		pr_err("%s: 32-bit PCI DMA not supported\n", dev->name);
-		return -EIO;
-	}
-
-	err = pci_request_regions(pci_dev, dev->name);
-	if (err < 0) {
-		pr_err("%s: Unable to get MMIO region\n", dev->name);
-		return err;
-	}
-
-	dev->mmio = pci_ioremap_bar(pci_dev, 0);
-	if (!dev->mmio) {
-		pr_err("%s: Unable to remap MMIO region\n", dev->name);
-		return -EIO;
-	}
-
-	reg_write(dev, SYS_SOFT_RST, 0x0F); /* Reset all subsystems */
-	mdelay(1);
-
-	reg_write(dev, SRST[0], 0x3F);
-	if (max_channels(dev) > 4)
-		reg_write(dev, SRST[1], 0x3F);
-	reg_write(dev, DMA_CMD, 0);
-	reg_write(dev, DMA_CHANNEL_ENABLE, 0);
-	reg_write(dev, DMA_CHANNEL_TIMEOUT, 0x3EFF0FF0);
-	reg_write(dev, DMA_TIMER_INTERVAL, 0x38000);
-	reg_write(dev, DMA_CONFIG, 0xFFFFFF04);
-
-	spin_lock_init(&dev->irq_lock);
-
-	err = devm_request_irq(&pci_dev->dev, pci_dev->irq, tw686x_irq,
-			       IRQF_SHARED, dev->name, dev);
-	if (err < 0) {
-		pr_err("%s: Unable to get IRQ\n", dev->name);
-		return err;
-	}
-
-	err = tw686x_kh_video_init(dev);
-	if (err)
-		return err;
-
-	pci_set_drvdata(pci_dev, dev);
-	return 0;
-}
-
-static void tw686x_remove(struct pci_dev *pci_dev)
-{
-	struct tw686x_dev *dev = pci_get_drvdata(pci_dev);
-
-	tw686x_kh_video_free(dev);
-}
-
-/* driver_data is number of A/V channels */
-static const struct pci_device_id tw686x_pci_tbl[] = {
-	{PCI_DEVICE(0x1797, 0x6864), .driver_data = 4},
-	/* not tested */
-	{PCI_DEVICE(0x1797, 0x6865), .driver_data = 4 | TYPE_SECOND_GEN},
-	/* TW6868 supports 8 A/V channels with an external TW2865 chip -
-	   not supported by the driver */
-	{PCI_DEVICE(0x1797, 0x6868), .driver_data = 4}, /* not tested */
-	{PCI_DEVICE(0x1797, 0x6869), .driver_data = 8 | TYPE_SECOND_GEN},
-	{}
-};
-
-static struct pci_driver tw686x_pci_driver = {
-	.name = "tw686x-kh",
-	.id_table = tw686x_pci_tbl,
-	.probe = tw686x_probe,
-	.remove = tw686x_remove,
-};
-
-MODULE_DESCRIPTION("Driver for video frame grabber cards based on Intersil/Techwell TW686[4589]");
-MODULE_AUTHOR("Krzysztof Halasa");
-MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(pci, tw686x_pci_tbl);
-module_pci_driver(tw686x_pci_driver);
diff --git a/drivers/staging/media/tw686x-kh/tw686x-kh-regs.h b/drivers/staging/media/tw686x-kh/tw686x-kh-regs.h
deleted file mode 100644
index 53e1889babd0..000000000000
--- a/drivers/staging/media/tw686x-kh/tw686x-kh-regs.h
+++ /dev/null
@@ -1,103 +0,0 @@
-/* DMA controller registers */
-#define REG8_1(a0) ((const u16[8]) {a0, a0 + 1, a0 + 2, a0 + 3,	\
-				   a0 + 4, a0 + 5, a0 + 6, a0 + 7})
-#define REG8_2(a0) ((const u16[8]) {a0, a0 + 2, a0 + 4, a0 + 6,	\
-				   a0 + 8, a0 + 0xA, a0 + 0xC, a0 + 0xE})
-#define REG8_8(a0) ((const u16[8]) {a0, a0 + 8, a0 + 0x10, a0 + 0x18,	\
-				   a0 + 0x20, a0 + 0x28, a0 + 0x30, a0 + 0x38})
-#define INT_STATUS		0x00
-#define PB_STATUS		0x01
-#define DMA_CMD			0x02
-#define VIDEO_FIFO_STATUS	0x03
-#define VIDEO_CHANNEL_ID	0x04
-#define VIDEO_PARSER_STATUS	0x05
-#define SYS_SOFT_RST		0x06
-#define DMA_PAGE_TABLE0_ADDR	((const u16[8]) {0x08, 0xD0, 0xD2, 0xD4, \
-						0xD6, 0xD8, 0xDA, 0xDC})
-#define DMA_PAGE_TABLE1_ADDR	((const u16[8]) {0x09, 0xD1, 0xD3, 0xD5, \
-						0xD7, 0xD9, 0xDB, 0xDD})
-#define DMA_CHANNEL_ENABLE	0x0A
-#define DMA_CONFIG		0x0B
-#define DMA_TIMER_INTERVAL	0x0C
-#define DMA_CHANNEL_TIMEOUT	0x0D
-#define VDMA_CHANNEL_CONFIG	REG8_1(0x10)
-#define ADMA_P_ADDR		REG8_2(0x18)
-#define ADMA_B_ADDR		REG8_2(0x19)
-#define DMA10_P_ADDR		0x28 /* ??? */
-#define DMA10_B_ADDR		0x29
-#define VIDEO_CONTROL1		0x2A
-#define VIDEO_CONTROL2		0x2B
-#define AUDIO_CONTROL1		0x2C
-#define AUDIO_CONTROL2		0x2D
-#define PHASE_REF		0x2E
-#define GPIO_REG		0x2F
-#define INTL_HBAR_CTRL		REG8_1(0x30)
-#define AUDIO_CONTROL3		0x38
-#define VIDEO_FIELD_CTRL	REG8_1(0x39)
-#define HSCALER_CTRL		REG8_1(0x42)
-#define VIDEO_SIZE		REG8_1(0x4A)
-#define VIDEO_SIZE_F2		REG8_1(0x52)
-#define MD_CONF			REG8_1(0x60)
-#define MD_INIT			REG8_1(0x68)
-#define MD_MAP0			REG8_1(0x70)
-#define VDMA_P_ADDR		REG8_8(0x80) /* not used in DMA SG mode */
-#define VDMA_WHP		REG8_8(0x81)
-#define VDMA_B_ADDR		REG8_8(0x82)
-#define VDMA_F2_P_ADDR		REG8_8(0x84)
-#define VDMA_F2_WHP		REG8_8(0x85)
-#define VDMA_F2_B_ADDR		REG8_8(0x86)
-#define EP_REG_ADDR		0xFE
-#define EP_REG_DATA		0xFF
-
-/* Video decoder registers */
-#define VDREG8(a0) ((const u16[8]) {			\
-	a0 + 0x000, a0 + 0x010, a0 + 0x020, a0 + 0x030,	\
-	a0 + 0x100, a0 + 0x110, a0 + 0x120, a0 + 0x130})
-#define VIDSTAT			VDREG8(0x100)
-#define BRIGHT			VDREG8(0x101)
-#define CONTRAST		VDREG8(0x102)
-#define SHARPNESS		VDREG8(0x103)
-#define SAT_U			VDREG8(0x104)
-#define SAT_V			VDREG8(0x105)
-#define HUE			VDREG8(0x106)
-#define CROP_HI			VDREG8(0x107)
-#define VDELAY_LO		VDREG8(0x108)
-#define VACTIVE_LO		VDREG8(0x109)
-#define HDELAY_LO		VDREG8(0x10A)
-#define HACTIVE_LO		VDREG8(0x10B)
-#define MVSN			VDREG8(0x10C)
-#define STATUS2			VDREG8(0x10C)
-#define SDT			VDREG8(0x10E)
-#define SDT_EN			VDREG8(0x10F)
-
-#define VSCALE_LO		VDREG8(0x144)
-#define SCALE_HI		VDREG8(0x145)
-#define HSCALE_LO		VDREG8(0x146)
-#define F2CROP_HI		VDREG8(0x147)
-#define F2VDELAY_LO		VDREG8(0x148)
-#define F2VACTIVE_LO		VDREG8(0x149)
-#define F2HDELAY_LO		VDREG8(0x14A)
-#define F2HACTIVE_LO		VDREG8(0x14B)
-#define F2VSCALE_LO		VDREG8(0x14C)
-#define F2SCALE_HI		VDREG8(0x14D)
-#define F2HSCALE_LO		VDREG8(0x14E)
-#define F2CNT			VDREG8(0x14F)
-
-#define VDREG2(a0) ((const u16[2]) {a0, a0 + 0x100})
-#define SRST			VDREG2(0x180)
-#define ACNTL			VDREG2(0x181)
-#define ACNTL2			VDREG2(0x182)
-#define CNTRL1			VDREG2(0x183)
-#define CKHY			VDREG2(0x184)
-#define SHCOR			VDREG2(0x185)
-#define CORING			VDREG2(0x186)
-#define CLMPG			VDREG2(0x187)
-#define IAGC			VDREG2(0x188)
-#define VCTRL1			VDREG2(0x18F)
-#define MISC1			VDREG2(0x194)
-#define LOOP			VDREG2(0x195)
-#define MISC2			VDREG2(0x196)
-
-#define CLMD			VDREG2(0x197)
-#define AIGAIN			((const u16[8]) {0x1D0, 0x1D1, 0x1D2, 0x1D3, \
-						 0x2D0, 0x2D1, 0x2D2, 0x2D3})
diff --git a/drivers/staging/media/tw686x-kh/tw686x-kh-video.c b/drivers/staging/media/tw686x-kh/tw686x-kh-video.c
deleted file mode 100644
index 9bf32aec2fc6..000000000000
--- a/drivers/staging/media/tw686x-kh/tw686x-kh-video.c
+++ /dev/null
@@ -1,813 +0,0 @@
-/*
- * Copyright (C) 2015 Industrial Research Institute for Automation
- * and Measurements PIAP
- *
- * Written by Krzysztof Ha?asa.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License
- * as published by the Free Software Foundation.
- */
-
-#include <linux/init.h>
-#include <linux/list.h>
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <media/v4l2-common.h>
-#include <media/v4l2-event.h>
-#include "tw686x-kh.h"
-#include "tw686x-kh-regs.h"
-
-#define MAX_SG_ENTRY_SIZE (/* 8192 - 128 */ 4096)
-#define MAX_SG_DESC_COUNT 256 /* PAL 704x576 needs up to 198 4-KB pages */
-
-static const struct tw686x_format formats[] = {
-	{
-		.name = "4:2:2 packed, UYVY", /* aka Y422 */
-		.fourcc = V4L2_PIX_FMT_UYVY,
-		.mode = 0,
-		.depth = 16,
-	}, {
-#if 0
-		.name = "4:2:0 packed, YUV",
-		.mode = 1,	/* non-standard */
-		.depth = 12,
-	}, {
-		.name = "4:1:1 packed, YUV",
-		.mode = 2,	/* non-standard */
-		.depth = 12,
-	}, {
-#endif
-		.name = "4:1:1 packed, YUV",
-		.fourcc = V4L2_PIX_FMT_Y41P,
-		.mode = 3,
-		.depth = 12,
-	}, {
-		.name = "15 bpp RGB",
-		.fourcc = V4L2_PIX_FMT_RGB555,
-		.mode = 4,
-		.depth = 16,
-	}, {
-		.name = "16 bpp RGB",
-		.fourcc = V4L2_PIX_FMT_RGB565,
-		.mode = 5,
-		.depth = 16,
-	}, {
-		.name = "4:2:2 packed, YUYV",
-		.fourcc = V4L2_PIX_FMT_YUYV,
-		.mode = 6,
-		.depth = 16,
-	}
-	/* mode 7 is "reserved" */
-};
-
-static const v4l2_std_id video_standards[7] = {
-	V4L2_STD_NTSC,
-	V4L2_STD_PAL,
-	V4L2_STD_SECAM,
-	V4L2_STD_NTSC_443,
-	V4L2_STD_PAL_M,
-	V4L2_STD_PAL_N,
-	V4L2_STD_PAL_60,
-};
-
-static const struct tw686x_format *format_by_fourcc(unsigned int fourcc)
-{
-	unsigned int cnt;
-
-	for (cnt = 0; cnt < ARRAY_SIZE(formats); cnt++)
-		if (formats[cnt].fourcc == fourcc)
-			return &formats[cnt];
-	return NULL;
-}
-
-static void tw686x_get_format(struct tw686x_video_channel *vc,
-			      struct v4l2_format *f)
-{
-	const struct tw686x_format *format;
-	unsigned int width, height, height_div = 1;
-
-	format = format_by_fourcc(f->fmt.pix.pixelformat);
-	if (!format) {
-		format = &formats[0];
-		f->fmt.pix.pixelformat = format->fourcc;
-	}
-
-	width = 704;
-	if (f->fmt.pix.width < width * 3 / 4 /* halfway */)
-		width /= 2;
-
-	height = (vc->video_standard & V4L2_STD_625_50) ? 576 : 480;
-	if (f->fmt.pix.height < height * 3 / 4 /* halfway */)
-		height_div = 2;
-
-	switch (f->fmt.pix.field) {
-	case V4L2_FIELD_TOP:
-	case V4L2_FIELD_BOTTOM:
-		height_div = 2;
-		break;
-	case V4L2_FIELD_SEQ_BT:
-		if (height_div > 1)
-			f->fmt.pix.field = V4L2_FIELD_BOTTOM;
-		break;
-	default:
-		if (height_div > 1)
-			f->fmt.pix.field = V4L2_FIELD_TOP;
-		else
-			f->fmt.pix.field = V4L2_FIELD_SEQ_TB;
-	}
-	height /= height_div;
-
-	f->fmt.pix.width = width;
-	f->fmt.pix.height = height;
-	f->fmt.pix.bytesperline = f->fmt.pix.width * format->depth / 8;
-	f->fmt.pix.sizeimage = f->fmt.pix.height * f->fmt.pix.bytesperline;
-	f->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
-}
-
-/* video queue operations */
-
-static int tw686x_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers,
-			      unsigned int *nplanes, unsigned int sizes[],
-			      struct device *alloc_devs[])
-{
-	struct tw686x_video_channel *vc = vb2_get_drv_priv(vq);
-	unsigned int size = vc->width * vc->height * vc->format->depth / 8;
-
-	if (*nbuffers < 2)
-		*nbuffers = 2;
-
-	if (*nplanes)
-		return sizes[0] < size ? -EINVAL : 0;
-
-	sizes[0] = size;
-	*nplanes = 1;		/* packed formats only */
-	return 0;
-}
-
-static void tw686x_buf_queue(struct vb2_buffer *vb)
-{
-	struct tw686x_video_channel *vc = vb2_get_drv_priv(vb->vb2_queue);
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct tw686x_vb2_buf *buf;
-
-	buf = container_of(vbuf, struct tw686x_vb2_buf, vb);
-
-	spin_lock(&vc->qlock);
-	list_add_tail(&buf->list, &vc->vidq_queued);
-	spin_unlock(&vc->qlock);
-}
-
-static void setup_descs(struct tw686x_video_channel *vc, unsigned int n)
-{
-loop:
-	while (!list_empty(&vc->vidq_queued)) {
-		struct vdma_desc *descs = vc->sg_descs[n];
-		struct tw686x_vb2_buf *buf;
-		struct sg_table *vbuf;
-		struct scatterlist *sg;
-		unsigned int buf_len, count = 0;
-		int i;
-
-		buf = list_first_entry(&vc->vidq_queued, struct tw686x_vb2_buf,
-				       list);
-		list_del(&buf->list);
-
-		buf_len = vc->width * vc->height * vc->format->depth / 8;
-		if (vb2_plane_size(&buf->vb.vb2_buf, 0) < buf_len) {
-			pr_err("Video buffer size too small\n");
-			vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
-			goto loop; /* try another */
-		}
-
-		vbuf = vb2_dma_sg_plane_desc(&buf->vb.vb2_buf, 0);
-		for_each_sg(vbuf->sgl, sg, vbuf->nents, i) {
-			dma_addr_t phys = sg_dma_address(sg);
-			unsigned int len = sg_dma_len(sg);
-
-			while (len && buf_len) {
-				unsigned int entry_len = min_t(unsigned int, len,
-							   MAX_SG_ENTRY_SIZE);
-				entry_len = min(entry_len, buf_len);
-				if (count == MAX_SG_DESC_COUNT) {
-					pr_err("Video buffer size too fragmented\n");
-					vb2_buffer_done(&buf->vb.vb2_buf,
-							VB2_BUF_STATE_ERROR);
-					goto loop;
-				}
-				descs[count].phys = cpu_to_le32(phys);
-				descs[count++].flags_length =
-					cpu_to_le32(0x40000000 /* available */ |
-						    entry_len);
-				phys += entry_len;
-				len -= entry_len;
-				buf_len -= entry_len;
-			}
-			if (!buf_len)
-				break;
-		}
-
-		/* clear the remaining entries */
-		while (count < MAX_SG_DESC_COUNT) {
-			descs[count].phys = 0;
-			descs[count++].flags_length = 0; /* unavailable */
-		}
-
-		buf->vb.vb2_buf.state = VB2_BUF_STATE_ACTIVE;
-		vc->curr_bufs[n] = buf;
-		return;
-	}
-	vc->curr_bufs[n] = NULL;
-}
-
-/* On TW6864 and TW6868, all channels share the pair of video DMA SG tables,
-   with 10-bit start_idx and end_idx determining start and end of frame buffer
-   for particular channel.
-   TW6868 with all its 8 channels would be problematic (only 127 SG entries per
-   channel) but we support only 4 channels on this chip anyway (the first
-   4 channels are driven with internal video decoder, the other 4 would require
-   an external TW286x part).
-
-   On TW6865 and TW6869, each channel has its own DMA SG table, with indexes
-   starting with 0. Both chips have complete sets of internal video decoders
-   (respectively 4 or 8-channel).
-
-   All chips have separate SG tables for two video frames. */
-
-static void setup_dma_cfg(struct tw686x_video_channel *vc)
-{
-	unsigned int field_width = 704;
-	unsigned int field_height = (vc->video_standard & V4L2_STD_625_50) ?
-		288 : 240;
-	unsigned int start_idx = is_second_gen(vc->dev) ? 0 :
-		vc->ch * MAX_SG_DESC_COUNT;
-	unsigned int end_idx = start_idx + MAX_SG_DESC_COUNT - 1;
-	u32 dma_cfg = (0 << 30) /* input selection */ |
-		(1 << 29) /* field2 dropped (if any) */ |
-		((vc->height < 300) << 28) /* field dropping */ |
-		(1 << 27) /* master */ |
-		(0 << 25) /* master channel (for slave only) */ |
-		(0 << 24) /* (no) vertical (line) decimation */ |
-		((vc->width < 400) << 23) /* horizontal decimation */ |
-		(vc->format->mode << 20) /* output video format */ |
-		(end_idx << 10) /* DMA end index */ |
-		start_idx /* DMA start index */;
-	u32 reg;
-
-	reg_write(vc->dev, VDMA_CHANNEL_CONFIG[vc->ch], dma_cfg);
-	reg_write(vc->dev, VIDEO_SIZE[vc->ch], (1 << 31) | (field_height << 16)
-		  | field_width);
-	reg = reg_read(vc->dev, VIDEO_CONTROL1);
-	if (vc->video_standard & V4L2_STD_625_50)
-		reg |= 1 << (vc->ch + 13);
-	else
-		reg &= ~(1 << (vc->ch + 13));
-	reg_write(vc->dev, VIDEO_CONTROL1, reg);
-}
-
-static int tw686x_start_streaming(struct vb2_queue *vq, unsigned int count)
-{
-	struct tw686x_video_channel *vc = vb2_get_drv_priv(vq);
-	struct tw686x_dev *dev = vc->dev;
-	u32 dma_ch_mask;
-	unsigned int n;
-
-	setup_dma_cfg(vc);
-
-	/* queue video buffers if available */
-	spin_lock(&vc->qlock);
-	for (n = 0; n < 2; n++)
-		setup_descs(vc, n);
-	spin_unlock(&vc->qlock);
-
-	dev->video_active |= 1 << vc->ch;
-	vc->seq = 0;
-	dma_ch_mask = reg_read(dev, DMA_CHANNEL_ENABLE) | (1 << vc->ch);
-	reg_write(dev, DMA_CHANNEL_ENABLE, dma_ch_mask);
-	reg_write(dev, DMA_CMD, (1 << 31) | dma_ch_mask);
-	return 0;
-}
-
-static void tw686x_stop_streaming(struct vb2_queue *vq)
-{
-	struct tw686x_video_channel *vc = vb2_get_drv_priv(vq);
-	struct tw686x_dev *dev = vc->dev;
-	u32 dma_ch_mask = reg_read(dev, DMA_CHANNEL_ENABLE);
-	u32 dma_cmd = reg_read(dev, DMA_CMD);
-	unsigned int n;
-
-	dma_ch_mask &= ~(1 << vc->ch);
-	reg_write(dev, DMA_CHANNEL_ENABLE, dma_ch_mask);
-
-	dev->video_active &= ~(1 << vc->ch);
-
-	dma_cmd &= ~(1 << vc->ch);
-	reg_write(dev, DMA_CMD, dma_cmd);
-
-	if (!dev->video_active) {
-		reg_write(dev, DMA_CMD, 0);
-		reg_write(dev, DMA_CHANNEL_ENABLE, 0);
-	}
-
-	spin_lock(&vc->qlock);
-	while (!list_empty(&vc->vidq_queued)) {
-		struct tw686x_vb2_buf *buf;
-
-		buf = list_entry(vc->vidq_queued.next, struct tw686x_vb2_buf,
-				 list);
-		list_del(&buf->list);
-		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
-	}
-
-	for (n = 0; n < 2; n++)
-		if (vc->curr_bufs[n])
-			vb2_buffer_done(&vc->curr_bufs[n]->vb.vb2_buf,
-					VB2_BUF_STATE_ERROR);
-
-	spin_unlock(&vc->qlock);
-}
-
-static struct vb2_ops tw686x_video_qops = {
-	.queue_setup		= tw686x_queue_setup,
-	.buf_queue		= tw686x_buf_queue,
-	.start_streaming	= tw686x_start_streaming,
-	.stop_streaming		= tw686x_stop_streaming,
-	.wait_prepare		= vb2_ops_wait_prepare,
-	.wait_finish		= vb2_ops_wait_finish,
-};
-
-static int tw686x_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct tw686x_video_channel *vc;
-	struct tw686x_dev *dev;
-	unsigned int ch;
-
-	vc = container_of(ctrl->handler, struct tw686x_video_channel,
-			  ctrl_handler);
-	dev = vc->dev;
-	ch = vc->ch;
-
-	switch (ctrl->id) {
-	case V4L2_CID_BRIGHTNESS:
-		reg_write(dev, BRIGHT[ch], ctrl->val & 0xFF);
-		return 0;
-
-	case V4L2_CID_CONTRAST:
-		reg_write(dev, CONTRAST[ch], ctrl->val);
-		return 0;
-
-	case V4L2_CID_SATURATION:
-		reg_write(dev, SAT_U[ch], ctrl->val);
-		reg_write(dev, SAT_V[ch], ctrl->val);
-		return 0;
-
-	case V4L2_CID_HUE:
-		reg_write(dev, HUE[ch], ctrl->val & 0xFF);
-		return 0;
-	}
-
-	return -EINVAL;
-}
-
-static const struct v4l2_ctrl_ops ctrl_ops = {
-	.s_ctrl = tw686x_s_ctrl,
-};
-
-static int tw686x_g_fmt_vid_cap(struct file *file, void *priv,
-				struct v4l2_format *f)
-{
-	struct tw686x_video_channel *vc = video_drvdata(file);
-
-	f->fmt.pix.width = vc->width;
-	f->fmt.pix.height = vc->height;
-	f->fmt.pix.field = vc->field;
-	f->fmt.pix.pixelformat = vc->format->fourcc;
-	f->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
-	f->fmt.pix.bytesperline = f->fmt.pix.width * vc->format->depth / 8;
-	f->fmt.pix.sizeimage = f->fmt.pix.height * f->fmt.pix.bytesperline;
-	return 0;
-}
-
-static int tw686x_try_fmt_vid_cap(struct file *file, void *priv,
-				  struct v4l2_format *f)
-{
-	tw686x_get_format(video_drvdata(file), f);
-	return 0;
-}
-
-static int tw686x_s_fmt_vid_cap(struct file *file, void *priv,
-				struct v4l2_format *f)
-{
-	struct tw686x_video_channel *vc = video_drvdata(file);
-
-	tw686x_get_format(vc, f);
-	vc->format = format_by_fourcc(f->fmt.pix.pixelformat);
-	vc->field = f->fmt.pix.field;
-	vc->width = f->fmt.pix.width;
-	vc->height = f->fmt.pix.height;
-	return 0;
-}
-
-static int tw686x_querycap(struct file *file, void *priv,
-			   struct v4l2_capability *cap)
-{
-	struct tw686x_video_channel *vc = video_drvdata(file);
-	struct tw686x_dev *dev = vc->dev;
-
-	strcpy(cap->driver, "tw686x-kh");
-	strcpy(cap->card, dev->name);
-	sprintf(cap->bus_info, "PCI:%s", pci_name(dev->pci_dev));
-	cap->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
-	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
-	return 0;
-}
-
-static int tw686x_s_std(struct file *file, void *priv, v4l2_std_id id)
-{
-	struct tw686x_video_channel *vc = video_drvdata(file);
-	unsigned int cnt;
-	u32 sdt = 0; /* default */
-
-	for (cnt = 0; cnt < ARRAY_SIZE(video_standards); cnt++)
-		if (id & video_standards[cnt]) {
-			sdt = cnt;
-			break;
-		}
-
-	reg_write(vc->dev, SDT[vc->ch], sdt);
-	vc->video_standard = video_standards[sdt];
-	return 0;
-}
-
-static int tw686x_g_std(struct file *file, void *priv, v4l2_std_id *id)
-{
-	struct tw686x_video_channel *vc = video_drvdata(file);
-
-	*id = vc->video_standard;
-	return 0;
-}
-
-static int tw686x_enum_fmt_vid_cap(struct file *file, void *priv,
-				   struct v4l2_fmtdesc *f)
-{
-	if (f->index >= ARRAY_SIZE(formats))
-		return -EINVAL;
-
-	strlcpy(f->description, formats[f->index].name, sizeof(f->description));
-	f->pixelformat = formats[f->index].fourcc;
-	return 0;
-}
-
-static int tw686x_g_parm(struct file *file, void *priv,
-			 struct v4l2_streamparm *sp)
-{
-	struct tw686x_video_channel *vc = video_drvdata(file);
-
-	if (sp->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-	memset(&sp->parm.capture, 0, sizeof(sp->parm.capture));
-	sp->parm.capture.capability = V4L2_CAP_TIMEPERFRAME;
-	v4l2_video_std_frame_period(vc->video_standard,
-				    &sp->parm.capture.timeperframe);
-
-	return 0;
-}
-
-static int tw686x_enum_input(struct file *file, void *priv,
-			     struct v4l2_input *inp)
-{
-	/* the chip has internal multiplexer, support can be added
-	   if the actual hw uses it */
-	if (inp->index)
-		return -EINVAL;
-
-	snprintf(inp->name, sizeof(inp->name), "Composite");
-	inp->type = V4L2_INPUT_TYPE_CAMERA;
-	inp->std = V4L2_STD_ALL;
-	inp->capabilities = V4L2_IN_CAP_STD;
-	return 0;
-}
-
-static int tw686x_g_input(struct file *file, void *priv, unsigned int *v)
-{
-	*v = 0;
-	return 0;
-}
-
-static int tw686x_s_input(struct file *file, void *priv, unsigned int v)
-{
-	if (v)
-		return -EINVAL;
-	return 0;
-}
-
-static const struct v4l2_file_operations tw686x_video_fops = {
-	.owner		= THIS_MODULE,
-	.open		= v4l2_fh_open,
-	.unlocked_ioctl	= video_ioctl2,
-	.release	= vb2_fop_release,
-	.poll		= vb2_fop_poll,
-	.read		= vb2_fop_read,
-	.mmap		= vb2_fop_mmap,
-};
-
-static const struct v4l2_ioctl_ops tw686x_video_ioctl_ops = {
-	.vidioc_querycap		= tw686x_querycap,
-	.vidioc_enum_fmt_vid_cap	= tw686x_enum_fmt_vid_cap,
-	.vidioc_g_fmt_vid_cap		= tw686x_g_fmt_vid_cap,
-	.vidioc_s_fmt_vid_cap		= tw686x_s_fmt_vid_cap,
-	.vidioc_try_fmt_vid_cap		= tw686x_try_fmt_vid_cap,
-	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
-	.vidioc_querybuf		= vb2_ioctl_querybuf,
-	.vidioc_qbuf			= vb2_ioctl_qbuf,
-	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
-	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
-	.vidioc_streamon		= vb2_ioctl_streamon,
-	.vidioc_streamoff		= vb2_ioctl_streamoff,
-	.vidioc_g_std			= tw686x_g_std,
-	.vidioc_s_std			= tw686x_s_std,
-	.vidioc_g_parm			= tw686x_g_parm,
-	.vidioc_enum_input		= tw686x_enum_input,
-	.vidioc_g_input			= tw686x_g_input,
-	.vidioc_s_input			= tw686x_s_input,
-	.vidioc_subscribe_event		= v4l2_ctrl_subscribe_event,
-	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
-};
-
-static int video_thread(void *arg)
-{
-	struct tw686x_dev *dev = arg;
-	DECLARE_WAITQUEUE(wait, current);
-
-	set_freezable();
-	add_wait_queue(&dev->video_thread_wait, &wait);
-
-	while (1) {
-		long timeout = schedule_timeout_interruptible(HZ);
-		unsigned int ch;
-
-		if (timeout == -ERESTARTSYS || kthread_should_stop())
-			break;
-
-		for (ch = 0; ch < max_channels(dev); ch++) {
-			struct tw686x_video_channel *vc;
-			unsigned long flags;
-			u32 request, n, stat = VB2_BUF_STATE_DONE;
-
-			vc = &dev->video_channels[ch];
-			if (!(dev->video_active & (1 << ch)))
-				continue;
-
-			spin_lock_irq(&dev->irq_lock);
-			request = dev->dma_requests & (0x01000001 << ch);
-			if (request)
-				dev->dma_requests &= ~request;
-			spin_unlock_irq(&dev->irq_lock);
-
-			if (!request)
-				continue;
-
-			request >>= ch;
-
-			/* handle channel events */
-			if ((request & 0x01000000) |
-			    (reg_read(dev, VIDEO_FIFO_STATUS) & (0x01010001 << ch)) |
-			    (reg_read(dev, VIDEO_PARSER_STATUS) & (0x00000101 << ch))) {
-				/* DMA Errors - reset channel */
-				u32 reg;
-
-				spin_lock_irqsave(&dev->irq_lock, flags);
-				reg = reg_read(dev, DMA_CMD);
-				/* Reset DMA channel */
-				reg_write(dev, DMA_CMD, reg & ~(1 << ch));
-				reg_write(dev, DMA_CMD, reg);
-				spin_unlock_irqrestore(&dev->irq_lock, flags);
-				stat = VB2_BUF_STATE_ERROR;
-			}
-
-			/* handle video stream */
-			mutex_lock(&vc->vb_mutex);
-			spin_lock(&vc->qlock);
-			n = !!(reg_read(dev, PB_STATUS) & (1 << ch));
-			if (vc->curr_bufs[n]) {
-				struct vb2_v4l2_buffer *vb;
-
-				vb = &vc->curr_bufs[n]->vb;
-				vb->vb2_buf.timestamp = ktime_get_ns();
-				vb->field = vc->field;
-				if (V4L2_FIELD_HAS_BOTH(vc->field))
-					vb->sequence = vc->seq++;
-				else
-					vb->sequence = (vc->seq++) / 2;
-				vb2_set_plane_payload(&vb->vb2_buf, 0,
-				      vc->width * vc->height * vc->format->depth / 8);
-				vb2_buffer_done(&vb->vb2_buf, stat);
-			}
-			setup_descs(vc, n);
-			spin_unlock(&vc->qlock);
-			mutex_unlock(&vc->vb_mutex);
-		}
-		try_to_freeze();
-	}
-
-	remove_wait_queue(&dev->video_thread_wait, &wait);
-	return 0;
-}
-
-int tw686x_kh_video_irq(struct tw686x_dev *dev)
-{
-	unsigned long flags, handled = 0;
-	u32 requests;
-
-	spin_lock_irqsave(&dev->irq_lock, flags);
-	requests = dev->dma_requests;
-	spin_unlock_irqrestore(&dev->irq_lock, flags);
-
-	if (requests & dev->video_active) {
-		wake_up_interruptible_all(&dev->video_thread_wait);
-		handled = 1;
-	}
-	return handled;
-}
-
-void tw686x_kh_video_free(struct tw686x_dev *dev)
-{
-	unsigned int ch, n;
-
-	if (dev->video_thread)
-		kthread_stop(dev->video_thread);
-
-	for (ch = 0; ch < max_channels(dev); ch++) {
-		struct tw686x_video_channel *vc = &dev->video_channels[ch];
-
-		v4l2_ctrl_handler_free(&vc->ctrl_handler);
-		if (vc->device)
-			video_unregister_device(vc->device);
-		for (n = 0; n < 2; n++) {
-			struct dma_desc *descs = &vc->sg_tables[n];
-
-			if (descs->virt)
-				pci_free_consistent(dev->pci_dev, descs->size,
-						    descs->virt, descs->phys);
-		}
-	}
-
-	v4l2_device_unregister(&dev->v4l2_dev);
-}
-
-#define SG_TABLE_SIZE (MAX_SG_DESC_COUNT * sizeof(struct vdma_desc))
-
-int tw686x_kh_video_init(struct tw686x_dev *dev)
-{
-	unsigned int ch, n;
-	int err;
-
-	init_waitqueue_head(&dev->video_thread_wait);
-
-	err = v4l2_device_register(&dev->pci_dev->dev, &dev->v4l2_dev);
-	if (err)
-		return err;
-
-	reg_write(dev, VIDEO_CONTROL1, 0); /* NTSC, disable scaler */
-	reg_write(dev, PHASE_REF, 0x00001518); /* Scatter-gather DMA mode */
-
-	/* setup required SG table sizes */
-	for (n = 0; n < 2; n++)
-		if (is_second_gen(dev)) {
-			/* TW 6865, TW6869 - each channel needs a pair of
-			   descriptor tables */
-			for (ch = 0; ch < max_channels(dev); ch++)
-				dev->video_channels[ch].sg_tables[n].size =
-					SG_TABLE_SIZE;
-
-		} else
-			/* TW 6864, TW6868 - we need to allocate a pair of
-			   descriptor tables, common for all channels.
-			   Each table will be bigger than 4 KB. */
-			dev->video_channels[0].sg_tables[n].size =
-				max_channels(dev) * SG_TABLE_SIZE;
-
-	/* allocate SG tables and initialize video channels */
-	for (ch = 0; ch < max_channels(dev); ch++) {
-		struct tw686x_video_channel *vc = &dev->video_channels[ch];
-		struct video_device *vdev;
-
-		mutex_init(&vc->vb_mutex);
-		spin_lock_init(&vc->qlock);
-		INIT_LIST_HEAD(&vc->vidq_queued);
-
-		vc->dev = dev;
-		vc->ch = ch;
-
-		/* default settings: NTSC */
-		vc->format = &formats[0];
-		vc->video_standard = V4L2_STD_NTSC;
-		reg_write(vc->dev, SDT[vc->ch], 0);
-		vc->field = V4L2_FIELD_SEQ_BT;
-		vc->width = 704;
-		vc->height = 480;
-
-		for (n = 0; n < 2; n++) {
-			void *cpu;
-
-			if (vc->sg_tables[n].size) {
-				unsigned int reg = n ? DMA_PAGE_TABLE1_ADDR[ch] :
-					DMA_PAGE_TABLE0_ADDR[ch];
-
-				cpu = pci_alloc_consistent(dev->pci_dev,
-							   vc->sg_tables[n].size,
-							   &vc->sg_tables[n].phys);
-				if (!cpu) {
-					pr_err("Error allocating video DMA scatter-gather tables\n");
-					err = -ENOMEM;
-					goto error;
-				}
-				vc->sg_tables[n].virt = cpu;
-				reg_write(dev, reg, vc->sg_tables[n].phys);
-			} else
-				cpu = dev->video_channels[0].sg_tables[n].virt +
-					ch * SG_TABLE_SIZE;
-
-			vc->sg_descs[n] = cpu;
-		}
-
-		reg_write(dev, VCTRL1[0], 0x24);
-		reg_write(dev, LOOP[0], 0xA5);
-		if (max_channels(dev) > 4) {
-			reg_write(dev, VCTRL1[1], 0x24);
-			reg_write(dev, LOOP[1], 0xA5);
-		}
-		reg_write(dev, VIDEO_FIELD_CTRL[ch], 0);
-		reg_write(dev, VDELAY_LO[ch], 0x14);
-
-		vdev = video_device_alloc();
-		if (!vdev) {
-			pr_warn("Unable to allocate video device\n");
-			err = -ENOMEM;
-			goto error;
-		}
-
-		vc->vidq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-		vc->vidq.io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
-		vc->vidq.drv_priv = vc;
-		vc->vidq.buf_struct_size = sizeof(struct tw686x_vb2_buf);
-		vc->vidq.ops = &tw686x_video_qops;
-		vc->vidq.mem_ops = &vb2_dma_sg_memops;
-		vc->vidq.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
-		vc->vidq.min_buffers_needed = 2;
-		vc->vidq.lock = &vc->vb_mutex;
-		vc->vidq.dev = &dev->pci_dev->dev;
-		vc->vidq.gfp_flags = GFP_DMA32;
-
-		err = vb2_queue_init(&vc->vidq);
-		if (err)
-			goto error;
-
-		strcpy(vdev->name, "TW686x-video");
-		snprintf(vdev->name, sizeof(vdev->name), "%s video", dev->name);
-		vdev->fops = &tw686x_video_fops;
-		vdev->ioctl_ops = &tw686x_video_ioctl_ops;
-		vdev->release = video_device_release;
-		vdev->v4l2_dev = &dev->v4l2_dev;
-		vdev->queue = &vc->vidq;
-		vdev->tvnorms = V4L2_STD_ALL;
-		vdev->minor = -1;
-		vdev->lock = &vc->vb_mutex;
-
-		dev->video_channels[ch].device = vdev;
-		video_set_drvdata(vdev, vc);
-		err = video_register_device(vdev, VFL_TYPE_GRABBER, -1);
-		if (err < 0)
-			goto error;
-
-		v4l2_ctrl_handler_init(&vc->ctrl_handler,
-				       4 /* number of controls */);
-		vdev->ctrl_handler = &vc->ctrl_handler;
-		v4l2_ctrl_new_std(&vc->ctrl_handler, &ctrl_ops,
-				  V4L2_CID_BRIGHTNESS, -128, 127, 1, 0);
-		v4l2_ctrl_new_std(&vc->ctrl_handler, &ctrl_ops,
-				  V4L2_CID_CONTRAST, 0, 255, 1, 64);
-		v4l2_ctrl_new_std(&vc->ctrl_handler, &ctrl_ops,
-				  V4L2_CID_SATURATION, 0, 255, 1, 128);
-		v4l2_ctrl_new_std(&vc->ctrl_handler, &ctrl_ops, V4L2_CID_HUE,
-				  -124, 127, 1, 0);
-		err = vc->ctrl_handler.error;
-		if (err)
-			goto error;
-
-		v4l2_ctrl_handler_setup(&vc->ctrl_handler);
-	}
-
-	dev->video_thread = kthread_run(video_thread, dev, "tw686x_video");
-	if (IS_ERR(dev->video_thread)) {
-		err = PTR_ERR(dev->video_thread);
-		goto error;
-	}
-
-	return 0;
-
-error:
-	tw686x_kh_video_free(dev);
-	return err;
-}
diff --git a/drivers/staging/media/tw686x-kh/tw686x-kh.h b/drivers/staging/media/tw686x-kh/tw686x-kh.h
deleted file mode 100644
index 6284a90d6fe3..000000000000
--- a/drivers/staging/media/tw686x-kh/tw686x-kh.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright (C) 2015 Industrial Research Institute for Automation
- * and Measurements PIAP
- *
- * Written by Krzysztof Ha?asa.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License
- * as published by the Free Software Foundation.
- */
-
-#include <linux/delay.h>
-#include <linux/freezer.h>
-#include <linux/interrupt.h>
-#include <linux/kthread.h>
-#include <linux/mutex.h>
-#include <linux/pci.h>
-#include <media/videobuf2-dma-sg.h>
-#include <linux/videodev2.h>
-#include <media/v4l2-common.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-ioctl.h>
-
-#define TYPE_MAX_CHANNELS 0x0F
-#define TYPE_SECOND_GEN   0x10
-
-struct tw686x_format {
-	char *name;
-	unsigned int fourcc;
-	unsigned int depth;
-	unsigned int mode;
-};
-
-struct dma_desc {
-	dma_addr_t phys;
-	void *virt;
-	unsigned int size;
-};
-
-struct vdma_desc {
-	__le32 flags_length;	/* 3 MSBits for flags, 13 LSBits for length */
-	__le32 phys;
-};
-
-struct tw686x_vb2_buf {
-	struct vb2_v4l2_buffer vb;
-	struct list_head list;
-};
-
-struct tw686x_video_channel {
-	struct tw686x_dev *dev;
-
-	struct vb2_queue vidq;
-	struct list_head vidq_queued;
-	struct video_device *device;
-	struct dma_desc sg_tables[2];
-	struct tw686x_vb2_buf *curr_bufs[2];
-	struct vdma_desc *sg_descs[2];
-
-	struct v4l2_ctrl_handler ctrl_handler;
-	const struct tw686x_format *format;
-	struct mutex vb_mutex;
-	spinlock_t qlock;
-	v4l2_std_id video_standard;
-	unsigned int width, height;
-	enum v4l2_field field; /* supported TOP, BOTTOM, SEQ_TB and SEQ_BT */
-	unsigned int seq;	       /* video field or frame counter */
-	unsigned int ch;
-};
-
-/* global device status */
-struct tw686x_dev {
-	spinlock_t irq_lock;
-
-	struct v4l2_device v4l2_dev;
-	struct snd_card *card;	/* sound card */
-
-	unsigned int video_active;	/* active video channel mask */
-
-	char name[32];
-	unsigned int type;
-	struct pci_dev *pci_dev;
-	__u32 __iomem *mmio;
-
-	struct task_struct *video_thread;
-	wait_queue_head_t video_thread_wait;
-	u32 dma_requests;
-
-	struct tw686x_video_channel video_channels[0];
-};
-
-static inline uint32_t reg_read(struct tw686x_dev *dev, unsigned int reg)
-{
-	return readl(dev->mmio + reg);
-}
-
-static inline void reg_write(struct tw686x_dev *dev, unsigned int reg,
-			     uint32_t value)
-{
-	writel(value, dev->mmio + reg);
-}
-
-static inline unsigned int max_channels(struct tw686x_dev *dev)
-{
-	return dev->type & TYPE_MAX_CHANNELS; /* 4 or 8 channels */
-}
-
-static inline unsigned int is_second_gen(struct tw686x_dev *dev)
-{
-	/* each channel has its own DMA SG table */
-	return dev->type & TYPE_SECOND_GEN;
-}
-
-int tw686x_kh_video_irq(struct tw686x_dev *dev);
-int tw686x_kh_video_init(struct tw686x_dev *dev);
-void tw686x_kh_video_free(struct tw686x_dev *dev);
diff --git a/drivers/staging/most/Documentation/ABI/sysfs-class-most.txt b/drivers/staging/most/Documentation/ABI/sysfs-class-most.txt
index 42ff0d8b2ead..48aa45acc953 100644
--- a/drivers/staging/most/Documentation/ABI/sysfs-class-most.txt
+++ b/drivers/staging/most/Documentation/ABI/sysfs-class-most.txt
@@ -51,6 +51,140 @@ Description:
 		uses.
 Users:
 
+What:		/sys/class/most/mostcore/devices/<mdev>/dci
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		If the network interface controller is attached via USB, a dci
+		directory is created that allows applications to use the
+		controller's direct communication interface (DCI) to exchange
+		information.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/arb_address
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to set an arbitrary DCI register address an
+		application wants to read from or write to.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/arb_value
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to read from or write to the arbitrary DCI register
+		whose address is stored in arb_address.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/mep_eui48_hi
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to check and configure the MAC address.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/mep_eui48_lo
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to check and configure the MAC address.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/mep_eui48_mi
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to check and configure the MAC address.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/mep_filter
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to check and configure the MEP filter address.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/mep_hash0
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to check and configure the MEP hash table.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/mep_hash1
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to check and configure the MEP hash table.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/mep_hash2
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to check and configure the MEP hash table.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/mep_hash3
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		This is used to check and configure the MEP hash table.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/ni_state
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		Indicates the current network interface state.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/node_address
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		Indicates the current node address.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/node_position
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		Indicates the current node position.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/packet_bandwidth
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		Indicates the configured packet bandwidth.
+Users:
+
+What:		/sys/class/most/mostcore/devices/<mdev>/dci/sync_ep
+Date:		June 2016
+KernelVersion:	4.9
+Contact:	Christian Gromm <christian.gromm@microchip.com>
+Description:
+		Triggers the controller's synchronization process for a certain
+		endpoint.
+Users:
+
 What:		/sys/class/most/mostcore/devices/<mdev>/<channel>/
 Date:		June 2015
 KernelVersion:	4.3
diff --git a/drivers/staging/most/aim-cdev/cdev.c b/drivers/staging/most/aim-cdev/cdev.c
index de4f76abfb47..7f51024dc5eb 100644
--- a/drivers/staging/most/aim-cdev/cdev.c
+++ b/drivers/staging/most/aim-cdev/cdev.c
@@ -57,7 +57,11 @@ static inline bool ch_has_mbo(struct aim_channel *c)
 
 static inline bool ch_get_mbo(struct aim_channel *c, struct mbo **mbo)
 {
-	*mbo = most_get_mbo(c->iface, c->channel_id, &cdev_aim);
+	if (!kfifo_peek(&c->fifo, mbo)) {
+		*mbo = most_get_mbo(c->iface, c->channel_id, &cdev_aim);
+		if (*mbo)
+			kfifo_in(&c->fifo, mbo, 1);
+	}
 	return *mbo;
 }
 
@@ -130,7 +134,7 @@ static int aim_open(struct inode *inode, struct file *filp)
 	if (!c->dev) {
 		pr_info("WARN: Device is destroyed\n");
 		mutex_unlock(&c->io_mutex);
-		return -EBUSY;
+		return -ENODEV;
 	}
 
 	if (c->access_ref) {
@@ -184,8 +188,7 @@ static ssize_t aim_write(struct file *filp, const char __user *buf,
 			 size_t count, loff_t *offset)
 {
 	int ret;
-	size_t actual_len;
-	size_t max_len;
+	size_t to_copy, left;
 	struct mbo *mbo = NULL;
 	struct aim_channel *c = filp->private_data;
 
@@ -201,27 +204,28 @@ static ssize_t aim_write(struct file *filp, const char __user *buf,
 	}
 
 	if (unlikely(!c->dev)) {
-		ret = -EPIPE;
+		ret = -ENODEV;
 		goto unlock;
 	}
 
-	max_len = c->cfg->buffer_size;
-	actual_len = min(count, max_len);
-	mbo->buffer_length = actual_len;
-
-	if (copy_from_user(mbo->virt_address, buf, mbo->buffer_length)) {
+	to_copy = min(count, c->cfg->buffer_size - c->mbo_offs);
+	left = copy_from_user(mbo->virt_address + c->mbo_offs, buf, to_copy);
+	if (left == to_copy) {
 		ret = -EFAULT;
-		goto put_mbo;
+		goto unlock;
 	}
 
-	ret = most_submit_mbo(mbo);
-	if (ret)
-		goto put_mbo;
+	c->mbo_offs += to_copy - left;
+	if (c->mbo_offs >= c->cfg->buffer_size ||
+	    c->cfg->data_type == MOST_CH_CONTROL ||
+	    c->cfg->data_type == MOST_CH_ASYNC) {
+		kfifo_skip(&c->fifo);
+		mbo->buffer_length = c->mbo_offs;
+		c->mbo_offs = 0;
+		most_submit_mbo(mbo);
+	}
 
-	mutex_unlock(&c->io_mutex);
-	return actual_len;
-put_mbo:
-	most_put_mbo(mbo);
+	ret = to_copy - left;
 unlock:
 	mutex_unlock(&c->io_mutex);
 	return ret;
@@ -256,7 +260,7 @@ aim_read(struct file *filp, char __user *buf, size_t count, loff_t *offset)
 	/* make sure we don't submit to gone devices */
 	if (unlikely(!c->dev)) {
 		mutex_unlock(&c->io_mutex);
-		return -EIO;
+		return -ENODEV;
 	}
 
 	to_copy = min_t(size_t,
@@ -290,7 +294,7 @@ static unsigned int aim_poll(struct file *filp, poll_table *wait)
 		if (!kfifo_is_empty(&c->fifo))
 			mask |= POLLIN | POLLRDNORM;
 	} else {
-		if (ch_has_mbo(c))
+		if (!kfifo_is_empty(&c->fifo) || ch_has_mbo(c))
 			mask |= POLLOUT | POLLWRNORM;
 	}
 	return mask;
@@ -366,7 +370,7 @@ static int aim_rx_completion(struct mbo *mbo)
 	spin_lock(&c->unlink);
 	if (!c->access_ref || !c->dev) {
 		spin_unlock(&c->unlink);
-		return -EFAULT;
+		return -ENODEV;
 	}
 	kfifo_in(&c->fifo, &mbo, 1);
 	spin_unlock(&c->unlink);
@@ -499,23 +503,27 @@ static struct most_aim cdev_aim = {
 
 static int __init mod_init(void)
 {
+	int err;
+
 	pr_info("init()\n");
 
 	INIT_LIST_HEAD(&channel_list);
 	spin_lock_init(&ch_list_lock);
 	ida_init(&minor_id);
 
-	if (alloc_chrdev_region(&aim_devno, 0, 50, "cdev") < 0)
-		return -EIO;
+	err = alloc_chrdev_region(&aim_devno, 0, 50, "cdev");
+	if (err < 0)
+		goto dest_ida;
 	major = MAJOR(aim_devno);
 
 	aim_class = class_create(THIS_MODULE, "most_cdev_aim");
 	if (IS_ERR(aim_class)) {
 		pr_err("no udev support\n");
+		err = PTR_ERR(aim_class);
 		goto free_cdev;
 	}
-
-	if (most_register_aim(&cdev_aim))
+	err = most_register_aim(&cdev_aim);
+	if (err)
 		goto dest_class;
 	return 0;
 
@@ -523,7 +531,9 @@ dest_class:
 	class_destroy(aim_class);
 free_cdev:
 	unregister_chrdev_region(aim_devno, 1);
-	return -EIO;
+dest_ida:
+	ida_destroy(&minor_id);
+	return err;
 }
 
 static void __exit mod_exit(void)
diff --git a/drivers/staging/most/aim-network/networking.c b/drivers/staging/most/aim-network/networking.c
index 2f42de44d051..4659a6450c04 100644
--- a/drivers/staging/most/aim-network/networking.c
+++ b/drivers/staging/most/aim-network/networking.c
@@ -298,15 +298,16 @@ static struct net_dev_context *get_net_dev_context(
 	struct most_interface *iface)
 {
 	struct net_dev_context *nd, *tmp;
+	unsigned long flags;
 
-	spin_lock(&list_lock);
+	spin_lock_irqsave(&list_lock, flags);
 	list_for_each_entry_safe(nd, tmp, &net_devices, list) {
 		if (nd->iface == iface) {
-			spin_unlock(&list_lock);
+			spin_unlock_irqrestore(&list_lock, flags);
 			return nd;
 		}
 	}
-	spin_unlock(&list_lock);
+	spin_unlock_irqrestore(&list_lock, flags);
 	return NULL;
 }
 
@@ -316,6 +317,7 @@ static int aim_probe_channel(struct most_interface *iface, int channel_idx,
 {
 	struct net_dev_context *nd;
 	struct net_dev_channel *ch;
+	unsigned long flags;
 
 	if (!iface)
 		return -EINVAL;
@@ -332,9 +334,9 @@ static int aim_probe_channel(struct most_interface *iface, int channel_idx,
 
 		nd->iface = iface;
 
-		spin_lock(&list_lock);
+		spin_lock_irqsave(&list_lock, flags);
 		list_add(&nd->list, &net_devices);
-		spin_unlock(&list_lock);
+		spin_unlock_irqrestore(&list_lock, flags);
 	}
 
 	ch = ccfg->direction == MOST_CH_TX ? &nd->tx : &nd->rx;
@@ -377,6 +379,7 @@ static int aim_disconnect_channel(struct most_interface *iface,
 {
 	struct net_dev_context *nd;
 	struct net_dev_channel *ch;
+	unsigned long flags;
 
 	nd = get_net_dev_context(iface);
 	if (!nd)
@@ -398,9 +401,9 @@ static int aim_disconnect_channel(struct most_interface *iface,
 	most_net_rm_netdev_safe(nd);
 
 	if (!nd->rx.linked && !nd->tx.linked) {
-		spin_lock(&list_lock);
+		spin_lock_irqsave(&list_lock, flags);
 		list_del(&nd->list);
-		spin_unlock(&list_lock);
+		spin_unlock_irqrestore(&list_lock, flags);
 		kfree(nd);
 	}
 
@@ -514,20 +517,21 @@ static int __init most_net_init(void)
 static void __exit most_net_exit(void)
 {
 	struct net_dev_context *nd, *tmp;
+	unsigned long flags;
 
-	spin_lock(&list_lock);
+	spin_lock_irqsave(&list_lock, flags);
 	list_for_each_entry_safe(nd, tmp, &net_devices, list) {
 		list_del(&nd->list);
-		spin_unlock(&list_lock);
+		spin_unlock_irqrestore(&list_lock, flags);
 		/*
 		 * do not call most_stop_channel() here, because channels are
 		 * going to be closed in ndo_stop() after unregister_netdev()
 		 */
 		most_net_rm_netdev_safe(nd);
 		kfree(nd);
-		spin_lock(&list_lock);
+		spin_lock_irqsave(&list_lock, flags);
 	}
-	spin_unlock(&list_lock);
+	spin_unlock_irqrestore(&list_lock, flags);
 
 	most_deregister_aim(&aim);
 	pr_info("most_net_exit()\n");
diff --git a/drivers/staging/most/aim-sound/sound.c b/drivers/staging/most/aim-sound/sound.c
index 9c645801cff4..e4198e5e064b 100644
--- a/drivers/staging/most/aim-sound/sound.c
+++ b/drivers/staging/most/aim-sound/sound.c
@@ -234,7 +234,6 @@ static int playback_thread(void *data)
 	while (!kthread_should_stop()) {
 		struct mbo *mbo = NULL;
 		bool period_elapsed = false;
-		int ret;
 
 		wait_event_interruptible(
 			channel->playback_waitq,
@@ -250,10 +249,7 @@ static int playback_thread(void *data)
 		else
 			memset(mbo->virt_address, 0, mbo->buffer_length);
 
-		ret = most_submit_mbo(mbo);
-		if (ret)
-			channel->is_stream_running = false;
-
+		most_submit_mbo(mbo);
 		if (period_elapsed)
 			snd_pcm_period_elapsed(channel->substream);
 	}
@@ -457,7 +453,7 @@ static snd_pcm_uframes_t pcm_pointer(struct snd_pcm_substream *substream)
 /**
  * Initialization of struct snd_pcm_ops
  */
-static struct snd_pcm_ops pcm_ops = {
+static const struct snd_pcm_ops pcm_ops = {
 	.open       = pcm_open,
 	.close      = pcm_close,
 	.ioctl      = snd_pcm_lib_ioctl,
@@ -611,7 +607,8 @@ static int audio_probe_channel(struct most_interface *iface, int channel_id,
 	channel->id = channel_id;
 	init_waitqueue_head(&channel->playback_waitq);
 
-	if (audio_set_hw_params(&channel->pcm_hardware, pcm_format, cfg))
+	ret = audio_set_hw_params(&channel->pcm_hardware, pcm_format, cfg);
+	if (ret)
 		goto err_free_card;
 
 	snprintf(card->driver, sizeof(card->driver), "%s", DRIVER_NAME);
diff --git a/drivers/staging/most/aim-v4l2/video.c b/drivers/staging/most/aim-v4l2/video.c
index 13abf7c275a4..e0748416aee5 100644
--- a/drivers/staging/most/aim-v4l2/video.c
+++ b/drivers/staging/most/aim-v4l2/video.c
@@ -79,7 +79,7 @@ static int aim_vdev_open(struct file *filp)
 	struct most_video_dev *mdev = video_drvdata(filp);
 	struct aim_fh *fh;
 
-	pr_info("aim_vdev_open()\n");
+	v4l2_info(&mdev->v4l2_dev, "aim_vdev_open()\n");
 
 	switch (vdev->vfl_type) {
 	case VFL_TYPE_GRABBER:
@@ -93,7 +93,7 @@ static int aim_vdev_open(struct file *filp)
 		return -ENOMEM;
 
 	if (!atomic_inc_and_test(&mdev->access_ref)) {
-		pr_err("too many clients\n");
+		v4l2_err(&mdev->v4l2_dev, "too many clients\n");
 		ret = -EBUSY;
 		goto err_dec;
 	}
@@ -106,7 +106,7 @@ static int aim_vdev_open(struct file *filp)
 
 	ret = most_start_channel(mdev->iface, mdev->ch_idx, &aim_info);
 	if (ret) {
-		pr_err("most_start_channel() failed\n");
+		v4l2_err(&mdev->v4l2_dev, "most_start_channel() failed\n");
 		goto err_rm;
 	}
 
@@ -128,7 +128,7 @@ static int aim_vdev_close(struct file *filp)
 	struct most_video_dev *mdev = fh->mdev;
 	struct mbo *mbo, *tmp;
 
-	pr_info("aim_vdev_close()\n");
+	v4l2_info(&mdev->v4l2_dev, "aim_vdev_close()\n");
 
 	/*
 	 * We need to put MBOs back before we call most_stop_channel()
@@ -139,15 +139,15 @@ static int aim_vdev_close(struct file *filp)
 	 * This must be implemented in core.
 	 */
 
-	spin_lock(&mdev->list_lock);
+	spin_lock_irq(&mdev->list_lock);
 	mdev->mute = true;
 	list_for_each_entry_safe(mbo, tmp, &mdev->pending_mbos, list) {
 		list_del(&mbo->list);
-		spin_unlock(&mdev->list_lock);
+		spin_unlock_irq(&mdev->list_lock);
 		most_put_mbo(mbo);
-		spin_lock(&mdev->list_lock);
+		spin_lock_irq(&mdev->list_lock);
 	}
-	spin_unlock(&mdev->list_lock);
+	spin_unlock_irq(&mdev->list_lock);
 	most_stop_channel(mdev->iface, mdev->ch_idx, &aim_info);
 	mdev->mute = false;
 
@@ -187,7 +187,7 @@ static ssize_t aim_vdev_read(struct file *filp, char __user *buf,
 		int const cnt = rem < count ? rem : count;
 
 		if (copy_to_user(buf, mbo->virt_address + fh->offs, cnt)) {
-			pr_err("read: copy_to_user failed\n");
+			v4l2_err(&mdev->v4l2_dev, "read: copy_to_user failed\n");
 			if (!ret)
 				ret = -EFAULT;
 			return ret;
@@ -200,9 +200,9 @@ static ssize_t aim_vdev_read(struct file *filp, char __user *buf,
 
 		if (cnt >= rem) {
 			fh->offs = 0;
-			spin_lock(&mdev->list_lock);
+			spin_lock_irq(&mdev->list_lock);
 			list_del(&mbo->list);
-			spin_unlock(&mdev->list_lock);
+			spin_unlock_irq(&mdev->list_lock);
 			most_put_mbo(mbo);
 		}
 	}
@@ -250,16 +250,16 @@ static int aim_set_format(struct most_video_dev *mdev, unsigned int cmd,
 	return 0;
 }
 
-static int vidioc_querycap(struct file *file, void  *priv,
+static int vidioc_querycap(struct file *file, void *priv,
 			   struct v4l2_capability *cap)
 {
 	struct aim_fh *fh = priv;
 	struct most_video_dev *mdev = fh->mdev;
 
-	pr_info("vidioc_querycap()\n");
+	v4l2_info(&mdev->v4l2_dev, "vidioc_querycap()\n");
 
 	strlcpy(cap->driver, "v4l2_most_aim", sizeof(cap->driver));
-	strlcpy(cap->card, "my_card", sizeof(cap->card));
+	strlcpy(cap->card, "MOST", sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info),
 		 "%s", mdev->iface->description);
 
@@ -270,10 +270,13 @@ static int vidioc_querycap(struct file *file, void  *priv,
 	return 0;
 }
 
-static int vidioc_enum_fmt_vid_cap(struct file *file, void  *priv,
+static int vidioc_enum_fmt_vid_cap(struct file *file, void *priv,
 				   struct v4l2_fmtdesc *f)
 {
-	pr_info("vidioc_enum_fmt_vid_cap() %d\n", f->index);
+	struct aim_fh *fh = priv;
+	struct most_video_dev *mdev = fh->mdev;
+
+	v4l2_info(&mdev->v4l2_dev, "vidioc_enum_fmt_vid_cap() %d\n", f->index);
 
 	if (f->index)
 		return -EINVAL;
@@ -289,7 +292,10 @@ static int vidioc_enum_fmt_vid_cap(struct file *file, void  *priv,
 static int vidioc_g_fmt_vid_cap(struct file *file, void *priv,
 				struct v4l2_format *f)
 {
-	pr_info("vidioc_g_fmt_vid_cap()\n");
+	struct aim_fh *fh = priv;
+	struct most_video_dev *mdev = fh->mdev;
+
+	v4l2_info(&mdev->v4l2_dev, "vidioc_g_fmt_vid_cap()\n");
 
 	aim_set_format_struct(f);
 	return 0;
@@ -298,7 +304,7 @@ static int vidioc_g_fmt_vid_cap(struct file *file, void *priv,
 static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 				  struct v4l2_format *f)
 {
-	struct aim_fh *fh  = priv;
+	struct aim_fh *fh = priv;
 	struct most_video_dev *mdev = fh->mdev;
 
 	return aim_set_format(mdev, VIDIOC_TRY_FMT, f);
@@ -307,7 +313,7 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 static int vidioc_s_fmt_vid_cap(struct file *file, void *priv,
 				struct v4l2_format *f)
 {
-	struct aim_fh *fh  = priv;
+	struct aim_fh *fh = priv;
 	struct most_video_dev *mdev = fh->mdev;
 
 	return aim_set_format(mdev, VIDIOC_S_FMT, f);
@@ -315,7 +321,10 @@ static int vidioc_s_fmt_vid_cap(struct file *file, void *priv,
 
 static int vidioc_g_std(struct file *file, void *priv, v4l2_std_id *norm)
 {
-	pr_info("vidioc_g_std()\n");
+	struct aim_fh *fh = priv;
+	struct most_video_dev *mdev = fh->mdev;
+
+	v4l2_info(&mdev->v4l2_dev, "vidioc_g_std()\n");
 
 	*norm = V4L2_STD_UNKNOWN;
 	return 0;
@@ -352,7 +361,7 @@ static int vidioc_s_input(struct file *file, void *priv, unsigned int index)
 	struct aim_fh *fh = priv;
 	struct most_video_dev *mdev = fh->mdev;
 
-	pr_info("vidioc_s_input(%d)\n", index);
+	v4l2_info(&mdev->v4l2_dev, "vidioc_s_input(%d)\n", index);
 
 	if (index >= V4L2_AIM_MAX_INPUT)
 		return -EINVAL;
@@ -360,7 +369,7 @@ static int vidioc_s_input(struct file *file, void *priv, unsigned int index)
 	return 0;
 }
 
-static struct v4l2_file_operations aim_fops = {
+static const struct v4l2_file_operations aim_fops = {
 	.owner      = THIS_MODULE,
 	.open       = aim_vdev_open,
 	.release    = aim_vdev_close,
@@ -393,45 +402,46 @@ static const struct video_device aim_videodev_template = {
 static struct most_video_dev *get_aim_dev(
 	struct most_interface *iface, int channel_idx)
 {
-	struct most_video_dev *mdev, *tmp;
+	struct most_video_dev *mdev;
+	unsigned long flags;
 
-	spin_lock(&list_lock);
-	list_for_each_entry_safe(mdev, tmp, &video_devices, list) {
+	spin_lock_irqsave(&list_lock, flags);
+	list_for_each_entry(mdev, &video_devices, list) {
 		if (mdev->iface == iface && mdev->ch_idx == channel_idx) {
-			spin_unlock(&list_lock);
+			spin_unlock_irqrestore(&list_lock, flags);
 			return mdev;
 		}
 	}
-	spin_unlock(&list_lock);
+	spin_unlock_irqrestore(&list_lock, flags);
 	return NULL;
 }
 
 static int aim_rx_data(struct mbo *mbo)
 {
+	unsigned long flags;
 	struct most_video_dev *mdev =
 		get_aim_dev(mbo->ifp, mbo->hdm_channel_id);
 
 	if (!mdev)
 		return -EIO;
 
-	spin_lock(&mdev->list_lock);
+	spin_lock_irqsave(&mdev->list_lock, flags);
 	if (unlikely(mdev->mute)) {
-		spin_unlock(&mdev->list_lock);
+		spin_unlock_irqrestore(&mdev->list_lock, flags);
 		return -EIO;
 	}
 
 	list_add_tail(&mbo->list, &mdev->pending_mbos);
-	spin_unlock(&mdev->list_lock);
+	spin_unlock_irqrestore(&mdev->list_lock, flags);
 	wake_up_interruptible(&mdev->wait_data);
 	return 0;
 }
 
 static int aim_register_videodev(struct most_video_dev *mdev)
 {
-	int retval = -ENOMEM;
 	int ret;
 
-	pr_info("aim_register_videodev()\n");
+	v4l2_info(&mdev->v4l2_dev, "aim_register_videodev()\n");
 
 	init_waitqueue_head(&mdev->wait_data);
 
@@ -444,27 +454,24 @@ static int aim_register_videodev(struct most_video_dev *mdev)
 	*mdev->vdev = aim_videodev_template;
 	mdev->vdev->v4l2_dev = &mdev->v4l2_dev;
 	mdev->vdev->lock = &mdev->lock;
-	strcpy(mdev->vdev->name, "most v4l2 aim video");
+	snprintf(mdev->vdev->name, sizeof(mdev->vdev->name), "MOST: %s",
+		 mdev->v4l2_dev.name);
 
 	/* Register the v4l2 device */
 	video_set_drvdata(mdev->vdev, mdev);
-	retval = video_register_device(mdev->vdev, VFL_TYPE_GRABBER, -1);
-	if (retval != 0) {
-		pr_err("video_register_device failed (%d)\n", retval);
-		ret = -ENODEV;
-		goto err_vbi_dev;
+	ret = video_register_device(mdev->vdev, VFL_TYPE_GRABBER, -1);
+	if (ret) {
+		v4l2_err(&mdev->v4l2_dev, "video_register_device failed (%d)\n",
+			 ret);
+		video_device_release(mdev->vdev);
 	}
 
-	return 0;
-
-err_vbi_dev:
-	video_device_release(mdev->vdev);
 	return ret;
 }
 
 static void aim_unregister_videodev(struct most_video_dev *mdev)
 {
-	pr_info("aim_unregister_videodev()\n");
+	v4l2_info(&mdev->v4l2_dev, "aim_unregister_videodev()\n");
 
 	video_unregister_device(mdev->vdev);
 }
@@ -485,7 +492,7 @@ static int aim_probe_channel(struct most_interface *iface, int channel_idx,
 	int ret;
 	struct most_video_dev *mdev = get_aim_dev(iface, channel_idx);
 
-	pr_info("aim_probe_channel()\n");
+	pr_info("aim_probe_channel(%s)\n", name);
 
 	if (mdev) {
 		pr_err("channel already linked\n");
@@ -498,8 +505,8 @@ static int aim_probe_channel(struct most_interface *iface, int channel_idx,
 	}
 
 	if (ccfg->data_type != MOST_CH_SYNC &&
-	    ccfg->data_type != MOST_CH_ISOC_AVP) {
-		pr_err("wrong channel type, expect sync or isoc_avp\n");
+	    ccfg->data_type != MOST_CH_ISOC) {
+		pr_err("wrong channel type, expect sync or isoc\n");
 		return -EINVAL;
 	}
 
@@ -516,8 +523,7 @@ static int aim_probe_channel(struct most_interface *iface, int channel_idx,
 	mdev->v4l2_dev.release = aim_v4l2_dev_release;
 
 	/* Create the v4l2_device */
-	strlcpy(mdev->v4l2_dev.name, "most_video_device",
-		sizeof(mdev->v4l2_dev.name));
+	strlcpy(mdev->v4l2_dev.name, name, sizeof(mdev->v4l2_dev.name));
 	ret = v4l2_device_register(NULL, &mdev->v4l2_dev);
 	if (ret) {
 		pr_err("v4l2_device_register() failed\n");
@@ -529,9 +535,10 @@ static int aim_probe_channel(struct most_interface *iface, int channel_idx,
 	if (ret)
 		goto err_unreg;
 
-	spin_lock(&list_lock);
+	spin_lock_irq(&list_lock);
 	list_add(&mdev->list, &video_devices);
-	spin_unlock(&list_lock);
+	spin_unlock_irq(&list_lock);
+	v4l2_info(&mdev->v4l2_dev, "aim_probe_channel() done\n");
 	return 0;
 
 err_unreg:
@@ -545,16 +552,16 @@ static int aim_disconnect_channel(struct most_interface *iface,
 {
 	struct most_video_dev *mdev = get_aim_dev(iface, channel_idx);
 
-	pr_info("aim_disconnect_channel()\n");
-
 	if (!mdev) {
 		pr_err("no such channel is linked\n");
 		return -ENOENT;
 	}
 
-	spin_lock(&list_lock);
+	v4l2_info(&mdev->v4l2_dev, "aim_disconnect_channel()\n");
+
+	spin_lock_irq(&list_lock);
 	list_del(&mdev->list);
-	spin_unlock(&list_lock);
+	spin_unlock_irq(&list_lock);
 
 	aim_unregister_videodev(mdev);
 	v4l2_device_disconnect(&mdev->v4l2_dev);
@@ -585,17 +592,17 @@ static void __exit aim_exit(void)
 	 * we simulate this call here.
 	 * This must be fixed in core.
 	 */
-	spin_lock(&list_lock);
+	spin_lock_irq(&list_lock);
 	list_for_each_entry_safe(mdev, tmp, &video_devices, list) {
 		list_del(&mdev->list);
-		spin_unlock(&list_lock);
+		spin_unlock_irq(&list_lock);
 
 		aim_unregister_videodev(mdev);
 		v4l2_device_disconnect(&mdev->v4l2_dev);
 		v4l2_device_put(&mdev->v4l2_dev);
-		spin_lock(&list_lock);
+		spin_lock_irq(&list_lock);
 	}
-	spin_unlock(&list_lock);
+	spin_unlock_irq(&list_lock);
 
 	most_deregister_aim(&aim_info);
 	BUG_ON(!list_empty(&video_devices));
diff --git a/drivers/staging/most/hdm-dim2/dim2_hal.c b/drivers/staging/most/hdm-dim2/dim2_hal.c
index 3c524506ee22..0b9816ce1761 100644
--- a/drivers/staging/most/hdm-dim2/dim2_hal.c
+++ b/drivers/staging/most/hdm-dim2/dim2_hal.c
@@ -2,7 +2,7 @@
  * dim2_hal.c - DIM2 HAL implementation
  * (MediaLB, Device Interface Macro IP, OS62420)
  *
- * Copyright (C) 2015, Microchip Technology Germany II GmbH & Co. KG
+ * Copyright (C) 2015-2016, Microchip Technology Germany II GmbH & Co. KG
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -20,18 +20,6 @@
 #include <linux/stddef.h>
 
 /*
- * The number of frames per sub-buffer for synchronous channels.
- * Allowed values: 1, 2, 4, 8, 16, 32, 64.
- */
-#define FRAMES_PER_SUBBUFF 16
-
-/*
- * Size factor for synchronous DBR buffer.
- * Minimal value is 4*FRAMES_PER_SUBBUFF.
- */
-#define SYNC_DBR_FACTOR (4u * (u16)FRAMES_PER_SUBBUFF)
-
-/*
  * Size factor for isochronous DBR buffer.
  * Minimal value is 3.
  */
@@ -61,12 +49,11 @@
 #define DBR_SIZE  (16 * 1024) /* specified by IP */
 #define DBR_BLOCK_SIZE  (DBR_SIZE / 32 / DBR_MAP_SIZE)
 
+#define ROUND_UP_TO(x, d)  (((x) + (d) - 1) / (d) * (d))
+
 /* -------------------------------------------------------------------------- */
 /* generic helper functions and macros */
 
-#define MLBC0_FCNT_VAL_MACRO(n) MLBC0_FCNT_VAL_ ## n ## FPSB
-#define MLBC0_FCNT_VAL(fpsb) MLBC0_FCNT_VAL_MACRO(fpsb)
-
 static inline u32 bit_mask(u8 position)
 {
 	return (u32)1 << position;
@@ -81,10 +68,20 @@ static inline bool dim_on_error(u8 error_id, const char *error_message)
 /* -------------------------------------------------------------------------- */
 /* types and local variables */
 
+struct async_tx_dbr {
+	u8 ch_addr;
+	u16 rpc;
+	u16 wpc;
+	u16 rest_size;
+	u16 sz_queue[CDT0_RPC_MASK + 1];
+};
+
 struct lld_global_vars_t {
 	bool dim_is_initialized;
 	bool mcm_is_initialized;
 	struct dim2_regs __iomem *dim2; /* DIM2 core base address */
+	struct async_tx_dbr atx_dbr;
+	u32 fcnt;
 	u32 dbr_map[DBR_MAP_SIZE];
 };
 
@@ -149,15 +146,34 @@ static void free_dbr(int offs, int size)
 
 /* -------------------------------------------------------------------------- */
 
-static u32 dim2_read_ctr(u32 ctr_addr, u16 mdat_idx)
+static void dim2_transfer_madr(u32 val)
 {
-	dimcb_io_write(&g.dim2->MADR, ctr_addr);
+	dimcb_io_write(&g.dim2->MADR, val);
 
-	/* wait till transfer is completed */
+	/* wait for transfer completion */
 	while ((dimcb_io_read(&g.dim2->MCTL) & 1) != 1)
 		continue;
 
 	dimcb_io_write(&g.dim2->MCTL, 0);   /* clear transfer complete */
+}
+
+static void dim2_clear_dbr(u16 addr, u16 size)
+{
+	enum { MADR_TB_BIT = 30, MADR_WNR_BIT = 31 };
+
+	u16 const end_addr = addr + size;
+	u32 const cmd = bit_mask(MADR_WNR_BIT) | bit_mask(MADR_TB_BIT);
+
+	dimcb_io_write(&g.dim2->MCTL, 0);   /* clear transfer complete */
+	dimcb_io_write(&g.dim2->MDAT0, 0);
+
+	for (; addr < end_addr; addr++)
+		dim2_transfer_madr(cmd | addr);
+}
+
+static u32 dim2_read_ctr(u32 ctr_addr, u16 mdat_idx)
+{
+	dim2_transfer_madr(ctr_addr);
 
 	return dimcb_io_read((&g.dim2->MDAT0) + mdat_idx);
 }
@@ -182,13 +198,7 @@ static void dim2_write_ctr_mask(u32 ctr_addr, const u32 *mask, const u32 *value)
 	dimcb_io_write(&g.dim2->MDWE2, mask[2]);
 	dimcb_io_write(&g.dim2->MDWE3, mask[3]);
 
-	dimcb_io_write(&g.dim2->MADR, bit_mask(MADR_WNR_BIT) | ctr_addr);
-
-	/* wait till transfer is completed */
-	while ((dimcb_io_read(&g.dim2->MCTL) & 1) != 1)
-		continue;
-
-	dimcb_io_write(&g.dim2->MCTL, 0);   /* clear transfer complete */
+	dim2_transfer_madr(bit_mask(MADR_WNR_BIT) | ctr_addr);
 }
 
 static inline void dim2_write_ctr(u32 ctr_addr, const u32 *value)
@@ -252,6 +262,13 @@ static void dim2_configure_cdt(u8 ch_addr, u16 dbr_address, u16 hw_buffer_size,
 	dim2_write_ctr(CDT + ch_addr, cdt);
 }
 
+static u16 dim2_rpc(u8 ch_addr)
+{
+	u32 cdt0 = dim2_read_ctr(CDT + ch_addr, 0);
+
+	return (cdt0 >> CDT0_RPC_SHIFT) & CDT0_RPC_MASK;
+}
+
 static void dim2_clear_cdt(u8 ch_addr)
 {
 	u32 cdt[4] = { 0, 0, 0, 0 };
@@ -356,6 +373,52 @@ static void dim2_clear_channel(u8 ch_addr)
 
 	dim2_clear_cat(MLB_CAT, ch_addr);
 	dim2_clear_cdt(ch_addr);
+
+	/* clear channel status bit */
+	dimcb_io_write(&g.dim2->ACSR0, bit_mask(ch_addr));
+}
+
+/* -------------------------------------------------------------------------- */
+/* trace async tx dbr fill state */
+
+static inline u16 norm_pc(u16 pc)
+{
+	return pc & CDT0_RPC_MASK;
+}
+
+static void dbrcnt_init(u8 ch_addr, u16 dbr_size)
+{
+	g.atx_dbr.rest_size = dbr_size;
+	g.atx_dbr.rpc = dim2_rpc(ch_addr);
+	g.atx_dbr.wpc = g.atx_dbr.rpc;
+}
+
+static void dbrcnt_enq(int buf_sz)
+{
+	g.atx_dbr.rest_size -= buf_sz;
+	g.atx_dbr.sz_queue[norm_pc(g.atx_dbr.wpc)] = buf_sz;
+	g.atx_dbr.wpc++;
+}
+
+u16 dim_dbr_space(struct dim_channel *ch)
+{
+	u16 cur_rpc;
+	struct async_tx_dbr *dbr = &g.atx_dbr;
+
+	if (ch->addr != dbr->ch_addr)
+		return 0xFFFF;
+
+	cur_rpc = dim2_rpc(ch->addr);
+
+	while (norm_pc(dbr->rpc) != cur_rpc) {
+		dbr->rest_size += dbr->sz_queue[norm_pc(dbr->rpc)];
+		dbr->rpc++;
+	}
+
+	if ((u16)(dbr->wpc - dbr->rpc) >= CDT0_RPC_MASK)
+		return 0;
+
+	return dbr->rest_size;
 }
 
 /* -------------------------------------------------------------------------- */
@@ -398,7 +461,8 @@ static inline bool check_packet_length(u32 packet_length)
 
 static inline bool check_bytes_per_frame(u32 bytes_per_frame)
 {
-	u16 const max_size = ((u16)CDT3_BD_MASK + 1u) / SYNC_DBR_FACTOR;
+	u16 const bd_factor = g.fcnt + 2;
+	u16 const max_size = ((u16)CDT3_BD_MASK + 1u) >> bd_factor;
 
 	if (bytes_per_frame <= 0)
 		return false; /* too small */
@@ -439,7 +503,7 @@ static inline u16 norm_sync_buffer_size(u16 buf_size, u16 bytes_per_frame)
 {
 	u16 n;
 	u16 const max_size = (u16)ADT1_ISOC_SYNC_BD_MASK + 1u;
-	u32 const unit = bytes_per_frame * (u16)FRAMES_PER_SUBBUFF;
+	u32 const unit = bytes_per_frame << g.fcnt;
 
 	if (buf_size > max_size)
 		buf_size = max_size;
@@ -479,7 +543,7 @@ static void dim2_initialize(bool enable_6pin, u8 mlb_clock)
 	dimcb_io_write(&g.dim2->MLBC0,
 		       enable_6pin << MLBC0_MLBPEN_BIT |
 		       mlb_clock << MLBC0_MLBCLK_SHIFT |
-		       MLBC0_FCNT_VAL(FRAMES_PER_SUBBUFF) << MLBC0_FCNT_SHIFT |
+		       g.fcnt << MLBC0_FCNT_SHIFT |
 		       true << MLBC0_MLBEN_BIT);
 
 	/* activate all HBI channels */
@@ -515,20 +579,17 @@ static inline bool service_channel(u8 ch_addr, u8 idx)
 {
 	u8 const shift = idx * 16;
 	u32 const adt1 = dim2_read_ctr(ADT + ch_addr, 1);
+	u32 mask[4] = { 0, 0, 0, 0 };
+	u32 adt_w[4] = { 0, 0, 0, 0 };
 
 	if (((adt1 >> (ADT1_DNE_BIT + shift)) & 1) == 0)
 		return false;
 
-	{
-		u32 mask[4] = { 0, 0, 0, 0 };
-		u32 adt_w[4] = { 0, 0, 0, 0 };
-
-		mask[1] =
-			bit_mask(ADT1_DNE_BIT + shift) |
-			bit_mask(ADT1_ERR_BIT + shift) |
-			bit_mask(ADT1_RDY_BIT + shift);
-		dim2_write_ctr_mask(ADT + ch_addr, mask, adt_w);
-	}
+	mask[1] =
+		bit_mask(ADT1_DNE_BIT + shift) |
+		bit_mask(ADT1_ERR_BIT + shift) |
+		bit_mask(ADT1_RDY_BIT + shift);
+	dim2_write_ctr_mask(ADT + ch_addr, mask, adt_w);
 
 	/* clear channel status bit */
 	dimcb_io_write(&g.dim2->ACSR0, bit_mask(ch_addr));
@@ -612,6 +673,9 @@ static bool channel_start(struct dim_channel *ch, u32 buf_addr, u16 buf_size)
 
 	++state->level;
 
+	if (ch->addr == g.atx_dbr.ch_addr)
+		dbrcnt_enq(buf_size);
+
 	if (ch->packet_length || ch->bytes_per_frame)
 		dim2_start_isoc_sync(ch->addr, state->idx1, buf_addr, buf_size);
 	else
@@ -650,7 +714,8 @@ static bool channel_detach_buffers(struct dim_channel *ch, u16 buffers_number)
 /* -------------------------------------------------------------------------- */
 /* API */
 
-u8 dim_startup(struct dim2_regs __iomem *dim_base_address, u32 mlb_clock)
+u8 dim_startup(struct dim2_regs __iomem *dim_base_address, u32 mlb_clock,
+	       u32 fcnt)
 {
 	g.dim_is_initialized = false;
 
@@ -662,7 +727,11 @@ u8 dim_startup(struct dim2_regs __iomem *dim_base_address, u32 mlb_clock)
 	if (mlb_clock >= 8)
 		return DIM_INIT_ERR_MLB_CLOCK;
 
+	if (fcnt > MLBC0_FCNT_MAX_VAL)
+		return DIM_INIT_ERR_MLB_CLOCK;
+
 	g.dim2 = dim_base_address;
+	g.fcnt = fcnt;
 	g.dbr_map[0] = 0;
 	g.dbr_map[1] = 0;
 
@@ -693,7 +762,7 @@ static u8 init_ctrl_async(struct dim_channel *ch, u8 type, u8 is_tx,
 	if (!check_channel_address(ch_address))
 		return DIM_INIT_ERR_CHANNEL_ADDRESS;
 
-	ch->dbr_size = hw_buffer_size;
+	ch->dbr_size = ROUND_UP_TO(hw_buffer_size, DBR_BLOCK_SIZE);
 	ch->dbr_addr = alloc_dbr(ch->dbr_size);
 	if (ch->dbr_addr >= DBR_SIZE)
 		return DIM_INIT_ERR_OUT_OF_MEMORY;
@@ -706,6 +775,12 @@ static u8 init_ctrl_async(struct dim_channel *ch, u8 type, u8 is_tx,
 	return DIM_NO_ERROR;
 }
 
+void dim_service_mlb_int_irq(void)
+{
+	dimcb_io_write(&g.dim2->MS0, 0);
+	dimcb_io_write(&g.dim2->MS1, 0);
+}
+
 u16 dim_norm_ctrl_async_buffer_size(u16 buf_size)
 {
 	return norm_ctrl_async_buffer_size(buf_size);
@@ -749,8 +824,16 @@ u8 dim_init_control(struct dim_channel *ch, u8 is_tx, u16 ch_address,
 u8 dim_init_async(struct dim_channel *ch, u8 is_tx, u16 ch_address,
 		  u16 max_buffer_size)
 {
-	return init_ctrl_async(ch, CAT_CT_VAL_ASYNC, is_tx, ch_address,
-			       max_buffer_size);
+	u8 ret = init_ctrl_async(ch, CAT_CT_VAL_ASYNC, is_tx, ch_address,
+				 max_buffer_size);
+
+	if (is_tx && !g.atx_dbr.ch_addr) {
+		g.atx_dbr.ch_addr = ch->addr;
+		dbrcnt_init(ch->addr, ch->dbr_size);
+		dimcb_io_write(&g.dim2->MIEN, bit_mask(20));
+	}
+
+	return ret;
 }
 
 u8 dim_init_isoc(struct dim_channel *ch, u8 is_tx, u16 ch_address,
@@ -781,6 +864,8 @@ u8 dim_init_isoc(struct dim_channel *ch, u8 is_tx, u16 ch_address,
 u8 dim_init_sync(struct dim_channel *ch, u8 is_tx, u16 ch_address,
 		 u16 bytes_per_frame)
 {
+	u16 bd_factor = g.fcnt + 2;
+
 	if (!g.dim_is_initialized || !ch)
 		return DIM_ERR_DRIVER_NOT_INITIALIZED;
 
@@ -790,13 +875,14 @@ u8 dim_init_sync(struct dim_channel *ch, u8 is_tx, u16 ch_address,
 	if (!check_bytes_per_frame(bytes_per_frame))
 		return DIM_ERR_BAD_CONFIG;
 
-	ch->dbr_size = bytes_per_frame * SYNC_DBR_FACTOR;
+	ch->dbr_size = bytes_per_frame << bd_factor;
 	ch->dbr_addr = alloc_dbr(ch->dbr_size);
 	if (ch->dbr_addr >= DBR_SIZE)
 		return DIM_INIT_ERR_OUT_OF_MEMORY;
 
 	sync_init(ch, ch_address / 2, bytes_per_frame);
 
+	dim2_clear_dbr(ch->dbr_addr, ch->dbr_size);
 	dim2_configure_channel(ch->addr, CAT_CT_VAL_SYNC, is_tx,
 			       ch->dbr_addr, ch->dbr_size, 0, true);
 
@@ -808,6 +894,11 @@ u8 dim_destroy_channel(struct dim_channel *ch)
 	if (!g.dim_is_initialized || !ch)
 		return DIM_ERR_DRIVER_NOT_INITIALIZED;
 
+	if (ch->addr == g.atx_dbr.ch_addr) {
+		dimcb_io_write(&g.dim2->MIEN, 0);
+		g.atx_dbr.ch_addr = 0;
+	}
+
 	dim2_clear_channel(ch->addr);
 	if (ch->dbr_addr < DBR_SIZE)
 		free_dbr(ch->dbr_addr, ch->dbr_size);
@@ -816,7 +907,7 @@ u8 dim_destroy_channel(struct dim_channel *ch)
 	return DIM_NO_ERROR;
 }
 
-void dim_service_irq(struct dim_channel *const *channels)
+void dim_service_ahb_int_irq(struct dim_channel *const *channels)
 {
 	bool state_changed;
 
@@ -848,10 +939,6 @@ void dim_service_irq(struct dim_channel *const *channels)
 			++ch;
 		}
 	} while (state_changed);
-
-	/* clear pending Interrupts */
-	dimcb_io_write(&g.dim2->MS0, 0);
-	dimcb_io_write(&g.dim2->MS1, 0);
 }
 
 u8 dim_service_channel(struct dim_channel *ch)
diff --git a/drivers/staging/most/hdm-dim2/dim2_hal.h b/drivers/staging/most/hdm-dim2/dim2_hal.h
index 1c924e869de7..6df6ea5f7da4 100644
--- a/drivers/staging/most/hdm-dim2/dim2_hal.h
+++ b/drivers/staging/most/hdm-dim2/dim2_hal.h
@@ -60,7 +60,8 @@ struct dim_channel {
 	u16 done_sw_buffers_number; /*< Done software buffers number. */
 };
 
-u8 dim_startup(struct dim2_regs __iomem *dim_base_address, u32 mlb_clock);
+u8 dim_startup(struct dim2_regs __iomem *dim_base_address, u32 mlb_clock,
+	       u32 fcnt);
 
 void dim_shutdown(void);
 
@@ -86,13 +87,17 @@ u8 dim_init_sync(struct dim_channel *ch, u8 is_tx, u16 ch_address,
 
 u8 dim_destroy_channel(struct dim_channel *ch);
 
-void dim_service_irq(struct dim_channel *const *channels);
+void dim_service_mlb_int_irq(void);
+
+void dim_service_ahb_int_irq(struct dim_channel *const *channels);
 
 u8 dim_service_channel(struct dim_channel *ch);
 
 struct dim_ch_state_t *dim_get_channel_state(struct dim_channel *ch,
 					     struct dim_ch_state_t *state_ptr);
 
+u16 dim_dbr_space(struct dim_channel *ch);
+
 bool dim_enqueue_buffer(struct dim_channel *ch, u32 buffer_addr,
 			u16 buffer_size);
 
diff --git a/drivers/staging/most/hdm-dim2/dim2_hdm.c b/drivers/staging/most/hdm-dim2/dim2_hdm.c
index a36449551513..78b2c3dd9bb3 100644
--- a/drivers/staging/most/hdm-dim2/dim2_hdm.c
+++ b/drivers/staging/most/hdm-dim2/dim2_hdm.c
@@ -1,7 +1,7 @@
 /*
  * dim2_hdm.c - MediaLB DIM2 Hardware Dependent Module
  *
- * Copyright (C) 2015, Microchip Technology Germany II GmbH & Co. KG
+ * Copyright (C) 2015-2016, Microchip Technology Germany II GmbH & Co. KG
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -45,15 +45,15 @@ module_param(clock_speed, charp, 0);
 MODULE_PARM_DESC(clock_speed, "MediaLB Clock Speed");
 
 /*
- * #############################################################################
+ * The parameter representing the number of frames per sub-buffer for
+ * synchronous channels.  Valid values: [0 .. 6].
  *
- * The define below activates an utility function used by HAL-simu
- * for calling DIM interrupt handler.
- * It is used only for TEST PURPOSE and shall be commented before release.
- *
- * #############################################################################
+ * The values 0, 1, 2, 3, 4, 5, 6 represent corresponding number of frames per
+ * sub-buffer 1, 2, 4, 8, 16, 32, 64.
  */
-/* #define ENABLE_HDM_TEST */
+static u8 fcnt = 4;  /* (1 << fcnt) frames per subbuffer */
+module_param(fcnt, byte, 0);
+MODULE_PARM_DESC(fcnt, "Num of frames per sub-buffer for sync channels as a power of 2");
 
 static DEFINE_SPINLOCK(dim_lock);
 
@@ -85,7 +85,6 @@ struct hdm_channel {
  * @most_iface: most interface structure
  * @capabilities: an array of channel capability data
  * @io_base: I/O register base address
- * @irq_ahb0: dim2 AHB0 irq number
  * @clk_speed: user selectable (through command line parameter) clock speed
  * @netinfo_task: thread to deliver network status
  * @netinfo_waitq: waitq for the thread to sleep
@@ -100,7 +99,6 @@ struct dim2_hdm {
 	struct most_interface most_iface;
 	char name[16 + sizeof "dim2-"];
 	void __iomem *io_base;
-	unsigned int irq_ahb0;
 	int clk_speed;
 	struct task_struct *netinfo_task;
 	wait_queue_head_t netinfo_waitq;
@@ -118,10 +116,6 @@ struct dim2_hdm {
 	(((p)[1] == 0x18) && ((p)[2] == 0x05) && ((p)[3] == 0x0C) && \
 	 ((p)[13] == 0x3C) && ((p)[14] == 0x00) && ((p)[15] == 0x0A))
 
-#if defined(ENABLE_HDM_TEST)
-static struct dim2_hdm *test_dev;
-#endif
-
 bool dim2_sysfs_get_state_cb(void)
 {
 	bool state;
@@ -212,7 +206,8 @@ static int startup_dim(struct platform_device *pdev)
 			return ret;
 	}
 
-	hal_ret = dim_startup(dev->io_base, dev->clk_speed);
+	pr_info("sync: num of frames per sub-buffer: %u\n", fcnt);
+	hal_ret = dim_startup(dev->io_base, dev->clk_speed, fcnt);
 	if (hal_ret != DIM_NO_ERROR) {
 		pr_err("dim_startup failed: %d\n", hal_ret);
 		if (pdata && pdata->destroy)
@@ -254,6 +249,11 @@ static int try_start_dim_transfer(struct hdm_channel *hdm_ch)
 	mbo = list_first_entry(head, struct mbo, list);
 	buf_size = mbo->buffer_length;
 
+	if (dim_dbr_space(&hdm_ch->ch) < buf_size) {
+		spin_unlock_irqrestore(&dim_lock, flags);
+		return -EAGAIN;
+	}
+
 	BUG_ON(mbo->bus_address == 0);
 	if (!dim_enqueue_buffer(&hdm_ch->ch, mbo->bus_address, buf_size)) {
 		list_del(head->next);
@@ -411,6 +411,22 @@ static struct dim_channel **get_active_channels(struct dim2_hdm *dev,
 	return buffer;
 }
 
+static irqreturn_t dim2_mlb_isr(int irq, void *_dev)
+{
+	struct dim2_hdm *dev = _dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dim_lock, flags);
+	dim_service_mlb_int_irq();
+	spin_unlock_irqrestore(&dim_lock, flags);
+
+	if (dev->atx_idx >= 0 && dev->hch[dev->atx_idx].is_initialized)
+		while (!try_start_dim_transfer(dev->hch + dev->atx_idx))
+			continue;
+
+	return IRQ_HANDLED;
+}
+
 /**
  * dim2_tasklet_fn - tasklet function
  * @data: private data
@@ -452,30 +468,14 @@ static irqreturn_t dim2_ahb_isr(int irq, void *_dev)
 	unsigned long flags;
 
 	spin_lock_irqsave(&dim_lock, flags);
-	dim_service_irq(get_active_channels(dev, buffer));
+	dim_service_ahb_int_irq(get_active_channels(dev, buffer));
 	spin_unlock_irqrestore(&dim_lock, flags);
 
-#if !defined(ENABLE_HDM_TEST)
 	dim2_tasklet.data = (unsigned long)dev;
 	tasklet_schedule(&dim2_tasklet);
-#else
-	dim2_tasklet_fn((unsigned long)dev);
-#endif
 	return IRQ_HANDLED;
 }
 
-#if defined(ENABLE_HDM_TEST)
-
-/*
- * Utility function used by HAL-simu for calling DIM interrupt handler.
- * It is used only for TEST PURPOSE.
- */
-void raise_dim_interrupt(void)
-{
-	(void)dim2_ahb_isr(0, test_dev);
-}
-#endif
-
 /**
  * complete_all_mbos - complete MBO's in a list
  * @head: list head
@@ -545,7 +545,7 @@ static int configure_channel(struct most_interface *most_iface, int ch_idx,
 				hdm_ch->name, buf_size, new_size);
 		spin_lock_irqsave(&dim_lock, flags);
 		hal_ret = dim_init_control(&hdm_ch->ch, is_tx, ch_addr,
-					   buf_size);
+					   is_tx ? new_size * 2 : new_size);
 		break;
 	case MOST_CH_ASYNC:
 		new_size = dim_norm_ctrl_async_buffer_size(buf_size);
@@ -558,9 +558,10 @@ static int configure_channel(struct most_interface *most_iface, int ch_idx,
 			pr_warn("%s: fixed buffer size (%d -> %d)\n",
 				hdm_ch->name, buf_size, new_size);
 		spin_lock_irqsave(&dim_lock, flags);
-		hal_ret = dim_init_async(&hdm_ch->ch, is_tx, ch_addr, buf_size);
+		hal_ret = dim_init_async(&hdm_ch->ch, is_tx, ch_addr,
+					 is_tx ? new_size * 2 : new_size);
 		break;
-	case MOST_CH_ISOC_AVP:
+	case MOST_CH_ISOC:
 		new_size = dim_norm_isoc_buffer_size(buf_size, sub_size);
 		if (new_size == 0) {
 			pr_err("%s: invalid sub-buffer size or too small buffer size\n",
@@ -705,12 +706,14 @@ static int poison_channel(struct most_interface *most_iface, int ch_idx)
 	if (!hdm_ch->is_initialized)
 		return -EPERM;
 
+	tasklet_disable(&dim2_tasklet);
 	spin_lock_irqsave(&dim_lock, flags);
 	hal_ret = dim_destroy_channel(&hdm_ch->ch);
 	hdm_ch->is_initialized = false;
 	if (ch_idx == dev->atx_idx)
 		dev->atx_idx = -1;
 	spin_unlock_irqrestore(&dim_lock, flags);
+	tasklet_enable(&dim2_tasklet);
 	if (hal_ret != DIM_NO_ERROR) {
 		pr_err("HAL Failed to close channel %s\n", hdm_ch->name);
 		ret = -EFAULT;
@@ -735,6 +738,7 @@ static int dim2_probe(struct platform_device *pdev)
 	struct resource *res;
 	int ret, i;
 	struct kobject *kobj;
+	int irq;
 
 	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
 	if (!dev)
@@ -743,29 +747,37 @@ static int dim2_probe(struct platform_device *pdev)
 	dev->atx_idx = -1;
 
 	platform_set_drvdata(pdev, dev);
-#if defined(ENABLE_HDM_TEST)
-	test_dev = dev;
-#else
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	dev->io_base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(dev->io_base))
 		return PTR_ERR(dev->io_base);
 
-	ret = platform_get_irq(pdev, 0);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to get irq\n");
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "failed to get ahb0_int irq\n");
 		return -ENODEV;
 	}
-	dev->irq_ahb0 = ret;
 
-	ret = devm_request_irq(&pdev->dev, dev->irq_ahb0, dim2_ahb_isr, 0,
-			       "mlb_ahb0", dev);
+	ret = devm_request_irq(&pdev->dev, irq, dim2_ahb_isr, 0,
+			       "dim2_ahb0_int", dev);
 	if (ret) {
-		dev_err(&pdev->dev, "failed to request IRQ: %d, err: %d\n",
-			dev->irq_ahb0, ret);
+		dev_err(&pdev->dev, "failed to request ahb0_int irq %d\n", irq);
 		return ret;
 	}
-#endif
+
+	irq = platform_get_irq(pdev, 1);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "failed to get mlb_int irq\n");
+		return -ENODEV;
+	}
+
+	ret = devm_request_irq(&pdev->dev, irq, dim2_mlb_isr, 0,
+			       "dim2_mlb_int", dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to request mlb_int irq %d\n", irq);
+		return ret;
+	}
+
 	init_waitqueue_head(&dev->netinfo_waitq);
 	dev->deliver_netinfo = 0;
 	dev->netinfo_task = kthread_run(&deliver_netinfo_thread, (void *)dev,
@@ -785,7 +797,7 @@ static int dim2_probe(struct platform_device *pdev)
 		cap->name_suffix = hdm_ch->name;
 		cap->direction = MOST_CH_RX | MOST_CH_TX;
 		cap->data_type = MOST_CH_CONTROL | MOST_CH_ASYNC |
-				 MOST_CH_ISOC_AVP | MOST_CH_SYNC;
+				 MOST_CH_ISOC | MOST_CH_SYNC;
 		cap->num_buffers_packet = MAX_BUFFERS_PACKET;
 		cap->buffer_size_packet = MAX_BUF_SIZE_PACKET;
 		cap->num_buffers_streaming = MAX_BUFFERS_STREAMING;
diff --git a/drivers/staging/most/hdm-dim2/dim2_reg.h b/drivers/staging/most/hdm-dim2/dim2_reg.h
index e0837b6b9ae1..01fe499411ff 100644
--- a/drivers/staging/most/hdm-dim2/dim2_reg.h
+++ b/drivers/staging/most/hdm-dim2/dim2_reg.h
@@ -77,13 +77,7 @@ enum {
 
 	MLBC0_FCNT_SHIFT = 15,
 	MLBC0_FCNT_MASK = 7,
-	MLBC0_FCNT_VAL_1FPSB = 0,
-	MLBC0_FCNT_VAL_2FPSB = 1,
-	MLBC0_FCNT_VAL_4FPSB = 2,
-	MLBC0_FCNT_VAL_8FPSB = 3,
-	MLBC0_FCNT_VAL_16FPSB = 4,
-	MLBC0_FCNT_VAL_32FPSB = 5,
-	MLBC0_FCNT_VAL_64FPSB = 6,
+	MLBC0_FCNT_MAX_VAL = 6,
 
 	MLBC0_MLBEN_BIT = 0,
 
@@ -123,6 +117,9 @@ enum {
 };
 
 enum {
+	CDT0_RPC_SHIFT = 16 + 11,
+	CDT0_RPC_MASK = DIM2_MASK(5),
+
 	CDT1_BS_ISOC_SHIFT = 0,
 	CDT1_BS_ISOC_MASK = DIM2_MASK(9),
 
diff --git a/drivers/staging/most/hdm-dim2/dim2_sysfs.c b/drivers/staging/most/hdm-dim2/dim2_sysfs.c
index 2b28e4a51131..d8b22f91d2c3 100644
--- a/drivers/staging/most/hdm-dim2/dim2_sysfs.c
+++ b/drivers/staging/most/hdm-dim2/dim2_sysfs.c
@@ -39,7 +39,7 @@ static struct attribute *bus_default_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group bus_attr_group = {
+static const struct attribute_group bus_attr_group = {
 	.attrs = bus_default_attrs,
 };
 
diff --git a/drivers/staging/most/hdm-usb/hdm_usb.c b/drivers/staging/most/hdm-usb/hdm_usb.c
index aeae071f2823..26c9adb29308 100644
--- a/drivers/staging/most/hdm-usb/hdm_usb.c
+++ b/drivers/staging/most/hdm-usb/hdm_usb.c
@@ -43,8 +43,9 @@
 
 #define USB_VENDOR_ID_SMSC	0x0424  /* VID: SMSC */
 #define USB_DEV_ID_BRDG		0xC001  /* PID: USB Bridge */
-#define USB_DEV_ID_INIC		0xCF18  /* PID: USB INIC */
-#define HW_RESYNC		0x0000
+#define USB_DEV_ID_OS81118	0xCF18  /* PID: USB OS81118 */
+#define USB_DEV_ID_OS81119	0xCF19  /* PID: USB OS81119 */
+#define USB_DEV_ID_OS81210	0xCF30  /* PID: USB OS81210 */
 /* DRCI Addresses */
 #define DRCI_REG_NI_STATE	0x0100
 #define DRCI_REG_PACKET_BW	0x0101
@@ -64,33 +65,30 @@
 #define DRCI_WRITE_REQ		0xA1
 
 /**
- * struct buf_anchor - used to create a list of pending URBs
- * @urb: pointer to USB request block
- * @clear_work_obj:
- * @list: linked list
- * @urb_completion:
- */
-struct buf_anchor {
-	struct urb *urb;
-	struct work_struct clear_work_obj;
-	struct list_head list;
-	struct completion urb_compl;
-};
-
-#define to_buf_anchor(w) container_of(w, struct buf_anchor, clear_work_obj)
-
-/**
  * struct most_dci_obj - Direct Communication Interface
  * @kobj:position in sysfs
  * @usb_device: pointer to the usb device
+ * @reg_addr: register address for arbitrary DCI access
  */
 struct most_dci_obj {
 	struct kobject kobj;
 	struct usb_device *usb_device;
+	u16 reg_addr;
 };
 
 #define to_dci_obj(p) container_of(p, struct most_dci_obj, kobj)
 
+struct most_dev;
+
+struct clear_hold_work {
+	struct work_struct ws;
+	struct most_dev *mdev;
+	unsigned int channel;
+	int pipe;
+};
+
+#define to_clear_hold_work(w) container_of(w, struct clear_hold_work, ws)
+
 /**
  * struct most_dev - holds all usb interface specific stuff
  * @parent: parent object in sysfs
@@ -104,10 +102,10 @@ struct most_dci_obj {
  * @link_stat: link status of hardware
  * @description: device description
  * @suffix: suffix for channel name
- * @anchor_list_lock: locks list access
+ * @channel_lock: synchronize channel access
  * @padding_active: indicates channel uses padding
  * @is_channel_healthy: health status table of each channel
- * @anchor_list: list of anchored items
+ * @busy_urbs: list of anchored items
  * @io_mutex: synchronize I/O with disconnect
  * @link_stat_timer: timer for link status reports
  * @poll_work_obj: work for polling link status
@@ -124,10 +122,11 @@ struct most_dev {
 	u16 link_stat;
 	char description[MAX_STRING_LEN];
 	char suffix[MAX_NUM_ENDPOINTS][MAX_SUFFIX_LEN];
-	spinlock_t anchor_list_lock[MAX_NUM_ENDPOINTS];
+	spinlock_t channel_lock[MAX_NUM_ENDPOINTS]; /* sync channel access */
 	bool padding_active[MAX_NUM_ENDPOINTS];
 	bool is_channel_healthy[MAX_NUM_ENDPOINTS];
-	struct list_head *anchor_list;
+	struct clear_hold_work clear_work[MAX_NUM_ENDPOINTS];
+	struct usb_anchor *busy_urbs;
 	struct mutex io_mutex;
 	struct timer_list link_stat_timer;
 	struct work_struct poll_work_obj;
@@ -191,40 +190,24 @@ static inline int drci_wr_reg(struct usb_device *dev, u16 reg, u16 data)
  * free_anchored_buffers - free device's anchored items
  * @mdev: the device
  * @channel: channel ID
+ * @status: status of MBO termination
  */
-static void free_anchored_buffers(struct most_dev *mdev, unsigned int channel)
+static void free_anchored_buffers(struct most_dev *mdev, unsigned int channel,
+				  enum mbo_status_flags status)
 {
 	struct mbo *mbo;
-	struct buf_anchor *anchor, *tmp;
-	unsigned long flags;
+	struct urb *urb;
 
-	spin_lock_irqsave(&mdev->anchor_list_lock[channel], flags);
-	list_for_each_entry_safe(anchor, tmp, &mdev->anchor_list[channel],
-				 list) {
-		struct urb *urb = anchor->urb;
-
-		spin_unlock_irqrestore(&mdev->anchor_list_lock[channel], flags);
-		if (likely(urb)) {
-			mbo = urb->context;
-			if (!irqs_disabled()) {
-				usb_kill_urb(urb);
-			} else {
-				usb_unlink_urb(urb);
-				wait_for_completion(&anchor->urb_compl);
-			}
-			if ((mbo) && (mbo->complete)) {
-				mbo->status = MBO_E_CLOSE;
-				mbo->processed_length = 0;
-				mbo->complete(mbo);
-			}
-			usb_free_urb(urb);
+	while ((urb = usb_get_from_anchor(&mdev->busy_urbs[channel]))) {
+		mbo = urb->context;
+		usb_kill_urb(urb);
+		if (mbo && mbo->complete) {
+			mbo->status = status;
+			mbo->processed_length = 0;
+			mbo->complete(mbo);
 		}
-		spin_lock_irqsave(&mdev->anchor_list_lock[channel], flags);
-		list_del(&anchor->list);
-		cancel_work_sync(&anchor->clear_work_obj);
-		kfree(anchor);
+		usb_free_urb(urb);
 	}
-	spin_unlock_irqrestore(&mdev->anchor_list_lock[channel], flags);
 }
 
 /**
@@ -241,7 +224,7 @@ static unsigned int get_stream_frame_size(struct most_channel_config *cfg)
 		return frame_size;
 	}
 	switch (cfg->data_type) {
-	case MOST_CH_ISOC_AVP:
+	case MOST_CH_ISOC:
 		frame_size = AV_PACKETS_PER_XACT * sub_size;
 		break;
 	case MOST_CH_SYNC:
@@ -274,22 +257,28 @@ static unsigned int get_stream_frame_size(struct most_channel_config *cfg)
  */
 static int hdm_poison_channel(struct most_interface *iface, int channel)
 {
-	struct most_dev *mdev;
+	struct most_dev *mdev = to_mdev(iface);
+	unsigned long flags;
+	spinlock_t *lock; /* temp. lock */
 
-	mdev = to_mdev(iface);
 	if (unlikely(!iface)) {
 		dev_warn(&mdev->usb_device->dev, "Poison: Bad interface.\n");
 		return -EIO;
 	}
-	if (unlikely((channel < 0) || (channel >= iface->num_channels))) {
+	if (unlikely(channel < 0 || channel >= iface->num_channels)) {
 		dev_warn(&mdev->usb_device->dev, "Channel ID out of range.\n");
 		return -ECHRNG;
 	}
 
+	lock = mdev->channel_lock + channel;
+	spin_lock_irqsave(lock, flags);
 	mdev->is_channel_healthy[channel] = false;
+	spin_unlock_irqrestore(lock, flags);
+
+	cancel_work_sync(&mdev->clear_work[channel].ws);
 
 	mutex_lock(&mdev->io_mutex);
-	free_anchored_buffers(mdev, channel);
+	free_anchored_buffers(mdev, channel, MBO_E_CLOSE);
 	if (mdev->padding_active[channel])
 		mdev->padding_active[channel] = false;
 
@@ -313,10 +302,10 @@ static int hdm_poison_channel(struct most_interface *iface, int channel)
 static int hdm_add_padding(struct most_dev *mdev, int channel, struct mbo *mbo)
 {
 	struct most_channel_config *conf = &mdev->conf[channel];
-	unsigned int j, num_frames, frame_size;
+	unsigned int frame_size = get_stream_frame_size(conf);
+	unsigned int j, num_frames;
 	u16 rd_addr, wr_addr;
 
-	frame_size = get_stream_frame_size(conf);
 	if (!frame_size)
 		return -EIO;
 	num_frames = mbo->buffer_length / frame_size;
@@ -350,10 +339,10 @@ static int hdm_add_padding(struct most_dev *mdev, int channel, struct mbo *mbo)
 static int hdm_remove_padding(struct most_dev *mdev, int channel,
 			      struct mbo *mbo)
 {
-	unsigned int j, num_frames, frame_size;
 	struct most_channel_config *const conf = &mdev->conf[channel];
+	unsigned int frame_size = get_stream_frame_size(conf);
+	unsigned int j, num_frames;
 
-	frame_size = get_stream_frame_size(conf);
 	if (!frame_size)
 		return -EIO;
 	num_frames = mbo->processed_length / USB_MTU;
@@ -380,37 +369,29 @@ static int hdm_remove_padding(struct most_dev *mdev, int channel,
  */
 static void hdm_write_completion(struct urb *urb)
 {
-	struct mbo *mbo;
-	struct buf_anchor *anchor;
-	struct most_dev *mdev;
-	struct device *dev;
-	unsigned int channel;
+	struct mbo *mbo = urb->context;
+	struct most_dev *mdev = to_mdev(mbo->ifp);
+	unsigned int channel = mbo->hdm_channel_id;
+	struct device *dev = &mdev->usb_device->dev;
+	spinlock_t *lock = mdev->channel_lock + channel;
 	unsigned long flags;
 
-	mbo = urb->context;
-	anchor = mbo->priv;
-	mdev = to_mdev(mbo->ifp);
-	channel = mbo->hdm_channel_id;
-	dev = &mdev->usb_device->dev;
-
-	if ((urb->status == -ENOENT) || (urb->status == -ECONNRESET) ||
-	    (!mdev->is_channel_healthy[channel])) {
-		complete(&anchor->urb_compl);
+	spin_lock_irqsave(lock, flags);
+	if (urb->status == -ENOENT || urb->status == -ECONNRESET ||
+	    !mdev->is_channel_healthy[channel]) {
+		spin_unlock_irqrestore(lock, flags);
 		return;
 	}
 
-	if (unlikely(urb->status && !(urb->status == -ENOENT ||
-				      urb->status == -ECONNRESET ||
-				      urb->status == -ESHUTDOWN))) {
+	if (unlikely(urb->status && urb->status != -ESHUTDOWN)) {
 		mbo->processed_length = 0;
 		switch (urb->status) {
 		case -EPIPE:
 			dev_warn(dev, "Broken OUT pipe detected\n");
-			most_stop_enqueue(&mdev->iface, channel);
-			mbo->status = MBO_E_INVAL;
-			usb_unlink_urb(urb);
-			INIT_WORK(&anchor->clear_work_obj, wq_clear_halt);
-			schedule_work(&anchor->clear_work_obj);
+			mdev->is_channel_healthy[channel] = false;
+			spin_unlock_irqrestore(lock, flags);
+			mdev->clear_work[channel].pipe = urb->pipe;
+			schedule_work(&mdev->clear_work[channel].ws);
 			return;
 		case -ENODEV:
 		case -EPROTO:
@@ -425,10 +406,7 @@ static void hdm_write_completion(struct urb *urb)
 		mbo->processed_length = urb->actual_length;
 	}
 
-	spin_lock_irqsave(&mdev->anchor_list_lock[channel], flags);
-	list_del(&anchor->list);
-	spin_unlock_irqrestore(&mdev->anchor_list_lock[channel], flags);
-	kfree(anchor);
+	spin_unlock_irqrestore(lock, flags);
 
 	if (likely(mbo->complete))
 		mbo->complete(mbo);
@@ -545,36 +523,29 @@ static void hdm_write_completion(struct urb *urb)
  */
 static void hdm_read_completion(struct urb *urb)
 {
-	struct mbo *mbo;
-	struct buf_anchor *anchor;
-	struct most_dev *mdev;
-	struct device *dev;
+	struct mbo *mbo = urb->context;
+	struct most_dev *mdev = to_mdev(mbo->ifp);
+	unsigned int channel = mbo->hdm_channel_id;
+	struct device *dev = &mdev->usb_device->dev;
+	spinlock_t *lock = mdev->channel_lock + channel;
 	unsigned long flags;
-	unsigned int channel;
-
-	mbo = urb->context;
-	anchor = mbo->priv;
-	mdev = to_mdev(mbo->ifp);
-	channel = mbo->hdm_channel_id;
-	dev = &mdev->usb_device->dev;
 
-	if ((urb->status == -ENOENT) || (urb->status == -ECONNRESET) ||
-	    (!mdev->is_channel_healthy[channel])) {
-		complete(&anchor->urb_compl);
+	spin_lock_irqsave(lock, flags);
+	if (urb->status == -ENOENT || urb->status == -ECONNRESET ||
+	    !mdev->is_channel_healthy[channel]) {
+		spin_unlock_irqrestore(lock, flags);
 		return;
 	}
 
-	if (unlikely(urb->status && !(urb->status == -ENOENT ||
-				      urb->status == -ECONNRESET ||
-				      urb->status == -ESHUTDOWN))) {
+	if (unlikely(urb->status && urb->status != -ESHUTDOWN)) {
 		mbo->processed_length = 0;
 		switch (urb->status) {
 		case -EPIPE:
 			dev_warn(dev, "Broken IN pipe detected\n");
-			mbo->status = MBO_E_INVAL;
-			usb_unlink_urb(urb);
-			INIT_WORK(&anchor->clear_work_obj, wq_clear_halt);
-			schedule_work(&anchor->clear_work_obj);
+			mdev->is_channel_healthy[channel] = false;
+			spin_unlock_irqrestore(lock, flags);
+			mdev->clear_work[channel].pipe = urb->pipe;
+			schedule_work(&mdev->clear_work[channel].ws);
 			return;
 		case -ENODEV:
 		case -EPROTO:
@@ -588,21 +559,15 @@ static void hdm_read_completion(struct urb *urb)
 		}
 	} else {
 		mbo->processed_length = urb->actual_length;
-		if (!mdev->padding_active[channel]) {
-			mbo->status = MBO_SUCCESS;
-		} else {
-			if (hdm_remove_padding(mdev, channel, mbo)) {
-				mbo->processed_length = 0;
-				mbo->status = MBO_E_INVAL;
-			} else {
-				mbo->status = MBO_SUCCESS;
-			}
+		mbo->status = MBO_SUCCESS;
+		if (mdev->padding_active[channel] &&
+		    hdm_remove_padding(mdev, channel, mbo)) {
+			mbo->processed_length = 0;
+			mbo->status = MBO_E_INVAL;
 		}
 	}
-	spin_lock_irqsave(&mdev->anchor_list_lock[channel], flags);
-	list_del(&anchor->list);
-	spin_unlock_irqrestore(&mdev->anchor_list_lock[channel], flags);
-	kfree(anchor);
+
+	spin_unlock_irqrestore(lock, flags);
 
 	if (likely(mbo->complete))
 		mbo->complete(mbo);
@@ -628,18 +593,16 @@ static int hdm_enqueue(struct most_interface *iface, int channel,
 		       struct mbo *mbo)
 {
 	struct most_dev *mdev;
-	struct buf_anchor *anchor;
 	struct most_channel_config *conf;
 	struct device *dev;
 	int retval = 0;
 	struct urb *urb;
-	unsigned long flags;
 	unsigned long length;
 	void *virt_address;
 
 	if (unlikely(!iface || !mbo))
 		return -EIO;
-	if (unlikely(iface->num_channels <= channel) || (channel < 0))
+	if (unlikely(iface->num_channels <= channel || channel < 0))
 		return -ECHRNG;
 
 	mdev = to_mdev(iface);
@@ -650,32 +613,15 @@ static int hdm_enqueue(struct most_interface *iface, int channel,
 		return -ENODEV;
 
 	urb = usb_alloc_urb(NO_ISOCHRONOUS_URB, GFP_ATOMIC);
-	if (!urb) {
-		dev_err(dev, "Failed to allocate URB\n");
+	if (!urb)
 		return -ENOMEM;
-	}
 
-	anchor = kzalloc(sizeof(*anchor), GFP_ATOMIC);
-	if (!anchor) {
-		retval = -ENOMEM;
+	if ((conf->direction & MOST_CH_TX) && mdev->padding_active[channel] &&
+	    hdm_add_padding(mdev, channel, mbo)) {
+		retval = -EIO;
 		goto _error;
 	}
 
-	anchor->urb = urb;
-	init_completion(&anchor->urb_compl);
-	mbo->priv = anchor;
-
-	spin_lock_irqsave(&mdev->anchor_list_lock[channel], flags);
-	list_add_tail(&anchor->list, &mdev->anchor_list[channel]);
-	spin_unlock_irqrestore(&mdev->anchor_list_lock[channel], flags);
-
-	if ((mdev->padding_active[channel]) &&
-	    (conf->direction & MOST_CH_TX))
-		if (hdm_add_padding(mdev, channel, mbo)) {
-			retval = -EIO;
-			goto _error_1;
-		}
-
 	urb->transfer_dma = mbo->bus_address;
 	virt_address = mbo->virt_address;
 	length = mbo->buffer_length;
@@ -688,7 +634,7 @@ static int hdm_enqueue(struct most_interface *iface, int channel,
 				  length,
 				  hdm_write_completion,
 				  mbo);
-		if (conf->data_type != MOST_CH_ISOC_AVP)
+		if (conf->data_type != MOST_CH_ISOC)
 			urb->transfer_flags |= URB_ZERO_PACKET;
 	} else {
 		usb_fill_bulk_urb(urb, mdev->usb_device,
@@ -701,6 +647,8 @@ static int hdm_enqueue(struct most_interface *iface, int channel,
 	}
 	urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
 
+	usb_anchor_urb(urb, &mdev->busy_urbs[channel]);
+
 	retval = usb_submit_urb(urb, GFP_KERNEL);
 	if (retval) {
 		dev_err(dev, "URB submit failed with error %d.\n", retval);
@@ -709,10 +657,7 @@ static int hdm_enqueue(struct most_interface *iface, int channel,
 	return 0;
 
 _error_1:
-	spin_lock_irqsave(&mdev->anchor_list_lock[channel], flags);
-	list_del(&anchor->list);
-	spin_unlock_irqrestore(&mdev->anchor_list_lock[channel], flags);
-	kfree(anchor);
+	usb_unanchor_urb(urb);
 _error:
 	usb_free_urb(urb);
 	return retval;
@@ -731,29 +676,30 @@ static int hdm_configure_channel(struct most_interface *iface, int channel,
 	unsigned int frame_size;
 	unsigned int temp_size;
 	unsigned int tail_space;
-	struct most_dev *mdev;
-	struct device *dev;
+	struct most_dev *mdev = to_mdev(iface);
+	struct device *dev = &mdev->usb_device->dev;
 
-	mdev = to_mdev(iface);
 	mdev->is_channel_healthy[channel] = true;
-	dev = &mdev->usb_device->dev;
+	mdev->clear_work[channel].channel = channel;
+	mdev->clear_work[channel].mdev = mdev;
+	INIT_WORK(&mdev->clear_work[channel].ws, wq_clear_halt);
 
 	if (unlikely(!iface || !conf)) {
 		dev_err(dev, "Bad interface or config pointer.\n");
 		return -EINVAL;
 	}
-	if (unlikely((channel < 0) || (channel >= iface->num_channels))) {
+	if (unlikely(channel < 0 || channel >= iface->num_channels)) {
 		dev_err(dev, "Channel ID out of range.\n");
 		return -EINVAL;
 	}
-	if ((!conf->num_buffers) || (!conf->buffer_size)) {
+	if (!conf->num_buffers || !conf->buffer_size) {
 		dev_err(dev, "Misconfig: buffer size or #buffers zero.\n");
 		return -EINVAL;
 	}
 
-	if (!(conf->data_type == MOST_CH_SYNC) &&
-	    !((conf->data_type == MOST_CH_ISOC_AVP) &&
-	      (conf->packets_per_xact != 0xFF))) {
+	if (conf->data_type != MOST_CH_SYNC &&
+	    !(conf->data_type == MOST_CH_ISOC &&
+	      conf->packets_per_xact != 0xFF)) {
 		mdev->padding_active[channel] = false;
 		goto exit;
 	}
@@ -762,7 +708,7 @@ static int hdm_configure_channel(struct most_interface *iface, int channel,
 	temp_size = conf->buffer_size;
 
 	frame_size = get_stream_frame_size(conf);
-	if ((frame_size == 0) || (frame_size > USB_MTU)) {
+	if (frame_size == 0 || frame_size > USB_MTU) {
 		dev_warn(dev, "Misconfig: frame size wrong\n");
 		return -EINVAL;
 	}
@@ -807,17 +753,17 @@ static int hdm_update_netinfo(struct most_dev *mdev)
 	if (!is_valid_ether_addr(mdev->hw_addr)) {
 		if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_HI, &hi) < 0) {
 			dev_err(dev, "Vendor request \"hw_addr_hi\" failed\n");
-			return -1;
+			return -EFAULT;
 		}
 
 		if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_MI, &mi) < 0) {
 			dev_err(dev, "Vendor request \"hw_addr_mid\" failed\n");
-			return -1;
+			return -EFAULT;
 		}
 
 		if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_LO, &lo) < 0) {
 			dev_err(dev, "Vendor request \"hw_addr_low\" failed\n");
-			return -1;
+			return -EFAULT;
 		}
 
 		mutex_lock(&mdev->io_mutex);
@@ -832,7 +778,7 @@ static int hdm_update_netinfo(struct most_dev *mdev)
 
 	if (drci_rd_reg(usb_device, DRCI_REG_NI_STATE, &link) < 0) {
 		dev_err(dev, "Vendor request \"link status\" failed\n");
-		return -1;
+		return -EFAULT;
 	}
 
 	mutex_lock(&mdev->io_mutex);
@@ -886,25 +832,22 @@ static void link_stat_timer_handler(unsigned long data)
  */
 static void wq_netinfo(struct work_struct *wq_obj)
 {
-	struct most_dev *mdev;
-	int i, prev_link_stat;
+	struct most_dev *mdev = to_mdev_from_work(wq_obj);
+	int i, prev_link_stat = mdev->link_stat;
 	u8 prev_hw_addr[6];
 
-	mdev = to_mdev_from_work(wq_obj);
-	prev_link_stat = mdev->link_stat;
-
 	for (i = 0; i < 6; i++)
 		prev_hw_addr[i] = mdev->hw_addr[i];
 
 	if (hdm_update_netinfo(mdev) < 0)
 		return;
-	if ((prev_link_stat != mdev->link_stat) ||
-	    (prev_hw_addr[0] != mdev->hw_addr[0]) ||
-	    (prev_hw_addr[1] != mdev->hw_addr[1]) ||
-	    (prev_hw_addr[2] != mdev->hw_addr[2]) ||
-	    (prev_hw_addr[3] != mdev->hw_addr[3]) ||
-	    (prev_hw_addr[4] != mdev->hw_addr[4]) ||
-	    (prev_hw_addr[5] != mdev->hw_addr[5]))
+	if (prev_link_stat != mdev->link_stat ||
+	    prev_hw_addr[0] != mdev->hw_addr[0] ||
+	    prev_hw_addr[1] != mdev->hw_addr[1] ||
+	    prev_hw_addr[2] != mdev->hw_addr[2] ||
+	    prev_hw_addr[3] != mdev->hw_addr[3] ||
+	    prev_hw_addr[4] != mdev->hw_addr[4] ||
+	    prev_hw_addr[5] != mdev->hw_addr[5])
 		most_deliver_netinfo(&mdev->iface, mdev->link_stat,
 				     &mdev->hw_addr[0]);
 }
@@ -917,33 +860,20 @@ static void wq_netinfo(struct work_struct *wq_obj)
  */
 static void wq_clear_halt(struct work_struct *wq_obj)
 {
-	struct buf_anchor *anchor;
-	struct most_dev *mdev;
-	struct mbo *mbo;
-	struct urb *urb;
-	unsigned int channel;
-	unsigned long flags;
+	struct clear_hold_work *clear_work = to_clear_hold_work(wq_obj);
+	struct most_dev *mdev = clear_work->mdev;
+	unsigned int channel = clear_work->channel;
+	int pipe = clear_work->pipe;
 
-	anchor = to_buf_anchor(wq_obj);
-	urb = anchor->urb;
-	mbo = urb->context;
-	mdev = to_mdev(mbo->ifp);
-	channel = mbo->hdm_channel_id;
-
-	if (usb_clear_halt(urb->dev, urb->pipe))
+	mutex_lock(&mdev->io_mutex);
+	most_stop_enqueue(&mdev->iface, channel);
+	free_anchored_buffers(mdev, channel, MBO_E_INVAL);
+	if (usb_clear_halt(mdev->usb_device, pipe))
 		dev_warn(&mdev->usb_device->dev, "Failed to reset endpoint.\n");
 
-	usb_free_urb(urb);
-	spin_lock_irqsave(&mdev->anchor_list_lock[channel], flags);
-	list_del(&anchor->list);
-	spin_unlock_irqrestore(&mdev->anchor_list_lock[channel], flags);
-
-	if (likely(mbo->complete))
-		mbo->complete(mbo);
-	if (mdev->conf[channel].direction & MOST_CH_TX)
-		most_resume_enqueue(&mdev->iface, channel);
-
-	kfree(anchor);
+	mdev->is_channel_healthy[channel] = true;
+	most_resume_enqueue(&mdev->iface, channel);
+	mutex_unlock(&mdev->io_mutex);
 }
 
 /**
@@ -958,7 +888,9 @@ static const struct file_operations hdm_usb_fops = {
  */
 static struct usb_device_id usbid[] = {
 	{ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_DEV_ID_BRDG), },
-	{ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_DEV_ID_INIC), },
+	{ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_DEV_ID_OS81118), },
+	{ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_DEV_ID_OS81119), },
+	{ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_DEV_ID_OS81210), },
 	{ } /* Terminating entry */
 };
 
@@ -970,6 +902,10 @@ static struct usb_device_id usbid[] = {
 	struct most_dci_attribute most_dci_attr_##_name = \
 		__ATTR(_name, S_IRUGO | S_IWUSR, show_value, store_value)
 
+#define MOST_DCI_WO_ATTR(_name) \
+	struct most_dci_attribute most_dci_attr_##_name = \
+		__ATTR(_name, S_IWUSR, NULL, store_value)
+
 /**
  * struct most_dci_attribute - to access the attributes of a dci object
  * @attr: attributes of a dci object
@@ -1046,45 +982,68 @@ static void most_dci_release(struct kobject *kobj)
 	kfree(dci_obj);
 }
 
+struct regs {
+	const char *name;
+	u16 reg;
+};
+
+static const struct regs ro_regs[] = {
+	{ "ni_state", DRCI_REG_NI_STATE },
+	{ "packet_bandwidth", DRCI_REG_PACKET_BW },
+	{ "node_address", DRCI_REG_NODE_ADDR },
+	{ "node_position", DRCI_REG_NODE_POS },
+};
+
+static const struct regs rw_regs[] = {
+	{ "mep_filter", DRCI_REG_MEP_FILTER },
+	{ "mep_hash0", DRCI_REG_HASH_TBL0 },
+	{ "mep_hash1", DRCI_REG_HASH_TBL1 },
+	{ "mep_hash2", DRCI_REG_HASH_TBL2 },
+	{ "mep_hash3", DRCI_REG_HASH_TBL3 },
+	{ "mep_eui48_hi", DRCI_REG_HW_ADDR_HI },
+	{ "mep_eui48_mi", DRCI_REG_HW_ADDR_MI },
+	{ "mep_eui48_lo", DRCI_REG_HW_ADDR_LO },
+};
+
+static int get_stat_reg_addr(const struct regs *regs, int size,
+			     const char *name, u16 *reg_addr)
+{
+	int i;
+
+	for (i = 0; i < size; i++) {
+		if (!strcmp(name, regs[i].name)) {
+			*reg_addr = regs[i].reg;
+			return 0;
+		}
+	}
+	return -EFAULT;
+}
+
+#define get_static_reg_addr(regs, name, reg_addr) \
+	get_stat_reg_addr(regs, ARRAY_SIZE(regs), name, reg_addr)
+
 static ssize_t show_value(struct most_dci_obj *dci_obj,
 			  struct most_dci_attribute *attr, char *buf)
 {
-	u16 tmp_val;
+	const char *name = attr->attr.name;
+	u16 val;
 	u16 reg_addr;
 	int err;
 
-	if (!strcmp(attr->attr.name, "ni_state"))
-		reg_addr = DRCI_REG_NI_STATE;
-	else if (!strcmp(attr->attr.name, "packet_bandwidth"))
-		reg_addr = DRCI_REG_PACKET_BW;
-	else if (!strcmp(attr->attr.name, "node_address"))
-		reg_addr = DRCI_REG_NODE_ADDR;
-	else if (!strcmp(attr->attr.name, "node_position"))
-		reg_addr = DRCI_REG_NODE_POS;
-	else if (!strcmp(attr->attr.name, "mep_filter"))
-		reg_addr = DRCI_REG_MEP_FILTER;
-	else if (!strcmp(attr->attr.name, "mep_hash0"))
-		reg_addr = DRCI_REG_HASH_TBL0;
-	else if (!strcmp(attr->attr.name, "mep_hash1"))
-		reg_addr = DRCI_REG_HASH_TBL1;
-	else if (!strcmp(attr->attr.name, "mep_hash2"))
-		reg_addr = DRCI_REG_HASH_TBL2;
-	else if (!strcmp(attr->attr.name, "mep_hash3"))
-		reg_addr = DRCI_REG_HASH_TBL3;
-	else if (!strcmp(attr->attr.name, "mep_eui48_hi"))
-		reg_addr = DRCI_REG_HW_ADDR_HI;
-	else if (!strcmp(attr->attr.name, "mep_eui48_mi"))
-		reg_addr = DRCI_REG_HW_ADDR_MI;
-	else if (!strcmp(attr->attr.name, "mep_eui48_lo"))
-		reg_addr = DRCI_REG_HW_ADDR_LO;
-	else
-		return -EIO;
+	if (!strcmp(name, "arb_address"))
+		return snprintf(buf, PAGE_SIZE, "%04x\n", dci_obj->reg_addr);
+
+	if (!strcmp(name, "arb_value"))
+		reg_addr = dci_obj->reg_addr;
+	else if (get_static_reg_addr(ro_regs, name, &reg_addr) &&
+		 get_static_reg_addr(rw_regs, name, &reg_addr))
+		return -EFAULT;
 
-	err = drci_rd_reg(dci_obj->usb_device, reg_addr, &tmp_val);
+	err = drci_rd_reg(dci_obj->usb_device, reg_addr, &val);
 	if (err < 0)
 		return err;
 
-	return snprintf(buf, PAGE_SIZE, "%04x\n", tmp_val);
+	return snprintf(buf, PAGE_SIZE, "%04x\n", val);
 }
 
 static ssize_t store_value(struct most_dci_obj *dci_obj,
@@ -1093,31 +1052,28 @@ static ssize_t store_value(struct most_dci_obj *dci_obj,
 {
 	u16 val;
 	u16 reg_addr;
-	int err;
-
-	if (!strcmp(attr->attr.name, "mep_filter"))
-		reg_addr = DRCI_REG_MEP_FILTER;
-	else if (!strcmp(attr->attr.name, "mep_hash0"))
-		reg_addr = DRCI_REG_HASH_TBL0;
-	else if (!strcmp(attr->attr.name, "mep_hash1"))
-		reg_addr = DRCI_REG_HASH_TBL1;
-	else if (!strcmp(attr->attr.name, "mep_hash2"))
-		reg_addr = DRCI_REG_HASH_TBL2;
-	else if (!strcmp(attr->attr.name, "mep_hash3"))
-		reg_addr = DRCI_REG_HASH_TBL3;
-	else if (!strcmp(attr->attr.name, "mep_eui48_hi"))
-		reg_addr = DRCI_REG_HW_ADDR_HI;
-	else if (!strcmp(attr->attr.name, "mep_eui48_mi"))
-		reg_addr = DRCI_REG_HW_ADDR_MI;
-	else if (!strcmp(attr->attr.name, "mep_eui48_lo"))
-		reg_addr = DRCI_REG_HW_ADDR_LO;
-	else
-		return -EIO;
+	const char *name = attr->attr.name;
+	int err = kstrtou16(buf, 16, &val);
 
-	err = kstrtou16(buf, 16, &val);
 	if (err)
 		return err;
 
+	if (!strcmp(name, "arb_address")) {
+		dci_obj->reg_addr = val;
+		return count;
+	}
+
+	if (!strcmp(name, "arb_value")) {
+		reg_addr = dci_obj->reg_addr;
+	} else if (!strcmp(name, "sync_ep")) {
+		u16 ep = val;
+
+		reg_addr = DRCI_REG_BASE + DRCI_COMMAND + ep * 16;
+		val = 1;
+	} else if (get_static_reg_addr(ro_regs, name, &reg_addr)) {
+		return -EFAULT;
+	}
+
 	err = drci_wr_reg(dci_obj->usb_device, reg_addr, val);
 	if (err < 0)
 		return err;
@@ -1129,6 +1085,7 @@ static MOST_DCI_RO_ATTR(ni_state);
 static MOST_DCI_RO_ATTR(packet_bandwidth);
 static MOST_DCI_RO_ATTR(node_address);
 static MOST_DCI_RO_ATTR(node_position);
+static MOST_DCI_WO_ATTR(sync_ep);
 static MOST_DCI_ATTR(mep_filter);
 static MOST_DCI_ATTR(mep_hash0);
 static MOST_DCI_ATTR(mep_hash1);
@@ -1137,6 +1094,8 @@ static MOST_DCI_ATTR(mep_hash3);
 static MOST_DCI_ATTR(mep_eui48_hi);
 static MOST_DCI_ATTR(mep_eui48_mi);
 static MOST_DCI_ATTR(mep_eui48_lo);
+static MOST_DCI_ATTR(arb_address);
+static MOST_DCI_ATTR(arb_value);
 
 /**
  * most_dci_def_attrs - array of default attribute files of the dci object
@@ -1146,6 +1105,7 @@ static struct attribute *most_dci_def_attrs[] = {
 	&most_dci_attr_packet_bandwidth.attr,
 	&most_dci_attr_node_address.attr,
 	&most_dci_attr_node_position.attr,
+	&most_dci_attr_sync_ep.attr,
 	&most_dci_attr_mep_filter.attr,
 	&most_dci_attr_mep_hash0.attr,
 	&most_dci_attr_mep_hash1.attr,
@@ -1154,6 +1114,8 @@ static struct attribute *most_dci_def_attrs[] = {
 	&most_dci_attr_mep_eui48_hi.attr,
 	&most_dci_attr_mep_eui48_mi.attr,
 	&most_dci_attr_mep_eui48_lo.attr,
+	&most_dci_attr_arb_address.attr,
+	&most_dci_attr_arb_value.attr,
 	NULL,
 };
 
@@ -1176,10 +1138,9 @@ static struct kobj_type most_dci_ktype = {
 static struct
 most_dci_obj *create_most_dci_obj(struct kobject *parent)
 {
-	struct most_dci_obj *most_dci;
+	struct most_dci_obj *most_dci = kzalloc(sizeof(*most_dci), GFP_KERNEL);
 	int retval;
 
-	most_dci = kzalloc(sizeof(*most_dci), GFP_KERNEL);
 	if (!most_dci)
 		return NULL;
 
@@ -1216,21 +1177,17 @@ static void destroy_most_dci_obj(struct most_dci_obj *p)
 static int
 hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 {
+	struct usb_host_interface *usb_iface_desc = interface->cur_altsetting;
+	struct usb_device *usb_dev = interface_to_usbdev(interface);
+	struct device *dev = &usb_dev->dev;
+	struct most_dev *mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
 	unsigned int i;
 	unsigned int num_endpoints;
 	struct most_channel_capability *tmp_cap;
-	struct most_dev *mdev;
-	struct usb_device *usb_dev;
-	struct device *dev;
-	struct usb_host_interface *usb_iface_desc;
 	struct usb_endpoint_descriptor *ep_desc;
 	int ret = 0;
 	int err;
 
-	usb_iface_desc = interface->cur_altsetting;
-	usb_dev = interface_to_usbdev(interface);
-	dev = &usb_dev->dev;
-	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
 	if (!mdev)
 		goto exit_ENOMEM;
 
@@ -1276,9 +1233,9 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 	if (!mdev->ep_address)
 		goto exit_free2;
 
-	mdev->anchor_list =
-		kcalloc(num_endpoints, sizeof(*mdev->anchor_list), GFP_KERNEL);
-	if (!mdev->anchor_list)
+	mdev->busy_urbs =
+		kcalloc(num_endpoints, sizeof(*mdev->busy_urbs), GFP_KERNEL);
+	if (!mdev->busy_urbs)
 		goto exit_free3;
 
 	tmp_cap = mdev->cap;
@@ -1297,21 +1254,21 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 		tmp_cap->num_buffers_packet = BUF_CHAIN_SIZE;
 		tmp_cap->num_buffers_streaming = BUF_CHAIN_SIZE;
 		tmp_cap->data_type = MOST_CH_CONTROL | MOST_CH_ASYNC |
-				     MOST_CH_ISOC_AVP | MOST_CH_SYNC;
+				     MOST_CH_ISOC | MOST_CH_SYNC;
 		if (usb_endpoint_dir_in(ep_desc))
 			tmp_cap->direction = MOST_CH_RX;
 		else
 			tmp_cap->direction = MOST_CH_TX;
 		tmp_cap++;
-		INIT_LIST_HEAD(&mdev->anchor_list[i]);
-		spin_lock_init(&mdev->anchor_list_lock[i]);
+		init_usb_anchor(&mdev->busy_urbs[i]);
+		spin_lock_init(&mdev->channel_lock[i]);
 		err = drci_wr_reg(usb_dev,
 				  DRCI_REG_BASE + DRCI_COMMAND +
 				  ep_desc->bEndpointAddress * 16,
 				  1);
 		if (err < 0)
-			pr_warn("DCI Sync for EP %02x failed",
-				ep_desc->bEndpointAddress);
+			dev_warn(dev, "DCI Sync for EP %02x failed",
+				 ep_desc->bEndpointAddress);
 	}
 	dev_notice(dev, "claimed gadget: Vendor=%4.4x ProdID=%4.4x Bus=%02x Device=%02x\n",
 		   le16_to_cpu(usb_dev->descriptor.idVendor),
@@ -1332,7 +1289,9 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 	}
 
 	mutex_lock(&mdev->io_mutex);
-	if (le16_to_cpu(usb_dev->descriptor.idProduct) == USB_DEV_ID_INIC) {
+	if (le16_to_cpu(usb_dev->descriptor.idProduct) == USB_DEV_ID_OS81118 ||
+	    le16_to_cpu(usb_dev->descriptor.idProduct) == USB_DEV_ID_OS81119 ||
+	    le16_to_cpu(usb_dev->descriptor.idProduct) == USB_DEV_ID_OS81210) {
 		/* this increments the reference count of the instance
 		 * object of the core
 		 */
@@ -1351,7 +1310,7 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 	return 0;
 
 exit_free4:
-	kfree(mdev->anchor_list);
+	kfree(mdev->busy_urbs);
 exit_free3:
 	kfree(mdev->ep_address);
 exit_free2:
@@ -1379,9 +1338,8 @@ exit_ENOMEM:
  */
 static void hdm_disconnect(struct usb_interface *interface)
 {
-	struct most_dev *mdev;
+	struct most_dev *mdev = usb_get_intfdata(interface);
 
-	mdev = usb_get_intfdata(interface);
 	mutex_lock(&mdev->io_mutex);
 	usb_set_intfdata(interface, NULL);
 	mdev->usb_device = NULL;
@@ -1393,7 +1351,7 @@ static void hdm_disconnect(struct usb_interface *interface)
 	destroy_most_dci_obj(mdev->dci);
 	most_deregister_interface(&mdev->iface);
 
-	kfree(mdev->anchor_list);
+	kfree(mdev->busy_urbs);
 	kfree(mdev->cap);
 	kfree(mdev->conf);
 	kfree(mdev->ep_address);
diff --git a/drivers/staging/most/mostcore/core.c b/drivers/staging/most/mostcore/core.c
index 7c619feb12d3..329109c0024f 100644
--- a/drivers/staging/most/mostcore/core.c
+++ b/drivers/staging/most/mostcore/core.c
@@ -33,7 +33,7 @@
 #define STRING_SIZE	80
 
 static struct class *most_class;
-static struct device *class_glue_dir;
+static struct device *core_dev;
 static struct ida mdev_id;
 static int dummy_num_buffers;
 
@@ -51,6 +51,7 @@ struct most_c_obj {
 	u16 channel_id;
 	bool is_poisoned;
 	struct mutex start_mutex;
+	struct mutex nq_mutex; /* nq thread synchronization */
 	int is_starving;
 	struct most_interface *iface;
 	struct most_inst_obj *inst;
@@ -82,10 +83,13 @@ struct most_inst_obj {
 static const struct {
 	int most_ch_data_type;
 	char *name;
-} ch_data_type[] = { { MOST_CH_CONTROL, "control\n" },
+} ch_data_type[] = {
+	{ MOST_CH_CONTROL, "control\n" },
 	{ MOST_CH_ASYNC, "async\n" },
 	{ MOST_CH_SYNC, "sync\n" },
-	{ MOST_CH_ISOC_AVP, "isoc_avp\n"} };
+	{ MOST_CH_ISOC, "isoc\n"},
+	{ MOST_CH_ISOC, "isoc_avp\n"},
+};
 
 #define to_inst_obj(d) container_of(d, struct most_inst_obj, kobj)
 
@@ -260,11 +264,11 @@ static ssize_t show_available_directions(struct most_c_obj *c,
 
 	strcpy(buf, "");
 	if (c->iface->channel_vector[i].direction & MOST_CH_RX)
-		strcat(buf, "dir_rx ");
+		strcat(buf, "rx ");
 	if (c->iface->channel_vector[i].direction & MOST_CH_TX)
-		strcat(buf, "dir_tx ");
+		strcat(buf, "tx ");
 	strcat(buf, "\n");
-	return strlen(buf) + 1;
+	return strlen(buf);
 }
 
 static ssize_t show_available_datatypes(struct most_c_obj *c,
@@ -280,10 +284,10 @@ static ssize_t show_available_datatypes(struct most_c_obj *c,
 		strcat(buf, "async ");
 	if (c->iface->channel_vector[i].data_type & MOST_CH_SYNC)
 		strcat(buf, "sync ");
-	if (c->iface->channel_vector[i].data_type & MOST_CH_ISOC_AVP)
-		strcat(buf, "isoc_avp ");
+	if (c->iface->channel_vector[i].data_type & MOST_CH_ISOC)
+		strcat(buf, "isoc ");
 	strcat(buf, "\n");
-	return strlen(buf) + 1;
+	return strlen(buf);
 }
 
 static
@@ -391,9 +395,9 @@ static ssize_t show_set_direction(struct most_c_obj *c,
 				  char *buf)
 {
 	if (c->cfg.direction & MOST_CH_TX)
-		return snprintf(buf, PAGE_SIZE, "dir_tx\n");
+		return snprintf(buf, PAGE_SIZE, "tx\n");
 	else if (c->cfg.direction & MOST_CH_RX)
-		return snprintf(buf, PAGE_SIZE, "dir_rx\n");
+		return snprintf(buf, PAGE_SIZE, "rx\n");
 	return snprintf(buf, PAGE_SIZE, "unconfigured\n");
 }
 
@@ -404,8 +408,12 @@ static ssize_t store_set_direction(struct most_c_obj *c,
 {
 	if (!strcmp(buf, "dir_rx\n")) {
 		c->cfg.direction = MOST_CH_RX;
+	} else if (!strcmp(buf, "rx\n")) {
+		c->cfg.direction = MOST_CH_RX;
 	} else if (!strcmp(buf, "dir_tx\n")) {
 		c->cfg.direction = MOST_CH_TX;
+	} else if (!strcmp(buf, "tx\n")) {
+		c->cfg.direction = MOST_CH_TX;
 	} else {
 		pr_info("WARN: invalid attribute settings\n");
 		return -EINVAL;
@@ -847,7 +855,23 @@ static ssize_t show_add_link(struct most_aim_obj *aim_obj,
 			     struct most_aim_attribute *attr,
 			     char *buf)
 {
-	return snprintf(buf, PAGE_SIZE, "%s\n", aim_obj->add_link);
+	struct most_c_obj *c;
+	struct most_inst_obj *i;
+	int offs = 0;
+
+	list_for_each_entry(i, &instance_list, list) {
+		list_for_each_entry(c, &i->channel_list, list) {
+			if (c->aim0.ptr == aim_obj->driver ||
+			    c->aim1.ptr == aim_obj->driver) {
+				offs += snprintf(buf + offs, PAGE_SIZE - offs,
+						 "%s:%s\n",
+						 kobject_name(&i->kobj),
+						 kobject_name(&c->kobj));
+			}
+		}
+	}
+
+	return offs;
 }
 
 /**
@@ -1131,18 +1155,18 @@ static inline void trash_mbo(struct mbo *mbo)
 	spin_unlock_irqrestore(&c->fifo_lock, flags);
 }
 
-static struct mbo *get_hdm_mbo(struct most_c_obj *c)
+static bool hdm_mbo_ready(struct most_c_obj *c)
 {
-	unsigned long flags;
-	struct mbo *mbo;
+	bool empty;
 
-	spin_lock_irqsave(&c->fifo_lock, flags);
-	if (c->enqueue_halt || list_empty(&c->halt_fifo))
-		mbo = NULL;
-	else
-		mbo = list_pop_mbo(&c->halt_fifo);
-	spin_unlock_irqrestore(&c->fifo_lock, flags);
-	return mbo;
+	if (c->enqueue_halt)
+		return false;
+
+	spin_lock_irq(&c->fifo_lock);
+	empty = list_empty(&c->halt_fifo);
+	spin_unlock_irq(&c->fifo_lock);
+
+	return !empty;
 }
 
 static void nq_hdm_mbo(struct mbo *mbo)
@@ -1160,20 +1184,32 @@ static int hdm_enqueue_thread(void *data)
 {
 	struct most_c_obj *c = data;
 	struct mbo *mbo;
+	int ret;
 	typeof(c->iface->enqueue) enqueue = c->iface->enqueue;
 
 	while (likely(!kthread_should_stop())) {
 		wait_event_interruptible(c->hdm_fifo_wq,
-					 (mbo = get_hdm_mbo(c)) ||
+					 hdm_mbo_ready(c) ||
 					 kthread_should_stop());
 
-		if (unlikely(!mbo))
+		mutex_lock(&c->nq_mutex);
+		spin_lock_irq(&c->fifo_lock);
+		if (unlikely(c->enqueue_halt || list_empty(&c->halt_fifo))) {
+			spin_unlock_irq(&c->fifo_lock);
+			mutex_unlock(&c->nq_mutex);
 			continue;
+		}
+
+		mbo = list_pop_mbo(&c->halt_fifo);
+		spin_unlock_irq(&c->fifo_lock);
 
 		if (c->cfg.direction == MOST_CH_RX)
 			mbo->buffer_length = c->cfg.buffer_size;
 
-		if (unlikely(enqueue(mbo->ifp, mbo->hdm_channel_id, mbo))) {
+		ret = enqueue(mbo->ifp, mbo->hdm_channel_id, mbo);
+		mutex_unlock(&c->nq_mutex);
+
+		if (unlikely(ret)) {
 			pr_err("hdm enqueue failed\n");
 			nq_hdm_mbo(mbo);
 			c->hdm_enqueue_task = NULL;
@@ -1294,17 +1330,14 @@ _exit:
 /**
  * most_submit_mbo - submits an MBO to fifo
  * @mbo: pointer to the MBO
- *
  */
-int most_submit_mbo(struct mbo *mbo)
+void most_submit_mbo(struct mbo *mbo)
 {
-	if (unlikely((!mbo) || (!mbo->context))) {
-		pr_err("Bad MBO or missing channel reference\n");
-		return -EINVAL;
-	}
+	if (WARN_ONCE(!mbo || !mbo->context,
+		      "bad mbo or missing channel reference\n"))
+		return;
 
 	nq_hdm_mbo(mbo);
-	return 0;
 }
 EXPORT_SYMBOL_GPL(most_submit_mbo);
 
@@ -1468,10 +1501,8 @@ static void most_read_completion(struct mbo *mbo)
 		return;
 	}
 
-	if (atomic_sub_and_test(1, &c->mbo_nq_level)) {
-		pr_info("WARN: rx device out of buffers\n");
+	if (atomic_sub_and_test(1, &c->mbo_nq_level))
 		c->is_starving = 1;
-	}
 
 	if (c->aim0.refs && c->aim0.ptr->rx_completion &&
 	    c->aim0.ptr->rx_completion(mbo) == 0)
@@ -1761,6 +1792,7 @@ struct kobject *most_register_interface(struct most_interface *iface)
 		init_completion(&c->cleanup);
 		atomic_set(&c->mbo_ref, 0);
 		mutex_init(&c->start_mutex);
+		mutex_init(&c->nq_mutex);
 		list_add_tail(&c->list, &inst->channel_list);
 	}
 	pr_info("registered new MOST device mdev%d (%s)\n",
@@ -1826,8 +1858,12 @@ void most_stop_enqueue(struct most_interface *iface, int id)
 {
 	struct most_c_obj *c = get_channel_by_iface(iface, id);
 
-	if (likely(c))
-		c->enqueue_halt = true;
+	if (!c)
+		return;
+
+	mutex_lock(&c->nq_mutex);
+	c->enqueue_halt = true;
+	mutex_unlock(&c->nq_mutex);
 }
 EXPORT_SYMBOL_GPL(most_stop_enqueue);
 
@@ -1843,9 +1879,12 @@ void most_resume_enqueue(struct most_interface *iface, int id)
 {
 	struct most_c_obj *c = get_channel_by_iface(iface, id);
 
-	if (unlikely(!c))
+	if (!c)
 		return;
+
+	mutex_lock(&c->nq_mutex);
 	c->enqueue_halt = false;
+	mutex_unlock(&c->nq_mutex);
 
 	wake_up_interruptible(&c->hdm_fifo_wq);
 }
@@ -1879,22 +1918,19 @@ static int __init most_init(void)
 		goto exit_class;
 	}
 
-	class_glue_dir =
-		device_create(most_class, NULL, 0, NULL, "mostcore");
-	if (IS_ERR(class_glue_dir)) {
-		err = PTR_ERR(class_glue_dir);
+	core_dev = device_create(most_class, NULL, 0, NULL, "mostcore");
+	if (IS_ERR(core_dev)) {
+		err = PTR_ERR(core_dev);
 		goto exit_driver;
 	}
 
-	most_aim_kset =
-		kset_create_and_add("aims", NULL, &class_glue_dir->kobj);
+	most_aim_kset = kset_create_and_add("aims", NULL, &core_dev->kobj);
 	if (!most_aim_kset) {
 		err = -ENOMEM;
 		goto exit_class_container;
 	}
 
-	most_inst_kset =
-		kset_create_and_add("devices", NULL, &class_glue_dir->kobj);
+	most_inst_kset = kset_create_and_add("devices", NULL, &core_dev->kobj);
 	if (!most_inst_kset) {
 		err = -ENOMEM;
 		goto exit_driver_kset;
diff --git a/drivers/staging/most/mostcore/mostcore.h b/drivers/staging/most/mostcore/mostcore.h
index 60e018e499ef..5f8339bd046f 100644
--- a/drivers/staging/most/mostcore/mostcore.h
+++ b/drivers/staging/most/mostcore/mostcore.h
@@ -56,7 +56,7 @@ enum most_channel_direction {
 enum most_channel_data_type {
 	MOST_CH_CONTROL = 1 << 0,
 	MOST_CH_ASYNC = 1 << 1,
-	MOST_CH_ISOC_AVP = 1 << 2,
+	MOST_CH_ISOC = 1 << 2,
 	MOST_CH_SYNC = 1 << 5,
 };
 
@@ -112,7 +112,7 @@ struct most_channel_capability {
 	u16 buffer_size_packet;
 	u16 num_buffers_streaming;
 	u16 buffer_size_streaming;
-	char *name_suffix;
+	const char *name_suffix;
 };
 
 /**
@@ -287,7 +287,7 @@ struct kobject *most_register_interface(struct most_interface *iface);
  * @intf_instance Pointer to the interface instance description.
  */
 void most_deregister_interface(struct most_interface *iface);
-int most_submit_mbo(struct mbo *mbo);
+void most_submit_mbo(struct mbo *mbo);
 
 /**
  * most_stop_enqueue - prevents core from enqueing MBOs
diff --git a/drivers/staging/netlogic/xlr_net.c b/drivers/staging/netlogic/xlr_net.c
index 99445d0fcf9c..552a7dcbf50b 100644
--- a/drivers/staging/netlogic/xlr_net.c
+++ b/drivers/staging/netlogic/xlr_net.c
@@ -192,7 +192,7 @@ static int xlr_set_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
 	return phy_ethtool_sset(phydev, ecmd);
 }
 
-static struct ethtool_ops xlr_ethtool_ops = {
+static const struct ethtool_ops xlr_ethtool_ops = {
 	.get_settings = xlr_get_settings,
 	.set_settings = xlr_set_settings,
 };
@@ -269,16 +269,6 @@ static void xlr_make_tx_desc(struct nlm_fmn_msg *msg, unsigned long addr,
 	msg->msg3 = 0;
 }
 
-static void __maybe_unused xlr_wakeup_queue(unsigned long dev)
-{
-	struct net_device *ndev = (struct net_device *)dev;
-	struct xlr_net_priv *priv = netdev_priv(ndev);
-	struct phy_device *phydev = xlr_get_phydev(priv);
-
-	if (phydev->link)
-		netif_tx_wake_queue(netdev_get_tx_queue(ndev, priv->wakeup_q));
-}
-
 static netdev_tx_t xlr_net_start_xmit(struct sk_buff *skb,
 				      struct net_device *ndev)
 {
@@ -413,7 +403,7 @@ static struct rtnl_link_stats64 *xlr_get_stats64(struct net_device *ndev,
 	return stats;
 }
 
-static struct net_device_ops xlr_netdev_ops = {
+static const struct net_device_ops xlr_netdev_ops = {
 	.ndo_open = xlr_net_open,
 	.ndo_stop = xlr_net_stop,
 	.ndo_start_xmit = xlr_net_start_xmit,
diff --git a/drivers/staging/octeon-usb/Kconfig b/drivers/staging/octeon-usb/Kconfig
index 16ea17ff3fd2..0b8f1d9c7056 100644
--- a/drivers/staging/octeon-usb/Kconfig
+++ b/drivers/staging/octeon-usb/Kconfig
@@ -6,5 +6,5 @@ config OCTEON_USB
 	  Networks' products in the Octeon family.
 
 	  To compile this driver as a module, choose M here. The module
-	  will be called octeon-usb.
+	  will be called octeon-hcd.
 
diff --git a/drivers/staging/octeon-usb/octeon-hcd.c b/drivers/staging/octeon-usb/octeon-hcd.c
index 17442b3ed849..9a7858a300fd 100644
--- a/drivers/staging/octeon-usb/octeon-hcd.c
+++ b/drivers/staging/octeon-usb/octeon-hcd.c
@@ -3292,7 +3292,6 @@ static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
 	spin_lock_irqsave(&usb->lock, flags);
 	port_status = cvmx_usb_get_status(usb);
 	spin_unlock_irqrestore(&usb->lock, flags);
-	buf[0] = 0;
 	buf[0] = port_status.connect_change << 1;
 
 	return buf[0] != 0;
diff --git a/drivers/staging/octeon/ethernet-mdio.c b/drivers/staging/octeon/ethernet-mdio.c
index e13a4ab46977..691e4a51ace4 100644
--- a/drivers/staging/octeon/ethernet-mdio.c
+++ b/drivers/staging/octeon/ethernet-mdio.c
@@ -34,48 +34,23 @@ static void cvm_oct_get_drvinfo(struct net_device *dev,
 	strlcpy(info->bus_info, "Builtin", sizeof(info->bus_info));
 }
 
-static int cvm_oct_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-	struct octeon_ethernet *priv = netdev_priv(dev);
-
-	if (priv->phydev)
-		return phy_ethtool_gset(priv->phydev, cmd);
-
-	return -EINVAL;
-}
-
-static int cvm_oct_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
-{
-	struct octeon_ethernet *priv = netdev_priv(dev);
-
-	if (!capable(CAP_NET_ADMIN))
-		return -EPERM;
-
-	if (priv->phydev)
-		return phy_ethtool_sset(priv->phydev, cmd);
-
-	return -EINVAL;
-}
-
 static int cvm_oct_nway_reset(struct net_device *dev)
 {
-	struct octeon_ethernet *priv = netdev_priv(dev);
-
 	if (!capable(CAP_NET_ADMIN))
 		return -EPERM;
 
-	if (priv->phydev)
-		return phy_start_aneg(priv->phydev);
+	if (dev->phydev)
+		return phy_start_aneg(dev->phydev);
 
 	return -EINVAL;
 }
 
 const struct ethtool_ops cvm_oct_ethtool_ops = {
 	.get_drvinfo = cvm_oct_get_drvinfo,
-	.get_settings = cvm_oct_get_settings,
-	.set_settings = cvm_oct_set_settings,
 	.nway_reset = cvm_oct_nway_reset,
 	.get_link = ethtool_op_get_link,
+	.get_link_ksettings = phy_ethtool_get_link_ksettings,
+	.set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
 /**
@@ -88,15 +63,13 @@ const struct ethtool_ops cvm_oct_ethtool_ops = {
  */
 int cvm_oct_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 {
-	struct octeon_ethernet *priv = netdev_priv(dev);
-
 	if (!netif_running(dev))
 		return -EINVAL;
 
-	if (!priv->phydev)
+	if (!dev->phydev)
 		return -EINVAL;
 
-	return phy_mii_ioctl(priv->phydev, rq, cmd);
+	return phy_mii_ioctl(dev->phydev, rq, cmd);
 }
 
 void cvm_oct_note_carrier(struct octeon_ethernet *priv,
@@ -119,9 +92,9 @@ void cvm_oct_adjust_link(struct net_device *dev)
 	cvmx_helper_link_info_t link_info;
 
 	link_info.u64		= 0;
-	link_info.s.link_up	= priv->phydev->link ? 1 : 0;
-	link_info.s.full_duplex = priv->phydev->duplex ? 1 : 0;
-	link_info.s.speed	= priv->phydev->speed;
+	link_info.s.link_up	= dev->phydev->link ? 1 : 0;
+	link_info.s.full_duplex = dev->phydev->duplex ? 1 : 0;
+	link_info.s.speed	= dev->phydev->speed;
 	priv->link_info		= link_info.u64;
 
 	/*
@@ -130,8 +103,8 @@ void cvm_oct_adjust_link(struct net_device *dev)
 	if (priv->poll)
 		priv->poll(dev);
 
-	if (priv->last_link != priv->phydev->link) {
-		priv->last_link = priv->phydev->link;
+	if (priv->last_link != dev->phydev->link) {
+		priv->last_link = dev->phydev->link;
 		cvmx_helper_link_set(priv->port, link_info);
 		cvm_oct_note_carrier(priv, link_info);
 	}
@@ -151,9 +124,8 @@ int cvm_oct_common_stop(struct net_device *dev)
 
 	priv->poll = NULL;
 
-	if (priv->phydev)
-		phy_disconnect(priv->phydev);
-	priv->phydev = NULL;
+	if (dev->phydev)
+		phy_disconnect(dev->phydev);
 
 	if (priv->last_link) {
 		link_info.u64 = 0;
@@ -176,6 +148,7 @@ int cvm_oct_phy_setup_device(struct net_device *dev)
 {
 	struct octeon_ethernet *priv = netdev_priv(dev);
 	struct device_node *phy_node;
+	struct phy_device *phydev = NULL;
 
 	if (!priv->of_node)
 		goto no_phy;
@@ -193,14 +166,15 @@ int cvm_oct_phy_setup_device(struct net_device *dev)
 	if (!phy_node)
 		goto no_phy;
 
-	priv->phydev = of_phy_connect(dev, phy_node, cvm_oct_adjust_link, 0,
-				      PHY_INTERFACE_MODE_GMII);
+	phydev = of_phy_connect(dev, phy_node, cvm_oct_adjust_link, 0,
+				PHY_INTERFACE_MODE_GMII);
+	of_node_put(phy_node);
 
-	if (!priv->phydev)
+	if (!phydev)
 		return -ENODEV;
 
 	priv->last_link = 0;
-	phy_start_aneg(priv->phydev);
+	phy_start_aneg(phydev);
 
 	return 0;
 no_phy:
diff --git a/drivers/staging/octeon/ethernet-rgmii.c b/drivers/staging/octeon/ethernet-rgmii.c
index 91b148cfcbdb..4e7304210bb9 100644
--- a/drivers/staging/octeon/ethernet-rgmii.c
+++ b/drivers/staging/octeon/ethernet-rgmii.c
@@ -117,7 +117,10 @@ static void cvm_oct_rgmii_poll(struct net_device *dev)
 	cvmx_helper_link_info_t link_info;
 	bool status_change;
 
-	link_info = cvmx_helper_link_autoconf(priv->port);
+	link_info = cvmx_helper_link_get(priv->port);
+	if (priv->link_info != link_info.u64 &&
+	    cvmx_helper_link_set(priv->port, link_info))
+		link_info.u64 = priv->link_info;
 	status_change = priv->link_info != link_info.u64;
 	priv->link_info = link_info.u64;
 
@@ -145,7 +148,7 @@ int cvm_oct_rgmii_open(struct net_device *dev)
 	if (ret)
 		return ret;
 
-	if (priv->phydev) {
+	if (dev->phydev) {
 		/*
 		 * In phydev mode, we need still periodic polling for the
 		 * preamble error checking, and we also need to call this
diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c
index a10fe3af9a9c..f0900d1c4d7b 100644
--- a/drivers/staging/octeon/ethernet-rx.c
+++ b/drivers/staging/octeon/ethernet-rx.c
@@ -43,21 +43,27 @@
 
 #include <asm/octeon/cvmx-gmxx-defs.h>
 
-static struct napi_struct cvm_oct_napi;
+static atomic_t oct_rx_ready = ATOMIC_INIT(0);
+
+static struct oct_rx_group {
+	int irq;
+	int group;
+	struct napi_struct napi;
+} oct_rx_group[16];
 
 /**
  * cvm_oct_do_interrupt - interrupt handler.
- * @cpl: Interrupt number. Unused
- * @dev_id: Cookie to identify the device. Unused
+ * @irq: Interrupt number.
+ * @napi_id: Cookie to identify the NAPI instance.
  *
  * The interrupt occurs whenever the POW has packets in our group.
  *
  */
-static irqreturn_t cvm_oct_do_interrupt(int cpl, void *dev_id)
+static irqreturn_t cvm_oct_do_interrupt(int irq, void *napi_id)
 {
 	/* Disable the IRQ and start napi_poll. */
-	disable_irq_nosync(OCTEON_IRQ_WORKQ0 + pow_receive_group);
-	napi_schedule(&cvm_oct_napi);
+	disable_irq_nosync(irq);
+	napi_schedule(napi_id);
 
 	return IRQ_HANDLED;
 }
@@ -143,14 +149,7 @@ static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work)
 	return 0;
 }
 
-/**
- * cvm_oct_napi_poll - the NAPI poll function.
- * @napi: The NAPI instance, or null if called from cvm_oct_poll_controller
- * @budget: Maximum number of packets to receive.
- *
- * Returns the number of packets processed.
- */
-static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)
+static int cvm_oct_poll(struct oct_rx_group *rx_group, int budget)
 {
 	const int	coreid = cvmx_get_core_num();
 	u64	old_group_mask;
@@ -172,13 +171,13 @@ static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)
 	if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
 		old_group_mask = cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid));
 		cvmx_write_csr(CVMX_SSO_PPX_GRP_MSK(coreid),
-			       1ull << pow_receive_group);
+			       BIT(rx_group->group));
 		cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); /* Flush */
 	} else {
 		old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid));
 		cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid),
 			       (old_group_mask & ~0xFFFFull) |
-			       1 << pow_receive_group);
+			       BIT(rx_group->group));
 	}
 
 	if (USE_ASYNC_IOBDMA) {
@@ -203,15 +202,15 @@ static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)
 		if (!work) {
 			if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
 				cvmx_write_csr(CVMX_SSO_WQ_IQ_DIS,
-					       1ull << pow_receive_group);
+					       BIT(rx_group->group));
 				cvmx_write_csr(CVMX_SSO_WQ_INT,
-					       1ull << pow_receive_group);
+					       BIT(rx_group->group));
 			} else {
 				union cvmx_pow_wq_int wq_int;
 
 				wq_int.u64 = 0;
-				wq_int.s.iq_dis = 1 << pow_receive_group;
-				wq_int.s.wq_int = 1 << pow_receive_group;
+				wq_int.s.iq_dis = BIT(rx_group->group);
+				wq_int.s.wq_int = BIT(rx_group->group);
 				cvmx_write_csr(CVMX_POW_WQ_INT, wq_int.u64);
 			}
 			break;
@@ -410,10 +409,28 @@ static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)
 	}
 	cvm_oct_rx_refill_pool(0);
 
-	if (rx_count < budget && napi) {
+	return rx_count;
+}
+
+/**
+ * cvm_oct_napi_poll - the NAPI poll function.
+ * @napi: The NAPI instance.
+ * @budget: Maximum number of packets to receive.
+ *
+ * Returns the number of packets processed.
+ */
+static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)
+{
+	struct oct_rx_group *rx_group = container_of(napi, struct oct_rx_group,
+						     napi);
+	int rx_count;
+
+	rx_count = cvm_oct_poll(rx_group, budget);
+
+	if (rx_count < budget) {
 		/* No more work */
 		napi_complete(napi);
-		enable_irq(OCTEON_IRQ_WORKQ0 + pow_receive_group);
+		enable_irq(rx_group->irq);
 	}
 	return rx_count;
 }
@@ -427,7 +444,17 @@ static int cvm_oct_napi_poll(struct napi_struct *napi, int budget)
  */
 void cvm_oct_poll_controller(struct net_device *dev)
 {
-	cvm_oct_napi_poll(NULL, 16);
+	int i;
+
+	if (!atomic_read(&oct_rx_ready))
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) {
+		if (!(pow_receive_groups & BIT(i)))
+			continue;
+
+		cvm_oct_poll(&oct_rx_group[i], 16);
+	}
 }
 #endif
 
@@ -446,54 +473,80 @@ void cvm_oct_rx_initialize(void)
 	if (!dev_for_napi)
 		panic("No net_devices were allocated.");
 
-	netif_napi_add(dev_for_napi, &cvm_oct_napi, cvm_oct_napi_poll,
-		       rx_napi_weight);
-	napi_enable(&cvm_oct_napi);
+	for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) {
+		int ret;
 
-	/* Register an IRQ handler to receive POW interrupts */
-	i = request_irq(OCTEON_IRQ_WORKQ0 + pow_receive_group,
-			cvm_oct_do_interrupt, 0, "Ethernet", cvm_oct_device);
+		if (!(pow_receive_groups & BIT(i)))
+			continue;
 
-	if (i)
-		panic("Could not acquire Ethernet IRQ %d\n",
-		      OCTEON_IRQ_WORKQ0 + pow_receive_group);
+		netif_napi_add(dev_for_napi, &oct_rx_group[i].napi,
+			       cvm_oct_napi_poll, rx_napi_weight);
+		napi_enable(&oct_rx_group[i].napi);
 
-	disable_irq_nosync(OCTEON_IRQ_WORKQ0 + pow_receive_group);
+		oct_rx_group[i].irq = OCTEON_IRQ_WORKQ0 + i;
+		oct_rx_group[i].group = i;
 
-	/* Enable POW interrupt when our port has at least one packet */
-	if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
-		union cvmx_sso_wq_int_thrx int_thr;
-		union cvmx_pow_wq_int_pc int_pc;
-
-		int_thr.u64 = 0;
-		int_thr.s.tc_en = 1;
-		int_thr.s.tc_thr = 1;
-		cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(pow_receive_group),
-			       int_thr.u64);
-
-		int_pc.u64 = 0;
-		int_pc.s.pc_thr = 5;
-		cvmx_write_csr(CVMX_SSO_WQ_INT_PC, int_pc.u64);
-	} else {
-		union cvmx_pow_wq_int_thrx int_thr;
-		union cvmx_pow_wq_int_pc int_pc;
-
-		int_thr.u64 = 0;
-		int_thr.s.tc_en = 1;
-		int_thr.s.tc_thr = 1;
-		cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group),
-			       int_thr.u64);
-
-		int_pc.u64 = 0;
-		int_pc.s.pc_thr = 5;
-		cvmx_write_csr(CVMX_POW_WQ_INT_PC, int_pc.u64);
-	}
+		/* Register an IRQ handler to receive POW interrupts */
+		ret = request_irq(oct_rx_group[i].irq, cvm_oct_do_interrupt, 0,
+				  "Ethernet", &oct_rx_group[i].napi);
+		if (ret)
+			panic("Could not acquire Ethernet IRQ %d\n",
+			      oct_rx_group[i].irq);
+
+		disable_irq_nosync(oct_rx_group[i].irq);
+
+		/* Enable POW interrupt when our port has at least one packet */
+		if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
+			union cvmx_sso_wq_int_thrx int_thr;
+			union cvmx_pow_wq_int_pc int_pc;
+
+			int_thr.u64 = 0;
+			int_thr.s.tc_en = 1;
+			int_thr.s.tc_thr = 1;
+			cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(i), int_thr.u64);
+
+			int_pc.u64 = 0;
+			int_pc.s.pc_thr = 5;
+			cvmx_write_csr(CVMX_SSO_WQ_INT_PC, int_pc.u64);
+		} else {
+			union cvmx_pow_wq_int_thrx int_thr;
+			union cvmx_pow_wq_int_pc int_pc;
+
+			int_thr.u64 = 0;
+			int_thr.s.tc_en = 1;
+			int_thr.s.tc_thr = 1;
+			cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), int_thr.u64);
 
-	/* Schedule NAPI now. This will indirectly enable the interrupt. */
-	napi_schedule(&cvm_oct_napi);
+			int_pc.u64 = 0;
+			int_pc.s.pc_thr = 5;
+			cvmx_write_csr(CVMX_POW_WQ_INT_PC, int_pc.u64);
+		}
+
+		/* Schedule NAPI now. This will indirectly enable the
+		 * interrupt.
+		 */
+		napi_schedule(&oct_rx_group[i].napi);
+	}
+	atomic_inc(&oct_rx_ready);
 }
 
 void cvm_oct_rx_shutdown(void)
 {
-	netif_napi_del(&cvm_oct_napi);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) {
+		if (!(pow_receive_groups & BIT(i)))
+			continue;
+
+		/* Disable POW interrupt */
+		if (OCTEON_IS_MODEL(OCTEON_CN68XX))
+			cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(i), 0);
+		else
+			cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), 0);
+
+		/* Free the interrupt handler */
+		free_irq(oct_rx_group[i].irq, cvm_oct_device);
+
+		netif_napi_del(&oct_rx_group[i].napi);
+	}
 }
diff --git a/drivers/staging/octeon/ethernet-util.h b/drivers/staging/octeon/ethernet-util.h
index 45f024bc5e33..617da8037a4d 100644
--- a/drivers/staging/octeon/ethernet-util.h
+++ b/drivers/staging/octeon/ethernet-util.h
@@ -32,12 +32,13 @@ static inline void *cvm_oct_get_buffer_ptr(union cvmx_buf_ptr packet_ptr)
  */
 static inline int INTERFACE(int ipd_port)
 {
-	int interface = cvmx_helper_get_interface_num(ipd_port);
+	int interface;
 
+	if (ipd_port == CVMX_PIP_NUM_INPUT_PORTS)
+		return 10;
+	interface = cvmx_helper_get_interface_num(ipd_port);
 	if (interface >= 0)
 		return interface;
-	else if (ipd_port == CVMX_PIP_NUM_INPUT_PORTS)
-		return 10;
 	panic("Illegal ipd_port %d passed to INTERFACE\n", ipd_port);
 }
 
diff --git a/drivers/staging/octeon/ethernet.c b/drivers/staging/octeon/ethernet.c
index e9cd5f242921..d02e3e31ed29 100644
--- a/drivers/staging/octeon/ethernet.c
+++ b/drivers/staging/octeon/ethernet.c
@@ -17,6 +17,8 @@
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/of_net.h>
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
 
 #include <net/dst.h>
 
@@ -35,17 +37,19 @@
 #include <asm/octeon/cvmx-fau.h>
 #include <asm/octeon/cvmx-ipd.h>
 #include <asm/octeon/cvmx-helper.h>
-
+#include <asm/octeon/cvmx-asxx-defs.h>
 #include <asm/octeon/cvmx-gmxx-defs.h>
 #include <asm/octeon/cvmx-smix-defs.h>
 
+#define OCTEON_MAX_MTU 65392
+
 static int num_packet_buffers = 1024;
 module_param(num_packet_buffers, int, 0444);
 MODULE_PARM_DESC(num_packet_buffers, "\n"
 	"\tNumber of packet buffers to allocate and store in the\n"
 	"\tFPA. By default, 1024 packet buffers are used.\n");
 
-int pow_receive_group = 15;
+static int pow_receive_group = 15;
 module_param(pow_receive_group, int, 0444);
 MODULE_PARM_DESC(pow_receive_group, "\n"
 	"\tPOW group to receive packets from. All ethernet hardware\n"
@@ -53,6 +57,15 @@ MODULE_PARM_DESC(pow_receive_group, "\n"
 	"\tgroup. Also any other software can submit packets to this\n"
 	"\tgroup for the kernel to process.");
 
+static int receive_group_order;
+module_param(receive_group_order, int, 0444);
+MODULE_PARM_DESC(receive_group_order, "\n"
+	"\tOrder (0..4) of receive groups to take into use. Ethernet hardware\n"
+	"\twill be configured to send incoming packets to multiple POW\n"
+	"\tgroups. pow_receive_group parameter is ignored when multiple\n"
+	"\tgroups are taken into use and groups are allocated starting\n"
+	"\tfrom 0. By default, a single group is used.\n");
+
 int pow_send_group = -1;
 module_param(pow_send_group, int, 0644);
 MODULE_PARM_DESC(pow_send_group, "\n"
@@ -86,6 +99,8 @@ int rx_napi_weight = 32;
 module_param(rx_napi_weight, int, 0444);
 MODULE_PARM_DESC(rx_napi_weight, "The NAPI WEIGHT parameter.");
 
+/* Mask indicating which receive groups are in use. */
+int pow_receive_groups;
 
 /*
  * cvm_oct_poll_queue_stopping - flag to indicate polling should stop.
@@ -237,21 +252,22 @@ static int cvm_oct_common_change_mtu(struct net_device *dev, int new_mtu)
 {
 	struct octeon_ethernet *priv = netdev_priv(dev);
 	int interface = INTERFACE(priv->port);
-	int index = INDEX(priv->port);
-#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
-	int vlan_bytes = 4;
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
+	int vlan_bytes = VLAN_HLEN;
 #else
 	int vlan_bytes = 0;
 #endif
+	int mtu_overhead = ETH_HLEN + ETH_FCS_LEN + vlan_bytes;
 
 	/*
 	 * Limit the MTU to make sure the ethernet packets are between
 	 * 64 bytes and 65535 bytes.
 	 */
-	if ((new_mtu + 14 + 4 + vlan_bytes < 64) ||
-	    (new_mtu + 14 + 4 + vlan_bytes > 65392)) {
+	if ((new_mtu + mtu_overhead < VLAN_ETH_ZLEN) ||
+	    (new_mtu + mtu_overhead > OCTEON_MAX_MTU)) {
 		pr_err("MTU must be between %d and %d.\n",
-		       64 - 14 - 4 - vlan_bytes, 65392 - 14 - 4 - vlan_bytes);
+		       VLAN_ETH_ZLEN - mtu_overhead,
+		       OCTEON_MAX_MTU - mtu_overhead);
 		return -EINVAL;
 	}
 	dev->mtu = new_mtu;
@@ -259,8 +275,9 @@ static int cvm_oct_common_change_mtu(struct net_device *dev, int new_mtu)
 	if ((interface < 2) &&
 	    (cvmx_helper_interface_get_mode(interface) !=
 		CVMX_HELPER_INTERFACE_MODE_SPI)) {
+		int index = INDEX(priv->port);
 		/* Add ethernet header and FCS, and VLAN if configured. */
-		int max_packet = new_mtu + 14 + 4 + vlan_bytes;
+		int max_packet = new_mtu + mtu_overhead;
 
 		if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 		    OCTEON_IS_MODEL(OCTEON_CN58XX)) {
@@ -275,7 +292,7 @@ static int cvm_oct_common_change_mtu(struct net_device *dev, int new_mtu)
 			union cvmx_pip_frm_len_chkx frm_len_chk;
 
 			frm_len_chk.u64 = 0;
-			frm_len_chk.s.minlen = 64;
+			frm_len_chk.s.minlen = VLAN_ETH_ZLEN;
 			frm_len_chk.s.maxlen = max_packet;
 			cvmx_write_csr(CVMX_PIP_FRM_LEN_CHKX(interface),
 				       frm_len_chk.u64);
@@ -300,12 +317,12 @@ static void cvm_oct_common_set_multicast_list(struct net_device *dev)
 	union cvmx_gmxx_prtx_cfg gmx_cfg;
 	struct octeon_ethernet *priv = netdev_priv(dev);
 	int interface = INTERFACE(priv->port);
-	int index = INDEX(priv->port);
 
 	if ((interface < 2) &&
 	    (cvmx_helper_interface_get_mode(interface) !=
 		CVMX_HELPER_INTERFACE_MODE_SPI)) {
 		union cvmx_gmxx_rxx_adr_ctl control;
+		int index = INDEX(priv->port);
 
 		control.u64 = 0;
 		control.s.bcst = 1;	/* Allow broadcast MAC addresses */
@@ -352,7 +369,6 @@ static int cvm_oct_set_mac_filter(struct net_device *dev)
 	struct octeon_ethernet *priv = netdev_priv(dev);
 	union cvmx_gmxx_prtx_cfg gmx_cfg;
 	int interface = INTERFACE(priv->port);
-	int index = INDEX(priv->port);
 
 	if ((interface < 2) &&
 	    (cvmx_helper_interface_get_mode(interface) !=
@@ -360,6 +376,7 @@ static int cvm_oct_set_mac_filter(struct net_device *dev)
 		int i;
 		u8 *ptr = dev->dev_addr;
 		u64 mac = 0;
+		int index = INDEX(priv->port);
 
 		for (i = 0; i < 6; i++)
 			mac = (mac << 8) | (u64)ptr[i];
@@ -457,10 +474,8 @@ int cvm_oct_common_init(struct net_device *dev)
 
 void cvm_oct_common_uninit(struct net_device *dev)
 {
-	struct octeon_ethernet *priv = netdev_priv(dev);
-
-	if (priv->phydev)
-		phy_disconnect(priv->phydev);
+	if (dev->phydev)
+		phy_disconnect(dev->phydev);
 }
 
 int cvm_oct_common_open(struct net_device *dev,
@@ -479,15 +494,17 @@ int cvm_oct_common_open(struct net_device *dev,
 
 	gmx_cfg.u64 = cvmx_read_csr(CVMX_GMXX_PRTX_CFG(index, interface));
 	gmx_cfg.s.en = 1;
+	if (octeon_has_feature(OCTEON_FEATURE_PKND))
+		gmx_cfg.s.pknd = priv->port;
 	cvmx_write_csr(CVMX_GMXX_PRTX_CFG(index, interface), gmx_cfg.u64);
 
 	if (octeon_is_simulation())
 		return 0;
 
-	if (priv->phydev) {
-		int r = phy_read_status(priv->phydev);
+	if (dev->phydev) {
+		int r = phy_read_status(dev->phydev);
 
-		if (r == 0 && priv->phydev->link == 0)
+		if (r == 0 && dev->phydev->link == 0)
 			netif_carrier_off(dev);
 		cvm_oct_adjust_link(dev);
 	} else {
@@ -510,8 +527,10 @@ void cvm_oct_link_poll(struct net_device *dev)
 	if (link_info.u64 == priv->link_info)
 		return;
 
-	link_info = cvmx_helper_link_autoconf(priv->port);
-	priv->link_info = link_info.u64;
+	if (cvmx_helper_link_set(priv->port, link_info))
+		link_info.u64 = priv->link_info;
+	else
+		priv->link_info = link_info.u64;
 
 	if (link_info.s.link_up) {
 		if (!netif_carrier_ok(dev))
@@ -649,6 +668,16 @@ static struct device_node *cvm_oct_node_for_port(struct device_node *pip,
 	return np;
 }
 
+static void cvm_set_rgmii_delay(struct device_node *np, int iface, int port)
+{
+	u32 delay_value;
+
+	if (!of_property_read_u32(np, "rx-delay", &delay_value))
+		cvmx_write_csr(CVMX_ASXX_RX_CLK_SETX(port, iface), delay_value);
+	if (!of_property_read_u32(np, "tx-delay", &delay_value))
+		cvmx_write_csr(CVMX_ASXX_TX_CLK_SETX(port, iface), delay_value);
+}
+
 static int cvm_oct_probe(struct platform_device *pdev)
 {
 	int num_interfaces;
@@ -665,11 +694,18 @@ static int cvm_oct_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-
 	cvm_oct_configure_common_hw();
 
 	cvmx_helper_initialize_packet_io_global();
 
+	if (receive_group_order) {
+		if (receive_group_order > 4)
+			receive_group_order = 4;
+		pow_receive_groups = (1 << (1 << receive_group_order)) - 1;
+	} else {
+		pow_receive_groups = BIT(pow_receive_group);
+	}
+
 	/* Change the input group for all ports before input is enabled */
 	num_interfaces = cvmx_helper_get_number_of_interfaces();
 	for (interface = 0; interface < num_interfaces; interface++) {
@@ -683,7 +719,37 @@ static int cvm_oct_probe(struct platform_device *pdev)
 
 			pip_prt_tagx.u64 =
 			    cvmx_read_csr(CVMX_PIP_PRT_TAGX(port));
-			pip_prt_tagx.s.grp = pow_receive_group;
+
+			if (receive_group_order) {
+				int tag_mask;
+
+				/* We support only 16 groups at the moment, so
+				 * always disable the two additional "hidden"
+				 * tag_mask bits on CN68XX.
+				 */
+				if (OCTEON_IS_MODEL(OCTEON_CN68XX))
+					pip_prt_tagx.u64 |= 0x3ull << 44;
+
+				tag_mask = ~((1 << receive_group_order) - 1);
+				pip_prt_tagx.s.grptagbase	= 0;
+				pip_prt_tagx.s.grptagmask	= tag_mask;
+				pip_prt_tagx.s.grptag		= 1;
+				pip_prt_tagx.s.tag_mode		= 0;
+				pip_prt_tagx.s.inc_prt_flag	= 1;
+				pip_prt_tagx.s.ip6_dprt_flag	= 1;
+				pip_prt_tagx.s.ip4_dprt_flag	= 1;
+				pip_prt_tagx.s.ip6_sprt_flag	= 1;
+				pip_prt_tagx.s.ip4_sprt_flag	= 1;
+				pip_prt_tagx.s.ip6_dst_flag	= 1;
+				pip_prt_tagx.s.ip4_dst_flag	= 1;
+				pip_prt_tagx.s.ip6_src_flag	= 1;
+				pip_prt_tagx.s.ip4_src_flag	= 1;
+				pip_prt_tagx.s.grp		= 0;
+			} else {
+				pip_prt_tagx.s.grptag	= 0;
+				pip_prt_tagx.s.grp	= pow_receive_group;
+			}
+
 			cvmx_write_csr(CVMX_PIP_PRT_TAGX(port),
 				       pip_prt_tagx.u64);
 		}
@@ -705,7 +771,6 @@ static int cvm_oct_probe(struct platform_device *pdev)
 	if ((pow_send_group != -1)) {
 		struct net_device *dev;
 
-		pr_info("\tConfiguring device for POW only access\n");
 		dev = alloc_etherdev(sizeof(struct octeon_ethernet));
 		if (dev) {
 			/* Initialize the device private structure. */
@@ -808,6 +873,8 @@ static int cvm_oct_probe(struct platform_device *pdev)
 			case CVMX_HELPER_INTERFACE_MODE_GMII:
 				dev->netdev_ops = &cvm_oct_rgmii_netdev_ops;
 				strcpy(dev->name, "eth%d");
+				cvm_set_rgmii_delay(priv->of_node, interface,
+						    port_index);
 				break;
 			}
 
@@ -844,17 +911,8 @@ static int cvm_oct_remove(struct platform_device *pdev)
 {
 	int port;
 
-	/* Disable POW interrupt */
-	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
-		cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(pow_receive_group), 0);
-	else
-		cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0);
-
 	cvmx_ipd_disable();
 
-	/* Free the interrupt handler */
-	free_irq(OCTEON_IRQ_WORKQ0 + pow_receive_group, cvm_oct_device);
-
 	atomic_inc_return(&cvm_oct_poll_queue_stopping);
 	cancel_delayed_work_sync(&cvm_oct_rx_refill_work);
 
@@ -878,7 +936,6 @@ static int cvm_oct_remove(struct platform_device *pdev)
 		}
 	}
 
-
 	cvmx_pko_shutdown();
 
 	cvmx_ipd_free_ptr();
diff --git a/drivers/staging/octeon/octeon-ethernet.h b/drivers/staging/octeon/octeon-ethernet.h
index 6275c15e0035..9c6852d61c0d 100644
--- a/drivers/staging/octeon/octeon-ethernet.h
+++ b/drivers/staging/octeon/octeon-ethernet.h
@@ -40,7 +40,6 @@ struct octeon_ethernet {
 	struct sk_buff_head tx_free_list[16];
 	/* Device statistics */
 	struct net_device_stats stats;
-	struct phy_device *phydev;
 	unsigned int last_speed;
 	unsigned int last_link;
 	/* Last negotiated link state */
@@ -73,7 +72,7 @@ void cvm_oct_link_poll(struct net_device *dev);
 
 extern int always_use_pow;
 extern int pow_send_group;
-extern int pow_receive_group;
+extern int pow_receive_groups;
 extern char pow_send_list[];
 extern struct net_device *cvm_oct_device[];
 extern atomic_t cvm_oct_poll_queue_stopping;
diff --git a/drivers/staging/olpc_dcon/olpc_dcon_xo_1_5.c b/drivers/staging/olpc_dcon/olpc_dcon_xo_1_5.c
index 6a4d379c16a3..1e23ef15b263 100644
--- a/drivers/staging/olpc_dcon/olpc_dcon_xo_1_5.c
+++ b/drivers/staging/olpc_dcon/olpc_dcon_xo_1_5.c
@@ -49,7 +49,7 @@ static void dcon_clear_irq(void)
 
 static int dcon_was_irq(void)
 {
-	u_int8_t tmp;
+	u8 tmp;
 
 	/* irq status will appear in PMIO_Rx50[6] on gpio12 */
 	tmp = inb(VX855_GPI_STATUS_CHG);
diff --git a/drivers/staging/rtl8188eu/core/rtw_ap.c b/drivers/staging/rtl8188eu/core/rtw_ap.c
index a5755358cc5d..553e8d50352f 100644
--- a/drivers/staging/rtl8188eu/core/rtw_ap.c
+++ b/drivers/staging/rtl8188eu/core/rtw_ap.c
@@ -78,9 +78,9 @@ static void update_BCNTIM(struct adapter *padapter)
 	/* update TIM IE */
 	p = rtw_get_ie(pie + _FIXED_IE_LENGTH_, _TIM_IE_, &tim_ielen,
 			pnetwork_mlmeext->IELength - _FIXED_IE_LENGTH_);
-	if (p != NULL && tim_ielen > 0) {
+	if (p && tim_ielen > 0) {
 		tim_ielen += 2;
-		premainder_ie = p+tim_ielen;
+		premainder_ie = p + tim_ielen;
 		tim_ie_offset = (int)(p - pie);
 		remainder_ielen = pnetwork_mlmeext->IELength -
 					tim_ie_offset - tim_ielen;
@@ -98,7 +98,7 @@ static void update_BCNTIM(struct adapter *padapter)
 				_SUPPORTEDRATES_IE_, &tmp_len,
 				(pnetwork_mlmeext->IELength -
 					_BEACON_IE_OFFSET_));
-		if (p !=  NULL)
+		if (p)
 			offset += tmp_len+2;
 
 		/* DS Parameter Set IE, len = 3 */
@@ -183,10 +183,10 @@ void rtw_add_bcn_ie(struct adapter *padapter, struct wlan_bssid_ex *pnetwork,
 		i += (pIE->Length + 2);
 	}
 
-	if (p != NULL && ielen > 0) {
+	if (p && ielen > 0) {
 		ielen += 2;
 
-		premainder_ie = p+ielen;
+		premainder_ie = p + ielen;
 
 		ie_offset = (int)(p - pie);
 
@@ -195,7 +195,7 @@ void rtw_add_bcn_ie(struct adapter *padapter, struct wlan_bssid_ex *pnetwork,
 		if (bmatch)
 			dst_ie = p;
 		else
-			dst_ie = p+ielen;
+			dst_ie = p + ielen;
 	}
 
 	if (remainder_ielen > 0) {
@@ -232,10 +232,10 @@ void rtw_remove_bcn_ie(struct adapter *padapter, struct wlan_bssid_ex *pnetwork,
 
 	p = rtw_get_ie(pie + _FIXED_IE_LENGTH_, index, &ielen,
 		       pnetwork->IELength - _FIXED_IE_LENGTH_);
-	if (p != NULL && ielen > 0) {
+	if (p && ielen > 0) {
 		ielen += 2;
 
-		premainder_ie = p+ielen;
+		premainder_ie = p + ielen;
 
 		ie_offset = (int)(p - pie);
 
@@ -385,8 +385,8 @@ void	expire_timeout_chk(struct adapter *padapter)
 			updated = ap_free_sta(padapter, psta, true, WLAN_REASON_DEAUTH_LEAVING);
 		} else {
 			/* TODO: Aging mechanism to digest frames in sleep_q to avoid running out of xmitframe */
-			if (psta->sleepq_len > (NR_XMITFRAME/pstapriv->asoc_list_cnt) &&
-			    padapter->xmitpriv.free_xmitframe_cnt < (NR_XMITFRAME/pstapriv->asoc_list_cnt/2)) {
+			if (psta->sleepq_len > (NR_XMITFRAME / pstapriv->asoc_list_cnt) &&
+			    padapter->xmitpriv.free_xmitframe_cnt < (NR_XMITFRAME / pstapriv->asoc_list_cnt / 2)) {
 				DBG_88E("%s sta:%pM, sleepq_len:%u, free_xmitframe_cnt:%u, asoc_list_cnt:%u, clear sleep_q\n", __func__,
 					(psta->hwaddr), psta->sleepq_len,
 					padapter->xmitpriv.free_xmitframe_cnt,
@@ -470,7 +470,7 @@ void add_RATid(struct adapter *padapter, struct sta_info *psta, u8 rssi_level)
 	/* b/g mode ra_bitmap */
 	for (i = 0; i < sizeof(psta->bssrateset); i++) {
 		if (psta->bssrateset[i])
-			tx_ra_bitmap |= rtw_get_bit_value_from_ieee_value(psta->bssrateset[i]&0x7f);
+			tx_ra_bitmap |= rtw_get_bit_value_from_ieee_value(psta->bssrateset[i] & 0x7f);
 	}
 	/* n mode ra_bitmap */
 	if (psta_ht->ht_option) {
@@ -481,8 +481,8 @@ void add_RATid(struct adapter *padapter, struct sta_info *psta, u8 rssi_level)
 			limit = 8;/*   1R */
 
 		for (i = 0; i < limit; i++) {
-			if (psta_ht->ht_cap.supp_mcs_set[i/8] & BIT(i%8))
-				tx_ra_bitmap |= BIT(i+12);
+			if (psta_ht->ht_cap.mcs.rx_mask[i / 8] & BIT(i % 8))
+				tx_ra_bitmap |= BIT(i + 12);
 		}
 
 		/* max short GI rate */
@@ -507,19 +507,19 @@ void add_RATid(struct adapter *padapter, struct sta_info *psta, u8 rssi_level)
 	psta->wireless_mode = sta_band;
 
 	raid = networktype_to_raid(sta_band);
-	init_rate = get_highest_rate_idx(tx_ra_bitmap&0x0fffffff)&0x3f;
+	init_rate = get_highest_rate_idx(tx_ra_bitmap & 0x0fffffff) & 0x3f;
 
 	if (psta->aid < NUM_STA) {
 		u8 arg = 0;
 
-		arg = psta->mac_id&0x1f;
+		arg = psta->mac_id & 0x1f;
 
 		arg |= BIT(7);/* support entry 2~31 */
 
 		if (shortGIrate)
 			arg |= BIT(5);
 
-		tx_ra_bitmap |= ((raid<<28)&0xf0000000);
+		tx_ra_bitmap |= ((raid << 28) & 0xf0000000);
 
 		DBG_88E("%s => mac_id:%d , raid:%d , bitmap = 0x%x, arg = 0x%x\n",
 			__func__, psta->mac_id, raid, tx_ra_bitmap, arg);
@@ -573,7 +573,7 @@ static void update_bmc_sta(struct adapter *padapter)
 		/* b/g mode ra_bitmap */
 		for (i = 0; i < supportRateNum; i++) {
 			if (psta->bssrateset[i])
-				tx_ra_bitmap |= rtw_get_bit_value_from_ieee_value(psta->bssrateset[i]&0x7f);
+				tx_ra_bitmap |= rtw_get_bit_value_from_ieee_value(psta->bssrateset[i] & 0x7f);
 		}
 
 		if (pcur_network->Configuration.DSConfig > 14) {
@@ -587,7 +587,7 @@ static void update_bmc_sta(struct adapter *padapter)
 		}
 
 		raid = networktype_to_raid(network_type);
-		init_rate = get_highest_rate_idx(tx_ra_bitmap&0x0fffffff)&0x3f;
+		init_rate = get_highest_rate_idx(tx_ra_bitmap & 0x0fffffff) & 0x3f;
 
 		/* ap mode */
 		rtw_hal_set_odm_var(padapter, HAL_ODM_STA_INFO, psta, true);
@@ -597,7 +597,7 @@ static void update_bmc_sta(struct adapter *padapter)
 
 			arg = psta->mac_id&0x1f;
 			arg |= BIT(7);
-			tx_ra_bitmap |= ((raid<<28)&0xf0000000);
+			tx_ra_bitmap |= ((raid << 28) & 0xf0000000);
 			DBG_88E("update_bmc_sta, mask = 0x%x, arg = 0x%x\n", tx_ra_bitmap, arg);
 
 			/* bitmap[0:27] = tx_rate_bitmap */
@@ -636,7 +636,7 @@ void update_sta_info_apmode(struct adapter *padapter, struct sta_info *psta)
 	struct ht_priv	*phtpriv_ap = &pmlmepriv->htpriv;
 	struct ht_priv	*phtpriv_sta = &psta->htpriv;
 
-	psta->mac_id = psta->aid+1;
+	psta->mac_id = psta->aid + 1;
 	DBG_88E("%s\n", __func__);
 
 	/* ap mode */
@@ -658,11 +658,15 @@ void update_sta_info_apmode(struct adapter *padapter, struct sta_info *psta)
 		phtpriv_sta->ampdu_enable = phtpriv_ap->ampdu_enable;
 
 		/* check if sta support s Short GI */
-		if ((phtpriv_sta->ht_cap.cap_info & phtpriv_ap->ht_cap.cap_info) & (IEEE80211_HT_CAP_SGI_20 | IEEE80211_HT_CAP_SGI_40))
+		if (le16_to_cpu(phtpriv_sta->ht_cap.cap_info &
+				phtpriv_ap->ht_cap.cap_info) &
+		    (IEEE80211_HT_CAP_SGI_20 | IEEE80211_HT_CAP_SGI_40))
 			phtpriv_sta->sgi = true;
 
 		/*  bwmode */
-		if ((phtpriv_sta->ht_cap.cap_info & phtpriv_ap->ht_cap.cap_info) & IEEE80211_HT_CAP_SUP_WIDTH) {
+		if (le16_to_cpu(phtpriv_sta->ht_cap.cap_info &
+				phtpriv_ap->ht_cap.cap_info) &
+		    IEEE80211_HT_CAP_SUP_WIDTH) {
 			phtpriv_sta->bwmode = pmlmeext->cur_bwmode;
 			phtpriv_sta->ch_offset = pmlmeext->cur_ch_offset;
 		}
@@ -702,12 +706,12 @@ static void update_hw_ht_param(struct adapter *padapter)
 
 	/* handle A-MPDU parameter field */
 	/*
-		AMPDU_para [1:0]:Max AMPDU Len => 0:8k , 1:16k, 2:32k, 3:64k
-		AMPDU_para [4:2]:Min MPDU Start Spacing
+		ampdu_params_info [1:0]:Max AMPDU Len => 0:8k , 1:16k, 2:32k, 3:64k
+		ampdu_params_info [4:2]:Min MPDU Start Spacing
 	*/
-	max_AMPDU_len = pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x03;
+	max_AMPDU_len = pmlmeinfo->HT_caps.ampdu_params_info & 0x03;
 
-	min_MPDU_spacing = (pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x1c) >> 2;
+	min_MPDU_spacing = (pmlmeinfo->HT_caps.ampdu_params_info & 0x1c) >> 2;
 
 	rtw_hal_set_hwreg(padapter, HW_VAR_AMPDU_MIN_SPACE, (u8 *)(&min_MPDU_spacing));
 
@@ -716,7 +720,7 @@ static void update_hw_ht_param(struct adapter *padapter)
 	/*  */
 	/*  Config SM Power Save setting */
 	/*  */
-	pmlmeinfo->SM_PS = (le16_to_cpu(pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info) & 0x0C) >> 2;
+	pmlmeinfo->SM_PS = (le16_to_cpu(pmlmeinfo->HT_caps.cap_info) & 0x0C) >> 2;
 	if (pmlmeinfo->SM_PS == WLAN_HT_CAP_SM_PS_STATIC)
 		DBG_88E("%s(): WLAN_HT_CAP_SM_PS_STATIC\n", __func__);
 }
@@ -746,7 +750,7 @@ static void start_bss_network(struct adapter *padapter, u8 *pbuf)
 	/* check if there is wps ie, */
 	/* if there is wpsie in beacon, the hostapd will update beacon twice when stating hostapd, */
 	/* and at first time the security ie (RSN/WPA IE) will not include in beacon. */
-	if (!rtw_get_wps_ie(pnetwork->IEs+_FIXED_IE_LENGTH_, pnetwork->IELength-_FIXED_IE_LENGTH_, NULL, NULL))
+	if (!rtw_get_wps_ie(pnetwork->IEs + _FIXED_IE_LENGTH_, pnetwork->IELength - _FIXED_IE_LENGTH_, NULL, NULL))
 		pmlmeext->bstart_bss = true;
 
 	/* todo: update wmm, ht cap */
@@ -799,7 +803,7 @@ static void start_bss_network(struct adapter *padapter, u8 *pbuf)
 	/* set channel, bwmode */
 	p = rtw_get_ie((pnetwork->IEs + sizeof(struct ndis_802_11_fixed_ie)), _HT_ADD_INFO_IE_, &ie_len, (pnetwork->IELength - sizeof(struct ndis_802_11_fixed_ie)));
 	if (p && ie_len) {
-		pht_info = (struct HT_info_element *)(p+2);
+		pht_info = (struct HT_info_element *)(p + 2);
 
 		if ((pregpriv->cbw40_enable) &&	 (pht_info->infos[0] & BIT(2))) {
 			/* switch to the 40M Hz mode */
@@ -930,15 +934,15 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	memset(supportRate, 0, NDIS_802_11_LENGTH_RATES_EX);
 	/*  get supported rates */
 	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _SUPPORTEDRATES_IE_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
-	if (p !=  NULL) {
-		memcpy(supportRate, p+2, ie_len);
+	if (p) {
+		memcpy(supportRate, p + 2, ie_len);
 		supportRateNum = ie_len;
 	}
 
 	/* get ext_supported rates */
 	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _EXT_SUPPORTEDRATES_IE_, &ie_len, pbss_network->IELength - _BEACON_IE_OFFSET_);
-	if (p !=  NULL) {
-		memcpy(supportRate+supportRateNum, p+2, ie_len);
+	if (p) {
+		memcpy(supportRate + supportRateNum, p + 2, ie_len);
 		supportRateNum += ie_len;
 	}
 
@@ -966,7 +970,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	psecuritypriv->wpa2_pairwise_cipher = _NO_PRIVACY_;
 	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _RSN_IE_2_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
 	if (p && ie_len > 0) {
-		if (rtw_parse_wpa2_ie(p, ie_len+2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
+		if (rtw_parse_wpa2_ie(p, ie_len + 2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
 			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
 
 			psecuritypriv->dot8021xalg = 1;/* psk,  todo:802.1x */
@@ -985,8 +989,8 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	for (p = ie + _BEACON_IE_OFFSET_;; p += (ie_len + 2)) {
 		p = rtw_get_ie(p, _SSN_IE_1_, &ie_len,
 			       (pbss_network->IELength - _BEACON_IE_OFFSET_ - (ie_len + 2)));
-		if ((p) && (!memcmp(p+2, OUI1, 4))) {
-			if (rtw_parse_wpa_ie(p, ie_len+2, &group_cipher,
+		if ((p) && (!memcmp(p + 2, OUI1, 4))) {
+			if (rtw_parse_wpa_ie(p, ie_len + 2, &group_cipher,
 					     &pairwise_cipher, NULL) == _SUCCESS) {
 				psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
 
@@ -1010,10 +1014,10 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 		for (p = ie + _BEACON_IE_OFFSET_;; p += (ie_len + 2)) {
 			p = rtw_get_ie(p, _VENDOR_SPECIFIC_IE_, &ie_len,
 				       (pbss_network->IELength - _BEACON_IE_OFFSET_ - (ie_len + 2)));
-			if ((p) && !memcmp(p+2, WMM_PARA_IE, 6)) {
+			if ((p) && !memcmp(p + 2, WMM_PARA_IE, 6)) {
 				pmlmepriv->qospriv.qos_option = 1;
 
-				*(p+8) |= BIT(7);/* QoS Info, support U-APSD */
+				*(p + 8) |= BIT(7);/* QoS Info, support U-APSD */
 
 				/* disable all ACM bits since the WMM admission control is not supported */
 				*(p + 10) &= ~BIT(4); /* BE */
@@ -1032,7 +1036,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 		       (pbss_network->IELength - _BEACON_IE_OFFSET_));
 	if (p && ie_len > 0) {
 		u8 rf_type;
-		struct rtw_ieee80211_ht_cap *pht_cap = (struct rtw_ieee80211_ht_cap *)(p+2);
+		struct ieee80211_ht_cap *pht_cap = (struct ieee80211_ht_cap *)(p + 2);
 
 		pHT_caps_ie = p;
 		ht_cap = true;
@@ -1042,7 +1046,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 
 		if ((psecuritypriv->wpa_pairwise_cipher & WPA_CIPHER_CCMP) ||
 		    (psecuritypriv->wpa2_pairwise_cipher & WPA_CIPHER_CCMP))
-			pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY&(0x07<<2));
+			pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & (0x07 << 2));
 		else
 			pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY&0x00);
 
@@ -1050,8 +1054,8 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 		pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_FACTOR & 0x03);
 
 		if (rf_type == RF_1T1R) {
-			pht_cap->supp_mcs_set[0] = 0xff;
-			pht_cap->supp_mcs_set[1] = 0x0;
+			pht_cap->mcs.rx_mask[0] = 0xff;
+			pht_cap->mcs.rx_mask[1] = 0x0;
 		}
 		memcpy(&pmlmepriv->htpriv.ht_cap, p+2, ie_len);
 	}
@@ -1143,7 +1147,7 @@ int rtw_acl_add_sta(struct adapter *padapter, u8 *addr)
 
 	DBG_88E("%s(acl_num =%d) =%pM\n", __func__, pacl_list->num, (addr));
 
-	if ((NUM_ACL-1) < pacl_list->num)
+	if ((NUM_ACL - 1) < pacl_list->num)
 		return -1;
 
 	spin_lock_bh(&(pacl_node_q->lock));
@@ -1422,7 +1426,8 @@ static int rtw_ht_operation_update(struct adapter *padapter)
 	if (pmlmepriv->num_sta_no_ht ||
 	    (pmlmepriv->ht_op_mode & HT_INFO_OPERATION_MODE_NON_GF_DEVS_PRESENT))
 		new_op_mode = OP_MODE_MIXED;
-	else if ((phtpriv_ap->ht_cap.cap_info & IEEE80211_HT_CAP_SUP_WIDTH) &&
+	else if ((le16_to_cpu(phtpriv_ap->ht_cap.cap_info) &
+		  IEEE80211_HT_CAP_SUP_WIDTH) &&
 		 pmlmepriv->num_sta_ht_20mhz)
 		new_op_mode = OP_MODE_20MHZ_HT_STA_ASSOCED;
 	else if (pmlmepriv->olbc_ht)
@@ -1552,7 +1557,7 @@ void bss_cap_update_on_sta_join(struct adapter *padapter, struct sta_info *psta)
 	}
 
 	if (psta->flags & WLAN_STA_HT) {
-		u16 ht_capab = psta->htpriv.ht_cap.cap_info;
+		u16 ht_capab = le16_to_cpu(psta->htpriv.ht_cap.cap_info);
 
 		DBG_88E("HT: STA %pM HT Capabilities Info: 0x%04x\n",
 			(psta->hwaddr), ht_capab);
@@ -1710,40 +1715,6 @@ u8 ap_free_sta(struct adapter *padapter, struct sta_info *psta,
 	return beacon_updated;
 }
 
-int rtw_ap_inform_ch_switch(struct adapter *padapter, u8 new_ch, u8 ch_offset)
-{
-	struct list_head *phead, *plist;
-	struct sta_info *psta = NULL;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
-	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-
-	if ((pmlmeinfo->state&0x03) != WIFI_FW_AP_STATE)
-		return 0;
-
-	DBG_88E(FUNC_NDEV_FMT" with ch:%u, offset:%u\n",
-		FUNC_NDEV_ARG(padapter->pnetdev), new_ch, ch_offset);
-
-	spin_lock_bh(&pstapriv->asoc_list_lock);
-	phead = &pstapriv->asoc_list;
-	plist = phead->next;
-
-	/* for each sta in asoc_queue */
-	while (phead != plist) {
-		psta = container_of(plist, struct sta_info, asoc_list);
-		plist = plist->next;
-
-		issue_action_spct_ch_switch(padapter, psta->hwaddr, new_ch, ch_offset);
-		psta->expire_to = min_t(unsigned int, pstapriv->expire_to * 2, 5);
-	}
-	spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-	issue_action_spct_ch_switch(padapter, bc_addr, new_ch, ch_offset);
-
-	return 0;
-}
-
 int rtw_sta_flush(struct adapter *padapter)
 {
 	struct list_head *phead, *plist;
@@ -1856,9 +1827,6 @@ void start_ap_mode(struct adapter *padapter)
 	pmlmepriv->wps_probe_resp_ie = NULL;
 	pmlmepriv->wps_assoc_resp_ie = NULL;
 
-	pmlmepriv->p2p_beacon_ie = NULL;
-	pmlmepriv->p2p_probe_resp_ie = NULL;
-
 	/* for ACL */
 	INIT_LIST_HEAD(&(pacl_list->acl_node_q.queue));
 	pacl_list->num = 0;
diff --git a/drivers/staging/rtl8188eu/core/rtw_cmd.c b/drivers/staging/rtl8188eu/core/rtw_cmd.c
index 77485235c615..f1f4788dbd86 100644
--- a/drivers/staging/rtl8188eu/core/rtw_cmd.c
+++ b/drivers/staging/rtl8188eu/core/rtw_cmd.c
@@ -27,8 +27,8 @@ No irqsave is necessary.
 
 int rtw_init_cmd_priv(struct cmd_priv *pcmdpriv)
 {
-	sema_init(&(pcmdpriv->cmd_queue_sema), 0);
-	sema_init(&(pcmdpriv->terminate_cmdthread_sema), 0);
+	init_completion(&pcmdpriv->cmd_queue_comp);
+	init_completion(&pcmdpriv->terminate_cmdthread_comp);
 
 	_rtw_init_queue(&(pcmdpriv->cmd_queue));
 	return _SUCCESS;
@@ -85,7 +85,7 @@ static int rtw_cmd_filter(struct cmd_priv *pcmdpriv, struct cmd_obj *cmd_obj)
 	/* To decide allow or not */
 	if ((pcmdpriv->padapter->pwrctrlpriv.bHWPwrPindetect) &&
 	    (!pcmdpriv->padapter->registrypriv.usbss_enable)) {
-		if (cmd_obj->cmdcode == GEN_CMD_CODE(_Set_Drv_Extra)) {
+		if (cmd_obj->cmdcode == _Set_Drv_Extra_CMD_) {
 			struct drvextra_cmd_parm	*pdrvextra_cmd_parm = (struct drvextra_cmd_parm	*)cmd_obj->parmbuf;
 
 			if (pdrvextra_cmd_parm->ec_id == POWER_SAVING_CTRL_WK_CID)
@@ -93,7 +93,7 @@ static int rtw_cmd_filter(struct cmd_priv *pcmdpriv, struct cmd_obj *cmd_obj)
 		}
 	}
 
-	if (cmd_obj->cmdcode == GEN_CMD_CODE(_SetChannelPlan))
+	if (cmd_obj->cmdcode == _SetChannelPlan_CMD_)
 		bAllow = true;
 
 	if ((!pcmdpriv->padapter->hw_init_completed && !bAllow) ||
@@ -122,7 +122,7 @@ u32 rtw_enqueue_cmd(struct cmd_priv *pcmdpriv, struct cmd_obj *cmd_obj)
 	res = _rtw_enqueue_cmd(&pcmdpriv->cmd_queue, cmd_obj);
 
 	if (res == _SUCCESS)
-		up(&pcmdpriv->cmd_queue_sema);
+		complete(&pcmdpriv->cmd_queue_comp);
 
 exit:
 
@@ -162,12 +162,12 @@ int rtw_cmd_thread(void *context)
 	allow_signal(SIGTERM);
 
 	pcmdpriv->cmdthd_running = true;
-	up(&pcmdpriv->terminate_cmdthread_sema);
+	complete(&pcmdpriv->terminate_cmdthread_comp);
 
 	RT_TRACE(_module_rtl871x_cmd_c_, _drv_info_, ("start r871x rtw_cmd_thread !!!!\n"));
 
 	while (1) {
-		if (_rtw_down_sema(&pcmdpriv->cmd_queue_sema) == _FAIL)
+		if (wait_for_completion_interruptible(&pcmdpriv->cmd_queue_comp))
 			break;
 
 		if (padapter->bDriverStopped ||
@@ -234,7 +234,7 @@ _next:
 		rtw_free_cmd_obj(pcmd);
 	}
 
-	up(&pcmdpriv->terminate_cmdthread_sema);
+	complete(&pcmdpriv->terminate_cmdthread_comp);
 
 
 	complete_and_exit(NULL, 0);
@@ -271,7 +271,7 @@ u8 rtw_sitesurvey_cmd(struct adapter  *padapter, struct ndis_802_11_ssid *ssid,
 
 	RT_TRACE(_module_rtl871x_cmd_c_, _drv_info_, ("%s: flush network queue\n", __func__));
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara, GEN_CMD_CODE(_SiteSurvey));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara, _SiteSurvey_CMD_);
 
 	/* psurveyPara->bsslimit = 48; */
 	psurveyPara->scan_mode = pmlmepriv->scan_mode;
@@ -305,8 +305,6 @@ u8 rtw_sitesurvey_cmd(struct adapter  *padapter, struct ndis_802_11_ssid *ssid,
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 
 	if (res == _SUCCESS) {
-		pmlmepriv->scan_start_time = jiffies;
-
 		mod_timer(&pmlmepriv->scan_to_timer,
 			  jiffies + msecs_to_jiffies(SCANNING_TIMEOUT));
 
@@ -491,7 +489,7 @@ u8 rtw_joinbss_cmd(struct adapter  *padapter, struct wlan_network *pnetwork)
 	pcmd->cmdsz = get_wlan_bssid_ex_sz(psecnetwork);/* get cmdsz before endian conversion */
 
 	INIT_LIST_HEAD(&pcmd->list);
-	pcmd->cmdcode = _JoinBss_CMD_;/* GEN_CMD_CODE(_JoinBss) */
+	pcmd->cmdcode = _JoinBss_CMD_;
 	pcmd->parmbuf = (unsigned char *)psecnetwork;
 	pcmd->rsp = NULL;
 	pcmd->rspsz = 0;
@@ -670,13 +668,13 @@ u8 rtw_addbareq_cmd(struct adapter *padapter, u8 tid, u8 *addr)
 	u8	res = _SUCCESS;
 
 
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_KERNEL);
+	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
 	if (!ph2c) {
 		res = _FAIL;
 		goto exit;
 	}
 
-	paddbareq_parm = kzalloc(sizeof(struct addBaReq_parm), GFP_KERNEL);
+	paddbareq_parm = kzalloc(sizeof(struct addBaReq_parm), GFP_ATOMIC);
 	if (!paddbareq_parm) {
 		kfree(ph2c);
 		res = _FAIL;
@@ -686,7 +684,7 @@ u8 rtw_addbareq_cmd(struct adapter *padapter, u8 tid, u8 *addr)
 	paddbareq_parm->tid = tid;
 	memcpy(paddbareq_parm->addr, addr, ETH_ALEN);
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, paddbareq_parm, GEN_CMD_CODE(_AddBAReq));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, paddbareq_parm, _AddBAReq_CMD_);
 
 	/* DBG_88E("rtw_addbareq_cmd, tid =%d\n", tid); */
 
@@ -724,7 +722,7 @@ u8 rtw_dynamic_chk_wk_cmd(struct adapter *padapter)
 	pdrvextra_cmd_parm->type_size = 0;
 	pdrvextra_cmd_parm->pbuf = (u8 *)padapter;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, _Set_Drv_Extra_CMD_);
 
 
 	/* rtw_enqueue_cmd(pcmdpriv, ph2c); */
@@ -767,7 +765,7 @@ u8 rtw_set_chplan_cmd(struct adapter *padapter, u8 chplan, u8 enqueue)
 			goto exit;
 		}
 
-		init_h2fwcmd_w_parm_no_rsp(pcmdobj, setChannelPlan_param, GEN_CMD_CODE(_SetChannelPlan));
+		init_h2fwcmd_w_parm_no_rsp(pcmdobj, setChannelPlan_param, _SetChannelPlan_CMD_);
 		res = rtw_enqueue_cmd(pcmdpriv, pcmdobj);
 	} else {
 		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
@@ -936,7 +934,7 @@ u8 rtw_lps_ctrl_wk_cmd(struct adapter *padapter, u8 lps_ctrl_type, u8 enqueue)
 		pdrvextra_cmd_parm->type_size = lps_ctrl_type;
 		pdrvextra_cmd_parm->pbuf = NULL;
 
-		init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+		init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, _Set_Drv_Extra_CMD_);
 
 		res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 	} else {
@@ -978,7 +976,7 @@ u8 rtw_rpt_timer_cfg_cmd(struct adapter *padapter, u16 min_time)
 	pdrvextra_cmd_parm->ec_id = RTP_TIMER_CFG_WK_CID;
 	pdrvextra_cmd_parm->type_size = min_time;
 	pdrvextra_cmd_parm->pbuf = NULL;
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, _Set_Drv_Extra_CMD_);
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 exit:
 
@@ -1020,7 +1018,7 @@ u8 rtw_antenna_select_cmd(struct adapter *padapter, u8 antenna, u8 enqueue)
 		pdrvextra_cmd_parm->ec_id = ANT_SELECT_WK_CID;
 		pdrvextra_cmd_parm->type_size = antenna;
 		pdrvextra_cmd_parm->pbuf = NULL;
-		init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+		init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, _Set_Drv_Extra_CMD_);
 
 		res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 	} else {
@@ -1048,7 +1046,7 @@ u8 rtw_ps_cmd(struct adapter *padapter)
 
 	pdrvextra_cmd_parm->ec_id = POWER_SAVING_CTRL_WK_CID;
 	pdrvextra_cmd_parm->pbuf = NULL;
-	init_h2fwcmd_w_parm_no_rsp(ppscmd, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ppscmd, pdrvextra_cmd_parm, _Set_Drv_Extra_CMD_);
 
 	return rtw_enqueue_cmd(pcmdpriv, ppscmd);
 }
@@ -1119,7 +1117,7 @@ u8 rtw_chk_hi_queue_cmd(struct adapter *padapter)
 	pdrvextra_cmd_parm->type_size = 0;
 	pdrvextra_cmd_parm->pbuf = NULL;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, _Set_Drv_Extra_CMD_);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 exit:
diff --git a/drivers/staging/rtl8188eu/core/rtw_debug.c b/drivers/staging/rtl8188eu/core/rtw_debug.c
index db5c952ac852..60d8c7b9f458 100644
--- a/drivers/staging/rtl8188eu/core/rtw_debug.c
+++ b/drivers/staging/rtl8188eu/core/rtw_debug.c
@@ -138,144 +138,6 @@ int proc_set_read_reg(struct file *file, const char __user *buffer,
 	return count;
 }
 
-int proc_get_fwstate(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	int len = 0;
-
-	len += snprintf(page + len, count - len, "fwstate=0x%x\n", get_fwstate(pmlmepriv));
-
-	*eof = 1;
-	return len;
-}
-
-int proc_get_sec_info(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-
-	int len = 0;
-
-	len += snprintf(page + len, count - len, "auth_alg=0x%x, enc_alg=0x%x, auth_type=0x%x, enc_type=0x%x\n",
-						psecuritypriv->dot11AuthAlgrthm, psecuritypriv->dot11PrivacyAlgrthm,
-						psecuritypriv->ndisauthtype, psecuritypriv->ndisencryptstatus);
-
-	*eof = 1;
-	return len;
-}
-
-int proc_get_mlmext_state(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info	*pmlmeinfo = &pmlmeext->mlmext_info;
-
-	int len = 0;
-
-	len += snprintf(page + len, count - len, "pmlmeinfo->state=0x%x\n", pmlmeinfo->state);
-
-	*eof = 1;
-	return len;
-}
-
-int proc_get_qos_option(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	int len = 0;
-
-	len += snprintf(page + len, count - len, "qos_option=%d\n", pmlmepriv->qospriv.qos_option);
-
-	*eof = 1;
-	return len;
-}
-
-int proc_get_ht_option(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	int len = 0;
-
-	len += snprintf(page + len, count - len, "ht_option=%d\n", pmlmepriv->htpriv.ht_option);
-	*eof = 1;
-	return len;
-}
-
-int proc_get_rf_info(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
-	int len = 0;
-
-	len += snprintf(page + len, count - len, "cur_ch=%d, cur_bw=%d, cur_ch_offset=%d\n",
-					pmlmeext->cur_channel, pmlmeext->cur_bwmode, pmlmeext->cur_ch_offset);
-	*eof = 1;
-	return len;
-}
-
-int proc_get_ap_info(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct sta_info *psta;
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	int len = 0;
-
-	psta = rtw_get_stainfo(pstapriv, cur_network->network.MacAddress);
-	if (psta) {
-		int i;
-		struct recv_reorder_ctrl *preorder_ctrl;
-
-		len += snprintf(page + len, count - len, "SSID=%s\n", cur_network->network.Ssid.Ssid);
-		len += snprintf(page + len, count - len, "sta's macaddr:%pM\n", psta->hwaddr);
-		len += snprintf(page + len, count - len, "cur_channel=%d, cur_bwmode=%d, cur_ch_offset=%d\n", pmlmeext->cur_channel, pmlmeext->cur_bwmode, pmlmeext->cur_ch_offset);
-		len += snprintf(page + len, count - len, "rtsen=%d, cts2slef=%d\n", psta->rtsen, psta->cts2self);
-		len += snprintf(page + len, count - len, "state=0x%x, aid=%d, macid=%d, raid=%d\n", psta->state, psta->aid, psta->mac_id, psta->raid);
-		len += snprintf(page + len, count - len, "qos_en=%d, ht_en=%d, init_rate=%d\n", psta->qos_option, psta->htpriv.ht_option, psta->init_rate);
-		len += snprintf(page + len, count - len, "bwmode=%d, ch_offset=%d, sgi=%d\n", psta->htpriv.bwmode, psta->htpriv.ch_offset, psta->htpriv.sgi);
-		len += snprintf(page + len, count - len, "ampdu_enable = %d\n", psta->htpriv.ampdu_enable);
-		len += snprintf(page + len, count - len, "agg_enable_bitmap=%x, candidate_tid_bitmap=%x\n", psta->htpriv.agg_enable_bitmap, psta->htpriv.candidate_tid_bitmap);
-
-		for (i = 0; i < 16; i++) {
-			preorder_ctrl = &psta->recvreorder_ctrl[i];
-			if (preorder_ctrl->enable)
-				len += snprintf(page + len, count - len, "tid=%d, indicate_seq=%d\n", i, preorder_ctrl->indicate_seq);
-		}
-	} else {
-		len += snprintf(page + len, count - len, "can't get sta's macaddr, cur_network's macaddr: %pM\n", cur_network->network.MacAddress);
-	}
-
-	*eof = 1;
-	return len;
-}
-
 int proc_get_adapter_state(char *page, char **start,
 			  off_t offset, int count,
 			  int *eof, void *data)
@@ -291,599 +153,6 @@ int proc_get_adapter_state(char *page, char **start,
 	return len;
 }
 
-int proc_get_trx_info(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct recv_priv  *precvpriv = &padapter->recvpriv;
-	int len = 0;
-
-	len += snprintf(page + len, count - len, "free_xmitbuf_cnt=%d, free_xmitframe_cnt=%d, free_ext_xmitbuf_cnt=%d, free_recvframe_cnt=%d\n",
-				pxmitpriv->free_xmitbuf_cnt, pxmitpriv->free_xmitframe_cnt, pxmitpriv->free_xmit_extbuf_cnt, precvpriv->free_recvframe_cnt);
-	len += snprintf(page + len, count - len, "rx_urb_pending_cn=%d\n", precvpriv->rx_pending_cnt);
-
-	*eof = 1;
-	return len;
-}
-
-int proc_get_mac_reg_dump1(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1;
-
-	len += snprintf(page + len, count - len, "\n======= MAC REG =======\n");
-
-	for (i = 0x0; i < 0x300; i += 4) {
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", usb_read32(padapter, i));
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-
-	*eof = 1;
-	return len;
-}
-
-int proc_get_mac_reg_dump2(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1;
-
-	len += snprintf(page + len, count - len, "\n======= MAC REG =======\n");
-	memset(page, 0, count);
-	for (i = 0x300; i < 0x600; i += 4) {
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", usb_read32(padapter, i));
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-
-	*eof = 1;
-	return len;
-}
-
-int proc_get_mac_reg_dump3(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1;
-
-	len += snprintf(page + len, count - len, "\n======= MAC REG =======\n");
-
-	for (i = 0x600; i < 0x800; i += 4) {
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", usb_read32(padapter, i));
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-
-	*eof = 1;
-	return len;
-}
-
-int proc_get_bb_reg_dump1(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1;
-
-	len += snprintf(page + len, count - len, "\n======= BB REG =======\n");
-	for (i = 0x800; i < 0xB00; i += 4) {
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", usb_read32(padapter, i));
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-	*eof = 1;
-	return len;
-}
-
-int proc_get_bb_reg_dump2(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1;
-
-	len += snprintf(page + len, count - len, "\n======= BB REG =======\n");
-	for (i = 0xB00; i < 0xE00; i += 4) {
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", usb_read32(padapter, i));
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-	*eof = 1;
-	return len;
-}
-
-int proc_get_bb_reg_dump3(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1;
-
-	len += snprintf(page + len, count - len, "\n======= BB REG =======\n");
-	for (i = 0xE00; i < 0x1000; i += 4) {
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", usb_read32(padapter, i));
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-	*eof = 1;
-	return len;
-}
-
-int proc_get_rf_reg_dump1(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1, path;
-	u32 value;
-
-	len += snprintf(page + len, count - len, "\n======= RF REG =======\n");
-	path = 1;
-	len += snprintf(page + len, count - len, "\nRF_Path(%x)\n", path);
-	for (i = 0; i < 0xC0; i++) {
-		value = rtw_hal_read_rfreg(padapter, path, i, 0xffffffff);
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x ", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", value);
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-	*eof = 1;
-	return len;
-}
-
-int proc_get_rf_reg_dump2(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1, path;
-	u32 value;
-
-	len += snprintf(page + len, count - len, "\n======= RF REG =======\n");
-	path = 1;
-	len += snprintf(page + len, count - len, "\nRF_Path(%x)\n", path);
-	for (i = 0xC0; i < 0x100; i++) {
-		value = rtw_hal_read_rfreg(padapter, path, i, 0xffffffff);
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x ", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", value);
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-	*eof = 1;
-	return len;
-}
-
-int proc_get_rf_reg_dump3(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1, path;
-	u32 value;
-
-	len += snprintf(page + len, count - len, "\n======= RF REG =======\n");
-	path = 2;
-	len += snprintf(page + len, count - len, "\nRF_Path(%x)\n", path);
-	for (i = 0; i < 0xC0; i++) {
-		value = rtw_hal_read_rfreg(padapter, path, i, 0xffffffff);
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x ", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", value);
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-
-	*eof = 1;
-	return len;
-}
-
-
-int proc_get_rf_reg_dump4(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-	int i, j = 1, path;
-	u32 value;
-
-	len += snprintf(page + len, count - len, "\n======= RF REG =======\n");
-	path = 2;
-	len += snprintf(page + len, count - len, "\nRF_Path(%x)\n", path);
-	for (i = 0xC0; i < 0x100; i++) {
-		value = rtw_hal_read_rfreg(padapter, path, i, 0xffffffff);
-		if (j%4 == 1)
-			len += snprintf(page + len, count - len, "0x%02x ", i);
-		len += snprintf(page + len, count - len, " 0x%08x ", value);
-		if ((j++)%4 == 0)
-			len += snprintf(page + len, count - len, "\n");
-	}
-	*eof = 1;
-	return len;
-}
-
-
-
-int proc_get_rx_signal(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	int len = 0;
-
-	len = snprintf(page + len, count,
-		"rssi:%d\n"
-		"rxpwdb:%d\n"
-		"signal_strength:%u\n"
-		"signal_qual:%u\n"
-		"noise:%u\n",
-		padapter->recvpriv.rssi,
-		padapter->recvpriv.rxpwdb,
-		padapter->recvpriv.signal_strength,
-		padapter->recvpriv.signal_qual,
-		padapter->recvpriv.noise
-		);
-
-	*eof = 1;
-	return len;
-}
-
-int proc_set_rx_signal(struct file *file, const char __user *buffer,
-		unsigned long count, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	char tmp[32];
-	u32 is_signal_dbg;
-	s32 signal_strength;
-
-	if (count < 1)
-		return -EFAULT;
-
-	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
-		int num = sscanf(tmp, "%u %u", &is_signal_dbg, &signal_strength);
-
-		is_signal_dbg = is_signal_dbg == 0 ? 0 : 1;
-		if (is_signal_dbg && num != 2)
-			return count;
-
-		signal_strength = clamp(signal_strength, 0, 100);
-
-		padapter->recvpriv.is_signal_dbg = is_signal_dbg;
-		padapter->recvpriv.signal_strength_dbg = signal_strength;
-
-		if (is_signal_dbg)
-			DBG_88E("set %s %u\n", "DBG_SIGNAL_STRENGTH", signal_strength);
-		else
-			DBG_88E("set %s\n", "HW_SIGNAL_STRENGTH");
-	}
-	return count;
-}
-
-int proc_get_ht_enable(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct registry_priv	*pregpriv = &padapter->registrypriv;
-	int len = 0;
-
-	if (pregpriv)
-		len += snprintf(page + len, count - len,
-			"%d\n",
-			pregpriv->ht_enable
-			);
-	*eof = 1;
-	return len;
-}
-
-int proc_set_ht_enable(struct file *file, const char __user *buffer,
-		unsigned long count, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct registry_priv	*pregpriv = &padapter->registrypriv;
-	char tmp[32];
-	s32 mode = 0;
-
-	if (count < 1)
-		return -EFAULT;
-
-	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
-		if (pregpriv) {
-			pregpriv->ht_enable = mode;
-			pr_info("ht_enable=%d\n", pregpriv->ht_enable);
-		}
-	}
-
-	return count;
-}
-
-int proc_get_cbw40_enable(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct registry_priv	*pregpriv = &padapter->registrypriv;
-
-	int len = 0;
-
-	if (pregpriv)
-		len += snprintf(page + len, count - len,
-			"%d\n",
-			pregpriv->cbw40_enable
-			);
-
-	*eof = 1;
-	return len;
-}
-
-int proc_set_cbw40_enable(struct file *file, const char __user *buffer,
-		unsigned long count, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct registry_priv	*pregpriv = &padapter->registrypriv;
-	char tmp[32];
-	s32 mode = 0;
-
-	if (count < 1)
-		return -EFAULT;
-
-	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
-		if (pregpriv) {
-			pregpriv->cbw40_enable = mode;
-			pr_info("cbw40_enable=%d\n", mode);
-		}
-	}
-	return count;
-}
-
-int proc_get_ampdu_enable(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct registry_priv	*pregpriv = &padapter->registrypriv;
-
-	int len = 0;
-
-	if (pregpriv)
-		len += snprintf(page + len, count - len,
-			"%d\n",
-			pregpriv->ampdu_enable
-			);
-
-	*eof = 1;
-	return len;
-}
-
-int proc_set_ampdu_enable(struct file *file, const char __user *buffer,
-		unsigned long count, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct registry_priv	*pregpriv = &padapter->registrypriv;
-	char tmp[32];
-	s32 mode = 0;
-
-	if (count < 1)
-		return -EFAULT;
-
-	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
-		if (pregpriv) {
-			pregpriv->ampdu_enable = mode;
-			pr_info("ampdu_enable=%d\n", mode);
-		}
-	}
-	return count;
-}
-
-int proc_get_two_path_rssi(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-
-	int len = 0;
-
-	if (padapter)
-		len += snprintf(page + len, count - len,
-			"%d %d\n",
-			padapter->recvpriv.RxRssi[0],
-			padapter->recvpriv.RxRssi[1]
-			);
-
-	*eof = 1;
-	return len;
-}
-
-int proc_get_rx_stbc(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct registry_priv	*pregpriv = &padapter->registrypriv;
-
-	int len = 0;
-
-	if (pregpriv)
-		len += snprintf(page + len, count - len,
-			"%d\n",
-			pregpriv->rx_stbc
-			);
-
-	*eof = 1;
-	return len;
-}
-
-int proc_set_rx_stbc(struct file *file, const char __user *buffer,
-		unsigned long count, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct registry_priv	*pregpriv = &padapter->registrypriv;
-	char tmp[32];
-	u32 mode = 0;
-
-	if (count < 1)
-		return -EFAULT;
-
-	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
-		if (pregpriv) {
-			pregpriv->rx_stbc = mode;
-			netdev_info(dev, "rx_stbc=%d\n", mode);
-		}
-	}
-	return count;
-}
-
-int proc_get_rssi_disp(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	*eof = 1;
-	return 0;
-}
-
-int proc_set_rssi_disp(struct file *file, const char __user *buffer,
-		unsigned long count, void *data)
-{
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	char tmp[32];
-	u32 enable = 0;
-
-	if (count < 1) {
-		DBG_88E("argument size is less than 1\n");
-		return -EFAULT;
-	}
-
-	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
-		int num = sscanf(tmp, "%x", &enable);
-
-		if (num !=  1) {
-			DBG_88E("invalid set_rssi_disp parameter!\n");
-			return count;
-		}
-
-		if (enable) {
-			DBG_88E("Turn On Rx RSSI Display Function\n");
-			padapter->bRxRSSIDisplay = enable;
-		} else {
-			DBG_88E("Turn Off Rx RSSI Display Function\n");
-			padapter->bRxRSSIDisplay = 0;
-		}
-	}
-	return count;
-}
-
-#ifdef CONFIG_88EU_AP_MODE
-
-int proc_get_all_sta_info(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data)
-{
-	struct sta_info *psta;
-	struct net_device *dev = data;
-	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	int i, j;
-	struct list_head *plist, *phead;
-	struct recv_reorder_ctrl *preorder_ctrl;
-	int len = 0;
-
-
-	len += snprintf(page + len, count - len, "sta_dz_bitmap=0x%x, tim_bitmap=0x%x\n", pstapriv->sta_dz_bitmap, pstapriv->tim_bitmap);
-
-	spin_lock_bh(&pstapriv->sta_hash_lock);
-
-	for (i = 0; i < NUM_STA; i++) {
-		phead = &pstapriv->sta_hash[i];
-		plist = phead->next;
-
-		while (phead != plist) {
-			psta = container_of(plist, struct sta_info, hash_list);
-
-			plist = plist->next;
-
-			len += snprintf(page + len, count - len, "sta's macaddr: %pM\n", psta->hwaddr);
-			len += snprintf(page + len, count - len, "rtsen=%d, cts2slef=%d\n", psta->rtsen, psta->cts2self);
-			len += snprintf(page + len, count - len, "state=0x%x, aid=%d, macid=%d, raid=%d\n", psta->state, psta->aid, psta->mac_id, psta->raid);
-			len += snprintf(page + len, count - len, "qos_en=%d, ht_en=%d, init_rate=%d\n", psta->qos_option, psta->htpriv.ht_option, psta->init_rate);
-			len += snprintf(page + len, count - len, "bwmode=%d, ch_offset=%d, sgi=%d\n", psta->htpriv.bwmode, psta->htpriv.ch_offset, psta->htpriv.sgi);
-			len += snprintf(page + len, count - len, "ampdu_enable = %d\n", psta->htpriv.ampdu_enable);
-			len += snprintf(page + len, count - len, "agg_enable_bitmap=%x, candidate_tid_bitmap=%x\n", psta->htpriv.agg_enable_bitmap, psta->htpriv.candidate_tid_bitmap);
-			len += snprintf(page + len, count - len, "sleepq_len=%d\n", psta->sleepq_len);
-			len += snprintf(page + len, count - len, "capability=0x%x\n", psta->capability);
-			len += snprintf(page + len, count - len, "flags=0x%x\n", psta->flags);
-			len += snprintf(page + len, count - len, "wpa_psk=0x%x\n", psta->wpa_psk);
-			len += snprintf(page + len, count - len, "wpa2_group_cipher=0x%x\n", psta->wpa2_group_cipher);
-			len += snprintf(page + len, count - len, "wpa2_pairwise_cipher=0x%x\n", psta->wpa2_pairwise_cipher);
-			len += snprintf(page + len, count - len, "qos_info=0x%x\n", psta->qos_info);
-			len += snprintf(page + len, count - len, "dot118021XPrivacy=0x%x\n", psta->dot118021XPrivacy);
-
-			for (j = 0; j < 16; j++) {
-				preorder_ctrl = &psta->recvreorder_ctrl[j];
-				if (preorder_ctrl->enable)
-					len += snprintf(page + len, count - len, "tid=%d, indicate_seq=%d\n", j, preorder_ctrl->indicate_seq);
-			}
-		}
-	}
-	spin_unlock_bh(&pstapriv->sta_hash_lock);
-
-	*eof = 1;
-	return len;
-}
-#endif
-
 int proc_get_best_channel(char *page, char **start,
 			  off_t offset, int count,
 			  int *eof, void *data)
diff --git a/drivers/staging/rtl8188eu/core/rtw_efuse.c b/drivers/staging/rtl8188eu/core/rtw_efuse.c
index fbce1f7e68ca..16cc7706a1e6 100644
--- a/drivers/staging/rtl8188eu/core/rtw_efuse.c
+++ b/drivers/staging/rtl8188eu/core/rtw_efuse.c
@@ -317,69 +317,6 @@ void efuse_ReadEFuse(struct adapter *Adapter, u8 efuseType, u16 _offset, u16 _si
 	}
 }
 
-/* Do not support BT */
-void EFUSE_GetEfuseDefinition(struct adapter *pAdapter, u8 efuseType, u8 type, void *pOut)
-{
-	switch (type) {
-	case TYPE_EFUSE_MAX_SECTION:
-		{
-			u8 *pMax_section;
-			pMax_section = pOut;
-			*pMax_section = EFUSE_MAX_SECTION_88E;
-		}
-		break;
-	case TYPE_EFUSE_REAL_CONTENT_LEN:
-		{
-			u16 *pu2Tmp;
-			pu2Tmp = pOut;
-			*pu2Tmp = EFUSE_REAL_CONTENT_LEN_88E;
-		}
-		break;
-	case TYPE_EFUSE_CONTENT_LEN_BANK:
-		{
-			u16 *pu2Tmp;
-			pu2Tmp = pOut;
-			*pu2Tmp = EFUSE_REAL_CONTENT_LEN_88E;
-		}
-		break;
-	case TYPE_AVAILABLE_EFUSE_BYTES_BANK:
-		{
-			u16 *pu2Tmp;
-			pu2Tmp = pOut;
-			*pu2Tmp = (u16)(EFUSE_REAL_CONTENT_LEN_88E-EFUSE_OOB_PROTECT_BYTES_88E);
-		}
-		break;
-	case TYPE_AVAILABLE_EFUSE_BYTES_TOTAL:
-		{
-			u16 *pu2Tmp;
-			pu2Tmp = pOut;
-			*pu2Tmp = (u16)(EFUSE_REAL_CONTENT_LEN_88E-EFUSE_OOB_PROTECT_BYTES_88E);
-		}
-		break;
-	case TYPE_EFUSE_MAP_LEN:
-		{
-			u16 *pu2Tmp;
-			pu2Tmp = pOut;
-			*pu2Tmp = (u16)EFUSE_MAP_LEN_88E;
-		}
-		break;
-	case TYPE_EFUSE_PROTECT_BYTES_BANK:
-		{
-			u8 *pu1Tmp;
-			pu1Tmp = pOut;
-			*pu1Tmp = (u8)(EFUSE_OOB_PROTECT_BYTES_88E);
-		}
-		break;
-	default:
-		{
-			u8 *pu1Tmp;
-			pu1Tmp = pOut;
-			*pu1Tmp = 0;
-		}
-		break;
-	}
-}
-
 u8 Efuse_WordEnableDataWrite(struct adapter *pAdapter, u16 efuse_addr, u8 word_en, u8 *data)
 {
 	u16	tmpaddr = 0;
@@ -483,14 +420,11 @@ int Efuse_PgPacketRead(struct adapter *pAdapter, u8 offset, u8 *data)
 	u8 hoffset = 0, hworden = 0;
 	u8 tmpidx = 0;
 	u8 tmpdata[8];
-	u8 max_section = 0;
 	u8 tmp_header = 0;
 
-	EFUSE_GetEfuseDefinition(pAdapter, EFUSE_WIFI, TYPE_EFUSE_MAX_SECTION, (void *)&max_section);
-
 	if (!data)
 		return false;
-	if (offset > max_section)
+	if (offset > EFUSE_MAX_SECTION_88E)
 		return false;
 
 	memset(data, 0xff, sizeof(u8) * PGPKT_DATA_SIZE);
@@ -591,12 +525,12 @@ static bool hal_EfuseFixHeaderProcess(struct adapter *pAdapter, u8 efuseType, st
 static bool hal_EfusePgPacketWrite2ByteHeader(struct adapter *pAdapter, u8 efuseType, u16 *pAddr, struct pgpkt *pTargetPkt)
 {
 	bool bRet = false;
-	u16	efuse_addr = *pAddr, efuse_max_available_len = 0;
+	u16 efuse_addr = *pAddr;
+	u16 efuse_max_available_len =
+		EFUSE_REAL_CONTENT_LEN_88E - EFUSE_OOB_PROTECT_BYTES_88E;
 	u8 pg_header = 0, tmp_header = 0, pg_header_temp = 0;
 	u8 repeatcnt = 0;
 
-	EFUSE_GetEfuseDefinition(pAdapter, efuseType, TYPE_AVAILABLE_EFUSE_BYTES_BANK, (void *)&efuse_max_available_len);
-
 	while (efuse_addr < efuse_max_available_len) {
 		pg_header = ((pTargetPkt->offset & 0x07) << 5) | 0x0F;
 		efuse_OneByteWrite(pAdapter, efuse_addr, pg_header);
@@ -769,12 +703,11 @@ static bool hal_EfusePartialWriteCheck(struct adapter *pAdapter, u8 efuseType, u
 	bool bRet = false;
 	u8 i, efuse_data = 0, cur_header = 0;
 	u8 matched_wden = 0, badworden = 0;
-	u16	startAddr = 0, efuse_max_available_len = 0, efuse_max = 0;
+	u16 startAddr = 0;
+	u16 efuse_max_available_len =
+		EFUSE_REAL_CONTENT_LEN_88E - EFUSE_OOB_PROTECT_BYTES_88E;
 	struct pgpkt curPkt;
 
-	EFUSE_GetEfuseDefinition(pAdapter, efuseType, TYPE_AVAILABLE_EFUSE_BYTES_BANK, (void *)&efuse_max_available_len);
-	EFUSE_GetEfuseDefinition(pAdapter, efuseType, TYPE_EFUSE_REAL_CONTENT_LEN, (void *)&efuse_max);
-
 	rtw_hal_get_hwreg(pAdapter, HW_VAR_EFUSE_BYTES, (u8 *)&startAddr);
 	startAddr %= EFUSE_REAL_CONTENT_LEN;
 
@@ -846,12 +779,7 @@ hal_EfusePgCheckAvailableAddr(
 		u8 efuseType
 	)
 {
-	u16	efuse_max_available_len = 0;
-
-	/* Change to check TYPE_EFUSE_MAP_LEN , because 8188E raw 256, logic map over 256. */
-	EFUSE_GetEfuseDefinition(pAdapter, EFUSE_WIFI, TYPE_EFUSE_MAP_LEN, (void *)&efuse_max_available_len);
-
-	if (Efuse_GetCurrentSize(pAdapter) >= efuse_max_available_len)
+	if (Efuse_GetCurrentSize(pAdapter) >= EFUSE_MAP_LEN_88E)
 		return false;
 	return true;
 }
@@ -977,13 +905,9 @@ void efuse_WordEnableDataRead(u8 word_en, u8 *sourdata, u8 *targetdata)
  */
 static void Efuse_ReadAllMap(struct adapter *pAdapter, u8 efuseType, u8 *Efuse)
 {
-	u16 mapLen = 0;
-
 	Efuse_PowerSwitch(pAdapter, false, true);
 
-	EFUSE_GetEfuseDefinition(pAdapter, efuseType, TYPE_EFUSE_MAP_LEN, (void *)&mapLen);
-
-	efuse_ReadEFuse(pAdapter, efuseType, 0, mapLen, Efuse);
+	efuse_ReadEFuse(pAdapter, efuseType, 0, EFUSE_MAP_LEN_88E, Efuse);
 
 	Efuse_PowerSwitch(pAdapter, false, false);
 }
@@ -996,12 +920,9 @@ void EFUSE_ShadowMapUpdate(
 	u8 efuseType)
 {
 	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(pAdapter);
-	u16 mapLen = 0;
-
-	EFUSE_GetEfuseDefinition(pAdapter, efuseType, TYPE_EFUSE_MAP_LEN, (void *)&mapLen);
 
 	if (pEEPROM->bautoload_fail_flag)
-		memset(pEEPROM->efuse_eeprom_data, 0xFF, mapLen);
+		memset(pEEPROM->efuse_eeprom_data, 0xFF, EFUSE_MAP_LEN_88E);
 	else
 		Efuse_ReadAllMap(pAdapter, efuseType, pEEPROM->efuse_eeprom_data);
 }
diff --git a/drivers/staging/rtl8188eu/core/rtw_ieee80211.c b/drivers/staging/rtl8188eu/core/rtw_ieee80211.c
index 0b0d78fe83ed..914c4923421b 100644
--- a/drivers/staging/rtl8188eu/core/rtw_ieee80211.c
+++ b/drivers/staging/rtl8188eu/core/rtw_ieee80211.c
@@ -155,59 +155,6 @@ u8 *rtw_set_ie
 	return pbuf + len + 2;
 }
 
-inline u8 *rtw_set_ie_ch_switch(u8 *buf, u32 *buf_len, u8 ch_switch_mode,
-	u8 new_ch, u8 ch_switch_cnt)
-{
-	u8 ie_data[3];
-
-	ie_data[0] = ch_switch_mode;
-	ie_data[1] = new_ch;
-	ie_data[2] = ch_switch_cnt;
-	return rtw_set_ie(buf, WLAN_EID_CHANNEL_SWITCH,  3, ie_data, buf_len);
-}
-
-inline u8 secondary_ch_offset_to_hal_ch_offset(u8 ch_offset)
-{
-	if (ch_offset == SCN)
-		return HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-	else if (ch_offset == SCA)
-		return HAL_PRIME_CHNL_OFFSET_UPPER;
-	else if (ch_offset == SCB)
-		return HAL_PRIME_CHNL_OFFSET_LOWER;
-
-	return HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-}
-
-inline u8 hal_ch_offset_to_secondary_ch_offset(u8 ch_offset)
-{
-	if (ch_offset == HAL_PRIME_CHNL_OFFSET_DONT_CARE)
-		return SCN;
-	else if (ch_offset == HAL_PRIME_CHNL_OFFSET_LOWER)
-		return SCB;
-	else if (ch_offset == HAL_PRIME_CHNL_OFFSET_UPPER)
-		return SCA;
-
-	return SCN;
-}
-
-inline u8 *rtw_set_ie_secondary_ch_offset(u8 *buf, u32 *buf_len, u8 secondary_ch_offset)
-{
-	return rtw_set_ie(buf, WLAN_EID_SECONDARY_CHANNEL_OFFSET,  1, &secondary_ch_offset, buf_len);
-}
-
-inline u8 *rtw_set_ie_mesh_ch_switch_parm(u8 *buf, u32 *buf_len, u8 ttl,
-	u8 flags, u16 reason, u16 precedence)
-{
-	u8 ie_data[6];
-
-	ie_data[0] = ttl;
-	ie_data[1] = flags;
-	*(u16 *)(ie_data + 2) = cpu_to_le16(reason);
-	*(u16 *)(ie_data + 4) = cpu_to_le16(precedence);
-
-	return rtw_set_ie(buf, 0x118,  6, ie_data, buf_len);
-}
-
 /*----------------------------------------------------------------------------
 index: the information element id index, limit is the limit for search
 -----------------------------------------------------------------------------*/
@@ -236,97 +183,6 @@ u8 *rtw_get_ie(u8 *pbuf, int index, int *len, int limit)
 	return NULL;
 }
 
-/**
- * rtw_get_ie_ex - Search specific IE from a series of IEs
- * @in_ie: Address of IEs to search
- * @in_len: Length limit from in_ie
- * @eid: Element ID to match
- * @oui: OUI to match
- * @oui_len: OUI length
- * @ie: If not NULL and the specific IE is found, the IE will be copied to the buf starting from the specific IE
- * @ielen: If not NULL and the specific IE is found, will set to the length of the entire IE
- *
- * Returns: The address of the specific IE found, or NULL
- */
-u8 *rtw_get_ie_ex(u8 *in_ie, uint in_len, u8 eid, u8 *oui, u8 oui_len, u8 *ie, uint *ielen)
-{
-	uint cnt;
-	u8 *target_ie = NULL;
-
-
-	if (ielen)
-		*ielen = 0;
-
-	if (!in_ie || in_len <= 0)
-		return target_ie;
-
-	cnt = 0;
-
-	while (cnt < in_len) {
-		if (eid == in_ie[cnt] && (!oui || !memcmp(&in_ie[cnt + 2], oui, oui_len))) {
-			target_ie = &in_ie[cnt];
-
-			if (ie)
-				memcpy(ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
-
-			if (ielen)
-				*ielen = in_ie[cnt + 1] + 2;
-
-			break;
-		} else {
-			cnt += in_ie[cnt + 1] + 2; /* goto next */
-		}
-	}
-	return target_ie;
-}
-
-/**
- * rtw_ies_remove_ie - Find matching IEs and remove
- * @ies: Address of IEs to search
- * @ies_len: Pointer of length of ies, will update to new length
- * @offset: The offset to start scarch
- * @eid: Element ID to match
- * @oui: OUI to match
- * @oui_len: OUI length
- *
- * Returns: _SUCCESS: ies is updated, _FAIL: not updated
- */
-int rtw_ies_remove_ie(u8 *ies, uint *ies_len, uint offset, u8 eid, u8 *oui, u8 oui_len)
-{
-	int ret = _FAIL;
-	u8 *target_ie;
-	u32 target_ielen;
-	u8 *start;
-	uint search_len;
-
-	if (!ies || !ies_len || *ies_len <= offset)
-		goto exit;
-
-	start = ies + offset;
-	search_len = *ies_len - offset;
-
-	while (1) {
-		target_ie = rtw_get_ie_ex(start, search_len, eid, oui, oui_len, NULL, &target_ielen);
-		if (target_ie && target_ielen) {
-			u8 buf[MAX_IE_SZ] = {0};
-			u8 *remain_ies = target_ie + target_ielen;
-			uint remain_len = search_len - (remain_ies - start);
-
-			memcpy(buf, remain_ies, remain_len);
-			memcpy(target_ie, buf, remain_len);
-			*ies_len = *ies_len - target_ielen;
-			ret = _SUCCESS;
-
-			start = target_ie;
-			search_len = remain_len;
-		} else {
-			break;
-		}
-	}
-exit:
-	return ret;
-}
-
 void rtw_set_supported_rate(u8 *SupportedRates, uint mode)
 {
 
@@ -1096,43 +952,6 @@ void rtw_macaddr_cfg(u8 *mac_addr)
 	DBG_88E("rtw_macaddr_cfg MAC Address  = %pM\n", (mac_addr));
 }
 
-void dump_ies(u8 *buf, u32 buf_len)
-{
-	u8 *pos = buf;
-	u8 id, len;
-
-	while (pos - buf <= buf_len) {
-		id = *pos;
-		len = *(pos + 1);
-
-		DBG_88E("%s ID:%u, LEN:%u\n", __func__, id, len);
-		dump_wps_ie(pos, len);
-
-		pos += (2 + len);
-	}
-}
-
-void dump_wps_ie(u8 *ie, u32 ie_len)
-{
-	u8 *pos = ie;
-	u16 id;
-	u16 len;
-	u8 *wps_ie;
-	uint wps_ielen;
-
-	wps_ie = rtw_get_wps_ie(ie, ie_len, NULL, &wps_ielen);
-	if (wps_ie != ie || wps_ielen == 0)
-		return;
-
-	pos += 6;
-	while (pos - ie < ie_len) {
-		id = get_unaligned_be16(pos);
-		len = get_unaligned_be16(pos + 2);
-		DBG_88E("%s ID:0x%04x, LEN:%u\n", __func__, id, len);
-		pos += (4 + len);
-	}
-}
-
 /* Baron adds to avoid FreeBSD warning */
 int ieee80211_is_empty_essid(const char *essid, int essid_len)
 {
@@ -1223,7 +1042,6 @@ void rtw_get_bcn_info(struct wlan_network *pnetwork)
 	__le16 le_tmp;
 	u16 wpa_len = 0, rsn_len = 0;
 	struct HT_info_element *pht_info = NULL;
-	struct rtw_ieee80211_ht_cap *pht_cap = NULL;
 	unsigned int		len;
 	unsigned char		*p;
 
@@ -1259,10 +1077,12 @@ void rtw_get_bcn_info(struct wlan_network *pnetwork)
 	/* parsing HT_CAP_IE */
 	p = rtw_get_ie(pnetwork->network.IEs + _FIXED_IE_LENGTH_, _HT_CAPABILITY_IE_, &len, pnetwork->network.IELength - _FIXED_IE_LENGTH_);
 	if (p && len > 0) {
-			pht_cap = (struct rtw_ieee80211_ht_cap *)(p + 2);
-			pnetwork->BcnInfo.ht_cap_info = pht_cap->cap_info;
+		struct ieee80211_ht_cap *ht_cap =
+			(struct ieee80211_ht_cap *)(p + 2);
+
+		pnetwork->BcnInfo.ht_cap_info = le16_to_cpu(ht_cap->cap_info);
 	} else {
-			pnetwork->BcnInfo.ht_cap_info = 0;
+		pnetwork->BcnInfo.ht_cap_info = 0;
 	}
 	/* parsing HT_INFO_IE */
 	p = rtw_get_ie(pnetwork->network.IEs + _FIXED_IE_LENGTH_, _HT_ADD_INFO_IE_, &len, pnetwork->network.IELength - _FIXED_IE_LENGTH_);
@@ -1335,58 +1155,3 @@ u16 rtw_mcs_rate(u8 rf_type, u8 bw_40MHz, u8 short_GI_20, u8 short_GI_40, unsign
 	}
 	return max_rate;
 }
-
-int rtw_action_frame_parse(const u8 *frame, u32 frame_len, u8 *category, u8 *action)
-{
-	const u8 *frame_body = frame + sizeof(struct rtw_ieee80211_hdr_3addr);
-	u16 fc;
-	u8 c, a = 0;
-
-	fc = le16_to_cpu(((struct rtw_ieee80211_hdr_3addr *)frame)->frame_ctl);
-
-	if ((fc & (RTW_IEEE80211_FCTL_FTYPE | RTW_IEEE80211_FCTL_STYPE)) !=
-	    (RTW_IEEE80211_FTYPE_MGMT | RTW_IEEE80211_STYPE_ACTION))
-		return false;
-
-	c = frame_body[0];
-
-	switch (c) {
-	case RTW_WLAN_CATEGORY_P2P: /* vendor-specific */
-		break;
-	default:
-		a = frame_body[1];
-	}
-
-	if (category)
-		*category = c;
-	if (action)
-		*action = a;
-
-	return true;
-}
-
-static const char *_action_public_str[] = {
-	"ACT_PUB_BSSCOEXIST",
-	"ACT_PUB_DSE_ENABLE",
-	"ACT_PUB_DSE_DEENABLE",
-	"ACT_PUB_DSE_REG_LOCATION",
-	"ACT_PUB_EXT_CHL_SWITCH",
-	"ACT_PUB_DSE_MSR_REQ",
-	"ACT_PUB_DSE_MSR_RPRT",
-	"ACT_PUB_MP",
-	"ACT_PUB_DSE_PWR_CONSTRAINT",
-	"ACT_PUB_VENDOR",
-	"ACT_PUB_GAS_INITIAL_REQ",
-	"ACT_PUB_GAS_INITIAL_RSP",
-	"ACT_PUB_GAS_COMEBACK_REQ",
-	"ACT_PUB_GAS_COMEBACK_RSP",
-	"ACT_PUB_TDLS_DISCOVERY_RSP",
-	"ACT_PUB_LOCATION_TRACK",
-	"ACT_PUB_RSVD",
-};
-
-const char *action_public_str(u8 action)
-{
-	action = min_t(u8, action, ACT_PUBLIC_MAX);
-	return _action_public_str[action];
-}
diff --git a/drivers/staging/rtl8188eu/core/rtw_ioctl_set.c b/drivers/staging/rtl8188eu/core/rtw_ioctl_set.c
index f85a6abec3a3..6ed23f4db38c 100644
--- a/drivers/staging/rtl8188eu/core/rtw_ioctl_set.c
+++ b/drivers/staging/rtl8188eu/core/rtw_ioctl_set.c
@@ -14,7 +14,6 @@
  ******************************************************************************/
 #define _RTW_IOCTL_SET_C_
 
-
 #include <osdep_service.h>
 #include <drv_types.h>
 #include <rtw_ioctl_set.h>
@@ -570,10 +569,8 @@ u16 rtw_get_cur_max_rate(struct adapter *adapter)
 	struct registry_priv *pregistrypriv = &adapter->registrypriv;
 	struct mlme_priv	*pmlmepriv = &adapter->mlmepriv;
 	struct wlan_bssid_ex  *pcur_bss = &pmlmepriv->cur_network.network;
-	struct rtw_ieee80211_ht_cap *pht_capie;
 	u8	rf_type = 0;
 	u8	bw_40MHz = 0, short_GI_20 = 0, short_GI_40 = 0;
-	u16	mcs_rate = 0;
 	u32	ht_ielen = 0;
 
 	if (adapter->registrypriv.mp_mode == 1) {
@@ -588,15 +585,11 @@ u16 rtw_get_cur_max_rate(struct adapter *adapter)
 	if (pmlmeext->cur_wireless_mode & (WIRELESS_11_24N|WIRELESS_11_5N)) {
 		p = rtw_get_ie(&pcur_bss->IEs[12], _HT_CAPABILITY_IE_, &ht_ielen, pcur_bss->IELength-12);
 		if (p && ht_ielen > 0) {
-			pht_capie = (struct rtw_ieee80211_ht_cap *)(p+2);
-
-			memcpy(&mcs_rate, pht_capie->supp_mcs_set, 2);
-
 			/* cur_bwmod is updated by beacon, pmlmeinfo is updated by association response */
 			bw_40MHz = (pmlmeext->cur_bwmode && (HT_INFO_HT_PARAM_REC_TRANS_CHNL_WIDTH & pmlmeinfo->HT_info.infos[0])) ? 1 : 0;
 
-			short_GI_20 = (le16_to_cpu(pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info) & IEEE80211_HT_CAP_SGI_20) ? 1 : 0;
-			short_GI_40 = (le16_to_cpu(pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info) & IEEE80211_HT_CAP_SGI_40) ? 1 : 0;
+			short_GI_20 = (le16_to_cpu(pmlmeinfo->HT_caps.cap_info) & IEEE80211_HT_CAP_SGI_20) ? 1 : 0;
+			short_GI_40 = (le16_to_cpu(pmlmeinfo->HT_caps.cap_info) & IEEE80211_HT_CAP_SGI_40) ? 1 : 0;
 
 			rtw_hal_get_hwreg(adapter, HW_VAR_RF_TYPE, (u8 *)(&rf_type));
 			max_rate = rtw_mcs_rate(
@@ -604,7 +597,7 @@ u16 rtw_get_cur_max_rate(struct adapter *adapter)
 				bw_40MHz & (pregistrypriv->cbw40_enable),
 				short_GI_20,
 				short_GI_40,
-				pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate
+				pmlmeinfo->HT_caps.mcs.rx_mask
 			);
 		}
 	} else {
diff --git a/drivers/staging/rtl8188eu/core/rtw_mlme.c b/drivers/staging/rtl8188eu/core/rtw_mlme.c
index 1456499b84bf..ee2dcd05010f 100644
--- a/drivers/staging/rtl8188eu/core/rtw_mlme.c
+++ b/drivers/staging/rtl8188eu/core/rtw_mlme.c
@@ -52,8 +52,6 @@ int rtw_init_mlme_priv(struct adapter *padapter)
 	_rtw_init_queue(&(pmlmepriv->free_bss_pool));
 	_rtw_init_queue(&(pmlmepriv->scanned_queue));
 
-	set_scanned_network_val(pmlmepriv, 0);
-
 	memset(&pmlmepriv->assoc_ssid, 0, sizeof(struct ndis_802_11_ssid));
 
 	pbuf = vzalloc(MAX_BSS_CNT * (sizeof(struct wlan_network)));
@@ -100,12 +98,6 @@ void rtw_free_mlme_priv_ie_data(struct mlme_priv *pmlmepriv)
 	rtw_free_mlme_ie_data(&pmlmepriv->wps_probe_req_ie, &pmlmepriv->wps_probe_req_ie_len);
 	rtw_free_mlme_ie_data(&pmlmepriv->wps_probe_resp_ie, &pmlmepriv->wps_probe_resp_ie_len);
 	rtw_free_mlme_ie_data(&pmlmepriv->wps_assoc_resp_ie, &pmlmepriv->wps_assoc_resp_ie_len);
-
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_beacon_ie, &pmlmepriv->p2p_beacon_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_probe_req_ie, &pmlmepriv->p2p_probe_req_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_probe_resp_ie, &pmlmepriv->p2p_probe_resp_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_go_probe_resp_ie, &pmlmepriv->p2p_go_probe_resp_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_assoc_req_ie, &pmlmepriv->p2p_assoc_req_ie_len);
 }
 #else
 void rtw_free_mlme_priv_ie_data(struct mlme_priv *pmlmepriv)
@@ -143,8 +135,6 @@ struct wlan_network *_rtw_alloc_network(struct mlme_priv *pmlmepriv)
 	pnetwork->aid = 0;
 	pnetwork->join_res = 0;
 
-	pmlmepriv->num_of_scanned++;
-
 exit:
 	spin_unlock_bh(&free_queue->lock);
 
@@ -175,7 +165,6 @@ static void _rtw_free_network(struct mlme_priv *pmlmepriv, struct wlan_network *
 	spin_lock_bh(&free_queue->lock);
 	list_del_init(&(pnetwork->list));
 	list_add_tail(&(pnetwork->list), &(free_queue->queue));
-	pmlmepriv->num_of_scanned--;
 	spin_unlock_bh(&free_queue->lock);
 }
 
@@ -189,7 +178,6 @@ void _rtw_free_network_nolock(struct	mlme_priv *pmlmepriv, struct wlan_network *
 		return;
 	list_del_init(&(pnetwork->list));
 	list_add_tail(&(pnetwork->list), get_list_head(free_queue));
-	pmlmepriv->num_of_scanned--;
 }
 
 /*
@@ -271,7 +259,6 @@ void rtw_generate_random_ibss(u8 *pibss)
 	pibss[3] = (u8)(curtime & 0xff);/* p[0]; */
 	pibss[4] = (u8)((curtime>>8) & 0xff);/* p[1]; */
 	pibss[5] = (u8)((curtime>>16) & 0xff);/* p[2]; */
-	return;
 }
 
 u8 *rtw_get_capability_from_ie(u8 *ie)
@@ -569,7 +556,6 @@ static int rtw_is_desired_network(struct adapter *adapter, struct wlan_network *
 void rtw_atimdone_event_callback(struct adapter *adapter, u8 *pbuf)
 {
 	RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_, ("receive atimdone_evet\n"));
-	return;
 }
 
 
@@ -732,7 +718,6 @@ static void free_scanqueue(struct	mlme_priv *pmlmepriv)
 		list_del_init(plist);
 		list_add_tail(plist, &free_queue->queue);
 		plist = ptemp;
-		pmlmepriv->num_of_scanned--;
 	}
 
 	spin_unlock_bh(&free_queue->lock);
@@ -1935,7 +1920,6 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 	u32 ielen, out_len;
 	enum ht_cap_ampdu_factor max_rx_ampdu_factor;
 	unsigned char *p;
-	struct rtw_ieee80211_ht_cap ht_capie;
 	unsigned char WMM_IE[] = {0x00, 0x50, 0xf2, 0x02, 0x00, 0x01, 0x00};
 	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 	struct qos_priv		*pqospriv = &pmlmepriv->qospriv;
@@ -1948,6 +1932,8 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 	p = rtw_get_ie(in_ie+12, _HT_CAPABILITY_IE_, &ielen, in_len-12);
 
 	if (p && ielen > 0) {
+		struct ieee80211_ht_cap ht_cap;
+
 		if (pqospriv->qos_option == 0) {
 			out_len = *pout_len;
 			rtw_set_ie(out_ie+out_len, _VENDOR_SPECIFIC_IE_,
@@ -1958,33 +1944,33 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 
 		out_len = *pout_len;
 
-		memset(&ht_capie, 0, sizeof(struct rtw_ieee80211_ht_cap));
+		memset(&ht_cap, 0, sizeof(struct ieee80211_ht_cap));
 
-		ht_capie.cap_info = IEEE80211_HT_CAP_SUP_WIDTH |
-				    IEEE80211_HT_CAP_SGI_20 |
-				    IEEE80211_HT_CAP_SGI_40 |
-				    IEEE80211_HT_CAP_TX_STBC |
-				    IEEE80211_HT_CAP_DSSSCCK40;
+		ht_cap.cap_info = cpu_to_le16(IEEE80211_HT_CAP_SUP_WIDTH |
+					      IEEE80211_HT_CAP_SGI_20 |
+					      IEEE80211_HT_CAP_SGI_40 |
+					      IEEE80211_HT_CAP_TX_STBC |
+					      IEEE80211_HT_CAP_DSSSCCK40);
 
 		rtw_hal_get_def_var(padapter, HAL_DEF_RX_PACKET_OFFSET, &rx_packet_offset);
 		rtw_hal_get_def_var(padapter, HAL_DEF_MAX_RECVBUF_SZ, &max_recvbuf_sz);
 
 		/*
-		AMPDU_para [1:0]:Max AMPDU Len => 0:8k , 1:16k, 2:32k, 3:64k
-		AMPDU_para [4:2]:Min MPDU Start Spacing
+		ampdu_params_info [1:0]:Max AMPDU Len => 0:8k , 1:16k, 2:32k, 3:64k
+		ampdu_params_info [4:2]:Min MPDU Start Spacing
 		*/
 
 		rtw_hal_get_def_var(padapter, HW_VAR_MAX_RX_AMPDU_FACTOR, &max_rx_ampdu_factor);
-		ht_capie.ampdu_params_info = (max_rx_ampdu_factor&0x03);
+		ht_cap.ampdu_params_info = max_rx_ampdu_factor & 0x03;
 
 		if (padapter->securitypriv.dot11PrivacyAlgrthm == _AES_)
-			ht_capie.ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY&(0x07<<2));
+			ht_cap.ampdu_params_info |= IEEE80211_HT_CAP_AMPDU_DENSITY & (0x07 << 2);
 		else
-			ht_capie.ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY&0x00);
-
+			ht_cap.ampdu_params_info |= IEEE80211_HT_CAP_AMPDU_DENSITY & 0x00;
 
 		rtw_set_ie(out_ie+out_len, _HT_CAPABILITY_IE_,
-			   sizeof(struct rtw_ieee80211_ht_cap), (unsigned char *)&ht_capie, pout_len);
+			   sizeof(struct ieee80211_ht_cap),
+			   (unsigned char *)&ht_cap, pout_len);
 
 		phtpriv->ht_option = true;
 
@@ -2000,9 +1986,6 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 /* the function is > passive_level (in critical_section) */
 void rtw_update_ht_cap(struct adapter *padapter, u8 *pie, uint ie_len)
 {
-	u8 *p, max_ampdu_sz;
-	int len;
-	struct rtw_ieee80211_ht_cap *pht_capie;
 	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 	struct ht_priv		*phtpriv = &pmlmepriv->htpriv;
 	struct registry_priv *pregistrypriv = &padapter->registrypriv;
@@ -2027,34 +2010,21 @@ void rtw_update_ht_cap(struct adapter *padapter, u8 *pie, uint ie_len)
 		phtpriv->ampdu_enable = true;
 	}
 
-
-	/* check Max Rx A-MPDU Size */
-	len = 0;
-	p = rtw_get_ie(pie+sizeof(struct ndis_802_11_fixed_ie), _HT_CAPABILITY_IE_, &len, ie_len-sizeof(struct ndis_802_11_fixed_ie));
-	if (p && len > 0) {
-		pht_capie = (struct rtw_ieee80211_ht_cap *)(p+2);
-		max_ampdu_sz = pht_capie->ampdu_params_info & IEEE80211_HT_CAP_AMPDU_FACTOR;
-		max_ampdu_sz = 1 << (max_ampdu_sz+3); /*  max_ampdu_sz (kbytes); */
-		phtpriv->rx_ampdu_maxlen = max_ampdu_sz;
-	}
-	len = 0;
-	p = rtw_get_ie(pie+sizeof(struct ndis_802_11_fixed_ie), _HT_ADD_INFO_IE_, &len, ie_len-sizeof(struct ndis_802_11_fixed_ie));
-
 	/* update cur_bwmode & cur_ch_offset */
 	if ((pregistrypriv->cbw40_enable) &&
-	    (le16_to_cpu(pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info) & BIT(1)) &&
+	    (le16_to_cpu(pmlmeinfo->HT_caps.cap_info) & BIT(1)) &&
 	    (pmlmeinfo->HT_info.infos[0] & BIT(2))) {
 		int i;
 		u8	rf_type;
 
-		padapter->HalFunc.GetHwRegHandler(padapter, HW_VAR_RF_TYPE, (u8 *)(&rf_type));
+		rtw_hal_get_hwreg(padapter, HW_VAR_RF_TYPE, (u8 *)(&rf_type));
 
 		/* update the MCS rates */
 		for (i = 0; i < 16; i++) {
 			if ((rf_type == RF_1T1R) || (rf_type == RF_1T2R))
-				pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate[i] &= MCS_rate_1R[i];
+				((u8 *)&pmlmeinfo->HT_caps.mcs)[i] &= MCS_rate_1R[i];
 			else
-				pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate[i] &= MCS_rate_2R[i];
+				((u8 *)&pmlmeinfo->HT_caps.mcs)[i] &= MCS_rate_2R[i];
 		}
 		/* switch to the 40M Hz mode according to the AP */
 		pmlmeext->cur_bwmode = HT_CHANNEL_WIDTH_40;
@@ -2072,7 +2042,7 @@ void rtw_update_ht_cap(struct adapter *padapter, u8 *pie, uint ie_len)
 	}
 
 	/*  Config SM Power Save setting */
-	pmlmeinfo->SM_PS = (le16_to_cpu(pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info) & 0x0C) >> 2;
+	pmlmeinfo->SM_PS = (le16_to_cpu(pmlmeinfo->HT_caps.cap_info) & 0x0C) >> 2;
 	if (pmlmeinfo->SM_PS == WLAN_HT_CAP_SM_PS_STATIC)
 		DBG_88E("%s(): WLAN_HT_CAP_SM_PS_STATIC\n", __func__);
 
diff --git a/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c b/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
index 7f32b39e5869..fb13df586441 100644
--- a/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
@@ -320,7 +320,7 @@ static void issue_beacon(struct adapter *padapter, int timeout_ms)
 	struct xmit_frame	*pmgntframe;
 	struct pkt_attrib	*pattrib;
 	unsigned char	*pframe;
-	struct rtw_ieee80211_hdr *pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	unsigned int	rate_len;
 	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
@@ -347,22 +347,22 @@ static void issue_beacon(struct adapter *padapter, int timeout_ms)
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
-	memcpy(pwlanhdr->addr1, bc_addr, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN);
-	memcpy(pwlanhdr->addr3, cur_network->MacAddress, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, bc_addr);
+	ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr3, cur_network->MacAddress);
 
 	SetSeqNum(pwlanhdr, 0/*pmlmeext->mgnt_seq*/);
 	/* pmlmeext->mgnt_seq++; */
 	SetFrameSubType(pframe, WIFI_BEACON);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pframe += sizeof(struct ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
 	if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE) {
 		int len_diff;
@@ -377,8 +377,8 @@ static void issue_beacon(struct adapter *padapter, int timeout_ms)
 			);
 		pframe += (cur_network->IELength+len_diff);
 		pattrib->pktlen += (cur_network->IELength+len_diff);
-		wps_ie = rtw_get_wps_ie(pmgntframe->buf_addr+TXDESC_OFFSET+sizeof(struct rtw_ieee80211_hdr_3addr)+_BEACON_IE_OFFSET_,
-			pattrib->pktlen-sizeof(struct rtw_ieee80211_hdr_3addr)-_BEACON_IE_OFFSET_, NULL, &wps_ielen);
+		wps_ie = rtw_get_wps_ie(pmgntframe->buf_addr+TXDESC_OFFSET+sizeof(struct ieee80211_hdr_3addr)+_BEACON_IE_OFFSET_,
+			pattrib->pktlen-sizeof(struct ieee80211_hdr_3addr)-_BEACON_IE_OFFSET_, NULL, &wps_ielen);
 		if (wps_ie && wps_ielen > 0)
 			rtw_get_wps_attr_content(wps_ie,  wps_ielen, WPS_ATTR_SELECTED_REGISTRAR, (u8 *)(&sr), NULL);
 		if (sr != 0)
@@ -461,7 +461,7 @@ static void issue_probersp(struct adapter *padapter, unsigned char *da)
 	struct xmit_frame			*pmgntframe;
 	struct pkt_attrib			*pattrib;
 	unsigned char					*pframe;
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	unsigned char					*mac, *bssid;
 	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
@@ -488,22 +488,22 @@ static void issue_probersp(struct adapter *padapter, unsigned char *da)
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
 	mac = myid(&(padapter->eeprompriv));
 	bssid = cur_network->MacAddress;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
-	memcpy(pwlanhdr->addr1, da, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
-	memcpy(pwlanhdr->addr3, bssid, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, da);
+	ether_addr_copy(pwlanhdr->addr2, mac);
+	ether_addr_copy(pwlanhdr->addr3, bssid);
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	SetFrameSubType(fctrl, WIFI_PROBERSP);
 
-	pattrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr);
 	pattrib->pktlen = pattrib->hdrlen;
 	pframe += pattrib->hdrlen;
 
@@ -609,7 +609,7 @@ static int issue_probereq(struct adapter *padapter, struct ndis_802_11_ssid *pss
 	struct xmit_frame		*pmgntframe;
 	struct pkt_attrib		*pattrib;
 	unsigned char			*pframe;
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	unsigned char			*mac;
 	unsigned char			bssrate[NumRates];
@@ -633,31 +633,31 @@ static int issue_probereq(struct adapter *padapter, struct ndis_802_11_ssid *pss
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
 	mac = myid(&(padapter->eeprompriv));
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
 	if (da) {
 		/*	unicast probe request frame */
-		memcpy(pwlanhdr->addr1, da, ETH_ALEN);
-		memcpy(pwlanhdr->addr3, da, ETH_ALEN);
+		ether_addr_copy(pwlanhdr->addr1, da);
+		ether_addr_copy(pwlanhdr->addr3, da);
 	} else {
 		/*	broadcast probe request frame */
-		memcpy(pwlanhdr->addr1, bc_addr, ETH_ALEN);
-		memcpy(pwlanhdr->addr3, bc_addr, ETH_ALEN);
+		ether_addr_copy(pwlanhdr->addr1, bc_addr);
+		ether_addr_copy(pwlanhdr->addr3, bc_addr);
 	}
 
-	memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr2, mac);
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	SetFrameSubType(pframe, WIFI_PROBEREQ);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pframe += sizeof(struct ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
 	if (pssid)
 		pframe = rtw_set_ie(pframe, _SSID_IE_, pssid->SsidLength, pssid->Ssid, &(pattrib->pktlen));
@@ -745,7 +745,7 @@ static void issue_auth(struct adapter *padapter, struct sta_info *psta,
 	struct xmit_frame *pmgntframe;
 	struct pkt_attrib *pattrib;
 	unsigned char *pframe;
-	struct rtw_ieee80211_hdr *pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	unsigned int val32;
 	u16 val16;
@@ -769,25 +769,27 @@ static void issue_auth(struct adapter *padapter, struct sta_info *psta,
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	SetFrameSubType(pframe, WIFI_AUTH);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pframe += sizeof(struct ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
 
 	if (psta) {/*  for AP mode */
 #ifdef CONFIG_88EU_AP_MODE
 
-		memcpy(pwlanhdr->addr1, psta->hwaddr, ETH_ALEN);
-		memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN);
-		memcpy(pwlanhdr->addr3, myid(&(padapter->eeprompriv)), ETH_ALEN);
+		ether_addr_copy(pwlanhdr->addr1, psta->hwaddr);
+		ether_addr_copy(pwlanhdr->addr2,
+				myid(&(padapter->eeprompriv)));
+		ether_addr_copy(pwlanhdr->addr3,
+				myid(&(padapter->eeprompriv)));
 
 
 		/*  setting auth algo number */
@@ -825,9 +827,9 @@ static void issue_auth(struct adapter *padapter, struct sta_info *psta,
 	} else {
 		__le32 le_tmp32;
 		__le16 le_tmp16;
-		memcpy(pwlanhdr->addr1, pnetwork->MacAddress, ETH_ALEN);
-		memcpy(pwlanhdr->addr2, myid(&padapter->eeprompriv), ETH_ALEN);
-		memcpy(pwlanhdr->addr3, pnetwork->MacAddress, ETH_ALEN);
+		ether_addr_copy(pwlanhdr->addr1, pnetwork->MacAddress);
+		ether_addr_copy(pwlanhdr->addr2, myid(&padapter->eeprompriv));
+		ether_addr_copy(pwlanhdr->addr3, pnetwork->MacAddress);
 
 		/*  setting auth algo number */
 		val16 = (pmlmeinfo->auth_algo == dot11AuthAlgrthm_Shared) ? 1 : 0;/*  0:OPEN System, 1:Shared key */
@@ -866,7 +868,7 @@ static void issue_auth(struct adapter *padapter, struct sta_info *psta,
 
 			SetPrivacy(fctrl);
 
-			pattrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+			pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr);
 
 			pattrib->encrypt = _WEP40_;
 
@@ -889,7 +891,7 @@ static void issue_asocrsp(struct adapter *padapter, unsigned short status,
 			  struct sta_info *pstat, int pkt_type)
 {
 	struct xmit_frame	*pmgntframe;
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	struct pkt_attrib *pattrib;
 	unsigned char	*pbuf, *pframe;
 	unsigned short val;
@@ -916,14 +918,15 @@ static void issue_asocrsp(struct adapter *padapter, unsigned short status,
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
-	memcpy((void *)GetAddr1Ptr(pwlanhdr), pstat->hwaddr, ETH_ALEN);
-	memcpy((void *)GetAddr2Ptr(pwlanhdr), myid(&(padapter->eeprompriv)), ETH_ALEN);
-	memcpy((void *)GetAddr3Ptr(pwlanhdr), pnetwork->MacAddress, ETH_ALEN);
+	ether_addr_copy((void *)GetAddr1Ptr(pwlanhdr), pstat->hwaddr);
+	ether_addr_copy((void *)GetAddr2Ptr(pwlanhdr),
+			myid(&(padapter->eeprompriv)));
+	ether_addr_copy((void *)GetAddr3Ptr(pwlanhdr), pnetwork->MacAddress);
 
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
@@ -933,7 +936,7 @@ static void issue_asocrsp(struct adapter *padapter, unsigned short status,
 	else
 		return;
 
-	pattrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr);
 	pattrib->pktlen += pattrib->hdrlen;
 	pframe += pattrib->hdrlen;
 
@@ -1017,7 +1020,7 @@ static void issue_assocreq(struct adapter *padapter)
 	struct xmit_frame	*pmgntframe;
 	struct pkt_attrib	*pattrib;
 	unsigned char		*pframe, *p;
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	unsigned int	i, j, ie_len, index = 0;
 	unsigned char	rf_type, bssrate[NumRates], sta_bssrate[NumRates];
@@ -1040,20 +1043,20 @@ static void issue_assocreq(struct adapter *padapter)
 
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
-	memcpy(pwlanhdr->addr1, pnetwork->MacAddress, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN);
-	memcpy(pwlanhdr->addr3, pnetwork->MacAddress, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, pnetwork->MacAddress);
+	ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr3, pnetwork->MacAddress);
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	SetFrameSubType(pframe, WIFI_ASSOCREQ);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pframe += sizeof(struct ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
 	/* caps */
 
@@ -1132,23 +1135,23 @@ static void issue_assocreq(struct adapter *padapter)
 	if (padapter->mlmepriv.htpriv.ht_option) {
 		p = rtw_get_ie((pmlmeinfo->network.IEs + sizeof(struct ndis_802_11_fixed_ie)), _HT_CAPABILITY_IE_, &ie_len, (pmlmeinfo->network.IELength - sizeof(struct ndis_802_11_fixed_ie)));
 		if ((p != NULL) && (!(is_ap_in_tkip(padapter)))) {
-			memcpy(&(pmlmeinfo->HT_caps), (p + 2), sizeof(struct HT_caps_element));
+			memcpy(&pmlmeinfo->HT_caps, p + 2, sizeof(struct ieee80211_ht_cap));
 
 			/* to disable 40M Hz support while gd_bw_40MHz_en = 0 */
 			if (pregpriv->cbw40_enable == 0)
-				pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info &= cpu_to_le16(~(BIT(6) | BIT(1)));
+				pmlmeinfo->HT_caps.cap_info &= cpu_to_le16(~(BIT(6) | BIT(1)));
 			else
-				pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info |= cpu_to_le16(BIT(1));
+				pmlmeinfo->HT_caps.cap_info |= cpu_to_le16(BIT(1));
 
 			/* todo: disable SM power save mode */
-			pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info |= cpu_to_le16(0x000c);
+			pmlmeinfo->HT_caps.cap_info |= cpu_to_le16(0x000c);
 
 			rtw_hal_get_hwreg(padapter, HW_VAR_RF_TYPE, (u8 *)(&rf_type));
 			switch (rf_type) {
 			case RF_1T1R:
 				if (pregpriv->rx_stbc)
-					pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info |= cpu_to_le16(0x0100);/* RX STBC One spatial stream */
-				memcpy(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_rate_1R, 16);
+					pmlmeinfo->HT_caps.cap_info |= cpu_to_le16(0x0100);/* RX STBC One spatial stream */
+				memcpy((u8 *)&pmlmeinfo->HT_caps.mcs, MCS_rate_1R, 16);
 				break;
 			case RF_2T2R:
 			case RF_1T2R:
@@ -1157,9 +1160,9 @@ static void issue_assocreq(struct adapter *padapter)
 				    ((pmlmeext->cur_wireless_mode & WIRELESS_11_24N) && (pregpriv->rx_stbc == 0x1)) || /* enable for 2.4GHz */
 				    (pregpriv->wifi_spec == 1)) {
 					DBG_88E("declare supporting RX STBC\n");
-					pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info |= cpu_to_le16(0x0200);/* RX STBC two spatial stream */
+					pmlmeinfo->HT_caps.cap_info |= cpu_to_le16(0x0200);/* RX STBC two spatial stream */
 				}
-				memcpy(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_rate_2R, 16);
+				memcpy(&pmlmeinfo->HT_caps.mcs, MCS_rate_2R, 16);
 				break;
 			}
 			pframe = rtw_set_ie(pframe, _HT_CAPABILITY_IE_, ie_len , (u8 *)(&(pmlmeinfo->HT_caps)), &(pattrib->pktlen));
@@ -1212,7 +1215,7 @@ static int _issue_nulldata(struct adapter *padapter, unsigned char *da, unsigned
 	struct xmit_frame			*pmgntframe;
 	struct pkt_attrib			*pattrib;
 	unsigned char					*pframe;
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	struct xmit_priv	*pxmitpriv;
 	struct mlme_ext_priv	*pmlmeext;
@@ -1239,9 +1242,9 @@ static int _issue_nulldata(struct adapter *padapter, unsigned char *da, unsigned
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
 	if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE)
@@ -1252,16 +1255,16 @@ static int _issue_nulldata(struct adapter *padapter, unsigned char *da, unsigned
 	if (power_mode)
 		SetPwrMgt(fctrl);
 
-	memcpy(pwlanhdr->addr1, da, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN);
-	memcpy(pwlanhdr->addr3, pnetwork->MacAddress, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, da);
+	ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr3, pnetwork->MacAddress);
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	SetFrameSubType(pframe, WIFI_DATA_NULL);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pframe += sizeof(struct ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
 	pattrib->last_txcmdsz = pattrib->pktlen;
 
@@ -1332,7 +1335,7 @@ static int _issue_qos_nulldata(struct adapter *padapter, unsigned char *da, u16
 	struct xmit_frame			*pmgntframe;
 	struct pkt_attrib			*pattrib;
 	unsigned char					*pframe;
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	unsigned short *qc;
 	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
@@ -1359,9 +1362,9 @@ static int _issue_qos_nulldata(struct adapter *padapter, unsigned char *da, u16
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
 	if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE)
@@ -1380,16 +1383,16 @@ static int _issue_qos_nulldata(struct adapter *padapter, unsigned char *da, u16
 
 	SetAckpolicy(qc, pattrib->ack_policy);
 
-	memcpy(pwlanhdr->addr1, da, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN);
-	memcpy(pwlanhdr->addr3, pnetwork->MacAddress, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, da);
+	ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr3, pnetwork->MacAddress);
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	SetFrameSubType(pframe, WIFI_QOS_DATA_NULL);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr_qos);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr_qos);
+	pframe += sizeof(struct ieee80211_qos_hdr);
+	pattrib->pktlen = sizeof(struct ieee80211_qos_hdr);
 
 	pattrib->last_txcmdsz = pattrib->pktlen;
 
@@ -1457,7 +1460,7 @@ static int _issue_deauth(struct adapter *padapter, unsigned char *da, unsigned s
 	struct xmit_frame			*pmgntframe;
 	struct pkt_attrib			*pattrib;
 	unsigned char					*pframe;
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
 	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
@@ -1478,21 +1481,21 @@ static int _issue_deauth(struct adapter *padapter, unsigned char *da, unsigned s
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
-	memcpy(pwlanhdr->addr1, da, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN);
-	memcpy(pwlanhdr->addr3, pnetwork->MacAddress, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, da);
+	ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr3, pnetwork->MacAddress);
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	SetFrameSubType(pframe, WIFI_DEAUTH);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pframe += sizeof(struct ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
 	le_tmp = cpu_to_le16(reason);
 	pframe = rtw_set_fixed_ie(pframe, _RSON_CODE_, &le_tmp,
@@ -1559,66 +1562,6 @@ exit:
 	return ret;
 }
 
-void issue_action_spct_ch_switch(struct adapter *padapter, u8 *ra, u8 new_ch, u8 ch_offset)
-{
-	struct xmit_frame			*pmgntframe;
-	struct pkt_attrib			*pattrib;
-	unsigned char				*pframe;
-	struct rtw_ieee80211_hdr	*pwlanhdr;
-	__le16 *fctrl;
-	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
-	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
-
-
-	DBG_88E(FUNC_NDEV_FMT" ra =%pM, ch:%u, offset:%u\n",
-		FUNC_NDEV_ARG(padapter->pnetdev), ra, new_ch, ch_offset);
-
-	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
-	if (pmgntframe == NULL)
-		return;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib(padapter, pattrib);
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
-
-	fctrl = &(pwlanhdr->frame_ctl);
-	*(fctrl) = 0;
-
-	memcpy(pwlanhdr->addr1, ra, ETH_ALEN); /* RA */
-	memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN); /* TA */
-	memcpy(pwlanhdr->addr3, ra, ETH_ALEN); /* DA = RA */
-
-	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
-	pmlmeext->mgnt_seq++;
-	SetFrameSubType(pframe, WIFI_ACTION);
-
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
-
-	/* category, action */
-	{
-		u8 category, action;
-		category = RTW_WLAN_CATEGORY_SPECTRUM_MGMT;
-		action = RTW_WLAN_ACTION_SPCT_CHL_SWITCH;
-
-		pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
-		pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
-	}
-
-	pframe = rtw_set_ie_ch_switch(pframe, &(pattrib->pktlen), 0, new_ch, 0);
-	pframe = rtw_set_ie_secondary_ch_offset(pframe, &(pattrib->pktlen),
-		hal_ch_offset_to_secondary_ch_offset(ch_offset));
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	dump_mgntframe(padapter, pmgntframe);
-}
-
 static void issue_action_BA(struct adapter *padapter, unsigned char *raddr,
 			    unsigned char action, unsigned short status)
 {
@@ -1633,7 +1576,7 @@ static void issue_action_BA(struct adapter *padapter, unsigned char *raddr,
 	struct xmit_frame *pmgntframe;
 	struct pkt_attrib *pattrib;
 	u8 *pframe;
-	struct rtw_ieee80211_hdr *pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
 	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
@@ -1656,21 +1599,21 @@ static void issue_action_BA(struct adapter *padapter, unsigned char *raddr,
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
-	memcpy(pwlanhdr->addr1, raddr, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN);
-	memcpy(pwlanhdr->addr3, pnetwork->MacAddress, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, raddr);
+	ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr3, pnetwork->MacAddress);
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	SetFrameSubType(pframe, WIFI_ACTION);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pframe += sizeof(struct ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
 	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
 	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
@@ -1779,7 +1722,7 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter)
 	struct xmit_frame			*pmgntframe;
 	struct pkt_attrib			*pattrib;
 	unsigned char				*pframe;
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	struct	wlan_network	*pnetwork = NULL;
 	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
@@ -1815,21 +1758,21 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter)
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
-	memcpy(pwlanhdr->addr1, cur_network->MacAddress, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN);
-	memcpy(pwlanhdr->addr3, cur_network->MacAddress, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, cur_network->MacAddress);
+	ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr3, cur_network->MacAddress);
 
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	SetFrameSubType(pframe, WIFI_ACTION);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pframe += sizeof(struct ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
 	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
 	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
@@ -2112,7 +2055,7 @@ static u8 collect_bss_info(struct adapter *padapter,
 	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
 	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
 
-	len = packet_len - sizeof(struct rtw_ieee80211_hdr_3addr);
+	len = packet_len - sizeof(struct ieee80211_hdr_3addr);
 
 	if (len > MAX_IE_SZ)
 		return _FAIL;
@@ -2142,7 +2085,7 @@ static u8 collect_bss_info(struct adapter *padapter,
 
 	/* below is to copy the information element */
 	bssid->IELength = len;
-	memcpy(bssid->IEs, (pframe + sizeof(struct rtw_ieee80211_hdr_3addr)), bssid->IELength);
+	memcpy(bssid->IEs, (pframe + sizeof(struct ieee80211_hdr_3addr)), bssid->IELength);
 
 	/* get the signal strength in dBM.raw data */
 	bssid->Rssi = precv_frame->attrib.phy_info.recvpower;
@@ -2219,7 +2162,7 @@ static u8 collect_bss_info(struct adapter *padapter,
 	if (subtype == WIFI_PROBEREQ) {
 		/*  FIXME */
 		bssid->InfrastructureMode = Ndis802_11Infrastructure;
-		memcpy(bssid->MacAddress, GetAddr2Ptr(pframe), ETH_ALEN);
+		ether_addr_copy(bssid->MacAddress, GetAddr2Ptr(pframe));
 		bssid->Privacy = 1;
 		return _SUCCESS;
 	}
@@ -2231,10 +2174,10 @@ static u8 collect_bss_info(struct adapter *padapter,
 
 	if (val16 & BIT(0)) {
 		bssid->InfrastructureMode = Ndis802_11Infrastructure;
-		memcpy(bssid->MacAddress, GetAddr2Ptr(pframe), ETH_ALEN);
+		ether_addr_copy(bssid->MacAddress, GetAddr2Ptr(pframe));
 	} else {
 		bssid->InfrastructureMode = Ndis802_11IBSS;
-		memcpy(bssid->MacAddress, GetAddr3Ptr(pframe), ETH_ALEN);
+		ether_addr_copy(bssid->MacAddress, GetAddr3Ptr(pframe));
 	}
 
 	if (val16 & BIT(4))
@@ -2249,10 +2192,10 @@ static u8 collect_bss_info(struct adapter *padapter,
 		struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 		p = rtw_get_ie(bssid->IEs + ie_offset, _HT_CAPABILITY_IE_, &len, bssid->IELength - ie_offset);
 		if (p && len > 0) {
-			struct HT_caps_element	*pHT_caps;
-			pHT_caps = (struct HT_caps_element *)(p + 2);
+			struct ieee80211_ht_cap *pHT_caps =
+				(struct ieee80211_ht_cap *)(p + 2);
 
-			if (le16_to_cpu(pHT_caps->u.HT_cap_element.HT_caps_info)&BIT(14))
+			if (le16_to_cpu(pHT_caps->cap_info) & BIT(14))
 				pmlmepriv->num_FortyMHzIntolerant++;
 		} else {
 			pmlmepriv->num_sta_no_ht++;
@@ -2684,7 +2627,7 @@ static unsigned int OnBeacon(struct adapter *padapter,
 			}
 
 			/* check the vendor of the assoc AP */
-			pmlmeinfo->assoc_AP_vendor = check_assoc_AP(pframe+sizeof(struct rtw_ieee80211_hdr_3addr), len-sizeof(struct rtw_ieee80211_hdr_3addr));
+			pmlmeinfo->assoc_AP_vendor = check_assoc_AP(pframe+sizeof(struct ieee80211_hdr_3addr), len-sizeof(struct ieee80211_hdr_3addr));
 
 			/* update TSF Value */
 			update_TSF(pmlmeext, pframe, len);
@@ -3296,13 +3239,15 @@ static unsigned int OnAssocReq(struct adapter *padapter,
 	}
 
 	/* save HT capabilities in the sta object */
-	memset(&pstat->htpriv.ht_cap, 0, sizeof(struct rtw_ieee80211_ht_cap));
-	if (elems.ht_capabilities && elems.ht_capabilities_len >= sizeof(struct rtw_ieee80211_ht_cap)) {
+	memset(&pstat->htpriv.ht_cap, 0, sizeof(struct ieee80211_ht_cap));
+	if (elems.ht_capabilities &&
+	    elems.ht_capabilities_len >= sizeof(struct ieee80211_ht_cap)) {
 		pstat->flags |= WLAN_STA_HT;
 
 		pstat->flags |= WLAN_STA_WME;
 
-		memcpy(&pstat->htpriv.ht_cap, elems.ht_capabilities, sizeof(struct rtw_ieee80211_ht_cap));
+		memcpy(&pstat->htpriv.ht_cap,
+		       elems.ht_capabilities, sizeof(struct ieee80211_ht_cap));
 	} else {
 		pstat->flags &= ~WLAN_STA_HT;
 	}
@@ -3639,7 +3584,7 @@ static unsigned int on_action_spct(struct adapter *padapter,
 	struct sta_info *psta = NULL;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	u8 *pframe = precv_frame->rx_data;
-	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 *frame_body = pframe + sizeof(struct ieee80211_hdr_3addr);
 	u8 category;
 	u8 action;
 
@@ -3713,7 +3658,7 @@ static unsigned int OnAction_back(struct adapter *padapter,
 	if (psta == NULL)
 		return _SUCCESS;
 
-	frame_body = (unsigned char *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+	frame_body = (unsigned char *)(pframe + sizeof(struct ieee80211_hdr_3addr));
 
 	category = frame_body[0];
 	if (category == RTW_WLAN_CATEGORY_BACK) { /*  representing Block Ack */
@@ -3800,7 +3745,7 @@ static unsigned int on_action_public_p2p(struct recv_frame *precv_frame)
 	u8 *pframe = precv_frame->rx_data;
 	u8 *frame_body;
 	u8 dialogToken = 0;
-	frame_body = (unsigned char *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+	frame_body = (unsigned char *)(pframe + sizeof(struct ieee80211_hdr_3addr));
 
 	dialogToken = frame_body[7];
 
@@ -3814,7 +3759,7 @@ static unsigned int on_action_public_vendor(struct recv_frame *precv_frame)
 {
 	unsigned int ret = _FAIL;
 	u8 *pframe = precv_frame->rx_data;
-	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 *frame_body = pframe + sizeof(struct ieee80211_hdr_3addr);
 
 	if (!memcmp(frame_body + 2, P2P_OUI, 4))
 		ret = on_action_public_p2p(precv_frame);
@@ -3826,7 +3771,7 @@ static unsigned int on_action_public_default(struct recv_frame *precv_frame, u8
 {
 	unsigned int ret = _FAIL;
 	u8 *pframe = precv_frame->rx_data;
-	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 *frame_body = pframe + sizeof(struct ieee80211_hdr_3addr);
 	u8 token;
 
 	token = frame_body[2];
@@ -3845,7 +3790,7 @@ static unsigned int on_action_public(struct adapter *padapter,
 {
 	unsigned int ret = _FAIL;
 	u8 *pframe = precv_frame->rx_data;
-	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 *frame_body = pframe + sizeof(struct ieee80211_hdr_3addr);
 	u8 category, action;
 
 	/* check RA matches or not */
@@ -3917,7 +3862,7 @@ static unsigned int OnAction(struct adapter *padapter,
 	unsigned char	*frame_body;
 	u8 *pframe = precv_frame->rx_data;
 
-	frame_body = (unsigned char *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+	frame_body = (unsigned char *)(pframe + sizeof(struct ieee80211_hdr_3addr));
 
 	category = frame_body[0];
 
@@ -4295,7 +4240,7 @@ void report_survey_event(struct adapter *padapter,
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = _Set_MLME_EVT_CMD_;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4304,7 +4249,7 @@ void report_survey_event(struct adapter *padapter,
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct survey_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_Survey);
+	pc2h_evt_hdr->ID = _Survey_EVT_;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	psurvey_evt = (struct survey_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
@@ -4345,7 +4290,7 @@ void report_surveydone_event(struct adapter *padapter)
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = _Set_MLME_EVT_CMD_;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4354,7 +4299,7 @@ void report_surveydone_event(struct adapter *padapter)
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct surveydone_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_SurveyDone);
+	pc2h_evt_hdr->ID = _SurveyDone_EVT_;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	psurveydone_evt = (struct surveydone_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
@@ -4389,7 +4334,7 @@ void report_join_res(struct adapter *padapter, int res)
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = _Set_MLME_EVT_CMD_;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4398,7 +4343,7 @@ void report_join_res(struct adapter *padapter, int res)
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct joinbss_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_JoinBss);
+	pc2h_evt_hdr->ID = _JoinBss_EVT_;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	pjoinbss_evt = (struct joinbss_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
@@ -4440,7 +4385,7 @@ void report_del_sta_event(struct adapter *padapter, unsigned char *MacAddr, unsi
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = _Set_MLME_EVT_CMD_;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4449,11 +4394,11 @@ void report_del_sta_event(struct adapter *padapter, unsigned char *MacAddr, unsi
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct stadel_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_DelSTA);
+	pc2h_evt_hdr->ID = _DelSTA_EVT_;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	pdel_sta_evt = (struct stadel_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
-	memcpy((unsigned char *)(&(pdel_sta_evt->macaddr)), MacAddr, ETH_ALEN);
+	ether_addr_copy((unsigned char *)(&(pdel_sta_evt->macaddr)), MacAddr);
 	memcpy((unsigned char *)(pdel_sta_evt->rsvd), (unsigned char *)(&reason), 2);
 
 
@@ -4493,7 +4438,7 @@ void report_add_sta_event(struct adapter *padapter, unsigned char *MacAddr, int
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = _Set_MLME_EVT_CMD_;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4502,11 +4447,11 @@ void report_add_sta_event(struct adapter *padapter, unsigned char *MacAddr, int
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct stassoc_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_AddSTA);
+	pc2h_evt_hdr->ID = _AddSTA_EVT_;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	padd_sta_evt = (struct stassoc_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
-	memcpy((unsigned char *)(&(padd_sta_evt->macaddr)), MacAddr, ETH_ALEN);
+	ether_addr_copy((unsigned char *)(&(padd_sta_evt->macaddr)), MacAddr);
 	padd_sta_evt->cam_id = cam_idx;
 
 	DBG_88E("report_add_sta_event: add STA\n");
@@ -4537,7 +4482,7 @@ void update_sta_info(struct adapter *padapter, struct sta_info *psta)
 
 		psta->htpriv.ampdu_enable = pmlmepriv->htpriv.ampdu_enable;
 
-		if (support_short_GI(padapter, &(pmlmeinfo->HT_caps)))
+		if (support_short_GI(padapter, &pmlmeinfo->HT_caps))
 			psta->htpriv.sgi = true;
 
 		psta->qos_option = true;
@@ -4904,7 +4849,7 @@ void survey_timer_hdl(unsigned long data)
 			goto exit_survey_timer_hdl;
 		}
 
-		init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara, GEN_CMD_CODE(_SiteSurvey));
+		init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara, _SiteSurvey_CMD_);
 		rtw_enqueue_cmd(pcmdpriv, ph2c);
 	}
 
@@ -5480,7 +5425,7 @@ u8 set_tx_beacon_cmd(struct adapter *padapter)
 				      pmlmeinfo->hidden_ssid_mode);
 	ptxBeacon_parm->IELength += len_diff;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, ptxBeacon_parm, GEN_CMD_CODE(_TX_Beacon));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, ptxBeacon_parm, _TX_Beacon_CMD_);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 
diff --git a/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c b/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c
index 59c6d8ab60f6..0b70fe7d3b72 100644
--- a/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c
+++ b/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c
@@ -38,7 +38,7 @@ static int rtw_hw_suspend(struct adapter *padapter)
 	LeaveAllPowerSaveMode(padapter);
 
 	DBG_88E("==> rtw_hw_suspend\n");
-	_enter_pwrlock(&pwrpriv->lock);
+	mutex_lock(&pwrpriv->mutex_lock);
 	pwrpriv->bips_processing = true;
 	/* s1. */
 	if (pnetdev) {
@@ -73,7 +73,7 @@ static int rtw_hw_suspend(struct adapter *padapter)
 	pwrpriv->rf_pwrstate = rf_off;
 	pwrpriv->bips_processing = false;
 
-	_exit_pwrlock(&pwrpriv->lock);
+	mutex_unlock(&pwrpriv->mutex_lock);
 
 	return 0;
 
@@ -90,12 +90,12 @@ static int rtw_hw_resume(struct adapter *padapter)
 
 	/* system resume */
 	DBG_88E("==> rtw_hw_resume\n");
-	_enter_pwrlock(&pwrpriv->lock);
+	mutex_lock(&pwrpriv->mutex_lock);
 	pwrpriv->bips_processing = true;
 	rtw_reset_drv_sw(padapter);
 
 	if (pm_netdev_open(pnetdev, false) != 0) {
-		_exit_pwrlock(&pwrpriv->lock);
+		mutex_unlock(&pwrpriv->mutex_lock);
 		goto error_exit;
 	}
 
@@ -113,7 +113,7 @@ static int rtw_hw_resume(struct adapter *padapter)
 	pwrpriv->rf_pwrstate = rf_on;
 	pwrpriv->bips_processing = false;
 
-	_exit_pwrlock(&pwrpriv->lock);
+	mutex_unlock(&pwrpriv->mutex_lock);
 
 
 	return 0;
@@ -138,7 +138,7 @@ void ips_enter(struct adapter *padapter)
 		return;
 	}
 
-	_enter_pwrlock(&pwrpriv->lock);
+	mutex_lock(&pwrpriv->mutex_lock);
 
 	pwrpriv->bips_processing = true;
 
@@ -159,7 +159,7 @@ void ips_enter(struct adapter *padapter)
 	}
 	pwrpriv->bips_processing = false;
 
-	_exit_pwrlock(&pwrpriv->lock);
+	mutex_unlock(&pwrpriv->mutex_lock);
 }
 
 int ips_leave(struct adapter *padapter)
@@ -171,7 +171,7 @@ int ips_leave(struct adapter *padapter)
 	int keyid;
 
 
-	_enter_pwrlock(&pwrpriv->lock);
+	mutex_lock(&pwrpriv->mutex_lock);
 
 	if ((pwrpriv->rf_pwrstate == rf_off) && (!pwrpriv->bips_processing)) {
 		pwrpriv->bips_processing = true;
@@ -205,7 +205,7 @@ int ips_leave(struct adapter *padapter)
 		pwrpriv->bpower_saving = false;
 	}
 
-	_exit_pwrlock(&pwrpriv->lock);
+	mutex_unlock(&pwrpriv->mutex_lock);
 
 	return result;
 }
@@ -504,7 +504,7 @@ void rtw_init_pwrctrl_priv(struct adapter *padapter)
 {
 	struct pwrctrl_priv *pwrctrlpriv = &padapter->pwrctrlpriv;
 
-	_init_pwrlock(&pwrctrlpriv->lock);
+	mutex_init(&pwrctrlpriv->mutex_lock);
 	pwrctrlpriv->rf_pwrstate = rf_on;
 	pwrctrlpriv->ips_enter_cnts = 0;
 	pwrctrlpriv->ips_leave_cnts = 0;
diff --git a/drivers/staging/rtl8188eu/core/rtw_recv.c b/drivers/staging/rtl8188eu/core/rtw_recv.c
index 977bb2532c3e..b87cbbbee054 100644
--- a/drivers/staging/rtl8188eu/core/rtw_recv.c
+++ b/drivers/staging/rtl8188eu/core/rtw_recv.c
@@ -39,7 +39,7 @@ static u8 rtw_rfc1042_header[] = {
        0xaa, 0xaa, 0x03, 0x00, 0x00, 0x00
 };
 
-void rtw_signal_stat_timer_hdl(unsigned long data);
+static void rtw_signal_stat_timer_hdl(unsigned long data);
 
 void _rtw_init_sta_recv_priv(struct sta_recv_priv *psta_recvpriv)
 {
@@ -73,7 +73,7 @@ int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *padapter)
 	if (!precvpriv->pallocated_frame_buf)
 		return _FAIL;
 
-	precvpriv->precv_frame_buf = (u8 *)N_BYTE_ALIGMENT((size_t)(precvpriv->pallocated_frame_buf), RXFRAME_ALIGN_SZ);
+	precvpriv->precv_frame_buf = PTR_ALIGN(precvpriv->pallocated_frame_buf, RXFRAME_ALIGN_SZ);
 
 	precvframe = (struct recv_frame *)precvpriv->precv_frame_buf;
 
@@ -2088,7 +2088,7 @@ _recv_entry_drop:
 	return ret;
 }
 
-void rtw_signal_stat_timer_hdl(unsigned long data)
+static void rtw_signal_stat_timer_hdl(unsigned long data)
 {
 	struct adapter *adapter = (struct adapter *)data;
 	struct recv_priv *recvpriv = &adapter->recvpriv;
diff --git a/drivers/staging/rtl8188eu/core/rtw_rf.c b/drivers/staging/rtl8188eu/core/rtw_rf.c
index 3fc1a8fd367c..e47be87fb402 100644
--- a/drivers/staging/rtl8188eu/core/rtw_rf.c
+++ b/drivers/staging/rtl8188eu/core/rtw_rf.c
@@ -19,7 +19,6 @@
 #include <recv_osdep.h>
 #include <xmit_osdep.h>
 
-
 struct ch_freq {
 	u32 channel;
 	u32 frequency;
diff --git a/drivers/staging/rtl8188eu/core/rtw_security.c b/drivers/staging/rtl8188eu/core/rtw_security.c
index 442a614a3726..85bb441a7214 100644
--- a/drivers/staging/rtl8188eu/core/rtw_security.c
+++ b/drivers/staging/rtl8188eu/core/rtw_security.c
@@ -233,7 +233,6 @@ void rtw_wep_decrypt(struct adapter  *padapter, u8 *precvframe)
 				 &crc, &payload[length-4]));
 		}
 	}
-	return;
 }
 
 /* 3		===== TKIP related ===== */
diff --git a/drivers/staging/rtl8188eu/core/rtw_sreset.c b/drivers/staging/rtl8188eu/core/rtw_sreset.c
index 13a5bf4730ab..a198c5779d50 100644
--- a/drivers/staging/rtl8188eu/core/rtw_sreset.c
+++ b/drivers/staging/rtl8188eu/core/rtw_sreset.c
@@ -16,18 +16,16 @@
 #include <rtw_sreset.h>
 #include <usb_ops_linux.h>
 
-void sreset_init_value(struct adapter *padapter)
+void rtw_hal_sreset_init(struct adapter *padapter)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(padapter);
-	struct sreset_priv *psrtpriv = &pHalData->srestpriv;
+	struct sreset_priv *psrtpriv = &padapter->HalData->srestpriv;
 
 	psrtpriv->Wifi_Error_Status = WIFI_STATUS_SUCCESS;
 }
 
 u8 sreset_get_wifi_status(struct adapter *padapter)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(padapter);
-	struct sreset_priv *psrtpriv = &pHalData->srestpriv;
+	struct sreset_priv *psrtpriv = &padapter->HalData->srestpriv;
 
 	u8 status = WIFI_STATUS_SUCCESS;
 	u32 val32 = 0;
@@ -54,6 +52,5 @@ u8 sreset_get_wifi_status(struct adapter *padapter)
 
 void sreset_set_wifi_error_status(struct adapter *padapter, u32 status)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(padapter);
-	pHalData->srestpriv.Wifi_Error_Status = status;
+	padapter->HalData->srestpriv.Wifi_Error_Status = status;
 }
diff --git a/drivers/staging/rtl8188eu/core/rtw_wlan_util.c b/drivers/staging/rtl8188eu/core/rtw_wlan_util.c
index 4410fe8d7c68..2a65ac702129 100644
--- a/drivers/staging/rtl8188eu/core/rtw_wlan_util.c
+++ b/drivers/staging/rtl8188eu/core/rtw_wlan_util.c
@@ -341,9 +341,6 @@ void set_channel_bwmode(struct adapter *padapter, unsigned char channel, unsigne
 {
 	u8 center_ch;
 
-	if (padapter->bNotifyChannelChange)
-		DBG_88E("[%s] ch = %d, offset = %d, bwmode = %d\n", __func__, channel, channel_offset, bwmode);
-
 	if ((bwmode == HT_CHANNEL_WIDTH_20) ||
 	    (channel_offset == HAL_PRIME_CHNL_OFFSET_DONT_CARE)) {
 		/* SelectChannel(padapter, channel); */
@@ -716,6 +713,7 @@ void HT_caps_handler(struct adapter *padapter, struct ndis_802_11_var_ie *pIE)
 	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
 	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
 	struct ht_priv			*phtpriv = &pmlmepriv->htpriv;
+	u8 *HT_cap = (u8 *)(&pmlmeinfo->HT_caps);
 
 	if (pIE == NULL)
 		return;
@@ -728,20 +726,20 @@ void HT_caps_handler(struct adapter *padapter, struct ndis_802_11_var_ie *pIE)
 	for (i = 0; i < (pIE->Length); i++) {
 		if (i != 2) {
 			/*	Got the endian issue here. */
-			pmlmeinfo->HT_caps.u.HT_cap[i] &= (pIE->data[i]);
+			HT_cap[i] &= (pIE->data[i]);
 		} else {
 			/* modify from  fw by Thomas 2010/11/17 */
-			if ((pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x3) > (pIE->data[i] & 0x3))
+			if ((pmlmeinfo->HT_caps.ampdu_params_info & 0x3) > (pIE->data[i] & 0x3))
 				max_AMPDU_len = pIE->data[i] & 0x3;
 			else
-				max_AMPDU_len = pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x3;
+				max_AMPDU_len = pmlmeinfo->HT_caps.ampdu_params_info & 0x3;
 
-			if ((pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x1c) > (pIE->data[i] & 0x1c))
-				min_MPDU_spacing = pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x1c;
+			if ((pmlmeinfo->HT_caps.ampdu_params_info & 0x1c) > (pIE->data[i] & 0x1c))
+				min_MPDU_spacing = pmlmeinfo->HT_caps.ampdu_params_info & 0x1c;
 			else
 				min_MPDU_spacing = pIE->data[i] & 0x1c;
 
-			pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para = max_AMPDU_len | min_MPDU_spacing;
+			pmlmeinfo->HT_caps.ampdu_params_info = max_AMPDU_len | min_MPDU_spacing;
 		}
 	}
 
@@ -750,9 +748,9 @@ void HT_caps_handler(struct adapter *padapter, struct ndis_802_11_var_ie *pIE)
 	/* update the MCS rates */
 	for (i = 0; i < 16; i++) {
 		if ((rf_type == RF_1T1R) || (rf_type == RF_1T2R))
-			pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate[i] &= MCS_rate_1R[i];
+			((u8 *)&pmlmeinfo->HT_caps.mcs)[i] &= MCS_rate_1R[i];
 		else
-			pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate[i] &= MCS_rate_2R[i];
+			((u8 *)&pmlmeinfo->HT_caps.mcs)[i] &= MCS_rate_2R[i];
 	}
 }
 
@@ -798,9 +796,9 @@ void HTOnAssocRsp(struct adapter *padapter)
 		AMPDU_para [1:0]:Max AMPDU Len => 0:8k , 1:16k, 2:32k, 3:64k
 		AMPDU_para [4:2]:Min MPDU Start Spacing
 	*/
-	max_AMPDU_len = pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x03;
+	max_AMPDU_len = pmlmeinfo->HT_caps.ampdu_params_info & 0x03;
 
-	min_MPDU_spacing = (pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x1c) >> 2;
+	min_MPDU_spacing = (pmlmeinfo->HT_caps.ampdu_params_info & 0x1c) >> 2;
 
 	rtw_hal_set_hwreg(padapter, HW_VAR_AMPDU_MIN_SPACE, (u8 *)(&min_MPDU_spacing));
 
@@ -872,7 +870,6 @@ int rtw_check_bcn_info(struct adapter  *Adapter, u8 *pframe, u32 packet_len)
 	u32 wpa_ielen = 0;
 	u8 *pbssid = GetAddr3Ptr(pframe);
 	struct HT_info_element *pht_info = NULL;
-	struct rtw_ieee80211_ht_cap *pht_cap = NULL;
 	u32 bcn_channel;
 	unsigned short	ht_cap_info;
 	unsigned char	ht_info_infos_0;
@@ -881,7 +878,7 @@ int rtw_check_bcn_info(struct adapter  *Adapter, u8 *pframe, u32 packet_len)
 	if (is_client_associated_to_ap(Adapter) == false)
 		return true;
 
-	len = packet_len - sizeof(struct rtw_ieee80211_hdr_3addr);
+	len = packet_len - sizeof(struct ieee80211_hdr_3addr);
 
 	if (len > MAX_IE_SZ) {
 		DBG_88E("%s IE too long for survey event\n", __func__);
@@ -907,14 +904,16 @@ int rtw_check_bcn_info(struct adapter  *Adapter, u8 *pframe, u32 packet_len)
 
 	/* below is to copy the information element */
 	bssid->IELength = len;
-	memcpy(bssid->IEs, (pframe + sizeof(struct rtw_ieee80211_hdr_3addr)), bssid->IELength);
+	memcpy(bssid->IEs, (pframe + sizeof(struct ieee80211_hdr_3addr)), bssid->IELength);
 
 	/* check bw and channel offset */
 	/* parsing HT_CAP_IE */
 	p = rtw_get_ie(bssid->IEs + _FIXED_IE_LENGTH_, _HT_CAPABILITY_IE_, &len, bssid->IELength - _FIXED_IE_LENGTH_);
 	if (p && len > 0) {
-		pht_cap = (struct rtw_ieee80211_ht_cap *)(p + 2);
-		ht_cap_info = pht_cap->cap_info;
+		struct ieee80211_ht_cap *ht_cap =
+			(struct ieee80211_ht_cap *)(p + 2);
+
+		ht_cap_info = le16_to_cpu(ht_cap->cap_info);
 	} else {
 		ht_cap_info = 0;
 	}
@@ -1243,16 +1242,17 @@ unsigned int update_supported_rate(unsigned char *ptn, unsigned int ptn_sz)
 	return mask;
 }
 
-unsigned int update_MSC_rate(struct HT_caps_element *pHT_caps)
+unsigned int update_MSC_rate(struct ieee80211_ht_cap *pHT_caps)
 {
 	unsigned int mask = 0;
 
-	mask = (pHT_caps->u.HT_cap_element.MCS_rate[0] << 12) | (pHT_caps->u.HT_cap_element.MCS_rate[1] << 20);
+	mask = (pHT_caps->mcs.rx_mask[0] << 12) |
+	       (pHT_caps->mcs.rx_mask[1] << 20);
 
 	return mask;
 }
 
-int support_short_GI(struct adapter *padapter, struct HT_caps_element *pHT_caps)
+int support_short_GI(struct adapter *padapter, struct ieee80211_ht_cap *pHT_caps)
 {
 	unsigned char					bit_offset;
 	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
@@ -1266,7 +1266,7 @@ int support_short_GI(struct adapter *padapter, struct HT_caps_element *pHT_caps)
 
 	bit_offset = (pmlmeext->cur_bwmode & HT_CHANNEL_WIDTH_40) ? 6 : 5;
 
-	if (__le16_to_cpu(pHT_caps->u.HT_cap_element.HT_caps_info) & (0x1 << bit_offset))
+	if (__le16_to_cpu(pHT_caps->cap_info) & (0x1 << bit_offset))
 		return _SUCCESS;
 	else
 		return _FAIL;
@@ -1508,7 +1508,7 @@ void update_wireless_mode(struct adapter *padapter)
 	SIFS_Timer = 0x0a0a0808;/* 0x0808 -> for CCK, 0x0a0a -> for OFDM */
 				/* change this value if having IOT issues. */
 
-	padapter->HalFunc.SetHwRegHandler(padapter, HW_VAR_RESP_SIFS,  (u8 *)&SIFS_Timer);
+	rtw_hal_set_hwreg(padapter, HW_VAR_RESP_SIFS,  (u8 *)&SIFS_Timer);
 
 	if (pmlmeext->cur_wireless_mode & WIRELESS_11B)
 		update_mgnt_tx_rate(padapter, IEEE80211_CCK_RATE_1MB);
@@ -1584,7 +1584,7 @@ void update_TSF(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint len)
 	u8 *pIE;
 	__le32 *pbuf;
 
-	pIE = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	pIE = pframe + sizeof(struct ieee80211_hdr_3addr);
 	pbuf = (__le32 *)pIE;
 
 	pmlmeext->TSFValue = le32_to_cpu(*(pbuf+1));
diff --git a/drivers/staging/rtl8188eu/core/rtw_xmit.c b/drivers/staging/rtl8188eu/core/rtw_xmit.c
index e0a5567f5942..0f8b8e0bffdf 100644
--- a/drivers/staging/rtl8188eu/core/rtw_xmit.c
+++ b/drivers/staging/rtl8188eu/core/rtw_xmit.c
@@ -57,8 +57,6 @@ s32	_rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 	/*  We don't need to memset padapter->XXX to zero, because adapter is allocated by vzalloc(). */
 
 	spin_lock_init(&pxmitpriv->lock);
-	sema_init(&pxmitpriv->xmit_sema, 0);
-	sema_init(&pxmitpriv->terminate_xmitthread_sema, 0);
 
 	/*
 	Please insert all the queue initializaiton using _rtw_init_queue below
@@ -88,7 +86,7 @@ s32	_rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 		res = _FAIL;
 		goto exit;
 	}
-	pxmitpriv->pxmit_frame_buf = (u8 *)N_BYTE_ALIGMENT((size_t)(pxmitpriv->pallocated_frame_buf), 4);
+	pxmitpriv->pxmit_frame_buf = PTR_ALIGN(pxmitpriv->pallocated_frame_buf, 4);
 	/* pxmitpriv->pxmit_frame_buf = pxmitpriv->pallocated_frame_buf + 4 - */
 	/* 						((size_t) (pxmitpriv->pallocated_frame_buf) &3); */
 
@@ -126,7 +124,7 @@ s32	_rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 		goto exit;
 	}
 
-	pxmitpriv->pxmitbuf = (u8 *)N_BYTE_ALIGMENT((size_t)(pxmitpriv->pallocated_xmitbuf), 4);
+	pxmitpriv->pxmitbuf = PTR_ALIGN(pxmitpriv->pallocated_xmitbuf, 4);
 	/* pxmitpriv->pxmitbuf = pxmitpriv->pallocated_xmitbuf + 4 - */
 	/* 						((size_t) (pxmitpriv->pallocated_xmitbuf) &3); */
 
@@ -144,9 +142,8 @@ s32	_rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 		if (res == _FAIL) {
 			msleep(10);
 			res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ));
-			if (res == _FAIL) {
+			if (res == _FAIL)
 				goto exit;
-			}
 		}
 
 		pxmitbuf->flags = XMIT_VO_QUEUE;
@@ -168,7 +165,7 @@ s32	_rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 		goto exit;
 	}
 
-	pxmitpriv->pxmit_extbuf = (u8 *)N_BYTE_ALIGMENT((size_t)(pxmitpriv->pallocated_xmit_extbuf), 4);
+	pxmitpriv->pxmit_extbuf = PTR_ALIGN(pxmitpriv->pallocated_xmit_extbuf, 4);
 
 	pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmit_extbuf;
 
@@ -199,8 +196,6 @@ s32	_rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 
 	pxmitpriv->txirp_cnt = 1;
 
-	sema_init(&(pxmitpriv->tx_retevt), 0);
-
 	/* per AC pending irp */
 	pxmitpriv->beq_cnt = 0;
 	pxmitpriv->bkq_cnt = 0;
@@ -252,9 +247,8 @@ void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv)
 		pxmitbuf++;
 	}
 
-	if (pxmitpriv->pallocated_xmit_extbuf) {
+	if (pxmitpriv->pallocated_xmit_extbuf)
 		vfree(pxmitpriv->pallocated_xmit_extbuf);
-	}
 
 	rtw_free_hwxmits(padapter);
 
@@ -408,7 +402,7 @@ static void set_qos(struct pkt_file *ppktfile, struct pkt_attrib *pattrib)
 		_rtw_pktfile_read(ppktfile, (u8 *)&ip_hdr, sizeof(ip_hdr));
 /* 		user_prio = (ntohs(ip_hdr.tos) >> 5) & 0x3; */
 		user_prio = ip_hdr.tos >> 5;
-	} else if (pattrib->ether_type == 0x888e) {
+	} else if (pattrib->ether_type == ETH_P_PAE) {
 		/*  "When priority processing of data frames is supported, */
 		/*  a STA's SME should send EAPOL-Key frames at the highest priority." */
 		user_prio = 7;
@@ -457,14 +451,14 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 
 	pattrib->pktlen = pktfile.pkt_len;
 
-	if (ETH_P_IP == pattrib->ether_type) {
+	if (pattrib->ether_type == ETH_P_IP) {
 		/*  The following is for DHCP and ARP packet, we use cck1M to tx these packets and let LPS awake some time */
 		/*  to prevent DHCP protocol fail */
 		u8 tmp[24];
 		_rtw_pktfile_read(&pktfile, &tmp[0], 24);
 		pattrib->dhcp_pkt = 0;
 		if (pktfile.pkt_len > 282) {/* MINIMUM_DHCP_PACKET_SIZE) { */
-			if (ETH_P_IP == pattrib->ether_type) {/*  IP header */
+			if (pattrib->ether_type == ETH_P_IP) {/*  IP header */
 				if (((tmp[21] == 68) && (tmp[23] == 67)) ||
 				    ((tmp[21] == 67) && (tmp[23] == 68))) {
 					/*  68 : UDP BOOTP client */
@@ -475,15 +469,15 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 				}
 			}
 		}
-	} else if (0x888e == pattrib->ether_type) {
+	} else if (pattrib->ether_type == ETH_P_PAE) {
 		DBG_88E_LEVEL(_drv_info_, "send eapol packet\n");
 	}
 
-	if ((pattrib->ether_type == 0x888e) || (pattrib->dhcp_pkt == 1))
+	if ((pattrib->ether_type == ETH_P_PAE) || (pattrib->dhcp_pkt == 1))
 		rtw_set_scan_deny(padapter, 3000);
 
 	/*  If EAPOL , ARP , OR DHCP packet, driver must be in active mode. */
-	if ((pattrib->ether_type == 0x0806) || (pattrib->ether_type == 0x888e) || (pattrib->dhcp_pkt == 1))
+	if ((pattrib->ether_type == ETH_P_ARP) || (pattrib->ether_type == ETH_P_PAE) || (pattrib->dhcp_pkt == 1))
 		rtw_lps_ctrl_wk_cmd(padapter, LPS_CTRL_SPECIAL_PACKET, 1);
 
 	bmcast = IS_MCAST(pattrib->ra);
@@ -515,8 +509,6 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 	}
 
 	pattrib->ack_policy = 0;
-	/*  get ether_hdr_len */
-	pattrib->pkt_hdrlen = ETH_HLEN;/* pattrib->ether_type == 0x8100) ? (14 + 4): 14; vlan tag */
 
 	pattrib->hdrlen = WLAN_HDR_A3_LEN;
 	pattrib->subtype = WIFI_DATA_TYPE;
@@ -539,8 +531,8 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 
 		pattrib->encrypt = 0;
 
-		if ((pattrib->ether_type != 0x888e) && !check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_, ("\npsta->ieee8021x_blocked == true,  pattrib->ether_type(%.4x) != 0x888e\n", pattrib->ether_type));
+		if ((pattrib->ether_type != ETH_P_PAE) && !check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
+			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_, ("\npsta->ieee8021x_blocked == true,  pattrib->ether_type(%.4x) != ETH_P_PAE\n", pattrib->ether_type));
 			res = _FAIL;
 			goto exit;
 		}
@@ -769,13 +761,13 @@ s32 rtw_make_wlanhdr(struct adapter *padapter, u8 *hdr, struct pkt_attrib *pattr
 {
 	u16 *qc;
 
-	struct rtw_ieee80211_hdr *pwlanhdr = (struct rtw_ieee80211_hdr *)hdr;
+	struct ieee80211_hdr *pwlanhdr = (struct ieee80211_hdr *)hdr;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct qos_priv *pqospriv = &pmlmepriv->qospriv;
 	u8 qos_option = false;
 
 	int res = _SUCCESS;
-	__le16 *fctrl = &pwlanhdr->frame_ctl;
+	__le16 *fctrl = &pwlanhdr->frame_control;
 
 	struct sta_info *psta;
 
@@ -785,11 +777,10 @@ s32 rtw_make_wlanhdr(struct adapter *padapter, u8 *hdr, struct pkt_attrib *pattr
 	if (pattrib->psta) {
 		psta = pattrib->psta;
 	} else {
-		if (bmcst) {
+		if (bmcst)
 			psta = rtw_get_bcmc_stainfo(padapter);
-		} else {
+		else
 			psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
-		}
 	}
 
 	memset(hdr, 0, WLANHDR_OFFSET);
@@ -999,7 +990,7 @@ s32 rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct
 	}
 
 	_rtw_open_pktfile(pkt, &pktfile);
-	_rtw_pktfile_read(&pktfile, NULL, pattrib->pkt_hdrlen);
+	_rtw_pktfile_read(&pktfile, NULL, ETH_HLEN);
 
 	frg_inx = 0;
 	frg_len = pxmitpriv->frag_len - 4;/* 2346-4 = 2342 */
@@ -1054,9 +1045,8 @@ s32 rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct
 			mpdu_len -= llc_sz;
 		}
 
-		if ((pattrib->icv_len > 0) && (pattrib->bswenc)) {
+		if ((pattrib->icv_len > 0) && (pattrib->bswenc))
 			mpdu_len -= pattrib->icv_len;
-		}
 
 		if (bmcst) {
 			/*  don't do fragment to broadcat/multicast packets */
@@ -1560,11 +1550,10 @@ s32 rtw_xmit_classifier(struct adapter *padapter, struct xmit_frame *pxmitframe)
 	int res = _SUCCESS;
 
 
-	if (pattrib->psta) {
+	if (pattrib->psta)
 		psta = pattrib->psta;
-	} else {
+	else
 		psta = rtw_get_stainfo(pstapriv, pattrib->ra);
-	}
 
 	if (psta == NULL) {
 		res = _FAIL;
@@ -1658,16 +1647,6 @@ u32 rtw_get_ff_hwaddr(struct xmit_frame *pxmitframe)
 	return addr;
 }
 
-static void do_queue_select(struct adapter *padapter, struct pkt_attrib *pattrib)
-{
-	u8 qsel;
-
-	qsel = pattrib->priority;
-	RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_, ("### do_queue_select priority=%d , qsel = %d\n", pattrib->priority, qsel));
-
-	pattrib->qsel = qsel;
-}
-
 /*
  * The main transmit(tx) entry
  *
@@ -1700,7 +1679,7 @@ s32 rtw_xmit(struct adapter *padapter, struct sk_buff **ppkt)
 
 	rtw_led_control(padapter, LED_CTL_TX);
 
-	do_queue_select(padapter, &pxmitframe->attrib);
+	pxmitframe->attrib.qsel = pxmitframe->attrib.priority;
 
 #ifdef CONFIG_88EU_AP_MODE
 	spin_lock_bh(&pxmitpriv->lock);
diff --git a/drivers/staging/rtl8188eu/hal/bb_cfg.c b/drivers/staging/rtl8188eu/hal/bb_cfg.c
index cce1ea259b76..134fa6c595a8 100644
--- a/drivers/staging/rtl8188eu/hal/bb_cfg.c
+++ b/drivers/staging/rtl8188eu/hal/bb_cfg.c
@@ -498,7 +498,7 @@ static u32 array_phy_reg_pg_8188e[] = {
 static void store_pwrindex_offset(struct adapter *adapter,
 				  u32 regaddr, u32 bitmask, u32 data)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapter);
+	struct hal_data_8188e *hal_data = adapter->HalData;
 	u32 * const power_level_offset =
 		hal_data->MCSTxPowerLevelOriginalOffset[hal_data->pwrGroupCnt];
 
@@ -518,8 +518,7 @@ static void store_pwrindex_offset(struct adapter *adapter,
 		power_level_offset[4] = data;
 	if (regaddr == rTxAGC_A_Mcs15_Mcs12) {
 		power_level_offset[5] = data;
-		if (hal_data->rf_type == RF_1T1R)
-			hal_data->pwrGroupCnt++;
+		hal_data->pwrGroupCnt++;
 	}
 	if (regaddr == rTxAGC_B_Rate18_06)
 		power_level_offset[8] = data;
@@ -537,8 +536,6 @@ static void store_pwrindex_offset(struct adapter *adapter,
 		power_level_offset[12] = data;
 	if (regaddr == rTxAGC_B_Mcs15_Mcs12) {
 		power_level_offset[13] = data;
-		if (hal_data->rf_type != RF_1T1R)
-			hal_data->pwrGroupCnt++;
 	}
 }
 
@@ -588,11 +585,10 @@ static bool config_bb_with_pgheader(struct adapter *adapt)
 
 static void rtl88e_phy_init_bb_rf_register_definition(struct adapter *adapter)
 {
-	struct hal_data_8188e		*hal_data = GET_HAL_DATA(adapter);
 	struct bb_reg_def               *reg[4];
 
-	reg[RF_PATH_A] = &hal_data->PHYRegDef[RF_PATH_A];
-	reg[RF_PATH_B] = &hal_data->PHYRegDef[RF_PATH_B];
+	reg[RF_PATH_A] = &adapter->HalData->PHYRegDef[RF_PATH_A];
+	reg[RF_PATH_B] = &adapter->HalData->PHYRegDef[RF_PATH_B];
 
 	reg[RF_PATH_A]->rfintfs = rFPGA0_XAB_RFInterfaceSW;
 	reg[RF_PATH_B]->rfintfs = rFPGA0_XAB_RFInterfaceSW;
@@ -652,13 +648,12 @@ static void rtl88e_phy_init_bb_rf_register_definition(struct adapter *adapter)
 static bool config_parafile(struct adapter *adapt)
 {
 	struct eeprom_priv *eeprom = GET_EEPROM_EFUSE_PRIV(adapt);
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
 
 	set_baseband_phy_config(adapt);
 
 	/* If EEPROM or EFUSE autoload OK, We must config by PHY_REG_PG.txt */
 	if (!eeprom->bautoload_fail_flag) {
-		hal_data->pwrGroupCnt = 0;
+		adapt->HalData->pwrGroupCnt = 0;
 		config_bb_with_pgheader(adapt);
 	}
 	set_baseband_agc_config(adapt);
@@ -668,7 +663,6 @@ static bool config_parafile(struct adapter *adapt)
 bool rtl88eu_phy_bb_config(struct adapter *adapt)
 {
 	int rtstatus = true;
-	struct hal_data_8188e	*hal_data = GET_HAL_DATA(adapt);
 	u32 regval;
 	u8 crystal_cap;
 
@@ -688,7 +682,7 @@ bool rtl88eu_phy_bb_config(struct adapter *adapt)
 	rtstatus = config_parafile(adapt);
 
 	/*  write 0x24[16:11] = 0x24[22:17] = crystal_cap */
-	crystal_cap = hal_data->CrystalCap & 0x3F;
+	crystal_cap = adapt->HalData->CrystalCap & 0x3F;
 	phy_set_bb_reg(adapt, REG_AFE_XTAL_CTRL, 0x7ff800,
 		       (crystal_cap | (crystal_cap << 6)));
 
diff --git a/drivers/staging/rtl8188eu/hal/hal_intf.c b/drivers/staging/rtl8188eu/hal/hal_intf.c
index 085f0fbd0c43..a11c7b4254f6 100644
--- a/drivers/staging/rtl8188eu/hal/hal_intf.c
+++ b/drivers/staging/rtl8188eu/hal/hal_intf.c
@@ -17,62 +17,6 @@
 #include <osdep_service.h>
 #include <drv_types.h>
 #include <hal_intf.h>
-#include <usb_hal.h>
-
-void rtw_hal_chip_configure(struct adapter *adapt)
-{
-	if (adapt->HalFunc.intf_chip_configure)
-		adapt->HalFunc.intf_chip_configure(adapt);
-}
-
-void rtw_hal_read_chip_info(struct adapter *adapt)
-{
-	if (adapt->HalFunc.read_adapter_info)
-		adapt->HalFunc.read_adapter_info(adapt);
-}
-
-void rtw_hal_read_chip_version(struct adapter *adapt)
-{
-	if (adapt->HalFunc.read_chip_version)
-		adapt->HalFunc.read_chip_version(adapt);
-}
-
-void rtw_hal_def_value_init(struct adapter *adapt)
-{
-	if (adapt->HalFunc.init_default_value)
-		adapt->HalFunc.init_default_value(adapt);
-}
-
-void rtw_hal_free_data(struct adapter *adapt)
-{
-	if (adapt->HalFunc.free_hal_data)
-		adapt->HalFunc.free_hal_data(adapt);
-}
-
-void rtw_hal_dm_init(struct adapter *adapt)
-{
-	if (adapt->HalFunc.dm_init)
-		adapt->HalFunc.dm_init(adapt);
-}
-
-void rtw_hal_sw_led_init(struct adapter *adapt)
-{
-	if (adapt->HalFunc.InitSwLeds)
-		adapt->HalFunc.InitSwLeds(adapt);
-}
-
-void rtw_hal_sw_led_deinit(struct adapter *adapt)
-{
-	if (adapt->HalFunc.DeInitSwLeds)
-		adapt->HalFunc.DeInitSwLeds(adapt);
-}
-
-u32 rtw_hal_power_on(struct adapter *adapt)
-{
-	if (adapt->HalFunc.hal_power_on)
-		return adapt->HalFunc.hal_power_on(adapt);
-	return _FAIL;
-}
 
 uint	 rtw_hal_init(struct adapter *adapt)
 {
@@ -80,15 +24,13 @@ uint	 rtw_hal_init(struct adapter *adapt)
 
 	adapt->hw_init_completed = false;
 
-	status = adapt->HalFunc.hal_init(adapt);
+	status = rtl8188eu_hal_init(adapt);
 
 	if (status == _SUCCESS) {
 		adapt->hw_init_completed = true;
 
 		if (adapt->registrypriv.notch_filter == 1)
 			rtw_hal_notch_filter(adapt, 1);
-
-		rtw_hal_reset_security_engine(adapt);
 	} else {
 		adapt->hw_init_completed = false;
 		DBG_88E("rtw_hal_init: hal__init fail\n");
@@ -104,7 +46,7 @@ uint rtw_hal_deinit(struct adapter *adapt)
 {
 	uint	status = _SUCCESS;
 
-	status = adapt->HalFunc.hal_deinit(adapt);
+	status = rtl8188eu_hal_deinit(adapt);
 
 	if (status == _SUCCESS)
 		adapt->hw_init_completed = false;
@@ -114,92 +56,6 @@ uint rtw_hal_deinit(struct adapter *adapt)
 	return status;
 }
 
-void rtw_hal_set_hwreg(struct adapter *adapt, u8 variable, u8 *val)
-{
-	if (adapt->HalFunc.SetHwRegHandler)
-		adapt->HalFunc.SetHwRegHandler(adapt, variable, val);
-}
-
-void rtw_hal_get_hwreg(struct adapter *adapt, u8 variable, u8 *val)
-{
-	if (adapt->HalFunc.GetHwRegHandler)
-		adapt->HalFunc.GetHwRegHandler(adapt, variable, val);
-}
-
-u8 rtw_hal_get_def_var(struct adapter *adapt,
-		       enum hal_def_variable var, void *val)
-{
-	if (adapt->HalFunc.GetHalDefVarHandler)
-		return adapt->HalFunc.GetHalDefVarHandler(adapt, var, val);
-	return _FAIL;
-}
-
-void rtw_hal_set_odm_var(struct adapter *adapt,
-			 enum hal_odm_variable var, void *val1,
-			 bool set)
-{
-	if (adapt->HalFunc.SetHalODMVarHandler)
-		adapt->HalFunc.SetHalODMVarHandler(adapt, var,
-						      val1, set);
-}
-
-u32 rtw_hal_inirp_init(struct adapter *adapt)
-{
-	u32 rst = _FAIL;
-
-	if (adapt->HalFunc.inirp_init)
-		rst = adapt->HalFunc.inirp_init(adapt);
-	else
-		DBG_88E(" %s HalFunc.inirp_init is NULL!!!\n", __func__);
-	return rst;
-}
-
-u32 rtw_hal_inirp_deinit(struct adapter *adapt)
-{
-	if (adapt->HalFunc.inirp_deinit)
-		return adapt->HalFunc.inirp_deinit(adapt);
-
-	return _FAIL;
-}
-
-s32 rtw_hal_xmit(struct adapter *adapt, struct xmit_frame *pxmitframe)
-{
-	if (adapt->HalFunc.hal_xmit)
-		return adapt->HalFunc.hal_xmit(adapt, pxmitframe);
-
-	return false;
-}
-
-s32 rtw_hal_mgnt_xmit(struct adapter *adapt, struct xmit_frame *pmgntframe)
-{
-	s32 ret = _FAIL;
-
-	if (adapt->HalFunc.mgnt_xmit)
-		ret = adapt->HalFunc.mgnt_xmit(adapt, pmgntframe);
-	return ret;
-}
-
-s32 rtw_hal_init_xmit_priv(struct adapter *adapt)
-{
-	if (adapt->HalFunc.init_xmit_priv)
-		return adapt->HalFunc.init_xmit_priv(adapt);
-	return _FAIL;
-}
-
-s32 rtw_hal_init_recv_priv(struct adapter *adapt)
-{
-	if (adapt->HalFunc.init_recv_priv)
-		return adapt->HalFunc.init_recv_priv(adapt);
-
-	return _FAIL;
-}
-
-void rtw_hal_free_recv_priv(struct adapter *adapt)
-{
-	if (adapt->HalFunc.free_recv_priv)
-		adapt->HalFunc.free_recv_priv(adapt);
-}
-
 void rtw_hal_update_ra_mask(struct adapter *adapt, u32 mac_id, u8 rssi_level)
 {
 	struct mlme_priv *pmlmepriv = &(adapt->mlmepriv);
@@ -215,86 +71,6 @@ void rtw_hal_update_ra_mask(struct adapter *adapt, u32 mac_id, u8 rssi_level)
 			add_RATid(adapt, psta, 0);/* todo: based on rssi_level*/
 #endif
 	} else {
-		if (adapt->HalFunc.UpdateRAMaskHandler)
-			adapt->HalFunc.UpdateRAMaskHandler(adapt, mac_id,
-							      rssi_level);
+		UpdateHalRAMask8188EUsb(adapt, mac_id, rssi_level);
 	}
 }
-
-void rtw_hal_add_ra_tid(struct adapter *adapt, u32 bitmap, u8 arg,
-			u8 rssi_level)
-{
-	if (adapt->HalFunc.Add_RateATid)
-		adapt->HalFunc.Add_RateATid(adapt, bitmap, arg,
-					       rssi_level);
-}
-
-u32 rtw_hal_read_rfreg(struct adapter *adapt, enum rf_radio_path rfpath,
-		       u32 regaddr, u32 bitmask)
-{
-	u32 data = 0;
-
-	if (adapt->HalFunc.read_rfreg)
-		data = adapt->HalFunc.read_rfreg(adapt, rfpath, regaddr,
-						    bitmask);
-	return data;
-}
-
-void rtw_hal_set_bwmode(struct adapter *adapt,
-			enum ht_channel_width bandwidth, u8 offset)
-{
-	if (adapt->HalFunc.set_bwmode_handler)
-		adapt->HalFunc.set_bwmode_handler(adapt, bandwidth,
-						     offset);
-}
-
-void rtw_hal_set_chan(struct adapter *adapt, u8 channel)
-{
-	if (adapt->HalFunc.set_channel_handler)
-		adapt->HalFunc.set_channel_handler(adapt, channel);
-}
-
-void rtw_hal_dm_watchdog(struct adapter *adapt)
-{
-	if (adapt->HalFunc.hal_dm_watchdog)
-		adapt->HalFunc.hal_dm_watchdog(adapt);
-}
-
-void rtw_hal_bcn_related_reg_setting(struct adapter *adapt)
-{
-	if (adapt->HalFunc.SetBeaconRelatedRegistersHandler)
-		adapt->HalFunc.SetBeaconRelatedRegistersHandler(adapt);
-}
-
-u8 rtw_hal_antdiv_before_linked(struct adapter *adapt)
-{
-	if (adapt->HalFunc.AntDivBeforeLinkHandler)
-		return adapt->HalFunc.AntDivBeforeLinkHandler(adapt);
-	return false;
-}
-
-void rtw_hal_antdiv_rssi_compared(struct adapter *adapt,
-				  struct wlan_bssid_ex *dst,
-				  struct wlan_bssid_ex *src)
-{
-	if (adapt->HalFunc.AntDivCompareHandler)
-		adapt->HalFunc.AntDivCompareHandler(adapt, dst, src);
-}
-
-void rtw_hal_sreset_init(struct adapter *adapt)
-{
-	if (adapt->HalFunc.sreset_init_value)
-		adapt->HalFunc.sreset_init_value(adapt);
-}
-
-void rtw_hal_notch_filter(struct adapter *adapter, bool enable)
-{
-	if (adapter->HalFunc.hal_notch_filter)
-		adapter->HalFunc.hal_notch_filter(adapter, enable);
-}
-
-void rtw_hal_reset_security_engine(struct adapter *adapter)
-{
-	if (adapter->HalFunc.hal_reset_security_engine)
-		adapter->HalFunc.hal_reset_security_engine(adapter);
-}
diff --git a/drivers/staging/rtl8188eu/hal/odm.c b/drivers/staging/rtl8188eu/hal/odm.c
index 57a127501694..d983a8029f4c 100644
--- a/drivers/staging/rtl8188eu/hal/odm.c
+++ b/drivers/staging/rtl8188eu/hal/odm.c
@@ -226,150 +226,6 @@ void ODM_DMWatchdog(struct odm_dm_struct *pDM_Odm)
 	odm_EdcaTurboCheck(pDM_Odm);
 }
 
-/*  Init /.. Fixed HW value. Only init time. */
-void ODM_CmnInfoInit(struct odm_dm_struct *pDM_Odm, enum odm_common_info_def CmnInfo, u32 Value)
-{
-	/*  This section is used for init value */
-	switch	(CmnInfo) {
-	/*  Fixed ODM value. */
-	case	ODM_CMNINFO_ABILITY:
-		pDM_Odm->SupportAbility = (u32)Value;
-		break;
-	case	ODM_CMNINFO_PLATFORM:
-		pDM_Odm->SupportPlatform = (u8)Value;
-		break;
-	case	ODM_CMNINFO_INTERFACE:
-		pDM_Odm->SupportInterface = (u8)Value;
-		break;
-	case	ODM_CMNINFO_MP_TEST_CHIP:
-		pDM_Odm->bIsMPChip = (u8)Value;
-		break;
-	case	ODM_CMNINFO_IC_TYPE:
-		pDM_Odm->SupportICType = Value;
-		break;
-	case	ODM_CMNINFO_CUT_VER:
-		pDM_Odm->CutVersion = (u8)Value;
-		break;
-	case	ODM_CMNINFO_RF_TYPE:
-		pDM_Odm->RFType = (u8)Value;
-		break;
-	case    ODM_CMNINFO_RF_ANTENNA_TYPE:
-		pDM_Odm->AntDivType = (u8)Value;
-		break;
-	case	ODM_CMNINFO_BOARD_TYPE:
-		pDM_Odm->BoardType = (u8)Value;
-		break;
-	case	ODM_CMNINFO_EXT_LNA:
-		pDM_Odm->ExtLNA = (u8)Value;
-		break;
-	case	ODM_CMNINFO_EXT_PA:
-		pDM_Odm->ExtPA = (u8)Value;
-		break;
-	case	ODM_CMNINFO_EXT_TRSW:
-		pDM_Odm->ExtTRSW = (u8)Value;
-		break;
-	case	ODM_CMNINFO_PATCH_ID:
-		pDM_Odm->PatchID = (u8)Value;
-		break;
-	case	ODM_CMNINFO_BINHCT_TEST:
-		pDM_Odm->bInHctTest = (bool)Value;
-		break;
-	case	ODM_CMNINFO_BWIFI_TEST:
-		pDM_Odm->bWIFITest = (bool)Value;
-		break;
-	case	ODM_CMNINFO_SMART_CONCURRENT:
-		pDM_Odm->bDualMacSmartConcurrent = (bool)Value;
-		break;
-	/* To remove the compiler warning, must add an empty default statement to handle the other values. */
-	default:
-		/* do nothing */
-		break;
-	}
-
-	/*  Tx power tracking BB swing table. */
-	/*  The base index = 12. +((12-n)/2)dB 13~?? = decrease tx pwr by -((n-12)/2)dB */
-	pDM_Odm->BbSwingIdxOfdm			= 12; /*  Set defalut value as index 12. */
-	pDM_Odm->BbSwingIdxOfdmCurrent	= 12;
-	pDM_Odm->BbSwingFlagOfdm		= false;
-}
-
-void ODM_CmnInfoHook(struct odm_dm_struct *pDM_Odm, enum odm_common_info_def CmnInfo, void *pValue)
-{
-	/*  */
-	/*  Hook call by reference pointer. */
-	/*  */
-	switch	(CmnInfo) {
-	/*  Dynamic call by reference pointer. */
-	case	ODM_CMNINFO_MAC_PHY_MODE:
-		pDM_Odm->pMacPhyMode = (u8 *)pValue;
-		break;
-	case	ODM_CMNINFO_TX_UNI:
-		pDM_Odm->pNumTxBytesUnicast = (u64 *)pValue;
-		break;
-	case	ODM_CMNINFO_RX_UNI:
-		pDM_Odm->pNumRxBytesUnicast = (u64 *)pValue;
-		break;
-	case	ODM_CMNINFO_WM_MODE:
-		pDM_Odm->pWirelessMode = (u8 *)pValue;
-		break;
-	case	ODM_CMNINFO_BAND:
-		pDM_Odm->pBandType = (u8 *)pValue;
-		break;
-	case	ODM_CMNINFO_SEC_CHNL_OFFSET:
-		pDM_Odm->pSecChOffset = (u8 *)pValue;
-		break;
-	case	ODM_CMNINFO_SEC_MODE:
-		pDM_Odm->pSecurity = (u8 *)pValue;
-		break;
-	case	ODM_CMNINFO_BW:
-		pDM_Odm->pBandWidth = (u8 *)pValue;
-		break;
-	case	ODM_CMNINFO_CHNL:
-		pDM_Odm->pChannel = (u8 *)pValue;
-		break;
-	case	ODM_CMNINFO_DMSP_GET_VALUE:
-		pDM_Odm->pbGetValueFromOtherMac = (bool *)pValue;
-		break;
-	case	ODM_CMNINFO_BUDDY_ADAPTOR:
-		pDM_Odm->pBuddyAdapter = (struct adapter **)pValue;
-		break;
-	case	ODM_CMNINFO_DMSP_IS_MASTER:
-		pDM_Odm->pbMasterOfDMSP = (bool *)pValue;
-		break;
-	case	ODM_CMNINFO_SCAN:
-		pDM_Odm->pbScanInProcess = (bool *)pValue;
-		break;
-	case	ODM_CMNINFO_POWER_SAVING:
-		pDM_Odm->pbPowerSaving = (bool *)pValue;
-		break;
-	case	ODM_CMNINFO_ONE_PATH_CCA:
-		pDM_Odm->pOnePathCCA = (u8 *)pValue;
-		break;
-	case	ODM_CMNINFO_DRV_STOP:
-		pDM_Odm->pbDriverStopped =  (bool *)pValue;
-		break;
-	case	ODM_CMNINFO_PNP_IN:
-		pDM_Odm->pbDriverIsGoingToPnpSetPowerSleep =  (bool *)pValue;
-		break;
-	case	ODM_CMNINFO_INIT_ON:
-		pDM_Odm->pinit_adpt_in_progress =  (bool *)pValue;
-		break;
-	case	ODM_CMNINFO_ANT_TEST:
-		pDM_Odm->pAntennaTest =  (u8 *)pValue;
-		break;
-	case	ODM_CMNINFO_NET_CLOSED:
-		pDM_Odm->pbNet_closed = (bool *)pValue;
-		break;
-	case    ODM_CMNINFO_MP_MODE:
-		pDM_Odm->mp_mode = (u8 *)pValue;
-		break;
-	/* To remove the compiler warning, must add an empty default statement to handle the other values. */
-	default:
-		/* do nothing */
-		break;
-	}
-}
-
 void ODM_CmnInfoPtrArrayHook(struct odm_dm_struct *pDM_Odm, enum odm_common_info_def CmnInfo, u16 Index, void *pValue)
 {
 	/*  Hook call by reference pointer. */
@@ -385,46 +241,6 @@ void ODM_CmnInfoPtrArrayHook(struct odm_dm_struct *pDM_Odm, enum odm_common_info
 	}
 }
 
-/*  Update Band/CHannel/.. The values are dynamic but non-per-packet. */
-void ODM_CmnInfoUpdate(struct odm_dm_struct *pDM_Odm, u32 CmnInfo, u64 Value)
-{
-	/*  */
-	/*  This init variable may be changed in run time. */
-	/*  */
-	switch	(CmnInfo) {
-	case	ODM_CMNINFO_ABILITY:
-		pDM_Odm->SupportAbility = (u32)Value;
-		break;
-	case	ODM_CMNINFO_RF_TYPE:
-		pDM_Odm->RFType = (u8)Value;
-		break;
-	case	ODM_CMNINFO_WIFI_DIRECT:
-		pDM_Odm->bWIFI_Direct = (bool)Value;
-		break;
-	case	ODM_CMNINFO_WIFI_DISPLAY:
-		pDM_Odm->bWIFI_Display = (bool)Value;
-		break;
-	case	ODM_CMNINFO_LINK:
-		pDM_Odm->bLinked = (bool)Value;
-		break;
-	case	ODM_CMNINFO_RSSI_MIN:
-		pDM_Odm->RSSI_Min = (u8)Value;
-		break;
-	case	ODM_CMNINFO_DBG_COMP:
-		pDM_Odm->DebugComponents = Value;
-		break;
-	case	ODM_CMNINFO_DBG_LEVEL:
-		pDM_Odm->DebugLevel = (u32)Value;
-		break;
-	case	ODM_CMNINFO_RA_THRESHOLD_HIGH:
-		pDM_Odm->RateAdaptive.HighRSSIThresh = (u8)Value;
-		break;
-	case	ODM_CMNINFO_RA_THRESHOLD_LOW:
-		pDM_Odm->RateAdaptive.LowRSSIThresh = (u8)Value;
-		break;
-	}
-}
-
 void odm_CommonInfoSelfInit(struct odm_dm_struct *pDM_Odm)
 {
 	struct adapter *adapter = pDM_Odm->Adapter;
@@ -469,7 +285,6 @@ void odm_CmnInfoInit_Debug(struct odm_dm_struct *pDM_Odm)
 	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("SupportInterface=%d\n", pDM_Odm->SupportInterface));
 	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("SupportICType=0x%x\n", pDM_Odm->SupportICType));
 	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("CutVersion=%d\n", pDM_Odm->CutVersion));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("RFType=%d\n", pDM_Odm->RFType));
 	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("BoardType=%d\n", pDM_Odm->BoardType));
 	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("ExtLNA=%d\n", pDM_Odm->ExtLNA));
 	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("ExtPA=%d\n", pDM_Odm->ExtPA));
@@ -947,37 +762,21 @@ u32 ODM_Get_Rate_Bitmap(struct odm_dm_struct *pDM_Odm, u32 macid, u32 ra_mask, u
 		break;
 	case (ODM_WM_B|ODM_WM_G|ODM_WM_N24G):
 	case (ODM_WM_A|ODM_WM_B|ODM_WM_G|ODM_WM_N24G):
-		if (pDM_Odm->RFType == ODM_1T2R || pDM_Odm->RFType == ODM_1T1R) {
-			if (rssi_level == DM_RATR_STA_HIGH) {
-				rate_bitmap = 0x000f0000;
-			} else if (rssi_level == DM_RATR_STA_MIDDLE) {
-				rate_bitmap = 0x000ff000;
-			} else {
-				if (*(pDM_Odm->pBandWidth) == ODM_BW40M)
-					rate_bitmap = 0x000ff015;
-				else
-					rate_bitmap = 0x000ff005;
-			}
+		if (rssi_level == DM_RATR_STA_HIGH) {
+			rate_bitmap = 0x000f0000;
+		} else if (rssi_level == DM_RATR_STA_MIDDLE) {
+			rate_bitmap = 0x000ff000;
 		} else {
-			if (rssi_level == DM_RATR_STA_HIGH) {
-				rate_bitmap = 0x0f8f0000;
-			} else if (rssi_level == DM_RATR_STA_MIDDLE) {
-				rate_bitmap = 0x0f8ff000;
-			} else {
-				if (*(pDM_Odm->pBandWidth) == ODM_BW40M)
-					rate_bitmap = 0x0f8ff015;
-				else
-					rate_bitmap = 0x0f8ff005;
-			}
+			if (*(pDM_Odm->pBandWidth) == ODM_BW40M)
+				rate_bitmap = 0x000ff015;
+			else
+				rate_bitmap = 0x000ff005;
 		}
 		break;
 	default:
 		/* case WIRELESS_11_24N: */
 		/* case WIRELESS_11_5N: */
-		if (pDM_Odm->RFType == RF_1T2R)
-			rate_bitmap = 0x000fffff;
-		else
-			rate_bitmap = 0x0fffffff;
+		rate_bitmap = 0x0fffffff;
 		break;
 	}
 
@@ -1096,8 +895,7 @@ bool ODM_RAStateCheck(struct odm_dm_struct *pDM_Odm, s32 RSSI, bool bForceUpdate
 void odm_DynamicTxPowerInit(struct odm_dm_struct *pDM_Odm)
 {
 	struct adapter *Adapter = pDM_Odm->Adapter;
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv	*pdmpriv = &pHalData->dmpriv;
+	struct dm_priv	*pdmpriv = &Adapter->HalData->dmpriv;
 	pdmpriv->bDynamicTxPowerEnable = false;
 	pdmpriv->LastDTPLvl = TxHighPwrLevel_Normal;
 	pdmpriv->DynamicTxHighPowerLvl = TxHighPwrLevel_Normal;
@@ -1122,8 +920,7 @@ void odm_RSSIMonitorCheck(struct odm_dm_struct *pDM_Odm)
 
 static void FindMinimumRSSI(struct adapter *pAdapter)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(pAdapter);
-	struct dm_priv	*pdmpriv = &pHalData->dmpriv;
+	struct dm_priv	*pdmpriv = &pAdapter->HalData->dmpriv;
 
 	/* 1 1.Unconditionally set RSSI */
 	pdmpriv->MinUndecoratedPWDBForDM = pdmpriv->EntryMinUndecoratedSmoothedPWDB;
@@ -1132,8 +929,7 @@ static void FindMinimumRSSI(struct adapter *pAdapter)
 void odm_RSSIMonitorCheckCE(struct odm_dm_struct *pDM_Odm)
 {
 	struct adapter *Adapter = pDM_Odm->Adapter;
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv	*pdmpriv = &pHalData->dmpriv;
+	struct dm_priv	*pdmpriv = &Adapter->HalData->dmpriv;
 	int	i;
 	int	tmpEntryMaxPWDB = 0, tmpEntryMinPWDB = 0xff;
 	u8	sta_cnt = 0;
@@ -1162,7 +958,7 @@ void odm_RSSIMonitorCheckCE(struct odm_dm_struct *pDM_Odm)
 
 	for (i = 0; i < sta_cnt; i++) {
 		if (PWDB_rssi[i] != 0) {
-			ODM_RA_SetRSSI_8188E(&pHalData->odmpriv,
+			ODM_RA_SetRSSI_8188E(&Adapter->HalData->odmpriv,
 					     PWDB_rssi[i] & 0xFF,
 					     (PWDB_rssi[i] >> 16) & 0xFF);
 		}
@@ -1179,8 +975,7 @@ void odm_RSSIMonitorCheckCE(struct odm_dm_struct *pDM_Odm)
 		pdmpriv->EntryMinUndecoratedSmoothedPWDB = 0;
 
 	FindMinimumRSSI(Adapter);
-	ODM_CmnInfoUpdate(&pHalData->odmpriv, ODM_CMNINFO_RSSI_MIN,
-			  pdmpriv->MinUndecoratedPWDBForDM);
+	Adapter->HalData->odmpriv.RSSI_Min = pdmpriv->MinUndecoratedPWDBForDM;
 }
 
 /* 3============================================================ */
@@ -1290,7 +1085,6 @@ void odm_EdcaTurboCheckCE(struct odm_dm_struct *pDM_Odm)
 	u64	cur_tx_bytes = 0;
 	u64	cur_rx_bytes = 0;
 	u8	bbtchange = false;
-	struct hal_data_8188e		*pHalData = GET_HAL_DATA(Adapter);
 	struct xmit_priv		*pxmitpriv = &(Adapter->xmitpriv);
 	struct recv_priv		*precvpriv = &(Adapter->recvpriv);
 	struct registry_priv	*pregpriv = &Adapter->registrypriv;
@@ -1344,7 +1138,8 @@ void odm_EdcaTurboCheckCE(struct odm_dm_struct *pDM_Odm)
 		/*  Turn Off EDCA turbo here. */
 		/*  Restore original EDCA according to the declaration of AP. */
 		 if (pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA) {
-			usb_write32(Adapter, REG_EDCA_BE_PARAM, pHalData->AcParam_BE);
+			usb_write32(Adapter, REG_EDCA_BE_PARAM,
+				    Adapter->HalData->AcParam_BE);
 			pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA = false;
 		}
 	}
diff --git a/drivers/staging/rtl8188eu/hal/phy.c b/drivers/staging/rtl8188eu/hal/phy.c
index a83bbea9be93..5192ef70bcfc 100644
--- a/drivers/staging/rtl8188eu/hal/phy.c
+++ b/drivers/staging/rtl8188eu/hal/phy.c
@@ -65,8 +65,7 @@ static u32 rf_serial_read(struct adapter *adapt,
 			enum rf_radio_path rfpath, u32 offset)
 {
 	u32 ret = 0;
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
-	struct bb_reg_def *phyreg = &hal_data->PHYRegDef[rfpath];
+	struct bb_reg_def *phyreg = &adapt->HalData->PHYRegDef[rfpath];
 	u32 tmplong, tmplong2;
 	u8 rfpi_enable = 0;
 
@@ -110,15 +109,14 @@ static void rf_serial_write(struct adapter *adapt,
 			    u32 data)
 {
 	u32 data_and_addr = 0;
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
-	struct bb_reg_def *phyreg = &hal_data->PHYRegDef[rfpath];
+	struct bb_reg_def *phyreg = &adapt->HalData->PHYRegDef[rfpath];
 
 	offset &= 0xff;
 	data_and_addr = ((offset<<20) | (data&0x000fffff)) & 0x0fffffff;
 	phy_set_bb_reg(adapt, phyreg->rf3wireOffset, bMaskDWord, data_and_addr);
 }
 
-u32 phy_query_rf_reg(struct adapter *adapt, enum rf_radio_path rf_path,
+u32 rtw_hal_read_rfreg(struct adapter *adapt, enum rf_radio_path rf_path,
 		     u32 reg_addr, u32 bit_mask)
 {
 	u32 original_value, readback_value, bit_shift;
@@ -147,14 +145,11 @@ void phy_set_rf_reg(struct adapter *adapt, enum rf_radio_path rf_path,
 static void get_tx_power_index(struct adapter *adapt, u8 channel, u8 *cck_pwr,
 			       u8 *ofdm_pwr, u8 *bw20_pwr, u8 *bw40_pwr)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 	u8 index = (channel - 1);
 	u8 TxCount = 0, path_nums;
 
-	if ((RF_1T2R == hal_data->rf_type) || (RF_1T1R == hal_data->rf_type))
-		path_nums = 1;
-	else
-		path_nums = 2;
+	path_nums = 1;
 
 	for (TxCount = 0; TxCount < path_nums; TxCount++) {
 		if (TxCount == RF_PATH_A) {
@@ -183,7 +178,7 @@ static void phy_power_index_check(struct adapter *adapt, u8 channel,
 				  u8 *cck_pwr, u8 *ofdm_pwr, u8 *bw20_pwr,
 				  u8 *bw40_pwr)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 
 	hal_data->CurrentCckTxPwrIdx = cck_pwr[0];
 	hal_data->CurrentOfdm24GTxPwrIdx = ofdm_pwr[0];
@@ -211,7 +206,7 @@ void phy_set_tx_power_level(struct adapter *adapt, u8 channel)
 
 static void phy_set_bw_mode_callback(struct adapter *adapt)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 	u8 reg_bw_opmode;
 	u8 reg_prsr_rsc;
 
@@ -274,10 +269,10 @@ static void phy_set_bw_mode_callback(struct adapter *adapt)
 		rtl88eu_phy_rf6052_set_bandwidth(adapt, hal_data->CurrentChannelBW);
 }
 
-void phy_set_bw_mode(struct adapter *adapt, enum ht_channel_width bandwidth,
+void rtw_hal_set_bwmode(struct adapter *adapt, enum ht_channel_width bandwidth,
 		     unsigned char offset)
 {
-	struct hal_data_8188e	*hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 	enum ht_channel_width tmp_bw = hal_data->CurrentChannelBW;
 
 	hal_data->CurrentChannelBW = bandwidth;
@@ -293,10 +288,7 @@ static void phy_sw_chnl_callback(struct adapter *adapt, u8 channel)
 {
 	u8 rf_path;
 	u32 param1, param2;
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
-
-	if (adapt->bNotifyChannelChange)
-		DBG_88E("[%s] ch = %d\n", __func__, channel);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 
 	phy_set_tx_power_level(adapt, channel);
 
@@ -310,9 +302,9 @@ static void phy_sw_chnl_callback(struct adapter *adapt, u8 channel)
 	}
 }
 
-void phy_sw_chnl(struct adapter *adapt, u8 channel)
+void rtw_hal_set_chan(struct adapter *adapt, u8 channel)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 	u8 tmpchannel = hal_data->CurrentChannel;
 
 	if (hal_data->rf_chip == RF_PSEUDO_11N)
@@ -407,7 +399,7 @@ static void dm_txpwr_track_setpwr(struct odm_dm_struct *dm_odm)
 
 void rtl88eu_dm_txpower_tracking_callback_thermalmeter(struct adapter *adapt)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 	u8 thermal_val = 0, delta, delta_lck, delta_iqk, offset;
 	u8 thermal_avg_count = 0;
 	u32 thermal_avg = 0;
@@ -439,7 +431,7 @@ void rtl88eu_dm_txpower_tracking_callback_thermalmeter(struct adapter *adapt)
 
 	dm_odm->RFCalibrateInfo.RegA24 = 0x090e1317;
 
-	thermal_val = (u8)phy_query_rf_reg(adapt, RF_PATH_A,
+	thermal_val = (u8)rtw_hal_read_rfreg(adapt, RF_PATH_A,
 					   RF_T_METER_88E, 0xfc00);
 
 	if (is2t)
@@ -632,8 +624,7 @@ static u8 phy_path_a_rx_iqk(struct adapter *adapt, bool configPathB)
 {
 	u32 reg_eac, reg_e94, reg_e9c, reg_ea4, u4tmp;
 	u8 result = 0x00;
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
-	struct odm_dm_struct *dm_odm = &hal_data->odmpriv;
+	struct odm_dm_struct *dm_odm = &adapt->HalData->odmpriv;
 
 	/* 1 Get TXIMR setting */
 	/* modify RXIQK mode table */
@@ -737,8 +728,7 @@ static u8 phy_path_b_iqk(struct adapter *adapt)
 {
 	u32 regeac, regeb4, regebc, regec4, regecc;
 	u8 result = 0x00;
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
-	struct odm_dm_struct *dm_odm = &hal_data->odmpriv;
+	struct odm_dm_struct *dm_odm = &adapt->HalData->odmpriv;
 
 	/* One shot, path B LOK & IQK */
 	phy_set_bb_reg(adapt, rIQK_AGC_Cont, bMaskDWord, 0x00000002);
@@ -954,17 +944,11 @@ static bool simularity_compare(struct adapter *adapt, s32 resulta[][8],
 			       u8 c1, u8 c2)
 {
 	u32 i, j, diff, sim_bitmap = 0, bound;
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
-	struct odm_dm_struct *dm_odm = &hal_data->odmpriv;
 	u8 final_candidate[2] = {0xFF, 0xFF};	/* for path A and path B */
 	bool result = true;
 	s32 tmp1 = 0, tmp2 = 0;
 
-	if ((dm_odm->RFType == ODM_2T2R) || (dm_odm->RFType == ODM_2T3R) ||
-	    (dm_odm->RFType == ODM_2T4R))
-		bound = 8;
-	else
-		bound = 4;
+	bound = 4;
 
 	for (i = 0; i < bound; i++) {
 		if ((i == 1) || (i == 3) || (i == 5) || (i == 7)) {
@@ -1033,8 +1017,7 @@ static bool simularity_compare(struct adapter *adapt, s32 resulta[][8],
 static void phy_iq_calibrate(struct adapter *adapt, s32 result[][8],
 			     u8 t, bool is2t)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
-	struct odm_dm_struct *dm_odm = &hal_data->odmpriv;
+	struct odm_dm_struct *dm_odm = &adapt->HalData->odmpriv;
 	u32 i;
 	u8 path_a_ok, path_b_ok;
 	u32 adda_reg[IQK_ADDA_REG_NUM] = {
@@ -1232,12 +1215,12 @@ static void phy_lc_calibrate(struct adapter *adapt, bool is2t)
 	if ((tmpreg&0x70) != 0) {
 		/* 1. Read original RF mode */
 		/* Path-A */
-		rf_a_mode = phy_query_rf_reg(adapt, RF_PATH_A, RF_AC,
+		rf_a_mode = rtw_hal_read_rfreg(adapt, RF_PATH_A, RF_AC,
 					     bMask12Bits);
 
 		/* Path-B */
 		if (is2t)
-			rf_b_mode = phy_query_rf_reg(adapt, RF_PATH_B, RF_AC,
+			rf_b_mode = rtw_hal_read_rfreg(adapt, RF_PATH_B, RF_AC,
 						     bMask12Bits);
 
 		/* 2. Set RF mode = standby mode */
@@ -1252,7 +1235,7 @@ static void phy_lc_calibrate(struct adapter *adapt, bool is2t)
 	}
 
 	/* 3. Read RF reg18 */
-	lc_cal = phy_query_rf_reg(adapt, RF_PATH_A, RF_CHNLBW, bMask12Bits);
+	lc_cal = rtw_hal_read_rfreg(adapt, RF_PATH_A, RF_CHNLBW, bMask12Bits);
 
 	/* 4. Set LC calibration begin bit15 */
 	phy_set_rf_reg(adapt, RF_PATH_A, RF_CHNLBW, bMask12Bits,
@@ -1279,8 +1262,7 @@ static void phy_lc_calibrate(struct adapter *adapt, bool is2t)
 
 void rtl88eu_phy_iq_calibrate(struct adapter *adapt, bool recovery)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
-	struct odm_dm_struct *dm_odm = &hal_data->odmpriv;
+	struct odm_dm_struct *dm_odm = &adapt->HalData->odmpriv;
 	s32 result[4][8];
 	u8 i, final, chn_index;
 	bool pathaok, pathbok;
@@ -1295,7 +1277,7 @@ void rtl88eu_phy_iq_calibrate(struct adapter *adapt, bool recovery)
 		rOFDM0_RxIQExtAnta};
 	bool is2t;
 
-	is2t = (dm_odm->RFType == ODM_2T2R) ? true : false;
+	is2t = false;
 
 	if (!(dm_odm->SupportAbility & ODM_RF_CALIBRATION))
 		return;
@@ -1391,7 +1373,7 @@ void rtl88eu_phy_iq_calibrate(struct adapter *adapt, bool recovery)
 				       (reg_ec4 == 0));
 	}
 
-	chn_index = get_right_chnl_for_iqk(hal_data->CurrentChannel);
+	chn_index = get_right_chnl_for_iqk(adapt->HalData->CurrentChannel);
 
 	if (final < 4) {
 		for (i = 0; i < IQK_Matrix_REG_NUM; i++)
@@ -1407,8 +1389,7 @@ void rtl88eu_phy_lc_calibrate(struct adapter *adapt)
 {
 	bool singletone = false, carrier_sup = false;
 	u32 timeout = 2000, timecount = 0;
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
-	struct odm_dm_struct *dm_odm = &hal_data->odmpriv;
+	struct odm_dm_struct *dm_odm = &adapt->HalData->odmpriv;
 
 	if (!(dm_odm->SupportAbility & ODM_RF_CALIBRATION))
 		return;
@@ -1422,12 +1403,7 @@ void rtl88eu_phy_lc_calibrate(struct adapter *adapt)
 
 	dm_odm->RFCalibrateInfo.bLCKInProgress = true;
 
-	if (dm_odm->RFType == ODM_2T2R) {
-		phy_lc_calibrate(adapt, true);
-	} else {
-		/* For 88C 1T1R */
-		phy_lc_calibrate(adapt, false);
-	}
+	phy_lc_calibrate(adapt, false);
 
 	dm_odm->RFCalibrateInfo.bLCKInProgress = false;
 }
diff --git a/drivers/staging/rtl8188eu/hal/rf.c b/drivers/staging/rtl8188eu/hal/rf.c
index 1596274eefc5..2f3edf0f850a 100644
--- a/drivers/staging/rtl8188eu/hal/rf.c
+++ b/drivers/staging/rtl8188eu/hal/rf.c
@@ -22,7 +22,7 @@
 void rtl88eu_phy_rf6052_set_bandwidth(struct adapter *adapt,
 				      enum ht_channel_width bandwidth)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 
 	switch (bandwidth) {
 	case HT_CHANNEL_WIDTH_20:
@@ -44,7 +44,7 @@ void rtl88eu_phy_rf6052_set_bandwidth(struct adapter *adapt,
 
 void rtl88eu_phy_rf6052_set_cck_txpower(struct adapter *adapt, u8 *powerlevel)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 	struct dm_priv *pdmpriv = &hal_data->dmpriv;
 	struct mlme_ext_priv *pmlmeext = &adapt->mlmeextpriv;
 	u32 tx_agc[2] = {0, 0}, tmpval = 0, pwrtrac_value;
@@ -129,7 +129,6 @@ static void getpowerbase88e(struct adapter *adapt, u8 *pwr_level_ofdm,
 			    u8 *pwr_level_bw20, u8 *pwr_level_bw40,
 			    u8 channel, u32 *ofdmbase, u32 *mcs_base)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
 	u32 powerbase0, powerbase1;
 	u8 i, powerlevel[2];
 
@@ -140,9 +139,9 @@ static void getpowerbase88e(struct adapter *adapt, u8 *pwr_level_ofdm,
 			     (powerbase0<<8) | powerbase0;
 		*(ofdmbase+i) = powerbase0;
 	}
-	for (i = 0; i < hal_data->NumTotalRFPath; i++) {
+	for (i = 0; i < adapt->HalData->NumTotalRFPath; i++) {
 		/* Check HT20 to HT40 diff */
-		if (hal_data->CurrentChannelBW == HT_CHANNEL_WIDTH_20)
+		if (adapt->HalData->CurrentChannelBW == HT_CHANNEL_WIDTH_20)
 			powerlevel[i] = pwr_level_bw20[i];
 		else
 			powerlevel[i] = pwr_level_bw40[i];
@@ -156,7 +155,7 @@ static void get_rx_power_val_by_reg(struct adapter *adapt, u8 channel,
 				    u8 index, u32 *powerbase0, u32 *powerbase1,
 				    u32 *out_val)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 	struct dm_priv	*pdmpriv = &hal_data->dmpriv;
 	u8 i, chnlGroup = 0, pwr_diff_limit[4], customer_pwr_limit;
 	s8 pwr_diff = 0;
@@ -286,7 +285,6 @@ void rtl88eu_phy_rf6052_set_ofdm_txpower(struct adapter *adapt,
 					 u8 *pwr_level_bw20,
 					 u8 *pwr_level_bw40, u8 channel)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
 	u32 write_val[2], powerbase0[2], powerbase1[2], pwrtrac_value;
 	u8 direction;
 	u8 index = 0;
@@ -294,8 +292,8 @@ void rtl88eu_phy_rf6052_set_ofdm_txpower(struct adapter *adapt,
 	getpowerbase88e(adapt, pwr_level_ofdm, pwr_level_bw20, pwr_level_bw40,
 			channel, &powerbase0[0], &powerbase1[0]);
 
-	rtl88eu_dm_txpower_track_adjust(&hal_data->odmpriv, 0, &direction,
-					&pwrtrac_value);
+	rtl88eu_dm_txpower_track_adjust(&adapt->HalData->odmpriv, 0,
+					&direction, &pwrtrac_value);
 
 	for (index = 0; index < 6; index++) {
 		get_rx_power_val_by_reg(adapt, channel, index,
diff --git a/drivers/staging/rtl8188eu/hal/rf_cfg.c b/drivers/staging/rtl8188eu/hal/rf_cfg.c
index 453f9e729067..dde64417e66a 100644
--- a/drivers/staging/rtl8188eu/hal/rf_cfg.c
+++ b/drivers/staging/rtl8188eu/hal/rf_cfg.c
@@ -19,7 +19,7 @@
 
 static bool check_condition(struct adapter *adapt, const u32  condition)
 {
-	struct odm_dm_struct *odm = &GET_HAL_DATA(adapt)->odmpriv;
+	struct odm_dm_struct *odm = &adapt->HalData->odmpriv;
 	u32 _board = odm->BoardType;
 	u32 _platform = odm->SupportPlatform;
 	u32 _interface = odm->SupportInterface;
@@ -228,7 +228,7 @@ static bool rtl88e_phy_config_rf_with_headerfile(struct adapter *adapt)
 
 static bool rf6052_conf_para(struct adapter *adapt)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 	u32 u4val = 0;
 	u8 rfpath;
 	bool rtstatus = true;
@@ -299,12 +299,9 @@ static bool rf6052_conf_para(struct adapter *adapt)
 
 static bool rtl88e_phy_rf6052_config(struct adapter *adapt)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *hal_data = adapt->HalData;
 
-	if (hal_data->rf_type == RF_1T1R)
-		hal_data->NumTotalRFPath = 1;
-	else
-		hal_data->NumTotalRFPath = 2;
+	hal_data->NumTotalRFPath = 1;
 
 	return rf6052_conf_para(adapt);
 }
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188e_cmd.c b/drivers/staging/rtl8188eu/hal/rtl8188e_cmd.c
index 2422c0297a50..d0f59b7836f1 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188e_cmd.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188e_cmd.c
@@ -60,7 +60,6 @@ static s32 FillH2CCmd_88E(struct adapter *adapt, u8 ElementID, u32 CmdLen, u8 *p
 	u8 h2c_box_num;
 	u32 msgbox_addr;
 	u32 msgbox_ex_addr;
-	struct hal_data_8188e *haldata = GET_HAL_DATA(adapt);
 	u8 cmd_idx, ext_cmd_len;
 	u32 h2c_cmd = 0;
 	u32 h2c_cmd_ex = 0;
@@ -81,7 +80,7 @@ static s32 FillH2CCmd_88E(struct adapter *adapt, u8 ElementID, u32 CmdLen, u8 *p
 
 	/* pay attention to if  race condition happened in  H2C cmd setting. */
 	do {
-		h2c_box_num = haldata->LastHMEBoxNum;
+		h2c_box_num = adapt->HalData->LastHMEBoxNum;
 
 		if (!_is_fw_read_cmd_down(adapt, h2c_box_num)) {
 			DBG_88E(" fw read cmd failed...\n");
@@ -110,7 +109,8 @@ static s32 FillH2CCmd_88E(struct adapter *adapt, u8 ElementID, u32 CmdLen, u8 *p
 		}
 		bcmd_down = true;
 
-		haldata->LastHMEBoxNum = (h2c_box_num+1) % RTL88E_MAX_H2C_BOX_NUMS;
+		adapt->HalData->LastHMEBoxNum =
+			(h2c_box_num+1) % RTL88E_MAX_H2C_BOX_NUMS;
 
 	} while ((!bcmd_down) && (retry_cnts--));
 
@@ -126,9 +126,9 @@ exit:
 /* bitmap[28:31]= Rate Adaptive id */
 /* arg[0:4] = macid */
 /* arg[5] = Short GI */
-void rtl8188e_Add_RateATid(struct adapter *pAdapter, u32 bitmap, u8 arg, u8 rssi_level)
+void rtw_hal_add_ra_tid(struct adapter *pAdapter, u32 bitmap, u8 arg, u8 rssi_level)
 {
-	struct hal_data_8188e *haldata = GET_HAL_DATA(pAdapter);
+	struct odm_dm_struct *odmpriv = &pAdapter->HalData->odmpriv;
 
 	u8 macid, init_rate, raid, shortGIrate = false;
 
@@ -138,7 +138,7 @@ void rtl8188e_Add_RateATid(struct adapter *pAdapter, u32 bitmap, u8 arg, u8 rssi
 	bitmap &= 0x0fffffff;
 
 	if (rssi_level != DM_RATR_STA_INIT)
-		bitmap = ODM_Get_Rate_Bitmap(&haldata->odmpriv, macid, bitmap, rssi_level);
+		bitmap = ODM_Get_Rate_Bitmap(odmpriv, macid, bitmap, rssi_level);
 
 	bitmap |= ((raid<<28)&0xf0000000);
 
@@ -156,7 +156,7 @@ void rtl8188e_Add_RateATid(struct adapter *pAdapter, u32 bitmap, u8 arg, u8 rssi
 	DBG_88E("%s=> mac_id:%d, raid:%d, ra_bitmap=0x%x, shortGIrate=0x%02x\n",
 		__func__, macid, raid, bitmap, shortGIrate);
 
-	ODM_RA_UpdateRateInfo_8188E(&(haldata->odmpriv), macid, raid, bitmap, shortGIrate);
+	ODM_RA_UpdateRateInfo_8188E(odmpriv, macid, raid, bitmap, shortGIrate);
 }
 
 void rtl8188e_set_FwPwrMode_cmd(struct adapter *adapt, u8 Mode)
@@ -219,7 +219,7 @@ void rtl8188e_set_FwMediaStatus_cmd(struct adapter *adapt, __le16 mstatus_rpt)
 
 static void ConstructBeacon(struct adapter *adapt, u8 *pframe, u32 *pLength)
 {
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	u32 rate_len, pktlen;
 	struct mlme_ext_priv *pmlmeext = &(adapt->mlmeextpriv);
@@ -227,20 +227,20 @@ static void ConstructBeacon(struct adapter *adapt, u8 *pframe, u32 *pLength)
 	struct wlan_bssid_ex		*cur_network = &(pmlmeinfo->network);
 	u8 bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
-	memcpy(pwlanhdr->addr1, bc_addr, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, myid(&(adapt->eeprompriv)), ETH_ALEN);
-	memcpy(pwlanhdr->addr3, cur_network->MacAddress, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, bc_addr);
+	ether_addr_copy(pwlanhdr->addr2, myid(&(adapt->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr3, cur_network->MacAddress);
 
 	SetSeqNum(pwlanhdr, 0/*pmlmeext->mgnt_seq*/);
 	SetFrameSubType(pframe, WIFI_BEACON);
 
-	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
-	pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pframe += sizeof(struct ieee80211_hdr_3addr);
+	pktlen = sizeof(struct ieee80211_hdr_3addr);
 
 	/* timestamp will be inserted by hardware */
 	pframe += 8;
@@ -304,16 +304,16 @@ _ConstructBeacon:
 
 static void ConstructPSPoll(struct adapter *adapt, u8 *pframe, u32 *pLength)
 {
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	struct mlme_ext_priv *pmlmeext = &(adapt->mlmeextpriv);
 	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
 	__le16 *fctrl;
 	struct wlan_bssid_ex *pnetwork = &(pmlmeinfo->network);
 
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
 	/*  Frame control. */
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 	SetPwrMgt(fctrl);
 	SetFrameSubType(pframe, WIFI_PSPOLL);
@@ -322,10 +322,10 @@ static void ConstructPSPoll(struct adapter *adapt, u8 *pframe, u32 *pLength)
 	SetDuration(pframe, (pmlmeinfo->aid | 0xc000));
 
 	/*  BSSID. */
-	memcpy(pwlanhdr->addr1, pnetwork->MacAddress, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, pnetwork->MacAddress);
 
 	/*  TA. */
-	memcpy(pwlanhdr->addr2, myid(&(adapt->eeprompriv)), ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr2, myid(&(adapt->eeprompriv)));
 
 	*pLength = 16;
 }
@@ -338,7 +338,7 @@ static void ConstructNullFunctionData(struct adapter *adapt, u8 *pframe,
 	u8 bEosp,
 	u8 bForcePowerSave)
 {
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	u32 pktlen;
 	struct mlme_priv *pmlmepriv = &adapt->mlmepriv;
@@ -347,9 +347,9 @@ static void ConstructNullFunctionData(struct adapter *adapt, u8 *pframe,
 	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
 	struct wlan_bssid_ex *pnetwork = &(pmlmeinfo->network);
 
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &pwlanhdr->frame_ctl;
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 	if (bForcePowerSave)
 		SetPwrMgt(fctrl);
@@ -357,40 +357,40 @@ static void ConstructNullFunctionData(struct adapter *adapt, u8 *pframe,
 	switch (cur_network->network.InfrastructureMode) {
 	case Ndis802_11Infrastructure:
 		SetToDs(fctrl);
-		memcpy(pwlanhdr->addr1, pnetwork->MacAddress, ETH_ALEN);
-		memcpy(pwlanhdr->addr2, myid(&(adapt->eeprompriv)), ETH_ALEN);
-		memcpy(pwlanhdr->addr3, StaAddr, ETH_ALEN);
+		ether_addr_copy(pwlanhdr->addr1, pnetwork->MacAddress);
+		ether_addr_copy(pwlanhdr->addr2, myid(&(adapt->eeprompriv)));
+		ether_addr_copy(pwlanhdr->addr3, StaAddr);
 		break;
 	case Ndis802_11APMode:
 		SetFrDs(fctrl);
-		memcpy(pwlanhdr->addr1, StaAddr, ETH_ALEN);
-		memcpy(pwlanhdr->addr2, pnetwork->MacAddress, ETH_ALEN);
-		memcpy(pwlanhdr->addr3, myid(&(adapt->eeprompriv)), ETH_ALEN);
+		ether_addr_copy(pwlanhdr->addr1, StaAddr);
+		ether_addr_copy(pwlanhdr->addr2, pnetwork->MacAddress);
+		ether_addr_copy(pwlanhdr->addr3, myid(&(adapt->eeprompriv)));
 		break;
 	case Ndis802_11IBSS:
 	default:
-		memcpy(pwlanhdr->addr1, StaAddr, ETH_ALEN);
-		memcpy(pwlanhdr->addr2, myid(&(adapt->eeprompriv)), ETH_ALEN);
-		memcpy(pwlanhdr->addr3, pnetwork->MacAddress, ETH_ALEN);
+		ether_addr_copy(pwlanhdr->addr1, StaAddr);
+		ether_addr_copy(pwlanhdr->addr2, myid(&(adapt->eeprompriv)));
+		ether_addr_copy(pwlanhdr->addr3, pnetwork->MacAddress);
 		break;
 	}
 
 	SetSeqNum(pwlanhdr, 0);
 
 	if (bQoS) {
-		struct rtw_ieee80211_hdr_3addr_qos *pwlanqoshdr;
+		struct ieee80211_qos_hdr *pwlanqoshdr;
 
 		SetFrameSubType(pframe, WIFI_QOS_DATA_NULL);
 
-		pwlanqoshdr = (struct rtw_ieee80211_hdr_3addr_qos *)pframe;
-		SetPriority(&pwlanqoshdr->qc, AC);
-		SetEOSP(&pwlanqoshdr->qc, bEosp);
+		pwlanqoshdr = (struct ieee80211_qos_hdr *)pframe;
+		SetPriority(&pwlanqoshdr->qos_ctrl, AC);
+		SetEOSP(&pwlanqoshdr->qos_ctrl, bEosp);
 
-		pktlen = sizeof(struct rtw_ieee80211_hdr_3addr_qos);
+		pktlen = sizeof(struct ieee80211_qos_hdr);
 	} else {
 		SetFrameSubType(pframe, WIFI_DATA_NULL);
 
-		pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+		pktlen = sizeof(struct ieee80211_hdr_3addr);
 	}
 
 	*pLength = pktlen;
@@ -398,7 +398,7 @@ static void ConstructNullFunctionData(struct adapter *adapt, u8 *pframe,
 
 static void ConstructProbeRsp(struct adapter *adapt, u8 *pframe, u32 *pLength, u8 *StaAddr, bool bHideSSID)
 {
-	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	u8 *mac, *bssid;
 	u32 pktlen;
@@ -406,21 +406,21 @@ static void ConstructProbeRsp(struct adapter *adapt, u8 *pframe, u32 *pLength, u
 	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
 	struct wlan_bssid_ex	*cur_network = &(pmlmeinfo->network);
 
-	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
 	mac = myid(&(adapt->eeprompriv));
 	bssid = cur_network->MacAddress;
 
-	fctrl = &(pwlanhdr->frame_ctl);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
-	memcpy(pwlanhdr->addr1, StaAddr, ETH_ALEN);
-	memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
-	memcpy(pwlanhdr->addr3, bssid, ETH_ALEN);
+	ether_addr_copy(pwlanhdr->addr1, StaAddr);
+	ether_addr_copy(pwlanhdr->addr2, mac);
+	ether_addr_copy(pwlanhdr->addr3, bssid);
 
 	SetSeqNum(pwlanhdr, 0);
 	SetFrameSubType(fctrl, WIFI_PROBERSP);
 
-	pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pktlen = sizeof(struct ieee80211_hdr_3addr);
 	pframe += pktlen;
 
 	if (cur_network->IELength > MAX_IE_SZ)
@@ -445,7 +445,6 @@ static void ConstructProbeRsp(struct adapter *adapt, u8 *pframe, u32 *pLength, u
 /*  2009.10.15 by tynli. */
 static void SetFwRsvdPagePkt(struct adapter *adapt, bool bDLFinished)
 {
-	struct hal_data_8188e *haldata;
 	struct xmit_frame	*pmgntframe;
 	struct pkt_attrib	*pattrib;
 	struct xmit_priv *pxmitpriv;
@@ -467,7 +466,6 @@ static void SetFwRsvdPagePkt(struct adapter *adapt, bool bDLFinished)
 		return;
 	}
 
-	haldata = GET_HAL_DATA(adapt);
 	pxmitpriv = &adapt->xmitpriv;
 	pmlmeext = &adapt->mlmeextpriv;
 	pmlmeinfo = &pmlmeext->mlmext_info;
@@ -487,7 +485,7 @@ static void SetFwRsvdPagePkt(struct adapter *adapt, bool bDLFinished)
 	if (PageNeed == 1)
 		PageNeed += 1;
 	PageNum += PageNeed;
-	haldata->FwRsvdPageStartOffset = PageNum;
+	adapt->HalData->FwRsvdPageStartOffset = PageNum;
 
 	BufIndex += PageNeed*128;
 
@@ -554,7 +552,7 @@ exit:
 
 void rtl8188e_set_FwJoinBssReport_cmd(struct adapter *adapt, u8 mstatus)
 {
-	struct hal_data_8188e *haldata = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *haldata = adapt->HalData;
 	struct mlme_ext_priv *pmlmeext = &(adapt->mlmeextpriv);
 	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
 	bool	bSendBeacon = false;
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188e_dm.c b/drivers/staging/rtl8188eu/hal/rtl8188e_dm.c
index 81f2931876f8..d04b7fbb71e1 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188e_dm.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188e_dm.c
@@ -42,43 +42,33 @@ static void dm_InitGPIOSetting(struct adapter *Adapter)
 /*  */
 static void Init_ODM_ComInfo_88E(struct adapter *Adapter)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *hal_data = Adapter->HalData;
 	struct dm_priv	*pdmpriv = &hal_data->dmpriv;
 	struct odm_dm_struct *dm_odm = &(hal_data->odmpriv);
-	u8 cut_ver;
 
 	/*  Init Value */
 	memset(dm_odm, 0, sizeof(*dm_odm));
 
 	dm_odm->Adapter = Adapter;
+	dm_odm->SupportPlatform = ODM_CE;
+	dm_odm->SupportICType = ODM_RTL8188E;
+	dm_odm->CutVersion = ODM_CUT_A;
+	dm_odm->bIsMPChip = hal_data->VersionID.ChipType == NORMAL_CHIP;
+	dm_odm->PatchID = hal_data->CustomerID;
+	dm_odm->bWIFITest = Adapter->registrypriv.wifi_spec;
 
-	ODM_CmnInfoInit(dm_odm, ODM_CMNINFO_PLATFORM, ODM_CE);
+	dm_odm->AntDivType = hal_data->TRxAntDivType;
 
-	ODM_CmnInfoInit(dm_odm, ODM_CMNINFO_IC_TYPE, ODM_RTL8188E);
-
-	cut_ver = ODM_CUT_A;
-
-	ODM_CmnInfoInit(dm_odm, ODM_CMNINFO_CUT_VER, cut_ver);
-
-	ODM_CmnInfoInit(dm_odm, ODM_CMNINFO_MP_TEST_CHIP, hal_data->VersionID.ChipType == NORMAL_CHIP ? true : false);
-
-	ODM_CmnInfoInit(dm_odm, ODM_CMNINFO_PATCH_ID, hal_data->CustomerID);
-	ODM_CmnInfoInit(dm_odm, ODM_CMNINFO_BWIFI_TEST, Adapter->registrypriv.wifi_spec);
-
-
-	if (hal_data->rf_type == RF_1T1R)
-		ODM_CmnInfoUpdate(dm_odm, ODM_CMNINFO_RF_TYPE, ODM_1T1R);
-	else if (hal_data->rf_type == RF_2T2R)
-		ODM_CmnInfoUpdate(dm_odm, ODM_CMNINFO_RF_TYPE, ODM_2T2R);
-	else if (hal_data->rf_type == RF_1T2R)
-		ODM_CmnInfoUpdate(dm_odm, ODM_CMNINFO_RF_TYPE, ODM_1T2R);
-
-	ODM_CmnInfoInit(dm_odm, ODM_CMNINFO_RF_ANTENNA_TYPE, hal_data->TRxAntDivType);
+	/*  Tx power tracking BB swing table. */
+	/*  The base index = 12. +((12-n)/2)dB 13~?? = decrease tx pwr by -((n-12)/2)dB */
+	dm_odm->BbSwingIdxOfdm = 12; /*  Set defalut value as index 12. */
+	dm_odm->BbSwingIdxOfdmCurrent = 12;
+	dm_odm->BbSwingFlagOfdm = false;
 
 	pdmpriv->InitODMFlag =	ODM_RF_CALIBRATION |
 				ODM_RF_TX_PWR_TRACK;
 
-	ODM_CmnInfoUpdate(dm_odm, ODM_CMNINFO_ABILITY, pdmpriv->InitODMFlag);
+	dm_odm->SupportAbility = pdmpriv->InitODMFlag;
 }
 
 static void Update_ODM_ComInfo_88E(struct adapter *Adapter)
@@ -86,7 +76,7 @@ static void Update_ODM_ComInfo_88E(struct adapter *Adapter)
 	struct mlme_ext_priv	*pmlmeext = &Adapter->mlmeextpriv;
 	struct mlme_priv	*pmlmepriv = &Adapter->mlmepriv;
 	struct pwrctrl_priv *pwrctrlpriv = &Adapter->pwrctrlpriv;
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *hal_data = Adapter->HalData;
 	struct odm_dm_struct *dm_odm = &(hal_data->odmpriv);
 	struct dm_priv	*pdmpriv = &hal_data->dmpriv;
 	int i;
@@ -109,20 +99,26 @@ static void Update_ODM_ComInfo_88E(struct adapter *Adapter)
 					ODM_RF_TX_PWR_TRACK;
 	}
 
-	ODM_CmnInfoUpdate(dm_odm, ODM_CMNINFO_ABILITY, pdmpriv->InitODMFlag);
-
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_TX_UNI, &(Adapter->xmitpriv.tx_bytes));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_RX_UNI, &(Adapter->recvpriv.rx_bytes));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_WM_MODE, &(pmlmeext->cur_wireless_mode));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_SEC_CHNL_OFFSET, &(hal_data->nCur40MhzPrimeSC));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_SEC_MODE, &(Adapter->securitypriv.dot11PrivacyAlgrthm));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_BW, &(hal_data->CurrentChannelBW));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_CHNL, &(hal_data->CurrentChannel));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_NET_CLOSED, &(Adapter->net_closed));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_MP_MODE, &(Adapter->registrypriv.mp_mode));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_SCAN, &(pmlmepriv->bScanInProcess));
-	ODM_CmnInfoHook(dm_odm, ODM_CMNINFO_POWER_SAVING, &(pwrctrlpriv->bpower_saving));
-	ODM_CmnInfoInit(dm_odm, ODM_CMNINFO_RF_ANTENNA_TYPE, hal_data->TRxAntDivType);
+	dm_odm->SupportAbility = pdmpriv->InitODMFlag;
+
+	dm_odm->pNumTxBytesUnicast = &Adapter->xmitpriv.tx_bytes;
+	dm_odm->pNumRxBytesUnicast = &Adapter->recvpriv.rx_bytes;
+	dm_odm->pWirelessMode = &pmlmeext->cur_wireless_mode;
+	dm_odm->pSecChOffset = &hal_data->nCur40MhzPrimeSC;
+	dm_odm->pSecurity = (u8 *)&Adapter->securitypriv.dot11PrivacyAlgrthm;
+	dm_odm->pBandWidth = (u8 *)&hal_data->CurrentChannelBW;
+	dm_odm->pChannel = &hal_data->CurrentChannel;
+	dm_odm->pbNet_closed = (bool *)&Adapter->net_closed;
+	dm_odm->mp_mode = &Adapter->registrypriv.mp_mode;
+	dm_odm->pbScanInProcess = (bool *)&pmlmepriv->bScanInProcess;
+	dm_odm->pbPowerSaving = (bool *)&pwrctrlpriv->bpower_saving;
+	dm_odm->AntDivType = hal_data->TRxAntDivType;
+
+	/*  Tx power tracking BB swing table. */
+	/*  The base index = 12. +((12-n)/2)dB 13~?? = decrease tx pwr by -((n-12)/2)dB */
+	dm_odm->BbSwingIdxOfdm = 12; /*  Set defalut value as index 12. */
+	dm_odm->BbSwingIdxOfdmCurrent = 12;
+	dm_odm->BbSwingFlagOfdm = false;
 
 	for (i = 0; i < NUM_STA; i++)
 		ODM_CmnInfoPtrArrayHook(dm_odm, ODM_CMNINFO_STA_STATUS, i, NULL);
@@ -130,23 +126,19 @@ static void Update_ODM_ComInfo_88E(struct adapter *Adapter)
 
 void rtl8188e_InitHalDm(struct adapter *Adapter)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(Adapter);
-	struct dm_priv	*pdmpriv = &hal_data->dmpriv;
-	struct odm_dm_struct *dm_odm = &(hal_data->odmpriv);
+	struct dm_priv	*pdmpriv = &Adapter->HalData->dmpriv;
+	struct odm_dm_struct *dm_odm = &(Adapter->HalData->odmpriv);
 
 	dm_InitGPIOSetting(Adapter);
 	pdmpriv->DM_Type = DM_Type_ByDriver;
 	pdmpriv->DMFlag = DYNAMIC_FUNC_DISABLE;
 	Update_ODM_ComInfo_88E(Adapter);
 	ODM_DMInit(dm_odm);
-	Adapter->fix_rate = 0xFF;
 }
 
-void rtl8188e_HalDmWatchDog(struct adapter *Adapter)
+void rtw_hal_dm_watchdog(struct adapter *Adapter)
 {
-	bool fw_ps_awake = true;
 	u8 hw_init_completed = false;
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(Adapter);
 	struct mlme_priv *pmlmepriv = NULL;
 	u8 bLinked = false;
 
@@ -155,13 +147,6 @@ void rtl8188e_HalDmWatchDog(struct adapter *Adapter)
 	if (!hw_init_completed)
 		goto skip_dm;
 
-	rtw_hal_get_hwreg(Adapter, HW_VAR_FWLPS_RF_ON, (u8 *)(&fw_ps_awake));
-
-	/*  Fw is under p2p powersaving mode, driver should stop dynamic mechanism. */
-	/*  modifed by thomas. 2011.06.11. */
-	if (Adapter->wdinfo.p2p_ps_mode)
-		fw_ps_awake = false;
-
 	/* ODM */
 	pmlmepriv = &Adapter->mlmepriv;
 
@@ -175,19 +160,18 @@ void rtl8188e_HalDmWatchDog(struct adapter *Adapter)
 			bLinked = true;
 	}
 
-	ODM_CmnInfoUpdate(&hal_data->odmpriv, ODM_CMNINFO_LINK, bLinked);
-	ODM_DMWatchdog(&hal_data->odmpriv);
+	Adapter->HalData->odmpriv.bLinked = bLinked;
+	ODM_DMWatchdog(&Adapter->HalData->odmpriv);
 skip_dm:
 	/*  Check GPIO to determine current RF on/off and Pbc status. */
 	/*  Check Hardware Radio ON/OFF or not */
 	return;
 }
 
-void rtl8188e_init_dm_priv(struct adapter *Adapter)
+void rtw_hal_dm_init(struct adapter *Adapter)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(Adapter);
-	struct dm_priv	*pdmpriv = &hal_data->dmpriv;
-	struct odm_dm_struct *podmpriv = &hal_data->odmpriv;
+	struct dm_priv	*pdmpriv = &Adapter->HalData->dmpriv;
+	struct odm_dm_struct *podmpriv = &Adapter->HalData->odmpriv;
 
 	memset(pdmpriv, 0, sizeof(struct dm_priv));
 	Init_ODM_ComInfo_88E(Adapter);
@@ -196,11 +180,9 @@ void rtl8188e_init_dm_priv(struct adapter *Adapter)
 
 /*  Add new function to reset the state of antenna diversity before link. */
 /*  Compare RSSI for deciding antenna */
-void AntDivCompare8188E(struct adapter *Adapter, struct wlan_bssid_ex *dst, struct wlan_bssid_ex *src)
+void rtw_hal_antdiv_rssi_compared(struct adapter *Adapter, struct wlan_bssid_ex *dst, struct wlan_bssid_ex *src)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(Adapter);
-
-	if (0 != hal_data->AntDivCfg) {
+	if (0 != Adapter->HalData->AntDivCfg) {
 		/* select optimum_antenna for before linked =>For antenna diversity */
 		if (dst->Rssi >=  src->Rssi) {/* keep org parameter */
 			src->Rssi = dst->Rssi;
@@ -210,15 +192,14 @@ void AntDivCompare8188E(struct adapter *Adapter, struct wlan_bssid_ex *dst, stru
 }
 
 /*  Add new function to reset the state of antenna diversity before link. */
-u8 AntDivBeforeLink8188E(struct adapter *Adapter)
+u8 rtw_hal_antdiv_before_linked(struct adapter *Adapter)
 {
-	struct hal_data_8188e *hal_data = GET_HAL_DATA(Adapter);
-	struct odm_dm_struct *dm_odm = &hal_data->odmpriv;
+	struct odm_dm_struct *dm_odm = &Adapter->HalData->odmpriv;
 	struct sw_ant_switch *dm_swat_tbl = &dm_odm->DM_SWAT_Table;
 	struct mlme_priv *pmlmepriv = &(Adapter->mlmepriv);
 
 	/*  Condition that does not need to use antenna diversity. */
-	if (hal_data->AntDivCfg == 0)
+	if (Adapter->HalData->AntDivCfg == 0)
 		return false;
 
 	if (check_fwstate(pmlmepriv, _FW_LINKED))
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188e_hal_init.c b/drivers/staging/rtl8188eu/hal/rtl8188e_hal_init.c
index 0b444fd3e550..385bc2f56f2f 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188e_hal_init.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188e_hal_init.c
@@ -108,28 +108,24 @@ void _8051Reset88E(struct adapter *padapter)
 
 void rtl8188e_InitializeFirmwareVars(struct adapter *padapter)
 {
-	struct hal_data_8188e *pHalData = GET_HAL_DATA(padapter);
-
 	/*  Init Fw LPS related. */
 	padapter->pwrctrlpriv.bFwCurrentInPSMode = false;
 
 	/*  Init H2C counter. by tynli. 2009.12.09. */
-	pHalData->LastHMEBoxNum = 0;
+	padapter->HalData->LastHMEBoxNum = 0;
 }
 
-static void rtl8188e_free_hal_data(struct adapter *padapter)
+void rtw_hal_free_data(struct adapter *padapter)
 {
 	kfree(padapter->HalData);
 	padapter->HalData = NULL;
 }
 
-static void ReadChipVersion8188E(struct adapter *padapter)
+void rtw_hal_read_chip_version(struct adapter *padapter)
 {
 	u32				value32;
 	struct HAL_VERSION		ChipVersion;
-	struct hal_data_8188e	*pHalData;
-
-	pHalData = GET_HAL_DATA(padapter);
+	struct hal_data_8188e *pHalData = padapter->HalData;
 
 	value32 = usb_read32(padapter, REG_SYS_CFG);
 	ChipVersion.ChipType = ((value32 & RTL_ID) ? TEST_CHIP : NORMAL_CHIP);
@@ -139,16 +135,13 @@ static void ReadChipVersion8188E(struct adapter *padapter)
 	dump_chip_info(ChipVersion);
 
 	pHalData->VersionID = ChipVersion;
-	pHalData->rf_type = RF_1T1R;
 	pHalData->NumTotalRFPath = 1;
-
-	MSG_88E("RF_Type is %x!!\n", pHalData->rf_type);
 }
 
-static void rtl8188e_SetHalODMVar(struct adapter *Adapter, enum hal_odm_variable eVariable, void *pValue1, bool bSet)
+void rtw_hal_set_odm_var(struct adapter *Adapter, enum hal_odm_variable eVariable, void *pValue1, bool bSet)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(Adapter);
-	struct odm_dm_struct *podmpriv = &pHalData->odmpriv;
+	struct odm_dm_struct *podmpriv = &Adapter->HalData->odmpriv;
+
 	switch (eVariable) {
 	case HAL_ODM_STA_INFO:
 		{
@@ -165,17 +158,17 @@ static void rtl8188e_SetHalODMVar(struct adapter *Adapter, enum hal_odm_variable
 		}
 		break;
 	case HAL_ODM_P2P_STATE:
-			ODM_CmnInfoUpdate(podmpriv, ODM_CMNINFO_WIFI_DIRECT, bSet);
+		podmpriv->bWIFI_Direct = bSet;
 		break;
 	case HAL_ODM_WIFI_DISPLAY_STATE:
-			ODM_CmnInfoUpdate(podmpriv, ODM_CMNINFO_WIFI_DISPLAY, bSet);
+		podmpriv->bWIFI_Display = bSet;
 		break;
 	default:
 		break;
 	}
 }
 
-static void hal_notch_filter_8188e(struct adapter *adapter, bool enable)
+void rtw_hal_notch_filter(struct adapter *adapter, bool enable)
 {
 	if (enable) {
 		DBG_88E("Enable notch filter\n");
@@ -185,32 +178,6 @@ static void hal_notch_filter_8188e(struct adapter *adapter, bool enable)
 		usb_write8(adapter, rOFDM0_RxDSP+1, usb_read8(adapter, rOFDM0_RxDSP+1) & ~BIT(1));
 	}
 }
-void rtl8188e_set_hal_ops(struct hal_ops *pHalFunc)
-{
-	pHalFunc->free_hal_data = &rtl8188e_free_hal_data;
-
-	pHalFunc->dm_init = &rtl8188e_init_dm_priv;
-
-	pHalFunc->read_chip_version = &ReadChipVersion8188E;
-
-	pHalFunc->set_bwmode_handler = &phy_set_bw_mode;
-	pHalFunc->set_channel_handler = &phy_sw_chnl;
-
-	pHalFunc->hal_dm_watchdog = &rtl8188e_HalDmWatchDog;
-
-	pHalFunc->Add_RateATid = &rtl8188e_Add_RateATid;
-
-	pHalFunc->AntDivBeforeLinkHandler = &AntDivBeforeLink8188E;
-	pHalFunc->AntDivCompareHandler = &AntDivCompare8188E;
-	pHalFunc->read_rfreg = &phy_query_rf_reg;
-
-	pHalFunc->sreset_init_value = &sreset_init_value;
-	pHalFunc->sreset_get_wifi_status  = &sreset_get_wifi_status;
-
-	pHalFunc->SetHalODMVarHandler = &rtl8188e_SetHalODMVar;
-
-	pHalFunc->hal_notch_filter = &hal_notch_filter_8188e;
-}
 
 /*  */
 /*  */
@@ -501,7 +468,7 @@ void Hal_ReadPowerSavingMode88E(struct adapter *padapter, u8 *hwinfo, bool AutoL
 
 void Hal_ReadTxPowerInfo88E(struct adapter *padapter, u8 *PROMContent, bool AutoLoadFail)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(padapter);
+	struct hal_data_8188e *pHalData = padapter->HalData;
 	struct txpowerinfo24g pwrInfo24G;
 	u8 rfPath, ch, group;
 	u8 bIn24G, TxCount;
@@ -553,7 +520,7 @@ void Hal_ReadTxPowerInfo88E(struct adapter *padapter, u8 *PROMContent, bool Auto
 
 void Hal_EfuseParseXtal_8188E(struct adapter *pAdapter, u8 *hwinfo, bool AutoLoadFail)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(pAdapter);
+	struct hal_data_8188e *pHalData = pAdapter->HalData;
 
 	if (!AutoLoadFail) {
 		pHalData->CrystalCap = hwinfo[EEPROM_XTAL_88E];
@@ -567,7 +534,7 @@ void Hal_EfuseParseXtal_8188E(struct adapter *pAdapter, u8 *hwinfo, bool AutoLoa
 
 void Hal_EfuseParseBoardType88E(struct adapter *pAdapter, u8 *hwinfo, bool AutoLoadFail)
 {
-	struct hal_data_8188e *pHalData = GET_HAL_DATA(pAdapter);
+	struct hal_data_8188e *pHalData = pAdapter->HalData;
 
 	if (!AutoLoadFail)
 		pHalData->BoardType = (hwinfo[EEPROM_RF_BOARD_OPTION_88E]
@@ -579,7 +546,7 @@ void Hal_EfuseParseBoardType88E(struct adapter *pAdapter, u8 *hwinfo, bool AutoL
 
 void Hal_EfuseParseEEPROMVer88E(struct adapter *padapter, u8 *hwinfo, bool AutoLoadFail)
 {
-	struct hal_data_8188e *pHalData = GET_HAL_DATA(padapter);
+	struct hal_data_8188e *pHalData = padapter->HalData;
 
 	if (!AutoLoadFail) {
 		pHalData->EEPROMVersion = hwinfo[EEPROM_VERSION_88E];
@@ -606,7 +573,7 @@ void rtl8188e_EfuseParseChnlPlan(struct adapter *padapter, u8 *hwinfo, bool Auto
 
 void Hal_EfuseParseCustomerID88E(struct adapter *padapter, u8 *hwinfo, bool AutoLoadFail)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(padapter);
+	struct hal_data_8188e	*pHalData = padapter->HalData;
 
 	if (!AutoLoadFail) {
 		pHalData->EEPROMCustomerID = hwinfo[EEPROM_CUSTOMERID_88E];
@@ -619,7 +586,7 @@ void Hal_EfuseParseCustomerID88E(struct adapter *padapter, u8 *hwinfo, bool Auto
 
 void Hal_ReadAntennaDiversity88E(struct adapter *pAdapter, u8 *PROMContent, bool AutoLoadFail)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(pAdapter);
+	struct hal_data_8188e *pHalData = pAdapter->HalData;
 	struct registry_priv	*registry_par = &pAdapter->registrypriv;
 
 	if (!AutoLoadFail) {
@@ -652,7 +619,7 @@ void Hal_ReadAntennaDiversity88E(struct adapter *pAdapter, u8 *PROMContent, bool
 
 void Hal_ReadThermalMeter_88E(struct adapter *Adapter, u8 *PROMContent, bool AutoloadFail)
 {
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *pHalData = Adapter->HalData;
 
 	/*  ThermalMeter from EEPROM */
 	if (!AutoloadFail)
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188e_rxdesc.c b/drivers/staging/rtl8188eu/hal/rtl8188e_rxdesc.c
index f110c961df70..fa2cfd5768de 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188e_rxdesc.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188e_rxdesc.c
@@ -57,10 +57,9 @@ static void process_link_qual(struct adapter *padapter,
 	signal_stat->avg_val = signal_stat->total_val / signal_stat->total_num;
 }
 
-void rtl8188e_process_phy_info(struct adapter *padapter, void *prframe)
+void rtl8188e_process_phy_info(struct adapter *padapter,
+		               struct recv_frame *precvframe)
 {
-	struct recv_frame *precvframe = prframe;
-
 	/*  Check RSSI */
 	process_rssi(padapter, precvframe);
 	/*  Check EVM */
@@ -140,7 +139,6 @@ void update_recvframe_phyinfo_88e(struct recv_frame *precvframe,
 {
 	struct adapter *padapter = precvframe->adapter;
 	struct rx_pkt_attrib *pattrib = &precvframe->attrib;
-	struct hal_data_8188e *pHalData = GET_HAL_DATA(padapter);
 	struct odm_phy_status_info *pPHYInfo  = (struct odm_phy_status_info *)(&pattrib->phy_info);
 	u8 *wlanhdr;
 	struct odm_per_pkt_info	pkt_info;
@@ -181,7 +179,8 @@ void update_recvframe_phyinfo_88e(struct recv_frame *precvframe,
 		pkt_info.StationID = psta->mac_id;
 	pkt_info.Rate = pattrib->mcs_rate;
 
-	ODM_PhyStatusQuery(&pHalData->odmpriv, pPHYInfo, (u8 *)pphy_status, &(pkt_info));
+	ODM_PhyStatusQuery(&padapter->HalData->odmpriv, pPHYInfo,
+			   (u8 *)pphy_status, &(pkt_info));
 
 	precvframe->psta = NULL;
 	if (pkt_info.bPacketMatchBSSID &&
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188eu_led.c b/drivers/staging/rtl8188eu/hal/rtl8188eu_led.c
index d9e677ef8f84..780666a755ee 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188eu_led.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188eu_led.c
@@ -40,14 +40,13 @@ void SwLedOn(struct adapter *padapter, struct LED_871x *pLed)
 void SwLedOff(struct adapter *padapter, struct LED_871x *pLed)
 {
 	u8	LedCfg;
-	struct hal_data_8188e	*pHalData = GET_HAL_DATA(padapter);
 
 	if (padapter->bSurpriseRemoved || padapter->bDriverStopped)
 		goto exit;
 
 	LedCfg = usb_read8(padapter, REG_LEDCFG2);/* 0x4E */
 
-	if (pHalData->bLedOpenDrain) {
+	if (padapter->HalData->bLedOpenDrain) {
 			/*  Open-drain arrangement for controlling the LED) */
 		LedCfg &= 0x90; /*  Set to software control. */
 		usb_write8(padapter, REG_LEDCFG2, (LedCfg | BIT(3)));
@@ -66,21 +65,20 @@ exit:
 
 /*	Description: */
 /*		Initialize all LED_871x objects. */
-void rtl8188eu_InitSwLeds(struct adapter *padapter)
+void rtw_hal_sw_led_init(struct adapter *padapter)
 {
 	struct led_priv *pledpriv = &(padapter->ledpriv);
-	struct hal_data_8188e   *haldata = GET_HAL_DATA(padapter);
 
 	pledpriv->bRegUseLed = true;
 	pledpriv->LedControlHandler = LedControl8188eu;
-	haldata->bLedOpenDrain = true;
+	padapter->HalData->bLedOpenDrain = true;
 
 	InitLed871x(padapter, &(pledpriv->SwLed0));
 }
 
 /*	Description: */
 /*		DeInitialize all LED_819xUsb objects. */
-void rtl8188eu_DeInitSwLeds(struct adapter *padapter)
+void rtw_hal_sw_led_deinit(struct adapter *padapter)
 {
 	struct led_priv	*ledpriv = &(padapter->ledpriv);
 
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188eu_recv.c b/drivers/staging/rtl8188eu/hal/rtl8188eu_recv.c
index 255d6f215091..d0495a16ff79 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188eu_recv.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188eu_recv.c
@@ -13,6 +13,7 @@
  *
  ******************************************************************************/
 #define _RTL8188EU_RECV_C_
+#include <linux/kmemleak.h>
 #include <osdep_service.h>
 #include <drv_types.h>
 #include <recv_osdep.h>
@@ -23,7 +24,7 @@
 
 #include <rtl8188e_hal.h>
 
-int	rtl8188eu_init_recv_priv(struct adapter *padapter)
+int	rtw_hal_init_recv_priv(struct adapter *padapter)
 {
 	struct recv_priv	*precvpriv = &padapter->recvpriv;
 	int	i, res = _SUCCESS;
@@ -72,6 +73,7 @@ int	rtl8188eu_init_recv_priv(struct adapter *padapter)
 					MAX_RECVBUF_SZ + RECVBUFF_ALIGN_SZ,
 					GFP_KERNEL);
 			if (pskb) {
+				kmemleak_not_leak(pskb);
 				pskb->dev = padapter->pnetdev;
 				tmpaddr = (size_t)pskb->data;
 				alignm = tmpaddr & (RECVBUFF_ALIGN_SZ-1);
@@ -87,7 +89,7 @@ exit:
 	return res;
 }
 
-void rtl8188eu_free_recv_priv(struct adapter *padapter)
+void rtw_hal_free_recv_priv(struct adapter *padapter)
 {
 	int	i;
 	struct recv_buf	*precvbuf;
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188eu_xmit.c b/drivers/staging/rtl8188eu/hal/rtl8188eu_xmit.c
index ec21d8c82eba..85650b2663ec 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188eu_xmit.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188eu_xmit.c
@@ -21,7 +21,7 @@
 #include <usb_ops_linux.h>
 #include <rtl8188e_hal.h>
 
-s32	rtl8188eu_init_xmit_priv(struct adapter *adapt)
+s32 rtw_hal_init_xmit_priv(struct adapter *adapt)
 {
 	struct xmit_priv	*pxmitpriv = &adapt->xmitpriv;
 
@@ -33,11 +33,7 @@ s32	rtl8188eu_init_xmit_priv(struct adapter *adapt)
 
 static u8 urb_zero_packet_chk(struct adapter *adapt, int sz)
 {
-	u8 set_tx_desc_offset;
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(adapt);
-	set_tx_desc_offset = (((sz + TXDESC_SIZE) %  haldata->UsbBulkOutSize) == 0) ? 1 : 0;
-
-	return set_tx_desc_offset;
+	return !((sz + TXDESC_SIZE) % adapt->HalData->UsbBulkOutSize);
 }
 
 static void rtl8188eu_cal_txdesc_chksum(struct tx_desc	*ptxdesc)
@@ -175,7 +171,7 @@ static s32 update_txdesc(struct xmit_frame *pxmitframe, u8 *pmem, s32 sz, u8 bag
 	u8 data_rate, pwr_status, offset;
 	struct adapter		*adapt = pxmitframe->padapter;
 	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(adapt);
+	struct odm_dm_struct *odmpriv = &adapt->HalData->odmpriv;
 	struct tx_desc	*ptxdesc = (struct tx_desc *)pmem;
 	struct mlme_ext_priv	*pmlmeext = &adapt->mlmeextpriv;
 	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
@@ -259,12 +255,12 @@ static s32 update_txdesc(struct xmit_frame *pxmitframe, u8 *pmem, s32 sz, u8 bag
 			ptxdesc->txdw5 |= cpu_to_le32(0x0001ff00);/* DATA/RTS  Rate FB LMT */
 
 			if (pattrib->ht_en) {
-				if (ODM_RA_GetShortGI_8188E(&haldata->odmpriv, pattrib->mac_id))
+				if (ODM_RA_GetShortGI_8188E(odmpriv, pattrib->mac_id))
 					ptxdesc->txdw5 |= cpu_to_le32(SGI);/* SGI */
 			}
-			data_rate = ODM_RA_GetDecisionRate_8188E(&haldata->odmpriv, pattrib->mac_id);
+			data_rate = ODM_RA_GetDecisionRate_8188E(odmpriv, pattrib->mac_id);
 			ptxdesc->txdw5 |= cpu_to_le32(data_rate & 0x3F);
-			pwr_status = ODM_RA_GetHwPwrStatus_8188E(&haldata->odmpriv, pattrib->mac_id);
+			pwr_status = ODM_RA_GetHwPwrStatus_8188E(odmpriv, pattrib->mac_id);
 			ptxdesc->txdw4 |= cpu_to_le32((pwr_status & 0x7) << PWR_STATUS_SHT);
 		} else {
 			/*  EAP data packet and ARP packet and DHCP. */
@@ -332,8 +328,7 @@ static s32 update_txdesc(struct xmit_frame *pxmitframe, u8 *pmem, s32 sz, u8 bag
 		ptxdesc->txdw4 |= cpu_to_le32(HW_SSN);	/*  Hw set sequence number */
 	}
 
-	rtl88eu_dm_set_tx_ant_by_tx_info(&haldata->odmpriv, pmem,
-					 pattrib->mac_id);
+	rtl88eu_dm_set_tx_ant_by_tx_info(odmpriv, pmem, pattrib->mac_id);
 
 	rtl8188eu_cal_txdesc_chksum(ptxdesc);
 	_dbg_dump_tx_info(adapt, pxmitframe->frame_tag, ptxdesc);
@@ -387,7 +382,7 @@ static s32 rtw_dump_xframe(struct adapter *adapt, struct xmit_frame *pxmitframe)
 		}
 		ff_hwaddr = rtw_get_ff_hwaddr(pxmitframe);
 
-		inner_ret = usb_write_port(adapt, ff_hwaddr, w_sz, (unsigned char *)pxmitbuf);
+		inner_ret = usb_write_port(adapt, ff_hwaddr, w_sz, pxmitbuf);
 
 		rtw_count_tx_stats(adapt, pxmitframe, sz);
 
@@ -424,11 +419,11 @@ static u32 xmitframe_need_length(struct xmit_frame *pxmitframe)
 	return len;
 }
 
-s32 rtl8188eu_xmitframe_complete(struct adapter *adapt, struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
+s32 rtl8188eu_xmitframe_complete(struct adapter *adapt, struct xmit_priv *pxmitpriv)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(adapt);
 	struct xmit_frame *pxmitframe = NULL;
 	struct xmit_frame *pfirstframe = NULL;
+	struct xmit_buf *pxmitbuf;
 
 	/*  aggregate variable */
 	struct hw_xmit *phwxmit;
@@ -441,7 +436,7 @@ s32 rtl8188eu_xmitframe_complete(struct adapter *adapt, struct xmit_priv *pxmitp
 	u32 pbuf_tail;	/*  last pkt tail */
 	u32 len;	/*  packet length, except TXDESC_SIZE and PKT_OFFSET */
 
-	u32 bulksize = haldata->UsbBulkOutSize;
+	u32 bulksize = adapt->HalData->UsbBulkOutSize;
 	u8 desc_cnt;
 	u32 bulkptr;
 
@@ -450,12 +445,9 @@ s32 rtl8188eu_xmitframe_complete(struct adapter *adapt, struct xmit_priv *pxmitp
 
 	RT_TRACE(_module_rtl8192c_xmit_c_, _drv_info_, ("+xmitframe_complete\n"));
 
-	/*  check xmitbuffer is ok */
-	if (pxmitbuf == NULL) {
-		pxmitbuf = rtw_alloc_xmitbuf(pxmitpriv);
-		if (pxmitbuf == NULL)
-			return false;
-	}
+	pxmitbuf = rtw_alloc_xmitbuf(pxmitpriv);
+	if (pxmitbuf == NULL)
+		return false;
 
 	/* 3 1. pick up first frame */
 	rtw_free_xmitframe(pxmitpriv, pxmitframe);
@@ -565,7 +557,7 @@ s32 rtl8188eu_xmitframe_complete(struct adapter *adapt, struct xmit_priv *pxmitp
 
 		if (pbuf < bulkptr) {
 			desc_cnt++;
-			if (desc_cnt == haldata->UsbTxAggDescNum)
+			if (desc_cnt == adapt->HalData->UsbTxAggDescNum)
 				break;
 		} else {
 			desc_cnt = 0;
@@ -594,7 +586,7 @@ s32 rtl8188eu_xmitframe_complete(struct adapter *adapt, struct xmit_priv *pxmitp
 
 	/* 3 4. write xmit buffer to USB FIFO */
 	ff_hwaddr = rtw_get_ff_hwaddr(pfirstframe);
-	usb_write_port(adapt, ff_hwaddr, pbuf_tail, (u8 *)pxmitbuf);
+	usb_write_port(adapt, ff_hwaddr, pbuf_tail, pxmitbuf);
 
 	/* 3 5. update statisitc */
 	pbuf_tail -= (pfirstframe->agg_num * TXDESC_SIZE);
@@ -607,24 +599,12 @@ s32 rtl8188eu_xmitframe_complete(struct adapter *adapt, struct xmit_priv *pxmitp
 	return true;
 }
 
-static s32 xmitframe_direct(struct adapter *adapt, struct xmit_frame *pxmitframe)
-{
-	s32 res;
-
-	res = rtw_xmitframe_coalesce(adapt, pxmitframe->pkt, pxmitframe);
-	if (res == _SUCCESS)
-		rtw_dump_xframe(adapt, pxmitframe);
-	else
-		DBG_88E("==> %s xmitframe_coalsece failed\n", __func__);
-	return res;
-}
-
 /*
  * Return
  *	true	dump packet directly
  *	false	enqueue packet
  */
-static s32 pre_xmitframe(struct adapter *adapt, struct xmit_frame *pxmitframe)
+s32 rtw_hal_xmit(struct adapter *adapt, struct xmit_frame *pxmitframe)
 {
 	s32 res;
 	struct xmit_buf *pxmitbuf = NULL;
@@ -650,7 +630,12 @@ static s32 pre_xmitframe(struct adapter *adapt, struct xmit_frame *pxmitframe)
 	pxmitframe->buf_addr = pxmitbuf->pbuf;
 	pxmitbuf->priv_data = pxmitframe;
 
-	if (xmitframe_direct(adapt, pxmitframe) != _SUCCESS) {
+	res = rtw_xmitframe_coalesce(adapt, pxmitframe->pkt, pxmitframe);
+
+	if (res == _SUCCESS) {
+		rtw_dump_xframe(adapt, pxmitframe);
+	} else {
+		DBG_88E("==> %s xmitframe_coalsece failed\n", __func__);
 		rtw_free_xmitbuf(pxmitpriv, pxmitbuf);
 		rtw_free_xmitframe(pxmitpriv, pxmitframe);
 	}
@@ -674,20 +659,10 @@ enqueue:
 	return false;
 }
 
-s32 rtl8188eu_mgnt_xmit(struct adapter *adapt, struct xmit_frame *pmgntframe)
+s32 rtw_hal_mgnt_xmit(struct adapter *adapt, struct xmit_frame *pmgntframe)
 {
 	struct xmit_priv *xmitpriv = &adapt->xmitpriv;
 
 	rtl88eu_mon_xmit_hook(adapt->pmondev, pmgntframe, xmitpriv->frag_len);
 	return rtw_dump_xframe(adapt, pmgntframe);
 }
-
-/*
- * Return
- *	true	dump packet directly ok
- *	false	temporary can't transmit packets to hardware
- */
-s32 rtl8188eu_hal_xmit(struct adapter *adapt, struct xmit_frame *pxmitframe)
-{
-	return pre_xmitframe(adapt, pxmitframe);
-}
diff --git a/drivers/staging/rtl8188eu/hal/usb_halinit.c b/drivers/staging/rtl8188eu/hal/usb_halinit.c
index 363f3a34ddce..7692ca495ee5 100644
--- a/drivers/staging/rtl8188eu/hal/usb_halinit.c
+++ b/drivers/staging/rtl8188eu/hal/usb_halinit.c
@@ -21,14 +21,13 @@
 #include <rtl8188e_hal.h>
 #include <rtl8188e_led.h>
 #include <rtw_iol.h>
-#include <usb_hal.h>
 #include <phy.h>
 
 #define		HAL_BB_ENABLE		1
 
 static void _ConfigNormalChipOutEP_8188E(struct adapter *adapt, u8 NumOutPipe)
 {
-	struct hal_data_8188e	*haldata	= GET_HAL_DATA(adapt);
+	struct hal_data_8188e *haldata = adapt->HalData;
 
 	switch (NumOutPipe) {
 	case	3:
@@ -51,13 +50,12 @@ static void _ConfigNormalChipOutEP_8188E(struct adapter *adapt, u8 NumOutPipe)
 
 static bool HalUsbSetQueuePipeMapping8188EUsb(struct adapter *adapt, u8 NumInPipe, u8 NumOutPipe)
 {
-	struct hal_data_8188e	*haldata	= GET_HAL_DATA(adapt);
 	bool			result		= false;
 
 	_ConfigNormalChipOutEP_8188E(adapt, NumOutPipe);
 
 	/*  Normal chip with one IN and one OUT doesn't have interrupt IN EP. */
-	if (haldata->OutEpNumber == 1) {
+	if (adapt->HalData->OutEpNumber == 1) {
 		if (NumInPipe != 1)
 			return result;
 	}
@@ -69,9 +67,9 @@ static bool HalUsbSetQueuePipeMapping8188EUsb(struct adapter *adapt, u8 NumInPip
 	return result;
 }
 
-static void rtl8188eu_interface_configure(struct adapter *adapt)
+void rtw_hal_chip_configure(struct adapter *adapt)
 {
-	struct hal_data_8188e	*haldata	= GET_HAL_DATA(adapt);
+	struct hal_data_8188e *haldata = adapt->HalData;
 	struct dvobj_priv	*pdvobjpriv = adapter_to_dvobj(adapt);
 
 	if (pdvobjpriv->ishighspeed)
@@ -94,12 +92,11 @@ static void rtl8188eu_interface_configure(struct adapter *adapt)
 				pdvobjpriv->RtNumInPipes, pdvobjpriv->RtNumOutPipes);
 }
 
-static u32 rtl8188eu_InitPowerOn(struct adapter *adapt)
+u32 rtw_hal_power_on(struct adapter *adapt)
 {
 	u16 value16;
 	/*  HW Power on sequence */
-	struct hal_data_8188e	*haldata	= GET_HAL_DATA(adapt);
-	if (haldata->bMacPwrCtrlOn)
+	if (adapt->HalData->bMacPwrCtrlOn)
 		return _SUCCESS;
 
 	if (!rtl88eu_pwrseqcmdparsing(adapt, PWR_CUT_ALL_MSK,
@@ -119,7 +116,7 @@ static u32 rtl8188eu_InitPowerOn(struct adapter *adapt)
 	/*  for SDIO - Set CR bit10 to enable 32k calibration. Suggested by SD1 Gimmy. Added by tynli. 2011.08.31. */
 
 	usb_write16(adapt, REG_CR, value16);
-	haldata->bMacPwrCtrlOn = true;
+	adapt->HalData->bMacPwrCtrlOn = true;
 
 	return _SUCCESS;
 }
@@ -129,18 +126,17 @@ static void _InitInterrupt(struct adapter *Adapter)
 {
 	u32 imr, imr_ex;
 	u8  usb_opt;
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
 
 	/* HISR write one to clear */
 	usb_write32(Adapter, REG_HISR_88E, 0xFFFFFFFF);
 	/*  HIMR - */
 	imr = IMR_PSTIMEOUT_88E | IMR_TBDER_88E | IMR_CPWM_88E | IMR_CPWM2_88E;
 	usb_write32(Adapter, REG_HIMR_88E, imr);
-	haldata->IntrMask[0] = imr;
+	Adapter->HalData->IntrMask[0] = imr;
 
 	imr_ex = IMR_TXERR_88E | IMR_RXERR_88E | IMR_TXFOVW_88E | IMR_RXFOVW_88E;
 	usb_write32(Adapter, REG_HIMRE_88E, imr_ex);
-	haldata->IntrMask[1] = imr_ex;
+	Adapter->HalData->IntrMask[1] = imr_ex;
 
 	/*  REG_USB_SPECIAL_OPTION - BIT(4) */
 	/*  0; Use interrupt endpoint to upload interrupt pkt */
@@ -157,7 +153,6 @@ static void _InitInterrupt(struct adapter *Adapter)
 
 static void _InitQueueReservedPage(struct adapter *Adapter)
 {
-	struct hal_data_8188e		*haldata = GET_HAL_DATA(Adapter);
 	struct registry_priv	*pregistrypriv = &Adapter->registrypriv;
 	u32 numHQ	= 0;
 	u32 numLQ	= 0;
@@ -168,14 +163,14 @@ static void _InitQueueReservedPage(struct adapter *Adapter)
 	bool bWiFiConfig = pregistrypriv->wifi_spec;
 
 	if (bWiFiConfig) {
-		if (haldata->OutEpQueueSel & TX_SELE_HQ)
+		if (Adapter->HalData->OutEpQueueSel & TX_SELE_HQ)
 			numHQ =  0x29;
 
-		if (haldata->OutEpQueueSel & TX_SELE_LQ)
+		if (Adapter->HalData->OutEpQueueSel & TX_SELE_LQ)
 			numLQ = 0x1C;
 
 		/*  NOTE: This step shall be proceed before writing REG_RQPN. */
-		if (haldata->OutEpQueueSel & TX_SELE_NQ)
+		if (Adapter->HalData->OutEpQueueSel & TX_SELE_NQ)
 			numNQ = 0x1C;
 		value8 = (u8)_NPQ(numNQ);
 		usb_write8(Adapter, REG_RQPN_NPQ, value8);
@@ -225,10 +220,9 @@ static void _InitNormalChipRegPriority(struct adapter *Adapter, u16 beQ,
 
 static void _InitNormalChipOneOutEpPriority(struct adapter *Adapter)
 {
-	struct hal_data_8188e	*haldata	= GET_HAL_DATA(Adapter);
-
 	u16 value = 0;
-	switch (haldata->OutEpQueueSel) {
+
+	switch (Adapter->HalData->OutEpQueueSel) {
 	case TX_SELE_HQ:
 		value = QUEUE_HIGH;
 		break;
@@ -247,13 +241,12 @@ static void _InitNormalChipOneOutEpPriority(struct adapter *Adapter)
 
 static void _InitNormalChipTwoOutEpPriority(struct adapter *Adapter)
 {
-	struct hal_data_8188e	*haldata	= GET_HAL_DATA(Adapter);
 	struct registry_priv *pregistrypriv = &Adapter->registrypriv;
 	u16 beQ, bkQ, viQ, voQ, mgtQ, hiQ;
 	u16 valueHi = 0;
 	u16 valueLow = 0;
 
-	switch (haldata->OutEpQueueSel) {
+	switch (Adapter->HalData->OutEpQueueSel) {
 	case (TX_SELE_HQ | TX_SELE_LQ):
 		valueHi = QUEUE_HIGH;
 		valueLow = QUEUE_LOW;
@@ -313,9 +306,7 @@ static void _InitNormalChipThreeOutEpPriority(struct adapter *Adapter)
 
 static void _InitQueuePriority(struct adapter *Adapter)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
-
-	switch (haldata->OutEpNumber) {
+	switch (Adapter->HalData->OutEpNumber) {
 	case 1:
 		_InitNormalChipOneOutEpPriority(Adapter);
 		break;
@@ -357,7 +348,7 @@ static void _InitDriverInfoSize(struct adapter *Adapter, u8 drvInfoSize)
 
 static void _InitWMACSetting(struct adapter *Adapter)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *haldata = Adapter->HalData;
 
 	haldata->ReceiveConfig = RCR_AAP | RCR_APM | RCR_AM | RCR_AB |
 				  RCR_CBSSID_DATA | RCR_CBSSID_BCN |
@@ -456,7 +447,7 @@ static void _InitRetryFunction(struct adapter *Adapter)
  */
 static void usb_AggSettingTxUpdate(struct adapter *Adapter)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *haldata = Adapter->HalData;
 	u32 value32;
 
 	if (Adapter->registrypriv.wifi_spec)
@@ -492,7 +483,7 @@ usb_AggSettingRxUpdate(
 		struct adapter *Adapter
 	)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *haldata = Adapter->HalData;
 	u8 valueDMA;
 	u8 valueUSB;
 
@@ -566,8 +557,6 @@ usb_AggSettingRxUpdate(
 
 static void InitUsbAggregationSetting(struct adapter *Adapter)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
-
 	/*  Tx aggregation setting */
 	usb_AggSettingTxUpdate(Adapter);
 
@@ -575,12 +564,12 @@ static void InitUsbAggregationSetting(struct adapter *Adapter)
 	usb_AggSettingRxUpdate(Adapter);
 
 	/*  201/12/10 MH Add for USB agg mode dynamic switch. */
-	haldata->UsbRxHighSpeedMode = false;
+	Adapter->HalData->UsbRxHighSpeedMode = false;
 }
 
 static void _InitBeaconParameters(struct adapter *Adapter)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *haldata = Adapter->HalData;
 
 	usb_write16(Adapter, REG_BCN_CTRL, 0x1010);
 
@@ -622,7 +611,7 @@ enum {
 
 static void _InitAntenna_Selection(struct adapter *Adapter)
 {
-	struct hal_data_8188e	*haldata	= GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *haldata = Adapter->HalData;
 
 	if (haldata->AntDivCfg == 0)
 		return;
@@ -672,13 +661,13 @@ enum rt_rf_power_state RfOnOffDetect(struct adapter *adapt)
 	return rfpowerstate;
 }	/*  HalDetectPwrDownMode */
 
-static u32 rtl8188eu_hal_init(struct adapter *Adapter)
+u32 rtl8188eu_hal_init(struct adapter *Adapter)
 {
 	u8 value8 = 0;
 	u16  value16;
 	u8 txpktbuf_bndy;
 	u32 status = _SUCCESS;
-	struct hal_data_8188e		*haldata = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *haldata = Adapter->HalData;
 	struct pwrctrl_priv		*pwrctrlpriv = &Adapter->pwrctrlpriv;
 	struct registry_priv	*pregistrypriv = &Adapter->registrypriv;
 	unsigned long init_start_time = jiffies;
@@ -702,7 +691,7 @@ static u32 rtl8188eu_hal_init(struct adapter *Adapter)
 	}
 
 	HAL_INIT_PROFILE_TAG(HAL_INIT_STAGES_INIT_PW_ON);
-	status = rtl8188eu_InitPowerOn(Adapter);
+	status = rtw_hal_power_on(Adapter);
 	if (status == _FAIL) {
 		RT_TRACE(_module_hci_hal_init_c_, _drv_err_, ("Failed to init power on!\n"));
 		goto exit;
@@ -810,8 +799,8 @@ static u32 rtl8188eu_hal_init(struct adapter *Adapter)
 	usb_write16(Adapter, REG_PKT_BE_BK_LIFE_TIME, 0x0400);	/*  unit: 256us. 256ms */
 
 	/* Keep RfRegChnlVal for later use. */
-	haldata->RfRegChnlVal[0] = phy_query_rf_reg(Adapter, (enum rf_radio_path)0, RF_CHNLBW, bRFRegOffsetMask);
-	haldata->RfRegChnlVal[1] = phy_query_rf_reg(Adapter, (enum rf_radio_path)1, RF_CHNLBW, bRFRegOffsetMask);
+	haldata->RfRegChnlVal[0] = rtw_hal_read_rfreg(Adapter, (enum rf_radio_path)0, RF_CHNLBW, bRFRegOffsetMask);
+	haldata->RfRegChnlVal[1] = rtw_hal_read_rfreg(Adapter, (enum rf_radio_path)1, RF_CHNLBW, bRFRegOffsetMask);
 
 HAL_INIT_PROFILE_TAG(HAL_INIT_STAGES_TURN_ON_BLOCK);
 	_BBTurnOnBlock(Adapter);
@@ -905,7 +894,6 @@ HAL_INIT_PROFILE_TAG(HAL_INIT_STAGES_END);
 static void CardDisableRTL8188EU(struct adapter *Adapter)
 {
 	u8 val8;
-	struct hal_data_8188e	*haldata	= GET_HAL_DATA(Adapter);
 
 	RT_TRACE(_module_hci_hal_init_c_, _drv_info_, ("CardDisableRTL8188EU\n"));
 
@@ -958,7 +946,7 @@ static void CardDisableRTL8188EU(struct adapter *Adapter)
 	val8 = usb_read8(Adapter, REG_GPIO_IO_SEL+1);
 	usb_write8(Adapter, REG_GPIO_IO_SEL+1, val8|0x0F);/* Reg0x43 */
 	usb_write32(Adapter, REG_BB_PAD_CTRL, 0x00080808);/* set LNA ,TRSW,EX_PA Pin to output mode */
-	haldata->bMacPwrCtrlOn = false;
+	Adapter->HalData->bMacPwrCtrlOn = false;
 	Adapter->bFWReady = false;
 }
 
@@ -972,7 +960,7 @@ static void rtl8192cu_hw_power_down(struct adapter *adapt)
 	usb_write16(adapt, REG_APS_FSMCO, 0x8812);
 }
 
-static u32 rtl8188eu_hal_deinit(struct adapter *Adapter)
+u32 rtl8188eu_hal_deinit(struct adapter *Adapter)
 {
 	DBG_88E("==> %s\n", __func__);
 
@@ -994,7 +982,7 @@ static u32 rtl8188eu_hal_deinit(struct adapter *Adapter)
 	return _SUCCESS;
 }
 
-static unsigned int rtl8188eu_inirp_init(struct adapter *Adapter)
+u32 rtw_hal_inirp_init(struct adapter *Adapter)
 {
 	u8 i;
 	struct recv_buf *precvbuf;
@@ -1029,17 +1017,6 @@ exit:
 	return status;
 }
 
-static unsigned int rtl8188eu_inirp_deinit(struct adapter *Adapter)
-{
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_, ("\n ===> usb_rx_deinit\n"));
-
-	usb_read_port_cancel(Adapter);
-
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_, ("\n <=== usb_rx_deinit\n"));
-
-	return _SUCCESS;
-}
-
 /*  */
 /*  */
 /*	EEPROM/EFUSE Content Parsing */
@@ -1047,7 +1024,7 @@ static unsigned int rtl8188eu_inirp_deinit(struct adapter *Adapter)
 /*  */
 static void Hal_EfuseParsePIDVID_8188EU(struct adapter *adapt, u8 *hwinfo, bool AutoLoadFail)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *haldata = adapt->HalData;
 
 	if (!AutoLoadFail) {
 		/*  VID, PID */
@@ -1132,12 +1109,10 @@ static void _ReadPROMContent(
 
 static void _ReadRFType(struct adapter *Adapter)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
-
-	haldata->rf_chip = RF_6052;
+	Adapter->HalData->rf_chip = RF_6052;
 }
 
-static void _ReadAdapterInfo8188EU(struct adapter *Adapter)
+void rtw_hal_read_chip_info(struct adapter *Adapter)
 {
 	unsigned long start = jiffies;
 
@@ -1157,7 +1132,7 @@ static void rtl8192cu_trigger_gpio_0(struct adapter *adapt)
 
 static void ResumeTxBeacon(struct adapter *adapt)
 {
-	struct hal_data_8188e *haldata = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *haldata = adapt->HalData;
 
 	/*  2010.03.01. Marked by tynli. No need to call workitem beacause we record the value */
 	/*  which should be read from register to a global variable. */
@@ -1171,7 +1146,7 @@ static void ResumeTxBeacon(struct adapter *adapt)
 
 static void StopTxBeacon(struct adapter *adapt)
 {
-	struct hal_data_8188e *haldata = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *haldata = adapt->HalData;
 
 	/*  2010.03.01. Marked by tynli. No need to call workitem beacause we record the value */
 	/*  which should be read from register to a global variable. */
@@ -1276,9 +1251,9 @@ static void hw_var_set_bcn_func(struct adapter *Adapter, u8 variable, u8 *val)
 		usb_write8(Adapter, bcn_ctrl_reg, usb_read8(Adapter, bcn_ctrl_reg)&(~(EN_BCN_FUNCTION | EN_TXBCN_RPT)));
 }
 
-static void SetHwReg8188EU(struct adapter *Adapter, u8 variable, u8 *val)
+void rtw_hal_set_hwreg(struct adapter *Adapter, u8 variable, u8 *val)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *haldata = Adapter->HalData;
 	struct dm_priv	*pdmpriv = &haldata->dmpriv;
 	struct odm_dm_struct *podmpriv = &haldata->odmpriv;
 
@@ -1426,17 +1401,8 @@ static void SetHwReg8188EU(struct adapter *Adapter, u8 variable, u8 *val)
 				/* enable update TSF */
 				usb_write8(Adapter, REG_BCN_CTRL, usb_read8(Adapter, REG_BCN_CTRL)&(~BIT(4)));
 			}
-			if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE) {
-				usb_write32(Adapter, REG_RCR, usb_read32(Adapter, REG_RCR)|RCR_CBSSID_BCN);
-			} else {
-				if (Adapter->in_cta_test) {
-					u32 v = usb_read32(Adapter, REG_RCR);
-					v &= ~(RCR_CBSSID_DATA | RCR_CBSSID_BCN);/*  RCR_ADF */
-					usb_write32(Adapter, REG_RCR, v);
-				} else {
-					usb_write32(Adapter, REG_RCR, usb_read32(Adapter, REG_RCR)|RCR_CBSSID_BCN);
-				}
-			}
+
+			usb_write32(Adapter, REG_RCR, usb_read32(Adapter, REG_RCR)|RCR_CBSSID_BCN);
 		}
 		break;
 	case HW_VAR_MLME_JOIN:
@@ -1449,13 +1415,7 @@ static void SetHwReg8188EU(struct adapter *Adapter, u8 variable, u8 *val)
 				/* enable to rx data frame.Accept all data frame */
 				usb_write16(Adapter, REG_RXFLTMAP2, 0xFFFF);
 
-				if (Adapter->in_cta_test) {
-					u32 v = usb_read32(Adapter, REG_RCR);
-					v &= ~(RCR_CBSSID_DATA | RCR_CBSSID_BCN);/*  RCR_ADF */
-					usb_write32(Adapter, REG_RCR, v);
-				} else {
-					usb_write32(Adapter, REG_RCR, usb_read32(Adapter, REG_RCR)|RCR_CBSSID_DATA|RCR_CBSSID_BCN);
-				}
+				usb_write32(Adapter, REG_RCR, usb_read32(Adapter, REG_RCR)|RCR_CBSSID_DATA|RCR_CBSSID_BCN);
 
 				if (check_fwstate(pmlmepriv, WIFI_STATION_STATE))
 					RetryLimit = (haldata->CustomerID == RT_CID_CCX) ? 7 : 48;
@@ -1525,9 +1485,6 @@ static void SetHwReg8188EU(struct adapter *Adapter, u8 variable, u8 *val)
 	case HW_VAR_SEC_CFG:
 		usb_write8(Adapter, REG_SECCFG, *((u8 *)val));
 		break;
-	case HW_VAR_DM_FLAG:
-		podmpriv->SupportAbility = *((u8 *)val);
-		break;
 	case HW_VAR_DM_FUNC_OP:
 		if (val[0])
 			podmpriv->BK_SupportAbility = podmpriv->SupportAbility;
@@ -1793,14 +1750,11 @@ static void SetHwReg8188EU(struct adapter *Adapter, u8 variable, u8 *val)
 	}
 }
 
-static void GetHwReg8188EU(struct adapter *Adapter, u8 variable, u8 *val)
+void rtw_hal_get_hwreg(struct adapter *Adapter, u8 variable, u8 *val)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
-	struct odm_dm_struct *podmpriv = &haldata->odmpriv;
-
 	switch (variable) {
 	case HW_VAR_BASIC_RATE:
-		*((u16 *)(val)) = haldata->BasicRateSet;
+		*((u16 *)(val)) = Adapter->HalData->BasicRateSet;
 	case HW_VAR_TXPAUSE:
 		val[0] = usb_read8(Adapter, REG_TXPAUSE);
 		break;
@@ -1808,11 +1762,8 @@ static void GetHwReg8188EU(struct adapter *Adapter, u8 variable, u8 *val)
 		/* BCN_VALID, BIT16 of REG_TDECTRL = BIT0 of REG_TDECTRL+2 */
 		val[0] = (BIT(0) & usb_read8(Adapter, REG_TDECTRL+2)) ? true : false;
 		break;
-	case HW_VAR_DM_FLAG:
-		val[0] = podmpriv->SupportAbility;
-		break;
 	case HW_VAR_RF_TYPE:
-		val[0] = haldata->rf_type;
+		val[0] = RF_1T1R;
 		break;
 	case HW_VAR_FWLPS_RF_ON:
 		{
@@ -1833,13 +1784,13 @@ static void GetHwReg8188EU(struct adapter *Adapter, u8 variable, u8 *val)
 		}
 		break;
 	case HW_VAR_CURRENT_ANTENNA:
-		val[0] = haldata->CurAntenna;
+		val[0] = Adapter->HalData->CurAntenna;
 		break;
 	case HW_VAR_EFUSE_BYTES: /*  To get EFUE total used bytes, added by Roger, 2008.12.22. */
-		*((u16 *)(val)) = haldata->EfuseUsedBytes;
+		*((u16 *)(val)) = Adapter->HalData->EfuseUsedBytes;
 		break;
 	case HW_VAR_APFM_ON_MAC:
-		*val = haldata->bMacPwrCtrlOn;
+		*val = Adapter->HalData->bMacPwrCtrlOn;
 		break;
 	case HW_VAR_CHK_HI_QUEUE_EMPTY:
 		*val = ((usb_read32(Adapter, REG_HGQ_INFORMATION)&0x0000ff00) == 0) ? true : false;
@@ -1853,14 +1804,13 @@ static void GetHwReg8188EU(struct adapter *Adapter, u8 variable, u8 *val)
 /*	Description: */
 /*		Query setting of specified variable. */
 /*  */
-static u8
-GetHalDefVar8188EUsb(
+u8 rtw_hal_get_def_var(
 		struct adapter *Adapter,
 		enum hal_def_variable eVariable,
 		void *pValue
 	)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(Adapter);
+	struct hal_data_8188e *haldata = Adapter->HalData;
 	u8 bResult = _SUCCESS;
 
 	switch (eVariable) {
@@ -1948,7 +1898,7 @@ GetHalDefVar8188EUsb(
 	return bResult;
 }
 
-static void UpdateHalRAMask8188EUsb(struct adapter *adapt, u32 mac_id, u8 rssi_level)
+void UpdateHalRAMask8188EUsb(struct adapter *adapt, u32 mac_id, u8 rssi_level)
 {
 	u8 init_rate = 0;
 	u8 networkType, raid;
@@ -1956,7 +1906,7 @@ static void UpdateHalRAMask8188EUsb(struct adapter *adapt, u32 mac_id, u8 rssi_l
 	u8 shortGIrate = false;
 	int	supportRateNum = 0;
 	struct sta_info	*psta;
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(adapt);
+	struct odm_dm_struct *odmpriv = &adapt->HalData->odmpriv;
 	struct mlme_ext_priv	*pmlmeext = &adapt->mlmeextpriv;
 	struct mlme_ext_info	*pmlmeinfo = &pmlmeext->mlmext_info;
 	struct wlan_bssid_ex	*cur_network = &pmlmeinfo->network;
@@ -1995,7 +1945,7 @@ static void UpdateHalRAMask8188EUsb(struct adapter *adapt, u32 mac_id, u8 rssi_l
 		break;
 	}
 
-	rate_bitmap = ODM_Get_Rate_Bitmap(&haldata->odmpriv, mac_id, mask, rssi_level);
+	rate_bitmap = ODM_Get_Rate_Bitmap(odmpriv, mac_id, mask, rssi_level);
 	DBG_88E("%s => mac_id:%d, networkType:0x%02x, mask:0x%08x\n\t ==> rssi_level:%d, rate_bitmap:0x%08x\n",
 		__func__, mac_id, networkType, mask, rssi_level, rate_bitmap);
 
@@ -2003,15 +1953,14 @@ static void UpdateHalRAMask8188EUsb(struct adapter *adapt, u32 mac_id, u8 rssi_l
 
 	init_rate = get_highest_rate_idx(mask)&0x3f;
 
-	ODM_RA_UpdateRateInfo_8188E(&haldata->odmpriv, mac_id,
-				    raid, mask, shortGIrate);
+	ODM_RA_UpdateRateInfo_8188E(odmpriv, mac_id, raid, mask, shortGIrate);
 
 	/* set ra_id */
 	psta->raid = raid;
 	psta->init_rate = init_rate;
 }
 
-static void SetBeaconRelatedRegisters8188EUsb(struct adapter *adapt)
+void rtw_hal_bcn_related_reg_setting(struct adapter *adapt)
 {
 	u32 value32;
 	struct mlme_ext_priv	*pmlmeext = &adapt->mlmeextpriv;
@@ -2045,13 +1994,12 @@ static void SetBeaconRelatedRegisters8188EUsb(struct adapter *adapt)
 	usb_write8(adapt, bcn_ctrl_reg, usb_read8(adapt, bcn_ctrl_reg) | BIT(1));
 }
 
-static void rtl8188eu_init_default_value(struct adapter *adapt)
+void rtw_hal_def_value_init(struct adapter *adapt)
 {
-	struct hal_data_8188e *haldata;
+	struct hal_data_8188e *haldata = adapt->HalData;
 	struct pwrctrl_priv *pwrctrlpriv;
 	u8 i;
 
-	haldata = GET_HAL_DATA(adapt);
 	pwrctrlpriv = &adapt->pwrctrlpriv;
 
 	/* init default value */
@@ -2067,43 +2015,3 @@ static void rtl8188eu_init_default_value(struct adapter *adapt)
 	for (i = 0; i < HP_THERMAL_NUM; i++)
 		haldata->odmpriv.RFCalibrateInfo.ThermalValue_HP[i] = 0;
 }
-
-void rtl8188eu_set_hal_ops(struct adapter *adapt)
-{
-	struct hal_ops	*halfunc = &adapt->HalFunc;
-
-
-	adapt->HalData = kzalloc(sizeof(struct hal_data_8188e), GFP_KERNEL);
-	if (!adapt->HalData)
-		DBG_88E("cant not alloc memory for HAL DATA\n");
-
-	halfunc->hal_power_on = rtl8188eu_InitPowerOn;
-	halfunc->hal_init = &rtl8188eu_hal_init;
-	halfunc->hal_deinit = &rtl8188eu_hal_deinit;
-
-	halfunc->inirp_init = &rtl8188eu_inirp_init;
-	halfunc->inirp_deinit = &rtl8188eu_inirp_deinit;
-
-	halfunc->init_xmit_priv = &rtl8188eu_init_xmit_priv;
-
-	halfunc->init_recv_priv = &rtl8188eu_init_recv_priv;
-	halfunc->free_recv_priv = &rtl8188eu_free_recv_priv;
-	halfunc->InitSwLeds = &rtl8188eu_InitSwLeds;
-	halfunc->DeInitSwLeds = &rtl8188eu_DeInitSwLeds;
-
-	halfunc->init_default_value = &rtl8188eu_init_default_value;
-	halfunc->intf_chip_configure = &rtl8188eu_interface_configure;
-	halfunc->read_adapter_info = &_ReadAdapterInfo8188EU;
-
-	halfunc->SetHwRegHandler = &SetHwReg8188EU;
-	halfunc->GetHwRegHandler = &GetHwReg8188EU;
-	halfunc->GetHalDefVarHandler = &GetHalDefVar8188EUsb;
-
-	halfunc->UpdateRAMaskHandler = &UpdateHalRAMask8188EUsb;
-	halfunc->SetBeaconRelatedRegistersHandler = &SetBeaconRelatedRegisters8188EUsb;
-
-	halfunc->hal_xmit = &rtl8188eu_hal_xmit;
-	halfunc->mgnt_xmit = &rtl8188eu_mgnt_xmit;
-
-	rtl8188e_set_hal_ops(halfunc);
-}
diff --git a/drivers/staging/rtl8188eu/include/Hal8188EPhyCfg.h b/drivers/staging/rtl8188eu/include/Hal8188EPhyCfg.h
index 8990748a1919..0976a761b280 100644
--- a/drivers/staging/rtl8188eu/include/Hal8188EPhyCfg.h
+++ b/drivers/staging/rtl8188eu/include/Hal8188EPhyCfg.h
@@ -158,24 +158,6 @@ struct bb_reg_def {
 				 * Path A and B */
 };
 
-struct ant_sel_ofdm {
-	u32 r_tx_antenna:4;
-	u32 r_ant_l:4;
-	u32 r_ant_non_ht:4;
-	u32 r_ant_ht1:4;
-	u32 r_ant_ht2:4;
-	u32 r_ant_ht_s1:4;
-	u32 r_ant_non_ht_s1:4;
-	u32 OFDM_TXSC:2;
-	u32 reserved:2;
-};
-
-struct ant_sel_cck {
-	u8 r_cckrx_enable_2:2;
-	u8 r_cckrx_enable:2;
-	u8 r_ccktx_enable:4;
-};
-
 /*------------------------------Define structure----------------------------*/
 
 
diff --git a/drivers/staging/rtl8188eu/include/HalHWImg8188E_FW.h b/drivers/staging/rtl8188eu/include/HalHWImg8188E_FW.h
deleted file mode 100644
index dbb55247b0c6..000000000000
--- a/drivers/staging/rtl8188eu/include/HalHWImg8188E_FW.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/******************************************************************************
-*
-* Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
-*
-* This program is free software; you can redistribute it and/or modify it
-* under the terms of version 2 of the GNU General Public License as
-* published by the Free Software Foundation.
-*
-* This program is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-* more details.
-*
-******************************************************************************/
-
-#ifndef __INC_FW_8188E_HW_IMG_H
-#define __INC_FW_8188E_HW_IMG_H
-
-
-/******************************************************************************
-*                           FW_AP.TXT
-******************************************************************************/
-/******************************************************************************
-*                           FW_WoWLAN.TXT
-******************************************************************************/
-#define ArrayLength_8188E_FW_WoWLAN 15764
-extern const u8 Array_8188E_FW_WoWLAN[ArrayLength_8188E_FW_WoWLAN];
-
-#endif
diff --git a/drivers/staging/rtl8188eu/include/basic_types.h b/drivers/staging/rtl8188eu/include/basic_types.h
index 2c1676d2ac6e..69c4d49f43ab 100644
--- a/drivers/staging/rtl8188eu/include/basic_types.h
+++ b/drivers/staging/rtl8188eu/include/basic_types.h
@@ -137,8 +137,4 @@ value to host byte ordering.*/
 		((((u8)__val) & BIT_LEN_MASK_8(__bitlen)) << (__bitoffset)) \
 		)
 
-/*  Get the N-bytes aligment offset from the current length */
-#define	N_BYTE_ALIGMENT(__value, __aligment) ((__aligment == 1) ? \
-	(__value) : (((__value + __aligment - 1) / __aligment) * __aligment))
-
 #endif /* __BASIC_TYPES_H__ */
diff --git a/drivers/staging/rtl8188eu/include/drv_types.h b/drivers/staging/rtl8188eu/include/drv_types.h
index 55506a7da1a4..32326fd1dd24 100644
--- a/drivers/staging/rtl8188eu/include/drv_types.h
+++ b/drivers/staging/rtl8188eu/include/drv_types.h
@@ -53,30 +53,17 @@
 #define SPEC_DEV_ID_ASSIGN_IFNAME	BIT(5)
 
 struct registry_priv {
-	u8	chip_version;
-	u8	rfintfs;
-	u8	lbkmode;
-	u8	hci;
 	struct ndis_802_11_ssid	ssid;
-	u8	network_mode;	/* infra, ad-hoc, auto */
 	u8	channel;/* ad-hoc support requirement */
 	u8	wireless_mode;/* A, B, G, auto */
-	u8	scan_mode;/* active, passive */
-	u8	radio_enable;
 	u8	preamble;/* long, short, auto */
 	u8	vrtl_carrier_sense;/* Enable, Disable, Auto */
 	u8	vcs_type;/* RTS/CTS, CTS-to-self */
 	u16	rts_thresh;
 	u16	frag_thresh;
-	u8	adhoc_tx_pwr;
-	u8	soft_ap;
 	u8	power_mgnt;
 	u8	ips_mode;
 	u8	smart_ps;
-	u8	long_retry_lmt;
-	u8	short_retry_lmt;
-	u16	busy_thresh;
-	u8	ack_policy;
 	u8	mp_mode;
 	u8	software_encrypt;
 	u8	software_decrypt;
@@ -84,11 +71,6 @@ struct registry_priv {
 	  /* UAPSD */
 	u8	wmm_enable;
 	u8	uapsd_enable;
-	u8	uapsd_max_sp;
-	u8	uapsd_acbk_en;
-	u8	uapsd_acbe_en;
-	u8	uapsd_acvi_en;
-	u8	uapsd_acvo_en;
 
 	struct wlan_bssid_ex    dev_network;
 
@@ -97,10 +79,6 @@ struct registry_priv {
 	u8	ampdu_enable;/* for tx */
 	u8	rx_stbc;
 	u8	ampdu_amsdu;/* A-MPDU Supports A-MSDU is permitted */
-	u8	lowrate_two_xmit;
-
-	u8	rf_config;
-	u8	low_power;
 
 	u8	wifi_spec;/*  !turbo_mode */
 
@@ -112,9 +90,6 @@ struct registry_priv {
 
 	u8	usbss_enable;/* 0:disable,1:enable */
 	u8	hwpdn_mode;/* 0:disable,1:enable,2:decide by EFUSE config */
-	u8	hwpwrp_detect;/* 0:disable,1:enable */
-
-	u8	hw_wps_pbc;/* 0:disable,1:enable */
 
 	u8	max_roaming_times; /*  the max number driver will try */
 
@@ -129,12 +104,6 @@ struct registry_priv {
 	bool	monitor_enable;
 };
 
-/* For registry parameters */
-#define RGTRY_OFT(field) ((u32)offsetof(struct registry_priv, field))
-#define RGTRY_SZ(field)   sizeof(((struct registry_priv *)0)->field)
-#define BSSID_OFT(field) ((u32)offsetofT(struct wlan_bssid_ex, field))
-#define BSSID_SZ(field)   sizeof(((struct wlan_bssid_ex *)0)->field)
-
 #define MAX_CONTINUAL_URB_ERR		4
 
 struct dvobj_priv {
@@ -149,16 +118,11 @@ struct dvobj_priv {
 	u8	Queue2Pipe[HW_QUEUE_ENTRY];/* for out pipe mapping */
 
 /*-------- below is for USB INTERFACE --------*/
-
-	u8	nr_endpoint;
 	u8	ishighspeed;
 	u8	RtNumInPipes;
 	u8	RtNumOutPipes;
-	int	ep_num[5]; /* endpoint number */
 	struct mutex  usb_vendor_req_mutex;
 
-	u8 *usb_vendor_req_buf;
-
 	struct usb_interface *pusbintf;
 	struct usb_device *pusbdev;
 };
@@ -184,14 +148,7 @@ struct adapter {
 	struct	eeprom_priv eeprompriv;
 	struct	led_priv	ledpriv;
 
-#ifdef CONFIG_88EU_AP_MODE
-	struct	hostapd_priv	*phostapdpriv;
-#endif
-
-	struct wifidirect_info	wdinfo;
-
-	void *HalData;
-	struct hal_ops	HalFunc;
+	struct hal_data_8188e *HalData;
 
 	s32	bDriverStopped;
 	s32	bSurpriseRemoved;
@@ -199,20 +156,11 @@ struct adapter {
 	u8	hw_init_completed;
 
 	void *cmdThread;
-	void *evtThread;
 	void (*intf_start)(struct adapter *adapter);
 	void (*intf_stop)(struct adapter *adapter);
 	struct  net_device *pnetdev;
 	struct  net_device *pmondev;
 
-	/*  used by rtw_rereg_nd_name related function */
-	struct rereg_nd_name_data {
-		struct  net_device *old_pnetdev;
-		char old_ifname[IFNAMSIZ];
-		u8 old_ips_mode;
-		u8 old_bRegUseLed;
-	} rereg_nd_name_priv;
-
 	int bup;
 	struct net_device_stats stats;
 	struct iw_statistics iwstats;
@@ -223,23 +171,12 @@ struct adapter {
 	u8 bReadPortCancel;
 	u8 bWritePortCancel;
 	u8 bRxRSSIDisplay;
-	/* The driver will show up the desired channel number
-	 * when this flag is 1. */
-	u8 bNotifyChannelChange;
 
 	struct mutex hw_init_mutex;
-
-	spinlock_t br_ext_lock;
-
-	u8	fix_rate;
-
-	unsigned char     in_cta_test;
 };
 
 #define adapter_to_dvobj(adapter) (adapter->dvobj)
 
-int rtw_handle_dualmac(struct adapter *adapter, bool init);
-
 static inline u8 *myid(struct eeprom_priv *peepriv)
 {
 	return peepriv->mac_addr;
diff --git a/drivers/staging/rtl8188eu/include/hal_intf.h b/drivers/staging/rtl8188eu/include/hal_intf.h
index eaf939bd4103..fa032b0c12ff 100644
--- a/drivers/staging/rtl8188eu/include/hal_intf.h
+++ b/drivers/staging/rtl8188eu/include/hal_intf.h
@@ -58,7 +58,6 @@ enum hw_variables {
 	HW_VAR_SEC_CFG,
 	HW_VAR_BCN_VALID,
 	HW_VAR_RF_TYPE,
-	HW_VAR_DM_FLAG,
 	HW_VAR_DM_FUNC_OP,
 	HW_VAR_DM_FUNC_SET,
 	HW_VAR_DM_FUNC_CLR,
@@ -139,81 +138,6 @@ enum hal_intf_ps_func {
 	HAL_MAX_ID,
 };
 
-struct hal_ops {
-	u32	(*hal_power_on)(struct adapter *padapter);
-	u32	(*hal_init)(struct adapter *padapter);
-	u32	(*hal_deinit)(struct adapter *padapter);
-
-	void	(*free_hal_data)(struct adapter *padapter);
-
-	u32	(*inirp_init)(struct adapter *padapter);
-	u32	(*inirp_deinit)(struct adapter *padapter);
-
-	s32	(*init_xmit_priv)(struct adapter *padapter);
-
-	s32	(*init_recv_priv)(struct adapter *padapter);
-	void	(*free_recv_priv)(struct adapter *padapter);
-
-	void	(*InitSwLeds)(struct adapter *padapter);
-	void	(*DeInitSwLeds)(struct adapter *padapter);
-
-	void	(*dm_init)(struct adapter *padapter);
-	void	(*read_chip_version)(struct adapter *padapter);
-
-	void	(*init_default_value)(struct adapter *padapter);
-
-	void	(*intf_chip_configure)(struct adapter *padapter);
-
-	void	(*read_adapter_info)(struct adapter *padapter);
-
-	s32	(*interrupt_handler)(struct adapter *padapter);
-
-	void	(*set_bwmode_handler)(struct adapter *padapter,
-				      enum ht_channel_width Bandwidth,
-				      u8 Offset);
-	void	(*set_channel_handler)(struct adapter *padapter, u8 channel);
-
-	void	(*hal_dm_watchdog)(struct adapter *padapter);
-
-	void	(*SetHwRegHandler)(struct adapter *padapter, u8	variable,
-				   u8 *val);
-	void	(*GetHwRegHandler)(struct adapter *padapter, u8	variable,
-				   u8 *val);
-
-	u8	(*GetHalDefVarHandler)(struct adapter *padapter,
-				       enum hal_def_variable eVariable,
-				       void *pValue);
-
-	void	(*SetHalODMVarHandler)(struct adapter *padapter,
-				       enum hal_odm_variable eVariable,
-				       void *pValue1, bool bSet);
-
-	void	(*UpdateRAMaskHandler)(struct adapter *padapter,
-				       u32 mac_id, u8 rssi_level);
-	void	(*SetBeaconRelatedRegistersHandler)(struct adapter *padapter);
-
-	void	(*Add_RateATid)(struct adapter *adapter, u32 bitmap, u8 arg,
-				u8 rssi_level);
-
-	u8	(*AntDivBeforeLinkHandler)(struct adapter *adapter);
-	void	(*AntDivCompareHandler)(struct adapter *adapter,
-					struct wlan_bssid_ex *dst,
-					struct wlan_bssid_ex *src);
-	s32	(*hal_xmit)(struct adapter *padapter,
-			    struct xmit_frame *pxmitframe);
-	s32 (*mgnt_xmit)(struct adapter *padapter,
-			 struct xmit_frame *pmgntframe);
-	u32	(*read_rfreg)(struct adapter *padapter,
-			      enum rf_radio_path eRFPath, u32 RegAddr,
-			      u32 BitMask);
-
-	void (*sreset_init_value)(struct adapter *padapter);
-	u8 (*sreset_get_wifi_status)(struct adapter *padapter);
-
-	void (*hal_notch_filter)(struct adapter *adapter, bool enable);
-	void (*hal_reset_security_engine)(struct adapter *adapter);
-};
-
 enum rt_eeprom_type {
 	EEPROM_93C46,
 	EEPROM_93C56,
@@ -235,6 +159,9 @@ enum hardware_type {
 
 #define is_boot_from_eeprom(adapter) (adapter->eeprompriv.EepromOrEfuse)
 
+void UpdateHalRAMask8188EUsb(struct adapter *adapt, u32 mac_id, u8 rssi_level);
+u32 rtl8188eu_hal_deinit(struct adapter *Adapter);
+u32 rtl8188eu_hal_init(struct adapter *Adapter);
 void rtw_hal_def_value_init(struct adapter *padapter);
 
 void	rtw_hal_free_data(struct adapter *padapter);
@@ -262,7 +189,7 @@ void rtw_hal_set_odm_var(struct adapter *padapter,
 			 bool bSet);
 
 u32	rtw_hal_inirp_init(struct adapter *padapter);
-u32	rtw_hal_inirp_deinit(struct adapter *padapter);
+void	rtw_hal_inirp_deinit(struct adapter *padapter);
 
 s32	rtw_hal_xmit(struct adapter *padapter, struct xmit_frame *pxmitframe);
 s32	rtw_hal_mgnt_xmit(struct adapter *padapter,
@@ -270,7 +197,7 @@ s32	rtw_hal_mgnt_xmit(struct adapter *padapter,
 
 s32	rtw_hal_init_xmit_priv(struct adapter *padapter);
 
-s32	rtw_hal_init_recv_priv(struct adapter *padapter);
+int	rtw_hal_init_recv_priv(struct adapter *padapter);
 void	rtw_hal_free_recv_priv(struct adapter *padapter);
 
 void rtw_hal_update_ra_mask(struct adapter *padapter, u32 mac_id, u8 level);
@@ -296,7 +223,6 @@ void	rtw_hal_antdiv_rssi_compared(struct adapter *padapter,
 void rtw_hal_sreset_init(struct adapter *padapter);
 
 void rtw_hal_notch_filter(struct adapter *adapter, bool enable);
-void rtw_hal_reset_security_engine(struct adapter *adapter);
 
 void indicate_wx_scan_complete_event(struct adapter *padapter);
 u8 rtw_do_join(struct adapter *padapter);
diff --git a/drivers/staging/rtl8188eu/include/ieee80211.h b/drivers/staging/rtl8188eu/include/ieee80211.h
index d8284c84f09c..fc58621368c1 100644
--- a/drivers/staging/rtl8188eu/include/ieee80211.h
+++ b/drivers/staging/rtl8188eu/include/ieee80211.h
@@ -239,7 +239,7 @@ struct ieee_param {
 			u16 capability;
 			int flags;
 			u8 tx_supp_rates[16];
-			struct rtw_ieee80211_ht_cap ht_cap;
+			struct ieee80211_ht_cap ht_cap;
 		} add_sta;
 		struct {
 			u8	reserved[2];/* for set max_num_sta */
@@ -264,7 +264,7 @@ struct sta_data {
 	u32 sta_set;
 	u8 tx_supp_rates[16];
 	u32 tx_supp_rates_len;
-	struct rtw_ieee80211_ht_cap ht_cap;
+	struct ieee80211_ht_cap ht_cap;
 	u64	rx_pkts;
 	u64	rx_bytes;
 	u64	rx_drops;
@@ -291,62 +291,6 @@ struct sta_data {
 /* this is stolen from ipw2200 driver */
 #define IEEE_IBSS_MAC_HASH_SIZE 31
 
-struct ieee_ibss_seq {
-	u8 mac[ETH_ALEN];
-	u16 seq_num;
-	u16 frag_num;
-	unsigned long packet_time;
-	struct list_head list;
-};
-
-struct rtw_ieee80211_hdr {
-	__le16 frame_ctl;
-	__le16 duration_id;
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-	u16 seq_ctl;
-	u8 addr4[ETH_ALEN];
-} __packed;
-
-struct rtw_ieee80211_hdr_3addr {
-	__le16 frame_ctl;
-	__le16 duration_id;
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-	u16 seq_ctl;
-} __packed;
-
-struct rtw_ieee80211_hdr_qos {
-	__le16 frame_ctl;
-	__le16 duration_id;
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-	u16 seq_ctl;
-	u8 addr4[ETH_ALEN];
-	u16	qc;
-}  __packed;
-
-struct rtw_ieee80211_hdr_3addr_qos {
-	__le16 frame_ctl;
-	__le16 duration_id;
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-	u16 seq_ctl;
-	u16     qc;
-}  __packed;
-
-struct eapol {
-	u8 snap[6];
-	u16 ethertype;
-	u8 version;
-	u8 type;
-	u16 length;
-} __packed;
-
 enum eap_type {
 	EAP_PACKET = 0,
 	EAPOL_START,
@@ -552,83 +496,12 @@ struct ieee80211_snap_hdr {
 #define IEEE80211_NUM_CCK_RATES		4
 #define IEEE80211_OFDM_SHIFT_MASK_A	4
 
-/* NOTE: This data is for statistical purposes; not all hardware provides this
- *       information for frames received.  Not setting these will not cause
- *       any adverse affects. */
-struct ieee80211_rx_stats {
-	/* u32 mac_time[2]; */
-	s8 rssi;
-	u8 signal;
-	u8 noise;
-	u8 received_channel;
-	u16 rate; /* in 100 kbps */
-	/* u8 control; */
-	u8 mask;
-	u8 freq;
-	u16 len;
-};
-
 /* IEEE 802.11 requires that STA supports concurrent reception of at least
  * three fragmented frames. This define can be increased to support more
  * concurrent frames, but it should be noted that each entry can consume about
  * 2 kB of RAM and increasing cache size will slow down frame reassembly. */
 #define IEEE80211_FRAG_CACHE_LEN 4
 
-struct ieee80211_frag_entry {
-	u32 first_frag_time;
-	uint seq;
-	uint last_frag;
-	uint qos;   /* jackson */
-	uint tid;	/* jackson */
-	struct sk_buff *skb;
-	u8 src_addr[ETH_ALEN];
-	u8 dst_addr[ETH_ALEN];
-};
-
-struct ieee80211_stats {
-	uint tx_unicast_frames;
-	uint tx_multicast_frames;
-	uint tx_fragments;
-	uint tx_unicast_octets;
-	uint tx_multicast_octets;
-	uint tx_deferred_transmissions;
-	uint tx_single_retry_frames;
-	uint tx_multiple_retry_frames;
-	uint tx_retry_limit_exceeded;
-	uint tx_discards;
-	uint rx_unicast_frames;
-	uint rx_multicast_frames;
-	uint rx_fragments;
-	uint rx_unicast_octets;
-	uint rx_multicast_octets;
-	uint rx_fcs_errors;
-	uint rx_discards_no_buffer;
-	uint tx_discards_wrong_sa;
-	uint rx_discards_undecryptable;
-	uint rx_message_in_msg_fragments;
-	uint rx_message_in_bad_msg_fragments;
-};
-
-struct ieee80211_softmac_stats {
-	uint rx_ass_ok;
-	uint rx_ass_err;
-	uint rx_probe_rq;
-	uint tx_probe_rs;
-	uint tx_beacons;
-	uint rx_auth_rq;
-	uint rx_auth_rs_ok;
-	uint rx_auth_rs_err;
-	uint tx_auth_rq;
-	uint no_auth_rs;
-	uint no_ass_rs;
-	uint tx_ass_rq;
-	uint rx_ass_rq;
-	uint tx_probe_rq;
-	uint reassoc;
-	uint swtxstop;
-	uint swtxawake;
-};
-
 #define SEC_KEY_1	(1<<0)
 #define SEC_KEY_2	(1<<1)
 #define SEC_KEY_3	(1<<2)
@@ -648,42 +521,6 @@ struct ieee80211_softmac_stats {
 #define WEP_KEYS 4
 #define WEP_KEY_LEN 13
 
-struct ieee80211_security {
-	u16 active_key:2,
-	enabled:1,
-	auth_mode:2,
-	auth_algo:4,
-	unicast_uses_group:1;
-	u8 key_sizes[WEP_KEYS];
-	u8 keys[WEP_KEYS][WEP_KEY_LEN];
-	u8 level;
-	u16 flags;
-} __packed;
-
-/*
-
- 802.11 data frame from AP
-
-      ,-------------------------------------------------------------------.
-Bytes |  2   |  2   |    6    |    6    |    6    |  2   | 0..2312 |   4  |
-      |------|------|---------|---------|---------|------|---------|------|
-Desc. | ctrl | dura |  DA/RA  |   TA    |    SA   | Sequ |  frame  |  fcs |
-      |      | tion | (BSSID) |	 |	 | ence |  data   |      |
-      `-------------------------------------------------------------------'
-
-Total: 28-2340 bytes
-
-*/
-
-struct ieee80211_header_data {
-	u16 frame_ctl;
-	u16 duration_id;
-	u8 addr1[6];
-	u8 addr2[6];
-	u8 addr3[6];
-	u16 seq_ctrl;
-};
-
 #define BEACON_PROBE_SSID_ID_POSITION 12
 
 /* Management Frame Information Element Types */
@@ -700,81 +537,9 @@ struct ieee80211_header_data {
 #define MFIE_TYPE_RATES_EX	50
 #define MFIE_TYPE_GENERIC	221
 
-struct ieee80211_info_element_hdr {
-	u8 id;
-	u8 len;
-} __packed;
-
-struct ieee80211_info_element {
-	u8 id;
-	u8 len;
-	u8 data[0];
-} __packed;
-
-/*
- * These are the data types that can make up management packets
- *
-	u16 auth_algorithm;
-	u16 auth_sequence;
-	u16 beacon_interval;
-	u16 capability;
-	u8 current_ap[ETH_ALEN];
-	u16 listen_interval;
-	struct {
-		u16 association_id:14, reserved:2;
-	} __packed;
-	u32 time_stamp[2];
-	u16 reason;
-	u16 status;
-*/
-
 #define IEEE80211_DEFAULT_TX_ESSID "Penguin"
 #define IEEE80211_DEFAULT_BASIC_RATE 10
 
-struct ieee80211_authentication {
-	struct ieee80211_header_data header;
-	u16 algorithm;
-	u16 transaction;
-	u16 status;
-	/* struct ieee80211_info_element_hdr info_element; */
-} __packed;
-
-struct ieee80211_probe_response {
-	struct ieee80211_header_data header;
-	u32 time_stamp[2];
-	u16 beacon_interval;
-	u16 capability;
-	struct ieee80211_info_element info_element;
-} __packed;
-
-struct ieee80211_probe_request {
-	struct ieee80211_header_data header;
-} __packed;
-
-struct ieee80211_assoc_request_frame {
-	struct rtw_ieee80211_hdr_3addr header;
-	u16 capability;
-	u16 listen_interval;
-	struct ieee80211_info_element_hdr info_element;
-} __packed;
-
-struct ieee80211_assoc_response_frame {
-	struct rtw_ieee80211_hdr_3addr header;
-	u16 capability;
-	u16 status;
-	u16 aid;
-} __packed;
-
-struct ieee80211_txb {
-	u8 nr_frags;
-	u8 encrypted;
-	u16 reserved;
-	u16 frag_size;
-	u16 payload_size;
-	struct sk_buff *fragments[0];
-};
-
-
 /* SWEEP TABLE ENTRIES NUMBER*/
 #define MAX_SWEEP_TAB_ENTRIES		  42
 #define MAX_SWEEP_TAB_ENTRIES_PER_PACKET  7
@@ -872,11 +637,6 @@ static inline int is_broadcast_mac_addr(const u8 *addr)
 #define CFG_IEEE80211_RESERVE_FCS (1<<0)
 #define CFG_IEEE80211_COMPUTE_FCS (1<<1)
 
-struct tx_pending {
-	int frag;
-	struct ieee80211_txb *txb;
-};
-
 #define MAXTID	16
 
 #define IEEE_A	    (1<<0)
@@ -1096,20 +856,8 @@ enum secondary_ch_offset {
 	SCA = 1, /* secondary channel above */
 	SCB = 3,  /* secondary channel below */
 };
-u8 secondary_ch_offset_to_hal_ch_offset(u8 ch_offset);
-u8 hal_ch_offset_to_secondary_ch_offset(u8 ch_offset);
-u8 *rtw_set_ie_ch_switch(u8 *buf, u32 *buf_len, u8 ch_switch_mode,
-			 u8 new_ch, u8 ch_switch_cnt);
-u8 *rtw_set_ie_secondary_ch_offset(u8 *buf, u32 *buf_len,
-				   u8 secondary_ch_offset);
-u8 *rtw_set_ie_mesh_ch_switch_parm(u8 *buf, u32 *buf_len, u8 ttl,
-				   u8 flags, u16 reason, u16 precedence);
 
 u8 *rtw_get_ie(u8 *pbuf, int index, int *len, int limit);
-u8 *rtw_get_ie_ex(u8 *in_ie, uint in_len, u8 eid, u8 *oui,
-		  u8 oui_len, u8 *ie, uint *ielen);
-int rtw_ies_remove_ie(u8 *ies, uint *ies_len, uint offset,
-		      u8 eid, u8 *oui, u8 oui_len);
 
 void rtw_set_supported_rate(u8 *SupportedRates, uint mode);
 
@@ -1133,19 +881,6 @@ u8 *rtw_get_wps_attr(u8 *wps_ie, uint wps_ielen, u16 target_attr_id,
 u8 *rtw_get_wps_attr_content(u8 *wps_ie, uint wps_ielen, u16 target_attr_id,
 			     u8 *buf_content, uint *len_content);
 
-/**
- * for_each_ie - iterate over continuous IEs
- * @ie:
- * @buf:
- * @buf_len:
- */
-#define for_each_ie(ie, buf, buf_len) \
-	for (ie = (void *)buf; (((u8 *)ie) - ((u8 *)buf) + 1) < buf_len;	\
-		ie = (void *)(((u8 *)ie) + *(((u8 *)ie)+1) + 2))
-
-void dump_ies(u8 *buf, u32 buf_len);
-void dump_wps_ie(u8 *ie, u32 ie_len);
-
 uint	rtw_get_rateset_len(u8	*rateset);
 
 struct registry_priv;
@@ -1167,8 +902,4 @@ void rtw_macaddr_cfg(u8 *mac_addr);
 u16 rtw_mcs_rate(u8 rf_type, u8 bw_40MHz, u8 short_GI_20, u8 short_GI_40,
 		 unsigned char *MCS_rate);
 
-int rtw_action_frame_parse(const u8 *frame, u32 frame_len, u8 *category,
-			   u8 *action);
-const char *action_public_str(u8 action);
-
 #endif /* IEEE80211_H */
diff --git a/drivers/staging/rtl8188eu/include/odm.h b/drivers/staging/rtl8188eu/include/odm.h
index dbebf17f36d3..805f52e108b2 100644
--- a/drivers/staging/rtl8188eu/include/odm.h
+++ b/drivers/staging/rtl8188eu/include/odm.h
@@ -315,22 +315,6 @@ enum odm_ability {
 	ODM_PSD2AFH		= 0x00000800
 };
 
-/*  2011/20/20 MH For MP driver RT_WLAN_STA =  struct sta_info */
-/*  Please declare below ODM relative info in your STA info structure. */
-
-struct odm_sta_info {
-	/*  Driver Write */
-	bool	bUsed;		/*  record the sta status link or not? */
-	u8	IOTPeer;	/*  Enum value.	HT_IOT_PEER_E */
-
-	/*  ODM Write */
-	/* 1 PHY_STATUS_INFO */
-	u8	RSSI_Path[4];		/*  */
-	u8	RSSI_Ave;
-	u8	RXEVM[4];
-	u8	RXSNR[4];
-};
-
 /*  2011/10/20 MH Define Common info enum for all team. */
 
 enum odm_common_info_def {
@@ -740,8 +724,6 @@ struct odm_dm_struct {
 	u32	SupportICType;
 	/*  Cut Version TestChip/A-cut/B-cut... = 0/1/2/3/... */
 	u8	CutVersion;
-	/*  RF Type 4T4R/3T3R/2T2R/1T2R/1T1R/... */
-	u8	RFType;
 	/*  Board Type Normal/HighPower/MiniCard/SLIM/Combo/. = 0/1/2/3/4/. */
 	u8	BoardType;
 	/*  with external LNA  NO/Yes = 0/1 */
diff --git a/drivers/staging/rtl8188eu/include/osdep_intf.h b/drivers/staging/rtl8188eu/include/osdep_intf.h
index 54fca79827e3..dbd7dc4f87dd 100644
--- a/drivers/staging/rtl8188eu/include/osdep_intf.h
+++ b/drivers/staging/rtl8188eu/include/osdep_intf.h
@@ -34,7 +34,6 @@ int rtw_ioctl(struct net_device *dev, struct ifreq *rq, int cmd);
 int rtw_init_netdev_name(struct net_device *pnetdev, const char *ifname);
 struct net_device *rtw_init_netdev(struct adapter *padapter);
 u16 rtw_recv_select_queue(struct sk_buff *skb);
-void rtw_proc_remove_one(struct net_device *dev);
 
 int pm_netdev_open(struct net_device *pnetdev, u8 bnormal);
 void rtw_ips_dev_unload(struct adapter *padapter);
diff --git a/drivers/staging/rtl8188eu/include/osdep_service.h b/drivers/staging/rtl8188eu/include/osdep_service.h
index 5475956c5ee5..9047b6dec9ed 100644
--- a/drivers/staging/rtl8188eu/include/osdep_service.h
+++ b/drivers/staging/rtl8188eu/include/osdep_service.h
@@ -35,7 +35,7 @@
 #include <asm/byteorder.h>
 #include <linux/atomic.h>
 #include <linux/io.h>
-#include <linux/semaphore.h>
+#include <linux/mutex.h>
 #include <linux/sem.h>
 #include <linux/sched.h>
 #include <linux/etherdevice.h>
@@ -44,7 +44,6 @@
 #include <linux/if_arp.h>
 #include <linux/rtnetlink.h>
 #include <linux/delay.h>
-#include <linux/proc_fs.h>	/*  Necessary because we use the proc fs */
 #include <linux/interrupt.h>	/*  for struct tasklet_struct */
 #include <linux/ip.h>
 #include <linux/kthread.h>
@@ -78,16 +77,12 @@ u8 *_rtw_malloc(u32 sz);
 
 void *rtw_malloc2d(int h, int w, int size);
 
-u32  _rtw_down_sema(struct semaphore *sema);
-
 void _rtw_init_queue(struct __queue *pqueue);
 
 struct rtw_netdev_priv_indicator {
 	void *priv;
-	u32 sizeof_priv;
 };
-struct net_device *rtw_alloc_etherdev_with_old_priv(int sizeof_priv,
-						    void *old_priv);
+struct net_device *rtw_alloc_etherdev_with_old_priv(void *old_priv);
 
 #define rtw_netdev_priv(netdev)					\
 	(((struct rtw_netdev_priv_indicator *)netdev_priv(netdev))->priv)
diff --git a/drivers/staging/rtl8188eu/include/phy.h b/drivers/staging/rtl8188eu/include/phy.h
index 9a9ab82a8ed3..cd387e998574 100644
--- a/drivers/staging/rtl8188eu/include/phy.h
+++ b/drivers/staging/rtl8188eu/include/phy.h
@@ -11,17 +11,13 @@ bool rtl88eu_phy_bb_config(struct adapter *adapt);
 
 u32 phy_query_bb_reg(struct adapter *adapt, u32 regaddr, u32 bitmask);
 void phy_set_bb_reg(struct adapter *adapt, u32 regaddr, u32 bitmask, u32 data);
-u32 phy_query_rf_reg(struct adapter *adapt, enum rf_radio_path rf_path,
+u32 rtw_hal_read_rfreg(struct adapter *adapt, enum rf_radio_path rf_path,
 		     u32 reg_addr, u32 bit_mask);
 void phy_set_rf_reg(struct adapter *adapt, enum rf_radio_path rf_path,
 		    u32 reg_addr, u32 bit_mask, u32 data);
 
 void phy_set_tx_power_level(struct adapter *adapt, u8 channel);
 
-void phy_set_bw_mode(struct adapter *adapt, enum ht_channel_width bandwidth,
-		     unsigned char offset);
-void phy_sw_chnl(struct adapter *adapt, u8 channel);
-
 void rtl88eu_dm_txpower_track_adjust(struct odm_dm_struct *dm_odm,
 				     u8 type, u8 *dir, u32 *out_write);
 
diff --git a/drivers/staging/rtl8188eu/include/recv_osdep.h b/drivers/staging/rtl8188eu/include/recv_osdep.h
index cad31587c30a..7550d58f6b5b 100644
--- a/drivers/staging/rtl8188eu/include/recv_osdep.h
+++ b/drivers/staging/rtl8188eu/include/recv_osdep.h
@@ -26,13 +26,9 @@ void _rtw_free_recv_priv(struct recv_priv *precvpriv);
 s32  rtw_recv_entry(struct recv_frame *precv_frame);
 int rtw_recv_indicatepkt(struct adapter *adapter,
 			 struct recv_frame *recv_frame);
-void rtw_recv_returnpacket(struct  net_device *cnxt, struct sk_buff *retpkt);
 
 void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup);
 
-int rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *padapter);
-void rtw_free_recv_priv(struct recv_priv *precvpriv);
-
 void rtw_os_recv_resource_alloc(struct recv_frame *recvfr);
 
 int rtw_os_recvbuf_resource_alloc(struct adapter *adapt, struct recv_buf *buf);
diff --git a/drivers/staging/rtl8188eu/include/rtl8188e_cmd.h b/drivers/staging/rtl8188eu/include/rtl8188e_cmd.h
index 4d7d804658c2..042b4ec656c8 100644
--- a/drivers/staging/rtl8188eu/include/rtl8188e_cmd.h
+++ b/drivers/staging/rtl8188eu/include/rtl8188e_cmd.h
@@ -49,12 +49,6 @@ enum RTL8188E_H2C_CMD_ID {
 	 H2C_RESET_TSF			= 0xc0,
 };
 
-struct cmd_msg_parm {
-	u8 eid; /* element id */
-	u8 sz; /*  sz */
-	u8 buf[6];
-};
-
 enum {
 	PWRS
 };
@@ -67,15 +61,6 @@ struct setpwrmode_parm {
 	u8 PwrState;/* AllON(0x0c),RFON(0x04),RFOFF(0x00) */
 };
 
-struct H2C_SS_RFOFF_PARAM {
-	u8 ROFOn; /*  1: on, 0:off */
-	u16 gpio_period; /*  unit: 1024 us */
-} __packed;
-
-struct joinbssrpt_parm {
-	u8 OpMode;	/*  RT_MEDIA_STATUS */
-};
-
 struct rsvdpage_loc {
 	u8 LocProbeRsp;
 	u8 LocPsPoll;
@@ -84,26 +69,9 @@ struct rsvdpage_loc {
 	u8 LocBTQosNull;
 };
 
-struct P2P_PS_Offload_t {
-	u8 Offload_En:1;
-	u8 role:1; /*  1: Owner, 0: Client */
-	u8 CTWindow_En:1;
-	u8 NoA0_En:1;
-	u8 NoA1_En:1;
-	u8 AllStaSleep:1; /*  Only valid in Owner */
-	u8 discovery:1;
-	u8 rsvd:1;
-};
-
-struct P2P_PS_CTWPeriod_t {
-	u8 CTWPeriod;	/* TU */
-};
-
 /*  host message to firmware cmd */
 void rtl8188e_set_FwPwrMode_cmd(struct adapter *padapter, u8 Mode);
 void rtl8188e_set_FwJoinBssReport_cmd(struct adapter *padapter, u8 mstatus);
-void rtl8188e_Add_RateATid(struct adapter *padapter, u32 bitmap, u8 arg,
-			   u8 rssi_level);
 
 void rtl8188e_set_FwMediaStatus_cmd(struct adapter *adapt, __le16 mstatus_rpt);
 
diff --git a/drivers/staging/rtl8188eu/include/rtl8188e_dm.h b/drivers/staging/rtl8188eu/include/rtl8188e_dm.h
index 4190112a50bf..c0ffd98d7617 100644
--- a/drivers/staging/rtl8188eu/include/rtl8188e_dm.h
+++ b/drivers/staging/rtl8188eu/include/rtl8188e_dm.h
@@ -45,12 +45,9 @@ struct	dm_priv {
 	u8	PowerIndex_backup[6];
 };
 
-void rtl8188e_init_dm_priv(struct adapter *adapt);
 void rtl8188e_InitHalDm(struct adapter *adapt);
-void rtl8188e_HalDmWatchDog(struct adapter *adapt);
 
 void AntDivCompare8188E(struct adapter *adapt, struct wlan_bssid_ex *dst,
 			struct wlan_bssid_ex *src);
-u8 AntDivBeforeLink8188E(struct adapter *adapt);
 
 #endif
diff --git a/drivers/staging/rtl8188eu/include/rtl8188e_hal.h b/drivers/staging/rtl8188eu/include/rtl8188e_hal.h
index 9dd5c293a54b..7c81e3f3d12e 100644
--- a/drivers/staging/rtl8188eu/include/rtl8188e_hal.h
+++ b/drivers/staging/rtl8188eu/include/rtl8188e_hal.h
@@ -202,7 +202,6 @@ struct hal_data_8188e {
 
 	/* rf_ctrl */
 	u8	rf_chip;
-	u8	rf_type;
 	u8	NumTotalRFPath;
 
 	u8	BoardType;
@@ -351,10 +350,6 @@ struct hal_data_8188e {
 	u8	UsbRxAggPageTimeout;
 };
 
-#define GET_HAL_DATA(__pAdapter)				\
-	((struct hal_data_8188e *)((__pAdapter)->HalData))
-#define GET_RF_TYPE(priv)		(GET_HAL_DATA(priv)->rf_type)
-
 /*  rtl8188e_hal_init.c */
 void _8051Reset88E(struct adapter *padapter);
 void rtl8188e_InitializeFirmwareVars(struct adapter *padapter);
@@ -385,8 +380,6 @@ void Hal_EfuseParseBoardType88E(struct adapter *pAdapter, u8 *hwinfo,
 void Hal_ReadPowerSavingMode88E(struct adapter *pAdapter, u8 *hwinfo,
 				bool AutoLoadFail);
 
-void rtl8188e_set_hal_ops(struct hal_ops *pHalFunc);
-
 /*  register */
 
 void rtl8188e_start_thread(struct adapter *padapter);
diff --git a/drivers/staging/rtl8188eu/include/rtl8188e_led.h b/drivers/staging/rtl8188eu/include/rtl8188e_led.h
index fca6d8c81e90..d1ad6aa8c1e0 100644
--- a/drivers/staging/rtl8188eu/include/rtl8188e_led.h
+++ b/drivers/staging/rtl8188eu/include/rtl8188e_led.h
@@ -22,8 +22,6 @@
 /*  */
 /*  Interface to manipulate LED objects. */
 /*  */
-void rtl8188eu_InitSwLeds(struct adapter *padapter);
-void rtl8188eu_DeInitSwLeds(struct adapter *padapter);
 void SwLedOn(struct adapter *padapter, struct LED_871x *pLed);
 void SwLedOff(struct adapter *padapter, struct LED_871x *pLed);
 
diff --git a/drivers/staging/rtl8188eu/include/rtl8188e_recv.h b/drivers/staging/rtl8188eu/include/rtl8188e_recv.h
index 54048bc826e5..80832a5f0732 100644
--- a/drivers/staging/rtl8188eu/include/rtl8188e_recv.h
+++ b/drivers/staging/rtl8188eu/include/rtl8188e_recv.h
@@ -51,12 +51,11 @@ enum rx_packet_type {
 };
 
 #define INTERRUPT_MSG_FORMAT_LEN 60
-s32 rtl8188eu_init_recv_priv(struct adapter *padapter);
-void rtl8188eu_free_recv_priv(struct adapter *padapter);
 void rtl8188eu_recv_hdl(struct adapter *padapter, struct recv_buf *precvbuf);
 void rtl8188eu_recv_tasklet(void *priv);
 void rtl8188e_query_rx_phy_status(struct recv_frame *fr, struct phy_stat *phy);
-void rtl8188e_process_phy_info(struct adapter *padapter, void *prframe);
+void rtl8188e_process_phy_info(struct adapter *padapter,
+			       struct recv_frame *prframe);
 void update_recvframe_phyinfo_88e(struct recv_frame *fra, struct phy_stat *phy);
 void update_recvframe_attrib_88e(struct recv_frame *fra,
 				 struct recv_stat *stat);
diff --git a/drivers/staging/rtl8188eu/include/rtl8188e_xmit.h b/drivers/staging/rtl8188eu/include/rtl8188e_xmit.h
index 65a63df2077f..66205b782721 100644
--- a/drivers/staging/rtl8188eu/include/rtl8188e_xmit.h
+++ b/drivers/staging/rtl8188eu/include/rtl8188e_xmit.h
@@ -154,14 +154,11 @@ struct txrpt_ccx_88e {
 void rtl8188e_fill_fake_txdesc(struct adapter *padapter, u8 *pDesc,
 			       u32 BufferLen, u8 IsPsPoll, u8 IsBTQosNull);
 s32 rtl8188eu_init_xmit_priv(struct adapter *padapter);
-s32 rtl8188eu_hal_xmit(struct adapter *padapter, struct xmit_frame *frame);
-s32 rtl8188eu_mgnt_xmit(struct adapter *padapter, struct xmit_frame *frame);
 s32 rtl8188eu_xmit_buf_handler(struct adapter *padapter);
 #define hal_xmit_handler rtl8188eu_xmit_buf_handler
 void rtl8188eu_xmit_tasklet(void *priv);
 s32 rtl8188eu_xmitframe_complete(struct adapter *padapter,
-				 struct xmit_priv *pxmitpriv,
-				 struct xmit_buf *pxmitbuf);
+				 struct xmit_priv *pxmitpriv);
 
 void dump_txrpt_ccx_88e(void *buf);
 void handle_txrpt_ccx_88e(struct adapter *adapter, u8 *buf);
diff --git a/drivers/staging/rtl8188eu/include/rtw_ap.h b/drivers/staging/rtl8188eu/include/rtw_ap.h
index b820684bc3fe..e8dd6d4407aa 100644
--- a/drivers/staging/rtl8188eu/include/rtw_ap.h
+++ b/drivers/staging/rtl8188eu/include/rtw_ap.h
@@ -50,7 +50,6 @@ void ap_sta_info_defer_update(struct adapter *padapter, struct sta_info *psta);
 u8 ap_free_sta(struct adapter *padapter, struct sta_info *psta,
 	       bool active, u16 reason);
 int rtw_sta_flush(struct adapter *padapter);
-int rtw_ap_inform_ch_switch(struct adapter *padapter, u8 new_ch, u8 ch_offset);
 void start_ap_mode(struct adapter *padapter);
 void stop_ap_mode(struct adapter *padapter);
 #endif /* end of CONFIG_88EU_AP_MODE */
diff --git a/drivers/staging/rtl8188eu/include/rtw_cmd.h b/drivers/staging/rtl8188eu/include/rtw_cmd.h
index 08ca59217cb7..18a6530c9dde 100644
--- a/drivers/staging/rtl8188eu/include/rtw_cmd.h
+++ b/drivers/staging/rtl8188eu/include/rtw_cmd.h
@@ -39,8 +39,8 @@ struct cmd_obj {
 };
 
 struct cmd_priv {
-	struct semaphore cmd_queue_sema;
-	struct semaphore terminate_cmdthread_sema;
+	struct completion cmd_queue_comp;
+	struct completion terminate_cmdthread_comp;
 	struct __queue cmd_queue;
 	u8 cmdthd_running;
 	struct adapter *padapter;
@@ -210,34 +210,6 @@ struct set_assocsta_rsp {
 };
 
 /*
-	Caller Ad-Hoc/AP
-
-	Command mode
-
-	This is to force fw to del an sta_data entry per driver's request
-
-	FW will invalidate the cam entry associated with it.
-
-*/
-struct del_assocsta_parm {
-	u8	addr[ETH_ALEN];
-};
-
-/*
-Caller Mode: AP/Ad-HoC(M)
-
-Notes: To notify fw that given staid has changed its power state
-
-Command Mode
-
-*/
-struct setstapwrstate_parm {
-	u8	staid;
-	u8	status;
-	u8	hwaddr[6];
-};
-
-/*
 	Notes: This command is used for H2C/C2H loopback testing
 
 	mac[0] == 0
@@ -312,8 +284,6 @@ struct SetChannelPlan_param {
 	u8 channel_plan;
 };
 
-#define GEN_CMD_CODE(cmd)	cmd ## _CMD_
-
 /*
 
 Result:
@@ -376,42 +346,42 @@ struct _cmd_callback {
 };
 
 enum rtw_h2c_cmd {
-	GEN_CMD_CODE(_JoinBss),
-	GEN_CMD_CODE(_DisConnect),
-	GEN_CMD_CODE(_CreateBss),
-	GEN_CMD_CODE(_SetOpMode),
-	GEN_CMD_CODE(_SiteSurvey),
-	GEN_CMD_CODE(_SetAuth),
-	GEN_CMD_CODE(_SetKey),
-	GEN_CMD_CODE(_SetStaKey),
-	GEN_CMD_CODE(_SetAssocSta),
-	GEN_CMD_CODE(_AddBAReq),
-	GEN_CMD_CODE(_SetChannel),
-	GEN_CMD_CODE(_TX_Beacon),
-	GEN_CMD_CODE(_Set_MLME_EVT),
-	GEN_CMD_CODE(_Set_Drv_Extra),
-	GEN_CMD_CODE(_SetChannelPlan),
+	_JoinBss_CMD_,
+	_DisConnect_CMD_,
+	_CreateBss_CMD_,
+	_SetOpMode_CMD_,
+	_SiteSurvey_CMD_,
+	_SetAuth_CMD_,
+	_SetKey_CMD_,
+	_SetStaKey_CMD_,
+	_SetAssocSta_CMD_,
+	_AddBAReq_CMD_,
+	_SetChannel_CMD_,
+	_TX_Beacon_CMD_,
+	_Set_MLME_EVT_CMD_,
+	_Set_Drv_Extra_CMD_,
+	_SetChannelPlan_CMD_,
 
 	MAX_H2CCMD
 };
 
 #ifdef _RTW_CMD_C_
 static struct _cmd_callback	rtw_cmd_callback[] = {
-	{GEN_CMD_CODE(_JoinBss), &rtw_joinbss_cmd_callback},
-	{GEN_CMD_CODE(_DisConnect), &rtw_disassoc_cmd_callback},
-	{GEN_CMD_CODE(_CreateBss), &rtw_createbss_cmd_callback},
-	{GEN_CMD_CODE(_SetOpMode), NULL},
-	{GEN_CMD_CODE(_SiteSurvey), &rtw_survey_cmd_callback},
-	{GEN_CMD_CODE(_SetAuth), NULL},
-	{GEN_CMD_CODE(_SetKey), NULL},
-	{GEN_CMD_CODE(_SetStaKey), &rtw_setstaKey_cmdrsp_callback},
-	{GEN_CMD_CODE(_SetAssocSta), &rtw_setassocsta_cmdrsp_callback},
-	{GEN_CMD_CODE(_AddBAReq), NULL},
-	{GEN_CMD_CODE(_SetChannel), NULL},
-	{GEN_CMD_CODE(_TX_Beacon), NULL},
-	{GEN_CMD_CODE(_Set_MLME_EVT), NULL},
-	{GEN_CMD_CODE(_Set_Drv_Extra), NULL},
-	{GEN_CMD_CODE(_SetChannelPlan), NULL},
+	{_JoinBss_CMD_, &rtw_joinbss_cmd_callback},
+	{_DisConnect_CMD_, &rtw_disassoc_cmd_callback},
+	{_CreateBss_CMD_, &rtw_createbss_cmd_callback},
+	{_SetOpMode_CMD_, NULL},
+	{_SiteSurvey_CMD_, &rtw_survey_cmd_callback},
+	{_SetAuth_CMD_, NULL},
+	{_SetKey_CMD_, NULL},
+	{_SetStaKey_CMD_, &rtw_setstaKey_cmdrsp_callback},
+	{_SetAssocSta_CMD_, &rtw_setassocsta_cmdrsp_callback},
+	{_AddBAReq_CMD_, NULL},
+	{_SetChannel_CMD_, NULL},
+	{_TX_Beacon_CMD_, NULL},
+	{_Set_MLME_EVT_CMD_, NULL},
+	{_Set_Drv_Extra_CMD_, NULL},
+	{_SetChannelPlan_CMD_, NULL},
 };
 #endif
 
diff --git a/drivers/staging/rtl8188eu/include/rtw_debug.h b/drivers/staging/rtl8188eu/include/rtw_debug.h
index 7ed4cada7efa..95590a1a7b1b 100644
--- a/drivers/staging/rtl8188eu/include/rtw_debug.h
+++ b/drivers/staging/rtl8188eu/include/rtw_debug.h
@@ -129,133 +129,12 @@ int proc_get_read_reg(char *page, char **start,
 int proc_set_read_reg(struct file *file, const char __user *buffer,
 		      unsigned long count, void *data);
 
-int proc_get_fwstate(char *page, char **start,
-		     off_t offset, int count,
-		     int *eof, void *data);
-int proc_get_sec_info(char *page, char **start,
-		      off_t offset, int count,
-		      int *eof, void *data);
-int proc_get_mlmext_state(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-int proc_get_qos_option(char *page, char **start,
-			off_t offset, int count,
-			int *eof, void *data);
-int proc_get_ht_option(char *page, char **start,
-		       off_t offset, int count,
-		       int *eof, void *data);
-int proc_get_rf_info(char *page, char **start,
-		     off_t offset, int count,
-		     int *eof, void *data);
-int proc_get_ap_info(char *page, char **start,
-		     off_t offset, int count,
-		     int *eof, void *data);
-
 int proc_get_adapter_state(char *page, char **start,
 			   off_t offset, int count,
 			   int *eof, void *data);
 
-int proc_get_trx_info(char *page, char **start,
-		      off_t offset, int count,
-		      int *eof, void *data);
-
-int proc_get_mac_reg_dump1(char *page, char **start,
-			   off_t offset, int count,
-			   int *eof, void *data);
-
-int proc_get_mac_reg_dump2(char *page, char **start,
-			   off_t offset, int count,
-			   int *eof, void *data);
-
-int proc_get_mac_reg_dump3(char *page, char **start,
-			   off_t offset, int count,
-			   int *eof, void *data);
-
-int proc_get_bb_reg_dump1(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-int proc_get_bb_reg_dump2(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-int proc_get_bb_reg_dump3(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-int proc_get_rf_reg_dump1(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-int proc_get_rf_reg_dump2(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-int proc_get_rf_reg_dump3(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-int proc_get_rf_reg_dump4(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-#ifdef CONFIG_88EU_AP_MODE
-
-int proc_get_all_sta_info(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-#endif
-
 int proc_get_best_channel(char *page, char **start,
 			  off_t offset, int count,
 			  int *eof, void *data);
 
-int proc_get_rx_signal(char *page, char **start,
-		       off_t offset, int count,
-		       int *eof, void *data);
-
-int proc_set_rx_signal(struct file *file, const char __user *buffer,
-		       unsigned long count, void *data);
-
-int proc_get_ht_enable(char *page, char **start,
-		       off_t offset, int count,
-		       int *eof, void *data);
-
-int proc_set_ht_enable(struct file *file, const char __user *buffer,
-		       unsigned long count, void *data);
-
-int proc_get_cbw40_enable(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-int proc_set_cbw40_enable(struct file *file, const char __user *buffer,
-			  unsigned long count, void *data);
-
-int proc_get_ampdu_enable(char *page, char **start,
-			  off_t offset, int count,
-			  int *eof, void *data);
-
-int proc_set_ampdu_enable(struct file *file, const char __user *buffer,
-			  unsigned long count, void *data);
-
-int proc_get_rx_stbc(char *page, char **start,
-		     off_t offset, int count,
-		     int *eof, void *data);
-
-int proc_set_rx_stbc(struct file *file, const char __user *buffer,
-		     unsigned long count, void *data);
-
-int proc_get_two_path_rssi(char *page, char **start,
-			   off_t offset, int count,
-			   int *eof, void *data);
-
-int proc_get_rssi_disp(char *page, char **start,
-		       off_t offset, int count,
-		       int *eof, void *data);
-
-int proc_set_rssi_disp(struct file *file, const char __user *buffer,
-		       unsigned long count, void *data);
-
 #endif	/* __RTW_DEBUG_H__ */
diff --git a/drivers/staging/rtl8188eu/include/rtw_efuse.h b/drivers/staging/rtl8188eu/include/rtw_efuse.h
index 9bfb10c302b5..168c12d3c0b4 100644
--- a/drivers/staging/rtl8188eu/include/rtw_efuse.h
+++ b/drivers/staging/rtl8188eu/include/rtw_efuse.h
@@ -34,16 +34,6 @@
 #define	EFUSE_WIFI				0
 #define	EFUSE_BT				1
 
-enum _EFUSE_DEF_TYPE {
-	TYPE_EFUSE_MAX_SECTION				= 0,
-	TYPE_EFUSE_REAL_CONTENT_LEN			= 1,
-	TYPE_AVAILABLE_EFUSE_BYTES_BANK		= 2,
-	TYPE_AVAILABLE_EFUSE_BYTES_TOTAL	= 3,
-	TYPE_EFUSE_MAP_LEN					= 4,
-	TYPE_EFUSE_PROTECT_BYTES_BANK		= 5,
-	TYPE_EFUSE_CONTENT_LEN_BANK			= 6,
-};
-
 /* E-Fuse */
 #define EFUSE_MAP_SIZE      512
 #define EFUSE_MAX_SIZE      256
@@ -95,8 +85,6 @@ struct efuse_hal {
 };
 
 u8 Efuse_CalculateWordCnts(u8 word_en);
-void EFUSE_GetEfuseDefinition(struct adapter *adapt, u8 type, u8 type1,
-			      void *out);
 u8 efuse_OneByteRead(struct adapter *adapter, u16 addr, u8 *data);
 u8 efuse_OneByteWrite(struct adapter *adapter, u16 addr, u8 data);
 
diff --git a/drivers/staging/rtl8188eu/include/rtw_event.h b/drivers/staging/rtl8188eu/include/rtw_event.h
index 5c34e567d341..1c5ebde97091 100644
--- a/drivers/staging/rtl8188eu/include/rtw_event.h
+++ b/drivers/staging/rtl8188eu/include/rtw_event.h
@@ -18,7 +18,7 @@
 #include <osdep_service.h>
 
 #include <wlan_bssdef.h>
-#include <linux/semaphore.h>
+#include <linux/mutex.h>
 #include <linux/sem.h>
 
 /*
@@ -71,12 +71,6 @@ struct stadel_event {
 	int mac_id;
 };
 
-struct addba_event {
-	unsigned int tid;
-};
-
-#define GEN_EVT_CODE(event)	event ## _EVT_
-
 struct fwevent {
 	u32	parmsize;
 	void (*event_callback)(struct adapter *dev, u8 *pbuf);
@@ -84,21 +78,6 @@ struct fwevent {
 
 #define C2HEVENT_SZ			32
 
-struct event_node {
-	unsigned char *node;
-	unsigned char evt_code;
-	unsigned short evt_sz;
-	int	*caller_ff_tail;
-	int	caller_ff_sz;
-};
-
-struct c2hevent_queue {
-	int	head;
-	int	tail;
-	struct	event_node	nodes[C2HEVENT_SZ];
-	unsigned char	seq;
-};
-
 #define NETWORK_QUEUE_SZ	4
 
 struct network_queue {
diff --git a/drivers/staging/rtl8188eu/include/rtw_ht.h b/drivers/staging/rtl8188eu/include/rtw_ht.h
index b45483fd069f..d842eade7f57 100644
--- a/drivers/staging/rtl8188eu/include/rtw_ht.h
+++ b/drivers/staging/rtl8188eu/include/rtw_ht.h
@@ -15,16 +15,11 @@
 #ifndef _RTW_HT_H_
 #define _RTW_HT_H_
 
-#include <osdep_service.h>
-#include "wifi.h"
+#include <linux/ieee80211.h>
 
 struct ht_priv {
 	u32	ht_option;
 	u32	ampdu_enable;/* for enable Tx A-MPDU */
-	u32	tx_amsdu_enable;/* for enable Tx A-MSDU */
-	u32	tx_amdsu_maxlen; /*  1: 8k, 0:4k ; default:8k, for tx */
-	u32	rx_ampdu_maxlen; /* for rx reordering ctrl win_sz,
-				  * updated when join_callback. */
 	u8	bwmode;/*  */
 	u8	ch_offset;/* PRIME_CHNL_OFFSET */
 	u8	sgi;/* short GI */
@@ -33,7 +28,7 @@ struct ht_priv {
 	u8	agg_enable_bitmap;
 	u8	candidate_tid_bitmap;
 
-	struct rtw_ieee80211_ht_cap ht_cap;
+	struct ieee80211_ht_cap ht_cap;
 };
 
 #endif	/* _RTL871X_HT_H_ */
diff --git a/drivers/staging/rtl8188eu/include/rtw_ioctl.h b/drivers/staging/rtl8188eu/include/rtw_ioctl.h
index 3a652df4b26c..a6b1c854a061 100644
--- a/drivers/staging/rtl8188eu/include/rtw_ioctl.h
+++ b/drivers/staging/rtl8188eu/include/rtw_ioctl.h
@@ -69,15 +69,6 @@ enum oid_type {
 	SET_OID
 };
 
-struct oid_funs_node {
-	unsigned int oid_start; /* the starting number for OID */
-	unsigned int oid_end; /* the ending number for OID */
-	struct oid_obj_priv *node_array;
-	unsigned int array_sz; /* the size of node_array */
-	int query_counter; /* count the number of query hits for this segment */
-	int set_counter; /* count the number of set hits for this segment */
-};
-
 struct oid_par_priv {
 	void		*adapter_context;
 	NDIS_OID	oid;
@@ -89,12 +80,6 @@ struct oid_par_priv {
 	u32		dbg;
 };
 
-struct oid_obj_priv {
-	unsigned char	dbg; /*  0: without OID debug message
-			      *  1: with OID debug message */
-	int (*oidfuns)(struct oid_par_priv *poid_par_priv);
-};
-
 #if defined(_RTW_MP_IOCTL_C_)
 static int oid_null_function(struct oid_par_priv *poid_par_priv) {
 	return NDIS_STATUS_SUCCESS;
diff --git a/drivers/staging/rtl8188eu/include/rtw_mlme.h b/drivers/staging/rtl8188eu/include/rtw_mlme.h
index 5d8bce0f58db..9434b869c5e9 100644
--- a/drivers/staging/rtl8188eu/include/rtw_mlme.h
+++ b/drivers/staging/rtl8188eu/include/rtw_mlme.h
@@ -301,12 +301,10 @@ struct mlme_priv {
 
 	u8 *nic_hdl;
 
-	u8 not_indic_disco;
 	struct list_head *pscanned;
 	struct __queue free_bss_pool;
 	struct __queue scanned_queue;
 	u8 *free_bss_buf;
-	u32	num_of_scanned;
 
 	struct ndis_802_11_ssid	assoc_ssid;
 	u8	assoc_bssid[6];
@@ -318,10 +316,8 @@ struct mlme_priv {
 	struct timer_list assoc_timer;
 
 	uint assoc_by_bssid;
-	uint assoc_by_rssi;
 
 	struct timer_list scan_to_timer; /*  driver itself handles scan_timeout status. */
-	u32 scan_start_time; /*  used to evaluate the time spent in scanning */
 
 	struct qos_priv qospriv;
 
@@ -387,17 +383,6 @@ struct mlme_priv {
 	u32 wps_probe_resp_ie_len;
 	u32 wps_assoc_resp_ie_len;
 
-	u8 *p2p_beacon_ie;
-	u8 *p2p_probe_req_ie;
-	u8 *p2p_probe_resp_ie;
-	u8 *p2p_go_probe_resp_ie; /* for GO */
-	u8 *p2p_assoc_req_ie;
-
-	u32 p2p_beacon_ie_len;
-	u32 p2p_probe_req_ie_len;
-	u32 p2p_probe_resp_ie_len;
-	u32 p2p_go_probe_resp_ie_len; /* for GO */
-	u32 p2p_assoc_req_ie_len;
 	spinlock_t bcn_update_lock;
 	u8		update_bcn;
 #endif /* if defined (CONFIG_88EU_AP_MODE) */
@@ -500,27 +485,6 @@ static inline void clr_fwstate_ex(struct mlme_priv *pmlmepriv, int state)
 	spin_unlock_bh(&pmlmepriv->lock);
 }
 
-static inline void up_scanned_network(struct mlme_priv *pmlmepriv)
-{
-	spin_lock_bh(&pmlmepriv->lock);
-	pmlmepriv->num_of_scanned++;
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
-static inline void down_scanned_network(struct mlme_priv *pmlmepriv)
-{
-	spin_lock_bh(&pmlmepriv->lock);
-	pmlmepriv->num_of_scanned--;
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
-static inline void set_scanned_network_val(struct mlme_priv *pmlmepriv, int val)
-{
-	spin_lock_bh(&pmlmepriv->lock);
-	pmlmepriv->num_of_scanned = val;
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
 u16 rtw_get_capability(struct wlan_bssid_ex *bss);
 void rtw_update_scanned_network(struct adapter *adapter,
 				struct wlan_bssid_ex *target);
diff --git a/drivers/staging/rtl8188eu/include/rtw_mlme_ext.h b/drivers/staging/rtl8188eu/include/rtw_mlme_ext.h
index 27382ff24a84..1b1caaf583c9 100644
--- a/drivers/staging/rtl8188eu/include/rtw_mlme_ext.h
+++ b/drivers/staging/rtl8188eu/include/rtw_mlme_ext.h
@@ -349,7 +349,7 @@ struct mlme_ext_info {
 
 	struct ADDBA_request	ADDBA_req;
 	struct WMM_para_element	WMM_param;
-	struct HT_caps_element	HT_caps;
+	struct ieee80211_ht_cap HT_caps;
 	struct HT_info_element	HT_info;
 	struct wlan_bssid_ex	network;/* join network or bss_network,
 					 * if in ap mode, it is the same
@@ -529,12 +529,12 @@ int update_sta_support_rate(struct adapter *padapter, u8 *pvar_ie,
 void update_sta_info(struct adapter *padapter, struct sta_info *psta);
 unsigned int update_basic_rate(unsigned char *ptn, unsigned int ptn_sz);
 unsigned int update_supported_rate(unsigned char *ptn, unsigned int ptn_sz);
-unsigned int update_MSC_rate(struct HT_caps_element *pHT_caps);
+unsigned int update_MSC_rate(struct ieee80211_ht_cap *pHT_caps);
 void Update_RA_Entry(struct adapter *padapter, u32 mac_id);
 void set_sta_rate(struct adapter *padapter, struct sta_info *psta);
 
 unsigned char get_highest_rate_idx(u32 mask);
-int support_short_GI(struct adapter *padapter, struct HT_caps_element *caps);
+int support_short_GI(struct adapter *padapter, struct ieee80211_ht_cap *caps);
 unsigned int is_ap_in_tkip(struct adapter *padapter);
 unsigned int is_ap_in_wep(struct adapter *padapter);
 unsigned int should_forbid_n_rate(struct adapter *padapter);
@@ -562,8 +562,6 @@ int issue_qos_nulldata(struct adapter *padapter, unsigned char *da,
 		       u16 tid, int try_cnt, int wait_ms);
 int issue_deauth(struct adapter *padapter, unsigned char *da,
 		 unsigned short reason);
-void issue_action_spct_ch_switch(struct adapter *padapter, u8 *ra, u8 new_ch,
-				 u8 ch_offset);
 unsigned int send_delba(struct adapter *padapter, u8 initiator, u8 *addr);
 unsigned int send_beacon(struct adapter *padapter);
 
@@ -627,27 +625,24 @@ u8 led_blink_hdl(struct adapter *padapter, unsigned char *pbuf);
 u8 set_csa_hdl(struct adapter *padapter, unsigned char *pbuf);
 u8 tdls_hdl(struct adapter *padapter, unsigned char *pbuf);
 
-#define GEN_DRV_CMD_HANDLER(size, cmd)	{size, &cmd ## _hdl},
-#define GEN_MLME_EXT_HANDLER(size, cmd)	{size, cmd},
-
 #ifdef _RTW_CMD_C_
 
 static struct cmd_hdl wlancmds[] = {
-	GEN_MLME_EXT_HANDLER(sizeof(struct wlan_bssid_ex), join_cmd_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct disconnect_parm), disconnect_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct wlan_bssid_ex), createbss_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct setopmode_parm), setopmode_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct sitesurvey_parm), sitesurvey_cmd_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct setauth_parm), setauth_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct setkey_parm), setkey_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct set_stakey_parm), set_stakey_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct set_assocsta_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct addBaReq_parm), add_ba_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct set_ch_parm), set_ch_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct wlan_bssid_ex), tx_beacon_hdl)
-	GEN_MLME_EXT_HANDLER(0, mlme_evt_hdl)
-	GEN_MLME_EXT_HANDLER(0, rtw_drvextra_cmd_hdl)
-	GEN_MLME_EXT_HANDLER(sizeof(struct SetChannelPlan_param), set_chplan_hdl)
+	{sizeof(struct wlan_bssid_ex), join_cmd_hdl},
+	{sizeof(struct disconnect_parm), disconnect_hdl},
+	{sizeof(struct wlan_bssid_ex), createbss_hdl},
+	{sizeof(struct setopmode_parm), setopmode_hdl},
+	{sizeof(struct sitesurvey_parm), sitesurvey_cmd_hdl},
+	{sizeof(struct setauth_parm), setauth_hdl},
+	{sizeof(struct setkey_parm), setkey_hdl},
+	{sizeof(struct set_stakey_parm), set_stakey_hdl},
+	{sizeof(struct set_assocsta_parm), NULL},
+	{sizeof(struct addBaReq_parm), add_ba_hdl},
+	{sizeof(struct set_ch_parm), set_ch_hdl},
+	{sizeof(struct wlan_bssid_ex), tx_beacon_hdl},
+	{0, mlme_evt_hdl},
+	{0, rtw_drvextra_cmd_hdl},
+	{sizeof(struct SetChannelPlan_param), set_chplan_hdl}
 };
 
 #endif
@@ -669,32 +664,32 @@ void rtw_dummy_event_callback(struct adapter *adapter, u8 *pbuf);
 void rtw_fwdbg_event_callback(struct adapter *adapter, u8 *pbuf);
 
 enum rtw_c2h_event {
-	GEN_EVT_CODE(_Read_MACREG) = 0, /*0*/
-	GEN_EVT_CODE(_Read_BBREG),
-	GEN_EVT_CODE(_Read_RFREG),
-	GEN_EVT_CODE(_Read_EEPROM),
-	GEN_EVT_CODE(_Read_EFUSE),
-	GEN_EVT_CODE(_Read_CAM),	/*5*/
-	GEN_EVT_CODE(_Get_BasicRate),
-	GEN_EVT_CODE(_Get_DataRate),
-	GEN_EVT_CODE(_Survey),	 /*8*/
-	GEN_EVT_CODE(_SurveyDone),	 /*9*/
-
-	GEN_EVT_CODE(_JoinBss), /*10*/
-	GEN_EVT_CODE(_AddSTA),
-	GEN_EVT_CODE(_DelSTA),
-	GEN_EVT_CODE(_AtimDone),
-	GEN_EVT_CODE(_TX_Report),
-	GEN_EVT_CODE(_CCX_Report),		/*15*/
-	GEN_EVT_CODE(_DTM_Report),
-	GEN_EVT_CODE(_TX_Rate_Statistics),
-	GEN_EVT_CODE(_C2HLBK),
-	GEN_EVT_CODE(_FWDBG),
-	GEN_EVT_CODE(_C2HFEEDBACK),             /*20*/
-	GEN_EVT_CODE(_ADDBA),
-	GEN_EVT_CODE(_C2HBCN),
-	GEN_EVT_CODE(_ReportPwrState),	/* filen: only for PCIE, USB */
-	GEN_EVT_CODE(_CloseRF),		/* filen: only for PCIE,
+	_Read_MACREG_EVT_ = 0, /*0*/
+	_Read_BBREG_EVT_,
+	_Read_RFREG_EVT_,
+	_Read_EEPROM_EVT_,
+	_Read_EFUSE_EVT_,
+	_Read_CAM_EVT_,	/*5*/
+	_Get_BasicRate_EVT_,
+	_Get_DataRate_EVT_,
+	_Survey_EVT_,	 /*8*/
+	_SurveyDone_EVT_,	 /*9*/
+
+	_JoinBss_EVT_, /*10*/
+	_AddSTA_EVT_,
+	_DelSTA_EVT_,
+	_AtimDone_EVT_,
+	_TX_Report_EVT_,
+	_CCX_Report_EVT_,		/*15*/
+	_DTM_Report_EVT_,
+	_TX_Rate_Statistics_EVT_,
+	_C2HLBK_EVT_,
+	_FWDBG_EVT_,
+	_C2HFEEDBACK_EVT_,             /*20*/
+	_ADDBA_EVT_,
+	_C2HBCN_EVT_,
+	_ReportPwrState_EVT_,	/* filen: only for PCIE, USB */
+	_CloseRF_EVT_,		/* filen: only for PCIE,
 					 * work around ASPM */
 	MAX_C2HEVT
 };
diff --git a/drivers/staging/rtl8188eu/include/rtw_pwrctrl.h b/drivers/staging/rtl8188eu/include/rtw_pwrctrl.h
index 9680e2eab62f..18a9e744fcbe 100644
--- a/drivers/staging/rtl8188eu/include/rtw_pwrctrl.h
+++ b/drivers/staging/rtl8188eu/include/rtw_pwrctrl.h
@@ -92,21 +92,6 @@ struct reportpwrstate_parm {
 	unsigned short rsvd;
 };
 
-static inline void _init_pwrlock(struct semaphore  *plock)
-{
-	sema_init(plock, 1);
-}
-
-static inline void _enter_pwrlock(struct semaphore  *plock)
-{
-	_rtw_down_sema(plock);
-}
-
-static inline void _exit_pwrlock(struct semaphore  *plock)
-{
-	up(plock);
-}
-
 #define LPS_DELAY_TIME	1*HZ /*  1 sec */
 
 #define EXE_PWR_NONE	0x01
@@ -157,7 +142,7 @@ enum { /*  for ips_mode */
 };
 
 struct pwrctrl_priv {
-	struct semaphore lock;
+	struct mutex mutex_lock;
 	volatile u8 rpwm; /*  requested power state for fw */
 	volatile u8 cpwm; /*  fw current power state. updated when
 			   * 1. read from HCPWM 2. driver lowers power level */
diff --git a/drivers/staging/rtl8188eu/include/rtw_recv.h b/drivers/staging/rtl8188eu/include/rtw_recv.h
index b0373b6216d6..49d973881a04 100644
--- a/drivers/staging/rtl8188eu/include/rtw_recv.h
+++ b/drivers/staging/rtl8188eu/include/rtw_recv.h
@@ -65,13 +65,6 @@ struct	stainfo_rxcache	{
 */
 };
 
-struct smooth_rssi_data {
-	u32	elements[100];	/* array to store values */
-	u32	index;			/* index to current array to store */
-	u32	total_num;		/* num of valid elements */
-	u32	total_val;		/* sum of valid elements */
-};
-
 struct signal_stat {
 	u8	update_req;		/* used to indicate */
 	u8	avg_val;		/* avg of valid elements */
@@ -246,7 +239,6 @@ struct recv_buf {
 struct recv_frame {
 	struct list_head list;
 	struct sk_buff	 *pkt;
-	struct sk_buff	 *pkt_newalloc;
 	struct adapter  *adapter;
 	struct rx_pkt_attrib attrib;
 	uint  len;
diff --git a/drivers/staging/rtl8188eu/include/rtw_security.h b/drivers/staging/rtl8188eu/include/rtw_security.h
index ca1247bce6e3..2663e60bb6c6 100644
--- a/drivers/staging/rtl8188eu/include/rtw_security.h
+++ b/drivers/staging/rtl8188eu/include/rtw_security.h
@@ -164,12 +164,6 @@ struct security_priv {
 	u8 bWepDefaultKeyIdxSet;
 };
 
-struct sha256_state {
-	u64 length;
-	u32 state[8], curlen;
-	u8 buf[64];
-};
-
 #define GET_ENCRY_ALGO(psecuritypriv, psta, encry_algo, bmcst)		\
 do {									\
 	switch (psecuritypriv->dot11AuthAlgrthm) {			\
diff --git a/drivers/staging/rtl8188eu/include/rtw_sreset.h b/drivers/staging/rtl8188eu/include/rtw_sreset.h
index ce027dfdecc5..4c4ccd564863 100644
--- a/drivers/staging/rtl8188eu/include/rtw_sreset.h
+++ b/drivers/staging/rtl8188eu/include/rtw_sreset.h
@@ -33,7 +33,6 @@ struct sreset_priv {
 #define	WIFI_RX_HANG			BIT(5)
 #define		WIFI_IF_NOT_EXIST	BIT(6)
 
-void sreset_init_value(struct adapter *padapter);
 u8 sreset_get_wifi_status(struct adapter *padapter);
 void sreset_set_wifi_error_status(struct adapter *padapter, u32 status);
 
diff --git a/drivers/staging/rtl8188eu/include/rtw_xmit.h b/drivers/staging/rtl8188eu/include/rtw_xmit.h
index a0853bab3edb..dd6b7a9a8d4a 100644
--- a/drivers/staging/rtl8188eu/include/rtw_xmit.h
+++ b/drivers/staging/rtl8188eu/include/rtw_xmit.h
@@ -113,7 +113,6 @@ struct pkt_attrib {
 	u8	dhcp_pkt;
 	u16	ether_type;
 	u16	seqnum;
-	u16	pkt_hdrlen;	/* the original 802.3 pkt header len */
 	u16	hdrlen;		/* the WLAN Header Len */
 	u32	pktlen;		/* the original 802.3 pkt raw_data len (not include
 				 * ether_hdr data) */
@@ -256,15 +255,8 @@ struct	hw_txqueue {
 	int	ac_tag;
 };
 
-struct agg_pkt_info {
-	u16 offset;
-	u16 pkt_len;
-};
-
 struct	xmit_priv {
 	spinlock_t lock;
-	struct semaphore xmit_sema;
-	struct semaphore terminate_xmitthread_sema;
 	struct __queue be_pending;
 	struct __queue bk_pending;
 	struct __queue vi_pending;
@@ -289,7 +281,6 @@ struct	xmit_priv {
 	u8	wmm_para_seq[4];/* sequence for wmm ac parameter strength
 				 * from large to small. it's value is 0->vo,
 				 * 1->vi, 2->be, 3->bk. */
-	struct semaphore tx_retevt;/* all tx return event; */
 	u8		txirp_cnt;/*  */
 	struct tasklet_struct xmit_tasklet;
 	/* per AC pending irp */
diff --git a/drivers/staging/rtl8188eu/include/usb_hal.h b/drivers/staging/rtl8188eu/include/usb_hal.h
deleted file mode 100644
index b1bf07a9013e..000000000000
--- a/drivers/staging/rtl8188eu/include/usb_hal.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __USB_HAL_H__
-#define __USB_HAL_H__
-
-void rtl8188eu_set_hal_ops(struct adapter *padapter);
-#define hal_set_hal_ops	rtl8188eu_set_hal_ops
-
-#endif /* __USB_HAL_H__ */
diff --git a/drivers/staging/rtl8188eu/include/usb_ops_linux.h b/drivers/staging/rtl8188eu/include/usb_ops_linux.h
index 220733314f8b..78d9b6e035bf 100644
--- a/drivers/staging/rtl8188eu/include/usb_ops_linux.h
+++ b/drivers/staging/rtl8188eu/include/usb_ops_linux.h
@@ -47,21 +47,6 @@
 #define usb_read_interrupt_complete(purb, regs)		\
 	usb_read_interrupt_complete(purb)
 
-static inline u8 rtw_usb_bulk_size_boundary(struct adapter *padapter,
-					    int buf_len)
-{
-	u8 rst = true;
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
-
-	if (pdvobjpriv->ishighspeed)
-		rst = (0 == (buf_len) % USB_HIGH_SPEED_BULK_SIZE) ?
-		      true : false;
-	else
-		rst = (0 == (buf_len) % USB_FULL_SPEED_BULK_SIZE) ?
-		      true : false;
-	return rst;
-}
-
 unsigned int ffaddr2pipehdl(struct dvobj_priv *pdvobj, u32 addr);
 
 u8 usb_read8(struct adapter *adapter, u32 addr);
@@ -75,7 +60,7 @@ int usb_write8(struct adapter *adapter, u32 addr, u8 val);
 int usb_write16(struct adapter *adapter, u32 addr, u16 val);
 int usb_write32(struct adapter *adapter, u32 addr, u32 val);
 
-u32 usb_write_port(struct adapter *adapter, u32 addr, u32 cnt, u8 *pmem);
+u32 usb_write_port(struct adapter *adapter, u32 addr, u32 cnt, struct xmit_buf *pmem);
 void usb_write_port_cancel(struct adapter *adapter);
 
 #endif
diff --git a/drivers/staging/rtl8188eu/include/wifi.h b/drivers/staging/rtl8188eu/include/wifi.h
index e7c512183619..9e08e6842eca 100644
--- a/drivers/staging/rtl8188eu/include/wifi.h
+++ b/drivers/staging/rtl8188eu/include/wifi.h
@@ -508,22 +508,6 @@ struct rtw_ieee80211_bar {
 #define IEEE80211_BAR_CTRL_ACK_POLICY_NORMAL     0x0000
 #define IEEE80211_BAR_CTRL_CBMTID_COMPRESSED_BA  0x0004
 
- /**
- * struct rtw_ieee80211_ht_cap - HT capabilities
- *
- * This structure refers to "HT capabilities element" as
- * described in 802.11n draft section 7.3.2.52
- */
-
-struct rtw_ieee80211_ht_cap {
-	unsigned short	cap_info;
-	unsigned char	ampdu_params_info;
-	unsigned char	supp_mcs_set[16];
-	unsigned short	extended_ht_cap_info;
-	unsigned int	tx_BF_cap_info;
-	unsigned char   antenna_selection_info;
-} __packed;
-
 /**
  * struct rtw_ieee80211_ht_cap - HT additional information
  *
@@ -538,20 +522,6 @@ struct ieee80211_ht_addt_info {
 	unsigned char	basic_set[16];
 } __packed;
 
-struct HT_caps_element {
-	union {
-		struct {
-			__le16	HT_caps_info;
-			unsigned char	AMPDU_para;
-			unsigned char	MCS_rate[16];
-			unsigned short	HT_ext_caps;
-			unsigned int	Beamforming_caps;
-			unsigned char	ASEL_caps;
-		} HT_cap_element;
-		unsigned char HT_cap[26];
-	} u;
-} __packed;
-
 struct HT_info_element {
 	unsigned char	primary_channel;
 	unsigned char	infos[5];
diff --git a/drivers/staging/rtl8188eu/include/wlan_bssdef.h b/drivers/staging/rtl8188eu/include/wlan_bssdef.h
index 560966cd7dfe..e1931dd04da0 100644
--- a/drivers/staging/rtl8188eu/include/wlan_bssdef.h
+++ b/drivers/staging/rtl8188eu/include/wlan_bssdef.h
@@ -123,40 +123,10 @@ enum ndis_802_11_wep_status {
 #define NDIS_802_11_AI_RESFI_STATUSCODE        2
 #define NDIS_802_11_AI_RESFI_ASSOCIATIONID     4
 
-struct ndis_802_11_ai_reqfi {
-    u16 Capabilities;
-    u16 ListenInterval;
-    unsigned char CurrentAPAddress[ETH_ALEN];
-};
-
-struct ndis_802_11_ai_resfi {
-    u16 Capabilities;
-    u16 StatusCode;
-    u16 AssociationId;
-};
-
-struct ndis_802_11_assoc_info {
-	u32  Length;
-	u16 AvailableRequestFixedIEs;
-	struct ndis_802_11_ai_reqfi    RequestFixedIEs;
-	u32  RequestIELength;
-	u32  OffsetRequestIEs;
-	u16 AvailableResponseFixedIEs;
-	struct ndis_802_11_ai_resfi    ResponseFixedIEs;
-	u32  ResponseIELength;
-	u32  OffsetResponseIEs;
-};
-
 enum ndis_802_11_reload_def {
 	Ndis802_11ReloadWEPKeys
 };
 
-struct ndis_802_11_remove_key {
-	u32                   Length;        /*  Length */
-	u32                   KeyIndex;
-	unsigned char BSSID[ETH_ALEN];
-};
-
 struct ndis_802_11_wep {
 	u32     Length;        /*  Length of this structure */
 	u32     KeyIndex;      /*  0 is the per-client key,
@@ -165,12 +135,6 @@ struct ndis_802_11_wep {
 	u8     KeyMaterial[16];/*  variable len depending on above field */
 };
 
-struct ndis_802_11_auth_req {
-	u32 Length;            /*  Length of structure */
-	unsigned char Bssid[ETH_ALEN];
-	u32 Flags;
-};
-
 enum ndis_802_11_status_type {
 	Ndis802_11StatusType_Authentication,
 	Ndis802_11StatusType_MediaStreamMode,
@@ -179,10 +143,6 @@ enum ndis_802_11_status_type {
 				    * an upper bound */
 };
 
-struct ndis_802_11_status_ind {
-	enum ndis_802_11_status_type StatusType;
-};
-
 /*  mask for authentication/integrity fields */
 #define NDIS_802_11_AUTH_REQUEST_AUTH_FIELDS        	0x0f
 #define NDIS_802_11_AUTH_REQUEST_REAUTH			0x01
@@ -193,21 +153,6 @@ struct ndis_802_11_status_ind {
 /*  MIC check time, 60 seconds. */
 #define MIC_CHECK_TIME	60000000
 
-struct ndis_802_11_auth_evt {
-	struct ndis_802_11_status_ind       Status;
-	struct ndis_802_11_auth_req  Request[1];
-};
-
-struct ndis_802_11_test {
-	u32 Length;
-	u32 Type;
-	union {
-		struct ndis_802_11_auth_evt AuthenticationEvent;
-		NDIS_802_11_RSSI RssiTrigger;
-	} tt;
-};
-
-
 #ifndef Ndis802_11APMode
 #define Ndis802_11APMode (Ndis802_11InfrastructureMax+1)
 #endif
@@ -297,32 +242,4 @@ enum UAPSD_MAX_SP {
 #define NUM_PRE_AUTH_KEY 16
 #define NUM_PMKID_CACHE NUM_PRE_AUTH_KEY
 
-/*
-*	WPA2
-*/
-
-struct pmkid_candidate {
-	unsigned char BSSID[ETH_ALEN];
-	u32 Flags;
-};
-
-struct ndis_802_11_pmkid_list {
-	u32 Version;       /*  Version of the structure */
-	u32 NumCandidates; /*  No. of pmkid candidates */
-	struct pmkid_candidate CandidateList[1];
-};
-
-struct ndis_802_11_auth_encrypt {
-	enum ndis_802_11_auth_mode AuthModeSupported;
-	enum ndis_802_11_wep_status EncryptStatusSupported;
-};
-
-struct ndis_802_11_cap {
-	u32  Length;
-	u32  Version;
-	u32  NoOfPMKIDs;
-	u32  NoOfAuthEncryptPairsSupported;
-	struct ndis_802_11_auth_encrypt AuthenticationEncryptionSupported[1];
-};
-
 #endif /* ifndef WLAN_BSSDEF_H_ */
diff --git a/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c b/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c
index 5672f014cc46..4de9dbc93380 100644
--- a/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c
+++ b/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c
@@ -132,12 +132,15 @@ static char *translate_scan(struct adapter *padapter,
 	p = rtw_get_ie(&pnetwork->network.IEs[12], _HT_CAPABILITY_IE_, &ht_ielen, pnetwork->network.IELength-12);
 
 	if (p && ht_ielen > 0) {
-		struct rtw_ieee80211_ht_cap *pht_capie;
+		struct ieee80211_ht_cap *pht_capie;
 		ht_cap = true;
-		pht_capie = (struct rtw_ieee80211_ht_cap *)(p+2);
-		memcpy(&mcs_rate, pht_capie->supp_mcs_set, 2);
-		bw_40MHz = (pht_capie->cap_info&IEEE80211_HT_CAP_SUP_WIDTH) ? 1 : 0;
-		short_GI = (pht_capie->cap_info&(IEEE80211_HT_CAP_SGI_20|IEEE80211_HT_CAP_SGI_40)) ? 1 : 0;
+		pht_capie = (struct ieee80211_ht_cap *)(p + 2);
+		memcpy(&mcs_rate, pht_capie->mcs.rx_mask, 2);
+		bw_40MHz = !!(le16_to_cpu(pht_capie->cap_info) &
+			      IEEE80211_HT_CAP_SUP_WIDTH);
+		short_GI = !!(le16_to_cpu(pht_capie->cap_info) &
+			      (IEEE80211_HT_CAP_SGI_20 |
+			       IEEE80211_HT_CAP_SGI_40));
 	}
 
 	/* Add the protocol name */
@@ -400,7 +403,7 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 			wep_key_len = wep_key_len <= 5 ? 5 : 13;
 			wep_total_len = wep_key_len + offsetof(struct ndis_802_11_wep, KeyMaterial);
 			pwep = (struct ndis_802_11_wep *)rtw_malloc(wep_total_len);
-			if (pwep == NULL) {
+			if (!pwep) {
 				RT_TRACE(_module_rtl871x_ioctl_os_c, _drv_err_, (" wpa_set_encryption: pwep allocate fail !!!\n"));
 				goto exit;
 			}
@@ -441,7 +444,7 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE | WIFI_MP_STATE)) { /* sta mode */
 			psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
-			if (psta == NULL) {
+			if (!psta) {
 				;
 			} else {
 				if (strcmp(param->u.crypt.alg, "none") != 0)
@@ -476,7 +479,7 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 				}
 			}
 			pbcmc_sta = rtw_get_bcmc_stainfo(padapter);
-			if (pbcmc_sta == NULL) {
+			if (!pbcmc_sta) {
 				;
 			} else {
 				/* Jeff: don't disable ieee8021x_blocked while clearing key */
@@ -502,9 +505,9 @@ static int rtw_set_wpa_ie(struct adapter *padapter, char *pie, unsigned short ie
 	int group_cipher = 0, pairwise_cipher = 0;
 	int ret = 0;
 
-	if ((ielen > MAX_WPA_IE_LEN) || (pie == NULL)) {
+	if ((ielen > MAX_WPA_IE_LEN) || (!pie)) {
 		_clr_fwstate_(&padapter->mlmepriv, WIFI_UNDER_WPS);
-		if (pie == NULL)
+		if (!pie)
 			return ret;
 		else
 			return -EINVAL;
@@ -512,7 +515,7 @@ static int rtw_set_wpa_ie(struct adapter *padapter, char *pie, unsigned short ie
 
 	if (ielen) {
 		buf = kmemdup(pie, ielen, GFP_KERNEL);
-		if (buf == NULL) {
+		if (!buf) {
 			ret =  -ENOMEM;
 			goto exit;
 		}
@@ -1049,7 +1052,7 @@ static int rtw_wx_set_mlme(struct net_device *dev,
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
 	struct iw_mlme *mlme = (struct iw_mlme *)extra;
 
-	if (mlme == NULL)
+	if (!mlme)
 		return -1;
 
 	DBG_88E("%s\n", __func__);
@@ -1896,7 +1899,7 @@ static int rtw_wx_set_enc_ext(struct net_device *dev,
 
 	param_len = sizeof(struct ieee_param) + pext->key_len;
 	param = (struct ieee_param *)rtw_malloc(param_len);
-	if (param == NULL)
+	if (!param)
 		return -1;
 
 	memset(param, 0, param_len);
@@ -2061,7 +2064,7 @@ static int wpa_supplicant_ioctl(struct net_device *dev, struct iw_point *p)
 	}
 
 	param = (struct ieee_param *)rtw_malloc(p->length);
-	if (param == NULL) {
+	if (!param) {
 		ret = -ENOMEM;
 		goto out;
 	}
@@ -2254,13 +2257,13 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 		}
 	}
 
-	if (strcmp(param->u.crypt.alg, "none") == 0 && (psta == NULL)) {
+	if (strcmp(param->u.crypt.alg, "none") == 0 && (!psta)) {
 		/* todo:clear default encryption keys */
 
 		DBG_88E("clear default encryption keys, keyid =%d\n", param->u.crypt.idx);
 		goto exit;
 	}
-	if (strcmp(param->u.crypt.alg, "WEP") == 0 && (psta == NULL)) {
+	if (strcmp(param->u.crypt.alg, "WEP") == 0 && (!psta)) {
 		DBG_88E("r871x_set_encryption, crypt.alg = WEP\n");
 		wep_key_idx = param->u.crypt.idx;
 		wep_key_len = param->u.crypt.key_len;
@@ -2274,7 +2277,7 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 			wep_key_len = wep_key_len <= 5 ? 5 : 13;
 			wep_total_len = wep_key_len + offsetof(struct ndis_802_11_wep, KeyMaterial);
 			pwep = (struct ndis_802_11_wep *)rtw_malloc(wep_total_len);
-			if (pwep == NULL) {
+			if (!pwep) {
 				DBG_88E(" r871x_set_encryption: pwep allocate fail !!!\n");
 				goto exit;
 			}
@@ -2528,7 +2531,8 @@ static int rtw_add_sta(struct net_device *dev, struct ieee_param *param)
 		if (WLAN_STA_HT&flags) {
 			psta->htpriv.ht_option = true;
 			psta->qos_option = 1;
-			memcpy((void *)&psta->htpriv.ht_cap, (void *)&param->u.add_sta.ht_cap, sizeof(struct rtw_ieee80211_ht_cap));
+			memcpy(&psta->htpriv.ht_cap, &param->u.add_sta.ht_cap,
+			       sizeof(struct ieee80211_ht_cap));
 		} else {
 			psta->htpriv.ht_option = false;
 		}
@@ -2624,7 +2628,8 @@ static int rtw_ioctl_get_sta_data(struct net_device *dev, struct ieee_param *par
 				      (psta->ht_20mhz_set << 5));
 		psta_data->tx_supp_rates_len =  psta->bssratelen;
 		memcpy(psta_data->tx_supp_rates, psta->bssrateset, psta->bssratelen);
-		memcpy(&psta_data->ht_cap, &psta->htpriv.ht_cap, sizeof(struct rtw_ieee80211_ht_cap));
+		memcpy(&psta_data->ht_cap,
+		       &psta->htpriv.ht_cap, sizeof(struct ieee80211_ht_cap));
 		psta_data->rx_pkts = psta->sta_stats.rx_data_pkts;
 		psta_data->rx_bytes = psta->sta_stats.rx_bytes;
 		psta_data->rx_drops = psta->sta_stats.rx_drops;
@@ -2699,7 +2704,7 @@ static int rtw_set_wps_beacon(struct net_device *dev, struct ieee_param *param,
 	if (ie_len > 0) {
 		pmlmepriv->wps_beacon_ie = rtw_malloc(ie_len);
 		pmlmepriv->wps_beacon_ie_len = ie_len;
-		if (pmlmepriv->wps_beacon_ie == NULL) {
+		if (!pmlmepriv->wps_beacon_ie) {
 			DBG_88E("%s()-%d: rtw_malloc() ERROR!\n", __func__, __LINE__);
 			return -EINVAL;
 		}
@@ -2734,7 +2739,7 @@ static int rtw_set_wps_probe_resp(struct net_device *dev, struct ieee_param *par
 	if (ie_len > 0) {
 		pmlmepriv->wps_probe_resp_ie = rtw_malloc(ie_len);
 		pmlmepriv->wps_probe_resp_ie_len = ie_len;
-		if (pmlmepriv->wps_probe_resp_ie == NULL) {
+		if (!pmlmepriv->wps_probe_resp_ie) {
 			DBG_88E("%s()-%d: rtw_malloc() ERROR!\n", __func__, __LINE__);
 			return -EINVAL;
 		}
@@ -2764,7 +2769,7 @@ static int rtw_set_wps_assoc_resp(struct net_device *dev, struct ieee_param *par
 	if (ie_len > 0) {
 		pmlmepriv->wps_assoc_resp_ie = rtw_malloc(ie_len);
 		pmlmepriv->wps_assoc_resp_ie_len = ie_len;
-		if (pmlmepriv->wps_assoc_resp_ie == NULL) {
+		if (!pmlmepriv->wps_assoc_resp_ie) {
 			DBG_88E("%s()-%d: rtw_malloc() ERROR!\n", __func__, __LINE__);
 			return -EINVAL;
 		}
@@ -2866,7 +2871,7 @@ static int rtw_hostapd_ioctl(struct net_device *dev, struct iw_point *p)
 	}
 
 	param = (struct ieee_param *)rtw_malloc(p->length);
-	if (param == NULL) {
+	if (!param) {
 		ret = -ENOMEM;
 		goto out;
 	}
@@ -2976,7 +2981,7 @@ static int rtw_wx_set_priv(struct net_device *dev,
 			pmlmepriv->wps_probe_req_ie = NULL;
 
 			pmlmepriv->wps_probe_req_ie = rtw_malloc(cp_sz);
-			if (pmlmepriv->wps_probe_req_ie == NULL) {
+			if (!pmlmepriv->wps_probe_req_ie) {
 				pr_info("%s()-%d: rtw_malloc() ERROR!\n", __func__, __LINE__);
 				ret =  -EINVAL;
 				goto FREE_EXT;
diff --git a/drivers/staging/rtl8188eu/os_dep/os_intfs.c b/drivers/staging/rtl8188eu/os_dep/os_intfs.c
index ae2caff030f1..40691f1ec507 100644
--- a/drivers/staging/rtl8188eu/os_dep/os_intfs.c
+++ b/drivers/staging/rtl8188eu/os_dep/os_intfs.c
@@ -23,8 +23,6 @@
 #include <rtw_ioctl.h>
 #include <rtl8188e_hal.h>
 
-#include <usb_hal.h>
-
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Realtek Wireless Lan Driver");
 MODULE_AUTHOR("Realtek Semiconductor Corp.");
@@ -33,11 +31,7 @@ MODULE_VERSION(DRIVERVERSION);
 #define RTW_NOTCH_FILTER 0 /* 0:Disable, 1:Enable, */
 
 /* module param defaults */
-static int rtw_chip_version;
-static int rtw_rfintfs = HWPI;
-static int rtw_lbkmode;/* RTL8712_AIR_TRX; */
 /* Ndis802_11Infrastructure; infra, ad-hoc, auto */
-static int rtw_network_mode = Ndis802_11IBSS;
 static int rtw_channel = 1;/* ad-hoc support requirement */
 static int rtw_wireless_mode = WIRELESS_11BG_24N;
 static int rtw_vrtl_carrier_sense = AUTO_VCS;
@@ -45,9 +39,6 @@ static int rtw_vcs_type = RTS_CTS;/*  */
 static int rtw_rts_thresh = 2347;/*  */
 static int rtw_frag_thresh = 2346;/*  */
 static int rtw_preamble = PREAMBLE_LONG;/* long, short, auto */
-static int rtw_scan_mode = 1;/* active, passive */
-static int rtw_adhoc_tx_pwr = 1;
-static int rtw_soft_ap;
 static int rtw_power_mgnt = 1;
 static int rtw_ips_mode = IPS_NORMAL;
 
@@ -57,11 +48,6 @@ module_param(rtw_ips_mode, int, 0644);
 MODULE_PARM_DESC(rtw_ips_mode, "The default IPS mode");
 
 static int rtw_debug = 1;
-static int rtw_radio_enable = 1;
-static int rtw_long_retry_lmt = 7;
-static int rtw_short_retry_lmt = 7;
-static int rtw_busy_thresh = 40;
-static int rtw_ack_policy = NORMAL_ACK;
 
 static int rtw_software_encrypt;
 static int rtw_software_decrypt;
@@ -70,11 +56,6 @@ static int rtw_acm_method;/*  0:By SW 1:By HW. */
 
 static int rtw_wmm_enable = 1;/*  default is set to enable the wmm. */
 static int rtw_uapsd_enable;
-static int rtw_uapsd_max_sp = NO_LIMIT;
-static int rtw_uapsd_acbk_en;
-static int rtw_uapsd_acbe_en;
-static int rtw_uapsd_acvi_en;
-static int rtw_uapsd_acvo_en;
 
 static int rtw_ht_enable = 1;
 /* 0 :disable, bit(0): enable 2.4g, bit(1): enable 5g */
@@ -89,11 +70,6 @@ static int rtw_ampdu_enable = 1;/* for enable tx_ampdu */
 static int rtw_rx_stbc = 1;
 static int rtw_ampdu_amsdu;/*  0: disabled, 1:enabled, 2:auto */
 
-/* Use 2 path Tx to transmit MCS0~7 and legacy mode */
-static int rtw_lowrate_two_xmit = 1;
-
-static int rtw_rf_config = RF_819X_MAX_TYPE;  /* auto */
-static int rtw_low_power;
 static int rtw_wifi_spec;
 static int rtw_channel_plan = RT_CHANNEL_DOMAIN_MAX;
 
@@ -111,10 +87,6 @@ static int rtw_enusbss;/* 0:disable, 1:enable */
 
 static int rtw_hwpdn_mode = 2;/* 0:disable, 1:enable, 2: by EFUSE config */
 
-static int rtw_hwpwrp_detect; /* HW power  ping detect 0:disable , 1:enable */
-
-static int rtw_hw_wps_pbc = 1;
-
 int rtw_mc2u_disable;
 
 static int rtw_80211d;
@@ -132,32 +104,22 @@ char *rtw_initmac;
 
 module_param(rtw_initmac, charp, 0644);
 module_param(rtw_channel_plan, int, 0644);
-module_param(rtw_chip_version, int, 0644);
-module_param(rtw_rfintfs, int, 0644);
-module_param(rtw_lbkmode, int, 0644);
-module_param(rtw_network_mode, int, 0644);
 module_param(rtw_channel, int, 0644);
 module_param(rtw_wmm_enable, int, 0644);
 module_param(rtw_vrtl_carrier_sense, int, 0644);
 module_param(rtw_vcs_type, int, 0644);
-module_param(rtw_busy_thresh, int, 0644);
 module_param(rtw_ht_enable, int, 0644);
 module_param(rtw_cbw40_enable, int, 0644);
 module_param(rtw_ampdu_enable, int, 0644);
 module_param(rtw_rx_stbc, int, 0644);
 module_param(rtw_ampdu_amsdu, int, 0644);
-module_param(rtw_lowrate_two_xmit, int, 0644);
-module_param(rtw_rf_config, int, 0644);
 module_param(rtw_power_mgnt, int, 0644);
 module_param(rtw_smart_ps, int, 0644);
-module_param(rtw_low_power, int, 0644);
 module_param(rtw_wifi_spec, int, 0644);
 module_param(rtw_antdiv_cfg, int, 0644);
 module_param(rtw_antdiv_type, int, 0644);
 module_param(rtw_enusbss, int, 0644);
 module_param(rtw_hwpdn_mode, int, 0644);
-module_param(rtw_hwpwrp_detect, int, 0644);
-module_param(rtw_hw_wps_pbc, int, 0644);
 
 static uint rtw_max_roaming_times = 2;
 module_param(rtw_max_roaming_times, uint, 0644);
@@ -185,361 +147,11 @@ MODULE_PARM_DESC(monitor_enable, "Enable monitor inferface (default: false)");
 static int netdev_open(struct net_device *pnetdev);
 static int netdev_close(struct net_device *pnetdev);
 
-/* dummy routines */
-void rtw_proc_remove_one(struct net_device *dev)
-{
-}
-
-static void rtw_proc_init_one(struct net_device *dev)
-{
-}
-
-#if 0	/* TODO: Convert these to /sys */
-static void rtw_proc_init_one(struct net_device *dev)
-{
-	struct proc_dir_entry *dir_dev = NULL;
-	struct proc_dir_entry *entry = NULL;
-	struct adapter	*padapter = rtw_netdev_priv(dev);
-	u8 rf_type;
-
-	if (rtw_proc == NULL) {
-		memcpy(rtw_proc_name, DRV_NAME, sizeof(DRV_NAME));
-
-		rtw_proc = create_proc_entry(rtw_proc_name, S_IFDIR,
-					     init_net.proc_net);
-		if (rtw_proc == NULL) {
-			DBG_88E(KERN_ERR "Unable to create rtw_proc directory\n");
-			return;
-		}
-
-		entry = create_proc_read_entry("ver_info", S_IFREG | S_IRUGO,
-					       rtw_proc, proc_get_drv_version,
-					       dev);
-		if (!entry) {
-			pr_info("Unable to create_proc_read_entry!\n");
-			return;
-		}
-	}
-
-	if (padapter->dir_dev == NULL) {
-		padapter->dir_dev = create_proc_entry(dev->name,
-					  S_IFDIR | S_IRUGO | S_IXUGO,
-					  rtw_proc);
-		dir_dev = padapter->dir_dev;
-		if (dir_dev == NULL) {
-			if (rtw_proc_cnt == 0 && rtw_proc) {
-				remove_proc_entry(rtw_proc_name, init_net.proc_net);
-				rtw_proc = NULL;
-			}
-
-			pr_info("Unable to create dir_dev directory\n");
-			return;
-		}
-	} else {
-		return;
-	}
-
-	rtw_proc_cnt++;
-
-	entry = create_proc_read_entry("write_reg", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_write_reg, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-	entry->write_proc = proc_set_write_reg;
-
-	entry = create_proc_read_entry("read_reg", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_read_reg, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-	entry->write_proc = proc_set_read_reg;
-
-
-	entry = create_proc_read_entry("fwstate", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_fwstate, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("sec_info", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_sec_info, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("mlmext_state", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_mlmext_state, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("qos_option", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_qos_option, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("ht_option", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_ht_option, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("rf_info", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_rf_info, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("ap_info", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_ap_info, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("adapter_state", S_IFREG | S_IRUGO,
-				   dir_dev, proc_getstruct adapter_state, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("trx_info", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_trx_info, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("mac_reg_dump1", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_mac_reg_dump1, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("mac_reg_dump2", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_mac_reg_dump2, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("mac_reg_dump3", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_mac_reg_dump3, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("bb_reg_dump1", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_bb_reg_dump1, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("bb_reg_dump2", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_bb_reg_dump2, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("bb_reg_dump3", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_bb_reg_dump3, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("rf_reg_dump1", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_rf_reg_dump1, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("rf_reg_dump2", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_rf_reg_dump2, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	rtw_hal_get_hwreg(padapter, HW_VAR_RF_TYPE, (u8 *)(&rf_type));
-	if ((RF_1T2R == rf_type) || (RF_1T1R == rf_type)) {
-		entry = create_proc_read_entry("rf_reg_dump3",
-					       S_IFREG | S_IRUGO, dir_dev,
-					       proc_get_rf_reg_dump3, dev);
-		if (!entry) {
-			pr_info("Unable to create_proc_read_entry!\n");
-			return;
-		}
-
-		entry = create_proc_read_entry("rf_reg_dump4",
-					       S_IFREG | S_IRUGO, dir_dev,
-					       proc_get_rf_reg_dump4, dev);
-		if (!entry) {
-			pr_info("Unable to create_proc_read_entry!\n");
-			return;
-		}
-	}
-
-#ifdef CONFIG_88EU_AP_MODE
-
-	entry = create_proc_read_entry("all_sta_info", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_all_sta_info, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-#endif
-
-	entry = create_proc_read_entry("best_channel", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_best_channel, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-
-	entry = create_proc_read_entry("rx_signal", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_rx_signal, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-	entry->write_proc = proc_set_rx_signal;
-	entry = create_proc_read_entry("ht_enable", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_ht_enable, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-	entry->write_proc = proc_set_ht_enable;
-
-	entry = create_proc_read_entry("cbw40_enable", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_cbw40_enable, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-	entry->write_proc = proc_set_cbw40_enable;
-
-	entry = create_proc_read_entry("ampdu_enable", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_ampdu_enable, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-	entry->write_proc = proc_set_ampdu_enable;
-
-	entry = create_proc_read_entry("rx_stbc", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_rx_stbc, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-	entry->write_proc = proc_set_rx_stbc;
-
-	entry = create_proc_read_entry("path_rssi", S_IFREG | S_IRUGO,
-					dir_dev, proc_get_two_path_rssi, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-	entry = create_proc_read_entry("rssi_disp", S_IFREG | S_IRUGO,
-				   dir_dev, proc_get_rssi_disp, dev);
-	if (!entry) {
-		pr_info("Unable to create_proc_read_entry!\n");
-		return;
-	}
-	entry->write_proc = proc_set_rssi_disp;
-}
-
-void rtw_proc_remove_one(struct net_device *dev)
-{
-	struct proc_dir_entry *dir_dev = NULL;
-	struct adapter	*padapter = rtw_netdev_priv(dev);
-	u8 rf_type;
-
-	dir_dev = padapter->dir_dev;
-	padapter->dir_dev = NULL;
-
-	if (dir_dev) {
-		remove_proc_entry("write_reg", dir_dev);
-		remove_proc_entry("read_reg", dir_dev);
-		remove_proc_entry("fwstate", dir_dev);
-		remove_proc_entry("sec_info", dir_dev);
-		remove_proc_entry("mlmext_state", dir_dev);
-		remove_proc_entry("qos_option", dir_dev);
-		remove_proc_entry("ht_option", dir_dev);
-		remove_proc_entry("rf_info", dir_dev);
-		remove_proc_entry("ap_info", dir_dev);
-		remove_proc_entry("adapter_state", dir_dev);
-		remove_proc_entry("trx_info", dir_dev);
-		remove_proc_entry("mac_reg_dump1", dir_dev);
-		remove_proc_entry("mac_reg_dump2", dir_dev);
-		remove_proc_entry("mac_reg_dump3", dir_dev);
-		remove_proc_entry("bb_reg_dump1", dir_dev);
-		remove_proc_entry("bb_reg_dump2", dir_dev);
-		remove_proc_entry("bb_reg_dump3", dir_dev);
-		remove_proc_entry("rf_reg_dump1", dir_dev);
-		remove_proc_entry("rf_reg_dump2", dir_dev);
-		rtw_hal_get_hwreg(padapter, HW_VAR_RF_TYPE, (u8 *)(&rf_type));
-		if ((RF_1T2R == rf_type) || (RF_1T1R == rf_type)) {
-			remove_proc_entry("rf_reg_dump3", dir_dev);
-			remove_proc_entry("rf_reg_dump4", dir_dev);
-		}
-#ifdef CONFIG_88EU_AP_MODE
-		remove_proc_entry("all_sta_info", dir_dev);
-#endif
-
-		remove_proc_entry("best_channel", dir_dev);
-		remove_proc_entry("rx_signal", dir_dev);
-		remove_proc_entry("cbw40_enable", dir_dev);
-		remove_proc_entry("ht_enable", dir_dev);
-		remove_proc_entry("ampdu_enable", dir_dev);
-		remove_proc_entry("rx_stbc", dir_dev);
-		remove_proc_entry("path_rssi", dir_dev);
-		remove_proc_entry("rssi_disp", dir_dev);
-		remove_proc_entry(dev->name, rtw_proc);
-		dir_dev = NULL;
-	} else {
-		return;
-	}
-	rtw_proc_cnt--;
-
-	if (rtw_proc_cnt == 0) {
-		if (rtw_proc) {
-			remove_proc_entry("ver_info", rtw_proc);
-
-			remove_proc_entry(rtw_proc_name, init_net.proc_net);
-			rtw_proc = NULL;
-		}
-	}
-}
-#endif
-
 static void loadparam(struct adapter *padapter, struct net_device *pnetdev)
 {
 	struct registry_priv  *registry_par = &padapter->registrypriv;
 
 	GlobalDebugLevel = rtw_debug;
-	registry_par->chip_version = (u8)rtw_chip_version;
-	registry_par->rfintfs = (u8)rtw_rfintfs;
-	registry_par->lbkmode = (u8)rtw_lbkmode;
-	registry_par->network_mode  = (u8)rtw_network_mode;
 
 	memcpy(registry_par->ssid.Ssid, "ANY", 3);
 	registry_par->ssid.SsidLength = 3;
@@ -551,17 +163,9 @@ static void loadparam(struct adapter *padapter, struct net_device *pnetdev)
 	registry_par->rts_thresh = (u16)rtw_rts_thresh;
 	registry_par->frag_thresh = (u16)rtw_frag_thresh;
 	registry_par->preamble = (u8)rtw_preamble;
-	registry_par->scan_mode = (u8)rtw_scan_mode;
-	registry_par->adhoc_tx_pwr = (u8)rtw_adhoc_tx_pwr;
-	registry_par->soft_ap =  (u8)rtw_soft_ap;
 	registry_par->smart_ps =  (u8)rtw_smart_ps;
 	registry_par->power_mgnt = (u8)rtw_power_mgnt;
 	registry_par->ips_mode = (u8)rtw_ips_mode;
-	registry_par->radio_enable = (u8)rtw_radio_enable;
-	registry_par->long_retry_lmt = (u8)rtw_long_retry_lmt;
-	registry_par->short_retry_lmt = (u8)rtw_short_retry_lmt;
-	registry_par->busy_thresh = (u16)rtw_busy_thresh;
-	registry_par->ack_policy = (u8)rtw_ack_policy;
 	registry_par->mp_mode = 0;
 	registry_par->software_encrypt = (u8)rtw_software_encrypt;
 	registry_par->software_decrypt = (u8)rtw_software_decrypt;
@@ -570,28 +174,18 @@ static void loadparam(struct adapter *padapter, struct net_device *pnetdev)
 	 /* UAPSD */
 	registry_par->wmm_enable = (u8)rtw_wmm_enable;
 	registry_par->uapsd_enable = (u8)rtw_uapsd_enable;
-	registry_par->uapsd_max_sp = (u8)rtw_uapsd_max_sp;
-	registry_par->uapsd_acbk_en = (u8)rtw_uapsd_acbk_en;
-	registry_par->uapsd_acbe_en = (u8)rtw_uapsd_acbe_en;
-	registry_par->uapsd_acvi_en = (u8)rtw_uapsd_acvi_en;
-	registry_par->uapsd_acvo_en = (u8)rtw_uapsd_acvo_en;
 
 	registry_par->ht_enable = (u8)rtw_ht_enable;
 	registry_par->cbw40_enable = (u8)rtw_cbw40_enable;
 	registry_par->ampdu_enable = (u8)rtw_ampdu_enable;
 	registry_par->rx_stbc = (u8)rtw_rx_stbc;
 	registry_par->ampdu_amsdu = (u8)rtw_ampdu_amsdu;
-	registry_par->lowrate_two_xmit = (u8)rtw_lowrate_two_xmit;
-	registry_par->rf_config = (u8)rtw_rf_config;
-	registry_par->low_power = (u8)rtw_low_power;
 	registry_par->wifi_spec = (u8)rtw_wifi_spec;
 	registry_par->channel_plan = (u8)rtw_channel_plan;
 	registry_par->accept_addba_req = true;
 	registry_par->antdiv_cfg = (u8)rtw_antdiv_cfg;
 	registry_par->antdiv_type = (u8)rtw_antdiv_type;
 	registry_par->hwpdn_mode = (u8)rtw_hwpdn_mode;
-	registry_par->hwpwrp_detect = (u8)rtw_hwpwrp_detect;
-	registry_par->hw_wps_pbc = (u8)rtw_hw_wps_pbc;
 
 	registry_par->max_roaming_times = (u8)rtw_max_roaming_times;
 
@@ -732,7 +326,7 @@ struct net_device *rtw_init_netdev(struct adapter *old_padapter)
 	RT_TRACE(_module_os_intfs_c_, _drv_info_, ("+init_net_dev\n"));
 
 	if (old_padapter != NULL)
-		pnetdev = rtw_alloc_etherdev_with_old_priv(sizeof(struct adapter), (void *)old_padapter);
+		pnetdev = rtw_alloc_etherdev_with_old_priv((void *)old_padapter);
 
 	if (!pnetdev)
 		return NULL;
@@ -762,7 +356,7 @@ static int rtw_start_drv_threads(struct adapter *padapter)
 		err = PTR_ERR(padapter->cmdThread);
 	else
 		/* wait for cmd_thread to run */
-		_rtw_down_sema(&padapter->cmdpriv.terminate_cmdthread_sema);
+		wait_for_completion_interruptible(&padapter->cmdpriv.terminate_cmdthread_comp);
 
 	return err;
 }
@@ -772,9 +366,9 @@ void rtw_stop_drv_threads(struct adapter *padapter)
 	RT_TRACE(_module_os_intfs_c_, _drv_info_, ("+rtw_stop_drv_threads\n"));
 
 	/* Below is to terminate rtw_cmd_thread & event_thread... */
-	up(&padapter->cmdpriv.cmd_queue_sema);
+	complete(&padapter->cmdpriv.cmd_queue_comp);
 	if (padapter->cmdThread)
-		_rtw_down_sema(&padapter->cmdpriv.terminate_cmdthread_sema);
+		wait_for_completion_interruptible(&padapter->cmdpriv.terminate_cmdthread_comp);
 
 }
 
@@ -821,7 +415,6 @@ static u8 rtw_init_default_value(struct adapter *padapter)
 	padapter->bReadPortCancel = false;
 	padapter->bWritePortCancel = false;
 	padapter->bRxRSSIDisplay = 0;
-	padapter->bNotifyChannelChange = 0;
 	return _SUCCESS;
 }
 
@@ -912,8 +505,6 @@ u8 rtw_init_drv_sw(struct adapter *padapter)
 
 	rtw_hal_sreset_init(padapter);
 
-	spin_lock_init(&padapter->br_ext_lock);
-
 exit:
 	RT_TRACE(_module_os_intfs_c_, _drv_info_, ("-rtw_init_drv_sw\n"));
 
@@ -961,12 +552,6 @@ u8 rtw_free_drv_sw(struct adapter *padapter)
 
 	RT_TRACE(_module_os_intfs_c_, _drv_info_, ("<== rtw_free_drv_sw\n"));
 
-	/* free the old_pnetdev */
-	if (padapter->rereg_nd_name_priv.old_pnetdev) {
-		free_netdev(padapter->rereg_nd_name_priv.old_pnetdev);
-		padapter->rereg_nd_name_priv.old_pnetdev = NULL;
-	}
-
 	mutex_destroy(&padapter->hw_init_mutex);
 
 	RT_TRACE(_module_os_intfs_c_, _drv_info_, ("-rtw_free_drv_sw\n"));
@@ -1013,7 +598,6 @@ static int _netdev_open(struct net_device *pnetdev)
 		}
 		if (padapter->intf_start)
 			padapter->intf_start(padapter);
-		rtw_proc_init_one(pnetdev);
 
 		rtw_led_control(padapter, LED_CTL_NO_LINK);
 
diff --git a/drivers/staging/rtl8188eu/os_dep/osdep_service.c b/drivers/staging/rtl8188eu/os_dep/osdep_service.c
index 764250b4ba86..7cd2655f27fe 100644
--- a/drivers/staging/rtl8188eu/os_dep/osdep_service.c
+++ b/drivers/staging/rtl8188eu/os_dep/osdep_service.c
@@ -55,21 +55,13 @@ void *rtw_malloc2d(int h, int w, int size)
 	return a;
 }
 
-u32 _rtw_down_sema(struct semaphore *sema)
-{
-	if (down_interruptible(sema))
-		return _FAIL;
-	return _SUCCESS;
-}
-
 void	_rtw_init_queue(struct __queue *pqueue)
 {
 	INIT_LIST_HEAD(&(pqueue->queue));
 	spin_lock_init(&(pqueue->lock));
 }
 
-struct net_device *rtw_alloc_etherdev_with_old_priv(int sizeof_priv,
-						    void *old_priv)
+struct net_device *rtw_alloc_etherdev_with_old_priv(void *old_priv)
 {
 	struct net_device *pnetdev;
 	struct rtw_netdev_priv_indicator *pnpi;
@@ -80,7 +72,6 @@ struct net_device *rtw_alloc_etherdev_with_old_priv(int sizeof_priv,
 
 	pnpi = netdev_priv(pnetdev);
 	pnpi->priv = old_priv;
-	pnpi->sizeof_priv = sizeof_priv;
 
 RETURN:
 	return pnetdev;
diff --git a/drivers/staging/rtl8188eu/os_dep/recv_linux.c b/drivers/staging/rtl8188eu/os_dep/recv_linux.c
index 0c44914ea3e6..103cdb4ed073 100644
--- a/drivers/staging/rtl8188eu/os_dep/recv_linux.c
+++ b/drivers/staging/rtl8188eu/os_dep/recv_linux.c
@@ -24,7 +24,6 @@
 /* alloc os related resource in struct recv_frame */
 void rtw_os_recv_resource_alloc(struct recv_frame *precvframe)
 {
-	precvframe->pkt_newalloc = NULL;
 	precvframe->pkt = NULL;
 }
 
diff --git a/drivers/staging/rtl8188eu/os_dep/usb_intf.c b/drivers/staging/rtl8188eu/os_dep/usb_intf.c
index 11d51a30170f..68e1e6bbe87f 100644
--- a/drivers/staging/rtl8188eu/os_dep/usb_intf.c
+++ b/drivers/staging/rtl8188eu/os_dep/usb_intf.c
@@ -25,9 +25,10 @@
 #include <osdep_intf.h>
 
 #include <usb_ops_linux.h>
-#include <usb_hal.h>
 #include <rtw_ioctl.h>
 
+#include "rtl8188e_hal.h"
+
 #define USB_VENDER_ID_REALTEK		0x0bda
 
 /* DID_USB_v916_20130116 */
@@ -79,9 +80,8 @@ static struct dvobj_priv *usb_dvobj_init(struct usb_interface *usb_intf)
 
 	pdvobjpriv->NumInterfaces = pconf_desc->bNumInterfaces;
 	pdvobjpriv->InterfaceNumber = piface_desc->bInterfaceNumber;
-	pdvobjpriv->nr_endpoint = piface_desc->bNumEndpoints;
 
-	for (i = 0; i < pdvobjpriv->nr_endpoint; i++) {
+	for (i = 0; i < piface_desc->bNumEndpoints; i++) {
 		int ep_num;
 		pendp_desc = &phost_iface->endpoint[i].desc;
 
@@ -98,7 +98,6 @@ static struct dvobj_priv *usb_dvobj_init(struct usb_interface *usb_intf)
 				ep_num;
 			pdvobjpriv->RtNumOutPipes++;
 		}
-		pdvobjpriv->ep_num[i] = ep_num;
 	}
 
 	if (pusbd->speed == USB_SPEED_HIGH)
@@ -107,13 +106,6 @@ static struct dvobj_priv *usb_dvobj_init(struct usb_interface *usb_intf)
 		pdvobjpriv->ishighspeed = false;
 
 	mutex_init(&pdvobjpriv->usb_vendor_req_mutex);
-	pdvobjpriv->usb_vendor_req_buf = kzalloc(MAX_USB_IO_CTL_SIZE, GFP_KERNEL);
-
-	if (!pdvobjpriv->usb_vendor_req_buf) {
-		usb_set_intfdata(usb_intf, NULL);
-		kfree(pdvobjpriv);
-		return NULL;
-	}
 	usb_get_dev(pusbd);
 
 	return pdvobjpriv;
@@ -141,7 +133,6 @@ static void usb_dvobj_deinit(struct usb_interface *usb_intf)
 			}
 		}
 
-		kfree(dvobj->usb_vendor_req_buf);
 		mutex_destroy(&dvobj->usb_vendor_req_mutex);
 		kfree(dvobj);
 	}
@@ -238,7 +229,7 @@ static int rtw_suspend(struct usb_interface *pusb_intf, pm_message_t message)
 	rtw_cancel_all_timer(padapter);
 	LeaveAllPowerSaveMode(padapter);
 
-	_enter_pwrlock(&pwrpriv->lock);
+	mutex_lock(&pwrpriv->mutex_lock);
 	/* s1. */
 	if (pnetdev) {
 		netif_carrier_off(pnetdev);
@@ -267,7 +258,7 @@ static int rtw_suspend(struct usb_interface *pusb_intf, pm_message_t message)
 	rtw_free_network_queue(padapter, true);
 
 	rtw_dev_unload(padapter);
-	_exit_pwrlock(&pwrpriv->lock);
+	mutex_unlock(&pwrpriv->mutex_lock);
 
 	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
 		rtw_indicate_scan_done(padapter, 1);
@@ -298,18 +289,20 @@ static int rtw_resume_process(struct adapter *padapter)
 		goto exit;
 	}
 
-	_enter_pwrlock(&pwrpriv->lock);
+	mutex_lock(&pwrpriv->mutex_lock);
 	rtw_reset_drv_sw(padapter);
 	pwrpriv->bkeepfwalive = false;
 
 	pr_debug("bkeepfwalive(%x)\n", pwrpriv->bkeepfwalive);
-	if (pm_netdev_open(pnetdev, true) != 0)
+	if (pm_netdev_open(pnetdev, true) != 0) {
+		mutex_unlock(&pwrpriv->mutex_lock);
 		goto exit;
+	}
 
 	netif_device_attach(pnetdev);
 	netif_carrier_on(pnetdev);
 
-	_exit_pwrlock(&pwrpriv->lock);
+	mutex_unlock(&pwrpriv->mutex_lock);
 
 	rtw_roaming(padapter, NULL);
 
@@ -369,8 +362,9 @@ static struct adapter *rtw_usb_if1_init(struct dvobj_priv *dvobj,
 		padapter->pmondev = pmondev;
 	}
 
-	/* step 2. hook HalFunc, allocate HalData */
-	hal_set_hal_ops(padapter);
+	padapter->HalData = kzalloc(sizeof(struct hal_data_8188e), GFP_KERNEL);
+	if (!padapter->HalData)
+		DBG_88E("cant not alloc memory for HAL DATA\n");
 
 	padapter->intf_start = &usb_intf_start;
 	padapter->intf_stop = &usb_intf_stop;
@@ -456,11 +450,9 @@ static void rtw_usb_if1_deinit(struct adapter *if1)
 	free_mlme_ap_info(if1);
 #endif
 
-	if (pnetdev) {
-		/* will call netdev_close() */
-		unregister_netdev(pnetdev);
-		rtw_proc_remove_one(pnetdev);
-	}
+	if (pnetdev)
+		unregister_netdev(pnetdev); /* will call netdev_close() */
+
 	rtl88eu_mon_deinit(if1->pmondev);
 	rtw_cancel_all_timer(if1);
 
diff --git a/drivers/staging/rtl8188eu/os_dep/usb_ops_linux.c b/drivers/staging/rtl8188eu/os_dep/usb_ops_linux.c
index ce1e1a135f1b..d0d591501b73 100644
--- a/drivers/staging/rtl8188eu/os_dep/usb_ops_linux.c
+++ b/drivers/staging/rtl8188eu/os_dep/usb_ops_linux.c
@@ -20,7 +20,7 @@
 
 static void interrupt_handler_8188eu(struct adapter *adapt, u16 pkt_len, u8 *pbuf)
 {
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *haldata = adapt->HalData;
 
 	if (pkt_len != INTERRUPT_MSG_FORMAT_LEN) {
 		DBG_88E("%s Invalid interrupt content length (%d)!\n", __func__, pkt_len);
@@ -48,7 +48,7 @@ static int recvbuf2recvframe(struct adapter *adapt, struct sk_buff *pskb)
 	struct sk_buff *pkt_copy = NULL;
 	struct recv_frame	*precvframe = NULL;
 	struct rx_pkt_attrib	*pattrib = NULL;
-	struct hal_data_8188e	*haldata = GET_HAL_DATA(adapt);
+	struct hal_data_8188e *haldata = adapt->HalData;
 	struct recv_priv	*precvpriv = &adapt->recvpriv;
 	struct __queue *pfree_recv_queue = &precvpriv->free_recv_queue;
 
@@ -251,7 +251,7 @@ static int usbctrl_vendorreq(struct adapter *adapt, u8 request, u16 value, u16 i
 	}
 
 	/*  Acquire IO memory for vendorreq */
-	pIo_buf = dvobjpriv->usb_vendor_req_buf;
+	pIo_buf = kmalloc(MAX_USB_IO_CTL_SIZE, GFP_ATOMIC);
 
 	if (pIo_buf == NULL) {
 		DBG_88E("[%s] pIo_buf == NULL\n", __func__);
@@ -285,8 +285,7 @@ static int usbctrl_vendorreq(struct adapter *adapt, u8 request, u16 value, u16 i
 				if (status == (-ESHUTDOWN) || status == -ENODEV) {
 					adapt->bSurpriseRemoved = true;
 				} else {
-					struct hal_data_8188e	*haldata = GET_HAL_DATA(adapt);
-					haldata->srestpriv.Wifi_Error_Status = USB_VEN_REQ_CMD_FAIL;
+					adapt->HalData->srestpriv.Wifi_Error_Status = USB_VEN_REQ_CMD_FAIL;
 				}
 			} else { /*  status != len && status >= 0 */
 				if (status > 0) {
@@ -303,6 +302,8 @@ static int usbctrl_vendorreq(struct adapter *adapt, u8 request, u16 value, u16 i
 		if ((value >= FW_8188E_START_ADDRESS && value <= FW_8188E_END_ADDRESS) || status == len)
 			break;
 	}
+	kfree(pIo_buf);
+
 release_mutex:
 	mutex_unlock(&dvobjpriv->usb_vendor_req_mutex);
 exit:
@@ -434,10 +435,7 @@ static void usb_read_port_complete(struct urb *purb, struct pt_regs *regs)
 			break;
 		case -EPROTO:
 		case -EOVERFLOW:
-			{
-				struct hal_data_8188e	*haldata = GET_HAL_DATA(adapt);
-				haldata->srestpriv.Wifi_Error_Status = USB_READ_PORT_FAIL;
-			}
+			adapt->HalData->srestpriv.Wifi_Error_Status = USB_READ_PORT_FAIL;
 			precvbuf->reuse = true;
 			usb_read_port(adapt, precvpriv->ff_hwaddr, 0, (unsigned char *)precvbuf);
 			break;
@@ -525,7 +523,7 @@ u32 usb_read_port(struct adapter *adapter, u32 addr, u32 cnt, u8 *rmem)
 	return ret;
 }
 
-void usb_read_port_cancel(struct adapter *padapter)
+void rtw_hal_inirp_deinit(struct adapter *padapter)
 {
 	int i;
 	struct recv_buf *precvbuf;
@@ -691,7 +689,7 @@ check_completion:
 	tasklet_hi_schedule(&pxmitpriv->xmit_tasklet);
 }
 
-u32 usb_write_port(struct adapter *padapter, u32 addr, u32 cnt, u8 *wmem)
+u32 usb_write_port(struct adapter *padapter, u32 addr, u32 cnt, struct xmit_buf *xmitbuf)
 {
 	unsigned long irqL;
 	unsigned int pipe;
@@ -700,8 +698,7 @@ u32 usb_write_port(struct adapter *padapter, u32 addr, u32 cnt, u8 *wmem)
 	struct urb *purb = NULL;
 	struct dvobj_priv	*pdvobj = adapter_to_dvobj(padapter);
 	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
-	struct xmit_buf *pxmitbuf = (struct xmit_buf *)wmem;
-	struct xmit_frame *pxmitframe = (struct xmit_frame *)pxmitbuf->priv_data;
+	struct xmit_frame *pxmitframe = (struct xmit_frame *)xmitbuf->priv_data;
 	struct usb_device *pusbd = pdvobj->pusbdev;
 
 
@@ -711,7 +708,7 @@ u32 usb_write_port(struct adapter *padapter, u32 addr, u32 cnt, u8 *wmem)
 	    (padapter->pwrctrlpriv.pnp_bstop_trx)) {
 		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
 			 ("usb_write_port:( padapter->bDriverStopped ||padapter->bSurpriseRemoved ||adapter->pwrctrlpriv.pnp_bstop_trx)!!!\n"));
-		rtw_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_TX_DENY);
+		rtw_sctx_done_err(&xmitbuf->sctx, RTW_SCTX_DONE_TX_DENY);
 		goto exit;
 	}
 
@@ -720,44 +717,44 @@ u32 usb_write_port(struct adapter *padapter, u32 addr, u32 cnt, u8 *wmem)
 	switch (addr) {
 	case VO_QUEUE_INX:
 		pxmitpriv->voq_cnt++;
-		pxmitbuf->flags = VO_QUEUE_INX;
+		xmitbuf->flags = VO_QUEUE_INX;
 		break;
 	case VI_QUEUE_INX:
 		pxmitpriv->viq_cnt++;
-		pxmitbuf->flags = VI_QUEUE_INX;
+		xmitbuf->flags = VI_QUEUE_INX;
 		break;
 	case BE_QUEUE_INX:
 		pxmitpriv->beq_cnt++;
-		pxmitbuf->flags = BE_QUEUE_INX;
+		xmitbuf->flags = BE_QUEUE_INX;
 		break;
 	case BK_QUEUE_INX:
 		pxmitpriv->bkq_cnt++;
-		pxmitbuf->flags = BK_QUEUE_INX;
+		xmitbuf->flags = BK_QUEUE_INX;
 		break;
 	case HIGH_QUEUE_INX:
-		pxmitbuf->flags = HIGH_QUEUE_INX;
+		xmitbuf->flags = HIGH_QUEUE_INX;
 		break;
 	default:
-		pxmitbuf->flags = MGT_QUEUE_INX;
+		xmitbuf->flags = MGT_QUEUE_INX;
 		break;
 	}
 
 	spin_unlock_irqrestore(&pxmitpriv->lock, irqL);
 
-	purb	= pxmitbuf->pxmit_urb[0];
+	purb	= xmitbuf->pxmit_urb[0];
 
 	/* translate DMA FIFO addr to pipehandle */
 	pipe = ffaddr2pipehdl(pdvobj, addr);
 
 	usb_fill_bulk_urb(purb, pusbd, pipe,
-			  pxmitframe->buf_addr, /*  pxmitbuf->pbuf */
+			  pxmitframe->buf_addr, /*  xmitbuf->pbuf */
 			  cnt,
 			  usb_write_port_complete,
-			  pxmitbuf);/* context is pxmitbuf */
+			  xmitbuf);/* context is xmitbuf */
 
 	status = usb_submit_urb(purb, GFP_ATOMIC);
 	if (status) {
-		rtw_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_WRITE_PORT_ERR);
+		rtw_sctx_done_err(&xmitbuf->sctx, RTW_SCTX_DONE_WRITE_PORT_ERR);
 		DBG_88E("usb_write_port, status =%d\n", status);
 		RT_TRACE(_module_hci_ops_os_c_, _drv_err_, ("usb_write_port(): usb_submit_urb, status =%x\n", status));
 
@@ -779,7 +776,7 @@ u32 usb_write_port(struct adapter *padapter, u32 addr, u32 cnt, u8 *wmem)
 
 exit:
 	if (ret != _SUCCESS)
-		rtw_free_xmitbuf(pxmitpriv, pxmitbuf);
+		rtw_free_xmitbuf(pxmitpriv, xmitbuf);
 	return ret;
 }
 
@@ -846,7 +843,7 @@ void rtl8188eu_xmit_tasklet(void *priv)
 			break;
 		}
 
-		ret = rtl8188eu_xmitframe_complete(adapt, pxmitpriv, NULL);
+		ret = rtl8188eu_xmitframe_complete(adapt, pxmitpriv);
 
 		if (!ret)
 			break;
diff --git a/drivers/staging/rtl8188eu/os_dep/xmit_linux.c b/drivers/staging/rtl8188eu/os_dep/xmit_linux.c
index 221e2750652e..4b1b04e00715 100644
--- a/drivers/staging/rtl8188eu/os_dep/xmit_linux.c
+++ b/drivers/staging/rtl8188eu/os_dep/xmit_linux.c
@@ -72,7 +72,7 @@ int rtw_os_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf *pxmitb
 	if (pxmitbuf->pallocated_buf == NULL)
 		return _FAIL;
 
-	pxmitbuf->pbuf = (u8 *)N_BYTE_ALIGMENT((size_t)(pxmitbuf->pallocated_buf), XMITBUF_ALIGN_SZ);
+	pxmitbuf->pbuf = PTR_ALIGN(pxmitbuf->pallocated_buf, XMITBUF_ALIGN_SZ);
 	pxmitbuf->dma_transfer_addr = 0;
 
 	for (i = 0; i < 8; i++) {
diff --git a/drivers/staging/rtl8192e/dot11d.c b/drivers/staging/rtl8192e/dot11d.c
index 4d8fb4158f6b..25725b158eca 100644
--- a/drivers/staging/rtl8192e/dot11d.c
+++ b/drivers/staging/rtl8192e/dot11d.c
@@ -51,9 +51,8 @@ void dot11d_init(struct rtllib_device *ieee)
 	pDot11dInfo->State = DOT11D_STATE_NONE;
 	pDot11dInfo->CountryIeLen = 0;
 	memset(pDot11dInfo->channel_map, 0, MAX_CHANNEL_NUMBER+1);
-	memset(pDot11dInfo->MaxTxPwrDbmList, 0xFF, MAX_CHANNEL_NUMBER+1);
+	memset(pDot11dInfo->MaxTxPwrDbmList, 0xFF, MAX_CHANNEL_NUMBER + 1);
 	RESET_CIE_WATCHDOG(ieee);
-
 }
 EXPORT_SYMBOL(dot11d_init);
 
@@ -99,14 +98,13 @@ void Dot11d_Channelmap(u8 channel_plan, struct rtllib_device *ieee)
 }
 EXPORT_SYMBOL(Dot11d_Channelmap);
 
-
 void Dot11d_Reset(struct rtllib_device *ieee)
 {
 	struct rt_dot11d_info *pDot11dInfo = GET_DOT11D_INFO(ieee);
 	u32 i;
 
-	memset(pDot11dInfo->channel_map, 0, MAX_CHANNEL_NUMBER+1);
-	memset(pDot11dInfo->MaxTxPwrDbmList, 0xFF, MAX_CHANNEL_NUMBER+1);
+	memset(pDot11dInfo->channel_map, 0, MAX_CHANNEL_NUMBER + 1);
+	memset(pDot11dInfo->MaxTxPwrDbmList, 0xFF, MAX_CHANNEL_NUMBER + 1);
 	for (i = 1; i <= 11; i++)
 		(pDot11dInfo->channel_map)[i] = 1;
 	for (i = 12; i <= 14; i++)
@@ -123,8 +121,8 @@ void Dot11d_UpdateCountryIe(struct rtllib_device *dev, u8 *pTaddr,
 	u8 i, j, NumTriples, MaxChnlNum;
 	struct chnl_txpow_triple *pTriple;
 
-	memset(pDot11dInfo->channel_map, 0, MAX_CHANNEL_NUMBER+1);
-	memset(pDot11dInfo->MaxTxPwrDbmList, 0xFF, MAX_CHANNEL_NUMBER+1);
+	memset(pDot11dInfo->channel_map, 0, MAX_CHANNEL_NUMBER + 1);
+	memset(pDot11dInfo->MaxTxPwrDbmList, 0xFF, MAX_CHANNEL_NUMBER + 1);
 	MaxChnlNum = 0;
 	NumTriples = (CoutryIeLen - 3) / 3;
 	pTriple = (struct chnl_txpow_triple *)(pCoutryIe + 3);
diff --git a/drivers/staging/rtl8192e/dot11d.h b/drivers/staging/rtl8192e/dot11d.h
index 735a199ebdcf..aac395f26652 100644
--- a/drivers/staging/rtl8192e/dot11d.h
+++ b/drivers/staging/rtl8192e/dot11d.h
@@ -47,8 +47,8 @@ struct rt_dot11d_info {
 	u8  CountryIeSrcAddr[6];
 	u8  CountryIeWatchdog;
 
-	u8  channel_map[MAX_CHANNEL_NUMBER+1];
-	u8  MaxTxPwrDbmList[MAX_CHANNEL_NUMBER+1];
+	u8  channel_map[MAX_CHANNEL_NUMBER + 1];
+	u8  MaxTxPwrDbmList[MAX_CHANNEL_NUMBER + 1];
 
 	enum dot11d_state State;
 };
@@ -78,6 +78,7 @@ static inline void RESET_CIE_WATCHDOG(struct rtllib_device *__pIeeeDev)
 {
 	GET_CIE_WATCHDOG(__pIeeeDev) = 0;
 }
+
 #define UPDATE_CIE_WATCHDOG(__pIeeeDev) (++GET_CIE_WATCHDOG(__pIeeeDev))
 
 void dot11d_init(struct rtllib_device *dev);
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c
index ba64a4f1b3a8..8d6bca61e7aa 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c
@@ -1487,8 +1487,8 @@ static void _rtl92e_query_rxphystatus(
 	struct phy_ofdm_rx_status_rxsc_sgien_exintfflag *prxsc;
 	u8 *prxpkt;
 	u8 i, max_spatial_stream, tmp_rxsnr, tmp_rxevm, rxsc_sgien_exflg;
-	char rx_pwr[4], rx_pwr_all = 0;
-	char rx_snrX, rx_evmX;
+	s8 rx_pwr[4], rx_pwr_all = 0;
+	s8 rx_snrX, rx_evmX;
 	u8 evm, pwdb_all;
 	u32 RSSI, total_rssi = 0;
 	u8 is_cck_rate = 0;
@@ -1613,7 +1613,7 @@ static void _rtl92e_query_rxphystatus(
 				     2) - 110;
 
 			tmp_rxsnr = pofdm_buf->rxsnr_X[i];
-			rx_snrX = (char)(tmp_rxsnr);
+			rx_snrX = (s8)(tmp_rxsnr);
 			rx_snrX /= 2;
 			priv->stats.rxSNRdB[i] = (long)rx_snrX;
 
@@ -1643,7 +1643,7 @@ static void _rtl92e_query_rxphystatus(
 
 		for (i = 0; i < max_spatial_stream; i++) {
 			tmp_rxevm = pofdm_buf->rxevm_X[i];
-			rx_evmX = (char)(tmp_rxevm);
+			rx_evmX = (s8)(tmp_rxevm);
 
 			rx_evmX /= 2;
 
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_hwimg.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_hwimg.c
index 29cefb599bab..d437a8efe933 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_hwimg.c
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_hwimg.c
@@ -11,7 +11,7 @@
  *
  * Contact Information:
  * wlanfae <wlanfae@realtek.com>
-******************************************************************************/
+ *****************************************************************************/
 /*Created on  2008/11/18,  3: 7*/
 
 #include "r8192E_hwimg.h"
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c
index 5e3bbe5c3ca4..dde492261451 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c
@@ -256,7 +256,7 @@ u32 rtl92e_get_rf_reg(struct net_device *dev, enum rf90_radio_path eRFPath,
 		return 0;
 	if (priv->rtllib->eRFPowerState != eRfOn && !priv->being_init_adapter)
 		return	0;
-	down(&priv->rf_sem);
+	mutex_lock(&priv->rf_mutex);
 	if (priv->Rf_Mode == RF_OP_By_FW) {
 		Original_Value = _rtl92e_phy_rf_fw_read(dev, eRFPath, RegAddr);
 		udelay(200);
@@ -265,7 +265,7 @@ u32 rtl92e_get_rf_reg(struct net_device *dev, enum rf90_radio_path eRFPath,
 	}
 	BitShift =  _rtl92e_calculate_bit_shift(BitMask);
 	Readback_Value = (Original_Value & BitMask) >> BitShift;
-	up(&priv->rf_sem);
+	mutex_unlock(&priv->rf_mutex);
 	return Readback_Value;
 }
 
@@ -630,7 +630,7 @@ void rtl92e_set_tx_power(struct net_device *dev, u8 channel)
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
 	u8	powerlevel = 0, powerlevelOFDM24G = 0;
-	char ant_pwr_diff;
+	s8	ant_pwr_diff;
 	u32	u4RegValue;
 
 	if (priv->epromtype == EEPROM_93C46) {
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_cam.c b/drivers/staging/rtl8192e/rtl8192e/rtl_cam.c
index 803c8b02a0c8..30f65af4d614 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_cam.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_cam.c
@@ -107,9 +107,9 @@ void rtl92e_set_key(struct net_device *dev, u8 EntryNo, u8 KeyIndex,
 					    __func__);
 				return;
 			}
-			down(&priv->rtllib->ips_sem);
+			mutex_lock(&priv->rtllib->ips_mutex);
 			rtl92e_ips_leave(dev);
-			up(&priv->rtllib->ips_sem);
+			mutex_unlock(&priv->rtllib->ips_mutex);
 		}
 	}
 	priv->rtllib->is_set_key = true;
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
index 13a5ddc2bea5..4c30eea45f89 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
@@ -38,7 +38,7 @@ static int channels = 0x3fff;
 static char *ifname = "wlan%d";
 
 
-static struct rtl819x_ops rtl819xp_ops = {
+static const struct rtl819x_ops rtl819xp_ops = {
 	.nic_type			= NIC_8192E,
 	.get_eeprom_size		= rtl92e_get_eeprom_size,
 	.init_adapter_variable		= rtl92e_init_variables,
@@ -993,8 +993,8 @@ static void _rtl92e_init_priv_lock(struct r8192_priv *priv)
 	spin_lock_init(&priv->irq_th_lock);
 	spin_lock_init(&priv->rf_ps_lock);
 	spin_lock_init(&priv->ps_lock);
-	sema_init(&priv->wx_sem, 1);
-	sema_init(&priv->rf_sem, 1);
+	mutex_init(&priv->wx_mutex);
+	mutex_init(&priv->rf_mutex);
 	mutex_init(&priv->mutex);
 }
 
@@ -1247,7 +1247,7 @@ static void _rtl92e_if_silent_reset(struct net_device *dev)
 
 RESET_START:
 
-		down(&priv->wx_sem);
+		mutex_lock(&priv->wx_mutex);
 
 		if (priv->rtllib->state == RTLLIB_LINKED)
 			rtl92e_leisure_ps_leave(dev);
@@ -1255,7 +1255,7 @@ RESET_START:
 		if (priv->up) {
 			netdev_info(dev, "%s():the driver is not up.\n",
 				    __func__);
-			up(&priv->wx_sem);
+			mutex_unlock(&priv->wx_mutex);
 			return;
 		}
 		priv->up = 0;
@@ -1277,14 +1277,14 @@ RESET_START:
 		rtllib_stop_scan_syncro(ieee);
 
 		if (ieee->state == RTLLIB_LINKED) {
-			SEM_DOWN_IEEE_WX(&ieee->wx_sem);
+			mutex_lock(&ieee->wx_mutex);
 			netdev_info(dev, "ieee->state is RTLLIB_LINKED\n");
 			rtllib_stop_send_beacons(priv->rtllib);
 			del_timer_sync(&ieee->associate_timer);
 			cancel_delayed_work(&ieee->associate_retry_wq);
 			rtllib_stop_scan(ieee);
 			netif_carrier_off(dev);
-			SEM_UP_IEEE_WX(&ieee->wx_sem);
+			mutex_unlock(&ieee->wx_mutex);
 		} else {
 			netdev_info(dev, "ieee->state is NOT LINKED\n");
 			rtllib_softmac_stop_protocol(priv->rtllib, 0, true);
@@ -1292,7 +1292,7 @@ RESET_START:
 
 		rtl92e_dm_backup_state(dev);
 
-		up(&priv->wx_sem);
+		mutex_unlock(&priv->wx_mutex);
 		RT_TRACE(COMP_RESET,
 			 "%s():<==========down process is finished\n",
 			 __func__);
@@ -1982,7 +1982,7 @@ void rtl92e_update_rx_statistics(struct r8192_priv *priv,
 					weighting) / 6;
 }
 
-u8 rtl92e_rx_db_to_percent(char antpower)
+u8 rtl92e_rx_db_to_percent(s8 antpower)
 {
 	if ((antpower <= -100) || (antpower >= 20))
 		return	0;
@@ -1993,9 +1993,9 @@ u8 rtl92e_rx_db_to_percent(char antpower)
 
 }	/* QueryRxPwrPercentage */
 
-u8 rtl92e_evm_db_to_percent(char value)
+u8 rtl92e_evm_db_to_percent(s8 value)
 {
-	char ret_val;
+	s8 ret_val;
 
 	ret_val = value;
 
@@ -2179,9 +2179,9 @@ static int _rtl92e_open(struct net_device *dev)
 	struct r8192_priv *priv = rtllib_priv(dev);
 	int ret;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 	ret = _rtl92e_try_up(dev);
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return ret;
 
 }
@@ -2206,11 +2206,11 @@ static int _rtl92e_close(struct net_device *dev)
 		rtllib_stop_scan(priv->rtllib);
 	}
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ret = _rtl92e_down(dev, true);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 
@@ -2242,11 +2242,11 @@ static void _rtl92e_restart(void *data)
 				  reset_wq);
 	struct net_device *dev = priv->rtllib->dev;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	rtl92e_commit(dev);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 }
 
 static void _rtl92e_set_multicast(struct net_device *dev)
@@ -2265,12 +2265,12 @@ static int _rtl92e_set_mac_adr(struct net_device *dev, void *mac)
 	struct r8192_priv *priv = rtllib_priv(dev);
 	struct sockaddr *addr = mac;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ether_addr_copy(dev->dev_addr, addr->sa_data);
 
 	schedule_work(&priv->reset_wq);
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return 0;
 }
@@ -2287,7 +2287,7 @@ static int _rtl92e_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 	struct iw_point *p = &wrq->u.data;
 	struct ieee_param *ipw = NULL;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	switch (cmd) {
 	case RTL_IOCTL_WPA_SUPPLICANT:
@@ -2393,7 +2393,7 @@ static int _rtl92e_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 	}
 
 out:
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 }
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.h b/drivers/staging/rtl8192e/rtl8192e/rtl_core.h
index f627fdc15a58..babc0b3bce95 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.h
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.h
@@ -375,8 +375,8 @@ struct r8192_priv {
 	struct tasklet_struct		irq_tx_tasklet;
 	struct tasklet_struct		irq_prepare_beacon_tasklet;
 
-	struct semaphore			wx_sem;
-	struct semaphore			rf_sem;
+	struct mutex				wx_mutex;
+	struct mutex				rf_mutex;
 	struct mutex				mutex;
 
 	struct rt_stats stats;
@@ -503,8 +503,8 @@ struct r8192_priv {
 	u32 Pwr_Track;
 	u8 CCKPresentAttentuation_20Mdefault;
 	u8 CCKPresentAttentuation_40Mdefault;
-	char CCKPresentAttentuation_difference;
-	char CCKPresentAttentuation;
+	s8 CCKPresentAttentuation_difference;
+	s8 CCKPresentAttentuation;
 	long undecorated_smoothed_pwdb;
 
 	u32 MCSTxPowerLevelOriginalOffset[6];
@@ -604,8 +604,8 @@ void rtl92e_update_rx_pkt_timestamp(struct net_device *dev,
 long rtl92e_translate_to_dbm(struct r8192_priv *priv, u8 signal_strength_index);
 void rtl92e_update_rx_statistics(struct r8192_priv *priv,
 				 struct rtllib_rx_stats *pprevious_stats);
-u8 rtl92e_evm_db_to_percent(char value);
-u8 rtl92e_rx_db_to_percent(char antpower);
+u8 rtl92e_evm_db_to_percent(s8 value);
+u8 rtl92e_rx_db_to_percent(s8 antpower);
 void rtl92e_copy_mpdu_stats(struct rtllib_rx_stats *psrc_stats,
 			    struct rtllib_rx_stats *ptarget_stats);
 bool rtl92e_enable_nic(struct net_device *dev);
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c b/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c
index 98e4d88d0e73..aa4b015c3cc7 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c
@@ -179,9 +179,9 @@ void rtl92e_ips_leave_wq(void *data)
 	struct net_device *dev = ieee->dev;
 	struct r8192_priv *priv = (struct r8192_priv *)rtllib_priv(dev);
 
-	down(&priv->rtllib->ips_sem);
+	mutex_lock(&priv->rtllib->ips_mutex);
 	rtl92e_ips_leave(dev);
-	up(&priv->rtllib->ips_sem);
+	mutex_unlock(&priv->rtllib->ips_mutex);
 }
 
 void rtl92e_rtllib_ips_leave_wq(struct net_device *dev)
@@ -209,9 +209,9 @@ void rtl92e_rtllib_ips_leave(struct net_device *dev)
 {
 	struct r8192_priv *priv = (struct r8192_priv *)rtllib_priv(dev);
 
-	down(&priv->rtllib->ips_sem);
+	mutex_lock(&priv->rtllib->ips_mutex);
 	rtl92e_ips_leave(dev);
-	up(&priv->rtllib->ips_sem);
+	mutex_unlock(&priv->rtllib->ips_mutex);
 }
 
 static bool _rtl92e_ps_set_mode(struct net_device *dev, u8 rtPsMode)
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c b/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c
index 70df6a1485d6..7413a100ca19 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c
@@ -65,11 +65,11 @@ static int _rtl92e_wx_set_rate(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ret = rtllib_wx_set_rate(priv->rtllib, info, wrqu, extra);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 }
@@ -84,11 +84,11 @@ static int _rtl92e_wx_set_rts(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ret = rtllib_wx_set_rts(priv->rtllib, info, wrqu, extra);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 }
@@ -114,11 +114,11 @@ static int _rtl92e_wx_set_power(struct net_device *dev,
 			    __func__);
 		return 0;
 	}
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ret = rtllib_wx_set_power(priv->rtllib, info, wrqu, extra);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 }
@@ -142,11 +142,11 @@ static int _rtl92e_wx_set_rawtx(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ret = rtllib_wx_set_rawtx(priv->rtllib, info, wrqu, extra);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 
@@ -158,12 +158,12 @@ static int _rtl92e_wx_force_reset(struct net_device *dev,
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	RT_TRACE(COMP_DBG, "%s(): force reset ! extra is %d\n",
 		 __func__, *extra);
 	priv->force_reset = *extra;
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return 0;
 
 }
@@ -177,7 +177,7 @@ static int _rtl92e_wx_adapter_power_status(struct net_device *dev,
 					(&(priv->rtllib->PowerSaveControl));
 	struct rtllib_device *ieee = priv->rtllib;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	RT_TRACE(COMP_POWER, "%s(): %s\n", __func__, (*extra == 6) ?
 		 "DC power" : "AC power");
@@ -193,7 +193,7 @@ static int _rtl92e_wx_adapter_power_status(struct net_device *dev,
 		ieee->ps = *extra;
 	}
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return 0;
 }
@@ -207,13 +207,13 @@ static int _rtl92e_wx_set_lps_awake_interval(struct net_device *dev,
 	struct rt_pwr_save_ctrl *pPSC = (struct rt_pwr_save_ctrl *)
 					(&(priv->rtllib->PowerSaveControl));
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	netdev_info(dev, "%s(): set lps awake interval ! extra is %d\n",
 		    __func__, *extra);
 
 	pPSC->RegMaxLPSAwakeIntvl = *extra;
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return 0;
 }
 
@@ -223,13 +223,13 @@ static int _rtl92e_wx_set_force_lps(struct net_device *dev,
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	netdev_info(dev,
 		    "%s(): force LPS ! extra is %d (1 is open 0 is close)\n",
 		    __func__, *extra);
 	priv->force_lps = *extra;
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return 0;
 
 }
@@ -266,7 +266,7 @@ static int _rtl92e_wx_set_mode(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 	rtState = priv->rtllib->eRFPowerState;
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 	if (wrqu->mode == IW_MODE_ADHOC || wrqu->mode == IW_MODE_MONITOR ||
 	    ieee->bNetPromiscuousMode) {
 		if (priv->rtllib->PowerSaveControl.bInactivePs) {
@@ -275,21 +275,21 @@ static int _rtl92e_wx_set_mode(struct net_device *dev,
 				    RF_CHANGE_BY_IPS) {
 					netdev_warn(dev, "%s(): RF is OFF.\n",
 						    __func__);
-					up(&priv->wx_sem);
+					mutex_unlock(&priv->wx_mutex);
 					return -1;
 				}
 				netdev_info(dev,
 					    "=========>%s(): rtl92e_ips_leave\n",
 					    __func__);
-				down(&priv->rtllib->ips_sem);
+				mutex_lock(&priv->rtllib->ips_mutex);
 				rtl92e_ips_leave(dev);
-				up(&priv->rtllib->ips_sem);
+				mutex_unlock(&priv->rtllib->ips_mutex);
 			}
 		}
 	}
 	ret = rtllib_wx_set_mode(priv->rtllib, a, wrqu, b);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return ret;
 }
 
@@ -425,7 +425,7 @@ static int _rtl92e_wx_set_scan(struct net_device *dev,
 		}
 	}
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	priv->rtllib->FirstIe_InScan = true;
 
@@ -436,15 +436,15 @@ static int _rtl92e_wx_set_scan(struct net_device *dev,
 				    RF_CHANGE_BY_IPS) {
 					netdev_warn(dev, "%s(): RF is OFF.\n",
 						    __func__);
-					up(&priv->wx_sem);
+					mutex_unlock(&priv->wx_mutex);
 					return -1;
 				}
 				RT_TRACE(COMP_PS,
 					 "=========>%s(): rtl92e_ips_leave\n",
 					 __func__);
-				down(&priv->rtllib->ips_sem);
+				mutex_lock(&priv->rtllib->ips_mutex);
 				rtl92e_ips_leave(dev);
-				up(&priv->rtllib->ips_sem);
+				mutex_unlock(&priv->rtllib->ips_mutex);
 			}
 		}
 		rtllib_stop_scan(priv->rtllib);
@@ -471,7 +471,7 @@ static int _rtl92e_wx_set_scan(struct net_device *dev,
 		ret = rtllib_wx_set_scan(priv->rtllib, a, wrqu, b);
 	}
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return ret;
 }
 
@@ -491,11 +491,11 @@ static int _rtl92e_wx_get_scan(struct net_device *dev,
 		return 0;
 
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ret = rtllib_wx_get_scan(priv->rtllib, a, wrqu, b);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 }
@@ -513,10 +513,10 @@ static int _rtl92e_wx_set_essid(struct net_device *dev,
 			    __func__);
 		return 0;
 	}
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 	ret = rtllib_wx_set_essid(priv->rtllib, a, wrqu, b);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 }
@@ -528,11 +528,11 @@ static int _rtl92e_wx_get_essid(struct net_device *dev,
 	int ret;
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ret = rtllib_wx_get_essid(priv->rtllib, a, wrqu, b);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 }
@@ -545,12 +545,12 @@ static int _rtl92e_wx_set_nick(struct net_device *dev,
 
 	if (wrqu->data.length > IW_ESSID_MAX_SIZE)
 		return -E2BIG;
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 	wrqu->data.length = min_t(size_t, wrqu->data.length,
 				  sizeof(priv->nick));
 	memset(priv->nick, 0, sizeof(priv->nick));
 	memcpy(priv->nick, extra, wrqu->data.length);
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return 0;
 
 }
@@ -561,11 +561,11 @@ static int _rtl92e_wx_get_nick(struct net_device *dev,
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 	wrqu->data.length = strlen(priv->nick);
 	memcpy(extra, priv->nick, wrqu->data.length);
 	wrqu->data.flags = 1;   /* active */
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return 0;
 }
 
@@ -579,11 +579,11 @@ static int _rtl92e_wx_set_freq(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ret = rtllib_wx_set_freq(priv->rtllib, a, wrqu, b);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return ret;
 }
 
@@ -644,11 +644,11 @@ static int _rtl92e_wx_set_wap(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	ret = rtllib_wx_set_wap(priv->rtllib, info, awrq, extra);
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return ret;
 
@@ -698,14 +698,14 @@ static int _rtl92e_wx_set_enc(struct net_device *dev,
 		return -ENETDOWN;
 
 	priv->rtllib->wx_set_enc = 1;
-	down(&priv->rtllib->ips_sem);
+	mutex_lock(&priv->rtllib->ips_mutex);
 	rtl92e_ips_leave(dev);
-	up(&priv->rtllib->ips_sem);
-	down(&priv->wx_sem);
+	mutex_unlock(&priv->rtllib->ips_mutex);
+	mutex_lock(&priv->wx_mutex);
 
 	RT_TRACE(COMP_SEC, "Setting SW wep key");
 	ret = rtllib_wx_set_encode(priv->rtllib, info, wrqu, key);
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 
 	if (wrqu->encoding.flags & IW_ENCODE_DISABLED) {
@@ -799,7 +799,7 @@ static int _rtl92e_wx_set_retry(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	if (wrqu->retry.flags & IW_RETRY_LIFETIME ||
 	    wrqu->retry.disabled) {
@@ -822,7 +822,7 @@ static int _rtl92e_wx_set_retry(struct net_device *dev,
 
 	rtl92e_commit(dev);
 exit:
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return err;
 }
@@ -875,7 +875,7 @@ static int _rtl92e_wx_set_sens(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 	if (priv->rf_set_sens == NULL) {
 		err = -1; /* we have not this support for this radio */
 		goto exit;
@@ -886,7 +886,7 @@ static int _rtl92e_wx_set_sens(struct net_device *dev,
 		err = -EINVAL;
 
 exit:
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return err;
 }
@@ -902,12 +902,12 @@ static int _rtl92e_wx_set_encode_ext(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	priv->rtllib->wx_set_enc = 1;
-	down(&priv->rtllib->ips_sem);
+	mutex_lock(&priv->rtllib->ips_mutex);
 	rtl92e_ips_leave(dev);
-	up(&priv->rtllib->ips_sem);
+	mutex_unlock(&priv->rtllib->ips_mutex);
 
 	ret = rtllib_wx_set_encode_ext(ieee, info, wrqu, extra);
 	{
@@ -969,7 +969,7 @@ static int _rtl92e_wx_set_encode_ext(struct net_device *dev,
 
 end_hw_sec:
 	priv->rtllib->wx_set_enc = 0;
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return ret;
 
 }
@@ -985,9 +985,9 @@ static int _rtl92e_wx_set_auth(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 	ret = rtllib_wx_set_auth(priv->rtllib, info, &(data->param), extra);
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return ret;
 }
 
@@ -1003,9 +1003,9 @@ static int _rtl92e_wx_set_mlme(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 	ret = rtllib_wx_set_mlme(priv->rtllib, info, wrqu, extra);
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return ret;
 }
 
@@ -1020,9 +1020,9 @@ static int _rtl92e_wx_set_gen_ie(struct net_device *dev,
 	if (priv->bHwRadioOff)
 		return 0;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 	ret = rtllib_wx_set_gen_ie(priv->rtllib, extra, data->data.length);
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 	return ret;
 }
 
@@ -1097,14 +1097,14 @@ static int _rtl92e_wx_get_promisc_mode(struct net_device *dev,
 	struct r8192_priv *priv = rtllib_priv(dev);
 	struct rtllib_device *ieee = priv->rtllib;
 
-	down(&priv->wx_sem);
+	mutex_lock(&priv->wx_mutex);
 
 	snprintf(extra, 45, "PromiscuousMode:%d, FilterSrcSTAFrame:%d",
 		 ieee->IntelPromiscuousModeInfo.bPromiscuousOn,
 		 ieee->IntelPromiscuousModeInfo.bFilterSourceStationFrame);
 	wrqu->data.length = strlen(extra) + 1;
 
-	up(&priv->wx_sem);
+	mutex_unlock(&priv->wx_mutex);
 
 	return 0;
 }
diff --git a/drivers/staging/rtl8192e/rtl819x_Qos.h b/drivers/staging/rtl8192e/rtl819x_Qos.h
index 463122db6d29..61da8f7475bb 100644
--- a/drivers/staging/rtl8192e/rtl819x_Qos.h
+++ b/drivers/staging/rtl8192e/rtl819x_Qos.h
@@ -169,9 +169,6 @@ union qos_tclas {
 	} TYPE2_8021Q;
 };
 
-#define IsACValid(ac)		((ac >= 0 && ac <= 7) ? true : false)
-
-
 union aci_aifsn {
 	u8	charData;
 
diff --git a/drivers/staging/rtl8192e/rtl819x_TSProc.c b/drivers/staging/rtl8192e/rtl819x_TSProc.c
index 2c8a526773ed..a966a8e490ab 100644
--- a/drivers/staging/rtl8192e/rtl819x_TSProc.c
+++ b/drivers/staging/rtl8192e/rtl819x_TSProc.c
@@ -306,6 +306,11 @@ static void MakeTSEntry(struct ts_common_info *pTsCommonInfo, u8 *Addr,
 	pTsCommonInfo->TClasNum = TCLAS_Num;
 }
 
+static bool IsACValid(unsigned int tid)
+{
+	return tid < 7;
+}
+
 bool GetTs(struct rtllib_device *ieee, struct ts_common_info **ppTS,
 	   u8 *Addr, u8 TID, enum tr_select TxRxSelect, bool bAddNewTs)
 {
diff --git a/drivers/staging/rtl8192e/rtllib.h b/drivers/staging/rtl8192e/rtllib.h
index 776e179d5bfd..b895a537d3e4 100644
--- a/drivers/staging/rtl8192e/rtllib.h
+++ b/drivers/staging/rtl8192e/rtllib.h
@@ -30,7 +30,7 @@
 #include <linux/jiffies.h>
 #include <linux/timer.h>
 #include <linux/sched.h>
-#include <linux/semaphore.h>
+#include <linux/mutex.h>
 
 #include <linux/delay.h>
 #include <linux/wireless.h>
@@ -521,7 +521,7 @@ enum wireless_mode {
 };
 
 #ifndef ETH_P_PAE
-#define ETH_P_PAE 0x888E /* Port Access Entity (IEEE 802.1X) */
+#define ETH_P_PAE	0x888E		/* Port Access Entity (IEEE 802.1X) */
 #define ETH_P_IP	0x0800		/* Internet Protocol packet	*/
 #define ETH_P_ARP	0x0806		/* Address Resolution packet	*/
 #endif /* ETH_P_PAE */
@@ -1651,9 +1651,9 @@ struct rtllib_device {
 	short proto_started;
 	short proto_stoppping;
 
-	struct semaphore wx_sem;
-	struct semaphore scan_sem;
-	struct semaphore ips_sem;
+	struct mutex wx_mutex;
+	struct mutex scan_mutex;
+	struct mutex ips_mutex;
 
 	spinlock_t mgmt_tx_lock;
 	spinlock_t beacon_lock;
@@ -2212,7 +2212,5 @@ void rtllib_indicate_packets(struct rtllib_device *ieee,
 void HTUseDefaultSetting(struct rtllib_device *ieee);
 #define RT_ASOC_RETRY_LIMIT	5
 u8 MgntQuery_TxRateExcludeCCKRates(struct rtllib_device *ieee);
-#define SEM_DOWN_IEEE_WX(psem) down(psem)
-#define SEM_UP_IEEE_WX(psem) up(psem)
 
 #endif /* RTLLIB_H */
diff --git a/drivers/staging/rtl8192e/rtllib_module.c b/drivers/staging/rtl8192e/rtllib_module.c
index f4f318abb299..9d5788e04dd5 100644
--- a/drivers/staging/rtl8192e/rtllib_module.c
+++ b/drivers/staging/rtl8192e/rtllib_module.c
@@ -138,7 +138,7 @@ struct net_device *alloc_rtllib(int sizeof_priv)
 	rtllib_softmac_init(ieee);
 
 	ieee->pHTInfo = kzalloc(sizeof(struct rt_hi_throughput), GFP_KERNEL);
-	if (ieee->pHTInfo == NULL)
+	if (!ieee->pHTInfo)
 		return NULL;
 
 	HTUpdateDefaultSetting(ieee);
diff --git a/drivers/staging/rtl8192e/rtllib_softmac.c b/drivers/staging/rtl8192e/rtllib_softmac.c
index 62154e3f4463..da74dc49b95e 100644
--- a/drivers/staging/rtl8192e/rtllib_softmac.c
+++ b/drivers/staging/rtl8192e/rtllib_softmac.c
@@ -276,8 +276,9 @@ inline void softmac_mgmt_xmit(struct sk_buff *skb, struct rtllib_device *ieee)
 	}
 }
 
-inline void softmac_ps_mgmt_xmit(struct sk_buff *skb,
-		struct rtllib_device *ieee)
+static inline void
+softmac_ps_mgmt_xmit(struct sk_buff *skb,
+		     struct rtllib_device *ieee)
 {
 	short single = ieee->softmac_features & IEEE_SOFTMAC_SINGLE_QUEUE;
 	struct rtllib_hdr_3addr  *header =
@@ -513,7 +514,7 @@ static void rtllib_softmac_scan_syncro(struct rtllib_device *ieee, u8 is_mesh)
 
 	ieee->be_scan_inprogress = true;
 
-	down(&ieee->scan_sem);
+	mutex_lock(&ieee->scan_mutex);
 
 	while (1) {
 		do {
@@ -566,7 +567,7 @@ out:
 		if (IS_DOT11D_ENABLE(ieee))
 			DOT11D_ScanComplete(ieee);
 	}
-	up(&ieee->scan_sem);
+	mutex_unlock(&ieee->scan_mutex);
 
 	ieee->be_scan_inprogress = false;
 
@@ -587,7 +588,7 @@ static void rtllib_softmac_scan_wq(void *data)
 	if (rtllib_act_scanning(ieee, true))
 		return;
 
-	down(&ieee->scan_sem);
+	mutex_lock(&ieee->scan_mutex);
 
 	if (ieee->eRFPowerState == eRfOff) {
 		netdev_info(ieee->dev,
@@ -618,7 +619,7 @@ static void rtllib_softmac_scan_wq(void *data)
 	schedule_delayed_work(&ieee->softmac_scan_wq,
 			      msecs_to_jiffies(RTLLIB_SOFTMAC_SCAN_TIME));
 
-	up(&ieee->scan_sem);
+	mutex_unlock(&ieee->scan_mutex);
 	return;
 
 out:
@@ -630,7 +631,7 @@ out1:
 	ieee->actscanning = false;
 	ieee->scan_watch_dog = 0;
 	ieee->scanning_continue = 0;
-	up(&ieee->scan_sem);
+	mutex_unlock(&ieee->scan_mutex);
 }
 
 
@@ -683,7 +684,7 @@ EXPORT_SYMBOL(rtllib_start_send_beacons);
 
 static void rtllib_softmac_stop_scan(struct rtllib_device *ieee)
 {
-	down(&ieee->scan_sem);
+	mutex_lock(&ieee->scan_mutex);
 	ieee->scan_watch_dog = 0;
 	if (ieee->scanning_continue == 1) {
 		ieee->scanning_continue = 0;
@@ -692,7 +693,7 @@ static void rtllib_softmac_stop_scan(struct rtllib_device *ieee)
 		cancel_delayed_work_sync(&ieee->softmac_scan_wq);
 	}
 
-	up(&ieee->scan_sem);
+	mutex_unlock(&ieee->scan_mutex);
 }
 
 void rtllib_stop_scan(struct rtllib_device *ieee)
@@ -753,7 +754,7 @@ static void rtllib_start_scan(struct rtllib_device *ieee)
 	}
 }
 
-/* called with wx_sem held */
+/* called with wx_mutex held */
 void rtllib_start_scan_syncro(struct rtllib_device *ieee, u8 is_mesh)
 {
 	if (IS_DOT11D_ENABLE(ieee)) {
@@ -770,8 +771,10 @@ void rtllib_start_scan_syncro(struct rtllib_device *ieee, u8 is_mesh)
 }
 EXPORT_SYMBOL(rtllib_start_scan_syncro);
 
-inline struct sk_buff *rtllib_authentication_req(struct rtllib_network *beacon,
-	struct rtllib_device *ieee, int challengelen, u8 *daddr)
+static inline struct sk_buff *
+rtllib_authentication_req(struct rtllib_network *beacon,
+			  struct rtllib_device *ieee,
+			  int challengelen, u8 *daddr)
 {
 	struct sk_buff *skb;
 	struct rtllib_authentication *auth;
@@ -1130,7 +1133,7 @@ static void rtllib_resp_to_probe(struct rtllib_device *ieee, u8 *dest)
 }
 
 
-inline int SecIsInPMKIDList(struct rtllib_device *ieee, u8 *bssid)
+static inline int SecIsInPMKIDList(struct rtllib_device *ieee, u8 *bssid)
 {
 	int i = 0;
 
@@ -1146,8 +1149,9 @@ inline int SecIsInPMKIDList(struct rtllib_device *ieee, u8 *bssid)
 	return i;
 }
 
-inline struct sk_buff *rtllib_association_req(struct rtllib_network *beacon,
-					      struct rtllib_device *ieee)
+static inline struct sk_buff *
+rtllib_association_req(struct rtllib_network *beacon,
+		       struct rtllib_device *ieee)
 {
 	struct sk_buff *skb;
 	struct rtllib_assoc_request_frame *hdr;
@@ -1590,7 +1594,7 @@ static void rtllib_associate_procedure_wq(void *data)
 	rtllib_stop_scan_syncro(ieee);
 	if (ieee->rtllib_ips_leave != NULL)
 		ieee->rtllib_ips_leave(ieee->dev);
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	if (ieee->data_hard_stop)
 		ieee->data_hard_stop(ieee->dev);
@@ -1605,14 +1609,14 @@ static void rtllib_associate_procedure_wq(void *data)
 			 __func__);
 		if (ieee->rtllib_ips_leave_wq != NULL)
 			ieee->rtllib_ips_leave_wq(ieee->dev);
-		up(&ieee->wx_sem);
+		mutex_unlock(&ieee->wx_mutex);
 		return;
 	}
 	ieee->associate_seq = 1;
 
 	rtllib_associate_step1(ieee, ieee->current_network.bssid);
 
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 }
 
 inline void rtllib_softmac_new_net(struct rtllib_device *ieee,
@@ -2209,8 +2213,9 @@ static void rtllib_process_action(struct rtllib_device *ieee,
 	}
 }
 
-inline int rtllib_rx_assoc_resp(struct rtllib_device *ieee, struct sk_buff *skb,
-				struct rtllib_rx_stats *rx_stats)
+static inline int
+rtllib_rx_assoc_resp(struct rtllib_device *ieee, struct sk_buff *skb,
+		     struct rtllib_rx_stats *rx_stats)
 {
 	u16 errcode;
 	int aid;
@@ -2344,8 +2349,9 @@ static void rtllib_rx_auth_resp(struct rtllib_device *ieee, struct sk_buff *skb)
 	}
 }
 
-inline int rtllib_rx_auth(struct rtllib_device *ieee, struct sk_buff *skb,
-			  struct rtllib_rx_stats *rx_stats)
+static inline int
+rtllib_rx_auth(struct rtllib_device *ieee, struct sk_buff *skb,
+	       struct rtllib_rx_stats *rx_stats)
 {
 
 	if (ieee->softmac_features & IEEE_SOFTMAC_ASSOCIATE) {
@@ -2361,7 +2367,8 @@ inline int rtllib_rx_auth(struct rtllib_device *ieee, struct sk_buff *skb,
 	return 0;
 }
 
-inline int rtllib_rx_deauth(struct rtllib_device *ieee, struct sk_buff *skb)
+static inline int
+rtllib_rx_deauth(struct rtllib_device *ieee, struct sk_buff *skb)
 {
 	struct rtllib_hdr_3addr *header = (struct rtllib_hdr_3addr *) skb->data;
 	u16 frame_ctl;
@@ -2582,16 +2589,16 @@ static void rtllib_start_ibss_wq(void *data)
 				     struct rtllib_device, start_ibss_wq);
 	/* iwconfig mode ad-hoc will schedule this and return
 	 * on the other hand this will block further iwconfig SET
-	 * operations because of the wx_sem hold.
+	 * operations because of the wx_mutex hold.
 	 * Anyway some most set operations set a flag to speed-up
 	 * (abort) this wq (when syncro scanning) before sleeping
-	 * on the semaphore
+	 * on the mutex
 	 */
 	if (!ieee->proto_started) {
 		netdev_info(ieee->dev, "==========oh driver down return\n");
 		return;
 	}
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	if (ieee->current_network.ssid_len == 0) {
 		strcpy(ieee->current_network.ssid, RTLLIB_DEFAULT_TX_ESSID);
@@ -2703,7 +2710,7 @@ static void rtllib_start_ibss_wq(void *data)
 
 	netif_carrier_on(ieee->dev);
 
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 }
 
 inline void rtllib_start_ibss(struct rtllib_device *ieee)
@@ -2711,7 +2718,7 @@ inline void rtllib_start_ibss(struct rtllib_device *ieee)
 	schedule_delayed_work(&ieee->start_ibss_wq, msecs_to_jiffies(150));
 }
 
-/* this is called only in user context, with wx_sem held */
+/* this is called only in user context, with wx_mutex held */
 static void rtllib_start_bss(struct rtllib_device *ieee)
 {
 	unsigned long flags;
@@ -2773,7 +2780,7 @@ static void rtllib_associate_retry_wq(void *data)
 				     struct rtllib_device, associate_retry_wq);
 	unsigned long flags;
 
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 	if (!ieee->proto_started)
 		goto exit;
 
@@ -2806,7 +2813,7 @@ static void rtllib_associate_retry_wq(void *data)
 
 	ieee->beinretry = false;
 exit:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 }
 
 static struct sk_buff *rtllib_get_beacon_(struct rtllib_device *ieee)
@@ -2853,9 +2860,9 @@ void rtllib_softmac_stop_protocol(struct rtllib_device *ieee, u8 mesh_flag,
 				  u8 shutdown)
 {
 	rtllib_stop_scan_syncro(ieee);
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 	rtllib_stop_protocol(ieee, shutdown);
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 }
 EXPORT_SYMBOL(rtllib_softmac_stop_protocol);
 
@@ -2902,9 +2909,9 @@ void rtllib_stop_protocol(struct rtllib_device *ieee, u8 shutdown)
 
 void rtllib_softmac_start_protocol(struct rtllib_device *ieee, u8 mesh_flag)
 {
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 	rtllib_start_protocol(ieee);
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 }
 EXPORT_SYMBOL(rtllib_softmac_start_protocol);
 
@@ -3034,9 +3041,9 @@ void rtllib_softmac_init(struct rtllib_device *ieee)
 	INIT_WORK_RSL(&ieee->wx_sync_scan_wq, (void *)rtllib_wx_sync_scan_wq,
 		      ieee);
 
-	sema_init(&ieee->wx_sem, 1);
-	sema_init(&ieee->scan_sem, 1);
-	sema_init(&ieee->ips_sem, 1);
+	mutex_init(&ieee->wx_mutex);
+	mutex_init(&ieee->scan_mutex);
+	mutex_init(&ieee->ips_mutex);
 
 	spin_lock_init(&ieee->mgmt_tx_lock);
 	spin_lock_init(&ieee->beacon_lock);
@@ -3049,7 +3056,7 @@ void rtllib_softmac_init(struct rtllib_device *ieee)
 
 void rtllib_softmac_free(struct rtllib_device *ieee)
 {
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 	kfree(ieee->pDot11dInfo);
 	ieee->pDot11dInfo = NULL;
 	del_timer_sync(&ieee->associate_timer);
@@ -3064,7 +3071,7 @@ void rtllib_softmac_free(struct rtllib_device *ieee)
 	cancel_work_sync(&ieee->associate_complete_wq);
 	cancel_work_sync(&ieee->ips_leave_wq);
 	cancel_work_sync(&ieee->wx_sync_scan_wq);
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 	tasklet_kill(&ieee->ps_task);
 }
 
@@ -3397,8 +3404,9 @@ static int rtllib_wpa_set_encryption(struct rtllib_device *ieee,
 	return ret;
 }
 
-inline struct sk_buff *rtllib_disauth_skb(struct rtllib_network *beacon,
-		struct rtllib_device *ieee, u16 asRsn)
+static inline struct sk_buff *
+rtllib_disauth_skb(struct rtllib_network *beacon,
+		   struct rtllib_device *ieee, u16 asRsn)
 {
 	struct sk_buff *skb;
 	struct rtllib_disauth *disauth;
@@ -3423,8 +3431,9 @@ inline struct sk_buff *rtllib_disauth_skb(struct rtllib_network *beacon,
 	return skb;
 }
 
-inline struct sk_buff *rtllib_disassociate_skb(struct rtllib_network *beacon,
-		struct rtllib_device *ieee, u16 asRsn)
+static inline struct sk_buff *
+rtllib_disassociate_skb(struct rtllib_network *beacon,
+			struct rtllib_device *ieee, u16 asRsn)
 {
 	struct sk_buff *skb;
 	struct rtllib_disassoc *disass;
@@ -3499,7 +3508,7 @@ int rtllib_wpa_supplicant_ioctl(struct rtllib_device *ieee, struct iw_point *p,
 	struct ieee_param *param;
 	int ret = 0;
 
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	if (p->length < sizeof(struct ieee_param) || !p->pointer) {
 		ret = -EINVAL;
@@ -3543,7 +3552,7 @@ int rtllib_wpa_supplicant_ioctl(struct rtllib_device *ieee, struct iw_point *p,
 
 	kfree(param);
 out:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 
 	return ret;
 }
diff --git a/drivers/staging/rtl8192e/rtllib_softmac_wx.c b/drivers/staging/rtl8192e/rtllib_softmac_wx.c
index 61ed8b0413e4..5f1412fc410d 100644
--- a/drivers/staging/rtl8192e/rtllib_softmac_wx.c
+++ b/drivers/staging/rtl8192e/rtllib_softmac_wx.c
@@ -35,7 +35,7 @@ int rtllib_wx_set_freq(struct rtllib_device *ieee, struct iw_request_info *a,
 	int ret;
 	struct iw_freq *fwrq = &wrqu->freq;
 
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	if (ieee->iw_mode == IW_MODE_INFRA) {
 		ret = 0;
@@ -81,7 +81,7 @@ int rtllib_wx_set_freq(struct rtllib_device *ieee, struct iw_request_info *a,
 
 	ret = 0;
 out:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 	return ret;
 }
 EXPORT_SYMBOL(rtllib_wx_set_freq);
@@ -146,7 +146,7 @@ int rtllib_wx_set_wap(struct rtllib_device *ieee,
 
 	rtllib_stop_scan_syncro(ieee);
 
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 	/* use ifconfig hw ether */
 	if (ieee->iw_mode == IW_MODE_MASTER) {
 		ret = -1;
@@ -185,7 +185,7 @@ int rtllib_wx_set_wap(struct rtllib_device *ieee,
 	if (ifup)
 		rtllib_start_protocol(ieee);
 out:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 	return ret;
 }
 EXPORT_SYMBOL(rtllib_wx_set_wap);
@@ -287,7 +287,7 @@ int rtllib_wx_set_mode(struct rtllib_device *ieee, struct iw_request_info *a,
 	int set_mode_status = 0;
 
 	rtllib_stop_scan_syncro(ieee);
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 	switch (wrqu->mode) {
 	case IW_MODE_MONITOR:
 	case IW_MODE_ADHOC:
@@ -322,7 +322,7 @@ int rtllib_wx_set_mode(struct rtllib_device *ieee, struct iw_request_info *a,
 	}
 
 out:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 	return set_mode_status;
 }
 EXPORT_SYMBOL(rtllib_wx_set_mode);
@@ -412,7 +412,7 @@ void rtllib_wx_sync_scan_wq(void *data)
 	rtllib_wake_all_queues(ieee);
 
 out:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 
 }
 
@@ -421,7 +421,7 @@ int rtllib_wx_set_scan(struct rtllib_device *ieee, struct iw_request_info *a,
 {
 	int ret = 0;
 
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	if (ieee->iw_mode == IW_MODE_MONITOR || !(ieee->proto_started)) {
 		ret = -1;
@@ -435,7 +435,7 @@ int rtllib_wx_set_scan(struct rtllib_device *ieee, struct iw_request_info *a,
 	}
 
 out:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 	return ret;
 }
 EXPORT_SYMBOL(rtllib_wx_set_scan);
@@ -450,7 +450,7 @@ int rtllib_wx_set_essid(struct rtllib_device *ieee,
 	unsigned long flags;
 
 	rtllib_stop_scan_syncro(ieee);
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	proto_started = ieee->proto_started;
 
@@ -492,7 +492,7 @@ int rtllib_wx_set_essid(struct rtllib_device *ieee,
 	if (proto_started)
 		rtllib_start_protocol(ieee);
 out:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 	return ret;
 }
 EXPORT_SYMBOL(rtllib_wx_set_essid);
@@ -514,7 +514,7 @@ int rtllib_wx_set_rawtx(struct rtllib_device *ieee,
 	int enable = (parms[0] > 0);
 	short prev = ieee->raw_tx;
 
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	if (enable)
 		ieee->raw_tx = 1;
@@ -536,7 +536,7 @@ int rtllib_wx_set_rawtx(struct rtllib_device *ieee,
 			netif_carrier_off(ieee->dev);
 	}
 
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 
 	return 0;
 }
@@ -575,7 +575,7 @@ int rtllib_wx_set_power(struct rtllib_device *ieee,
 		return -1;
 	}
 
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	if (wrqu->power.disabled) {
 		RT_TRACE(COMP_DBG, "===>%s(): power disable\n", __func__);
@@ -611,7 +611,7 @@ int rtllib_wx_set_power(struct rtllib_device *ieee,
 
 	}
 exit:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 	return ret;
 
 }
@@ -622,7 +622,7 @@ int rtllib_wx_get_power(struct rtllib_device *ieee,
 				 struct iw_request_info *info,
 				 union iwreq_data *wrqu, char *extra)
 {
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	if (ieee->ps == RTLLIB_PS_DISABLED) {
 		wrqu->power.disabled = 1;
@@ -648,7 +648,7 @@ int rtllib_wx_get_power(struct rtllib_device *ieee,
 		wrqu->power.flags |= IW_POWER_UNICAST_R;
 
 exit:
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 	return 0;
 
 }
diff --git a/drivers/staging/rtl8192e/rtllib_tx.c b/drivers/staging/rtl8192e/rtllib_tx.c
index 58fc70ec5f2f..78a3ad5b231f 100644
--- a/drivers/staging/rtl8192e/rtllib_tx.c
+++ b/drivers/staging/rtl8192e/rtllib_tx.c
@@ -1,31 +1,31 @@
 /******************************************************************************
-
-  Copyright(c) 2003 - 2004 Intel Corporation. All rights reserved.
-
-  This program is free software; you can redistribute it and/or modify it
-  under the terms of version 2 of the GNU General Public License as
-  published by the Free Software Foundation.
-
-  This program is distributed in the hope that it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-  more details.
-
-  The full GNU General Public License is included in this distribution in the
-  file called LICENSE.
-
-  Contact Information:
-  James P. Ketrenos <ipw2100-admin@linux.intel.com>
-  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
-
-******************************************************************************
-
-  Few modifications for Realtek's Wi-Fi drivers by
-  Andrea Merello <andrea.merello@gmail.com>
-
-  A special thanks goes to Realtek for their support !
-
-******************************************************************************/
+ *
+ * Copyright(c) 2003 - 2004 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * James P. Ketrenos <ipw2100-admin@linux.intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ *****************************************************************************
+ *
+ * Few modifications for Realtek's Wi-Fi drivers by
+ * Andrea Merello <andrea.merello@gmail.com>
+ *
+ * A special thanks goes to Realtek for their support !
+ *
+ *****************************************************************************/
 
 #include <linux/compiler.h>
 #include <linux/errno.h>
@@ -731,17 +731,19 @@ static int rtllib_xmit_inter(struct sk_buff *skb, struct net_device *dev)
 		if (qos_actived) {
 			hdr_len = RTLLIB_3ADDR_LEN + 2;
 
-		/* in case we are a client verify acm is not set for this ac */
-		while (unlikely(ieee->wmm_acm & (0x01 << skb->priority))) {
-			netdev_info(ieee->dev, "skb->priority = %x\n",
-				    skb->priority);
-			if (wme_downgrade_ac(skb))
-				break;
-			netdev_info(ieee->dev, "converted skb->priority = %x\n",
-			       skb->priority);
-		 }
+			/* in case we are a client verify acm is not set for this ac */
+			while (unlikely(ieee->wmm_acm & (0x01 << skb->priority))) {
+				netdev_info(ieee->dev, "skb->priority = %x\n",
+						skb->priority);
+				if (wme_downgrade_ac(skb))
+					break;
+				netdev_info(ieee->dev, "converted skb->priority = %x\n",
+					   skb->priority);
+			}
+
 			qos_ctl |= skb->priority;
 			header.qos_ctl = cpu_to_le16(qos_ctl & RTLLIB_QOS_TID);
+
 		} else {
 			hdr_len = RTLLIB_3ADDR_LEN;
 		}
@@ -981,6 +983,7 @@ static int rtllib_xmit_inter(struct sk_buff *skb, struct net_device *dev)
 	return 1;
 
 }
+
 int rtllib_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	memset(skb->cb, 0, sizeof(skb->cb));
diff --git a/drivers/staging/rtl8192e/rtllib_wx.c b/drivers/staging/rtl8192e/rtllib_wx.c
index 84e6272f28cd..b1500ee9a5cf 100644
--- a/drivers/staging/rtl8192e/rtllib_wx.c
+++ b/drivers/staging/rtl8192e/rtllib_wx.c
@@ -263,7 +263,7 @@ int rtllib_wx_get_scan(struct rtllib_device *ieee,
 	int err = 0;
 
 	netdev_dbg(ieee->dev, "Getting scan\n");
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 	spin_lock_irqsave(&ieee->lock, flags);
 
 	list_for_each_entry(network, &ieee->network_list, list) {
@@ -287,7 +287,7 @@ int rtllib_wx_get_scan(struct rtllib_device *ieee,
 	}
 
 	spin_unlock_irqrestore(&ieee->lock, flags);
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 	wrqu->data.length = ev -  extra;
 	wrqu->data.flags = 0;
 
@@ -689,7 +689,7 @@ int rtllib_wx_set_mlme(struct rtllib_device *ieee,
 	if (ieee->state != RTLLIB_LINKED)
 		return -ENOLINK;
 
-	down(&ieee->wx_sem);
+	mutex_lock(&ieee->wx_mutex);
 
 	switch (mlme->cmd) {
 	case IW_MLME_DEAUTH:
@@ -716,11 +716,11 @@ int rtllib_wx_set_mlme(struct rtllib_device *ieee,
 		ieee->current_network.ssid_len = 0;
 		break;
 	default:
-		up(&ieee->wx_sem);
+		mutex_unlock(&ieee->wx_mutex);
 		return -EOPNOTSUPP;
 	}
 
-	up(&ieee->wx_sem);
+	mutex_unlock(&ieee->wx_mutex);
 
 	return 0;
 }
diff --git a/drivers/staging/rtl8192u/ieee80211/Makefile b/drivers/staging/rtl8192u/ieee80211/Makefile
index b5d0c2eb045b..9e3f432e5355 100644
--- a/drivers/staging/rtl8192u/ieee80211/Makefile
+++ b/drivers/staging/rtl8192u/ieee80211/Makefile
@@ -1,7 +1,6 @@
 NIC_SELECT = RTL8192U
 
-ccflags-y := -I$(TOPDIR)/drivers/net/wireless
-ccflags-y += -O2
+ccflags-y := -O2
 ccflags-y += -DJACKSON_NEW_8187 -DJACKSON_NEW_RX
 
 ieee80211-rsl-objs := ieee80211_rx.o \
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211.h b/drivers/staging/rtl8192u/ieee80211/ieee80211.h
index 09e9499b7f9d..077ea13eb1e7 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211.h
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211.h
@@ -746,7 +746,7 @@ struct ieee80211_rx_stats {
 	bool		  bisrxaggrsubframe;
 	bool		  bPacketBeacon;	//cosa add for rssi
 	bool		  bToSelfBA;		//cosa add for rssi
-	char	  cck_adc_pwdb[4];	//cosa add for rx path selection
+	s8		  cck_adc_pwdb[4];	//cosa add for rx path selection
 	u16		  Seq_Num;
 
 };
@@ -1814,7 +1814,7 @@ struct ieee80211_device {
 	u32 wpax_type_notify; //{added by David, 2006.9.26}
 
 	/* QoS related flag */
-	char init_wmmparam_flag;
+	s8  init_wmmparam_flag;
 	/* set on initialization */
 	u8  qos_support;
 
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
index 051c2be842d0..89cbc077a48d 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
@@ -1027,7 +1027,7 @@ int ieee80211_rx(struct ieee80211_device *ieee, struct sk_buff *skb,
 				ieee,
 				(PTS_COMMON_INFO *) &pRxTS,
 				hdr->addr2,
-				(u8)Frame_QoSTID((u8 *)(skb->data)),
+				Frame_QoSTID((u8 *)(skb->data)),
 				RX_DIR,
 				true))
 		{
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_softmac.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_softmac.c
index 49db1b75cd05..d7d85b3f19c4 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_softmac.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_softmac.c
@@ -284,7 +284,8 @@ inline void softmac_mgmt_xmit(struct sk_buff *skb, struct ieee80211_device *ieee
 	}
 }
 
-inline void softmac_ps_mgmt_xmit(struct sk_buff *skb, struct ieee80211_device *ieee)
+static inline void
+softmac_ps_mgmt_xmit(struct sk_buff *skb, struct ieee80211_device *ieee)
 {
 
 	short single = ieee->softmac_features & IEEE_SOFTMAC_SINGLE_QUEUE;
@@ -320,7 +321,7 @@ inline void softmac_ps_mgmt_xmit(struct sk_buff *skb, struct ieee80211_device *i
 	//dev_kfree_skb_any(skb);//edit by thomas
 }
 
-inline struct sk_buff *ieee80211_probe_req(struct ieee80211_device *ieee)
+static inline struct sk_buff *ieee80211_probe_req(struct ieee80211_device *ieee)
 {
 	unsigned int len, rate_len;
 	u8 *tag;
@@ -640,8 +641,9 @@ void ieee80211_start_scan_syncro(struct ieee80211_device *ieee)
 }
 EXPORT_SYMBOL(ieee80211_start_scan_syncro);
 
-inline struct sk_buff *ieee80211_authentication_req(struct ieee80211_network *beacon,
-	struct ieee80211_device *ieee, int challengelen)
+static inline struct sk_buff *
+ieee80211_authentication_req(struct ieee80211_network *beacon,
+			     struct ieee80211_device *ieee, int challengelen)
 {
 	struct sk_buff *skb;
 	struct ieee80211_authentication *auth;
@@ -806,7 +808,7 @@ static struct sk_buff *ieee80211_probe_resp(struct ieee80211_device *ieee, u8 *d
 		*(tag++) = 2;
 
 		put_unaligned_le16(ieee->current_network.atim_window,
-				   (u8 *)tag);
+				   tag);
 		tag+=2;
 	}
 
@@ -978,7 +980,9 @@ static void ieee80211_resp_to_probe(struct ieee80211_device *ieee, u8 *dest)
 }
 
 
-inline struct sk_buff *ieee80211_association_req(struct ieee80211_network *beacon,struct ieee80211_device *ieee)
+static inline struct sk_buff *
+ieee80211_association_req(struct ieee80211_network *beacon,
+			  struct ieee80211_device *ieee)
 {
 	struct sk_buff *skb;
 	//unsigned long flags;
@@ -3091,7 +3095,7 @@ static int ieee80211_wpa_set_encryption(struct ieee80211_device *ieee,
 	return ret;
 }
 
-inline struct sk_buff *ieee80211_disassociate_skb(
+static inline struct sk_buff *ieee80211_disassociate_skb(
 							struct ieee80211_network *beacon,
 							struct ieee80211_device *ieee,
 							u8	asRsn)
diff --git a/drivers/staging/rtl8192u/ieee80211/rtl819x_BAProc.c b/drivers/staging/rtl8192u/ieee80211/rtl819x_BAProc.c
index 28737ec65186..98fbb6ef484d 100644
--- a/drivers/staging/rtl8192u/ieee80211/rtl819x_BAProc.c
+++ b/drivers/staging/rtl8192u/ieee80211/rtl819x_BAProc.c
@@ -354,7 +354,7 @@ int ieee80211_rx_ADDBAReq(struct ieee80211_device *ieee, struct sk_buff *skb)
 
 	req = (struct rtl_80211_hdr_3addr *) skb->data;
 	tag = (u8 *)req;
-	dst = (u8 *)(&req->addr2[0]);
+	dst = &req->addr2[0];
 	tag += sizeof(struct rtl_80211_hdr_3addr);
 	pDialogToken = tag + 2;  //category+action
 	pBaParamSet = (PBA_PARAM_SET)(tag + 3);   //+DialogToken
@@ -452,7 +452,7 @@ int ieee80211_rx_ADDBARsp(struct ieee80211_device *ieee, struct sk_buff *skb)
 	}
 	rsp = (struct rtl_80211_hdr_3addr *)skb->data;
 	tag = (u8 *)rsp;
-	dst = (u8 *)(&rsp->addr2[0]);
+	dst = &rsp->addr2[0];
 	tag += sizeof(struct rtl_80211_hdr_3addr);
 	pDialogToken = tag + 2;
 	pStatusCode = (u16 *)(tag + 3);
@@ -590,7 +590,7 @@ int ieee80211_rx_DELBA(struct ieee80211_device *ieee, struct sk_buff *skb)
 
 	IEEE80211_DEBUG_DATA(IEEE80211_DL_DATA|IEEE80211_DL_BA, skb->data, skb->len);
 	delba = (struct rtl_80211_hdr_3addr *)skb->data;
-	dst = (u8 *)(&delba->addr2[0]);
+	dst = &delba->addr2[0];
 	pDelBaParamSet = (PDELBA_PARAM_SET)&delba->payload[2];
 
 	if(pDelBaParamSet->field.Initiator == 1)
diff --git a/drivers/staging/rtl8192u/r8192U.h b/drivers/staging/rtl8192u/r8192U.h
index 821afc0ddac5..0b7b04ea0910 100644
--- a/drivers/staging/rtl8192u/r8192U.h
+++ b/drivers/staging/rtl8192u/r8192U.h
@@ -533,7 +533,7 @@ typedef struct _rt_9x_tx_rate_history {
 	u32             ht_mcs[4][16];
 } rt_tx_rahis_t, *prt_tx_rahis_t;
 typedef struct _RT_SMOOTH_DATA_4RF {
-	char    elements[4][100]; /* array to store values */
+	s8    elements[4][100]; /* array to store values */
 	u32     index;            /* index to current array to store */
 	u32     TotalNum;         /* num of valid elements */
 	u32     TotalVal[4];      /* sum of valid elements */
@@ -1031,7 +1031,7 @@ typedef struct r8192_priv {
 	s8 cck_present_attentuation;
 	u8 cck_present_attentuation_20Mdefault;
 	u8 cck_present_attentuation_40Mdefault;
-	char cck_present_attentuation_difference;
+	s8 cck_present_attentuation_difference;
 	bool btxpower_tracking;
 	bool bcck_in_ch14;
 	bool btxpowerdata_readfromEEPORM;
diff --git a/drivers/staging/rtl8192u/r8192U_core.c b/drivers/staging/rtl8192u/r8192U_core.c
index dd0970facdf5..457eeb5f5239 100644
--- a/drivers/staging/rtl8192u/r8192U_core.c
+++ b/drivers/staging/rtl8192u/r8192U_core.c
@@ -114,9 +114,9 @@ static int channels = 0x3fff;
 
 
 
-module_param(ifname, charp, S_IRUGO | S_IWUSR);
-module_param(hwwep, int, S_IRUGO | S_IWUSR);
-module_param(channels, int, S_IRUGO | S_IWUSR);
+module_param(ifname, charp, 0644);
+module_param(hwwep, int, 0644);
+module_param(channels, int, 0644);
 
 MODULE_PARM_DESC(ifname, " Net interface name, wlan%d=default");
 MODULE_PARM_DESC(hwwep, " Try to use hardware security support. ");
@@ -922,47 +922,6 @@ void rtl8192_rtx_disable(struct net_device *dev)
 	skb_queue_purge(&priv->skb_queue);
 }
 
-inline u16 ieeerate2rtlrate(int rate)
-{
-	switch (rate) {
-	case 10:
-		return 0;
-	case 20:
-		return 1;
-	case 55:
-		return 2;
-	case 110:
-		return 3;
-	case 60:
-		return 4;
-	case 90:
-		return 5;
-	case 120:
-		return 6;
-	case 180:
-		return 7;
-	case 240:
-		return 8;
-	case 360:
-		return 9;
-	case 480:
-		return 10;
-	case 540:
-		return 11;
-	default:
-		return 3;
-	}
-}
-
-static u16 rtl_rate[] = {10, 20, 55, 110, 60, 90, 120, 180, 240, 360, 480, 540};
-inline u16 rtl8192_rate2rate(short rate)
-{
-	if (rate > 11)
-		return 0;
-	return rtl_rate[rate];
-}
-
-
 /* The prototype of rx_isr has changed since one version of Linux Kernel */
 static void rtl8192_rx_isr(struct urb *urb)
 {
@@ -1319,14 +1278,6 @@ void rtl819xusb_beacon_tx(struct net_device *dev, u16  tx_rate)
 
 }
 
-inline u8 rtl8192_IsWirelessBMode(u16 rate)
-{
-	if (((rate <= 110) && (rate != 60) && (rate != 90)) || (rate == 220))
-		return 1;
-	else
-		return 0;
-}
-
 short rtl819xU_tx_cmd(struct net_device *dev, struct sk_buff *skb)
 {
 	struct r8192_priv *priv = ieee80211_priv(dev);
@@ -1702,11 +1653,8 @@ short rtl8192_tx(struct net_device *dev, struct sk_buff *skb)
 		}
 		if (bSend0Byte) {
 			tx_urb_zero = usb_alloc_urb(0, GFP_ATOMIC);
-			if (!tx_urb_zero) {
-				RT_TRACE(COMP_ERR,
-					 "can't alloc urb for zero byte\n");
+			if (!tx_urb_zero)
 				return -ENOMEM;
-			}
 			usb_fill_bulk_urb(tx_urb_zero, udev,
 					  usb_sndbulkpipe(udev, idx_pipe),
 					  &zero, 0, tx_zero_isr, dev);
@@ -4209,7 +4157,7 @@ static void rtl8192_process_phyinfo(struct r8192_priv *priv, u8 *buffer,
  *
  * Return:		0-100 percentage
  *---------------------------------------------------------------------------*/
-static u8 rtl819x_query_rxpwrpercentage(char antpower)
+static u8 rtl819x_query_rxpwrpercentage(s8 antpower)
 {
 	if ((antpower <= -100) || (antpower >= 20))
 		return	0;
@@ -4220,9 +4168,9 @@ static u8 rtl819x_query_rxpwrpercentage(char antpower)
 
 }	/* QueryRxPwrPercentage */
 
-static u8 rtl819x_evm_dbtopercentage(char value)
+static u8 rtl819x_evm_dbtopercentage(s8 value)
 {
-	char ret_val;
+	s8 ret_val;
 
 	ret_val = value;
 
@@ -4297,8 +4245,8 @@ static void rtl8192_query_rxphystatus(struct r8192_priv *priv,
 	phy_ofdm_rx_status_rxsc_sgien_exintfflag *prxsc;
 	u8	*prxpkt;
 	u8	i, max_spatial_stream, tmp_rxsnr, tmp_rxevm, rxsc_sgien_exflg;
-	char	rx_pwr[4], rx_pwr_all = 0;
-	char	rx_snrX, rx_evmX;
+	s8	rx_pwr[4], rx_pwr_all = 0;
+	s8	rx_snrX, rx_evmX;
 	u8	evm, pwdb_all;
 	u32	RSSI, total_rssi = 0;
 	u8	is_cck_rate = 0;
@@ -4423,7 +4371,7 @@ static void rtl8192_query_rxphystatus(struct r8192_priv *priv,
 
 			/* Get Rx snr value in DB */
 			tmp_rxsnr =	pofdm_buf->rxsnr_X[i];
-			rx_snrX = (char)(tmp_rxsnr);
+			rx_snrX = (s8)(tmp_rxsnr);
 			rx_snrX /= 2;
 			priv->stats.rxSNRdB[i] = (long)rx_snrX;
 
@@ -4457,7 +4405,7 @@ static void rtl8192_query_rxphystatus(struct r8192_priv *priv,
 
 		for (i = 0; i < max_spatial_stream; i++) {
 			tmp_rxevm =	pofdm_buf->rxevm_X[i];
-			rx_evmX = (char)(tmp_rxevm);
+			rx_evmX = (s8)(tmp_rxevm);
 
 			/* Do not use shift operation like "rx_evmX >>= 1"
 			 * because the compiler of free build environment will
@@ -4475,10 +4423,10 @@ static void rtl8192_query_rxphystatus(struct r8192_priv *priv,
 				 */
 				pstats->SignalQuality =
 					precord_stats->SignalQuality =
-					(u8)(evm & 0xff);
+					evm & 0xff;
 			pstats->RxMIMOSignalQuality[i] =
 				precord_stats->RxMIMOSignalQuality[i] =
-				(u8)(evm & 0xff);
+				evm & 0xff;
 		}
 
 
@@ -5013,8 +4961,7 @@ static int rtl8192_usb_probe(struct usb_interface *intf,
 
 	dev->netdev_ops = &rtl8192_netdev_ops;
 
-	dev->wireless_handlers =
-		(struct iw_handler_def *)&r8192_wx_handlers_def;
+	dev->wireless_handlers = &r8192_wx_handlers_def;
 
 	dev->type = ARPHRD_ETHER;
 
@@ -5222,7 +5169,8 @@ void setKey(struct net_device *dev, u8 EntryNo, u8 KeyIndex, u16 KeyType,
 		} else {
 			/* Key Material */
 			if (KeyContent) {
-				write_nic_dword(dev, WCAMI, (u32)(*(KeyContent + i - 2)));
+				write_nic_dword(dev, WCAMI,
+						*(KeyContent + i - 2));
 				write_nic_dword(dev, RWCAM, TargetCommand);
 			}
 		}
diff --git a/drivers/staging/rtl8192u/r8192U_dm.c b/drivers/staging/rtl8192u/r8192U_dm.c
index 1e0e53c9c314..9209aad0515e 100644
--- a/drivers/staging/rtl8192u/r8192U_dm.c
+++ b/drivers/staging/rtl8192u/r8192U_dm.c
@@ -150,7 +150,7 @@ void deinit_hal_dm(struct net_device *dev)
 #ifdef USB_RX_AGGREGATION_SUPPORT
 void dm_CheckRxAggregation(struct net_device *dev)
 {
-	struct r8192_priv *priv = ieee80211_priv((struct net_device *)dev);
+	struct r8192_priv *priv = ieee80211_priv(dev);
 	PRT_HIGH_THROUGHPUT	pHTInfo = priv->ieee80211->pHTInfo;
 	static unsigned long	lastTxOkCnt;
 	static unsigned long	lastRxOkCnt;
@@ -2346,7 +2346,7 @@ dm_CheckEdcaTurbo_EXIT:
 
 static void dm_init_ctstoself(struct net_device *dev)
 {
-	struct r8192_priv *priv = ieee80211_priv((struct net_device *)dev);
+	struct r8192_priv *priv = ieee80211_priv(dev);
 
 	priv->ieee80211->bCTSToSelfEnable = true;
 	priv->ieee80211->CTSToSelfTH = CTSToSelfTHVal;
@@ -2354,7 +2354,7 @@ static void dm_init_ctstoself(struct net_device *dev)
 
 static void dm_ctstoself(struct net_device *dev)
 {
-	struct r8192_priv *priv = ieee80211_priv((struct net_device *)dev);
+	struct r8192_priv *priv = ieee80211_priv(dev);
 	PRT_HIGH_THROUGHPUT	pHTInfo = priv->ieee80211->pHTInfo;
 	static unsigned long				lastTxOkCnt;
 	static unsigned long				lastRxOkCnt;
diff --git a/drivers/staging/rtl8712/ieee80211.c b/drivers/staging/rtl8712/ieee80211.c
index 8918654b44ed..5dc3b5b9bfff 100644
--- a/drivers/staging/rtl8712/ieee80211.c
+++ b/drivers/staging/rtl8712/ieee80211.c
@@ -145,7 +145,7 @@ static void set_supported_rate(u8 *rates, uint mode)
 	case WIRELESS_11BG:
 		memcpy(rates, WIFI_CCKRATES, IEEE80211_CCK_RATE_LEN);
 		memcpy(rates + IEEE80211_CCK_RATE_LEN, WIFI_OFDMRATES,
-			IEEE80211_NUM_OFDM_RATESLEN);
+		       IEEE80211_NUM_OFDM_RATESLEN);
 		break;
 	}
 }
@@ -188,24 +188,24 @@ int r8712_generate_ie(struct registry_priv *pregistrypriv)
 	ie += 2;
 	/*SSID*/
 	ie = r8712_set_ie(ie, _SSID_IE_, pdev_network->Ssid.SsidLength,
-		    pdev_network->Ssid.Ssid, &sz);
+			  pdev_network->Ssid.Ssid, &sz);
 	/*supported rates*/
 	set_supported_rate(pdev_network->rates, pregistrypriv->wireless_mode);
 	rateLen = r8712_get_rateset_len(pdev_network->rates);
 	if (rateLen > 8) {
 		ie = r8712_set_ie(ie, _SUPPORTEDRATES_IE_, 8,
-			    pdev_network->rates, &sz);
+				  pdev_network->rates, &sz);
 		ie = r8712_set_ie(ie, _EXT_SUPPORTEDRATES_IE_, (rateLen - 8),
-			    (pdev_network->rates + 8), &sz);
+				  (pdev_network->rates + 8), &sz);
 	} else
 		ie = r8712_set_ie(ie, _SUPPORTEDRATES_IE_,
-			    rateLen, pdev_network->rates, &sz);
+				  rateLen, pdev_network->rates, &sz);
 	/*DS parameter set*/
 	ie = r8712_set_ie(ie, _DSSET_IE_, 1,
-		    (u8 *)&(pdev_network->Configuration.DSConfig), &sz);
+			  (u8 *)&(pdev_network->Configuration.DSConfig), &sz);
 	/*IBSS Parameter Set*/
 	ie = r8712_set_ie(ie, _IBSS_PARA_IE_, 2,
-		    (u8 *)&(pdev_network->Configuration.ATIMWindow), &sz);
+			  (u8 *)&(pdev_network->Configuration.ATIMWindow), &sz);
 	return sz;
 }
 
@@ -220,8 +220,7 @@ unsigned char *r8712_get_wpa_ie(unsigned char *pie, int *wpa_ie_len, int limit)
 		pbuf = r8712_get_ie(pbuf, _WPA_IE_ID_, &len, limit);
 		if (pbuf) {
 			/*check if oui matches...*/
-			if (memcmp((pbuf + 2), wpa_oui_type,
-			    sizeof(wpa_oui_type)))
+			if (memcmp((pbuf + 2), wpa_oui_type, sizeof(wpa_oui_type)))
 				goto check_next_ie;
 			/*check version...*/
 			memcpy((u8 *)&val16, (pbuf + 6), sizeof(val16));
@@ -279,7 +278,7 @@ static int r8712_get_wpa2_cipher_suite(u8 *s)
 }
 
 int r8712_parse_wpa_ie(u8 *wpa_ie, int wpa_ie_len, int *group_cipher,
-		 int *pairwise_cipher)
+		       int *pairwise_cipher)
 {
 	int i;
 	int left, count;
@@ -322,7 +321,7 @@ int r8712_parse_wpa_ie(u8 *wpa_ie, int wpa_ie_len, int *group_cipher,
 }
 
 int r8712_parse_wpa2_ie(u8 *rsn_ie, int rsn_ie_len, int *group_cipher,
-		  int *pairwise_cipher)
+			int *pairwise_cipher)
 {
 	int i;
 	int left, count;
@@ -365,7 +364,7 @@ int r8712_parse_wpa2_ie(u8 *rsn_ie, int rsn_ie_len, int *group_cipher,
 }
 
 int r8712_get_sec_ie(u8 *in_ie, uint in_len, u8 *rsn_ie, u16 *rsn_len,
-	       u8 *wpa_ie, u16 *wpa_len)
+		     u8 *wpa_ie, u16 *wpa_len)
 {
 	u8 authmode;
 	u8 wpa_oui[4] = {0x0, 0x50, 0xf2, 0x01};
@@ -383,7 +382,7 @@ int r8712_get_sec_ie(u8 *in_ie, uint in_len, u8 *rsn_ie, u16 *rsn_len,
 		} else {
 			if (authmode == _WPA2_IE_ID_) {
 				memcpy(rsn_ie, &in_ie[cnt],
-					in_ie[cnt + 1] + 2);
+				       in_ie[cnt + 1] + 2);
 				*rsn_len = in_ie[cnt + 1] + 2;
 				cnt += in_ie[cnt + 1] + 2;  /*get next*/
 			} else {
diff --git a/drivers/staging/rtl8712/os_intfs.c b/drivers/staging/rtl8712/os_intfs.c
index 57211f7e68a5..cbe4de05d26b 100644
--- a/drivers/staging/rtl8712/os_intfs.c
+++ b/drivers/staging/rtl8712/os_intfs.c
@@ -243,9 +243,9 @@ static u32 start_drv_threads(struct _adapter *padapter)
 void r8712_stop_drv_threads(struct _adapter *padapter)
 {
 	/*Below is to terminate r8712_cmd_thread & event_thread...*/
-	up(&padapter->cmdpriv.cmd_queue_sema);
+	complete(&padapter->cmdpriv.cmd_queue_comp);
 	if (padapter->cmdThread)
-		_down_sema(&padapter->cmdpriv.terminate_cmdthread_sema);
+		wait_for_completion_interruptible(&padapter->cmdpriv.terminate_cmdthread_comp);
 	padapter->cmdpriv.cmd_seq = 1;
 }
 
@@ -425,7 +425,7 @@ static int netdev_open(struct net_device *pnetdev)
 	else
 		netif_wake_queue(pnetdev);
 
-	 if (video_mode)
+	if (video_mode)
 		enable_video_mode(padapter, cbw40_enable);
 	/* start driver mlme relation timer */
 	start_drv_timers(padapter);
diff --git a/drivers/staging/rtl8712/osdep_intf.h b/drivers/staging/rtl8712/osdep_intf.h
index aa0ec74af511..5d37e1f951cf 100644
--- a/drivers/staging/rtl8712/osdep_intf.h
+++ b/drivers/staging/rtl8712/osdep_intf.h
@@ -36,7 +36,7 @@ struct intf_priv {
 	/* when in USB, IO is through interrupt in/out endpoints */
 	struct usb_device *udev;
 	struct urb *piorw_urb;
-	struct semaphore io_retevt;
+	struct completion io_retevt_comp;
 };
 
 int r871x_ioctl(struct net_device *dev, struct ifreq *rq, int cmd);
diff --git a/drivers/staging/rtl8712/osdep_service.h b/drivers/staging/rtl8712/osdep_service.h
index ad041c96fdb8..c9ea50daffff 100644
--- a/drivers/staging/rtl8712/osdep_service.h
+++ b/drivers/staging/rtl8712/osdep_service.h
@@ -57,13 +57,6 @@ struct	__queue	{
 		spin_lock_init(&((pqueue)->lock));	\
 	} while (0)
 
-static inline u32 _down_sema(struct semaphore *sema)
-{
-	if (down_interruptible(sema))
-		return _FAIL;
-	return _SUCCESS;
-}
-
 static inline u32 end_of_queue_search(struct list_head *head,
 		struct list_head *plist)
 {
diff --git a/drivers/staging/rtl8712/recv_linux.c b/drivers/staging/rtl8712/recv_linux.c
index 735a0eadd98c..576c15d25a0f 100644
--- a/drivers/staging/rtl8712/recv_linux.c
+++ b/drivers/staging/rtl8712/recv_linux.c
@@ -60,7 +60,6 @@ int r8712_os_recvbuf_resource_alloc(struct _adapter *padapter,
 	if (!precvbuf->purb)
 		res = _FAIL;
 	precvbuf->pskb = NULL;
-	precvbuf->reuse = false;
 	precvbuf->pallocated_buf = NULL;
 	precvbuf->pbuf = NULL;
 	precvbuf->pdata = NULL;
diff --git a/drivers/staging/rtl8712/rtl8712_cmd.c b/drivers/staging/rtl8712/rtl8712_cmd.c
index 13c018340ff2..9f61583af150 100644
--- a/drivers/staging/rtl8712/rtl8712_cmd.c
+++ b/drivers/staging/rtl8712/rtl8712_cmd.c
@@ -264,9 +264,9 @@ static struct cmd_obj *cmd_hdl_filter(struct _adapter *padapter,
 		 */
 		if (padapter->pwrctrlpriv.pwr_mode > PS_MODE_ACTIVE) {
 			padapter->pwrctrlpriv.pwr_mode = PS_MODE_ACTIVE;
-			_enter_pwrlock(&(padapter->pwrctrlpriv.lock));
+			mutex_lock(&padapter->pwrctrlpriv.mutex_lock);
 			r8712_set_rpwm(padapter, PS_STATE_S4);
-			up(&(padapter->pwrctrlpriv.lock));
+			mutex_unlock(&padapter->pwrctrlpriv.mutex_lock);
 		}
 		pcmd_r = pcmd;
 		break;
@@ -322,7 +322,7 @@ int r8712_cmd_thread(void *context)
 
 	allow_signal(SIGTERM);
 	while (1) {
-		if ((_down_sema(&(pcmdpriv->cmd_queue_sema))) == _FAIL)
+		if (wait_for_completion_interruptible(&pcmdpriv->cmd_queue_comp))
 			break;
 		if (padapter->bDriverStopped || padapter->bSurpriseRemoved)
 			break;
@@ -395,10 +395,10 @@ _next:
 			}
 			if (pcmd->cmdcode == GEN_CMD_CODE(_SetPwrMode)) {
 				if (padapter->pwrctrlpriv.bSleep) {
-					_enter_pwrlock(&(padapter->
-						       pwrctrlpriv.lock));
+					mutex_lock(&padapter->
+						       pwrctrlpriv.mutex_lock);
 					r8712_set_rpwm(padapter, PS_STATE_S2);
-					up(&padapter->pwrctrlpriv.lock);
+					mutex_unlock(&padapter->pwrctrlpriv.mutex_lock);
 				}
 			}
 			r8712_free_cmd_obj(pcmd);
@@ -420,7 +420,7 @@ _next:
 			break;
 		r8712_free_cmd_obj(pcmd);
 	} while (1);
-	up(&pcmdpriv->terminate_cmdthread_sema);
+	complete(&pcmdpriv->terminate_cmdthread_comp);
 	thread_exit();
 }
 
diff --git a/drivers/staging/rtl8712/rtl8712_efuse.c b/drivers/staging/rtl8712/rtl8712_efuse.c
index 76f60ba5ee9b..205298e23656 100644
--- a/drivers/staging/rtl8712/rtl8712_efuse.c
+++ b/drivers/staging/rtl8712/rtl8712_efuse.c
@@ -248,7 +248,7 @@ u8 r8712_efuse_pg_packet_read(struct _adapter *padapter, u8 offset, u8 *data)
 	u8 tmpdata[PGPKT_DATA_SIZE];
 	u8 ret = true;
 
-	if (data == NULL)
+	if (!data)
 		return false;
 	if (offset > 0x0f)
 		return false;
diff --git a/drivers/staging/rtl8712/rtl8712_led.c b/drivers/staging/rtl8712/rtl8712_led.c
index 9055827cccf8..a8e237e480c9 100644
--- a/drivers/staging/rtl8712/rtl8712_led.c
+++ b/drivers/staging/rtl8712/rtl8712_led.c
@@ -43,7 +43,7 @@
 #define LED_BLINK_LINK_INTERVAL_ALPHA		500
 #define LED_BLINK_SCAN_INTERVAL_ALPHA		180
 #define LED_BLINK_FASTER_INTERVAL_ALPHA		50
-#define LED_BLINK_WPS_SUCESS_INTERVAL_ALPHA	5000
+#define LED_BLINK_WPS_SUCCESS_INTERVAL_ALPHA	5000
 
 /*===========================================================================
  * LED object.
@@ -51,17 +51,19 @@
  */
 enum _LED_STATE_871x {
 	LED_UNKNOWN = 0,
-	LED_ON = 1,
-	LED_OFF = 2,
+	LED_STATE_ON = 1,
+	LED_STATE_OFF = 2,
 	LED_BLINK_NORMAL = 3,
 	LED_BLINK_SLOWLY = 4,
 	LED_POWER_ON_BLINK = 5,
 	LED_SCAN_BLINK = 6, /* LED is blinking during scanning period,
 			     * the # of times to blink is depend on time
-			     * for scanning. */
+			     * for scanning.
+			     */
 	LED_NO_LINK_BLINK = 7, /* LED is blinking during no link state. */
 	LED_BLINK_StartToBlink = 8,/* Customized for Sercomm Printer
-				    * Server case */
+				    * Server case
+				    */
 	LED_BLINK_WPS = 9,	/* LED is blinkg during WPS communication */
 	LED_TXRX_BLINK = 10,
 	LED_BLINK_WPS_STOP = 11,	/*for ALPHA */
@@ -92,7 +94,7 @@ static void InitLed871x(struct _adapter *padapter, struct LED_871x *pLed,
 	nic = padapter->pnetdev;
 	pLed->padapter = padapter;
 	pLed->LedPin = LedPin;
-	pLed->CurrLedState = LED_OFF;
+	pLed->CurrLedState = LED_STATE_OFF;
 	pLed->bLedOn = false;
 	pLed->bLedBlinkInProgress = false;
 	pLed->BlinkTimes = 0;
@@ -110,7 +112,8 @@ static void DeInitLed871x(struct LED_871x *pLed)
 {
 	del_timer_sync(&pLed->BlinkTimer);
 	/* We should reset bLedBlinkInProgress if we cancel
-	 * the LedControlTimer, */
+	 * the LedControlTimer,
+	 */
 	pLed->bLedBlinkInProgress = false;
 }
 
@@ -208,7 +211,7 @@ static void SwLedBlink(struct LED_871x *pLed)
 	u8 bStopBlinking = false;
 
 	/* Change LED according to BlinkingLedState specified. */
-	if (pLed->BlinkingLedState == LED_ON)
+	if (pLed->BlinkingLedState == LED_STATE_ON)
 		SwLedOn(padapter, pLed);
 	else
 		SwLedOff(padapter, pLed);
@@ -248,10 +251,10 @@ static void SwLedBlink(struct LED_871x *pLed)
 		pLed->bLedBlinkInProgress = false;
 	} else {
 		/* Assign LED state to toggle. */
-		if (pLed->BlinkingLedState == LED_ON)
-			pLed->BlinkingLedState = LED_OFF;
+		if (pLed->BlinkingLedState == LED_STATE_ON)
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 
 		/* Schedule a timer to toggle LED state. */
 		switch (pLed->CurrLedState) {
@@ -288,7 +291,7 @@ static void SwLedBlink1(struct LED_871x *pLed)
 	if (peeprompriv->CustomerID == RT_CID_819x_CAMEO)
 		pLed = &(ledpriv->SwLed1);
 	/* Change LED according to BlinkingLedState specified. */
-	if (pLed->BlinkingLedState == LED_ON)
+	if (pLed->BlinkingLedState == LED_STATE_ON)
 		SwLedOn(padapter, pLed);
 	else
 		SwLedOff(padapter, pLed);
@@ -312,17 +315,17 @@ static void SwLedBlink1(struct LED_871x *pLed)
 	switch (pLed->CurrLedState) {
 	case LED_BLINK_SLOWLY:
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 		break;
 	case LED_BLINK_NORMAL:
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_LINK_INTERVAL_ALPHA));
 		break;
@@ -335,27 +338,27 @@ static void SwLedBlink1(struct LED_871x *pLed)
 				pLed->bLedLinkBlinkInProgress = true;
 				pLed->CurrLedState = LED_BLINK_NORMAL;
 				if (pLed->bLedOn)
-					pLed->BlinkingLedState = LED_OFF;
+					pLed->BlinkingLedState = LED_STATE_OFF;
 				else
-					pLed->BlinkingLedState = LED_ON;
+					pLed->BlinkingLedState = LED_STATE_ON;
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_LINK_INTERVAL_ALPHA));
 			} else if (!check_fwstate(pmlmepriv, _FW_LINKED)) {
 				pLed->bLedNoLinkBlinkInProgress = true;
 				pLed->CurrLedState = LED_BLINK_SLOWLY;
 				if (pLed->bLedOn)
-					pLed->BlinkingLedState = LED_OFF;
+					pLed->BlinkingLedState = LED_STATE_OFF;
 				else
-					pLed->BlinkingLedState = LED_ON;
+					pLed->BlinkingLedState = LED_STATE_ON;
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 			}
 			pLed->bLedScanBlinkInProgress = false;
 		} else {
 			 if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -369,18 +372,18 @@ static void SwLedBlink1(struct LED_871x *pLed)
 				pLed->bLedLinkBlinkInProgress = true;
 				pLed->CurrLedState = LED_BLINK_NORMAL;
 				if (pLed->bLedOn)
-					pLed->BlinkingLedState = LED_OFF;
+					pLed->BlinkingLedState = LED_STATE_OFF;
 				else
-					pLed->BlinkingLedState = LED_ON;
+					pLed->BlinkingLedState = LED_STATE_ON;
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_LINK_INTERVAL_ALPHA));
 			} else if (!check_fwstate(pmlmepriv, _FW_LINKED)) {
 				pLed->bLedNoLinkBlinkInProgress = true;
 				pLed->CurrLedState = LED_BLINK_SLOWLY;
 				if (pLed->bLedOn)
-					pLed->BlinkingLedState = LED_OFF;
+					pLed->BlinkingLedState = LED_STATE_OFF;
 				else
-					pLed->BlinkingLedState = LED_ON;
+					pLed->BlinkingLedState = LED_STATE_ON;
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 			}
@@ -388,26 +391,26 @@ static void SwLedBlink1(struct LED_871x *pLed)
 			pLed->bLedBlinkInProgress = false;
 		} else {
 			 if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
 		break;
 	case LED_BLINK_WPS:
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		break;
 	case LED_BLINK_WPS_STOP:	/* WPS success */
-		if (pLed->BlinkingLedState == LED_ON) {
-			pLed->BlinkingLedState = LED_OFF;
+		if (pLed->BlinkingLedState == LED_STATE_ON) {
+			pLed->BlinkingLedState = LED_STATE_OFF;
 			mod_timer(&pLed->BlinkTimer, jiffies +
-				  msecs_to_jiffies(LED_BLINK_WPS_SUCESS_INTERVAL_ALPHA));
+				  msecs_to_jiffies(LED_BLINK_WPS_SUCCESS_INTERVAL_ALPHA));
 			bStopBlinking = false;
 		} else {
 			bStopBlinking = true;
@@ -416,9 +419,9 @@ static void SwLedBlink1(struct LED_871x *pLed)
 			pLed->bLedLinkBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_NORMAL;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_LINK_INTERVAL_ALPHA));
 		}
@@ -436,7 +439,7 @@ static void SwLedBlink2(struct LED_871x *pLed)
 	u8 bStopBlinking = false;
 
 	/* Change LED according to BlinkingLedState specified. */
-	if (pLed->BlinkingLedState == LED_ON)
+	if (pLed->BlinkingLedState == LED_STATE_ON)
 		SwLedOn(padapter, pLed);
 	else
 		SwLedOff(padapter, pLed);
@@ -447,20 +450,20 @@ static void SwLedBlink2(struct LED_871x *pLed)
 			bStopBlinking = true;
 		if (bStopBlinking) {
 			if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-				pLed->CurrLedState = LED_ON;
-				pLed->BlinkingLedState = LED_ON;
+				pLed->CurrLedState = LED_STATE_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 				SwLedOn(padapter, pLed);
 			} else if (!check_fwstate(pmlmepriv, _FW_LINKED)) {
-				pLed->CurrLedState = LED_OFF;
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->CurrLedState = LED_STATE_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 				SwLedOff(padapter, pLed);
 			}
 			pLed->bLedScanBlinkInProgress = false;
 		} else {
 			 if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -471,20 +474,20 @@ static void SwLedBlink2(struct LED_871x *pLed)
 			bStopBlinking = true;
 		if (bStopBlinking) {
 			if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-				pLed->CurrLedState = LED_ON;
-				pLed->BlinkingLedState = LED_ON;
+				pLed->CurrLedState = LED_STATE_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 				SwLedOn(padapter, pLed);
 			} else if (!check_fwstate(pmlmepriv, _FW_LINKED)) {
-				pLed->CurrLedState = LED_OFF;
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->CurrLedState = LED_STATE_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 				SwLedOff(padapter, pLed);
 			}
 			pLed->bLedBlinkInProgress = false;
 		} else {
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
@@ -501,7 +504,7 @@ static void SwLedBlink3(struct LED_871x *pLed)
 	u8 bStopBlinking = false;
 
 	/* Change LED according to BlinkingLedState specified. */
-	if (pLed->BlinkingLedState == LED_ON)
+	if (pLed->BlinkingLedState == LED_STATE_ON)
 		SwLedOn(padapter, pLed);
 	else
 		if (pLed->CurrLedState != LED_BLINK_WPS_STOP)
@@ -513,22 +516,22 @@ static void SwLedBlink3(struct LED_871x *pLed)
 			bStopBlinking = true;
 		if (bStopBlinking) {
 			if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-				pLed->CurrLedState = LED_ON;
-				pLed->BlinkingLedState = LED_ON;
+				pLed->CurrLedState = LED_STATE_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 				if (!pLed->bLedOn)
 					SwLedOn(padapter, pLed);
 			} else if (!check_fwstate(pmlmepriv, _FW_LINKED)) {
-				pLed->CurrLedState = LED_OFF;
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->CurrLedState = LED_STATE_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 				if (pLed->bLedOn)
 					SwLedOff(padapter, pLed);
 			}
 			pLed->bLedScanBlinkInProgress = false;
 		} else {
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -539,46 +542,46 @@ static void SwLedBlink3(struct LED_871x *pLed)
 			bStopBlinking = true;
 		if (bStopBlinking) {
 			if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-				pLed->CurrLedState = LED_ON;
-				pLed->BlinkingLedState = LED_ON;
+				pLed->CurrLedState = LED_STATE_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 				if (!pLed->bLedOn)
 					SwLedOn(padapter, pLed);
 			} else if (!check_fwstate(pmlmepriv, _FW_LINKED)) {
-				pLed->CurrLedState = LED_OFF;
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->CurrLedState = LED_STATE_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 				if (pLed->bLedOn)
 					SwLedOff(padapter, pLed);
 			}
 			pLed->bLedBlinkInProgress = false;
 		} else {
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
 		break;
 	case LED_BLINK_WPS:
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		break;
 	case LED_BLINK_WPS_STOP:	/*WPS success*/
-		if (pLed->BlinkingLedState == LED_ON) {
-			pLed->BlinkingLedState = LED_OFF;
+		if (pLed->BlinkingLedState == LED_STATE_ON) {
+			pLed->BlinkingLedState = LED_STATE_OFF;
 			mod_timer(&pLed->BlinkTimer, jiffies +
-				  msecs_to_jiffies(LED_BLINK_WPS_SUCESS_INTERVAL_ALPHA));
+				  msecs_to_jiffies(LED_BLINK_WPS_SUCCESS_INTERVAL_ALPHA));
 			bStopBlinking = false;
 		} else {
 			bStopBlinking = true;
 		}
 		if (bStopBlinking) {
-			pLed->CurrLedState = LED_ON;
-			pLed->BlinkingLedState = LED_ON;
+			pLed->CurrLedState = LED_STATE_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			SwLedOn(padapter, pLed);
 			pLed->bLedWPSBlinkInProgress = false;
 		}
@@ -596,32 +599,32 @@ static void SwLedBlink4(struct LED_871x *pLed)
 	u8 bStopBlinking = false;
 
 	/* Change LED according to BlinkingLedState specified. */
-	if (pLed->BlinkingLedState == LED_ON)
+	if (pLed->BlinkingLedState == LED_STATE_ON)
 		SwLedOn(padapter, pLed);
 	else
 		SwLedOff(padapter, pLed);
 	if (!pLed1->bLedWPSBlinkInProgress &&
 	    pLed1->BlinkingLedState == LED_UNKNOWN) {
-		pLed1->BlinkingLedState = LED_OFF;
-		pLed1->CurrLedState = LED_OFF;
+		pLed1->BlinkingLedState = LED_STATE_OFF;
+		pLed1->CurrLedState = LED_STATE_OFF;
 		SwLedOff(padapter, pLed1);
 	}
 	switch (pLed->CurrLedState) {
 	case LED_BLINK_SLOWLY:
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 		break;
 	case LED_BLINK_StartToBlink:
 		if (pLed->bLedOn) {
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SLOWLY_INTERVAL));
 		} else {
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_NORMAL_INTERVAL));
 		}
@@ -634,17 +637,17 @@ static void SwLedBlink4(struct LED_871x *pLed)
 			pLed->bLedNoLinkBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_SLOWLY;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 			pLed->bLedScanBlinkInProgress = false;
 		} else {
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -657,37 +660,37 @@ static void SwLedBlink4(struct LED_871x *pLed)
 			pLed->bLedNoLinkBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_SLOWLY;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 			pLed->bLedBlinkInProgress = false;
 		} else {
 			 if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
 		break;
 	case LED_BLINK_WPS:
 		if (pLed->bLedOn) {
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SLOWLY_INTERVAL));
 		} else {
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_NORMAL_INTERVAL));
 		}
 		break;
 	case LED_BLINK_WPS_STOP:	/*WPS authentication fail*/
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_NORMAL_INTERVAL));
 		break;
@@ -701,14 +704,14 @@ static void SwLedBlink4(struct LED_871x *pLed)
 		}
 		if (bStopBlinking) {
 			pLed->BlinkTimes = 10;
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_LINK_INTERVAL_ALPHA));
 		} else {
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_NORMAL_INTERVAL));
 		}
@@ -724,7 +727,7 @@ static void SwLedBlink5(struct LED_871x *pLed)
 	u8 bStopBlinking = false;
 
 	/* Change LED according to BlinkingLedState specified. */
-	if (pLed->BlinkingLedState == LED_ON)
+	if (pLed->BlinkingLedState == LED_STATE_ON)
 		SwLedOn(padapter, pLed);
 	else
 		SwLedOff(padapter, pLed);
@@ -734,17 +737,17 @@ static void SwLedBlink5(struct LED_871x *pLed)
 		if (pLed->BlinkTimes == 0)
 			bStopBlinking = true;
 		if (bStopBlinking) {
-			pLed->CurrLedState = LED_ON;
-			pLed->BlinkingLedState = LED_ON;
+			pLed->CurrLedState = LED_STATE_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			if (!pLed->bLedOn)
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 			pLed->bLedScanBlinkInProgress = false;
 		} else {
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -754,17 +757,17 @@ static void SwLedBlink5(struct LED_871x *pLed)
 		if (pLed->BlinkTimes == 0)
 			bStopBlinking = true;
 		if (bStopBlinking) {
-			pLed->CurrLedState = LED_ON;
-			pLed->BlinkingLedState = LED_ON;
+			pLed->CurrLedState = LED_STATE_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			if (!pLed->bLedOn)
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 			pLed->bLedBlinkInProgress = false;
 		} else {
 			 if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
@@ -780,7 +783,7 @@ static void SwLedBlink6(struct LED_871x *pLed)
 	u8 bStopBlinking = false;
 
 	/* Change LED according to BlinkingLedState specified. */
-	if (pLed->BlinkingLedState == LED_ON)
+	if (pLed->BlinkingLedState == LED_STATE_ON)
 		SwLedOn(padapter, pLed);
 	else
 		SwLedOff(padapter, pLed);
@@ -790,25 +793,25 @@ static void SwLedBlink6(struct LED_871x *pLed)
 		if (pLed->BlinkTimes == 0)
 			bStopBlinking = true;
 		if (bStopBlinking) {
-			pLed->CurrLedState = LED_ON;
-			pLed->BlinkingLedState = LED_ON;
+			pLed->CurrLedState = LED_STATE_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			if (!pLed->bLedOn)
 				SwLedOn(padapter, pLed);
 			pLed->bLedBlinkInProgress = false;
 		} else {
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
 		break;
 	case LED_BLINK_WPS:
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		break;
@@ -827,7 +830,8 @@ static void BlinkTimerCallback(unsigned long data)
 	struct LED_871x  *pLed = (struct LED_871x *)data;
 
 	/* This fixed the crash problem on Fedora 12 when trying to do the
-	 * insmod;ifconfig up;rmmod commands. */
+	 * insmod;ifconfig up;rmmod commands.
+	 */
 	if (pLed->padapter->bSurpriseRemoved || pLed->padapter->bDriverStopped)
 		return;
 	schedule_work(&pLed->BlinkWorkItem);
@@ -908,9 +912,9 @@ static void SwLedControlMode1(struct _adapter *padapter,
 			pLed->bLedNoLinkBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_SLOWLY;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 		}
@@ -931,9 +935,9 @@ static void SwLedControlMode1(struct _adapter *padapter,
 			pLed->bLedLinkBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_NORMAL;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_LINK_INTERVAL_ALPHA));
 		}
@@ -961,9 +965,9 @@ static void SwLedControlMode1(struct _adapter *padapter,
 			pLed->CurrLedState = LED_SCAN_BLINK;
 			pLed->BlinkTimes = 24;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		 }
@@ -986,9 +990,9 @@ static void SwLedControlMode1(struct _adapter *padapter,
 			pLed->CurrLedState = LED_TXRX_BLINK;
 			pLed->BlinkTimes = 2;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
@@ -1016,9 +1020,9 @@ static void SwLedControlMode1(struct _adapter *padapter,
 			pLed->bLedWPSBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_WPS;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -1046,11 +1050,11 @@ static void SwLedControlMode1(struct _adapter *padapter,
 			pLed->bLedWPSBlinkInProgress = true;
 		pLed->CurrLedState = LED_BLINK_WPS_STOP;
 		if (pLed->bLedOn) {
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 			mod_timer(&pLed->BlinkTimer, jiffies +
-				  msecs_to_jiffies(LED_BLINK_WPS_SUCESS_INTERVAL_ALPHA));
+				  msecs_to_jiffies(LED_BLINK_WPS_SUCCESS_INTERVAL_ALPHA));
 		} else {
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer,
 				  jiffies + msecs_to_jiffies(0));
 		}
@@ -1063,15 +1067,15 @@ static void SwLedControlMode1(struct _adapter *padapter,
 		pLed->bLedNoLinkBlinkInProgress = true;
 		pLed->CurrLedState = LED_BLINK_SLOWLY;
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 		break;
 	case LED_CTL_POWER_OFF:
-		pLed->CurrLedState = LED_OFF;
-		pLed->BlinkingLedState = LED_OFF;
+		pLed->CurrLedState = LED_STATE_OFF;
+		pLed->BlinkingLedState = LED_STATE_OFF;
 		if (pLed->bLedNoLinkBlinkInProgress) {
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedNoLinkBlinkInProgress = false;
@@ -1123,9 +1127,9 @@ static void SwLedControlMode2(struct _adapter *padapter,
 			pLed->CurrLedState = LED_SCAN_BLINK;
 			pLed->BlinkTimes = 24;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		 }
@@ -1142,17 +1146,17 @@ static void SwLedControlMode2(struct _adapter *padapter,
 			pLed->CurrLedState = LED_TXRX_BLINK;
 			pLed->BlinkTimes = 2;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
 		break;
 
 	case LED_CTL_LINK:
-		pLed->CurrLedState = LED_ON;
-		pLed->BlinkingLedState = LED_ON;
+		pLed->CurrLedState = LED_STATE_ON;
+		pLed->BlinkingLedState = LED_STATE_ON;
 		if (pLed->bLedBlinkInProgress) {
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedBlinkInProgress = false;
@@ -1178,8 +1182,8 @@ static void SwLedControlMode2(struct _adapter *padapter,
 				pLed->bLedScanBlinkInProgress = false;
 			}
 			pLed->bLedWPSBlinkInProgress = true;
-			pLed->CurrLedState = LED_ON;
-			pLed->BlinkingLedState = LED_ON;
+			pLed->CurrLedState = LED_STATE_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer,
 				  jiffies + msecs_to_jiffies(0));
 		 }
@@ -1187,16 +1191,16 @@ static void SwLedControlMode2(struct _adapter *padapter,
 
 	case LED_CTL_STOP_WPS:
 		pLed->bLedWPSBlinkInProgress = false;
-		pLed->CurrLedState = LED_ON;
-		pLed->BlinkingLedState = LED_ON;
+		pLed->CurrLedState = LED_STATE_ON;
+		pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer,
 			  jiffies + msecs_to_jiffies(0));
 		break;
 
 	case LED_CTL_STOP_WPS_FAIL:
 		pLed->bLedWPSBlinkInProgress = false;
-		pLed->CurrLedState = LED_OFF;
-		pLed->BlinkingLedState = LED_OFF;
+		pLed->CurrLedState = LED_STATE_OFF;
+		pLed->BlinkingLedState = LED_STATE_OFF;
 		mod_timer(&pLed->BlinkTimer,
 			  jiffies + msecs_to_jiffies(0));
 		break;
@@ -1204,15 +1208,15 @@ static void SwLedControlMode2(struct _adapter *padapter,
 	case LED_CTL_START_TO_LINK:
 	case LED_CTL_NO_LINK:
 		if (!IS_LED_BLINKING(pLed)) {
-			pLed->CurrLedState = LED_OFF;
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->CurrLedState = LED_STATE_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 			mod_timer(&pLed->BlinkTimer,
 				  jiffies + msecs_to_jiffies(0));
 		}
 		break;
 	case LED_CTL_POWER_OFF:
-		pLed->CurrLedState = LED_OFF;
-		pLed->BlinkingLedState = LED_OFF;
+		pLed->CurrLedState = LED_STATE_OFF;
+		pLed->BlinkingLedState = LED_STATE_OFF;
 		if (pLed->bLedBlinkInProgress) {
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedBlinkInProgress = false;
@@ -1255,9 +1259,9 @@ static void SwLedControlMode3(struct _adapter *padapter,
 			pLed->CurrLedState = LED_SCAN_BLINK;
 			pLed->BlinkTimes = 24;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -1273,9 +1277,9 @@ static void SwLedControlMode3(struct _adapter *padapter,
 			pLed->CurrLedState = LED_TXRX_BLINK;
 			pLed->BlinkTimes = 2;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
@@ -1283,8 +1287,8 @@ static void SwLedControlMode3(struct _adapter *padapter,
 	case LED_CTL_LINK:
 		if (IS_LED_WPS_BLINKING(pLed))
 			return;
-		pLed->CurrLedState = LED_ON;
-		pLed->BlinkingLedState = LED_ON;
+		pLed->CurrLedState = LED_STATE_ON;
+		pLed->BlinkingLedState = LED_STATE_ON;
 		if (pLed->bLedBlinkInProgress) {
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedBlinkInProgress = false;
@@ -1310,9 +1314,9 @@ static void SwLedControlMode3(struct _adapter *padapter,
 			pLed->bLedWPSBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_WPS;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -1326,11 +1330,11 @@ static void SwLedControlMode3(struct _adapter *padapter,
 		}
 		pLed->CurrLedState = LED_BLINK_WPS_STOP;
 		if (pLed->bLedOn) {
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 			mod_timer(&pLed->BlinkTimer, jiffies +
-				  msecs_to_jiffies(LED_BLINK_WPS_SUCESS_INTERVAL_ALPHA));
+				  msecs_to_jiffies(LED_BLINK_WPS_SUCCESS_INTERVAL_ALPHA));
 		} else {
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer,
 				  jiffies + msecs_to_jiffies(0));
 		}
@@ -1340,23 +1344,23 @@ static void SwLedControlMode3(struct _adapter *padapter,
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedWPSBlinkInProgress = false;
 		}
-		pLed->CurrLedState = LED_OFF;
-		pLed->BlinkingLedState = LED_OFF;
+		pLed->CurrLedState = LED_STATE_OFF;
+		pLed->BlinkingLedState = LED_STATE_OFF;
 		mod_timer(&pLed->BlinkTimer,
 			  jiffies + msecs_to_jiffies(0));
 		break;
 	case LED_CTL_START_TO_LINK:
 	case LED_CTL_NO_LINK:
 		if (!IS_LED_BLINKING(pLed)) {
-			pLed->CurrLedState = LED_OFF;
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->CurrLedState = LED_STATE_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 			mod_timer(&pLed->BlinkTimer,
 				  jiffies + msecs_to_jiffies(0));
 		}
 		break;
 	case LED_CTL_POWER_OFF:
-		pLed->CurrLedState = LED_OFF;
-		pLed->BlinkingLedState = LED_OFF;
+		pLed->CurrLedState = LED_STATE_OFF;
+		pLed->BlinkingLedState = LED_STATE_OFF;
 		if (pLed->bLedBlinkInProgress) {
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedBlinkInProgress = false;
@@ -1390,8 +1394,8 @@ static void SwLedControlMode4(struct _adapter *padapter,
 		if (pLed1->bLedWPSBlinkInProgress) {
 			pLed1->bLedWPSBlinkInProgress = false;
 			del_timer(&pLed1->BlinkTimer);
-			pLed1->BlinkingLedState = LED_OFF;
-			pLed1->CurrLedState = LED_OFF;
+			pLed1->BlinkingLedState = LED_STATE_OFF;
+			pLed1->CurrLedState = LED_STATE_OFF;
 			if (pLed1->bLedOn)
 				mod_timer(&pLed->BlinkTimer,
 					  jiffies + msecs_to_jiffies(0));
@@ -1411,11 +1415,11 @@ static void SwLedControlMode4(struct _adapter *padapter,
 			pLed->bLedStartToLinkBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_StartToBlink;
 			if (pLed->bLedOn) {
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_SLOWLY_INTERVAL));
 			} else {
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_NORMAL_INTERVAL));
 			}
@@ -1428,8 +1432,8 @@ static void SwLedControlMode4(struct _adapter *padapter,
 			if (pLed1->bLedWPSBlinkInProgress) {
 				pLed1->bLedWPSBlinkInProgress = false;
 				del_timer(&pLed1->BlinkTimer);
-				pLed1->BlinkingLedState = LED_OFF;
-				pLed1->CurrLedState = LED_OFF;
+				pLed1->BlinkingLedState = LED_STATE_OFF;
+				pLed1->CurrLedState = LED_STATE_OFF;
 				if (pLed1->bLedOn)
 					mod_timer(&pLed->BlinkTimer,
 						  jiffies + msecs_to_jiffies(0));
@@ -1446,9 +1450,9 @@ static void SwLedControlMode4(struct _adapter *padapter,
 			pLed->bLedNoLinkBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_SLOWLY;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 		}
@@ -1472,9 +1476,9 @@ static void SwLedControlMode4(struct _adapter *padapter,
 			pLed->CurrLedState = LED_SCAN_BLINK;
 			pLed->BlinkTimes = 24;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -1493,9 +1497,9 @@ static void SwLedControlMode4(struct _adapter *padapter,
 			pLed->CurrLedState = LED_TXRX_BLINK;
 			pLed->BlinkTimes = 2;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
@@ -1505,8 +1509,8 @@ static void SwLedControlMode4(struct _adapter *padapter,
 		if (pLed1->bLedWPSBlinkInProgress) {
 			pLed1->bLedWPSBlinkInProgress = false;
 			del_timer(&pLed1->BlinkTimer);
-			pLed1->BlinkingLedState = LED_OFF;
-			pLed1->CurrLedState = LED_OFF;
+			pLed1->BlinkingLedState = LED_STATE_OFF;
+			pLed1->CurrLedState = LED_STATE_OFF;
 			if (pLed1->bLedOn)
 				mod_timer(&pLed->BlinkTimer,
 					  jiffies + msecs_to_jiffies(0));
@@ -1527,11 +1531,11 @@ static void SwLedControlMode4(struct _adapter *padapter,
 			pLed->bLedWPSBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_WPS;
 			if (pLed->bLedOn) {
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_SLOWLY_INTERVAL));
 			} else {
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 				mod_timer(&pLed->BlinkTimer, jiffies +
 					  msecs_to_jiffies(LED_BLINK_NORMAL_INTERVAL));
 			}
@@ -1545,9 +1549,9 @@ static void SwLedControlMode4(struct _adapter *padapter,
 		pLed->bLedNoLinkBlinkInProgress = true;
 		pLed->CurrLedState = LED_BLINK_SLOWLY;
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 		break;
@@ -1559,9 +1563,9 @@ static void SwLedControlMode4(struct _adapter *padapter,
 		pLed->bLedNoLinkBlinkInProgress = true;
 		pLed->CurrLedState = LED_BLINK_SLOWLY;
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 		/*LED1 settings*/
@@ -1571,9 +1575,9 @@ static void SwLedControlMode4(struct _adapter *padapter,
 			pLed1->bLedWPSBlinkInProgress = true;
 		pLed1->CurrLedState = LED_BLINK_WPS_STOP;
 		if (pLed1->bLedOn)
-			pLed1->BlinkingLedState = LED_OFF;
+			pLed1->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed1->BlinkingLedState = LED_ON;
+			pLed1->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_NORMAL_INTERVAL));
 		break;
@@ -1585,9 +1589,9 @@ static void SwLedControlMode4(struct _adapter *padapter,
 		pLed->bLedNoLinkBlinkInProgress = true;
 		pLed->CurrLedState = LED_BLINK_SLOWLY;
 		if (pLed->bLedOn)
-			pLed->BlinkingLedState = LED_OFF;
+			pLed->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed->BlinkingLedState = LED_ON;
+			pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 		/*LED1 settings*/
@@ -1598,15 +1602,15 @@ static void SwLedControlMode4(struct _adapter *padapter,
 		pLed1->CurrLedState = LED_BLINK_WPS_STOP_OVERLAP;
 		pLed1->BlinkTimes = 10;
 		if (pLed1->bLedOn)
-			pLed1->BlinkingLedState = LED_OFF;
+			pLed1->BlinkingLedState = LED_STATE_OFF;
 		else
-			pLed1->BlinkingLedState = LED_ON;
+			pLed1->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer, jiffies +
 			  msecs_to_jiffies(LED_BLINK_NORMAL_INTERVAL));
 		break;
 	case LED_CTL_POWER_OFF:
-		pLed->CurrLedState = LED_OFF;
-		pLed->BlinkingLedState = LED_OFF;
+		pLed->CurrLedState = LED_STATE_OFF;
+		pLed->BlinkingLedState = LED_STATE_OFF;
 		if (pLed->bLedNoLinkBlinkInProgress) {
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedNoLinkBlinkInProgress = false;
@@ -1660,8 +1664,8 @@ static void SwLedControlMode5(struct _adapter *padapter,
 	case LED_CTL_LINK:	/* solid blue */
 		if (pLed->CurrLedState == LED_SCAN_BLINK)
 			return;
-		pLed->CurrLedState = LED_ON;
-		pLed->BlinkingLedState = LED_ON;
+		pLed->CurrLedState = LED_STATE_ON;
+		pLed->BlinkingLedState = LED_STATE_ON;
 		pLed->bLedBlinkInProgress = false;
 		mod_timer(&pLed->BlinkTimer,
 			  jiffies + msecs_to_jiffies(0));
@@ -1679,9 +1683,9 @@ static void SwLedControlMode5(struct _adapter *padapter,
 			pLed->CurrLedState = LED_SCAN_BLINK;
 			pLed->BlinkTimes = 24;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -1695,16 +1699,16 @@ static void SwLedControlMode5(struct _adapter *padapter,
 			pLed->CurrLedState = LED_TXRX_BLINK;
 			pLed->BlinkTimes = 2;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
 		break;
 	case LED_CTL_POWER_OFF:
-		pLed->CurrLedState = LED_OFF;
-		pLed->BlinkingLedState = LED_OFF;
+		pLed->CurrLedState = LED_STATE_OFF;
+		pLed->BlinkingLedState = LED_STATE_OFF;
 		if (pLed->bLedBlinkInProgress) {
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedBlinkInProgress = false;
@@ -1731,8 +1735,8 @@ static void SwLedControlMode6(struct _adapter *padapter,
 	case LED_CTL_SITE_SURVEY:
 		if (IS_LED_WPS_BLINKING(pLed))
 				return;
-		pLed->CurrLedState = LED_ON;
-		pLed->BlinkingLedState = LED_ON;
+		pLed->CurrLedState = LED_STATE_ON;
+		pLed->BlinkingLedState = LED_STATE_ON;
 		pLed->bLedBlinkInProgress = false;
 		mod_timer(&(pLed->BlinkTimer), jiffies + msecs_to_jiffies(0));
 		break;
@@ -1746,9 +1750,9 @@ static void SwLedControlMode6(struct _adapter *padapter,
 			pLed->CurrLedState = LED_TXRX_BLINK;
 			pLed->BlinkTimes = 2;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 		}
@@ -1763,9 +1767,9 @@ static void SwLedControlMode6(struct _adapter *padapter,
 			pLed->bLedWPSBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_WPS;
 			if (pLed->bLedOn)
-				pLed->BlinkingLedState = LED_OFF;
+				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
-				pLed->BlinkingLedState = LED_ON;
+				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		}
@@ -1776,14 +1780,14 @@ static void SwLedControlMode6(struct _adapter *padapter,
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedWPSBlinkInProgress = false;
 		}
-		pLed->CurrLedState = LED_ON;
-		pLed->BlinkingLedState = LED_ON;
+		pLed->CurrLedState = LED_STATE_ON;
+		pLed->BlinkingLedState = LED_STATE_ON;
 		mod_timer(&pLed->BlinkTimer,
 			  jiffies + msecs_to_jiffies(0));
 		break;
 	case LED_CTL_POWER_OFF:
-		pLed->CurrLedState = LED_OFF;
-		pLed->BlinkingLedState = LED_OFF;
+		pLed->CurrLedState = LED_STATE_OFF;
+		pLed->BlinkingLedState = LED_STATE_OFF;
 		if (pLed->bLedBlinkInProgress) {
 			del_timer(&pLed->BlinkTimer);
 			pLed->bLedBlinkInProgress = false;
diff --git a/drivers/staging/rtl8712/rtl8712_recv.c b/drivers/staging/rtl8712/rtl8712_recv.c
index f25b34c7d115..66f0e0a35167 100644
--- a/drivers/staging/rtl8712/rtl8712_recv.c
+++ b/drivers/staging/rtl8712/rtl8712_recv.c
@@ -60,7 +60,7 @@ int r8712_init_recv_priv(struct recv_priv *precvpriv, struct _adapter *padapter)
 	_init_queue(&precvpriv->free_recv_buf_queue);
 	precvpriv->pallocated_recv_buf =
 		kzalloc(NR_RECVBUFF * sizeof(struct recv_buf) + 4, GFP_ATOMIC);
-	if (precvpriv->pallocated_recv_buf == NULL)
+	if (!precvpriv->pallocated_recv_buf)
 		return _FAIL;
 	precvpriv->precv_buf = precvpriv->pallocated_recv_buf + 4 -
 			      ((addr_t) (precvpriv->pallocated_recv_buf) & 3);
@@ -163,7 +163,8 @@ static void update_recvframe_attrib_from_recvstat(struct rx_pkt_attrib *pattrib,
 	drvinfo_sz = (le32_to_cpu(prxstat->rxdw0) & 0x000f0000) >> 16;
 	drvinfo_sz <<= 3;
 	/*TODO:
-	 * Offset 0 */
+	 * Offset 0
+	 */
 	pattrib->bdecrypted = ((le32_to_cpu(prxstat->rxdw0) & BIT(27)) >> 27)
 				 ? 0 : 1;
 	pattrib->crc_err = (le32_to_cpu(prxstat->rxdw0) & BIT(14)) >> 14;
@@ -210,7 +211,8 @@ static union recv_frame *recvframe_defrag(struct _adapter *adapter,
 	curfragnum = 0;
 	if (curfragnum != pfhdr->attrib.frag_num) {
 		/*the first fragment number must be 0
-		 *free the whole queue*/
+		 *free the whole queue
+		 */
 		r8712_free_recvframe(prframe, pfree_recv_queue);
 		r8712_free_recvframe_queue(defrag_q, pfree_recv_queue);
 		return NULL;
@@ -224,18 +226,21 @@ static union recv_frame *recvframe_defrag(struct _adapter *adapter,
 		/*check the fragment sequence  (2nd ~n fragment frame) */
 		if (curfragnum != pnfhdr->attrib.frag_num) {
 			/* the fragment number must increase  (after decache)
-			 * release the defrag_q & prframe */
+			 * release the defrag_q & prframe
+			 */
 			r8712_free_recvframe(prframe, pfree_recv_queue);
 			r8712_free_recvframe_queue(defrag_q, pfree_recv_queue);
 			return NULL;
 		}
 		curfragnum++;
 		/* copy the 2nd~n fragment frame's payload to the first fragment
-		 * get the 2nd~last fragment frame's payload */
+		 * get the 2nd~last fragment frame's payload
+		 */
 		wlanhdr_offset = pnfhdr->attrib.hdrlen + pnfhdr->attrib.iv_len;
 		recvframe_pull(pnextrframe, wlanhdr_offset);
 		/* append  to first fragment frame's tail (if privacy frame,
-		 * pull the ICV) */
+		 * pull the ICV)
+		 */
 		recvframe_pull_tail(prframe, pfhdr->attrib.icv_len);
 		memcpy(pfhdr->rx_tail, pnfhdr->rx_data, pnfhdr->len);
 		recvframe_put(prframe, pnfhdr->len);
@@ -269,7 +274,7 @@ union recv_frame *r8712_recvframe_chk_defrag(struct _adapter *padapter,
 	fragnum = pfhdr->attrib.frag_num;
 	psta_addr = pfhdr->attrib.ta;
 	psta = r8712_get_stainfo(pstapriv, psta_addr);
-	if (psta == NULL)
+	if (!psta)
 		pdefrag_q = NULL;
 	else
 		pdefrag_q = &psta->sta_recvpriv.defrag_q;
@@ -278,7 +283,8 @@ union recv_frame *r8712_recvframe_chk_defrag(struct _adapter *padapter,
 		prtnframe = precv_frame;/*isn't a fragment frame*/
 	if (ismfrag == 1) {
 		/* 0~(n-1) fragment frame
-		 * enqueue to defraf_g */
+		 * enqueue to defraf_g
+		 */
 		if (pdefrag_q != NULL) {
 			if (fragnum == 0) {
 				/*the first fragment*/
@@ -294,7 +300,8 @@ union recv_frame *r8712_recvframe_chk_defrag(struct _adapter *padapter,
 			prtnframe = NULL;
 		} else {
 			/* can't find this ta's defrag_queue, so free this
-			 * recv_frame */
+			 * recv_frame
+			 */
 			r8712_free_recvframe(precv_frame, pfree_recv_queue);
 			prtnframe = NULL;
 		}
@@ -302,7 +309,8 @@ union recv_frame *r8712_recvframe_chk_defrag(struct _adapter *padapter,
 	}
 	if ((ismfrag == 0) && (fragnum != 0)) {
 		/* the last fragment frame
-		 * enqueue the last fragment */
+		 * enqueue the last fragment
+		 */
 		if (pdefrag_q != NULL) {
 			phead = &pdefrag_q->queue;
 			list_add_tail(&pfhdr->list, phead);
@@ -311,7 +319,8 @@ union recv_frame *r8712_recvframe_chk_defrag(struct _adapter *padapter,
 			prtnframe = precv_frame;
 		} else {
 			/* can't find this ta's defrag_queue, so free this
-			 *  recv_frame */
+			 *  recv_frame
+			 */
 			r8712_free_recvframe(precv_frame, pfree_recv_queue);
 			prtnframe = NULL;
 		}
@@ -391,7 +400,8 @@ static int amsdu_to_msdu(struct _adapter *padapter, union recv_frame *prframe)
 		   eth_type != ETH_P_AARP && eth_type != ETH_P_IPX) ||
 		   !memcmp(sub_skb->data, bridge_tunnel_header, SNAP_SIZE))) {
 			/* remove RFC1042 or Bridge-Tunnel encapsulation and
-			 * replace EtherType */
+			 * replace EtherType
+			 */
 			skb_pull(sub_skb, SNAP_SIZE);
 			memcpy(skb_push(sub_skb, ETH_ALEN), pattrib->src,
 				ETH_ALEN);
@@ -530,7 +540,8 @@ int r8712_recv_indicatepkts_in_order(struct _adapter *padapter,
 		preorder_ctrl->indicate_seq = pattrib->seq_num;
 	}
 	/* Prepare indication list and indication.
-	 * Check if there is any packet need indicate. */
+	 * Check if there is any packet need indicate.
+	 */
 	while (!list_empty(phead)) {
 		prframe = container_of(plist, union recv_frame, u.list);
 		pattrib = &prframe->u.hdr.attrib;
@@ -757,7 +768,8 @@ static void query_rx_phy_status(struct _adapter *padapter,
 			/* Modify the RF RNA gain value to -40, -20,
 			 * -2, 14 by Jenyu's suggestion
 			 * Note: different RF with the different
-			 * RNA gain. */
+			 * RNA gain.
+			 */
 			case 0x3:
 				rx_pwr_all = -40 - (pcck_buf->cck_agc_rpt &
 					     0x3e);
@@ -842,7 +854,8 @@ static void query_rx_phy_status(struct _adapter *padapter,
 			total_rssi += rssi;
 		}
 		/* (2)PWDB, Average PWDB cacluated by hardware (for
-		 * rate adaptive) */
+		 * rate adaptive)
+		 */
 		rx_pwr_all = (((pphy_head[PHY_STAT_PWDB_ALL_SHT]) >> 1) & 0x7f)
 			     - 106;
 		pwdb_all = query_rx_pwr_percentage(rx_pwr_all);
@@ -870,7 +883,8 @@ static void query_rx_phy_status(struct _adapter *padapter,
 	}
 	/* UI BSS List signal strength(in percentage), make it good looking,
 	 * from 0~100. It is assigned to the BSS List in
-	 * GetValueFromBeaconOrProbeRsp(). */
+	 * GetValueFromBeaconOrProbeRsp().
+	 */
 	if (bcck_rate)
 		prframe->u.hdr.attrib.signal_strength =
 			 (u8)r8712_signal_scale_mapping(pwdb_all);
@@ -985,15 +999,15 @@ int recv_func(struct _adapter *padapter, void *pcontext)
 	}
 	process_phy_info(padapter, prframe);
 	prframe = r8712_decryptor(padapter, prframe);
-	if (prframe == NULL) {
+	if (!prframe) {
 		retval = _FAIL;
 		goto _exit_recv_func;
 	}
 	prframe = r8712_recvframe_chk_defrag(padapter, prframe);
-	if (prframe == NULL)
+	if (!prframe)
 		goto _exit_recv_func;
 	prframe = r8712_portctrl(padapter, prframe);
-	if (prframe == NULL) {
+	if (!prframe) {
 		retval = _FAIL;
 		goto _exit_recv_func;
 	}
@@ -1027,10 +1041,12 @@ static int recvbuf2recvframe(struct _adapter *padapter, struct sk_buff *pskb)
 	transfer_len = pskb->len;
 	/* Test throughput with Netgear 3700 (No security) with Chariot 3T3R
 	 * pairs. The packet count will be a big number so that the containing
-	 * packet will effect the Rx reordering. */
+	 * packet will effect the Rx reordering.
+	 */
 	if (transfer_len < pkt_len) {
 		/* In this case, it means the MAX_RECVBUF_SZ is too small to
-		 * get the data from 8712u. */
+		 * get the data from 8712u.
+		 */
 		return _FAIL;
 	}
 	do {
@@ -1049,7 +1065,7 @@ static int recvbuf2recvframe(struct _adapter *padapter, struct sk_buff *pskb)
 		if ((le32_to_cpu(prxstat->rxdw0) >> 23) & 0x01)
 			shift_sz = 2;
 		precvframe = r8712_alloc_recvframe(pfree_recv_queue);
-		if (precvframe == NULL)
+		if (!precvframe)
 			goto  _exit_recvbuf2recvframe;
 		INIT_LIST_HEAD(&precvframe->u.hdr.list);
 		precvframe->u.hdr.precvbuf = NULL; /*can't access the precvbuf*/
@@ -1057,14 +1073,16 @@ static int recvbuf2recvframe(struct _adapter *padapter, struct sk_buff *pskb)
 		tmp_len = pkt_len + drvinfo_sz + RXDESC_SIZE;
 		pkt_offset = (u16)round_up(tmp_len, 128);
 		/* for first fragment packet, driver need allocate 1536 +
-		 * drvinfo_sz + RXDESC_SIZE to defrag packet. */
+		 * drvinfo_sz + RXDESC_SIZE to defrag packet.
+		 */
 		if ((mf == 1) && (frag == 0))
 			/*1658+6=1664, 1664 is 128 alignment.*/
 			alloc_sz = max_t(u16, tmp_len, 1658);
 		else
 			alloc_sz = tmp_len;
 		/* 2 is for IP header 4 bytes alignment in QoS packet case.
-		 * 4 is for skb->data 4 bytes alignment. */
+		 * 4 is for skb->data 4 bytes alignment.
+		 */
 		alloc_sz += 6;
 		pkt_copy = netdev_alloc_skb(padapter->pnetdev, alloc_sz);
 		if (pkt_copy) {
diff --git a/drivers/staging/rtl8712/rtl8712_recv.h b/drivers/staging/rtl8712/rtl8712_recv.h
index fd9e3fc4c226..0b0c2730aac5 100644
--- a/drivers/staging/rtl8712/rtl8712_recv.h
+++ b/drivers/staging/rtl8712/rtl8712_recv.h
@@ -61,7 +61,8 @@ struct recv_stat {
 struct phy_cck_rx_status {
 	/* For CCK rate descriptor. This is a unsigned 8:1 variable.
 	 * LSB bit present 0.5. And MSB 7 bts present a signed value.
-	 * Range from -64~+63.5. */
+	 * Range from -64~+63.5.
+	 */
 	u8	adc_pwdb_X[4];
 	u8	sq_rpt;
 	u8	cck_agc_rpt;
@@ -103,7 +104,6 @@ struct recv_buf {
 	struct _adapter  *adapter;
 	struct urb *purb;
 	_pkt *pskb;
-	u8 reuse;
 	u8  irp_pending;
 	u32  transfer_len;
 	uint  len;
@@ -116,13 +116,13 @@ struct recv_buf {
 };
 
 /*
-	head  ----->
-		data  ----->
-			payload
-		tail  ----->
-	end   ----->
-	len = (unsigned int )(tail - data);
-*/
+ *	head  ----->
+ *		data  ----->
+ *			payload
+ *		tail  ----->
+ *	end   ----->
+ *	len = (unsigned int )(tail - data);
+ */
 struct recv_frame_hdr {
 	struct list_head list;
 	_pkt	*pkt;
diff --git a/drivers/staging/rtl8712/rtl8712_spec.h b/drivers/staging/rtl8712/rtl8712_spec.h
index af11b44b1140..51e042815cc9 100644
--- a/drivers/staging/rtl8712/rtl8712_spec.h
+++ b/drivers/staging/rtl8712/rtl8712_spec.h
@@ -83,7 +83,8 @@
 
 #define CMD_ADDR_MAPPING_SHIFT		2	/*SDIO CMD ADDR MAPPING,
 						 *shift 2 bit for match
-						 * offset[14:2]*/
+						 * offset[14:2]
+						 */
 
 /*Offset for SDIO LOCAL*/
 #define	OFFSET_SDIO_LOCAL				0x0FFF
diff --git a/drivers/staging/rtl8712/rtl8712_syscfg_bitdef.h b/drivers/staging/rtl8712/rtl8712_syscfg_bitdef.h
index eed09c872fc9..2e66d28d6918 100644
--- a/drivers/staging/rtl8712/rtl8712_syscfg_bitdef.h
+++ b/drivers/staging/rtl8712/rtl8712_syscfg_bitdef.h
@@ -68,11 +68,13 @@
 #define SYS_CLKSEL		BIT(SYS_CLKSEL_SHT) /* System Clock 80MHz*/
 #define PS_CLKSEL_SHT		1
 #define PS_CLKSEL		BIT(PS_CLKSEL_SHT) /*System power save
-						    * clock select.*/
+						    * clock select.
+						    */
 #define CPU_CLKSEL_SHT		2
 #define CPU_CLKSEL		BIT(CPU_CLKSEL_SHT) /* System Clock select,
 						     * 1: AFE source,
-						     * 0: System clock(L-Bus)*/
+						     * 0: System clock(L-Bus)
+						     */
 #define INT32K_EN_SHT		3
 #define INT32K_EN		BIT(INT32K_EN_SHT)
 #define MACSLP_SHT		4
@@ -85,10 +87,12 @@
 #define RING_CLK_EN		BIT(RING_CLK_EN_SHT)
 #define SWHW_SEL_SHT		14
 #define SWHW_SEL		BIT(SWHW_SEL_SHT) /* Load done,
-						   * control path switch.*/
+						   * control path switch.
+						   */
 #define FWHW_SEL_SHT		15
 #define FWHW_SEL		BIT(FWHW_SEL_SHT) /* Sleep exit,
-						   * control path switch.*/
+						   * control path switch.
+						   */
 
 /*9346CR*/
 #define	_VPDIDX_MSK		0xFF00
@@ -118,10 +122,12 @@
 #define	AFE_MISC_E32_EN		BIT(AFE_MISC_E32_EN_SHT)
 #define	AFE_MISC_MBEN_SHT	1
 #define	AFE_MISC_MBEN		BIT(AFE_MISC_MBEN_SHT)/* Enable AFE Macro
-						       * Block's Mbias.*/
+						       * Block's Mbias.
+						       */
 #define	AFE_MISC_BGEN_SHT	0
 #define	AFE_MISC_BGEN		BIT(AFE_MISC_BGEN_SHT)/* Enable AFE Macro
-						       * Block's Bandgap.*/
+						       * Block's Bandgap.
+						       */
 
 
 /*--------------------------------------------------------------------------*/
@@ -149,10 +155,12 @@
 
 /* EFUSE_CTRL*/
 #define EF_FLAG			BIT(31)		/* Access Flag, Write:1;
-						 *	        Read:0*/
+						 *	        Read:0
+						 */
 #define EF_PGPD			0x70000000	/* E-fuse Program time*/
 #define EF_RDT			0x0F000000	/* E-fuse read time: in the
-						 * unit of cycle time*/
+						 * unit of cycle time
+						 */
 #define EF_PDN_EN		BIT(19)		/* EFuse Power down enable*/
 #define ALD_EN			BIT(18)		/* Autoload Enable*/
 #define EF_ADDR			0x0003FF00	/* Access Address*/
@@ -164,7 +172,8 @@
 /* EFUSE_CLK_CTRL*/
 #define EFUSE_CLK_EN		BIT(1)		/* E-Fuse Clock Enable*/
 #define EFUSE_CLK_SEL		BIT(0)		/* E-Fuse Clock Select,
-						 * 0:500K, 1:40M*/
+						 * 0:500K, 1:40M
+						 */
 
 #endif	/*__RTL8712_SYSCFG_BITDEF_H__*/
 
diff --git a/drivers/staging/rtl8712/rtl8712_xmit.c b/drivers/staging/rtl8712/rtl8712_xmit.c
index 7e0b94503dfc..c4f03a602a2e 100644
--- a/drivers/staging/rtl8712/rtl8712_xmit.c
+++ b/drivers/staging/rtl8712/rtl8712_xmit.c
@@ -535,7 +535,8 @@ static void update_txdesc(struct xmit_frame *pxmitframe, uint *pmem, int sz)
 		 * seqnum per tid. about usb using 4-endpoint, qsel points out
 		 * the correct mapping between AC&Endpoint,
 		 * the purpose is that correct mapping lets the MAC release
-		 * the AC Queue list correctly. */
+		 * the AC Queue list correctly.
+		 */
 		ptxdesc->txdw3 = cpu_to_le32((pattrib->priority << SEQ_SHT) &
 				 0x0fff0000);
 		if ((pattrib->ether_type != 0x888e) &&
@@ -586,7 +587,8 @@ static void update_txdesc(struct xmit_frame *pxmitframe, uint *pmem, int sz)
 		 * per tid. about usb using 4-endpoint, qsel points out the
 		 * correct mapping between AC&Endpoint,
 		 * the purpose is that correct mapping let the MAC releases
-		 * the AC Queue list correctly. */
+		 * the AC Queue list correctly.
+		 */
 		ptxdesc->txdw3 = cpu_to_le32((pattrib->priority << SEQ_SHT) &
 					      0x0fff0000);
 		/* offset 16 */
@@ -627,7 +629,7 @@ int r8712_xmitframe_complete(struct _adapter *padapter,
 
 	phwxmits = pxmitpriv->hwxmits;
 	hwentry = pxmitpriv->hwxmit_entry;
-	if (pxmitbuf == NULL) {
+	if (!pxmitbuf) {
 		pxmitbuf = r8712_alloc_xmitbuf(pxmitpriv);
 		if (!pxmitbuf)
 			return false;
@@ -686,7 +688,8 @@ int r8712_xmitframe_complete(struct _adapter *padapter,
 				res = r8712_xmitframe_coalesce(padapter,
 					pxmitframe->pkt, pxmitframe);
 			/* always return ndis_packet after
-			 * r8712_xmitframe_coalesce */
+			 * r8712_xmitframe_coalesce
+			 */
 			r8712_xmit_complete(padapter, pxmitframe);
 		}
 		if (res == _SUCCESS)
diff --git a/drivers/staging/rtl8712/rtl871x_cmd.c b/drivers/staging/rtl8712/rtl871x_cmd.c
index aed03cfbb1ba..b7ee5e63af33 100644
--- a/drivers/staging/rtl8712/rtl871x_cmd.c
+++ b/drivers/staging/rtl8712/rtl871x_cmd.c
@@ -51,14 +51,14 @@
 #include "mlme_osdep.h"
 
 /*
-Caller and the r8712_cmd_thread can protect cmd_q by spin_lock.
-No irqsave is necessary.
-*/
+ * Caller and the r8712_cmd_thread can protect cmd_q by spin_lock.
+ * No irqsave is necessary.
+ */
 
 static sint _init_cmd_priv(struct cmd_priv *pcmdpriv)
 {
-	sema_init(&(pcmdpriv->cmd_queue_sema), 0);
-	sema_init(&(pcmdpriv->terminate_cmdthread_sema), 0);
+	init_completion(&pcmdpriv->cmd_queue_comp);
+	init_completion(&pcmdpriv->terminate_cmdthread_comp);
 
 	_init_queue(&(pcmdpriv->cmd_queue));
 
@@ -66,13 +66,13 @@ static sint _init_cmd_priv(struct cmd_priv *pcmdpriv)
 	pcmdpriv->cmd_seq = 1;
 	pcmdpriv->cmd_allocated_buf = kmalloc(MAX_CMDSZ + CMDBUFF_ALIGN_SZ,
 					      GFP_ATOMIC);
-	if (pcmdpriv->cmd_allocated_buf == NULL)
+	if (!pcmdpriv->cmd_allocated_buf)
 		return _FAIL;
 	pcmdpriv->cmd_buf = pcmdpriv->cmd_allocated_buf  +  CMDBUFF_ALIGN_SZ -
 			    ((addr_t)(pcmdpriv->cmd_allocated_buf) &
 			    (CMDBUFF_ALIGN_SZ - 1));
 	pcmdpriv->rsp_allocated_buf = kmalloc(MAX_RSPSZ + 4, GFP_ATOMIC);
-	if (pcmdpriv->rsp_allocated_buf == NULL)
+	if (!pcmdpriv->rsp_allocated_buf)
 		return _FAIL;
 	pcmdpriv->rsp_buf = pcmdpriv->rsp_allocated_buf  +  4 -
 			    ((addr_t)(pcmdpriv->rsp_allocated_buf) & 3);
@@ -88,7 +88,7 @@ static sint _init_evt_priv(struct evt_priv *pevtpriv)
 	pevtpriv->event_seq = 0;
 	pevtpriv->evt_allocated_buf = kmalloc(MAX_EVTSZ + 4, GFP_ATOMIC);
 
-	if (pevtpriv->evt_allocated_buf == NULL)
+	if (!pevtpriv->evt_allocated_buf)
 		return _FAIL;
 	pevtpriv->evt_buf = pevtpriv->evt_allocated_buf  +  4 -
 			    ((addr_t)(pevtpriv->evt_allocated_buf) & 3);
@@ -110,20 +110,20 @@ static void _free_cmd_priv(struct cmd_priv *pcmdpriv)
 }
 
 /*
-Calling Context:
-
-_enqueue_cmd can only be called between kernel thread,
-since only spin_lock is used.
-
-ISR/Call-Back functions can't call this sub-function.
-
-*/
+ * Calling Context:
+ *
+ * _enqueue_cmd can only be called between kernel thread,
+ * since only spin_lock is used.
+ *
+ * ISR/Call-Back functions can't call this sub-function.
+ *
+ */
 
 static sint _enqueue_cmd(struct  __queue *queue, struct cmd_obj *obj)
 {
 	unsigned long irqL;
 
-	if (obj == NULL)
+	if (!obj)
 		return _SUCCESS;
 	spin_lock_irqsave(&queue->lock, irqL);
 	list_add_tail(&obj->list, &queue->queue);
@@ -172,7 +172,7 @@ u32 r8712_enqueue_cmd(struct cmd_priv *pcmdpriv, struct cmd_obj *obj)
 	if (pcmdpriv->padapter->eeprompriv.bautoload_fail_flag)
 		return _FAIL;
 	res = _enqueue_cmd(&pcmdpriv->cmd_queue, obj);
-	up(&pcmdpriv->cmd_queue_sema);
+	complete(&pcmdpriv->cmd_queue_comp);
 	return res;
 }
 
@@ -181,7 +181,7 @@ u32 r8712_enqueue_cmd_ex(struct cmd_priv *pcmdpriv, struct cmd_obj *obj)
 	unsigned long irqL;
 	struct  __queue *queue;
 
-	if (obj == NULL)
+	if (!obj)
 		return _SUCCESS;
 	if (pcmdpriv->padapter->eeprompriv.bautoload_fail_flag)
 		return _FAIL;
@@ -189,7 +189,7 @@ u32 r8712_enqueue_cmd_ex(struct cmd_priv *pcmdpriv, struct cmd_obj *obj)
 	spin_lock_irqsave(&queue->lock, irqL);
 	list_add_tail(&obj->list, &queue->queue);
 	spin_unlock_irqrestore(&queue->lock, irqL);
-	up(&pcmdpriv->cmd_queue_sema);
+	complete(&pcmdpriv->cmd_queue_comp);
 	return _SUCCESS;
 }
 
@@ -211,11 +211,11 @@ void r8712_free_cmd_obj(struct cmd_obj *pcmd)
 }
 
 /*
-r8712_sitesurvey_cmd(~)
-	### NOTE:#### (!!!!)
-	MUST TAKE CARE THAT BEFORE CALLING THIS FUNC,
-	 YOU SHOULD HAVE LOCKED pmlmepriv->lock
-*/
+ *	r8712_sitesurvey_cmd(~)
+ *		### NOTE:#### (!!!!)
+ *		MUST TAKE CARE THAT BEFORE CALLING THIS FUNC,
+ *		YOU SHOULD HAVE LOCKED pmlmepriv->lock
+ */
 u8 r8712_sitesurvey_cmd(struct _adapter *padapter,
 			struct ndis_802_11_ssid *pssid)
 {
@@ -477,7 +477,7 @@ u8 r8712_joinbss_cmd(struct _adapter  *padapter, struct wlan_network *pnetwork)
 		}
 	}
 	psecnetwork = &psecuritypriv->sec_bss;
-	if (psecnetwork == NULL) {
+	if (!psecnetwork) {
 		kfree(pcmd);
 		return _FAIL;
 	}
@@ -491,8 +491,9 @@ u8 r8712_joinbss_cmd(struct _adapter  *padapter, struct wlan_network *pnetwork)
 		memcpy(&psecuritypriv->authenticator_ie[1],
 			&psecnetwork->IEs[12], (256 - 1));
 	psecnetwork->IELength = 0;
-	/* If the driver wants to use the bssid to create the connection.
-	 * If not,  we copy the connecting AP's MAC address to it so that
+	/*
+	 * If the driver wants to use the bssid to create the connection.
+	 * If not, we copy the connecting AP's MAC address to it so that
 	 * the driver just has the bssid information for PMKIDList searching.
 	 */
 	if (!pmlmepriv->assoc_by_bssid)
@@ -519,7 +520,8 @@ u8 r8712_joinbss_cmd(struct _adapter  *padapter, struct wlan_network *pnetwork)
 		}
 	}
 	if (pregistrypriv->ht_enable) {
-		/* For WEP mode, we will use the bg mode to do the connection
+		/*
+		 * For WEP mode, we will use the bg mode to do the connection
 		 * to avoid some IOT issues, especially for Realtek 8192u
 		 * SoftAP.
 		 */
@@ -882,16 +884,16 @@ void r8712_createbss_cmd_callback(struct _adapter *padapter,
 		if (!psta) {
 			psta = r8712_alloc_stainfo(&padapter->stapriv,
 						   pnetwork->MacAddress);
-			if (psta == NULL)
+			if (!psta)
 				goto createbss_cmd_fail;
 		}
 		r8712_indicate_connect(padapter);
 	} else {
 		pwlan = _r8712_alloc_network(pmlmepriv);
-		if (pwlan == NULL) {
+		if (!pwlan) {
 			pwlan = r8712_get_oldest_wlan_network(
 				&pmlmepriv->scanned_queue);
-			if (pwlan == NULL)
+			if (!pwlan)
 				goto createbss_cmd_fail;
 			pwlan->last_scanned = jiffies;
 		} else
@@ -904,8 +906,10 @@ void r8712_createbss_cmd_callback(struct _adapter *padapter,
 			(r8712_get_wlan_bssid_ex_sz(pnetwork)));
 		if (pmlmepriv->fw_state & _FW_UNDER_LINKING)
 			pmlmepriv->fw_state ^= _FW_UNDER_LINKING;
-		/* we will set _FW_LINKED when there is one more sat to
-		 * join us (stassoc_event_callback) */
+		/*
+		 * we will set _FW_LINKED when there is one more sat to
+		 * join us (stassoc_event_callback)
+		 */
 	}
 createbss_cmd_fail:
 	spin_unlock_irqrestore(&pmlmepriv->lock, irqL);
@@ -921,7 +925,7 @@ void r8712_setstaKey_cmdrsp_callback(struct _adapter *padapter,
 	struct sta_info *psta = r8712_get_stainfo(pstapriv,
 						  psetstakey_rsp->addr);
 
-	if (psta == NULL)
+	if (!psta)
 		goto exit;
 	psta->aid = psta->mac_id = psetstakey_rsp->keyid; /*CAM_ID(CAM_ENTRY)*/
 exit:
@@ -941,7 +945,7 @@ void r8712_setassocsta_cmdrsp_callback(struct _adapter *padapter,
 	struct sta_info *psta = r8712_get_stainfo(pstapriv,
 						  passocsta_parm->addr);
 
-	if (psta == NULL)
+	if (!psta)
 		return;
 	psta->aid = psta->mac_id = passocsta_rsp->cam_id;
 	spin_lock_irqsave(&pmlmepriv->lock, irqL);
diff --git a/drivers/staging/rtl8712/rtl871x_cmd.h b/drivers/staging/rtl8712/rtl871x_cmd.h
index e4a2a50c85de..3284dcf2f1a9 100644
--- a/drivers/staging/rtl8712/rtl871x_cmd.h
+++ b/drivers/staging/rtl8712/rtl871x_cmd.h
@@ -50,8 +50,8 @@ struct cmd_obj {
 };
 
 struct cmd_priv {
-	struct semaphore cmd_queue_sema;
-	struct semaphore terminate_cmdthread_sema;
+	struct completion cmd_queue_comp;
+	struct completion terminate_cmdthread_comp;
 	struct  __queue	cmd_queue;
 	u8 cmd_seq;
 	u8 *cmd_buf;	/*shall be non-paged, and 4 bytes aligned*/
@@ -185,10 +185,12 @@ struct setauth_parm {
  */
 struct setkey_parm {
 	u8	algorithm;	/* encryption algorithm, could be none, wep40,
-				 * TKIP, CCMP, wep104 */
+				 * TKIP, CCMP, wep104
+				 */
 	u8	keyid;
 	u8	grpkey;		/* 1: this is the grpkey for 802.1x.
-				 * 0: this is the unicast key for 802.1x */
+				 * 0: this is the unicast key for 802.1x
+				 */
 	u8	key[16];	/* this could be 40 or 104 */
 };
 
@@ -215,15 +217,15 @@ struct SetMacAddr_param {
 };
 
 /*
-Caller Ad-Hoc/AP
-
-Command -Rsp(AID == CAMID) mode
-
-This is to force fw to add an sta_data entry per driver's request.
-
-FW will write an cam entry associated with it.
-
-*/
+ *	Caller Ad-Hoc/AP
+ *
+ *	Command -Rsp(AID == CAMID) mode
+ *
+ *	This is to force fw to add an sta_data entry per driver's request.
+ *
+ *	FW will write an cam entry associated with it.
+ *
+ */
 struct set_assocsta_parm {
 	u8	addr[ETH_ALEN];
 };
@@ -234,27 +236,27 @@ struct set_assocsta_rsp {
 };
 
 /*
-	Caller Ad-Hoc/AP
-
-	Command mode
-
-	This is to force fw to del an sta_data entry per driver's request
-
-	FW will invalidate the cam entry associated with it.
-
-*/
+ *	Caller Ad-Hoc/AP
+ *
+ *	Command mode
+ *
+ *	This is to force fw to del an sta_data entry per driver's request
+ *
+ *	FW will invalidate the cam entry associated with it.
+ *
+ */
 struct del_assocsta_parm {
 	u8	addr[ETH_ALEN];
 };
 
 /*
-Caller Mode: AP/Ad-HoC(M)
-
-Notes: To notify fw that given staid has changed its power state
-
-Command Mode
-
-*/
+ *	Caller Mode: AP/Ad-HoC(M)
+ *
+ *	Notes: To notify fw that given staid has changed its power state
+ *
+ *	Command Mode
+ *
+ */
 struct setstapwrstate_parm {
 	u8	staid;
 	u8	status;
@@ -262,25 +264,25 @@ struct setstapwrstate_parm {
 };
 
 /*
-Caller Mode: Any
-
-Notes: To setup the basic rate of RTL8711
-
-Command Mode
-
-*/
+ *	Caller Mode: Any
+ *
+ *	Notes: To setup the basic rate of RTL8711
+ *
+ *	Command Mode
+ *
+ */
 struct	setbasicrate_parm {
 	u8	basicrates[NumRates];
 };
 
 /*
-Caller Mode: Any
-
-Notes: To read the current basic rate
-
-Command-Rsp Mode
-
-*/
+ *	Caller Mode: Any
+ *
+ *	Notes: To read the current basic rate
+ *
+ *	Command-Rsp Mode
+ *
+ */
 struct getbasicrate_parm {
 	u32 rsvd;
 };
@@ -290,13 +292,13 @@ struct getbasicrate_rsp {
 };
 
 /*
-Caller Mode: Any
-
-Notes: To setup the data rate of RTL8711
-
-Command Mode
-
-*/
+ *	Caller Mode: Any
+ *
+ *	Notes: To setup the data rate of RTL8711
+ *
+ *	Command Mode
+ *
+ */
 struct setdatarate_parm {
 	u8	mac_id;
 	u8	datarates[NumRates];
@@ -332,13 +334,13 @@ struct SetChannelPlan_param {
 };
 
 /*
-Caller Mode: Any
-
-Notes: To read the current data rate
-
-Command-Rsp Mode
-
-*/
+ *	Caller Mode: Any
+ *
+ *	Notes: To read the current data rate
+ *
+ *	Command-Rsp Mode
+ *
+ */
 struct getdatarate_parm {
 	u32 rsvd;
 
@@ -349,36 +351,36 @@ struct getdatarate_rsp {
 
 
 /*
-Caller Mode: Any
-AP: AP can use the info for the contents of beacon frame
-Infra: STA can use the info when sitesurveying
-Ad-HoC(M): Like AP
-Ad-HoC(C): Like STA
-
-
-Notes: To set the phy capability of the NIC
-
-Command Mode
-
-*/
+ *	Caller Mode: Any
+ *	AP: AP can use the info for the contents of beacon frame
+ *	Infra: STA can use the info when sitesurveying
+ *	Ad-HoC(M): Like AP
+ *	Ad-HoC(C): Like STA
+ *
+ *
+ *	Notes: To set the phy capability of the NIC
+ *
+ *	Command Mode
+ *
+ */
 
 /*
-Caller Mode: Any
-
-Notes: To set the channel/modem/band
-This command will be used when channel/modem/band is changed.
-
-Command Mode
-
-*/
+ *	Caller Mode: Any
+ *
+ *	Notes: To set the channel/modem/band
+ *	This command will be used when channel/modem/band is changed.
+ *
+ *	Command Mode
+ *
+ */
 /*
-Caller Mode: Any
-
-Notes: To get the current setting of channel/modem/band
-
-Command-Rsp Mode
-
-*/
+ *	Caller Mode: Any
+ *
+ *	Notes: To get the current setting of channel/modem/band
+ *
+ *	Command-Rsp Mode
+ *
+ */
 struct	getphy_rsp {
 	u8	rfchannel;
 	u8	modem;
@@ -428,58 +430,58 @@ struct getrfintfs_parm {
 };
 
 /*
-	Notes: This command is used for H2C/C2H loopback testing
-
-	mac[0] == 0
-	==> CMD mode, return H2C_SUCCESS.
-	The following condition must be ture under CMD mode
-		mac[1] == mac[4], mac[2] == mac[3], mac[0]=mac[5]= 0;
-		s0 == 0x1234, s1 == 0xabcd, w0 == 0x78563412, w1 == 0x5aa5def7;
-		s2 == (b1 << 8 | b0);
-
-	mac[0] == 1
-	==> CMD_RSP mode, return H2C_SUCCESS_RSP
-
-	The rsp layout shall be:
-	rsp:			parm:
-		mac[0]  =   mac[5];
-		mac[1]  =   mac[4];
-		mac[2]  =   mac[3];
-		mac[3]  =   mac[2];
-		mac[4]  =   mac[1];
-		mac[5]  =   mac[0];
-		s0		=   s1;
-		s1		=   swap16(s0);
-		w0		=	swap32(w1);
-		b0		=	b1
-		s2		=	s0 + s1
-		b1		=	b0
-		w1		=	w0
-
-	mac[0] ==	2
-	==> CMD_EVENT mode, return	H2C_SUCCESS
-	The event layout shall be:
-	event:	     parm:
-	mac[0]  =   mac[5];
-	mac[1]  =   mac[4];
-	mac[2]  =   event's sequence number, starting from 1 to parm's marc[3]
-	mac[3]  =   mac[2];
-	mac[4]  =   mac[1];
-	mac[5]  =   mac[0];
-	s0		=   swap16(s0) - event.mac[2];
-	s1		=   s1 + event.mac[2];
-	w0		=	swap32(w0);
-	b0		=	b1
-	s2		=	s0 + event.mac[2]
-	b1		=	b0
-	w1		=	swap32(w1) - event.mac[2];
-
-	parm->mac[3] is the total event counts that host requested.
-
-
-	event will be the same with the cmd's param.
-
-*/
+ *	Notes: This command is used for H2C/C2H loopback testing
+ *
+ *	mac[0] == 0
+ *	==> CMD mode, return H2C_SUCCESS.
+ *	The following condition must be ture under CMD mode
+ *		mac[1] == mac[4], mac[2] == mac[3], mac[0]=mac[5]= 0;
+ *		s0 == 0x1234, s1 == 0xabcd, w0 == 0x78563412, w1 == 0x5aa5def7;
+ *		s2 == (b1 << 8 | b0);
+ *
+ *	mac[0] == 1
+ *	==> CMD_RSP mode, return H2C_SUCCESS_RSP
+ *
+ *	The rsp layout shall be:
+ *	rsp:			parm:
+ *		mac[0]  =   mac[5];
+ *		mac[1]  =   mac[4];
+ *		mac[2]  =   mac[3];
+ *		mac[3]  =   mac[2];
+ *		mac[4]  =   mac[1];
+ *		mac[5]  =   mac[0];
+ *		s0		=   s1;
+ *		s1		=   swap16(s0);
+ *		w0		=	swap32(w1);
+ *		b0		=	b1
+ *		s2		=	s0 + s1
+ *		b1		=	b0
+ *		w1		=	w0
+ *
+ *	mac[0] ==	2
+ *	==> CMD_EVENT mode, return	H2C_SUCCESS
+ *	The event layout shall be:
+ *	event:	     parm:
+ *	mac[0]  =   mac[5];
+ *	mac[1]  =   mac[4];
+ *	mac[2]  =   event's sequence number, starting from 1 to parm's marc[3]
+ *	mac[3]  =   mac[2];
+ *	mac[4]  =   mac[1];
+ *	mac[5]  =   mac[0];
+ *	s0		=   swap16(s0) - event.mac[2];
+ *	s1		=   s1 + event.mac[2];
+ *	w0		=	swap32(w0);
+ *	b0		=	b1
+ *	s2		=	s0 + event.mac[2]
+ *	b1		=	b0
+ *	w1		=	swap32(w1) - event.mac[2];
+ *
+ *	parm->mac[3] is the total event counts that host requested.
+ *
+ *
+ *	event will be the same with the cmd's param.
+ *
+ */
 
 /* CMD param Formart for DRV INTERNAL CMD HDL*/
 struct drvint_cmd_parm {
@@ -570,7 +572,8 @@ struct setpwrmode_parm  {
 	u8	bcn_rx_en;
 	u8	bcn_pass_cnt;	  /* fw report one beacon information to
 				   * driver  when it receives bcn_pass_cnt
-				   *  beacons. */
+				   * beacons.
+				   */
 	u8	bcn_to;		  /* beacon TO (ms). ¡§=0¡¨ no limit.*/
 	u16	bcn_itv;
 	u8	app_itv; /* only for VOIP mode. */
diff --git a/drivers/staging/rtl8712/rtl871x_ht.h b/drivers/staging/rtl8712/rtl871x_ht.h
index 41872d937408..513f458ea07c 100644
--- a/drivers/staging/rtl8712/rtl871x_ht.h
+++ b/drivers/staging/rtl8712/rtl871x_ht.h
@@ -36,7 +36,8 @@ struct ht_priv {
 	unsigned int	tx_amsdu_enable;/*for enable Tx A-MSDU */
 	unsigned int	tx_amdsu_maxlen; /* 1: 8k, 0:4k ; default:8k, for tx */
 	unsigned int	rx_ampdu_maxlen; /* for rx reordering ctrl win_sz,
-					  *  updated when join_callback. */
+					  * updated when join_callback.
+					  */
 	struct ieee80211_ht_cap ht_cap;
 };
 
diff --git a/drivers/staging/rtl8712/rtl871x_ioctl.h b/drivers/staging/rtl8712/rtl871x_ioctl.h
index c9218be5bb4f..08bcb3b41bbd 100644
--- a/drivers/staging/rtl8712/rtl871x_ioctl.h
+++ b/drivers/staging/rtl8712/rtl871x_ioctl.h
@@ -68,7 +68,8 @@ struct oid_par_priv {
 
 struct oid_obj_priv {
 	unsigned char	dbg; /* 0: without OID debug message
-			      * 1: with OID debug message */
+			      * 1: with OID debug message
+			      */
 	uint (*oidfuns)(struct oid_par_priv *poid_par_priv);
 };
 
diff --git a/drivers/staging/rtl8712/rtl871x_ioctl_linux.c b/drivers/staging/rtl8712/rtl871x_ioctl_linux.c
index e205adf24da2..475e7904fe45 100644
--- a/drivers/staging/rtl8712/rtl871x_ioctl_linux.c
+++ b/drivers/staging/rtl8712/rtl871x_ioctl_linux.c
@@ -1976,9 +1976,9 @@ static int r871x_get_ap_info(struct net_device *dev,
 	if (pdata->length >= 32) {
 		if (copy_from_user(data, pdata->pointer, 32))
 			return -EINVAL;
-                data[32] = 0;
+		data[32] = 0;
 	} else {
-		 return -EINVAL;
+		return -EINVAL;
 	}
 	spin_lock_irqsave(&(pmlmepriv->scanned_queue.lock), irqL);
 	phead = &queue->queue;
diff --git a/drivers/staging/rtl8712/rtl871x_ioctl_set.c b/drivers/staging/rtl8712/rtl871x_ioctl_set.c
index 56760cda8e89..0aaf2aab6dd0 100644
--- a/drivers/staging/rtl8712/rtl871x_ioctl_set.c
+++ b/drivers/staging/rtl8712/rtl871x_ioctl_set.c
@@ -140,7 +140,8 @@ u8 r8712_set_802_11_bssid(struct _adapter *padapter, u8 *bssid)
 		    ETH_ALEN)) {
 			if (!check_fwstate(pmlmepriv, WIFI_STATION_STATE))
 				goto _Abort_Set_BSSID; /* driver is in
-						* WIFI_ADHOC_MASTER_STATE */
+						* WIFI_ADHOC_MASTER_STATE
+						*/
 		} else {
 			r8712_disassoc_cmd(padapter);
 			if (check_fwstate(pmlmepriv, _FW_LINKED))
@@ -203,7 +204,8 @@ void r8712_set_802_11_ssid(struct _adapter *padapter,
 					}
 				} else {
 					goto _Abort_Set_SSID; /* driver is in
-						  * WIFI_ADHOC_MASTER_STATE */
+						* WIFI_ADHOC_MASTER_STATE
+						*/
 				}
 			}
 		} else {
@@ -254,12 +256,14 @@ void r8712_set_802_11_infrastructure_mode(struct _adapter *padapter,
 		    (*pold_state == Ndis802_11IBSS)) {
 			/* will clr Linked_state before this function,
 			 * we must have checked whether issue dis-assoc_cmd or
-			 * not */
+			 * not
+			 */
 			r8712_ind_disconnect(padapter);
 		}
 		*pold_state = networktype;
 		/* clear WIFI_STATION_STATE; WIFI_AP_STATE; WIFI_ADHOC_STATE;
-		 * WIFI_ADHOC_MASTER_STATE */
+		 * WIFI_ADHOC_MASTER_STATE
+		 */
 		_clr_fwstate_(pmlmepriv, WIFI_STATION_STATE | WIFI_AP_STATE |
 			      WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE);
 		switch (networktype) {
diff --git a/drivers/staging/rtl8712/rtl871x_led.h b/drivers/staging/rtl8712/rtl871x_led.h
index eb612053a3dd..adfbc400a18d 100644
--- a/drivers/staging/rtl8712/rtl871x_led.h
+++ b/drivers/staging/rtl8712/rtl871x_led.h
@@ -72,14 +72,17 @@ enum LED_STRATEGY_871x {
 	SW_LED_MODE0, /* SW control 1 LED via GPIO0. It is default option. */
 	SW_LED_MODE1, /* 2 LEDs, through LED0 and LED1. For ALPHA. */
 	SW_LED_MODE2, /* SW control 1 LED via GPIO0,
-		       * custom for AzWave 8187 minicard. */
+		       * custom for AzWave 8187 minicard.
+		       */
 	SW_LED_MODE3, /* SW control 1 LED via GPIO0,
-		       *  customized for Sercomm Printer Server case.*/
+		       * customized for Sercomm Printer Server case.
+		       */
 	SW_LED_MODE4, /*for Edimax / Belkin*/
 	SW_LED_MODE5, /*for Sercomm / Belkin*/
 	SW_LED_MODE6, /*for WNC / Corega*/
 	HW_LED, /* HW control 2 LEDs, LED0 and LED1 (there are 4 different
-		 * control modes, see MAC.CONFIG1 for details.)*/
+		 * control modes, see MAC.CONFIG1 for details.)
+		 */
 };
 
 struct LED_871x {
@@ -96,7 +99,8 @@ struct LED_871x {
 	u8			bLedWPSBlinkInProgress;
 	u32			BlinkTimes; /* No. times to toggle for blink.*/
 	u32			BlinkingLedState; /* Next state for blinking,
-						   * either LED_ON or OFF.*/
+						   * either LED_ON or OFF.
+						   */
 
 	struct timer_list	BlinkTimer; /* Timer object for led blinking.*/
 	struct work_struct	BlinkWorkItem; /* Workitem used by BlinkTimer */
@@ -115,7 +119,8 @@ struct led_priv {
 
 /*===========================================================================
  * Interface to manipulate LED objects.
- *===========================================================================*/
+ *===========================================================================
+ */
 void r8712_InitSwLeds(struct _adapter *padapter);
 void r8712_DeInitSwLeds(struct _adapter *padapter);
 void LedControl871x(struct _adapter *padapter, enum LED_CTL_MODE LedAction);
diff --git a/drivers/staging/rtl8712/rtl871x_mlme.c b/drivers/staging/rtl8712/rtl871x_mlme.c
index 772bf9fa9592..c1feef3da26c 100644
--- a/drivers/staging/rtl8712/rtl871x_mlme.c
+++ b/drivers/staging/rtl8712/rtl871x_mlme.c
@@ -403,7 +403,8 @@ static void update_scanned_network(struct _adapter *adapter,
 
 
 	/* If we didn't find a match, then get a new network slot to initialize
-	 * with this beacon's information */
+	 * with this beacon's information
+	 */
 	if (end_of_queue_search(phead, plist)) {
 		if (list_empty(&pmlmepriv->free_bss_pool.queue)) {
 			/* If there are no more slots, expire the oldest */
@@ -926,7 +927,8 @@ void r8712_stassoc_event_callback(struct _adapter *adapter, u8 *pbuf)
 	if (psta != NULL) {
 		/*the sta have been in sta_info_queue => do nothing
 		 *(between drv has received this event before and
-		 *  fw have not yet to set key to CAM_ENTRY) */
+		 * fw have not yet to set key to CAM_ENTRY)
+		 */
 		return;
 	}
 
@@ -1171,7 +1173,8 @@ int r8712_select_and_join_from_scan(struct mlme_priv *pmlmepriv)
 		     pmlmepriv->assoc_ssid.SsidLength))) {
 			if (pmlmepriv->assoc_by_rssi) {
 				/* if the ssid is the same, select the bss
-				 *  which has the max rssi*/
+				 * which has the max rssi
+				 */
 				if (pnetwork_max_rssi) {
 					if (pnetwork->network.Rssi >
 					    pnetwork_max_rssi->network.Rssi)
@@ -1352,7 +1355,8 @@ static int SecIsInPMKIDList(struct _adapter *Adapter, u8 *bssid)
 		i = -1; /* Could not find. */
 	} else {
 		; /* There is one Pre-Authentication Key for the
-		   * specific BSSID. */
+		   * specific BSSID.
+		   */
 	}
 	return i;
 }
@@ -1430,7 +1434,8 @@ sint r8712_restruct_sec_ie(struct _adapter *adapter, u8 *in_ie,
 	if (match) {
 		if (sec_ie[0] == _WPA_IE_ID_) {
 			/* parsing SSN IE to select required encryption
-			 * algorithm, and set the bc/mc encryption algorithm */
+			 * algorithm, and set the bc/mc encryption algorithm
+			 */
 			while (true) {
 				/*check wpa_oui tag*/
 				if (memcmp(&sec_ie[2], &wpa_oui[0], 4)) {
@@ -1444,7 +1449,8 @@ sint r8712_restruct_sec_ie(struct _adapter *adapter, u8 *in_ie,
 				}
 				if (!memcmp(&sec_ie[8], &wpa_oui[0], 3)) {
 					/* get bc/mc encryption type (group
-					 * key type)*/
+					 * key type)
+					 */
 					switch (sec_ie[11]) {
 					case 0x0: /*none*/
 						psecuritypriv->XGrpPrivacy =
@@ -1482,7 +1488,8 @@ sint r8712_restruct_sec_ie(struct _adapter *adapter, u8 *in_ie,
 					} /*else the uncst_oui is match*/
 				} else { /*mixed mode, unicast_enc_type > 1*/
 					/*select the uncst_oui and remove
-					 * the other uncst_oui*/
+					 * the other uncst_oui
+					 */
 					cnt = sec_ie[12];
 					remove_cnt = (cnt - 1) * 4;
 					sec_ie[12] = 0x01;
@@ -1499,7 +1506,8 @@ sint r8712_restruct_sec_ie(struct _adapter *adapter, u8 *in_ie,
 		}
 		if (authmode == _WPA2_IE_ID_) {
 			/* parsing RSN IE to select required encryption
-			 * algorithm, and set the bc/mc encryption algorithm */
+			 * algorithm, and set the bc/mc encryption algorithm
+			 */
 			while (true) {
 				if ((sec_ie[2] != 0x01) || (sec_ie[3] != 0x0)) {
 					/*IE Ver error*/
@@ -1543,7 +1551,8 @@ sint r8712_restruct_sec_ie(struct _adapter *adapter, u8 *in_ie,
 					} /*else the uncst_oui is match*/
 				} else { /*mixed mode, unicast_enc_type > 1*/
 					/*select the uncst_oui and remove the
-					 * other uncst_oui*/
+					 * other uncst_oui
+					 */
 					cnt = sec_ie[8];
 					remove_cnt = (cnt - 1) * 4;
 					sec_ie[8] = 0x01;
@@ -1667,7 +1676,8 @@ void r8712_joinbss_reset(struct _adapter *padapter)
 	struct ht_priv		*phtpriv = &pmlmepriv->htpriv;
 
 	/* todo: if you want to do something io/reg/hw setting before join_bss,
-	 * please add code here */
+	 * please add code here
+	 */
 	phtpriv->ampdu_enable = false;/*reset to disabled*/
 	for (i = 0; i < 16; i++)
 		phtpriv->baddbareq_issued[i] = false;/*reset it*/
diff --git a/drivers/staging/rtl8712/rtl871x_mlme.h b/drivers/staging/rtl8712/rtl871x_mlme.h
index 61e0eb745d21..ddaaab058b2f 100644
--- a/drivers/staging/rtl8712/rtl871x_mlme.h
+++ b/drivers/staging/rtl8712/rtl871x_mlme.h
@@ -47,16 +47,20 @@
 #define   WIFI_ADHOC_MASTER_STATE 0x00000040
 #define   WIFI_UNDER_LINKING	0x00000080
 #define WIFI_SITE_MONITOR	0x00000800	/* to indicate the station
-						 * is under site surveying*/
+						 * is under site surveying
+						 */
 #define	WIFI_MP_STATE		0x00010000
 #define	WIFI_MP_CTX_BACKGROUND	0x00020000	/* in cont. tx background*/
 #define	WIFI_MP_CTX_ST		0x00040000	/* in cont. tx with
-						 *  single-tone*/
+						 * single-tone
+						 */
 #define	WIFI_MP_CTX_BACKGROUND_PENDING	0x00080000 /* pending in cont, tx
-					* background due to out of skb*/
+						    * background due to out of skb
+						    */
 #define	WIFI_MP_CTX_CCK_HW	0x00100000	/* in continuous tx*/
 #define	WIFI_MP_CTX_CCK_CS	0x00200000	/* in cont, tx with carrier
-						 * suppression*/
+						 * suppression
+						 */
 #define   WIFI_MP_LPBK_STATE	0x00400000
 
 #define _FW_UNDER_LINKING	WIFI_UNDER_LINKING
diff --git a/drivers/staging/rtl8712/rtl871x_mp.c b/drivers/staging/rtl8712/rtl871x_mp.c
index 5e4fda1890f5..3c10a2c848c8 100644
--- a/drivers/staging/rtl8712/rtl871x_mp.c
+++ b/drivers/staging/rtl8712/rtl871x_mp.c
@@ -376,7 +376,8 @@ void r8712_SwitchBandwidth(struct _adapter *pAdapter)
 		/* Use PHY_REG.txt default value. Do not need to change.
 		 * Correct the tx power for CCK rate in 40M.
 		 * Set Control channel to upper or lower. These settings are
-		 * required only for 40MHz */
+		 * required only for 40MHz
+		 */
 		set_bb_reg(pAdapter, rCCK0_System, bCCKSideBand,
 			   (HAL_PRIME_CHNL_OFFSET_DONT_CARE >> 1));
 		set_bb_reg(pAdapter, rOFDM1_LSTF, 0xC00,
diff --git a/drivers/staging/rtl8712/rtl871x_mp.h b/drivers/staging/rtl8712/rtl871x_mp.h
index 3f3b2e73b7d2..8df452e3e3ce 100644
--- a/drivers/staging/rtl8712/rtl871x_mp.h
+++ b/drivers/staging/rtl8712/rtl871x_mp.h
@@ -108,7 +108,8 @@ struct mp_priv {
 	unsigned char network_macaddr[6];
 	/*Testing Flag*/
 	u32 mode;/*0 for normal type packet,
-		  * 1 for loopback packet (16bytes TXCMD)*/
+		  * 1 for loopback packet (16bytes TXCMD)
+		  */
 	sint prev_fw_state;
 	u8 *pallocated_mp_xmitframe_buf;
 	u8 *pmp_xmtframe_buf;
diff --git a/drivers/staging/rtl8712/rtl871x_mp_ioctl.h b/drivers/staging/rtl8712/rtl871x_mp_ioctl.h
index 8dc898024e07..1102451a733d 100644
--- a/drivers/staging/rtl8712/rtl871x_mp_ioctl.h
+++ b/drivers/staging/rtl8712/rtl871x_mp_ioctl.h
@@ -158,28 +158,37 @@ static const struct oid_obj_priv oid_rtl_seg_81_80_00[] = {
 	{1, oid_null_function},		/*0x05	OID_RT_PRO_SET_SCRAMBLER*/
 	{1, oid_null_function},		/*0x06	OID_RT_PRO_SET_FILTER_BB*/
 	{1, oid_null_function},		/*0x07
-					 * OID_RT_PRO_SET_MANUAL_DIVERS_BB*/
+					 * OID_RT_PRO_SET_MANUAL_DIVERS_BB
+					 */
 	{1, oid_rt_pro_set_channel_direct_call_hdl},	/*0x08*/
 	{1, oid_null_function},		/*0x09
-				* OID_RT_PRO_SET_SLEEP_MODE_DIRECT_CALL*/
+				* OID_RT_PRO_SET_SLEEP_MODE_DIRECT_CALL
+				*/
 	{1, oid_null_function},		/*0x0A
-				* OID_RT_PRO_SET_WAKE_MODE_DIRECT_CALL*/
+				* OID_RT_PRO_SET_WAKE_MODE_DIRECT_CALL
+				*/
 	{1, oid_rt_pro_set_continuous_tx_hdl},	/*0x0B
-				* OID_RT_PRO_SET_TX_CONTINUOUS_DIRECT_CALL*/
+				* OID_RT_PRO_SET_TX_CONTINUOUS_DIRECT_CALL
+				*/
 	{1, oid_rt_pro_set_single_carrier_tx_hdl}, /*0x0C
-				* OID_RT_PRO_SET_SINGLE_CARRIER_TX_CONTINUOUS*/
+				* OID_RT_PRO_SET_SINGLE_CARRIER_TX_CONTINUOUS
+				*/
 	{1, oid_null_function},		/*0x0D
-				* OID_RT_PRO_SET_TX_ANTENNA_BB*/
+				* OID_RT_PRO_SET_TX_ANTENNA_BB
+				*/
 	{1, oid_rt_pro_set_antenna_bb_hdl},		/*0x0E*/
 	{1, oid_null_function},		/*0x0F	OID_RT_PRO_SET_CR_SCRAMBLER*/
 	{1, oid_null_function},		/*0x10	OID_RT_PRO_SET_CR_NEW_FILTER*/
 	{1, oid_rt_pro_set_tx_power_control_hdl}, /*0x11
-				* OID_RT_PRO_SET_TX_POWER_CONTROL*/
+				* OID_RT_PRO_SET_TX_POWER_CONTROL
+				*/
 	{1, oid_null_function},		/*0x12	OID_RT_PRO_SET_CR_TX_CONFIG*/
 	{1, oid_null_function},		/*0x13
-					 * OID_RT_PRO_GET_TX_POWER_CONTROL*/
+					 * OID_RT_PRO_GET_TX_POWER_CONTROL
+					 */
 	{1, oid_null_function},		/*0x14
-					 * OID_RT_PRO_GET_CR_SIGNAL_QUALITY*/
+					 * OID_RT_PRO_GET_CR_SIGNAL_QUALITY
+					 */
 	{1, oid_null_function},		/*0x15	OID_RT_PRO_SET_CR_SETPOINT*/
 	{1, oid_null_function},		/*0x16	OID_RT_PRO_SET_INTEGRATOR*/
 	{1, oid_null_function},		/*0x17	OID_RT_PRO_SET_SIGNAL_QUALITY*/
@@ -203,13 +212,17 @@ static const struct oid_obj_priv oid_rtl_seg_81_80_20[] = {
 	{1, oid_rt_pro_query_rx_packet_received_hdl},	/*0x26*/
 	{1, oid_rt_pro_query_rx_packet_crc32_error_hdl},/*0x27*/
 	{1, oid_null_function},		/*0x28
-					 *OID_RT_PRO_QUERY_CURRENT_ADDRESS*/
+					 *OID_RT_PRO_QUERY_CURRENT_ADDRESS
+					 */
 	{1, oid_null_function},		/*0x29
-					 *OID_RT_PRO_QUERY_PERMANENT_ADDRESS*/
+					 *OID_RT_PRO_QUERY_PERMANENT_ADDRESS
+					 */
 	{1, oid_null_function},		/*0x2A
-				 *OID_RT_PRO_SET_PHILIPS_RF_PARAMETERS*/
+				 *OID_RT_PRO_SET_PHILIPS_RF_PARAMETERS
+				 */
 	{1, oid_rt_pro_set_carrier_suppression_tx_hdl},/*0x2B
-				 *OID_RT_PRO_SET_CARRIER_SUPPRESSION_TX*/
+				 *OID_RT_PRO_SET_CARRIER_SUPPRESSION_TX
+				 */
 	{1, oid_null_function},		/*0x2C	OID_RT_PRO_RECEIVE_PACKET*/
 	{1, oid_null_function},		/*0x2D	OID_RT_PRO_WRITE_EEPROM_BYTE*/
 	{1, oid_null_function},		/*0x2E	OID_RT_PRO_READ_EEPROM_BYTE*/
diff --git a/drivers/staging/rtl8712/rtl871x_mp_phy_regdef.h b/drivers/staging/rtl8712/rtl871x_mp_phy_regdef.h
index 2e9120a21a0b..11bcfb7bf77c 100644
--- a/drivers/staging/rtl8712/rtl871x_mp_phy_regdef.h
+++ b/drivers/staging/rtl8712/rtl871x_mp_phy_regdef.h
@@ -82,7 +82,8 @@
  * 3. Page8(0x800)
  */
 #define	rFPGA0_RFMOD			0x800	/*RF mode & CCK TxSC RF
-						 * BW Setting?? */
+						 * BW Setting??
+						 */
 #define	rFPGA0_TxInfo			0x804	/* Status report?? */
 #define	rFPGA0_PSDFunction		0x808
 #define	rFPGA0_TxGainStage		0x80c	/* Set TX PWR init gain? */
@@ -119,7 +120,8 @@
 
 #define	rFPGA0_AnalogParameter1		0x880	/* Crystal cap setting
 						 * RF-R/W protection
-						 * for parameter4?? */
+						 * for parameter4??
+						 */
 #define	rFPGA0_AnalogParameter2		0x884
 #define	rFPGA0_AnalogParameter3		0x888	/* Useless now */
 #define	rFPGA0_AnalogParameter4		0x88c
@@ -146,7 +148,8 @@
  * 5. PageA(0xA00)
  *
  * Set Control channel to upper or lower.
- * These settings are required only for 40MHz */
+ * These settings are required only for 40MHz
+ */
 #define	rCCK0_System			0xa00
 
 #define	rCCK0_AFESetting		0xa04	/* Disable init gain now */
@@ -155,20 +158,23 @@
 #define	rCCK0_RxAGC1			0xa0c
 /* AGC default value, saturation level
  * Antenna Diversity, RX AGC, LNA Threshold, RX LNA Threshold useless now.
- * Not the same as 90 series */
+ * Not the same as 90 series
+ */
 #define	rCCK0_RxAGC2			0xa10	/* AGC & DAGC */
 
 #define	rCCK0_RxHP			0xa14
 
 #define	rCCK0_DSPParameter1		0xa18	/* Timing recovery & Channel
-						 * estimation threshold */
+						 * estimation threshold
+						 */
 #define	rCCK0_DSPParameter2		0xa1c	/* SQ threshold */
 
 #define	rCCK0_TxFilter1			0xa20
 #define	rCCK0_TxFilter2			0xa24
 #define	rCCK0_DebugPort			0xa28	/* debug port and Tx filter3 */
 #define	rCCK0_FalseAlarmReport		0xa2c	/* 0xa2d useless now 0xa30-a4f
-						 * channel report */
+						 * channel report
+						 */
 #define	rCCK0_TRSSIReport		0xa50
 #define	rCCK0_RxReport			0xa54   /* 0xa57 */
 #define	rCCK0_FACounterLower		0xa5c   /* 0xa5b */
@@ -193,11 +199,13 @@
 #define	rOFDM0_XDRxIQImbalance		0xc2c
 
 #define	rOFDM0_RxDetector1		0xc30  /* PD,BW & SBD DM tune
-						* init gain */
+						* init gain
+						*/
 #define	rOFDM0_RxDetector2		0xc34  /* SBD & Fame Sync. */
 #define	rOFDM0_RxDetector3		0xc38  /* Frame Sync. */
 #define	rOFDM0_RxDetector4		0xc3c  /* PD, SBD, Frame Sync &
-						* Short-GI */
+						* Short-GI
+						*/
 
 #define	rOFDM0_RxDSP			0xc40  /* Rx Sync Path */
 #define	rOFDM0_CFOandDAGC		0xc44  /* CFO & DAGC */
@@ -283,7 +291,8 @@
 #define	rTxAGC_Mcs15_Mcs12		0xe1c
 
 /* Analog- control in RX_WAIT_CCA : REG: EE0
- * [Analog- Power & Control Register] */
+ * [Analog- Power & Control Register]
+ */
 #define		rRx_Wait_CCCA		0xe70
 #define	rAnapar_Ctrl_BB			0xee0
 
@@ -371,7 +380,8 @@
 /*
  * Bit Mask
  *
- * 1. Page1(0x100) */
+ * 1. Page1(0x100)
+ */
 #define	bBBResetB			0x100	/* Useless now? */
 #define	bGlobalResetB			0x200
 #define	bOFDMTxStart			0x4
@@ -918,7 +928,8 @@
 #define	bPesudoNoiseState_D      0xffff0000
 
 /* 7. RF Register
- * Zebra1 */
+ * Zebra1
+ */
 #define	bZebra1_HSSIEnable        0x8		/* Useless */
 #define	bZebra1_TRxControl        0xc00
 #define	bZebra1_TRxGainSetting    0x07f
diff --git a/drivers/staging/rtl8712/rtl871x_pwrctrl.c b/drivers/staging/rtl8712/rtl871x_pwrctrl.c
index bf10d6db50e8..d464c136dd98 100644
--- a/drivers/staging/rtl8712/rtl871x_pwrctrl.c
+++ b/drivers/staging/rtl8712/rtl871x_pwrctrl.c
@@ -52,7 +52,8 @@ void r8712_set_rpwm(struct _adapter *padapter, u8 val8)
 		pwrpriv->cpwm = val8;
 		break;
 	case PS_STATE_S2:/* only for USB normal powersave mode use,
-			  * temp mark some code. */
+			  * temp mark some code.
+			  */
 	case PS_STATE_S3:
 	case PS_STATE_S4:
 		pwrpriv->cpwm = val8;
@@ -103,14 +104,14 @@ void r8712_cpwm_int_hdl(struct _adapter *padapter,
 	if (pwrpriv->cpwm_tog == ((preportpwrstate->state) & 0x80))
 		return;
 	del_timer(&padapter->pwrctrlpriv.rpwm_check_timer);
-	_enter_pwrlock(&pwrpriv->lock);
+	mutex_lock(&pwrpriv->mutex_lock);
 	pwrpriv->cpwm = (preportpwrstate->state) & 0xf;
 	if (pwrpriv->cpwm >= PS_STATE_S2) {
 		if (pwrpriv->alives & CMD_ALIVE)
-			up(&(pcmdpriv->cmd_queue_sema));
+			complete(&(pcmdpriv->cmd_queue_comp));
 	}
 	pwrpriv->cpwm_tog = (preportpwrstate->state) & 0x80;
-	up(&pwrpriv->lock);
+	mutex_unlock(&pwrpriv->mutex_lock);
 }
 
 static inline void register_task_alive(struct pwrctrl_priv *pwrctrl, uint tag)
@@ -141,10 +142,10 @@ static void SetPSModeWorkItemCallback(struct work_struct *work)
 	struct _adapter *padapter = container_of(pwrpriv,
 				    struct _adapter, pwrctrlpriv);
 	if (!pwrpriv->bSleep) {
-		_enter_pwrlock(&pwrpriv->lock);
+		mutex_lock(&pwrpriv->mutex_lock);
 		if (pwrpriv->pwr_mode == PS_MODE_ACTIVE)
 			r8712_set_rpwm(padapter, PS_STATE_S4);
-		up(&pwrpriv->lock);
+		mutex_unlock(&pwrpriv->mutex_lock);
 	}
 }
 
@@ -155,11 +156,11 @@ static void rpwm_workitem_callback(struct work_struct *work)
 	struct _adapter *padapter = container_of(pwrpriv,
 				    struct _adapter, pwrctrlpriv);
 	if (pwrpriv->cpwm != pwrpriv->rpwm) {
-		_enter_pwrlock(&pwrpriv->lock);
+		mutex_lock(&pwrpriv->mutex_lock);
 		r8712_read8(padapter, SDIO_HCPWM);
 		pwrpriv->rpwm_retry = 1;
 		r8712_set_rpwm(padapter, pwrpriv->rpwm);
-		up(&pwrpriv->lock);
+		mutex_unlock(&pwrpriv->mutex_lock);
 	}
 }
 
@@ -175,7 +176,7 @@ void r8712_init_pwrctrl_priv(struct _adapter *padapter)
 	struct pwrctrl_priv *pwrctrlpriv = &padapter->pwrctrlpriv;
 
 	memset((unsigned char *)pwrctrlpriv, 0, sizeof(struct pwrctrl_priv));
-	sema_init(&pwrctrlpriv->lock, 1);
+	mutex_init(&pwrctrlpriv->mutex_lock);
 	pwrctrlpriv->cpwm = PS_STATE_S4;
 	pwrctrlpriv->pwr_mode = PS_MODE_ACTIVE;
 	pwrctrlpriv->smart_ps = 0;
@@ -207,13 +208,13 @@ sint r8712_register_cmd_alive(struct _adapter *padapter)
 	uint res = _SUCCESS;
 	struct pwrctrl_priv *pwrctrl = &padapter->pwrctrlpriv;
 
-	_enter_pwrlock(&pwrctrl->lock);
+	mutex_lock(&pwrctrl->mutex_lock);
 	register_task_alive(pwrctrl, CMD_ALIVE);
 	if (pwrctrl->cpwm < PS_STATE_S2) {
 		r8712_set_rpwm(padapter, PS_STATE_S3);
 		res = _FAIL;
 	}
-	up(&pwrctrl->lock);
+	mutex_unlock(&pwrctrl->mutex_lock);
 	return res;
 }
 
@@ -229,7 +230,7 @@ void r8712_unregister_cmd_alive(struct _adapter *padapter)
 {
 	struct pwrctrl_priv *pwrctrl = &padapter->pwrctrlpriv;
 
-	_enter_pwrlock(&pwrctrl->lock);
+	mutex_lock(&pwrctrl->mutex_lock);
 	unregister_task_alive(pwrctrl, CMD_ALIVE);
 	if ((pwrctrl->cpwm > PS_STATE_S2) &&
 	   (pwrctrl->pwr_mode > PS_MODE_ACTIVE)) {
@@ -239,5 +240,5 @@ void r8712_unregister_cmd_alive(struct _adapter *padapter)
 			r8712_set_rpwm(padapter, PS_STATE_S0);
 		}
 	}
-	up(&pwrctrl->lock);
+	mutex_unlock(&pwrctrl->mutex_lock);
 }
diff --git a/drivers/staging/rtl8712/rtl871x_pwrctrl.h b/drivers/staging/rtl8712/rtl871x_pwrctrl.h
index dbfb55523545..c82fdf85d474 100644
--- a/drivers/staging/rtl8712/rtl871x_pwrctrl.h
+++ b/drivers/staging/rtl8712/rtl871x_pwrctrl.h
@@ -87,16 +87,12 @@ struct reportpwrstate_parm {
 	unsigned short rsvd;
 };
 
-static inline void _enter_pwrlock(struct semaphore *plock)
-{
-	_down_sema(plock);
-}
-
 struct	pwrctrl_priv {
-	struct semaphore lock;
+	struct mutex mutex_lock;
 	/*volatile*/ u8 rpwm; /* requested power state for fw */
 	/* fw current power state. updated when 1. read from HCPWM or
-	 * 2. driver lowers power level */
+	 * 2. driver lowers power level
+	 */
 	/*volatile*/ u8 cpwm;
 	/*volatile*/ u8 tog; /* toggling */
 	/*volatile*/ u8 cpwm_tog; /* toggling */
diff --git a/drivers/staging/rtl8712/rtl871x_recv.c b/drivers/staging/rtl8712/rtl871x_recv.c
index 23c143890252..cbd2e51ba42b 100644
--- a/drivers/staging/rtl8712/rtl871x_recv.c
+++ b/drivers/staging/rtl8712/rtl871x_recv.c
@@ -265,7 +265,8 @@ union recv_frame *r8712_portctrl(struct _adapter *adapter,
 
 		if ((psta != NULL) && (psta->ieee8021x_blocked)) {
 			/* blocked
-			 * only accept EAPOL frame */
+			 * only accept EAPOL frame
+			 */
 			if (ether_type == 0x888e) {
 				prtnframe = precv_frame;
 			} else {
@@ -277,7 +278,8 @@ union recv_frame *r8712_portctrl(struct _adapter *adapter,
 		} else {
 			/* allowed
 			 * check decryption status, and decrypt the
-			 * frame if needed */
+			 * frame if needed
+			 */
 			prtnframe = precv_frame;
 			/* check is the EAPOL frame or not (Rekey) */
 			if (ether_type == 0x888e) {
@@ -334,19 +336,22 @@ static sint sta2sta_data_frame(struct _adapter *adapter,
 		sta_addr = pattrib->src;
 	} else if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
 		/* For Station mode, sa and bssid should always be BSSID,
-		 * and DA is my mac-address */
+		 * and DA is my mac-address
+		 */
 		if (memcmp(pattrib->bssid, pattrib->src, ETH_ALEN))
 			return _FAIL;
 	       sta_addr = pattrib->bssid;
 	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 		if (bmcast) {
 			/* For AP mode, if DA == MCAST, then BSSID should
-			 * be also MCAST */
+			 * be also MCAST
+			 */
 			if (!IS_MCAST(pattrib->bssid))
 				return _FAIL;
 		} else { /* not mc-frame */
 			/* For AP mode, if DA is non-MCAST, then it must be
-			 *  BSSID, and bssid == BSSID */
+			 * BSSID, and bssid == BSSID
+			 */
 			if (memcmp(pattrib->bssid, pattrib->dst, ETH_ALEN))
 				return _FAIL;
 			sta_addr = pattrib->src;
@@ -391,7 +396,8 @@ static sint ap2sta_data_frame(struct _adapter *adapter,
 		if ((GetFrameSubType(ptr)) == WIFI_DATA_NULL)
 			return _FAIL;
 		/* drop QoS-SubType Data, including QoS NULL,
-		 * excluding QoS-Data */
+		 * excluding QoS-Data
+		 */
 		if ((GetFrameSubType(ptr) & WIFI_QOS_DATA_TYPE) ==
 		     WIFI_QOS_DATA_TYPE) {
 			if (GetFrameSubType(ptr) & (BIT(4) | BIT(5) | BIT(6)))
@@ -445,7 +451,8 @@ static sint sta2ap_data_frame(struct _adapter *adapter,
 	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 		/* For AP mode, if DA is non-MCAST, then it must be BSSID,
 		 * and bssid == BSSID
-		 * For AP mode, RA=BSSID, TX=STA(SRC_ADDR), A3=DST_ADDR */
+		 * For AP mode, RA=BSSID, TX=STA(SRC_ADDR), A3=DST_ADDR
+		 */
 		if (memcmp(pattrib->bssid, mybssid, ETH_ALEN))
 			return _FAIL;
 		*psta = r8712_get_stainfo(pstapriv, pattrib->src);
@@ -619,7 +626,8 @@ sint r8712_wlanhdr_to_ethhdr(union recv_frame *precvframe)
 	    (memcmp(psnap_type, (void *)SNAP_ETH_TYPE_APPLETALK_AARP, 2))) ||
 	     !memcmp(psnap, (void *)bridge_tunnel_header, SNAP_SIZE)) {
 		/* remove RFC1042 or Bridge-Tunnel encapsulation and
-		 * replace EtherType */
+		 * replace EtherType
+		 */
 		bsnaphdr = true;
 	} else {
 		/* Leave Ethernet header part of hdr and full payload */
diff --git a/drivers/staging/rtl8712/rtl871x_recv.h b/drivers/staging/rtl8712/rtl871x_recv.h
index 77487bb9d3c0..f419943ad75d 100644
--- a/drivers/staging/rtl8712/rtl871x_recv.h
+++ b/drivers/staging/rtl8712/rtl871x_recv.h
@@ -153,7 +153,8 @@ static inline u8 *get_recvframe_data(union recv_frame *precvframe)
 static inline u8 *recvframe_pull(union recv_frame *precvframe, sint sz)
 {
 	/* used for extract sz bytes from rx_data, update rx_data and return
-	 *  the updated rx_data to the caller */
+	 * the updated rx_data to the caller
+	 */
 	if (precvframe == NULL)
 		return NULL;
 	precvframe->u.hdr.rx_data += sz;
@@ -169,7 +170,8 @@ static inline u8 *recvframe_put(union recv_frame *precvframe, sint sz)
 {
 	/* used for append sz bytes from ptr to rx_tail, update rx_tail and
 	 * return the updated rx_tail to the caller
-	 * after putting, rx_tail must be still larger than rx_end. */
+	 * after putting, rx_tail must be still larger than rx_end.
+	 */
 	if (precvframe == NULL)
 		return NULL;
 	precvframe->u.hdr.rx_tail += sz;
@@ -186,7 +188,8 @@ static inline u8 *recvframe_pull_tail(union recv_frame *precvframe, sint sz)
 	/* rmv data from rx_tail (by yitsen)
 	 * used for extract sz bytes from rx_end, update rx_end and return the
 	 * updated rx_end to the caller
-	 * after pulling, rx_end must be still larger than rx_data. */
+	 * after pulling, rx_end must be still larger than rx_data.
+	 */
 	if (precvframe == NULL)
 		return NULL;
 	precvframe->u.hdr.rx_tail -= sz;
diff --git a/drivers/staging/rtl8712/rtl871x_security.h b/drivers/staging/rtl8712/rtl871x_security.h
index 2295f0e64dc2..fa952e17975b 100644
--- a/drivers/staging/rtl8712/rtl871x_security.h
+++ b/drivers/staging/rtl8712/rtl871x_security.h
@@ -90,18 +90,23 @@ struct RT_PMKID_LIST {
 
 struct security_priv {
 	u32 AuthAlgrthm;		/* 802.11 auth, could be open, shared,
-					 * 8021x and authswitch */
+					 * 8021x and authswitch
+					 */
 	u32 PrivacyAlgrthm;		/* This specify the privacy for shared
-					 * auth. algorithm. */
+					 * auth. algorithm.
+					 */
 	u32 PrivacyKeyIndex;		/* this is only valid for legendary
-					 * wep, 0~3 for key id. */
+					 * wep, 0~3 for key id.
+					 */
 	union Keytype DefKey[4];	/* this is only valid for def. key */
 	u32 DefKeylen[4];
 	u32 XGrpPrivacy;		/* This specify the privacy algthm.
-					 * used for Grp key */
+					 * used for Grp key
+					 */
 	u32 XGrpKeyid;			/* key id used for Grp Key */
 	union Keytype	XGrpKey[2];	/* 802.1x Group Key, for
-					 * inx0 and inx1 */
+					 * inx0 and inx1
+					 */
 	union Keytype	XGrptxmickey[2];
 	union Keytype	XGrprxmickey[2];
 	union pn48 Grptxpn;		/* PN48 used for Grp Key xmit. */
@@ -118,9 +123,11 @@ struct security_priv {
 	s32	sw_encrypt;	/* from registry_priv */
 	s32	sw_decrypt;	/* from registry_priv */
 	s32	hw_decrypted;	/* if the rx packets is hw_decrypted==false,
-				 * it means the hw has not been ready. */
+				 * it means the hw has not been ready.
+				 */
 	u32 ndisauthtype;	/* keeps the auth_type & enc_status from upper
-				 * layer ioctl(wpa_supplicant or wzc) */
+				 * layer ioctl(wpa_supplicant or wzc)
+				 */
 	u32 ndisencryptstatus;
 	struct wlan_bssid_ex sec_bss;  /* for joinbss (h2c buffer) usage */
 	struct NDIS_802_11_WEP ndiswep;
@@ -136,7 +143,8 @@ struct security_priv {
 	u32 btkip_countermeasure_time;
 	/*-------------------------------------------------------------------
 	 * For WPA2 Pre-Authentication.
-	 *------------------------------------------------------------------ */
+	 *------------------------------------------------------------------
+	 **/
 	struct RT_PMKID_LIST		PMKIDList[NUM_PMKID_CACHE];
 	u8				PMKIDIndex;
 };
diff --git a/drivers/staging/rtl8712/rtl871x_sta_mgt.c b/drivers/staging/rtl8712/rtl871x_sta_mgt.c
index e11ce2896893..e2d75e4c473f 100644
--- a/drivers/staging/rtl8712/rtl871x_sta_mgt.c
+++ b/drivers/staging/rtl8712/rtl871x_sta_mgt.c
@@ -188,7 +188,8 @@ void r8712_free_stainfo(struct _adapter *padapter, struct sta_info *psta)
 	_r8712_init_sta_xmit_priv(&psta->sta_xmitpriv);
 	_r8712_init_sta_recv_priv(&psta->sta_recvpriv);
 	/* for A-MPDU Rx reordering buffer control,
-	 * cancel reordering_ctrl_timer */
+	 * cancel reordering_ctrl_timer
+	 */
 	for (i = 0; i < 16; i++) {
 		preorder_ctrl = &psta->recvreorder_ctrl[i];
 		del_timer(&preorder_ctrl->reordering_ctrl_timer);
diff --git a/drivers/staging/rtl8712/rtl871x_xmit.c b/drivers/staging/rtl8712/rtl871x_xmit.c
index 99256baafd38..be38364c8a7c 100644
--- a/drivers/staging/rtl8712/rtl871x_xmit.c
+++ b/drivers/staging/rtl8712/rtl871x_xmit.c
@@ -204,7 +204,8 @@ sint r8712_update_attrib(struct _adapter *padapter, _pkt *pkt,
 
 {
 	/*If driver xmit ARP packet, driver can set ps mode to initial
-	 * setting. It stands for getting DHCP or fix IP.*/
+	 * setting. It stands for getting DHCP or fix IP.
+	 */
 	if (pattrib->ether_type == 0x0806) {
 		if (padapter->pwrctrlpriv.pwr_mode !=
 		    padapter->registrypriv.power_mgnt) {
@@ -232,7 +233,8 @@ sint r8712_update_attrib(struct _adapter *padapter, _pkt *pkt,
 		if (pattrib->ether_type != 0x8712)
 			return _FAIL;
 		/* for mp storing the txcmd per packet,
-		 * according to the info of txcmd to update pattrib */
+		 * according to the info of txcmd to update pattrib
+		 */
 		/*get MP_TXDESC_SIZE bytes txcmd per packet*/
 		_r8712_pktfile_read(&pktfile, (u8 *)&txdesc, TXDESC_SIZE);
 		memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
@@ -244,7 +246,8 @@ sint r8712_update_attrib(struct _adapter *padapter, _pkt *pkt,
 	if (pattrib->ether_type == ETH_P_IP) {
 		/* The following is for DHCP and ARP packet, we use cck1M to
 		 * tx these packets and let LPS awake some time
-		 * to prevent DHCP protocol fail */
+		 * to prevent DHCP protocol fail
+		 */
 		u8 tmp[24];
 
 		_r8712_pktfile_read(&pktfile, &tmp[0], 24);
@@ -255,7 +258,8 @@ sint r8712_update_attrib(struct _adapter *padapter, _pkt *pkt,
 					((tmp[21] == 67) && (tmp[23] == 68))) {
 					/* 68 : UDP BOOTP client
 					 * 67 : UDP BOOTP server
-					 * Use low rate to send DHCP packet.*/
+					 * Use low rate to send DHCP packet.
+					 */
 					pattrib->dhcp_pkt = 1;
 				}
 			}
@@ -337,7 +341,8 @@ sint r8712_update_attrib(struct _adapter *padapter, _pkt *pkt,
 	else
 		pattrib->bswenc = false;
 	/* if in MP_STATE, update pkt_attrib from mp_txcmd, and overwrite
-	 * some settings above.*/
+	 * some settings above.
+	 */
 	if (check_fwstate(pmlmepriv, WIFI_MP_STATE))
 		pattrib->priority = (txdesc.txdw1 >> QSEL_SHT) & 0x1f;
 	return _SUCCESS;
@@ -438,7 +443,8 @@ static sint xmitframe_addmic(struct _adapter *padapter,
 			}
 			r8712_secgetmic(&micdata, &(mic[0]));
 			/* add mic code  and add the mic code length in
-			 * last_txcmdsz */
+			 * last_txcmdsz
+			 */
 			memcpy(payload, &(mic[0]), 8);
 			pattrib->last_txcmdsz += 8;
 			payload = payload - pattrib->last_txcmdsz + 8;
diff --git a/drivers/staging/rtl8712/rtl871x_xmit.h b/drivers/staging/rtl8712/rtl871x_xmit.h
index a9633c3f73d0..d899d0c6d3a6 100644
--- a/drivers/staging/rtl8712/rtl871x_xmit.h
+++ b/drivers/staging/rtl8712/rtl871x_xmit.h
@@ -58,7 +58,8 @@ do { \
 } while (0)
 
 /* Fixed the Big Endian bug when doing the Tx.
- * The Linksys WRH54G will check this.*/
+ * The Linksys WRH54G will check this.
+ */
 #define TKIP_IV(pattrib_iv, txpn, keyidx)\
 do { \
 	pattrib_iv[0] = txpn._byte_.TSC1;\
@@ -105,7 +106,8 @@ struct pkt_attrib {
 	u16	seqnum;
 	u16	ether_type;
 	u16	pktlen;		/* the original 802.3 pkt raw_data len
-				 * (not include ether_hdr data) */
+				 * (not include ether_hdr data)
+				 */
 	u16	last_txcmdsz;
 
 	u8	pkt_hdrlen;	/*the original 802.3 pkt header len*/
@@ -119,7 +121,8 @@ struct pkt_attrib {
 
 	u8	priority;
 	u8	encrypt;	/* when 0 indicate no encrypt. when non-zero,
-				 * indicate the encrypt algorithm*/
+				 * indicate the encrypt algorithm
+				 */
 	u8	iv_len;
 	u8	icv_len;
 	unsigned char iv[8];
@@ -176,7 +179,8 @@ struct sta_xmit_priv {
 	spinlock_t lock;
 	sint	option;
 	sint	apsd_setting;	/* When bit mask is on, the associated edca
-				 * queue supports APSD.*/
+				 * queue supports APSD.
+				 */
 	struct tx_servq	be_q;	/* priority == 0,3 */
 	struct tx_servq	bk_q;	/* priority == 1,2*/
 	struct tx_servq	vi_q;	/*priority == 4,5*/
diff --git a/drivers/staging/rtl8712/usb_halinit.c b/drivers/staging/rtl8712/usb_halinit.c
index ad21df16c2bd..0b159850f5a2 100644
--- a/drivers/staging/rtl8712/usb_halinit.c
+++ b/drivers/staging/rtl8712/usb_halinit.c
@@ -196,7 +196,8 @@ u8 r8712_usb_hal_bus_init(struct _adapter *padapter)
 		msleep(20);
 		/* Revised POS, */
 		/* Enable AFE Macro Block's Bandgap and Enable AFE Macro
-		 * Block's Mbias */
+		 * Block's Mbias
+		 */
 		r8712_write8(padapter, SPS0_CTRL + 1, 0x53);
 		r8712_write8(padapter, SPS0_CTRL, 0x57);
 		val8 = r8712_read8(padapter, AFE_MISC);
diff --git a/drivers/staging/rtl8712/usb_intf.c b/drivers/staging/rtl8712/usb_intf.c
index c1a0ca490546..897d4621a5ce 100644
--- a/drivers/staging/rtl8712/usb_intf.c
+++ b/drivers/staging/rtl8712/usb_intf.c
@@ -301,7 +301,8 @@ void rtl871x_intf_stop(struct _adapter *padapter)
 	/*disable_hw_interrupt*/
 	if (!padapter->bSurpriseRemoved) {
 		/*device still exists, so driver can do i/o operation
-		 * TODO: */
+		 * TODO:
+		 */
 	}
 
 	/* cancel in irp */
@@ -368,7 +369,7 @@ static const struct device_type wlan_type = {
  *
  * notes: drv_init() is called when the bus driver has located a card for us
  * to support. We accept the new device by returning 0.
-*/
+ */
 static int r871xu_drv_init(struct usb_interface *pusb_intf,
 			   const struct usb_device_id *pdid)
 {
@@ -611,7 +612,8 @@ error:
 }
 
 /* rmmod module & unplug(SurpriseRemoved) will call r871xu_dev_remove()
- * => how to recognize both */
+ * => how to recognize both
+ */
 static void r871xu_dev_remove(struct usb_interface *pusb_intf)
 {
 	struct net_device *pnetdev = usb_get_intfdata(pusb_intf);
@@ -635,12 +637,14 @@ static void r871xu_dev_remove(struct usb_interface *pusb_intf)
 		r8712_free_drv_sw(padapter);
 
 		/* decrease the reference count of the usb device structure
-		 * when disconnect */
+		 * when disconnect
+		 */
 		usb_put_dev(udev);
 	}
 	/* If we didn't unplug usb dongle and remove/insert module, driver
 	 * fails on sitesurvey for the first time when device is up.
-	 * Reset usb port for sitesurvey fail issue. */
+	 * Reset usb port for sitesurvey fail issue.
+	 */
 	if (udev->state != USB_STATE_NOTATTACHED)
 		usb_reset_device(udev);
 }
diff --git a/drivers/staging/rtl8712/usb_ops_linux.c b/drivers/staging/rtl8712/usb_ops_linux.c
index 6f12345709c2..fc6bb0be2a28 100644
--- a/drivers/staging/rtl8712/usb_ops_linux.c
+++ b/drivers/staging/rtl8712/usb_ops_linux.c
@@ -50,7 +50,7 @@ uint r8712_usb_init_intf_priv(struct intf_priv *pintfpriv)
 	pintfpriv->piorw_urb = usb_alloc_urb(0, GFP_ATOMIC);
 	if (!pintfpriv->piorw_urb)
 		return _FAIL;
-	sema_init(&(pintfpriv->io_retevt), 0);
+	init_completion(&pintfpriv->io_retevt_comp);
 	return _SUCCESS;
 }
 
@@ -163,7 +163,7 @@ static void usb_write_mem_complete(struct urb *purb)
 		else
 			padapter->bSurpriseRemoved = true;
 	}
-	up(&pintfpriv->io_retevt);
+	complete(&pintfpriv->io_retevt_comp);
 }
 
 void r8712_usb_write_mem(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *wmem)
@@ -187,7 +187,7 @@ void r8712_usb_write_mem(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *wmem)
 			  wmem, cnt, usb_write_mem_complete,
 			  pio_queue);
 	usb_submit_urb(piorw_urb, GFP_ATOMIC);
-	_down_sema(&pintfpriv->io_retevt);
+	wait_for_completion_interruptible(&pintfpriv->io_retevt_comp);
 }
 
 static void r8712_usb_read_port_complete(struct urb *purb)
@@ -202,26 +202,23 @@ static void r8712_usb_read_port_complete(struct urb *purb)
 	if (purb->status == 0) { /* SUCCESS */
 		if ((purb->actual_length > (MAX_RECVBUF_SZ)) ||
 		    (purb->actual_length < RXDESC_SIZE)) {
-			precvbuf->reuse = true;
 			r8712_read_port(padapter, precvpriv->ff_hwaddr, 0,
 				  (unsigned char *)precvbuf);
 		} else {
+			_pkt *pskb = precvbuf->pskb;
+
 			precvbuf->transfer_len = purb->actual_length;
 			pbuf = (uint *)precvbuf->pbuf;
 			isevt = le32_to_cpu(*(pbuf + 1)) & 0x1ff;
 			if ((isevt & 0x1ff) == 0x1ff) {
 				r8712_rxcmd_event_hdl(padapter, pbuf);
-				precvbuf->reuse = true;
+				skb_queue_tail(&precvpriv->rx_skb_queue, pskb);
 				r8712_read_port(padapter, precvpriv->ff_hwaddr,
 						0, (unsigned char *)precvbuf);
 			} else {
-				_pkt *pskb = precvbuf->pskb;
-
 				skb_put(pskb, purb->actual_length);
 				skb_queue_tail(&precvpriv->rx_skb_queue, pskb);
 				tasklet_hi_schedule(&precvpriv->recv_tasklet);
-				precvbuf->pskb = NULL;
-				precvbuf->reuse = false;
 				r8712_read_port(padapter, precvpriv->ff_hwaddr,
 						0, (unsigned char *)precvbuf);
 			}
@@ -241,7 +238,6 @@ static void r8712_usb_read_port_complete(struct urb *purb)
 			}
 			/* Fall through. */
 		case -EPROTO:
-			precvbuf->reuse = true;
 			r8712_read_port(padapter, precvpriv->ff_hwaddr, 0,
 				  (unsigned char *)precvbuf);
 			break;
@@ -270,51 +266,43 @@ u32 r8712_usb_read_port(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *rmem)
 	struct usb_device *pusbd = pdvobj->pusbdev;
 
 	if (adapter->bDriverStopped || adapter->bSurpriseRemoved ||
-	    adapter->pwrctrlpriv.pnp_bstop_trx)
+	    adapter->pwrctrlpriv.pnp_bstop_trx || !precvbuf)
 		return _FAIL;
-	if (precvbuf->reuse || !precvbuf->pskb) {
-		precvbuf->pskb = skb_dequeue(&precvpriv->free_recv_skb_queue);
-		if (precvbuf->pskb != NULL)
-			precvbuf->reuse = true;
+	r8712_init_recvbuf(adapter, precvbuf);
+	/* Try to use skb from the free queue */
+	precvbuf->pskb = skb_dequeue(&precvpriv->free_recv_skb_queue);
+
+	if (!precvbuf->pskb) {
+		precvbuf->pskb = netdev_alloc_skb(adapter->pnetdev,
+				 MAX_RECVBUF_SZ + RECVBUFF_ALIGN_SZ);
+		if (!precvbuf->pskb)
+			return _FAIL;
+		tmpaddr = (addr_t)precvbuf->pskb->data;
+		alignment = tmpaddr & (RECVBUFF_ALIGN_SZ - 1);
+		skb_reserve(precvbuf->pskb,
+			    (RECVBUFF_ALIGN_SZ - alignment));
+		precvbuf->phead = precvbuf->pskb->head;
+		precvbuf->pdata = precvbuf->pskb->data;
+		precvbuf->ptail = skb_tail_pointer(precvbuf->pskb);
+		precvbuf->pend = skb_end_pointer(precvbuf->pskb);
+		precvbuf->pbuf = precvbuf->pskb->data;
+	} else { /* skb is reused */
+		precvbuf->phead = precvbuf->pskb->head;
+		precvbuf->pdata = precvbuf->pskb->data;
+		precvbuf->ptail = skb_tail_pointer(precvbuf->pskb);
+		precvbuf->pend = skb_end_pointer(precvbuf->pskb);
+		precvbuf->pbuf = precvbuf->pskb->data;
 	}
-	if (precvbuf != NULL) {
-		r8712_init_recvbuf(adapter, precvbuf);
-		/* re-assign for linux based on skb */
-		if (!precvbuf->reuse || !precvbuf->pskb) {
-			precvbuf->pskb = netdev_alloc_skb(adapter->pnetdev,
-					 MAX_RECVBUF_SZ + RECVBUFF_ALIGN_SZ);
-			if (!precvbuf->pskb)
-				return _FAIL;
-			tmpaddr = (addr_t)precvbuf->pskb->data;
-			alignment = tmpaddr & (RECVBUFF_ALIGN_SZ - 1);
-			skb_reserve(precvbuf->pskb,
-				    (RECVBUFF_ALIGN_SZ - alignment));
-			precvbuf->phead = precvbuf->pskb->head;
-			precvbuf->pdata = precvbuf->pskb->data;
-			precvbuf->ptail = skb_tail_pointer(precvbuf->pskb);
-			precvbuf->pend = skb_end_pointer(precvbuf->pskb);
-			precvbuf->pbuf = precvbuf->pskb->data;
-		} else { /* reuse skb */
-			precvbuf->phead = precvbuf->pskb->head;
-			precvbuf->pdata = precvbuf->pskb->data;
-			precvbuf->ptail = skb_tail_pointer(precvbuf->pskb);
-			precvbuf->pend = skb_end_pointer(precvbuf->pskb);
-			precvbuf->pbuf = precvbuf->pskb->data;
-			precvbuf->reuse = false;
-		}
-		purb = precvbuf->purb;
-		/* translate DMA FIFO addr to pipehandle */
-		pipe = ffaddr2pipehdl(pdvobj, addr);
-		usb_fill_bulk_urb(purb, pusbd, pipe,
-				  precvbuf->pbuf, MAX_RECVBUF_SZ,
-				  r8712_usb_read_port_complete,
-				  precvbuf);
-		err = usb_submit_urb(purb, GFP_ATOMIC);
-		if ((err) && (err != (-EPERM)))
-			ret = _FAIL;
-	} else {
+	purb = precvbuf->purb;
+	/* translate DMA FIFO addr to pipehandle */
+	pipe = ffaddr2pipehdl(pdvobj, addr);
+	usb_fill_bulk_urb(purb, pusbd, pipe,
+			  precvbuf->pbuf, MAX_RECVBUF_SZ,
+			  r8712_usb_read_port_complete,
+			  precvbuf);
+	err = usb_submit_urb(purb, GFP_ATOMIC);
+	if ((err) && (err != (-EPERM)))
 		ret = _FAIL;
-	}
 	return ret;
 }
 
diff --git a/drivers/staging/rtl8712/wifi.h b/drivers/staging/rtl8712/wifi.h
index 7a352c45344f..b8af9656e6da 100644
--- a/drivers/staging/rtl8712/wifi.h
+++ b/drivers/staging/rtl8712/wifi.h
@@ -376,7 +376,8 @@ static inline unsigned char *get_hdr_bssid(unsigned char *pframe)
 
 /*-----------------------------------------------------------------------------
 			Below is for the security related definition
-------------------------------------------------------------------------------*/
+ *-----------------------------------------------------------------------------
+ */
 #define _RESERVED_FRAME_TYPE_	0
 #define _SKB_FRAME_TYPE_	2
 #define _PRE_ALLOCMEM_		1
@@ -420,7 +421,8 @@ static inline unsigned char *get_hdr_bssid(unsigned char *pframe)
 
 /* ---------------------------------------------------------------------------
 					Below is the fixed elements...
------------------------------------------------------------------------------*/
+ * ---------------------------------------------------------------------------
+ */
 #define _AUTH_ALGM_NUM_			2
 #define _AUTH_SEQ_NUM_			2
 #define _BEACON_ITERVAL_		2
@@ -448,20 +450,23 @@ static inline unsigned char *get_hdr_bssid(unsigned char *pframe)
 
 /*-----------------------------------------------------------------------------
 				Below is the definition for 802.11i / 802.1x
-------------------------------------------------------------------------------*/
+ *------------------------------------------------------------------------------
+ */
 #define _IEEE8021X_MGT_			1	/*WPA */
 #define _IEEE8021X_PSK_			2	/* WPA with pre-shared key */
 
 /*-----------------------------------------------------------------------------
 				Below is the definition for WMM
-------------------------------------------------------------------------------*/
+ *------------------------------------------------------------------------------
+ */
 #define _WMM_IE_Length_				7  /* for WMM STA */
 #define _WMM_Para_Element_Length_		24
 
 
 /*-----------------------------------------------------------------------------
 				Below is the definition for 802.11n
-------------------------------------------------------------------------------*/
+ *------------------------------------------------------------------------------
+ */
 
 /* block-ack parameters */
 #define IEEE80211_ADDBA_PARAM_POLICY_MASK 0x0002
diff --git a/drivers/staging/rtl8712/wlan_bssdef.h b/drivers/staging/rtl8712/wlan_bssdef.h
index fda5707c4acd..86a88b493a43 100644
--- a/drivers/staging/rtl8712/wlan_bssdef.h
+++ b/drivers/staging/rtl8712/wlan_bssdef.h
@@ -171,7 +171,8 @@ struct NDIS_802_11_REMOVE_KEY {
 struct NDIS_802_11_WEP {
 	u32 Length;		  /* Length of this structure */
 	u32 KeyIndex;		  /* 0 is the per-client key,
-				   * 1-N are the global keys */
+				   * 1-N are the global keys
+				   */
 	u32 KeyLength;		  /* length of key in bytes */
 	u8  KeyMaterial[16];      /* variable length depending on above field */
 };
@@ -194,7 +195,8 @@ struct	wlan_network {
 	struct list_head list;
 	int	network_type;	/*refer to ieee80211.h for WIRELESS_11A/B/G */
 	int	fixed;		/* set to fixed when not to be removed asi
-				 * site-surveying */
+				 * site-surveying
+				 */
 	unsigned int	last_scanned; /*timestamp for the network */
 	int	aid;		/*will only be valid when a BSS is joined. */
 	int	join_res;
diff --git a/drivers/staging/rtl8712/xmit_linux.c b/drivers/staging/rtl8712/xmit_linux.c
index 695f9b9fc749..4ee4136b5c28 100644
--- a/drivers/staging/rtl8712/xmit_linux.c
+++ b/drivers/staging/rtl8712/xmit_linux.c
@@ -31,6 +31,7 @@
 #include <linux/usb.h>
 #include <linux/ip.h>
 #include <linux/if_ether.h>
+#include <linux/kmemleak.h>
 
 #include "osdep_service.h"
 #include "drv_types.h"
@@ -91,7 +92,8 @@ void r8712_set_qos(struct pkt_file *ppktfile, struct pkt_attrib *pattrib)
 	} else {
 		/* "When priority processing of data frames is supported,
 		 * a STA's SME should send EAPOL-Key frames at the highest
-		 * priority." */
+		 * priority."
+		 */
 
 		if (pattrib->ether_type == 0x888e)
 			UserPriority = 7;
@@ -132,6 +134,7 @@ int r8712_xmit_resource_alloc(struct _adapter *padapter,
 			netdev_err(padapter->pnetdev, "pxmitbuf->pxmit_urb[i] == NULL\n");
 			return _FAIL;
 		}
+		kmemleak_not_leak(pxmitbuf->pxmit_urb[i]);
 	}
 	return _SUCCESS;
 }
@@ -162,16 +165,16 @@ int r8712_xmit_entry(_pkt *pkt, struct  net_device *pnetdev)
 	struct _adapter *padapter = netdev_priv(pnetdev);
 	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
 
-	if (!r8712_if_up(padapter)) {
+	if (!r8712_if_up(padapter))
 		goto _xmit_entry_drop;
-	}
+
 	pxmitframe = r8712_alloc_xmitframe(pxmitpriv);
-	if (!pxmitframe) {
+	if (!pxmitframe)
 		goto _xmit_entry_drop;
-	}
-	if ((!r8712_update_attrib(padapter, pkt, &pxmitframe->attrib))) {
+
+	if ((!r8712_update_attrib(padapter, pkt, &pxmitframe->attrib)))
 		goto _xmit_entry_drop;
-	}
+
 	padapter->ledpriv.LedControlHandler(padapter, LED_CTL_TX);
 	pxmitframe->pkt = pkt;
 	if (r8712_pre_xmit(padapter, pxmitframe)) {
diff --git a/drivers/staging/rtl8723au/Kconfig b/drivers/staging/rtl8723au/Kconfig
deleted file mode 100644
index 277c1ab69317..000000000000
--- a/drivers/staging/rtl8723au/Kconfig
+++ /dev/null
@@ -1,33 +0,0 @@
-config R8723AU
-	tristate "Realtek RTL8723AU Wireless LAN NIC driver (deprecated)"
-	depends on USB && WLAN && RFKILL
-	select WIRELESS_EXT
-	select WEXT_PRIV
-	select CFG80211
-	default n
-	---help---
-	This option adds the Realtek RTL8723AU USB device such as found in
-	the Lenovo Yoga 13 tablet. If built as a module, it will be called r8723au.
-
-	Note: This driver is deprecated and scheduled to be removed in a
-	future kernel release. Please use rtl8xxxu instead.
-
-if R8723AU
-
-config 8723AU_AP_MODE
-	bool "Realtek RTL8723AU AP mode"
-	default y
-	---help---
-	This option enables Access Point mode. Unless you know that your system
-	will never be used as an AP, or the target system has limited memory,
-	"Y" should be selected.
-
-config 8723AU_BT_COEXIST
-	bool "Realtek RTL8723AU BlueTooth Coexistence"
-	default y
-	---help---
-	This option enables icoexistence with BlueTooth communications for the r8723au driver.
-	Unless you know that this driver will never by used with BT, or the target system has
-	limited memory, "Y" should be selected.
-
-endif
diff --git a/drivers/staging/rtl8723au/Makefile b/drivers/staging/rtl8723au/Makefile
deleted file mode 100644
index 3e8989018a88..000000000000
--- a/drivers/staging/rtl8723au/Makefile
+++ /dev/null
@@ -1,53 +0,0 @@
-r8723au-y :=				\
-		core/rtw_cmd.o		\
-		core/rtw_efuse.o	\
-		core/rtw_ieee80211.o	\
-		core/rtw_mlme.o		\
-		core/rtw_mlme_ext.o	\
-		core/rtw_pwrctrl.o	\
-		core/rtw_recv.o		\
-		core/rtw_security.o	\
-		core/rtw_sreset.o	\
-		core/rtw_sta_mgt.o	\
-		core/rtw_xmit.o		\
-		core/rtw_wlan_util.o	\
-		hal/hal_com.o		\
-		hal/hal_intf.o		\
-		hal/Hal8723PwrSeq.o	\
-		hal/Hal8723UHWImg_CE.o	\
-		hal/HalDMOutSrc8723A_CE.o \
-		hal/HalHWImg8723A_BB.o	\
-		hal/HalHWImg8723A_MAC.o	\
-		hal/HalHWImg8723A_RF.o	\
-		hal/HalPwrSeqCmd.o	\
-		hal/odm_RegConfig8723A.o \
-		hal/odm_debug.o		\
-		hal/odm_interface.o	\
-		hal/odm_HWConfig.o	\
-		hal/odm.o		\
-		hal/rtl8723a_cmd.o	\
-		hal/rtl8723a_dm.o	\
-		hal/rtl8723a_hal_init.o	\
-		hal/rtl8723a_phycfg.o	\
-		hal/rtl8723a_rf6052.o	\
-		hal/rtl8723a_rxdesc.o	\
-		hal/rtl8723a_sreset.o	\
-		hal/rtl8723au_recv.o	\
-		hal/rtl8723au_xmit.o	\
-		hal/usb_halinit.o	\
-		hal/usb_ops_linux.o	\
-		os_dep/ioctl_cfg80211.o	\
-		os_dep/mlme_linux.o	\
-		os_dep/os_intfs.o	\
-		os_dep/recv_linux.o	\
-		os_dep/usb_intf.o	\
-		os_dep/usb_ops_linux.o	\
-		os_dep/xmit_linux.o
-
-r8723au-$(CONFIG_8723AU_BT_COEXIST) += hal/rtl8723a_bt-coexist.o
-r8723au-$(CONFIG_8723AU_AP_MODE) += core/rtw_ap.o
-
-obj-$(CONFIG_R8723AU)	:= r8723au.o
-
-ccflags-y += $(call cc-option,-Wtype-limits,)
-ccflags-y += -D__CHECK_ENDIAN__ -I$(src)/include
diff --git a/drivers/staging/rtl8723au/TODO b/drivers/staging/rtl8723au/TODO
deleted file mode 100644
index 42b86e478df8..000000000000
--- a/drivers/staging/rtl8723au/TODO
+++ /dev/null
@@ -1,16 +0,0 @@
-TODO:
-- find and remove code valid only for 5 HGz. Many of the obvious
-  ones have been removed, but things like channel > 14 still exist.
-- find and remove any code for other chips that is left over
-- convert any remaining unusual variable types
-- find codes that can use %pM and %Nph formatting
-- checkpatch.pl fixes - most of the remaining ones are lines too long. Many
-  of them will require refactoring
-- merge Realtek's bugfixes and new features into the driver
-- switch to use MAC80211
-
-A mac80211 driver for this hardware already was integrated at
-drivers/net/wireless/realtek/rtl8xxxu/
-
-Please send any patches to Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
-Jes Sorensen <Jes.Sorensen@redhat.com>, and Larry Finger <Larry.Finger@lwfinger.net>.
diff --git a/drivers/staging/rtl8723au/core/rtw_ap.c b/drivers/staging/rtl8723au/core/rtw_ap.c
deleted file mode 100644
index aad686da3cf0..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_ap.c
+++ /dev/null
@@ -1,1738 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_AP_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <linux/ieee80211.h>
-#include <wifi.h>
-#include <rtl8723a_cmd.h>
-#include <rtl8723a_hal.h>
-#include <asm/unaligned.h>
-#include <rtw_mlme_ext.h>
-
-void init_mlme_ap_info23a(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct wlan_acl_pool *pacl_list = &pstapriv->acl_list;
-
-	spin_lock_init(&pmlmepriv->bcn_update_lock);
-
-	/* for ACL */
-	_rtw_init_queue23a(&pacl_list->acl_node_q);
-
-	start_ap_mode23a(padapter);
-}
-
-void free_mlme_ap_info23a(struct rtw_adapter *padapter)
-{
-	struct sta_info *psta = NULL;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	pmlmepriv->update_bcn = false;
-	pmlmeext->bstart_bss = false;
-
-	rtw_sta_flush23a(padapter);
-
-	pmlmeinfo->state = MSR_NOLINK;
-
-	/* free_assoc_sta_resources */
-	rtw_free_all_stainfo23a(padapter);
-
-	/* free bc/mc sta_info */
-	psta = rtw_get_bcmc_stainfo23a(padapter);
-	spin_lock_bh(&pstapriv->sta_hash_lock);
-	rtw_free_stainfo23a(padapter, psta);
-	spin_unlock_bh(&pstapriv->sta_hash_lock);
-}
-
-static void update_BCNTIM(struct rtw_adapter *padapter)
-{
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork_mlmeext = &pmlmeinfo->network;
-	unsigned char *pie = pnetwork_mlmeext->IEs;
-	u8 *p, *dst_ie, *premainder_ie = NULL, *pbackup_remainder_ie = NULL;
-	uint offset, tmp_len, tim_ielen, tim_ie_offset, remainder_ielen;
-
-	p = rtw_get_ie23a(pie, WLAN_EID_TIM, &tim_ielen,
-			  pnetwork_mlmeext->IELength);
-	if (p != NULL && tim_ielen > 0) {
-		tim_ielen += 2;
-
-		premainder_ie = p+tim_ielen;
-
-		tim_ie_offset = (int)(p - pie);
-
-		remainder_ielen = pnetwork_mlmeext->IELength - tim_ie_offset - tim_ielen;
-
-		/* append TIM IE from dst_ie offset */
-		dst_ie = p;
-	} else {
-		tim_ielen = 0;
-
-		/* calculate head_len */
-		offset = 0;
-
-		/* get ssid_ie len */
-		p = rtw_get_ie23a(pie, WLAN_EID_SSID,
-				  &tmp_len, pnetwork_mlmeext->IELength);
-		if (p != NULL)
-			offset += tmp_len+2;
-
-		/*  get supported rates len */
-		p = rtw_get_ie23a(pie, WLAN_EID_SUPP_RATES,
-				  &tmp_len, pnetwork_mlmeext->IELength);
-		if (p !=  NULL)
-			offset += tmp_len+2;
-
-		/* DS Parameter Set IE, len = 3 */
-		offset += 3;
-
-		premainder_ie = pie + offset;
-
-		remainder_ielen = pnetwork_mlmeext->IELength - offset - tim_ielen;
-
-		/* append TIM IE from offset */
-		dst_ie = pie + offset;
-	}
-
-	if (remainder_ielen > 0) {
-		pbackup_remainder_ie = kmalloc(remainder_ielen, GFP_ATOMIC);
-		if (pbackup_remainder_ie && premainder_ie)
-			memcpy(pbackup_remainder_ie, premainder_ie, remainder_ielen);
-	}
-
-	*dst_ie++ = WLAN_EID_TIM;
-
-	if ((pstapriv->tim_bitmap&0xff00) && (pstapriv->tim_bitmap&0x00fc))
-		tim_ielen = 5;
-	else
-		tim_ielen = 4;
-
-	*dst_ie++ = tim_ielen;
-
-	*dst_ie++ = 0; /* DTIM count */
-	*dst_ie++ = 1; /* DTIM period */
-
-	if (pstapriv->tim_bitmap & BIT(0)) /* for bc/mc frames */
-		*dst_ie++ = BIT(0);	   /* bitmap ctrl */
-	else
-		*dst_ie++ = 0;
-
-	if (tim_ielen == 4) {
-		*dst_ie++ = pstapriv->tim_bitmap & 0xff;
-	} else if (tim_ielen == 5) {
-		put_unaligned_le16(pstapriv->tim_bitmap, dst_ie);
-		dst_ie += 2;
-	}
-
-	/* copy remainder IE */
-	if (pbackup_remainder_ie) {
-		memcpy(dst_ie, pbackup_remainder_ie, remainder_ielen);
-
-		kfree(pbackup_remainder_ie);
-	}
-
-	offset =  (uint)(dst_ie - pie);
-	pnetwork_mlmeext->IELength = offset + remainder_ielen;
-
-	set_tx_beacon_cmd23a(padapter);
-}
-
-static u8 chk_sta_is_alive(struct sta_info *psta)
-{
-	u8 ret = false;
-
-	if ((psta->sta_stats.last_rx_data_pkts +
-	    psta->sta_stats.last_rx_ctrl_pkts) !=
-	    (psta->sta_stats.rx_data_pkts + psta->sta_stats.rx_ctrl_pkts))
-		ret = true;
-
-	sta_update_last_rx_pkts(psta);
-
-	return ret;
-}
-
-void expire_timeout_chk23a(struct rtw_adapter *padapter)
-{
-	struct list_head *phead;
-	u8 updated = 0;
-	struct sta_info *psta, *ptmp;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	u8 chk_alive_num = 0;
-	struct sta_info *chk_alive_list[NUM_STA];
-	int i;
-
-	spin_lock_bh(&pstapriv->auth_list_lock);
-	phead = &pstapriv->auth_list;
-	/* check auth_queue */
-	list_for_each_entry_safe(psta, ptmp, phead, auth_list) {
-		if (psta->expire_to > 0) {
-			psta->expire_to--;
-			if (psta->expire_to == 0) {
-				list_del_init(&psta->auth_list);
-				pstapriv->auth_list_cnt--;
-
-				DBG_8723A("auth expire %pM\n", psta->hwaddr);
-
-				spin_unlock_bh(&pstapriv->auth_list_lock);
-
-				spin_lock_bh(&pstapriv->sta_hash_lock);
-				rtw_free_stainfo23a(padapter, psta);
-				spin_unlock_bh(&pstapriv->sta_hash_lock);
-
-				spin_lock_bh(&pstapriv->auth_list_lock);
-			}
-		}
-	}
-	spin_unlock_bh(&pstapriv->auth_list_lock);
-
-	spin_lock_bh(&pstapriv->asoc_list_lock);
-	phead = &pstapriv->asoc_list;
-	/* check asoc_queue */
-	list_for_each_entry_safe(psta, ptmp, phead, asoc_list) {
-		if (chk_sta_is_alive(psta) || !psta->expire_to) {
-			psta->expire_to = pstapriv->expire_to;
-			psta->keep_alive_trycnt = 0;
-		} else {
-			psta->expire_to--;
-		}
-
-		if (psta->expire_to <= 0) {
-			struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-			if (padapter->registrypriv.wifi_spec == 1) {
-				psta->expire_to = pstapriv->expire_to;
-				continue;
-			}
-
-			if (psta->state & WIFI_SLEEP_STATE) {
-				if (!(psta->state & WIFI_STA_ALIVE_CHK_STATE)) {
-					/*
-					 * check if alive by another method
-					 * if station is at ps mode.
-					 */
-					psta->expire_to = pstapriv->expire_to;
-					psta->state |= WIFI_STA_ALIVE_CHK_STATE;
-					/*
-					 * update bcn with tim_bitmap
-					 * for this station
-					 */
-					pstapriv->tim_bitmap |= CHKBIT(psta->aid);
-					update_beacon23a(padapter, WLAN_EID_TIM, NULL, false);
-
-					if (!pmlmeext->active_keep_alive_check)
-						continue;
-				}
-			}
-
-			if (pmlmeext->active_keep_alive_check) {
-				chk_alive_list[chk_alive_num++] = psta;
-				continue;
-			}
-
-			list_del_init(&psta->asoc_list);
-			pstapriv->asoc_list_cnt--;
-
-			DBG_8723A("asoc expire %pM, state = 0x%x\n",
-				  psta->hwaddr, psta->state);
-			updated = ap_free_sta23a(padapter, psta, false, WLAN_REASON_DEAUTH_LEAVING);
-		} else {
-			/*
-			 * TODO: Aging mechanism to digest frames in
-			 * sleep_q to avoid running out of xmitframe
-			 */
-			if (psta->sleepq_len > (NR_XMITFRAME/pstapriv->asoc_list_cnt)
-				&& padapter->xmitpriv.free_xmitframe_cnt < ((NR_XMITFRAME/pstapriv->asoc_list_cnt)/2)
-			) {
-				DBG_8723A("%s sta:%pM, sleepq_len:%u, free_xmitframe_cnt:%u, asoc_list_cnt:%u, clear sleep_q\n",
-					  __func__,
-					  psta->hwaddr,
-					  psta->sleepq_len,
-					  padapter->xmitpriv.free_xmitframe_cnt,
-					  pstapriv->asoc_list_cnt);
-				wakeup_sta_to_xmit23a(padapter, psta);
-			}
-		}
-	}
-	spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-	if (chk_alive_num) {
-
-		u8 backup_oper_channel = 0;
-		struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-		/*
-		 * switch to correct channel of current
-		 * network before issue keep-alive frames
-		 */
-		if (rtw_get_oper_ch23a(padapter) != pmlmeext->cur_channel) {
-			backup_oper_channel = rtw_get_oper_ch23a(padapter);
-			SelectChannel23a(padapter, pmlmeext->cur_channel);
-		}
-
-	/* issue null data to check sta alive */
-	for (i = 0; i < chk_alive_num; i++) {
-
-		int ret = _FAIL;
-
-		psta = chk_alive_list[i];
-		if (!(psta->state & _FW_LINKED))
-			continue;
-
-		if (psta->state & WIFI_SLEEP_STATE)
-			ret = issue_nulldata23a(padapter, psta->hwaddr, 0, 1, 50);
-		else
-			ret = issue_nulldata23a(padapter, psta->hwaddr, 0, 3, 50);
-
-		psta->keep_alive_trycnt++;
-		if (ret == _SUCCESS) {
-			DBG_8723A("asoc check, sta(%pM) is alive\n",
-				  psta->hwaddr);
-			psta->expire_to = pstapriv->expire_to;
-			psta->keep_alive_trycnt = 0;
-			continue;
-		} else if (psta->keep_alive_trycnt <= 3) {
-			DBG_8723A("ack check for asoc expire, keep_alive_trycnt =%d\n", psta->keep_alive_trycnt);
-			psta->expire_to = 1;
-			continue;
-		}
-
-		psta->keep_alive_trycnt = 0;
-
-		DBG_8723A("asoc expire %pM, state = 0x%x\n",
-			  psta->hwaddr, psta->state);
-		spin_lock_bh(&pstapriv->asoc_list_lock);
-		if (!list_empty(&psta->asoc_list)) {
-			list_del_init(&psta->asoc_list);
-			pstapriv->asoc_list_cnt--;
-			updated = ap_free_sta23a(padapter, psta, false, WLAN_REASON_DEAUTH_LEAVING);
-		}
-		spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-	}
-
-	if (backup_oper_channel > 0) /* back to original operation channel */
-		SelectChannel23a(padapter, backup_oper_channel);
-}
-
-	associated_clients_update23a(padapter, updated);
-}
-
-void add_RATid23a(struct rtw_adapter *padapter, struct sta_info *psta, u8 rssi_level)
-{
-	int i;
-	u8 rf_type;
-	u32 init_rate = 0;
-	unsigned char sta_band = 0, raid, shortGIrate = false;
-	unsigned char limit;
-	unsigned int tx_ra_bitmap = 0;
-	struct ht_priv *psta_ht = NULL;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_bssid_ex *pcur_network = &pmlmepriv->cur_network.network;
-
-	if (psta)
-		psta_ht = &psta->htpriv;
-	else
-		return;
-
-	if (!(psta->state & _FW_LINKED))
-		return;
-
-	/* b/g mode ra_bitmap */
-	for (i = 0; i < sizeof(psta->bssrateset); i++) {
-		if (psta->bssrateset[i])
-			tx_ra_bitmap |= rtw_get_bit_value_from_ieee_value23a(psta->bssrateset[i]&0x7f);
-	}
-	/* n mode ra_bitmap */
-	if (psta_ht->ht_option) {
-		rf_type = rtl8723a_get_rf_type(padapter);
-
-		if (rf_type == RF_2T2R)
-			limit = 16; /* 2R */
-		else
-			limit = 8;  /* 1R */
-
-		for (i = 0; i < limit; i++) {
-			if (psta_ht->ht_cap.mcs.rx_mask[i / 8] & BIT(i % 8))
-				tx_ra_bitmap |= BIT(i + 12);
-		}
-
-		/* max short GI rate */
-		shortGIrate = psta_ht->sgi;
-	}
-
-	if (pcur_network->DSConfig > 14) {
-		/*  5G band */
-		if (tx_ra_bitmap & 0xffff000)
-			sta_band |= WIRELESS_11_5N | WIRELESS_11A;
-		else
-			sta_band |= WIRELESS_11A;
-	} else {
-		if (tx_ra_bitmap & 0xffff000)
-			sta_band |= WIRELESS_11_24N | WIRELESS_11G | WIRELESS_11B;
-		else if (tx_ra_bitmap & 0xff0)
-			sta_band |= WIRELESS_11G | WIRELESS_11B;
-		else
-			sta_band |= WIRELESS_11B;
-	}
-
-	psta->wireless_mode = sta_band;
-
-	raid = networktype_to_raid23a(sta_band);
-	init_rate = get_highest_rate_idx23a(tx_ra_bitmap&0x0fffffff)&0x3f;
-
-	if (psta->aid < NUM_STA) {
-		u8 arg;
-
-		arg = psta->mac_id&0x1f;
-
-		arg |= BIT(7); /* support entry 2~31 */
-
-		if (shortGIrate == true)
-			arg |= BIT(5);
-
-		tx_ra_bitmap |= ((raid<<28)&0xf0000000);
-
-		DBG_8723A("%s => mac_id:%d , raid:%d , bitmap = 0x%x, arg = "
-			  "0x%x\n",
-			  __func__, psta->mac_id, raid, tx_ra_bitmap, arg);
-
-		/* bitmap[0:27] = tx_rate_bitmap */
-		/* bitmap[28:31]= Rate Adaptive id */
-		/* arg[0:4] = macid */
-		/* arg[5] = Short GI */
-		rtl8723a_add_rateatid(padapter, tx_ra_bitmap, arg, rssi_level);
-
-		if (shortGIrate == true)
-			init_rate |= BIT(6);
-
-		/* set ra_id, init_rate */
-		psta->raid = raid;
-		psta->init_rate = init_rate;
-
-	} else
-		DBG_8723A("station aid %d exceed the max number\n", psta->aid);
-}
-
-static void update_bmc_sta(struct rtw_adapter *padapter)
-{
-	u32 init_rate = 0;
-	unsigned char network_type, raid;
-	int i, supportRateNum = 0;
-	unsigned int tx_ra_bitmap = 0;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_bssid_ex *pcur_network = &pmlmepriv->cur_network.network;
-	struct sta_info *psta = rtw_get_bcmc_stainfo23a(padapter);
-
-	if (psta) {
-		psta->aid = 0; /* default set to 0 */
-		psta->mac_id = psta->aid + 1;
-
-		psta->qos_option = 0;
-		psta->htpriv.ht_option = false;
-
-		psta->ieee8021x_blocked = 0;
-
-		memset((void *)&psta->sta_stats, 0,
-		       sizeof(struct stainfo_stats));
-
-		/* prepare for add_RATid23a */
-		supportRateNum = rtw_get_rateset_len23a((u8 *)&pcur_network->SupportedRates);
-		network_type = rtw_check_network_type23a((u8 *)&pcur_network->SupportedRates, supportRateNum, 1);
-
-		memcpy(psta->bssrateset, &pcur_network->SupportedRates, supportRateNum);
-		psta->bssratelen = supportRateNum;
-
-		/* b/g mode ra_bitmap */
-		for (i = 0; i < supportRateNum; i++) {
-			if (psta->bssrateset[i])
-				tx_ra_bitmap |= rtw_get_bit_value_from_ieee_value23a(psta->bssrateset[i]&0x7f);
-		}
-
-		if (pcur_network->DSConfig > 14) {
-			/* force to A mode. 5G doesn't support CCK rates */
-			network_type = WIRELESS_11A;
-			tx_ra_bitmap = 0x150; /* 6, 12, 24 Mbps */
-		} else {
-			/* force to b mode */
-			network_type = WIRELESS_11B;
-			tx_ra_bitmap = 0xf;
-		}
-
-		raid = networktype_to_raid23a(network_type);
-		init_rate = get_highest_rate_idx23a(tx_ra_bitmap&0x0fffffff)&0x3f;
-
-		/* ap mode */
-		rtl8723a_SetHalODMVar(padapter, HAL_ODM_STA_INFO, psta, true);
-
-		{
-			u8 arg;
-
-			arg = psta->mac_id&0x1f;
-
-			arg |= BIT(7);
-
-			tx_ra_bitmap |= ((raid<<28)&0xf0000000);
-
-			DBG_8723A("update_bmc_sta, mask = 0x%x, arg = 0x%x\n", tx_ra_bitmap, arg);
-
-			/* bitmap[0:27] = tx_rate_bitmap */
-			/* bitmap[28:31]= Rate Adaptive id */
-			/* arg[0:4] = macid */
-			/* arg[5] = Short GI */
-			rtl8723a_add_rateatid(padapter, tx_ra_bitmap, arg, 0);
-		}
-
-		/* set ra_id, init_rate */
-		psta->raid = raid;
-		psta->init_rate = init_rate;
-
-		spin_lock_bh(&psta->lock);
-		psta->state = _FW_LINKED;
-		spin_unlock_bh(&psta->lock);
-
-	} else
-		DBG_8723A("add_RATid23a_bmc_sta error!\n");
-}
-
-/*
- * AID: 1~MAX for sta and 0 for bc/mc in ap/adhoc mode
- * MAC_ID = AID+1 for sta in ap/adhoc mode
- * MAC_ID = 1 for bc/mc for sta/ap/adhoc
- * MAC_ID = 0 for bssid for sta/ap/adhoc
- * CAM_ID = 0~3 for default key, cmd_id = macid + 3, macid = aid + 1;
- */
-void update_sta_info23a_apmode23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct ht_priv *phtpriv_ap = &pmlmepriv->htpriv;
-	struct ht_priv *phtpriv_sta = &psta->htpriv;
-	/* set intf_tag to if1 */
-
-	psta->mac_id = psta->aid+1;
-	DBG_8723A("%s\n", __func__);
-
-	/* ap mode */
-	rtl8723a_SetHalODMVar(padapter, HAL_ODM_STA_INFO, psta, true);
-
-	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
-		psta->ieee8021x_blocked = true;
-	else
-		psta->ieee8021x_blocked = false;
-
-	/* update sta's cap */
-
-	/* ERP */
-	VCS_update23a(padapter, psta);
-	/* HT related cap */
-	if (phtpriv_sta->ht_option) {
-		/* check if sta supports rx ampdu */
-		phtpriv_sta->ampdu_enable = phtpriv_ap->ampdu_enable;
-
-		/* check if sta support s Short GI */
-		if ((phtpriv_sta->ht_cap.cap_info & phtpriv_ap->ht_cap.cap_info) & cpu_to_le16(IEEE80211_HT_CAP_SGI_20|IEEE80211_HT_CAP_SGI_40))
-			phtpriv_sta->sgi = true;
-
-		/* bwmode */
-		if ((phtpriv_sta->ht_cap.cap_info & phtpriv_ap->ht_cap.cap_info) & cpu_to_le16(IEEE80211_HT_CAP_SUP_WIDTH_20_40)) {
-			/* phtpriv_sta->bwmode = HT_CHANNEL_WIDTH_40; */
-			phtpriv_sta->bwmode = pmlmeext->cur_bwmode;
-			phtpriv_sta->ch_offset = pmlmeext->cur_ch_offset;
-
-		}
-
-		psta->qos_option = true;
-
-	} else {
-		phtpriv_sta->ampdu_enable = false;
-
-		phtpriv_sta->sgi = false;
-		phtpriv_sta->bwmode = HT_CHANNEL_WIDTH_20;
-		phtpriv_sta->ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-	}
-
-	/* Rx AMPDU */
-	send_delba23a(padapter, 0, psta->hwaddr); /* recipient */
-
-	/* TX AMPDU */
-	send_delba23a(padapter, 1, psta->hwaddr); /* originator */
-	phtpriv_sta->agg_enable_bitmap = 0x0;/* reset */
-	phtpriv_sta->candidate_tid_bitmap = 0x0;/* reset */
-
-	/* todo: init other variables */
-
-	memset((void *)&psta->sta_stats, 0, sizeof(struct stainfo_stats));
-
-	spin_lock_bh(&psta->lock);
-	psta->state |= _FW_LINKED;
-	spin_unlock_bh(&psta->lock);
-}
-
-static void update_hw_ht_param(struct rtw_adapter *padapter)
-{
-	unsigned char max_AMPDU_len;
-	unsigned char min_MPDU_spacing;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	DBG_8723A("%s\n", __func__);
-	/*
-	 * handle A-MPDU parameter field
-	 * AMPDU_para [1:0]:Max AMPDU Len => 0:8k, 1:16k, 2:32k, 3:64k
-	 * AMPDU_para [4:2]:Min MPDU Start Spacing
-	*/
-	max_AMPDU_len = pmlmeinfo->ht_cap.ampdu_params_info &
-		IEEE80211_HT_AMPDU_PARM_FACTOR;
-
-	min_MPDU_spacing = (pmlmeinfo->ht_cap.ampdu_params_info &
-			    IEEE80211_HT_AMPDU_PARM_DENSITY) >> 2;
-
-	rtl8723a_set_ampdu_min_space(padapter, min_MPDU_spacing);
-	rtl8723a_set_ampdu_factor(padapter, max_AMPDU_len);
-
-	/* Config SM Power Save setting */
-	pmlmeinfo->SM_PS = (le16_to_cpu(pmlmeinfo->ht_cap.cap_info) &
-			    IEEE80211_HT_CAP_SM_PS) >> 2;
-	if (pmlmeinfo->SM_PS == WLAN_HT_CAP_SM_PS_STATIC)
-		DBG_8723A("%s(): WLAN_HT_CAP_SM_PS_STATIC\n", __func__);
-}
-
-static void start_bss_network(struct rtw_adapter *padapter, u8 *pbuf)
-{
-	const u8 *p;
-	u8 val8, cur_channel, cur_bwmode, cur_ch_offset;
-	u16 bcn_interval;
-	u32 acparm;
-	struct registry_priv *pregpriv = &padapter->registrypriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct wlan_bssid_ex *pnetwork = &pmlmepriv->cur_network.network;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork_mlmeext = &pmlmeinfo->network;
-	struct ieee80211_ht_operation *pht_info = NULL;
-
-	bcn_interval = (u16)pnetwork->beacon_interval;
-	cur_channel = pnetwork->DSConfig;
-	cur_bwmode = HT_CHANNEL_WIDTH_20;
-	cur_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-
-	/*
-	 * check if there is wps ie
-	 * if there is wpsie in beacon the hostapd will
-	 * update beacon twice when stating hostapd
-	 * and at first time the security
-	 * ie (RSN/WPA IE) will not include in beacon
-	 */
-	if (!cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-				     WLAN_OUI_TYPE_MICROSOFT_WPS,
-				     pnetwork->IEs,
-				     pnetwork->IELength))
-		pmlmeext->bstart_bss = true;
-
-	/* todo: update wmm, ht cap */
-	/* pmlmeinfo->WMM_enable; */
-	/* pmlmeinfo->HT_enable; */
-	if (pmlmepriv->qos_option)
-		pmlmeinfo->WMM_enable = true;
-	if (pmlmepriv->htpriv.ht_option) {
-		pmlmeinfo->WMM_enable = true;
-		pmlmeinfo->HT_enable = true;
-
-		update_hw_ht_param(padapter);
-	}
-
-	if (pmlmepriv->cur_network.join_res != true) {
-		/*
-		 * setting only at first time
-		 * WEP Key will be set before this
-		 * function, do not clear CAM.
-		 */
-		if (psecuritypriv->dot11PrivacyAlgrthm !=
-		    WLAN_CIPHER_SUITE_WEP40 &&
-		    psecuritypriv->dot11PrivacyAlgrthm !=
-		    WLAN_CIPHER_SUITE_WEP104)
-			flush_all_cam_entry23a(padapter);	/* clear CAM */
-	}
-
-	/* set MSR to AP_Mode */
-	rtl8723a_set_media_status(padapter, MSR_AP);
-
-	/* Set BSSID REG */
-	hw_var_set_bssid(padapter, pnetwork->MacAddress);
-
-	/* Set EDCA param reg */
-	acparm = 0x002F3217; /* VO */
-	rtl8723a_set_ac_param_vo(padapter, acparm);
-	acparm = 0x005E4317; /* VI */
-	rtl8723a_set_ac_param_vi(padapter, acparm);
-	acparm = 0x005ea42b;
-	rtl8723a_set_ac_param_be(padapter, acparm);
-	acparm = 0x0000A444; /* BK */
-	rtl8723a_set_ac_param_bk(padapter, acparm);
-
-	/* Set Security */
-	val8 = (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) ?
-		0xcc : 0xcf;
-	rtl8723a_set_sec_cfg(padapter, val8);
-
-	/* Beacon Control related register */
-	rtl8723a_set_beacon_interval(padapter, bcn_interval);
-
-	UpdateBrateTbl23a(padapter, pnetwork->SupportedRates);
-	HalSetBrateCfg23a(padapter, pnetwork->SupportedRates);
-
-	if (!pmlmepriv->cur_network.join_res) {
-		/* setting only at  first time */
-
-		/* disable dynamic functions, such as high power, DIG */
-
-		/* turn on all dynamic functions */
-		rtl8723a_odm_support_ability_set(padapter,
-						 DYNAMIC_ALL_FUNC_ENABLE);
-	}
-	/* set channel, bwmode */
-
-	p = cfg80211_find_ie(WLAN_EID_HT_OPERATION, pnetwork->IEs,
-			     pnetwork->IELength);
-	if (p && p[1]) {
-		pht_info = (struct ieee80211_ht_operation *)(p + 2);
-
-		if (pregpriv->cbw40_enable && pht_info->ht_param &
-		    IEEE80211_HT_PARAM_CHAN_WIDTH_ANY) {
-			/* switch to the 40M Hz mode */
-			cur_bwmode = HT_CHANNEL_WIDTH_40;
-			switch (pht_info->ht_param &
-				IEEE80211_HT_PARAM_CHA_SEC_OFFSET) {
-			case IEEE80211_HT_PARAM_CHA_SEC_ABOVE:
-				/*
-				 * pmlmeext->cur_ch_offset =
-				 * HAL_PRIME_CHNL_OFFSET_LOWER;
-				 */
-				cur_ch_offset = HAL_PRIME_CHNL_OFFSET_LOWER;
-				break;
-			case IEEE80211_HT_PARAM_CHA_SEC_BELOW:
-				cur_ch_offset = HAL_PRIME_CHNL_OFFSET_UPPER;
-				break;
-			default:
-				cur_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-				break;
-			}
-		}
-	}
-	/*
-	 * TODO: need to judge the phy parameters
-	 * on concurrent mode for single phy
-	 */
-	set_channel_bwmode23a(padapter, cur_channel, cur_ch_offset, cur_bwmode);
-
-	DBG_8723A("CH =%d, BW =%d, offset =%d\n", cur_channel, cur_bwmode,
-		  cur_ch_offset);
-
-	pmlmeext->cur_channel = cur_channel;
-	pmlmeext->cur_bwmode = cur_bwmode;
-	pmlmeext->cur_ch_offset = cur_ch_offset;
-	pmlmeext->cur_wireless_mode = pmlmepriv->cur_network.network_type;
-
-	/* update cur_wireless_mode */
-	update_wireless_mode23a(padapter);
-
-	/* update capability after cur_wireless_mode updated */
-	update_capinfo23a(padapter, pnetwork->capability);
-
-	/* let pnetwork_mlmeext == pnetwork_mlme. */
-	memcpy(pnetwork_mlmeext, pnetwork, pnetwork->Length);
-
-	if (pmlmeext->bstart_bss) {
-		update_beacon23a(padapter, WLAN_EID_TIM, NULL, false);
-
-		/* issue beacon frame */
-		if (send_beacon23a(padapter) == _FAIL)
-			DBG_8723A("issue_beacon23a, fail!\n");
-	}
-
-	/* update bc/mc sta_info */
-	update_bmc_sta(padapter);
-}
-
-int rtw_check_beacon_data23a(struct rtw_adapter *padapter,
-			     struct ieee80211_mgmt *mgmt, unsigned int len)
-{
-	int ret = _SUCCESS;
-	u8 *p;
-	u8 *pHT_caps_ie = NULL;
-	u8 *pHT_info_ie = NULL;
-	struct sta_info *psta = NULL;
-	u16 ht_cap = false;
-	uint ie_len = 0;
-	int group_cipher, pairwise_cipher;
-	u8 channel, network_type, supportRate[NDIS_802_11_LENGTH_RATES_EX];
-	int supportRateNum = 0;
-	u8 WMM_PARA_IE[] = {0x00, 0x50, 0xf2, 0x02, 0x01, 0x01};
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_bssid_ex *pbss_network = &pmlmepriv->cur_network.network;
-	u8 *ie = pbss_network->IEs;
-	u8 *pbuf = mgmt->u.beacon.variable;
-
-	len -= offsetof(struct ieee80211_mgmt, u.beacon.variable);
-	/* SSID */
-	/* Supported rates */
-	/* DS Params */
-	/* WLAN_EID_COUNTRY */
-	/* ERP Information element */
-	/* Extended supported rates */
-	/* WPA/WPA2 */
-	/* Wi-Fi Wireless Multimedia Extensions */
-	/* ht_capab, ht_oper */
-	/* WPS IE */
-
-	DBG_8723A("%s, len =%d\n", __func__, len);
-
-	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE))
-		return _FAIL;
-
-	if (len > MAX_IE_SZ)
-		return _FAIL;
-
-	pbss_network->IELength = len;
-
-	memset(ie, 0, MAX_IE_SZ);
-
-	memcpy(ie, pbuf, pbss_network->IELength);
-
-	if (pbss_network->ifmode != NL80211_IFTYPE_AP &&
-	    pbss_network->ifmode != NL80211_IFTYPE_P2P_GO)
-		return _FAIL;
-
-	pbss_network->Rssi = 0;
-
-	memcpy(pbss_network->MacAddress, myid(&padapter->eeprompriv), ETH_ALEN);
-
-	/* SSID */
-	p = rtw_get_ie23a(ie, WLAN_EID_SSID, &ie_len, pbss_network->IELength);
-	if (p && ie_len > 0) {
-		memset(&pbss_network->Ssid, 0, sizeof(struct cfg80211_ssid));
-		memcpy(pbss_network->Ssid.ssid, (p + 2), ie_len);
-		pbss_network->Ssid.ssid_len = ie_len;
-	}
-
-	/* channel */
-	channel = 0;
-	p = rtw_get_ie23a(ie, WLAN_EID_DS_PARAMS, &ie_len,
-			  pbss_network->IELength);
-	if (p && ie_len > 0)
-		channel = *(p + 2);
-
-	pbss_network->DSConfig = channel;
-
-	memset(supportRate, 0, NDIS_802_11_LENGTH_RATES_EX);
-	/*  get supported rates */
-	p = rtw_get_ie23a(ie, WLAN_EID_SUPP_RATES, &ie_len,
-			  pbss_network->IELength);
-	if (p) {
-		memcpy(supportRate, p+2, ie_len);
-		supportRateNum = ie_len;
-	}
-
-	/* get ext_supported rates */
-	p = rtw_get_ie23a(ie, WLAN_EID_EXT_SUPP_RATES,
-			  &ie_len, pbss_network->IELength);
-	if (p) {
-		memcpy(supportRate+supportRateNum, p+2, ie_len);
-		supportRateNum += ie_len;
-	}
-
-	network_type = rtw_check_network_type23a(supportRate,
-						 supportRateNum, channel);
-
-	rtw_set_supported_rate23a(pbss_network->SupportedRates, network_type);
-
-	/* parsing ERP_IE */
-	p = rtw_get_ie23a(ie, WLAN_EID_ERP_INFO, &ie_len,
-			  pbss_network->IELength);
-	if (p && ie_len > 0)
-		ERP_IE_handler23a(padapter, p);
-
-	/* update privacy/security */
-	if (pbss_network->capability & BIT(4))
-		pbss_network->Privacy = 1;
-	else
-		pbss_network->Privacy = 0;
-
-	psecuritypriv->wpa_psk = 0;
-
-	/* wpa2 */
-	group_cipher = 0; pairwise_cipher = 0;
-	psecuritypriv->wpa2_group_cipher = 0;
-	psecuritypriv->wpa2_pairwise_cipher = 0;
-	p = rtw_get_ie23a(ie, WLAN_EID_RSN, &ie_len,
-			  pbss_network->IELength);
-	if (p && ie_len > 0) {
-		if (rtw_parse_wpa2_ie23a(p, ie_len+2, &group_cipher,
-					 &pairwise_cipher, NULL) == _SUCCESS) {
-			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
-
-			psecuritypriv->dot8021xalg = 1; /* psk, todo:802.1x */
-			psecuritypriv->wpa_psk |= BIT(1);
-
-			psecuritypriv->wpa2_group_cipher = group_cipher;
-			psecuritypriv->wpa2_pairwise_cipher = pairwise_cipher;
-		}
-	}
-
-	/* wpa */
-	ie_len = 0;
-	group_cipher = 0;
-	pairwise_cipher = 0;
-	psecuritypriv->wpa_group_cipher = 0;
-	psecuritypriv->wpa_pairwise_cipher = 0;
-	for (p = ie; ; p += (ie_len + 2)) {
-		p = rtw_get_ie23a(p, WLAN_EID_VENDOR_SPECIFIC, &ie_len,
-				  pbss_network->IELength - (ie_len + 2));
-		if ((p) && (!memcmp(p+2, RTW_WPA_OUI23A_TYPE, 4))) {
-			if (rtw_parse_wpa_ie23a(p, ie_len+2, &group_cipher,
-						&pairwise_cipher, NULL) == _SUCCESS) {
-				psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
-
-				/* psk,  todo:802.1x */
-				psecuritypriv->dot8021xalg = 1;
-
-				psecuritypriv->wpa_psk |= BIT(0);
-
-				psecuritypriv->wpa_group_cipher = group_cipher;
-				psecuritypriv->wpa_pairwise_cipher = pairwise_cipher;
-			}
-			break;
-		}
-
-		if (!p || !ie_len)
-			break;
-	}
-
-	/* wmm */
-	ie_len = 0;
-	pmlmepriv->qos_option = 0;
-	if (pregistrypriv->wmm_enable) {
-		for (p = ie; ; p += (ie_len + 2)) {
-			p = rtw_get_ie23a(p, WLAN_EID_VENDOR_SPECIFIC, &ie_len,
-					  (pbss_network->IELength -
-					   (ie_len + 2)));
-			if ((p) && !memcmp(p+2, WMM_PARA_IE, 6)) {
-				pmlmepriv->qos_option = 1;
-
-				*(p + 8) |= BIT(7);/* QoS Info:support U-APSD */
-
-				/* disable all ACM bits since the WMM admission
-				 * control is not supported
-				 */
-				*(p + 10) &= ~BIT(4); /* BE */
-				*(p + 14) &= ~BIT(4); /* BK */
-				*(p + 18) &= ~BIT(4); /* VI */
-				*(p + 22) &= ~BIT(4); /* VO */
-				break;
-			}
-			if ((p == NULL) || (ie_len == 0))
-				break;
-		}
-	}
-	/* parsing HT_CAP_IE */
-	p = rtw_get_ie23a(ie, WLAN_EID_HT_CAPABILITY, &ie_len,
-			  pbss_network->IELength);
-	if (p && ie_len > 0) {
-		u8 rf_type;
-
-		struct ieee80211_ht_cap *pht_cap = (struct ieee80211_ht_cap *)(p+2);
-
-		pHT_caps_ie = p;
-
-		ht_cap = true;
-		network_type |= WIRELESS_11_24N;
-
-		rf_type = rtl8723a_get_rf_type(padapter);
-
-		if ((psecuritypriv->wpa_pairwise_cipher & WPA_CIPHER_CCMP) ||
-		    (psecuritypriv->wpa2_pairwise_cipher & WPA_CIPHER_CCMP))
-			pht_cap->ampdu_params_info |= (IEEE80211_HT_AMPDU_PARM_DENSITY & (0x07<<2));
-		else
-			pht_cap->ampdu_params_info |= (IEEE80211_HT_AMPDU_PARM_DENSITY&0x00);
-
-		/* set  Max Rx AMPDU size  to 64K */
-		pht_cap->ampdu_params_info |= (IEEE80211_HT_AMPDU_PARM_FACTOR & 0x03);
-
-		if (rf_type == RF_1T1R) {
-			pht_cap->mcs.rx_mask[0] = 0xff;
-			pht_cap->mcs.rx_mask[1] = 0x0;
-		}
-
-		memcpy(&pmlmepriv->htpriv.ht_cap, p+2, ie_len);
-	}
-
-	/* parsing HT_INFO_IE */
-	p = rtw_get_ie23a(ie, WLAN_EID_HT_OPERATION, &ie_len,
-			  pbss_network->IELength);
-	if (p && ie_len > 0)
-		pHT_info_ie = p;
-
-	pmlmepriv->cur_network.network_type = network_type;
-
-	pmlmepriv->htpriv.ht_option = false;
-
-	/* ht_cap */
-	if (pregistrypriv->ht_enable && ht_cap) {
-		pmlmepriv->htpriv.ht_option = true;
-		pmlmepriv->qos_option = 1;
-
-		if (pregistrypriv->ampdu_enable == 1)
-			pmlmepriv->htpriv.ampdu_enable = true;
-
-		HT_caps_handler23a(padapter, pHT_caps_ie);
-
-		HT_info_handler23a(padapter, pHT_info_ie);
-	}
-
-	pbss_network->Length = get_wlan_bssid_ex_sz(pbss_network);
-
-	/* issue beacon to start bss network */
-	start_bss_network(padapter, (u8 *)pbss_network);
-
-	/* alloc sta_info for ap itself */
-	psta = rtw_get_stainfo23a(&padapter->stapriv, pbss_network->MacAddress);
-	if (!psta) {
-		psta = rtw_alloc_stainfo23a(&padapter->stapriv,
-					    pbss_network->MacAddress,
-					    GFP_KERNEL);
-		if (!psta)
-			return _FAIL;
-	}
-	/* fix bug of flush_cam_entry at STOP AP mode */
-	psta->state |= WIFI_AP_STATE;
-	rtw_indicate_connect23a(padapter);
-
-	/* for check if already set beacon */
-	pmlmepriv->cur_network.join_res = true;
-
-	return ret;
-}
-
-void rtw_set_macaddr_acl23a(struct rtw_adapter *padapter, int mode)
-{
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct wlan_acl_pool *pacl_list = &pstapriv->acl_list;
-
-	DBG_8723A("%s, mode =%d\n", __func__, mode);
-
-	pacl_list->mode = mode;
-}
-
-static void update_bcn_erpinfo_ie(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork = &pmlmeinfo->network;
-	unsigned char *p, *ie = pnetwork->IEs;
-	u32 len = 0;
-
-	DBG_8723A("%s, ERP_enable =%d\n", __func__, pmlmeinfo->ERP_enable);
-
-	if (!pmlmeinfo->ERP_enable)
-		return;
-
-	/* parsing ERP_IE */
-	p = rtw_get_ie23a(ie, WLAN_EID_ERP_INFO, &len, pnetwork->IELength);
-	if (p && len > 0) {
-		if (pmlmepriv->num_sta_non_erp == 1)
-			p[2] |= WLAN_ERP_NON_ERP_PRESENT |
-				WLAN_ERP_USE_PROTECTION;
-		else
-			p[2] &= ~(WLAN_ERP_NON_ERP_PRESENT |
-				  WLAN_ERP_USE_PROTECTION);
-
-		if (pmlmepriv->num_sta_no_short_preamble > 0)
-			p[2] |= WLAN_ERP_BARKER_PREAMBLE;
-		else
-			p[2] &= ~(WLAN_ERP_BARKER_PREAMBLE);
-
-		ERP_IE_handler23a(padapter, p);
-	}
-}
-
-static void update_bcn_wpa_ie(struct rtw_adapter *padapter)
-{
-	DBG_8723A("%s\n", __func__);
-}
-
-static void update_bcn_wmm_ie(struct rtw_adapter *padapter)
-{
-	DBG_8723A("%s\n", __func__);
-}
-
-static void update_bcn_wps_ie(struct rtw_adapter *padapter)
-{
-	DBG_8723A("%s\n", __func__);
-}
-
-static void update_bcn_p2p_ie(struct rtw_adapter *padapter)
-{
-}
-
-static void update_bcn_vendor_spec_ie(struct rtw_adapter *padapter, u8 *oui)
-{
-	DBG_8723A("%s\n", __func__);
-
-	if (!memcmp(RTW_WPA_OUI23A_TYPE, oui, 4))
-		update_bcn_wpa_ie(padapter);
-	else if (!memcmp(WMM_OUI23A, oui, 4))
-		update_bcn_wmm_ie(padapter);
-	else if (!memcmp(WPS_OUI23A, oui, 4))
-		update_bcn_wps_ie(padapter);
-	else if (!memcmp(P2P_OUI23A, oui, 4))
-		update_bcn_p2p_ie(padapter);
-	else
-		DBG_8723A("unknown OUI type!\n");
-}
-
-void update_beacon23a(struct rtw_adapter *padapter, u8 ie_id, u8 *oui, u8 tx)
-{
-	struct mlme_priv *pmlmepriv;
-	struct mlme_ext_priv *pmlmeext;
-	/* struct mlme_ext_info *pmlmeinfo; */
-
-	/* DBG_8723A("%s\n", __func__); */
-
-	if (!padapter)
-		return;
-
-	pmlmepriv = &padapter->mlmepriv;
-	pmlmeext = &padapter->mlmeextpriv;
-	/* pmlmeinfo = &pmlmeext->mlmext_info; */
-
-	if (false == pmlmeext->bstart_bss)
-		return;
-
-	spin_lock_bh(&pmlmepriv->bcn_update_lock);
-
-	switch (ie_id) {
-	case WLAN_EID_TIM:
-		update_BCNTIM(padapter);
-		break;
-
-	case WLAN_EID_ERP_INFO:
-		update_bcn_erpinfo_ie(padapter);
-		break;
-
-	case WLAN_EID_VENDOR_SPECIFIC:
-		update_bcn_vendor_spec_ie(padapter, oui);
-		break;
-
-	default:
-		break;
-	}
-
-	pmlmepriv->update_bcn = true;
-
-	spin_unlock_bh(&pmlmepriv->bcn_update_lock);
-
-	if (tx)
-		set_tx_beacon_cmd23a(padapter);
-}
-
-/*
- * op_mode
- * Set to 0 (HT pure) under the following conditions
- *	- all STAs in the BSS are 20/40 MHz HT in 20/40 MHz BSS or
- *	- all STAs in the BSS are 20 MHz HT in 20 MHz BSS
- * Set to 1 (HT non-member protection) if there may be non-HT STAs
- *	in both the primary and the secondary channel
- * Set to 2 if only HT STAs are associated in BSS,
- *	however and at least one 20 MHz HT STA is associated
- * Set to 3 (HT mixed mode) when one or more non-HT STAs are associated
- *	(currently non-GF HT station is considered as non-HT STA also)
-*/
-static int rtw_ht_operation_update(struct rtw_adapter *padapter)
-{
-	u16 cur_op_mode, new_op_mode;
-	int op_mode_changes = 0;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv_ap = &pmlmepriv->htpriv;
-
-	if (pmlmepriv->htpriv.ht_option)
-		return 0;
-
-	/* if (!iface->conf->ieee80211n || iface->conf->ht_op_mode_fixed) */
-	/*	return 0; */
-
-	DBG_8723A("%s current operation mode = 0x%X\n",
-		   __func__, pmlmepriv->ht_op_mode);
-
-	if (!(pmlmepriv->ht_op_mode & IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT)
-	    && pmlmepriv->num_sta_ht_no_gf) {
-		pmlmepriv->ht_op_mode |=
-			IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT;
-		op_mode_changes++;
-	} else if ((pmlmepriv->ht_op_mode &
-		    IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT) &&
-		   pmlmepriv->num_sta_ht_no_gf == 0) {
-		pmlmepriv->ht_op_mode &=
-			~IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT;
-		op_mode_changes++;
-	}
-
-	if (!(pmlmepriv->ht_op_mode & IEEE80211_HT_OP_MODE_NON_HT_STA_PRSNT) &&
-	    (pmlmepriv->num_sta_no_ht || pmlmepriv->olbc_ht)) {
-		pmlmepriv->ht_op_mode |= IEEE80211_HT_OP_MODE_NON_HT_STA_PRSNT;
-		op_mode_changes++;
-	} else if ((pmlmepriv->ht_op_mode &
-		    IEEE80211_HT_OP_MODE_NON_HT_STA_PRSNT) &&
-		   (pmlmepriv->num_sta_no_ht == 0 && !pmlmepriv->olbc_ht)) {
-		pmlmepriv->ht_op_mode &=
-			~IEEE80211_HT_OP_MODE_NON_HT_STA_PRSNT;
-		op_mode_changes++;
-	}
-
-	/*
-	 * Note: currently we switch to the MIXED op mode if HT non-greenfield
-	 * station is associated. Probably it's a theoretical case, since
-	 * it looks like all known HT STAs support greenfield.
-	 */
-	if (pmlmepriv->num_sta_no_ht ||
-	    (pmlmepriv->ht_op_mode & IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT))
-		new_op_mode = IEEE80211_HT_OP_MODE_PROTECTION_NONHT_MIXED;
-	else if ((le16_to_cpu(phtpriv_ap->ht_cap.cap_info) &
-		  IEEE80211_HT_CAP_SUP_WIDTH_20_40) &&
-		 pmlmepriv->num_sta_ht_20mhz)
-		new_op_mode = IEEE80211_HT_OP_MODE_PROTECTION_20MHZ;
-	else if (pmlmepriv->olbc_ht)
-		new_op_mode = IEEE80211_HT_OP_MODE_PROTECTION_NONMEMBER;
-	else
-		new_op_mode = IEEE80211_HT_OP_MODE_PROTECTION_NONE;
-
-	cur_op_mode = pmlmepriv->ht_op_mode & IEEE80211_HT_OP_MODE_PROTECTION;
-	if (cur_op_mode != new_op_mode) {
-		pmlmepriv->ht_op_mode &= ~IEEE80211_HT_OP_MODE_PROTECTION;
-		pmlmepriv->ht_op_mode |= new_op_mode;
-		op_mode_changes++;
-	}
-
-	DBG_8723A("%s new operation mode = 0x%X changes =%d\n",
-		   __func__, pmlmepriv->ht_op_mode, op_mode_changes);
-
-	return op_mode_changes;
-}
-
-void associated_clients_update23a(struct rtw_adapter *padapter, u8 updated)
-{
-	/* update associated stations cap. */
-	if (updated == true) {
-		struct list_head *phead;
-		struct sta_info *psta, *ptmp;
-		struct sta_priv *pstapriv = &padapter->stapriv;
-
-		spin_lock_bh(&pstapriv->asoc_list_lock);
-		phead = &pstapriv->asoc_list;
-		list_for_each_entry_safe(psta, ptmp, phead, asoc_list)
-			VCS_update23a(padapter, psta);
-		spin_unlock_bh(&pstapriv->asoc_list_lock);
-	}
-}
-
-/* called > TSR LEVEL for USB or SDIO Interface */
-void bss_cap_update_on_sta_join23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	u8 beacon_updated = false;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	if (!(psta->flags & WLAN_STA_SHORT_PREAMBLE)) {
-		if (!psta->no_short_preamble_set) {
-			psta->no_short_preamble_set = 1;
-
-			pmlmepriv->num_sta_no_short_preamble++;
-
-			if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
-				(pmlmepriv->num_sta_no_short_preamble == 1)) {
-				beacon_updated = true;
-				update_beacon23a(padapter, 0xFF, NULL, true);
-			}
-
-		}
-	} else {
-		if (psta->no_short_preamble_set) {
-			psta->no_short_preamble_set = 0;
-
-			pmlmepriv->num_sta_no_short_preamble--;
-
-			if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
-				(pmlmepriv->num_sta_no_short_preamble == 0)) {
-				beacon_updated = true;
-				update_beacon23a(padapter, 0xFF, NULL, true);
-			}
-
-		}
-	}
-
-	if (psta->flags & WLAN_STA_NONERP) {
-		if (!psta->nonerp_set) {
-			psta->nonerp_set = 1;
-
-			pmlmepriv->num_sta_non_erp++;
-
-			if (pmlmepriv->num_sta_non_erp == 1) {
-				beacon_updated = true;
-				update_beacon23a(padapter, WLAN_EID_ERP_INFO, NULL, true);
-			}
-		}
-
-	} else {
-		if (psta->nonerp_set) {
-			psta->nonerp_set = 0;
-
-			pmlmepriv->num_sta_non_erp--;
-
-			if (pmlmepriv->num_sta_non_erp == 0) {
-				beacon_updated = true;
-				update_beacon23a(padapter, WLAN_EID_ERP_INFO, NULL, true);
-			}
-		}
-
-	}
-
-	if (!(psta->capability & WLAN_CAPABILITY_SHORT_SLOT_TIME)) {
-		if (!psta->no_short_slot_time_set) {
-			psta->no_short_slot_time_set = 1;
-
-			pmlmepriv->num_sta_no_short_slot_time++;
-
-			if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
-				 (pmlmepriv->num_sta_no_short_slot_time == 1)) {
-				beacon_updated = true;
-				update_beacon23a(padapter, 0xFF, NULL, true);
-			}
-
-		}
-	} else {
-		if (psta->no_short_slot_time_set) {
-			psta->no_short_slot_time_set = 0;
-
-			pmlmepriv->num_sta_no_short_slot_time--;
-
-			if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
-				 (pmlmepriv->num_sta_no_short_slot_time == 0)) {
-				beacon_updated = true;
-				update_beacon23a(padapter, 0xFF, NULL, true);
-			}
-		}
-	}
-
-	if (psta->flags & WLAN_STA_HT) {
-		u16 ht_capab = le16_to_cpu(psta->htpriv.ht_cap.cap_info);
-
-		DBG_8723A("HT: STA %pM HT Capabilities Info: 0x%04x\n",
-			  psta->hwaddr, ht_capab);
-
-		if (psta->no_ht_set) {
-			psta->no_ht_set = 0;
-			pmlmepriv->num_sta_no_ht--;
-		}
-
-		if ((ht_capab & IEEE80211_HT_CAP_GRN_FLD) == 0) {
-			if (!psta->no_ht_gf_set) {
-				psta->no_ht_gf_set = 1;
-				pmlmepriv->num_sta_ht_no_gf++;
-			}
-			DBG_8723A("%s STA %pM - no greenfield, num of non-gf stations %d\n",
-				  __func__, psta->hwaddr,
-				  pmlmepriv->num_sta_ht_no_gf);
-		}
-
-		if ((ht_capab & IEEE80211_HT_CAP_SUP_WIDTH_20_40) == 0) {
-			if (!psta->ht_20mhz_set) {
-				psta->ht_20mhz_set = 1;
-				pmlmepriv->num_sta_ht_20mhz++;
-			}
-			DBG_8723A("%s STA %pM - 20 MHz HT, num of 20MHz HT STAs %d\n",
-				  __func__, psta->hwaddr,
-				  pmlmepriv->num_sta_ht_20mhz);
-		}
-
-	} else {
-		if (!psta->no_ht_set) {
-			psta->no_ht_set = 1;
-			pmlmepriv->num_sta_no_ht++;
-		}
-		if (pmlmepriv->htpriv.ht_option) {
-			DBG_8723A("%s STA %pM - no HT, num of non-HT stations %d\n",
-				  __func__, psta->hwaddr,
-				  pmlmepriv->num_sta_no_ht);
-		}
-	}
-
-	if (rtw_ht_operation_update(padapter) > 0) {
-		update_beacon23a(padapter, WLAN_EID_HT_CAPABILITY, NULL, false);
-		update_beacon23a(padapter, WLAN_EID_HT_OPERATION, NULL, true);
-	}
-
-	/* update associated stations cap. */
-	associated_clients_update23a(padapter,  beacon_updated);
-
-	DBG_8723A("%s, updated =%d\n", __func__, beacon_updated);
-}
-
-u8 bss_cap_update_on_sta_leave23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	u8 beacon_updated = false;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	if (!psta)
-		return beacon_updated;
-
-	if (psta->no_short_preamble_set) {
-		psta->no_short_preamble_set = 0;
-		pmlmepriv->num_sta_no_short_preamble--;
-		if (pmlmeext->cur_wireless_mode > WIRELESS_11B
-		    && pmlmepriv->num_sta_no_short_preamble == 0) {
-			beacon_updated = true;
-			update_beacon23a(padapter, 0xFF, NULL, true);
-		}
-	}
-
-	if (psta->nonerp_set) {
-		psta->nonerp_set = 0;
-		pmlmepriv->num_sta_non_erp--;
-		if (pmlmepriv->num_sta_non_erp == 0) {
-			beacon_updated = true;
-			update_beacon23a(padapter, WLAN_EID_ERP_INFO,
-					 NULL, true);
-		}
-	}
-
-	if (psta->no_short_slot_time_set) {
-		psta->no_short_slot_time_set = 0;
-		pmlmepriv->num_sta_no_short_slot_time--;
-		if (pmlmeext->cur_wireless_mode > WIRELESS_11B
-		    && pmlmepriv->num_sta_no_short_slot_time == 0) {
-			beacon_updated = true;
-			update_beacon23a(padapter, 0xFF, NULL, true);
-		}
-	}
-
-	if (psta->no_ht_gf_set) {
-		psta->no_ht_gf_set = 0;
-		pmlmepriv->num_sta_ht_no_gf--;
-	}
-
-	if (psta->no_ht_set) {
-		psta->no_ht_set = 0;
-		pmlmepriv->num_sta_no_ht--;
-	}
-
-	if (psta->ht_20mhz_set) {
-		psta->ht_20mhz_set = 0;
-		pmlmepriv->num_sta_ht_20mhz--;
-	}
-
-	if (rtw_ht_operation_update(padapter) > 0) {
-		update_beacon23a(padapter, WLAN_EID_HT_CAPABILITY, NULL, false);
-		update_beacon23a(padapter, WLAN_EID_HT_OPERATION, NULL, true);
-	}
-
-	/* update associated stations cap. */
-
-	DBG_8723A("%s, updated =%d\n", __func__, beacon_updated);
-
-	return beacon_updated;
-}
-
-u8 ap_free_sta23a(struct rtw_adapter *padapter, struct sta_info *psta, bool active, u16 reason)
-{
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	u8 beacon_updated = false;
-
-	if (!psta)
-		return beacon_updated;
-
-	if (active) {
-		/* tear down Rx AMPDU */
-		send_delba23a(padapter, 0, psta->hwaddr); /* recipient */
-
-		/* tear down TX AMPDU */
-		send_delba23a(padapter, 1, psta->hwaddr); /* originator */
-
-		issue_deauth23a(padapter, psta->hwaddr, reason);
-	}
-
-	psta->htpriv.agg_enable_bitmap = 0x0;	 /* reset */
-	psta->htpriv.candidate_tid_bitmap = 0x0; /* reset */
-
-	/* report_del_sta_event23a(padapter, psta->hwaddr, reason); */
-
-	/* clear cam entry / key */
-	/* clear_cam_entry23a(padapter, (psta->mac_id + 3)); */
-	rtw_clearstakey_cmd23a(padapter, (u8 *)psta, (u8)(psta->mac_id + 3),
-			       true);
-
-	spin_lock_bh(&psta->lock);
-	psta->state &= ~_FW_LINKED;
-	spin_unlock_bh(&psta->lock);
-
-	rtw_cfg80211_indicate_sta_disassoc(padapter, psta->hwaddr, reason);
-
-	report_del_sta_event23a(padapter, psta->hwaddr, reason);
-
-	beacon_updated = bss_cap_update_on_sta_leave23a(padapter, psta);
-
-	spin_lock_bh(&pstapriv->sta_hash_lock);
-	rtw_free_stainfo23a(padapter, psta);
-	spin_unlock_bh(&pstapriv->sta_hash_lock);
-
-	return beacon_updated;
-}
-
-int rtw_sta_flush23a(struct rtw_adapter *padapter)
-{
-	struct list_head *phead;
-	struct sta_info *psta, *ptmp;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-	u8 chk_alive_num = 0;
-	struct sta_info *chk_alive_list[NUM_STA];
-	int i;
-
-	DBG_8723A("%s(%s)\n", __func__, padapter->pnetdev->name);
-
-	if ((pmlmeinfo->state&0x03) != MSR_AP)
-		return 0;
-
-	spin_lock_bh(&pstapriv->asoc_list_lock);
-	phead = &pstapriv->asoc_list;
-	list_for_each_entry_safe(psta, ptmp, phead, asoc_list) {
-		/* Remove sta from asoc_list */
-		list_del_init(&psta->asoc_list);
-		pstapriv->asoc_list_cnt--;
-
-		/* Keep sta for ap_free_sta23a() beyond this asoc_list loop */
-		chk_alive_list[chk_alive_num++] = psta;
-	}
-	spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-	/* For each sta in chk_alive_list, call ap_free_sta23a */
-	for (i = 0; i < chk_alive_num; i++)
-		ap_free_sta23a(padapter, chk_alive_list[i], true,
-			       WLAN_REASON_DEAUTH_LEAVING);
-
-	issue_deauth23a(padapter, bc_addr, WLAN_REASON_DEAUTH_LEAVING);
-
-	associated_clients_update23a(padapter, true);
-
-	return 0;
-}
-
-/* called > TSR LEVEL for USB or SDIO Interface */
-void sta_info_update23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	int flags = psta->flags;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	/* update wmm cap. */
-	if (WLAN_STA_WME&flags)
-		psta->qos_option = 1;
-	else
-		psta->qos_option = 0;
-
-	if (pmlmepriv->qos_option == 0)
-		psta->qos_option = 0;
-
-	/* update 802.11n ht cap. */
-	if (WLAN_STA_HT&flags) {
-		psta->htpriv.ht_option = true;
-		psta->qos_option = 1;
-	} else {
-		psta->htpriv.ht_option = false;
-	}
-
-	if (!pmlmepriv->htpriv.ht_option)
-		psta->htpriv.ht_option = false;
-
-	update_sta_info23a_apmode23a(padapter, psta);
-}
-
-/* called >= TSR LEVEL for USB or SDIO Interface */
-void ap_sta_info_defer_update23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	if (psta->state & _FW_LINKED) {
-		/* add ratid */
-		add_RATid23a(padapter, psta, 0);/* DM_RATR_STA_INIT */
-	}
-}
-
-/* restore hw setting from sw data structures */
-void rtw_ap_restore_network(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *mlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct sta_info *psta, *ptmp;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct list_head *phead;
-	u8 chk_alive_num = 0;
-	struct sta_info *chk_alive_list[NUM_STA];
-	int i;
-
-	rtw_setopmode_cmd23a(padapter, NL80211_IFTYPE_AP);
-
-	set_channel_bwmode23a(padapter, pmlmeext->cur_channel, pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode);
-
-	start_bss_network(padapter, (u8 *)&mlmepriv->cur_network.network);
-
-	if (padapter->securitypriv.dot11PrivacyAlgrthm ==
-	    WLAN_CIPHER_SUITE_TKIP ||
-	    padapter->securitypriv.dot11PrivacyAlgrthm ==
-	    WLAN_CIPHER_SUITE_CCMP) {
-		/* restore group key, WEP keys is restored in ips_leave23a() */
-		rtw_set_key23a(padapter, psecuritypriv,
-			       psecuritypriv->dot118021XGrpKeyid, 0);
-	}
-
-	/* per sta pairwise key and settings */
-	if (padapter->securitypriv.dot11PrivacyAlgrthm !=
-	    WLAN_CIPHER_SUITE_TKIP &&
-	    padapter->securitypriv.dot11PrivacyAlgrthm !=
-	    WLAN_CIPHER_SUITE_CCMP) {
-		return;
-	}
-
-	spin_lock_bh(&pstapriv->asoc_list_lock);
-	phead = &pstapriv->asoc_list;
-	list_for_each_entry_safe(psta, ptmp, phead, asoc_list)
-		chk_alive_list[chk_alive_num++] = psta;
-	spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-	for (i = 0; i < chk_alive_num; i++) {
-		psta = chk_alive_list[i];
-
-		if (psta->state & _FW_LINKED) {
-			Update_RA_Entry23a(padapter, psta);
-			/* pairwise key */
-			rtw_setstakey_cmd23a(padapter, (unsigned char *)psta, true);
-		}
-	}
-}
-
-void start_ap_mode23a(struct rtw_adapter *padapter)
-{
-	int i;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct wlan_acl_pool *pacl_list = &pstapriv->acl_list;
-
-	pmlmepriv->update_bcn = false;
-
-	/* init_mlme_ap_info23a(padapter); */
-	pmlmeext->bstart_bss = false;
-
-	pmlmepriv->num_sta_non_erp = 0;
-
-	pmlmepriv->num_sta_no_short_slot_time = 0;
-
-	pmlmepriv->num_sta_no_short_preamble = 0;
-
-	pmlmepriv->num_sta_ht_no_gf = 0;
-	pmlmepriv->num_sta_no_ht = 0;
-	pmlmepriv->num_sta_ht_20mhz = 0;
-
-	pmlmepriv->olbc = false;
-
-	pmlmepriv->olbc_ht = false;
-
-	pmlmepriv->ht_op_mode = 0;
-
-	for (i = 0; i < NUM_STA; i++)
-		pstapriv->sta_aid[i] = NULL;
-
-	/* for ACL */
-	INIT_LIST_HEAD(&pacl_list->acl_node_q.queue);
-	pacl_list->num = 0;
-	pacl_list->mode = 0;
-	for (i = 0; i < NUM_ACL; i++) {
-		INIT_LIST_HEAD(&pacl_list->aclnode[i].list);
-		pacl_list->aclnode[i].valid = false;
-	}
-}
-
-void stop_ap_mode23a(struct rtw_adapter *padapter)
-{
-	struct list_head *phead;
-	struct rtw_wlan_acl_node *paclnode, *ptmp;
-	struct sta_info *psta = NULL;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct wlan_acl_pool *pacl_list = &pstapriv->acl_list;
-	struct rtw_queue *pacl_node_q = &pacl_list->acl_node_q;
-
-	pmlmepriv->update_bcn = false;
-	pmlmeext->bstart_bss = false;
-
-	/*
-	 * reset and init security priv , this can
-	 * refine with rtw_reset_securitypriv23a
-	 */
-	memset((unsigned char *)&padapter->securitypriv, 0, sizeof(struct security_priv));
-	padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen;
-	padapter->securitypriv.ndisencryptstatus = Ndis802_11WEPDisabled;
-
-	/* for ACL */
-	spin_lock_bh(&pacl_node_q->lock);
-	phead = get_list_head(pacl_node_q);
-	list_for_each_entry_safe(paclnode, ptmp, phead, list) {
-		if (paclnode->valid == true) {
-			paclnode->valid = false;
-			list_del_init(&paclnode->list);
-			pacl_list->num--;
-		}
-	}
-	spin_unlock_bh(&pacl_node_q->lock);
-
-	DBG_8723A("%s, free acl_node_queue, num =%d\n",
-		  __func__, pacl_list->num);
-
-	rtw_sta_flush23a(padapter);
-
-	/* free_assoc_sta_resources */
-	rtw_free_all_stainfo23a(padapter);
-
-	psta = rtw_get_bcmc_stainfo23a(padapter);
-	spin_lock_bh(&pstapriv->sta_hash_lock);
-	rtw_free_stainfo23a(padapter, psta);
-	spin_unlock_bh(&pstapriv->sta_hash_lock);
-
-	rtw_init_bcmc_stainfo23a(padapter);
-
-	rtw23a_free_mlme_priv_ie_data(pmlmepriv);
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_cmd.c b/drivers/staging/rtl8723au/core/rtw_cmd.c
deleted file mode 100644
index cd4e0f05d82f..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_cmd.c
+++ /dev/null
@@ -1,1470 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_CMD_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <recv_osdep.h>
-#include <mlme_osdep.h>
-#include <rtl8723a_cmd.h>
-#include <rtw_sreset.h>
-
-static struct cmd_hdl wlancmds[] = {
-	GEN_DRV_CMD_HANDLER(0, NULL) /*0*/
-	GEN_DRV_CMD_HANDLER(0, NULL)
-	GEN_DRV_CMD_HANDLER(0, NULL)
-	GEN_DRV_CMD_HANDLER(0, NULL)
-	GEN_DRV_CMD_HANDLER(0, NULL)
-	GEN_DRV_CMD_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL) /*10*/
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct wlan_bssid_ex), join_cmd_hdl23a) /*14*/
-	GEN_MLME_EXT_HANDLER(sizeof(struct disconnect_parm), disconnect_hdl23a)
-	GEN_MLME_EXT_HANDLER(sizeof(struct wlan_bssid_ex), createbss_hdl23a)
-	GEN_MLME_EXT_HANDLER(sizeof(struct setopmode_parm), setopmode_hdl23a)
-	GEN_MLME_EXT_HANDLER(sizeof(struct sitesurvey_parm), sitesurvey_cmd_hdl23a) /*18*/
-	GEN_MLME_EXT_HANDLER(sizeof(struct setauth_parm), setauth_hdl23a)
-	GEN_MLME_EXT_HANDLER(sizeof(struct setkey_parm), setkey_hdl23a) /*20*/
-	GEN_MLME_EXT_HANDLER(sizeof(struct set_stakey_parm), set_stakey_hdl23a)
-	GEN_MLME_EXT_HANDLER(sizeof(struct set_assocsta_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct del_assocsta_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct setstapwrstate_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct setbasicrate_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct getbasicrate_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct setdatarate_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct getdatarate_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct setphyinfo_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct getphyinfo_parm), NULL)  /*30*/
-	GEN_MLME_EXT_HANDLER(sizeof(struct setphy_parm), NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct getphy_parm), NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)	/*40*/
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct addBaReq_parm), add_ba_hdl23a)
-	GEN_MLME_EXT_HANDLER(sizeof(struct set_ch_parm), set_ch_hdl23a) /* 46 */
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL) /*50*/
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(0, NULL)
-	GEN_MLME_EXT_HANDLER(sizeof(struct Tx_Beacon_param), tx_beacon_hdl23a) /*55*/
-
-	GEN_MLME_EXT_HANDLER(0, mlme_evt_hdl23a) /*56*/
-	GEN_MLME_EXT_HANDLER(0, rtw_drvextra_cmd_hdl23a) /*57*/
-
-	GEN_MLME_EXT_HANDLER(0, h2c_msg_hdl23a) /*58*/
-	GEN_MLME_EXT_HANDLER(sizeof(struct SetChannelPlan_param), set_chplan_hdl23a) /*59*/
-	GEN_MLME_EXT_HANDLER(sizeof(struct LedBlink_param), led_blink_hdl23a) /*60*/
-
-	GEN_MLME_EXT_HANDLER(sizeof(struct SetChannelSwitch_param), set_csa_hdl23a) /*61*/
-	GEN_MLME_EXT_HANDLER(sizeof(struct TDLSoption_param), tdls_hdl23a) /*62*/
-};
-
-struct _cmd_callback	rtw_cmd_callback[] = {
-	{GEN_CMD_CODE(_Read_MACREG), NULL}, /*0*/
-	{GEN_CMD_CODE(_Write_MACREG), NULL},
-	{GEN_CMD_CODE(_Read_BBREG), &rtw_getbbrfreg_cmdrsp_callback23a},
-	{GEN_CMD_CODE(_Write_BBREG), NULL},
-	{GEN_CMD_CODE(_Read_RFREG), &rtw_getbbrfreg_cmdrsp_callback23a},
-	{GEN_CMD_CODE(_Write_RFREG), NULL}, /*5*/
-	{GEN_CMD_CODE(_Read_EEPROM), NULL},
-	{GEN_CMD_CODE(_Write_EEPROM), NULL},
-	{GEN_CMD_CODE(_Read_EFUSE), NULL},
-	{GEN_CMD_CODE(_Write_EFUSE), NULL},
-
-	{GEN_CMD_CODE(_Read_CAM),	NULL},	/*10*/
-	{GEN_CMD_CODE(_Write_CAM),	 NULL},
-	{GEN_CMD_CODE(_setBCNITV), NULL},
-	{GEN_CMD_CODE(_setMBIDCFG), NULL},
-	{GEN_CMD_CODE(_JoinBss), &rtw_joinbss_cmd23a_callback},  /*14*/
-	{GEN_CMD_CODE(_DisConnect), &rtw_disassoc_cmd23a_callback}, /*15*/
-	{GEN_CMD_CODE(_CreateBss), &rtw_createbss_cmd23a_callback},
-	{GEN_CMD_CODE(_SetOpMode), NULL},
-	{GEN_CMD_CODE(_SiteSurvey), &rtw_survey_cmd_callback23a}, /*18*/
-	{GEN_CMD_CODE(_SetAuth), NULL},
-
-	{GEN_CMD_CODE(_SetKey), NULL},	/*20*/
-	{GEN_CMD_CODE(_SetStaKey), &rtw_setstaKey_cmdrsp_callback23a},
-	{GEN_CMD_CODE(_SetAssocSta), &rtw_setassocsta_cmdrsp_callback23a},
-	{GEN_CMD_CODE(_DelAssocSta), NULL},
-	{GEN_CMD_CODE(_SetStaPwrState), NULL},
-	{GEN_CMD_CODE(_SetBasicRate), NULL}, /*25*/
-	{GEN_CMD_CODE(_GetBasicRate), NULL},
-	{GEN_CMD_CODE(_SetDataRate), NULL},
-	{GEN_CMD_CODE(_GetDataRate), NULL},
-	{GEN_CMD_CODE(_SetPhyInfo), NULL},
-
-	{GEN_CMD_CODE(_GetPhyInfo), NULL}, /*30*/
-	{GEN_CMD_CODE(_SetPhy), NULL},
-	{GEN_CMD_CODE(_GetPhy), NULL},
-	{GEN_CMD_CODE(_readRssi), NULL},
-	{GEN_CMD_CODE(_readGain), NULL},
-	{GEN_CMD_CODE(_SetAtim), NULL}, /*35*/
-	{GEN_CMD_CODE(_SetPwrMode), NULL},
-	{GEN_CMD_CODE(_JoinbssRpt), NULL},
-	{GEN_CMD_CODE(_SetRaTable), NULL},
-	{GEN_CMD_CODE(_GetRaTable), NULL},
-
-	{GEN_CMD_CODE(_GetCCXReport), NULL}, /*40*/
-	{GEN_CMD_CODE(_GetDTMReport),	NULL},
-	{GEN_CMD_CODE(_GetTXRateStatistics), NULL},
-	{GEN_CMD_CODE(_SetUsbSuspend), NULL},
-	{GEN_CMD_CODE(_SetH2cLbk), NULL},
-	{GEN_CMD_CODE(_AddBAReq), NULL}, /*45*/
-	{GEN_CMD_CODE(_SetChannel), NULL},		/*46*/
-	{GEN_CMD_CODE(_SetTxPower), NULL},
-	{GEN_CMD_CODE(_SwitchAntenna), NULL},
-	{GEN_CMD_CODE(_SetCrystalCap), NULL},
-	{GEN_CMD_CODE(_SetSingleCarrierTx), NULL},	/*50*/
-
-	{GEN_CMD_CODE(_SetSingleToneTx), NULL}, /*51*/
-	{GEN_CMD_CODE(_SetCarrierSuppressionTx), NULL},
-	{GEN_CMD_CODE(_SetContinuousTx), NULL},
-	{GEN_CMD_CODE(_SwitchBandwidth), NULL},		/*54*/
-	{GEN_CMD_CODE(_TX_Beacon), NULL},/*55*/
-
-	{GEN_CMD_CODE(_Set_MLME_EVT), NULL},/*56*/
-	{GEN_CMD_CODE(_Set_Drv_Extra), NULL},/*57*/
-	{GEN_CMD_CODE(_Set_H2C_MSG), NULL},/*58*/
-	{GEN_CMD_CODE(_SetChannelPlan), NULL},/*59*/
-	{GEN_CMD_CODE(_LedBlink), NULL},/*60*/
-
-	{GEN_CMD_CODE(_SetChannelSwitch), NULL},/*61*/
-	{GEN_CMD_CODE(_TDLS), NULL},/*62*/
-};
-
-/*
-Caller and the rtw_cmd_thread23a can protect cmd_q by spin_lock.
-No irqsave is necessary.
-*/
-
-int rtw_init_cmd_priv23a(struct cmd_priv *pcmdpriv)
-{
-	int res = _SUCCESS;
-
-	pcmdpriv->cmd_issued_cnt = 0;
-	pcmdpriv->cmd_done_cnt = 0;
-	pcmdpriv->rsp_cnt = 0;
-
-	pcmdpriv->wq = alloc_workqueue("rtl8723au_cmd", 0, 1);
-	if (!pcmdpriv->wq)
-		res = _FAIL;
-
-	return res;
-}
-
-/* forward definition */
-
-static void rtw_irq_work(struct work_struct *work);
-
-u32 rtw_init_evt_priv23a(struct evt_priv *pevtpriv)
-{
-	pevtpriv->wq = alloc_workqueue("rtl8723au_evt", 0, 1);
-
-	INIT_WORK(&pevtpriv->irq_wk, rtw_irq_work);
-
-	return _SUCCESS;
-}
-
-void rtw_free_evt_priv23a(struct evt_priv *pevtpriv)
-{
-	cancel_work_sync(&pevtpriv->irq_wk);
-}
-
-static int rtw_cmd_filter(struct cmd_priv *pcmdpriv, struct cmd_obj *cmd_obj)
-{
-	/* set to true to allow enqueuing cmd when hw_init_completed is false */
-	u8 bAllow = false;
-
-	if (cmd_obj->cmdcode == GEN_CMD_CODE(_SetChannelPlan))
-		bAllow = true;
-
-	if (pcmdpriv->padapter->hw_init_completed == false && bAllow == false)
-		return _FAIL;
-	return _SUCCESS;
-}
-
-static void rtw_cmd_work(struct work_struct *work);
-
-int rtw_enqueue_cmd23a(struct cmd_priv *pcmdpriv, struct cmd_obj *cmd_obj)
-{
-	int res = _FAIL;
-
-	if (!cmd_obj)
-		goto exit;
-
-	cmd_obj->padapter = pcmdpriv->padapter;
-
-	res = rtw_cmd_filter(pcmdpriv, cmd_obj);
-	if (res == _FAIL) {
-		rtw_free_cmd_obj23a(cmd_obj);
-		goto exit;
-	}
-
-	INIT_WORK(&cmd_obj->work, rtw_cmd_work);
-
-	res = queue_work(pcmdpriv->wq, &cmd_obj->work);
-
-	if (!res) {
-		netdev_err(pcmdpriv->padapter->pnetdev,
-			   "%s: Call to queue_work() failed\n", __func__);
-		res = _FAIL;
-	} else
-		res = _SUCCESS;
-exit:
-
-	return res;
-}
-
-void rtw_free_cmd_obj23a(struct cmd_obj *pcmd)
-{
-
-	if (pcmd->cmdcode != _JoinBss_CMD_ &&
-	    pcmd->cmdcode != _CreateBss_CMD_) {
-		/* free parmbuf in cmd_obj */
-		kfree(pcmd->parmbuf);
-	}
-
-	if (pcmd->rsp) {
-		if (pcmd->rspsz != 0) {
-			/* free rsp in cmd_obj */
-			kfree(pcmd->rsp);
-		}
-	}
-
-	kfree(pcmd);
-}
-
-static void rtw_cmd_work(struct work_struct *work)
-{
-	int (*cmd_hdl)(struct rtw_adapter *padapter, const u8 *pbuf);
-	void (*pcmd_callback)(struct rtw_adapter *dev, struct cmd_obj *pcmd);
-	struct cmd_priv *pcmdpriv;
-	struct cmd_obj *pcmd = container_of(work, struct cmd_obj, work);
-
-	pcmdpriv = &pcmd->padapter->cmdpriv;
-
-	if (rtw_cmd_filter(pcmdpriv, pcmd) == _FAIL) {
-		pcmd->res = H2C_DROPPED;
-		goto post_process;
-	}
-
-	pcmdpriv->cmd_issued_cnt++;
-
-	pcmd->cmdsz = ALIGN(pcmd->cmdsz, 4);
-
-	if (pcmd->cmdcode < (sizeof(wlancmds)/sizeof(struct cmd_hdl))) {
-		cmd_hdl = wlancmds[pcmd->cmdcode].h2cfuns;
-
-		if (cmd_hdl)
-			pcmd->res = cmd_hdl(pcmd->padapter, pcmd->parmbuf);
-		else
-			pcmd->res = H2C_DROPPED;
-	} else
-		pcmd->res = H2C_PARAMETERS_ERROR;
-
-post_process:
-	/* call callback function for post-processed */
-	if (pcmd->cmdcode < ARRAY_SIZE(rtw_cmd_callback)) {
-		pcmd_callback =	rtw_cmd_callback[pcmd->cmdcode].callback;
-		if (!pcmd_callback) {
-			RT_TRACE(_module_rtl871x_cmd_c_, _drv_info_,
-				 "mlme_cmd_hdl(): pcmd_callback = 0x%p, cmdcode = 0x%x\n",
-				 pcmd_callback, pcmd->cmdcode);
-			rtw_free_cmd_obj23a(pcmd);
-		} else {
-			/* need consider that free cmd_obj in
-			   rtw_cmd_callback */
-			pcmd_callback(pcmd->padapter, pcmd);
-		}
-	} else {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "%s: cmdcode = 0x%x callback not defined!\n",
-			 __func__, pcmd->cmdcode);
-		rtw_free_cmd_obj23a(pcmd);
-	}
-}
-
-
-int rtw_sitesurvey_cmd23a(struct rtw_adapter *padapter,
-			  struct cfg80211_ssid *ssid, int ssid_num,
-			  struct rtw_ieee80211_channel *ch, int ch_num)
-{
-	int res = _FAIL;
-	struct cmd_obj *ph2c;
-	struct sitesurvey_parm *psurveyPara;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	if (check_fwstate(pmlmepriv, _FW_LINKED))
-		rtw_lps_ctrl_wk_cmd23a(padapter, LPS_CTRL_SCAN, 1);
-
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!ph2c)
-		return _FAIL;
-
-	psurveyPara = kzalloc(sizeof(struct sitesurvey_parm), GFP_ATOMIC);
-	if (!psurveyPara) {
-		kfree(ph2c);
-		return _FAIL;
-	}
-
-	rtw_free_network_queue23a(padapter);
-
-	RT_TRACE(_module_rtl871x_cmd_c_, _drv_info_,
-		 "%s: flush network queue\n", __func__);
-
-	init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara,
-				   GEN_CMD_CODE(_SiteSurvey));
-
-	/* psurveyPara->bsslimit = 48; */
-	psurveyPara->scan_mode = pmlmepriv->scan_mode;
-
-	/* prepare ssid list */
-	if (ssid) {
-		int i;
-
-		for (i = 0; i < ssid_num && i < RTW_SSID_SCAN_AMOUNT; i++) {
-			if (ssid[i].ssid_len) {
-				memcpy(&psurveyPara->ssid[i], &ssid[i],
-				       sizeof(struct cfg80211_ssid));
-				psurveyPara->ssid_num++;
-			}
-		}
-	}
-
-	/* prepare channel list */
-	if (ch) {
-		int i;
-
-		for (i = 0; i < ch_num && i < RTW_CHANNEL_SCAN_AMOUNT; i++) {
-			if (ch[i].hw_value &&
-			    !(ch[i].flags & IEEE80211_CHAN_DISABLED)) {
-				memcpy(&psurveyPara->ch[i], &ch[i],
-				       sizeof(struct rtw_ieee80211_channel));
-				psurveyPara->ch_num++;
-			}
-		}
-	}
-
-	set_fwstate(pmlmepriv, _FW_UNDER_SURVEY);
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-
-	if (res == _SUCCESS) {
-		mod_timer(&pmlmepriv->scan_to_timer, jiffies +
-			  msecs_to_jiffies(SCANNING_TIMEOUT));
-
-		pmlmepriv->scan_interval = SCAN_INTERVAL;/*  30*2 sec = 60sec */
-	} else
-		_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY);
-
-	return res;
-}
-
-void rtw_getbbrfreg_cmdrsp_callback23a(struct rtw_adapter *padapter,
-				       struct cmd_obj *pcmd)
-{
-	kfree(pcmd->parmbuf);
-	kfree(pcmd);
-}
-
-int rtw_createbss_cmd23a(struct rtw_adapter  *padapter)
-{
-	struct cmd_obj *pcmd;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_bssid_ex *pdev_network;
-	u8 res = _SUCCESS;
-
-	pdev_network = &padapter->registrypriv.dev_network;
-
-	if (pmlmepriv->assoc_ssid.ssid_len == 0) {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_info_,
-			 "createbss for Any SSid:%s\n",
-			 pmlmepriv->assoc_ssid.ssid);
-	} else {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_info_,
-			 "createbss for SSid:%s\n",
-			 pmlmepriv->assoc_ssid.ssid);
-	}
-
-	pcmd = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!pcmd) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	pcmd->cmdcode = _CreateBss_CMD_;
-	pcmd->parmbuf = (unsigned char *)pdev_network;
-	pcmd->cmdsz = get_wlan_bssid_ex_sz(pdev_network);
-	pcmd->rsp = NULL;
-	pcmd->rspsz = 0;
-
-	pdev_network->Length = pcmd->cmdsz;
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, pcmd);
-
-exit:
-
-	return res;
-}
-
-int rtw_joinbss_cmd23a(struct rtw_adapter *padapter,
-		       struct wlan_network *pnetwork)
-{
-	int res = _SUCCESS;
-	struct wlan_bssid_ex *psecnetwork;
-	struct cmd_obj *pcmd;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-	enum nl80211_iftype ifmode;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	ifmode = pnetwork->network.ifmode;
-
-	if (pmlmepriv->assoc_ssid.ssid_len == 0) {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_info_,
-			 "+Join cmd: Any SSid\n");
-	} else {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_notice_,
-			 "+Join cmd: SSid =[%s]\n",
-			 pmlmepriv->assoc_ssid.ssid);
-	}
-
-	pcmd = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!pcmd) {
-		res = _FAIL;
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "rtw_joinbss_cmd23a: memory allocate for cmd_obj fail!!!\n");
-		goto exit;
-	}
-
-	/* for hidden ap to set fw_state here */
-	if (!check_fwstate(pmlmepriv, WIFI_STATION_STATE|WIFI_ADHOC_STATE)) {
-		switch (ifmode) {
-		case NL80211_IFTYPE_ADHOC:
-			set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
-			break;
-		case NL80211_IFTYPE_P2P_CLIENT:
-		case NL80211_IFTYPE_STATION:
-			set_fwstate(pmlmepriv, WIFI_STATION_STATE);
-			break;
-		default:
-			break;
-		}
-	}
-
-	psecnetwork = &psecuritypriv->sec_bss;
-	if (!psecnetwork) {
-		kfree(pcmd);
-		res = _FAIL;
-
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "rtw_joinbss_cmd23a :psecnetwork == NULL!!!\n");
-
-		goto exit;
-	}
-
-	memset(psecnetwork, 0, sizeof(struct wlan_bssid_ex));
-
-	memcpy(psecnetwork, &pnetwork->network,
-	       get_wlan_bssid_ex_sz(&pnetwork->network));
-
-	psecnetwork->IELength = 0;
-	/*  Added by Albert 2009/02/18 */
-	/*  If the the driver wants to use the bssid to create the
-	 *  connection. If not,  we have to copy the connecting AP's
-	 *  MAC address to it so that the driver just has the bssid
-	 *  information for PMKIDList searching. */
-
-	if (pmlmepriv->assoc_by_bssid == false)
-		ether_addr_copy(&pmlmepriv->assoc_bssid[0],
-				&pnetwork->network.MacAddress[0]);
-
-	psecnetwork->IELength =
-		rtw_restruct_sec_ie23a(padapter, &pnetwork->network.IEs[0],
-				       &psecnetwork->IEs[0],
-				       pnetwork->network.IELength);
-
-	pmlmepriv->qos_option = 0;
-
-	if (pregistrypriv->wmm_enable) {
-		u32 tmp_len;
-
-		tmp_len = rtw_restruct_wmm_ie23a(padapter,
-						 &pnetwork->network.IEs[0],
-						 &psecnetwork->IEs[0],
-						 pnetwork->network.IELength,
-						 psecnetwork->IELength);
-
-		if (psecnetwork->IELength != tmp_len) {
-			psecnetwork->IELength = tmp_len;
-			/* There is WMM IE in this corresp. beacon */
-			pmlmepriv->qos_option = 1;
-		} else {
-			/* There is no WMM IE in this corresp. beacon */
-			pmlmepriv->qos_option = 0;
-		}
-	}
-
-	phtpriv->ht_option = false;
-	if (pregistrypriv->ht_enable) {
-		u32 algo = padapter->securitypriv.dot11PrivacyAlgrthm;
-		/*	Added by Albert 2010/06/23 */
-		/*	For the WEP mode, we will use the bg mode to do
-			the connection to avoid some IOT issue. */
-		/*	Especially for Realtek 8192u SoftAP. */
-		if (algo != WLAN_CIPHER_SUITE_WEP40 &&
-		    algo != WLAN_CIPHER_SUITE_WEP104 &&
-		    algo != WLAN_CIPHER_SUITE_TKIP) {
-			/* rtw_restructure_ht_ie23a */
-			rtw_restructure_ht_ie23a(padapter,
-						 &pnetwork->network.IEs[0],
-						 &psecnetwork->IEs[0],
-						 pnetwork->network.IELength,
-						 &psecnetwork->IELength);
-		}
-	}
-
-	pmlmeinfo->assoc_AP_vendor =
-		check_assoc_AP23a(pnetwork->network.IEs,
-				  pnetwork->network.IELength);
-
-	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_TENDA)
-		padapter->pwrctrlpriv.smart_ps = 0;
-	else
-		padapter->pwrctrlpriv.smart_ps =
-			padapter->registrypriv.smart_ps;
-
-	DBG_8723A("%s: smart_ps =%d\n", __func__,
-		  padapter->pwrctrlpriv.smart_ps);
-
-	/* get cmdsz before endian conversion */
-	pcmd->cmdsz = get_wlan_bssid_ex_sz(psecnetwork);
-
-	pcmd->cmdcode = _JoinBss_CMD_;/* GEN_CMD_CODE(_JoinBss) */
-	pcmd->parmbuf = (unsigned char *)psecnetwork;
-	pcmd->rsp = NULL;
-	pcmd->rspsz = 0;
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, pcmd);
-exit:
-
-	return res;
-}
-
-int rtw_disassoc_cmd23a(struct rtw_adapter *padapter, u32 deauth_timeout_ms,
-			bool enqueue)
-{
-	struct cmd_obj *cmdobj = NULL;
-	struct disconnect_parm *param = NULL;
-	struct cmd_priv *cmdpriv = &padapter->cmdpriv;
-	int res = _SUCCESS;
-
-	RT_TRACE(_module_rtl871x_cmd_c_, _drv_notice_,
-		 "+rtw_disassoc_cmd23a\n");
-
-	/* prepare cmd parameter */
-	param = kzalloc(sizeof(*param), GFP_ATOMIC);
-	if (param == NULL) {
-		res = _FAIL;
-		goto exit;
-	}
-	param->deauth_timeout_ms = deauth_timeout_ms;
-
-	if (enqueue) {
-		/* need enqueue, prepare cmd_obj and enqueue */
-		cmdobj = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-		if (!cmdobj) {
-			res = _FAIL;
-			kfree(param);
-			goto exit;
-		}
-		init_h2fwcmd_w_parm_no_rsp(cmdobj, param, _DisConnect_CMD_);
-		res = rtw_enqueue_cmd23a(cmdpriv, cmdobj);
-	} else {
-		/* no need to enqueue, do the cmd hdl directly and
-		   free cmd parameter */
-		if (disconnect_hdl23a(padapter, (u8 *)param) != H2C_SUCCESS)
-			res = _FAIL;
-		kfree(param);
-	}
-
-exit:
-	return res;
-}
-
-int rtw_setopmode_cmd23a(struct rtw_adapter *padapter,
-			 enum nl80211_iftype ifmode)
-{
-	struct	cmd_obj *ph2c;
-	struct	setopmode_parm *psetop;
-	struct	cmd_priv   *pcmdpriv = &padapter->cmdpriv;
-	int res = _SUCCESS;
-
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_KERNEL);
-	if (!ph2c) {
-		res = false;
-		goto exit;
-	}
-	psetop = kzalloc(sizeof(struct setopmode_parm), GFP_KERNEL);
-
-	if (!psetop) {
-		kfree(ph2c);
-		res = false;
-		goto exit;
-	}
-
-	init_h2fwcmd_w_parm_no_rsp(ph2c, psetop, _SetOpMode_CMD_);
-	psetop->mode = ifmode;
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-exit:
-	return res;
-}
-
-int rtw_setstakey_cmd23a(struct rtw_adapter *padapter, u8 *psta, u8 unicast_key)
-{
-	struct cmd_obj *ph2c;
-	struct set_stakey_parm *psetstakey_para;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	struct set_stakey_rsp *psetstakey_rsp = NULL;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct sta_info *sta = (struct sta_info *)psta;
-	int res = _SUCCESS;
-
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_KERNEL);
-	if (!ph2c) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	psetstakey_para = kzalloc(sizeof(struct set_stakey_parm), GFP_KERNEL);
-	if (!psetstakey_para) {
-		kfree(ph2c);
-		res = _FAIL;
-		goto exit;
-	}
-
-	psetstakey_rsp = kzalloc(sizeof(struct set_stakey_rsp), GFP_KERNEL);
-	if (!psetstakey_rsp) {
-		kfree(ph2c);
-		kfree(psetstakey_para);
-		res = _FAIL;
-		goto exit;
-	}
-
-	init_h2fwcmd_w_parm_no_rsp(ph2c, psetstakey_para, _SetStaKey_CMD_);
-	ph2c->rsp = (u8 *) psetstakey_rsp;
-	ph2c->rspsz = sizeof(struct set_stakey_rsp);
-
-	ether_addr_copy(psetstakey_para->addr, sta->hwaddr);
-
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
-		psetstakey_para->algorithm =
-			(unsigned char)psecuritypriv->dot11PrivacyAlgrthm;
-	} else {
-		GET_ENCRY_ALGO(psecuritypriv, sta, psetstakey_para->algorithm,
-			       false);
-	}
-
-	if (unicast_key == true) {
-		memcpy(&psetstakey_para->key, &sta->dot118021x_UncstKey, 16);
-	} else {
-		int idx = psecuritypriv->dot118021XGrpKeyid;
-
-		memcpy(&psetstakey_para->key,
-		       &psecuritypriv->dot118021XGrpKey[idx].skey, 16);
-	}
-
-	/* jeff: set this because at least sw key is ready */
-	padapter->securitypriv.busetkipkey = 1;
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-
-exit:
-
-	return res;
-}
-
-int rtw_clearstakey_cmd23a(struct rtw_adapter *padapter, u8 *psta, u8 entry,
-			   u8 enqueue)
-{
-	struct cmd_obj *ph2c;
-	struct set_stakey_parm *psetstakey_para;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	struct set_stakey_rsp *psetstakey_rsp = NULL;
-	struct sta_info *sta = (struct sta_info *)psta;
-	int res = _SUCCESS;
-
-	if (!enqueue) {
-		clear_cam_entry23a(padapter, entry);
-	} else {
-		ph2c = kzalloc(sizeof(struct cmd_obj), GFP_KERNEL);
-		if (!ph2c) {
-			res = _FAIL;
-			goto exit;
-		}
-
-		psetstakey_para = kzalloc(sizeof(struct set_stakey_parm),
-					  GFP_KERNEL);
-		if (!psetstakey_para) {
-			kfree(ph2c);
-			res = _FAIL;
-			goto exit;
-		}
-
-		psetstakey_rsp = kzalloc(sizeof(struct set_stakey_rsp),
-					 GFP_KERNEL);
-		if (!psetstakey_rsp) {
-			kfree(ph2c);
-			kfree(psetstakey_para);
-			res = _FAIL;
-			goto exit;
-		}
-
-		init_h2fwcmd_w_parm_no_rsp(ph2c, psetstakey_para,
-					   _SetStaKey_CMD_);
-		ph2c->rsp = (u8 *) psetstakey_rsp;
-		ph2c->rspsz = sizeof(struct set_stakey_rsp);
-
-		ether_addr_copy(psetstakey_para->addr, sta->hwaddr);
-
-		psetstakey_para->algorithm = 0;
-
-		psetstakey_para->id = entry;
-
-		res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-	}
-exit:
-	return res;
-}
-
-int rtw_addbareq_cmd23a(struct rtw_adapter *padapter, u8 tid, u8 *addr)
-{
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	struct cmd_obj *ph2c;
-	struct addBaReq_parm *paddbareq_parm;
-	int res = _SUCCESS;
-
-	if (tid >= MAXTID) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!ph2c) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	paddbareq_parm = kzalloc(sizeof(struct addBaReq_parm), GFP_ATOMIC);
-	if (!paddbareq_parm) {
-		kfree(ph2c);
-		res = _FAIL;
-		goto exit;
-	}
-
-	paddbareq_parm->tid = tid;
-	ether_addr_copy(paddbareq_parm->addr, addr);
-
-	init_h2fwcmd_w_parm_no_rsp(ph2c, paddbareq_parm,
-				   GEN_CMD_CODE(_AddBAReq));
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-exit:
-	return res;
-}
-
-int rtw_dynamic_chk_wk_cmd23a(struct rtw_adapter *padapter)
-{
-	struct cmd_obj *ph2c;
-	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	int res = _SUCCESS;
-
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!ph2c) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC);
-	if (!pdrvextra_cmd_parm) {
-		kfree(ph2c);
-		res = _FAIL;
-		goto exit;
-	}
-
-	pdrvextra_cmd_parm->ec_id = DYNAMIC_CHK_WK_CID;
-	pdrvextra_cmd_parm->type_size = 0;
-	pdrvextra_cmd_parm->pbuf = (u8 *)padapter;
-
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm,
-				   GEN_CMD_CODE(_Set_Drv_Extra));
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-exit:
-
-	return res;
-}
-
-static void traffic_status_watchdog(struct rtw_adapter *padapter)
-{
-	u8 bEnterPS;
-	u8 bBusyTraffic = false, bTxBusyTraffic = false, bRxBusyTraffic = false;
-	u8 bHigherBusyTraffic = false, bHigherBusyRxTraffic = false;
-	u8 bHigherBusyTxTraffic = false;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	int BusyThreshold = 100;
-	struct rt_link_detect *ldi = &pmlmepriv->LinkDetectInfo;
-
-	/*  */
-	/*  Determine if our traffic is busy now */
-	/*  */
-	if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-		if (rtl8723a_BT_coexist(padapter))
-			BusyThreshold = 50;
-		else if (ldi->bBusyTraffic)
-			BusyThreshold = 75;
-		/*  if we raise bBusyTraffic in last watchdog, using
-		    lower threshold. */
-		if (ldi->NumRxOkInPeriod > BusyThreshold ||
-		    ldi->NumTxOkInPeriod > BusyThreshold) {
-			bBusyTraffic = true;
-
-			if (ldi->NumRxOkInPeriod > ldi->NumTxOkInPeriod)
-				bRxBusyTraffic = true;
-			else
-				bTxBusyTraffic = true;
-		}
-
-		/*  Higher Tx/Rx data. */
-		if (ldi->NumRxOkInPeriod > 4000 ||
-		    ldi->NumTxOkInPeriod > 4000) {
-			bHigherBusyTraffic = true;
-
-			if (ldi->NumRxOkInPeriod > ldi->NumTxOkInPeriod)
-				bHigherBusyRxTraffic = true;
-			else
-				bHigherBusyTxTraffic = true;
-		}
-
-		if (!rtl8723a_BT_coexist(padapter) ||
-		    !rtl8723a_BT_using_antenna_1(padapter)) {
-		/*  check traffic for  powersaving. */
-			if (((ldi->NumRxUnicastOkInPeriod +
-			      ldi->NumTxOkInPeriod) > 8) ||
-			    ldi->NumRxUnicastOkInPeriod > 2)
-				bEnterPS = false;
-			else
-				bEnterPS = true;
-
-			/*  LeisurePS only work in infra mode. */
-			if (bEnterPS)
-				LPS_Enter23a(padapter);
-			else
-				LPS_Leave23a(padapter);
-		}
-	} else
-		LPS_Leave23a(padapter);
-
-	ldi->NumRxOkInPeriod = 0;
-	ldi->NumTxOkInPeriod = 0;
-	ldi->NumRxUnicastOkInPeriod = 0;
-	ldi->bBusyTraffic = bBusyTraffic;
-	ldi->bTxBusyTraffic = bTxBusyTraffic;
-	ldi->bRxBusyTraffic = bRxBusyTraffic;
-	ldi->bHigherBusyTraffic = bHigherBusyTraffic;
-	ldi->bHigherBusyRxTraffic = bHigherBusyRxTraffic;
-	ldi->bHigherBusyTxTraffic = bHigherBusyTxTraffic;
-}
-
-static void dynamic_chk_wk_hdl(struct rtw_adapter *padapter, u8 *pbuf, int sz)
-{
-	struct mlme_priv *pmlmepriv;
-
-	padapter = (struct rtw_adapter *)pbuf;
-	pmlmepriv = &padapter->mlmepriv;
-
-#ifdef CONFIG_8723AU_AP_MODE
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
-		expire_timeout_chk23a(padapter);
-#endif
-
-	rtl8723a_sreset_xmit_status_check(padapter);
-
-	linked_status_chk23a(padapter);
-	traffic_status_watchdog(padapter);
-
-	rtl8723a_HalDmWatchDog(padapter);
-
-	/*  */
-	/*  BT-Coexist */
-	/*  */
-	rtl8723a_BT_do_coexist(padapter);
-}
-
-static void lps_ctrl_wk_hdl(struct rtw_adapter *padapter, u8 lps_ctrl_type)
-{
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	u8 mstatus;
-
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE))
-		return;
-
-	switch (lps_ctrl_type) {
-	case LPS_CTRL_SCAN:
-		rtl8723a_BT_wifiscan_notify(padapter, true);
-		if (!rtl8723a_BT_using_antenna_1(padapter)) {
-			if (check_fwstate(pmlmepriv, _FW_LINKED))
-				LPS_Leave23a(padapter);
-			}
-		break;
-	case LPS_CTRL_JOINBSS:
-		LPS_Leave23a(padapter);
-		break;
-	case LPS_CTRL_CONNECT:
-		mstatus = 1;/* connect */
-		/*  Reset LPS Setting */
-		padapter->pwrctrlpriv.LpsIdleCount = 0;
-		rtl8723a_set_FwJoinBssReport_cmd(padapter, 1);
-		rtl8723a_BT_mediastatus_notify(padapter, mstatus);
-		break;
-	case LPS_CTRL_DISCONNECT:
-		mstatus = 0;/* disconnect */
-		rtl8723a_BT_mediastatus_notify(padapter, mstatus);
-		if (!rtl8723a_BT_using_antenna_1(padapter))
-			LPS_Leave23a(padapter);
-		rtl8723a_set_FwJoinBssReport_cmd(padapter, 0);
-		break;
-	case LPS_CTRL_SPECIAL_PACKET:
-		pwrpriv->DelayLPSLastTimeStamp = jiffies;
-		rtl8723a_BT_specialpacket_notify(padapter);
-		if (!rtl8723a_BT_using_antenna_1(padapter))
-			LPS_Leave23a(padapter);
-		break;
-	case LPS_CTRL_LEAVE:
-		rtl8723a_BT_lps_leave(padapter);
-		if (!rtl8723a_BT_using_antenna_1(padapter))
-			LPS_Leave23a(padapter);
-		break;
-
-	default:
-		break;
-	}
-}
-
-int rtw_lps_ctrl_wk_cmd23a(struct rtw_adapter *padapter,
-			   u8 lps_ctrl_type, u8 enqueue)
-{
-	struct cmd_obj *ph2c;
-	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	int res = _SUCCESS;
-
-	if (enqueue) {
-		ph2c = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-		if (!ph2c) {
-			res = _FAIL;
-			goto exit;
-		}
-
-		pdrvextra_cmd_parm = kzalloc(sizeof(struct drvextra_cmd_parm),
-					     GFP_ATOMIC);
-		if (!pdrvextra_cmd_parm) {
-			kfree(ph2c);
-			res = _FAIL;
-			goto exit;
-		}
-
-		pdrvextra_cmd_parm->ec_id = LPS_CTRL_WK_CID;
-		pdrvextra_cmd_parm->type_size = lps_ctrl_type;
-		pdrvextra_cmd_parm->pbuf = NULL;
-
-		init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm,
-					   GEN_CMD_CODE(_Set_Drv_Extra));
-
-		res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-	} else
-		lps_ctrl_wk_hdl(padapter, lps_ctrl_type);
-exit:
-
-	return res;
-}
-
-int rtw_ps_cmd23a(struct rtw_adapter *padapter)
-{
-	struct cmd_obj *ppscmd;
-	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	int res = _SUCCESS;
-
-	ppscmd = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!ppscmd) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	pdrvextra_cmd_parm = kzalloc(sizeof(struct drvextra_cmd_parm),
-				     GFP_ATOMIC);
-	if (!pdrvextra_cmd_parm) {
-		kfree(ppscmd);
-		res = _FAIL;
-		goto exit;
-	}
-
-	pdrvextra_cmd_parm->ec_id = POWER_SAVING_CTRL_WK_CID;
-	pdrvextra_cmd_parm->pbuf = NULL;
-	init_h2fwcmd_w_parm_no_rsp(ppscmd, pdrvextra_cmd_parm,
-				   GEN_CMD_CODE(_Set_Drv_Extra));
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ppscmd);
-exit:
-
-	return res;
-}
-
-#ifdef CONFIG_8723AU_AP_MODE
-
-static void rtw_chk_hi_queue_hdl(struct rtw_adapter *padapter)
-{
-	int cnt = 0;
-	struct sta_info *psta_bmc;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-
-	psta_bmc = rtw_get_bcmc_stainfo23a(padapter);
-	if (!psta_bmc)
-		return;
-
-	if (psta_bmc->sleepq_len == 0) {
-		bool val;
-
-		val = rtl8723a_chk_hi_queue_empty(padapter);
-
-		while (!val) {
-			msleep(100);
-
-			cnt++;
-
-			if (cnt > 10)
-				break;
-
-			val = rtl8723a_chk_hi_queue_empty(padapter);
-		}
-
-		if (cnt <= 10) {
-			pstapriv->tim_bitmap &= ~BIT(0);
-			pstapriv->sta_dz_bitmap &= ~BIT(0);
-
-			update_beacon23a(padapter, WLAN_EID_TIM, NULL, false);
-		} else /* re check again */
-			rtw_chk_hi_queue_cmd23a(padapter);
-	}
-}
-
-int rtw_chk_hi_queue_cmd23a(struct rtw_adapter *padapter)
-{
-	struct cmd_obj *ph2c;
-	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	int res = _SUCCESS;
-
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!ph2c) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	pdrvextra_cmd_parm = kzalloc(sizeof(struct drvextra_cmd_parm),
-				     GFP_ATOMIC);
-	if (!pdrvextra_cmd_parm) {
-		kfree(ph2c);
-		res = _FAIL;
-		goto exit;
-	}
-
-	pdrvextra_cmd_parm->ec_id = CHECK_HIQ_WK_CID;
-	pdrvextra_cmd_parm->type_size = 0;
-	pdrvextra_cmd_parm->pbuf = NULL;
-
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm,
-				   GEN_CMD_CODE(_Set_Drv_Extra));
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-exit:
-
-	return res;
-}
-#endif
-
-int rtw_c2h_wk_cmd23a(struct rtw_adapter *padapter, u8 *c2h_evt)
-{
-	struct cmd_obj *ph2c;
-	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	int res = _SUCCESS;
-
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!ph2c) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	pdrvextra_cmd_parm = kzalloc(sizeof(struct drvextra_cmd_parm),
-				     GFP_ATOMIC);
-	if (!pdrvextra_cmd_parm) {
-		kfree(ph2c);
-		res = _FAIL;
-		goto exit;
-	}
-
-	pdrvextra_cmd_parm->ec_id = C2H_WK_CID;
-	pdrvextra_cmd_parm->type_size = c2h_evt?16:0;
-	pdrvextra_cmd_parm->pbuf = c2h_evt;
-
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm,
-				   GEN_CMD_CODE(_Set_Drv_Extra));
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-
-exit:
-
-	return res;
-}
-
-static int c2h_evt_hdl(struct rtw_adapter *adapter, struct c2h_evt_hdr *c2h_evt)
-{
-	int ret = _FAIL;
-	u8 buf[16];
-
-	if (!c2h_evt) {
-		/* No c2h event in cmd_obj, read c2h event before handling*/
-		if (c2h_evt_read23a(adapter, buf) == _SUCCESS) {
-			c2h_evt = (struct c2h_evt_hdr *)buf;
-
-			ret = c2h_handler_8723a(adapter, c2h_evt);
-		}
-	} else
-		ret = c2h_handler_8723a(adapter, c2h_evt);
-
-	return ret;
-}
-
-static void rtw_irq_work(struct work_struct *work)
-{
-	struct evt_priv *evtpriv;
-	struct rtw_adapter *adapter;
-
-	evtpriv = container_of(work, struct evt_priv, irq_wk);
-	adapter = container_of(evtpriv, struct rtw_adapter, evtpriv);
-
-	c2h_evt_clear23a(adapter);
-}
-
-void rtw_evt_work(struct work_struct *work)
-{
-	struct evt_work *ework;
-	struct rtw_adapter *adapter;
-
-	ework = container_of(work, struct evt_work, work);
-	adapter = ework->adapter;
-
-	c2h_evt_clear23a(adapter);
-
-	if (!c2h_evt_exist(&ework->u.c2h_evt)) {
-		kfree(ework);
-		return;
-	}
-
-	if (c2h_id_filter_ccx_8723a(ework->u.c2h_evt.id) == true) {
-		/* Handle CCX report here */
-		c2h_handler_8723a(adapter, &ework->u.c2h_evt);
-		kfree(ework);
-	} else {
-		/*
-		 * Enqueue into cmd_thread for others.
-		 * ework will be turned into a c2h_evt and freed once it
-		 * has been consumed.
-		 */
-		rtw_c2h_wk_cmd23a(adapter, (u8 *)&ework->u.c2h_evt);
-	}
-}
-
-int rtw_drvextra_cmd_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	const struct drvextra_cmd_parm *pdrvextra_cmd;
-
-	if (!pbuf)
-		return H2C_PARAMETERS_ERROR;
-
-	pdrvextra_cmd = (struct drvextra_cmd_parm *)pbuf;
-
-	switch (pdrvextra_cmd->ec_id) {
-	case DYNAMIC_CHK_WK_CID:
-		dynamic_chk_wk_hdl(padapter, pdrvextra_cmd->pbuf,
-				   pdrvextra_cmd->type_size);
-		break;
-	case POWER_SAVING_CTRL_WK_CID:
-		rtw_ps_processor23a(padapter);
-		break;
-	case LPS_CTRL_WK_CID:
-		lps_ctrl_wk_hdl(padapter, (u8)pdrvextra_cmd->type_size);
-		break;
-#ifdef CONFIG_8723AU_AP_MODE
-	case CHECK_HIQ_WK_CID:
-		rtw_chk_hi_queue_hdl(padapter);
-		break;
-#endif /* CONFIG_8723AU_AP_MODE */
-	case C2H_WK_CID:
-		c2h_evt_hdl(padapter,
-			    (struct c2h_evt_hdr *)pdrvextra_cmd->pbuf);
-		break;
-
-	default:
-		break;
-	}
-
-	if (pdrvextra_cmd->pbuf && (pdrvextra_cmd->type_size > 0)) {
-		kfree(pdrvextra_cmd->pbuf);
-		/*
-		 * No need to set pdrvextra_cmd->pbuf = NULL as we were
-		 * operating on a copy of the original pcmd->parmbuf
-		 * created in rtw_cmd_work().
-		 */
-	}
-
-	return H2C_SUCCESS;
-}
-
-void rtw_survey_cmd_callback23a(struct rtw_adapter *padapter,
-				struct cmd_obj *pcmd)
-{
-	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	if (pcmd->res == H2C_DROPPED) {
-		/* TODO: cancel timer and do timeout handler directly... */
-		/* need to make timeout handlerOS independent */
-		mod_timer(&pmlmepriv->scan_to_timer,
-			  jiffies + msecs_to_jiffies(1));
-	} else if (pcmd->res != H2C_SUCCESS) {
-		mod_timer(&pmlmepriv->scan_to_timer,
-			  jiffies + msecs_to_jiffies(1));
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "********Error: MgntActrtw_set_802_11_bssid23a_LIST_SCAN Fail ************\n");
-	}
-
-	/*  free cmd */
-	rtw_free_cmd_obj23a(pcmd);
-}
-
-void rtw_disassoc_cmd23a_callback(struct rtw_adapter *padapter,
-				  struct cmd_obj *pcmd)
-{
-	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	if (pcmd->res != H2C_SUCCESS) {
-		spin_lock_bh(&pmlmepriv->lock);
-		set_fwstate(pmlmepriv, _FW_LINKED);
-		spin_unlock_bh(&pmlmepriv->lock);
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "***Error: disconnect_cmd_callback Fail ***\n");
-		return;
-	}
-
-	/*  free cmd */
-	rtw_free_cmd_obj23a(pcmd);
-}
-
-void rtw_joinbss_cmd23a_callback(struct rtw_adapter *padapter,
-				 struct cmd_obj *pcmd)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	if (pcmd->res == H2C_DROPPED) {
-		/* TODO: cancel timer and do timeout handler directly... */
-		/* need to make timeout handlerOS independent */
-		mod_timer(&pmlmepriv->assoc_timer,
-			  jiffies + msecs_to_jiffies(1));
-	} else if (pcmd->res != H2C_SUCCESS) {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "********Error:rtw_select_and_join_from_scanned_queue Wait Sema  Fail ************\n");
-		mod_timer(&pmlmepriv->assoc_timer,
-			  jiffies + msecs_to_jiffies(1));
-	}
-
-	rtw_free_cmd_obj23a(pcmd);
-}
-
-void rtw_createbss_cmd23a_callback(struct rtw_adapter *padapter,
-				   struct cmd_obj *pcmd)
-{
-	struct sta_info *psta;
-	struct wlan_network *pwlan;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_bssid_ex *pnetwork = (struct wlan_bssid_ex *)pcmd->parmbuf;
-	struct wlan_network *tgt_network = &pmlmepriv->cur_network;
-	struct rtw_queue *scanned_queue = &pmlmepriv->scanned_queue;
-
-	if (pcmd->res != H2C_SUCCESS) {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "********Error: rtw_createbss_cmd23a_callback  Fail ************\n");
-		mod_timer(&pmlmepriv->assoc_timer,
-			  jiffies + msecs_to_jiffies(1));
-	}
-
-	del_timer_sync(&pmlmepriv->assoc_timer);
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		psta = rtw_get_stainfo23a(&padapter->stapriv,
-					  pnetwork->MacAddress);
-		if (!psta) {
-			psta = rtw_alloc_stainfo23a(&padapter->stapriv,
-						    pnetwork->MacAddress,
-						    GFP_KERNEL);
-			if (!psta) {
-				RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-					 "Can't alloc sta_info when createbss_cmd_callback\n");
-				goto createbss_cmd_fail;
-			}
-		}
-
-		spin_lock_bh(&pmlmepriv->lock);
-		rtw_indicate_connect23a(padapter);
-		spin_unlock_bh(&pmlmepriv->lock);
-	} else {
-		pwlan = rtw_alloc_network(pmlmepriv, GFP_KERNEL);
-		spin_lock_bh(&scanned_queue->lock);
-		if (!pwlan) {
-			pwlan = rtw_get_oldest_wlan_network23a(scanned_queue);
-			if (!pwlan) {
-				RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-					 "Error:  can't get pwlan in rtw23a_joinbss_event_cb\n");
-				spin_unlock_bh(&scanned_queue->lock);
-				goto createbss_cmd_fail;
-			}
-			pwlan->last_scanned = jiffies;
-		} else {
-			list_add_tail(&pwlan->list,
-				      &scanned_queue->queue);
-		}
-
-		pnetwork->Length = get_wlan_bssid_ex_sz(pnetwork);
-		memcpy(&pwlan->network, pnetwork, pnetwork->Length);
-		/* pwlan->fixed = true; */
-
-		/* list_add_tail(&pwlan->list,
-		   &pmlmepriv->scanned_queue.queue); */
-
-		/*  copy pdev_network information to
-		    pmlmepriv->cur_network */
-		memcpy(&tgt_network->network, pnetwork,
-		       get_wlan_bssid_ex_sz(pnetwork));
-
-		/*  reset DSConfig */
-
-		clr_fwstate(pmlmepriv, _FW_UNDER_LINKING);
-
-		/*  we will set _FW_LINKED when there is one more sat to
-		    join us (rtw_stassoc_event_callback23a) */
-		spin_unlock_bh(&scanned_queue->lock);
-	}
-
-createbss_cmd_fail:
-
-	rtw_free_cmd_obj23a(pcmd);
-}
-
-void rtw_setstaKey_cmdrsp_callback23a(struct rtw_adapter *padapter,
-				      struct cmd_obj *pcmd)
-{
-	struct sta_priv *pstapriv;
-	struct set_stakey_rsp *psetstakey_rsp;
-	struct sta_info *psta;
-
-	pstapriv = &padapter->stapriv;
-	psetstakey_rsp = (struct set_stakey_rsp *) (pcmd->rsp);
-	psta = rtw_get_stainfo23a(pstapriv, psetstakey_rsp->addr);
-
-	if (!psta) {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "ERROR: rtw_setstaKey_cmdrsp_callback23a => can't get sta_info\n");
-		goto exit;
-	}
-
-exit:
-
-	rtw_free_cmd_obj23a(pcmd);
-}
-
-void rtw_setassocsta_cmdrsp_callback23a(struct rtw_adapter *padapter,
-					struct cmd_obj *pcmd)
-{
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct set_assocsta_parm *passocsta_parm;
-	struct set_assocsta_rsp *passocsta_rsp;
-	struct sta_info *psta;
-
-	passocsta_parm = (struct set_assocsta_parm *)(pcmd->parmbuf);
-	passocsta_rsp = (struct set_assocsta_rsp *) (pcmd->rsp);
-	psta = rtw_get_stainfo23a(pstapriv, passocsta_parm->addr);
-
-	if (psta == NULL) {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "ERROR: setassocsta_cmdrsp_callbac => can't get sta_info\n");
-		goto exit;
-	}
-
-	psta->aid = psta->mac_id = passocsta_rsp->cam_id;
-
-	spin_lock_bh(&pmlmepriv->lock);
-
-	if (check_fwstate(pmlmepriv, WIFI_MP_STATE) &&
-	    check_fwstate(pmlmepriv, _FW_UNDER_LINKING))
-		_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
-
-	set_fwstate(pmlmepriv, _FW_LINKED);
-	spin_unlock_bh(&pmlmepriv->lock);
-
-exit:
-	rtw_free_cmd_obj23a(pcmd);
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_efuse.c b/drivers/staging/rtl8723au/core/rtw_efuse.c
deleted file mode 100644
index 359ef4197e94..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_efuse.c
+++ /dev/null
@@ -1,538 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_EFUSE_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#include <rtw_efuse.h>
-#include <rtl8723a_hal.h>
-#include <usb_ops_linux.h>
-
-#define REG_EFUSE_CTRL		0x0030
-#define EFUSE_CTRL		REG_EFUSE_CTRL	/* E-Fuse Control */
-
-#define VOLTAGE_V25		0x03
-#define LDOE25_SHIFT		28
-
-/*
- * When we want to enable write operation, we should change to
- * pwr on state. When we stop write, we should switch to 500k mode
- * and disable LDO 2.5V.
- */
-static void Efuse_PowerSwitch(struct rtw_adapter *padapter,
-			      u8 bWrite, u8 PwrState)
-{
-	u8 tempval;
-	u16 tmpV16;
-
-	if (PwrState == true) {
-		rtl8723au_write8(padapter, REG_EFUSE_ACCESS, EFUSE_ACCESS_ON);
-
-		/*
-		 * 1.2V Power: From VDDON with Power
-		 * Cut(0x0000h[15]), default valid
-		 */
-		tmpV16 = rtl8723au_read16(padapter, REG_SYS_ISO_CTRL);
-		if (!(tmpV16 & PWC_EV12V)) {
-			tmpV16 |= PWC_EV12V;
-			rtl8723au_write16(padapter, REG_SYS_ISO_CTRL, tmpV16);
-		}
-		/* Reset: 0x0000h[28], default valid */
-		tmpV16 = rtl8723au_read16(padapter, REG_SYS_FUNC_EN);
-		if (!(tmpV16 & FEN_ELDR)) {
-			tmpV16 |= FEN_ELDR;
-			rtl8723au_write16(padapter, REG_SYS_FUNC_EN, tmpV16);
-		}
-
-		/*
-		 * Clock: Gated(0x0008h[5]) 8M(0x0008h[1])
-		 * clock from ANA, default valid
-		 */
-		tmpV16 = rtl8723au_read16(padapter, REG_SYS_CLKR);
-		if ((!(tmpV16 & LOADER_CLK_EN)) || (!(tmpV16 & ANA8M))) {
-			tmpV16 |= (LOADER_CLK_EN | ANA8M);
-			rtl8723au_write16(padapter, REG_SYS_CLKR, tmpV16);
-		}
-
-		if (bWrite == true) {
-			/*  Enable LDO 2.5V before read/write action */
-			tempval = rtl8723au_read8(padapter, EFUSE_TEST + 3);
-			tempval &= 0x0F;
-			tempval |= (VOLTAGE_V25 << 4);
-			rtl8723au_write8(padapter, EFUSE_TEST + 3,
-					 tempval | 0x80);
-		}
-	} else {
-		rtl8723au_write8(padapter, REG_EFUSE_ACCESS, EFUSE_ACCESS_OFF);
-
-		if (bWrite == true) {
-			/*  Disable LDO 2.5V after read/write action */
-			tempval = rtl8723au_read8(padapter, EFUSE_TEST + 3);
-			rtl8723au_write8(padapter, EFUSE_TEST + 3,
-					 tempval & 0x7F);
-		}
-	}
-}
-
-u16 Efuse_GetCurrentSize23a(struct rtw_adapter *pAdapter, u8 efuseType)
-{
-	u16 ret = 0;
-
-	if (efuseType == EFUSE_WIFI)
-		ret = rtl8723a_EfuseGetCurrentSize_WiFi(pAdapter);
-	else
-		ret = rtl8723a_EfuseGetCurrentSize_BT(pAdapter);
-
-	return ret;
-}
-
-/* Get current efuse area enabled word */
-u8 Efuse_CalculateWordCnts23a(u8 word_en)
-{
-	return hweight8((~word_en) & 0xf);
-}
-
-/*
- * Description: Execute E-Fuse read byte operation.
- *
- * Assumptions: 1. Boot from E-Fuse and successfully auto-load.
- *              2. PASSIVE_LEVEL (USB interface)
- */
-void ReadEFuseByte23a(struct rtw_adapter *Adapter, u16 _offset, u8 *pbuf)
-{
-	u32	value32;
-	u8	readbyte;
-	u16	retry;
-
-	/* Write Address */
-	rtl8723au_write8(Adapter, EFUSE_CTRL+1, (_offset & 0xff));
-	readbyte = rtl8723au_read8(Adapter, EFUSE_CTRL+2);
-	rtl8723au_write8(Adapter, EFUSE_CTRL+2,
-			 ((_offset >> 8) & 0x03) | (readbyte & 0xfc));
-
-	/* Write bit 32 0 */
-	readbyte = rtl8723au_read8(Adapter, EFUSE_CTRL+3);
-	rtl8723au_write8(Adapter, EFUSE_CTRL+3, readbyte & 0x7f);
-
-	/* Check bit 32 read-ready */
-	retry = 0;
-	value32 = rtl8723au_read32(Adapter, EFUSE_CTRL);
-	while (!((value32 >> 24) & 0x80) && retry < 10000) {
-		value32 = rtl8723au_read32(Adapter, EFUSE_CTRL);
-		retry++;
-	}
-
-	/*
-	 * Added suggested delay. This fixes the problem that
-	 * Efuse read error in high temperature condition.
-	 * Designer says that there shall be some delay after
-	 * ready bit is set, or the result will always stay
-	 * on last data we read.
-	 */
-	udelay(50);
-	value32 = rtl8723au_read32(Adapter, EFUSE_CTRL);
-
-	*pbuf = (u8)(value32 & 0xff);
-}
-
-void EFUSE_GetEfuseDefinition23a(struct rtw_adapter *pAdapter, u8 efuseType,
-				 u8 type, void *pOut)
-{
-	u8 *pu1Tmp;
-	u16 *pu2Tmp;
-	u8 *pMax_section;
-
-	switch (type) {
-	case TYPE_EFUSE_MAX_SECTION:
-		pMax_section = pOut;
-
-		if (efuseType == EFUSE_WIFI)
-			*pMax_section = EFUSE_MAX_SECTION_8723A;
-		else
-			*pMax_section = EFUSE_BT_MAX_SECTION;
-		break;
-
-	case TYPE_EFUSE_REAL_CONTENT_LEN:
-		pu2Tmp = pOut;
-
-		if (efuseType == EFUSE_WIFI)
-			*pu2Tmp = EFUSE_REAL_CONTENT_LEN_8723A;
-		else
-			*pu2Tmp = EFUSE_BT_REAL_CONTENT_LEN;
-		break;
-
-	case TYPE_AVAILABLE_EFUSE_BYTES_BANK:
-		pu2Tmp = pOut;
-
-		if (efuseType == EFUSE_WIFI)
-			*pu2Tmp = (EFUSE_REAL_CONTENT_LEN_8723A -
-				   EFUSE_OOB_PROTECT_BYTES);
-		else
-			*pu2Tmp = (EFUSE_BT_REAL_BANK_CONTENT_LEN -
-				   EFUSE_PROTECT_BYTES_BANK);
-		break;
-
-	case TYPE_AVAILABLE_EFUSE_BYTES_TOTAL:
-		pu2Tmp = pOut;
-
-		if (efuseType == EFUSE_WIFI)
-			*pu2Tmp = (EFUSE_REAL_CONTENT_LEN_8723A -
-				   EFUSE_OOB_PROTECT_BYTES);
-		else
-			*pu2Tmp = (EFUSE_BT_REAL_CONTENT_LEN -
-				   (EFUSE_PROTECT_BYTES_BANK * 3));
-		break;
-
-	case TYPE_EFUSE_MAP_LEN:
-		pu2Tmp = pOut;
-
-		if (efuseType == EFUSE_WIFI)
-			*pu2Tmp = EFUSE_MAP_LEN_8723A;
-		else
-			*pu2Tmp = EFUSE_BT_MAP_LEN;
-		break;
-
-	case TYPE_EFUSE_PROTECT_BYTES_BANK:
-		pu1Tmp = pOut;
-
-		if (efuseType == EFUSE_WIFI)
-			*pu1Tmp = EFUSE_OOB_PROTECT_BYTES;
-		else
-			*pu1Tmp = EFUSE_PROTECT_BYTES_BANK;
-		break;
-
-	case TYPE_EFUSE_CONTENT_LEN_BANK:
-		pu2Tmp = pOut;
-
-		if (efuseType == EFUSE_WIFI)
-			*pu2Tmp = EFUSE_REAL_CONTENT_LEN_8723A;
-		else
-			*pu2Tmp = EFUSE_BT_REAL_BANK_CONTENT_LEN;
-		break;
-
-	default:
-		pu1Tmp = pOut;
-		*pu1Tmp = 0;
-		break;
-	}
-}
-
-/* Copy from WMAC for EFUSE read 1 byte. */
-u8 EFUSE_Read1Byte23a(struct rtw_adapter *Adapter, u16 Address)
-{
-	u8	data;
-	u8	Bytetemp = {0x00};
-	u8	temp = {0x00};
-	u32	k = 0;
-	u16	contentLen = 0;
-
-	EFUSE_GetEfuseDefinition23a(Adapter, EFUSE_WIFI,
-				 TYPE_EFUSE_REAL_CONTENT_LEN,
-				 (void *)&contentLen);
-
-	if (Address < contentLen) { /* E-fuse 512Byte */
-		/* Write E-fuse Register address bit0~7 */
-		temp = Address & 0xFF;
-		rtl8723au_write8(Adapter, EFUSE_CTRL+1, temp);
-		Bytetemp = rtl8723au_read8(Adapter, EFUSE_CTRL+2);
-		/* Write E-fuse Register address bit8~9 */
-		temp = ((Address >> 8) & 0x03) | (Bytetemp & 0xFC);
-		rtl8723au_write8(Adapter, EFUSE_CTRL+2, temp);
-
-		/* Write 0x30[31]= 0 */
-		Bytetemp = rtl8723au_read8(Adapter, EFUSE_CTRL+3);
-		temp = Bytetemp & 0x7F;
-		rtl8723au_write8(Adapter, EFUSE_CTRL+3, temp);
-
-		/* Wait Write-ready (0x30[31]= 1) */
-		Bytetemp = rtl8723au_read8(Adapter, EFUSE_CTRL+3);
-		while (!(Bytetemp & 0x80)) {
-			Bytetemp = rtl8723au_read8(Adapter, EFUSE_CTRL+3);
-			k++;
-			if (k == 1000) {
-				k = 0;
-				break;
-			}
-		}
-		data = rtl8723au_read8(Adapter, EFUSE_CTRL);
-		return data;
-	}
-	return 0xFF;
-}
-
-/* Read one byte from real Efuse. */
-int efuse_OneByteRead23a(struct rtw_adapter *pAdapter, u16 addr, u8 *data)
-{
-	u8	tmpidx = 0;
-	int	bResult;
-
-	/*  -----------------e-fuse reg ctrl ---------------------------- */
-	/* address */
-	rtl8723au_write8(pAdapter, EFUSE_CTRL + 1, (u8)(addr & 0xff));
-	rtl8723au_write8(pAdapter, EFUSE_CTRL + 2,
-			 ((u8)((addr >> 8) & 0x03)) |
-			 (rtl8723au_read8(pAdapter, EFUSE_CTRL + 2) & 0xFC));
-
-	rtl8723au_write8(pAdapter, EFUSE_CTRL + 3, 0x72); /* read cmd */
-
-	while (!(0x80 & rtl8723au_read8(pAdapter, EFUSE_CTRL + 3)) &&
-	       (tmpidx < 100))
-		tmpidx++;
-	if (tmpidx < 100) {
-		*data = rtl8723au_read8(pAdapter, EFUSE_CTRL);
-		bResult = _SUCCESS;
-	} else {
-		*data = 0xff;
-		bResult = _FAIL;
-	}
-	return bResult;
-}
-
-/* Write one byte to reald Efuse. */
-int efuse_OneByteWrite23a(struct rtw_adapter *pAdapter, u16 addr, u8 data)
-{
-	u8	tmpidx = 0;
-	int	bResult;
-
-	/* return	0; */
-
-	/*  -----------------e-fuse reg ctrl ------------------------- */
-	/* address */
-	rtl8723au_write8(pAdapter, EFUSE_CTRL + 1, (u8)(addr & 0xff));
-	rtl8723au_write8(pAdapter, EFUSE_CTRL + 2,
-			 (rtl8723au_read8(pAdapter, EFUSE_CTRL + 2) & 0xFC) |
-			 (u8)((addr >> 8) & 0x03));
-	rtl8723au_write8(pAdapter, EFUSE_CTRL, data); /* data */
-
-	rtl8723au_write8(pAdapter, EFUSE_CTRL + 3, 0xF2); /* write cmd */
-
-	while ((0x80 & rtl8723au_read8(pAdapter, EFUSE_CTRL + 3)) &&
-	       (tmpidx < 100)) {
-		tmpidx++;
-	}
-
-	if (tmpidx < 100)
-		bResult = _SUCCESS;
-	else
-		bResult = _FAIL;
-
-	return bResult;
-}
-
-/* Read allowed word in current efuse section data. */
-void efuse_WordEnableDataRead23a(u8 word_en, u8 *sourdata, u8 *targetdata)
-{
-	if (!(word_en&BIT(0))) {
-		targetdata[0] = sourdata[0];
-		targetdata[1] = sourdata[1];
-	}
-	if (!(word_en&BIT(1))) {
-		targetdata[2] = sourdata[2];
-		targetdata[3] = sourdata[3];
-	}
-	if (!(word_en&BIT(2))) {
-		targetdata[4] = sourdata[4];
-		targetdata[5] = sourdata[5];
-	}
-	if (!(word_en&BIT(3))) {
-		targetdata[6] = sourdata[6];
-		targetdata[7] = sourdata[7];
-	}
-}
-
-static int efuse_read8(struct rtw_adapter *padapter, u16 address, u8 *value)
-{
-	return efuse_OneByteRead23a(padapter, address, value);
-}
-
-static int efuse_write8(struct rtw_adapter *padapter, u16 address, u8 *value)
-{
-	return efuse_OneByteWrite23a(padapter, address, *value);
-}
-
-/* read/write raw efuse data */
-int rtw_efuse_access23a(struct rtw_adapter *padapter, u8 bWrite, u16 start_addr,
-			u16 cnts, u8 *data)
-{
-	int i = 0;
-	u16 real_content_len = 0, max_available_size = 0;
-	int res = _FAIL;
-	int (*rw8)(struct rtw_adapter *, u16, u8*);
-
-	EFUSE_GetEfuseDefinition23a(padapter, EFUSE_WIFI,
-				 TYPE_EFUSE_REAL_CONTENT_LEN,
-				 (void *)&real_content_len);
-	EFUSE_GetEfuseDefinition23a(padapter, EFUSE_WIFI,
-				 TYPE_AVAILABLE_EFUSE_BYTES_TOTAL,
-				 (void *)&max_available_size);
-
-	if (start_addr > real_content_len)
-		return _FAIL;
-
-	if (true == bWrite) {
-		if ((start_addr + cnts) > max_available_size)
-			return _FAIL;
-		rw8 = &efuse_write8;
-	} else
-		rw8 = &efuse_read8;
-
-	Efuse_PowerSwitch(padapter, bWrite, true);
-
-	/* e-fuse one byte read/write */
-	for (i = 0; i < cnts; i++) {
-		if (start_addr >= real_content_len) {
-			res = _FAIL;
-			break;
-		}
-
-		res = rw8(padapter, start_addr++, data++);
-		if (res == _FAIL)
-			break;
-	}
-
-	Efuse_PowerSwitch(padapter, bWrite, false);
-
-	return res;
-}
-
-u16 efuse_GetMaxSize23a(struct rtw_adapter *padapter)
-{
-	u16 max_size;
-
-	EFUSE_GetEfuseDefinition23a(padapter, EFUSE_WIFI,
-				 TYPE_AVAILABLE_EFUSE_BYTES_TOTAL,
-				 (void *)&max_size);
-	return max_size;
-}
-
-int rtw_efuse_map_read23a(struct rtw_adapter *padapter,
-			  u16 addr, u16 cnts, u8 *data)
-{
-	u16 mapLen = 0;
-
-	EFUSE_GetEfuseDefinition23a(padapter, EFUSE_WIFI,
-				 TYPE_EFUSE_MAP_LEN, (void *)&mapLen);
-
-	if ((addr + cnts) > mapLen)
-		return _FAIL;
-
-	Efuse_PowerSwitch(padapter, false, true);
-
-	rtl8723a_readefuse(padapter, EFUSE_WIFI, addr, cnts, data);
-
-	Efuse_PowerSwitch(padapter, false, false);
-
-	return _SUCCESS;
-}
-
-int rtw_BT_efuse_map_read23a(struct rtw_adapter *padapter,
-			     u16 addr, u16 cnts, u8 *data)
-{
-	u16 mapLen = 0;
-
-	EFUSE_GetEfuseDefinition23a(padapter, EFUSE_BT,
-				 TYPE_EFUSE_MAP_LEN, (void *)&mapLen);
-
-	if ((addr + cnts) > mapLen)
-		return _FAIL;
-
-	Efuse_PowerSwitch(padapter, false, true);
-
-	rtl8723a_readefuse(padapter, EFUSE_BT, addr, cnts, data);
-
-	Efuse_PowerSwitch(padapter, false, false);
-
-	return _SUCCESS;
-}
-
-/* Read All Efuse content */
-static void Efuse_ReadAllMap(struct rtw_adapter *pAdapter, u8 efuseType,
-			     u8 *Efuse)
-{
-	u16	mapLen = 0;
-
-	Efuse_PowerSwitch(pAdapter, false, true);
-
-	EFUSE_GetEfuseDefinition23a(pAdapter, efuseType, TYPE_EFUSE_MAP_LEN,
-				 (void *)&mapLen);
-
-	rtl8723a_readefuse(pAdapter, efuseType, 0, mapLen, Efuse);
-
-	Efuse_PowerSwitch(pAdapter, false, false);
-}
-
-/*
- * Functions:	efuse_ShadowRead1Byte
- *		efuse_ShadowRead2Byte
- *		efuse_ShadowRead4Byte
- *
- * Read from efuse init map by one/two/four bytes
- */
-static void efuse_ShadowRead1Byte(struct rtw_adapter *pAdapter, u16 Offset,
-				  u8 *Value)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(pAdapter);
-
-	*Value = pEEPROM->efuse_eeprom_data[Offset];
-}
-
-static void efuse_ShadowRead2Byte(struct rtw_adapter *pAdapter, u16 Offset,
-				  u16 *Value)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(pAdapter);
-
-	*Value = pEEPROM->efuse_eeprom_data[Offset];
-	*Value |= pEEPROM->efuse_eeprom_data[Offset+1]<<8;
-}
-
-static void efuse_ShadowRead4Byte(struct rtw_adapter *pAdapter, u16 Offset,
-				  u32 *Value)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(pAdapter);
-
-	*Value = pEEPROM->efuse_eeprom_data[Offset];
-	*Value |= pEEPROM->efuse_eeprom_data[Offset+1]<<8;
-	*Value |= pEEPROM->efuse_eeprom_data[Offset+2]<<16;
-	*Value |= pEEPROM->efuse_eeprom_data[Offset+3]<<24;
-}
-
-/* Transfer current EFUSE content to shadow init and modify map. */
-void EFUSE_ShadowMapUpdate23a(struct rtw_adapter *pAdapter, u8 efuseType)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(pAdapter);
-	u16	mapLen = 0;
-
-	EFUSE_GetEfuseDefinition23a(pAdapter, efuseType,
-				 TYPE_EFUSE_MAP_LEN, (void *)&mapLen);
-
-	if (pEEPROM->bautoload_fail_flag == true)
-		memset(pEEPROM->efuse_eeprom_data, 0xFF, mapLen);
-	else
-		Efuse_ReadAllMap(pAdapter, efuseType,
-				 pEEPROM->efuse_eeprom_data);
-}
-
-/* Read from efuse init map */
-void EFUSE_ShadowRead23a(struct rtw_adapter *pAdapter, u8 Type,
-			 u16 Offset, u32 *Value)
-{
-	if (Type == 1)
-		efuse_ShadowRead1Byte(pAdapter, Offset, (u8 *)Value);
-	else if (Type == 2)
-		efuse_ShadowRead2Byte(pAdapter, Offset, (u16 *)Value);
-	else if (Type == 4)
-		efuse_ShadowRead4Byte(pAdapter, Offset, (u32 *)Value);
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_ieee80211.c b/drivers/staging/rtl8723au/core/rtw_ieee80211.c
deleted file mode 100644
index 07a6490a83d6..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_ieee80211.c
+++ /dev/null
@@ -1,855 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _IEEE80211_C
-
-#include <drv_types.h>
-#include <linux/ieee80211.h>
-#include <ieee80211.h>
-#include <wifi.h>
-#include <osdep_service.h>
-#include <wlan_bssdef.h>
-
-u8 RTW_WPA_OUI23A_TYPE[] = { 0x00, 0x50, 0xf2, 1 };
-u16 RTW_WPA_VERSION23A = 1;
-u8 WPA_AUTH_KEY_MGMT_NONE23A[] = { 0x00, 0x50, 0xf2, 0 };
-u8 WPA_AUTH_KEY_MGMT_UNSPEC_802_1X23A[] = { 0x00, 0x50, 0xf2, 1 };
-u8 WPA_AUTH_KEY_MGMT_PSK_OVER_802_1X23A[] = { 0x00, 0x50, 0xf2, 2 };
-u8 WPA_CIPHER_SUITE_NONE23A[] = { 0x00, 0x50, 0xf2, 0 };
-u8 WPA_CIPHER_SUITE_WEP4023A[] = { 0x00, 0x50, 0xf2, 1 };
-u8 WPA_CIPHER_SUITE_TKIP23A[] = { 0x00, 0x50, 0xf2, 2 };
-u8 WPA_CIPHER_SUITE_WRAP23A[] = { 0x00, 0x50, 0xf2, 3 };
-u8 WPA_CIPHER_SUITE_CCMP23A[] = { 0x00, 0x50, 0xf2, 4 };
-u8 WPA_CIPHER_SUITE_WEP10423A[] = { 0x00, 0x50, 0xf2, 5 };
-
-u8 RSN_AUTH_KEY_MGMT_UNSPEC_802_1X23A[] = { 0x00, 0x0f, 0xac, 1 };
-u8 RSN_AUTH_KEY_MGMT_PSK_OVER_802_1X23A[] = { 0x00, 0x0f, 0xac, 2 };
-u8 RSN_CIPHER_SUITE_NONE23A[] = { 0x00, 0x0f, 0xac, 0 };
-u8 RSN_CIPHER_SUITE_WEP4023A[] = { 0x00, 0x0f, 0xac, 1 };
-u8 RSN_CIPHER_SUITE_TKIP23A[] = { 0x00, 0x0f, 0xac, 2 };
-u8 RSN_CIPHER_SUITE_WRAP23A[] = { 0x00, 0x0f, 0xac, 3 };
-u8 RSN_CIPHER_SUITE_CCMP23A[] = { 0x00, 0x0f, 0xac, 4 };
-u8 RSN_CIPHER_SUITE_WEP10423A[] = { 0x00, 0x0f, 0xac, 5 };
-/*  */
-/*  for adhoc-master to generate ie and provide supported-rate to fw */
-/*  */
-
-static u8 WIFI_CCKRATES[] = {
-	IEEE80211_CCK_RATE_1MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_CCK_RATE_2MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_CCK_RATE_5MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_CCK_RATE_11MB | IEEE80211_BASIC_RATE_MASK
-};
-
-static u8 WIFI_OFDMRATES[] = {
-	IEEE80211_OFDM_RATE_6MB,
-	IEEE80211_OFDM_RATE_9MB,
-	IEEE80211_OFDM_RATE_12MB,
-	IEEE80211_OFDM_RATE_18MB,
-	IEEE80211_OFDM_RATE_24MB,
-	IEEE80211_OFDM_RATE_36MB,
-	IEEE80211_OFDM_RATE_48MB,
-	IEEE80211_OFDM_RATE_54MB
-};
-
-int rtw_get_bit_value_from_ieee_value23a(u8 val)
-{
-	unsigned char dot11_rate_table[]=
-		{2, 4, 11, 22, 12, 18, 24, 36, 48, 72, 96, 108, 0};
-
-	int i = 0;
-
-	while (dot11_rate_table[i] != 0) {
-		if (dot11_rate_table[i] == val)
-			return BIT(i);
-		i++;
-	}
-	return 0;
-}
-
-static bool rtw_is_cckrates_included(u8 *rate)
-{
-	u32 i = 0;
-
-	while (rate[i]) {
-		if ((rate[i] & 0x7f) == 2 || (rate[i] & 0x7f) == 4 ||
-		    (rate[i] & 0x7f) == 11 || (rate[i] & 0x7f) == 22)
-			return true;
-		i++;
-	}
-
-	return false;
-}
-
-static bool rtw_is_cckratesonly_included(u8 *rate)
-{
-	u32 i = 0;
-
-	while (rate[i]) {
-		if ((rate[i] & 0x7f) != 2 && (rate[i] & 0x7f) != 4 &&
-		    (rate[i] & 0x7f) != 11 && (rate[i] & 0x7f) != 22)
-			return false;
-
-		i++;
-	}
-
-	return true;
-}
-
-int rtw_check_network_type23a(unsigned char *rate, int ratelen, int channel)
-{
-	if (channel > 14) {
-		if (rtw_is_cckrates_included(rate))
-			return WIRELESS_INVALID;
-		else
-			return WIRELESS_11A;
-	} else {  /*  could be pure B, pure G, or B/G */
-		if (rtw_is_cckratesonly_included(rate))
-			return WIRELESS_11B;
-		else if (rtw_is_cckrates_included(rate))
-			return	WIRELESS_11BG;
-		else
-			return WIRELESS_11G;
-	}
-}
-
-/*  rtw_set_ie23a will update frame length */
-u8 *rtw_set_ie23a(u8 *pbuf, int index, uint len, const u8 *source, uint *frlen)
-{
-
-	*pbuf = (u8)index;
-
-	*(pbuf + 1) = (u8)len;
-
-	if (len > 0)
-		memcpy((void *)(pbuf + 2), (void *)source, len);
-
-	*frlen = *frlen + (len + 2);
-
-	return pbuf + len + 2;
-}
-
-inline u8 *rtw_set_ie23a_ch_switch (u8 *buf, u32 *buf_len, u8 ch_switch_mode,
-				u8 new_ch, u8 ch_switch_cnt)
-{
-	u8 ie_data[3];
-
-	ie_data[0] = ch_switch_mode;
-	ie_data[1] = new_ch;
-	ie_data[2] = ch_switch_cnt;
-	return rtw_set_ie23a(buf, WLAN_EID_CHANNEL_SWITCH,  3, ie_data, buf_len);
-}
-
-inline u8 hal_ch_offset_to_secondary_ch_offset23a(u8 ch_offset)
-{
-	if (ch_offset == HAL_PRIME_CHNL_OFFSET_LOWER)
-		return IEEE80211_HT_PARAM_CHA_SEC_BELOW;
-	else if (ch_offset == HAL_PRIME_CHNL_OFFSET_UPPER)
-		return IEEE80211_HT_PARAM_CHA_SEC_ABOVE;
-
-	return IEEE80211_HT_PARAM_CHA_SEC_NONE;
-}
-
-inline u8 *rtw_set_ie23a_secondary_ch_offset(u8 *buf, u32 *buf_len,
-					  u8 secondary_ch_offset)
-{
-	return rtw_set_ie23a(buf, WLAN_EID_SECONDARY_CHANNEL_OFFSET,
-			  1, &secondary_ch_offset, buf_len);
-}
-
-/*----------------------------------------------------------------------------
-index: the information element id index, limit is the limit for search
------------------------------------------------------------------------------*/
-u8 *rtw_get_ie23a(u8 *pbuf, int index, int *len, int limit)
-{
-	int tmp, i;
-	u8 *p;
-
-	if (limit < 1) {
-
-		return NULL;
-	}
-
-	p = pbuf;
-	i = 0;
-	*len = 0;
-	while (1) {
-		if (*p == index) {
-			*len = *(p + 1);
-			return p;
-		} else {
-			tmp = *(p + 1);
-			p += (tmp + 2);
-			i += (tmp + 2);
-		}
-		if (i >= limit)
-			break;
-	}
-
-	return NULL;
-}
-
-/**
- * rtw_get_ie23a_ex - Search specific IE from a series of IEs
- * @in_ie: Address of IEs to search
- * @in_len: Length limit from in_ie
- * @eid: Element ID to match
- * @oui: OUI to match
- * @oui_len: OUI length
- * @ie: If not NULL and the specific IE is found, the IE will be copied
- *      to the buf starting from the specific IE
- * @ielen: If not NULL and the specific IE is found, will set to the length
- *         of the entire IE
- *
- * Returns: The address of the specific IE found, or NULL
- */
-u8 *rtw_get_ie23a_ex(u8 *in_ie, uint in_len, u8 eid, u8 *oui, u8 oui_len,
-		  u8 *ie, uint *ielen)
-{
-	uint cnt;
-	u8 *target_ie = NULL;
-
-	if (ielen)
-		*ielen = 0;
-
-	if (!in_ie || in_len <= 0)
-		return target_ie;
-
-	cnt = 0;
-
-	while (cnt < in_len) {
-		if (eid == in_ie[cnt] &&
-		    (!oui || !memcmp(&in_ie[cnt+2], oui, oui_len))) {
-			target_ie = &in_ie[cnt];
-
-			if (ie)
-				memcpy(ie, &in_ie[cnt], in_ie[cnt+1]+2);
-
-			if (ielen)
-				*ielen = in_ie[cnt+1]+2;
-			break;
-		} else {
-			cnt += in_ie[cnt + 1] + 2; /* goto next */
-		}
-	}
-
-	return target_ie;
-}
-
-/**
- * rtw_ies_remove_ie23a - Find matching IEs and remove
- * @ies: Address of IEs to search
- * @ies_len: Pointer of length of ies, will update to new length
- * @offset: The offset to start search
- * @eid: Element ID to match
- * @oui: OUI to match
- * @oui_len: OUI length
- *
- * Returns: _SUCCESS: ies is updated, _FAIL: not updated
- */
-int rtw_ies_remove_ie23a(u8 *ies, uint *ies_len, uint offset, u8 eid,
-		      u8 *oui, u8 oui_len)
-{
-	int ret = _FAIL;
-	u8 *target_ie;
-	u32 target_ielen;
-	u8 *start;
-	uint search_len;
-
-	if (!ies || !ies_len || *ies_len <= offset)
-		goto exit;
-
-	start = ies + offset;
-	search_len = *ies_len - offset;
-
-	while (1) {
-		target_ie = rtw_get_ie23a_ex(start, search_len, eid, oui, oui_len,
-					  NULL, &target_ielen);
-		if (target_ie && target_ielen) {
-			u8 buf[MAX_IE_SZ] = {0};
-			u8 *remain_ies = target_ie + target_ielen;
-			uint remain_len = search_len - (remain_ies - start);
-
-			memcpy(buf, remain_ies, remain_len);
-			memcpy(target_ie, buf, remain_len);
-			*ies_len = *ies_len - target_ielen;
-			ret = _SUCCESS;
-
-			start = target_ie;
-			search_len = remain_len;
-		} else {
-			break;
-		}
-	}
-exit:
-	return ret;
-}
-
-void rtw_set_supported_rate23a(u8 *SupportedRates, uint mode)
-{
-
-
-	memset(SupportedRates, 0, NDIS_802_11_LENGTH_RATES_EX);
-
-	switch (mode) {
-	case WIRELESS_11B:
-		memcpy(SupportedRates, WIFI_CCKRATES, IEEE80211_CCK_RATE_LEN);
-		break;
-
-	case WIRELESS_11G:
-	case WIRELESS_11A:
-	case WIRELESS_11_5N:
-	case WIRELESS_11A_5N:/* Todo: no basic rate for ofdm ? */
-		memcpy(SupportedRates, WIFI_OFDMRATES,
-		       IEEE80211_NUM_OFDM_RATESLEN);
-		break;
-
-	case WIRELESS_11BG:
-	case WIRELESS_11G_24N:
-	case WIRELESS_11_24N:
-	case WIRELESS_11BG_24N:
-		memcpy(SupportedRates, WIFI_CCKRATES, IEEE80211_CCK_RATE_LEN);
-		memcpy(SupportedRates + IEEE80211_CCK_RATE_LEN, WIFI_OFDMRATES,
-		       IEEE80211_NUM_OFDM_RATESLEN);
-		break;
-	}
-
-}
-
-uint rtw_get_rateset_len23a(u8 *rateset)
-{
-	uint i = 0;
-
-	while(1) {
-		if (rateset[i] == 0)
-			break;
-
-		if (i > 12)
-			break;
-
-		i++;
-	}
-
-	return i;
-}
-
-int rtw_generate_ie23a(struct registry_priv *pregistrypriv)
-{
-	u8	wireless_mode;
-	int	sz = 0, rateLen;
-	struct wlan_bssid_ex *pdev_network = &pregistrypriv->dev_network;
-	u8	*ie = pdev_network->IEs;
-	u16	cap;
-
-	pdev_network->tsf = 0;
-
-	cap = WLAN_CAPABILITY_IBSS;
-
-	if (pregistrypriv->preamble == PREAMBLE_SHORT)
-		cap |= WLAN_CAPABILITY_SHORT_PREAMBLE;
-
-	if (pdev_network->Privacy)
-		cap |= WLAN_CAPABILITY_PRIVACY;
-
-	pdev_network->capability = cap;
-
-	/* SSID */
-	ie = rtw_set_ie23a(ie, WLAN_EID_SSID, pdev_network->Ssid.ssid_len,
-			pdev_network->Ssid.ssid, &sz);
-
-	/* supported rates */
-	if (pregistrypriv->wireless_mode == WIRELESS_11ABGN) {
-		if (pdev_network->DSConfig > 14)
-			wireless_mode = WIRELESS_11A_5N;
-		else
-			wireless_mode = WIRELESS_11BG_24N;
-	} else {
-		wireless_mode = pregistrypriv->wireless_mode;
-	}
-
-	rtw_set_supported_rate23a(pdev_network->SupportedRates, wireless_mode) ;
-
-	rateLen = rtw_get_rateset_len23a(pdev_network->SupportedRates);
-
-	if (rateLen > 8) {
-		ie = rtw_set_ie23a(ie, WLAN_EID_SUPP_RATES, 8,
-				pdev_network->SupportedRates, &sz);
-		/* ie = rtw_set_ie23a(ie, _EXT_SUPPORTEDRATES_IE_, (rateLen - 8), (pdev_network->SupportedRates + 8), &sz); */
-	} else {
-		ie = rtw_set_ie23a(ie, WLAN_EID_SUPP_RATES, rateLen,
-				pdev_network->SupportedRates, &sz);
-	}
-
-	/* DS parameter set */
-	ie = rtw_set_ie23a(ie, WLAN_EID_DS_PARAMS, 1,
-			   (u8 *)&pdev_network->DSConfig, &sz);
-
-	/* IBSS Parameter Set */
-
-	ie = rtw_set_ie23a(ie, WLAN_EID_IBSS_PARAMS, 2,
-			   (u8 *)&pdev_network->ATIMWindow, &sz);
-
-	if (rateLen > 8) {
-		ie = rtw_set_ie23a(ie, WLAN_EID_EXT_SUPP_RATES, (rateLen - 8),
-				(pdev_network->SupportedRates + 8), &sz);
-	}
-
-
-
-	/* return _SUCCESS; */
-
-	return sz;
-}
-
-static int rtw_get_wpa_cipher_suite(const u8 *s)
-{
-	if (!memcmp(s, WPA_CIPHER_SUITE_NONE23A, WPA_SELECTOR_LEN))
-		return WPA_CIPHER_NONE;
-	if (!memcmp(s, WPA_CIPHER_SUITE_WEP4023A, WPA_SELECTOR_LEN))
-		return WPA_CIPHER_WEP40;
-	if (!memcmp(s, WPA_CIPHER_SUITE_TKIP23A, WPA_SELECTOR_LEN))
-		return WPA_CIPHER_TKIP;
-	if (!memcmp(s, WPA_CIPHER_SUITE_CCMP23A, WPA_SELECTOR_LEN))
-		return WPA_CIPHER_CCMP;
-	if (!memcmp(s, WPA_CIPHER_SUITE_WEP10423A, WPA_SELECTOR_LEN))
-		return WPA_CIPHER_WEP104;
-
-	return 0;
-}
-
-static int rtw_get_wpa2_cipher_suite(const u8 *s)
-{
-	if (!memcmp(s, RSN_CIPHER_SUITE_NONE23A, RSN_SELECTOR_LEN))
-		return WPA_CIPHER_NONE;
-	if (!memcmp(s, RSN_CIPHER_SUITE_WEP4023A, RSN_SELECTOR_LEN))
-		return WPA_CIPHER_WEP40;
-	if (!memcmp(s, RSN_CIPHER_SUITE_TKIP23A, RSN_SELECTOR_LEN))
-		return WPA_CIPHER_TKIP;
-	if (!memcmp(s, RSN_CIPHER_SUITE_CCMP23A, RSN_SELECTOR_LEN))
-		return WPA_CIPHER_CCMP;
-	if (!memcmp(s, RSN_CIPHER_SUITE_WEP10423A, RSN_SELECTOR_LEN))
-		return WPA_CIPHER_WEP104;
-
-	return 0;
-}
-
-int rtw_parse_wpa_ie23a(const u8 *wpa_ie, int wpa_ie_len, int *group_cipher,
-			int *pairwise_cipher, int *is_8021x)
-{
-	int i, ret = _SUCCESS;
-	int left, count;
-	const u8 *pos;
-
-	if (wpa_ie_len <= 0) {
-		/* No WPA IE - fail silently */
-		return _FAIL;
-	}
-
-	if (wpa_ie[1] != (u8)(wpa_ie_len - 2))
-		return _FAIL;
-
-	pos = wpa_ie;
-
-	pos += 8;
-	left = wpa_ie_len - 8;
-
-	/* group_cipher */
-	if (left >= WPA_SELECTOR_LEN) {
-
-		*group_cipher = rtw_get_wpa_cipher_suite(pos);
-
-		pos += WPA_SELECTOR_LEN;
-		left -= WPA_SELECTOR_LEN;
-	} else if (left > 0) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "%s: ie length mismatch, %u too much\n",
-			 __func__, left);
-
-		return _FAIL;
-	}
-
-	/* pairwise_cipher */
-	if (left >= 2) {
-                /* count = le16_to_cpu(*(u16*)pos); */
-		count = get_unaligned_le16(pos);
-		pos += 2;
-		left -= 2;
-
-		if (count == 0 || left < count * WPA_SELECTOR_LEN) {
-			RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-				 "%s: ie count botch (pairwise), count %u left %u\n",
-				 __func__, count, left);
-			return _FAIL;
-		}
-
-		for (i = 0; i < count; i++) {
-			*pairwise_cipher |= rtw_get_wpa_cipher_suite(pos);
-
-			pos += WPA_SELECTOR_LEN;
-			left -= WPA_SELECTOR_LEN;
-		}
-	} else if (left == 1) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "%s: ie too short (for key mgmt)\n", __func__);
-		return _FAIL;
-	}
-
-	if (is_8021x) {
-		if (left >= 6) {
-			pos += 2;
-			if (!memcmp(pos, RTW_WPA_OUI23A_TYPE, 4)) {
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-					 "%s : there has 802.1x auth\n",
-					 __func__);
-				*is_8021x = 1;
-			}
-		}
-	}
-
-	return ret;
-}
-
-int rtw_parse_wpa2_ie23a(const u8 *rsn_ie, int rsn_ie_len, int *group_cipher,
-		      int *pairwise_cipher, int *is_8021x)
-{
-	int i, ret = _SUCCESS;
-	int left, count;
-	const u8 *pos;
-	u8 SUITE_1X[4] = {0x00, 0x0f, 0xac, 0x01};
-
-	if (rsn_ie_len <= 0) {
-		/* No RSN IE - fail silently */
-		return _FAIL;
-	}
-
-	if (*rsn_ie != WLAN_EID_RSN || *(rsn_ie+1) != (u8)(rsn_ie_len - 2)) {
-		return _FAIL;
-	}
-
-	pos = rsn_ie;
-	pos += 4;
-	left = rsn_ie_len - 4;
-
-	/* group_cipher */
-	if (left >= RSN_SELECTOR_LEN) {
-		*group_cipher = rtw_get_wpa2_cipher_suite(pos);
-
-		pos += RSN_SELECTOR_LEN;
-		left -= RSN_SELECTOR_LEN;
-	} else if (left > 0) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "%s: ie length mismatch, %u too much\n",
-			 __func__, left);
-		return _FAIL;
-	}
-
-	/* pairwise_cipher */
-	if (left >= 2) {
-	        /* count = le16_to_cpu(*(u16*)pos); */
-		count = get_unaligned_le16(pos);
-		pos += 2;
-		left -= 2;
-
-		if (count == 0 || left < count * RSN_SELECTOR_LEN) {
-			RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-				 "%s: ie count botch (pairwise), count %u left %u\n",
-				 __func__, count, left);
-			return _FAIL;
-		}
-
-		for (i = 0; i < count; i++) {
-			*pairwise_cipher |= rtw_get_wpa2_cipher_suite(pos);
-
-			pos += RSN_SELECTOR_LEN;
-			left -= RSN_SELECTOR_LEN;
-		}
-	} else if (left == 1) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "%s: ie too short (for key mgmt)\n",  __func__);
-
-		return _FAIL;
-	}
-
-	if (is_8021x) {
-		if (left >= 6) {
-			pos += 2;
-			if (!memcmp(pos, SUITE_1X, 4)) {
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-					 "%s (): there has 802.1x auth\n",
-					 __func__);
-				*is_8021x = 1;
-			}
-		}
-	}
-
-	return ret;
-}
-
-/**
- * rtw_get_wps_attr23a - Search a specific WPS attribute from a given WPS IE
- * @wps_ie: Address of WPS IE to search
- * @wps_ielen: Length limit from wps_ie
- * @target_attr_id: The attribute ID of WPS attribute to search
- * @buf_attr: If not NULL and the WPS attribute is found, WPS attribute
- *            will be copied to the buf starting from buf_attr
- * @len_attr: If not NULL and the WPS attribute is found, will set to the
- *            length of the entire WPS attribute
- *
- * Returns: the address of the specific WPS attribute found, or NULL
- */
-const u8 *rtw_get_wps_attr23a(const u8 *wps_ie, uint wps_ielen,
-			      u16 target_attr_id, u8 *buf_attr, u32 *len_attr)
-{
-	const u8 *attr_ptr = NULL;
-	const u8 *target_attr_ptr = NULL;
-	u8 wps_oui[4] = {0x00, 0x50, 0xF2, 0x04};
-
-	if (len_attr)
-		*len_attr = 0;
-
-	if (wps_ie[0] != WLAN_EID_VENDOR_SPECIFIC ||
-	    memcmp(wps_ie + 2, wps_oui, 4)) {
-		return attr_ptr;
-	}
-
-	/*  6 = 1(Element ID) + 1(Length) + 4(WPS OUI) */
-	attr_ptr = wps_ie + 6; /* goto first attr */
-
-	while (attr_ptr - wps_ie < wps_ielen) {
-		/*  4 = 2(Attribute ID) + 2(Length) */
-		u16 attr_id = get_unaligned_be16(attr_ptr);
-		u16 attr_data_len = get_unaligned_be16(attr_ptr + 2);
-		u16 attr_len = attr_data_len + 4;
-
-		/* DBG_8723A("%s attr_ptr:%p, id:%u, length:%u\n", __func__, attr_ptr, attr_id, attr_data_len); */
-		if (attr_id == target_attr_id) {
-			target_attr_ptr = attr_ptr;
-
-			if (buf_attr)
-				memcpy(buf_attr, attr_ptr, attr_len);
-
-			if (len_attr)
-				*len_attr = attr_len;
-
-			break;
-		} else {
-			attr_ptr += attr_len; /* goto next */
-		}
-	}
-
-	return target_attr_ptr;
-}
-
-/**
- * rtw_get_wps_attr_content23a - Search a specific WPS attribute content
- * from a given WPS IE
- * @wps_ie: Address of WPS IE to search
- * @wps_ielen: Length limit from wps_ie
- * @target_attr_id: The attribute ID of WPS attribute to search
- * @buf_content: If not NULL and the WPS attribute is found, WPS attribute
- *               content will be copied to the buf starting from buf_content
- * @len_content: If not NULL and the WPS attribute is found, will set to the
- *               length of the WPS attribute content
- *
- * Returns: the address of the specific WPS attribute content found, or NULL
- */
-const u8 *rtw_get_wps_attr_content23a(const u8 *wps_ie, uint wps_ielen,
-				      u16 target_attr_id, u8 *buf_content)
-{
-	const u8 *attr_ptr;
-	u32 attr_len;
-
-	attr_ptr = rtw_get_wps_attr23a(wps_ie, wps_ielen, target_attr_id,
-				    NULL, &attr_len);
-
-	if (attr_ptr && attr_len) {
-		if (buf_content)
-			memcpy(buf_content, attr_ptr + 4, attr_len - 4);
-
-		return attr_ptr + 4;
-	}
-
-	return NULL;
-}
-
-static int rtw_get_cipher_info(struct wlan_network *pnetwork)
-{
-	const u8 *pbuf;
-	int group_cipher = 0, pairwise_cipher = 0, is8021x = 0;
-	int ret = _FAIL;
-	int r, plen;
-	char *pie;
-
-	pie = pnetwork->network.IEs;
-	plen = pnetwork->network.IELength;
-
-	pbuf = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-				       WLAN_OUI_TYPE_MICROSOFT_WPA, pie, plen);
-
-	if (pbuf && pbuf[1] > 0) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-			 "rtw_get_cipher_info: wpa_ielen: %d\n", pbuf[1]);
-		r = rtw_parse_wpa_ie23a(pbuf, pbuf[1] + 2, &group_cipher,
-				     &pairwise_cipher, &is8021x);
-		if (r == _SUCCESS) {
-			pnetwork->BcnInfo.pairwise_cipher = pairwise_cipher;
-			pnetwork->BcnInfo.group_cipher = group_cipher;
-			pnetwork->BcnInfo.is_8021x = is8021x;
-			RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-				 "%s: pnetwork->pairwise_cipher: %d, is_8021x is %d\n",
-				 __func__, pnetwork->BcnInfo.pairwise_cipher,
-				 pnetwork->BcnInfo.is_8021x);
-			ret = _SUCCESS;
-		}
-	} else {
-		pbuf = cfg80211_find_ie(WLAN_EID_RSN, pie, plen);
-
-		if (pbuf && pbuf[1] > 0) {
-			RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-				 "get RSN IE\n");
-			r = rtw_parse_wpa2_ie23a(pbuf, pbuf[1] + 2,
-					      &group_cipher, &pairwise_cipher,
-					      &is8021x);
-			if (r == _SUCCESS) {
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-					 "get RSN IE  OK!!!\n");
-				pnetwork->BcnInfo.pairwise_cipher =
-					pairwise_cipher;
-				pnetwork->BcnInfo.group_cipher = group_cipher;
-				pnetwork->BcnInfo.is_8021x = is8021x;
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-					 "%s: pnetwork->pairwise_cipher: %d,pnetwork->group_cipher is %d, is_8021x is %d\n",
-					 __func__,
-					 pnetwork->BcnInfo.pairwise_cipher,
-					 pnetwork->BcnInfo.group_cipher,
-					 pnetwork->BcnInfo.is_8021x);
-				ret = _SUCCESS;
-			}
-		}
-	}
-
-	return ret;
-}
-
-void rtw_get_bcn_info23a(struct wlan_network *pnetwork)
-{
-	u8 bencrypt = 0;
-	int pie_len;
-	u8 *pie;
-	const u8 *p;
-
-	if (pnetwork->network.capability & WLAN_CAPABILITY_PRIVACY) {
-		bencrypt = 1;
-		pnetwork->network.Privacy = 1;
-	} else
-		pnetwork->BcnInfo.encryp_protocol = ENCRYP_PROTOCOL_OPENSYS;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "%s: ssid =%s\n", __func__, pnetwork->network.Ssid.ssid);
-
-	pie = pnetwork->network.IEs;
-	pie_len = pnetwork->network.IELength;
-
-	p = cfg80211_find_ie(WLAN_EID_RSN, pie, pie_len);
-	if (p && p[1]) {
-		pnetwork->BcnInfo.encryp_protocol = ENCRYP_PROTOCOL_WPA2;
-	} else if (cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-					   WLAN_OUI_TYPE_MICROSOFT_WPA,
-					   pie, pie_len)) {
-		pnetwork->BcnInfo.encryp_protocol = ENCRYP_PROTOCOL_WPA;
-	} else {
-		if (bencrypt)
-			pnetwork->BcnInfo.encryp_protocol = ENCRYP_PROTOCOL_WEP;
-	}
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "%s: pnetwork->encryp_protocol is %x\n", __func__,
-		 pnetwork->BcnInfo.encryp_protocol);
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "%s: pnetwork->encryp_protocol is %x\n", __func__,
-		 pnetwork->BcnInfo.encryp_protocol);
-	rtw_get_cipher_info(pnetwork);
-
-	/* get bwmode and ch_offset */
-}
-
-/* show MCS rate, unit: 100Kbps */
-u16 rtw_mcs_rate23a(u8 rf_type, u8 bw_40MHz, u8 short_GI_20, u8 short_GI_40,
-		    struct ieee80211_mcs_info *mcs)
-{
-	u16 max_rate = 0;
-
-	if (rf_type == RF_1T1R) {
-		if (mcs->rx_mask[0] & BIT(7))
-			max_rate = (bw_40MHz) ? ((short_GI_40)?1500:1350):
-				((short_GI_20)?722:650);
-		else if (mcs->rx_mask[0] & BIT(6))
-			max_rate = (bw_40MHz) ? ((short_GI_40)?1350:1215):
-				((short_GI_20)?650:585);
-		else if (mcs->rx_mask[0] & BIT(5))
-			max_rate = (bw_40MHz) ? ((short_GI_40)?1200:1080):
-				((short_GI_20)?578:520);
-		else if (mcs->rx_mask[0] & BIT(4))
-			max_rate = (bw_40MHz) ? ((short_GI_40)?900:810):
-				((short_GI_20)?433:390);
-		else if (mcs->rx_mask[0] & BIT(3))
-			max_rate = (bw_40MHz) ? ((short_GI_40)?600:540):
-				((short_GI_20)?289:260);
-		else if (mcs->rx_mask[0] & BIT(2))
-			max_rate = (bw_40MHz) ? ((short_GI_40)?450:405):
-				((short_GI_20)?217:195);
-		else if (mcs->rx_mask[0] & BIT(1))
-			max_rate = (bw_40MHz) ? ((short_GI_40)?300:270):
-				((short_GI_20)?144:130);
-		else if (mcs->rx_mask[0] & BIT(0))
-			max_rate = (bw_40MHz) ? ((short_GI_40)?150:135):
-				((short_GI_20)?72:65);
-	} else {
-		if (mcs->rx_mask[1]) {
-			if (mcs->rx_mask[1] & BIT(7))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?3000:2700):((short_GI_20)?1444:1300);
-			else if (mcs->rx_mask[1] & BIT(6))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?2700:2430):((short_GI_20)?1300:1170);
-			else if (mcs->rx_mask[1] & BIT(5))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?2400:2160):((short_GI_20)?1156:1040);
-			else if (mcs->rx_mask[1] & BIT(4))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?1800:1620):((short_GI_20)?867:780);
-			else if (mcs->rx_mask[1] & BIT(3))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?1200:1080):((short_GI_20)?578:520);
-			else if (mcs->rx_mask[1] & BIT(2))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?900:810):((short_GI_20)?433:390);
-			else if (mcs->rx_mask[1] & BIT(1))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?600:540):((short_GI_20)?289:260);
-			else if (mcs->rx_mask[1] & BIT(0))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?300:270):((short_GI_20)?144:130);
-		} else {
-			if (mcs->rx_mask[0] & BIT(7))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?1500:1350):((short_GI_20)?722:650);
-			else if (mcs->rx_mask[0] & BIT(6))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?1350:1215):((short_GI_20)?650:585);
-			else if (mcs->rx_mask[0] & BIT(5))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?1200:1080):((short_GI_20)?578:520);
-			else if (mcs->rx_mask[0] & BIT(4))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?900:810):((short_GI_20)?433:390);
-			else if (mcs->rx_mask[0] & BIT(3))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?600:540):((short_GI_20)?289:260);
-			else if (mcs->rx_mask[0] & BIT(2))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?450:405):((short_GI_20)?217:195);
-			else if (mcs->rx_mask[0] & BIT(1))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?300:270):((short_GI_20)?144:130);
-			else if (mcs->rx_mask[0] & BIT(0))
-				max_rate = (bw_40MHz) ? ((short_GI_40)?150:135):((short_GI_20)?72:65);
-		}
-	}
-	return max_rate;
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_mlme.c b/drivers/staging/rtl8723au/core/rtw_mlme.c
deleted file mode 100644
index a786fc4bdb53..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_mlme.c
+++ /dev/null
@@ -1,2314 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_MLME_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <recv_osdep.h>
-#include <xmit_osdep.h>
-#include <hal_intf.h>
-#include <mlme_osdep.h>
-#include <sta_info.h>
-#include <linux/ieee80211.h>
-#include <wifi.h>
-#include <wlan_bssdef.h>
-#include <rtw_sreset.h>
-
-static struct wlan_network *
-rtw_select_candidate_from_queue(struct mlme_priv *pmlmepriv);
-static int rtw_do_join(struct rtw_adapter *padapter);
-
-static void rtw_init_mlme_timer(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	setup_timer(&pmlmepriv->assoc_timer, rtw23a_join_to_handler,
-		    (unsigned long)padapter);
-
-	setup_timer(&pmlmepriv->scan_to_timer, rtw_scan_timeout_handler23a,
-		    (unsigned long)padapter);
-
-	setup_timer(&pmlmepriv->dynamic_chk_timer,
-		    rtw_dynamic_check_timer_handler, (unsigned long)padapter);
-
-	setup_timer(&pmlmepriv->set_scan_deny_timer,
-		    rtw_set_scan_deny_timer_hdl, (unsigned long)padapter);
-}
-
-int rtw_init_mlme_priv23a(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	pmlmepriv->nic_hdl = padapter;
-
-	pmlmepriv->fw_state = 0;
-	pmlmepriv->cur_network.network.ifmode = NL80211_IFTYPE_UNSPECIFIED;
-	/*  1: active, 0: pasive. Maybe someday we should rename this
-	    varable to "active_mode" (Jeff) */
-	pmlmepriv->scan_mode = SCAN_ACTIVE;
-
-	spin_lock_init(&pmlmepriv->lock);
-	_rtw_init_queue23a(&pmlmepriv->scanned_queue);
-
-	memset(&pmlmepriv->assoc_ssid, 0, sizeof(struct cfg80211_ssid));
-
-	rtw_clear_scan_deny(padapter);
-
-	rtw_init_mlme_timer(padapter);
-	return _SUCCESS;
-}
-
-#ifdef CONFIG_8723AU_AP_MODE
-static void rtw_free_mlme_ie_data(u8 **ppie, u32 *plen)
-{
-	if (*ppie) {
-		kfree(*ppie);
-		*plen = 0;
-		*ppie = NULL;
-	}
-}
-#endif
-
-void rtw23a_free_mlme_priv_ie_data(struct mlme_priv *pmlmepriv)
-{
-#ifdef CONFIG_8723AU_AP_MODE
-	kfree(pmlmepriv->assoc_req);
-	kfree(pmlmepriv->assoc_rsp);
-	rtw_free_mlme_ie_data(&pmlmepriv->wps_probe_req_ie,
-			      &pmlmepriv->wps_probe_req_ie_len);
-#endif
-}
-
-void rtw_free_mlme_priv23a(struct mlme_priv *pmlmepriv)
-{
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-		 "rtw_free_mlme_priv23a\n");
-
-	rtw23a_free_mlme_priv_ie_data(pmlmepriv);
-}
-
-struct wlan_network *rtw_alloc_network(struct mlme_priv *pmlmepriv, gfp_t gfp)
-{
-	struct wlan_network *pnetwork;
-
-	pnetwork = kzalloc(sizeof(struct wlan_network), gfp);
-	if (pnetwork) {
-		INIT_LIST_HEAD(&pnetwork->list);
-		pnetwork->network_type = 0;
-		pnetwork->fixed = false;
-		pnetwork->last_scanned = jiffies;
-		pnetwork->join_res = 0;
-	}
-
-	return pnetwork;
-}
-
-static void _rtw_free_network23a(struct mlme_priv *pmlmepriv,
-				 struct wlan_network *pnetwork)
-{
-	if (!pnetwork)
-		return;
-
-	if (pnetwork->fixed == true)
-		return;
-
-	list_del_init(&pnetwork->list);
-
-	kfree(pnetwork);
-}
-
-/*
- return the wlan_network with the matching addr
-
- Shall be called under atomic context... to avoid possible racing condition...
-*/
-struct wlan_network *
-rtw_find_network23a(struct rtw_queue *scanned_queue, u8 *addr)
-{
-	struct list_head *phead, *plist;
-	struct wlan_network *pnetwork = NULL;
-
-	if (is_zero_ether_addr(addr)) {
-		pnetwork = NULL;
-		goto exit;
-	}
-
-	/* spin_lock_bh(&scanned_queue->lock); */
-
-	phead = get_list_head(scanned_queue);
-	plist = phead->next;
-
-	while (plist != phead) {
-		pnetwork = container_of(plist, struct wlan_network, list);
-
-		if (ether_addr_equal(addr, pnetwork->network.MacAddress))
-			break;
-
-		plist = plist->next;
-	}
-
-	if (plist == phead)
-		pnetwork = NULL;
-
-	/* spin_unlock_bh(&scanned_queue->lock); */
-
-exit:
-
-	return pnetwork;
-}
-
-void rtw_free_network_queue23a(struct rtw_adapter *padapter)
-{
-	struct list_head *phead;
-	struct wlan_network *pnetwork, *ptmp;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct rtw_queue *scanned_queue = &pmlmepriv->scanned_queue;
-
-	spin_lock_bh(&scanned_queue->lock);
-	phead = get_list_head(scanned_queue);
-	list_for_each_entry_safe(pnetwork, ptmp, phead, list)
-		_rtw_free_network23a(pmlmepriv, pnetwork);
-	spin_unlock_bh(&scanned_queue->lock);
-}
-
-int rtw_if_up23a(struct rtw_adapter *padapter)
-{
-	int res;
-
-	if (padapter->bDriverStopped || padapter->bSurpriseRemoved ||
-	    !check_fwstate(&padapter->mlmepriv, _FW_LINKED)) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-			 "rtw_if_up23a:bDriverStopped(%d) OR bSurpriseRemoved(%d)\n",
-			 padapter->bDriverStopped, padapter->bSurpriseRemoved);
-		res = false;
-	} else
-		res =  true;
-
-	return res;
-}
-
-void rtw_generate_random_ibss23a(u8 *pibss)
-{
-	unsigned long curtime = jiffies;
-
-	pibss[0] = 0x02;  /* in ad-hoc mode bit1 must set to 1 */
-	pibss[1] = 0x11;
-	pibss[2] = 0x87;
-	pibss[3] = curtime & 0xff;/* p[0]; */
-	pibss[4] = (curtime >> 8) & 0xff;/* p[1]; */
-	pibss[5] = (curtime >> 16) & 0xff;/* p[2]; */
-}
-
-void rtw_set_roaming(struct rtw_adapter *adapter, u8 to_roaming)
-{
-	if (to_roaming == 0)
-		adapter->mlmepriv.to_join = false;
-	adapter->mlmepriv.to_roaming = to_roaming;
-}
-
-static void _rtw_roaming(struct rtw_adapter *padapter,
-			 struct wlan_network *tgt_network)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network *pnetwork;
-	int do_join_r;
-
-	if (tgt_network)
-		pnetwork = tgt_network;
-	else
-		pnetwork = &pmlmepriv->cur_network;
-
-	if (padapter->mlmepriv.to_roaming > 0) {
-		DBG_8723A("roaming from %s(%pM), length:%d\n",
-			  pnetwork->network.Ssid.ssid,
-			  pnetwork->network.MacAddress,
-			  pnetwork->network.Ssid.ssid_len);
-		memcpy(&pmlmepriv->assoc_ssid, &pnetwork->network.Ssid,
-		       sizeof(struct cfg80211_ssid));
-
-		pmlmepriv->assoc_by_bssid = false;
-
-		while (1) {
-			do_join_r = rtw_do_join(padapter);
-			if (do_join_r == _SUCCESS)
-				break;
-			else {
-				DBG_8723A("roaming do_join return %d\n",
-					  do_join_r);
-				pmlmepriv->to_roaming--;
-
-				if (padapter->mlmepriv.to_roaming > 0)
-					continue;
-				else {
-					DBG_8723A("%s(%d) -to roaming fail, "
-						  "indicate_disconnect\n",
-						  __func__, __LINE__);
-					rtw_indicate_disconnect23a(padapter);
-					break;
-				}
-			}
-		}
-	}
-}
-
-void rtw23a_roaming(struct rtw_adapter *padapter,
-		    struct wlan_network *tgt_network)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	spin_lock_bh(&pmlmepriv->lock);
-	_rtw_roaming(padapter, tgt_network);
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
-static void rtw_free_network_nolock(struct mlme_priv *pmlmepriv,
-				    struct wlan_network *pnetwork)
-{
-	_rtw_free_network23a(pmlmepriv, pnetwork);
-}
-
-bool rtw_is_same_ibss23a(struct rtw_adapter *adapter,
-			 struct wlan_network *pnetwork)
-{
-	int ret;
-	struct security_priv *psecuritypriv = &adapter->securitypriv;
-
-	if (psecuritypriv->dot11PrivacyAlgrthm != 0 &&
-	    pnetwork->network.Privacy == 0)
-		ret = false;
-	else if (psecuritypriv->dot11PrivacyAlgrthm == 0 &&
-		 pnetwork->network.Privacy == 1)
-		ret = false;
-	else
-		ret = true;
-
-	return ret;
-}
-
-inline int is_same_ess(struct wlan_bssid_ex *a, struct wlan_bssid_ex *b);
-inline int is_same_ess(struct wlan_bssid_ex *a, struct wlan_bssid_ex *b)
-{
-	return (a->Ssid.ssid_len == b->Ssid.ssid_len) &&
-		!memcmp(a->Ssid.ssid, b->Ssid.ssid, a->Ssid.ssid_len);
-}
-
-int is_same_network23a(struct wlan_bssid_ex *src, struct wlan_bssid_ex *dst)
-{
-	u16 s_cap, d_cap;
-
-	s_cap = src->capability;
-	d_cap = dst->capability;
-
-	return ((src->Ssid.ssid_len == dst->Ssid.ssid_len) &&
-		/*	(src->DSConfig == dst->DSConfig) && */
-		ether_addr_equal(src->MacAddress, dst->MacAddress) &&
-		!memcmp(src->Ssid.ssid, dst->Ssid.ssid, src->Ssid.ssid_len) &&
-		(s_cap & WLAN_CAPABILITY_IBSS) ==
-		(d_cap & WLAN_CAPABILITY_IBSS) &&
-		(s_cap & WLAN_CAPABILITY_ESS) == (d_cap & WLAN_CAPABILITY_ESS));
-}
-
-struct wlan_network *
-rtw_get_oldest_wlan_network23a(struct rtw_queue *scanned_queue)
-{
-	struct list_head *phead;
-	struct wlan_network *pwlan;
-	struct wlan_network *oldest = NULL;
-
-	phead = get_list_head(scanned_queue);
-	list_for_each_entry(pwlan, phead, list) {
-		if (pwlan->fixed != true) {
-			if (!oldest || time_after(oldest->last_scanned,
-						  pwlan->last_scanned))
-				oldest = pwlan;
-		}
-	}
-
-	return oldest;
-}
-
-void update_network23a(struct wlan_bssid_ex *dst, struct wlan_bssid_ex *src,
-		       struct rtw_adapter *padapter, bool update_ie)
-{
-	u8 ss_ori = dst->SignalStrength;
-	u8 sq_ori = dst->SignalQuality;
-	long rssi_ori = dst->Rssi;
-
-	u8 ss_smp = src->SignalStrength;
-	u8 sq_smp = src->SignalQuality;
-	long rssi_smp = src->Rssi;
-
-	u8 ss_final;
-	u8 sq_final;
-	long rssi_final;
-
-	DBG_8723A("%s %s(%pM, ch%u) ss_ori:%3u, sq_ori:%3u, rssi_ori:%3ld, "
-		  "ss_smp:%3u, sq_smp:%3u, rssi_smp:%3ld\n",
-		  __func__, src->Ssid.ssid, src->MacAddress,
-		  src->DSConfig, ss_ori, sq_ori, rssi_ori,
-		  ss_smp, sq_smp, rssi_smp
-	);
-
-	/* The rule below is 1/5 for sample value, 4/5 for history value */
-	if (check_fwstate(&padapter->mlmepriv, _FW_LINKED) &&
-	    is_same_network23a(&padapter->mlmepriv.cur_network.network, src)) {
-		/* Take the recvpriv's value for the connected AP*/
-		ss_final = padapter->recvpriv.signal_strength;
-		sq_final = padapter->recvpriv.signal_qual;
-		/* the rssi value here is undecorated, and will be
-		   used for antenna diversity */
-		if (sq_smp != 101) /* from the right channel */
-			rssi_final = (src->Rssi+dst->Rssi*4)/5;
-		else
-			rssi_final = rssi_ori;
-	} else {
-		if (sq_smp != 101) { /* from the right channel */
-			ss_final = ((u32)src->SignalStrength +
-				    (u32)dst->SignalStrength * 4) / 5;
-			sq_final = ((u32)src->SignalQuality +
-				    (u32)dst->SignalQuality * 4) / 5;
-			rssi_final = src->Rssi+dst->Rssi * 4 / 5;
-		} else {
-			/* bss info not receiving from the right channel, use
-			   the original RX signal infos */
-			ss_final = dst->SignalStrength;
-			sq_final = dst->SignalQuality;
-			rssi_final = dst->Rssi;
-		}
-
-	}
-
-	if (update_ie)
-		memcpy(dst, src, get_wlan_bssid_ex_sz(src));
-
-	dst->SignalStrength = ss_final;
-	dst->SignalQuality = sq_final;
-	dst->Rssi = rssi_final;
-
-	DBG_8723A("%s %s(%pM), SignalStrength:%u, SignalQuality:%u, "
-		  "RawRSSI:%ld\n",  __func__, dst->Ssid.ssid, dst->MacAddress,
-		  dst->SignalStrength, dst->SignalQuality, dst->Rssi);
-}
-
-static void update_current_network(struct rtw_adapter *adapter,
-				   struct wlan_bssid_ex *pnetwork)
-{
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-
-	if (check_fwstate(pmlmepriv, _FW_LINKED) &&
-	    is_same_network23a(&pmlmepriv->cur_network.network, pnetwork)) {
-		update_network23a(&pmlmepriv->cur_network.network,
-				  pnetwork, adapter, true);
-
-		rtw_update_protection23a(adapter,
-					 pmlmepriv->cur_network.network.IEs,
-					 pmlmepriv->cur_network.network.IELength);
-	}
-}
-
-/*
-
-Caller must hold pmlmepriv->lock first.
-
-*/
-static void rtw_update_scanned_network(struct rtw_adapter *adapter,
-				       struct wlan_bssid_ex *target)
-{
-	struct list_head *plist, *phead;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct wlan_network *pnetwork = NULL;
-	struct wlan_network *oldest = NULL;
-	struct rtw_queue *queue = &pmlmepriv->scanned_queue;
-	u32 bssid_ex_sz;
-	int found = 0;
-
-	spin_lock_bh(&queue->lock);
-	phead = get_list_head(queue);
-	list_for_each(plist, phead) {
-		pnetwork = container_of(plist, struct wlan_network, list);
-
-		if (is_same_network23a(&pnetwork->network, target)) {
-			found = 1;
-			break;
-		}
-		if (!oldest || time_after(oldest->last_scanned,
-					  pnetwork->last_scanned))
-			oldest = pnetwork;
-	}
-
-	/* If we didn't find a match, then get a new network slot to initialize
-	 * with this beacon's information */
-	if (!found) {
-		pnetwork = rtw_alloc_network(pmlmepriv, GFP_ATOMIC);
-		if (!pnetwork) {
-			if (!oldest) {
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-					 "something wrong here\n");
-				goto exit;
-			}
-			pnetwork = oldest;
-		} else
-			list_add_tail(&pnetwork->list, &queue->queue);
-
-		bssid_ex_sz = get_wlan_bssid_ex_sz(target);
-		target->Length = bssid_ex_sz;
-		memcpy(&pnetwork->network, target, bssid_ex_sz);
-
-		/*  variable initialize */
-		pnetwork->fixed = false;
-		pnetwork->last_scanned = jiffies;
-
-		pnetwork->network_type = 0;
-		pnetwork->join_res = 0;
-
-		/* bss info not receiving from the right channel */
-		if (pnetwork->network.SignalQuality == 101)
-			pnetwork->network.SignalQuality = 0;
-	} else {
-		/*
-		 * we have an entry and we are going to update it. But
-		 * this entry may be already expired. In this case we
-		 * do the same as we found a new net and call the
-		 * new_net handler
-		 */
-		bool update_ie = true;
-
-		pnetwork->last_scanned = jiffies;
-
-		/* target.reserved == 1, means that scanned network is
-		 * a bcn frame. */
-		if (pnetwork->network.IELength > target->IELength &&
-		    target->reserved == 1)
-			update_ie = false;
-
-		update_network23a(&pnetwork->network, target, adapter,
-				  update_ie);
-	}
-
-exit:
-	spin_unlock_bh(&queue->lock);
-}
-
-static void rtw_add_network(struct rtw_adapter *adapter,
-			    struct wlan_bssid_ex *pnetwork)
-{
-	update_current_network(adapter, pnetwork);
-	rtw_update_scanned_network(adapter, pnetwork);
-}
-
-/* select the desired network based on the capability of the (i)bss. */
-/*  check items: (1) security */
-/*			   (2) network_type */
-/*			   (3) WMM */
-/*			   (4) HT */
-/*                      (5) others */
-static int rtw_is_desired_network(struct rtw_adapter *adapter,
-				  struct wlan_network *pnetwork)
-{
-	struct security_priv *psecuritypriv = &adapter->securitypriv;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	u32 desired_encmode;
-	u32 privacy;
-	int bselected = true;
-
-	desired_encmode = psecuritypriv->ndisencryptstatus;
-	privacy = pnetwork->network.Privacy;
-
-	if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
-		if (cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-					    WLAN_OUI_TYPE_MICROSOFT_WPA,
-					    pnetwork->network.IEs,
-					    pnetwork->network.IELength))
-			return true;
-		else
-			return false;
-	}
-	if (adapter->registrypriv.wifi_spec == 1) {
-		/* for  correct flow of 8021X  to do.... */
-		if (desired_encmode == Ndis802_11EncryptionDisabled &&
-		    privacy != 0)
-			bselected = false;
-	}
-
-	if (desired_encmode != Ndis802_11EncryptionDisabled && privacy == 0) {
-		DBG_8723A("desired_encmode: %d, privacy: %d\n",
-			  desired_encmode, privacy);
-		bselected = false;
-	}
-
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
-		if (pnetwork->network.ifmode !=
-		    pmlmepriv->cur_network.network.ifmode)
-			bselected = false;
-	}
-
-	return bselected;
-}
-
-void rtw_survey_event_cb23a(struct rtw_adapter *adapter, const u8 *pbuf)
-{
-	u32 len;
-	struct wlan_bssid_ex *pnetwork;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct survey_event *survey = (struct survey_event *)pbuf;
-
-	pnetwork = survey->bss;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "rtw_survey_event_cb23a, ssid=%s\n", pnetwork->Ssid.ssid);
-
-	len = get_wlan_bssid_ex_sz(pnetwork);
-	if (len > (sizeof(struct wlan_bssid_ex))) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "****rtw_survey_event_cb23a: return a wrong bss ***\n");
-		return;
-	}
-
-	spin_lock_bh(&pmlmepriv->lock);
-
-	/*  update IBSS_network 's timestamp */
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
-		if (ether_addr_equal(pmlmepriv->cur_network.network.MacAddress,
-				     pnetwork->MacAddress)) {
-			struct wlan_network *ibss_wlan;
-
-			pmlmepriv->cur_network.network.beacon_interval =
-				pnetwork->beacon_interval;
-			pmlmepriv->cur_network.network.capability =
-				pnetwork->capability;
-			pmlmepriv->cur_network.network.tsf = pnetwork->tsf;
-			spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-			ibss_wlan = rtw_find_network23a(
-				&pmlmepriv->scanned_queue,
-				pnetwork->MacAddress);
-			if (ibss_wlan) {
-				pmlmepriv->cur_network.network.beacon_interval =
-					ibss_wlan->network.beacon_interval;
-				pmlmepriv->cur_network.network.capability =
-					ibss_wlan->network.capability;
-				pmlmepriv->cur_network.network.tsf =
-					ibss_wlan->network.tsf;
-				spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-				goto exit;
-			}
-			spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-		}
-	}
-
-	/*  lock pmlmepriv->lock when you accessing network_q */
-	if (!check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) {
-		if (pnetwork->Ssid.ssid[0] == 0)
-			pnetwork->Ssid.ssid_len = 0;
-
-		rtw_add_network(adapter, pnetwork);
-	}
-
-exit:
-
-	spin_unlock_bh(&pmlmepriv->lock);
-
-	kfree(survey->bss);
-	survey->bss = NULL;
-}
-
-void
-rtw_surveydone_event_callback23a(struct rtw_adapter *adapter, const u8 *pbuf)
-{
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
-	int ret;
-
-	spin_lock_bh(&pmlmepriv->lock);
-
-	if (pmlmepriv->wps_probe_req_ie) {
-		pmlmepriv->wps_probe_req_ie_len = 0;
-		kfree(pmlmepriv->wps_probe_req_ie);
-		pmlmepriv->wps_probe_req_ie = NULL;
-	}
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "rtw_surveydone_event_callback23a: fw_state:%x\n",
-		 get_fwstate(pmlmepriv));
-
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY)) {
-		del_timer_sync(&pmlmepriv->scan_to_timer);
-
-		_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY);
-	} else {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "nic status =%x, survey done event comes too late!\n",
-			 get_fwstate(pmlmepriv));
-	}
-
-	rtw_set_signal_stat_timer(&adapter->recvpriv);
-
-	if (pmlmepriv->to_join == true) {
-		set_fwstate(pmlmepriv, _FW_UNDER_LINKING);
-		if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
-			ret = rtw_select_and_join_from_scanned_queue23a(
-				pmlmepriv);
-			if (ret != _SUCCESS)
-				rtw_do_join_adhoc(adapter);
-		} else {
-			pmlmepriv->to_join = false;
-			ret = rtw_select_and_join_from_scanned_queue23a(
-				pmlmepriv);
-			if (ret != _SUCCESS) {
-				DBG_8723A("try_to_join, but select scanning "
-					  "queue fail, to_roaming:%d\n",
-					  adapter->mlmepriv.to_roaming);
-				if (adapter->mlmepriv.to_roaming) {
-					if (--pmlmepriv->to_roaming == 0 ||
-					    rtw_sitesurvey_cmd23a(
-						    adapter,
-						    &pmlmepriv->assoc_ssid, 1,
-						    NULL, 0) != _SUCCESS) {
-						rtw_set_roaming(adapter, 0);
-						rtw_free_assoc_resources23a(
-							adapter, 1);
-						rtw_indicate_disconnect23a(
-							adapter);
-					} else
-						pmlmepriv->to_join = true;
-				}
-				_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
-			}
-		}
-	}
-
-	spin_unlock_bh(&pmlmepriv->lock);
-
-	rtw_os_xmit_schedule23a(adapter);
-
-	if (pmlmeext->sitesurvey_res.bss_cnt == 0)
-		rtw_sreset_reset(adapter);
-
-	rtw_cfg80211_surveydone_event_callback(adapter);
-}
-
-static void free_scanqueue(struct mlme_priv *pmlmepriv)
-{
-	struct wlan_network *pnetwork, *ptemp;
-	struct rtw_queue *scan_queue = &pmlmepriv->scanned_queue;
-	struct list_head *phead;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_notice_, "+free_scanqueue\n");
-	spin_lock_bh(&scan_queue->lock);
-	phead = get_list_head(scan_queue);
-	list_for_each_entry_safe(pnetwork, ptemp, phead, list) {
-		pnetwork->fixed = false;
-		_rtw_free_network23a(pmlmepriv, pnetwork);
-	}
-	spin_unlock_bh(&scan_queue->lock);
-}
-
-/*
- *rtw_free_assoc_resources23a: the caller has to lock pmlmepriv->lock
- */
-void rtw_free_assoc_resources23a(struct rtw_adapter *adapter,
-				 int lock_scanned_queue)
-{
-	struct wlan_network *pwlan;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct sta_priv *pstapriv = &adapter->stapriv;
-	struct wlan_network *tgt_network = &pmlmepriv->cur_network;
-	struct sta_info *psta;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_notice_,
-		 "+rtw_free_assoc_resources23a\n");
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "tgt_network->network.MacAddress=%pM ssid=%s\n",
-		 tgt_network->network.MacAddress,
-		 tgt_network->network.Ssid.ssid);
-
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE|WIFI_AP_STATE)) {
-		psta = rtw_get_stainfo23a(&adapter->stapriv,
-					  tgt_network->network.MacAddress);
-
-		spin_lock_bh(&pstapriv->sta_hash_lock);
-		rtw_free_stainfo23a(adapter,  psta);
-		spin_unlock_bh(&pstapriv->sta_hash_lock);
-	}
-
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE |
-			  WIFI_ADHOC_MASTER_STATE|WIFI_AP_STATE)) {
-		rtw_free_all_stainfo23a(adapter);
-
-		psta = rtw_get_bcmc_stainfo23a(adapter);
-		spin_lock_bh(&pstapriv->sta_hash_lock);
-		rtw_free_stainfo23a(adapter, psta);
-		spin_unlock_bh(&pstapriv->sta_hash_lock);
-
-		rtw_init_bcmc_stainfo23a(adapter);
-	}
-
-	if (lock_scanned_queue)
-		spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-
-	pwlan = rtw_find_network23a(&pmlmepriv->scanned_queue,
-				    tgt_network->network.MacAddress);
-	if (pwlan)
-		pwlan->fixed = false;
-	else
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "rtw_free_assoc_resources23a : pwlan== NULL\n");
-
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) &&
-	    adapter->stapriv.asoc_sta_count == 1)
-		rtw_free_network_nolock(pmlmepriv, pwlan);
-
-	if (lock_scanned_queue)
-		spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-
-	pmlmepriv->key_mask = 0;
-}
-
-/*
-*rtw_indicate_connect23a: the caller has to lock pmlmepriv->lock
-*/
-void rtw_indicate_connect23a(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-		 "+rtw_indicate_connect23a\n");
-
-	pmlmepriv->to_join = false;
-
-	if (!check_fwstate(&padapter->mlmepriv, _FW_LINKED)) {
-		set_fwstate(pmlmepriv, _FW_LINKED);
-
-		rtw_cfg80211_indicate_connect(padapter);
-
-		netif_carrier_on(padapter->pnetdev);
-
-		if (padapter->pid[2] != 0)
-			kill_pid(find_vpid(padapter->pid[2]), SIGALRM, 1);
-	}
-
-	rtw_set_roaming(padapter, 0);
-
-	rtw_set_scan_deny(padapter, 3000);
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-		 "-rtw_indicate_connect23a: fw_state=0x%08x\n",
-		 get_fwstate(pmlmepriv));
-}
-
-/*
- *rtw_indicate_disconnect23a: the caller has to lock pmlmepriv->lock
- */
-void rtw_indicate_disconnect23a(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-		 "+rtw_indicate_disconnect23a\n");
-
-	_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING|WIFI_UNDER_WPS);
-
-	/* DBG_8723A("clear wps when %s\n", __func__); */
-
-	if (padapter->mlmepriv.to_roaming > 0)
-		_clr_fwstate_(pmlmepriv, _FW_LINKED);
-
-	if (check_fwstate(&padapter->mlmepriv, _FW_LINKED) ||
-	    padapter->mlmepriv.to_roaming <= 0) {
-		rtw_os_indicate_disconnect23a(padapter);
-
-		/* set ips_deny_time to avoid enter IPS before LPS leave */
-		padapter->pwrctrlpriv.ips_deny_time =
-			jiffies + msecs_to_jiffies(3000);
-
-		_clr_fwstate_(pmlmepriv, _FW_LINKED);
-
-		rtw_clear_scan_deny(padapter);
-	}
-
-	rtw_lps_ctrl_wk_cmd23a(padapter, LPS_CTRL_DISCONNECT, 1);
-}
-
-void rtw_scan_abort23a(struct rtw_adapter *adapter)
-{
-	unsigned long start;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
-
-	start = jiffies;
-	pmlmeext->scan_abort = true;
-	while (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY) &&
-	       jiffies_to_msecs(jiffies - start) <= 200) {
-		if (adapter->bDriverStopped || adapter->bSurpriseRemoved)
-			break;
-
-		DBG_8723A("%s(%s): fw_state = _FW_UNDER_SURVEY!\n",
-			  __func__, adapter->pnetdev->name);
-		msleep(20);
-	}
-
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY)) {
-		if (!adapter->bDriverStopped && !adapter->bSurpriseRemoved)
-			DBG_8723A("%s(%s): waiting for scan_abort time out!\n",
-				  __func__, adapter->pnetdev->name);
-		rtw_cfg80211_indicate_scan_done(wdev_to_priv(adapter->rtw_wdev),
-						true);
-	}
-	pmlmeext->scan_abort = false;
-}
-
-static struct sta_info *
-rtw_joinbss_update_stainfo(struct rtw_adapter *padapter,
-			   struct wlan_network *pnetwork)
-{
-	int i;
-	struct sta_info *bmc_sta, *psta;
-	struct recv_reorder_ctrl *preorder_ctrl;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-
-	psta = rtw_get_stainfo23a(pstapriv, pnetwork->network.MacAddress);
-	if (!psta)
-		psta = rtw_alloc_stainfo23a(pstapriv,
-					    pnetwork->network.MacAddress,
-					    GFP_ATOMIC);
-
-	if (psta) { /* update ptarget_sta */
-		DBG_8723A("%s\n", __func__);
-
-		psta->aid  = pnetwork->join_res;
-		psta->mac_id = 0;
-
-		/* sta mode */
-		rtl8723a_SetHalODMVar(padapter, HAL_ODM_STA_INFO, psta, true);
-
-		/* security related */
-		if (padapter->securitypriv.dot11AuthAlgrthm ==
-		    dot11AuthAlgrthm_8021X) {
-			padapter->securitypriv.binstallGrpkey = 0;
-			padapter->securitypriv.busetkipkey = 0;
-
-			psta->ieee8021x_blocked = true;
-			psta->dot118021XPrivacy =
-				padapter->securitypriv.dot11PrivacyAlgrthm;
-
-			memset(&psta->dot118021x_UncstKey, 0,
-			       sizeof (union Keytype));
-
-			memset(&psta->dot11tkiprxmickey, 0,
-			       sizeof (union Keytype));
-			memset(&psta->dot11tkiptxmickey, 0,
-			       sizeof (union Keytype));
-
-			memset(&psta->dot11txpn, 0, sizeof (union pn48));
-			memset(&psta->dot11rxpn, 0, sizeof (union pn48));
-		}
-
-		/*	Commented by Albert 2012/07/21 */
-		/*	When doing the WPS, the wps_ie_len won't equal to 0 */
-		/*	And the Wi-Fi driver shouldn't allow the data packet
-			to be transmitted. */
-		if (padapter->securitypriv.wps_ie_len != 0) {
-			psta->ieee8021x_blocked = true;
-			padapter->securitypriv.wps_ie_len = 0;
-		}
-
-		/* for A-MPDU Rx reordering buffer control for bmc_sta &
-		 * sta_info */
-		/* if A-MPDU Rx is enabled, resetting
-		   rx_ordering_ctrl wstart_b(indicate_seq) to default
-		   value = 0xffff */
-		/* todo: check if AP can send A-MPDU packets */
-		for (i = 0; i < 16 ; i++) {
-			/* preorder_ctrl = &precvpriv->recvreorder_ctrl[i]; */
-			preorder_ctrl = &psta->recvreorder_ctrl[i];
-			preorder_ctrl->enable = false;
-			preorder_ctrl->indicate_seq = 0xffff;
-			preorder_ctrl->wend_b = 0xffff;
-			/* max_ampdu_sz; ex. 32(kbytes) -> wsize_b = 32 */
-			preorder_ctrl->wsize_b = 64;
-		}
-
-		bmc_sta = rtw_get_bcmc_stainfo23a(padapter);
-		if (bmc_sta) {
-			for (i = 0; i < 16 ; i++) {
-				preorder_ctrl = &bmc_sta->recvreorder_ctrl[i];
-				preorder_ctrl->enable = false;
-				preorder_ctrl->indicate_seq = 0xffff;
-				preorder_ctrl->wend_b = 0xffff;
-				/* max_ampdu_sz; ex. 32(kbytes) ->
-				   wsize_b = 32 */
-				preorder_ctrl->wsize_b = 64;
-			}
-		}
-
-		/* misc. */
-		update_sta_info23a(padapter, psta);
-
-	}
-
-	return psta;
-}
-
-/* pnetwork : returns from rtw23a_joinbss_event_cb */
-/* ptarget_wlan: found from scanned_queue */
-static void
-rtw_joinbss_update_network23a(struct rtw_adapter *padapter,
-			      struct wlan_network *ptarget_wlan,
-			      struct wlan_network  *pnetwork)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
-
-	DBG_8723A("%s\n", __func__);
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "fw_state:%x, BSSID:%pM\n",
-		 get_fwstate(pmlmepriv),
-		 pnetwork->network.MacAddress);
-
-	/*  why not use ptarget_wlan?? */
-	memcpy(&cur_network->network, &pnetwork->network,
-	       pnetwork->network.Length);
-	/*  some IEs in pnetwork is wrong, so we should use ptarget_wlan IEs */
-	cur_network->network.IELength = ptarget_wlan->network.IELength;
-	memcpy(&cur_network->network.IEs[0], &ptarget_wlan->network.IEs[0],
-	       MAX_IE_SZ);
-
-	cur_network->network.capability = ptarget_wlan->network.capability;
-	cur_network->network.beacon_interval =
-		ptarget_wlan->network.beacon_interval;
-	cur_network->network.tsf = ptarget_wlan->network.tsf;
-
-	rtw_set_signal_stat_timer(&padapter->recvpriv);
-	padapter->recvpriv.signal_strength =
-		ptarget_wlan->network.SignalStrength;
-	padapter->recvpriv.signal_qual = ptarget_wlan->network.SignalQuality;
-	/*
-	 * the ptarget_wlan->network.Rssi is raw data, we use
-	 * ptarget_wlan->network.SignalStrength instead (has scaled)
-	 */
-	DBG_8723A("%s signal_strength:%3u, signal_qual:%3u\n",
-		  __func__, padapter->recvpriv.signal_strength,
-		  padapter->recvpriv.signal_qual);
-	rtw_set_signal_stat_timer(&padapter->recvpriv);
-
-	/* update fw_state will clr _FW_UNDER_LINKING here indirectly */
-	switch (pnetwork->network.ifmode) {
-	case NL80211_IFTYPE_P2P_CLIENT:
-	case NL80211_IFTYPE_STATION:
-		if (pmlmepriv->fw_state & WIFI_UNDER_WPS)
-			pmlmepriv->fw_state = WIFI_STATION_STATE|WIFI_UNDER_WPS;
-		else
-			pmlmepriv->fw_state = WIFI_STATION_STATE;
-		break;
-	case NL80211_IFTYPE_ADHOC:
-		pmlmepriv->fw_state = WIFI_ADHOC_STATE;
-		break;
-	default:
-		pmlmepriv->fw_state = WIFI_NULL_STATE;
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "Invalid network_mode\n");
-		break;
-	}
-
-	rtw_update_protection23a(padapter, cur_network->network.IEs,
-				 cur_network->network.IELength);
-
-	rtw_update_ht_cap23a(padapter, cur_network->network.IEs,
-			     cur_network->network.IELength);
-}
-
-/*
- * Notes:
- * the function could be > passive_level (the same context as Rx tasklet)
- * pnetwork : returns from rtw23a_joinbss_event_cb
- * ptarget_wlan: found from scanned_queue
- * if join_res > 0, for (fw_state==WIFI_STATION_STATE),
- * we check if  "ptarget_sta" & "ptarget_wlan" exist.
- * if join_res > 0, for (fw_state==WIFI_ADHOC_STATE),
- * we only check if "ptarget_wlan" exist.
- * if join_res > 0, update "cur_network->network" from "pnetwork->network"
- * if (ptarget_wlan !=NULL).
- */
-
-void rtw_joinbss_event_prehandle23a(struct rtw_adapter *adapter, u8 *pbuf)
-{
-	struct sta_info *ptarget_sta, *pcur_sta;
-	struct sta_priv *pstapriv = &adapter->stapriv;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct wlan_network *pnetwork = (struct wlan_network *)pbuf;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
-	struct wlan_network *pcur_wlan, *ptarget_wlan = NULL;
-	bool the_same_macaddr;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "joinbss event call back received with res=%d\n",
-		 pnetwork->join_res);
-
-	if (pmlmepriv->assoc_ssid.ssid_len == 0) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "@@@@@   joinbss event call back  for Any SSid\n");
-	} else {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "@@@@@   rtw23a_joinbss_event_cb for SSid:%s\n",
-			 pmlmepriv->assoc_ssid.ssid);
-	}
-
-	if (ether_addr_equal(pnetwork->network.MacAddress,
-			     cur_network->network.MacAddress))
-		the_same_macaddr = true;
-	else
-		the_same_macaddr = false;
-
-	pnetwork->network.Length = get_wlan_bssid_ex_sz(&pnetwork->network);
-	if (pnetwork->network.Length > sizeof(struct wlan_bssid_ex)) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "***joinbss_evt_callback return a wrong bss ***\n");
-		return;
-	}
-
-	spin_lock_bh(&pmlmepriv->lock);
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "rtw23a_joinbss_event_cb !! _enter_critical\n");
-
-	if (pnetwork->join_res > 0) {
-		spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-		if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) {
-			/* s1. find ptarget_wlan */
-			if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-				if (the_same_macaddr) {
-					ptarget_wlan = rtw_find_network23a(&pmlmepriv->scanned_queue, cur_network->network.MacAddress);
-				} else {
-					pcur_wlan = rtw_find_network23a(&pmlmepriv->scanned_queue, cur_network->network.MacAddress);
-					if (pcur_wlan)
-						pcur_wlan->fixed = false;
-
-					pcur_sta = rtw_get_stainfo23a(pstapriv, cur_network->network.MacAddress);
-					if (pcur_sta) {
-						spin_lock_bh(&pstapriv->sta_hash_lock);
-						rtw_free_stainfo23a(adapter,
-								    pcur_sta);
-						spin_unlock_bh(&pstapriv->sta_hash_lock);
-					}
-
-					ptarget_wlan = rtw_find_network23a(&pmlmepriv->scanned_queue, pnetwork->network.MacAddress);
-					if (check_fwstate(pmlmepriv,
-							  WIFI_STATION_STATE)) {
-						if (ptarget_wlan)
-							ptarget_wlan->fixed =
-								true;
-					}
-				}
-
-			} else {
-				ptarget_wlan = rtw_find_network23a(
-					&pmlmepriv->scanned_queue,
-					pnetwork->network.MacAddress);
-				if (check_fwstate(pmlmepriv,
-						  WIFI_STATION_STATE)) {
-					if (ptarget_wlan)
-						ptarget_wlan->fixed = true;
-				}
-			}
-
-			/* s2. update cur_network */
-			if (ptarget_wlan)
-				rtw_joinbss_update_network23a(adapter,
-							      ptarget_wlan,
-							      pnetwork);
-			else {
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-					 "Can't find ptarget_wlan when joinbss_event callback\n");
-				spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-				goto ignore_joinbss_callback;
-			}
-
-			/* s3. find ptarget_sta & update ptarget_sta after
-			   update cur_network only for station mode */
-			if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
-				ptarget_sta = rtw_joinbss_update_stainfo(
-					adapter, pnetwork);
-				if (!ptarget_sta) {
-					RT_TRACE(_module_rtl871x_mlme_c_,
-						 _drv_err_,
-						 "Can't update stainfo when joinbss_event callback\n");
-					spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-					goto ignore_joinbss_callback;
-				}
-			}
-
-			/* s4. indicate connect */
-			if (check_fwstate(pmlmepriv, WIFI_STATION_STATE))
-				rtw_indicate_connect23a(adapter);
-			else {
-				/* adhoc mode will rtw_indicate_connect23a
-				   when rtw_stassoc_event_callback23a */
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-					 "adhoc mode, fw_state:%x\n",
-					 get_fwstate(pmlmepriv));
-			}
-
-			/* s5. Cancle assoc_timer */
-			del_timer_sync(&pmlmepriv->assoc_timer);
-
-			RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-				 "Cancle assoc_timer\n");
-		} else {
-			RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-				 "rtw23a_joinbss_event_cb err: fw_state:%x\n",
-				 get_fwstate(pmlmepriv));
-			spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-			goto ignore_joinbss_callback;
-		}
-		spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-	} else if (pnetwork->join_res == -4) {
-		rtw_reset_securitypriv23a(adapter);
-		mod_timer(&pmlmepriv->assoc_timer,
-			  jiffies + msecs_to_jiffies(1));
-
-		/* rtw_free_assoc_resources23a(adapter, 1); */
-
-		if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) {
-			RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-				 "fail! clear _FW_UNDER_LINKING ^^^fw_state=%x\n",
-				 get_fwstate(pmlmepriv));
-			_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
-		}
-	} else {
-		/* if join_res < 0 (join fails), then try again */
-		mod_timer(&pmlmepriv->assoc_timer,
-			  jiffies + msecs_to_jiffies(1));
-		_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
-	}
-
-ignore_joinbss_callback:
-
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
-void rtw23a_joinbss_event_cb(struct rtw_adapter *adapter, const u8 *pbuf)
-{
-	struct wlan_network *pnetwork = (struct wlan_network *)pbuf;
-
-	mlmeext_joinbss_event_callback23a(adapter, pnetwork->join_res);
-
-	rtw_os_xmit_schedule23a(adapter);
-}
-
-void rtw_stassoc_event_callback23a(struct rtw_adapter *adapter, const u8 *pbuf)
-{
-	struct sta_info *psta;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct stassoc_event *pstassoc = (struct stassoc_event *)pbuf;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
-	struct wlan_network *ptarget_wlan;
-
-	if (rtw_access_ctrl23a(adapter, pstassoc->macaddr) == false)
-		return;
-
-#ifdef CONFIG_8723AU_AP_MODE
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		psta = rtw_get_stainfo23a(&adapter->stapriv, pstassoc->macaddr);
-		if (psta) {
-			/* bss_cap_update_on_sta_join23a(adapter, psta); */
-			/* sta_info_update23a(adapter, psta); */
-			ap_sta_info_defer_update23a(adapter, psta);
-		}
-		return;
-	}
-#endif
-	/* for AD-HOC mode */
-	psta = rtw_get_stainfo23a(&adapter->stapriv, pstassoc->macaddr);
-	if (psta != NULL) {
-		/* the sta have been in sta_info_queue => do nothing */
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "Error: rtw_stassoc_event_callback23a: sta has been in sta_hash_queue\n");
-		/* between drv has received this event before and
-		   fw have not yet to set key to CAM_ENTRY) */
-		return;
-	}
-
-	psta = rtw_alloc_stainfo23a(&adapter->stapriv, pstassoc->macaddr,
-		GFP_KERNEL);
-	if (!psta) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "Can't alloc sta_info when rtw_stassoc_event_callback23a\n");
-		return;
-	}
-
-	/* to do : init sta_info variable */
-	psta->qos_option = 0;
-	psta->mac_id = (uint)pstassoc->cam_id;
-	/* psta->aid = (uint)pstassoc->cam_id; */
-	DBG_8723A("%s\n", __func__);
-	/* for ad-hoc mode */
-	rtl8723a_SetHalODMVar(adapter, HAL_ODM_STA_INFO, psta, true);
-
-	if (adapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
-		psta->dot118021XPrivacy =
-			adapter->securitypriv.dot11PrivacyAlgrthm;
-
-	psta->ieee8021x_blocked = false;
-
-	spin_lock_bh(&pmlmepriv->lock);
-
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
-		if (adapter->stapriv.asoc_sta_count == 2) {
-			spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-			ptarget_wlan =
-				rtw_find_network23a(&pmlmepriv->scanned_queue,
-						    cur_network->network.MacAddress);
-			if (ptarget_wlan)
-				ptarget_wlan->fixed = true;
-			spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-			/*  a sta + bc/mc_stainfo (not Ibss_stainfo) */
-			rtw_indicate_connect23a(adapter);
-		}
-	}
-
-	spin_unlock_bh(&pmlmepriv->lock);
-
-	mlmeext_sta_add_event_callback23a(adapter, psta);
-}
-
-void rtw_stadel_event_callback23a(struct rtw_adapter *adapter, const u8 *pbuf)
-{
-	int mac_id;
-	struct sta_info *psta;
-	struct wlan_network *pwlan;
-	struct wlan_bssid_ex *pdev_network;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct stadel_event *pstadel = (struct stadel_event *)pbuf;
-	struct sta_priv *pstapriv = &adapter->stapriv;
-	struct wlan_network *tgt_network = &pmlmepriv->cur_network;
-
-	psta = rtw_get_stainfo23a(&adapter->stapriv, pstadel->macaddr);
-	if (psta)
-		mac_id = psta->mac_id;
-	else
-		mac_id = pstadel->mac_id;
-
-	DBG_8723A("%s(mac_id=%d)=%pM\n", __func__, mac_id, pstadel->macaddr);
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
-		return;
-
-	mlmeext_sta_del_event_callback23a(adapter);
-
-	spin_lock_bh(&pmlmepriv->lock);
-
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
-		if (adapter->mlmepriv.to_roaming > 0) {
-			/* this stadel_event is caused by roaming,
-			   decrease to_roaming */
-			pmlmepriv->to_roaming--;
-		} else if (adapter->mlmepriv.to_roaming == 0)
-			rtw_set_roaming(adapter, adapter->registrypriv.max_roaming_times);
-		if (*((u16 *)pstadel->rsvd) != WLAN_REASON_EXPIRATION_CHK)
-			rtw_set_roaming(adapter, 0); /* don't roam */
-
-		rtw_free_uc_swdec_pending_queue23a(adapter);
-
-		rtw_free_assoc_resources23a(adapter, 1);
-		rtw_indicate_disconnect23a(adapter);
-		spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-		/*  remove the network entry in scanned_queue */
-		pwlan = rtw_find_network23a(&pmlmepriv->scanned_queue,
-					    tgt_network->network.MacAddress);
-		if (pwlan) {
-			pwlan->fixed = false;
-			rtw_free_network_nolock(pmlmepriv, pwlan);
-		}
-		spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-
-		_rtw_roaming(adapter, tgt_network);
-	}
-
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
-
-		spin_lock_bh(&pstapriv->sta_hash_lock);
-		rtw_free_stainfo23a(adapter,  psta);
-		spin_unlock_bh(&pstapriv->sta_hash_lock);
-
-		/* a sta + bc/mc_stainfo (not Ibss_stainfo) */
-		if (adapter->stapriv.asoc_sta_count == 1) {
-			spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-			/* free old ibss network */
-			/* pwlan = rtw_find_network23a(
-			   &pmlmepriv->scanned_queue, pstadel->macaddr); */
-			pwlan = rtw_find_network23a(&pmlmepriv->scanned_queue,
-						    tgt_network->network.MacAddress);
-			if (pwlan) {
-				pwlan->fixed = false;
-				rtw_free_network_nolock(pmlmepriv, pwlan);
-			}
-			spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-			/* re-create ibss */
-			pdev_network = &adapter->registrypriv.dev_network;
-
-			memcpy(pdev_network, &tgt_network->network,
-			       get_wlan_bssid_ex_sz(&tgt_network->network));
-
-			rtw_do_join_adhoc(adapter);
-		}
-	}
-
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
-/*
-* rtw23a_join_to_handler - Timeout/failure handler for CMD JoinBss
-* @adapter: pointer to _adapter structure
-*/
-void rtw23a_join_to_handler (unsigned long data)
-{
-	struct rtw_adapter *adapter = (struct rtw_adapter *)data;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	int do_join_r;
-
-	DBG_8723A("%s, fw_state=%x\n", __func__, get_fwstate(pmlmepriv));
-
-	if (adapter->bDriverStopped || adapter->bSurpriseRemoved)
-		return;
-
-	spin_lock_bh(&pmlmepriv->lock);
-
-	if (adapter->mlmepriv.to_roaming > 0) {
-		/* join timeout caused by roaming */
-		while (1) {
-			pmlmepriv->to_roaming--;
-			if (adapter->mlmepriv.to_roaming != 0) {
-				/* try another */
-				DBG_8723A("%s try another roaming\n", __func__);
-				do_join_r = rtw_do_join(adapter);
-				if (do_join_r != _SUCCESS) {
-					DBG_8723A("%s roaming do_join return "
-						  "%d\n", __func__ , do_join_r);
-					continue;
-				}
-				break;
-			} else {
-				DBG_8723A("%s We've try roaming but fail\n",
-					  __func__);
-				rtw_indicate_disconnect23a(adapter);
-				break;
-			}
-		}
-	} else {
-		rtw_indicate_disconnect23a(adapter);
-		free_scanqueue(pmlmepriv);/*  */
-
-		/* indicate disconnect for the case that join_timeout and
-		   check_fwstate != FW_LINKED */
-		rtw_cfg80211_indicate_disconnect(adapter);
-	}
-
-	spin_unlock_bh(&pmlmepriv->lock);
-
-}
-
-/*
-* rtw_scan_timeout_handler23a - Timeout/Failure handler for CMD SiteSurvey
-* @data: pointer to _adapter structure
-*/
-void rtw_scan_timeout_handler23a(unsigned long data)
-{
-	struct rtw_adapter *adapter = (struct rtw_adapter *)data;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-
-	DBG_8723A("%s(%s): fw_state =%x\n", __func__, adapter->pnetdev->name,
-		  get_fwstate(pmlmepriv));
-
-	spin_lock_bh(&pmlmepriv->lock);
-
-	_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY);
-
-	spin_unlock_bh(&pmlmepriv->lock);
-
-	rtw_cfg80211_indicate_scan_done(wdev_to_priv(adapter->rtw_wdev), true);
-}
-
-void rtw_dynamic_check_timer_handler(unsigned long data)
-{
-	struct rtw_adapter *adapter = (struct rtw_adapter *)data;
-
-	if (adapter->hw_init_completed == false)
-		goto out;
-
-	if (adapter->bDriverStopped == true ||
-	    adapter->bSurpriseRemoved == true)
-		goto out;
-
-	if (adapter->net_closed == true)
-		goto out;
-
-	rtw_dynamic_chk_wk_cmd23a(adapter);
-
-out:
-	mod_timer(&adapter->mlmepriv.dynamic_chk_timer,
-		  jiffies + msecs_to_jiffies(2000));
-}
-
-inline bool rtw_is_scan_deny(struct rtw_adapter *adapter)
-{
-	struct mlme_priv *mlmepriv = &adapter->mlmepriv;
-
-	return (atomic_read(&mlmepriv->set_scan_deny) != 0) ? true : false;
-}
-
-void rtw_clear_scan_deny(struct rtw_adapter *adapter)
-{
-	struct mlme_priv *mlmepriv = &adapter->mlmepriv;
-
-	atomic_set(&mlmepriv->set_scan_deny, 0);
-}
-
-void rtw_set_scan_deny_timer_hdl(unsigned long data)
-{
-	struct rtw_adapter *adapter = (struct rtw_adapter *)data;
-
-	rtw_clear_scan_deny(adapter);
-}
-
-void rtw_set_scan_deny(struct rtw_adapter *adapter, u32 ms)
-{
-	struct mlme_priv *mlmepriv = &adapter->mlmepriv;
-
-	atomic_set(&mlmepriv->set_scan_deny, 1);
-	mod_timer(&mlmepriv->set_scan_deny_timer,
-		  jiffies + msecs_to_jiffies(ms));
-}
-
-#if defined(IEEE80211_SCAN_RESULT_EXPIRE)
-#define RTW_SCAN_RESULT_EXPIRE  \
-	((IEEE80211_SCAN_RESULT_EXPIRE / (HZ*1000)) - 1000) /* 3000 -1000 */
-#else
-#define RTW_SCAN_RESULT_EXPIRE 2000
-#endif
-
-/*
-* Select a new join candidate from the original @param candidate and
-*     @param competitor
-* @return true: candidate is updated
-* @return false: candidate is not updated
-*/
-static int rtw_check_join_candidate(struct mlme_priv *pmlmepriv,
-				    struct wlan_network **candidate,
-				    struct wlan_network *competitor)
-{
-	int updated = false;
-	struct rtw_adapter *adapter;
-
-	adapter = container_of(pmlmepriv, struct rtw_adapter, mlmepriv);
-
-	/* check bssid, if needed */
-	if (pmlmepriv->assoc_by_bssid == true) {
-		if (!ether_addr_equal(competitor->network.MacAddress,
-				      pmlmepriv->assoc_bssid))
-			goto exit;
-	}
-
-	/* check ssid, if needed */
-	if (pmlmepriv->assoc_ssid.ssid_len) {
-		if (competitor->network.Ssid.ssid_len !=
-		    pmlmepriv->assoc_ssid.ssid_len ||
-		    memcmp(competitor->network.Ssid.ssid,
-			   pmlmepriv->assoc_ssid.ssid,
-			   pmlmepriv->assoc_ssid.ssid_len))
-			goto exit;
-	}
-
-	if (rtw_is_desired_network(adapter, competitor) == false)
-		goto exit;
-
-	if (adapter->mlmepriv.to_roaming > 0) {
-		unsigned int passed;
-
-		passed = jiffies_to_msecs(jiffies - competitor->last_scanned);
-		if (passed >= RTW_SCAN_RESULT_EXPIRE ||
-		    is_same_ess(&competitor->network,
-				&pmlmepriv->cur_network.network) == false)
-			goto exit;
-	}
-
-	if (!*candidate ||
-	    (*candidate)->network.Rssi<competitor->network.Rssi) {
-		*candidate = competitor;
-		updated = true;
-	}
-
-	if (updated) {
-		DBG_8723A("[by_bssid:%u][assoc_ssid:%s][to_roaming:%u] new candidate: %s(%pM) rssi:%d\n",
-			  pmlmepriv->assoc_by_bssid,
-			  pmlmepriv->assoc_ssid.ssid,
-			  adapter->mlmepriv.to_roaming,
-			  (*candidate)->network.Ssid.ssid,
-			  (*candidate)->network.MacAddress,
-			  (int)(*candidate)->network.Rssi);
-	}
-
-exit:
-	return updated;
-}
-
-/*
-Calling context:
-The caller of the sub-routine will be in critical section...
-
-The caller must hold the following spinlock
-
-pmlmepriv->lock
-
-*/
-
-static int rtw_do_join(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	int ret;
-
-	pmlmepriv->cur_network.join_res = -2;
-
-	set_fwstate(pmlmepriv, _FW_UNDER_LINKING);
-
-	pmlmepriv->to_join = true;
-
-	ret = rtw_select_and_join_from_scanned_queue23a(pmlmepriv);
-	if (ret == _SUCCESS) {
-		pmlmepriv->to_join = false;
-	} else {
-		if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
-			/* switch to ADHOC_MASTER */
-			ret = rtw_do_join_adhoc(padapter);
-			if (ret != _SUCCESS)
-				goto exit;
-		} else {
-			/*  can't associate ; reset under-linking */
-			_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
-
-			ret = _FAIL;
-			pmlmepriv->to_join = false;
-		}
-	}
-
-exit:
-	return ret;
-}
-
-static struct wlan_network *
-rtw_select_candidate_from_queue(struct mlme_priv *pmlmepriv)
-{
-	struct wlan_network *pnetwork, *ptmp, *candidate = NULL;
-	struct rtw_queue *queue = &pmlmepriv->scanned_queue;
-	struct list_head *phead;
-
-	spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-	phead = get_list_head(queue);
-	list_for_each_entry_safe(pnetwork, ptmp, phead, list)
-		rtw_check_join_candidate(pmlmepriv, &candidate, pnetwork);
-	spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-
-	return candidate;
-}
-
-
-int rtw_do_join_adhoc(struct rtw_adapter *adapter)
-{
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct wlan_bssid_ex *pdev_network;
-	u8 *ibss;
-	int ret;
-
-	pdev_network = &adapter->registrypriv.dev_network;
-	ibss = adapter->registrypriv.dev_network.MacAddress;
-
-	_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY);
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-		 "switching to adhoc master\n");
-
-	memcpy(&pdev_network->Ssid, &pmlmepriv->assoc_ssid,
-	       sizeof(struct cfg80211_ssid));
-
-	rtw_update_registrypriv_dev_network23a(adapter);
-	rtw_generate_random_ibss23a(ibss);
-
-	pmlmepriv->fw_state = WIFI_ADHOC_MASTER_STATE;
-
-	ret = rtw_createbss_cmd23a(adapter);
-	if (ret != _SUCCESS) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "Error =>rtw_createbss_cmd23a status FAIL\n");
-	} else  {
-		pmlmepriv->to_join = false;
-	}
-
-	return ret;
-}
-
-int rtw_do_join_network(struct rtw_adapter *adapter,
-			struct wlan_network *candidate)
-{
-	int ret;
-
-	/*  check for situation of  _FW_LINKED */
-	if (check_fwstate(&adapter->mlmepriv, _FW_LINKED)) {
-		DBG_8723A("%s: _FW_LINKED while ask_for_joinbss!\n", __func__);
-
-		rtw_disassoc_cmd23a(adapter, 0, true);
-		rtw_indicate_disconnect23a(adapter);
-		rtw_free_assoc_resources23a(adapter, 0);
-	}
-	set_fwstate(&adapter->mlmepriv, _FW_UNDER_LINKING);
-
-	ret = rtw_joinbss_cmd23a(adapter, candidate);
-
-	if (ret == _SUCCESS)
-		mod_timer(&adapter->mlmepriv.assoc_timer,
-			  jiffies + msecs_to_jiffies(MAX_JOIN_TIMEOUT));
-
-	return ret;
-}
-
-int rtw_select_and_join_from_scanned_queue23a(struct mlme_priv *pmlmepriv)
-{
-	struct rtw_adapter *adapter;
-	struct wlan_network *candidate = NULL;
-	int ret;
-
-	adapter = pmlmepriv->nic_hdl;
-
-	candidate = rtw_select_candidate_from_queue(pmlmepriv);
-	if (!candidate) {
-		DBG_8723A("%s: return _FAIL(candidate == NULL)\n", __func__);
-		ret = _FAIL;
-		goto exit;
-	} else {
-		DBG_8723A("%s: candidate: %s(%pM, ch:%u)\n",
-			  __func__,
-			  candidate->network.Ssid.ssid,
-			  candidate->network.MacAddress,
-			  candidate->network.DSConfig);
-	}
-
-	ret = rtw_do_join_network(adapter, candidate);
-
-exit:
-	return ret;
-}
-
-int rtw_set_auth23a(struct rtw_adapter *adapter,
-		    struct security_priv *psecuritypriv)
-{
-	struct cmd_obj *pcmd;
-	struct setauth_parm *psetauthparm;
-	struct cmd_priv *pcmdpriv = &adapter->cmdpriv;
-	int res = _SUCCESS;
-
-	pcmd = kzalloc(sizeof(struct cmd_obj), GFP_KERNEL);
-	if (!pcmd) {
-		res = _FAIL;  /* try again */
-		goto exit;
-	}
-
-	psetauthparm = kzalloc(sizeof(struct setauth_parm), GFP_KERNEL);
-	if (!psetauthparm) {
-		kfree(pcmd);
-		res = _FAIL;
-		goto exit;
-	}
-
-	psetauthparm->mode = (unsigned char)psecuritypriv->dot11AuthAlgrthm;
-
-	pcmd->cmdcode = _SetAuth_CMD_;
-	pcmd->parmbuf = (unsigned char *)psetauthparm;
-	pcmd->cmdsz =  (sizeof(struct setauth_parm));
-	pcmd->rsp = NULL;
-	pcmd->rspsz = 0;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-		 "after enqueue set_auth_cmd, auth_mode=%x\n",
-		 psecuritypriv->dot11AuthAlgrthm);
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, pcmd);
-
-exit:
-
-	return res;
-}
-
-int rtw_set_key23a(struct rtw_adapter *adapter,
-		   struct security_priv *psecuritypriv, int keyid, u8 set_tx)
-{
-	u8 keylen;
-	struct cmd_obj *pcmd;
-	struct setkey_parm *psetkeyparm;
-	struct cmd_priv *pcmdpriv = &adapter->cmdpriv;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	int res = _SUCCESS;
-
-	if (keyid >= 4) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	pcmd = kzalloc(sizeof(struct cmd_obj), GFP_KERNEL);
-	if (!pcmd) {
-		res = _FAIL;  /* try again */
-		goto exit;
-	}
-	psetkeyparm = kzalloc(sizeof(struct setkey_parm), GFP_KERNEL);
-	if (!psetkeyparm) {
-		kfree(pcmd);
-		res = _FAIL;
-		goto exit;
-	}
-
-	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) {
-		psetkeyparm->algorithm = (unsigned char)
-			psecuritypriv->dot118021XGrpPrivacy;
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "rtw_set_key23a: psetkeyparm->algorithm = (unsigned char)psecuritypriv->dot118021XGrpPrivacy =%d\n",
-			 psetkeyparm->algorithm);
-	} else {
-		psetkeyparm->algorithm = (u8)psecuritypriv->dot11PrivacyAlgrthm;
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "rtw_set_key23a: psetkeyparm->algorithm = (u8)psecuritypriv->dot11PrivacyAlgrthm =%d\n",
-			 psetkeyparm->algorithm);
-	}
-	psetkeyparm->keyid = keyid;/* 0~3 */
-	psetkeyparm->set_tx = set_tx;
-	if (is_wep_enc(psetkeyparm->algorithm))
-		pmlmepriv->key_mask |= BIT(psetkeyparm->keyid);
-
-	DBG_8723A("==> rtw_set_key23a algorithm(%x), keyid(%x), key_mask(%x)\n",
-		  psetkeyparm->algorithm, psetkeyparm->keyid,
-		  pmlmepriv->key_mask);
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-		 "rtw_set_key23a: psetkeyparm->algorithm =%d psetkeyparm->keyid = (u8)keyid =%d\n",
-		 psetkeyparm->algorithm, keyid);
-
-	switch (psetkeyparm->algorithm) {
-	case WLAN_CIPHER_SUITE_WEP40:
-		keylen = 5;
-		memcpy(&psetkeyparm->key[0],
-		       &psecuritypriv->wep_key[keyid].key, keylen);
-		break;
-	case WLAN_CIPHER_SUITE_WEP104:
-		keylen = 13;
-		memcpy(&psetkeyparm->key[0],
-		       &psecuritypriv->wep_key[keyid].key, keylen);
-		break;
-	case WLAN_CIPHER_SUITE_TKIP:
-		keylen = 16;
-		memcpy(&psetkeyparm->key,
-		       &psecuritypriv->dot118021XGrpKey[keyid], keylen);
-		psetkeyparm->grpkey = 1;
-		break;
-	case WLAN_CIPHER_SUITE_CCMP:
-		keylen = 16;
-		memcpy(&psetkeyparm->key,
-		       &psecuritypriv->dot118021XGrpKey[keyid], keylen);
-		psetkeyparm->grpkey = 1;
-		break;
-	default:
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "rtw_set_key23a:psecuritypriv->dot11PrivacyAlgrthm = %x (must be 1 or 2 or 4 or 5)\n",
-			 psecuritypriv->dot11PrivacyAlgrthm);
-		res = _FAIL;
-		kfree(pcmd);
-		kfree(psetkeyparm);
-		goto exit;
-	}
-
-	pcmd->cmdcode = _SetKey_CMD_;
-	pcmd->parmbuf = (u8 *)psetkeyparm;
-	pcmd->cmdsz =  (sizeof(struct setkey_parm));
-	pcmd->rsp = NULL;
-	pcmd->rspsz = 0;
-
-	/* sema_init(&pcmd->cmd_sem, 0); */
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, pcmd);
-
-exit:
-
-	return res;
-}
-
-/* adjust IEs for rtw_joinbss_cmd23a in WMM */
-int rtw_restruct_wmm_ie23a(struct rtw_adapter *adapter, u8 *in_ie,
-			   u8 *out_ie, uint in_len, uint initial_out_len)
-{
-	int ielength;
-	const u8 *p;
-
-	ielength = initial_out_len;
-
-	p = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-				    WLAN_OUI_TYPE_MICROSOFT_WMM,
-				    in_ie, in_len);
-
-	if (p && p[1]) {
-		memcpy(out_ie + initial_out_len, p, 9);
-
-		out_ie[initial_out_len + 1] = 7;
-		out_ie[initial_out_len + 6] = 0;
-		out_ie[initial_out_len + 8] = 0;
-
-		ielength += 9;
-	}
-
-	return ielength;
-}
-
-/*  */
-/*  Ported from 8185: IsInPreAuthKeyList().
-    (Renamed from SecIsInPreAuthKeyList(), 2006-10-13.) */
-/*  Added by Annie, 2006-05-07. */
-/*  */
-/*  Search by BSSID, */
-/*  Return Value: */
-/*		-1	:if there is no pre-auth key in the  table */
-/*		>= 0	:if there is pre-auth key, and   return the entry id */
-/*  */
-/*  */
-
-static int SecIsInPMKIDList(struct rtw_adapter *Adapter, u8 *bssid)
-{
-	struct security_priv *psecuritypriv = &Adapter->securitypriv;
-	int i = 0;
-
-	do {
-		if (psecuritypriv->PMKIDList[i].bUsed &&
-                    ether_addr_equal(psecuritypriv->PMKIDList[i].Bssid, bssid)) {
-			break;
-		} else {
-			i++;
-			/* continue; */
-		}
-	} while (i < NUM_PMKID_CACHE);
-
-	if (i == NUM_PMKID_CACHE)
-		i = -1;/*  Could not find. */
-	else {
-		/*  There is one Pre-Authentication Key for
-		    the specific BSSID. */
-	}
-
-	return i;
-}
-
-/*  */
-/*  Check the RSN IE length */
-/*  If the RSN IE length <= 20, the RSN IE didn't include
-    the PMKID information */
-/*  0-11th element in the array are the fixed IE */
-/*  12th element in the array is the IE */
-/*  13th element in the array is the IE length */
-/*  */
-
-static int rtw_append_pmkid(struct rtw_adapter *Adapter, int iEntry,
-			    u8 *ie, uint ie_len)
-{
-	struct security_priv *psecuritypriv = &Adapter->securitypriv;
-
-	if (ie[1] <= 20) {
-		/*  The RSN IE didn't include the PMK ID,
-		    append the PMK information */
-			ie[ie_len] = 1;
-			ie_len++;
-			ie[ie_len] = 0;	/* PMKID count = 0x0100 */
-			ie_len++;
-			memcpy(&ie[ie_len],
-			       &psecuritypriv->PMKIDList[iEntry].PMKID, 16);
-
-			ie_len += 16;
-			ie[1] += 18;/* PMKID length = 2+16 */
-	}
-	return ie_len;
-}
-
-int rtw_restruct_sec_ie23a(struct rtw_adapter *adapter, u8 *in_ie, u8 *out_ie,
-			   uint in_len)
-{
-	u8 authmode;
-	uint ielength;
-	int iEntry;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct security_priv *psecuritypriv = &adapter->securitypriv;
-	uint ndisauthmode = psecuritypriv->ndisauthtype;
-	uint ndissecuritytype = psecuritypriv->ndisencryptstatus;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_notice_,
-		 "+rtw_restruct_sec_ie23a: ndisauthmode=%d ndissecuritytype=%d\n",
-		 ndisauthmode, ndissecuritytype);
-
-	ielength = 0;
-	if (ndisauthmode == Ndis802_11AuthModeWPA ||
-	    ndisauthmode == Ndis802_11AuthModeWPAPSK)
-		authmode = WLAN_EID_VENDOR_SPECIFIC;
-	if (ndisauthmode == Ndis802_11AuthModeWPA2 ||
-	    ndisauthmode == Ndis802_11AuthModeWPA2PSK)
-		authmode = WLAN_EID_RSN;
-
-	if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
-		memcpy(out_ie + ielength, psecuritypriv->wps_ie,
-		       psecuritypriv->wps_ie_len);
-
-		ielength += psecuritypriv->wps_ie_len;
-	} else if (authmode == WLAN_EID_VENDOR_SPECIFIC ||
-		   authmode == WLAN_EID_RSN) {
-		/* copy RSN or SSN */
-		memcpy(&out_ie[ielength], &psecuritypriv->supplicant_ie[0],
-		       psecuritypriv->supplicant_ie[1] + 2);
-		ielength += psecuritypriv->supplicant_ie[1] + 2;
-	}
-
-	iEntry = SecIsInPMKIDList(adapter, pmlmepriv->assoc_bssid);
-	if (iEntry < 0)
-		return ielength;
-	else {
-		if (authmode == WLAN_EID_RSN)
-			ielength = rtw_append_pmkid(adapter, iEntry,
-						    out_ie, ielength);
-	}
-
-	return ielength;
-}
-
-void rtw_init_registrypriv_dev_network23a(struct rtw_adapter *adapter)
-{
-	struct registry_priv *pregistrypriv = &adapter->registrypriv;
-	struct eeprom_priv *peepriv = &adapter->eeprompriv;
-	struct wlan_bssid_ex    *pdev_network = &pregistrypriv->dev_network;
-	u8 *myhwaddr = myid(peepriv);
-
-	ether_addr_copy(pdev_network->MacAddress, myhwaddr);
-
-	memcpy(&pdev_network->Ssid, &pregistrypriv->ssid,
-	       sizeof(struct cfg80211_ssid));
-
-	pdev_network->beacon_interval = 100;
-}
-
-void rtw_update_registrypriv_dev_network23a(struct rtw_adapter *adapter)
-{
-	int sz = 0;
-	struct registry_priv *pregistrypriv = &adapter->registrypriv;
-	struct wlan_bssid_ex *pdev_network = &pregistrypriv->dev_network;
-	struct security_priv *psecuritypriv = &adapter->securitypriv;
-	struct wlan_network *cur_network = &adapter->mlmepriv.cur_network;
-	/* struct	xmit_priv	*pxmitpriv = &adapter->xmitpriv; */
-
-	pdev_network->Privacy =
-		(psecuritypriv->dot11PrivacyAlgrthm > 0 ? 1 : 0);
-
-	pdev_network->Rssi = 0;
-
-	pdev_network->DSConfig = pregistrypriv->channel;
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "pregistrypriv->channel =%d, pdev_network->DSConfig = 0x%x\n",
-		 pregistrypriv->channel, pdev_network->DSConfig);
-
-	if (cur_network->network.ifmode == NL80211_IFTYPE_ADHOC)
-		pdev_network->ATIMWindow = 0;
-
-	pdev_network->ifmode = cur_network->network.ifmode;
-
-	/*  1. Supported rates */
-	/*  2. IE */
-
-	sz = rtw_generate_ie23a(pregistrypriv);
-
-	pdev_network->IELength = sz;
-
-	pdev_network->Length =
-		get_wlan_bssid_ex_sz(pdev_network);
-
-	/* notes: translate IELength & Length after assign the
-	   Length to cmdsz in createbss_cmd(); */
-	/* pdev_network->IELength = cpu_to_le32(sz); */
-}
-
-/* the function is at passive_level */
-void rtw_joinbss_reset23a(struct rtw_adapter *padapter)
-{
-	u8 threshold;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-
-	/* todo: if you want to do something io/reg/hw setting
-	   before join_bss, please add code here */
-
-	pmlmepriv->num_FortyMHzIntolerant = 0;
-
-	pmlmepriv->num_sta_no_ht = 0;
-
-	phtpriv->ampdu_enable = false;/* reset to disabled */
-
-	/*  TH = 1 => means that invalidate usb rx aggregation */
-	/*  TH = 0 => means that validate usb rx aggregation, use init value. */
-	if (phtpriv->ht_option) {
-		if (padapter->registrypriv.wifi_spec == 1)
-			threshold = 1;
-		else
-			threshold = 0;
-	} else
-		threshold = 1;
-
-	rtl8723a_set_rxdma_agg_pg_th(padapter, threshold);
-}
-
-/* the function is >= passive_level */
-bool rtw_restructure_ht_ie23a(struct rtw_adapter *padapter, u8 *in_ie,
-			      u8 *out_ie, uint in_len, uint *pout_len)
-{
-	u32 out_len;
-	int max_rx_ampdu_factor;
-	unsigned char *pframe;
-	const u8 *p;
-	struct ieee80211_ht_cap ht_capie;
-	u8 WMM_IE[7] = {0x00, 0x50, 0xf2, 0x02, 0x00, 0x01, 0x00};
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-
-	phtpriv->ht_option = false;
-
-	p = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY, in_ie, in_len);
-
-	if (p && p[1] > 0) {
-		u32 rx_packet_offset, max_recvbuf_sz;
-
-		if (pmlmepriv->qos_option == 0) {
-			out_len = *pout_len;
-			pframe = rtw_set_ie23a(out_ie + out_len,
-					       WLAN_EID_VENDOR_SPECIFIC,
-					       sizeof(WMM_IE), WMM_IE,
-					       pout_len);
-
-			pmlmepriv->qos_option = 1;
-		}
-
-		out_len = *pout_len;
-
-		memset(&ht_capie, 0, sizeof(struct ieee80211_ht_cap));
-
-		ht_capie.cap_info = cpu_to_le16(IEEE80211_HT_CAP_SUP_WIDTH_20_40 |
-			IEEE80211_HT_CAP_SGI_20 | IEEE80211_HT_CAP_SGI_40 |
-			IEEE80211_HT_CAP_TX_STBC | IEEE80211_HT_CAP_DSSSCCK40);
-
-		GetHalDefVar8192CUsb(padapter, HAL_DEF_RX_PACKET_OFFSET,
-				     &rx_packet_offset);
-		GetHalDefVar8192CUsb(padapter, HAL_DEF_MAX_RECVBUF_SZ,
-				     &max_recvbuf_sz);
-
-		GetHalDefVar8192CUsb(padapter, HW_VAR_MAX_RX_AMPDU_FACTOR,
-				     &max_rx_ampdu_factor);
-		ht_capie.ampdu_params_info = max_rx_ampdu_factor & 0x03;
-
-		if (padapter->securitypriv.dot11PrivacyAlgrthm ==
-		    WLAN_CIPHER_SUITE_CCMP)
-			ht_capie.ampdu_params_info |=
-				(IEEE80211_HT_AMPDU_PARM_DENSITY& (0x07 << 2));
-		else
-			ht_capie.ampdu_params_info |=
-				(IEEE80211_HT_AMPDU_PARM_DENSITY & 0x00);
-
-		pframe = rtw_set_ie23a(out_ie + out_len, WLAN_EID_HT_CAPABILITY,
-				    sizeof(struct ieee80211_ht_cap),
-				    (unsigned char *)&ht_capie, pout_len);
-
-		phtpriv->ht_option = true;
-
-		p = cfg80211_find_ie(WLAN_EID_HT_OPERATION, in_ie, in_len);
-		if (p && (p[1] == sizeof(struct ieee80211_ht_operation))) {
-			out_len = *pout_len;
-			pframe = rtw_set_ie23a(out_ie + out_len,
-					       WLAN_EID_HT_OPERATION,
-					       p[1], p + 2 , pout_len);
-		}
-	}
-
-	return phtpriv->ht_option;
-}
-
-/* the function is > passive_level (in critical_section) */
-void rtw_update_ht_cap23a(struct rtw_adapter *padapter, u8 *pie, uint ie_len)
-{
-	u8 max_ampdu_sz;
-	const u8 *p;
-	struct ieee80211_ht_cap *pht_capie;
-	struct ieee80211_ht_operation *pht_addtinfo;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (!phtpriv->ht_option)
-		return;
-
-	if ((!pmlmeinfo->HT_info_enable) || (!pmlmeinfo->HT_caps_enable))
-		return;
-
-	DBG_8723A("+rtw_update_ht_cap23a()\n");
-
-	/* maybe needs check if ap supports rx ampdu. */
-	if (!phtpriv->ampdu_enable && pregistrypriv->ampdu_enable == 1) {
-		if (pregistrypriv->wifi_spec == 1)
-			phtpriv->ampdu_enable = false;
-		else
-			phtpriv->ampdu_enable = true;
-	} else if (pregistrypriv->ampdu_enable == 2)
-		phtpriv->ampdu_enable = true;
-
-	/* check Max Rx A-MPDU Size */
-	p = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY, pie, ie_len);
-
-	if (p && p[1] > 0) {
-		pht_capie = (struct ieee80211_ht_cap *)(p + 2);
-		max_ampdu_sz = pht_capie->ampdu_params_info &
-			IEEE80211_HT_AMPDU_PARM_FACTOR;
-		/*  max_ampdu_sz (kbytes); */
-		max_ampdu_sz = 1 << (max_ampdu_sz + 3);
-
-		phtpriv->rx_ampdu_maxlen = max_ampdu_sz;
-	}
-
-	p = cfg80211_find_ie(WLAN_EID_HT_OPERATION, pie, ie_len);
-	if (p && p[1] > 0) {
-		pht_addtinfo = (struct ieee80211_ht_operation *)(p + 2);
-		/* todo: */
-	}
-
-	/* update cur_bwmode & cur_ch_offset */
-	if (pregistrypriv->cbw40_enable &&
-	    pmlmeinfo->ht_cap.cap_info &
-	    cpu_to_le16(IEEE80211_HT_CAP_SUP_WIDTH_20_40) &&
-	    pmlmeinfo->HT_info.ht_param & IEEE80211_HT_PARAM_CHAN_WIDTH_ANY) {
-		int i;
-		u8 rf_type;
-
-		rf_type = rtl8723a_get_rf_type(padapter);
-
-		/* update the MCS rates */
-		for (i = 0; i < IEEE80211_HT_MCS_MASK_LEN; i++) {
-			if (rf_type == RF_1T1R || rf_type == RF_1T2R)
-				pmlmeinfo->ht_cap.mcs.rx_mask[i] &=
-					MCS_rate_1R23A[i];
-			else
-				pmlmeinfo->ht_cap.mcs.rx_mask[i] &=
-					MCS_rate_2R23A[i];
-		}
-		/* switch to the 40M Hz mode according to the AP */
-		pmlmeext->cur_bwmode = HT_CHANNEL_WIDTH_40;
-		switch (pmlmeinfo->HT_info.ht_param &
-			IEEE80211_HT_PARAM_CHA_SEC_OFFSET) {
-		case IEEE80211_HT_PARAM_CHA_SEC_ABOVE:
-			pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_LOWER;
-			break;
-
-		case IEEE80211_HT_PARAM_CHA_SEC_BELOW:
-			pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_UPPER;
-			break;
-
-		default:
-			pmlmeext->cur_ch_offset =
-				HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-			break;
-		}
-	}
-
-	/*  */
-	/*  Config SM Power Save setting */
-	/*  */
-	pmlmeinfo->SM_PS =
-		(le16_to_cpu(pmlmeinfo->ht_cap.cap_info) &
-		 IEEE80211_HT_CAP_SM_PS) >> IEEE80211_HT_CAP_SM_PS_SHIFT;
-	if (pmlmeinfo->SM_PS == WLAN_HT_CAP_SM_PS_STATIC)
-		DBG_8723A("%s(): WLAN_HT_CAP_SM_PS_STATIC\n", __func__);
-
-	/*  */
-	/*  Config current HT Protection mode. */
-	/*  */
-	pmlmeinfo->HT_protection =
-		le16_to_cpu(pmlmeinfo->HT_info.operation_mode) &
-		IEEE80211_HT_OP_MODE_PROTECTION;
-}
-
-void rtw_issue_addbareq_cmd23a(struct rtw_adapter *padapter,
-			       struct xmit_frame *pxmitframe)
-{
-	u8 issued;
-	int priority;
-	struct sta_info *psta;
-	struct ht_priv	*phtpriv;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-	s32 bmcst = is_multicast_ether_addr(pattrib->ra);
-
-	if (bmcst || padapter->mlmepriv.LinkDetectInfo.NumTxOkInPeriod < 100)
-		return;
-
-	priority = pattrib->priority;
-
-	if (pattrib->psta)
-		psta = pattrib->psta;
-	else {
-		DBG_8723A("%s, call rtw_get_stainfo23a()\n", __func__);
-		psta = rtw_get_stainfo23a(&padapter->stapriv, pattrib->ra);
-	}
-
-	if (!psta) {
-		DBG_8723A("%s, psta == NUL\n", __func__);
-		return;
-	}
-
-	if (!(psta->state &_FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n",
-			  __func__, psta->state);
-		return;
-	}
-
-	phtpriv = &psta->htpriv;
-
-	if (phtpriv->ht_option && phtpriv->ampdu_enable) {
-		issued = (phtpriv->agg_enable_bitmap>>priority)&0x1;
-		issued |= (phtpriv->candidate_tid_bitmap>>priority)&0x1;
-
-		if (issued == 0) {
-			DBG_8723A("rtw_issue_addbareq_cmd23a, p =%d\n",
-				  priority);
-			psta->htpriv.candidate_tid_bitmap |= BIT(priority);
-			rtw_addbareq_cmd23a(padapter, (u8) priority,
-					    pattrib->ra);
-		}
-	}
-}
-
-int rtw_linked_check(struct rtw_adapter *padapter)
-{
-	if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) ||
-	    check_fwstate(&padapter->mlmepriv,
-			  WIFI_ADHOC_STATE|WIFI_ADHOC_MASTER_STATE)) {
-		if (padapter->stapriv.asoc_sta_count > 2)
-			return true;
-	} else {	/* Station mode */
-		if (check_fwstate(&padapter->mlmepriv, _FW_LINKED))
-			return true;
-	}
-	return false;
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_mlme_ext.c b/drivers/staging/rtl8723au/core/rtw_mlme_ext.c
deleted file mode 100644
index 7dd1540ebedd..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_mlme_ext.c
+++ /dev/null
@@ -1,6187 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_MLME_EXT_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <wifi.h>
-#include <rtw_mlme_ext.h>
-#include <wlan_bssdef.h>
-#include <mlme_osdep.h>
-#include <recv_osdep.h>
-#include <linux/ieee80211.h>
-#include <rtl8723a_hal.h>
-
-static int OnAssocReq23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAssocRsp23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnProbeReq23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnProbeRsp23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int DoReserved23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnBeacon23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAtim23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnDisassoc23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAuth23aClient23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnDeAuth23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAction23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-
-static int on_action_spct23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAction23a_qos(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAction23a_dls(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAction23a_back23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int on_action_public23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAction23a_ht(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAction23a_wmm(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static int OnAction23a_p2p(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-
-static void issue_assocreq(struct rtw_adapter *padapter);
-static void issue_probereq(struct rtw_adapter *padapter,
-			   struct cfg80211_ssid *pssid, u8 *da);
-static int issue_probereq_ex(struct rtw_adapter *padapter,
-			     struct cfg80211_ssid *pssid,
-			     u8 *da, int try_cnt, int wait_ms);
-static void issue_probersp(struct rtw_adapter *padapter, unsigned char *da);
-static void issue_auth(struct rtw_adapter *padapter, struct sta_info *psta,
-		       unsigned short status);
-static int issue_deauth_ex(struct rtw_adapter *padapter, u8 *da,
-			   unsigned short reason, int try_cnt, int wait_ms);
-static void start_clnt_assoc(struct rtw_adapter *padapter);
-static void start_clnt_auth(struct rtw_adapter *padapter);
-static void start_clnt_join(struct rtw_adapter *padapter);
-static void start_create_ibss(struct rtw_adapter *padapter);
-static struct wlan_bssid_ex *collect_bss_info(struct rtw_adapter *padapter,
-					      struct recv_frame *precv_frame);
-
-#ifdef CONFIG_8723AU_AP_MODE
-static int OnAuth23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-static void issue_assocrsp(struct rtw_adapter *padapter, unsigned short status,
-			   struct sta_info *pstat, u16 pkt_type);
-#endif
-
-static struct mlme_handler mlme_sta_tbl[]={
-	{"OnAssocReq23a",		&OnAssocReq23a},
-	{"OnAssocRsp23a",		&OnAssocRsp23a},
-	{"OnReAssocReq",	&OnAssocReq23a},
-	{"OnReAssocRsp",	&OnAssocRsp23a},
-	{"OnProbeReq23a",		&OnProbeReq23a},
-	{"OnProbeRsp23a",		&OnProbeRsp23a},
-
-	/*----------------------------------------------------------
-					below 2 are reserved
-	-----------------------------------------------------------*/
-	{"DoReserved23a",		&DoReserved23a},
-	{"DoReserved23a",		&DoReserved23a},
-	{"OnBeacon23a",		&OnBeacon23a},
-	{"OnATIM",		&OnAtim23a},
-	{"OnDisassoc23a",		&OnDisassoc23a},
-	{"OnAuth23a",		&OnAuth23aClient23a},
-	{"OnDeAuth23a",		&OnDeAuth23a},
-	{"OnAction23a",		&OnAction23a},
-};
-
-static struct action_handler OnAction23a_tbl[]={
-	{WLAN_CATEGORY_SPECTRUM_MGMT, "ACTION_SPECTRUM_MGMT", on_action_spct23a},
-	{WLAN_CATEGORY_QOS, "ACTION_QOS", &OnAction23a_qos},
-	{WLAN_CATEGORY_DLS, "ACTION_DLS", &OnAction23a_dls},
-	{WLAN_CATEGORY_BACK, "ACTION_BACK", &OnAction23a_back23a},
-	{WLAN_CATEGORY_PUBLIC, "ACTION_PUBLIC", on_action_public23a},
-	{WLAN_CATEGORY_HT, "ACTION_HT",	&OnAction23a_ht},
-	{WLAN_CATEGORY_SA_QUERY, "ACTION_SA_QUERY", &DoReserved23a},
-	{WLAN_CATEGORY_WMM, "ACTION_WMM", &OnAction23a_wmm},
-	{WLAN_CATEGORY_VENDOR_SPECIFIC, "ACTION_P2P", &OnAction23a_p2p},
-};
-
-static u8	null_addr[ETH_ALEN]= {0, 0, 0, 0, 0, 0};
-
-/**************************************************
-OUI definitions for the vendor specific IE
-***************************************************/
-unsigned char WMM_OUI23A[] = {0x00, 0x50, 0xf2, 0x02};
-unsigned char WPS_OUI23A[] = {0x00, 0x50, 0xf2, 0x04};
-unsigned char P2P_OUI23A[] = {0x50, 0x6F, 0x9A, 0x09};
-unsigned char WFD_OUI23A[] = {0x50, 0x6F, 0x9A, 0x0A};
-
-unsigned char WMM_INFO_OUI23A[] = {0x00, 0x50, 0xf2, 0x02, 0x00, 0x01};
-unsigned char WMM_PARA_OUI23A[] = {0x00, 0x50, 0xf2, 0x02, 0x01, 0x01};
-
-static unsigned char REALTEK_96B_IE[] = {0x00, 0xe0, 0x4c, 0x02, 0x01, 0x20};
-
-/********************************************************
-MCS rate definitions
-*********************************************************/
-unsigned char MCS_rate_2R23A[16] = {
-	0xff, 0xff, 0x0, 0x0, 0x01, 0x0, 0x0, 0x0,
-	0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
-unsigned char MCS_rate_1R23A[16] = {
-	0xff, 0x00, 0x0, 0x0, 0x01, 0x0, 0x0, 0x0,
-	0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
-
-/********************************************************
-ChannelPlan definitions
-*********************************************************/
-
-static struct rt_channel_plan_2g RTW_ChannelPlan2G[RT_CHANNEL_DOMAIN_2G_MAX] = {
-	/*  0x00, RT_CHANNEL_DOMAIN_2G_WORLD , Passive scan CH 12, 13 */
-	{{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, 13},
-	/*  0x01, RT_CHANNEL_DOMAIN_2G_ETSI1 */
-	{{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, 13},
-	/*  0x02, RT_CHANNEL_DOMAIN_2G_FCC1 */
-	{{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, 11},
-	/*  0x03, RT_CHANNEL_DOMAIN_2G_MIKK1 */
-	{{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, 14},
-	/*  0x04, RT_CHANNEL_DOMAIN_2G_ETSI2 */
-	{{10, 11, 12, 13}, 4},
-	/*  0x05, RT_CHANNEL_DOMAIN_2G_NULL */
-	{{}, 0},
-};
-
-static struct rt_channel_plan_5g RTW_ChannelPlan5G[RT_CHANNEL_DOMAIN_5G_MAX] = {
-	/*  0x00, RT_CHANNEL_DOMAIN_5G_NULL */
-	{{}, 0},
-	/*  0x01, RT_CHANNEL_DOMAIN_5G_ETSI1 */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112,
-	  116, 120, 124, 128, 132, 136, 140}, 19},
-	/*  0x02, RT_CHANNEL_DOMAIN_5G_ETSI2 */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112,
-	  116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165}, 24},
-	/*  0x03, RT_CHANNEL_DOMAIN_5G_ETSI3 */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112,
-	  116, 120, 124, 128, 132, 149, 153, 157, 161, 165}, 22},
-	/*  0x04, RT_CHANNEL_DOMAIN_5G_FCC1 */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112,
-	  116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165}, 24},
-	/*  0x05, RT_CHANNEL_DOMAIN_5G_FCC2 */
-	{{36, 40, 44, 48, 149, 153, 157, 161, 165}, 9},
-	/*  0x06, RT_CHANNEL_DOMAIN_5G_FCC3 */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165}, 13},
-	/*  0x07, RT_CHANNEL_DOMAIN_5G_FCC4 */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161}, 12},
-	/*  0x08, RT_CHANNEL_DOMAIN_5G_FCC5 */
-	{{149, 153, 157, 161, 165}, 5},
-	/*  0x09, RT_CHANNEL_DOMAIN_5G_FCC6 */
-	{{36, 40, 44, 48, 52, 56, 60, 64}, 8},
-	/*  0x0A, RT_CHANNEL_DOMAIN_5G_FCC7_IC1 */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112,
-	  116, 136, 140, 149, 153, 157, 161, 165}, 20},
-	/*  0x0B, RT_CHANNEL_DOMAIN_5G_KCC1 */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112,
-	  116, 120, 124, 149, 153, 157, 161, 165}, 20},
-	/*  0x0C, RT_CHANNEL_DOMAIN_5G_MKK1 */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112,
-	  116, 120, 124, 128, 132, 136, 140}, 19},
-	/*  0x0D, RT_CHANNEL_DOMAIN_5G_MKK2 */
-	{{36, 40, 44, 48, 52, 56, 60, 64}, 8},
-	/*  0x0E, RT_CHANNEL_DOMAIN_5G_MKK3 */
-	{{100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140}, 11},
-	/*  0x0F, RT_CHANNEL_DOMAIN_5G_NCC1 */
-	{{56, 60, 64, 100, 104, 108, 112, 116, 136, 140, 149,
-	  153, 157, 161, 165}, 15},
-	/*  0x10, RT_CHANNEL_DOMAIN_5G_NCC2 */
-	{{56, 60, 64, 149, 153, 157, 161, 165}, 8},
-
-	/*  Driver self defined for old channel plan Compatible,
-	    Remember to modify if have new channel plan definition ===== */
-	/*  0x11, RT_CHANNEL_DOMAIN_5G_FCC */
-	{{36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112,
-	  116, 132, 136, 140, 149, 153, 157, 161, 165}, 21},
-	/*  0x12, RT_CHANNEL_DOMAIN_5G_JAPAN_NO_DFS */
-	{{36, 40, 44, 48}, 4},
-	/*  0x13, RT_CHANNEL_DOMAIN_5G_FCC4_NO_DFS */
-	{{36, 40, 44, 48, 149, 153, 157, 161}, 8},
-};
-
-static struct rt_channel_plan_map RTW_ChannelPlanMap[RT_CHANNEL_DOMAIN_MAX] = {
-	/*  0x00 ~ 0x1F , Old Define ===== */
-	{0x02, 0x11},	/* 0x00, RT_CHANNEL_DOMAIN_FCC */
-	{0x02, 0x0A},	/* 0x01, RT_CHANNEL_DOMAIN_IC */
-	{0x01, 0x01},	/* 0x02, RT_CHANNEL_DOMAIN_ETSI */
-	{0x01, 0x00},	/* 0x03, RT_CHANNEL_DOMAIN_SPAIN */
-	{0x01, 0x00},	/* 0x04, RT_CHANNEL_DOMAIN_FRANCE */
-	{0x03, 0x00},	/* 0x05, RT_CHANNEL_DOMAIN_MKK */
-	{0x03, 0x00},	/* 0x06, RT_CHANNEL_DOMAIN_MKK1 */
-	{0x01, 0x09},	/* 0x07, RT_CHANNEL_DOMAIN_ISRAEL */
-	{0x03, 0x09},	/* 0x08, RT_CHANNEL_DOMAIN_TELEC */
-	{0x03, 0x00},	/* 0x09, RT_CHANNEL_DOMAIN_GLOBAL_DOAMIN */
-	{0x00, 0x00},	/* 0x0A, RT_CHANNEL_DOMAIN_WORLD_WIDE_13 */
-	{0x02, 0x0F},	/* 0x0B, RT_CHANNEL_DOMAIN_TAIWAN */
-	{0x01, 0x08},	/* 0x0C, RT_CHANNEL_DOMAIN_CHINA */
-	{0x02, 0x06},	/* 0x0D, RT_CHANNEL_DOMAIN_SINGAPORE_INDIA_MEXICO */
-	{0x02, 0x0B},	/* 0x0E, RT_CHANNEL_DOMAIN_KOREA */
-	{0x02, 0x09},	/* 0x0F, RT_CHANNEL_DOMAIN_TURKEY */
-	{0x01, 0x01},	/* 0x10, RT_CHANNEL_DOMAIN_JAPAN */
-	{0x02, 0x05},	/* 0x11, RT_CHANNEL_DOMAIN_FCC_NO_DFS */
-	{0x01, 0x12},	/* 0x12, RT_CHANNEL_DOMAIN_JAPAN_NO_DFS */
-	{0x00, 0x04},	/* 0x13, RT_CHANNEL_DOMAIN_WORLD_WIDE_5G */
-	{0x02, 0x10},	/* 0x14, RT_CHANNEL_DOMAIN_TAIWAN_NO_DFS */
-	{0x00, 0x12},	/* 0x15, RT_CHANNEL_DOMAIN_ETSI_NO_DFS */
-	{0x00, 0x13},	/* 0x16, RT_CHANNEL_DOMAIN_KOREA_NO_DFS */
-	{0x03, 0x12},	/* 0x17, RT_CHANNEL_DOMAIN_JAPAN_NO_DFS */
-	{0x05, 0x08},	/* 0x18, RT_CHANNEL_DOMAIN_PAKISTAN_NO_DFS */
-	{0x02, 0x08},	/* 0x19, RT_CHANNEL_DOMAIN_TAIWAN2_NO_DFS */
-	{0x00, 0x00},	/* 0x1A, */
-	{0x00, 0x00},	/* 0x1B, */
-	{0x00, 0x00},	/* 0x1C, */
-	{0x00, 0x00},	/* 0x1D, */
-	{0x00, 0x00},	/* 0x1E, */
-	{0x05, 0x04},	/* 0x1F, RT_CHANNEL_DOMAIN_WORLD_WIDE_ONLY_5G */
-	/*  0x20 ~ 0x7F , New Define ===== */
-	{0x00, 0x00},	/* 0x20, RT_CHANNEL_DOMAIN_WORLD_NULL */
-	{0x01, 0x00},	/* 0x21, RT_CHANNEL_DOMAIN_ETSI1_NULL */
-	{0x02, 0x00},	/* 0x22, RT_CHANNEL_DOMAIN_FCC1_NULL */
-	{0x03, 0x00},	/* 0x23, RT_CHANNEL_DOMAIN_MKK1_NULL */
-	{0x04, 0x00},	/* 0x24, RT_CHANNEL_DOMAIN_ETSI2_NULL */
-	{0x02, 0x04},	/* 0x25, RT_CHANNEL_DOMAIN_FCC1_FCC1 */
-	{0x00, 0x01},	/* 0x26, RT_CHANNEL_DOMAIN_WORLD_ETSI1 */
-	{0x03, 0x0C},	/* 0x27, RT_CHANNEL_DOMAIN_MKK1_MKK1 */
-	{0x00, 0x0B},	/* 0x28, RT_CHANNEL_DOMAIN_WORLD_KCC1 */
-	{0x00, 0x05},	/* 0x29, RT_CHANNEL_DOMAIN_WORLD_FCC2 */
-	{0x00, 0x00},	/* 0x2A, */
-	{0x00, 0x00},	/* 0x2B, */
-	{0x00, 0x00},	/* 0x2C, */
-	{0x00, 0x00},	/* 0x2D, */
-	{0x00, 0x00},	/* 0x2E, */
-	{0x00, 0x00},	/* 0x2F, */
-	{0x00, 0x06},	/* 0x30, RT_CHANNEL_DOMAIN_WORLD_FCC3 */
-	{0x00, 0x07},	/* 0x31, RT_CHANNEL_DOMAIN_WORLD_FCC4 */
-	{0x00, 0x08},	/* 0x32, RT_CHANNEL_DOMAIN_WORLD_FCC5 */
-	{0x00, 0x09},	/* 0x33, RT_CHANNEL_DOMAIN_WORLD_FCC6 */
-	{0x02, 0x0A},	/* 0x34, RT_CHANNEL_DOMAIN_FCC1_FCC7 */
-	{0x00, 0x02},	/* 0x35, RT_CHANNEL_DOMAIN_WORLD_ETSI2 */
-	{0x00, 0x03},	/* 0x36, RT_CHANNEL_DOMAIN_WORLD_ETSI3 */
-	{0x03, 0x0D},	/* 0x37, RT_CHANNEL_DOMAIN_MKK1_MKK2 */
-	{0x03, 0x0E},	/* 0x38, RT_CHANNEL_DOMAIN_MKK1_MKK3 */
-	{0x02, 0x0F},	/* 0x39, RT_CHANNEL_DOMAIN_FCC1_NCC1 */
-	{0x00, 0x00},	/* 0x3A, */
-	{0x00, 0x00},	/* 0x3B, */
-	{0x00, 0x00},	/* 0x3C, */
-	{0x00, 0x00},	/* 0x3D, */
-	{0x00, 0x00},	/* 0x3E, */
-	{0x00, 0x00},	/* 0x3F, */
-	{0x02, 0x10},	/* 0x40, RT_CHANNEL_DOMAIN_FCC1_NCC2 */
-	{0x03, 0x00},	/* 0x41, RT_CHANNEL_DOMAIN_GLOBAL_DOAMIN_2G */
-};
-
-static struct rt_channel_plan_map RTW_CHANNEL_PLAN_MAP_REALTEK_DEFINE =
-{0x03, 0x02}; /* use the conbination for max channel numbers */
-
-static void dummy_event_callback(struct rtw_adapter *adapter, const u8 *pbuf)
-{
-}
-
-static struct fwevent wlanevents[] =
-{
-	{0, &dummy_event_callback},	/*0*/
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, &rtw_survey_event_cb23a},		/*8*/
-	{sizeof (struct surveydone_event), &rtw_surveydone_event_callback23a},
-	{0, &rtw23a_joinbss_event_cb},		/*10*/
-	{sizeof(struct stassoc_event), &rtw_stassoc_event_callback23a},
-	{sizeof(struct stadel_event), &rtw_stadel_event_callback23a},
-	{0, &dummy_event_callback},
-	{0, &dummy_event_callback},
-	{0, NULL},	/*15*/
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, &dummy_event_callback},
-	{0, NULL},	 /*20*/
-	{0, NULL},
-	{0, NULL},
-	{0, &dummy_event_callback},
-	{0, NULL},
-};
-
-
-static void rtw_correct_TSF(struct rtw_adapter *padapter)
-{
-	hw_var_set_correct_tsf(padapter);
-}
-
-static void
-rtw_update_TSF(struct mlme_ext_priv *pmlmeext, struct ieee80211_mgmt *mgmt)
-{
-	pmlmeext->TSFValue = get_unaligned_le64(&mgmt->u.beacon.timestamp);
-}
-
-/*
- * Search the @param channel_num in given @param channel_set
- * @ch_set: the given channel set
- * @ch: the given channel number
- *
- * return the index of channel_num in channel_set, -1 if not found
- */
-int rtw_ch_set_search_ch23a(struct rt_channel_info *ch_set, const u32 ch)
-{
-	int i;
-
-	for (i = 0; ch_set[i]. ChannelNum != 0; i++) {
-		if (ch == ch_set[i].ChannelNum)
-			break;
-	}
-
-	if (i >= ch_set[i].ChannelNum)
-		return -1;
-	return i;
-}
-
-/****************************************************************************
-
-Following are the initialization functions for WiFi MLME
-
-*****************************************************************************/
-
-int init_hw_mlme_ext23a(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	set_channel_bwmode23a(padapter, pmlmeext->cur_channel,
-			      pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode);
-	return _SUCCESS;
-}
-
-static void init_mlme_ext_priv23a_value(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	unsigned char	mixed_datarate[NumRates] = {
-		_1M_RATE_, _2M_RATE_, _5M_RATE_, _11M_RATE_, _6M_RATE_,
-		_9M_RATE_, _12M_RATE_, _18M_RATE_, _24M_RATE_, _36M_RATE_,
-		_48M_RATE_, _54M_RATE_, 0xff};
-	unsigned char	mixed_basicrate[NumRates] = {
-		_1M_RATE_, _2M_RATE_, _5M_RATE_, _11M_RATE_, _6M_RATE_,
-		_12M_RATE_, _24M_RATE_, 0xff,};
-
-	atomic_set(&pmlmeext->event_seq, 0);
-	/* reset to zero when disconnect at client mode */
-	pmlmeext->mgnt_seq = 0;
-
-	pmlmeext->cur_channel = padapter->registrypriv.channel;
-	pmlmeext->cur_bwmode = HT_CHANNEL_WIDTH_20;
-	pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-
-	pmlmeext->retry = 0;
-
-	pmlmeext->cur_wireless_mode = padapter->registrypriv.wireless_mode;
-
-	memcpy(pmlmeext->datarate, mixed_datarate, NumRates);
-	memcpy(pmlmeext->basicrate, mixed_basicrate, NumRates);
-
-	if (pmlmeext->cur_channel > 14)
-		pmlmeext->tx_rate = IEEE80211_OFDM_RATE_6MB;
-	else
-		pmlmeext->tx_rate = IEEE80211_CCK_RATE_1MB;
-
-	pmlmeext->sitesurvey_res.state = SCAN_DISABLE;
-	pmlmeext->sitesurvey_res.channel_idx = 0;
-	pmlmeext->sitesurvey_res.bss_cnt = 0;
-	pmlmeext->scan_abort = false;
-
-	pmlmeinfo->state = MSR_NOLINK;
-	pmlmeinfo->reauth_count = 0;
-	pmlmeinfo->reassoc_count = 0;
-	pmlmeinfo->link_count = 0;
-	pmlmeinfo->auth_seq = 0;
-	pmlmeinfo->auth_algo = dot11AuthAlgrthm_Open;
-	pmlmeinfo->key_index = 0;
-	pmlmeinfo->iv = 0;
-
-	pmlmeinfo->enc_algo = 0;
-	pmlmeinfo->authModeToggle = 0;
-
-	memset(pmlmeinfo->chg_txt, 0, 128);
-
-	pmlmeinfo->slotTime = SHORT_SLOT_TIME;
-	pmlmeinfo->preamble_mode = PREAMBLE_AUTO;
-
-	pmlmeinfo->dialogToken = 0;
-
-	pmlmeext->action_public_rxseq = 0xffff;
-	pmlmeext->action_public_dialog_token = 0xff;
-}
-
-static int has_channel(struct rt_channel_info *channel_set,
-		       u8 chanset_size, u8 chan) {
-	int i;
-
-	for (i = 0; i < chanset_size; i++) {
-		if (channel_set[i].ChannelNum == chan)
-			return 1;
-	}
-
-	return 0;
-}
-
-static void init_channel_list(struct rtw_adapter *padapter,
-			      struct rt_channel_info *channel_set,
-			      u8 chanset_size,
-			      struct p2p_channels *channel_list)
-{
-	struct p2p_oper_class_map op_class[] = {
-		{ IEEE80211G,  81,   1,  13,  1, BW20 },
-		{ IEEE80211G,  82,  14,  14,  1, BW20 },
-		{ IEEE80211A, 115,  36,  48,  4, BW20 },
-		{ IEEE80211A, 116,  36,  44,  8, BW40PLUS },
-		{ IEEE80211A, 117,  40,  48,  8, BW40MINUS },
-		{ IEEE80211A, 124, 149, 161,  4, BW20 },
-		{ IEEE80211A, 125, 149, 169,  4, BW20 },
-		{ IEEE80211A, 126, 149, 157,  8, BW40PLUS },
-		{ IEEE80211A, 127, 153, 161,  8, BW40MINUS },
-		{ -1, 0, 0, 0, 0, BW20 }
-	};
-
-	int cla, op;
-
-	cla = 0;
-
-	for (op = 0; op_class[op].op_class; op++) {
-		u8 ch;
-		struct p2p_oper_class_map *o = &op_class[op];
-		struct p2p_reg_class *reg = NULL;
-
-		for (ch = o->min_chan; ch <= o->max_chan; ch += o->inc) {
-			if (!has_channel(channel_set, chanset_size, ch))
-				continue;
-
-			if ((0 == padapter->registrypriv.ht_enable) &&
-			    (o->inc == 8))
-				continue;
-
-			if ((0 == (padapter->registrypriv.cbw40_enable & BIT(1))) &&
-				((BW40MINUS == o->bw) || (BW40PLUS == o->bw)))
-				continue;
-
-			if (reg == NULL) {
-				reg = &channel_list->reg_class[cla];
-				cla++;
-				reg->reg_class = o->op_class;
-				reg->channels = 0;
-			}
-			reg->channel[reg->channels] = ch;
-			reg->channels++;
-		}
-	}
-	channel_list->reg_classes = cla;
-}
-
-static u8 init_channel_set(struct rtw_adapter *padapter, u8 cplan,
-			   struct rt_channel_info *c_set)
-{
-	u8 i, ch_size = 0;
-	u8 b5GBand = false, b2_4GBand = false;
-	u8 Index2G = 0, Index5G = 0;
-
-	memset(c_set, 0, sizeof(struct rt_channel_info) * MAX_CHANNEL_NUM);
-
-	if (cplan >= RT_CHANNEL_DOMAIN_MAX &&
-	    cplan != RT_CHANNEL_DOMAIN_REALTEK_DEFINE) {
-		DBG_8723A("ChannelPlan ID %x error !!!!!\n", cplan);
-		return ch_size;
-	}
-
-	if (padapter->registrypriv.wireless_mode & WIRELESS_11G) {
-		b2_4GBand = true;
-		if (RT_CHANNEL_DOMAIN_REALTEK_DEFINE == cplan)
-			Index2G = RTW_CHANNEL_PLAN_MAP_REALTEK_DEFINE.Index2G;
-		else
-			Index2G = RTW_ChannelPlanMap[cplan].Index2G;
-	}
-
-	if (padapter->registrypriv.wireless_mode & WIRELESS_11A) {
-		b5GBand = true;
-		if (RT_CHANNEL_DOMAIN_REALTEK_DEFINE == cplan)
-			Index5G = RTW_CHANNEL_PLAN_MAP_REALTEK_DEFINE.Index5G;
-		else
-			Index5G = RTW_ChannelPlanMap[cplan].Index5G;
-	}
-
-	if (b2_4GBand) {
-		for (i = 0; i < RTW_ChannelPlan2G[Index2G].Len; i++) {
-			c_set[ch_size].ChannelNum =
-				RTW_ChannelPlan2G[Index2G].Channel[i];
-
-			if ((RT_CHANNEL_DOMAIN_GLOBAL_DOAMIN == cplan) ||
-			    /* Channel 1~11 is active, and 12~14 is passive */
-			    RT_CHANNEL_DOMAIN_GLOBAL_DOAMIN_2G == cplan) {
-				if (c_set[ch_size].ChannelNum >= 1 &&
-				    c_set[ch_size].ChannelNum <= 11)
-					c_set[ch_size].ScanType = SCAN_ACTIVE;
-				else if (c_set[ch_size].ChannelNum >= 12 &&
-					 c_set[ch_size].ChannelNum  <= 14)
-					c_set[ch_size].ScanType = SCAN_PASSIVE;
-			} else if (RT_CHANNEL_DOMAIN_WORLD_WIDE_13 == cplan ||
-				   RT_CHANNEL_DOMAIN_WORLD_WIDE_5G == cplan ||
-				   RT_CHANNEL_DOMAIN_2G_WORLD == Index2G) {
-				/*  channel 12~13, passive scan */
-				if (c_set[ch_size].ChannelNum <= 11)
-					c_set[ch_size].ScanType = SCAN_ACTIVE;
-				else
-					c_set[ch_size].ScanType = SCAN_PASSIVE;
-			} else
-				c_set[ch_size].ScanType = SCAN_ACTIVE;
-
-			ch_size++;
-		}
-	}
-
-	if (b5GBand) {
-		for (i = 0; i < RTW_ChannelPlan5G[Index5G].Len; i++) {
-			if (RTW_ChannelPlan5G[Index5G].Channel[i] <= 48 ||
-			    RTW_ChannelPlan5G[Index5G].Channel[i] >= 149) {
-				c_set[ch_size].ChannelNum =
-					RTW_ChannelPlan5G[Index5G].Channel[i];
-				if (RT_CHANNEL_DOMAIN_WORLD_WIDE_5G == cplan) {
-					/* passive scan for all 5G channels */
-					c_set[ch_size].ScanType =
-						SCAN_PASSIVE;
-				} else
-					c_set[ch_size].ScanType =
-						SCAN_ACTIVE;
-				DBG_8723A("%s(): channel_set[%d].ChannelNum = "
-					  "%d\n", __func__, ch_size,
-					  c_set[ch_size].ChannelNum);
-				ch_size++;
-			}
-		}
-	}
-
-	return ch_size;
-}
-
-int init_mlme_ext_priv23a(struct rtw_adapter *padapter)
-{
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	pmlmeext->padapter = padapter;
-
-	init_mlme_ext_priv23a_value(padapter);
-	pmlmeinfo->bAcceptAddbaReq = pregistrypriv->bAcceptAddbaReq;
-
-	init_mlme_ext_timer23a(padapter);
-
-#ifdef CONFIG_8723AU_AP_MODE
-	init_mlme_ap_info23a(padapter);
-#endif
-
-	pmlmeext->max_chan_nums = init_channel_set(padapter,
-						   pmlmepriv->ChannelPlan,
-						   pmlmeext->channel_set);
-	init_channel_list(padapter, pmlmeext->channel_set,
-			  pmlmeext->max_chan_nums, &pmlmeext->channel_list);
-
-	pmlmeext->chan_scan_time = SURVEY_TO;
-	pmlmeext->mlmeext_init = true;
-
-	pmlmeext->active_keep_alive_check = true;
-	return _SUCCESS;
-}
-
-void free_mlme_ext_priv23a (struct mlme_ext_priv *pmlmeext)
-{
-	struct rtw_adapter *padapter = pmlmeext->padapter;
-
-	if (!padapter)
-		return;
-
-	if (padapter->bDriverStopped == true) {
-		del_timer_sync(&pmlmeext->survey_timer);
-		del_timer_sync(&pmlmeext->link_timer);
-		/* del_timer_sync(&pmlmeext->ADDBA_timer); */
-	}
-}
-
-static void
-_mgt_dispatcher23a(struct rtw_adapter *padapter, struct mlme_handler *ptable,
-		   struct recv_frame *precv_frame)
-{
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-
-	if (ptable->func) {
-		/* receive the frames that ra(a1) is my address
-		   or ra(a1) is bc address. */
-		if (!ether_addr_equal(hdr->addr1, myid(&padapter->eeprompriv))&&
-		    !is_broadcast_ether_addr(hdr->addr1))
-			return;
-
-		ptable->func(padapter, precv_frame);
-	}
-}
-
-void mgt_dispatcher23a(struct rtw_adapter *padapter,
-		    struct recv_frame *precv_frame)
-{
-	struct mlme_handler *ptable;
-#ifdef CONFIG_8723AU_AP_MODE
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-#endif /* CONFIG_8723AU_AP_MODE */
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-	struct sta_info *psta;
-	u16 stype;
-	int index;
-
-	if (!ieee80211_is_mgmt(mgmt->frame_control))
-		return;
-
-	/* receive the frames that ra(a1) is my address or ra(a1) is
-	   bc address. */
-	if (!ether_addr_equal(mgmt->da, myid(&padapter->eeprompriv)) &&
-	    !is_broadcast_ether_addr(mgmt->da))
-		return;
-
-	ptable = mlme_sta_tbl;
-
-	stype = le16_to_cpu(mgmt->frame_control) & IEEE80211_FCTL_STYPE;
-	index = stype >> 4;
-
-	if (index > 13) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "Currently we do not support reserved sub-fr-type =%d\n",
-			 index);
-		return;
-	}
-	ptable += index;
-
-	psta = rtw_get_stainfo23a(&padapter->stapriv, mgmt->sa);
-
-	if (psta) {
-		if (ieee80211_has_retry(mgmt->frame_control)) {
-			if (precv_frame->attrib.seq_num ==
-			    psta->RxMgmtFrameSeqNum) {
-				/* drop the duplicate management frame */
-				DBG_8723A("Drop duplicate management frame "
-					  "with seq_num = %d.\n",
-					  precv_frame->attrib.seq_num);
-				return;
-			}
-		}
-		psta->RxMgmtFrameSeqNum = precv_frame->attrib.seq_num;
-	}
-
-#ifdef CONFIG_8723AU_AP_MODE
-	switch (stype) {
-	case IEEE80211_STYPE_AUTH:
-		if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
-			ptable->func = &OnAuth23a;
-		else
-			ptable->func = &OnAuth23aClient23a;
-		/* pass through */
-	case IEEE80211_STYPE_ASSOC_REQ:
-	case IEEE80211_STYPE_REASSOC_REQ:
-		_mgt_dispatcher23a(padapter, ptable, precv_frame);
-		break;
-	case IEEE80211_STYPE_PROBE_REQ:
-		if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
-			_mgt_dispatcher23a(padapter, ptable, precv_frame);
-		else
-			_mgt_dispatcher23a(padapter, ptable, precv_frame);
-		break;
-	case IEEE80211_STYPE_BEACON:
-		_mgt_dispatcher23a(padapter, ptable, precv_frame);
-		break;
-	case IEEE80211_STYPE_ACTION:
-		/* if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) */
-		_mgt_dispatcher23a(padapter, ptable, precv_frame);
-		break;
-	default:
-		_mgt_dispatcher23a(padapter, ptable, precv_frame);
-		break;
-	}
-#else
-	_mgt_dispatcher23a(padapter, ptable, precv_frame);
-#endif
-}
-
-/****************************************************************************
-
-Following are the callback functions for each subtype of the management frames
-
-*****************************************************************************/
-
-static int
-OnProbeReq23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	const u8 *ie;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur = &pmlmeinfo->network;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-	int len = skb->len;
-
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE))
-		return _SUCCESS;
-
-	if (!check_fwstate(pmlmepriv, _FW_LINKED) &&
-	    !check_fwstate(pmlmepriv,
-			   WIFI_ADHOC_MASTER_STATE | WIFI_AP_STATE))
-		return _SUCCESS;
-
-	if (unlikely(!ieee80211_is_probe_req(mgmt->frame_control))) {
-		printk(KERN_WARNING "%s: Received non probe request frame\n",
-		       __func__);
-		return _FAIL;
-	}
-
-	len -= offsetof(struct ieee80211_mgmt, u.probe_req.variable);
-
-	ie = cfg80211_find_ie(WLAN_EID_SSID, mgmt->u.probe_req.variable, len);
-
-	/* check (wildcard) SSID */
-	if (!ie)
-		goto out;
-
-	if ((ie[1] && memcmp(ie + 2, cur->Ssid.ssid, cur->Ssid.ssid_len)) ||
-	    (ie[1] == 0 && pmlmeinfo->hidden_ssid_mode)) {
-		return _SUCCESS;
-	}
-
-	if (check_fwstate(pmlmepriv, _FW_LINKED) &&
-	    pmlmepriv->cur_network.join_res)
-		issue_probersp(padapter, mgmt->sa);
-
-out:
-	return _SUCCESS;
-}
-
-static int
-OnProbeRsp23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	if (pmlmeext->sitesurvey_res.state == SCAN_PROCESS) {
-		report_survey_event23a(padapter, precv_frame);
-		return _SUCCESS;
-	}
-
-	return _SUCCESS;
-}
-
-static int
-OnBeacon23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	int cam_idx;
-	struct sta_info	*psta;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct sta_priv	*pstapriv = &padapter->stapriv;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-	int pkt_len = skb->len;
-	struct wlan_bssid_ex *pbss;
-	int ret = _SUCCESS;
-	u8 *p, *pie;
-	int pie_len;
-	u32 ielen = 0;
-
-	pie = mgmt->u.beacon.variable;
-	pie_len = pkt_len - offsetof(struct ieee80211_mgmt, u.beacon.variable);
-	p = rtw_get_ie23a(pie, WLAN_EID_EXT_SUPP_RATES, &ielen, pie_len);
-	if (p && ielen > 0) {
-		if (p[1 + ielen] == 0x2D && p[2 + ielen] != 0x2D) {
-			/* Invalid value 0x2D is detected in Extended Supported
-			 * Rates (ESR) IE. Try to fix the IE length to avoid
-			 * failed Beacon parsing.
-			 */
-			DBG_8723A("[WIFIDBG] Error in ESR IE is detected in "
-				  "Beacon of BSSID: %pM. Fix the length of "
-				  "ESR IE to avoid failed Beacon parsing.\n",
-				  mgmt->bssid);
-			p[1] = ielen - 1;
-		}
-	}
-
-	if (pmlmeext->sitesurvey_res.state == SCAN_PROCESS) {
-		report_survey_event23a(padapter, precv_frame);
-		return _SUCCESS;
-	}
-
-	if (!ether_addr_equal(mgmt->bssid,
-			      get_my_bssid23a(&pmlmeinfo->network)))
-		goto out;
-
-	if (pmlmeinfo->state & WIFI_FW_AUTH_NULL) {
-		/* we should update current network before auth,
-		   or some IE is wrong */
-		pbss = collect_bss_info(padapter, precv_frame);
-		if (pbss) {
-			update_network23a(&pmlmepriv->cur_network.network, pbss,
-					  padapter, true);
-			rtw_get_bcn_info23a(&pmlmepriv->cur_network);
-			kfree(pbss);
-		}
-
-		/* check the vendor of the assoc AP */
-		pmlmeinfo->assoc_AP_vendor =
-			check_assoc_AP23a((u8 *)&mgmt->u.beacon, pkt_len -
-					  offsetof(struct ieee80211_mgmt, u));
-
-		/* update TSF Value */
-		rtw_update_TSF(pmlmeext, mgmt);
-
-		/* start auth */
-		start_clnt_auth(padapter);
-
-		return _SUCCESS;
-	}
-
-	if (((pmlmeinfo->state & 0x03) == MSR_AP) &&
-	    (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS)) {
-		psta = rtw_get_stainfo23a(pstapriv, mgmt->sa);
-		if (psta) {
-			ret = rtw_check_bcn_info23a(padapter, mgmt, pkt_len);
-			if (ret != _SUCCESS) {
-				DBG_8723A_LEVEL(_drv_always_, "ap has changed, "
-						"disconnect now\n");
-				receive_disconnect23a(padapter, pmlmeinfo->network.MacAddress, 65535);
-				return _SUCCESS;
-			}
-			/* update WMM, ERP in the beacon */
-			/* todo: the timer is used instead of
-			   the number of the beacon received */
-			if ((sta_rx_pkts(psta) & 0xf) == 0) {
-				/* DBG_8723A("update_bcn_info\n"); */
-				update_beacon23a_info(padapter, mgmt,
-						      pkt_len, psta);
-			}
-		}
-	} else if ((pmlmeinfo->state&0x03) == MSR_ADHOC) {
-		psta = rtw_get_stainfo23a(pstapriv, mgmt->sa);
-		if (psta) {
-			/* update WMM, ERP in the beacon */
-			/* todo: the timer is used instead of the
-			   number of the beacon received */
-			if ((sta_rx_pkts(psta) & 0xf) == 0) {
-				/* DBG_8723A("update_bcn_info\n"); */
-				update_beacon23a_info(padapter, mgmt,
-						      pkt_len, psta);
-			}
-		} else {
-			/* allocate a new CAM entry for IBSS station */
-			cam_idx = allocate_fw_sta_entry23a(padapter);
-			if (cam_idx == NUM_STA)
-				goto out;
-
-			/* get supported rate */
-			if (update_sta_support_rate23a(padapter, pie, pie_len,
-						       cam_idx) == _FAIL) {
-				pmlmeinfo->FW_sta_info[cam_idx].status = 0;
-				goto out;
-			}
-
-			/* update TSF Value */
-			rtw_update_TSF(pmlmeext, mgmt);
-
-			/* report sta add event */
-			report_add_sta_event23a(padapter, mgmt->sa,
-						cam_idx);
-		}
-	}
-
-out:
-
-	return _SUCCESS;
-}
-
-#ifdef CONFIG_8723AU_AP_MODE
-static int
-OnAuth23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	static struct sta_info stat;
-	struct sta_info *pstat = NULL;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-	u8 *pframe;
-	const u8 *p;
-	unsigned char *sa;
-	u16 auth_mode, seq, algorithm;
-	int status, len = skb->len;
-
-	if ((pmlmeinfo->state & 0x03) != MSR_AP)
-		return _FAIL;
-
-	DBG_8723A("+OnAuth23a\n");
-
-	sa = mgmt->sa;
-
-	auth_mode = psecuritypriv->dot11AuthAlgrthm;
-
-	pframe = mgmt->u.auth.variable;
-	len = skb->len - offsetof(struct ieee80211_mgmt, u.auth.variable);
-
-	seq = le16_to_cpu(mgmt->u.auth.auth_transaction);
-	algorithm = le16_to_cpu(mgmt->u.auth.auth_alg);
-
-	DBG_8723A("auth alg =%x, seq =%X\n", algorithm, seq);
-
-	if (auth_mode == 2 &&
-	    psecuritypriv->dot11PrivacyAlgrthm != WLAN_CIPHER_SUITE_WEP40 &&
-	    psecuritypriv->dot11PrivacyAlgrthm != WLAN_CIPHER_SUITE_WEP104)
-		auth_mode = 0;
-
-	/*  rx a shared-key auth but shared not enabled, or */
-	/*  rx a open-system auth but shared-key is enabled */
-	if ((algorithm != WLAN_AUTH_OPEN && auth_mode == 0) ||
-	    (algorithm == WLAN_AUTH_OPEN && auth_mode == 1)) {
-		DBG_8723A("auth rejected due to bad alg [alg =%d, auth_mib "
-			  "=%d] %02X%02X%02X%02X%02X%02X\n",
-			  algorithm, auth_mode,
-			  sa[0], sa[1], sa[2], sa[3], sa[4], sa[5]);
-
-		status = WLAN_STATUS_NOT_SUPPORTED_AUTH_ALG;
-
-		goto auth_fail;
-	}
-
-	if (rtw_access_ctrl23a(padapter, sa) == false) {
-		status = WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA;
-		goto auth_fail;
-	}
-
-	pstat = rtw_get_stainfo23a(pstapriv, sa);
-	if (!pstat) {
-		/*  allocate a new one */
-		DBG_8723A("going to alloc stainfo for sa =%pM\n", sa);
-		pstat = rtw_alloc_stainfo23a(pstapriv, sa, GFP_ATOMIC);
-		if (!pstat) {
-			DBG_8723A(" Exceed the upper limit of supported "
-				  "clients...\n");
-			status = WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA;
-			goto auth_fail;
-		}
-
-		pstat->state = WIFI_FW_AUTH_NULL;
-		pstat->auth_seq = 0;
-
-		/* pstat->flags = 0; */
-		/* pstat->capability = 0; */
-	} else {
-		spin_lock_bh(&pstapriv->asoc_list_lock);
-		if (!list_empty(&pstat->asoc_list)) {
-			list_del_init(&pstat->asoc_list);
-			pstapriv->asoc_list_cnt--;
-			if (pstat->expire_to > 0) {
-				/* TODO: STA re_auth within expire_to */
-			}
-		}
-		spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-		if (seq == 1) {
-			/* TODO: STA re_auth and auth timeout */
-		}
-	}
-
-	spin_lock_bh(&pstapriv->auth_list_lock);
-	if (list_empty(&pstat->auth_list)) {
-		list_add_tail(&pstat->auth_list, &pstapriv->auth_list);
-		pstapriv->auth_list_cnt++;
-	}
-	spin_unlock_bh(&pstapriv->auth_list_lock);
-
-	if (pstat->auth_seq == 0)
-		pstat->expire_to = pstapriv->auth_to;
-
-	if ((pstat->auth_seq + 1) != seq) {
-		DBG_8723A("(1)auth rejected because out of seq [rx_seq =%d, "
-			  "exp_seq =%d]!\n", seq, pstat->auth_seq+1);
-		status = WLAN_STATUS_UNKNOWN_AUTH_TRANSACTION;
-		goto auth_fail;
-	}
-
-	if (algorithm == WLAN_AUTH_OPEN && (auth_mode == 0 || auth_mode == 2)) {
-		if (seq == 1) {
-			pstat->state &= ~WIFI_FW_AUTH_NULL;
-			pstat->state |= WIFI_FW_AUTH_SUCCESS;
-			pstat->expire_to = pstapriv->assoc_to;
-			pstat->authalg = algorithm;
-		} else {
-			DBG_8723A("(2)auth rejected because out of seq "
-				  "[rx_seq =%d, exp_seq =%d]!\n",
-				  seq, pstat->auth_seq+1);
-			status = WLAN_STATUS_UNKNOWN_AUTH_TRANSACTION;
-			goto auth_fail;
-		}
-	} else { /*  shared system or auto authentication */
-		if (seq == 1) {
-			/* prepare for the challenging txt... */
-			pstat->state &= ~WIFI_FW_AUTH_NULL;
-			pstat->state |= WIFI_FW_AUTH_STATE;
-			pstat->authalg = algorithm;
-			pstat->auth_seq = 2;
-		} else if (seq == 3) {
-			/* checking for challenging txt... */
-			DBG_8723A("checking for challenging txt...\n");
-
-			p = cfg80211_find_ie(WLAN_EID_CHALLENGE, pframe, len);
-			if (!p || p[1] <= 0) {
-				DBG_8723A("auth rejected because challenge "
-					  "failure!(1)\n");
-				status = WLAN_STATUS_CHALLENGE_FAIL;
-				goto auth_fail;
-			}
-
-			if (!memcmp(p + 2, pstat->chg_txt, 128)) {
-				pstat->state &= ~WIFI_FW_AUTH_STATE;
-				pstat->state |= WIFI_FW_AUTH_SUCCESS;
-				/*  challenging txt is correct... */
-				pstat->expire_to =  pstapriv->assoc_to;
-			} else {
-				DBG_8723A("auth rejected because challenge "
-					  "failure!\n");
-				status = WLAN_STATUS_CHALLENGE_FAIL;
-				goto auth_fail;
-			}
-		} else {
-			DBG_8723A("(3)auth rejected because out of seq "
-				  "[rx_seq =%d, exp_seq =%d]!\n",
-				  seq, pstat->auth_seq+1);
-			status = WLAN_STATUS_UNKNOWN_AUTH_TRANSACTION;
-			goto auth_fail;
-		}
-	}
-
-	/*  Now, we are going to issue_auth... */
-	pstat->auth_seq = seq + 1;
-
-	issue_auth(padapter, pstat, WLAN_STATUS_SUCCESS);
-
-	if (pstat->state & WIFI_FW_AUTH_SUCCESS)
-		pstat->auth_seq = 0;
-
-	return _SUCCESS;
-
-auth_fail:
-
-	if (pstat)
-		rtw_free_stainfo23a(padapter, pstat);
-
-	pstat = &stat;
-	memset((char *)pstat, '\0', sizeof(stat));
-	pstat->auth_seq = 2;
-	ether_addr_copy(pstat->hwaddr, sa);
-
-	issue_auth(padapter, pstat, (unsigned short)status);
-
-	return _FAIL;
-}
-#endif
-
-static int
-OnAuth23aClient23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	unsigned int seq, status, algthm;
-	unsigned int go2asoc = 0;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-	const u8 *p;
-	u8 *pie;
-	int plen = skb->len;
-
-	DBG_8723A("%s\n", __func__);
-
-	/* check A1 matches or not */
-	if (!ether_addr_equal(myid(&padapter->eeprompriv), mgmt->da))
-		return _SUCCESS;
-
-	if (!(pmlmeinfo->state & WIFI_FW_AUTH_STATE))
-		return _SUCCESS;
-
-	pie = mgmt->u.auth.variable;
-	plen -= offsetof(struct ieee80211_mgmt, u.auth.variable);
-
-	algthm = le16_to_cpu(mgmt->u.auth.auth_alg);
-	seq = le16_to_cpu(mgmt->u.auth.auth_transaction);
-	status = le16_to_cpu(mgmt->u.auth.status_code);
-
-	if (status) {
-		DBG_8723A("clnt auth fail, status: %d\n", status);
-		/*  pmlmeinfo->auth_algo == dot11AuthAlgrthm_Auto) */
-		if (status == WLAN_STATUS_NOT_SUPPORTED_AUTH_ALG) {
-			if (pmlmeinfo->auth_algo == dot11AuthAlgrthm_Shared)
-				pmlmeinfo->auth_algo = dot11AuthAlgrthm_Open;
-			else
-				pmlmeinfo->auth_algo = dot11AuthAlgrthm_Shared;
-			/* pmlmeinfo->reauth_count = 0; */
-		}
-
-		set_link_timer(pmlmeext, 1);
-		goto authclnt_fail;
-	}
-
-	if (seq == 2) {
-		if (pmlmeinfo->auth_algo == dot11AuthAlgrthm_Shared) {
-			/*  legendary shared system */
-			p = cfg80211_find_ie(WLAN_EID_CHALLENGE, pie, plen);
-
-			if (!p) {
-				/* DBG_8723A("marc: no challenge text?\n"); */
-				goto authclnt_fail;
-			}
-
-			memcpy((void *)(pmlmeinfo->chg_txt), p + 2, p[1]);
-			pmlmeinfo->auth_seq = 3;
-			issue_auth(padapter, NULL, 0);
-			set_link_timer(pmlmeext, REAUTH_TO);
-
-			return _SUCCESS;
-		} else {
-			/*  open system */
-			go2asoc = 1;
-		}
-	} else if (seq == 4) {
-		if (pmlmeinfo->auth_algo == dot11AuthAlgrthm_Shared)
-			go2asoc = 1;
-		else
-			goto authclnt_fail;
-	} else {
-		/*  this is also illegal */
-		/* DBG_8723A("marc: clnt auth failed due to illegal seq =%x\n",
-		   seq); */
-		goto authclnt_fail;
-	}
-
-	if (go2asoc) {
-		DBG_8723A_LEVEL(_drv_always_, "auth success, start assoc\n");
-		start_clnt_assoc(padapter);
-		return _SUCCESS;
-	}
-
-authclnt_fail:
-
-	/* pmlmeinfo->state &= ~(WIFI_FW_AUTH_STATE); */
-
-	return _FAIL;
-}
-
-#ifdef CONFIG_8723AU_AP_MODE
-static int rtw_validate_vendor_specific_ies(const u8 *pos, int elen)
-{
-	unsigned int oui;
-
-	/* first 3 bytes in vendor specific information element are the IEEE
-	 * OUI of the vendor. The following byte is used a vendor specific
-	 * sub-type. */
-	if (elen < 4) {
-		DBG_8723A("short vendor specific information element "
-			  "ignored (len =%i)\n", elen);
-		return -EINVAL;
-	}
-
-	oui = RTW_GET_BE24(pos);
-	switch (oui) {
-	case WLAN_OUI_MICROSOFT:
-		/* Microsoft/Wi-Fi information elements are further typed and
-		 * subtyped */
-		switch (pos[3]) {
-		case WLAN_OUI_TYPE_MICROSOFT_WPA:
-			/* Microsoft OUI (00:50:F2) with OUI Type 1:
-			 * real WPA information element */
-			break;
-		case WLAN_OUI_TYPE_MICROSOFT_WMM:
-			if (elen < 5) {
-				DBG_8723A("short WME information element "
-					  "ignored (len =%i)\n", elen);
-				return -EINVAL;
-			}
-			switch (pos[4]) {
-			case WME_OUI_SUBTYPE_INFORMATION_ELEMENT:
-			case WME_OUI_SUBTYPE_PARAMETER_ELEMENT:
-				break;
-			case WME_OUI_SUBTYPE_TSPEC_ELEMENT:
-				break;
-			default:
-				DBG_8723A("unknown WME information element "
-					  "ignored (subtype =%d len =%i)\n",
-					   pos[4], elen);
-				return -EINVAL;
-			}
-			break;
-		case WLAN_OUI_TYPE_MICROSOFT_WPS:
-			/* Wi-Fi Protected Setup (WPS) IE */
-			break;
-		default:
-			DBG_8723A("Unknown Microsoft information element "
-				  "ignored (type =%d len =%i)\n",
-				  pos[3], elen);
-			return -EINVAL;
-		}
-		break;
-
-	case OUI_BROADCOM:
-		switch (pos[3]) {
-		case VENDOR_HT_CAPAB_OUI_TYPE:
-			break;
-		default:
-			DBG_8723A("Unknown Broadcom information element "
-				  "ignored (type =%d len =%i)\n", pos[3], elen);
-			return -EINVAL;
-		}
-		break;
-
-	default:
-		DBG_8723A("unknown vendor specific information element "
-			  "ignored (vendor OUI %02x:%02x:%02x len =%i)\n",
-			   pos[0], pos[1], pos[2], elen);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int rtw_validate_frame_ies(const u8 *start, uint len)
-{
-	const u8 *pos = start;
-	int left = len;
-	int unknown = 0;
-
-	while (left >= 2) {
-		u8 id, elen;
-
-		id = *pos++;
-		elen = *pos++;
-		left -= 2;
-
-		if (elen > left) {
-			DBG_8723A("%s: IEEE 802.11 failed (id =%d elen =%d "
-				  "left =%i)\n", __func__, id, elen, left);
-			return -EINVAL;
-		}
-
-		switch (id) {
-		case WLAN_EID_SSID:
-		case WLAN_EID_SUPP_RATES:
-		case WLAN_EID_FH_PARAMS:
-		case WLAN_EID_DS_PARAMS:
-		case WLAN_EID_CF_PARAMS:
-		case WLAN_EID_TIM:
-		case WLAN_EID_IBSS_PARAMS:
-		case WLAN_EID_CHALLENGE:
-		case WLAN_EID_ERP_INFO:
-		case WLAN_EID_EXT_SUPP_RATES:
-			break;
-		case WLAN_EID_VENDOR_SPECIFIC:
-			if (rtw_validate_vendor_specific_ies(pos, elen))
-				unknown++;
-			break;
-		case WLAN_EID_RSN:
-		case WLAN_EID_PWR_CAPABILITY:
-		case WLAN_EID_SUPPORTED_CHANNELS:
-		case WLAN_EID_MOBILITY_DOMAIN:
-		case WLAN_EID_FAST_BSS_TRANSITION:
-		case WLAN_EID_TIMEOUT_INTERVAL:
-		case WLAN_EID_HT_CAPABILITY:
-		case WLAN_EID_HT_OPERATION:
-		default:
-			unknown++;
-			DBG_8723A("%s IEEE 802.11 ignored unknown element "
-				  "(id =%d elen =%d)\n", __func__, id, elen);
-			break;
-		}
-
-		left -= elen;
-		pos += elen;
-	}
-
-	if (left)
-		return -EINVAL;
-
-	return 0;
-}
-#endif
-
-static int
-OnAssocReq23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-#ifdef CONFIG_8723AU_AP_MODE
-	u16 capab_info, listen_interval;
-	struct sta_info	*pstat;
-	unsigned char reassoc;
-	int i, wpa_ie_len, left;
-	unsigned char supportRate[16];
-	int supportRateNum;
-	unsigned short status = WLAN_STATUS_SUCCESS;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur = &pmlmeinfo->network;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-	const u8 *pos, *p, *wpa_ie, *wps_ie;
-	u8 *pframe = skb->data;
-	uint pkt_len = skb->len;
-	int r;
-
-	if ((pmlmeinfo->state & 0x03) != MSR_AP)
-		return _FAIL;
-
-	left = pkt_len - sizeof(struct ieee80211_hdr_3addr);
-	if (ieee80211_is_assoc_req(mgmt->frame_control)) {
-		reassoc = 0;
-		pos = mgmt->u.assoc_req.variable;
-		left -= offsetof(struct ieee80211_mgmt, u.assoc_req.variable);
-	} else { /*  WIFI_REASSOCREQ */
-		reassoc = 1;
-		pos = mgmt->u.reassoc_req.variable;
-		left -= offsetof(struct ieee80211_mgmt, u.reassoc_req.variable);
-	}
-
-	if (left < 0) {
-		DBG_8723A("handle_assoc(reassoc =%d) - too short payload "
-			  "(len =%lu)\n", reassoc, (unsigned long)pkt_len);
-		return _FAIL;
-	}
-
-	pstat = rtw_get_stainfo23a(pstapriv, mgmt->sa);
-	if (!pstat) {
-		status = WLAN_REASON_CLASS2_FRAME_FROM_NONAUTH_STA;
-		goto asoc_class2_error;
-	}
-
-	/* These two are located at the same offsets whether it's an
-	 * assoc_req or a reassoc_req */
-	capab_info = get_unaligned_le16(&mgmt->u.assoc_req.capab_info);
-	listen_interval =
-		get_unaligned_le16(&mgmt->u.assoc_req.listen_interval);
-
-	DBG_8723A("%s\n", __func__);
-
-	/*  check if this stat has been successfully authenticated/assocated */
-	if (!(pstat->state & WIFI_FW_AUTH_SUCCESS)) {
-		if (!(pstat->state & WIFI_FW_ASSOC_SUCCESS)) {
-			status = WLAN_REASON_CLASS2_FRAME_FROM_NONAUTH_STA;
-			goto asoc_class2_error;
-		} else {
-			pstat->state &= (~WIFI_FW_ASSOC_SUCCESS);
-			pstat->state |= WIFI_FW_ASSOC_STATE;
-		}
-	} else {
-		pstat->state &= (~WIFI_FW_AUTH_SUCCESS);
-		pstat->state |= WIFI_FW_ASSOC_STATE;
-	}
-
-	pstat->capability = capab_info;
-
-	/* now parse all ieee802_11 ie to point to elems */
-
-	if (rtw_validate_frame_ies(pos, left)) {
-		DBG_8723A("STA %pM sent invalid association request\n",
-			  pstat->hwaddr);
-		status = WLAN_STATUS_UNSPECIFIED_FAILURE;
-		goto OnAssocReq23aFail;
-	}
-
-	/*  now we should check all the fields... */
-	/*  checking SSID */
-	p = cfg80211_find_ie(WLAN_EID_SSID, pos, left);
-	if (!p || p[1] == 0) {
-		/*  broadcast ssid, however it is not allowed in assocreq */
-		DBG_8723A("STA %pM sent invalid association request lacking an SSID\n",
-			  pstat->hwaddr);
-		status = WLAN_STATUS_UNSPECIFIED_FAILURE;
-		goto OnAssocReq23aFail;
-	} else {
-		/*  check if ssid match */
-		if (memcmp(p + 2, cur->Ssid.ssid, cur->Ssid.ssid_len))
-			status = WLAN_STATUS_UNSPECIFIED_FAILURE;
-
-		if (p[1] != cur->Ssid.ssid_len)
-			status = WLAN_STATUS_UNSPECIFIED_FAILURE;
-	}
-
-	if (status != WLAN_STATUS_SUCCESS)
-		goto OnAssocReq23aFail;
-
-	/*  check if the supported rate is ok */
-	p = cfg80211_find_ie(WLAN_EID_SUPP_RATES, pos, left);
-	if (!p) {
-		DBG_8723A("Rx a sta assoc-req which supported rate is "
-			  "empty!\n");
-		/*  use our own rate set as statoin used */
-		/* memcpy(supportRate, AP_BSSRATE, AP_BSSRATE_LEN); */
-		/* supportRateNum = AP_BSSRATE_LEN; */
-
-		status = WLAN_STATUS_UNSPECIFIED_FAILURE;
-		goto OnAssocReq23aFail;
-	} else {
-		memcpy(supportRate, p + 2, p[1]);
-		supportRateNum = p[1];
-
-		p = cfg80211_find_ie(WLAN_EID_EXT_SUPP_RATES, pos, left);
-		if (p) {
-			if (supportRateNum <= sizeof(supportRate)) {
-				memcpy(supportRate+supportRateNum, p + 2, p[1]);
-				supportRateNum += p[1];
-			}
-		}
-	}
-
-	/* todo: mask supportRate between AP & STA -> move to update raid */
-	/* get_matched_rate(pmlmeext, supportRate, &supportRateNum, 0); */
-
-	/* update station supportRate */
-	pstat->bssratelen = supportRateNum;
-	memcpy(pstat->bssrateset, supportRate, supportRateNum);
-	Update23aTblForSoftAP(pstat->bssrateset, pstat->bssratelen);
-
-	/* check RSN/WPA/WPS */
-	pstat->dot8021xalg = 0;
-	pstat->wpa_psk = 0;
-	pstat->wpa_group_cipher = 0;
-	pstat->wpa2_group_cipher = 0;
-	pstat->wpa_pairwise_cipher = 0;
-	pstat->wpa2_pairwise_cipher = 0;
-	memset(pstat->wpa_ie, 0, sizeof(pstat->wpa_ie));
-
-	wpa_ie = cfg80211_find_ie(WLAN_EID_RSN, pos, left);
-	if (!wpa_ie)
-		wpa_ie = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-						 WLAN_OUI_TYPE_MICROSOFT_WPA,
-						 pos, left);
-	if (wpa_ie) {
-		int group_cipher = 0, pairwise_cipher = 0;
-
-		wpa_ie_len = wpa_ie[1];
-		if (psecuritypriv->wpa_psk & BIT(1)) {
-			r = rtw_parse_wpa2_ie23a(wpa_ie, wpa_ie_len + 2,
-						 &group_cipher,
-						 &pairwise_cipher, NULL);
-			if (r == _SUCCESS) {
-				pstat->dot8021xalg = 1;/* psk,  todo:802.1x */
-				pstat->wpa_psk |= BIT(1);
-
-				pstat->wpa2_group_cipher = group_cipher &
-					psecuritypriv->wpa2_group_cipher;
-				pstat->wpa2_pairwise_cipher = pairwise_cipher &
-					psecuritypriv->wpa2_pairwise_cipher;
-			} else
-				status = WLAN_STATUS_INVALID_IE;
-		} else if (psecuritypriv->wpa_psk & BIT(0)) {
-			r = rtw_parse_wpa_ie23a(wpa_ie, wpa_ie_len + 2,
-						&group_cipher, &pairwise_cipher,
-						NULL);
-			if (r == _SUCCESS) {
-				pstat->dot8021xalg = 1;/* psk,  todo:802.1x */
-				pstat->wpa_psk |= BIT(0);
-
-				pstat->wpa_group_cipher = group_cipher &
-					psecuritypriv->wpa_group_cipher;
-				pstat->wpa_pairwise_cipher = pairwise_cipher &
-					psecuritypriv->wpa_pairwise_cipher;
-			} else
-				status = WLAN_STATUS_INVALID_IE;
-		} else {
-			wpa_ie = NULL;
-			wpa_ie_len = 0;
-		}
-		if (wpa_ie && status == WLAN_STATUS_SUCCESS) {
-			if (!pstat->wpa_group_cipher)
-				status = WLAN_STATUS_INVALID_GROUP_CIPHER;
-
-			if (!pstat->wpa_pairwise_cipher)
-				status = WLAN_STATUS_INVALID_PAIRWISE_CIPHER;
-		}
-	}
-
-	if (status != WLAN_STATUS_SUCCESS)
-		goto OnAssocReq23aFail;
-
-	pstat->flags &= ~(WLAN_STA_WPS | WLAN_STA_MAYBE_WPS);
-
-	wps_ie = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-					 WLAN_OUI_TYPE_MICROSOFT_WPS,
-					 pos, left);
-
-	if (!wpa_ie) {
-		if (wps_ie) {
-			DBG_8723A("STA included WPS IE in (Re)Association "
-				  "Request - assume WPS is used\n");
-			pstat->flags |= WLAN_STA_WPS;
-		} else {
-			DBG_8723A("STA did not include WPA/RSN IE in (Re)"
-				   "Association Request - possible WPS use\n");
-			pstat->flags |= WLAN_STA_MAYBE_WPS;
-		}
-	} else {
-		int copy_len;
-
-		if (psecuritypriv->wpa_psk == 0) {
-			DBG_8723A("STA %pM: WPA/RSN IE in association request, but AP don't support WPA/RSN\n",
-				  pstat->hwaddr);
-
-			status = WLAN_STATUS_INVALID_IE;
-
-			goto OnAssocReq23aFail;
-		}
-
-		if (wps_ie) {
-			DBG_8723A("STA included WPS IE in (Re)Association "
-				  "Request - WPS is used\n");
-			pstat->flags |= WLAN_STA_WPS;
-			copy_len = 0;
-		} else {
-			copy_len = ((wpa_ie_len + 2) > sizeof(pstat->wpa_ie)) ?
-				sizeof(pstat->wpa_ie) : (wpa_ie_len + 2);
-		}
-
-		if (copy_len > 0)
-			memcpy(pstat->wpa_ie, wpa_ie - 2, copy_len);
-	}
-
-	/*  check if there is WMM IE & support WWM-PS */
-	pstat->flags &= ~WLAN_STA_WME;
-	pstat->qos_option = 0;
-	pstat->qos_info = 0;
-	pstat->has_legacy_ac = true;
-	pstat->uapsd_vo = 0;
-	pstat->uapsd_vi = 0;
-	pstat->uapsd_be = 0;
-	pstat->uapsd_bk = 0;
-	if (pmlmepriv->qos_option) {
-		const u8 *end = pos + left;
-
-		p = pos;
-
-		for (;;) {
-			left = end - p;
-			p = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-						    WLAN_OUI_TYPE_MICROSOFT_WMM,
-						    p, left);
-			if (p) {
-				pstat->flags |= WLAN_STA_WME;
-
-				pstat->qos_option = 1;
-				pstat->qos_info = *(p + 8);
-
-				pstat->max_sp_len =
-					(pstat->qos_info >> 5) & 0x3;
-
-				if ((pstat->qos_info & 0xf) != 0xf)
-					pstat->has_legacy_ac = true;
-				else
-					pstat->has_legacy_ac = false;
-
-				if (pstat->qos_info & 0xf) {
-					if (pstat->qos_info & BIT(0))
-						pstat->uapsd_vo = BIT(0)|BIT(1);
-					else
-						pstat->uapsd_vo = 0;
-
-					if (pstat->qos_info & BIT(1))
-						pstat->uapsd_vi = BIT(0)|BIT(1);
-					else
-						pstat->uapsd_vi = 0;
-
-					if (pstat->qos_info & BIT(2))
-						pstat->uapsd_bk = BIT(0)|BIT(1);
-					else
-						pstat->uapsd_bk = 0;
-
-					if (pstat->qos_info & BIT(3))
-						pstat->uapsd_be = BIT(0)|BIT(1);
-					else
-						pstat->uapsd_be = 0;
-
-					break;
-				}
-			} else {
-				break;
-			}
-			p = p + p[1] + 2;
-		}
-	}
-
-	/* save HT capabilities in the sta object */
-	memset(&pstat->htpriv.ht_cap, 0, sizeof(struct ieee80211_ht_cap));
-	p = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY, pos, left);
-
-	if (p && p[1] >= sizeof(struct ieee80211_ht_cap)) {
-		pstat->flags |= WLAN_STA_HT;
-
-		pstat->flags |= WLAN_STA_WME;
-
-		memcpy(&pstat->htpriv.ht_cap, p + 2,
-		       sizeof(struct ieee80211_ht_cap));
-	} else
-		pstat->flags &= ~WLAN_STA_HT;
-
-	if (!pmlmepriv->htpriv.ht_option && pstat->flags & WLAN_STA_HT){
-		status = WLAN_STATUS_UNSPECIFIED_FAILURE;
-		goto OnAssocReq23aFail;
-	}
-
-	if (pstat->flags & WLAN_STA_HT &&
-	    (pstat->wpa2_pairwise_cipher & WPA_CIPHER_TKIP ||
-	     pstat->wpa_pairwise_cipher & WPA_CIPHER_TKIP)) {
-		DBG_8723A("HT: %pM tried to use TKIP with HT association\n",
-			  pstat->hwaddr);
-
-		/* status = WLAN_STATUS_CIPHER_REJECTED_PER_POLICY; */
-		/* goto OnAssocReq23aFail; */
-	}
-
-	pstat->flags |= WLAN_STA_NONERP;
-	for (i = 0; i < pstat->bssratelen; i++) {
-		if ((pstat->bssrateset[i] & 0x7f) > 22) {
-			pstat->flags &= ~WLAN_STA_NONERP;
-			break;
-		}
-	}
-
-	if (pstat->capability & WLAN_CAPABILITY_SHORT_PREAMBLE)
-		pstat->flags |= WLAN_STA_SHORT_PREAMBLE;
-	else
-		pstat->flags &= ~WLAN_STA_SHORT_PREAMBLE;
-
-	if (status != WLAN_STATUS_SUCCESS)
-		goto OnAssocReq23aFail;
-
-	/* TODO: identify_proprietary_vendor_ie(); */
-	/*  Realtek proprietary IE */
-	/*  identify if this is Broadcom sta */
-	/*  identify if this is ralink sta */
-	/*  Customer proprietary IE */
-
-	/* get a unique AID */
-	if (pstat->aid > 0) {
-		DBG_8723A("  old AID %d\n", pstat->aid);
-	} else {
-		for (pstat->aid = 1; pstat->aid <= NUM_STA; pstat->aid++)
-			if (pstapriv->sta_aid[pstat->aid - 1] == NULL)
-				break;
-
-		if (pstat->aid > NUM_STA)
-			pstat->aid = NUM_STA;
-		if (pstat->aid > pstapriv->max_num_sta) {
-
-			pstat->aid = 0;
-
-			DBG_8723A("  no room for more AIDs\n");
-
-			status = WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA;
-
-			goto OnAssocReq23aFail;
-		} else {
-			pstapriv->sta_aid[pstat->aid - 1] = pstat;
-			DBG_8723A("allocate new AID = (%d)\n", pstat->aid);
-		}
-	}
-
-	pstat->state &= ~WIFI_FW_ASSOC_STATE;
-	pstat->state |= WIFI_FW_ASSOC_SUCCESS;
-
-	spin_lock_bh(&pstapriv->auth_list_lock);
-	if (!list_empty(&pstat->auth_list)) {
-		list_del_init(&pstat->auth_list);
-		pstapriv->auth_list_cnt--;
-	}
-	spin_unlock_bh(&pstapriv->auth_list_lock);
-
-	spin_lock_bh(&pstapriv->asoc_list_lock);
-	if (list_empty(&pstat->asoc_list)) {
-		pstat->expire_to = pstapriv->expire_to;
-		list_add_tail(&pstat->asoc_list, &pstapriv->asoc_list);
-		pstapriv->asoc_list_cnt++;
-	}
-	spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-	/*  now the station is qualified to join our BSS... */
-	if (pstat->state & WIFI_FW_ASSOC_SUCCESS &&
-	    status == WLAN_STATUS_SUCCESS) {
-		/* 1 bss_cap_update & sta_info_update23a */
-		bss_cap_update_on_sta_join23a(padapter, pstat);
-		sta_info_update23a(padapter, pstat);
-
-		/* issue assoc rsp before notify station join event. */
-		if (ieee80211_is_assoc_req(mgmt->frame_control))
-			issue_assocrsp(padapter, status, pstat,
-				       IEEE80211_STYPE_ASSOC_RESP);
-		else
-			issue_assocrsp(padapter, status, pstat,
-				       IEEE80211_STYPE_REASSOC_RESP);
-
-		/* 2 - report to upper layer */
-		DBG_8723A("indicate_sta_join_event to upper layer - hostapd\n");
-		rtw_cfg80211_indicate_sta_assoc(padapter, pframe, pkt_len);
-
-		/* 3-(1) report sta add event */
-		report_add_sta_event23a(padapter, pstat->hwaddr, pstat->aid);
-	}
-
-	return _SUCCESS;
-
-asoc_class2_error:
-
-	issue_deauth23a(padapter, mgmt->sa, status);
-	return _FAIL;
-
-OnAssocReq23aFail:
-
-	pstat->aid = 0;
-	if (ieee80211_is_assoc_req(mgmt->frame_control))
-		issue_assocrsp(padapter, status, pstat,
-			       IEEE80211_STYPE_ASSOC_RESP);
-	else
-		issue_assocrsp(padapter, status, pstat,
-			       IEEE80211_STYPE_REASSOC_RESP);
-
-#endif /* CONFIG_8723AU_AP_MODE */
-
-	return _FAIL;
-}
-
-static int
-OnAssocRsp23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *pmgmt = (struct ieee80211_mgmt *) skb->data;
-	int res;
-	unsigned short status;
-	const u8 *p, *pie;
-	u8 *pframe = skb->data;
-	int pkt_len = skb->len;
-	int pielen;
-
-	DBG_8723A("%s\n", __func__);
-
-	/* check A1 matches or not */
-	if (!ether_addr_equal(myid(&padapter->eeprompriv), pmgmt->da))
-		return _SUCCESS;
-
-	if (!(pmlmeinfo->state & (WIFI_FW_AUTH_SUCCESS | WIFI_FW_ASSOC_STATE)))
-		return _SUCCESS;
-
-	if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS)
-		return _SUCCESS;
-
-	del_timer_sync(&pmlmeext->link_timer);
-
-	/* status */
-	status = le16_to_cpu(pmgmt->u.assoc_resp.status_code);
-	if (status > 0)	{
-		DBG_8723A("assoc reject, status code: %d\n", status);
-		pmlmeinfo->state = MSR_NOLINK;
-		res = -4;
-		goto report_assoc_result;
-	}
-
-	/* get capabilities */
-	pmlmeinfo->capability = le16_to_cpu(pmgmt->u.assoc_resp.capab_info);
-
-	/* set slot time */
-	pmlmeinfo->slotTime = (pmlmeinfo->capability & BIT(10))? 9: 20;
-
-	/* AID */
-	res = pmlmeinfo->aid = le16_to_cpu(pmgmt->u.assoc_resp.aid) & 0x3fff;
-
-	pie = pframe + offsetof(struct ieee80211_mgmt, u.assoc_resp.variable);
-	pielen = pkt_len -
-		offsetof(struct ieee80211_mgmt, u.assoc_resp.variable);
-
-	p = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY,
-			     pmgmt->u.assoc_resp.variable, pielen);
-	if (p && p[1])
-		HT_caps_handler23a(padapter, p);
-
-	p = cfg80211_find_ie(WLAN_EID_HT_OPERATION,
-			     pmgmt->u.assoc_resp.variable, pielen);
-	if (p && p[1])
-		HT_info_handler23a(padapter, p);
-
-	p = cfg80211_find_ie(WLAN_EID_ERP_INFO,
-			     pmgmt->u.assoc_resp.variable, pielen);
-	if (p && p[1])
-		ERP_IE_handler23a(padapter, p);
-
-	pie = pframe + offsetof(struct ieee80211_mgmt, u.assoc_resp.variable);
-	while (true) {
-		p = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-					    WLAN_OUI_TYPE_MICROSOFT_WMM,
-					    pie, pframe + pkt_len - pie);
-		if (!p)
-			break;
-
-		pie = p + p[1] + 2;
-		/* if this IE is too short, try the next */
-		if (p[1] <= 4)
-			continue;
-		/* if this IE is WMM params, we found what we wanted */
-		if (p[6] == 1)
-			break;
-	}
-
-	if (p && p[1])
-		WMM_param_handler23a(padapter, p);
-
-	pmlmeinfo->state &= ~WIFI_FW_ASSOC_STATE;
-	pmlmeinfo->state |= WIFI_FW_ASSOC_SUCCESS;
-
-	/* Update Basic Rate Table for spec, 2010-12-28 , by thomas */
-	UpdateBrateTbl23a(padapter, pmlmeinfo->network.SupportedRates);
-
-report_assoc_result:
-	pmlmepriv->assoc_rsp_len = 0;
-	if (res > 0) {
-		kfree(pmlmepriv->assoc_rsp);
-		pmlmepriv->assoc_rsp = kmalloc(pkt_len, GFP_ATOMIC);
-		if (pmlmepriv->assoc_rsp) {
-			memcpy(pmlmepriv->assoc_rsp, pframe, pkt_len);
-			pmlmepriv->assoc_rsp_len = pkt_len;
-		}
-	} else
-		kfree(pmlmepriv->assoc_rsp);
-
-	report_join_res23a(padapter, res);
-
-	return _SUCCESS;
-}
-
-static int
-OnDeAuth23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	unsigned short reason;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-
-	if (!ether_addr_equal(mgmt->bssid,
-			      get_my_bssid23a(&pmlmeinfo->network)))
-		return _SUCCESS;
-
-	reason = le16_to_cpu(mgmt->u.deauth.reason_code);
-
-	DBG_8723A("%s Reason code(%d)\n", __func__, reason);
-
-#ifdef CONFIG_8723AU_AP_MODE
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		struct sta_info *psta;
-		struct sta_priv *pstapriv = &padapter->stapriv;
-
-		DBG_8723A_LEVEL(_drv_always_, "ap recv deauth reason code(%d) "
-				"sta:%pM\n", reason, mgmt->sa);
-
-		psta = rtw_get_stainfo23a(pstapriv, mgmt->sa);
-		if (psta) {
-			u8 updated = 0;
-
-			spin_lock_bh(&pstapriv->asoc_list_lock);
-			if (!list_empty(&psta->asoc_list)) {
-				list_del_init(&psta->asoc_list);
-				pstapriv->asoc_list_cnt--;
-				updated = ap_free_sta23a(padapter, psta,
-						      false, reason);
-			}
-			spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-			associated_clients_update23a(padapter, updated);
-		}
-
-		return _SUCCESS;
-	} else
-#endif
-	{
-		DBG_8723A_LEVEL(_drv_always_, "sta recv deauth reason code(%d) "
-				"sta:%pM\n", reason, mgmt->bssid);
-
-		receive_disconnect23a(padapter, mgmt->bssid, reason);
-	}
-	pmlmepriv->LinkDetectInfo.bBusyTraffic = false;
-
-	return _SUCCESS;
-}
-
-static int
-OnDisassoc23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	unsigned short reason;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-
-	if (!ether_addr_equal(mgmt->bssid,
-			      get_my_bssid23a(&pmlmeinfo->network)))
-		return _SUCCESS;
-
-	reason = le16_to_cpu(mgmt->u.disassoc.reason_code);
-
-	DBG_8723A("%s Reason code(%d)\n", __func__, reason);
-
-#ifdef CONFIG_8723AU_AP_MODE
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		struct sta_info *psta;
-		struct sta_priv *pstapriv = &padapter->stapriv;
-
-		DBG_8723A_LEVEL(_drv_always_, "ap recv disassoc reason code(%d)"
-				" sta:%pM\n", reason, mgmt->sa);
-
-		psta = rtw_get_stainfo23a(pstapriv, mgmt->sa);
-		if (psta) {
-			u8 updated = 0;
-
-			spin_lock_bh(&pstapriv->asoc_list_lock);
-			if (!list_empty(&psta->asoc_list)) {
-				list_del_init(&psta->asoc_list);
-				pstapriv->asoc_list_cnt--;
-				updated = ap_free_sta23a(padapter, psta,
-							 false, reason);
-			}
-			spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-			associated_clients_update23a(padapter, updated);
-		}
-
-		return _SUCCESS;
-	} else
-#endif
-	{
-		DBG_8723A_LEVEL(_drv_always_, "ap recv disassoc reason "
-				"code(%d) sta:%pM\n", reason, mgmt->bssid);
-
-		receive_disconnect23a(padapter, mgmt->bssid, reason);
-	}
-	pmlmepriv->LinkDetectInfo.bBusyTraffic = false;
-	return _SUCCESS;
-}
-
-static int
-OnAtim23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	DBG_8723A("%s\n", __func__);
-	return _SUCCESS;
-}
-
-static int
-on_action_spct23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	return _FAIL;
-}
-
-static int
-OnAction23a_qos(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	return _SUCCESS;
-}
-
-static int
-OnAction23a_dls(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	return _SUCCESS;
-}
-
-static int OnAction23a_back23a(struct rtw_adapter *padapter,
-			       struct recv_frame *precv_frame)
-{
-	u8 *addr;
-	struct sta_info *psta = NULL;
-	struct recv_reorder_ctrl *preorder_ctrl;
-	unsigned char category, action;
-	unsigned short tid, status, capab, params, reason_code = 0;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-
-	/* check RA matches or not */
-	if (!ether_addr_equal(myid(&padapter->eeprompriv), mgmt->da))
-		return _SUCCESS;
-
-	DBG_8723A("%s\n", __func__);
-
-	if ((pmlmeinfo->state&0x03) != MSR_AP)
-		if (!(pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS))
-			return _SUCCESS;
-
-	addr = mgmt->sa;
-	psta = rtw_get_stainfo23a(pstapriv, addr);
-
-	if (!psta)
-		return _SUCCESS;
-
-	category = mgmt->u.action.category;
-	if (category == WLAN_CATEGORY_BACK) { /*  representing Block Ack */
-		if (!pmlmeinfo->HT_enable)
-			return _SUCCESS;
-		/* action_code is located in the same place for all
-		   action events, so pick any */
-		action = mgmt->u.action.u.wme_action.action_code;
-		DBG_8723A("%s, action =%d\n", __func__, action);
-		switch (action) {
-		case WLAN_ACTION_ADDBA_REQ: /* ADDBA request */
-			memcpy(&pmlmeinfo->ADDBA_req,
-			       &mgmt->u.action.u.addba_req.dialog_token,
-			       sizeof(struct ADDBA_request));
-			process_addba_req23a(padapter,
-					     (u8 *)&pmlmeinfo->ADDBA_req, addr);
-			if (pmlmeinfo->bAcceptAddbaReq == true)
-				issue_action_BA23a(padapter, addr,
-						   WLAN_ACTION_ADDBA_RESP, 0);
-			else {
-				/* reject ADDBA Req */
-				issue_action_BA23a(padapter, addr,
-						   WLAN_ACTION_ADDBA_RESP, 37);
-			}
-			break;
-		case WLAN_ACTION_ADDBA_RESP: /* ADDBA response */
-			status = get_unaligned_le16(
-				&mgmt->u.action.u.addba_resp.status);
-			capab = get_unaligned_le16(
-				&mgmt->u.action.u.addba_resp.capab);
-			tid = (capab & IEEE80211_ADDBA_PARAM_TID_MASK) >> 2;
-			if (status == 0) {	/* successful */
-				DBG_8723A("agg_enable for TID =%d\n", tid);
-				psta->htpriv.agg_enable_bitmap |= BIT(tid);
-				psta->htpriv.candidate_tid_bitmap &= ~BIT(tid);
-			} else
-				psta->htpriv.agg_enable_bitmap &= ~BIT(tid);
-			break;
-
-		case WLAN_ACTION_DELBA: /* DELBA */
-			params = get_unaligned_le16(
-				&mgmt->u.action.u.delba.params);
-			tid = params >> 12;
-
-			if (params & IEEE80211_DELBA_PARAM_INITIATOR_MASK) {
-				preorder_ctrl = &psta->recvreorder_ctrl[tid];
-				preorder_ctrl->enable = false;
-				preorder_ctrl->indicate_seq = 0xffff;
-			} else {
-				psta->htpriv.agg_enable_bitmap &= ~BIT(tid);
-				psta->htpriv.candidate_tid_bitmap &= ~BIT(tid);
-			}
-			reason_code = get_unaligned_le16(
-				&mgmt->u.action.u.delba.reason_code);
-			/* todo: how to notify the host while receiving
-			   DELETE BA */
-			break;
-		default:
-			break;
-		}
-	}
-	return _SUCCESS;
-}
-
-static int on_action_public23a(struct rtw_adapter *padapter,
-			       struct recv_frame *precv_frame)
-{
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	u8 *pframe = skb->data;
-	int freq, channel;
-
-	/* check RA matches or not */
-	if (!ether_addr_equal(myid(&padapter->eeprompriv), hdr->addr1))
-		return _FAIL;
-
-	channel = rtw_get_oper_ch23a(padapter);
-
-	if (channel <= RTW_CH_MAX_2G_CHANNEL)
-		freq = ieee80211_channel_to_frequency(channel,
-						      NL80211_BAND_2GHZ);
-	else
-		freq = ieee80211_channel_to_frequency(channel,
-						      NL80211_BAND_5GHZ);
-
-	if (cfg80211_rx_mgmt(padapter->rtw_wdev, freq, 0, pframe,
-			     skb->len, 0))
-		return _SUCCESS;
-
-	return _FAIL;
-}
-
-static int
-OnAction23a_ht(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	return _SUCCESS;
-}
-
-static int
-OnAction23a_wmm(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	return _SUCCESS;
-}
-
-static int
-OnAction23a_p2p(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	return _SUCCESS;
-}
-
-static int
-OnAction23a(struct rtw_adapter *padapter, struct recv_frame *precv_frame)
-{
-	int i;
-	u8 category;
-	struct action_handler *ptable;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-
-	category = mgmt->u.action.category;
-
-	for (i = 0; i < ARRAY_SIZE(OnAction23a_tbl); i++) {
-		ptable = &OnAction23a_tbl[i];
-
-		if (category == ptable->num)
-			ptable->func(padapter, precv_frame);
-	}
-
-	return _SUCCESS;
-}
-
-static int DoReserved23a(struct rtw_adapter *padapter,
-			 struct recv_frame *precv_frame)
-{
-	return _SUCCESS;
-}
-
-struct xmit_frame *alloc_mgtxmitframe23a(struct xmit_priv *pxmitpriv)
-{
-	struct xmit_frame *pmgntframe;
-	struct xmit_buf *pxmitbuf;
-
-	pmgntframe = rtw_alloc_xmitframe23a_ext(pxmitpriv);
-
-	if (!pmgntframe) {
-		DBG_8723A("%s(%s): alloc xmitframe fail\n", __func__,
-			  pxmitpriv->adapter->pnetdev->name);
-		goto exit;
-	}
-
-	pxmitbuf = rtw_alloc_xmitbuf23a_ext(pxmitpriv);
-	if (!pxmitbuf) {
-		DBG_8723A("%s(%s): alloc xmitbuf fail\n", __func__,
-			  pxmitpriv->adapter->pnetdev->name);
-		rtw_free_xmitframe23a(pxmitpriv, pmgntframe);
-		pmgntframe = NULL;
-		goto exit;
-	}
-
-	pmgntframe->frame_tag = MGNT_FRAMETAG;
-	pmgntframe->pxmitbuf = pxmitbuf;
-	pmgntframe->buf_addr = pxmitbuf->pbuf;
-	pxmitbuf->priv_data = pmgntframe;
-
-exit:
-	return pmgntframe;
-}
-
-/****************************************************************************
-
-Following are some TX functions for WiFi MLME
-
-*****************************************************************************/
-
-void update_mgnt_tx_rate23a(struct rtw_adapter *padapter, u8 rate)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	pmlmeext->tx_rate = rate;
-	DBG_8723A("%s(): rate = %x\n", __func__, rate);
-}
-
-void update_mgntframe_attrib23a(struct rtw_adapter *padapter,
-				struct pkt_attrib *pattrib)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	memset((u8 *)pattrib, 0, sizeof(struct pkt_attrib));
-
-	pattrib->hdrlen = 24;
-	pattrib->nr_frags = 1;
-	pattrib->priority = 7;
-	pattrib->mac_id = 0;
-	pattrib->qsel = 0x12;
-
-	pattrib->pktlen = 0;
-
-	if (pmlmeext->cur_wireless_mode & WIRELESS_11B)
-		pattrib->raid = 6;/* b mode */
-	else
-		pattrib->raid = 5;/* a/g mode */
-
-	pattrib->encrypt = 0;
-	pattrib->bswenc = false;
-
-	pattrib->qos_en = false;
-	pattrib->ht_en = false;
-	pattrib->bwmode = HT_CHANNEL_WIDTH_20;
-	pattrib->ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-	pattrib->sgi = false;
-
-	pattrib->seqnum = pmlmeext->mgnt_seq;
-
-	pattrib->retry_ctrl = true;
-}
-
-void dump_mgntframe23a(struct rtw_adapter *padapter,
-		       struct xmit_frame *pmgntframe)
-{
-	if (padapter->bSurpriseRemoved == true ||
-	    padapter->bDriverStopped == true)
-		return;
-
-	rtl8723au_mgnt_xmit(padapter, pmgntframe);
-}
-
-int dump_mgntframe23a_and_wait(struct rtw_adapter *padapter,
-			       struct xmit_frame *pmgntframe, int timeout_ms)
-{
-	int ret = _FAIL;
-	unsigned long irqL;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct xmit_buf *pxmitbuf = pmgntframe->pxmitbuf;
-	struct submit_ctx sctx;
-
-	if (padapter->bSurpriseRemoved == true ||
-	    padapter->bDriverStopped == true)
-		return ret;
-
-	rtw_sctx_init23a(&sctx, timeout_ms);
-	pxmitbuf->sctx = &sctx;
-
-	ret = rtl8723au_mgnt_xmit(padapter, pmgntframe);
-
-	if (ret == _SUCCESS)
-		ret = rtw_sctx_wait23a(&sctx);
-
-	spin_lock_irqsave(&pxmitpriv->lock_sctx, irqL);
-	pxmitbuf->sctx = NULL;
-	spin_unlock_irqrestore(&pxmitpriv->lock_sctx, irqL);
-
-	return ret;
-}
-
-int dump_mgntframe23a_and_wait_ack23a(struct rtw_adapter *padapter,
-				      struct xmit_frame *pmgntframe)
-{
-	int ret = _FAIL;
-	u32 timeout_ms = 500;/*   500ms */
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	if (padapter->bSurpriseRemoved == true ||
-	    padapter->bDriverStopped == true)
-		return _FAIL;
-
-	mutex_lock(&pxmitpriv->ack_tx_mutex);
-	pxmitpriv->ack_tx = true;
-
-	pmgntframe->ack_report = 1;
-	if (rtl8723au_mgnt_xmit(padapter, pmgntframe) == _SUCCESS)
-		ret = rtw_ack_tx_wait23a(pxmitpriv, timeout_ms);
-
-	pxmitpriv->ack_tx = false;
-	mutex_unlock(&pxmitpriv->ack_tx_mutex);
-
-	return ret;
-}
-
-static int update_hidden_ssid(u8 *ies, u32 ies_len, u8 hidden_ssid_mode)
-{
-	u8 *ssid_ie;
-	int ssid_len_ori;
-	int len_diff = 0;
-	u8 *next_ie;
-	u32 remain_len;
-
-	ssid_ie = rtw_get_ie23a(ies,  WLAN_EID_SSID, &ssid_len_ori, ies_len);
-
-	/* DBG_8723A("%s hidden_ssid_mode:%u, ssid_ie:%p, ssid_len_ori:%d\n",
-	   __func__, hidden_ssid_mode, ssid_ie, ssid_len_ori); */
-
-	if (ssid_ie && ssid_len_ori > 0) {
-		switch (hidden_ssid_mode) {
-		case 1:
-			next_ie = ssid_ie + 2 + ssid_len_ori;
-			remain_len = ies_len -(next_ie-ies);
-
-			ssid_ie[1] = 0;
-			memcpy(ssid_ie+2, next_ie, remain_len);
-			len_diff -= ssid_len_ori;
-
-			break;
-		case 2:
-			memset(&ssid_ie[2], 0, ssid_len_ori);
-			break;
-		default:
-			break;
-		}
-	}
-
-	return len_diff;
-}
-
-void issue_beacon23a(struct rtw_adapter *padapter, int timeout_ms)
-{
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	struct ieee80211_mgmt *mgmt;
-	unsigned int rate_len;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-	u8 bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-	const u8 *wps_ie;
-	u8 sr = 0;
-	int len_diff;
-
-	/* DBG_8723A("%s\n", __func__); */
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe) {
-		DBG_8723A("%s, alloc mgnt frame fail\n", __func__);
-		return;
-	}
-#ifdef CONFIG_8723AU_AP_MODE
-	spin_lock_bh(&pmlmepriv->bcn_update_lock);
-#endif
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-	pattrib->qsel = 0x10;
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	mgmt = (struct ieee80211_mgmt *)pframe;
-
-	mgmt->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_BEACON);
-	mgmt->seq_ctrl = 0;
-
-	ether_addr_copy(mgmt->da, bc_addr);
-	ether_addr_copy(mgmt->sa, myid(&padapter->eeprompriv));
-	ether_addr_copy(mgmt->bssid, get_my_bssid23a(cur_network));
-
-	/* timestamp will be inserted by hardware */
-
-	put_unaligned_le16(cur_network->beacon_interval,
-			   &mgmt->u.beacon.beacon_int);
-
-	put_unaligned_le16(cur_network->capability,
-			   &mgmt->u.beacon.capab_info);
-
-	pframe = mgmt->u.beacon.variable;
-	pattrib->pktlen = offsetof(struct ieee80211_mgmt, u.beacon.variable);
-
-	if ((pmlmeinfo->state & 0x03) == MSR_AP) {
-		u8 *iebuf;
-		int buflen;
-		/* DBG_8723A("ie len =%d\n", cur_network->IELength); */
-		memcpy(pframe, cur_network->IEs, cur_network->IELength);
-		len_diff = update_hidden_ssid(pframe, cur_network->IELength,
-					      pmlmeinfo->hidden_ssid_mode);
-		pframe += (cur_network->IELength+len_diff);
-		pattrib->pktlen += (cur_network->IELength+len_diff);
-
-		iebuf = mgmt->u.beacon.variable;
-		buflen = pattrib->pktlen -
-			offsetof(struct ieee80211_mgmt, u.beacon.variable);
-		wps_ie = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-						 WLAN_OUI_TYPE_MICROSOFT_WPS,
-						 iebuf, buflen);
-
-		if (wps_ie && wps_ie[1] > 0) {
-			rtw_get_wps_attr_content23a(wps_ie, wps_ie[1],
-						    WPS_ATTR_SELECTED_REGISTRAR,
-						    (u8 *)&sr);
-		}
-		if (sr != 0)
-			set_fwstate(pmlmepriv, WIFI_UNDER_WPS);
-		else
-			_clr_fwstate_(pmlmepriv, WIFI_UNDER_WPS);
-
-		goto _issue_bcn;
-	}
-
-	/*  SSID */
-	pframe = rtw_set_ie23a(pframe, WLAN_EID_SSID,
-			       cur_network->Ssid.ssid_len,
-			       cur_network->Ssid.ssid, &pattrib->pktlen);
-
-	/*  supported rates... */
-	rate_len = rtw_get_rateset_len23a(cur_network->SupportedRates);
-	pframe = rtw_set_ie23a(pframe, WLAN_EID_SUPP_RATES,
-			       ((rate_len > 8)? 8: rate_len),
-			       cur_network->SupportedRates, &pattrib->pktlen);
-
-	/*  DS parameter set */
-	pframe = rtw_set_ie23a(pframe, WLAN_EID_DS_PARAMS, 1, (unsigned char *)
-			       &cur_network->DSConfig, &pattrib->pktlen);
-
-	/* if ((pmlmeinfo->state&0x03) == MSR_ADHOC) */
-	{
-		u8 erpinfo = 0;
-		u32 ATIMWindow;
-		/*  IBSS Parameter Set... */
-		/* ATIMWindow = cur->ATIMWindow; */
-		ATIMWindow = 0;
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_IBSS_PARAMS, 2,
-				       (unsigned char *)&ATIMWindow,
-				       &pattrib->pktlen);
-
-		/* ERP IE */
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_ERP_INFO, 1,
-				       &erpinfo, &pattrib->pktlen);
-	}
-
-	/*  EXTERNDED SUPPORTED RATE */
-	if (rate_len > 8)
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_EXT_SUPP_RATES,
-				       rate_len - 8,
-				       cur_network->SupportedRates + 8,
-				       &pattrib->pktlen);
-
-	/* todo:HT for adhoc */
-
-_issue_bcn:
-
-#ifdef CONFIG_8723AU_AP_MODE
-	pmlmepriv->update_bcn = false;
-
-	spin_unlock_bh(&pmlmepriv->bcn_update_lock);
-#endif
-
-	if ((pattrib->pktlen + TXDESC_SIZE) > 512) {
-		DBG_8723A("beacon frame too large\n");
-		return;
-	}
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	/* DBG_8723A("issue bcn_sz =%d\n", pattrib->last_txcmdsz); */
-	if (timeout_ms > 0)
-		dump_mgntframe23a_and_wait(padapter, pmgntframe, timeout_ms);
-	else
-		dump_mgntframe23a(padapter, pmgntframe);
-}
-
-static void issue_probersp(struct rtw_adapter *padapter, unsigned char *da)
-{
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	struct ieee80211_mgmt *mgmt;
-	unsigned char *mac, *bssid;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-#ifdef CONFIG_8723AU_AP_MODE
-	const u8 *pwps_ie;
-	u8 *ssid_ie;
-	int ssid_ielen;
-	int ssid_ielen_diff;
-	u8 buf[MAX_IE_SZ];
-#endif
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-	unsigned int rate_len;
-
-	/* DBG_8723A("%s\n", __func__); */
-
-	if (cur_network->IELength > MAX_IE_SZ)
-		return;
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe) {
-		DBG_8723A("%s, alloc mgnt frame fail\n", __func__);
-		return;
-	}
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *)pmgntframe->buf_addr + TXDESC_OFFSET;
-	mgmt = (struct ieee80211_mgmt *)pframe;
-
-	mac = myid(&padapter->eeprompriv);
-	bssid = cur_network->MacAddress;
-
-	mgmt->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_PROBE_RESP);
-
-	ether_addr_copy(mgmt->da, da);
-	ether_addr_copy(mgmt->sa, mac);
-	ether_addr_copy(mgmt->bssid, bssid);
-
-	mgmt->seq_ctrl = cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-	pmlmeext->mgnt_seq++;
-
-	pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr);
-
-	/* timestamp will be inserted by hardware */
-	put_unaligned_le16(cur_network->beacon_interval,
-			   &mgmt->u.probe_resp.beacon_int);
-
-	put_unaligned_le16(cur_network->capability,
-			   &mgmt->u.probe_resp.capab_info);
-
-	pframe = mgmt->u.probe_resp.variable;
-	pattrib->pktlen =
-		offsetof(struct ieee80211_mgmt, u.probe_resp.variable);
-
-	/* below for ad-hoc mode */
-
-#ifdef CONFIG_8723AU_AP_MODE
-	if ((pmlmeinfo->state & 0x03) == MSR_AP) {
-		pwps_ie = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-						  WLAN_OUI_TYPE_MICROSOFT_WPS,
-						  cur_network->IEs,
-						  cur_network->IELength);
-
-		memcpy(pframe, cur_network->IEs, cur_network->IELength);
-		pframe += cur_network->IELength;
-		pattrib->pktlen += cur_network->IELength;
-
-		/* retrieve SSID IE from cur_network->Ssid */
-
-		ssid_ie = rtw_get_ie23a(mgmt->u.probe_resp.variable,
-					WLAN_EID_SSID, &ssid_ielen,
-					pframe - mgmt->u.probe_resp.variable);
-
-		ssid_ielen_diff = cur_network->Ssid.ssid_len - ssid_ielen;
-
-		if (ssid_ie && cur_network->Ssid.ssid_len) {
-			uint remainder_ielen;
-			u8 *remainder_ie;
-
-			remainder_ie = ssid_ie + 2;
-
-			remainder_ielen = pframe - remainder_ie;
-
-			DBG_8723A_LEVEL(_drv_warning_, "%s(%s): "
-					"remainder_ielen > MAX_IE_SZ\n",
-					__func__, padapter->pnetdev->name);
-			if (remainder_ielen > MAX_IE_SZ)
-				remainder_ielen = MAX_IE_SZ;
-
-			memcpy(buf, remainder_ie, remainder_ielen);
-			memcpy(remainder_ie + ssid_ielen_diff, buf,
-			       remainder_ielen);
-			*(ssid_ie + 1) = cur_network->Ssid.ssid_len;
-			memcpy(ssid_ie + 2, cur_network->Ssid.ssid,
-			       cur_network->Ssid.ssid_len);
-
-			pframe += ssid_ielen_diff;
-			pattrib->pktlen += ssid_ielen_diff;
-		}
-	} else
-#endif
-	{
-		/*  SSID */
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SSID,
-				       cur_network->Ssid.ssid_len,
-				       cur_network->Ssid.ssid,
-				       &pattrib->pktlen);
-
-		/*  supported rates... */
-		rate_len = rtw_get_rateset_len23a(cur_network->SupportedRates);
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SUPP_RATES,
-				       ((rate_len > 8)? 8: rate_len),
-				       cur_network->SupportedRates,
-				       &pattrib->pktlen);
-
-		/*  DS parameter set */
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_DS_PARAMS, 1,
-				       (unsigned char *)&cur_network->DSConfig,
-				       &pattrib->pktlen);
-
-		if ((pmlmeinfo->state & 0x03) == MSR_ADHOC) {
-			u8 erpinfo = 0;
-			u32 ATIMWindow;
-			/*  IBSS Parameter Set... */
-			/* ATIMWindow = cur->ATIMWindow; */
-			ATIMWindow = 0;
-			pframe = rtw_set_ie23a(pframe, WLAN_EID_IBSS_PARAMS, 2,
-					       (unsigned char *)&ATIMWindow,
-					       &pattrib->pktlen);
-
-			/* ERP IE */
-			pframe = rtw_set_ie23a(pframe, WLAN_EID_ERP_INFO, 1,
-					       &erpinfo, &pattrib->pktlen);
-		}
-
-		/*  EXTERNDED SUPPORTED RATE */
-		if (rate_len > 8)
-			pframe = rtw_set_ie23a(pframe, WLAN_EID_EXT_SUPP_RATES,
-					       rate_len - 8,
-					       cur_network->SupportedRates + 8,
-					       &pattrib->pktlen);
-
-		/* todo:HT for adhoc */
-	}
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	dump_mgntframe23a(padapter, pmgntframe);
-}
-
-static int _issue_probereq(struct rtw_adapter *padapter,
-			   struct cfg80211_ssid *pssid, u8 *da, int wait_ack)
-{
-	int ret = _FAIL;
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	struct ieee80211_hdr *pwlanhdr;
-	unsigned char *mac;
-	unsigned char bssrate[NumRates];
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	int bssrate_len = 0;
-	u8 bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_notice_,
-		 "+%s\n", __func__);
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe)
-		goto exit;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct ieee80211_hdr *)pframe;
-
-	mac = myid(&padapter->eeprompriv);
-
-	pwlanhdr->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
-					      IEEE80211_STYPE_PROBE_REQ);
-
-	if (da) {
-		/*	unicast probe request frame */
-		ether_addr_copy(pwlanhdr->addr1, da);
-		ether_addr_copy(pwlanhdr->addr3, da);
-	} else {
-		/*	broadcast probe request frame */
-		ether_addr_copy(pwlanhdr->addr1, bc_addr);
-		ether_addr_copy(pwlanhdr->addr3, bc_addr);
-	}
-
-	ether_addr_copy(pwlanhdr->addr2, mac);
-
-	pwlanhdr->seq_ctrl =
-		cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-
-	pmlmeext->mgnt_seq++;
-
-	pframe += sizeof (struct ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof (struct ieee80211_hdr_3addr);
-
-	if (pssid)
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SSID, pssid->ssid_len,
-				       pssid->ssid, &pattrib->pktlen);
-	else
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SSID, 0, NULL,
-				       &pattrib->pktlen);
-
-	get_rate_set23a(padapter, bssrate, &bssrate_len);
-
-	if (bssrate_len > 8) {
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SUPP_RATES, 8,
-				       bssrate, &pattrib->pktlen);
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_EXT_SUPP_RATES,
-				       (bssrate_len - 8), (bssrate + 8),
-				       &pattrib->pktlen);
-	} else {
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SUPP_RATES,
-				       bssrate_len, bssrate, &pattrib->pktlen);
-	}
-
-	/* add wps_ie for wps2.0 */
-	if (pmlmepriv->wps_probe_req_ie_len>0 && pmlmepriv->wps_probe_req_ie) {
-		memcpy(pframe, pmlmepriv->wps_probe_req_ie,
-		       pmlmepriv->wps_probe_req_ie_len);
-		pframe += pmlmepriv->wps_probe_req_ie_len;
-		pattrib->pktlen += pmlmepriv->wps_probe_req_ie_len;
-	}
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_notice_,
-		 "issuing probe_req, tx_len =%d\n", pattrib->last_txcmdsz);
-
-	if (wait_ack) {
-		ret = dump_mgntframe23a_and_wait_ack23a(padapter, pmgntframe);
-	} else {
-		dump_mgntframe23a(padapter, pmgntframe);
-		ret = _SUCCESS;
-	}
-
-exit:
-	return ret;
-}
-
-static inline void issue_probereq(struct rtw_adapter *padapter,
-				  struct cfg80211_ssid *pssid, u8 *da)
-{
-	_issue_probereq(padapter, pssid, da, false);
-}
-
-static int issue_probereq_ex(struct rtw_adapter *padapter,
-			     struct cfg80211_ssid *pssid, u8 *da,
-			     int try_cnt, int wait_ms)
-{
-	int ret;
-	int i = 0;
-	unsigned long start = jiffies;
-
-	do {
-		ret = _issue_probereq(padapter, pssid, da,
-				      wait_ms > 0 ? true : false);
-
-		i++;
-
-		if (padapter->bDriverStopped || padapter->bSurpriseRemoved)
-			break;
-
-		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
-			msleep(wait_ms);
-
-	} while((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
-
-	if (ret != _FAIL) {
-		ret = _SUCCESS;
-		goto exit;
-	}
-
-	if (try_cnt && wait_ms) {
-		if (da)
-			DBG_8723A("%s(%s): to %pM, ch:%u%s, %d/%d in %u ms\n",
-				  __func__, padapter->pnetdev->name,
-				  da, rtw_get_oper_ch23a(padapter),
-				  ret == _SUCCESS ? ", acked" : "", i, try_cnt,
-				  jiffies_to_msecs(jiffies - start));
-		else
-			DBG_8723A("%s(%s):, ch:%u%s, %d/%d in %u ms\n",
-				  __func__, padapter->pnetdev->name,
-				  rtw_get_oper_ch23a(padapter),
-				  ret == _SUCCESS ? ", acked" : "", i, try_cnt,
-				  jiffies_to_msecs(jiffies - start));
-	}
-exit:
-	return ret;
-}
-
-/*  if psta == NULL, indiate we are station(client) now... */
-static void issue_auth(struct rtw_adapter *padapter, struct sta_info *psta,
-		       unsigned short status)
-{
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	struct ieee80211_mgmt *mgmt;
-	unsigned int val32;
-	u16 auth_algo;
-	int use_shared_key = 0;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe)
-		return;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	mgmt = (struct ieee80211_mgmt *)pframe;
-
-	mgmt->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_AUTH);
-	mgmt->seq_ctrl = cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-	pmlmeext->mgnt_seq++;
-
-	pattrib->pktlen = offsetof(struct ieee80211_mgmt, u.auth.variable);
-
-	if (psta) { /*  for AP mode */
-#ifdef CONFIG_8723AU_AP_MODE
-		unsigned short val16;
-
-		ether_addr_copy(mgmt->da, psta->hwaddr);
-		ether_addr_copy(mgmt->sa, myid(&padapter->eeprompriv));
-		ether_addr_copy(mgmt->bssid, myid(&padapter->eeprompriv));
-
-		/*  setting auth algo number */
-		val16 = (u16)psta->authalg;
-
-		if (status != WLAN_STATUS_SUCCESS)
-			val16 = 0;
-
-		if (val16)
-			use_shared_key = 1;
-
-		mgmt->u.auth.auth_alg = cpu_to_le16(val16);
-
-		/*  setting auth seq number */
-		mgmt->u.auth.auth_transaction =
-			cpu_to_le16((u16)psta->auth_seq);
-
-		/*  setting status code... */
-		mgmt->u.auth.status_code = cpu_to_le16(status);
-
-		pframe = mgmt->u.auth.variable;
-		/*  added challenging text... */
-		if ((psta->auth_seq == 2) &&
-		    (psta->state & WIFI_FW_AUTH_STATE) && (use_shared_key == 1))
-			pframe = rtw_set_ie23a(pframe, WLAN_EID_CHALLENGE, 128,
-					       psta->chg_txt, &pattrib->pktlen);
-#endif
-	} else {
-		struct ieee80211_mgmt *iv_mgmt;
-
-		ether_addr_copy(mgmt->da, get_my_bssid23a(&pmlmeinfo->network));
-		ether_addr_copy(mgmt->sa, myid(&padapter->eeprompriv));
-		ether_addr_copy(mgmt->bssid,
-				get_my_bssid23a(&pmlmeinfo->network));
-
-		/*  setting auth algo number */
-		/*  0:OPEN System, 1:Shared key */
-		if (pmlmeinfo->auth_algo == dot11AuthAlgrthm_Shared) {
-			use_shared_key = 1;
-			auth_algo = WLAN_AUTH_SHARED_KEY;
-		} else
-			auth_algo = WLAN_AUTH_OPEN;
-
-		/* DBG_8723A("%s auth_algo = %s auth_seq =%d\n", __func__,
-		   (pmlmeinfo->auth_algo == 0)?"OPEN":"SHARED",
-		   pmlmeinfo->auth_seq); */
-
-		/* setting IV for auth seq #3 */
-		if ((pmlmeinfo->auth_seq == 3) &&
-		    (pmlmeinfo->state & WIFI_FW_AUTH_STATE) &&
-		    (use_shared_key == 1)) {
-			u32 *piv = (u32 *)&mgmt->u.auth;
-
-			iv_mgmt = (struct ieee80211_mgmt *)(pframe + 4);
-			/* DBG_8723A("==> iv(%d), key_index(%d)\n",
-			   pmlmeinfo->iv, pmlmeinfo->key_index); */
-			val32 = (pmlmeinfo->iv & 0x3fffffff) |
-				(pmlmeinfo->key_index << 30);
-			pmlmeinfo->iv++;
-			put_unaligned_le32(val32, piv);
-
-			pattrib->pktlen += 4;
-
-			pattrib->iv_len = IEEE80211_WEP_IV_LEN;
-		} else
-			iv_mgmt = mgmt;
-
-		iv_mgmt->u.auth.auth_alg = cpu_to_le16(auth_algo);
-
-		/*  setting auth seq number */
-		iv_mgmt->u.auth.auth_transaction =
-			cpu_to_le16(pmlmeinfo->auth_seq);
-
-		/*  setting status code... */
-		iv_mgmt->u.auth.status_code = cpu_to_le16(status);
-
-		pframe = iv_mgmt->u.auth.variable;
-
-		/*  then checking to see if sending challenging text... */
-		if ((pmlmeinfo->auth_seq == 3) &&
-		    (pmlmeinfo->state & WIFI_FW_AUTH_STATE) &&
-		    (use_shared_key == 1)) {
-			pframe = rtw_set_ie23a(pframe, WLAN_EID_CHALLENGE, 128,
-					       pmlmeinfo->chg_txt,
-					       &pattrib->pktlen);
-
-			mgmt->frame_control |=
-				cpu_to_le16(IEEE80211_FCTL_PROTECTED);
-
-			pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr);
-
-			pattrib->encrypt = WLAN_CIPHER_SUITE_WEP40;
-
-			pattrib->icv_len = IEEE80211_WEP_ICV_LEN;
-
-			pattrib->pktlen += pattrib->icv_len;
-		}
-	}
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	rtw_wep_encrypt23a(padapter, pmgntframe);
-	DBG_8723A("%s\n", __func__);
-	dump_mgntframe23a(padapter, pmgntframe);
-}
-
-#ifdef CONFIG_8723AU_AP_MODE
-static void issue_assocrsp(struct rtw_adapter *padapter, unsigned short status,
-			   struct sta_info *pstat, u16 pkt_type)
-{
-	struct xmit_frame *pmgntframe;
-	struct ieee80211_mgmt *mgmt;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork = &pmlmeinfo->network;
-	const u8 *p;
-	u8 *ie = pnetwork->IEs;
-
-	DBG_8723A("%s\n", __func__);
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe)
-		return;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	mgmt = (struct ieee80211_mgmt *)pframe;
-
-	mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT | pkt_type);
-
-	ether_addr_copy(mgmt->da, pstat->hwaddr);
-	ether_addr_copy(mgmt->sa, myid(&padapter->eeprompriv));
-	ether_addr_copy(mgmt->bssid, get_my_bssid23a(&pmlmeinfo->network));
-
-	mgmt->seq_ctrl = cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-
-	pmlmeext->mgnt_seq++;
-
-	pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr);
-	pattrib->pktlen =
-		offsetof(struct ieee80211_mgmt, u.assoc_resp.variable);
-
-	mgmt->u.assoc_resp.capab_info = cpu_to_le16(pnetwork->capability);
-	mgmt->u.assoc_resp.status_code = cpu_to_le16(status);
-	mgmt->u.assoc_resp.aid = cpu_to_le16(pstat->aid | BIT(14) | BIT(15));
-
-	pframe = mgmt->u.assoc_resp.variable;
-
-	if (pstat->bssratelen <= 8) {
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SUPP_RATES,
-				       pstat->bssratelen, pstat->bssrateset,
-				       &pattrib->pktlen);
-	} else {
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SUPP_RATES, 8,
-				       pstat->bssrateset, &pattrib->pktlen);
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_EXT_SUPP_RATES,
-				       pstat->bssratelen - 8,
-				       pstat->bssrateset + 8, &pattrib->pktlen);
-	}
-
-	if (pstat->flags & WLAN_STA_HT && pmlmepriv->htpriv.ht_option) {
-		/* FILL HT CAP INFO IE */
-		/* p = hostapd_eid_ht_capabilities_info(hapd, p); */
-		p = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY, ie,
-				     pnetwork->IELength);
-		if (p && p[1]) {
-			memcpy(pframe, p, p[1] + 2);
-			pframe += (p[1] + 2);
-			pattrib->pktlen += (p[1] + 2);
-		}
-
-		/* FILL HT ADD INFO IE */
-		/* p = hostapd_eid_ht_operation(hapd, p); */
-		p = cfg80211_find_ie(WLAN_EID_HT_OPERATION, ie,
-				     pnetwork->IELength);
-		if (p && p[1] > 0) {
-			memcpy(pframe, p, p[1] + 2);
-			pframe += (p[1] + 2);
-			pattrib->pktlen += (p[1] + 2);
-		}
-	}
-
-	/* FILL WMM IE */
-	if (pstat->flags & WLAN_STA_WME && pmlmepriv->qos_option) {
-		unsigned char WMM_PARA_IE[] = {0x00, 0x50, 0xf2, 0x02,
-					       0x01, 0x01};
-		int ie_len = 0;
-
-		for (p = ie; ; p += (ie_len + 2)) {
-			p = cfg80211_find_ie(WLAN_EID_VENDOR_SPECIFIC, p,
-					     pnetwork->IELength - (ie_len + 2));
-			if (p)
-				ie_len = p[1];
-			else
-				ie_len = 0;
-			if (p && !memcmp(p + 2, WMM_PARA_IE, 6)) {
-				memcpy(pframe, p, ie_len + 2);
-				pframe += (ie_len + 2);
-				pattrib->pktlen += (ie_len + 2);
-
-				break;
-			}
-
-			if (!p || ie_len == 0)
-				break;
-		}
-	}
-
-	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_REALTEK) {
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_VENDOR_SPECIFIC, 6,
-				       REALTEK_96B_IE, &pattrib->pktlen);
-	}
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	dump_mgntframe23a(padapter, pmgntframe);
-}
-#endif
-
-static void issue_assocreq(struct rtw_adapter *padapter)
-{
-	int ret = _FAIL;
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	const u8 *p;
-	struct ieee80211_mgmt *mgmt;
-	unsigned int i, j, index = 0;
-	unsigned char rf_type, bssrate[NumRates], sta_bssrate[NumRates];
-	struct registry_priv *pregpriv = &padapter->registrypriv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	int bssrate_len = 0, sta_bssrate_len = 0, pie_len;
-	u8 *pie;
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe)
-		goto exit;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *)pmgntframe->buf_addr + TXDESC_OFFSET;
-	mgmt = (struct ieee80211_mgmt *)pframe;
-
-	mgmt->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_ASSOC_REQ);
-
-	ether_addr_copy(mgmt->da, get_my_bssid23a(&pmlmeinfo->network));
-	ether_addr_copy(mgmt->sa, myid(&padapter->eeprompriv));
-	ether_addr_copy(mgmt->bssid, get_my_bssid23a(&pmlmeinfo->network));
-
-	mgmt->seq_ctrl = cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-	pmlmeext->mgnt_seq++;
-
-	/* caps */
-	put_unaligned_le16(pmlmeinfo->network.capability,
-			   &mgmt->u.assoc_req.capab_info);
-	/* todo: listen interval for power saving */
-	put_unaligned_le16(3, &mgmt->u.assoc_req.listen_interval);
-
-	pframe = mgmt->u.assoc_req.variable;
-	pattrib->pktlen = offsetof(struct ieee80211_mgmt, u.assoc_req.variable);
-
-	/* SSID */
-	pframe = rtw_set_ie23a(pframe, WLAN_EID_SSID,
-			       pmlmeinfo->network.Ssid.ssid_len,
-			       pmlmeinfo->network.Ssid.ssid, &pattrib->pktlen);
-
-	/* supported rate & extended supported rate */
-
-	get_rate_set23a(padapter, sta_bssrate, &sta_bssrate_len);
-	/* DBG_8723A("sta_bssrate_len =%d\n", sta_bssrate_len); */
-
-	/*  for JAPAN, channel 14 can only uses B Mode(CCK) */
-	if (pmlmeext->cur_channel == 14)
-		sta_bssrate_len = 4;
-
-	/* for (i = 0; i < sta_bssrate_len; i++) { */
-	/*	DBG_8723A("sta_bssrate[%d]=%02X\n", i, sta_bssrate[i]); */
-	/*  */
-
-	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
-		if (pmlmeinfo->network.SupportedRates[i] == 0)
-			break;
-		DBG_8723A("network.SupportedRates[%d]=%02X\n", i,
-			  pmlmeinfo->network.SupportedRates[i]);
-	}
-
-	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
-		if (pmlmeinfo->network.SupportedRates[i] == 0)
-			break;
-
-		/*  Check if the AP's supported rates are also
-		    supported by STA. */
-		for (j = 0; j < sta_bssrate_len; j++) {
-			 /*  Avoid the proprietary data rate (22Mbps) of
-			     Handlink WSG-4000 AP */
-			if ((pmlmeinfo->network.SupportedRates[i] |
-			     IEEE80211_BASIC_RATE_MASK) ==
-			    (sta_bssrate[j] | IEEE80211_BASIC_RATE_MASK)) {
-				/* DBG_8723A("match i = %d, j =%d\n", i, j); */
-				break;
-			}
-		}
-
-		if (j == sta_bssrate_len) {
-			/*  the rate is not supported by STA */
-			DBG_8723A("%s(): the rate[%d]=%02X is not supported by "
-				  "STA!\n", __func__, i,
-				  pmlmeinfo->network.SupportedRates[i]);
-		} else {
-			/*  the rate is supported by STA */
-			bssrate[index++] = pmlmeinfo->network.SupportedRates[i];
-		}
-	}
-
-	bssrate_len = index;
-	DBG_8723A("bssrate_len = %d\n", bssrate_len);
-
-	if (bssrate_len == 0) {
-		rtw_free_xmitbuf23a(pxmitpriv, pmgntframe->pxmitbuf);
-		rtw_free_xmitframe23a(pxmitpriv, pmgntframe);
-		goto exit; /* don't connect to AP if no joint supported rate */
-	}
-
-	if (bssrate_len > 8) {
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SUPP_RATES, 8,
-				       bssrate, &pattrib->pktlen);
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_EXT_SUPP_RATES,
-				       (bssrate_len - 8), (bssrate + 8),
-				       &pattrib->pktlen);
-	} else
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_SUPP_RATES,
-				       bssrate_len, bssrate, &pattrib->pktlen);
-
-	/* RSN */
-
-	pie = pmlmeinfo->network.IEs;
-	pie_len = pmlmeinfo->network.IELength;
-
-	p = cfg80211_find_ie(WLAN_EID_RSN, pie, pie_len);
-	if (p)
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_RSN, p[1], p + 2,
-				       &pattrib->pktlen);
-
-	/* HT caps */
-	if (padapter->mlmepriv.htpriv.ht_option) {
-		p = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY, pie, pie_len);
-
-		if (p && !is_ap_in_tkip23a(padapter)) {
-			struct ieee80211_ht_cap *cap = &pmlmeinfo->ht_cap;
-
-			memcpy(cap, p + 2, sizeof(struct ieee80211_ht_cap));
-
-			/* to disable 40M Hz support while gd_bw_40MHz_en = 0 */
-			if (pregpriv->cbw40_enable == 0) {
-				cap->cap_info &= ~cpu_to_le16(
-					IEEE80211_HT_CAP_SGI_40 |
-					IEEE80211_HT_CAP_SUP_WIDTH_20_40);
-			} else {
-				cap->cap_info |= cpu_to_le16(
-					IEEE80211_HT_CAP_SUP_WIDTH_20_40);
-			}
-
-			/* todo: disable SM power save mode */
-			cap->cap_info |= cpu_to_le16(IEEE80211_HT_CAP_SM_PS);
-
-			rf_type = rtl8723a_get_rf_type(padapter);
-			/* switch (pregpriv->rf_config) */
-			switch (rf_type) {
-			case RF_1T1R:
-				/* RX STBC One spatial stream */
-				if (pregpriv->rx_stbc)
-					cap->cap_info |= cpu_to_le16(1 << IEEE80211_HT_CAP_RX_STBC_SHIFT);
-
-				memcpy(&cap->mcs, MCS_rate_1R23A, 16);
-				break;
-
-			case RF_2T2R:
-			case RF_1T2R:
-			default:
-				/* enable for 2.4/5 GHz */
-				if (pregpriv->rx_stbc == 0x3 ||
-				    (pmlmeext->cur_wireless_mode &
-				     WIRELESS_11_24N &&
-				     /* enable for 2.4GHz */
-				     pregpriv->rx_stbc == 0x1) ||
-				    (pmlmeext->cur_wireless_mode &
-				     WIRELESS_11_5N &&
-				     pregpriv->rx_stbc == 0x2) ||
-				    /* enable for 5GHz */
-				    pregpriv->wifi_spec == 1) {
-					DBG_8723A("declare supporting RX "
-						  "STBC\n");
-					/* RX STBC two spatial stream */
-					cap->cap_info |= cpu_to_le16(2 << IEEE80211_HT_CAP_RX_STBC_SHIFT);
-				}
-				memcpy(&cap->mcs, MCS_rate_2R23A, 16);
-				break;
-			}
-
-			if (rtl8723a_BT_coexist(padapter) &&
-			    rtl8723a_BT_using_antenna_1(padapter)) {
-				/*  set to 8K */
-				cap->ampdu_params_info &=
-					~IEEE80211_HT_AMPDU_PARM_FACTOR;
-/*				cap->ampdu_params_info |= MAX_AMPDU_FACTOR_8K */
-			}
-
-			pframe = rtw_set_ie23a(pframe, WLAN_EID_HT_CAPABILITY,
-					       p[1], (u8 *)&pmlmeinfo->ht_cap,
-					       &pattrib->pktlen);
-		}
-	}
-
-	/* vendor specific IE, such as WPA, WMM, WPS */
-	for (i = 0;  i < pmlmeinfo->network.IELength;) {
-		p = pmlmeinfo->network.IEs + i;
-
-		switch (p[0]) {
-		case WLAN_EID_VENDOR_SPECIFIC:
-			if (!memcmp(p + 2, RTW_WPA_OUI23A_TYPE, 4) ||
-			    !memcmp(p + 2, WMM_OUI23A, 4) ||
-			    !memcmp(p + 2, WPS_OUI23A, 4)) {
-				u8 plen = p[1];
-
-				if (!padapter->registrypriv.wifi_spec) {
-					/* Commented by Kurt 20110629 */
-					/* In some older APs, WPS handshake */
-					/* would be fail if we append vender
-					   extensions informations to AP */
-					if (!memcmp(p + 2, WPS_OUI23A, 4))
-						plen = 14;
-				}
-				pframe = rtw_set_ie23a(pframe,
-						       WLAN_EID_VENDOR_SPECIFIC,
-						       plen, p + 2,
-						       &pattrib->pktlen);
-			}
-			break;
-
-		default:
-			break;
-		}
-
-		i += p[1] + 2;
-	}
-
-	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_REALTEK)
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_VENDOR_SPECIFIC, 6,
-				       REALTEK_96B_IE, &pattrib->pktlen);
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-	dump_mgntframe23a(padapter, pmgntframe);
-
-	ret = _SUCCESS;
-
-exit:
-	pmlmepriv->assoc_req_len = 0;
-	if (ret == _SUCCESS) {
-		kfree(pmlmepriv->assoc_req);
-		pmlmepriv->assoc_req = kmalloc(pattrib->pktlen, GFP_ATOMIC);
-		if (pmlmepriv->assoc_req) {
-			memcpy(pmlmepriv->assoc_req, mgmt, pattrib->pktlen);
-			pmlmepriv->assoc_req_len = pattrib->pktlen;
-		}
-	} else
-		kfree(pmlmepriv->assoc_req);
-}
-
-/* when wait_ack is true, this function should be called at process context */
-static int _issue_nulldata23a(struct rtw_adapter *padapter, unsigned char *da,
-			      unsigned int power_mode, int wait_ack)
-{
-	int ret = _FAIL;
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	struct ieee80211_hdr *pwlanhdr;
-	struct xmit_priv *pxmitpriv;
-	struct mlme_ext_priv *pmlmeext;
-	struct mlme_ext_info *pmlmeinfo;
-
-	/* DBG_8723A("%s:%d\n", __func__, power_mode); */
-
-	if (!padapter)
-		goto exit;
-
-	pxmitpriv = &padapter->xmitpriv;
-	pmlmeext = &padapter->mlmeextpriv;
-	pmlmeinfo = &pmlmeext->mlmext_info;
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe)
-		goto exit;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-	pattrib->retry_ctrl = false;
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct ieee80211_hdr *)pframe;
-
-	pwlanhdr->frame_control = cpu_to_le16(IEEE80211_FTYPE_DATA |
-					      IEEE80211_STYPE_NULLFUNC);
-
-	if ((pmlmeinfo->state&0x03) == MSR_AP)
-		pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_FROMDS);
-	else if ((pmlmeinfo->state&0x03) == MSR_INFRA)
-		pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_TODS);
-
-	if (power_mode)
-		pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
-
-	ether_addr_copy(pwlanhdr->addr1, da);
-	ether_addr_copy(pwlanhdr->addr2, myid(&padapter->eeprompriv));
-	ether_addr_copy(pwlanhdr->addr3, get_my_bssid23a(&pmlmeinfo->network));
-
-	pwlanhdr->seq_ctrl =
-		cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-	pmlmeext->mgnt_seq++;
-
-	pframe += sizeof(struct ieee80211_hdr_3addr);
-	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	if (wait_ack)
-		ret = dump_mgntframe23a_and_wait_ack23a(padapter, pmgntframe);
-	else {
-		dump_mgntframe23a(padapter, pmgntframe);
-		ret = _SUCCESS;
-	}
-
-exit:
-	return ret;
-}
-
-/* when wait_ms >0 , this function should be called at process context */
-/* da == NULL for station mode */
-int issue_nulldata23a(struct rtw_adapter *padapter, unsigned char *da,
-		      unsigned int power_mode, int try_cnt, int wait_ms)
-{
-	int ret;
-	int i = 0;
-	unsigned long start = jiffies;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	/* da == NULL, assume it's null data for sta to ap*/
-	if (da == NULL)
-		da = get_my_bssid23a(&pmlmeinfo->network);
-
-	do {
-		ret = _issue_nulldata23a(padapter, da, power_mode,
-					 wait_ms > 0 ? true : false);
-
-		i++;
-
-		if (padapter->bDriverStopped || padapter->bSurpriseRemoved)
-			break;
-
-		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
-			msleep(wait_ms);
-
-	} while((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
-
-	if (ret != _FAIL) {
-		ret = _SUCCESS;
-		goto exit;
-	}
-
-	if (try_cnt && wait_ms) {
-		if (da)
-			DBG_8723A("%s(%s): to %pM, ch:%u%s, %d/%d in %u ms\n",
-				  __func__, padapter->pnetdev->name,
-				  da, rtw_get_oper_ch23a(padapter),
-				  ret == _SUCCESS ? ", acked" : "", i, try_cnt,
-				  jiffies_to_msecs(jiffies - start));
-		else
-			DBG_8723A("%s(%s):, ch:%u%s, %d/%d in %u ms\n",
-				  __func__, padapter->pnetdev->name,
-				  rtw_get_oper_ch23a(padapter),
-				  ret == _SUCCESS ? ", acked" : "", i, try_cnt,
-				  jiffies_to_msecs(jiffies - start));
-	}
-exit:
-	return ret;
-}
-
-/* when wait_ack is true, this function should be called at process context */
-static int _issue_qos_nulldata23a(struct rtw_adapter *padapter,
-				  unsigned char *da, u16 tid, int wait_ack)
-{
-	int ret = _FAIL;
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	struct ieee80211_qos_hdr *pwlanhdr;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	DBG_8723A("%s\n", __func__);
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe)
-		goto exit;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-
-	pattrib->hdrlen += 2;
-	pattrib->qos_en = true;
-	pattrib->eosp = 1;
-	pattrib->ack_policy = 0;
-	pattrib->mdata = 0;
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-	pwlanhdr = (struct ieee80211_qos_hdr *)pframe;
-
-	pwlanhdr->frame_control = cpu_to_le16(IEEE80211_FTYPE_DATA |
-					      IEEE80211_STYPE_QOS_NULLFUNC);
-
-	if ((pmlmeinfo->state&0x03) == MSR_AP)
-		pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_FROMDS);
-	else if ((pmlmeinfo->state&0x03) == MSR_INFRA)
-		pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_TODS);
-
-	if (pattrib->mdata)
-		pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_MOREDATA);
-
-	pwlanhdr->qos_ctrl = cpu_to_le16(tid & IEEE80211_QOS_CTL_TID_MASK);
-	pwlanhdr->qos_ctrl |= cpu_to_le16((pattrib->ack_policy << 5) &
-					  IEEE80211_QOS_CTL_ACK_POLICY_MASK);
-	if (pattrib->eosp)
-		pwlanhdr->qos_ctrl |= cpu_to_le16(IEEE80211_QOS_CTL_EOSP);
-
-	ether_addr_copy(pwlanhdr->addr1, da);
-	ether_addr_copy(pwlanhdr->addr2, myid(&padapter->eeprompriv));
-	ether_addr_copy(pwlanhdr->addr3, get_my_bssid23a(&pmlmeinfo->network));
-
-	pwlanhdr->seq_ctrl =
-		cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-	pmlmeext->mgnt_seq++;
-
-	pframe += sizeof(struct ieee80211_qos_hdr);
-	pattrib->pktlen = sizeof(struct ieee80211_qos_hdr);
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	if (wait_ack)
-		ret = dump_mgntframe23a_and_wait_ack23a(padapter, pmgntframe);
-	else {
-		dump_mgntframe23a(padapter, pmgntframe);
-		ret = _SUCCESS;
-	}
-
-exit:
-	return ret;
-}
-
-/* when wait_ms >0 , this function should be called at process context */
-/* da == NULL for station mode */
-int issue_qos_nulldata23a(struct rtw_adapter *padapter, unsigned char *da,
-			  u16 tid, int try_cnt, int wait_ms)
-{
-	int ret;
-	int i = 0;
-	unsigned long start = jiffies;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	/* da == NULL, assume it's null data for sta to ap*/
-	if (da == NULL)
-		da = get_my_bssid23a(&pmlmeinfo->network);
-
-	do {
-		ret = _issue_qos_nulldata23a(padapter, da, tid,
-					     wait_ms > 0 ? true : false);
-
-		i++;
-
-		if (padapter->bDriverStopped || padapter->bSurpriseRemoved)
-			break;
-
-		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
-			msleep(wait_ms);
-	} while((i < try_cnt) && ((ret == _FAIL)||(wait_ms == 0)));
-
-	if (ret != _FAIL) {
-		ret = _SUCCESS;
-		goto exit;
-	}
-
-	if (try_cnt && wait_ms) {
-		if (da)
-			DBG_8723A("%s(%s): to %pM, ch:%u%s, %d/%d in %u ms\n",
-				  __func__, padapter->pnetdev->name,
-				  da, rtw_get_oper_ch23a(padapter),
-				  ret == _SUCCESS ? ", acked" : "", i, try_cnt,
-				  jiffies_to_msecs(jiffies - start));
-		else
-			DBG_8723A("%s(%s):, ch:%u%s, %d/%d in %u ms\n",
-				  __func__, padapter->pnetdev->name,
-				  rtw_get_oper_ch23a(padapter),
-				  ret == _SUCCESS ? ", acked" : "", i, try_cnt,
-				  jiffies_to_msecs(jiffies - start));
-	}
-exit:
-	return ret;
-}
-
-static int _issue_deauth(struct rtw_adapter *padapter, unsigned char *da,
-			 unsigned short reason, u8 wait_ack)
-{
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	struct ieee80211_mgmt *mgmt;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	int ret = _FAIL;
-
-	/* DBG_8723A("%s to %pM\n", __func__, da); */
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe)
-		goto exit;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-	pattrib->retry_ctrl = false;
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	mgmt = (struct ieee80211_mgmt *)(pmgntframe->buf_addr + TXDESC_OFFSET);
-
-	mgmt->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_DEAUTH);
-
-	ether_addr_copy(mgmt->da, da);
-	ether_addr_copy(mgmt->sa, myid(&padapter->eeprompriv));
-	ether_addr_copy(mgmt->bssid, get_my_bssid23a(&pmlmeinfo->network));
-
-	mgmt->seq_ctrl = cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-	pmlmeext->mgnt_seq++;
-
-	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr) + 2;
-
-	mgmt->u.deauth.reason_code = cpu_to_le16(reason);
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	if (wait_ack)
-		ret = dump_mgntframe23a_and_wait_ack23a(padapter, pmgntframe);
-	else {
-		dump_mgntframe23a(padapter, pmgntframe);
-		ret = _SUCCESS;
-	}
-
-exit:
-	return ret;
-}
-
-int issue_deauth23a(struct rtw_adapter *padapter, unsigned char *da,
-		    unsigned short reason)
-{
-	DBG_8723A("%s to %pM\n", __func__, da);
-	return _issue_deauth(padapter, da, reason, false);
-}
-
-static int issue_deauth_ex(struct rtw_adapter *padapter, u8 *da,
-			   unsigned short reason, int try_cnt, int wait_ms)
-{
-	int ret;
-	int i = 0;
-	unsigned long start = jiffies;
-
-	do {
-		ret = _issue_deauth(padapter, da, reason,
-				    wait_ms >0 ? true : false);
-
-		i++;
-
-		if (padapter->bDriverStopped || padapter->bSurpriseRemoved)
-			break;
-
-		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
-			msleep(wait_ms);
-
-	} while((i < try_cnt) && ((ret == _FAIL)||(wait_ms == 0)));
-
-	if (ret != _FAIL) {
-		ret = _SUCCESS;
-		goto exit;
-	}
-
-	if (try_cnt && wait_ms) {
-		if (da)
-			DBG_8723A("%s(%s): to %pM, ch:%u%s, %d/%d in %u ms\n",
-				  __func__, padapter->pnetdev->name,
-				  da, rtw_get_oper_ch23a(padapter),
-				  ret == _SUCCESS ? ", acked" : "", i, try_cnt,
-				  jiffies_to_msecs(jiffies - start));
-		else
-			DBG_8723A("%s(%s):, ch:%u%s, %d/%d in %u ms\n",
-				  __func__, padapter->pnetdev->name,
-				  rtw_get_oper_ch23a(padapter),
-				  ret == _SUCCESS ? ", acked" : "", i, try_cnt,
-				  jiffies_to_msecs(jiffies - start));
-	}
-exit:
-	return ret;
-}
-
-void issue_action_spct_ch_switch23a(struct rtw_adapter *padapter,
-				    u8 *ra, u8 new_ch, u8 ch_offset)
-{
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	struct ieee80211_mgmt *mgmt;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	DBG_8723A("%s(%s): ra=%pM, ch:%u, offset:%u\n",
-		  __func__, padapter->pnetdev->name, ra, new_ch, ch_offset);
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe)
-		return;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	mgmt = (struct ieee80211_mgmt *)(pmgntframe->buf_addr + TXDESC_OFFSET);
-
-	mgmt->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_ACTION);
-
-	ether_addr_copy(mgmt->da, ra); /* RA */
-	ether_addr_copy(mgmt->sa, myid(&padapter->eeprompriv)); /* TA */
-	ether_addr_copy(mgmt->bssid, ra); /* DA = RA */
-
-	mgmt->seq_ctrl = cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-	pmlmeext->mgnt_seq++;
-
-	mgmt->u.action.category = WLAN_CATEGORY_SPECTRUM_MGMT;
-	mgmt->u.action.u.chan_switch.action_code = WLAN_ACTION_SPCT_CHL_SWITCH;
-
-	pframe = mgmt->u.action.u.chan_switch.variable;
-	pattrib->pktlen = offsetof(struct ieee80211_mgmt,
-				   u.action.u.chan_switch.variable);
-
-	pframe = rtw_set_ie23a_ch_switch (pframe, &pattrib->pktlen, 0,
-					  new_ch, 0);
-	pframe = rtw_set_ie23a_secondary_ch_offset(pframe, &pattrib->pktlen,
-		hal_ch_offset_to_secondary_ch_offset23a(ch_offset));
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	dump_mgntframe23a(padapter, pmgntframe);
-}
-
-void issue_action_BA23a(struct rtw_adapter *padapter,
-			const unsigned char *raddr,
-			unsigned char action, unsigned short status)
-{
-	u16 start_seq;
-	u16 BA_para_set;
-	u16 BA_starting_seqctrl;
-	u16 BA_para;
-	int max_rx_ampdu_factor;
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	struct ieee80211_mgmt *mgmt;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct sta_info *psta;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct registry_priv *pregpriv = &padapter->registrypriv;
-	u8 tendaAPMac[] = {0xC8, 0x3A, 0x35};
-
-	DBG_8723A("%s, action =%d, status =%d\n", __func__, action, status);
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe)
-		return;
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	mgmt = (struct ieee80211_mgmt *)(pmgntframe->buf_addr + TXDESC_OFFSET);
-
-	mgmt->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_ACTION);
-
-	ether_addr_copy(mgmt->da, raddr);
-	ether_addr_copy(mgmt->sa, myid(&padapter->eeprompriv));
-	ether_addr_copy(mgmt->bssid, get_my_bssid23a(&pmlmeinfo->network));
-
-	mgmt->seq_ctrl = cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-	pmlmeext->mgnt_seq++;
-
-	mgmt->u.action.category = WLAN_CATEGORY_BACK;
-
-	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr) + 1;
-
-	switch (action) {
-	case WLAN_ACTION_ADDBA_REQ:
-		pattrib->pktlen += sizeof(mgmt->u.action.u.addba_req);
-
-		mgmt->u.action.u.addba_req.action_code = action;
-
-		do {
-			pmlmeinfo->dialogToken++;
-		} while (pmlmeinfo->dialogToken == 0);
-
-		mgmt->u.action.u.addba_req.dialog_token =
-			pmlmeinfo->dialogToken;
-
-		if (rtl8723a_BT_coexist(padapter) &&
-		    rtl8723a_BT_using_antenna_1(padapter) &&
-		    (pmlmeinfo->assoc_AP_vendor != broadcomAP ||
-		     memcmp(raddr, tendaAPMac, 3))) {
-			/*  A-MSDU NOT Supported */
-			BA_para_set = 0;
-			/*  immediate Block Ack */
-			BA_para_set |= (1 << 1) &
-				IEEE80211_ADDBA_PARAM_POLICY_MASK;
-			/*  TID */
-			BA_para_set |= (status << 2) &
-				IEEE80211_ADDBA_PARAM_TID_MASK;
-			/*  max buffer size is 8 MSDU */
-			BA_para_set |= (8 << 6) &
-				IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK;
-		} else {
-			/* immediate ack & 64 buffer size */
-			BA_para_set = 0x1002 | ((status & 0xf) << 2);
-		}
-
-		put_unaligned_le16(BA_para_set,
-				   &mgmt->u.action.u.addba_req.capab);
-
-		/*  5ms */
-		put_unaligned_le16(5000, &mgmt->u.action.u.addba_req.timeout);
-
-		psta = rtw_get_stainfo23a(pstapriv, raddr);
-		if (psta) {
-			int idx;
-
-			idx = status & 0x07;
-			start_seq =
-				(psta->sta_xmitpriv.txseq_tid[idx] & 0xfff) + 1;
-
-			DBG_8723A("BA_starting_seqctrl = %d for TID =%d\n",
-				  start_seq, idx);
-
-			psta->BA_starting_seqctrl[idx] = start_seq;
-
-			BA_starting_seqctrl = start_seq << 4;
-		} else
-			BA_starting_seqctrl = 0;
-
-		put_unaligned_le16(BA_starting_seqctrl,
-				   &mgmt->u.action.u.addba_req.start_seq_num);
-
-		break;
-
-	case WLAN_ACTION_ADDBA_RESP:
-		pattrib->pktlen += sizeof(mgmt->u.action.u.addba_resp);
-
-		mgmt->u.action.u.addba_resp.action_code = action;
-		mgmt->u.action.u.addba_resp.dialog_token =
-			pmlmeinfo->ADDBA_req.dialog_token;
-		put_unaligned_le16(status,
-				   &mgmt->u.action.u.addba_resp.status);
-
-		GetHalDefVar8192CUsb(padapter, HW_VAR_MAX_RX_AMPDU_FACTOR,
-				     &max_rx_ampdu_factor);
-
-		BA_para = le16_to_cpu(pmlmeinfo->ADDBA_req.BA_para_set) & 0x3f;
-		if (max_rx_ampdu_factor == IEEE80211_HT_MAX_AMPDU_64K)
-			BA_para_set = BA_para | 0x1000; /* 64 buffer size */
-		else if (max_rx_ampdu_factor == IEEE80211_HT_MAX_AMPDU_32K)
-			BA_para_set = BA_para | 0x0800; /* 32 buffer size */
-		else if (max_rx_ampdu_factor == IEEE80211_HT_MAX_AMPDU_16K)
-			BA_para_set = BA_para | 0x0400; /* 16 buffer size */
-		else if (max_rx_ampdu_factor == IEEE80211_HT_MAX_AMPDU_8K)
-			BA_para_set = BA_para | 0x0200; /* 8 buffer size */
-		else
-			BA_para_set = BA_para | 0x1000; /* 64 buffer size */
-
-		if (rtl8723a_BT_coexist(padapter) &&
-		    rtl8723a_BT_using_antenna_1(padapter) &&
-		    (pmlmeinfo->assoc_AP_vendor != broadcomAP ||
-		     memcmp(raddr, tendaAPMac, 3))) {
-			/*  max buffer size is 8 MSDU */
-			BA_para_set &= ~IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK;
-			BA_para_set |= (8 << 6) &
-				IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK;
-		}
-
-		if (pregpriv->ampdu_amsdu == 0)/* disabled */
-			BA_para_set &= ~BIT(0);
-		else if (pregpriv->ampdu_amsdu == 1)/* enabled */
-			BA_para_set |= BIT(0);
-
-		put_unaligned_le16(BA_para_set,
-				   &mgmt->u.action.u.addba_resp.capab);
-
-		mgmt->u.action.u.addba_resp.timeout
-			= pmlmeinfo->ADDBA_req.BA_timeout_value;
-
-		pattrib->pktlen += 8;
-		break;
-	case WLAN_ACTION_DELBA:
-		pattrib->pktlen += sizeof(mgmt->u.action.u.delba);
-
-		mgmt->u.action.u.delba.action_code = action;
-		BA_para_set = (status & 0x1F) << 3;
-		mgmt->u.action.u.delba.params = cpu_to_le16(BA_para_set);
-		mgmt->u.action.u.delba.reason_code =
-			cpu_to_le16(WLAN_REASON_QSTA_NOT_USE);
-
-		pattrib->pktlen += 5;
-		break;
-	default:
-		break;
-	}
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	dump_mgntframe23a(padapter, pmgntframe);
-}
-
-int send_delba23a(struct rtw_adapter *padapter, u8 initiator, u8 *addr)
-{
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct sta_info *psta = NULL;
-	/* struct recv_reorder_ctrl *preorder_ctrl; */
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	u16 tid;
-
-	if ((pmlmeinfo->state&0x03) != MSR_AP)
-		if (!(pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS))
-			return _SUCCESS;
-
-	psta = rtw_get_stainfo23a(pstapriv, addr);
-	if (psta == NULL)
-		return _SUCCESS;
-
-	if (initiator == 0) {  /*  recipient */
-		for (tid = 0; tid < MAXTID; tid++) {
-			if (psta->recvreorder_ctrl[tid].enable == true) {
-				DBG_8723A("rx agg disable tid(%d)\n", tid);
-				issue_action_BA23a(padapter, addr, WLAN_ACTION_DELBA, (((tid <<1) |initiator)&0x1F));
-				psta->recvreorder_ctrl[tid].enable = false;
-				psta->recvreorder_ctrl[tid].indicate_seq = 0xffff;
-			}
-		}
-	} else if (initiator == 1) { /*  originator */
-		for (tid = 0; tid < MAXTID; tid++) {
-			if (psta->htpriv.agg_enable_bitmap & BIT(tid)) {
-				DBG_8723A("tx agg disable tid(%d)\n", tid);
-				issue_action_BA23a(padapter, addr, WLAN_ACTION_DELBA, (((tid <<1) |initiator)&0x1F));
-				psta->htpriv.agg_enable_bitmap &= ~BIT(tid);
-				psta->htpriv.candidate_tid_bitmap &= ~BIT(tid);
-
-			}
-		}
-	}
-	return _SUCCESS;
-}
-
-int send_beacon23a(struct rtw_adapter *padapter)
-{
-	bool bxmitok;
-	int issue = 0;
-	int poll = 0;
-	unsigned long start = jiffies;
-	unsigned int passing_time;
-
-	rtl8723a_bcn_valid(padapter);
-	do {
-		issue_beacon23a(padapter, 100);
-		issue++;
-		do {
-			yield();
-			bxmitok = rtl8723a_get_bcn_valid(padapter);
-			poll++;
-		} while ((poll % 10) != 0 && !bxmitok &&
-			 !padapter->bSurpriseRemoved &&
-			 !padapter->bDriverStopped);
-
-	} while (!bxmitok && issue<100 && !padapter->bSurpriseRemoved &&
-		 !padapter->bDriverStopped);
-
-	if (padapter->bSurpriseRemoved || padapter->bDriverStopped)
-		return _FAIL;
-
-	passing_time = jiffies_to_msecs(jiffies - start);
-
-	if (!bxmitok) {
-		DBG_8723A("%s fail! %u ms\n", __func__, passing_time);
-		return _FAIL;
-	} else {
-
-		if (passing_time > 100 || issue > 3)
-			DBG_8723A("%s success, issue:%d, poll:%d, %u ms\n",
-				  __func__, issue, poll, passing_time);
-		return _SUCCESS;
-	}
-}
-
-/****************************************************************************
-
-Following are some utitity functions for WiFi MLME
-
-*****************************************************************************/
-
-bool IsLegal5GChannel(struct rtw_adapter *Adapter, u8 channel)
-{
-
-	int i = 0;
-	u8 Channel_5G[45] = {36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58,
-			     60, 62, 64, 100, 102, 104, 106, 108, 110, 112,
-			     114, 116, 118, 120, 122, 124, 126, 128, 130, 132,
-			     134, 136, 138, 140, 149, 151, 153, 155, 157, 159,
-			     161, 163, 165};
-	for (i = 0; i < sizeof(Channel_5G); i++)
-		if (channel == Channel_5G[i])
-			return true;
-	return false;
-}
-
-static void rtw_site_survey(struct rtw_adapter *padapter)
-{
-	unsigned char survey_channel = 0;
-	enum rt_scan_type ScanType = SCAN_PASSIVE;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct rtw_ieee80211_channel *ch;
-
-	if (pmlmeext->sitesurvey_res.channel_idx <
-	    pmlmeext->sitesurvey_res.ch_num) {
-		ch = &pmlmeext->sitesurvey_res.ch[pmlmeext->sitesurvey_res.channel_idx];
-		survey_channel = ch->hw_value;
-		ScanType = (ch->flags & IEEE80211_CHAN_NO_IR) ?
-			SCAN_PASSIVE : SCAN_ACTIVE;
-	}
-
-	if (survey_channel != 0) {
-		/* PAUSE 4-AC Queue when site_survey */
-		if (pmlmeext->sitesurvey_res.channel_idx == 0)
-			set_channel_bwmode23a(padapter, survey_channel,
-					      HAL_PRIME_CHNL_OFFSET_DONT_CARE,
-					      HT_CHANNEL_WIDTH_20);
-		else
-			SelectChannel23a(padapter, survey_channel);
-
-		if (ScanType == SCAN_ACTIVE) /* obey the channel plan setting... */
-		{
-			int i;
-
-			for (i = 0;i<RTW_SSID_SCAN_AMOUNT;i++) {
-				if (pmlmeext->sitesurvey_res.ssid[i].ssid_len) {
-					/* todo: to issue two probe req??? */
-					issue_probereq(padapter, &pmlmeext->sitesurvey_res.ssid[i], NULL);
-					/* msleep(SURVEY_TO>>1); */
-					issue_probereq(padapter, &pmlmeext->sitesurvey_res.ssid[i], NULL);
-				}
-			}
-
-			if (pmlmeext->sitesurvey_res.scan_mode == SCAN_ACTIVE) {
-				/* todo: to issue two probe req??? */
-				issue_probereq(padapter, NULL, NULL);
-				/* msleep(SURVEY_TO>>1); */
-				issue_probereq(padapter, NULL, NULL);
-			}
-		}
-
-		set_survey_timer(pmlmeext, pmlmeext->chan_scan_time);
-	} else {
-		/*	channel number is 0 or this channel is not valid. */
-		pmlmeext->sitesurvey_res.state = SCAN_COMPLETE;
-
-		/* switch back to the original channel */
-
-		set_channel_bwmode23a(padapter, pmlmeext->cur_channel,
-				      pmlmeext->cur_ch_offset,
-				      pmlmeext->cur_bwmode);
-
-		/* flush 4-AC Queue after rtw_site_survey */
-		/* val8 = 0; */
-
-		/* config MSR */
-		rtl8723a_set_media_status(padapter, pmlmeinfo->state & 0x3);
-
-		/* restore RX GAIN */
-		rtl8723a_set_initial_gain(padapter, 0xff);
-		/* turn on dynamic functions */
-		rtl8723a_odm_support_ability_restore(padapter);
-
-		if (is_client_associated_to_ap23a(padapter) == true)
-			issue_nulldata23a(padapter, NULL, 0, 3, 500);
-
-		rtl8723a_mlme_sitesurvey(padapter, 0);
-
-		report_surveydone_event23a(padapter);
-
-		pmlmeext->chan_scan_time = SURVEY_TO;
-		pmlmeext->sitesurvey_res.state = SCAN_DISABLE;
-	}
-}
-
-/* collect bss info from Beacon and Probe request/response frames. */
-static struct wlan_bssid_ex *collect_bss_info(struct rtw_adapter *padapter,
-					      struct recv_frame *precv_frame)
-{
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *) skb->data;
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *bssid;
-	const u8 *p;
-	u8 *pie;
-	unsigned int length;
-	int i;
-
-	length = skb->len;
-
-	bssid = kzalloc(sizeof(struct wlan_bssid_ex), GFP_ATOMIC);
-	if (!bssid)
-		return NULL;
-
-	if (ieee80211_is_beacon(mgmt->frame_control)) {
-		length -= offsetof(struct ieee80211_mgmt, u.beacon.variable);
-		pie = mgmt->u.beacon.variable;
-		bssid->reserved = 1;
-		bssid->capability =
-			get_unaligned_le16(&mgmt->u.beacon.capab_info);
-		bssid->beacon_interval =
-			get_unaligned_le16(&mgmt->u.beacon.beacon_int);
-		bssid->tsf = get_unaligned_le64(&mgmt->u.beacon.timestamp);
-	} else if (ieee80211_is_probe_req(mgmt->frame_control)) {
-		length -= offsetof(struct ieee80211_mgmt, u.probe_req.variable);
-		pie = mgmt->u.probe_req.variable;
-		bssid->reserved = 2;
-		bssid->capability = 0;
-		bssid->beacon_interval =
-			padapter->registrypriv.dev_network.beacon_interval;
-		bssid->tsf = 0;
-	} else if (ieee80211_is_probe_resp(mgmt->frame_control)) {
-		length -=
-			offsetof(struct ieee80211_mgmt, u.probe_resp.variable);
-		pie = mgmt->u.probe_resp.variable;
-		bssid->reserved = 3;
-		bssid->capability =
-			get_unaligned_le16(&mgmt->u.probe_resp.capab_info);
-		bssid->beacon_interval =
-			get_unaligned_le16(&mgmt->u.probe_resp.beacon_int);
-		bssid->tsf = get_unaligned_le64(&mgmt->u.probe_resp.timestamp);
-	} else {
-		length -= offsetof(struct ieee80211_mgmt, u.beacon.variable);
-		pie = mgmt->u.beacon.variable;
-		bssid->reserved = 0;
-		bssid->capability =
-			get_unaligned_le16(&mgmt->u.beacon.capab_info);
-		bssid->beacon_interval =
-			padapter->registrypriv.dev_network.beacon_interval;
-		bssid->tsf = 0;
-	}
-
-	if (length > MAX_IE_SZ) {
-		/* DBG_8723A("IE too long for survey event\n"); */
-		kfree(bssid);
-		return NULL;
-	}
-
-	bssid->Length = offsetof(struct wlan_bssid_ex, IEs) + length;
-
-	/* below is to copy the information element */
-	bssid->IELength = length;
-	memcpy(bssid->IEs, pie, bssid->IELength);
-
-	/* get the signal strength */
-	/*  in dBM.raw data */
-	bssid->Rssi = precv_frame->attrib.phy_info.RecvSignalPower;
-	bssid->SignalQuality =
-		precv_frame->attrib.phy_info.SignalQuality;/* in percentage */
-	bssid->SignalStrength =
-		precv_frame->attrib.phy_info.SignalStrength;/* in percentage */
-
-	/*  checking SSID */
-	p = cfg80211_find_ie(WLAN_EID_SSID, bssid->IEs, bssid->IELength);
-
-	if (!p) {
-		DBG_8723A("marc: cannot find SSID for survey event\n");
-		goto fail;
-	}
-
-	if (p[1] > IEEE80211_MAX_SSID_LEN) {
-		DBG_8723A("%s()-%d: IE too long (%d) for survey "
-			  "event\n", __func__, __LINE__, p[1]);
-		goto fail;
-	}
-	memcpy(bssid->Ssid.ssid, p + 2, p[1]);
-	bssid->Ssid.ssid_len = p[1];
-
-	/* checking rate info... */
-	i = 0;
-	p = cfg80211_find_ie(WLAN_EID_SUPP_RATES, bssid->IEs, bssid->IELength);
-	if (p) {
-		if (p[1] > NDIS_802_11_LENGTH_RATES_EX) {
-			DBG_8723A("%s()-%d: IE too long (%d) for survey "
-				  "event\n", __func__, __LINE__, p[1]);
-			goto fail;
-		}
-		memcpy(bssid->SupportedRates, p + 2, p[1]);
-		i = p[1];
-	}
-
-	p = cfg80211_find_ie(WLAN_EID_EXT_SUPP_RATES, bssid->IEs,
-			     bssid->IELength);
-	if (p) {
-		if (p[1] > (NDIS_802_11_LENGTH_RATES_EX-i)) {
-			DBG_8723A("%s()-%d: IE too long (%d) for survey "
-				  "event\n", __func__, __LINE__, p[1]);
-			goto fail;
-		}
-		memcpy(bssid->SupportedRates + i, p + 2, p[1]);
-	}
-
-	/*  Checking for DSConfig */
-	p = cfg80211_find_ie(WLAN_EID_DS_PARAMS, bssid->IEs, bssid->IELength);
-
-	bssid->DSConfig = 0;
-
-	if (p) {
-		bssid->DSConfig = p[2];
-	} else {/*  In 5G, some ap do not have DSSET IE */
-		/*  checking HT info for channel */
-		p = cfg80211_find_ie(WLAN_EID_HT_OPERATION, bssid->IEs,
-				     bssid->IELength);
-		if (p) {
-			struct ieee80211_ht_operation *HT_info =
-				(struct ieee80211_ht_operation *)(p + 2);
-			bssid->DSConfig = HT_info->primary_chan;
-		} else /*  use current channel */
-			bssid->DSConfig = rtw_get_oper_ch23a(padapter);
-	}
-
-	if (ieee80211_is_probe_req(mgmt->frame_control)) {
-		/*  FIXME */
-		bssid->ifmode = NL80211_IFTYPE_STATION;
-		ether_addr_copy(bssid->MacAddress, mgmt->sa);
-		bssid->Privacy = 1;
-		return bssid;
-	}
-
-	if (bssid->capability & WLAN_CAPABILITY_ESS) {
-		bssid->ifmode = NL80211_IFTYPE_STATION;
-		ether_addr_copy(bssid->MacAddress, mgmt->sa);
-	} else {
-		bssid->ifmode = NL80211_IFTYPE_ADHOC;
-		ether_addr_copy(bssid->MacAddress, mgmt->bssid);
-	}
-
-	if (bssid->capability & WLAN_CAPABILITY_PRIVACY)
-		bssid->Privacy = 1;
-	else
-		bssid->Privacy = 0;
-
-	bssid->ATIMWindow = 0;
-
-	/* 20/40 BSS Coexistence check */
-	if (pregistrypriv->wifi_spec == 1 &&
-	    pmlmeinfo->bwmode_updated == false) {
-		struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-		p = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY, bssid->IEs,
-				     bssid->IELength);
-		if (p && p[1] > 0) {
-			struct ieee80211_ht_cap *pHT_caps;
-
-			pHT_caps = (struct ieee80211_ht_cap *)(p + 2);
-
-			if (pHT_caps->cap_info &
-			    cpu_to_le16(IEEE80211_HT_CAP_40MHZ_INTOLERANT))
-				pmlmepriv->num_FortyMHzIntolerant++;
-		} else
-			pmlmepriv->num_sta_no_ht++;
-	}
-
-
-	/*  mark bss info receiving from nearby channel as SignalQuality 101 */
-	if (bssid->DSConfig != rtw_get_oper_ch23a(padapter))
-		bssid->SignalQuality = 101;
-
-	return bssid;
-fail:
-	kfree (bssid);
-	return NULL;
-}
-
-static void start_create_ibss(struct rtw_adapter *padapter)
-{
-	unsigned short caps;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork = &pmlmeinfo->network;
-
-	pmlmeext->cur_channel = (u8)pnetwork->DSConfig;
-	pmlmeinfo->bcn_interval = pnetwork->beacon_interval;
-
-	/* update wireless mode */
-	update_wireless_mode23a(padapter);
-
-	/* update capability */
-	caps = pnetwork->capability;
-	update_capinfo23a(padapter, caps);
-	if (caps & WLAN_CAPABILITY_IBSS) {	/* adhoc master */
-		rtl8723a_set_sec_cfg(padapter, 0xcf);
-
-		/* switch channel */
-		/* SelectChannel23a(padapter, pmlmeext->cur_channel, HAL_PRIME_CHNL_OFFSET_DONT_CARE); */
-		set_channel_bwmode23a(padapter, pmlmeext->cur_channel, HAL_PRIME_CHNL_OFFSET_DONT_CARE, HT_CHANNEL_WIDTH_20);
-
-		rtl8723a_SetBeaconRelatedRegisters(padapter);
-
-		/* set msr to MSR_ADHOC */
-		pmlmeinfo->state = MSR_ADHOC;
-		rtl8723a_set_media_status(padapter, pmlmeinfo->state & 0x3);
-
-		/* issue beacon */
-		if (send_beacon23a(padapter) == _FAIL) {
-			RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-				 "issuing beacon frame fail....\n");
-
-			report_join_res23a(padapter, -1);
-			pmlmeinfo->state = MSR_NOLINK;
-		} else {
-			hw_var_set_bssid(padapter, padapter->registrypriv.dev_network.MacAddress);
-			hw_var_set_mlme_join(padapter, 0);
-
-			report_join_res23a(padapter, 1);
-			pmlmeinfo->state |= WIFI_FW_ASSOC_SUCCESS;
-		}
-	} else {
-		DBG_8723A("%s: invalid cap:%x\n", __func__, caps);
-		return;
-	}
-}
-
-static void start_clnt_join(struct rtw_adapter *padapter)
-{
-	unsigned short caps;
-	u8 val8;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork = &pmlmeinfo->network;
-	int beacon_timeout;
-
-	pmlmeext->cur_channel = (u8)pnetwork->DSConfig;
-	pmlmeinfo->bcn_interval = pnetwork->beacon_interval;
-
-	/* update wireless mode */
-	update_wireless_mode23a(padapter);
-
-	/* update capability */
-	caps = pnetwork->capability;
-	update_capinfo23a(padapter, caps);
-	if (caps & WLAN_CAPABILITY_ESS) {
-		/* switch channel */
-		set_channel_bwmode23a(padapter, pmlmeext->cur_channel, pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode);
-
-		rtl8723a_set_media_status(padapter, MSR_INFRA);
-
-		val8 = (pmlmeinfo->auth_algo == dot11AuthAlgrthm_8021X) ?
-			0xcc: 0xcf;
-
-		rtl8723a_set_sec_cfg(padapter, val8);
-
-		/* switch channel */
-		/* set_channel_bwmode23a(padapter, pmlmeext->cur_channel, pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode); */
-
-		/* here wait for receiving the beacon to start auth */
-		/* and enable a timer */
-		beacon_timeout = decide_wait_for_beacon_timeout23a(pmlmeinfo->bcn_interval);
-		set_link_timer(pmlmeext, beacon_timeout);
-		mod_timer(&padapter->mlmepriv.assoc_timer, jiffies +
-			  msecs_to_jiffies((REAUTH_TO * REAUTH_LIMIT) + (REASSOC_TO*REASSOC_LIMIT) + beacon_timeout));
-		pmlmeinfo->state = WIFI_FW_AUTH_NULL | MSR_INFRA;
-	} else if (caps & WLAN_CAPABILITY_IBSS) {	/* adhoc client */
-		rtl8723a_set_media_status(padapter, MSR_ADHOC);
-
-		rtl8723a_set_sec_cfg(padapter, 0xcf);
-
-		/* switch channel */
-		set_channel_bwmode23a(padapter, pmlmeext->cur_channel, pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode);
-
-		rtl8723a_SetBeaconRelatedRegisters(padapter);
-
-		pmlmeinfo->state = MSR_ADHOC;
-
-		report_join_res23a(padapter, 1);
-	} else {
-		/* DBG_8723A("marc: invalid cap:%x\n", caps); */
-		return;
-	}
-}
-
-static void start_clnt_auth(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	del_timer_sync(&pmlmeext->link_timer);
-
-	pmlmeinfo->state &= (~WIFI_FW_AUTH_NULL);
-	pmlmeinfo->state |= WIFI_FW_AUTH_STATE;
-
-	pmlmeinfo->auth_seq = 1;
-	pmlmeinfo->reauth_count = 0;
-	pmlmeinfo->reassoc_count = 0;
-	pmlmeinfo->link_count = 0;
-	pmlmeext->retry = 0;
-
-	/*  Because of AP's not receiving deauth before */
-	/*  AP may: 1)not response auth or 2)deauth us after link is complete */
-	/*  issue deauth before issuing auth to deal with the situation */
-	/*	Commented by Albert 2012/07/21 */
-	/*	For the Win8 P2P connection, it will be hard to have a
-		successful connection if this Wi-Fi doesn't connect to it. */
-	issue_deauth23a(padapter, (&pmlmeinfo->network)->MacAddress,
-			WLAN_REASON_DEAUTH_LEAVING);
-
-	DBG_8723A_LEVEL(_drv_always_, "start auth\n");
-	issue_auth(padapter, NULL, 0);
-
-	set_link_timer(pmlmeext, REAUTH_TO);
-}
-
-static void start_clnt_assoc(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	del_timer_sync(&pmlmeext->link_timer);
-
-	pmlmeinfo->state &= (~(WIFI_FW_AUTH_NULL | WIFI_FW_AUTH_STATE));
-	pmlmeinfo->state |= (WIFI_FW_AUTH_SUCCESS | WIFI_FW_ASSOC_STATE);
-
-	issue_assocreq(padapter);
-
-	set_link_timer(pmlmeext, REASSOC_TO);
-}
-
-int receive_disconnect23a(struct rtw_adapter *padapter,
-			  unsigned char *MacAddr, unsigned short reason)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	/* check A3 */
-	if (!ether_addr_equal(MacAddr, get_my_bssid23a(&pmlmeinfo->network)))
-		return _SUCCESS;
-
-	DBG_8723A("%s\n", __func__);
-
-	if ((pmlmeinfo->state&0x03) == MSR_INFRA) {
-		if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) {
-			pmlmeinfo->state = MSR_NOLINK;
-			report_del_sta_event23a(padapter, MacAddr, reason);
-
-		} else if (pmlmeinfo->state & WIFI_FW_LINKING_STATE) {
-			pmlmeinfo->state = MSR_NOLINK;
-			report_join_res23a(padapter, -2);
-		}
-	}
-
-	return _SUCCESS;
-}
-
-static void process_80211d(struct rtw_adapter *padapter,
-			   struct wlan_bssid_ex *bssid)
-{
-	struct registry_priv *pregistrypriv;
-	struct mlme_ext_priv *pmlmeext;
-	struct rt_channel_info *chplan_new;
-	u8 channel;
-	u8 i;
-
-	pregistrypriv = &padapter->registrypriv;
-	pmlmeext = &padapter->mlmeextpriv;
-
-	/*  Adjust channel plan by AP Country IE */
-	if (pregistrypriv->enable80211d &&
-	    !pmlmeext->update_channel_plan_by_ap_done) {
-		const u8 *ie, *p;
-		struct rt_channel_plan chplan_ap;
-		struct rt_channel_info chplan_sta[MAX_CHANNEL_NUM];
-		u8 country[4];
-		u8 fcn; /*  first channel number */
-		u8 noc; /*  number of channel */
-		u8 j, k;
-
-		ie = cfg80211_find_ie(WLAN_EID_COUNTRY, bssid->IEs,
-				      bssid->IELength);
-		if (!ie || ie[1] < IEEE80211_COUNTRY_IE_MIN_LEN)
-			return;
-
-		p = ie + 2;
-		ie += ie[1];
-		ie += 2;
-
-		memcpy(country, p, 3);
-		country[3] = '\0';
-
-		p += 3;
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_notice_,
-			 "%s: 802.11d country =%s\n", __func__, country);
-
-		i = 0;
-		while ((ie - p) >= 3) {
-			fcn = *(p++);
-			noc = *(p++);
-			p++;
-
-			for (j = 0; j < noc; j++) {
-				if (fcn <= 14)
-					channel = fcn + j; /*  2.4 GHz */
-				else
-					channel = fcn + j * 4; /*  5 GHz */
-
-				chplan_ap.Channel[i++] = channel;
-			}
-		}
-		chplan_ap.Len = i;
-
-		memcpy(chplan_sta, pmlmeext->channel_set, sizeof(chplan_sta));
-		memset(pmlmeext->channel_set, 0, sizeof(pmlmeext->channel_set));
-		chplan_new = pmlmeext->channel_set;
-
-		i = j = k = 0;
-		if (pregistrypriv->wireless_mode & WIRELESS_11G) {
-			do {
-				if (i == MAX_CHANNEL_NUM ||
-				    chplan_sta[i].ChannelNum == 0 ||
-				    chplan_sta[i].ChannelNum > 14)
-					break;
-
-				if (j == chplan_ap.Len ||
-				    chplan_ap.Channel[j] > 14)
-					break;
-
-				if (chplan_sta[i].ChannelNum ==
-				    chplan_ap.Channel[j]) {
-					chplan_new[k].ChannelNum =
-						chplan_ap.Channel[j];
-					chplan_new[k].ScanType = SCAN_ACTIVE;
-					i++;
-					j++;
-					k++;
-				} else if (chplan_sta[i].ChannelNum <
-					   chplan_ap.Channel[j]) {
-					chplan_new[k].ChannelNum =
-						chplan_sta[i].ChannelNum;
-					chplan_new[k].ScanType =
-						SCAN_PASSIVE;
-					i++;
-					k++;
-				} else if (chplan_sta[i].ChannelNum >
-					   chplan_ap.Channel[j]) {
-					chplan_new[k].ChannelNum =
-						chplan_ap.Channel[j];
-					chplan_new[k].ScanType =
-						SCAN_ACTIVE;
-					j++;
-					k++;
-				}
-			} while (1);
-
-			/*  change AP not support channel to Passive scan */
-			while (i < MAX_CHANNEL_NUM &&
-			       chplan_sta[i].ChannelNum != 0 &&
-			       chplan_sta[i].ChannelNum <= 14) {
-				chplan_new[k].ChannelNum =
-					chplan_sta[i].ChannelNum;
-				chplan_new[k].ScanType = SCAN_PASSIVE;
-				i++;
-				k++;
-			}
-
-			/*  add channel AP supported */
-			while (j < chplan_ap.Len && chplan_ap.Channel[j] <= 14){
-				chplan_new[k].ChannelNum = chplan_ap.Channel[j];
-				chplan_new[k].ScanType = SCAN_ACTIVE;
-				j++;
-				k++;
-			}
-		} else {
-			/*  keep original STA 2.4G channel plan */
-			while (i < MAX_CHANNEL_NUM &&
-			       chplan_sta[i].ChannelNum != 0 &&
-			       chplan_sta[i].ChannelNum <= 14) {
-				chplan_new[k].ChannelNum =
-					chplan_sta[i].ChannelNum;
-				chplan_new[k].ScanType = chplan_sta[i].ScanType;
-				i++;
-				k++;
-			}
-
-			/*  skip AP 2.4G channel plan */
-			while (j < chplan_ap.Len && chplan_ap.Channel[j] <= 14)
-				j++;
-		}
-
-		if (pregistrypriv->wireless_mode & WIRELESS_11A) {
-			do {
-				if (i == MAX_CHANNEL_NUM ||
-				    chplan_sta[i].ChannelNum == 0)
-					break;
-
-				if (j == chplan_ap.Len ||
-				    chplan_ap.Channel[j] == 0)
-					break;
-
-				if (chplan_sta[i].ChannelNum ==
-				    chplan_ap.Channel[j]) {
-					chplan_new[k].ChannelNum =
-						chplan_ap.Channel[j];
-					chplan_new[k].ScanType = SCAN_ACTIVE;
-					i++;
-					j++;
-					k++;
-				} else if (chplan_sta[i].ChannelNum <
-					   chplan_ap.Channel[j]) {
-					chplan_new[k].ChannelNum =
-						chplan_sta[i].ChannelNum;
-					chplan_new[k].ScanType = SCAN_PASSIVE;
-					i++;
-					k++;
-				} else if (chplan_sta[i].ChannelNum >
-					   chplan_ap.Channel[j]) {
-					chplan_new[k].ChannelNum =
-						chplan_ap.Channel[j];
-					chplan_new[k].ScanType = SCAN_ACTIVE;
-					j++;
-					k++;
-				}
-			} while (1);
-
-			/*  change AP not support channel to Passive scan */
-			while (i < MAX_CHANNEL_NUM &&
-			       chplan_sta[i].ChannelNum != 0) {
-				chplan_new[k].ChannelNum =
-					chplan_sta[i].ChannelNum;
-				chplan_new[k].ScanType = SCAN_PASSIVE;
-				i++;
-				k++;
-			}
-
-			/*  add channel AP supported */
-			while (j < chplan_ap.Len && chplan_ap.Channel[j] != 0) {
-				chplan_new[k].ChannelNum = chplan_ap.Channel[j];
-				chplan_new[k].ScanType = SCAN_ACTIVE;
-				j++;
-				k++;
-			}
-		} else {
-			/*  keep original STA 5G channel plan */
-			while (i < MAX_CHANNEL_NUM &&
-			       chplan_sta[i].ChannelNum != 0) {
-				chplan_new[k].ChannelNum =
-					chplan_sta[i].ChannelNum;
-				chplan_new[k].ScanType = chplan_sta[i].ScanType;
-				i++;
-				k++;
-			}
-		}
-		pmlmeext->update_channel_plan_by_ap_done = 1;
-	}
-
-	/*  If channel is used by AP, set channel scan type to active */
-	channel = bssid->DSConfig;
-	chplan_new = pmlmeext->channel_set;
-	i = 0;
-	while (i < MAX_CHANNEL_NUM && chplan_new[i].ChannelNum != 0) {
-		if (chplan_new[i].ChannelNum == channel) {
-			if (chplan_new[i].ScanType == SCAN_PASSIVE) {
-				/* 5G Bnad 2, 3 (DFS) doesn't change
-				   to active scan */
-				if (channel >= 52 && channel <= 144)
-					break;
-
-				chplan_new[i].ScanType = SCAN_ACTIVE;
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_notice_,
-					 "%s: change channel %d scan type from passive to active\n",
-					 __func__, channel);
-			}
-			break;
-		}
-		i++;
-	}
-}
-
-/****************************************************************************
-
-Following are the functions to report events
-
-*****************************************************************************/
-
-void report_survey_event23a(struct rtw_adapter *padapter,
-			    struct recv_frame *precv_frame)
-{
-	struct cmd_obj *pcmd_obj;
-	u8 *pevtcmd;
-	u32 cmdsz;
-	struct survey_event *psurvey_evt;
-	struct C2HEvent_Header *pc2h_evt_hdr;
-	struct mlme_ext_priv *pmlmeext;
-	struct cmd_priv *pcmdpriv;
-
-	if (!padapter)
-		return;
-
-	pmlmeext = &padapter->mlmeextpriv;
-	pcmdpriv = &padapter->cmdpriv;
-
-	pcmd_obj = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!pcmd_obj)
-		return;
-
-	cmdsz = sizeof(struct survey_event) + sizeof(struct C2HEvent_Header);
-	pevtcmd = kzalloc(cmdsz, GFP_ATOMIC);
-	if (!pevtcmd) {
-		kfree(pcmd_obj);
-		return;
-	}
-
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
-	pcmd_obj->cmdsz = cmdsz;
-	pcmd_obj->parmbuf = pevtcmd;
-
-	pcmd_obj->rsp = NULL;
-	pcmd_obj->rspsz  = 0;
-
-	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
-	pc2h_evt_hdr->len = sizeof(struct survey_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_Survey);
-	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
-
-	psurvey_evt = (struct survey_event*)(pevtcmd + sizeof(struct C2HEvent_Header));
-
-	psurvey_evt->bss = collect_bss_info(padapter, precv_frame);
-	if (!psurvey_evt->bss) {
-		kfree(pcmd_obj);
-		kfree(pevtcmd);
-		return;
-	}
-
-	process_80211d(padapter, psurvey_evt->bss);
-
-	rtw_enqueue_cmd23a(pcmdpriv, pcmd_obj);
-
-	pmlmeext->sitesurvey_res.bss_cnt++;
-}
-
-void report_surveydone_event23a(struct rtw_adapter *padapter)
-{
-	struct cmd_obj *pcmd_obj;
-	u8 *pevtcmd;
-	u32 cmdsz;
-	struct surveydone_event *psurveydone_evt;
-	struct C2HEvent_Header *pc2h_evt_hdr;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-
-	pcmd_obj = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!pcmd_obj)
-		return;
-
-	cmdsz = sizeof(struct surveydone_event) + sizeof(struct C2HEvent_Header);
-	pevtcmd = kzalloc(cmdsz, GFP_ATOMIC);
-	if (!pevtcmd) {
-		kfree(pcmd_obj);
-		return;
-	}
-
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
-	pcmd_obj->cmdsz = cmdsz;
-	pcmd_obj->parmbuf = pevtcmd;
-
-	pcmd_obj->rsp = NULL;
-	pcmd_obj->rspsz  = 0;
-
-	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
-	pc2h_evt_hdr->len = sizeof(struct surveydone_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_SurveyDone);
-	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
-
-	psurveydone_evt = (struct surveydone_event*)(pevtcmd + sizeof(struct C2HEvent_Header));
-	psurveydone_evt->bss_cnt = pmlmeext->sitesurvey_res.bss_cnt;
-
-	DBG_8723A("survey done event(%x)\n", psurveydone_evt->bss_cnt);
-
-	rtw_enqueue_cmd23a(pcmdpriv, pcmd_obj);
-}
-
-void report_join_res23a(struct rtw_adapter *padapter, int res)
-{
-	struct cmd_obj *pcmd_obj;
-	u8 *pevtcmd;
-	u32 cmdsz;
-	struct joinbss_event		*pjoinbss_evt;
-	struct C2HEvent_Header	*pc2h_evt_hdr;
-	struct mlme_ext_priv		*pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-
-	pcmd_obj = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!pcmd_obj)
-		return;
-
-	cmdsz = sizeof(struct joinbss_event) + sizeof(struct C2HEvent_Header);
-	pevtcmd = kzalloc(cmdsz, GFP_ATOMIC);
-	if (!pevtcmd) {
-		kfree(pcmd_obj);
-		return;
-	}
-
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
-	pcmd_obj->cmdsz = cmdsz;
-	pcmd_obj->parmbuf = pevtcmd;
-
-	pcmd_obj->rsp = NULL;
-	pcmd_obj->rspsz  = 0;
-
-	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
-	pc2h_evt_hdr->len = sizeof(struct joinbss_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_JoinBss);
-	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
-
-	pjoinbss_evt = (struct joinbss_event*)(pevtcmd + sizeof(struct C2HEvent_Header));
-	memcpy((unsigned char *)&pjoinbss_evt->network.network,
-	       &pmlmeinfo->network, sizeof(struct wlan_bssid_ex));
-	pjoinbss_evt->network.join_res = res;
-
-	DBG_8723A("report_join_res23a(%d)\n", res);
-
-	rtw_joinbss_event_prehandle23a(padapter, (u8 *)&pjoinbss_evt->network);
-
-	rtw_enqueue_cmd23a(pcmdpriv, pcmd_obj);
-}
-
-void report_del_sta_event23a(struct rtw_adapter *padapter,
-			     unsigned char *MacAddr, unsigned short reason)
-{
-	struct cmd_obj *pcmd_obj;
-	u8 *pevtcmd;
-	u32 cmdsz;
-	struct sta_info *psta;
-	int mac_id;
-	struct stadel_event *pdel_sta_evt;
-	struct C2HEvent_Header *pc2h_evt_hdr;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-
-	pcmd_obj = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!pcmd_obj)
-		return;
-
-	cmdsz = sizeof(struct stadel_event) + sizeof(struct C2HEvent_Header);
-	pevtcmd = kzalloc(cmdsz, GFP_ATOMIC);
-	if (!pevtcmd) {
-		kfree(pcmd_obj);
-		return;
-	}
-
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
-	pcmd_obj->cmdsz = cmdsz;
-	pcmd_obj->parmbuf = pevtcmd;
-
-	pcmd_obj->rsp = NULL;
-	pcmd_obj->rspsz  = 0;
-
-	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
-	pc2h_evt_hdr->len = sizeof(struct stadel_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_DelSTA);
-	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
-
-	pdel_sta_evt = (struct stadel_event*)(pevtcmd + sizeof(struct C2HEvent_Header));
-	ether_addr_copy((unsigned char *)&pdel_sta_evt->macaddr, MacAddr);
-	memcpy((unsigned char *)pdel_sta_evt->rsvd, (unsigned char *)&reason,
-	       2);
-
-	psta = rtw_get_stainfo23a(&padapter->stapriv, MacAddr);
-	if (psta)
-		mac_id = (int)psta->mac_id;
-	else
-		mac_id = -1;
-
-	pdel_sta_evt->mac_id = mac_id;
-
-	DBG_8723A("report_del_sta_event23a: delete STA, mac_id =%d\n", mac_id);
-
-	rtw_enqueue_cmd23a(pcmdpriv, pcmd_obj);
-}
-
-void report_add_sta_event23a(struct rtw_adapter *padapter,
-			     unsigned char *MacAddr, int cam_idx)
-{
-	struct cmd_obj *pcmd_obj;
-	u8 *pevtcmd;
-	u32 cmdsz;
-	struct stassoc_event *padd_sta_evt;
-	struct C2HEvent_Header *pc2h_evt_hdr;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-
-	pcmd_obj = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!pcmd_obj)
-		return;
-
-	cmdsz = sizeof(struct stassoc_event) + sizeof(struct C2HEvent_Header);
-	pevtcmd = kzalloc(cmdsz, GFP_ATOMIC);
-	if (!pevtcmd) {
-		kfree(pcmd_obj);
-		return;
-	}
-
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
-	pcmd_obj->cmdsz = cmdsz;
-	pcmd_obj->parmbuf = pevtcmd;
-
-	pcmd_obj->rsp = NULL;
-	pcmd_obj->rspsz  = 0;
-
-	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
-	pc2h_evt_hdr->len = sizeof(struct stassoc_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_AddSTA);
-	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
-
-	padd_sta_evt = (struct stassoc_event*)(pevtcmd + sizeof(struct C2HEvent_Header));
-	ether_addr_copy((unsigned char *)&padd_sta_evt->macaddr, MacAddr);
-	padd_sta_evt->cam_id = cam_idx;
-
-	DBG_8723A("report_add_sta_event23a: add STA\n");
-
-	rtw_enqueue_cmd23a(pcmdpriv, pcmd_obj);
-}
-
-/****************************************************************************
-
-Following are the event callback functions
-
-*****************************************************************************/
-
-/* for sta/adhoc mode */
-void update_sta_info23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	/* ERP */
-	VCS_update23a(padapter, psta);
-
-	/* HT */
-	if (pmlmepriv->htpriv.ht_option) {
-		psta->htpriv.ht_option = true;
-
-		psta->htpriv.ampdu_enable = pmlmepriv->htpriv.ampdu_enable;
-
-		if (support_short_GI23a(padapter, &pmlmeinfo->ht_cap))
-			psta->htpriv.sgi = true;
-
-		psta->qos_option = true;
-
-	} else {
-		psta->htpriv.ht_option = false;
-
-		psta->htpriv.ampdu_enable = false;
-
-		psta->htpriv.sgi = false;
-		psta->qos_option = false;
-
-	}
-	psta->htpriv.bwmode = pmlmeext->cur_bwmode;
-	psta->htpriv.ch_offset = pmlmeext->cur_ch_offset;
-
-	psta->htpriv.agg_enable_bitmap = 0x0;/* reset */
-	psta->htpriv.candidate_tid_bitmap = 0x0;/* reset */
-
-	/* QoS */
-	if (pmlmepriv->qos_option)
-		psta->qos_option = true;
-
-	psta->state = _FW_LINKED;
-}
-
-void mlmeext_joinbss_event_callback23a(struct rtw_adapter *padapter,
-				       int join_res)
-{
-	struct sta_info *psta, *psta_bmc;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-
-	if (join_res < 0) {
-		hw_var_set_mlme_join(padapter, 1);
-		hw_var_set_bssid(padapter, null_addr);
-
-		/* restore to initial setting. */
-		update_tx_basic_rate23a(padapter,
-					padapter->registrypriv.wireless_mode);
-
-		goto exit_mlmeext_joinbss_event_callback23a;
-	}
-
-	if ((pmlmeinfo->state&0x03) == MSR_ADHOC) {
-		/* for bc/mc */
-		psta_bmc = rtw_get_bcmc_stainfo23a(padapter);
-		if (psta_bmc) {
-			pmlmeinfo->FW_sta_info[psta_bmc->mac_id].psta = psta_bmc;
-			update_bmc_sta_support_rate23a(padapter, psta_bmc->mac_id);
-			Update_RA_Entry23a(padapter, psta_bmc);
-		}
-	}
-
-	/* turn on dynamic functions */
-	rtl8723a_odm_support_ability_set(padapter, DYNAMIC_ALL_FUNC_ENABLE);
-
-	/*  update IOT-releated issue */
-	update_IOT_info23a(padapter);
-
-	HalSetBrateCfg23a(padapter, cur_network->SupportedRates);
-
-	/* BCN interval */
-	rtl8723a_set_beacon_interval(padapter, pmlmeinfo->bcn_interval);
-
-	/* update capability */
-	update_capinfo23a(padapter, pmlmeinfo->capability);
-
-	/* WMM, Update EDCA param */
-	WMMOnAssocRsp23a(padapter);
-
-	/* HT */
-	HTOnAssocRsp23a(padapter);
-
-	/* Set cur_channel&cur_bwmode&cur_ch_offset */
-	set_channel_bwmode23a(padapter, pmlmeext->cur_channel, pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode);
-
-	psta = rtw_get_stainfo23a(pstapriv, cur_network->MacAddress);
-	if (psta) { /* only for infra. mode */
-		pmlmeinfo->FW_sta_info[psta->mac_id].psta = psta;
-
-		/* DBG_8723A("set_sta_rate23a\n"); */
-
-		psta->wireless_mode = pmlmeext->cur_wireless_mode;
-
-		/* set per sta rate after updating HT cap. */
-		set_sta_rate23a(padapter, psta);
-	}
-
-	hw_var_set_mlme_join(padapter, 2);
-
-	if ((pmlmeinfo->state&0x03) == MSR_INFRA) {
-		/*  correcting TSF */
-		rtw_correct_TSF(padapter);
-
-		/* set_link_timer(pmlmeext, DISCONNECT_TO); */
-	}
-
-	rtw_lps_ctrl_wk_cmd23a(padapter, LPS_CTRL_CONNECT, 0);
-
-exit_mlmeext_joinbss_event_callback23a:
-	DBG_8723A("=>%s\n", __func__);
-}
-
-void mlmeext_sta_add_event_callback23a(struct rtw_adapter *padapter,
-				       struct sta_info *psta)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	DBG_8723A("%s\n", __func__);
-
-	if ((pmlmeinfo->state & 0x03) == MSR_ADHOC) {
-	/* adhoc master or sta_count>1 */
-		if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) {
-			/* nothing to do */
-		} else { /* adhoc client */
-			/*  correcting TSF */
-			rtw_correct_TSF(padapter);
-
-			/* start beacon */
-			if (send_beacon23a(padapter) != _SUCCESS) {
-				pmlmeinfo->FW_sta_info[psta->mac_id].status = 0;
-
-				pmlmeinfo->state ^= MSR_ADHOC;
-
-				return;
-			}
-
-			pmlmeinfo->state |= WIFI_FW_ASSOC_SUCCESS;
-		}
-		hw_var_set_mlme_join(padapter, 2);
-	}
-
-	pmlmeinfo->FW_sta_info[psta->mac_id].psta = psta;
-
-	/* rate radaptive */
-	Update_RA_Entry23a(padapter, psta);
-
-	/* update adhoc sta_info */
-	update_sta_info23a(padapter, psta);
-}
-
-void mlmeext_sta_del_event_callback23a(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (is_client_associated_to_ap23a(padapter) ||
-	    is_IBSS_empty23a(padapter)) {
-		/* set_opmode_cmd(padapter, infra_client_with_mlme); */
-
-		hw_var_set_mlme_disconnect(padapter);
-		hw_var_set_bssid(padapter, null_addr);
-
-		/* restore to initial setting. */
-		update_tx_basic_rate23a(padapter,
-					padapter->registrypriv.wireless_mode);
-
-		/* switch to the 20M Hz mode after disconnect */
-		pmlmeext->cur_bwmode = HT_CHANNEL_WIDTH_20;
-		pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-
-		set_channel_bwmode23a(padapter, pmlmeext->cur_channel,
-				      pmlmeext->cur_ch_offset,
-				      pmlmeext->cur_bwmode);
-
-		flush_all_cam_entry23a(padapter);
-
-		pmlmeinfo->state = MSR_NOLINK;
-
-		/* set MSR to no link state -> infra. mode */
-		rtl8723a_set_media_status(padapter, MSR_INFRA);
-
-		del_timer_sync(&pmlmeext->link_timer);
-	}
-}
-
-static u8 chk_ap_is_alive(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	u8 ret = false;
-
-	if (sta_rx_data_pkts(psta) == sta_last_rx_data_pkts(psta) &&
-	    sta_rx_beacon_pkts(psta) == sta_last_rx_beacon_pkts(psta) &&
-	    sta_rx_probersp_pkts(psta) == sta_last_rx_probersp_pkts(psta))
-		ret = false;
-	else
-		ret = true;
-
-	sta_update_last_rx_pkts(psta);
-	return ret;
-}
-
-void linked_status_chk23a(struct rtw_adapter *padapter)
-{
-	u32 i;
-	struct sta_info *psta;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-
-	if (is_client_associated_to_ap23a(padapter)) {
-		/* linked infrastructure client mode */
-
-		int tx_chk = _SUCCESS, rx_chk = _SUCCESS;
-		int rx_chk_limit;
-
-		rx_chk_limit = 4;
-
-		psta = rtw_get_stainfo23a(pstapriv,
-					  pmlmeinfo->network.MacAddress);
-		if (psta) {
-			bool is_p2p_enable = false;
-
-			if (chk_ap_is_alive(padapter, psta) == false)
-				rx_chk = _FAIL;
-
-			if (pxmitpriv->last_tx_pkts == pxmitpriv->tx_pkts)
-				tx_chk = _FAIL;
-
-			if (pmlmeext->active_keep_alive_check &&
-			    (rx_chk == _FAIL || tx_chk == _FAIL)) {
-				u8 backup_oper_channel = 0;
-
-				/* switch to correct channel of current
-				   network  before issue keep-alive frames */
-				if (rtw_get_oper_ch23a(padapter) !=
-				    pmlmeext->cur_channel) {
-					backup_oper_channel =
-						rtw_get_oper_ch23a(padapter);
-					SelectChannel23a(padapter,
-							 pmlmeext->cur_channel);
-				}
-
-				if (rx_chk != _SUCCESS)
-					issue_probereq_ex(padapter, &pmlmeinfo->network.Ssid, psta->hwaddr, 3, 1);
-
-				if ((tx_chk != _SUCCESS &&
-				     pmlmeinfo->link_count++ == 0xf) ||
-				    rx_chk != _SUCCESS) {
-					tx_chk = issue_nulldata23a(padapter,
-								   psta->hwaddr,
-								   0, 3, 1);
-					/* if tx acked and p2p disabled,
-					   set rx_chk _SUCCESS to reset retry
-					   count */
-					if (tx_chk == _SUCCESS &&
-					    !is_p2p_enable)
-						rx_chk = _SUCCESS;
-				}
-
-				/* back to the original operation channel */
-				if (backup_oper_channel>0)
-					SelectChannel23a(padapter,
-							 backup_oper_channel);
-			} else {
-				if (rx_chk != _SUCCESS) {
-					if (pmlmeext->retry == 0) {
-						issue_probereq(padapter, &pmlmeinfo->network.Ssid, pmlmeinfo->network.MacAddress);
-						issue_probereq(padapter, &pmlmeinfo->network.Ssid, pmlmeinfo->network.MacAddress);
-						issue_probereq(padapter, &pmlmeinfo->network.Ssid, pmlmeinfo->network.MacAddress);
-					}
-				}
-
-				if (tx_chk != _SUCCESS &&
-				    pmlmeinfo->link_count++ == 0xf)
-					tx_chk = issue_nulldata23a(padapter,
-								   NULL, 0, 1,
-								   0);
-			}
-
-			if (rx_chk == _FAIL) {
-				pmlmeext->retry++;
-				if (pmlmeext->retry > rx_chk_limit) {
-					DBG_8723A_LEVEL(_drv_always_,
-							"%s(%s): disconnect or "
-							"roaming\n", __func__,
-							padapter->pnetdev->name);
-					receive_disconnect23a(padapter, pmlmeinfo->network.MacAddress,
-						WLAN_REASON_EXPIRATION_CHK);
-					return;
-				}
-			} else
-				pmlmeext->retry = 0;
-
-			if (tx_chk == _FAIL)
-				pmlmeinfo->link_count &= 0xf;
-			else {
-				pxmitpriv->last_tx_pkts = pxmitpriv->tx_pkts;
-				pmlmeinfo->link_count = 0;
-			}
-
-		}
-	} else if (is_client_associated_to_ibss23a(padapter)) {
-		/* linked IBSS mode */
-		/* for each assoc list entry to check the rx pkt counter */
-		for (i = IBSS_START_MAC_ID; i < NUM_STA; i++) {
-			if (pmlmeinfo->FW_sta_info[i].status == 1) {
-				psta = pmlmeinfo->FW_sta_info[i].psta;
-
-				if (!psta)
-					continue;
-
-				if (pmlmeinfo->FW_sta_info[i].rx_pkt ==
-				    sta_rx_pkts(psta)) {
-
-					if (pmlmeinfo->FW_sta_info[i].retry<3) {
-						pmlmeinfo->FW_sta_info[i].retry++;
-					} else {
-						pmlmeinfo->FW_sta_info[i].retry = 0;
-						pmlmeinfo->FW_sta_info[i].status = 0;
-						report_del_sta_event23a(padapter, psta->hwaddr,
-							65535/*  indicate disconnect caused by no rx */
-						);
-					}
-				} else {
-					pmlmeinfo->FW_sta_info[i].retry = 0;
-					pmlmeinfo->FW_sta_info[i].rx_pkt = (u32)sta_rx_pkts(psta);
-				}
-			}
-		}
-		/* set_link_timer(pmlmeext, DISCONNECT_TO); */
-	}
-}
-
-static void survey_timer_hdl(unsigned long data)
-{
-	struct rtw_adapter *padapter = (struct rtw_adapter *)data;
-	struct cmd_obj *ph2c;
-	struct sitesurvey_parm *psurveyPara;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	/* issue rtw_sitesurvey_cmd23a */
-	if (pmlmeext->sitesurvey_res.state > SCAN_START) {
-		if (pmlmeext->sitesurvey_res.state ==  SCAN_PROCESS)
-			pmlmeext->sitesurvey_res.channel_idx++;
-
-		if (pmlmeext->scan_abort == true) {
-			pmlmeext->sitesurvey_res.channel_idx =
-				pmlmeext->sitesurvey_res.ch_num;
-			DBG_8723A("%s idx:%d\n", __func__,
-				  pmlmeext->sitesurvey_res.channel_idx);
-
-			pmlmeext->scan_abort = false;/* reset */
-		}
-
-		ph2c = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-		if (!ph2c)
-			goto exit_survey_timer_hdl;
-
-		psurveyPara = kzalloc(sizeof(struct sitesurvey_parm),
-					GFP_ATOMIC);
-		if (!psurveyPara) {
-			kfree(ph2c);
-			goto exit_survey_timer_hdl;
-		}
-
-		init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara,
-					   GEN_CMD_CODE(_SiteSurvey));
-		rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-	}
-
-exit_survey_timer_hdl:
-	return;
-}
-
-static void link_timer_hdl(unsigned long data)
-{
-	struct rtw_adapter *padapter = (struct rtw_adapter *)data;
-	/* static unsigned int		rx_pkt = 0; */
-	/* static u64				tx_cnt = 0; */
-	/* struct xmit_priv *pxmitpriv = &padapter->xmitpriv; */
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	/* struct sta_priv		*pstapriv = &padapter->stapriv; */
-
-	if (pmlmeinfo->state & WIFI_FW_AUTH_NULL) {
-		DBG_8723A("link_timer_hdl:no beacon while connecting\n");
-		pmlmeinfo->state = MSR_NOLINK;
-		report_join_res23a(padapter, -3);
-	} else if (pmlmeinfo->state & WIFI_FW_AUTH_STATE) {
-		/* re-auth timer */
-		if (++pmlmeinfo->reauth_count > REAUTH_LIMIT) {
-			/* if (pmlmeinfo->auth_algo != dot11AuthAlgrthm_Auto) */
-			/*  */
-				pmlmeinfo->state = 0;
-				report_join_res23a(padapter, -1);
-				return;
-			/*  */
-			/* else */
-			/*  */
-			/* pmlmeinfo->auth_algo = dot11AuthAlgrthm_Shared; */
-			/* pmlmeinfo->reauth_count = 0; */
-			/*  */
-		}
-
-		DBG_8723A("link_timer_hdl: auth timeout and try again\n");
-		pmlmeinfo->auth_seq = 1;
-		issue_auth(padapter, NULL, 0);
-		set_link_timer(pmlmeext, REAUTH_TO);
-	} else if (pmlmeinfo->state & WIFI_FW_ASSOC_STATE) {
-		/* re-assoc timer */
-		if (++pmlmeinfo->reassoc_count > REASSOC_LIMIT) {
-			pmlmeinfo->state = MSR_NOLINK;
-			report_join_res23a(padapter, -2);
-			return;
-		}
-
-		DBG_8723A("link_timer_hdl: assoc timeout and try again\n");
-		issue_assocreq(padapter);
-		set_link_timer(pmlmeext, REASSOC_TO);
-	}
-}
-
-static void addba_timer_hdl(unsigned long data)
-{
-	struct sta_info *psta = (struct sta_info *)data;
-	struct ht_priv *phtpriv;
-
-	if (!psta)
-		return;
-
-	phtpriv = &psta->htpriv;
-
-	if (phtpriv->ht_option && phtpriv->ampdu_enable) {
-		if (phtpriv->candidate_tid_bitmap)
-			phtpriv->candidate_tid_bitmap = 0x0;
-	}
-}
-
-void init_addba_retry_timer23a(struct sta_info *psta)
-{
-	setup_timer(&psta->addba_retry_timer, addba_timer_hdl,
-		    (unsigned long)psta);
-}
-
-void init_mlme_ext_timer23a(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	setup_timer(&pmlmeext->survey_timer, survey_timer_hdl,
-		    (unsigned long)padapter);
-
-	setup_timer(&pmlmeext->link_timer, link_timer_hdl,
-		    (unsigned long)padapter);
-}
-
-int NULL_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	return H2C_SUCCESS;
-}
-
-int setopmode_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	enum nl80211_iftype type;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	const struct setopmode_parm *psetop = (struct setopmode_parm *)pbuf;
-
-	switch (psetop->mode) {
-	case NL80211_IFTYPE_P2P_GO:
-	case NL80211_IFTYPE_AP:
-		pmlmeinfo->state = MSR_AP;
-		type = MSR_AP;
-		break;
-	case NL80211_IFTYPE_P2P_CLIENT:
-	case NL80211_IFTYPE_STATION:
-		/*  clear state */
-		pmlmeinfo->state &= ~(BIT(0)|BIT(1));
-		/* set to STATION_STATE */
-		pmlmeinfo->state |= MSR_INFRA;
-		type = MSR_INFRA;
-		break;
-	case NL80211_IFTYPE_ADHOC:
-		type = MSR_ADHOC;
-		break;
-	default:
-		type = MSR_NOLINK;
-		break;
-	}
-
-	hw_var_set_opmode(padapter, type);
-	/* Set_NETYPE0_MSR(padapter, type); */
-
-	return H2C_SUCCESS;
-}
-
-int createbss_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork = &pmlmeinfo->network;
-	const struct wlan_bssid_ex *pparm = (struct wlan_bssid_ex *)pbuf;
-	/* u32	initialgain; */
-
-	if (pparm->ifmode == NL80211_IFTYPE_AP ||
-	    pparm->ifmode == NL80211_IFTYPE_P2P_GO) {
-#ifdef CONFIG_8723AU_AP_MODE
-		if (pmlmeinfo->state == MSR_AP) {
-			/* todo: */
-			return H2C_SUCCESS;
-		}
-#endif
-	}
-
-	/* below is for ad-hoc master */
-	if (pparm->ifmode == NL80211_IFTYPE_ADHOC) {
-		rtw_joinbss_reset23a(padapter);
-
-		pmlmeext->cur_bwmode = HT_CHANNEL_WIDTH_20;
-		pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-		pmlmeinfo->ERP_enable = 0;
-		pmlmeinfo->WMM_enable = 0;
-		pmlmeinfo->HT_enable = 0;
-		pmlmeinfo->HT_caps_enable = 0;
-		pmlmeinfo->HT_info_enable = 0;
-
-		/* disable dynamic functions, such as high power, DIG */
-		rtl8723a_odm_support_ability_backup(padapter);
-
-		rtl8723a_odm_support_ability_clr(padapter,
-						 DYNAMIC_FUNC_DISABLE);
-
-		/* cancel link timer */
-		del_timer_sync(&pmlmeext->link_timer);
-
-		/* clear CAM */
-		flush_all_cam_entry23a(padapter);
-
-		if (pparm->IELength > MAX_IE_SZ)/* Check pbuf->IELength */
-			return H2C_PARAMETERS_ERROR;
-
-		memcpy(pnetwork, pparm, sizeof(struct wlan_bssid_ex));
-
-		start_create_ibss(padapter);
-	}
-
-	return H2C_SUCCESS;
-}
-
-int join_cmd_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	struct registry_priv *pregpriv = &padapter->registrypriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork = &pmlmeinfo->network;
-	const struct wlan_bssid_ex *pparm = (struct wlan_bssid_ex *)pbuf;
-	struct ieee80211_ht_operation *pht_info;
-	u32 i;
-	u8 *p;
-	/* u32	initialgain; */
-	/* u32	acparm; */
-
-	/* check already connecting to AP or not */
-	if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) {
-		if (pmlmeinfo->state & MSR_INFRA)
-			issue_deauth_ex(padapter, pnetwork->MacAddress,
-					WLAN_REASON_DEAUTH_LEAVING, 5, 100);
-
-		pmlmeinfo->state = MSR_NOLINK;
-
-		/* clear CAM */
-		flush_all_cam_entry23a(padapter);
-
-		del_timer_sync(&pmlmeext->link_timer);
-
-		/* set MSR to nolink -> infra. mode */
-		rtl8723a_set_media_status(padapter, MSR_INFRA);
-
-		hw_var_set_mlme_disconnect(padapter);
-	}
-
-	rtw_joinbss_reset23a(padapter);
-
-	pmlmeext->cur_bwmode = HT_CHANNEL_WIDTH_20;
-	pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-	pmlmeinfo->ERP_enable = 0;
-	pmlmeinfo->WMM_enable = 0;
-	pmlmeinfo->HT_enable = 0;
-	pmlmeinfo->HT_caps_enable = 0;
-	pmlmeinfo->HT_info_enable = 0;
-	pmlmeinfo->bwmode_updated = false;
-	/* pmlmeinfo->assoc_AP_vendor = HT_IOT_PEER_MAX; */
-
-	if (pparm->IELength > MAX_IE_SZ)/* Check pbuf->IELength */
-		return H2C_PARAMETERS_ERROR;
-
-	memcpy(pnetwork, pbuf, sizeof(struct wlan_bssid_ex));
-
-	/* Check AP vendor to move rtw_joinbss_cmd23a() */
-	/* pmlmeinfo->assoc_AP_vendor = check_assoc_AP23a(pnetwork->IEs,
-	   pnetwork->IELength); */
-
-	for (i = 0; i < pnetwork->IELength;) {
-		p = pnetwork->IEs + i;
-
-		switch (p[0]) {
-		case WLAN_EID_VENDOR_SPECIFIC:/* Get WMM IE. */
-			if (!memcmp(p + 2, WMM_OUI23A, 4))
-				pmlmeinfo->WMM_enable = 1;
-			break;
-
-		case WLAN_EID_HT_CAPABILITY:	/* Get HT Cap IE. */
-			pmlmeinfo->HT_caps_enable = 1;
-			break;
-
-		case WLAN_EID_HT_OPERATION:	/* Get HT Info IE. */
-			pmlmeinfo->HT_info_enable = 1;
-
-			/* spec case only for cisco's ap because cisco's ap
-			 * issue assoc rsp using mcs rate @40MHz or @20MHz */
-			pht_info = (struct ieee80211_ht_operation *)(p + 2);
-
-			if (pregpriv->cbw40_enable &&
-			    (pht_info->ht_param &
-			     IEEE80211_HT_PARAM_CHAN_WIDTH_ANY)) {
-				/* switch to the 40M Hz mode according to AP */
-				pmlmeext->cur_bwmode = HT_CHANNEL_WIDTH_40;
-				switch (pht_info->ht_param &
-					IEEE80211_HT_PARAM_CHA_SEC_OFFSET) {
-				case IEEE80211_HT_PARAM_CHA_SEC_ABOVE:
-					pmlmeext->cur_ch_offset =
-						HAL_PRIME_CHNL_OFFSET_LOWER;
-					break;
-
-				case IEEE80211_HT_PARAM_CHA_SEC_BELOW:
-					pmlmeext->cur_ch_offset =
-						HAL_PRIME_CHNL_OFFSET_UPPER;
-					break;
-
-				default:
-					pmlmeext->cur_ch_offset =
-						HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-					break;
-				}
-
-				DBG_8723A("set ch/bw before connected\n");
-			}
-			break;
-
-		default:
-			break;
-		}
-
-		i += (p[1] + 2);
-	}
-
-	hw_var_set_bssid(padapter, pmlmeinfo->network.MacAddress);
-	hw_var_set_mlme_join(padapter, 0);
-
-	/* cancel link timer */
-	del_timer_sync(&pmlmeext->link_timer);
-
-	start_clnt_join(padapter);
-
-	return H2C_SUCCESS;
-}
-
-int disconnect_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	const struct disconnect_parm *param = (struct disconnect_parm *)pbuf;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork = &pmlmeinfo->network;
-
-	if (is_client_associated_to_ap23a(padapter)) {
-		issue_deauth_ex(padapter, pnetwork->MacAddress,
-				WLAN_REASON_DEAUTH_LEAVING,
-				param->deauth_timeout_ms/100, 100);
-	}
-
-	/* set_opmode_cmd(padapter, infra_client_with_mlme); */
-
-	/* pmlmeinfo->state = MSR_NOLINK; */
-
-	hw_var_set_mlme_disconnect(padapter);
-	hw_var_set_bssid(padapter, null_addr);
-
-	/* restore to initial setting. */
-	update_tx_basic_rate23a(padapter, padapter->registrypriv.wireless_mode);
-
-	if ((pmlmeinfo->state & 0x03) == MSR_ADHOC ||
-	    (pmlmeinfo->state & 0x03) == MSR_AP)
-		rtl8723a_set_bcn_func(padapter, 0);	/* Stop BCN */
-
-	/* set MSR to no link state -> infra. mode */
-	rtl8723a_set_media_status(padapter, MSR_INFRA);
-
-	pmlmeinfo->state = MSR_NOLINK;
-
-	/* switch to the 20M Hz mode after disconnect */
-	pmlmeext->cur_bwmode = HT_CHANNEL_WIDTH_20;
-	pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-
-	set_channel_bwmode23a(padapter, pmlmeext->cur_channel,
-			      pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode);
-
-	flush_all_cam_entry23a(padapter);
-
-	del_timer_sync(&pmlmeext->link_timer);
-
-	rtw_free_uc_swdec_pending_queue23a(padapter);
-
-	return H2C_SUCCESS;
-}
-
-static int
-rtw_scan_ch_decision(struct rtw_adapter *padapter,
-		     struct rtw_ieee80211_channel *out, u32 out_num,
-		     const struct rtw_ieee80211_channel *in, u32 in_num)
-{
-	int i, j;
-	int scan_ch_num = 0;
-	int set_idx;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	/* clear out first */
-	memset(out, 0, sizeof(struct rtw_ieee80211_channel)*out_num);
-
-	/* acquire channels from in */
-	j = 0;
-	for (i = 0;i<in_num;i++) {
-		if (in[i].hw_value &&
-		    !(in[i].flags & IEEE80211_CHAN_DISABLED) &&
-		    (set_idx = rtw_ch_set_search_ch23a(pmlmeext->channel_set,
-						       in[i].hw_value)) >= 0) {
-			memcpy(&out[j], &in[i],
-			       sizeof(struct rtw_ieee80211_channel));
-
-			if (pmlmeext->channel_set[set_idx].ScanType ==
-			    SCAN_PASSIVE)
-				out[j].flags &= IEEE80211_CHAN_NO_IR;
-
-			j++;
-		}
-		if (j>= out_num)
-			break;
-	}
-
-	/* if out is empty, use channel_set as default */
-	if (j == 0) {
-		for (i = 0;i<pmlmeext->max_chan_nums;i++) {
-			out[i].hw_value = pmlmeext->channel_set[i].ChannelNum;
-
-			if (pmlmeext->channel_set[i].ScanType == SCAN_PASSIVE)
-				out[i].flags &= IEEE80211_CHAN_NO_IR;
-
-			j++;
-		}
-	}
-
-	if (padapter->setband == GHZ_24) {			/*  2.4G */
-		for (i = 0; i < j ; i++) {
-			if (out[i].hw_value > 35)
-				memset(&out[i], 0,
-				       sizeof(struct rtw_ieee80211_channel));
-			else
-				scan_ch_num++;
-		}
-		j = scan_ch_num;
-	} else if  (padapter->setband == GHZ_50) {		/*  5G */
-		for (i = 0; i < j ; i++) {
-			if (out[i].hw_value > 35) {
-				memcpy(&out[scan_ch_num++], &out[i],
-				       sizeof(struct rtw_ieee80211_channel));
-			}
-		}
-		j = scan_ch_num;
-	} else
-		{}
-
-	return j;
-}
-
-int sitesurvey_cmd_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	const struct sitesurvey_parm *pparm = (struct sitesurvey_parm *)pbuf;
-	u8 bdelayscan = false;
-	u32 initialgain;
-	u32 i;
-
-	if (pmlmeext->sitesurvey_res.state == SCAN_DISABLE) {
-		pmlmeext->sitesurvey_res.state = SCAN_START;
-		pmlmeext->sitesurvey_res.bss_cnt = 0;
-		pmlmeext->sitesurvey_res.channel_idx = 0;
-
-		for (i = 0; i < RTW_SSID_SCAN_AMOUNT; i++) {
-			if (pparm->ssid[i].ssid_len) {
-				memcpy(pmlmeext->sitesurvey_res.ssid[i].ssid,
-				       pparm->ssid[i].ssid,
-				       IEEE80211_MAX_SSID_LEN);
-				pmlmeext->sitesurvey_res.ssid[i].ssid_len =
-					pparm->ssid[i].ssid_len;
-			} else {
-				pmlmeext->sitesurvey_res.ssid[i].ssid_len = 0;
-			}
-		}
-
-		pmlmeext->sitesurvey_res.ch_num =
-			rtw_scan_ch_decision(padapter,
-					     pmlmeext->sitesurvey_res.ch,
-					     RTW_CHANNEL_SCAN_AMOUNT,
-					     pparm->ch, pparm->ch_num);
-
-		pmlmeext->sitesurvey_res.scan_mode = pparm->scan_mode;
-
-		/* issue null data if associating to the AP */
-		if (is_client_associated_to_ap23a(padapter)) {
-			pmlmeext->sitesurvey_res.state = SCAN_TXNULL;
-
-			/* switch to correct channel of current network
-			   before issue keep-alive frames */
-			if (rtw_get_oper_ch23a(padapter) !=
-			    pmlmeext->cur_channel)
-				SelectChannel23a(padapter,
-						 pmlmeext->cur_channel);
-
-			issue_nulldata23a(padapter, NULL, 1, 3, 500);
-
-			bdelayscan = true;
-		}
-
-		if (bdelayscan) {
-			/* delay 50ms to protect nulldata(1). */
-			set_survey_timer(pmlmeext, 50);
-			return H2C_SUCCESS;
-		}
-	}
-
-	if (pmlmeext->sitesurvey_res.state == SCAN_START ||
-	    pmlmeext->sitesurvey_res.state == SCAN_TXNULL) {
-		/* disable dynamic functions, such as high power, DIG */
-		rtl8723a_odm_support_ability_backup(padapter);
-		rtl8723a_odm_support_ability_clr(padapter,
-						 DYNAMIC_FUNC_DISABLE);
-
-		/* config the initial gain under scanning, need to
-		   write the BB registers */
-		if (wdev_to_priv(padapter->rtw_wdev)->p2p_enabled == true)
-			initialgain = 0x30;
-		else
-			initialgain = 0x1E;
-
-		rtl8723a_set_initial_gain(padapter, initialgain);
-
-		/* set MSR to no link state */
-		rtl8723a_set_media_status(padapter, MSR_NOLINK);
-
-		rtl8723a_mlme_sitesurvey(padapter, 1);
-
-		pmlmeext->sitesurvey_res.state = SCAN_PROCESS;
-	}
-
-	rtw_site_survey(padapter);
-
-	return H2C_SUCCESS;
-}
-
-int setauth_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	const struct setauth_parm *pparm = (struct setauth_parm *)pbuf;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (pparm->mode < 4)
-		pmlmeinfo->auth_algo = pparm->mode;
-
-	return H2C_SUCCESS;
-}
-
-int setkey_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	unsigned short ctrl;
-	const struct setkey_parm *pparm = (struct setkey_parm *)pbuf;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	unsigned char null_sta[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
-
-	/* main tx key for wep. */
-	if (pparm->set_tx)
-		pmlmeinfo->key_index = pparm->keyid;
-
-	/* write cam */
-	ctrl = BIT(15) | (pparm->algorithm) << 2 | pparm->keyid;
-
-	DBG_8723A_LEVEL(_drv_always_, "set group key to hw: alg:%d(WEP40-1 "
-			"WEP104-5 TKIP-2 AES-4) keyid:%d\n",
-			pparm->algorithm, pparm->keyid);
-	rtl8723a_cam_write(padapter, pparm->keyid, ctrl, null_sta, pparm->key);
-
-	/* allow multicast packets to driver */
-	rtl8723a_on_rcr_am(padapter);
-
-	return H2C_SUCCESS;
-}
-
-int set_stakey_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	u16 ctrl = 0;
-	u8 cam_id;/* cam_entry */
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	const struct set_stakey_parm *pparm = (struct set_stakey_parm *)pbuf;
-
-	/* cam_entry: */
-	/* 0~3 for default key */
-
-	/* for concurrent mode (ap+sta): */
-	/* default key is disable, using sw encrypt/decrypt */
-	/* cam_entry = 4  for sta mode (macid = 0) */
-	/* cam_entry(macid+3) = 5 ~ N for ap mode (aid = 1~N, macid = 2 ~N) */
-
-	/* for concurrent mode (sta+sta): */
-	/* default key is disable, using sw encrypt/decrypt */
-	/* cam_entry = 4 mapping to macid = 0 */
-	/* cam_entry = 5 mapping to macid = 2 */
-
-	cam_id = 4;
-
-	DBG_8723A_LEVEL(_drv_always_, "set pairwise key to hw: alg:%d(WEP40-1 "
-			"WEP104-5 TKIP-2 AES-4) camid:%d\n",
-			pparm->algorithm, cam_id);
-	if ((pmlmeinfo->state & 0x03) == MSR_AP) {
-		struct sta_info *psta;
-		struct sta_priv *pstapriv = &padapter->stapriv;
-
-		if (pparm->algorithm == 0) {	/*  clear cam entry */
-			clear_cam_entry23a(padapter, pparm->id);
-			return H2C_SUCCESS_RSP;
-		}
-
-		psta = rtw_get_stainfo23a(pstapriv, pparm->addr);
-		if (psta) {
-			ctrl = BIT(15) | (pparm->algorithm << 2);
-
-			DBG_8723A("r871x_set_stakey_hdl23a(): enc_algorithm "
-				  "=%d\n", pparm->algorithm);
-
-			if (psta->mac_id < 1 || psta->mac_id > (NUM_STA - 4)) {
-				DBG_8723A("r871x_set_stakey_hdl23a():set_stakey"
-					  " failed, mac_id(aid) =%d\n",
-					  psta->mac_id);
-				return H2C_REJECTED;
-			}
-
-			/* 0~3 for default key, cmd_id = macid + 3,
-			   macid = aid+1; */
-			cam_id = psta->mac_id + 3;
-
-			DBG_8723A("Write CAM, mac_addr =%pM, "
-				  "cam_entry =%d\n", pparm->addr, cam_id);
-
-			rtl8723a_cam_write(padapter, cam_id, ctrl,
-					   pparm->addr, pparm->key);
-
-			return H2C_SUCCESS_RSP;
-		} else {
-			DBG_8723A("r871x_set_stakey_hdl23a(): sta has been "
-				  "free\n");
-			return H2C_REJECTED;
-		}
-	}
-
-	/* below for sta mode */
-
-	if (pparm->algorithm == 0) {	/*  clear cam entry */
-		clear_cam_entry23a(padapter, pparm->id);
-		return H2C_SUCCESS;
-	}
-
-	ctrl = BIT(15) | (pparm->algorithm << 2);
-
-	rtl8723a_cam_write(padapter, cam_id, ctrl, pparm->addr, pparm->key);
-
-	pmlmeinfo->enc_algo = pparm->algorithm;
-
-	return H2C_SUCCESS;
-}
-
-int add_ba_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	const struct addBaReq_parm *pparm = (struct addBaReq_parm *)pbuf;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct sta_info *psta;
-
-	psta = rtw_get_stainfo23a(&padapter->stapriv, pparm->addr);
-
-	if (!psta)
-		return H2C_SUCCESS;
-
-	if (((pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) &&
-	     pmlmeinfo->HT_enable) ||
-	    (pmlmeinfo->state & 0x03) == MSR_AP) {
-		issue_action_BA23a(padapter, pparm->addr,
-				   WLAN_ACTION_ADDBA_REQ, (u16)pparm->tid);
-		mod_timer(&psta->addba_retry_timer,
-			  jiffies + msecs_to_jiffies(ADDBA_TO));
-	} else
-		psta->htpriv.candidate_tid_bitmap &= ~BIT(pparm->tid);
-
-	return H2C_SUCCESS;
-}
-
-int set_tx_beacon_cmd23a(struct rtw_adapter *padapter)
-{
-	struct cmd_obj *ph2c;
-	struct Tx_Beacon_param *ptxBeacon_parm;
-	struct cmd_priv	*pcmdpriv = &padapter->cmdpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	u8 res = _SUCCESS;
-	int len_diff = 0;
-
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_ATOMIC);
-	if (!ph2c) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	ptxBeacon_parm = kzalloc(sizeof(struct Tx_Beacon_param), GFP_ATOMIC);
-	if (!ptxBeacon_parm) {
-		kfree(ph2c);
-		res = _FAIL;
-		goto exit;
-	}
-
-	memcpy(&ptxBeacon_parm->network, &pmlmeinfo->network,
-	       sizeof(struct wlan_bssid_ex));
-
-	len_diff = update_hidden_ssid(ptxBeacon_parm->network.IEs,
-				      ptxBeacon_parm->network.IELength,
-				      pmlmeinfo->hidden_ssid_mode);
-	ptxBeacon_parm->network.IELength += len_diff;
-
-	init_h2fwcmd_w_parm_no_rsp(ph2c, ptxBeacon_parm,
-				   GEN_CMD_CODE(_TX_Beacon));
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-
-exit:
-	return res;
-}
-
-int mlme_evt_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	u8 evt_code, evt_seq;
-	u16 evt_sz;
-	const struct C2HEvent_Header *c2h;
-	void (*event_callback)(struct rtw_adapter *dev, const u8 *pbuf);
-
-	c2h = (struct C2HEvent_Header *)pbuf;
-	evt_sz = c2h->len;
-	evt_seq = c2h->seq;
-	evt_code = c2h->ID;
-
-	/*  checking if event code is valid */
-	if (evt_code >= MAX_C2HEVT) {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "Event Code(%d) mismatch!\n", evt_code);
-		goto _abort_event_;
-	}
-
-	/*  checking if event size match the event parm size */
-	if (wlanevents[evt_code].parmsize != 0 &&
-	    wlanevents[evt_code].parmsize != evt_sz) {
-		RT_TRACE(_module_rtl871x_cmd_c_, _drv_err_,
-			 "Event(%d) Parm Size mismatch (%d vs %d)!\n",
-			 evt_code, wlanevents[evt_code].parmsize, evt_sz);
-		goto _abort_event_;
-	}
-
-	event_callback = wlanevents[evt_code].event_callback;
-	event_callback(padapter, pbuf + sizeof(struct C2HEvent_Header));
-
-_abort_event_:
-
-	return H2C_SUCCESS;
-}
-
-int h2c_msg_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	if (!pbuf)
-		return H2C_PARAMETERS_ERROR;
-
-	return H2C_SUCCESS;
-}
-
-int tx_beacon_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	if (send_beacon23a(padapter) == _FAIL) {
-		DBG_8723A("issue_beacon23a, fail!\n");
-		return H2C_PARAMETERS_ERROR;
-	}
-#ifdef CONFIG_8723AU_AP_MODE
-	else { /* tx bc/mc frames after update TIM */
-		struct sta_info *psta_bmc;
-		struct list_head *phead;
-		struct xmit_frame *pxmitframe, *ptmp;
-		struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-		struct sta_priv *pstapriv = &padapter->stapriv;
-
-		/* for BC/MC Frames */
-		psta_bmc = rtw_get_bcmc_stainfo23a(padapter);
-		if (!psta_bmc)
-			return H2C_SUCCESS;
-
-		if (pstapriv->tim_bitmap & BIT(0) && psta_bmc->sleepq_len > 0) {
-			msleep(10);/*  10ms, ATIM(HIQ) Windows */
-			/* spin_lock_bh(&psta_bmc->sleep_q.lock); */
-			spin_lock_bh(&pxmitpriv->lock);
-
-			phead = get_list_head(&psta_bmc->sleep_q);
-
-			list_for_each_entry_safe(pxmitframe, ptmp,
-						 phead, list) {
-
-				list_del_init(&pxmitframe->list);
-
-				psta_bmc->sleepq_len--;
-				if (psta_bmc->sleepq_len>0)
-					pxmitframe->attrib.mdata = 1;
-				else
-					pxmitframe->attrib.mdata = 0;
-
-				pxmitframe->attrib.triggered = 1;
-
-				pxmitframe->attrib.qsel = 0x11;/* HIQ */
-
-				rtl8723au_hal_xmitframe_enqueue(padapter,
-								pxmitframe);
-			}
-			/* spin_unlock_bh(&psta_bmc->sleep_q.lock); */
-			spin_unlock_bh(&pxmitpriv->lock);
-		}
-	}
-#endif
-
-	return H2C_SUCCESS;
-}
-
-int set_ch_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	const struct set_ch_parm *set_ch_parm;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	if (!pbuf)
-		return H2C_PARAMETERS_ERROR;
-
-	set_ch_parm = (struct set_ch_parm *)pbuf;
-
-	DBG_8723A("%s(%s): ch:%u, bw:%u, ch_offset:%u\n", __func__,
-		  padapter->pnetdev->name, set_ch_parm->ch,
-		  set_ch_parm->bw, set_ch_parm->ch_offset);
-
-	pmlmeext->cur_channel = set_ch_parm->ch;
-	pmlmeext->cur_ch_offset = set_ch_parm->ch_offset;
-	pmlmeext->cur_bwmode = set_ch_parm->bw;
-
-	set_channel_bwmode23a(padapter, set_ch_parm->ch,
-			      set_ch_parm->ch_offset, set_ch_parm->bw);
-
-	return H2C_SUCCESS;
-}
-
-int set_chplan_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	const struct SetChannelPlan_param *setChannelPlan_param;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	if (!pbuf)
-		return H2C_PARAMETERS_ERROR;
-
-	setChannelPlan_param = (struct SetChannelPlan_param *)pbuf;
-
-	pmlmeext->max_chan_nums =
-		init_channel_set(padapter, setChannelPlan_param->channel_plan,
-				 pmlmeext->channel_set);
-	init_channel_list(padapter, pmlmeext->channel_set,
-			  pmlmeext->max_chan_nums, &pmlmeext->channel_list);
-
-	return H2C_SUCCESS;
-}
-
-int led_blink_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	struct LedBlink_param *ledBlink_param;
-
-	if (!pbuf)
-		return H2C_PARAMETERS_ERROR;
-
-	ledBlink_param = (struct LedBlink_param *)pbuf;
-
-	return H2C_SUCCESS;
-}
-
-int set_csa_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	return H2C_REJECTED;
-}
-
-/*  TDLS_WRCR		: write RCR DATA BIT */
-/*  TDLS_SD_PTI		: issue peer traffic indication */
-/*  TDLS_CS_OFF		: go back to the channel linked with AP,
-			  terminating channel switch procedure */
-/*  TDLS_INIT_CH_SEN	: init channel sensing, receive all data and
-			  mgnt frame */
-/*  TDLS_DONE_CH_SEN	: channel sensing and report candidate channel */
-/*  TDLS_OFF_CH		: first time set channel to off channel */
-/*  TDLS_BASE_CH	: go back tp the channel linked with AP when set
-			  base channel as target channel */
-/*  TDLS_P_OFF_CH	: periodically go to off channel */
-/*  TDLS_P_BASE_CH	: periodically go back to base channel */
-/*  TDLS_RS_RCR		: restore RCR */
-/*  TDLS_CKALV_PH1	: check alive timer phase1 */
-/*  TDLS_CKALV_PH2	: check alive timer phase2 */
-/*  TDLS_FREE_STA	: free tdls sta */
-int tdls_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf)
-{
-	return H2C_REJECTED;
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_pwrctrl.c b/drivers/staging/rtl8723au/core/rtw_pwrctrl.c
deleted file mode 100644
index 7488a104935b..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_pwrctrl.c
+++ /dev/null
@@ -1,606 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_PWRCTRL_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <osdep_intf.h>
-#include <rtl8723a_cmd.h>
-#include <rtw_sreset.h>
-
-#include <rtl8723a_bt_intf.h>
-#include <usb_ops_linux.h>
-
-void ips_enter23a(struct rtw_adapter *padapter)
-{
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-
-	down(&pwrpriv->lock);
-
-	pwrpriv->bips_processing = true;
-
-	/*  syn ips_mode with request */
-	pwrpriv->ips_mode = pwrpriv->ips_mode_req;
-
-	pwrpriv->ips_enter23a_cnts++;
-	DBG_8723A("==>ips_enter23a cnts:%d\n", pwrpriv->ips_enter23a_cnts);
-	rtl8723a_BT_disable_coexist(padapter);
-
-	if (pwrpriv->change_rfpwrstate == rf_off) {
-		pwrpriv->bpower_saving = true;
-		DBG_8723A_LEVEL(_drv_always_, "nolinked power save enter\n");
-
-		if (pwrpriv->ips_mode == IPS_LEVEL_2)
-			pwrpriv->bkeepfwalive = true;
-
-		rtw_ips_pwr_down23a(padapter);
-		pwrpriv->rf_pwrstate = rf_off;
-	}
-	pwrpriv->bips_processing = false;
-
-	up(&pwrpriv->lock);
-}
-
-int ips_leave23a(struct rtw_adapter *padapter)
-{
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	int result = _SUCCESS;
-	int keyid;
-
-	down(&pwrpriv->lock);
-
-	if (pwrpriv->rf_pwrstate == rf_off && !pwrpriv->bips_processing) {
-		pwrpriv->bips_processing = true;
-		pwrpriv->change_rfpwrstate = rf_on;
-		pwrpriv->ips_leave23a_cnts++;
-		DBG_8723A("==>ips_leave23a cnts:%d\n",
-			  pwrpriv->ips_leave23a_cnts);
-
-		result = rtw_ips_pwr_up23a(padapter);
-		if (result == _SUCCESS)
-			pwrpriv->rf_pwrstate = rf_on;
-
-		DBG_8723A_LEVEL(_drv_always_, "nolinked power save leave\n");
-
-		if (psecuritypriv->dot11PrivacyAlgrthm ==
-		    WLAN_CIPHER_SUITE_WEP40 ||
-		    psecuritypriv->dot11PrivacyAlgrthm ==
-		    WLAN_CIPHER_SUITE_WEP104) {
-			DBG_8723A("==>%s, channel(%d), processing(%x)\n",
-				  __func__, padapter->mlmeextpriv.cur_channel,
-				  pwrpriv->bips_processing);
-			set_channel_bwmode23a(padapter,
-					      padapter->mlmeextpriv.cur_channel,
-					      HAL_PRIME_CHNL_OFFSET_DONT_CARE,
-					      HT_CHANNEL_WIDTH_20);
-			for (keyid = 0; keyid < 4; keyid++) {
-				if (pmlmepriv->key_mask & BIT(keyid)) {
-					if (keyid ==
-					    psecuritypriv->dot11PrivacyKeyIndex)
-						result = rtw_set_key23a(padapter, psecuritypriv, keyid, 1);
-					else
-						result = rtw_set_key23a(padapter, psecuritypriv, keyid, 0);
-				}
-			}
-		}
-
-		DBG_8723A("==> ips_leave23a.....LED(0x%08x)...\n",
-			  rtl8723au_read32(padapter, 0x4c));
-		pwrpriv->bips_processing = false;
-
-		pwrpriv->bkeepfwalive = false;
-		pwrpriv->bpower_saving = false;
-	}
-
-	up(&pwrpriv->lock);
-
-	return result;
-}
-
-
-static bool rtw_pwr_unassociated_idle(struct rtw_adapter *adapter)
-{
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct xmit_priv *pxmit_priv = &adapter->xmitpriv;
-
-	bool ret = false;
-
-	if (time_after_eq(adapter->pwrctrlpriv.ips_deny_time, jiffies))
-		goto exit;
-
-	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE|WIFI_SITE_MONITOR) ||
-	    check_fwstate(pmlmepriv, WIFI_UNDER_LINKING|WIFI_UNDER_WPS) ||
-	    check_fwstate(pmlmepriv, WIFI_AP_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE|WIFI_ADHOC_STATE)){
-		goto exit;
-	}
-
-	if (pxmit_priv->free_xmitbuf_cnt != NR_XMITBUFF ||
-		pxmit_priv->free_xmit_extbuf_cnt != NR_XMIT_EXTBUFF) {
-		DBG_8723A_LEVEL(_drv_always_,
-				"There are some pkts to transmit\n");
-		DBG_8723A_LEVEL(_drv_info_, "free_xmitbuf_cnt: %d, "
-				"free_xmit_extbuf_cnt: %d\n",
-				pxmit_priv->free_xmitbuf_cnt,
-				pxmit_priv->free_xmit_extbuf_cnt);
-		goto exit;
-	}
-
-	ret = true;
-
-exit:
-	return ret;
-}
-
-void rtw_ps_processor23a(struct rtw_adapter *padapter)
-{
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	pwrpriv->ps_processing = true;
-
-	if (pwrpriv->bips_processing == true)
-		goto exit;
-
-	if (pwrpriv->ips_mode_req == IPS_NONE)
-		goto exit;
-
-	if (!rtw_pwr_unassociated_idle(padapter))
-		goto exit;
-
-	if (pwrpriv->rf_pwrstate == rf_on &&
-	    (pwrpriv->pwr_state_check_cnts % 4) == 0) {
-		DBG_8723A("==>%s .fw_state(%x)\n", __func__,
-			  get_fwstate(pmlmepriv));
-		pwrpriv->change_rfpwrstate = rf_off;
-		ips_enter23a(padapter);
-	}
-exit:
-	rtw_set_pwr_state_check_timer(&padapter->pwrctrlpriv);
-	pwrpriv->ps_processing = false;
-}
-
-static void pwr_state_check_handler(unsigned long data)
-{
-	struct rtw_adapter *padapter = (struct rtw_adapter *)data;
-
-	rtw_ps_cmd23a(padapter);
-}
-
-/*
- *
- * Parameters
- *   padapter
- *   pslv	power state level, only could be PS_STATE_S0 ~ PS_STATE_S4
- *
- */
-void rtw_set_rpwm23a(struct rtw_adapter *padapter, u8 pslv)
-{
-	u8 rpwm;
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-
-	pslv = PS_STATE(pslv);
-
-	if (pwrpriv->btcoex_rfon) {
-		if (pslv < PS_STATE_S4)
-			pslv = PS_STATE_S3;
-	}
-
-	if (pwrpriv->rpwm == pslv) {
-		RT_TRACE(_module_rtl871x_pwrctrl_c_, _drv_err_,
-			 "%s: Already set rpwm[0x%02X], new = 0x%02X!\n",
-			 __func__, pwrpriv->rpwm, pslv);
-		return;
-	}
-
-	if (padapter->bSurpriseRemoved == true ||
-	    padapter->hw_init_completed == false) {
-		RT_TRACE(_module_rtl871x_pwrctrl_c_, _drv_err_,
-			 "%s: SurpriseRemoved(%d) hw_init_completed(%d)\n",
-			 __func__, padapter->bSurpriseRemoved,
-			 padapter->hw_init_completed);
-
-		pwrpriv->cpwm = PS_STATE_S4;
-
-		return;
-	}
-
-	if (padapter->bDriverStopped == true) {
-		RT_TRACE(_module_rtl871x_pwrctrl_c_, _drv_err_,
-			 "%s: change power state(0x%02X) when DriverStopped\n",
-			 __func__, pslv);
-
-		if (pslv < PS_STATE_S2) {
-			RT_TRACE(_module_rtl871x_pwrctrl_c_, _drv_err_,
-				 "%s: Reject to enter PS_STATE(0x%02X) lower than S2 when DriverStopped!!\n",
-				 __func__, pslv);
-			return;
-		}
-	}
-
-	rpwm = pslv | pwrpriv->tog;
-	RT_TRACE(_module_rtl871x_pwrctrl_c_, _drv_notice_,
-		 "rtw_set_rpwm23a: rpwm = 0x%02x cpwm = 0x%02x\n",
-		 rpwm, pwrpriv->cpwm);
-
-	pwrpriv->rpwm = pslv;
-
-	rtl8723a_set_rpwm(padapter, rpwm);
-
-	pwrpriv->tog += 0x80;
-	pwrpriv->cpwm = pslv;
-}
-
-static bool PS_RDY_CHECK(struct rtw_adapter *padapter)
-{
-	unsigned long delta_time;
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	delta_time = jiffies - pwrpriv->DelayLPSLastTimeStamp;
-
-	if (delta_time < LPS_DELAY_TIME)
-		return false;
-
-	if (!check_fwstate(pmlmepriv, _FW_LINKED) ||
-	    check_fwstate(pmlmepriv, _FW_UNDER_SURVEY) ||
-	    check_fwstate(pmlmepriv, WIFI_AP_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE))
-		return false;
-	if (pwrpriv->bInSuspend)
-		return false;
-	if (padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X &&
-	    !padapter->securitypriv.binstallGrpkey) {
-		DBG_8723A("Group handshake still in progress !!!\n");
-		return false;
-	}
-	if (!rtw_cfg80211_pwr_mgmt(padapter))
-		return false;
-
-	return true;
-}
-
-void rtw_set_ps_mode23a(struct rtw_adapter *padapter, u8 ps_mode,
-			u8 smart_ps, u8 bcn_ant_mode)
-{
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-
-	RT_TRACE(_module_rtl871x_pwrctrl_c_, _drv_notice_,
-		 "%s: PowerMode =%d Smart_PS =%d\n",
-		 __func__, ps_mode, smart_ps);
-
-	if (ps_mode > PM_Card_Disable) {
-		RT_TRACE(_module_rtl871x_pwrctrl_c_, _drv_err_,
-			 "ps_mode:%d error\n", ps_mode);
-		return;
-	}
-
-	if (pwrpriv->pwr_mode == ps_mode) {
-		if (PS_MODE_ACTIVE == ps_mode)
-			return;
-
-		if (pwrpriv->smart_ps == smart_ps &&
-		    pwrpriv->bcn_ant_mode == bcn_ant_mode)
-			return;
-	}
-
-	if (ps_mode == PS_MODE_ACTIVE) {
-		DBG_8723A("rtw_set_ps_mode23a: Leave 802.11 power save\n");
-
-		pwrpriv->pwr_mode = ps_mode;
-		rtw_set_rpwm23a(padapter, PS_STATE_S4);
-		rtl8723a_set_FwPwrMode_cmd(padapter, ps_mode);
-		pwrpriv->bFwCurrentInPSMode = false;
-	} else {
-		if (PS_RDY_CHECK(padapter) ||
-		    rtl8723a_BT_using_antenna_1(padapter)) {
-			DBG_8723A("%s: Enter 802.11 power save\n", __func__);
-
-			pwrpriv->bFwCurrentInPSMode = true;
-			pwrpriv->pwr_mode = ps_mode;
-			pwrpriv->smart_ps = smart_ps;
-			pwrpriv->bcn_ant_mode = bcn_ant_mode;
-			rtl8723a_set_FwPwrMode_cmd(padapter, ps_mode);
-
-			rtw_set_rpwm23a(padapter, PS_STATE_S2);
-		}
-	}
-}
-
-/*
- * Return:
- *	0:	Leave OK
- *	-1:	Timeout
- *	-2:	Other error
- */
-s32 LPS_RF_ON_check23a(struct rtw_adapter *padapter, u32 delay_ms)
-{
-	unsigned long start_time, end_time;
-	u8 bAwake = false;
-	s32 err = 0;
-
-	start_time = jiffies;
-	end_time = start_time + msecs_to_jiffies(delay_ms);
-
-	while (1) {
-		bAwake = rtl8723a_get_fwlps_rf_on(padapter);
-		if (bAwake == true)
-			break;
-
-		if (padapter->bSurpriseRemoved == true) {
-			err = -2;
-			DBG_8723A("%s: device surprise removed!!\n", __func__);
-			break;
-		}
-
-		if (time_after(jiffies, end_time)) {
-			err = -1;
-			DBG_8723A("%s: Wait for FW LPS leave more than %u "
-				  "ms!\n", __func__, delay_ms);
-			break;
-		}
-		udelay(100);
-	}
-
-	return err;
-}
-
-/*	Description: */
-/*		Enter the leisure power save mode. */
-void LPS_Enter23a(struct rtw_adapter *padapter)
-{
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-
-	if (!PS_RDY_CHECK(padapter))
-		return;
-
-	if (pwrpriv->bLeisurePs) {
-		/*  Idle for a while if we connect to AP a while ago. */
-		if (pwrpriv->LpsIdleCount >= 2) { /*   4 Sec */
-			if (pwrpriv->pwr_mode == PS_MODE_ACTIVE) {
-				pwrpriv->bpower_saving = true;
-				DBG_8723A("%s smart_ps:%d\n", __func__,
-					  pwrpriv->smart_ps);
-				/* For Tenda W311R IOT issue */
-				rtw_set_ps_mode23a(padapter,
-						   pwrpriv->power_mgnt,
-						   pwrpriv->smart_ps, 0);
-			}
-		} else
-			pwrpriv->LpsIdleCount++;
-	}
-}
-
-/*	Description: */
-/*		Leave the leisure power save mode. */
-void LPS_Leave23a(struct rtw_adapter *padapter)
-{
-#define LPS_LEAVE_TIMEOUT_MS 100
-
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-
-	if (pwrpriv->bLeisurePs) {
-		if (pwrpriv->pwr_mode != PS_MODE_ACTIVE) {
-			rtw_set_ps_mode23a(padapter, PS_MODE_ACTIVE, 0, 0);
-
-			if (pwrpriv->pwr_mode == PS_MODE_ACTIVE)
-				LPS_RF_ON_check23a(padapter,
-						   LPS_LEAVE_TIMEOUT_MS);
-		}
-	}
-
-	pwrpriv->bpower_saving = false;
-}
-
-/*  Description: Leave all power save mode: LPS, FwLPS, IPS if needed. */
-/*  Move code to function by tynli. 2010.03.26. */
-void LeaveAllPowerSaveMode23a(struct rtw_adapter *Adapter)
-{
-	struct mlme_priv *pmlmepriv = &Adapter->mlmepriv;
-	u8 enqueue = 0;
-
-	/* DBG_8723A("%s.....\n", __func__); */
-	if (check_fwstate(pmlmepriv, _FW_LINKED))
-		rtw_lps_ctrl_wk_cmd23a(Adapter, LPS_CTRL_LEAVE, enqueue);
-}
-
-void rtw_init_pwrctrl_priv23a(struct rtw_adapter *padapter)
-{
-	struct pwrctrl_priv *pwrctrlpriv = &padapter->pwrctrlpriv;
-
-	sema_init(&pwrctrlpriv->lock, 1);
-	pwrctrlpriv->rf_pwrstate = rf_on;
-	pwrctrlpriv->ips_enter23a_cnts = 0;
-	pwrctrlpriv->ips_leave23a_cnts = 0;
-	pwrctrlpriv->bips_processing = false;
-
-	pwrctrlpriv->ips_mode = padapter->registrypriv.ips_mode;
-	pwrctrlpriv->ips_mode_req = padapter->registrypriv.ips_mode;
-
-	pwrctrlpriv->pwr_state_check_interval = RTW_PWR_STATE_CHK_INTERVAL;
-	pwrctrlpriv->pwr_state_check_cnts = 0;
-	pwrctrlpriv->bInSuspend = false;
-	pwrctrlpriv->bkeepfwalive = false;
-
-	pwrctrlpriv->LpsIdleCount = 0;
-
-	/*  PS_MODE_MIN; */
-	pwrctrlpriv->power_mgnt = padapter->registrypriv.power_mgnt;
-	pwrctrlpriv->bLeisurePs =
-		(PS_MODE_ACTIVE != pwrctrlpriv->power_mgnt)?true:false;
-
-	pwrctrlpriv->bFwCurrentInPSMode = false;
-
-	pwrctrlpriv->rpwm = 0;
-	pwrctrlpriv->cpwm = PS_STATE_S4;
-
-	pwrctrlpriv->pwr_mode = PS_MODE_ACTIVE;
-	pwrctrlpriv->smart_ps = padapter->registrypriv.smart_ps;
-	pwrctrlpriv->bcn_ant_mode = 0;
-
-	pwrctrlpriv->tog = 0x80;
-
-	pwrctrlpriv->btcoex_rfon = false;
-
-	setup_timer(&pwrctrlpriv->pwr_state_check_timer,
-		    pwr_state_check_handler, (unsigned long)padapter);
-}
-
-void rtw_free_pwrctrl_priv(struct rtw_adapter *adapter)
-{
-}
-
-inline void rtw_set_ips_deny23a(struct rtw_adapter *padapter, u32 ms)
-{
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-
-	pwrpriv->ips_deny_time = jiffies + msecs_to_jiffies(ms);
-}
-
-/*
-* rtw_pwr_wakeup - Wake the NIC up from: 1)IPS. 2)USB autosuspend
-* @adapter: pointer to _adapter structure
-* @ips_deffer_ms: the ms will prevent from falling into IPS after wakeup
-* Return _SUCCESS or _FAIL
-*/
-
-int _rtw_pwr_wakeup23a(struct rtw_adapter *padapter, u32 ips_deffer_ms, const char *caller)
-{
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	int ret = _SUCCESS;
-	unsigned long start = jiffies;
-	unsigned long new_deny_time;
-
-	new_deny_time = jiffies + msecs_to_jiffies(ips_deffer_ms);
-
-	if (time_before(pwrpriv->ips_deny_time, new_deny_time))
-		pwrpriv->ips_deny_time = new_deny_time;
-
-	if (pwrpriv->ps_processing) {
-		DBG_8723A("%s wait ps_processing...\n", __func__);
-		while (pwrpriv->ps_processing &&
-		       jiffies_to_msecs(jiffies - start) <= 3000)
-			msleep(10);
-		if (pwrpriv->ps_processing)
-			DBG_8723A("%s wait ps_processing timeout\n", __func__);
-		else
-			DBG_8723A("%s wait ps_processing done\n", __func__);
-	}
-
-	if (rtw_sreset_inprogress(padapter)) {
-		DBG_8723A("%s wait sreset_inprogress...\n", __func__);
-		while (rtw_sreset_inprogress(padapter) &&
-		       jiffies_to_msecs(jiffies - start) <= 4000)
-			msleep(10);
-		if (rtw_sreset_inprogress(padapter))
-			DBG_8723A("%s wait sreset_inprogress timeout\n",
-				  __func__);
-		else
-			DBG_8723A("%s wait sreset_inprogress done\n", __func__);
-	}
-
-	if (pwrpriv->bInSuspend) {
-		DBG_8723A("%s wait bInSuspend...\n", __func__);
-		while (pwrpriv->bInSuspend &&
-		       (jiffies_to_msecs(jiffies - start) <= 3000)) {
-			msleep(10);
-		}
-		if (pwrpriv->bInSuspend)
-			DBG_8723A("%s wait bInSuspend timeout\n", __func__);
-		else
-			DBG_8723A("%s wait bInSuspend done\n", __func__);
-	}
-
-	/* System suspend is not allowed to wakeup */
-	if (pwrpriv->bInSuspend) {
-		ret = _FAIL;
-		goto exit;
-	}
-
-	/* I think this should be check in IPS, LPS, autosuspend functions... */
-	if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-		ret = _SUCCESS;
-		goto exit;
-	}
-
-	if (rf_off == pwrpriv->rf_pwrstate) {
-		DBG_8723A("%s call ips_leave23a....\n", __func__);
-		if (ips_leave23a(padapter)== _FAIL) {
-			DBG_8723A("======> ips_leave23a fail.............\n");
-			ret = _FAIL;
-			goto exit;
-		}
-	}
-
-	/* TODO: the following checking need to be merged... */
-	if (padapter->bDriverStopped || !padapter->bup ||
-	    !padapter->hw_init_completed) {
-		DBG_8723A("%s: bDriverStopped =%d, bup =%d, hw_init_completed "
-			  "=%u\n", caller, padapter->bDriverStopped,
-			  padapter->bup, padapter->hw_init_completed);
-		ret = _FAIL;
-		goto exit;
-	}
-
-exit:
-	new_deny_time = jiffies + msecs_to_jiffies(ips_deffer_ms);
-	if (time_before(pwrpriv->ips_deny_time, new_deny_time))
-		pwrpriv->ips_deny_time = new_deny_time;
-	return ret;
-}
-
-int rtw_pm_set_lps23a(struct rtw_adapter *padapter, u8 mode)
-{
-	int ret = 0;
-	struct pwrctrl_priv *pwrctrlpriv = &padapter->pwrctrlpriv;
-
-	if (mode < PS_MODE_NUM) {
-		if (pwrctrlpriv->power_mgnt != mode) {
-			if (PS_MODE_ACTIVE == mode)
-				LeaveAllPowerSaveMode23a(padapter);
-			else
-				pwrctrlpriv->LpsIdleCount = 2;
-			pwrctrlpriv->power_mgnt = mode;
-			pwrctrlpriv->bLeisurePs =
-				(PS_MODE_ACTIVE != pwrctrlpriv->power_mgnt) ?
-				true:false;
-		}
-	} else
-		ret = -EINVAL;
-
-	return ret;
-}
-
-int rtw_pm_set_ips23a(struct rtw_adapter *padapter, u8 mode)
-{
-	struct pwrctrl_priv *pwrctrlpriv = &padapter->pwrctrlpriv;
-
-	if (mode != IPS_NORMAL && mode != IPS_LEVEL_2 && mode != IPS_NONE)
-		return -EINVAL;
-
-	pwrctrlpriv->ips_mode_req = mode;
-	if (mode == IPS_NONE) {
-		DBG_8723A("%s %s\n", __func__, "IPS_NONE");
-		if (padapter->bSurpriseRemoved == 0 &&
-		    rtw_pwr_wakeup(padapter) == _FAIL)
-			return -EFAULT;
-	}
-
-	return 0;
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_recv.c b/drivers/staging/rtl8723au/core/rtw_recv.c
deleted file mode 100644
index 150dabc2a58d..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_recv.c
+++ /dev/null
@@ -1,2204 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_RECV_C_
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <recv_osdep.h>
-#include <mlme_osdep.h>
-#include <linux/ip.h>
-#include <linux/if_ether.h>
-#include <usb_ops.h>
-#include <linux/ieee80211.h>
-#include <wifi.h>
-#include <rtl8723a_recv.h>
-#include <rtl8723a_xmit.h>
-
-void rtw_signal_stat_timer_hdl23a(unsigned long data);
-
-void _rtw_init_sta_recv_priv23a(struct sta_recv_priv *psta_recvpriv)
-{
-
-
-
-	spin_lock_init(&psta_recvpriv->lock);
-
-	/* for (i = 0; i<MAX_RX_NUMBLKS; i++) */
-	/*	_rtw_init_queue23a(&psta_recvpriv->blk_strms[i]); */
-
-	_rtw_init_queue23a(&psta_recvpriv->defrag_q);
-
-
-}
-
-int _rtw_init_recv_priv23a(struct recv_priv *precvpriv,
-			struct rtw_adapter *padapter)
-{
-	struct recv_frame *precvframe;
-	int i;
-	int res = _SUCCESS;
-
-	spin_lock_init(&precvpriv->lock);
-
-	_rtw_init_queue23a(&precvpriv->free_recv_queue);
-	_rtw_init_queue23a(&precvpriv->recv_pending_queue);
-	_rtw_init_queue23a(&precvpriv->uc_swdec_pending_queue);
-
-	precvpriv->adapter = padapter;
-
-	for (i = 0; i < NR_RECVFRAME ; i++) {
-		precvframe = kzalloc(sizeof(struct recv_frame), GFP_KERNEL);
-		if (!precvframe)
-			break;
-		INIT_LIST_HEAD(&precvframe->list);
-
-		list_add_tail(&precvframe->list,
-			      &precvpriv->free_recv_queue.queue);
-
-		precvframe->adapter = padapter;
-		precvframe++;
-	}
-
-	precvpriv->free_recvframe_cnt = i;
-	precvpriv->rx_pending_cnt = 1;
-
-	res = rtl8723au_init_recv_priv(padapter);
-
-	setup_timer(&precvpriv->signal_stat_timer, rtw_signal_stat_timer_hdl23a,
-		    (unsigned long)padapter);
-
-	precvpriv->signal_stat_sampling_interval = 1000; /* ms */
-
-	rtw_set_signal_stat_timer(precvpriv);
-
-	return res;
-}
-
-void _rtw_free_recv_priv23a(struct recv_priv *precvpriv)
-{
-	struct rtw_adapter *padapter = precvpriv->adapter;
-	struct recv_frame *precvframe, *ptmp;
-
-	rtw_free_uc_swdec_pending_queue23a(padapter);
-
-	list_for_each_entry_safe(precvframe, ptmp,
-				 &precvpriv->free_recv_queue.queue, list) {
-		list_del_init(&precvframe->list);
-		kfree(precvframe);
-	}
-
-	rtl8723au_free_recv_priv(padapter);
-}
-
-struct recv_frame *rtw_alloc_recvframe23a(struct rtw_queue *pfree_recv_queue)
-{
-	struct recv_frame *pframe;
-	struct rtw_adapter *padapter;
-	struct recv_priv *precvpriv;
-
-	spin_lock_bh(&pfree_recv_queue->lock);
-
-	pframe = list_first_entry_or_null(&pfree_recv_queue->queue,
-					  struct recv_frame, list);
-	if (pframe) {
-		list_del_init(&pframe->list);
-		padapter = pframe->adapter;
-		if (padapter) {
-			precvpriv = &padapter->recvpriv;
-			if (pfree_recv_queue == &precvpriv->free_recv_queue)
-				precvpriv->free_recvframe_cnt--;
-		}
-	}
-
-	spin_unlock_bh(&pfree_recv_queue->lock);
-
-	return pframe;
-}
-
-int rtw_free_recvframe23a(struct recv_frame *precvframe)
-{
-	struct rtw_adapter *padapter = precvframe->adapter;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-	struct rtw_queue *pfree_recv_queue;
-
-	if (precvframe->pkt) {
-		dev_kfree_skb_any(precvframe->pkt);/* free skb by driver */
-		precvframe->pkt = NULL;
-	}
-
-	pfree_recv_queue = &precvpriv->free_recv_queue;
-	spin_lock_bh(&pfree_recv_queue->lock);
-
-	list_del_init(&precvframe->list);
-
-	list_add_tail(&precvframe->list, get_list_head(pfree_recv_queue));
-
-	if (padapter) {
-		if (pfree_recv_queue == &precvpriv->free_recv_queue)
-			precvpriv->free_recvframe_cnt++;
-	}
-
-	spin_unlock_bh(&pfree_recv_queue->lock);
-
-
-
-	return _SUCCESS;
-}
-
-int rtw_enqueue_recvframe23a(struct recv_frame *precvframe, struct rtw_queue *queue)
-{
-	struct rtw_adapter *padapter = precvframe->adapter;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-
-	spin_lock_bh(&queue->lock);
-
-	list_del_init(&precvframe->list);
-
-	list_add_tail(&precvframe->list, get_list_head(queue));
-
-	if (padapter) {
-		if (queue == &precvpriv->free_recv_queue)
-			precvpriv->free_recvframe_cnt++;
-	}
-
-	spin_unlock_bh(&queue->lock);
-
-	return _SUCCESS;
-}
-
-/*
-caller : defrag ; recvframe_chk_defrag23a in recv_thread  (passive)
-pframequeue: defrag_queue : will be accessed in recv_thread  (passive)
-
-using spinlock to protect
-
-*/
-
-static void rtw_free_recvframe23a_queue(struct rtw_queue *pframequeue)
-{
-	struct recv_frame *hdr, *ptmp;
-	struct list_head *phead;
-
-	spin_lock(&pframequeue->lock);
-	phead = get_list_head(pframequeue);
-	list_for_each_entry_safe(hdr, ptmp, phead, list)
-		rtw_free_recvframe23a(hdr);
-	spin_unlock(&pframequeue->lock);
-}
-
-u32 rtw_free_uc_swdec_pending_queue23a(struct rtw_adapter *adapter)
-{
-	u32 cnt = 0;
-	struct recv_frame *pending_frame;
-
-	while ((pending_frame = rtw_alloc_recvframe23a(&adapter->recvpriv.uc_swdec_pending_queue))) {
-		rtw_free_recvframe23a(pending_frame);
-		DBG_8723A("%s: dequeue uc_swdec_pending_queue\n", __func__);
-		cnt++;
-	}
-
-	return cnt;
-}
-
-struct recv_buf *rtw_dequeue_recvbuf23a (struct rtw_queue *queue)
-{
-	unsigned long irqL;
-	struct recv_buf *precvbuf;
-
-	spin_lock_irqsave(&queue->lock, irqL);
-
-	precvbuf = list_first_entry_or_null(&queue->queue,
-					    struct recv_buf, list);
-	if (precvbuf)
-		list_del_init(&precvbuf->list);
-
-	spin_unlock_irqrestore(&queue->lock, irqL);
-
-	return precvbuf;
-}
-
-int recvframe_chkmic(struct rtw_adapter *adapter,
-		     struct recv_frame *precvframe);
-int recvframe_chkmic(struct rtw_adapter *adapter,
-		     struct recv_frame *precvframe) {
-
-	int	i, res = _SUCCESS;
-	u32	datalen;
-	u8	miccode[8];
-	u8	bmic_err = false, brpt_micerror = true;
-	u8	*pframe, *payload, *pframemic;
-	u8	*mickey;
-	struct	sta_info *stainfo;
-	struct	rx_pkt_attrib *prxattrib = &precvframe->attrib;
-	struct	security_priv *psecuritypriv = &adapter->securitypriv;
-
-	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-
-	stainfo = rtw_get_stainfo23a(&adapter->stapriv, &prxattrib->ta[0]);
-
-	if (prxattrib->encrypt == WLAN_CIPHER_SUITE_TKIP) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-			 "recvframe_chkmic:prxattrib->encrypt == WLAN_CIPHER_SUITE_TKIP\n");
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-			 "recvframe_chkmic:da = %pM\n", prxattrib->ra);
-
-		/* calculate mic code */
-		if (stainfo != NULL) {
-			if (is_multicast_ether_addr(prxattrib->ra)) {
-				mickey = &psecuritypriv->dot118021XGrprxmickey[prxattrib->key_index].skey[0];
-
-				RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-					 "recvframe_chkmic: bcmc key\n");
-
-				if (!psecuritypriv->binstallGrpkey) {
-					res = _FAIL;
-					RT_TRACE(_module_rtl871x_recv_c_,
-						 _drv_err_,
-						 "recvframe_chkmic:didn't install group key!\n");
-					DBG_8723A("\n recvframe_chkmic:didn't "
-						  "install group key!!!!!!\n");
-					goto exit;
-				}
-			} else {
-				mickey = &stainfo->dot11tkiprxmickey.skey[0];
-				RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-					 "recvframe_chkmic: unicast key\n");
-			}
-
-			/* icv_len included the mic code */
-			datalen = precvframe->pkt->len-prxattrib->
-				hdrlen-prxattrib->iv_len-prxattrib->icv_len - 8;
-			pframe = precvframe->pkt->data;
-			payload = pframe + prxattrib->hdrlen +
-				prxattrib->iv_len;
-
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-				 "prxattrib->iv_len =%d prxattrib->icv_len =%d\n",
-				 prxattrib->iv_len, prxattrib->icv_len);
-
-			/* care the length of the data */
-			rtw_seccalctkipmic23a(mickey, pframe, payload,
-					   datalen, &miccode[0],
-					   (unsigned char)prxattrib->priority);
-
-			pframemic = payload + datalen;
-
-			bmic_err = false;
-
-			for (i = 0; i < 8; i++) {
-				if (miccode[i] != *(pframemic + i)) {
-					RT_TRACE(_module_rtl871x_recv_c_,
-						 _drv_err_,
-						 "recvframe_chkmic:miccode[%d](%02x) != *(pframemic+%d)(%02x)\n",
-						 i, miccode[i],
-						 i, *(pframemic + i));
-					bmic_err = true;
-				}
-			}
-
-			if (bmic_err == true) {
-				int i;
-
-				RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-					 "*(pframemic-8)-*(pframemic-1) =%*phC\n",
-					 8, pframemic - 8);
-				RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-					 "*(pframemic-16)-*(pframemic-9) =%*phC\n",
-					 8, pframemic - 16);
-
-				RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-					 "====== demp packet (len =%d) ======\n",
-					 precvframe->pkt->len);
-				for (i = 0; i < precvframe->pkt->len; i = i + 8) {
-					RT_TRACE(_module_rtl871x_recv_c_,
-						 _drv_err_, "%*phC\n",
-						 8, precvframe->pkt->data + i);
-				}
-				RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-					 "====== demp packet end [len =%d]======\n",
-					 precvframe->pkt->len);
-				RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-					 "hrdlen =%d\n", prxattrib->hdrlen);
-
-				RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-					 "ra = %pM psecuritypriv->binstallGrpkey =%d\n",
-					 prxattrib->ra,
-					 psecuritypriv->binstallGrpkey);
-
-				/*  double check key_index for some timing
-				    issue, cannot compare with
-				    psecuritypriv->dot118021XGrpKeyid also
-				    cause timing issue */
-				if ((is_multicast_ether_addr(prxattrib->ra)) &&
-				    (prxattrib->key_index !=
-				     pmlmeinfo->key_index))
-					brpt_micerror = false;
-
-				if ((prxattrib->bdecrypted == true) &&
-				    (brpt_micerror == true)) {
-					rtw_handle_tkip_mic_err23a(adapter, (u8)is_multicast_ether_addr(prxattrib->ra));
-					RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-						 "mic error :prxattrib->bdecrypted =%d\n",
-						 prxattrib->bdecrypted);
-					DBG_8723A(" mic error :prxattrib->"
-						  "bdecrypted =%d\n",
-						  prxattrib->bdecrypted);
-				} else {
-					RT_TRACE(_module_rtl871x_recv_c_,
-						 _drv_err_,
-						 "mic error :prxattrib->bdecrypted =%d\n",
-						 prxattrib->bdecrypted);
-					DBG_8723A(" mic error :prxattrib->"
-						  "bdecrypted =%d\n",
-						  prxattrib->bdecrypted);
-				}
-
-				res = _FAIL;
-			} else {
-				/* mic checked ok */
-				if (!psecuritypriv->bcheck_grpkey &&
-				    is_multicast_ether_addr(prxattrib->ra)) {
-					psecuritypriv->bcheck_grpkey = 1;
-					RT_TRACE(_module_rtl871x_recv_c_,
-						 _drv_err_,
-						 "psecuritypriv->bcheck_grpkey = true\n");
-				}
-			}
-		} else {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "recvframe_chkmic: rtw_get_stainfo23a ==NULL!!!\n");
-		}
-
-		skb_trim(precvframe->pkt, precvframe->pkt->len - 8);
-	}
-
-exit:
-
-
-
-	return res;
-}
-
-/* decrypt and set the ivlen, icvlen of the recv_frame */
-struct recv_frame *decryptor(struct rtw_adapter *padapter,
-			     struct recv_frame *precv_frame);
-struct recv_frame *decryptor(struct rtw_adapter *padapter,
-			     struct recv_frame *precv_frame)
-{
-	struct rx_pkt_attrib *prxattrib = &precv_frame->attrib;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct recv_frame *return_packet = precv_frame;
-	int res = _SUCCESS;
-
-	RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-		 "prxstat->decrypted =%x prxattrib->encrypt = 0x%03x\n",
-		 prxattrib->bdecrypted, prxattrib->encrypt);
-
-	if (prxattrib->encrypt > 0) {
-		u8 *iv = precv_frame->pkt->data + prxattrib->hdrlen;
-
-		prxattrib->key_index = (((iv[3]) >> 6) & 0x3);
-
-		if (prxattrib->key_index > WEP_KEYS) {
-			DBG_8723A("prxattrib->key_index(%d) > WEP_KEYS\n",
-				  prxattrib->key_index);
-
-			switch (prxattrib->encrypt) {
-			case WLAN_CIPHER_SUITE_WEP40:
-			case WLAN_CIPHER_SUITE_WEP104:
-				prxattrib->key_index =
-					psecuritypriv->dot11PrivacyKeyIndex;
-				break;
-			case WLAN_CIPHER_SUITE_TKIP:
-			case WLAN_CIPHER_SUITE_CCMP:
-			default:
-				prxattrib->key_index =
-					psecuritypriv->dot118021XGrpKeyid;
-				break;
-			}
-		}
-	}
-
-	if ((prxattrib->encrypt > 0) && ((prxattrib->bdecrypted == 0))) {
-		psecuritypriv->hw_decrypted = 0;
-		switch (prxattrib->encrypt) {
-		case WLAN_CIPHER_SUITE_WEP40:
-		case WLAN_CIPHER_SUITE_WEP104:
-			rtw_wep_decrypt23a(padapter, precv_frame);
-			break;
-		case WLAN_CIPHER_SUITE_TKIP:
-			res = rtw_tkip_decrypt23a(padapter, precv_frame);
-			break;
-		case WLAN_CIPHER_SUITE_CCMP:
-			res = rtw_aes_decrypt23a(padapter, precv_frame);
-			break;
-		default:
-			break;
-		}
-	} else if (prxattrib->bdecrypted == 1 && prxattrib->encrypt > 0 &&
-		   (psecuritypriv->busetkipkey == 1 ||
-		    prxattrib->encrypt != WLAN_CIPHER_SUITE_TKIP)) {
-			psecuritypriv->hw_decrypted = 1;
-	}
-
-	if (res == _FAIL) {
-		rtw_free_recvframe23a(return_packet);
-		return_packet = NULL;
-	}
-
-
-
-	return return_packet;
-}
-
-/* set the security information in the recv_frame */
-static struct recv_frame *portctrl(struct rtw_adapter *adapter,
-				   struct recv_frame *precv_frame)
-{
-	u8 *psta_addr, *ptr;
-	uint auth_alg;
-	struct recv_frame *pfhdr;
-	struct sta_info *psta;
-	struct sta_priv *pstapriv ;
-	struct recv_frame *prtnframe;
-	u16 ether_type;
-	u16 eapol_type = ETH_P_PAE;/* for Funia BD's WPA issue */
-	struct rx_pkt_attrib *pattrib;
-
-	pstapriv = &adapter->stapriv;
-
-	auth_alg = adapter->securitypriv.dot11AuthAlgrthm;
-
-	pfhdr = precv_frame;
-	pattrib = &pfhdr->attrib;
-	psta_addr = pattrib->ta;
-	psta = rtw_get_stainfo23a(pstapriv, psta_addr);
-
-	RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-		 "########portctrl:adapter->securitypriv.dot11AuthAlgrthm =%d\n",
-		 adapter->securitypriv.dot11AuthAlgrthm);
-
-	prtnframe = precv_frame;
-
-	if (auth_alg == dot11AuthAlgrthm_8021X) {
-		/* get ether_type */
-		ptr = pfhdr->pkt->data + pfhdr->attrib.hdrlen;
-
-		ether_type = (ptr[6] << 8) | ptr[7];
-
-		if (psta && psta->ieee8021x_blocked) {
-			/* blocked */
-			/* only accept EAPOL frame */
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-				 "########portctrl:psta->ieee8021x_blocked ==1\n");
-
-			if (ether_type != eapol_type) {
-				/* free this frame */
-				rtw_free_recvframe23a(precv_frame);
-				prtnframe = NULL;
-			}
-		}
-	}
-
-	return prtnframe;
-}
-
-int recv_decache(struct recv_frame *precv_frame, u8 bretry,
-		 struct stainfo_rxcache *prxcache);
-int recv_decache(struct recv_frame *precv_frame, u8 bretry,
-		 struct stainfo_rxcache *prxcache)
-{
-	int tid = precv_frame->attrib.priority;
-
-	u16 seq_ctrl = ((precv_frame->attrib.seq_num & 0xffff) << 4) |
-		(precv_frame->attrib.frag_num & 0xf);
-
-
-
-	if (tid > 15) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_notice_,
-			 "recv_decache, (tid>15)! seq_ctrl = 0x%x, tid = 0x%x\n",
-			 seq_ctrl, tid);
-
-		return _FAIL;
-	}
-
-	if (1) { /* if (bretry) */
-		if (seq_ctrl == prxcache->tid_rxseq[tid]) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_notice_,
-				 "recv_decache, seq_ctrl = 0x%x, tid = 0x%x, tid_rxseq = 0x%x\n",
-				 seq_ctrl, tid, prxcache->tid_rxseq[tid]);
-
-			return _FAIL;
-		}
-	}
-
-	prxcache->tid_rxseq[tid] = seq_ctrl;
-
-
-
-	return _SUCCESS;
-}
-
-void process23a_pwrbit_data(struct rtw_adapter *padapter,
-			 struct recv_frame *precv_frame);
-void process23a_pwrbit_data(struct rtw_adapter *padapter,
-			 struct recv_frame *precv_frame)
-{
-#ifdef CONFIG_8723AU_AP_MODE
-	unsigned char pwrbit;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	struct rx_pkt_attrib *pattrib = &precv_frame->attrib;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct sta_info *psta;
-
-	psta = rtw_get_stainfo23a(pstapriv, pattrib->src);
-
-	if (psta) {
-		pwrbit = ieee80211_has_pm(hdr->frame_control);
-
-		if (pwrbit) {
-			if (!(psta->state & WIFI_SLEEP_STATE))
-				stop_sta_xmit23a(padapter, psta);
-		} else {
-			if (psta->state & WIFI_SLEEP_STATE)
-				wakeup_sta_to_xmit23a(padapter, psta);
-		}
-	}
-
-#endif
-}
-
-void process_wmmps_data(struct rtw_adapter *padapter,
-			struct recv_frame *precv_frame);
-void process_wmmps_data(struct rtw_adapter *padapter,
-			struct recv_frame *precv_frame)
-{
-#ifdef CONFIG_8723AU_AP_MODE
-	struct rx_pkt_attrib *pattrib = &precv_frame->attrib;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct sta_info *psta;
-
-	psta = rtw_get_stainfo23a(pstapriv, pattrib->src);
-
-	if (!psta)
-		return;
-
-
-	if (!psta->qos_option)
-		return;
-
-	if (!(psta->qos_info & 0xf))
-		return;
-
-	if (psta->state & WIFI_SLEEP_STATE) {
-		u8 wmmps_ac = 0;
-
-		switch (pattrib->priority) {
-		case 1:
-		case 2:
-			wmmps_ac = psta->uapsd_bk & BIT(1);
-			break;
-		case 4:
-		case 5:
-			wmmps_ac = psta->uapsd_vi & BIT(1);
-			break;
-		case 6:
-		case 7:
-			wmmps_ac = psta->uapsd_vo & BIT(1);
-			break;
-		case 0:
-		case 3:
-		default:
-			wmmps_ac = psta->uapsd_be & BIT(1);
-			break;
-		}
-
-		if (wmmps_ac) {
-			if (psta->sleepq_ac_len > 0) {
-				/* process received triggered frame */
-				xmit_delivery_enabled_frames23a(padapter, psta);
-			} else {
-				/* issue one qos null frame with More data bit = 0 and the EOSP bit set (= 1) */
-				issue_qos_nulldata23a(padapter, psta->hwaddr,
-						   (u16)pattrib->priority,
-						   0, 0);
-			}
-		}
-	}
-
-#endif
-}
-
-static void count_rx_stats(struct rtw_adapter *padapter,
-			   struct recv_frame *prframe, struct sta_info *sta)
-{
-	int sz;
-	struct sta_info *psta = NULL;
-	struct stainfo_stats *pstats = NULL;
-	struct rx_pkt_attrib *pattrib = & prframe->attrib;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-
-	sz = prframe->pkt->len;
-	precvpriv->rx_bytes += sz;
-
-	padapter->mlmepriv.LinkDetectInfo.NumRxOkInPeriod++;
-
-	if ((!is_broadcast_ether_addr(pattrib->dst)) &&
-	    (!is_multicast_ether_addr(pattrib->dst)))
-		padapter->mlmepriv.LinkDetectInfo.NumRxUnicastOkInPeriod++;
-
-	if (sta)
-		psta = sta;
-	else
-		psta = prframe->psta;
-
-	if (psta) {
-		pstats = &psta->sta_stats;
-
-		pstats->rx_data_pkts++;
-		pstats->rx_bytes += sz;
-	}
-}
-
-static int sta2sta_data_frame(struct rtw_adapter *adapter,
-			      struct recv_frame *precv_frame,
-			      struct sta_info**psta)
-{
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	int ret = _SUCCESS;
-	struct rx_pkt_attrib *pattrib = & precv_frame->attrib;
-	struct sta_priv *pstapriv = &adapter->stapriv;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	u8 *mybssid  = get_bssid(pmlmepriv);
-	u8 *myhwaddr = myid(&adapter->eeprompriv);
-	u8 *sta_addr = NULL;
-	int bmcast = is_multicast_ether_addr(pattrib->dst);
-
-
-
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
-
-		/*  filter packets that SA is myself or multicast or broadcast */
-		if (ether_addr_equal(myhwaddr, pattrib->src)) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "SA == myself\n");
-			ret = _FAIL;
-			goto exit;
-		}
-
-		if (!ether_addr_equal(myhwaddr, pattrib->dst) && !bmcast) {
-			ret = _FAIL;
-			goto exit;
-		}
-
-		if (ether_addr_equal(pattrib->bssid, "\x0\x0\x0\x0\x0\x0") ||
-		    ether_addr_equal(mybssid, "\x0\x0\x0\x0\x0\x0") ||
-		    !ether_addr_equal(pattrib->bssid, mybssid)) {
-			ret = _FAIL;
-			goto exit;
-		}
-
-		sta_addr = pattrib->src;
-	} else if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
-		/*  For Station mode, sa and bssid should always be BSSID,
-		    and DA is my mac-address */
-		if (!ether_addr_equal(pattrib->bssid, pattrib->src)) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "bssid != TA under STATION_MODE; drop pkt\n");
-			ret = _FAIL;
-			goto exit;
-		}
-
-		sta_addr = pattrib->bssid;
-
-	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		if (bmcast) {
-			/*  For AP mode, if DA == MCAST, then BSSID should be also MCAST */
-			if (!is_multicast_ether_addr(pattrib->bssid)) {
-				ret = _FAIL;
-				goto exit;
-			}
-		} else { /*  not mc-frame */
-			/*  For AP mode, if DA is non-MCAST, then it must
-			    be BSSID, and bssid == BSSID */
-			if (!ether_addr_equal(pattrib->bssid, pattrib->dst)) {
-				ret = _FAIL;
-				goto exit;
-			}
-
-			sta_addr = pattrib->src;
-		}
-	} else if (check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
-		ether_addr_copy(pattrib->dst, hdr->addr1);
-		ether_addr_copy(pattrib->src, hdr->addr2);
-		ether_addr_copy(pattrib->bssid, hdr->addr3);
-		ether_addr_copy(pattrib->ra, pattrib->dst);
-		ether_addr_copy(pattrib->ta, pattrib->src);
-
-		sta_addr = mybssid;
-	} else {
-		ret  = _FAIL;
-	}
-
-	if (bmcast)
-		*psta = rtw_get_bcmc_stainfo23a(adapter);
-	else
-		*psta = rtw_get_stainfo23a(pstapriv, sta_addr); /*  get ap_info */
-
-	if (*psta == NULL) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "can't get psta under sta2sta_data_frame ; drop pkt\n");
-		ret = _FAIL;
-		goto exit;
-	}
-
-exit:
-
-	return ret;
-}
-
-int ap2sta_data_frame(struct rtw_adapter *adapter,
-		      struct recv_frame *precv_frame,
-		      struct sta_info **psta);
-int ap2sta_data_frame(struct rtw_adapter *adapter,
-		      struct recv_frame *precv_frame,
-		      struct sta_info **psta)
-{
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	struct rx_pkt_attrib *pattrib = & precv_frame->attrib;
-	int ret = _SUCCESS;
-	struct sta_priv *pstapriv = &adapter->stapriv;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	u8 *mybssid  = get_bssid(pmlmepriv);
-	u8 *myhwaddr = myid(&adapter->eeprompriv);
-	int bmcast = is_multicast_ether_addr(pattrib->dst);
-
-
-
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) &&
-	    (check_fwstate(pmlmepriv, _FW_LINKED) ||
-	     check_fwstate(pmlmepriv, _FW_UNDER_LINKING))) {
-
-		/* filter packets that SA is myself or multicast or broadcast */
-		if (ether_addr_equal(myhwaddr, pattrib->src)) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "SA == myself\n");
-			ret = _FAIL;
-			goto exit;
-		}
-
-		/*  da should be for me */
-		if (!ether_addr_equal(myhwaddr, pattrib->dst) && !bmcast) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-				 "ap2sta_data_frame:  compare DA failed; DA=%pM\n",
-				 pattrib->dst);
-			ret = _FAIL;
-			goto exit;
-		}
-
-		/*  check BSSID */
-		if (ether_addr_equal(pattrib->bssid, "\x0\x0\x0\x0\x0\x0") ||
-		    ether_addr_equal(mybssid, "\x0\x0\x0\x0\x0\x0") ||
-		    !ether_addr_equal(pattrib->bssid, mybssid)) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-				 "ap2sta_data_frame:  compare BSSID failed; BSSID=%pM\n",
-				 pattrib->bssid);
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-				 "mybssid=%pM\n", mybssid);
-
-			if (!bmcast) {
-				DBG_8723A("issue_deauth23a to the nonassociated ap=%pM for the reason(7)\n",
-					  pattrib->bssid);
-				issue_deauth23a(adapter, pattrib->bssid,
-					     WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
-			}
-
-			ret = _FAIL;
-			goto exit;
-		}
-
-		if (bmcast)
-			*psta = rtw_get_bcmc_stainfo23a(adapter);
-		else
-			/*  get ap_info */
-			*psta = rtw_get_stainfo23a(pstapriv, pattrib->bssid);
-
-		if (*psta == NULL) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "ap2sta: can't get psta under STATION_MODE; drop pkt\n");
-			ret = _FAIL;
-			goto exit;
-		}
-
-		if (ieee80211_is_nullfunc(hdr->frame_control)) {
-			/* No data, will not indicate to upper layer,
-			   temporily count it here */
-			count_rx_stats(adapter, precv_frame, *psta);
-			ret = RTW_RX_HANDLED;
-			goto exit;
-		}
-
-	} else if (check_fwstate(pmlmepriv, WIFI_MP_STATE) &&
-		   check_fwstate(pmlmepriv, _FW_LINKED)) {
-		ether_addr_copy(pattrib->dst, hdr->addr1);
-		ether_addr_copy(pattrib->src, hdr->addr2);
-		ether_addr_copy(pattrib->bssid, hdr->addr3);
-		ether_addr_copy(pattrib->ra, pattrib->dst);
-		ether_addr_copy(pattrib->ta, pattrib->src);
-
-		/*  */
-		ether_addr_copy(pattrib->bssid,  mybssid);
-
-		/*  get sta_info */
-		*psta = rtw_get_stainfo23a(pstapriv, pattrib->bssid);
-		if (*psta == NULL) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "can't get psta under MP_MODE ; drop pkt\n");
-			ret = _FAIL;
-			goto exit;
-		}
-	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		/* Special case */
-		ret = RTW_RX_HANDLED;
-		goto exit;
-	} else {
-		if (ether_addr_equal(myhwaddr, pattrib->dst) && !bmcast) {
-			*psta = rtw_get_stainfo23a(pstapriv, pattrib->bssid);
-			if (*psta == NULL) {
-				DBG_8723A("issue_deauth23a to the ap=%pM for the reason(7)\n",
-					  pattrib->bssid);
-
-				issue_deauth23a(adapter, pattrib->bssid,
-					     WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
-			}
-		}
-
-		ret = _FAIL;
-	}
-
-exit:
-
-
-
-	return ret;
-}
-
-int sta2ap_data_frame(struct rtw_adapter *adapter,
-		      struct recv_frame *precv_frame,
-		      struct sta_info **psta);
-int sta2ap_data_frame(struct rtw_adapter *adapter,
-		      struct recv_frame *precv_frame,
-		      struct sta_info **psta)
-{
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	struct rx_pkt_attrib *pattrib = & precv_frame->attrib;
-	struct sta_priv *pstapriv = &adapter->stapriv;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	unsigned char *mybssid = get_bssid(pmlmepriv);
-	int ret = _SUCCESS;
-
-
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		/* For AP mode, RA = BSSID, TX = STA(SRC_ADDR), A3 = DST_ADDR */
-		if (!ether_addr_equal(pattrib->bssid, mybssid)) {
-			ret = _FAIL;
-			goto exit;
-		}
-
-		*psta = rtw_get_stainfo23a(pstapriv, pattrib->src);
-		if (*psta == NULL) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "can't get psta under AP_MODE; drop pkt\n");
-			DBG_8723A("issue_deauth23a to sta=%pM for the reason(7)\n",
-				  pattrib->src);
-
-			issue_deauth23a(adapter, pattrib->src,
-				     WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
-
-			ret = RTW_RX_HANDLED;
-			goto exit;
-		}
-
-		process23a_pwrbit_data(adapter, precv_frame);
-
-		/* We only get here if it's a data frame, so no need to
-		 * confirm data frame type first */
-		if (ieee80211_is_data_qos(hdr->frame_control))
-			process_wmmps_data(adapter, precv_frame);
-
-		if (ieee80211_is_nullfunc(hdr->frame_control)) {
-			/* No data, will not indicate to upper layer,
-			   temporily count it here */
-			count_rx_stats(adapter, precv_frame, *psta);
-			ret = RTW_RX_HANDLED;
-			goto exit;
-		}
-	} else {
-		u8 *myhwaddr = myid(&adapter->eeprompriv);
-
-		if (!ether_addr_equal(pattrib->ra, myhwaddr)) {
-			ret = RTW_RX_HANDLED;
-			goto exit;
-		}
-		DBG_8723A("issue_deauth23a to sta=%pM for the reason(7)\n",
-			  pattrib->src);
-		issue_deauth23a(adapter, pattrib->src,
-			     WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
-		ret = RTW_RX_HANDLED;
-		goto exit;
-	}
-
-exit:
-
-
-
-	return ret;
-}
-
-static int validate_recv_ctrl_frame(struct rtw_adapter *padapter,
-				    struct recv_frame *precv_frame)
-{
-#ifdef CONFIG_8723AU_AP_MODE
-	struct rx_pkt_attrib *pattrib = &precv_frame->attrib;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-
-	if (!ieee80211_is_ctl(hdr->frame_control))
-		return _FAIL;
-
-	/* receive the frames that ra(a1) is my address */
-	if (!ether_addr_equal(hdr->addr1, myid(&padapter->eeprompriv)))
-		return _FAIL;
-
-	/* only handle ps-poll */
-	if (ieee80211_is_pspoll(hdr->frame_control)) {
-		struct ieee80211_pspoll *psp = (struct ieee80211_pspoll *)hdr;
-		u16 aid;
-		u8 wmmps_ac = 0;
-		struct sta_info *psta = NULL;
-
-		aid = le16_to_cpu(psp->aid) & 0x3fff;
-		psta = rtw_get_stainfo23a(pstapriv, hdr->addr2);
-
-		if (!psta || psta->aid != aid)
-			return _FAIL;
-
-		/* for rx pkt statistics */
-		psta->sta_stats.rx_ctrl_pkts++;
-
-		switch (pattrib->priority) {
-		case 1:
-		case 2:
-			wmmps_ac = psta->uapsd_bk & BIT(0);
-			break;
-		case 4:
-		case 5:
-			wmmps_ac = psta->uapsd_vi & BIT(0);
-			break;
-		case 6:
-		case 7:
-			wmmps_ac = psta->uapsd_vo & BIT(0);
-			break;
-		case 0:
-		case 3:
-		default:
-			wmmps_ac = psta->uapsd_be & BIT(0);
-			break;
-		}
-
-		if (wmmps_ac)
-			return _FAIL;
-
-		if (psta->state & WIFI_STA_ALIVE_CHK_STATE) {
-			DBG_8723A("%s alive check-rx ps-poll\n", __func__);
-			psta->expire_to = pstapriv->expire_to;
-			psta->state ^= WIFI_STA_ALIVE_CHK_STATE;
-		}
-
-		if ((psta->state & WIFI_SLEEP_STATE) &&
-		    (pstapriv->sta_dz_bitmap & CHKBIT(psta->aid))) {
-			struct list_head *xmitframe_phead;
-			struct xmit_frame *pxmitframe;
-			struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-			spin_lock_bh(&pxmitpriv->lock);
-
-			xmitframe_phead = get_list_head(&psta->sleep_q);
-			pxmitframe = list_first_entry_or_null(xmitframe_phead,
-							      struct xmit_frame,
-							      list);
-			if (pxmitframe) {
-				list_del_init(&pxmitframe->list);
-
-				psta->sleepq_len--;
-
-				if (psta->sleepq_len>0)
-					pxmitframe->attrib.mdata = 1;
-				else
-					pxmitframe->attrib.mdata = 0;
-
-				pxmitframe->attrib.triggered = 1;
-
-				rtl8723au_hal_xmitframe_enqueue(padapter,
-								pxmitframe);
-
-				if (psta->sleepq_len == 0) {
-					pstapriv->tim_bitmap &= ~CHKBIT(psta->aid);
-					update_beacon23a(padapter, WLAN_EID_TIM,
-							 NULL, false);
-				}
-
-				spin_unlock_bh(&pxmitpriv->lock);
-
-			} else {
-				spin_unlock_bh(&pxmitpriv->lock);
-
-				if (pstapriv->tim_bitmap & CHKBIT(psta->aid)) {
-					if (psta->sleepq_len == 0) {
-						DBG_8723A("no buffered packets "
-							  "to xmit\n");
-
-						/* issue nulldata with More data bit = 0 to indicate we have no buffered packets */
-						issue_nulldata23a(padapter,
-							       psta->hwaddr,
-							       0, 0, 0);
-					} else {
-						DBG_8723A("error!psta->sleepq"
-							  "_len =%d\n",
-							  psta->sleepq_len);
-						psta->sleepq_len = 0;
-					}
-
-					pstapriv->tim_bitmap &= ~CHKBIT(psta->aid);
-
-					update_beacon23a(padapter, WLAN_EID_TIM,
-							 NULL, false);
-				}
-			}
-		}
-	}
-
-#endif
-	return _FAIL;
-}
-
-struct recv_frame *recvframe_chk_defrag23a(struct rtw_adapter *padapter,
-					struct recv_frame *precv_frame);
-static int validate_recv_mgnt_frame(struct rtw_adapter *padapter,
-				    struct recv_frame *precv_frame)
-{
-	struct sta_info *psta;
-	struct sk_buff *skb;
-	struct ieee80211_hdr *hdr;
-
-	RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-		 "+validate_recv_mgnt_frame\n");
-
-	precv_frame = recvframe_chk_defrag23a(padapter, precv_frame);
-	if (precv_frame == NULL) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_notice_,
-			 "%s: fragment packet\n", __func__);
-		return _SUCCESS;
-	}
-
-	skb = precv_frame->pkt;
-	hdr = (struct ieee80211_hdr *) skb->data;
-
-		/* for rx pkt statistics */
-	psta = rtw_get_stainfo23a(&padapter->stapriv, hdr->addr2);
-	if (psta) {
-		psta->sta_stats.rx_mgnt_pkts++;
-
-		if (ieee80211_is_beacon(hdr->frame_control))
-			psta->sta_stats.rx_beacon_pkts++;
-		else if (ieee80211_is_probe_req(hdr->frame_control))
-			psta->sta_stats.rx_probereq_pkts++;
-		else if (ieee80211_is_probe_resp(hdr->frame_control)) {
-			if (ether_addr_equal(padapter->eeprompriv.mac_addr,
-				    hdr->addr1))
-				psta->sta_stats.rx_probersp_pkts++;
-			else if (is_broadcast_ether_addr(hdr->addr1) ||
-				 is_multicast_ether_addr(hdr->addr1))
-				psta->sta_stats.rx_probersp_bm_pkts++;
-			else
-				psta->sta_stats.rx_probersp_uo_pkts++;
-		}
-	}
-
-	mgt_dispatcher23a(padapter, precv_frame);
-
-	return _SUCCESS;
-}
-
-static int validate_recv_data_frame(struct rtw_adapter *adapter,
-				    struct recv_frame *precv_frame)
-{
-	u8 bretry;
-	u8 *psa, *pda;
-	struct sta_info *psta = NULL;
-	struct rx_pkt_attrib *pattrib = & precv_frame->attrib;
-	struct security_priv *psecuritypriv = &adapter->securitypriv;
-	int ret = _SUCCESS;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-
-
-
-	bretry = ieee80211_has_retry(hdr->frame_control);
-	pda = ieee80211_get_DA(hdr);
-	psa = ieee80211_get_SA(hdr);
-
-	ether_addr_copy(pattrib->dst, pda);
-	ether_addr_copy(pattrib->src, psa);
-
-	switch (hdr->frame_control &
-		cpu_to_le16(IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS)) {
-	case cpu_to_le16(0):
-		ether_addr_copy(pattrib->bssid, hdr->addr3);
-		ether_addr_copy(pattrib->ra, pda);
-		ether_addr_copy(pattrib->ta, psa);
-		ret = sta2sta_data_frame(adapter, precv_frame, &psta);
-		break;
-
-	case cpu_to_le16(IEEE80211_FCTL_FROMDS):
-		ether_addr_copy(pattrib->bssid, hdr->addr2);
-		ether_addr_copy(pattrib->ra, pda);
-		ether_addr_copy(pattrib->ta, hdr->addr2);
-		ret = ap2sta_data_frame(adapter, precv_frame, &psta);
-		break;
-
-	case cpu_to_le16(IEEE80211_FCTL_TODS):
-		ether_addr_copy(pattrib->bssid, hdr->addr1);
-		ether_addr_copy(pattrib->ra, hdr->addr1);
-		ether_addr_copy(pattrib->ta, psa);
-		ret = sta2ap_data_frame(adapter, precv_frame, &psta);
-		break;
-
-	case cpu_to_le16(IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS):
-		/*
-		 * There is no BSSID in this case, but the driver has been
-		 * using addr1 so far, so keep it for now.
-		 */
-		ether_addr_copy(pattrib->bssid, hdr->addr1);
-		ether_addr_copy(pattrib->ra, hdr->addr1);
-		ether_addr_copy(pattrib->ta, hdr->addr2);
-		ret = _FAIL;
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_, "case 3\n");
-		break;
-	}
-
-	if ((ret == _FAIL) || (ret == RTW_RX_HANDLED))
-		goto exit;
-
-	if (!psta) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "after to_fr_ds_chk; psta == NULL\n");
-		ret = _FAIL;
-		goto exit;
-	}
-
-	precv_frame->psta = psta;
-
-	pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr);
-	if (ieee80211_has_a4(hdr->frame_control))
-		pattrib->hdrlen += ETH_ALEN;
-
-	/* parsing QC field */
-	if (pattrib->qos == 1) {
-		__le16 *qptr = (__le16 *)ieee80211_get_qos_ctl(hdr);
-		u16 qos_ctrl = le16_to_cpu(*qptr);
-
-		pattrib->priority = qos_ctrl & IEEE80211_QOS_CTL_TID_MASK;
-		pattrib->ack_policy = (qos_ctrl >> 5) & 3;
-		pattrib->amsdu =
-			(qos_ctrl & IEEE80211_QOS_CTL_A_MSDU_PRESENT) >> 7;
-		pattrib->hdrlen += IEEE80211_QOS_CTL_LEN;
-
-		if (pattrib->priority != 0 && pattrib->priority != 3) {
-			adapter->recvpriv.bIsAnyNonBEPkts = true;
-		}
-	} else {
-		pattrib->priority = 0;
-		pattrib->ack_policy = 0;
-		pattrib->amsdu = 0;
-	}
-
-	if (pattrib->order) { /* HT-CTRL 11n */
-		pattrib->hdrlen += 4;
-	}
-
-	precv_frame->preorder_ctrl = &psta->recvreorder_ctrl[pattrib->priority];
-
-	/*  decache, drop duplicate recv packets */
-	if (recv_decache(precv_frame, bretry, &psta->sta_recvpriv.rxcache) ==
-	    _FAIL) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "decache : drop pkt\n");
-		ret = _FAIL;
-		goto exit;
-	}
-
-	if (pattrib->privacy) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-			 "validate_recv_data_frame:pattrib->privacy =%x\n",
-			 pattrib->privacy);
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-			 "^^^^^^^^^^^is_multicast_ether_addr(pattrib->ra(0x%02x)) =%d^^^^^^^^^^^^^^^6\n",
-			 pattrib->ra[0],
-			 is_multicast_ether_addr(pattrib->ra));
-
-		GET_ENCRY_ALGO(psecuritypriv, psta, pattrib->encrypt,
-			       is_multicast_ether_addr(pattrib->ra));
-
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-			 "pattrib->encrypt =%d\n", pattrib->encrypt);
-
-		switch (pattrib->encrypt) {
-		case WLAN_CIPHER_SUITE_WEP40:
-		case WLAN_CIPHER_SUITE_WEP104:
-			pattrib->iv_len = IEEE80211_WEP_IV_LEN;
-			pattrib->icv_len = IEEE80211_WEP_ICV_LEN;
-			break;
-		case WLAN_CIPHER_SUITE_TKIP:
-			pattrib->iv_len = IEEE80211_TKIP_IV_LEN;
-			pattrib->icv_len = IEEE80211_TKIP_ICV_LEN;
-			break;
-		case WLAN_CIPHER_SUITE_CCMP:
-			pattrib->iv_len = IEEE80211_CCMP_HDR_LEN;
-			pattrib->icv_len = IEEE80211_CCMP_MIC_LEN;
-			break;
-		default:
-			pattrib->iv_len = 0;
-			pattrib->icv_len = 0;
-			break;
-		}
-	} else {
-		pattrib->encrypt = 0;
-		pattrib->iv_len = 0;
-		pattrib->icv_len = 0;
-	}
-
-exit:
-
-
-
-	return ret;
-}
-
-static void dump_rx_pkt(struct sk_buff *skb, u16 type, int level)
-{
-	int i;
-	u8 *ptr;
-
-	if ((level == 1) ||
-	    ((level == 2) && (type == IEEE80211_FTYPE_MGMT)) ||
-	    ((level == 3) && (type == IEEE80211_FTYPE_DATA))) {
-
-		ptr = skb->data;
-
-		DBG_8723A("#############################\n");
-
-		for (i = 0; i < 64; i = i + 8)
-			DBG_8723A("%*phC:\n", 8, ptr + i);
-		DBG_8723A("#############################\n");
-	}
-}
-
-static int validate_recv_frame(struct rtw_adapter *adapter,
-			       struct recv_frame *precv_frame)
-{
-	/* shall check frame subtype, to / from ds, da, bssid */
-
-	/* then call check if rx seq/frag. duplicated. */
-	u8 type;
-	u8 subtype;
-	int retval = _SUCCESS;
-	struct rx_pkt_attrib *pattrib = & precv_frame->attrib;
-	struct sk_buff *skb = precv_frame->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	u8 ver;
-	u8 bDumpRxPkt;
-	u16 seq_ctrl, fctl;
-
-	fctl = le16_to_cpu(hdr->frame_control);
-	ver = fctl & IEEE80211_FCTL_VERS;
-	type = fctl & IEEE80211_FCTL_FTYPE;
-	subtype = fctl & IEEE80211_FCTL_STYPE;
-
-	/* add version chk */
-	if (ver != 0) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "validate_recv_data_frame fail! (ver!= 0)\n");
-		retval = _FAIL;
-		goto exit;
-	}
-
-	seq_ctrl = le16_to_cpu(hdr->seq_ctrl);
-	pattrib->frag_num = seq_ctrl & IEEE80211_SCTL_FRAG;
-	pattrib->seq_num = seq_ctrl >> 4;
-
-	pattrib->pw_save = ieee80211_has_pm(hdr->frame_control);
-	pattrib->mfrag = ieee80211_has_morefrags(hdr->frame_control);
-	pattrib->mdata = ieee80211_has_moredata(hdr->frame_control);
-	pattrib->privacy = ieee80211_has_protected(hdr->frame_control);
-	pattrib->order = ieee80211_has_order(hdr->frame_control);
-
-	GetHalDefVar8192CUsb(adapter, HAL_DEF_DBG_DUMP_RXPKT, &bDumpRxPkt);
-
-	if (unlikely(bDumpRxPkt == 1))
-		dump_rx_pkt(skb, type, bDumpRxPkt);
-
-	switch (type) {
-	case IEEE80211_FTYPE_MGMT:
-		retval = validate_recv_mgnt_frame(adapter, precv_frame);
-		if (retval == _FAIL) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "validate_recv_mgnt_frame fail\n");
-		}
-		retval = _FAIL; /*  only data frame return _SUCCESS */
-		break;
-	case IEEE80211_FTYPE_CTL:
-		retval = validate_recv_ctrl_frame(adapter, precv_frame);
-		if (retval == _FAIL) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "validate_recv_ctrl_frame fail\n");
-		}
-		retval = _FAIL; /*  only data frame return _SUCCESS */
-		break;
-	case IEEE80211_FTYPE_DATA:
-		pattrib->qos = (subtype & IEEE80211_STYPE_QOS_DATA) ? 1 : 0;
-		retval = validate_recv_data_frame(adapter, precv_frame);
-		if (retval == _FAIL) {
-			struct recv_priv *precvpriv = &adapter->recvpriv;
-
-			precvpriv->rx_drop++;
-		}
-		break;
-	default:
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "validate_recv_data_frame fail! type = 0x%x\n", type);
-		retval = _FAIL;
-		break;
-	}
-
-exit:
-	return retval;
-}
-
-/* remove the wlanhdr and add the eth_hdr */
-
-static int wlanhdr_to_ethhdr (struct recv_frame *precvframe)
-{
-	u16	eth_type, len, hdrlen;
-	u8	bsnaphdr;
-	u8	*psnap;
-	struct rtw_adapter *adapter = precvframe->adapter;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-
-	struct sk_buff *skb = precvframe->pkt;
-	u8 *ptr;
-	struct rx_pkt_attrib *pattrib = &precvframe->attrib;
-
-
-
-	ptr = skb->data;
-	hdrlen = pattrib->hdrlen;
-	psnap = ptr + hdrlen;
-	eth_type = (psnap[6] << 8) | psnap[7];
-	/* convert hdr + possible LLC headers into Ethernet header */
-	if ((ether_addr_equal(psnap, rfc1042_header) &&
-	     eth_type != ETH_P_AARP && eth_type != ETH_P_IPX) ||
-	    ether_addr_equal(psnap, bridge_tunnel_header)) {
-		/* remove RFC1042 or Bridge-Tunnel encapsulation
-		   and replace EtherType */
-		bsnaphdr = true;
-		hdrlen += SNAP_SIZE;
-	} else {
-		/* Leave Ethernet header part of hdr and full payload */
-		bsnaphdr = false;
-		eth_type = (psnap[0] << 8) | psnap[1];
-	}
-
-	len = skb->len - hdrlen;
-
-	RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-		 "=== pattrib->hdrlen: %x,  pattrib->iv_len:%x ===\n",
-		 pattrib->hdrlen,  pattrib->iv_len);
-
-	pattrib->eth_type = eth_type;
-	if (check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
-		ptr += hdrlen;
-		*ptr = 0x87;
-		*(ptr + 1) = 0x12;
-
-		eth_type = 0x8712;
-		/*  append rx status for mp test packets */
-
-		ptr = skb_pull(skb, (hdrlen - sizeof(struct ethhdr) + 2) - 24);
-		memcpy(ptr, skb->head, 24);
-		ptr += 24;
-	} else {
-		ptr = skb_pull(skb, (hdrlen - sizeof(struct ethhdr) +
-				     (bsnaphdr ? 2:0)));
-	}
-
-	ether_addr_copy(ptr, pattrib->dst);
-	ether_addr_copy(ptr + ETH_ALEN, pattrib->src);
-
-	if (!bsnaphdr) {
-		put_unaligned_be16(len, ptr + 12);
-	}
-
-
-	return _SUCCESS;
-}
-
-/* perform defrag */
-struct recv_frame *recvframe_defrag(struct rtw_adapter *adapter,
-				    struct rtw_queue *defrag_q);
-struct recv_frame *recvframe_defrag(struct rtw_adapter *adapter,
-				    struct rtw_queue *defrag_q)
-{
-	struct list_head *phead;
-	u8 wlanhdr_offset;
-	u8 curfragnum;
-	struct recv_frame *pnfhdr, *ptmp;
-	struct recv_frame *prframe, *pnextrframe;
-	struct rtw_queue *pfree_recv_queue;
-	struct sk_buff *skb;
-
-	curfragnum = 0;
-	pfree_recv_queue = &adapter->recvpriv.free_recv_queue;
-
-	phead = get_list_head(defrag_q);
-	prframe = list_first_entry(phead, struct recv_frame, list);
-	list_del_init(&prframe->list);
-	skb = prframe->pkt;
-
-	if (curfragnum != prframe->attrib.frag_num) {
-		/* the first fragment number must be 0 */
-		/* free the whole queue */
-		rtw_free_recvframe23a(prframe);
-		rtw_free_recvframe23a_queue(defrag_q);
-
-		return NULL;
-	}
-
-	curfragnum++;
-
-	list_for_each_entry_safe(pnfhdr, ptmp, phead, list) {
-		pnextrframe = (struct recv_frame *)pnfhdr;
-		/* check the fragment sequence  (2nd ~n fragment frame) */
-
-		if (curfragnum != pnfhdr->attrib.frag_num) {
-			/* the fragment number must be increasing
-			   (after decache) */
-			/* release the defrag_q & prframe */
-			rtw_free_recvframe23a(prframe);
-			rtw_free_recvframe23a_queue(defrag_q);
-			return NULL;
-		}
-
-		curfragnum++;
-
-		/* copy the 2nd~n fragment frame's payload to the
-		   first fragment */
-		/* get the 2nd~last fragment frame's payload */
-
-		wlanhdr_offset = pnfhdr->attrib.hdrlen + pnfhdr->attrib.iv_len;
-
-		skb_pull(pnfhdr->pkt, wlanhdr_offset);
-
-		/* append  to first fragment frame's tail
-		   (if privacy frame, pull the ICV) */
-
-		skb_trim(skb, skb->len - prframe->attrib.icv_len);
-
-		memcpy(skb_tail_pointer(skb), pnfhdr->pkt->data,
-		       pnfhdr->pkt->len);
-
-		skb_put(skb, pnfhdr->pkt->len);
-
-		prframe->attrib.icv_len = pnfhdr->attrib.icv_len;
-	}
-
-	/* free the defrag_q queue and return the prframe */
-	rtw_free_recvframe23a_queue(defrag_q);
-
-	RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-		 "Performance defrag!!!!!\n");
-
-	return prframe;
-}
-
-/* check if need to defrag, if needed queue the frame to defrag_q */
-struct recv_frame *recvframe_chk_defrag23a(struct rtw_adapter *padapter,
-					struct recv_frame *precv_frame)
-{
-	u8	ismfrag;
-	u8	fragnum;
-	u8	*psta_addr;
-	struct recv_frame *pfhdr;
-	struct sta_info *psta;
-	struct sta_priv *pstapriv;
-	struct list_head *phead;
-	struct recv_frame *prtnframe = NULL;
-	struct rtw_queue *pfree_recv_queue, *pdefrag_q;
-
-
-
-	pstapriv = &padapter->stapriv;
-
-	pfhdr = precv_frame;
-
-	pfree_recv_queue = &padapter->recvpriv.free_recv_queue;
-
-	/* need to define struct of wlan header frame ctrl */
-	ismfrag = pfhdr->attrib.mfrag;
-	fragnum = pfhdr->attrib.frag_num;
-
-	psta_addr = pfhdr->attrib.ta;
-	psta = rtw_get_stainfo23a(pstapriv, psta_addr);
-	if (!psta) {
-		struct ieee80211_hdr *hdr =
-			(struct ieee80211_hdr *) pfhdr->pkt->data;
-		if (!ieee80211_is_data(hdr->frame_control)) {
-			psta = rtw_get_bcmc_stainfo23a(padapter);
-			pdefrag_q = &psta->sta_recvpriv.defrag_q;
-		} else
-			pdefrag_q = NULL;
-	} else
-		pdefrag_q = &psta->sta_recvpriv.defrag_q;
-
-	if ((ismfrag == 0) && (fragnum == 0)) {
-		prtnframe = precv_frame;/* isn't a fragment frame */
-	}
-
-	if (ismfrag == 1) {
-		/* 0~(n-1) fragment frame */
-		/* enqueue to defraf_g */
-		if (pdefrag_q != NULL) {
-			if (fragnum == 0) {
-				/* the first fragment */
-				if (!list_empty(&pdefrag_q->queue)) {
-					/* free current defrag_q */
-					rtw_free_recvframe23a_queue(pdefrag_q);
-				}
-			}
-
-			/* Then enqueue the 0~(n-1) fragment into the
-			   defrag_q */
-
-			/* spin_lock(&pdefrag_q->lock); */
-			phead = get_list_head(pdefrag_q);
-			list_add_tail(&pfhdr->list, phead);
-			/* spin_unlock(&pdefrag_q->lock); */
-
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-				 "Enqueuq: ismfrag = %d, fragnum = %d\n",
-				 ismfrag, fragnum);
-
-			prtnframe = NULL;
-
-		} else {
-			/* can't find this ta's defrag_queue,
-			   so free this recv_frame */
-			rtw_free_recvframe23a(precv_frame);
-			prtnframe = NULL;
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "Free because pdefrag_q == NULL: ismfrag = %d, fragnum = %d\n",
-				 ismfrag, fragnum);
-		}
-	}
-
-	if ((ismfrag == 0) && (fragnum != 0)) {
-		/* the last fragment frame */
-		/* enqueue the last fragment */
-		if (pdefrag_q != NULL) {
-			/* spin_lock(&pdefrag_q->lock); */
-			phead = get_list_head(pdefrag_q);
-			list_add_tail(&pfhdr->list, phead);
-			/* spin_unlock(&pdefrag_q->lock); */
-
-			/* call recvframe_defrag to defrag */
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-				 "defrag: ismfrag = %d, fragnum = %d\n",
-				 ismfrag, fragnum);
-			precv_frame = recvframe_defrag(padapter, pdefrag_q);
-			prtnframe = precv_frame;
-		} else {
-			/* can't find this ta's defrag_queue,
-			   so free this recv_frame */
-			rtw_free_recvframe23a(precv_frame);
-			prtnframe = NULL;
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "Free because pdefrag_q == NULL: ismfrag = %d, fragnum = %d\n",
-				 ismfrag, fragnum);
-		}
-
-	}
-
-	if ((prtnframe != NULL) && (prtnframe->attrib.privacy))	{
-		/* after defrag we must check tkip mic code */
-		if (recvframe_chkmic(padapter,  prtnframe) == _FAIL) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "recvframe_chkmic(padapter,  prtnframe) ==_FAIL\n");
-			rtw_free_recvframe23a(prtnframe);
-			prtnframe = NULL;
-		}
-	}
-
-
-
-	return prtnframe;
-}
-
-int amsdu_to_msdu(struct rtw_adapter *padapter, struct recv_frame *prframe);
-int amsdu_to_msdu(struct rtw_adapter *padapter, struct recv_frame *prframe)
-{
-	struct rx_pkt_attrib *pattrib;
-	struct sk_buff *skb, *sub_skb;
-	struct sk_buff_head skb_list;
-
-	pattrib = &prframe->attrib;
-
-	skb = prframe->pkt;
-	skb_pull(skb, prframe->attrib.hdrlen);
-	__skb_queue_head_init(&skb_list);
-
-	ieee80211_amsdu_to_8023s(skb, &skb_list, NULL, 0, 0, false);
-
-	while (!skb_queue_empty(&skb_list)) {
-		sub_skb = __skb_dequeue(&skb_list);
-
-		sub_skb->protocol = eth_type_trans(sub_skb, padapter->pnetdev);
-		sub_skb->dev = padapter->pnetdev;
-
-		sub_skb->ip_summed = CHECKSUM_NONE;
-
-		netif_rx(sub_skb);
-	}
-
-	prframe->pkt = NULL;
-	rtw_free_recvframe23a(prframe);
-	return _SUCCESS;
-}
-
-int check_indicate_seq(struct recv_reorder_ctrl *preorder_ctrl, u16 seq_num);
-int check_indicate_seq(struct recv_reorder_ctrl *preorder_ctrl, u16 seq_num)
-{
-	u8	wsize = preorder_ctrl->wsize_b;
-	u16	wend = (preorder_ctrl->indicate_seq + wsize -1) & 0xFFF;
-
-	/*  Rx Reorder initialize condition. */
-	if (preorder_ctrl->indicate_seq == 0xFFFF)
-		preorder_ctrl->indicate_seq = seq_num;
-
-	/*  Drop out the packet which SeqNum is smaller than WinStart */
-	if (SN_LESS(seq_num, preorder_ctrl->indicate_seq))
-		return false;
-
-	/*  */
-	/*  Sliding window manipulation. Conditions includes: */
-	/*  1. Incoming SeqNum is equal to WinStart =>Window shift 1 */
-	/*  2. Incoming SeqNum is larger than the WinEnd => Window shift N */
-	/*  */
-	if (SN_EQUAL(seq_num, preorder_ctrl->indicate_seq)) {
-		preorder_ctrl->indicate_seq =
-			(preorder_ctrl->indicate_seq + 1) & 0xFFF;
-	} else if (SN_LESS(wend, seq_num)) {
-		/*  boundary situation, when seq_num cross 0xFFF */
-		if (seq_num >= (wsize - 1))
-			preorder_ctrl->indicate_seq = seq_num + 1 -wsize;
-		else
-			preorder_ctrl->indicate_seq = 0xFFF - (wsize - (seq_num + 1)) + 1;
-	}
-	return true;
-}
-
-static int enqueue_reorder_recvframe23a(struct recv_reorder_ctrl *preorder_ctrl,
-					struct recv_frame *prframe)
-{
-	struct rx_pkt_attrib *pattrib = &prframe->attrib;
-	struct rtw_queue *ppending_recvframe_queue;
-	struct list_head *phead, *plist, *ptmp;
-	struct recv_frame *hdr;
-	struct rx_pkt_attrib *pnextattrib;
-
-	ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
-	phead = get_list_head(ppending_recvframe_queue);
-
-	list_for_each_safe(plist, ptmp, phead) {
-		hdr = container_of(plist, struct recv_frame, list);
-		pnextattrib = &hdr->attrib;
-
-		if (SN_LESS(pnextattrib->seq_num, pattrib->seq_num)) {
-			continue;
-		} else if (SN_EQUAL(pnextattrib->seq_num, pattrib->seq_num)) {
-			/* Duplicate entry is found!! Do not insert current entry. */
-			return false;
-		} else {
-			break;
-		}
-
-	}
-
-	list_del_init(&prframe->list);
-
-	list_add_tail(&prframe->list, plist);
-
-	return true;
-}
-
-int recv_indicatepkts_in_order(struct rtw_adapter *padapter,
-			       struct recv_reorder_ctrl *preorder_ctrl,
-			       int bforced);
-int recv_indicatepkts_in_order(struct rtw_adapter *padapter,
-			       struct recv_reorder_ctrl *preorder_ctrl,
-			       int bforced)
-{
-	struct list_head *phead, *plist;
-	struct recv_frame *prframe;
-	struct rx_pkt_attrib *pattrib;
-	int bPktInBuf = false;
-	struct recv_priv *precvpriv;
-	struct rtw_queue *ppending_recvframe_queue;
-
-	precvpriv = &padapter->recvpriv;
-	ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
-	phead =	get_list_head(ppending_recvframe_queue);
-	plist = phead->next;
-
-	/*  Handling some condition for forced indicate case. */
-	if (bforced) {
-		if (list_empty(phead)) {
-			return true;
-		}
-
-		prframe = container_of(plist, struct recv_frame, list);
-		pattrib = &prframe->attrib;
-		preorder_ctrl->indicate_seq = pattrib->seq_num;
-	}
-
-	/*  Prepare indication list and indication. */
-	/*  Check if there is any packet need indicate. */
-	while (!list_empty(phead)) {
-
-		prframe = container_of(plist, struct recv_frame, list);
-		pattrib = &prframe->attrib;
-
-		if (!SN_LESS(preorder_ctrl->indicate_seq, pattrib->seq_num)) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_notice_,
-				 "recv_indicatepkts_in_order: indicate =%d seq =%d amsdu =%d\n",
-				 preorder_ctrl->indicate_seq,
-				 pattrib->seq_num, pattrib->amsdu);
-
-			plist = plist->next;
-			list_del_init(&prframe->list);
-
-			if (SN_EQUAL(preorder_ctrl->indicate_seq,
-				     pattrib->seq_num))	{
-				preorder_ctrl->indicate_seq =
-					(preorder_ctrl->indicate_seq + 1)&0xFFF;
-			}
-
-			if (!pattrib->amsdu) {
-				if ((padapter->bDriverStopped == false) &&
-				    (padapter->bSurpriseRemoved == false)) {
-					rtw_recv_indicatepkt23a(padapter, prframe);
-				}
-			} else {
-				if (amsdu_to_msdu(padapter, prframe) !=
-				    _SUCCESS)
-					rtw_free_recvframe23a(prframe);
-			}
-
-			/* Update local variables. */
-			bPktInBuf = false;
-
-		} else {
-			bPktInBuf = true;
-			break;
-		}
-
-	}
-
-	return bPktInBuf;
-}
-
-int recv_indicatepkt_reorder(struct rtw_adapter *padapter,
-			     struct recv_frame *prframe);
-int recv_indicatepkt_reorder(struct rtw_adapter *padapter,
-			     struct recv_frame *prframe)
-{
-	int retval = _SUCCESS;
-	struct rx_pkt_attrib *pattrib;
-	struct recv_reorder_ctrl *preorder_ctrl;
-	struct rtw_queue *ppending_recvframe_queue;
-
-	pattrib = &prframe->attrib;
-	preorder_ctrl = prframe->preorder_ctrl;
-	ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
-
-	if (!pattrib->amsdu) {
-		/* s1. */
-		wlanhdr_to_ethhdr(prframe);
-
-		if ((pattrib->qos!= 1) || (pattrib->eth_type == ETH_P_ARP) ||
-		    (pattrib->ack_policy != 0)) {
-			if ((padapter->bDriverStopped == false) &&
-			    (padapter->bSurpriseRemoved == false)) {
-				RT_TRACE(_module_rtl871x_recv_c_, _drv_notice_,
-					 "@@@@  recv_indicatepkt_reorder -recv_func recv_indicatepkt\n");
-
-				rtw_recv_indicatepkt23a(padapter, prframe);
-				return _SUCCESS;
-			}
-
-			return _FAIL;
-		}
-
-		if (preorder_ctrl->enable == false) {
-			/* indicate this recv_frame */
-			preorder_ctrl->indicate_seq = pattrib->seq_num;
-			rtw_recv_indicatepkt23a(padapter, prframe);
-
-			preorder_ctrl->indicate_seq =
-				(preorder_ctrl->indicate_seq + 1) % 4096;
-			return _SUCCESS;
-		}
-	} else {
-		 /* temp filter -> means didn't support A-MSDUs in a A-MPDU */
-		if (preorder_ctrl->enable == false) {
-			preorder_ctrl->indicate_seq = pattrib->seq_num;
-			retval = amsdu_to_msdu(padapter, prframe);
-
-			preorder_ctrl->indicate_seq =
-				(preorder_ctrl->indicate_seq + 1) % 4096;
-			return retval;
-		}
-	}
-
-	spin_lock_bh(&ppending_recvframe_queue->lock);
-
-	RT_TRACE(_module_rtl871x_recv_c_, _drv_notice_,
-		 "recv_indicatepkt_reorder: indicate =%d seq =%d\n",
-		 preorder_ctrl->indicate_seq, pattrib->seq_num);
-
-	/* s2. check if winstart_b(indicate_seq) needs to been updated */
-	if (!check_indicate_seq(preorder_ctrl, pattrib->seq_num)) {
-		goto _err_exit;
-	}
-
-	/* s3. Insert all packet into Reorder Queue to maintain its ordering. */
-	if (!enqueue_reorder_recvframe23a(preorder_ctrl, prframe)) {
-		goto _err_exit;
-	}
-
-	/* s4. */
-	/*  Indication process. */
-	/*  After Packet dropping and Sliding Window shifting as above,
-	    we can now just indicate the packets */
-	/*  with the SeqNum smaller than latest WinStart and buffer
-	    other packets. */
-	/*  */
-	/*  For Rx Reorder condition: */
-	/*  1. All packets with SeqNum smaller than WinStart => Indicate */
-	/*  2. All packets with SeqNum larger than or equal to WinStart =>
-	    Buffer it. */
-	/*  */
-
-	if (recv_indicatepkts_in_order(padapter, preorder_ctrl, false) == true) {
-		mod_timer(&preorder_ctrl->reordering_ctrl_timer,
-			  jiffies + msecs_to_jiffies(REORDER_WAIT_TIME));
-		spin_unlock_bh(&ppending_recvframe_queue->lock);
-	} else {
-		spin_unlock_bh(&ppending_recvframe_queue->lock);
-		del_timer_sync(&preorder_ctrl->reordering_ctrl_timer);
-	}
-	return _SUCCESS;
-
-_err_exit:
-
-	spin_unlock_bh(&ppending_recvframe_queue->lock);
-	return _FAIL;
-}
-
-void rtw_reordering_ctrl_timeout_handler23a(unsigned long pcontext)
-{
-	struct recv_reorder_ctrl *preorder_ctrl;
-	struct rtw_adapter *padapter;
-	struct rtw_queue *ppending_recvframe_queue;
-
-	preorder_ctrl = (struct recv_reorder_ctrl *)pcontext;
-	padapter = preorder_ctrl->padapter;
-	ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
-
-	if (padapter->bDriverStopped || padapter->bSurpriseRemoved) {
-		return;
-	}
-
-	spin_lock_bh(&ppending_recvframe_queue->lock);
-
-	if (recv_indicatepkts_in_order(padapter, preorder_ctrl, true) == true) {
-		mod_timer(&preorder_ctrl->reordering_ctrl_timer,
-			  jiffies + msecs_to_jiffies(REORDER_WAIT_TIME));
-	}
-
-	spin_unlock_bh(&ppending_recvframe_queue->lock);
-}
-
-int process_recv_indicatepkts(struct rtw_adapter *padapter,
-			      struct recv_frame *prframe);
-int process_recv_indicatepkts(struct rtw_adapter *padapter,
-			      struct recv_frame *prframe)
-{
-	int retval = _SUCCESS;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-
-	if (phtpriv->ht_option == true) { /* B/G/N Mode */
-		/*  including perform A-MPDU Rx Ordering Buffer Control */
-		if (recv_indicatepkt_reorder(padapter, prframe) != _SUCCESS) {
-			if ((padapter->bDriverStopped == false) &&
-			    (padapter->bSurpriseRemoved == false)) {
-				retval = _FAIL;
-				return retval;
-			}
-		}
-	} else { /* B/G mode */
-		retval = wlanhdr_to_ethhdr(prframe);
-		if (retval != _SUCCESS) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "wlanhdr_to_ethhdr: drop pkt\n");
-			return retval;
-		}
-
-		if ((padapter->bDriverStopped == false) &&
-		    (padapter->bSurpriseRemoved == false)) {
-			/* indicate this recv_frame */
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_notice_,
-				 "@@@@ process_recv_indicatepkts- recv_func recv_indicatepkt\n");
-			rtw_recv_indicatepkt23a(padapter, prframe);
-		} else {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_notice_,
-				 "@@@@ process_recv_indicatepkts- recv_func free_indicatepkt\n");
-
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_notice_,
-				 "recv_func:bDriverStopped(%d) OR bSurpriseRemoved(%d)\n",
-				 padapter->bDriverStopped,
-				 padapter->bSurpriseRemoved);
-			retval = _FAIL;
-			return retval;
-		}
-
-	}
-
-	return retval;
-}
-
-static int recv_func_prehandle(struct rtw_adapter *padapter,
-			       struct recv_frame *rframe)
-{
-	int ret;
-
-	/* check the frame crtl field and decache */
-	ret = validate_recv_frame(padapter, rframe);
-	if (ret != _SUCCESS) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-			 "recv_func: validate_recv_frame fail! drop pkt\n");
-		rtw_free_recvframe23a(rframe);
-		goto exit;
-	}
-
-exit:
-	return ret;
-}
-
-static int recv_func_posthandle(struct rtw_adapter *padapter,
-				struct recv_frame *prframe)
-{
-	int ret = _SUCCESS;
-	struct recv_frame *orig_prframe = prframe;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-
-	/*  DATA FRAME */
-	prframe = decryptor(padapter, prframe);
-	if (prframe == NULL) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "decryptor: drop pkt\n");
-		ret = _FAIL;
-		goto _recv_data_drop;
-	}
-
-	prframe = recvframe_chk_defrag23a(padapter, prframe);
-	if (!prframe) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "recvframe_chk_defrag23a: drop pkt\n");
-		goto _recv_data_drop;
-	}
-
-	/*
-	 * Pull off crypto headers
-	 */
-	if (prframe->attrib.iv_len > 0) {
-		skb_pull(prframe->pkt, prframe->attrib.iv_len);
-	}
-
-	if (prframe->attrib.icv_len > 0) {
-		skb_trim(prframe->pkt,
-			 prframe->pkt->len - prframe->attrib.icv_len);
-	}
-
-	prframe = portctrl(padapter, prframe);
-	if (!prframe) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "portctrl: drop pkt\n");
-		ret = _FAIL;
-		goto _recv_data_drop;
-	}
-
-	count_rx_stats(padapter, prframe, NULL);
-
-	ret = process_recv_indicatepkts(padapter, prframe);
-	if (ret != _SUCCESS) {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "recv_func: process_recv_indicatepkts fail!\n");
-		rtw_free_recvframe23a(orig_prframe);/* free this recv_frame */
-		goto _recv_data_drop;
-	}
-	return ret;
-
-_recv_data_drop:
-	precvpriv->rx_drop++;
-	return ret;
-}
-
-int rtw_recv_entry23a(struct recv_frame *rframe)
-{
-	int ret, r;
-	struct rtw_adapter *padapter = rframe->adapter;
-	struct rx_pkt_attrib *prxattrib = &rframe->attrib;
-	struct recv_priv *recvpriv = &padapter->recvpriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct mlme_priv *mlmepriv = &padapter->mlmepriv;
-
-	/* check if need to handle uc_swdec_pending_queue*/
-	if (check_fwstate(mlmepriv, WIFI_STATION_STATE) &&
-	    psecuritypriv->busetkipkey)	{
-		struct recv_frame *pending_frame;
-
-		while ((pending_frame = rtw_alloc_recvframe23a(&padapter->recvpriv.uc_swdec_pending_queue))) {
-			r = recv_func_posthandle(padapter, pending_frame);
-			if (r == _SUCCESS)
-				DBG_8723A("%s: dequeue uc_swdec_pending_queue\n", __func__);
-		}
-	}
-
-	ret = recv_func_prehandle(padapter, rframe);
-
-	if (ret == _SUCCESS) {
-		/* check if need to enqueue into uc_swdec_pending_queue*/
-		if (check_fwstate(mlmepriv, WIFI_STATION_STATE) &&
-		    !is_multicast_ether_addr(prxattrib->ra) &&
-		    prxattrib->encrypt > 0 &&
-		    (prxattrib->bdecrypted == 0) &&
-		    !is_wep_enc(psecuritypriv->dot11PrivacyAlgrthm) &&
-		    !psecuritypriv->busetkipkey) {
-			rtw_enqueue_recvframe23a(rframe, &padapter->recvpriv.uc_swdec_pending_queue);
-			DBG_8723A("%s: no key, enqueue uc_swdec_pending_queue\n", __func__);
-			goto exit;
-		}
-
-		ret = recv_func_posthandle(padapter, rframe);
-
-		recvpriv->rx_pkts++;
-	}
-
-exit:
-	return ret;
-}
-
-void rtw_signal_stat_timer_hdl23a(unsigned long data)
-{
-	struct rtw_adapter *adapter = (struct rtw_adapter *)data;
-	struct recv_priv *recvpriv = &adapter->recvpriv;
-
-	u32 tmp_s, tmp_q;
-	u8 avg_signal_strength = 0;
-	u8 avg_signal_qual = 0;
-	u32 num_signal_strength = 0;
-	u32 num_signal_qual = 0;
-	u8 _alpha = 3;	/* this value is based on converging_constant = 5000 */
-			/* and sampling_interval = 1000 */
-
-	if (recvpriv->signal_strength_data.update_req == 0) {
-		/*  update_req is clear, means we got rx */
-		avg_signal_strength = recvpriv->signal_strength_data.avg_val;
-		num_signal_strength = recvpriv->signal_strength_data.total_num;
-		/*  after avg_vals are acquired, we can re-stat */
-		/* the signal values */
-		recvpriv->signal_strength_data.update_req = 1;
-	}
-
-	if (recvpriv->signal_qual_data.update_req == 0) {
-		/*  update_req is clear, means we got rx */
-		avg_signal_qual = recvpriv->signal_qual_data.avg_val;
-		num_signal_qual = recvpriv->signal_qual_data.total_num;
-		/*  after avg_vals are acquired, we can re-stat */
-		/*the signal values */
-		recvpriv->signal_qual_data.update_req = 1;
-	}
-
-	/* update value of signal_strength, rssi, signal_qual */
-	if (!check_fwstate(&adapter->mlmepriv, _FW_UNDER_SURVEY)) {
-		tmp_s = avg_signal_strength + (_alpha - 1) *
-			 recvpriv->signal_strength;
-		if (tmp_s %_alpha)
-			tmp_s = tmp_s / _alpha + 1;
-		else
-			tmp_s = tmp_s / _alpha;
-		if (tmp_s > 100)
-			tmp_s = 100;
-
-		tmp_q = avg_signal_qual + (_alpha - 1) * recvpriv->signal_qual;
-		if (tmp_q %_alpha)
-			tmp_q = tmp_q / _alpha + 1;
-		else
-			tmp_q = tmp_q / _alpha;
-		if (tmp_q > 100)
-			tmp_q = 100;
-
-		recvpriv->signal_strength = tmp_s;
-		recvpriv->signal_qual = tmp_q;
-
-		DBG_8723A("%s signal_strength:%3u, signal_qual:%3u, "
-			  "num_signal_strength:%u, num_signal_qual:%u\n",
-			  __func__, recvpriv->signal_strength,
-			  recvpriv->signal_qual, num_signal_strength,
-			  num_signal_qual);
-	}
-
-	rtw_set_signal_stat_timer(recvpriv);
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_security.c b/drivers/staging/rtl8723au/core/rtw_security.c
deleted file mode 100644
index 5a4cfdf1ebd4..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_security.c
+++ /dev/null
@@ -1,1630 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define  _RTW_SECURITY_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <wifi.h>
-#include <osdep_intf.h>
-
-/* WEP related ===== */
-
-#define CRC32_POLY 0x04c11db7
-
-struct arc4context {
-	u32 x;
-	u32 y;
-	u8 state[256];
-};
-
-static void arcfour_init(struct arc4context *parc4ctx, u8 *key, u32 key_len)
-{
-	u32 t, u;
-	u32 keyindex;
-	u32 stateindex;
-	u8 *state;
-	u32 counter;
-
-	state = parc4ctx->state;
-	parc4ctx->x = 0;
-	parc4ctx->y = 0;
-	for (counter = 0; counter < 256; counter++)
-		state[counter] = (u8)counter;
-	keyindex = 0;
-	stateindex = 0;
-	for (counter = 0; counter < 256; counter++) {
-		t = state[counter];
-		stateindex = (stateindex + key[keyindex] + t) & 0xff;
-		u = state[stateindex];
-		state[stateindex] = (u8)t;
-		state[counter] = (u8)u;
-		if (++keyindex >= key_len)
-			keyindex = 0;
-	}
-
-}
-
-static u32 arcfour_byte(struct arc4context *parc4ctx)
-{
-	u32 x;
-	u32 y;
-	u32 sx, sy;
-	u8 *state;
-
-	state = parc4ctx->state;
-	x = (parc4ctx->x + 1) & 0xff;
-	sx = state[x];
-	y = (sx + parc4ctx->y) & 0xff;
-	sy = state[y];
-	parc4ctx->x = x;
-	parc4ctx->y = y;
-	state[y] = (u8)sx;
-	state[x] = (u8)sy;
-
-	return state[(sx + sy) & 0xff];
-}
-
-static void arcfour_encrypt(struct arc4context *parc4ctx, u8 *dest,
-			    u8 *src, u32 len)
-{
-	u32 i;
-
-	for (i = 0; i < len; i++)
-		dest[i] = src[i] ^ (unsigned char)arcfour_byte(parc4ctx);
-}
-
-static int bcrc32initialized;
-static u32 crc32_table[256];
-
-static u8 crc32_reverseBit(u8 data)
-{
-	u8 retval = ((data << 7) & 0x80) | ((data << 5) & 0x40) |
-		((data << 3) & 0x20) | ((data << 1) & 0x10) |
-		((data >> 1) & 0x08) | ((data >> 3) & 0x04) |
-		((data >> 5) & 0x02) | ((data >> 7) & 0x01);
-	return retval;
-}
-
-static void crc32_init(void)
-{
-	int i, j;
-	u32 c;
-	u8 *p, *p1;
-	u8 k;
-
-	if (bcrc32initialized == 1)
-		return;
-
-	p = (u8 *) &c;
-	c = 0x12340000;
-
-	for (i = 0; i < 256; ++i) {
-		k = crc32_reverseBit((u8)i);
-
-		for (c = ((u32)k) << 24, j = 8; j > 0; --j)
-			c = c & 0x80000000 ? (c << 1) ^ CRC32_POLY : (c << 1);
-
-		p1 = (u8 *)&crc32_table[i];
-
-		p1[0] = crc32_reverseBit(p[3]);
-		p1[1] = crc32_reverseBit(p[2]);
-		p1[2] = crc32_reverseBit(p[1]);
-		p1[3] = crc32_reverseBit(p[0]);
-	}
-
-	bcrc32initialized = 1;
-}
-
-static u32 getcrc32(u8 *buf, int len)
-{
-	u8 *p;
-	u32 crc;
-
-	if (bcrc32initialized == 0)
-		crc32_init();
-
-	crc = 0xffffffff; /* preload shift register, per CRC-32 spec */
-
-	for (p = buf; len > 0; ++p, --len)
-		crc = crc32_table[(crc ^ *p) & 0xff] ^ (crc >> 8);
-
-	return ~crc; /* transmit complement, per CRC-32 spec */
-}
-
-/* Need to consider the fragment  situation */
-void rtw_wep_encrypt23a(struct rtw_adapter *padapter,
-		     struct xmit_frame *pxmitframe)
-{
-	/*  exclude ICV */
-	__le32 crc;
-	struct arc4context mycontext;
-	int curfragnum, length, index;
-	u32 keylength;
-	u8 *pframe, *payload, *iv; /* wepkey */
-	u8 wepkey[16];
-	u8 hw_hdr_offset = 0;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	if (!pxmitframe->buf_addr)
-		return;
-
-	hw_hdr_offset = TXDESC_OFFSET;
-
-	pframe = pxmitframe->buf_addr + hw_hdr_offset;
-
-	/* start to encrypt each fragment */
-	if (pattrib->encrypt != WLAN_CIPHER_SUITE_WEP40 &&
-	    pattrib->encrypt != WLAN_CIPHER_SUITE_WEP104)
-		return;
-
-	index = psecuritypriv->dot11PrivacyKeyIndex;
-	keylength = psecuritypriv->wep_key[index].keylen;
-
-	for (curfragnum = 0; curfragnum < pattrib->nr_frags ; curfragnum++) {
-		iv = pframe + pattrib->hdrlen;
-		memcpy(&wepkey[0], iv, 3);
-		memcpy(&wepkey[3], &psecuritypriv->wep_key[index].key,
-		       keylength);
-		payload = pframe + pattrib->iv_len + pattrib->hdrlen;
-
-		if ((curfragnum + 1) == pattrib->nr_frags) {
-			/* the last fragment */
-			length = pattrib->last_txcmdsz - pattrib->hdrlen -
-				pattrib->iv_len - pattrib->icv_len;
-
-			crc = cpu_to_le32(getcrc32(payload, length));
-
-			arcfour_init(&mycontext, wepkey, 3 + keylength);
-			arcfour_encrypt(&mycontext, payload, payload, length);
-			arcfour_encrypt(&mycontext, payload + length,
-					(char *)&crc, 4);
-		} else {
-			length = pxmitpriv->frag_len - pattrib->hdrlen -
-				pattrib->iv_len - pattrib->icv_len;
-			crc = cpu_to_le32(getcrc32(payload, length));
-			arcfour_init(&mycontext, wepkey, 3 + keylength);
-			arcfour_encrypt(&mycontext, payload, payload, length);
-			arcfour_encrypt(&mycontext, payload + length,
-					(char *)&crc, 4);
-
-			pframe += pxmitpriv->frag_len;
-			pframe = PTR_ALIGN(pframe, 4);
-		}
-	}
-
-}
-
-void rtw_wep_decrypt23a(struct rtw_adapter *padapter,
-		     struct recv_frame *precvframe)
-{
-	/*  exclude ICV */
-	u32 actual_crc, expected_crc;
-	struct arc4context mycontext;
-	int length;
-	u32 keylength;
-	u8 *pframe, *payload, *iv, wepkey[16];
-	u8 keyindex;
-	struct rx_pkt_attrib *prxattrib = &precvframe->attrib;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct sk_buff *skb = precvframe->pkt;
-
-	pframe = skb->data;
-
-	/* start to decrypt recvframe */
-	if (prxattrib->encrypt != WLAN_CIPHER_SUITE_WEP40 &&
-	    prxattrib->encrypt != WLAN_CIPHER_SUITE_WEP104)
-		return;
-
-	iv = pframe + prxattrib->hdrlen;
-	/* keyindex = (iv[3]&0x3); */
-	keyindex = prxattrib->key_index;
-	keylength = psecuritypriv->wep_key[keyindex].keylen;
-	memcpy(&wepkey[0], iv, 3);
-	/* memcpy(&wepkey[3], &psecuritypriv->dot11DefKey[psecuritypriv->dot11PrivacyKeyIndex].skey[0], keylength); */
-	memcpy(&wepkey[3], &psecuritypriv->wep_key[keyindex].key, keylength);
-	length = skb->len - prxattrib->hdrlen - prxattrib->iv_len;
-
-	payload = pframe + prxattrib->iv_len + prxattrib->hdrlen;
-
-	/* decrypt payload include icv */
-	arcfour_init(&mycontext, wepkey, 3 + keylength);
-	arcfour_encrypt(&mycontext, payload, payload, length);
-
-	/* calculate icv and compare the icv */
-	actual_crc = getcrc32(payload, length - 4);
-	expected_crc = get_unaligned_le32(&payload[length - 4]);
-
-	if (actual_crc != expected_crc) {
-		RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-			 "%s:icv CRC mismatch: "
-			 "actual: %08x, expected: %08x\n",
-			 __func__, actual_crc, expected_crc);
-	}
-}
-
-/* 3		===== TKIP related ===== */
-
-static u32 secmicgetuint32(u8 *p)
-/*  Convert from Byte[] to u32 in a portable way */
-{
-	s32 i;
-	u32 res = 0;
-
-	for (i = 0; i < 4; i++)
-		res |= ((u32)(*p++)) << (8 * i);
-
-	return res;
-}
-
-static void secmicputuint32(u8 *p, u32 val)
-/*  Convert from long to Byte[] in a portable way */
-{
-	long i;
-
-	for (i = 0; i < 4; i++) {
-		*p++ = (u8) (val & 0xff);
-		val >>= 8;
-	}
-
-}
-
-static void secmicclear(struct mic_data *pmicdata)
-{
-/*  Reset the state to the empty message. */
-
-	pmicdata->L = pmicdata->K0;
-	pmicdata->R = pmicdata->K1;
-	pmicdata->nBytesInM = 0;
-	pmicdata->M = 0;
-
-}
-
-void rtw_secmicsetkey23a(struct mic_data *pmicdata, u8 *key)
-{
-	/*  Set the key */
-
-	pmicdata->K0 = secmicgetuint32(key);
-	pmicdata->K1 = secmicgetuint32(key + 4);
-	/*  and reset the message */
-	secmicclear(pmicdata);
-
-}
-
-void rtw_secmicappend23abyte23a(struct mic_data *pmicdata, u8 b)
-{
-
-	/*  Append the byte to our word-sized buffer */
-	pmicdata->M |= ((unsigned long)b) << (8 * pmicdata->nBytesInM);
-	pmicdata->nBytesInM++;
-	/*  Process the word if it is full. */
-	if (pmicdata->nBytesInM >= 4) {
-		pmicdata->L ^= pmicdata->M;
-		pmicdata->R ^= ROL32(pmicdata->L, 17);
-		pmicdata->L += pmicdata->R;
-		pmicdata->R ^= ((pmicdata->L & 0xff00ff00) >> 8) | ((pmicdata->L & 0x00ff00ff) << 8);
-		pmicdata->L += pmicdata->R;
-		pmicdata->R ^= ROL32(pmicdata->L, 3);
-		pmicdata->L += pmicdata->R;
-		pmicdata->R ^= ROR32(pmicdata->L, 2);
-		pmicdata->L += pmicdata->R;
-		/*  Clear the buffer */
-		pmicdata->M = 0;
-		pmicdata->nBytesInM = 0;
-	}
-
-}
-
-void rtw_secmicappend23a(struct mic_data *pmicdata, u8 *src, u32 nbytes)
-{
-
-	/*  This is simple */
-	while (nbytes > 0) {
-		rtw_secmicappend23abyte23a(pmicdata, *src++);
-		nbytes--;
-	}
-
-}
-
-void rtw_secgetmic23a(struct mic_data *pmicdata, u8 *dst)
-{
-
-	/*  Append the minimum padding */
-	rtw_secmicappend23abyte23a(pmicdata, 0x5a);
-	rtw_secmicappend23abyte23a(pmicdata, 0);
-	rtw_secmicappend23abyte23a(pmicdata, 0);
-	rtw_secmicappend23abyte23a(pmicdata, 0);
-	rtw_secmicappend23abyte23a(pmicdata, 0);
-	/*  and then zeroes until the length is a multiple of 4 */
-	while (pmicdata->nBytesInM != 0)
-		rtw_secmicappend23abyte23a(pmicdata, 0);
-	/*  The appendByte function has already computed the result. */
-	secmicputuint32(dst, pmicdata->L);
-	secmicputuint32(dst + 4, pmicdata->R);
-	/*  Reset to the empty message. */
-	secmicclear(pmicdata);
-
-}
-
-void rtw_seccalctkipmic23a(u8 *key, u8 *header, u8 *data, u32 data_len,
-			   u8 *mic_code, u8 pri)
-{
-
-	struct mic_data micdata;
-	u8 priority[4] = {0x0, 0x0, 0x0, 0x0};
-
-	rtw_secmicsetkey23a(&micdata, key);
-	priority[0] = pri;
-
-	/* Michael MIC pseudo header: DA, SA, 3 x 0, Priority */
-	if (header[1]&1) { /* ToDS == 1 */
-			rtw_secmicappend23a(&micdata, &header[16], 6); /* DA */
-		if (header[1]&2) /* From Ds == 1 */
-			rtw_secmicappend23a(&micdata, &header[24], 6);
-		else
-			rtw_secmicappend23a(&micdata, &header[10], 6);
-	} else { /* ToDS == 0 */
-		rtw_secmicappend23a(&micdata, &header[4], 6); /* DA */
-		if (header[1]&2) /* From Ds == 1 */
-			rtw_secmicappend23a(&micdata, &header[16], 6);
-		else
-			rtw_secmicappend23a(&micdata, &header[10], 6);
-
-	}
-	rtw_secmicappend23a(&micdata, &priority[0], 4);
-
-	rtw_secmicappend23a(&micdata, data, data_len);
-
-	rtw_secgetmic23a(&micdata, mic_code);
-
-}
-
-/* macros for extraction/creation of unsigned char/unsigned short values  */
-#define RotR1(v16)   ((((v16) >> 1) & 0x7FFF) ^ (((v16) & 1) << 15))
-#define   Lo8(v16)   ((u8)((v16)       & 0x00FF))
-#define   Hi8(v16)   ((u8)(((v16) >> 8) & 0x00FF))
-#define  Lo16(v32)   ((u16)((v32)       & 0xFFFF))
-#define  Hi16(v32)   ((u16)(((v32) >> 16) & 0xFFFF))
-#define  Mk16(hi, lo) ((lo) ^ (((u16)(hi)) << 8))
-
-/* select the Nth 16-bit word of the temporal key unsigned char array TK[] */
-#define  TK16(N)     Mk16(tk[2 * (N) + 1], tk[2 * (N)])
-
-/* S-box lookup: 16 bits --> 16 bits */
-#define _S_(v16)     (Sbox1[0][Lo8(v16)] ^ Sbox1[1][Hi8(v16)])
-
-/* fixed algorithm "parameters" */
-#define PHASE1_LOOP_CNT   8    /* this needs to be "big enough"     */
-#define TA_SIZE           6    /*  48-bit transmitter address       */
-#define TK_SIZE          16    /* 128-bit temporal key              */
-#define P1K_SIZE         10    /*  80-bit Phase1 key                */
-#define RC4_KEY_SIZE     16    /* 128-bit RC4KEY (104 bits unknown) */
-
-/* 2-unsigned char by 2-unsigned char subset of the full AES S-box table */
-static const unsigned short Sbox1[2][256] = {
-	/* Sbox for hash (can be in ROM) */
-	{
-		0xC6A5, 0xF884, 0xEE99, 0xF68D, 0xFF0D, 0xD6BD, 0xDEB1, 0x9154,
-		0x6050, 0x0203, 0xCEA9, 0x567D, 0xE719, 0xB562, 0x4DE6, 0xEC9A,
-		0x8F45, 0x1F9D, 0x8940, 0xFA87, 0xEF15, 0xB2EB, 0x8EC9, 0xFB0B,
-		0x41EC, 0xB367, 0x5FFD, 0x45EA, 0x23BF, 0x53F7, 0xE496, 0x9B5B,
-		0x75C2, 0xE11C, 0x3DAE, 0x4C6A, 0x6C5A, 0x7E41, 0xF502, 0x834F,
-		0x685C, 0x51F4, 0xD134, 0xF908, 0xE293, 0xAB73, 0x6253, 0x2A3F,
-		0x080C, 0x9552, 0x4665, 0x9D5E, 0x3028, 0x37A1, 0x0A0F, 0x2FB5,
-		0x0E09, 0x2436, 0x1B9B, 0xDF3D, 0xCD26, 0x4E69, 0x7FCD, 0xEA9F,
-		0x121B, 0x1D9E, 0x5874, 0x342E, 0x362D, 0xDCB2, 0xB4EE, 0x5BFB,
-		0xA4F6, 0x764D, 0xB761, 0x7DCE, 0x527B, 0xDD3E, 0x5E71, 0x1397,
-		0xA6F5, 0xB968, 0x0000, 0xC12C, 0x4060, 0xE31F, 0x79C8, 0xB6ED,
-		0xD4BE, 0x8D46, 0x67D9, 0x724B, 0x94DE, 0x98D4, 0xB0E8, 0x854A,
-		0xBB6B, 0xC52A, 0x4FE5, 0xED16, 0x86C5, 0x9AD7, 0x6655, 0x1194,
-		0x8ACF, 0xE910, 0x0406, 0xFE81, 0xA0F0, 0x7844, 0x25BA, 0x4BE3,
-		0xA2F3, 0x5DFE, 0x80C0, 0x058A, 0x3FAD, 0x21BC, 0x7048, 0xF104,
-		0x63DF, 0x77C1, 0xAF75, 0x4263, 0x2030, 0xE51A, 0xFD0E, 0xBF6D,
-		0x814C, 0x1814, 0x2635, 0xC32F, 0xBEE1, 0x35A2, 0x88CC, 0x2E39,
-		0x9357, 0x55F2, 0xFC82, 0x7A47, 0xC8AC, 0xBAE7, 0x322B, 0xE695,
-		0xC0A0, 0x1998, 0x9ED1, 0xA37F, 0x4466, 0x547E, 0x3BAB, 0x0B83,
-		0x8CCA, 0xC729, 0x6BD3, 0x283C, 0xA779, 0xBCE2, 0x161D, 0xAD76,
-		0xDB3B, 0x6456, 0x744E, 0x141E, 0x92DB, 0x0C0A, 0x486C, 0xB8E4,
-		0x9F5D, 0xBD6E, 0x43EF, 0xC4A6, 0x39A8, 0x31A4, 0xD337, 0xF28B,
-		0xD532, 0x8B43, 0x6E59, 0xDAB7, 0x018C, 0xB164, 0x9CD2, 0x49E0,
-		0xD8B4, 0xACFA, 0xF307, 0xCF25, 0xCAAF, 0xF48E, 0x47E9, 0x1018,
-		0x6FD5, 0xF088, 0x4A6F, 0x5C72, 0x3824, 0x57F1, 0x73C7, 0x9751,
-		0xCB23, 0xA17C, 0xE89C, 0x3E21, 0x96DD, 0x61DC, 0x0D86, 0x0F85,
-		0xE090, 0x7C42, 0x71C4, 0xCCAA, 0x90D8, 0x0605, 0xF701, 0x1C12,
-		0xC2A3, 0x6A5F, 0xAEF9, 0x69D0, 0x1791, 0x9958, 0x3A27, 0x27B9,
-		0xD938, 0xEB13, 0x2BB3, 0x2233, 0xD2BB, 0xA970, 0x0789, 0x33A7,
-		0x2DB6, 0x3C22, 0x1592, 0xC920, 0x8749, 0xAAFF, 0x5078, 0xA57A,
-		0x038F, 0x59F8, 0x0980, 0x1A17, 0x65DA, 0xD731, 0x84C6, 0xD0B8,
-		0x82C3, 0x29B0, 0x5A77, 0x1E11, 0x7BCB, 0xA8FC, 0x6DD6, 0x2C3A,
-	},
-	{ /* second half of table is unsigned char-reversed version of first! */
-		0xA5C6, 0x84F8, 0x99EE, 0x8DF6, 0x0DFF, 0xBDD6, 0xB1DE, 0x5491,
-		0x5060, 0x0302, 0xA9CE, 0x7D56, 0x19E7, 0x62B5, 0xE64D, 0x9AEC,
-		0x458F, 0x9D1F, 0x4089, 0x87FA, 0x15EF, 0xEBB2, 0xC98E, 0x0BFB,
-		0xEC41, 0x67B3, 0xFD5F, 0xEA45, 0xBF23, 0xF753, 0x96E4, 0x5B9B,
-		0xC275, 0x1CE1, 0xAE3D, 0x6A4C, 0x5A6C, 0x417E, 0x02F5, 0x4F83,
-		0x5C68, 0xF451, 0x34D1, 0x08F9, 0x93E2, 0x73AB, 0x5362, 0x3F2A,
-		0x0C08, 0x5295, 0x6546, 0x5E9D, 0x2830, 0xA137, 0x0F0A, 0xB52F,
-		0x090E, 0x3624, 0x9B1B, 0x3DDF, 0x26CD, 0x694E, 0xCD7F, 0x9FEA,
-		0x1B12, 0x9E1D, 0x7458, 0x2E34, 0x2D36, 0xB2DC, 0xEEB4, 0xFB5B,
-		0xF6A4, 0x4D76, 0x61B7, 0xCE7D, 0x7B52, 0x3EDD, 0x715E, 0x9713,
-		0xF5A6, 0x68B9, 0x0000, 0x2CC1, 0x6040, 0x1FE3, 0xC879, 0xEDB6,
-		0xBED4, 0x468D, 0xD967, 0x4B72, 0xDE94, 0xD498, 0xE8B0, 0x4A85,
-		0x6BBB, 0x2AC5, 0xE54F, 0x16ED, 0xC586, 0xD79A, 0x5566, 0x9411,
-		0xCF8A, 0x10E9, 0x0604, 0x81FE, 0xF0A0, 0x4478, 0xBA25, 0xE34B,
-		0xF3A2, 0xFE5D, 0xC080, 0x8A05, 0xAD3F, 0xBC21, 0x4870, 0x04F1,
-		0xDF63, 0xC177, 0x75AF, 0x6342, 0x3020, 0x1AE5, 0x0EFD, 0x6DBF,
-		0x4C81, 0x1418, 0x3526, 0x2FC3, 0xE1BE, 0xA235, 0xCC88, 0x392E,
-		0x5793, 0xF255, 0x82FC, 0x477A, 0xACC8, 0xE7BA, 0x2B32, 0x95E6,
-		0xA0C0, 0x9819, 0xD19E, 0x7FA3, 0x6644, 0x7E54, 0xAB3B, 0x830B,
-		0xCA8C, 0x29C7, 0xD36B, 0x3C28, 0x79A7, 0xE2BC, 0x1D16, 0x76AD,
-		0x3BDB, 0x5664, 0x4E74, 0x1E14, 0xDB92, 0x0A0C, 0x6C48, 0xE4B8,
-		0x5D9F, 0x6EBD, 0xEF43, 0xA6C4, 0xA839, 0xA431, 0x37D3, 0x8BF2,
-		0x32D5, 0x438B, 0x596E, 0xB7DA, 0x8C01, 0x64B1, 0xD29C, 0xE049,
-		0xB4D8, 0xFAAC, 0x07F3, 0x25CF, 0xAFCA, 0x8EF4, 0xE947, 0x1810,
-		0xD56F, 0x88F0, 0x6F4A, 0x725C, 0x2438, 0xF157, 0xC773, 0x5197,
-		0x23CB, 0x7CA1, 0x9CE8, 0x213E, 0xDD96, 0xDC61, 0x860D, 0x850F,
-		0x90E0, 0x427C, 0xC471, 0xAACC, 0xD890, 0x0506, 0x01F7, 0x121C,
-		0xA3C2, 0x5F6A, 0xF9AE, 0xD069, 0x9117, 0x5899, 0x273A, 0xB927,
-		0x38D9, 0x13EB, 0xB32B, 0x3322, 0xBBD2, 0x70A9, 0x8907, 0xA733,
-		0xB62D, 0x223C, 0x9215, 0x20C9, 0x4987, 0xFFAA, 0x7850, 0x7AA5,
-		0x8F03, 0xF859, 0x8009, 0x171A, 0xDA65, 0x31D7, 0xC684, 0xB8D0,
-		0xC382, 0xB029, 0x775A, 0x111E, 0xCB7B, 0xFCA8, 0xD66D, 0x3A2C,
-	}
-};
-
- /*
-**********************************************************************
-* Routine: Phase 1 -- generate P1K, given TA, TK, IV32
-*
-* Inputs:
-*     tk[]      = temporal key                         [128 bits]
-*     ta[]      = transmitter's MAC address            [ 48 bits]
-*     iv32      = upper 32 bits of IV                  [ 32 bits]
-* Output:
-*     p1k[]     = Phase 1 key                          [ 80 bits]
-*
-* Note:
-*     This function only needs to be called every 2**16 packets,
-*     although in theory it could be called every packet.
-*
-**********************************************************************
-*/
-static void phase1(u16 *p1k, const u8 *tk, const u8 *ta, u32 iv32)
-{
-	int  i;
-
-	/* Initialize the 80 bits of P1K[] from IV32 and TA[0..5] */
-	p1k[0]      = Lo16(iv32);
-	p1k[1]      = Hi16(iv32);
-	p1k[2]      = Mk16(ta[1], ta[0]); /* use TA[] as little-endian */
-	p1k[3]      = Mk16(ta[3], ta[2]);
-	p1k[4]      = Mk16(ta[5], ta[4]);
-
-	/* Now compute an unbalanced Feistel cipher with 80-bit block */
-	/* size on the 80-bit block P1K[], using the 128-bit key TK[] */
-	for (i = 0; i < PHASE1_LOOP_CNT; i++) {
-		/* Each add operation here is mod 2**16 */
-		p1k[0] += _S_(p1k[4] ^ TK16((i & 1) + 0));
-		p1k[1] += _S_(p1k[0] ^ TK16((i & 1) + 2));
-		p1k[2] += _S_(p1k[1] ^ TK16((i & 1) + 4));
-		p1k[3] += _S_(p1k[2] ^ TK16((i & 1) + 6));
-		p1k[4] += _S_(p1k[3] ^ TK16((i & 1) + 0));
-		p1k[4] +=  (unsigned short) i; /* avoid "slide attacks" */
-		}
-
-}
-
-/*
-**********************************************************************
-* Routine: Phase 2 -- generate RC4KEY, given TK, P1K, IV16
-*
-* Inputs:
-*     tk[]      = Temporal key                         [128 bits]
-*     p1k[]     = Phase 1 output key                   [ 80 bits]
-*     iv16      = low 16 bits of IV counter            [ 16 bits]
-* Output:
-*     rc4key[]  = the key used to encrypt the packet   [128 bits]
-*
-* Note:
-*     The value {TA, IV32, IV16} for Phase1/Phase2 must be unique
-*     across all packets using the same key TK value. Then, for a
-*     given value of TK[], this TKIP48 construction guarantees that
-*     the final RC4KEY value is unique across all packets.
-*
-* Suggested implementation optimization: if PPK[] is "overlaid"
-*     appropriately on RC4KEY[], there is no need for the final
-*     for loop below that copies the PPK[] result into RC4KEY[].
-*
-**********************************************************************
-*/
-static void phase2(u8 *rc4key, const u8 *tk, const u16 *p1k, u16 iv16)
-{
-	int  i;
-	u16 PPK[6]; /* temporary key for mixing    */
-
-	/* Note: all adds in the PPK[] equations below are mod 2**16 */
-	for (i = 0; i < 5; i++)
-		PPK[i] = p1k[i]; /* first, copy P1K to PPK */
-
-	PPK[5] = p1k[4] + iv16; /* next,  add in IV16 */
-
-	/* Bijective non-linear mixing of the 96 bits of PPK[0..5] */
-	PPK[0] += _S_(PPK[5] ^ TK16(0)); /* Mix key in each "round" */
-	PPK[1] += _S_(PPK[0] ^ TK16(1));
-	PPK[2] += _S_(PPK[1] ^ TK16(2));
-	PPK[3] += _S_(PPK[2] ^ TK16(3));
-	PPK[4] += _S_(PPK[3] ^ TK16(4));
-	PPK[5] += _S_(PPK[4] ^ TK16(5)); /* Total # S-box lookups == 6 */
-
-	/* Final sweep: bijective, "linear". Rotates kill LSB correlations */
-	PPK[0] +=  RotR1(PPK[5] ^ TK16(6));
-	PPK[1] +=  RotR1(PPK[0] ^ TK16(7)); /* Use all of TK[] in Phase2 */
-	PPK[2] +=  RotR1(PPK[1]);
-	PPK[3] +=  RotR1(PPK[2]);
-	PPK[4] +=  RotR1(PPK[3]);
-	PPK[5] +=  RotR1(PPK[4]);
-	/* Note: At this point, for a given key TK[0..15], the 96-bit output */
-	/*       value PPK[0..5] is guaranteed to be unique, as a function   */
-	/*       of the 96-bit "input" value   {TA, IV32, IV16}. That is,    */
-	/*       P1K is now a keyed permutation of {TA, IV32, IV16}.         */
-
-	/* Set RC4KEY[0..3], which includes "cleartext" portion of RC4 key   */
-	rc4key[0] = Hi8(iv16);                 /* RC4KEY[0..2] is the WEP IV */
-	rc4key[1] = (Hi8(iv16) | 0x20) & 0x7F; /* Help avoid weak (FMS) keys */
-	rc4key[2] = Lo8(iv16);
-	rc4key[3] = Lo8((PPK[5] ^ TK16(0)) >> 1);
-
-	/* Copy 96 bits of PPK[0..5] to RC4KEY[4..15]  (little-endian) */
-	for (i = 0; i < 6; i++) {
-		rc4key[4 + 2 * i] = Lo8(PPK[i]);
-		rc4key[5 + 2 * i] = Hi8(PPK[i]);
-	}
-
-}
-
-/* The hlen isn't include the IV */
-int rtw_tkip_encrypt23a(struct rtw_adapter *padapter,
-			struct xmit_frame *pxmitframe)
-{
-	u16 pnl;
-	u32 pnh;
-	u8 rc4key[16];
-	u8 ttkey[16];
-	__le32 crc;
-	u8 hw_hdr_offset = 0;
-	struct arc4context mycontext;
-	int curfragnum, length;
-	u8 *pframe, *payload, *iv, *prwskey;
-	union pn48 dot11txpn;
-	struct sta_info *stainfo;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	int res = _SUCCESS;
-
-	if (pattrib->encrypt != WLAN_CIPHER_SUITE_TKIP)
-		return _FAIL;
-
-	if (!pxmitframe->buf_addr)
-		return _FAIL;
-
-	hw_hdr_offset = TXDESC_OFFSET;
-
-	pframe = pxmitframe->buf_addr + hw_hdr_offset;
-
-	if (pattrib->psta)
-		stainfo = pattrib->psta;
-	else {
-		DBG_8723A("%s, call rtw_get_stainfo()\n", __func__);
-		stainfo = rtw_get_stainfo23a(&padapter->stapriv,
-					     &pattrib->ra[0]);
-	}
-
-	if (!stainfo) {
-		RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-			 "%s: stainfo == NULL!!!\n", __func__);
-		DBG_8723A("%s, psta == NUL\n", __func__);
-		return _FAIL;
-	}
-
-	RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-		 "%s: stainfo!= NULL!!!\n", __func__);
-
-	if (!(stainfo->state & _FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n", __func__, stainfo->state);
-		return _FAIL;
-	}
-
-	if (is_multicast_ether_addr(pattrib->ra))
-		prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey;
-	else
-		prwskey = &stainfo->dot118021x_UncstKey.skey[0];
-
-	/* 4 start to encrypt each fragment */
-	for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
-		iv = pframe + pattrib->hdrlen;
-		payload = pframe + pattrib->iv_len + pattrib->hdrlen;
-
-		GET_TKIP_PN(iv, dot11txpn);
-
-		pnl = (u16)(dot11txpn.val);
-		pnh = (u32)(dot11txpn.val>>16);
-
-		phase1((u16 *)&ttkey[0], prwskey, &pattrib->ta[0], pnh);
-
-		phase2(&rc4key[0], prwskey, (u16 *)&ttkey[0], pnl);
-
-		if ((curfragnum + 1) == pattrib->nr_frags) { /* 4 the last fragment */
-			length = (pattrib->last_txcmdsz -
-				  pattrib->hdrlen -
-				  pattrib->iv_len -
-				  pattrib->icv_len);
-
-			RT_TRACE(_module_rtl871x_security_c_, _drv_info_,
-				 "pattrib->iv_len =%x, pattrib->icv_len =%x\n",
-				 pattrib->iv_len,
-				 pattrib->icv_len);
-			crc = cpu_to_le32(getcrc32(payload, length));
-
-			arcfour_init(&mycontext, rc4key, 16);
-			arcfour_encrypt(&mycontext, payload, payload, length);
-			arcfour_encrypt(&mycontext, payload + length,
-					(char *)&crc, 4);
-
-		} else {
-			length = (pxmitpriv->frag_len -
-				  pattrib->hdrlen -
-				  pattrib->iv_len -
-				  pattrib->icv_len);
-
-			crc = cpu_to_le32(getcrc32(payload, length));
-			arcfour_init(&mycontext, rc4key, 16);
-			arcfour_encrypt(&mycontext, payload, payload, length);
-			arcfour_encrypt(&mycontext, payload + length,
-					(char *)&crc, 4);
-
-			pframe += pxmitpriv->frag_len;
-			pframe  = PTR_ALIGN(pframe, 4);
-		}
-	}
-
-	return res;
-}
-
-/* The hlen isn't include the IV */
-int rtw_tkip_decrypt23a(struct rtw_adapter *padapter,
-			struct recv_frame *precvframe)
-{
-	u16 pnl;
-	u32 pnh;
-	u8 rc4key[16];
-	u8 ttkey[16];
-	u32 actual_crc, expected_crc;
-	struct arc4context mycontext;
-	int length;
-	u8 *pframe, *payload, *iv, *prwskey;
-	union pn48 dot11txpn;
-	struct sta_info *stainfo;
-	struct rx_pkt_attrib *prxattrib = &precvframe->attrib;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct sk_buff *skb = precvframe->pkt;
-	int res = _SUCCESS;
-
-	if (prxattrib->encrypt != WLAN_CIPHER_SUITE_TKIP)
-		return _FAIL;
-
-	pframe = skb->data;
-
-	stainfo = rtw_get_stainfo23a(&padapter->stapriv,
-				     &prxattrib->ta[0]);
-	if (!stainfo) {
-		RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-			 "%s: stainfo == NULL!!!\n", __func__);
-		return _FAIL;
-	}
-
-	/* 4 start to decrypt recvframe */
-	if (is_multicast_ether_addr(prxattrib->ra)) {
-		if (psecuritypriv->binstallGrpkey == 0) {
-			res = _FAIL;
-			DBG_8723A("%s:rx bc/mc packets, but didn't install group key!!!!!!!!!!\n", __func__);
-			goto exit;
-		}
-		prwskey = psecuritypriv->dot118021XGrpKey[prxattrib->key_index].skey;
-	} else {
-		RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-			 "%s: stainfo!= NULL!!!\n", __func__);
-		prwskey = &stainfo->dot118021x_UncstKey.skey[0];
-	}
-
-	iv = pframe + prxattrib->hdrlen;
-	payload = pframe + prxattrib->iv_len + prxattrib->hdrlen;
-	length = skb->len - prxattrib->hdrlen - prxattrib->iv_len;
-
-	GET_TKIP_PN(iv, dot11txpn);
-
-	pnl = (u16)(dot11txpn.val);
-	pnh = (u32)(dot11txpn.val>>16);
-
-	phase1((u16 *)&ttkey[0], prwskey, &prxattrib->ta[0], pnh);
-	phase2(&rc4key[0], prwskey, (unsigned short *)&ttkey[0], pnl);
-
-	/* 4 decrypt payload include icv */
-	arcfour_init(&mycontext, rc4key, 16);
-	arcfour_encrypt(&mycontext, payload, payload, length);
-
-	actual_crc = getcrc32(payload, length - 4);
-	expected_crc = get_unaligned_le32(&payload[length - 4]);
-
-	if (actual_crc != expected_crc) {
-		RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-			 "%s:icv CRC mismatch: "
-			 "actual: %08x, expected: %08x\n",
-			 __func__, actual_crc, expected_crc);
-		res = _FAIL;
-	}
-
-exit:
-	return res;
-}
-
-/* 3			===== AES related ===== */
-
-#define MAX_MSG_SIZE	2048
-/*****************************/
-/******** SBOX Table *********/
-/*****************************/
-
-static  u8 sbox_table[256] = {
-	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
-	0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
-	0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
-	0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
-	0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc,
-	0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
-	0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a,
-	0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
-	0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
-	0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
-	0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b,
-	0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
-	0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85,
-	0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
-	0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
-	0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
-	0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17,
-	0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
-	0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88,
-	0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
-	0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
-	0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
-	0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9,
-	0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
-	0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6,
-	0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
-	0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
-	0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
-	0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94,
-	0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
-	0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68,
-	0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
-};
-
-/*****************************/
-/**** Function Prototypes ****/
-/*****************************/
-
-static void construct_mic_header2(u8 *mic_header2, u8 *mpdu, int a4_exists,
-				  int qc_exists);
-
-static void xor_128(u8 *a, u8 *b, u8 *out)
-{
-	int i;
-
-	for (i = 0; i < 16; i++)
-		out[i] = a[i] ^ b[i];
-}
-
-static void xor_32(u8 *a, u8 *b, u8 *out)
-{
-	int i;
-
-	for (i = 0; i < 4; i++)
-		out[i] = a[i] ^ b[i];
-}
-
-static u8 sbox(u8 a)
-{
-	return sbox_table[(int)a];
-}
-
-static void next_key(u8 *key, int round)
-{
-	u8 rcon;
-	u8 sbox_key[4];
-	u8 rcon_table[12] = {
-		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
-		0x1b, 0x36, 0x36, 0x36
-	};
-
-	sbox_key[0] = sbox(key[13]);
-	sbox_key[1] = sbox(key[14]);
-	sbox_key[2] = sbox(key[15]);
-	sbox_key[3] = sbox(key[12]);
-
-	rcon = rcon_table[round];
-
-	xor_32(&key[0], sbox_key, &key[0]);
-	key[0] = key[0] ^ rcon;
-
-	xor_32(&key[4], &key[0], &key[4]);
-	xor_32(&key[8], &key[4], &key[8]);
-	xor_32(&key[12], &key[8], &key[12]);
-
-}
-
-static void byte_sub(u8 *in, u8 *out)
-{
-	int i;
-
-	for (i = 0; i < 16; i++)
-		out[i] = sbox(in[i]);
-}
-
-static void shift_row(u8 *in, u8 *out)
-{
-
-	out[0] =  in[0];
-	out[1] =  in[5];
-	out[2] =  in[10];
-	out[3] =  in[15];
-	out[4] =  in[4];
-	out[5] =  in[9];
-	out[6] =  in[14];
-	out[7] =  in[3];
-	out[8] =  in[8];
-	out[9] =  in[13];
-	out[10] = in[2];
-	out[11] = in[7];
-	out[12] = in[12];
-	out[13] = in[1];
-	out[14] = in[6];
-	out[15] = in[11];
-
-}
-
-static void mix_column(u8 *in, u8 *out)
-{
-	int i;
-	u8 add1b[4];
-	u8 add1bf7[4];
-	u8 rotl[4];
-	u8 swap_halfs[4];
-	u8 andf7[4];
-	u8 rotr[4];
-	u8 temp[4];
-	u8 tempb[4];
-
-	for (i = 0; i < 4; i++) {
-		if ((in[i] & 0x80) == 0x80)
-			add1b[i] = 0x1b;
-		else
-			add1b[i] = 0x00;
-	}
-
-	swap_halfs[0] = in[2]; /* Swap halfs */
-	swap_halfs[1] = in[3];
-	swap_halfs[2] = in[0];
-	swap_halfs[3] = in[1];
-
-	rotl[0] = in[3]; /* Rotate left 8 bits */
-	rotl[1] = in[0];
-	rotl[2] = in[1];
-	rotl[3] = in[2];
-
-	andf7[0] = in[0] & 0x7f;
-	andf7[1] = in[1] & 0x7f;
-	andf7[2] = in[2] & 0x7f;
-	andf7[3] = in[3] & 0x7f;
-
-	for (i = 3; i > 0; i--) { /* logical shift left 1 bit */
-		andf7[i] = andf7[i] << 1;
-		if ((andf7[i - 1] & 0x80) == 0x80)
-			andf7[i] = (andf7[i] | 0x01);
-	}
-	andf7[0] = andf7[0] << 1;
-	andf7[0] = andf7[0] & 0xfe;
-
-	xor_32(add1b, andf7, add1bf7);
-
-	xor_32(in, add1bf7, rotr);
-
-	temp[0] = rotr[0]; /* Rotate right 8 bits */
-	rotr[0] = rotr[1];
-	rotr[1] = rotr[2];
-	rotr[2] = rotr[3];
-	rotr[3] = temp[0];
-
-	xor_32(add1bf7, rotr, temp);
-	xor_32(swap_halfs, rotl, tempb);
-	xor_32(temp, tempb, out);
-
-}
-
-static void aes128k128d(u8 *key, u8 *data, u8 *ciphertext)
-{
-	int round;
-	int i;
-	u8 intermediatea[16];
-	u8 intermediateb[16];
-	u8 round_key[16];
-
-	for (i = 0; i < 16; i++)
-		round_key[i] = key[i];
-
-	for (round = 0; round < 11; round++) {
-		if (round == 0) {
-			xor_128(round_key, data, ciphertext);
-			next_key(round_key, round);
-		} else if (round == 10) {
-			byte_sub(ciphertext, intermediatea);
-			shift_row(intermediatea, intermediateb);
-			xor_128(intermediateb, round_key, ciphertext);
-		} else { /* 1 - 9 */
-			byte_sub(ciphertext, intermediatea);
-			shift_row(intermediatea, intermediateb);
-			mix_column(&intermediateb[0], &intermediatea[0]);
-			mix_column(&intermediateb[4], &intermediatea[4]);
-			mix_column(&intermediateb[8], &intermediatea[8]);
-			mix_column(&intermediateb[12], &intermediatea[12]);
-			xor_128(intermediatea, round_key, ciphertext);
-			next_key(round_key, round);
-		}
-	}
-
-}
-
-/************************************************/
-/* construct_mic_iv()                           */
-/* Builds the MIC IV from header fields and PN  */
-/************************************************/
-static void construct_mic_iv(u8 *mic_iv, int qc_exists, int a4_exists, u8 *mpdu,
-			     uint payload_length, u8 *pn_vector)
-{
-	int i;
-
-	mic_iv[0] = 0x59;
-	if (qc_exists && a4_exists)
-		mic_iv[1] = mpdu[30] & 0x0f; /* QoS_TC        */
-	if (qc_exists && !a4_exists)
-		mic_iv[1] = mpdu[24] & 0x0f; /* mute bits 7-4 */
-	if (!qc_exists)
-		mic_iv[1] = 0x00;
-	for (i = 2; i < 8; i++)
-		mic_iv[i] = mpdu[i + 8]; /* mic_iv[2:7] = A2[0:5] = mpdu[10:15] */
-	for (i = 8; i < 14; i++)
-		mic_iv[i] = pn_vector[13 - i]; /* mic_iv[8:13] = PN[5:0] */
-	mic_iv[14] = (unsigned char)(payload_length / 256);
-	mic_iv[15] = (unsigned char)(payload_length % 256);
-}
-
-/************************************************/
-/* construct_mic_header1()                      */
-/* Builds the first MIC header block from       */
-/* header fields.                               */
-/************************************************/
-static void construct_mic_header1(u8 *mic_header1, int header_length, u8 *mpdu)
-{
-	mic_header1[0] = (u8)((header_length - 2) / 256);
-	mic_header1[1] = (u8)((header_length - 2) % 256);
-	mic_header1[2] = mpdu[0] & 0xcf; /* Mute CF poll & CF ack bits */
-	mic_header1[3] = mpdu[1] & 0xc7; /* Mute retry, more data and pwr mgt bits */
-	mic_header1[4] = mpdu[4]; /* A1 */
-	mic_header1[5] = mpdu[5];
-	mic_header1[6] = mpdu[6];
-	mic_header1[7] = mpdu[7];
-	mic_header1[8] = mpdu[8];
-	mic_header1[9] = mpdu[9];
-	mic_header1[10] = mpdu[10]; /* A2 */
-	mic_header1[11] = mpdu[11];
-	mic_header1[12] = mpdu[12];
-	mic_header1[13] = mpdu[13];
-	mic_header1[14] = mpdu[14];
-	mic_header1[15] = mpdu[15];
-
-}
-
-/************************************************/
-/* construct_mic_header2()                      */
-/* Builds the last MIC header block from        */
-/* header fields.                               */
-/************************************************/
-static void construct_mic_header2(u8 *mic_header2, u8 *mpdu, int a4_exists,
-				  int qc_exists)
-{
-	int i;
-
-	for (i = 0; i < 16; i++)
-		mic_header2[i] = 0x00;
-
-	mic_header2[0] = mpdu[16]; /* A3 */
-	mic_header2[1] = mpdu[17];
-	mic_header2[2] = mpdu[18];
-	mic_header2[3] = mpdu[19];
-	mic_header2[4] = mpdu[20];
-	mic_header2[5] = mpdu[21];
-
-	mic_header2[6] = 0x00;
-	mic_header2[7] = 0x00; /* mpdu[23]; */
-
-	if (!qc_exists && a4_exists) {
-		for (i = 0; i < 6; i++)
-			mic_header2[8+i] = mpdu[24+i]; /* A4 */
-	}
-
-	if (qc_exists && !a4_exists) {
-		mic_header2[8] = mpdu[24] & 0x0f; /* mute bits 15 - 4 */
-		mic_header2[9] = mpdu[25] & 0x00;
-	}
-
-	if (qc_exists && a4_exists) {
-		for (i = 0; i < 6; i++)
-			mic_header2[8+i] = mpdu[24+i]; /* A4 */
-
-		mic_header2[14] = mpdu[30] & 0x0f;
-		mic_header2[15] = mpdu[31] & 0x00;
-	}
-
-}
-
-/************************************************/
-/* construct_mic_header2()                      */
-/* Builds the last MIC header block from        */
-/* header fields.                               */
-/************************************************/
-static void construct_ctr_preload(u8 *ctr_preload, int a4_exists, int qc_exists,
-				  u8 *mpdu, u8 *pn_vector, int c)
-{
-	int i = 0;
-
-	for (i = 0; i < 16; i++)
-		ctr_preload[i] = 0x00;
-
-	i = 0;
-
-	ctr_preload[0] = 0x01; /* flag */
-	if (qc_exists && a4_exists)
-		ctr_preload[1] = mpdu[30] & 0x0f; /* QoC_Control */
-	if (qc_exists && !a4_exists)
-		ctr_preload[1] = mpdu[24] & 0x0f;
-
-	for (i = 2; i < 8; i++)
-		ctr_preload[i] = mpdu[i + 8]; /* ctr_preload[2:7] = A2[0:5] = mpdu[10:15] */
-	for (i = 8; i < 14; i++)
-		ctr_preload[i] = pn_vector[13 - i]; /* ctr_preload[8:13] = PN[5:0] */
-	ctr_preload[14] =  (unsigned char) (c / 256); /* Ctr */
-	ctr_preload[15] =  (unsigned char) (c % 256);
-
-}
-
-/************************************/
-/* bitwise_xor()                    */
-/* A 128 bit, bitwise exclusive or  */
-/************************************/
-static void bitwise_xor(u8 *ina, u8 *inb, u8 *out)
-{
-	int i;
-
-	for (i = 0; i < 16; i++)
-		out[i] = ina[i] ^ inb[i];
-}
-
-static int aes_cipher(u8 *key, uint hdrlen, u8 *pframe, uint plen)
-{
-	uint qc_exists, a4_exists, i, j, payload_remainder,
-	     num_blocks, payload_index;
-	u8 pn_vector[6];
-	u8 mic_iv[16];
-	u8 mic_header1[16];
-	u8 mic_header2[16];
-	u8 ctr_preload[16];
-	/* Intermediate Buffers */
-	u8 chain_buffer[16];
-	u8 aes_out[16];
-	u8 padded_buffer[16];
-	u8 mic[8];
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)pframe;
-	u16 frsubtype = le16_to_cpu(hdr->frame_control) & IEEE80211_FCTL_STYPE;
-
-	memset((void *)mic_iv, 0, 16);
-	memset((void *)mic_header1, 0, 16);
-	memset((void *)mic_header2, 0, 16);
-	memset((void *)ctr_preload, 0, 16);
-	memset((void *)chain_buffer, 0, 16);
-	memset((void *)aes_out, 0, 16);
-	memset((void *)padded_buffer, 0, 16);
-
-	if ((hdrlen == sizeof(struct ieee80211_hdr_3addr) ||
-	    (hdrlen == sizeof(struct ieee80211_qos_hdr))))
-		a4_exists = 0;
-	else
-		a4_exists = 1;
-
-	if (ieee80211_is_data(hdr->frame_control)) {
-		if ((frsubtype == IEEE80211_STYPE_DATA_CFACK) ||
-		    (frsubtype == IEEE80211_STYPE_DATA_CFPOLL) ||
-		    (frsubtype == IEEE80211_STYPE_DATA_CFACKPOLL)) {
-			qc_exists = 1;
-			if (hdrlen != sizeof(struct ieee80211_qos_hdr))
-				hdrlen += 2;
-		} else if ((frsubtype == IEEE80211_STYPE_QOS_DATA) ||
-			   (frsubtype == IEEE80211_STYPE_QOS_DATA_CFACK) ||
-			   (frsubtype == IEEE80211_STYPE_QOS_DATA_CFPOLL) ||
-			   (frsubtype == IEEE80211_STYPE_QOS_DATA_CFACKPOLL)) {
-			if (hdrlen != sizeof(struct ieee80211_qos_hdr))
-				hdrlen += 2;
-			qc_exists = 1;
-		} else {
-			qc_exists = 0;
-		}
-	} else {
-		qc_exists = 0;
-	}
-	pn_vector[0] = pframe[hdrlen];
-	pn_vector[1] = pframe[hdrlen + 1];
-	pn_vector[2] = pframe[hdrlen + 4];
-	pn_vector[3] = pframe[hdrlen + 5];
-	pn_vector[4] = pframe[hdrlen + 6];
-	pn_vector[5] = pframe[hdrlen + 7];
-
-	construct_mic_iv(mic_iv, qc_exists, a4_exists, pframe, plen, pn_vector);
-
-	construct_mic_header1(mic_header1, hdrlen, pframe);
-	construct_mic_header2(mic_header2, pframe, a4_exists, qc_exists);
-
-	payload_remainder = plen % 16;
-	num_blocks = plen / 16;
-
-	/* Find start of payload */
-	payload_index = hdrlen + 8;
-
-	/* Calculate MIC */
-	aes128k128d(key, mic_iv, aes_out);
-	bitwise_xor(aes_out, mic_header1, chain_buffer);
-	aes128k128d(key, chain_buffer, aes_out);
-	bitwise_xor(aes_out, mic_header2, chain_buffer);
-	aes128k128d(key, chain_buffer, aes_out);
-
-	for (i = 0; i < num_blocks; i++) {
-		bitwise_xor(aes_out, &pframe[payload_index], chain_buffer);
-
-		payload_index += 16;
-		aes128k128d(key, chain_buffer, aes_out);
-	}
-
-	/* Add on the final payload block if it needs padding */
-	if (payload_remainder > 0) {
-		for (j = 0; j < 16; j++)
-			padded_buffer[j] = 0x00;
-		for (j = 0; j < payload_remainder; j++)
-			padded_buffer[j] = pframe[payload_index++];
-		bitwise_xor(aes_out, padded_buffer, chain_buffer);
-		aes128k128d(key, chain_buffer, aes_out);
-	}
-
-	for (j = 0; j < 8; j++)
-		mic[j] = aes_out[j];
-
-	/* Insert MIC into payload */
-	for (j = 0; j < 8; j++)
-		pframe[payload_index + j] = mic[j];
-
-	payload_index = hdrlen + 8;
-	for (i = 0; i < num_blocks; i++) {
-		construct_ctr_preload(ctr_preload, a4_exists, qc_exists,
-				      pframe, pn_vector, i + 1);
-		aes128k128d(key, ctr_preload, aes_out);
-		bitwise_xor(aes_out, &pframe[payload_index], chain_buffer);
-		for (j = 0; j < 16; j++)
-			pframe[payload_index++] = chain_buffer[j];
-	}
-
-	if (payload_remainder > 0) {
-		/* If there is a short final block, then pad it,
-		 * encrypt it and copy the unpadded part back
-		 */
-		construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe,
-				      pn_vector, num_blocks + 1);
-
-		for (j = 0; j < 16; j++)
-			padded_buffer[j] = 0x00;
-		for (j = 0; j < payload_remainder; j++)
-			padded_buffer[j] = pframe[payload_index + j];
-		aes128k128d(key, ctr_preload, aes_out);
-		bitwise_xor(aes_out, padded_buffer, chain_buffer);
-		for (j = 0; j < payload_remainder; j++)
-			pframe[payload_index++] = chain_buffer[j];
-	}
-
-	/* Encrypt the MIC */
-	construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe,
-			      pn_vector, 0);
-
-	for (j = 0; j < 16; j++)
-		padded_buffer[j] = 0x00;
-	for (j = 0; j < 8; j++)
-		padded_buffer[j] = pframe[j + hdrlen + 8 + plen];
-
-	aes128k128d(key, ctr_preload, aes_out);
-	bitwise_xor(aes_out, padded_buffer, chain_buffer);
-	for (j = 0; j < 8; j++)
-		pframe[payload_index++] = chain_buffer[j];
-
-	return _SUCCESS;
-}
-
-int rtw_aes_encrypt23a(struct rtw_adapter *padapter,
-		       struct xmit_frame *pxmitframe)
-{	/* exclude ICV */
-	/* Intermediate Buffers */
-	int curfragnum, length;
-	u8 *pframe, *prwskey;
-	u8 hw_hdr_offset = 0;
-	struct sta_info *stainfo;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	int res = _SUCCESS;
-
-	if (!pxmitframe->buf_addr)
-		return _FAIL;
-
-	hw_hdr_offset = TXDESC_OFFSET;
-
-	pframe = pxmitframe->buf_addr + hw_hdr_offset;
-
-	/* 4 start to encrypt each fragment */
-	if (pattrib->encrypt != WLAN_CIPHER_SUITE_CCMP)
-		return _FAIL;
-
-	if (pattrib->psta) {
-		stainfo = pattrib->psta;
-	} else {
-		DBG_8723A("%s, call rtw_get_stainfo23a()\n", __func__);
-		stainfo = rtw_get_stainfo23a(&padapter->stapriv, &pattrib->ra[0]);
-	}
-
-	if (!stainfo) {
-		RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-			 "%s: stainfo == NULL!!!\n", __func__);
-		DBG_8723A("%s, psta == NUL\n", __func__);
-		res = _FAIL;
-		goto out;
-	}
-	if (!(stainfo->state & _FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n",
-			  __func__, stainfo->state);
-		return _FAIL;
-	}
-	RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-		 "%s: stainfo!= NULL!!!\n", __func__);
-
-	if (is_multicast_ether_addr(pattrib->ra))
-		prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey;
-	else
-		prwskey = &stainfo->dot118021x_UncstKey.skey[0];
-
-	for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
-		/* 4 the last fragment */
-		if ((curfragnum + 1) == pattrib->nr_frags) {
-			length = pattrib->last_txcmdsz -
-				pattrib->hdrlen-pattrib->iv_len -
-				pattrib->icv_len;
-
-			aes_cipher(prwskey, pattrib->hdrlen, pframe, length);
-		} else {
-			length = pxmitpriv->frag_len-pattrib->hdrlen -
-				pattrib->iv_len - pattrib->icv_len;
-
-			aes_cipher(prwskey, pattrib->hdrlen, pframe, length);
-			pframe += pxmitpriv->frag_len;
-			pframe = PTR_ALIGN(pframe, 4);
-		}
-	}
-out:
-	return res;
-}
-
-static int aes_decipher(u8 *key, uint hdrlen, u8 *pframe, uint plen)
-{
-	static u8 message[MAX_MSG_SIZE];
-	uint qc_exists, a4_exists, i, j, payload_remainder,
-	     num_blocks, payload_index;
-	int res = _SUCCESS;
-	u8 pn_vector[6];
-	u8 mic_iv[16];
-	u8 mic_header1[16];
-	u8 mic_header2[16];
-	u8 ctr_preload[16];
-	/* Intermediate Buffers */
-	u8 chain_buffer[16];
-	u8 aes_out[16];
-	u8 padded_buffer[16];
-	u8 mic[8];
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)pframe;
-	u16 frsubtype = le16_to_cpu(hdr->frame_control) & IEEE80211_FCTL_STYPE;
-
-	memset((void *)mic_iv, 0, 16);
-	memset((void *)mic_header1, 0, 16);
-	memset((void *)mic_header2, 0, 16);
-	memset((void *)ctr_preload, 0, 16);
-	memset((void *)chain_buffer, 0, 16);
-	memset((void *)aes_out, 0, 16);
-	memset((void *)padded_buffer, 0, 16);
-
-	/* start to decrypt the payload */
-
-	num_blocks = (plen - 8) / 16; /* plen including llc, payload_length and mic) */
-
-	payload_remainder = (plen - 8) % 16;
-
-	pn_vector[0]  = pframe[hdrlen];
-	pn_vector[1]  = pframe[hdrlen + 1];
-	pn_vector[2]  = pframe[hdrlen + 4];
-	pn_vector[3]  = pframe[hdrlen + 5];
-	pn_vector[4]  = pframe[hdrlen + 6];
-	pn_vector[5]  = pframe[hdrlen + 7];
-
-	if ((hdrlen == sizeof(struct ieee80211_hdr_3addr) ||
-	    (hdrlen == sizeof(struct ieee80211_qos_hdr))))
-		a4_exists = 0;
-	else
-		a4_exists = 1;
-
-	if (ieee80211_is_data(hdr->frame_control)) {
-		if ((frsubtype == IEEE80211_STYPE_DATA_CFACK) ||
-		    (frsubtype == IEEE80211_STYPE_DATA_CFPOLL) ||
-		    (frsubtype == IEEE80211_STYPE_DATA_CFACKPOLL)) {
-			qc_exists = 1;
-			if (hdrlen != sizeof(struct ieee80211_hdr_3addr))
-				hdrlen += 2;
-		} else if ((frsubtype == IEEE80211_STYPE_QOS_DATA) ||
-			   (frsubtype == IEEE80211_STYPE_QOS_DATA_CFACK) ||
-			   (frsubtype == IEEE80211_STYPE_QOS_DATA_CFPOLL) ||
-			   (frsubtype == IEEE80211_STYPE_QOS_DATA_CFACKPOLL)) {
-			if (hdrlen != sizeof(struct ieee80211_hdr_3addr))
-				hdrlen += 2;
-			qc_exists = 1;
-		} else {
-			qc_exists = 0;
-		}
-	} else {
-		qc_exists = 0;
-	}
-
-	/* now, decrypt pframe with hdrlen offset and plen long */
-
-	payload_index = hdrlen + 8; /*  8 is for extiv */
-
-	for (i = 0; i < num_blocks; i++) {
-		construct_ctr_preload(ctr_preload, a4_exists, qc_exists,
-				      pframe, pn_vector, i + 1);
-
-		aes128k128d(key, ctr_preload, aes_out);
-		bitwise_xor(aes_out, &pframe[payload_index], chain_buffer);
-
-		for (j = 0; j < 16; j++)
-			pframe[payload_index++] = chain_buffer[j];
-	}
-
-	if (payload_remainder > 0) {
-		/* If there is a short final block, then pad it,
-		 * encrypt it and copy the unpadded part back
-		 */
-		construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe,
-				      pn_vector, num_blocks + 1);
-
-		for (j = 0; j < 16; j++)
-			padded_buffer[j] = 0x00;
-		for (j = 0; j < payload_remainder; j++)
-			padded_buffer[j] = pframe[payload_index + j];
-		aes128k128d(key, ctr_preload, aes_out);
-		bitwise_xor(aes_out, padded_buffer, chain_buffer);
-		for (j = 0; j < payload_remainder; j++)
-			pframe[payload_index++] = chain_buffer[j];
-	}
-
-	/* start to calculate the mic */
-	if ((hdrlen + plen + 8) <= MAX_MSG_SIZE)
-		memcpy(message, pframe, (hdrlen + plen + 8)); /* 8 is for ext iv len */
-
-	pn_vector[0] = pframe[hdrlen];
-	pn_vector[1] = pframe[hdrlen + 1];
-	pn_vector[2] = pframe[hdrlen + 4];
-	pn_vector[3] = pframe[hdrlen + 5];
-	pn_vector[4] = pframe[hdrlen + 6];
-	pn_vector[5] = pframe[hdrlen + 7];
-
-	construct_mic_iv(mic_iv, qc_exists, a4_exists, message,
-			 plen - 8, pn_vector);
-
-	construct_mic_header1(mic_header1, hdrlen, message);
-	construct_mic_header2(mic_header2, message, a4_exists, qc_exists);
-
-	payload_remainder = (plen - 8) % 16;
-	num_blocks = (plen - 8) / 16;
-
-	/* Find start of payload */
-	payload_index = hdrlen + 8;
-
-	/* Calculate MIC */
-	aes128k128d(key, mic_iv, aes_out);
-	bitwise_xor(aes_out, mic_header1, chain_buffer);
-	aes128k128d(key, chain_buffer, aes_out);
-	bitwise_xor(aes_out, mic_header2, chain_buffer);
-	aes128k128d(key, chain_buffer, aes_out);
-
-	for (i = 0; i < num_blocks; i++) {
-		bitwise_xor(aes_out, &message[payload_index], chain_buffer);
-
-		payload_index += 16;
-		aes128k128d(key, chain_buffer, aes_out);
-	}
-
-	/* Add on the final payload block if it needs padding */
-	if (payload_remainder > 0) {
-		for (j = 0; j < 16; j++)
-			padded_buffer[j] = 0x00;
-		for (j = 0; j < payload_remainder; j++)
-			padded_buffer[j] = message[payload_index++];
-		bitwise_xor(aes_out, padded_buffer, chain_buffer);
-		aes128k128d(key, chain_buffer, aes_out);
-	}
-
-	for (j = 0 ; j < 8; j++)
-		mic[j] = aes_out[j];
-
-	/* Insert MIC into payload */
-	for (j = 0; j < 8; j++)
-		message[payload_index + j] = mic[j];
-
-	payload_index = hdrlen + 8;
-	for (i = 0; i < num_blocks; i++) {
-		construct_ctr_preload(ctr_preload, a4_exists, qc_exists,
-				      message, pn_vector, i + 1);
-		aes128k128d(key, ctr_preload, aes_out);
-		bitwise_xor(aes_out, &message[payload_index], chain_buffer);
-		for (j = 0; j < 16; j++)
-			message[payload_index++] = chain_buffer[j];
-	}
-
-	if (payload_remainder > 0) {
-		/* If there is a short final block, then pad it,
-		 * encrypt it and copy the unpadded part back
-		 */
-		construct_ctr_preload(ctr_preload, a4_exists, qc_exists,
-				      message, pn_vector, num_blocks + 1);
-
-		for (j = 0; j < 16; j++)
-			padded_buffer[j] = 0x00;
-		for (j = 0; j < payload_remainder; j++)
-			padded_buffer[j] = message[payload_index + j];
-		aes128k128d(key, ctr_preload, aes_out);
-		bitwise_xor(aes_out, padded_buffer, chain_buffer);
-		for (j = 0; j < payload_remainder; j++)
-			message[payload_index++] = chain_buffer[j];
-	}
-
-	/* Encrypt the MIC */
-	construct_ctr_preload(ctr_preload, a4_exists, qc_exists, message,
-			      pn_vector, 0);
-
-	for (j = 0; j < 16; j++)
-		padded_buffer[j] = 0x00;
-	for (j = 0; j < 8; j++)
-		padded_buffer[j] = message[j + hdrlen + 8 + plen - 8];
-
-	aes128k128d(key, ctr_preload, aes_out);
-	bitwise_xor(aes_out, padded_buffer, chain_buffer);
-	for (j = 0; j < 8; j++)
-		message[payload_index++] = chain_buffer[j];
-
-	/* compare the mic */
-	for (i = 0; i < 8; i++) {
-		if (pframe[hdrlen + 8 + plen - 8 + i] != message[hdrlen + 8 + plen - 8 + i]) {
-			RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-				 "%s:mic check error mic[%d]: pframe(%x) != message(%x)\n",
-				 __func__, i,
-				 pframe[hdrlen + 8 + plen - 8 + i],
-				 message[hdrlen + 8 + plen - 8 + i]);
-			DBG_8723A("%s:mic check error mic[%d]: pframe(%x) != message(%x)\n",
-				  __func__, i,
-				  pframe[hdrlen + 8 + plen - 8 + i],
-				  message[hdrlen + 8 + plen - 8 + i]);
-			res = _FAIL;
-		}
-	}
-	return res;
-}
-
-int rtw_aes_decrypt23a(struct rtw_adapter *padapter,
-		       struct recv_frame *precvframe)
-{	/*  exclude ICV */
-	struct sta_info *stainfo;
-	struct rx_pkt_attrib *prxattrib = &precvframe->attrib;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct sk_buff *skb = precvframe->pkt;
-	int length;
-	u8 *pframe, *prwskey;
-	int res = _SUCCESS;
-
-	pframe = skb->data;
-	/* 4 start to encrypt each fragment */
-	if (prxattrib->encrypt != WLAN_CIPHER_SUITE_CCMP)
-		return _FAIL;
-
-	stainfo = rtw_get_stainfo23a(&padapter->stapriv, &prxattrib->ta[0]);
-	if (!stainfo) {
-		RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-			 "%s: stainfo == NULL!!!\n", __func__);
-		res = _FAIL;
-		goto exit;
-	}
-
-	RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-		 "%s: stainfo!= NULL!!!\n", __func__);
-
-	if (is_multicast_ether_addr(prxattrib->ra)) {
-		/* in concurrent we should use sw decrypt in
-		 * group key, so we remove this message
-		 */
-		if (!psecuritypriv->binstallGrpkey) {
-			res = _FAIL;
-			DBG_8723A("%s:rx bc/mc packets, but didn't install "
-				  "group key!!!!!!!!!!\n", __func__);
-			goto exit;
-		}
-		prwskey = psecuritypriv->dot118021XGrpKey[prxattrib->key_index].skey;
-		if (psecuritypriv->dot118021XGrpKeyid != prxattrib->key_index) {
-			DBG_8723A("not match packet_index =%d, install_index ="
-				  "%d\n", prxattrib->key_index,
-				  psecuritypriv->dot118021XGrpKeyid);
-			res = _FAIL;
-			goto exit;
-		}
-	} else {
-		prwskey = &stainfo->dot118021x_UncstKey.skey[0];
-	}
-
-	length = skb->len - prxattrib->hdrlen -	prxattrib->iv_len;
-
-	res = aes_decipher(prwskey, prxattrib->hdrlen, pframe, length);
-exit:
-	return res;
-}
-
-void rtw_use_tkipkey_handler23a(void *function_context)
-{
-	struct rtw_adapter *padapter = function_context;
-
-	RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-		 "^^^%s ^^^\n", __func__);
-	padapter->securitypriv.busetkipkey = 1;
-	RT_TRACE(_module_rtl871x_security_c_, _drv_err_,
-		 "^^^%s padapter->securitypriv.busetkipkey =%d^^^\n",
-		 __func__, padapter->securitypriv.busetkipkey);
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_sreset.c b/drivers/staging/rtl8723au/core/rtw_sreset.c
deleted file mode 100644
index 29a29d92a6ac..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_sreset.c
+++ /dev/null
@@ -1,214 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#include <rtw_sreset.h>
-#include <usb_ops_linux.h>
-
-void rtw_sreset_init(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct sreset_priv *psrtpriv = &pHalData->srestpriv;
-
-	mutex_init(&psrtpriv->silentreset_mutex);
-	psrtpriv->silent_reset_inprogress = false;
-	psrtpriv->last_tx_time = 0;
-	psrtpriv->last_tx_complete_time = 0;
-}
-
-void rtw_sreset_reset_value(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct sreset_priv *psrtpriv = &pHalData->srestpriv;
-
-	psrtpriv->silent_reset_inprogress = false;
-	psrtpriv->last_tx_time = 0;
-	psrtpriv->last_tx_complete_time = 0;
-}
-
-bool rtw_sreset_inprogress(struct rtw_adapter *padapter)
-{
-	struct rtw_adapter *primary_adapter = GET_PRIMARY_ADAPTER(padapter);
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(primary_adapter);
-
-	return pHalData->srestpriv.silent_reset_inprogress;
-}
-
-static void sreset_restore_security_station(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *mlmepriv = &padapter->mlmepriv;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct sta_info *psta;
-	struct mlme_ext_info *pmlmeinfo = &padapter->mlmeextpriv.mlmext_info;
-	u8 val8;
-
-	if (pmlmeinfo->auth_algo == dot11AuthAlgrthm_8021X)
-		val8 = 0xcc;
-	else
-		val8 = 0xcf;
-
-	rtl8723a_set_sec_cfg(padapter, val8);
-
-	if (padapter->securitypriv.dot11PrivacyAlgrthm ==
-	    WLAN_CIPHER_SUITE_TKIP ||
-	    padapter->securitypriv.dot11PrivacyAlgrthm ==
-	    WLAN_CIPHER_SUITE_CCMP) {
-		psta = rtw_get_stainfo23a(pstapriv, get_bssid(mlmepriv));
-		if (psta == NULL) {
-			/* DEBUG_ERR(("Set wpa_set_encryption: Obtain Sta_info fail\n")); */
-		} else {
-			/* pairwise key */
-			rtw_setstakey_cmd23a(padapter, (unsigned char *)psta, true);
-			/* group key */
-			rtw_set_key23a(padapter,&padapter->securitypriv, padapter->securitypriv.dot118021XGrpKeyid, 0);
-		}
-	}
-}
-
-static void sreset_restore_network_station(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *mlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	u8 threshold;
-
-	rtw_setopmode_cmd23a(padapter, NL80211_IFTYPE_STATION);
-
-	/*  TH = 1 => means that invalidate usb rx aggregation */
-	/*  TH = 0 => means that validate usb rx aggregation, use init value. */
-	if (mlmepriv->htpriv.ht_option) {
-		if (padapter->registrypriv.wifi_spec == 1)
-			threshold = 1;
-		else
-			threshold = 0;
-	} else
-		threshold = 1;
-
-	rtl8723a_set_rxdma_agg_pg_th(padapter, threshold);
-
-	set_channel_bwmode23a(padapter, pmlmeext->cur_channel,
-			      pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode);
-
-	hw_var_set_bssid(padapter, pmlmeinfo->network.MacAddress);
-	hw_var_set_mlme_join(padapter, 0);
-
-	rtl8723a_set_media_status(padapter, pmlmeinfo->state & 0x3);
-
-	mlmeext_joinbss_event_callback23a(padapter, 1);
-	/* restore Sequence No. */
-	rtl8723au_write8(padapter, REG_NQOS_SEQ, padapter->xmitpriv.nqos_ssn);
-
-	sreset_restore_security_station(padapter);
-}
-
-static void sreset_restore_network_status(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *mlmepriv = &padapter->mlmepriv;
-
-	if (check_fwstate(mlmepriv, WIFI_STATION_STATE)) {
-		DBG_8723A("%s(%s): fwstate:0x%08x - WIFI_STATION_STATE\n",
-			  __func__, padapter->pnetdev->name,
-			  get_fwstate(mlmepriv));
-		sreset_restore_network_station(padapter);
-#ifdef CONFIG_8723AU_AP_MODE
-	} else if (check_fwstate(mlmepriv, WIFI_AP_STATE)) {
-		DBG_8723A("%s(%s): fwstate:0x%08x - WIFI_AP_STATE\n",
-			  __func__, padapter->pnetdev->name,
-			  get_fwstate(mlmepriv));
-		rtw_ap_restore_network(padapter);
-#endif
-	} else if (check_fwstate(mlmepriv, WIFI_ADHOC_STATE)) {
-		DBG_8723A("%s(%s): fwstate:0x%08x - WIFI_ADHOC_STATE\n",
-			  __func__, padapter->pnetdev->name,
-			  get_fwstate(mlmepriv));
-	} else {
-		DBG_8723A("%s(%s): fwstate:0x%08x - ???\n", __func__,
-			  padapter->pnetdev->name, get_fwstate(mlmepriv));
-	}
-}
-
-static void sreset_stop_adapter(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
-
-	if (padapter == NULL)
-		return;
-
-	DBG_8723A("%s(%s)\n", __func__, padapter->pnetdev->name);
-
-	if (!rtw_netif_queue_stopped(padapter->pnetdev))
-		netif_tx_stop_all_queues(padapter->pnetdev);
-
-	rtw_cancel_all_timer23a(padapter);
-
-	/* TODO: OS and HCI independent */
-	tasklet_kill(&pxmitpriv->xmit_tasklet);
-
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
-		rtw_scan_abort23a(padapter);
-
-	if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING))
-		rtw23a_join_to_handler((unsigned long)padapter);
-}
-
-static void sreset_start_adapter(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
-
-	if (padapter == NULL)
-		return;
-
-	DBG_8723A("%s(%s)\n", __func__, padapter->pnetdev->name);
-
-	if (check_fwstate(pmlmepriv, _FW_LINKED))
-		sreset_restore_network_status(padapter);
-
-	/* TODO: OS and HCI independent */
-	tasklet_hi_schedule(&pxmitpriv->xmit_tasklet);
-
-	mod_timer(&padapter->mlmepriv.dynamic_chk_timer,
-		  jiffies + msecs_to_jiffies(2000));
-
-	if (rtw_netif_queue_stopped(padapter->pnetdev))
-		netif_tx_wake_all_queues(padapter->pnetdev);
-}
-
-void rtw_sreset_reset(struct rtw_adapter *active_adapter)
-{
-	struct rtw_adapter *padapter = GET_PRIMARY_ADAPTER(active_adapter);
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct sreset_priv *psrtpriv = &pHalData->srestpriv;
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-	unsigned long start = jiffies;
-
-	DBG_8723A("%s\n", __func__);
-
-	mutex_lock(&psrtpriv->silentreset_mutex);
-	psrtpriv->silent_reset_inprogress = true;
-	pwrpriv->change_rfpwrstate = rf_off;
-
-	sreset_stop_adapter(padapter);
-
-	ips_enter23a(padapter);
-	ips_leave23a(padapter);
-
-	sreset_start_adapter(padapter);
-	psrtpriv->silent_reset_inprogress = false;
-	mutex_unlock(&psrtpriv->silentreset_mutex);
-
-	DBG_8723A("%s done in %d ms\n", __func__,
-		  jiffies_to_msecs(jiffies - start));
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_sta_mgt.c b/drivers/staging/rtl8723au/core/rtw_sta_mgt.c
deleted file mode 100644
index a9b778c45d44..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_sta_mgt.c
+++ /dev/null
@@ -1,439 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_STA_MGT_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <recv_osdep.h>
-#include <xmit_osdep.h>
-#include <mlme_osdep.h>
-#include <sta_info.h>
-#include <rtl8723a_hal.h>
-
-static const u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-
-static void _rtw_init_stainfo(struct sta_info *psta)
-{
-	memset((u8 *)psta, 0, sizeof(struct sta_info));
-	spin_lock_init(&psta->lock);
-	INIT_LIST_HEAD(&psta->list);
-	INIT_LIST_HEAD(&psta->hash_list);
-	_rtw_init_queue23a(&psta->sleep_q);
-	psta->sleepq_len = 0;
-	_rtw_init_sta_xmit_priv23a(&psta->sta_xmitpriv);
-	_rtw_init_sta_recv_priv23a(&psta->sta_recvpriv);
-#ifdef CONFIG_8723AU_AP_MODE
-	INIT_LIST_HEAD(&psta->asoc_list);
-	INIT_LIST_HEAD(&psta->auth_list);
-	psta->expire_to = 0;
-	psta->flags = 0;
-	psta->capability = 0;
-	psta->bpairwise_key_installed = false;
-	psta->nonerp_set = 0;
-	psta->no_short_slot_time_set = 0;
-	psta->no_short_preamble_set = 0;
-	psta->no_ht_gf_set = 0;
-	psta->no_ht_set = 0;
-	psta->ht_20mhz_set = 0;
-	psta->keep_alive_trycnt = 0;
-#endif	/*  CONFIG_8723AU_AP_MODE */
-}
-
-int _rtw_init_sta_priv23a(struct sta_priv *pstapriv)
-{
-	int i;
-
-	spin_lock_init(&pstapriv->sta_hash_lock);
-	pstapriv->asoc_sta_count = 0;
-	for (i = 0; i < NUM_STA; i++)
-		INIT_LIST_HEAD(&pstapriv->sta_hash[i]);
-
-#ifdef CONFIG_8723AU_AP_MODE
-	pstapriv->sta_dz_bitmap = 0;
-	pstapriv->tim_bitmap = 0;
-	INIT_LIST_HEAD(&pstapriv->asoc_list);
-	INIT_LIST_HEAD(&pstapriv->auth_list);
-	spin_lock_init(&pstapriv->asoc_list_lock);
-	spin_lock_init(&pstapriv->auth_list_lock);
-	pstapriv->asoc_list_cnt = 0;
-	pstapriv->auth_list_cnt = 0;
-	pstapriv->auth_to = 3; /*  3*2 = 6 sec */
-	pstapriv->assoc_to = 3;
-	/* pstapriv->expire_to = 900;  900*2 = 1800 sec = 30 min,
-	    expire after no any traffic. */
-	/* pstapriv->expire_to = 30;  30*2 = 60 sec = 1 min,
-	    expire after no any traffic. */
-	pstapriv->expire_to = 3; /*  3*2 = 6 sec */
-	pstapriv->max_num_sta = NUM_STA;
-#endif
-	return _SUCCESS;
-}
-
-int _rtw_free_sta_priv23a(struct sta_priv *pstapriv)
-{
-	struct list_head *phead;
-	struct sta_info *psta, *ptmp;
-	struct recv_reorder_ctrl *preorder_ctrl;
-	int index;
-
-	if (pstapriv) {
-		/*	delete all reordering_ctrl_timer		*/
-		spin_lock_bh(&pstapriv->sta_hash_lock);
-		for (index = 0; index < NUM_STA; index++) {
-			phead = &pstapriv->sta_hash[index];
-			list_for_each_entry_safe(psta, ptmp, phead, hash_list) {
-				int i;
-
-				for (i = 0; i < 16 ; i++) {
-					preorder_ctrl = &psta->recvreorder_ctrl[i];
-					del_timer_sync(&preorder_ctrl->reordering_ctrl_timer);
-				}
-			}
-		}
-		spin_unlock_bh(&pstapriv->sta_hash_lock);
-		/*===============================*/
-	}
-	return _SUCCESS;
-}
-
-struct sta_info *
-rtw_alloc_stainfo23a(struct sta_priv *pstapriv, const u8 *hwaddr, gfp_t gfp)
-{
-	struct list_head	*phash_list;
-	struct sta_info	*psta;
-	struct recv_reorder_ctrl *preorder_ctrl;
-	s32	index;
-	int i = 0;
-	u16  wRxSeqInitialValue = 0xffff;
-
-	psta = kmalloc(sizeof(struct sta_info), gfp);
-	if (!psta)
-		return NULL;
-
-	spin_lock_bh(&pstapriv->sta_hash_lock);
-
-	_rtw_init_stainfo(psta);
-
-	psta->padapter = pstapriv->padapter;
-
-	ether_addr_copy(psta->hwaddr, hwaddr);
-
-	index = wifi_mac_hash(hwaddr);
-
-	RT_TRACE(_module_rtl871x_sta_mgt_c_, _drv_info_,
-		 "rtw_alloc_stainfo23a: index  = %x\n", index);
-	if (index >= NUM_STA) {
-		RT_TRACE(_module_rtl871x_sta_mgt_c_, _drv_err_,
-			 "ERROR => rtw_alloc_stainfo23a: index >= NUM_STA\n");
-		psta = NULL;
-		goto exit;
-	}
-	phash_list = &pstapriv->sta_hash[index];
-
-	list_add_tail(&psta->hash_list, phash_list);
-
-	pstapriv->asoc_sta_count++;
-
-/*  For the SMC router, the sequence number of first packet of WPS
-     handshake will be 0. */
-/*  In this case, this packet will be dropped by recv_decache function
-    if we use the 0x00 as the default value for tid_rxseq variable. */
-/*  So, we initialize the tid_rxseq variable as the 0xffff. */
-
-	for (i = 0; i < 16; i++)
-		memcpy(&psta->sta_recvpriv.rxcache.tid_rxseq[i],
-			&wRxSeqInitialValue, 2);
-
-	RT_TRACE(_module_rtl871x_sta_mgt_c_, _drv_info_,
-		 "alloc number_%d stainfo  with hwaddr = %pM\n",
-		 pstapriv->asoc_sta_count, hwaddr);
-
-	init_addba_retry_timer23a(psta);
-
-	/* for A-MPDU Rx reordering buffer control */
-	for (i = 0; i < 16; i++) {
-		preorder_ctrl = &psta->recvreorder_ctrl[i];
-
-		preorder_ctrl->padapter = pstapriv->padapter;
-
-		preorder_ctrl->enable = false;
-
-		preorder_ctrl->indicate_seq = 0xffff;
-		preorder_ctrl->wend_b = 0xffff;
-		/* preorder_ctrl->wsize_b = (NR_RECVBUFF-2); */
-		preorder_ctrl->wsize_b = 64;/* 64; */
-
-		_rtw_init_queue23a(&preorder_ctrl->pending_recvframe_queue);
-
-		rtw_init_recv_timer23a(preorder_ctrl);
-	}
-	/* init for DM */
-	psta->rssi_stat.UndecoratedSmoothedPWDB = (-1);
-	psta->rssi_stat.UndecoratedSmoothedCCK = (-1);
-
-	/* init for the sequence number of received management frame */
-	psta->RxMgmtFrameSeqNum = 0xffff;
-exit:
-	spin_unlock_bh(&pstapriv->sta_hash_lock);
-	return psta;
-}
-
-/*  using pstapriv->sta_hash_lock to protect */
-int rtw_free_stainfo23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	struct recv_reorder_ctrl *preorder_ctrl;
-	struct	sta_xmit_priv	*pstaxmitpriv;
-	struct	xmit_priv	*pxmitpriv = &padapter->xmitpriv;
-	struct	sta_priv *pstapriv = &padapter->stapriv;
-	struct hw_xmit *phwxmit;
-	int i;
-
-	if (!psta)
-		goto exit;
-
-	spin_lock_bh(&psta->lock);
-	psta->state &= ~_FW_LINKED;
-	spin_unlock_bh(&psta->lock);
-
-	pstaxmitpriv = &psta->sta_xmitpriv;
-
-	spin_lock_bh(&pxmitpriv->lock);
-
-	rtw_free_xmitframe_queue23a(pxmitpriv, &psta->sleep_q);
-	psta->sleepq_len = 0;
-
-	/* vo */
-	rtw_free_xmitframe_queue23a(pxmitpriv, &pstaxmitpriv->vo_q.sta_pending);
-	list_del_init(&pstaxmitpriv->vo_q.tx_pending);
-	phwxmit = pxmitpriv->hwxmits;
-	phwxmit->accnt -= pstaxmitpriv->vo_q.qcnt;
-	pstaxmitpriv->vo_q.qcnt = 0;
-
-	/* vi */
-	rtw_free_xmitframe_queue23a(pxmitpriv, &pstaxmitpriv->vi_q.sta_pending);
-	list_del_init(&pstaxmitpriv->vi_q.tx_pending);
-	phwxmit = pxmitpriv->hwxmits+1;
-	phwxmit->accnt -= pstaxmitpriv->vi_q.qcnt;
-	pstaxmitpriv->vi_q.qcnt = 0;
-
-	/* be */
-	rtw_free_xmitframe_queue23a(pxmitpriv, &pstaxmitpriv->be_q.sta_pending);
-	list_del_init(&pstaxmitpriv->be_q.tx_pending);
-	phwxmit = pxmitpriv->hwxmits+2;
-	phwxmit->accnt -= pstaxmitpriv->be_q.qcnt;
-	pstaxmitpriv->be_q.qcnt = 0;
-
-	/* bk */
-	rtw_free_xmitframe_queue23a(pxmitpriv, &pstaxmitpriv->bk_q.sta_pending);
-	list_del_init(&pstaxmitpriv->bk_q.tx_pending);
-	phwxmit = pxmitpriv->hwxmits+3;
-	phwxmit->accnt -= pstaxmitpriv->bk_q.qcnt;
-	pstaxmitpriv->bk_q.qcnt = 0;
-
-	spin_unlock_bh(&pxmitpriv->lock);
-
-	list_del_init(&psta->hash_list);
-	RT_TRACE(_module_rtl871x_sta_mgt_c_, _drv_err_,
-		 "free number_%d stainfo  with hwaddr = %pM\n",
-		 pstapriv->asoc_sta_count, psta->hwaddr);
-	pstapriv->asoc_sta_count--;
-
-	/*  re-init sta_info; 20061114  will be init in alloc_stainfo */
-	/* _rtw_init_sta_xmit_priv23a(&psta->sta_xmitpriv); */
-	/* _rtw_init_sta_recv_priv23a(&psta->sta_recvpriv); */
-
-	del_timer_sync(&psta->addba_retry_timer);
-
-	/* for A-MPDU Rx reordering buffer control,
-	   cancel reordering_ctrl_timer */
-	for (i = 0; i < 16; i++) {
-		struct list_head	*phead, *plist;
-		struct recv_frame *prframe;
-		struct rtw_queue *ppending_recvframe_queue;
-
-		preorder_ctrl = &psta->recvreorder_ctrl[i];
-
-		del_timer_sync(&preorder_ctrl->reordering_ctrl_timer);
-
-		ppending_recvframe_queue =
-			&preorder_ctrl->pending_recvframe_queue;
-
-		spin_lock_bh(&ppending_recvframe_queue->lock);
-		phead =		get_list_head(ppending_recvframe_queue);
-		plist = phead->next;
-
-		while (!list_empty(phead)) {
-			prframe = container_of(plist, struct recv_frame, list);
-			plist = plist->next;
-			list_del_init(&prframe->list);
-			rtw_free_recvframe23a(prframe);
-		}
-		spin_unlock_bh(&ppending_recvframe_queue->lock);
-	}
-	if (!(psta->state & WIFI_AP_STATE))
-		rtl8723a_SetHalODMVar(padapter, HAL_ODM_STA_INFO, psta, false);
-#ifdef CONFIG_8723AU_AP_MODE
-	spin_lock_bh(&pstapriv->auth_list_lock);
-	if (!list_empty(&psta->auth_list)) {
-		list_del_init(&psta->auth_list);
-		pstapriv->auth_list_cnt--;
-	}
-	spin_unlock_bh(&pstapriv->auth_list_lock);
-
-	psta->expire_to = 0;
-
-	psta->sleepq_ac_len = 0;
-	psta->qos_info = 0;
-
-	psta->max_sp_len = 0;
-	psta->uapsd_bk = 0;
-	psta->uapsd_be = 0;
-	psta->uapsd_vi = 0;
-	psta->uapsd_vo = 0;
-
-	psta->has_legacy_ac = 0;
-
-	pstapriv->sta_dz_bitmap &= ~CHKBIT(psta->aid);
-	pstapriv->tim_bitmap &= ~CHKBIT(psta->aid);
-
-	if ((psta->aid > 0) && (pstapriv->sta_aid[psta->aid - 1] == psta)) {
-		pstapriv->sta_aid[psta->aid - 1] = NULL;
-		psta->aid = 0;
-	}
-#endif	/*  CONFIG_8723AU_AP_MODE */
-
-	kfree(psta);
-exit:
-	return _SUCCESS;
-}
-
-/*  free all stainfo which in sta_hash[all] */
-void rtw_free_all_stainfo23a(struct rtw_adapter *padapter)
-{
-	struct list_head *phead;
-	struct sta_info *psta, *ptmp;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct sta_info *pbcmc_stainfo = rtw_get_bcmc_stainfo23a(padapter);
-	s32 index;
-
-	if (pstapriv->asoc_sta_count == 1)
-		return;
-
-	spin_lock_bh(&pstapriv->sta_hash_lock);
-	for (index = 0; index < NUM_STA; index++) {
-		phead = &pstapriv->sta_hash[index];
-		list_for_each_entry_safe(psta, ptmp, phead, hash_list) {
-			if (pbcmc_stainfo != psta)
-				rtw_free_stainfo23a(padapter, psta);
-		}
-	}
-	spin_unlock_bh(&pstapriv->sta_hash_lock);
-}
-
-/* any station allocated can be searched by hash list */
-struct sta_info *rtw_get_stainfo23a(struct sta_priv *pstapriv, const u8 *hwaddr)
-{
-	struct list_head *phead;
-	struct sta_info *pos, *psta = NULL;
-	u32 index;
-	const u8 *addr;
-
-	if (!hwaddr)
-		return NULL;
-
-	if (is_multicast_ether_addr(hwaddr))
-		addr = bc_addr;
-	else
-		addr = hwaddr;
-
-	index = wifi_mac_hash(addr);
-
-	spin_lock_bh(&pstapriv->sta_hash_lock);
-	phead = &pstapriv->sta_hash[index];
-	list_for_each_entry(pos, phead, hash_list) {
-		psta = pos;
-
-		/*  if found the matched address */
-		if (ether_addr_equal(psta->hwaddr, addr))
-			break;
-
-		psta = NULL;
-	}
-	spin_unlock_bh(&pstapriv->sta_hash_lock);
-	return psta;
-}
-
-int rtw_init_bcmc_stainfo23a(struct rtw_adapter *padapter)
-{
-	struct	sta_priv *pstapriv = &padapter->stapriv;
-	struct sta_info		*psta;
-	struct tx_servq	*ptxservq;
-	int res = _SUCCESS;
-
-	psta = rtw_alloc_stainfo23a(pstapriv, bc_addr, GFP_KERNEL);
-	if (!psta) {
-		res = _FAIL;
-		RT_TRACE(_module_rtl871x_sta_mgt_c_, _drv_err_,
-			 "rtw_alloc_stainfo23a fail\n");
-		return res;
-	}
-	/*  default broadcast & multicast use macid 1 */
-	psta->mac_id = 1;
-
-	ptxservq = &psta->sta_xmitpriv.be_q;
-	return _SUCCESS;
-}
-
-struct sta_info *rtw_get_bcmc_stainfo23a(struct rtw_adapter *padapter)
-{
-	struct sta_info		*psta;
-	struct sta_priv		*pstapriv = &padapter->stapriv;
-
-	psta = rtw_get_stainfo23a(pstapriv, bc_addr);
-	return psta;
-}
-
-bool rtw_access_ctrl23a(struct rtw_adapter *padapter, u8 *mac_addr)
-{
-	bool res = true;
-#ifdef CONFIG_8723AU_AP_MODE
-	struct list_head *phead;
-	struct rtw_wlan_acl_node *paclnode;
-	bool match = false;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct wlan_acl_pool *pacl_list = &pstapriv->acl_list;
-	struct rtw_queue *pacl_node_q = &pacl_list->acl_node_q;
-
-	spin_lock_bh(&pacl_node_q->lock);
-	phead = get_list_head(pacl_node_q);
-	list_for_each_entry(paclnode, phead, list) {
-		if (ether_addr_equal(paclnode->addr, mac_addr)) {
-			if (paclnode->valid) {
-				match = true;
-				break;
-			}
-		}
-	}
-	spin_unlock_bh(&pacl_node_q->lock);
-
-	if (pacl_list->mode == 1)/* accept unless in deny list */
-		res = (match) ?  false : true;
-	else if (pacl_list->mode == 2)/* deny unless in accept list */
-		res = (match) ?  true : false;
-	else
-		 res = true;
-#endif
-	return res;
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_wlan_util.c b/drivers/staging/rtl8723au/core/rtw_wlan_util.c
deleted file mode 100644
index 694cf17f82cf..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_wlan_util.c
+++ /dev/null
@@ -1,1537 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_WLAN_UTIL_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <linux/ieee80211.h>
-#include <wifi.h>
-#include <rtl8723a_spec.h>
-
-static unsigned char ARTHEROS_OUI1[] = {0x00, 0x03, 0x7f};
-static unsigned char ARTHEROS_OUI2[] = {0x00, 0x13, 0x74};
-
-static unsigned char BROADCOM_OUI1[] = {0x00, 0x10, 0x18};
-static unsigned char BROADCOM_OUI2[] = {0x00, 0x0a, 0xf7};
-
-static unsigned char CISCO_OUI[] = {0x00, 0x40, 0x96};
-static unsigned char MARVELL_OUI[] = {0x00, 0x50, 0x43};
-static unsigned char RALINK_OUI[] = {0x00, 0x0c, 0x43};
-static unsigned char REALTEK_OUI[] = {0x00, 0xe0, 0x4c};
-static unsigned char AIRGOCAP_OUI[] = {0x00, 0x0a, 0xf5};
-static unsigned char EPIGRAM_OUI[] = {0x00, 0x90, 0x4c};
-
-static unsigned char WPA_TKIP_CIPHER[4] = {0x00, 0x50, 0xf2, 0x02};
-static unsigned char RSN_TKIP_CIPHER[4] = {0x00, 0x0f, 0xac, 0x02};
-
-#define R2T_PHY_DELAY		0
-
-/* define WAIT_FOR_BCN_TO_MIN	3000 */
-#define WAIT_FOR_BCN_TO_MIN	6000
-#define WAIT_FOR_BCN_TO_MAX	20000
-
-static u8 rtw_basic_rate_cck[4] = {
-	IEEE80211_CCK_RATE_1MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_CCK_RATE_2MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_CCK_RATE_5MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_CCK_RATE_11MB | IEEE80211_BASIC_RATE_MASK
-};
-
-static u8 rtw_basic_rate_ofdm[3] = {
-	IEEE80211_OFDM_RATE_6MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_OFDM_RATE_12MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_OFDM_RATE_24MB | IEEE80211_BASIC_RATE_MASK
-};
-
-static u8 rtw_basic_rate_mix[7] = {
-	IEEE80211_CCK_RATE_1MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_CCK_RATE_2MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_CCK_RATE_5MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_CCK_RATE_11MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_OFDM_RATE_6MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_OFDM_RATE_12MB | IEEE80211_BASIC_RATE_MASK,
-	IEEE80211_OFDM_RATE_24MB | IEEE80211_BASIC_RATE_MASK
-};
-
-int cckrates_included23a(unsigned char *rate, int ratelen)
-{
-	int	i;
-
-	for (i = 0; i < ratelen; i++) {
-		if  (((rate[i]) & 0x7f) == 2 || ((rate[i]) & 0x7f) == 4 ||
-		     ((rate[i]) & 0x7f) == 11  || ((rate[i]) & 0x7f) == 22)
-			return true;
-	}
-
-	return false;
-}
-
-int cckratesonly_included23a(unsigned char *rate, int ratelen)
-{
-	int	i;
-
-	for (i = 0; i < ratelen; i++) {
-		if  (((rate[i]) & 0x7f) != 2 && ((rate[i]) & 0x7f) != 4 &&
-		     ((rate[i]) & 0x7f) != 11 && ((rate[i]) & 0x7f) != 22)
-		return false;
-	}
-
-	return true;
-}
-
-unsigned char networktype_to_raid23a(unsigned char network_type)
-{
-	unsigned char raid;
-
-	switch (network_type) {
-	case WIRELESS_11B:
-		raid = RATR_INX_WIRELESS_B;
-		break;
-	case WIRELESS_11A:
-	case WIRELESS_11G:
-		raid = RATR_INX_WIRELESS_G;
-		break;
-	case WIRELESS_11BG:
-		raid = RATR_INX_WIRELESS_GB;
-		break;
-	case WIRELESS_11_24N:
-	case WIRELESS_11_5N:
-		raid = RATR_INX_WIRELESS_N;
-		break;
-	case WIRELESS_11A_5N:
-	case WIRELESS_11G_24N:
-		raid = RATR_INX_WIRELESS_NG;
-		break;
-	case WIRELESS_11BG_24N:
-		raid = RATR_INX_WIRELESS_NGB;
-		break;
-	default:
-		raid = RATR_INX_WIRELESS_GB;
-		break;
-	}
-	return raid;
-}
-
-u8 judge_network_type23a(struct rtw_adapter *padapter,
-			 unsigned char *rate, int ratelen)
-{
-	u8 network_type = 0;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (pmlmeext->cur_channel > 14) {
-		if (pmlmeinfo->HT_enable)
-			network_type = WIRELESS_11_5N;
-		network_type |= WIRELESS_11A;
-	} else {
-		if (pmlmeinfo->HT_enable)
-			network_type = WIRELESS_11_24N;
-
-		if ((cckratesonly_included23a(rate, ratelen)) == true)
-			network_type |= WIRELESS_11B;
-		else if ((cckrates_included23a(rate, ratelen)) == true)
-			network_type |= WIRELESS_11BG;
-		else
-			network_type |= WIRELESS_11G;
-	}
-	return	network_type;
-}
-
-static unsigned char ratetbl_val_2wifirate(unsigned char rate)
-{
-	unsigned char val = 0;
-
-	switch (rate & 0x7f) {
-	case 0:
-		val = IEEE80211_CCK_RATE_1MB;
-		break;
-	case 1:
-		val = IEEE80211_CCK_RATE_2MB;
-		break;
-	case 2:
-		val = IEEE80211_CCK_RATE_5MB;
-		break;
-	case 3:
-		val = IEEE80211_CCK_RATE_11MB;
-		break;
-	case 4:
-		val = IEEE80211_OFDM_RATE_6MB;
-		break;
-	case 5:
-		val = IEEE80211_OFDM_RATE_9MB;
-		break;
-	case 6:
-		val = IEEE80211_OFDM_RATE_12MB;
-		break;
-	case 7:
-		val = IEEE80211_OFDM_RATE_18MB;
-		break;
-	case 8:
-		val = IEEE80211_OFDM_RATE_24MB;
-		break;
-	case 9:
-		val = IEEE80211_OFDM_RATE_36MB;
-		break;
-	case 10:
-		val = IEEE80211_OFDM_RATE_48MB;
-		break;
-	case 11:
-		val = IEEE80211_OFDM_RATE_54MB;
-		break;
-	}
-	return val;
-}
-
-static int is_basicrate(struct rtw_adapter *padapter, unsigned char rate)
-{
-	int i;
-	unsigned char val;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	for (i = 0; i < NumRates; i++) {
-		val = pmlmeext->basicrate[i];
-
-		if (val != 0xff && val != 0xfe) {
-			if (rate == ratetbl_val_2wifirate(val))
-				return true;
-		}
-	}
-
-	return false;
-}
-
-static unsigned int ratetbl2rateset(struct rtw_adapter *padapter,
-				    unsigned char *rateset)
-{
-	int i;
-	unsigned char rate;
-	unsigned int len = 0;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	for (i = 0; i < NumRates; i++) {
-		rate = pmlmeext->datarate[i];
-
-		switch (rate) {
-		case 0xff:
-			return len;
-		case 0xfe:
-			continue;
-		default:
-			rate = ratetbl_val_2wifirate(rate);
-
-			if (is_basicrate(padapter, rate) == true)
-				rate |= IEEE80211_BASIC_RATE_MASK;
-
-			rateset[len] = rate;
-			len++;
-			break;
-		}
-	}
-	return len;
-}
-
-void get_rate_set23a(struct rtw_adapter *padapter,
-		     unsigned char *pbssrate, int *bssrate_len)
-{
-	unsigned char supportedrates[NumRates];
-
-	memset(supportedrates, 0, NumRates);
-	*bssrate_len = ratetbl2rateset(padapter, supportedrates);
-	memcpy(pbssrate, supportedrates, *bssrate_len);
-}
-
-void UpdateBrateTbl23a(struct rtw_adapter *Adapter, u8 *mBratesOS)
-{
-	u8 i;
-	u8 rate;
-
-	/*  1M, 2M, 5.5M, 11M, 6M, 12M, 24M are mandatory. */
-	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
-		rate = mBratesOS[i] & 0x7f;
-		switch (rate) {
-		case IEEE80211_CCK_RATE_1MB:
-		case IEEE80211_CCK_RATE_2MB:
-		case IEEE80211_CCK_RATE_5MB:
-		case IEEE80211_CCK_RATE_11MB:
-		case IEEE80211_OFDM_RATE_6MB:
-		case IEEE80211_OFDM_RATE_12MB:
-		case IEEE80211_OFDM_RATE_24MB:
-			mBratesOS[i] |= IEEE80211_BASIC_RATE_MASK;
-			break;
-		default:
-			break;
-		}
-	}
-}
-
-void Update23aTblForSoftAP(u8 *bssrateset, u32 bssratelen)
-{
-	u8 i;
-	u8 rate;
-
-	for (i = 0; i < bssratelen; i++) {
-		rate = bssrateset[i] & 0x7f;
-		switch (rate) {
-		case IEEE80211_CCK_RATE_1MB:
-		case IEEE80211_CCK_RATE_2MB:
-		case IEEE80211_CCK_RATE_5MB:
-		case IEEE80211_CCK_RATE_11MB:
-			bssrateset[i] |= IEEE80211_BASIC_RATE_MASK;
-			break;
-		}
-	}
-}
-
-inline u8 rtw_get_oper_ch23a(struct rtw_adapter *adapter)
-{
-	return adapter_to_dvobj(adapter)->oper_channel;
-}
-
-inline void rtw_set_oper_ch23a(struct rtw_adapter *adapter, u8 ch)
-{
-	adapter_to_dvobj(adapter)->oper_channel = ch;
-}
-
-inline void rtw_set_oper_bw23a(struct rtw_adapter *adapter, u8 bw)
-{
-	adapter_to_dvobj(adapter)->oper_bwmode = bw;
-}
-
-inline void rtw_set_oper_ch23aoffset23a(struct rtw_adapter *adapter, u8 offset)
-{
-	adapter_to_dvobj(adapter)->oper_ch_offset = offset;
-}
-
-void SelectChannel23a(struct rtw_adapter *padapter, unsigned char channel)
-{
-	mutex_lock(&adapter_to_dvobj(padapter)->setch_mutex);
-
-	/* saved channel info */
-	rtw_set_oper_ch23a(padapter, channel);
-
-	PHY_SwChnl8723A(padapter, channel);
-
-	mutex_unlock(&adapter_to_dvobj(padapter)->setch_mutex);
-}
-
-static void set_bwmode(struct rtw_adapter *padapter, unsigned short bwmode,
-		       unsigned char channel_offset)
-{
-	mutex_lock(&adapter_to_dvobj(padapter)->setbw_mutex);
-
-	/* saved bw info */
-	rtw_set_oper_bw23a(padapter, bwmode);
-	rtw_set_oper_ch23aoffset23a(padapter, channel_offset);
-
-	PHY_SetBWMode23a8723A(padapter, (enum ht_channel_width)bwmode,
-			      channel_offset);
-
-	mutex_unlock(&adapter_to_dvobj(padapter)->setbw_mutex);
-}
-
-void set_channel_bwmode23a(struct rtw_adapter *padapter, unsigned char channel,
-			   unsigned char channel_offset, unsigned short bwmode)
-{
-	u8 center_ch;
-
-	if (bwmode == HT_CHANNEL_WIDTH_20 ||
-	    channel_offset == HAL_PRIME_CHNL_OFFSET_DONT_CARE) {
-		/* SelectChannel23a(padapter, channel); */
-		center_ch = channel;
-	} else {
-		/* switch to the proper channel */
-		if (channel_offset == HAL_PRIME_CHNL_OFFSET_LOWER) {
-			/* SelectChannel23a(padapter, channel + 2); */
-			center_ch = channel + 2;
-		} else {
-			/* SelectChannel23a(padapter, channel - 2); */
-			center_ch = channel - 2;
-		}
-	}
-
-	/* set Channel */
-	mutex_lock(&adapter_to_dvobj(padapter)->setch_mutex);
-
-	/* saved channel/bw info */
-	rtw_set_oper_ch23a(padapter, channel);
-	rtw_set_oper_bw23a(padapter, bwmode);
-	rtw_set_oper_ch23aoffset23a(padapter, channel_offset);
-
-	PHY_SwChnl8723A(padapter, center_ch); /*  set center channel */
-
-	mutex_unlock(&adapter_to_dvobj(padapter)->setch_mutex);
-
-	set_bwmode(padapter, bwmode, channel_offset);
-}
-
-inline u8 *get_my_bssid23a(struct wlan_bssid_ex *pnetwork)
-{
-	return pnetwork->MacAddress;
-}
-
-bool is_client_associated_to_ap23a(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext;
-	struct mlme_ext_info *pmlmeinfo;
-
-	if (!padapter)
-		return false;
-
-	pmlmeext = &padapter->mlmeextpriv;
-	pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS &&
-	    (pmlmeinfo->state & 0x03) == MSR_INFRA)
-		return true;
-	else
-		return false;
-}
-
-bool is_client_associated_to_ibss23a(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS &&
-	    (pmlmeinfo->state & 0x03) == MSR_ADHOC)
-		return true;
-	else
-		return false;
-}
-
-bool is_IBSS_empty23a(struct rtw_adapter *padapter)
-{
-	unsigned int i;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	for (i = IBSS_START_MAC_ID; i < NUM_STA; i++) {
-		if (pmlmeinfo->FW_sta_info[i].status == 1)
-			return false;
-	}
-
-	return true;
-}
-
-unsigned int decide_wait_for_beacon_timeout23a(unsigned int bcn_interval)
-{
-	if ((bcn_interval << 2) < WAIT_FOR_BCN_TO_MIN)
-		return WAIT_FOR_BCN_TO_MIN;
-	else if ((bcn_interval << 2) > WAIT_FOR_BCN_TO_MAX)
-		return WAIT_FOR_BCN_TO_MAX;
-	else
-		return bcn_interval << 2;
-}
-
-void clear_cam_entry23a(struct rtw_adapter *padapter, u8 entry)
-{
-	unsigned char null_sta[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
-
-	unsigned char null_key[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-				    0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-				    0x00, 0x00, 0x00, 0x00};
-
-	rtl8723a_cam_write(padapter, entry, 0, null_sta, null_key);
-}
-
-int allocate_fw_sta_entry23a(struct rtw_adapter *padapter)
-{
-	unsigned int mac_id;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	for (mac_id = IBSS_START_MAC_ID; mac_id < NUM_STA; mac_id++) {
-		if (pmlmeinfo->FW_sta_info[mac_id].status == 0) {
-			pmlmeinfo->FW_sta_info[mac_id].status = 1;
-			pmlmeinfo->FW_sta_info[mac_id].retry = 0;
-			break;
-		}
-	}
-
-	return mac_id;
-}
-
-void flush_all_cam_entry23a(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	rtl8723a_cam_invalidate_all(padapter);
-
-	memset(pmlmeinfo->FW_sta_info, 0, sizeof(pmlmeinfo->FW_sta_info));
-}
-
-int WMM_param_handler23a(struct rtw_adapter *padapter, const u8 *p)
-{
-	/* struct registry_priv	*pregpriv = &padapter->registrypriv; */
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (pmlmepriv->qos_option == 0) {
-		pmlmeinfo->WMM_enable = 0;
-		return _FAIL;
-	}
-
-	pmlmeinfo->WMM_enable = 1;
-	memcpy(&pmlmeinfo->WMM_param, p + 2 + 6,
-	       sizeof(struct WMM_para_element));
-	return true;
-}
-
-void WMMOnAssocRsp23a(struct rtw_adapter *padapter)
-{
-	u8 ACI, ACM, AIFS, ECWMin, ECWMax, aSifsTime;
-	u8 acm_mask;
-	u16 TXOP;
-	u32 acParm, i;
-	u32 edca[4], inx[4];
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct registry_priv *pregpriv = &padapter->registrypriv;
-
-	if (pmlmeinfo->WMM_enable == 0) {
-		padapter->mlmepriv.acm_mask = 0;
-		return;
-	}
-
-	acm_mask = 0;
-
-	if (pmlmeext->cur_wireless_mode == WIRELESS_11B)
-		aSifsTime = 10;
-	else
-		aSifsTime = 16;
-
-	for (i = 0; i < 4; i++) {
-		ACI = (pmlmeinfo->WMM_param.ac_param[i].ACI_AIFSN >> 5) & 0x03;
-		ACM = (pmlmeinfo->WMM_param.ac_param[i].ACI_AIFSN >> 4) & 0x01;
-
-		/* AIFS = AIFSN * slot time + SIFS - r2t phy delay */
-		AIFS = (pmlmeinfo->WMM_param.ac_param[i].ACI_AIFSN & 0x0f) *
-			pmlmeinfo->slotTime + aSifsTime;
-
-		ECWMin = pmlmeinfo->WMM_param.ac_param[i].CW & 0x0f;
-		ECWMax = (pmlmeinfo->WMM_param.ac_param[i].CW & 0xf0) >> 4;
-		TXOP = le16_to_cpu(pmlmeinfo->WMM_param.ac_param[i].TXOP_limit);
-
-		acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
-
-		switch (ACI) {
-		case 0x0:
-			rtl8723a_set_ac_param_be(padapter, acParm);
-			acm_mask |= (ACM? BIT(1):0);
-			edca[XMIT_BE_QUEUE] = acParm;
-			break;
-		case 0x1:
-			rtl8723a_set_ac_param_bk(padapter, acParm);
-			/* acm_mask |= (ACM? BIT(0):0); */
-			edca[XMIT_BK_QUEUE] = acParm;
-			break;
-		case 0x2:
-			rtl8723a_set_ac_param_vi(padapter, acParm);
-			acm_mask |= (ACM? BIT(2):0);
-			edca[XMIT_VI_QUEUE] = acParm;
-			break;
-		case 0x3:
-			rtl8723a_set_ac_param_vo(padapter, acParm);
-			acm_mask |= (ACM? BIT(3):0);
-			edca[XMIT_VO_QUEUE] = acParm;
-			break;
-		}
-
-		DBG_8723A("WMM(%x): %x, %x\n", ACI, ACM, acParm);
-	}
-
-	if (padapter->registrypriv.acm_method == 1)
-		rtl8723a_set_acm_ctrl(padapter, acm_mask);
-	else
-		padapter->mlmepriv.acm_mask = acm_mask;
-
-	inx[0] = 0; inx[1] = 1; inx[2] = 2; inx[3] = 3;
-
-	if (pregpriv->wifi_spec == 1) {
-		u32 j, change_inx = false;
-
-		/* entry indx: 0->vo, 1->vi, 2->be, 3->bk. */
-		for (i = 0; i < 4; i++) {
-			for (j = i+1; j < 4; j++) {
-				/* compare CW and AIFS */
-				if ((edca[j] & 0xFFFF) < (edca[i] & 0xFFFF)) {
-					change_inx = true;
-				} else if ((edca[j] & 0xFFFF) ==
-					   (edca[i] & 0xFFFF)) {
-					/* compare TXOP */
-					if ((edca[j] >> 16) > (edca[i] >> 16))
-						change_inx = true;
-				}
-
-				if (change_inx) {
-					swap(edca[i], edca[j]);
-					swap(inx[i], inx[j]);
-					change_inx = false;
-				}
-			}
-		}
-	}
-
-	for (i = 0; i<4; i++) {
-		pxmitpriv->wmm_para_seq[i] = inx[i];
-		DBG_8723A("wmm_para_seq(%d): %d\n", i,
-			  pxmitpriv->wmm_para_seq[i]);
-	}
-}
-
-static void bwmode_update_check(struct rtw_adapter *padapter, const u8 *p)
-{
-	struct ieee80211_ht_operation *pHT_info;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-	unsigned char new_bwmode;
-	unsigned char new_ch_offset;
-
-	if (!p)
-		return;
-	if (!phtpriv->ht_option)
-		return;
-	if (p[1] != sizeof(struct ieee80211_ht_operation))
-		return;
-
-	pHT_info = (struct ieee80211_ht_operation *)(p + 2);
-
-	if ((pHT_info->ht_param & IEEE80211_HT_PARAM_CHAN_WIDTH_ANY) &&
-	    pregistrypriv->cbw40_enable) {
-		new_bwmode = HT_CHANNEL_WIDTH_40;
-
-		switch (pHT_info->ht_param & IEEE80211_HT_PARAM_CHA_SEC_OFFSET){
-		case IEEE80211_HT_PARAM_CHA_SEC_ABOVE:
-			new_ch_offset = HAL_PRIME_CHNL_OFFSET_LOWER;
-			break;
-		case IEEE80211_HT_PARAM_CHA_SEC_BELOW:
-			new_ch_offset = HAL_PRIME_CHNL_OFFSET_UPPER;
-			break;
-		default:
-			new_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-			break;
-		}
-	} else {
-		new_bwmode = HT_CHANNEL_WIDTH_20;
-		new_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-	}
-
-	if (new_bwmode != pmlmeext->cur_bwmode ||
-	    new_ch_offset != pmlmeext->cur_ch_offset) {
-		pmlmeinfo->bwmode_updated = true;
-
-		pmlmeext->cur_bwmode = new_bwmode;
-		pmlmeext->cur_ch_offset = new_ch_offset;
-
-		/* update HT info also */
-		HT_info_handler23a(padapter, p);
-	} else
-		pmlmeinfo->bwmode_updated = false;
-
-	if (pmlmeinfo->bwmode_updated) {
-		struct sta_info *psta;
-		struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-		struct sta_priv	*pstapriv = &padapter->stapriv;
-
-		/* set_channel_bwmode23a(padapter, pmlmeext->cur_channel,
-		   pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode); */
-
-		/* update ap's stainfo */
-		psta = rtw_get_stainfo23a(pstapriv, cur_network->MacAddress);
-		if (psta) {
-			struct ht_priv *phtpriv_sta = &psta->htpriv;
-
-			if (phtpriv_sta->ht_option) {
-				/*  bwmode */
-				phtpriv_sta->bwmode = pmlmeext->cur_bwmode;
-				phtpriv_sta->ch_offset =
-					pmlmeext->cur_ch_offset;
-			} else {
-				phtpriv_sta->bwmode = HT_CHANNEL_WIDTH_20;
-				phtpriv_sta->ch_offset =
-					HAL_PRIME_CHNL_OFFSET_DONT_CARE;
-			}
-		}
-	}
-}
-
-void HT_caps_handler23a(struct rtw_adapter *padapter, const u8 *p)
-{
-	unsigned int i;
-	u8 rf_type;
-	u8 max_AMPDU_len, min_MPDU_spacing;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-	struct ieee80211_ht_cap *cap;
-	u8 *dstcap;
-
-	if (!p)
-		return;
-
-	if (!phtpriv->ht_option)
-		return;
-
-	pmlmeinfo->HT_caps_enable = 1;
-
-	cap = &pmlmeinfo->ht_cap;
-	dstcap = (u8 *)cap;
-	for (i = 0; i < p[1]; i++) {
-		if (i != 2) {
-			dstcap[i] &= p[i + 2];
-		} else {
-			/* modify from  fw by Thomas 2010/11/17 */
-			if ((cap->ampdu_params_info &
-			     IEEE80211_HT_AMPDU_PARM_FACTOR) >
-			    (p[i + 2] & IEEE80211_HT_AMPDU_PARM_FACTOR))
-				max_AMPDU_len = p[i + 2] &
-					IEEE80211_HT_AMPDU_PARM_FACTOR;
-			else
-				max_AMPDU_len = cap->ampdu_params_info &
-					IEEE80211_HT_AMPDU_PARM_FACTOR;
-
-			if ((cap->ampdu_params_info &
-			     IEEE80211_HT_AMPDU_PARM_DENSITY) >
-			    (p[i + 2] & IEEE80211_HT_AMPDU_PARM_DENSITY))
-				min_MPDU_spacing = cap->ampdu_params_info &
-					IEEE80211_HT_AMPDU_PARM_DENSITY;
-			else
-				min_MPDU_spacing = p[i + 2] &
-					IEEE80211_HT_AMPDU_PARM_DENSITY;
-
-			cap->ampdu_params_info =
-				max_AMPDU_len | min_MPDU_spacing;
-		}
-	}
-
-	rf_type = rtl8723a_get_rf_type(padapter);
-
-	/* update the MCS rates */
-	for (i = 0; i < IEEE80211_HT_MCS_MASK_LEN; i++) {
-		if (rf_type == RF_1T1R || rf_type == RF_1T2R)
-			cap->mcs.rx_mask[i] &= MCS_rate_1R23A[i];
-		else
-			cap->mcs.rx_mask[i] &= MCS_rate_2R23A[i];
-	}
-}
-
-void HT_info_handler23a(struct rtw_adapter *padapter, const u8 *p)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-
-	if (!p)
-		return;
-
-	if (!phtpriv->ht_option)
-		return;
-
-	if (p[1] != sizeof(struct ieee80211_ht_operation))
-		return;
-
-	pmlmeinfo->HT_info_enable = 1;
-	memcpy(&pmlmeinfo->HT_info, p + 2, p[1]);
-}
-
-void HTOnAssocRsp23a(struct rtw_adapter *padapter)
-{
-	unsigned char max_AMPDU_len;
-	unsigned char min_MPDU_spacing;
-	/* struct registry_priv	 *pregpriv = &padapter->registrypriv; */
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	DBG_8723A("%s\n", __func__);
-
-	if (pmlmeinfo->HT_info_enable && pmlmeinfo->HT_caps_enable)
-		pmlmeinfo->HT_enable = 1;
-	else {
-		pmlmeinfo->HT_enable = 0;
-		/* set_channel_bwmode23a(padapter, pmlmeext->cur_channel,
-		   pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode); */
-		return;
-	}
-
-	/* handle A-MPDU parameter field */
-	/*
-		AMPDU_para [1:0]:Max AMPDU Len => 0:8k , 1:16k, 2:32k, 3:64k
-		AMPDU_para [4:2]:Min MPDU Start Spacing
-	*/
-	max_AMPDU_len = pmlmeinfo->ht_cap.ampdu_params_info &
-		IEEE80211_HT_AMPDU_PARM_FACTOR;
-
-	min_MPDU_spacing =
-		(pmlmeinfo->ht_cap.ampdu_params_info &
-		 IEEE80211_HT_AMPDU_PARM_DENSITY) >> 2;
-
-	rtl8723a_set_ampdu_min_space(padapter, min_MPDU_spacing);
-	rtl8723a_set_ampdu_factor(padapter, max_AMPDU_len);
-}
-
-void ERP_IE_handler23a(struct rtw_adapter *padapter, const u8 *p)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (p[1] > 1)
-		return;
-
-	pmlmeinfo->ERP_enable = 1;
-	memcpy(&pmlmeinfo->ERP_IE, p + 2, p[1]);
-}
-
-void VCS_update23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	struct registry_priv *pregpriv = &padapter->registrypriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	switch (pregpriv->vrtl_carrier_sense) { /* 0:off 1:on 2:auto */
-	case 0: /* off */
-		psta->rtsen = 0;
-		psta->cts2self = 0;
-		break;
-	case 1: /* on */
-		if (pregpriv->vcs_type == RTS_CTS) {
-			psta->rtsen = 1;
-			psta->cts2self = 0;
-		} else {
-			psta->rtsen = 0;
-			psta->cts2self = 1;
-		}
-		break;
-	case 2: /* auto */
-	default:
-		if (pmlmeinfo->ERP_enable && pmlmeinfo->ERP_IE & BIT(1)) {
-			if (pregpriv->vcs_type == RTS_CTS) {
-				psta->rtsen = 1;
-				psta->cts2self = 0;
-			} else {
-				psta->rtsen = 0;
-				psta->cts2self = 1;
-			}
-		} else {
-			psta->rtsen = 0;
-			psta->cts2self = 0;
-		}
-		break;
-	}
-}
-
-int rtw_check_bcn_info23a(struct rtw_adapter *Adapter,
-			  struct ieee80211_mgmt *mgmt, u32 pkt_len)
-{
-	struct wlan_network *cur_network = &Adapter->mlmepriv.cur_network;
-	struct ieee80211_ht_operation *pht_info;
-	unsigned short val16;
-	u8 crypto, bcn_channel;
-	int group_cipher = 0, pairwise_cipher = 0, is_8021x = 0, r;
-	int pie_len, ssid_len, privacy;
-	const u8 *p, *ssid;
-
-	if (!is_client_associated_to_ap23a(Adapter))
-		return _SUCCESS;
-
-	if (unlikely(!ieee80211_is_beacon(mgmt->frame_control))) {
-		printk(KERN_WARNING "%s: received a non beacon frame!\n",
-		       __func__);
-		return _FAIL;
-	}
-
-	if (!ether_addr_equal(cur_network->network.MacAddress, mgmt->bssid)) {
-		DBG_8723A("%s: linked but recv other bssid bcn %pM %pM\n",
-			  __func__, mgmt->bssid,
-			  cur_network->network.MacAddress);
-		return _FAIL;
-	}
-
-	/* check bw and channel offset */
-	/* parsing HT_CAP_IE */
-	pie_len = pkt_len - offsetof(struct ieee80211_mgmt, u.beacon.variable);
-
-	/* Checking for channel */
-	p = cfg80211_find_ie(WLAN_EID_DS_PARAMS, mgmt->u.beacon.variable,
-			     pie_len);
-	if (p)
-		bcn_channel = p[2];
-	else {
-		/* In 5G, some ap do not have DSSET IE checking HT
-		   info for channel */
-		p = cfg80211_find_ie(WLAN_EID_HT_OPERATION,
-				     mgmt->u.beacon.variable, pie_len);
-
-		if (p && p[1] > 0) {
-			pht_info = (struct ieee80211_ht_operation *)(p + 2);
-			bcn_channel = pht_info->primary_chan;
-		} else { /* we don't find channel IE, so don't check it */
-			DBG_8723A("Oops: %s we don't find channel IE, so don't "
-				  "check it\n", __func__);
-			bcn_channel = Adapter->mlmeextpriv.cur_channel;
-		}
-	}
-	if (bcn_channel != Adapter->mlmeextpriv.cur_channel) {
-		DBG_8723A("%s beacon channel:%d cur channel:%d disconnect\n",
-			  __func__, bcn_channel,
-			  Adapter->mlmeextpriv.cur_channel);
-		goto _mismatch;
-	}
-
-	/* checking SSID */
-	p = cfg80211_find_ie(WLAN_EID_SSID, mgmt->u.beacon.variable, pie_len);
-	if (p && p[1]) {
-		ssid = p + 2;
-		ssid_len = p[1];
-	} else {
-		DBG_8723A("%s marc: cannot find SSID for survey event\n",
-			  __func__);
-		ssid = NULL;
-		ssid_len = 0;
-	}
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "%s bssid.Ssid.Ssid:%s bssid.Ssid.SsidLength:%d cur_network->network.Ssid.Ssid:%s len:%d\n",
-		 __func__, ssid, ssid_len, cur_network->network.Ssid.ssid,
-		 cur_network->network.Ssid.ssid_len);
-
-	if (ssid_len != cur_network->network.Ssid.ssid_len || ssid_len > 32 ||
-	    (ssid_len &&
-	     memcmp(ssid, cur_network->network.Ssid.ssid, ssid_len))) {
-		DBG_8723A("%s(), SSID is not match return FAIL\n", __func__);
-		goto _mismatch;
-	}
-
-	/* check encryption info */
-	val16 = le16_to_cpu(mgmt->u.beacon.capab_info);
-
-	if (val16 & WLAN_CAPABILITY_PRIVACY)
-		privacy = 1;
-	else
-		privacy = 0;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-		 "%s(): cur_network->network.Privacy is %d, bssid.Privacy is %d\n",
-		 __func__, cur_network->network.Privacy, privacy);
-	if (cur_network->network.Privacy != privacy) {
-		DBG_8723A("%s(), privacy is not match return FAIL\n", __func__);
-		goto _mismatch;
-	}
-
-	p = cfg80211_find_ie(WLAN_EID_RSN, mgmt->u.beacon.variable, pie_len);
-	if (p && p[1]) {
-		crypto = ENCRYP_PROTOCOL_WPA2;
-		if (p && p[1]) {
-			r = rtw_parse_wpa2_ie23a(p, p[1] + 2, &group_cipher,
-						 &pairwise_cipher, &is_8021x);
-			if (r == _SUCCESS)
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-					 "%s pnetwork->pairwise_cipher: %d, pnetwork->group_cipher: %d, is_802x : %d\n",
-					 __func__, pairwise_cipher,
-					 group_cipher, is_8021x);
-			}
-	} else {
-		p = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-					    WLAN_OUI_TYPE_MICROSOFT_WPA,
-					    mgmt->u.beacon.variable, pie_len);
-		if (p && p[1]) {
-			crypto = ENCRYP_PROTOCOL_WPA;
-			r = rtw_parse_wpa_ie23a(p, p[1] + 2, &group_cipher,
-						&pairwise_cipher, &is_8021x);
-			if (r == _SUCCESS)
-				RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_,
-					 "%s pnetwork->pairwise_cipher: %d, group_cipher is %d, is_8021x is %d\n",
-					 __func__, pairwise_cipher,
-					 group_cipher, is_8021x);
-		} else {
-			if (privacy)
-				crypto = ENCRYP_PROTOCOL_WEP;
-			else
-				crypto = ENCRYP_PROTOCOL_OPENSYS;
-		}
-	}
-
-	if (cur_network->BcnInfo.encryp_protocol != crypto) {
-		DBG_8723A("%s(): encryption mismatch, return FAIL\n", __func__);
-		goto _mismatch;
-	}
-
-	if (crypto == ENCRYP_PROTOCOL_WPA || crypto == ENCRYP_PROTOCOL_WPA2) {
-		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
-			 "%s cur_network->group_cipher is %d: %d\n", __func__,
-			 cur_network->BcnInfo.group_cipher, group_cipher);
-		if (pairwise_cipher != cur_network->BcnInfo.pairwise_cipher ||
-		    group_cipher != cur_network->BcnInfo.group_cipher) {
-			DBG_8723A("%s pairwise_cipher(%x:%x) or group_cipher "
-				  "(%x:%x) is not match, return FAIL\n",
-				  __func__, pairwise_cipher,
-				  cur_network->BcnInfo.pairwise_cipher,
-				  group_cipher,
-				  cur_network->BcnInfo.group_cipher);
-			goto _mismatch;
-		}
-
-		if (is_8021x != cur_network->BcnInfo.is_8021x) {
-			DBG_8723A("%s authentication is not match, return "
-				  "FAIL\n", __func__);
-			goto _mismatch;
-		}
-	}
-
-	return _SUCCESS;
-
-_mismatch:
-
-	return _FAIL;
-}
-
-void update_beacon23a_info(struct rtw_adapter *padapter,
-			   struct ieee80211_mgmt *mgmt,
-			   uint pkt_len, struct sta_info *psta)
-{
-	unsigned int len;
-	const u8 *p;
-
-	len = pkt_len - offsetof(struct ieee80211_mgmt, u.beacon.variable);
-
-	p = cfg80211_find_ie(WLAN_EID_HT_OPERATION, mgmt->u.beacon.variable,
-			     len);
-	if (p)
-		bwmode_update_check(padapter, p);
-
-	p = cfg80211_find_ie(WLAN_EID_ERP_INFO, mgmt->u.beacon.variable, len);
-	if (p) {
-		ERP_IE_handler23a(padapter, p);
-		VCS_update23a(padapter, psta);
-	}
-}
-
-bool is_ap_in_tkip23a(struct rtw_adapter *padapter)
-{
-	u32 i;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-	const u8 *p;
-
-	if (cur_network->capability & WLAN_CAPABILITY_PRIVACY) {
-		for (i = 0; i < pmlmeinfo->network.IELength;) {
-			p = pmlmeinfo->network.IEs + i;
-
-			switch (p[0]) {
-			case WLAN_EID_VENDOR_SPECIFIC:
-				if (!memcmp(p + 2, RTW_WPA_OUI23A_TYPE, 4) &&
-				    !memcmp(p + 2 + 12, WPA_TKIP_CIPHER, 4))
-					return true;
-				break;
-			case WLAN_EID_RSN:
-				if (!memcmp(p + 2 + 8, RSN_TKIP_CIPHER, 4))
-					return true;
-				break;
-			default:
-				break;
-			}
-			i += (p[1] + 2);
-		}
-		return false;
-	} else
-		return false;
-}
-
-bool should_forbid_n_rate23a(struct rtw_adapter *padapter)
-{
-	u32 i;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_bssid_ex  *cur_network = &pmlmepriv->cur_network.network;
-	const u8 *p;
-
-	if (cur_network->capability & WLAN_CAPABILITY_PRIVACY) {
-		for (i = 0; i < cur_network->IELength;) {
-			p = cur_network->IEs + i;
-
-			switch (p[0]) {
-			case WLAN_EID_VENDOR_SPECIFIC:
-				if (!memcmp(p + 2, RTW_WPA_OUI23A_TYPE, 4) &&
-				    (!memcmp(p + 2 + 12,
-					     WPA_CIPHER_SUITE_CCMP23A, 4) ||
-				     !memcmp(p + 2 + 16,
-					     WPA_CIPHER_SUITE_CCMP23A, 4)))
-					return false;
-				break;
-			case WLAN_EID_RSN:
-				if (!memcmp(p + 2 + 8,
-					    RSN_CIPHER_SUITE_CCMP23A, 4) ||
-				    !memcmp(p + 2 + 12,
-					    RSN_CIPHER_SUITE_CCMP23A, 4))
-				return false;
-			default:
-				break;
-			}
-
-			i += (p[1] + 2);
-		}
-		return true;
-	} else {
-		return false;
-	}
-}
-
-bool is_ap_in_wep23a(struct rtw_adapter *padapter)
-{
-	u32 i;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-	const u8 *p;
-
-	if (cur_network->capability & WLAN_CAPABILITY_PRIVACY) {
-		for (i = 0; i < pmlmeinfo->network.IELength;) {
-			p = pmlmeinfo->network.IEs + i;
-
-			switch (p[0]) {
-			case WLAN_EID_VENDOR_SPECIFIC:
-				if (!memcmp(p + 2, RTW_WPA_OUI23A_TYPE, 4))
-					return false;
-				break;
-			case WLAN_EID_RSN:
-				return false;
-
-			default:
-				break;
-			}
-
-			i += (p[1] + 2);
-		}
-
-		return true;
-	} else
-		return false;
-}
-
-static int wifirate2_ratetbl_inx23a(unsigned char rate)
-{
-	int inx = 0;
-
-	rate = rate & 0x7f;
-
-	switch (rate) {
-	case 54*2:
-		inx = 11;
-		break;
-	case 48*2:
-		inx = 10;
-		break;
-	case 36*2:
-		inx = 9;
-		break;
-	case 24*2:
-		inx = 8;
-		break;
-	case 18*2:
-		inx = 7;
-		break;
-	case 12*2:
-		inx = 6;
-		break;
-	case 9*2:
-		inx = 5;
-		break;
-	case 6*2:
-		inx = 4;
-		break;
-	case 11*2:
-		inx = 3;
-		break;
-	case 11:
-		inx = 2;
-		break;
-	case 2*2:
-		inx = 1;
-		break;
-	case 1*2:
-		inx = 0;
-		break;
-	}
-	return inx;
-}
-
-unsigned int update_basic_rate23a(unsigned char *ptn, unsigned int ptn_sz)
-{
-	unsigned int i, num_of_rate;
-	unsigned int mask = 0;
-
-	num_of_rate = (ptn_sz > NumRates)? NumRates: ptn_sz;
-
-	for (i = 0; i < num_of_rate; i++) {
-		if ((*(ptn + i)) & 0x80)
-			mask |= 0x1 << wifirate2_ratetbl_inx23a(*(ptn + i));
-	}
-	return mask;
-}
-
-unsigned int update_supported_rate23a(unsigned char *ptn, unsigned int ptn_sz)
-{
-	unsigned int i, num_of_rate;
-	unsigned int mask = 0;
-
-	num_of_rate = (ptn_sz > NumRates) ? NumRates : ptn_sz;
-
-	for (i = 0; i < num_of_rate; i++)
-		mask |= 0x1 << wifirate2_ratetbl_inx23a(*(ptn + i));
-	return mask;
-}
-
-unsigned int update_MSC_rate23a(struct ieee80211_ht_cap *pHT_caps)
-{
-	unsigned int mask;
-
-	mask = pHT_caps->mcs.rx_mask[0] << 12 |
-		pHT_caps->mcs.rx_mask[1] << 20;
-
-	return mask;
-}
-
-int support_short_GI23a(struct rtw_adapter *padapter,
-			struct ieee80211_ht_cap *pHT_caps)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	unsigned char bit_offset;
-
-	if (!pmlmeinfo->HT_enable)
-		return _FAIL;
-	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_RALINK)
-		return _FAIL;
-	bit_offset = (pmlmeext->cur_bwmode & HT_CHANNEL_WIDTH_40)? 6: 5;
-
-	if (pHT_caps->cap_info & cpu_to_le16(0x1 << bit_offset))
-		return _SUCCESS;
-	else
-		return _FAIL;
-}
-
-unsigned char get_highest_rate_idx23a(u32 mask)
-{
-	int i;
-	unsigned char rate_idx = 0;
-
-	for (i = 27; i >= 0; i--) {
-		if (mask & BIT(i)) {
-			rate_idx = i;
-			break;
-		}
-	}
-	return rate_idx;
-}
-
-void Update_RA_Entry23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	rtw_hal_update_ra_mask23a(psta, 0);
-}
-
-static void enable_rate_adaptive(struct rtw_adapter *padapter,
-				 struct sta_info *psta)
-{
-	Update_RA_Entry23a(padapter, psta);
-}
-
-void set_sta_rate23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	/* rate adaptive */
-	enable_rate_adaptive(padapter, psta);
-}
-
-/*  Update RRSR and Rate for USERATE */
-void update_tx_basic_rate23a(struct rtw_adapter *padapter, u8 wirelessmode)
-{
-	unsigned char supported_rates[NDIS_802_11_LENGTH_RATES_EX];
-
-	memset(supported_rates, 0, NDIS_802_11_LENGTH_RATES_EX);
-
-	if (wirelessmode == WIRELESS_11B) {
-		memcpy(supported_rates, rtw_basic_rate_cck, 4);
-	} else if (wirelessmode & WIRELESS_11B) {
-		memcpy(supported_rates, rtw_basic_rate_mix, 7);
-	} else {
-		memcpy(supported_rates, rtw_basic_rate_ofdm, 3);
-	}
-
-	if (wirelessmode & WIRELESS_11B)
-		update_mgnt_tx_rate23a(padapter, IEEE80211_CCK_RATE_1MB);
-	else
-		update_mgnt_tx_rate23a(padapter, IEEE80211_OFDM_RATE_6MB);
-
-	HalSetBrateCfg23a(padapter, supported_rates);
-}
-
-unsigned char check_assoc_AP23a(u8 *pframe, uint len)
-{
-	int i;
-	u8 epigram_vendor_flag;
-	u8 ralink_vendor_flag;
-	const u8 *p;
-
-	epigram_vendor_flag = 0;
-	ralink_vendor_flag = 0;
-
-	for (i = 0; i < len;) {
-		p = pframe + i;
-
-		switch (p[0]) {
-		case WLAN_EID_VENDOR_SPECIFIC:
-			if (!memcmp(p + 2, ARTHEROS_OUI1, 3) ||
-			    !memcmp(p + 2, ARTHEROS_OUI2, 3)) {
-				DBG_8723A("link to Artheros AP\n");
-				return HT_IOT_PEER_ATHEROS;
-			} else if (!memcmp(p + 2, BROADCOM_OUI1, 3) ||
-				   !memcmp(p + 2, BROADCOM_OUI2, 3)) {
-				DBG_8723A("link to Broadcom AP\n");
-				return HT_IOT_PEER_BROADCOM;
-			} else if (!memcmp(p + 2, MARVELL_OUI, 3)) {
-				DBG_8723A("link to Marvell AP\n");
-				return HT_IOT_PEER_MARVELL;
-			} else if (!memcmp(p + 2, RALINK_OUI, 3)) {
-				if (!ralink_vendor_flag)
-					ralink_vendor_flag = 1;
-				else {
-					DBG_8723A("link to Ralink AP\n");
-					return HT_IOT_PEER_RALINK;
-				}
-			} else if (!memcmp(p + 2, CISCO_OUI, 3)) {
-				DBG_8723A("link to Cisco AP\n");
-				return HT_IOT_PEER_CISCO;
-			} else if (!memcmp(p + 2, REALTEK_OUI, 3)) {
-				DBG_8723A("link to Realtek 96B\n");
-				return HT_IOT_PEER_REALTEK;
-			} else if (!memcmp(p + 2, AIRGOCAP_OUI, 3)) {
-				DBG_8723A("link to Airgo Cap\n");
-				return HT_IOT_PEER_AIRGO;
-			} else if (!memcmp(p + 2, EPIGRAM_OUI, 3)) {
-				epigram_vendor_flag = 1;
-				if (ralink_vendor_flag) {
-					DBG_8723A("link to Tenda W311R AP\n");
-					return HT_IOT_PEER_TENDA;
-				} else
-					DBG_8723A("Capture EPIGRAM_OUI\n");
-			} else
-				break;
-		default:
-			break;
-		}
-
-		i += (p[1] + 2);
-	}
-
-	if (ralink_vendor_flag && !epigram_vendor_flag) {
-		DBG_8723A("link to Ralink AP\n");
-		return HT_IOT_PEER_RALINK;
-	} else if (ralink_vendor_flag && epigram_vendor_flag) {
-		DBG_8723A("link to Tenda W311R AP\n");
-		return HT_IOT_PEER_TENDA;
-	} else {
-		DBG_8723A("link to new AP\n");
-		return HT_IOT_PEER_UNKNOWN;
-	}
-}
-
-void update_IOT_info23a(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	switch (pmlmeinfo->assoc_AP_vendor) {
-	case HT_IOT_PEER_MARVELL:
-		pmlmeinfo->turboMode_cts2self = 1;
-		pmlmeinfo->turboMode_rtsen = 0;
-		break;
-	case HT_IOT_PEER_RALINK:
-		pmlmeinfo->turboMode_cts2self = 0;
-		pmlmeinfo->turboMode_rtsen = 1;
-		/* disable high power */
-		rtl8723a_odm_support_ability_clr(padapter, (u32)
-						 ~DYNAMIC_BB_DYNAMIC_TXPWR);
-		break;
-	case HT_IOT_PEER_REALTEK:
-		/* rtw_write16(padapter, 0x4cc, 0xffff); */
-		/* rtw_write16(padapter, 0x546, 0x01c0); */
-		/* disable high power */
-		rtl8723a_odm_support_ability_clr(padapter, (u32)
-						 ~DYNAMIC_BB_DYNAMIC_TXPWR);
-		break;
-	default:
-		pmlmeinfo->turboMode_cts2self = 0;
-		pmlmeinfo->turboMode_rtsen = 1;
-		break;
-	}
-}
-
-void update_capinfo23a(struct rtw_adapter *Adapter, u16 updateCap)
-{
-	struct mlme_ext_priv *pmlmeext = &Adapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (updateCap & cShortPreamble) {
-		/*  Short Preamble */
-		if (pmlmeinfo->preamble_mode != PREAMBLE_SHORT) {
-			/*  PREAMBLE_LONG or PREAMBLE_AUTO */
-			pmlmeinfo->preamble_mode = PREAMBLE_SHORT;
-			rtl8723a_ack_preamble(Adapter, true);
-		}
-	} else { /*  Long Preamble */
-		if (pmlmeinfo->preamble_mode != PREAMBLE_LONG) {
-			/*  PREAMBLE_SHORT or PREAMBLE_AUTO */
-			pmlmeinfo->preamble_mode = PREAMBLE_LONG;
-			rtl8723a_ack_preamble(Adapter, false);
-		}
-	}
-	if (updateCap & cIBSS) {
-		/* Filen: See 802.11-2007 p.91 */
-		pmlmeinfo->slotTime = NON_SHORT_SLOT_TIME;
-	} else {
-		/* Filen: See 802.11-2007 p.90 */
-		if (pmlmeext->cur_wireless_mode &
-		    (WIRELESS_11G | WIRELESS_11_24N)) {
-			if (updateCap & cShortSlotTime) { /*  Short Slot Time */
-				if (pmlmeinfo->slotTime != SHORT_SLOT_TIME)
-					pmlmeinfo->slotTime = SHORT_SLOT_TIME;
-			} else { /*  Long Slot Time */
-				if (pmlmeinfo->slotTime != NON_SHORT_SLOT_TIME)
-					pmlmeinfo->slotTime =
-						NON_SHORT_SLOT_TIME;
-			}
-		} else if (pmlmeext->cur_wireless_mode &
-			   (WIRELESS_11A | WIRELESS_11_5N)) {
-			pmlmeinfo->slotTime = SHORT_SLOT_TIME;
-		} else {
-			/* B Mode */
-			pmlmeinfo->slotTime = NON_SHORT_SLOT_TIME;
-		}
-	}
-	rtl8723a_set_slot_time(Adapter, pmlmeinfo->slotTime);
-}
-
-void update_wireless_mode23a(struct rtw_adapter *padapter)
-{
-	int ratelen, network_type = 0;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-	unsigned char *rate = cur_network->SupportedRates;
-
-	ratelen = rtw_get_rateset_len23a(cur_network->SupportedRates);
-
-	if ((pmlmeinfo->HT_info_enable) && (pmlmeinfo->HT_caps_enable))
-		pmlmeinfo->HT_enable = 1;
-
-	if (pmlmeext->cur_channel > 14) {
-		if (pmlmeinfo->HT_enable)
-			network_type = WIRELESS_11_5N;
-		network_type |= WIRELESS_11A;
-	} else {
-		if (pmlmeinfo->HT_enable)
-			network_type = WIRELESS_11_24N;
-
-		if (cckratesonly_included23a(rate, ratelen) == true)
-			network_type |= WIRELESS_11B;
-		else if (cckrates_included23a(rate, ratelen) == true)
-			network_type |= WIRELESS_11BG;
-		else
-			network_type |= WIRELESS_11G;
-	}
-
-	pmlmeext->cur_wireless_mode =
-		network_type & padapter->registrypriv.wireless_mode;
-
-	/* 0x0808 -> for CCK, 0x0a0a -> for OFDM */
-	/* change this value if having IOT issues. */
-	rtl8723a_set_resp_sifs(padapter, 0x08, 0x08, 0x0a, 0x0a);
-
-	if (pmlmeext->cur_wireless_mode & WIRELESS_11B)
-		update_mgnt_tx_rate23a(padapter, IEEE80211_CCK_RATE_1MB);
-	 else
-		update_mgnt_tx_rate23a(padapter, IEEE80211_OFDM_RATE_6MB);
-}
-
-void update_bmc_sta_support_rate23a(struct rtw_adapter *padapter, u32 mac_id)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (pmlmeext->cur_wireless_mode & WIRELESS_11B) {
-		/*  Only B, B/G, and B/G/N AP could use CCK rate */
-		memcpy((pmlmeinfo->FW_sta_info[mac_id].SupportedRates),
-		       rtw_basic_rate_cck, 4);
-	} else {
-		memcpy(pmlmeinfo->FW_sta_info[mac_id].SupportedRates,
-		       rtw_basic_rate_ofdm, 3);
-	}
-}
-
-int update_sta_support_rate23a(struct rtw_adapter *padapter, u8 *pvar_ie,
-			       uint var_ie_len, int cam_idx)
-{
-	int supportRateNum = 0;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	const u8 *p;
-
-	p = cfg80211_find_ie(WLAN_EID_SUPP_RATES, pvar_ie, var_ie_len);
-	if (!p)
-		return _FAIL;
-
-	memcpy(pmlmeinfo->FW_sta_info[cam_idx].SupportedRates, p + 2, p[1]);
-	supportRateNum = p[1];
-
-	p = cfg80211_find_ie(WLAN_EID_EXT_SUPP_RATES, pvar_ie, var_ie_len);
-	if (p)
-		memcpy(pmlmeinfo->FW_sta_info[cam_idx].SupportedRates +
-		       supportRateNum, p + 2, p[1]);
-	return _SUCCESS;
-}
-
-void process_addba_req23a(struct rtw_adapter *padapter,
-			  u8 *paddba_req, u8 *addr)
-{
-	struct sta_info *psta;
-	u16 tid, start_seq, param;
-	struct recv_reorder_ctrl *preorder_ctrl;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct ADDBA_request *preq = (struct ADDBA_request *)paddba_req;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	psta = rtw_get_stainfo23a(pstapriv, addr);
-
-	if (psta) {
-		start_seq = le16_to_cpu(preq->BA_starting_seqctrl) >> 4;
-
-		param = le16_to_cpu(preq->BA_para_set);
-		tid = (param >> 2) & 0x0f;
-
-		preorder_ctrl = &psta->recvreorder_ctrl[tid];
-
-		preorder_ctrl->indicate_seq = 0xffff;
-
-		preorder_ctrl->enable = (pmlmeinfo->bAcceptAddbaReq == true) ?
-			true : false;
-	}
-}
diff --git a/drivers/staging/rtl8723au/core/rtw_xmit.c b/drivers/staging/rtl8723au/core/rtw_xmit.c
deleted file mode 100644
index 3de40cfa5f3b..000000000000
--- a/drivers/staging/rtl8723au/core/rtw_xmit.c
+++ /dev/null
@@ -1,2341 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTW_XMIT_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <wifi.h>
-#include <osdep_intf.h>
-#include <linux/ip.h>
-#include <usb_ops.h>
-#include <rtl8723a_xmit.h>
-
-static void _init_txservq(struct tx_servq *ptxservq)
-{
-
-	INIT_LIST_HEAD(&ptxservq->tx_pending);
-	_rtw_init_queue23a(&ptxservq->sta_pending);
-	ptxservq->qcnt = 0;
-
-}
-
-void	_rtw_init_sta_xmit_priv23a(struct sta_xmit_priv *psta_xmitpriv)
-{
-
-	spin_lock_init(&psta_xmitpriv->lock);
-
-	/* for (i = 0 ; i < MAX_NUMBLKS; i++) */
-	/*	_init_txservq(&psta_xmitpriv->blk_q[i]); */
-
-	_init_txservq(&psta_xmitpriv->be_q);
-	_init_txservq(&psta_xmitpriv->bk_q);
-	_init_txservq(&psta_xmitpriv->vi_q);
-	_init_txservq(&psta_xmitpriv->vo_q);
-	INIT_LIST_HEAD(&psta_xmitpriv->legacy_dz);
-	INIT_LIST_HEAD(&psta_xmitpriv->apsd);
-
-}
-
-int _rtw_init_xmit_priv23a(struct xmit_priv *pxmitpriv,
-			   struct rtw_adapter *padapter)
-{
-	int i;
-	struct xmit_buf *pxmitbuf;
-	struct xmit_frame *pxframe;
-	int res = _SUCCESS;
-	u32 max_xmit_extbuf_size = MAX_XMIT_EXTBUF_SZ;
-	u32 num_xmit_extbuf = NR_XMIT_EXTBUFF;
-
-	spin_lock_init(&pxmitpriv->lock);
-	spin_lock_init(&pxmitpriv->lock_sctx);
-	sema_init(&pxmitpriv->xmit_sema, 0);
-	sema_init(&pxmitpriv->terminate_xmitthread_sema, 0);
-
-	pxmitpriv->adapter = padapter;
-
-	_rtw_init_queue23a(&pxmitpriv->be_pending);
-	_rtw_init_queue23a(&pxmitpriv->bk_pending);
-	_rtw_init_queue23a(&pxmitpriv->vi_pending);
-	_rtw_init_queue23a(&pxmitpriv->vo_pending);
-	_rtw_init_queue23a(&pxmitpriv->bm_pending);
-
-	_rtw_init_queue23a(&pxmitpriv->free_xmit_queue);
-
-	for (i = 0; i < NR_XMITFRAME; i++) {
-		pxframe = kzalloc(sizeof(struct xmit_frame), GFP_KERNEL);
-		if (!pxframe)
-			break;
-		INIT_LIST_HEAD(&pxframe->list);
-
-		pxframe->padapter = padapter;
-		pxframe->frame_tag = NULL_FRAMETAG;
-
-		list_add_tail(&pxframe->list,
-			      &pxmitpriv->free_xmit_queue.queue);
-	}
-
-	pxmitpriv->free_xmitframe_cnt = i;
-
-	pxmitpriv->frag_len = MAX_FRAG_THRESHOLD;
-
-	/* init xmit_buf */
-	_rtw_init_queue23a(&pxmitpriv->free_xmitbuf_queue);
-	INIT_LIST_HEAD(&pxmitpriv->xmitbuf_list);
-	_rtw_init_queue23a(&pxmitpriv->pending_xmitbuf_queue);
-
-	for (i = 0; i < NR_XMITBUFF; i++) {
-		pxmitbuf = kzalloc(sizeof(struct xmit_buf), GFP_KERNEL);
-		if (!pxmitbuf)
-			goto fail;
-		INIT_LIST_HEAD(&pxmitbuf->list);
-		INIT_LIST_HEAD(&pxmitbuf->list2);
-
-		pxmitbuf->padapter = padapter;
-
-		/* Tx buf allocation may fail sometimes, so sleep and retry. */
-		res = rtw_os_xmit_resource_alloc23a(padapter, pxmitbuf,
-						 (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ));
-		if (res == _FAIL) {
-			goto fail;
-		}
-
-		list_add_tail(&pxmitbuf->list,
-			      &pxmitpriv->free_xmitbuf_queue.queue);
-		list_add_tail(&pxmitbuf->list2,
-			      &pxmitpriv->xmitbuf_list);
-	}
-
-	pxmitpriv->free_xmitbuf_cnt = NR_XMITBUFF;
-
-	/* init xframe_ext queue,  the same count as extbuf  */
-	_rtw_init_queue23a(&pxmitpriv->free_xframe_ext_queue);
-
-	for (i = 0; i < num_xmit_extbuf; i++) {
-		pxframe = kzalloc(sizeof(struct xmit_frame), GFP_KERNEL);
-		if (!pxframe)
-			break;
-		INIT_LIST_HEAD(&pxframe->list);
-
-		pxframe->padapter = padapter;
-		pxframe->frame_tag = NULL_FRAMETAG;
-
-		pxframe->pkt = NULL;
-
-		pxframe->buf_addr = NULL;
-		pxframe->pxmitbuf = NULL;
-
-		pxframe->ext_tag = 1;
-
-		list_add_tail(&pxframe->list,
-			      &pxmitpriv->free_xframe_ext_queue.queue);
-	}
-	pxmitpriv->free_xframe_ext_cnt = i;
-
-	/*  Init xmit extension buff */
-	_rtw_init_queue23a(&pxmitpriv->free_xmit_extbuf_queue);
-	INIT_LIST_HEAD(&pxmitpriv->xmitextbuf_list);
-
-	for (i = 0; i < num_xmit_extbuf; i++) {
-		pxmitbuf = kzalloc(sizeof(struct xmit_buf), GFP_KERNEL);
-		if (!pxmitbuf)
-			goto fail;
-		INIT_LIST_HEAD(&pxmitbuf->list);
-		INIT_LIST_HEAD(&pxmitbuf->list2);
-
-		pxmitbuf->padapter = padapter;
-
-		/* Tx buf allocation may fail sometimes, so sleep and retry. */
-		res = rtw_os_xmit_resource_alloc23a(padapter, pxmitbuf,
-						 max_xmit_extbuf_size + XMITBUF_ALIGN_SZ);
-		if (res == _FAIL) {
-			goto exit;
-		}
-
-		list_add_tail(&pxmitbuf->list,
-			      &pxmitpriv->free_xmit_extbuf_queue.queue);
-		list_add_tail(&pxmitbuf->list2,
-			      &pxmitpriv->xmitextbuf_list);
-	}
-
-	pxmitpriv->free_xmit_extbuf_cnt = num_xmit_extbuf;
-
-	rtw_alloc_hwxmits23a(padapter);
-	rtw_init_hwxmits23a(pxmitpriv->hwxmits, pxmitpriv->hwxmit_entry);
-
-	for (i = 0; i < 4; i ++)
-		pxmitpriv->wmm_para_seq[i] = i;
-
-	sema_init(&pxmitpriv->tx_retevt, 0);
-
-	pxmitpriv->ack_tx = false;
-	mutex_init(&pxmitpriv->ack_tx_mutex);
-	rtw_sctx_init23a(&pxmitpriv->ack_tx_ops, 0);
-	tasklet_init(&padapter->xmitpriv.xmit_tasklet,
-		     (void(*)(unsigned long))rtl8723au_xmit_tasklet,
-		     (unsigned long)padapter);
-
-exit:
-
-	return res;
-fail:
-	goto exit;
-}
-
-void _rtw_free_xmit_priv23a(struct xmit_priv *pxmitpriv)
-{
-	struct rtw_adapter *padapter = pxmitpriv->adapter;
-	struct xmit_frame *pxframe, *ptmp;
-	struct xmit_buf *pxmitbuf, *ptmp2;
-
-	list_for_each_entry_safe(pxframe, ptmp,
-				 &pxmitpriv->free_xmit_queue.queue, list) {
-		list_del_init(&pxframe->list);
-		rtw_os_xmit_complete23a(padapter, pxframe);
-		kfree(pxframe);
-	}
-
-	list_for_each_entry_safe(pxmitbuf, ptmp2,
-				 &pxmitpriv->xmitbuf_list, list2) {
-		list_del_init(&pxmitbuf->list2);
-		rtw_os_xmit_resource_free23a(padapter, pxmitbuf);
-		kfree(pxmitbuf);
-	}
-
-	/* free xframe_ext queue,  the same count as extbuf  */
-	list_for_each_entry_safe(pxframe, ptmp,
-				 &pxmitpriv->free_xframe_ext_queue.queue,
-				 list) {
-		list_del_init(&pxframe->list);
-		rtw_os_xmit_complete23a(padapter, pxframe);
-		kfree(pxframe);
-	}
-
-	/*  free xmit extension buff */
-	list_for_each_entry_safe(pxmitbuf, ptmp2,
-				 &pxmitpriv->xmitextbuf_list, list2) {
-		list_del_init(&pxmitbuf->list2);
-		rtw_os_xmit_resource_free23a(padapter, pxmitbuf);
-		kfree(pxmitbuf);
-	}
-
-	rtw_free_hwxmits23a(padapter);
-	mutex_destroy(&pxmitpriv->ack_tx_mutex);
-}
-
-static void update_attrib_vcs_info(struct rtw_adapter *padapter, struct xmit_frame *pxmitframe)
-{
-	u32	sz;
-	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
-	struct sta_info	*psta = pattrib->psta;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	if (pattrib->psta) {
-		psta = pattrib->psta;
-	} else {
-		DBG_8723A("%s, call rtw_get_stainfo23a()\n", __func__);
-		psta = rtw_get_stainfo23a(&padapter->stapriv, &pattrib->ra[0]);
-	}
-
-	if (psta == NULL) {
-		DBG_8723A("%s, psta == NUL\n", __func__);
-		return;
-	}
-
-	if (!(psta->state &_FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n", __func__, psta->state);
-		return;
-	}
-
-	if (pattrib->nr_frags != 1)
-		sz = padapter->xmitpriv.frag_len;
-	else /* no frag */
-		sz = pattrib->last_txcmdsz;
-
-	/*  (1) RTS_Threshold is compared to the MPDU, not MSDU. */
-	/*  (2) If there are more than one frag in  this MSDU, only the first frag uses protection frame. */
-	/*		Other fragments are protected by previous fragment. */
-	/*		So we only need to check the length of first fragment. */
-	if (pmlmeext->cur_wireless_mode < WIRELESS_11_24N  || padapter->registrypriv.wifi_spec) {
-		if (sz > padapter->registrypriv.rts_thresh) {
-			pattrib->vcs_mode = RTS_CTS;
-		} else {
-			if (psta->rtsen)
-				pattrib->vcs_mode = RTS_CTS;
-			else if (psta->cts2self)
-				pattrib->vcs_mode = CTS_TO_SELF;
-			else
-				pattrib->vcs_mode = NONE_VCS;
-		}
-	} else {
-		while (true) {
-			/* IOT action */
-			if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_ATHEROS &&
-			    pattrib->ampdu_en &&
-			    padapter->securitypriv.dot11PrivacyAlgrthm ==
-			    WLAN_CIPHER_SUITE_CCMP) {
-				pattrib->vcs_mode = CTS_TO_SELF;
-				break;
-			}
-
-			/* check ERP protection */
-			if (psta->rtsen || psta->cts2self) {
-				if (psta->rtsen)
-					pattrib->vcs_mode = RTS_CTS;
-				else if (psta->cts2self)
-					pattrib->vcs_mode = CTS_TO_SELF;
-
-				break;
-			}
-
-			/* check HT op mode */
-			if (pattrib->ht_en) {
-				u8 HTOpMode = pmlmeinfo->HT_protection;
-
-				if ((pmlmeext->cur_bwmode && (HTOpMode == 2 || HTOpMode == 3)) ||
-				    (!pmlmeext->cur_bwmode && HTOpMode == 3)) {
-					pattrib->vcs_mode = RTS_CTS;
-					break;
-				}
-			}
-
-			/* check rts */
-			if (sz > padapter->registrypriv.rts_thresh) {
-				pattrib->vcs_mode = RTS_CTS;
-				break;
-			}
-
-			/* to do list: check MIMO power save condition. */
-
-			/* check AMPDU aggregation for TXOP */
-			if (pattrib->ampdu_en) {
-				pattrib->vcs_mode = RTS_CTS;
-				break;
-			}
-
-			pattrib->vcs_mode = NONE_VCS;
-			break;
-		}
-	}
-}
-
-static void update_attrib_phy_info(struct pkt_attrib *pattrib, struct sta_info *psta)
-{
-	/*if (psta->rtsen)
-		pattrib->vcs_mode = RTS_CTS;
-	else if (psta->cts2self)
-		pattrib->vcs_mode = CTS_TO_SELF;
-	else
-		pattrib->vcs_mode = NONE_VCS;*/
-
-	pattrib->mdata = 0;
-	pattrib->eosp = 0;
-	pattrib->triggered = 0;
-
-	/* qos_en, ht_en, init rate, , bw, ch_offset, sgi */
-	pattrib->qos_en = psta->qos_option;
-
-	pattrib->raid = psta->raid;
-	pattrib->ht_en = psta->htpriv.ht_option;
-	pattrib->bwmode = psta->htpriv.bwmode;
-	pattrib->ch_offset = psta->htpriv.ch_offset;
-	pattrib->sgi = psta->htpriv.sgi;
-	pattrib->ampdu_en = false;
-
-	pattrib->retry_ctrl = false;
-}
-
-u8 qos_acm23a(u8 acm_mask, u8 priority)
-{
-	u8 change_priority = priority;
-
-	switch (priority) {
-	case 0:
-	case 3:
-		if (acm_mask & BIT(1))
-			change_priority = 1;
-		break;
-	case 1:
-	case 2:
-		break;
-	case 4:
-	case 5:
-		if (acm_mask & BIT(2))
-			change_priority = 0;
-		break;
-	case 6:
-	case 7:
-		if (acm_mask & BIT(3))
-			change_priority = 5;
-		break;
-	default:
-		DBG_8723A("qos_acm23a(): invalid pattrib->priority: %d!!!\n",
-			  priority);
-		change_priority = 0;
-		break;
-	}
-
-	return change_priority;
-}
-
-static void set_qos(struct sk_buff *skb, struct pkt_attrib *pattrib)
-{
-	u8 *pframe = skb->data;
-	struct iphdr *ip_hdr;
-	u8 UserPriority = 0;
-
-	/*  get UserPriority from IP hdr */
-	if (pattrib->ether_type == ETH_P_IP) {
-		ip_hdr = (struct iphdr *)(pframe + ETH_HLEN);
-		UserPriority = ip_hdr->tos >> 5;
-	} else if (pattrib->ether_type == ETH_P_PAE) {
-		/*  "When priority processing of data frames is supported, */
-		/*  a STA's SME should send EAPOL-Key frames at the highest
-		    priority." */
-		UserPriority = 7;
-	}
-
-	pattrib->priority = UserPriority;
-	pattrib->hdrlen = sizeof(struct ieee80211_qos_hdr);
-	pattrib->type = IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_DATA;
-}
-
-static int update_attrib(struct rtw_adapter *padapter,
-			 struct sk_buff *skb, struct pkt_attrib *pattrib)
-{
-	struct sta_info *psta = NULL;
-	int bmcast;
-	struct sta_priv	*pstapriv = &padapter->stapriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	int res = _SUCCESS;
-	struct ethhdr *ehdr = (struct ethhdr *) skb->data;
-
-	pattrib->ether_type = ntohs(ehdr->h_proto);
-
-	ether_addr_copy(pattrib->dst, ehdr->h_dest);
-	ether_addr_copy(pattrib->src, ehdr->h_source);
-
-	pattrib->pctrl = 0;
-
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
-		ether_addr_copy(pattrib->ra, pattrib->dst);
-		ether_addr_copy(pattrib->ta, pattrib->src);
-	} else if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
-		ether_addr_copy(pattrib->ra, get_bssid(pmlmepriv));
-		ether_addr_copy(pattrib->ta, pattrib->src);
-	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		ether_addr_copy(pattrib->ra, pattrib->dst);
-		ether_addr_copy(pattrib->ta, get_bssid(pmlmepriv));
-	}
-
-	pattrib->pktlen = skb->len - ETH_HLEN;
-
-	if (pattrib->ether_type == ETH_P_IP) {
-		/*  The following is for DHCP and ARP packet, we use cck1M
-		    to tx these packets and let LPS awake some time */
-		/*  to prevent DHCP protocol fail */
-		pattrib->dhcp_pkt = 0;
-		/* MINIMUM_DHCP_PACKET_SIZE) { */
-		if (pattrib->pktlen > 282 + 24) {
-			if (pattrib->ether_type == ETH_P_IP) {/*  IP header */
-				u8 *pframe = skb->data;
-
-				pframe += ETH_HLEN;
-
-				if ((pframe[21] == 68 && pframe[23] == 67) ||
-				    (pframe[21] == 67 && pframe[23] == 68)) {
-					/*  68 : UDP BOOTP client */
-					/*  67 : UDP BOOTP server */
-					RT_TRACE(_module_rtl871x_xmit_c_,
-						 _drv_err_,
-						 "======================update_attrib: get DHCP Packet\n");
-					pattrib->dhcp_pkt = 1;
-				}
-			}
-		}
-	} else if (pattrib->ether_type == ETH_P_PAE) {
-		DBG_8723A_LEVEL(_drv_always_, "send eapol packet\n");
-	}
-
-	if ((pattrib->ether_type == ETH_P_PAE) || (pattrib->dhcp_pkt == 1)) {
-		rtw_set_scan_deny(padapter, 3000);
-	}
-
-	/*  If EAPOL , ARP , OR DHCP packet, driver must be in active mode. */
-	if ((pattrib->ether_type == ETH_P_ARP) ||
-	    (pattrib->ether_type == ETH_P_PAE) || (pattrib->dhcp_pkt == 1)) {
-		rtw_lps_ctrl_wk_cmd23a(padapter, LPS_CTRL_SPECIAL_PACKET, 1);
-	}
-
-	bmcast = is_multicast_ether_addr(pattrib->ra);
-
-	/*  get sta_info */
-	if (bmcast) {
-		psta = rtw_get_bcmc_stainfo23a(padapter);
-	} else {
-		psta = rtw_get_stainfo23a(pstapriv, pattrib->ra);
-		if (psta == NULL) { /*  if we cannot get psta => drrp the pkt */
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_alert_,
-				 "update_attrib => get sta_info fail, ra:%pM\n",
-				 pattrib->ra);
-			res = _FAIL;
-			goto exit;
-		} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) &&
-			   (!(psta->state & _FW_LINKED))) {
-			res = _FAIL;
-			goto exit;
-		}
-	}
-
-	if (psta) {
-		pattrib->mac_id = psta->mac_id;
-		/* DBG_8723A("%s ==> mac_id(%d)\n", __func__, pattrib->mac_id); */
-		pattrib->psta = psta;
-	} else {
-		/*  if we cannot get psta => drop the pkt */
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_alert_,
-			 "update_attrib => get sta_info fail, ra:%pM\n",
-			 pattrib->ra);
-		res = _FAIL;
-		goto exit;
-	}
-
-	pattrib->ack_policy = 0;
-	/*  get ether_hdr_len */
-
-	/* pattrib->ether_type == 0x8100) ? (14 + 4): 14; vlan tag */
-	pattrib->pkt_hdrlen = ETH_HLEN;
-
-	pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr);
-	pattrib->type = IEEE80211_FTYPE_DATA;
-	pattrib->priority = 0;
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE | WIFI_ADHOC_STATE |
-			  WIFI_ADHOC_MASTER_STATE)) {
-		if (psta->qos_option)
-			set_qos(skb, pattrib);
-	} else {
-		if (pmlmepriv->qos_option) {
-			set_qos(skb, pattrib);
-
-			if (pmlmepriv->acm_mask != 0) {
-				pattrib->priority = qos_acm23a(pmlmepriv->acm_mask,
-							    pattrib->priority);
-			}
-		}
-	}
-
-	if (psta->ieee8021x_blocked == true) {
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-			 "psta->ieee8021x_blocked == true\n");
-
-		pattrib->encrypt = 0;
-
-		if ((pattrib->ether_type != ETH_P_PAE) &&
-		    !check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-				 "psta->ieee8021x_blocked == true,  pattrib->ether_type(%.4x) != 0x888e\n",
-				 pattrib->ether_type);
-			res = _FAIL;
-			goto exit;
-		}
-	} else {
-		GET_ENCRY_ALGO(psecuritypriv, psta, pattrib->encrypt, bmcast);
-
-		switch (psecuritypriv->dot11AuthAlgrthm) {
-		case dot11AuthAlgrthm_Open:
-		case dot11AuthAlgrthm_Shared:
-		case dot11AuthAlgrthm_Auto:
-			pattrib->key_idx =
-				(u8)psecuritypriv->dot11PrivacyKeyIndex;
-			break;
-		case dot11AuthAlgrthm_8021X:
-			if (bmcast)
-				pattrib->key_idx =
-					(u8)psecuritypriv->dot118021XGrpKeyid;
-			else
-				pattrib->key_idx = 0;
-			break;
-		default:
-			pattrib->key_idx = 0;
-			break;
-		}
-
-	}
-
-	switch (pattrib->encrypt) {
-	case WLAN_CIPHER_SUITE_WEP40:
-	case WLAN_CIPHER_SUITE_WEP104:
-		pattrib->iv_len = IEEE80211_WEP_IV_LEN;
-		pattrib->icv_len = IEEE80211_WEP_ICV_LEN;
-		break;
-
-	case WLAN_CIPHER_SUITE_TKIP:
-		pattrib->iv_len = IEEE80211_TKIP_IV_LEN;
-		pattrib->icv_len = IEEE80211_TKIP_ICV_LEN;
-
-		if (!padapter->securitypriv.busetkipkey) {
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-				 "padapter->securitypriv.busetkipkey(%d) == false drop packet\n",
-				 padapter->securitypriv.busetkipkey);
-			res = _FAIL;
-			goto exit;
-		}
-
-		break;
-	case WLAN_CIPHER_SUITE_CCMP:
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "pattrib->encrypt =%d (WLAN_CIPHER_SUITE_CCMP)\n",
-			 pattrib->encrypt);
-		pattrib->iv_len = IEEE80211_CCMP_HDR_LEN;
-		pattrib->icv_len = IEEE80211_CCMP_MIC_LEN;
-		break;
-
-	default:
-		pattrib->iv_len = 0;
-		pattrib->icv_len = 0;
-		break;
-	}
-
-	RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-		 "update_attrib: encrypt =%d\n", pattrib->encrypt);
-
-	if (pattrib->encrypt && !psecuritypriv->hw_decrypted) {
-		pattrib->bswenc = true;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-			 "update_attrib: encrypt =%d bswenc = true\n",
-			 pattrib->encrypt);
-	} else {
-		pattrib->bswenc = false;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "update_attrib: bswenc = false\n");
-	}
-	update_attrib_phy_info(pattrib, psta);
-
-exit:
-
-	return res;
-}
-
-static int xmitframe_addmic(struct rtw_adapter *padapter,
-			    struct xmit_frame *pxmitframe) {
-	struct mic_data micdata;
-	struct sta_info *stainfo;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	int curfragnum, length;
-	u8 *pframe, *payload, mic[8];
-	u8 priority[4]= {0x0, 0x0, 0x0, 0x0};
-	u8 hw_hdr_offset = 0;
-	int bmcst = is_multicast_ether_addr(pattrib->ra);
-
-	if (pattrib->psta) {
-		stainfo = pattrib->psta;
-	} else {
-		DBG_8723A("%s, call rtw_get_stainfo23a()\n", __func__);
-		stainfo = rtw_get_stainfo23a(&padapter->stapriv, &pattrib->ra[0]);
-	}
-
-	if (!stainfo) {
-		DBG_8723A("%s, psta == NUL\n", __func__);
-		return _FAIL;
-	}
-
-	if (!(stainfo->state &_FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n",
-			  __func__, stainfo->state);
-		return _FAIL;
-	}
-
-	hw_hdr_offset = TXDESC_OFFSET;
-
-	if (pattrib->encrypt == WLAN_CIPHER_SUITE_TKIP) {
-		/* encode mic code */
-		if (stainfo) {
-			u8 null_key[16]={0x0, 0x0, 0x0, 0x0,
-					 0x0, 0x0, 0x0, 0x0,
-					 0x0, 0x0, 0x0, 0x0,
-					 0x0, 0x0, 0x0, 0x0};
-
-			pframe = pxmitframe->buf_addr + hw_hdr_offset;
-
-			if (bmcst) {
-				if (!memcmp(psecuritypriv->dot118021XGrptxmickey[psecuritypriv->dot118021XGrpKeyid].skey, null_key, 16)) {
-					return _FAIL;
-				}
-				/* start to calculate the mic code */
-				rtw_secmicsetkey23a(&micdata, psecuritypriv->dot118021XGrptxmickey[psecuritypriv->dot118021XGrpKeyid].skey);
-			} else {
-				if (!memcmp(&stainfo->dot11tkiptxmickey.skey[0],
-					    null_key, 16)) {
-					return _FAIL;
-				}
-				/* start to calculate the mic code */
-				rtw_secmicsetkey23a(&micdata, &stainfo->dot11tkiptxmickey.skey[0]);
-			}
-
-			if (pframe[1] & 1) {   /* ToDS == 1 */
-				/* DA */
-				rtw_secmicappend23a(&micdata, &pframe[16], 6);
-				if (pframe[1] & 2)  /* From Ds == 1 */
-					rtw_secmicappend23a(&micdata,
-							 &pframe[24], 6);
-				else
-					rtw_secmicappend23a(&micdata,
-							 &pframe[10], 6);
-			} else {	/* ToDS == 0 */
-				/* DA */
-				rtw_secmicappend23a(&micdata, &pframe[4], 6);
-				if (pframe[1] & 2)  /* From Ds == 1 */
-					rtw_secmicappend23a(&micdata,
-							 &pframe[16], 6);
-				else
-					rtw_secmicappend23a(&micdata,
-							 &pframe[10], 6);
-			}
-
-			/* if (pmlmepriv->qos_option == 1) */
-			if (pattrib->qos_en)
-				priority[0] = (u8)pxmitframe->attrib.priority;
-
-			rtw_secmicappend23a(&micdata, &priority[0], 4);
-
-			payload = pframe;
-
-			for (curfragnum = 0; curfragnum < pattrib->nr_frags;
-			     curfragnum++) {
-				payload = PTR_ALIGN(payload, 4);
-				RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-					 "=== curfragnum =%d, pframe = 0x%.2x, 0x%.2x, 0x%.2x, 0x%.2x, 0x%.2x, 0x%.2x, 0x%.2x, 0x%.2x,!!!\n",
-					 curfragnum, *payload, *(payload + 1),
-					 *(payload + 2), *(payload + 3),
-					 *(payload + 4), *(payload + 5),
-					 *(payload + 6), *(payload + 7));
-
-				payload = payload + pattrib->hdrlen +
-					pattrib->iv_len;
-				RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-					 "curfragnum =%d pattrib->hdrlen =%d pattrib->iv_len =%d\n",
-					 curfragnum,
-					 pattrib->hdrlen, pattrib->iv_len);
-				if ((curfragnum + 1) == pattrib->nr_frags) {
-					length = pattrib->last_txcmdsz -
-						pattrib->hdrlen -
-						pattrib->iv_len -
-						((pattrib->bswenc) ?
-						 pattrib->icv_len : 0);
-					rtw_secmicappend23a(&micdata, payload,
-							 length);
-					payload = payload + length;
-				} else {
-					length = pxmitpriv->frag_len -
-						pattrib->hdrlen -
-						pattrib->iv_len -
-						((pattrib->bswenc) ?
-						 pattrib->icv_len : 0);
-					rtw_secmicappend23a(&micdata, payload,
-							 length);
-					payload = payload + length +
-						pattrib->icv_len;
-					RT_TRACE(_module_rtl871x_xmit_c_,
-						 _drv_err_,
-						 "curfragnum =%d length =%d pattrib->icv_len =%d\n",
-						 curfragnum, length,
-						 pattrib->icv_len);
-				}
-			}
-			rtw_secgetmic23a(&micdata, &mic[0]);
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-				 "xmitframe_addmic: before add mic code!!\n");
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-				 "xmitframe_addmic: pattrib->last_txcmdsz =%d!!!\n",
-				 pattrib->last_txcmdsz);
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-				 "xmitframe_addmic: mic[0]= 0x%.2x , mic[1]=0x%.2x , mic[2]= 0x%.2x , mic[3]= 0x%.2x\nmic[4]= 0x%.2x , mic[5]= 0x%.2x , mic[6]= 0x%.2x , mic[7]= 0x%.2x !!!!\n",
-				 mic[0], mic[1], mic[2], mic[3],
-				 mic[4], mic[5], mic[6], mic[7]);
-			/* add mic code  and add the mic code length
-			   in last_txcmdsz */
-
-			memcpy(payload, &mic[0], 8);
-			pattrib->last_txcmdsz += 8;
-
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-				 "======== last pkt ========\n");
-			payload = payload - pattrib->last_txcmdsz + 8;
-			for (curfragnum = 0; curfragnum < pattrib->last_txcmdsz;
-			     curfragnum = curfragnum + 8) {
-				RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-					 "%.2x,  %.2x,  %.2x,  %.2x,  %.2x,  %.2x,  %.2x,  %.2x\n",
-					 *(payload + curfragnum),
-					 *(payload + curfragnum + 1),
-					 *(payload + curfragnum + 2),
-					 *(payload + curfragnum + 3),
-					 *(payload + curfragnum + 4),
-					 *(payload + curfragnum + 5),
-					 *(payload + curfragnum + 6),
-					 *(payload + curfragnum + 7));
-			}
-		} else {
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-				 "xmitframe_addmic: rtw_get_stainfo23a ==NULL!!!\n");
-		}
-	}
-
-	return _SUCCESS;
-}
-
-static int xmitframe_swencrypt(struct rtw_adapter *padapter,
-			       struct xmit_frame *pxmitframe)
-{
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-
-	/* if ((psecuritypriv->sw_encrypt)||(pattrib->bswenc)) */
-	if (pattrib->bswenc) {
-		/* DBG_8723A("start xmitframe_swencrypt\n"); */
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_alert_,
-			 "### xmitframe_swencrypt\n");
-		switch (pattrib->encrypt) {
-		case WLAN_CIPHER_SUITE_WEP40:
-		case WLAN_CIPHER_SUITE_WEP104:
-			rtw_wep_encrypt23a(padapter, pxmitframe);
-			break;
-		case WLAN_CIPHER_SUITE_TKIP:
-			rtw_tkip_encrypt23a(padapter, pxmitframe);
-			break;
-		case WLAN_CIPHER_SUITE_CCMP:
-			rtw_aes_encrypt23a(padapter, pxmitframe);
-			break;
-		default:
-				break;
-		}
-
-	} else {
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_notice_,
-			 "### xmitframe_hwencrypt\n");
-	}
-
-	return _SUCCESS;
-}
-
-static int rtw_make_wlanhdr(struct rtw_adapter *padapter, u8 *hdr,
-			    struct pkt_attrib *pattrib)
-{
-	struct ieee80211_hdr *pwlanhdr = (struct ieee80211_hdr *)hdr;
-	struct ieee80211_qos_hdr *qoshdr;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	u8 qos_option = false;
-	int res = _SUCCESS;
-
-	struct sta_info *psta;
-
-	int bmcst = is_multicast_ether_addr(pattrib->ra);
-
-	if (pattrib->psta) {
-		psta = pattrib->psta;
-	} else {
-		DBG_8723A("%s, call rtw_get_stainfo23a()\n", __func__);
-		if (bmcst) {
-			psta = rtw_get_bcmc_stainfo23a(padapter);
-		} else {
-			psta = rtw_get_stainfo23a(&padapter->stapriv, pattrib->ra);
-		}
-	}
-
-	if (psta == NULL) {
-		DBG_8723A("%s, psta == NUL\n", __func__);
-		return _FAIL;
-	}
-
-	if (!(psta->state &_FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n", __func__, psta->state);
-		return _FAIL;
-	}
-
-	memset(hdr, 0, WLANHDR_OFFSET);
-
-	pwlanhdr->frame_control = cpu_to_le16(pattrib->type);
-
-	if (pattrib->type & IEEE80211_FTYPE_DATA) {
-		if (check_fwstate(pmlmepriv,  WIFI_STATION_STATE)) {
-			/* to_ds = 1, fr_ds = 0; */
-			/* Data transfer to AP */
-			pwlanhdr->frame_control |=
-				cpu_to_le16(IEEE80211_FCTL_TODS);
-			ether_addr_copy(pwlanhdr->addr1, get_bssid(pmlmepriv));
-			ether_addr_copy(pwlanhdr->addr2, pattrib->src);
-			ether_addr_copy(pwlanhdr->addr3, pattrib->dst);
-
-			if (pmlmepriv->qos_option)
-				qos_option = true;
-
-		} else if (check_fwstate(pmlmepriv,  WIFI_AP_STATE)) {
-			/* to_ds = 0, fr_ds = 1; */
-			pwlanhdr->frame_control |=
-				cpu_to_le16(IEEE80211_FCTL_FROMDS);
-			ether_addr_copy(pwlanhdr->addr1, pattrib->dst);
-			ether_addr_copy(pwlanhdr->addr2, get_bssid(pmlmepriv));
-			ether_addr_copy(pwlanhdr->addr3, pattrib->src);
-
-			if (psta->qos_option)
-				qos_option = true;
-		} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
-			   check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
-			ether_addr_copy(pwlanhdr->addr1, pattrib->dst);
-			ether_addr_copy(pwlanhdr->addr2, pattrib->src);
-			ether_addr_copy(pwlanhdr->addr3, get_bssid(pmlmepriv));
-
-			if (psta->qos_option)
-				qos_option = true;
-		}
-		else {
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-				 "fw_state:%x is not allowed to xmit frame\n",
-				 get_fwstate(pmlmepriv));
-			res = _FAIL;
-			goto exit;
-		}
-		if (pattrib->mdata)
-			pwlanhdr->frame_control |=
-				cpu_to_le16(IEEE80211_FCTL_MOREDATA);
-		if (pattrib->encrypt)
-			pwlanhdr->frame_control |=
-				cpu_to_le16(IEEE80211_FCTL_PROTECTED);
-		if (qos_option) {
-			qoshdr = (struct ieee80211_qos_hdr *)hdr;
-
-			qoshdr->qos_ctrl = cpu_to_le16(
-				pattrib->priority & IEEE80211_QOS_CTL_TID_MASK);
-
-			qoshdr->qos_ctrl |= cpu_to_le16(
-				(pattrib->ack_policy << 5) &
-				IEEE80211_QOS_CTL_ACK_POLICY_MASK);
-
-			if (pattrib->eosp)
-				qoshdr->qos_ctrl |=
-					cpu_to_le16(IEEE80211_QOS_CTL_EOSP);
-		}
-		/* TODO: fill HT Control Field */
-
-		/* Update Seq Num will be handled by f/w */
-		if (psta) {
-			psta->sta_xmitpriv.txseq_tid[pattrib->priority]++;
-			psta->sta_xmitpriv.txseq_tid[pattrib->priority] &= 0xFFF;
-			pattrib->seqnum = psta->sta_xmitpriv.txseq_tid[pattrib->priority];
-			/* We dont need to worry about frag bits here */
-			pwlanhdr->seq_ctrl = cpu_to_le16(IEEE80211_SN_TO_SEQ(
-							      pattrib->seqnum));
-			/* check if enable ampdu */
-			if (pattrib->ht_en && psta->htpriv.ampdu_enable) {
-				if (pattrib->priority >= 16)
-					printk(KERN_WARNING "%s: Invalid "
-					       "pattrib->priority %i\n",
-					       __func__, pattrib->priority);
-				if (psta->htpriv.agg_enable_bitmap &
-				    BIT(pattrib->priority))
-					pattrib->ampdu_en = true;
-			}
-			/* re-check if enable ampdu by BA_starting_seqctrl */
-			if (pattrib->ampdu_en) {
-				u16 tx_seq;
-
-				tx_seq = psta->BA_starting_seqctrl[pattrib->priority & 0x0f];
-
-				/* check BA_starting_seqctrl */
-				if (SN_LESS(pattrib->seqnum, tx_seq)) {
-					/* DBG_8723A("tx ampdu seqnum(%d) < tx_seq(%d)\n", pattrib->seqnum, tx_seq); */
-					pattrib->ampdu_en = false;/* AGG BK */
-				} else if (SN_EQUAL(pattrib->seqnum, tx_seq)) {
-					psta->BA_starting_seqctrl[pattrib->priority & 0x0f] = (tx_seq+1)&0xfff;
-					pattrib->ampdu_en = true;/* AGG EN */
-				} else {
-					/* DBG_8723A("tx ampdu over run\n"); */
-					psta->BA_starting_seqctrl[pattrib->priority & 0x0f] = (pattrib->seqnum+1)&0xfff;
-					pattrib->ampdu_en = true;/* AGG EN */
-				}
-			}
-		}
-	}
-exit:
-	return res;
-}
-
-s32 rtw_txframes_pending23a(struct rtw_adapter *padapter)
-{
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	return (!list_empty(&pxmitpriv->be_pending.queue)) ||
-		(!list_empty(&pxmitpriv->bk_pending.queue)) ||
-		(!list_empty(&pxmitpriv->vi_pending.queue)) ||
-		(!list_empty(&pxmitpriv->vo_pending.queue));
-}
-
-s32 rtw_txframes_sta_ac_pending23a(struct rtw_adapter *padapter,
-				struct pkt_attrib *pattrib)
-{
-	struct sta_info *psta;
-	struct tx_servq *ptxservq;
-	int priority = pattrib->priority;
-
-	if (pattrib->psta) {
-		psta = pattrib->psta;
-	} else {
-		DBG_8723A("%s, call rtw_get_stainfo23a()\n", __func__);
-		psta = rtw_get_stainfo23a(&padapter->stapriv, &pattrib->ra[0]);
-	}
-	if (psta == NULL) {
-		DBG_8723A("%s, psta == NUL\n", __func__);
-		return 0;
-	}
-	if (!(psta->state &_FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n", __func__,
-			  psta->state);
-		return 0;
-	}
-	switch (priority) {
-	case 1:
-	case 2:
-		ptxservq = &psta->sta_xmitpriv.bk_q;
-		break;
-	case 4:
-	case 5:
-		ptxservq = &psta->sta_xmitpriv.vi_q;
-		break;
-	case 6:
-	case 7:
-		ptxservq = &psta->sta_xmitpriv.vo_q;
-		break;
-	case 0:
-	case 3:
-	default:
-		ptxservq = &psta->sta_xmitpriv.be_q;
-		break;
-	}
-	return ptxservq->qcnt;
-}
-
-/* Logical Link Control(LLC) SubNetwork Attachment Point(SNAP) header
- * IEEE LLC/SNAP header contains 8 octets
- * First 3 octets comprise the LLC portion
- * SNAP portion, 5 octets, is divided into two fields:
- *	Organizationally Unique Identifier(OUI), 3 octets,
- *	type, defined by that organization, 2 octets.
- */
-static int rtw_put_snap(u8 *data, u16 h_proto)
-{
-	if (h_proto == ETH_P_IPX || h_proto == ETH_P_AARP)
-		ether_addr_copy(data, bridge_tunnel_header);
-	else
-		ether_addr_copy(data, rfc1042_header);
-
-	data += ETH_ALEN;
-	put_unaligned_be16(h_proto, data);
-	return ETH_ALEN + sizeof(u16);
-}
-
-/*
-
-This sub-routine will perform all the following:
-
-1. remove 802.3 header.
-2. create wlan_header, based on the info in pxmitframe
-3. append sta's iv/ext-iv
-4. append LLC
-5. move frag chunk from pframe to pxmitframe->mem
-6. apply sw-encrypt, if necessary.
-
-*/
-int rtw_xmitframe_coalesce23a(struct rtw_adapter *padapter, struct sk_buff *skb,
-			      struct xmit_frame *pxmitframe)
-{
-	struct sta_info *psta;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-	struct ieee80211_hdr *hdr;
-	s32 frg_inx, frg_len, mpdu_len, llc_sz, mem_sz;
-	u8 *pframe, *mem_start;
-	u8 hw_hdr_offset;
-	u8 *pbuf_start;
-	u8 *pdata = skb->data;
-	int data_len = skb->len;
-	s32 bmcst = is_multicast_ether_addr(pattrib->ra);
-	int res = _SUCCESS;
-
-	if (pattrib->psta)
-		psta = pattrib->psta;
-	else {
-		DBG_8723A("%s, call rtw_get_stainfo23a()\n", __func__);
-		psta = rtw_get_stainfo23a(&padapter->stapriv, pattrib->ra);
-	}
-
-	if (!psta) {
-		DBG_8723A("%s, psta == NUL\n", __func__);
-		return _FAIL;
-	}
-
-	if (!(psta->state &_FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n",
-			  __func__, psta->state);
-		return _FAIL;
-	}
-
-	if (!pxmitframe->buf_addr) {
-		DBG_8723A("==> %s buf_addr == NULL\n", __func__);
-		return _FAIL;
-	}
-
-	pbuf_start = pxmitframe->buf_addr;
-
-	hw_hdr_offset = TXDESC_OFFSET;
-
-	mem_start = pbuf_start + hw_hdr_offset;
-
-	if (rtw_make_wlanhdr(padapter, mem_start, pattrib) == _FAIL) {
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-			 "%s: rtw_make_wlanhdr fail; drop pkt\n", __func__);
-		res = _FAIL;
-		goto exit;
-	}
-
-	pdata += pattrib->pkt_hdrlen;
-	data_len -= pattrib->pkt_hdrlen;
-
-	frg_inx = 0;
-	frg_len = pxmitpriv->frag_len - 4;/* 2346-4 = 2342 */
-
-	while (1) {
-		llc_sz = 0;
-
-		mpdu_len = frg_len;
-
-		pframe = mem_start;
-		hdr = (struct ieee80211_hdr *)mem_start;
-
-		pframe += pattrib->hdrlen;
-		mpdu_len -= pattrib->hdrlen;
-
-		/* adding icv, if necessary... */
-		if (pattrib->iv_len) {
-			if (psta) {
-				switch (pattrib->encrypt) {
-				case WLAN_CIPHER_SUITE_WEP40:
-				case WLAN_CIPHER_SUITE_WEP104:
-					WEP_IV(pattrib->iv, psta->dot11txpn,
-					       pattrib->key_idx);
-					break;
-				case WLAN_CIPHER_SUITE_TKIP:
-					if (bmcst)
-						TKIP_IV(pattrib->iv,
-							psta->dot11txpn,
-							pattrib->key_idx);
-					else
-						TKIP_IV(pattrib->iv,
-							psta->dot11txpn, 0);
-					break;
-				case WLAN_CIPHER_SUITE_CCMP:
-					if (bmcst)
-						AES_IV(pattrib->iv,
-						       psta->dot11txpn,
-						       pattrib->key_idx);
-					else
-						AES_IV(pattrib->iv,
-						       psta->dot11txpn, 0);
-					break;
-				}
-			}
-
-			memcpy(pframe, pattrib->iv, pattrib->iv_len);
-
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_notice_,
-				 "rtw_xmiaframe_coalesce23a: keyid =%d pattrib->iv[3]=%.2x pframe =%.2x %.2x %.2x %.2x\n",
-				 padapter->securitypriv.dot11PrivacyKeyIndex,
-				 pattrib->iv[3], *pframe, *(pframe+1),
-				 *(pframe+2), *(pframe+3));
-			pframe += pattrib->iv_len;
-			mpdu_len -= pattrib->iv_len;
-		}
-		if (frg_inx == 0) {
-			llc_sz = rtw_put_snap(pframe, pattrib->ether_type);
-			pframe += llc_sz;
-			mpdu_len -= llc_sz;
-		}
-
-		if (pattrib->icv_len > 0 && pattrib->bswenc)
-			mpdu_len -= pattrib->icv_len;
-
-		if (bmcst)
-			/*  don't do fragment to broadcast/multicast packets */
-			mem_sz = min_t(s32, data_len, pattrib->pktlen);
-		else
-			mem_sz = min_t(s32, data_len, mpdu_len);
-
-		memcpy(pframe, pdata, mem_sz);
-
-		pframe += mem_sz;
-		pdata += mem_sz;
-		data_len -= mem_sz;
-
-		if ((pattrib->icv_len >0) && (pattrib->bswenc)) {
-			memcpy(pframe, pattrib->icv, pattrib->icv_len);
-			pframe += pattrib->icv_len;
-		}
-
-		frg_inx++;
-
-		if (bmcst || data_len <= 0) {
-			pattrib->nr_frags = frg_inx;
-
-			pattrib->last_txcmdsz = pattrib->hdrlen +
-						pattrib->iv_len +
-						((pattrib->nr_frags == 1) ?
-						llc_sz : 0) +
-						((pattrib->bswenc) ?
-						pattrib->icv_len : 0) + mem_sz;
-			hdr->frame_control &=
-				~cpu_to_le16(IEEE80211_FCTL_MOREFRAGS);
-
-			break;
-		} else {
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-				 "%s: There're still something in packet!\n",
-				 __func__);
-		}
-		hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_MOREFRAGS);
-
-		mem_start = PTR_ALIGN(pframe, 4) + hw_hdr_offset;
-		memcpy(mem_start, pbuf_start + hw_hdr_offset, pattrib->hdrlen);
-	}
-
-	if (xmitframe_addmic(padapter, pxmitframe) == _FAIL) {
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-			 "xmitframe_addmic(padapter, pxmitframe) == _FAIL\n");
-		DBG_8723A("xmitframe_addmic(padapter, pxmitframe) == _FAIL\n");
-		res = _FAIL;
-		goto exit;
-	}
-
-	xmitframe_swencrypt(padapter, pxmitframe);
-
-	if (bmcst == false)
-		update_attrib_vcs_info(padapter, pxmitframe);
-	else
-		pattrib->vcs_mode = NONE_VCS;
-
-exit:
-	return res;
-}
-
-void rtw_update_protection23a(struct rtw_adapter *padapter, u8 *ie, uint ie_len)
-{
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	uint protection;
-	const u8 *p;
-
-	switch (pregistrypriv->vrtl_carrier_sense) {
-	case DISABLE_VCS:
-		pxmitpriv->vcs = NONE_VCS;
-		break;
-	case ENABLE_VCS:
-		break;
-	case AUTO_VCS:
-	default:
-		p = cfg80211_find_ie(WLAN_EID_ERP_INFO, ie, ie_len);
-		if (!p)
-			pxmitpriv->vcs = NONE_VCS;
-		else {
-			protection = (*(p + 2)) & BIT(1);
-			if (protection) {
-				if (pregistrypriv->vcs_type == RTS_CTS)
-					pxmitpriv->vcs = RTS_CTS;
-				else
-					pxmitpriv->vcs = CTS_TO_SELF;
-			} else {
-				pxmitpriv->vcs = NONE_VCS;
-			}
-		}
-		break;
-	}
-}
-
-void rtw_count_tx_stats23a(struct rtw_adapter *padapter, struct xmit_frame *pxmitframe, int sz)
-{
-	struct sta_info *psta = NULL;
-	struct stainfo_stats *pstats = NULL;
-	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
-	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
-
-	if (pxmitframe->frame_tag == DATA_FRAMETAG) {
-		pxmitpriv->tx_bytes += sz;
-		pmlmepriv->LinkDetectInfo.NumTxOkInPeriod++;
-
-		psta = pxmitframe->attrib.psta;
-		if (psta) {
-			pstats = &psta->sta_stats;
-			pstats->tx_pkts++;
-			pstats->tx_bytes += sz;
-		}
-	}
-}
-
-struct xmit_buf *rtw_alloc_xmitbuf23a_ext(struct xmit_priv *pxmitpriv)
-{
-	unsigned long irqL;
-	struct xmit_buf *pxmitbuf =  NULL;
-	struct list_head *phead;
-	struct rtw_queue *pfree_queue = &pxmitpriv->free_xmit_extbuf_queue;
-
-	spin_lock_irqsave(&pfree_queue->lock, irqL);
-
-	phead = get_list_head(pfree_queue);
-
-	if (!list_empty(phead)) {
-		pxmitbuf = list_first_entry(phead, struct xmit_buf, list);
-
-		list_del_init(&pxmitbuf->list);
-
-		pxmitpriv->free_xmit_extbuf_cnt--;
-		pxmitbuf->priv_data = NULL;
-		pxmitbuf->ext_tag = true;
-
-		if (pxmitbuf->sctx) {
-			DBG_8723A("%s pxmitbuf->sctx is not NULL\n", __func__);
-			rtw23a_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_BUF_ALLOC);
-		}
-	}
-
-	spin_unlock_irqrestore(&pfree_queue->lock, irqL);
-
-	return pxmitbuf;
-}
-
-int rtw_free_xmitbuf_ext23a(struct xmit_priv *pxmitpriv,
-			    struct xmit_buf *pxmitbuf)
-{
-	unsigned long irqL;
-	struct rtw_queue *pfree_queue = &pxmitpriv->free_xmit_extbuf_queue;
-
-	if (pxmitbuf == NULL)
-		return _FAIL;
-
-	spin_lock_irqsave(&pfree_queue->lock, irqL);
-
-	list_del_init(&pxmitbuf->list);
-
-	list_add_tail(&pxmitbuf->list, get_list_head(pfree_queue));
-	pxmitpriv->free_xmit_extbuf_cnt++;
-
-	spin_unlock_irqrestore(&pfree_queue->lock, irqL);
-
-	return _SUCCESS;
-}
-
-struct xmit_buf *rtw_alloc_xmitbuf23a(struct xmit_priv *pxmitpriv)
-{
-	unsigned long irqL;
-	struct xmit_buf *pxmitbuf =  NULL;
-	struct list_head *phead;
-	struct rtw_queue *pfree_xmitbuf_queue = &pxmitpriv->free_xmitbuf_queue;
-
-	/* DBG_8723A("+rtw_alloc_xmitbuf23a\n"); */
-
-	spin_lock_irqsave(&pfree_xmitbuf_queue->lock, irqL);
-
-	phead = get_list_head(pfree_xmitbuf_queue);
-
-	if (!list_empty(phead)) {
-		pxmitbuf = list_first_entry(phead, struct xmit_buf, list);
-
-		list_del_init(&pxmitbuf->list);
-
-		pxmitpriv->free_xmitbuf_cnt--;
-		pxmitbuf->priv_data = NULL;
-		pxmitbuf->ext_tag = false;
-		pxmitbuf->flags = XMIT_VO_QUEUE;
-
-		if (pxmitbuf->sctx) {
-			DBG_8723A("%s pxmitbuf->sctx is not NULL\n", __func__);
-			rtw23a_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_BUF_ALLOC);
-		}
-	}
-
-	spin_unlock_irqrestore(&pfree_xmitbuf_queue->lock, irqL);
-
-	return pxmitbuf;
-}
-
-int rtw_free_xmitbuf23a(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
-{
-	unsigned long irqL;
-	struct rtw_queue *pfree_xmitbuf_queue = &pxmitpriv->free_xmitbuf_queue;
-
-	/* DBG_8723A("+rtw_free_xmitbuf23a\n"); */
-
-	if (pxmitbuf == NULL)
-		return _FAIL;
-
-	if (pxmitbuf->sctx) {
-		DBG_8723A("%s pxmitbuf->sctx is not NULL\n", __func__);
-		rtw23a_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_BUF_FREE);
-	}
-
-	if (pxmitbuf->ext_tag) {
-		rtw_free_xmitbuf_ext23a(pxmitpriv, pxmitbuf);
-	} else {
-		spin_lock_irqsave(&pfree_xmitbuf_queue->lock, irqL);
-
-		list_del_init(&pxmitbuf->list);
-
-		list_add_tail(&pxmitbuf->list,
-			      get_list_head(pfree_xmitbuf_queue));
-
-		pxmitpriv->free_xmitbuf_cnt++;
-		spin_unlock_irqrestore(&pfree_xmitbuf_queue->lock, irqL);
-	}
-
-	return _SUCCESS;
-}
-
-static void rtw_init_xmitframe(struct xmit_frame *pxframe)
-{
-	if (pxframe !=  NULL) {
-		/* default value setting */
-		pxframe->buf_addr = NULL;
-		pxframe->pxmitbuf = NULL;
-
-		memset(&pxframe->attrib, 0, sizeof(struct pkt_attrib));
-		/* pxframe->attrib.psta = NULL; */
-
-		pxframe->frame_tag = DATA_FRAMETAG;
-
-		pxframe->pkt = NULL;
-		pxframe->pkt_offset = 1;/* default use pkt_offset to fill tx desc */
-
-		pxframe->ack_report = 0;
-	}
-}
-
-/*
-Calling context:
-1. OS_TXENTRY
-2. RXENTRY (rx_thread or RX_ISR/RX_CallBack)
-
-If we turn on USE_RXTHREAD, then, no need for critical section.
-Otherwise, we must use _enter/_exit critical to protect free_xmit_queue...
-
-Must be very very cautious...
-
-*/
-static struct xmit_frame *rtw_alloc_xmitframe(struct xmit_priv *pxmitpriv)
-{
-	struct xmit_frame *pxframe;
-	struct rtw_queue *pfree_xmit_queue = &pxmitpriv->free_xmit_queue;
-
-	spin_lock_bh(&pfree_xmit_queue->lock);
-
-	pxframe = list_first_entry_or_null(&pfree_xmit_queue->queue,
-					   struct xmit_frame, list);
-	if (!pxframe) {
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "rtw_alloc_xmitframe:%d\n",
-			 pxmitpriv->free_xmitframe_cnt);
-	} else {
-		list_del_init(&pxframe->list);
-		pxmitpriv->free_xmitframe_cnt--;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "rtw_alloc_xmitframe():free_xmitframe_cnt =%d\n",
-			 pxmitpriv->free_xmitframe_cnt);
-	}
-
-	spin_unlock_bh(&pfree_xmit_queue->lock);
-
-	rtw_init_xmitframe(pxframe);
-
-	return pxframe;
-}
-
-struct xmit_frame *rtw_alloc_xmitframe23a_ext(struct xmit_priv *pxmitpriv)
-{
-	struct xmit_frame *pxframe;
-	struct rtw_queue *queue = &pxmitpriv->free_xframe_ext_queue;
-
-	spin_lock_bh(&queue->lock);
-
-	pxframe = list_first_entry_or_null(&queue->queue,
-					   struct xmit_frame, list);
-	if (!pxframe) {
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "rtw_alloc_xmitframe23a_ext:%d\n",
-			 pxmitpriv->free_xframe_ext_cnt);
-	} else {
-		list_del_init(&pxframe->list);
-		pxmitpriv->free_xframe_ext_cnt--;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "rtw_alloc_xmitframe23a_ext():free_xmitframe_cnt =%d\n",
-			 pxmitpriv->free_xframe_ext_cnt);
-	}
-
-	spin_unlock_bh(&queue->lock);
-
-	rtw_init_xmitframe(pxframe);
-
-	return pxframe;
-}
-
-s32 rtw_free_xmitframe23a(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitframe)
-{
-	struct rtw_queue *queue = NULL;
-	struct rtw_adapter *padapter = pxmitpriv->adapter;
-	struct sk_buff *pndis_pkt = NULL;
-
-	if (pxmitframe == NULL) {
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-			 "====== rtw_free_xmitframe23a():pxmitframe == NULL!!!!!!!!!!\n");
-		goto exit;
-	}
-
-	if (pxmitframe->pkt) {
-		pndis_pkt = pxmitframe->pkt;
-		pxmitframe->pkt = NULL;
-	}
-
-	if (pxmitframe->ext_tag == 0)
-		queue = &pxmitpriv->free_xmit_queue;
-	else if (pxmitframe->ext_tag == 1)
-		queue = &pxmitpriv->free_xframe_ext_queue;
-
-	if (!queue)
-		goto check_pkt_complete;
-	spin_lock_bh(&queue->lock);
-
-	list_del_init(&pxmitframe->list);
-	list_add_tail(&pxmitframe->list, get_list_head(queue));
-	if (pxmitframe->ext_tag == 0) {
-		pxmitpriv->free_xmitframe_cnt++;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_debug_,
-			 "rtw_free_xmitframe23a():free_xmitframe_cnt =%d\n",
-			 pxmitpriv->free_xmitframe_cnt);
-	} else if (pxmitframe->ext_tag == 1) {
-		pxmitpriv->free_xframe_ext_cnt++;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_debug_,
-			 "rtw_free_xmitframe23a():free_xframe_ext_cnt =%d\n",
-			 pxmitpriv->free_xframe_ext_cnt);
-	}
-
-	spin_unlock_bh(&queue->lock);
-
-check_pkt_complete:
-
-	if (pndis_pkt)
-		rtw_os_pkt_complete23a(padapter, pndis_pkt);
-
-exit:
-
-	return _SUCCESS;
-}
-
-void rtw_free_xmitframe_queue23a(struct xmit_priv *pxmitpriv,
-				 struct rtw_queue *pframequeue)
-{
-	struct list_head *phead;
-	struct xmit_frame *pxmitframe, *ptmp;
-
-	spin_lock_bh(&pframequeue->lock);
-	phead = get_list_head(pframequeue);
-	list_for_each_entry_safe(pxmitframe, ptmp, phead, list)
-		rtw_free_xmitframe23a(pxmitpriv, pxmitframe);
-	spin_unlock_bh(&pframequeue->lock);
-
-}
-
-int rtw_xmitframe_enqueue23a(struct rtw_adapter *padapter,
-			     struct xmit_frame *pxmitframe)
-{
-	if (rtw_xmit23a_classifier(padapter, pxmitframe) == _FAIL) {
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-			 "rtw_xmitframe_enqueue23a: drop xmit pkt for classifier fail\n");
-		return _FAIL;
-	}
-
-	return _SUCCESS;
-}
-
-static struct xmit_frame *
-dequeue_one_xmitframe(struct xmit_priv *pxmitpriv, struct hw_xmit *phwxmit,
-		      struct tx_servq *ptxservq, struct rtw_queue *pframe_queue)
-{
-	struct list_head *phead;
-	struct xmit_frame *pxmitframe = NULL;
-
-	phead = get_list_head(pframe_queue);
-
-	if (!list_empty(phead)) {
-		pxmitframe = list_first_entry(phead, struct xmit_frame, list);
-		list_del_init(&pxmitframe->list);
-		ptxservq->qcnt--;
-	}
-	return pxmitframe;
-}
-
-struct xmit_frame *
-rtw_dequeue_xframe23a(struct xmit_priv *pxmitpriv, struct hw_xmit *phwxmit_i,
-		   int entry)
-{
-	struct list_head *sta_phead;
-	struct hw_xmit *phwxmit;
-	struct tx_servq *ptxservq = NULL, *ptmp;
-	struct rtw_queue *pframe_queue = NULL;
-	struct xmit_frame *pxmitframe = NULL;
-	struct rtw_adapter *padapter = pxmitpriv->adapter;
-	struct registry_priv	*pregpriv = &padapter->registrypriv;
-	int i, inx[4];
-
-	inx[0] = 0;
-	inx[1] = 1;
-	inx[2] = 2;
-	inx[3] = 3;
-	if (pregpriv->wifi_spec == 1) {
-		int j;
-
-		for (j = 0; j < 4; j++)
-			inx[j] = pxmitpriv->wmm_para_seq[j];
-	}
-
-	spin_lock_bh(&pxmitpriv->lock);
-
-	for (i = 0; i < entry; i++) {
-		phwxmit = phwxmit_i + inx[i];
-
-		sta_phead = get_list_head(phwxmit->sta_queue);
-		list_for_each_entry_safe(ptxservq, ptmp, sta_phead,
-					 tx_pending) {
-			pframe_queue = &ptxservq->sta_pending;
-
-			pxmitframe = dequeue_one_xmitframe(pxmitpriv, phwxmit, ptxservq, pframe_queue);
-
-			if (pxmitframe) {
-				phwxmit->accnt--;
-
-				/* Remove sta node when there is no pending packets. */
-				/* must be done after get_next and
-				   before break */
-				if (list_empty(&pframe_queue->queue))
-					list_del_init(&ptxservq->tx_pending);
-				goto exit;
-			}
-		}
-	}
-exit:
-	spin_unlock_bh(&pxmitpriv->lock);
-	return pxmitframe;
-}
-
-struct tx_servq *rtw_get_sta_pending23a(struct rtw_adapter *padapter, struct sta_info *psta, int up, u8 *ac)
-{
-	struct tx_servq *ptxservq = NULL;
-
-	switch (up) {
-	case 1:
-	case 2:
-		ptxservq = &psta->sta_xmitpriv.bk_q;
-		*(ac) = 3;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "rtw_get_sta_pending23a : BK\n");
-		break;
-	case 4:
-	case 5:
-		ptxservq = &psta->sta_xmitpriv.vi_q;
-		*(ac) = 1;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "rtw_get_sta_pending23a : VI\n");
-		break;
-	case 6:
-	case 7:
-		ptxservq = &psta->sta_xmitpriv.vo_q;
-		*(ac) = 0;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "rtw_get_sta_pending23a : VO\n");
-		break;
-	case 0:
-	case 3:
-	default:
-		ptxservq = &psta->sta_xmitpriv.be_q;
-		*(ac) = 2;
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "rtw_get_sta_pending23a : BE\n");
-		break;
-	}
-	return ptxservq;
-}
-
-/*
- * Will enqueue pxmitframe to the proper queue,
- * and indicate it to xx_pending list.....
- */
-int rtw_xmit23a_classifier(struct rtw_adapter *padapter,
-			   struct xmit_frame *pxmitframe)
-{
-	struct sta_info	*psta;
-	struct tx_servq	*ptxservq;
-	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
-	struct sta_priv	*pstapriv = &padapter->stapriv;
-	struct hw_xmit	*phwxmits =  padapter->xmitpriv.hwxmits;
-	u8	ac_index;
-	int res = _SUCCESS;
-
-	if (pattrib->psta) {
-		psta = pattrib->psta;
-	} else {
-		DBG_8723A("%s, call rtw_get_stainfo23a()\n", __func__);
-		psta = rtw_get_stainfo23a(pstapriv, pattrib->ra);
-	}
-	if (psta == NULL) {
-		res = _FAIL;
-		DBG_8723A("rtw_xmit23a_classifier: psta == NULL\n");
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-			 "rtw_xmit23a_classifier: psta == NULL\n");
-		goto exit;
-	}
-	if (!(psta->state & _FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n", __func__,
-			  psta->state);
-		return _FAIL;
-	}
-	ptxservq = rtw_get_sta_pending23a(padapter, psta, pattrib->priority,
-				       (u8 *)(&ac_index));
-
-	if (list_empty(&ptxservq->tx_pending)) {
-		list_add_tail(&ptxservq->tx_pending,
-			      get_list_head(phwxmits[ac_index].sta_queue));
-	}
-
-	list_add_tail(&pxmitframe->list, get_list_head(&ptxservq->sta_pending));
-	ptxservq->qcnt++;
-	phwxmits[ac_index].accnt++;
-exit:
-	return res;
-}
-
-void rtw_alloc_hwxmits23a(struct rtw_adapter *padapter)
-{
-	struct hw_xmit *hwxmits;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	int size;
-
-	pxmitpriv->hwxmit_entry = HWXMIT_ENTRY;
-
-	size = sizeof(struct hw_xmit) * (pxmitpriv->hwxmit_entry + 1);
-	pxmitpriv->hwxmits = kzalloc(size, GFP_KERNEL);
-
-	hwxmits = pxmitpriv->hwxmits;
-
-	if (pxmitpriv->hwxmit_entry == 5) {
-		/* pxmitpriv->bmc_txqueue.head = 0; */
-		/* hwxmits[0] .phwtxqueue = &pxmitpriv->bmc_txqueue; */
-		hwxmits[0] .sta_queue = &pxmitpriv->bm_pending;
-
-		/* pxmitpriv->vo_txqueue.head = 0; */
-		/* hwxmits[1] .phwtxqueue = &pxmitpriv->vo_txqueue; */
-		hwxmits[1] .sta_queue = &pxmitpriv->vo_pending;
-
-		/* pxmitpriv->vi_txqueue.head = 0; */
-		/* hwxmits[2] .phwtxqueue = &pxmitpriv->vi_txqueue; */
-		hwxmits[2] .sta_queue = &pxmitpriv->vi_pending;
-
-		/* pxmitpriv->bk_txqueue.head = 0; */
-		/* hwxmits[3] .phwtxqueue = &pxmitpriv->bk_txqueue; */
-		hwxmits[3] .sta_queue = &pxmitpriv->bk_pending;
-
-		/* pxmitpriv->be_txqueue.head = 0; */
-		/* hwxmits[4] .phwtxqueue = &pxmitpriv->be_txqueue; */
-		hwxmits[4] .sta_queue = &pxmitpriv->be_pending;
-
-	} else if (pxmitpriv->hwxmit_entry == 4) {
-
-		/* pxmitpriv->vo_txqueue.head = 0; */
-		/* hwxmits[0] .phwtxqueue = &pxmitpriv->vo_txqueue; */
-		hwxmits[0] .sta_queue = &pxmitpriv->vo_pending;
-
-		/* pxmitpriv->vi_txqueue.head = 0; */
-		/* hwxmits[1] .phwtxqueue = &pxmitpriv->vi_txqueue; */
-		hwxmits[1] .sta_queue = &pxmitpriv->vi_pending;
-
-		/* pxmitpriv->be_txqueue.head = 0; */
-		/* hwxmits[2] .phwtxqueue = &pxmitpriv->be_txqueue; */
-		hwxmits[2] .sta_queue = &pxmitpriv->be_pending;
-
-		/* pxmitpriv->bk_txqueue.head = 0; */
-		/* hwxmits[3] .phwtxqueue = &pxmitpriv->bk_txqueue; */
-		hwxmits[3] .sta_queue = &pxmitpriv->bk_pending;
-	} else {
-
-	}
-}
-
-void rtw_free_hwxmits23a(struct rtw_adapter *padapter)
-{
-	struct hw_xmit *hwxmits;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	hwxmits = pxmitpriv->hwxmits;
-	kfree(hwxmits);
-}
-
-void rtw_init_hwxmits23a(struct hw_xmit *phwxmit, int entry)
-{
-	int i;
-
-	for (i = 0; i < entry; i++, phwxmit++)
-		phwxmit->accnt = 0;
-}
-
-u32 rtw_get_ff_hwaddr23a(struct xmit_frame *pxmitframe)
-{
-	u32 addr;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-
-	switch (pattrib->qsel) {
-	case 0:
-	case 3:
-		addr = BE_QUEUE_INX;
-		break;
-	case 1:
-	case 2:
-		addr = BK_QUEUE_INX;
-		break;
-	case 4:
-	case 5:
-		addr = VI_QUEUE_INX;
-		break;
-	case 6:
-	case 7:
-		addr = VO_QUEUE_INX;
-		break;
-	case 0x10:
-		addr = BCN_QUEUE_INX;
-		break;
-	case 0x11:/* BC/MC in PS (HIQ) */
-		addr = HIGH_QUEUE_INX;
-		break;
-	case 0x12:
-	default:
-		addr = MGT_QUEUE_INX;
-		break;
-	}
-
-	return addr;
-}
-
-/*
- * The main transmit(tx) entry
- *
- * Return
- *	1	enqueue
- *	0	success, hardware will handle this xmit frame(packet)
- *	<0	fail
- */
-int rtw_xmit23a(struct rtw_adapter *padapter, struct sk_buff *skb)
-{
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct xmit_frame *pxmitframe = NULL;
-	int res;
-
-	pxmitframe = rtw_alloc_xmitframe(pxmitpriv);
-
-	if (pxmitframe == NULL) {
-		RT_TRACE(_module_xmit_osdep_c_, _drv_err_,
-			 "rtw_xmit23a: no more pxmitframe\n");
-		return -1;
-	}
-
-	res = update_attrib(padapter, skb, &pxmitframe->attrib);
-
-	if (res == _FAIL) {
-		RT_TRACE(_module_xmit_osdep_c_, _drv_err_,
-			 "rtw_xmit23a: update attrib fail\n");
-		rtw_free_xmitframe23a(pxmitpriv, pxmitframe);
-		return -1;
-	}
-	pxmitframe->pkt = skb;
-
-	pxmitframe->attrib.qsel = pxmitframe->attrib.priority;
-
-#ifdef CONFIG_8723AU_AP_MODE
-	spin_lock_bh(&pxmitpriv->lock);
-	if (xmitframe_enqueue_for_sleeping_sta23a(padapter, pxmitframe)) {
-		spin_unlock_bh(&pxmitpriv->lock);
-		return 1;
-	}
-	spin_unlock_bh(&pxmitpriv->lock);
-#endif
-
-	if (rtl8723au_hal_xmit(padapter, pxmitframe) == false)
-		return 1;
-
-	return 0;
-}
-
-#if defined(CONFIG_8723AU_AP_MODE)
-
-int xmitframe_enqueue_for_sleeping_sta23a(struct rtw_adapter *padapter, struct xmit_frame *pxmitframe)
-{
-	int ret = false;
-	struct sta_info *psta = NULL;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	int bmcst = is_multicast_ether_addr(pattrib->ra);
-
-	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE))
-		return ret;
-
-	if (pattrib->psta) {
-		psta = pattrib->psta;
-	} else {
-		DBG_8723A("%s, call rtw_get_stainfo23a()\n", __func__);
-		psta = rtw_get_stainfo23a(pstapriv, pattrib->ra);
-	}
-
-	if (psta == NULL) {
-		DBG_8723A("%s, psta == NUL\n", __func__);
-		return false;
-	}
-
-	if (!(psta->state & _FW_LINKED)) {
-		DBG_8723A("%s, psta->state(0x%x) != _FW_LINKED\n", __func__,
-			  psta->state);
-		return false;
-	}
-
-	if (pattrib->triggered == 1) {
-		if (bmcst)
-			pattrib->qsel = 0x11;/* HIQ */
-		return ret;
-	}
-
-	if (bmcst) {
-		spin_lock_bh(&psta->sleep_q.lock);
-
-		if (pstapriv->sta_dz_bitmap) {
-			/* if anyone sta is in ps mode */
-			list_del_init(&pxmitframe->list);
-
-			/* spin_lock_bh(&psta->sleep_q.lock); */
-
-			list_add_tail(&pxmitframe->list, get_list_head(&psta->sleep_q));
-
-			psta->sleepq_len++;
-
-			pstapriv->tim_bitmap |= BIT(0);/*  */
-			pstapriv->sta_dz_bitmap |= BIT(0);
-
-			/* DBG_8723A("enqueue, sq_len =%d, tim =%x\n", psta->sleepq_len, pstapriv->tim_bitmap); */
-
-			/* tx bc/mc packets after update bcn */
-			update_beacon23a(padapter, WLAN_EID_TIM, NULL, false);
-
-			/* spin_unlock_bh(&psta->sleep_q.lock); */
-
-			ret = true;
-
-		}
-
-		spin_unlock_bh(&psta->sleep_q.lock);
-
-		return ret;
-
-	}
-
-	spin_lock_bh(&psta->sleep_q.lock);
-
-	if (psta->state&WIFI_SLEEP_STATE) {
-		u8 wmmps_ac = 0;
-
-		if (pstapriv->sta_dz_bitmap & CHKBIT(psta->aid)) {
-			list_del_init(&pxmitframe->list);
-
-			/* spin_lock_bh(&psta->sleep_q.lock); */
-
-			list_add_tail(&pxmitframe->list, get_list_head(&psta->sleep_q));
-
-			psta->sleepq_len++;
-
-			switch (pattrib->priority) {
-			case 1:
-			case 2:
-				wmmps_ac = psta->uapsd_bk & BIT(0);
-				break;
-			case 4:
-			case 5:
-				wmmps_ac = psta->uapsd_vi & BIT(0);
-				break;
-			case 6:
-			case 7:
-				wmmps_ac = psta->uapsd_vo & BIT(0);
-				break;
-			case 0:
-			case 3:
-			default:
-				wmmps_ac = psta->uapsd_be & BIT(0);
-				break;
-			}
-
-			if (wmmps_ac)
-				psta->sleepq_ac_len++;
-
-			if (((psta->has_legacy_ac) && (!wmmps_ac)) ||
-			   ((!psta->has_legacy_ac) && (wmmps_ac))) {
-				pstapriv->tim_bitmap |= CHKBIT(psta->aid);
-
-				if (psta->sleepq_len == 1) {
-					/* update BCN for TIM IE */
-					update_beacon23a(padapter, WLAN_EID_TIM,
-							 NULL, false);
-				}
-			}
-
-			/* spin_unlock_bh(&psta->sleep_q.lock); */
-
-			/* if (psta->sleepq_len > (NR_XMITFRAME>>3)) */
-			/*  */
-			/*	wakeup_sta_to_xmit23a(padapter, psta); */
-			/*  */
-
-			ret = true;
-
-		}
-
-	}
-
-	spin_unlock_bh(&psta->sleep_q.lock);
-
-	return ret;
-}
-
-static void
-dequeue_xmitframes_to_sleeping_queue(struct rtw_adapter *padapter,
-				     struct sta_info *psta,
-				     struct rtw_queue *pframequeue)
-{
-	int ret;
-	struct list_head *phead;
-	u8 ac_index;
-	struct tx_servq	*ptxservq;
-	struct pkt_attrib *pattrib;
-	struct xmit_frame *pxmitframe, *ptmp;
-	struct hw_xmit *phwxmits = padapter->xmitpriv.hwxmits;
-
-	phead = get_list_head(pframequeue);
-	list_for_each_entry_safe(pxmitframe, ptmp, phead, list) {
-		ret = xmitframe_enqueue_for_sleeping_sta23a(padapter, pxmitframe);
-
-		if (ret == true) {
-			pattrib = &pxmitframe->attrib;
-
-			ptxservq = rtw_get_sta_pending23a(padapter, psta, pattrib->priority, (u8 *)(&ac_index));
-
-			ptxservq->qcnt--;
-			phwxmits[ac_index].accnt--;
-		} else {
-			/* DBG_8723A("xmitframe_enqueue_for_sleeping_sta23a return false\n"); */
-		}
-	}
-}
-
-void stop_sta_xmit23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	struct sta_info *psta_bmc;
-	struct sta_xmit_priv *pstaxmitpriv;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	pstaxmitpriv = &psta->sta_xmitpriv;
-
-	/* for BC/MC Frames */
-	psta_bmc = rtw_get_bcmc_stainfo23a(padapter);
-
-	spin_lock_bh(&pxmitpriv->lock);
-
-	psta->state |= WIFI_SLEEP_STATE;
-
-	pstapriv->sta_dz_bitmap |= CHKBIT(psta->aid);
-
-	dequeue_xmitframes_to_sleeping_queue(padapter, psta, &pstaxmitpriv->vo_q.sta_pending);
-	list_del_init(&pstaxmitpriv->vo_q.tx_pending);
-
-	dequeue_xmitframes_to_sleeping_queue(padapter, psta, &pstaxmitpriv->vi_q.sta_pending);
-	list_del_init(&pstaxmitpriv->vi_q.tx_pending);
-
-	dequeue_xmitframes_to_sleeping_queue(padapter, psta,
-					     &pstaxmitpriv->be_q.sta_pending);
-	list_del_init(&pstaxmitpriv->be_q.tx_pending);
-
-	dequeue_xmitframes_to_sleeping_queue(padapter, psta,
-					     &pstaxmitpriv->bk_q.sta_pending);
-	list_del_init(&pstaxmitpriv->bk_q.tx_pending);
-
-	/* for BC/MC Frames */
-	pstaxmitpriv = &psta_bmc->sta_xmitpriv;
-	dequeue_xmitframes_to_sleeping_queue(padapter, psta_bmc,
-					     &pstaxmitpriv->be_q.sta_pending);
-	list_del_init(&pstaxmitpriv->be_q.tx_pending);
-
-	spin_unlock_bh(&pxmitpriv->lock);
-}
-
-void wakeup_sta_to_xmit23a(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	u8 update_mask = 0, wmmps_ac = 0;
-	struct sta_info *psta_bmc;
-	struct list_head *phead;
-	struct xmit_frame *pxmitframe = NULL, *ptmp;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	spin_lock_bh(&pxmitpriv->lock);
-	phead = get_list_head(&psta->sleep_q);
-	list_for_each_entry_safe(pxmitframe, ptmp, phead, list) {
-		list_del_init(&pxmitframe->list);
-
-		switch (pxmitframe->attrib.priority) {
-		case 1:
-		case 2:
-			wmmps_ac = psta->uapsd_bk & BIT(1);
-			break;
-		case 4:
-		case 5:
-			wmmps_ac = psta->uapsd_vi & BIT(1);
-			break;
-		case 6:
-		case 7:
-			wmmps_ac = psta->uapsd_vo & BIT(1);
-			break;
-		case 0:
-		case 3:
-		default:
-			wmmps_ac = psta->uapsd_be & BIT(1);
-			break;
-		}
-
-		psta->sleepq_len--;
-		if (psta->sleepq_len > 0)
-			pxmitframe->attrib.mdata = 1;
-		else
-			pxmitframe->attrib.mdata = 0;
-
-		if (wmmps_ac) {
-			psta->sleepq_ac_len--;
-			if (psta->sleepq_ac_len > 0) {
-				pxmitframe->attrib.mdata = 1;
-				pxmitframe->attrib.eosp = 0;
-			} else {
-				pxmitframe->attrib.mdata = 0;
-				pxmitframe->attrib.eosp = 1;
-			}
-		}
-
-		pxmitframe->attrib.triggered = 1;
-		rtl8723au_hal_xmitframe_enqueue(padapter, pxmitframe);
-	}
-
-	if (psta->sleepq_len == 0) {
-		pstapriv->tim_bitmap &= ~CHKBIT(psta->aid);
-
-		/* update BCN for TIM IE */
-		update_mask = BIT(0);
-
-		if (psta->state&WIFI_SLEEP_STATE)
-			psta->state ^= WIFI_SLEEP_STATE;
-
-		if (psta->state & WIFI_STA_ALIVE_CHK_STATE) {
-			psta->expire_to = pstapriv->expire_to;
-			psta->state ^= WIFI_STA_ALIVE_CHK_STATE;
-		}
-
-		pstapriv->sta_dz_bitmap &= ~CHKBIT(psta->aid);
-	}
-	/* spin_unlock_bh(&psta->sleep_q.lock); */
-	spin_unlock_bh(&pxmitpriv->lock);
-
-	/* for BC/MC Frames */
-	psta_bmc = rtw_get_bcmc_stainfo23a(padapter);
-	if (!psta_bmc)
-		return;
-
-	if ((pstapriv->sta_dz_bitmap&0xfffe) == 0x0) {
-		/* no any sta in ps mode */
-		spin_lock_bh(&pxmitpriv->lock);
-		phead = get_list_head(&psta_bmc->sleep_q);
-		list_for_each_entry_safe(pxmitframe, ptmp, phead, list) {
-			list_del_init(&pxmitframe->list);
-
-			psta_bmc->sleepq_len--;
-			if (psta_bmc->sleepq_len > 0)
-				pxmitframe->attrib.mdata = 1;
-			else
-				pxmitframe->attrib.mdata = 0;
-
-			pxmitframe->attrib.triggered = 1;
-			rtl8723au_hal_xmitframe_enqueue(padapter, pxmitframe);
-		}
-		if (psta_bmc->sleepq_len == 0) {
-			pstapriv->tim_bitmap &= ~BIT(0);
-			pstapriv->sta_dz_bitmap &= ~BIT(0);
-
-			/* update BCN for TIM IE */
-			/* update_BCNTIM(padapter); */
-			update_mask |= BIT(1);
-		}
-		/* spin_unlock_bh(&psta_bmc->sleep_q.lock); */
-		spin_unlock_bh(&pxmitpriv->lock);
-	}
-
-	if (update_mask)
-		update_beacon23a(padapter, WLAN_EID_TIM, NULL, false);
-}
-
-void xmit_delivery_enabled_frames23a(struct rtw_adapter *padapter,
-				  struct sta_info *psta)
-{
-	u8 wmmps_ac = 0;
-	struct list_head *phead;
-	struct xmit_frame *pxmitframe, *ptmp;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	/* spin_lock_bh(&psta->sleep_q.lock); */
-	spin_lock_bh(&pxmitpriv->lock);
-	phead = get_list_head(&psta->sleep_q);
-	list_for_each_entry_safe(pxmitframe, ptmp, phead, list) {
-		switch (pxmitframe->attrib.priority) {
-		case 1:
-		case 2:
-			wmmps_ac = psta->uapsd_bk & BIT(1);
-			break;
-		case 4:
-		case 5:
-			wmmps_ac = psta->uapsd_vi & BIT(1);
-			break;
-		case 6:
-		case 7:
-			wmmps_ac = psta->uapsd_vo & BIT(1);
-			break;
-		case 0:
-		case 3:
-		default:
-			wmmps_ac = psta->uapsd_be & BIT(1);
-			break;
-		}
-
-		if (!wmmps_ac)
-			continue;
-
-		list_del_init(&pxmitframe->list);
-
-		psta->sleepq_len--;
-		psta->sleepq_ac_len--;
-
-		if (psta->sleepq_ac_len > 0) {
-			pxmitframe->attrib.mdata = 1;
-			pxmitframe->attrib.eosp = 0;
-		} else {
-			pxmitframe->attrib.mdata = 0;
-			pxmitframe->attrib.eosp = 1;
-		}
-
-		pxmitframe->attrib.triggered = 1;
-
-		rtl8723au_hal_xmitframe_enqueue(padapter, pxmitframe);
-
-		if ((psta->sleepq_ac_len == 0) && (!psta->has_legacy_ac) &&
-		    (wmmps_ac)) {
-			pstapriv->tim_bitmap &= ~CHKBIT(psta->aid);
-
-			/* update BCN for TIM IE */
-			update_beacon23a(padapter, WLAN_EID_TIM, NULL, false);
-		}
-	}
-	spin_unlock_bh(&pxmitpriv->lock);
-}
-
-#endif
-
-void rtw_sctx_init23a(struct submit_ctx *sctx, int timeout_ms)
-{
-	sctx->timeout_ms = timeout_ms;
-	init_completion(&sctx->done);
-	sctx->status = RTW_SCTX_SUBMITTED;
-}
-
-int rtw_sctx_wait23a(struct submit_ctx *sctx)
-{
-	int ret = _FAIL;
-	unsigned long expire;
-	int status = 0;
-
-	expire = sctx->timeout_ms ? msecs_to_jiffies(sctx->timeout_ms) :
-		 MAX_SCHEDULE_TIMEOUT;
-	if (!wait_for_completion_timeout(&sctx->done, expire)) {
-		/* timeout, do something?? */
-		status = RTW_SCTX_DONE_TIMEOUT;
-		DBG_8723A("%s timeout\n", __func__);
-	} else {
-		status = sctx->status;
-	}
-
-	if (status == RTW_SCTX_DONE_SUCCESS)
-		ret = _SUCCESS;
-
-	return ret;
-}
-
-static bool rtw_sctx_chk_waring_status(int status)
-{
-	switch (status) {
-	case RTW_SCTX_DONE_UNKNOWN:
-	case RTW_SCTX_DONE_BUF_ALLOC:
-	case RTW_SCTX_DONE_BUF_FREE:
-	case RTW_SCTX_DONE_DRV_STOP:
-	case RTW_SCTX_DONE_DEV_REMOVE:
-		return true;
-	default:
-		return false;
-	}
-}
-
-void rtw23a_sctx_done_err(struct submit_ctx **sctx, int status)
-{
-	if (*sctx) {
-		if (rtw_sctx_chk_waring_status(status))
-			DBG_8723A("%s status:%d\n", __func__, status);
-		(*sctx)->status = status;
-		complete(&(*sctx)->done);
-		*sctx = NULL;
-	}
-}
-
-int rtw_ack_tx_wait23a(struct xmit_priv *pxmitpriv, u32 timeout_ms)
-{
-	struct submit_ctx *pack_tx_ops = &pxmitpriv->ack_tx_ops;
-
-	pack_tx_ops->timeout_ms = timeout_ms;
-	pack_tx_ops->status = RTW_SCTX_SUBMITTED;
-
-	return rtw_sctx_wait23a(pack_tx_ops);
-}
-
diff --git a/drivers/staging/rtl8723au/hal/Hal8723PwrSeq.c b/drivers/staging/rtl8723au/hal/Hal8723PwrSeq.c
deleted file mode 100644
index 747f86cddeb9..000000000000
--- a/drivers/staging/rtl8723au/hal/Hal8723PwrSeq.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#include "Hal8723PwrSeq.h"
-
-/*
-    drivers should parse below arrays and do the corresponding actions
-*/
-/* 3 Power on  Array */
-struct wlan_pwr_cfg rtl8723AU_power_on_flow[RTL8723A_TRANS_CARDEMU_TO_ACT_STEPS+RTL8723A_TRANS_END_STEPS] = {
-	RTL8723A_TRANS_CARDEMU_TO_ACT
-	RTL8723A_TRANS_END
-};
-
-/* 3 Radio off GPIO Array */
-struct wlan_pwr_cfg rtl8723AU_radio_off_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_END_STEPS] = {
-	RTL8723A_TRANS_ACT_TO_CARDEMU
-	RTL8723A_TRANS_END
-};
-
-/* 3 Card Disable Array */
-struct wlan_pwr_cfg rtl8723AU_card_disable_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_PDN_STEPS+RTL8723A_TRANS_END_STEPS] = {
-	RTL8723A_TRANS_ACT_TO_CARDEMU
-	RTL8723A_TRANS_CARDEMU_TO_CARDDIS
-	RTL8723A_TRANS_END
-};
-
-/* 3 Card Enable Array */
-struct wlan_pwr_cfg rtl8723AU_card_enable_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_PDN_STEPS+RTL8723A_TRANS_END_STEPS] = {
-	RTL8723A_TRANS_CARDDIS_TO_CARDEMU
-	RTL8723A_TRANS_CARDEMU_TO_ACT
-	RTL8723A_TRANS_END
-};
-
-/* 3 Suspend Array */
-struct wlan_pwr_cfg rtl8723AU_suspend_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_SUS_STEPS+RTL8723A_TRANS_END_STEPS] = {
-	RTL8723A_TRANS_ACT_TO_CARDEMU
-	RTL8723A_TRANS_CARDEMU_TO_SUS
-	RTL8723A_TRANS_END
-};
-
-/* 3 Resume Array */
-struct wlan_pwr_cfg rtl8723AU_resume_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_SUS_STEPS+RTL8723A_TRANS_END_STEPS] = {
-	RTL8723A_TRANS_SUS_TO_CARDEMU
-	RTL8723A_TRANS_CARDEMU_TO_ACT
-	RTL8723A_TRANS_END
-};
-
-/* 3 HWPDN Array */
-struct wlan_pwr_cfg rtl8723AU_hwpdn_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_PDN_STEPS+RTL8723A_TRANS_END_STEPS] = {
-	RTL8723A_TRANS_ACT_TO_CARDEMU
-	RTL8723A_TRANS_CARDEMU_TO_PDN
-	RTL8723A_TRANS_END
-};
-
-/* 3 Enter LPS */
-struct wlan_pwr_cfg rtl8723AU_enter_lps_flow[RTL8723A_TRANS_ACT_TO_LPS_STEPS+RTL8723A_TRANS_END_STEPS] = {
-	/* FW behavior */
-	RTL8723A_TRANS_ACT_TO_LPS
-	RTL8723A_TRANS_END
-};
-
-/* 3 Leave LPS */
-struct wlan_pwr_cfg rtl8723AU_leave_lps_flow[RTL8723A_TRANS_LPS_TO_ACT_STEPS+RTL8723A_TRANS_END_STEPS] = {
-	/* FW behavior */
-	RTL8723A_TRANS_LPS_TO_ACT
-	RTL8723A_TRANS_END
-};
diff --git a/drivers/staging/rtl8723au/hal/Hal8723UHWImg_CE.c b/drivers/staging/rtl8723au/hal/Hal8723UHWImg_CE.c
deleted file mode 100644
index 56833da63ced..000000000000
--- a/drivers/staging/rtl8723au/hal/Hal8723UHWImg_CE.c
+++ /dev/null
@@ -1,136 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-/*Created on  2013/01/14, 15:51*/
-#include "odm_precomp.h"
-
-u32 Rtl8723UPHY_REG_Array_PG[Rtl8723UPHY_REG_Array_PGLength] = {
-	0xe00, 0xffffffff, 0x0a0c0c0c,
-	0xe04, 0xffffffff, 0x02040608,
-	0xe08, 0x0000ff00, 0x00000000,
-	0x86c, 0xffffff00, 0x00000000,
-	0xe10, 0xffffffff, 0x0a0c0d0e,
-	0xe14, 0xffffffff, 0x02040608,
-	0xe18, 0xffffffff, 0x0a0c0d0e,
-	0xe1c, 0xffffffff, 0x02040608,
-	0x830, 0xffffffff, 0x0a0c0c0c,
-	0x834, 0xffffffff, 0x02040608,
-	0x838, 0xffffff00, 0x00000000,
-	0x86c, 0x000000ff, 0x00000000,
-	0x83c, 0xffffffff, 0x0a0c0d0e,
-	0x848, 0xffffffff, 0x02040608,
-	0x84c, 0xffffffff, 0x0a0c0d0e,
-	0x868, 0xffffffff, 0x02040608,
-	0xe00, 0xffffffff, 0x00000000,
-	0xe04, 0xffffffff, 0x00000000,
-	0xe08, 0x0000ff00, 0x00000000,
-	0x86c, 0xffffff00, 0x00000000,
-	0xe10, 0xffffffff, 0x00000000,
-	0xe14, 0xffffffff, 0x00000000,
-	0xe18, 0xffffffff, 0x00000000,
-	0xe1c, 0xffffffff, 0x00000000,
-	0x830, 0xffffffff, 0x00000000,
-	0x834, 0xffffffff, 0x00000000,
-	0x838, 0xffffff00, 0x00000000,
-	0x86c, 0x000000ff, 0x00000000,
-	0x83c, 0xffffffff, 0x00000000,
-	0x848, 0xffffffff, 0x00000000,
-	0x84c, 0xffffffff, 0x00000000,
-	0x868, 0xffffffff, 0x00000000,
-	0xe00, 0xffffffff, 0x04040404,
-	0xe04, 0xffffffff, 0x00020204,
-	0xe08, 0x0000ff00, 0x00000000,
-	0x86c, 0xffffff00, 0x00000000,
-	0xe10, 0xffffffff, 0x06060606,
-	0xe14, 0xffffffff, 0x00020406,
-	0xe18, 0xffffffff, 0x00000000,
-	0xe1c, 0xffffffff, 0x00000000,
-	0x830, 0xffffffff, 0x04040404,
-	0x834, 0xffffffff, 0x00020204,
-	0x838, 0xffffff00, 0x00000000,
-	0x86c, 0x000000ff, 0x00000000,
-	0x83c, 0xffffffff, 0x06060606,
-	0x848, 0xffffffff, 0x00020406,
-	0x84c, 0xffffffff, 0x00000000,
-	0x868, 0xffffffff, 0x00000000,
-	0xe00, 0xffffffff, 0x00000000,
-	0xe04, 0xffffffff, 0x00000000,
-	0xe08, 0x0000ff00, 0x00000000,
-	0x86c, 0xffffff00, 0x00000000,
-	0xe10, 0xffffffff, 0x00000000,
-	0xe14, 0xffffffff, 0x00000000,
-	0xe18, 0xffffffff, 0x00000000,
-	0xe1c, 0xffffffff, 0x00000000,
-	0x830, 0xffffffff, 0x00000000,
-	0x834, 0xffffffff, 0x00000000,
-	0x838, 0xffffff00, 0x00000000,
-	0x86c, 0x000000ff, 0x00000000,
-	0x83c, 0xffffffff, 0x00000000,
-	0x848, 0xffffffff, 0x00000000,
-	0x84c, 0xffffffff, 0x00000000,
-	0x868, 0xffffffff, 0x00000000,
-	0xe00, 0xffffffff, 0x00000000,
-	0xe04, 0xffffffff, 0x00000000,
-	0xe08, 0x0000ff00, 0x00000000,
-	0x86c, 0xffffff00, 0x00000000,
-	0xe10, 0xffffffff, 0x00000000,
-	0xe14, 0xffffffff, 0x00000000,
-	0xe18, 0xffffffff, 0x00000000,
-	0xe1c, 0xffffffff, 0x00000000,
-	0x830, 0xffffffff, 0x00000000,
-	0x834, 0xffffffff, 0x00000000,
-	0x838, 0xffffff00, 0x00000000,
-	0x86c, 0x000000ff, 0x00000000,
-	0x83c, 0xffffffff, 0x00000000,
-	0x848, 0xffffffff, 0x00000000,
-	0x84c, 0xffffffff, 0x00000000,
-	0x868, 0xffffffff, 0x00000000,
-	0xe00, 0xffffffff, 0x04040404,
-	0xe04, 0xffffffff, 0x00020204,
-	0xe08, 0x0000ff00, 0x00000000,
-	0x86c, 0xffffff00, 0x00000000,
-	0xe10, 0xffffffff, 0x00000000,
-	0xe14, 0xffffffff, 0x00000000,
-	0xe18, 0xffffffff, 0x00000000,
-	0xe1c, 0xffffffff, 0x00000000,
-	0x830, 0xffffffff, 0x04040404,
-	0x834, 0xffffffff, 0x00020204,
-	0x838, 0xffffff00, 0x00000000,
-	0x86c, 0x000000ff, 0x00000000,
-	0x83c, 0xffffffff, 0x00000000,
-	0x848, 0xffffffff, 0x00000000,
-	0x84c, 0xffffffff, 0x00000000,
-	0x868, 0xffffffff, 0x00000000,
-	0xe00, 0xffffffff, 0x00000000,
-	0xe04, 0xffffffff, 0x00000000,
-	0xe08, 0x0000ff00, 0x00000000,
-	0x86c, 0xffffff00, 0x00000000,
-	0xe10, 0xffffffff, 0x00000000,
-	0xe14, 0xffffffff, 0x00000000,
-	0xe18, 0xffffffff, 0x00000000,
-	0xe1c, 0xffffffff, 0x00000000,
-	0x830, 0xffffffff, 0x00000000,
-	0x834, 0xffffffff, 0x00000000,
-	0x838, 0xffffff00, 0x00000000,
-	0x86c, 0x000000ff, 0x00000000,
-	0x83c, 0xffffffff, 0x00000000,
-	0x848, 0xffffffff, 0x00000000,
-	0x84c, 0xffffffff, 0x00000000,
-	0x868, 0xffffffff, 0x00000000,
-	};
-
-u32 Rtl8723UMACPHY_Array_PG[Rtl8723UMACPHY_Array_PGLength] = {
-	0x0,
-};
diff --git a/drivers/staging/rtl8723au/hal/HalDMOutSrc8723A_CE.c b/drivers/staging/rtl8723au/hal/HalDMOutSrc8723A_CE.c
deleted file mode 100644
index 3f9ec9e00e16..000000000000
--- a/drivers/staging/rtl8723au/hal/HalDMOutSrc8723A_CE.c
+++ /dev/null
@@ -1,1097 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-/*  Description: */
-/*  This file is for 92CE/92CU dynamic mechanism only */
-
-/*  include files */
-
-#include "odm_precomp.h"
-#include <usb_ops_linux.h>
-
-#define		DPK_DELTA_MAPPING_NUM	13
-#define		index_mapping_HP_NUM	15
-/* 091212 chiyokolin */
-static void
-odm_TXPowerTrackingCallback_ThermalMeter_92C(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-	u8 ThermalValue = 0, delta, delta_LCK, delta_IQK, delta_HP;
-	int ele_A, ele_D, TempCCk, X, value32;
-	int Y, ele_C;
-	s8 OFDM_index[2], CCK_index = 0, OFDM_index_old[2] = {0};
-	s8 CCK_index_old = 0;
-	int i = 0;
-	u8 OFDM_min_index = 6, rf; /* OFDM BB Swing should be less than +3.0dB*/
-	u8 ThermalValue_HP_count = 0;
-	u32 ThermalValue_HP = 0;
-	s32 index_mapping_HP[index_mapping_HP_NUM] = {
-		0, 1, 3, 4, 6,
-		7, 9, 10, 12, 13,
-		15, 16, 18, 19, 21
-	};
-	s8 index_HP;
-
-	pdmpriv->TXPowerTrackingCallbackCnt++;	/* cosa add for debug */
-	pdmpriv->bTXPowerTrackingInit = true;
-
-	if (pHalData->CurrentChannel == 14 && !pdmpriv->bCCKinCH14)
-		pdmpriv->bCCKinCH14 = true;
-	else if (pHalData->CurrentChannel != 14 && pdmpriv->bCCKinCH14)
-		pdmpriv->bCCKinCH14 = false;
-
-	ThermalValue = (u8)PHY_QueryRFReg(Adapter, RF_PATH_A, RF_T_METER,
-					  0x1f);/*  0x24: RF Reg[4:0]	 */
-
-	rtl8723a_phy_ap_calibrate(Adapter, (ThermalValue -
-				  pHalData->EEPROMThermalMeter));
-
-	if (pHalData->rf_type == RF_2T2R)
-		rf = 2;
-	else
-		rf = 1;
-
-	if (ThermalValue) {
-		/* Query OFDM path A default setting	 */
-		ele_D = rtl8723au_read32(Adapter, rOFDM0_XATxIQImbalance) &
-			bMaskOFDM_D;
-		for (i = 0; i < OFDM_TABLE_SIZE_92C; i++) {
-			/* find the index */
-			if (ele_D == (OFDMSwingTable23A[i]&bMaskOFDM_D)) {
-				OFDM_index_old[0] = (u8)i;
-				break;
-			}
-		}
-
-		/* Query OFDM path B default setting  */
-		if (pHalData->rf_type == RF_2T2R) {
-			ele_D = rtl8723au_read32(Adapter,
-						 rOFDM0_XBTxIQImbalance);
-			ele_D &= bMaskOFDM_D;
-			for (i = 0; i < OFDM_TABLE_SIZE_92C; i++) {	/* find the index  */
-				if (ele_D == (OFDMSwingTable23A[i]&bMaskOFDM_D)) {
-					OFDM_index_old[1] = (u8)i;
-					break;
-				}
-			}
-		}
-
-		/* Query CCK default setting From 0xa24 */
-		TempCCk = rtl8723au_read32(Adapter, rCCK0_TxFilter2) & bMaskCCK;
-		for (i = 0 ; i < CCK_TABLE_SIZE ; i++) {
-			if (pdmpriv->bCCKinCH14) {
-				if (!memcmp(&TempCCk,
-					    &CCKSwingTable_Ch1423A[i][2], 4)) {
-					CCK_index_old = (u8)i;
-					break;
-				}
-			} else {
-				if (!memcmp(&TempCCk,
-					    &CCKSwingTable_Ch1_Ch1323A[i][2], 4)) {
-					CCK_index_old = (u8)i;
-					break;
-				}
-			}
-		}
-
-		if (!pdmpriv->ThermalValue) {
-			pdmpriv->ThermalValue = pHalData->EEPROMThermalMeter;
-			pdmpriv->ThermalValue_LCK = ThermalValue;
-			pdmpriv->ThermalValue_IQK = ThermalValue;
-			pdmpriv->ThermalValue_DPK = pHalData->EEPROMThermalMeter;
-
-			for (i = 0; i < rf; i++) {
-				pdmpriv->OFDM_index_HP[i] = OFDM_index_old[i];
-				pdmpriv->OFDM_index[i] = OFDM_index_old[i];
-			}
-			pdmpriv->CCK_index_HP = CCK_index_old;
-			pdmpriv->CCK_index = CCK_index_old;
-		}
-
-		if (pHalData->BoardType == BOARD_USB_High_PA) {
-			pdmpriv->ThermalValue_HP[pdmpriv->ThermalValue_HP_index] = ThermalValue;
-			pdmpriv->ThermalValue_HP_index++;
-			if (pdmpriv->ThermalValue_HP_index == HP_THERMAL_NUM)
-				pdmpriv->ThermalValue_HP_index = 0;
-
-			for (i = 0; i < HP_THERMAL_NUM; i++) {
-				if (pdmpriv->ThermalValue_HP[i]) {
-					ThermalValue_HP += pdmpriv->ThermalValue_HP[i];
-					ThermalValue_HP_count++;
-				}
-			}
-
-			if (ThermalValue_HP_count)
-				ThermalValue = (u8)(ThermalValue_HP / ThermalValue_HP_count);
-		}
-
-		delta = (ThermalValue > pdmpriv->ThermalValue) ?
-			(ThermalValue - pdmpriv->ThermalValue) :
-			(pdmpriv->ThermalValue - ThermalValue);
-		if (pHalData->BoardType == BOARD_USB_High_PA) {
-			if (pdmpriv->bDoneTxpower)
-				delta_HP = (ThermalValue > pdmpriv->ThermalValue) ?
-					   (ThermalValue - pdmpriv->ThermalValue) :
-					   (pdmpriv->ThermalValue - ThermalValue);
-			else
-				delta_HP = ThermalValue > pHalData->EEPROMThermalMeter ?
-					   (ThermalValue - pHalData->EEPROMThermalMeter) :
-					   (pHalData->EEPROMThermalMeter - ThermalValue);
-		} else {
-			delta_HP = 0;
-		}
-		delta_LCK = (ThermalValue > pdmpriv->ThermalValue_LCK) ?
-			    (ThermalValue - pdmpriv->ThermalValue_LCK) :
-			    (pdmpriv->ThermalValue_LCK - ThermalValue);
-		delta_IQK = (ThermalValue > pdmpriv->ThermalValue_IQK) ?
-			    (ThermalValue - pdmpriv->ThermalValue_IQK) :
-			    (pdmpriv->ThermalValue_IQK - ThermalValue);
-
-		if (delta_LCK > 1) {
-			pdmpriv->ThermalValue_LCK = ThermalValue;
-			rtl8723a_phy_lc_calibrate(Adapter);
-		}
-
-		if ((delta > 0 || delta_HP > 0) && pdmpriv->TxPowerTrackControl) {
-			if (pHalData->BoardType == BOARD_USB_High_PA) {
-				pdmpriv->bDoneTxpower = true;
-				delta_HP = ThermalValue > pHalData->EEPROMThermalMeter ?
-					   (ThermalValue - pHalData->EEPROMThermalMeter) :
-					   (pHalData->EEPROMThermalMeter - ThermalValue);
-
-				if (delta_HP > index_mapping_HP_NUM-1)
-					index_HP = index_mapping_HP[index_mapping_HP_NUM-1];
-				else
-					index_HP = index_mapping_HP[delta_HP];
-
-				if (ThermalValue > pHalData->EEPROMThermalMeter) {
-					/* set larger Tx power */
-					for (i = 0; i < rf; i++)
-						OFDM_index[i] = pdmpriv->OFDM_index_HP[i] - index_HP;
-					CCK_index = pdmpriv->CCK_index_HP - index_HP;
-				} else {
-					for (i = 0; i < rf; i++)
-						OFDM_index[i] = pdmpriv->OFDM_index_HP[i] + index_HP;
-					CCK_index = pdmpriv->CCK_index_HP + index_HP;
-				}
-
-				delta_HP = (ThermalValue > pdmpriv->ThermalValue) ?
-					   (ThermalValue - pdmpriv->ThermalValue) :
-					   (pdmpriv->ThermalValue - ThermalValue);
-			} else {
-				if (ThermalValue > pdmpriv->ThermalValue) {
-					for (i = 0; i < rf; i++)
-						pdmpriv->OFDM_index[i] -= delta;
-					pdmpriv->CCK_index -= delta;
-				} else {
-					for (i = 0; i < rf; i++)
-						pdmpriv->OFDM_index[i] += delta;
-					pdmpriv->CCK_index += delta;
-				}
-			}
-
-			/* no adjust */
-			if (pHalData->BoardType != BOARD_USB_High_PA) {
-				if (ThermalValue > pHalData->EEPROMThermalMeter) {
-					for (i = 0; i < rf; i++)
-						OFDM_index[i] = pdmpriv->OFDM_index[i]+1;
-					CCK_index = pdmpriv->CCK_index+1;
-				} else {
-					for (i = 0; i < rf; i++)
-						OFDM_index[i] = pdmpriv->OFDM_index[i];
-					CCK_index = pdmpriv->CCK_index;
-				}
-			}
-			for (i = 0; i < rf; i++) {
-				if (OFDM_index[i] > (OFDM_TABLE_SIZE_92C-1))
-					OFDM_index[i] = (OFDM_TABLE_SIZE_92C-1);
-				else if (OFDM_index[i] < OFDM_min_index)
-					OFDM_index[i] = OFDM_min_index;
-			}
-
-			if (CCK_index > (CCK_TABLE_SIZE-1))
-				CCK_index = CCK_TABLE_SIZE-1;
-			else if (CCK_index < 0)
-				CCK_index = 0;
-		}
-
-		if (pdmpriv->TxPowerTrackControl &&
-		    (delta != 0 || delta_HP != 0)) {
-			/* Adujst OFDM Ant_A according to IQK result */
-			ele_D = (OFDMSwingTable23A[OFDM_index[0]] & 0xFFC00000)>>22;
-			X = pdmpriv->RegE94;
-			Y = pdmpriv->RegE9C;
-
-			if (X != 0) {
-				if ((X & 0x00000200) != 0)
-					X = X | 0xFFFFFC00;
-				ele_A = ((X * ele_D)>>8)&0x000003FF;
-
-				/* new element C = element D x Y */
-				if ((Y & 0x00000200) != 0)
-					Y = Y | 0xFFFFFC00;
-				ele_C = ((Y * ele_D)>>8)&0x000003FF;
-
-				/* write new elements A, C, D to regC80 and regC94, element B is always 0 */
-				value32 = (ele_D<<22)|((ele_C&0x3F)<<16)|ele_A;
-				rtl8723au_write32(Adapter,
-						  rOFDM0_XATxIQImbalance,
-						  value32);
-
-				value32 = (ele_C&0x000003C0)>>6;
-				PHY_SetBBReg(Adapter, rOFDM0_XCTxAFE, bMaskH4Bits, value32);
-
-				value32 = ((X * ele_D)>>7)&0x01;
-				PHY_SetBBReg(Adapter, rOFDM0_ECCAThreshold,
-					     BIT(31), value32);
-
-				value32 = ((Y * ele_D)>>7)&0x01;
-				PHY_SetBBReg(Adapter, rOFDM0_ECCAThreshold,
-					     BIT(29), value32);
-			} else {
-				rtl8723au_write32(Adapter,
-						  rOFDM0_XATxIQImbalance,
-						  OFDMSwingTable23A[OFDM_index[0]]);
-				PHY_SetBBReg(Adapter, rOFDM0_XCTxAFE,
-					     bMaskH4Bits, 0x00);
-				PHY_SetBBReg(Adapter, rOFDM0_ECCAThreshold,
-					     BIT(31) | BIT(29), 0x00);
-			}
-
-			/* Adjust CCK according to IQK result */
-			if (!pdmpriv->bCCKinCH14) {
-				rtl8723au_write8(Adapter, 0xa22, CCKSwingTable_Ch1_Ch1323A[CCK_index][0]);
-				rtl8723au_write8(Adapter, 0xa23, CCKSwingTable_Ch1_Ch1323A[CCK_index][1]);
-				rtl8723au_write8(Adapter, 0xa24, CCKSwingTable_Ch1_Ch1323A[CCK_index][2]);
-				rtl8723au_write8(Adapter, 0xa25, CCKSwingTable_Ch1_Ch1323A[CCK_index][3]);
-				rtl8723au_write8(Adapter, 0xa26, CCKSwingTable_Ch1_Ch1323A[CCK_index][4]);
-				rtl8723au_write8(Adapter, 0xa27, CCKSwingTable_Ch1_Ch1323A[CCK_index][5]);
-				rtl8723au_write8(Adapter, 0xa28, CCKSwingTable_Ch1_Ch1323A[CCK_index][6]);
-				rtl8723au_write8(Adapter, 0xa29, CCKSwingTable_Ch1_Ch1323A[CCK_index][7]);
-			} else {
-				rtl8723au_write8(Adapter, 0xa22, CCKSwingTable_Ch1423A[CCK_index][0]);
-				rtl8723au_write8(Adapter, 0xa23, CCKSwingTable_Ch1423A[CCK_index][1]);
-				rtl8723au_write8(Adapter, 0xa24, CCKSwingTable_Ch1423A[CCK_index][2]);
-				rtl8723au_write8(Adapter, 0xa25, CCKSwingTable_Ch1423A[CCK_index][3]);
-				rtl8723au_write8(Adapter, 0xa26, CCKSwingTable_Ch1423A[CCK_index][4]);
-				rtl8723au_write8(Adapter, 0xa27, CCKSwingTable_Ch1423A[CCK_index][5]);
-				rtl8723au_write8(Adapter, 0xa28, CCKSwingTable_Ch1423A[CCK_index][6]);
-				rtl8723au_write8(Adapter, 0xa29, CCKSwingTable_Ch1423A[CCK_index][7]);
-			}
-
-			if (pHalData->rf_type == RF_2T2R) {
-				ele_D = (OFDMSwingTable23A[(u8)OFDM_index[1]] & 0xFFC00000)>>22;
-
-				/* new element A = element D x X */
-				X = pdmpriv->RegEB4;
-				Y = pdmpriv->RegEBC;
-
-				if (X != 0) {
-					if ((X & 0x00000200) != 0)	/* consider minus */
-						X = X | 0xFFFFFC00;
-					ele_A = ((X * ele_D)>>8)&0x000003FF;
-
-					/* new element C = element D x Y */
-					if ((Y & 0x00000200) != 0)
-						Y = Y | 0xFFFFFC00;
-					ele_C = ((Y * ele_D)>>8)&0x00003FF;
-
-					/* write new elements A, C, D to regC88 and regC9C, element B is always 0 */
-					value32 = (ele_D<<22)|((ele_C&0x3F)<<16) | ele_A;
-					rtl8723au_write32(Adapter, rOFDM0_XBTxIQImbalance, value32);
-
-					value32 = (ele_C&0x000003C0)>>6;
-					PHY_SetBBReg(Adapter, rOFDM0_XDTxAFE, bMaskH4Bits, value32);
-
-					value32 = ((X * ele_D)>>7)&0x01;
-					PHY_SetBBReg(Adapter,
-						     rOFDM0_ECCAThreshold,
-						     BIT(27), value32);
-
-					value32 = ((Y * ele_D)>>7)&0x01;
-					PHY_SetBBReg(Adapter,
-						     rOFDM0_ECCAThreshold,
-						     BIT(25), value32);
-				} else {
-					rtl8723au_write32(Adapter,
-							  rOFDM0_XBTxIQImbalance,
-							  OFDMSwingTable23A[OFDM_index[1]]);
-					PHY_SetBBReg(Adapter,
-						     rOFDM0_XDTxAFE,
-						     bMaskH4Bits, 0x00);
-					PHY_SetBBReg(Adapter,
-						     rOFDM0_ECCAThreshold,
-						     BIT(27) | BIT(25), 0x00);
-				}
-			}
-
-		}
-		if (delta_IQK > 3) {
-			pdmpriv->ThermalValue_IQK = ThermalValue;
-			rtl8723a_phy_iq_calibrate(Adapter, false);
-		}
-
-		/* update thermal meter value */
-		if (pdmpriv->TxPowerTrackControl)
-			pdmpriv->ThermalValue = ThermalValue;
-	}
-	pdmpriv->TXPowercount = 0;
-}
-
-/*	Description: */
-/*		- Dispatch TxPower Tracking direct call ONLY for 92s. */
-/*		- We shall NOT schedule Workitem within PASSIVE LEVEL, which will cause system resource */
-/*		   leakage under some platform. */
-/*	Assumption: */
-/*		PASSIVE_LEVEL when this routine is called. */
-static void ODM_TXPowerTracking92CDirectCall(struct rtw_adapter *Adapter)
-{
-	odm_TXPowerTrackingCallback_ThermalMeter_92C(Adapter);
-}
-
-void rtl8723a_odm_check_tx_power_tracking(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-
-	if (!pdmpriv->TM_Trigger) {		/* at least delay 1 sec */
-		PHY_SetRFReg(Adapter, RF_PATH_A, RF_T_METER, bRFRegOffsetMask, 0x60);
-
-		pdmpriv->TM_Trigger = 1;
-		return;
-	} else {
-		ODM_TXPowerTracking92CDirectCall(Adapter);
-		pdmpriv->TM_Trigger = 0;
-	}
-}
-
-/*	IQK */
-#define MAX_TOLERANCE		5
-#define IQK_DELAY_TIME		1	/* ms */
-
-static u8 _PHY_PathA_IQK(struct rtw_adapter *pAdapter, bool configPathB)
-{
-	u32 regEAC, regE94, regE9C, regEA4;
-	u8 result = 0x00;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter);
-
-	/* path-A IQK setting */
-	rtl8723au_write32(pAdapter, rTx_IQK_Tone_A, 0x10008c1f);
-	rtl8723au_write32(pAdapter, rRx_IQK_Tone_A, 0x10008c1f);
-	rtl8723au_write32(pAdapter, rTx_IQK_PI_A, 0x82140102);
-
-	rtl8723au_write32(pAdapter, rRx_IQK_PI_A, configPathB ? 0x28160202 :
-		IS_81xxC_VENDOR_UMC_B_CUT(pHalData->VersionID)?0x28160202:0x28160502);
-
-	/* path-B IQK setting */
-	if (configPathB) {
-		rtl8723au_write32(pAdapter, rTx_IQK_Tone_B, 0x10008c22);
-		rtl8723au_write32(pAdapter, rRx_IQK_Tone_B, 0x10008c22);
-		rtl8723au_write32(pAdapter, rTx_IQK_PI_B, 0x82140102);
-		rtl8723au_write32(pAdapter, rRx_IQK_PI_B, 0x28160202);
-	}
-
-	/* LO calibration setting */
-	rtl8723au_write32(pAdapter, rIQK_AGC_Rsp, 0x001028d1);
-
-	/* One shot, path A LOK & IQK */
-	rtl8723au_write32(pAdapter, rIQK_AGC_Pts, 0xf9000000);
-	rtl8723au_write32(pAdapter, rIQK_AGC_Pts, 0xf8000000);
-
-	/*  delay x ms */
-	/* PlatformStallExecution(IQK_DELAY_TIME*1000); */
-	udelay(IQK_DELAY_TIME*1000);
-
-	/*  Check failed */
-	regEAC = rtl8723au_read32(pAdapter, rRx_Power_After_IQK_A_2);
-	regE94 = rtl8723au_read32(pAdapter, rTx_Power_Before_IQK_A);
-	regE9C = rtl8723au_read32(pAdapter, rTx_Power_After_IQK_A);
-	regEA4 = rtl8723au_read32(pAdapter, rRx_Power_Before_IQK_A_2);
-
-	if (!(regEAC & BIT(28)) &&
-	    (((regE94 & 0x03FF0000)>>16) != 0x142) &&
-	    (((regE9C & 0x03FF0000)>>16) != 0x42))
-		result |= 0x01;
-	else			/* if Tx not OK, ignore Rx */
-		return result;
-
-	if (!(regEAC & BIT(27)) && /* if Tx is OK, check whether Rx is OK */
-	    (((regEA4 & 0x03FF0000)>>16) != 0x132) &&
-	    (((regEAC & 0x03FF0000)>>16) != 0x36))
-		result |= 0x02;
-	else
-		DBG_8723A("Path A Rx IQK fail!!\n");
-	return result;
-}
-
-static u8 _PHY_PathB_IQK(struct rtw_adapter *pAdapter)
-{
-	u32 regEAC, regEB4, regEBC, regEC4, regECC;
-	u8 result = 0x00;
-
-	/* One shot, path B LOK & IQK */
-	rtl8723au_write32(pAdapter, rIQK_AGC_Cont, 0x00000002);
-	rtl8723au_write32(pAdapter, rIQK_AGC_Cont, 0x00000000);
-
-	/*  delay x ms */
-	udelay(IQK_DELAY_TIME*1000);
-
-	/*  Check failed */
-	regEAC = rtl8723au_read32(pAdapter, rRx_Power_After_IQK_A_2);
-	regEB4 = rtl8723au_read32(pAdapter, rTx_Power_Before_IQK_B);
-	regEBC = rtl8723au_read32(pAdapter, rTx_Power_After_IQK_B);
-	regEC4 = rtl8723au_read32(pAdapter, rRx_Power_Before_IQK_B_2);
-	regECC = rtl8723au_read32(pAdapter, rRx_Power_After_IQK_B_2);
-
-	if (!(regEAC & BIT(31)) &&
-	    (((regEB4 & 0x03FF0000)>>16) != 0x142) &&
-	    (((regEBC & 0x03FF0000)>>16) != 0x42))
-		result |= 0x01;
-	else
-		return result;
-
-	if (!(regEAC & BIT(30)) &&
-	    (((regEC4 & 0x03FF0000)>>16) != 0x132) &&
-	    (((regECC & 0x03FF0000)>>16) != 0x36))
-		result |= 0x02;
-	else
-		DBG_8723A("Path B Rx IQK fail!!\n");
-	return result;
-}
-
-static void _PHY_PathAFillIQKMatrix(struct rtw_adapter *pAdapter,
-	bool bIQKOK,
-	int result[][8],
-	u8 final_candidate,
-	bool bTxOnly
-	)
-{
-	u32 Oldval_0, X, TX0_A, reg;
-	s32 Y, TX0_C;
-
-	DBG_8723A("Path A IQ Calibration %s !\n", (bIQKOK)?"Success":"Failed");
-
-	if (final_candidate == 0xFF) {
-		return;
-	} else if (bIQKOK) {
-		Oldval_0 = rtl8723au_read32(pAdapter, rOFDM0_XATxIQImbalance);
-		Oldval_0 = (Oldval_0 >> 22) & 0x3FF;
-
-		X = result[final_candidate][0];
-		if ((X & 0x00000200) != 0)
-			X = X | 0xFFFFFC00;
-		TX0_A = (X * Oldval_0) >> 8;
-		PHY_SetBBReg(pAdapter, rOFDM0_XATxIQImbalance, 0x3FF, TX0_A);
-		PHY_SetBBReg(pAdapter, rOFDM0_ECCAThreshold, BIT(31),
-			     ((X * Oldval_0>>7) & 0x1));
-
-		Y = result[final_candidate][1];
-		if ((Y & 0x00000200) != 0)
-			Y = Y | 0xFFFFFC00;
-		TX0_C = (Y * Oldval_0) >> 8;
-		PHY_SetBBReg(pAdapter, rOFDM0_XCTxAFE, 0xF0000000,
-			     ((TX0_C&0x3C0)>>6));
-		PHY_SetBBReg(pAdapter, rOFDM0_XATxIQImbalance, 0x003F0000,
-			     (TX0_C&0x3F));
-		PHY_SetBBReg(pAdapter, rOFDM0_ECCAThreshold, BIT(29),
-			     ((Y * Oldval_0>>7) & 0x1));
-
-		if (bTxOnly) {
-			DBG_8723A("_PHY_PathAFillIQKMatrix only Tx OK\n");
-			return;
-		}
-
-		reg = result[final_candidate][2];
-		PHY_SetBBReg(pAdapter, rOFDM0_XARxIQImbalance, 0x3FF, reg);
-
-		reg = result[final_candidate][3] & 0x3F;
-		PHY_SetBBReg(pAdapter, rOFDM0_XARxIQImbalance, 0xFC00, reg);
-
-		reg = (result[final_candidate][3] >> 6) & 0xF;
-		PHY_SetBBReg(pAdapter, rOFDM0_RxIQExtAnta, 0xF0000000, reg);
-	}
-}
-
-static void _PHY_PathBFillIQKMatrix(struct rtw_adapter *pAdapter, bool bIQKOK, int result[][8], u8 final_candidate, bool bTxOnly)
-{
-	u32 Oldval_1, X, TX1_A, reg;
-	s32 Y, TX1_C;
-
-	DBG_8723A("Path B IQ Calibration %s !\n", (bIQKOK)?"Success":"Failed");
-
-	if (final_candidate == 0xFF) {
-		return;
-	} else if (bIQKOK) {
-		Oldval_1 = rtl8723au_read32(pAdapter, rOFDM0_XBTxIQImbalance);
-		Oldval_1 = (Oldval_1 >> 22) & 0x3FF;
-
-		X = result[final_candidate][4];
-		if ((X & 0x00000200) != 0)
-			X = X | 0xFFFFFC00;
-		TX1_A = (X * Oldval_1) >> 8;
-		PHY_SetBBReg(pAdapter, rOFDM0_XBTxIQImbalance, 0x3FF, TX1_A);
-		PHY_SetBBReg(pAdapter, rOFDM0_ECCAThreshold, BIT(27),
-			     ((X * Oldval_1 >> 7) & 0x1));
-
-		Y = result[final_candidate][5];
-		if ((Y & 0x00000200) != 0)
-			Y = Y | 0xFFFFFC00;
-		TX1_C = (Y * Oldval_1) >> 8;
-		PHY_SetBBReg(pAdapter, rOFDM0_XDTxAFE, 0xF0000000,
-			     ((TX1_C & 0x3C0) >> 6));
-		PHY_SetBBReg(pAdapter, rOFDM0_XBTxIQImbalance, 0x003F0000,
-			     (TX1_C & 0x3F));
-		PHY_SetBBReg(pAdapter, rOFDM0_ECCAThreshold, BIT(25),
-			     ((Y * Oldval_1 >> 7) & 0x1));
-
-		if (bTxOnly)
-			return;
-
-		reg = result[final_candidate][6];
-		PHY_SetBBReg(pAdapter, rOFDM0_XBRxIQImbalance, 0x3FF, reg);
-
-		reg = result[final_candidate][7] & 0x3F;
-		PHY_SetBBReg(pAdapter, rOFDM0_XBRxIQImbalance, 0xFC00, reg);
-
-		reg = (result[final_candidate][7] >> 6) & 0xF;
-		PHY_SetBBReg(pAdapter, rOFDM0_AGCRSSITable, 0x0000F000, reg);
-	}
-}
-
-static void _PHY_SaveADDARegisters(struct rtw_adapter *pAdapter, u32 *ADDAReg, u32 *ADDABackup, u32 RegisterNum)
-{
-	u32 i;
-
-	for (i = 0 ; i < RegisterNum ; i++) {
-		ADDABackup[i] = rtl8723au_read32(pAdapter, ADDAReg[i]);
-	}
-}
-
-static void _PHY_SaveMACRegisters(struct rtw_adapter *pAdapter, u32 *MACReg,
-				  u32 *MACBackup)
-{
-	u32 i;
-
-	for (i = 0 ; i < (IQK_MAC_REG_NUM - 1); i++) {
-		MACBackup[i] = rtl8723au_read8(pAdapter, MACReg[i]);
-	}
-	MACBackup[i] = rtl8723au_read32(pAdapter, MACReg[i]);
-}
-
-static void _PHY_ReloadADDARegisters(struct rtw_adapter *pAdapter,
-				     u32 *ADDAReg, u32 *ADDABackup,
-				     u32 RegiesterNum)
-{
-	u32 i;
-
-	for (i = 0 ; i < RegiesterNum ; i++) {
-		rtl8723au_write32(pAdapter, ADDAReg[i], ADDABackup[i]);
-	}
-}
-
-static void _PHY_ReloadMACRegisters(struct rtw_adapter *pAdapter,
-				    u32 *MACReg, u32 *MACBackup)
-{
-	u32 i;
-
-	for (i = 0 ; i < (IQK_MAC_REG_NUM - 1); i++)
-		rtl8723au_write8(pAdapter, MACReg[i], (u8)MACBackup[i]);
-
-	rtl8723au_write32(pAdapter, MACReg[i], MACBackup[i]);
-}
-
-static void _PHY_PathADDAOn(struct rtw_adapter *pAdapter, u32 *ADDAReg,
-			    bool isPathAOn, bool is2T)
-{
-	u32 pathOn;
-	u32 i;
-
-	pathOn = isPathAOn ? 0x04db25a4 : 0x0b1b25a4;
-	if (!is2T) {
-		pathOn = 0x0bdb25a0;
-		rtl8723au_write32(pAdapter, ADDAReg[0], 0x0b1b25a0);
-	} else {
-		rtl8723au_write32(pAdapter, ADDAReg[0], pathOn);
-	}
-
-	for (i = 1 ; i < IQK_ADDA_REG_NUM ; i++)
-		rtl8723au_write32(pAdapter, ADDAReg[i], pathOn);
-}
-
-static void _PHY_MACSettingCalibration(struct rtw_adapter *pAdapter,
-				       u32 *MACReg, u32 *MACBackup)
-{
-	u32 i = 0;
-
-	rtl8723au_write8(pAdapter, MACReg[i], 0x3F);
-
-	for (i = 1 ; i < (IQK_MAC_REG_NUM - 1); i++) {
-		rtl8723au_write8(pAdapter, MACReg[i],
-				 (u8)(MACBackup[i] & ~BIT(3)));
-	}
-	rtl8723au_write8(pAdapter, MACReg[i], (u8)(MACBackup[i] & ~BIT(5)));
-}
-
-static void _PHY_PathAStandBy(struct rtw_adapter *pAdapter)
-{
-	rtl8723au_write32(pAdapter, rFPGA0_IQK, 0x0);
-	rtl8723au_write32(pAdapter, 0x840, 0x00010000);
-	rtl8723au_write32(pAdapter, rFPGA0_IQK, 0x80800000);
-}
-
-static void _PHY_PIModeSwitch(struct rtw_adapter *pAdapter, bool PIMode)
-{
-	u32 mode;
-
-	mode = PIMode ? 0x01000100 : 0x01000000;
-	rtl8723au_write32(pAdapter, 0x820, mode);
-	rtl8723au_write32(pAdapter, 0x828, mode);
-}
-
-/*
-return false => do IQK again
-*/
-static bool _PHY_SimularityCompare(struct rtw_adapter *pAdapter, int result[][8], u8 c1, u8 c2)
-{
-	u32 i, j, diff, SimularityBitMap, bound = 0;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter);
-	u8 final_candidate[2] = {0xFF, 0xFF};	/* for path A and path B */
-	bool bResult = true;
-
-	if (pHalData->rf_type == RF_2T2R)
-		bound = 8;
-	else
-		bound = 4;
-
-	SimularityBitMap = 0;
-
-	for (i = 0; i < bound; i++) {
-		diff = (result[c1][i] > result[c2][i]) ? (result[c1][i] - result[c2][i]) : (result[c2][i] - result[c1][i]);
-		if (diff > MAX_TOLERANCE) {
-			if ((i == 2 || i == 6) && !SimularityBitMap) {
-				if (result[c1][i]+result[c1][i+1] == 0)
-					final_candidate[(i/4)] = c2;
-				else if (result[c2][i]+result[c2][i+1] == 0)
-					final_candidate[(i/4)] = c1;
-				else
-					SimularityBitMap = SimularityBitMap|(1<<i);
-			} else {
-				SimularityBitMap = SimularityBitMap|(1<<i);
-			}
-		}
-	}
-
-	if (SimularityBitMap == 0) {
-		for (i = 0; i < (bound/4); i++) {
-			if (final_candidate[i] != 0xFF) {
-				for (j = i*4; j < (i+1)*4-2; j++)
-					result[3][j] = result[final_candidate[i]][j];
-				bResult = false;
-			}
-		}
-		return bResult;
-	} else if (!(SimularityBitMap & 0x0F)) {
-		/* path A OK */
-		for (i = 0; i < 4; i++)
-			result[3][i] = result[c1][i];
-		return false;
-	} else if (!(SimularityBitMap & 0xF0) && pHalData->rf_type == RF_2T2R) {
-		/* path B OK */
-		for (i = 4; i < 8; i++)
-			result[3][i] = result[c1][i];
-		return false;
-	} else {
-		return false;
-	}
-}
-
-static void _PHY_IQCalibrate(struct rtw_adapter *pAdapter, int result[][8], u8 t, bool is2T)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-	u32 i;
-	u8 PathAOK, PathBOK;
-	u32 ADDA_REG[IQK_ADDA_REG_NUM] = {
-		rFPGA0_XCD_SwitchControl, rBlue_Tooth,
-		rRx_Wait_CCA, rTx_CCK_RFON,
-		rTx_CCK_BBON, rTx_OFDM_RFON,
-		rTx_OFDM_BBON, rTx_To_Rx,
-		rTx_To_Tx, rRx_CCK,
-		rRx_OFDM, rRx_Wait_RIFS,
-		rRx_TO_Rx, rStandby,
-		rSleep, rPMPD_ANAEN
-	};
-
-	u32 IQK_MAC_REG[IQK_MAC_REG_NUM] = {
-		REG_TXPAUSE, REG_BCN_CTRL,
-		REG_BCN_CTRL_1, REG_GPIO_MUXCFG
-	};
-
-	u32 IQK_BB_REG_92C[IQK_BB_REG_NUM] = {
-		rOFDM0_TRxPathEnable, rOFDM0_TRMuxPar,
-		rFPGA0_XCD_RFInterfaceSW, rConfig_AntA, rConfig_AntB,
-		rFPGA0_XAB_RFInterfaceSW, rFPGA0_XA_RFInterfaceOE,
-		rFPGA0_XB_RFInterfaceOE, rFPGA0_RFMOD
-	};
-
-	const u32 retryCount = 2;
-
-	/*  Note: IQ calibration must be performed after loading  */
-	/*		PHY_REG.txt , and radio_a, radio_b.txt	 */
-
-	u32 bbvalue;
-
-	if (t == 0) {
-		bbvalue = rtl8723au_read32(pAdapter, rFPGA0_RFMOD);
-
-		/*  Save ADDA parameters, turn Path A ADDA on */
-		_PHY_SaveADDARegisters(pAdapter, ADDA_REG, pdmpriv->ADDA_backup, IQK_ADDA_REG_NUM);
-		_PHY_SaveMACRegisters(pAdapter, IQK_MAC_REG, pdmpriv->IQK_MAC_backup);
-		_PHY_SaveADDARegisters(pAdapter, IQK_BB_REG_92C, pdmpriv->IQK_BB_backup, IQK_BB_REG_NUM);
-	}
-	_PHY_PathADDAOn(pAdapter, ADDA_REG, true, is2T);
-
-	if (t == 0)
-		pdmpriv->bRfPiEnable = (u8)
-			PHY_QueryBBReg(pAdapter, rFPGA0_XA_HSSIParameter1,
-				       BIT(8));
-
-	if (!pdmpriv->bRfPiEnable) {
-		/*  Switch BB to PI mode to do IQ Calibration. */
-		_PHY_PIModeSwitch(pAdapter, true);
-	}
-
-	PHY_SetBBReg(pAdapter, rFPGA0_RFMOD, BIT(24), 0x00);
-	rtl8723au_write32(pAdapter, rOFDM0_TRxPathEnable, 0x03a05600);
-	rtl8723au_write32(pAdapter, rOFDM0_TRMuxPar, 0x000800e4);
-	rtl8723au_write32(pAdapter, rFPGA0_XCD_RFInterfaceSW, 0x22204000);
-	PHY_SetBBReg(pAdapter, rFPGA0_XAB_RFInterfaceSW, BIT(10), 0x01);
-	PHY_SetBBReg(pAdapter, rFPGA0_XAB_RFInterfaceSW, BIT(26), 0x01);
-	PHY_SetBBReg(pAdapter, rFPGA0_XA_RFInterfaceOE, BIT(10), 0x00);
-	PHY_SetBBReg(pAdapter, rFPGA0_XB_RFInterfaceOE, BIT(10), 0x00);
-
-	if (is2T) {
-		rtl8723au_write32(pAdapter,
-				  rFPGA0_XA_LSSIParameter, 0x00010000);
-		rtl8723au_write32(pAdapter,
-				  rFPGA0_XB_LSSIParameter, 0x00010000);
-	}
-
-	/* MAC settings */
-	_PHY_MACSettingCalibration(pAdapter, IQK_MAC_REG, pdmpriv->IQK_MAC_backup);
-
-	/* Page B init */
-	rtl8723au_write32(pAdapter, rConfig_AntA, 0x00080000);
-
-	if (is2T)
-		rtl8723au_write32(pAdapter, rConfig_AntB, 0x00080000);
-
-	/*  IQ calibration setting */
-	rtl8723au_write32(pAdapter, rFPGA0_IQK, 0x80800000);
-	rtl8723au_write32(pAdapter, rTx_IQK, 0x01007c00);
-	rtl8723au_write32(pAdapter, rRx_IQK, 0x01004800);
-
-	for (i = 0 ; i < retryCount ; i++) {
-		PathAOK = _PHY_PathA_IQK(pAdapter, is2T);
-		if (PathAOK == 0x03) {
-				DBG_8723A("Path A IQK Success!!\n");
-				result[t][0] = (rtl8723au_read32(pAdapter, rTx_Power_Before_IQK_A)&0x3FF0000)>>16;
-				result[t][1] = (rtl8723au_read32(pAdapter, rTx_Power_After_IQK_A)&0x3FF0000)>>16;
-				result[t][2] = (rtl8723au_read32(pAdapter, rRx_Power_Before_IQK_A_2)&0x3FF0000)>>16;
-				result[t][3] = (rtl8723au_read32(pAdapter, rRx_Power_After_IQK_A_2)&0x3FF0000)>>16;
-			break;
-		} else if (i == (retryCount-1) && PathAOK == 0x01) {
-			/* Tx IQK OK */
-			DBG_8723A("Path A IQK Only  Tx Success!!\n");
-
-			result[t][0] = (rtl8723au_read32(pAdapter, rTx_Power_Before_IQK_A)&0x3FF0000)>>16;
-			result[t][1] = (rtl8723au_read32(pAdapter, rTx_Power_After_IQK_A)&0x3FF0000)>>16;
-		}
-	}
-
-	if (0x00 == PathAOK) {
-		DBG_8723A("Path A IQK failed!!\n");
-	}
-
-	if (is2T) {
-		_PHY_PathAStandBy(pAdapter);
-
-		/*  Turn Path B ADDA on */
-		_PHY_PathADDAOn(pAdapter, ADDA_REG, false, is2T);
-
-		for (i = 0 ; i < retryCount ; i++) {
-			PathBOK = _PHY_PathB_IQK(pAdapter);
-			if (PathBOK == 0x03) {
-				DBG_8723A("Path B IQK Success!!\n");
-				result[t][4] = (rtl8723au_read32(pAdapter, rTx_Power_Before_IQK_B)&0x3FF0000)>>16;
-				result[t][5] = (rtl8723au_read32(pAdapter, rTx_Power_After_IQK_B)&0x3FF0000)>>16;
-				result[t][6] = (rtl8723au_read32(pAdapter, rRx_Power_Before_IQK_B_2)&0x3FF0000)>>16;
-				result[t][7] = (rtl8723au_read32(pAdapter, rRx_Power_After_IQK_B_2)&0x3FF0000)>>16;
-				break;
-			} else if (i == (retryCount - 1) && PathBOK == 0x01) {
-				/* Tx IQK OK */
-				DBG_8723A("Path B Only Tx IQK Success!!\n");
-				result[t][4] = (rtl8723au_read32(pAdapter, rTx_Power_Before_IQK_B)&0x3FF0000)>>16;
-				result[t][5] = (rtl8723au_read32(pAdapter, rTx_Power_After_IQK_B)&0x3FF0000)>>16;
-			}
-		}
-
-		if (0x00 == PathBOK) {
-			DBG_8723A("Path B IQK failed!!\n");
-		}
-	}
-
-	/* Back to BB mode, load original value */
-	rtl8723au_write32(pAdapter, rFPGA0_IQK, 0);
-
-	if (t != 0) {
-		if (!pdmpriv->bRfPiEnable) {
-			/*  Switch back BB to SI mode after finish IQ Calibration. */
-			_PHY_PIModeSwitch(pAdapter, false);
-		}
-
-		/*  Reload ADDA power saving parameters */
-		_PHY_ReloadADDARegisters(pAdapter, ADDA_REG, pdmpriv->ADDA_backup, IQK_ADDA_REG_NUM);
-
-		/*  Reload MAC parameters */
-		_PHY_ReloadMACRegisters(pAdapter, IQK_MAC_REG, pdmpriv->IQK_MAC_backup);
-
-		/*  Reload BB parameters */
-		_PHY_ReloadADDARegisters(pAdapter, IQK_BB_REG_92C, pdmpriv->IQK_BB_backup, IQK_BB_REG_NUM);
-
-		/*  Restore RX initial gain */
-		rtl8723au_write32(pAdapter,
-				  rFPGA0_XA_LSSIParameter, 0x00032ed3);
-		if (is2T) {
-			rtl8723au_write32(pAdapter,
-					  rFPGA0_XB_LSSIParameter, 0x00032ed3);
-		}
-
-		/* load 0xe30 IQC default value */
-		rtl8723au_write32(pAdapter, rTx_IQK_Tone_A, 0x01008c00);
-		rtl8723au_write32(pAdapter, rRx_IQK_Tone_A, 0x01008c00);
-
-	}
-}
-
-static void _PHY_LCCalibrate(struct rtw_adapter *pAdapter, bool is2T)
-{
-	u8 tmpReg;
-	u32 RF_Amode = 0, RF_Bmode = 0, LC_Cal;
-
-	/* Check continuous TX and Packet TX */
-	tmpReg = rtl8723au_read8(pAdapter, 0xd03);
-
-	if ((tmpReg&0x70) != 0) {
-		/* Deal with contisuous TX case */
-		/* disable all continuous TX */
-		rtl8723au_write8(pAdapter, 0xd03, tmpReg&0x8F);
-	} else {
-		/*  Deal with Packet TX case */
-		/*  block all queues */
-		rtl8723au_write8(pAdapter, REG_TXPAUSE, 0xFF);
-	}
-
-	if ((tmpReg&0x70) != 0) {
-		/* 1. Read original RF mode */
-		/* Path-A */
-		RF_Amode = PHY_QueryRFReg(pAdapter, RF_PATH_A, RF_AC, bMask12Bits);
-
-		/* Path-B */
-		if (is2T)
-			RF_Bmode = PHY_QueryRFReg(pAdapter, RF_PATH_B, RF_AC, bMask12Bits);
-
-		/* 2. Set RF mode = standby mode */
-		/* Path-A */
-		PHY_SetRFReg(pAdapter, RF_PATH_A, RF_AC, bMask12Bits, (RF_Amode&0x8FFFF)|0x10000);
-
-		/* Path-B */
-		if (is2T)
-			PHY_SetRFReg(pAdapter, RF_PATH_B, RF_AC, bMask12Bits, (RF_Bmode&0x8FFFF)|0x10000);
-	}
-
-	/* 3. Read RF reg18 */
-	LC_Cal = PHY_QueryRFReg(pAdapter, RF_PATH_A, RF_CHNLBW, bMask12Bits);
-
-	/* 4. Set LC calibration begin */
-	PHY_SetRFReg(pAdapter, RF_PATH_A, RF_CHNLBW, bMask12Bits, LC_Cal|0x08000);
-
-	msleep(100);
-
-	/* Restore original situation */
-	if ((tmpReg&0x70) != 0) {	/* Deal with contuous TX case  */
-		/* Path-A */
-		rtl8723au_write8(pAdapter, 0xd03, tmpReg);
-		PHY_SetRFReg(pAdapter, RF_PATH_A, RF_AC, bMask12Bits, RF_Amode);
-
-		/* Path-B */
-		if (is2T)
-			PHY_SetRFReg(pAdapter, RF_PATH_B, RF_AC, bMask12Bits, RF_Bmode);
-	} else /*  Deal with Packet TX case */
-		rtl8723au_write8(pAdapter, REG_TXPAUSE, 0x00);
-}
-
-/* Analog Pre-distortion calibration */
-#define		APK_BB_REG_NUM	8
-#define		APK_CURVE_REG_NUM 4
-#define		PATH_NUM		2
-
-void rtl8723a_phy_iq_calibrate(struct rtw_adapter *pAdapter, bool bReCovery)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-	s32 result[4][8];	/* last is final result */
-	u8 i, final_candidate;
-	bool bPathAOK, bPathBOK;
-	s32 RegE94, RegE9C, RegEA4, RegEAC, RegEB4, RegEBC, RegEC4;
-	s32 RegECC, RegTmp = 0;
-	bool is12simular, is13simular, is23simular;
-	bool bStartContTx = false, bSingleTone = false;
-	bool bCarrierSuppression = false;
-	u32 IQK_BB_REG_92C[IQK_BB_REG_NUM] = {
-		rOFDM0_XARxIQImbalance, rOFDM0_XBRxIQImbalance,
-		rOFDM0_ECCAThreshold, rOFDM0_AGCRSSITable,
-		rOFDM0_XATxIQImbalance, rOFDM0_XBTxIQImbalance,
-		rOFDM0_XCTxAFE, rOFDM0_XDTxAFE,
-		rOFDM0_RxIQExtAnta
-	};
-
-	/* ignore IQK when continuous Tx */
-	if (bStartContTx || bSingleTone || bCarrierSuppression)
-		return;
-
-	if (bReCovery) {
-		_PHY_ReloadADDARegisters(pAdapter, IQK_BB_REG_92C, pdmpriv->IQK_BB_backup_recover, 9);
-		return;
-	}
-	DBG_8723A("IQK:Start!!!\n");
-
-	for (i = 0; i < 8; i++) {
-		result[0][i] = 0;
-		result[1][i] = 0;
-		result[2][i] = 0;
-		result[3][i] = 0;
-	}
-	final_candidate = 0xff;
-	bPathAOK = false;
-	bPathBOK = false;
-	is12simular = false;
-	is23simular = false;
-	is13simular = false;
-
-	for (i = 0; i < 3; i++) {
-		if (pHalData->rf_type == RF_2T2R)
-			_PHY_IQCalibrate(pAdapter, result, i, true);
-		else /*  For 88C 1T1R */
-			_PHY_IQCalibrate(pAdapter, result, i, false);
-
-		if (i == 1) {
-			is12simular = _PHY_SimularityCompare(pAdapter, result, 0, 1);
-			if (is12simular) {
-				final_candidate = 0;
-				break;
-			}
-		}
-
-		if (i == 2) {
-			is13simular = _PHY_SimularityCompare(pAdapter, result, 0, 2);
-			if (is13simular) {
-				final_candidate = 0;
-				break;
-			}
-
-			is23simular = _PHY_SimularityCompare(pAdapter, result, 1, 2);
-			if (is23simular) {
-				final_candidate = 1;
-			} else {
-				for (i = 0; i < 8; i++)
-					RegTmp += result[3][i];
-
-				if (RegTmp != 0)
-					final_candidate = 3;
-				else
-					final_candidate = 0xFF;
-			}
-		}
-	}
-
-	for (i = 0; i < 4; i++) {
-		RegE94 = result[i][0];
-		RegE9C = result[i][1];
-		RegEA4 = result[i][2];
-		RegEAC = result[i][3];
-		RegEB4 = result[i][4];
-		RegEBC = result[i][5];
-		RegEC4 = result[i][6];
-		RegECC = result[i][7];
-	}
-
-	if (final_candidate != 0xff) {
-		RegE94 = result[final_candidate][0];
-		pdmpriv->RegE94 =  RegE94;
-		RegE9C = result[final_candidate][1];
-		pdmpriv->RegE9C = RegE9C;
-		RegEA4 = result[final_candidate][2];
-		RegEAC = result[final_candidate][3];
-		RegEB4 = result[final_candidate][4];
-		pdmpriv->RegEB4 = RegEB4;
-		RegEBC = result[final_candidate][5];
-		pdmpriv->RegEBC = RegEBC;
-		RegEC4 = result[final_candidate][6];
-		RegECC = result[final_candidate][7];
-		DBG_8723A("IQK: final_candidate is %x\n", final_candidate);
-		DBG_8723A("IQK: RegE94 =%x RegE9C =%x RegEA4 =%x RegEAC =%x RegEB4 =%x RegEBC =%x RegEC4 =%x RegECC =%x\n ",
-			  RegE94, RegE9C, RegEA4, RegEAC, RegEB4, RegEBC, RegEC4, RegECC);
-		bPathAOK = bPathBOK = true;
-	} else {
-		RegE94 = RegEB4 = pdmpriv->RegE94 = pdmpriv->RegEB4 = 0x100;	/* X default value */
-		RegE9C = RegEBC = pdmpriv->RegE9C = pdmpriv->RegEBC = 0x0;		/* Y default value */
-	}
-
-	if ((RegE94 != 0)/*&&(RegEA4 != 0)*/)
-		_PHY_PathAFillIQKMatrix(pAdapter, bPathAOK, result, final_candidate, (RegEA4 == 0));
-
-	if (pHalData->rf_type == RF_2T2R) {
-		if ((RegEB4 != 0)/*&&(RegEC4 != 0)*/)
-			_PHY_PathBFillIQKMatrix(pAdapter, bPathBOK, result,
-						final_candidate, (RegEC4 == 0));
-	}
-
-	_PHY_SaveADDARegisters(pAdapter, IQK_BB_REG_92C, pdmpriv->IQK_BB_backup_recover, 9);
-}
-
-void rtl8723a_phy_lc_calibrate(struct rtw_adapter *pAdapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter);
-	struct mlme_ext_priv *pmlmeext = &pAdapter->mlmeextpriv;
-	bool bStartContTx = false, bSingleTone = false, bCarrierSuppression = false;
-
-	/* ignore IQK when continuous Tx */
-	if (bStartContTx || bSingleTone || bCarrierSuppression)
-		return;
-
-	if (pmlmeext->sitesurvey_res.state == SCAN_PROCESS)
-		return;
-
-	if (pHalData->rf_type == RF_2T2R)
-		_PHY_LCCalibrate(pAdapter, true);
-	else	/*  For 88C 1T1R */
-		_PHY_LCCalibrate(pAdapter, false);
-}
-
-void
-rtl8723a_phy_ap_calibrate(struct rtw_adapter *pAdapter, char delta)
-{
-}
diff --git a/drivers/staging/rtl8723au/hal/HalHWImg8723A_BB.c b/drivers/staging/rtl8723au/hal/HalHWImg8723A_BB.c
deleted file mode 100644
index 8d3ea6c0cbe6..000000000000
--- a/drivers/staging/rtl8723au/hal/HalHWImg8723A_BB.c
+++ /dev/null
@@ -1,565 +0,0 @@
-/******************************************************************************
-*
-* Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
-*
-* This program is free software; you can redistribute it and/or modify it
-* under the terms of version 2 of the GNU General Public License as
-* published by the Free Software Foundation.
-*
-* This program is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-* more details.
-*
-******************************************************************************/
-
-#include "odm_precomp.h"
-
-static bool CheckCondition(const u32  Condition, const u32  Hex)
-{
-	u32 _board     = (Hex & 0x000000FF);
-	u32 _interface = (Hex & 0x0000FF00) >> 8;
-	u32 _platform  = (Hex & 0x00FF0000) >> 16;
-	u32 cond = Condition;
-
-	if (Condition == 0xCDCDCDCD)
-		return true;
-
-	cond = Condition & 0x000000FF;
-	if ((_board == cond) && cond != 0x00)
-		return false;
-
-	cond = Condition & 0x0000FF00;
-	cond >>= 8;
-	if ((_interface & cond) == 0 && cond != 0x07)
-		return false;
-
-	cond = Condition & 0x00FF0000;
-	cond >>= 16;
-	if ((_platform & cond) == 0 && cond != 0x0F)
-		return false;
-	return true;
-}
-
-/******************************************************************************
-*                           AGC_TAB_1T.TXT
-******************************************************************************/
-
-static u32 Array_AGC_TAB_1T_8723A[] = {
-	0xC78, 0x7B000001,
-	0xC78, 0x7B010001,
-	0xC78, 0x7B020001,
-	0xC78, 0x7B030001,
-	0xC78, 0x7B040001,
-	0xC78, 0x7B050001,
-	0xC78, 0x7A060001,
-	0xC78, 0x79070001,
-	0xC78, 0x78080001,
-	0xC78, 0x77090001,
-	0xC78, 0x760A0001,
-	0xC78, 0x750B0001,
-	0xC78, 0x740C0001,
-	0xC78, 0x730D0001,
-	0xC78, 0x720E0001,
-	0xC78, 0x710F0001,
-	0xC78, 0x70100001,
-	0xC78, 0x6F110001,
-	0xC78, 0x6E120001,
-	0xC78, 0x6D130001,
-	0xC78, 0x6C140001,
-	0xC78, 0x6B150001,
-	0xC78, 0x6A160001,
-	0xC78, 0x69170001,
-	0xC78, 0x68180001,
-	0xC78, 0x67190001,
-	0xC78, 0x661A0001,
-	0xC78, 0x651B0001,
-	0xC78, 0x641C0001,
-	0xC78, 0x631D0001,
-	0xC78, 0x621E0001,
-	0xC78, 0x611F0001,
-	0xC78, 0x60200001,
-	0xC78, 0x49210001,
-	0xC78, 0x48220001,
-	0xC78, 0x47230001,
-	0xC78, 0x46240001,
-	0xC78, 0x45250001,
-	0xC78, 0x44260001,
-	0xC78, 0x43270001,
-	0xC78, 0x42280001,
-	0xC78, 0x41290001,
-	0xC78, 0x402A0001,
-	0xC78, 0x262B0001,
-	0xC78, 0x252C0001,
-	0xC78, 0x242D0001,
-	0xC78, 0x232E0001,
-	0xC78, 0x222F0001,
-	0xC78, 0x21300001,
-	0xC78, 0x20310001,
-	0xC78, 0x06320001,
-	0xC78, 0x05330001,
-	0xC78, 0x04340001,
-	0xC78, 0x03350001,
-	0xC78, 0x02360001,
-	0xC78, 0x01370001,
-	0xC78, 0x00380001,
-	0xC78, 0x00390001,
-	0xC78, 0x003A0001,
-	0xC78, 0x003B0001,
-	0xC78, 0x003C0001,
-	0xC78, 0x003D0001,
-	0xC78, 0x003E0001,
-	0xC78, 0x003F0001,
-	0xC78, 0x7B400001,
-	0xC78, 0x7B410001,
-	0xC78, 0x7B420001,
-	0xC78, 0x7B430001,
-	0xC78, 0x7B440001,
-	0xC78, 0x7B450001,
-	0xC78, 0x7A460001,
-	0xC78, 0x79470001,
-	0xC78, 0x78480001,
-	0xC78, 0x77490001,
-	0xC78, 0x764A0001,
-	0xC78, 0x754B0001,
-	0xC78, 0x744C0001,
-	0xC78, 0x734D0001,
-	0xC78, 0x724E0001,
-	0xC78, 0x714F0001,
-	0xC78, 0x70500001,
-	0xC78, 0x6F510001,
-	0xC78, 0x6E520001,
-	0xC78, 0x6D530001,
-	0xC78, 0x6C540001,
-	0xC78, 0x6B550001,
-	0xC78, 0x6A560001,
-	0xC78, 0x69570001,
-	0xC78, 0x68580001,
-	0xC78, 0x67590001,
-	0xC78, 0x665A0001,
-	0xC78, 0x655B0001,
-	0xC78, 0x645C0001,
-	0xC78, 0x635D0001,
-	0xC78, 0x625E0001,
-	0xC78, 0x615F0001,
-	0xC78, 0x60600001,
-	0xC78, 0x49610001,
-	0xC78, 0x48620001,
-	0xC78, 0x47630001,
-	0xC78, 0x46640001,
-	0xC78, 0x45650001,
-	0xC78, 0x44660001,
-	0xC78, 0x43670001,
-	0xC78, 0x42680001,
-	0xC78, 0x41690001,
-	0xC78, 0x406A0001,
-	0xC78, 0x266B0001,
-	0xC78, 0x256C0001,
-	0xC78, 0x246D0001,
-	0xC78, 0x236E0001,
-	0xC78, 0x226F0001,
-	0xC78, 0x21700001,
-	0xC78, 0x20710001,
-	0xC78, 0x06720001,
-	0xC78, 0x05730001,
-	0xC78, 0x04740001,
-	0xC78, 0x03750001,
-	0xC78, 0x02760001,
-	0xC78, 0x01770001,
-	0xC78, 0x00780001,
-	0xC78, 0x00790001,
-	0xC78, 0x007A0001,
-	0xC78, 0x007B0001,
-	0xC78, 0x007C0001,
-	0xC78, 0x007D0001,
-	0xC78, 0x007E0001,
-	0xC78, 0x007F0001,
-	0xC78, 0x3800001E,
-	0xC78, 0x3801001E,
-	0xC78, 0x3802001E,
-	0xC78, 0x3803001E,
-	0xC78, 0x3804001E,
-	0xC78, 0x3805001E,
-	0xC78, 0x3806001E,
-	0xC78, 0x3807001E,
-	0xC78, 0x3808001E,
-	0xC78, 0x3C09001E,
-	0xC78, 0x3E0A001E,
-	0xC78, 0x400B001E,
-	0xC78, 0x440C001E,
-	0xC78, 0x480D001E,
-	0xC78, 0x4C0E001E,
-	0xC78, 0x500F001E,
-	0xC78, 0x5210001E,
-	0xC78, 0x5611001E,
-	0xC78, 0x5A12001E,
-	0xC78, 0x5E13001E,
-	0xC78, 0x6014001E,
-	0xC78, 0x6015001E,
-	0xC78, 0x6016001E,
-	0xC78, 0x6217001E,
-	0xC78, 0x6218001E,
-	0xC78, 0x6219001E,
-	0xC78, 0x621A001E,
-	0xC78, 0x621B001E,
-	0xC78, 0x621C001E,
-	0xC78, 0x621D001E,
-	0xC78, 0x621E001E,
-	0xC78, 0x621F001E,
-};
-
-#define READ_NEXT_PAIR(v1, v2, i)			\
-	do {						\
-		i += 2; v1 = Array[i]; v2 = Array[i+1];	\
-	} while (0)
-
-void ODM_ReadAndConfig_AGC_TAB_1T_8723A(struct dm_odm_t *pDM_Odm)
-{
-	u32 hex;
-	u32 i;
-	u8 platform = 0x04;
-	u8 board = pDM_Odm->BoardType;
-	u32 ArrayLen = ARRAY_SIZE(Array_AGC_TAB_1T_8723A);
-	u32 *Array = Array_AGC_TAB_1T_8723A;
-
-	hex = board;
-	hex += ODM_ITRF_USB << 8;
-	hex += platform << 16;
-	hex += 0xFF000000;
-	for (i = 0; i < ArrayLen; i += 2) {
-		u32 v1 = Array[i];
-		u32 v2 = Array[i+1];
-
-		/*  This (offset, data) pair meets the condition. */
-		if (v1 < 0xCDCDCDCD) {
-			odm_ConfigBB_AGC_8723A(pDM_Odm, v1, v2);
-			continue;
-		} else {
-			if (!CheckCondition(Array[i], hex)) {
-				/* Discard the following (offset, data) pairs */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-				i -= 2; /*  prevent from for-loop += 2 */
-			} else {
-				/*  Configure matched pairs and skip to
-				    end of if-else. */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2) {
-					odm_ConfigBB_AGC_8723A(pDM_Odm, v1, v2);
-					READ_NEXT_PAIR(v1, v2, i);
-				}
-				while (v2 != 0xDEAD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-			}
-		}
-	}
-}
-
-/******************************************************************************
-*                           PHY_REG_1T.TXT
-******************************************************************************/
-
-static u32 Array_PHY_REG_1T_8723A[] = {
-	0x800, 0x80040000,
-	0x804, 0x00000003,
-	0x808, 0x0000FC00,
-	0x80C, 0x0000000A,
-	0x810, 0x10001331,
-	0x814, 0x020C3D10,
-	0x818, 0x02200385,
-	0x81C, 0x00000000,
-	0x820, 0x01000100,
-	0x824, 0x00390004,
-	0x828, 0x00000000,
-	0x82C, 0x00000000,
-	0x830, 0x00000000,
-	0x834, 0x00000000,
-	0x838, 0x00000000,
-	0x83C, 0x00000000,
-	0x840, 0x00010000,
-	0x844, 0x00000000,
-	0x848, 0x00000000,
-	0x84C, 0x00000000,
-	0x850, 0x00000000,
-	0x854, 0x00000000,
-	0x858, 0x569A569A,
-	0x85C, 0x001B25A4,
-	0x860, 0x66F60110,
-	0x864, 0x061F0130,
-	0x868, 0x00000000,
-	0x86C, 0x32323200,
-	0x870, 0x07000760,
-	0x874, 0x22004000,
-	0x878, 0x00000808,
-	0x87C, 0x00000000,
-	0x880, 0xC0083070,
-	0x884, 0x000004D5,
-	0x888, 0x00000000,
-	0x88C, 0xCCC000C0,
-	0x890, 0x00000800,
-	0x894, 0xFFFFFFFE,
-	0x898, 0x40302010,
-	0x89C, 0x00706050,
-	0x900, 0x00000000,
-	0x904, 0x00000023,
-	0x908, 0x00000000,
-	0x90C, 0x81121111,
-	0xA00, 0x00D047C8,
-	0xA04, 0x80FF000C,
-	0xA08, 0x8C838300,
-	0xA0C, 0x2E68120F,
-	0xA10, 0x9500BB78,
-	0xA14, 0x11144028,
-	0xA18, 0x00881117,
-	0xA1C, 0x89140F00,
-	0xA20, 0x1A1B0000,
-	0xA24, 0x090E1317,
-	0xA28, 0x00000204,
-	0xA2C, 0x00D30000,
-	0xA70, 0x101FBF00,
-	0xA74, 0x00000007,
-	0xA78, 0x00000900,
-	0xC00, 0x48071D40,
-	0xC04, 0x03A05611,
-	0xC08, 0x000000E4,
-	0xC0C, 0x6C6C6C6C,
-	0xC10, 0x08800000,
-	0xC14, 0x40000100,
-	0xC18, 0x08800000,
-	0xC1C, 0x40000100,
-	0xC20, 0x00000000,
-	0xC24, 0x00000000,
-	0xC28, 0x00000000,
-	0xC2C, 0x00000000,
-	0xC30, 0x69E9AC44,
-	0xFF0F011F, 0xABCD,
-	0xC34, 0x469652CF,
-	0xCDCDCDCD, 0xCDCD,
-	0xC34, 0x469652AF,
-	0xFF0F011F, 0xDEAD,
-	0xC38, 0x49795994,
-	0xC3C, 0x0A97971C,
-	0xC40, 0x1F7C403F,
-	0xC44, 0x000100B7,
-	0xC48, 0xEC020107,
-	0xC4C, 0x007F037F,
-	0xC50, 0x69543420,
-	0xC54, 0x43BC0094,
-	0xC58, 0x69543420,
-	0xC5C, 0x433C0094,
-	0xC60, 0x00000000,
-	0xFF0F011F, 0xABCD,
-	0xC64, 0x7116848B,
-	0xCDCDCDCD, 0xCDCD,
-	0xC64, 0x7112848B,
-	0xFF0F011F, 0xDEAD,
-	0xC68, 0x47C00BFF,
-	0xC6C, 0x00000036,
-	0xC70, 0x2C7F000D,
-	0xC74, 0x018610DB,
-	0xC78, 0x0000001F,
-	0xC7C, 0x00B91612,
-	0xC80, 0x40000100,
-	0xC84, 0x20F60000,
-	0xC88, 0x40000100,
-	0xC8C, 0x20200000,
-	0xC90, 0x00121820,
-	0xC94, 0x00000000,
-	0xC98, 0x00121820,
-	0xC9C, 0x00007F7F,
-	0xCA0, 0x00000000,
-	0xCA4, 0x00000080,
-	0xCA8, 0x00000000,
-	0xCAC, 0x00000000,
-	0xCB0, 0x00000000,
-	0xCB4, 0x00000000,
-	0xCB8, 0x00000000,
-	0xCBC, 0x28000000,
-	0xCC0, 0x00000000,
-	0xCC4, 0x00000000,
-	0xCC8, 0x00000000,
-	0xCCC, 0x00000000,
-	0xCD0, 0x00000000,
-	0xCD4, 0x00000000,
-	0xCD8, 0x64B22427,
-	0xCDC, 0x00766932,
-	0xCE0, 0x00222222,
-	0xCE4, 0x00000000,
-	0xCE8, 0x37644302,
-	0xCEC, 0x2F97D40C,
-	0xD00, 0x00080740,
-	0xD04, 0x00020401,
-	0xD08, 0x0000907F,
-	0xD0C, 0x20010201,
-	0xD10, 0xA0633333,
-	0xD14, 0x3333BC43,
-	0xD18, 0x7A8F5B6B,
-	0xD2C, 0xCC979975,
-	0xD30, 0x00000000,
-	0xD34, 0x80608000,
-	0xD38, 0x00000000,
-	0xD3C, 0x00027293,
-	0xD40, 0x00000000,
-	0xD44, 0x00000000,
-	0xD48, 0x00000000,
-	0xD4C, 0x00000000,
-	0xD50, 0x6437140A,
-	0xD54, 0x00000000,
-	0xD58, 0x00000000,
-	0xD5C, 0x30032064,
-	0xD60, 0x4653DE68,
-	0xD64, 0x04518A3C,
-	0xD68, 0x00002101,
-	0xD6C, 0x2A201C16,
-	0xD70, 0x1812362E,
-	0xD74, 0x322C2220,
-	0xD78, 0x000E3C24,
-	0xE00, 0x2A2A2A2A,
-	0xE04, 0x2A2A2A2A,
-	0xE08, 0x03902A2A,
-	0xE10, 0x2A2A2A2A,
-	0xE14, 0x2A2A2A2A,
-	0xE18, 0x2A2A2A2A,
-	0xE1C, 0x2A2A2A2A,
-	0xE28, 0x00000000,
-	0xE30, 0x1000DC1F,
-	0xE34, 0x10008C1F,
-	0xE38, 0x02140102,
-	0xE3C, 0x681604C2,
-	0xE40, 0x01007C00,
-	0xE44, 0x01004800,
-	0xE48, 0xFB000000,
-	0xE4C, 0x000028D1,
-	0xE50, 0x1000DC1F,
-	0xE54, 0x10008C1F,
-	0xE58, 0x02140102,
-	0xE5C, 0x28160D05,
-	0xE60, 0x00000008,
-	0xE68, 0x001B25A4,
-	0xE6C, 0x631B25A0,
-	0xE70, 0x631B25A0,
-	0xE74, 0x081B25A0,
-	0xE78, 0x081B25A0,
-	0xE7C, 0x081B25A0,
-	0xE80, 0x081B25A0,
-	0xE84, 0x631B25A0,
-	0xE88, 0x081B25A0,
-	0xE8C, 0x631B25A0,
-	0xED0, 0x631B25A0,
-	0xED4, 0x631B25A0,
-	0xED8, 0x631B25A0,
-	0xEDC, 0x001B25A0,
-	0xEE0, 0x001B25A0,
-	0xEEC, 0x6B1B25A0,
-	0xF14, 0x00000003,
-	0xF4C, 0x00000000,
-	0xF00, 0x00000300,
-};
-
-void ODM_ReadAndConfig_PHY_REG_1T_8723A(struct dm_odm_t *pDM_Odm)
-{
-	u32 hex = 0;
-	u32 i = 0;
-	u8  platform = 0x04;
-	u8  board = pDM_Odm->BoardType;
-	u32 ArrayLen = ARRAY_SIZE(Array_PHY_REG_1T_8723A);
-	u32 *Array = Array_PHY_REG_1T_8723A;
-
-	hex += board;
-	hex += ODM_ITRF_USB << 8;
-	hex += platform << 16;
-	hex += 0xFF000000;
-	for (i = 0; i < ArrayLen; i += 2) {
-		u32 v1 = Array[i];
-		u32 v2 = Array[i+1];
-
-		/*  This (offset, data) pair meets the condition. */
-		if (v1 < 0xCDCDCDCD) {
-			odm_ConfigBB_PHY_8723A(pDM_Odm, v1, v2);
-			continue;
-		} else {
-			if (!CheckCondition(Array[i], hex)) {
-				/* Discard the following (offset, data) pairs */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-				i -= 2; /*  prevent from for-loop += 2 */
-			} else {
-				/*  Configure matched pairs and skip to
-				    end of if-else. */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2) {
-					odm_ConfigBB_PHY_8723A(pDM_Odm, v1, v2);
-					READ_NEXT_PAIR(v1, v2, i);
-				}
-				while (v2 != 0xDEAD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-			}
-		}
-	}
-}
-
-/******************************************************************************
-*                           PHY_REG_MP.TXT
-******************************************************************************/
-
-static u32 Array_PHY_REG_MP_8723A[] = {
-	0xC30, 0x69E9AC4A,
-	0xC3C, 0x0A979718,
-};
-
-void ODM_ReadAndConfig_PHY_REG_MP_8723A(struct dm_odm_t *pDM_Odm)
-{
-	u32 hex = 0;
-	u32 i;
-	u8 platform = 0x04;
-	u8 board = pDM_Odm->BoardType;
-	u32 ArrayLen = ARRAY_SIZE(Array_PHY_REG_MP_8723A);
-	u32 *Array = Array_PHY_REG_MP_8723A;
-
-	hex += board;
-	hex += ODM_ITRF_USB << 8;
-	hex += platform << 16;
-	hex += 0xFF000000;
-	for (i = 0; i < ArrayLen; i += 2) {
-		u32 v1 = Array[i];
-		u32 v2 = Array[i+1];
-
-		/*  This (offset, data) pair meets the condition. */
-		if (v1 < 0xCDCDCDCD) {
-			odm_ConfigBB_PHY_8723A(pDM_Odm, v1, v2);
-			continue;
-		} else {
-			if (!CheckCondition(Array[i], hex)) {
-				/* Discard the following (offset, data) pairs */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-				i -= 2; /*  prevent from for-loop += 2 */
-			} else {
-				/* Configure matched pairs and skip to
-				   end of if-else. */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2) {
-					odm_ConfigBB_PHY_8723A(pDM_Odm, v1, v2);
-					READ_NEXT_PAIR(v1, v2, i);
-				}
-				while (v2 != 0xDEAD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-			}
-		}
-	}
-}
diff --git a/drivers/staging/rtl8723au/hal/HalHWImg8723A_MAC.c b/drivers/staging/rtl8723au/hal/HalHWImg8723A_MAC.c
deleted file mode 100644
index 9bf685905e68..000000000000
--- a/drivers/staging/rtl8723au/hal/HalHWImg8723A_MAC.c
+++ /dev/null
@@ -1,187 +0,0 @@
-/******************************************************************************
-*
-* Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
-*
-* This program is free software; you can redistribute it and/or modify it
-* under the terms of version 2 of the GNU General Public License as
-* published by the Free Software Foundation.
-*
-* This program is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-* more details.
-*
-******************************************************************************/
-
-#include "odm_precomp.h"
-
-static bool CheckCondition(const u32  Condition, const u32  Hex)
-{
-	u32 _board     = (Hex & 0x000000FF);
-	u32 _interface = (Hex & 0x0000FF00) >> 8;
-	u32 _platform  = (Hex & 0x00FF0000) >> 16;
-	u32 cond = Condition;
-
-	if (Condition == 0xCDCDCDCD)
-		return true;
-
-	cond = Condition & 0x000000FF;
-	if ((_board == cond) && cond != 0x00)
-		return false;
-
-	cond = Condition & 0x0000FF00;
-	cond >>= 8;
-	if ((_interface & cond) == 0 && cond != 0x07)
-		return false;
-
-	cond = Condition & 0x00FF0000;
-	cond >>= 16;
-	if ((_platform & cond) == 0 && cond != 0x0F)
-		return false;
-	return true;
-}
-
-/******************************************************************************
-*                           MAC_REG.TXT
-******************************************************************************/
-
-static u32 Array_MAC_REG_8723A[] = {
-		0x420, 0x00000080,
-		0x423, 0x00000000,
-		0x430, 0x00000000,
-		0x431, 0x00000000,
-		0x432, 0x00000000,
-		0x433, 0x00000001,
-		0x434, 0x00000004,
-		0x435, 0x00000005,
-		0x436, 0x00000006,
-		0x437, 0x00000007,
-		0x438, 0x00000000,
-		0x439, 0x00000000,
-		0x43A, 0x00000000,
-		0x43B, 0x00000001,
-		0x43C, 0x00000004,
-		0x43D, 0x00000005,
-		0x43E, 0x00000006,
-		0x43F, 0x00000007,
-		0x440, 0x0000005D,
-		0x441, 0x00000001,
-		0x442, 0x00000000,
-		0x444, 0x00000015,
-		0x445, 0x000000F0,
-		0x446, 0x0000000F,
-		0x447, 0x00000000,
-		0x458, 0x00000041,
-		0x459, 0x000000A8,
-		0x45A, 0x00000072,
-		0x45B, 0x000000B9,
-		0x460, 0x00000066,
-		0x461, 0x00000066,
-		0x462, 0x00000008,
-		0x463, 0x00000003,
-		0x4C8, 0x000000FF,
-		0x4C9, 0x00000008,
-		0x4CC, 0x000000FF,
-		0x4CD, 0x000000FF,
-		0x4CE, 0x00000001,
-		0x500, 0x00000026,
-		0x501, 0x000000A2,
-		0x502, 0x0000002F,
-		0x503, 0x00000000,
-		0x504, 0x00000028,
-		0x505, 0x000000A3,
-		0x506, 0x0000005E,
-		0x507, 0x00000000,
-		0x508, 0x0000002B,
-		0x509, 0x000000A4,
-		0x50A, 0x0000005E,
-		0x50B, 0x00000000,
-		0x50C, 0x0000004F,
-		0x50D, 0x000000A4,
-		0x50E, 0x00000000,
-		0x50F, 0x00000000,
-		0x512, 0x0000001C,
-		0x514, 0x0000000A,
-		0x515, 0x00000010,
-		0x516, 0x0000000A,
-		0x517, 0x00000010,
-		0x51A, 0x00000016,
-		0x524, 0x0000000F,
-		0x525, 0x0000004F,
-		0x546, 0x00000040,
-		0x547, 0x00000000,
-		0x550, 0x00000010,
-		0x551, 0x00000010,
-		0x559, 0x00000002,
-		0x55A, 0x00000002,
-		0x55D, 0x000000FF,
-		0x605, 0x00000030,
-		0x608, 0x0000000E,
-		0x609, 0x0000002A,
-		0x652, 0x00000020,
-		0x63C, 0x0000000A,
-		0x63D, 0x0000000A,
-		0x63E, 0x0000000E,
-		0x63F, 0x0000000E,
-		0x66E, 0x00000005,
-		0x700, 0x00000021,
-		0x701, 0x00000043,
-		0x702, 0x00000065,
-		0x703, 0x00000087,
-		0x708, 0x00000021,
-		0x709, 0x00000043,
-		0x70A, 0x00000065,
-		0x70B, 0x00000087,
-};
-
-void ODM_ReadAndConfig_MAC_REG_8723A(struct dm_odm_t *pDM_Odm)
-{
-	#define READ_NEXT_PAIR(v1, v2, i)			\
-		 do {						\
-			i += 2; v1 = Array[i]; v2 = Array[i+1];	\
-		 } while (0)
-
-	u32     hex         = 0;
-	u32     i           = 0;
-	u8     platform    = 0x04;
-	u8     board       = pDM_Odm->BoardType;
-	u32     ArrayLen    = ARRAY_SIZE(Array_MAC_REG_8723A);
-	u32 *Array       = Array_MAC_REG_8723A;
-
-	hex += board;
-	hex += ODM_ITRF_USB << 8;
-	hex += platform << 16;
-	hex += 0xFF000000;
-	for (i = 0; i < ArrayLen; i += 2) {
-		u32 v1 = Array[i];
-		u32 v2 = Array[i+1];
-
-		/*  This (offset, data) pair meets the condition. */
-		if (v1 < 0xCDCDCDCD) {
-			odm_ConfigMAC_8723A(pDM_Odm, v1, (u8)v2);
-			continue;
-		} else {
-			if (!CheckCondition(Array[i], hex)) {
-				/* Discard the following (offset, data) pairs. */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-				i -= 2; /*  prevent from for-loop += 2 */
-			} else {
-				/*  Configure matched pairs and skip to end of if-else. */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2) {
-					odm_ConfigMAC_8723A(pDM_Odm, v1, (u8)v2);
-					READ_NEXT_PAIR(v1, v2, i);
-				}
-
-				while (v2 != 0xDEAD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-			}
-		}
-	}
-}
diff --git a/drivers/staging/rtl8723au/hal/HalHWImg8723A_RF.c b/drivers/staging/rtl8723au/hal/HalHWImg8723A_RF.c
deleted file mode 100644
index 286f3ea3d263..000000000000
--- a/drivers/staging/rtl8723au/hal/HalHWImg8723A_RF.c
+++ /dev/null
@@ -1,259 +0,0 @@
-/******************************************************************************
-*
-* Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
-*
-* This program is free software; you can redistribute it and/or modify it
-* under the terms of version 2 of the GNU General Public License as
-* published by the Free Software Foundation.
-*
-* This program is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-* more details.
-*
-******************************************************************************/
-
-#include "odm_precomp.h"
-
-static bool CheckCondition(const u32  Condition, const u32  Hex)
-{
-	u32 _board     = (Hex & 0x000000FF);
-	u32 _interface = (Hex & 0x0000FF00) >> 8;
-	u32 _platform  = (Hex & 0x00FF0000) >> 16;
-	u32 cond = Condition;
-
-	if (Condition == 0xCDCDCDCD)
-		return true;
-
-	cond = Condition & 0x000000FF;
-	if ((_board == cond) && cond != 0x00)
-		return false;
-
-	cond = Condition & 0x0000FF00;
-	cond >>= 8;
-	if ((_interface & cond) == 0 && cond != 0x07)
-		return false;
-
-	cond = Condition & 0x00FF0000;
-	cond >>= 16;
-	if ((_platform & cond) == 0 && cond != 0x0F)
-		return false;
-	return true;
-}
-
-/******************************************************************************
-*                           RadioA_1T.TXT
-******************************************************************************/
-
-static u32 Array_RadioA_1T_8723A[] = {
-	0x000, 0x00030159,
-	0x001, 0x00031284,
-	0x002, 0x00098000,
-	0xFF0F011F, 0xABCD,
-	0x003, 0x00018C63,
-	0xCDCDCDCD, 0xCDCD,
-	0x003, 0x00039C63,
-	0xFF0F011F, 0xDEAD,
-	0x004, 0x000210E7,
-	0x009, 0x0002044F,
-	0x00A, 0x0001A3F1,
-	0x00B, 0x00014787,
-	0x00C, 0x000896FE,
-	0x00D, 0x0000E02C,
-	0x00E, 0x00039CE7,
-	0x00F, 0x00000451,
-	0x019, 0x00000000,
-	0x01A, 0x00030355,
-	0x01B, 0x00060A00,
-	0x01C, 0x000FC378,
-	0x01D, 0x000A1250,
-	0x01E, 0x0000024F,
-	0x01F, 0x00000000,
-	0x020, 0x0000B614,
-	0x021, 0x0006C000,
-	0x022, 0x00000000,
-	0x023, 0x00001558,
-	0x024, 0x00000060,
-	0x025, 0x00000483,
-	0x026, 0x0004F000,
-	0x027, 0x000EC7D9,
-	0x028, 0x00057730,
-	0x029, 0x00004783,
-	0x02A, 0x00000001,
-	0x02B, 0x00021334,
-	0x02A, 0x00000000,
-	0x02B, 0x00000054,
-	0x02A, 0x00000001,
-	0x02B, 0x00000808,
-	0x02B, 0x00053333,
-	0x02C, 0x0000000C,
-	0x02A, 0x00000002,
-	0x02B, 0x00000808,
-	0x02B, 0x0005B333,
-	0x02C, 0x0000000D,
-	0x02A, 0x00000003,
-	0x02B, 0x00000808,
-	0x02B, 0x00063333,
-	0x02C, 0x0000000D,
-	0x02A, 0x00000004,
-	0x02B, 0x00000808,
-	0x02B, 0x0006B333,
-	0x02C, 0x0000000D,
-	0x02A, 0x00000005,
-	0x02B, 0x00000808,
-	0x02B, 0x00073333,
-	0x02C, 0x0000000D,
-	0x02A, 0x00000006,
-	0x02B, 0x00000709,
-	0x02B, 0x0005B333,
-	0x02C, 0x0000000D,
-	0x02A, 0x00000007,
-	0x02B, 0x00000709,
-	0x02B, 0x00063333,
-	0x02C, 0x0000000D,
-	0x02A, 0x00000008,
-	0x02B, 0x0000060A,
-	0x02B, 0x0004B333,
-	0x02C, 0x0000000D,
-	0x02A, 0x00000009,
-	0x02B, 0x0000060A,
-	0x02B, 0x00053333,
-	0x02C, 0x0000000D,
-	0x02A, 0x0000000A,
-	0x02B, 0x0000060A,
-	0x02B, 0x0005B333,
-	0x02C, 0x0000000D,
-	0x02A, 0x0000000B,
-	0x02B, 0x0000060A,
-	0x02B, 0x00063333,
-	0x02C, 0x0000000D,
-	0x02A, 0x0000000C,
-	0x02B, 0x0000060A,
-	0x02B, 0x0006B333,
-	0x02C, 0x0000000D,
-	0x02A, 0x0000000D,
-	0x02B, 0x0000060A,
-	0x02B, 0x00073333,
-	0x02C, 0x0000000D,
-	0x02A, 0x0000000E,
-	0x02B, 0x0000050B,
-	0x02B, 0x00066666,
-	0x02C, 0x0000001A,
-	0x02A, 0x000E0000,
-	0x010, 0x0004000F,
-	0x011, 0x000E31FC,
-	0x010, 0x0006000F,
-	0x011, 0x000FF9F8,
-	0x010, 0x0002000F,
-	0x011, 0x000203F9,
-	0x010, 0x0003000F,
-	0x011, 0x000FF500,
-	0x010, 0x00000000,
-	0x011, 0x00000000,
-	0x010, 0x0008000F,
-	0x011, 0x0003F100,
-	0x010, 0x0009000F,
-	0x011, 0x00023100,
-	0x012, 0x00032000,
-	0x012, 0x00071000,
-	0x012, 0x000B0000,
-	0x012, 0x000FC000,
-	0x013, 0x000287B3,
-	0x013, 0x000244B7,
-	0x013, 0x000204AB,
-	0x013, 0x0001C49F,
-	0x013, 0x00018493,
-	0x013, 0x0001429B,
-	0x013, 0x00010299,
-	0x013, 0x0000C29C,
-	0x013, 0x000081A0,
-	0x013, 0x000040AC,
-	0x013, 0x00000020,
-	0x014, 0x0001944C,
-	0x014, 0x00059444,
-	0x014, 0x0009944C,
-	0x014, 0x000D9444,
-	0xFF0F011F, 0xABCD,
-	0x015, 0x0000F424,
-	0x015, 0x0004F424,
-	0x015, 0x0008F424,
-	0x015, 0x000CF424,
-	0xCDCDCDCD, 0xCDCD,
-	0x015, 0x0000F474,
-	0x015, 0x0004F477,
-	0x015, 0x0008F455,
-	0x015, 0x000CF455,
-	0xFF0F011F, 0xDEAD,
-	0x016, 0x00000339,
-	0x016, 0x00040339,
-	0x016, 0x00080339,
-	0xFF0F011F, 0xABCD,
-	0x016, 0x000C0356,
-	0xCDCDCDCD, 0xCDCD,
-	0x016, 0x000C0366,
-	0xFF0F011F, 0xDEAD,
-	0x000, 0x00010159,
-	0x018, 0x0000F401,
-	0x0FE, 0x00000000,
-	0x0FE, 0x00000000,
-	0x01F, 0x00000003,
-	0x0FE, 0x00000000,
-	0x0FE, 0x00000000,
-	0x01E, 0x00000247,
-	0x01F, 0x00000000,
-	0x000, 0x00030159,
-};
-
-void ODM_ReadAndConfig_RadioA_1T_8723A(struct dm_odm_t *pDM_Odm)
-{
-	#define READ_NEXT_PAIR(v1, v2, i)			\
-		 do {						\
-			 i += 2; v1 = Array[i]; v2 = Array[i+1];\
-		 } while (0)
-
-	u32     hex         = 0;
-	u32     i           = 0;
-	u8     platform    = 0x04;
-	u8     board       = pDM_Odm->BoardType;
-	u32     ArrayLen    = ARRAY_SIZE(Array_RadioA_1T_8723A);
-	u32 *Array = Array_RadioA_1T_8723A;
-
-	hex += board;
-	hex += ODM_ITRF_USB << 8;
-	hex += platform << 16;
-	hex += 0xFF000000;
-
-	for (i = 0; i < ArrayLen; i += 2) {
-		u32 v1 = Array[i];
-		u32 v2 = Array[i+1];
-
-		/*  This (offset, data) pair meets the condition. */
-		if (v1 < 0xCDCDCDCD) {
-			odm_ConfigRFReg_8723A(pDM_Odm, v1, v2, RF_PATH_A, v1);
-			continue;
-		} else {
-			if (!CheckCondition(Array[i], hex)) {
-				/*  Discard the following (offset, data) pairs. */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-				i -= 2; /*  prevent from for-loop += 2 */
-			} else {
-				/*  Configure matched pairs and skip to end of if-else. */
-				READ_NEXT_PAIR(v1, v2, i);
-				while (v2 != 0xDEAD &&
-				       v2 != 0xCDEF &&
-				       v2 != 0xCDCD && i < ArrayLen - 2) {
-					odm_ConfigRFReg_8723A(pDM_Odm, v1, v2,
-							      RF_PATH_A, v1);
-					READ_NEXT_PAIR(v1, v2, i);
-				}
-
-				while (v2 != 0xDEAD && i < ArrayLen - 2)
-					READ_NEXT_PAIR(v1, v2, i);
-			}
-		}
-	}
-}
diff --git a/drivers/staging/rtl8723au/hal/HalPwrSeqCmd.c b/drivers/staging/rtl8723au/hal/HalPwrSeqCmd.c
deleted file mode 100644
index 0a3d96e840cc..000000000000
--- a/drivers/staging/rtl8723au/hal/HalPwrSeqCmd.c
+++ /dev/null
@@ -1,156 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-/*++
-Copyright (c) Realtek Semiconductor Corp. All rights reserved.
-
-Module Name:
-	HalPwrSeqCmd.c
-
-Abstract:
-	Implement HW Power sequence configuration CMD handling routine for
-	Realtek devices.
-
-Major Change History:
-	When       Who               What
-	---------- ---------------   -------------------------------
-	2011-10-26 Lucas            Modify to be compatible with SD4-CE driver.
-	2011-07-07 Roger            Create.
-
---*/
-#include <HalPwrSeqCmd.h>
-#include <usb_ops_linux.h>
-
-/*  */
-/*	Description: */
-/*		This routine deal with the Power Configuration CMDs parsing
-		for RTL8723/RTL8188E Series IC. */
-/*  */
-/*	Assumption: */
-/*		We should follow specific format which was released from
-		HW SD. */
-/*  */
-/*	2011.07.07, added by Roger. */
-/*  */
-u8 HalPwrSeqCmdParsing23a(struct rtw_adapter *padapter, u8 CutVersion,
-		       u8 FabVersion, u8 InterfaceType,
-		       struct wlan_pwr_cfg PwrSeqCmd[])
-{
-	struct wlan_pwr_cfg PwrCfgCmd;
-	u8 bPollingBit;
-	u32 AryIdx = 0;
-	u8 value;
-	u32 offset;
-	u32 pollingCount = 0;	/*  polling autoload done. */
-	u32 maxPollingCnt = 5000;
-
-	do {
-		PwrCfgCmd = PwrSeqCmd[AryIdx];
-
-		RT_TRACE(_module_hal_init_c_, _drv_info_,
-			 "HalPwrSeqCmdParsing23a: offset(%#x) cut_msk(%#x) fab_msk(%#x) interface_msk(%#x) base(%#x) cmd(%#x) msk(%#x) value(%#x)\n",
-			 GET_PWR_CFG_OFFSET(PwrCfgCmd),
-			 GET_PWR_CFG_CUT_MASK(PwrCfgCmd),
-			 GET_PWR_CFG_FAB_MASK(PwrCfgCmd),
-			 GET_PWR_CFG_INTF_MASK(PwrCfgCmd),
-			 GET_PWR_CFG_BASE(PwrCfgCmd),
-			 GET_PWR_CFG_CMD(PwrCfgCmd),
-			 GET_PWR_CFG_MASK(PwrCfgCmd),
-			 GET_PWR_CFG_VALUE(PwrCfgCmd));
-
-		/* 2 Only Handle the command whose FAB, CUT, and Interface are
-		   matched */
-		if ((GET_PWR_CFG_FAB_MASK(PwrCfgCmd) & FabVersion) &&
-		    (GET_PWR_CFG_CUT_MASK(PwrCfgCmd) & CutVersion) &&
-		    (GET_PWR_CFG_INTF_MASK(PwrCfgCmd) & InterfaceType)) {
-			switch (GET_PWR_CFG_CMD(PwrCfgCmd)) {
-			case PWR_CMD_READ:
-				RT_TRACE(_module_hal_init_c_, _drv_info_,
-					 "HalPwrSeqCmdParsing23a: PWR_CMD_READ\n");
-				break;
-
-			case PWR_CMD_WRITE:
-				RT_TRACE(_module_hal_init_c_, _drv_info_,
-					 "HalPwrSeqCmdParsing23a: PWR_CMD_WRITE\n");
-				offset = GET_PWR_CFG_OFFSET(PwrCfgCmd);
-
-				/*  Read the value from system register */
-				value = rtl8723au_read8(padapter, offset);
-
-				value &= ~(GET_PWR_CFG_MASK(PwrCfgCmd));
-				value |= (GET_PWR_CFG_VALUE(PwrCfgCmd) &
-					  GET_PWR_CFG_MASK(PwrCfgCmd));
-
-				/*  Write the value back to system register */
-				rtl8723au_write8(padapter, offset, value);
-				break;
-
-			case PWR_CMD_POLLING:
-				RT_TRACE(_module_hal_init_c_, _drv_info_,
-					 "HalPwrSeqCmdParsing23a: PWR_CMD_POLLING\n");
-
-				bPollingBit = false;
-				offset = GET_PWR_CFG_OFFSET(PwrCfgCmd);
-				do {
-					value = rtl8723au_read8(padapter,
-								offset);
-
-					value &= GET_PWR_CFG_MASK(PwrCfgCmd);
-					if (value ==
-					    (GET_PWR_CFG_VALUE(PwrCfgCmd) &
-					     GET_PWR_CFG_MASK(PwrCfgCmd)))
-						bPollingBit = true;
-					else
-						udelay(10);
-
-					if (pollingCount++ > maxPollingCnt) {
-						DBG_8723A("Fail to polling "
-							  "Offset[%#x]\n",
-							  offset);
-						return false;
-					}
-				} while (!bPollingBit);
-
-				break;
-
-			case PWR_CMD_DELAY:
-				RT_TRACE(_module_hal_init_c_, _drv_info_,
-					 "HalPwrSeqCmdParsing23a: PWR_CMD_DELAY\n");
-				if (GET_PWR_CFG_VALUE(PwrCfgCmd) ==
-				    PWRSEQ_DELAY_US)
-					udelay(GET_PWR_CFG_OFFSET(PwrCfgCmd));
-				else
-					udelay(GET_PWR_CFG_OFFSET(PwrCfgCmd) *
-					       1000);
-				break;
-
-			case PWR_CMD_END:
-				/*  When this command is parsed, end
-				    the process */
-				RT_TRACE(_module_hal_init_c_, _drv_info_,
-					 "HalPwrSeqCmdParsing23a: PWR_CMD_END\n");
-				return true;
-
-			default:
-				RT_TRACE(_module_hal_init_c_, _drv_err_,
-					 "HalPwrSeqCmdParsing23a: Unknown CMD!!\n");
-				break;
-			}
-		}
-
-		AryIdx++;	/* Add Array Index */
-	} while (1);
-
-	return true;
-}
diff --git a/drivers/staging/rtl8723au/hal/hal_com.c b/drivers/staging/rtl8723au/hal/hal_com.c
deleted file mode 100644
index 9d7b11b63957..000000000000
--- a/drivers/staging/rtl8723au/hal/hal_com.c
+++ /dev/null
@@ -1,853 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#include <hal_intf.h>
-#include <hal_com.h>
-#include <rtl8723a_hal.h>
-#include <usb_ops_linux.h>
-
-#define _HAL_INIT_C_
-
-#define	EEPROM_CHANNEL_PLAN_BY_HW_MASK	0x80
-
-/* return the final channel plan decision */
-/* hw_channel_plan:  channel plan from HW (efuse/eeprom) */
-/* sw_channel_plan:  channel plan from SW (registry/module param) */
-/* def_channel_plan: channel plan used when the former two is invalid */
-u8 hal_com_get_channel_plan23a(struct rtw_adapter *padapter, u8 hw_channel_plan,
-			    u8 sw_channel_plan, u8 def_channel_plan,
-			    bool AutoLoadFail)
-{
-	u8 swConfig;
-	u8 chnlPlan;
-
-	swConfig = true;
-	if (!AutoLoadFail) {
-		if (!rtw_is_channel_plan_valid(sw_channel_plan))
-			swConfig = false;
-		if (hw_channel_plan & EEPROM_CHANNEL_PLAN_BY_HW_MASK)
-			swConfig = false;
-	}
-
-	if (swConfig == true)
-		chnlPlan = sw_channel_plan;
-	else
-		chnlPlan = hw_channel_plan & (~EEPROM_CHANNEL_PLAN_BY_HW_MASK);
-
-	if (!rtw_is_channel_plan_valid(chnlPlan))
-		chnlPlan = def_channel_plan;
-
-	return chnlPlan;
-}
-
-u8 MRateToHwRate23a(u8 rate)
-{
-	u8 ret = DESC_RATE1M;
-
-	switch (rate) {
-		/*  CCK and OFDM non-HT rates */
-	case IEEE80211_CCK_RATE_1MB:
-		ret = DESC_RATE1M;
-		break;
-	case IEEE80211_CCK_RATE_2MB:
-		ret = DESC_RATE2M;
-		break;
-	case IEEE80211_CCK_RATE_5MB:
-		ret = DESC_RATE5_5M;
-		break;
-	case IEEE80211_CCK_RATE_11MB:
-		ret = DESC_RATE11M;
-		break;
-	case IEEE80211_OFDM_RATE_6MB:
-		ret = DESC_RATE6M;
-		break;
-	case IEEE80211_OFDM_RATE_9MB:
-		ret = DESC_RATE9M;
-		break;
-	case IEEE80211_OFDM_RATE_12MB:
-		ret = DESC_RATE12M;
-		break;
-	case IEEE80211_OFDM_RATE_18MB:
-		ret = DESC_RATE18M;
-		break;
-	case IEEE80211_OFDM_RATE_24MB:
-		ret = DESC_RATE24M;
-		break;
-	case IEEE80211_OFDM_RATE_36MB:
-		ret = DESC_RATE36M;
-		break;
-	case IEEE80211_OFDM_RATE_48MB:
-		ret = DESC_RATE48M;
-		break;
-	case IEEE80211_OFDM_RATE_54MB:
-		ret = DESC_RATE54M;
-		break;
-
-		/*  HT rates since here */
-		/* case MGN_MCS0:	ret = DESC_RATEMCS0;    break; */
-		/* case MGN_MCS1:	ret = DESC_RATEMCS1;    break; */
-		/* case MGN_MCS2:	ret = DESC_RATEMCS2;    break; */
-		/* case MGN_MCS3:	ret = DESC_RATEMCS3;    break; */
-		/* case MGN_MCS4:	ret = DESC_RATEMCS4;    break; */
-		/* case MGN_MCS5:	ret = DESC_RATEMCS5;    break; */
-		/* case MGN_MCS6:	ret = DESC_RATEMCS6;    break; */
-		/* case MGN_MCS7:	ret = DESC_RATEMCS7;    break; */
-
-	default:
-		break;
-	}
-	return ret;
-}
-
-void HalSetBrateCfg23a(struct rtw_adapter *padapter, u8 *mBratesOS)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 i, is_brate, brate;
-	u16 brate_cfg = 0;
-	u8 rate_index;
-
-	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
-		is_brate = mBratesOS[i] & IEEE80211_BASIC_RATE_MASK;
-		brate = mBratesOS[i] & 0x7f;
-
-		if (is_brate) {
-			switch (brate) {
-			case IEEE80211_CCK_RATE_1MB:
-				brate_cfg |= RATE_1M;
-				break;
-			case IEEE80211_CCK_RATE_2MB:
-				brate_cfg |= RATE_2M;
-				break;
-			case IEEE80211_CCK_RATE_5MB:
-				brate_cfg |= RATE_5_5M;
-				break;
-			case IEEE80211_CCK_RATE_11MB:
-				brate_cfg |= RATE_11M;
-				break;
-			case IEEE80211_OFDM_RATE_6MB:
-				brate_cfg |= RATE_6M;
-				break;
-			case IEEE80211_OFDM_RATE_9MB:
-				brate_cfg |= RATE_9M;
-				break;
-			case IEEE80211_OFDM_RATE_12MB:
-				brate_cfg |= RATE_12M;
-				break;
-			case IEEE80211_OFDM_RATE_18MB:
-				brate_cfg |= RATE_18M;
-				break;
-			case IEEE80211_OFDM_RATE_24MB:
-				brate_cfg |= RATE_24M;
-				break;
-			case IEEE80211_OFDM_RATE_36MB:
-				brate_cfg |= RATE_36M;
-				break;
-			case IEEE80211_OFDM_RATE_48MB:
-				brate_cfg |= RATE_48M;
-				break;
-			case IEEE80211_OFDM_RATE_54MB:
-				brate_cfg |= RATE_54M;
-				break;
-			}
-		}
-	}
-
-	/*  2007.01.16, by Emily */
-	/*  Select RRSR (in Legacy-OFDM and CCK) */
-	/*  For 8190, we select only 24M, 12M, 6M, 11M, 5.5M, 2M,
-	    and 1M from the Basic rate. */
-	/*  We do not use other rates. */
-	/* 2011.03.30 add by Luke Lee */
-	/* CCK 2M ACK should be disabled for some BCM and Atheros AP IOT */
-	/* because CCK 2M has poor TXEVM */
-	/* CCK 5.5M & 11M ACK should be enabled for better
-	   performance */
-
-	brate_cfg = (brate_cfg | 0xd) & 0x15d;
-	pHalData->BasicRateSet = brate_cfg;
-	brate_cfg |= 0x01;	/*  default enable 1M ACK rate */
-	DBG_8723A("HW_VAR_BASIC_RATE: BrateCfg(%#x)\n", brate_cfg);
-
-	/*  Set RRSR rate table. */
-	rtl8723au_write8(padapter, REG_RRSR, brate_cfg & 0xff);
-	rtl8723au_write8(padapter, REG_RRSR + 1, (brate_cfg >> 8) & 0xff);
-	rtl8723au_write8(padapter, REG_RRSR + 2,
-			 rtl8723au_read8(padapter, REG_RRSR + 2) & 0xf0);
-
-	rate_index = 0;
-	/*  Set RTS initial rate */
-	while (brate_cfg > 0x1) {
-		brate_cfg >>= 1;
-		rate_index++;
-	}
-		/*  Ziv - Check */
-	rtl8723au_write8(padapter, REG_INIRTS_RATE_SEL, rate_index);
-}
-
-static void _OneOutPipeMapping(struct rtw_adapter *pAdapter)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(pAdapter);
-
-	pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0];	/* VO */
-	pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[0];	/* VI */
-	pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[0];	/* BE */
-	pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[0];	/* BK */
-
-	pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0];	/* BCN */
-	pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0];	/* MGT */
-	pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0];	/* HIGH */
-	pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0];	/* TXCMD */
-}
-
-static void _TwoOutPipeMapping(struct rtw_adapter *pAdapter, bool bWIFICfg)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(pAdapter);
-
-	if (bWIFICfg) {		/* WMM */
-		/*    BK,   BE,   VI,   VO,   BCN,  CMD,  MGT, HIGH, HCCA */
-		/*     0,    1,    0,    1,     0,    0,    0,    0,    0 }; */
-		/* 0:H, 1:L */
-		pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[1]; /* VO */
-		pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[0]; /* VI */
-		pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[1]; /* BE */
-		pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[0]; /* BK */
-
-		pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0]; /* BCN */
-		pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0]; /* MGT */
-		pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0]; /* HIGH */
-		pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0]; /* TXCMD*/
-	} else {		/* typical setting */
-		/*    BK,   BE,   VI,   VO,   BCN,  CMD,  MGT, HIGH, HCCA */
-		/*     1,    1,    0,    0,     0,    0,    0,    0,    0 }; */
-		/* 0:H, 1:L */
-		pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0]; /* VO */
-		pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[0]; /* VI */
-		pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[1]; /* BE */
-		pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[1]; /* BK */
-
-		pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0]; /* BCN */
-		pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0]; /* MGT */
-		pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0]; /* HIGH */
-		pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0]; /* TXCMD*/
-	}
-}
-
-static void _ThreeOutPipeMapping(struct rtw_adapter *pAdapter, bool bWIFICfg)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(pAdapter);
-
-	if (bWIFICfg) {		/* for WMM */
-		/*    BK,   BE,   VI,   VO,   BCN,  CMD,  MGT, HIGH, HCCA */
-		/*     1,    2,    1,    0,     0,    0,    0,    0,    0 }; */
-		/* 0:H, 1:N, 2:L */
-		pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0]; /* VO */
-		pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[1]; /* VI */
-		pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[2]; /* BE */
-		pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[1]; /* BK */
-
-		pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0]; /* BCN */
-		pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0]; /* MGT */
-		pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0]; /* HIGH */
-		pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0]; /* TXCMD*/
-	} else {		/* typical setting */
-		/*    BK,   BE,   VI,   VO,   BCN,  CMD,  MGT, HIGH, HCCA */
-		/*     2,    2,    1,    0,     0,    0,    0,    0,    0 }; */
-		/* 0:H, 1:N, 2:L */
-		pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0]; /* VO */
-		pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[1]; /* VI */
-		pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[2]; /* BE */
-		pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[2]; /* BK */
-
-		pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0]; /* BCN */
-		pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0]; /* MGT */
-		pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0]; /* HIGH */
-		pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0]; /* TXCMD*/
-	}
-}
-
-bool Hal_MappingOutPipe23a(struct rtw_adapter *pAdapter, u8 NumOutPipe)
-{
-	struct registry_priv *pregistrypriv = &pAdapter->registrypriv;
-	bool bWIFICfg = (pregistrypriv->wifi_spec) ? true : false;
-	bool result = true;
-
-	switch (NumOutPipe) {
-	case 2:
-		_TwoOutPipeMapping(pAdapter, bWIFICfg);
-		break;
-	case 3:
-		_ThreeOutPipeMapping(pAdapter, bWIFICfg);
-		break;
-	case 1:
-		_OneOutPipeMapping(pAdapter);
-		break;
-	default:
-		result = false;
-		break;
-	}
-
-	return result;
-}
-
-/*
-* C2H event format:
-* Field	 TRIGGER		CONTENT	   CMD_SEQ	CMD_LEN		 CMD_ID
-* BITS	 [127:120]	[119:16]      [15:8]		  [7:4]		   [3:0]
-*/
-
-void c2h_evt_clear23a(struct rtw_adapter *adapter)
-{
-	rtl8723au_write8(adapter, REG_C2HEVT_CLEAR, C2H_EVT_HOST_CLOSE);
-}
-
-int c2h_evt_read23a(struct rtw_adapter *adapter, u8 *buf)
-{
-	int ret = _FAIL;
-	struct c2h_evt_hdr *c2h_evt;
-	int i;
-	u8 trigger;
-
-	if (buf == NULL)
-		goto exit;
-
-	trigger = rtl8723au_read8(adapter, REG_C2HEVT_CLEAR);
-
-	if (trigger == C2H_EVT_HOST_CLOSE)
-		goto exit;	/* Not ready */
-	if (trigger != C2H_EVT_FW_CLOSE)
-		goto clear_evt;	/* Not a valid value */
-
-	c2h_evt = (struct c2h_evt_hdr *)buf;
-
-	memset(c2h_evt, 0, 16);
-
-	*buf = rtl8723au_read8(adapter, REG_C2HEVT_MSG_NORMAL);
-	*(buf + 1) = rtl8723au_read8(adapter, REG_C2HEVT_MSG_NORMAL + 1);
-
-	RT_PRINT_DATA(_module_hal_init_c_, _drv_info_, "c2h_evt_read23a(): ",
-		      &c2h_evt, sizeof(c2h_evt));
-
-	if (0) {
-		DBG_8723A("%s id:%u, len:%u, seq:%u, trigger:0x%02x\n",
-			  __func__, c2h_evt->id, c2h_evt->plen, c2h_evt->seq,
-			  trigger);
-	}
-
-	/* Read the content */
-	for (i = 0; i < c2h_evt->plen; i++)
-		c2h_evt->payload[i] = rtl8723au_read8(adapter,
-						REG_C2HEVT_MSG_NORMAL +
-						sizeof(*c2h_evt) + i);
-
-	RT_PRINT_DATA(_module_hal_init_c_, _drv_info_,
-		      "c2h_evt_read23a(): Command Content:\n", c2h_evt->payload,
-		      c2h_evt->plen);
-
-	ret = _SUCCESS;
-
-clear_evt:
-	/*
-	 * Clear event to notify FW we have read the command.
-	 * If this field isn't clear, the FW won't update the
-	 * next command message.
-	 */
-	c2h_evt_clear23a(adapter);
-exit:
-	return ret;
-}
-
-void
-rtl8723a_set_ampdu_min_space(struct rtw_adapter *padapter, u8 MinSpacingToSet)
-{
-	u8 SecMinSpace;
-
-	if (MinSpacingToSet <= 7) {
-		switch (padapter->securitypriv.dot11PrivacyAlgrthm) {
-		case 0:
-		case WLAN_CIPHER_SUITE_CCMP:
-			SecMinSpace = 0;
-			break;
-
-		case WLAN_CIPHER_SUITE_WEP40:
-		case WLAN_CIPHER_SUITE_WEP104:
-		case WLAN_CIPHER_SUITE_TKIP:
-			SecMinSpace = 6;
-			break;
-		default:
-			SecMinSpace = 7;
-			break;
-		}
-
-		if (MinSpacingToSet < SecMinSpace)
-			MinSpacingToSet = SecMinSpace;
-
-		MinSpacingToSet |=
-			rtl8723au_read8(padapter, REG_AMPDU_MIN_SPACE) & 0xf8;
-		rtl8723au_write8(padapter, REG_AMPDU_MIN_SPACE,
-				 MinSpacingToSet);
-	}
-}
-
-void rtl8723a_set_ampdu_factor(struct rtw_adapter *padapter, u8 FactorToSet)
-{
-	u8 RegToSet_Normal[4] = { 0x41, 0xa8, 0x72, 0xb9 };
-	u8 MaxAggNum;
-	u8 *pRegToSet;
-	u8 index = 0;
-
-	pRegToSet = RegToSet_Normal;	/*  0xb972a841; */
-
-	if (rtl8723a_BT_enabled(padapter) &&
-	    rtl8723a_BT_using_antenna_1(padapter))
-		MaxAggNum = 0x8;
-	else
-		MaxAggNum = 0xF;
-
-	if (FactorToSet <= 3) {
-		FactorToSet = 1 << (FactorToSet + 2);
-		if (FactorToSet > MaxAggNum)
-			FactorToSet = MaxAggNum;
-
-		for (index = 0; index < 4; index++) {
-			if ((pRegToSet[index] & 0xf0) > (FactorToSet << 4))
-				pRegToSet[index] = (pRegToSet[index] & 0x0f) |
-					(FactorToSet << 4);
-
-			if ((pRegToSet[index] & 0x0f) > FactorToSet)
-				pRegToSet[index] = (pRegToSet[index] & 0xf0) |
-					FactorToSet;
-
-			rtl8723au_write8(padapter, REG_AGGLEN_LMT + index,
-					 pRegToSet[index]);
-		}
-	}
-}
-
-void rtl8723a_set_acm_ctrl(struct rtw_adapter *padapter, u8 ctrl)
-{
-	u8 hwctrl = 0;
-
-	if (ctrl != 0) {
-		hwctrl |= AcmHw_HwEn;
-
-		if (ctrl & BIT(1))	/*  BE */
-			hwctrl |= AcmHw_BeqEn;
-
-		if (ctrl & BIT(2))	/*  VI */
-			hwctrl |= AcmHw_ViqEn;
-
-		if (ctrl & BIT(3))	/*  VO */
-			hwctrl |= AcmHw_VoqEn;
-	}
-
-	DBG_8723A("[HW_VAR_ACM_CTRL] Write 0x%02X\n", hwctrl);
-	rtl8723au_write8(padapter, REG_ACMHWCTRL, hwctrl);
-}
-
-void rtl8723a_set_media_status(struct rtw_adapter *padapter, u8 status)
-{
-	u8 val8;
-
-	val8 = rtl8723au_read8(padapter, MSR) & 0x0c;
-	val8 |= status;
-	rtl8723au_write8(padapter, MSR, val8);
-}
-
-void rtl8723a_set_media_status1(struct rtw_adapter *padapter, u8 status)
-{
-	u8 val8;
-
-	val8 = rtl8723au_read8(padapter, MSR) & 0x03;
-	val8 |= status << 2;
-	rtl8723au_write8(padapter, MSR, val8);
-}
-
-void rtl8723a_set_bcn_func(struct rtw_adapter *padapter, u8 val)
-{
-	if (val)
-		SetBcnCtrlReg23a(padapter, EN_BCN_FUNCTION | EN_TXBCN_RPT, 0);
-	else
-		SetBcnCtrlReg23a(padapter, 0, EN_BCN_FUNCTION | EN_TXBCN_RPT);
-}
-
-void rtl8723a_check_bssid(struct rtw_adapter *padapter, u8 val)
-{
-	u32 val32;
-
-	val32 = rtl8723au_read32(padapter, REG_RCR);
-	if (val)
-		val32 |= RCR_CBSSID_DATA | RCR_CBSSID_BCN;
-	else
-		val32 &= ~(RCR_CBSSID_DATA | RCR_CBSSID_BCN);
-	rtl8723au_write32(padapter, REG_RCR, val32);
-}
-
-void rtl8723a_mlme_sitesurvey(struct rtw_adapter *padapter, u8 flag)
-{
-	if (flag) {	/* under sitesurvey */
-		u32 v32;
-
-		/*  config RCR to receive different BSSID & not
-		    to receive data frame */
-		v32 = rtl8723au_read32(padapter, REG_RCR);
-		v32 &= ~(RCR_CBSSID_BCN);
-		rtl8723au_write32(padapter, REG_RCR, v32);
-		/*  reject all data frame */
-		rtl8723au_write16(padapter, REG_RXFLTMAP2, 0);
-
-		/*  disable update TSF */
-		SetBcnCtrlReg23a(padapter, DIS_TSF_UDT, 0);
-	} else {	/* sitesurvey done */
-
-		struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-		struct mlme_ext_info *pmlmeinfo;
-		u32 v32;
-
-		pmlmeinfo = &pmlmeext->mlmext_info;
-
-		if ((is_client_associated_to_ap23a(padapter) == true) ||
-		    ((pmlmeinfo->state & 0x03) == MSR_ADHOC) ||
-		    ((pmlmeinfo->state & 0x03) == MSR_AP)) {
-			/*  enable to rx data frame */
-			rtl8723au_write16(padapter, REG_RXFLTMAP2, 0xFFFF);
-
-			/*  enable update TSF */
-			SetBcnCtrlReg23a(padapter, 0, DIS_TSF_UDT);
-		}
-
-		v32 = rtl8723au_read32(padapter, REG_RCR);
-		v32 |= RCR_CBSSID_BCN;
-		rtl8723au_write32(padapter, REG_RCR, v32);
-	}
-
-	rtl8723a_BT_wifiscan_notify(padapter, flag ? true : false);
-}
-
-void rtl8723a_on_rcr_am(struct rtw_adapter *padapter)
-{
-	rtl8723au_write32(padapter, REG_RCR,
-		    rtl8723au_read32(padapter, REG_RCR) | RCR_AM);
-	DBG_8723A("%s, %d, RCR = %x\n", __func__, __LINE__,
-		  rtl8723au_read32(padapter, REG_RCR));
-}
-
-void rtl8723a_off_rcr_am(struct rtw_adapter *padapter)
-{
-	rtl8723au_write32(padapter, REG_RCR,
-		    rtl8723au_read32(padapter, REG_RCR) & (~RCR_AM));
-	DBG_8723A("%s, %d, RCR = %x\n", __func__, __LINE__,
-		  rtl8723au_read32(padapter, REG_RCR));
-}
-
-void rtl8723a_set_slot_time(struct rtw_adapter *padapter, u8 slottime)
-{
-	u8 u1bAIFS, aSifsTime;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	rtl8723au_write8(padapter, REG_SLOT, slottime);
-
-	if (pmlmeinfo->WMM_enable == 0) {
-		if (pmlmeext->cur_wireless_mode == WIRELESS_11B)
-			aSifsTime = 10;
-		else
-			aSifsTime = 16;
-
-		u1bAIFS = aSifsTime + (2 * pmlmeinfo->slotTime);
-
-		/*  <Roger_EXP> Temporary removed, 2008.06.20. */
-		rtl8723au_write8(padapter, REG_EDCA_VO_PARAM, u1bAIFS);
-		rtl8723au_write8(padapter, REG_EDCA_VI_PARAM, u1bAIFS);
-		rtl8723au_write8(padapter, REG_EDCA_BE_PARAM, u1bAIFS);
-		rtl8723au_write8(padapter, REG_EDCA_BK_PARAM, u1bAIFS);
-	}
-}
-
-void rtl8723a_ack_preamble(struct rtw_adapter *padapter, u8 bShortPreamble)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 regTmp;
-
-	/*  Joseph marked out for Netgear 3500 TKIP
-	    channel 7 issue.(Temporarily) */
-	regTmp = (pHalData->nCur40MhzPrimeSC) << 5;
-	/* regTmp = 0; */
-	if (bShortPreamble)
-		regTmp |= 0x80;
-	rtl8723au_write8(padapter, REG_RRSR + 2, regTmp);
-}
-
-void rtl8723a_set_sec_cfg(struct rtw_adapter *padapter, u8 sec)
-{
-	rtl8723au_write8(padapter, REG_SECCFG, sec);
-}
-
-void rtl8723a_cam_empty_entry(struct rtw_adapter *padapter, u8 ucIndex)
-{
-	u8 i;
-	u32 ulCommand = 0;
-	u32 ulContent = 0;
-	u32 ulEncAlgo = CAM_AES;
-
-	for (i = 0; i < CAM_CONTENT_COUNT; i++) {
-		/*  filled id in CAM config 2 byte */
-		if (i == 0) {
-			ulContent |= (ucIndex & 0x03) |
-				((u16) (ulEncAlgo) << 2);
-			/* ulContent |= CAM_VALID; */
-		} else {
-			ulContent = 0;
-		}
-		/*  polling bit, and No Write enable, and address */
-		ulCommand = CAM_CONTENT_COUNT * ucIndex + i;
-		ulCommand = ulCommand | CAM_POLLINIG | CAM_WRITE;
-		/*  write content 0 is equall to mark invalid */
-		/* delay_ms(40); */
-		rtl8723au_write32(padapter, WCAMI, ulContent);
-		/* delay_ms(40); */
-		rtl8723au_write32(padapter, REG_CAMCMD, ulCommand);
-	}
-}
-
-void rtl8723a_cam_invalidate_all(struct rtw_adapter *padapter)
-{
-	rtl8723au_write32(padapter, REG_CAMCMD, CAM_POLLINIG | BIT(30));
-}
-
-void rtl8723a_cam_write(struct rtw_adapter *padapter,
-			u8 entry, u16 ctrl, const u8 *mac, const u8 *key)
-{
-	u32 cmd;
-	unsigned int i, val, addr;
-	int j;
-
-	addr = entry << 3;
-
-	for (j = 5; j >= 0; j--) {
-		switch (j) {
-		case 0:
-			val = ctrl | (mac[0] << 16) | (mac[1] << 24);
-			break;
-		case 1:
-			val = mac[2] | (mac[3] << 8) |
-				(mac[4] << 16) | (mac[5] << 24);
-			break;
-		default:
-			i = (j - 2) << 2;
-			val = key[i] | (key[i+1] << 8) |
-				(key[i+2] << 16) | (key[i+3] << 24);
-			break;
-		}
-
-		rtl8723au_write32(padapter, WCAMI, val);
-		cmd = CAM_POLLINIG | CAM_WRITE | (addr + j);
-		rtl8723au_write32(padapter, REG_CAMCMD, cmd);
-
-		/* DBG_8723A("%s => cam write: %x, %x\n", __func__, cmd, val);*/
-	}
-}
-
-void rtl8723a_fifo_cleanup(struct rtw_adapter *padapter)
-{
-#define RW_RELEASE_EN		BIT(18)
-#define RXDMA_IDLE		BIT(17)
-
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-	u8 trycnt = 100;
-
-	/*  pause tx */
-	rtl8723au_write8(padapter, REG_TXPAUSE, 0xff);
-
-	/*  keep sn */
-	padapter->xmitpriv.nqos_ssn = rtl8723au_read8(padapter, REG_NQOS_SEQ);
-
-	if (pwrpriv->bkeepfwalive != true) {
-		u32 v32;
-
-		/*  RX DMA stop */
-		v32 = rtl8723au_read32(padapter, REG_RXPKT_NUM);
-		v32 |= RW_RELEASE_EN;
-		rtl8723au_write32(padapter, REG_RXPKT_NUM, v32);
-		do {
-			v32 = rtl8723au_read32(padapter,
-					       REG_RXPKT_NUM) & RXDMA_IDLE;
-			if (!v32)
-				break;
-		} while (trycnt--);
-		if (trycnt == 0)
-			DBG_8723A("Stop RX DMA failed......\n");
-
-		/*  RQPN Load 0 */
-		rtl8723au_write16(padapter, REG_RQPN_NPQ, 0);
-		rtl8723au_write32(padapter, REG_RQPN, 0x80000000);
-		mdelay(10);
-	}
-}
-
-void rtl8723a_bcn_valid(struct rtw_adapter *padapter)
-{
-	/* BCN_VALID, BIT16 of REG_TDECTRL = BIT0 of REG_TDECTRL+2,
-	   write 1 to clear, Clear by sw */
-	rtl8723au_write8(padapter, REG_TDECTRL + 2,
-			 rtl8723au_read8(padapter, REG_TDECTRL + 2) | BIT(0));
-}
-
-bool rtl8723a_get_bcn_valid(struct rtw_adapter *padapter)
-{
-	bool retval;
-
-	retval = (rtl8723au_read8(padapter, REG_TDECTRL + 2) & BIT(0)) ? true : false;
-
-	return retval;
-}
-
-void rtl8723a_set_beacon_interval(struct rtw_adapter *padapter, u16 interval)
-{
-	rtl8723au_write16(padapter, REG_BCN_INTERVAL, interval);
-}
-
-void rtl8723a_set_resp_sifs(struct rtw_adapter *padapter,
-			    u8 r2t1, u8 r2t2, u8 t2t1, u8 t2t2)
-{
-	/* SIFS_Timer = 0x0a0a0808; */
-	/* RESP_SIFS for CCK */
-	/*  SIFS_T2T_CCK (0x08) */
-	rtl8723au_write8(padapter, REG_R2T_SIFS, r2t1);
-	/* SIFS_R2T_CCK(0x08) */
-	rtl8723au_write8(padapter, REG_R2T_SIFS + 1, r2t2);
-	/* RESP_SIFS for OFDM */
-	/* SIFS_T2T_OFDM (0x0a) */
-	rtl8723au_write8(padapter, REG_T2T_SIFS, t2t1);
-	/* SIFS_R2T_OFDM(0x0a) */
-	rtl8723au_write8(padapter, REG_T2T_SIFS + 1, t2t2);
-}
-
-void rtl8723a_set_ac_param_vo(struct rtw_adapter *padapter, u32 vo)
-{
-	rtl8723au_write32(padapter, REG_EDCA_VO_PARAM, vo);
-}
-
-void rtl8723a_set_ac_param_vi(struct rtw_adapter *padapter, u32 vi)
-{
-	rtl8723au_write32(padapter, REG_EDCA_VI_PARAM, vi);
-}
-
-void rtl8723a_set_ac_param_be(struct rtw_adapter *padapter, u32 be)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	pHalData->AcParam_BE = be;
-	rtl8723au_write32(padapter, REG_EDCA_BE_PARAM, be);
-}
-
-void rtl8723a_set_ac_param_bk(struct rtw_adapter *padapter, u32 bk)
-{
-	rtl8723au_write32(padapter, REG_EDCA_BK_PARAM, bk);
-}
-
-void rtl8723a_set_rxdma_agg_pg_th(struct rtw_adapter *padapter, u8 val)
-{
-	rtl8723au_write8(padapter, REG_RXDMA_AGG_PG_TH, val);
-}
-
-void rtl8723a_set_initial_gain(struct rtw_adapter *padapter, u32 rx_gain)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct dig_t *pDigTable = &pHalData->odmpriv.DM_DigTable;
-
-	if (rx_gain == 0xff)	/* restore rx gain */
-		ODM_Write_DIG23a(&pHalData->odmpriv, pDigTable->BackupIGValue);
-	else {
-		pDigTable->BackupIGValue = pDigTable->CurIGValue;
-		ODM_Write_DIG23a(&pHalData->odmpriv, rx_gain);
-	}
-}
-
-void rtl8723a_odm_support_ability_restore(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	pHalData->odmpriv.SupportAbility = pHalData->odmpriv.BK_SupportAbility;
-}
-
-void rtl8723a_odm_support_ability_backup(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	pHalData->odmpriv.BK_SupportAbility = pHalData->odmpriv.SupportAbility;
-}
-
-void rtl8723a_odm_support_ability_set(struct rtw_adapter *padapter, u32 val)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (val == DYNAMIC_ALL_FUNC_ENABLE)
-		pHalData->odmpriv.SupportAbility = pHalData->dmpriv.InitODMFlag;
-	else
-		pHalData->odmpriv.SupportAbility |= val;
-}
-
-void rtl8723a_odm_support_ability_clr(struct rtw_adapter *padapter, u32 val)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	pHalData->odmpriv.SupportAbility &= val;
-}
-
-void rtl8723a_set_rpwm(struct rtw_adapter *padapter, u8 val)
-{
-	rtl8723au_write8(padapter, REG_USB_HRPWM, val);
-}
-
-u8 rtl8723a_get_rf_type(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	return pHalData->rf_type;
-}
-
-bool rtl8723a_get_fwlps_rf_on(struct rtw_adapter *padapter)
-{
-	bool retval;
-	u32 valRCR;
-
-	/*  When we halt NIC, we should check if FW LPS is leave. */
-
-	if ((padapter->bSurpriseRemoved == true) ||
-	    (padapter->pwrctrlpriv.rf_pwrstate == rf_off)) {
-		/*  If it is in HW/SW Radio OFF or IPS state, we do
-		    not check Fw LPS Leave, because Fw is unload. */
-		retval = true;
-	} else {
-		valRCR = rtl8723au_read32(padapter, REG_RCR);
-		if (valRCR & 0x00070000)
-			retval = false;
-		else
-			retval = true;
-	}
-
-	return retval;
-}
-
-bool rtl8723a_chk_hi_queue_empty(struct rtw_adapter *padapter)
-{
-	u32 hgq;
-
-	hgq = rtl8723au_read32(padapter, REG_HGQ_INFORMATION);
-
-	return ((hgq & 0x0000ff00) == 0) ? true : false;
-}
diff --git a/drivers/staging/rtl8723au/hal/hal_intf.c b/drivers/staging/rtl8723au/hal/hal_intf.c
deleted file mode 100644
index 5383e692546f..000000000000
--- a/drivers/staging/rtl8723au/hal/hal_intf.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#define _HAL_INTF_C_
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#include <hal_intf.h>
-
-#include <rtl8723a_hal.h>
-
-void rtw_hal_update_ra_mask23a(struct sta_info *psta, u8 rssi_level)
-{
-	struct rtw_adapter *padapter;
-	struct mlme_priv *pmlmepriv;
-
-	if (!psta)
-		return;
-
-	padapter = psta->padapter;
-
-	pmlmepriv = &padapter->mlmepriv;
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-#ifdef CONFIG_8723AU_AP_MODE
-		add_RATid23a(padapter, psta, rssi_level);
-#endif
-	} else
-		rtl8723a_update_ramask(padapter, psta->mac_id, rssi_level);
-}
diff --git a/drivers/staging/rtl8723au/hal/odm.c b/drivers/staging/rtl8723au/hal/odm.c
deleted file mode 100644
index e279c34b3fc6..000000000000
--- a/drivers/staging/rtl8723au/hal/odm.c
+++ /dev/null
@@ -1,1732 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#include "odm_precomp.h"
-#include "usb_ops_linux.h"
-
-static const u16 dB_Invert_Table[8][12] = {
-	{1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4},
-	{4, 5, 6, 6, 7, 8, 9, 10, 11, 13, 14, 16},
-	{18, 20, 22, 25, 28, 32, 35, 40, 45, 50, 56, 63},
-	{71, 79, 89, 100, 112, 126, 141, 158, 178, 200, 224, 251},
-	{282, 316, 355, 398, 447, 501, 562, 631, 708, 794, 891, 1000},
-	{1122, 1259, 1413, 1585, 1778, 1995, 2239, 2512, 2818, 3162, 3548, 3981},
-	{4467, 5012, 5623, 6310, 7079, 7943, 8913, 10000, 11220, 12589, 14125, 15849},
-	{17783, 19953, 22387, 25119, 28184, 31623, 35481, 39811, 44668, 50119, 56234, 65535}
-};
-
-static u32 EDCAParam[HT_IOT_PEER_MAX][3] = {          /*  UL			DL */
-	{0x5ea42b, 0x5ea42b, 0x5ea42b}, /* 0:unknown AP */
-	{0xa44f, 0x5ea44f, 0x5e431c}, /*  1:realtek AP */
-	{0x5ea42b, 0x5ea42b, 0x5ea42b}, /*  2:unknown AP => realtek_92SE */
-	{0x5ea32b, 0x5ea42b, 0x5e4322}, /*  3:broadcom AP */
-	{0x5ea422, 0x00a44f, 0x00a44f}, /*  4:ralink AP */
-	{0x5ea322, 0x00a630, 0x00a44f}, /*  5:atheros AP */
-	{0x5e4322, 0x5e4322, 0x5e4322},/*  6:cisco AP */
-	{0x5ea44f, 0x00a44f, 0x5ea42b}, /*  8:marvell AP */
-	{0x5ea42b, 0x5ea42b, 0x5ea42b}, /*  10:unknown AP => 92U AP */
-	{0x5ea42b, 0xa630, 0x5e431c}, /*  11:airgocap AP */
-};
-
-/*  EDCA Parameter for AP/ADSL   by Mingzhi 2011-11-22 */
-
-/*  Global var */
-u32 OFDMSwingTable23A[OFDM_TABLE_SIZE_92D] = {
-	0x7f8001fe, /*  0, +6.0dB */
-	0x788001e2, /*  1, +5.5dB */
-	0x71c001c7, /*  2, +5.0dB */
-	0x6b8001ae, /*  3, +4.5dB */
-	0x65400195, /*  4, +4.0dB */
-	0x5fc0017f, /*  5, +3.5dB */
-	0x5a400169, /*  6, +3.0dB */
-	0x55400155, /*  7, +2.5dB */
-	0x50800142, /*  8, +2.0dB */
-	0x4c000130, /*  9, +1.5dB */
-	0x47c0011f, /*  10, +1.0dB */
-	0x43c0010f, /*  11, +0.5dB */
-	0x40000100, /*  12, +0dB */
-	0x3c8000f2, /*  13, -0.5dB */
-	0x390000e4, /*  14, -1.0dB */
-	0x35c000d7, /*  15, -1.5dB */
-	0x32c000cb, /*  16, -2.0dB */
-	0x300000c0, /*  17, -2.5dB */
-	0x2d4000b5, /*  18, -3.0dB */
-	0x2ac000ab, /*  19, -3.5dB */
-	0x288000a2, /*  20, -4.0dB */
-	0x26000098, /*  21, -4.5dB */
-	0x24000090, /*  22, -5.0dB */
-	0x22000088, /*  23, -5.5dB */
-	0x20000080, /*  24, -6.0dB */
-	0x1e400079, /*  25, -6.5dB */
-	0x1c800072, /*  26, -7.0dB */
-	0x1b00006c, /*  27. -7.5dB */
-	0x19800066, /*  28, -8.0dB */
-	0x18000060, /*  29, -8.5dB */
-	0x16c0005b, /*  30, -9.0dB */
-	0x15800056, /*  31, -9.5dB */
-	0x14400051, /*  32, -10.0dB */
-	0x1300004c, /*  33, -10.5dB */
-	0x12000048, /*  34, -11.0dB */
-	0x11000044, /*  35, -11.5dB */
-	0x10000040, /*  36, -12.0dB */
-	0x0f00003c,/*  37, -12.5dB */
-	0x0e400039,/*  38, -13.0dB */
-	0x0d800036,/*  39, -13.5dB */
-	0x0cc00033,/*  40, -14.0dB */
-	0x0c000030,/*  41, -14.5dB */
-	0x0b40002d,/*  42, -15.0dB */
-};
-
-u8 CCKSwingTable_Ch1_Ch1323A[CCK_TABLE_SIZE][8] = {
-	{0x36, 0x35, 0x2e, 0x25, 0x1c, 0x12, 0x09, 0x04}, /*  0, +0dB */
-	{0x33, 0x32, 0x2b, 0x23, 0x1a, 0x11, 0x08, 0x04}, /*  1, -0.5dB */
-	{0x30, 0x2f, 0x29, 0x21, 0x19, 0x10, 0x08, 0x03}, /*  2, -1.0dB */
-	{0x2d, 0x2d, 0x27, 0x1f, 0x18, 0x0f, 0x08, 0x03}, /*  3, -1.5dB */
-	{0x2b, 0x2a, 0x25, 0x1e, 0x16, 0x0e, 0x07, 0x03}, /*  4, -2.0dB */
-	{0x28, 0x28, 0x22, 0x1c, 0x15, 0x0d, 0x07, 0x03}, /*  5, -2.5dB */
-	{0x26, 0x25, 0x21, 0x1b, 0x14, 0x0d, 0x06, 0x03}, /*  6, -3.0dB */
-	{0x24, 0x23, 0x1f, 0x19, 0x13, 0x0c, 0x06, 0x03}, /*  7, -3.5dB */
-	{0x22, 0x21, 0x1d, 0x18, 0x11, 0x0b, 0x06, 0x02}, /*  8, -4.0dB */
-	{0x20, 0x20, 0x1b, 0x16, 0x11, 0x08, 0x05, 0x02}, /*  9, -4.5dB */
-	{0x1f, 0x1e, 0x1a, 0x15, 0x10, 0x0a, 0x05, 0x02}, /*  10, -5.0dB */
-	{0x1d, 0x1c, 0x18, 0x14, 0x0f, 0x0a, 0x05, 0x02}, /*  11, -5.5dB */
-	{0x1b, 0x1a, 0x17, 0x13, 0x0e, 0x09, 0x04, 0x02}, /*  12, -6.0dB */
-	{0x1a, 0x19, 0x16, 0x12, 0x0d, 0x09, 0x04, 0x02}, /*  13, -6.5dB */
-	{0x18, 0x17, 0x15, 0x11, 0x0c, 0x08, 0x04, 0x02}, /*  14, -7.0dB */
-	{0x17, 0x16, 0x13, 0x10, 0x0c, 0x08, 0x04, 0x02}, /*  15, -7.5dB */
-	{0x16, 0x15, 0x12, 0x0f, 0x0b, 0x07, 0x04, 0x01}, /*  16, -8.0dB */
-	{0x14, 0x14, 0x11, 0x0e, 0x0b, 0x07, 0x03, 0x02}, /*  17, -8.5dB */
-	{0x13, 0x13, 0x10, 0x0d, 0x0a, 0x06, 0x03, 0x01}, /*  18, -9.0dB */
-	{0x12, 0x12, 0x0f, 0x0c, 0x09, 0x06, 0x03, 0x01}, /*  19, -9.5dB */
-	{0x11, 0x11, 0x0f, 0x0c, 0x09, 0x06, 0x03, 0x01}, /*  20, -10.0dB */
-	{0x10, 0x10, 0x0e, 0x0b, 0x08, 0x05, 0x03, 0x01}, /*  21, -10.5dB */
-	{0x0f, 0x0f, 0x0d, 0x0b, 0x08, 0x05, 0x03, 0x01}, /*  22, -11.0dB */
-	{0x0e, 0x0e, 0x0c, 0x0a, 0x08, 0x05, 0x02, 0x01}, /*  23, -11.5dB */
-	{0x0d, 0x0d, 0x0c, 0x0a, 0x07, 0x05, 0x02, 0x01}, /*  24, -12.0dB */
-	{0x0d, 0x0c, 0x0b, 0x09, 0x07, 0x04, 0x02, 0x01}, /*  25, -12.5dB */
-	{0x0c, 0x0c, 0x0a, 0x09, 0x06, 0x04, 0x02, 0x01}, /*  26, -13.0dB */
-	{0x0b, 0x0b, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x01}, /*  27, -13.5dB */
-	{0x0b, 0x0a, 0x09, 0x08, 0x06, 0x04, 0x02, 0x01}, /*  28, -14.0dB */
-	{0x0a, 0x0a, 0x09, 0x07, 0x05, 0x03, 0x02, 0x01}, /*  29, -14.5dB */
-	{0x0a, 0x09, 0x08, 0x07, 0x05, 0x03, 0x02, 0x01}, /*  30, -15.0dB */
-	{0x09, 0x09, 0x08, 0x06, 0x05, 0x03, 0x01, 0x01}, /*  31, -15.5dB */
-	{0x09, 0x08, 0x07, 0x06, 0x04, 0x03, 0x01, 0x01}	/*  32, -16.0dB */
-};
-
-u8 CCKSwingTable_Ch1423A[CCK_TABLE_SIZE][8] = {
-	{0x36, 0x35, 0x2e, 0x1b, 0x00, 0x00, 0x00, 0x00}, /*  0, +0dB */
-	{0x33, 0x32, 0x2b, 0x19, 0x00, 0x00, 0x00, 0x00}, /*  1, -0.5dB */
-	{0x30, 0x2f, 0x29, 0x18, 0x00, 0x00, 0x00, 0x00}, /*  2, -1.0dB */
-	{0x2d, 0x2d, 0x17, 0x17, 0x00, 0x00, 0x00, 0x00}, /*  3, -1.5dB */
-	{0x2b, 0x2a, 0x25, 0x15, 0x00, 0x00, 0x00, 0x00}, /*  4, -2.0dB */
-	{0x28, 0x28, 0x24, 0x14, 0x00, 0x00, 0x00, 0x00}, /*  5, -2.5dB */
-	{0x26, 0x25, 0x21, 0x13, 0x00, 0x00, 0x00, 0x00}, /*  6, -3.0dB */
-	{0x24, 0x23, 0x1f, 0x12, 0x00, 0x00, 0x00, 0x00}, /*  7, -3.5dB */
-	{0x22, 0x21, 0x1d, 0x11, 0x00, 0x00, 0x00, 0x00}, /*  8, -4.0dB */
-	{0x20, 0x20, 0x1b, 0x10, 0x00, 0x00, 0x00, 0x00}, /*  9, -4.5dB */
-	{0x1f, 0x1e, 0x1a, 0x0f, 0x00, 0x00, 0x00, 0x00}, /*  10, -5.0dB */
-	{0x1d, 0x1c, 0x18, 0x0e, 0x00, 0x00, 0x00, 0x00}, /*  11, -5.5dB */
-	{0x1b, 0x1a, 0x17, 0x0e, 0x00, 0x00, 0x00, 0x00}, /*  12, -6.0dB */
-	{0x1a, 0x19, 0x16, 0x0d, 0x00, 0x00, 0x00, 0x00}, /*  13, -6.5dB */
-	{0x18, 0x17, 0x15, 0x0c, 0x00, 0x00, 0x00, 0x00}, /*  14, -7.0dB */
-	{0x17, 0x16, 0x13, 0x0b, 0x00, 0x00, 0x00, 0x00}, /*  15, -7.5dB */
-	{0x16, 0x15, 0x12, 0x0b, 0x00, 0x00, 0x00, 0x00}, /*  16, -8.0dB */
-	{0x14, 0x14, 0x11, 0x0a, 0x00, 0x00, 0x00, 0x00}, /*  17, -8.5dB */
-	{0x13, 0x13, 0x10, 0x0a, 0x00, 0x00, 0x00, 0x00}, /*  18, -9.0dB */
-	{0x12, 0x12, 0x0f, 0x09, 0x00, 0x00, 0x00, 0x00}, /*  19, -9.5dB */
-	{0x11, 0x11, 0x0f, 0x09, 0x00, 0x00, 0x00, 0x00}, /*  20, -10.0dB */
-	{0x10, 0x10, 0x0e, 0x08, 0x00, 0x00, 0x00, 0x00}, /*  21, -10.5dB */
-	{0x0f, 0x0f, 0x0d, 0x08, 0x00, 0x00, 0x00, 0x00}, /*  22, -11.0dB */
-	{0x0e, 0x0e, 0x0c, 0x07, 0x00, 0x00, 0x00, 0x00}, /*  23, -11.5dB */
-	{0x0d, 0x0d, 0x0c, 0x07, 0x00, 0x00, 0x00, 0x00}, /*  24, -12.0dB */
-	{0x0d, 0x0c, 0x0b, 0x06, 0x00, 0x00, 0x00, 0x00}, /*  25, -12.5dB */
-	{0x0c, 0x0c, 0x0a, 0x06, 0x00, 0x00, 0x00, 0x00}, /*  26, -13.0dB */
-	{0x0b, 0x0b, 0x0a, 0x06, 0x00, 0x00, 0x00, 0x00}, /*  27, -13.5dB */
-	{0x0b, 0x0a, 0x09, 0x05, 0x00, 0x00, 0x00, 0x00}, /*  28, -14.0dB */
-	{0x0a, 0x0a, 0x09, 0x05, 0x00, 0x00, 0x00, 0x00}, /*  29, -14.5dB */
-	{0x0a, 0x09, 0x08, 0x05, 0x00, 0x00, 0x00, 0x00}, /*  30, -15.0dB */
-	{0x09, 0x09, 0x08, 0x05, 0x00, 0x00, 0x00, 0x00}, /*  31, -15.5dB */
-	{0x09, 0x08, 0x07, 0x04, 0x00, 0x00, 0x00, 0x00}	/*  32, -16.0dB */
-};
-
-/*  Local Function predefine. */
-
-/* START------------COMMON INFO RELATED--------------- */
-void odm_CommonInfoSelfInit23a(struct dm_odm_t *pDM_Odm);
-
-static void odm_CommonInfoSelfUpdate(struct hal_data_8723a *pHalData);
-
-void odm_CmnInfoInit_Debug23a(struct dm_odm_t *pDM_Odm);
-
-void odm_CmnInfoUpdate_Debug23a(struct dm_odm_t *pDM_Odm);
-
-/* START---------------DIG--------------------------- */
-void odm_FalseAlarmCounterStatistics23a(struct dm_odm_t *pDM_Odm);
-
-void odm_DIG23aInit(struct dm_odm_t *pDM_Odm);
-
-void odm_DIG23a(struct rtw_adapter *adapter);
-
-void odm_CCKPacketDetectionThresh23a(struct dm_odm_t *pDM_Odm);
-/* END---------------DIG--------------------------- */
-
-/* START-------BB POWER SAVE----------------------- */
-void odm23a_DynBBPSInit(struct dm_odm_t *pDM_Odm);
-
-
-/* END---------BB POWER SAVE----------------------- */
-
-void odm_DynamicTxPower23aInit(struct dm_odm_t *pDM_Odm);
-
-static void odm_RSSIMonitorCheck(struct dm_odm_t *pDM_Odm);
-void odm_DynamicTxPower23a(struct dm_odm_t *pDM_Odm);
-
-static void odm_RefreshRateAdaptiveMask(struct dm_odm_t *pDM_Odm);
-
-void odm_RateAdaptiveMaskInit23a(struct dm_odm_t *pDM_Odm);
-
-static void odm_TXPowerTrackingInit(struct dm_odm_t *pDM_Odm);
-
-static void odm_EdcaTurboCheck23a(struct dm_odm_t *pDM_Odm);
-static void ODM_EdcaTurboInit23a(struct dm_odm_t *pDM_Odm);
-
-#define		RxDefaultAnt1		0x65a9
-#define	RxDefaultAnt2		0x569a
-
-bool odm_StaDefAntSel(struct dm_odm_t *pDM_Odm,
-	u32 OFDM_Ant1_Cnt,
-	u32 OFDM_Ant2_Cnt,
-	u32 CCK_Ant1_Cnt,
-	u32 CCK_Ant2_Cnt,
-	u8 *pDefAnt
-	);
-
-void odm_SetRxIdleAnt(struct dm_odm_t *pDM_Odm,
-	u8 Ant,
-	bool   bDualPath
-);
-
-/* 3 Export Interface */
-
-/*  2011/09/21 MH Add to describe different team necessary resource allocate?? */
-void ODM23a_DMInit(struct dm_odm_t *pDM_Odm)
-{
-	/* For all IC series */
-	odm_CommonInfoSelfInit23a(pDM_Odm);
-	odm_CmnInfoInit_Debug23a(pDM_Odm);
-	odm_DIG23aInit(pDM_Odm);
-	odm_RateAdaptiveMaskInit23a(pDM_Odm);
-
-	odm23a_DynBBPSInit(pDM_Odm);
-	odm_DynamicTxPower23aInit(pDM_Odm);
-	odm_TXPowerTrackingInit(pDM_Odm);
-	ODM_EdcaTurboInit23a(pDM_Odm);
-}
-
-/*  2011/09/20 MH This is the entry pointer for all team to execute HW out source DM. */
-/*  You can not add any dummy function here, be care, you can only use DM structure */
-/*  to perform any new ODM_DM. */
-void ODM_DMWatchdog23a(struct rtw_adapter *adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(adapter);
-	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
-	struct pwrctrl_priv *pwrctrlpriv = &adapter->pwrctrlpriv;
-
-	/* 2012.05.03 Luke: For all IC series */
-	odm_CmnInfoUpdate_Debug23a(pDM_Odm);
-	odm_CommonInfoSelfUpdate(pHalData);
-	odm_FalseAlarmCounterStatistics23a(pDM_Odm);
-	odm_RSSIMonitorCheck(pDM_Odm);
-
-	/* 8723A or 8189ES platform */
-	/* NeilChen--2012--08--24-- */
-	/* Fix Leave LPS issue */
-	if ((pDM_Odm->Adapter->pwrctrlpriv.pwr_mode != PS_MODE_ACTIVE) &&/*  in LPS mode */
-	    (pDM_Odm->SupportICType & ODM_RTL8723A)) {
-			ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("----Step1: odm_DIG23a is in LPS mode\n"));
-			ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("---Step2: 8723AS is in LPS mode\n"));
-			odm_DIG23abyRSSI_LPS(pDM_Odm);
-	} else {
-		odm_DIG23a(adapter);
-	}
-
-	odm_CCKPacketDetectionThresh23a(pDM_Odm);
-
-	if (pwrctrlpriv->bpower_saving)
-		return;
-
-	odm_RefreshRateAdaptiveMask(pDM_Odm);
-
-
-	odm_EdcaTurboCheck23a(pDM_Odm);
-}
-
-/*  */
-/*  Init /.. Fixed HW value. Only init time. */
-/*  */
-void ODM_CmnInfoInit23a(struct dm_odm_t *pDM_Odm,
-		enum odm_cmninfo CmnInfo,
-		u32 Value
-	)
-{
-	/* ODM_RT_TRACE(pDM_Odm,); */
-
-	/*  */
-	/*  This section is used for init value */
-	/*  */
-	switch	(CmnInfo) {
-	/*  Fixed ODM value. */
-	case	ODM_CMNINFO_MP_TEST_CHIP:
-		pDM_Odm->bIsMPChip = (u8)Value;
-		break;
-	case	ODM_CMNINFO_IC_TYPE:
-		pDM_Odm->SupportICType = Value;
-		break;
-	case	ODM_CMNINFO_CUT_VER:
-		pDM_Odm->CutVersion = (u8)Value;
-		break;
-	case	ODM_CMNINFO_FAB_VER:
-		pDM_Odm->FabVersion = (u8)Value;
-		break;
-	case	ODM_CMNINFO_BOARD_TYPE:
-		pDM_Odm->BoardType = (u8)Value;
-		break;
-	case	ODM_CMNINFO_EXT_LNA:
-		pDM_Odm->ExtLNA = (u8)Value;
-		break;
-	case	ODM_CMNINFO_EXT_PA:
-		pDM_Odm->ExtPA = (u8)Value;
-		break;
-	case	ODM_CMNINFO_EXT_TRSW:
-		pDM_Odm->ExtTRSW = (u8)Value;
-		break;
-	case	ODM_CMNINFO_BINHCT_TEST:
-		pDM_Odm->bInHctTest = (bool)Value;
-		break;
-	case	ODM_CMNINFO_BWIFI_TEST:
-		pDM_Odm->bWIFITest = (bool)Value;
-		break;
-	case	ODM_CMNINFO_SMART_CONCURRENT:
-		pDM_Odm->bDualMacSmartConcurrent = (bool)Value;
-		break;
-	/* To remove the compiler warning, must add an empty default statement to handle the other values. */
-	default:
-		/* do nothing */
-		break;
-	}
-}
-
-void ODM_CmnInfoPtrArrayHook23a(struct dm_odm_t *pDM_Odm, enum odm_cmninfo CmnInfo,
-				u16 Index, void *pValue)
-{
-	/*  Hook call by reference pointer. */
-	switch	(CmnInfo) {
-	/*  Dynamic call by reference pointer. */
-	case	ODM_CMNINFO_STA_STATUS:
-		pDM_Odm->pODM_StaInfo[Index] = (struct sta_info *)pValue;
-		break;
-	/* To remove the compiler warning, must add an empty default statement to handle the other values. */
-	default:
-		/* do nothing */
-		break;
-	}
-}
-
-/*  Update Band/CHannel/.. The values are dynamic but non-per-packet. */
-void ODM_CmnInfoUpdate23a(struct dm_odm_t *pDM_Odm, u32 CmnInfo, u64 Value)
-{
-	/*  This init variable may be changed in run time. */
-	switch	(CmnInfo) {
-	case	ODM_CMNINFO_WIFI_DIRECT:
-		pDM_Odm->bWIFI_Direct = (bool)Value;
-		break;
-	case	ODM_CMNINFO_WIFI_DISPLAY:
-		pDM_Odm->bWIFI_Display = (bool)Value;
-		break;
-	case	ODM_CMNINFO_LINK:
-		pDM_Odm->bLinked = (bool)Value;
-		break;
-	case	ODM_CMNINFO_RSSI_MIN:
-		pDM_Odm->RSSI_Min = (u8)Value;
-		break;
-	case	ODM_CMNINFO_DBG_COMP:
-		pDM_Odm->DebugComponents = Value;
-		break;
-	case	ODM_CMNINFO_DBG_LEVEL:
-		pDM_Odm->DebugLevel = (u32)Value;
-		break;
-	case	ODM_CMNINFO_RA_THRESHOLD_HIGH:
-		pDM_Odm->RateAdaptive.HighRSSIThresh = (u8)Value;
-		break;
-	case	ODM_CMNINFO_RA_THRESHOLD_LOW:
-		pDM_Odm->RateAdaptive.LowRSSIThresh = (u8)Value;
-		break;
-	}
-
-}
-
-void odm_CommonInfoSelfInit23a(struct dm_odm_t *pDM_Odm)
-{
-	u32 val32;
-
-	val32 = rtl8723au_read32(pDM_Odm->Adapter, rFPGA0_XA_HSSIParameter2);
-	if (val32 & BIT(9))
-		pDM_Odm->bCckHighPower = true;
-	else
-		pDM_Odm->bCckHighPower = false;
-
-	pDM_Odm->RFPathRxEnable =
-		rtl8723au_read32(pDM_Odm->Adapter, rOFDM0_TRxPathEnable) & 0x0F;
-
-	ODM_InitDebugSetting23a(pDM_Odm);
-}
-
-static void odm_CommonInfoSelfUpdate(struct hal_data_8723a *pHalData)
-{
-	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
-	struct sta_info *pEntry;
-	u8 EntryCnt = 0;
-	u8 i;
-
-	for (i = 0; i < ODM_ASSOCIATE_ENTRY_NUM; i++) {
-		pEntry = pDM_Odm->pODM_StaInfo[i];
-		if (pEntry)
-			EntryCnt++;
-	}
-	if (EntryCnt == 1)
-		pDM_Odm->bOneEntryOnly = true;
-	else
-		pDM_Odm->bOneEntryOnly = false;
-}
-
-void odm_CmnInfoInit_Debug23a(struct dm_odm_t *pDM_Odm)
-{
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("odm_CmnInfoInit_Debug23a ==>\n"));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("SupportAbility = 0x%x\n", pDM_Odm->SupportAbility));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("SupportICType = 0x%x\n", pDM_Odm->SupportICType));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("CutVersion =%d\n", pDM_Odm->CutVersion));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("FabVersion =%d\n", pDM_Odm->FabVersion));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("BoardType =%d\n", pDM_Odm->BoardType));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("ExtLNA =%d\n", pDM_Odm->ExtLNA));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("ExtPA =%d\n", pDM_Odm->ExtPA));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("ExtTRSW =%d\n", pDM_Odm->ExtTRSW));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("bInHctTest =%d\n", pDM_Odm->bInHctTest));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("bWIFITest =%d\n", pDM_Odm->bWIFITest));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("bDualMacSmartConcurrent =%d\n", pDM_Odm->bDualMacSmartConcurrent));
-
-}
-
-void odm_CmnInfoUpdate_Debug23a(struct dm_odm_t *pDM_Odm)
-{
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("odm_CmnInfoUpdate_Debug23a ==>\n"));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("bWIFI_Direct =%d\n", pDM_Odm->bWIFI_Direct));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("bWIFI_Display =%d\n", pDM_Odm->bWIFI_Display));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("bLinked =%d\n", pDM_Odm->bLinked));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("RSSI_Min =%d\n", pDM_Odm->RSSI_Min));
-}
-
-void ODM_Write_DIG23a(struct dm_odm_t *pDM_Odm,	u8 CurrentIGI)
-{
-	struct rtw_adapter *adapter = pDM_Odm->Adapter;
-	struct dig_t *pDM_DigTable = &pDM_Odm->DM_DigTable;
-	u32 val32;
-
-	if (pDM_DigTable->CurIGValue != CurrentIGI) {
-		val32 = rtl8723au_read32(adapter, ODM_REG_IGI_A_11N);
-		val32 &= ~ODM_BIT_IGI_11N;
-		val32 |= CurrentIGI;
-		rtl8723au_write32(adapter, ODM_REG_IGI_A_11N, val32);
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-			     ("CurrentIGI(0x%02x). \n", CurrentIGI));
-		pDM_DigTable->CurIGValue = CurrentIGI;
-	}
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-		     ("ODM_Write_DIG23a():CurrentIGI = 0x%x \n", CurrentIGI));
-}
-
-/* Need LPS mode for CE platform --2012--08--24--- */
-/* 8723AS/8189ES */
-void odm_DIG23abyRSSI_LPS(struct dm_odm_t *pDM_Odm)
-{
-	struct rtw_adapter *pAdapter = pDM_Odm->Adapter;
-	struct false_alarm_stats *pFalseAlmCnt = &pDM_Odm->FalseAlmCnt;
-	u8 RSSI_Lower = DM_DIG_MIN_NIC;   /* 0x1E or 0x1C */
-	u8 bFwCurrentInPSMode = false;
-	u8 CurrentIGI = pDM_Odm->RSSI_Min;
-
-	if (!(pDM_Odm->SupportICType & ODM_RTL8723A))
-		return;
-
-	CurrentIGI = CurrentIGI+RSSI_OFFSET_DIG;
-	bFwCurrentInPSMode = pAdapter->pwrctrlpriv.bFwCurrentInPSMode;
-
-	/*  Using FW PS mode to make IGI */
-	if (bFwCurrentInPSMode) {
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-			     ("---Neil---odm_DIG23a is in LPS mode\n"));
-		/* Adjust by  FA in LPS MODE */
-		if (pFalseAlmCnt->Cnt_all > DM_DIG_FA_TH2_LPS)
-			CurrentIGI = CurrentIGI+2;
-		else if (pFalseAlmCnt->Cnt_all > DM_DIG_FA_TH1_LPS)
-			CurrentIGI = CurrentIGI+1;
-		else if (pFalseAlmCnt->Cnt_all < DM_DIG_FA_TH0_LPS)
-			CurrentIGI = CurrentIGI-1;
-	} else {
-		CurrentIGI = RSSI_Lower;
-	}
-
-	/* Lower bound checking */
-
-	/* RSSI Lower bound check */
-	if ((pDM_Odm->RSSI_Min-10) > DM_DIG_MIN_NIC)
-		RSSI_Lower = (pDM_Odm->RSSI_Min-10);
-	else
-		RSSI_Lower = DM_DIG_MIN_NIC;
-
-	/* Upper and Lower Bound checking */
-	 if (CurrentIGI > DM_DIG_MAX_NIC)
-		CurrentIGI = DM_DIG_MAX_NIC;
-	 else if (CurrentIGI < RSSI_Lower)
-		CurrentIGI = RSSI_Lower;
-
-	ODM_Write_DIG23a(pDM_Odm, CurrentIGI);
-}
-
-void odm_DIG23aInit(struct dm_odm_t *pDM_Odm)
-{
-	struct dig_t *pDM_DigTable = &pDM_Odm->DM_DigTable;
-	u32 val32;
-
-	val32 = rtl8723au_read32(pDM_Odm->Adapter, ODM_REG_IGI_A_11N);
-	pDM_DigTable->CurIGValue = val32 & ODM_BIT_IGI_11N;
-
-	pDM_DigTable->RssiLowThresh	= DM_DIG_THRESH_LOW;
-	pDM_DigTable->RssiHighThresh	= DM_DIG_THRESH_HIGH;
-	pDM_DigTable->FALowThresh	= DM_FALSEALARM_THRESH_LOW;
-	pDM_DigTable->FAHighThresh	= DM_FALSEALARM_THRESH_HIGH;
-	if (pDM_Odm->BoardType == ODM_BOARD_HIGHPWR) {
-		pDM_DigTable->rx_gain_range_max = DM_DIG_MAX_NIC;
-		pDM_DigTable->rx_gain_range_min = DM_DIG_MIN_NIC;
-	} else {
-		pDM_DigTable->rx_gain_range_max = DM_DIG_MAX_NIC;
-		pDM_DigTable->rx_gain_range_min = DM_DIG_MIN_NIC;
-	}
-	pDM_DigTable->BackoffVal = DM_DIG_BACKOFF_DEFAULT;
-	pDM_DigTable->BackoffVal_range_max = DM_DIG_BACKOFF_MAX;
-	pDM_DigTable->BackoffVal_range_min = DM_DIG_BACKOFF_MIN;
-	pDM_DigTable->PreCCK_CCAThres = 0xFF;
-	pDM_DigTable->CurCCK_CCAThres = 0x83;
-	pDM_DigTable->ForbiddenIGI = DM_DIG_MIN_NIC;
-	pDM_DigTable->LargeFAHit = 0;
-	pDM_DigTable->Recover_cnt = 0;
-	pDM_DigTable->DIG_Dynamic_MIN_0 = DM_DIG_MIN_NIC;
-	pDM_DigTable->DIG_Dynamic_MIN_1 = DM_DIG_MIN_NIC;
-	pDM_DigTable->bMediaConnect_0 = false;
-	pDM_DigTable->bMediaConnect_1 = false;
-}
-
-void odm_DIG23a(struct rtw_adapter *adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(adapter);
-	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
-	struct dig_t *pDM_DigTable = &pDM_Odm->DM_DigTable;
-	struct false_alarm_stats *pFalseAlmCnt = &pDM_Odm->FalseAlmCnt;
-	u8 DIG_Dynamic_MIN;
-	u8 DIG_MaxOfMin;
-	bool FirstConnect, FirstDisConnect;
-	u8 dm_dig_max, dm_dig_min;
-	u8 CurrentIGI = pDM_DigTable->CurIGValue;
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-		     ("odm_DIG23a() ==>\n"));
-	if (adapter->mlmepriv.bScanInProcess) {
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-			     ("odm_DIG23a() Return: In Scan Progress \n"));
-		return;
-	}
-
-	DIG_Dynamic_MIN = pDM_DigTable->DIG_Dynamic_MIN_0;
-	FirstConnect = (pDM_Odm->bLinked) && (!pDM_DigTable->bMediaConnect_0);
-	FirstDisConnect = (!pDM_Odm->bLinked) &&
-		(pDM_DigTable->bMediaConnect_0);
-
-	/* 1 Boundary Decision */
-	if ((pDM_Odm->SupportICType & ODM_RTL8723A) &&
-	    (pDM_Odm->BoardType == ODM_BOARD_HIGHPWR || pDM_Odm->ExtLNA)) {
-		dm_dig_max = DM_DIG_MAX_NIC_HP;
-		dm_dig_min = DM_DIG_MIN_NIC_HP;
-		DIG_MaxOfMin = DM_DIG_MAX_AP_HP;
-	} else {
-		dm_dig_max = DM_DIG_MAX_NIC;
-		dm_dig_min = DM_DIG_MIN_NIC;
-		DIG_MaxOfMin = DM_DIG_MAX_AP;
-	}
-
-	if (pDM_Odm->bLinked) {
-	      /* 2 8723A Series, offset need to be 10 */
-		if (pDM_Odm->SupportICType == ODM_RTL8723A) {
-			/* 2 Upper Bound */
-			if ((pDM_Odm->RSSI_Min + 10) > DM_DIG_MAX_NIC)
-				pDM_DigTable->rx_gain_range_max = DM_DIG_MAX_NIC;
-			else if ((pDM_Odm->RSSI_Min + 10) < DM_DIG_MIN_NIC)
-				pDM_DigTable->rx_gain_range_max = DM_DIG_MIN_NIC;
-			else
-				pDM_DigTable->rx_gain_range_max = pDM_Odm->RSSI_Min + 10;
-
-			/* 2 If BT is Concurrent, need to set Lower Bound */
-			DIG_Dynamic_MIN = DM_DIG_MIN_NIC;
-		} else {
-			/* 2 Modify DIG upper bound */
-			if ((pDM_Odm->RSSI_Min + 20) > dm_dig_max)
-				pDM_DigTable->rx_gain_range_max = dm_dig_max;
-			else if ((pDM_Odm->RSSI_Min + 20) < dm_dig_min)
-				pDM_DigTable->rx_gain_range_max = dm_dig_min;
-			else
-				pDM_DigTable->rx_gain_range_max = pDM_Odm->RSSI_Min + 20;
-
-			/* 2 Modify DIG lower bound */
-			if (pDM_Odm->bOneEntryOnly) {
-				if (pDM_Odm->RSSI_Min < dm_dig_min)
-					DIG_Dynamic_MIN = dm_dig_min;
-				else if (pDM_Odm->RSSI_Min > DIG_MaxOfMin)
-					DIG_Dynamic_MIN = DIG_MaxOfMin;
-				else
-					DIG_Dynamic_MIN = pDM_Odm->RSSI_Min;
-				ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-					     ("odm_DIG23a() : bOneEntryOnly = true,  DIG_Dynamic_MIN = 0x%x\n",
-					     DIG_Dynamic_MIN));
-				ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-					     ("odm_DIG23a() : pDM_Odm->RSSI_Min =%d\n",
-					     pDM_Odm->RSSI_Min));
-			} else {
-				DIG_Dynamic_MIN = dm_dig_min;
-			}
-		}
-	} else {
-		pDM_DigTable->rx_gain_range_max = dm_dig_max;
-		DIG_Dynamic_MIN = dm_dig_min;
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a() : No Link\n"));
-	}
-
-	/* 1 Modify DIG lower bound, deal with abnormally large false alarm */
-	if (pFalseAlmCnt->Cnt_all > 10000) {
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-			     ("dm_DIG(): Abnornally false alarm case. \n"));
-
-		if (pDM_DigTable->LargeFAHit != 3)
-			pDM_DigTable->LargeFAHit++;
-		if (pDM_DigTable->ForbiddenIGI < CurrentIGI) {
-			pDM_DigTable->ForbiddenIGI = CurrentIGI;
-			pDM_DigTable->LargeFAHit = 1;
-		}
-
-		if (pDM_DigTable->LargeFAHit >= 3) {
-			if ((pDM_DigTable->ForbiddenIGI+1) > pDM_DigTable->rx_gain_range_max)
-				pDM_DigTable->rx_gain_range_min = pDM_DigTable->rx_gain_range_max;
-			else
-				pDM_DigTable->rx_gain_range_min = (pDM_DigTable->ForbiddenIGI + 1);
-			pDM_DigTable->Recover_cnt = 3600; /* 3600 = 2hr */
-		}
-	} else {
-		/* Recovery mechanism for IGI lower bound */
-		if (pDM_DigTable->Recover_cnt != 0) {
-			pDM_DigTable->Recover_cnt--;
-		} else {
-			if (pDM_DigTable->LargeFAHit < 3) {
-				if ((pDM_DigTable->ForbiddenIGI - 1) < DIG_Dynamic_MIN) {
-					pDM_DigTable->ForbiddenIGI = DIG_Dynamic_MIN; /* DM_DIG_MIN; */
-					pDM_DigTable->rx_gain_range_min = DIG_Dynamic_MIN; /* DM_DIG_MIN; */
-					ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-						     ("odm_DIG23a(): Normal Case: At Lower Bound\n"));
-				} else {
-					pDM_DigTable->ForbiddenIGI--;
-					pDM_DigTable->rx_gain_range_min = (pDM_DigTable->ForbiddenIGI + 1);
-					ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD,
-						     ("odm_DIG23a(): Normal Case: Approach Lower Bound\n"));
-				}
-			} else {
-				pDM_DigTable->LargeFAHit = 0;
-			}
-		}
-	}
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a(): pDM_DigTable->LargeFAHit =%d\n", pDM_DigTable->LargeFAHit));
-
-	/* 1 Adjust initial gain by false alarm */
-	if (pDM_Odm->bLinked) {
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a(): DIG AfterLink\n"));
-		if (FirstConnect) {
-			CurrentIGI = pDM_Odm->RSSI_Min;
-			ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("DIG: First Connect\n"));
-		} else {
-			if (pFalseAlmCnt->Cnt_all > DM_DIG_FA_TH2)
-				CurrentIGI = CurrentIGI + 4;/* pDM_DigTable->CurIGValue = pDM_DigTable->PreIGValue+2; */
-			else if (pFalseAlmCnt->Cnt_all > DM_DIG_FA_TH1)
-				CurrentIGI = CurrentIGI + 2;/* pDM_DigTable->CurIGValue = pDM_DigTable->PreIGValue+1; */
-			else if (pFalseAlmCnt->Cnt_all < DM_DIG_FA_TH0)
-				CurrentIGI = CurrentIGI - 2;/* pDM_DigTable->CurIGValue = pDM_DigTable->PreIGValue-1; */
-		}
-	} else {
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a(): DIG BeforeLink\n"));
-		if (FirstDisConnect) {
-			CurrentIGI = pDM_DigTable->rx_gain_range_min;
-			ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a(): First DisConnect \n"));
-		} else {
-			/* 2012.03.30 LukeLee: enable DIG before link but with very high thresholds */
-			if (pFalseAlmCnt->Cnt_all > 10000)
-				CurrentIGI = CurrentIGI + 2;/* pDM_DigTable->CurIGValue = pDM_DigTable->PreIGValue+2; */
-			else if (pFalseAlmCnt->Cnt_all > 8000)
-				CurrentIGI = CurrentIGI + 1;/* pDM_DigTable->CurIGValue = pDM_DigTable->PreIGValue+1; */
-			else if (pFalseAlmCnt->Cnt_all < 500)
-				CurrentIGI = CurrentIGI - 1;/* pDM_DigTable->CurIGValue = pDM_DigTable->PreIGValue-1; */
-			ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a(): England DIG \n"));
-		}
-	}
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a(): DIG End Adjust IGI\n"));
-	/* 1 Check initial gain by upper/lower bound */
-	if (CurrentIGI > pDM_DigTable->rx_gain_range_max)
-		CurrentIGI = pDM_DigTable->rx_gain_range_max;
-	if (CurrentIGI < pDM_DigTable->rx_gain_range_min)
-		CurrentIGI = pDM_DigTable->rx_gain_range_min;
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a(): rx_gain_range_max = 0x%x, rx_gain_range_min = 0x%x\n",
-		pDM_DigTable->rx_gain_range_max, pDM_DigTable->rx_gain_range_min));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a(): TotalFA =%d\n", pFalseAlmCnt->Cnt_all));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG23a(): CurIGValue = 0x%x\n", CurrentIGI));
-
-	/* 2 High power RSSI threshold */
-
-	ODM_Write_DIG23a(pDM_Odm, CurrentIGI);/* ODM_Write_DIG23a(pDM_Odm, pDM_DigTable->CurIGValue); */
-	pDM_DigTable->bMediaConnect_0 = pDM_Odm->bLinked;
-	pDM_DigTable->DIG_Dynamic_MIN_0 = DIG_Dynamic_MIN;
-}
-
-/* 3 ============================================================ */
-/* 3 FASLE ALARM CHECK */
-/* 3 ============================================================ */
-
-void odm_FalseAlarmCounterStatistics23a(struct dm_odm_t *pDM_Odm)
-{
-	struct rtw_adapter *adapter = pDM_Odm->Adapter;
-	struct false_alarm_stats *FalseAlmCnt = &pDM_Odm->FalseAlmCnt;
-	u32 ret_value, val32;
-
-	/* hold ofdm counter */
-	/* hold page C counter */
-	val32 = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_HOLDC_11N);
-	val32 |= BIT(31);
-	rtl8723au_write32(adapter, ODM_REG_OFDM_FA_HOLDC_11N, val32);
-	/* hold page D counter */
-	val32 = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_RSTD_11N);
-	val32 |= BIT(31);
-	rtl8723au_write32(adapter, ODM_REG_OFDM_FA_RSTD_11N, val32);
-	ret_value = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_TYPE1_11N);
-	FalseAlmCnt->Cnt_Fast_Fsync = (ret_value&0xffff);
-	FalseAlmCnt->Cnt_SB_Search_fail = (ret_value & 0xffff0000)>>16;
-	ret_value = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_TYPE2_11N);
-	FalseAlmCnt->Cnt_OFDM_CCA = (ret_value&0xffff);
-	FalseAlmCnt->Cnt_Parity_Fail = (ret_value & 0xffff0000)>>16;
-	ret_value = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_TYPE3_11N);
-	FalseAlmCnt->Cnt_Rate_Illegal = (ret_value&0xffff);
-	FalseAlmCnt->Cnt_Crc8_fail = (ret_value & 0xffff0000)>>16;
-	ret_value = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_TYPE4_11N);
-	FalseAlmCnt->Cnt_Mcs_fail = (ret_value&0xffff);
-
-	FalseAlmCnt->Cnt_Ofdm_fail = FalseAlmCnt->Cnt_Parity_Fail +
-		FalseAlmCnt->Cnt_Rate_Illegal +
-		FalseAlmCnt->Cnt_Crc8_fail +
-		FalseAlmCnt->Cnt_Mcs_fail +
-		FalseAlmCnt->Cnt_Fast_Fsync +
-		FalseAlmCnt->Cnt_SB_Search_fail;
-	/* hold cck counter */
-	val32 = rtl8723au_read32(adapter, ODM_REG_CCK_FA_RST_11N);
-	val32 |= (BIT(12) | BIT(14));
-	rtl8723au_write32(adapter, ODM_REG_CCK_FA_RST_11N, val32);
-
-	ret_value = rtl8723au_read32(adapter, ODM_REG_CCK_FA_LSB_11N) & 0xff;
-	FalseAlmCnt->Cnt_Cck_fail = ret_value;
-	ret_value = rtl8723au_read32(adapter, ODM_REG_CCK_FA_MSB_11N) >> 16;
-	FalseAlmCnt->Cnt_Cck_fail += (ret_value & 0xff00);
-
-	ret_value = rtl8723au_read32(adapter, ODM_REG_CCK_CCA_CNT_11N);
-	FalseAlmCnt->Cnt_CCK_CCA =
-		((ret_value&0xFF)<<8) | ((ret_value&0xFF00)>>8);
-
-	FalseAlmCnt->Cnt_all = (FalseAlmCnt->Cnt_Fast_Fsync +
-				FalseAlmCnt->Cnt_SB_Search_fail +
-				FalseAlmCnt->Cnt_Parity_Fail +
-				FalseAlmCnt->Cnt_Rate_Illegal +
-				FalseAlmCnt->Cnt_Crc8_fail +
-				FalseAlmCnt->Cnt_Mcs_fail +
-				FalseAlmCnt->Cnt_Cck_fail);
-
-	FalseAlmCnt->Cnt_CCA_all =
-		FalseAlmCnt->Cnt_OFDM_CCA + FalseAlmCnt->Cnt_CCK_CCA;
-
-	if (pDM_Odm->SupportICType >= ODM_RTL8723A) {
-		/* reset false alarm counter registers */
-		val32 = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_RSTC_11N);
-		val32 |= BIT(31);
-		rtl8723au_write32(adapter, ODM_REG_OFDM_FA_RSTC_11N, val32);
-		val32 = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_RSTC_11N);
-		val32 &= ~BIT(31);
-		rtl8723au_write32(adapter, ODM_REG_OFDM_FA_RSTC_11N, val32);
-
-		val32 = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_RSTD_11N);
-		val32 |= BIT(27);
-		rtl8723au_write32(adapter, ODM_REG_OFDM_FA_RSTD_11N, val32);
-		val32 = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_RSTD_11N);
-		val32 &= ~BIT(27);
-		rtl8723au_write32(adapter, ODM_REG_OFDM_FA_RSTD_11N, val32);
-
-		/* update ofdm counter */
-		 /* update page C counter */
-		val32 = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_HOLDC_11N);
-		val32 &= ~BIT(31);
-		rtl8723au_write32(adapter, ODM_REG_OFDM_FA_HOLDC_11N, val32);
-
-		 /* update page D counter */
-		val32 = rtl8723au_read32(adapter, ODM_REG_OFDM_FA_RSTD_11N);
-		val32 &= ~BIT(31);
-		rtl8723au_write32(adapter, ODM_REG_OFDM_FA_RSTD_11N, val32);
-
-		/* reset CCK CCA counter */
-		val32 = rtl8723au_read32(adapter, ODM_REG_CCK_FA_RST_11N);
-		val32 &= ~(BIT(12) | BIT(13) | BIT(14) | BIT(15));
-		rtl8723au_write32(adapter, ODM_REG_CCK_FA_RST_11N, val32);
-
-		val32 = rtl8723au_read32(adapter, ODM_REG_CCK_FA_RST_11N);
-		val32 |= (BIT(13) | BIT(15));
-		rtl8723au_write32(adapter, ODM_REG_CCK_FA_RST_11N, val32);
-	}
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_FA_CNT, ODM_DBG_LOUD,
-		     ("Enter odm_FalseAlarmCounterStatistics23a\n"));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_FA_CNT, ODM_DBG_LOUD,
-		     ("Cnt_Fast_Fsync =%d, Cnt_SB_Search_fail =%d\n",
-		      FalseAlmCnt->Cnt_Fast_Fsync,
-		      FalseAlmCnt->Cnt_SB_Search_fail));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_FA_CNT, ODM_DBG_LOUD,
-		     ("Cnt_Parity_Fail =%d, Cnt_Rate_Illegal =%d\n",
-		      FalseAlmCnt->Cnt_Parity_Fail,
-		      FalseAlmCnt->Cnt_Rate_Illegal));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_FA_CNT, ODM_DBG_LOUD,
-		     ("Cnt_Crc8_fail =%d, Cnt_Mcs_fail =%d\n",
-		      FalseAlmCnt->Cnt_Crc8_fail, FalseAlmCnt->Cnt_Mcs_fail));
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_FA_CNT, ODM_DBG_LOUD,
-		     ("Cnt_Cck_fail =%d\n", FalseAlmCnt->Cnt_Cck_fail));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_FA_CNT, ODM_DBG_LOUD,
-		     ("Cnt_Ofdm_fail =%d\n", FalseAlmCnt->Cnt_Ofdm_fail));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_FA_CNT, ODM_DBG_LOUD,
-		     ("Total False Alarm =%d\n", FalseAlmCnt->Cnt_all));
-}
-
-/* 3 ============================================================ */
-/* 3 CCK Packet Detect Threshold */
-/* 3 ============================================================ */
-
-void odm_CCKPacketDetectionThresh23a(struct dm_odm_t *pDM_Odm)
-{
-	struct false_alarm_stats *FalseAlmCnt = &pDM_Odm->FalseAlmCnt;
-	u8 CurCCK_CCAThres;
-
-	if (pDM_Odm->ExtLNA)
-		return;
-
-	if (pDM_Odm->bLinked) {
-		if (pDM_Odm->RSSI_Min > 25) {
-			CurCCK_CCAThres = 0xcd;
-		} else if (pDM_Odm->RSSI_Min <= 25 && pDM_Odm->RSSI_Min > 10) {
-			CurCCK_CCAThres = 0x83;
-		} else {
-			if (FalseAlmCnt->Cnt_Cck_fail > 1000)
-				CurCCK_CCAThres = 0x83;
-			else
-				CurCCK_CCAThres = 0x40;
-		}
-	} else {
-		if (FalseAlmCnt->Cnt_Cck_fail > 1000)
-			CurCCK_CCAThres = 0x83;
-		else
-			CurCCK_CCAThres = 0x40;
-	}
-
-	ODM_Write_CCK_CCA_Thres23a(pDM_Odm, CurCCK_CCAThres);
-}
-
-void ODM_Write_CCK_CCA_Thres23a(struct dm_odm_t *pDM_Odm, u8 CurCCK_CCAThres)
-{
-	struct dig_t *pDM_DigTable = &pDM_Odm->DM_DigTable;
-
-	if (pDM_DigTable->CurCCK_CCAThres != CurCCK_CCAThres)
-		rtl8723au_write8(pDM_Odm->Adapter, ODM_REG(CCK_CCA, pDM_Odm),
-				 CurCCK_CCAThres);
-	pDM_DigTable->PreCCK_CCAThres = pDM_DigTable->CurCCK_CCAThres;
-	pDM_DigTable->CurCCK_CCAThres = CurCCK_CCAThres;
-}
-
-/* 3 ============================================================ */
-/* 3 BB Power Save */
-/* 3 ============================================================ */
-void odm23a_DynBBPSInit(struct dm_odm_t *pDM_Odm)
-{
-	struct dynamic_pwr_sav *pDM_PSTable = &pDM_Odm->DM_PSTable;
-
-	pDM_PSTable->PreCCAState = CCA_MAX;
-	pDM_PSTable->CurCCAState = CCA_MAX;
-	pDM_PSTable->PreRFState = RF_MAX;
-	pDM_PSTable->CurRFState = RF_MAX;
-	pDM_PSTable->Rssi_val_min = 0;
-	pDM_PSTable->initialize = 0;
-}
-
-
-void ODM_RF_Saving23a(struct dm_odm_t *pDM_Odm, u8 bForceInNormal)
-{
-	struct dynamic_pwr_sav *pDM_PSTable = &pDM_Odm->DM_PSTable;
-	struct rtw_adapter *adapter = pDM_Odm->Adapter;
-	u32 val32;
-	u8 Rssi_Up_bound = 30;
-	u8 Rssi_Low_bound = 25;
-	if (pDM_PSTable->initialize == 0) {
-
-		pDM_PSTable->Reg874 =
-			rtl8723au_read32(adapter, 0x874) & 0x1CC000;
-		pDM_PSTable->RegC70 =
-			rtl8723au_read32(adapter, 0xc70) & BIT(3);
-		pDM_PSTable->Reg85C =
-			rtl8723au_read32(adapter, 0x85c) & 0xFF000000;
-		pDM_PSTable->RegA74 = rtl8723au_read32(adapter, 0xa74) & 0xF000;
-		pDM_PSTable->initialize = 1;
-	}
-
-	if (!bForceInNormal) {
-		if (pDM_Odm->RSSI_Min != 0xFF) {
-			if (pDM_PSTable->PreRFState == RF_Normal) {
-				if (pDM_Odm->RSSI_Min >= Rssi_Up_bound)
-					pDM_PSTable->CurRFState = RF_Save;
-				else
-					pDM_PSTable->CurRFState = RF_Normal;
-			} else {
-				if (pDM_Odm->RSSI_Min <= Rssi_Low_bound)
-					pDM_PSTable->CurRFState = RF_Normal;
-				else
-					pDM_PSTable->CurRFState = RF_Save;
-			}
-		} else {
-			pDM_PSTable->CurRFState = RF_MAX;
-		}
-	} else {
-		pDM_PSTable->CurRFState = RF_Normal;
-	}
-
-	if (pDM_PSTable->PreRFState != pDM_PSTable->CurRFState) {
-		if (pDM_PSTable->CurRFState == RF_Save) {
-			/*  <tynli_note> 8723 RSSI report will be wrong.
-			 * Set 0x874[5]= 1 when enter BB power saving mode. */
-			/*  Suggested by SD3 Yu-Nan. 2011.01.20. */
-			/* Reg874[5]= 1b'1 */
-			if (pDM_Odm->SupportICType == ODM_RTL8723A) {
-				val32 = rtl8723au_read32(adapter, 0x874);
-				val32 |= BIT(5);
-				rtl8723au_write32(adapter, 0x874, val32);
-			}
-			/* Reg874[20:18]= 3'b010 */
-			val32 = rtl8723au_read32(adapter, 0x874);
-			val32 &= ~(BIT(18) | BIT(20));
-			val32 |= BIT(19);
-			rtl8723au_write32(adapter, 0x874, val32);
-			/* RegC70[3]= 1'b0 */
-			val32 = rtl8723au_read32(adapter, 0xc70);
-			val32 &= ~BIT(3);
-			rtl8723au_write32(adapter, 0xc70, val32);
-			/* Reg85C[31:24]= 0x63 */
-			val32 = rtl8723au_read32(adapter, 0x85c);
-			val32 &= 0x00ffffff;
-			val32 |= 0x63000000;
-			rtl8723au_write32(adapter, 0x85c, val32);
-			/* Reg874[15:14]= 2'b10 */
-			val32 = rtl8723au_read32(adapter, 0x874);
-			val32 &= ~BIT(14);
-			val32 |= BIT(15);
-			rtl8723au_write32(adapter, 0x874, val32);
-			/* RegA75[7:4]= 0x3 */
-			val32 = rtl8723au_read32(adapter, 0xa74);
-			val32 &= ~(BIT(14) | BIT(15));
-			val32 |= (BIT(12) | BIT(13));
-			rtl8723au_write32(adapter, 0xa74, val32);
-			/* Reg818[28]= 1'b0 */
-			val32 = rtl8723au_read32(adapter, 0x818);
-			val32 &= ~BIT(28);
-			rtl8723au_write32(adapter, 0x818, val32);
-			/* Reg818[28]= 1'b1 */
-			val32 = rtl8723au_read32(adapter, 0x818);
-			val32 |= BIT(28);
-			rtl8723au_write32(adapter, 0x818, val32);
-		} else {
-			val32 = rtl8723au_read32(adapter, 0x874);
-			val32 |= pDM_PSTable->Reg874;
-			rtl8723au_write32(adapter, 0x874, val32);
-
-			val32 = rtl8723au_read32(adapter, 0xc70);
-			val32 |= pDM_PSTable->RegC70;
-			rtl8723au_write32(adapter, 0xc70, val32);
-
-			val32 = rtl8723au_read32(adapter, 0x85c);
-			val32 |= pDM_PSTable->Reg85C;
-			rtl8723au_write32(adapter, 0x85c, val32);
-
-			val32 = rtl8723au_read32(adapter, 0xa74);
-			val32 |= pDM_PSTable->RegA74;
-			rtl8723au_write32(adapter, 0xa74, val32);
-
-			val32 = rtl8723au_read32(adapter, 0x818);
-			val32 &= ~BIT(28);
-			rtl8723au_write32(adapter, 0x818, val32);
-
-			/* Reg874[5]= 1b'0 */
-			if (pDM_Odm->SupportICType == ODM_RTL8723A) {
-				val32 = rtl8723au_read32(adapter, 0x874);
-				val32 &= ~BIT(5);
-				rtl8723au_write32(adapter, 0x874, val32);
-			}
-		}
-		pDM_PSTable->PreRFState = pDM_PSTable->CurRFState;
-	}
-}
-
-/* 3 ============================================================ */
-/* 3 RATR MASK */
-/* 3 ============================================================ */
-/* 3 ============================================================ */
-/* 3 Rate Adaptive */
-/* 3 ============================================================ */
-
-void odm_RateAdaptiveMaskInit23a(struct dm_odm_t *pDM_Odm)
-{
-	struct odm_rate_adapt *pOdmRA = &pDM_Odm->RateAdaptive;
-
-	pOdmRA->Type = DM_Type_ByDriver;
-
-	pOdmRA->RATRState = DM_RATR_STA_INIT;
-	pOdmRA->HighRSSIThresh = 50;
-	pOdmRA->LowRSSIThresh = 20;
-}
-
-u32 ODM_Get_Rate_Bitmap23a(struct hal_data_8723a *pHalData, u32 macid,
-			   u32 ra_mask, u8 rssi_level)
-{
-	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
-	struct sta_info *pEntry;
-	u32 rate_bitmap = 0x0fffffff;
-	u8 WirelessMode;
-
-	pEntry = pDM_Odm->pODM_StaInfo[macid];
-	if (!pEntry)
-		return ra_mask;
-
-	WirelessMode = pEntry->wireless_mode;
-
-	switch (WirelessMode) {
-	case ODM_WM_B:
-		if (ra_mask & 0x0000000c)		/* 11M or 5.5M enable */
-			rate_bitmap = 0x0000000d;
-		else
-			rate_bitmap = 0x0000000f;
-		break;
-	case (ODM_WM_A|ODM_WM_G):
-		if (rssi_level == DM_RATR_STA_HIGH)
-			rate_bitmap = 0x00000f00;
-		else
-			rate_bitmap = 0x00000ff0;
-		break;
-	case (ODM_WM_B|ODM_WM_G):
-		if (rssi_level == DM_RATR_STA_HIGH)
-			rate_bitmap = 0x00000f00;
-		else if (rssi_level == DM_RATR_STA_MIDDLE)
-			rate_bitmap = 0x00000ff0;
-		else
-			rate_bitmap = 0x00000ff5;
-		break;
-	case (ODM_WM_B|ODM_WM_G|ODM_WM_N24G):
-	case (ODM_WM_A|ODM_WM_B|ODM_WM_G|ODM_WM_N24G):
-		if (pHalData->rf_type == RF_1T2R ||
-		    pHalData->rf_type == RF_1T1R) {
-			if (rssi_level == DM_RATR_STA_HIGH) {
-				rate_bitmap = 0x000f0000;
-			} else if (rssi_level == DM_RATR_STA_MIDDLE) {
-				rate_bitmap = 0x000ff000;
-			} else {
-				if (pHalData->CurrentChannelBW ==
-				    HT_CHANNEL_WIDTH_40)
-					rate_bitmap = 0x000ff015;
-				else
-					rate_bitmap = 0x000ff005;
-			}
-		} else {
-			if (rssi_level == DM_RATR_STA_HIGH) {
-				rate_bitmap = 0x0f8f0000;
-			} else if (rssi_level == DM_RATR_STA_MIDDLE) {
-				rate_bitmap = 0x0f8ff000;
-			} else {
-				if (pHalData->CurrentChannelBW ==
-				    HT_CHANNEL_WIDTH_40)
-					rate_bitmap = 0x0f8ff015;
-				else
-					rate_bitmap = 0x0f8ff005;
-			}
-		}
-		break;
-	default:
-		/* case WIRELESS_11_24N: */
-		/* case WIRELESS_11_5N: */
-		if (pHalData->rf_type == RF_1T2R)
-			rate_bitmap = 0x000fffff;
-		else
-			rate_bitmap = 0x0fffffff;
-		break;
-	}
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_RA_MASK, ODM_DBG_LOUD,
-	(" ==> rssi_level:0x%02x, WirelessMode:0x%02x, rate_bitmap:0x%08x \n",
-	 rssi_level, WirelessMode, rate_bitmap));
-
-	return rate_bitmap;
-}
-
-/*-----------------------------------------------------------------------------
- * Function:	odm_RefreshRateAdaptiveMask()
- *
- * Overview:	Update rate table mask according to rssi
- *
- * Input:		NONE
- *
- * Output:		NONE
- *
- * Return:		NONE
- *
- * Revised History:
- *When		Who		Remark
- *05/27/2009	hpfan	Create Version 0.
- *
- *---------------------------------------------------------------------------*/
-static void odm_RefreshRateAdaptiveMask(struct dm_odm_t *pDM_Odm)
-{
-	struct rtw_adapter *pAdapter = pDM_Odm->Adapter;
-	u32 smoothed;
-	u8 i;
-
-	if (pAdapter->bDriverStopped) {
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_RA_MASK, ODM_DBG_TRACE,
-			     ("<---- %s: driver is going to unload\n",
-			      __func__));
-		return;
-	}
-
-	for (i = 0; i < ODM_ASSOCIATE_ENTRY_NUM; i++) {
-		struct sta_info *pstat = pDM_Odm->pODM_StaInfo[i];
-		if (pstat) {
-			smoothed = pstat->rssi_stat.UndecoratedSmoothedPWDB;
-			if (ODM_RAStateCheck23a(pDM_Odm, smoothed, false,
-						&pstat->rssi_level)) {
-				ODM_RT_TRACE(pDM_Odm, ODM_COMP_RA_MASK,
-					     ODM_DBG_LOUD,
-					     ("RSSI:%d, RSSI_LEVEL:%d\n",
-					      smoothed,
-					      pstat->rssi_level));
-				rtw_hal_update_ra_mask23a(pstat,
-							  pstat->rssi_level);
-			}
-		}
-	}
-}
-
-/*  Return Value: bool */
-/*  - true: RATRState is changed. */
-bool ODM_RAStateCheck23a(struct dm_odm_t *pDM_Odm, s32 RSSI, bool bForceUpdate,
-			 u8 *pRATRState)
-{
-	struct odm_rate_adapt *pRA = &pDM_Odm->RateAdaptive;
-	const u8 GoUpGap = 5;
-	u8 HighRSSIThreshForRA = pRA->HighRSSIThresh;
-	u8 LowRSSIThreshForRA = pRA->LowRSSIThresh;
-	u8 RATRState;
-
-	/*  Threshold Adjustment: */
-	/*  when RSSI state trends to go up one or two levels, make sure RSSI is high enough. */
-	/*  Here GoUpGap is added to solve the boundary's level alternation issue. */
-	switch (*pRATRState) {
-	case DM_RATR_STA_INIT:
-	case DM_RATR_STA_HIGH:
-		break;
-	case DM_RATR_STA_MIDDLE:
-		HighRSSIThreshForRA += GoUpGap;
-		break;
-	case DM_RATR_STA_LOW:
-		HighRSSIThreshForRA += GoUpGap;
-		LowRSSIThreshForRA += GoUpGap;
-		break;
-	default:
-		ODM_RT_ASSERT(pDM_Odm, false, ("wrong rssi level setting %d !",
-					       *pRATRState));
-		break;
-	}
-
-	/*  Decide RATRState by RSSI. */
-	if (RSSI > HighRSSIThreshForRA)
-		RATRState = DM_RATR_STA_HIGH;
-	else if (RSSI > LowRSSIThreshForRA)
-		RATRState = DM_RATR_STA_MIDDLE;
-	else
-		RATRState = DM_RATR_STA_LOW;
-
-	if (*pRATRState != RATRState || bForceUpdate) {
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_RA_MASK, ODM_DBG_LOUD,
-			     ("RSSI Level %d -> %d\n", *pRATRState, RATRState));
-		*pRATRState = RATRState;
-		return true;
-	}
-	return false;
-}
-
-/* 3 ============================================================ */
-/* 3 Dynamic Tx Power */
-/* 3 ============================================================ */
-
-void odm_DynamicTxPower23aInit(struct dm_odm_t *pDM_Odm)
-{
-	struct rtw_adapter *Adapter = pDM_Odm->Adapter;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-
-	/*
-	 * This is never changed, so we should be able to clean up the
-	 * code checking for different values in rtl8723a_rf6052.c
-	 */
-	pdmpriv->DynamicTxHighPowerLvl = TxHighPwrLevel_Normal;
-}
-
-static void
-FindMinimumRSSI(struct rtw_adapter *pAdapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
-
-	/* 1 1.Determine the minimum RSSI */
-
-	if (!pDM_Odm->bLinked && !pdmpriv->EntryMinUndecoratedSmoothedPWDB)
-		pdmpriv->MinUndecoratedPWDBForDM = 0;
-	else
-		pdmpriv->MinUndecoratedPWDBForDM =
-			pdmpriv->EntryMinUndecoratedSmoothedPWDB;
-}
-
-static void odm_RSSIMonitorCheck(struct dm_odm_t *pDM_Odm)
-{
-	struct rtw_adapter *Adapter = pDM_Odm->Adapter;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-	int i;
-	int MaxDB = 0, MinDB = 0xff;
-	u8 sta_cnt = 0;
-	u32 tmpdb;
-	u32 PWDB_rssi[NUM_STA] = {0};/* 0~15]:MACID, [16~31]:PWDB_rssi */
-	struct sta_info *psta;
-
-	if (!pDM_Odm->bLinked)
-		return;
-
-	for (i = 0; i < ODM_ASSOCIATE_ENTRY_NUM; i++) {
-		psta = pDM_Odm->pODM_StaInfo[i];
-		if (psta) {
-			if (psta->rssi_stat.UndecoratedSmoothedPWDB < MinDB)
-				MinDB = psta->rssi_stat.UndecoratedSmoothedPWDB;
-
-			if (psta->rssi_stat.UndecoratedSmoothedPWDB > MaxDB)
-				MaxDB = psta->rssi_stat.UndecoratedSmoothedPWDB;
-
-			if (psta->rssi_stat.UndecoratedSmoothedPWDB != -1) {
-				tmpdb = psta->rssi_stat.UndecoratedSmoothedPWDB;
-				PWDB_rssi[sta_cnt++] = psta->mac_id |
-					(tmpdb << 16);
-			}
-		}
-	}
-
-	for (i = 0; i < sta_cnt; i++) {
-		if (PWDB_rssi[i] != (0))
-			rtl8723a_set_rssi_cmd(Adapter, PWDB_rssi[i]);
-	}
-
-	pdmpriv->EntryMaxUndecoratedSmoothedPWDB = MaxDB;
-
-	if (MinDB != 0xff) /*  If associated entry is found */
-		pdmpriv->EntryMinUndecoratedSmoothedPWDB = MinDB;
-	else
-		pdmpriv->EntryMinUndecoratedSmoothedPWDB = 0;
-
-	FindMinimumRSSI(Adapter);/* get pdmpriv->MinUndecoratedPWDBForDM */
-
-	ODM_CmnInfoUpdate23a(&pHalData->odmpriv, ODM_CMNINFO_RSSI_MIN,
-			     pdmpriv->MinUndecoratedPWDBForDM);
-}
-
-/* endif */
-/* 3 ============================================================ */
-/* 3 Tx Power Tracking */
-/* 3 ============================================================ */
-
-static void odm_TXPowerTrackingInit(struct dm_odm_t *pDM_Odm)
-{
-	struct rtw_adapter *Adapter = pDM_Odm->Adapter;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-
-	pdmpriv->bTXPowerTracking = true;
-	pdmpriv->TXPowercount = 0;
-	pdmpriv->bTXPowerTrackingInit = false;
-	pdmpriv->TxPowerTrackControl = true;
-	MSG_8723A("pdmpriv->TxPowerTrackControl = %d\n",
-		  pdmpriv->TxPowerTrackControl);
-
-	pDM_Odm->RFCalibrateInfo.TxPowerTrackControl = true;
-}
-
-/* EDCA Turbo */
-static void ODM_EdcaTurboInit23a(struct dm_odm_t *pDM_Odm)
-{
-	struct rtw_adapter *Adapter = pDM_Odm->Adapter;
-
-	pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA = false;
-	Adapter->recvpriv.bIsAnyNonBEPkts = false;
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_EDCA_TURBO, ODM_DBG_LOUD,
-		     ("Orginial VO PARAM: 0x%x\n",
-		      rtl8723au_read32(Adapter, ODM_EDCA_VO_PARAM)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_EDCA_TURBO, ODM_DBG_LOUD,
-		     ("Orginial VI PARAM: 0x%x\n",
-		      rtl8723au_read32(Adapter, ODM_EDCA_VI_PARAM)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_EDCA_TURBO, ODM_DBG_LOUD,
-		     ("Orginial BE PARAM: 0x%x\n",
-		      rtl8723au_read32(Adapter, ODM_EDCA_BE_PARAM)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_EDCA_TURBO, ODM_DBG_LOUD,
-		     ("Orginial BK PARAM: 0x%x\n",
-		      rtl8723au_read32(Adapter, ODM_EDCA_BK_PARAM)));
-}
-
-static void odm_EdcaTurboCheck23a(struct dm_odm_t *pDM_Odm)
-{
-	struct rtw_adapter *Adapter = pDM_Odm->Adapter;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct xmit_priv *pxmitpriv = &Adapter->xmitpriv;
-	struct recv_priv *precvpriv = &Adapter->recvpriv;
-	struct registry_priv *pregpriv = &Adapter->registrypriv;
-	struct mlme_ext_priv *pmlmeext = &Adapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	u32 trafficIndex;
-	u32 edca_param;
-	u64 cur_tx_bytes;
-	u64 cur_rx_bytes;
-
-	/*  For AP/ADSL use struct rtl8723a_priv * */
-	/*  For CE/NIC use struct rtw_adapter * */
-
-	/*
-	 * 2011/09/29 MH In HW integration first stage, we provide 4
-	 * different handle to operate at the same time. In the stage2/3,
-	 * we need to prive universal interface and merge all HW dynamic
-	 * mechanism.
-	 */
-
-	if ((pregpriv->wifi_spec == 1))/*  (pmlmeinfo->HT_enable == 0)) */
-		goto dm_CheckEdcaTurbo_EXIT;
-
-	if (pmlmeinfo->assoc_AP_vendor >=  HT_IOT_PEER_MAX)
-		goto dm_CheckEdcaTurbo_EXIT;
-
-	if (rtl8723a_BT_disable_EDCA_turbo(Adapter))
-		goto dm_CheckEdcaTurbo_EXIT;
-
-	/*  Check if the status needs to be changed. */
-	if (!precvpriv->bIsAnyNonBEPkts) {
-		cur_tx_bytes = pxmitpriv->tx_bytes - pxmitpriv->last_tx_bytes;
-		cur_rx_bytes = precvpriv->rx_bytes - precvpriv->last_rx_bytes;
-
-		/* traffic, TX or RX */
-		if ((pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_RALINK) ||
-		    (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_ATHEROS)) {
-			if (cur_tx_bytes > (cur_rx_bytes << 2)) {
-				/*  Uplink TP is present. */
-				trafficIndex = UP_LINK;
-			} else { /*  Balance TP is present. */
-				trafficIndex = DOWN_LINK;
-			}
-		} else {
-			if (cur_rx_bytes > (cur_tx_bytes << 2)) {
-				/*  Downlink TP is present. */
-				trafficIndex = DOWN_LINK;
-			} else { /*  Balance TP is present. */
-				trafficIndex = UP_LINK;
-			}
-		}
-
-		if ((pDM_Odm->DM_EDCA_Table.prv_traffic_idx != trafficIndex) ||
-		    (!pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA)) {
-			if ((pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_CISCO) &&
-			    (pmlmeext->cur_wireless_mode & WIRELESS_11_24N))
-				edca_param = EDCAParam[pmlmeinfo->assoc_AP_vendor][trafficIndex];
-			else
-				edca_param = EDCAParam[HT_IOT_PEER_UNKNOWN][trafficIndex];
-			rtl8723au_write32(Adapter, REG_EDCA_BE_PARAM,
-					  edca_param);
-
-			pDM_Odm->DM_EDCA_Table.prv_traffic_idx = trafficIndex;
-		}
-
-		pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA = true;
-	} else {
-		/*  Turn Off EDCA turbo here. */
-		/*  Restore original EDCA according to the declaration of AP. */
-		if (pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA) {
-			rtl8723au_write32(Adapter, REG_EDCA_BE_PARAM,
-					  pHalData->AcParam_BE);
-			pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA = false;
-		}
-	}
-
-dm_CheckEdcaTurbo_EXIT:
-	/*  Set variables for next time. */
-	precvpriv->bIsAnyNonBEPkts = false;
-	pxmitpriv->last_tx_bytes = pxmitpriv->tx_bytes;
-	precvpriv->last_rx_bytes = precvpriv->rx_bytes;
-}
-
-u32 GetPSDData(struct dm_odm_t *pDM_Odm, unsigned int point,
-	       u8 initial_gain_psd)
-{
-	struct rtw_adapter *adapter = pDM_Odm->Adapter;
-	u32 psd_report, val32;
-
-	/* Set DCO frequency index, offset = (40MHz/SamplePts)*point */
-	val32 = rtl8723au_read32(adapter, 0x808);
-	val32 &= ~0x3ff;
-	val32 |= (point & 0x3ff);
-	rtl8723au_write32(adapter, 0x808, val32);
-
-	/* Start PSD calculation, Reg808[22]= 0->1 */
-	val32 = rtl8723au_read32(adapter, 0x808);
-	val32 |= BIT(22);
-	rtl8723au_write32(adapter, 0x808, val32);
-	/* Need to wait for HW PSD report */
-	udelay(30);
-	val32 = rtl8723au_read32(adapter, 0x808);
-	val32 &= ~BIT(22);
-	rtl8723au_write32(adapter, 0x808, val32);
-	/* Read PSD report, Reg8B4[15:0] */
-	psd_report = rtl8723au_read32(adapter, 0x8B4) & 0x0000FFFF;
-
-	psd_report = (u32)(ConvertTo_dB23a(psd_report)) +
-		(u32)(initial_gain_psd-0x1c);
-
-	return psd_report;
-}
-
-u32 ConvertTo_dB23a(u32 Value)
-{
-	u8 i;
-	u8 j;
-	u32 dB;
-
-	Value = Value & 0xFFFF;
-
-	for (i = 0; i < 8; i++) {
-		if (Value <= dB_Invert_Table[i][11])
-			break;
-	}
-
-	if (i >= 8)
-		return 96;	/*  maximum 96 dB */
-
-	for (j = 0; j < 12; j++) {
-		if (Value <= dB_Invert_Table[i][j])
-			break;
-	}
-
-	dB = i*12 + j + 1;
-
-	return dB;
-}
-
-/*  */
-/*  Description: */
-/* Set Single/Dual Antenna default setting for products that do not
- * do detection in advance. */
-/*  */
-/*  Added by Joseph, 2012.03.22 */
-/*  */
-void ODM_SingleDualAntennaDefaultSetting(struct dm_odm_t *pDM_Odm)
-{
-	struct sw_ant_sw *pDM_SWAT_Table = &pDM_Odm->DM_SWAT_Table;
-
-	pDM_SWAT_Table->ANTA_ON = true;
-	pDM_SWAT_Table->ANTB_ON = true;
-}
-
-/* 2 8723A ANT DETECT */
-
-static void odm_PHY_SaveAFERegisters(struct dm_odm_t *pDM_Odm, u32 *AFEReg,
-				     u32 *AFEBackup, u32 RegisterNum)
-{
-	u32 i;
-
-	for (i = 0 ; i < RegisterNum ; i++)
-		AFEBackup[i] = rtl8723au_read32(pDM_Odm->Adapter, AFEReg[i]);
-}
-
-static void odm_PHY_ReloadAFERegisters(struct dm_odm_t *pDM_Odm, u32 *AFEReg,
-				       u32 *AFEBackup, u32 RegiesterNum)
-{
-	u32 i;
-
-	for (i = 0 ; i < RegiesterNum; i++)
-		rtl8723au_write32(pDM_Odm->Adapter, AFEReg[i], AFEBackup[i]);
-}
-
-/* 2 8723A ANT DETECT */
-/*  Description: */
-/* Implement IQK single tone for RF DPK loopback and BB PSD scanning. */
-/* This function is cooperated with BB team Neil. */
-bool ODM_SingleDualAntennaDetection(struct dm_odm_t *pDM_Odm, u8 mode)
-{
-	struct sw_ant_sw *pDM_SWAT_Table = &pDM_Odm->DM_SWAT_Table;
-	struct rtw_adapter *adapter = pDM_Odm->Adapter;
-	u32 CurrentChannel, RfLoopReg;
-	u8 n;
-	u32 Reg88c, Regc08, Reg874, Regc50, val32;
-	u8 initial_gain = 0x5a;
-	u32 PSD_report_tmp;
-	u32 AntA_report = 0x0, AntB_report = 0x0, AntO_report = 0x0;
-	bool bResult = true;
-	u32 AFE_Backup[16];
-	u32 AFE_REG_8723A[16] = {
-		rRx_Wait_CCA, rTx_CCK_RFON,
-		rTx_CCK_BBON, rTx_OFDM_RFON,
-		rTx_OFDM_BBON, rTx_To_Rx,
-		rTx_To_Tx, rRx_CCK,
-		rRx_OFDM, rRx_Wait_RIFS,
-		rRx_TO_Rx, rStandby,
-		rSleep, rPMPD_ANAEN,
-		rFPGA0_XCD_SwitchControl, rBlue_Tooth};
-
-	if (!(pDM_Odm->SupportICType & ODM_RTL8723A))
-		return bResult;
-
-	if (!(pDM_Odm->SupportAbility&ODM_BB_ANT_DIV))
-		return bResult;
-	/* 1 Backup Current RF/BB Settings */
-
-	CurrentChannel = ODM_GetRFReg(pDM_Odm, RF_PATH_A, ODM_CHANNEL,
-				      bRFRegOffsetMask);
-	RfLoopReg = ODM_GetRFReg(pDM_Odm, RF_PATH_A, 0x00, bRFRegOffsetMask);
-	/*  change to Antenna A */
-	val32 = rtl8723au_read32(adapter, rFPGA0_XA_RFInterfaceOE);
-	val32 &= ~0x300;
-	val32 |= 0x100;		/* Enable antenna A */
-	rtl8723au_write32(adapter, rFPGA0_XA_RFInterfaceOE, val32);
-
-	/*  Step 1: USE IQK to transmitter single tone */
-
-	udelay(10);
-
-	/* Store A Path Register 88c, c08, 874, c50 */
-	Reg88c = rtl8723au_read32(adapter, rFPGA0_AnalogParameter4);
-	Regc08 = rtl8723au_read32(adapter, rOFDM0_TRMuxPar);
-	Reg874 = rtl8723au_read32(adapter, rFPGA0_XCD_RFInterfaceSW);
-	Regc50 = rtl8723au_read32(adapter, rOFDM0_XAAGCCore1);
-
-	/*  Store AFE Registers */
-	odm_PHY_SaveAFERegisters(pDM_Odm, AFE_REG_8723A, AFE_Backup, 16);
-
-	/* Set PSD 128 pts */
-	val32 = rtl8723au_read32(adapter, rFPGA0_PSDFunction);
-	val32 &= ~(BIT(14) | BIT(15));
-	rtl8723au_write32(adapter, rFPGA0_PSDFunction, val32);
-
-	/*  To SET CH1 to do */
-	ODM_SetRFReg(pDM_Odm, RF_PATH_A, ODM_CHANNEL, bRFRegOffsetMask, 0x01);
-
-	/*  AFE all on step */
-	rtl8723au_write32(adapter, rRx_Wait_CCA, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rTx_CCK_RFON, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rTx_CCK_BBON, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rTx_OFDM_RFON, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rTx_OFDM_BBON, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rTx_To_Rx, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rTx_To_Tx, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rRx_CCK, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rRx_OFDM, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rRx_Wait_RIFS, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rRx_TO_Rx, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rStandby, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rSleep, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rPMPD_ANAEN, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rFPGA0_XCD_SwitchControl, 0x6FDB25A4);
-	rtl8723au_write32(adapter, rBlue_Tooth, 0x6FDB25A4);
-
-	/*  3 wire Disable */
-	rtl8723au_write32(adapter, rFPGA0_AnalogParameter4, 0xCCF000C0);
-
-	/* BB IQK Setting */
-	rtl8723au_write32(adapter, rOFDM0_TRMuxPar, 0x000800E4);
-	rtl8723au_write32(adapter, rFPGA0_XCD_RFInterfaceSW, 0x22208000);
-
-	/* IQK setting tone@ 4.34Mhz */
-	rtl8723au_write32(adapter, rTx_IQK_Tone_A, 0x10008C1C);
-	rtl8723au_write32(adapter, rTx_IQK, 0x01007c00);
-
-	/* Page B init */
-	rtl8723au_write32(adapter, rConfig_AntA, 0x00080000);
-	rtl8723au_write32(adapter, rConfig_AntA, 0x0f600000);
-	rtl8723au_write32(adapter, rRx_IQK, 0x01004800);
-	rtl8723au_write32(adapter, rRx_IQK_Tone_A, 0x10008c1f);
-	rtl8723au_write32(adapter, rTx_IQK_PI_A, 0x82150008);
-	rtl8723au_write32(adapter, rRx_IQK_PI_A, 0x28150008);
-	rtl8723au_write32(adapter, rIQK_AGC_Rsp, 0x001028d0);
-
-	/* RF loop Setting */
-	ODM_SetRFReg(pDM_Odm, RF_PATH_A, 0x0, 0xFFFFF, 0x50008);
-
-	/* IQK Single tone start */
-	rtl8723au_write32(adapter, rFPGA0_IQK, 0x80800000);
-	rtl8723au_write32(adapter, rIQK_AGC_Pts, 0xf8000000);
-	udelay(1000);
-	PSD_report_tmp = 0x0;
-
-	for (n = 0; n < 2; n++) {
-		PSD_report_tmp =  GetPSDData(pDM_Odm, 14, initial_gain);
-		if (PSD_report_tmp > AntA_report)
-			AntA_report = PSD_report_tmp;
-	}
-
-	PSD_report_tmp = 0x0;
-
-	val32 = rtl8723au_read32(adapter, rFPGA0_XA_RFInterfaceOE);
-	val32 &= ~0x300;
-	val32 |= 0x200;		/* Enable antenna B */
-	rtl8723au_write32(adapter, rFPGA0_XA_RFInterfaceOE, val32);
-	udelay(10);
-
-	for (n = 0; n < 2; n++) {
-		PSD_report_tmp =  GetPSDData(pDM_Odm, 14, initial_gain);
-		if (PSD_report_tmp > AntB_report)
-			AntB_report = PSD_report_tmp;
-	}
-
-	/*  change to open case */
-	/*  change to Ant A and B all open case */
-	val32 = rtl8723au_read32(adapter, rFPGA0_XA_RFInterfaceOE);
-	val32 &= ~0x300;
-	rtl8723au_write32(adapter, rFPGA0_XA_RFInterfaceOE, val32);
-	udelay(10);
-
-	for (n = 0; n < 2; n++) {
-		PSD_report_tmp =  GetPSDData(pDM_Odm, 14, initial_gain);
-		if (PSD_report_tmp > AntO_report)
-			AntO_report = PSD_report_tmp;
-	}
-
-	/* Close IQK Single Tone function */
-	rtl8723au_write32(adapter, rFPGA0_IQK, 0x00000000);
-	PSD_report_tmp = 0x0;
-
-	/* 1 Return to antanna A */
-	val32 = rtl8723au_read32(adapter, rFPGA0_XA_RFInterfaceOE);
-	val32 &= ~0x300;
-	val32 |= 0x100;		/* Enable antenna A */
-	rtl8723au_write32(adapter, rFPGA0_XA_RFInterfaceOE, val32);
-	rtl8723au_write32(adapter, rFPGA0_AnalogParameter4, Reg88c);
-	rtl8723au_write32(adapter, rOFDM0_TRMuxPar, Regc08);
-	rtl8723au_write32(adapter, rFPGA0_XCD_RFInterfaceSW, Reg874);
-	val32 = rtl8723au_read32(adapter, rOFDM0_XAAGCCore1);
-	val32 &= ~0x7f;
-	val32 |= 0x40;
-	rtl8723au_write32(adapter, rOFDM0_XAAGCCore1, val32);
-
-	rtl8723au_write32(adapter, rOFDM0_XAAGCCore1, Regc50);
-	ODM_SetRFReg(pDM_Odm, RF_PATH_A, RF_CHNLBW, bRFRegOffsetMask,
-		     CurrentChannel);
-	ODM_SetRFReg(pDM_Odm, RF_PATH_A, 0x00, bRFRegOffsetMask, RfLoopReg);
-
-	/* Reload AFE Registers */
-	odm_PHY_ReloadAFERegisters(pDM_Odm, AFE_REG_8723A, AFE_Backup, 16);
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV, ODM_DBG_LOUD,
-		     ("psd_report_A[%d]= %d \n", 2416, AntA_report));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV, ODM_DBG_LOUD,
-		     ("psd_report_B[%d]= %d \n", 2416, AntB_report));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV, ODM_DBG_LOUD,
-		     ("psd_report_O[%d]= %d \n", 2416, AntO_report));
-
-	/* 2 Test Ant B based on Ant A is ON */
-	if (mode == ANTTESTB) {
-		if (AntA_report >= 100) {
-			if (AntB_report > (AntA_report+1)) {
-				pDM_SWAT_Table->ANTB_ON = false;
-				ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV, ODM_DBG_LOUD, ("ODM_SingleDualAntennaDetection(): Single Antenna A\n"));
-			} else {
-				pDM_SWAT_Table->ANTB_ON = true;
-				ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV, ODM_DBG_LOUD, ("ODM_SingleDualAntennaDetection(): Dual Antenna is A and B\n"));
-			}
-		} else {
-			ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV, ODM_DBG_LOUD, ("ODM_SingleDualAntennaDetection(): Need to check again\n"));
-			pDM_SWAT_Table->ANTB_ON = false; /*  Set Antenna B off as default */
-			bResult = false;
-		}
-	} else if (mode == ANTTESTALL) {
-		/* 2 Test Ant A and B based on DPDT Open */
-		if ((AntO_report >= 100) & (AntO_report < 118)) {
-			if (AntA_report > (AntO_report+1)) {
-				pDM_SWAT_Table->ANTA_ON = false;
-				ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV,
-					     ODM_DBG_LOUD, ("Ant A is OFF"));
-			} else {
-				pDM_SWAT_Table->ANTA_ON = true;
-				ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV,
-					     ODM_DBG_LOUD, ("Ant A is ON"));
-			}
-
-			if (AntB_report > (AntO_report+2)) {
-				pDM_SWAT_Table->ANTB_ON = false;
-				ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV,
-					     ODM_DBG_LOUD, ("Ant B is OFF"));
-			} else {
-				pDM_SWAT_Table->ANTB_ON = true;
-				ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV,
-					     ODM_DBG_LOUD, ("Ant B is ON"));
-			}
-		}
-	} else {
-		ODM_RT_TRACE(pDM_Odm, ODM_COMP_ANT_DIV, ODM_DBG_LOUD,
-		("ODM_SingleDualAntennaDetection(): Need to check again\n"));
-		/*  Set Antenna A on as default */
-		pDM_SWAT_Table->ANTA_ON = true;
-		/*  Set Antenna B off as default */
-		pDM_SWAT_Table->ANTB_ON = false;
-		bResult = false;
-	}
-
-	return bResult;
-}
diff --git a/drivers/staging/rtl8723au/hal/odm_HWConfig.c b/drivers/staging/rtl8723au/hal/odm_HWConfig.c
deleted file mode 100644
index 0562f61bd1dc..000000000000
--- a/drivers/staging/rtl8723au/hal/odm_HWConfig.c
+++ /dev/null
@@ -1,396 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-/*  */
-/*  include files */
-/*  */
-
-#include "odm_precomp.h"
-
-static u8 odm_QueryRxPwrPercentage(s8 AntPower)
-{
-	if ((AntPower <= -100) || (AntPower >= 20))
-		return	0;
-	else if (AntPower >= 0)
-		return	100;
-	else
-		return	100 + AntPower;
-}
-
-static s32 odm_SignalScaleMapping_92CSeries(struct dm_odm_t *pDM_Odm, s32 CurrSig)
-{
-	s32 RetSig = 0;
-
-	if (CurrSig >= 51 && CurrSig <= 100)
-		RetSig = 100;
-	else if (CurrSig >= 41 && CurrSig <= 50)
-		RetSig = 80 + ((CurrSig - 40)*2);
-	else if (CurrSig >= 31 && CurrSig <= 40)
-		RetSig = 66 + (CurrSig - 30);
-	else if (CurrSig >= 21 && CurrSig <= 30)
-		RetSig = 54 + (CurrSig - 20);
-	else if (CurrSig >= 10 && CurrSig <= 20)
-		RetSig = 42 + (((CurrSig - 10) * 2) / 3);
-	else if (CurrSig >= 5 && CurrSig <= 9)
-		RetSig = 22 + (((CurrSig - 5) * 3) / 2);
-	else if (CurrSig >= 1 && CurrSig <= 4)
-		RetSig = 6 + (((CurrSig - 1) * 3) / 2);
-	else
-		RetSig = CurrSig;
-
-	return RetSig;
-}
-
-static s32 odm_SignalScaleMapping(struct dm_odm_t *pDM_Odm, s32 CurrSig)
-{
-	return odm_SignalScaleMapping_92CSeries(pDM_Odm, CurrSig);
-}
-
-static u8
-odm_EVMdbToPercentage(
-	s8 Value
-  )
-{
-	/*  */
-	/*  -33dB~0dB to 0%~99% */
-	/*  */
-	s8 ret_val;
-
-	ret_val = Value;
-
-	if (ret_val >= 0)
-		ret_val = 0;
-	if (ret_val <= -33)
-		ret_val = -33;
-
-	ret_val = 0 - ret_val;
-	ret_val *= 3;
-
-	if (ret_val == 99)
-		ret_val = 100;
-
-	return ret_val;
-}
-
-static void odm_RxPhyStatus92CSeries_Parsing(struct dm_odm_t *pDM_Odm,
-					     struct phy_info *pPhyInfo,
-					     u8 *pPhyStatus,
-					     struct odm_packet_info *pPktinfo)
-{
-	struct phy_status_rpt *pPhyStaRpt = (struct phy_status_rpt *)pPhyStatus;
-	u8 i, Max_spatial_stream;
-	s8 rx_pwr[4], rx_pwr_all = 0;
-	u8 EVM, PWDB_ALL = 0, PWDB_ALL_BT;
-	u8 RSSI, total_rssi = 0;
-	u8 isCCKrate = 0;
-	u8 rf_rx_num = 0;
-	u8 cck_highpwr = 0;
-
-	isCCKrate = (pPktinfo->Rate <= DESC92C_RATE11M) ? true : false;
-	pPhyInfo->RxMIMOSignalQuality[RF_PATH_A] = -1;
-	pPhyInfo->RxMIMOSignalQuality[RF_PATH_B] = -1;
-
-	if (isCCKrate) {
-		u8 report;
-		u8 cck_agc_rpt;
-
-		pDM_Odm->PhyDbgInfo.NumQryPhyStatusCCK++;
-		/*  (1)Hardware does not provide RSSI for CCK */
-		/*  (2)PWDB, Average PWDB cacluated by hardware (for rate adaptive) */
-
-		cck_highpwr = pDM_Odm->bCckHighPower;
-
-		cck_agc_rpt =  pPhyStaRpt->cck_agc_rpt_ofdm_cfosho_a;
-
-		/* The RSSI formula should be modified according to the gain table */
-		if (!cck_highpwr) {
-			report = (cck_agc_rpt & 0xc0)>>6;
-			switch (report) {
-			/*  Modify the RF RNA gain value to -40, -20, -2, 14 by Jenyu's suggestion */
-			/*  Note: different RF with the different RNA gain. */
-			case 0x3:
-				rx_pwr_all = -46 - (cck_agc_rpt & 0x3e);
-				break;
-			case 0x2:
-				rx_pwr_all = -26 - (cck_agc_rpt & 0x3e);
-				break;
-			case 0x1:
-				rx_pwr_all = -12 - (cck_agc_rpt & 0x3e);
-				break;
-			case 0x0:
-				rx_pwr_all = 16 - (cck_agc_rpt & 0x3e);
-				break;
-			}
-		} else {
-			report = (cck_agc_rpt & 0x60)>>5;
-			switch (report) {
-			case 0x3:
-				rx_pwr_all = -46 - ((cck_agc_rpt & 0x1f)<<1);
-				break;
-			case 0x2:
-				rx_pwr_all = -26 - ((cck_agc_rpt & 0x1f)<<1);
-				break;
-			case 0x1:
-				rx_pwr_all = -12 - ((cck_agc_rpt & 0x1f)<<1);
-				break;
-			case 0x0:
-				rx_pwr_all = 16 - ((cck_agc_rpt & 0x1f)<<1);
-				break;
-			}
-		}
-
-		PWDB_ALL = odm_QueryRxPwrPercentage(rx_pwr_all);
-
-		/* Modification for ext-LNA board */
-		if (pDM_Odm->BoardType == ODM_BOARD_HIGHPWR) {
-			if ((cck_agc_rpt>>7) == 0) {
-				PWDB_ALL = (PWDB_ALL > 94) ? 100 : (PWDB_ALL+6);
-			} else {
-				if (PWDB_ALL > 38)
-					PWDB_ALL -= 16;
-				else
-					PWDB_ALL = (PWDB_ALL <= 16) ? (PWDB_ALL>>2) : (PWDB_ALL-12);
-			}
-
-			/* CCK modification */
-			if (PWDB_ALL > 25 && PWDB_ALL <= 60)
-				PWDB_ALL += 6;
-		} else { /* Modification for int-LNA board */
-			if (PWDB_ALL > 99)
-				PWDB_ALL -= 8;
-			else if (PWDB_ALL > 50 && PWDB_ALL <= 68)
-				PWDB_ALL += 4;
-		}
-		pPhyInfo->RxPWDBAll = PWDB_ALL;
-		pPhyInfo->BTRxRSSIPercentage = PWDB_ALL;
-		pPhyInfo->RecvSignalPower = rx_pwr_all;
-		/*  (3) Get Signal Quality (EVM) */
-		if (pPktinfo->bPacketMatchBSSID) {
-			u8	SQ, SQ_rpt;
-
-			SQ_rpt = pPhyStaRpt->cck_sig_qual_ofdm_pwdb_all;
-
-			if (SQ_rpt > 64)
-				SQ = 0;
-			else if (SQ_rpt < 20)
-				SQ = 100;
-			else
-				SQ = ((64-SQ_rpt) * 100) / 44;
-
-			pPhyInfo->SignalQuality = SQ;
-			pPhyInfo->RxMIMOSignalQuality[RF_PATH_A] = SQ;
-			pPhyInfo->RxMIMOSignalQuality[RF_PATH_B] = -1;
-		}
-	} else { /* is OFDM rate */
-		pDM_Odm->PhyDbgInfo.NumQryPhyStatusOFDM++;
-
-		/*  (1)Get RSSI for HT rate */
-
-		for (i = RF_PATH_A; i < RF_PATH_MAX; i++) {
-			/*  2008/01/30 MH we will judge RF RX path now. */
-			if (pDM_Odm->RFPathRxEnable & BIT(i))
-				rf_rx_num++;
-
-			rx_pwr[i] = ((pPhyStaRpt->path_agc[i].gain & 0x3F)*2) - 110;
-
-			pPhyInfo->RxPwr[i] = rx_pwr[i];
-
-			/* Translate DBM to percentage. */
-			RSSI = odm_QueryRxPwrPercentage(rx_pwr[i]);
-			total_rssi += RSSI;
-
-			/* Modification for ext-LNA board */
-			if (pDM_Odm->BoardType == ODM_BOARD_HIGHPWR) {
-				if ((pPhyStaRpt->path_agc[i].trsw) == 1)
-					RSSI = (RSSI > 94) ? 100 : (RSSI+6);
-				else
-					RSSI = (RSSI <= 16) ? (RSSI>>3) : (RSSI-16);
-
-				if ((RSSI <= 34) && (RSSI >= 4))
-					RSSI -= 4;
-			}
-
-			pPhyInfo->RxMIMOSignalStrength[i] = (u8) RSSI;
-
-			/* Get Rx snr value in DB */
-			pPhyInfo->RxSNR[i] = pDM_Odm->PhyDbgInfo.RxSNRdB[i] = (s32)(pPhyStaRpt->path_rxsnr[i]/2);
-		}
-
-		/*  (2)PWDB, Average PWDB cacluated by hardware (for rate adaptive) */
-		rx_pwr_all = (((pPhyStaRpt->cck_sig_qual_ofdm_pwdb_all) >> 1) & 0x7f)-110;
-
-		PWDB_ALL = odm_QueryRxPwrPercentage(rx_pwr_all);
-		PWDB_ALL_BT = PWDB_ALL;
-
-		pPhyInfo->RxPWDBAll = PWDB_ALL;
-		pPhyInfo->BTRxRSSIPercentage = PWDB_ALL_BT;
-		pPhyInfo->RxPower = rx_pwr_all;
-		pPhyInfo->RecvSignalPower = rx_pwr_all;
-
-		/*  (3)EVM of HT rate */
-		if (pPktinfo->Rate >= DESC92C_RATEMCS8 && pPktinfo->Rate <= DESC92C_RATEMCS15)
-			Max_spatial_stream = 2; /* both spatial stream make sense */
-		else
-			Max_spatial_stream = 1; /* only spatial stream 1 makes sense */
-
-		for (i = 0; i < Max_spatial_stream; i++) {
-			/*  Do not use shift operation like "rx_evmX >>= 1" because the compilor of free build environment */
-			/*  fill most significant bit to "zero" when doing shifting operation which may change a negative */
-			/*  value to positive one, then the dbm value (which is supposed to be negative)  is not correct anymore. */
-			EVM = odm_EVMdbToPercentage((pPhyStaRpt->stream_rxevm[i]));	/* dbm */
-
-			if (pPktinfo->bPacketMatchBSSID) {
-				if (i == RF_PATH_A) {
-					/*  Fill value in RFD, Get the first spatial stream only */
-					pPhyInfo->SignalQuality = (u8)(EVM & 0xff);
-				}
-				pPhyInfo->RxMIMOSignalQuality[i] = (u8)(EVM & 0xff);
-			}
-		}
-	}
-	/* UI BSS List signal strength(in percentage), make it good looking, from 0~100. */
-	/* It is assigned to the BSS List in GetValueFromBeaconOrProbeRsp(). */
-	if (isCCKrate) {
-		pPhyInfo->SignalStrength = (u8)(odm_SignalScaleMapping(pDM_Odm, PWDB_ALL));/* PWDB_ALL; */
-	} else {
-		if (rf_rx_num != 0)
-			pPhyInfo->SignalStrength = (u8)(odm_SignalScaleMapping(pDM_Odm, total_rssi /= rf_rx_num));
-	}
-}
-
-static void odm_Process_RSSIForDM(struct dm_odm_t *pDM_Odm,
-				  struct phy_info *pPhyInfo,
-				  struct odm_packet_info *pPktinfo)
-{
-	s32 UndecoratedSmoothedPWDB, UndecoratedSmoothedCCK;
-	s32 UndecoratedSmoothedOFDM, RSSI_Ave;
-	u8 isCCKrate = 0;
-	u8 RSSI_max, RSSI_min, i;
-	u32 OFDM_pkt = 0;
-	u32 Weighting = 0;
-	struct sta_info *pEntry;
-
-	if (pPktinfo->StationID == 0xFF)
-		return;
-
-	pEntry = pDM_Odm->pODM_StaInfo[pPktinfo->StationID];
-	if (!pEntry)
-		return;
-	if ((!pPktinfo->bPacketMatchBSSID))
-		return;
-
-	isCCKrate = (pPktinfo->Rate <= DESC92C_RATE11M) ? true : false;
-
-	/* Smart Antenna Debug Message------------------*/
-
-	UndecoratedSmoothedCCK =  pEntry->rssi_stat.UndecoratedSmoothedCCK;
-	UndecoratedSmoothedOFDM = pEntry->rssi_stat.UndecoratedSmoothedOFDM;
-	UndecoratedSmoothedPWDB = pEntry->rssi_stat.UndecoratedSmoothedPWDB;
-
-	if (pPktinfo->bPacketToSelf || pPktinfo->bPacketBeacon) {
-		if (!isCCKrate) { /* ofdm rate */
-			if (pPhyInfo->RxMIMOSignalStrength[RF_PATH_B] == 0) {
-				RSSI_Ave = pPhyInfo->RxMIMOSignalStrength[RF_PATH_A];
-			} else {
-				if (pPhyInfo->RxMIMOSignalStrength[RF_PATH_A] > pPhyInfo->RxMIMOSignalStrength[RF_PATH_B]) {
-					RSSI_max = pPhyInfo->RxMIMOSignalStrength[RF_PATH_A];
-					RSSI_min = pPhyInfo->RxMIMOSignalStrength[RF_PATH_B];
-				} else {
-					RSSI_max = pPhyInfo->RxMIMOSignalStrength[RF_PATH_B];
-					RSSI_min = pPhyInfo->RxMIMOSignalStrength[RF_PATH_A];
-				}
-				if ((RSSI_max - RSSI_min) < 3)
-					RSSI_Ave = RSSI_max;
-				else if ((RSSI_max - RSSI_min) < 6)
-					RSSI_Ave = RSSI_max - 1;
-				else if ((RSSI_max - RSSI_min) < 10)
-					RSSI_Ave = RSSI_max - 2;
-				else
-					RSSI_Ave = RSSI_max - 3;
-			}
-
-			/* 1 Process OFDM RSSI */
-			if (UndecoratedSmoothedOFDM <= 0) {
-				/*  initialize */
-				UndecoratedSmoothedOFDM = pPhyInfo->RxPWDBAll;
-			} else {
-				if (pPhyInfo->RxPWDBAll > (u32)UndecoratedSmoothedOFDM) {
-					UndecoratedSmoothedOFDM =
-							(((UndecoratedSmoothedOFDM)*(Rx_Smooth_Factor-1)) +
-							(RSSI_Ave)) / (Rx_Smooth_Factor);
-					UndecoratedSmoothedOFDM = UndecoratedSmoothedOFDM + 1;
-				} else {
-					UndecoratedSmoothedOFDM =
-							(((UndecoratedSmoothedOFDM)*(Rx_Smooth_Factor-1)) +
-							(RSSI_Ave)) / (Rx_Smooth_Factor);
-				}
-			}
-			pEntry->rssi_stat.PacketMap =
-				(pEntry->rssi_stat.PacketMap<<1) | BIT(0);
-		} else {
-			RSSI_Ave = pPhyInfo->RxPWDBAll;
-
-			/* 1 Process CCK RSSI */
-			if (UndecoratedSmoothedCCK <= 0) {
-				/*  initialize */
-				UndecoratedSmoothedCCK = pPhyInfo->RxPWDBAll;
-			} else {
-				if (pPhyInfo->RxPWDBAll > (u32)UndecoratedSmoothedCCK) {
-					UndecoratedSmoothedCCK =
-							(((UndecoratedSmoothedCCK)*(Rx_Smooth_Factor-1)) +
-							(pPhyInfo->RxPWDBAll)) / (Rx_Smooth_Factor);
-					UndecoratedSmoothedCCK = UndecoratedSmoothedCCK + 1;
-				} else {
-					UndecoratedSmoothedCCK =
-							(((UndecoratedSmoothedCCK)*(Rx_Smooth_Factor-1)) +
-							(pPhyInfo->RxPWDBAll)) / (Rx_Smooth_Factor);
-				}
-			}
-			pEntry->rssi_stat.PacketMap = pEntry->rssi_stat.PacketMap<<1;
-		}
-
-		/* 2011.07.28 LukeLee: modified to prevent unstable CCK RSSI */
-		if (pEntry->rssi_stat.ValidBit >= 64)
-			pEntry->rssi_stat.ValidBit = 64;
-		else
-			pEntry->rssi_stat.ValidBit++;
-
-		for (i = 0; i < pEntry->rssi_stat.ValidBit; i++)
-			OFDM_pkt +=
-				(u8)(pEntry->rssi_stat.PacketMap>>i) & BIT(0);
-
-		if (pEntry->rssi_stat.ValidBit == 64) {
-			Weighting = ((OFDM_pkt<<4) > 64)?64:(OFDM_pkt<<4);
-			UndecoratedSmoothedPWDB = (Weighting*UndecoratedSmoothedOFDM+(64-Weighting)*UndecoratedSmoothedCCK)>>6;
-		} else {
-			if (pEntry->rssi_stat.ValidBit != 0)
-				UndecoratedSmoothedPWDB = (OFDM_pkt*UndecoratedSmoothedOFDM+(pEntry->rssi_stat.ValidBit-OFDM_pkt)*UndecoratedSmoothedCCK)/pEntry->rssi_stat.ValidBit;
-			else
-				UndecoratedSmoothedPWDB = 0;
-		}
-		pEntry->rssi_stat.UndecoratedSmoothedCCK = UndecoratedSmoothedCCK;
-		pEntry->rssi_stat.UndecoratedSmoothedOFDM = UndecoratedSmoothedOFDM;
-		pEntry->rssi_stat.UndecoratedSmoothedPWDB = UndecoratedSmoothedPWDB;
-	}
-}
-
-void ODM_PhyStatusQuery23a(struct dm_odm_t *pDM_Odm, struct phy_info *pPhyInfo,
-			   u8 *pPhyStatus, struct odm_packet_info *pPktinfo)
-{
-	odm_RxPhyStatus92CSeries_Parsing(pDM_Odm, pPhyInfo,
-					 pPhyStatus, pPktinfo);
-
-	odm_Process_RSSIForDM(pDM_Odm, pPhyInfo, pPktinfo);
-}
diff --git a/drivers/staging/rtl8723au/hal/odm_RegConfig8723A.c b/drivers/staging/rtl8723au/hal/odm_RegConfig8723A.c
deleted file mode 100644
index a63c6cb88bc9..000000000000
--- a/drivers/staging/rtl8723au/hal/odm_RegConfig8723A.c
+++ /dev/null
@@ -1,88 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#include "odm_precomp.h"
-#include "usb_ops_linux.h"
-
-void
-odm_ConfigRFReg_8723A(
-	struct dm_odm_t *pDM_Odm,
-	u32					Addr,
-	u32					Data,
-	enum RF_RADIO_PATH     RF_PATH,
-	u32				    RegAddr
-	)
-{
-	if (Addr == 0xfe) {
-		msleep(50);
-	} else if (Addr == 0xfd) {
-		mdelay(5);
-	} else if (Addr == 0xfc) {
-		mdelay(1);
-	} else if (Addr == 0xfb) {
-		udelay(50);
-	} else if (Addr == 0xfa) {
-		udelay(5);
-	} else if (Addr == 0xf9) {
-		udelay(1);
-	} else {
-		ODM_SetRFReg(pDM_Odm, RF_PATH, RegAddr, bRFRegOffsetMask, Data);
-		/*  Add 1us delay between BB/RF register setting. */
-		udelay(1);
-	}
-}
-
-void odm_ConfigMAC_8723A(struct dm_odm_t *pDM_Odm, u32 addr, u8	data)
-{
-	rtl8723au_write8(pDM_Odm->Adapter, addr, data);
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_INIT, ODM_DBG_LOUD,
-		     ("===> %s: [MAC_REG] %08X %08X\n", __func__, addr, data));
-}
-
-void odm_ConfigBB_AGC_8723A(struct dm_odm_t *pDM_Odm, u32 addr, u32 data)
-{
-	rtl8723au_write32(pDM_Odm->Adapter, addr, data);
-	/*  Add 1us delay between BB/RF register setting. */
-	udelay(1);
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_INIT, ODM_DBG_LOUD,
-		     ("===> %s: [AGC_TAB] %08X %08X\n", __func__, addr, data));
-}
-
-void
-odm_ConfigBB_PHY_8723A(struct dm_odm_t *pDM_Odm, u32 addr, u32 data)
-{
-	if (addr == 0xfe)
-		msleep(50);
-	else if (addr == 0xfd)
-		mdelay(5);
-	else if (addr == 0xfc)
-		mdelay(1);
-	else if (addr == 0xfb)
-		udelay(50);
-	else if (addr == 0xfa)
-		udelay(5);
-	else if (addr == 0xf9)
-		udelay(1);
-	else if (addr == 0xa24)
-		pDM_Odm->RFCalibrateInfo.RegA24 = data;
-	rtl8723au_write32(pDM_Odm->Adapter, addr, data);
-
-	/*  Add 1us delay between BB/RF register setting. */
-	udelay(1);
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_INIT, ODM_DBG_LOUD,
-		     ("===> %s: [PHY_REG] %08X %08X\n", __func__, addr, data));
-}
diff --git a/drivers/staging/rtl8723au/hal/odm_debug.c b/drivers/staging/rtl8723au/hal/odm_debug.c
deleted file mode 100644
index cb2bdda6b0eb..000000000000
--- a/drivers/staging/rtl8723au/hal/odm_debug.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#include "odm_precomp.h"
-
-void ODM_InitDebugSetting23a(struct dm_odm_t *pDM_Odm)
-{
-	pDM_Odm->DebugLevel = ODM_DBG_TRACE;
-	pDM_Odm->DebugComponents = 0;
-}
-
-u32 GlobalDebugLevel23A;
-
-void rt_trace(int comp, int level, const char *fmt, ...)
-{
-	struct va_format vaf;
-	va_list args;
-
-	va_start(args, fmt);
-
-	vaf.fmt = fmt;
-	vaf.va = &args;
-
-	pr_info(DRIVER_PREFIX " [0x%08x,%d] %pV", comp, level, &vaf);
-
-	va_end(args);
-}
diff --git a/drivers/staging/rtl8723au/hal/odm_interface.c b/drivers/staging/rtl8723au/hal/odm_interface.c
deleted file mode 100644
index d8f67902708e..000000000000
--- a/drivers/staging/rtl8723au/hal/odm_interface.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-/*  */
-/*  include files */
-/*  */
-
-#include "odm_precomp.h"
-/*  */
-/*  ODM IO Relative API. */
-/*  */
-#include <usb_ops_linux.h>
-
-void ODM_SetRFReg(
-	struct dm_odm_t *pDM_Odm,
-	enum RF_RADIO_PATH	eRFPath,
-	u32				RegAddr,
-	u32				BitMask,
-	u32				Data
-	)
-{
-	struct rtw_adapter *Adapter = pDM_Odm->Adapter;
-
-	PHY_SetRFReg(Adapter, eRFPath, RegAddr, BitMask, Data);
-}
-
-u32 ODM_GetRFReg(
-	struct dm_odm_t *pDM_Odm,
-	enum RF_RADIO_PATH	eRFPath,
-	u32				RegAddr,
-	u32				BitMask
-	)
-{
-	struct rtw_adapter *Adapter = pDM_Odm->Adapter;
-
-	return PHY_QueryRFReg(Adapter, eRFPath, RegAddr, BitMask);
-}
diff --git a/drivers/staging/rtl8723au/hal/rtl8723a_bt-coexist.c b/drivers/staging/rtl8723au/hal/rtl8723a_bt-coexist.c
deleted file mode 100644
index bfcbd7a349cf..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723a_bt-coexist.c
+++ /dev/null
@@ -1,11265 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- *published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#include <drv_types.h>
-#include <rtl8723a_hal.h>
-#include <usb_ops_linux.h>
-
-#define DIS_PS_RX_BCN
-
-u32 BTCoexDbgLevel = _bt_dbg_off_;
-
-#define RTPRINT(_Comp, _Level, Fmt)\
-do {\
-	if ((BTCoexDbgLevel == _bt_dbg_on_)) {\
-		printk Fmt;\
-	}					\
-} while (0)
-
-#define RTPRINT_ADDR(dbgtype, dbgflag, printstr, _Ptr)\
-if ((BTCoexDbgLevel == _bt_dbg_on_)) {\
-	u32 __i;						\
-	u8 *ptr = (u8 *)_Ptr;	\
-	printk printstr;				\
-	printk(" ");					\
-	for (__i = 0; __i < 6; __i++)		\
-		printk("%02X%s", ptr[__i], (__i == 5)?"":"-");		\
-	printk("\n");							\
-}
-#define RTPRINT_DATA(dbgtype, dbgflag, _TitleString, _HexData, _HexDataLen)\
-if ((BTCoexDbgLevel == _bt_dbg_on_)) {\
-	u32 __i;						\
-	u8 *ptr = (u8 *)_HexData;				\
-	printk(_TitleString);					\
-	for (__i = 0; __i < (u32)_HexDataLen; __i++) {		\
-		printk("%02X%s", ptr[__i], (((__i + 1) % 4) == 0)?"  ":" ");\
-		if (((__i + 1) % 16) == 0)			\
-			printk("\n");				\
-	}								\
-	printk("\n");							\
-}
-/*  Added by Annie, 2005-11-22. */
-#define MAX_STR_LEN	64
-/*  I want to see ASCII 33 to 126 only. Otherwise, I print '?'. */
-#define PRINTABLE(_ch)	(_ch >= ' ' && _ch <= '~')
-#define RT_PRINT_STR(_Comp, _Level, _TitleString, _Ptr, _Len)		\
-	{								\
-		u32 __i;						\
-		u8 buffer[MAX_STR_LEN];					\
-		u32 length = (_Len < MAX_STR_LEN) ? _Len : (MAX_STR_LEN-1);\
-		memset(buffer, 0, MAX_STR_LEN);				\
-		memcpy(buffer, (u8 *)_Ptr, length);			\
-		for (__i = 0; __i < length; __i++) {			\
-			if (!PRINTABLE(buffer[__i]))			\
-				buffer[__i] = '?';			\
-		}							\
-		buffer[length] = '\0';					\
-		printk(_TitleString);					\
-		printk(": %d, <%s>\n", _Len, buffer);			\
-	}
-
-#define DCMD_Printf(...)
-#define RT_ASSERT(...)
-
-
-#define GetDefaultAdapter(padapter)	padapter
-
-#define PlatformZeroMemory(ptr, sz)	memset(ptr, 0, sz)
-
-#define GET_UNDECORATED_AVERAGE_RSSI(padapter)	\
-			(GET_HAL_DATA(padapter)->dmpriv.EntryMinUndecoratedSmoothedPWDB)
-#define RT_RF_CHANGE_SOURCE u32
-
-enum {
-	RT_JOIN_INFRA   = 1,
-	RT_JOIN_IBSS  = 2,
-	RT_START_IBSS = 3,
-	RT_NO_ACTION  = 4,
-};
-
-/*  power saving */
-
-/*  ===== Below this line is sync from SD7 driver COMMOM/BT.c ===== */
-
-static u8 BT_Operation(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->BtOperationOn)
-		return true;
-	else
-		return false;
-}
-
-static u8 BT_IsLegalChannel(struct rtw_adapter *padapter, u8 channel)
-{
-	struct rt_channel_info *pChanneList = NULL;
-	u8 channelLen, i;
-
-	pChanneList = padapter->mlmeextpriv.channel_set;
-	channelLen = padapter->mlmeextpriv.max_chan_nums;
-
-	for (i = 0; i < channelLen; i++) {
-		RTPRINT(FIOCTL, IOCTL_STATE,
-			("Check if chnl(%d) in channel plan contains bt target chnl(%d) for BT connection\n",
-			 pChanneList[i].ChannelNum, channel));
-		if ((channel == pChanneList[i].ChannelNum) ||
-		    (channel == pChanneList[i].ChannelNum + 2))
-			return channel;
-	}
-	return 0;
-}
-
-void BT_SignalCompensation(struct rtw_adapter *padapter, u8 *rssi_wifi, u8 *rssi_bt)
-{
-	BTDM_SignalCompensation(padapter, rssi_wifi, rssi_bt);
-}
-
-void rtl8723a_BT_wifiscan_notify(struct rtw_adapter *padapter, u8 scanType)
-{
-	BTHCI_WifiScanNotify(padapter, scanType);
-	BTDM_CheckAntSelMode(padapter);
-	BTDM_WifiScanNotify(padapter, scanType);
-}
-
-void rtl8723a_BT_wifiassociate_notify(struct rtw_adapter *padapter, u8 action)
-{
-	/*  action : */
-	/*  true = associate start */
-	/*  false = associate finished */
-	if (action)
-		BTDM_CheckAntSelMode(padapter);
-
-	BTDM_WifiAssociateNotify(padapter, action);
-}
-
-void BT_HaltProcess(struct rtw_adapter *padapter)
-{
-	BTDM_ForHalt(padapter);
-}
-
-/*  ===== End of sync from SD7 driver COMMOM/BT.c ===== */
-
-#define i64fmt		"ll"
-#define UINT64_C(v)  (v)
-
-#define FillOctetString(_os, _octet, _len)		\
-	(_os).Octet = (u8 *)(_octet);			\
-	(_os).Length = (_len);
-
-static enum rt_status PlatformIndicateBTEvent(
-	struct rtw_adapter *padapter,
-	void						*pEvntData,
-	u32						dataLen
-	)
-{
-	enum rt_status	rt_status = RT_STATUS_FAILURE;
-
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT_DETAIL, ("BT event start, %d bytes data to Transferred!!\n", dataLen));
-	RTPRINT_DATA(FIOCTL, IOCTL_BT_EVENT_DETAIL, "To transfer Hex Data :\n",
-		pEvntData, dataLen);
-
-	BT_EventParse(padapter, pEvntData, dataLen);
-
-	printk(KERN_WARNING "%s: Linux has no way to report BT event!!\n", __func__);
-
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT_DETAIL, ("BT event end, %s\n",
-		(rt_status == RT_STATUS_SUCCESS) ? "SUCCESS" : "FAIL"));
-
-	return rt_status;
-}
-
-/*  ===== Below this line is sync from SD7 driver COMMOM/bt_hci.c ===== */
-
-static u8 bthci_GetLocalChannel(struct rtw_adapter *padapter)
-{
-	return padapter->mlmeextpriv.cur_channel;
-}
-
-static u8 bthci_GetCurrentEntryNum(struct rtw_adapter *padapter, u8 PhyHandle)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	u8 i;
-
-	for (i = 0; i < MAX_BT_ASOC_ENTRY_NUM; i++) {
-		if ((pBTInfo->BtAsocEntry[i].bUsed) &&
-		    (pBTInfo->BtAsocEntry[i].PhyLinkCmdData.BtPhyLinkhandle == PhyHandle))
-			return i;
-	}
-
-	return 0xFF;
-}
-
-static void bthci_DecideBTChannel(struct rtw_adapter *padapter, u8 EntryNum)
-{
-/*PMGNT_INFO pMgntInfo = &padapter->MgntInfo; */
-	struct mlme_priv *pmlmepriv;
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_hci_info *pBtHciInfo;
-	struct chnl_txpower_triple *pTriple_subband = NULL;
-	struct common_triple *pTriple;
-	u8 i, j, localchnl, firstRemoteLegalChnlInTriplet = 0;
-	u8 regulatory_skipLen = 0;
-	u8 subbandTripletCnt = 0;
-
-	pmlmepriv = &padapter->mlmepriv;
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtHciInfo = &pBTInfo->BtHciInfo;
-
-	pBtMgnt->CheckChnlIsSuit = true;
-	localchnl = bthci_GetLocalChannel(padapter);
-
-	pTriple = (struct common_triple *)
-		&pBtHciInfo->BTPreChnllist[COUNTRY_STR_LEN];
-
-	/*  contains country string, len is 3 */
-	for (i = 0; i < (pBtHciInfo->BtPreChnlListLen-COUNTRY_STR_LEN); i += 3, pTriple++) {
-		/*  */
-		/*  check every triplet, an triplet may be */
-		/*  regulatory extension identifier or sub-band triplet */
-		/*  */
-		if (pTriple->byte_1st == 0xc9) {
-			/*  Regulatory Extension Identifier, skip it */
-			RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO),
-				("Find Regulatory ID, regulatory class = %d\n", pTriple->byte_2nd));
-			regulatory_skipLen += 3;
-			pTriple_subband = NULL;
-			continue;
-		} else {	/*  Sub-band triplet */
-			RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Find Sub-band triplet \n"));
-			subbandTripletCnt++;
-			pTriple_subband = (struct chnl_txpower_triple *)pTriple;
-			/*  if remote first legal channel not found, then find first remote channel */
-			/*  and it's legal for our channel plan. */
-
-			/*  search the sub-band triplet and find if remote channel is legal to our channel plan. */
-			for (j = pTriple_subband->FirstChnl; j < (pTriple_subband->FirstChnl+pTriple_subband->NumChnls); j++) {
-				RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), (" Check if chnl(%d) is legal\n", j));
-				if (BT_IsLegalChannel(padapter, j)) {
-					/*  remote channel is legal for our channel plan. */
-					firstRemoteLegalChnlInTriplet = j;
-					RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO),
-						("Find first remote legal channel : %d\n",
-						firstRemoteLegalChnlInTriplet));
-
-					/*  If we find a remote legal channel in the sub-band triplet */
-					/*  and only BT connection is established(local not connect to any AP or IBSS), */
-					/*  then we just switch channel to remote channel. */
-					if (!(check_fwstate(pmlmepriv, WIFI_ASOC_STATE|WIFI_ADHOC_STATE|WIFI_AP_STATE) ||
-					    BTHCI_HsConnectionEstablished(padapter))) {
-						pBtMgnt->BTChannel = firstRemoteLegalChnlInTriplet;
-						RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Remote legal channel (%d) is selected, Local not connect to any!!\n", pBtMgnt->BTChannel));
-						return;
-					} else {
-						if ((localchnl >= firstRemoteLegalChnlInTriplet) &&
-						    (localchnl < (pTriple_subband->FirstChnl+pTriple_subband->NumChnls))) {
-							pBtMgnt->BTChannel = localchnl;
-							RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Local channel (%d) is selected, wifi or BT connection exists\n", pBtMgnt->BTChannel));
-							return;
-						}
-					}
-					break;
-				}
-			}
-		}
-	}
-
-	if (subbandTripletCnt) {
-		/* if any preferred channel triplet exists */
-		RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("There are %d sub band triplet exists, ", subbandTripletCnt));
-		if (firstRemoteLegalChnlInTriplet == 0) {
-			/* no legal channel is found, reject the connection. */
-			RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("no legal channel is found!!\n"));
-		} else {
-			/*  Remote Legal channel is found but not match to local */
-			/* wifi connection exists), so reject the connection. */
-			RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO),
-				("Remote Legal channel is found but not match to local(wifi connection exists)!!\n"));
-		}
-		pBtMgnt->CheckChnlIsSuit = false;
-	} else {
-		/*  There are not any preferred channel triplet exists */
-		/*  Use current legal channel as the bt channel. */
-		RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("No sub band triplet exists!!\n"));
-	}
-	pBtMgnt->BTChannel = localchnl;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Local channel (%d) is selected!!\n", pBtMgnt->BTChannel));
-}
-
-/* Success:return true */
-/* Fail:return false */
-static u8 bthci_GetAssocInfo(struct rtw_adapter *padapter, u8 EntryNum)
-{
-	struct bt_30info *pBTInfo;
-	struct bt_hci_info *pBtHciInfo;
-	u8 tempBuf[256];
-	u8 i = 0;
-	u8 BaseMemoryShift = 0;
-	u16	TotalLen = 0;
-	struct amp_assoc_structure *pAmpAsoc;
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("GetAssocInfo start\n"));
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtHciInfo = &pBTInfo->BtHciInfo;
-
-	if (pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.LenSoFar == 0) {
-		if (pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.AMPAssocRemLen < (MAX_AMP_ASSOC_FRAG_LEN))
-			TotalLen = pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.AMPAssocRemLen;
-		else if (pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.AMPAssocRemLen == (MAX_AMP_ASSOC_FRAG_LEN))
-			TotalLen = MAX_AMP_ASSOC_FRAG_LEN;
-	} else if (pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.LenSoFar > 0)
-		TotalLen = pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.LenSoFar;
-
-	while ((pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.LenSoFar >= BaseMemoryShift) || TotalLen > BaseMemoryShift) {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_DETAIL, ("GetAssocInfo, TotalLen =%d, BaseMemoryShift =%d\n", TotalLen, BaseMemoryShift));
-		memcpy(tempBuf,
-			(u8 *)pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.AMPAssocfragment+BaseMemoryShift,
-			TotalLen-BaseMemoryShift);
-		RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD_DETAIL, "GetAssocInfo :\n",
-			tempBuf, TotalLen-BaseMemoryShift);
-
-		pAmpAsoc = (struct amp_assoc_structure *)tempBuf;
-		le16_to_cpus(&pAmpAsoc->Length);
-		BaseMemoryShift += 3 + pAmpAsoc->Length;
-
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("TypeID = 0x%x, ", pAmpAsoc->TypeID));
-		RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD, "Hex Data: \n", pAmpAsoc->Data, pAmpAsoc->Length);
-		switch (pAmpAsoc->TypeID) {
-		case AMP_MAC_ADDR:
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("==> AMP_MAC_ADDR\n"));
-			if (pAmpAsoc->Length > 6)
-				return false;
-			memcpy(pBTInfo->BtAsocEntry[EntryNum].BTRemoteMACAddr, pAmpAsoc->Data, 6);
-			RTPRINT_ADDR(FIOCTL, IOCTL_BT_HCICMD, ("Remote Mac address \n"), pBTInfo->BtAsocEntry[EntryNum].BTRemoteMACAddr);
-			break;
-		case AMP_PREFERRED_CHANNEL_LIST:
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("==> AMP_PREFERRED_CHANNEL_LIST\n"));
-			pBtHciInfo->BtPreChnlListLen = pAmpAsoc->Length;
-			memcpy(pBtHciInfo->BTPreChnllist,
-				pAmpAsoc->Data,
-				pBtHciInfo->BtPreChnlListLen);
-			RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD, "Preferred channel list : \n", pBtHciInfo->BTPreChnllist, pBtHciInfo->BtPreChnlListLen);
-			bthci_DecideBTChannel(padapter, EntryNum);
-			break;
-		case AMP_CONNECTED_CHANNEL:
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("==> AMP_CONNECTED_CHANNEL\n"));
-			pBtHciInfo->BTConnectChnlListLen = pAmpAsoc->Length;
-			memcpy(pBtHciInfo->BTConnectChnllist,
-				pAmpAsoc->Data,
-				pBtHciInfo->BTConnectChnlListLen);
-			break;
-		case AMP_80211_PAL_CAP_LIST:
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("==> AMP_80211_PAL_CAP_LIST\n"));
-			pBTInfo->BtAsocEntry[EntryNum].BTCapability = *(u32 *)(pAmpAsoc->Data);
-			if (pBTInfo->BtAsocEntry[EntryNum].BTCapability & 0x00000001) {
-				/*  TODO: */
-
-				/* Signifies PAL capable of utilizing received activity reports. */
-			}
-			if (pBTInfo->BtAsocEntry[EntryNum].BTCapability & 0x00000002) {
-				/*  TODO: */
-				/* Signifies PAL is capable of utilizing scheduling information received in an activity reports. */
-			}
-			break;
-		case AMP_80211_PAL_VISION:
-			pBtHciInfo->BTPalVersion = *(u8 *)(pAmpAsoc->Data);
-			pBtHciInfo->BTPalCompanyID = *(u16 *)(((u8 *)(pAmpAsoc->Data))+1);
-			pBtHciInfo->BTPalsubversion = *(u16 *)(((u8 *)(pAmpAsoc->Data))+3);
-			RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("==> AMP_80211_PAL_VISION PalVersion  0x%x, PalCompanyID  0x%x, Palsubversion 0x%x\n",
-				pBtHciInfo->BTPalVersion,
-				pBtHciInfo->BTPalCompanyID,
-				pBtHciInfo->BTPalsubversion));
-			break;
-		default:
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("==> Unsupport TypeID !!\n"));
-			break;
-		}
-		i++;
-	}
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("GetAssocInfo end\n"));
-
-	return true;
-}
-
-static u8 bthci_AddEntry(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	u8 i;
-
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-
-	for (i = 0; i < MAX_BT_ASOC_ENTRY_NUM; i++) {
-		if (pBTInfo->BtAsocEntry[i].bUsed == false) {
-			pBTInfo->BtAsocEntry[i].bUsed = true;
-			pBtMgnt->CurrentConnectEntryNum = i;
-			break;
-		}
-	}
-
-	if (i == MAX_BT_ASOC_ENTRY_NUM) {
-		RTPRINT(FIOCTL, IOCTL_STATE, ("bthci_AddEntry(), Add entry fail!!\n"));
-		return false;
-	}
-	return true;
-}
-
-static u8 bthci_DiscardTxPackets(struct rtw_adapter *padapter, u16 LLH)
-{
-	return false;
-}
-
-static u8
-bthci_CheckLogLinkBehavior(
-	struct rtw_adapter *padapter,
-	struct hci_flow_spec			TxFlowSpec
-	)
-{
-	u8 ID = TxFlowSpec.Identifier;
-	u8 ServiceType = TxFlowSpec.ServiceType;
-	u16	MaxSDUSize = TxFlowSpec.MaximumSDUSize;
-	u32	SDUInterArrivatime = TxFlowSpec.SDUInterArrivalTime;
-	u8 match = false;
-
-	switch (ID) {
-	case 1:
-		if (ServiceType == BT_LL_BE) {
-			match = true;
-			RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Logical Link Type =  TX best effort flowspec\n"));
-		} else if ((ServiceType == BT_LL_GU) && (MaxSDUSize == 0xffff)) {
-			match = true;
-			RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Logical Link Type =  RX guaranteed latency flowspec\n"));
-		} else if ((ServiceType == BT_LL_GU) && (MaxSDUSize == 2500)) {
-			RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Logical Link Type =  RX guaranteed Large latency flowspec\n"));
-		}
-		break;
-	case 2:
-		if (ServiceType == BT_LL_BE) {
-			match = true;
-			RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Logical Link Type =  RX best effort flowspec\n"));
-
-		}
-		break;
-	case 3:
-		if ((ServiceType == BT_LL_GU) && (MaxSDUSize == 1492)) {
-			match = true;
-			RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Logical Link Type =  TX guaranteed latency flowspec\n"));
-		} else if ((ServiceType == BT_LL_GU) && (MaxSDUSize == 2500)) {
-			RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Logical Link Type =  TX guaranteed Large latency flowspec\n"));
-		}
-		break;
-	case 4:
-		if (ServiceType == BT_LL_BE) {
-			if ((SDUInterArrivatime == 0xffffffff) && (ServiceType == BT_LL_BE) && (MaxSDUSize == 1492)) {
-				match = true;
-				RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Logical Link Type =  TX/RX aggregated best effort flowspec\n"));
-			}
-		} else if (ServiceType == BT_LL_GU) {
-			if (SDUInterArrivatime == 100) {
-				match = true;
-				RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Logical Link Type =  TX/RX guaranteed bandwidth flowspec\n"));
-			}
-		}
-		break;
-	default:
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Logical Link Type =  Unknow Type !!!!!!!!\n"));
-		break;
-	}
-
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO),
-		("ID = 0x%x, ServiceType = 0x%x, MaximumSDUSize = 0x%x, SDUInterArrivalTime = 0x%x, AccessLatency = 0x%x, FlushTimeout = 0x%x\n",
-		TxFlowSpec.Identifier, TxFlowSpec.ServiceType, MaxSDUSize,
-		SDUInterArrivatime, TxFlowSpec.AccessLatency, TxFlowSpec.FlushTimeout));
-	return match;
-}
-
-static u16 bthci_AssocMACAddr(struct rtw_adapter *padapter, void	*pbuf)
-{
-	struct amp_assoc_structure *pAssoStrc = (struct amp_assoc_structure *)pbuf;
-	pAssoStrc->TypeID = AMP_MAC_ADDR;
-	pAssoStrc->Length = 0x06;
-	memcpy(&pAssoStrc->Data[0], padapter->eeprompriv.mac_addr, 6);
-	RTPRINT_DATA(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO),
-		     ("AssocMACAddr : \n"), pAssoStrc, pAssoStrc->Length+3);
-
-	return pAssoStrc->Length + 3;
-}
-
-static u16
-bthci_PALCapabilities(
-	struct rtw_adapter *padapter,
-	void	*pbuf
-	)
-{
-	struct amp_assoc_structure *pAssoStrc = (struct amp_assoc_structure *)pbuf;
-
-	pAssoStrc->TypeID = AMP_80211_PAL_CAP_LIST;
-	pAssoStrc->Length = 0x04;
-
-	pAssoStrc->Data[0] = 0x00;
-	pAssoStrc->Data[1] = 0x00;
-
-	RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD_DETAIL, ("PALCapabilities:\n"), pAssoStrc, pAssoStrc->Length+3);
-	RTPRINT(FIOCTL, IOCTL_BT_LOGO, ("PALCapabilities \n"));
-
-	RTPRINT(FIOCTL, IOCTL_BT_LOGO, (" TypeID = 0x%x,\n Length = 0x%x,\n Content = 0x0000\n",
-		pAssoStrc->TypeID,
-		pAssoStrc->Length));
-
-	return pAssoStrc->Length + 3;
-}
-
-static u16 bthci_AssocPreferredChannelList(struct rtw_adapter *padapter,
-					   void *pbuf, u8 EntryNum)
-{
-	struct bt_30info *pBTInfo;
-	struct amp_assoc_structure *pAssoStrc;
-	struct amp_pref_chnl_regulatory *pReg;
-	struct chnl_txpower_triple *pTriple;
-	char ctrString[3] = {'X', 'X', 'X'};
-	u32 len = 0;
-	u8 preferredChnl;
-
-	pBTInfo = GET_BT_INFO(padapter);
-	pAssoStrc = (struct amp_assoc_structure *)pbuf;
-	pReg = (struct amp_pref_chnl_regulatory *)&pAssoStrc->Data[3];
-
-	preferredChnl = bthci_GetLocalChannel(padapter);
-	pAssoStrc->TypeID = AMP_PREFERRED_CHANNEL_LIST;
-
-	/*  locale unknown */
-	memcpy(&pAssoStrc->Data[0], &ctrString[0], 3);
-	pReg->reXId = 201;
-	pReg->regulatoryClass = 254;
-	pReg->coverageClass = 0;
-	len += 6;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD | IOCTL_BT_LOGO), ("PREFERRED_CHNL_LIST\n"));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD | IOCTL_BT_LOGO), ("XXX, 201, 254, 0\n"));
-	/*  at the following, chnl 1~11 should be contained */
-	pTriple = (struct chnl_txpower_triple *)&pAssoStrc->Data[len];
-
-	/*  (1) if any wifi or bt HS connection exists */
-	if ((pBTInfo->BtAsocEntry[EntryNum].AMPRole == AMP_BTAP_CREATOR) ||
-	    (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE |
-			   WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE |
-			   WIFI_AP_STATE)) ||
-	    BTHCI_HsConnectionEstablished(padapter)) {
-		pTriple->FirstChnl = preferredChnl;
-		pTriple->NumChnls = 1;
-		pTriple->MaxTxPowerInDbm = 20;
-		len += 3;
-		RTPRINT(FIOCTL, (IOCTL_BT_HCICMD | IOCTL_BT_LOGO), ("First Channel = %d, Channel Num = %d, MaxDbm = %d\n",
-			pTriple->FirstChnl,
-			pTriple->NumChnls,
-			pTriple->MaxTxPowerInDbm));
-	}
-
-	pAssoStrc->Length = (u16)len;
-	RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD, ("AssocPreferredChannelList : \n"), pAssoStrc, pAssoStrc->Length+3);
-
-	return pAssoStrc->Length + 3;
-}
-
-static u16 bthci_AssocPALVer(struct rtw_adapter *padapter, void *pbuf)
-{
-	struct amp_assoc_structure *pAssoStrc = (struct amp_assoc_structure *)pbuf;
-	u8 *pu1Tmp;
-	u16	*pu2Tmp;
-
-	pAssoStrc->TypeID = AMP_80211_PAL_VISION;
-	pAssoStrc->Length = 0x5;
-	pu1Tmp = &pAssoStrc->Data[0];
-	*pu1Tmp = 0x1;	/*  PAL Version */
-	pu2Tmp = (u16 *)&pAssoStrc->Data[1];
-	*pu2Tmp = 0x5D;	/*  SIG Company identifier of 802.11 PAL vendor */
-	pu2Tmp = (u16 *)&pAssoStrc->Data[3];
-	*pu2Tmp = 0x1;	/*  PAL Sub-version specifier */
-
-	RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD_DETAIL, ("AssocPALVer : \n"), pAssoStrc, pAssoStrc->Length+3);
-	RTPRINT(FIOCTL, IOCTL_BT_LOGO, ("AssocPALVer \n"));
-
-	RTPRINT(FIOCTL, IOCTL_BT_LOGO, (" TypeID = 0x%x,\n Length = 0x%x,\n PAL Version = 0x01,\n PAL vendor = 0x01,\n PAL Sub-version specifier = 0x01\n",
-		pAssoStrc->TypeID,
-		pAssoStrc->Length));
-	return pAssoStrc->Length + 3;
-}
-
-static u8 bthci_CheckRfStateBeforeConnect(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo;
-	enum rt_rf_power_state		RfState;
-
-	pBTInfo = GET_BT_INFO(padapter);
-
-	RfState = padapter->pwrctrlpriv.rf_pwrstate;
-
-	if (RfState != rf_on) {
-		mod_timer(&pBTInfo->BTPsDisableTimer,
-			  jiffies + msecs_to_jiffies(50));
-		return false;
-	}
-	return true;
-}
-
-static void bthci_ResponderStartToScan(struct rtw_adapter *padapter)
-{
-}
-
-static u8 bthci_PhyLinkConnectionInProgress(struct rtw_adapter *padapter, u8 PhyLinkHandle)
-{
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->bPhyLinkInProgress &&
-		(pBtMgnt->BtCurrentPhyLinkhandle == PhyLinkHandle))
-		return true;
-	return false;
-}
-
-static void bthci_ResetFlowSpec(struct rtw_adapter *padapter, u8 EntryNum, u8 index)
-{
-	struct bt_30info *pBTinfo;
-
-	pBTinfo = GET_BT_INFO(padapter);
-
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].BtLogLinkhandle = 0;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].BtPhyLinkhandle = 0;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].bLLCompleteEventIsSet = false;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].bLLCancelCMDIsSetandComplete = false;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].BtTxFlowSpecID = 0;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].TxPacketCount = 0;
-
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Tx_Flow_Spec.Identifier = 0x01;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Tx_Flow_Spec.ServiceType = SERVICE_BEST_EFFORT;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Tx_Flow_Spec.MaximumSDUSize = 0xffff;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Tx_Flow_Spec.SDUInterArrivalTime = 0xffffffff;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Tx_Flow_Spec.AccessLatency = 0xffffffff;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Tx_Flow_Spec.FlushTimeout = 0xffffffff;
-
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Rx_Flow_Spec.Identifier = 0x01;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Rx_Flow_Spec.ServiceType = SERVICE_BEST_EFFORT;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Rx_Flow_Spec.MaximumSDUSize = 0xffff;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Rx_Flow_Spec.SDUInterArrivalTime = 0xffffffff;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Rx_Flow_Spec.AccessLatency = 0xffffffff;
-	pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[index].Rx_Flow_Spec.FlushTimeout = 0xffffffff;
-}
-
-static void bthci_ResetEntry(struct rtw_adapter *padapter, u8 EntryNum)
-{
-	struct bt_30info *pBTinfo;
-	struct bt_mgnt *pBtMgnt;
-	u8 j;
-
-	pBTinfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTinfo->BtMgnt;
-
-	pBTinfo->BtAsocEntry[EntryNum].bUsed = false;
-	pBTinfo->BtAsocEntry[EntryNum].BtCurrentState = HCI_STATE_DISCONNECTED;
-	pBTinfo->BtAsocEntry[EntryNum].BtNextState = HCI_STATE_DISCONNECTED;
-
-	pBTinfo->BtAsocEntry[EntryNum].AmpAsocCmdData.AMPAssocRemLen = 0;
-	pBTinfo->BtAsocEntry[EntryNum].AmpAsocCmdData.BtPhyLinkhandle = 0;
-	if (pBTinfo->BtAsocEntry[EntryNum].AmpAsocCmdData.AMPAssocfragment != NULL)
-		memset(pBTinfo->BtAsocEntry[EntryNum].AmpAsocCmdData.AMPAssocfragment, 0, TOTAL_ALLOCIATE_ASSOC_LEN);
-	pBTinfo->BtAsocEntry[EntryNum].AmpAsocCmdData.LenSoFar = 0;
-
-	pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKeyType = 0;
-	pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle = 0;
-	memset(pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKey, 0,
-	       pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKeyLen);
-	pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKeyLen = 0;
-
-	/* 0x640; 0.625ms*1600 = 1000ms, 0.625ms*16000 = 10000ms */
-	pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.LinkSuperversionTimeout = 0x3e80;
-
-	pBTinfo->BtAsocEntry[EntryNum].AMPRole = AMP_BTAP_NONE;
-
-	pBTinfo->BtAsocEntry[EntryNum].mAssoc = false;
-	pBTinfo->BtAsocEntry[EntryNum].b4waySuccess = false;
-
-	/*  Reset BT WPA */
-	pBTinfo->BtAsocEntry[EntryNum].KeyReplayCounter = 0;
-	pBTinfo->BtAsocEntry[EntryNum].BTWPAAuthState = STATE_WPA_AUTH_UNINITIALIZED;
-
-	pBTinfo->BtAsocEntry[EntryNum].bSendSupervisionPacket = false;
-	pBTinfo->BtAsocEntry[EntryNum].NoRxPktCnt = 0;
-	pBTinfo->BtAsocEntry[EntryNum].ShortRangeMode = 0;
-	pBTinfo->BtAsocEntry[EntryNum].rxSuvpPktCnt = 0;
-
-	for (j = 0; j < MAX_LOGICAL_LINK_NUM; j++)
-		bthci_ResetFlowSpec(padapter, EntryNum, j);
-
-	pBtMgnt->BTAuthCount = 0;
-	pBtMgnt->BTAsocCount = 0;
-	pBtMgnt->BTCurrentConnectType = BT_DISCONNECT;
-	pBtMgnt->BTReceiveConnectPkt = BT_DISCONNECT;
-
-	HALBT_RemoveKey(padapter, EntryNum);
-}
-
-static void bthci_RemoveEntryByEntryNum(struct rtw_adapter *padapter, u8 EntryNum)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	bthci_ResetEntry(padapter, EntryNum);
-
-	if (pBtMgnt->CurrentBTConnectionCnt > 0)
-		pBtMgnt->CurrentBTConnectionCnt--;
-
-	RTPRINT(FIOCTL, IOCTL_STATE, ("[BT Flag], CurrentBTConnectionCnt = %d!!\n",
-		pBtMgnt->CurrentBTConnectionCnt));
-
-	if (pBtMgnt->CurrentBTConnectionCnt > 0) {
-		pBtMgnt->BtOperationOn = true;
-	} else {
-		pBtMgnt->BtOperationOn = false;
-		RTPRINT(FIOCTL, IOCTL_STATE, ("[BT Flag], Bt Operation OFF!!\n"));
-	}
-
-	if (!pBtMgnt->BtOperationOn) {
-		del_timer_sync(&pBTInfo->BTHCIDiscardAclDataTimer);
-		del_timer_sync(&pBTInfo->BTBeaconTimer);
-		pBtMgnt->bStartSendSupervisionPkt = false;
-	}
-}
-
-static u8
-bthci_CommandCompleteHeader(
-	u8 *pbuf,
-	u16		OGF,
-	u16		OCF,
-	enum hci_status	status
-	)
-{
-	struct packet_irp_hcievent_data *PPacketIrpEvent = (struct packet_irp_hcievent_data *)pbuf;
-	u8 NumHCI_Comm = 0x1;
-
-	PPacketIrpEvent->EventCode = HCI_EVENT_COMMAND_COMPLETE;
-	PPacketIrpEvent->Data[0] = NumHCI_Comm;	/* packet # */
-	PPacketIrpEvent->Data[1] = HCIOPCODELOW(OCF, OGF);
-	PPacketIrpEvent->Data[2] = HCIOPCODEHIGHT(OCF, OGF);
-
-	if (OGF == OGF_EXTENSION) {
-		if (OCF == HCI_SET_RSSI_VALUE) {
-			RTPRINT(FIOCTL, (IOCTL_BT_EVENT_PERIODICAL),
-				("[BT event], CommandComplete, Num_HCI_Comm = 0x%x, Opcode = 0x%02x%02x, status = 0x%x, OGF = 0x%x, OCF = 0x%x\n",
-				NumHCI_Comm, (HCIOPCODEHIGHT(OCF, OGF)), (HCIOPCODELOW(OCF, OGF)), status, OGF, OCF));
-		} else {
-			RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_EXT),
-				("[BT event], CommandComplete, Num_HCI_Comm = 0x%x, Opcode = 0x%02x%02x, status = 0x%x, OGF = 0x%x, OCF = 0x%x\n",
-				NumHCI_Comm, (HCIOPCODEHIGHT(OCF, OGF)), (HCIOPCODELOW(OCF, OGF)), status, OGF, OCF));
-		}
-	} else {
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO),
-			("[BT event], CommandComplete, Num_HCI_Comm = 0x%x, Opcode = 0x%02x%02x, status = 0x%x, OGF = 0x%x, OCF = 0x%x\n",
-			NumHCI_Comm, (HCIOPCODEHIGHT(OCF, OGF)), (HCIOPCODELOW(OCF, OGF)), status, OGF, OCF));
-	}
-	return 3;
-}
-
-static u8 bthci_ExtensionEventHeaderRtk(u8 *pbuf, u8 extensionEvent)
-{
-	struct packet_irp_hcievent_data *PPacketIrpEvent = (struct packet_irp_hcievent_data *)pbuf;
-	PPacketIrpEvent->EventCode = HCI_EVENT_EXTENSION_RTK;
-	PPacketIrpEvent->Data[0] = extensionEvent;	/* extension event code */
-
-	return 1;
-}
-
-static enum rt_status
-bthci_IndicateEvent(
-	struct rtw_adapter *padapter,
-	void		*pEvntData,
-	u32		dataLen
-	)
-{
-	return PlatformIndicateBTEvent(padapter, pEvntData, dataLen);
-}
-
-static void
-bthci_EventWriteRemoteAmpAssoc(
-	struct rtw_adapter *padapter,
-	enum hci_status	status,
-	u8 PLHandle
-	)
-{
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_STATUS_PARAMETERS,
-		HCI_WRITE_REMOTE_AMP_ASSOC,
-		status);
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("PhyLinkHandle = 0x%x, status = %d\n", PLHandle, status));
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	pRetPar[1] = PLHandle;
-	len += 2;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-}
-
-static void
-bthci_EventEnhancedFlushComplete(
-	struct rtw_adapter *padapter,
-	u16					LLH
-	)
-{
-	u8 localBuf[4] = "";
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT, ("EventEnhancedFlushComplete, LLH = 0x%x\n", LLH));
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_ENHANCED_FLUSH_COMPLETE;
-	PPacketIrpEvent->Length = 2;
-	/* Logical link handle */
-	PPacketIrpEvent->Data[0] = TWOBYTE_LOWBYTE(LLH);
-	PPacketIrpEvent->Data[1] = TWOBYTE_HIGHTBYTE(LLH);
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 4);
-}
-
-static void
-bthci_EventShortRangeModeChangeComplete(
-	struct rtw_adapter *padapter,
-	enum hci_status				HciStatus,
-	u8 		ShortRangeState,
-	u8 		EntryNum
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[5] = "";
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	if (!(pBtHciInfo->BTEventMaskPage2 & EMP2_HCI_EVENT_SHORT_RANGE_MODE_CHANGE_COMPLETE)) {
-		RTPRINT(FIOCTL, IOCTL_BT_EVENT,
-			("[BT event], Short Range Mode Change Complete, Ignore to send this event due to event mask page 2\n"));
-		return;
-	}
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT, ("[BT event], Short Range Mode Change Complete, Status = %d\n , PLH = 0x%x\n, Short_Range_Mode_State = 0x%x\n",
-		HciStatus, pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle, ShortRangeState));
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_SHORT_RANGE_MODE_CHANGE_COMPLETE;
-	PPacketIrpEvent->Length = 3;
-	PPacketIrpEvent->Data[0] = HciStatus;
-	PPacketIrpEvent->Data[1] = pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle;
-	PPacketIrpEvent->Data[2] = ShortRangeState;
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 5);
-}
-
-static void bthci_EventSendFlowSpecModifyComplete(struct rtw_adapter *padapter,
-						  enum hci_status HciStatus,
-						  u16 logicHandle)
-{
-	u8 localBuf[5] = "";
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-
-	if (!(pBtHciInfo->BTEventMaskPage2 & EMP2_HCI_EVENT_FLOW_SPEC_MODIFY_COMPLETE)) {
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO),
-			("[BT event], Flow Spec Modify Complete, Ignore to send this event due to event mask page 2\n"));
-		return;
-	}
-	RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO),
-		("[BT event], Flow Spec Modify Complete, status = 0x%x, LLH = 0x%x\n", HciStatus, logicHandle));
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_FLOW_SPEC_MODIFY_COMPLETE;
-	PPacketIrpEvent->Length = 3;
-
-	PPacketIrpEvent->Data[0] = HciStatus;
-	/* Logical link handle */
-	PPacketIrpEvent->Data[1] = TWOBYTE_LOWBYTE(logicHandle);
-	PPacketIrpEvent->Data[2] = TWOBYTE_HIGHTBYTE(logicHandle);
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 5);
-}
-
-static void
-bthci_EventExtWifiScanNotify(
-	struct rtw_adapter *padapter,
-	u8 			scanType
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	u8 len = 0;
-	u8 localBuf[7] = "";
-	u8 *pRetPar;
-	u8 *pu1Temp;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	if (!pBtMgnt->BtOperationOn)
-		return;
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_ExtensionEventHeaderRtk(&localBuf[0], HCI_EVENT_EXT_WIFI_SCAN_NOTIFY);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pu1Temp = (u8 *)&pRetPar[0];
-	*pu1Temp = scanType;
-	len += 1;
-
-	PPacketIrpEvent->Length = len;
-
-	if (bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2) == RT_STATUS_SUCCESS) {
-		RTPRINT(FIOCTL, IOCTL_BT_EVENT, ("[BT event], Wifi scan notify, scan type = %d\n",
-			scanType));
-	}
-}
-
-static void
-bthci_EventAMPReceiverReport(
-	struct rtw_adapter *padapter,
-	u8 Reason
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-
-	if (pBtHciInfo->bTestNeedReport) {
-		u8 localBuf[20] = "";
-		u32	*pu4Temp;
-		u16	*pu2Temp;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), (" HCI_EVENT_AMP_RECEIVER_REPORT\n"));
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-		PPacketIrpEvent->EventCode = HCI_EVENT_AMP_RECEIVER_REPORT;
-		PPacketIrpEvent->Length = 2;
-
-		PPacketIrpEvent->Data[0] = pBtHciInfo->TestCtrType;
-
-		PPacketIrpEvent->Data[1] = Reason;
-
-		pu4Temp = (u32 *)&PPacketIrpEvent->Data[2];
-		*pu4Temp = pBtHciInfo->TestEventType;
-
-		pu2Temp = (u16 *)&PPacketIrpEvent->Data[6];
-		*pu2Temp = pBtHciInfo->TestNumOfFrame;
-
-		pu2Temp = (u16 *)&PPacketIrpEvent->Data[8];
-		*pu2Temp = pBtHciInfo->TestNumOfErrFrame;
-
-		pu4Temp = (u32 *)&PPacketIrpEvent->Data[10];
-		*pu4Temp = pBtHciInfo->TestNumOfBits;
-
-		pu4Temp = (u32 *)&PPacketIrpEvent->Data[14];
-		*pu4Temp = pBtHciInfo->TestNumOfErrBits;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, 20);
-
-		/* Return to Idel state with RX and TX off. */
-
-	}
-
-	pBtHciInfo->TestNumOfFrame = 0x00;
-}
-
-static void
-bthci_EventChannelSelected(
-	struct rtw_adapter *padapter,
-	u8 	EntryNum
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[3] = "";
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	if (!(pBtHciInfo->BTEventMaskPage2 & EMP2_HCI_EVENT_CHANNEL_SELECT)) {
-		RTPRINT(FIOCTL, IOCTL_BT_EVENT,
-			("[BT event], Channel Selected, Ignore to send this event due to event mask page 2\n"));
-		return;
-	}
-
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT|IOCTL_STATE,
-		("[BT event], Channel Selected, PhyLinkHandle %d\n",
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle));
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_CHANNEL_SELECT;
-	PPacketIrpEvent->Length = 1;
-	PPacketIrpEvent->Data[0] = pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle;
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 3);
-}
-
-static void
-bthci_EventDisconnectPhyLinkComplete(
-	struct rtw_adapter *padapter,
-	enum hci_status				HciStatus,
-	enum hci_status				Reason,
-	u8 		EntryNum
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[5] = "";
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	if (!(pBtHciInfo->BTEventMaskPage2 & EMP2_HCI_EVENT_DISCONNECT_PHY_LINK_COMPLETE)) {
-		RTPRINT(FIOCTL, IOCTL_BT_EVENT,
-			("[BT event], Disconnect Physical Link Complete, Ignore to send this event due to event mask page 2\n"));
-		return;
-	}
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT,
-		("[BT event], Disconnect Physical Link Complete, Status = 0x%x, PLH = 0x%x Reason = 0x%x\n",
-		HciStatus, pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle, Reason));
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_DISCONNECT_PHY_LINK_COMPLETE;
-	PPacketIrpEvent->Length = 3;
-	PPacketIrpEvent->Data[0] = HciStatus;
-	PPacketIrpEvent->Data[1] = pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle;
-	PPacketIrpEvent->Data[2] = Reason;
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 5);
-}
-
-static void
-bthci_EventPhysicalLinkComplete(
-	struct rtw_adapter *padapter,
-	enum hci_status				HciStatus,
-	u8 		EntryNum,
-	u8 		PLHandle
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-	u8 localBuf[4] = "";
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u8 PL_handle;
-
-	pBtMgnt->bPhyLinkInProgress = false;
-	pBtDbg->dbgHciInfo.hciCmdPhyLinkStatus = HciStatus;
-	if (!(pBtHciInfo->BTEventMaskPage2 & EMP2_HCI_EVENT_PHY_LINK_COMPLETE)) {
-		RTPRINT(FIOCTL, IOCTL_BT_EVENT,
-			("[BT event], Physical Link Complete, Ignore to send this event due to event mask page 2\n"));
-		return;
-	}
-
-	if (EntryNum == 0xff) {
-		/*  connection not started yet, just use the input physical link handle to response. */
-		PL_handle = PLHandle;
-	} else {
-		/*  connection is under progress, use the phy link handle we recorded. */
-		PL_handle  = pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle;
-		pBTInfo->BtAsocEntry[EntryNum].bNeedPhysLinkCompleteEvent = false;
-	}
-
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT, ("[BT event], Physical Link Complete, Status = 0x%x PhyLinkHandle = 0x%x\n", HciStatus,
-		PL_handle));
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_PHY_LINK_COMPLETE;
-	PPacketIrpEvent->Length = 2;
-
-	PPacketIrpEvent->Data[0] = HciStatus;
-	PPacketIrpEvent->Data[1] = PL_handle;
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 4);
-
-}
-
-static void
-bthci_EventCommandStatus(
-	struct rtw_adapter *padapter,
-	u8 		OGF,
-	u16					OCF,
-	enum hci_status				HciStatus
-	)
-{
-
-	u8 localBuf[6] = "";
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u8 Num_Hci_Comm = 0x1;
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT,
-		("[BT event], CommandStatus, Opcode = 0x%02x%02x, OGF = 0x%x,  OCF = 0x%x, Status = 0x%x, Num_HCI_COMM = 0x%x\n",
-		(HCIOPCODEHIGHT(OCF, OGF)), (HCIOPCODELOW(OCF, OGF)), OGF, OCF, HciStatus, Num_Hci_Comm));
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_COMMAND_STATUS;
-	PPacketIrpEvent->Length = 4;
-	PPacketIrpEvent->Data[0] = HciStatus;	/* current pending */
-	PPacketIrpEvent->Data[1] = Num_Hci_Comm;	/* packet # */
-	PPacketIrpEvent->Data[2] = HCIOPCODELOW(OCF, OGF);
-	PPacketIrpEvent->Data[3] = HCIOPCODEHIGHT(OCF, OGF);
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 6);
-
-}
-
-static void
-bthci_EventLogicalLinkComplete(
-	struct rtw_adapter *padapter,
-	enum hci_status				HciStatus,
-	u8 		PhyLinkHandle,
-	u16					LogLinkHandle,
-	u8 		LogLinkIndex,
-	u8 		EntryNum
-	)
-{
-/*PMGNT_INFO pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[7] = "";
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	if (!(pBtHciInfo->BTEventMaskPage2 & EMP2_HCI_EVENT_LOGICAL_LINK_COMPLETE)) {
-		RTPRINT(FIOCTL, IOCTL_BT_EVENT,
-			("[BT event], Logical Link Complete, Ignore to send this event due to event mask page 2\n"));
-		return;
-	}
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT, ("[BT event], Logical Link Complete, PhyLinkHandle = 0x%x,  LogLinkHandle = 0x%x, Status = 0x%x\n",
-		PhyLinkHandle, LogLinkHandle, HciStatus));
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_LOGICAL_LINK_COMPLETE;
-	PPacketIrpEvent->Length = 5;
-
-	PPacketIrpEvent->Data[0] = HciStatus;/* status code */
-	/* Logical link handle */
-	PPacketIrpEvent->Data[1] = TWOBYTE_LOWBYTE(LogLinkHandle);
-	PPacketIrpEvent->Data[2] = TWOBYTE_HIGHTBYTE(LogLinkHandle);
-	/* Physical link handle */
-	PPacketIrpEvent->Data[3] = TWOBYTE_LOWBYTE(PhyLinkHandle);
-	/* corresponding Tx flow spec ID */
-	if (HciStatus == HCI_STATUS_SUCCESS) {
-		PPacketIrpEvent->Data[4] =
-			pBTInfo->BtAsocEntry[EntryNum].LogLinkCmdData[LogLinkIndex].Tx_Flow_Spec.Identifier;
-	} else {
-		PPacketIrpEvent->Data[4] = 0x0;
-	}
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 7);
-}
-
-static void
-bthci_EventDisconnectLogicalLinkComplete(
-	struct rtw_adapter *padapter,
-	enum hci_status				HciStatus,
-	u16					LogLinkHandle,
-	enum hci_status				Reason
-	)
-{
-	u8 localBuf[6] = "";
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	if (!(pBtHciInfo->BTEventMaskPage2 & EMP2_HCI_EVENT_DISCONNECT_LOGICAL_LINK_COMPLETE)) {
-		RTPRINT(FIOCTL, IOCTL_BT_EVENT, ("[BT event], Disconnect Logical Link Complete, Ignore to send this event due to event mask page 2\n"));
-		return;
-	}
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT, ("[BT event], Disconnect Logical Link Complete, Status = 0x%x, LLH = 0x%x Reason = 0x%x\n", HciStatus, LogLinkHandle, Reason));
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_DISCONNECT_LOGICAL_LINK_COMPLETE;
-	PPacketIrpEvent->Length = 4;
-
-	PPacketIrpEvent->Data[0] = HciStatus;
-	/* Logical link handle */
-	PPacketIrpEvent->Data[1] = TWOBYTE_LOWBYTE(LogLinkHandle);
-	PPacketIrpEvent->Data[2] = TWOBYTE_HIGHTBYTE(LogLinkHandle);
-	/* Disconnect reason */
-	PPacketIrpEvent->Data[3] = Reason;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 6);
-}
-
-static void
-bthci_EventFlushOccurred(
-	struct rtw_adapter *padapter,
-	u16					LogLinkHandle
-	)
-{
-	u8 localBuf[4] = "";
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	RTPRINT(FIOCTL, IOCTL_BT_EVENT, ("bthci_EventFlushOccurred(), LLH = 0x%x\n", LogLinkHandle));
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	PPacketIrpEvent->EventCode = HCI_EVENT_FLUSH_OCCRUED;
-	PPacketIrpEvent->Length = 2;
-	/* Logical link handle */
-	PPacketIrpEvent->Data[0] = TWOBYTE_LOWBYTE(LogLinkHandle);
-	PPacketIrpEvent->Data[1] = TWOBYTE_HIGHTBYTE(LogLinkHandle);
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, 4);
-}
-
-static enum hci_status
-bthci_BuildPhysicalLink(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd,
-	u16	OCF
-)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	u8 EntryNum, PLH;
-
-	/* Send HCI Command status event to AMP. */
-	bthci_EventCommandStatus(padapter,
-			LINK_CONTROL_COMMANDS,
-			OCF,
-			HCI_STATUS_SUCCESS);
-
-	PLH = *((u8 *)pHciCmd->Data);
-
-	/*  Check if resource or bt connection is under progress, if yes, reject the link creation. */
-	if (!bthci_AddEntry(padapter)) {
-		status = HCI_STATUS_CONNECT_RJT_LIMIT_RESOURCE;
-		bthci_EventPhysicalLinkComplete(padapter, status, INVALID_ENTRY_NUM, PLH);
-		return status;
-	}
-
-	EntryNum = pBtMgnt->CurrentConnectEntryNum;
-	pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle = PLH;
-	pBtMgnt->BtCurrentPhyLinkhandle = PLH;
-
-	if (pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.AMPAssocfragment == NULL) {
-		RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Create/Accept PhysicalLink, AMP controller is busy\n"));
-		status = HCI_STATUS_CONTROLLER_BUSY;
-		bthci_EventPhysicalLinkComplete(padapter, status, INVALID_ENTRY_NUM, PLH);
-		return status;
-	}
-
-	/*  Record Key and the info */
-	pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKeyLen = (*((u8 *)pHciCmd->Data+1));
-	pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKeyType = (*((u8 *)pHciCmd->Data+2));
-	memcpy(pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKey,
-		(((u8 *)pHciCmd->Data+3)), pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKeyLen);
-	memcpy(pBTInfo->BtAsocEntry[EntryNum].PMK, pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKey, PMK_LEN);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("BuildPhysicalLink, EntryNum = %d, PLH = 0x%x  KeyLen = 0x%x, KeyType = 0x%x\n",
-		EntryNum, pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle,
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKeyLen,
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKeyType));
-	RTPRINT_DATA(FIOCTL, (IOCTL_BT_LOGO|IOCTL_BT_HCICMD), ("BtAMPKey\n"), pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKey,
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtAMPKeyLen);
-	RTPRINT_DATA(FIOCTL, (IOCTL_BT_LOGO|IOCTL_BT_HCICMD), ("PMK\n"), pBTInfo->BtAsocEntry[EntryNum].PMK,
-		PMK_LEN);
-
-	if (OCF == HCI_CREATE_PHYSICAL_LINK) {
-		/* These macros require braces */
-		BTHCI_SM_WITH_INFO(padapter, HCI_STATE_DISCONNECTED, STATE_CMD_CREATE_PHY_LINK, EntryNum);
-	} else if (OCF == HCI_ACCEPT_PHYSICAL_LINK) {
-		BTHCI_SM_WITH_INFO(padapter, HCI_STATE_DISCONNECTED, STATE_CMD_ACCEPT_PHY_LINK, EntryNum);
-	}
-
-	return status;
-}
-
-static void
-bthci_BuildLogicalLink(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd,
-	u16 OCF
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTinfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTinfo->BtMgnt;
-	u8 PhyLinkHandle, EntryNum;
-	static u16 AssignLogHandle = 1;
-
-	struct hci_flow_spec	TxFlowSpec;
-	struct hci_flow_spec	RxFlowSpec;
-	u32	MaxSDUSize, ArriveTime, Bandwidth;
-
-	PhyLinkHandle = *((u8 *)pHciCmd->Data);
-
-	EntryNum = bthci_GetCurrentEntryNum(padapter, PhyLinkHandle);
-
-	memcpy(&TxFlowSpec,
-		&pHciCmd->Data[1], sizeof(struct hci_flow_spec));
-	memcpy(&RxFlowSpec,
-		&pHciCmd->Data[17], sizeof(struct hci_flow_spec));
-
-	MaxSDUSize = TxFlowSpec.MaximumSDUSize;
-	ArriveTime = TxFlowSpec.SDUInterArrivalTime;
-
-	if (bthci_CheckLogLinkBehavior(padapter, TxFlowSpec) && bthci_CheckLogLinkBehavior(padapter, RxFlowSpec))
-		Bandwidth = BTTOTALBANDWIDTH;
-	else if (MaxSDUSize == 0xffff && ArriveTime == 0xffffffff)
-		Bandwidth = BTTOTALBANDWIDTH;
-	else
-		Bandwidth = MaxSDUSize*8*1000/(ArriveTime+244);
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD,
-		("BuildLogicalLink, PhyLinkHandle = 0x%x, MaximumSDUSize = 0x%x, SDUInterArrivalTime = 0x%x, Bandwidth = 0x%x\n",
-		PhyLinkHandle, MaxSDUSize, ArriveTime, Bandwidth));
-
-	if (EntryNum == 0xff) {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Invalid Physical Link handle = 0x%x, status = HCI_STATUS_UNKNOW_CONNECT_ID, return\n", PhyLinkHandle));
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-
-		/* When we receive Create/Accept logical link command, we should send command status event first. */
-		bthci_EventCommandStatus(padapter,
-			LINK_CONTROL_COMMANDS,
-			OCF,
-			status);
-		return;
-	}
-
-	if (!pBtMgnt->bLogLinkInProgress) {
-		if (bthci_PhyLinkConnectionInProgress(padapter, PhyLinkHandle)) {
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Physical link connection in progress, status = HCI_STATUS_CMD_DISALLOW, return\n"));
-			status = HCI_STATUS_CMD_DISALLOW;
-
-			pBtMgnt->bPhyLinkInProgressStartLL = true;
-			/* When we receive Create/Accept logical link command, we should send command status event first. */
-			bthci_EventCommandStatus(padapter,
-				LINK_CONTROL_COMMANDS,
-				OCF,
-				status);
-
-			return;
-		}
-
-		if (Bandwidth > BTTOTALBANDWIDTH) {
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("status = HCI_STATUS_QOS_REJECT, Bandwidth = 0x%x, return\n", Bandwidth));
-			status = HCI_STATUS_QOS_REJECT;
-
-			/* When we receive Create/Accept logical link command, we should send command status event first. */
-			bthci_EventCommandStatus(padapter,
-				LINK_CONTROL_COMMANDS,
-				OCF,
-				status);
-		} else {
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("status = HCI_STATUS_SUCCESS\n"));
-			status = HCI_STATUS_SUCCESS;
-
-			/* When we receive Create/Accept logical link command, we should send command status event first. */
-			bthci_EventCommandStatus(padapter,
-				LINK_CONTROL_COMMANDS,
-				OCF,
-				status);
-
-		}
-
-		if (pBTinfo->BtAsocEntry[EntryNum].BtCurrentState != HCI_STATE_CONNECTED) {
-			bthci_EventLogicalLinkComplete(padapter,
-				HCI_STATUS_CMD_DISALLOW, 0, 0, 0, EntryNum);
-		} else {
-			u8 i, find = 0;
-
-			pBtMgnt->bLogLinkInProgress = true;
-
-			/*  find an unused logical link index and copy the data */
-			for (i = 0; i < MAX_LOGICAL_LINK_NUM; i++) {
-				if (pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].BtLogLinkhandle == 0) {
-					enum hci_status LogCompEventstatus = HCI_STATUS_SUCCESS;
-
-					pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].BtPhyLinkhandle = *((u8 *)pHciCmd->Data);
-					pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].BtLogLinkhandle = AssignLogHandle;
-					RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("BuildLogicalLink, EntryNum = %d, physical link handle = 0x%x, logical link handle = 0x%x\n",
-						EntryNum, pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle,
-								  pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].BtLogLinkhandle));
-					memcpy(&pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].Tx_Flow_Spec,
-						&TxFlowSpec, sizeof(struct hci_flow_spec));
-					memcpy(&pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].Rx_Flow_Spec,
-						&RxFlowSpec, sizeof(struct hci_flow_spec));
-
-					pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].bLLCompleteEventIsSet = false;
-
-					if (pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].bLLCancelCMDIsSetandComplete)
-						LogCompEventstatus = HCI_STATUS_UNKNOW_CONNECT_ID;
-					bthci_EventLogicalLinkComplete(padapter,
-						LogCompEventstatus,
-						pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].BtPhyLinkhandle,
-						pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].BtLogLinkhandle, i, EntryNum);
-
-					pBTinfo->BtAsocEntry[EntryNum].LogLinkCmdData[i].bLLCompleteEventIsSet = true;
-
-					find = 1;
-					pBtMgnt->BtCurrentLogLinkhandle = AssignLogHandle;
-					AssignLogHandle++;
-					break;
-				}
-			}
-
-			if (!find) {
-				bthci_EventLogicalLinkComplete(padapter,
-					HCI_STATUS_CONNECT_RJT_LIMIT_RESOURCE, 0, 0, 0, EntryNum);
-			}
-			pBtMgnt->bLogLinkInProgress = false;
-		}
-	} else {
-		bthci_EventLogicalLinkComplete(padapter,
-			HCI_STATUS_CONTROLLER_BUSY, 0, 0, 0, EntryNum);
-	}
-
-}
-
-static void
-bthci_StartBeaconAndConnect(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd,
-	u8 CurrentAssocNum
-	)
-{
-/*PMGNT_INFO pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("StartBeaconAndConnect, CurrentAssocNum =%d, AMPRole =%d\n",
-		CurrentAssocNum,
-		pBTInfo->BtAsocEntry[CurrentAssocNum].AMPRole));
-
-	if (!pBtMgnt->CheckChnlIsSuit) {
-		bthci_EventPhysicalLinkComplete(padapter, HCI_STATUS_CONNECT_REJ_NOT_SUIT_CHNL_FOUND, CurrentAssocNum, INVALID_PL_HANDLE);
-		bthci_RemoveEntryByEntryNum(padapter, CurrentAssocNum);
-		return;
-	}
-
-	if (pBTInfo->BtAsocEntry[CurrentAssocNum].AMPRole == AMP_BTAP_CREATOR) {
-		snprintf((char *)pBTInfo->BtAsocEntry[CurrentAssocNum].BTSsidBuf, 32,
-			 "AMP-%pMF", padapter->eeprompriv.mac_addr);
-	} else if (pBTInfo->BtAsocEntry[CurrentAssocNum].AMPRole == AMP_BTAP_JOINER) {
-		snprintf((char *)pBTInfo->BtAsocEntry[CurrentAssocNum].BTSsidBuf, 32,
-			 "AMP-%pMF", pBTInfo->BtAsocEntry[CurrentAssocNum].BTRemoteMACAddr);
-	}
-
-	FillOctetString(pBTInfo->BtAsocEntry[CurrentAssocNum].BTSsid, pBTInfo->BtAsocEntry[CurrentAssocNum].BTSsidBuf, 21);
-	pBTInfo->BtAsocEntry[CurrentAssocNum].BTSsid.Length = 21;
-
-	/* To avoid set the start ap or connect twice, or the original connection will be disconnected. */
-	if (!pBtMgnt->bBTConnectInProgress) {
-		pBtMgnt->bBTConnectInProgress = true;
-		RTPRINT(FIOCTL, IOCTL_STATE, ("[BT Flag], BT Connect in progress ON!!\n"));
-		BTHCI_SM_WITH_INFO(padapter, HCI_STATE_STARTING, STATE_CMD_MAC_START_COMPLETE, CurrentAssocNum);
-
-		/*  20100325 Joseph: Check RF ON/OFF. */
-		/*  If RF OFF, it reschedule connecting operation after 50ms. */
-		if (!bthci_CheckRfStateBeforeConnect(padapter))
-			return;
-
-		if (pBTInfo->BtAsocEntry[CurrentAssocNum].AMPRole == AMP_BTAP_CREATOR) {
-			/* These macros need braces */
-			BTHCI_SM_WITH_INFO(padapter, HCI_STATE_CONNECTING, STATE_CMD_MAC_CONNECT_COMPLETE, CurrentAssocNum);
-		} else if (pBTInfo->BtAsocEntry[CurrentAssocNum].AMPRole == AMP_BTAP_JOINER) {
-			bthci_ResponderStartToScan(padapter);
-		}
-	}
-	RT_PRINT_STR(_module_rtl871x_mlme_c_, _drv_notice_,
-		     "StartBeaconAndConnect, SSID:\n",
-		     pBTInfo->BtAsocEntry[pBtMgnt->CurrentConnectEntryNum].BTSsid.Octet,
-		     pBTInfo->BtAsocEntry[pBtMgnt->CurrentConnectEntryNum].BTSsid.Length);
-}
-
-static void bthci_ResetBtMgnt(struct bt_mgnt *pBtMgnt)
-{
-	pBtMgnt->BtOperationOn = false;
-	pBtMgnt->bBTConnectInProgress = false;
-	pBtMgnt->bLogLinkInProgress = false;
-	pBtMgnt->bPhyLinkInProgress = false;
-	pBtMgnt->bPhyLinkInProgressStartLL = false;
-	pBtMgnt->DisconnectEntryNum = 0xff;
-	pBtMgnt->bStartSendSupervisionPkt = false;
-	pBtMgnt->JoinerNeedSendAuth = false;
-	pBtMgnt->CurrentBTConnectionCnt = 0;
-	pBtMgnt->BTCurrentConnectType = BT_DISCONNECT;
-	pBtMgnt->BTReceiveConnectPkt = BT_DISCONNECT;
-	pBtMgnt->BTAuthCount = 0;
-	pBtMgnt->btLogoTest = 0;
-}
-
-static void bthci_ResetBtHciInfo(struct bt_hci_info *pBtHciInfo)
-{
-	pBtHciInfo->BTEventMask = 0;
-	pBtHciInfo->BTEventMaskPage2 = 0;
-	pBtHciInfo->ConnAcceptTimeout =  10000;
-	pBtHciInfo->PageTimeout  =  0x30;
-	pBtHciInfo->LocationDomainAware = 0x0;
-	pBtHciInfo->LocationDomain = 0x5858;
-	pBtHciInfo->LocationDomainOptions = 0x58;
-	pBtHciInfo->LocationOptions = 0x0;
-	pBtHciInfo->FlowControlMode = 0x1;	/*  0:Packet based data flow control mode(BR/EDR), 1: Data block based data flow control mode(AMP). */
-
-	pBtHciInfo->enFlush_LLH = 0;
-	pBtHciInfo->FLTO_LLH = 0;
-
-	/* Test command only */
-	pBtHciInfo->bTestIsEnd = true;
-	pBtHciInfo->bInTestMode = false;
-	pBtHciInfo->bTestNeedReport = false;
-	pBtHciInfo->TestScenario = 0xff;
-	pBtHciInfo->TestReportInterval = 0x01;
-	pBtHciInfo->TestCtrType = 0x5d;
-	pBtHciInfo->TestEventType = 0x00;
-	pBtHciInfo->TestNumOfFrame = 0;
-	pBtHciInfo->TestNumOfErrFrame = 0;
-	pBtHciInfo->TestNumOfBits = 0;
-	pBtHciInfo->TestNumOfErrBits = 0;
-}
-
-static void bthci_ResetBtSec(struct rtw_adapter *padapter, struct bt_security *pBtSec)
-{
-/*PMGNT_INFO	pMgntInfo = &padapter->MgntInfo; */
-
-	/*  Set BT used HW or SW encrypt !! */
-	if (GET_HAL_DATA(padapter)->bBTMode)
-		pBtSec->bUsedHwEncrypt = true;
-	else
-		pBtSec->bUsedHwEncrypt = false;
-	RT_TRACE(_module_rtl871x_security_c_, _drv_info_,
-		 "%s: bUsedHwEncrypt =%d\n", __func__, pBtSec->bUsedHwEncrypt);
-
-	pBtSec->RSNIE.Octet = pBtSec->RSNIEBuf;
-}
-
-static void bthci_ResetBtExtInfo(struct bt_mgnt *pBtMgnt)
-{
-	u8 i;
-
-	for (i = 0; i < MAX_BT_ASOC_ENTRY_NUM; i++) {
-		pBtMgnt->ExtConfig.linkInfo[i].ConnectHandle = 0;
-		pBtMgnt->ExtConfig.linkInfo[i].IncomingTrafficMode = 0;
-		pBtMgnt->ExtConfig.linkInfo[i].OutgoingTrafficMode = 0;
-		pBtMgnt->ExtConfig.linkInfo[i].BTProfile = BT_PROFILE_NONE;
-		pBtMgnt->ExtConfig.linkInfo[i].BTCoreSpec = BT_SPEC_2_1_EDR;
-		pBtMgnt->ExtConfig.linkInfo[i].BT_RSSI = 0;
-		pBtMgnt->ExtConfig.linkInfo[i].TrafficProfile = BT_PROFILE_NONE;
-		pBtMgnt->ExtConfig.linkInfo[i].linkRole = BT_LINK_MASTER;
-	}
-
-	pBtMgnt->ExtConfig.CurrentConnectHandle = 0;
-	pBtMgnt->ExtConfig.CurrentIncomingTrafficMode = 0;
-	pBtMgnt->ExtConfig.CurrentOutgoingTrafficMode = 0;
-	pBtMgnt->ExtConfig.MIN_BT_RSSI = 0;
-	pBtMgnt->ExtConfig.NumberOfHandle = 0;
-	pBtMgnt->ExtConfig.NumberOfSCO = 0;
-	pBtMgnt->ExtConfig.CurrentBTStatus = 0;
-	pBtMgnt->ExtConfig.HCIExtensionVer = 0;
-
-	pBtMgnt->ExtConfig.bManualControl = false;
-	pBtMgnt->ExtConfig.bBTBusy = false;
-	pBtMgnt->ExtConfig.bBTA2DPBusy = false;
-}
-
-static enum hci_status bthci_CmdReset(struct rtw_adapter *_padapter, u8 bNeedSendEvent)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct rtw_adapter *padapter;
-/*PMGNT_INFO pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_hci_info *pBtHciInfo;
-	struct bt_security *pBtSec;
-	struct bt_dgb *pBtDbg;
-	u8 i;
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("bthci_CmdReset()\n"));
-
-	padapter = GetDefaultAdapter(_padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtHciInfo = &pBTInfo->BtHciInfo;
-	pBtSec = &pBTInfo->BtSec;
-	pBtDbg = &pBTInfo->BtDbg;
-
-	pBTInfo->padapter = padapter;
-
-	for (i = 0; i < MAX_BT_ASOC_ENTRY_NUM; i++)
-		bthci_ResetEntry(padapter, i);
-
-	bthci_ResetBtMgnt(pBtMgnt);
-	bthci_ResetBtHciInfo(pBtHciInfo);
-	bthci_ResetBtSec(padapter, pBtSec);
-
-	pBtMgnt->BTChannel = BT_Default_Chnl;
-	pBtMgnt->CheckChnlIsSuit = true;
-
-	pBTInfo->BTBeaconTmrOn = false;
-
-	pBtMgnt->bCreateSpportQos = true;
-
-	del_timer_sync(&pBTInfo->BTHCIDiscardAclDataTimer);
-	del_timer_sync(&pBTInfo->BTBeaconTimer);
-
-	HALBT_SetRtsCtsNoLenLimit(padapter);
-	/*  */
-	/*  Maybe we need to take care Group != AES case !! */
-	/*  now we Pairwise and Group all used AES !! */
-
-	bthci_ResetBtExtInfo(pBtMgnt);
-
-	/* send command complete event here when all data are received. */
-	if (bNeedSendEvent) {
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_SET_EVENT_MASK_COMMAND,
-			HCI_RESET,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWriteRemoteAMPAssoc(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-	u8 CurrentAssocNum;
-	u8 PhyLinkHandle;
-
-	pBtDbg->dbgHciInfo.hciCmdCntWriteRemoteAmpAssoc++;
-	PhyLinkHandle = *((u8 *)pHciCmd->Data);
-	CurrentAssocNum = bthci_GetCurrentEntryNum(padapter, PhyLinkHandle);
-
-	if (CurrentAssocNum == 0xff) {
-		RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("WriteRemoteAMPAssoc, No such Handle in the Entry\n"));
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-		bthci_EventWriteRemoteAmpAssoc(padapter, status, PhyLinkHandle);
-		return status;
-	}
-
-	if (pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.AMPAssocfragment == NULL) {
-		RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("WriteRemoteAMPAssoc, AMP controller is busy\n"));
-		status = HCI_STATUS_CONTROLLER_BUSY;
-		bthci_EventWriteRemoteAmpAssoc(padapter, status, PhyLinkHandle);
-		return status;
-	}
-
-	pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.BtPhyLinkhandle = PhyLinkHandle;/* u8 *)pHciCmd->Data); */
-	pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.LenSoFar = *((u16 *)((u8 *)pHciCmd->Data+1));
-	pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.AMPAssocRemLen = *((u16 *)((u8 *)pHciCmd->Data+3));
-
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("WriteRemoteAMPAssoc, LenSoFar = 0x%x, AssocRemLen = 0x%x\n",
-		pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.LenSoFar,
-		pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.AMPAssocRemLen));
-
-	RTPRINT_DATA(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO),
-		     ("WriteRemoteAMPAssoc fragment \n"),
-		     pHciCmd->Data,
-		     pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.AMPAssocRemLen+5);
-	if ((pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.AMPAssocRemLen) > MAX_AMP_ASSOC_FRAG_LEN) {
-		memcpy(((u8 *)pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.AMPAssocfragment+(pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.LenSoFar*(sizeof(u8)))),
-			(u8 *)pHciCmd->Data+5,
-			MAX_AMP_ASSOC_FRAG_LEN);
-	} else {
-		memcpy((u8 *)(pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.AMPAssocfragment)+(pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.LenSoFar*(sizeof(u8))),
-			((u8 *)pHciCmd->Data+5),
-			(pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.AMPAssocRemLen));
-
-		RTPRINT_DATA(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), "WriteRemoteAMPAssoc :\n",
-			pHciCmd->Data+5, pBTInfo->BtAsocEntry[CurrentAssocNum].AmpAsocCmdData.AMPAssocRemLen);
-
-		if (!bthci_GetAssocInfo(padapter, CurrentAssocNum))
-			status = HCI_STATUS_INVALID_HCI_CMD_PARA_VALUE;
-
-		bthci_EventWriteRemoteAmpAssoc(padapter, status, PhyLinkHandle);
-
-		bthci_StartBeaconAndConnect(padapter, pHciCmd, CurrentAssocNum);
-	}
-
-	return status;
-}
-
-/* 7.3.13 */
-static enum hci_status bthci_CmdReadConnectionAcceptTimeout(struct rtw_adapter *padapter)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-/*PMGNT_INFO		pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[8] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp;
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_READ_CONNECTION_ACCEPT_TIMEOUT,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	pu2Temp = (u16 *)&pRetPar[1];		/*  Conn_Accept_Timeout */
-	*pu2Temp = pBtHciInfo->ConnAcceptTimeout;
-	len += 3;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-/* 7.3.14 */
-static enum hci_status
-bthci_CmdWriteConnectionAcceptTimeout(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u16	*pu2Temp;
-	u8 localBuf[6] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	pu2Temp = (u16 *)&pHciCmd->Data[0];
-	pBtHciInfo->ConnAcceptTimeout = *pu2Temp;
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_DETAIL, ("ConnAcceptTimeout = 0x%x",
-		pBtHciInfo->ConnAcceptTimeout));
-
-	/* send command complete event here when all data are received. */
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_WRITE_CONNECTION_ACCEPT_TIMEOUT,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	len += 1;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdReadPageTimeout(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[8] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp;
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_READ_PAGE_TIMEOUT,
-		status);
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Read PageTimeout = 0x%x\n", pBtHciInfo->PageTimeout));
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	pu2Temp = (u16 *)&pRetPar[1];		/*  Page_Timeout */
-	*pu2Temp = pBtHciInfo->PageTimeout;
-	len += 3;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWritePageTimeout(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u16	*pu2Temp;
-
-	pu2Temp = (u16 *)&pHciCmd->Data[0];
-	pBtHciInfo->PageTimeout = *pu2Temp;
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Write PageTimeout = 0x%x\n",
-		pBtHciInfo->PageTimeout));
-
-	/* send command complete event here when all data are received. */
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_SET_EVENT_MASK_COMMAND,
-			HCI_WRITE_PAGE_TIMEOUT,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdReadLinkSupervisionTimeout(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTinfo = GET_BT_INFO(padapter);
-	u8 physicalLinkHandle, EntryNum;
-
-	physicalLinkHandle = *((u8 *)pHciCmd->Data);
-
-	EntryNum = bthci_GetCurrentEntryNum(padapter, physicalLinkHandle);
-
-	if (EntryNum == 0xff) {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("ReadLinkSupervisionTimeout, No such Handle in the Entry\n"));
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-		return status;
-	}
-
-	if (pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle != physicalLinkHandle)
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-
-	{
-		u8 localBuf[10] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-		u16 *pu2Temp;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_SET_EVENT_MASK_COMMAND,
-			HCI_READ_LINK_SUPERVISION_TIMEOUT,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;
-		pRetPar[1] = pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle;
-		pRetPar[2] = 0;
-		pu2Temp = (u16 *)&pRetPar[3];		/*  Conn_Accept_Timeout */
-		*pu2Temp = pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.LinkSuperversionTimeout;
-		len += 5;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWriteLinkSupervisionTimeout(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTinfo = GET_BT_INFO(padapter);
-	u8 physicalLinkHandle, EntryNum;
-
-	physicalLinkHandle = *((u8 *)pHciCmd->Data);
-
-	EntryNum = bthci_GetCurrentEntryNum(padapter, physicalLinkHandle);
-
-	if (EntryNum == 0xff) {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("WriteLinkSupervisionTimeout, No such Handle in the Entry\n"));
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-	} else {
-		if (pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle != physicalLinkHandle) {
-			status = HCI_STATUS_UNKNOW_CONNECT_ID;
-		} else {
-			pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.LinkSuperversionTimeout = *((u16 *)(((u8 *)pHciCmd->Data)+2));
-			RTPRINT(FIOCTL, IOCTL_STATE, ("BT Write LinkSuperversionTimeout[%d] = 0x%x\n",
-				EntryNum, pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.LinkSuperversionTimeout));
-		}
-	}
-
-	{
-		u8 localBuf[8] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_SET_EVENT_MASK_COMMAND,
-			HCI_WRITE_LINK_SUPERVISION_TIMEOUT,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;
-		pRetPar[1] = pBTinfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle;
-		pRetPar[2] = 0;
-		len += 3;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdEnhancedFlush(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTinfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTinfo->BtHciInfo;
-	u16		logicHandle;
-	u8 Packet_Type;
-
-	logicHandle = *((u16 *)&pHciCmd->Data[0]);
-	Packet_Type = pHciCmd->Data[2];
-
-	if (Packet_Type != 0)
-		status = HCI_STATUS_INVALID_HCI_CMD_PARA_VALUE;
-	else
-		pBtHciInfo->enFlush_LLH = logicHandle;
-
-	if (bthci_DiscardTxPackets(padapter, pBtHciInfo->enFlush_LLH))
-		bthci_EventFlushOccurred(padapter, pBtHciInfo->enFlush_LLH);
-
-	/*  should send command status event */
-	bthci_EventCommandStatus(padapter,
-			OGF_SET_EVENT_MASK_COMMAND,
-			HCI_ENHANCED_FLUSH,
-			status);
-
-	if (pBtHciInfo->enFlush_LLH) {
-		bthci_EventEnhancedFlushComplete(padapter, pBtHciInfo->enFlush_LLH);
-		pBtHciInfo->enFlush_LLH = 0;
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdReadLogicalLinkAcceptTimeout(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-/*PMGNT_INFO		pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[8] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp;
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_READ_LOGICAL_LINK_ACCEPT_TIMEOUT,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;
-
-	pu2Temp = (u16 *)&pRetPar[1];		/*  Conn_Accept_Timeout */
-	*pu2Temp = pBtHciInfo->LogicalAcceptTimeout;
-	len += 3;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWriteLogicalLinkAcceptTimeout(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-/*PMGNT_INFO		pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[6] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	pBtHciInfo->LogicalAcceptTimeout = *((u16 *)pHciCmd->Data);
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_WRITE_LOGICAL_LINK_ACCEPT_TIMEOUT,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;
-
-	len += 1;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	return status;
-}
-
-static enum hci_status
-bthci_CmdSetEventMask(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-/*PMGNT_INFO		pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 *pu8Temp;
-	u8 localBuf[6] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	pu8Temp = (u8 *)&pHciCmd->Data[0];
-	pBtHciInfo->BTEventMask = *pu8Temp;
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_DETAIL, ("BTEventMask = 0x%"i64fmt"x\n",
-		pBtHciInfo->BTEventMask));
-
-	/* send command complete event here when all data are received. */
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_SET_EVENT_MASK,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	len += 1;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-/*  7.3.69 */
-static enum hci_status
-bthci_CmdSetEventMaskPage2(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 *pu8Temp;
-	u8 localBuf[6] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	pu8Temp = (u8 *)&pHciCmd->Data[0];
-	pBtHciInfo->BTEventMaskPage2 = *pu8Temp;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("BTEventMaskPage2 = 0x%"i64fmt"x\n",
-		pBtHciInfo->BTEventMaskPage2));
-
-	/* send command complete event here when all data are received. */
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_SET_EVENT_MASK_PAGE_2,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	len += 1;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdReadLocationData(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[12] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp;
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_READ_LOCATION_DATA,
-		status);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("DomainAware = 0x%x\n", pBtHciInfo->LocationDomainAware));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Domain = 0x%x\n", pBtHciInfo->LocationDomain));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("DomainOptions = 0x%x\n", pBtHciInfo->LocationDomainOptions));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Options = 0x%x\n", pBtHciInfo->LocationOptions));
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;
-
-	pRetPar[1] = pBtHciInfo->LocationDomainAware;	/* 0x0;	 Location_Domain_Aware */
-	pu2Temp = (u16 *)&pRetPar[2];					/*  Location_Domain */
-	*pu2Temp = pBtHciInfo->LocationDomain;		/* 0x5858; */
-	pRetPar[4] = pBtHciInfo->LocationDomainOptions;	/* 0x58;	Location_Domain_Options */
-	pRetPar[5] = pBtHciInfo->LocationOptions;		/* 0x0;	Location_Options */
-	len += 6;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWriteLocationData(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u16	*pu2Temp;
-	u8 localBuf[6] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	pBtHciInfo->LocationDomainAware = pHciCmd->Data[0];
-	pu2Temp = (u16 *)&pHciCmd->Data[1];
-	pBtHciInfo->LocationDomain = *pu2Temp;
-	pBtHciInfo->LocationDomainOptions = pHciCmd->Data[3];
-	pBtHciInfo->LocationOptions = pHciCmd->Data[4];
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("DomainAware = 0x%x\n", pBtHciInfo->LocationDomainAware));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Domain = 0x%x\n", pBtHciInfo->LocationDomain));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("DomainOptions = 0x%x\n", pBtHciInfo->LocationDomainOptions));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Options = 0x%x\n", pBtHciInfo->LocationOptions));
-
-	/* send command complete event here when all data are received. */
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_WRITE_LOCATION_DATA,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	len += 1;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdReadFlowControlMode(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[7] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_READ_FLOW_CONTROL_MODE,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;
-	pRetPar[1] = pBtHciInfo->FlowControlMode;	/*  Flow Control Mode */
-	len += 2;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWriteFlowControlMode(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[6] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	pBtHciInfo->FlowControlMode = pHciCmd->Data[0];
-
-	/* send command complete event here when all data are received. */
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_WRITE_FLOW_CONTROL_MODE,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	len += 1;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdReadBestEffortFlushTimeout(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTinfo = GET_BT_INFO(padapter);
-	u16 i, j, logicHandle;
-	u32 BestEffortFlushTimeout = 0xffffffff;
-	u8 find = 0;
-
-	logicHandle = *((u16 *)pHciCmd->Data);
-	/*  find an matched logical link index and copy the data */
-	for (j = 0; j < MAX_BT_ASOC_ENTRY_NUM; j++) {
-		for (i = 0; i < MAX_LOGICAL_LINK_NUM; i++) {
-			if (pBTinfo->BtAsocEntry[j].LogLinkCmdData[i].BtLogLinkhandle == logicHandle) {
-				BestEffortFlushTimeout = pBTinfo->BtAsocEntry[j].LogLinkCmdData[i].BestEffortFlushTimeout;
-				find = 1;
-				break;
-			}
-		}
-	}
-
-	if (!find)
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-
-	{
-		u8 localBuf[10] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-		u32 *pu4Temp;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_SET_EVENT_MASK_COMMAND,
-			HCI_READ_BEST_EFFORT_FLUSH_TIMEOUT,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;
-		pu4Temp = (u32 *)&pRetPar[1];	/*  Best_Effort_Flush_Timeout */
-		*pu4Temp = BestEffortFlushTimeout;
-		len += 5;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWriteBestEffortFlushTimeout(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTinfo = GET_BT_INFO(padapter);
-	u16 i, j, logicHandle;
-	u32 BestEffortFlushTimeout = 0xffffffff;
-	u8 find = 0;
-
-	logicHandle = *((u16 *)pHciCmd->Data);
-	BestEffortFlushTimeout = *((u32 *)(pHciCmd->Data+1));
-
-	/*  find an matched logical link index and copy the data */
-	for (j = 0; j < MAX_BT_ASOC_ENTRY_NUM; j++) {
-		for (i = 0; i < MAX_LOGICAL_LINK_NUM; i++) {
-			if (pBTinfo->BtAsocEntry[j].LogLinkCmdData[i].BtLogLinkhandle == logicHandle) {
-				pBTinfo->BtAsocEntry[j].LogLinkCmdData[i].BestEffortFlushTimeout = BestEffortFlushTimeout;
-				find = 1;
-				break;
-			}
-		}
-	}
-
-	if (!find)
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_SET_EVENT_MASK_COMMAND,
-			HCI_WRITE_BEST_EFFORT_FLUSH_TIMEOUT,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-	return status;
-}
-
-static enum hci_status
-bthci_CmdShortRangeMode(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	u8 PhyLinkHandle, EntryNum, ShortRangeMode;
-
-	PhyLinkHandle = pHciCmd->Data[0];
-	ShortRangeMode = pHciCmd->Data[1];
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("PLH = 0x%x, Short_Range_Mode = 0x%x\n", PhyLinkHandle, ShortRangeMode));
-
-	EntryNum = bthci_GetCurrentEntryNum(padapter, PhyLinkHandle);
-	if (EntryNum != 0xff) {
-		pBTInfo->BtAsocEntry[EntryNum].ShortRangeMode = ShortRangeMode;
-	} else {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("No such PLH(0x%x)\n", PhyLinkHandle));
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-	}
-
-	bthci_EventCommandStatus(padapter,
-			OGF_SET_EVENT_MASK_COMMAND,
-			HCI_SHORT_RANGE_MODE,
-			status);
-
-	bthci_EventShortRangeModeChangeComplete(padapter, status, ShortRangeMode, EntryNum);
-
-	return status;
-}
-
-static enum hci_status bthci_CmdReadLocalSupportedCommands(struct rtw_adapter *padapter)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar, *pSupportedCmds;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	/*  send command complete event here when all data are received. */
-	PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_INFORMATIONAL_PARAMETERS,
-		HCI_READ_LOCAL_SUPPORTED_COMMANDS,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	len += 1;
-	pSupportedCmds = &pRetPar[1];
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[5]= 0xc0\nBit [6]= Set Event Mask, [7]= Reset\n"));
-	pSupportedCmds[5] = 0xc0;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[6]= 0x01\nBit [0]= Set Event Filter\n"));
-	pSupportedCmds[6] = 0x01;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[7]= 0x0c\nBit [2]= Read Connection Accept Timeout, [3]= Write Connection Accept Timeout\n"));
-	pSupportedCmds[7] = 0x0c;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[10]= 0x80\nBit [7]= Host Number Of Completed Packets\n"));
-	pSupportedCmds[10] = 0x80;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[11]= 0x03\nBit [0]= Read Link Supervision Timeout, [1]= Write Link Supervision Timeout\n"));
-	pSupportedCmds[11] = 0x03;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[14]= 0xa8\nBit [3]= Read Local Version Information, [5]= Read Local Supported Features, [7]= Read Buffer Size\n"));
-	pSupportedCmds[14] = 0xa8;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[15]= 0x1c\nBit [2]= Read Failed Contact Count, [3]= Reset Failed Contact Count, [4]= Get Link Quality\n"));
-	pSupportedCmds[15] = 0x1c;
-	/* pSupportedCmds[16] = 0x04; */
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[19]= 0x40\nBit [6]= Enhanced Flush\n"));
-	pSupportedCmds[19] = 0x40;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[21]= 0xff\nBit [0]= Create Physical Link, [1]= Accept Physical Link, [2]= Disconnect Physical Link, [3]= Create Logical Link\n"));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("	[4]= Accept Logical Link, [5]= Disconnect Logical Link, [6]= Logical Link Cancel, [7]= Flow Spec Modify\n"));
-	pSupportedCmds[21] = 0xff;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[22]= 0xff\nBit [0]= Read Logical Link Accept Timeout, [1]= Write Logical Link Accept Timeout, [2]= Set Event Mask Page 2, [3]= Read Location Data\n"));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("	[4]= Write Location Data, [5]= Read Local AMP Info, [6]= Read Local AMP_ASSOC, [7]= Write Remote AMP_ASSOC\n"));
-	pSupportedCmds[22] = 0xff;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[23]= 0x07\nBit [0]= Read Flow Control Mode, [1]= Write Flow Control Mode, [2]= Read Data Block Size\n"));
-	pSupportedCmds[23] = 0x07;
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD|IOCTL_BT_LOGO), ("Octet[24]= 0x1c\nBit [2]= Read Best Effort Flush Timeout, [3]= Write Best Effort Flush Timeout, [4]= Short Range Mode\n"));
-	pSupportedCmds[24] = 0x1c;
-	len += 64;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status bthci_CmdReadLocalSupportedFeatures(struct rtw_adapter *padapter)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	/* send command complete event here when all data are received. */
-	PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_INFORMATIONAL_PARAMETERS,
-		HCI_READ_LOCAL_SUPPORTED_FEATURES,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	len += 9;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	return status;
-}
-
-static enum hci_status bthci_CmdReadLocalAMPAssoc(struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-	u8 PhyLinkHandle, EntryNum;
-
-	pBtDbg->dbgHciInfo.hciCmdCntReadLocalAmpAssoc++;
-	PhyLinkHandle = *((u8 *)pHciCmd->Data);
-	EntryNum = bthci_GetCurrentEntryNum(padapter, PhyLinkHandle);
-
-	if ((EntryNum == 0xff) && PhyLinkHandle != 0) {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("ReadLocalAMPAssoc, EntryNum = %d  !!!!!, physical link handle = 0x%x\n",
-		EntryNum, PhyLinkHandle));
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-	} else if (pBtMgnt->bPhyLinkInProgressStartLL) {
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-		pBtMgnt->bPhyLinkInProgressStartLL = false;
-	} else {
-		pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.BtPhyLinkhandle = *((u8 *)pHciCmd->Data);
-		pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.LenSoFar = *((u16 *)((u8 *)pHciCmd->Data+1));
-		pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.MaxRemoteASSOCLen = *((u16 *)((u8 *)pHciCmd->Data+3));
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_DETAIL, ("ReadLocalAMPAssoc, LenSoFar =%d, MaxRemoteASSOCLen =%d\n",
-			pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.LenSoFar,
-			pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.MaxRemoteASSOCLen));
-	}
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("ReadLocalAMPAssoc, EntryNum = %d  !!!!!, physical link handle = 0x%x, LengthSoFar = %x  \n",
-		EntryNum, PhyLinkHandle, pBTInfo->BtAsocEntry[EntryNum].AmpAsocCmdData.LenSoFar));
-
-	/* send command complete event here when all data are received. */
-	{
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		/* PVOID buffer = padapter->IrpHCILocalbuf.Ptr; */
-		u8 localBuf[TmpLocalBufSize] = "";
-		u16	*pRemainLen;
-		u32	totalLen = 0;
-		u16	typeLen = 0, remainLen = 0, ret_index = 0;
-		u8 *pRetPar;
-
-		PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-		/* PPacketIrpEvent = (struct packet_irp_hcievent_data *)(buffer); */
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		totalLen += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_STATUS_PARAMETERS,
-			HCI_READ_LOCAL_AMP_ASSOC,
-			status);
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("ReadLocalAMPAssoc, Remaining_Len =%d  \n", remainLen));
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[totalLen];
-		pRetPar[0] = status;		/* status */
-		pRetPar[1] = *((u8 *)pHciCmd->Data);
-		pRemainLen = (u16 *)&pRetPar[2];	/* AMP_ASSOC_Remaining_Length */
-		totalLen += 4;	/* 0]~[3] */
-		ret_index = 4;
-
-		typeLen = bthci_AssocMACAddr(padapter, &pRetPar[ret_index]);
-		totalLen += typeLen;
-		remainLen += typeLen;
-		ret_index += typeLen;
-		typeLen = bthci_AssocPreferredChannelList(padapter, &pRetPar[ret_index], EntryNum);
-		totalLen += typeLen;
-		remainLen += typeLen;
-		ret_index += typeLen;
-		typeLen = bthci_PALCapabilities(padapter, &pRetPar[ret_index]);
-		totalLen += typeLen;
-		remainLen += typeLen;
-		ret_index += typeLen;
-		typeLen = bthci_AssocPALVer(padapter, &pRetPar[ret_index]);
-		totalLen += typeLen;
-		remainLen += typeLen;
-		PPacketIrpEvent->Length = (u8)totalLen;
-		*pRemainLen = remainLen;	/*  AMP_ASSOC_Remaining_Length */
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("ReadLocalAMPAssoc, Remaining_Len =%d  \n", remainLen));
-		RTPRINT_DATA(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("AMP_ASSOC_fragment : \n"), PPacketIrpEvent->Data, totalLen);
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, totalLen+2);
-	}
-
-	return status;
-}
-
-static enum hci_status bthci_CmdReadFailedContactCounter(struct rtw_adapter *padapter,
-		       struct packet_irp_hcicmd_data *pHciCmd)
-{
-
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 handle;
-
-	handle = *((u16 *)pHciCmd->Data);
-	/* send command complete event here when all data are received. */
-	PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_STATUS_PARAMETERS,
-		HCI_READ_FAILED_CONTACT_COUNTER,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	pRetPar[1] = TWOBYTE_LOWBYTE(handle);
-	pRetPar[2] = TWOBYTE_HIGHTBYTE(handle);
-	pRetPar[3] = TWOBYTE_LOWBYTE(pBtHciInfo->FailContactCount);
-	pRetPar[4] = TWOBYTE_HIGHTBYTE(pBtHciInfo->FailContactCount);
-	len += 5;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdResetFailedContactCounter(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status		status = HCI_STATUS_SUCCESS;
-/*PMGNT_INFO		pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u16		handle;
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-	handle = *((u16 *)pHciCmd->Data);
-	pBtHciInfo->FailContactCount = 0;
-
-	/* send command complete event here when all data are received. */
-	PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-	/* PPacketIrpEvent = (struct packet_irp_hcievent_data *)(buffer); */
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_STATUS_PARAMETERS,
-		HCI_RESET_FAILED_CONTACT_COUNTER,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	pRetPar[1] = TWOBYTE_LOWBYTE(handle);
-	pRetPar[2] = TWOBYTE_HIGHTBYTE(handle);
-	len += 3;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	return status;
-}
-
-/*  */
-/*  BT 3.0+HS [Vol 2] 7.4.1 */
-/*  */
-static enum hci_status
-bthci_CmdReadLocalVersionInformation(
-	struct rtw_adapter *padapter
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	/* send command complete event here when all data are received. */
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp;
-
-	PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_INFORMATIONAL_PARAMETERS,
-		HCI_READ_LOCAL_VERSION_INFORMATION,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	pRetPar[1] = 0x05;			/*  HCI_Version */
-	pu2Temp = (u16 *)&pRetPar[2];		/*  HCI_Revision */
-	*pu2Temp = 0x0001;
-	pRetPar[4] = 0x05;			/*  LMP/PAL_Version */
-	pu2Temp = (u16 *)&pRetPar[5];		/*  Manufacturer_Name */
-	*pu2Temp = 0x005d;
-	pu2Temp = (u16 *)&pRetPar[7];		/*  LMP/PAL_Subversion */
-	*pu2Temp = 0x0001;
-	len += 9;
-	PPacketIrpEvent->Length = len;
-
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("LOCAL_VERSION_INFORMATION\n"));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("Status  %x\n", status));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("HCI_Version = 0x05\n"));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("HCI_Revision = 0x0001\n"));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("LMP/PAL_Version = 0x05\n"));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("Manufacturer_Name = 0x0001\n"));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("LMP/PAL_Subversion = 0x0001\n"));
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-/* 7.4.7 */
-static enum hci_status bthci_CmdReadDataBlockSize(struct rtw_adapter *padapter)
-{
-	enum hci_status			status = HCI_STATUS_SUCCESS;
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp;
-
-	PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_INFORMATIONAL_PARAMETERS,
-		HCI_READ_DATA_BLOCK_SIZE,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = HCI_STATUS_SUCCESS;	/* status */
-	pu2Temp = (u16 *)&pRetPar[1];		/*  Max_ACL_Data_Packet_Length */
-	*pu2Temp = Max80211PALPDUSize;
-
-	pu2Temp = (u16 *)&pRetPar[3];		/*  Data_Block_Length */
-	*pu2Temp = Max80211PALPDUSize;
-	pu2Temp = (u16 *)&pRetPar[5];		/*  Total_Num_Data_Blocks */
-	*pu2Temp = BTTotalDataBlockNum;
-	len += 7;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-/*  7.4.5 */
-static enum hci_status bthci_CmdReadBufferSize(struct rtw_adapter *padapter)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp;
-
-	PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-	/* PPacketIrpEvent = (struct packet_irp_hcievent_data *)(buffer); */
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_INFORMATIONAL_PARAMETERS,
-		HCI_READ_BUFFER_SIZE,
-		status);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Synchronous_Data_Packet_Length = 0x%x\n", BTSynDataPacketLength));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Total_Num_ACL_Data_Packets = 0x%x\n", BTTotalDataBlockNum));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("Total_Num_Synchronous_Data_Packets = 0x%x\n", BTTotalDataBlockNum));
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	pu2Temp = (u16 *)&pRetPar[1];		/*  HC_ACL_Data_Packet_Length */
-	*pu2Temp = Max80211PALPDUSize;
-
-	pRetPar[3] = BTSynDataPacketLength;	/*  HC_Synchronous_Data_Packet_Length */
-	pu2Temp = (u16 *)&pRetPar[4];		/*  HC_Total_Num_ACL_Data_Packets */
-	*pu2Temp = BTTotalDataBlockNum;
-	pu2Temp = (u16 *)&pRetPar[6];		/*  HC_Total_Num_Synchronous_Data_Packets */
-	*pu2Temp = BTTotalDataBlockNum;
-	len += 8;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status bthci_CmdReadLocalAMPInfo(struct rtw_adapter *padapter)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-	struct pwrctrl_priv *ppwrctrl = &padapter->pwrctrlpriv;
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp;
-	u32 *pu4Temp;
-	u32	TotalBandwidth = BTTOTALBANDWIDTH, MaxBandGUBandwidth = BTMAXBANDGUBANDWIDTH;
-	u8 ControlType = 0x01, AmpStatus = 0x01;
-	u32	MaxFlushTimeout = 10000, BestEffortFlushTimeout = 5000;
-	u16 MaxPDUSize = Max80211PALPDUSize, PalCap = 0x1, AmpAssocLen = Max80211AMPASSOCLen, MinLatency = 20;
-
-	if ((ppwrctrl->rfoff_reason & RF_CHANGE_BY_HW) ||
-	    (ppwrctrl->rfoff_reason & RF_CHANGE_BY_SW)) {
-		AmpStatus = AMP_STATUS_NO_CAPACITY_FOR_BT;
-	}
-
-	PlatformZeroMemory(&localBuf[0], TmpLocalBufSize);
-	/* PPacketIrpEvent = (struct packet_irp_hcievent_data *)(buffer); */
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_STATUS_PARAMETERS,
-		HCI_READ_LOCAL_AMP_INFO,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;			/* status */
-	pRetPar[1] = AmpStatus;			/*  AMP_Status */
-	pu4Temp = (u32 *)&pRetPar[2];		/*  Total_Bandwidth */
-	*pu4Temp = TotalBandwidth;		/* 0x19bfcc00;0x7530; */
-	pu4Temp = (u32 *)&pRetPar[6];		/*  Max_Guaranteed_Bandwidth */
-	*pu4Temp = MaxBandGUBandwidth;		/* 0x19bfcc00;0x4e20; */
-	pu4Temp = (u32 *)&pRetPar[10];		/*  Min_Latency */
-	*pu4Temp = MinLatency;			/* 150; */
-	pu4Temp = (u32 *)&pRetPar[14];		/*  Max_PDU_Size */
-	*pu4Temp = MaxPDUSize;
-	pRetPar[18] = ControlType;		/*  Controller_Type */
-	pu2Temp = (u16 *)&pRetPar[19];		/*  PAL_Capabilities */
-	*pu2Temp = PalCap;
-	pu2Temp = (u16 *)&pRetPar[21];		/*  AMP_ASSOC_Length */
-	*pu2Temp = AmpAssocLen;
-	pu4Temp = (u32 *)&pRetPar[23];		/*  Max_Flush_Timeout */
-	*pu4Temp = MaxFlushTimeout;
-	pu4Temp = (u32 *)&pRetPar[27];		/*  Best_Effort_Flush_Timeout */
-	*pu4Temp = BestEffortFlushTimeout;
-	len += 31;
-	PPacketIrpEvent->Length = len;
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("AmpStatus = 0x%x\n",
-		AmpStatus));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("TotalBandwidth = 0x%x, MaxBandGUBandwidth = 0x%x, MinLatency = 0x%x, \n MaxPDUSize = 0x%x, ControlType = 0x%x\n",
-		TotalBandwidth, MaxBandGUBandwidth, MinLatency, MaxPDUSize, ControlType));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("PalCap = 0x%x, AmpAssocLen = 0x%x, MaxFlushTimeout = 0x%x, BestEffortFlushTimeout = 0x%x\n",
-		PalCap, AmpAssocLen, MaxFlushTimeout, BestEffortFlushTimeout));
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	return status;
-}
-
-static enum hci_status
-bthci_CmdCreatePhysicalLink(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-
-	pBtDbg->dbgHciInfo.hciCmdCntCreatePhyLink++;
-
-	return bthci_BuildPhysicalLink(padapter,
-		pHciCmd, HCI_CREATE_PHYSICAL_LINK);
-}
-
-static enum hci_status
-bthci_CmdReadLinkQuality(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status			status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	u16				PLH;
-	u8 	EntryNum, LinkQuality = 0x55;
-
-	PLH = *((u16 *)&pHciCmd->Data[0]);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("PLH = 0x%x\n", PLH));
-
-	EntryNum = bthci_GetCurrentEntryNum(padapter, (u8)PLH);
-	if (EntryNum == 0xff) {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("No such PLH(0x%x)\n", PLH));
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-	}
-
-	{
-		u8 localBuf[11] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_STATUS_PARAMETERS,
-			HCI_READ_LINK_QUALITY,
-			status);
-
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, (" PLH = 0x%x\n Link Quality = 0x%x\n", PLH, LinkQuality));
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;			/* status */
-		*((u16 *)&pRetPar[1]) = pBTInfo->BtAsocEntry[EntryNum].PhyLinkCmdData.BtPhyLinkhandle;	/*  Handle */
-		pRetPar[3] = 0x55;	/* Link Quailty */
-		len += 4;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdCreateLogicalLink(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-
-	pBtDbg->dbgHciInfo.hciCmdCntCreateLogLink++;
-
-	bthci_BuildLogicalLink(padapter, pHciCmd,
-		HCI_CREATE_LOGICAL_LINK);
-
-	return HCI_STATUS_SUCCESS;
-}
-
-static enum hci_status
-bthci_CmdAcceptLogicalLink(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-
-	pBtDbg->dbgHciInfo.hciCmdCntAcceptLogLink++;
-
-	bthci_BuildLogicalLink(padapter, pHciCmd,
-		HCI_ACCEPT_LOGICAL_LINK);
-
-	return HCI_STATUS_SUCCESS;
-}
-
-static enum hci_status
-bthci_CmdDisconnectLogicalLink(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-/*PMGNT_INFO pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTinfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTinfo->BtMgnt;
-	struct bt_dgb *pBtDbg = &pBTinfo->BtDbg;
-	u16	logicHandle;
-	u8 i, j, find = 0, LogLinkCount = 0;
-
-	pBtDbg->dbgHciInfo.hciCmdCntDisconnectLogLink++;
-
-	logicHandle = *((u16 *)pHciCmd->Data);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("DisconnectLogicalLink, logicHandle = 0x%x\n", logicHandle));
-
-	/*  find an created logical link index and clear the data */
-	for (j = 0; j < MAX_BT_ASOC_ENTRY_NUM; j++) {
-		for (i = 0; i < MAX_LOGICAL_LINK_NUM; i++) {
-			if (pBTinfo->BtAsocEntry[j].LogLinkCmdData[i].BtLogLinkhandle == logicHandle) {
-				RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("DisconnectLogicalLink, logicHandle is matched  0x%x\n", logicHandle));
-				bthci_ResetFlowSpec(padapter, j, i);
-				find = 1;
-				pBtMgnt->DisconnectEntryNum = j;
-				break;
-			}
-		}
-	}
-
-	if (!find)
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-
-	/*  To check each */
-	for (i = 0; i < MAX_LOGICAL_LINK_NUM; i++) {
-		if (pBTinfo->BtAsocEntry[pBtMgnt->DisconnectEntryNum].LogLinkCmdData[i].BtLogLinkhandle != 0)
-			LogLinkCount++;
-	}
-
-	/* When we receive Create logical link command, we should send command status event first. */
-	bthci_EventCommandStatus(padapter,
-			LINK_CONTROL_COMMANDS,
-			HCI_DISCONNECT_LOGICAL_LINK,
-			status);
-	/*  */
-	/* When we determines the logical link is established, we should send command complete event. */
-	/*  */
-	if (status == HCI_STATUS_SUCCESS) {
-		bthci_EventDisconnectLogicalLinkComplete(padapter, status,
-			logicHandle, HCI_STATUS_CONNECT_TERMINATE_LOCAL_HOST);
-	}
-
-	if (LogLinkCount == 0)
-		mod_timer(&pBTinfo->BTDisconnectPhyLinkTimer,
-			  jiffies + msecs_to_jiffies(100));
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdLogicalLinkCancel(struct rtw_adapter *padapter,
-			   struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTinfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTinfo->BtMgnt;
-	u8 CurrentEntryNum, CurrentLogEntryNum;
-
-	u8 physicalLinkHandle, TxFlowSpecID, i;
-	u16	CurrentLogicalHandle;
-
-	physicalLinkHandle = *((u8 *)pHciCmd->Data);
-	TxFlowSpecID = *(((u8 *)pHciCmd->Data)+1);
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("LogicalLinkCancel, physicalLinkHandle = 0x%x, TxFlowSpecID = 0x%x\n",
-		physicalLinkHandle, TxFlowSpecID));
-
-	CurrentEntryNum = pBtMgnt->CurrentConnectEntryNum;
-	CurrentLogicalHandle = pBtMgnt->BtCurrentLogLinkhandle;
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("CurrentEntryNum = 0x%x, CurrentLogicalHandle = 0x%x\n",
-		CurrentEntryNum, CurrentLogicalHandle));
-
-	CurrentLogEntryNum = 0xff;
-	for (i = 0; i < MAX_LOGICAL_LINK_NUM; i++) {
-		if ((CurrentLogicalHandle == pBTinfo->BtAsocEntry[CurrentEntryNum].LogLinkCmdData[i].BtLogLinkhandle) &&
-			(physicalLinkHandle == pBTinfo->BtAsocEntry[CurrentEntryNum].LogLinkCmdData[i].BtPhyLinkhandle)) {
-			CurrentLogEntryNum = i;
-			break;
-		}
-	}
-
-	if (CurrentLogEntryNum == 0xff) {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("LogicalLinkCancel, CurrentLogEntryNum == 0xff !!!!\n"));
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-		return status;
-	} else {
-		if (pBTinfo->BtAsocEntry[CurrentEntryNum].LogLinkCmdData[CurrentLogEntryNum].bLLCompleteEventIsSet) {
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("LogicalLinkCancel, LLCompleteEventIsSet!!!!\n"));
-			status = HCI_STATUS_ACL_CONNECT_EXISTS;
-		}
-	}
-
-	{
-		u8 localBuf[8] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			LINK_CONTROL_COMMANDS,
-			HCI_LOGICAL_LINK_CANCEL,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-		pRetPar[1] = pBTinfo->BtAsocEntry[CurrentEntryNum].LogLinkCmdData[CurrentLogEntryNum].BtPhyLinkhandle;
-		pRetPar[2] = pBTinfo->BtAsocEntry[CurrentEntryNum].LogLinkCmdData[CurrentLogEntryNum].BtTxFlowSpecID;
-		len += 3;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	pBTinfo->BtAsocEntry[CurrentEntryNum].LogLinkCmdData[CurrentLogEntryNum].bLLCancelCMDIsSetandComplete = true;
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdFlowSpecModify(struct rtw_adapter *padapter,
-			struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-/*PMGNT_INFO pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTinfo = GET_BT_INFO(padapter);
-	u8 i, j, find = 0;
-	u16 logicHandle;
-
-	logicHandle = *((u16 *)pHciCmd->Data);
-	/*  find an matched logical link index and copy the data */
-	for (j = 0; j < MAX_BT_ASOC_ENTRY_NUM; j++) {
-		for (i = 0; i < MAX_LOGICAL_LINK_NUM; i++) {
-			if (pBTinfo->BtAsocEntry[j].LogLinkCmdData[i].BtLogLinkhandle == logicHandle) {
-				memcpy(&pBTinfo->BtAsocEntry[j].LogLinkCmdData[i].Tx_Flow_Spec,
-					&pHciCmd->Data[2], sizeof(struct hci_flow_spec));
-				memcpy(&pBTinfo->BtAsocEntry[j].LogLinkCmdData[i].Rx_Flow_Spec,
-					&pHciCmd->Data[18], sizeof(struct hci_flow_spec));
-
-				bthci_CheckLogLinkBehavior(padapter, pBTinfo->BtAsocEntry[j].LogLinkCmdData[i].Tx_Flow_Spec);
-				find = 1;
-				break;
-			}
-		}
-	}
-	RTPRINT(FIOCTL, IOCTL_BT_LOGO, ("FlowSpecModify, LLH = 0x%x, \n", logicHandle));
-
-	/* When we receive Flow Spec Modify command, we should send command status event first. */
-	bthci_EventCommandStatus(padapter,
-		LINK_CONTROL_COMMANDS,
-		HCI_FLOW_SPEC_MODIFY,
-		HCI_STATUS_SUCCESS);
-
-	if (!find)
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-
-	bthci_EventSendFlowSpecModifyComplete(padapter, status, logicHandle);
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdAcceptPhysicalLink(struct rtw_adapter *padapter,
-			    struct packet_irp_hcicmd_data *pHciCmd)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-
-	pBtDbg->dbgHciInfo.hciCmdCntAcceptPhyLink++;
-
-	return bthci_BuildPhysicalLink(padapter,
-		pHciCmd, HCI_ACCEPT_PHYSICAL_LINK);
-}
-
-static enum hci_status
-bthci_CmdDisconnectPhysicalLink(struct rtw_adapter *padapter,
-				struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-	u8 PLH, CurrentEntryNum, PhysLinkDisconnectReason;
-
-	pBtDbg->dbgHciInfo.hciCmdCntDisconnectPhyLink++;
-
-	PLH = *((u8 *)pHciCmd->Data);
-	PhysLinkDisconnectReason = *((u8 *)pHciCmd->Data+1);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_DISCONNECT_PHYSICAL_LINK  PhyHandle = 0x%x, Reason = 0x%x\n",
-		PLH, PhysLinkDisconnectReason));
-
-	CurrentEntryNum = bthci_GetCurrentEntryNum(padapter, PLH);
-
-	if (CurrentEntryNum == 0xff) {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD,
-			("DisconnectPhysicalLink, No such Handle in the Entry\n"));
-		status = HCI_STATUS_UNKNOW_CONNECT_ID;
-	} else {
-		pBTInfo->BtAsocEntry[CurrentEntryNum].PhyLinkDisconnectReason =
-			(enum hci_status)PhysLinkDisconnectReason;
-	}
-	/* Send HCI Command status event to AMP. */
-	bthci_EventCommandStatus(padapter, LINK_CONTROL_COMMANDS,
-				 HCI_DISCONNECT_PHYSICAL_LINK, status);
-
-	if (status != HCI_STATUS_SUCCESS)
-		return status;
-
-	/* The macros below require { and } in the if statement */
-	if (pBTInfo->BtAsocEntry[CurrentEntryNum].BtCurrentState == HCI_STATE_DISCONNECTED) {
-		BTHCI_SM_WITH_INFO(padapter, HCI_STATE_DISCONNECTED, STATE_CMD_DISCONNECT_PHY_LINK, CurrentEntryNum);
-	} else {
-		BTHCI_SM_WITH_INFO(padapter, HCI_STATE_DISCONNECTING, STATE_CMD_DISCONNECT_PHY_LINK, CurrentEntryNum);
-	}
-	return status;
-}
-
-static enum hci_status
-bthci_CmdSetACLLinkDataFlowMode(struct rtw_adapter *padapter,
-				struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	u8 localBuf[8] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp;
-
-	pBtMgnt->ExtConfig.CurrentConnectHandle = *((u16 *)pHciCmd->Data);
-	pBtMgnt->ExtConfig.CurrentIncomingTrafficMode = *((u8 *)pHciCmd->Data)+2;
-	pBtMgnt->ExtConfig.CurrentOutgoingTrafficMode = *((u8 *)pHciCmd->Data)+3;
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("Connection Handle = 0x%x, Incoming Traffic mode = 0x%x, Outgoing Traffic mode = 0x%x",
-		pBtMgnt->ExtConfig.CurrentConnectHandle,
-		pBtMgnt->ExtConfig.CurrentIncomingTrafficMode,
-		pBtMgnt->ExtConfig.CurrentOutgoingTrafficMode));
-
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_EXTENSION,
-		HCI_SET_ACL_LINK_DATA_FLOW_MODE,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-
-	pu2Temp = (u16 *)&pRetPar[1];
-	*pu2Temp = pBtMgnt->ExtConfig.CurrentConnectHandle;
-	len += 3;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	return status;
-}
-
-static enum hci_status
-bthci_CmdSetACLLinkStatus(struct rtw_adapter *padapter,
-			  struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-	u8 i;
-	u8 *pTriple;
-
-	pBtDbg->dbgHciInfo.hciCmdCntSetAclLinkStatus++;
-	RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD_EXT, "SetACLLinkStatus, Hex Data :\n",
-			&pHciCmd->Data[0], pHciCmd->Length);
-
-	/*  Only Core Stack v251 and later version support this command. */
-	pBtMgnt->bSupportProfile = true;
-
-	pBtMgnt->ExtConfig.NumberOfHandle = *((u8 *)pHciCmd->Data);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("NumberOfHandle = 0x%x\n", pBtMgnt->ExtConfig.NumberOfHandle));
-
-	pTriple = &pHciCmd->Data[1];
-	for (i = 0; i < pBtMgnt->ExtConfig.NumberOfHandle; i++) {
-		pBtMgnt->ExtConfig.linkInfo[i].ConnectHandle = *((u16 *)&pTriple[0]);
-		pBtMgnt->ExtConfig.linkInfo[i].IncomingTrafficMode = pTriple[2];
-		pBtMgnt->ExtConfig.linkInfo[i].OutgoingTrafficMode = pTriple[3];
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT,
-			("Connection_Handle = 0x%x, Incoming Traffic mode = 0x%x, Outgoing Traffic Mode = 0x%x\n",
-			pBtMgnt->ExtConfig.linkInfo[i].ConnectHandle,
-			pBtMgnt->ExtConfig.linkInfo[i].IncomingTrafficMode,
-			pBtMgnt->ExtConfig.linkInfo[i].OutgoingTrafficMode));
-		pTriple += 4;
-	}
-
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_SET_ACL_LINK_STATUS,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdSetSCOLinkStatus(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-
-	pBtDbg->dbgHciInfo.hciCmdCntSetScoLinkStatus++;
-	pBtMgnt->ExtConfig.NumberOfSCO = *((u8 *)pHciCmd->Data);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("NumberOfSCO = 0x%x\n",
-		pBtMgnt->ExtConfig.NumberOfSCO));
-
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_SET_SCO_LINK_STATUS,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdSetRSSIValue(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	s8		min_bt_rssi = 0;
-	u8 i;
-	for (i = 0; i < pBtMgnt->ExtConfig.NumberOfHandle; i++) {
-		if (pBtMgnt->ExtConfig.linkInfo[i].ConnectHandle == *((u16 *)&pHciCmd->Data[0])) {
-			pBtMgnt->ExtConfig.linkInfo[i].BT_RSSI = (s8)(pHciCmd->Data[2]);
-			RTPRINT(FIOCTL, IOCTL_BT_EVENT_PERIODICAL,
-			("Connection_Handle = 0x%x, RSSI = %d \n",
-			pBtMgnt->ExtConfig.linkInfo[i].ConnectHandle,
-			pBtMgnt->ExtConfig.linkInfo[i].BT_RSSI));
-		}
-		/*  get the minimum bt rssi value */
-		if (pBtMgnt->ExtConfig.linkInfo[i].BT_RSSI <= min_bt_rssi)
-			min_bt_rssi = pBtMgnt->ExtConfig.linkInfo[i].BT_RSSI;
-	}
-
-	pBtMgnt->ExtConfig.MIN_BT_RSSI = min_bt_rssi;
-	RTPRINT(FBT, BT_TRACE, ("[bt rssi], the min rssi is %d\n", min_bt_rssi));
-
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_SET_RSSI_VALUE,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdSetCurrentBluetoothStatus(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-/*PMGNT_INFO	pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	pBtMgnt->ExtConfig.CurrentBTStatus = *((u8 *)&pHciCmd->Data[0]);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("SetCurrentBluetoothStatus, CurrentBTStatus = 0x%x\n",
-		pBtMgnt->ExtConfig.CurrentBTStatus));
-
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_SET_CURRENT_BLUETOOTH_STATUS,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-		len += 1;
-
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdExtensionVersionNotify(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-
-	pBtDbg->dbgHciInfo.hciCmdCntExtensionVersionNotify++;
-	RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD_EXT, "ExtensionVersionNotify, Hex Data :\n",
-			&pHciCmd->Data[0], pHciCmd->Length);
-
-	pBtMgnt->ExtConfig.HCIExtensionVer = *((u16 *)&pHciCmd->Data[0]);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCIExtensionVer = 0x%x\n", pBtMgnt->ExtConfig.HCIExtensionVer));
-
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_EXTENSION_VERSION_NOTIFY,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdLinkStatusNotify(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-	u8 i;
-	u8 *pTriple;
-
-	pBtDbg->dbgHciInfo.hciCmdCntLinkStatusNotify++;
-	RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD_EXT, "LinkStatusNotify, Hex Data :\n",
-			&pHciCmd->Data[0], pHciCmd->Length);
-
-	/*  Current only RTL8723 support this command. */
-	pBtMgnt->bSupportProfile = true;
-
-	pBtMgnt->ExtConfig.NumberOfHandle = *((u8 *)pHciCmd->Data);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("NumberOfHandle = 0x%x\n", pBtMgnt->ExtConfig.NumberOfHandle));
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCIExtensionVer = %d\n", pBtMgnt->ExtConfig.HCIExtensionVer));
-
-	pTriple = &pHciCmd->Data[1];
-	for (i = 0; i < pBtMgnt->ExtConfig.NumberOfHandle; i++) {
-		if (pBtMgnt->ExtConfig.HCIExtensionVer < 1) {
-			pBtMgnt->ExtConfig.linkInfo[i].ConnectHandle = *((u16 *)&pTriple[0]);
-			pBtMgnt->ExtConfig.linkInfo[i].BTProfile = pTriple[2];
-			pBtMgnt->ExtConfig.linkInfo[i].BTCoreSpec = pTriple[3];
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT,
-				("Connection_Handle = 0x%x, BTProfile =%d, BTSpec =%d\n",
-				pBtMgnt->ExtConfig.linkInfo[i].ConnectHandle,
-				pBtMgnt->ExtConfig.linkInfo[i].BTProfile,
-				pBtMgnt->ExtConfig.linkInfo[i].BTCoreSpec));
-			pTriple += 4;
-		} else if (pBtMgnt->ExtConfig.HCIExtensionVer >= 1) {
-			pBtMgnt->ExtConfig.linkInfo[i].ConnectHandle = *((u16 *)&pTriple[0]);
-			pBtMgnt->ExtConfig.linkInfo[i].BTProfile = pTriple[2];
-			pBtMgnt->ExtConfig.linkInfo[i].BTCoreSpec = pTriple[3];
-			pBtMgnt->ExtConfig.linkInfo[i].linkRole = pTriple[4];
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT,
-				("Connection_Handle = 0x%x, BTProfile =%d, BTSpec =%d, LinkRole =%d\n",
-				pBtMgnt->ExtConfig.linkInfo[i].ConnectHandle,
-				pBtMgnt->ExtConfig.linkInfo[i].BTProfile,
-				pBtMgnt->ExtConfig.linkInfo[i].BTCoreSpec,
-				pBtMgnt->ExtConfig.linkInfo[i].linkRole));
-			pTriple += 5;
-		}
-
-	}
-	BTHCI_UpdateBTProfileRTKToMoto(padapter);
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_LINK_STATUS_NOTIFY,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdBtOperationNotify(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD_EXT, "Bt Operation notify, Hex Data :\n",
-			&pHciCmd->Data[0], pHciCmd->Length);
-
-	pBtMgnt->ExtConfig.btOperationCode = *((u8 *)pHciCmd->Data);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("btOperationCode = 0x%x\n", pBtMgnt->ExtConfig.btOperationCode));
-	switch (pBtMgnt->ExtConfig.btOperationCode) {
-	case HCI_BT_OP_NONE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : Operation None!!\n"));
-		break;
-	case HCI_BT_OP_INQUIRY_START:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : Inquire start!!\n"));
-		break;
-	case HCI_BT_OP_INQUIRY_FINISH:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : Inquire finished!!\n"));
-		break;
-	case HCI_BT_OP_PAGING_START:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : Paging is started!!\n"));
-		break;
-	case HCI_BT_OP_PAGING_SUCCESS:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : Paging complete successfully!!\n"));
-		break;
-	case HCI_BT_OP_PAGING_UNSUCCESS:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : Paging complete unsuccessfully!!\n"));
-		break;
-	case HCI_BT_OP_PAIRING_START:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : Pairing start!!\n"));
-		break;
-	case HCI_BT_OP_PAIRING_FINISH:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : Pairing finished!!\n"));
-		break;
-	case HCI_BT_OP_BT_DEV_ENABLE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : BT Device is enabled!!\n"));
-		break;
-	case HCI_BT_OP_BT_DEV_DISABLE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : BT Device is disabled!!\n"));
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[bt operation] : Unknown, error!!\n"));
-		break;
-	}
-	BTDM_AdjustForBtOperation(padapter);
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_BT_OPERATION_NOTIFY,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdEnableWifiScanNotify(struct rtw_adapter *padapter,
-			      struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT_DATA(FIOCTL, IOCTL_BT_HCICMD_EXT, "Enable Wifi scan notify, Hex Data :\n",
-			&pHciCmd->Data[0], pHciCmd->Length);
-
-	pBtMgnt->ExtConfig.bEnableWifiScanNotify = *((u8 *)pHciCmd->Data);
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("bEnableWifiScanNotify = %d\n", pBtMgnt->ExtConfig.bEnableWifiScanNotify));
-
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_ENABLE_WIFI_SCAN_NOTIFY,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWIFICurrentChannel(struct rtw_adapter *padapter,
-			    struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	u8 chnl = pmlmeext->cur_channel;
-
-	if (pmlmeext->cur_bwmode == HT_CHANNEL_WIDTH_40) {
-		if (pmlmeext->cur_ch_offset == HAL_PRIME_CHNL_OFFSET_UPPER)
-			chnl += 2;
-		else if (pmlmeext->cur_ch_offset == HAL_PRIME_CHNL_OFFSET_LOWER)
-			chnl -= 2;
-	}
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("Current Channel  = 0x%x\n", chnl));
-
-	{
-		u8 localBuf[8] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_WIFI_CURRENT_CHANNEL,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-		pRetPar[1] = chnl;			/* current channel */
-		len += 2;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWIFICurrentBandwidth(struct rtw_adapter *padapter,
-			      struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	enum ht_channel_width bw;
-	u8 CurrentBW = 0;
-
-	bw = padapter->mlmeextpriv.cur_bwmode;
-
-	if (bw == HT_CHANNEL_WIDTH_20)
-		CurrentBW = 0;
-	else if (bw == HT_CHANNEL_WIDTH_40)
-		CurrentBW = 1;
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("Current BW = 0x%x\n",
-		CurrentBW));
-
-	{
-		u8 localBuf[8] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_WIFI_CURRENT_BANDWIDTH,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-		pRetPar[1] = CurrentBW;		/* current BW */
-		len += 2;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdWIFIConnectionStatus(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	u8 connectStatus = HCI_WIFI_NOT_CONNECTED;
-
-	if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE)) {
-		if (padapter->stapriv.asoc_sta_count >= 3)
-			connectStatus = HCI_WIFI_CONNECTED;
-		else
-			connectStatus = HCI_WIFI_NOT_CONNECTED;
-	} else if (check_fwstate(&padapter->mlmepriv, WIFI_ADHOC_STATE|WIFI_ADHOC_MASTER_STATE|WIFI_ASOC_STATE)) {
-		connectStatus = HCI_WIFI_CONNECTED;
-	} else if (check_fwstate(&padapter->mlmepriv, WIFI_UNDER_LINKING)) {
-		connectStatus = HCI_WIFI_CONNECT_IN_PROGRESS;
-	} else {
-		connectStatus = HCI_WIFI_NOT_CONNECTED;
-	}
-
-	{
-		u8 localBuf[8] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_EXTENSION,
-			HCI_WIFI_CONNECTION_STATUS,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;			/* status */
-		pRetPar[1] = connectStatus;	/* connect status */
-		len += 2;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdEnableDeviceUnderTestMode(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-
-	pBtHciInfo->bInTestMode = true;
-	pBtHciInfo->bTestIsEnd = false;
-
-	/* send command complete event here when all data are received. */
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_TESTING_COMMANDS,
-			HCI_ENABLE_DEVICE_UNDER_TEST_MODE,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdAMPTestEnd(struct rtw_adapter *padapter,
-		    struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-
-	if (!pBtHciInfo->bInTestMode) {
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Not in Test mode, return status = HCI_STATUS_CMD_DISALLOW\n"));
-		status = HCI_STATUS_CMD_DISALLOW;
-		return status;
-	}
-
-	pBtHciInfo->bTestIsEnd = true;
-
-	del_timer_sync(&pBTInfo->BTTestSendPacketTimer);
-
-	rtl8723a_check_bssid(padapter, true);
-
-	/* send command complete event here when all data are received. */
-	{
-		u8 localBuf[4] = "";
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("AMP Test End Event \n"));
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-		PPacketIrpEvent->EventCode = HCI_EVENT_AMP_TEST_END;
-		PPacketIrpEvent->Length = 2;
-
-		PPacketIrpEvent->Data[0] = status;
-		PPacketIrpEvent->Data[1] = pBtHciInfo->TestScenario;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, 4);
-	}
-
-	bthci_EventAMPReceiverReport(padapter, 0x01);
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdAMPTestCommand(struct rtw_adapter *padapter,
-			struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-
-	if (!pBtHciInfo->bInTestMode) {
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Not in Test mode, return status = HCI_STATUS_CMD_DISALLOW\n"));
-		status = HCI_STATUS_CMD_DISALLOW;
-		return status;
-	}
-
-	pBtHciInfo->TestScenario = *((u8 *)pHciCmd->Data);
-
-	if (pBtHciInfo->TestScenario == 0x01)
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("TX Single Test \n"));
-	else if (pBtHciInfo->TestScenario == 0x02)
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Receive Frame Test \n"));
-	else
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("No Such Test !!!!!!!!!!!!!!!!!! \n"));
-
-	if (pBtHciInfo->bTestIsEnd) {
-		u8 localBuf[5] = "";
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("AMP Test End Event \n"));
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-		PPacketIrpEvent->EventCode = HCI_EVENT_AMP_TEST_END;
-		PPacketIrpEvent->Length = 2;
-
-		PPacketIrpEvent->Data[0] = status;
-		PPacketIrpEvent->Data[1] = pBtHciInfo->TestScenario ;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, 4);
-
-		/* Return to Idel state with RX and TX off. */
-
-		return status;
-	}
-
-	/*  should send command status event */
-	bthci_EventCommandStatus(padapter,
-			OGF_TESTING_COMMANDS,
-			HCI_AMP_TEST_COMMAND,
-			status);
-
-	/* The HCI_AMP_Start Test Event shall be generated when the */
-	/* HCI_AMP_Test_Command has completed and the first data is ready to be sent */
-	/* or received. */
-
-	{
-		u8 localBuf[5] = "";
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), (" HCI_AMP_Start Test Event \n"));
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-		PPacketIrpEvent->EventCode = HCI_EVENT_AMP_START_TEST;
-		PPacketIrpEvent->Length = 2;
-
-		PPacketIrpEvent->Data[0] = status;
-		PPacketIrpEvent->Data[1] = pBtHciInfo->TestScenario ;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, 4);
-
-		/* Return to Idel state with RX and TX off. */
-	}
-
-	if (pBtHciInfo->TestScenario == 0x01) {
-		/*
-			When in a transmitter test scenario and the frames/bursts count have been
-			transmitted the HCI_AMP_Test_End event shall be sent.
-		*/
-		mod_timer(&pBTInfo->BTTestSendPacketTimer,
-			  jiffies + msecs_to_jiffies(50));
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("TX Single Test \n"));
-	} else if (pBtHciInfo->TestScenario == 0x02) {
-		rtl8723a_check_bssid(padapter, false);
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_BT_LOGO), ("Receive Frame Test \n"));
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdEnableAMPReceiverReports(struct rtw_adapter *padapter,
-				  struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-
-	if (!pBtHciInfo->bInTestMode) {
-		status = HCI_STATUS_CMD_DISALLOW;
-		/* send command complete event here when all data are received. */
-		{
-			u8 localBuf[6] = "";
-			u8 *pRetPar;
-			u8 len = 0;
-			struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-			PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-			len += bthci_CommandCompleteHeader(&localBuf[0],
-				OGF_TESTING_COMMANDS,
-				HCI_ENABLE_AMP_RECEIVER_REPORTS,
-				status);
-
-			/*  Return parameters starts from here */
-			pRetPar = &PPacketIrpEvent->Data[len];
-			pRetPar[0] = status;		/* status */
-			len += 1;
-			PPacketIrpEvent->Length = len;
-
-			bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-		}
-		return status;
-	}
-
-	pBtHciInfo->bTestNeedReport = *((u8 *)pHciCmd->Data);
-	pBtHciInfo->TestReportInterval = (*((u8 *)pHciCmd->Data+2));
-
-	bthci_EventAMPReceiverReport(padapter, 0x00);
-
-	/* send command complete event here when all data are received. */
-	{
-		u8 localBuf[6] = "";
-		u8 *pRetPar;
-		u8 len = 0;
-		struct packet_irp_hcievent_data *PPacketIrpEvent;
-
-		PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-		len += bthci_CommandCompleteHeader(&localBuf[0],
-			OGF_TESTING_COMMANDS,
-			HCI_ENABLE_AMP_RECEIVER_REPORTS,
-			status);
-
-		/*  Return parameters starts from here */
-		pRetPar = &PPacketIrpEvent->Data[len];
-		pRetPar[0] = status;		/* status */
-		len += 1;
-		PPacketIrpEvent->Length = len;
-
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-	}
-
-	return status;
-}
-
-static enum hci_status
-bthci_CmdHostBufferSize(struct rtw_adapter *padapter,
-			struct packet_irp_hcicmd_data *pHciCmd)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	u8 localBuf[6] = "";
-	u8 *pRetPar;
-	u8 len = 0;
-
-	pBTInfo->BtAsocEntry[pBtMgnt->CurrentConnectEntryNum].ACLPacketsData.ACLDataPacketLen = *((u16 *)pHciCmd->Data);
-	pBTInfo->BtAsocEntry[pBtMgnt->CurrentConnectEntryNum].SyncDataPacketLen = *((u8 *)(pHciCmd->Data+2));
-	pBTInfo->BtAsocEntry[pBtMgnt->CurrentConnectEntryNum].TotalNumACLDataPackets = *((u16 *)(pHciCmd->Data+3));
-	pBTInfo->BtAsocEntry[pBtMgnt->CurrentConnectEntryNum].TotalSyncNumDataPackets = *((u16 *)(pHciCmd->Data+5));
-
-	/* send command complete event here when all data are received. */
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	len += bthci_CommandCompleteHeader(&localBuf[0],
-		OGF_SET_EVENT_MASK_COMMAND,
-		HCI_HOST_BUFFER_SIZE,
-		status);
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[len];
-	pRetPar[0] = status;		/* status */
-	len += 1;
-	PPacketIrpEvent->Length = len;
-
-	bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-
-	return status;
-}
-
-static enum hci_status
-bthci_UnknownCMD(struct rtw_adapter *padapter, struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_UNKNOW_HCI_CMD;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-
-	pBtDbg->dbgHciInfo.hciCmdCntUnknown++;
-	bthci_EventCommandStatus(padapter,
-			(u8)pHciCmd->OGF,
-			pHciCmd->OCF,
-			status);
-
-	return status;
-}
-
-static enum hci_status
-bthci_HandleOGFInformationalParameters(struct rtw_adapter *padapter,
-				       struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-
-	switch (pHciCmd->OCF) {
-	case HCI_READ_LOCAL_VERSION_INFORMATION:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_LOCAL_VERSION_INFORMATION\n"));
-		status = bthci_CmdReadLocalVersionInformation(padapter);
-		break;
-	case HCI_READ_LOCAL_SUPPORTED_COMMANDS:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_LOCAL_SUPPORTED_COMMANDS\n"));
-		status = bthci_CmdReadLocalSupportedCommands(padapter);
-		break;
-	case HCI_READ_LOCAL_SUPPORTED_FEATURES:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_LOCAL_SUPPORTED_FEATURES\n"));
-		status = bthci_CmdReadLocalSupportedFeatures(padapter);
-		break;
-	case HCI_READ_BUFFER_SIZE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_BUFFER_SIZE\n"));
-		status = bthci_CmdReadBufferSize(padapter);
-		break;
-	case HCI_READ_DATA_BLOCK_SIZE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_DATA_BLOCK_SIZE\n"));
-		status = bthci_CmdReadDataBlockSize(padapter);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("bthci_HandleOGFInformationalParameters(), Unknown case = 0x%x\n", pHciCmd->OCF));
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_UNKNOWN_COMMAND\n"));
-		status = bthci_UnknownCMD(padapter, pHciCmd);
-		break;
-	}
-	return status;
-}
-
-static enum hci_status
-bthci_HandleOGFSetEventMaskCMD(struct rtw_adapter *padapter,
-			       struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-
-	switch (pHciCmd->OCF) {
-	case HCI_SET_EVENT_MASK:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_SET_EVENT_MASK\n"));
-		status = bthci_CmdSetEventMask(padapter, pHciCmd);
-		break;
-	case HCI_RESET:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_RESET\n"));
-		status = bthci_CmdReset(padapter, true);
-		break;
-	case HCI_READ_CONNECTION_ACCEPT_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_CONNECTION_ACCEPT_TIMEOUT\n"));
-		status = bthci_CmdReadConnectionAcceptTimeout(padapter);
-		break;
-	case HCI_SET_EVENT_FILTER:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_SET_EVENT_FILTER\n"));
-		break;
-	case HCI_WRITE_CONNECTION_ACCEPT_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_WRITE_CONNECTION_ACCEPT_TIMEOUT\n"));
-		status = bthci_CmdWriteConnectionAcceptTimeout(padapter, pHciCmd);
-		break;
-	case HCI_READ_PAGE_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_PAGE_TIMEOUT\n"));
-		status = bthci_CmdReadPageTimeout(padapter, pHciCmd);
-		break;
-	case HCI_WRITE_PAGE_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_WRITE_PAGE_TIMEOUT\n"));
-		status = bthci_CmdWritePageTimeout(padapter, pHciCmd);
-		break;
-	case HCI_HOST_NUMBER_OF_COMPLETED_PACKETS:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_HOST_NUMBER_OF_COMPLETED_PACKETS\n"));
-		break;
-	case HCI_READ_LINK_SUPERVISION_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_LINK_SUPERVISION_TIMEOUT\n"));
-		status = bthci_CmdReadLinkSupervisionTimeout(padapter, pHciCmd);
-		break;
-	case HCI_WRITE_LINK_SUPERVISION_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_WRITE_LINK_SUPERVISION_TIMEOUT\n"));
-		status = bthci_CmdWriteLinkSupervisionTimeout(padapter, pHciCmd);
-		break;
-	case HCI_ENHANCED_FLUSH:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_ENHANCED_FLUSH\n"));
-		status = bthci_CmdEnhancedFlush(padapter, pHciCmd);
-		break;
-	case HCI_READ_LOGICAL_LINK_ACCEPT_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_LOGICAL_LINK_ACCEPT_TIMEOUT\n"));
-		status = bthci_CmdReadLogicalLinkAcceptTimeout(padapter, pHciCmd);
-		break;
-	case HCI_WRITE_LOGICAL_LINK_ACCEPT_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_WRITE_LOGICAL_LINK_ACCEPT_TIMEOUT\n"));
-		status = bthci_CmdWriteLogicalLinkAcceptTimeout(padapter, pHciCmd);
-		break;
-	case HCI_SET_EVENT_MASK_PAGE_2:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_SET_EVENT_MASK_PAGE_2\n"));
-		status = bthci_CmdSetEventMaskPage2(padapter, pHciCmd);
-		break;
-	case HCI_READ_LOCATION_DATA:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_LOCATION_DATA\n"));
-		status = bthci_CmdReadLocationData(padapter, pHciCmd);
-		break;
-	case HCI_WRITE_LOCATION_DATA:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_WRITE_LOCATION_DATA\n"));
-		status = bthci_CmdWriteLocationData(padapter, pHciCmd);
-		break;
-	case HCI_READ_FLOW_CONTROL_MODE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_FLOW_CONTROL_MODE\n"));
-		status = bthci_CmdReadFlowControlMode(padapter, pHciCmd);
-		break;
-	case HCI_WRITE_FLOW_CONTROL_MODE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_WRITE_FLOW_CONTROL_MODE\n"));
-		status = bthci_CmdWriteFlowControlMode(padapter, pHciCmd);
-		break;
-	case HCI_READ_BEST_EFFORT_FLUSH_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_BEST_EFFORT_FLUSH_TIMEOUT\n"));
-		status = bthci_CmdReadBestEffortFlushTimeout(padapter, pHciCmd);
-		break;
-	case HCI_WRITE_BEST_EFFORT_FLUSH_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_WRITE_BEST_EFFORT_FLUSH_TIMEOUT\n"));
-		status = bthci_CmdWriteBestEffortFlushTimeout(padapter, pHciCmd);
-		break;
-	case HCI_SHORT_RANGE_MODE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_SHORT_RANGE_MODE\n"));
-		status = bthci_CmdShortRangeMode(padapter, pHciCmd);
-		break;
-	case HCI_HOST_BUFFER_SIZE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_HOST_BUFFER_SIZE\n"));
-		status = bthci_CmdHostBufferSize(padapter, pHciCmd);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("bthci_HandleOGFSetEventMaskCMD(), Unknown case = 0x%x\n", pHciCmd->OCF));
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_UNKNOWN_COMMAND\n"));
-		status = bthci_UnknownCMD(padapter, pHciCmd);
-		break;
-	}
-	return status;
-}
-
-static enum hci_status
-bthci_HandleOGFStatusParameters(struct rtw_adapter *padapter,
-				struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-
-	switch (pHciCmd->OCF) {
-	case HCI_READ_FAILED_CONTACT_COUNTER:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_FAILED_CONTACT_COUNTER\n"));
-		status = bthci_CmdReadFailedContactCounter(padapter, pHciCmd);
-		break;
-	case HCI_RESET_FAILED_CONTACT_COUNTER:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_RESET_FAILED_CONTACT_COUNTER\n"));
-		status = bthci_CmdResetFailedContactCounter(padapter, pHciCmd);
-		break;
-	case HCI_READ_LINK_QUALITY:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_LINK_QUALITY\n"));
-		status = bthci_CmdReadLinkQuality(padapter, pHciCmd);
-		break;
-	case HCI_READ_RSSI:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_RSSI\n"));
-		break;
-	case HCI_READ_LOCAL_AMP_INFO:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_LOCAL_AMP_INFO\n"));
-		status = bthci_CmdReadLocalAMPInfo(padapter);
-		break;
-	case HCI_READ_LOCAL_AMP_ASSOC:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_READ_LOCAL_AMP_ASSOC\n"));
-		status = bthci_CmdReadLocalAMPAssoc(padapter, pHciCmd);
-		break;
-	case HCI_WRITE_REMOTE_AMP_ASSOC:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_WRITE_REMOTE_AMP_ASSOC\n"));
-		status = bthci_CmdWriteRemoteAMPAssoc(padapter, pHciCmd);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("bthci_HandleOGFStatusParameters(), Unknown case = 0x%x\n", pHciCmd->OCF));
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_UNKNOWN_COMMAND\n"));
-		status = bthci_UnknownCMD(padapter, pHciCmd);
-		break;
-	}
-	return status;
-}
-
-static enum hci_status
-bthci_HandleOGFLinkControlCMD(struct rtw_adapter *padapter,
-			      struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-
-	switch (pHciCmd->OCF) {
-	case HCI_CREATE_PHYSICAL_LINK:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_CREATE_PHYSICAL_LINK\n"));
-		status = bthci_CmdCreatePhysicalLink(padapter, pHciCmd);
-		break;
-	case HCI_ACCEPT_PHYSICAL_LINK:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_ACCEPT_PHYSICAL_LINK\n"));
-		status = bthci_CmdAcceptPhysicalLink(padapter, pHciCmd);
-		break;
-	case HCI_DISCONNECT_PHYSICAL_LINK:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_DISCONNECT_PHYSICAL_LINK\n"));
-		status = bthci_CmdDisconnectPhysicalLink(padapter, pHciCmd);
-		break;
-	case HCI_CREATE_LOGICAL_LINK:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_CREATE_LOGICAL_LINK\n"));
-		status = bthci_CmdCreateLogicalLink(padapter, pHciCmd);
-		break;
-	case HCI_ACCEPT_LOGICAL_LINK:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_ACCEPT_LOGICAL_LINK\n"));
-		status = bthci_CmdAcceptLogicalLink(padapter, pHciCmd);
-		break;
-	case HCI_DISCONNECT_LOGICAL_LINK:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_DISCONNECT_LOGICAL_LINK\n"));
-		status = bthci_CmdDisconnectLogicalLink(padapter, pHciCmd);
-		break;
-	case HCI_LOGICAL_LINK_CANCEL:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_LOGICAL_LINK_CANCEL\n"));
-		status = bthci_CmdLogicalLinkCancel(padapter, pHciCmd);
-		break;
-	case HCI_FLOW_SPEC_MODIFY:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_FLOW_SPEC_MODIFY\n"));
-		status = bthci_CmdFlowSpecModify(padapter, pHciCmd);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("bthci_HandleOGFLinkControlCMD(), Unknown case = 0x%x\n", pHciCmd->OCF));
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_UNKNOWN_COMMAND\n"));
-		status = bthci_UnknownCMD(padapter, pHciCmd);
-		break;
-	}
-	return status;
-}
-
-static enum hci_status
-bthci_HandleOGFTestingCMD(struct rtw_adapter *padapter,
-			  struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	switch (pHciCmd->OCF) {
-	case HCI_ENABLE_DEVICE_UNDER_TEST_MODE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_ENABLE_DEVICE_UNDER_TEST_MODE\n"));
-		bthci_CmdEnableDeviceUnderTestMode(padapter, pHciCmd);
-		break;
-	case HCI_AMP_TEST_END:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_AMP_TEST_END\n"));
-		bthci_CmdAMPTestEnd(padapter, pHciCmd);
-		break;
-	case HCI_AMP_TEST_COMMAND:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_AMP_TEST_COMMAND\n"));
-		bthci_CmdAMPTestCommand(padapter, pHciCmd);
-		break;
-	case HCI_ENABLE_AMP_RECEIVER_REPORTS:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_ENABLE_AMP_RECEIVER_REPORTS\n"));
-		bthci_CmdEnableAMPReceiverReports(padapter, pHciCmd);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_UNKNOWN_COMMAND\n"));
-		status = bthci_UnknownCMD(padapter, pHciCmd);
-		break;
-	}
-	return status;
-}
-
-static enum hci_status
-bthci_HandleOGFExtension(struct rtw_adapter *padapter,
-			 struct packet_irp_hcicmd_data *pHciCmd)
-{
-	enum hci_status status = HCI_STATUS_SUCCESS;
-	switch (pHciCmd->OCF) {
-	case HCI_SET_ACL_LINK_DATA_FLOW_MODE:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_SET_ACL_LINK_DATA_FLOW_MODE\n"));
-		status = bthci_CmdSetACLLinkDataFlowMode(padapter, pHciCmd);
-		break;
-	case HCI_SET_ACL_LINK_STATUS:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_SET_ACL_LINK_STATUS\n"));
-		status = bthci_CmdSetACLLinkStatus(padapter, pHciCmd);
-		break;
-	case HCI_SET_SCO_LINK_STATUS:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_SET_SCO_LINK_STATUS\n"));
-		status = bthci_CmdSetSCOLinkStatus(padapter, pHciCmd);
-		break;
-	case HCI_SET_RSSI_VALUE:
-		RTPRINT(FIOCTL, IOCTL_BT_EVENT_PERIODICAL, ("HCI_SET_RSSI_VALUE\n"));
-		status = bthci_CmdSetRSSIValue(padapter, pHciCmd);
-		break;
-	case HCI_SET_CURRENT_BLUETOOTH_STATUS:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_SET_CURRENT_BLUETOOTH_STATUS\n"));
-		status = bthci_CmdSetCurrentBluetoothStatus(padapter, pHciCmd);
-		break;
-	/* The following is for RTK8723 */
-
-	case HCI_EXTENSION_VERSION_NOTIFY:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_EXTENSION_VERSION_NOTIFY\n"));
-		status = bthci_CmdExtensionVersionNotify(padapter, pHciCmd);
-		break;
-	case HCI_LINK_STATUS_NOTIFY:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_LINK_STATUS_NOTIFY\n"));
-		status = bthci_CmdLinkStatusNotify(padapter, pHciCmd);
-		break;
-	case HCI_BT_OPERATION_NOTIFY:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_BT_OPERATION_NOTIFY\n"));
-		status = bthci_CmdBtOperationNotify(padapter, pHciCmd);
-		break;
-	case HCI_ENABLE_WIFI_SCAN_NOTIFY:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_ENABLE_WIFI_SCAN_NOTIFY\n"));
-		status = bthci_CmdEnableWifiScanNotify(padapter, pHciCmd);
-		break;
-
-	/* The following is for IVT */
-	case HCI_WIFI_CURRENT_CHANNEL:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_WIFI_CURRENT_CHANNEL\n"));
-		status = bthci_CmdWIFICurrentChannel(padapter, pHciCmd);
-		break;
-	case HCI_WIFI_CURRENT_BANDWIDTH:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_WIFI_CURRENT_BANDWIDTH\n"));
-		status = bthci_CmdWIFICurrentBandwidth(padapter, pHciCmd);
-		break;
-	case HCI_WIFI_CONNECTION_STATUS:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_WIFI_CONNECTION_STATUS\n"));
-		status = bthci_CmdWIFIConnectionStatus(padapter, pHciCmd);
-		break;
-
-	default:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("HCI_UNKNOWN_COMMAND\n"));
-		status = bthci_UnknownCMD(padapter, pHciCmd);
-		break;
-	}
-	return status;
-}
-
-static void
-bthci_StateStarting(struct rtw_adapter *padapter,
-		    enum hci_state_with_cmd StateCmd, u8 EntryNum)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT(FIOCTL, IOCTL_STATE, ("[BT state], [Starting], "));
-	switch (StateCmd) {
-	case STATE_CMD_CONNECT_ACCEPT_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_CONNECT_ACCEPT_TIMEOUT\n"));
-		pBTInfo->BtAsocEntry[EntryNum].PhysLinkCompleteStatus = HCI_STATUS_CONNECT_ACCEPT_TIMEOUT;
-		pBtMgnt->bNeedNotifyAMPNoCap = true;
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-		break;
-	case STATE_CMD_DISCONNECT_PHY_LINK:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_DISCONNECT_PHY_LINK\n"));
-
-		bthci_EventDisconnectPhyLinkComplete(padapter,
-		HCI_STATUS_SUCCESS,
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkDisconnectReason,
-		EntryNum);
-
-		del_timer_sync(&pBTInfo->BTHCIJoinTimeoutTimer);
-
-		pBTInfo->BtAsocEntry[EntryNum].PhysLinkCompleteStatus = HCI_STATUS_UNKNOW_CONNECT_ID;
-
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-		break;
-	case STATE_CMD_MAC_START_COMPLETE:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_MAC_START_COMPLETE\n"));
-		if (pBTInfo->BtAsocEntry[EntryNum].AMPRole == AMP_BTAP_CREATOR)
-			bthci_EventChannelSelected(padapter, EntryNum);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("State command(%d) is Wrong !!!\n", StateCmd));
-		break;
-	}
-}
-
-static void
-bthci_StateConnecting(struct rtw_adapter *padapter,
-		      enum hci_state_with_cmd StateCmd, u8 EntryNum)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT(FIOCTL, IOCTL_STATE, ("[BT state], [Connecting], "));
-	switch (StateCmd) {
-	case STATE_CMD_CONNECT_ACCEPT_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_CONNECT_ACCEPT_TIMEOUT\n"));
-		pBTInfo->BtAsocEntry[EntryNum].PhysLinkCompleteStatus = HCI_STATUS_CONNECT_ACCEPT_TIMEOUT;
-		pBtMgnt->bNeedNotifyAMPNoCap = true;
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-		break;
-	case STATE_CMD_MAC_CONNECT_COMPLETE:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_MAC_CONNECT_COMPLETE\n"));
-
-		if (pBTInfo->BtAsocEntry[EntryNum].AMPRole == AMP_BTAP_JOINER) {
-			RT_TRACE(_module_rtl871x_security_c_, _drv_info_,
-				 "StateConnecting\n");
-		}
-		break;
-	case STATE_CMD_DISCONNECT_PHY_LINK:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_DISCONNECT_PHY_LINK\n"));
-
-		bthci_EventDisconnectPhyLinkComplete(padapter,
-		HCI_STATUS_SUCCESS,
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkDisconnectReason,
-		EntryNum);
-
-		pBTInfo->BtAsocEntry[EntryNum].PhysLinkCompleteStatus = HCI_STATUS_UNKNOW_CONNECT_ID;
-
-		del_timer_sync(&pBTInfo->BTHCIJoinTimeoutTimer);
-
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-
-		break;
-	case STATE_CMD_MAC_CONNECT_CANCEL_INDICATE:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_MAC_CONNECT_CANCEL_INDICATE\n"));
-		pBTInfo->BtAsocEntry[EntryNum].PhysLinkCompleteStatus = HCI_STATUS_CONTROLLER_BUSY;
-		/*  Because this state cmd is caused by the BTHCI_EventAMPStatusChange(), */
-		/*  we don't need to send event in the following BTHCI_DisconnectPeer() again. */
-		pBtMgnt->bNeedNotifyAMPNoCap = false;
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("State command(%d) is Wrong !!!\n", StateCmd));
-		break;
-	}
-}
-
-static void
-bthci_StateConnected(struct rtw_adapter *padapter,
-		     enum hci_state_with_cmd StateCmd, u8 EntryNum)
-{
-/*PMGNT_INFO pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	u8 i;
-	u16 logicHandle = 0;
-
-	RTPRINT(FIOCTL, IOCTL_STATE, ("[BT state], [Connected], "));
-	switch (StateCmd) {
-	case STATE_CMD_DISCONNECT_PHY_LINK:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_DISCONNECT_PHY_LINK\n"));
-
-		/* When we are trying to disconnect the phy link, we should disconnect log link first, */
-		for (i = 0; i < MAX_LOGICAL_LINK_NUM; i++) {
-			if (pBTInfo->BtAsocEntry[EntryNum].LogLinkCmdData->BtLogLinkhandle != 0) {
-				logicHandle = pBTInfo->BtAsocEntry[EntryNum].LogLinkCmdData->BtLogLinkhandle;
-
-				bthci_EventDisconnectLogicalLinkComplete(padapter, HCI_STATUS_SUCCESS,
-					logicHandle, pBTInfo->BtAsocEntry[EntryNum].PhyLinkDisconnectReason);
-
-				pBTInfo->BtAsocEntry[EntryNum].LogLinkCmdData->BtLogLinkhandle = 0;
-			}
-		}
-
-		bthci_EventDisconnectPhyLinkComplete(padapter,
-		HCI_STATUS_SUCCESS,
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkDisconnectReason,
-		EntryNum);
-
-		del_timer_sync(&pBTInfo->BTHCIJoinTimeoutTimer);
-
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-		break;
-
-	case STATE_CMD_MAC_DISCONNECT_INDICATE:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_MAC_DISCONNECT_INDICATE\n"));
-
-		bthci_EventDisconnectPhyLinkComplete(padapter,
-		HCI_STATUS_SUCCESS,
-		/*  TODO: Remote Host not local host */
-		HCI_STATUS_CONNECT_TERMINATE_LOCAL_HOST,
-		EntryNum);
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-
-		break;
-	case STATE_CMD_ENTER_STATE:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_ENTER_STATE\n"));
-
-		if (pBtMgnt->bBTConnectInProgress) {
-			pBtMgnt->bBTConnectInProgress = false;
-			RTPRINT(FIOCTL, IOCTL_STATE, ("[BT Flag], BT Connect in progress OFF!!\n"));
-		}
-		pBTInfo->BtAsocEntry[EntryNum].BtCurrentState = HCI_STATE_CONNECTED;
-		pBTInfo->BtAsocEntry[EntryNum].b4waySuccess = true;
-		pBtMgnt->bStartSendSupervisionPkt = true;
-
-		/*  for rate adaptive */
-
-		rtl8723a_update_ramask(padapter,
-				       MAX_FW_SUPPORT_MACID_NUM-1-EntryNum, 0);
-
-		HalSetBrateCfg23a(padapter, padapter->mlmepriv.cur_network.network.SupportedRates);
-		BTDM_SetFwChnlInfo(padapter, RT_MEDIA_CONNECT);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("State command(%d) is Wrong !!!\n", StateCmd));
-		break;
-	}
-}
-
-static void
-bthci_StateAuth(struct rtw_adapter *padapter, enum hci_state_with_cmd StateCmd,
-		u8 EntryNum)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT(FIOCTL, IOCTL_STATE, ("[BT state], [Authenticating], "));
-	switch (StateCmd) {
-	case STATE_CMD_CONNECT_ACCEPT_TIMEOUT:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_CONNECT_ACCEPT_TIMEOUT\n"));
-		pBTInfo->BtAsocEntry[EntryNum].PhysLinkCompleteStatus = HCI_STATUS_CONNECT_ACCEPT_TIMEOUT;
-		pBtMgnt->bNeedNotifyAMPNoCap = true;
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-		break;
-	case STATE_CMD_DISCONNECT_PHY_LINK:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_DISCONNECT_PHY_LINK\n"));
-		bthci_EventDisconnectPhyLinkComplete(padapter,
-		HCI_STATUS_SUCCESS,
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkDisconnectReason,
-		EntryNum);
-
-		pBTInfo->BtAsocEntry[EntryNum].PhysLinkCompleteStatus = HCI_STATUS_UNKNOW_CONNECT_ID;
-
-		del_timer_sync(&pBTInfo->BTHCIJoinTimeoutTimer);
-
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-		break;
-	case STATE_CMD_4WAY_FAILED:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_4WAY_FAILED\n"));
-
-		pBTInfo->BtAsocEntry[EntryNum].PhysLinkCompleteStatus = HCI_STATUS_AUTH_FAIL;
-		pBtMgnt->bNeedNotifyAMPNoCap = true;
-
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-
-		del_timer_sync(&pBTInfo->BTHCIJoinTimeoutTimer);
-		break;
-	case STATE_CMD_4WAY_SUCCESSED:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_4WAY_SUCCESSED\n"));
-
-		bthci_EventPhysicalLinkComplete(padapter, HCI_STATUS_SUCCESS, EntryNum, INVALID_PL_HANDLE);
-
-		del_timer_sync(&pBTInfo->BTHCIJoinTimeoutTimer);
-
-		BTHCI_SM_WITH_INFO(padapter, HCI_STATE_CONNECTED, STATE_CMD_ENTER_STATE, EntryNum);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("State command(%d) is Wrong !!!\n", StateCmd));
-		break;
-	}
-}
-
-static void
-bthci_StateDisconnecting(struct rtw_adapter *padapter,
-			 enum hci_state_with_cmd StateCmd, u8 EntryNum)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT(FIOCTL, IOCTL_STATE, ("[BT state], [Disconnecting], "));
-	switch (StateCmd) {
-	case STATE_CMD_MAC_CONNECT_CANCEL_INDICATE:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_MAC_CONNECT_CANCEL_INDICATE\n"));
-		if (pBTInfo->BtAsocEntry[EntryNum].bNeedPhysLinkCompleteEvent) {
-			bthci_EventPhysicalLinkComplete(padapter,
-				pBTInfo->BtAsocEntry[EntryNum].PhysLinkCompleteStatus,
-				EntryNum, INVALID_PL_HANDLE);
-		}
-
-		if (pBtMgnt->bBTConnectInProgress) {
-			pBtMgnt->bBTConnectInProgress = false;
-			RTPRINT(FIOCTL, IOCTL_STATE, ("[BT Flag], BT Connect in progress OFF!!\n"));
-		}
-
-		BTHCI_SM_WITH_INFO(padapter, HCI_STATE_DISCONNECTED, STATE_CMD_ENTER_STATE, EntryNum);
-		break;
-	case STATE_CMD_DISCONNECT_PHY_LINK:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_DISCONNECT_PHY_LINK\n"));
-
-		bthci_EventDisconnectPhyLinkComplete(padapter,
-		HCI_STATUS_SUCCESS,
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkDisconnectReason,
-		EntryNum);
-
-		del_timer_sync(&pBTInfo->BTHCIJoinTimeoutTimer);
-
-		BTHCI_DisconnectPeer(padapter, EntryNum);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("State command(%d) is Wrong !!!\n", StateCmd));
-		break;
-	}
-}
-
-static void
-bthci_StateDisconnected(struct rtw_adapter *padapter,
-			enum hci_state_with_cmd StateCmd, u8 EntryNum)
-{
-/*PMGNT_INFO pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT(FIOCTL, IOCTL_STATE, ("[BT state], [Disconnected], "));
-	switch (StateCmd) {
-	case STATE_CMD_CREATE_PHY_LINK:
-	case STATE_CMD_ACCEPT_PHY_LINK:
-		if (StateCmd == STATE_CMD_CREATE_PHY_LINK)
-			RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_CREATE_PHY_LINK\n"));
-		else
-			RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_ACCEPT_PHY_LINK\n"));
-
-		RTPRINT(FIOCTL, IOCTL_STATE, ("[BT PS], Disable IPS and LPS\n"));
-		ips_leave23a(padapter);
-		LPS_Leave23a(padapter);
-
-		pBtMgnt->bPhyLinkInProgress = true;
-		pBtMgnt->BTCurrentConnectType = BT_DISCONNECT;
-		pBtMgnt->CurrentBTConnectionCnt++;
-		RTPRINT(FIOCTL, IOCTL_STATE, ("[BT Flag], CurrentBTConnectionCnt = %d\n",
-			pBtMgnt->CurrentBTConnectionCnt));
-		pBtMgnt->BtOperationOn = true;
-		RTPRINT(FIOCTL, IOCTL_STATE, ("[BT Flag], Bt Operation ON!! CurrentConnectEntryNum = %d\n",
-			pBtMgnt->CurrentConnectEntryNum));
-
-		if (pBtMgnt->bBTConnectInProgress) {
-			bthci_EventPhysicalLinkComplete(padapter, HCI_STATUS_CONTROLLER_BUSY, INVALID_ENTRY_NUM, pBtMgnt->BtCurrentPhyLinkhandle);
-			bthci_RemoveEntryByEntryNum(padapter, EntryNum);
-			return;
-		}
-
-		if (StateCmd == STATE_CMD_CREATE_PHY_LINK)
-			pBTInfo->BtAsocEntry[EntryNum].AMPRole = AMP_BTAP_CREATOR;
-		else
-			pBTInfo->BtAsocEntry[EntryNum].AMPRole = AMP_BTAP_JOINER;
-
-		/*  1. MAC not yet in selected channel */
-		while (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE|WIFI_SITE_MONITOR)) {
-			RTPRINT(FIOCTL, IOCTL_STATE, ("Scan/Roaming/Wifi Link is in Progress, wait 200 ms\n"));
-			mdelay(200);
-		}
-		/*  2. MAC already in selected channel */
-		RTPRINT(FIOCTL, IOCTL_STATE, ("Channel is Ready\n"));
-		mod_timer(&pBTInfo->BTHCIJoinTimeoutTimer,
-			  jiffies + msecs_to_jiffies(pBtHciInfo->ConnAcceptTimeout));
-
-		pBTInfo->BtAsocEntry[EntryNum].bNeedPhysLinkCompleteEvent = true;
-		break;
-	case STATE_CMD_DISCONNECT_PHY_LINK:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_DISCONNECT_PHY_LINK\n"));
-
-		del_timer_sync(&pBTInfo->BTHCIJoinTimeoutTimer);
-
-		bthci_EventDisconnectPhyLinkComplete(padapter,
-		HCI_STATUS_SUCCESS,
-		pBTInfo->BtAsocEntry[EntryNum].PhyLinkDisconnectReason,
-		EntryNum);
-
-		if (pBTInfo->BtAsocEntry[EntryNum].bNeedPhysLinkCompleteEvent) {
-			bthci_EventPhysicalLinkComplete(padapter,
-				HCI_STATUS_UNKNOW_CONNECT_ID,
-				EntryNum, INVALID_PL_HANDLE);
-		}
-
-		if (pBtMgnt->bBTConnectInProgress) {
-			pBtMgnt->bBTConnectInProgress = false;
-			RTPRINT(FIOCTL, IOCTL_STATE, ("[BT Flag], BT Connect in progress OFF!!\n"));
-		}
-		BTHCI_SM_WITH_INFO(padapter, HCI_STATE_DISCONNECTED, STATE_CMD_ENTER_STATE, EntryNum);
-		bthci_RemoveEntryByEntryNum(padapter, EntryNum);
-		break;
-	case STATE_CMD_ENTER_STATE:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("STATE_CMD_ENTER_STATE\n"));
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_STATE, ("State command(%d) is Wrong !!!\n", StateCmd));
-		break;
-	}
-}
-
-void BTHCI_EventParse(struct rtw_adapter *padapter, void *pEvntData, u32 dataLen)
-{
-}
-
-u8 BTHCI_HsConnectionEstablished(struct rtw_adapter *padapter)
-{
-	u8 bBtConnectionExist = false;
-	struct bt_30info *pBtinfo = GET_BT_INFO(padapter);
-	u8 i;
-
-	for (i = 0; i < MAX_BT_ASOC_ENTRY_NUM; i++) {
-		if (pBtinfo->BtAsocEntry[i].b4waySuccess) {
-			bBtConnectionExist = true;
-			break;
-		}
-	}
-
-/*RTPRINT(FIOCTL, IOCTL_STATE, (" BTHCI_HsConnectionEstablished(), connection exist = %d\n", bBtConnectionExist)); */
-
-	return bBtConnectionExist;
-}
-
-static u8
-BTHCI_CheckProfileExist(struct rtw_adapter *padapter,
-			enum bt_traffic_mode_profile Profile)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	u8 IsPRofile = false;
-	u8 i = 0;
-
-	for (i = 0; i < pBtMgnt->ExtConfig.NumberOfHandle; i++) {
-		if (pBtMgnt->ExtConfig.linkInfo[i].TrafficProfile == Profile) {
-			IsPRofile = true;
-			break;
-		}
-	}
-
-	return IsPRofile;
-}
-
-void BTHCI_UpdateBTProfileRTKToMoto(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	u8 i = 0;
-
-	pBtMgnt->ExtConfig.NumberOfSCO = 0;
-
-	for (i = 0; i < pBtMgnt->ExtConfig.NumberOfHandle; i++) {
-		pBtMgnt->ExtConfig.linkInfo[i].TrafficProfile = BT_PROFILE_NONE;
-
-		if (pBtMgnt->ExtConfig.linkInfo[i].BTProfile == BT_PROFILE_SCO)
-			pBtMgnt->ExtConfig.NumberOfSCO++;
-
-		pBtMgnt->ExtConfig.linkInfo[i].TrafficProfile = pBtMgnt->ExtConfig.linkInfo[i].BTProfile;
-		switch (pBtMgnt->ExtConfig.linkInfo[i].TrafficProfile) {
-		case BT_PROFILE_SCO:
-			break;
-		case BT_PROFILE_PAN:
-			pBtMgnt->ExtConfig.linkInfo[i].IncomingTrafficMode = BT_MOTOR_EXT_BE;
-			pBtMgnt->ExtConfig.linkInfo[i].OutgoingTrafficMode = BT_MOTOR_EXT_BE;
-			break;
-		case BT_PROFILE_A2DP:
-			pBtMgnt->ExtConfig.linkInfo[i].IncomingTrafficMode = BT_MOTOR_EXT_GULB;
-			pBtMgnt->ExtConfig.linkInfo[i].OutgoingTrafficMode = BT_MOTOR_EXT_GULB;
-			break;
-		case BT_PROFILE_HID:
-			pBtMgnt->ExtConfig.linkInfo[i].IncomingTrafficMode = BT_MOTOR_EXT_GUL;
-			pBtMgnt->ExtConfig.linkInfo[i].OutgoingTrafficMode = BT_MOTOR_EXT_BE;
-			break;
-		default:
-			break;
-		}
-	}
-
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], RTK, NumberOfHandle = %d, NumberOfSCO = %d\n",
-		pBtMgnt->ExtConfig.NumberOfHandle, pBtMgnt->ExtConfig.NumberOfSCO));
-}
-
-void BTHCI_WifiScanNotify(struct rtw_adapter *padapter, u8 scanType)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bEnableWifiScanNotify)
-		bthci_EventExtWifiScanNotify(padapter, scanType);
-}
-
-void
-BTHCI_StateMachine(
-	struct rtw_adapter *padapter,
-	u8 		StateToEnter,
-	enum hci_state_with_cmd		StateCmd,
-	u8 		EntryNum
-	)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (EntryNum == 0xff) {
-		RTPRINT(FIOCTL, IOCTL_STATE, (" StateMachine, error EntryNum = 0x%x \n", EntryNum));
-		return;
-	}
-	RTPRINT(FIOCTL, IOCTL_STATE, (" StateMachine, EntryNum = 0x%x, CurrentState = 0x%x, BtNextState = 0x%x,  StateCmd = 0x%x , StateToEnter = 0x%x\n",
-		EntryNum, pBTInfo->BtAsocEntry[EntryNum].BtCurrentState, pBTInfo->BtAsocEntry[EntryNum].BtNextState, StateCmd, StateToEnter));
-
-	if (pBTInfo->BtAsocEntry[EntryNum].BtNextState & StateToEnter) {
-		pBTInfo->BtAsocEntry[EntryNum].BtCurrentState = StateToEnter;
-
-		switch (StateToEnter) {
-		case HCI_STATE_STARTING:
-			pBTInfo->BtAsocEntry[EntryNum].BtNextState = HCI_STATE_DISCONNECTING | HCI_STATE_CONNECTING;
-			bthci_StateStarting(padapter, StateCmd, EntryNum);
-			break;
-		case HCI_STATE_CONNECTING:
-			pBTInfo->BtAsocEntry[EntryNum].BtNextState = HCI_STATE_CONNECTING | HCI_STATE_DISCONNECTING | HCI_STATE_AUTHENTICATING;
-			bthci_StateConnecting(padapter, StateCmd, EntryNum);
-			break;
-		case HCI_STATE_AUTHENTICATING:
-			pBTInfo->BtAsocEntry[EntryNum].BtNextState = HCI_STATE_DISCONNECTING | HCI_STATE_CONNECTED;
-			bthci_StateAuth(padapter, StateCmd, EntryNum);
-			break;
-		case HCI_STATE_CONNECTED:
-			pBTInfo->BtAsocEntry[EntryNum].BtNextState = HCI_STATE_CONNECTED | HCI_STATE_DISCONNECTING;
-			bthci_StateConnected(padapter, StateCmd, EntryNum);
-			break;
-		case HCI_STATE_DISCONNECTING:
-			pBTInfo->BtAsocEntry[EntryNum].BtNextState = HCI_STATE_DISCONNECTED | HCI_STATE_DISCONNECTING;
-			bthci_StateDisconnecting(padapter, StateCmd, EntryNum);
-			break;
-		case HCI_STATE_DISCONNECTED:
-			pBTInfo->BtAsocEntry[EntryNum].BtNextState = HCI_STATE_DISCONNECTED | HCI_STATE_STARTING | HCI_STATE_CONNECTING;
-			bthci_StateDisconnected(padapter, StateCmd, EntryNum);
-			break;
-		default:
-			RTPRINT(FIOCTL, IOCTL_STATE, (" StateMachine, Unknown state to enter!!!\n"));
-			break;
-		}
-	} else {
-		RTPRINT(FIOCTL, IOCTL_STATE, (" StateMachine, Wrong state to enter\n"));
-	}
-
-	/*  20100325 Joseph: Disable/Enable IPS/LPS according to BT status. */
-	if (!pBtMgnt->bBTConnectInProgress && !pBtMgnt->BtOperationOn) {
-		RTPRINT(FIOCTL, IOCTL_STATE, ("[BT PS], ips_enter23a()\n"));
-		ips_enter23a(padapter);
-	}
-}
-
-void BTHCI_DisconnectPeer(struct rtw_adapter *padapter, u8 EntryNum)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, (" BTHCI_DisconnectPeer()\n"));
-
-	BTHCI_SM_WITH_INFO(padapter, HCI_STATE_DISCONNECTING, STATE_CMD_MAC_CONNECT_CANCEL_INDICATE, EntryNum);
-
-	if (pBTInfo->BtAsocEntry[EntryNum].bUsed) {
-/*BTPKT_SendDeauthentication(padapter, pBTInfo->BtAsocEntry[EntryNum].BTRemoteMACAddr, unspec_reason); not porting yet */
-	}
-
-	if (pBtMgnt->bBTConnectInProgress) {
-		pBtMgnt->bBTConnectInProgress = false;
-		RTPRINT(FIOCTL, IOCTL_STATE, ("[BT Flag], BT Connect in progress OFF!!\n"));
-	}
-
-	bthci_RemoveEntryByEntryNum(padapter, EntryNum);
-
-	if (pBtMgnt->bNeedNotifyAMPNoCap) {
-		RTPRINT(FIOCTL, IOCTL_STATE, ("[BT AMPStatus], set to invalid in BTHCI_DisconnectPeer()\n"));
-		BTHCI_EventAMPStatusChange(padapter, AMP_STATUS_NO_CAPACITY_FOR_BT);
-	}
-}
-
-void BTHCI_EventNumOfCompletedDataBlocks(struct rtw_adapter *padapter)
-{
-/*PMGNT_INFO pMgntInfo = &padapter->MgntInfo; */
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_hci_info *pBtHciInfo = &pBTInfo->BtHciInfo;
-	u8 localBuf[TmpLocalBufSize] = "";
-	u8 *pRetPar, *pTriple;
-	u8 len = 0, i, j, handleNum = 0;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u16 *pu2Temp, *pPackets, *pHandle, *pDblocks;
-	u8 sent = 0;
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-
-	if (!(pBtHciInfo->BTEventMaskPage2 & EMP2_HCI_EVENT_NUM_OF_COMPLETE_DATA_BLOCKS)) {
-		RTPRINT(FIOCTL, IOCTL_BT_EVENT, ("[BT event], Num Of Completed DataBlocks, Ignore to send NumOfCompletedDataBlocksEvent due to event mask page 2\n"));
-		return;
-	}
-
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[0];
-	pTriple = &pRetPar[3];
-	for (j = 0; j < MAX_BT_ASOC_ENTRY_NUM; j++) {
-
-		for (i = 0; i < MAX_LOGICAL_LINK_NUM; i++) {
-			if (pBTInfo->BtAsocEntry[j].LogLinkCmdData[i].BtLogLinkhandle) {
-				handleNum++;
-				pHandle = (u16 *)&pTriple[0];	/*  Handle[i] */
-				pPackets = (u16 *)&pTriple[2];	/*  Num_Of_Completed_Packets[i] */
-				pDblocks = (u16 *)&pTriple[4];	/*  Num_Of_Completed_Blocks[i] */
-				*pHandle = pBTInfo->BtAsocEntry[j].LogLinkCmdData[i].BtLogLinkhandle;
-				*pPackets = (u16)pBTInfo->BtAsocEntry[j].LogLinkCmdData[i].TxPacketCount;
-				*pDblocks = (u16)pBTInfo->BtAsocEntry[j].LogLinkCmdData[i].TxPacketCount;
-				if (pBTInfo->BtAsocEntry[j].LogLinkCmdData[i].TxPacketCount) {
-					sent = 1;
-					RTPRINT(FIOCTL, IOCTL_BT_EVENT_DETAIL,
-						("[BT event], Num Of Completed DataBlocks, Handle = 0x%x, Num_Of_Completed_Packets = 0x%x, Num_Of_Completed_Blocks = 0x%x\n",
-					*pHandle, *pPackets, *pDblocks));
-				}
-				pBTInfo->BtAsocEntry[j].LogLinkCmdData[i].TxPacketCount = 0;
-				len += 6;
-				pTriple += len;
-			}
-		}
-	}
-
-	pRetPar[2] = handleNum;				/*  Number_of_Handles */
-	len += 1;
-	pu2Temp = (u16 *)&pRetPar[0];
-	*pu2Temp = BTTotalDataBlockNum;
-	len += 2;
-
-	PPacketIrpEvent->EventCode = HCI_EVENT_NUM_OF_COMPLETE_DATA_BLOCKS;
-	PPacketIrpEvent->Length = len;
-	if (handleNum && sent)
-		bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2);
-}
-
-void BTHCI_EventAMPStatusChange(struct rtw_adapter *padapter, u8 AMP_Status)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct packet_irp_hcievent_data *PPacketIrpEvent;
-	u8 len = 0;
-	u8 localBuf[7] = "";
-	u8 *pRetPar;
-
-	if (AMP_Status == AMP_STATUS_NO_CAPACITY_FOR_BT) {
-		pBtMgnt->BTNeedAMPStatusChg = true;
-		pBtMgnt->bNeedNotifyAMPNoCap = false;
-
-		BTHCI_DisconnectAll(padapter);
-	} else if (AMP_Status == AMP_STATUS_FULL_CAPACITY_FOR_BT) {
-		pBtMgnt->BTNeedAMPStatusChg = false;
-	}
-
-	PPacketIrpEvent = (struct packet_irp_hcievent_data *)(&localBuf[0]);
-	/*  Return parameters starts from here */
-	pRetPar = &PPacketIrpEvent->Data[0];
-
-	pRetPar[0] = 0;	/*  Status */
-	len += 1;
-	pRetPar[1] = AMP_Status;	/*  AMP_Status */
-	len += 1;
-
-	PPacketIrpEvent->EventCode = HCI_EVENT_AMP_STATUS_CHANGE;
-	PPacketIrpEvent->Length = len;
-	if (bthci_IndicateEvent(padapter, PPacketIrpEvent, len+2) == RT_STATUS_SUCCESS)
-		RTPRINT(FIOCTL, (IOCTL_BT_EVENT|IOCTL_STATE), ("[BT event], AMP Status Change, AMP_Status = %d\n", AMP_Status));
-}
-
-void BTHCI_DisconnectAll(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	u8 i;
-
-	RTPRINT(FIOCTL, IOCTL_STATE, (" DisconnectALL()\n"));
-
-	for (i = 0; i < MAX_BT_ASOC_ENTRY_NUM; i++) {
-		if (pBTInfo->BtAsocEntry[i].b4waySuccess) {
-			BTHCI_SM_WITH_INFO(padapter, HCI_STATE_CONNECTED, STATE_CMD_DISCONNECT_PHY_LINK, i);
-		} else if (pBTInfo->BtAsocEntry[i].bUsed) {
-			if (pBTInfo->BtAsocEntry[i].BtCurrentState == HCI_STATE_CONNECTING) {
-				BTHCI_SM_WITH_INFO(padapter, HCI_STATE_CONNECTING, STATE_CMD_MAC_CONNECT_CANCEL_INDICATE, i);
-			} else if (pBTInfo->BtAsocEntry[i].BtCurrentState == HCI_STATE_DISCONNECTING) {
-				BTHCI_SM_WITH_INFO(padapter, HCI_STATE_DISCONNECTING, STATE_CMD_MAC_CONNECT_CANCEL_INDICATE, i);
-			}
-		}
-	}
-}
-
-enum hci_status
-BTHCI_HandleHCICMD(
-	struct rtw_adapter *padapter,
-	struct packet_irp_hcicmd_data *pHciCmd
-	)
-{
-	enum hci_status	status = HCI_STATUS_SUCCESS;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("\n"));
-	RTPRINT(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), ("HCI Command start, OGF = 0x%x, OCF = 0x%x, Length = 0x%x\n",
-		pHciCmd->OGF, pHciCmd->OCF, pHciCmd->Length));
-	if (pHciCmd->Length) {
-		RTPRINT_DATA(FIOCTL, (IOCTL_BT_HCICMD_DETAIL|IOCTL_BT_LOGO), "HCI Command, Hex Data :\n",
-			&pHciCmd->Data[0], pHciCmd->Length);
-	}
-	if (pHciCmd->OGF == OGF_EXTENSION) {
-		if (pHciCmd->OCF == HCI_SET_RSSI_VALUE)
-			RTPRINT(FIOCTL, IOCTL_BT_EVENT_PERIODICAL, ("[BT cmd], "));
-		else
-			RTPRINT(FIOCTL, IOCTL_BT_HCICMD_EXT, ("[BT cmd], "));
-	} else {
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("[BT cmd], "));
-	}
-
-	pBtDbg->dbgHciInfo.hciCmdCnt++;
-
-	switch (pHciCmd->OGF) {
-	case LINK_CONTROL_COMMANDS:
-		status = bthci_HandleOGFLinkControlCMD(padapter, pHciCmd);
-		break;
-	case HOLD_MODE_COMMAND:
-		break;
-	case OGF_SET_EVENT_MASK_COMMAND:
-		status = bthci_HandleOGFSetEventMaskCMD(padapter, pHciCmd);
-		break;
-	case OGF_INFORMATIONAL_PARAMETERS:
-		status = bthci_HandleOGFInformationalParameters(padapter, pHciCmd);
-		break;
-	case OGF_STATUS_PARAMETERS:
-		status = bthci_HandleOGFStatusParameters(padapter, pHciCmd);
-		break;
-	case OGF_TESTING_COMMANDS:
-		status = bthci_HandleOGFTestingCMD(padapter, pHciCmd);
-		break;
-	case OGF_EXTENSION:
-		status = bthci_HandleOGFExtension(padapter, pHciCmd);
-		break;
-	default:
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI Command(), Unknown OGF = 0x%x\n", pHciCmd->OGF));
-		RTPRINT(FIOCTL, IOCTL_BT_HCICMD, ("HCI_UNKNOWN_COMMAND\n"));
-		status = bthci_UnknownCMD(padapter, pHciCmd);
-		break;
-	}
-	RTPRINT(FIOCTL, IOCTL_BT_HCICMD_DETAIL, ("HCI Command execution end!!\n"));
-
-	return status;
-}
-
-/*  ===== End of sync from SD7 driver COMMOM/bt_hci.c ===== */
-
-static const char *const BtStateString[] = {
-	"BT_DISABLED",
-	"BT_NO_CONNECTION",
-	"BT_CONNECT_IDLE",
-	"BT_INQ_OR_PAG",
-	"BT_ACL_ONLY_BUSY",
-	"BT_SCO_ONLY_BUSY",
-	"BT_ACL_SCO_BUSY",
-	"BT_ACL_INQ_OR_PAG",
-	"BT_STATE_NOT_DEFINED"
-};
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtc87231Ant.c ===== */
-
-static void btdm_SetFwIgnoreWlanAct(struct rtw_adapter *padapter, u8 bEnable)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 H2C_Parameter[1] = {0};
-
-	if (bEnable) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT Ignore Wlan_Act !!\n"));
-		H2C_Parameter[0] |= BIT(0);		/*  function enable */
-		pHalData->bt_coexist.bFWCoexistAllOff = false;
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT don't ignore Wlan_Act !!\n"));
-	}
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], set FW for BT Ignore Wlan_Act, write 0x25 = 0x%02x\n",
-		H2C_Parameter[0]));
-
-	FillH2CCmd(padapter, BT_IGNORE_WLAN_ACT_EID, 1, H2C_Parameter);
-}
-
-static void btdm_NotifyFwScan(struct rtw_adapter *padapter, u8 scanType)
-{
-	u8 H2C_Parameter[1] = {0};
-
-	if (scanType == true)
-		H2C_Parameter[0] = 0x1;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], Notify FW for wifi scan, write 0x3b = 0x%02x\n",
-		H2C_Parameter[0]));
-
-	FillH2CCmd(padapter, 0x3b, 1, H2C_Parameter);
-}
-
-static void btdm_1AntSetPSMode(struct rtw_adapter *padapter,
-			       u8 enable, u8 smartps, u8 mode)
-{
-	struct pwrctrl_priv *pwrctrl;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], Current LPS(%s, %d), smartps =%d\n", enable == true?"ON":"OFF", mode, smartps));
-
-	pwrctrl = &padapter->pwrctrlpriv;
-
-	if (enable == true) {
-		rtw_set_ps_mode23a(padapter, PS_MODE_MIN, smartps, mode);
-	} else {
-		rtw_set_ps_mode23a(padapter, PS_MODE_ACTIVE, 0, 0);
-		LPS_RF_ON_check23a(padapter, 100);
-	}
-}
-
-static void btdm_1AntTSFSwitch(struct rtw_adapter *padapter, u8 enable)
-{
-	u8 oldVal, newVal;
-
-	oldVal = rtl8723au_read8(padapter, 0x550);
-
-	if (enable)
-		newVal = oldVal | EN_BCN_FUNCTION;
-	else
-		newVal = oldVal & ~EN_BCN_FUNCTION;
-
-	if (oldVal != newVal)
-		rtl8723au_write8(padapter, 0x550, newVal);
-}
-
-static u8 btdm_Is1AntPsTdmaStateChange(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_1ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm1Ant;
-
-	if ((pBtdm8723->bPrePsTdmaOn != pBtdm8723->bCurPsTdmaOn) ||
-		(pBtdm8723->prePsTdma != pBtdm8723->curPsTdma))
-		return true;
-	else
-		return false;
-}
-
-/*  Before enter TDMA, make sure Power Saving is enable! */
-static void
-btdm_1AntPsTdma(
-	struct rtw_adapter *padapter,
-	u8 bTurnOn,
-	u8 type
-	)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_1ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm1Ant;
-
-	pBtdm8723->bCurPsTdmaOn = bTurnOn;
-	pBtdm8723->curPsTdma = type;
-	if (bTurnOn) {
-		switch (type) {
-		case 1:	/*  A2DP Level-1 or FTP/OPP */
-		default:
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  wide duration for WiFi */
-				BTDM_SetFw3a(padapter, 0xd3, 0x1a, 0x1a, 0x0, 0x58);
-			}
-			break;
-		case 2:	/*  A2DP Level-2 */
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  normal duration for WiFi */
-				BTDM_SetFw3a(padapter, 0xd3, 0x12, 0x12, 0x0, 0x58);
-			}
-			break;
-		case 3:	/*  BT FTP/OPP */
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  normal duration for WiFi */
-				BTDM_SetFw3a(padapter, 0xd3, 0x30, 0x03, 0x10, 0x58);
-
-			}
-			break;
-		case 4:	/*  for wifi scan & BT is connected */
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  protect 3 beacons in 3-beacon period & no Tx pause at BT slot */
-				BTDM_SetFw3a(padapter, 0x93, 0x15, 0x03, 0x14, 0x0);
-			}
-			break;
-		case 5:	/*  for WiFi connected-busy & BT is Non-Connected-Idle */
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  SCO mode, Ant fixed at WiFi, WLAN_Act toggle */
-				BTDM_SetFw3a(padapter, 0x61, 0x15, 0x03, 0x31, 0x00);
-			}
-			break;
-		case 9:	/*  ACL high-retry type - 2 */
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  narrow duration for WiFi */
-				BTDM_SetFw3a(padapter, 0xd3, 0xa, 0xa, 0x0, 0x58); /* narrow duration for WiFi */
-			}
-			break;
-		case 10: /*  for WiFi connect idle & BT ACL busy or WiFi Connected-Busy & BT is Inquiry */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0x13, 0xa, 0xa, 0x0, 0x40);
-			break;
-		case 11: /*  ACL high-retry type - 3 */
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  narrow duration for WiFi */
-				BTDM_SetFw3a(padapter, 0xd3, 0x05, 0x05, 0x00, 0x58);
-			}
-			break;
-		case 12: /*  for WiFi Connected-Busy & BT is Connected-Idle */
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  Allow High-Pri BT */
-				BTDM_SetFw3a(padapter, 0xeb, 0x0a, 0x03, 0x31, 0x18);
-			}
-			break;
-		case 20: /*  WiFi only busy , TDMA mode for power saving */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0x13, 0x25, 0x25, 0x00, 0x00);
-			break;
-		case 27: /*  WiFi DHCP/Site Survey & BT SCO busy */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0xa3, 0x25, 0x03, 0x31, 0x98);
-			break;
-		case 28: /*  WiFi DHCP/Site Survey & BT idle */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0x69, 0x25, 0x03, 0x31, 0x00);
-			break;
-		case 29: /*  WiFi DHCP/Site Survey & BT ACL busy */
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				BTDM_SetFw3a(padapter, 0xeb, 0x1a, 0x1a, 0x01, 0x18);
-				rtl8723au_write32(padapter, 0x6c0, 0x5afa5afa);
-				rtl8723au_write32(padapter, 0x6c4, 0x5afa5afa);
-			}
-			break;
-		case 30: /*  WiFi idle & BT Inquiry */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0x93, 0x15, 0x03, 0x14, 0x00);
-			break;
-		case 31:  /*  BT HID */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0xd3, 0x1a, 0x1a, 0x00, 0x58);
-			break;
-		case 32:  /*  BT SCO & Inquiry */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0xab, 0x0a, 0x03, 0x11, 0x98);
-			break;
-		case 33:  /*  BT SCO & WiFi site survey */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0xa3, 0x25, 0x03, 0x30, 0x98);
-			break;
-		case 34:  /*  BT HID & WiFi site survey */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0xd3, 0x1a, 0x1a, 0x00, 0x18);
-			break;
-		case 35:  /*  BT HID & WiFi Connecting */
-			if (btdm_Is1AntPsTdmaStateChange(padapter))
-				BTDM_SetFw3a(padapter, 0xe3, 0x1a, 0x1a, 0x00, 0x18);
-			break;
-		}
-	} else {
-		/*  disable PS-TDMA */
-		switch (type) {
-		case 8:
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  Antenna control by PTA, 0x870 = 0x310 */
-				BTDM_SetFw3a(padapter, 0x8, 0x0, 0x0, 0x0, 0x0);
-			}
-			break;
-		case 0:
-		default:
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  Antenna control by PTA, 0x870 = 0x310 */
-				BTDM_SetFw3a(padapter, 0x0, 0x0, 0x0, 0x8, 0x0);
-			}
-			/*  Switch Antenna to BT */
-			rtl8723au_write16(padapter, 0x860, 0x210);
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], 0x860 = 0x210, Switch Antenna to BT\n"));
-			break;
-		case 9:
-			if (btdm_Is1AntPsTdmaStateChange(padapter)) {
-				/*  Antenna control by PTA, 0x870 = 0x310 */
-				BTDM_SetFw3a(padapter, 0x0, 0x0, 0x0, 0x8, 0x0);
-			}
-			/*  Switch Antenna to WiFi */
-			rtl8723au_write16(padapter, 0x860, 0x110);
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], 0x860 = 0x110, Switch Antenna to WiFi\n"));
-			break;
-		}
-	}
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], Current TDMA(%s, %d)\n",
-		pBtdm8723->bCurPsTdmaOn?"ON":"OFF", pBtdm8723->curPsTdma));
-
-	/*  update pre state */
-	pBtdm8723->bPrePsTdmaOn = pBtdm8723->bCurPsTdmaOn;
-	pBtdm8723->prePsTdma = pBtdm8723->curPsTdma;
-}
-
-static void
-_btdm_1AntSetPSTDMA(struct rtw_adapter *padapter, u8 bPSEn, u8 smartps,
-		    u8 psOption, u8 bTDMAOn, u8 tdmaType)
-{
-	struct pwrctrl_priv *pwrctrl;
-	struct hal_data_8723a *pHalData;
-	struct btdm_8723a_1ant *pBtdm8723;
-	u8 psMode;
-	u8 bSwitchPS;
-
-	if (!check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE) &&
-	    (get_fwstate(&padapter->mlmepriv) != WIFI_NULL_STATE)) {
-		btdm_1AntPsTdma(padapter, bTDMAOn, tdmaType);
-		return;
-	}
-	psOption &= ~BIT(0);
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTCoex], Set LPS(%s, %d) TDMA(%s, %d)\n",
-		bPSEn == true?"ON":"OFF", psOption,
-		bTDMAOn == true?"ON":"OFF", tdmaType));
-
-	pwrctrl = &padapter->pwrctrlpriv;
-	pHalData = GET_HAL_DATA(padapter);
-	pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm1Ant;
-
-	if (bPSEn) {
-		if (pBtdm8723->bWiFiHalt) {
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], Enable PS Fail, WiFi in Halt!!\n"));
-			return;
-		}
-
-		if (pwrctrl->bInSuspend) {
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], Enable PS Fail, WiFi in Suspend!!\n"));
-			return;
-		}
-
-		if (padapter->bDriverStopped) {
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], Enable PS Fail, WiFi driver stopped!!\n"));
-			return;
-		}
-
-		if (padapter->bSurpriseRemoved) {
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], Enable PS Fail, WiFi Surprise Removed!!\n"));
-			return;
-		}
-
-		psMode = PS_MODE_MIN;
-	} else {
-		psMode = PS_MODE_ACTIVE;
-		psOption = 0;
-	}
-
-	if (psMode != pwrctrl->pwr_mode) {
-		bSwitchPS = true;
-	} else if (psMode != PS_MODE_ACTIVE) {
-		if (psOption != pwrctrl->bcn_ant_mode)
-			bSwitchPS = true;
-		else if (smartps != pwrctrl->smart_ps)
-			bSwitchPS = true;
-		else
-			bSwitchPS = false;
-	} else {
-		bSwitchPS = false;
-	}
-
-	if (bSwitchPS) {
-		/*  disable TDMA */
-		if (pBtdm8723->bCurPsTdmaOn) {
-			if (!bTDMAOn) {
-				btdm_1AntPsTdma(padapter, false, tdmaType);
-			} else {
-				if (!rtl8723a_BT_enabled(padapter) ||
-				    (pHalData->bt_coexist.halCoex8723.c2hBtInfo == BT_INFO_STATE_NO_CONNECTION) ||
-				    (pHalData->bt_coexist.halCoex8723.c2hBtInfo == BT_INFO_STATE_CONNECT_IDLE) ||
-				    (tdmaType == 29))
-					btdm_1AntPsTdma(padapter, false, 9);
-				else
-					btdm_1AntPsTdma(padapter, false, 0);
-			}
-		}
-
-		/*  change Power Save State */
-		btdm_1AntSetPSMode(padapter, bPSEn, smartps, psOption);
-	}
-
-	btdm_1AntPsTdma(padapter, bTDMAOn, tdmaType);
-}
-
-static void
-btdm_1AntSetPSTDMA(struct rtw_adapter *padapter, u8 bPSEn,
-		   u8 psOption, u8 bTDMAOn, u8 tdmaType)
-{
-	_btdm_1AntSetPSTDMA(padapter, bPSEn, 0, psOption, bTDMAOn, tdmaType);
-}
-
-static void btdm_1AntWifiParaAdjust(struct rtw_adapter *padapter, u8 bEnable)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_1ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm1Ant;
-
-	if (bEnable) {
-		pBtdm8723->curWifiPara = 1;
-		if (pBtdm8723->preWifiPara != pBtdm8723->curWifiPara)
-			BTDM_SetSwPenaltyTxRateAdaptive(padapter, BT_TX_RATE_ADAPTIVE_LOW_PENALTY);
-	} else {
-		pBtdm8723->curWifiPara = 2;
-		if (pBtdm8723->preWifiPara != pBtdm8723->curWifiPara)
-			BTDM_SetSwPenaltyTxRateAdaptive(padapter, BT_TX_RATE_ADAPTIVE_NORMAL);
-	}
-
-}
-
-static void btdm_1AntPtaParaReload(struct rtw_adapter *padapter)
-{
-	/*  PTA parameter */
-	rtl8723au_write8(padapter, 0x6cc, 0x0);		/*  1-Ant coex */
-	rtl8723au_write32(padapter, 0x6c8, 0xffff);	/*  wifi break table */
-	rtl8723au_write32(padapter, 0x6c4, 0x55555555);	/*  coex table */
-
-	/*  Antenna switch control parameter */
-	rtl8723au_write32(padapter, 0x858, 0xaaaaaaaa);
-	if (IS_8723A_A_CUT(GET_HAL_DATA(padapter)->VersionID)) {
-		/*  SPDT(connected with TRSW) control by hardware PTA */
-		rtl8723au_write32(padapter, 0x870, 0x0);
-		rtl8723au_write8(padapter, 0x40, 0x24);
-	} else {
-		rtl8723au_write8(padapter, 0x40, 0x20);
-		/*  set antenna at bt side if ANTSW is software control */
-		rtl8723au_write16(padapter, 0x860, 0x210);
-		/*  SPDT(connected with TRSW) control by hardware PTA */
-		rtl8723au_write32(padapter, 0x870, 0x300);
-		/*  ANTSW keep by GNT_BT */
-		rtl8723au_write32(padapter, 0x874, 0x22804000);
-	}
-
-	/*  coexistence parameters */
-	rtl8723au_write8(padapter, 0x778, 0x1);	/*  enable RTK mode PTA */
-
-	/*  BT don't ignore WLAN_Act */
-	btdm_SetFwIgnoreWlanAct(padapter, false);
-}
-
-/*
- * Return
- *1: upgrade (add WiFi duration time)
- *0: keep
- *-1: downgrade (add BT duration time)
- */
-static s8 btdm_1AntTdmaJudgement(struct rtw_adapter *padapter, u8 retry)
-{
-	struct hal_data_8723a *pHalData;
-	struct btdm_8723a_1ant *pBtdm8723;
-	static s8 up, dn, m = 1, WaitCount;
-	s8 ret;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm1Ant;
-	ret = 0;
-
-	if (pBtdm8723->psTdmaMonitorCnt == 0) {
-		up = 0;
-		dn = 0;
-		m = 1;
-		WaitCount = 0;
-	} else {
-		WaitCount++;
-	}
-
-	if (retry == 0) {
-	/*  no retry in the last 2-second duration */
-		up++;
-		dn--;
-		if (dn < 0)
-			dn = 0;
-		if (up >= 3*m) {
-			/*  retry = 0 in consecutive 3m*(2s), add WiFi duration */
-			ret = 1;
-			up = 0;
-			dn = 0;
-			WaitCount = 0;
-		}
-	} else if (retry <= 3) {
-		/*  retry<= 3 in the last 2-second duration */
-		up--;
-		dn++;
-		if (up < 0)
-			up = 0;
-
-		if (dn == 2) {
-			/*  retry<= 3 in consecutive 2*(2s), minus WiFi duration (add BT duration) */
-			ret = -1;
-
-			/*  record how many time downgrad WiFi duration */
-			if (WaitCount <= 2)
-				m++;
-			else
-				m = 1;
-			/*  the max number of m is 20 */
-			/*  the longest time of upgrade WiFi duration is 20*3*2s = 120s */
-			if (m >= 20)
-				m = 20;
-			up = 0;
-			dn = 0;
-			WaitCount = 0;
-		}
-	} else {
-		/*  retry count > 3 */
-		/*  retry>3, minus WiFi duration (add BT duration) */
-		ret = -1;
-
-		/*  record how many time downgrad WiFi duration */
-		if (WaitCount == 1)
-			m++;
-		else
-			m = 1;
-		if (m >= 20)
-			m = 20;
-
-		up = 0;
-		dn = 0;
-		WaitCount = 0;
-	}
-	return ret;
-}
-
-static void btdm_1AntTdmaDurationAdjustForACL(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_1ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm1Ant;
-
-	if (pBtdm8723->psTdmaGlobalCnt != pBtdm8723->psTdmaMonitorCnt) {
-		pBtdm8723->psTdmaMonitorCnt = 0;
-		pBtdm8723->psTdmaGlobalCnt = 0;
-	}
-	if (pBtdm8723->psTdmaMonitorCnt == 0) {
-		btdm_1AntSetPSTDMA(padapter, true, 0, true, 2);
-		pBtdm8723->psTdmaDuAdjType = 2;
-	} else {
-		/*  Now we only have 4 level Ps Tdma, */
-		/*  if that's not the following 4 level(will changed by wifi scan, dhcp...), */
-		/*  then we have to adjust it back to the previous record one. */
-		if ((pBtdm8723->curPsTdma != 1) &&
-		    (pBtdm8723->curPsTdma != 2) &&
-		    (pBtdm8723->curPsTdma != 9) &&
-		    (pBtdm8723->curPsTdma != 11)) {
-			btdm_1AntSetPSTDMA(padapter, true, 0, true, pBtdm8723->psTdmaDuAdjType);
-		} else {
-			s32 judge;
-
-			judge = btdm_1AntTdmaJudgement(padapter, pHalData->bt_coexist.halCoex8723.btRetryCnt);
-			if (judge == -1) {
-				if (pBtdm8723->curPsTdma == 1) {
-					/*  Decrease WiFi duration for high BT retry */
-					if (pHalData->bt_coexist.halCoex8723.btInfoExt)
-						pBtdm8723->psTdmaDuAdjType = 9;
-					else
-						pBtdm8723->psTdmaDuAdjType = 2;
-					btdm_1AntSetPSTDMA(padapter, true, 0, true, pBtdm8723->psTdmaDuAdjType);
-				} else if (pBtdm8723->curPsTdma == 2) {
-					btdm_1AntSetPSTDMA(padapter, true, 0, true, 9);
-					pBtdm8723->psTdmaDuAdjType = 9;
-				} else if (pBtdm8723->curPsTdma == 9) {
-					btdm_1AntSetPSTDMA(padapter, true, 0, true, 11);
-					pBtdm8723->psTdmaDuAdjType = 11;
-				}
-			} else if (judge == 1) {
-				if (pBtdm8723->curPsTdma == 11) {
-					btdm_1AntSetPSTDMA(padapter, true, 0, true, 9);
-					pBtdm8723->psTdmaDuAdjType = 9;
-				} else if (pBtdm8723->curPsTdma == 9) {
-					if (pHalData->bt_coexist.halCoex8723.btInfoExt)
-						pBtdm8723->psTdmaDuAdjType = 9;
-					else
-						pBtdm8723->psTdmaDuAdjType = 2;
-					btdm_1AntSetPSTDMA(padapter, true, 0, true, pBtdm8723->psTdmaDuAdjType);
-				} else if (pBtdm8723->curPsTdma == 2) {
-					if (pHalData->bt_coexist.halCoex8723.btInfoExt)
-						pBtdm8723->psTdmaDuAdjType = 9;
-					else
-						pBtdm8723->psTdmaDuAdjType = 1;
-					btdm_1AntSetPSTDMA(padapter, true, 0, true, pBtdm8723->psTdmaDuAdjType);
-				}
-			}
-		}
-		RTPRINT(FBT, BT_TRACE,
-			("[BTCoex], ACL current TDMA(%s, %d)\n",
-			(pBtdm8723->bCurPsTdmaOn ? "ON" : "OFF"), pBtdm8723->curPsTdma));
-	}
-	pBtdm8723->psTdmaMonitorCnt++;
-}
-
-static void btdm_1AntCoexProcessForWifiConnect(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv;
-	struct hal_data_8723a *pHalData;
-	struct bt_coexist_8723a *pBtCoex;
-	struct btdm_8723a_1ant *pBtdm8723;
-	u8 BtState;
-
-	pmlmepriv = &padapter->mlmepriv;
-	pHalData = GET_HAL_DATA(padapter);
-	pBtCoex = &pHalData->bt_coexist.halCoex8723;
-	pBtdm8723 = &pBtCoex->btdm1Ant;
-	BtState = pBtCoex->c2hBtInfo;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], WiFi is %s\n",
-				BTDM_IsWifiBusy(padapter)?"Busy":"IDLE"));
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT is %s\n",
-				BtStateString[BtState]));
-
-	padapter->pwrctrlpriv.btcoex_rfon = false;
-
-	if (!BTDM_IsWifiBusy(padapter) &&
-	    !check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) &&
-	    (BtState == BT_INFO_STATE_NO_CONNECTION ||
-	     BtState == BT_INFO_STATE_CONNECT_IDLE)) {
-		switch (BtState) {
-		case BT_INFO_STATE_NO_CONNECTION:
-			_btdm_1AntSetPSTDMA(padapter, true, 2, 0x26, false, 9);
-			break;
-		case BT_INFO_STATE_CONNECT_IDLE:
-			_btdm_1AntSetPSTDMA(padapter, true, 2, 0x26, false, 0);
-			break;
-		}
-	} else {
-		switch (BtState) {
-		case BT_INFO_STATE_NO_CONNECTION:
-		case BT_INFO_STATE_CONNECT_IDLE:
-			/*  WiFi is Busy */
-			btdm_1AntSetPSTDMA(padapter, false, 0, true, 5);
-			rtl8723au_write32(padapter, 0x6c0, 0x5a5a5a5a);
-			rtl8723au_write32(padapter, 0x6c4, 0x5a5a5a5a);
-			break;
-		case BT_INFO_STATE_ACL_INQ_OR_PAG:
-			RTPRINT(FBT, BT_TRACE,
-				("[BTCoex], BT PROFILE is "
-				 "BT_INFO_STATE_ACL_INQ_OR_PAG\n"));
-		case BT_INFO_STATE_INQ_OR_PAG:
-			padapter->pwrctrlpriv.btcoex_rfon = true;
-			btdm_1AntSetPSTDMA(padapter, true, 0, true, 30);
-			break;
-		case BT_INFO_STATE_SCO_ONLY_BUSY:
-		case BT_INFO_STATE_ACL_SCO_BUSY:
-			if (true == pBtCoex->bC2hBtInquiryPage)
-				btdm_1AntSetPSTDMA(padapter, false, 0,
-						   true, 32);
-			else {
-#ifdef BTCOEX_CMCC_TEST
-				btdm_1AntSetPSTDMA(padapter, false, 0,
-						   true, 23);
-#else /*  !BTCOEX_CMCC_TEST */
-				btdm_1AntSetPSTDMA(padapter, false, 0,
-						   false, 8);
-				rtl8723au_write32(padapter, 0x6c0, 0x5a5a5a5a);
-				rtl8723au_write32(padapter, 0x6c4, 0x5a5a5a5a);
-#endif /*  !BTCOEX_CMCC_TEST */
-			}
-			break;
-		case BT_INFO_STATE_ACL_ONLY_BUSY:
-			padapter->pwrctrlpriv.btcoex_rfon = true;
-			if (pBtCoex->c2hBtProfile == BT_INFO_HID) {
-				RTPRINT(FBT, BT_TRACE,
-					("[BTCoex], BT PROFILE is HID\n"));
-				btdm_1AntSetPSTDMA(padapter, true, 0, true, 31);
-			} else if (pBtCoex->c2hBtProfile == BT_INFO_FTP) {
-				RTPRINT(FBT, BT_TRACE,
-					("[BTCoex], BT PROFILE is FTP/OPP\n"));
-				btdm_1AntSetPSTDMA(padapter, true, 0, true, 3);
-			} else if (pBtCoex->c2hBtProfile == (BT_INFO_A2DP|BT_INFO_FTP)) {
-				RTPRINT(FBT, BT_TRACE,
-					("[BTCoex], BT PROFILE is A2DP_FTP\n"));
-				btdm_1AntSetPSTDMA(padapter, true, 0, true, 11);
-			} else {
-				if (pBtCoex->c2hBtProfile == BT_INFO_A2DP)
-					RTPRINT(FBT, BT_TRACE,
-						("[BTCoex], BT PROFILE is "
-						 "A2DP\n"));
-				else
-					RTPRINT(FBT, BT_TRACE,
-						("[BTCoex], BT PROFILE is "
-						 "UNKNOWN(0x%02X)! Use A2DP "
-						 "Profile\n",
-						 pBtCoex->c2hBtProfile));
-				btdm_1AntTdmaDurationAdjustForACL(padapter);
-			}
-			break;
-		}
-	}
-
-	pBtdm8723->psTdmaGlobalCnt++;
-}
-
-static void
-btdm_1AntUpdateHalRAMask(struct rtw_adapter *padapter, u32 mac_id, u32 filter)
-{
-	u8 init_rate = 0;
-	u8 raid, arg;
-	u32 mask;
-	u8 shortGIrate = false;
-	int supportRateNum = 0;
-	struct sta_info	*psta;
-	struct hal_data_8723a *pHalData;
-	struct dm_priv *pdmpriv;
-	struct mlme_ext_priv *pmlmeext;
-	struct mlme_ext_info *pmlmeinfo;
-	struct wlan_bssid_ex *cur_network;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], %s, MACID =%d, filter = 0x%08x!!\n",
-				__func__, mac_id, filter));
-
-	pHalData = GET_HAL_DATA(padapter);
-	pdmpriv = &pHalData->dmpriv;
-	pmlmeext = &padapter->mlmeextpriv;
-	pmlmeinfo = &pmlmeext->mlmext_info;
-	cur_network = &pmlmeinfo->network;
-
-	if (mac_id >= NUM_STA) { /* CAM_SIZE */
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], %s, MACID =%d illegal!!\n",
-					__func__, mac_id));
-		return;
-	}
-
-	psta = pmlmeinfo->FW_sta_info[mac_id].psta;
-	if (!psta) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], %s, Can't find station!!\n",
-					__func__));
-		return;
-	}
-
-	raid = psta->raid;
-
-	switch (mac_id) {
-	case 0:/*  for infra mode */
-		supportRateNum =
-			rtw_get_rateset_len23a(cur_network->SupportedRates);
-		mask = update_supported_rate23a(cur_network->SupportedRates,
-						supportRateNum);
-		mask |= (pmlmeinfo->HT_enable) ?
-			update_MSC_rate23a(&pmlmeinfo->ht_cap):0;
-		if (support_short_GI23a(padapter, &pmlmeinfo->ht_cap))
-			shortGIrate = true;
-		break;
-	case 1:/* for broadcast/multicast */
-		supportRateNum = rtw_get_rateset_len23a(
-			pmlmeinfo->FW_sta_info[mac_id].SupportedRates);
-		mask = update_basic_rate23a(cur_network->SupportedRates,
-					    supportRateNum);
-		break;
-	default: /* for each sta in IBSS */
-		supportRateNum = rtw_get_rateset_len23a(
-			pmlmeinfo->FW_sta_info[mac_id].SupportedRates);
-		mask = update_supported_rate23a(cur_network->SupportedRates,
-						supportRateNum);
-		break;
-	}
-	mask |= ((raid<<28)&0xf0000000);
-	mask &= 0xffffffff;
-	mask &= ~filter;
-	init_rate = get_highest_rate_idx23a(mask)&0x3f;
-
-	arg = mac_id&0x1f;/* MACID */
-	arg |= BIT(7);
-	if (true == shortGIrate)
-		arg |= BIT(5);
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTCoex], Update FW RAID entry, MASK = 0x%08x, "
-		 "arg = 0x%02x\n", mask, arg));
-
-	rtl8723a_set_raid_cmd(padapter, mask, arg);
-
-	psta->init_rate = init_rate;
-	pdmpriv->INIDATA_RATE[mac_id] = init_rate;
-}
-
-static void
-btdm_1AntUpdateHalRAMaskForSCO(struct rtw_adapter *padapter, u8 forceUpdate)
-{
-	struct btdm_8723a_1ant *pBtdm8723;
-	struct sta_priv *pstapriv;
-	struct wlan_bssid_ex *cur_network;
-	struct sta_info *psta;
-	u32 macid;
-	u32 filter = 0;
-
-	pBtdm8723 = &GET_HAL_DATA(padapter)->bt_coexist.halCoex8723.btdm1Ant;
-
-	if (pBtdm8723->bRAChanged == true && forceUpdate == false)
-		return;
-
-	pstapriv = &padapter->stapriv;
-	cur_network = &padapter->mlmeextpriv.mlmext_info.network;
-	psta = rtw_get_stainfo23a(pstapriv, cur_network->MacAddress);
-	macid = psta->mac_id;
-
-	filter |= BIT(_1M_RATE_);
-	filter |= BIT(_2M_RATE_);
-	filter |= BIT(_5M_RATE_);
-	filter |= BIT(_11M_RATE_);
-	filter |= BIT(_6M_RATE_);
-	filter |= BIT(_9M_RATE_);
-
-	btdm_1AntUpdateHalRAMask(padapter, macid, filter);
-
-	pBtdm8723->bRAChanged = true;
-}
-
-static void btdm_1AntRecoverHalRAMask(struct rtw_adapter *padapter)
-{
-	struct btdm_8723a_1ant *pBtdm8723;
-	struct sta_priv *pstapriv;
-	struct wlan_bssid_ex *cur_network;
-	struct sta_info *psta;
-
-	pBtdm8723 = &GET_HAL_DATA(padapter)->bt_coexist.halCoex8723.btdm1Ant;
-
-	if (pBtdm8723->bRAChanged == false)
-		return;
-
-	pstapriv = &padapter->stapriv;
-	cur_network = &padapter->mlmeextpriv.mlmext_info.network;
-	psta = rtw_get_stainfo23a(pstapriv, cur_network->MacAddress);
-
-	Update_RA_Entry23a(padapter, psta);
-
-	pBtdm8723->bRAChanged = false;
-}
-
-static void
-btdm_1AntBTStateChangeHandler(struct rtw_adapter *padapter,
-			      enum bt_state_1ant oldState,
-			      enum bt_state_1ant newState)
-{
-	struct hal_data_8723a *phaldata;
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT state change, %s => %s\n",
-				BtStateString[oldState],
-				BtStateString[newState]));
-
-	/*  BT default ignore wlan active, */
-	/*  WiFi MUST disable this when BT is enable */
-	if (newState > BT_INFO_STATE_DISABLED)
-		btdm_SetFwIgnoreWlanAct(padapter, false);
-
-	if ((check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE)) &&
-	    (BTDM_IsWifiConnectionExist(padapter))) {
-		if ((newState == BT_INFO_STATE_SCO_ONLY_BUSY) ||
-		    (newState == BT_INFO_STATE_ACL_SCO_BUSY)) {
-			btdm_1AntUpdateHalRAMaskForSCO(padapter, false);
-		} else {
-			/*  Recover original RA setting */
-			btdm_1AntRecoverHalRAMask(padapter);
-		}
-	} else {
-		phaldata = GET_HAL_DATA(padapter);
-		phaldata->bt_coexist.halCoex8723.btdm1Ant.bRAChanged = false;
-	}
-
-	if (oldState == newState)
-		return;
-
-	if (oldState == BT_INFO_STATE_ACL_ONLY_BUSY) {
-		struct hal_data_8723a *Hal = GET_HAL_DATA(padapter);
-		Hal->bt_coexist.halCoex8723.btdm1Ant.psTdmaMonitorCnt = 0;
-		Hal->bt_coexist.halCoex8723.btdm1Ant.psTdmaMonitorCntForSCO = 0;
-	}
-
-	if ((oldState == BT_INFO_STATE_SCO_ONLY_BUSY) ||
-	    (oldState == BT_INFO_STATE_ACL_SCO_BUSY)) {
-		struct hal_data_8723a *Hal = GET_HAL_DATA(padapter);
-		Hal->bt_coexist.halCoex8723.btdm1Ant.psTdmaMonitorCntForSCO = 0;
-	}
-
-	/*  Active 2Ant mechanism when BT Connected */
-	if ((oldState == BT_INFO_STATE_DISABLED) ||
-	    (oldState == BT_INFO_STATE_NO_CONNECTION)) {
-		if ((newState != BT_INFO_STATE_DISABLED) &&
-		    (newState != BT_INFO_STATE_NO_CONNECTION)) {
-			BTDM_SetSwRfRxLpfCorner(padapter,
-						BT_RF_RX_LPF_CORNER_SHRINK);
-			BTDM_AGCTable(padapter, BT_AGCTABLE_ON);
-			BTDM_BBBackOffLevel(padapter, BT_BB_BACKOFF_ON);
-		}
-	} else {
-		if ((newState == BT_INFO_STATE_DISABLED) ||
-		    (newState == BT_INFO_STATE_NO_CONNECTION)) {
-			BTDM_SetSwRfRxLpfCorner(padapter,
-						BT_RF_RX_LPF_CORNER_RESUME);
-			BTDM_AGCTable(padapter, BT_AGCTABLE_OFF);
-			BTDM_BBBackOffLevel(padapter, BT_BB_BACKOFF_OFF);
-		}
-	}
-}
-
-static void btdm_1AntBtCoexistHandler(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_coexist_8723a *pBtCoex8723;
-	struct btdm_8723a_1ant *pBtdm8723;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBtCoex8723 = &pHalData->bt_coexist.halCoex8723;
-	pBtdm8723 = &pBtCoex8723->btdm1Ant;
-	padapter->pwrctrlpriv.btcoex_rfon = false;
-	if (!rtl8723a_BT_enabled(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT is disabled\n"));
-
-		if (BTDM_IsWifiConnectionExist(padapter)) {
-			RTPRINT(FBT, BT_TRACE,
-				("[BTCoex], wifi is connected\n"));
-
-			if (BTDM_IsWifiBusy(padapter)) {
-				RTPRINT(FBT, BT_TRACE,
-					("[BTCoex], Wifi is busy\n"));
-				btdm_1AntSetPSTDMA(padapter, false, 0,
-						   false, 9);
-			} else {
-				RTPRINT(FBT, BT_TRACE,
-					("[BTCoex], Wifi is idle\n"));
-				_btdm_1AntSetPSTDMA(padapter, true, 2, 1,
-						    false, 9);
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE,
-				("[BTCoex], wifi is disconnected\n"));
-
-			btdm_1AntSetPSTDMA(padapter, false, 0, false, 9);
-		}
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT is enabled\n"));
-
-		if (BTDM_IsWifiConnectionExist(padapter)) {
-			RTPRINT(FBT, BT_TRACE,
-				("[BTCoex], wifi is connected\n"));
-
-			btdm_1AntWifiParaAdjust(padapter, true);
-			btdm_1AntCoexProcessForWifiConnect(padapter);
-		} else {
-			RTPRINT(FBT, BT_TRACE,
-				("[BTCoex], wifi is disconnected\n"));
-
-			/*  Antenna switch at BT side(0x870 = 0x300,
-			    0x860 = 0x210) after PSTDMA off */
-			btdm_1AntWifiParaAdjust(padapter, false);
-			btdm_1AntSetPSTDMA(padapter, false, 0, false, 0);
-		}
-	}
-
-	btdm_1AntBTStateChangeHandler(padapter, pBtCoex8723->prec2hBtInfo,
-				      pBtCoex8723->c2hBtInfo);
-	pBtCoex8723->prec2hBtInfo = pBtCoex8723->c2hBtInfo;
-}
-
-void BTDM_1AntSignalCompensation(struct rtw_adapter *padapter,
-				 u8 *rssi_wifi, u8 *rssi_bt)
-{
-	struct hal_data_8723a *pHalData;
-	struct btdm_8723a_1ant *pBtdm8723;
-	u8 RSSI_WiFi_Cmpnstn, RSSI_BT_Cmpnstn;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm1Ant;
-	RSSI_WiFi_Cmpnstn = 0;
-	RSSI_BT_Cmpnstn = 0;
-
-	switch (pBtdm8723->curPsTdma) {
-	case 1: /*  WiFi 52ms */
-		RSSI_WiFi_Cmpnstn = 11; /*  22*0.48 */
-		break;
-	case 2: /*  WiFi 36ms */
-		RSSI_WiFi_Cmpnstn = 14; /*  22*0.64 */
-		break;
-	case 9: /*  WiFi 20ms */
-		RSSI_WiFi_Cmpnstn = 18; /*  22*0.80 */
-		break;
-	case 11: /*  WiFi 10ms */
-		RSSI_WiFi_Cmpnstn = 20; /*  22*0.90 */
-		break;
-	case 4: /*  WiFi 21ms */
-		RSSI_WiFi_Cmpnstn = 17; /*  22*0.79 */
-		break;
-	case 16: /*  WiFi 24ms */
-		RSSI_WiFi_Cmpnstn = 18; /*  22*0.76 */
-		break;
-	case 18: /*  WiFi 37ms */
-		RSSI_WiFi_Cmpnstn = 14; /*  22*0.64 */
-		break;
-	case 23: /* Level-1, Antenna switch to BT at all time */
-	case 24: /* Level-2, Antenna switch to BT at all time */
-	case 25: /* Level-3a, Antenna switch to BT at all time */
-	case 26: /* Level-3b, Antenna switch to BT at all time */
-	case 27: /* Level-3b, Antenna switch to BT at all time */
-	case 33: /* BT SCO & WiFi site survey */
-		RSSI_WiFi_Cmpnstn = 22;
-		break;
-	default:
-		break;
-	}
-
-	if (rssi_wifi && RSSI_WiFi_Cmpnstn) {
-		RTPRINT(FBT, BT_TRACE,
-			("[BTCoex], 1AntSgnlCmpnstn, case %d, WiFiCmpnstn "
-			 "=%d(%d => %d)\n", pBtdm8723->curPsTdma,
-			 RSSI_WiFi_Cmpnstn, *rssi_wifi,
-			 *rssi_wifi+RSSI_WiFi_Cmpnstn));
-		*rssi_wifi += RSSI_WiFi_Cmpnstn;
-	}
-
-	if (rssi_bt && RSSI_BT_Cmpnstn) {
-		RTPRINT(FBT, BT_TRACE,
-			("[BTCoex], 1AntSgnlCmpnstn, case %d, BTCmpnstn "
-			 "=%d(%d => %d)\n", pBtdm8723->curPsTdma,
-			 RSSI_BT_Cmpnstn, *rssi_bt, *rssi_bt+RSSI_BT_Cmpnstn));
-		*rssi_bt += RSSI_BT_Cmpnstn;
-	}
-}
-
-static void BTDM_1AntParaInit(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_coexist_8723a *pBtCoex;
-	struct btdm_8723a_1ant *pBtdm8723;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBtCoex = &pHalData->bt_coexist.halCoex8723;
-	pBtdm8723 = &pBtCoex->btdm1Ant;
-
-	/*  Enable counter statistics */
-	rtl8723au_write8(padapter, 0x76e, 0x4);
-	btdm_1AntPtaParaReload(padapter);
-
-	pBtdm8723->wifiRssiThresh = 48;
-
-	pBtdm8723->bWiFiHalt = false;
-	pBtdm8723->bRAChanged = false;
-
-	if ((pBtCoex->c2hBtInfo != BT_INFO_STATE_DISABLED) &&
-	    (pBtCoex->c2hBtInfo != BT_INFO_STATE_NO_CONNECTION)) {
-		BTDM_SetSwRfRxLpfCorner(padapter, BT_RF_RX_LPF_CORNER_SHRINK);
-		BTDM_AGCTable(padapter, BT_AGCTABLE_ON);
-		BTDM_BBBackOffLevel(padapter, BT_BB_BACKOFF_ON);
-	}
-}
-
-static void BTDM_1AntForHalt(struct rtw_adapter *padapter)
-{
-	RTPRINT(FBT, BT_TRACE, ("\n[BTCoex], 1Ant for halt\n"));
-
-	GET_HAL_DATA(padapter)->bt_coexist.halCoex8723.btdm1Ant.bWiFiHalt =
-		true;
-
-	btdm_1AntWifiParaAdjust(padapter, false);
-
-	/*  don't use btdm_1AntSetPSTDMA() here */
-	/*  it will call rtw_set_ps_mode23a() and request pwrpriv->lock. */
-	/*  This will lead to deadlock, if this function is called in IPS */
-	/*  Lucas@20130205 */
-	btdm_1AntPsTdma(padapter, false, 0);
-
-	btdm_SetFwIgnoreWlanAct(padapter, true);
-}
-
-static void BTDM_1AntLpsLeave(struct rtw_adapter *padapter)
-{
-	RTPRINT(FBT, BT_TRACE, ("\n[BTCoex], 1Ant for LPS Leave\n"));
-
-	/*  Prevent from entering LPS again */
-	GET_HAL_DATA(padapter)->bt_coexist.halCoex8723.btdm1Ant.bWiFiHalt =
-		true;
-
-	btdm_1AntSetPSTDMA(padapter, false, 0, false, 8);
-/*btdm_1AntPsTdma(padapter, false, 8); */
-}
-
-static void BTDM_1AntWifiAssociateNotify(struct rtw_adapter *padapter, u8 type)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	RTPRINT(FBT, BT_TRACE,
-		("\n[BTCoex], 1Ant for associate, type =%d\n", type));
-
-	if (type) {
-		rtl8723a_CheckAntenna_Selection(padapter);
-		if (!rtl8723a_BT_enabled(padapter))
-			btdm_1AntSetPSTDMA(padapter, false, 0, false, 9);
-		else {
-			struct bt_coexist_8723a *pBtCoex;
-			u8 BtState;
-
-			pBtCoex = &pHalData->bt_coexist.halCoex8723;
-			BtState = pBtCoex->c2hBtInfo;
-
-			btdm_1AntTSFSwitch(padapter, true);
-
-			if (BtState == BT_INFO_STATE_NO_CONNECTION ||
-			    BtState == BT_INFO_STATE_CONNECT_IDLE) {
-				btdm_1AntSetPSTDMA(padapter, false, 0,
-						   true, 28);
-			} else if (BtState == BT_INFO_STATE_SCO_ONLY_BUSY ||
-				   BtState == BT_INFO_STATE_ACL_SCO_BUSY) {
-				btdm_1AntSetPSTDMA(padapter, false, 0,
-						   false, 8);
-				rtl8723au_write32(padapter, 0x6c0, 0x5a5a5a5a);
-				rtl8723au_write32(padapter, 0x6c4, 0x5a5a5a5a);
-			} else if (BtState == BT_INFO_STATE_ACL_ONLY_BUSY ||
-				   BtState == BT_INFO_STATE_ACL_INQ_OR_PAG) {
-				if (pBtCoex->c2hBtProfile == BT_INFO_HID)
-					btdm_1AntSetPSTDMA(padapter, false, 0,
-							   true, 35);
-				else
-					btdm_1AntSetPSTDMA(padapter, false, 0,
-							   true, 29);
-			}
-		}
-	} else {
-		if (!rtl8723a_BT_enabled(padapter)) {
-			if (!BTDM_IsWifiConnectionExist(padapter)) {
-				btdm_1AntPsTdma(padapter, false, 0);
-				btdm_1AntTSFSwitch(padapter, false);
-			}
-		}
-
-		btdm_1AntBtCoexistHandler(padapter);
-	}
-}
-
-static void
-BTDM_1AntMediaStatusNotify(struct rtw_adapter *padapter,
-			   enum rt_media_status mstatus)
-{
-	struct bt_coexist_8723a *pBtCoex;
-
-	pBtCoex = &GET_HAL_DATA(padapter)->bt_coexist.halCoex8723;
-
-	RTPRINT(FBT, BT_TRACE,
-		("\n\n[BTCoex]******************************\n"));
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], MediaStatus, WiFi %s !!\n",
-			mstatus == RT_MEDIA_CONNECT?"CONNECT":"DISCONNECT"));
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex]******************************\n"));
-
-	if (RT_MEDIA_CONNECT == mstatus) {
-		if (check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE)) {
-			if (pBtCoex->c2hBtInfo == BT_INFO_STATE_SCO_ONLY_BUSY ||
-			    pBtCoex->c2hBtInfo == BT_INFO_STATE_ACL_SCO_BUSY)
-				btdm_1AntUpdateHalRAMaskForSCO(padapter, true);
-		}
-
-		padapter->pwrctrlpriv.DelayLPSLastTimeStamp = jiffies;
-		BTDM_1AntForDhcp(padapter);
-	} else {
-		/* DBG_8723A("%s rtl8723a_DeinitAntenna_Selection\n",
-		   __func__); */
-		rtl8723a_DeinitAntenna_Selection(padapter);
-		btdm_1AntBtCoexistHandler(padapter);
-		pBtCoex->btdm1Ant.bRAChanged = false;
-	}
-}
-
-void BTDM_1AntForDhcp(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	u8 BtState;
-	struct bt_coexist_8723a *pBtCoex;
-	struct btdm_8723a_1ant *pBtdm8723;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBtCoex = &pHalData->bt_coexist.halCoex8723;
-	BtState = pBtCoex->c2hBtInfo;
-	pBtdm8723 = &pBtCoex->btdm1Ant;
-
-	RTPRINT(FBT, BT_TRACE, ("\n[BTCoex], 1Ant for DHCP\n"));
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], 1Ant for DHCP, WiFi is %s\n",
-				BTDM_IsWifiBusy(padapter)?"Busy":"IDLE"));
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], 1Ant for DHCP, %s\n",
-				BtStateString[BtState]));
-
-	BTDM_1AntWifiAssociateNotify(padapter, true);
-}
-
-static void BTDM_1AntWifiScanNotify(struct rtw_adapter *padapter, u8 scanType)
-{
-	struct hal_data_8723a *pHalData;
-	u8 BtState;
-	struct bt_coexist_8723a *pBtCoex;
-	struct btdm_8723a_1ant *pBtdm8723;
-
-	pHalData = GET_HAL_DATA(padapter);
-	BtState = pHalData->bt_coexist.halCoex8723.c2hBtInfo;
-	pBtCoex = &pHalData->bt_coexist.halCoex8723;
-	pBtdm8723 = &pBtCoex->btdm1Ant;
-
-	RTPRINT(FBT, BT_TRACE, ("\n[BTCoex], 1Ant for wifi scan =%d!!\n",
-				scanType));
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], 1Ant for wifi scan, WiFi is %s\n",
-				BTDM_IsWifiBusy(padapter)?"Busy":"IDLE"));
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], 1Ant for wifi scan, %s\n",
-				BtStateString[BtState]));
-
-	if (scanType) {
-		rtl8723a_CheckAntenna_Selection(padapter);
-		if (!rtl8723a_BT_enabled(padapter)) {
-			btdm_1AntSetPSTDMA(padapter, false, 0, false, 9);
-		} else if (BTDM_IsWifiConnectionExist(padapter) == false) {
-			BTDM_1AntWifiAssociateNotify(padapter, true);
-		} else {
-			if ((BtState == BT_INFO_STATE_SCO_ONLY_BUSY) ||
-			    (BtState == BT_INFO_STATE_ACL_SCO_BUSY)) {
-				if (pBtCoex->bC2hBtInquiryPage) {
-					btdm_1AntSetPSTDMA(padapter, false, 0,
-							   true, 32);
-				} else {
-					padapter->pwrctrlpriv.btcoex_rfon =
-						true;
-					btdm_1AntSetPSTDMA(padapter, true, 0,
-							   true, 33);
-				}
-			} else if (true == pBtCoex->bC2hBtInquiryPage) {
-				padapter->pwrctrlpriv.btcoex_rfon = true;
-				btdm_1AntSetPSTDMA(padapter, true, 0, true, 30);
-			} else if (BtState == BT_INFO_STATE_ACL_ONLY_BUSY) {
-				padapter->pwrctrlpriv.btcoex_rfon = true;
-				if (pBtCoex->c2hBtProfile == BT_INFO_HID)
-					btdm_1AntSetPSTDMA(padapter, true, 0,
-							   true, 34);
-				else
-					btdm_1AntSetPSTDMA(padapter, true, 0,
-							   true, 4);
-			} else {
-				padapter->pwrctrlpriv.btcoex_rfon = true;
-				btdm_1AntSetPSTDMA(padapter, true, 0, true, 5);
-			}
-		}
-
-		btdm_NotifyFwScan(padapter, 1);
-	} else {
-		/*  WiFi_Finish_Scan */
-		btdm_NotifyFwScan(padapter, 0);
-		btdm_1AntBtCoexistHandler(padapter);
-	}
-}
-
-static void BTDM_1AntFwC2hBtInfo8723A(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_coexist_8723a *pBtCoex;
-	u8 u1tmp, btState;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtCoex = &pHalData->bt_coexist.halCoex8723;
-
-	u1tmp = pBtCoex->c2hBtInfoOriginal;
-	/*  sco BUSY bit is not used on voice over PCM platform */
-	btState = u1tmp & 0xF;
-	pBtCoex->c2hBtProfile = u1tmp & 0xE0;
-
-	/*  default set bt to idle state. */
-	pBtMgnt->ExtConfig.bBTBusy = false;
-	pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_BT_IDLE;
-
-	/*  check BIT2 first ==> check if bt is under inquiry or page scan */
-	if (btState & BIT(2))
-		pBtCoex->bC2hBtInquiryPage = true;
-	else
-		pBtCoex->bC2hBtInquiryPage = false;
-	btState &= ~BIT(2);
-
-	if (!(btState & BIT(0)))
-		pBtCoex->c2hBtInfo = BT_INFO_STATE_NO_CONNECTION;
-	else {
-		if (btState == 0x1)
-			pBtCoex->c2hBtInfo = BT_INFO_STATE_CONNECT_IDLE;
-		else if (btState == 0x9) {
-			if (pBtCoex->bC2hBtInquiryPage == true)
-				pBtCoex->c2hBtInfo =
-					BT_INFO_STATE_ACL_INQ_OR_PAG;
-			else
-				pBtCoex->c2hBtInfo =
-					BT_INFO_STATE_ACL_ONLY_BUSY;
-			pBtMgnt->ExtConfig.bBTBusy = true;
-		} else if (btState == 0x3) {
-			pBtCoex->c2hBtInfo = BT_INFO_STATE_SCO_ONLY_BUSY;
-			pBtMgnt->ExtConfig.bBTBusy = true;
-		} else if (btState == 0xb) {
-			pBtCoex->c2hBtInfo = BT_INFO_STATE_ACL_SCO_BUSY;
-			pBtMgnt->ExtConfig.bBTBusy = true;
-		} else
-			pBtCoex->c2hBtInfo = BT_INFO_STATE_MAX;
-		if (pBtMgnt->ExtConfig.bBTBusy)
-			pHalData->bt_coexist.CurrentState &=
-				~BT_COEX_STATE_BT_IDLE;
-	}
-
-	if (BT_INFO_STATE_NO_CONNECTION == pBtCoex->c2hBtInfo ||
-	    BT_INFO_STATE_CONNECT_IDLE == pBtCoex->c2hBtInfo) {
-		if (pBtCoex->bC2hBtInquiryPage)
-			pBtCoex->c2hBtInfo = BT_INFO_STATE_INQ_OR_PAG;
-	}
-
-	RTPRINT(FBT, BT_TRACE, ("[BTC2H], %s(%d)\n",
-			BtStateString[pBtCoex->c2hBtInfo], pBtCoex->c2hBtInfo));
-
-	if (pBtCoex->c2hBtProfile != BT_INFO_HID)
-		pBtCoex->c2hBtProfile &= ~BT_INFO_HID;
-}
-
-void BTDM_1AntBtCoexist8723A(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv;
-	struct hal_data_8723a *pHalData;
-	unsigned long delta_time;
-
-	pmlmepriv = &padapter->mlmepriv;
-	pHalData = GET_HAL_DATA(padapter);
-
-	if (check_fwstate(pmlmepriv, WIFI_SITE_MONITOR)) {
-		/*  already done in BTDM_1AntForScan() */
-		RTPRINT(FBT, BT_TRACE,
-			("[BTCoex], wifi is under scan progress!!\n"));
-		return;
-	}
-
-	if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING)) {
-		RTPRINT(FBT, BT_TRACE,
-			("[BTCoex], wifi is under link progress!!\n"));
-		return;
-	}
-
-	/*  under DHCP(Special packet) */
-	delta_time = jiffies - padapter->pwrctrlpriv.DelayLPSLastTimeStamp;
-	delta_time = jiffies_to_msecs(delta_time);
-	if (delta_time < 500) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], wifi is under DHCP "
-					"progress(%li ms)!!\n", delta_time));
-		return;
-	}
-
-	BTDM_CheckWiFiState(padapter);
-
-	btdm_1AntBtCoexistHandler(padapter);
-}
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtc87231Ant.c ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtc87232Ant.c ===== */
-
-/*  local function start with btdm_ */
-static u8 btdm_ActionAlgorithm(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-	u8 bScoExist = false, bBtLinkExist = false, bBtHsModeExist = false;
-	u8 algorithm = BT_2ANT_COEX_ALGO_UNDEFINED;
-
-	if (pBtMgnt->ExtConfig.NumberOfHandle)
-		bBtLinkExist = true;
-	if (pBtMgnt->ExtConfig.NumberOfSCO)
-		bScoExist = true;
-	if (BT_HsConnectionEstablished(padapter))
-		bBtHsModeExist = true;
-
-	/*  here we get BT status first */
-	/*  1) initialize */
-	pBtdm8723->btStatus = BT_2ANT_BT_STATUS_IDLE;
-
-	if ((bScoExist) || (bBtHsModeExist) ||
-	    (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID))) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO or HID or HS exists, set BT non-idle !!!\n"));
-		pBtdm8723->btStatus = BT_2ANT_BT_STATUS_NON_IDLE;
-	} else {
-		/*  A2dp profile */
-		if ((pBtMgnt->ExtConfig.NumberOfHandle == 1) &&
-		    (BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP))) {
-			if (BTDM_BtTxRxCounterL(padapter) < 100) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], A2DP, low priority tx+rx < 100, set BT connected-idle!!!\n"));
-				pBtdm8723->btStatus = BT_2ANT_BT_STATUS_CONNECTED_IDLE;
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], A2DP, low priority tx+rx >= 100, set BT non-idle!!!\n"));
-				pBtdm8723->btStatus = BT_2ANT_BT_STATUS_NON_IDLE;
-			}
-		}
-		/*  Pan profile */
-		if ((pBtMgnt->ExtConfig.NumberOfHandle == 1) &&
-		    (BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN))) {
-			if (BTDM_BtTxRxCounterL(padapter) < 600) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], PAN, low priority tx+rx < 600, set BT connected-idle!!!\n"));
-				pBtdm8723->btStatus = BT_2ANT_BT_STATUS_CONNECTED_IDLE;
-			} else {
-				if (pHalData->bt_coexist.halCoex8723.lowPriorityTx) {
-					if ((pHalData->bt_coexist.halCoex8723.lowPriorityRx /
-					    pHalData->bt_coexist.halCoex8723.lowPriorityTx) > 9) {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], PAN, low priority rx/tx > 9, set BT connected-idle!!!\n"));
-						pBtdm8723->btStatus = BT_2ANT_BT_STATUS_CONNECTED_IDLE;
-					}
-				}
-			}
-			if (BT_2ANT_BT_STATUS_CONNECTED_IDLE != pBtdm8723->btStatus) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], PAN, set BT non-idle!!!\n"));
-				pBtdm8723->btStatus = BT_2ANT_BT_STATUS_NON_IDLE;
-			}
-		}
-		/*  Pan+A2dp profile */
-		if ((pBtMgnt->ExtConfig.NumberOfHandle == 2) &&
-		    (BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) &&
-		    (BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN))) {
-			if (BTDM_BtTxRxCounterL(padapter) < 600) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], PAN+A2DP, low priority tx+rx < 600, set BT connected-idle!!!\n"));
-				pBtdm8723->btStatus = BT_2ANT_BT_STATUS_CONNECTED_IDLE;
-			} else {
-				if (pHalData->bt_coexist.halCoex8723.lowPriorityTx) {
-					if ((pHalData->bt_coexist.halCoex8723.lowPriorityRx /
-					    pHalData->bt_coexist.halCoex8723.lowPriorityTx) > 9) {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], PAN+A2DP, low priority rx/tx > 9, set BT connected-idle!!!\n"));
-						pBtdm8723->btStatus = BT_2ANT_BT_STATUS_CONNECTED_IDLE;
-					}
-				}
-			}
-			if (BT_2ANT_BT_STATUS_CONNECTED_IDLE != pBtdm8723->btStatus) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], PAN+A2DP, set BT non-idle!!!\n"));
-				pBtdm8723->btStatus = BT_2ANT_BT_STATUS_NON_IDLE;
-			}
-		}
-	}
-	if (BT_2ANT_BT_STATUS_IDLE != pBtdm8723->btStatus)
-		pBtMgnt->ExtConfig.bBTBusy = true;
-	else
-		pBtMgnt->ExtConfig.bBTBusy = false;
-
-	if (!bBtLinkExist) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], No profile exists!!!\n"));
-		return algorithm;
-	}
-
-	if (pBtMgnt->ExtConfig.NumberOfHandle == 1) {
-		if (bScoExist) {
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO only\n"));
-			algorithm = BT_2ANT_COEX_ALGO_SCO;
-		} else {
-			if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID)) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID only\n"));
-				algorithm = BT_2ANT_COEX_ALGO_HID;
-			} else if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], A2DP only\n"));
-				algorithm = BT_2ANT_COEX_ALGO_A2DP;
-			} else if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN)) {
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], PAN(HS) only\n"));
-					algorithm = BT_2ANT_COEX_ALGO_PANHS;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], PAN(EDR) only\n"));
-					algorithm = BT_2ANT_COEX_ALGO_PANEDR;
-				}
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! NO matched profile for NumberOfHandle =%d \n",
-				pBtMgnt->ExtConfig.NumberOfHandle));
-			}
-		}
-	} else if (pBtMgnt->ExtConfig.NumberOfHandle == 2) {
-		if (bScoExist) {
-			if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID)) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO + HID\n"));
-				algorithm = BT_2ANT_COEX_ALGO_HID;
-			} else if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO + A2DP\n"));
-			} else if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN)) {
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO + PAN(HS)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_SCO;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO + PAN(EDR)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_PANEDR_HID;
-				}
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO exists but why NO matched ACL profile for NumberOfHandle =%d\n",
-				pBtMgnt->ExtConfig.NumberOfHandle));
-			}
-		} else {
-			if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID) &&
-			    BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + A2DP\n"));
-				algorithm = BT_2ANT_COEX_ALGO_HID_A2DP;
-		} else if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID) &&
-			   BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN)) {
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + PAN(HS)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_HID_A2DP;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + PAN(EDR)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_PANEDR_HID;
-				}
-			} else if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN) &&
-				   BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], A2DP + PAN(HS)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_A2DP;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], A2DP + PAN(EDR)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_PANEDR_A2DP;
-				}
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! NO matched profile for NumberOfHandle =%d\n",
-					pBtMgnt->ExtConfig.NumberOfHandle));
-			}
-		}
-	} else if (pBtMgnt->ExtConfig.NumberOfHandle == 3) {
-		if (bScoExist) {
-			if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID) &&
-			    BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO + HID + A2DP\n"));
-			} else if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID) &&
-				   BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN)) {
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO + HID + PAN(HS)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_HID_A2DP;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO + HID + PAN(EDR)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_PANEDR_HID;
-				}
-			} else if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN) &&
-				   BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO + A2DP + PAN(HS)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_SCO;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO + A2DP + PAN(EDR)\n"));
-				}
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO exists but why NO matched profile for NumberOfHandle =%d\n",
-					pBtMgnt->ExtConfig.NumberOfHandle));
-			}
-		} else {
-			if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID) &&
-			    BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN) &&
-			    BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + A2DP + PAN(HS)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_HID_A2DP_PANHS;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + A2DP + PAN(EDR)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_HID_A2DP_PANEDR;
-				}
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! NO matched profile for NumberOfHandle =%d\n",
-					pBtMgnt->ExtConfig.NumberOfHandle));
-			}
-		}
-	} else if (pBtMgnt->ExtConfig.NumberOfHandle >= 3) {
-		if (bScoExist) {
-			if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID) &&
-			    BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN) &&
-			    BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-				if (bBtHsModeExist)
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO + HID + A2DP + PAN(HS)\n"));
-				else
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO + HID + A2DP + PAN(EDR)\n"));
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO exists but why NO matched profile for NumberOfHandle =%d\n",
-					pBtMgnt->ExtConfig.NumberOfHandle));
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! NO matched profile for NumberOfHandle =%d\n",
-				pBtMgnt->ExtConfig.NumberOfHandle));
-		}
-	}
-	return algorithm;
-}
-
-static u8 btdm_NeedToDecBtPwr(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 bRet = false;
-
-	if (BT_Operation(padapter)) {
-		if (pHalData->dmpriv.EntryMinUndecoratedSmoothedPWDB > 47) {
-			RTPRINT(FBT, BT_TRACE, ("Need to decrease bt power for HS mode!!\n"));
-			bRet = true;
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("NO Need to decrease bt power for HS mode!!\n"));
-		}
-	} else {
-		if (BTDM_IsWifiConnectionExist(padapter)) {
-			RTPRINT(FBT, BT_TRACE, ("Need to decrease bt power for Wifi is connected!!\n"));
-			bRet = true;
-		}
-	}
-	return bRet;
-}
-
-static void
-btdm_SetCoexTable(struct rtw_adapter *padapter, u32 val0x6c0,
-		  u32 val0x6c8, u8 val0x6cc)
-{
-	RTPRINT(FBT, BT_TRACE, ("set coex table, set 0x6c0 = 0x%x\n", val0x6c0));
-	rtl8723au_write32(padapter, 0x6c0, val0x6c0);
-
-	RTPRINT(FBT, BT_TRACE, ("set coex table, set 0x6c8 = 0x%x\n", val0x6c8));
-	rtl8723au_write32(padapter, 0x6c8, val0x6c8);
-
-	RTPRINT(FBT, BT_TRACE, ("set coex table, set 0x6cc = 0x%x\n", val0x6cc));
-	rtl8723au_write8(padapter, 0x6cc, val0x6cc);
-}
-
-static void
-btdm_SetSwFullTimeDacSwing(struct rtw_adapter *padapter, u8 bSwDacSwingOn,
-			   u32 swDacSwingLvl)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (bSwDacSwingOn) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], SwDacSwing = 0x%x\n", swDacSwingLvl));
-		PHY_SetBBReg(padapter, 0x880, 0xff000000, swDacSwingLvl);
-		pHalData->bt_coexist.bSWCoexistAllOff = false;
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], SwDacSwing Off!\n"));
-		PHY_SetBBReg(padapter, 0x880, 0xff000000, 0xc0);
-	}
-}
-
-static void
-btdm_SetFwDacSwingLevel(struct rtw_adapter *padapter, u8 dacSwingLvl)
-{
-	u8 H2C_Parameter[1] = {0};
-
-	H2C_Parameter[0] = dacSwingLvl;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], Set Dac Swing Level = 0x%x\n", dacSwingLvl));
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], write 0x29 = 0x%x\n", H2C_Parameter[0]));
-
-	FillH2CCmd(padapter, 0x29, 1, H2C_Parameter);
-}
-
-static void btdm_2AntDecBtPwr(struct rtw_adapter *padapter, u8 bDecBtPwr)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTCoex], Dec BT power = %s\n",
-		((bDecBtPwr) ? "ON" : "OFF")));
-	pBtdm8723->bCurDecBtPwr = bDecBtPwr;
-
-	if (pBtdm8723->bPreDecBtPwr == pBtdm8723->bCurDecBtPwr)
-		return;
-
-	BTDM_SetFwDecBtPwr(padapter, pBtdm8723->bCurDecBtPwr);
-
-	pBtdm8723->bPreDecBtPwr = pBtdm8723->bCurDecBtPwr;
-}
-
-static void
-btdm_2AntFwDacSwingLvl(struct rtw_adapter *padapter, u8 fwDacSwingLvl)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], set FW Dac Swing level = %d\n",  fwDacSwingLvl));
-	pBtdm8723->curFwDacSwingLvl = fwDacSwingLvl;
-
-	/* RTPRINT(FBT, BT_TRACE, ("[BTCoex], preFwDacSwingLvl =%d, curFwDacSwingLvl =%d\n", */
-	/*pBtdm8723->preFwDacSwingLvl, pBtdm8723->curFwDacSwingLvl)); */
-
-	if (pBtdm8723->preFwDacSwingLvl == pBtdm8723->curFwDacSwingLvl)
-		return;
-
-	btdm_SetFwDacSwingLevel(padapter, pBtdm8723->curFwDacSwingLvl);
-
-	pBtdm8723->preFwDacSwingLvl = pBtdm8723->curFwDacSwingLvl;
-}
-
-static void
-btdm_2AntRfShrink(struct rtw_adapter *padapter, u8 bRxRfShrinkOn)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTCoex], turn Rx RF Shrink = %s\n",
-		((bRxRfShrinkOn) ? "ON" : "OFF")));
-	pBtdm8723->bCurRfRxLpfShrink = bRxRfShrinkOn;
-
-	/* RTPRINT(FBT, BT_TRACE, ("[BTCoex], bPreRfRxLpfShrink =%d, bCurRfRxLpfShrink =%d\n", */
-	/*pBtdm8723->bPreRfRxLpfShrink, pBtdm8723->bCurRfRxLpfShrink)); */
-
-	if (pBtdm8723->bPreRfRxLpfShrink == pBtdm8723->bCurRfRxLpfShrink)
-		return;
-
-	BTDM_SetSwRfRxLpfCorner(padapter, (u8)pBtdm8723->bCurRfRxLpfShrink);
-
-	pBtdm8723->bPreRfRxLpfShrink = pBtdm8723->bCurRfRxLpfShrink;
-}
-
-static void
-btdm_2AntLowPenaltyRa(struct rtw_adapter *padapter, u8 bLowPenaltyRa)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTCoex], turn LowPenaltyRA = %s\n",
-		((bLowPenaltyRa) ? "ON" : "OFF")));
-	pBtdm8723->bCurLowPenaltyRa = bLowPenaltyRa;
-
-	/* RTPRINT(FBT, BT_TRACE, ("[BTCoex], bPreLowPenaltyRa =%d, bCurLowPenaltyRa =%d\n", */
-	/*pBtdm8723->bPreLowPenaltyRa, pBtdm8723->bCurLowPenaltyRa)); */
-
-	if (pBtdm8723->bPreLowPenaltyRa == pBtdm8723->bCurLowPenaltyRa)
-		return;
-
-	BTDM_SetSwPenaltyTxRateAdaptive(padapter, (u8)pBtdm8723->bCurLowPenaltyRa);
-
-	pBtdm8723->bPreLowPenaltyRa = pBtdm8723->bCurLowPenaltyRa;
-}
-
-static void
-btdm_2AntDacSwing(struct rtw_adapter *padapter,
-		  u8 bDacSwingOn, u32 dacSwingLvl)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTCoex], turn DacSwing =%s, dacSwingLvl = 0x%x\n",
-		(bDacSwingOn ? "ON" : "OFF"), dacSwingLvl));
-	pBtdm8723->bCurDacSwingOn = bDacSwingOn;
-	pBtdm8723->curDacSwingLvl = dacSwingLvl;
-
-	if ((pBtdm8723->bPreDacSwingOn == pBtdm8723->bCurDacSwingOn) &&
-	    (pBtdm8723->preDacSwingLvl == pBtdm8723->curDacSwingLvl))
-		return;
-
-	mdelay(30);
-	btdm_SetSwFullTimeDacSwing(padapter, bDacSwingOn, dacSwingLvl);
-
-	pBtdm8723->bPreDacSwingOn = pBtdm8723->bCurDacSwingOn;
-	pBtdm8723->preDacSwingLvl = pBtdm8723->curDacSwingLvl;
-}
-
-static void btdm_2AntAdcBackOff(struct rtw_adapter *padapter, u8 bAdcBackOff)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTCoex], turn AdcBackOff = %s\n",
-		((bAdcBackOff) ? "ON" : "OFF")));
-	pBtdm8723->bCurAdcBackOff = bAdcBackOff;
-
-	if (pBtdm8723->bPreAdcBackOff == pBtdm8723->bCurAdcBackOff)
-		return;
-
-	BTDM_BBBackOffLevel(padapter, (u8)pBtdm8723->bCurAdcBackOff);
-
-	pBtdm8723->bPreAdcBackOff = pBtdm8723->bCurAdcBackOff;
-}
-
-static void btdm_2AntAgcTable(struct rtw_adapter *padapter, u8 bAgcTableEn)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTCoex], %s Agc Table\n", ((bAgcTableEn) ? "Enable" : "Disable")));
-	pBtdm8723->bCurAgcTableEn = bAgcTableEn;
-
-	/* RTPRINT(FBT, BT_TRACE, ("[BTCoex], bPreAgcTableEn =%d, bCurAgcTableEn =%d\n", */
-	/*pBtdm8723->bPreAgcTableEn, pBtdm8723->bCurAgcTableEn)); */
-
-	if (pBtdm8723->bPreAgcTableEn == pBtdm8723->bCurAgcTableEn)
-		return;
-
-	BTDM_AGCTable(padapter, (u8)bAgcTableEn);
-
-	pBtdm8723->bPreAgcTableEn = pBtdm8723->bCurAgcTableEn;
-}
-
-static void
-btdm_2AntCoexTable(struct rtw_adapter *padapter,
-		   u32 val0x6c0, u32 val0x6c8, u8 val0x6cc)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], write Coex Table 0x6c0 = 0x%x, 0x6c8 = 0x%x, 0x6cc = 0x%x\n",
-		val0x6c0, val0x6c8, val0x6cc));
-	pBtdm8723->curVal0x6c0 = val0x6c0;
-	pBtdm8723->curVal0x6c8 = val0x6c8;
-	pBtdm8723->curVal0x6cc = val0x6cc;
-
-	/* RTPRINT(FBT, BT_TRACE, ("[BTCoex], preVal0x6c0 = 0x%x, preVal0x6c8 = 0x%x, preVal0x6cc = 0x%x !!\n", */
-	/*pBtdm8723->preVal0x6c0, pBtdm8723->preVal0x6c8, pBtdm8723->preVal0x6cc)); */
-	/* RTPRINT(FBT, BT_TRACE, ("[BTCoex], curVal0x6c0 = 0x%x, curVal0x6c8 = 0x%x, curVal0x6cc = 0x%x !!\n", */
-	/*pBtdm8723->curVal0x6c0, pBtdm8723->curVal0x6c8, pBtdm8723->curVal0x6cc)); */
-
-	if ((pBtdm8723->preVal0x6c0 == pBtdm8723->curVal0x6c0) &&
-	    (pBtdm8723->preVal0x6c8 == pBtdm8723->curVal0x6c8) &&
-	    (pBtdm8723->preVal0x6cc == pBtdm8723->curVal0x6cc))
-		return;
-
-	btdm_SetCoexTable(padapter, val0x6c0, val0x6c8, val0x6cc);
-
-	pBtdm8723->preVal0x6c0 = pBtdm8723->curVal0x6c0;
-	pBtdm8723->preVal0x6c8 = pBtdm8723->curVal0x6c8;
-	pBtdm8723->preVal0x6cc = pBtdm8723->curVal0x6cc;
-}
-
-static void btdm_2AntIgnoreWlanAct(struct rtw_adapter *padapter, u8 bEnable)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTCoex], turn Ignore WlanAct %s\n", (bEnable ? "ON" : "OFF")));
-	pBtdm8723->bCurIgnoreWlanAct = bEnable;
-
-
-	if (pBtdm8723->bPreIgnoreWlanAct == pBtdm8723->bCurIgnoreWlanAct)
-		return;
-
-	btdm_SetFwIgnoreWlanAct(padapter, bEnable);
-	pBtdm8723->bPreIgnoreWlanAct = pBtdm8723->bCurIgnoreWlanAct;
-}
-
-static void
-btdm_2AntSetFw3a(struct rtw_adapter *padapter, u8 byte1, u8 byte2,
-		 u8 byte3, u8 byte4, u8 byte5)
-{
-	u8 H2C_Parameter[5] = {0};
-
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	/*  byte1[1:0] != 0 means enable pstdma */
-	/*  for 2Ant bt coexist, if byte1 != 0 means enable pstdma */
-	if (byte1)
-		pHalData->bt_coexist.bFWCoexistAllOff = false;
-	H2C_Parameter[0] = byte1;
-	H2C_Parameter[1] = byte2;
-	H2C_Parameter[2] = byte3;
-	H2C_Parameter[3] = byte4;
-	H2C_Parameter[4] = byte5;
-
-	pHalData->bt_coexist.fw3aVal[0] = byte1;
-	pHalData->bt_coexist.fw3aVal[1] = byte2;
-	pHalData->bt_coexist.fw3aVal[2] = byte3;
-	pHalData->bt_coexist.fw3aVal[3] = byte4;
-	pHalData->bt_coexist.fw3aVal[4] = byte5;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], FW write 0x3a(5bytes) = 0x%x%08x\n",
-		H2C_Parameter[0],
-		H2C_Parameter[1]<<24|H2C_Parameter[2]<<16|H2C_Parameter[3]<<8|H2C_Parameter[4]));
-
-	FillH2CCmd(padapter, 0x3a, 5, H2C_Parameter);
-	}
-
-static void btdm_2AntPsTdma(struct rtw_adapter *padapter, u8 bTurnOn, u8 type)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-	u32			btTxRxCnt = 0;
-	u8 bTurnOnByCnt = false;
-	u8 psTdmaTypeByCnt = 0;
-
-	btTxRxCnt = BTDM_BtTxRxCounterH(padapter)+BTDM_BtTxRxCounterL(padapter);
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT TxRx Counters = %d\n", btTxRxCnt));
-	if (btTxRxCnt > 3000) {
-		bTurnOnByCnt = true;
-		psTdmaTypeByCnt = 8;
-
-		RTPRINT(FBT, BT_TRACE,
-			("[BTCoex], For BTTxRxCounters, turn %s PS TDMA, type =%d\n",
-			(bTurnOnByCnt ? "ON" : "OFF"), psTdmaTypeByCnt));
-		pBtdm8723->bCurPsTdmaOn = bTurnOnByCnt;
-		pBtdm8723->curPsTdma = psTdmaTypeByCnt;
-	} else {
-		RTPRINT(FBT, BT_TRACE,
-			("[BTCoex], turn %s PS TDMA, type =%d\n",
-			(bTurnOn ? "ON" : "OFF"), type));
-		pBtdm8723->bCurPsTdmaOn = bTurnOn;
-		pBtdm8723->curPsTdma = type;
-	}
-
-	if ((pBtdm8723->bPrePsTdmaOn == pBtdm8723->bCurPsTdmaOn) &&
-	    (pBtdm8723->prePsTdma == pBtdm8723->curPsTdma))
-		return;
-
-	if (bTurnOn) {
-		switch (type) {
-		case 1:
-		default:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x1a, 0x1a, 0xa1, 0x98);
-			break;
-		case 2:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x12, 0x12, 0xa1, 0x98);
-			break;
-		case 3:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0xa, 0xa, 0xa1, 0x98);
-			break;
-		case 4:
-			btdm_2AntSetFw3a(padapter, 0xa3, 0x5, 0x5, 0xa1, 0x80);
-			break;
-		case 5:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x1a, 0x1a, 0x20, 0x98);
-			break;
-		case 6:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x12, 0x12, 0x20, 0x98);
-			break;
-		case 7:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0xa, 0xa, 0x20, 0x98);
-			break;
-		case 8:
-			btdm_2AntSetFw3a(padapter, 0xa3, 0x5, 0x5, 0x20, 0x80);
-			break;
-		case 9:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x1a, 0x1a, 0xa1, 0x98);
-			break;
-		case 10:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x12, 0x12, 0xa1, 0x98);
-			break;
-		case 11:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0xa, 0xa, 0xa1, 0x98);
-			break;
-		case 12:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x5, 0x5, 0xa1, 0x98);
-			break;
-		case 13:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x1a, 0x1a, 0x20, 0x98);
-			break;
-		case 14:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x12, 0x12, 0x20, 0x98);
-			break;
-		case 15:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0xa, 0xa, 0x20, 0x98);
-			break;
-		case 16:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x5, 0x5, 0x20, 0x98);
-			break;
-		case 17:
-			btdm_2AntSetFw3a(padapter, 0xa3, 0x2f, 0x2f, 0x20, 0x80);
-			break;
-		case 18:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x5, 0x5, 0xa1, 0x98);
-			break;
-		case 19:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x25, 0x25, 0xa1, 0x98);
-			break;
-		case 20:
-			btdm_2AntSetFw3a(padapter, 0xe3, 0x25, 0x25, 0x20, 0x98);
-			break;
-		}
-	} else {
-		/*  disable PS tdma */
-		switch (type) {
-		case 0:
-			btdm_2AntSetFw3a(padapter, 0x0, 0x0, 0x0, 0x8, 0x0);
-			break;
-		case 1:
-			btdm_2AntSetFw3a(padapter, 0x0, 0x0, 0x0, 0x0, 0x0);
-			break;
-		default:
-			btdm_2AntSetFw3a(padapter, 0x0, 0x0, 0x0, 0x8, 0x0);
-			break;
-		}
-	}
-
-	/*  update pre state */
-	pBtdm8723->bPrePsTdmaOn =  pBtdm8723->bCurPsTdmaOn;
-	pBtdm8723->prePsTdma = pBtdm8723->curPsTdma;
-}
-
-static void btdm_2AntBtInquiryPage(struct rtw_adapter *padapter)
-{
-	btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-	btdm_2AntIgnoreWlanAct(padapter, false);
-	btdm_2AntPsTdma(padapter, true, 8);
-}
-
-static u8 btdm_HoldForBtInqPage(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u32 curTime = jiffies;
-
-	if (pHalData->bt_coexist.halCoex8723.bC2hBtInquiryPage) {
-		/*  bt inquiry or page is started. */
-		if (pHalData->bt_coexist.halCoex8723.btInqPageStartTime == 0) {
-			pHalData->bt_coexist.halCoex8723.btInqPageStartTime = curTime;
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT Inquiry/page is started at time : 0x%lx \n",
-			pHalData->bt_coexist.halCoex8723.btInqPageStartTime));
-		}
-	}
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT Inquiry/page started time : 0x%lx, curTime : 0x%x \n",
-		pHalData->bt_coexist.halCoex8723.btInqPageStartTime, curTime));
-
-	if (pHalData->bt_coexist.halCoex8723.btInqPageStartTime) {
-		if (((curTime - pHalData->bt_coexist.halCoex8723.btInqPageStartTime)/1000000) >= 10) {
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], BT Inquiry/page >= 10sec!!!"));
-			pHalData->bt_coexist.halCoex8723.btInqPageStartTime = 0;
-		}
-	}
-
-	if (pHalData->bt_coexist.halCoex8723.btInqPageStartTime) {
-		btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-		btdm_2AntIgnoreWlanAct(padapter, false);
-		btdm_2AntPsTdma(padapter, true, 8);
-		return true;
-	} else {
-		return false;
-	}
-}
-
-static u8 btdm_Is2Ant8723ACommonAction(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-	u8 bCommon = false;
-
-	RTPRINT(FBT, BT_TRACE, ("%s :BTDM_IsWifiConnectionExist =%x check_fwstate =%x pmlmepriv->fw_state = 0x%x\n", __func__, BTDM_IsWifiConnectionExist(padapter), check_fwstate(&padapter->mlmepriv, (_FW_UNDER_SURVEY|_FW_UNDER_LINKING)), padapter->mlmepriv.fw_state));
-
-	if ((!BTDM_IsWifiConnectionExist(padapter)) &&
-	    (!check_fwstate(&padapter->mlmepriv, (_FW_UNDER_SURVEY|_FW_UNDER_LINKING))) &&
-	    (BT_2ANT_BT_STATUS_IDLE == pBtdm8723->btStatus)) {
-		RTPRINT(FBT, BT_TRACE, ("Wifi idle + Bt idle!!\n"));
-
-		btdm_2AntLowPenaltyRa(padapter, false);
-		btdm_2AntRfShrink(padapter, false);
-		btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-
-		btdm_2AntIgnoreWlanAct(padapter, false);
-		btdm_2AntPsTdma(padapter, false, 0);
-		btdm_2AntFwDacSwingLvl(padapter, 0x20);
-		btdm_2AntDecBtPwr(padapter, false);
-
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, false);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-
-		bCommon = true;
-	} else if (((BTDM_IsWifiConnectionExist(padapter)) ||
-		   (check_fwstate(&padapter->mlmepriv, (_FW_UNDER_SURVEY|_FW_UNDER_LINKING)))) &&
-		   (BT_2ANT_BT_STATUS_IDLE == pBtdm8723->btStatus)) {
-		RTPRINT(FBT, BT_TRACE, ("Wifi non-idle + BT idle!!\n"));
-
-		btdm_2AntLowPenaltyRa(padapter, true);
-		btdm_2AntRfShrink(padapter, false);
-		btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-
-		btdm_2AntIgnoreWlanAct(padapter, false);
-		btdm_2AntPsTdma(padapter, false, 0);
-		btdm_2AntFwDacSwingLvl(padapter, 0x20);
-		btdm_2AntDecBtPwr(padapter, true);
-
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, false);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-
-		bCommon = true;
-	} else if ((!BTDM_IsWifiConnectionExist(padapter)) &&
-		   (!check_fwstate(&padapter->mlmepriv, (_FW_UNDER_SURVEY|_FW_UNDER_LINKING))) &&
-		   (BT_2ANT_BT_STATUS_CONNECTED_IDLE == pBtdm8723->btStatus)) {
-		RTPRINT(FBT, BT_TRACE, ("Wifi idle + Bt connected idle!!\n"));
-
-		btdm_2AntLowPenaltyRa(padapter, true);
-		btdm_2AntRfShrink(padapter, true);
-		btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-
-		btdm_2AntIgnoreWlanAct(padapter, false);
-		btdm_2AntPsTdma(padapter, false, 0);
-		btdm_2AntFwDacSwingLvl(padapter, 0x20);
-		btdm_2AntDecBtPwr(padapter, false);
-
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, false);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-
-		bCommon = true;
-	} else if (((BTDM_IsWifiConnectionExist(padapter)) ||
-		   (check_fwstate(&padapter->mlmepriv, (_FW_UNDER_SURVEY|_FW_UNDER_LINKING)))) &&
-		   (BT_2ANT_BT_STATUS_CONNECTED_IDLE == pBtdm8723->btStatus)) {
-		RTPRINT(FBT, BT_TRACE, ("Wifi non-idle + Bt connected idle!!\n"));
-
-		btdm_2AntLowPenaltyRa(padapter, true);
-		btdm_2AntRfShrink(padapter, true);
-		btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-
-		btdm_2AntIgnoreWlanAct(padapter, false);
-		btdm_2AntPsTdma(padapter, false, 0);
-		btdm_2AntFwDacSwingLvl(padapter, 0x20);
-		btdm_2AntDecBtPwr(padapter, true);
-
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, false);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-
-		bCommon = true;
-	} else if ((!BTDM_IsWifiConnectionExist(padapter)) &&
-		   (!check_fwstate(&padapter->mlmepriv, (_FW_UNDER_SURVEY|_FW_UNDER_LINKING))) &&
-		   (BT_2ANT_BT_STATUS_NON_IDLE == pBtdm8723->btStatus)) {
-		RTPRINT(FBT, BT_TRACE, ("Wifi idle + BT non-idle!!\n"));
-
-		btdm_2AntLowPenaltyRa(padapter, true);
-		btdm_2AntRfShrink(padapter, true);
-		btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-
-		btdm_2AntIgnoreWlanAct(padapter, false);
-		btdm_2AntPsTdma(padapter, false, 0);
-		btdm_2AntFwDacSwingLvl(padapter, 0x20);
-		btdm_2AntDecBtPwr(padapter, false);
-
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, false);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-
-		bCommon = true;
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("Wifi non-idle + BT non-idle!!\n"));
-		btdm_2AntLowPenaltyRa(padapter, true);
-		btdm_2AntRfShrink(padapter, true);
-		btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-		btdm_2AntIgnoreWlanAct(padapter, false);
-		btdm_2AntFwDacSwingLvl(padapter, 0x20);
-
-		bCommon = false;
-	}
-	return bCommon;
-}
-
-static void
-btdm_2AntTdmaDurationAdjust(struct rtw_adapter *padapter, u8 bScoHid,
-			    u8 bTxPause, u8 maxInterval)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-	static s32		up, dn, m, n, WaitCount;
-	s32			result;   /* 0: no change, +1: increase WiFi duration, -1: decrease WiFi duration */
-	u8 retryCount = 0;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], TdmaDurationAdjust()\n"));
-
-	if (pBtdm8723->bResetTdmaAdjust) {
-		pBtdm8723->bResetTdmaAdjust = false;
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], first run TdmaDurationAdjust()!!\n"));
-		if (bScoHid) {
-			if (bTxPause) {
-				btdm_2AntPsTdma(padapter, true, 15);
-				pBtdm8723->psTdmaDuAdjType = 15;
-			} else {
-				btdm_2AntPsTdma(padapter, true, 11);
-				pBtdm8723->psTdmaDuAdjType = 11;
-			}
-		} else {
-			if (bTxPause) {
-				btdm_2AntPsTdma(padapter, true, 7);
-				pBtdm8723->psTdmaDuAdjType = 7;
-			} else {
-				btdm_2AntPsTdma(padapter, true, 3);
-				pBtdm8723->psTdmaDuAdjType = 3;
-			}
-		}
-		up = 0;
-		dn = 0;
-		m = 1;
-		n = 3;
-		result = 0;
-		WaitCount = 0;
-	} else {
-		/* accquire the BT TRx retry count from BT_Info byte2 */
-		retryCount = pHalData->bt_coexist.halCoex8723.btRetryCnt;
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], retryCount = %d\n", retryCount));
-		result = 0;
-		WaitCount++;
-
-		if (retryCount == 0) {  /*  no retry in the last 2-second duration */
-			up++;
-			dn--;
-
-			if (dn <= 0)
-				dn = 0;
-
-			if (up >= n) {	/*  if ³sÄò n ­Ó2¬í retry count¬°0, «h½Õ¼eWiFi duration */
-				WaitCount = 0;
-				n = 3;
-				up = 0;
-				dn = 0;
-				result = 1;
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Increase wifi duration!!\n"));
-			}
-		} else if (retryCount <= 3) {	/*  <= 3 retry in the last 2-second duration */
-			up--;
-			dn++;
-
-			if (up <= 0)
-				up = 0;
-
-			if (dn == 2) {	/*  if ³sÄò 2 ­Ó2¬í retry count< 3, «h½Õ¯¶WiFi duration */
-				if (WaitCount <= 2)
-					m++; /*  ÁקK¤@ª½¦b¨â­Ólevel¤¤¨Ó¦^ */
-				else
-					m = 1;
-
-				if (m >= 20) /* m ³Ì¤j­È = 20 ' ³Ì¤j120¬í recheck¬O§_½Õ¾ã WiFi duration. */
-					m = 20;
-
-				n = 3*m;
-				up = 0;
-				dn = 0;
-				WaitCount = 0;
-				result = -1;
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], Decrease wifi duration for retryCounter<3!!\n"));
-			}
-		} else {  /* retry count > 3, ¥u­n1¦¸ retry count > 3, «h½Õ¯¶WiFi duration */
-			if (WaitCount == 1)
-				m++; /*  ÁקK¤@ª½¦b¨â­Ólevel¤¤¨Ó¦^ */
-			else
-				m = 1;
-
-			if (m >= 20) /* m ³Ì¤j­È = 20 ' ³Ì¤j120¬í recheck¬O§_½Õ¾ã WiFi duration. */
-				m = 20;
-			n = 3*m;
-			up = 0;
-			dn = 0;
-			WaitCount = 0;
-			result = -1;
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], Decrease wifi duration for retryCounter>3!!\n"));
-		}
-
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], max Interval = %d\n", maxInterval));
-		if (maxInterval == 1) {
-			if (bTxPause) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], TxPause = 1\n"));
-				if (pBtdm8723->curPsTdma == 1) {
-					btdm_2AntPsTdma(padapter, true, 5);
-					pBtdm8723->psTdmaDuAdjType = 5;
-				} else if (pBtdm8723->curPsTdma == 2) {
-					btdm_2AntPsTdma(padapter, true, 6);
-					pBtdm8723->psTdmaDuAdjType = 6;
-				} else if (pBtdm8723->curPsTdma == 3) {
-					btdm_2AntPsTdma(padapter, true, 7);
-					pBtdm8723->psTdmaDuAdjType = 7;
-				} else if (pBtdm8723->curPsTdma == 4) {
-					btdm_2AntPsTdma(padapter, true, 8);
-					pBtdm8723->psTdmaDuAdjType = 8;
-				}
-				if (pBtdm8723->curPsTdma == 9) {
-					btdm_2AntPsTdma(padapter, true, 13);
-					pBtdm8723->psTdmaDuAdjType = 13;
-				} else if (pBtdm8723->curPsTdma == 10) {
-					btdm_2AntPsTdma(padapter, true, 14);
-					pBtdm8723->psTdmaDuAdjType = 14;
-				} else if (pBtdm8723->curPsTdma == 11) {
-					btdm_2AntPsTdma(padapter, true, 15);
-					pBtdm8723->psTdmaDuAdjType = 15;
-				} else if (pBtdm8723->curPsTdma == 12) {
-					btdm_2AntPsTdma(padapter, true, 16);
-					pBtdm8723->psTdmaDuAdjType = 16;
-				}
-
-				if (result == -1) {
-					if (pBtdm8723->curPsTdma == 5) {
-						btdm_2AntPsTdma(padapter, true, 6);
-						pBtdm8723->psTdmaDuAdjType = 6;
-					} else if (pBtdm8723->curPsTdma == 6) {
-						btdm_2AntPsTdma(padapter, true, 7);
-						pBtdm8723->psTdmaDuAdjType = 7;
-					} else if (pBtdm8723->curPsTdma == 7) {
-						btdm_2AntPsTdma(padapter, true, 8);
-						pBtdm8723->psTdmaDuAdjType = 8;
-					} else if (pBtdm8723->curPsTdma == 13) {
-						btdm_2AntPsTdma(padapter, true, 14);
-						pBtdm8723->psTdmaDuAdjType = 14;
-					} else if (pBtdm8723->curPsTdma == 14) {
-						btdm_2AntPsTdma(padapter, true, 15);
-						pBtdm8723->psTdmaDuAdjType = 15;
-					} else if (pBtdm8723->curPsTdma == 15) {
-						btdm_2AntPsTdma(padapter, true, 16);
-						pBtdm8723->psTdmaDuAdjType = 16;
-					}
-				} else if (result == 1) {
-					if (pBtdm8723->curPsTdma == 8) {
-						btdm_2AntPsTdma(padapter, true, 7);
-						pBtdm8723->psTdmaDuAdjType = 7;
-					} else if (pBtdm8723->curPsTdma == 7) {
-						btdm_2AntPsTdma(padapter, true, 6);
-						pBtdm8723->psTdmaDuAdjType = 6;
-					} else if (pBtdm8723->curPsTdma == 6) {
-						btdm_2AntPsTdma(padapter, true, 5);
-						pBtdm8723->psTdmaDuAdjType = 5;
-					} else if (pBtdm8723->curPsTdma == 16) {
-						btdm_2AntPsTdma(padapter, true, 15);
-						pBtdm8723->psTdmaDuAdjType = 15;
-					} else if (pBtdm8723->curPsTdma == 15) {
-						btdm_2AntPsTdma(padapter, true, 14);
-						pBtdm8723->psTdmaDuAdjType = 14;
-					} else if (pBtdm8723->curPsTdma == 14) {
-						btdm_2AntPsTdma(padapter, true, 13);
-						pBtdm8723->psTdmaDuAdjType = 13;
-					}
-				}
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], TxPause = 0\n"));
-				if (pBtdm8723->curPsTdma == 5) {
-					btdm_2AntPsTdma(padapter, true, 1);
-					pBtdm8723->psTdmaDuAdjType = 1;
-				} else if (pBtdm8723->curPsTdma == 6) {
-					btdm_2AntPsTdma(padapter, true, 2);
-					pBtdm8723->psTdmaDuAdjType = 2;
-				} else if (pBtdm8723->curPsTdma == 7) {
-					btdm_2AntPsTdma(padapter, true, 3);
-					pBtdm8723->psTdmaDuAdjType = 3;
-				} else if (pBtdm8723->curPsTdma == 8) {
-					btdm_2AntPsTdma(padapter, true, 4);
-					pBtdm8723->psTdmaDuAdjType = 4;
-				}
-				if (pBtdm8723->curPsTdma == 13) {
-					btdm_2AntPsTdma(padapter, true, 9);
-					pBtdm8723->psTdmaDuAdjType = 9;
-				} else if (pBtdm8723->curPsTdma == 14) {
-					btdm_2AntPsTdma(padapter, true, 10);
-					pBtdm8723->psTdmaDuAdjType = 10;
-				} else if (pBtdm8723->curPsTdma == 15) {
-					btdm_2AntPsTdma(padapter, true, 11);
-					pBtdm8723->psTdmaDuAdjType = 11;
-				} else if (pBtdm8723->curPsTdma == 16) {
-					btdm_2AntPsTdma(padapter, true, 12);
-					pBtdm8723->psTdmaDuAdjType = 12;
-				}
-
-				if (result == -1) {
-					if (pBtdm8723->curPsTdma == 1) {
-						btdm_2AntPsTdma(padapter, true, 2);
-						pBtdm8723->psTdmaDuAdjType = 2;
-					} else if (pBtdm8723->curPsTdma == 2) {
-						btdm_2AntPsTdma(padapter, true, 3);
-						pBtdm8723->psTdmaDuAdjType = 3;
-					} else if (pBtdm8723->curPsTdma == 3) {
-						btdm_2AntPsTdma(padapter, true, 4);
-						pBtdm8723->psTdmaDuAdjType = 4;
-					} else if (pBtdm8723->curPsTdma == 9) {
-						btdm_2AntPsTdma(padapter, true, 10);
-						pBtdm8723->psTdmaDuAdjType = 10;
-					} else if (pBtdm8723->curPsTdma == 10) {
-						btdm_2AntPsTdma(padapter, true, 11);
-						pBtdm8723->psTdmaDuAdjType = 11;
-					} else if (pBtdm8723->curPsTdma == 11) {
-						btdm_2AntPsTdma(padapter, true, 12);
-						pBtdm8723->psTdmaDuAdjType = 12;
-					}
-				} else if (result == 1) {
-					if (pBtdm8723->curPsTdma == 4) {
-						btdm_2AntPsTdma(padapter, true, 3);
-						pBtdm8723->psTdmaDuAdjType = 3;
-					} else if (pBtdm8723->curPsTdma == 3) {
-						btdm_2AntPsTdma(padapter, true, 2);
-						pBtdm8723->psTdmaDuAdjType = 2;
-					} else if (pBtdm8723->curPsTdma == 2) {
-						btdm_2AntPsTdma(padapter, true, 1);
-						pBtdm8723->psTdmaDuAdjType = 1;
-					} else if (pBtdm8723->curPsTdma == 12) {
-						btdm_2AntPsTdma(padapter, true, 11);
-						pBtdm8723->psTdmaDuAdjType = 11;
-					} else if (pBtdm8723->curPsTdma == 11) {
-						btdm_2AntPsTdma(padapter, true, 10);
-						pBtdm8723->psTdmaDuAdjType = 10;
-					} else if (pBtdm8723->curPsTdma == 10) {
-						btdm_2AntPsTdma(padapter, true, 9);
-						pBtdm8723->psTdmaDuAdjType = 9;
-					}
-				}
-			}
-		} else if (maxInterval == 2) {
-			if (bTxPause) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], TxPause = 1\n"));
-				if (pBtdm8723->curPsTdma == 1) {
-					btdm_2AntPsTdma(padapter, true, 6);
-					pBtdm8723->psTdmaDuAdjType = 6;
-				} else if (pBtdm8723->curPsTdma == 2) {
-					btdm_2AntPsTdma(padapter, true, 6);
-					pBtdm8723->psTdmaDuAdjType = 6;
-				} else if (pBtdm8723->curPsTdma == 3) {
-					btdm_2AntPsTdma(padapter, true, 7);
-					pBtdm8723->psTdmaDuAdjType = 7;
-				} else if (pBtdm8723->curPsTdma == 4) {
-					btdm_2AntPsTdma(padapter, true, 8);
-					pBtdm8723->psTdmaDuAdjType = 8;
-				}
-				if (pBtdm8723->curPsTdma == 9) {
-					btdm_2AntPsTdma(padapter, true, 14);
-					pBtdm8723->psTdmaDuAdjType = 14;
-				} else if (pBtdm8723->curPsTdma == 10) {
-					btdm_2AntPsTdma(padapter, true, 14);
-					pBtdm8723->psTdmaDuAdjType = 14;
-				} else if (pBtdm8723->curPsTdma == 11) {
-					btdm_2AntPsTdma(padapter, true, 15);
-					pBtdm8723->psTdmaDuAdjType = 15;
-				} else if (pBtdm8723->curPsTdma == 12) {
-					btdm_2AntPsTdma(padapter, true, 16);
-					pBtdm8723->psTdmaDuAdjType = 16;
-				}
-				if (result == -1) {
-					if (pBtdm8723->curPsTdma == 5) {
-						btdm_2AntPsTdma(padapter, true, 6);
-						pBtdm8723->psTdmaDuAdjType = 6;
-					} else if (pBtdm8723->curPsTdma == 6) {
-						btdm_2AntPsTdma(padapter, true, 7);
-						pBtdm8723->psTdmaDuAdjType = 7;
-					} else if (pBtdm8723->curPsTdma == 7) {
-						btdm_2AntPsTdma(padapter, true, 8);
-						pBtdm8723->psTdmaDuAdjType = 8;
-					} else if (pBtdm8723->curPsTdma == 13) {
-						btdm_2AntPsTdma(padapter, true, 14);
-						pBtdm8723->psTdmaDuAdjType = 14;
-					} else if (pBtdm8723->curPsTdma == 14) {
-						btdm_2AntPsTdma(padapter, true, 15);
-						pBtdm8723->psTdmaDuAdjType = 15;
-					} else if (pBtdm8723->curPsTdma == 15) {
-						btdm_2AntPsTdma(padapter, true, 16);
-						pBtdm8723->psTdmaDuAdjType = 16;
-					}
-				} else if (result == 1) {
-					if (pBtdm8723->curPsTdma == 8) {
-						btdm_2AntPsTdma(padapter, true, 7);
-						pBtdm8723->psTdmaDuAdjType = 7;
-					} else if (pBtdm8723->curPsTdma == 7) {
-						btdm_2AntPsTdma(padapter, true, 6);
-						pBtdm8723->psTdmaDuAdjType = 6;
-					} else if (pBtdm8723->curPsTdma == 6) {
-						btdm_2AntPsTdma(padapter, true, 6);
-						pBtdm8723->psTdmaDuAdjType = 6;
-					} else if (pBtdm8723->curPsTdma == 16) {
-						btdm_2AntPsTdma(padapter, true, 15);
-						pBtdm8723->psTdmaDuAdjType = 15;
-					} else if (pBtdm8723->curPsTdma == 15) {
-						btdm_2AntPsTdma(padapter, true, 14);
-						pBtdm8723->psTdmaDuAdjType = 14;
-					} else if (pBtdm8723->curPsTdma == 14) {
-						btdm_2AntPsTdma(padapter, true, 14);
-						pBtdm8723->psTdmaDuAdjType = 14;
-					}
-				}
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], TxPause = 0\n"));
-				if (pBtdm8723->curPsTdma == 5) {
-					btdm_2AntPsTdma(padapter, true, 2);
-					pBtdm8723->psTdmaDuAdjType = 2;
-				} else if (pBtdm8723->curPsTdma == 6) {
-					btdm_2AntPsTdma(padapter, true, 2);
-					pBtdm8723->psTdmaDuAdjType = 2;
-				} else if (pBtdm8723->curPsTdma == 7) {
-					btdm_2AntPsTdma(padapter, true, 3);
-					pBtdm8723->psTdmaDuAdjType = 3;
-				} else if (pBtdm8723->curPsTdma == 8) {
-					btdm_2AntPsTdma(padapter, true, 4);
-					pBtdm8723->psTdmaDuAdjType = 4;
-				}
-				if (pBtdm8723->curPsTdma == 13) {
-					btdm_2AntPsTdma(padapter, true, 10);
-					pBtdm8723->psTdmaDuAdjType = 10;
-				} else if (pBtdm8723->curPsTdma == 14) {
-					btdm_2AntPsTdma(padapter, true, 10);
-					pBtdm8723->psTdmaDuAdjType = 10;
-				} else if (pBtdm8723->curPsTdma == 15) {
-					btdm_2AntPsTdma(padapter, true, 11);
-					pBtdm8723->psTdmaDuAdjType = 11;
-				} else if (pBtdm8723->curPsTdma == 16) {
-					btdm_2AntPsTdma(padapter, true, 12);
-					pBtdm8723->psTdmaDuAdjType = 12;
-				}
-				if (result == -1) {
-					if (pBtdm8723->curPsTdma == 1) {
-						btdm_2AntPsTdma(padapter, true, 2);
-						pBtdm8723->psTdmaDuAdjType = 2;
-					} else if (pBtdm8723->curPsTdma == 2) {
-						btdm_2AntPsTdma(padapter, true, 3);
-						pBtdm8723->psTdmaDuAdjType = 3;
-					} else if (pBtdm8723->curPsTdma == 3) {
-						btdm_2AntPsTdma(padapter, true, 4);
-						pBtdm8723->psTdmaDuAdjType = 4;
-					} else if (pBtdm8723->curPsTdma == 9) {
-						btdm_2AntPsTdma(padapter, true, 10);
-						pBtdm8723->psTdmaDuAdjType = 10;
-					} else if (pBtdm8723->curPsTdma == 10) {
-						btdm_2AntPsTdma(padapter, true, 11);
-						pBtdm8723->psTdmaDuAdjType = 11;
-					} else if (pBtdm8723->curPsTdma == 11) {
-						btdm_2AntPsTdma(padapter, true, 12);
-						pBtdm8723->psTdmaDuAdjType = 12;
-					}
-				} else if (result == 1) {
-					if (pBtdm8723->curPsTdma == 4) {
-						btdm_2AntPsTdma(padapter, true, 3);
-						pBtdm8723->psTdmaDuAdjType = 3;
-					} else if (pBtdm8723->curPsTdma == 3) {
-						btdm_2AntPsTdma(padapter, true, 2);
-						pBtdm8723->psTdmaDuAdjType = 2;
-					} else if (pBtdm8723->curPsTdma == 2) {
-						btdm_2AntPsTdma(padapter, true, 2);
-						pBtdm8723->psTdmaDuAdjType = 2;
-					} else if (pBtdm8723->curPsTdma == 12) {
-						btdm_2AntPsTdma(padapter, true, 11);
-						pBtdm8723->psTdmaDuAdjType = 11;
-					} else if (pBtdm8723->curPsTdma == 11) {
-						btdm_2AntPsTdma(padapter, true, 10);
-						pBtdm8723->psTdmaDuAdjType = 10;
-					} else if (pBtdm8723->curPsTdma == 10) {
-						btdm_2AntPsTdma(padapter, true, 10);
-						pBtdm8723->psTdmaDuAdjType = 10;
-					}
-				}
-			}
-		} else if (maxInterval == 3) {
-			if (bTxPause) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], TxPause = 1\n"));
-				if (pBtdm8723->curPsTdma == 1) {
-					btdm_2AntPsTdma(padapter, true, 7);
-					pBtdm8723->psTdmaDuAdjType = 7;
-				} else if (pBtdm8723->curPsTdma == 2) {
-					btdm_2AntPsTdma(padapter, true, 7);
-					pBtdm8723->psTdmaDuAdjType = 7;
-				} else if (pBtdm8723->curPsTdma == 3) {
-					btdm_2AntPsTdma(padapter, true, 7);
-					pBtdm8723->psTdmaDuAdjType = 7;
-				} else if (pBtdm8723->curPsTdma == 4) {
-					btdm_2AntPsTdma(padapter, true, 8);
-					pBtdm8723->psTdmaDuAdjType = 8;
-				}
-				if (pBtdm8723->curPsTdma == 9) {
-					btdm_2AntPsTdma(padapter, true, 15);
-					pBtdm8723->psTdmaDuAdjType = 15;
-				} else if (pBtdm8723->curPsTdma == 10) {
-					btdm_2AntPsTdma(padapter, true, 15);
-					pBtdm8723->psTdmaDuAdjType = 15;
-				} else if (pBtdm8723->curPsTdma == 11) {
-					btdm_2AntPsTdma(padapter, true, 15);
-					pBtdm8723->psTdmaDuAdjType = 15;
-				} else if (pBtdm8723->curPsTdma == 12) {
-					btdm_2AntPsTdma(padapter, true, 16);
-					pBtdm8723->psTdmaDuAdjType = 16;
-				}
-				if (result == -1) {
-					if (pBtdm8723->curPsTdma == 5) {
-						btdm_2AntPsTdma(padapter, true, 7);
-						pBtdm8723->psTdmaDuAdjType = 7;
-					} else if (pBtdm8723->curPsTdma == 6) {
-						btdm_2AntPsTdma(padapter, true, 7);
-						pBtdm8723->psTdmaDuAdjType = 7;
-					} else if (pBtdm8723->curPsTdma == 7) {
-						btdm_2AntPsTdma(padapter, true, 8);
-						pBtdm8723->psTdmaDuAdjType = 8;
-					} else if (pBtdm8723->curPsTdma == 13) {
-						btdm_2AntPsTdma(padapter, true, 15);
-						pBtdm8723->psTdmaDuAdjType = 15;
-					} else if (pBtdm8723->curPsTdma == 14) {
-						btdm_2AntPsTdma(padapter, true, 15);
-						pBtdm8723->psTdmaDuAdjType = 15;
-					} else if (pBtdm8723->curPsTdma == 15) {
-						btdm_2AntPsTdma(padapter, true, 16);
-						pBtdm8723->psTdmaDuAdjType = 16;
-					}
-				} else if (result == 1) {
-					if (pBtdm8723->curPsTdma == 8) {
-						btdm_2AntPsTdma(padapter, true, 7);
-						pBtdm8723->psTdmaDuAdjType = 7;
-					} else if (pBtdm8723->curPsTdma == 7) {
-						btdm_2AntPsTdma(padapter, true, 7);
-						pBtdm8723->psTdmaDuAdjType = 7;
-					} else if (pBtdm8723->curPsTdma == 6) {
-						btdm_2AntPsTdma(padapter, true, 7);
-						pBtdm8723->psTdmaDuAdjType = 7;
-					} else if (pBtdm8723->curPsTdma == 16) {
-						btdm_2AntPsTdma(padapter, true, 15);
-						pBtdm8723->psTdmaDuAdjType = 15;
-					} else if (pBtdm8723->curPsTdma == 15) {
-						btdm_2AntPsTdma(padapter, true, 15);
-						pBtdm8723->psTdmaDuAdjType = 15;
-					} else if (pBtdm8723->curPsTdma == 14) {
-						btdm_2AntPsTdma(padapter, true, 15);
-						pBtdm8723->psTdmaDuAdjType = 15;
-					}
-				}
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], TxPause = 0\n"));
-				if (pBtdm8723->curPsTdma == 5) {
-					btdm_2AntPsTdma(padapter, true, 3);
-					pBtdm8723->psTdmaDuAdjType = 3;
-				} else if (pBtdm8723->curPsTdma == 6) {
-					btdm_2AntPsTdma(padapter, true, 3);
-					pBtdm8723->psTdmaDuAdjType = 3;
-				} else if (pBtdm8723->curPsTdma == 7) {
-					btdm_2AntPsTdma(padapter, true, 3);
-					pBtdm8723->psTdmaDuAdjType = 3;
-				} else if (pBtdm8723->curPsTdma == 8) {
-					btdm_2AntPsTdma(padapter, true, 4);
-					pBtdm8723->psTdmaDuAdjType = 4;
-				}
-				if (pBtdm8723->curPsTdma == 13) {
-					btdm_2AntPsTdma(padapter, true, 11);
-					pBtdm8723->psTdmaDuAdjType = 11;
-				} else if (pBtdm8723->curPsTdma == 14) {
-					btdm_2AntPsTdma(padapter, true, 11);
-					pBtdm8723->psTdmaDuAdjType = 11;
-				} else if (pBtdm8723->curPsTdma == 15) {
-					btdm_2AntPsTdma(padapter, true, 11);
-					pBtdm8723->psTdmaDuAdjType = 11;
-				} else if (pBtdm8723->curPsTdma == 16) {
-					btdm_2AntPsTdma(padapter, true, 12);
-					pBtdm8723->psTdmaDuAdjType = 12;
-				}
-				if (result == -1) {
-					if (pBtdm8723->curPsTdma == 1) {
-						btdm_2AntPsTdma(padapter, true, 3);
-						pBtdm8723->psTdmaDuAdjType = 3;
-					} else if (pBtdm8723->curPsTdma == 2) {
-						btdm_2AntPsTdma(padapter, true, 3);
-						pBtdm8723->psTdmaDuAdjType = 3;
-					} else if (pBtdm8723->curPsTdma == 3) {
-						btdm_2AntPsTdma(padapter, true, 4);
-						pBtdm8723->psTdmaDuAdjType = 4;
-					} else if (pBtdm8723->curPsTdma == 9) {
-						btdm_2AntPsTdma(padapter, true, 11);
-						pBtdm8723->psTdmaDuAdjType = 11;
-					} else if (pBtdm8723->curPsTdma == 10) {
-						btdm_2AntPsTdma(padapter, true, 11);
-						pBtdm8723->psTdmaDuAdjType = 11;
-					} else if (pBtdm8723->curPsTdma == 11) {
-						btdm_2AntPsTdma(padapter, true, 12);
-						pBtdm8723->psTdmaDuAdjType = 12;
-					}
-				} else if (result == 1) {
-					if (pBtdm8723->curPsTdma == 4) {
-						btdm_2AntPsTdma(padapter, true, 3);
-						pBtdm8723->psTdmaDuAdjType = 3;
-					} else if (pBtdm8723->curPsTdma == 3) {
-						btdm_2AntPsTdma(padapter, true, 3);
-						pBtdm8723->psTdmaDuAdjType = 3;
-					} else if (pBtdm8723->curPsTdma == 2) {
-						btdm_2AntPsTdma(padapter, true, 3);
-						pBtdm8723->psTdmaDuAdjType = 3;
-					} else if (pBtdm8723->curPsTdma == 12) {
-						btdm_2AntPsTdma(padapter, true, 11);
-						pBtdm8723->psTdmaDuAdjType = 11;
-					} else if (pBtdm8723->curPsTdma == 11) {
-						btdm_2AntPsTdma(padapter, true, 11);
-						pBtdm8723->psTdmaDuAdjType = 11;
-					} else if (pBtdm8723->curPsTdma == 10) {
-						btdm_2AntPsTdma(padapter, true, 11);
-						pBtdm8723->psTdmaDuAdjType = 11;
-					}
-				}
-			}
-		}
-	}
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], PsTdma type : recordPsTdma =%d\n", pBtdm8723->psTdmaDuAdjType));
-	/*  if current PsTdma not match with the recorded one (when scan, dhcp...), */
-	/*  then we have to adjust it back to the previous record one. */
-	if (pBtdm8723->curPsTdma != pBtdm8723->psTdmaDuAdjType) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], PsTdma type dismatch!!!, curPsTdma =%d, recordPsTdma =%d\n",
-			pBtdm8723->curPsTdma, pBtdm8723->psTdmaDuAdjType));
-
-		if (!check_fwstate(&padapter->mlmepriv, _FW_UNDER_SURVEY|_FW_UNDER_LINKING))
-			btdm_2AntPsTdma(padapter, true, pBtdm8723->psTdmaDuAdjType);
-		else
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], roaming/link/scan is under progress, will adjust next time!!!\n"));
-	}
-}
-
-/*  default Action */
-/*  SCO only or SCO+PAN(HS) */
-static void btdm_2Ant8723ASCOAction(struct rtw_adapter *padapter)
-{
-	u8 btRssiState, btRssiState1;
-
-	if (btdm_NeedToDecBtPwr(padapter))
-		btdm_2AntDecBtPwr(padapter, true);
-	else
-		btdm_2AntDecBtPwr(padapter, false);
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		/*  fw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntPsTdma(padapter, true, 11);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			btdm_2AntPsTdma(padapter, true, 15);
-		}
-
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, true);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		btRssiState1 = BTDM_CheckCoexRSSIState1(padapter, 2, 27, 0);
-
-		/*  fw mechanism */
-		if ((btRssiState1 == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState1 == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntPsTdma(padapter, true, 11);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			btdm_2AntPsTdma(padapter, true, 15);
-		}
-
-		/*  sw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-static void btdm_2Ant8723AHIDAction(struct rtw_adapter *padapter)
-{
-	u8 btRssiState, btRssiState1;
-
-	if (btdm_NeedToDecBtPwr(padapter))
-		btdm_2AntDecBtPwr(padapter, true);
-	else
-		btdm_2AntDecBtPwr(padapter, false);
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-			/*  fw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntPsTdma(padapter, true, 9);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			btdm_2AntPsTdma(padapter, true, 13);
-		}
-
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, false);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		btRssiState1 = BTDM_CheckCoexRSSIState1(padapter, 2, 27, 0);
-
-		/*  fw mechanism */
-		if ((btRssiState1 == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState1 == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntPsTdma(padapter, true, 9);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			btdm_2AntPsTdma(padapter, true, 13);
-		}
-
-		/*  sw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-/* A2DP only / PAN(EDR) only/ A2DP+PAN(HS) */
-static void btdm_2Ant8723AA2DPAction(struct rtw_adapter *padapter)
-{
-	u8 btRssiState, btRssiState1;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 btInfoExt = pHalData->bt_coexist.halCoex8723.btInfoExt;
-
-	if (btdm_NeedToDecBtPwr(padapter))
-		btdm_2AntDecBtPwr(padapter, true);
-	else
-		btdm_2AntDecBtPwr(padapter, false);
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-
-		/*  fw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, false, false, 3);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, false, false, 1);
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, false, true, 3);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-			btdm_2AntTdmaDurationAdjust(padapter, false, true, 1);
-			}
-		}
-
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, true);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		btRssiState1 = BTDM_CheckCoexRSSIState1(padapter, 2, 27, 0);
-
-		/*  fw mechanism */
-		if ((btRssiState1 == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState1 == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, false, false, 3);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, false, false, 1);
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, false, true, 3);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, false, true, 1);
-			}
-		}
-
-		/*  sw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-static void btdm_2Ant8723APANEDRAction(struct rtw_adapter *padapter)
-{
-	u8 btRssiState, btRssiState1;
-
-	if (btdm_NeedToDecBtPwr(padapter))
-		btdm_2AntDecBtPwr(padapter, true);
-	else
-		btdm_2AntDecBtPwr(padapter, false);
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-
-		/*  fw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntPsTdma(padapter, true, 2);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntPsTdma(padapter, true, 6);
-		}
-
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, true);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		btRssiState1 = BTDM_CheckCoexRSSIState1(padapter, 2, 27, 0);
-
-		/*  fw mechanism */
-		if ((btRssiState1 == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState1 == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntPsTdma(padapter, true, 2);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			btdm_2AntPsTdma(padapter, true, 6);
-		}
-
-		/*  sw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-/* PAN(HS) only */
-static void btdm_2Ant8723APANHSAction(struct rtw_adapter *padapter)
-{
-	u8 btRssiState;
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 47, 0);
-		/*  fw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			btdm_2AntDecBtPwr(padapter, true);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntDecBtPwr(padapter, false);
-		}
-		btdm_2AntPsTdma(padapter, false, 0);
-
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, true);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 47, 0);
-
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high\n"));
-			/*  fw mechanism */
-			btdm_2AntDecBtPwr(padapter, true);
-			btdm_2AntPsTdma(padapter, false, 0);
-
-			/*  sw mechanism */
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low\n"));
-			/*  fw mechanism */
-			btdm_2AntDecBtPwr(padapter, false);
-			btdm_2AntPsTdma(padapter, false, 0);
-
-			/*  sw mechanism */
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-/* PAN(EDR)+A2DP */
-static void btdm_2Ant8723APANEDRA2DPAction(struct rtw_adapter *padapter)
-{
-	u8 btRssiState, btRssiState1, btInfoExt;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	btInfoExt = pHalData->bt_coexist.halCoex8723.btInfoExt;
-
-	if (btdm_NeedToDecBtPwr(padapter))
-		btdm_2AntDecBtPwr(padapter, true);
-	else
-		btdm_2AntDecBtPwr(padapter, false);
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			/*  fw mechanism */
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntPsTdma(padapter, true, 4);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntPsTdma(padapter, true, 2);
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			/*  fw mechanism */
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntPsTdma(padapter, true, 8);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntPsTdma(padapter, true, 6);
-			}
-		}
-
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, true);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		btRssiState1 = BTDM_CheckCoexRSSIState1(padapter, 2, 27, 0);
-
-		if ((btRssiState1 == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState1 == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			/*  fw mechanism */
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntPsTdma(padapter, true, 4);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntPsTdma(padapter, true, 2);
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			/*  fw mechanism */
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntPsTdma(padapter, true, 8);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntPsTdma(padapter, true, 6);
-			}
-		}
-
-		/*  sw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-static void btdm_2Ant8723APANEDRHIDAction(struct rtw_adapter *padapter)
-{
-	u8 btRssiState, btRssiState1;
-
-	if (btdm_NeedToDecBtPwr(padapter))
-		btdm_2AntDecBtPwr(padapter, true);
-	else
-		btdm_2AntDecBtPwr(padapter, false);
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		/*  fw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntPsTdma(padapter, true, 10);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntPsTdma(padapter, true, 14);
-		}
-
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, true);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		btRssiState1 = BTDM_CheckCoexRSSIState1(padapter, 2, 27, 0);
-
-		/*  fw mechanism */
-		if ((btRssiState1 == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState1 == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntPsTdma(padapter, true, 10);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			btdm_2AntPsTdma(padapter, true, 14);
-		}
-
-		/*  sw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-/*  HID+A2DP+PAN(EDR) */
-static void btdm_2Ant8723AHIDA2DPPANEDRAction(struct rtw_adapter *padapter)
-{
-	u8 btRssiState, btRssiState1, btInfoExt;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	btInfoExt = pHalData->bt_coexist.halCoex8723.btInfoExt;
-
-	if (btdm_NeedToDecBtPwr(padapter))
-		btdm_2AntDecBtPwr(padapter, true);
-	else
-		btdm_2AntDecBtPwr(padapter, false);
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntPsTdma(padapter, true, 12);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntPsTdma(padapter, true, 10);
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntPsTdma(padapter, true, 16);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntPsTdma(padapter, true, 14);
-			}
-		}
-
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, true);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState1 = BTDM_CheckCoexRSSIState1(padapter, 2, 37, 0);
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 27, 0);
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntPsTdma(padapter, true, 12);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntPsTdma(padapter, true, 10);
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntPsTdma(padapter, true, 16);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntPsTdma(padapter, true, 14);
-			}
-		}
-
-		/*  sw mechanism */
-		if ((btRssiState1 == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState1 == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-static void btdm_2Ant8723AHIDA2DPAction(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 btRssiState, btRssiState1, btInfoExt;
-
-	btInfoExt = pHalData->bt_coexist.halCoex8723.btInfoExt;
-
-	if (btdm_NeedToDecBtPwr(padapter))
-		btdm_2AntDecBtPwr(padapter, true);
-	else
-		btdm_2AntDecBtPwr(padapter, false);
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, true, false, 3);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, true, false, 1);
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, true, true, 3);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, true, true, 1);
-			}
-		}
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, true);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		btRssiState1 = BTDM_CheckCoexRSSIState(padapter, 2, 27, 0);
-
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, true, false, 3);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, true, false, 1);
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			if (btInfoExt&BIT(0)) {	/* a2dp rate, 1:basic /0:edr */
-				RTPRINT(FBT, BT_TRACE, ("a2dp basic rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, true, true, 3);
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("a2dp edr rate \n"));
-				btdm_2AntTdmaDurationAdjust(padapter, true, true, 1);
-			}
-		}
-		if ((btRssiState1 == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState1 == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			/*  sw mechanism */
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			/*  sw mechanism */
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-static void btdm_2Ant8723AA2dp(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 btRssiState, btRssiState1, btInfoExt;
-
-	btInfoExt = pHalData->bt_coexist.halCoex8723.btInfoExt;
-
-	if (btdm_NeedToDecBtPwr(padapter))
-		btdm_2AntDecBtPwr(padapter, true);
-	else
-		btdm_2AntDecBtPwr(padapter, false);
-	/*  coex table */
-	btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-	btdm_2AntIgnoreWlanAct(padapter, false);
-
-	if (BTDM_IsHT40(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("HT40\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 37, 0);
-		/*  fw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntTdmaDurationAdjust(padapter, false, false, 1);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			btdm_2AntTdmaDurationAdjust(padapter, false, true, 1);
-		}
-
-		/*  sw mechanism */
-		btdm_2AntAgcTable(padapter, false);
-		btdm_2AntAdcBackOff(padapter, true);
-		btdm_2AntDacSwing(padapter, false, 0xc0);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("HT20 or Legacy\n"));
-		btRssiState = BTDM_CheckCoexRSSIState(padapter, 2, 47, 0);
-		btRssiState1 = BTDM_CheckCoexRSSIState1(padapter, 2, 27, 0);
-
-		/*  fw mechanism */
-		if ((btRssiState1 == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState1 == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 high \n"));
-			PlatformEFIOWrite1Byte(padapter, 0x883, 0x40);
-			btdm_2AntTdmaDurationAdjust(padapter, false, false, 1);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi-1 low \n"));
-			btdm_2AntTdmaDurationAdjust(padapter, false, true, 1);
-		}
-
-		/*  sw mechanism */
-		if ((btRssiState == BT_RSSI_STATE_HIGH) ||
-		    (btRssiState == BT_RSSI_STATE_STAY_HIGH)) {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi high \n"));
-			btdm_2AntAgcTable(padapter, true);
-			btdm_2AntAdcBackOff(padapter, true);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("Wifi rssi low \n"));
-			btdm_2AntAgcTable(padapter, false);
-			btdm_2AntAdcBackOff(padapter, false);
-			btdm_2AntDacSwing(padapter, false, 0xc0);
-		}
-	}
-}
-
-/*  extern function start with BTDM_ */
-static void BTDM_2AntParaInit(struct rtw_adapter *padapter)
-{
-
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], 2Ant Parameter Init!!\n"));
-
-	/*  Enable counter statistics */
-	rtl8723au_write8(padapter, 0x76e, 0x4);
-	rtl8723au_write8(padapter, 0x778, 0x3);
-	rtl8723au_write8(padapter, 0x40, 0x20);
-
-	/*  force to reset coex mechanism */
-	pBtdm8723->preVal0x6c0 = 0x0;
-	btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-
-	pBtdm8723->bPrePsTdmaOn = true;
-	btdm_2AntPsTdma(padapter, false, 0);
-
-	pBtdm8723->preFwDacSwingLvl = 0x10;
-	btdm_2AntFwDacSwingLvl(padapter, 0x20);
-
-	pBtdm8723->bPreDecBtPwr = true;
-	btdm_2AntDecBtPwr(padapter, false);
-
-	pBtdm8723->bPreAgcTableEn = true;
-	btdm_2AntAgcTable(padapter, false);
-
-	pBtdm8723->bPreAdcBackOff = true;
-	btdm_2AntAdcBackOff(padapter, false);
-
-	pBtdm8723->bPreLowPenaltyRa = true;
-	btdm_2AntLowPenaltyRa(padapter, false);
-
-	pBtdm8723->bPreRfRxLpfShrink = true;
-	btdm_2AntRfShrink(padapter, false);
-
-	pBtdm8723->bPreDacSwingOn = true;
-	btdm_2AntDacSwing(padapter, false, 0xc0);
-
-	pBtdm8723->bPreIgnoreWlanAct = true;
-	btdm_2AntIgnoreWlanAct(padapter, false);
-}
-
-static void BTDM_2AntHwCoexAllOff8723A(struct rtw_adapter *padapter)
-{
-	btdm_2AntCoexTable(padapter, 0x55555555, 0xffff, 0x3);
-}
-
-static void BTDM_2AntFwCoexAllOff8723A(struct rtw_adapter *padapter)
-{
-	btdm_2AntIgnoreWlanAct(padapter, false);
-	btdm_2AntPsTdma(padapter, false, 0);
-	btdm_2AntFwDacSwingLvl(padapter, 0x20);
-	btdm_2AntDecBtPwr(padapter, false);
-}
-
-static void BTDM_2AntSwCoexAllOff8723A(struct rtw_adapter *padapter)
-{
-	btdm_2AntAgcTable(padapter, false);
-	btdm_2AntAdcBackOff(padapter, false);
-	btdm_2AntLowPenaltyRa(padapter, false);
-	btdm_2AntRfShrink(padapter, false);
-	btdm_2AntDacSwing(padapter, false, 0xc0);
-}
-
-static void BTDM_2AntFwC2hBtInfo8723A(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-	u8 btInfo = 0;
-	u8 algorithm = BT_2ANT_COEX_ALGO_UNDEFINED;
-	u8 bBtLinkExist = false, bBtHsModeExist = false;
-
-	btInfo = pHalData->bt_coexist.halCoex8723.c2hBtInfoOriginal;
-	pBtdm8723->btStatus = BT_2ANT_BT_STATUS_IDLE;
-
-	/*  check BIT2 first ==> check if bt is under inquiry or page scan */
-	if (btInfo & BIT(2)) {
-		if (!pHalData->bt_coexist.halCoex8723.bC2hBtInquiryPage) {
-			pBtMgnt->ExtConfig.bHoldForBtOperation = true;
-			pBtMgnt->ExtConfig.bHoldPeriodCnt = 1;
-			btdm_2AntBtInquiryPage(padapter);
-		} else {
-			pBtMgnt->ExtConfig.bHoldPeriodCnt++;
-			btdm_HoldForBtInqPage(padapter);
-		}
-		pHalData->bt_coexist.halCoex8723.bC2hBtInquiryPage = true;
-
-	} else {
-		pHalData->bt_coexist.halCoex8723.bC2hBtInquiryPage = false;
-		pBtMgnt->ExtConfig.bHoldForBtOperation = false;
-		pBtMgnt->ExtConfig.bHoldPeriodCnt = 0;
-
-	}
-	RTPRINT(FBT, BT_TRACE,
-		("[BTC2H], pHalData->bt_coexist.halCoex8723.bC2hBtInquiryPage =%x pBtMgnt->ExtConfig.bHoldPeriodCnt =%x pBtMgnt->ExtConfig.bHoldForBtOperation =%x\n",
-		pHalData->bt_coexist.halCoex8723.bC2hBtInquiryPage,
-		pBtMgnt->ExtConfig.bHoldPeriodCnt,
-		pBtMgnt->ExtConfig.bHoldForBtOperation));
-
-	RTPRINT(FBT, BT_TRACE,
-		("[BTC2H],   btInfo =%x   pHalData->bt_coexist.halCoex8723.c2hBtInfoOriginal =%x\n",
-		btInfo, pHalData->bt_coexist.halCoex8723.c2hBtInfoOriginal));
-	if (btInfo&BT_INFO_ACL) {
-		RTPRINT(FBT, BT_TRACE, ("[BTC2H], BTInfo: bConnect = true   btInfo =%x\n", btInfo));
-		bBtLinkExist = true;
-		if (((btInfo&(BT_INFO_FTP|BT_INFO_A2DP|BT_INFO_HID|BT_INFO_SCO_BUSY)) != 0) ||
-		    pHalData->bt_coexist.halCoex8723.btRetryCnt > 0) {
-			pBtdm8723->btStatus = BT_2ANT_BT_STATUS_NON_IDLE;
-		} else {
-			pBtdm8723->btStatus = BT_2ANT_BT_STATUS_CONNECTED_IDLE;
-		}
-
-		if (btInfo&BT_INFO_SCO || btInfo&BT_INFO_SCO_BUSY) {
-			if (btInfo&BT_INFO_FTP || btInfo&BT_INFO_A2DP || btInfo&BT_INFO_HID) {
-				switch (btInfo&0xe0) {
-				case BT_INFO_HID:
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO + HID\n"));
-					algorithm = BT_2ANT_COEX_ALGO_HID;
-					break;
-				case BT_INFO_A2DP:
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], Error!!! SCO + A2DP\n"));
-					break;
-				case BT_INFO_FTP:
-					if (bBtHsModeExist) {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO + PAN(HS)\n"));
-						algorithm = BT_2ANT_COEX_ALGO_SCO;
-					} else {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO + PAN(EDR)\n"));
-						algorithm = BT_2ANT_COEX_ALGO_PANEDR_HID;
-					}
-					break;
-				case (BT_INFO_HID | BT_INFO_A2DP):
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + A2DP\n"));
-					algorithm = BT_2ANT_COEX_ALGO_HID_A2DP;
-					break;
-				case (BT_INFO_HID | BT_INFO_FTP):
-					if (bBtHsModeExist) {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + PAN(HS)\n"));
-						algorithm = BT_2ANT_COEX_ALGO_HID_A2DP;
-					} else {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + PAN(EDR)\n"));
-						algorithm = BT_2ANT_COEX_ALGO_PANEDR_HID;
-					}
-					break;
-				case (BT_INFO_A2DP | BT_INFO_FTP):
-					if (bBtHsModeExist) {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], A2DP + PAN(HS)\n"));
-						algorithm = BT_2ANT_COEX_ALGO_A2DP;
-					} else {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], A2DP + PAN(EDR)\n"));
-						algorithm = BT_2ANT_COEX_ALGO_PANEDR_A2DP;
-					}
-					break;
-				case (BT_INFO_HID | BT_INFO_A2DP | BT_INFO_FTP):
-					if (bBtHsModeExist) {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + A2DP + PAN(HS)\n"));
-						algorithm = BT_2ANT_COEX_ALGO_HID_A2DP;
-					} else {
-						RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + A2DP + PAN(EDR)\n"));
-						algorithm = BT_2ANT_COEX_ALGO_HID_A2DP_PANEDR;
-					}
-					break;
-				}
-			} else {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], SCO only\n"));
-				algorithm = BT_2ANT_COEX_ALGO_SCO;
-			}
-		} else {
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex], non SCO\n"));
-			switch (btInfo&0xe0) {
-			case BT_INFO_HID:
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID\n"));
-				algorithm = BT_2ANT_COEX_ALGO_HID;
-				break;
-			case BT_INFO_A2DP:
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex],  A2DP\n"));
-				algorithm = BT_2ANT_COEX_ALGO_A2DP;
-				break;
-			case BT_INFO_FTP:
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], PAN(EDR)\n"));
-				algorithm = BT_2ANT_COEX_ALGO_PANEDR_HID;
-				break;
-			case (BT_INFO_HID | BT_INFO_A2DP):
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + A2DP\n"));
-				algorithm = BT_2ANT_COEX_ALGO_HID_A2DP;
-				break;
-			case (BT_INFO_HID|BT_INFO_FTP):
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + PAN(HS)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_HID_A2DP;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + PAN(EDR)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_PANEDR_HID;
-				}
-				break;
-			case (BT_INFO_A2DP|BT_INFO_FTP):
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], A2DP + PAN(HS)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_A2DP;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], A2DP + PAN(EDR)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_PANEDR_A2DP;
-				}
-				break;
-			case (BT_INFO_HID|BT_INFO_A2DP|BT_INFO_FTP):
-				if (bBtHsModeExist) {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + A2DP + PAN(HS)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_HID_A2DP;
-				} else {
-					RTPRINT(FBT, BT_TRACE, ("[BTCoex], HID + A2DP + PAN(EDR)\n"));
-					algorithm = BT_2ANT_COEX_ALGO_HID_A2DP_PANEDR;
-				}
-				break;
-			}
-
-		}
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("[BTC2H], BTInfo: bConnect = false\n"));
-		pBtdm8723->btStatus = BT_2ANT_BT_STATUS_IDLE;
-	}
-
-	pBtdm8723->curAlgorithm = algorithm;
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], Algorithm = %d \n", pBtdm8723->curAlgorithm));
-
-/* From */
-	BTDM_CheckWiFiState(padapter);
-	if (pBtMgnt->ExtConfig.bManualControl) {
-		RTPRINT(FBT, BT_TRACE, ("Action Manual control, won't execute bt coexist mechanism!!\n"));
-		return;
-	}
-}
-
-void BTDM_2AntBtCoexist8723A(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct bt_dgb *pBtDbg = &pBTInfo->BtDbg;
-	u8 btInfoOriginal = 0;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct btdm_8723a_2ant *pBtdm8723 = &pHalData->bt_coexist.halCoex8723.btdm2Ant;
-
-	if (BTDM_BtProfileSupport(padapter)) {
-		if (pBtMgnt->ExtConfig.bHoldForBtOperation) {
-			RTPRINT(FBT, BT_TRACE, ("Action for BT Operation adjust!!\n"));
-			return;
-		}
-		if (pBtMgnt->ExtConfig.bHoldPeriodCnt) {
-			RTPRINT(FBT, BT_TRACE, ("Hold BT inquiry/page scan setting (cnt = %d)!!\n",
-				pBtMgnt->ExtConfig.bHoldPeriodCnt));
-			if (pBtMgnt->ExtConfig.bHoldPeriodCnt >= 11) {
-				pBtMgnt->ExtConfig.bHoldPeriodCnt = 0;
-				/*  next time the coexist parameters should be reset again. */
-			} else {
-				pBtMgnt->ExtConfig.bHoldPeriodCnt++;
-			}
-			return;
-		}
-
-		if (pBtDbg->dbgCtrl)
-			RTPRINT(FBT, BT_TRACE, ("[Dbg control], "));
-
-		pBtdm8723->curAlgorithm = btdm_ActionAlgorithm(padapter);
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], Algorithm = %d \n", pBtdm8723->curAlgorithm));
-
-		if (btdm_Is2Ant8723ACommonAction(padapter)) {
-			RTPRINT(FBT, BT_TRACE, ("Action 2-Ant common.\n"));
-			pBtdm8723->bResetTdmaAdjust = true;
-		} else {
-			if (pBtdm8723->curAlgorithm != pBtdm8723->preAlgorithm) {
-				RTPRINT(FBT, BT_TRACE, ("[BTCoex], preAlgorithm =%d, curAlgorithm =%d\n",
-				pBtdm8723->preAlgorithm, pBtdm8723->curAlgorithm));
-				pBtdm8723->bResetTdmaAdjust = true;
-			}
-			switch (pBtdm8723->curAlgorithm) {
-			case BT_2ANT_COEX_ALGO_SCO:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = SCO.\n"));
-				btdm_2Ant8723ASCOAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_HID:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = HID.\n"));
-				btdm_2Ant8723AHIDAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_A2DP:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = A2DP.\n"));
-				btdm_2Ant8723AA2DPAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_PANEDR:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = PAN(EDR).\n"));
-				btdm_2Ant8723APANEDRAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_PANHS:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = HS mode.\n"));
-				btdm_2Ant8723APANHSAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_PANEDR_A2DP:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = PAN+A2DP.\n"));
-				btdm_2Ant8723APANEDRA2DPAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_PANEDR_HID:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = PAN(EDR)+HID.\n"));
-				btdm_2Ant8723APANEDRHIDAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_HID_A2DP_PANEDR:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = HID+A2DP+PAN.\n"));
-				btdm_2Ant8723AHIDA2DPPANEDRAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_HID_A2DP:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = HID+A2DP.\n"));
-				btdm_2Ant8723AHIDA2DPAction(padapter);
-				break;
-			default:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = 0.\n"));
-				btdm_2Ant8723AA2DPAction(padapter);
-				break;
-			}
-			pBtdm8723->preAlgorithm = pBtdm8723->curAlgorithm;
-		}
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex] Get bt info by fw!!\n"));
-		/* msg shows c2h rsp for bt_info is received or not. */
-		if (pHalData->bt_coexist.halCoex8723.bC2hBtInfoReqSent)
-			RTPRINT(FBT, BT_TRACE, ("[BTCoex] c2h for btInfo not rcvd yet!!\n"));
-
-		btInfoOriginal = pHalData->bt_coexist.halCoex8723.c2hBtInfoOriginal;
-
-		if (pBtMgnt->ExtConfig.bHoldForBtOperation) {
-			RTPRINT(FBT, BT_TRACE, ("Action for BT Operation adjust!!\n"));
-			return;
-		}
-		if (pBtMgnt->ExtConfig.bHoldPeriodCnt) {
-			RTPRINT(FBT, BT_TRACE,
-				("Hold BT inquiry/page scan setting (cnt = %d)!!\n",
-				pBtMgnt->ExtConfig.bHoldPeriodCnt));
-			if (pBtMgnt->ExtConfig.bHoldPeriodCnt >= 11) {
-				pBtMgnt->ExtConfig.bHoldPeriodCnt = 0;
-				/*  next time the coexist parameters should be reset again. */
-			} else {
-				 pBtMgnt->ExtConfig.bHoldPeriodCnt++;
-			}
-			return;
-		}
-
-		if (pBtDbg->dbgCtrl)
-			RTPRINT(FBT, BT_TRACE, ("[Dbg control], "));
-		if (btdm_Is2Ant8723ACommonAction(padapter)) {
-			RTPRINT(FBT, BT_TRACE, ("Action 2-Ant common.\n"));
-			pBtdm8723->bResetTdmaAdjust = true;
-		} else {
-			if (pBtdm8723->curAlgorithm != pBtdm8723->preAlgorithm) {
-				RTPRINT(FBT, BT_TRACE,
-					("[BTCoex], preAlgorithm =%d, curAlgorithm =%d\n",
-					pBtdm8723->preAlgorithm,
-					pBtdm8723->curAlgorithm));
-				pBtdm8723->bResetTdmaAdjust = true;
-			}
-			switch (pBtdm8723->curAlgorithm) {
-			case BT_2ANT_COEX_ALGO_SCO:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = SCO.\n"));
-				btdm_2Ant8723ASCOAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_HID:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = HID.\n"));
-				btdm_2Ant8723AHIDAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_A2DP:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = A2DP.\n"));
-				btdm_2Ant8723AA2dp(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_PANEDR:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = PAN(EDR).\n"));
-				btdm_2Ant8723APANEDRAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_PANHS:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = HS mode.\n"));
-				btdm_2Ant8723APANHSAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_PANEDR_A2DP:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = PAN+A2DP.\n"));
-				btdm_2Ant8723APANEDRA2DPAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_PANEDR_HID:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = PAN(EDR)+HID.\n"));
-				btdm_2Ant8723APANEDRHIDAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_HID_A2DP_PANEDR:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = HID+A2DP+PAN.\n"));
-				btdm_2Ant8723AHIDA2DPPANEDRAction(padapter);
-				break;
-			case BT_2ANT_COEX_ALGO_HID_A2DP:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = HID+A2DP.\n"));
-				btdm_2Ant8723AHIDA2DPAction(padapter);
-				break;
-			default:
-				RTPRINT(FBT, BT_TRACE, ("Action 2-Ant, algorithm = 0.\n"));
-				btdm_2Ant8723AA2DPAction(padapter);
-				break;
-			}
-			pBtdm8723->preAlgorithm = pBtdm8723->curAlgorithm;
-		}
-	}
-}
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtc87232Ant.c ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtc8723.c ===== */
-
-static u8 btCoexDbgBuf[BT_TMP_BUF_SIZE];
-
-static const char *const BtProfileString[] = {
-	"NONE",
-	"A2DP",
-	"PAN",
-	"HID",
-	"SCO",
-};
-
-static const char *const BtSpecString[] = {
-	"1.0b",
-	"1.1",
-	"1.2",
-	"2.0+EDR",
-	"2.1+EDR",
-	"3.0+HS",
-	"4.0",
-};
-
-static const char *const BtLinkRoleString[] = {
-	"Master",
-	"Slave",
-};
-
-static u8 btdm_BtWifiAntNum(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct bt_coexist_8723a *pBtCoex = &pHalData->bt_coexist.halCoex8723;
-
-	if (Ant_x2 == pHalData->bt_coexist.BT_Ant_Num) {
-		if (Ant_x2 == pBtCoex->TotalAntNum)
-			return Ant_x2;
-		else
-			return Ant_x1;
-	} else {
-		return Ant_x1;
-	}
-	return Ant_x2;
-}
-
-static void btdm_BtHwCountersMonitor(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u32	regHPTxRx, regLPTxRx, u4Tmp;
-	u32	regHPTx = 0, regHPRx = 0, regLPTx = 0, regLPRx = 0;
-
-	regHPTxRx = REG_HIGH_PRIORITY_TXRX;
-	regLPTxRx = REG_LOW_PRIORITY_TXRX;
-
-	u4Tmp = rtl8723au_read32(padapter, regHPTxRx);
-	regHPTx = u4Tmp & bMaskLWord;
-	regHPRx = (u4Tmp & bMaskHWord)>>16;
-
-	u4Tmp = rtl8723au_read32(padapter, regLPTxRx);
-	regLPTx = u4Tmp & bMaskLWord;
-	regLPRx = (u4Tmp & bMaskHWord)>>16;
-
-	pHalData->bt_coexist.halCoex8723.highPriorityTx = regHPTx;
-	pHalData->bt_coexist.halCoex8723.highPriorityRx = regHPRx;
-	pHalData->bt_coexist.halCoex8723.lowPriorityTx = regLPTx;
-	pHalData->bt_coexist.halCoex8723.lowPriorityRx = regLPRx;
-
-	RTPRINT(FBT, BT_TRACE, ("High Priority Tx/Rx = %d / %d\n", regHPTx, regHPRx));
-	RTPRINT(FBT, BT_TRACE, ("Low Priority Tx/Rx = %d / %d\n", regLPTx, regLPRx));
-
-	/*  reset counter */
-	rtl8723au_write8(padapter, 0x76e, 0xc);
-}
-
-/*  This function check if 8723 bt is disabled */
-static void btdm_BtEnableDisableCheck8723A(struct rtw_adapter *padapter)
-{
-	u8 btAlife = true;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-#ifdef CHECK_BT_EXIST_FROM_REG
-	u8 val8;
-
-	/*  ox68[28]= 1 => BT enable; otherwise disable */
-	val8 = rtl8723au_read8(padapter, 0x6B);
-	if (!(val8 & BIT(4)))
-		btAlife = false;
-
-	if (btAlife)
-		pHalData->bt_coexist.bCurBtDisabled = false;
-	else
-		pHalData->bt_coexist.bCurBtDisabled = true;
-#else
-	if (pHalData->bt_coexist.halCoex8723.highPriorityTx == 0 &&
-	    pHalData->bt_coexist.halCoex8723.highPriorityRx == 0 &&
-	    pHalData->bt_coexist.halCoex8723.lowPriorityTx == 0 &&
-	    pHalData->bt_coexist.halCoex8723.lowPriorityRx == 0)
-		btAlife = false;
-	if (pHalData->bt_coexist.halCoex8723.highPriorityTx == 0xeaea &&
-	    pHalData->bt_coexist.halCoex8723.highPriorityRx == 0xeaea &&
-	    pHalData->bt_coexist.halCoex8723.lowPriorityTx == 0xeaea &&
-	    pHalData->bt_coexist.halCoex8723.lowPriorityRx == 0xeaea)
-		btAlife = false;
-	if (pHalData->bt_coexist.halCoex8723.highPriorityTx == 0xffff &&
-	    pHalData->bt_coexist.halCoex8723.highPriorityRx == 0xffff &&
-	    pHalData->bt_coexist.halCoex8723.lowPriorityTx == 0xffff &&
-	    pHalData->bt_coexist.halCoex8723.lowPriorityRx == 0xffff)
-		btAlife = false;
-	if (btAlife) {
-		pHalData->bt_coexist.btActiveZeroCnt = 0;
-		pHalData->bt_coexist.bCurBtDisabled = false;
-		RTPRINT(FBT, BT_TRACE, ("8723A BT is enabled !!\n"));
-	} else {
-		pHalData->bt_coexist.btActiveZeroCnt++;
-		RTPRINT(FBT, BT_TRACE, ("8723A bt all counters = 0, %d times!!\n",
-				pHalData->bt_coexist.btActiveZeroCnt));
-		if (pHalData->bt_coexist.btActiveZeroCnt >= 2) {
-			pHalData->bt_coexist.bCurBtDisabled = true;
-			RTPRINT(FBT, BT_TRACE, ("8723A BT is disabled !!\n"));
-		}
-	}
-#endif
-
-	if (!pHalData->bt_coexist.bCurBtDisabled) {
-		if (BTDM_IsWifiConnectionExist(padapter))
-			BTDM_SetFwChnlInfo(padapter, RT_MEDIA_CONNECT);
-		else
-			BTDM_SetFwChnlInfo(padapter, RT_MEDIA_DISCONNECT);
-	}
-
-	if (pHalData->bt_coexist.bPreBtDisabled !=
-	    pHalData->bt_coexist.bCurBtDisabled) {
-		RTPRINT(FBT, BT_TRACE, ("8723A BT is from %s to %s!!\n",
-			(pHalData->bt_coexist.bPreBtDisabled ? "disabled":"enabled"),
-			(pHalData->bt_coexist.bCurBtDisabled ? "disabled":"enabled")));
-		pHalData->bt_coexist.bPreBtDisabled = pHalData->bt_coexist.bCurBtDisabled;
-	}
-}
-
-static void btdm_BTCoexist8723AHandler(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-
-	pHalData = GET_HAL_DATA(padapter);
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x2) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], 2 Ant mechanism\n"));
-		BTDM_2AntBtCoexist8723A(padapter);
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], 1 Ant mechanism\n"));
-		BTDM_1AntBtCoexist8723A(padapter);
-	}
-
-	if (!BTDM_IsSameCoexistState(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], Coexist State[bitMap] change from 0x%"i64fmt"x to 0x%"i64fmt"x\n",
-			pHalData->bt_coexist.PreviousState,
-			pHalData->bt_coexist.CurrentState));
-		pHalData->bt_coexist.PreviousState = pHalData->bt_coexist.CurrentState;
-
-		RTPRINT(FBT, BT_TRACE, ("["));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_BT30)
-			RTPRINT(FBT, BT_TRACE, ("BT 3.0, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_WIFI_HT20)
-			RTPRINT(FBT, BT_TRACE, ("HT20, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_WIFI_HT40)
-			RTPRINT(FBT, BT_TRACE, ("HT40, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_WIFI_LEGACY)
-			RTPRINT(FBT, BT_TRACE, ("Legacy, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_WIFI_RSSI_LOW)
-			RTPRINT(FBT, BT_TRACE, ("Rssi_Low, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_WIFI_RSSI_MEDIUM)
-			RTPRINT(FBT, BT_TRACE, ("Rssi_Mid, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_WIFI_RSSI_HIGH)
-			RTPRINT(FBT, BT_TRACE, ("Rssi_High, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_WIFI_IDLE)
-			RTPRINT(FBT, BT_TRACE, ("Wifi_Idle, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_WIFI_UPLINK)
-			RTPRINT(FBT, BT_TRACE, ("Wifi_Uplink, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_WIFI_DOWNLINK)
-			RTPRINT(FBT, BT_TRACE, ("Wifi_Downlink, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_BT_IDLE)
-			RTPRINT(FBT, BT_TRACE, ("BT_idle, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_PROFILE_HID)
-			RTPRINT(FBT, BT_TRACE, ("PRO_HID, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_PROFILE_A2DP)
-			RTPRINT(FBT, BT_TRACE, ("PRO_A2DP, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_PROFILE_PAN)
-			RTPRINT(FBT, BT_TRACE, ("PRO_PAN, "));
-		if (pHalData->bt_coexist.CurrentState & BT_COEX_STATE_PROFILE_SCO)
-			RTPRINT(FBT, BT_TRACE, ("PRO_SCO, "));
-		RTPRINT(FBT, BT_TRACE, ("]\n"));
-	}
-}
-
-/*  extern function start with BTDM_ */
-u32 BTDM_BtTxRxCounterH(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u32	counters;
-
-	counters = pHalData->bt_coexist.halCoex8723.highPriorityTx+
-		pHalData->bt_coexist.halCoex8723.highPriorityRx;
-	return counters;
-}
-
-u32 BTDM_BtTxRxCounterL(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u32	counters;
-
-	counters = pHalData->bt_coexist.halCoex8723.lowPriorityTx+
-		pHalData->bt_coexist.halCoex8723.lowPriorityRx;
-	return counters;
-}
-
-void BTDM_SetFwChnlInfo(struct rtw_adapter *padapter, enum rt_media_status mstatus)
-{
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	u8 H2C_Parameter[3] = {0};
-	u8 chnl;
-
-	/*  opMode */
-	if (RT_MEDIA_CONNECT == mstatus)
-		H2C_Parameter[0] = 0x1;	/*  0: disconnected, 1:connected */
-
-	if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) {
-		/*  channel */
-		chnl = pmlmeext->cur_channel;
-		if (BTDM_IsHT40(padapter)) {
-			if (pmlmeext->cur_ch_offset == HAL_PRIME_CHNL_OFFSET_UPPER)
-				chnl -= 2;
-			else if (pmlmeext->cur_ch_offset == HAL_PRIME_CHNL_OFFSET_LOWER)
-				chnl += 2;
-		}
-		H2C_Parameter[1] = chnl;
-	} else {	/*  check if HS link is exists */
-		/*  channel */
-		if (BT_Operation(padapter))
-			H2C_Parameter[1] = pBtMgnt->BTChannel;
-		else
-			H2C_Parameter[1] = pmlmeext->cur_channel;
-	}
-
-	if (BTDM_IsHT40(padapter))
-		H2C_Parameter[2] = 0x30;
-	else
-		H2C_Parameter[2] = 0x20;
-
-	FillH2CCmd(padapter, 0x19, 3, H2C_Parameter);
-}
-
-u8 BTDM_IsWifiConnectionExist(struct rtw_adapter *padapter)
-{
-	u8 bRet = false;
-
-	if (BTHCI_HsConnectionEstablished(padapter))
-		bRet = true;
-
-	if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == true)
-		bRet = true;
-
-	return bRet;
-}
-
-void BTDM_SetFw3a(
-	struct rtw_adapter *padapter,
-	u8 byte1,
-	u8 byte2,
-	u8 byte3,
-	u8 byte4,
-	u8 byte5
-	)
-{
-	u8 H2C_Parameter[5] = {0};
-
-	if (rtl8723a_BT_using_antenna_1(padapter)) {
-		if ((!check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE)) &&
-		    (get_fwstate(&padapter->mlmepriv) != WIFI_NULL_STATE)) {
-			/*  for softap mode */
-			struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-			struct bt_coexist_8723a *pBtCoex = &pHalData->bt_coexist.halCoex8723;
-			u8 BtState = pBtCoex->c2hBtInfo;
-
-			if ((BtState != BT_INFO_STATE_NO_CONNECTION) &&
-			    (BtState != BT_INFO_STATE_CONNECT_IDLE)) {
-				if (byte1 & BIT(4)) {
-					byte1 &= ~BIT(4);
-					byte1 |= BIT(5);
-				}
-
-				byte5 |= BIT(5);
-				if (byte5 & BIT(6))
-					byte5 &= ~BIT(6);
-			}
-		}
-	}
-
-	H2C_Parameter[0] = byte1;
-	H2C_Parameter[1] = byte2;
-	H2C_Parameter[2] = byte3;
-	H2C_Parameter[3] = byte4;
-	H2C_Parameter[4] = byte5;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], FW write 0x3a(5bytes) = 0x%02x%08x\n",
-		H2C_Parameter[0],
-		H2C_Parameter[1]<<24|H2C_Parameter[2]<<16|H2C_Parameter[3]<<8|H2C_Parameter[4]));
-
-	FillH2CCmd(padapter, 0x3a, 5, H2C_Parameter);
-}
-
-void BTDM_QueryBtInformation(struct rtw_adapter *padapter)
-{
-	u8 H2C_Parameter[1] = {0};
-	struct hal_data_8723a *pHalData;
-	struct bt_coexist_8723a *pBtCoex;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBtCoex = &pHalData->bt_coexist.halCoex8723;
-
-	if (!rtl8723a_BT_enabled(padapter)) {
-		pBtCoex->c2hBtInfo = BT_INFO_STATE_DISABLED;
-		pBtCoex->bC2hBtInfoReqSent = false;
-		return;
-	}
-
-	if (pBtCoex->c2hBtInfo == BT_INFO_STATE_DISABLED)
-		pBtCoex->c2hBtInfo = BT_INFO_STATE_NO_CONNECTION;
-
-	if (pBtCoex->bC2hBtInfoReqSent == true)
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], didn't recv previous BtInfo report!\n"));
-	else
-		pBtCoex->bC2hBtInfoReqSent = true;
-
-	H2C_Parameter[0] |= BIT(0);	/*  trigger */
-
-/*RTPRINT(FBT, BT_TRACE, ("[BTCoex], Query Bt information, write 0x38 = 0x%x\n", */
-/*H2C_Parameter[0])); */
-
-	FillH2CCmd(padapter, 0x38, 1, H2C_Parameter);
-}
-
-void BTDM_SetSwRfRxLpfCorner(struct rtw_adapter *padapter, u8 type)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (BT_RF_RX_LPF_CORNER_SHRINK == type) {
-		/* Shrink RF Rx LPF corner */
-		RTPRINT(FBT, BT_TRACE, ("Shrink RF Rx LPF corner!!\n"));
-		PHY_SetRFReg(padapter, PathA, 0x1e, bRFRegOffsetMask, 0xf0ff7);
-		pHalData->bt_coexist.bSWCoexistAllOff = false;
-	} else if (BT_RF_RX_LPF_CORNER_RESUME == type) {
-		/* Resume RF Rx LPF corner */
-		RTPRINT(FBT, BT_TRACE, ("Resume RF Rx LPF corner!!\n"));
-		PHY_SetRFReg(padapter, PathA, 0x1e, bRFRegOffsetMask, pHalData->bt_coexist.BtRfRegOrigin1E);
-	}
-}
-
-void
-BTDM_SetSwPenaltyTxRateAdaptive(
-	struct rtw_adapter *padapter,
-	u8 raType
-	)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 tmpU1;
-
-	tmpU1 = rtl8723au_read8(padapter, 0x4fd);
-	tmpU1 |= BIT(0);
-	if (BT_TX_RATE_ADAPTIVE_LOW_PENALTY == raType) {
-		tmpU1 &= ~BIT(2);
-		pHalData->bt_coexist.bSWCoexistAllOff = false;
-	} else if (BT_TX_RATE_ADAPTIVE_NORMAL == raType) {
-		tmpU1 |= BIT(2);
-	}
-
-	rtl8723au_write8(padapter, 0x4fd, tmpU1);
-}
-
-void BTDM_SetFwDecBtPwr(struct rtw_adapter *padapter, u8 bDecBtPwr)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 H2C_Parameter[1] = {0};
-
-	H2C_Parameter[0] = 0;
-
-	if (bDecBtPwr) {
-		H2C_Parameter[0] |= BIT(1);
-		pHalData->bt_coexist.bFWCoexistAllOff = false;
-	}
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], decrease Bt Power : %s, write 0x21 = 0x%x\n",
-		(bDecBtPwr ? "Yes!!" : "No!!"), H2C_Parameter[0]));
-
-	FillH2CCmd(padapter, 0x21, 1, H2C_Parameter);
-}
-
-u8 BTDM_BtProfileSupport(struct rtw_adapter *padapter)
-{
-	u8 bRet = false;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (pBtMgnt->bSupportProfile &&
-	    !pHalData->bt_coexist.halCoex8723.bForceFwBtInfo)
-		bRet = true;
-
-	return bRet;
-}
-
-static void BTDM_AdjustForBtOperation8723A(struct rtw_adapter *padapter)
-{
-	/* BTDM_2AntAdjustForBtOperation8723(padapter); */
-}
-
-static void BTDM_FwC2hBtRssi8723A(struct rtw_adapter *padapter, u8 *tmpBuf)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 percent, u1tmp;
-
-	u1tmp = tmpBuf[0];
-	percent = u1tmp*2+10;
-
-	pHalData->bt_coexist.halCoex8723.btRssi = percent;
-/*RTPRINT(FBT, BT_TRACE, ("[BTC2H], BT RSSI =%d\n", percent)); */
-}
-
-void
-rtl8723a_fw_c2h_BT_info(struct rtw_adapter *padapter, u8 *tmpBuf, u8 length)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_coexist_8723a *pBtCoex;
-	u8 i;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtCoex = &pHalData->bt_coexist.halCoex8723;
-
-	pBtCoex->bC2hBtInfoReqSent = false;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTC2H], BT info[%d]=[", length));
-
-	pBtCoex->btRetryCnt = 0;
-	for (i = 0; i < length; i++) {
-		switch (i) {
-		case 0:
-			pBtCoex->c2hBtInfoOriginal = tmpBuf[i];
-			break;
-		case 1:
-			pBtCoex->btRetryCnt = tmpBuf[i];
-			break;
-		case 2:
-			BTDM_FwC2hBtRssi8723A(padapter, &tmpBuf[i]);
-			break;
-		case 3:
-			pBtCoex->btInfoExt = tmpBuf[i]&BIT(0);
-			break;
-		}
-
-		if (i == length-1)
-			RTPRINT(FBT, BT_TRACE, ("0x%02x]\n", tmpBuf[i]));
-		else
-			RTPRINT(FBT, BT_TRACE, ("0x%02x, ", tmpBuf[i]));
-	}
-	RTPRINT(FBT, BT_TRACE, ("[BTC2H], BT RSSI =%d\n", pBtCoex->btRssi));
-	if (pBtCoex->btInfoExt)
-		RTPRINT(FBT, BT_TRACE, ("[BTC2H], pBtCoex->btInfoExt =%x\n", pBtCoex->btInfoExt));
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-		BTDM_1AntFwC2hBtInfo8723A(padapter);
-	else
-		BTDM_2AntFwC2hBtInfo8723A(padapter);
-
-	if (pBtMgnt->ExtConfig.bManualControl) {
-		RTPRINT(FBT, BT_TRACE, ("%s: Action Manual control!!\n", __func__));
-		return;
-	}
-
-	btdm_BTCoexist8723AHandler(padapter);
-}
-
-static void BTDM_Display8723ABtCoexInfo(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct bt_coexist_8723a *pBtCoex = &pHalData->bt_coexist.halCoex8723;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	u8 u1Tmp, u1Tmp1, u1Tmp2, i, btInfoExt, psTdmaCase = 0;
-	u32 u4Tmp[4];
-	u8 antNum = Ant_x2;
-
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n ============[BT Coexist info]============");
-	DCMD_Printf(btCoexDbgBuf);
-
-	if (!rtl8723a_BT_coexist(padapter)) {
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n BT not exists !!!");
-		DCMD_Printf(btCoexDbgBuf);
-		return;
-	}
-
-	antNum = btdm_BtWifiAntNum(padapter);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %d/%d ", "Ant mechanism PG/Now run :", \
-		((pHalData->bt_coexist.BT_Ant_Num == Ant_x2) ? 2 : 1), ((antNum == Ant_x2) ? 2 : 1));
-	DCMD_Printf(btCoexDbgBuf);
-
-	if (pBtMgnt->ExtConfig.bManualControl) {
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s", "[Action Manual control]!!");
-		DCMD_Printf(btCoexDbgBuf);
-	} else {
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %s / %d", "BT stack/ hci ext ver", \
-			((pBtMgnt->bSupportProfile) ? "Yes" : "No"), pBtMgnt->ExtConfig.HCIExtensionVer);
-		DCMD_Printf(btCoexDbgBuf);
-	}
-
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\n %-35s = / %d", "Dot11 channel / BT channel", \
-		pBtMgnt->BTChannel);
-		DCMD_Printf(btCoexDbgBuf);
-
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\n %-35s = %d / %d / %d", "Wifi/BT/HS rssi", \
-		BTDM_GetRxSS(padapter),
-		pHalData->bt_coexist.halCoex8723.btRssi,
-		pHalData->dmpriv.EntryMinUndecoratedSmoothedPWDB);
-			DCMD_Printf(btCoexDbgBuf);
-
-	if (!pBtMgnt->ExtConfig.bManualControl) {
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\n %-35s = %s / %s ", "WIfi status",
-			((BTDM_Legacy(padapter)) ? "Legacy" : (((BTDM_IsHT40(padapter)) ? "HT40" : "HT20"))),
-			((!BTDM_IsWifiBusy(padapter)) ? "idle" : ((BTDM_IsWifiUplink(padapter)) ? "uplink" : "downlink")));
-		DCMD_Printf(btCoexDbgBuf);
-
-		if (pBtMgnt->bSupportProfile) {
-			snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %d / %d / %d / %d", "SCO/HID/PAN/A2DP",
-				((BTHCI_CheckProfileExist(padapter, BT_PROFILE_SCO)) ? 1 : 0),
-				((BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID)) ? 1 : 0),
-				((BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN)) ? 1 : 0),
-				((BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) ? 1 : 0));
-		DCMD_Printf(btCoexDbgBuf);
-
-			for (i = 0; i < pBtMgnt->ExtConfig.NumberOfHandle; i++) {
-				if (pBtMgnt->ExtConfig.HCIExtensionVer >= 1) {
-					snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %s/ %s/ %s", "Bt link type/spec/role",
-						BtProfileString[pBtMgnt->ExtConfig.linkInfo[i].BTProfile],
-						BtSpecString[pBtMgnt->ExtConfig.linkInfo[i].BTCoreSpec],
-						BtLinkRoleString[pBtMgnt->ExtConfig.linkInfo[i].linkRole]);
-					DCMD_Printf(btCoexDbgBuf);
-
-					btInfoExt = pHalData->bt_coexist.halCoex8723.btInfoExt;
-					snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %s", "A2DP rate", \
-						 (btInfoExt & BIT(0)) ?
-						 "Basic rate" : "EDR rate");
-					DCMD_Printf(btCoexDbgBuf);
-				} else {
-					snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %s/ %s", "Bt link type/spec", \
-						BtProfileString[pBtMgnt->ExtConfig.linkInfo[i].BTProfile],
-						BtSpecString[pBtMgnt->ExtConfig.linkInfo[i].BTCoreSpec]);
-					DCMD_Printf(btCoexDbgBuf);
-				}
-			}
-		}
-	}
-
-	/*  Sw mechanism */
-	if (!pBtMgnt->ExtConfig.bManualControl) {
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s", "============[Sw BT Coex mechanism]============");
-		DCMD_Printf(btCoexDbgBuf);
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %d ", "AGC Table", \
-			pBtCoex->btdm2Ant.bCurAgcTableEn);
-		DCMD_Printf(btCoexDbgBuf);
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %d ", "ADC Backoff", \
-			pBtCoex->btdm2Ant.bCurAdcBackOff);
-		DCMD_Printf(btCoexDbgBuf);
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %d ", "Low penalty RA", \
-			pBtCoex->btdm2Ant.bCurLowPenaltyRa);
-		DCMD_Printf(btCoexDbgBuf);
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %d ", "RF Rx LPF Shrink", \
-			pBtCoex->btdm2Ant.bCurRfRxLpfShrink);
-		DCMD_Printf(btCoexDbgBuf);
-	}
-	u4Tmp[0] = PHY_QueryRFReg(padapter, PathA, 0x1e, 0xff0);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x/ 0x%x", "RF-A, 0x1e[11:4]/original val", \
-		u4Tmp[0], pHalData->bt_coexist.BtRfRegOrigin1E);
-	DCMD_Printf(btCoexDbgBuf);
-
-	/*  Fw mechanism */
-	if (!pBtMgnt->ExtConfig.bManualControl) {
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s", "============[Fw BT Coex mechanism]============");
-		DCMD_Printf(btCoexDbgBuf);
-	}
-	if (!pBtMgnt->ExtConfig.bManualControl) {
-		if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-			psTdmaCase = pHalData->bt_coexist.halCoex8723.btdm1Ant.curPsTdma;
-		else
-			psTdmaCase = pHalData->bt_coexist.halCoex8723.btdm2Ant.curPsTdma;
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %*ph case-%d",
-			 "PS TDMA(0x3a)", 5, pHalData->bt_coexist.fw3aVal, psTdmaCase);
-		DCMD_Printf(btCoexDbgBuf);
-
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %d ", "Decrease Bt Power", \
-			pBtCoex->btdm2Ant.bCurDecBtPwr);
-		DCMD_Printf(btCoexDbgBuf);
-	}
-	u1Tmp = rtl8723au_read8(padapter, 0x778);
-	u1Tmp1 = rtl8723au_read8(padapter, 0x783);
-	u1Tmp2 = rtl8723au_read8(padapter, 0x796);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x", "0x778/ 0x783/ 0x796", \
-		u1Tmp, u1Tmp1, u1Tmp2);
-	DCMD_Printf(btCoexDbgBuf);
-
-	if (!pBtMgnt->ExtConfig.bManualControl) {
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x / 0x%x", "Sw DacSwing Ctrl/Val", \
-			pBtCoex->btdm2Ant.bCurDacSwingOn, pBtCoex->btdm2Ant.curDacSwingLvl);
-		DCMD_Printf(btCoexDbgBuf);
-	}
-	u4Tmp[0] =  rtl8723au_read32(padapter, 0x880);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x", "0x880", \
-		u4Tmp[0]);
-	DCMD_Printf(btCoexDbgBuf);
-
-	/*  Hw mechanism */
-	if (!pBtMgnt->ExtConfig.bManualControl) {
-		snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s", "============[Hw BT Coex mechanism]============");
-		DCMD_Printf(btCoexDbgBuf);
-	}
-
-	u1Tmp = rtl8723au_read8(padapter, 0x40);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x", "0x40", \
-		u1Tmp);
-	DCMD_Printf(btCoexDbgBuf);
-
-	u4Tmp[0] = rtl8723au_read32(padapter, 0x550);
-	u1Tmp = rtl8723au_read8(padapter, 0x522);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x/0x%x", "0x550(bcn contrl)/0x522", \
-		u4Tmp[0], u1Tmp);
-	DCMD_Printf(btCoexDbgBuf);
-
-	u4Tmp[0] = rtl8723au_read32(padapter, 0x484);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x", "0x484(rate adaptive)", \
-		u4Tmp[0]);
-	DCMD_Printf(btCoexDbgBuf);
-
-	u4Tmp[0] = rtl8723au_read32(padapter, 0x50);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x", "0xc50(dig)", \
-		u4Tmp[0]);
-	DCMD_Printf(btCoexDbgBuf);
-
-	u4Tmp[0] = rtl8723au_read32(padapter, 0xda0);
-	u4Tmp[1] = rtl8723au_read32(padapter, 0xda4);
-	u4Tmp[2] = rtl8723au_read32(padapter, 0xda8);
-	u4Tmp[3] = rtl8723au_read32(padapter, 0xdac);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x", "0xda0/0xda4/0xda8/0xdac(FA cnt)", \
-		u4Tmp[0], u4Tmp[1], u4Tmp[2], u4Tmp[3]);
-	DCMD_Printf(btCoexDbgBuf);
-
-	u4Tmp[0] = rtl8723au_read32(padapter, 0x6c0);
-	u4Tmp[1] = rtl8723au_read32(padapter, 0x6c4);
-	u4Tmp[2] = rtl8723au_read32(padapter, 0x6c8);
-	u1Tmp = rtl8723au_read8(padapter, 0x6cc);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x", "0x6c0/0x6c4/0x6c8/0x6cc(coexTable)", \
-		u4Tmp[0], u4Tmp[1], u4Tmp[2], u1Tmp);
-	DCMD_Printf(btCoexDbgBuf);
-
-	/* u4Tmp = rtl8723au_read32(padapter, 0x770); */
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %d / %d", "0x770(Hi pri Rx[31:16]/Tx[15:0])", \
-		pHalData->bt_coexist.halCoex8723.highPriorityRx,
-		pHalData->bt_coexist.halCoex8723.highPriorityTx);
-	DCMD_Printf(btCoexDbgBuf);
-	/* u4Tmp = rtl8723au_read32(padapter, 0x774); */
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = %d / %d", "0x774(Lo pri Rx[31:16]/Tx[15:0])", \
-		pHalData->bt_coexist.halCoex8723.lowPriorityRx,
-		pHalData->bt_coexist.halCoex8723.lowPriorityTx);
-	DCMD_Printf(btCoexDbgBuf);
-
-	/*  Tx mgnt queue hang or not, 0x41b should = 0xf, ex: 0xd ==>hang */
-	u1Tmp = rtl8723au_read8(padapter, 0x41b);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x", "0x41b (hang chk == 0xf)", \
-		u1Tmp);
-	DCMD_Printf(btCoexDbgBuf);
-	snprintf(btCoexDbgBuf, BT_TMP_BUF_SIZE, "\r\n %-35s = 0x%x", "lastHMEBoxNum", \
-		pHalData->LastHMEBoxNum);
-	DCMD_Printf(btCoexDbgBuf);
-}
-
-static void
-BTDM_8723ASignalCompensation(struct rtw_adapter *padapter,
-			     u8 *rssi_wifi, u8 *rssi_bt)
-{
-	if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-		BTDM_1AntSignalCompensation(padapter, rssi_wifi, rssi_bt);
-}
-
-static void BTDM_8723AInit(struct rtw_adapter *padapter)
-{
-	if (btdm_BtWifiAntNum(padapter) == Ant_x2)
-		BTDM_2AntParaInit(padapter);
-	else
-		BTDM_1AntParaInit(padapter);
-}
-
-static void BTDM_HWCoexAllOff8723A(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x2)
-		BTDM_2AntHwCoexAllOff8723A(padapter);
-}
-
-static void BTDM_FWCoexAllOff8723A(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x2)
-		BTDM_2AntFwCoexAllOff8723A(padapter);
-}
-
-static void BTDM_SWCoexAllOff8723A(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x2)
-		BTDM_2AntSwCoexAllOff8723A(padapter);
-}
-
-static void
-BTDM_Set8723ABtCoexCurrAntNum(struct rtw_adapter *padapter, u8 antNum)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct bt_coexist_8723a *pBtCoex = &pHalData->bt_coexist.halCoex8723;
-
-	if (antNum == 1)
-		pBtCoex->TotalAntNum = Ant_x1;
-	else if (antNum == 2)
-		pBtCoex->TotalAntNum = Ant_x2;
-}
-
-void rtl8723a_BT_lps_leave(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-		BTDM_1AntLpsLeave(padapter);
-}
-
-static void BTDM_ForHalt8723A(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-		BTDM_1AntForHalt(padapter);
-}
-
-static void BTDM_WifiScanNotify8723A(struct rtw_adapter *padapter, u8 scanType)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-		BTDM_1AntWifiScanNotify(padapter, scanType);
-}
-
-static void
-BTDM_WifiAssociateNotify8723A(struct rtw_adapter *padapter, u8 action)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-		BTDM_1AntWifiAssociateNotify(padapter, action);
-}
-
-static void
-BTDM_MediaStatusNotify8723A(struct rtw_adapter *padapter,
-			    enum rt_media_status mstatus)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], MediaStatusNotify, %s\n",
-		mstatus?"connect":"disconnect"));
-
-	BTDM_SetFwChnlInfo(padapter, mstatus);
-
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-		BTDM_1AntMediaStatusNotify(padapter, mstatus);
-}
-
-static void BTDM_ForDhcp8723A(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-
-	if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-		BTDM_1AntForDhcp(padapter);
-}
-
-bool rtl8723a_BT_using_antenna_1(struct rtw_adapter *padapter)
-{
-	if (btdm_BtWifiAntNum(padapter) == Ant_x1)
-		return true;
-	else
-		return false;
-}
-
-static void BTDM_BTCoexist8723A(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_coexist_8723a *pBtCoex;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtCoex = &pHalData->bt_coexist.halCoex8723;
-
-	RTPRINT(FBT, BT_TRACE, ("[BTCoex], beacon RSSI = 0x%x(%d)\n",
-		pHalData->dmpriv.EntryMinUndecoratedSmoothedPWDB,
-		pHalData->dmpriv.EntryMinUndecoratedSmoothedPWDB));
-
-	btdm_BtHwCountersMonitor(padapter);
-	btdm_BtEnableDisableCheck8723A(padapter);
-
-	if (pBtMgnt->ExtConfig.bManualControl) {
-		RTPRINT(FBT, BT_TRACE, ("%s: Action Manual control!!\n", __func__));
-		return;
-	}
-
-	if (pBtCoex->bC2hBtInfoReqSent) {
-		if (!rtl8723a_BT_enabled(padapter)) {
-			pBtCoex->c2hBtInfo = BT_INFO_STATE_DISABLED;
-		} else {
-			if (pBtCoex->c2hBtInfo == BT_INFO_STATE_DISABLED)
-				pBtCoex->c2hBtInfo = BT_INFO_STATE_NO_CONNECTION;
-		}
-
-		btdm_BTCoexist8723AHandler(padapter);
-	} else if (!rtl8723a_BT_enabled(padapter)) {
-		pBtCoex->c2hBtInfo = BT_INFO_STATE_DISABLED;
-		btdm_BTCoexist8723AHandler(padapter);
-	}
-
-	BTDM_QueryBtInformation(padapter);
-}
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtc8723.c ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtcCsr1Ant.c ===== */
-
-/*  local function start with btdm_ */
-/*  extern function start with BTDM_ */
-
-static void BTDM_SetAntenna(struct rtw_adapter *padapter, u8 who)
-{
-}
-
-void
-BTDM_SingleAnt(
-	struct rtw_adapter *padapter,
-	u8 bSingleAntOn,
-	u8 bInterruptOn,
-	u8 bMultiNAVOn
-	)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 H2C_Parameter[3] = {0};
-
-	if (pHalData->bt_coexist.BT_Ant_Num != Ant_x1)
-		return;
-
-	H2C_Parameter[2] = 0;
-	H2C_Parameter[1] = 0;
-	H2C_Parameter[0] = 0;
-
-	if (bInterruptOn) {
-		H2C_Parameter[2] |= 0x02;	/* BIT1 */
-		pHalData->bt_coexist.bFWCoexistAllOff = false;
-	}
-	pHalData->bt_coexist.bInterruptOn = bInterruptOn;
-
-	if (bSingleAntOn) {
-		H2C_Parameter[2] |= 0x10;	/* BIT4 */
-		pHalData->bt_coexist.bFWCoexistAllOff = false;
-	}
-	pHalData->bt_coexist.bSingleAntOn = bSingleAntOn;
-
-	if (bMultiNAVOn) {
-		H2C_Parameter[2] |= 0x20;	/* BIT5 */
-		pHalData->bt_coexist.bFWCoexistAllOff = false;
-	}
-	pHalData->bt_coexist.bMultiNAVOn = bMultiNAVOn;
-
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], SingleAntenna =[%s:%s:%s], write 0xe = 0x%x\n",
-		bSingleAntOn?"ON":"OFF", bInterruptOn?"ON":"OFF", bMultiNAVOn?"ON":"OFF",
-		H2C_Parameter[0]<<16|H2C_Parameter[1]<<8|H2C_Parameter[2]));
-}
-
-void BTDM_CheckBTIdleChange1Ant(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-/*PMGNT_INFO		pMgntInfo = &padapter->MgntInfo; */
-	u8 	stateChange = false;
-	u32			BT_Polling, Ratio_Act, Ratio_STA;
-	u32				BT_Active, BT_State;
-	u32				regBTActive = 0, regBTState = 0, regBTPolling = 0;
-
-	if (!rtl8723a_BT_coexist(padapter))
-		return;
-	if (pBtMgnt->ExtConfig.bManualControl)
-		return;
-	if (pHalData->bt_coexist.BT_CoexistType != BT_CSR_BC8)
-		return;
-	if (pHalData->bt_coexist.BT_Ant_Num != Ant_x1)
-		return;
-
-	/*  The following we only consider CSR BC8 and fw version should be >= 62 */
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], FirmwareVersion = 0x%x(%d)\n",
-	pHalData->FirmwareVersion, pHalData->FirmwareVersion));
-	regBTActive = REG_BT_ACTIVE;
-	regBTState = REG_BT_STATE;
-	if (pHalData->FirmwareVersion >= FW_VER_BT_REG1)
-		regBTPolling = REG_BT_POLLING1;
-	else
-		regBTPolling = REG_BT_POLLING;
-
-	BT_Active = rtl8723au_read32(padapter, regBTActive);
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], BT_Active(0x%x) =%x\n", regBTActive, BT_Active));
-	BT_Active = BT_Active & 0x00ffffff;
-
-	BT_State = rtl8723au_read32(padapter, regBTState);
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], BT_State(0x%x) =%x\n", regBTState, BT_State));
-	BT_State = BT_State & 0x00ffffff;
-
-	BT_Polling = rtl8723au_read32(padapter, regBTPolling);
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], BT_Polling(0x%x) =%x\n", regBTPolling, BT_Polling));
-
-	if (BT_Active == 0xffffffff && BT_State == 0xffffffff && BT_Polling == 0xffffffff)
-		return;
-	if (BT_Polling == 0)
-		return;
-
-	Ratio_Act = BT_Active*1000/BT_Polling;
-	Ratio_STA = BT_State*1000/BT_Polling;
-
-	pHalData->bt_coexist.Ratio_Tx = Ratio_Act;
-	pHalData->bt_coexist.Ratio_PRI = Ratio_STA;
-
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], Ratio_Act =%d\n", Ratio_Act));
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], Ratio_STA =%d\n", Ratio_STA));
-
-	if (Ratio_STA < 60 && Ratio_Act < 500) {	/*  BT PAN idle */
-		pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_BT_PAN_IDLE;
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT_PAN_DOWNLINK;
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT_PAN_UPLINK;
-	} else {
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT_PAN_IDLE;
-
-		if (Ratio_STA) {
-			/*  Check if BT PAN (under BT 2.1) is uplink or downlink */
-			if ((Ratio_Act/Ratio_STA) < 2) {
-				/*  BT PAN Uplink */
-				pHalData->bt_coexist.BT21TrafficStatistics.bTxBusyTraffic = true;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_BT_PAN_UPLINK;
-				pHalData->bt_coexist.BT21TrafficStatistics.bRxBusyTraffic = false;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT_PAN_DOWNLINK;
-			} else {
-				/*  BT PAN downlink */
-				pHalData->bt_coexist.BT21TrafficStatistics.bTxBusyTraffic = false;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT_PAN_UPLINK;
-				pHalData->bt_coexist.BT21TrafficStatistics.bRxBusyTraffic = true;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_BT_PAN_DOWNLINK;
-			}
-		} else {
-			/*  BT PAN downlink */
-			pHalData->bt_coexist.BT21TrafficStatistics.bTxBusyTraffic = false;
-			pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT_PAN_UPLINK;
-			pHalData->bt_coexist.BT21TrafficStatistics.bRxBusyTraffic = true;
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_BT_PAN_DOWNLINK;
-		}
-	}
-
-	/*  Check BT is idle or not */
-	if (pBtMgnt->ExtConfig.NumberOfHandle == 0 &&
-	    pBtMgnt->ExtConfig.NumberOfSCO == 0) {
-		pBtMgnt->ExtConfig.bBTBusy = false;
-		pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_BT_IDLE;
-	} else {
-		if (Ratio_STA < 60) {
-			pBtMgnt->ExtConfig.bBTBusy = false;
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_BT_IDLE;
-		} else {
-			pBtMgnt->ExtConfig.bBTBusy = true;
-			pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT_IDLE;
-		}
-	}
-
-	if (pBtMgnt->ExtConfig.NumberOfHandle == 0 &&
-	    pBtMgnt->ExtConfig.NumberOfSCO == 0) {
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT_RSSI_LOW;
-		pBtMgnt->ExtConfig.MIN_BT_RSSI = 0;
-		BTDM_SetAntenna(padapter, BTDM_ANT_BT_IDLE);
-	} else {
-		if (pBtMgnt->ExtConfig.MIN_BT_RSSI <= -5) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_BT_RSSI_LOW;
-			RTPRINT(FBT, BT_TRACE, ("[DM][BT], core stack notify bt rssi Low\n"));
-		} else {
-			pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT_RSSI_LOW;
-			RTPRINT(FBT, BT_TRACE, ("[DM][BT], core stack notify bt rssi Normal\n"));
-		}
-	}
-
-	if (pHalData->bt_coexist.bBTBusyTraffic != pBtMgnt->ExtConfig.bBTBusy) {
-		/*  BT idle or BT non-idle */
-		pHalData->bt_coexist.bBTBusyTraffic = pBtMgnt->ExtConfig.bBTBusy;
-		stateChange = true;
-	}
-
-	if (stateChange) {
-		if (!pBtMgnt->ExtConfig.bBTBusy)
-			RTPRINT(FBT, BT_TRACE, ("[DM][BT], BT is idle or disable\n"));
-		else
-			RTPRINT(FBT, BT_TRACE, ("[DM][BT], BT is non-idle\n"));
-	}
-	if (!pBtMgnt->ExtConfig.bBTBusy) {
-		RTPRINT(FBT, BT_TRACE, ("[DM][BT], BT is idle or disable\n"));
-		if (check_fwstate(&padapter->mlmepriv, WIFI_UNDER_LINKING|WIFI_SITE_MONITOR) == true)
-			BTDM_SetAntenna(padapter, BTDM_ANT_WIFI);
-	}
-}
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtcCsr1Ant.c ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtcCsr2Ant.c ===== */
-
-/*  local function start with btdm_ */
-
-/*  Note: */
-/*  In the following, FW should be done before SW mechanism. */
-/*  BTDM_Balance(), BTDM_DiminishWiFi(), BT_NAV() should be done */
-/*  before BTDM_AGCTable(), BTDM_BBBackOffLevel(), btdm_DacSwing(). */
-
-/*  extern function start with BTDM_ */
-
-void
-BTDM_DiminishWiFi(
-	struct rtw_adapter *padapter,
-	u8 bDACOn,
-	u8 bInterruptOn,
-	u8 DACSwingLevel,
-	u8 bNAVOn
-	)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 H2C_Parameter[3] = {0};
-
-	if (pHalData->bt_coexist.BT_Ant_Num != Ant_x2)
-		return;
-
-	if ((pHalData->bt_coexist.CurrentState & BT_COEX_STATE_BT_RSSI_LOW) &&
-	    (DACSwingLevel == 0x20)) {
-		RTPRINT(FBT, BT_TRACE, ("[BT]DiminishWiFi 0x20 original, but set 0x18 for Low RSSI!\n"));
-		DACSwingLevel = 0x18;
-	}
-
-	H2C_Parameter[2] = 0;
-	H2C_Parameter[1] = DACSwingLevel;
-	H2C_Parameter[0] = 0;
-	if (bDACOn) {
-		H2C_Parameter[2] |= 0x01;	/* BIT0 */
-		if (bInterruptOn)
-			H2C_Parameter[2] |= 0x02;	/* BIT1 */
-		pHalData->bt_coexist.bFWCoexistAllOff = false;
-	}
-	if (bNAVOn) {
-		H2C_Parameter[2] |= 0x08;	/* BIT3 */
-		pHalData->bt_coexist.bFWCoexistAllOff = false;
-	}
-
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], bDACOn = %s, bInterruptOn = %s, write 0xe = 0x%x\n",
-		bDACOn?"ON":"OFF", bInterruptOn?"ON":"OFF",
-		H2C_Parameter[0]<<16|H2C_Parameter[1]<<8|H2C_Parameter[2]));
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], bNAVOn = %s\n",
-		bNAVOn?"ON":"OFF"));
-}
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtcCsr2Ant.c ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtCoexist.c ===== */
-
-/*  local function */
-static void btdm_ResetFWCoexState(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	pHalData->bt_coexist.CurrentState = 0;
-	pHalData->bt_coexist.PreviousState = 0;
-}
-
-static void btdm_InitBtCoexistDM(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	/*  20100415 Joseph: Restore RF register 0x1E and 0x1F value for further usage. */
-	pHalData->bt_coexist.BtRfRegOrigin1E = PHY_QueryRFReg(padapter, PathA, RF_RCK1, bRFRegOffsetMask);
-	pHalData->bt_coexist.BtRfRegOrigin1F = PHY_QueryRFReg(padapter, PathA, RF_RCK2, 0xf0);
-
-	pHalData->bt_coexist.CurrentState = 0;
-	pHalData->bt_coexist.PreviousState = 0;
-
-	BTDM_8723AInit(padapter);
-	pHalData->bt_coexist.bInitlized = true;
-}
-
-/*  */
-/*  extern function */
-/*  */
-void BTDM_CheckAntSelMode(struct rtw_adapter *padapter)
-{
-}
-
-void BTDM_FwC2hBtRssi(struct rtw_adapter *padapter, u8 *tmpBuf)
-{
-	BTDM_FwC2hBtRssi8723A(padapter, tmpBuf);
-}
-
-void BTDM_DisplayBtCoexInfo(struct rtw_adapter *padapter)
-{
-	BTDM_Display8723ABtCoexInfo(padapter);
-}
-
-void BTDM_RejectAPAggregatedPacket(struct rtw_adapter *padapter, u8 bReject)
-{
-}
-
-u8 BTDM_IsHT40(struct rtw_adapter *padapter)
-{
-	u8 isht40 = true;
-	enum ht_channel_width bw;
-
-	bw = padapter->mlmeextpriv.cur_bwmode;
-
-	if (bw == HT_CHANNEL_WIDTH_20)
-		isht40 = false;
-	else if (bw == HT_CHANNEL_WIDTH_40)
-		isht40 = true;
-
-	return isht40;
-}
-
-u8 BTDM_Legacy(struct rtw_adapter *padapter)
-{
-	struct mlme_ext_priv *pmlmeext;
-	u8 isLegacy = false;
-
-	pmlmeext = &padapter->mlmeextpriv;
-	if ((pmlmeext->cur_wireless_mode == WIRELESS_11B) ||
-		(pmlmeext->cur_wireless_mode == WIRELESS_11G) ||
-		(pmlmeext->cur_wireless_mode == WIRELESS_11BG))
-		isLegacy = true;
-
-	return isLegacy;
-}
-
-void BTDM_CheckWiFiState(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct mlme_priv *pmlmepriv;
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pmlmepriv = &padapter->mlmepriv;
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pmlmepriv->LinkDetectInfo.bBusyTraffic) {
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_IDLE;
-
-		if (pmlmepriv->LinkDetectInfo.bTxBusyTraffic)
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_UPLINK;
-		else
-			pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_UPLINK;
-
-		if (pmlmepriv->LinkDetectInfo.bRxBusyTraffic)
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_DOWNLINK;
-		else
-			pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_DOWNLINK;
-	} else {
-		pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_IDLE;
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_UPLINK;
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_DOWNLINK;
-	}
-
-	if (BTDM_Legacy(padapter)) {
-		pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_LEGACY;
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_HT20;
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_HT40;
-	} else {
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_LEGACY;
-		if (BTDM_IsHT40(padapter)) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_HT40;
-			pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_HT20;
-		} else {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_HT20;
-			pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_HT40;
-		}
-	}
-
-	if (pBtMgnt->BtOperationOn)
-		pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_BT30;
-	else
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_BT30;
-}
-
-s32 BTDM_GetRxSS(struct rtw_adapter *padapter)
-{
-/*PMGNT_INFO		pMgntInfo = &padapter->MgntInfo; */
-	struct mlme_priv *pmlmepriv;
-	struct hal_data_8723a *pHalData;
-	s32			UndecoratedSmoothedPWDB = 0;
-
-	pmlmepriv = &padapter->mlmepriv;
-	pHalData = GET_HAL_DATA(padapter);
-
-	if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-		UndecoratedSmoothedPWDB = GET_UNDECORATED_AVERAGE_RSSI(padapter);
-	} else { /*  associated entry pwdb */
-		UndecoratedSmoothedPWDB = pHalData->dmpriv.EntryMinUndecoratedSmoothedPWDB;
-		/* pHalData->BT_EntryMinUndecoratedSmoothedPWDB */
-	}
-	RTPRINT(FBT, BT_TRACE, ("BTDM_GetRxSS() = %d\n", UndecoratedSmoothedPWDB));
-	return UndecoratedSmoothedPWDB;
-}
-
-static s32 BTDM_GetRxBeaconSS(struct rtw_adapter *padapter)
-{
-/*PMGNT_INFO		pMgntInfo = &padapter->MgntInfo; */
-	struct mlme_priv *pmlmepriv;
-	struct hal_data_8723a *pHalData;
-	s32			pwdbBeacon = 0;
-
-	pmlmepriv = &padapter->mlmepriv;
-	pHalData = GET_HAL_DATA(padapter);
-
-	if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-		/* pwdbBeacon = pHalData->dmpriv.UndecoratedSmoothedBeacon; */
-		pwdbBeacon = pHalData->dmpriv.EntryMinUndecoratedSmoothedPWDB;
-	}
-	RTPRINT(FBT, BT_TRACE, ("BTDM_GetRxBeaconSS() = %d\n", pwdbBeacon));
-	return pwdbBeacon;
-}
-
-/*  Get beacon rssi state */
-u8 BTDM_CheckCoexBcnRssiState(struct rtw_adapter *padapter, u8 levelNum,
-			      u8 RssiThresh, u8 RssiThresh1)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	s32 pwdbBeacon = 0;
-	u8 bcnRssiState = 0;
-
-	pwdbBeacon = BTDM_GetRxBeaconSS(padapter);
-
-	if (levelNum == 2) {
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_MEDIUM;
-
-		if ((pHalData->bt_coexist.preRssiStateBeacon == BT_RSSI_STATE_LOW) ||
-		    (pHalData->bt_coexist.preRssiStateBeacon == BT_RSSI_STATE_STAY_LOW)) {
-			if (pwdbBeacon >= (RssiThresh+BT_FW_COEX_THRESH_TOL)) {
-				bcnRssiState = BT_RSSI_STATE_HIGH;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_BEACON_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state switch to High\n"));
-			} else {
-				bcnRssiState = BT_RSSI_STATE_STAY_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state stay at Low\n"));
-			}
-		} else {
-			if (pwdbBeacon < RssiThresh) {
-				bcnRssiState = BT_RSSI_STATE_LOW;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_BEACON_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state switch to Low\n"));
-			} else {
-				bcnRssiState = BT_RSSI_STATE_STAY_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state stay at High\n"));
-			}
-		}
-	} else if (levelNum == 3) {
-		if (RssiThresh > RssiThresh1) {
-			RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON thresh error!!\n"));
-			return pHalData->bt_coexist.preRssiStateBeacon;
-		}
-
-		if ((pHalData->bt_coexist.preRssiStateBeacon == BT_RSSI_STATE_LOW) ||
-		    (pHalData->bt_coexist.preRssiStateBeacon == BT_RSSI_STATE_STAY_LOW)) {
-			if (pwdbBeacon >= (RssiThresh+BT_FW_COEX_THRESH_TOL)) {
-				bcnRssiState = BT_RSSI_STATE_MEDIUM;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_BEACON_MEDIUM;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state switch to Medium\n"));
-			} else {
-				bcnRssiState = BT_RSSI_STATE_STAY_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state stay at Low\n"));
-			}
-		} else if ((pHalData->bt_coexist.preRssiStateBeacon == BT_RSSI_STATE_MEDIUM) ||
-			   (pHalData->bt_coexist.preRssiStateBeacon == BT_RSSI_STATE_STAY_MEDIUM)) {
-			if (pwdbBeacon >= (RssiThresh1+BT_FW_COEX_THRESH_TOL)) {
-				bcnRssiState = BT_RSSI_STATE_HIGH;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_BEACON_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_MEDIUM;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state switch to High\n"));
-			} else if (pwdbBeacon < RssiThresh) {
-				bcnRssiState = BT_RSSI_STATE_LOW;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_BEACON_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_MEDIUM;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state switch to Low\n"));
-			} else {
-				bcnRssiState = BT_RSSI_STATE_STAY_MEDIUM;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state stay at Medium\n"));
-			}
-		} else {
-			if (pwdbBeacon < RssiThresh1) {
-				bcnRssiState = BT_RSSI_STATE_MEDIUM;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_BEACON_MEDIUM;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_BEACON_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state switch to Medium\n"));
-			} else {
-				bcnRssiState = BT_RSSI_STATE_STAY_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_BEACON state stay at High\n"));
-			}
-		}
-	}
-
-	pHalData->bt_coexist.preRssiStateBeacon = bcnRssiState;
-
-	return bcnRssiState;
-}
-
-u8 BTDM_CheckCoexRSSIState1(struct rtw_adapter *padapter, u8 levelNum,
-			    u8 RssiThresh, u8 RssiThresh1)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	s32 UndecoratedSmoothedPWDB = 0;
-	u8 btRssiState = 0;
-
-	UndecoratedSmoothedPWDB = BTDM_GetRxSS(padapter);
-
-	if (levelNum == 2) {
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_MEDIUM;
-
-		if ((pHalData->bt_coexist.preRssiState1 == BT_RSSI_STATE_LOW) ||
-		    (pHalData->bt_coexist.preRssiState1 == BT_RSSI_STATE_STAY_LOW)) {
-			if (UndecoratedSmoothedPWDB >= (RssiThresh+BT_FW_COEX_THRESH_TOL)) {
-				btRssiState = BT_RSSI_STATE_HIGH;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_1_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state switch to High\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state stay at Low\n"));
-			}
-		} else {
-			if (UndecoratedSmoothedPWDB < RssiThresh) {
-				btRssiState = BT_RSSI_STATE_LOW;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_1_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state switch to Low\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state stay at High\n"));
-			}
-		}
-	} else if (levelNum == 3) {
-		if (RssiThresh > RssiThresh1) {
-			RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 thresh error!!\n"));
-			return pHalData->bt_coexist.preRssiState1;
-		}
-
-		if ((pHalData->bt_coexist.preRssiState1 == BT_RSSI_STATE_LOW) ||
-		    (pHalData->bt_coexist.preRssiState1 == BT_RSSI_STATE_STAY_LOW)) {
-			if (UndecoratedSmoothedPWDB >= (RssiThresh+BT_FW_COEX_THRESH_TOL)) {
-				btRssiState = BT_RSSI_STATE_MEDIUM;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_1_MEDIUM;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state switch to Medium\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state stay at Low\n"));
-			}
-		} else if ((pHalData->bt_coexist.preRssiState1 == BT_RSSI_STATE_MEDIUM) ||
-			   (pHalData->bt_coexist.preRssiState1 == BT_RSSI_STATE_STAY_MEDIUM)) {
-			if (UndecoratedSmoothedPWDB >= (RssiThresh1+BT_FW_COEX_THRESH_TOL)) {
-				btRssiState = BT_RSSI_STATE_HIGH;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_1_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_MEDIUM;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state switch to High\n"));
-			} else if (UndecoratedSmoothedPWDB < RssiThresh) {
-				btRssiState = BT_RSSI_STATE_LOW;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_1_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_MEDIUM;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state switch to Low\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_MEDIUM;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state stay at Medium\n"));
-			}
-		} else {
-			if (UndecoratedSmoothedPWDB < RssiThresh1) {
-				btRssiState = BT_RSSI_STATE_MEDIUM;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_1_MEDIUM;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_1_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state switch to Medium\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI_1 state stay at High\n"));
-			}
-		}
-	}
-
-	pHalData->bt_coexist.preRssiState1 = btRssiState;
-
-	return btRssiState;
-}
-
-u8 BTDM_CheckCoexRSSIState(struct rtw_adapter *padapter, u8 levelNum,
-			   u8 RssiThresh, u8 RssiThresh1)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	s32 UndecoratedSmoothedPWDB = 0;
-	u8 btRssiState = 0;
-
-	UndecoratedSmoothedPWDB = BTDM_GetRxSS(padapter);
-
-	if (levelNum == 2) {
-		pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_MEDIUM;
-
-		if ((pHalData->bt_coexist.preRssiState == BT_RSSI_STATE_LOW) ||
-		    (pHalData->bt_coexist.preRssiState == BT_RSSI_STATE_STAY_LOW)) {
-			if (UndecoratedSmoothedPWDB >= (RssiThresh+BT_FW_COEX_THRESH_TOL)) {
-				btRssiState = BT_RSSI_STATE_HIGH;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state switch to High\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state stay at Low\n"));
-			}
-		} else {
-			if (UndecoratedSmoothedPWDB < RssiThresh) {
-				btRssiState = BT_RSSI_STATE_LOW;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state switch to Low\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state stay at High\n"));
-			}
-		}
-	} else if (levelNum == 3) {
-		if (RssiThresh > RssiThresh1) {
-			RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI thresh error!!\n"));
-			return pHalData->bt_coexist.preRssiState;
-		}
-
-		if ((pHalData->bt_coexist.preRssiState == BT_RSSI_STATE_LOW) ||
-		    (pHalData->bt_coexist.preRssiState == BT_RSSI_STATE_STAY_LOW)) {
-			if (UndecoratedSmoothedPWDB >= (RssiThresh+BT_FW_COEX_THRESH_TOL)) {
-				btRssiState = BT_RSSI_STATE_MEDIUM;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_MEDIUM;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state switch to Medium\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state stay at Low\n"));
-			}
-		} else if ((pHalData->bt_coexist.preRssiState == BT_RSSI_STATE_MEDIUM) ||
-			   (pHalData->bt_coexist.preRssiState == BT_RSSI_STATE_STAY_MEDIUM)) {
-			if (UndecoratedSmoothedPWDB >= (RssiThresh1+BT_FW_COEX_THRESH_TOL)) {
-				btRssiState = BT_RSSI_STATE_HIGH;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_MEDIUM;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state switch to High\n"));
-			} else if (UndecoratedSmoothedPWDB < RssiThresh) {
-				btRssiState = BT_RSSI_STATE_LOW;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_LOW;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_MEDIUM;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state switch to Low\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_MEDIUM;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state stay at Medium\n"));
-			}
-		} else {
-			if (UndecoratedSmoothedPWDB < RssiThresh1) {
-				btRssiState = BT_RSSI_STATE_MEDIUM;
-				pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_WIFI_RSSI_MEDIUM;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_HIGH;
-				pHalData->bt_coexist.CurrentState &= ~BT_COEX_STATE_WIFI_RSSI_LOW;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state switch to Medium\n"));
-			} else {
-				btRssiState = BT_RSSI_STATE_STAY_HIGH;
-				RTPRINT(FBT, BT_TRACE, ("[DM][BT], RSSI state stay at High\n"));
-			}
-		}
-	}
-
-	pHalData->bt_coexist.preRssiState = btRssiState;
-
-	return btRssiState;
-}
-
-bool rtl8723a_BT_disable_EDCA_turbo(struct rtw_adapter *padapter)
-{
-	struct bt_mgnt *pBtMgnt;
-	struct hal_data_8723a *pHalData;
-	u8 bBtChangeEDCA = false;
-	u32 EDCA_BT_BE = 0x5ea42b, cur_EDCA_reg;
-	bool bRet = false;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBtMgnt = &pHalData->BtInfo.BtMgnt;
-
-	if (!rtl8723a_BT_coexist(padapter)) {
-		bRet = false;
-		pHalData->bt_coexist.lastBtEdca = 0;
-		return bRet;
-	}
-	if (!((pBtMgnt->bSupportProfile) ||
-	    (pHalData->bt_coexist.BT_CoexistType == BT_CSR_BC8))) {
-		bRet = false;
-		pHalData->bt_coexist.lastBtEdca = 0;
-		return bRet;
-	}
-
-	if (rtl8723a_BT_using_antenna_1(padapter)) {
-		bRet = false;
-		pHalData->bt_coexist.lastBtEdca = 0;
-		return bRet;
-	}
-
-	if (pHalData->bt_coexist.exec_cnt < 3)
-		pHalData->bt_coexist.exec_cnt++;
-	else
-		pHalData->bt_coexist.bEDCAInitialized = true;
-
-	/*  When BT is non idle */
-	if (!(pHalData->bt_coexist.CurrentState & BT_COEX_STATE_BT_IDLE)) {
-		RTPRINT(FBT, BT_TRACE, ("BT state non idle, set bt EDCA\n"));
-
-		/* aggr_num = 0x0909; */
-		if (pHalData->odmpriv.DM_EDCA_Table.bCurrentTurboEDCA) {
-			bBtChangeEDCA = true;
-			pHalData->odmpriv.DM_EDCA_Table.bCurrentTurboEDCA = false;
-			pHalData->dmpriv.prv_traffic_idx = 3;
-		}
-		cur_EDCA_reg = rtl8723au_read32(padapter, REG_EDCA_BE_PARAM);
-
-		if (cur_EDCA_reg != EDCA_BT_BE)
-			bBtChangeEDCA = true;
-		if (bBtChangeEDCA || !pHalData->bt_coexist.bEDCAInitialized) {
-			rtl8723au_write32(padapter, REG_EDCA_BE_PARAM,
-					  EDCA_BT_BE);
-			pHalData->bt_coexist.lastBtEdca = EDCA_BT_BE;
-		}
-		bRet = true;
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("BT state idle, set original EDCA\n"));
-		pHalData->bt_coexist.lastBtEdca = 0;
-		bRet = false;
-	}
-	return bRet;
-}
-
-void
-BTDM_Balance(
-	struct rtw_adapter *padapter,
-	u8 bBalanceOn,
-	u8 ms0,
-	u8 ms1
-	)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 H2C_Parameter[3] = {0};
-
-	if (bBalanceOn) {
-		H2C_Parameter[2] = 1;
-		H2C_Parameter[1] = ms1;
-		H2C_Parameter[0] = ms0;
-		pHalData->bt_coexist.bFWCoexistAllOff = false;
-	} else {
-		H2C_Parameter[2] = 0;
-		H2C_Parameter[1] = 0;
-		H2C_Parameter[0] = 0;
-	}
-	pHalData->bt_coexist.bBalanceOn = bBalanceOn;
-
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], Balance =[%s:%dms:%dms], write 0xc = 0x%x\n",
-		bBalanceOn?"ON":"OFF", ms0, ms1,
-		H2C_Parameter[0]<<16|H2C_Parameter[1]<<8|H2C_Parameter[2]));
-
-	FillH2CCmd(padapter, 0xc, 3, H2C_Parameter);
-}
-
-void BTDM_AGCTable(struct rtw_adapter *padapter, u8 type)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	if (type == BT_AGCTABLE_OFF) {
-		RTPRINT(FBT, BT_TRACE, ("[BT]AGCTable Off!\n"));
-		rtl8723au_write32(padapter, 0xc78, 0x641c0001);
-		rtl8723au_write32(padapter, 0xc78, 0x631d0001);
-		rtl8723au_write32(padapter, 0xc78, 0x621e0001);
-		rtl8723au_write32(padapter, 0xc78, 0x611f0001);
-		rtl8723au_write32(padapter, 0xc78, 0x60200001);
-
-		PHY_SetRFReg(padapter, PathA, RF_RX_AGC_HP, bRFRegOffsetMask, 0x32000);
-		PHY_SetRFReg(padapter, PathA, RF_RX_AGC_HP, bRFRegOffsetMask, 0x71000);
-		PHY_SetRFReg(padapter, PathA, RF_RX_AGC_HP, bRFRegOffsetMask, 0xb0000);
-		PHY_SetRFReg(padapter, PathA, RF_RX_AGC_HP, bRFRegOffsetMask, 0xfc000);
-		PHY_SetRFReg(padapter, PathA, RF_RX_G1, bRFRegOffsetMask, 0x30355);
-
-		pHalData->bt_coexist.b8723aAgcTableOn = false;
-	} else if (type == BT_AGCTABLE_ON) {
-		RTPRINT(FBT, BT_TRACE, ("[BT]AGCTable On!\n"));
-		rtl8723au_write32(padapter, 0xc78, 0x4e1c0001);
-		rtl8723au_write32(padapter, 0xc78, 0x4d1d0001);
-		rtl8723au_write32(padapter, 0xc78, 0x4c1e0001);
-		rtl8723au_write32(padapter, 0xc78, 0x4b1f0001);
-		rtl8723au_write32(padapter, 0xc78, 0x4a200001);
-
-		PHY_SetRFReg(padapter, PathA, RF_RX_AGC_HP, bRFRegOffsetMask, 0xdc000);
-		PHY_SetRFReg(padapter, PathA, RF_RX_AGC_HP, bRFRegOffsetMask, 0x90000);
-		PHY_SetRFReg(padapter, PathA, RF_RX_AGC_HP, bRFRegOffsetMask, 0x51000);
-		PHY_SetRFReg(padapter, PathA, RF_RX_AGC_HP, bRFRegOffsetMask, 0x12000);
-		PHY_SetRFReg(padapter, PathA, RF_RX_G1, bRFRegOffsetMask, 0x00355);
-
-		pHalData->bt_coexist.b8723aAgcTableOn = true;
-
-		pHalData->bt_coexist.bSWCoexistAllOff = false;
-	}
-}
-
-void BTDM_BBBackOffLevel(struct rtw_adapter *padapter, u8 type)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (type == BT_BB_BACKOFF_OFF) {
-		RTPRINT(FBT, BT_TRACE, ("[BT]BBBackOffLevel Off!\n"));
-		rtl8723au_write32(padapter, 0xc04, 0x3a05611);
-	} else if (type == BT_BB_BACKOFF_ON) {
-		RTPRINT(FBT, BT_TRACE, ("[BT]BBBackOffLevel On!\n"));
-		rtl8723au_write32(padapter, 0xc04, 0x3a07611);
-		pHalData->bt_coexist.bSWCoexistAllOff = false;
-	}
-}
-
-void BTDM_FWCoexAllOff(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	RTPRINT(FBT, BT_TRACE, ("BTDM_FWCoexAllOff()\n"));
-	if (pHalData->bt_coexist.bFWCoexistAllOff)
-		return;
-	RTPRINT(FBT, BT_TRACE, ("BTDM_FWCoexAllOff(), real Do\n"));
-
-	BTDM_FWCoexAllOff8723A(padapter);
-
-	pHalData->bt_coexist.bFWCoexistAllOff = true;
-}
-
-void BTDM_SWCoexAllOff(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	RTPRINT(FBT, BT_TRACE, ("BTDM_SWCoexAllOff()\n"));
-	if (pHalData->bt_coexist.bSWCoexistAllOff)
-		return;
-	RTPRINT(FBT, BT_TRACE, ("BTDM_SWCoexAllOff(), real Do\n"));
-	BTDM_SWCoexAllOff8723A(padapter);
-
-	pHalData->bt_coexist.bSWCoexistAllOff = true;
-}
-
-void BTDM_HWCoexAllOff(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	RTPRINT(FBT, BT_TRACE, ("BTDM_HWCoexAllOff()\n"));
-	if (pHalData->bt_coexist.bHWCoexistAllOff)
-		return;
-	RTPRINT(FBT, BT_TRACE, ("BTDM_HWCoexAllOff(), real Do\n"));
-
-	BTDM_HWCoexAllOff8723A(padapter);
-
-	pHalData->bt_coexist.bHWCoexistAllOff = true;
-}
-
-void BTDM_CoexAllOff(struct rtw_adapter *padapter)
-{
-	BTDM_FWCoexAllOff(padapter);
-	BTDM_SWCoexAllOff(padapter);
-	BTDM_HWCoexAllOff(padapter);
-}
-
-void rtl8723a_BT_disable_coexist(struct rtw_adapter *padapter)
-{
-	struct pwrctrl_priv *ppwrctrl = &padapter->pwrctrlpriv;
-
-	if (!rtl8723a_BT_coexist(padapter))
-		return;
-
-	/*  8723 1Ant doesn't need to turn off bt coexist mechanism. */
-	if (rtl8723a_BT_using_antenna_1(padapter))
-		return;
-
-	/*  Before enter IPS, turn off FW BT Co-exist mechanism */
-	if (ppwrctrl->reg_rfoff == rf_on) {
-		RTPRINT(FBT, BT_TRACE, ("[BT][DM], Before enter IPS, turn off all Coexist DM\n"));
-		btdm_ResetFWCoexState(padapter);
-		BTDM_CoexAllOff(padapter);
-		BTDM_SetAntenna(padapter, BTDM_ANT_BT);
-	}
-}
-
-void BTDM_SignalCompensation(struct rtw_adapter *padapter, u8 *rssi_wifi, u8 *rssi_bt)
-{
-	BTDM_8723ASignalCompensation(padapter, rssi_wifi, rssi_bt);
-}
-
-void rtl8723a_BT_do_coexist(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (!rtl8723a_BT_coexist(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("[DM][BT], BT not exists!!\n"));
-		return;
-	}
-
-	if (!pHalData->bt_coexist.bInitlized) {
-		RTPRINT(FBT, BT_TRACE, ("[DM][BT], btdm_InitBtCoexistDM()\n"));
-		btdm_InitBtCoexistDM(padapter);
-	}
-
-	RTPRINT(FBT, BT_TRACE, ("\n\n[DM][BT], BTDM start!!\n"));
-
-	BTDM_PWDBMonitor(padapter);
-
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], HW type is 8723\n"));
-	BTDM_BTCoexist8723A(padapter);
-	RTPRINT(FBT, BT_TRACE, ("[DM][BT], BTDM end!!\n\n"));
-}
-
-void BTDM_UpdateCoexState(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (!BTDM_IsSameCoexistState(padapter)) {
-		RTPRINT(FBT, BT_TRACE, ("[BTCoex], Coexist State[bitMap] change from 0x%"i64fmt"x to 0x%"i64fmt"x,  changeBits = 0x%"i64fmt"x\n",
-			pHalData->bt_coexist.PreviousState,
-			pHalData->bt_coexist.CurrentState,
-			(pHalData->bt_coexist.PreviousState^pHalData->bt_coexist.CurrentState)));
-		pHalData->bt_coexist.PreviousState = pHalData->bt_coexist.CurrentState;
-	}
-}
-
-u8 BTDM_IsSameCoexistState(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (pHalData->bt_coexist.PreviousState == pHalData->bt_coexist.CurrentState) {
-		return true;
-	} else {
-		RTPRINT(FBT, BT_TRACE, ("[DM][BT], Coexist state changed!!\n"));
-		return false;
-	}
-}
-
-void BTDM_PWDBMonitor(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(GetDefaultAdapter(padapter));
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 H2C_Parameter[3] = {0};
-	s32 tmpBTEntryMaxPWDB = 0, tmpBTEntryMinPWDB = 0xff;
-	u8 i;
-
-	if (pBtMgnt->BtOperationOn) {
-		for (i = 0; i < MAX_BT_ASOC_ENTRY_NUM; i++) {
-			if (pBTInfo->BtAsocEntry[i].bUsed) {
-				if (pBTInfo->BtAsocEntry[i].UndecoratedSmoothedPWDB < tmpBTEntryMinPWDB)
-					tmpBTEntryMinPWDB = pBTInfo->BtAsocEntry[i].UndecoratedSmoothedPWDB;
-				if (pBTInfo->BtAsocEntry[i].UndecoratedSmoothedPWDB > tmpBTEntryMaxPWDB)
-					tmpBTEntryMaxPWDB = pBTInfo->BtAsocEntry[i].UndecoratedSmoothedPWDB;
-				/*  Report every BT connection (HS mode) RSSI to FW */
-				H2C_Parameter[2] = (u8)(pBTInfo->BtAsocEntry[i].UndecoratedSmoothedPWDB & 0xFF);
-				H2C_Parameter[0] = (MAX_FW_SUPPORT_MACID_NUM-1-i);
-				RTPRINT(FDM, DM_BT30, ("RSSI report for BT[%d], H2C_Par = 0x%x\n", i, H2C_Parameter[0]));
-				FillH2CCmd(padapter, RSSI_SETTING_EID, 3, H2C_Parameter);
-				RTPRINT_ADDR(FDM, (DM_PWDB|DM_BT30), ("BT_Entry Mac :"),
-					     pBTInfo->BtAsocEntry[i].BTRemoteMACAddr)
-				RTPRINT(FDM, (DM_PWDB|DM_BT30),
-					("BT rx pwdb[%d] = 0x%x(%d)\n", i,
-					pBTInfo->BtAsocEntry[i].UndecoratedSmoothedPWDB,
-					pBTInfo->BtAsocEntry[i].UndecoratedSmoothedPWDB));
-			}
-		}
-		if (tmpBTEntryMaxPWDB != 0) {	/*  If associated entry is found */
-			pHalData->dmpriv.BT_EntryMaxUndecoratedSmoothedPWDB = tmpBTEntryMaxPWDB;
-			RTPRINT(FDM, (DM_PWDB|DM_BT30), ("BT_EntryMaxPWDB = 0x%x(%d)\n",
-				tmpBTEntryMaxPWDB, tmpBTEntryMaxPWDB));
-		} else {
-			pHalData->dmpriv.BT_EntryMaxUndecoratedSmoothedPWDB = 0;
-		}
-		if (tmpBTEntryMinPWDB != 0xff) { /*  If associated entry is found */
-			pHalData->dmpriv.BT_EntryMinUndecoratedSmoothedPWDB = tmpBTEntryMinPWDB;
-			RTPRINT(FDM, (DM_PWDB|DM_BT30), ("BT_EntryMinPWDB = 0x%x(%d)\n",
-				tmpBTEntryMinPWDB, tmpBTEntryMinPWDB));
-		} else {
-			pHalData->dmpriv.BT_EntryMinUndecoratedSmoothedPWDB = 0;
-		}
-	}
-}
-
-u8 BTDM_IsBTBusy(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_mgnt *pBtMgnt = &pBTInfo->BtMgnt;
-
-	if (pBtMgnt->ExtConfig.bBTBusy)
-		return true;
-	else
-		return false;
-}
-
-u8 BTDM_IsWifiBusy(struct rtw_adapter *padapter)
-{
-/*PMGNT_INFO		pMgntInfo = &GetDefaultAdapter(padapter)->MgntInfo; */
-	struct mlme_priv *pmlmepriv = &GetDefaultAdapter(padapter)->mlmepriv;
-	struct bt_30info *pBTInfo = GET_BT_INFO(padapter);
-	struct bt_traffic *pBtTraffic = &pBTInfo->BtTraffic;
-
-	if (pmlmepriv->LinkDetectInfo.bBusyTraffic ||
-		pBtTraffic->Bt30TrafficStatistics.bTxBusyTraffic ||
-		pBtTraffic->Bt30TrafficStatistics.bRxBusyTraffic)
-		return true;
-	else
-		return false;
-}
-
-u8 BTDM_IsCoexistStateChanged(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (pHalData->bt_coexist.PreviousState == pHalData->bt_coexist.CurrentState)
-		return false;
-	else
-		return true;
-}
-
-u8 BTDM_IsWifiUplink(struct rtw_adapter *padapter)
-{
-/*PMGNT_INFO		pMgntInfo = &GetDefaultAdapter(padapter)->MgntInfo; */
-	struct mlme_priv *pmlmepriv;
-	struct bt_30info *pBTInfo;
-	struct bt_traffic *pBtTraffic;
-
-	pmlmepriv = &padapter->mlmepriv;
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtTraffic = &pBTInfo->BtTraffic;
-
-	if ((pmlmepriv->LinkDetectInfo.bTxBusyTraffic) ||
-		(pBtTraffic->Bt30TrafficStatistics.bTxBusyTraffic))
-		return true;
-	else
-		return false;
-}
-
-u8 BTDM_IsWifiDownlink(struct rtw_adapter *padapter)
-{
-/*PMGNT_INFO		pMgntInfo = &GetDefaultAdapter(padapter)->MgntInfo; */
-	struct mlme_priv *pmlmepriv;
-	struct bt_30info *pBTInfo;
-	struct bt_traffic *pBtTraffic;
-
-	pmlmepriv = &padapter->mlmepriv;
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtTraffic = &pBTInfo->BtTraffic;
-
-	if ((pmlmepriv->LinkDetectInfo.bRxBusyTraffic) ||
-		(pBtTraffic->Bt30TrafficStatistics.bRxBusyTraffic))
-		return true;
-	else
-		return false;
-}
-
-u8 BTDM_IsBTHSMode(struct rtw_adapter *padapter)
-{
-/*PMGNT_INFO		pMgntInfo = &GetDefaultAdapter(padapter)->MgntInfo; */
-	struct hal_data_8723a *pHalData;
-	struct bt_mgnt *pBtMgnt;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBtMgnt = &pHalData->BtInfo.BtMgnt;
-
-	if (pBtMgnt->BtOperationOn)
-		return true;
-	else
-		return false;
-}
-
-u8 BTDM_IsBTUplink(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (pHalData->bt_coexist.BT21TrafficStatistics.bTxBusyTraffic)
-		return true;
-	else
-		return false;
-}
-
-u8 BTDM_IsBTDownlink(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (pHalData->bt_coexist.BT21TrafficStatistics.bRxBusyTraffic)
-		return true;
-	else
-		return false;
-}
-
-void BTDM_AdjustForBtOperation(struct rtw_adapter *padapter)
-{
-	RTPRINT(FBT, BT_TRACE, ("[BT][DM], BTDM_AdjustForBtOperation()\n"));
-	BTDM_AdjustForBtOperation8723A(padapter);
-}
-
-void BTDM_SetBtCoexCurrAntNum(struct rtw_adapter *padapter, u8 antNum)
-{
-	BTDM_Set8723ABtCoexCurrAntNum(padapter, antNum);
-}
-
-void BTDM_ForHalt(struct rtw_adapter *padapter)
-{
-	if (!rtl8723a_BT_coexist(padapter))
-		return;
-
-	BTDM_ForHalt8723A(padapter);
-	GET_HAL_DATA(padapter)->bt_coexist.bInitlized = false;
-}
-
-void BTDM_WifiScanNotify(struct rtw_adapter *padapter, u8 scanType)
-{
-	if (!rtl8723a_BT_coexist(padapter))
-		return;
-
-	BTDM_WifiScanNotify8723A(padapter, scanType);
-}
-
-void BTDM_WifiAssociateNotify(struct rtw_adapter *padapter, u8 action)
-{
-	if (!rtl8723a_BT_coexist(padapter))
-		return;
-
-	BTDM_WifiAssociateNotify8723A(padapter, action);
-}
-
-void rtl8723a_BT_mediastatus_notify(struct rtw_adapter *padapter,
-				    enum rt_media_status mstatus)
-{
-	if (!rtl8723a_BT_coexist(padapter))
-		return;
-
-	BTDM_MediaStatusNotify8723A(padapter, mstatus);
-}
-
-void rtl8723a_BT_specialpacket_notify(struct rtw_adapter *padapter)
-{
-	if (!rtl8723a_BT_coexist(padapter))
-		return;
-
-	BTDM_ForDhcp8723A(padapter);
-}
-
-void BTDM_ResetActionProfileState(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	pHalData->bt_coexist.CurrentState &= ~\
-		(BT_COEX_STATE_PROFILE_HID|BT_COEX_STATE_PROFILE_A2DP|
-		BT_COEX_STATE_PROFILE_PAN|BT_COEX_STATE_PROFILE_SCO);
-}
-
-u8 BTDM_IsActionSCO(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_dgb *pBtDbg;
-	u8 bRet;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtDbg = &pBTInfo->BtDbg;
-	bRet = false;
-
-	if (pBtDbg->dbgCtrl) {
-		if (pBtDbg->dbgProfile == BT_DBG_PROFILE_SCO) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_PROFILE_SCO;
-			bRet = true;
-		}
-	} else {
-		if (pBtMgnt->ExtConfig.NumberOfSCO > 0) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_PROFILE_SCO;
-			bRet = true;
-		}
-	}
-	return bRet;
-}
-
-u8 BTDM_IsActionHID(struct rtw_adapter *padapter)
-{
-	struct bt_30info *pBTInfo;
-	struct hal_data_8723a *pHalData;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_dgb *pBtDbg;
-	u8 bRet;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtDbg = &pBTInfo->BtDbg;
-	bRet = false;
-
-	if (pBtDbg->dbgCtrl) {
-		if (pBtDbg->dbgProfile == BT_DBG_PROFILE_HID) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_PROFILE_HID;
-			bRet = true;
-		}
-	} else {
-		if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID) &&
-		    pBtMgnt->ExtConfig.NumberOfHandle == 1) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_PROFILE_HID;
-			bRet = true;
-		}
-	}
-	return bRet;
-}
-
-u8 BTDM_IsActionA2DP(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_dgb *pBtDbg;
-	u8 bRet;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtDbg = &pBTInfo->BtDbg;
-	bRet = false;
-
-	if (pBtDbg->dbgCtrl) {
-		if (pBtDbg->dbgProfile == BT_DBG_PROFILE_A2DP) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_PROFILE_A2DP;
-			bRet = true;
-		}
-	} else {
-		if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP) &&
-		    pBtMgnt->ExtConfig.NumberOfHandle == 1) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_PROFILE_A2DP;
-			bRet = true;
-		}
-	}
-	return bRet;
-}
-
-u8 BTDM_IsActionPAN(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_dgb *pBtDbg;
-	u8 bRet;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtDbg = &pBTInfo->BtDbg;
-	bRet = false;
-
-	if (pBtDbg->dbgCtrl) {
-		if (pBtDbg->dbgProfile == BT_DBG_PROFILE_PAN) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_PROFILE_PAN;
-			bRet = true;
-		}
-	} else {
-		if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN) &&
-		    pBtMgnt->ExtConfig.NumberOfHandle == 1) {
-			pHalData->bt_coexist.CurrentState |= BT_COEX_STATE_PROFILE_PAN;
-			bRet = true;
-		}
-	}
-	return bRet;
-}
-
-u8 BTDM_IsActionHIDA2DP(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_30info *pBTInfo;
-	struct bt_mgnt *pBtMgnt;
-	struct bt_dgb *pBtDbg;
-	u8 bRet;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtMgnt = &pBTInfo->BtMgnt;
-	pBtDbg = &pBTInfo->BtDbg;
-	bRet = false;
-
-	if (pBtDbg->dbgCtrl) {
-		if (pBtDbg->dbgProfile == BT_DBG_PROFILE_HID_A2DP) {
-			pHalData->bt_coexist.CurrentState |= (BT_COEX_STATE_PROFILE_HID|BT_COEX_STATE_PROFILE_A2DP);
-			bRet = true;
-		}
-	} else {
-		if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID) &&
-		    BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-			pHalData->bt_coexist.CurrentState |= (BT_COEX_STATE_PROFILE_HID|BT_COEX_STATE_PROFILE_A2DP);
-			bRet = true;
-		}
-	}
-	return bRet;
-}
-
-u8 BTDM_IsActionHIDPAN(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_30info *pBTInfo;
-	struct bt_dgb *pBtDbg;
-	u8 bRet;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtDbg = &pBTInfo->BtDbg;
-	bRet = false;
-
-	if (pBtDbg->dbgCtrl) {
-		if (pBtDbg->dbgProfile == BT_DBG_PROFILE_HID_PAN) {
-			pHalData->bt_coexist.CurrentState |= (BT_COEX_STATE_PROFILE_HID|BT_COEX_STATE_PROFILE_PAN);
-			bRet = true;
-		}
-	} else {
-		if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_HID) &&
-		    BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN)) {
-			pHalData->bt_coexist.CurrentState |= (BT_COEX_STATE_PROFILE_HID|BT_COEX_STATE_PROFILE_PAN);
-			bRet = true;
-		}
-	}
-	return bRet;
-}
-
-u8 BTDM_IsActionPANA2DP(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct bt_30info *pBTInfo;
-	struct bt_dgb *pBtDbg;
-	u8 bRet;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pBTInfo = GET_BT_INFO(padapter);
-	pBtDbg = &pBTInfo->BtDbg;
-	bRet = false;
-
-	if (pBtDbg->dbgCtrl) {
-		if (pBtDbg->dbgProfile == BT_DBG_PROFILE_PAN_A2DP) {
-			pHalData->bt_coexist.CurrentState |= (BT_COEX_STATE_PROFILE_PAN|BT_COEX_STATE_PROFILE_A2DP);
-			bRet = true;
-		}
-	} else {
-		if (BTHCI_CheckProfileExist(padapter, BT_PROFILE_PAN) && BTHCI_CheckProfileExist(padapter, BT_PROFILE_A2DP)) {
-			pHalData->bt_coexist.CurrentState |= (BT_COEX_STATE_PROFILE_PAN|BT_COEX_STATE_PROFILE_A2DP);
-			bRet = true;
-		}
-	}
-	return bRet;
-}
-
-bool rtl8723a_BT_enabled(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (pHalData->bt_coexist.bCurBtDisabled)
-		return false;
-	else
-		return true;
-}
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtCoexist.c ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/HalBT.c ===== */
-
-/*  */
-/*local function */
-/*  */
-
-static void halbt_InitHwConfig8723A(struct rtw_adapter *padapter)
-{
-}
-
-/*  */
-/*extern function */
-/*  */
-u8 HALBT_GetPGAntNum(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	return pHalData->bt_coexist.BT_Ant_Num;
-}
-
-void HALBT_SetKey(struct rtw_adapter *padapter, u8 EntryNum)
-{
-	struct bt_30info *pBTinfo;
-	struct bt_asoc_entry *pBtAssocEntry;
-	u16				usConfig = 0;
-
-	pBTinfo = GET_BT_INFO(padapter);
-	pBtAssocEntry = &pBTinfo->BtAsocEntry[EntryNum];
-
-	pBtAssocEntry->HwCAMIndex = BT_HWCAM_STAR + EntryNum;
-
-	usConfig = CAM_VALID | (CAM_AES << 2);
-	rtl8723a_cam_write(padapter, pBtAssocEntry->HwCAMIndex, usConfig,
-			   pBtAssocEntry->BTRemoteMACAddr,
-			   pBtAssocEntry->PTK + TKIP_ENC_KEY_POS);
-}
-
-void HALBT_RemoveKey(struct rtw_adapter *padapter, u8 EntryNum)
-{
-	struct bt_30info *pBTinfo;
-	struct bt_asoc_entry *pBtAssocEntry;
-
-	pBTinfo = GET_BT_INFO(padapter);
-	pBtAssocEntry = &pBTinfo->BtAsocEntry[EntryNum];
-
-	if (pBTinfo->BtAsocEntry[EntryNum].HwCAMIndex != 0) {
-		/*  ToDo : add New HALBT_RemoveKey function !! */
-		if (pBtAssocEntry->HwCAMIndex >= BT_HWCAM_STAR &&
-		    pBtAssocEntry->HwCAMIndex < HALF_CAM_ENTRY)
-			rtl8723a_cam_empty_entry(padapter,
-						 pBtAssocEntry->HwCAMIndex);
-		pBTinfo->BtAsocEntry[EntryNum].HwCAMIndex = 0;
-	}
-}
-
-void rtl8723a_BT_init_hal_vars(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-
-	pHalData = GET_HAL_DATA(padapter);
-
-	pHalData->bt_coexist.BluetoothCoexist = pHalData->EEPROMBluetoothCoexist;
-	pHalData->bt_coexist.BT_Ant_Num = pHalData->EEPROMBluetoothAntNum;
-	pHalData->bt_coexist.BT_CoexistType = pHalData->EEPROMBluetoothType;
-	pHalData->bt_coexist.BT_Ant_isolation = pHalData->EEPROMBluetoothAntIsolation;
-	pHalData->bt_coexist.bt_radiosharedtype = pHalData->EEPROMBluetoothRadioShared;
-
-	RT_TRACE(_module_hal_init_c_, _drv_info_,
-		 "BT Coexistance = 0x%x\n", rtl8723a_BT_coexist(padapter));
-
-	if (rtl8723a_BT_coexist(padapter)) {
-		if (pHalData->bt_coexist.BT_Ant_Num == Ant_x2) {
-			BTDM_SetBtCoexCurrAntNum(padapter, 2);
-			RT_TRACE(_module_hal_init_c_, _drv_info_,
-				 "BlueTooth BT_Ant_Num = Antx2\n");
-		} else if (pHalData->bt_coexist.BT_Ant_Num == Ant_x1) {
-			BTDM_SetBtCoexCurrAntNum(padapter, 1);
-			RT_TRACE(_module_hal_init_c_, _drv_info_,
-				 "BlueTooth BT_Ant_Num = Antx1\n");
-		}
-		pHalData->bt_coexist.bBTBusyTraffic = false;
-		pHalData->bt_coexist.bBTTrafficModeSet = false;
-		pHalData->bt_coexist.bBTNonTrafficModeSet = false;
-		pHalData->bt_coexist.CurrentState = 0;
-		pHalData->bt_coexist.PreviousState = 0;
-
-		RT_TRACE(_module_hal_init_c_, _drv_info_,
-			 "bt_radiosharedType = 0x%x\n",
-			 pHalData->bt_coexist.bt_radiosharedtype);
-	}
-}
-
-bool rtl8723a_BT_coexist(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (pHalData->bt_coexist.BluetoothCoexist)
-		return true;
-	else
-		return false;
-}
-
-u8 HALBT_BTChipType(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	return pHalData->bt_coexist.BT_CoexistType;
-}
-
-void rtl8723a_BT_init_hwconfig(struct rtw_adapter *padapter)
-{
-	halbt_InitHwConfig8723A(padapter);
-	rtl8723a_BT_do_coexist(padapter);
-}
-
-void HALBT_SetRtsCtsNoLenLimit(struct rtw_adapter *padapter)
-{
-}
-
-/*  ===== End of sync from SD7 driver HAL/HalBT.c ===== */
-
-void rtl8723a_dual_antenna_detection(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct dm_odm_t *pDM_Odm;
-	struct sw_ant_sw *pDM_SWAT_Table;
-	u8 i;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pDM_Odm = &pHalData->odmpriv;
-	pDM_SWAT_Table = &pDM_Odm->DM_SWAT_Table;
-
-	/*  */
-	/*  <Roger_Notes> RTL8723A Single and Dual antenna dynamic detection
-	    mechanism when RF power state is on. */
-	/*  We should take power tracking, IQK, LCK, RCK RF read/write
-	    operation into consideration. */
-	/*  2011.12.15. */
-	/*  */
-	if (!pHalData->bAntennaDetected) {
-		u8 btAntNum = BT_GetPGAntNum(padapter);
-
-		/*  Set default antenna B status */
-		if (btAntNum == Ant_x2)
-			pDM_SWAT_Table->ANTB_ON = true;
-		else if (btAntNum == Ant_x1)
-			pDM_SWAT_Table->ANTB_ON = false;
-		else
-			pDM_SWAT_Table->ANTB_ON = true;
-
-		if (pHalData->CustomerID != RT_CID_TOSHIBA) {
-			for (i = 0; i < MAX_ANTENNA_DETECTION_CNT; i++) {
-				if (ODM_SingleDualAntennaDetection
-				    (&pHalData->odmpriv, ANTTESTALL) == true)
-					break;
-			}
-
-			/*  Set default antenna number for BT coexistence */
-			if (btAntNum == Ant_x2)
-				BT_SetBtCoexCurrAntNum(padapter,
-						       pDM_SWAT_Table->
-						       ANTB_ON ? 2 : 1);
-		}
-		pHalData->bAntennaDetected = true;
-	}
-}
diff --git a/drivers/staging/rtl8723au/hal/rtl8723a_cmd.c b/drivers/staging/rtl8723au/hal/rtl8723a_cmd.c
deleted file mode 100644
index 2230f4c539ec..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723a_cmd.c
+++ /dev/null
@@ -1,755 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTL8723A_CMD_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <recv_osdep.h>
-#include <mlme_osdep.h>
-#include <rtl8723a_hal.h>
-#include <usb_ops_linux.h>
-
-#define RTL92C_MAX_H2C_BOX_NUMS		4
-#define RTL92C_MAX_CMD_LEN		5
-#define MESSAGE_BOX_SIZE		4
-#define EX_MESSAGE_BOX_SIZE		2
-
-static u8 _is_fw_read_cmd_down(struct rtw_adapter *padapter, u8 msgbox_num)
-{
-	u8 read_down = false;
-	int	retry_cnts = 100;
-	u8 valid;
-
-	do {
-		valid = rtl8723au_read8(padapter, REG_HMETFR) & BIT(msgbox_num);
-		if (0 == valid)
-			read_down = true;
-	} while ((!read_down) && (retry_cnts--));
-
-	return read_down;
-}
-
-/*****************************************
-* H2C Msg format :
-*| 31 - 8		|7		| 6 - 0	|
-*| h2c_msg	|Ext_bit	|CMD_ID	|
-*
-******************************************/
-int FillH2CCmd(struct rtw_adapter *padapter, u8 ElementID, u32 CmdLen,
-	       u8 *pCmdBuffer)
-{
-	u8 bcmd_down = false;
-	s32 retry_cnts = 100;
-	u8 h2c_box_num;
-	u32 msgbox_addr;
-	u32 msgbox_ex_addr;
-	struct hal_data_8723a *pHalData;
-	u32 h2c_cmd = 0;
-	u16 h2c_cmd_ex = 0;
-	int ret = _FAIL;
-
-	padapter = GET_PRIMARY_ADAPTER(padapter);
-	pHalData = GET_HAL_DATA(padapter);
-
-	mutex_lock(&adapter_to_dvobj(padapter)->h2c_fwcmd_mutex);
-
-	if (!pCmdBuffer)
-		goto exit;
-	if (CmdLen > RTL92C_MAX_CMD_LEN)
-		goto exit;
-	if (padapter->bSurpriseRemoved == true)
-		goto exit;
-
-	/* pay attention to if  race condition happened in  H2C cmd setting. */
-	do {
-		h2c_box_num = pHalData->LastHMEBoxNum;
-
-		if (!_is_fw_read_cmd_down(padapter, h2c_box_num)) {
-			DBG_8723A(" fw read cmd failed...\n");
-			goto exit;
-		}
-
-		if (CmdLen <= 3) {
-			memcpy((u8 *)(&h2c_cmd)+1, pCmdBuffer, CmdLen);
-		} else {
-			memcpy((u8 *)(&h2c_cmd_ex), pCmdBuffer, EX_MESSAGE_BOX_SIZE);
-			memcpy((u8 *)(&h2c_cmd)+1, pCmdBuffer+2, (CmdLen-EX_MESSAGE_BOX_SIZE));
-			*(u8 *)(&h2c_cmd) |= BIT(7);
-		}
-
-		*(u8 *)(&h2c_cmd) |= ElementID;
-
-		if (h2c_cmd & BIT(7)) {
-			msgbox_ex_addr = REG_HMEBOX_EXT_0 + (h2c_box_num * EX_MESSAGE_BOX_SIZE);
-			h2c_cmd_ex = le16_to_cpu(h2c_cmd_ex);
-			rtl8723au_write16(padapter, msgbox_ex_addr, h2c_cmd_ex);
-		}
-		msgbox_addr = REG_HMEBOX_0 + (h2c_box_num * MESSAGE_BOX_SIZE);
-		h2c_cmd = le32_to_cpu(h2c_cmd);
-		rtl8723au_write32(padapter, msgbox_addr, h2c_cmd);
-
-		bcmd_down = true;
-
-		pHalData->LastHMEBoxNum = (h2c_box_num+1) % RTL92C_MAX_H2C_BOX_NUMS;
-
-	} while ((!bcmd_down) && (retry_cnts--));
-
-	ret = _SUCCESS;
-
-exit:
-	mutex_unlock(&adapter_to_dvobj(padapter)->h2c_fwcmd_mutex);
-	return ret;
-}
-
-int rtl8723a_set_rssi_cmd(struct rtw_adapter *padapter, u32 param)
-{
-	__le32 cmd = cpu_to_le32(param);
-
-	FillH2CCmd(padapter, RSSI_SETTING_EID, 3, (void *)&cmd);
-
-	return _SUCCESS;
-}
-
-int rtl8723a_set_raid_cmd(struct rtw_adapter *padapter, u32 mask, u8 arg)
-{
-	u8 buf[5];
-
-	memset(buf, 0, 5);
-	put_unaligned_le32(mask, buf);
-	buf[4]  = arg;
-
-	FillH2CCmd(padapter, MACID_CONFIG_EID, 5, buf);
-
-	return _SUCCESS;
-}
-
-/* bitmap[0:27] = tx_rate_bitmap */
-/* bitmap[28:31]= Rate Adaptive id */
-/* arg[0:4] = macid */
-/* arg[5] = Short GI */
-void rtl8723a_add_rateatid(struct rtw_adapter *pAdapter, u32 bitmap, u8 arg, u8 rssi_level)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter);
-	u8 macid = arg & 0x1f;
-	u32 raid = bitmap & 0xf0000000;
-
-	bitmap &= 0x0fffffff;
-	if (rssi_level != DM_RATR_STA_INIT)
-		bitmap = ODM_Get_Rate_Bitmap23a(pHalData, macid, bitmap,
-						rssi_level);
-
-	bitmap |= raid;
-
-	rtl8723a_set_raid_cmd(pAdapter, bitmap, arg);
-}
-
-void rtl8723a_set_FwPwrMode_cmd(struct rtw_adapter *padapter, u8 Mode)
-{
-	struct setpwrmode_parm H2CSetPwrMode;
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	DBG_8723A("%s: Mode =%d SmartPS =%d UAPSD =%d BcnMode = 0x%02x\n", __func__,
-			Mode, pwrpriv->smart_ps, padapter->registrypriv.uapsd_enable, pwrpriv->bcn_ant_mode);
-
-	/*  Forece leave RF low power mode for 1T1R to
-	    prevent conficting setting in Fw power */
-	/*  saving sequence. 2010.06.07. Added by tynli.
-	    Suggested by SD3 yschang. */
-	if (Mode != PS_MODE_ACTIVE && pHalData->rf_type != RF_2T2R)
-		ODM_RF_Saving23a(&pHalData->odmpriv, true);
-
-	H2CSetPwrMode.Mode = Mode;
-	H2CSetPwrMode.SmartPS = pwrpriv->smart_ps;
-	H2CSetPwrMode.AwakeInterval = 1;
-	H2CSetPwrMode.bAllQueueUAPSD = padapter->registrypriv.uapsd_enable;
-	H2CSetPwrMode.BcnAntMode = pwrpriv->bcn_ant_mode;
-
-	FillH2CCmd(padapter, SET_PWRMODE_EID, sizeof(H2CSetPwrMode), (u8 *)&H2CSetPwrMode);
-
-}
-
-static void
-ConstructBeacon(struct rtw_adapter *padapter, u8 *pframe, u32 *pLength)
-{
-	struct ieee80211_mgmt *mgmt;
-	u32 rate_len, pktlen;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-	u8 bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-
-	/* DBG_8723A("%s\n", __func__); */
-
-	mgmt = (struct ieee80211_mgmt *)pframe;
-
-	mgmt->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_BEACON);
-
-	ether_addr_copy(mgmt->da, bc_addr);
-	ether_addr_copy(mgmt->sa, myid(&padapter->eeprompriv));
-	ether_addr_copy(mgmt->bssid, get_my_bssid23a(cur_network));
-
-	/* A Beacon frame shouldn't have fragment bits set */
-	mgmt->seq_ctrl = 0;
-
-	/* timestamp will be inserted by hardware */
-
-	put_unaligned_le16(cur_network->beacon_interval,
-			   &mgmt->u.beacon.beacon_int);
-
-	put_unaligned_le16(cur_network->capability,
-			   &mgmt->u.beacon.capab_info);
-
-	pframe = mgmt->u.beacon.variable;
-	pktlen = offsetof(struct ieee80211_mgmt, u.beacon.variable);
-
-	if ((pmlmeinfo->state&0x03) == MSR_AP) {
-		/* DBG_8723A("ie len =%d\n", cur_network->IELength); */
-		pktlen += cur_network->IELength;
-		memcpy(pframe, cur_network->IEs, pktlen);
-
-		goto _ConstructBeacon;
-	}
-
-	/* below for ad-hoc mode */
-
-	/*  SSID */
-	pframe = rtw_set_ie23a(pframe, WLAN_EID_SSID,
-			       cur_network->Ssid.ssid_len,
-			       cur_network->Ssid.ssid, &pktlen);
-
-	/*  supported rates... */
-	rate_len = rtw_get_rateset_len23a(cur_network->SupportedRates);
-	pframe = rtw_set_ie23a(pframe, WLAN_EID_SUPP_RATES, ((rate_len > 8) ?
-			       8 : rate_len), cur_network->SupportedRates, &pktlen);
-
-	/*  DS parameter set */
-	pframe = rtw_set_ie23a(pframe, WLAN_EID_DS_PARAMS, 1, (unsigned char *)
-			       &cur_network->DSConfig, &pktlen);
-
-	if ((pmlmeinfo->state&0x03) == MSR_ADHOC) {
-		u32 ATIMWindow;
-		/*  IBSS Parameter Set... */
-		/* ATIMWindow = cur->ATIMWindow; */
-		ATIMWindow = 0;
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_IBSS_PARAMS, 2,
-				       (unsigned char *)&ATIMWindow, &pktlen);
-	}
-
-	/* todo: ERP IE */
-
-	/*  EXTERNDED SUPPORTED RATE */
-	if (rate_len > 8)
-		pframe = rtw_set_ie23a(pframe, WLAN_EID_EXT_SUPP_RATES,
-				       (rate_len - 8),
-				       (cur_network->SupportedRates + 8),
-				       &pktlen);
-
-	/* todo:HT for adhoc */
-
-_ConstructBeacon:
-
-	if ((pktlen + TXDESC_SIZE) > 512) {
-		DBG_8723A("beacon frame too large\n");
-		return;
-	}
-
-	*pLength = pktlen;
-
-	/* DBG_8723A("%s bcn_sz =%d\n", __func__, pktlen); */
-
-}
-
-static void ConstructPSPoll(struct rtw_adapter *padapter,
-			    u8 *pframe, u32 *pLength)
-{
-	struct ieee80211_hdr *pwlanhdr;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	pwlanhdr = (struct ieee80211_hdr *)pframe;
-
-	/*  Frame control. */
-	pwlanhdr->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_PSPOLL);
-	pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
-
-	/*  AID. */
-	pwlanhdr->duration_id = cpu_to_le16(pmlmeinfo->aid | 0xc000);
-
-	/*  BSSID. */
-	memcpy(pwlanhdr->addr1, get_my_bssid23a(&pmlmeinfo->network), ETH_ALEN);
-
-	/*  TA. */
-	memcpy(pwlanhdr->addr2, myid(&padapter->eeprompriv), ETH_ALEN);
-
-	*pLength = 16;
-}
-
-static void
-ConstructNullFunctionData(struct rtw_adapter *padapter, u8 *pframe,
-			  u32 *pLength, u8 *StaAddr, u8 bQoS, u8 AC,
-			  u8 bEosp, u8 bForcePowerSave)
-{
-	struct ieee80211_hdr *pwlanhdr;
-	u32 pktlen;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	pwlanhdr = (struct ieee80211_hdr *)pframe;
-
-	pwlanhdr->frame_control = 0;
-	pwlanhdr->seq_ctrl = 0;
-
-	if (bForcePowerSave)
-		pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
-
-	switch (cur_network->network.ifmode) {
-	case NL80211_IFTYPE_P2P_CLIENT:
-	case NL80211_IFTYPE_STATION:
-		pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_TODS);
-		memcpy(pwlanhdr->addr1,
-		       get_my_bssid23a(&pmlmeinfo->network), ETH_ALEN);
-		memcpy(pwlanhdr->addr2, myid(&padapter->eeprompriv),
-		       ETH_ALEN);
-		memcpy(pwlanhdr->addr3, StaAddr, ETH_ALEN);
-		break;
-	case NL80211_IFTYPE_P2P_GO:
-	case NL80211_IFTYPE_AP:
-		pwlanhdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_FROMDS);
-		memcpy(pwlanhdr->addr1, StaAddr, ETH_ALEN);
-		memcpy(pwlanhdr->addr2,
-		       get_my_bssid23a(&pmlmeinfo->network), ETH_ALEN);
-		memcpy(pwlanhdr->addr3, myid(&padapter->eeprompriv),
-		       ETH_ALEN);
-		break;
-	case NL80211_IFTYPE_ADHOC:
-	default:
-		memcpy(pwlanhdr->addr1, StaAddr, ETH_ALEN);
-		memcpy(pwlanhdr->addr2, myid(&padapter->eeprompriv), ETH_ALEN);
-		memcpy(pwlanhdr->addr3,
-		       get_my_bssid23a(&pmlmeinfo->network), ETH_ALEN);
-		break;
-	}
-
-	if (bQoS == true) {
-		struct ieee80211_qos_hdr *qoshdr;
-		qoshdr = (struct ieee80211_qos_hdr *)pframe;
-
-		qoshdr->frame_control |=
-			cpu_to_le16(IEEE80211_FTYPE_DATA |
-				    IEEE80211_STYPE_QOS_NULLFUNC);
-
-		qoshdr->qos_ctrl = cpu_to_le16(AC & IEEE80211_QOS_CTL_TID_MASK);
-		if (bEosp)
-			qoshdr->qos_ctrl |= cpu_to_le16(IEEE80211_QOS_CTL_EOSP);
-
-		pktlen = sizeof(struct ieee80211_qos_hdr);
-	} else {
-		pwlanhdr->frame_control |=
-			cpu_to_le16(IEEE80211_FTYPE_DATA |
-				    IEEE80211_STYPE_NULLFUNC);
-
-		pktlen = sizeof(struct ieee80211_hdr_3addr);
-	}
-
-	*pLength = pktlen;
-}
-
-static void ConstructProbeRsp(struct rtw_adapter *padapter, u8 *pframe,
-			      u32 *pLength, u8 *StaAddr, bool bHideSSID)
-{
-	struct ieee80211_mgmt *mgmt;
-	u8 *mac, *bssid;
-	u32 pktlen;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-
-	/* DBG_8723A("%s\n", __func__); */
-
-	mgmt = (struct ieee80211_mgmt *)pframe;
-
-	mac = myid(&padapter->eeprompriv);
-	bssid = cur_network->MacAddress;
-
-	mgmt->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_PROBE_RESP);
-
-	mgmt->seq_ctrl = 0;
-
-	memcpy(mgmt->da, StaAddr, ETH_ALEN);
-	memcpy(mgmt->sa, mac, ETH_ALEN);
-	memcpy(mgmt->bssid, bssid, ETH_ALEN);
-
-	put_unaligned_le64(cur_network->tsf,
-			   &mgmt->u.probe_resp.timestamp);
-	put_unaligned_le16(cur_network->beacon_interval,
-			   &mgmt->u.probe_resp.beacon_int);
-	put_unaligned_le16(cur_network->capability,
-			   &mgmt->u.probe_resp.capab_info);
-
-	pktlen = offsetof(struct ieee80211_mgmt, u.probe_resp.variable);
-
-	if (cur_network->IELength > MAX_IE_SZ)
-		return;
-
-	memcpy(mgmt->u.probe_resp.variable, cur_network->IEs,
-	       cur_network->IELength);
-	pktlen += (cur_network->IELength);
-
-	*pLength = pktlen;
-}
-
-/*  */
-/*  Description: Fill the reserved packets that FW will use to RSVD page. */
-/*			Now we just send 4 types packet to rsvd page. */
-/*			(1)Beacon, (2)Ps-poll, (3)Null data, (4)ProbeRsp. */
-/*	Input: */
-/*	    bDLFinished - false: At the first time we will send all the packets as a large packet to Hw, */
-/*						so we need to set the packet length to total lengh. */
-/*			      true: At the second time, we should send the first packet (default:beacon) */
-/*						to Hw again and set the lengh in descriptor to the real beacon lengh. */
-/*  2009.10.15 by tynli. */
-static void SetFwRsvdPagePkt(struct rtw_adapter *padapter, bool bDLFinished)
-{
-	struct hal_data_8723a *pHalData;
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	struct xmit_priv *pxmitpriv;
-	struct mlme_ext_priv *pmlmeext;
-	struct mlme_ext_info *pmlmeinfo;
-	u32 BeaconLength = 0, ProbeRspLength = 0, PSPollLength;
-	u32 NullDataLength, QosNullLength, BTQosNullLength;
-	u8 *ReservedPagePacket;
-	u8 PageNum, PageNeed, TxDescLen;
-	u16 BufIndex;
-	u32 TotalPacketLen;
-	struct rsvdpage_loc	RsvdPageLoc;
-
-	DBG_8723A("%s\n", __func__);
-
-	ReservedPagePacket = kzalloc(1000, GFP_KERNEL);
-	if (ReservedPagePacket == NULL) {
-		DBG_8723A("%s: alloc ReservedPagePacket fail!\n", __func__);
-		return;
-	}
-
-	pHalData = GET_HAL_DATA(padapter);
-	pxmitpriv = &padapter->xmitpriv;
-	pmlmeext = &padapter->mlmeextpriv;
-	pmlmeinfo = &pmlmeext->mlmext_info;
-
-	TxDescLen = TXDESC_SIZE;
-	PageNum = 0;
-
-	/* 3 (1) beacon */
-	BufIndex = TXDESC_OFFSET;
-	ConstructBeacon(padapter, &ReservedPagePacket[BufIndex], &BeaconLength);
-
-	/*  When we count the first page size, we need to reserve description size for the RSVD */
-	/*  packet, it will be filled in front of the packet in TXPKTBUF. */
-	PageNeed = (u8)PageNum_128(TxDescLen + BeaconLength);
-	/*  To reserved 2 pages for beacon buffer. 2010.06.24. */
-	if (PageNeed == 1)
-		PageNeed += 1;
-	PageNum += PageNeed;
-	pHalData->FwRsvdPageStartOffset = PageNum;
-
-	BufIndex += PageNeed*128;
-
-	/* 3 (2) ps-poll */
-	RsvdPageLoc.LocPsPoll = PageNum;
-	ConstructPSPoll(padapter, &ReservedPagePacket[BufIndex], &PSPollLength);
-	rtl8723a_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], PSPollLength, true, false);
-
-	PageNeed = (u8)PageNum_128(TxDescLen + PSPollLength);
-	PageNum += PageNeed;
-
-	BufIndex += PageNeed*128;
-
-	/* 3 (3) null data */
-	RsvdPageLoc.LocNullData = PageNum;
-	ConstructNullFunctionData(padapter, &ReservedPagePacket[BufIndex],
-				  &NullDataLength,
-				  get_my_bssid23a(&pmlmeinfo->network),
-				  false, 0, 0, false);
-	rtl8723a_fill_fake_txdesc(padapter,
-				  &ReservedPagePacket[BufIndex-TxDescLen],
-				  NullDataLength, false, false);
-
-	PageNeed = (u8)PageNum_128(TxDescLen + NullDataLength);
-	PageNum += PageNeed;
-
-	BufIndex += PageNeed*128;
-
-	/* 3 (4) probe response */
-	RsvdPageLoc.LocProbeRsp = PageNum;
-	ConstructProbeRsp(
-		padapter,
-		&ReservedPagePacket[BufIndex],
-		&ProbeRspLength,
-		get_my_bssid23a(&pmlmeinfo->network),
-		false);
-	rtl8723a_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], ProbeRspLength, false, false);
-
-	PageNeed = (u8)PageNum_128(TxDescLen + ProbeRspLength);
-	PageNum += PageNeed;
-
-	BufIndex += PageNeed*128;
-
-	/* 3 (5) Qos null data */
-	RsvdPageLoc.LocQosNull = PageNum;
-	ConstructNullFunctionData(
-		padapter,
-		&ReservedPagePacket[BufIndex],
-		&QosNullLength,
-		get_my_bssid23a(&pmlmeinfo->network),
-		true, 0, 0, false);
-	rtl8723a_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], QosNullLength, false, false);
-
-	PageNeed = (u8)PageNum_128(TxDescLen + QosNullLength);
-	PageNum += PageNeed;
-
-	BufIndex += PageNeed*128;
-
-	/* 3 (6) BT Qos null data */
-	RsvdPageLoc.LocBTQosNull = PageNum;
-	ConstructNullFunctionData(
-		padapter,
-		&ReservedPagePacket[BufIndex],
-		&BTQosNullLength,
-		get_my_bssid23a(&pmlmeinfo->network),
-		true, 0, 0, false);
-	rtl8723a_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], BTQosNullLength, false, true);
-
-	TotalPacketLen = BufIndex + BTQosNullLength;
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (pmgntframe == NULL)
-		goto exit;
-
-	/*  update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-	pattrib->qsel = 0x10;
-	pattrib->pktlen = pattrib->last_txcmdsz = TotalPacketLen - TXDESC_OFFSET;
-	memcpy(pmgntframe->buf_addr, ReservedPagePacket, TotalPacketLen);
-
-	rtl8723au_mgnt_xmit(padapter, pmgntframe);
-
-	DBG_8723A("%s: Set RSVD page location to Fw\n", __func__);
-	FillH2CCmd(padapter, RSVD_PAGE_EID, sizeof(RsvdPageLoc), (u8 *)&RsvdPageLoc);
-
-exit:
-	kfree(ReservedPagePacket);
-}
-
-void rtl8723a_set_FwJoinBssReport_cmd(struct rtw_adapter *padapter, u8 mstatus)
-{
-	struct joinbssrpt_parm	JoinBssRptParm;
-	struct hal_data_8723a	*pHalData = GET_HAL_DATA(padapter);
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	DBG_8723A("%s mstatus(%x)\n", __func__, mstatus);
-
-	if (mstatus == 1) {
-		bool bRecover = false;
-		u8 v8;
-
-		/*  We should set AID, correct TSF, HW seq enable before set JoinBssReport to Fw in 88/92C. */
-		/*  Suggested by filen. Added by tynli. */
-		rtl8723au_write16(padapter, REG_BCN_PSR_RPT,
-				  0xC000|pmlmeinfo->aid);
-		/*  Do not set TSF again here or vWiFi beacon DMA INT will not work. */
-		/* correct_TSF23a(padapter, pmlmeext); */
-		/*  Hw sequende enable by dedault. 2010.06.23. by tynli. */
-		/* rtl8723au_write16(padapter, REG_NQOS_SEQ, ((pmlmeext->mgnt_seq+100)&0xFFF)); */
-		/* rtl8723au_write8(padapter, REG_HWSEQ_CTRL, 0xFF); */
-
-		/*  set REG_CR bit 8 */
-		v8 = rtl8723au_read8(padapter, REG_CR+1);
-		v8 |= BIT(0); /*  ENSWBCN */
-		rtl8723au_write8(padapter,  REG_CR+1, v8);
-
-		/*  Disable Hw protection for a time which revserd for Hw sending beacon. */
-		/*  Fix download reserved page packet fail that access collision with the protection time. */
-		/*  2010.05.11. Added by tynli. */
-/*			SetBcnCtrlReg23a(padapter, 0, BIT(3)); */
-/*			SetBcnCtrlReg23a(padapter, BIT(4), 0); */
-		SetBcnCtrlReg23a(padapter, BIT(4), BIT(3));
-
-		/*  Set FWHW_TXQ_CTRL 0x422[6]= 0 to tell Hw the packet is not a real beacon frame. */
-		if (pHalData->RegFwHwTxQCtrl & BIT(6))
-			bRecover = true;
-
-		/*  To tell Hw the packet is not a real beacon frame. */
-		/* U1bTmp = rtl8723au_read8(padapter, REG_FWHW_TXQ_CTRL+2); */
-		rtl8723au_write8(padapter, REG_FWHW_TXQ_CTRL + 2,
-				 pHalData->RegFwHwTxQCtrl & ~BIT(6));
-		pHalData->RegFwHwTxQCtrl &= ~BIT(6);
-		SetFwRsvdPagePkt(padapter, 0);
-
-		/*  2010.05.11. Added by tynli. */
-		SetBcnCtrlReg23a(padapter, BIT(3), BIT(4));
-
-		/*  To make sure that if there exists an adapter which would like to send beacon. */
-		/*  If exists, the origianl value of 0x422[6] will be 1, we should check this to */
-		/*  prevent from setting 0x422[6] to 0 after download reserved page, or it will cause */
-		/*  the beacon cannot be sent by HW. */
-		/*  2010.06.23. Added by tynli. */
-		if (bRecover) {
-			rtl8723au_write8(padapter, REG_FWHW_TXQ_CTRL + 2,
-					 pHalData->RegFwHwTxQCtrl | BIT(6));
-			pHalData->RegFwHwTxQCtrl |= BIT(6);
-		}
-
-		/*  Clear CR[8] or beacon packet will not be send to TxBuf anymore. */
-		v8 = rtl8723au_read8(padapter, REG_CR+1);
-		v8 &= ~BIT(0); /*  ~ENSWBCN */
-		rtl8723au_write8(padapter, REG_CR+1, v8);
-	}
-
-	JoinBssRptParm.OpMode = mstatus;
-
-	FillH2CCmd(padapter, JOINBSS_RPT_EID, sizeof(JoinBssRptParm), (u8 *)&JoinBssRptParm);
-
-}
-
-#ifdef CONFIG_8723AU_BT_COEXIST
-static void SetFwRsvdPagePkt_BTCoex(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	struct xmit_priv *pxmitpriv;
-	struct mlme_ext_priv *pmlmeext;
-	struct mlme_ext_info *pmlmeinfo;
-	u8 fakemac[6] = {0x00, 0xe0, 0x4c, 0x00, 0x00, 0x00};
-	u32 NullDataLength, BTQosNullLength;
-	u8 *ReservedPagePacket;
-	u8 PageNum, PageNeed, TxDescLen;
-	u16 BufIndex;
-	u32 TotalPacketLen;
-	struct rsvdpage_loc	RsvdPageLoc;
-
-	DBG_8723A("+%s\n", __func__);
-
-	ReservedPagePacket = kzalloc(1024, GFP_KERNEL);
-	if (ReservedPagePacket == NULL) {
-		DBG_8723A("%s: alloc ReservedPagePacket fail!\n", __func__);
-		return;
-	}
-
-	pHalData = GET_HAL_DATA(padapter);
-	pxmitpriv = &padapter->xmitpriv;
-	pmlmeext = &padapter->mlmeextpriv;
-	pmlmeinfo = &pmlmeext->mlmext_info;
-
-	TxDescLen = TXDESC_SIZE;
-	PageNum = 0;
-
-	/* 3 (1) beacon */
-	BufIndex = TXDESC_OFFSET;
-	/*  skip Beacon Packet */
-	PageNeed = 3;
-
-	PageNum += PageNeed;
-	pHalData->FwRsvdPageStartOffset = PageNum;
-
-	BufIndex += PageNeed*128;
-
-	/* 3 (3) null data */
-	RsvdPageLoc.LocNullData = PageNum;
-	ConstructNullFunctionData(
-		padapter,
-		&ReservedPagePacket[BufIndex],
-		&NullDataLength,
-		fakemac,
-		false, 0, 0, false);
-	rtl8723a_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], NullDataLength, false, false);
-
-	PageNeed = (u8)PageNum_128(TxDescLen + NullDataLength);
-	PageNum += PageNeed;
-
-	BufIndex += PageNeed*128;
-
-	/* 3 (6) BT Qos null data */
-	RsvdPageLoc.LocBTQosNull = PageNum;
-	ConstructNullFunctionData(
-		padapter,
-		&ReservedPagePacket[BufIndex],
-		&BTQosNullLength,
-		fakemac,
-		true, 0, 0, false);
-	rtl8723a_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], BTQosNullLength, false, true);
-
-	TotalPacketLen = BufIndex + BTQosNullLength;
-
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (pmgntframe == NULL)
-		goto exit;
-
-	/*  update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-	pattrib->qsel = 0x10;
-	pattrib->pktlen = pattrib->last_txcmdsz = TotalPacketLen - TXDESC_OFFSET;
-	memcpy(pmgntframe->buf_addr, ReservedPagePacket, TotalPacketLen);
-
-	rtl8723au_mgnt_xmit(padapter, pmgntframe);
-
-	DBG_8723A("%s: Set RSVD page location to Fw\n", __func__);
-	FillH2CCmd(padapter, RSVD_PAGE_EID, sizeof(RsvdPageLoc), (u8 *)&RsvdPageLoc);
-
-exit:
-	kfree(ReservedPagePacket);
-}
-
-void rtl8723a_set_BTCoex_AP_mode_FwRsvdPkt_cmd(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	u8 bRecover = false;
-
-	DBG_8723A("+%s\n", __func__);
-
-	pHalData = GET_HAL_DATA(padapter);
-
-	/*  Set FWHW_TXQ_CTRL 0x422[6]= 0 to tell Hw the packet is not a real beacon frame. */
-	if (pHalData->RegFwHwTxQCtrl & BIT(6))
-		bRecover = true;
-
-	/*  To tell Hw the packet is not a real beacon frame. */
-	pHalData->RegFwHwTxQCtrl &= ~BIT(6);
-	rtl8723au_write8(padapter, REG_FWHW_TXQ_CTRL + 2,
-			 pHalData->RegFwHwTxQCtrl);
-	SetFwRsvdPagePkt_BTCoex(padapter);
-
-	/*  To make sure that if there exists an adapter which would like to send beacon. */
-	/*  If exists, the origianl value of 0x422[6] will be 1, we should check this to */
-	/*  prevent from setting 0x422[6] to 0 after download reserved page, or it will cause */
-	/*  the beacon cannot be sent by HW. */
-	/*  2010.06.23. Added by tynli. */
-	if (bRecover) {
-		pHalData->RegFwHwTxQCtrl |= BIT(6);
-		rtl8723au_write8(padapter, REG_FWHW_TXQ_CTRL + 2,
-				 pHalData->RegFwHwTxQCtrl);
-	}
-}
-#endif
diff --git a/drivers/staging/rtl8723au/hal/rtl8723a_dm.c b/drivers/staging/rtl8723au/hal/rtl8723a_dm.c
deleted file mode 100644
index 1e831f2d1caf..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723a_dm.c
+++ /dev/null
@@ -1,194 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-/*  */
-/*  Description: */
-/*  */
-/*  This file is for 92CE/92CU dynamic mechanism only */
-/*  */
-/*  */
-/*  */
-#define _RTL8723A_DM_C_
-
-/*  */
-/*  include files */
-/*  */
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#include <rtl8723a_hal.h>
-#include <usb_ops_linux.h>
-
-/*  */
-/*  Global var */
-/*  */
-
-static void dm_CheckPbcGPIO(struct rtw_adapter *padapter)
-{
-	u8	tmp1byte;
-	u8	bPbcPressed = false;
-
-	if (!padapter->registrypriv.hw_wps_pbc)
-		return;
-
-	tmp1byte = rtl8723au_read8(padapter, GPIO_IO_SEL);
-	tmp1byte |= (HAL_8192C_HW_GPIO_WPS_BIT);
-	/* enable GPIO[2] as output mode */
-	rtl8723au_write8(padapter, GPIO_IO_SEL, tmp1byte);
-
-	tmp1byte &= ~(HAL_8192C_HW_GPIO_WPS_BIT);
-	/* reset the floating voltage level */
-	rtl8723au_write8(padapter,  GPIO_IN, tmp1byte);
-
-	tmp1byte = rtl8723au_read8(padapter, GPIO_IO_SEL);
-	tmp1byte &= ~(HAL_8192C_HW_GPIO_WPS_BIT);
-	/* enable GPIO[2] as input mode */
-	rtl8723au_write8(padapter, GPIO_IO_SEL, tmp1byte);
-
-	tmp1byte = rtl8723au_read8(padapter, GPIO_IN);
-
-	if (tmp1byte == 0xff)
-		return;
-
-	if (tmp1byte&HAL_8192C_HW_GPIO_WPS_BIT)
-		bPbcPressed = true;
-
-	if (bPbcPressed) {
-		/*  Here we only set bPbcPressed to true */
-		/*  After trigger PBC, the variable will be set to false */
-		DBG_8723A("CheckPbcGPIO - PBC is pressed\n");
-
-		if (padapter->pid[0] == 0) {
-			/* 0 is the default value and it means the application
-			 * monitors the HW PBC doesn't privde its pid to driver.
-			 */
-			return;
-		}
-
-		kill_pid(find_vpid(padapter->pid[0]), SIGUSR1, 1);
-	}
-}
-
-/*  Initialize GPIO setting registers */
-/*  functions */
-
-void rtl8723a_init_dm_priv(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv	*pdmpriv = &pHalData->dmpriv;
-	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
-	u8 cut_ver, fab_ver;
-
-	memset(pdmpriv, 0, sizeof(struct dm_priv));
-	memset(pDM_Odm, 0, sizeof(*pDM_Odm));
-
-	pDM_Odm->Adapter = Adapter;
-
-	ODM_CmnInfoInit23a(pDM_Odm, ODM_CMNINFO_IC_TYPE, ODM_RTL8723A);
-
-	if (IS_8723A_A_CUT(pHalData->VersionID)) {
-		fab_ver = ODM_UMC;
-		cut_ver = ODM_CUT_A;
-	} else if (IS_8723A_B_CUT(pHalData->VersionID)) {
-		fab_ver = ODM_UMC;
-		cut_ver = ODM_CUT_B;
-	} else {
-		fab_ver = ODM_TSMC;
-		cut_ver = ODM_CUT_A;
-	}
-	ODM_CmnInfoInit23a(pDM_Odm, ODM_CMNINFO_FAB_VER, fab_ver);
-	ODM_CmnInfoInit23a(pDM_Odm, ODM_CMNINFO_CUT_VER, cut_ver);
-	ODM_CmnInfoInit23a(pDM_Odm, ODM_CMNINFO_MP_TEST_CHIP, IS_NORMAL_CHIP(pHalData->VersionID));
-
-	ODM_CmnInfoInit23a(pDM_Odm, ODM_CMNINFO_BOARD_TYPE, pHalData->BoardType);
-
-	if (pHalData->BoardType == BOARD_USB_High_PA) {
-		ODM_CmnInfoInit23a(pDM_Odm, ODM_CMNINFO_EXT_LNA, true);
-		ODM_CmnInfoInit23a(pDM_Odm, ODM_CMNINFO_EXT_PA, true);
-	}
-	ODM_CmnInfoInit23a(pDM_Odm, ODM_CMNINFO_BWIFI_TEST, Adapter->registrypriv.wifi_spec);
-}
-
-static void Update_ODM_ComInfo_8723a(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
-	struct dm_priv	*pdmpriv = &pHalData->dmpriv;
-	int i;
-	pdmpriv->InitODMFlag = 0;
-	/*  Pointer reference */
-	rtl8723a_odm_support_ability_set(Adapter, DYNAMIC_ALL_FUNC_ENABLE);
-
-	for (i = 0; i < NUM_STA; i++)
-		ODM_CmnInfoPtrArrayHook23a(pDM_Odm, ODM_CMNINFO_STA_STATUS, i, NULL);
-}
-
-void rtl8723a_InitHalDm(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv	*pdmpriv = &pHalData->dmpriv;
-	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
-	u8	i;
-
-	Update_ODM_ComInfo_8723a(Adapter);
-	ODM23a_DMInit(pDM_Odm);
-	/*  Save REG_INIDATA_RATE_SEL value for TXDESC. */
-	for (i = 0; i < 32; i++)
-		pdmpriv->INIDATA_RATE[i] = rtl8723au_read8(Adapter, REG_INIDATA_RATE_SEL+i) & 0x3f;
-}
-
-void
-rtl8723a_HalDmWatchDog(
-	struct rtw_adapter *Adapter
-	)
-{
-	bool		bFwCurrentInPSMode = false;
-	bool		bFwPSAwake = true;
-	u8 bLinked = false;
-	u8 hw_init_completed = false;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv	*pdmpriv = &pHalData->dmpriv;
-
-	hw_init_completed = Adapter->hw_init_completed;
-
-	if (hw_init_completed == false)
-		goto skip_dm;
-
-	bFwCurrentInPSMode = Adapter->pwrctrlpriv.bFwCurrentInPSMode;
-	bFwPSAwake = rtl8723a_get_fwlps_rf_on(Adapter);
-
-	if (!bFwCurrentInPSMode && bFwPSAwake) {
-		/*  Read REG_INIDATA_RATE_SEL value for TXDESC. */
-		if (check_fwstate(&Adapter->mlmepriv, WIFI_STATION_STATE)) {
-			pdmpriv->INIDATA_RATE[0] = rtl8723au_read8(Adapter, REG_INIDATA_RATE_SEL) & 0x3f;
-		} else {
-			u8	i;
-			for (i = 1 ; i < (Adapter->stapriv.asoc_sta_count + 1); i++)
-				pdmpriv->INIDATA_RATE[i] = rtl8723au_read8(Adapter, (REG_INIDATA_RATE_SEL+i)) & 0x3f;
-		}
-	}
-
-	/* ODM */
-	if (rtw_linked_check(Adapter))
-		bLinked = true;
-
-	ODM_CmnInfoUpdate23a(&pHalData->odmpriv, ODM_CMNINFO_LINK, bLinked);
-	ODM_DMWatchdog23a(Adapter);
-
-skip_dm:
-
-	/*  Check GPIO to determine current RF on/off and Pbc status. */
-	/*  Check Hardware Radio ON/OFF or not */
-	dm_CheckPbcGPIO(Adapter);
-}
diff --git a/drivers/staging/rtl8723au/hal/rtl8723a_hal_init.c b/drivers/staging/rtl8723au/hal/rtl8723a_hal_init.c
deleted file mode 100644
index 1ea0af499ce9..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723a_hal_init.c
+++ /dev/null
@@ -1,2076 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _HAL_INIT_C_
-
-#include <linux/firmware.h>
-#include <drv_types.h>
-#include <rtw_efuse.h>
-
-#include <rtl8723a_hal.h>
-#include <usb_ops_linux.h>
-
-static void _FWDownloadEnable(struct rtw_adapter *padapter, bool enable)
-{
-	u8 tmp;
-
-	if (enable) {
-		/*  8051 enable */
-		tmp = rtl8723au_read8(padapter, REG_SYS_FUNC_EN + 1);
-		rtl8723au_write8(padapter, REG_SYS_FUNC_EN + 1, tmp | 0x04);
-
-		/*  MCU firmware download enable. */
-		tmp = rtl8723au_read8(padapter, REG_MCUFWDL);
-		rtl8723au_write8(padapter, REG_MCUFWDL, tmp | 0x01);
-
-		/*  8051 reset */
-		tmp = rtl8723au_read8(padapter, REG_MCUFWDL + 2);
-		rtl8723au_write8(padapter, REG_MCUFWDL + 2, tmp & 0xf7);
-	} else {
-		/*  MCU firmware download disable. */
-		tmp = rtl8723au_read8(padapter, REG_MCUFWDL);
-		rtl8723au_write8(padapter, REG_MCUFWDL, tmp & 0xfe);
-
-		/*  Reserved for fw extension. */
-		rtl8723au_write8(padapter, REG_MCUFWDL + 1, 0x00);
-	}
-}
-
-static int
-_PageWrite(struct rtw_adapter *padapter, u32 page, void *buffer, u32 size)
-{
-	u8 value8;
-	u8 u8Page = (u8) (page & 0x07);
-
-	if (size > MAX_PAGE_SIZE)
-		return _FAIL;
-
-	value8 = (rtl8723au_read8(padapter, REG_MCUFWDL + 2) & 0xF8) | u8Page;
-	rtl8723au_write8(padapter, REG_MCUFWDL + 2, value8);
-
-	return rtl8723au_writeN(padapter, FW_8723A_START_ADDRESS, size, buffer);
-}
-
-static int _WriteFW(struct rtw_adapter *padapter, void *buffer, u32 size)
-{
-	/*  Since we need dynamic decide method of dwonload fw, so we
-	    call this function to get chip version. */
-	/*  We can remove _ReadChipVersion from ReadpadapterInfo8192C later. */
-	int ret = _SUCCESS;
-	u32 pageNums, remainSize;
-	u32 page, offset;
-	u8 *bufferPtr = (u8 *) buffer;
-
-	pageNums = size / MAX_PAGE_SIZE;
-	/* RT_ASSERT((pageNums <= 4),
-	   ("Page numbers should not greater then 4 \n")); */
-	remainSize = size % MAX_PAGE_SIZE;
-
-	for (page = 0; page < pageNums; page++) {
-		offset = page * MAX_PAGE_SIZE;
-		ret = _PageWrite(padapter, page, bufferPtr + offset,
-				 MAX_PAGE_SIZE);
-
-		if (ret == _FAIL)
-			goto exit;
-	}
-	if (remainSize) {
-		offset = pageNums * MAX_PAGE_SIZE;
-		page = pageNums;
-		ret = _PageWrite(padapter, page, bufferPtr + offset,
-				 remainSize);
-
-		if (ret == _FAIL)
-			goto exit;
-	}
-	RT_TRACE(_module_hal_init_c_, _drv_info_,
-		 "_WriteFW Done- for Normal chip.\n");
-
-exit:
-	return ret;
-}
-
-static int _FWFreeToGo(struct rtw_adapter *padapter)
-{
-	u32 counter = 0;
-	u32 value32;
-
-	/*  polling CheckSum report */
-	do {
-		value32 = rtl8723au_read32(padapter, REG_MCUFWDL);
-		if (value32 & FWDL_ChkSum_rpt)
-			break;
-	} while (counter++ < POLLING_READY_TIMEOUT_COUNT);
-
-	if (counter >= POLLING_READY_TIMEOUT_COUNT) {
-		RT_TRACE(_module_hal_init_c_, _drv_err_,
-			 "%s: chksum report fail! REG_MCUFWDL:0x%08x\n",
-			 __func__, value32);
-		return _FAIL;
-	}
-	RT_TRACE(_module_hal_init_c_, _drv_info_,
-		 "%s: Checksum report OK! REG_MCUFWDL:0x%08x\n", __func__,
-		 value32);
-
-	value32 = rtl8723au_read32(padapter, REG_MCUFWDL);
-	value32 |= MCUFWDL_RDY;
-	value32 &= ~WINTINI_RDY;
-	rtl8723au_write32(padapter, REG_MCUFWDL, value32);
-
-	/*  polling for FW ready */
-	counter = 0;
-	do {
-		value32 = rtl8723au_read32(padapter, REG_MCUFWDL);
-		if (value32 & WINTINI_RDY) {
-			RT_TRACE(_module_hal_init_c_, _drv_info_,
-				 "%s: Polling FW ready success!! REG_MCUFWDL:0x%08x\n",
-				 __func__, value32);
-			return _SUCCESS;
-		}
-		udelay(5);
-	} while (counter++ < POLLING_READY_TIMEOUT_COUNT);
-
-	RT_TRACE(_module_hal_init_c_, _drv_err_,
-		 "%s: Polling FW ready fail!! REG_MCUFWDL:0x%08x\n",
-		 __func__, value32);
-	return _FAIL;
-}
-
-#define IS_FW_81xxC(padapter)	(((GET_HAL_DATA(padapter))->FirmwareSignature & 0xFFF0) == 0x88C0)
-
-void rtl8723a_FirmwareSelfReset(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 u1bTmp;
-	u8 Delay = 100;
-
-	if (!(IS_FW_81xxC(padapter) &&
-	      ((pHalData->FirmwareVersion < 0x21) ||
-	       (pHalData->FirmwareVersion == 0x21 &&
-		pHalData->FirmwareSubVersion < 0x01)))) {
-		/*  after 88C Fw v33.1 */
-		/* 0x1cf = 0x20. Inform 8051 to reset. 2009.12.25. tynli_test */
-		rtl8723au_write8(padapter, REG_HMETFR + 3, 0x20);
-
-		u1bTmp = rtl8723au_read8(padapter, REG_SYS_FUNC_EN + 1);
-		while (u1bTmp & BIT(2)) {
-			Delay--;
-			if (Delay == 0)
-				break;
-			udelay(50);
-			u1bTmp = rtl8723au_read8(padapter, REG_SYS_FUNC_EN + 1);
-		}
-		RT_TRACE(_module_hal_init_c_, _drv_info_,
-			 "-%s: 8051 reset success (%d)\n", __func__,
-			 Delay);
-
-		if ((Delay == 0)) {
-			/* force firmware reset */
-			u1bTmp = rtl8723au_read8(padapter, REG_SYS_FUNC_EN + 1);
-			rtl8723au_write8(padapter, REG_SYS_FUNC_EN + 1,
-					 u1bTmp & ~BIT(2));
-		}
-	}
-}
-
-/*  */
-/*	Description: */
-/*		Download 8192C firmware code. */
-/*  */
-/*  */
-int rtl8723a_FirmwareDownload(struct rtw_adapter *padapter)
-{
-	int rtStatus = _SUCCESS;
-	u8 writeFW_retry = 0;
-	unsigned long fwdl_start_time;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
-	struct device *device = dvobj_to_dev(dvobj);
-	struct rt_8723a_firmware_hdr *pFwHdr = NULL;
-	const struct firmware *fw;
-	char *fw_name;
-	u8 *firmware_buf = NULL;
-	u8 *buf;
-	int fw_size;
-	static int log_version;
-
-	RT_TRACE(_module_hal_init_c_, _drv_info_, "+%s\n", __func__);
-
-	if (IS_8723A_A_CUT(pHalData->VersionID)) {
-		fw_name = "rtlwifi/rtl8723aufw_A.bin";
-		RT_TRACE(_module_hal_init_c_, _drv_info_,
-			 "rtl8723a_FirmwareDownload: R8723FwImageArray_UMC for RTL8723A A CUT\n");
-	} else if (IS_8723A_B_CUT(pHalData->VersionID)) {
-		/*  WLAN Fw. */
-		if (padapter->registrypriv.wifi_spec == 1) {
-			fw_name = "rtlwifi/rtl8723aufw_B_NoBT.bin";
-			DBG_8723A(" Rtl8723_FwUMCBCutImageArrayWithoutBT for "
-				  "RTL8723A B CUT\n");
-		} else {
-			if (rtl8723a_BT_coexist(padapter)) {
-				fw_name = "rtlwifi/rtl8723aufw_B.bin";
-				DBG_8723A(" Rtl8723_FwUMCBCutImageArrayWithBT "
-					  "for RTL8723A B CUT\n");
-			} else {
-				fw_name = "rtlwifi/rtl8723aufw_B_NoBT.bin";
-				DBG_8723A(" Rtl8723_FwUMCBCutImageArrayWithout "
-					  "BT for RTL8723A B CUT\n");
-			}
-		}
-	} else {
-		/*  <Roger_TODO> We should download proper RAM Code here
-		    to match the ROM code. */
-		RT_TRACE(_module_hal_init_c_, _drv_err_,
-			 "%s: unknown version!\n", __func__);
-		rtStatus = _FAIL;
-		goto Exit;
-	}
-
-	pr_info("rtl8723au: Loading firmware %s\n", fw_name);
-	if (request_firmware(&fw, fw_name, device)) {
-		pr_err("rtl8723au: request_firmware load failed\n");
-		rtStatus = _FAIL;
-		goto Exit;
-	}
-	if (!fw) {
-		pr_err("rtl8723au: Firmware %s not available\n", fw_name);
-		rtStatus = _FAIL;
-		goto Exit;
-	}
-	firmware_buf = kmemdup(fw->data, fw->size, GFP_KERNEL);
-	fw_size = fw->size;
-	release_firmware(fw);
-	if (!firmware_buf) {
-		rtStatus = _FAIL;
-		goto Exit;
-	}
-	buf = firmware_buf;
-
-	/*  To Check Fw header. Added by tynli. 2009.12.04. */
-	pFwHdr = (struct rt_8723a_firmware_hdr *)firmware_buf;
-
-	pHalData->FirmwareVersion = le16_to_cpu(pFwHdr->Version);
-	pHalData->FirmwareSubVersion = pFwHdr->Subversion;
-	pHalData->FirmwareSignature = le16_to_cpu(pFwHdr->Signature);
-
-	DBG_8723A("%s: fw_ver =%d fw_subver =%d sig = 0x%x\n",
-		  __func__, pHalData->FirmwareVersion,
-		  pHalData->FirmwareSubVersion, pHalData->FirmwareSignature);
-
-	if (!log_version++)
-		pr_info("%sFirmware Version %d, SubVersion %d, Signature "
-			"0x%x\n", DRIVER_PREFIX, pHalData->FirmwareVersion,
-			pHalData->FirmwareSubVersion,
-			pHalData->FirmwareSignature);
-
-	if (IS_FW_HEADER_EXIST(pFwHdr)) {
-		/*  Shift 32 bytes for FW header */
-		buf = buf + 32;
-		fw_size = fw_size - 32;
-	}
-
-	/*  Suggested by Filen. If 8051 is running in RAM code, driver should
-	    inform Fw to reset by itself, */
-	/*  or it will cause download Fw fail. 2010.02.01. by tynli. */
-	if (rtl8723au_read8(padapter, REG_MCUFWDL) & RAM_DL_SEL) {
-		/* 8051 RAM code */
-		rtl8723a_FirmwareSelfReset(padapter);
-		rtl8723au_write8(padapter, REG_MCUFWDL, 0x00);
-	}
-
-	_FWDownloadEnable(padapter, true);
-	fwdl_start_time = jiffies;
-	while (1) {
-		/* reset the FWDL chksum */
-		rtl8723au_write8(padapter, REG_MCUFWDL,
-				 rtl8723au_read8(padapter, REG_MCUFWDL) |
-				 FWDL_ChkSum_rpt);
-
-		rtStatus = _WriteFW(padapter, buf, fw_size);
-
-		if (rtStatus == _SUCCESS ||
-		    (jiffies_to_msecs(jiffies - fwdl_start_time) > 500 &&
-		     writeFW_retry++ >= 3))
-			break;
-
-		DBG_8723A("%s writeFW_retry:%u, time after fwdl_start_time:"
-			  "%ums\n", __func__, writeFW_retry,
-			  jiffies_to_msecs(jiffies - fwdl_start_time));
-	}
-	_FWDownloadEnable(padapter, false);
-	if (_SUCCESS != rtStatus) {
-		DBG_8723A("DL Firmware failed!\n");
-		goto Exit;
-	}
-
-	rtStatus = _FWFreeToGo(padapter);
-	if (_SUCCESS != rtStatus) {
-		RT_TRACE(_module_hal_init_c_, _drv_err_,
-			 "DL Firmware failed!\n");
-		goto Exit;
-	}
-	RT_TRACE(_module_hal_init_c_, _drv_info_,
-		 "Firmware is ready to run!\n");
-
-Exit:
-	kfree(firmware_buf);
-	return rtStatus;
-}
-
-void rtl8723a_InitializeFirmwareVars(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	/*  Init Fw LPS related. */
-	padapter->pwrctrlpriv.bFwCurrentInPSMode = false;
-
-	/*  Init H2C counter. by tynli. 2009.12.09. */
-	pHalData->LastHMEBoxNum = 0;
-}
-
-/*  */
-/*				Efuse related code */
-/*  */
-static u8
-hal_EfuseSwitchToBank(struct rtw_adapter *padapter, u8 bank)
-{
-	u8 bRet = false;
-	u32 value32 = 0;
-
-	DBG_8723A("%s: Efuse switch bank to %d\n", __func__, bank);
-	value32 = rtl8723au_read32(padapter, EFUSE_TEST);
-	bRet = true;
-	switch (bank) {
-	case 0:
-		value32 = (value32 & ~EFUSE_SEL_MASK) |
-			EFUSE_SEL(EFUSE_WIFI_SEL_0);
-		break;
-	case 1:
-		value32 = (value32 & ~EFUSE_SEL_MASK) |
-			EFUSE_SEL(EFUSE_BT_SEL_0);
-		break;
-	case 2:
-		value32 = (value32 & ~EFUSE_SEL_MASK) |
-			EFUSE_SEL(EFUSE_BT_SEL_1);
-		break;
-	case 3:
-		value32 = (value32 & ~EFUSE_SEL_MASK) |
-			EFUSE_SEL(EFUSE_BT_SEL_2);
-		break;
-	default:
-		value32 = (value32 & ~EFUSE_SEL_MASK) |
-			EFUSE_SEL(EFUSE_WIFI_SEL_0);
-		bRet = false;
-		break;
-	}
-	rtl8723au_write32(padapter, EFUSE_TEST, value32);
-
-	return bRet;
-}
-
-static void
-hal_ReadEFuse_WiFi(struct rtw_adapter *padapter,
-		   u16 _offset, u16 _size_byte, u8 *pbuf)
-{
-	u8 *efuseTbl = NULL;
-	u16 eFuse_Addr = 0;
-	u8 offset, wden;
-	u8 efuseHeader, efuseExtHdr, efuseData;
-	u16 i, total, used;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	/*  Do NOT excess total size of EFuse table.
-	    Added by Roger, 2008.11.10. */
-	if ((_offset + _size_byte) > EFUSE_MAP_LEN_8723A) {
-		DBG_8723A("%s: Invalid offset(%#x) with read bytes(%#x)!!\n",
-			  __func__, _offset, _size_byte);
-		return;
-	}
-
-	efuseTbl = kmalloc(EFUSE_MAP_LEN_8723A, GFP_KERNEL);
-	if (!efuseTbl)
-		return;
-	/*  0xff will be efuse default value instead of 0x00. */
-	memset(efuseTbl, 0xFF, EFUSE_MAP_LEN_8723A);
-
-	/*  switch bank back to bank 0 for later BT and wifi use. */
-	hal_EfuseSwitchToBank(padapter, 0);
-
-	while (AVAILABLE_EFUSE_ADDR(eFuse_Addr)) {
-		ReadEFuseByte23a(padapter, eFuse_Addr++, &efuseHeader);
-		if (efuseHeader == 0xFF) {
-			DBG_8723A("%s: data end at address =%#x\n", __func__,
-				  eFuse_Addr);
-			break;
-		}
-
-		/*  Check PG header for section num. */
-		if (EXT_HEADER(efuseHeader)) {	/* extended header */
-			offset = GET_HDR_OFFSET_2_0(efuseHeader);
-
-			ReadEFuseByte23a(padapter, eFuse_Addr++, &efuseExtHdr);
-			if (ALL_WORDS_DISABLED(efuseExtHdr))
-				continue;
-
-			offset |= ((efuseExtHdr & 0xF0) >> 1);
-			wden = efuseExtHdr & 0x0F;
-		} else {
-			offset = (efuseHeader >> 4) & 0x0f;
-			wden = efuseHeader & 0x0f;
-		}
-
-		if (offset < EFUSE_MAX_SECTION_8723A) {
-			u16 addr;
-			/*  Get word enable value from PG header */
-
-			addr = offset * PGPKT_DATA_SIZE;
-			for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) {
-				/* Check word enable condition in the section */
-				if (!(wden & (0x01 << i))) {
-					ReadEFuseByte23a(padapter, eFuse_Addr++,
-						      &efuseData);
-					efuseTbl[addr] = efuseData;
-
-					ReadEFuseByte23a(padapter, eFuse_Addr++,
-						      &efuseData);
-					efuseTbl[addr + 1] = efuseData;
-				}
-				addr += 2;
-			}
-		} else {
-			DBG_8723A(KERN_ERR "%s: offset(%d) is illegal!!\n",
-				  __func__, offset);
-			eFuse_Addr += Efuse_CalculateWordCnts23a(wden) * 2;
-		}
-	}
-
-	/*  Copy from Efuse map to output pointer memory!!! */
-	for (i = 0; i < _size_byte; i++)
-		pbuf[i] = efuseTbl[_offset + i];
-
-	/*  Calculate Efuse utilization */
-	EFUSE_GetEfuseDefinition23a(padapter, EFUSE_WIFI,
-				 TYPE_AVAILABLE_EFUSE_BYTES_TOTAL, &total);
-	used = eFuse_Addr - 1;
-	pHalData->EfuseUsedBytes = used;
-
-	kfree(efuseTbl);
-}
-
-static void
-hal_ReadEFuse_BT(struct rtw_adapter *padapter,
-		 u16 _offset, u16 _size_byte, u8 *pbuf)
-{
-	u8 *efuseTbl;
-	u8 bank;
-	u16 eFuse_Addr;
-	u8 efuseHeader, efuseExtHdr, efuseData;
-	u8 offset, wden;
-	u16 i, total, used;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	/*  Do NOT excess total size of EFuse table.
-	    Added by Roger, 2008.11.10. */
-	if ((_offset + _size_byte) > EFUSE_BT_MAP_LEN) {
-		DBG_8723A("%s: Invalid offset(%#x) with read bytes(%#x)!!\n",
-			  __func__, _offset, _size_byte);
-		return;
-	}
-
-	efuseTbl = kmalloc(EFUSE_BT_MAP_LEN, GFP_KERNEL);
-	if (!efuseTbl)
-		return;
-	/*  0xff will be efuse default value instead of 0x00. */
-	memset(efuseTbl, 0xFF, EFUSE_BT_MAP_LEN);
-
-	EFUSE_GetEfuseDefinition23a(padapter, EFUSE_BT,
-				 TYPE_AVAILABLE_EFUSE_BYTES_BANK, &total);
-
-	for (bank = 1; bank < EFUSE_MAX_BANK; bank++) {
-		if (hal_EfuseSwitchToBank(padapter, bank) == false) {
-			DBG_8723A("%s: hal_EfuseSwitchToBank Fail!!\n",
-				  __func__);
-			goto exit;
-		}
-
-		eFuse_Addr = 0;
-
-		while (AVAILABLE_EFUSE_ADDR(eFuse_Addr)) {
-			ReadEFuseByte23a(padapter, eFuse_Addr++, &efuseHeader);
-			if (efuseHeader == 0xFF)
-				break;
-
-			/*  Check PG header for section num. */
-			if (EXT_HEADER(efuseHeader)) {	/* extended header */
-				offset = GET_HDR_OFFSET_2_0(efuseHeader);
-
-				ReadEFuseByte23a(padapter, eFuse_Addr++,
-					      &efuseExtHdr);
-				if (ALL_WORDS_DISABLED(efuseExtHdr))
-					continue;
-
-				offset |= ((efuseExtHdr & 0xF0) >> 1);
-				wden = efuseExtHdr & 0x0F;
-			} else {
-				offset = (efuseHeader >> 4) & 0x0f;
-				wden = efuseHeader & 0x0f;
-			}
-
-			if (offset < EFUSE_BT_MAX_SECTION) {
-				u16 addr;
-
-				/*  Get word enable value from PG header */
-
-				addr = offset * PGPKT_DATA_SIZE;
-				for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) {
-					/*  Check word enable condition in
-					    the section */
-					if (!(wden & (0x01 << i))) {
-						ReadEFuseByte23a(padapter,
-							      eFuse_Addr++,
-							      &efuseData);
-						efuseTbl[addr] = efuseData;
-
-						ReadEFuseByte23a(padapter,
-							      eFuse_Addr++,
-							      &efuseData);
-						efuseTbl[addr + 1] = efuseData;
-					}
-					addr += 2;
-				}
-			} else {
-				DBG_8723A(KERN_ERR
-					  "%s: offset(%d) is illegal!!\n",
-					  __func__, offset);
-				eFuse_Addr += Efuse_CalculateWordCnts23a(wden) * 2;
-			}
-		}
-
-		if ((eFuse_Addr - 1) < total) {
-			DBG_8723A("%s: bank(%d) data end at %#x\n",
-				  __func__, bank, eFuse_Addr - 1);
-			break;
-		}
-	}
-
-	/*  switch bank back to bank 0 for later BT and wifi use. */
-	hal_EfuseSwitchToBank(padapter, 0);
-
-	/*  Copy from Efuse map to output pointer memory!!! */
-	for (i = 0; i < _size_byte; i++)
-		pbuf[i] = efuseTbl[_offset + i];
-
-	/*  */
-	/*  Calculate Efuse utilization. */
-	/*  */
-	EFUSE_GetEfuseDefinition23a(padapter, EFUSE_BT,
-				 TYPE_AVAILABLE_EFUSE_BYTES_TOTAL, &total);
-	used = (EFUSE_BT_REAL_BANK_CONTENT_LEN * (bank - 1)) + eFuse_Addr - 1;
-	pHalData->BTEfuseUsedBytes = used;
-
-exit:
-	kfree(efuseTbl);
-}
-
-void
-rtl8723a_readefuse(struct rtw_adapter *padapter,
-		   u8 efuseType, u16 _offset, u16 _size_byte, u8 *pbuf)
-{
-	if (efuseType == EFUSE_WIFI)
-		hal_ReadEFuse_WiFi(padapter, _offset, _size_byte, pbuf);
-	else
-		hal_ReadEFuse_BT(padapter, _offset, _size_byte, pbuf);
-}
-
-u16 rtl8723a_EfuseGetCurrentSize_WiFi(struct rtw_adapter *padapter)
-{
-	u16 efuse_addr = 0;
-	u8 hoffset = 0, hworden = 0;
-	u8 efuse_data, word_cnts = 0;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	efuse_addr = pHalData->EfuseUsedBytes;
-
-	DBG_8723A("%s: start_efuse_addr = 0x%X\n", __func__, efuse_addr);
-
-	/*  switch bank back to bank 0 for later BT and wifi use. */
-	hal_EfuseSwitchToBank(padapter, 0);
-
-	while (AVAILABLE_EFUSE_ADDR(efuse_addr)) {
-		if (efuse_OneByteRead23a(padapter, efuse_addr, &efuse_data) ==
-		    _FAIL) {
-			DBG_8723A(KERN_ERR "%s: efuse_OneByteRead23a Fail! "
-				  "addr = 0x%X !!\n", __func__, efuse_addr);
-			break;
-		}
-
-		if (efuse_data == 0xFF)
-			break;
-
-		if (EXT_HEADER(efuse_data)) {
-			hoffset = GET_HDR_OFFSET_2_0(efuse_data);
-			efuse_addr++;
-			efuse_OneByteRead23a(padapter, efuse_addr, &efuse_data);
-			if (ALL_WORDS_DISABLED(efuse_data))
-				continue;
-
-			hoffset |= ((efuse_data & 0xF0) >> 1);
-			hworden = efuse_data & 0x0F;
-		} else {
-			hoffset = (efuse_data >> 4) & 0x0F;
-			hworden = efuse_data & 0x0F;
-		}
-
-		word_cnts = Efuse_CalculateWordCnts23a(hworden);
-		efuse_addr += (word_cnts * 2) + 1;
-	}
-
-	pHalData->EfuseUsedBytes = efuse_addr;
-
-	DBG_8723A("%s: CurrentSize =%d\n", __func__, efuse_addr);
-
-	return efuse_addr;
-}
-
-u16 rtl8723a_EfuseGetCurrentSize_BT(struct rtw_adapter *padapter)
-{
-	u16 btusedbytes;
-	u16 efuse_addr;
-	u8 bank, startBank;
-	u8 hoffset = 0, hworden = 0;
-	u8 efuse_data, word_cnts = 0;
-	u16 retU2 = 0;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	btusedbytes = pHalData->BTEfuseUsedBytes;
-
-	efuse_addr = (u16) ((btusedbytes % EFUSE_BT_REAL_BANK_CONTENT_LEN));
-	startBank = (u8) (1 + (btusedbytes / EFUSE_BT_REAL_BANK_CONTENT_LEN));
-
-	DBG_8723A("%s: start from bank =%d addr = 0x%X\n", __func__, startBank,
-		  efuse_addr);
-
-	EFUSE_GetEfuseDefinition23a(padapter, EFUSE_BT,
-				 TYPE_AVAILABLE_EFUSE_BYTES_BANK, &retU2);
-
-	for (bank = startBank; bank < EFUSE_MAX_BANK; bank++) {
-		if (hal_EfuseSwitchToBank(padapter, bank) == false) {
-			DBG_8723A(KERN_ERR "%s: switch bank(%d) Fail!!\n",
-				  __func__, bank);
-			bank = EFUSE_MAX_BANK;
-			break;
-		}
-
-		/*  only when bank is switched we have to reset
-		    the efuse_addr. */
-		if (bank != startBank)
-			efuse_addr = 0;
-
-		while (AVAILABLE_EFUSE_ADDR(efuse_addr)) {
-			if (efuse_OneByteRead23a(padapter, efuse_addr,
-					      &efuse_data) == _FAIL) {
-				DBG_8723A(KERN_ERR "%s: efuse_OneByteRead23a Fail!"
-					  " addr = 0x%X !!\n",
-					  __func__, efuse_addr);
-				bank = EFUSE_MAX_BANK;
-				break;
-			}
-
-			if (efuse_data == 0xFF)
-				break;
-
-			if (EXT_HEADER(efuse_data)) {
-				hoffset = GET_HDR_OFFSET_2_0(efuse_data);
-				efuse_addr++;
-				efuse_OneByteRead23a(padapter, efuse_addr,
-						  &efuse_data);
-				if (ALL_WORDS_DISABLED(efuse_data)) {
-					efuse_addr++;
-					continue;
-				}
-
-				hoffset |= ((efuse_data & 0xF0) >> 1);
-				hworden = efuse_data & 0x0F;
-			} else {
-				hoffset = (efuse_data >> 4) & 0x0F;
-				hworden = efuse_data & 0x0F;
-			}
-			word_cnts = Efuse_CalculateWordCnts23a(hworden);
-			/* read next header */
-			efuse_addr += (word_cnts * 2) + 1;
-		}
-
-		/*  Check if we need to check next bank efuse */
-		if (efuse_addr < retU2)
-			break;	/*  don't need to check next bank. */
-	}
-
-	retU2 = ((bank - 1) * EFUSE_BT_REAL_BANK_CONTENT_LEN) + efuse_addr;
-	pHalData->BTEfuseUsedBytes = retU2;
-
-	DBG_8723A("%s: CurrentSize =%d\n", __func__, retU2);
-	return retU2;
-}
-
-void rtl8723a_read_chip_version(struct rtw_adapter *padapter)
-{
-	u32 value32;
-	struct hal_version ChipVersion;
-	struct hal_data_8723a *pHalData;
-
-	pHalData = GET_HAL_DATA(padapter);
-
-	value32 = rtl8723au_read32(padapter, REG_SYS_CFG);
-	ChipVersion.ICType = CHIP_8723A;
-	ChipVersion.ChipType = ((value32 & RTL_ID) ? TEST_CHIP : NORMAL_CHIP);
-	pHalData->rf_type = RF_1T1R;
-	ChipVersion.VendorType =
-		((value32 & VENDOR_ID) ? CHIP_VENDOR_UMC : CHIP_VENDOR_TSMC);
-	ChipVersion.CUTVersion = (value32 & CHIP_VER_RTL_MASK) >> CHIP_VER_RTL_SHIFT;	/*  IC version (CUT) */
-
-	/*  For regulator mode. by tynli. 2011.01.14 */
-	pHalData->RegulatorMode = ((value32 & SPS_SEL) ?
-				   RT_LDO_REGULATOR : RT_SWITCHING_REGULATOR);
-
-	value32 = rtl8723au_read32(padapter, REG_GPIO_OUTSTS);
-	/*  ROM code version. */
-	ChipVersion.ROMVer = (value32 & RF_RL_ID) >> 20;
-
-	/*  For multi-function consideration. Added by Roger, 2010.10.06. */
-	pHalData->MultiFunc = RT_MULTI_FUNC_NONE;
-	value32 = rtl8723au_read32(padapter, REG_MULTI_FUNC_CTRL);
-	pHalData->MultiFunc |=
-		((value32 & WL_FUNC_EN) ? RT_MULTI_FUNC_WIFI : 0);
-	pHalData->MultiFunc |= ((value32 & BT_FUNC_EN) ? RT_MULTI_FUNC_BT : 0);
-	pHalData->MultiFunc |=
-		((value32 & GPS_FUNC_EN) ? RT_MULTI_FUNC_GPS : 0);
-	pHalData->PolarityCtl =
-		((value32 & WL_HWPDN_SL) ? RT_POLARITY_HIGH_ACT :
-		 RT_POLARITY_LOW_ACT);
-	pHalData->VersionID = ChipVersion;
-
-	MSG_8723A("RF_Type is %x!!\n", pHalData->rf_type);
-}
-
-/*  */
-/*  */
-/*  20100209 Joseph: */
-/*  This function is used only for 92C to set REG_BCN_CTRL(0x550) register. */
-/*  We just reserve the value of the register in variable
-    pHalData->RegBcnCtrlVal and then operate */
-/*  the value of the register via atomic operation. */
-/*  This prevents from race condition when setting this register. */
-/*  The value of pHalData->RegBcnCtrlVal is initialized in
-    HwConfigureRTL8192CE() function. */
-/*  */
-void SetBcnCtrlReg23a(struct rtw_adapter *padapter, u8 SetBits, u8 ClearBits)
-{
-	u8 val8;
-
-	val8 = rtl8723au_read8(padapter, REG_BCN_CTRL);
-	val8 |= SetBits;
-	val8 &= ~ClearBits;
-
-	rtl8723au_write8(padapter, REG_BCN_CTRL, val8);
-}
-
-void rtl8723a_InitBeaconParameters(struct rtw_adapter *padapter)
-{
-	rtl8723au_write16(padapter, REG_BCN_CTRL, 0x1010);
-
-	/*  TODO: Remove these magic number */
-	rtl8723au_write16(padapter, REG_TBTT_PROHIBIT, 0x6404);	/*  ms */
-	/*  Firmware will control REG_DRVERLYINT when power saving is enable, */
-	/*  so don't set this register on STA mode. */
-	if (check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE) == false)
-		rtl8723au_write8(padapter, REG_DRVERLYINT,
-				 DRIVER_EARLY_INT_TIME);
-	/*  2ms */
-	rtl8723au_write8(padapter, REG_BCNDMATIM, BCN_DMA_ATIME_INT_TIME);
-
-	/*  Suggested by designer timchen. Change beacon AIFS to the
-	    largest number beacause test chip does not contension before
-	    sending beacon. by tynli. 2009.11.03 */
-	rtl8723au_write16(padapter, REG_BCNTCFG, 0x660F);
-}
-
-static void ResumeTxBeacon(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	/*  2010.03.01. Marked by tynli. No need to call workitem beacause
-	    we record the value */
-	/*  which should be read from register to a global variable. */
-
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_, "+ResumeTxBeacon\n");
-
-	pHalData->RegFwHwTxQCtrl |= BIT(6);
-	rtl8723au_write8(padapter, REG_FWHW_TXQ_CTRL + 2,
-			 pHalData->RegFwHwTxQCtrl);
-	rtl8723au_write8(padapter, REG_TBTT_PROHIBIT + 1, 0xff);
-	pHalData->RegReg542 |= BIT(0);
-	rtl8723au_write8(padapter, REG_TBTT_PROHIBIT + 2, pHalData->RegReg542);
-}
-
-static void StopTxBeacon(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	/*  2010.03.01. Marked by tynli. No need to call workitem beacause
-	    we record the value */
-	/*  which should be read from register to a global variable. */
-
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_, "+StopTxBeacon\n");
-
-	pHalData->RegFwHwTxQCtrl &= ~BIT(6);
-	rtl8723au_write8(padapter, REG_FWHW_TXQ_CTRL + 2,
-			 pHalData->RegFwHwTxQCtrl);
-	rtl8723au_write8(padapter, REG_TBTT_PROHIBIT + 1, 0x64);
-	pHalData->RegReg542 &= ~BIT(0);
-	rtl8723au_write8(padapter, REG_TBTT_PROHIBIT + 2, pHalData->RegReg542);
-}
-
-static void _BeaconFunctionEnable(struct rtw_adapter *padapter, u8 Enable,
-				  u8 Linked)
-{
-	SetBcnCtrlReg23a(padapter, DIS_TSF_UDT | EN_BCN_FUNCTION | DIS_BCNQ_SUB,
-		      0);
-	rtl8723au_write8(padapter, REG_RD_CTRL + 1, 0x6F);
-}
-
-void rtl8723a_SetBeaconRelatedRegisters(struct rtw_adapter *padapter)
-{
-	u32 value32;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	/* reset TSF, enable update TSF, correcting TSF On Beacon */
-
-	/* REG_BCN_INTERVAL */
-	/* REG_BCNDMATIM */
-	/* REG_ATIMWND */
-	/* REG_TBTT_PROHIBIT */
-	/* REG_DRVERLYINT */
-	/* REG_BCN_MAX_ERR */
-	/* REG_BCNTCFG (0x510) */
-	/* REG_DUAL_TSF_RST */
-	/* REG_BCN_CTRL (0x550) */
-
-	/*  */
-	/*  ATIM window */
-	/*  */
-	rtl8723au_write16(padapter, REG_ATIMWND, 2);
-
-	/*  */
-	/*  Beacon interval (in unit of TU). */
-	/*  */
-	rtl8723au_write16(padapter, REG_BCN_INTERVAL, pmlmeinfo->bcn_interval);
-
-	rtl8723a_InitBeaconParameters(padapter);
-
-	rtl8723au_write8(padapter, REG_SLOT, 0x09);
-
-	/*  */
-	/*  Reset TSF Timer to zero, added by Roger. 2008.06.24 */
-	/*  */
-	value32 = rtl8723au_read32(padapter, REG_TCR);
-	value32 &= ~TSFRST;
-	rtl8723au_write32(padapter, REG_TCR, value32);
-
-	value32 |= TSFRST;
-	rtl8723au_write32(padapter, REG_TCR, value32);
-
-	/*  NOTE: Fix test chip's bug (about contention windows's randomness) */
-	if (check_fwstate(&padapter->mlmepriv, WIFI_ADHOC_STATE |
-			  WIFI_ADHOC_MASTER_STATE | WIFI_AP_STATE) == true) {
-		rtl8723au_write8(padapter, REG_RXTSF_OFFSET_CCK, 0x50);
-		rtl8723au_write8(padapter, REG_RXTSF_OFFSET_OFDM, 0x50);
-	}
-
-	_BeaconFunctionEnable(padapter, true, true);
-
-	ResumeTxBeacon(padapter);
-	SetBcnCtrlReg23a(padapter, DIS_BCNQ_SUB, 0);
-}
-
-void rtl8723a_SetHalODMVar(struct rtw_adapter *Adapter,
-			   enum hal_odm_variable eVariable,
-			   void *pValue1, bool bSet)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_odm_t *podmpriv = &pHalData->odmpriv;
-	switch (eVariable) {
-	case HAL_ODM_STA_INFO:
-	{
-		struct sta_info *psta = (struct sta_info *)pValue1;
-
-		if (bSet) {
-			DBG_8723A("Set STA_(%d) info\n", psta->mac_id);
-			ODM_CmnInfoPtrArrayHook23a(podmpriv,
-						ODM_CMNINFO_STA_STATUS,
-						psta->mac_id, psta);
-		} else {
-			DBG_8723A("Clean STA_(%d) info\n", psta->mac_id);
-				ODM_CmnInfoPtrArrayHook23a(podmpriv,
-							ODM_CMNINFO_STA_STATUS,
-							psta->mac_id, NULL);
-		}
-	}
-		break;
-	case HAL_ODM_P2P_STATE:
-		ODM_CmnInfoUpdate23a(podmpriv, ODM_CMNINFO_WIFI_DIRECT, bSet);
-		break;
-	case HAL_ODM_WIFI_DISPLAY_STATE:
-		ODM_CmnInfoUpdate23a(podmpriv, ODM_CMNINFO_WIFI_DISPLAY, bSet);
-		break;
-	default:
-		break;
-	}
-}
-
-void rtl8723a_notch_filter(struct rtw_adapter *adapter, bool enable)
-{
-	if (enable) {
-		DBG_8723A("Enable notch filter\n");
-		rtl8723au_write8(adapter, rOFDM0_RxDSP + 1,
-				 rtl8723au_read8(adapter, rOFDM0_RxDSP + 1) |
-				 BIT(1));
-	} else {
-		DBG_8723A("Disable notch filter\n");
-		rtl8723au_write8(adapter, rOFDM0_RxDSP + 1,
-			   rtl8723au_read8(adapter, rOFDM0_RxDSP + 1) &
-				 ~BIT(1));
-	}
-}
-
-bool c2h_id_filter_ccx_8723a(u8 id)
-{
-	bool ret = false;
-	if (id == C2H_CCX_TX_RPT)
-		ret = true;
-
-	return ret;
-}
-
-int c2h_handler_8723a(struct rtw_adapter *padapter, struct c2h_evt_hdr *c2h_evt)
-{
-	int ret = _SUCCESS;
-	u8 i = 0;
-
-	if (c2h_evt == NULL) {
-		DBG_8723A("%s c2h_evt is NULL\n", __func__);
-		ret = _FAIL;
-		goto exit;
-	}
-
-	switch (c2h_evt->id) {
-	case C2H_DBG:
-		RT_TRACE(_module_hal_init_c_, _drv_info_,
-			 "C2HCommandHandler: %s\n", c2h_evt->payload);
-		break;
-
-	case C2H_CCX_TX_RPT:
-		handle_txrpt_ccx_8723a(padapter, c2h_evt->payload);
-		break;
-	case C2H_EXT_RA_RPT:
-		break;
-	case C2H_HW_INFO_EXCH:
-		RT_TRACE(_module_hal_init_c_, _drv_info_,
-			 "[BT], C2H_HW_INFO_EXCH\n");
-		for (i = 0; i < c2h_evt->plen; i++) {
-			RT_TRACE(_module_hal_init_c_, _drv_info_,
-				 "[BT], tmpBuf[%d]= 0x%x\n", i,
-				 c2h_evt->payload[i]);
-		}
-		break;
-
-	case C2H_C2H_H2C_TEST:
-		RT_TRACE(_module_hal_init_c_, _drv_info_,
-			 "[BT], C2H_H2C_TEST\n");
-		RT_TRACE(_module_hal_init_c_, _drv_info_,
-			 "[BT], tmpBuf[0]/[1]/[2]/[3]/[4]= 0x%x/ 0x%x/ 0x%x/ 0x%x/ 0x%x\n",
-			 c2h_evt->payload[0],
-			 c2h_evt->payload[1], c2h_evt->payload[2],
-			 c2h_evt->payload[3], c2h_evt->payload[4]);
-		break;
-
-	case C2H_BT_INFO:
-		DBG_8723A("%s ,  Got  C2H_BT_INFO \n", __func__);
-		rtl8723a_fw_c2h_BT_info(padapter,
-					c2h_evt->payload, c2h_evt->plen);
-		break;
-
-	default:
-		ret = _FAIL;
-		break;
-	}
-
-exit:
-	return ret;
-}
-
-void handle_txrpt_ccx_8723a(struct rtw_adapter *adapter, void *buf)
-{
-	struct txrpt_ccx_8723a *txrpt_ccx = buf;
-	struct submit_ctx *pack_tx_ops = &adapter->xmitpriv.ack_tx_ops;
-
-	if (txrpt_ccx->int_ccx && adapter->xmitpriv.ack_tx) {
-		if (txrpt_ccx->pkt_ok)
-			rtw23a_sctx_done_err(&pack_tx_ops,
-					     RTW_SCTX_DONE_SUCCESS);
-		else
-			rtw23a_sctx_done_err(&pack_tx_ops,
-					     RTW_SCTX_DONE_CCX_PKT_FAIL);
-	}
-}
-
-void rtl8723a_InitAntenna_Selection(struct rtw_adapter *padapter)
-{
-	u8 val;
-
-	val = rtl8723au_read8(padapter, REG_LEDCFG2);
-	/*  Let 8051 take control antenna setting */
-	val |= BIT(7);		/*  DPDT_SEL_EN, 0x4C[23] */
-	rtl8723au_write8(padapter, REG_LEDCFG2, val);
-}
-
-void rtl8723a_CheckAntenna_Selection(struct rtw_adapter *padapter)
-{
-	u8 val;
-
-	val = rtl8723au_read8(padapter, REG_LEDCFG2);
-	/*  Let 8051 take control antenna setting */
-	if (!(val & BIT(7))) {
-		val |= BIT(7);	/*  DPDT_SEL_EN, 0x4C[23] */
-		rtl8723au_write8(padapter, REG_LEDCFG2, val);
-	}
-}
-
-void rtl8723a_DeinitAntenna_Selection(struct rtw_adapter *padapter)
-{
-	u8 val;
-
-	val = rtl8723au_read8(padapter, REG_LEDCFG2);
-	/*  Let 8051 take control antenna setting */
-	val &= ~BIT(7);		/*  DPDT_SEL_EN, clear 0x4C[23] */
-	rtl8723au_write8(padapter, REG_LEDCFG2, val);
-}
-
-void rtl8723a_init_default_value(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData;
-	struct dm_priv *pdmpriv;
-	u8 i;
-
-	pHalData = GET_HAL_DATA(padapter);
-	pdmpriv = &pHalData->dmpriv;
-
-	/*  init default value */
-	pHalData->bIQKInitialized = false;
-	if (!padapter->pwrctrlpriv.bkeepfwalive)
-		pHalData->LastHMEBoxNum = 0;
-
-	pHalData->bIQKInitialized = false;
-
-	/*  init dm default value */
-	pdmpriv->TM_Trigger = 0;	/* for IQK */
-/*	pdmpriv->binitialized = false; */
-/*	pdmpriv->prv_traffic_idx = 3; */
-/*	pdmpriv->initialize = 0; */
-
-	pdmpriv->ThermalValue_HP_index = 0;
-	for (i = 0; i < HP_THERMAL_NUM; i++)
-		pdmpriv->ThermalValue_HP[i] = 0;
-
-	/*  init Efuse variables */
-	pHalData->EfuseUsedBytes = 0;
-	pHalData->BTEfuseUsedBytes = 0;
-}
-
-u8 GetEEPROMSize8723A(struct rtw_adapter *padapter)
-{
-	u8 size = 0;
-	u32 cr;
-
-	cr = rtl8723au_read16(padapter, REG_9346CR);
-	/*  6: EEPROM used is 93C46, 4: boot from E-Fuse. */
-	size = (cr & BOOT_FROM_EEPROM) ? 6 : 4;
-
-	MSG_8723A("EEPROM type is %s\n", size == 4 ? "E-FUSE" : "93C46");
-
-	return size;
-}
-
-/*  */
-/*  */
-/*  LLT R/W/Init function */
-/*  */
-/*  */
-static int _LLTWrite(struct rtw_adapter *padapter, u32 address, u32 data)
-{
-	int status = _SUCCESS;
-	s32 count = 0;
-	u32 value = _LLT_INIT_ADDR(address) | _LLT_INIT_DATA(data) |
-		    _LLT_OP(_LLT_WRITE_ACCESS);
-	u16 LLTReg = REG_LLT_INIT;
-
-	rtl8723au_write32(padapter, LLTReg, value);
-
-	/* polling */
-	do {
-		value = rtl8723au_read32(padapter, LLTReg);
-		if (_LLT_NO_ACTIVE == _LLT_OP_VALUE(value))
-			break;
-
-		if (count > POLLING_LLT_THRESHOLD) {
-			RT_TRACE(_module_hal_init_c_, _drv_err_,
-				 "Failed to polling write LLT done at address %d!\n",
-				 address);
-			status = _FAIL;
-			break;
-		}
-	} while (count++);
-
-	return status;
-}
-
-int InitLLTTable23a(struct rtw_adapter *padapter, u32 boundary)
-{
-	int status = _SUCCESS;
-	u32 i;
-	u32 txpktbuf_bndy = boundary;
-	u32 Last_Entry_Of_TxPktBuf = LAST_ENTRY_OF_TX_PKT_BUFFER;
-
-	for (i = 0; i < (txpktbuf_bndy - 1); i++) {
-		status = _LLTWrite(padapter, i, i + 1);
-		if (status != _SUCCESS)
-			return status;
-	}
-
-	/*  end of list */
-	status = _LLTWrite(padapter, (txpktbuf_bndy - 1), 0xFF);
-	if (status != _SUCCESS)
-		return status;
-
-	/*  Make the other pages as ring buffer */
-	/*  This ring buffer is used as beacon buffer if we config this
-	    MAC as two MAC transfer. */
-	/*  Otherwise used as local loopback buffer. */
-	for (i = txpktbuf_bndy; i < Last_Entry_Of_TxPktBuf; i++) {
-		status = _LLTWrite(padapter, i, (i + 1));
-		if (_SUCCESS != status)
-			return status;
-	}
-
-	/*  Let last entry point to the start entry of ring buffer */
-	status = _LLTWrite(padapter, Last_Entry_Of_TxPktBuf, txpktbuf_bndy);
-
-	return status;
-}
-
-static void _DisableGPIO(struct rtw_adapter *padapter)
-{
-/***************************************
-j. GPIO_PIN_CTRL 0x44[31:0]= 0x000
-k.Value = GPIO_PIN_CTRL[7:0]
-l. GPIO_PIN_CTRL 0x44[31:0] = 0x00FF0000 | (value <<8); write external PIN level
-m. GPIO_MUXCFG 0x42 [15:0] = 0x0780
-n. LEDCFG 0x4C[15:0] = 0x8080
-***************************************/
-	u32 value32;
-	u32 u4bTmp;
-
-	/* 1. Disable GPIO[7:0] */
-	rtl8723au_write16(padapter, REG_GPIO_PIN_CTRL + 2, 0x0000);
-	value32 = rtl8723au_read32(padapter, REG_GPIO_PIN_CTRL) & 0xFFFF00FF;
-	u4bTmp = value32 & 0x000000FF;
-	value32 |= ((u4bTmp << 8) | 0x00FF0000);
-	rtl8723au_write32(padapter, REG_GPIO_PIN_CTRL, value32);
-
-	/*  */
-	/*  <Roger_Notes> For RTL8723u multi-function configuration which
-	    was autoload from Efuse offset 0x0a and 0x0b, */
-	/*  WLAN HW GPIO[9], GPS HW GPIO[10] and BT HW GPIO[11]. */
-	/*  Added by Roger, 2010.10.07. */
-	/*  */
-	/* 2. Disable GPIO[8] and GPIO[12] */
-
-	/*  Configure all pins as input mode. */
-	rtl8723au_write16(padapter, REG_GPIO_IO_SEL_2, 0x0000);
-	value32 = rtl8723au_read32(padapter, REG_GPIO_PIN_CTRL_2) & 0xFFFF001F;
-	u4bTmp = value32 & 0x0000001F;
-	/*  Set pin 8, 10, 11 and pin 12 to output mode. */
-	value32 |= ((u4bTmp << 8) | 0x001D0000);
-	rtl8723au_write32(padapter, REG_GPIO_PIN_CTRL_2, value32);
-
-	/* 3. Disable LED0 & 1 */
-	rtl8723au_write16(padapter, REG_LEDCFG0, 0x8080);
-}				/* end of _DisableGPIO() */
-
-static void _DisableRFAFEAndResetBB8192C(struct rtw_adapter *padapter)
-{
-/**************************************
-a.	TXPAUSE 0x522[7:0] = 0xFF		Pause MAC TX queue
-b.	RF path 0 offset 0x00 = 0x00		disable RF
-c.	APSD_CTRL 0x600[7:0] = 0x40
-d.	SYS_FUNC_EN 0x02[7:0] = 0x16		reset BB state machine
-e.	SYS_FUNC_EN 0x02[7:0] = 0x14		reset BB state machine
-***************************************/
-	u8 value8;
-
-	rtl8723au_write8(padapter, REG_TXPAUSE, 0xFF);
-
-	PHY_SetRFReg(padapter, RF_PATH_A, 0x0, bMaskByte0, 0x0);
-
-	value8 = APSDOFF;
-	rtl8723au_write8(padapter, REG_APSD_CTRL, value8);	/* 0x40 */
-
-	/*  Set BB reset at first */
-	value8 = FEN_USBD | FEN_USBA | FEN_BB_GLB_RSTn;
-	rtl8723au_write8(padapter, REG_SYS_FUNC_EN, value8);	/* 0x16 */
-
-	/*  Set global reset. */
-	value8 &= ~FEN_BB_GLB_RSTn;
-	rtl8723au_write8(padapter, REG_SYS_FUNC_EN, value8);	/* 0x14 */
-
-	/*  2010/08/12 MH We need to set BB/GLBAL reset to save power
-	    for SS mode. */
-}
-
-static void _ResetDigitalProcedure1_92C(struct rtw_adapter *padapter,
-					bool bWithoutHWSM)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (IS_FW_81xxC(padapter) && (pHalData->FirmwareVersion <= 0x20)) {
-	/*****************************
-	f.	MCUFWDL 0x80[7:0]= 0		reset MCU ready status
-	g.	SYS_FUNC_EN 0x02[10]= 0		reset MCU register, (8051 reset)
-	h.	SYS_FUNC_EN 0x02[15-12]= 5	reset MAC register, DCORE
-	i.     SYS_FUNC_EN 0x02[10]= 1		enable MCU register,
-						(8051 enable)
-	******************************/
-		u16 valu16;
-		rtl8723au_write8(padapter, REG_MCUFWDL, 0);
-
-		valu16 = rtl8723au_read16(padapter, REG_SYS_FUNC_EN);
-		/* reset MCU , 8051 */
-		rtl8723au_write16(padapter, REG_SYS_FUNC_EN,
-				  valu16 & ~FEN_CPUEN);
-
-		valu16 = rtl8723au_read16(padapter, REG_SYS_FUNC_EN) & 0x0FFF;
-		/* reset MAC */
-		rtl8723au_write16(padapter, REG_SYS_FUNC_EN,
-				  valu16 | FEN_HWPDN | FEN_ELDR);
-
-		valu16 = rtl8723au_read16(padapter, REG_SYS_FUNC_EN);
-		/* enable MCU , 8051 */
-		rtl8723au_write16(padapter, REG_SYS_FUNC_EN,
-				  valu16 | FEN_CPUEN);
-	} else {
-		u8 retry_cnts = 0;
-		u8 val8;
-
-		val8 = rtl8723au_read8(padapter, REG_MCUFWDL);
-
-		/*  2010/08/12 MH For USB SS, we can not stop 8051 when we
-		    are trying to enter IPS/HW&SW radio off. For
-		    S3/S4/S5/Disable, we can stop 8051 because */
-		/*  we will init FW when power on again. */
-		/*  If we want to SS mode, we can not reset 8051. */
-		if ((val8 & BIT(1)) && padapter->bFWReady) {
-			/* IF fw in RAM code, do reset */
-			/*  2010/08/25 MH According to RD alfred's
-			    suggestion, we need to disable other */
-			/*  HRCV INT to influence 8051 reset. */
-			rtl8723au_write8(padapter, REG_FWIMR, 0x20);
-			/*  2011/02/15 MH According to Alex's
-			    suggestion, close mask to prevent
-			    incorrect FW write operation. */
-			rtl8723au_write8(padapter, REG_FTIMR, 0x00);
-			rtl8723au_write8(padapter, REG_FSIMR, 0x00);
-
-			/* 8051 reset by self */
-			rtl8723au_write8(padapter, REG_HMETFR + 3, 0x20);
-
-			while ((retry_cnts++ < 100) &&
-			       (rtl8723au_read16(padapter, REG_SYS_FUNC_EN) &
-				FEN_CPUEN)) {
-				udelay(50);	/* us */
-			}
-
-			if (retry_cnts >= 100) {
-				/* Reset MAC and Enable 8051 */
-				rtl8723au_write8(padapter,
-						 REG_SYS_FUNC_EN + 1, 0x50);
-				mdelay(10);
-			}
-		}
-		/* Reset MAC and Enable 8051 */
-		rtl8723au_write8(padapter, REG_SYS_FUNC_EN + 1, 0x54);
-		rtl8723au_write8(padapter, REG_MCUFWDL, 0);
-	}
-
-	if (bWithoutHWSM) {
-	/*****************************
-		Without HW auto state machine
-	g.	SYS_CLKR 0x08[15:0] = 0x30A3		disable MAC clock
-	h.	AFE_PLL_CTRL 0x28[7:0] = 0x80		disable AFE PLL
-	i.	AFE_XTAL_CTRL 0x24[15:0] = 0x880F	gated AFE DIG_CLOCK
-	j.	SYS_ISO_CTRL 0x00[7:0] = 0xF9		isolated digital to PON
-	******************************/
-		/* modify to 0x70A3 by Scott. */
-		rtl8723au_write16(padapter, REG_SYS_CLKR, 0x70A3);
-		rtl8723au_write8(padapter, REG_AFE_PLL_CTRL, 0x80);
-		rtl8723au_write16(padapter, REG_AFE_XTAL_CTRL, 0x880F);
-		rtl8723au_write8(padapter, REG_SYS_ISO_CTRL, 0xF9);
-	} else {
-		/*  Disable all RF/BB power */
-		rtl8723au_write8(padapter, REG_RF_CTRL, 0x00);
-	}
-}
-
-static void _ResetDigitalProcedure2(struct rtw_adapter *padapter)
-{
-/*****************************
-k.	SYS_FUNC_EN 0x03[7:0] = 0x44		disable ELDR runction
-l.	SYS_CLKR 0x08[15:0] = 0x3083		disable ELDR clock
-m.	SYS_ISO_CTRL 0x01[7:0] = 0x83		isolated ELDR to PON
-******************************/
-	/* modify to 0x70a3 by Scott. */
-	rtl8723au_write16(padapter, REG_SYS_CLKR, 0x70a3);
-	/* modify to 0x82 by Scott. */
-	rtl8723au_write8(padapter, REG_SYS_ISO_CTRL + 1, 0x82);
-}
-
-static void _DisableAnalog(struct rtw_adapter *padapter, bool bWithoutHWSM)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u16 value16;
-	u8 value8;
-
-	if (bWithoutHWSM) {
-	/*****************************
-	n.	LDOA15_CTRL 0x20[7:0] = 0x04	disable A15 power
-	o.	LDOV12D_CTRL 0x21[7:0] = 0x54	disable digital core power
-	r.	When driver call disable, the ASIC will turn off remaining
-		clock automatically
-	******************************/
-
-		rtl8723au_write8(padapter, REG_LDOA15_CTRL, 0x04);
-		/* rtl8723au_write8(padapter, REG_LDOV12D_CTRL, 0x54); */
-
-		value8 = rtl8723au_read8(padapter, REG_LDOV12D_CTRL);
-		value8 &= ~LDV12_EN;
-		rtl8723au_write8(padapter, REG_LDOV12D_CTRL, value8);
-	}
-
-	/*****************************
-	h.	SPS0_CTRL 0x11[7:0] = 0x23		enter PFM mode
-	i.	APS_FSMCO 0x04[15:0] = 0x4802		set USB suspend
-	******************************/
-	value8 = 0x23;
-	if (IS_81xxC_VENDOR_UMC_B_CUT(pHalData->VersionID))
-		value8 |= BIT(3);
-
-	rtl8723au_write8(padapter, REG_SPS0_CTRL, value8);
-
-	if (bWithoutHWSM) {
-		/* value16 |= (APDM_HOST | FSM_HSUS |/PFM_ALDN); */
-		/*  2010/08/31 According to Filen description, we need to
-		    use HW to shut down 8051 automatically. */
-		/*  Because suspend operation need the asistance of 8051
-		    to wait for 3ms. */
-		value16 = APDM_HOST | AFSM_HSUS | PFM_ALDN;
-	} else {
-		value16 = APDM_HOST | AFSM_HSUS | PFM_ALDN;
-	}
-
-	rtl8723au_write16(padapter, REG_APS_FSMCO, value16);	/* 0x4802 */
-
-	rtl8723au_write8(padapter, REG_RSV_CTRL, 0x0e);
-}
-
-/*  HW Auto state machine */
-int CardDisableHWSM(struct rtw_adapter *padapter, u8 resetMCU)
-{
-	if (padapter->bSurpriseRemoved)
-		return _SUCCESS;
-
-	/*  RF Off Sequence ==== */
-	_DisableRFAFEAndResetBB8192C(padapter);
-
-	/*   ==== Reset digital sequence   ====== */
-	_ResetDigitalProcedure1_92C(padapter, false);
-
-	/*   ==== Pull GPIO PIN to balance level and LED control ====== */
-	_DisableGPIO(padapter);
-
-	/*   ==== Disable analog sequence === */
-	_DisableAnalog(padapter, false);
-
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "======> Card disable finished.\n");
-
-	return _SUCCESS;
-}
-
-/*  without HW Auto state machine */
-int CardDisableWithoutHWSM(struct rtw_adapter *padapter)
-{
-	if (padapter->bSurpriseRemoved)
-		return _SUCCESS;
-
-	/*  RF Off Sequence ==== */
-	_DisableRFAFEAndResetBB8192C(padapter);
-
-	/*   ==== Reset digital sequence   ====== */
-	_ResetDigitalProcedure1_92C(padapter, true);
-
-	/*   ==== Pull GPIO PIN to balance level and LED control ====== */
-	_DisableGPIO(padapter);
-
-	/*   ==== Reset digital sequence   ====== */
-	_ResetDigitalProcedure2(padapter);
-
-	/*   ==== Disable analog sequence === */
-	_DisableAnalog(padapter, true);
-
-	return _SUCCESS;
-}
-
-void Hal_InitPGData(struct rtw_adapter *padapter, u8 *PROMContent)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
-
-	if (!pEEPROM->bautoload_fail_flag) {	/*  autoload OK. */
-		if (!pEEPROM->EepromOrEfuse) {
-			/*  Read EFUSE real map to shadow. */
-			EFUSE_ShadowMapUpdate23a(padapter, EFUSE_WIFI);
-			memcpy(PROMContent, pEEPROM->efuse_eeprom_data,
-			       HWSET_MAX_SIZE);
-		}
-	} else {
-		RT_TRACE(_module_hci_hal_init_c_, _drv_notice_,
-			 "AutoLoad Fail reported from CR9346!!\n");
-		/* update to default value 0xFF */
-		if (!pEEPROM->EepromOrEfuse)
-			EFUSE_ShadowMapUpdate23a(padapter, EFUSE_WIFI);
-		memcpy(PROMContent, pEEPROM->efuse_eeprom_data,
-		       HWSET_MAX_SIZE);
-	}
-}
-
-void Hal_EfuseParseIDCode(struct rtw_adapter *padapter, u8 *hwinfo)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
-/*	struct hal_data_8723a	*pHalData = GET_HAL_DATA(padapter); */
-	u16 EEPROMId;
-
-	/*  Checl 0x8129 again for making sure autoload status!! */
-	EEPROMId = le16_to_cpu(*((__le16 *) hwinfo));
-	if (EEPROMId != RTL_EEPROM_ID) {
-		DBG_8723A("EEPROM ID(%#x) is invalid!!\n", EEPROMId);
-		pEEPROM->bautoload_fail_flag = true;
-	} else {
-		pEEPROM->bautoload_fail_flag = false;
-	}
-
-	RT_TRACE(_module_hal_init_c_, _drv_info_,
-		 "EEPROM ID = 0x%04x\n", EEPROMId);
-}
-
-static void
-Hal_ReadPowerValueFromPROM_8723A(struct txpowerinfo *pwrInfo,
-				 u8 *PROMContent, bool AutoLoadFail)
-{
-	u32 rfPath, eeAddr, group, rfPathMax = 1;
-
-	memset(pwrInfo, 0, sizeof(*pwrInfo));
-
-	if (AutoLoadFail) {
-		for (group = 0; group < MAX_CHNL_GROUP; group++) {
-			for (rfPath = 0; rfPath < rfPathMax; rfPath++) {
-				pwrInfo->CCKIndex[rfPath][group] =
-					EEPROM_Default_TxPowerLevel;
-				pwrInfo->HT40_1SIndex[rfPath][group] =
-					EEPROM_Default_TxPowerLevel;
-				pwrInfo->HT40_2SIndexDiff[rfPath][group] =
-					EEPROM_Default_HT40_2SDiff;
-				pwrInfo->HT20IndexDiff[rfPath][group] =
-					EEPROM_Default_HT20_Diff;
-				pwrInfo->OFDMIndexDiff[rfPath][group] =
-					EEPROM_Default_LegacyHTTxPowerDiff;
-				pwrInfo->HT40MaxOffset[rfPath][group] =
-					EEPROM_Default_HT40_PwrMaxOffset;
-				pwrInfo->HT20MaxOffset[rfPath][group] =
-					EEPROM_Default_HT20_PwrMaxOffset;
-			}
-		}
-		pwrInfo->TSSI_A[0] = EEPROM_Default_TSSI;
-		return;
-	}
-
-	for (rfPath = 0; rfPath < rfPathMax; rfPath++) {
-		for (group = 0; group < MAX_CHNL_GROUP; group++) {
-			eeAddr =
-			    EEPROM_CCK_TX_PWR_INX_8723A + (rfPath * 3) + group;
-
-			pwrInfo->CCKIndex[rfPath][group] = PROMContent[eeAddr];
-			if (pwrInfo->CCKIndex[rfPath][group] > 63)
-				pwrInfo->CCKIndex[rfPath][group] =
-					EEPROM_Default_TxPowerLevel;
-
-			eeAddr = EEPROM_HT40_1S_TX_PWR_INX_8723A +
-				(rfPath * 3) + group;
-			pwrInfo->HT40_1SIndex[rfPath][group] =
-				PROMContent[eeAddr];
-			if (pwrInfo->HT40_1SIndex[rfPath][group] > 63)
-				pwrInfo->HT40_1SIndex[rfPath][group] =
-					EEPROM_Default_TxPowerLevel;
-		}
-	}
-
-	for (group = 0; group < MAX_CHNL_GROUP; group++) {
-		for (rfPath = 0; rfPath < rfPathMax; rfPath++) {
-			pwrInfo->HT40_2SIndexDiff[rfPath][group] = 0;
-			pwrInfo->HT20IndexDiff[rfPath][group] =
-				(PROMContent
-				 [EEPROM_HT20_TX_PWR_INX_DIFF_8723A +
-				  group] >> (rfPath * 4)) & 0xF;
-			/* 4bit sign number to 8 bit sign number */
-			if (pwrInfo->HT20IndexDiff[rfPath][group] & BIT(3))
-				pwrInfo->HT20IndexDiff[rfPath][group] |= 0xF0;
-
-			pwrInfo->OFDMIndexDiff[rfPath][group] =
-				(PROMContent[EEPROM_OFDM_TX_PWR_INX_DIFF_8723A +
-					     group] >> (rfPath * 4)) & 0xF;
-
-			pwrInfo->HT40MaxOffset[rfPath][group] =
-				(PROMContent[EEPROM_HT40_MAX_PWR_OFFSET_8723A +
-					     group] >> (rfPath * 4)) & 0xF;
-
-			pwrInfo->HT20MaxOffset[rfPath][group] =
-				(PROMContent[EEPROM_HT20_MAX_PWR_OFFSET_8723A +
-					     group] >> (rfPath * 4)) & 0xF;
-		}
-	}
-
-	pwrInfo->TSSI_A[0] = PROMContent[EEPROM_TSSI_A_8723A];
-}
-
-static u8 Hal_GetChnlGroup(u8 chnl)
-{
-	u8 group = 0;
-
-	if (chnl < 3)		/*  Cjanel 1-3 */
-		group = 0;
-	else if (chnl < 9)	/*  Channel 4-9 */
-		group = 1;
-	else			/*  Channel 10-14 */
-		group = 2;
-
-	return group;
-}
-
-void
-Hal_EfuseParsetxpowerinfo_8723A(struct rtw_adapter *padapter,
-				u8 *PROMContent, bool AutoLoadFail)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct txpowerinfo pwrInfo;
-	u8 rfPath, ch, group, rfPathMax = 1;
-	u8 pwr, diff;
-
-	Hal_ReadPowerValueFromPROM_8723A(&pwrInfo, PROMContent, AutoLoadFail);
-	for (rfPath = 0; rfPath < rfPathMax; rfPath++) {
-		for (ch = 0; ch < CHANNEL_MAX_NUMBER; ch++) {
-			group = Hal_GetChnlGroup(ch);
-
-			pHalData->TxPwrLevelCck[rfPath][ch] =
-				pwrInfo.CCKIndex[rfPath][group];
-			pHalData->TxPwrLevelHT40_1S[rfPath][ch] =
-				pwrInfo.HT40_1SIndex[rfPath][group];
-
-			pHalData->TxPwrHt20Diff[rfPath][ch] =
-				pwrInfo.HT20IndexDiff[rfPath][group];
-			pHalData->TxPwrLegacyHtDiff[rfPath][ch] =
-				pwrInfo.OFDMIndexDiff[rfPath][group];
-			pHalData->PwrGroupHT20[rfPath][ch] =
-				pwrInfo.HT20MaxOffset[rfPath][group];
-			pHalData->PwrGroupHT40[rfPath][ch] =
-				pwrInfo.HT40MaxOffset[rfPath][group];
-
-			pwr = pwrInfo.HT40_1SIndex[rfPath][group];
-			diff = pwrInfo.HT40_2SIndexDiff[rfPath][group];
-
-			pHalData->TxPwrLevelHT40_2S[rfPath][ch] =
-			    (pwr > diff) ? (pwr - diff) : 0;
-		}
-	}
-	for (rfPath = 0; rfPath < RF_PATH_MAX; rfPath++) {
-		for (ch = 0; ch < CHANNEL_MAX_NUMBER; ch++) {
-			RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-				 "RF(%u)-Ch(%u) [CCK / HT40_1S / HT40_2S] = [0x%x / 0x%x / 0x%x]\n",
-				 rfPath, ch,
-				 pHalData->TxPwrLevelCck[rfPath][ch],
-				 pHalData->TxPwrLevelHT40_1S[rfPath][ch],
-				 pHalData->TxPwrLevelHT40_2S[rfPath][ch]);
-
-		}
-	}
-	for (ch = 0; ch < CHANNEL_MAX_NUMBER; ch++) {
-		RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-			 "RF-A Ht20 to HT40 Diff[%u] = 0x%x(%d)\n", ch,
-			 pHalData->TxPwrHt20Diff[RF_PATH_A][ch],
-			 pHalData->TxPwrHt20Diff[RF_PATH_A][ch]);
-	}
-	for (ch = 0; ch < CHANNEL_MAX_NUMBER; ch++)
-		RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-			 "RF-A Legacy to Ht40 Diff[%u] = 0x%x\n", ch,
-			 pHalData->TxPwrLegacyHtDiff[RF_PATH_A][ch]);
-	for (ch = 0; ch < CHANNEL_MAX_NUMBER; ch++) {
-		RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-			 "RF-B Ht20 to HT40 Diff[%u] = 0x%x(%d)\n", ch,
-			 pHalData->TxPwrHt20Diff[RF_PATH_B][ch],
-			 pHalData->TxPwrHt20Diff[RF_PATH_B][ch]);
-	}
-	for (ch = 0; ch < CHANNEL_MAX_NUMBER; ch++)
-		RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-			 "RF-B Legacy to HT40 Diff[%u] = 0x%x\n", ch,
-			 pHalData->TxPwrLegacyHtDiff[RF_PATH_B][ch]);
-	if (!AutoLoadFail) {
-		struct registry_priv *registry_par = &padapter->registrypriv;
-		if (registry_par->regulatory_tid == 0xff) {
-			if (PROMContent[RF_OPTION1_8723A] == 0xff)
-				pHalData->EEPROMRegulatory = 0;
-			else
-				pHalData->EEPROMRegulatory =
-					PROMContent[RF_OPTION1_8723A] & 0x7;
-		} else {
-			pHalData->EEPROMRegulatory =
-			    registry_par->regulatory_tid;
-		}
-	} else {
-		pHalData->EEPROMRegulatory = 0;
-	}
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "EEPROMRegulatory = 0x%x\n", pHalData->EEPROMRegulatory);
-
-	if (!AutoLoadFail)
-		pHalData->bTXPowerDataReadFromEEPORM = true;
-}
-
-void
-Hal_EfuseParseBTCoexistInfo_8723A(struct rtw_adapter *padapter,
-				  u8 *hwinfo, bool AutoLoadFail)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	u8 tempval;
-	u32 tmpu4;
-
-	if (!AutoLoadFail) {
-		tmpu4 = rtl8723au_read32(padapter, REG_MULTI_FUNC_CTRL);
-		if (tmpu4 & BT_FUNC_EN)
-			pHalData->EEPROMBluetoothCoexist = 1;
-		else
-			pHalData->EEPROMBluetoothCoexist = 0;
-		pHalData->EEPROMBluetoothType = BT_RTL8723A;
-
-		/*  The following need to be checked with newer version of */
-		/*  eeprom spec */
-		tempval = hwinfo[RF_OPTION4_8723A];
-		pHalData->EEPROMBluetoothAntNum = (tempval & 0x1);
-		pHalData->EEPROMBluetoothAntIsolation = (tempval & 0x10) >> 4;
-		pHalData->EEPROMBluetoothRadioShared = (tempval & 0x20) >> 5;
-	} else {
-		pHalData->EEPROMBluetoothCoexist = 0;
-		pHalData->EEPROMBluetoothType = BT_RTL8723A;
-		pHalData->EEPROMBluetoothAntNum = Ant_x2;
-		pHalData->EEPROMBluetoothAntIsolation = 0;
-		pHalData->EEPROMBluetoothRadioShared = BT_Radio_Shared;
-	}
-
-	rtl8723a_BT_init_hal_vars(padapter);
-}
-
-void
-Hal_EfuseParseEEPROMVer(struct rtw_adapter *padapter,
-			u8 *hwinfo, bool AutoLoadFail)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (!AutoLoadFail)
-		pHalData->EEPROMVersion = hwinfo[EEPROM_VERSION_8723A];
-	else
-		pHalData->EEPROMVersion = 1;
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "Hal_EfuseParseEEPROMVer(), EEVer = %d\n",
-		 pHalData->EEPROMVersion);
-}
-
-void
-rtl8723a_EfuseParseChnlPlan(struct rtw_adapter *padapter,
-			    u8 *hwinfo, bool AutoLoadFail)
-{
-	padapter->mlmepriv.ChannelPlan =
-		hal_com_get_channel_plan23a(padapter, hwinfo ?
-					 hwinfo[EEPROM_ChannelPlan_8723A]:0xFF,
-					 padapter->registrypriv.channel_plan,
-					 RT_CHANNEL_DOMAIN_WORLD_WIDE_13,
-					 AutoLoadFail);
-
-	DBG_8723A("mlmepriv.ChannelPlan = 0x%02x\n",
-		  padapter->mlmepriv.ChannelPlan);
-}
-
-void
-Hal_EfuseParseCustomerID(struct rtw_adapter *padapter,
-			 u8 *hwinfo, bool AutoLoadFail)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	if (!AutoLoadFail) {
-		pHalData->EEPROMCustomerID = hwinfo[EEPROM_CustomID_8723A];
-		pHalData->EEPROMSubCustomerID =
-		    hwinfo[EEPROM_SubCustomID_8723A];
-	} else {
-		pHalData->EEPROMCustomerID = 0;
-		pHalData->EEPROMSubCustomerID = 0;
-	}
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "EEPROM Customer ID: 0x%2x\n", pHalData->EEPROMCustomerID);
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "EEPROM SubCustomer ID: 0x%02x\n",
-		 pHalData->EEPROMSubCustomerID);
-}
-
-void
-Hal_EfuseParseAntennaDiversity(struct rtw_adapter *padapter,
-			       u8 *hwinfo, bool AutoLoadFail)
-{
-}
-
-void
-Hal_EfuseParseRateIndicationOption(struct rtw_adapter *padapter,
-				   u8 *hwinfo, bool AutoLoadFail)
-{
-}
-
-void
-Hal_EfuseParseXtal_8723A(struct rtw_adapter *pAdapter,
-			 u8 *hwinfo, u8 AutoLoadFail)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter);
-
-	if (!AutoLoadFail) {
-		pHalData->CrystalCap = hwinfo[EEPROM_XTAL_K_8723A];
-		if (pHalData->CrystalCap == 0xFF)
-			pHalData->CrystalCap = EEPROM_Default_CrystalCap_8723A;
-	} else {
-		pHalData->CrystalCap = EEPROM_Default_CrystalCap_8723A;
-	}
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "%s: CrystalCap = 0x%2x\n", __func__,
-		 pHalData->CrystalCap);
-}
-
-void
-Hal_EfuseParseThermalMeter_8723A(struct rtw_adapter *padapter,
-				 u8 *PROMContent, bool AutoloadFail)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-
-	/*  */
-	/*  ThermalMeter from EEPROM */
-	/*  */
-	if (!AutoloadFail)
-		pHalData->EEPROMThermalMeter =
-		    PROMContent[EEPROM_THERMAL_METER_8723A];
-	else
-		pHalData->EEPROMThermalMeter = EEPROM_Default_ThermalMeter;
-
-	if ((pHalData->EEPROMThermalMeter == 0xff) || AutoloadFail) {
-		pHalData->bAPKThermalMeterIgnore = true;
-		pHalData->EEPROMThermalMeter = EEPROM_Default_ThermalMeter;
-	}
-
-	DBG_8723A("%s: ThermalMeter = 0x%x\n", __func__,
-		  pHalData->EEPROMThermalMeter);
-}
-
-static void rtl8723a_cal_txdesc_chksum(struct tx_desc *ptxdesc)
-{
-	__le16 *usPtr = (__le16 *)ptxdesc;
-	u32 count = 16;		/*  (32 bytes / 2 bytes per XOR) => 16 times */
-	u32 index;
-	u16 checksum = 0;
-
-	/*  Clear first */
-	ptxdesc->txdw7 &= cpu_to_le32(0xffff0000);
-
-	for (index = 0; index < count; index++)
-		checksum ^= le16_to_cpu(usPtr[index]);
-
-	ptxdesc->txdw7 |= cpu_to_le32(checksum & 0x0000ffff);
-}
-
-/*
- *  Description: In normal chip, we should send some packet to Hw which
- *  will be used by Fw in FW LPS mode. The function is to fill the Tx
- * descriptor of this packets, then
- */
-/*			Fw can tell Hw to send these packet derectly. */
-/*  Added by tynli. 2009.10.15. */
-/*  */
-void rtl8723a_fill_fake_txdesc(struct rtw_adapter *padapter, u8 *pDesc,
-			       u32 BufferLen, u8 IsPsPoll, u8 IsBTQosNull)
-{
-	struct tx_desc *ptxdesc;
-
-	/*  Clear all status */
-	ptxdesc = (struct tx_desc *)pDesc;
-	memset(pDesc, 0, TXDESC_SIZE);
-
-	/* offset 0 */
-	/* own, bFirstSeg, bLastSeg; */
-	ptxdesc->txdw0 |= cpu_to_le32(OWN | FSG | LSG);
-
-	/* 32 bytes for TX Desc */
-	ptxdesc->txdw0 |= cpu_to_le32(((TXDESC_SIZE + OFFSET_SZ) <<
-				       OFFSET_SHT) & 0x00ff0000);
-
-	/*  Buffer size + command header */
-	ptxdesc->txdw0 |= cpu_to_le32(BufferLen & 0x0000ffff);
-
-	/* offset 4 */
-	/*  Fixed queue of Mgnt queue */
-	ptxdesc->txdw1 |= cpu_to_le32((QSLT_MGNT << QSEL_SHT) & 0x00001f00);
-
-	/* Set NAVUSEHDR to prevent Ps-poll AId filed to be changed
-	   to error vlaue by Hw. */
-	if (IsPsPoll) {
-		ptxdesc->txdw1 |= cpu_to_le32(NAVUSEHDR);
-	} else {
-		/*  Hw set sequence number */
-		ptxdesc->txdw4 |= cpu_to_le32(BIT(7));
-		/* set bit3 to 1. Suugested by TimChen. 2009.12.29. */
-		ptxdesc->txdw3 |= cpu_to_le32((8 << 28));
-	}
-
-	if (true == IsBTQosNull)
-		ptxdesc->txdw2 |= cpu_to_le32(BIT(23));	/*  BT NULL */
-
-	/* offset 16 */
-	ptxdesc->txdw4 |= cpu_to_le32(BIT(8));	/* driver uses rate */
-
-	/*  USB interface drop packet if the checksum of descriptor isn't
-	    correct. */
-	/*  Using this checksum can let hardware recovery from packet bulk
-	    out error (e.g. Cancel URC, Bulk out error.). */
-	rtl8723a_cal_txdesc_chksum(ptxdesc);
-}
-
-void hw_var_set_opmode(struct rtw_adapter *padapter, u8 mode)
-{
-	u8 val8;
-
-	if (mode == MSR_INFRA || mode == MSR_NOLINK) {
-		StopTxBeacon(padapter);
-
-		/*  disable atim wnd */
-		val8 = DIS_TSF_UDT | EN_BCN_FUNCTION | DIS_ATIM;
-		SetBcnCtrlReg23a(padapter, val8, ~val8);
-	} else if (mode == MSR_ADHOC) {
-		ResumeTxBeacon(padapter);
-
-		val8 = DIS_TSF_UDT | EN_BCN_FUNCTION | DIS_BCNQ_SUB;
-		SetBcnCtrlReg23a(padapter, val8, ~val8);
-	} else if (mode == MSR_AP) {
-		/*  add NULL Data and BT NULL Data Packets to FW RSVD Page */
-		rtl8723a_set_BTCoex_AP_mode_FwRsvdPkt_cmd(padapter);
-
-		ResumeTxBeacon(padapter);
-
-		val8 = DIS_TSF_UDT | DIS_BCNQ_SUB;
-		SetBcnCtrlReg23a(padapter, val8, ~val8);
-
-		/*  Set RCR */
-		/* rtl8723au_write32(padapter, REG_RCR, 0x70002a8e);
-		   CBSSID_DATA must set to 0 */
-		/* CBSSID_DATA must set to 0 */
-		rtl8723au_write32(padapter, REG_RCR, 0x7000228e);
-		/*  enable to rx data frame */
-		rtl8723au_write16(padapter, REG_RXFLTMAP2, 0xFFFF);
-		/*  enable to rx ps-poll */
-		rtl8723au_write16(padapter, REG_RXFLTMAP1, 0x0400);
-
-		/*  Beacon Control related register for first time */
-		/*  2ms */
-		rtl8723au_write8(padapter, REG_BCNDMATIM, 0x02);
-		/*  5ms */
-		rtl8723au_write8(padapter, REG_DRVERLYINT, 0x05);
-		/*  10ms for port0 */
-		rtl8723au_write8(padapter, REG_ATIMWND, 0x0a);
-		rtl8723au_write16(padapter, REG_BCNTCFG, 0x00);
-		rtl8723au_write16(padapter, REG_TBTT_PROHIBIT, 0xff04);
-		/*  +32767 (~32ms) */
-		rtl8723au_write16(padapter, REG_TSFTR_SYN_OFFSET, 0x7fff);
-
-		/*  reset TSF */
-		rtl8723au_write8(padapter, REG_DUAL_TSF_RST, BIT(0));
-
-		/*  enable BCN Function */
-		/*  don't enable update TSF (due to TSF update when
-		    beacon/probe rsp are received) */
-		val8 = DIS_TSF_UDT | EN_BCN_FUNCTION |
-		       EN_TXBCN_RPT | DIS_BCNQ_SUB;
-		SetBcnCtrlReg23a(padapter, val8, ~val8);
-	}
-
-	val8 = rtl8723au_read8(padapter, MSR);
-	val8 = (val8 & 0xC) | mode;
-	rtl8723au_write8(padapter, MSR, val8);
-}
-
-void hw_var_set_macaddr(struct rtw_adapter *padapter, u8 *val)
-{
-	u8 idx = 0;
-	u32 reg_macid;
-
-	reg_macid = REG_MACID;
-
-	for (idx = 0; idx < 6; idx++)
-		rtl8723au_write8(padapter, (reg_macid + idx), val[idx]);
-}
-
-void hw_var_set_bssid(struct rtw_adapter *padapter, u8 *val)
-{
-	u8 idx = 0;
-	u32 reg_bssid;
-
-	reg_bssid = REG_BSSID;
-
-	for (idx = 0; idx < 6; idx++)
-		rtl8723au_write8(padapter, (reg_bssid + idx), val[idx]);
-}
-
-void hw_var_set_correct_tsf(struct rtw_adapter *padapter)
-{
-	u64 tsf;
-	u32 reg_tsftr;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-
-	/* tsf = pmlmeext->TSFValue - ((u32)pmlmeext->TSFValue %
-	   (pmlmeinfo->bcn_interval*1024)) - 1024; us */
-	tsf = pmlmeext->TSFValue -
-		do_div(pmlmeext->TSFValue,
-		       (pmlmeinfo->bcn_interval * 1024)) - 1024;	/* us */
-
-	if (((pmlmeinfo->state & 0x03) == MSR_ADHOC) ||
-	    ((pmlmeinfo->state & 0x03) == MSR_AP)) {
-		/* pHalData->RegTxPause |= STOP_BCNQ;BIT(6) */
-		/* rtl8723au_write8(padapter, REG_TXPAUSE,
-		   (rtl8723au_read8(Adapter, REG_TXPAUSE)|BIT(6))); */
-		StopTxBeacon(padapter);
-	}
-
-	reg_tsftr = REG_TSFTR;
-
-	/*  disable related TSF function */
-	SetBcnCtrlReg23a(padapter, 0, EN_BCN_FUNCTION);
-
-	rtl8723au_write32(padapter, reg_tsftr, tsf);
-	rtl8723au_write32(padapter, reg_tsftr + 4, tsf >> 32);
-
-	/* enable related TSF function */
-	SetBcnCtrlReg23a(padapter, EN_BCN_FUNCTION, 0);
-
-	if (((pmlmeinfo->state & 0x03) == MSR_ADHOC) ||
-	    ((pmlmeinfo->state & 0x03) == MSR_AP))
-		ResumeTxBeacon(padapter);
-}
-
-void hw_var_set_mlme_disconnect(struct rtw_adapter *padapter)
-{
-	/*  reject all data frames */
-	rtl8723au_write16(padapter, REG_RXFLTMAP2, 0);
-
-	/*  reset TSF */
-	rtl8723au_write8(padapter, REG_DUAL_TSF_RST, BIT(0));
-
-	/*  disable update TSF */
-	SetBcnCtrlReg23a(padapter, DIS_TSF_UDT, 0);
-}
-
-void hw_var_set_mlme_join(struct rtw_adapter *padapter, u8 type)
-{
-	u8 RetryLimit = 0x30;
-
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	if (type == 0) {	/*  prepare to join */
-		u32 v32;
-
-		/*  enable to rx data frame.Accept all data frame */
-		/* rtl8723au_write32(padapter, REG_RCR,
-		   rtl8723au_read32(padapter, REG_RCR)|RCR_ADF); */
-		rtl8723au_write16(padapter, REG_RXFLTMAP2, 0xFFFF);
-
-		v32 = rtl8723au_read32(padapter, REG_RCR);
-		v32 |= RCR_CBSSID_DATA | RCR_CBSSID_BCN;
-		rtl8723au_write32(padapter, REG_RCR, v32);
-
-		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) == true)
-			RetryLimit =
-			    (pHalData->CustomerID == RT_CID_CCX) ? 7 : 48;
-		else		/*  Ad-hoc Mode */
-			RetryLimit = 0x7;
-	} else if (type == 1) {	/*  joinbss_event callback when join res < 0 */
-		/*  config RCR to receive different BSSID & not to
-		    receive data frame during linking */
-		rtl8723au_write16(padapter, REG_RXFLTMAP2, 0);
-	} else if (type == 2) {	/*  sta add event callback */
-		/*  enable update TSF */
-		SetBcnCtrlReg23a(padapter, 0, DIS_TSF_UDT);
-
-		if (check_fwstate(pmlmepriv,
-				  WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE)) {
-			/*  fixed beacon issue for 8191su........... */
-			rtl8723au_write8(padapter, 0x542, 0x02);
-			RetryLimit = 0x7;
-		}
-	}
-
-	rtl8723au_write16(padapter, REG_RL,
-			  RetryLimit << RETRY_LIMIT_SHORT_SHIFT | RetryLimit <<
-			  RETRY_LIMIT_LONG_SHIFT);
-
-	switch (type) {
-	case 0:
-		/*  prepare to join */
-		rtl8723a_BT_wifiassociate_notify(padapter, true);
-		break;
-	case 1:
-		/*  joinbss_event callback when join res < 0 */
-		rtl8723a_BT_wifiassociate_notify(padapter, false);
-		break;
-	case 2:
-		/*  sta add event callback */
-/*		BT_WifiMediaStatusNotify(padapter, RT_MEDIA_CONNECT); */
-		break;
-	}
-}
diff --git a/drivers/staging/rtl8723au/hal/rtl8723a_phycfg.c b/drivers/staging/rtl8723au/hal/rtl8723a_phycfg.c
deleted file mode 100644
index 06a6c3eeeb33..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723a_phycfg.c
+++ /dev/null
@@ -1,961 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTL8723A_PHYCFG_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#include <rtl8723a_hal.h>
-#include <usb_ops_linux.h>
-
-/*---------------------------Define Local Constant---------------------------*/
-/* Channel switch:The size of command tables for switch channel*/
-#define MAX_PRECMD_CNT 16
-#define MAX_RFDEPENDCMD_CNT 16
-#define MAX_POSTCMD_CNT 16
-
-#define MAX_DOZE_WAITING_TIMES_9x 64
-
-/*---------------------------Define Local Constant---------------------------*/
-
-/*------------------------Define global variable-----------------------------*/
-
-/*------------------------Define local variable------------------------------*/
-
-/*--------------------Define export function prototype-----------------------*/
-/*  Please refer to header file */
-/*--------------------Define export function prototype-----------------------*/
-
-/*----------------------------Function Body----------------------------------*/
-/*  */
-/*  1. BB register R/W API */
-/*  */
-
-/**
-* Function:	phy_CalculateBitShift
-*
-* OverView:	Get shifted position of the BitMask
-*
-* Input:
-*			u32		BitMask,
-*
-* Output:	none
-* Return:		u32		Return the shift bit bit position of the mask
-*/
-static	u32 phy_CalculateBitShift(u32 BitMask)
-{
-	u32 i;
-
-	for (i = 0; i <= 31; i++) {
-		if (((BitMask>>i) & 0x1) == 1)
-			break;
-	}
-
-	return i;
-}
-
-/**
-* Function:	PHY_QueryBBReg
-*
-* OverView:	Read "sepcific bits" from BB register
-*
-* Input:
-*	struct rtw_adapter *	Adapter,
-*	u32			RegAddr,	Target address to be readback
-*	u32			BitMask		Target bit position in the
-*						target address to be readback
-* Output:
-*	None
-* Return:
-*	u32			Data		The readback register value
-* Note:
-*	This function is equal to "GetRegSetting" in PHY programming guide
-*/
-u32
-PHY_QueryBBReg(struct rtw_adapter *Adapter, u32 RegAddr, u32 BitMask)
-{
-	u32	ReturnValue = 0, OriginalValue, BitShift;
-
-	OriginalValue = rtl8723au_read32(Adapter, RegAddr);
-	BitShift = phy_CalculateBitShift(BitMask);
-	ReturnValue = (OriginalValue & BitMask) >> BitShift;
-	return ReturnValue;
-}
-
-/**
-* Function:	PHY_SetBBReg
-*
-* OverView:	Write "Specific bits" to BB register (page 8~)
-*
-* Input:
-*	struct rtw_adapter *	Adapter,
-*	u32			RegAddr,	Target address to be modified
-*	u32			BitMask		Target bit position in the
-*						target address to be modified
-*	u32			Data		The new register value in the
-*						target bit position of the
-*						 target address
-*
-* Output:
-*	None
-* Return:
-*	None
-* Note:
-*	This function is equal to "PutRegSetting" in PHY programming guide
-*/
-
-void
-PHY_SetBBReg(struct rtw_adapter *Adapter, u32 RegAddr, u32 BitMask, u32	Data)
-{
-	u32 OriginalValue, BitShift;
-
-	if (BitMask != bMaskDWord) {/* if not "double word" write */
-		OriginalValue = rtl8723au_read32(Adapter, RegAddr);
-		BitShift = phy_CalculateBitShift(BitMask);
-		Data = (OriginalValue & (~BitMask)) | (Data << BitShift);
-	}
-
-	rtl8723au_write32(Adapter, RegAddr, Data);
-
-	/* RTPRINT(FPHY, PHY_BBW, ("BBW MASK = 0x%lx Addr[0x%lx]= 0x%lx\n", BitMask, RegAddr, Data)); */
-}
-
-/*  */
-/*  2. RF register R/W API */
-/*  */
-
-/**
-* Function:	phy_RFSerialRead
-*
-* OverView:	Read regster from RF chips
-*
-* Input:
-*		struct rtw_adapter *		Adapter,
-*		enum RF_RADIO_PATH	eRFPath,	Radio path of A/B/C/D
-*		u32 Offset,			The target address to be read
-*
-* Output:	None
-* Return:	u32			reback value
-* Note:		Threre are three types of serial operations:
-*		1. Software serial write
-*		2. Hardware LSSI-Low Speed Serial Interface
-*		3. Hardware HSSI-High speed
-*		serial write. Driver need to implement (1) and (2).
-*		This function is equal to the combination of RF_ReadReg() and
-*		RFLSSIRead()
-*/
-static u32
-phy_RFSerialRead(struct rtw_adapter *Adapter, enum RF_RADIO_PATH eRFPath,
-		 u32 Offset)
-{
-	u32 retValue = 0;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct bb_reg_define *pPhyReg = &pHalData->PHYRegDef[eRFPath];
-	u32 NewOffset;
-	u32 tmplong, tmplong2;
-	u8 RfPiEnable = 0;
-	/*  */
-	/*  Make sure RF register offset is correct */
-	/*  */
-	Offset &= 0x3f;
-
-	/*  */
-	/*  Switch page for 8256 RF IC */
-	/*  */
-	NewOffset = Offset;
-
-	/*  2009/06/17 MH We can not execute IO for power save or
-	    other accident mode. */
-	/* if (RT_CANNOT_IO(Adapter)) */
-	/*  */
-	/*	RTPRINT(FPHY, PHY_RFR, ("phy_RFSerialRead return all one\n")); */
-	/*	return	0xFFFFFFFF; */
-	/*  */
-
-	/*  For 92S LSSI Read RFLSSIRead */
-	/*  For RF A/B write 0x824/82c(does not work in the future) */
-	/*  We must use 0x824 for RF A and B to execute read trigger */
-	tmplong = rtl8723au_read32(Adapter, rFPGA0_XA_HSSIParameter2);
-	if (eRFPath == RF_PATH_A)
-		tmplong2 = tmplong;
-	else
-		tmplong2 = rtl8723au_read32(Adapter, pPhyReg->rfHSSIPara2);
-
-	tmplong2 = (tmplong2 & ~bLSSIReadAddress) |
-		(NewOffset << 23) | bLSSIReadEdge;	/* T65 RF */
-
-	rtl8723au_write32(Adapter, rFPGA0_XA_HSSIParameter2,
-			  tmplong & (~bLSSIReadEdge));
-	udelay(10);/*  PlatformStallExecution(10); */
-
-	rtl8723au_write32(Adapter, pPhyReg->rfHSSIPara2, tmplong2);
-	udelay(100);/* PlatformStallExecution(100); */
-
-	rtl8723au_write32(Adapter, rFPGA0_XA_HSSIParameter2,
-			  tmplong | bLSSIReadEdge);
-	udelay(10);/* PlatformStallExecution(10); */
-
-	if (eRFPath == RF_PATH_A)
-		RfPiEnable = (u8)PHY_QueryBBReg(Adapter,
-						rFPGA0_XA_HSSIParameter1,
-						BIT(8));
-	else if (eRFPath == RF_PATH_B)
-		RfPiEnable = (u8)PHY_QueryBBReg(Adapter,
-						rFPGA0_XB_HSSIParameter1,
-						BIT(8));
-
-	if (RfPiEnable)	{
-		/* Read from BBreg8b8, 12 bits for 8190, 20bits for T65 RF */
-		retValue = PHY_QueryBBReg(Adapter, pPhyReg->rfLSSIReadBackPi,
-					  bLSSIReadBackData);
-		/* DBG_8723A("Readback from RF-PI : 0x%x\n", retValue); */
-	} else {
-		/* Read from BBreg8a0, 12 bits for 8190, 20 bits for T65 RF */
-		retValue = PHY_QueryBBReg(Adapter, pPhyReg->rfLSSIReadBack,
-					  bLSSIReadBackData);
-		/* DBG_8723A("Readback from RF-SI : 0x%x\n", retValue); */
-	}
-	/* DBG_8723A("RFR-%d Addr[0x%x]= 0x%x\n", eRFPath, pPhyReg->rfLSSIReadBack, retValue); */
-
-	return retValue;
-}
-
-/**
-* Function:	phy_RFSerialWrite
-*
-* OverView:	Write data to RF register (page 8~)
-*
-* Input:
-*	struct rtw_adapter *		Adapter,
-*	enum RF_RADIO_PATH	eRFPath,	Radio path of A/B/C/D
-*	u32 Offset,			The target address to be read
-*	u32 Data			The new register Data in the target
-*					bit position of the target to be read
-*
-* Output:
-*	None
-* Return:
-*	None
-* Note:
-*	Threre are three types of serial operations:
-*		1. Software serial write
-*		2. Hardware LSSI-Low Speed Serial Interface
-*		3. Hardware HSSI-High speed
-*		serial write. Driver need to implement (1) and (2).
-*		This function is equal to the combination of RF_ReadReg() and
-*		RFLSSIRead()
-*
-* Note:	  For RF8256 only
-* The total count of RTL8256(Zebra4) register is around 36 bit it only employs
-* 4-bit RF address. RTL8256 uses "register mode control bit"
-* (Reg00[12], Reg00[10]) to access register address bigger than 0xf.
-* See "Appendix-4 in PHY Configuration programming guide" for more details.
-* Thus, we define a sub-finction for RTL8526 register address conversion
-* ===========================================================
-* Register Mode:	RegCTL[1]	RegCTL[0]	Note
-*			(Reg00[12])	(Reg00[10])
-* ===========================================================
-* Reg_Mode0		0		x		Reg 0 ~15(0x0 ~ 0xf)
-* ------------------------------------------------------------------
-* Reg_Mode1		1		0		Reg 16 ~30(0x1 ~ 0xf)
-* ------------------------------------------------------------------
-* Reg_Mode2		1		1		Reg 31 ~ 45(0x1 ~ 0xf)
-* ------------------------------------------------------------------
-*
-*	2008/09/02	MH	Add 92S RF definition
-*/
-static	void
-phy_RFSerialWrite(struct rtw_adapter *Adapter, enum RF_RADIO_PATH eRFPath,
-		  u32 Offset, u32 Data)
-{
-	u32 DataAndAddr = 0;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct bb_reg_define *pPhyReg = &pHalData->PHYRegDef[eRFPath];
-	u32 NewOffset;
-
-	/*  2009/06/17 MH We can not execute IO for power save or
-	    other accident mode. */
-	/* if (RT_CANNOT_IO(Adapter)) */
-	/*  */
-	/*	RTPRINT(FPHY, PHY_RFW, ("phy_RFSerialWrite stop\n")); */
-	/*	return; */
-	/*  */
-
-	Offset &= 0x3f;
-
-	/*  */
-	/*  Shadow Update */
-	/*  */
-	/* PHY_RFShadowWrite(Adapter, eRFPath, Offset, Data); */
-
-	/*  */
-	/*  Switch page for 8256 RF IC */
-	/*  */
-	NewOffset = Offset;
-
-	/*  */
-	/*  Put write addr in [5:0]  and write data in [31:16] */
-	/*  */
-	/* DataAndAddr = (Data<<16) | (NewOffset&0x3f); */
-	/*  T65 RF */
-	DataAndAddr = ((NewOffset<<20) | (Data&0x000fffff)) & 0x0fffffff;
-
-	/*  */
-	/*  Write Operation */
-	/*  */
-	rtl8723au_write32(Adapter, pPhyReg->rf3wireOffset, DataAndAddr);
-}
-
-/**
-* Function:	PHY_QueryRFReg
-*
-* OverView:	Query "Specific bits" to RF register (page 8~)
-*
-* Input:
-*	struct rtw_adapter *		Adapter,
-*	enum RF_RADIO_PATH	eRFPath,	Radio path of A/B/C/D
-*	u32 RegAddr,			The target address to be read
-*	u32BitMask			The target bit position in the target
-*					address	to be read
-*
-* Output:
-*	None
-* Return:
-*	u32				Readback value
-* Note:
-*	This function is equal to "GetRFRegSetting" in PHY programming guide
-*/
-u32
-PHY_QueryRFReg(struct rtw_adapter *Adapter, enum RF_RADIO_PATH eRFPath,
-	       u32 RegAddr, u32 BitMask)
-{
-	u32 Original_Value, Readback_Value, BitShift;
-	/* struct hal_data_8723a	*pHalData = GET_HAL_DATA(Adapter); */
-	/* u8	RFWaitCounter = 0; */
-	/* _irqL	irqL; */
-
-	Original_Value = phy_RFSerialRead(Adapter, eRFPath, RegAddr);
-
-	BitShift =  phy_CalculateBitShift(BitMask);
-	Readback_Value = (Original_Value & BitMask) >> BitShift;
-
-	return Readback_Value;
-}
-
-/**
-* Function:	PHY_SetRFReg
-*
-* OverView:	Write "Specific bits" to RF register (page 8~)
-*
-* Input:
-*	struct rtw_adapter *		Adapter,
-*	enum RF_RADIO_PATH	eRFPath,	Radio path of A/B/C/D
-*	u32 RegAddr,			The target address to be modified
-*	u32 BitMask			The target bit position in the target
-*					address to be modified
-*	u32 Data			The new register Data in the target
-*					bit position of the target address
-*
-* Output:
-*	None
-* Return:
-*	None
-* Note:	This function is equal to "PutRFRegSetting" in PHY programming guide
-*/
-void
-PHY_SetRFReg(struct rtw_adapter *Adapter, enum RF_RADIO_PATH eRFPath,
-	     u32 RegAddr, u32 BitMask, u32 Data)
-{
-	/* struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter); */
-	/* u8 RFWaitCounter	= 0; */
-	u32 Original_Value, BitShift;
-
-	/*  RF data is 12 bits only */
-	if (BitMask != bRFRegOffsetMask) {
-		Original_Value = phy_RFSerialRead(Adapter, eRFPath, RegAddr);
-		BitShift =  phy_CalculateBitShift(BitMask);
-		Data = (Original_Value & (~BitMask)) | (Data << BitShift);
-	}
-
-	phy_RFSerialWrite(Adapter, eRFPath, RegAddr, Data);
-}
-
-/*  3. Initial MAC/BB/RF config by reading MAC/BB/RF txt. */
-
-/*-----------------------------------------------------------------------------
- * Function:    PHY_MACConfig8723A
- *
- * Overview:	Condig MAC by header file or parameter file.
- *
- * Input:       NONE
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- * Revised History:
- *  When		Who		Remark
- *  08/12/2008	MHC		Create Version 0.
- *
- *---------------------------------------------------------------------------*/
-int PHY_MACConfig8723A(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	/*  */
-	/*  Config MAC */
-	/*  */
-	ODM_ReadAndConfig_MAC_REG_8723A(&pHalData->odmpriv);
-
-	/*  2010.07.13 AMPDU aggregation number 9 */
-	rtl8723au_write8(Adapter, REG_MAX_AGGR_NUM, 0x0A);
-	if (pHalData->rf_type == RF_2T2R &&
-	    BOARD_USB_DONGLE == pHalData->BoardType)
-		rtl8723au_write8(Adapter, 0x40, 0x04);
-
-	return _SUCCESS;
-}
-
-/**
-* Function:	phy_InitBBRFRegisterDefinition
-*
-* OverView:	Initialize Register definition offset for Radio Path A/B/C/D
-*
-* Input:
-*			struct rtw_adapter *		Adapter,
-*
-* Output:	None
-* Return:		None
-* Note:
-*	The initialization value is constant and it should never be changes
-*/
-static	void
-phy_InitBBRFRegisterDefinition(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	/*  RF Interface Sowrtware Control */
-	 /*  16 LSBs if read 32-bit from 0x870 */
-	pHalData->PHYRegDef[RF_PATH_A].rfintfs = rFPGA0_XAB_RFInterfaceSW;
-	 /*  16 MSBs if read 32-bit from 0x870 (16-bit for 0x872) */
-	pHalData->PHYRegDef[RF_PATH_B].rfintfs = rFPGA0_XAB_RFInterfaceSW;
-
-	/*  RF Interface Readback Value */
-	/*  16 LSBs if read 32-bit from 0x8E0 */
-	pHalData->PHYRegDef[RF_PATH_A].rfintfi = rFPGA0_XAB_RFInterfaceRB;
-	/*  16 MSBs if read 32-bit from 0x8E0 (16-bit for 0x8E2) */
-	pHalData->PHYRegDef[RF_PATH_B].rfintfi = rFPGA0_XAB_RFInterfaceRB;
-
-	/*  RF Interface Output (and Enable) */
-	/*  16 LSBs if read 32-bit from 0x860 */
-	pHalData->PHYRegDef[RF_PATH_A].rfintfo = rFPGA0_XA_RFInterfaceOE;
-	 /*  16 LSBs if read 32-bit from 0x864 */
-	pHalData->PHYRegDef[RF_PATH_B].rfintfo = rFPGA0_XB_RFInterfaceOE;
-
-	/*  RF Interface (Output and)  Enable */
-	 /*  16 MSBs if read 32-bit from 0x860 (16-bit for 0x862) */
-	pHalData->PHYRegDef[RF_PATH_A].rfintfe = rFPGA0_XA_RFInterfaceOE;
-	/*  16 MSBs if read 32-bit from 0x864 (16-bit for 0x866) */
-	pHalData->PHYRegDef[RF_PATH_B].rfintfe = rFPGA0_XB_RFInterfaceOE;
-
-	/* Addr of LSSI. Wirte RF register by driver */
-	pHalData->PHYRegDef[RF_PATH_A].rf3wireOffset = rFPGA0_XA_LSSIParameter;
-	pHalData->PHYRegDef[RF_PATH_B].rf3wireOffset = rFPGA0_XB_LSSIParameter;
-
-	/*  RF parameter */
-	/* BB Band Select */
-	pHalData->PHYRegDef[RF_PATH_A].rfLSSI_Select = rFPGA0_XAB_RFParameter;
-	pHalData->PHYRegDef[RF_PATH_B].rfLSSI_Select = rFPGA0_XAB_RFParameter;
-
-	/*  Tx AGC Gain Stage (same for all path. Should we remove this?) */
-	pHalData->PHYRegDef[RF_PATH_A].rfTxGainStage = rFPGA0_TxGainStage;
-	pHalData->PHYRegDef[RF_PATH_B].rfTxGainStage = rFPGA0_TxGainStage;
-
-	/*  Tranceiver A~D HSSI Parameter-1 */
-	/* wire control parameter1 */
-	pHalData->PHYRegDef[RF_PATH_A].rfHSSIPara1 = rFPGA0_XA_HSSIParameter1;
-	/* wire control parameter1 */
-	pHalData->PHYRegDef[RF_PATH_B].rfHSSIPara1 = rFPGA0_XB_HSSIParameter1;
-
-	/*  Tranceiver A~D HSSI Parameter-2 */
-	/* wire control parameter2 */
-	pHalData->PHYRegDef[RF_PATH_A].rfHSSIPara2 = rFPGA0_XA_HSSIParameter2;
-	/* wire control parameter2 */
-	pHalData->PHYRegDef[RF_PATH_B].rfHSSIPara2 = rFPGA0_XB_HSSIParameter2;
-
-	/*  RF switch Control */
-	pHalData->PHYRegDef[RF_PATH_A].rfSwitchControl =
-		rFPGA0_XAB_SwitchControl; /* TR/Ant switch control */
-	pHalData->PHYRegDef[RF_PATH_B].rfSwitchControl =
-		rFPGA0_XAB_SwitchControl;
-
-	/*  AGC control 1 */
-	pHalData->PHYRegDef[RF_PATH_A].rfAGCControl1 = rOFDM0_XAAGCCore1;
-	pHalData->PHYRegDef[RF_PATH_B].rfAGCControl1 = rOFDM0_XBAGCCore1;
-
-	/*  AGC control 2 */
-	pHalData->PHYRegDef[RF_PATH_A].rfAGCControl2 = rOFDM0_XAAGCCore2;
-	pHalData->PHYRegDef[RF_PATH_B].rfAGCControl2 = rOFDM0_XBAGCCore2;
-
-	/*  RX AFE control 1 */
-	pHalData->PHYRegDef[RF_PATH_A].rfRxIQImbalance = rOFDM0_XARxIQImbalance;
-	pHalData->PHYRegDef[RF_PATH_B].rfRxIQImbalance = rOFDM0_XBRxIQImbalance;
-
-	/*  RX AFE control 1 */
-	pHalData->PHYRegDef[RF_PATH_A].rfRxAFE = rOFDM0_XARxAFE;
-	pHalData->PHYRegDef[RF_PATH_B].rfRxAFE = rOFDM0_XBRxAFE;
-
-	/*  Tx AFE control 1 */
-	pHalData->PHYRegDef[RF_PATH_A].rfTxIQImbalance = rOFDM0_XATxIQImbalance;
-	pHalData->PHYRegDef[RF_PATH_B].rfTxIQImbalance = rOFDM0_XBTxIQImbalance;
-
-	/*  Tx AFE control 2 */
-	pHalData->PHYRegDef[RF_PATH_A].rfTxAFE = rOFDM0_XATxAFE;
-	pHalData->PHYRegDef[RF_PATH_B].rfTxAFE = rOFDM0_XBTxAFE;
-
-	/*  Tranceiver LSSI Readback SI mode */
-	pHalData->PHYRegDef[RF_PATH_A].rfLSSIReadBack = rFPGA0_XA_LSSIReadBack;
-	pHalData->PHYRegDef[RF_PATH_B].rfLSSIReadBack = rFPGA0_XB_LSSIReadBack;
-
-	/*  Tranceiver LSSI Readback PI mode */
-	pHalData->PHYRegDef[RF_PATH_A].rfLSSIReadBackPi =
-		TransceiverA_HSPI_Readback;
-	pHalData->PHYRegDef[RF_PATH_B].rfLSSIReadBackPi =
-		TransceiverB_HSPI_Readback;
-}
-
-/*  The following is for High Power PA */
-static void
-storePwrIndexDiffRateOffset(struct rtw_adapter *Adapter, u32 RegAddr,
-			    u32 BitMask, u32 Data)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	if (RegAddr == rTxAGC_A_Rate18_06) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][0] = Data;
-	}
-	if (RegAddr == rTxAGC_A_Rate54_24) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][1] = Data;
-	}
-	if (RegAddr == rTxAGC_A_CCK1_Mcs32) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][6] = Data;
-	}
-	if (RegAddr == rTxAGC_B_CCK11_A_CCK2_11 && BitMask == 0xffffff00) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][7] = Data;
-	}
-	if (RegAddr == rTxAGC_A_Mcs03_Mcs00) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][2] = Data;
-	}
-	if (RegAddr == rTxAGC_A_Mcs07_Mcs04) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][3] = Data;
-	}
-	if (RegAddr == rTxAGC_A_Mcs11_Mcs08) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][4] = Data;
-	}
-	if (RegAddr == rTxAGC_A_Mcs15_Mcs12) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][5] = Data;
-	}
-	if (RegAddr == rTxAGC_B_Rate18_06) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][8] = Data;
-	}
-	if (RegAddr == rTxAGC_B_Rate54_24) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][9] = Data;
-	}
-	if (RegAddr == rTxAGC_B_CCK1_55_Mcs32) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][14] = Data;
-	}
-	if (RegAddr == rTxAGC_B_CCK11_A_CCK2_11 && BitMask == 0x000000ff) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][15] = Data;
-	}
-	if (RegAddr == rTxAGC_B_Mcs03_Mcs00) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][10] = Data;
-	}
-	if (RegAddr == rTxAGC_B_Mcs07_Mcs04) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][11] = Data;
-	}
-	if (RegAddr == rTxAGC_B_Mcs11_Mcs08) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][12] = Data;
-	}
-	if (RegAddr == rTxAGC_B_Mcs15_Mcs12) {
-		pHalData->MCSTxPowerLevelOriginalOffset[pHalData->pwrGroupCnt][13] = Data;
-		pHalData->pwrGroupCnt++;
-	}
-}
-
-/*-----------------------------------------------------------------------------
- * Function:	phy_ConfigBBWithPgHeaderFile
- *
- * Overview:	Config PHY_REG_PG array
- *
- * Input:       NONE
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- * Revised History:
- * When		Who	Remark
- * 11/06/2008	MHC	Add later!!!!!!.. Please modify for new files!!!!
- * 11/10/2008	tynli	Modify to mew files.
- *---------------------------------------------------------------------------*/
-static	int
-phy_ConfigBBWithPgHeaderFile(struct rtw_adapter *Adapter)
-{
-	int i;
-	u32 *Rtl819XPHY_REGArray_Table_PG;
-	u16 PHY_REGArrayPGLen;
-
-	PHY_REGArrayPGLen = Rtl8723_PHY_REG_Array_PGLength;
-	Rtl819XPHY_REGArray_Table_PG = (u32 *)Rtl8723_PHY_REG_Array_PG;
-
-	for (i = 0; i < PHY_REGArrayPGLen; i = i + 3) {
-		storePwrIndexDiffRateOffset(Adapter,
-					    Rtl819XPHY_REGArray_Table_PG[i],
-					    Rtl819XPHY_REGArray_Table_PG[i+1],
-					    Rtl819XPHY_REGArray_Table_PG[i+2]);
-	}
-
-	return _SUCCESS;
-}
-
-static void
-phy_BB8192C_Config_1T(struct rtw_adapter *Adapter)
-{
-	/* for path - B */
-	PHY_SetBBReg(Adapter, rFPGA0_TxInfo, 0x3, 0x2);
-	PHY_SetBBReg(Adapter, rFPGA1_TxInfo, 0x300033, 0x200022);
-
-	/*  20100519 Joseph: Add for 1T2R config. Suggested by Kevin,
-	    Jenyu and Yunan. */
-	PHY_SetBBReg(Adapter, rCCK0_AFESetting, bMaskByte3, 0x45);
-	PHY_SetBBReg(Adapter, rOFDM0_TRxPathEnable, bMaskByte0, 0x23);
-	/*  B path first AGC */
-	PHY_SetBBReg(Adapter, rOFDM0_AGCParameter1, 0x30, 0x1);
-
-	PHY_SetBBReg(Adapter, 0xe74, 0x0c000000, 0x2);
-	PHY_SetBBReg(Adapter, 0xe78, 0x0c000000, 0x2);
-	PHY_SetBBReg(Adapter, 0xe7c, 0x0c000000, 0x2);
-	PHY_SetBBReg(Adapter, 0xe80, 0x0c000000, 0x2);
-	PHY_SetBBReg(Adapter, 0xe88, 0x0c000000, 0x2);
-}
-
-static int
-phy_BB8723a_Config_ParaFile(struct rtw_adapter *Adapter)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(Adapter);
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	int rtStatus = _SUCCESS;
-
-	/*  */
-	/*  1. Read PHY_REG.TXT BB INIT!! */
-	/*  We will separate as 88C / 92C according to chip version */
-	/*  */
-	ODM_ReadAndConfig_PHY_REG_1T_8723A(&pHalData->odmpriv);
-
-	/*  */
-	/*  20100318 Joseph: Config 2T2R to 1T2R if necessary. */
-	/*  */
-	if (pHalData->rf_type == RF_1T2R) {
-		phy_BB8192C_Config_1T(Adapter);
-		DBG_8723A("phy_BB8723a_Config_ParaFile():Config to 1T!!\n");
-	}
-
-	/*  */
-	/*  2. If EEPROM or EFUSE autoload OK, We must config by
-	    PHY_REG_PG.txt */
-	/*  */
-	if (pEEPROM->bautoload_fail_flag == false) {
-		pHalData->pwrGroupCnt = 0;
-
-		rtStatus = phy_ConfigBBWithPgHeaderFile(Adapter);
-	}
-
-	if (rtStatus != _SUCCESS)
-		goto phy_BB8190_Config_ParaFile_Fail;
-
-	/*  */
-	/*  3. BB AGC table Initialization */
-	/*  */
-	ODM_ReadAndConfig_AGC_TAB_1T_8723A(&pHalData->odmpriv);
-
-phy_BB8190_Config_ParaFile_Fail:
-
-	return rtStatus;
-}
-
-int
-PHY_BBConfig8723A(struct rtw_adapter *Adapter)
-{
-	int rtStatus = _SUCCESS;
-	struct hal_data_8723a	*pHalData = GET_HAL_DATA(Adapter);
-	u8 TmpU1B = 0;
-	u8 CrystalCap;
-
-	phy_InitBBRFRegisterDefinition(Adapter);
-
-	/*  Suggested by Scott. tynli_test. 2010.12.30. */
-	/* 1. 0x28[1] = 1 */
-	TmpU1B = rtl8723au_read8(Adapter, REG_AFE_PLL_CTRL);
-	udelay(2);
-	rtl8723au_write8(Adapter, REG_AFE_PLL_CTRL, TmpU1B | BIT(1));
-	udelay(2);
-
-	/* 2. 0x29[7:0] = 0xFF */
-	rtl8723au_write8(Adapter, REG_AFE_PLL_CTRL+1, 0xff);
-	udelay(2);
-
-	/* 3. 0x02[1:0] = 2b'11 */
-	TmpU1B = rtl8723au_read8(Adapter, REG_SYS_FUNC_EN);
-	rtl8723au_write8(Adapter, REG_SYS_FUNC_EN,
-			 (TmpU1B | FEN_BB_GLB_RSTn | FEN_BBRSTB));
-
-	/* 4. 0x25[6] = 0 */
-	TmpU1B = rtl8723au_read8(Adapter, REG_AFE_XTAL_CTRL + 1);
-	rtl8723au_write8(Adapter, REG_AFE_XTAL_CTRL+1, TmpU1B & ~BIT(6));
-
-	/* 5. 0x24[20] = 0	Advised by SD3 Alex Wang. 2011.02.09. */
-	TmpU1B = rtl8723au_read8(Adapter, REG_AFE_XTAL_CTRL+2);
-	rtl8723au_write8(Adapter, REG_AFE_XTAL_CTRL+2, TmpU1B & ~BIT(4));
-
-	/* 6. 0x1f[7:0] = 0x07 */
-	rtl8723au_write8(Adapter, REG_RF_CTRL, 0x07);
-
-	/*  */
-	/*  Config BB and AGC */
-	/*  */
-	rtStatus = phy_BB8723a_Config_ParaFile(Adapter);
-
-/* only for B-cut */
-	if (pHalData->EEPROMVersion >= 0x01) {
-		CrystalCap = pHalData->CrystalCap & 0x3F;
-		PHY_SetBBReg(Adapter, REG_MAC_PHY_CTRL, 0xFFF000,
-			     (CrystalCap | (CrystalCap << 6)));
-	}
-
-	rtl8723au_write32(Adapter, REG_LDOA15_CTRL, 0x01572505);
-	return rtStatus;
-}
-
-/*-----------------------------------------------------------------------------
- * Function:    SetTxPowerLevel8723A()
- *
- * Overview:    This function is export to "HalCommon" moudule
- *			We must consider RF path later!!!!!!!
- *
- * Input:       struct rtw_adapter *		Adapter
- *			u8		channel
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- *---------------------------------------------------------------------------*/
-void PHY_SetTxPowerLevel8723A(struct rtw_adapter *Adapter, u8 channel)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	u8 cckpwr[2], ofdmpwr[2];	/*  [0]:RF-A, [1]:RF-B */
-	int i = channel - 1;
-
-	if (pHalData->bTXPowerDataReadFromEEPORM == false)
-		return;
-
-	/*  1. CCK */
-	cckpwr[RF_PATH_A] = pHalData->TxPwrLevelCck[RF_PATH_A][i];
-	cckpwr[RF_PATH_B] = pHalData->TxPwrLevelCck[RF_PATH_B][i];
-
-	/*  2. OFDM for 1S or 2S */
-	if (GET_RF_TYPE(Adapter) == RF_1T2R ||
-	    GET_RF_TYPE(Adapter) == RF_1T1R) {
-		/*  Read HT 40 OFDM TX power */
-		ofdmpwr[RF_PATH_A] = pHalData->TxPwrLevelHT40_1S[RF_PATH_A][i];
-		ofdmpwr[RF_PATH_B] = pHalData->TxPwrLevelHT40_1S[RF_PATH_B][i];
-	} else if (GET_RF_TYPE(Adapter) == RF_2T2R) {
-		/*  Read HT 40 OFDM TX power */
-		ofdmpwr[RF_PATH_A] = pHalData->TxPwrLevelHT40_2S[RF_PATH_A][i];
-		ofdmpwr[RF_PATH_B] = pHalData->TxPwrLevelHT40_2S[RF_PATH_B][i];
-	}
-
-	rtl823a_phy_rf6052setccktxpower(Adapter, &cckpwr[0]);
-	rtl8723a_PHY_RF6052SetOFDMTxPower(Adapter, &ofdmpwr[0], channel);
-}
-
-/*-----------------------------------------------------------------------------
- * Function:    PHY_SetBWMode23aCallback8192C()
- *
- * Overview:    Timer callback function for SetSetBWMode23a
- *
- * Input:		PRT_TIMER		pTimer
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- * Note:
- *	(1) We do not take j mode into consideration now
- *	(2) Will two workitem of "switch channel" and
- *	    "switch channel bandwidth" run concurrently?
- *---------------------------------------------------------------------------*/
-static void
-_PHY_SetBWMode23a92C(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	u8 regBwOpMode;
-	u8 regRRSR_RSC;
-
-	if (Adapter->bDriverStopped)
-		return;
-
-	/* 3 */
-	/* 3<1>Set MAC register */
-	/* 3 */
-
-	regBwOpMode = rtl8723au_read8(Adapter, REG_BWOPMODE);
-	regRRSR_RSC = rtl8723au_read8(Adapter, REG_RRSR+2);
-
-	switch (pHalData->CurrentChannelBW) {
-	case HT_CHANNEL_WIDTH_20:
-		regBwOpMode |= BW_OPMODE_20MHZ;
-		rtl8723au_write8(Adapter, REG_BWOPMODE, regBwOpMode);
-		break;
-	case HT_CHANNEL_WIDTH_40:
-		regBwOpMode &= ~BW_OPMODE_20MHZ;
-		rtl8723au_write8(Adapter, REG_BWOPMODE, regBwOpMode);
-		regRRSR_RSC = (regRRSR_RSC & 0x90) |
-			(pHalData->nCur40MhzPrimeSC << 5);
-		rtl8723au_write8(Adapter, REG_RRSR+2, regRRSR_RSC);
-		break;
-
-	default:
-		break;
-	}
-
-	/* 3 */
-	/* 3<2>Set PHY related register */
-	/* 3 */
-	switch (pHalData->CurrentChannelBW) {
-		/* 20 MHz channel*/
-	case HT_CHANNEL_WIDTH_20:
-		PHY_SetBBReg(Adapter, rFPGA0_RFMOD, bRFMOD, 0x0);
-		PHY_SetBBReg(Adapter, rFPGA1_RFMOD, bRFMOD, 0x0);
-		PHY_SetBBReg(Adapter, rFPGA0_AnalogParameter2, BIT(10), 1);
-
-		break;
-
-		/* 40 MHz channel*/
-	case HT_CHANNEL_WIDTH_40:
-		PHY_SetBBReg(Adapter, rFPGA0_RFMOD, bRFMOD, 0x1);
-		PHY_SetBBReg(Adapter, rFPGA1_RFMOD, bRFMOD, 0x1);
-
-		/*  Set Control channel to upper or lower. These settings
-		    are required only for 40MHz */
-		PHY_SetBBReg(Adapter, rCCK0_System, bCCKSideBand,
-			     (pHalData->nCur40MhzPrimeSC >> 1));
-		PHY_SetBBReg(Adapter, rOFDM1_LSTF, 0xC00,
-			     pHalData->nCur40MhzPrimeSC);
-		PHY_SetBBReg(Adapter, rFPGA0_AnalogParameter2, BIT(10), 0);
-
-		PHY_SetBBReg(Adapter, 0x818, BIT(26) | BIT(27),
-			     (pHalData->nCur40MhzPrimeSC ==
-			      HAL_PRIME_CHNL_OFFSET_LOWER) ? 2:1);
-		break;
-
-	default:
-		break;
-	}
-	/* Skip over setting of J-mode in BB register here. Default value
-	   is "None J mode". Emily 20070315 */
-
-	/*  Added it for 20/40 mhz switch time evaluation by guangan 070531 */
-	/* NowL = PlatformEFIORead4Byte(Adapter, TSFR); */
-	/* NowH = PlatformEFIORead4Byte(Adapter, TSFR+4); */
-	/* EndTime = ((u64)NowH << 32) + NowL; */
-
-	rtl8723a_phy_rf6052set_bw(Adapter, pHalData->CurrentChannelBW);
-}
-
- /*-----------------------------------------------------------------------------
- * Function:   SetBWMode23a8190Pci()
- *
- * Overview:  This function is export to "HalCommon" moudule
- *
- * Input:		struct rtw_adapter *			Adapter
- *			enum ht_channel_width	Bandwidth	20M or 40M
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- * Note:		We do not take j mode into consideration now
- *---------------------------------------------------------------------------*/
-void
-PHY_SetBWMode23a8723A(struct rtw_adapter *Adapter,
-		   enum ht_channel_width Bandwidth, unsigned char Offset)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	enum ht_channel_width tmpBW = pHalData->CurrentChannelBW;
-
-	pHalData->CurrentChannelBW = Bandwidth;
-
-	pHalData->nCur40MhzPrimeSC = Offset;
-
-	if ((!Adapter->bDriverStopped) && (!Adapter->bSurpriseRemoved))
-		_PHY_SetBWMode23a92C(Adapter);
-	else
-		pHalData->CurrentChannelBW = tmpBW;
-}
-
-static void _PHY_SwChnl8723A(struct rtw_adapter *Adapter, u8 channel)
-{
-	enum RF_RADIO_PATH eRFPath;
-	u32 param1, param2;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	/* s1. pre common command - CmdID_SetTxPowerLevel */
-	PHY_SetTxPowerLevel8723A(Adapter, channel);
-
-	/* s2. RF dependent command - CmdID_RF_WriteReg,
-	   param1 = RF_CHNLBW, param2 = channel */
-	param1 = RF_CHNLBW;
-	param2 = channel;
-	for (eRFPath = 0; eRFPath < pHalData->NumTotalRFPath; eRFPath++) {
-		pHalData->RfRegChnlVal[eRFPath] =
-			(pHalData->RfRegChnlVal[eRFPath] & 0xfffffc00) | param2;
-		PHY_SetRFReg(Adapter, eRFPath, param1,
-			     bRFRegOffsetMask, pHalData->RfRegChnlVal[eRFPath]);
-	}
-
-	/* s3. post common command - CmdID_End, None */
-}
-
-void PHY_SwChnl8723A(struct rtw_adapter *Adapter, u8 channel)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	u8 tmpchannel = pHalData->CurrentChannel;
-	bool  result = true;
-
-	if (channel == 0)
-		channel = 1;
-
-	pHalData->CurrentChannel = channel;
-
-	if ((!Adapter->bDriverStopped) && (!Adapter->bSurpriseRemoved)) {
-		_PHY_SwChnl8723A(Adapter, channel);
-
-		if (!result)
-			pHalData->CurrentChannel = tmpchannel;
-	} else {
-		pHalData->CurrentChannel = tmpchannel;
-	}
-}
diff --git a/drivers/staging/rtl8723au/hal/rtl8723a_rf6052.c b/drivers/staging/rtl8723au/hal/rtl8723a_rf6052.c
deleted file mode 100644
index 24c0ff3d82bc..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723a_rf6052.c
+++ /dev/null
@@ -1,503 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-/******************************************************************************
- *
- *
- * Module:	rtl8192c_rf6052.c	(Source C File)
- *
- * Note:	Provide RF 6052 series relative API.
- *
- * Function:
- *
- * Export:
- *
- * Abbrev:
- *
- * History:
- * Data			Who		Remark
- *
- * 09/25/2008	MHC		Create initial version.
- * 11/05/2008	MHC		Add API for tw power setting.
- *
- *
-******************************************************************************/
-
-#define _RTL8723A_RF6052_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#include <rtl8723a_hal.h>
-#include <usb_ops_linux.h>
-
-/*-----------------------------------------------------------------------------
- * Function:    PHY_RF6052SetBandwidth()
- *
- * Overview:    This function is called by SetBWMode23aCallback8190Pci() only
- *
- * Input:       struct rtw_adapter *				Adapter
- *			WIRELESS_BANDWIDTH_E	Bandwidth	20M or 40M
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- * Note:		For RF type 0222D
- *---------------------------------------------------------------------------*/
-void rtl8723a_phy_rf6052set_bw(struct rtw_adapter *Adapter,
-			       enum ht_channel_width Bandwidth)	/* 20M or 40M */
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	switch (Bandwidth) {
-	case HT_CHANNEL_WIDTH_20:
-		pHalData->RfRegChnlVal[0] =
-			(pHalData->RfRegChnlVal[0] & 0xfffff3ff) | 0x0400;
-		PHY_SetRFReg(Adapter, RF_PATH_A, RF_CHNLBW, bRFRegOffsetMask,
-			     pHalData->RfRegChnlVal[0]);
-		break;
-	case HT_CHANNEL_WIDTH_40:
-		pHalData->RfRegChnlVal[0] =
-			(pHalData->RfRegChnlVal[0] & 0xfffff3ff);
-		PHY_SetRFReg(Adapter, RF_PATH_A, RF_CHNLBW, bRFRegOffsetMask,
-			     pHalData->RfRegChnlVal[0]);
-		break;
-	default:
-		break;
-	}
-}
-
-/*-----------------------------------------------------------------------------
- * Function:	PHY_RF6052SetCckTxPower
- *
- * Overview:
- *
- * Input:       NONE
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- * Revised History:
- * When			Who		Remark
- * 11/05/2008	MHC		Simulate 8192series..
- *
- *---------------------------------------------------------------------------*/
-
-void rtl823a_phy_rf6052setccktxpower(struct rtw_adapter *Adapter,
-				     u8 *pPowerlevel)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-	struct mlme_ext_priv *pmlmeext = &Adapter->mlmeextpriv;
-	u32 TxAGC[2] = {0, 0}, tmpval = 0;
-	u8 idx1, idx2;
-	u8 *ptr;
-
-	if (pmlmeext->sitesurvey_res.state == SCAN_PROCESS) {
-		TxAGC[RF_PATH_A] = 0x3f3f3f3f;
-		TxAGC[RF_PATH_B] = 0x3f3f3f3f;
-
-		for (idx1 = RF_PATH_A; idx1 <= RF_PATH_B; idx1++) {
-			TxAGC[idx1] = pPowerlevel[idx1] |
-				(pPowerlevel[idx1] << 8) |
-				(pPowerlevel[idx1] << 16) |
-				(pPowerlevel[idx1] << 24);
-			/*
-			 * 2010/10/18 MH For external PA module. We need
-			 * to limit power index to be less than 0x20.
-			 */
-			if (TxAGC[idx1] > 0x20 && pHalData->ExternalPA)
-				TxAGC[idx1] = 0x20;
-		}
-	} else {
-/*  20100427 Joseph: Driver dynamic Tx power shall not affect Tx
- *  power. It shall be determined by power training mechanism. */
-/*  Currently, we cannot fully disable driver dynamic tx power
- *  mechanism because it is referenced by BT coexist mechanism. */
-/*  In the future, two mechanism shall be separated from each other
- *  and maintained independently. Thanks for Lanhsin's reminder. */
-		if (pdmpriv->DynamicTxHighPowerLvl == TxHighPwrLevel_Level1) {
-			TxAGC[RF_PATH_A] = 0x10101010;
-			TxAGC[RF_PATH_B] = 0x10101010;
-		} else if (pdmpriv->DynamicTxHighPowerLvl ==
-			   TxHighPwrLevel_Level2) {
-			TxAGC[RF_PATH_A] = 0x00000000;
-			TxAGC[RF_PATH_B] = 0x00000000;
-		} else {
-			for (idx1 = RF_PATH_A; idx1 <= RF_PATH_B; idx1++) {
-				TxAGC[idx1] = pPowerlevel[idx1] |
-					(pPowerlevel[idx1] << 8) |
-					(pPowerlevel[idx1] << 16) |
-					(pPowerlevel[idx1] << 24);
-			}
-
-			if (pHalData->EEPROMRegulatory == 0) {
-				tmpval = (pHalData->MCSTxPowerLevelOriginalOffset[0][6]) +
-						(pHalData->MCSTxPowerLevelOriginalOffset[0][7]<<8);
-				TxAGC[RF_PATH_A] += tmpval;
-
-				tmpval = (pHalData->MCSTxPowerLevelOriginalOffset[0][14]) +
-						(pHalData->MCSTxPowerLevelOriginalOffset[0][15]<<24);
-				TxAGC[RF_PATH_B] += tmpval;
-			}
-		}
-	}
-
-	for (idx1 = RF_PATH_A; idx1 <= RF_PATH_B; idx1++) {
-		ptr = (u8 *)(&TxAGC[idx1]);
-		for (idx2 = 0; idx2 < 4; idx2++) {
-			if (*ptr > RF6052_MAX_TX_PWR)
-				*ptr = RF6052_MAX_TX_PWR;
-			ptr++;
-		}
-	}
-
-	/*  rf-A cck tx power */
-	tmpval = TxAGC[RF_PATH_A] & 0xff;
-	PHY_SetBBReg(Adapter, rTxAGC_A_CCK1_Mcs32, bMaskByte1, tmpval);
-	tmpval = TxAGC[RF_PATH_A] >> 8;
-	PHY_SetBBReg(Adapter, rTxAGC_B_CCK11_A_CCK2_11, 0xffffff00, tmpval);
-
-	/*  rf-B cck tx power */
-	tmpval = TxAGC[RF_PATH_B] >> 24;
-	PHY_SetBBReg(Adapter, rTxAGC_B_CCK11_A_CCK2_11, bMaskByte0, tmpval);
-	tmpval = TxAGC[RF_PATH_B] & 0x00ffffff;
-	PHY_SetBBReg(Adapter, rTxAGC_B_CCK1_55_Mcs32, 0xffffff00, tmpval);
-}	/* PHY_RF6052SetCckTxPower */
-
-/*  powerbase0 for OFDM rates */
-/*  powerbase1 for HT MCS rates */
-static void getPowerBase(struct rtw_adapter *Adapter, u8 *pPowerLevel,
-			 u8 Channel, u32 *OfdmBase, u32 *MCSBase)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	u32 ofdm, mcs;
-	u8 Legacy_pwrdiff = 0;
-	s8 HT20_pwrdiff = 0;
-	u8 i, powerlevel[2];
-
-	for (i = 0; i < 2; i++) {
-		powerlevel[i] = pPowerLevel[i];
-		Legacy_pwrdiff = pHalData->TxPwrLegacyHtDiff[i][Channel-1];
-		ofdm = powerlevel[i] + Legacy_pwrdiff;
-
-		ofdm = ofdm << 24 | ofdm << 16 | ofdm << 8 | ofdm;
-		*(OfdmBase + i) = ofdm;
-	}
-
-	for (i = 0; i < 2; i++) {
-		/* Check HT20 to HT40 diff */
-		if (pHalData->CurrentChannelBW == HT_CHANNEL_WIDTH_20) {
-			HT20_pwrdiff = pHalData->TxPwrHt20Diff[i][Channel-1];
-			powerlevel[i] += HT20_pwrdiff;
-		}
-		mcs = powerlevel[i];
-		mcs = mcs << 24 | mcs << 16 | mcs << 8 | mcs;
-		*(MCSBase + i) = mcs;
-	}
-}
-
-static void
-getTxPowerWriteValByRegulatory(struct rtw_adapter *Adapter, u8 Channel,
-			       u8 index, u32 *powerBase0, u32 *powerBase1,
-			       u32 *pOutWriteVal)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct dm_priv	*pdmpriv = &pHalData->dmpriv;
-	u8 i, chnlGroup = 0, pwr_diff_limit[4];
-	u32 writeVal, customer_limit, rf;
-
-	/*  Index 0 & 1 = legacy OFDM, 2-5 = HT_MCS rate */
-	for (rf = 0; rf < 2; rf++) {
-		switch (pHalData->EEPROMRegulatory) {
-		case 0:	/*  Realtek better performance */
-			/*  increase power diff defined by Realtek for
-			 *  large power */
-			chnlGroup = 0;
-			writeVal = pHalData->MCSTxPowerLevelOriginalOffset[chnlGroup][index+(rf?8:0)] +
-				((index < 2) ? powerBase0[rf] : powerBase1[rf]);
-			break;
-		case 1:	/*  Realtek regulatory */
-			/*  increase power diff defined by Realtek for
-			 *  regulatory */
-			if (pHalData->pwrGroupCnt == 1)
-				chnlGroup = 0;
-			if (pHalData->pwrGroupCnt >= 3) {
-				if (Channel <= 3)
-					chnlGroup = 0;
-				else if (Channel >= 4 && Channel <= 9)
-					chnlGroup = 1;
-				else if (Channel > 9)
-					chnlGroup = 2;
-
-				if (pHalData->CurrentChannelBW ==
-				    HT_CHANNEL_WIDTH_20)
-					chnlGroup++;
-				else
-					chnlGroup += 4;
-			}
-			writeVal = pHalData->MCSTxPowerLevelOriginalOffset[chnlGroup][index+(rf?8:0)] +
-				   ((index < 2) ? powerBase0[rf] :
-				    powerBase1[rf]);
-			break;
-		case 2:	/*  Better regulatory */
-			/*  don't increase any power diff */
-			writeVal = (index < 2) ? powerBase0[rf] :
-				    powerBase1[rf];
-			break;
-		case 3:	/*  Customer defined power diff. */
-			chnlGroup = 0;
-
-			for (i = 0; i < 4; i++) {
-				pwr_diff_limit[i] = (u8)((pHalData->MCSTxPowerLevelOriginalOffset[chnlGroup][index +
-						    (rf ? 8 : 0)]&(0x7f << (i*8))) >> (i*8));
-				if (pHalData->CurrentChannelBW == HT_CHANNEL_WIDTH_40) {
-					if (pwr_diff_limit[i] > pHalData->PwrGroupHT40[rf][Channel-1])
-						pwr_diff_limit[i] = pHalData->PwrGroupHT40[rf][Channel-1];
-				} else {
-					if (pwr_diff_limit[i] > pHalData->PwrGroupHT20[rf][Channel-1])
-						pwr_diff_limit[i] = pHalData->PwrGroupHT20[rf][Channel-1];
-				}
-			}
-			customer_limit = (pwr_diff_limit[3]<<24) | (pwr_diff_limit[2]<<16) |
-							(pwr_diff_limit[1]<<8) | (pwr_diff_limit[0]);
-			writeVal = customer_limit + ((index<2)?powerBase0[rf]:powerBase1[rf]);
-			break;
-		default:
-			chnlGroup = 0;
-			writeVal = pHalData->MCSTxPowerLevelOriginalOffset[chnlGroup][index+(rf?8:0)] +
-					((index < 2) ? powerBase0[rf] : powerBase1[rf]);
-			break;
-		}
-
-/*  20100427 Joseph: Driver dynamic Tx power shall not affect Tx power.
-    It shall be determined by power training mechanism. */
-/*  Currently, we cannot fully disable driver dynamic tx power mechanism
-    because it is referenced by BT coexist mechanism. */
-/*  In the future, two mechanism shall be separated from each other and
-    maintained independently. Thanks for Lanhsin's reminder. */
-
-		if (pdmpriv->DynamicTxHighPowerLvl == TxHighPwrLevel_Level1)
-			writeVal = 0x14141414;
-		else if (pdmpriv->DynamicTxHighPowerLvl ==
-			 TxHighPwrLevel_Level2)
-			writeVal = 0x00000000;
-
-		/* 20100628 Joseph: High power mode for BT-Coexist mechanism. */
-		/* This mechanism is only applied when
-		   Driver-Highpower-Mechanism is OFF. */
-		if (pdmpriv->DynamicTxHighPowerLvl == TxHighPwrLevel_BT1)
-			writeVal = writeVal - 0x06060606;
-		else if (pdmpriv->DynamicTxHighPowerLvl == TxHighPwrLevel_BT2)
-			writeVal = writeVal;
-		*(pOutWriteVal + rf) = writeVal;
-	}
-}
-
-static void writeOFDMPowerReg(struct rtw_adapter *Adapter, u8 index,
-			      u32 *pValue)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	u16 RegOffset_A[6] = {
-		rTxAGC_A_Rate18_06, rTxAGC_A_Rate54_24,
-		rTxAGC_A_Mcs03_Mcs00, rTxAGC_A_Mcs07_Mcs04,
-		rTxAGC_A_Mcs11_Mcs08, rTxAGC_A_Mcs15_Mcs12
-	};
-	u16 RegOffset_B[6] = {
-		rTxAGC_B_Rate18_06, rTxAGC_B_Rate54_24,
-		rTxAGC_B_Mcs03_Mcs00, rTxAGC_B_Mcs07_Mcs04,
-		rTxAGC_B_Mcs11_Mcs08, rTxAGC_B_Mcs15_Mcs12
-	};
-	u8 i, rf, pwr_val[4];
-	u32 writeVal;
-	u16 RegOffset;
-
-	for (rf = 0; rf < 2; rf++) {
-		writeVal = pValue[rf];
-		for (i = 0; i < 4; i++) {
-			pwr_val[i] = (u8)((writeVal &
-					   (0x7f << (i * 8))) >> (i * 8));
-			if (pwr_val[i] > RF6052_MAX_TX_PWR)
-				pwr_val[i]  = RF6052_MAX_TX_PWR;
-		}
-		writeVal = pwr_val[3] << 24 | pwr_val[2] << 16 |
-			pwr_val[1] << 8 | pwr_val[0];
-
-		if (rf == 0)
-			RegOffset = RegOffset_A[index];
-		else
-			RegOffset = RegOffset_B[index];
-
-		rtl8723au_write32(Adapter, RegOffset, writeVal);
-
-		/*  201005115 Joseph: Set Tx Power diff for Tx power
-		    training mechanism. */
-		if (((pHalData->rf_type == RF_2T2R) &&
-		    (RegOffset == rTxAGC_A_Mcs15_Mcs12 ||
-		     RegOffset == rTxAGC_B_Mcs15_Mcs12)) ||
-		    ((pHalData->rf_type != RF_2T2R) &&
-		     (RegOffset == rTxAGC_A_Mcs07_Mcs04 ||
-		      RegOffset == rTxAGC_B_Mcs07_Mcs04))) {
-			writeVal = pwr_val[3];
-			if (RegOffset == rTxAGC_A_Mcs15_Mcs12 ||
-			    RegOffset == rTxAGC_A_Mcs07_Mcs04)
-				RegOffset = 0xc90;
-			if (RegOffset == rTxAGC_B_Mcs15_Mcs12 ||
-			    RegOffset == rTxAGC_B_Mcs07_Mcs04)
-				RegOffset = 0xc98;
-			for (i = 0; i < 3; i++) {
-				if (i != 2)
-					writeVal = (writeVal > 8) ?
-						(writeVal - 8) : 0;
-				else
-					writeVal = (writeVal > 6) ?
-						(writeVal - 6) : 0;
-				rtl8723au_write8(Adapter, RegOffset + i,
-						 (u8)writeVal);
-			}
-		}
-	}
-}
-/*-----------------------------------------------------------------------------
- * Function:	PHY_RF6052SetOFDMTxPower
- *
- * Overview:	For legacy and HY OFDM, we must read EEPROM TX power index for
- *		different channel and read original value in TX power
- *		register area from 0xe00. We increase offset and
- *		original value to be correct tx pwr.
- *
- * Input:       NONE
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- * Revised History:
- * When			Remark
- * 11/05/2008	MHC	Simulate 8192 series method.
- * 01/06/2009	MHC	1. Prevent Path B tx power overflow or
- *			underflow dure to A/B pwr difference or
- *			legacy/HT pwr diff.
- *			2. We concern with path B legacy/HT OFDM difference.
- * 01/22/2009	MHC	Support new EPRO format from SD3.
- *
- *---------------------------------------------------------------------------*/
-void rtl8723a_PHY_RF6052SetOFDMTxPower(struct rtw_adapter *Adapter,
-				       u8 *pPowerLevel, u8 Channel)
-{
-	u32 writeVal[2], powerBase0[2], powerBase1[2];
-	u8 index = 0;
-
-	getPowerBase(Adapter, pPowerLevel, Channel,
-		     &powerBase0[0], &powerBase1[0]);
-
-	for (index = 0; index < 6; index++) {
-		getTxPowerWriteValByRegulatory(Adapter, Channel, index,
-			&powerBase0[0], &powerBase1[0], &writeVal[0]);
-
-		writeOFDMPowerReg(Adapter, index, &writeVal[0]);
-	}
-}
-
-static int phy_RF6052_Config_ParaFile(struct rtw_adapter *Adapter)
-{
-	u32 u4RegValue = 0;
-	u8 eRFPath;
-	struct bb_reg_define *pPhyReg;
-	int rtStatus = _SUCCESS;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	/* 3----------------------------------------------------------------- */
-	/* 3 <2> Initialize RF */
-	/* 3----------------------------------------------------------------- */
-	for (eRFPath = 0; eRFPath < pHalData->NumTotalRFPath; eRFPath++) {
-
-		pPhyReg = &pHalData->PHYRegDef[eRFPath];
-
-		/*----Store original RFENV control type----*/
-		switch (eRFPath) {
-		case RF_PATH_A:
-			u4RegValue = PHY_QueryBBReg(Adapter, pPhyReg->rfintfs,
-						    bRFSI_RFENV);
-			break;
-		case RF_PATH_B:
-			u4RegValue = PHY_QueryBBReg(Adapter, pPhyReg->rfintfs,
-						    bRFSI_RFENV << 16);
-			break;
-		}
-
-		/*----Set RF_ENV enable----*/
-		PHY_SetBBReg(Adapter, pPhyReg->rfintfe, bRFSI_RFENV << 16, 0x1);
-		udelay(1);/* PlatformStallExecution(1); */
-
-		/*----Set RF_ENV output high----*/
-		PHY_SetBBReg(Adapter, pPhyReg->rfintfo, bRFSI_RFENV, 0x1);
-		udelay(1);/* PlatformStallExecution(1); */
-
-		/* Set bit number of Address and Data for RF register */
-		PHY_SetBBReg(Adapter, pPhyReg->rfHSSIPara2, b3WireAddressLength,
-			     0x0);	/*  Set 1 to 4 bits for 8255 */
-		udelay(1);/* PlatformStallExecution(1); */
-
-		PHY_SetBBReg(Adapter, pPhyReg->rfHSSIPara2, b3WireDataLength,
-			     0x0);	/*  Set 0 to 12  bits for 8255 */
-		udelay(1);/* PlatformStallExecution(1); */
-
-		/*----Initialize RF fom connfiguration file----*/
-		switch (eRFPath) {
-		case RF_PATH_A:
-			ODM_ReadAndConfig_RadioA_1T_8723A(&pHalData->odmpriv);
-			break;
-		case RF_PATH_B:
-			break;
-		}
-
-		/*----Restore RFENV control type----*/
-		switch (eRFPath) {
-		case RF_PATH_A:
-			PHY_SetBBReg(Adapter, pPhyReg->rfintfs,
-				     bRFSI_RFENV, u4RegValue);
-			break;
-		case RF_PATH_B:
-			PHY_SetBBReg(Adapter, pPhyReg->rfintfs,
-				     bRFSI_RFENV << 16, u4RegValue);
-			break;
-		}
-
-		if (rtStatus != _SUCCESS) {
-			goto phy_RF6052_Config_ParaFile_Fail;
-		}
-	}
-phy_RF6052_Config_ParaFile_Fail:
-	return rtStatus;
-}
-
-int PHY_RF6052_Config8723A(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	/*  Initialize general global value */
-	/*  TODO: Extend RF_PATH_C and RF_PATH_D in the future */
-	if (pHalData->rf_type == RF_1T1R)
-		pHalData->NumTotalRFPath = 1;
-	else
-		pHalData->NumTotalRFPath = 2;
-
-	/*  Config BB and RF */
-	return phy_RF6052_Config_ParaFile(Adapter);
-}
-
-/* End of HalRf6052.c */
diff --git a/drivers/staging/rtl8723au/hal/rtl8723a_rxdesc.c b/drivers/staging/rtl8723au/hal/rtl8723a_rxdesc.c
deleted file mode 100644
index 81b5efe649fa..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723a_rxdesc.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTL8723A_REDESC_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <rtl8723a_hal.h>
-
-static void process_rssi(struct rtw_adapter *padapter,
-			 struct recv_frame *prframe)
-{
-	struct rx_pkt_attrib *pattrib = &prframe->attrib;
-	struct signal_stat *signal_stat = &padapter->recvpriv.signal_strength_data;
-
-	if (signal_stat->update_req) {
-		signal_stat->total_num = 0;
-		signal_stat->total_val = 0;
-		signal_stat->update_req = 0;
-	}
-
-	signal_stat->total_num++;
-	signal_stat->total_val  += pattrib->phy_info.SignalStrength;
-	signal_stat->avg_val = signal_stat->total_val / signal_stat->total_num;
-}
-
-static void process_link_qual(struct rtw_adapter *padapter,
-			      struct recv_frame *prframe)
-{
-	struct rx_pkt_attrib *pattrib;
-	struct signal_stat *signal_stat;
-
-	if (prframe == NULL || padapter == NULL)
-		return;
-
-	pattrib = &prframe->attrib;
-	signal_stat = &padapter->recvpriv.signal_qual_data;
-
-	if (signal_stat->update_req) {
-		signal_stat->total_num = 0;
-		signal_stat->total_val = 0;
-		signal_stat->update_req = 0;
-	}
-
-	signal_stat->total_num++;
-	signal_stat->total_val  += pattrib->phy_info.SignalQuality;
-	signal_stat->avg_val = signal_stat->total_val / signal_stat->total_num;
-}
-
-/* void rtl8723a_process_phy_info(struct rtw_adapter *padapter, union recv_frame *prframe) */
-void rtl8723a_process_phy_info(struct rtw_adapter *padapter, void *prframe)
-{
-	struct recv_frame *precvframe = prframe;
-	/*  Check RSSI */
-	process_rssi(padapter, precvframe);
-	/*  Check EVM */
-	process_link_qual(padapter,  precvframe);
-}
diff --git a/drivers/staging/rtl8723au/hal/rtl8723a_sreset.c b/drivers/staging/rtl8723au/hal/rtl8723a_sreset.c
deleted file mode 100644
index 3c46294b8788..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723a_sreset.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTL8723A_SRESET_C_
-
-#include <rtl8723a_sreset.h>
-#include <rtl8723a_hal.h>
-#include <usb_ops_linux.h>
-
-void rtl8723a_sreset_xmit_status_check(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a	*pHalData = GET_HAL_DATA(padapter);
-	struct sreset_priv *psrtpriv = &pHalData->srestpriv;
-
-	unsigned long current_time;
-	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
-	unsigned int diff_time;
-	u32 txdma_status;
-
-	txdma_status = rtl8723au_read32(padapter, REG_TXDMA_STATUS);
-	if (txdma_status != 0) {
-		DBG_8723A("%s REG_TXDMA_STATUS:0x%08x\n", __func__, txdma_status);
-		rtw_sreset_reset(padapter);
-	}
-
-	current_time = jiffies;
-
-	if (0 == pxmitpriv->free_xmitbuf_cnt || 0 == pxmitpriv->free_xmit_extbuf_cnt) {
-
-		diff_time = jiffies_to_msecs(jiffies - psrtpriv->last_tx_time);
-
-		if (diff_time > 2000) {
-			if (psrtpriv->last_tx_complete_time == 0) {
-				psrtpriv->last_tx_complete_time = current_time;
-			} else {
-				diff_time = jiffies_to_msecs(jiffies - psrtpriv->last_tx_complete_time);
-				if (diff_time > 4000) {
-					DBG_8723A("%s tx hang\n", __func__);
-					rtw_sreset_reset(padapter);
-				}
-			}
-		}
-	}
-}
diff --git a/drivers/staging/rtl8723au/hal/rtl8723au_recv.c b/drivers/staging/rtl8723au/hal/rtl8723au_recv.c
deleted file mode 100644
index 0fec84bcb5d9..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723au_recv.c
+++ /dev/null
@@ -1,267 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTL8192CU_RECV_C_
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <recv_osdep.h>
-#include <mlme_osdep.h>
-#include <linux/ip.h>
-#include <linux/if_ether.h>
-#include <usb_ops.h>
-#include <wifi.h>
-#include <rtl8723a_hal.h>
-
-int rtl8723au_init_recv_priv(struct rtw_adapter *padapter)
-{
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-	int i, size, res = _SUCCESS;
-	struct recv_buf *precvbuf;
-	unsigned long tmpaddr;
-	unsigned long alignment;
-	struct sk_buff *pskb;
-
-	tasklet_init(&precvpriv->recv_tasklet,
-		     (void(*)(unsigned long))rtl8723au_recv_tasklet,
-		     (unsigned long)padapter);
-
-	precvpriv->int_in_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!precvpriv->int_in_urb)
-		DBG_8723A("alloc_urb for interrupt in endpoint fail !!!!\n");
-	precvpriv->int_in_buf = kzalloc(USB_INTR_CONTENT_LENGTH, GFP_KERNEL);
-	if (!precvpriv->int_in_buf)
-		DBG_8723A("alloc_mem for interrupt in endpoint fail !!!!\n");
-
-	size = NR_RECVBUFF * sizeof(struct recv_buf);
-	precvpriv->precv_buf = kzalloc(size, GFP_KERNEL);
-	if (!precvpriv->precv_buf) {
-		res = _FAIL;
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-			 "alloc recv_buf fail!\n");
-		goto exit;
-	}
-
-	precvbuf = (struct recv_buf *)precvpriv->precv_buf;
-
-	for (i = 0; i < NR_RECVBUFF; i++) {
-		INIT_LIST_HEAD(&precvbuf->list);
-
-		precvbuf->purb = usb_alloc_urb(0, GFP_KERNEL);
-		if (!precvbuf->purb)
-			break;
-
-		precvbuf->adapter = padapter;
-
-		precvbuf++;
-	}
-
-	skb_queue_head_init(&precvpriv->rx_skb_queue);
-	skb_queue_head_init(&precvpriv->free_recv_skb_queue);
-
-	for (i = 0; i < NR_PREALLOC_RECV_SKB; i++) {
-		size = MAX_RECVBUF_SZ + RECVBUFF_ALIGN_SZ;
-		pskb = __netdev_alloc_skb(padapter->pnetdev, size, GFP_KERNEL);
-
-		if (pskb) {
-			pskb->dev = padapter->pnetdev;
-
-			tmpaddr = (unsigned long)pskb->data;
-			alignment = tmpaddr & (RECVBUFF_ALIGN_SZ-1);
-			skb_reserve(pskb, (RECVBUFF_ALIGN_SZ - alignment));
-
-			skb_queue_tail(&precvpriv->free_recv_skb_queue, pskb);
-		}
-
-		pskb = NULL;
-	}
-
-exit:
-	return res;
-}
-
-void rtl8723au_free_recv_priv(struct rtw_adapter *padapter)
-{
-	int	i;
-	struct recv_buf	*precvbuf;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-
-	precvbuf = (struct recv_buf *)precvpriv->precv_buf;
-
-	for (i = 0; i < NR_RECVBUFF; i++) {
-		usb_free_urb(precvbuf->purb);
-
-		if (precvbuf->pskb)
-			dev_kfree_skb_any(precvbuf->pskb);
-
-		precvbuf++;
-	}
-
-	kfree(precvpriv->precv_buf);
-
-	usb_free_urb(precvpriv->int_in_urb);
-	kfree(precvpriv->int_in_buf);
-
-	if (skb_queue_len(&precvpriv->rx_skb_queue))
-		DBG_8723A(KERN_WARNING "rx_skb_queue not empty\n");
-
-	skb_queue_purge(&precvpriv->rx_skb_queue);
-
-	if (skb_queue_len(&precvpriv->free_recv_skb_queue)) {
-		DBG_8723A(KERN_WARNING "free_recv_skb_queue not empty, %d\n",
-			  skb_queue_len(&precvpriv->free_recv_skb_queue));
-	}
-
-	skb_queue_purge(&precvpriv->free_recv_skb_queue);
-}
-
-struct recv_stat_cpu {
-	u32 rxdw0;
-	u32 rxdw1;
-	u32 rxdw2;
-	u32 rxdw3;
-	u32 rxdw4;
-	u32 rxdw5;
-};
-
-void update_recvframe_attrib(struct recv_frame *precvframe,
-			     struct recv_stat *prxstat)
-{
-	struct rx_pkt_attrib *pattrib;
-	struct recv_stat_cpu report;
-	struct rxreport_8723a *prxreport;
-
-	report.rxdw0 = le32_to_cpu(prxstat->rxdw0);
-	report.rxdw1 = le32_to_cpu(prxstat->rxdw1);
-	report.rxdw2 = le32_to_cpu(prxstat->rxdw2);
-	report.rxdw3 = le32_to_cpu(prxstat->rxdw3);
-	report.rxdw4 = le32_to_cpu(prxstat->rxdw4);
-	report.rxdw5 = le32_to_cpu(prxstat->rxdw5);
-
-	prxreport = (struct rxreport_8723a *)&report;
-
-	pattrib = &precvframe->attrib;
-	memset(pattrib, 0, sizeof(struct rx_pkt_attrib));
-
-	/*  update rx report to recv_frame attribute */
-	pattrib->pkt_len = (u16)prxreport->pktlen;
-	pattrib->drvinfo_sz = (u8)(prxreport->drvinfosize << 3);
-	pattrib->physt = (u8)prxreport->physt;
-
-	pattrib->crc_err = (u8)prxreport->crc32;
-	pattrib->icv_err = (u8)prxreport->icverr;
-
-	pattrib->bdecrypted = (u8)(prxreport->swdec ? 0 : 1);
-	pattrib->encrypt = (u8)prxreport->security;
-
-	pattrib->qos = (u8)prxreport->qos;
-	pattrib->priority = (u8)prxreport->tid;
-
-	pattrib->amsdu = (u8)prxreport->amsdu;
-
-	pattrib->seq_num = (u16)prxreport->seq;
-	pattrib->frag_num = (u8)prxreport->frag;
-	pattrib->mfrag = (u8)prxreport->mf;
-	pattrib->mdata = (u8)prxreport->md;
-
-	pattrib->mcs_rate = (u8)prxreport->rxmcs;
-	pattrib->rxht = (u8)prxreport->rxht;
-}
-
-void update_recvframe_phyinfo(struct recv_frame *precvframe,
-			      struct phy_stat *pphy_status)
-{
-	struct rtw_adapter *padapter = precvframe->adapter;
-	struct rx_pkt_attrib *pattrib = &precvframe->attrib;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct phy_info *pPHYInfo = &pattrib->phy_info;
-	struct odm_packet_info pkt_info;
-	u8 *sa = NULL, *da;
-	struct sta_priv *pstapriv;
-	struct sta_info *psta;
-	struct sk_buff *skb = precvframe->pkt;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	bool matchbssid = false;
-	u8 *bssid;
-
-	matchbssid = !ieee80211_is_ctl(hdr->frame_control) &&
-		      !pattrib->icv_err && !pattrib->crc_err;
-
-	if (matchbssid) {
-		switch (hdr->frame_control &
-			cpu_to_le16(IEEE80211_FCTL_TODS |
-				    IEEE80211_FCTL_FROMDS)) {
-		case cpu_to_le16(IEEE80211_FCTL_TODS):
-			bssid = hdr->addr1;
-			break;
-		case cpu_to_le16(IEEE80211_FCTL_FROMDS):
-			bssid = hdr->addr2;
-			break;
-		case cpu_to_le16(0):
-			bssid = hdr->addr3;
-			break;
-		default:
-			bssid = NULL;
-			matchbssid = false;
-		}
-
-		if (bssid)
-			matchbssid = ether_addr_equal(
-				get_bssid(&padapter->mlmepriv), bssid);
-	}
-
-	pkt_info.bPacketMatchBSSID = matchbssid;
-
-	da = ieee80211_get_DA(hdr);
-	pkt_info.bPacketToSelf = pkt_info.bPacketMatchBSSID &&
-		(!memcmp(da, myid(&padapter->eeprompriv), ETH_ALEN));
-
-	pkt_info.bPacketBeacon = pkt_info.bPacketMatchBSSID &&
-		ieee80211_is_beacon(hdr->frame_control);
-
-	pkt_info.StationID = 0xFF;
-	if (pkt_info.bPacketBeacon) {
-		if (check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE) == true)
-			sa = padapter->mlmepriv.cur_network.network.MacAddress;
-		/* to do Ad-hoc */
-	} else {
-		sa = ieee80211_get_SA(hdr);
-	}
-
-	pstapriv = &padapter->stapriv;
-	psta = rtw_get_stainfo23a(pstapriv, sa);
-	if (psta) {
-		pkt_info.StationID = psta->mac_id;
-		/* printk("%s ==> StationID(%d)\n", __func__, pkt_info.StationID); */
-	}
-	pkt_info.Rate = pattrib->mcs_rate;
-
-	ODM_PhyStatusQuery23a(&pHalData->odmpriv, pPHYInfo,
-			      (u8 *)pphy_status, &pkt_info);
-	precvframe->psta = NULL;
-	if (pkt_info.bPacketMatchBSSID &&
-	    (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) == true)) {
-		if (psta) {
-			precvframe->psta = psta;
-			rtl8723a_process_phy_info(padapter, precvframe);
-		}
-	} else if (pkt_info.bPacketToSelf || pkt_info.bPacketBeacon) {
-		if (check_fwstate(&padapter->mlmepriv,
-				  WIFI_ADHOC_STATE|WIFI_ADHOC_MASTER_STATE) ==
-		    true) {
-			if (psta)
-				precvframe->psta = psta;
-		}
-		rtl8723a_process_phy_info(padapter, precvframe);
-	}
-}
diff --git a/drivers/staging/rtl8723au/hal/rtl8723au_xmit.c b/drivers/staging/rtl8723au/hal/rtl8723au_xmit.c
deleted file mode 100644
index 14746dd8db78..000000000000
--- a/drivers/staging/rtl8723au/hal/rtl8723au_xmit.c
+++ /dev/null
@@ -1,520 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RTL8192C_XMIT_C_
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <wifi.h>
-#include <osdep_intf.h>
-#include <usb_ops.h>
-/* include <rtl8192c_hal.h> */
-#include <rtl8723a_hal.h>
-
-static int urb_zero_packet_chk(struct rtw_adapter *padapter, int sz)
-{
-	int blnSetTxDescOffset;
-	struct dvobj_priv	*pdvobj = adapter_to_dvobj(padapter);
-
-	if (pdvobj->ishighspeed) {
-		if (((sz + TXDESC_SIZE) % 512) == 0)
-			blnSetTxDescOffset = 1;
-		else
-			blnSetTxDescOffset = 0;
-	} else {
-		if (((sz + TXDESC_SIZE) % 64) == 0)
-			blnSetTxDescOffset = 1;
-		else
-			blnSetTxDescOffset = 0;
-	}
-	return blnSetTxDescOffset;
-}
-
-static void rtl8192cu_cal_txdesc_chksum(struct tx_desc	*ptxdesc)
-{
-		__le16	*usPtr = (__le16 *)ptxdesc;
-		u32 count = 16;		/*  (32 bytes / 2 bytes per XOR) => 16 times */
-		u32 index;
-		u16 checksum = 0;
-
-		/* Clear first */
-		ptxdesc->txdw7 &= cpu_to_le32(0xffff0000);
-
-		for (index = 0 ; index < count ; index++)
-			checksum = checksum ^ le16_to_cpu(*(usPtr + index));
-
-		ptxdesc->txdw7 |= cpu_to_le32(0x0000ffff&checksum);
-}
-
-static void fill_txdesc_sectype(struct pkt_attrib *pattrib, struct tx_desc *ptxdesc)
-{
-	if ((pattrib->encrypt > 0) && !pattrib->bswenc) {
-		switch (pattrib->encrypt) {
-		/* SEC_TYPE */
-		case WLAN_CIPHER_SUITE_WEP40:
-		case WLAN_CIPHER_SUITE_WEP104:
-			ptxdesc->txdw1 |= cpu_to_le32((0x01<<22)&0x00c00000);
-			break;
-		case WLAN_CIPHER_SUITE_TKIP:
-			/* ptxdesc->txdw1 |= cpu_to_le32((0x02<<22)&0x00c00000); */
-			ptxdesc->txdw1 |= cpu_to_le32((0x01<<22)&0x00c00000);
-			break;
-		case WLAN_CIPHER_SUITE_CCMP:
-			ptxdesc->txdw1 |= cpu_to_le32((0x03<<22)&0x00c00000);
-			break;
-		case 0:
-		default:
-			break;
-		}
-	}
-}
-
-static void fill_txdesc_vcs(struct pkt_attrib *pattrib, __le32 *pdw)
-{
-	/* DBG_8723A("cvs_mode =%d\n", pattrib->vcs_mode); */
-
-	switch (pattrib->vcs_mode) {
-	case RTS_CTS:
-		*pdw |= cpu_to_le32(BIT(12));
-		break;
-	case CTS_TO_SELF:
-		*pdw |= cpu_to_le32(BIT(11));
-		break;
-	case NONE_VCS:
-	default:
-		break;
-	}
-
-	if (pattrib->vcs_mode) {
-		*pdw |= cpu_to_le32(BIT(13));
-
-		/*  Set RTS BW */
-		if (pattrib->ht_en) {
-			*pdw |= (pattrib->bwmode&HT_CHANNEL_WIDTH_40) ?	cpu_to_le32(BIT(27)) : 0;
-
-			if (pattrib->ch_offset == HAL_PRIME_CHNL_OFFSET_LOWER)
-				*pdw |= cpu_to_le32((0x01<<28)&0x30000000);
-			else if (pattrib->ch_offset == HAL_PRIME_CHNL_OFFSET_UPPER)
-				*pdw |= cpu_to_le32((0x02<<28)&0x30000000);
-			else if (pattrib->ch_offset == HAL_PRIME_CHNL_OFFSET_DONT_CARE)
-				*pdw |= 0;
-			else
-				*pdw |= cpu_to_le32((0x03<<28)&0x30000000);
-		}
-	}
-}
-
-static void fill_txdesc_phy(struct pkt_attrib *pattrib, __le32 *pdw)
-{
-	if (pattrib->ht_en) {
-		*pdw |= (pattrib->bwmode&HT_CHANNEL_WIDTH_40) ? cpu_to_le32(BIT(25)) : 0;
-
-		if (pattrib->ch_offset == HAL_PRIME_CHNL_OFFSET_LOWER)
-			*pdw |= cpu_to_le32((0x01<<20)&0x003f0000);
-		else if (pattrib->ch_offset == HAL_PRIME_CHNL_OFFSET_UPPER)
-			*pdw |= cpu_to_le32((0x02<<20)&0x003f0000);
-		else if (pattrib->ch_offset == HAL_PRIME_CHNL_OFFSET_DONT_CARE)
-			*pdw |= 0;
-		else
-			*pdw |= cpu_to_le32((0x03<<20)&0x003f0000);
-	}
-}
-
-static s32 update_txdesc(struct xmit_frame *pxmitframe, u8 *pmem, s32 sz)
-{
-	int	pull = 0;
-	uint	qsel;
-	struct rtw_adapter	*padapter = pxmitframe->padapter;
-	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
-	struct hal_data_8723a	*pHalData = GET_HAL_DATA(padapter);
-	struct dm_priv	*pdmpriv = &pHalData->dmpriv;
-	struct tx_desc	*ptxdesc = (struct tx_desc *)pmem;
-	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info	*pmlmeinfo = &pmlmeext->mlmext_info;
-	int	bmcst = is_multicast_ether_addr(pattrib->ra);
-
-	if (urb_zero_packet_chk(padapter, sz) == 0) {
-		ptxdesc = (struct tx_desc *)(pmem+PACKET_OFFSET_SZ);
-		pull = 1;
-		pxmitframe->pkt_offset--;
-	}
-
-	memset(ptxdesc, 0, sizeof(struct tx_desc));
-
-	if (pxmitframe->frame_tag == DATA_FRAMETAG) {
-		/* offset 4 */
-		ptxdesc->txdw1 |= cpu_to_le32(pattrib->mac_id&0x1f);
-
-		qsel = (uint)(pattrib->qsel & 0x0000001f);
-		ptxdesc->txdw1 |= cpu_to_le32((qsel << QSEL_SHT) & 0x00001f00);
-
-		ptxdesc->txdw1 |= cpu_to_le32((pattrib->raid<<16) & 0x000f0000);
-
-		fill_txdesc_sectype(pattrib, ptxdesc);
-
-		if (pattrib->ampdu_en)
-			ptxdesc->txdw1 |= cpu_to_le32(BIT(5));/* AGG EN */
-		else
-			ptxdesc->txdw1 |= cpu_to_le32(BIT(6));/* AGG BK */
-
-		/* offset 8 */
-
-		/* offset 12 */
-		ptxdesc->txdw3 |= cpu_to_le32((pattrib->seqnum<<16)&0xffff0000);
-
-		/* offset 16 , offset 20 */
-		if (pattrib->qos_en)
-			ptxdesc->txdw4 |= cpu_to_le32(BIT(6));/* QoS */
-
-		if ((pattrib->ether_type != 0x888e) &&
-		    (pattrib->ether_type != 0x0806) &&
-		    (pattrib->dhcp_pkt != 1)) {
-			/* Non EAP & ARP & DHCP type data packet */
-
-			fill_txdesc_vcs(pattrib, &ptxdesc->txdw4);
-			fill_txdesc_phy(pattrib, &ptxdesc->txdw4);
-
-			ptxdesc->txdw4 |= cpu_to_le32(0x00000008);/* RTS Rate = 24M */
-			ptxdesc->txdw5 |= cpu_to_le32(0x0001ff00);/*  */
-
-			/* use REG_INIDATA_RATE_SEL value */
-			ptxdesc->txdw5 |= cpu_to_le32(pdmpriv->INIDATA_RATE[pattrib->mac_id]);
-		} else {
-			/*  EAP data packet and ARP packet. */
-			/*  Use the 1M data rate to send the EAP/ARP packet. */
-			/*  This will maybe make the handshake smooth. */
-
-			ptxdesc->txdw1 |= cpu_to_le32(BIT(6));/* AGG BK */
-
-			ptxdesc->txdw4 |= cpu_to_le32(BIT(8));/* driver uses rate */
-
-			if (pmlmeinfo->preamble_mode == PREAMBLE_SHORT)
-				ptxdesc->txdw4 |= cpu_to_le32(BIT(24));/*  DATA_SHORT */
-
-			ptxdesc->txdw5 |= cpu_to_le32(MRateToHwRate23a(pmlmeext->tx_rate));
-		}
-	} else if (pxmitframe->frame_tag == MGNT_FRAMETAG) {
-		/* offset 4 */
-		ptxdesc->txdw1 |= cpu_to_le32(pattrib->mac_id&0x1f);
-
-		qsel = (uint)(pattrib->qsel&0x0000001f);
-		ptxdesc->txdw1 |= cpu_to_le32((qsel<<QSEL_SHT)&0x00001f00);
-
-		ptxdesc->txdw1 |= cpu_to_le32((pattrib->raid<<16) & 0x000f0000);
-
-		/* offset 8 */
-		/* CCX-TXRPT ack for xmit mgmt frames. */
-		if (pxmitframe->ack_report)
-			ptxdesc->txdw2 |= cpu_to_le32(BIT(19));
-
-		/* offset 12 */
-		ptxdesc->txdw3 |= cpu_to_le32((pattrib->seqnum<<16)&0xffff0000);
-
-		/* offset 16 */
-		ptxdesc->txdw4 |= cpu_to_le32(BIT(8));/* driver uses rate */
-
-		/* offset 20 */
-		ptxdesc->txdw5 |= cpu_to_le32(BIT(17));/* retry limit enable */
-		ptxdesc->txdw5 |= cpu_to_le32(0x00180000);/* retry limit = 6 */
-
-		ptxdesc->txdw5 |= cpu_to_le32(MRateToHwRate23a(pmlmeext->tx_rate));
-	} else if (pxmitframe->frame_tag == TXAGG_FRAMETAG) {
-		DBG_8723A("pxmitframe->frame_tag == TXAGG_FRAMETAG\n");
-	} else {
-		DBG_8723A("pxmitframe->frame_tag = %d\n",
-			  pxmitframe->frame_tag);
-
-		/* offset 4 */
-		ptxdesc->txdw1 |= cpu_to_le32((4)&0x1f);/* CAM_ID(MAC_ID) */
-
-		ptxdesc->txdw1 |= cpu_to_le32((6<<16) & 0x000f0000);/* raid */
-
-		/* offset 8 */
-
-		/* offset 12 */
-		ptxdesc->txdw3 |= cpu_to_le32((pattrib->seqnum<<16)&0xffff0000);
-
-		/* offset 16 */
-		ptxdesc->txdw4 |= cpu_to_le32(BIT(8));/* driver uses rate */
-
-		/* offset 20 */
-		ptxdesc->txdw5 |= cpu_to_le32(MRateToHwRate23a(pmlmeext->tx_rate));
-	}
-
-	/*  (1) The sequence number of each non-Qos frame / broadcast / multicast / */
-	/*  mgnt frame should be controlled by Hw because Fw will also send null data */
-	/*  which we cannot control when Fw LPS enable. */
-	/*  --> default enable non-Qos data sequense number. 2010.06.23. by tynli. */
-	/*  (2) Enable HW SEQ control for beacon packet, because we use Hw beacon. */
-	/*  (3) Use HW Qos SEQ to control the seq num of Ext port non-Qos packets. */
-	if (!pattrib->qos_en) {
-		/*  Hw set sequence number */
-		ptxdesc->txdw4 |= cpu_to_le32(BIT(7));
-		/* set bit3 to 1. */
-		ptxdesc->txdw3 |= cpu_to_le32((8 << 28));
-	}
-
-	/* offset 0 */
-	ptxdesc->txdw0 |= cpu_to_le32(sz&0x0000ffff);
-	ptxdesc->txdw0 |= cpu_to_le32(OWN | FSG | LSG);
-	ptxdesc->txdw0 |= cpu_to_le32(((TXDESC_SIZE+OFFSET_SZ)<<OFFSET_SHT)&0x00ff0000);/* 32 bytes for TX Desc */
-
-	if (bmcst)
-		ptxdesc->txdw0 |= cpu_to_le32(BIT(24));
-
-	RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-		 "offset0-txdesc = 0x%x\n", ptxdesc->txdw0);
-
-	/* offset 4 */
-	/*  pkt_offset, unit:8 bytes padding */
-	if (pxmitframe->pkt_offset > 0)
-		ptxdesc->txdw1 |= cpu_to_le32((pxmitframe->pkt_offset << 26) & 0x7c000000);
-
-	rtl8192cu_cal_txdesc_chksum(ptxdesc);
-	return pull;
-}
-
-static int rtw_dump_xframe(struct rtw_adapter *padapter,
-			   struct xmit_frame *pxmitframe)
-{
-	int ret = _SUCCESS;
-	int inner_ret = _SUCCESS;
-	int t, sz, w_sz, pull = 0;
-	u8 *mem_addr;
-	u32 ff_hwaddr;
-	struct xmit_buf *pxmitbuf = pxmitframe->pxmitbuf;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	if (pxmitframe->frame_tag == DATA_FRAMETAG &&
-	    pxmitframe->attrib.ether_type != ETH_P_ARP &&
-	    pxmitframe->attrib.ether_type != ETH_P_PAE &&
-	    pxmitframe->attrib.dhcp_pkt != 1)
-		rtw_issue_addbareq_cmd23a(padapter, pxmitframe);
-
-	mem_addr = pxmitframe->buf_addr;
-
-	RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_, "rtw_dump_xframe()\n");
-
-	for (t = 0; t < pattrib->nr_frags; t++) {
-		if (inner_ret != _SUCCESS && ret == _SUCCESS)
-			ret = _FAIL;
-
-		if (t != (pattrib->nr_frags - 1)) {
-			RT_TRACE(_module_rtl871x_xmit_c_, _drv_err_,
-				 "pattrib->nr_frags =%d\n", pattrib->nr_frags);
-
-			sz = pxmitpriv->frag_len;
-			sz = sz - 4 - pattrib->icv_len;
-		} else {
-			/* no frag */
-			sz = pattrib->last_txcmdsz;
-		}
-
-		pull = update_txdesc(pxmitframe, mem_addr, sz);
-
-		if (pull) {
-			mem_addr += PACKET_OFFSET_SZ; /* pull txdesc head */
-
-			pxmitframe->buf_addr = mem_addr;
-
-			w_sz = sz + TXDESC_SIZE;
-		} else {
-			w_sz = sz + TXDESC_SIZE + PACKET_OFFSET_SZ;
-		}
-
-		ff_hwaddr = rtw_get_ff_hwaddr23a(pxmitframe);
-		inner_ret = rtl8723au_write_port(padapter, ff_hwaddr,
-						 w_sz, pxmitbuf);
-		rtw_count_tx_stats23a(padapter, pxmitframe, sz);
-
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "rtw_write_port, w_sz =%d\n", w_sz);
-
-		mem_addr += w_sz;
-
-		mem_addr = PTR_ALIGN(mem_addr, 4);
-	}
-
-	rtw_free_xmitframe23a(pxmitpriv, pxmitframe);
-
-	if  (ret != _SUCCESS)
-		rtw23a_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_UNKNOWN);
-
-	return ret;
-}
-
-bool rtl8723au_xmitframe_complete(struct rtw_adapter *padapter,
-				  struct xmit_priv *pxmitpriv,
-				  struct xmit_buf *pxmitbuf)
-{
-	struct hw_xmit *phwxmits;
-	struct xmit_frame *pxmitframe;
-	int hwentry;
-	int res = _SUCCESS, xcnt = 0;
-
-	phwxmits = pxmitpriv->hwxmits;
-	hwentry = pxmitpriv->hwxmit_entry;
-
-	RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_, "xmitframe_complete()\n");
-
-	if (pxmitbuf == NULL) {
-		pxmitbuf = rtw_alloc_xmitbuf23a(pxmitpriv);
-		if (!pxmitbuf)
-			return false;
-	}
-	pxmitframe =  rtw_dequeue_xframe23a(pxmitpriv, phwxmits, hwentry);
-
-	if (pxmitframe) {
-		pxmitframe->pxmitbuf = pxmitbuf;
-
-		pxmitframe->buf_addr = pxmitbuf->pbuf;
-
-		pxmitbuf->priv_data = pxmitframe;
-
-		if (pxmitframe->frame_tag == DATA_FRAMETAG) {
-			if (pxmitframe->attrib.priority <= 15)/* TID0~15 */
-				res = rtw_xmitframe_coalesce23a(padapter, pxmitframe->pkt, pxmitframe);
-
-			rtw_os_xmit_complete23a(padapter, pxmitframe);/* always return ndis_packet after rtw_xmitframe_coalesce23a */
-		}
-
-		RT_TRACE(_module_rtl871x_xmit_c_, _drv_info_,
-			 "xmitframe_complete(): rtw_dump_xframe\n");
-
-		if (res == _SUCCESS) {
-			rtw_dump_xframe(padapter, pxmitframe);
-		} else {
-			rtw_free_xmitbuf23a(pxmitpriv, pxmitbuf);
-			rtw_free_xmitframe23a(pxmitpriv, pxmitframe);
-		}
-		xcnt++;
-	} else {
-		rtw_free_xmitbuf23a(pxmitpriv, pxmitbuf);
-		return false;
-	}
-	return true;
-}
-
-static int xmitframe_direct(struct rtw_adapter *padapter,
-			    struct xmit_frame *pxmitframe)
-{
-	int res;
-
-	res = rtw_xmitframe_coalesce23a(padapter, pxmitframe->pkt, pxmitframe);
-	if (res == _SUCCESS)
-		rtw_dump_xframe(padapter, pxmitframe);
-	return res;
-}
-
-/*
- * Return
- *	true	dump packet directly
- *	false	enqueue packet
- */
-bool rtl8723au_hal_xmit(struct rtw_adapter *padapter,
-			struct xmit_frame *pxmitframe)
-{
-	int res;
-	struct xmit_buf *pxmitbuf = NULL;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct pkt_attrib *pattrib = &pxmitframe->attrib;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	pattrib->qsel = pattrib->priority;
-	spin_lock_bh(&pxmitpriv->lock);
-
-#ifdef CONFIG_8723AU_AP_MODE
-	if (xmitframe_enqueue_for_sleeping_sta23a(padapter, pxmitframe)) {
-		struct sta_info *psta;
-		struct sta_priv *pstapriv = &padapter->stapriv;
-
-		spin_unlock_bh(&pxmitpriv->lock);
-
-		if (pattrib->psta)
-			psta = pattrib->psta;
-		else
-			psta = rtw_get_stainfo23a(pstapriv, pattrib->ra);
-
-		if (psta) {
-			if (psta->sleepq_len > (NR_XMITFRAME>>3))
-				wakeup_sta_to_xmit23a(padapter, psta);
-		}
-
-		return false;
-	}
-#endif
-
-	if (rtw_txframes_sta_ac_pending23a(padapter, pattrib) > 0)
-		goto enqueue;
-
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY|_FW_UNDER_LINKING) == true)
-		goto enqueue;
-
-	pxmitbuf = rtw_alloc_xmitbuf23a(pxmitpriv);
-	if (pxmitbuf == NULL)
-		goto enqueue;
-
-	spin_unlock_bh(&pxmitpriv->lock);
-
-	pxmitframe->pxmitbuf = pxmitbuf;
-	pxmitframe->buf_addr = pxmitbuf->pbuf;
-	pxmitbuf->priv_data = pxmitframe;
-
-	if (xmitframe_direct(padapter, pxmitframe) != _SUCCESS) {
-		rtw_free_xmitbuf23a(pxmitpriv, pxmitbuf);
-		rtw_free_xmitframe23a(pxmitpriv, pxmitframe);
-	}
-	return true;
-
-enqueue:
-	res = rtw_xmitframe_enqueue23a(padapter, pxmitframe);
-	spin_unlock_bh(&pxmitpriv->lock);
-
-	if (res != _SUCCESS) {
-		RT_TRACE(_module_xmit_osdep_c_, _drv_err_,
-			 "pre_xmitframe: enqueue xmitframe fail\n");
-		rtw_free_xmitframe23a(pxmitpriv, pxmitframe);
-
-		/*  Trick, make the statistics correct */
-		pxmitpriv->tx_pkts--;
-		pxmitpriv->tx_drop++;
-		return true;
-	}
-	return false;
-}
-
-int rtl8723au_mgnt_xmit(struct rtw_adapter *padapter,
-			struct xmit_frame *pmgntframe)
-{
-	return rtw_dump_xframe(padapter, pmgntframe);
-}
-
-int rtl8723au_hal_xmitframe_enqueue(struct rtw_adapter *padapter,
-				    struct xmit_frame *pxmitframe)
-{
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	int err;
-
-	err = rtw_xmitframe_enqueue23a(padapter, pxmitframe);
-	if (err != _SUCCESS) {
-		rtw_free_xmitframe23a(pxmitpriv, pxmitframe);
-
-		/*  Trick, make the statistics correct */
-		pxmitpriv->tx_pkts--;
-		pxmitpriv->tx_drop++;
-	} else {
-		tasklet_hi_schedule(&pxmitpriv->xmit_tasklet);
-	}
-	return err;
-}
diff --git a/drivers/staging/rtl8723au/hal/usb_halinit.c b/drivers/staging/rtl8723au/hal/usb_halinit.c
deleted file mode 100644
index fa47aebf8b98..000000000000
--- a/drivers/staging/rtl8723au/hal/usb_halinit.c
+++ /dev/null
@@ -1,1269 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _HCI_HAL_INIT_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <rtw_efuse.h>
-
-#include <HalPwrSeqCmd.h>
-#include <Hal8723PwrSeq.h>
-#include <rtl8723a_hal.h>
-#include <linux/ieee80211.h>
-
-#include <usb_ops.h>
-
-static void phy_SsPwrSwitch92CU(struct rtw_adapter *Adapter,
-				enum rt_rf_power_state eRFPowerState);
-
-static void
-_ConfigChipOutEP(struct rtw_adapter *pAdapter, u8 NumOutPipe)
-{
-	u8 value8;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter);
-
-	pHalData->OutEpQueueSel = 0;
-	pHalData->OutEpNumber = 0;
-
-	/*  Normal and High queue */
-	value8 = rtl8723au_read8(pAdapter, (REG_NORMAL_SIE_EP + 1));
-
-	if (value8 & USB_NORMAL_SIE_EP_MASK) {
-		pHalData->OutEpQueueSel |= TX_SELE_HQ;
-		pHalData->OutEpNumber++;
-	}
-
-	if ((value8 >> USB_NORMAL_SIE_EP_SHIFT) & USB_NORMAL_SIE_EP_MASK) {
-		pHalData->OutEpQueueSel |= TX_SELE_NQ;
-		pHalData->OutEpNumber++;
-	}
-
-	/*  Low queue */
-	value8 = rtl8723au_read8(pAdapter, (REG_NORMAL_SIE_EP + 2));
-	if (value8 & USB_NORMAL_SIE_EP_MASK) {
-		pHalData->OutEpQueueSel |= TX_SELE_LQ;
-		pHalData->OutEpNumber++;
-	}
-
-	/*  TODO: Error recovery for this case */
-	/* RT_ASSERT((NumOutPipe == pHalData->OutEpNumber),
-	   ("Out EP number isn't match! %d(Descriptor) != %d (SIE reg)\n",
-	   (u32)NumOutPipe, (u32)pHalData->OutEpNumber)); */
-}
-
-bool rtl8723au_chip_configure(struct rtw_adapter *padapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
-	u8 NumInPipe = pdvobjpriv->RtNumInPipes;
-	u8 NumOutPipe = pdvobjpriv->RtNumOutPipes;
-
-	_ConfigChipOutEP(padapter, NumOutPipe);
-
-	/*  Normal chip with one IN and one OUT doesn't have interrupt IN EP. */
-	if (pHalData->OutEpNumber == 1) {
-		if (NumInPipe != 1)
-			return false;
-	}
-
-	return Hal_MappingOutPipe23a(padapter, NumOutPipe);
-}
-
-static int _InitPowerOn(struct rtw_adapter *padapter)
-{
-	u16 value16;
-	u8 value8;
-
-	/*  RSV_CTRL 0x1C[7:0] = 0x00
-	    unlock ISO/CLK/Power control register */
-	rtl8723au_write8(padapter, REG_RSV_CTRL, 0x0);
-
-	/*  HW Power on sequence */
-	if (!HalPwrSeqCmdParsing23a(padapter, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK,
-				 PWR_INTF_USB_MSK, rtl8723AU_card_enable_flow))
-		return _FAIL;
-
-	/*  0x04[19] = 1, suggest by Jackie 2011.05.09, reset 8051 */
-	value8 = rtl8723au_read8(padapter, REG_APS_FSMCO+2);
-	rtl8723au_write8(padapter, REG_APS_FSMCO + 2, value8 | BIT(3));
-
-	/*  Enable MAC DMA/WMAC/SCHEDULE/SEC block */
-	/*  Set CR bit10 to enable 32k calibration. Suggested by SD1 Gimmy.
-	    Added by tynli. 2011.08.31. */
-	value16 = rtl8723au_read16(padapter, REG_CR);
-	value16 |= (HCI_TXDMA_EN | HCI_RXDMA_EN | TXDMA_EN | RXDMA_EN |
-		    PROTOCOL_EN | SCHEDULE_EN | MACTXEN | MACRXEN |
-		    ENSEC | CALTMR_EN);
-	rtl8723au_write16(padapter, REG_CR, value16);
-
-	/* for Efuse PG, suggest by Jackie 2011.11.23 */
-	PHY_SetBBReg(padapter, REG_EFUSE_CTRL, BIT(28)|BIT(29)|BIT(30), 0x06);
-
-	return _SUCCESS;
-}
-
-/*  Shall USB interface init this? */
-static void _InitInterrupt(struct rtw_adapter *Adapter)
-{
-	u32 value32;
-
-	/*  HISR - turn all on */
-	value32 = 0xFFFFFFFF;
-	rtl8723au_write32(Adapter, REG_HISR, value32);
-
-	/*  HIMR - turn all on */
-	rtl8723au_write32(Adapter, REG_HIMR, value32);
-}
-
-static void _InitQueueReservedPage(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct registry_priv *pregistrypriv = &Adapter->registrypriv;
-	u32 numHQ = 0;
-	u32 numLQ = 0;
-	u32 numNQ = 0;
-	u32 numPubQ;
-	u32 value32;
-	u8 value8;
-	bool bWiFiConfig = pregistrypriv->wifi_spec;
-
-	/* RT_ASSERT((outEPNum>= 2), ("for WMM , number of out-ep "
-	   "must more than or equal to 2!\n")); */
-
-	numPubQ = bWiFiConfig ? WMM_NORMAL_PAGE_NUM_PUBQ : NORMAL_PAGE_NUM_PUBQ;
-
-	if (pHalData->OutEpQueueSel & TX_SELE_HQ) {
-		numHQ = bWiFiConfig ?
-			WMM_NORMAL_PAGE_NUM_HPQ : NORMAL_PAGE_NUM_HPQ;
-	}
-
-	if (pHalData->OutEpQueueSel & TX_SELE_LQ) {
-		numLQ = bWiFiConfig ?
-			WMM_NORMAL_PAGE_NUM_LPQ : NORMAL_PAGE_NUM_LPQ;
-	}
-	/*  NOTE: This step shall be proceed before
-	    writting REG_RQPN. */
-	if (pHalData->OutEpQueueSel & TX_SELE_NQ) {
-		numNQ = bWiFiConfig ?
-			WMM_NORMAL_PAGE_NUM_NPQ : NORMAL_PAGE_NUM_NPQ;
-	}
-	value8 = (u8)_NPQ(numNQ);
-	rtl8723au_write8(Adapter, REG_RQPN_NPQ, value8);
-
-	/*  TX DMA */
-	value32 = _HPQ(numHQ) | _LPQ(numLQ) | _PUBQ(numPubQ) | LD_RQPN;
-	rtl8723au_write32(Adapter, REG_RQPN, value32);
-}
-
-static void _InitTxBufferBoundary(struct rtw_adapter *Adapter)
-{
-	struct registry_priv *pregistrypriv = &Adapter->registrypriv;
-
-	u8 txpktbuf_bndy;
-
-	if (!pregistrypriv->wifi_spec)
-		txpktbuf_bndy = TX_PAGE_BOUNDARY;
-	else /* for WMM */
-		txpktbuf_bndy = WMM_NORMAL_TX_PAGE_BOUNDARY;
-
-	rtl8723au_write8(Adapter, REG_TXPKTBUF_BCNQ_BDNY, txpktbuf_bndy);
-	rtl8723au_write8(Adapter, REG_TXPKTBUF_MGQ_BDNY, txpktbuf_bndy);
-	rtl8723au_write8(Adapter, REG_TXPKTBUF_WMAC_LBK_BF_HD, txpktbuf_bndy);
-	rtl8723au_write8(Adapter, REG_TRXFF_BNDY, txpktbuf_bndy);
-	rtl8723au_write8(Adapter, REG_TDECTRL+1, txpktbuf_bndy);
-}
-
-static void _InitPageBoundary(struct rtw_adapter *Adapter)
-{
-	/*  RX Page Boundary */
-	/* srand(static_cast<unsigned int>(time(NULL))); */
-	u16 rxff_bndy = 0x27FF;/* rand() % 1) ? 0x27FF : 0x23FF; */
-
-	rtl8723au_write16(Adapter, (REG_TRXFF_BNDY + 2), rxff_bndy);
-
-	/*  TODO: ?? shall we set tx boundary? */
-}
-
-static void
-_InitNormalChipRegPriority(struct rtw_adapter *Adapter, u16 beQ, u16 bkQ,
-			   u16 viQ, u16 voQ, u16 mgtQ, u16 hiQ)
-{
-	u16 value16 = rtl8723au_read16(Adapter, REG_TRXDMA_CTRL) & 0x7;
-
-	value16 |= _TXDMA_BEQ_MAP(beQ) | _TXDMA_BKQ_MAP(bkQ) |
-		_TXDMA_VIQ_MAP(viQ) | _TXDMA_VOQ_MAP(voQ) |
-		_TXDMA_MGQ_MAP(mgtQ) | _TXDMA_HIQ_MAP(hiQ);
-
-	rtl8723au_write16(Adapter, REG_TRXDMA_CTRL, value16);
-}
-
-static void _InitNormalChipOneOutEpPriority(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	u16 value = 0;
-
-	switch (pHalData->OutEpQueueSel) {
-	case TX_SELE_HQ:
-		value = QUEUE_HIGH;
-		break;
-	case TX_SELE_LQ:
-		value = QUEUE_LOW;
-		break;
-	case TX_SELE_NQ:
-		value = QUEUE_NORMAL;
-		break;
-	default:
-		/* RT_ASSERT(false, ("Shall not reach here!\n")); */
-		break;
-	}
-
-	_InitNormalChipRegPriority(Adapter, value, value, value,
-				   value, value, value);
-}
-
-static void _InitNormalChipTwoOutEpPriority(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct registry_priv *pregistrypriv = &Adapter->registrypriv;
-	u16 beQ, bkQ, viQ, voQ, mgtQ, hiQ;
-	u16 valueHi = 0;
-	u16 valueLow = 0;
-
-	switch (pHalData->OutEpQueueSel) {
-	case (TX_SELE_HQ | TX_SELE_LQ):
-		valueHi = QUEUE_HIGH;
-		valueLow = QUEUE_LOW;
-		break;
-	case (TX_SELE_NQ | TX_SELE_LQ):
-		valueHi = QUEUE_NORMAL;
-		valueLow = QUEUE_LOW;
-		break;
-	case (TX_SELE_HQ | TX_SELE_NQ):
-		valueHi = QUEUE_HIGH;
-		valueLow = QUEUE_NORMAL;
-		break;
-	default:
-		/* RT_ASSERT(false, ("Shall not reach here!\n")); */
-		break;
-	}
-
-	if (!pregistrypriv->wifi_spec) {
-		beQ = valueLow;
-		bkQ = valueLow;
-		viQ = valueHi;
-		voQ = valueHi;
-		mgtQ = valueHi;
-		hiQ = valueHi;
-	} else {/* for WMM , CONFIG_OUT_EP_WIFI_MODE */
-		beQ = valueLow;
-		bkQ = valueHi;
-		viQ = valueHi;
-		voQ = valueLow;
-		mgtQ = valueHi;
-		hiQ = valueHi;
-	}
-
-	_InitNormalChipRegPriority(Adapter, beQ, bkQ, viQ, voQ, mgtQ, hiQ);
-}
-
-static void _InitNormalChipThreeOutEpPriority(struct rtw_adapter *Adapter)
-{
-	struct registry_priv *pregistrypriv = &Adapter->registrypriv;
-	u16 beQ, bkQ, viQ, voQ, mgtQ, hiQ;
-
-	if (!pregistrypriv->wifi_spec) {/*  typical setting */
-		beQ = QUEUE_LOW;
-		bkQ = QUEUE_LOW;
-		viQ = QUEUE_NORMAL;
-		voQ = QUEUE_HIGH;
-		mgtQ = QUEUE_HIGH;
-		hiQ = QUEUE_HIGH;
-	} else {/*  for WMM */
-		beQ = QUEUE_LOW;
-		bkQ = QUEUE_NORMAL;
-		viQ = QUEUE_NORMAL;
-		voQ = QUEUE_HIGH;
-		mgtQ = QUEUE_HIGH;
-		hiQ = QUEUE_HIGH;
-	}
-	_InitNormalChipRegPriority(Adapter, beQ, bkQ, viQ, voQ, mgtQ, hiQ);
-}
-
-static void _InitQueuePriority(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	switch (pHalData->OutEpNumber) {
-	case 1:
-		_InitNormalChipOneOutEpPriority(Adapter);
-		break;
-	case 2:
-		_InitNormalChipTwoOutEpPriority(Adapter);
-		break;
-	case 3:
-		_InitNormalChipThreeOutEpPriority(Adapter);
-		break;
-	default:
-		/* RT_ASSERT(false, ("Shall not reach here!\n")); */
-		break;
-	}
-}
-
-static void _InitTransferPageSize(struct rtw_adapter *Adapter)
-{
-	/*  Tx page size is always 128. */
-
-	u8 value8;
-	value8 = _PSRX(PBP_128) | _PSTX(PBP_128);
-	rtl8723au_write8(Adapter, REG_PBP, value8);
-}
-
-static void _InitDriverInfoSize(struct rtw_adapter *Adapter, u8 drvInfoSize)
-{
-	rtl8723au_write8(Adapter, REG_RX_DRVINFO_SZ, drvInfoSize);
-}
-
-static void _InitWMACSetting(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	/*  don't turn on AAP, it will allow all packets to driver */
-	pHalData->ReceiveConfig = RCR_APM | RCR_AM | RCR_AB | RCR_CBSSID_DATA |
-				  RCR_CBSSID_BCN | RCR_APP_ICV | RCR_AMF |
-				  RCR_HTC_LOC_CTRL | RCR_APP_MIC |
-				  RCR_APP_PHYSTS;
-
-	/*  some REG_RCR will be modified later by
-	    phy_ConfigMACWithHeaderFile() */
-	rtl8723au_write32(Adapter, REG_RCR, pHalData->ReceiveConfig);
-
-	/*  Accept all multicast address */
-	rtl8723au_write32(Adapter, REG_MAR, 0xFFFFFFFF);
-	rtl8723au_write32(Adapter, REG_MAR + 4, 0xFFFFFFFF);
-
-	/*  Accept all data frames */
-	/* value16 = 0xFFFF; */
-	/* rtl8723au_write16(Adapter, REG_RXFLTMAP2, value16); */
-
-	/*  2010.09.08 hpfan */
-	/*  Since ADF is removed from RCR, ps-poll will not be indicate
-	    to driver, */
-	/*  RxFilterMap should mask ps-poll to guarantee AP mode can
-	    rx ps-poll. */
-	/* value16 = 0x400; */
-	/* rtl8723au_write16(Adapter, REG_RXFLTMAP1, value16); */
-
-	/*  Accept all management frames */
-	/* value16 = 0xFFFF; */
-	/* rtl8723au_write16(Adapter, REG_RXFLTMAP0, value16); */
-
-	/* enable RX_SHIFT bits */
-	/* rtl8723au_write8(Adapter, REG_TRXDMA_CTRL, rtl8723au_read8(Adapter,
-	   REG_TRXDMA_CTRL)|BIT(1)); */
-}
-
-static void _InitAdaptiveCtrl(struct rtw_adapter *Adapter)
-{
-	u16 value16;
-	u32 value32;
-
-	/*  Response Rate Set */
-	value32 = rtl8723au_read32(Adapter, REG_RRSR);
-	value32 &= ~RATE_BITMAP_ALL;
-	value32 |= RATE_RRSR_CCK_ONLY_1M;
-	rtl8723au_write32(Adapter, REG_RRSR, value32);
-
-	/*  CF-END Threshold */
-	/* m_spIoBase->rtl8723au_write8(REG_CFEND_TH, 0x1); */
-
-	/*  SIFS (used in NAV) */
-	value16 = _SPEC_SIFS_CCK(0x10) | _SPEC_SIFS_OFDM(0x10);
-	rtl8723au_write16(Adapter, REG_SPEC_SIFS, value16);
-
-	/*  Retry Limit */
-	value16 = _LRL(0x30) | _SRL(0x30);
-	rtl8723au_write16(Adapter, REG_RL, value16);
-}
-
-static void _InitRateFallback(struct rtw_adapter *Adapter)
-{
-	/*  Set Data Auto Rate Fallback Retry Count register. */
-	rtl8723au_write32(Adapter, REG_DARFRC, 0x00000000);
-	rtl8723au_write32(Adapter, REG_DARFRC+4, 0x10080404);
-	rtl8723au_write32(Adapter, REG_RARFRC, 0x04030201);
-	rtl8723au_write32(Adapter, REG_RARFRC+4, 0x08070605);
-}
-
-static void _InitEDCA(struct rtw_adapter *Adapter)
-{
-	/*  Set Spec SIFS (used in NAV) */
-	rtl8723au_write16(Adapter, REG_SPEC_SIFS, 0x100a);
-	rtl8723au_write16(Adapter, REG_MAC_SPEC_SIFS, 0x100a);
-
-	/*  Set SIFS for CCK */
-	rtl8723au_write16(Adapter, REG_SIFS_CTX, 0x100a);
-
-	/*  Set SIFS for OFDM */
-	rtl8723au_write16(Adapter, REG_SIFS_TRX, 0x100a);
-
-	/*  TXOP */
-	rtl8723au_write32(Adapter, REG_EDCA_BE_PARAM, 0x005EA42B);
-	rtl8723au_write32(Adapter, REG_EDCA_BK_PARAM, 0x0000A44F);
-	rtl8723au_write32(Adapter, REG_EDCA_VI_PARAM, 0x005EA324);
-	rtl8723au_write32(Adapter, REG_EDCA_VO_PARAM, 0x002FA226);
-}
-
-static void _InitRDGSetting(struct rtw_adapter *Adapter)
-{
-	rtl8723au_write8(Adapter, REG_RD_CTRL, 0xFF);
-	rtl8723au_write16(Adapter, REG_RD_NAV_NXT, 0x200);
-	rtl8723au_write8(Adapter, REG_RD_RESP_PKT_TH, 0x05);
-}
-
-static void _InitRetryFunction(struct rtw_adapter *Adapter)
-{
-	u8 value8;
-
-	value8 = rtl8723au_read8(Adapter, REG_FWHW_TXQ_CTRL);
-	value8 |= EN_AMPDU_RTY_NEW;
-	rtl8723au_write8(Adapter, REG_FWHW_TXQ_CTRL, value8);
-
-	/*  Set ACK timeout */
-	rtl8723au_write8(Adapter, REG_ACKTO, 0x40);
-}
-
-static void _InitRFType(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	pHalData->rf_type = RF_1T1R;
-}
-
-/*  Set CCK and OFDM Block "ON" */
-static void _BBTurnOnBlock(struct rtw_adapter *Adapter)
-{
-	PHY_SetBBReg(Adapter, rFPGA0_RFMOD, bCCKEn, 0x1);
-	PHY_SetBBReg(Adapter, rFPGA0_RFMOD, bOFDMEn, 0x1);
-}
-
-#define MgntActSet_RF_State(...)
-static void _RfPowerSave(struct rtw_adapter *padapter)
-{
-}
-
-enum {
-	Antenna_Lfet = 1,
-	Antenna_Right = 2,
-};
-
-enum rt_rf_power_state RfOnOffDetect23a(struct rtw_adapter *pAdapter)
-{
-	/* struct hal_data_8723a *pHalData = GET_HAL_DATA(pAdapter); */
-	u8 val8;
-	enum rt_rf_power_state rfpowerstate = rf_off;
-
-	rtl8723au_write8(pAdapter, REG_MAC_PINMUX_CFG,
-			 rtl8723au_read8(pAdapter,
-					 REG_MAC_PINMUX_CFG) & ~BIT(3));
-	val8 = rtl8723au_read8(pAdapter, REG_GPIO_IO_SEL);
-	DBG_8723A("GPIO_IN =%02x\n", val8);
-	rfpowerstate = (val8 & BIT(3)) ? rf_on : rf_off;
-
-	return rfpowerstate;
-}
-
-int rtl8723au_hal_init(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	struct pwrctrl_priv *pwrctrlpriv = &Adapter->pwrctrlpriv;
-	struct registry_priv *pregistrypriv = &Adapter->registrypriv;
-	u8 val8 = 0;
-	u32 boundary;
-	int status = _SUCCESS;
-	bool mac_on;
-
-	unsigned long init_start_time = jiffies;
-
-	Adapter->hw_init_completed = false;
-
-	if (Adapter->pwrctrlpriv.bkeepfwalive) {
-		phy_SsPwrSwitch92CU(Adapter, rf_on);
-
-		if (pHalData->bIQKInitialized) {
-			rtl8723a_phy_iq_calibrate(Adapter, true);
-		} else {
-			rtl8723a_phy_iq_calibrate(Adapter, false);
-			pHalData->bIQKInitialized = true;
-		}
-		rtl8723a_odm_check_tx_power_tracking(Adapter);
-		rtl8723a_phy_lc_calibrate(Adapter);
-
-		goto exit;
-	}
-
-	/*  Check if MAC has already power on. by tynli. 2011.05.27. */
-	val8 = rtl8723au_read8(Adapter, REG_CR);
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "%s: REG_CR 0x100 = 0x%02x\n", __func__, val8);
-	/* Fix 92DU-VC S3 hang with the reason is that secondary mac is not
-	   initialized. */
-	/* 0x100 value of first mac is 0xEA while 0x100 value of secondary
-	   is 0x00 */
-	if (val8 == 0xEA) {
-		mac_on = false;
-	} else {
-		mac_on = true;
-		RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-			 "%s: MAC has already power on\n", __func__);
-	}
-
-	status = _InitPowerOn(Adapter);
-	if (status == _FAIL) {
-		RT_TRACE(_module_hci_hal_init_c_, _drv_err_,
-			 "Failed to init power on!\n");
-		goto exit;
-	}
-
-	if (!pregistrypriv->wifi_spec) {
-		boundary = TX_PAGE_BOUNDARY;
-	} else {
-		/*  for WMM */
-		boundary = WMM_NORMAL_TX_PAGE_BOUNDARY;
-	}
-
-	if (!mac_on) {
-		status =  InitLLTTable23a(Adapter, boundary);
-		if (status == _FAIL) {
-			RT_TRACE(_module_hci_hal_init_c_, _drv_err_,
-				 "Failed to init LLT table\n");
-			goto exit;
-		}
-	}
-
-	if (pHalData->bRDGEnable)
-		_InitRDGSetting(Adapter);
-
-	status = rtl8723a_FirmwareDownload(Adapter);
-	if (status != _SUCCESS) {
-		Adapter->bFWReady = false;
-		DBG_8723A("fw download fail!\n");
-		goto exit;
-	} else {
-		Adapter->bFWReady = true;
-		DBG_8723A("fw download ok!\n");
-	}
-
-	rtl8723a_InitializeFirmwareVars(Adapter);
-
-	if (pwrctrlpriv->reg_rfoff == true) {
-		pwrctrlpriv->rf_pwrstate = rf_off;
-	}
-
-	/*  2010/08/09 MH We need to check if we need to turnon or off RF after detecting */
-	/*  HW GPIO pin. Before PHY_RFConfig8192C. */
-	/* HalDetectPwrDownMode(Adapter); */
-	/*  2010/08/26 MH If Efuse does not support sective suspend then disable the function. */
-	/* HalDetectSelectiveSuspendMode(Adapter); */
-
-	/*  Set RF type for BB/RF configuration */
-	_InitRFType(Adapter);/* _ReadRFType() */
-
-	/*  Save target channel */
-	/*  <Roger_Notes> Current Channel will be updated again later. */
-	pHalData->CurrentChannel = 6;/* default set to 6 */
-
-	status = PHY_MACConfig8723A(Adapter);
-	if (status == _FAIL) {
-		DBG_8723A("PHY_MACConfig8723A fault !!\n");
-		goto exit;
-	}
-
-	/*  */
-	/* d. Initialize BB related configurations. */
-	/*  */
-	status = PHY_BBConfig8723A(Adapter);
-	if (status == _FAIL) {
-		DBG_8723A("PHY_BBConfig8723A fault !!\n");
-		goto exit;
-	}
-
-	/*  Add for tx power by rate fine tune. We need to call the function after BB config. */
-	/*  Because the tx power by rate table is inited in BB config. */
-
-	status = PHY_RF6052_Config8723A(Adapter);
-	if (status == _FAIL) {
-		DBG_8723A("PHY_RF6052_Config8723A failed!!\n");
-		goto exit;
-	}
-
-	/* reducing 80M spur */
-	rtl8723au_write32(Adapter, REG_AFE_XTAL_CTRL, 0x0381808d);
-	rtl8723au_write32(Adapter, REG_AFE_PLL_CTRL, 0xf0ffff83);
-	rtl8723au_write32(Adapter, REG_AFE_PLL_CTRL, 0xf0ffff82);
-	rtl8723au_write32(Adapter, REG_AFE_PLL_CTRL, 0xf0ffff83);
-
-	/* RFSW Control */
-	/* 0x804[14]= 0 */
-	rtl8723au_write32(Adapter, rFPGA0_TxInfo, 0x00000003);
-	/* 0x870[6:5]= b'11 */
-	rtl8723au_write32(Adapter, rFPGA0_XAB_RFInterfaceSW, 0x07000760);
-	/* 0x860[6:5]= b'00 */
-	rtl8723au_write32(Adapter, rFPGA0_XA_RFInterfaceOE, 0x66F60210);
-
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "%s: 0x870 = value 0x%x\n", __func__,
-		 rtl8723au_read32(Adapter, 0x870));
-
-	/*  */
-	/*  Joseph Note: Keep RfRegChnlVal for later use. */
-	/*  */
-	pHalData->RfRegChnlVal[0] = PHY_QueryRFReg(Adapter, RF_PATH_A,
-						   RF_CHNLBW, bRFRegOffsetMask);
-	pHalData->RfRegChnlVal[1] = PHY_QueryRFReg(Adapter, RF_PATH_B,
-						   RF_CHNLBW, bRFRegOffsetMask);
-
-	if (!mac_on) {
-		_InitQueueReservedPage(Adapter);
-		_InitTxBufferBoundary(Adapter);
-	}
-	_InitQueuePriority(Adapter);
-	_InitPageBoundary(Adapter);
-	_InitTransferPageSize(Adapter);
-
-	/*  Get Rx PHY status in order to report RSSI and others. */
-	_InitDriverInfoSize(Adapter, DRVINFO_SZ);
-
-	_InitInterrupt(Adapter);
-	hw_var_set_macaddr(Adapter, Adapter->eeprompriv.mac_addr);
-	rtl8723a_set_media_status(Adapter, MSR_INFRA);
-	_InitWMACSetting(Adapter);
-	_InitAdaptiveCtrl(Adapter);
-	_InitEDCA(Adapter);
-	_InitRateFallback(Adapter);
-	_InitRetryFunction(Adapter);
-	rtl8723a_InitBeaconParameters(Adapter);
-
-	_BBTurnOnBlock(Adapter);
-	/* NicIFSetMacAddress(padapter, padapter->PermanentAddress); */
-
-	rtl8723a_cam_invalidate_all(Adapter);
-
-	/*  2010/12/17 MH We need to set TX power according to EFUSE content at first. */
-	PHY_SetTxPowerLevel8723A(Adapter, pHalData->CurrentChannel);
-
-	rtl8723a_InitAntenna_Selection(Adapter);
-
-	/*  HW SEQ CTRL */
-	/* set 0x0 to 0xFF by tynli. Default enable HW SEQ NUM. */
-	rtl8723au_write8(Adapter, REG_HWSEQ_CTRL, 0xFF);
-
-	/*  */
-	/*  Disable BAR, suggested by Scott */
-	/*  2010.04.09 add by hpfan */
-	/*  */
-	rtl8723au_write32(Adapter, REG_BAR_MODE_CTRL, 0x0201ffff);
-
-	if (pregistrypriv->wifi_spec)
-		rtl8723au_write16(Adapter, REG_FAST_EDCA_CTRL, 0);
-
-	/*  Move by Neo for USB SS from above setp */
-	_RfPowerSave(Adapter);
-
-	/*  2010/08/26 MH Merge from 8192CE. */
-	/* sherry masked that it has been done in _RfPowerSave */
-	/* 20110927 */
-	/* recovery for 8192cu and 9723Au 20111017 */
-	if (pwrctrlpriv->rf_pwrstate == rf_on) {
-		if (pHalData->bIQKInitialized) {
-			rtl8723a_phy_iq_calibrate(Adapter, true);
-		} else {
-			rtl8723a_phy_iq_calibrate(Adapter, false);
-			pHalData->bIQKInitialized = true;
-		}
-
-		rtl8723a_odm_check_tx_power_tracking(Adapter);
-
-		rtl8723a_phy_lc_calibrate(Adapter);
-
-		rtl8723a_dual_antenna_detection(Adapter);
-	}
-
-	/* fixed USB interface interference issue */
-	rtl8723au_write8(Adapter, 0xfe40, 0xe0);
-	rtl8723au_write8(Adapter, 0xfe41, 0x8d);
-	rtl8723au_write8(Adapter, 0xfe42, 0x80);
-	rtl8723au_write32(Adapter, 0x20c, 0xfd0320);
-	/* Solve too many protocol error on USB bus */
-	if (!IS_81xxC_VENDOR_UMC_A_CUT(pHalData->VersionID)) {
-		/*  0xE6 = 0x94 */
-		rtl8723au_write8(Adapter, 0xFE40, 0xE6);
-		rtl8723au_write8(Adapter, 0xFE41, 0x94);
-		rtl8723au_write8(Adapter, 0xFE42, 0x80);
-
-		/*  0xE0 = 0x19 */
-		rtl8723au_write8(Adapter, 0xFE40, 0xE0);
-		rtl8723au_write8(Adapter, 0xFE41, 0x19);
-		rtl8723au_write8(Adapter, 0xFE42, 0x80);
-
-		/*  0xE5 = 0x91 */
-		rtl8723au_write8(Adapter, 0xFE40, 0xE5);
-		rtl8723au_write8(Adapter, 0xFE41, 0x91);
-		rtl8723au_write8(Adapter, 0xFE42, 0x80);
-
-		/*  0xE2 = 0x81 */
-		rtl8723au_write8(Adapter, 0xFE40, 0xE2);
-		rtl8723au_write8(Adapter, 0xFE41, 0x81);
-		rtl8723au_write8(Adapter, 0xFE42, 0x80);
-
-	}
-
-/*	_InitPABias(Adapter); */
-
-	/*  Init BT hw config. */
-	rtl8723a_BT_init_hwconfig(Adapter);
-
-	rtl8723a_InitHalDm(Adapter);
-
-	val8 = DIV_ROUND_UP(WiFiNavUpperUs, HAL_8723A_NAV_UPPER_UNIT);
-	rtl8723au_write8(Adapter, REG_NAV_UPPER, val8);
-
-	/*  2011/03/09 MH debug only, UMC-B cut pass 2500 S5 test, but we need to fin root cause. */
-	if (((rtl8723au_read32(Adapter, rFPGA0_RFMOD) & 0xFF000000) !=
-	     0x83000000)) {
-		PHY_SetBBReg(Adapter, rFPGA0_RFMOD, BIT(24), 1);
-		RT_TRACE(_module_hci_hal_init_c_, _drv_err_,
-			 "%s: IQK fail recover\n", __func__);
-	}
-
-	/* ack for xmit mgmt frames. */
-	rtl8723au_write32(Adapter, REG_FWHW_TXQ_CTRL,
-			  rtl8723au_read32(Adapter, REG_FWHW_TXQ_CTRL)|BIT(12));
-
-exit:
-	if (status == _SUCCESS) {
-		Adapter->hw_init_completed = true;
-
-		if (Adapter->registrypriv.notch_filter == 1)
-			rtl8723a_notch_filter(Adapter, 1);
-	}
-
-	DBG_8723A("%s in %dms\n", __func__,
-		  jiffies_to_msecs(jiffies - init_start_time));
-	return status;
-}
-
-static void phy_SsPwrSwitch92CU(struct rtw_adapter *Adapter,
-				enum rt_rf_power_state eRFPowerState)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	u8 sps0;
-
-	sps0 = rtl8723au_read8(Adapter, REG_SPS0_CTRL);
-
-	switch (eRFPowerState) {
-	case rf_on:
-		/*  1. Enable MAC Clock. Can not be enabled now. */
-		/* WriteXBYTE(REG_SYS_CLKR+1,
-		   ReadXBYTE(REG_SYS_CLKR+1) | BIT(3)); */
-
-		/*  2. Force PWM, Enable SPS18_LDO_Marco_Block */
-		rtl8723au_write8(Adapter, REG_SPS0_CTRL,
-				 sps0 | BIT(0) | BIT(3));
-
-		/*  3. restore BB, AFE control register. */
-		/* RF */
-		if (pHalData->rf_type ==  RF_2T2R)
-			PHY_SetBBReg(Adapter, rFPGA0_XAB_RFParameter,
-				     0x380038, 1);
-		else
-			PHY_SetBBReg(Adapter, rFPGA0_XAB_RFParameter,
-				     0x38, 1);
-		PHY_SetBBReg(Adapter, rOFDM0_TRxPathEnable, 0xf0, 1);
-		PHY_SetBBReg(Adapter, rFPGA0_RFMOD, BIT(1), 0);
-
-		/* AFE */
-		if (pHalData->rf_type ==  RF_2T2R)
-			rtl8723au_write32(Adapter, rRx_Wait_CCA, 0x63DB25A0);
-		else if (pHalData->rf_type ==  RF_1T1R)
-			rtl8723au_write32(Adapter, rRx_Wait_CCA, 0x631B25A0);
-
-		/*  4. issue 3-wire command that RF set to Rx idle
-		    mode. This is used to re-write the RX idle mode. */
-		/*  We can only prvide a usual value instead and then
-		    HW will modify the value by itself. */
-		PHY_SetRFReg(Adapter, RF_PATH_A, RF_AC,
-			     bRFRegOffsetMask, 0x32D95);
-		if (pHalData->rf_type ==  RF_2T2R) {
-			PHY_SetRFReg(Adapter, RF_PATH_B, RF_AC,
-				     bRFRegOffsetMask, 0x32D95);
-		}
-		break;
-	case rf_sleep:
-	case rf_off:
-		if (IS_81xxC_VENDOR_UMC_B_CUT(pHalData->VersionID))
-			sps0 &= ~BIT(0);
-		else
-			sps0 &= ~(BIT(0) | BIT(3));
-
-		RT_TRACE(_module_hal_init_c_, _drv_err_, "SS LVL1\n");
-		/*  Disable RF and BB only for SelectSuspend. */
-
-		/*  1. Set BB/RF to shutdown. */
-		/*	(1) Reg878[5:3]= 0	RF rx_code for
-						preamble power saving */
-		/*	(2)Reg878[21:19]= 0	Turn off RF-B */
-		/*	(3) RegC04[7:4]= 0	Turn off all paths
-						for packet detection */
-		/*	(4) Reg800[1] = 1	enable preamble power saving */
-		Adapter->pwrctrlpriv.PS_BBRegBackup[PSBBREG_RF0] =
-			rtl8723au_read32(Adapter, rFPGA0_XAB_RFParameter);
-		Adapter->pwrctrlpriv.PS_BBRegBackup[PSBBREG_RF1] =
-			rtl8723au_read32(Adapter, rOFDM0_TRxPathEnable);
-		Adapter->pwrctrlpriv.PS_BBRegBackup[PSBBREG_RF2] =
-			rtl8723au_read32(Adapter, rFPGA0_RFMOD);
-		if (pHalData->rf_type ==  RF_2T2R)
-			PHY_SetBBReg(Adapter, rFPGA0_XAB_RFParameter,
-				     0x380038, 0);
-		else if (pHalData->rf_type ==  RF_1T1R)
-			PHY_SetBBReg(Adapter, rFPGA0_XAB_RFParameter, 0x38, 0);
-		PHY_SetBBReg(Adapter, rOFDM0_TRxPathEnable, 0xf0, 0);
-		PHY_SetBBReg(Adapter, rFPGA0_RFMOD, BIT(1), 1);
-
-		/*  2 .AFE control register to power down. bit[30:22] */
-		Adapter->pwrctrlpriv.PS_BBRegBackup[PSBBREG_AFE0] =
-			rtl8723au_read32(Adapter, rRx_Wait_CCA);
-		if (pHalData->rf_type ==  RF_2T2R)
-			rtl8723au_write32(Adapter, rRx_Wait_CCA, 0x00DB25A0);
-		else if (pHalData->rf_type ==  RF_1T1R)
-			rtl8723au_write32(Adapter, rRx_Wait_CCA, 0x001B25A0);
-
-		/*  3. issue 3-wire command that RF set to power down.*/
-		PHY_SetRFReg(Adapter, RF_PATH_A, RF_AC, bRFRegOffsetMask, 0);
-		if (pHalData->rf_type ==  RF_2T2R)
-			PHY_SetRFReg(Adapter, RF_PATH_B, RF_AC,
-				     bRFRegOffsetMask, 0);
-
-		/*  4. Force PFM , disable SPS18_LDO_Marco_Block */
-		rtl8723au_write8(Adapter, REG_SPS0_CTRL, sps0);
-		break;
-	default:
-		break;
-	}
-}
-
-static void CardDisableRTL8723U(struct rtw_adapter *Adapter)
-{
-	u8		u1bTmp;
-
-	DBG_8723A("CardDisableRTL8723U\n");
-	/*  USB-MF Card Disable Flow */
-	/*  1. Run LPS WL RFOFF flow */
-	HalPwrSeqCmdParsing23a(Adapter, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK,
-			    PWR_INTF_USB_MSK, rtl8723AU_enter_lps_flow);
-
-	/*  2. 0x1F[7:0] = 0		turn off RF */
-	rtl8723au_write8(Adapter, REG_RF_CTRL, 0x00);
-
-	/*	==== Reset digital sequence   ====== */
-	if ((rtl8723au_read8(Adapter, REG_MCUFWDL) & BIT(7)) &&
-	    Adapter->bFWReady) /* 8051 RAM code */
-		rtl8723a_FirmwareSelfReset(Adapter);
-
-	/*  Reset MCU. Suggested by Filen. 2011.01.26. by tynli. */
-	u1bTmp = rtl8723au_read8(Adapter, REG_SYS_FUNC_EN+1);
-	rtl8723au_write8(Adapter, REG_SYS_FUNC_EN+1, u1bTmp & ~BIT(2));
-
-	/*  g.	MCUFWDL 0x80[1:0]= 0		reset MCU ready status */
-	rtl8723au_write8(Adapter, REG_MCUFWDL, 0x00);
-
-	/*	==== Reset digital sequence end ====== */
-	/*  Card disable power action flow */
-	HalPwrSeqCmdParsing23a(Adapter, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK,
-			       PWR_INTF_USB_MSK,
-			       rtl8723AU_card_disable_flow);
-
-	/*  Reset MCU IO Wrapper, added by Roger, 2011.08.30. */
-	u1bTmp = rtl8723au_read8(Adapter, REG_RSV_CTRL + 1);
-	rtl8723au_write8(Adapter, REG_RSV_CTRL+1, u1bTmp & ~BIT(0));
-	u1bTmp = rtl8723au_read8(Adapter, REG_RSV_CTRL + 1);
-	rtl8723au_write8(Adapter, REG_RSV_CTRL+1, u1bTmp | BIT(0));
-
-	/*  7. RSV_CTRL 0x1C[7:0] = 0x0E  lock ISO/CLK/Power control register */
-	rtl8723au_write8(Adapter, REG_RSV_CTRL, 0x0e);
-}
-
-int rtl8723au_hal_deinit(struct rtw_adapter *padapter)
-{
-	DBG_8723A("==> %s\n", __func__);
-
-#ifdef CONFIG_8723AU_BT_COEXIST
-	BT_HaltProcess(padapter);
-#endif
-	/*  2011/02/18 To Fix RU LNA  power leakage problem. We need to
-	    execute below below in Adapter init and halt sequence.
-	    According to EEchou's opinion, we can enable the ability for all */
-	/*  IC. Accord to johnny's opinion, only RU need the support. */
-	CardDisableRTL8723U(padapter);
-
-	padapter->hw_init_completed = false;
-
-	return _SUCCESS;
-}
-
-int rtl8723au_inirp_init(struct rtw_adapter *Adapter)
-{
-	u8 i;
-	struct recv_buf *precvbuf;
-	int status;
-	struct recv_priv *precvpriv = &Adapter->recvpriv;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-
-	status = _SUCCESS;
-
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_, "===> usb_inirp_init\n");
-
-	/* issue Rx irp to receive data */
-	precvbuf = (struct recv_buf *)precvpriv->precv_buf;
-	for (i = 0; i < NR_RECVBUFF; i++) {
-		if (rtl8723au_read_port(Adapter, 0, precvbuf) == _FAIL) {
-			RT_TRACE(_module_hci_hal_init_c_, _drv_err_,
-				 "usb_rx_init: usb_read_port error\n");
-			status = _FAIL;
-			goto exit;
-		}
-		precvbuf++;
-	}
-	if (rtl8723au_read_interrupt(Adapter) == _FAIL) {
-		RT_TRACE(_module_hci_hal_init_c_, _drv_err_,
-			 "%s: usb_read_interrupt error\n", __func__);
-		status = _FAIL;
-	}
-	pHalData->IntrMask[0] = rtl8723au_read32(Adapter, REG_USB_HIMR);
-	MSG_8723A("pHalData->IntrMask = 0x%04x\n", pHalData->IntrMask[0]);
-	pHalData->IntrMask[0] |= UHIMR_C2HCMD|UHIMR_CPWM;
-	rtl8723au_write32(Adapter, REG_USB_HIMR, pHalData->IntrMask[0]);
-exit:
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "<=== usb_inirp_init\n");
-	return status;
-}
-
-int rtl8723au_inirp_deinit(struct rtw_adapter *Adapter)
-{
-	struct hal_data_8723a	*pHalData = GET_HAL_DATA(Adapter);
-
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "===> usb_rx_deinit\n");
-	rtl8723au_read_port_cancel(Adapter);
-	pHalData->IntrMask[0] = rtl8723au_read32(Adapter, REG_USB_HIMR);
-	MSG_8723A("%s pHalData->IntrMask = 0x%04x\n", __func__,
-		  pHalData->IntrMask[0]);
-	pHalData->IntrMask[0] = 0x0;
-	rtl8723au_write32(Adapter, REG_USB_HIMR, pHalData->IntrMask[0]);
-	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
-		 "<=== usb_rx_deinit\n");
-	return _SUCCESS;
-}
-
-static void _ReadBoardType(struct rtw_adapter *Adapter, u8 *PROMContent,
-			   bool AutoloadFail)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	u8 boardType = BOARD_USB_DONGLE;
-
-	if (AutoloadFail) {
-		if (IS_8723_SERIES(pHalData->VersionID))
-			pHalData->rf_type = RF_1T1R;
-		else
-			pHalData->rf_type = RF_2T2R;
-		pHalData->BoardType = boardType;
-		return;
-	}
-
-	boardType = PROMContent[EEPROM_NORMAL_BoardType];
-	boardType &= BOARD_TYPE_NORMAL_MASK;/* bit[7:5] */
-	boardType >>= 5;
-
-	pHalData->BoardType = boardType;
-	MSG_8723A("_ReadBoardType(%x)\n", pHalData->BoardType);
-
-	if (boardType == BOARD_USB_High_PA)
-		pHalData->ExternalPA = 1;
-}
-
-static void Hal_EfuseParseMACAddr_8723AU(struct rtw_adapter *padapter,
-					 u8 *hwinfo, bool AutoLoadFail)
-{
-	u16 i;
-	u8 sMacAddr[ETH_ALEN] = {0x00, 0xE0, 0x4C, 0x87, 0x23, 0x00};
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
-
-	if (AutoLoadFail) {
-		for (i = 0; i < 6; i++)
-			pEEPROM->mac_addr[i] = sMacAddr[i];
-	} else {
-		/* Read Permanent MAC address */
-		memcpy(pEEPROM->mac_addr, &hwinfo[EEPROM_MAC_ADDR_8723AU],
-		       ETH_ALEN);
-	}
-
-	RT_TRACE(_module_hci_hal_init_c_, _drv_notice_,
-		 "Hal_EfuseParseMACAddr_8723AU: Permanent Address =%pM\n",
-		 pEEPROM->mac_addr);
-}
-
-static void readAdapterInfo(struct rtw_adapter *padapter)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
-	/* struct hal_data_8723a * pHalData = GET_HAL_DATA(padapter); */
-	u8 hwinfo[HWSET_MAX_SIZE];
-
-	Hal_InitPGData(padapter, hwinfo);
-	Hal_EfuseParseIDCode(padapter, hwinfo);
-	Hal_EfuseParseEEPROMVer(padapter, hwinfo,
-				pEEPROM->bautoload_fail_flag);
-	Hal_EfuseParseMACAddr_8723AU(padapter, hwinfo,
-				     pEEPROM->bautoload_fail_flag);
-	Hal_EfuseParsetxpowerinfo_8723A(padapter, hwinfo,
-					pEEPROM->bautoload_fail_flag);
-	_ReadBoardType(padapter, hwinfo, pEEPROM->bautoload_fail_flag);
-	Hal_EfuseParseBTCoexistInfo_8723A(padapter, hwinfo,
-					  pEEPROM->bautoload_fail_flag);
-
-	rtl8723a_EfuseParseChnlPlan(padapter, hwinfo,
-				    pEEPROM->bautoload_fail_flag);
-	Hal_EfuseParseThermalMeter_8723A(padapter, hwinfo,
-					 pEEPROM->bautoload_fail_flag);
-/*	_ReadRFSetting(Adapter, PROMContent, pEEPROM->bautoload_fail_flag); */
-/*	_ReadPSSetting(Adapter, PROMContent, pEEPROM->bautoload_fail_flag); */
-	Hal_EfuseParseAntennaDiversity(padapter, hwinfo,
-				       pEEPROM->bautoload_fail_flag);
-
-	Hal_EfuseParseEEPROMVer(padapter, hwinfo, pEEPROM->bautoload_fail_flag);
-	Hal_EfuseParseCustomerID(padapter, hwinfo,
-				 pEEPROM->bautoload_fail_flag);
-	Hal_EfuseParseRateIndicationOption(padapter, hwinfo,
-					   pEEPROM->bautoload_fail_flag);
-	Hal_EfuseParseXtal_8723A(padapter, hwinfo,
-				 pEEPROM->bautoload_fail_flag);
-
-	/* hal_CustomizedBehavior_8723U(Adapter); */
-
-/*	Adapter->bDongle = (PROMContent[EEPROM_EASY_REPLACEMENT] == 1)? 0: 1; */
-	DBG_8723A("%s(): REPLACEMENT = %x\n", __func__, padapter->bDongle);
-}
-
-static void _ReadPROMContent(struct rtw_adapter *Adapter)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(Adapter);
-	u8 eeValue;
-
-	eeValue = rtl8723au_read8(Adapter, REG_9346CR);
-	/*  To check system boot selection. */
-	pEEPROM->EepromOrEfuse = (eeValue & BOOT_FROM_EEPROM) ? true : false;
-	pEEPROM->bautoload_fail_flag = (eeValue & EEPROM_EN) ? false : true;
-
-	DBG_8723A("Boot from %s, Autoload %s !\n",
-		  (pEEPROM->EepromOrEfuse ? "EEPROM" : "EFUSE"),
-		  (pEEPROM->bautoload_fail_flag ? "Fail" : "OK"));
-
-	readAdapterInfo(Adapter);
-}
-
-/*  */
-/*	Description: */
-/*		We should set Efuse cell selection to WiFi cell in default. */
-/*  */
-/*	Assumption: */
-/*		PASSIVE_LEVEL */
-/*  */
-/*	Added by Roger, 2010.11.23. */
-/*  */
-static void hal_EfuseCellSel(struct rtw_adapter *Adapter)
-{
-	u32 value32;
-
-	value32 = rtl8723au_read32(Adapter, EFUSE_TEST);
-	value32 = (value32 & ~EFUSE_SEL_MASK) | EFUSE_SEL(EFUSE_WIFI_SEL_0);
-	rtl8723au_write32(Adapter, EFUSE_TEST, value32);
-}
-
-void rtl8723a_read_adapter_info(struct rtw_adapter *Adapter)
-{
-	unsigned long start = jiffies;
-
-	/*  Read EEPROM size before call any EEPROM function */
-	Adapter->EepromAddressSize = GetEEPROMSize8723A(Adapter);
-
-	MSG_8723A("====> _ReadAdapterInfo8723AU\n");
-
-	hal_EfuseCellSel(Adapter);
-
-	_ReadPROMContent(Adapter);
-
-	MSG_8723A("<==== _ReadAdapterInfo8723AU in %d ms\n",
-		  jiffies_to_msecs(jiffies - start));
-}
-
-/*  */
-/*	Description: */
-/*		Query setting of specified variable. */
-/*  */
-int GetHalDefVar8192CUsb(struct rtw_adapter *Adapter,
-			 enum hal_def_variable eVariable, void *pValue)
-{
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(Adapter);
-	int bResult = _SUCCESS;
-
-	switch (eVariable) {
-	case HAL_DEF_UNDERCORATEDSMOOTHEDPWDB:
-		*((int *)pValue) = pHalData->dmpriv.UndecoratedSmoothedPWDB;
-		break;
-	case HAL_DEF_IS_SUPPORT_ANT_DIV:
-		break;
-	case HAL_DEF_CURRENT_ANTENNA:
-		break;
-	case HAL_DEF_DRVINFO_SZ:
-		*((u32 *)pValue) = DRVINFO_SZ;
-		break;
-	case HAL_DEF_MAX_RECVBUF_SZ:
-		*((u32 *)pValue) = MAX_RECVBUF_SZ;
-		break;
-	case HAL_DEF_RX_PACKET_OFFSET:
-		*((u32 *)pValue) = RXDESC_SIZE + DRVINFO_SZ;
-		break;
-	case HAL_DEF_DBG_DUMP_RXPKT:
-		*((u8 *)pValue) = pHalData->bDumpRxPkt;
-		break;
-	case HAL_DEF_DBG_DM_FUNC:
-		*((u32 *)pValue) = pHalData->odmpriv.SupportAbility;
-		break;
-	case HW_VAR_MAX_RX_AMPDU_FACTOR:
-		*((u32 *)pValue) = IEEE80211_HT_MAX_AMPDU_64K;
-		break;
-	case HW_DEF_ODM_DBG_FLAG:
-	{
-		struct dm_odm_t	*pDM_Odm = &pHalData->odmpriv;
-		printk("pDM_Odm->DebugComponents = 0x%llx\n",
-		       pDM_Odm->DebugComponents);
-	}
-		break;
-	default:
-		bResult = _FAIL;
-		break;
-	}
-
-	return bResult;
-}
-
-void rtl8723a_update_ramask(struct rtw_adapter *padapter,
-			    u32 mac_id, u8 rssi_level)
-{
-	struct sta_info	*psta;
-	struct FW_Sta_Info *fw_sta;
-	struct hal_data_8723a *pHalData = GET_HAL_DATA(padapter);
-	struct dm_priv *pdmpriv = &pHalData->dmpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
-	u8 init_rate, networkType, raid, arg;
-	u32 mask, rate_bitmap;
-	u8 shortGIrate = false;
-	int supportRateNum;
-
-	if (mac_id >= NUM_STA) /* CAM_SIZE */
-		return;
-
-	psta = pmlmeinfo->FW_sta_info[mac_id].psta;
-	if (psta == NULL)
-		return;
-
-	switch (mac_id) {
-	case 0:/*  for infra mode */
-		supportRateNum =
-			rtw_get_rateset_len23a(cur_network->SupportedRates);
-		networkType = judge_network_type23a(padapter,
-						 cur_network->SupportedRates,
-						 supportRateNum) & 0xf;
-		/* pmlmeext->cur_wireless_mode = networkType; */
-		raid = networktype_to_raid23a(networkType);
-
-		mask = update_supported_rate23a(cur_network->SupportedRates,
-					     supportRateNum);
-		mask |= (pmlmeinfo->HT_enable) ?
-			update_MSC_rate23a(&pmlmeinfo->ht_cap) : 0;
-
-		if (support_short_GI23a(padapter, &pmlmeinfo->ht_cap))
-			shortGIrate = true;
-		break;
-
-	case 1:/* for broadcast/multicast */
-		fw_sta = &pmlmeinfo->FW_sta_info[mac_id]; 
-		supportRateNum = rtw_get_rateset_len23a(fw_sta->SupportedRates);
-		if (pmlmeext->cur_wireless_mode & WIRELESS_11B)
-			networkType = WIRELESS_11B;
-		else
-			networkType = WIRELESS_11G;
-		raid = networktype_to_raid23a(networkType);
-
-		mask = update_basic_rate23a(cur_network->SupportedRates,
-					 supportRateNum);
-		break;
-
-	default: /* for each sta in IBSS */
-		fw_sta = &pmlmeinfo->FW_sta_info[mac_id]; 
-		supportRateNum = rtw_get_rateset_len23a(fw_sta->SupportedRates);
-		networkType = judge_network_type23a(padapter,
-						    fw_sta->SupportedRates,
-						    supportRateNum) & 0xf;
-		/* pmlmeext->cur_wireless_mode = networkType; */
-		raid = networktype_to_raid23a(networkType);
-
-		mask = update_supported_rate23a(cur_network->SupportedRates,
-						supportRateNum);
-
-		/* todo: support HT in IBSS */
-		break;
-	}
-
-	/* mask &= 0x0fffffff; */
-	rate_bitmap = ODM_Get_Rate_Bitmap23a(pHalData, mac_id, mask,
-					     rssi_level);
-	DBG_8723A("%s => mac_id:%d, networkType:0x%02x, "
-		  "mask:0x%08x\n\t ==> rssi_level:%d, rate_bitmap:0x%08x\n",
-		  __func__, mac_id, networkType, mask, rssi_level, rate_bitmap);
-
-	mask &= rate_bitmap;
-	mask |= ((raid << 28) & 0xf0000000);
-
-	init_rate = get_highest_rate_idx23a(mask) & 0x3f;
-
-	arg = mac_id & 0x1f;/* MACID */
-	arg |= BIT(7);
-
-	if (shortGIrate == true)
-		arg |= BIT(5);
-
-	DBG_8723A("update raid entry, mask = 0x%x, arg = 0x%x\n", mask, arg);
-
-	rtl8723a_set_raid_cmd(padapter, mask, arg);
-
-	/* set ra_id */
-	psta->raid = raid;
-	psta->init_rate = init_rate;
-
-	/* set correct initial date rate for each mac_id */
-	pdmpriv->INIDATA_RATE[mac_id] = init_rate;
-}
diff --git a/drivers/staging/rtl8723au/hal/usb_ops_linux.c b/drivers/staging/rtl8723au/hal/usb_ops_linux.c
deleted file mode 100644
index 5c81ff48252e..000000000000
--- a/drivers/staging/rtl8723au/hal/usb_ops_linux.c
+++ /dev/null
@@ -1,690 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _HCI_OPS_OS_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <osdep_intf.h>
-#include <usb_ops.h>
-#include <recv_osdep.h>
-#include <rtl8723a_hal.h>
-#include <rtl8723a_recv.h>
-
-u8 rtl8723au_read8(struct rtw_adapter *padapter, u16 addr)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
-	struct usb_device *udev = pdvobjpriv->pusbdev;
-	int len;
-	u8 data;
-
-	mutex_lock(&pdvobjpriv->usb_vendor_req_mutex);
-	len = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
-			      REALTEK_USB_VENQT_CMD_REQ, REALTEK_USB_VENQT_READ,
-			      addr, 0, &pdvobjpriv->usb_buf.val8, sizeof(data),
-			      RTW_USB_CONTROL_MSG_TIMEOUT);
-
-	data = pdvobjpriv->usb_buf.val8;
-	mutex_unlock(&pdvobjpriv->usb_vendor_req_mutex);
-
-	return data;
-}
-
-u16 rtl8723au_read16(struct rtw_adapter *padapter, u16 addr)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
-	struct usb_device *udev = pdvobjpriv->pusbdev;
-	int len;
-	u16 data;
-
-	mutex_lock(&pdvobjpriv->usb_vendor_req_mutex);
-	len = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
-			      REALTEK_USB_VENQT_CMD_REQ, REALTEK_USB_VENQT_READ,
-			      addr, 0, &pdvobjpriv->usb_buf.val16, sizeof(data),
-			      RTW_USB_CONTROL_MSG_TIMEOUT);
-
-	data = le16_to_cpu(pdvobjpriv->usb_buf.val16);
-	mutex_unlock(&pdvobjpriv->usb_vendor_req_mutex);
-
-	return data;
-}
-
-u32 rtl8723au_read32(struct rtw_adapter *padapter, u16 addr)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
-	struct usb_device *udev = pdvobjpriv->pusbdev;
-	int len;
-	u32 data;
-
-	mutex_lock(&pdvobjpriv->usb_vendor_req_mutex);
-	len = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
-			      REALTEK_USB_VENQT_CMD_REQ, REALTEK_USB_VENQT_READ,
-			      addr, 0, &pdvobjpriv->usb_buf.val32, sizeof(data),
-			      RTW_USB_CONTROL_MSG_TIMEOUT);
-
-	data = le32_to_cpu(pdvobjpriv->usb_buf.val32);
-	mutex_unlock(&pdvobjpriv->usb_vendor_req_mutex);
-
-	return data;
-}
-
-int rtl8723au_write8(struct rtw_adapter *padapter, u16 addr, u8 val)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
-	struct usb_device *udev = pdvobjpriv->pusbdev;
-	int ret;
-
-	mutex_lock(&pdvobjpriv->usb_vendor_req_mutex);
-	pdvobjpriv->usb_buf.val8 = val;
-
-	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
-			      REALTEK_USB_VENQT_CMD_REQ,
-			      REALTEK_USB_VENQT_WRITE,
-			      addr, 0, &pdvobjpriv->usb_buf.val8, sizeof(val),
-			      RTW_USB_CONTROL_MSG_TIMEOUT);
-
-	if (ret != sizeof(val))
-		ret = _FAIL;
-	else
-		ret = _SUCCESS;
-
-	mutex_unlock(&pdvobjpriv->usb_vendor_req_mutex);
-	return ret;
-}
-
-int rtl8723au_write16(struct rtw_adapter *padapter, u16 addr, u16 val)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
-	struct usb_device *udev = pdvobjpriv->pusbdev;
-	int ret;
-
-	mutex_lock(&pdvobjpriv->usb_vendor_req_mutex);
-	pdvobjpriv->usb_buf.val16 = cpu_to_le16(val);
-
-	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
-			      REALTEK_USB_VENQT_CMD_REQ,
-			      REALTEK_USB_VENQT_WRITE,
-			      addr, 0, &pdvobjpriv->usb_buf.val16, sizeof(val),
-			      RTW_USB_CONTROL_MSG_TIMEOUT);
-
-	if (ret != sizeof(val))
-		ret = _FAIL;
-	else
-		ret = _SUCCESS;
-
-	mutex_unlock(&pdvobjpriv->usb_vendor_req_mutex);
-	return ret;
-}
-
-int rtl8723au_write32(struct rtw_adapter *padapter, u16 addr, u32 val)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
-	struct usb_device *udev = pdvobjpriv->pusbdev;
-	int ret;
-
-	mutex_lock(&pdvobjpriv->usb_vendor_req_mutex);
-	pdvobjpriv->usb_buf.val32 = cpu_to_le32(val);
-
-	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
-			      REALTEK_USB_VENQT_CMD_REQ,
-			      REALTEK_USB_VENQT_WRITE,
-			      addr, 0, &pdvobjpriv->usb_buf.val32, sizeof(val),
-			      RTW_USB_CONTROL_MSG_TIMEOUT);
-
-	if (ret != sizeof(val))
-		ret = _FAIL;
-	else
-		ret = _SUCCESS;
-
-	mutex_unlock(&pdvobjpriv->usb_vendor_req_mutex);
-	return ret;
-}
-
-int rtl8723au_writeN(struct rtw_adapter *padapter, u16 addr, u16 len, u8 *buf)
-{
-	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
-	struct usb_device *udev = pdvobjpriv->pusbdev;
-	int ret;
-
-	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
-			      REALTEK_USB_VENQT_CMD_REQ,
-			      REALTEK_USB_VENQT_WRITE,
-			      addr, 0, buf, len, RTW_USB_CONTROL_MSG_TIMEOUT);
-
-	if (ret != len)
-		return _FAIL;
-	return _SUCCESS;
-}
-
-/*
- * Description:
- *	Recognize the interrupt content by reading the interrupt
- *	register or content and masking interrupt mask (IMR)
- *	if it is our NIC's interrupt. After recognizing, we may clear
- *	the all interrupts (ISR).
- * Arguments:
- *	[in] Adapter -
- *		The adapter context.
- *	[in] pContent -
- *		Under PCI interface, this field is ignord.
- *		Under USB interface, the content is the interrupt
- *		content pointer.
- *		Under SDIO interface, this is the interrupt type which
- *		is Local interrupt or system interrupt.
- *	[in] ContentLen -
- *		The length in byte of pContent.
- * Return:
- *	If any interrupt matches the mask (IMR), return true, and
- *	return false otherwise.
- */
-static bool
-InterruptRecognized8723AU(struct rtw_adapter *Adapter, void *pContent,
-			  u32 ContentLen)
-{
-	struct hal_data_8723a	*pHalData = GET_HAL_DATA(Adapter);
-	u8 *buffer = (u8 *)pContent;
-	struct reportpwrstate_parm report;
-
-	memcpy(&pHalData->IntArray[0], &buffer[USB_INTR_CONTENT_HISR_OFFSET],
-	       4);
-	pHalData->IntArray[0] &= pHalData->IntrMask[0];
-
-	/* For HISR extension. Added by tynli. 2009.10.07. */
-	memcpy(&pHalData->IntArray[1],
-	       &buffer[USB_INTR_CONTENT_HISRE_OFFSET], 4);
-	pHalData->IntArray[1] &= pHalData->IntrMask[1];
-
-	/* We sholud remove this function later because DDK suggest
-	 * not to executing too many operations in MPISR  */
-
-	memcpy(&report.state, &buffer[USB_INTR_CPWM_OFFSET], 1);
-
-	return (pHalData->IntArray[0] & pHalData->IntrMask[0]) != 0 ||
-		(pHalData->IntArray[1] & pHalData->IntrMask[1]) != 0;
-}
-
-static void usb_read_interrupt_complete(struct urb *purb)
-{
-	int err;
-	struct rtw_adapter *padapter = (struct rtw_adapter *)purb->context;
-
-	if (padapter->bSurpriseRemoved || padapter->bDriverStopped ||
-	    padapter->bReadPortCancel) {
-		DBG_8723A("%s() RX Warning! bDriverStopped(%d) OR "
-			  "bSurpriseRemoved(%d) bReadPortCancel(%d)\n",
-			  __func__, padapter->bDriverStopped,
-			  padapter->bSurpriseRemoved,
-			  padapter->bReadPortCancel);
-		return;
-	}
-
-	if (purb->status == 0) {
-		struct c2h_evt_hdr *c2h_evt;
-
-		c2h_evt = (struct c2h_evt_hdr *)purb->transfer_buffer;
-
-		if (purb->actual_length > USB_INTR_CONTENT_LENGTH) {
-			DBG_8723A("usb_read_interrupt_complete: purb->actual_"
-				  "length > USB_INTR_CONTENT_LENGTH\n");
-			goto urb_submit;
-		}
-
-		InterruptRecognized8723AU(padapter, purb->transfer_buffer,
-					  purb->actual_length);
-
-		if (c2h_evt_exist(c2h_evt)) {
-			if (c2h_id_filter_ccx_8723a(c2h_evt->id)) {
-				/* Handle CCX report here */
-				handle_txrpt_ccx_8723a(padapter, (void *)
-						       c2h_evt->payload);
-				schedule_work(&padapter->evtpriv.irq_wk);
-			} else {
-				struct evt_work *c2w;
-				int res;
-
-				c2w = kmalloc(sizeof(struct evt_work),
-						GFP_ATOMIC);
-
-				if (!c2w)
-					goto urb_submit;
-
-				c2w->adapter = padapter;
-				INIT_WORK(&c2w->work, rtw_evt_work);
-				memcpy(c2w->u.buf, purb->transfer_buffer, 16);
-
-				res = queue_work(padapter->evtpriv.wq,
-						 &c2w->work);
-
-				if (!res) {
-					printk(KERN_ERR "%s: Call to "
-					       "queue_work() failed\n",
-					       __func__);
-					kfree(c2w);
-					goto urb_submit;
-				}
-			}
-		}
-
-urb_submit:
-		err = usb_submit_urb(purb, GFP_ATOMIC);
-		if (err && (err != -EPERM)) {
-			DBG_8723A("cannot submit interrupt in-token(err = "
-				  "0x%08x), urb_status = %d\n",
-				  err, purb->status);
-		}
-	} else {
-		DBG_8723A("###=> usb_read_interrupt_complete => urb "
-			  "status(%d)\n", purb->status);
-
-		switch (purb->status) {
-		case -EINVAL:
-		case -EPIPE:
-		case -ENODEV:
-		case -ESHUTDOWN:
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "usb_read_port_complete:bSurpriseRemoved =true\n");
-			/* Fall Through here */
-		case -ENOENT:
-			padapter->bDriverStopped = true;
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "usb_read_port_complete:bDriverStopped =true\n");
-			break;
-		case -EPROTO:
-			break;
-		case -EINPROGRESS:
-			DBG_8723A("ERROR: URB IS IN PROGRESS!\n");
-			break;
-		default:
-			break;
-		}
-	}
-}
-
-int rtl8723au_read_interrupt(struct rtw_adapter *adapter)
-{
-	int err;
-	unsigned int pipe;
-	int ret = _SUCCESS;
-	struct dvobj_priv *pdvobj = adapter_to_dvobj(adapter);
-	struct recv_priv *precvpriv = &adapter->recvpriv;
-	struct usb_device *pusbd = pdvobj->pusbdev;
-
-	/* translate DMA FIFO addr to pipehandle */
-	pipe = usb_rcvintpipe(pusbd, pdvobj->RtInPipe[1]);
-
-	usb_fill_int_urb(precvpriv->int_in_urb, pusbd, pipe,
-			 precvpriv->int_in_buf, USB_INTR_CONTENT_LENGTH,
-			 usb_read_interrupt_complete, adapter, 1);
-
-	err = usb_submit_urb(precvpriv->int_in_urb, GFP_ATOMIC);
-	if (err && (err != -EPERM)) {
-		DBG_8723A("cannot submit interrupt in-token(err = 0x%08x),"
-			  "urb_status = %d\n", err,
-			  precvpriv->int_in_urb->status);
-		ret = _FAIL;
-	}
-
-	return ret;
-}
-
-static int recvbuf2recvframe(struct rtw_adapter *padapter, struct sk_buff *pskb)
-{
-	u8	*pbuf;
-	u8	shift_sz = 0;
-	u16	pkt_cnt;
-	u32	pkt_offset, skb_len, alloc_sz;
-	int	transfer_len;
-	struct recv_stat *prxstat;
-	struct phy_stat	*pphy_info;
-	struct sk_buff *pkt_copy;
-	struct recv_frame *precvframe;
-	struct rx_pkt_attrib *pattrib;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-	struct rtw_queue *pfree_recv_queue = &precvpriv->free_recv_queue;
-
-	transfer_len = (int)pskb->len;
-	pbuf = pskb->data;
-
-	prxstat = (struct recv_stat *)pbuf;
-	pkt_cnt = (le32_to_cpu(prxstat->rxdw2) >> 16) & 0xff;
-
-	do {
-		RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-			 "recvbuf2recvframe: rxdesc = offsset 0:0x%08x, 4:0x%08x, 8:0x%08x, C:0x%08x\n",
-			 prxstat->rxdw0, prxstat->rxdw1,
-			 prxstat->rxdw2, prxstat->rxdw4);
-
-		prxstat = (struct recv_stat *)pbuf;
-
-		precvframe = rtw_alloc_recvframe23a(pfree_recv_queue);
-		if (!precvframe) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "recvbuf2recvframe: precvframe == NULL\n");
-			DBG_8723A("%s()-%d: rtw_alloc_recvframe23a() failed! RX "
-				  "Drop!\n", __func__, __LINE__);
-			goto _exit_recvbuf2recvframe;
-		}
-
-		INIT_LIST_HEAD(&precvframe->list);
-
-		update_recvframe_attrib(precvframe, prxstat);
-
-		pattrib = &precvframe->attrib;
-
-		if (pattrib->crc_err) {
-			DBG_8723A("%s()-%d: RX Warning! rx CRC ERROR !!\n",
-				  __func__, __LINE__);
-			rtw_free_recvframe23a(precvframe);
-			goto _exit_recvbuf2recvframe;
-		}
-
-		pkt_offset = RXDESC_SIZE + pattrib->drvinfo_sz +
-			pattrib->shift_sz + pattrib->pkt_len;
-
-		if (pattrib->pkt_len <= 0 || pkt_offset > transfer_len) {
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_info_,
-				 "recvbuf2recvframe: pkt_len<= 0\n");
-			DBG_8723A("%s()-%d: RX Warning!\n",
-				  __func__, __LINE__);
-			rtw_free_recvframe23a(precvframe);
-			goto _exit_recvbuf2recvframe;
-		}
-
-		/*	Modified by Albert 20101213 */
-		/*	For 8 bytes IP header alignment. */
-		/*	Qos data, wireless lan header length is 26 */
-		if (pattrib->qos)
-			shift_sz = 6;
-		else
-			shift_sz = 0;
-
-		skb_len = pattrib->pkt_len;
-
-		/* for first fragment packet, driver need allocate
-		 * 1536+drvinfo_sz+RXDESC_SIZE to defrag packet.
-		 * modify alloc_sz for recvive crc error packet
-		 * by thomas 2011-06-02 */
-		if (pattrib->mfrag == 1 && pattrib->frag_num == 0) {
-			/* alloc_sz = 1664;	1664 is 128 alignment. */
-			if (skb_len <= 1650)
-				alloc_sz = 1664;
-			else
-				alloc_sz = skb_len + 14;
-		} else {
-			alloc_sz = skb_len;
-		/*  6 is for IP header 8 bytes alignment in QoS packet case. */
-		/*  8 is for skb->data 4 bytes alignment. */
-			alloc_sz += 14;
-		}
-
-		pkt_copy = netdev_alloc_skb(padapter->pnetdev, alloc_sz);
-		if (pkt_copy) {
-			pkt_copy->dev = padapter->pnetdev;
-			precvframe->pkt = pkt_copy;
-			/* force pkt_copy->data at 8-byte alignment address */
-			skb_reserve(pkt_copy, 8 -
-				    ((unsigned long)(pkt_copy->data) & 7));
-			/*force ip_hdr at 8-byte alignment address
-			  according to shift_sz. */
-			skb_reserve(pkt_copy, shift_sz);
-			memcpy(pkt_copy->data, pbuf + pattrib->shift_sz +
-			       pattrib->drvinfo_sz + RXDESC_SIZE, skb_len);
-			skb_put(pkt_copy, skb_len);
-		} else {
-			if (pattrib->mfrag == 1 && pattrib->frag_num == 0) {
-				DBG_8723A("recvbuf2recvframe: alloc_skb fail, "
-					  "drop frag frame \n");
-				rtw_free_recvframe23a(precvframe);
-				goto _exit_recvbuf2recvframe;
-			}
-
-			precvframe->pkt = skb_clone(pskb, GFP_ATOMIC);
-			if (!precvframe->pkt) {
-				DBG_8723A("recvbuf2recvframe: skb_clone "
-					  "fail\n");
-				rtw_free_recvframe23a(precvframe);
-				goto _exit_recvbuf2recvframe;
-			}
-		}
-
-		if (pattrib->physt) {
-			pphy_info = (struct phy_stat *)(pbuf + RXDESC_OFFSET);
-			update_recvframe_phyinfo(precvframe, pphy_info);
-		}
-
-		if (rtw_recv_entry23a(precvframe) != _SUCCESS)
-			RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
-				 "recvbuf2recvframe: rtw_recv_entry23a(precvframe) != _SUCCESS\n");
-
-		pkt_cnt--;
-		transfer_len -= pkt_offset;
-		pbuf += pkt_offset;
-		precvframe = NULL;
-		pkt_copy = NULL;
-
-		if (transfer_len > 0 && pkt_cnt == 0)
-			pkt_cnt = (le32_to_cpu(prxstat->rxdw2)>>16) & 0xff;
-
-	} while (transfer_len > 0 && pkt_cnt > 0);
-
-_exit_recvbuf2recvframe:
-
-	return _SUCCESS;
-}
-
-void rtl8723au_recv_tasklet(void *priv)
-{
-	struct sk_buff *pskb;
-	struct rtw_adapter *padapter = (struct rtw_adapter *)priv;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-
-	while (NULL != (pskb = skb_dequeue(&precvpriv->rx_skb_queue))) {
-		if (padapter->bDriverStopped || padapter->bSurpriseRemoved) {
-			DBG_8723A("recv_tasklet => bDriverStopped or "
-				  "bSurpriseRemoved \n");
-			dev_kfree_skb_any(pskb);
-			break;
-		}
-
-		recvbuf2recvframe(padapter, pskb);
-		skb_reset_tail_pointer(pskb);
-
-		pskb->len = 0;
-
-		skb_queue_tail(&precvpriv->free_recv_skb_queue, pskb);
-	}
-}
-
-static void usb_read_port_complete(struct urb *purb)
-{
-	struct recv_buf *precvbuf = (struct recv_buf *)purb->context;
-	struct rtw_adapter *padapter = (struct rtw_adapter *)precvbuf->adapter;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-
-	RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-		 "usb_read_port_complete!!!\n");
-
-	precvpriv->rx_pending_cnt--;
-
-	if (padapter->bSurpriseRemoved || padapter->bDriverStopped ||
-	    padapter->bReadPortCancel) {
-		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-			 "usb_read_port_complete:bDriverStopped(%d) OR bSurpriseRemoved(%d)\n",
-			 padapter->bDriverStopped, padapter->bSurpriseRemoved);
-
-		DBG_8723A("%s()-%d: RX Warning! bDriverStopped(%d) OR "
-			  "bSurpriseRemoved(%d) bReadPortCancel(%d)\n",
-			  __func__, __LINE__, padapter->bDriverStopped,
-			  padapter->bSurpriseRemoved, padapter->bReadPortCancel);
-		return;
-	}
-
-	if (purb->status == 0) {
-		if (purb->actual_length > MAX_RECVBUF_SZ ||
-		    purb->actual_length < RXDESC_SIZE) {
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "usb_read_port_complete: (purb->actual_length > MAX_RECVBUF_SZ) || (purb->actual_length < RXDESC_SIZE)\n");
-			rtl8723au_read_port(padapter, 0, precvbuf);
-			DBG_8723A("%s()-%d: RX Warning!\n",
-				  __func__, __LINE__);
-		} else {
-			rtw_reset_continual_urb_error(
-				adapter_to_dvobj(padapter));
-
-			skb_put(precvbuf->pskb, purb->actual_length);
-			skb_queue_tail(&precvpriv->rx_skb_queue,
-				       precvbuf->pskb);
-
-			if (skb_queue_len(&precvpriv->rx_skb_queue) <= 1)
-				tasklet_schedule(&precvpriv->recv_tasklet);
-
-			precvbuf->pskb = NULL;
-			rtl8723au_read_port(padapter, 0, precvbuf);
-		}
-	} else {
-		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-			 "usb_read_port_complete : purb->status(%d) != 0\n",
-			 purb->status);
-		skb_put(precvbuf->pskb, purb->actual_length);
-		precvbuf->pskb = NULL;
-
-		DBG_8723A("###=> usb_read_port_complete => urb status(%d)\n",
-			  purb->status);
-
-		if (rtw_inc_and_chk_continual_urb_error(
-			    adapter_to_dvobj(padapter))) {
-			padapter->bSurpriseRemoved = true;
-		}
-
-		switch (purb->status) {
-		case -EINVAL:
-		case -EPIPE:
-		case -ENODEV:
-		case -ESHUTDOWN:
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "usb_read_port_complete:bSurpriseRemoved = true\n");
-			/* Intentional fall through here */
-		case -ENOENT:
-			padapter->bDriverStopped = true;
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "usb_read_port_complete:bDriverStopped = true\n");
-			break;
-		case -EPROTO:
-		case -EOVERFLOW:
-			rtl8723au_read_port(padapter, 0, precvbuf);
-			break;
-		case -EINPROGRESS:
-			DBG_8723A("ERROR: URB IS IN PROGRESS!\n");
-			break;
-		default:
-			break;
-		}
-	}
-}
-
-int rtl8723au_read_port(struct rtw_adapter *adapter, u32 cnt,
-			struct recv_buf *precvbuf)
-{
-	struct urb *purb;
-	struct dvobj_priv *pdvobj = adapter_to_dvobj(adapter);
-	struct recv_priv *precvpriv = &adapter->recvpriv;
-	struct usb_device *pusbd = pdvobj->pusbdev;
-	int err;
-	unsigned int pipe;
-	unsigned long tmpaddr;
-	unsigned long alignment;
-	int ret = _SUCCESS;
-
-	if (adapter->bDriverStopped || adapter->bSurpriseRemoved) {
-		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-			 "usb_read_port:(padapter->bDriverStopped ||padapter->bSurpriseRemoved)!!!\n");
-		return _FAIL;
-	}
-
-	if (!precvbuf) {
-		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-			 "usb_read_port:precvbuf == NULL\n");
-		return _FAIL;
-	}
-
-	if (!precvbuf->pskb)
-		precvbuf->pskb = skb_dequeue(&precvpriv->free_recv_skb_queue);
-
-	/* re-assign for linux based on skb */
-	if (!precvbuf->pskb) {
-		precvbuf->pskb = netdev_alloc_skb(adapter->pnetdev, MAX_RECVBUF_SZ + RECVBUFF_ALIGN_SZ);
-		if (precvbuf->pskb == NULL) {
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "init_recvbuf(): alloc_skb fail!\n");
-			return _FAIL;
-		}
-
-		tmpaddr = (unsigned long)precvbuf->pskb->data;
-		alignment = tmpaddr & (RECVBUFF_ALIGN_SZ-1);
-		skb_reserve(precvbuf->pskb, (RECVBUFF_ALIGN_SZ - alignment));
-	}
-
-	precvpriv->rx_pending_cnt++;
-
-	purb = precvbuf->purb;
-
-	/* translate DMA FIFO addr to pipehandle */
-	pipe = usb_rcvbulkpipe(pusbd, pdvobj->RtInPipe[0]);
-
-	usb_fill_bulk_urb(purb, pusbd, pipe, precvbuf->pskb->data,
-			  MAX_RECVBUF_SZ, usb_read_port_complete,
-			  precvbuf);/* context is precvbuf */
-
-	err = usb_submit_urb(purb, GFP_ATOMIC);
-	if ((err) && (err != -EPERM)) {
-		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-			 "cannot submit rx in-token(err = 0x%.8x), URB_STATUS = 0x%.8x\n",
-			 err, purb->status);
-		DBG_8723A("cannot submit rx in-token(err = 0x%08x), urb_status "
-			  "= %d\n", err, purb->status);
-		ret = _FAIL;
-	}
-	return ret;
-}
-
-void rtl8723au_xmit_tasklet(void *priv)
-{
-	int ret;
-	struct rtw_adapter *padapter = (struct rtw_adapter *)priv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	if (check_fwstate(&padapter->mlmepriv, _FW_UNDER_SURVEY))
-		return;
-
-	while (1) {
-		if (padapter->bDriverStopped || padapter->bSurpriseRemoved ||
-		    padapter->bWritePortCancel) {
-			DBG_8723A("xmit_tasklet => bDriverStopped or "
-				  "bSurpriseRemoved or bWritePortCancel\n");
-			break;
-		}
-
-		ret = rtl8723au_xmitframe_complete(padapter, pxmitpriv, NULL);
-
-		if (!ret)
-			break;
-	}
-}
-
-void rtl8723au_set_hw_type(struct rtw_adapter *padapter)
-{
-	padapter->chip_type = RTL8723A;
-	padapter->HardwareType = HARDWARE_TYPE_RTL8723AU;
-	DBG_8723A("CHIP TYPE: RTL8723A\n");
-}
diff --git a/drivers/staging/rtl8723au/include/Hal8723APhyCfg.h b/drivers/staging/rtl8723au/include/Hal8723APhyCfg.h
deleted file mode 100644
index bcf36579f43a..000000000000
--- a/drivers/staging/rtl8723au/include/Hal8723APhyCfg.h
+++ /dev/null
@@ -1,162 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- *
- ******************************************************************************/
-#ifndef __INC_HAL8723PHYCFG_H__
-#define __INC_HAL8723PHYCFG_H__
-
-/*------------------------------Define structure----------------------------*/
-enum RF_RADIO_PATH {
-	RF_PATH_A = 0,			/* Radio Path A */
-	RF_PATH_B = 1,			/* Radio Path B */
-	RF_PATH_MAX			/* Max RF number 90 support */
-};
-
-#define CHANNEL_MAX_NUMBER		14	/*  14 is the max channel number */
-
-enum WIRELESS_MODE {
-	WIRELESS_MODE_UNKNOWN	= 0x00,
-	WIRELESS_MODE_A		= BIT(2),
-	WIRELESS_MODE_B		= BIT(0),
-	WIRELESS_MODE_G		= BIT(1),
-	WIRELESS_MODE_AUTO	= BIT(5),
-	WIRELESS_MODE_N_24G	= BIT(3),
-	WIRELESS_MODE_N_5G	= BIT(4),
-	WIRELESS_MODE_AC	= BIT(6)
-};
-
-struct bb_reg_define {
-	u32 rfintfs;		/*  set software control: */
-				/*		0x870~0x877[8 bytes] */
-	u32 rfintfi;		/*  readback data: */
-				/*		0x8e0~0x8e7[8 bytes] */
-	u32 rfintfo;		/*  output data: */
-				/*		0x860~0x86f [16 bytes] */
-	u32 rfintfe;		/*  output enable: */
-				/*		0x860~0x86f [16 bytes] */
-	u32 rf3wireOffset;	/*  LSSI data: */
-				/*		0x840~0x84f [16 bytes] */
-	u32 rfLSSI_Select;	/*  BB Band Select: */
-				/*		0x878~0x87f [8 bytes] */
-	u32 rfTxGainStage;	/*  Tx gain stage: */
-				/*		0x80c~0x80f [4 bytes] */
-	u32 rfHSSIPara1;	/*  wire parameter control1 : */
-				/*		0x820~0x823, 0x828~0x82b, 0x830~0x833, 0x838~0x83b [16 bytes] */
-	u32 rfHSSIPara2;	/*  wire parameter control2 : */
-				/*		0x824~0x827, 0x82c~0x82f, 0x834~0x837, 0x83c~0x83f [16 bytes] */
-	u32 rfSwitchControl; /* Tx Rx antenna control : */
-				/*		0x858~0x85f [16 bytes] */
-	u32 rfAGCControl1;	/* AGC parameter control1 : */
-				/*	0xc50~0xc53, 0xc58~0xc5b, 0xc60~0xc63, 0xc68~0xc6b [16 bytes] */
-	u32 rfAGCControl2;	/* AGC parameter control2 : */
-				/*		0xc54~0xc57, 0xc5c~0xc5f, 0xc64~0xc67, 0xc6c~0xc6f [16 bytes] */
-	u32 rfRxIQImbalance; /* OFDM Rx IQ imbalance matrix : */
-				/*		0xc14~0xc17, 0xc1c~0xc1f, 0xc24~0xc27, 0xc2c~0xc2f [16 bytes] */
-	u32 rfRxAFE;		/* Rx IQ DC ofset and Rx digital filter, Rx DC notch filter : */
-				/*	0xc10~0xc13, 0xc18~0xc1b, 0xc20~0xc23, 0xc28~0xc2b [16 bytes] */
-	u32 rfTxIQImbalance; /* OFDM Tx IQ imbalance matrix */
-				/*	0xc80~0xc83, 0xc88~0xc8b, 0xc90~0xc93, 0xc98~0xc9b [16 bytes] */
-	u32 rfTxAFE;		/* Tx IQ DC Offset and Tx DFIR type */
-				/*	0xc84~0xc87, 0xc8c~0xc8f, 0xc94~0xc97, 0xc9c~0xc9f [16 bytes] */
-	u32 rfLSSIReadBack;	/* LSSI RF readback data SI mode */
-				/*	0x8a0~0x8af [16 bytes] */
-	u32 rfLSSIReadBackPi;	/* LSSI RF readback data PI mode 0x8b8-8bc for Path A and B */
-};
-
-struct r_antenna_sel_ofdm {
-	u32			r_tx_antenna:4;
-	u32			r_ant_l:4;
-	u32			r_ant_non_ht:4;
-	u32			r_ant_ht1:4;
-	u32			r_ant_ht2:4;
-	u32			r_ant_ht_s1:4;
-	u32			r_ant_non_ht_s1:4;
-	u32			OFDM_TXSC:2;
-	u32			Reserved:2;
-};
-
-struct r_antenna_sel_cck {
-	u8			r_cckrx_enable_2:2;
-	u8			r_cckrx_enable:2;
-	u8			r_ccktx_enable:4;
-};
-
-/*------------------------------Define structure----------------------------*/
-
-
-/*------------------------Export global variable----------------------------*/
-/*------------------------Export global variable----------------------------*/
-
-
-/*------------------------Export Macro Definition---------------------------*/
-/*------------------------Export Macro Definition---------------------------*/
-
-
-/*--------------------------Exported Function prototype---------------------*/
-/*  */
-/*  BB and RF register read/write */
-/*  */
-u32	PHY_QueryBBReg(struct rtw_adapter *Adapter, u32 RegAddr,
-		       u32 BitMask);
-void	PHY_SetBBReg(struct rtw_adapter *Adapter, u32 RegAddr,
-		     u32 BitMask, u32 Data);
-u32	PHY_QueryRFReg(struct rtw_adapter *Adapter,
-		       enum RF_RADIO_PATH	eRFPath, u32 RegAddr,
-		       u32 BitMask);
-void	PHY_SetRFReg(struct rtw_adapter *Adapter,
-		     enum RF_RADIO_PATH eRFPath, u32 RegAddr,
-		     u32 BitMask,  u32	Data);
-
-/*  */
-/*  BB TX Power R/W */
-/*  */
-void PHY_SetTxPowerLevel8723A(struct rtw_adapter *Adapter, u8 channel);
-
-/*  */
-/*  Switch bandwidth for 8723A */
-/*  */
-void	PHY_SetBWMode23a8723A(struct rtw_adapter *pAdapter,
-			   enum ht_channel_width ChnlWidth,
-			   unsigned char Offset);
-
-/*  */
-/*  channel switch related funciton */
-/*  */
-void	PHY_SwChnl8723A(struct rtw_adapter *pAdapter, u8 channel);
-				/*  Call after initialization */
-void ChkFwCmdIoDone(struct rtw_adapter *Adapter);
-
-/*  */
-/*  Modify the value of the hw register when beacon interval be changed. */
-/*  */
-void
-rtl8192c_PHY_SetBeaconHwReg(struct rtw_adapter *Adapter, u16 BeaconInterval);
-
-
-void PHY_SwitchEphyParameter(struct rtw_adapter *Adapter);
-
-void PHY_EnableHostClkReq(struct rtw_adapter *Adapter);
-
-bool
-SetAntennaConfig92C(struct rtw_adapter *Adapter, u8 DefaultAnt);
-
-/*--------------------------Exported Function prototype---------------------*/
-
-#define PHY_SetMacReg	PHY_SetBBReg
-
-/* MAC/BB/RF HAL config */
-int PHY_BBConfig8723A(struct rtw_adapter *Adapter);
-s32 PHY_MACConfig8723A(struct rtw_adapter *padapter);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/Hal8723APhyReg.h b/drivers/staging/rtl8723au/include/Hal8723APhyReg.h
deleted file mode 100644
index 759928f78d6d..000000000000
--- a/drivers/staging/rtl8723au/include/Hal8723APhyReg.h
+++ /dev/null
@@ -1,1078 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- *
- ******************************************************************************/
-#ifndef __INC_HAL8723APHYREG_H__
-#define __INC_HAL8723APHYREG_H__
-
-/*  1. PMAC duplicate register due to connection: RF_Mode, TRxRN, NumOf L-STF */
-/*  1. Page1(0x100) */
-#define rPMAC_Reset				0x100
-#define rPMAC_TxStart				0x104
-#define rPMAC_TxLegacySIG			0x108
-#define rPMAC_TxHTSIG1				0x10c
-#define rPMAC_TxHTSIG2				0x110
-#define rPMAC_PHYDebug				0x114
-#define rPMAC_TxPacketNum			0x118
-#define rPMAC_TxIdle				0x11c
-#define rPMAC_TxMACHeader0			0x120
-#define rPMAC_TxMACHeader1			0x124
-#define rPMAC_TxMACHeader2			0x128
-#define rPMAC_TxMACHeader3			0x12c
-#define rPMAC_TxMACHeader4			0x130
-#define rPMAC_TxMACHeader5			0x134
-#define rPMAC_TxDataType			0x138
-#define rPMAC_TxRandomSeed			0x13c
-#define rPMAC_CCKPLCPPreamble			0x140
-#define rPMAC_CCKPLCPHeader			0x144
-#define rPMAC_CCKCRC16				0x148
-#define rPMAC_OFDMRxCRC32OK			0x170
-#define rPMAC_OFDMRxCRC32Er			0x174
-#define rPMAC_OFDMRxParityEr			0x178
-#define rPMAC_OFDMRxCRC8Er			0x17c
-#define rPMAC_CCKCRxRC16Er			0x180
-#define rPMAC_CCKCRxRC32Er			0x184
-#define rPMAC_CCKCRxRC32OK			0x188
-#define rPMAC_TxStatus				0x18c
-
-/*  2. Page2(0x200) */
-/*  The following two definition are only used for USB interface. */
-#define RF_BB_CMD_ADDR		0x02c0	/*  RF/BB read/write command address. */
-#define RF_BB_CMD_DATA		0x02c4	/*  RF/BB read/write command data. */
-
-/*  3. Page8(0x800) */
-#define rFPGA0_RFMOD		0x800	/* RF mode & CCK TxSC  RF BW Setting?? */
-
-#define rFPGA0_TxInfo		0x804	/*  Status report?? */
-#define rFPGA0_PSDFunction	0x808
-
-#define rFPGA0_TxGainStage	0x80c	/*  Set TX PWR init gain? */
-
-#define rFPGA0_RFTiming1	0x810	/*  Useless now */
-#define rFPGA0_RFTiming2	0x814
-
-#define rFPGA0_XA_HSSIParameter1	0x820	/*  RF 3 wire register */
-#define rFPGA0_XA_HSSIParameter2	0x824
-#define rFPGA0_XB_HSSIParameter1	0x828
-#define rFPGA0_XB_HSSIParameter2	0x82c
-#define rTxAGC_B_Rate18_06		0x830
-#define rTxAGC_B_Rate54_24		0x834
-#define rTxAGC_B_CCK1_55_Mcs32		0x838
-#define rTxAGC_B_Mcs03_Mcs00		0x83c
-
-#define rTxAGC_B_Mcs07_Mcs04		0x848
-#define rTxAGC_B_Mcs11_Mcs08		0x84c
-
-#define rFPGA0_XA_LSSIParameter		0x840
-#define rFPGA0_XB_LSSIParameter		0x844
-
-#define rFPGA0_RFWakeUpParameter	0x850	/*  Useless now */
-#define rFPGA0_RFSleepUpParameter	0x854
-
-#define rFPGA0_XAB_SwitchControl	0x858	/*  RF Channel switch */
-#define rFPGA0_XCD_SwitchControl	0x85c
-
-#define rFPGA0_XA_RFInterfaceOE		0x860	/*  RF Channel switch */
-#define rFPGA0_XB_RFInterfaceOE		0x864
-
-#define rTxAGC_B_Mcs15_Mcs12		0x868
-#define rTxAGC_B_CCK11_A_CCK2_11	0x86c
-
-#define rFPGA0_XAB_RFInterfaceSW	0x870	/*  RF Interface Software Control */
-#define rFPGA0_XCD_RFInterfaceSW	0x874
-
-#define rFPGA0_XAB_RFParameter		0x878	/*  RF Parameter */
-#define rFPGA0_XCD_RFParameter		0x87c
-
-#define rFPGA0_AnalogParameter1		0x880	/*  Crystal cap setting RF-R/W protection for parameter4?? */
-#define rFPGA0_AnalogParameter2		0x884
-#define rFPGA0_AnalogParameter3		0x888	/*  Useless now */
-#define rFPGA0_AnalogParameter4		0x88c
-
-#define rFPGA0_XA_LSSIReadBack		0x8a0	/*  Tranceiver LSSI Readback */
-#define rFPGA0_XB_LSSIReadBack		0x8a4
-#define rFPGA0_XC_LSSIReadBack		0x8a8
-#define rFPGA0_XD_LSSIReadBack		0x8ac
-
-#define rFPGA0_PSDReport		0x8b4	/*  Useless now */
-#define TransceiverA_HSPI_Readback	0x8b8	/*  Transceiver A HSPI Readback */
-#define TransceiverB_HSPI_Readback	0x8bc	/*  Transceiver B HSPI Readback */
-#define rFPGA0_XAB_RFInterfaceRB	0x8e0	/*  Useless now RF Interface Readback Value */
-#define rFPGA0_XCD_RFInterfaceRB	0x8e4	/*  Useless now */
-
-/*  4. Page9(0x900) */
-#define rFPGA1_RFMOD			0x900	/* RF mode & OFDM TxSC RF BW Setting?? */
-
-#define rFPGA1_TxBlock			0x904	/*  Useless now */
-#define rFPGA1_DebugSelect		0x908	/*  Useless now */
-#define rFPGA1_TxInfo			0x90c	/*  Useless now Status report?? */
-
-/*  5. PageA(0xA00) */
-/*  Set Control channel to upper or lower. These settings are required only for 40MHz */
-#define rCCK0_System			0xa00
-
-#define rCCK0_AFESetting		0xa04	/*  Disable init gain now Select RX path by RSSI */
-#define rCCK0_CCA			0xa08	/*  Disable init gain now Init gain */
-
-#define rCCK0_RxAGC1			0xa0c	/* AGC default value, saturation level Antenna Diversity, RX AGC, LNA Threshold, RX LNA Threshold useless now. Not the same as 90 series */
-#define rCCK0_RxAGC2			0xa10	/* AGC & DAGC */
-
-#define rCCK0_RxHP			0xa14
-
-#define rCCK0_DSPParameter1		0xa18	/* Timing recovery & Channel estimation threshold */
-#define rCCK0_DSPParameter2		0xa1c	/* SQ threshold */
-
-#define rCCK0_TxFilter1			0xa20
-#define rCCK0_TxFilter2			0xa24
-#define rCCK0_DebugPort			0xa28	/* debug port and Tx filter3 */
-#define rCCK0_FalseAlarmReport		0xa2c	/* 0xa2d	useless now 0xa30-a4f channel report */
-#define rCCK0_TRSSIReport		0xa50
-#define rCCK0_RxReport			0xa54  /* 0xa57 */
-#define rCCK0_FACounterLower		0xa5c  /* 0xa5b */
-#define rCCK0_FACounterUpper		0xa58  /* 0xa5c */
-/*  PageB(0xB00) */
-#define rPdp_AntA			0xb00
-#define rPdp_AntA_4			0xb04
-#define rConfig_Pmpd_AntA		0xb28
-#define rConfig_AntA			0xb68
-#define rConfig_AntB			0xb6c
-#define rPdp_AntB			0xb70
-#define rPdp_AntB_4			0xb74
-#define rConfig_Pmpd_AntB		0xb98
-#define rAPK				0xbd8
-
-/*  6. PageC(0xC00) */
-#define rOFDM0_LSTF			0xc00
-
-#define rOFDM0_TRxPathEnable		0xc04
-#define rOFDM0_TRMuxPar			0xc08
-#define rOFDM0_TRSWIsolation		0xc0c
-
-#define rOFDM0_XARxAFE			0xc10  /* RxIQ DC offset, Rx digital filter, DC notch filter */
-#define rOFDM0_XARxIQImbalance		0xc14  /* RxIQ imblance matrix */
-#define rOFDM0_XBRxAFE			0xc18
-#define rOFDM0_XBRxIQImbalance		0xc1c
-#define rOFDM0_XCRxAFE			0xc20
-#define rOFDM0_XCRxIQImbalance		0xc24
-#define rOFDM0_XDRxAFE			0xc28
-#define rOFDM0_XDRxIQImbalance		0xc2c
-
-#define rOFDM0_RxDetector1		0xc30  /* PD,BW & SBD	DM tune init gain */
-#define rOFDM0_RxDetector2		0xc34  /* SBD & Fame Sync. */
-#define rOFDM0_RxDetector3		0xc38  /* Frame Sync. */
-#define rOFDM0_RxDetector4		0xc3c  /* PD, SBD, Frame Sync & Short-GI */
-
-#define rOFDM0_RxDSP			0xc40  /* Rx Sync Path */
-#define rOFDM0_CFOandDAGC		0xc44  /* CFO & DAGC */
-#define rOFDM0_CCADropThreshold	0xc48 /* CCA Drop threshold */
-#define rOFDM0_ECCAThreshold		0xc4c /*  energy CCA */
-
-#define rOFDM0_XAAGCCore1		0xc50	/*  DIG */
-#define rOFDM0_XAAGCCore2		0xc54
-#define rOFDM0_XBAGCCore1		0xc58
-#define rOFDM0_XBAGCCore2		0xc5c
-#define rOFDM0_XCAGCCore1		0xc60
-#define rOFDM0_XCAGCCore2		0xc64
-#define rOFDM0_XDAGCCore1		0xc68
-#define rOFDM0_XDAGCCore2		0xc6c
-
-#define rOFDM0_AGCParameter1		0xc70
-#define rOFDM0_AGCParameter2		0xc74
-#define rOFDM0_AGCRSSITable		0xc78
-#define rOFDM0_HTSTFAGC			0xc7c
-
-#define rOFDM0_XATxIQImbalance		0xc80	/*  TX PWR TRACK and DIG */
-#define rOFDM0_XATxAFE			0xc84
-#define rOFDM0_XBTxIQImbalance		0xc88
-#define rOFDM0_XBTxAFE			0xc8c
-#define rOFDM0_XCTxIQImbalance		0xc90
-#define rOFDM0_XCTxAFE			0xc94
-#define rOFDM0_XDTxIQImbalance		0xc98
-#define rOFDM0_XDTxAFE			0xc9c
-
-#define rOFDM0_RxIQExtAnta		0xca0
-#define rOFDM0_TxCoeff1			0xca4
-#define rOFDM0_TxCoeff2			0xca8
-#define rOFDM0_TxCoeff3			0xcac
-#define rOFDM0_TxCoeff4			0xcb0
-#define rOFDM0_TxCoeff5			0xcb4
-#define rOFDM0_TxCoeff6			0xcb8
-#define rOFDM0_RxHPParameter		0xce0
-#define rOFDM0_TxPseudoNoiseWgt		0xce4
-#define rOFDM0_FrameSync		0xcf0
-#define rOFDM0_DFSReport		0xcf4
-
-/*  7. PageD(0xD00) */
-#define rOFDM1_LSTF			0xd00
-#define rOFDM1_TRxPathEnable		0xd04
-
-#define rOFDM1_CFO			0xd08	/*  No setting now */
-#define rOFDM1_CSI1			0xd10
-#define rOFDM1_SBD			0xd14
-#define rOFDM1_CSI2			0xd18
-#define rOFDM1_CFOTracking		0xd2c
-#define rOFDM1_TRxMesaure1		0xd34
-#define rOFDM1_IntfDet			0xd3c
-#define rOFDM1_PseudoNoiseStateAB	0xd50
-#define rOFDM1_PseudoNoiseStateCD	0xd54
-#define rOFDM1_RxPseudoNoiseWgt		0xd58
-
-#define rOFDM_PHYCounter1		0xda0  /* cca, parity fail */
-#define rOFDM_PHYCounter2		0xda4  /* rate illegal, crc8 fail */
-#define rOFDM_PHYCounter3		0xda8  /* MCS not support */
-
-#define rOFDM_ShortCFOAB		0xdac	/*  No setting now */
-#define rOFDM_ShortCFOCD		0xdb0
-#define rOFDM_LongCFOAB			0xdb4
-#define rOFDM_LongCFOCD			0xdb8
-#define rOFDM_TailCFOAB			0xdbc
-#define rOFDM_TailCFOCD			0xdc0
-#define rOFDM_PWMeasure1		0xdc4
-#define rOFDM_PWMeasure2		0xdc8
-#define rOFDM_BWReport			0xdcc
-#define rOFDM_AGCReport			0xdd0
-#define rOFDM_RxSNR			0xdd4
-#define rOFDM_RxEVMCSI			0xdd8
-#define rOFDM_SIGReport			0xddc
-
-
-/*  8. PageE(0xE00) */
-#define rTxAGC_A_Rate18_06		0xe00
-#define rTxAGC_A_Rate54_24		0xe04
-#define rTxAGC_A_CCK1_Mcs32		0xe08
-#define rTxAGC_A_Mcs03_Mcs00		0xe10
-#define rTxAGC_A_Mcs07_Mcs04		0xe14
-#define rTxAGC_A_Mcs11_Mcs08		0xe18
-#define rTxAGC_A_Mcs15_Mcs12		0xe1c
-
-#define rFPGA0_IQK			0xe28
-#define rTx_IQK_Tone_A			0xe30
-#define rRx_IQK_Tone_A			0xe34
-#define rTx_IQK_PI_A			0xe38
-#define rRx_IQK_PI_A			0xe3c
-
-#define rTx_IQK				0xe40
-#define rRx_IQK				0xe44
-#define rIQK_AGC_Pts			0xe48
-#define rIQK_AGC_Rsp			0xe4c
-#define rTx_IQK_Tone_B			0xe50
-#define rRx_IQK_Tone_B			0xe54
-#define rTx_IQK_PI_B			0xe58
-#define rRx_IQK_PI_B			0xe5c
-#define rIQK_AGC_Cont			0xe60
-
-#define rBlue_Tooth			0xe6c
-#define rRx_Wait_CCA			0xe70
-#define rTx_CCK_RFON			0xe74
-#define rTx_CCK_BBON			0xe78
-#define rTx_OFDM_RFON			0xe7c
-#define rTx_OFDM_BBON			0xe80
-#define rTx_To_Rx			0xe84
-#define rTx_To_Tx			0xe88
-#define rRx_CCK				0xe8c
-
-#define rTx_Power_Before_IQK_A		0xe94
-#define rTx_Power_After_IQK_A		0xe9c
-
-#define rRx_Power_Before_IQK_A		0xea0
-#define rRx_Power_Before_IQK_A_2	0xea4
-#define rRx_Power_After_IQK_A		0xea8
-#define rRx_Power_After_IQK_A_2		0xeac
-
-#define rTx_Power_Before_IQK_B		0xeb4
-#define rTx_Power_After_IQK_B		0xebc
-
-#define rRx_Power_Before_IQK_B		0xec0
-#define rRx_Power_Before_IQK_B_2	0xec4
-#define rRx_Power_After_IQK_B		0xec8
-#define rRx_Power_After_IQK_B_2		0xecc
-
-#define rRx_OFDM			0xed0
-#define rRx_Wait_RIFS			0xed4
-#define rRx_TO_Rx			0xed8
-#define rStandby			0xedc
-#define rSleep				0xee0
-#define rPMPD_ANAEN			0xeec
-
-/*  7. RF Register 0x00-0x2E (RF 8256) */
-/*     RF-0222D 0x00-3F */
-/* Zebra1 */
-#define rZebra1_HSSIEnable		0x0	/*  Useless now */
-#define rZebra1_TRxEnable1		0x1
-#define rZebra1_TRxEnable2		0x2
-#define rZebra1_AGC			0x4
-#define rZebra1_ChargePump		0x5
-#define rZebra1_Channel			0x7	/*  RF channel switch */
-
-#define rZebra1_TxGain			0x8	/*  Useless now */
-#define rZebra1_TxLPF			0x9
-#define rZebra1_RxLPF			0xb
-#define rZebra1_RxHPFCorner		0xc
-
-/* Zebra4 */
-#define rGlobalCtrl			0	/*  Useless now */
-#define rRTL8256_TxLPF			19
-#define rRTL8256_RxLPF			11
-
-/* RTL8258 */
-#define rRTL8258_TxLPF			0x11	/*  Useless now */
-#define rRTL8258_RxLPF			0x13
-#define rRTL8258_RSSILPF		0xa
-
-/*  RL6052 Register definition */
-#define RF_AC				0x00
-#define RF_IQADJ_G1			0x01
-#define RF_IQADJ_G2			0x02
-#define RF_BS_PA_APSET_G1_G4		0x03
-#define RF_BS_PA_APSET_G5_G8		0x04
-#define RF_POW_TRSW			0x05
-#define RF_GAIN_RX			0x06
-#define RF_GAIN_TX			0x07
-#define RF_TXM_IDAC			0x08
-#define RF_IPA_G			0x09
-#define RF_TXBIAS_G			0x0A
-#define RF_TXPA_AG			0x0B
-#define RF_IPA_A			0x0C
-#define RF_TXBIAS_A			0x0D
-#define RF_BS_PA_APSET_G9_G11		0x0E
-#define RF_BS_IQGEN			0x0F
-#define RF_MODE1			0x10
-#define RF_MODE2			0x11
-#define RF_RX_AGC_HP			0x12
-#define RF_TX_AGC			0x13
-#define RF_BIAS				0x14
-#define RF_IPA				0x15
-#define RF_TXBIAS			0x16
-#define RF_POW_ABILITY			0x17
-#define RF_MODE_AG			0x18
-#define rRfChannel			0x18	/*  RF channel and BW switch */
-#define RF_CHNLBW			0x18	/*  RF channel and BW switch */
-#define RF_TOP				0x19
-#define RF_RX_G1			0x1A
-#define RF_RX_G2			0x1B
-#define RF_RX_BB2			0x1C
-#define RF_RX_BB1			0x1D
-#define RF_RCK1				0x1E
-#define RF_RCK2				0x1F
-#define RF_TX_G1			0x20
-#define RF_TX_G2			0x21
-#define RF_TX_G3			0x22
-#define RF_TX_BB1			0x23
-#define RF_T_METER			0x24
-#define RF_SYN_G1			0x25	/*  RF TX Power control */
-#define RF_SYN_G2			0x26	/*  RF TX Power control */
-#define RF_SYN_G3			0x27	/*  RF TX Power control */
-#define RF_SYN_G4			0x28	/*  RF TX Power control */
-#define RF_SYN_G5			0x29	/*  RF TX Power control */
-#define RF_SYN_G6			0x2A	/*  RF TX Power control */
-#define RF_SYN_G7			0x2B	/*  RF TX Power control */
-#define RF_SYN_G8			0x2C	/*  RF TX Power control */
-
-#define RF_RCK_OS			0x30	/*  RF TX PA control */
-
-#define RF_TXPA_G1			0x31	/*  RF TX PA control */
-#define RF_TXPA_G2			0x32	/*  RF TX PA control */
-#define RF_TXPA_G3			0x33	/*  RF TX PA control */
-
-/* Bit Mask */
-/*  1. Page1(0x100) */
-#define bBBResetB			0x100	/*  Useless now? */
-#define bGlobalResetB			0x200
-#define bOFDMTxStart			0x4
-#define bCCKTxStart			0x8
-#define bCRC32Debug			0x100
-#define bPMACLoopback			0x10
-#define bTxLSIG				0xffffff
-#define bOFDMTxRate			0xf
-#define bOFDMTxReserved			0x10
-#define bOFDMTxLength			0x1ffe0
-#define bOFDMTxParity			0x20000
-#define bTxHTSIG1			0xffffff
-#define bTxHTMCSRate			0x7f
-#define bTxHTBW				0x80
-#define bTxHTLength			0xffff00
-#define bTxHTSIG2			0xffffff
-#define bTxHTSmoothing			0x1
-#define bTxHTSounding			0x2
-#define bTxHTReserved			0x4
-#define bTxHTAggreation			0x8
-#define bTxHTSTBC			0x30
-#define bTxHTAdvanceCoding		0x40
-#define bTxHTShortGI			0x80
-#define bTxHTNumberHT_LTF		0x300
-#define bTxHTCRC8			0x3fc00
-#define bCounterReset			0x10000
-#define bNumOfOFDMTx			0xffff
-#define bNumOfCCKTx			0xffff0000
-#define bTxIdleInterval			0xffff
-#define bOFDMService			0xffff0000
-#define bTxMACHeader			0xffffffff
-#define bTxDataInit			0xff
-#define bTxHTMode			0x100
-#define bTxDataType			0x30000
-#define bTxRandomSeed			0xffffffff
-#define bCCKTxPreamble			0x1
-#define bCCKTxSFD			0xffff0000
-#define bCCKTxSIG			0xff
-#define bCCKTxService			0xff00
-#define bCCKLengthExt			0x8000
-#define bCCKTxLength			0xffff0000
-#define bCCKTxCRC16			0xffff
-#define bCCKTxStatus			0x1
-#define bOFDMTxStatus			0x2
-
-#define IS_BB_REG_OFFSET_92S(_Offset)			\
-	((_Offset >= 0x800) && (_Offset <= 0xfff))
-
-/*  2. Page8(0x800) */
-#define bRFMOD				0x1	/*  Reg 0x800 rFPGA0_RFMOD */
-#define bJapanMode			0x2
-#define bCCKTxSC			0x30
-#define bCCKEn				0x1000000
-#define bOFDMEn				0x2000000
-
-#define bOFDMRxADCPhase			0x10000	/*  Useless now */
-#define bOFDMTxDACPhase			0x40000
-#define bXATxAGC			0x3f
-
-#define bAntennaSelect			0x0300
-
-#define bXBTxAGC			0xf00	/*  Reg 80c rFPGA0_TxGainStage */
-#define bXCTxAGC			0xf000
-#define bXDTxAGC			0xf0000
-
-#define bPAStart			0xf0000000	/*  Useless now */
-#define bTRStart			0x00f00000
-#define bRFStart			0x0000f000
-#define bBBStart			0x000000f0
-#define bBBCCKStart			0x0000000f
-#define bPAEnd				0xf          /* Reg0x814 */
-#define bTREnd				0x0f000000
-#define bRFEnd				0x000f0000
-#define bCCAMask			0x000000f0   /* T2R */
-#define bR2RCCAMask			0x00000f00
-#define bHSSI_R2TDelay			0xf8000000
-#define bHSSI_T2RDelay			0xf80000
-#define bContTxHSSI			0x400     /* chane gain at continue Tx */
-#define bIGFromCCK			0x200
-#define bAGCAddress			0x3f
-#define bRxHPTx				0x7000
-#define bRxHPT2R			0x38000
-#define bRxHPCCKIni			0xc0000
-#define bAGCTxCode			0xc00000
-#define bAGCRxCode			0x300000
-
-#define b3WireDataLength		0x800	/*  Reg 0x820~84f rFPGA0_XA_HSSIParameter1 */
-#define b3WireAddressLength		0x400
-
-#define b3WireRFPowerDown		0x1	/*  Useless now */
-/* define bHWSISelect			0x8 */
-#define b5GPAPEPolarity			0x40000000
-#define b2GPAPEPolarity			0x80000000
-#define bRFSW_TxDefaultAnt		0x3
-#define bRFSW_TxOptionAnt		0x30
-#define bRFSW_RxDefaultAnt		0x300
-#define bRFSW_RxOptionAnt		0x3000
-#define bRFSI_3WireData			0x1
-#define bRFSI_3WireClock		0x2
-#define bRFSI_3WireLoad			0x4
-#define bRFSI_3WireRW			0x8
-#define bRFSI_3Wire			0xf
-
-#define bRFSI_RFENV			0x10	/*  Reg 0x870 rFPGA0_XAB_RFInterfaceSW */
-
-#define bRFSI_TRSW			0x20	/*  Useless now */
-#define bRFSI_TRSWB			0x40
-#define bRFSI_ANTSW			0x100
-#define bRFSI_ANTSWB			0x200
-#define bRFSI_PAPE			0x400
-#define bRFSI_PAPE5G			0x800
-#define bBandSelect			0x1
-#define bHTSIG2_GI			0x80
-#define bHTSIG2_Smoothing		0x01
-#define bHTSIG2_Sounding		0x02
-#define bHTSIG2_Aggreaton		0x08
-#define bHTSIG2_STBC			0x30
-#define bHTSIG2_AdvCoding		0x40
-#define bHTSIG2_NumOfHTLTF		0x300
-#define bHTSIG2_CRC8			0x3fc
-#define bHTSIG1_MCS			0x7f
-#define bHTSIG1_BandWidth		0x80
-#define bHTSIG1_HTLength		0xffff
-#define bLSIG_Rate			0xf
-#define bLSIG_Reserved			0x10
-#define bLSIG_Length			0x1fffe
-#define bLSIG_Parity			0x20
-#define bCCKRxPhase			0x4
-
-#define bLSSIReadAddress		0x7f800000   /*  T65 RF */
-
-#define bLSSIReadEdge			0x80000000   /* LSSI "Read" edge signal */
-
-#define bLSSIReadBackData		0xfffff		/*  T65 RF */
-
-#define bLSSIReadOKFlag			0x1000	/*  Useless now */
-#define bCCKSampleRate			0x8       /* 0: 44MHz, 1:88MHz */
-#define bRegulator0Standby		0x1
-#define bRegulatorPLLStandby		0x2
-#define bRegulator1Standby		0x4
-#define bPLLPowerUp			0x8
-#define bDPLLPowerUp			0x10
-#define bDA10PowerUp			0x20
-#define bAD7PowerUp			0x200
-#define bDA6PowerUp			0x2000
-#define bXtalPowerUp			0x4000
-#define b40MDClkPowerUP			0x8000
-#define bDA6DebugMode			0x20000
-#define bDA6Swing			0x380000
-
-#define bADClkPhase			0x4000000	/*  Reg 0x880 rFPGA0_AnalogParameter1 20/40 CCK support switch 40/80 BB MHZ */
-
-#define b80MClkDelay			0x18000000	/*  Useless */
-#define bAFEWatchDogEnable		0x20000000
-
-#define bXtalCap01			0xc0000000	/*  Reg 0x884 rFPGA0_AnalogParameter2 Crystal cap */
-#define bXtalCap23			0x3
-#define bXtalCap92x			0x0f000000
-#define		bXtalCap		0x0f000000
-
-#define bIntDifClkEnable		0x400	/*  Useless */
-#define bExtSigClkEnable		0x800
-#define bBandgapMbiasPowerUp		0x10000
-#define bAD11SHGain			0xc0000
-#define bAD11InputRange			0x700000
-#define bAD11OPCurrent			0x3800000
-#define bIPathLoopback			0x4000000
-#define bQPathLoopback			0x8000000
-#define bAFELoopback			0x10000000
-#define bDA10Swing			0x7e0
-#define bDA10Reverse			0x800
-#define bDAClkSource			0x1000
-#define bAD7InputRange			0x6000
-#define bAD7Gain			0x38000
-#define bAD7OutputCMMode		0x40000
-#define bAD7InputCMMode			0x380000
-#define bAD7Current			0xc00000
-#define bRegulatorAdjust		0x7000000
-#define bAD11PowerUpAtTx		0x1
-#define bDA10PSAtTx			0x10
-#define bAD11PowerUpAtRx		0x100
-#define bDA10PSAtRx			0x1000
-#define bCCKRxAGCFormat			0x200
-#define bPSDFFTSamplepPoint		0xc000
-#define bPSDAverageNum			0x3000
-#define bIQPathControl			0xc00
-#define bPSDFreq			0x3ff
-#define bPSDAntennaPath			0x30
-#define bPSDIQSwitch			0x40
-#define bPSDRxTrigger			0x400000
-#define bPSDTxTrigger			0x80000000
-#define bPSDSineToneScale		0x7f000000
-#define bPSDReport			0xffff
-
-/*  3. Page9(0x900) */
-#define bOFDMTxSC			0x30000000	/*  Useless */
-#define bCCKTxOn			0x1
-#define bOFDMTxOn			0x2
-#define bDebugPage			0xfff  /* reset debug page and also HWord, LWord */
-#define bDebugItem			0xff   /* reset debug page and LWord */
-#define bAntL				0x10
-#define bAntNonHT			0x100
-#define bAntHT1				0x1000
-#define bAntHT2				0x10000
-#define bAntHT1S1			0x100000
-#define bAntNonHTS1			0x1000000
-
-/*  4. PageA(0xA00) */
-#define bCCKBBMode			0x3	/*  Useless */
-#define bCCKTxPowerSaving		0x80
-#define bCCKRxPowerSaving		0x40
-
-#define bCCKSideBand			0x10	/*  Reg 0xa00 rCCK0_System 20/40 switch */
-
-#define bCCKScramble			0x8	/*  Useless */
-#define bCCKAntDiversity		0x8000
-#define bCCKCarrierRecovery		0x4000
-#define bCCKTxRate			0x3000
-#define bCCKDCCancel			0x0800
-#define bCCKISICancel			0x0400
-#define bCCKMatchFilter			0x0200
-#define bCCKEqualizer			0x0100
-#define bCCKPreambleDetect		0x800000
-#define bCCKFastFalseCCA		0x400000
-#define bCCKChEstStart			0x300000
-#define bCCKCCACount			0x080000
-#define bCCKcs_lim			0x070000
-#define bCCKBistMode			0x80000000
-#define bCCKCCAMask			0x40000000
-#define bCCKTxDACPhase			0x4
-#define bCCKRxADCPhase			0x20000000   /* r_rx_clk */
-#define bCCKr_cp_mode0			0x0100
-#define bCCKTxDCOffset			0xf0
-#define bCCKRxDCOffset			0xf
-#define bCCKCCAMode			0xc000
-#define bCCKFalseCS_lim			0x3f00
-#define bCCKCS_ratio			0xc00000
-#define bCCKCorgBit_sel			0x300000
-#define bCCKPD_lim			0x0f0000
-#define bCCKNewCCA			0x80000000
-#define bCCKRxHPofIG			0x8000
-#define bCCKRxIG			0x7f00
-#define bCCKLNAPolarity			0x800000
-#define bCCKRx1stGain			0x7f0000
-#define bCCKRFExtend			0x20000000 /* CCK Rx Iinital gain polarity */
-#define bCCKRxAGCSatLevel		0x1f000000
-#define bCCKRxAGCSatCount		0xe0
-#define bCCKRxRFSettle			0x1f       /* AGCsamp_dly */
-#define bCCKFixedRxAGC			0x8000
-/* define bCCKRxAGCFormat		0x4000   remove to HSSI register 0x824 */
-#define bCCKAntennaPolarity		0x2000
-#define bCCKTxFilterType		0x0c00
-#define bCCKRxAGCReportType		0x0300
-#define bCCKRxDAGCEn			0x80000000
-#define bCCKRxDAGCPeriod		0x20000000
-#define bCCKRxDAGCSatLevel		0x1f000000
-#define bCCKTimingRecovery		0x800000
-#define bCCKTxC0			0x3f0000
-#define bCCKTxC1			0x3f000000
-#define bCCKTxC2			0x3f
-#define bCCKTxC3			0x3f00
-#define bCCKTxC4			0x3f0000
-#define bCCKTxC5			0x3f000000
-#define bCCKTxC6			0x3f
-#define bCCKTxC7			0x3f00
-#define bCCKDebugPort			0xff0000
-#define bCCKDACDebug			0x0f000000
-#define bCCKFalseAlarmEnable		0x8000
-#define bCCKFalseAlarmRead		0x4000
-#define bCCKTRSSI			0x7f
-#define bCCKRxAGCReport			0xfe
-#define bCCKRxReport_AntSel		0x80000000
-#define bCCKRxReport_MFOff		0x40000000
-#define bCCKRxRxReport_SQLoss		0x20000000
-#define bCCKRxReport_Pktloss		0x10000000
-#define bCCKRxReport_Lockedbit		0x08000000
-#define bCCKRxReport_RateError		0x04000000
-#define bCCKRxReport_RxRate		0x03000000
-#define bCCKRxFACounterLower		0xff
-#define bCCKRxFACounterUpper		0xff000000
-#define bCCKRxHPAGCStart		0xe000
-#define bCCKRxHPAGCFinal		0x1c00
-#define bCCKRxFalseAlarmEnable		0x8000
-#define bCCKFACounterFreeze		0x4000
-#define bCCKTxPathSel			0x10000000
-#define bCCKDefaultRxPath		0xc000000
-#define bCCKOptionRxPath		0x3000000
-
-/*  5. PageC(0xC00) */
-#define bNumOfSTF			0x3	/*  Useless */
-#define bShift_L			0xc0
-#define bGI_TH				0xc
-#define bRxPathA			0x1
-#define bRxPathB			0x2
-#define bRxPathC			0x4
-#define bRxPathD			0x8
-#define bTxPathA			0x1
-#define bTxPathB			0x2
-#define bTxPathC			0x4
-#define bTxPathD			0x8
-#define bTRSSIFreq			0x200
-#define bADCBackoff			0x3000
-#define bDFIRBackoff			0xc000
-#define bTRSSILatchPhase		0x10000
-#define bRxIDCOffset			0xff
-#define bRxQDCOffset			0xff00
-#define bRxDFIRMode			0x1800000
-#define bRxDCNFType			0xe000000
-#define bRXIQImb_A			0x3ff
-#define bRXIQImb_B			0xfc00
-#define bRXIQImb_C			0x3f0000
-#define bRXIQImb_D			0xffc00000
-#define bDC_dc_Notch			0x60000
-#define bRxNBINotch			0x1f000000
-#define bPD_TH				0xf
-#define bPD_TH_Opt2			0xc000
-#define bPWED_TH			0x700
-#define bIfMF_Win_L			0x800
-#define bPD_Option			0x1000
-#define bMF_Win_L			0xe000
-#define bBW_Search_L			0x30000
-#define bwin_enh_L			0xc0000
-#define bBW_TH				0x700000
-#define bED_TH2				0x3800000
-#define bBW_option			0x4000000
-#define bRatio_TH			0x18000000
-#define bWindow_L			0xe0000000
-#define bSBD_Option			0x1
-#define bFrame_TH			0x1c
-#define bFS_Option			0x60
-#define bDC_Slope_check			0x80
-#define bFGuard_Counter_DC_L		0xe00
-#define bFrame_Weight_Short		0x7000
-#define bSub_Tune			0xe00000
-#define bFrame_DC_Length		0xe000000
-#define bSBD_start_offset		0x30000000
-#define bFrame_TH_2			0x7
-#define bFrame_GI2_TH			0x38
-#define bGI2_Sync_en			0x40
-#define bSarch_Short_Early		0x300
-#define bSarch_Short_Late		0xc00
-#define bSarch_GI2_Late			0x70000
-#define bCFOAntSum			0x1
-#define bCFOAcc				0x2
-#define bCFOStartOffset			0xc
-#define bCFOLookBack			0x70
-#define bCFOSumWeight			0x80
-#define bDAGCEnable			0x10000
-#define bTXIQImb_A			0x3ff
-#define bTXIQImb_B			0xfc00
-#define bTXIQImb_C			0x3f0000
-#define bTXIQImb_D			0xffc00000
-#define bTxIDCOffset			0xff
-#define bTxQDCOffset			0xff00
-#define bTxDFIRMode			0x10000
-#define bTxPesudoNoiseOn		0x4000000
-#define bTxPesudoNoise_A		0xff
-#define bTxPesudoNoise_B		0xff00
-#define bTxPesudoNoise_C		0xff0000
-#define bTxPesudoNoise_D		0xff000000
-#define bCCADropOption			0x20000
-#define bCCADropThres			0xfff00000
-#define bEDCCA_H			0xf
-#define bEDCCA_L			0xf0
-#define bLambda_ED			0x300
-#define bRxInitialGain			0x7f
-#define bRxAntDivEn			0x80
-#define bRxAGCAddressForLNA		0x7f00
-#define bRxHighPowerFlow		0x8000
-#define bRxAGCFreezeThres		0xc0000
-#define bRxFreezeStep_AGC1		0x300000
-#define bRxFreezeStep_AGC2		0xc00000
-#define bRxFreezeStep_AGC3		0x3000000
-#define bRxFreezeStep_AGC0		0xc000000
-#define bRxRssi_Cmp_En			0x10000000
-#define bRxQuickAGCEn			0x20000000
-#define bRxAGCFreezeThresMode		0x40000000
-#define bRxOverFlowCheckType		0x80000000
-#define bRxAGCShift			0x7f
-#define bTRSW_Tri_Only			0x80
-#define bPowerThres			0x300
-#define bRxAGCEn			0x1
-#define bRxAGCTogetherEn		0x2
-#define bRxAGCMin			0x4
-#define bRxHP_Ini			0x7
-#define bRxHP_TRLNA			0x70
-#define bRxHP_RSSI			0x700
-#define bRxHP_BBP1			0x7000
-#define bRxHP_BBP2			0x70000
-#define bRxHP_BBP3			0x700000
-#define bRSSI_H				0x7f0000     /* the threshold for high power */
-#define bRSSI_Gen			0x7f000000   /* the threshold for ant diversity */
-#define bRxSettle_TRSW			0x7
-#define bRxSettle_LNA			0x38
-#define bRxSettle_RSSI			0x1c0
-#define bRxSettle_BBP			0xe00
-#define bRxSettle_RxHP			0x7000
-#define bRxSettle_AntSW_RSSI		0x38000
-#define bRxSettle_AntSW			0xc0000
-#define bRxProcessTime_DAGC		0x300000
-#define bRxSettle_HSSI			0x400000
-#define bRxProcessTime_BBPPW		0x800000
-#define bRxAntennaPowerShift		0x3000000
-#define bRSSITableSelect		0xc000000
-#define bRxHP_Final			0x7000000
-#define bRxHTSettle_BBP			0x7
-#define bRxHTSettle_HSSI		0x8
-#define bRxHTSettle_RxHP		0x70
-#define bRxHTSettle_BBPPW		0x80
-#define bRxHTSettle_Idle		0x300
-#define bRxHTSettle_Reserved		0x1c00
-#define bRxHTRxHPEn			0x8000
-#define bRxHTAGCFreezeThres		0x30000
-#define bRxHTAGCTogetherEn		0x40000
-#define bRxHTAGCMin			0x80000
-#define bRxHTAGCEn			0x100000
-#define bRxHTDAGCEn			0x200000
-#define bRxHTRxHP_BBP			0x1c00000
-#define bRxHTRxHP_Final			0xe0000000
-#define bRxPWRatioTH			0x3
-#define bRxPWRatioEn			0x4
-#define bRxMFHold			0x3800
-#define bRxPD_Delay_TH1			0x38
-#define bRxPD_Delay_TH2			0x1c0
-#define bRxPD_DC_COUNT_MAX		0x600
-/* define bRxMF_Hold           	    0x3800 */
-#define bRxPD_Delay_TH			0x8000
-#define bRxProcess_Delay		0xf0000
-#define bRxSearchrange_GI2_Early	0x700000
-#define bRxFrame_Guard_Counter_L	0x3800000
-#define bRxSGI_Guard_L			0xc000000
-#define bRxSGI_Search_L			0x30000000
-#define bRxSGI_TH			0xc0000000
-#define bDFSCnt0			0xff
-#define bDFSCnt1			0xff00
-#define bDFSFlag			0xf0000
-#define bMFWeightSum			0x300000
-#define bMinIdxTH			0x7f000000
-#define bDAFormat			0x40000
-#define bTxChEmuEnable			0x01000000
-#define bTRSWIsolation_A		0x7f
-#define bTRSWIsolation_B		0x7f00
-#define bTRSWIsolation_C		0x7f0000
-#define bTRSWIsolation_D		0x7f000000
-#define bExtLNAGain			0x7c00
-
-/*  6. PageE(0xE00) */
-#define bSTBCEn				0x4	/*  Useless */
-#define bAntennaMapping			0x10
-#define bNss				0x20
-#define bCFOAntSumD			0x200
-#define bPHYCounterReset		0x8000000
-#define bCFOReportGet			0x4000000
-#define bOFDMContinueTx			0x10000000
-#define bOFDMSingleCarrier		0x20000000
-#define bOFDMSingleTone			0x40000000
-/* define bRxPath1                 0x01 */
-/* define bRxPath2                 0x02 */
-/* define bRxPath3                 0x04 */
-/* define bRxPath4                 0x08 */
-/* define bTxPath1                 0x10 */
-/* define bTxPath2                 0x20 */
-#define bHTDetect			0x100
-#define bCFOEn				0x10000
-#define bCFOValue			0xfff00000
-#define bSigTone_Re			0x3f
-#define bSigTone_Im			0x7f00
-#define bCounter_CCA			0xffff
-#define bCounter_ParityFail		0xffff0000
-#define bCounter_RateIllegal		0xffff
-#define bCounter_CRC8Fail		0xffff0000
-#define bCounter_MCSNoSupport		0xffff
-#define bCounter_FastSync		0xffff
-#define bShortCFO			0xfff
-#define bShortCFOTLength		12   /* total */
-#define bShortCFOFLength		11   /* fraction */
-#define bLongCFO			0x7ff
-#define bLongCFOTLength			11
-#define bLongCFOFLength			11
-#define bTailCFO			0x1fff
-#define bTailCFOTLength			13
-#define bTailCFOFLength			12
-#define bmax_en_pwdB			0xffff
-#define bCC_power_dB			0xffff0000
-#define bnoise_pwdB			0xffff
-#define bPowerMeasTLength		10
-#define bPowerMeasFLength		3
-#define bRx_HT_BW			0x1
-#define bRxSC				0x6
-#define bRx_HT				0x8
-#define bNB_intf_det_on			0x1
-#define bIntf_win_len_cfg		0x30
-#define bNB_Intf_TH_cfg			0x1c0
-#define bRFGain				0x3f
-#define bTableSel			0x40
-#define bTRSW				0x80
-#define bRxSNR_A			0xff
-#define bRxSNR_B			0xff00
-#define bRxSNR_C			0xff0000
-#define bRxSNR_D			0xff000000
-#define bSNREVMTLength			8
-#define bSNREVMFLength			1
-#define bCSI1st				0xff
-#define bCSI2nd				0xff00
-#define bRxEVM1st			0xff0000
-#define bRxEVM2nd			0xff000000
-#define bSIGEVM				0xff
-#define bPWDB				0xff00
-#define bSGIEN				0x10000
-
-#define bSFactorQAM1			0xf	/*  Useless */
-#define bSFactorQAM2			0xf0
-#define bSFactorQAM3			0xf00
-#define bSFactorQAM4			0xf000
-#define bSFactorQAM5			0xf0000
-#define bSFactorQAM6			0xf0000
-#define bSFactorQAM7			0xf00000
-#define bSFactorQAM8			0xf000000
-#define bSFactorQAM9			0xf0000000
-#define bCSIScheme			0x100000
-
-#define bNoiseLvlTopSet			0x3	/*  Useless */
-#define bChSmooth			0x4
-#define bChSmoothCfg1			0x38
-#define bChSmoothCfg2			0x1c0
-#define bChSmoothCfg3			0xe00
-#define bChSmoothCfg4			0x7000
-#define bMRCMode			0x800000
-#define bTHEVMCfg			0x7000000
-
-#define bLoopFitType			0x1	/*  Useless */
-#define bUpdCFO				0x40
-#define bUpdCFOOffData			0x80
-#define bAdvUpdCFO			0x100
-#define bAdvTimeCtrl			0x800
-#define bUpdClko			0x1000
-#define bFC				0x6000
-#define bTrackingMode			0x8000
-#define bPhCmpEnable			0x10000
-#define bUpdClkoLTF			0x20000
-#define bComChCFO			0x40000
-#define bCSIEstiMode			0x80000
-#define bAdvUpdEqz			0x100000
-#define bUChCfg				0x7000000
-#define bUpdEqz				0x8000000
-
-/* Rx Pseduo noise */
-#define bRxPesudoNoiseOn		0x20000000	/*  Useless */
-#define bRxPesudoNoise_A		0xff
-#define bRxPesudoNoise_B		0xff00
-#define bRxPesudoNoise_C		0xff0000
-#define bRxPesudoNoise_D		0xff000000
-#define bPesudoNoiseState_A		0xffff
-#define bPesudoNoiseState_B		0xffff0000
-#define bPesudoNoiseState_C		0xffff
-#define bPesudoNoiseState_D		0xffff0000
-
-/* 7. RF Register */
-/* Zebra1 */
-#define bZebra1_HSSIEnable		0x8		/*  Useless */
-#define bZebra1_TRxControl		0xc00
-#define bZebra1_TRxGainSetting		0x07f
-#define bZebra1_RxCorner		0xc00
-#define bZebra1_TxChargePump		0x38
-#define bZebra1_RxChargePump		0x7
-#define bZebra1_ChannelNum		0xf80
-#define bZebra1_TxLPFBW			0x400
-#define bZebra1_RxLPFBW			0x600
-
-/* Zebra4 */
-#define bRTL8256RegModeCtrl1		0x100	/*  Useless */
-#define bRTL8256RegModeCtrl0		0x40
-#define bRTL8256_TxLPFBW		0x18
-#define bRTL8256_RxLPFBW		0x600
-
-/* RTL8258 */
-#define bRTL8258_TxLPFBW		0xc	/*  Useless */
-#define bRTL8258_RxLPFBW		0xc00
-#define bRTL8258_RSSILPFBW		0xc0
-
-
-/*  Other Definition */
-
-/* byte endable for sb_write */
-#define bByte0				0x1	/*  Useless */
-#define bByte1				0x2
-#define bByte2				0x4
-#define bByte3				0x8
-#define bWord0				0x3
-#define bWord1				0xc
-#define bDWord				0xf
-
-/* for PutRegsetting & GetRegSetting BitMask */
-#define bMaskByte0			0xff	/*  Reg 0xc50 rOFDM0_XAAGCCore~0xC6f */
-#define bMaskByte1			0xff00
-#define bMaskByte2			0xff0000
-#define bMaskByte3			0xff000000
-#define bMaskHWord			0xffff0000
-#define bMaskLWord			0x0000ffff
-#define bMaskDWord			0xffffffff
-#define bMask12Bits			0xfff
-#define bMaskH4Bits			0xf0000000
-#define bMaskOFDM_D			0xffc00000
-#define bMaskCCK			0x3f3f3f3f
-
-/* for PutRFRegsetting & GetRFRegSetting BitMask */
-#define		bRFRegOffsetMask	0xfffff
-
-#define bDisable			0x0
-
-#define LeftAntenna			0x0	/*  Useless */
-#define RightAntenna			0x1
-
-#define tCheckTxStatus			500   /* 500ms Useless */
-#define tUpdateRxCounter		100   /* 100ms */
-
-#define rateCCK				0	/*  Useless */
-#define rateOFDM			1
-#define rateHT				2
-
-/* define Register-End */
-#define bPMAC_End			0x1ff	/*  Useless */
-#define bFPGAPHY0_End			0x8ff
-#define bFPGAPHY1_End			0x9ff
-#define bCCKPHY0_End			0xaff
-#define bOFDMPHY0_End			0xcff
-#define bOFDMPHY1_End			0xdff
-
-/* define max debug item in each debug page */
-/* define bMaxItem_FPGA_PHY0        0x9 */
-/* define bMaxItem_FPGA_PHY1        0x3 */
-/* define bMaxItem_PHY_11B          0x16 */
-/* define bMaxItem_OFDM_PHY0        0x29 */
-/* define bMaxItem_OFDM_PHY1        0x0 */
-
-#define bPMACControl			0x0	/*  Useless */
-#define bWMACControl			0x1
-#define bWNICControl			0x2
-
-#define PathA				0x0	/*  Useless */
-#define PathB				0x1
-#define PathC				0x2
-#define PathD				0x3
-
-/*  PageB(0xB00) */
-#define rPdp_AntA			0xb00
-#define rPdp_AntA_4			0xb04
-#define rPdp_AntA_8			0xb08
-#define rPdp_AntA_C			0xb0c
-#define rPdp_AntA_18			0xb18
-#define rPdp_AntA_1C			0xb1c
-#define rPdp_AntA_20			0xb20
-#define rPdp_AntA_24			0xb24
-
-#define rConfig_Pmpd_AntA		0xb28
-#define rConfig_ram64x16		0xb2c
-
-#define rBndA				0xb30
-#define rHssiPar			0xb34
-
-#define rConfig_AntA			0xb68
-#define rConfig_AntB			0xb6c
-
-#define rPdp_AntB			0xb70
-#define rPdp_AntB_4			0xb74
-#define rPdp_AntB_8			0xb78
-#define rPdp_AntB_C			0xb7c
-#define rPdp_AntB_10			0xb80
-#define rPdp_AntB_14			0xb84
-#define rPdp_AntB_18			0xb88
-#define rPdp_AntB_1C			0xb8c
-#define rPdp_AntB_20			0xb90
-#define rPdp_AntB_24			0xb94
-
-#define rConfig_Pmpd_AntB		0xb98
-
-#define rBndB				0xba0
-
-#define rAPK				0xbd8
-#define rPm_Rx0_AntA			0xbdc
-#define rPm_Rx1_AntA			0xbe0
-#define rPm_Rx2_AntA			0xbe4
-#define rPm_Rx3_AntA			0xbe8
-#define rPm_Rx0_AntB			0xbec
-#define rPm_Rx1_AntB			0xbf0
-#define rPm_Rx2_AntB			0xbf4
-#define rPm_Rx3_AntB			0xbf8
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/Hal8723PwrSeq.h b/drivers/staging/rtl8723au/include/Hal8723PwrSeq.h
deleted file mode 100644
index 3771d6bb5774..000000000000
--- a/drivers/staging/rtl8723au/include/Hal8723PwrSeq.h
+++ /dev/null
@@ -1,126 +0,0 @@
-#ifndef __HAL8723PWRSEQ_H__
-#define __HAL8723PWRSEQ_H__
-/*
-	Check document WM-20110607-Paul-RTL8723A_Power_Architecture-R02.vsd
-	There are 6 HW Power States:
-	0: POFF--Power Off
-	1: PDN--Power Down
-	2: CARDEMU--Card Emulation
-	3: ACT--Active Mode
-	4: LPS--Low Power State
-	5: SUS--Suspend
-
-	The transision from different states are defined below
-	TRANS_CARDEMU_TO_ACT
-	TRANS_ACT_TO_CARDEMU
-	TRANS_CARDEMU_TO_SUS
-	TRANS_SUS_TO_CARDEMU
-	TRANS_CARDEMU_TO_PDN
-	TRANS_ACT_TO_LPS
-	TRANS_LPS_TO_ACT
-
-	TRANS_END
-*/
-#include "HalPwrSeqCmd.h"
-#include "rtl8723a_spec.h"
-
-#define	RTL8723A_TRANS_CARDEMU_TO_ACT_STEPS	15
-#define	RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS	15
-#define	RTL8723A_TRANS_CARDEMU_TO_SUS_STEPS	15
-#define	RTL8723A_TRANS_SUS_TO_CARDEMU_STEPS	15
-#define	RTL8723A_TRANS_CARDEMU_TO_PDN_STEPS	15
-#define	RTL8723A_TRANS_PDN_TO_CARDEMU_STEPS	15
-#define	RTL8723A_TRANS_ACT_TO_LPS_STEPS	15
-#define	RTL8723A_TRANS_LPS_TO_ACT_STEPS	15
-#define	RTL8723A_TRANS_END_STEPS	1
-
-
-/* format
- * { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value },  comments here
- */
-#define RTL8723A_TRANS_CARDEMU_TO_ACT														\
-	{0x0020, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)}, /*0x20[0] = 1b'1 enable LDOA12 MACRO block for all interface*/ \
-	{0x0067, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), 0}, /*0x67[0] = 0 to disable BT_GPS_SEL pins*/	\
-	{0x0001, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_DELAY, 1, PWRSEQ_DELAY_MS},/*Delay 1ms*/   \
-	{0x0000, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(5), 0}, /*0x00[5] = 1b'0 release analog Ips to digital , 1:isolation*/   \
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(2), 0},/* disable SW LPS 0x04[10]= 0*/	\
-	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(1), BIT(1)},/* wait till 0x04[17] = 1    power ready*/	\
-	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/* release WLON reset  0x04[16]= 1*/ \
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(7), 0},/* disable HWPDN 0x04[15]= 0*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, (BIT(4)|BIT(3)), 0},/* disable WL suspend*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/* polling until return 0*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(0), 0},/**/	\
-	{0x004E, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(7), 1},/*0x4C[23] = 0x4E[7] = 1, switch DPDT_SEL_P output from WL BB */\
-
-#define RTL8723A_TRANS_ACT_TO_CARDEMU													\
-	{0x001F, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0},/*0x1F[7:0] = 0 turn off RF*/	\
-	{0x004E, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(7), 0},/*0x4C[23] = 0x4E[7] = 0, switch DPDT_SEL_P output from register 0x65[2] */\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), BIT(1)}, /*0x04[9] = 1 turn off MAC by HW state machine*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(1), 0}, /*wait till 0x04[9] = 0 polling until return 0 to disable*/	\
-	{0x0000, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(5), BIT(5)}, /*0x00[5] = 1b'1 analog Ips to digital , 1:isolation*/   \
-	{0x0020, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0}, /*0x20[0] = 1b'0 disable LDOA12 MACRO block*/   \
-
-
-#define RTL8723A_TRANS_CARDEMU_TO_SUS													\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3)|BIT(4), BIT(3)}, /*0x04[12:11] = 2b'01 enable WL suspend*/
-
-#define RTL8723A_TRANS_SUS_TO_CARDEMU													\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3) | BIT(7), 0}, /*clear suspend enable and power down enable*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3)|BIT(4), 0}, /*0x04[12:11] = 2b'01enable WL suspend*/
-
-#define RTL8723A_TRANS_CARDEMU_TO_CARDDIS													\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3)|BIT(4), BIT(3)}, /*0x04[12:11] = 2b'01 enable WL suspend*/	\
-	{0x004A, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)}, /*0x48[16] = 1 to enable GPIO9 as EXT WAKEUP*/
-
-#define RTL8723A_TRANS_CARDDIS_TO_CARDEMU													\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3) | BIT(7), 0}, /*clear suspend enable and power down enable*/	\
-	{0x004A, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0}, /*0x48[16] = 0 to disable GPIO9 as EXT WAKEUP*/   \
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3)|BIT(4), 0}, /*0x04[12:11] = 2b'01enable WL suspend*/
-
-#define RTL8723A_TRANS_CARDEMU_TO_PDN												\
-	{0x0007, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x20}, /*0x07[7:0] = 0x20 SOP option to disable BG/MB/ACK/SWR*/   \
-	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0},/* 0x04[16] = 0*/\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(7), BIT(7)},/* 0x04[15] = 1*/
-
-#define RTL8723A_TRANS_PDN_TO_CARDEMU												\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(7), 0},/* 0x04[15] = 0*/
-
-#define RTL8723A_TRANS_ACT_TO_LPS														\
-	{0x0522, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0xFF},/*Tx Pause*/	\
-	{0x05F8, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, 0xFF, 0},/*Should be zero if no packet is transmitting*/	\
-	{0x05F9, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, 0xFF, 0},/*Should be zero if no packet is transmitting*/	\
-	{0x05FA, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, 0xFF, 0},/*Should be zero if no packet is transmitting*/	\
-	{0x05FB, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, 0xFF, 0},/*Should be zero if no packet is transmitting*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0},/*CCK and OFDM are disabled, and clock are gated*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_DELAY, 0, PWRSEQ_DELAY_US},/*Delay 1us*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), 0},/*Whole BB is reset*/	\
-	{0x0100, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x03},/*Reset MAC TRX*/	\
-	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), 0},/*check if removed later*/	\
-	{0x0553, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(5), BIT(5)},/*Respond TxOK to scheduler*/
-
-#define RTL8723A_TRANS_LPS_TO_ACT															\
-	{0xFE58, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x84}, /*USB RPWM*/\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_DELAY, 0, PWRSEQ_DELAY_MS}, /*Delay*/\
-	{0x0008, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), 0}, /*.	0x08[4] = 0		 switch TSF to 40M*/\
-	{0x0109, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(7), 0}, /*Polling 0x109[7]= 0  TSF in 40M*/\
-	{0x0029, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(6)|BIT(7), 0}, /*.	0x29[7:6] = 2b'00	 enable BB clock*/\
-	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), BIT(1)}, /*.	0x101[1] = 1*/\
-	{0x0100, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0xFF}, /*.	0x100[7:0] = 0xFF	 enable WMAC TRX*/\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1)|BIT(0), BIT(1)|BIT(0)}, /*.	0x02[1:0] = 2b'11	 enable BB macro*/\
-	{0x0522, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0}, /*.	0x522 = 0*/
-
-#define RTL8723A_TRANS_END															\
-	{0xFFFF, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, 0, PWR_CMD_END, 0, 0},
-
-
-extern struct wlan_pwr_cfg rtl8723AU_power_on_flow[RTL8723A_TRANS_CARDEMU_TO_ACT_STEPS+RTL8723A_TRANS_END_STEPS];
-extern struct wlan_pwr_cfg rtl8723AU_radio_off_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_END_STEPS];
-extern struct wlan_pwr_cfg rtl8723AU_card_disable_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_PDN_STEPS+RTL8723A_TRANS_END_STEPS];
-extern struct wlan_pwr_cfg rtl8723AU_card_enable_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_PDN_STEPS+RTL8723A_TRANS_END_STEPS];
-extern struct wlan_pwr_cfg rtl8723AU_suspend_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_SUS_STEPS+RTL8723A_TRANS_END_STEPS];
-extern struct wlan_pwr_cfg rtl8723AU_resume_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_SUS_STEPS+RTL8723A_TRANS_END_STEPS];
-extern struct wlan_pwr_cfg rtl8723AU_hwpdn_flow[RTL8723A_TRANS_ACT_TO_CARDEMU_STEPS+RTL8723A_TRANS_CARDEMU_TO_PDN_STEPS+RTL8723A_TRANS_END_STEPS];
-extern struct wlan_pwr_cfg rtl8723AU_enter_lps_flow[RTL8723A_TRANS_ACT_TO_LPS_STEPS+RTL8723A_TRANS_END_STEPS];
-extern struct wlan_pwr_cfg rtl8723AU_leave_lps_flow[RTL8723A_TRANS_LPS_TO_ACT_STEPS+RTL8723A_TRANS_END_STEPS];
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/Hal8723UHWImg_CE.h b/drivers/staging/rtl8723au/include/Hal8723UHWImg_CE.h
deleted file mode 100644
index c834b3a738d7..000000000000
--- a/drivers/staging/rtl8723au/include/Hal8723UHWImg_CE.h
+++ /dev/null
@@ -1,29 +0,0 @@
-#ifndef __INC_HAL8723U_FW_IMG_H
-#define __INC_HAL8723U_FW_IMG_H
-
-/*Created on  2013/01/14, 15:51*/
-
-/* FW v16 enable usb interrupt */
-#define Rtl8723UImgArrayLength 22172
-extern u8 Rtl8723UFwImgArray[Rtl8723UImgArrayLength];
-#define Rtl8723UBTImgArrayLength 1
-extern u8 Rtl8723UFwBTImgArray[Rtl8723UBTImgArrayLength];
-
-#define Rtl8723UUMCBCutImgArrayWithBTLength 24118
-#define Rtl8723UUMCBCutImgArrayWithoutBTLength 19200
-
-extern u8 Rtl8723UFwUMCBCutImgArrayWithBT[Rtl8723UUMCBCutImgArrayWithBTLength];
-extern u8 Rtl8723UFwUMCBCutImgArrayWithoutBT[Rtl8723UUMCBCutImgArrayWithoutBTLength];
-
-#define Rtl8723SUMCBCutMPImgArrayLength 24174
-extern const u8 Rtl8723SFwUMCBCutMPImgArray[Rtl8723SUMCBCutMPImgArrayLength];
-
-#define Rtl8723EBTImgArrayLength 15276
-extern u8 Rtl8723EFwBTImgArray[Rtl8723EBTImgArrayLength];
-
-#define Rtl8723UPHY_REG_Array_PGLength 336
-extern u32 Rtl8723UPHY_REG_Array_PG[Rtl8723UPHY_REG_Array_PGLength];
-#define Rtl8723UMACPHY_Array_PGLength 1
-extern u32 Rtl8723UMACPHY_Array_PG[Rtl8723UMACPHY_Array_PGLength];
-
-#endif /* ifndef __INC_HAL8723U_FW_IMG_H */
diff --git a/drivers/staging/rtl8723au/include/HalDMOutSrc8723A.h b/drivers/staging/rtl8723au/include/HalDMOutSrc8723A.h
deleted file mode 100644
index d7651f7a665b..000000000000
--- a/drivers/staging/rtl8723au/include/HalDMOutSrc8723A.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- *
- ******************************************************************************/
-#ifndef	__RTL8723A_ODM_H__
-#define __RTL8723A_ODM_H__
-/*  */
-
-#define	RSSI_CCK	0
-#define	RSSI_OFDM	1
-#define	RSSI_DEFAULT	2
-
-#define IQK_MAC_REG_NUM		4
-#define IQK_ADDA_REG_NUM		16
-#define IQK_BB_REG_NUM			9
-#define HP_THERMAL_NUM		8
-
-
-/*  */
-/*  structure and define */
-/*  */
-
-
-
-
-/*------------------------Export global variable----------------------------*/
-/*------------------------Export global variable----------------------------*/
-/*------------------------Export Marco Definition---------------------------*/
-/* define DM_MultiSTA_InitGainChangeNotify(Event) {DM_DigTable.CurMultiSTAConnectState = Event;} */
-
-
-/*  */
-/*  function prototype */
-/*  */
-
-/*  */
-/*  IQ calibrate */
-/*  */
-void rtl8723a_phy_iq_calibrate(struct rtw_adapter *pAdapter, bool bReCovery);
-
-/*  */
-/*  LC calibrate */
-/*  */
-void rtl8723a_phy_lc_calibrate(struct rtw_adapter *pAdapter);
-
-/*  */
-/*  AP calibrate */
-/*  */
-void rtl8723a_phy_ap_calibrate(struct rtw_adapter *pAdapter, char delta);
-
-void rtl8723a_odm_check_tx_power_tracking(struct rtw_adapter *Adapter);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/HalHWImg8723A_BB.h b/drivers/staging/rtl8723au/include/HalHWImg8723A_BB.h
deleted file mode 100644
index 127609404652..000000000000
--- a/drivers/staging/rtl8723au/include/HalHWImg8723A_BB.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/******************************************************************************
-*
-* Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
-*
-* This program is free software; you can redistribute it and/or modify it
-* under the terms of version 2 of the GNU General Public License as
-* published by the Free Software Foundation.
-*
-* This program is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-* more details.
-*
-*
-******************************************************************************/
-
-#ifndef __INC_BB_8723A_HW_IMG_H
-#define __INC_BB_8723A_HW_IMG_H
-
-/******************************************************************************
-*                           AGC_TAB_1T.TXT
-******************************************************************************/
-
-void ODM_ReadAndConfig_AGC_TAB_1T_8723A(struct dm_odm_t *pDM_Odm);
-
-/******************************************************************************
-*                           PHY_REG_1T.TXT
-******************************************************************************/
-
-void ODM_ReadAndConfig_PHY_REG_1T_8723A(struct dm_odm_t *pDM_Odm);
-
-/******************************************************************************
-*                           PHY_REG_MP.TXT
-******************************************************************************/
-
-void ODM_ReadAndConfig_PHY_REG_MP_8723A(struct dm_odm_t *pDM_Odm);
-
-#endif /*  end of HWIMG_SUPPORT */
diff --git a/drivers/staging/rtl8723au/include/HalHWImg8723A_FW.h b/drivers/staging/rtl8723au/include/HalHWImg8723A_FW.h
deleted file mode 100644
index 7ee363b99b49..000000000000
--- a/drivers/staging/rtl8723au/include/HalHWImg8723A_FW.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/******************************************************************************
-*
-* Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
-*
-* This program is free software; you can redistribute it and/or modify it
-* under the terms of version 2 of the GNU General Public License as
-* published by the Free Software Foundation.
-*
-* This program is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-* more details.
-*
-*
-******************************************************************************/
-
-#ifndef __INC_FW_8723A_HW_IMG_H
-#define __INC_FW_8723A_HW_IMG_H
-
-
-/******************************************************************************
-*                           rtl8723fw_B.TXT
-******************************************************************************/
-
-void ODM_ReadFirmware_8723A_rtl8723fw_B(struct dm_odm_t *pDM_Odm,
-					u8 *pFirmware, u32 *pFirmwareSize);
-
-#endif /*  end of HWIMG_SUPPORT */
diff --git a/drivers/staging/rtl8723au/include/HalHWImg8723A_MAC.h b/drivers/staging/rtl8723au/include/HalHWImg8723A_MAC.h
deleted file mode 100644
index 201be1f87292..000000000000
--- a/drivers/staging/rtl8723au/include/HalHWImg8723A_MAC.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/******************************************************************************
-*
-* Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
-*
-* This program is free software; you can redistribute it and/or modify it
-* under the terms of version 2 of the GNU General Public License as
-* published by the Free Software Foundation.
-*
-* This program is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-* more details.
-*
-*
-******************************************************************************/
-
-#ifndef __INC_MAC_8723A_HW_IMG_H
-#define __INC_MAC_8723A_HW_IMG_H
-
-/******************************************************************************
-*                           MAC_REG.TXT
-******************************************************************************/
-
-void ODM_ReadAndConfig_MAC_REG_8723A(struct dm_odm_t *pDM_Odm);
-
-#endif /*  end of HWIMG_SUPPORT */
diff --git a/drivers/staging/rtl8723au/include/HalHWImg8723A_RF.h b/drivers/staging/rtl8723au/include/HalHWImg8723A_RF.h
deleted file mode 100644
index c9af1c375339..000000000000
--- a/drivers/staging/rtl8723au/include/HalHWImg8723A_RF.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/******************************************************************************
-*
-* Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
-*
-* This program is free software; you can redistribute it and/or modify it
-* under the terms of version 2 of the GNU General Public License as
-* published by the Free Software Foundation.
-*
-* This program is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-* more details.
-*
-******************************************************************************/
-
-#ifndef __INC_RF_8723A_HW_IMG_H
-#define __INC_RF_8723A_HW_IMG_H
-
-/******************************************************************************
-*                           RadioA_1T.TXT
-******************************************************************************/
-
-void ODM_ReadAndConfig_RadioA_1T_8723A(struct dm_odm_t *pDM_Odm);
-
-#endif /*  end of HWIMG_SUPPORT */
diff --git a/drivers/staging/rtl8723au/include/HalPwrSeqCmd.h b/drivers/staging/rtl8723au/include/HalPwrSeqCmd.h
deleted file mode 100644
index 12e03a36f2d3..000000000000
--- a/drivers/staging/rtl8723au/include/HalPwrSeqCmd.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- *
- ******************************************************************************/
-#ifndef __HALPWRSEQCMD_H__
-#define __HALPWRSEQCMD_H__
-
-#include <drv_types.h>
-
-/*---------------------------------------------*/
-/*---------------------------------------------*/
-#define PWR_CMD_READ			0x00
-     /*  offset: the read register offset */
-     /*  msk: the mask of the read value */
-     /*  value: N/A, left by 0 */
-     /*  note: dirver shall implement this function by read & msk */
-
-#define PWR_CMD_WRITE			0x01
-     /*  offset: the read register offset */
-     /*  msk: the mask of the write bits */
-     /*  value: write value */
-     /*  note: driver shall implement this cmd by read & msk after write */
-
-#define PWR_CMD_POLLING			0x02
-     /*  offset: the read register offset */
-     /*  msk: the mask of the polled value */
-     /*  value: the value to be polled, masked by the msd field. */
-     /*  note: driver shall implement this cmd by */
-     /*  do{ */
-     /*  if( (Read(offset) & msk) == (value & msk) ) */
-     /*  break; */
-     /*  } while(not timeout); */
-
-#define PWR_CMD_DELAY			0x03
-     /*  offset: the value to delay */
-     /*  msk: N/A */
-     /*  value: the unit of delay, 0: us, 1: ms */
-
-#define PWR_CMD_END				0x04
-     /*  offset: N/A */
-     /*  msk: N/A */
-     /*  value: N/A */
-
-/*---------------------------------------------*/
-/* 3 The value of base: 4 bits */
-/*---------------------------------------------*/
-   /*  define the base address of each block */
-#define PWR_BASEADDR_MAC		0x00
-#define PWR_BASEADDR_USB		0x01
-#define PWR_BASEADDR_PCIE		0x02
-#define PWR_BASEADDR_SDIO		0x03
-
-/*---------------------------------------------*/
-/* 3 The value of interface_msk: 4 bits */
-/*---------------------------------------------*/
-#define	PWR_INTF_SDIO_MSK		BIT(0)
-#define	PWR_INTF_USB_MSK		BIT(1)
-#define	PWR_INTF_PCI_MSK		BIT(2)
-#define	PWR_INTF_ALL_MSK		(BIT(0)|BIT(1)|BIT(2)|BIT(3))
-
-/*---------------------------------------------*/
-/* 3 The value of fab_msk: 4 bits */
-/*---------------------------------------------*/
-#define	PWR_FAB_TSMC_MSK		BIT(0)
-#define	PWR_FAB_UMC_MSK			BIT(1)
-#define	PWR_FAB_ALL_MSK			(BIT(0)|BIT(1)|BIT(2)|BIT(3))
-
-/*---------------------------------------------*/
-/* 3 The value of cut_msk: 8 bits */
-/*---------------------------------------------*/
-#define	PWR_CUT_TESTCHIP_MSK	BIT(0)
-#define	PWR_CUT_A_MSK			BIT(1)
-#define	PWR_CUT_B_MSK			BIT(2)
-#define	PWR_CUT_C_MSK			BIT(3)
-#define	PWR_CUT_D_MSK			BIT(4)
-#define	PWR_CUT_E_MSK			BIT(5)
-#define	PWR_CUT_F_MSK			BIT(6)
-#define	PWR_CUT_G_MSK			BIT(7)
-#define	PWR_CUT_ALL_MSK			0xFF
-
-
-enum pwrseq_delay_unit {
-	PWRSEQ_DELAY_US,
-	PWRSEQ_DELAY_MS,
-};
-
-struct wlan_pwr_cfg {
-	u16 offset;
-	u8 cut_msk;
-	u8 fab_msk:4;
-	u8 interface_msk:4;
-	u8 base:4;
-	u8 cmd:4;
-	u8 msk;
-	u8 value;
-};
-
-
-#define GET_PWR_CFG_OFFSET(__PWR_CMD)		__PWR_CMD.offset
-#define GET_PWR_CFG_CUT_MASK(__PWR_CMD)		__PWR_CMD.cut_msk
-#define GET_PWR_CFG_FAB_MASK(__PWR_CMD)		__PWR_CMD.fab_msk
-#define GET_PWR_CFG_INTF_MASK(__PWR_CMD)	__PWR_CMD.interface_msk
-#define GET_PWR_CFG_BASE(__PWR_CMD)			__PWR_CMD.base
-#define GET_PWR_CFG_CMD(__PWR_CMD)			__PWR_CMD.cmd
-#define GET_PWR_CFG_MASK(__PWR_CMD)			__PWR_CMD.msk
-#define GET_PWR_CFG_VALUE(__PWR_CMD)		__PWR_CMD.value
-
-
-/*  */
-/*	Prototype of protected function. */
-/*  */
-u8 HalPwrSeqCmdParsing23a(
-	struct rtw_adapter		*padapter,
-	u8				CutVersion,
-	u8				FabVersion,
-	u8				InterfaceType,
-	struct wlan_pwr_cfg	PwrCfgCmd[]);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/HalVerDef.h b/drivers/staging/rtl8723au/include/HalVerDef.h
deleted file mode 100644
index 2a0e4ea7afad..000000000000
--- a/drivers/staging/rtl8723au/include/HalVerDef.h
+++ /dev/null
@@ -1,114 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __HAL_VERSION_DEF_H__
-#define __HAL_VERSION_DEF_H__
-
-enum hal_ic_type {
-	CHIP_8192S		=	0,
-	CHIP_8188C	=	1,
-	CHIP_8192C	=	2,
-	CHIP_8192D	=	3,
-	CHIP_8723A	=	4,
-	CHIP_8188E	=	5,
-	CHIP_8881A	=	6,
-	CHIP_8812A	=	7,
-	CHIP_8821A	=	8,
-	CHIP_8723B	=	9,
-	CHIP_8192E		=	10,
-};
-
-enum hal_chip_type {
-	TEST_CHIP		=	0,
-	NORMAL_CHIP		=	1,
-	FPGA			=	2,
-};
-
-enum hal_cut_version {
-	A_CUT_VERSION		=	0,
-	B_CUT_VERSION		=	1,
-	C_CUT_VERSION		=	2,
-	D_CUT_VERSION		=	3,
-	E_CUT_VERSION		=	4,
-	F_CUT_VERSION		=	5,
-	G_CUT_VERSION		=	6,
-};
-
-/*  HAL_Manufacturer */
-enum hal_vendor {
-	CHIP_VENDOR_TSMC	=	0,
-	CHIP_VENDOR_UMC		=	1,
-};
-
-struct hal_version {
-	enum hal_ic_type	ICType;
-	enum hal_chip_type	ChipType;
-	enum hal_cut_version	CUTVersion;
-	enum hal_vendor		VendorType;
-	u8			ROMVer;
-};
-
-/*  Get element */
-#define GET_CVID_IC_TYPE(version)	((version).ICType)
-#define GET_CVID_CHIP_TYPE(version)	((version).ChipType)
-#define GET_CVID_MANUFACTUER(version)	((version).VendorType)
-#define GET_CVID_CUT_VERSION(version)	((version).CUTVersion)
-#define GET_CVID_ROM_VERSION(version)	(((version).ROMVer) & ROM_VERSION_MASK)
-
-/* Common Macro. -- */
-
-#define IS_81XXC(version)			\
-	(((GET_CVID_IC_TYPE(version) == CHIP_8192C) ||	\
-	 (GET_CVID_IC_TYPE(version) == CHIP_8188C)) ? true : false)
-#define IS_8723_SERIES(version)			\
-	((GET_CVID_IC_TYPE(version) == CHIP_8723A) ? true : false)
-
-#define IS_TEST_CHIP(version)			\
-	((GET_CVID_CHIP_TYPE(version) == TEST_CHIP) ? true : false)
-#define IS_NORMAL_CHIP(version)			\
-	((GET_CVID_CHIP_TYPE(version) == NORMAL_CHIP) ? true : false)
-
-#define IS_A_CUT(version)			\
-	((GET_CVID_CUT_VERSION(version) == A_CUT_VERSION) ? true : false)
-#define IS_B_CUT(version)			\
-	((GET_CVID_CUT_VERSION(version) == B_CUT_VERSION) ? true : false)
-#define IS_C_CUT(version)			\
-	((GET_CVID_CUT_VERSION(version) == C_CUT_VERSION) ? true : false)
-#define IS_D_CUT(version)			\
-	((GET_CVID_CUT_VERSION(version) == D_CUT_VERSION) ? true : false)
-#define IS_E_CUT(version)			\
-	((GET_CVID_CUT_VERSION(version) == E_CUT_VERSION) ? true : false)
-
-#define IS_CHIP_VENDOR_TSMC(version)		\
-	((GET_CVID_MANUFACTUER(version) == CHIP_VENDOR_TSMC) ? true : false)
-#define IS_CHIP_VENDOR_UMC(version)		\
-	((GET_CVID_MANUFACTUER(version) == CHIP_VENDOR_UMC) ? true : false)
-
-/* Chip version Macro. -- */
-
-#define IS_81xxC_VENDOR_UMC_A_CUT(version)			\
-	(IS_81XXC(version)?(IS_CHIP_VENDOR_UMC(version) ?	\
-	(IS_A_CUT(version) ? true : false) : false) : false)
-#define IS_81xxC_VENDOR_UMC_B_CUT(version)			\
-	(IS_81XXC(version) ? (IS_CHIP_VENDOR_UMC(version) ?	\
-	 (IS_B_CUT(version) ? true : false) : false): false)
-#define IS_81xxC_VENDOR_UMC_C_CUT(version)			\
-	(IS_81XXC(version)?(IS_CHIP_VENDOR_UMC(version) ?	\
-	(IS_C_CUT(version) ? true : false) : false) : false)
-#define IS_8723A_A_CUT(version)				\
-	((IS_8723_SERIES(version)) ? (IS_A_CUT(version) ? true : false) : false)
-#define IS_8723A_B_CUT(version)					\
-	((IS_8723_SERIES(version)) ? (IS_B_CUT(version) ? true : false) : false)
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/drv_types.h b/drivers/staging/rtl8723au/include/drv_types.h
deleted file mode 100644
index e83463aeb9b1..000000000000
--- a/drivers/staging/rtl8723au/include/drv_types.h
+++ /dev/null
@@ -1,274 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-/*-----------------------------------------------------------------------------
-
-	For type defines and data structure defines
-
-------------------------------------------------------------------------------*/
-
-
-#ifndef __DRV_TYPES_H__
-#define __DRV_TYPES_H__
-
-#include <osdep_service.h>
-#include <wlan_bssdef.h>
-
-
-enum _NIC_VERSION {
-	RTL8711_NIC,
-	RTL8712_NIC,
-	RTL8713_NIC,
-	RTL8716_NIC
-
-};
-
-
-#include <rtw_ht.h>
-
-#include <rtw_cmd.h>
-#include <rtw_xmit.h>
-#include <rtw_recv.h>
-#include <hal_intf.h>
-#include <hal_com.h>
-#include <rtw_security.h>
-#include <rtw_pwrctrl.h>
-#include <rtw_io.h>
-#include <rtw_eeprom.h>
-#include <sta_info.h>
-#include <rtw_mlme.h>
-#include <rtw_debug.h>
-#include <rtw_rf.h>
-#include <rtw_event.h>
-#include <rtw_mlme_ext.h>
-#include <rtw_ap.h>
-
-#include "ioctl_cfg80211.h"
-
-struct registry_priv {
-	u8	chip_version;
-	u8	rfintfs;
-	struct	cfg80211_ssid ssid;
-	u8	channel;/* ad-hoc support requirement */
-	u8	wireless_mode;/* A, B, G, auto */
-	u8	scan_mode;/* active, passive */
-	u8	preamble;/* long, short, auto */
-	u8	vrtl_carrier_sense;/* Enable, Disable, Auto */
-	u8	vcs_type;/* RTS/CTS, CTS-to-self */
-	u16	rts_thresh;
-	u16  frag_thresh;
-	u8	adhoc_tx_pwr;
-	u8	soft_ap;
-	u8	power_mgnt;
-	u8	ips_mode;
-	u8	smart_ps;
-	u8	long_retry_lmt;
-	u8	short_retry_lmt;
-	u16	busy_thresh;
-	u8	ack_policy;
-	u8	software_encrypt;
-	u8	software_decrypt;
-	u8	acm_method;
-	  /* UAPSD */
-	u8	wmm_enable;
-	u8	uapsd_enable;
-
-	struct wlan_bssid_ex    dev_network;
-
-	u8	ht_enable;
-	u8	cbw40_enable;
-	u8	ampdu_enable;/* for tx */
-	u8	rx_stbc;
-	u8	ampdu_amsdu;/* A-MPDU Supports A-MSDU is permitted */
-	u8	lowrate_two_xmit;
-
-	u8	rf_config;
-	u8	low_power;
-
-	u8	wifi_spec;/*  !turbo_mode */
-
-	u8	channel_plan;
-#ifdef CONFIG_8723AU_BT_COEXIST
-	u8	btcoex;
-	u8	bt_iso;
-	u8	bt_sco;
-	u8	bt_ampdu;
-#endif
-	bool	bAcceptAddbaReq;
-
-	u8	antdiv_cfg;
-	u8	antdiv_type;
-
-	u8	hwpdn_mode;/* 0:disable,1:enable,2:decide by EFUSE config */
-	u8	hwpwrp_detect;/* 0:disable,1:enable */
-
-	u8	hw_wps_pbc;/* 0:disable,1:enable */
-
-	u8	max_roaming_times; /* max number driver will try to roaming */
-
-	u8 enable80211d;
-
-	u8 ifname[16];
-	u8 if2name[16];
-
-	u8 notch_filter;
-
-	u8 regulatory_tid;
-};
-
-
-#define MAX_CONTINUAL_URB_ERR 4
-
-#define GET_PRIMARY_ADAPTER(padapter)					\
-	(((struct rtw_adapter *)padapter)->dvobj->if1)
-
-enum _IFACE_ID {
-	IFACE_ID0, /* maping to PRIMARY_ADAPTER */
-	IFACE_ID1, /* maping to SECONDARY_ADAPTER */
-	IFACE_ID2,
-	IFACE_ID3,
-	IFACE_ID_MAX,
-};
-
-struct dvobj_priv {
-	struct rtw_adapter *if1; /* PRIMARY_ADAPTER */
-	struct rtw_adapter *if2; /* SECONDARY_ADAPTER */
-
-	/* for local/global synchronization */
-	struct mutex hw_init_mutex;
-	struct mutex h2c_fwcmd_mutex;
-	struct mutex setch_mutex;
-	struct mutex setbw_mutex;
-
-	unsigned char	oper_channel; /* saved chan info when set chan bw */
-	unsigned char	oper_bwmode;
-	unsigned char	oper_ch_offset;/* PRIME_CHNL_OFFSET */
-
-	struct rtw_adapter *padapters[IFACE_ID_MAX];
-	u8 iface_nums; /*  total number of ifaces used runtime */
-
-	/* For 92D, DMDP have 2 interface. */
-	u8	InterfaceNumber;
-	u8	NumInterfaces;
-
-	/* In /Out Pipe information */
-	int	RtInPipe[2];
-	int	RtOutPipe[3];
-	u8	Queue2Pipe[HW_QUEUE_ENTRY];/* for out pipe mapping */
-
-/*-------- below is for USB INTERFACE --------*/
-
-	u8	nr_endpoint;
-	u8	ishighspeed;
-	u8	RtNumInPipes;
-	u8	RtNumOutPipes;
-	int	ep_num[5]; /* endpoint number */
-
-	struct mutex usb_vendor_req_mutex;
-
-	union {
-		__le32 val32;
-		__le16 val16;
-		u8 val8;
-	} usb_buf;
-
-	struct usb_interface *pusbintf;
-	struct usb_device *pusbdev;
-	atomic_t continual_urb_error;
-
-/*-------- below is for PCIE INTERFACE --------*/
-
-};
-
-static inline struct device *dvobj_to_dev(struct dvobj_priv *dvobj)
-{
-	/* todo: get interface type from dvobj and the return the dev accordingly */
-	return &dvobj->pusbintf->dev;
-}
-
-enum _IFACE_TYPE {
-	IFACE_PORT0, /* mapping to port0 for C/D series chips */
-	IFACE_PORT1, /* mapping to port1 for C/D series chip */
-	MAX_IFACE_PORT,
-};
-
-enum _ADAPTER_TYPE {
-	PRIMARY_ADAPTER,
-	SECONDARY_ADAPTER,
-	MAX_ADAPTER,
-};
-
-struct rtw_adapter {
-	int	pid[3];/* process id from UI, 0:wps, 1:hostapd, 2:dhcpcd */
-	int	bDongle;/* build-in module or external dongle */
-	u16	chip_type;
-	u16	HardwareType;
-
-	struct dvobj_priv *dvobj;
-	struct	mlme_priv mlmepriv;
-	struct	mlme_ext_priv mlmeextpriv;
-	struct	cmd_priv	cmdpriv;
-	struct	evt_priv	evtpriv;
-	struct	xmit_priv	xmitpriv;
-	struct	recv_priv	recvpriv;
-	struct	sta_priv	stapriv;
-	struct	security_priv	securitypriv;
-	struct	registry_priv	registrypriv;
-	struct	pwrctrl_priv	pwrctrlpriv;
-	struct	eeprom_priv eeprompriv;
-
-	u32	setband;
-
-	void *HalData;
-
-	s32	bDriverStopped;
-	s32	bSurpriseRemoved;
-	s32  bCardDisableWOHSM;
-
-	u32	IsrContent;
-	u32	ImrContent;
-
-	u8	EepromAddressSize;
-	u8	hw_init_completed;
-	u8	bDriverIsGoingToUnload;
-	u8	init_adpt_in_progress;
-	u8	bHaltInProgress;
-
-	struct net_device *pnetdev;
-
-	/*  used by rtw_rereg_nd_name related function */
-	int bup;
-	struct net_device_stats stats;
-
-	struct wireless_dev *rtw_wdev;
-	int net_closed;
-
-	u8 bFWReady;
-	u8 bReadPortCancel;
-	u8 bWritePortCancel;
-
-	/* extend to support multi interface */
-	/* IFACE_ID0 is equals to PRIMARY_ADAPTER */
-	/* IFACE_ID1 is equals to SECONDARY_ADAPTER */
-	u8 iface_id;
-};
-
-#define adapter_to_dvobj(adapter) (adapter->dvobj)
-
-static inline u8 *myid(struct eeprom_priv *peepriv)
-{
-	return peepriv->mac_addr;
-}
-
-#endif /* __DRV_TYPES_H__ */
diff --git a/drivers/staging/rtl8723au/include/hal_com.h b/drivers/staging/rtl8723au/include/hal_com.h
deleted file mode 100644
index 9c50320b2100..000000000000
--- a/drivers/staging/rtl8723au/include/hal_com.h
+++ /dev/null
@@ -1,182 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __HAL_COMMON_H__
-#define __HAL_COMMON_H__
-
-/*  */
-/*        Rate Definition */
-/*  */
-/* CCK */
-#define	RATR_1M					0x00000001
-#define	RATR_2M					0x00000002
-#define	RATR_55M				0x00000004
-#define	RATR_11M				0x00000008
-/* OFDM */
-#define	RATR_6M					0x00000010
-#define	RATR_9M					0x00000020
-#define	RATR_12M				0x00000040
-#define	RATR_18M				0x00000080
-#define	RATR_24M				0x00000100
-#define	RATR_36M				0x00000200
-#define	RATR_48M				0x00000400
-#define	RATR_54M				0x00000800
-/* MCS 1 Spatial Stream */
-#define	RATR_MCS0				0x00001000
-#define	RATR_MCS1				0x00002000
-#define	RATR_MCS2				0x00004000
-#define	RATR_MCS3				0x00008000
-#define	RATR_MCS4				0x00010000
-#define	RATR_MCS5				0x00020000
-#define	RATR_MCS6				0x00040000
-#define	RATR_MCS7				0x00080000
-/* MCS 2 Spatial Stream */
-#define	RATR_MCS8				0x00100000
-#define	RATR_MCS9				0x00200000
-#define	RATR_MCS10				0x00400000
-#define	RATR_MCS11				0x00800000
-#define	RATR_MCS12				0x01000000
-#define	RATR_MCS13				0x02000000
-#define	RATR_MCS14				0x04000000
-#define	RATR_MCS15				0x08000000
-
-/* CCK */
-#define RATE_1M					BIT(0)
-#define RATE_2M					BIT(1)
-#define RATE_5_5M				BIT(2)
-#define RATE_11M				BIT(3)
-/* OFDM */
-#define RATE_6M					BIT(4)
-#define RATE_9M					BIT(5)
-#define RATE_12M				BIT(6)
-#define RATE_18M				BIT(7)
-#define RATE_24M				BIT(8)
-#define RATE_36M				BIT(9)
-#define RATE_48M				BIT(10)
-#define RATE_54M				BIT(11)
-
-/*------------------------------ Tx Desc definition Macro ------------------------*/
-/* pragma mark -- Tx Desc related definition. -- */
-/*  */
-/*  */
-/*	Rate */
-/*  */
-/*  CCK Rates, TxHT = 0 */
-#define DESC_RATE1M				0x00
-#define DESC_RATE2M				0x01
-#define DESC_RATE5_5M				0x02
-#define DESC_RATE11M				0x03
-
-/*  OFDM Rates, TxHT = 0 */
-#define DESC_RATE6M				0x04
-#define DESC_RATE9M				0x05
-#define DESC_RATE12M				0x06
-#define DESC_RATE18M				0x07
-#define DESC_RATE24M				0x08
-#define DESC_RATE36M				0x09
-#define DESC_RATE48M				0x0a
-#define DESC_RATE54M				0x0b
-
-/*  MCS Rates, TxHT = 1 */
-#define DESC_RATEMCS0				0x0c
-#define DESC_RATEMCS1				0x0d
-#define DESC_RATEMCS2				0x0e
-#define DESC_RATEMCS3				0x0f
-#define DESC_RATEMCS4				0x10
-#define DESC_RATEMCS5				0x11
-#define DESC_RATEMCS6				0x12
-#define DESC_RATEMCS7				0x13
-#define DESC_RATEMCS8				0x14
-#define DESC_RATEMCS9				0x15
-#define DESC_RATEMCS10				0x16
-#define DESC_RATEMCS11				0x17
-#define DESC_RATEMCS12				0x18
-#define DESC_RATEMCS13				0x19
-#define DESC_RATEMCS14				0x1a
-#define DESC_RATEMCS15				0x1b
-#define DESC_RATEMCS15_SG			0x1c
-#define DESC_RATEMCS32				0x20
-
-#define REG_P2P_CTWIN					0x0572 /*  1 Byte long (in unit of TU) */
-#define REG_NOA_DESC_SEL				0x05CF
-#define REG_NOA_DESC_DURATION		0x05E0
-#define REG_NOA_DESC_INTERVAL			0x05E4
-#define REG_NOA_DESC_START			0x05E8
-#define REG_NOA_DESC_COUNT			0x05EC
-
-#include "HalVerDef.h"
-
-
-u8	/* return the final channel plan decision */
-hal_com_get_channel_plan23a(
-	struct rtw_adapter	*padapter,
-	u8			hw_channel_plan,	/* channel plan from HW (efuse/eeprom) */
-	u8			sw_channel_plan,	/* channel plan from SW (registry/module param) */
-	u8			def_channel_plan,	/* channel plan used when the former two is invalid */
-	bool		AutoLoadFail
-	);
-
-u8	MRateToHwRate23a(u8 rate);
-
-void HalSetBrateCfg23a(struct rtw_adapter *padapter, u8 *mBratesOS);
-
-bool
-Hal_MappingOutPipe23a(struct rtw_adapter *pAdapter, u8 NumOutPipe);
-
-void c2h_evt_clear23a(struct rtw_adapter *adapter);
-s32 c2h_evt_read23a(struct rtw_adapter *adapter, u8 *buf);
-
-void rtl8723a_set_ampdu_min_space(struct rtw_adapter *padapter, u8 MinSpacingToSet);
-void rtl8723a_set_ampdu_factor(struct rtw_adapter *padapter, u8 FactorToSet);
-void rtl8723a_set_acm_ctrl(struct rtw_adapter *padapter, u8 ctrl);
-void rtl8723a_set_media_status(struct rtw_adapter *padapter, u8 status);
-void rtl8723a_set_media_status1(struct rtw_adapter *padapter, u8 status);
-void rtl8723a_set_bcn_func(struct rtw_adapter *padapter, u8 val);
-void rtl8723a_check_bssid(struct rtw_adapter *padapter, u8 val);
-void rtl8723a_mlme_sitesurvey(struct rtw_adapter *padapter, u8 flag);
-void rtl8723a_on_rcr_am(struct rtw_adapter *padapter);
-void rtl8723a_off_rcr_am(struct rtw_adapter *padapter);
-void rtl8723a_set_slot_time(struct rtw_adapter *padapter, u8 slottime);
-void rtl8723a_ack_preamble(struct rtw_adapter *padapter, u8 bShortPreamble);
-void rtl8723a_set_sec_cfg(struct rtw_adapter *padapter, u8 sec);
-void rtl8723a_cam_empty_entry(struct rtw_adapter *padapter, u8 ucIndex);
-void rtl8723a_cam_invalidate_all(struct rtw_adapter *padapter);
-void rtl8723a_cam_write(struct rtw_adapter *padapter,
-			u8 entry, u16 ctrl, const u8 *mac, const u8 *key);
-void rtl8723a_fifo_cleanup(struct rtw_adapter *padapter);
-void rtl8723a_set_apfm_on_mac(struct rtw_adapter *padapter, u8 val);
-void rtl8723a_bcn_valid(struct rtw_adapter *padapter);
-bool rtl8723a_get_bcn_valid(struct rtw_adapter *padapter);
-void rtl8723a_set_beacon_interval(struct rtw_adapter *padapter, u16 interval);
-void rtl8723a_set_resp_sifs(struct rtw_adapter *padapter,
-			    u8 r2t1, u8 r2t2, u8 t2t1, u8 t2t2);
-void rtl8723a_set_ac_param_vo(struct rtw_adapter *padapter, u32 vo);
-void rtl8723a_set_ac_param_vi(struct rtw_adapter *padapter, u32 vi);
-void rtl8723a_set_ac_param_be(struct rtw_adapter *padapter, u32 be);
-void rtl8723a_set_ac_param_bk(struct rtw_adapter *padapter, u32 bk);
-void rtl8723a_set_rxdma_agg_pg_th(struct rtw_adapter *padapter, u8 val);
-void rtl8723a_set_initial_gain(struct rtw_adapter *padapter, u32 rx_gain);
-
-void rtl8723a_odm_support_ability_write(struct rtw_adapter *padapter, u32 val);
-void rtl8723a_odm_support_ability_backup(struct rtw_adapter *padapter);
-void rtl8723a_odm_support_ability_restore(struct rtw_adapter *padapter);
-void rtl8723a_odm_support_ability_set(struct rtw_adapter *padapter, u32 val);
-void rtl8723a_odm_support_ability_clr(struct rtw_adapter *padapter, u32 val);
-
-void rtl8723a_set_rpwm(struct rtw_adapter *padapter, u8 val);
-u8 rtl8723a_get_rf_type(struct rtw_adapter *padapter);
-bool rtl8723a_get_fwlps_rf_on(struct rtw_adapter *padapter);
-bool rtl8723a_chk_hi_queue_empty(struct rtw_adapter *padapter);
-
-#endif /* __HAL_COMMON_H__ */
diff --git a/drivers/staging/rtl8723au/include/hal_intf.h b/drivers/staging/rtl8723au/include/hal_intf.h
deleted file mode 100644
index b924d47fcfbc..000000000000
--- a/drivers/staging/rtl8723au/include/hal_intf.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __HAL_INTF_H__
-#define __HAL_INTF_H__
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-enum _CHIP_TYPE {
-	NULL_CHIP_TYPE,
-	RTL8712_8188S_8191S_8192S,
-	RTL8188C_8192C,
-	RTL8192D,
-	RTL8723A,
-	RTL8188E,
-	MAX_CHIP_TYPE
-};
-
-enum hal_def_variable {
-	HAL_DEF_UNDERCORATEDSMOOTHEDPWDB,
-	HAL_DEF_IS_SUPPORT_ANT_DIV,
-	HAL_DEF_CURRENT_ANTENNA,
-	HAL_DEF_DRVINFO_SZ,
-	HAL_DEF_MAX_RECVBUF_SZ,
-	HAL_DEF_RX_PACKET_OFFSET,
-	HAL_DEF_DBG_DUMP_RXPKT,/* for dbg */
-	HAL_DEF_DBG_DM_FUNC,/* for dbg */
-	HAL_DEF_RA_DECISION_RATE,
-	HAL_DEF_RA_SGI,
-	HAL_DEF_PT_PWR_STATUS,
-	HW_VAR_MAX_RX_AMPDU_FACTOR,
-	HW_DEF_RA_INFO_DUMP,
-	HAL_DEF_DBG_DUMP_TXPKT,
-	HW_DEF_FA_CNT_DUMP,
-	HW_DEF_ODM_DBG_FLAG,
-};
-
-enum hal_odm_variable {
-	HAL_ODM_STA_INFO,
-	HAL_ODM_P2P_STATE,
-	HAL_ODM_WIFI_DISPLAY_STATE,
-};
-
-enum rt_eeprom_type {
-	EEPROM_93C46,
-	EEPROM_93C56,
-	EEPROM_BOOT_EFUSE,
-};
-
-
-
-#define RF_CHANGE_BY_INIT	0
-#define RF_CHANGE_BY_IPS	BIT(28)
-#define RF_CHANGE_BY_PS		BIT(29)
-#define RF_CHANGE_BY_HW		BIT(30)
-#define RF_CHANGE_BY_SW		BIT(31)
-
-enum hardware_type {
-	HARDWARE_TYPE_RTL8180,
-	HARDWARE_TYPE_RTL8185,
-	HARDWARE_TYPE_RTL8187,
-	HARDWARE_TYPE_RTL8188,
-	HARDWARE_TYPE_RTL8190P,
-	HARDWARE_TYPE_RTL8192E,
-	HARDWARE_TYPE_RTL819xU,
-	HARDWARE_TYPE_RTL8192SE,
-	HARDWARE_TYPE_RTL8192SU,
-	HARDWARE_TYPE_RTL8192CE,
-	HARDWARE_TYPE_RTL8192CU,
-	HARDWARE_TYPE_RTL8192DE,
-	HARDWARE_TYPE_RTL8192DU,
-	HARDWARE_TYPE_RTL8723AE,
-	HARDWARE_TYPE_RTL8723AU,
-	HARDWARE_TYPE_RTL8723AS,
-	HARDWARE_TYPE_RTL8188EE,
-	HARDWARE_TYPE_RTL8188EU,
-	HARDWARE_TYPE_RTL8188ES,
-	HARDWARE_TYPE_MAX,
-};
-
-#define GET_EEPROM_EFUSE_PRIV(adapter) (&adapter->eeprompriv)
-
-void rtw_hal_def_value_init23a(struct rtw_adapter *padapter);
-int pm_netdev_open23a(struct net_device *pnetdev, u8 bnormal);
-
-int rtl8723au_hal_init(struct rtw_adapter *padapter);
-int rtl8723au_hal_deinit(struct rtw_adapter *padapter);
-void rtw_hal_stop(struct rtw_adapter *padapter);
-
-void rtw_hal_update_ra_mask23a(struct sta_info *psta, u8 rssi_level);
-void	rtw_hal_clone_data(struct rtw_adapter *dst_padapter, struct rtw_adapter *src_padapter);
-
-void hw_var_set_correct_tsf(struct rtw_adapter *padapter);
-void hw_var_set_mlme_disconnect(struct rtw_adapter *padapter);
-void hw_var_set_opmode(struct rtw_adapter *padapter, u8 mode);
-void hw_var_set_macaddr(struct rtw_adapter *padapter, u8 *val);
-void hw_var_set_bssid(struct rtw_adapter *padapter, u8 *val);
-void hw_var_set_mlme_join(struct rtw_adapter *padapter, u8 type);
-
-int GetHalDefVar8192CUsb(struct rtw_adapter *Adapter,
-			 enum hal_def_variable eVariable, void *pValue);
-
-#endif /* __HAL_INTF_H__ */
diff --git a/drivers/staging/rtl8723au/include/ieee80211.h b/drivers/staging/rtl8723au/include/ieee80211.h
deleted file mode 100644
index 634102e1bda6..000000000000
--- a/drivers/staging/rtl8723au/include/ieee80211.h
+++ /dev/null
@@ -1,341 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __IEEE80211_H
-#define __IEEE80211_H
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include "linux/ieee80211.h"
-#include "wifi.h"
-
-#include <linux/wireless.h>
-
-#if (WIRELESS_EXT < 22)
-#error "Obsolete pre 2007 wireless extensions are not supported"
-#endif
-
-
-#ifdef CONFIG_8723AU_AP_MODE
-
-/* STA flags */
-#define WLAN_STA_AUTH BIT(0)
-#define WLAN_STA_ASSOC BIT(1)
-#define WLAN_STA_PS BIT(2)
-#define WLAN_STA_TIM BIT(3)
-#define WLAN_STA_PERM BIT(4)
-#define WLAN_STA_AUTHORIZED BIT(5)
-#define WLAN_STA_PENDING_POLL BIT(6) /* pending activity poll not ACKed */
-#define WLAN_STA_SHORT_PREAMBLE BIT(7)
-#define WLAN_STA_PREAUTH BIT(8)
-#define WLAN_STA_WME BIT(9)
-#define WLAN_STA_MFP BIT(10)
-#define WLAN_STA_HT BIT(11)
-#define WLAN_STA_WPS BIT(12)
-#define WLAN_STA_MAYBE_WPS BIT(13)
-#define WLAN_STA_NONERP BIT(31)
-
-#endif
-
-#define WPA_CIPHER_NONE		BIT(0)
-#define WPA_CIPHER_WEP40	BIT(1)
-#define WPA_CIPHER_WEP104	BIT(2)
-#define WPA_CIPHER_TKIP		BIT(3)
-#define WPA_CIPHER_CCMP		BIT(4)
-
-
-
-#define WPA_SELECTOR_LEN 4
-extern u8 RTW_WPA_OUI23A_TYPE[] ;
-extern u16 RTW_WPA_VERSION23A ;
-extern u8 WPA_AUTH_KEY_MGMT_NONE23A[];
-extern u8 WPA_AUTH_KEY_MGMT_UNSPEC_802_1X23A[];
-extern u8 WPA_AUTH_KEY_MGMT_PSK_OVER_802_1X23A[];
-extern u8 WPA_CIPHER_SUITE_NONE23A[];
-extern u8 WPA_CIPHER_SUITE_WEP4023A[];
-extern u8 WPA_CIPHER_SUITE_TKIP23A[];
-extern u8 WPA_CIPHER_SUITE_WRAP23A[];
-extern u8 WPA_CIPHER_SUITE_CCMP23A[];
-extern u8 WPA_CIPHER_SUITE_WEP10423A[];
-
-
-#define RSN_HEADER_LEN 4
-#define RSN_SELECTOR_LEN 4
-
-extern u8 RSN_AUTH_KEY_MGMT_UNSPEC_802_1X23A[];
-extern u8 RSN_AUTH_KEY_MGMT_PSK_OVER_802_1X23A[];
-extern u8 RSN_CIPHER_SUITE_NONE23A[];
-extern u8 RSN_CIPHER_SUITE_WEP4023A[];
-extern u8 RSN_CIPHER_SUITE_TKIP23A[];
-extern u8 RSN_CIPHER_SUITE_WRAP23A[];
-extern u8 RSN_CIPHER_SUITE_CCMP23A[];
-extern u8 RSN_CIPHER_SUITE_WEP10423A[];
-
-enum ratr_table_mode {
-	RATR_INX_WIRELESS_NGB = 0,	/*  BGN 40 Mhz 2SS 1SS */
-	RATR_INX_WIRELESS_NG = 1,	/*  GN or N */
-	RATR_INX_WIRELESS_NB = 2,	/*  BGN 20 Mhz 2SS 1SS  or BN */
-	RATR_INX_WIRELESS_N = 3,
-	RATR_INX_WIRELESS_GB = 4,
-	RATR_INX_WIRELESS_G = 5,
-	RATR_INX_WIRELESS_B = 6,
-	RATR_INX_WIRELESS_MC = 7,
-	RATR_INX_WIRELESS_AC_N = 8,
-};
-
-enum NETWORK_TYPE
-{
-	WIRELESS_INVALID = 0,
-	/* Sub-Element */
-	/*  tx: cck only , rx: cck only, hw: cck */
-	WIRELESS_11B = BIT(0),
-	/*  tx: ofdm only, rx: ofdm & cck, hw: cck & ofdm */
-	WIRELESS_11G = BIT(1),
-	/*  tx: ofdm only, rx: ofdm only, hw: ofdm only */
-	WIRELESS_11A = BIT(2),
-	/*  tx: MCS only, rx: MCS & cck, hw: MCS & cck */
-	WIRELESS_11_24N = BIT(3),
-	/*  tx: MCS only, rx: MCS & ofdm, hw: ofdm only */
-	WIRELESS_11_5N = BIT(4),
-	/* WIRELESS_AUTO		= BIT(5), */
-	WIRELESS_AC = BIT(6),
-
-	/* Combination */
-	 /*  tx: cck & ofdm, rx: cck & ofdm & MCS, hw: cck & ofdm */
-	WIRELESS_11BG = WIRELESS_11B|WIRELESS_11G,
-	/*  tx: ofdm & MCS, rx: ofdm & cck & MCS, hw: cck & ofdm */
-	WIRELESS_11G_24N = WIRELESS_11G | WIRELESS_11_24N,
-	/*  tx: ofdm & MCS, rx: ofdm & MCS, hw: ofdm only */
-	WIRELESS_11A_5N = WIRELESS_11A | WIRELESS_11_5N,
-	/*  tx: ofdm & cck & MCS, rx: ofdm & cck & MCS, hw: ofdm & cck */
-	WIRELESS_11BG_24N = WIRELESS_11B | WIRELESS_11G | WIRELESS_11_24N,
-	/*  tx: ofdm & MCS, rx: ofdm & MCS, hw: ofdm only */
-	WIRELESS_11AGN = WIRELESS_11A | WIRELESS_11G | WIRELESS_11_24N |
-				WIRELESS_11_5N,
-	WIRELESS_11ABGN = WIRELESS_11A | WIRELESS_11B | WIRELESS_11G |
-				WIRELESS_11_24N | WIRELESS_11_5N,
-};
-
-#define SUPPORTED_24G_NETTYPE_MSK (WIRELESS_11B | WIRELESS_11G | WIRELESS_11_24N)
-#define SUPPORTED_5G_NETTYPE_MSK (WIRELESS_11A | WIRELESS_11_5N)
-
-#define IsSupported24G(NetType) (NetType & SUPPORTED_24G_NETTYPE_MSK ? true : false)
-#define IsSupported5G(NetType) (NetType & SUPPORTED_5G_NETTYPE_MSK ? true : false)
-
-#define IsEnableHWCCK(NetType) IsSupported24G(NetType)
-#define IsEnableHWOFDM(NetType) (NetType & (WIRELESS_11G|WIRELESS_11_24N|SUPPORTED_5G_NETTYPE_MSK) ? true : false)
-
-#define IsSupportedRxCCK(NetType) IsEnableHWCCK(NetType)
-#define IsSupportedRxOFDM(NetType) IsEnableHWOFDM(NetType)
-#define IsSupportedRxMCS(NetType) IsEnableHWOFDM(NetType)
-
-#define IsSupportedTxCCK(NetType) (NetType & (WIRELESS_11B) ? true : false)
-#define IsSupportedTxOFDM(NetType) (NetType & (WIRELESS_11G|WIRELESS_11A) ? true : false)
-#define IsSupportedTxMCS(NetType) (NetType & (WIRELESS_11_24N|WIRELESS_11_5N) ? true : false)
-
-
-#define MIN_FRAG_THRESHOLD     256U
-#define	MAX_FRAG_THRESHOLD     2346U
-
-/* QoS,QOS */
-#define NORMAL_ACK			0
-#define NO_ACK				1
-#define NON_EXPLICIT_ACK		2
-#define BLOCK_ACK			3
-
-/* IEEE 802.11 defines */
-
-#define P80211_OUI_LEN 3
-
-struct ieee80211_snap_hdr {
-	u8    dsap;   /* always 0xAA */
-	u8    ssap;   /* always 0xAA */
-	u8    ctrl;   /* always 0x03 */
-	u8    oui[P80211_OUI_LEN];    /* organizational universal id */
-} __attribute__ ((packed));
-
-
-#define SNAP_SIZE sizeof(struct ieee80211_snap_hdr)
-
-#define WLAN_REASON_JOIN_WRONG_CHANNEL       65534
-#define WLAN_REASON_EXPIRATION_CHK 65535
-
-#define IEEE80211_CCK_RATE_LEN			4
-#define IEEE80211_NUM_OFDM_RATESLEN	8
-
-
-#define IEEE80211_CCK_RATE_1MB		        0x02
-#define IEEE80211_CCK_RATE_2MB		        0x04
-#define IEEE80211_CCK_RATE_5MB		        0x0B
-#define IEEE80211_CCK_RATE_11MB		        0x16
-#define IEEE80211_OFDM_RATE_LEN			8
-#define IEEE80211_OFDM_RATE_6MB		        0x0C
-#define IEEE80211_OFDM_RATE_9MB		        0x12
-#define IEEE80211_OFDM_RATE_12MB		0x18
-#define IEEE80211_OFDM_RATE_18MB		0x24
-#define IEEE80211_OFDM_RATE_24MB		0x30
-#define IEEE80211_OFDM_RATE_36MB		0x48
-#define IEEE80211_OFDM_RATE_48MB		0x60
-#define IEEE80211_OFDM_RATE_54MB		0x6C
-#define IEEE80211_BASIC_RATE_MASK		0x80
-
-#define IEEE80211_CCK_RATE_1MB_MASK		(1<<0)
-#define IEEE80211_CCK_RATE_2MB_MASK		(1<<1)
-#define IEEE80211_CCK_RATE_5MB_MASK		(1<<2)
-#define IEEE80211_CCK_RATE_11MB_MASK		(1<<3)
-#define IEEE80211_OFDM_RATE_6MB_MASK		(1<<4)
-#define IEEE80211_OFDM_RATE_9MB_MASK		(1<<5)
-#define IEEE80211_OFDM_RATE_12MB_MASK		(1<<6)
-#define IEEE80211_OFDM_RATE_18MB_MASK		(1<<7)
-#define IEEE80211_OFDM_RATE_24MB_MASK		(1<<8)
-#define IEEE80211_OFDM_RATE_36MB_MASK		(1<<9)
-#define IEEE80211_OFDM_RATE_48MB_MASK		(1<<10)
-#define IEEE80211_OFDM_RATE_54MB_MASK		(1<<11)
-
-#define IEEE80211_CCK_RATES_MASK	        0x0000000F
-#define IEEE80211_CCK_BASIC_RATES_MASK	(IEEE80211_CCK_RATE_1MB_MASK | \
-	IEEE80211_CCK_RATE_2MB_MASK)
-#define IEEE80211_CCK_DEFAULT_RATES_MASK	(IEEE80211_CCK_BASIC_RATES_MASK | \
-	IEEE80211_CCK_RATE_5MB_MASK | \
-	IEEE80211_CCK_RATE_11MB_MASK)
-
-#define IEEE80211_OFDM_RATES_MASK		0x00000FF0
-#define IEEE80211_OFDM_BASIC_RATES_MASK	(IEEE80211_OFDM_RATE_6MB_MASK | \
-	IEEE80211_OFDM_RATE_12MB_MASK | \
-	IEEE80211_OFDM_RATE_24MB_MASK)
-#define IEEE80211_OFDM_DEFAULT_RATES_MASK	(IEEE80211_OFDM_BASIC_RATES_MASK | \
-	IEEE80211_OFDM_RATE_9MB_MASK  | \
-	IEEE80211_OFDM_RATE_18MB_MASK | \
-	IEEE80211_OFDM_RATE_36MB_MASK | \
-	IEEE80211_OFDM_RATE_48MB_MASK | \
-	IEEE80211_OFDM_RATE_54MB_MASK)
-#define IEEE80211_DEFAULT_RATES_MASK (IEEE80211_OFDM_DEFAULT_RATES_MASK | \
-				      IEEE80211_CCK_DEFAULT_RATES_MASK)
-
-#define IEEE80211_NUM_OFDM_RATES	    8
-#define IEEE80211_NUM_CCK_RATES	            4
-#define IEEE80211_OFDM_SHIFT_MASK_A         4
-
-#define WEP_KEYS 4
-
-
-/* MAX_RATES_LENGTH needs to be 12.  The spec says 8, and many APs
- * only use 8, and then use extended rates for the remaining supported
- * rates.  Other APs, however, stick all of their supported rates on the
- * main rates information element... */
-#define MAX_RATES_LENGTH	12
-#define MAX_RATES_EX_LENGTH	16
-#define MAX_CHANNEL_NUMBER	161
-#define RTW_CH_MAX_2G_CHANNEL	14	/* Max channel in 2G band */
-
-#define MAX_WPA_IE_LEN		256
-#define MAX_WPS_IE_LEN		256
-#define MAX_P2P_IE_LEN		256
-#define MAX_WFD_IE_LEN		128
-
-/*
-join_res:
--1: authentication fail
--2: association fail
-> 0: TID
-*/
-
-#define MAXTID	16
-
-#define WME_OUI_TYPE 2
-#define WME_OUI_SUBTYPE_INFORMATION_ELEMENT 0
-#define WME_OUI_SUBTYPE_PARAMETER_ELEMENT 1
-#define WME_OUI_SUBTYPE_TSPEC_ELEMENT 2
-#define WME_VERSION 1
-
-
-#define OUI_BROADCOM 0x00904c /* Broadcom (Epigram) */
-
-#define VENDOR_HT_CAPAB_OUI_TYPE 0x33 /* 00-90-4c:0x33 */
-
-/* Represent channel details, subset of ieee80211_channel */
-struct rtw_ieee80211_channel {
-	/* enum nl80211_band band; */
-	/* u16 center_freq; */
-	u16 hw_value;
-	u32 flags;
-	/* int max_antenna_gain; */
-	/* int max_power; */
-	/* int max_reg_power; */
-	/* bool beacon_found; */
-	/* u32 orig_flags; */
-	/* int orig_mag; */
-	/* int orig_mpwr; */
-};
-
-#define CHAN_FMT \
-	/*"band:%d, "*/ \
-	/*"center_freq:%u, "*/ \
-	"hw_value:%u, " \
-	"flags:0x%08x" \
-	/*"max_antenna_gain:%d\n"*/ \
-	/*"max_power:%d\n"*/ \
-	/*"max_reg_power:%d\n"*/ \
-	/*"beacon_found:%u\n"*/ \
-	/*"orig_flags:0x%08x\n"*/ \
-	/*"orig_mag:%d\n"*/ \
-	/*"orig_mpwr:%d\n"*/
-
-#define CHAN_ARG(channel) \
-	/*(channel)->band*/ \
-	/*, (channel)->center_freq*/ \
-	(channel)->hw_value \
-	, (channel)->flags \
-	/*, (channel)->max_antenna_gain*/ \
-	/*, (channel)->max_power*/ \
-	/*, (channel)->max_reg_power*/ \
-	/*, (channel)->beacon_found*/ \
-	/*, (channel)->orig_flags*/ \
-	/*, (channel)->orig_mag*/ \
-	/*, (channel)->orig_mpwr*/ \
-
-u8 *rtw_set_ie23a(u8 *pbuf, int index, uint len, const u8 *source, uint *frlen);
-
-u8 hal_ch_offset_to_secondary_ch_offset23a(u8 ch_offset);
-u8 *rtw_set_ie23a_ch_switch(u8 *buf, u32 *buf_len, u8 ch_switch_mode, u8 new_ch, u8 ch_switch_cnt);
-u8 *rtw_set_ie23a_secondary_ch_offset(u8 *buf, u32 *buf_len, u8 secondary_ch_offset);
-
-u8 *rtw_get_ie23a(u8*pbuf, int index, int *len, int limit);
-u8 *rtw_get_ie23a_ex(u8 *in_ie, uint in_len, u8 eid, u8 *oui, u8 oui_len, u8 *ie, uint *ielen);
-int rtw_ies_remove_ie23a(u8 *ies, uint *ies_len, uint offset, u8 eid, u8 *oui, u8 oui_len);
-
-void rtw_set_supported_rate23a(u8 *SupportedRates, uint mode);
-
-int rtw_parse_wpa_ie23a(const u8* wpa_ie, int wpa_ie_len, int *group_cipher, int *pairwise_cipher, int *is_8021x);
-int rtw_parse_wpa2_ie23a(const u8* wpa_ie, int wpa_ie_len, int *group_cipher, int *pairwise_cipher, int *is_8021x);
-
-const u8 *rtw_get_wps_attr23a(const u8 *wps_ie, uint wps_ielen, u16 target_attr_id ,u8 *buf_attr, u32 *len_attr);
-const u8 *rtw_get_wps_attr_content23a(const u8 *wps_ie, uint wps_ielen, u16 target_attr_id ,u8 *buf_content);
-
-uint	rtw_get_rateset_len23a(u8	*rateset);
-
-struct registry_priv;
-int rtw_generate_ie23a(struct registry_priv *pregistrypriv);
-
-
-int rtw_get_bit_value_from_ieee_value23a(u8 val);
-
-int rtw_check_network_type23a(unsigned char *rate, int ratelen, int channel);
-
-void rtw_get_bcn_info23a(struct wlan_network *pnetwork);
-
-u16 rtw_mcs_rate23a(u8 rf_type, u8 bw_40MHz, u8 short_GI_20, u8 short_GI_40,
-		    struct ieee80211_mcs_info *mcs);
-
-#endif /* IEEE80211_H */
diff --git a/drivers/staging/rtl8723au/include/ioctl_cfg80211.h b/drivers/staging/rtl8723au/include/ioctl_cfg80211.h
deleted file mode 100644
index 3a4ead54f948..000000000000
--- a/drivers/staging/rtl8723au/include/ioctl_cfg80211.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __IOCTL_CFG80211_H__
-#define __IOCTL_CFG80211_H__
-
-struct rtw_wdev_priv {
-	struct wireless_dev *rtw_wdev;
-
-	struct rtw_adapter *padapter;
-
-	struct cfg80211_scan_request *scan_request;
-	spinlock_t scan_req_lock;
-
-	struct net_device *pmon_ndev;/* for monitor interface */
-	char ifname_mon[IFNAMSIZ + 1]; /* name for monitor interface */
-
-	u8 p2p_enabled;
-
-	bool power_mgmt;
-};
-
-#define wdev_to_priv(w) ((struct rtw_wdev_priv *)(wdev_priv(w)))
-
-#define wiphy_to_adapter(x)					\
-	(struct rtw_adapter *)(((struct rtw_wdev_priv *)	\
-	wiphy_priv(x))->padapter)
-
-#define wiphy_to_wdev(x)					\
-	(struct wireless_dev *)(((struct rtw_wdev_priv *)	\
-	wiphy_priv(x))->rtw_wdev)
-
-int rtw_wdev_alloc(struct rtw_adapter *padapter, struct device *dev);
-void rtw_wdev_free(struct wireless_dev *wdev);
-void rtw_wdev_unregister(struct wireless_dev *wdev);
-
-void rtw_cfg80211_init_wiphy(struct rtw_adapter *padapter);
-
-void rtw_cfg80211_surveydone_event_callback(struct rtw_adapter *padapter);
-
-void rtw_cfg80211_indicate_connect(struct rtw_adapter *padapter);
-void rtw_cfg80211_indicate_disconnect(struct rtw_adapter *padapter);
-void rtw_cfg80211_indicate_scan_done(struct rtw_wdev_priv *pwdev_priv,
-				     bool aborted);
-
-#ifdef CONFIG_8723AU_AP_MODE
-void rtw_cfg80211_indicate_sta_assoc(struct rtw_adapter *padapter,
-				     u8 *pmgmt_frame, uint frame_len);
-void rtw_cfg80211_indicate_sta_disassoc(struct rtw_adapter *padapter,
-					unsigned char *da, unsigned short reason);
-#endif /* CONFIG_8723AU_AP_MODE */
-
-bool rtw_cfg80211_pwr_mgmt(struct rtw_adapter *adapter);
-
-#endif /* __IOCTL_CFG80211_H__ */
diff --git a/drivers/staging/rtl8723au/include/mlme_osdep.h b/drivers/staging/rtl8723au/include/mlme_osdep.h
deleted file mode 100644
index 4bb5525b7a68..000000000000
--- a/drivers/staging/rtl8723au/include/mlme_osdep.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef	__MLME_OSDEP_H_
-#define __MLME_OSDEP_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-void rtw_os_indicate_disconnect23a(struct rtw_adapter *adapter);
-void rtw_reset_securitypriv23a(struct rtw_adapter *adapter);
-
-#endif	/* _MLME_OSDEP_H_ */
diff --git a/drivers/staging/rtl8723au/include/odm.h b/drivers/staging/rtl8723au/include/odm.h
deleted file mode 100644
index 24f2f28c473f..000000000000
--- a/drivers/staging/rtl8723au/include/odm.h
+++ /dev/null
@@ -1,860 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-
-#ifndef	__HALDMOUTSRC_H__
-#define __HALDMOUTSRC_H__
-
-/*  */
-/*  Definition */
-/*  */
-/*  */
-/*  2011/09/22 MH Define all team supprt ability. */
-/*  */
-
-/*  */
-/*  2011/09/22 MH Define for all teams. Please Define the constan in your precomp header. */
-/*  */
-/* define		DM_ODM_SUPPORT_AP			0 */
-/* define		DM_ODM_SUPPORT_ADSL			0 */
-/* define		DM_ODM_SUPPORT_CE			0 */
-/* define		DM_ODM_SUPPORT_MP			1 */
-
-#define	TP_MODE		0
-#define	RSSI_MODE		1
-#define	TRAFFIC_LOW	0
-#define	TRAFFIC_HIGH	1
-
-
-/*  */
-/* 3 Tx Power Tracking */
-/* 3============================================================ */
-#define		DPK_DELTA_MAPPING_NUM	13
-#define		index_mapping_HP_NUM	15
-
-
-/*  */
-/* 3 PSD Handler */
-/* 3============================================================ */
-
-#define	AFH_PSD		1	/* 0:normal PSD scan, 1: only do 20 pts PSD */
-#define	MODE_40M		0	/* 0:20M, 1:40M */
-#define	PSD_TH2		3
-#define	PSD_CHMIN		20   /*  Minimum channel number for BT AFH */
-#define	SIR_STEP_SIZE	3
-#define   Smooth_Size_1		5
-#define	Smooth_TH_1	3
-#define   Smooth_Size_2		10
-#define	Smooth_TH_2	4
-#define   Smooth_Size_3		20
-#define	Smooth_TH_3	4
-#define   Smooth_Step_Size 5
-#define	Adaptive_SIR	1
-#define	PSD_RESCAN		4
-#define	PSD_SCAN_INTERVAL	700 /* ms */
-
-/* 8723A High Power IGI Setting */
-#define DM_DIG_HIGH_PWR_IGI_LOWER_BOUND	0x22
-#define DM_DIG_Gmode_HIGH_PWR_IGI_LOWER_BOUND 0x28
-#define DM_DIG_HIGH_PWR_THRESHOLD	0x3a
-
-/*  LPS define */
-#define DM_DIG_FA_TH0_LPS				4 /*  4 in lps */
-#define DM_DIG_FA_TH1_LPS				15 /*  15 lps */
-#define DM_DIG_FA_TH2_LPS				30 /*  30 lps */
-#define RSSI_OFFSET_DIG					0x05;
-
-/* ANT Test */
-#define			ANTTESTALL		0x00		/* Ant A or B will be Testing */
-#define		ANTTESTA		0x01		/* Ant A will be Testing */
-#define		ANTTESTB		0x02		/* Ant B will be testing */
-
-
-/*  */
-/*  structure and define */
-/*  */
-
-struct  dig_t {
-	u8		Dig_Enable_Flag;
-	u8		Dig_Ext_Port_Stage;
-
-	int		RssiLowThresh;
-	int		RssiHighThresh;
-
-	u32		FALowThresh;
-	u32		FAHighThresh;
-
-	u8		CurSTAConnectState;
-	u8		PreSTAConnectState;
-	u8		CurMultiSTAConnectState;
-
-	u8		PreIGValue;
-	u8		CurIGValue;
-	u8		BackupIGValue;
-
-	s8		BackoffVal;
-	s8		BackoffVal_range_max;
-	s8		BackoffVal_range_min;
-	u8		rx_gain_range_max;
-	u8		rx_gain_range_min;
-	u8		Rssi_val_min;
-
-	u8		PreCCK_CCAThres;
-	u8		CurCCK_CCAThres;
-	u8		PreCCKPDState;
-	u8		CurCCKPDState;
-
-	u8		LargeFAHit;
-	u8		ForbiddenIGI;
-	u32		Recover_cnt;
-
-	u8		DIG_Dynamic_MIN_0;
-	u8		DIG_Dynamic_MIN_1;
-	bool		bMediaConnect_0;
-	bool		bMediaConnect_1;
-
-	u32		RSSI_max;
-};
-
-struct dynamic_pwr_sav {
-	u8		PreCCAState;
-	u8		CurCCAState;
-
-	u8		PreRFState;
-	u8		CurRFState;
-
-	int		    Rssi_val_min;
-
-	u8		initialize;
-	u32		Reg874, RegC70, Reg85C, RegA74;
-};
-
-struct false_alarm_stats {
-	u32	Cnt_Parity_Fail;
-	u32	Cnt_Rate_Illegal;
-	u32	Cnt_Crc8_fail;
-	u32	Cnt_Mcs_fail;
-	u32	Cnt_Ofdm_fail;
-	u32	Cnt_Cck_fail;
-	u32	Cnt_all;
-	u32	Cnt_Fast_Fsync;
-	u32	Cnt_SB_Search_fail;
-	u32	Cnt_OFDM_CCA;
-	u32	Cnt_CCK_CCA;
-	u32	Cnt_CCA_all;
-	u32	Cnt_BW_USC;	/* Gary */
-	u32	Cnt_BW_LSC;	/* Gary */
-};
-
-#define ASSOCIATE_ENTRY_NUM					32 /*  Max size of AsocEntry[]. */
-#define	ODM_ASSOCIATE_ENTRY_NUM				ASSOCIATE_ENTRY_NUM
-
-/*  This indicates two different the steps. */
-/*  In SWAW_STEP_PEAK, driver needs to switch antenna and listen to the signal on the air. */
-/*  In SWAW_STEP_DETERMINE, driver just compares the signal captured in SWAW_STEP_PEAK */
-/*  with original RSSI to determine if it is necessary to switch antenna. */
-#define SWAW_STEP_PEAK		0
-#define SWAW_STEP_DETERMINE	1
-
-#define	TP_MODE		0
-#define	RSSI_MODE		1
-#define	TRAFFIC_LOW	0
-#define	TRAFFIC_HIGH	1
-
-struct sw_ant_sw {
-	u8		try_flag;
-	s32		PreRSSI;
-	u8		CurAntenna;
-	u8		PreAntenna;
-	u8		RSSI_Trying;
-	u8		TestMode;
-	u8		bTriggerAntennaSwitch;
-	u8		SelectAntennaMap;
-	u8		RSSI_target;
-
-	/*  Before link Antenna Switch check */
-	u8		SWAS_NoLink_State;
-	u32		SWAS_NoLink_BK_Reg860;
-	bool		ANTA_ON;	/* To indicate Ant A is or not */
-	bool		ANTB_ON;	/* To indicate Ant B is on or not */
-
-	s32		RSSI_sum_A;
-	s32		RSSI_sum_B;
-	s32		RSSI_cnt_A;
-	s32		RSSI_cnt_B;
-
-	u64		lastTxOkCnt;
-	u64		lastRxOkCnt;
-	u64		TXByteCnt_A;
-	u64		TXByteCnt_B;
-	u64		RXByteCnt_A;
-	u64		RXByteCnt_B;
-	u8		TrafficLoad;
-};
-
-struct edca_turbo {
-	bool bCurrentTurboEDCA;
-	u32	prv_traffic_idx; /*  edca turbo */
-};
-
-struct odm_rate_adapt {
-	u8	Type;		/*  DM_Type_ByFW/DM_Type_ByDriver */
-	u8	HighRSSIThresh;	/*  if RSSI > HighRSSIThresh	=> RATRState is DM_RATR_STA_HIGH */
-	u8	LowRSSIThresh;	/*  if RSSI <= LowRSSIThresh	=> RATRState is DM_RATR_STA_LOW */
-	u8	RATRState;	/*  Current RSSI level, DM_RATR_STA_HIGH/DM_RATR_STA_MIDDLE/DM_RATR_STA_LOW */
-	u32	LastRATR;	/*  RATR Register Content */
-};
-
-#define IQK_MAC_REG_NUM		4
-#define IQK_ADDA_REG_NUM		16
-#define IQK_BB_REG_NUM_MAX	10
-#define IQK_BB_REG_NUM		9
-#define HP_THERMAL_NUM		8
-
-#define AVG_THERMAL_NUM		8
-#define IQK_Matrix_REG_NUM	8
-#define IQK_Matrix_Settings_NUM	1+24+21
-
-#define		DM_Type_ByFW			0
-#define		DM_Type_ByDriver		1
-
-/*  Declare for common info */
-
-struct odm_phy_dbg_info {
-	/* ODM Write,debug info */
-	s8		RxSNRdB[RF_PATH_MAX];
-	u64		NumQryPhyStatus;
-	u64		NumQryPhyStatusCCK;
-	u64		NumQryPhyStatusOFDM;
-	/* Others */
-	s32		RxEVM[RF_PATH_MAX];
-
-};
-
-struct odm_packet_info {
-	u8		Rate;
-	u8		StationID;
-	bool		bPacketMatchBSSID;
-	bool		bPacketToSelf;
-	bool		bPacketBeacon;
-};
-
-
-enum {
-	/*  BB Team */
-	ODM_DIG			= 0x00000001,
-	ODM_HIGH_POWER		= 0x00000002,
-	ODM_CCK_CCA_TH		= 0x00000004,
-	ODM_FA_STATISTICS	= 0x00000008,
-	ODM_RAMASK		= 0x00000010,
-	ODM_RSSI_MONITOR	= 0x00000020,
-	ODM_SW_ANTDIV		= 0x00000040,
-	ODM_HW_ANTDIV		= 0x00000080,
-	ODM_BB_PWRSV		= 0x00000100,
-	ODM_2TPATHDIV		= 0x00000200,
-	ODM_1TPATHDIV		= 0x00000400,
-	ODM_PSD2AFH		= 0x00000800
-};
-
-/*  */
-/*  2011/10/20 MH Define Common info enum for all team. */
-/*  */
-
-enum odm_cmninfo {
-	/*  Fixed value: */
-	/*  */
-
-	ODM_CMNINFO_MP_TEST_CHIP = 2,
-	ODM_CMNINFO_IC_TYPE,			/*  enum odm_ic_type_def */
-	ODM_CMNINFO_CUT_VER,			/*  enum odm_cut_version */
-	ODM_CMNINFO_FAB_VER,			/*  enum odm_fab_version */
-	ODM_CMNINFO_BOARD_TYPE,			/*  enum odm_board_type */
-	ODM_CMNINFO_EXT_LNA,			/*  true */
-	ODM_CMNINFO_EXT_PA,
-	ODM_CMNINFO_EXT_TRSW,
-	ODM_CMNINFO_BINHCT_TEST,
-	ODM_CMNINFO_BWIFI_TEST,
-	ODM_CMNINFO_SMART_CONCURRENT,
-
-
-	/*  */
-	/*  Dynamic value: */
-	/*  */
-	ODM_CMNINFO_MP_MODE,
-
-	ODM_CMNINFO_WIFI_DIRECT,
-	ODM_CMNINFO_WIFI_DISPLAY,
-	ODM_CMNINFO_LINK,
-	ODM_CMNINFO_RSSI_MIN,
-	ODM_CMNINFO_DBG_COMP,				/*  u64 */
-	ODM_CMNINFO_DBG_LEVEL,				/*  u32 */
-	ODM_CMNINFO_RA_THRESHOLD_HIGH,		/*  u8 */
-	ODM_CMNINFO_RA_THRESHOLD_LOW,		/*  u8 */
-	ODM_CMNINFO_RF_ANTENNA_TYPE,		/*  u8 */
-	ODM_CMNINFO_BT_DISABLED,
-	ODM_CMNINFO_BT_OPERATION,
-	ODM_CMNINFO_BT_DIG,
-	ODM_CMNINFO_BT_BUSY,					/* Check Bt is using or not */
-	ODM_CMNINFO_BT_DISABLE_EDCA,
-
-	/*  */
-	/*  Dynamic ptr array hook itms. */
-	/*  */
-	ODM_CMNINFO_STA_STATUS,
-	ODM_CMNINFO_PHY_STATUS,
-	ODM_CMNINFO_MAC_STATUS,
-
-	ODM_CMNINFO_MAX,
-};
-
-/*  Define ODM support ability.  ODM_CMNINFO_ABILITY */
-enum {
-	/*  BB ODM section BIT 0-15 */
-	ODM_BB_ANT_DIV				= BIT(6),
-};
-
-/*	ODM_CMNINFO_INTERFACE */
-enum odm_interface_def {
-	ODM_ITRF_PCIE	=	0x1,
-	ODM_ITRF_USB	=	0x2,
-	ODM_ITRF_SDIO	=	0x4,
-	ODM_ITRF_ALL	=	0x7,
-};
-
-/*  ODM_CMNINFO_IC_TYPE */
-enum odm_ic_type_def {
-	ODM_RTL8192S	=	BIT(0),
-	ODM_RTL8192C	=	BIT(1),
-	ODM_RTL8192D	=	BIT(2),
-	ODM_RTL8723A	=	BIT(3),
-	ODM_RTL8188E	=	BIT(4),
-	ODM_RTL8812	=	BIT(5),
-	ODM_RTL8821	=	BIT(6),
-};
-
-/* ODM_CMNINFO_CUT_VER */
-enum odm_cut_version {
-	ODM_CUT_A		=	1,
-	ODM_CUT_B		=	2,
-	ODM_CUT_C		=	3,
-	ODM_CUT_D		=	4,
-	ODM_CUT_E		=	5,
-	ODM_CUT_F		=	6,
-	ODM_CUT_TEST		=	7,
-};
-
-/*  ODM_CMNINFO_FAB_VER */
-enum odm_fab_version {
-	ODM_TSMC	=	0,
-	ODM_UMC		=	1,
-};
-
-/*  For example 1T2R (A+AB = BIT0|BIT4|BIT5) */
-enum rf_path_def {
-	ODM_RF_TX_A	=	BIT(0),
-	ODM_RF_TX_B	=	BIT(1),
-	ODM_RF_TX_C	=	BIT(2),
-	ODM_RF_TX_D	=	BIT(3),
-	ODM_RF_RX_A	=	BIT(4),
-	ODM_RF_RX_B	=	BIT(5),
-	ODM_RF_RX_C	=	BIT(6),
-	ODM_RF_RX_D	=	BIT(7),
-};
-
-/*  ODM Dynamic common info value definition */
-
-enum odm_mac_phy_mode {
-	ODM_SMSP	= 0,
-	ODM_DMSP	= 1,
-	ODM_DMDP	= 2,
-};
-
-
-enum odm_bt_coexist {
-	ODM_BT_BUSY		= 1,
-	ODM_BT_ON		= 2,
-	ODM_BT_OFF		= 3,
-	ODM_BT_NONE		= 4,
-};
-
-/*  ODM_CMNINFO_OP_MODE */
-enum odm_operation_mode {
-	ODM_NO_LINK		= BIT(0),
-	ODM_LINK		= BIT(1),
-	ODM_SCAN		= BIT(2),
-	ODM_POWERSAVE		= BIT(3),
-	ODM_AP_MODE		= BIT(4),
-	ODM_CLIENT_MODE		= BIT(5),
-	ODM_AD_HOC		= BIT(6),
-	ODM_WIFI_DIRECT		= BIT(7),
-	ODM_WIFI_DISPLAY	= BIT(8),
-};
-
-/*  ODM_CMNINFO_WM_MODE */
-enum odm_wireless_mode {
-	ODM_WM_UNKNOW		= 0x0,
-	ODM_WM_B		= BIT(0),
-	ODM_WM_G		= BIT(1),
-	ODM_WM_A		= BIT(2),
-	ODM_WM_N24G		= BIT(3),
-	ODM_WM_N5G		= BIT(4),
-	ODM_WM_AUTO		= BIT(5),
-	ODM_WM_AC		= BIT(6),
-};
-
-/*  ODM_CMNINFO_BAND */
-enum odm_band_type {
-	ODM_BAND_2_4G		= BIT(0),
-	ODM_BAND_5G		= BIT(1),
-
-};
-
-/*  ODM_CMNINFO_SEC_CHNL_OFFSET */
-enum odm_sec_chnl_offset {
-	ODM_DONT_CARE		= 0,
-	ODM_BELOW		= 1,
-	ODM_ABOVE		= 2
-};
-
-/*  ODM_CMNINFO_CHNL */
-
-/*  ODM_CMNINFO_BOARD_TYPE */
-enum odm_board_type {
-	ODM_BOARD_NORMAL	= 0,
-	ODM_BOARD_HIGHPWR	= 1,
-	ODM_BOARD_MINICARD	= 2,
-	ODM_BOARD_SLIM		= 3,
-	ODM_BOARD_COMBO		= 4,
-
-};
-
-/*  ODM_CMNINFO_ONE_PATH_CCA */
-enum odm_cca_path {
-	ODM_CCA_2R			= 0,
-	ODM_CCA_1R_A			= 1,
-	ODM_CCA_1R_B			= 2,
-};
-
-struct iqk_matrix_regs_set {
-	bool	bIQKDone;
-	s32	Value[1][IQK_Matrix_REG_NUM];
-};
-
-struct odm_rf_cal_t {
-	/* for tx power tracking */
-
-	u32	RegA24; /*  for TempCCK */
-	s32	RegE94;
-	s32	RegE9C;
-	s32	RegEB4;
-	s32	RegEBC;
-
-	/* u8 bTXPowerTracking; */
-	u8		TXPowercount;
-	bool bTXPowerTrackingInit;
-	bool bTXPowerTracking;
-	u8		TxPowerTrackControl; /* for mp mode, turn off txpwrtracking as default */
-	u8		TM_Trigger;
-	u8		InternalPA5G[2];	/* pathA / pathB */
-
-	u8		ThermalMeter[2];    /*  ThermalMeter, index 0 for RFIC0, and 1 for RFIC1 */
-	u8		ThermalValue;
-	u8		ThermalValue_LCK;
-	u8		ThermalValue_IQK;
-	u8	ThermalValue_DPK;
-	u8	ThermalValue_AVG[AVG_THERMAL_NUM];
-	u8	ThermalValue_AVG_index;
-	u8	ThermalValue_RxGain;
-	u8	ThermalValue_Crystal;
-	u8	ThermalValue_DPKstore;
-	u8	ThermalValue_DPKtrack;
-	bool	TxPowerTrackingInProgress;
-	bool	bDPKenable;
-
-	bool	bReloadtxpowerindex;
-	u8	bRfPiEnable;
-	u32	TXPowerTrackingCallbackCnt; /* cosa add for debug */
-
-	u8	bCCKinCH14;
-	u8	CCK_index;
-	u8	OFDM_index[2];
-	bool bDoneTxpower;
-
-	u8	ThermalValue_HP[HP_THERMAL_NUM];
-	u8	ThermalValue_HP_index;
-	struct iqk_matrix_regs_set IQKMatrixRegSetting[IQK_Matrix_Settings_NUM];
-
-	u8	Delta_IQK;
-	u8	Delta_LCK;
-
-	/* for IQK */
-	u32	RegC04;
-	u32	Reg874;
-	u32	RegC08;
-	u32	RegB68;
-	u32	RegB6C;
-	u32	Reg870;
-	u32	Reg860;
-	u32	Reg864;
-
-	bool	bIQKInitialized;
-	bool bLCKInProgress;
-	bool	bAntennaDetected;
-	u32	ADDA_backup[IQK_ADDA_REG_NUM];
-	u32	IQK_MAC_backup[IQK_MAC_REG_NUM];
-	u32	IQK_BB_backup_recover[9];
-	u32	IQK_BB_backup[IQK_BB_REG_NUM];
-
-	/* for APK */
-	u32	APKoutput[2][2]; /* path A/B; output1_1a/output1_2a */
-	u8	bAPKdone;
-	u8	bAPKThermalMeterIgnore;
-	u8	bDPdone;
-	u8	bDPPathAOK;
-	u8	bDPPathBOK;
-};
-
-enum ant_dif_type {
-	NO_ANTDIV			= 0xFF,
-	CG_TRX_HW_ANTDIV		= 0x01,
-	CGCS_RX_HW_ANTDIV		= 0x02,
-	FIXED_HW_ANTDIV			= 0x03,
-	CG_TRX_SMART_ANTDIV		= 0x04,
-	CGCS_RX_SW_ANTDIV		= 0x05,
-};
-
-/*  2011/09/22 MH Copy from SD4 defined structure. We use to support PHY DM integration. */
-struct dm_odm_t {
-	/*  */
-	/*	Add for different team use temporarily */
-	/*  */
-	struct rtw_adapter	*Adapter;		/*  For CE/NIC team */
-
-	u64			DebugComponents;
-	u32			DebugLevel;
-
-/*  ODM HANDLE, DRIVER NEEDS NOT TO HOOK------ */
-	bool			bCckHighPower;
-	u8			RFPathRxEnable;		/*  ODM_CMNINFO_RFPATH_ENABLE */
-/*  ODM HANDLE, DRIVER NEEDS NOT TO HOOK------ */
-
-/* 1  COMMON INFORMATION */
-
-	/*  Init Value */
-/* HOOK BEFORE REG INIT----------- */
-	/*  ODM Support Ability DIG/RATR/TX_PWR_TRACK/ ¡K¡K = 1/2/3/¡K */
-	u32			SupportAbility;
-	/*  ODM composite or independent. Bit oriented/ 92C+92D+ .... or any other type = 1/2/3/... */
-	u32			SupportICType;
-	/*  Cut Version TestChip/A-cut/B-cut... = 0/1/2/3/... */
-	u8			CutVersion;
-	/*  Fab Version TSMC/UMC = 0/1 */
-	u8			FabVersion;
-	/*  Board Type Normal/HighPower/MiniCard/SLIM/Combo/... = 0/1/2/3/4/... */
-	u8			BoardType;
-	/*  with external LNA  NO/Yes = 0/1 */
-	u8			ExtLNA;
-	/*  with external PA  NO/Yes = 0/1 */
-	u8			ExtPA;
-	/*  with external TRSW  NO/Yes = 0/1 */
-	u8			ExtTRSW;
-	bool			bInHctTest;
-	bool			bWIFITest;
-
-	bool			bDualMacSmartConcurrent;
-	u32			BK_SupportAbility;
-/* HOOK BEFORE REG INIT----------- */
-
-	/*  */
-	/*  Dynamic Value */
-	/*  */
-/*  POINTER REFERENCE----------- */
-
-	u8			u8_temp;
-	bool			bool_temp;
-	struct rtw_adapter	*PADAPTER_temp;
-
-/*  POINTER REFERENCE----------- */
-	/*  */
-/* CALL BY VALUE------------- */
-	bool			bWIFI_Direct;
-	bool			bWIFI_Display;
-	bool			bLinked;
-	u8			RSSI_Min;
-	u8			InterfaceIndex; /*  Add for 92D  dual MAC: 0--Mac0 1--Mac1 */
-	bool		bIsMPChip;
-	bool			bOneEntryOnly;
-	/*  Common info for BTDM */
-	bool			bBtDisabled;			/*  BT is disabled */
-	bool			bBtHsOperation;		/*  BT HS mode is under progress */
-	u8			btHsDigVal;			/*  use BT rssi to decide the DIG value */
-	bool			bBtDisableEdcaTurbo;	/*  Under some condition, don't enable the EDCA Turbo */
-	bool			bBtBusy;			/*  BT is busy. */
-/* CALL BY VALUE------------- */
-
-	/* 2 Define STA info. */
-	/*  _ODM_STA_INFO */
-	/*  2012/01/12 MH For MP, we need to reduce one array pointer for default port.?? */
-	struct sta_info *		pODM_StaInfo[ODM_ASSOCIATE_ENTRY_NUM];
-
-	/*  Latest packet phy info (ODM write) */
-	struct odm_phy_dbg_info	 PhyDbgInfo;
-	/* PHY_INFO_88E		PhyInfo; */
-
-	/*  Latest packet phy info (ODM write) */
-	/* MAC_INFO_88E		MacInfo; */
-
-	/*  Different Team independt structure?? */
-
-	/*  */
-	/* TX_RTP_CMN		TX_retrpo; */
-	/* TX_RTP_88E		TX_retrpo; */
-	/* TX_RTP_8195		TX_retrpo; */
-
-	/*  */
-	/* ODM Structure */
-	/*  */
-	struct dig_t	DM_DigTable;
-	struct dynamic_pwr_sav		DM_PSTable;
-	struct false_alarm_stats	FalseAlmCnt;
-	struct false_alarm_stats	FlaseAlmCntBuddyAdapter;
-	struct sw_ant_sw		DM_SWAT_Table;
-
-	struct edca_turbo		DM_EDCA_Table;
-	u32		WMMEDCA_BE;
-	/*  Copy from SD4 structure */
-	/*  */
-	/*  ================================================== */
-	/*  */
-
-	/* PSD */
-	u8			RSSI_BT;		/* come from BT */
-	struct odm_rate_adapt	RateAdaptive;
-
-
-	struct odm_rf_cal_t	RFCalibrateInfo;
-};	/*  DM_Dynamic_Mechanism_Structure */
-
-enum odm_rf_content {
-	odm_radioa_txt = 0x1000,
-	odm_radiob_txt = 0x1001,
-	odm_radioc_txt = 0x1002,
-	odm_radiod_txt = 0x1003
-};
-
-/*  Status code */
-enum rt_status {
-	RT_STATUS_SUCCESS,
-	RT_STATUS_FAILURE,
-	RT_STATUS_PENDING,
-	RT_STATUS_RESOURCE,
-	RT_STATUS_INVALID_CONTEXT,
-	RT_STATUS_INVALID_PARAMETER,
-	RT_STATUS_NOT_SUPPORT,
-	RT_STATUS_OS_API_FAILED,
-};
-
-/* include "odm_function.h" */
-
-/* 3=========================================================== */
-/* 3 DIG */
-/* 3=========================================================== */
-
-enum dm_dig_op {
-	DIG_TYPE_THRESH_HIGH	= 0,
-	DIG_TYPE_THRESH_LOW	= 1,
-	DIG_TYPE_BACKOFF		= 2,
-	DIG_TYPE_RX_GAIN_MIN	= 3,
-	DIG_TYPE_RX_GAIN_MAX	= 4,
-	DIG_TYPE_ENABLE			= 5,
-	DIG_TYPE_DISABLE		= 6,
-	DIG_OP_TYPE_MAX
-};
-
-#define		DM_DIG_THRESH_HIGH			40
-#define		DM_DIG_THRESH_LOW			35
-
-#define		DM_SCAN_RSSI_TH				0x14 /* scan return issue for LC */
-
-
-#define		DM_FALSEALARM_THRESH_LOW	400
-#define		DM_FALSEALARM_THRESH_HIGH	1000
-
-#define		DM_DIG_MAX_NIC				0x4e
-#define		DM_DIG_MIN_NIC				0x1e
-
-#define		DM_DIG_MAX_AP				0x32
-#define		DM_DIG_MIN_AP				0x20
-
-#define		DM_DIG_MAX_NIC_HP			0x46
-#define		DM_DIG_MIN_NIC_HP			0x2e
-
-#define		DM_DIG_MAX_AP_HP				0x42
-#define		DM_DIG_MIN_AP_HP				0x30
-
-/* vivi 92c&92d has different definition, 20110504 */
-/* this is for 92c */
-#define		DM_DIG_FA_TH0				0x200
-#define		DM_DIG_FA_TH1				0x300
-#define		DM_DIG_FA_TH2				0x400
-/* this is for 92d */
-#define		DM_DIG_FA_TH0_92D			0x100
-#define		DM_DIG_FA_TH1_92D			0x400
-#define		DM_DIG_FA_TH2_92D			0x600
-
-#define		DM_DIG_BACKOFF_MAX			12
-#define		DM_DIG_BACKOFF_MIN			-4
-#define		DM_DIG_BACKOFF_DEFAULT		10
-
-/* 3=========================================================== */
-/* 3 AGC RX High Power Mode */
-/* 3=========================================================== */
-#define          LNA_Low_Gain_1                      0x64
-#define          LNA_Low_Gain_2                      0x5A
-#define          LNA_Low_Gain_3                      0x58
-
-#define          FA_RXHP_TH1                           5000
-#define          FA_RXHP_TH2                           1500
-#define          FA_RXHP_TH3                             800
-#define          FA_RXHP_TH4                             600
-#define          FA_RXHP_TH5                             500
-
-/* 3=========================================================== */
-/* 3 EDCA */
-/* 3=========================================================== */
-
-/* 3=========================================================== */
-/* 3 Dynamic Tx Power */
-/* 3=========================================================== */
-/* Dynamic Tx Power Control Threshold */
-#define		TX_POWER_NEAR_FIELD_THRESH_LVL2	74
-#define		TX_POWER_NEAR_FIELD_THRESH_LVL1	67
-#define		TX_POWER_NEAR_FIELD_THRESH_AP		0x3F
-
-#define		TxHighPwrLevel_Normal		0
-#define		TxHighPwrLevel_Level1		1
-#define		TxHighPwrLevel_Level2		2
-#define		TxHighPwrLevel_BT1			3
-#define		TxHighPwrLevel_BT2			4
-#define		TxHighPwrLevel_15			5
-#define		TxHighPwrLevel_35			6
-#define		TxHighPwrLevel_50			7
-#define		TxHighPwrLevel_70			8
-#define		TxHighPwrLevel_100			9
-
-/* 3=========================================================== */
-/* 3 Rate Adaptive */
-/* 3=========================================================== */
-#define		DM_RATR_STA_INIT			0
-#define		DM_RATR_STA_HIGH			1
-#define			DM_RATR_STA_MIDDLE		2
-#define			DM_RATR_STA_LOW			3
-
-/* 3=========================================================== */
-/* 3 BB Power Save */
-/* 3=========================================================== */
-
-
-enum dm_1r_cca {
-	CCA_1R =0,
-	CCA_2R = 1,
-	CCA_MAX = 2,
-};
-
-enum dm_rf_def {
-	RF_Save =0,
-	RF_Normal = 1,
-	RF_MAX = 2,
-};
-
-/* 3=========================================================== */
-/* 3 Antenna Diversity */
-/* 3=========================================================== */
-enum dm_swas {
-	Antenna_A = 1,
-	Antenna_B = 2,
-	Antenna_MAX = 3,
-};
-
-/*  Maximal number of antenna detection mechanism needs to perform, added by Roger, 2011.12.28. */
-#define	MAX_ANTENNA_DETECTION_CNT	10
-
-/*  */
-/*  Extern Global Variables. */
-/*  */
-#define	OFDM_TABLE_SIZE_92C	37
-#define	OFDM_TABLE_SIZE_92D	43
-#define	CCK_TABLE_SIZE		33
-
-extern	u32 OFDMSwingTable23A[OFDM_TABLE_SIZE_92D];
-extern	u8 CCKSwingTable_Ch1_Ch1323A[CCK_TABLE_SIZE][8];
-extern	u8 CCKSwingTable_Ch1423A [CCK_TABLE_SIZE][8];
-
-
-
-/*  20100514 Joseph: Add definition for antenna switching test after link. */
-/*  This indicates two different the steps. */
-/*  In SWAW_STEP_PEAK, driver needs to switch antenna and listen to the signal on the air. */
-/*  In SWAW_STEP_DETERMINE, driver just compares the signal captured in SWAW_STEP_PEAK */
-/*  with original RSSI to determine if it is necessary to switch antenna. */
-#define SWAW_STEP_PEAK		0
-#define SWAW_STEP_DETERMINE	1
-
-struct hal_data_8723a;
-
-void ODM_Write_DIG23a(struct dm_odm_t *pDM_Odm,	u8	CurrentIGI);
-void ODM_Write_CCK_CCA_Thres23a(struct dm_odm_t *pDM_Odm, u8	CurCCK_CCAThres);
-
-void ODM_SetAntenna(struct dm_odm_t *pDM_Odm, u8 Antenna);
-
-
-#define dm_RF_Saving	ODM_RF_Saving23a
-void ODM_RF_Saving23a(struct dm_odm_t *pDM_Odm, u8 bForceInNormal);
-
-#define dm_CheckTXPowerTracking		ODM_TXPowerTrackingCheck23a
-void ODM_TXPowerTrackingCheck23a(struct dm_odm_t *pDM_Odm);
-
-bool ODM_RAStateCheck23a(struct dm_odm_t *pDM_Odm, s32 RSSI, bool bForceUpdate,
-		      u8 *pRATRState);
-
-
-u32 ConvertTo_dB23a(u32 Value);
-
-u32 GetPSDData(struct dm_odm_t *pDM_Odm, unsigned int point, u8 initial_gain_psd);
-
-void odm_DIG23abyRSSI_LPS(struct dm_odm_t *pDM_Odm);
-
-u32 ODM_Get_Rate_Bitmap23a(struct hal_data_8723a *pHalData, u32 macid, u32 ra_mask, u8 rssi_level);
-
-
-void ODM23a_DMInit(struct dm_odm_t *pDM_Odm);
-
-void ODM_DMWatchdog23a(struct rtw_adapter *adapter);
-
-void ODM_CmnInfoInit23a(struct dm_odm_t *pDM_Odm, enum odm_cmninfo	CmnInfo, u32 Value);
-
-void ODM_CmnInfoPtrArrayHook23a(struct dm_odm_t *pDM_Odm, enum odm_cmninfo	CmnInfo, u16 Index, void *pValue);
-
-void ODM_CmnInfoUpdate23a(struct dm_odm_t *pDM_Odm, u32 CmnInfo, u64 Value);
-
-void ODM_ResetIQKResult(struct dm_odm_t *pDM_Odm);
-
-void ODM_AntselStatistics_88C(struct dm_odm_t *pDM_Odm, u8 MacId, u32 PWDBAll, bool isCCKrate);
-
-void ODM_SingleDualAntennaDefaultSetting(struct dm_odm_t *pDM_Odm);
-
-bool ODM_SingleDualAntennaDetection(struct dm_odm_t *pDM_Odm, u8 mode);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/odm_HWConfig.h b/drivers/staging/rtl8723au/include/odm_HWConfig.h
deleted file mode 100644
index c748d5fb47fa..000000000000
--- a/drivers/staging/rtl8723au/include/odm_HWConfig.h
+++ /dev/null
@@ -1,153 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-
-#ifndef	__HALHWOUTSRC_H__
-#define __HALHWOUTSRC_H__
-
-#include <Hal8723APhyCfg.h>
-
-/*  */
-/*  Definition */
-/*  */
-/*  */
-/*  */
-/*  CCK Rates, TxHT = 0 */
-#define DESC92C_RATE1M					0x00
-#define DESC92C_RATE2M					0x01
-#define DESC92C_RATE5_5M				0x02
-#define DESC92C_RATE11M				0x03
-
-/*  OFDM Rates, TxHT = 0 */
-#define DESC92C_RATE6M					0x04
-#define DESC92C_RATE9M					0x05
-#define DESC92C_RATE12M				0x06
-#define DESC92C_RATE18M				0x07
-#define DESC92C_RATE24M				0x08
-#define DESC92C_RATE36M				0x09
-#define DESC92C_RATE48M				0x0a
-#define DESC92C_RATE54M				0x0b
-
-/*  MCS Rates, TxHT = 1 */
-#define DESC92C_RATEMCS0				0x0c
-#define DESC92C_RATEMCS1				0x0d
-#define DESC92C_RATEMCS2				0x0e
-#define DESC92C_RATEMCS3				0x0f
-#define DESC92C_RATEMCS4				0x10
-#define DESC92C_RATEMCS5				0x11
-#define DESC92C_RATEMCS6				0x12
-#define DESC92C_RATEMCS7				0x13
-#define DESC92C_RATEMCS8				0x14
-#define DESC92C_RATEMCS9				0x15
-#define DESC92C_RATEMCS10				0x16
-#define DESC92C_RATEMCS11				0x17
-#define DESC92C_RATEMCS12				0x18
-#define DESC92C_RATEMCS13				0x19
-#define DESC92C_RATEMCS14				0x1a
-#define DESC92C_RATEMCS15				0x1b
-#define DESC92C_RATEMCS15_SG			0x1c
-#define DESC92C_RATEMCS32				0x20
-
-
-/*  */
-/*  structure and define */
-/*  */
-
-struct phy_rx_agc_info {
-	#ifdef __LITTLE_ENDIAN
-		u8	gain:7, trsw:1;
-	#else
-		u8	trsw:1, gain:7;
-	#endif
-};
-
-struct phy_status_rpt {
-	struct phy_rx_agc_info path_agc[RF_PATH_MAX];
-	u8	ch_corr[RF_PATH_MAX];
-	u8	cck_sig_qual_ofdm_pwdb_all;
-	u8	cck_agc_rpt_ofdm_cfosho_a;
-	u8	cck_rpt_b_ofdm_cfosho_b;
-	u8	rsvd_1;/* ch_corr_msb; */
-	u8	noise_power_db_msb;
-	u8	path_cfotail[RF_PATH_MAX];
-	u8	pcts_mask[RF_PATH_MAX];
-	s8	stream_rxevm[RF_PATH_MAX];
-	u8	path_rxsnr[RF_PATH_MAX];
-	u8	noise_power_db_lsb;
-	u8	rsvd_2[3];
-	u8	stream_csi[RF_PATH_MAX];
-	u8	stream_target_csi[RF_PATH_MAX];
-	s8	sig_evm;
-	u8	rsvd_3;
-
-#ifdef __LITTLE_ENDIAN
-	u8	antsel_rx_keep_2:1;	/* ex_intf_flg:1; */
-	u8	sgi_en:1;
-	u8	rxsc:2;
-	u8	idle_long:1;
-	u8	r_ant_train_en:1;
-	u8	ant_sel_b:1;
-	u8	ant_sel:1;
-#else	/*  _BIG_ENDIAN_ */
-	u8	ant_sel:1;
-	u8	ant_sel_b:1;
-	u8	r_ant_train_en:1;
-	u8	idle_long:1;
-	u8	rxsc:2;
-	u8	sgi_en:1;
-	u8	antsel_rx_keep_2:1;	/* ex_intf_flg:1; */
-#endif
-};
-
-
-struct phy_status_rpt_8195 {
-	struct phy_rx_agc_info path_agc[2];
-	u8	ch_num[2];
-	u8	cck_sig_qual_ofdm_pwdb_all;
-	u8	cck_agc_rpt_ofdm_cfosho_a;
-	u8	cck_bb_pwr_ofdm_cfosho_b;
-	u8    cck_rx_path;	/* CCK_RX_PATH [3:0] (with regA07[3:0] definition) */
-	u8	rsvd_1;
-	u8	path_cfotail[2];
-	u8	pcts_mask[2];
-	s8	stream_rxevm[2];
-	u8	path_rxsnr[2];
-	u8	rsvd_2[2];
-	u8	stream_snr[2];
-	u8	stream_csi[2];
-	u8	rsvd_3[2];
-	s8	sig_evm;
-	u8	rsvd_4;
-#ifdef __LITTLE_ENDIAN
-	u8	antidx_anta:3;
-	u8	antidx_antb:3;
-	u8	rsvd_5:2;
-#else	/*  _BIG_ENDIAN_ */
-	u8	rsvd_5:2;
-	u8	antidx_antb:3;
-	u8	antidx_anta:3;
-#endif
-};
-
-
-void
-ODM_PhyStatusQuery23a(
-	struct dm_odm_t *pDM_Odm,
-	struct phy_info *pPhyInfo,
-	u8 *						pPhyStatus,
-	struct odm_packet_info *pPktinfo
-	);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/odm_RegConfig8723A.h b/drivers/staging/rtl8723au/include/odm_RegConfig8723A.h
deleted file mode 100644
index f2a54d829ed5..000000000000
--- a/drivers/staging/rtl8723au/include/odm_RegConfig8723A.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __INC_ODM_REGCONFIG_H_8723A
-#define __INC_ODM_REGCONFIG_H_8723A
-
-void odm_ConfigRFReg_8723A(struct dm_odm_t *pDM_Odm, u32 Addr, u32 Data,
-			   enum RF_RADIO_PATH RF_PATH, u32 RegAddr);
-
-void odm_ConfigMAC_8723A(struct dm_odm_t *pDM_Odm, u32 Addr, u8 Data);
-
-void odm_ConfigBB_AGC_8723A(struct dm_odm_t *pDM_Odm, u32 addr, u32 data);
-
-void odm_ConfigBB_PHY_8723A(struct dm_odm_t *pDM_Odm, u32 addr, u32 data);
-
-#endif /*  end of SUPPORT */
diff --git a/drivers/staging/rtl8723au/include/odm_RegDefine11N.h b/drivers/staging/rtl8723au/include/odm_RegDefine11N.h
deleted file mode 100644
index 27782154f502..000000000000
--- a/drivers/staging/rtl8723au/include/odm_RegDefine11N.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#ifndef	__ODM_REGDEFINE11N_H__
-#define __ODM_REGDEFINE11N_H__
-
-
-/* 2 RF REG LIST */
-#define	ODM_REG_RF_MODE_11N			0x00
-#define	ODM_REG_RF_0B_11N			0x0B
-#define	ODM_REG_CHNBW_11N			0x18
-#define	ODM_REG_T_METER_11N			0x24
-#define	ODM_REG_RF_25_11N			0x25
-#define	ODM_REG_RF_26_11N			0x26
-#define	ODM_REG_RF_27_11N			0x27
-#define	ODM_REG_RF_2B_11N			0x2B
-#define	ODM_REG_RF_2C_11N			0x2C
-#define	ODM_REG_RXRF_A3_11N			0x3C
-#define	ODM_REG_T_METER_92D_11N			0x42
-#define	ODM_REG_T_METER_88E_11N			0x42
-
-
-
-/* 2 BB REG LIST */
-/* PAGE 8 */
-#define	ODM_REG_BB_CTRL_11N			0x800
-#define	ODM_REG_RF_PIN_11N			0x804
-#define	ODM_REG_PSD_CTRL_11N			0x808
-#define	ODM_REG_TX_ANT_CTRL_11N			0x80C
-#define	ODM_REG_BB_PWR_SAV5_11N			0x818
-#define	ODM_REG_CCK_RPT_FORMAT_11N		0x824
-#define	ODM_REG_RX_DEFUALT_A_11N		0x858
-#define	ODM_REG_RX_DEFUALT_B_11N		0x85A
-#define	ODM_REG_BB_PWR_SAV3_11N			0x85C
-#define	ODM_REG_ANTSEL_CTRL_11N			0x860
-#define	ODM_REG_RX_ANT_CTRL_11N			0x864
-#define	ODM_REG_PIN_CTRL_11N			0x870
-#define	ODM_REG_BB_PWR_SAV1_11N			0x874
-#define	ODM_REG_ANTSEL_PATH_11N			0x878
-#define	ODM_REG_BB_3WIRE_11N			0x88C
-#define	ODM_REG_SC_CNT_11N			0x8C4
-#define	ODM_REG_PSD_DATA_11N			0x8B4
-/* PAGE 9 */
-#define	ODM_REG_ANT_MAPPING1_11N		0x914
-#define	ODM_REG_ANT_MAPPING2_11N		0x918
-/* PAGE A */
-#define	ODM_REG_CCK_ANTDIV_PARA1_11N		0xA00
-#define	ODM_REG_CCK_CCA_11N			0xA0A
-#define	ODM_REG_CCK_ANTDIV_PARA2_11N		0xA0C
-#define	ODM_REG_CCK_ANTDIV_PARA3_11N		0xA10
-#define	ODM_REG_CCK_ANTDIV_PARA4_11N		0xA14
-#define	ODM_REG_CCK_FILTER_PARA1_11N		0xA22
-#define	ODM_REG_CCK_FILTER_PARA2_11N		0xA23
-#define	ODM_REG_CCK_FILTER_PARA3_11N		0xA24
-#define	ODM_REG_CCK_FILTER_PARA4_11N		0xA25
-#define	ODM_REG_CCK_FILTER_PARA5_11N		0xA26
-#define	ODM_REG_CCK_FILTER_PARA6_11N		0xA27
-#define	ODM_REG_CCK_FILTER_PARA7_11N		0xA28
-#define	ODM_REG_CCK_FILTER_PARA8_11N		0xA29
-#define	ODM_REG_CCK_FA_RST_11N			0xA2C
-#define	ODM_REG_CCK_FA_MSB_11N			0xA58
-#define	ODM_REG_CCK_FA_LSB_11N			0xA5C
-#define	ODM_REG_CCK_CCA_CNT_11N			0xA60
-#define	ODM_REG_BB_PWR_SAV4_11N			0xA74
-/* PAGE B */
-#define	ODM_REG_LNA_SWITCH_11N			0xB2C
-#define	ODM_REG_PATH_SWITCH_11N			0xB30
-#define	ODM_REG_RSSI_CTRL_11N			0xB38
-#define	ODM_REG_CONFIG_ANTA_11N			0xB68
-#define	ODM_REG_RSSI_BT_11N			0xB9C
-/* PAGE C */
-#define	ODM_REG_OFDM_FA_HOLDC_11N		0xC00
-#define	ODM_REG_RX_PATH_11N			0xC04
-#define	ODM_REG_TRMUX_11N			0xC08
-#define	ODM_REG_OFDM_FA_RSTC_11N		0xC0C
-#define	ODM_REG_RXIQI_MATRIX_11N		0xC14
-#define	ODM_REG_TXIQK_MATRIX_LSB1_11N		0xC4C
-#define	ODM_REG_IGI_A_11N			0xC50
-#define	ODM_REG_ANTDIV_PARA2_11N		0xC54
-#define	ODM_REG_IGI_B_11N			0xC58
-#define	ODM_REG_ANTDIV_PARA3_11N		0xC5C
-#define	ODM_REG_BB_PWR_SAV2_11N			0xC70
-#define	ODM_REG_RX_OFF_11N			0xC7C
-#define	ODM_REG_TXIQK_MATRIXA_11N		0xC80
-#define	ODM_REG_TXIQK_MATRIXB_11N		0xC88
-#define	ODM_REG_TXIQK_MATRIXA_LSB2_11N		0xC94
-#define	ODM_REG_TXIQK_MATRIXB_LSB2_11N		0xC9C
-#define	ODM_REG_RXIQK_MATRIX_LSB_11N		0xCA0
-#define	ODM_REG_ANTDIV_PARA1_11N		0xCA4
-#define	ODM_REG_OFDM_FA_TYPE1_11N		0xCF0
-/* PAGE D */
-#define	ODM_REG_OFDM_FA_RSTD_11N		0xD00
-#define	ODM_REG_OFDM_FA_TYPE2_11N		0xDA0
-#define	ODM_REG_OFDM_FA_TYPE3_11N		0xDA4
-#define	ODM_REG_OFDM_FA_TYPE4_11N		0xDA8
-/* PAGE E */
-#define	ODM_REG_TXAGC_A_6_18_11N		0xE00
-#define	ODM_REG_TXAGC_A_24_54_11N		0xE04
-#define	ODM_REG_TXAGC_A_1_MCS32_11N		0xE08
-#define	ODM_REG_TXAGC_A_MCS0_3_11N		0xE10
-#define	ODM_REG_TXAGC_A_MCS4_7_11N		0xE14
-#define	ODM_REG_TXAGC_A_MCS8_11_11N		0xE18
-#define	ODM_REG_TXAGC_A_MCS12_15_11N		0xE1C
-#define	ODM_REG_FPGA0_IQK_11N			0xE28
-#define	ODM_REG_TXIQK_TONE_A_11N		0xE30
-#define	ODM_REG_RXIQK_TONE_A_11N		0xE34
-#define	ODM_REG_TXIQK_PI_A_11N			0xE38
-#define	ODM_REG_RXIQK_PI_A_11N			0xE3C
-#define	ODM_REG_TXIQK_11N			0xE40
-#define	ODM_REG_RXIQK_11N			0xE44
-#define	ODM_REG_IQK_AGC_PTS_11N			0xE48
-#define	ODM_REG_IQK_AGC_RSP_11N			0xE4C
-#define	ODM_REG_BLUETOOTH_11N			0xE6C
-#define	ODM_REG_RX_WAIT_CCA_11N			0xE70
-#define	ODM_REG_TX_CCK_RFON_11N			0xE74
-#define	ODM_REG_TX_CCK_BBON_11N			0xE78
-#define	ODM_REG_OFDM_RFON_11N			0xE7C
-#define	ODM_REG_OFDM_BBON_11N			0xE80
-#define ODM_REG_TX2RX_11N			0xE84
-#define	ODM_REG_TX2TX_11N			0xE88
-#define	ODM_REG_RX_CCK_11N			0xE8C
-#define	ODM_REG_RX_OFDM_11N			0xED0
-#define	ODM_REG_RX_WAIT_RIFS_11N		0xED4
-#define	ODM_REG_RX2RX_11N			0xED8
-#define	ODM_REG_STANDBY_11N			0xEDC
-#define	ODM_REG_SLEEP_11N			0xEE0
-#define	ODM_REG_PMPD_ANAEN_11N			0xEEC
-
-
-
-
-
-
-
-/* 2 MAC REG LIST */
-#define	ODM_REG_BB_RST_11N			0x02
-#define	ODM_REG_ANTSEL_PIN_11N			0x4C
-#define	ODM_REG_EARLY_MODE_11N			0x4D0
-#define	ODM_REG_RSSI_MONITOR_11N		0x4FE
-#define	ODM_REG_EDCA_VO_11N			0x500
-#define	ODM_REG_EDCA_VI_11N			0x504
-#define	ODM_REG_EDCA_BE_11N			0x508
-#define	ODM_REG_EDCA_BK_11N			0x50C
-#define	ODM_REG_TXPAUSE_11N			0x522
-#define	ODM_REG_RESP_TX_11N			0x6D8
-#define	ODM_REG_ANT_TRAIN_PARA1_11N		0x7b0
-#define	ODM_REG_ANT_TRAIN_PARA2_11N		0x7b4
-
-
-/* DIG Related */
-#define	ODM_BIT_IGI_11N				0x0000007F
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/odm_debug.h b/drivers/staging/rtl8723au/include/odm_debug.h
deleted file mode 100644
index c4b375a6f409..000000000000
--- a/drivers/staging/rtl8723au/include/odm_debug.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-
-#ifndef	__ODM_DBG_H__
-#define __ODM_DBG_H__
-
-
-/*  */
-/*	Define the debug levels */
-/*  */
-/*	1.	DBG_TRACE and DBG_LOUD are used for normal cases. */
-/*	So that, they can help SW engineer to develop or trace states changed */
-/*	and also help HW enginner to trace every operation to and from HW, */
-/*	e.g IO, Tx, Rx. */
-/*  */
-/*	2.	DBG_WARNNING and DBG_SERIOUS are used for unusual or error cases, */
-/*	which help us to debug SW or HW. */
-/*  */
-/*  */
-/*  */
-/*	Never used in a call to ODM_RT_TRACE()! */
-/*  */
-#define ODM_DBG_OFF					1
-
-/*  */
-/*	Fatal bug. */
-/*	For example, Tx/Rx/IO locked up, OS hangs, memory access violation, */
-/*	resource allocation failed, unexpected HW behavior, HW BUG and so on. */
-/*  */
-#define ODM_DBG_SERIOUS				2
-
-/*  */
-/*	Abnormal, rare, or unexpeted cases. */
-/*	For example, IRP/Packet/OID canceled, device suprisely unremoved and so on. */
-/*  */
-#define ODM_DBG_WARNING				3
-
-/*  */
-/*	Normal case with useful information about current SW or HW state. */
-/*	For example, Tx/Rx descriptor to fill, Tx/Rx descriptor completed status, */
-/*	SW protocol state change, dynamic mechanism state change and so on. */
-/*  */
-#define ODM_DBG_LOUD					4
-
-/*  */
-/*	Normal case with detail execution flow or information. */
-/*  */
-#define ODM_DBG_TRACE					5
-
-/*  */
-/*  Define the tracing components */
-/*  */
-/*  */
-/* BB Functions */
-#define ODM_COMP_DIG				BIT(0)
-#define ODM_COMP_RA_MASK			BIT(1)
-#define ODM_COMP_DYNAMIC_TXPWR			BIT(2)
-#define ODM_COMP_FA_CNT				BIT(3)
-#define ODM_COMP_RSSI_MONITOR			BIT(4)
-#define ODM_COMP_CCK_PD				BIT(5)
-#define ODM_COMP_ANT_DIV			BIT(6)
-#define ODM_COMP_PWR_SAVE			BIT(7)
-#define ODM_COMP_PWR_TRAIN			BIT(8)
-#define ODM_COMP_RATE_ADAPTIVE			BIT(9)
-#define ODM_COMP_PATH_DIV			BIT(10)
-#define ODM_COMP_PSD				BIT(11)
-#define ODM_COMP_DYNAMIC_PRICCA			BIT(12)
-#define ODM_COMP_RXHP				BIT(13)
-/* MAC Functions */
-#define ODM_COMP_EDCA_TURBO			BIT(16)
-#define ODM_COMP_EARLY_MODE			BIT(17)
-/* RF Functions */
-#define ODM_COMP_TX_PWR_TRACK			BIT(24)
-#define ODM_COMP_RX_GAIN_TRACK			BIT(25)
-#define ODM_COMP_CALIBRATION			BIT(26)
-/* Common Functions */
-#define ODM_COMP_COMMON				BIT(30)
-#define ODM_COMP_INIT				BIT(31)
-
-/*------------------------Export Macro Definition---------------------------*/
-	#define RT_PRINTK(fmt, args...)	printk("%s(): " fmt, __func__, ## args);
-
-#ifndef ASSERT
-	#define ASSERT(expr)
-#endif
-
-#define ODM_RT_TRACE(pDM_Odm, comp, level, fmt)							\
-		if(((comp) & pDM_Odm->DebugComponents) && (level <= pDM_Odm->DebugLevel))	\
-		{										\
-			printk("[ODM-8723A] ");						\
-			RT_PRINTK fmt;								\
-		}
-
-#define ODM_RT_ASSERT(pDM_Odm, expr, fmt)							\
-		if(!(expr)) {									\
-			printk("Assertion failed! %s at ......\n", #expr);			\
-			printk("      ......%s,%s,line=%d\n", __FILE__, __func__, __LINE__);\
-			RT_PRINTK fmt;								\
-			ASSERT(false);								\
-		}
-
-void ODM_InitDebugSetting23a(struct dm_odm_t *pDM_Odm);
-
-#endif	/*  __ODM_DBG_H__ */
diff --git a/drivers/staging/rtl8723au/include/odm_interface.h b/drivers/staging/rtl8723au/include/odm_interface.h
deleted file mode 100644
index 1d3bf03b59ea..000000000000
--- a/drivers/staging/rtl8723au/include/odm_interface.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-
-#ifndef	__ODM_INTERFACE_H__
-#define __ODM_INTERFACE_H__
-
-
-/*  _cat: implemented by Token-Pasting Operator. */
-
-/*===================================
-
-#define ODM_REG_DIG_11N		0xC50
-#define ODM_REG_DIG_11AC	0xDDD
-
-ODM_REG(DIG,_pDM_Odm)
-=====================================*/
-
-#define _reg_11N(_name)			ODM_REG_##_name##_11N
-#define _reg_11AC(_name)		ODM_REG_##_name##_11AC
-#define _bit_11N(_name)			ODM_BIT_##_name##_11N
-#define _bit_11AC(_name)		ODM_BIT_##_name##_11AC
-
-#define _cat(_name, _func)					\
-	(							\
-		_func##_11N(_name)				\
-	)
-
-/*  _name: name of register or bit. */
-/*  Example: "ODM_REG(R_A_AGC_CORE1, pDM_Odm)" */
-/*         gets "ODM_R_A_AGC_CORE1" or "ODM_R_A_AGC_CORE1_8192C", depends on SupportICType. */
-#define ODM_REG(_name, _pDM_Odm)	_cat(_name, _reg)
-#define ODM_BIT(_name, _pDM_Odm)	_cat(_name, _bit)
-
-/*  */
-/*  2012/02/17 MH For non-MP compile pass only. Linux does not support workitem. */
-/*  Suggest HW team to use thread instead of workitem. Windows also support the feature. */
-/*  */
-typedef void (*RT_WORKITEM_CALL_BACK)(struct work_struct *pContext);
-
-/*  */
-/*  =========== EXtern Function Prototype */
-/*  */
-
-void ODM_SetRFReg(struct dm_odm_t *pDM_Odm, enum RF_RADIO_PATH eRFPath,
-		  u32 RegAddr, u32 BitMask, u32 Data);
-u32 ODM_GetRFReg(struct dm_odm_t *pDM_Odm, enum RF_RADIO_PATH eRFPath,
-		 u32 RegAddr, u32 BitMask);
-
-#endif	/*  __ODM_INTERFACE_H__ */
diff --git a/drivers/staging/rtl8723au/include/odm_precomp.h b/drivers/staging/rtl8723au/include/odm_precomp.h
deleted file mode 100644
index fb793c8ba7f8..000000000000
--- a/drivers/staging/rtl8723au/include/odm_precomp.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#ifndef	__ODM_PRECOMP_H__
-#define __ODM_PRECOMP_H__
-
-/* 2 Config Flags and Structs - defined by each ODM Type */
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <hal_intf.h>
-
-
-/* 2 Hardware Parameter Files */
-#include "Hal8723UHWImg_CE.h"
-
-
-/* 2 OutSrc Header Files */
-
-#include "odm.h"
-#include "odm_HWConfig.h"
-#include "odm_debug.h"
-#include "odm_RegDefine11N.h"
-
-#include "HalDMOutSrc8723A.h" /* for IQK,LCK,Power-tracking */
-#include "rtl8723a_hal.h"
-
-#include "odm_interface.h"
-#include "odm_reg.h"
-
-#include "HalHWImg8723A_MAC.h"
-#include "HalHWImg8723A_RF.h"
-#include "HalHWImg8723A_BB.h"
-#include "HalHWImg8723A_FW.h"
-#include "odm_RegConfig8723A.h"
-
-#endif	/*  __ODM_PRECOMP_H__ */
diff --git a/drivers/staging/rtl8723au/include/odm_reg.h b/drivers/staging/rtl8723au/include/odm_reg.h
deleted file mode 100644
index c18433120fe8..000000000000
--- a/drivers/staging/rtl8723au/include/odm_reg.h
+++ /dev/null
@@ -1,111 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-/*  */
-/*  File Name: odm_reg.h */
-/*  */
-/*  Description: */
-/*  */
-/*  This file is for general register definition. */
-/*  */
-/*  */
-/*  */
-#ifndef	__HAL_ODM_REG_H__
-#define __HAL_ODM_REG_H__
-
-/*  */
-/*  Register Definition */
-/*  */
-
-/* MAC REG */
-#define	ODM_BB_RESET					0x002
-#define	ODM_DUMMY						0x4fe
-#define	ODM_EDCA_VO_PARAM			0x500
-#define	ODM_EDCA_VI_PARAM			0x504
-#define	ODM_EDCA_BE_PARAM			0x508
-#define	ODM_EDCA_BK_PARAM			0x50C
-#define	ODM_TXPAUSE					0x522
-
-/* BB REG */
-#define	ODM_FPGA_PHY0_PAGE8			0x800
-#define	ODM_PSD_SETTING				0x808
-#define	ODM_AFE_SETTING				0x818
-#define	ODM_TXAGC_B_6_18				0x830
-#define	ODM_TXAGC_B_24_54			0x834
-#define	ODM_TXAGC_B_MCS32_5			0x838
-#define	ODM_TXAGC_B_MCS0_MCS3		0x83c
-#define	ODM_TXAGC_B_MCS4_MCS7		0x848
-#define	ODM_TXAGC_B_MCS8_MCS11		0x84c
-#define	ODM_ANALOG_REGISTER			0x85c
-#define	ODM_RF_INTERFACE_OUTPUT		0x860
-#define	ODM_TXAGC_B_MCS12_MCS15	0x868
-#define	ODM_TXAGC_B_11_A_2_11		0x86c
-#define	ODM_AD_DA_LSB_MASK			0x874
-#define	ODM_ENABLE_3_WIRE			0x88c
-#define	ODM_PSD_REPORT				0x8b4
-#define	ODM_R_ANT_SELECT				0x90c
-#define	ODM_CCK_ANT_SELECT			0xa07
-#define	ODM_CCK_PD_THRESH			0xa0a
-#define	ODM_CCK_RF_REG1				0xa11
-#define	ODM_CCK_MATCH_FILTER			0xa20
-#define	ODM_CCK_RAKE_MAC				0xa2e
-#define	ODM_CCK_CNT_RESET			0xa2d
-#define	ODM_CCK_TX_DIVERSITY			0xa2f
-#define	ODM_CCK_FA_CNT_MSB			0xa5b
-#define	ODM_CCK_FA_CNT_LSB			0xa5c
-#define	ODM_CCK_NEW_FUNCTION		0xa75
-#define	ODM_OFDM_PHY0_PAGE_C		0xc00
-#define	ODM_OFDM_RX_ANT				0xc04
-#define	ODM_R_A_RXIQI					0xc14
-#define	ODM_R_A_AGC_CORE1			0xc50
-#define	ODM_R_A_AGC_CORE2			0xc54
-#define	ODM_R_B_AGC_CORE1			0xc58
-#define	ODM_R_AGC_PAR					0xc70
-#define	ODM_R_HTSTF_AGC_PAR			0xc7c
-#define	ODM_TX_PWR_TRAINING_A		0xc90
-#define	ODM_TX_PWR_TRAINING_B		0xc98
-#define	ODM_OFDM_FA_CNT1				0xcf0
-#define	ODM_OFDM_PHY0_PAGE_D		0xd00
-#define	ODM_OFDM_FA_CNT2				0xda0
-#define	ODM_OFDM_FA_CNT3				0xda4
-#define	ODM_OFDM_FA_CNT4				0xda8
-#define	ODM_TXAGC_A_6_18				0xe00
-#define	ODM_TXAGC_A_24_54			0xe04
-#define	ODM_TXAGC_A_1_MCS32			0xe08
-#define	ODM_TXAGC_A_MCS0_MCS3		0xe10
-#define	ODM_TXAGC_A_MCS4_MCS7		0xe14
-#define	ODM_TXAGC_A_MCS8_MCS11		0xe18
-#define	ODM_TXAGC_A_MCS12_MCS15		0xe1c
-
-/* RF REG */
-#define	ODM_GAIN_SETTING				0x00
-#define	ODM_CHANNEL					0x18
-
-/* Ant Detect Reg */
-#define	ODM_DPDT						0x300
-
-/* PSD Init */
-#define	ODM_PSDREG					0x808
-
-/* 92D Path Div */
-#define	PATHDIV_REG					0xB30
-#define	PATHDIV_TRI					0xBA0
-
-/*  */
-/*  Bitmap Definition */
-/*  */
-
-#define	BIT_FA_RESET					BIT(0)
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/osdep_intf.h b/drivers/staging/rtl8723au/include/osdep_intf.h
deleted file mode 100644
index a157eb2e78df..000000000000
--- a/drivers/staging/rtl8723au/include/osdep_intf.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#ifndef __OSDEP_INTF_H_
-#define __OSDEP_INTF_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-int rtw_init_drv_sw23a(struct rtw_adapter *padapter);
-int rtw_free_drv_sw23a(struct rtw_adapter *padapter);
-int rtw_reset_drv_sw23a(struct rtw_adapter *padapter);
-
-void rtw_cancel_all_timer23a(struct rtw_adapter *padapter);
-
-int rtw_init_netdev23a_name23a(struct net_device *pnetdev, const char *ifname);
-struct net_device *rtw_init_netdev23a(struct rtw_adapter *padapter);
-
-u16 rtw_recv_select_queue23a(struct sk_buff *skb);
-
-void rtw_ips_dev_unload23a(struct rtw_adapter *padapter);
-
-int rtw_ips_pwr_up23a(struct rtw_adapter *padapter);
-void rtw_ips_pwr_down23a(struct rtw_adapter *padapter);
-
-int rtw_drv_register_netdev(struct rtw_adapter *padapter);
-void rtw_ndev_destructor(struct net_device *ndev);
-
-int rtl8723au_inirp_init(struct rtw_adapter *Adapter);
-int rtl8723au_inirp_deinit(struct rtw_adapter *Adapter);
-void rtl8723a_usb_intf_stop(struct rtw_adapter *padapter);
-
-#endif	/* _OSDEP_INTF_H_ */
diff --git a/drivers/staging/rtl8723au/include/osdep_service.h b/drivers/staging/rtl8723au/include/osdep_service.h
deleted file mode 100644
index 98250b12e9f2..000000000000
--- a/drivers/staging/rtl8723au/include/osdep_service.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __OSDEP_SERVICE_H_
-#define __OSDEP_SERVICE_H_
-
-#define _FAIL		0
-#define _SUCCESS	1
-#define RTW_RX_HANDLED 2
-
-#include <linux/spinlock.h>
-#include <linux/compiler.h>
-#include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/module.h>
-#include <linux/kref.h>
-#include <linux/netdevice.h>
-#include <linux/skbuff.h>
-#include <linux/uaccess.h>
-#include <asm/byteorder.h>
-#include <linux/atomic.h>
-#include <linux/io.h>
-#include <linux/semaphore.h>
-#include <linux/sem.h>
-#include <linux/sched.h>
-#include <linux/etherdevice.h>
-#include <linux/wireless.h>
-#include <linux/if_arp.h>
-#include <linux/rtnetlink.h>
-#include <linux/delay.h>
-#include <linux/interrupt.h>	/*  for struct tasklet_struct */
-#include <linux/ip.h>
-
-#include <net/ieee80211_radiotap.h>
-#include <net/cfg80211.h>
-
-struct rtw_queue {
-	struct	list_head	queue;
-	spinlock_t		lock;
-};
-
-static inline struct list_head *get_list_head(struct rtw_queue *queue)
-{
-	return &queue->queue;
-}
-
-static inline int rtw_netif_queue_stopped(struct net_device *pnetdev)
-{
-	return (netif_tx_queue_stopped(netdev_get_tx_queue(pnetdev, 0)) &&
-		netif_tx_queue_stopped(netdev_get_tx_queue(pnetdev, 1)) &&
-		netif_tx_queue_stopped(netdev_get_tx_queue(pnetdev, 2)) &&
-		netif_tx_queue_stopped(netdev_get_tx_queue(pnetdev, 3)));
-}
-
-static inline u32 CHKBIT(u32 x)
-{
-	WARN_ON(x >= 32);
-	if (x >= 32)
-		return 0;
-	return BIT(x);
-}
-
-extern unsigned char MCS_rate_2R23A[16];
-
-extern unsigned char	MCS_rate_2R23A[16];
-extern unsigned char	MCS_rate_1R23A[16];
-
-void	_rtw_init_queue23a(struct rtw_queue *pqueue);
-
-/* Macros for handling unaligned memory accesses */
-
-#define RTW_GET_BE24(a) ((((u32) (a)[0]) << 16) | (((u32) (a)[1]) << 8) | \
-			 ((u32) (a)[2]))
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/recv_osdep.h b/drivers/staging/rtl8723au/include/recv_osdep.h
deleted file mode 100644
index c2d3f1bd5948..000000000000
--- a/drivers/staging/rtl8723au/include/recv_osdep.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RECV_OSDEP_H_
-#define __RECV_OSDEP_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-int _rtw_init_recv_priv23a(struct recv_priv *precvpriv, struct rtw_adapter *padapter);
-void _rtw_free_recv_priv23a (struct recv_priv *precvpriv);
-
-int rtw_recv_entry23a(struct recv_frame *precv_frame);
-int rtw_recv_indicatepkt23a(struct rtw_adapter *adapter, struct recv_frame *precv_frame);
-
-void rtw_handle_tkip_mic_err23a(struct rtw_adapter *padapter, u8 bgroup);
-
-int	rtw_init_recv_priv(struct recv_priv *precvpriv, struct rtw_adapter *padapter);
-void rtw_free_recv_priv (struct recv_priv *precvpriv);
-
-int rtw_os_recv_resource_init(struct recv_priv *precvpriv, struct rtw_adapter *padapter);
-
-void rtw_init_recv_timer23a(struct recv_reorder_ctrl *preorder_ctrl);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_bt-coexist.h b/drivers/staging/rtl8723au/include/rtl8723a_bt-coexist.h
deleted file mode 100644
index 7add5dfe015f..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_bt-coexist.h
+++ /dev/null
@@ -1,1627 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTL8723A_BT_COEXIST_H__
-#define __RTL8723A_BT_COEXIST_H__
-
-#include <drv_types.h>
-#include "odm_precomp.h"
-
-
-/*  HEADER/PlatformDef.h */
-enum rt_media_status {
-	RT_MEDIA_DISCONNECT	= 0,
-	RT_MEDIA_CONNECT	= 1
-};
-
-/*  ===== Below this line is sync from SD7 driver COMMON/BT.h ===== */
-
-#define	BT_TMP_BUF_SIZE		100
-
-void BT_SignalCompensation(struct rtw_adapter *padapter,
-			   u8 *rssi_wifi, u8 *rssi_bt);
-void BT_HaltProcess(struct rtw_adapter *padapter);
-void BT_LpsLeave(struct rtw_adapter *padapter);
-
-
-#define	BT_HsConnectionEstablished(Adapter)		false
-/*  ===== End of sync from SD7 driver COMMON/BT.h ===== */
-
-/*  HEADER/SecurityType.h */
-#define TKIP_ENC_KEY_POS		32		/* KEK_LEN+KEK_LEN) */
-#define MAXRSNIELEN				256
-
-/*  COMMON/Protocol802_11.h */
-/*  */
-/*       802.11 Management frame Status Code field */
-/*  */
-struct octet_string {
-	u8		*Octet;
-	u16		Length;
-};
-
-
-/*  AES_CCMP specific */
-enum {
-	AESCCMP_BLK_SIZE		=   16,     /*  # octets in an AES block */
-	AESCCMP_MAX_PACKET		=   4*512,  /*  largest packet size */
-	AESCCMP_N_RESERVED		=   0,      /*  reserved nonce octet value */
-	AESCCMP_A_DATA			=   0x40,   /*  the Adata bit in the flags */
-	AESCCMP_M_SHIFT			=   3,      /*  how much to shift the 3-bit M field */
-	AESCCMP_L_SHIFT			=   0,      /*  how much to shift the 3-bit L field */
-	AESCCMP_L_SIZE			=   2,       /*  size of the l(m) length field (in octets) */
-	AESCCMP_OFFSET_SC		=	22,
-	AESCCMP_OFFSET_DURATION	=	4,
-	AESCCMP_OFFSET_A2		=	10,
-	AESCCMP_OFFSET_A4		=	24,
-	AESCCMP_QC_TID_MASK		=	0x0f,
-	AESCCMP_BLK_SIZE_TOTAL	=   16*16,     /*  Added by Annie for CKIP AES MIC BSOD, 2006-08-17. */
-											/*  16*8 < 4*60  Resove to 16*16 */
-};
-
-/*  Key Length */
-#define PMK_LEN				32
-#define PTK_LEN_TKIP			64
-#define GTK_LEN				32
-#define KEY_NONCE_LEN			32
-
-
-/*  COMMON/Dot11d.h */
-struct chnl_txpower_triple {
-	u8 FirstChnl;
-	u8 NumChnls;
-	s8 MaxTxPowerInDbm;
-};
-
-
-/*  ===== Below this line is sync from SD7 driver COMMON/bt_hci.h ===== */
-/*  The following is for BT 3.0 + HS HCI COMMAND ERRORS CODES */
-
-#define Max80211PALPDUSize			1492
-#define Max80211AMPASSOCLen			672
-#define MinGUserPrio					4
-#define MaxGUserPrio					7
-#define BEUserPrio0						0
-#define BEUserPrio1						3
-#define Max80211BeaconPeriod		2000
-#define ShortRangeModePowerMax		4
-
-#define BT_Default_Chnl					10
-#define ACLDataHeaderLen				4
-
-#define BTTotalDataBlockNum			0x100
-#define BTLocalBufNum					0x200
-#define BTMaxDataBlockLen				0x800
-#define BTTOTALBANDWIDTH				0x7530
-#define BTMAXBANDGUBANDWIDTH		0x4e20
-#define TmpLocalBufSize					0x100
-#define BTSynDataPacketLength			0xff
-/*  */
-
-#define BTMaxAuthCount					5
-#define BTMaxAsocCount					5
-
-#define MAX_LOGICAL_LINK_NUM			2	/* temporarily define */
-#define MAX_BT_ASOC_ENTRY_NUM		2	/* temporarily define */
-
-#define INVALID_PL_HANDLE				0xff
-#define INVALID_ENTRY_NUM				0xff
-/*  */
-
-#define CAM_BT_START_INDEX		(HALF_CAM_ENTRY - 4)   /*  MAX_BT_ASOC_ENTRY_NUM : 4 !!! */
-#define BT_HWCAM_STAR			CAM_BT_START_INDEX  /*  We used  HALF_CAM_ENTRY ~ HALF_CAM_ENTRY -MAX_BT_ASOC_ENTRY_NUM */
-
-enum hci_status {
-	HCI_STATUS_SUCCESS			= 0x00, /* Success */
-	HCI_STATUS_UNKNOW_HCI_CMD		= 0x01, /* Unknown HCI Command */
-	HCI_STATUS_UNKNOW_CONNECT_ID		= 0X02, /* Unknown Connection Identifier */
-	HCI_STATUS_HW_FAIL			= 0X03, /* Hardware Failure */
-	HCI_STATUS_PAGE_TIMEOUT			= 0X04, /* Page Timeout */
-	HCI_STATUS_AUTH_FAIL			= 0X05, /* Authentication Failure */
-	HCI_STATUS_PIN_OR_KEY_MISSING		= 0X06, /* PIN or Key Missing */
-	HCI_STATUS_MEM_CAP_EXCEED		= 0X07, /* Memory Capacity Exceeded */
-	HCI_STATUS_CONNECT_TIMEOUT		= 0X08, /* Connection Timeout */
-	HCI_STATUS_CONNECT_LIMIT		= 0X09, /* Connection Limit Exceeded */
-	HCI_STATUS_SYN_CONNECT_LIMIT		= 0X0a, /* Synchronous Connection Limit To A Device Exceeded */
-	HCI_STATUS_ACL_CONNECT_EXISTS		= 0X0b, /* ACL Connection Already Exists */
-	HCI_STATUS_CMD_DISALLOW			= 0X0c, /* Command Disallowed */
-	HCI_STATUS_CONNECT_RJT_LIMIT_RESOURCE	= 0X0d, /* Connection Rejected due to Limited Resources */
-	HCI_STATUS_CONNECT_RJT_SEC_REASON	= 0X0e, /* Connection Rejected Due To Security Reasons */
-	HCI_STATUS_CONNECT_RJT_UNACCEPT_BD_ADDR	= 0X0f, /* Connection Rejected due to Unacceptable BD_ADDR */
-	HCI_STATUS_CONNECT_ACCEPT_TIMEOUT	= 0X10, /* Connection Accept Timeout Exceeded */
-	HCI_STATUS_UNSUPPORT_FEATURE_PARA_VALUE	= 0X11, /* Unsupported Feature or Parameter Value */
-	HCI_STATUS_INVALID_HCI_CMD_PARA_VALUE	= 0X12, /* Invalid HCI Command Parameters */
-	HCI_STATUS_REMOTE_USER_TERMINATE_CONNECT = 0X13, /* Remote User Terminated Connection */
-	HCI_STATUS_REMOTE_DEV_TERMINATE_LOW_RESOURCE = 0X14, /* Remote Device Terminated Connection due to Low Resources */
-	HCI_STATUS_REMOTE_DEV_TERMINATE_CONNECT_POWER_OFF = 0X15, /* Remote Device Terminated Connection due to Power Off */
-	HCI_STATUS_CONNECT_TERMINATE_LOCAL_HOST	= 0X16, /* Connection Terminated By Local Host */
-	HCI_STATUS_REPEATE_ATTEMPT		= 0X17, /* Repeated Attempts */
-	HCI_STATUS_PAIR_NOT_ALLOW		= 0X18, /* Pairing Not Allowed */
-	HCI_STATUS_UNKNOW_LMP_PDU		= 0X19, /* Unknown LMP PDU */
-	HCI_STATUS_UNSUPPORT_REMOTE_LMP_FEATURE	= 0X1a, /* Unsupported Remote Feature / Unsupported LMP Feature */
-	HCI_STATUS_SOC_OFFSET_REJECT		= 0X1b, /* SCO Offset Rejected */
-	HCI_STATUS_SOC_INTERVAL_REJECT		= 0X1c, /* SCO Interval Rejected */
-	HCI_STATUS_SOC_AIR_MODE_REJECT		= 0X1d,/* SCO Air Mode Rejected */
-	HCI_STATUS_INVALID_LMP_PARA		= 0X1e, /* Invalid LMP Parameters */
-	HCI_STATUS_UNSPECIFIC_ERROR		= 0X1f, /* Unspecified Error */
-	HCI_STATUS_UNSUPPORT_LMP_PARA_VALUE	= 0X20, /* Unsupported LMP Parameter Value */
-	HCI_STATUS_ROLE_CHANGE_NOT_ALLOW	= 0X21, /* Role Change Not Allowed */
-	HCI_STATUS_LMP_RESPONSE_TIMEOUT		= 0X22, /* LMP Response Timeout */
-	HCI_STATUS_LMP_ERROR_TRANSACTION_COLLISION = 0X23, /* LMP Error Transaction Collision */
-	HCI_STATUS_LMP_PDU_NOT_ALLOW		= 0X24, /* LMP PDU Not Allowed */
-	HCI_STATUS_ENCRYPTION_MODE_NOT_ALLOW	= 0X25, /* Encryption Mode Not Acceptable */
-	HCI_STATUS_LINK_KEY_CAN_NOT_CHANGE	= 0X26, /* Link Key Can Not be Changed */
-	HCI_STATUS_REQUEST_QOS_NOT_SUPPORT	= 0X27, /* Requested QoS Not Supported */
-	HCI_STATUS_INSTANT_PASSED		= 0X28, /* Instant Passed */
-	HCI_STATUS_PAIRING_UNIT_KEY_NOT_SUPPORT = 0X29, /* Pairing With Unit Key Not Supported */
-	HCI_STATUS_DIFFERENT_TRANSACTION_COLLISION = 0X2a, /* Different Transaction Collision */
-	HCI_STATUS_RESERVE_1			= 0X2b, /* Reserved */
-	HCI_STATUS_QOS_UNACCEPT_PARA		= 0X2c, /* QoS Unacceptable Parameter */
-	HCI_STATUS_QOS_REJECT			= 0X2d, /* QoS Rejected */
-	HCI_STATUS_CHNL_CLASSIFICATION_NOT_SUPPORT = 0X2e, /* Channel Classification Not Supported */
-	HCI_STATUS_INSUFFICIENT_SECURITY	= 0X2f, /* Insufficient Security */
-	HCI_STATUS_PARA_OUT_OF_RANGE		= 0x30, /* Parameter Out Of Mandatory Range */
-	HCI_STATUS_RESERVE_2			= 0X31, /* Reserved */
-	HCI_STATUS_ROLE_SWITCH_PENDING		= 0X32, /* Role Switch Pending */
-	HCI_STATUS_RESERVE_3			= 0X33, /* Reserved */
-	HCI_STATUS_RESERVE_SOLT_VIOLATION	= 0X34, /* Reserved Slot Violation */
-	HCI_STATUS_ROLE_SWITCH_FAIL		= 0X35, /* Role Switch Failed */
-	HCI_STATUS_EXTEND_INQUIRY_RSP_TOO_LARGE	= 0X36, /* Extended Inquiry Response Too Large */
-	HCI_STATUS_SEC_SIMPLE_PAIRING_NOT_SUPPORT = 0X37, /* Secure Simple Pairing Not Supported By Host. */
-	HCI_STATUS_HOST_BUSY_PAIRING		= 0X38, /* Host Busy - Pairing */
-	HCI_STATUS_CONNECT_REJ_NOT_SUIT_CHNL_FOUND = 0X39, /* Connection Rejected due to No Suitable Channel Found */
-	HCI_STATUS_CONTROLLER_BUSY		= 0X3a  /* CONTROLLER BUSY */
-};
-
-/*  */
-/*  The following is for BT 3.0 + HS HCI COMMAND */
-/*  */
-
-/* bit 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 */
-/*	 |	OCF			             |	   OGF       | */
-/*  */
-
-/* OGF 0x01 */
-#define LINK_CONTROL_COMMANDS			0x01
-enum link_control_commands {
-	HCI_INQUIRY					= 0x0001,
-	HCI_INQUIRY_CANCEL				= 0x0002,
-	HCI_PERIODIC_INQUIRY_MODE			= 0x0003,
-	HCI_EXIT_PERIODIC_INQUIRY_MODE			= 0x0004,
-	HCI_CREATE_CONNECTION				= 0x0005,
-	HCI_DISCONNECT					= 0x0006,
-	HCI_CREATE_CONNECTION_CANCEL			= 0x0008,
-	HCI_ACCEPT_CONNECTIONREQUEST			= 0x0009,
-	HCI_REJECT_CONNECTION_REQUEST			= 0x000a,
-	HCI_LINK_KEY_REQUEST_REPLY			= 0x000b,
-	HCI_LINK_KEY_REQUEST_NEGATIVE_REPLY		= 0x000c,
-	HCI_PIN_CODE_REQUEST_REPLY			= 0x000d,
-	HCI_PIN_CODE_REQUEST_NEGATIVE_REPLY		= 0x000e,
-	HCI_CHANGE_CONNECTION_PACKET_TYPE		= 0x000f,
-	HCI_AUTHENTICATION_REQUESTED			= 0x0011,
-	HCI_SET_CONNECTION_ENCRYPTION			= 0x0013,
-	HCI_CHANGE_CONNECTION_LINK_KEY			= 0x0015,
-	HCI_MASTER_LINK_KEY				= 0x0017,
-	HCI_REMOTE_NAME_REQUEST				= 0x0019,
-	HCI_REMOTE_NAME_REQUEST_CANCEL			= 0x001a,
-	HCI_READ_REMOTE_SUPPORTED_FEATURES		= 0x001b,
-	HCI_READ_REMOTE_EXTENDED_FEATURES		= 0x001c,
-	HCI_READ_REMOTE_VERSION_INFORMATION		= 0x001d,
-	HCI_READ_CLOCK_OFFSET				= 0x001f,
-	HCI_READ_LMP_HANDLE				= 0x0020,
-	HCI_SETUP_SYNCHRONOUS_CONNECTION		= 0x0028,
-	HCI_ACCEPT_SYNCHRONOUS_CONNECTION_REQUEST	= 0x0029,
-	HCI_REJECT_SYNCHRONOUS_CONNECTION_REQUEST	= 0x002a,
-	HCI_IO_CAPABILITY_REQUEST_REPLY			= 0x002b,
-	HCI_USER_CONFIRMATION_REQUEST_REPLY		= 0x002c,
-	HCI_USER_CONFIRMATION_REQUEST_NEGATIVE_REPLY	= 0x002d,
-	HCI_USER_PASSKEY_REQUEST_REPLY			= 0x002e,
-	HCI_USER_PASSKEY_REQUESTNEGATIVE_REPLY		= 0x002f,
-	HCI_REMOTE_OOB_DATA_REQUEST_REPLY		= 0x0030,
-	HCI_REMOTE_OOB_DATA_REQUEST_NEGATIVE_REPLY	= 0x0033,
-	HCI_IO_CAPABILITY_REQUEST_NEGATIVE_REPLY	= 0x0034,
-	HCI_CREATE_PHYSICAL_LINK			= 0x0035,
-	HCI_ACCEPT_PHYSICAL_LINK			= 0x0036,
-	HCI_DISCONNECT_PHYSICAL_LINK			= 0x0037,
-	HCI_CREATE_LOGICAL_LINK				= 0x0038,
-	HCI_ACCEPT_LOGICAL_LINK				= 0x0039,
-	HCI_DISCONNECT_LOGICAL_LINK			= 0x003a,
-	HCI_LOGICAL_LINK_CANCEL				= 0x003b,
-	HCI_FLOW_SPEC_MODIFY				= 0x003c
-};
-
-/* OGF 0x02 */
-#define HOLD_MODE_COMMAND				0x02
-enum hold_mode_command {
-	HCI_HOLD_MODE					= 0x0001,
-	HCI_SNIFF_MODE					= 0x0002,
-	HCI_EXIT_SNIFF_MODE				= 0x0003,
-	HCI_PARK_STATE					= 0x0005,
-	HCI_EXIT_PARK_STATE				= 0x0006,
-	HCI_QOS_SETUP					= 0x0007,
-	HCI_ROLE_DISCOVERY				= 0x0009,
-	HCI_SWITCH_ROLE					= 0x000b,
-	HCI_READ_LINK_POLICY_SETTINGS			= 0x000c,
-	HCI_WRITE_LINK_POLICY_SETTINGS			= 0x000d,
-	HCI_READ_DEFAULT_LINK_POLICY_SETTINGS		= 0x000e,
-	HCI_WRITE_DEFAULT_LINK_POLICY_SETTINGS		= 0x000f,
-	HCI_FLOW_SPECIFICATION				= 0x0010,
-	HCI_SNIFF_SUBRATING				= 0x0011
-};
-
-/* OGF 0x03 */
-#define OGF_SET_EVENT_MASK_COMMAND			0x03
-enum set_event_mask_command {
-	HCI_SET_EVENT_MASK				= 0x0001,
-	HCI_RESET					= 0x0003,
-	HCI_SET_EVENT_FILTER				= 0x0005,
-	HCI_FLUSH					= 0x0008,
-	HCI_READ_PIN_TYPE				= 0x0009,
-	HCI_WRITE_PIN_TYPE				= 0x000a,
-	HCI_CREATE_NEW_UNIT_KEY				= 0x000b,
-	HCI_READ_STORED_LINK_KEY			= 0x000d,
-	HCI_WRITE_STORED_LINK_KEY			= 0x0011,
-	HCI_DELETE_STORED_LINK_KEY			= 0x0012,
-	HCI_WRITE_LOCAL_NAME				= 0x0013,
-	HCI_READ_LOCAL_NAME				= 0x0014,
-	HCI_READ_CONNECTION_ACCEPT_TIMEOUT		= 0x0015,
-	HCI_WRITE_CONNECTION_ACCEPT_TIMEOUT		= 0x0016,
-	HCI_READ_PAGE_TIMEOUT				= 0x0017,
-	HCI_WRITE_PAGE_TIMEOUT				= 0x0018,
-	HCI_READ_SCAN_ENABLE				= 0x0019,
-	HCI_WRITE_SCAN_ENABLE				= 0x001a,
-	HCI_READ_PAGE_SCAN_ACTIVITY			= 0x001b,
-	HCI_WRITE_PAGE_SCAN_ACTIVITY			= 0x001c,
-	HCI_READ_INQUIRY_SCAN_ACTIVITY			= 0x001d,
-	HCI_WRITE_INQUIRY_SCAN_ACTIVITY			= 0x001e,
-	HCI_READ_AUTHENTICATION_ENABLE			= 0x001f,
-	HCI_WRITE_AUTHENTICATION_ENABLE			= 0x0020,
-	HCI_READ_CLASS_OF_DEVICE			= 0x0023,
-	HCI_WRITE_CLASS_OF_DEVICE			= 0x0024,
-	HCI_READ_VOICE_SETTING				= 0x0025,
-	HCI_WRITE_VOICE_SETTING				= 0x0026,
-	HCI_READ_AUTOMATIC_FLUSH_TIMEOUT		= 0x0027,
-	HCI_WRITE_AUTOMATIC_FLUSH_TIMEOUT		= 0x0028,
-	HCI_READ_NUM_BROADCAST_RETRANSMISSIONS		= 0x0029,
-	HCI_WRITE_NUM_BROADCAST_RETRANSMISSIONS		= 0x002a,
-	HCI_READ_HOLD_MODE_ACTIVITY			= 0x002b,
-	HCI_WRITE_HOLD_MODE_ACTIVITY			= 0x002c,
-	HCI_READ_SYNCHRONOUS_FLOW_CONTROL_ENABLE	= 0x002e,
-	HCI_WRITE_SYNCHRONOUS_FLOW_CONTROL_ENABLE	= 0x002f,
-	HCI_SET_CONTROLLER_TO_HOST_FLOW_CONTROL		= 0x0031,
-	HCI_HOST_BUFFER_SIZE				= 0x0033,
-	HCI_HOST_NUMBER_OF_COMPLETED_PACKETS		= 0x0035,
-	HCI_READ_LINK_SUPERVISION_TIMEOUT		= 0x0036,
-	HCI_WRITE_LINK_SUPERVISION_TIMEOUT		= 0x0037,
-	HCI_READ_NUMBER_OF_SUPPORTED_IAC		= 0x0038,
-	HCI_READ_CURRENT_IAC_LAP			= 0x0039,
-	HCI_WRITE_CURRENT_IAC_LAP			= 0x003a,
-	HCI_READ_PAGE_SCAN_MODE				= 0x003d,
-	HCI_WRITE_PAGE_SCAN_MODE			= 0x003e,
-	HCI_SET_AFH_HOST_CHANNEL_CLASSIFICATION		= 0x003f,
-	HCI_READ_INQUIRY_SCAN_TYPE			= 0x0042,
-	HCI_WRITE_INQUIRY_SCAN_TYPE			= 0x0043,
-	HCI_READ_INQUIRY_MODE				= 0x0044,
-	HCI_WRITE_INQUIRY_MODE				= 0x0045,
-	HCI_READ_PAGE_SCAN_TYPE				= 0x0046,
-	HCI_WRITE_PAGE_SCAN_TYPE			= 0x0047,
-	HCI_READ_AFH_CHANNEL_ASSESSMENT_MODE		= 0x0048,
-	HCI_WRITE_AFH_CHANNEL_ASSESSMENT_MODE		= 0x0049,
-	HCI_READ_EXTENDED_INQUIRY_RESPONSE		= 0x0051,
-	HCI_WRITE_EXTENDED_INQUIRY_RESPONSE		= 0x0052,
-	HCI_REFRESH_ENCRYPTION_KEY			= 0x0053,
-	HCI_READ_SIMPLE_PAIRING_MODE			= 0x0055,
-	HCI_WRITE_SIMPLE_PAIRING_MODE			= 0x0056,
-	HCI_READ_LOCAL_OOB_DATA				= 0x0057,
-	HCI_READ_INQUIRY_RESPONSE_TRANSMIT_POWER_LEVEL	= 0x0058,
-	HCI_WRITE_INQUIRY_TRANSMIT_POWER_LEVEL		= 0x0059,
-	HCI_READ_DEFAULT_ERRONEOUS_DATA_REPORTING	= 0x005a,
-	HCI_WRITE_DEFAULT_ERRONEOUS_DATA_REPORTING	= 0x005b,
-	HCI_ENHANCED_FLUSH				= 0x005f,
-	HCI_SEND_KEYPRESS_NOTIFICATION			= 0x0060,
-	HCI_READ_LOGICAL_LINK_ACCEPT_TIMEOUT		= 0x0061,
-	HCI_WRITE_LOGICAL_LINK_ACCEPT_TIMEOUT		= 0x0062,
-	HCI_SET_EVENT_MASK_PAGE_2			= 0x0063,
-	HCI_READ_LOCATION_DATA				= 0x0064,
-	HCI_WRITE_LOCATION_DATA				= 0x0065,
-	HCI_READ_FLOW_CONTROL_MODE			= 0x0066,
-	HCI_WRITE_FLOW_CONTROL_MODE			= 0x0067,
-	HCI_READ_ENHANCE_TRANSMIT_POWER_LEVEL		= 0x0068,
-	HCI_READ_BEST_EFFORT_FLUSH_TIMEOUT		= 0x0069,
-	HCI_WRITE_BEST_EFFORT_FLUSH_TIMEOUT		= 0x006a,
-	HCI_SHORT_RANGE_MODE				= 0x006b
-};
-
-/* OGF 0x04 */
-#define OGF_INFORMATIONAL_PARAMETERS			0x04
-enum informational_params {
-	HCI_READ_LOCAL_VERSION_INFORMATION		= 0x0001,
-	HCI_READ_LOCAL_SUPPORTED_COMMANDS		= 0x0002,
-	HCI_READ_LOCAL_SUPPORTED_FEATURES		= 0x0003,
-	HCI_READ_LOCAL_EXTENDED_FEATURES		= 0x0004,
-	HCI_READ_BUFFER_SIZE				= 0x0005,
-	HCI_READ_BD_ADDR				= 0x0009,
-	HCI_READ_DATA_BLOCK_SIZE			= 0x000a
-};
-
-/* OGF 0x05 */
-#define OGF_STATUS_PARAMETERS				0x05
-enum status_params {
-	HCI_READ_FAILED_CONTACT_COUNTER			= 0x0001,
-	HCI_RESET_FAILED_CONTACT_COUNTER		= 0x0002,
-	HCI_READ_LINK_QUALITY				= 0x0003,
-	HCI_READ_RSSI					= 0x0005,
-	HCI_READ_AFH_CHANNEL_MAP			= 0x0006,
-	HCI_READ_CLOCK					= 0x0007,
-	HCI_READ_ENCRYPTION_KEY_SIZE			= 0x0008,
-	HCI_READ_LOCAL_AMP_INFO				= 0x0009,
-	HCI_READ_LOCAL_AMP_ASSOC			= 0x000a,
-	HCI_WRITE_REMOTE_AMP_ASSOC			= 0x000b
-};
-
-/* OGF 0x06 */
-#define OGF_TESTING_COMMANDS				0x06
-enum testing_commands {
-	HCI_READ_LOOPBACK_MODE				= 0x0001,
-	HCI_WRITE_LOOPBACK_MODE				= 0x0002,
-	HCI_ENABLE_DEVICE_UNDER_TEST_MODE		= 0x0003,
-	HCI_WRITE_SIMPLE_PAIRING_DEBUG_MODE		= 0x0004,
-	HCI_ENABLE_AMP_RECEIVER_REPORTS			= 0x0007,
-	HCI_AMP_TEST_END				= 0x0008,
-	HCI_AMP_TEST_COMMAND				= 0x0009
-};
-
-/* OGF 0x3f */
-#define OGF_EXTENSION					0X3f
-enum hci_extension_commands {
-	HCI_SET_ACL_LINK_DATA_FLOW_MODE			= 0x0010,
-	HCI_SET_ACL_LINK_STATUS				= 0x0020,
-	HCI_SET_SCO_LINK_STATUS				= 0x0030,
-	HCI_SET_RSSI_VALUE				= 0x0040,
-	HCI_SET_CURRENT_BLUETOOTH_STATUS		= 0x0041,
-
-	/* The following is for RTK8723 */
-	HCI_EXTENSION_VERSION_NOTIFY			= 0x0100,
-	HCI_LINK_STATUS_NOTIFY				= 0x0101,
-	HCI_BT_OPERATION_NOTIFY				= 0x0102,
-	HCI_ENABLE_WIFI_SCAN_NOTIFY			= 0x0103,
-
-
-	/* The following is for IVT */
-	HCI_WIFI_CURRENT_CHANNEL			= 0x0300,
-	HCI_WIFI_CURRENT_BANDWIDTH			= 0x0301,
-	HCI_WIFI_CONNECTION_STATUS			= 0x0302,
-};
-
-enum bt_spec {
-	BT_SPEC_1_0_b					= 0x00,
-	BT_SPEC_1_1					= 0x01,
-	BT_SPEC_1_2					= 0x02,
-	BT_SPEC_2_0_EDR					= 0x03,
-	BT_SPEC_2_1_EDR					= 0x04,
-	BT_SPEC_3_0_HS					= 0x05,
-	BT_SPEC_4_0					= 0x06
-};
-
-/*  The following is for BT 3.0 + HS EVENTS */
-enum hci_event {
-	HCI_EVENT_INQUIRY_COMPLETE			= 0x01,
-	HCI_EVENT_INQUIRY_RESULT			= 0x02,
-	HCI_EVENT_CONNECTION_COMPLETE			= 0x03,
-	HCI_EVENT_CONNECTION_REQUEST			= 0x04,
-	HCI_EVENT_DISCONNECTION_COMPLETE		= 0x05,
-	HCI_EVENT_AUTHENTICATION_COMPLETE		= 0x06,
-	HCI_EVENT_REMOTE_NAME_REQUEST_COMPLETE		= 0x07,
-	HCI_EVENT_ENCRYPTION_CHANGE			= 0x08,
-	HCI_EVENT_CHANGE_LINK_KEY_COMPLETE		= 0x09,
-	HCI_EVENT_MASTER_LINK_KEY_COMPLETE		= 0x0a,
-	HCI_EVENT_READ_REMOTE_SUPPORT_FEATURES_COMPLETE	= 0x0b,
-	HCI_EVENT_READ_REMOTE_VER_INFO_COMPLETE		= 0x0c,
-	HCI_EVENT_QOS_SETUP_COMPLETE			= 0x0d,
-	HCI_EVENT_COMMAND_COMPLETE			= 0x0e,
-	HCI_EVENT_COMMAND_STATUS			= 0x0f,
-	HCI_EVENT_HARDWARE_ERROR			= 0x10,
-	HCI_EVENT_FLUSH_OCCRUED				= 0x11,
-	HCI_EVENT_ROLE_CHANGE				= 0x12,
-	HCI_EVENT_NUMBER_OF_COMPLETE_PACKETS		= 0x13,
-	HCI_EVENT_MODE_CHANGE				= 0x14,
-	HCI_EVENT_RETURN_LINK_KEYS			= 0x15,
-	HCI_EVENT_PIN_CODE_REQUEST			= 0x16,
-	HCI_EVENT_LINK_KEY_REQUEST			= 0x17,
-	HCI_EVENT_LINK_KEY_NOTIFICATION			= 0x18,
-	HCI_EVENT_LOOPBACK_COMMAND			= 0x19,
-	HCI_EVENT_DATA_BUFFER_OVERFLOW			= 0x1a,
-	HCI_EVENT_MAX_SLOTS_CHANGE			= 0x1b,
-	HCI_EVENT_READ_CLOCK_OFFSET_COMPLETE		= 0x1c,
-	HCI_EVENT_CONNECT_PACKET_TYPE_CHANGE		= 0x1d,
-	HCI_EVENT_QOS_VIOLATION				= 0x1e,
-	HCI_EVENT_PAGE_SCAN_REPETITION_MODE_CHANGE	= 0x20,
-	HCI_EVENT_FLOW_SEPC_COMPLETE			= 0x21,
-	HCI_EVENT_INQUIRY_RESULT_WITH_RSSI		= 0x22,
-	HCI_EVENT_READ_REMOTE_EXT_FEATURES_COMPLETE	= 0x23,
-	HCI_EVENT_SYNC_CONNECT_COMPLETE			= 0x2c,
-	HCI_EVENT_SYNC_CONNECT_CHANGE			= 0x2d,
-	HCI_EVENT_SNIFFER_SUBRATING			= 0x2e,
-	HCI_EVENT_EXTENTED_INQUIRY_RESULT		= 0x2f,
-	HCI_EVENT_ENCRYPTION_KEY_REFLASH_COMPLETE	= 0x30,
-	HCI_EVENT_IO_CAPIBILITY_COMPLETE		= 0x31,
-	HCI_EVENT_IO_CAPIBILITY_RESPONSE		= 0x32,
-	HCI_EVENT_USER_CONFIRMTION_REQUEST		= 0x33,
-	HCI_EVENT_USER_PASSKEY_REQUEST			= 0x34,
-	HCI_EVENT_REMOTE_OOB_DATA_REQUEST		= 0x35,
-	HCI_EVENT_SIMPLE_PAIRING_COMPLETE		= 0x36,
-	HCI_EVENT_LINK_SUPERVISION_TIMEOUT_CHANGE	= 0x38,
-	HCI_EVENT_ENHANCED_FLUSH_COMPLETE		= 0x39,
-	HCI_EVENT_USER_PASSKEY_NOTIFICATION		= 0x3b,
-	HCI_EVENT_KEYPRESS_NOTIFICATION			= 0x3c,
-	HCI_EVENT_REMOTE_HOST_SUPPORT_FEATURES_NOTIFICATION	= 0x3d,
-	HCI_EVENT_PHY_LINK_COMPLETE			= 0x40,
-	HCI_EVENT_CHANNEL_SELECT			= 0x41,
-	HCI_EVENT_DISCONNECT_PHY_LINK_COMPLETE		= 0x42,
-	HCI_EVENT_PHY_LINK_LOSS_EARLY_WARNING		= 0x43,
-	HCI_EVENT_PHY_LINK_RECOVER			= 0x44,
-	HCI_EVENT_LOGICAL_LINK_COMPLETE			= 0x45,
-	HCI_EVENT_DISCONNECT_LOGICAL_LINK_COMPLETE	= 0x46,
-	HCI_EVENT_FLOW_SPEC_MODIFY_COMPLETE		= 0x47,
-	HCI_EVENT_NUM_OF_COMPLETE_DATA_BLOCKS		= 0x48,
-	HCI_EVENT_AMP_START_TEST			= 0x49,
-	HCI_EVENT_AMP_TEST_END				= 0x4a,
-	HCI_EVENT_AMP_RECEIVER_REPORT			= 0x4b,
-	HCI_EVENT_SHORT_RANGE_MODE_CHANGE_COMPLETE	= 0x4c,
-	HCI_EVENT_AMP_STATUS_CHANGE			= 0x4d,
-	HCI_EVENT_EXTENSION_RTK				= 0xfe,
-	HCI_EVENT_EXTENSION_MOTO			= 0xff,
-};
-
-enum hci_extension_event_moto {
-	HCI_EVENT_GET_BT_RSSI				= 0x01,
-};
-
-enum hci_extension_event {
-	HCI_EVENT_EXT_WIFI_SCAN_NOTIFY			= 0x01,
-};
-
-enum hci_event_mask_page_2 {
-	EMP2_HCI_EVENT_PHY_LINK_COMPLETE		= 0x0000000000000001,
-	EMP2_HCI_EVENT_CHANNEL_SELECT			= 0x0000000000000002,
-	EMP2_HCI_EVENT_DISCONNECT_PHY_LINK_COMPLETE	= 0x0000000000000004,
-	EMP2_HCI_EVENT_PHY_LINK_LOSS_EARLY_WARNING	= 0x0000000000000008,
-	EMP2_HCI_EVENT_PHY_LINK_RECOVER			= 0x0000000000000010,
-	EMP2_HCI_EVENT_LOGICAL_LINK_COMPLETE		= 0x0000000000000020,
-	EMP2_HCI_EVENT_DISCONNECT_LOGICAL_LINK_COMPLETE	= 0x0000000000000040,
-	EMP2_HCI_EVENT_FLOW_SPEC_MODIFY_COMPLETE	= 0x0000000000000080,
-	EMP2_HCI_EVENT_NUM_OF_COMPLETE_DATA_BLOCKS	= 0x0000000000000100,
-	EMP2_HCI_EVENT_AMP_START_TEST			= 0x0000000000000200,
-	EMP2_HCI_EVENT_AMP_TEST_END			= 0x0000000000000400,
-	EMP2_HCI_EVENT_AMP_RECEIVER_REPORT		= 0x0000000000000800,
-	EMP2_HCI_EVENT_SHORT_RANGE_MODE_CHANGE_COMPLETE	= 0x0000000000001000,
-	EMP2_HCI_EVENT_AMP_STATUS_CHANGE		= 0x0000000000002000,
-};
-
-enum hci_state_machine {
-	HCI_STATE_STARTING			= 0x01,
-	HCI_STATE_CONNECTING			= 0x02,
-	HCI_STATE_AUTHENTICATING		= 0x04,
-	HCI_STATE_CONNECTED			= 0x08,
-	HCI_STATE_DISCONNECTING			= 0x10,
-	HCI_STATE_DISCONNECTED			= 0x20
-};
-
-enum amp_assoc_structure_type {
-	AMP_MAC_ADDR				= 0x01,
-	AMP_PREFERRED_CHANNEL_LIST		= 0x02,
-	AMP_CONNECTED_CHANNEL			= 0x03,
-	AMP_80211_PAL_CAP_LIST			= 0x04,
-	AMP_80211_PAL_VISION			= 0x05,
-	AMP_RESERVED_FOR_TESTING		= 0x33
-};
-
-enum amp_btap_type {
-	AMP_BTAP_NONE,
-	AMP_BTAP_CREATOR,
-	AMP_BTAP_JOINER
-};
-
-enum hci_state_with_cmd {
-	STATE_CMD_CREATE_PHY_LINK,
-	STATE_CMD_ACCEPT_PHY_LINK,
-	STATE_CMD_DISCONNECT_PHY_LINK,
-	STATE_CMD_CONNECT_ACCEPT_TIMEOUT,
-	STATE_CMD_MAC_START_COMPLETE,
-	STATE_CMD_MAC_START_FAILED,
-	STATE_CMD_MAC_CONNECT_COMPLETE,
-	STATE_CMD_MAC_CONNECT_FAILED,
-	STATE_CMD_MAC_DISCONNECT_INDICATE,
-	STATE_CMD_MAC_CONNECT_CANCEL_INDICATE,
-	STATE_CMD_4WAY_FAILED,
-	STATE_CMD_4WAY_SUCCESSED,
-	STATE_CMD_ENTER_STATE,
-	STATE_CMD_NO_SUCH_CMD,
-};
-
-enum hci_service_type {
-	SERVICE_NO_TRAFFIC,
-	SERVICE_BEST_EFFORT,
-	SERVICE_GUARANTEE
-};
-
-enum hci_traffic_mode {
-	TRAFFIC_MODE_BEST_EFFORT			= 0x00,
-	TRAFFIC_MODE_GUARANTEED_LATENCY			= 0x01,
-	TRAFFIC_MODE_GUARANTEED_BANDWIDTH		= 0x02,
-	TRAFFIC_MODE_GUARANTEED_LATENCY_AND_BANDWIDTH	= 0x03
-};
-
-#define HCIOPCODE(_OCF, _OGF)		(_OGF<<10|_OCF)
-#define HCIOPCODELOW(_OCF, _OGF)	(u8)(HCIOPCODE(_OCF, _OGF)&0x00ff)
-#define HCIOPCODEHIGHT(_OCF, _OGF)	(u8)(HCIOPCODE(_OCF, _OGF)>>8)
-
-#define TWOBYTE_HIGHTBYTE(_DATA)	(u8)(_DATA>>8)
-#define TWOBYTE_LOWBYTE(_DATA)		(u8)(_DATA)
-
-enum amp_status {
-	AMP_STATUS_AVA_PHY_PWR_DWN		= 0x0,
-	AMP_STATUS_BT_USE_ONLY			= 0x1,
-	AMP_STATUS_NO_CAPACITY_FOR_BT		= 0x2,
-	AMP_STATUS_LOW_CAPACITY_FOR_BT		= 0x3,
-	AMP_STATUS_MEDIUM_CAPACITY_FOR_BT	= 0x4,
-	AMP_STATUS_HIGH_CAPACITY_FOR_BT		= 0x5,
-	AMP_STATUS_FULL_CAPACITY_FOR_BT		= 0x6
-};
-
-enum bt_wpa_msg_type {
-	Type_BT_4way1st	= 0,
-	Type_BT_4way2nd	= 1,
-	Type_BT_4way3rd	= 2,
-	Type_BT_4way4th	= 3,
-	Type_BT_unknow	= 4
-};
-
-enum bt_connect_type {
-	BT_CONNECT_AUTH_REQ			= 0x00,
-	BT_CONNECT_AUTH_RSP			= 0x01,
-	BT_CONNECT_ASOC_REQ			= 0x02,
-	BT_CONNECT_ASOC_RSP			= 0x03,
-	BT_DISCONNECT				= 0x04
-};
-
-enum bt_ll_service_type {
-	BT_LL_BE = 0x01,
-	BT_LL_GU = 0x02
-};
-
-enum bt_ll_flowspec {
-	BT_TX_BE_FS,			/* TX best effort flowspec */
-	BT_RX_BE_FS,			/* RX best effort flowspec */
-	BT_TX_GU_FS,			/* TX guaranteed latency flowspec */
-	BT_RX_GU_FS,			/* RX guaranteed latency flowspec */
-	BT_TX_BE_AGG_FS,		/* TX aggregated best effort flowspec */
-	BT_RX_BE_AGG_FS,		/* RX aggregated best effort flowspec */
-	BT_TX_GU_BW_FS,			/* TX guaranteed bandwidth flowspec */
-	BT_RX_GU_BW_FS,			/* RX guaranteed bandwidth flowspec */
-	BT_TX_GU_LARGE_FS,		/* TX guaranteed latency flowspec, for testing only */
-	BT_RX_GU_LARGE_FS,		/* RX guaranteed latency flowspec, for testing only */
-};
-
-enum bt_traffic_mode {
-	BT_MOTOR_EXT_BE		= 0x00, /* Best Effort. Default. for HCRP, PAN, SDP, RFCOMM-based profiles like FTP, OPP, SPP, DUN, etc. */
-	BT_MOTOR_EXT_GUL	= 0x01, /* Guaranteed Latency. This type of traffic is used e.g. for HID and AVRCP. */
-	BT_MOTOR_EXT_GUB	= 0X02, /* Guaranteed Bandwidth. */
-	BT_MOTOR_EXT_GULB	= 0X03  /* Guaranteed Latency and Bandwidth. for A2DP and VDP. */
-};
-
-enum bt_traffic_mode_profile {
-	BT_PROFILE_NONE,
-	BT_PROFILE_A2DP,
-	BT_PROFILE_PAN,
-	BT_PROFILE_HID,
-	BT_PROFILE_SCO
-};
-
-enum bt_link_role {
-	BT_LINK_MASTER	= 0,
-	BT_LINK_SLAVE	= 1
-};
-
-enum bt_state_wpa_auth {
-	STATE_WPA_AUTH_UNINITIALIZED,
-	STATE_WPA_AUTH_WAIT_PACKET_1, /*  Join */
-	STATE_WPA_AUTH_WAIT_PACKET_2, /*  Creat */
-	STATE_WPA_AUTH_WAIT_PACKET_3,
-	STATE_WPA_AUTH_WAIT_PACKET_4,
-	STATE_WPA_AUTH_SUCCESSED
-};
-
-#define BT_WPA_AUTH_TIMEOUT_PERIOD		1000
-#define BTMaxWPAAuthReTransmitCoun		5
-
-#define MAX_AMP_ASSOC_FRAG_LEN			248
-#define TOTAL_ALLOCIATE_ASSOC_LEN			1000
-
-struct hci_flow_spec {
-	u8				Identifier;
-	u8				ServiceType;
-	u16				MaximumSDUSize;
-	u32				SDUInterArrivalTime;
-	u32				AccessLatency;
-	u32				FlushTimeout;
-};
-
-struct hci_log_link_cmd_data {
-	u8				BtPhyLinkhandle;
-	u16				BtLogLinkhandle;
-	u8				BtTxFlowSpecID;
-	struct hci_flow_spec		Tx_Flow_Spec;
-	struct hci_flow_spec		Rx_Flow_Spec;
-	u32				TxPacketCount;
-	u32				BestEffortFlushTimeout;
-
-	u8				bLLCompleteEventIsSet;
-
-	u8				bLLCancelCMDIsSetandComplete;
-};
-
-struct hci_phy_link_cmd_data {
-	/* Physical_Link_Handle */
-	u8				BtPhyLinkhandle;
-
-	u16				LinkSuperversionTimeout;
-
-	/* u16				SuperTimeOutCnt; */
-
-	/* Dedicated_AMP_Key_Length */
-	u8				BtAMPKeyLen;
-	/* Dedicated_AMP_Key_Type */
-	u8				BtAMPKeyType;
-	/* Dedicated_AMP_Key */
-	u8				BtAMPKey[PMK_LEN];
-};
-
-struct amp_assoc_structure {
-	/* TYPE ID */
-	u8				TypeID;
-	/* Length */
-	u16				Length;
-	/* Value */
-	u8				Data[1];
-};
-
-struct amp_pref_chnl_regulatory {
-	u8				reXId;
-	u8				regulatoryClass;
-	u8				coverageClass;
-};
-
-struct amp_assoc_cmd_data {
-	/* Physical_Link_Handle */
-	u8				BtPhyLinkhandle;
-	/* Length_So_Far */
-	u16				LenSoFar;
-
-	u16				MaxRemoteASSOCLen;
-	/* AMP_ASSOC_Remaining_Length */
-	u16				AMPAssocRemLen;
-	/* AMP_ASSOC_fragment */
-	void				*AMPAssocfragment;
-};
-
-struct hci_link_info {
-	u16				ConnectHandle;
-	u8				IncomingTrafficMode;
-	u8				OutgoingTrafficMode;
-	u8				BTProfile;
-	u8				BTCoreSpec;
-	s8				BT_RSSI;
-	u8				TrafficProfile;
-	u8				linkRole;
-};
-
-struct hci_ext_config {
-	struct hci_link_info		linkInfo[MAX_BT_ASOC_ENTRY_NUM];
-	u8				btOperationCode;
-	u16				CurrentConnectHandle;
-	u8				CurrentIncomingTrafficMode;
-	u8				CurrentOutgoingTrafficMode;
-	s8				MIN_BT_RSSI;
-	u8				NumberOfHandle;
-	u8				NumberOfSCO;
-	u8				CurrentBTStatus;
-	u16				HCIExtensionVer;
-
-	/* Bt coexist related */
-	u8				btProfileCase;
-	u8				btProfileAction;
-	u8				bManualControl;
-	u8				bBTBusy;
-	u8				bBTA2DPBusy;
-	u8				bEnableWifiScanNotify;
-
-	u8				bHoldForBtOperation;
-	u32				bHoldPeriodCnt;
-};
-
-struct hci_acl_packet_data {
-	u16				ACLDataPacketLen;
-	u8				SyncDataPacketLen;
-	u16				TotalNumACLDataPackets;
-	u16				TotalSyncNumDataPackets;
-};
-
-struct hci_phy_link_bss_info {
-	u16				bdCap;	/*  capability information */
-};
-
-struct packet_irp_hcicmd_data {
-	u16		OCF:10;
-	u16		OGF:6;
-	u8		Length;
-	u8		Data[20];
-};
-
-struct bt_asoc_entry {
-	u8						bUsed;
-	u8						mAssoc;
-	u8						b4waySuccess;
-	u8						Bssid[6];
-	struct hci_phy_link_cmd_data		PhyLinkCmdData;
-
-	struct hci_log_link_cmd_data		LogLinkCmdData[MAX_LOGICAL_LINK_NUM];
-
-	struct hci_acl_packet_data			ACLPacketsData;
-
-	struct amp_assoc_cmd_data		AmpAsocCmdData;
-	struct octet_string				BTSsid;
-	u8						BTSsidBuf[33];
-
-	enum hci_status						PhyLinkDisconnectReason;
-
-	u8						bSendSupervisionPacket;
-	/* u8						CurrentSuervisionPacketSendNum; */
-	/* u8						LastSuervisionPacketSendNum; */
-	u32						NoRxPktCnt;
-	/* Is Creator or Joiner */
-	enum amp_btap_type				AMPRole;
-
-	/* BT current state */
-	u8						BtCurrentState;
-	/* BT next state */
-	u8						BtNextState;
-
-	u8						bNeedPhysLinkCompleteEvent;
-
-	enum hci_status					PhysLinkCompleteStatus;
-
-	u8						BTRemoteMACAddr[6];
-
-	u32						BTCapability;
-
-	u8						SyncDataPacketLen;
-
-	u16						TotalSyncNumDataPackets;
-	u16						TotalNumACLDataPackets;
-
-	u8						ShortRangeMode;
-
-	u8						PTK[PTK_LEN_TKIP];
-	u8						GTK[GTK_LEN];
-	u8						ANonce[KEY_NONCE_LEN];
-	u8						SNonce[KEY_NONCE_LEN];
-	u64						KeyReplayCounter;
-	u8						WPAAuthReplayCount;
-	u8						AESKeyBuf[AESCCMP_BLK_SIZE_TOTAL];
-	u8						PMK[PMK_LEN];
-	enum bt_state_wpa_auth			BTWPAAuthState;
-	s32						UndecoratedSmoothedPWDB;
-
-	/*  Add for HW security !! */
-	u8						HwCAMIndex;  /*  Cam index */
-	u8						bPeerQosSta;
-
-	u32						rxSuvpPktCnt;
-};
-
-struct bt_traffic_statistics {
-	u8				bTxBusyTraffic;
-	u8				bRxBusyTraffic;
-	u8				bIdle;
-	u32				TxPktCntInPeriod;
-	u32				RxPktCntInPeriod;
-	u64				TxPktLenInPeriod;
-	u64				RxPktLenInPeriod;
-};
-
-struct bt_mgnt {
-	u8				bBTConnectInProgress;
-	u8				bLogLinkInProgress;
-	u8				bPhyLinkInProgress;
-	u8				bPhyLinkInProgressStartLL;
-	u8				BtCurrentPhyLinkhandle;
-	u16				BtCurrentLogLinkhandle;
-	u8				CurrentConnectEntryNum;
-	u8				DisconnectEntryNum;
-	u8				CurrentBTConnectionCnt;
-	enum bt_connect_type		BTCurrentConnectType;
-	enum bt_connect_type		BTReceiveConnectPkt;
-	u8				BTAuthCount;
-	u8				BTAsocCount;
-	u8				bStartSendSupervisionPkt;
-	u8				BtOperationOn;
-	u8				BTNeedAMPStatusChg;
-	u8				JoinerNeedSendAuth;
-	struct hci_phy_link_bss_info	bssDesc;
-	struct hci_ext_config		ExtConfig;
-	u8				bNeedNotifyAMPNoCap;
-	u8				bCreateSpportQos;
-	u8				bSupportProfile;
-	u8				BTChannel;
-	u8				CheckChnlIsSuit;
-	u8				bBtScan;
-	u8				btLogoTest;
-};
-
-struct bt_hci_dgb_info {
-	u32				hciCmdCnt;
-	u32				hciCmdCntUnknown;
-	u32				hciCmdCntCreatePhyLink;
-	u32				hciCmdCntAcceptPhyLink;
-	u32				hciCmdCntDisconnectPhyLink;
-	u32				hciCmdPhyLinkStatus;
-	u32				hciCmdCntCreateLogLink;
-	u32				hciCmdCntAcceptLogLink;
-	u32				hciCmdCntDisconnectLogLink;
-	u32				hciCmdCntReadLocalAmpAssoc;
-	u32				hciCmdCntWriteRemoteAmpAssoc;
-	u32				hciCmdCntSetAclLinkStatus;
-	u32				hciCmdCntSetScoLinkStatus;
-	u32				hciCmdCntExtensionVersionNotify;
-	u32				hciCmdCntLinkStatusNotify;
-};
-
-struct bt_irp_dgb_info {
-	u32				irpMJCreate;
-	/*  Io Control */
-	u32				irpIoControl;
-	u32				irpIoCtrlHciCmd;
-	u32				irpIoCtrlHciEvent;
-	u32				irpIoCtrlHciTxData;
-	u32				irpIoCtrlHciRxData;
-	u32				irpIoCtrlUnknown;
-
-	u32				irpIoCtrlHciTxData1s;
-};
-
-struct bt_packet_dgb_info {
-	u32				btPktTxProbReq;
-	u32				btPktRxProbReq;
-	u32				btPktRxProbReqFail;
-	u32				btPktTxProbRsp;
-	u32				btPktRxProbRsp;
-	u32				btPktTxAuth;
-	u32				btPktRxAuth;
-	u32				btPktRxAuthButDrop;
-	u32				btPktTxAssocReq;
-	u32				btPktRxAssocReq;
-	u32				btPktRxAssocReqButDrop;
-	u32				btPktTxAssocRsp;
-	u32				btPktRxAssocRsp;
-	u32				btPktTxDisassoc;
-	u32				btPktRxDisassoc;
-	u32				btPktRxDeauth;
-	u32				btPktTx4way1st;
-	u32				btPktRx4way1st;
-	u32				btPktTx4way2nd;
-	u32				btPktRx4way2nd;
-	u32				btPktTx4way3rd;
-	u32				btPktRx4way3rd;
-	u32				btPktTx4way4th;
-	u32				btPktRx4way4th;
-	u32				btPktTxLinkSuperReq;
-	u32				btPktRxLinkSuperReq;
-	u32				btPktTxLinkSuperRsp;
-	u32				btPktRxLinkSuperRsp;
-	u32				btPktTxData;
-	u32				btPktRxData;
-};
-
-struct bt_dgb {
-	u8				dbgCtrl;
-	u32				dbgProfile;
-	struct bt_hci_dgb_info		dbgHciInfo;
-	struct bt_irp_dgb_info		dbgIrpInfo;
-	struct bt_packet_dgb_info	dbgBtPkt;
-};
-
-struct bt_hci_info {
-	/* 802.11 Pal version specifier */
-	u8				BTPalVersion;
-	u16				BTPalCompanyID;
-	u16				BTPalsubversion;
-
-	/* Connected channel list */
-	u16				BTConnectChnlListLen;
-	u8				BTConnectChnllist[64];
-
-	/* Fail contact counter */
-	u16				FailContactCount;
-
-	/* Event mask */
-	u64				BTEventMask;
-	u64				BTEventMaskPage2;
-
-	/* timeout var */
-	u16				ConnAcceptTimeout;
-	u16				LogicalAcceptTimeout;
-	u16				PageTimeout;
-
-	u8				LocationDomainAware;
-	u16				LocationDomain;
-	u8				LocationDomainOptions;
-	u8				LocationOptions;
-
-	u8				FlowControlMode;
-
-	/* Preferred channel list */
-	u16				BtPreChnlListLen;
-	u8				BTPreChnllist[64];
-
-	u16				enFlush_LLH;	/* enhanced flush handle */
-	u16				FLTO_LLH;		/* enhanced flush handle */
-
-	/*  */
-	/* Test command only. */
-	u8				bInTestMode;
-	u8				bTestIsEnd;
-	u8				bTestNeedReport;
-	u8				TestScenario;
-	u8				TestReportInterval;
-	u8				TestCtrType;
-	u32				TestEventType;
-	u16				TestNumOfFrame;
-	u16				TestNumOfErrFrame;
-	u16				TestNumOfBits;
-	u16				TestNumOfErrBits;
-	/*  */
-};
-
-struct bt_traffic {
-	/*  Add for check replay data */
-	u8					LastRxUniFragNum;
-	u16					LastRxUniSeqNum;
-
-	/* s32					EntryMaxUndecoratedSmoothedPWDB; */
-	/* s32					EntryMinUndecoratedSmoothedPWDB; */
-
-	struct bt_traffic_statistics		Bt30TrafficStatistics;
-};
-
-#define RT_WORK_ITEM struct work_struct
-
-struct bt_security {
-	/*  WPA auth state
-	 *  May need to remove to BTSecInfo ... 
-	 * enum bt_state_wpa_auth BTWPAAuthState;
-	 */
-	struct octet_string	RSNIE;
-	u8			RSNIEBuf[MAXRSNIELEN];
-	u8			bRegNoEncrypt;
-	u8			bUsedHwEncrypt;
-};
-
-struct bt_30info {
-	struct rtw_adapter	*padapter;
-	struct bt_asoc_entry		BtAsocEntry[MAX_BT_ASOC_ENTRY_NUM];
-	struct bt_mgnt				BtMgnt;
-	struct bt_dgb				BtDbg;
-	struct bt_hci_info			BtHciInfo;
-	struct bt_traffic			BtTraffic;
-	struct bt_security			BtSec;
-	RT_WORK_ITEM		HCICmdWorkItem;
-	struct timer_list BTHCICmdTimer;
-	RT_WORK_ITEM		BTPsDisableWorkItem;
-	RT_WORK_ITEM		BTConnectWorkItem;
-	struct timer_list BTHCIDiscardAclDataTimer;
-	struct timer_list BTHCIJoinTimeoutTimer;
-	struct timer_list BTTestSendPacketTimer;
-	struct timer_list BTDisconnectPhyLinkTimer;
-	struct timer_list BTBeaconTimer;
-	u8				BTBeaconTmrOn;
-
-	struct timer_list BTPsDisableTimer;
-
-	void *				pBtChnlList;
-};
-
-struct packet_irp_acl_data {
-	u16		Handle:12;
-	u16		PB_Flag:2;
-	u16		BC_Flag:2;
-	u16		Length;
-	u8		Data[1];
-};
-
-struct packet_irp_hcievent_data {
-	u8		EventCode;
-	u8		Length;
-	u8		Data[20];
-};
-
-struct common_triple {
-	u8 byte_1st;
-	u8 byte_2nd;
-	u8 byte_3rd;
-};
-
-#define COUNTRY_STR_LEN		3	/*  country string len = 3 */
-
-#define LOCAL_PMK	0
-
-enum hci_wifi_connect_status {
-	HCI_WIFI_NOT_CONNECTED			= 0x0,
-	HCI_WIFI_CONNECTED			= 0x1,
-	HCI_WIFI_CONNECT_IN_PROGRESS		= 0x2,
-};
-
-enum hci_ext_bp_operation {
-	HCI_BT_OP_NONE				= 0x0,
-	HCI_BT_OP_INQUIRY_START			= 0x1,
-	HCI_BT_OP_INQUIRY_FINISH		= 0x2,
-	HCI_BT_OP_PAGING_START			= 0x3,
-	HCI_BT_OP_PAGING_SUCCESS		= 0x4,
-	HCI_BT_OP_PAGING_UNSUCCESS		= 0x5,
-	HCI_BT_OP_PAIRING_START			= 0x6,
-	HCI_BT_OP_PAIRING_FINISH		= 0x7,
-	HCI_BT_OP_BT_DEV_ENABLE			= 0x8,
-	HCI_BT_OP_BT_DEV_DISABLE		= 0x9,
-	HCI_BT_OP_MAX
-};
-
-#define BTHCI_SM_WITH_INFO(_Adapter, _StateToEnter, _StateCmd, _EntryNum)	\
-{										\
-	RTPRINT(FIOCTL, IOCTL_STATE, ("[BT state change] caused by ""%s"", line =%d\n", __func__, __LINE__));							\
-	BTHCI_StateMachine(_Adapter, _StateToEnter, _StateCmd, _EntryNum);\
-}
-
-void BTHCI_EventParse(struct rtw_adapter *padapter, void *pEvntData,
-		      u32 dataLen);
-#define BT_EventParse BTHCI_EventParse
-u8 BTHCI_HsConnectionEstablished(struct rtw_adapter *padapter);
-void BTHCI_UpdateBTProfileRTKToMoto(struct rtw_adapter *padapter);
-void BTHCI_WifiScanNotify(struct rtw_adapter *padapter, u8 scanType);
-void BTHCI_StateMachine(struct rtw_adapter *padapter, u8 StateToEnter,
-			enum hci_state_with_cmd StateCmd, u8 EntryNum);
-void BTHCI_DisconnectPeer(struct rtw_adapter *padapter, u8 EntryNum);
-void BTHCI_EventNumOfCompletedDataBlocks(struct rtw_adapter *padapter);
-void BTHCI_EventAMPStatusChange(struct rtw_adapter *padapter, u8 AMP_Status);
-void BTHCI_DisconnectAll(struct rtw_adapter *padapter);
-enum hci_status BTHCI_HandleHCICMD(struct rtw_adapter *padapter,
-				   struct packet_irp_hcicmd_data *pHciCmd);
-
-/*  ===== End of sync from SD7 driver COMMON/bt_hci.h ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtc87231Ant.h ===== */
-#define GET_BT_INFO(padapter)	(&GET_HAL_DATA(padapter)->BtInfo)
-
-#define	BTC_FOR_SCAN_START				1
-#define	BTC_FOR_SCAN_FINISH				0
-
-#define	BT_TXRX_CNT_THRES_1				1200
-#define	BT_TXRX_CNT_THRES_2				1400
-#define	BT_TXRX_CNT_THRES_3				3000
-#define	BT_TXRX_CNT_LEVEL_0				0	/*  < 1200 */
-#define	BT_TXRX_CNT_LEVEL_1				1	/*  >= 1200 && < 1400 */
-#define	BT_TXRX_CNT_LEVEL_2				2	/*  >= 1400 */
-#define	BT_TXRX_CNT_LEVEL_3				3	/*  >= 3000 */
-
-enum bt_state_1ant {
-	BT_INFO_STATE_DISABLED			= 0,
-	BT_INFO_STATE_NO_CONNECTION		= 1,
-	BT_INFO_STATE_CONNECT_IDLE		= 2,
-	BT_INFO_STATE_INQ_OR_PAG		= 3,
-	BT_INFO_STATE_ACL_ONLY_BUSY		= 4,
-	BT_INFO_STATE_SCO_ONLY_BUSY		= 5,
-	BT_INFO_STATE_ACL_SCO_BUSY		= 6,
-	BT_INFO_STATE_ACL_INQ_OR_PAG		= 7,
-	BT_INFO_STATE_MAX			= 8
-};
-
-struct btdm_8723a_1ant {
-	u8		prePsTdma;
-	u8		curPsTdma;
-	u8		psTdmaDuAdjType;
-	u8		bPrePsTdmaOn;
-	u8		bCurPsTdmaOn;
-	u8		preWifiPara;
-	u8		curWifiPara;
-	u8		preCoexWifiCon;
-	u8		curCoexWifiCon;
-	u8		wifiRssiThresh;
-
-	u32		psTdmaMonitorCnt;
-	u32		psTdmaGlobalCnt;
-
-	/* DurationAdjust For SCO */
-	u32		psTdmaMonitorCntForSCO;
-	u8		psTdmaDuAdjTypeForSCO;
-	u8		RSSI_WiFi_Last;
-	u8		RSSI_BT_Last;
-
-	u8		bWiFiHalt;
-	u8		bRAChanged;
-};
-
-void BTDM_1AntSignalCompensation(struct rtw_adapter *padapter,
-				 u8 *rssi_wifi, u8 *rssi_bt);
-void BTDM_1AntForDhcp(struct rtw_adapter *padapter);
-void BTDM_1AntBtCoexist8723A(struct rtw_adapter *padapter);
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtc87231Ant.h ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtc87232Ant.h ===== */
-enum bt_2ant_bt_status {
-	BT_2ANT_BT_STATUS_IDLE			= 0x0,
-	BT_2ANT_BT_STATUS_CONNECTED_IDLE	= 0x1,
-	BT_2ANT_BT_STATUS_NON_IDLE		= 0x2,
-	BT_2ANT_BT_STATUS_MAX
-};
-
-enum bt_2ant_coex_algo {
-	BT_2ANT_COEX_ALGO_UNDEFINED			= 0x0,
-	BT_2ANT_COEX_ALGO_SCO				= 0x1,
-	BT_2ANT_COEX_ALGO_HID				= 0x2,
-	BT_2ANT_COEX_ALGO_A2DP				= 0x3,
-	BT_2ANT_COEX_ALGO_PANEDR			= 0x4,
-	BT_2ANT_COEX_ALGO_PANHS				= 0x5,
-	BT_2ANT_COEX_ALGO_PANEDR_A2DP		= 0x6,
-	BT_2ANT_COEX_ALGO_PANEDR_HID		= 0x7,
-	BT_2ANT_COEX_ALGO_HID_A2DP_PANEDR	= 0x8,
-	BT_2ANT_COEX_ALGO_HID_A2DP			= 0x9,
-	BT_2ANT_COEX_ALGO_HID_A2DP_PANHS	= 0xA,
-	BT_2ANT_COEX_ALGO_MAX				= 0xB,
-};
-
-struct btdm_8723a_2ant {
-	u8	bPreDecBtPwr;
-	u8	bCurDecBtPwr;
-
-	u8	preWlanActHi;
-	u8	curWlanActHi;
-	u8	preWlanActLo;
-	u8	curWlanActLo;
-
-	u8	preFwDacSwingLvl;
-	u8	curFwDacSwingLvl;
-
-	u8	bPreRfRxLpfShrink;
-	u8	bCurRfRxLpfShrink;
-
-	u8	bPreLowPenaltyRa;
-	u8	bCurLowPenaltyRa;
-
-	u8	preBtRetryIndex;
-	u8	curBtRetryIndex;
-
-	u8	bPreDacSwingOn;
-	u32	preDacSwingLvl;
-	u8	bCurDacSwingOn;
-	u32	curDacSwingLvl;
-
-	u8	bPreAdcBackOff;
-	u8	bCurAdcBackOff;
-
-	u8	bPreAgcTableEn;
-	u8	bCurAgcTableEn;
-
-	u32	preVal0x6c0;
-	u32	curVal0x6c0;
-	u32	preVal0x6c8;
-	u32	curVal0x6c8;
-	u8	preVal0x6cc;
-	u8	curVal0x6cc;
-
-	u8	bCurIgnoreWlanAct;
-	u8	bPreIgnoreWlanAct;
-
-	u8	prePsTdma;
-	u8	curPsTdma;
-	u8	psTdmaDuAdjType;
-	u8	bPrePsTdmaOn;
-	u8	bCurPsTdmaOn;
-
-	u8	preAlgorithm;
-	u8	curAlgorithm;
-	u8	bResetTdmaAdjust;
-
-	u8	btStatus;
-};
-
-void BTDM_2AntBtCoexist8723A(struct rtw_adapter *padapter);
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtc87232Ant.h ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtc8723.h ===== */
-
-#define	BT_Q_PKT_OFF		0
-#define	BT_Q_PKT_ON		1
-
-#define	BT_TX_PWR_OFF		0
-#define	BT_TX_PWR_ON		1
-
-/*  TDMA mode definition */
-#define	TDMA_2ANT			0
-#define	TDMA_1ANT			1
-#define	TDMA_NAV_OFF		0
-#define	TDMA_NAV_ON		1
-#define	TDMA_DAC_SWING_OFF	0
-#define	TDMA_DAC_SWING_ON	1
-
-#define	BT_RSSI_LEVEL_H	0
-#define	BT_RSSI_LEVEL_M	1
-#define	BT_RSSI_LEVEL_L	2
-
-/*  PTA mode related definition */
-#define	BT_PTA_MODE_OFF		0
-#define	BT_PTA_MODE_ON		1
-
-/*  Penalty Tx Rate Adaptive */
-#define	BT_TX_RATE_ADAPTIVE_NORMAL			0
-#define	BT_TX_RATE_ADAPTIVE_LOW_PENALTY	1
-
-/*  RF Corner */
-#define	BT_RF_RX_LPF_CORNER_RESUME			0
-#define	BT_RF_RX_LPF_CORNER_SHRINK			1
-
-#define BT_INFO_ACL			BIT(0)
-#define BT_INFO_SCO			BIT(1)
-#define BT_INFO_INQ_PAG		BIT(2)
-#define BT_INFO_ACL_BUSY	BIT(3)
-#define BT_INFO_SCO_BUSY	BIT(4)
-#define BT_INFO_HID			BIT(5)
-#define BT_INFO_A2DP		BIT(6)
-#define BT_INFO_FTP			BIT(7)
-
-
-
-struct bt_coexist_8723a {
-	u32					highPriorityTx;
-	u32					highPriorityRx;
-	u32					lowPriorityTx;
-	u32					lowPriorityRx;
-	u8					btRssi;
-	u8					TotalAntNum;
-	u8					bC2hBtInfoSupport;
-	u8					c2hBtInfo;
-	u8					c2hBtInfoOriginal;
-	u8					prec2hBtInfo; /*  for 1Ant */
-	u8					bC2hBtInquiryPage;
-	unsigned long				btInqPageStartTime; /*  for 2Ant */
-	u8					c2hBtProfile; /*  for 1Ant */
-	u8					btRetryCnt;
-	u8					btInfoExt;
-	u8					bC2hBtInfoReqSent;
-	u8					bForceFwBtInfo;
-	u8					bForceA2dpSink;
-	struct btdm_8723a_2ant			btdm2Ant;
-	struct btdm_8723a_1ant			btdm1Ant;
-};
-
-void BTDM_SetFwChnlInfo(struct rtw_adapter *padapter,
-			enum rt_media_status mstatus);
-u8 BTDM_IsWifiConnectionExist(struct rtw_adapter *padapter);
-void BTDM_SetFw3a(struct rtw_adapter *padapter, u8 byte1, u8 byte2, u8 byte3,
-		  u8 byte4, u8 byte5);
-void BTDM_QueryBtInformation(struct rtw_adapter *padapter);
-void BTDM_SetSwRfRxLpfCorner(struct rtw_adapter *padapter, u8 type);
-void BTDM_SetSwPenaltyTxRateAdaptive(struct rtw_adapter *padapter, u8 raType);
-void BTDM_SetFwDecBtPwr(struct rtw_adapter *padapter, u8 bDecBtPwr);
-u8 BTDM_BtProfileSupport(struct rtw_adapter *padapter);
-void BTDM_LpsLeave(struct rtw_adapter *padapter);
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtc8723.h ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtcCsr1Ant.h ===== */
-
-enum BT_A2DP_INDEX{
-	BT_A2DP_INDEX0		= 0,			/*  32, 12; the most critical for BT */
-	BT_A2DP_INDEX1,					/*  12, 24 */
-	BT_A2DP_INDEX2,					/*  0, 0 */
-	BT_A2DP_INDEX_MAX
-};
-
-#define BT_A2DP_STATE_NOT_ENTERED		0
-#define BT_A2DP_STATE_DETECTING		1
-#define BT_A2DP_STATE_DETECTED			2
-
-#define BTDM_ANT_BT_IDLE				0
-#define BTDM_ANT_WIFI					1
-#define BTDM_ANT_BT						2
-
-
-void BTDM_SingleAnt(struct rtw_adapter *padapter, u8 bSingleAntOn,
-		    u8 bInterruptOn, u8 bMultiNAVOn);
-void BTDM_CheckBTIdleChange1Ant(struct rtw_adapter *padapter);
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtcCsr1Ant.h ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtcCsr2Ant.h ===== */
-
-/*  */
-/*  For old core stack before v251 */
-/*  */
-#define BT_RSSI_STATE_NORMAL_POWER	BIT(0)
-#define BT_RSSI_STATE_AMDPU_OFF		BIT(1)
-#define BT_RSSI_STATE_SPECIAL_LOW	BIT(2)
-#define BT_RSSI_STATE_BG_EDCA_LOW	BIT(3)
-#define BT_RSSI_STATE_TXPOWER_LOW	BIT(4)
-
-#define	BT_DACSWING_OFF				0
-#define	BT_DACSWING_M4				1
-#define	BT_DACSWING_M7				2
-#define	BT_DACSWING_M10				3
-
-void BTDM_DiminishWiFi(struct rtw_adapter *Adapter, u8 bDACOn, u8 bInterruptOn,
-		       u8 DACSwingLevel, u8 bNAVOn);
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtcCsr2Ant.h ===== */
-
-/*  HEADER/TypeDef.h */
-#define MAX_FW_SUPPORT_MACID_NUM			64
-
-/*  ===== Below this line is sync from SD7 driver HAL/BTCoexist/HalBtCoexist.h ===== */
-
-#define	FW_VER_BT_REG			62
-#define	FW_VER_BT_REG1		74
-#define	REG_BT_ACTIVE			0x444
-#define	REG_BT_STATE			0x448
-#define	REG_BT_POLLING1		0x44c
-#define	REG_BT_POLLING			0x700
-
-#define	REG_BT_ACTIVE_OLD		0x488
-#define	REG_BT_STATE_OLD		0x48c
-#define	REG_BT_POLLING_OLD	0x490
-
-/*  The reg define is for 8723 */
-#define	REG_HIGH_PRIORITY_TXRX			0x770
-#define	REG_LOW_PRIORITY_TXRX			0x774
-
-#define BT_FW_COEX_THRESH_TOL			6
-#define BT_FW_COEX_THRESH_20				20
-#define BT_FW_COEX_THRESH_23				23
-#define BT_FW_COEX_THRESH_25				25
-#define BT_FW_COEX_THRESH_30				30
-#define BT_FW_COEX_THRESH_35				35
-#define BT_FW_COEX_THRESH_40				40
-#define BT_FW_COEX_THRESH_45				45
-#define BT_FW_COEX_THRESH_47				47
-#define BT_FW_COEX_THRESH_50				50
-#define BT_FW_COEX_THRESH_55				55
-#define BT_FW_COEX_THRESH_65				65
-
-#define BT_COEX_STATE_BT30			BIT(0)
-#define BT_COEX_STATE_WIFI_HT20			BIT(1)
-#define BT_COEX_STATE_WIFI_HT40			BIT(2)
-#define BT_COEX_STATE_WIFI_LEGACY		BIT(3)
-
-#define BT_COEX_STATE_WIFI_RSSI_LOW		BIT(4)
-#define BT_COEX_STATE_WIFI_RSSI_MEDIUM		BIT(5)
-#define BT_COEX_STATE_WIFI_RSSI_HIGH		BIT(6)
-#define BT_COEX_STATE_DEC_BT_POWER		BIT(7)
-
-#define BT_COEX_STATE_WIFI_IDLE			BIT(8)
-#define BT_COEX_STATE_WIFI_UPLINK		BIT(9)
-#define BT_COEX_STATE_WIFI_DOWNLINK		BIT(10)
-
-#define BT_COEX_STATE_BT_INQ_PAGE		BIT(11)
-#define BT_COEX_STATE_BT_IDLE			BIT(12)
-#define BT_COEX_STATE_BT_UPLINK			BIT(13)
-#define BT_COEX_STATE_BT_DOWNLINK		BIT(14)
-/*  */
-/*  Todo: Remove these definitions */
-#define BT_COEX_STATE_BT_PAN_IDLE		BIT(15)
-#define BT_COEX_STATE_BT_PAN_UPLINK		BIT(16)
-#define BT_COEX_STATE_BT_PAN_DOWNLINK		BIT(17)
-#define BT_COEX_STATE_BT_A2DP_IDLE		BIT(18)
-/*  */
-#define BT_COEX_STATE_BT_RSSI_LOW		BIT(19)
-
-#define BT_COEX_STATE_PROFILE_HID		BIT(20)
-#define BT_COEX_STATE_PROFILE_A2DP		BIT(21)
-#define BT_COEX_STATE_PROFILE_PAN		BIT(22)
-#define BT_COEX_STATE_PROFILE_SCO		BIT(23)
-
-#define BT_COEX_STATE_WIFI_RSSI_1_LOW		BIT(24)
-#define BT_COEX_STATE_WIFI_RSSI_1_MEDIUM	BIT(25)
-#define BT_COEX_STATE_WIFI_RSSI_1_HIGH		BIT(26)
-
-#define BT_COEX_STATE_WIFI_RSSI_BEACON_LOW	BIT(27)
-#define BT_COEX_STATE_WIFI_RSSI_BEACON_MEDIUM	BIT(28)
-#define BT_COEX_STATE_WIFI_RSSI_BEACON_HIGH	BIT(29)
-
-
-#define BT_COEX_STATE_BTINFO_COMMON		BIT(30)
-#define BT_COEX_STATE_BTINFO_B_HID_SCOESCO	BIT(31)
-#define BT_COEX_STATE_BTINFO_B_FTP_A2DP		BIT(32)
-
-#define BT_COEX_STATE_BT_CNT_LEVEL_0		BIT(33)
-#define BT_COEX_STATE_BT_CNT_LEVEL_1		BIT(34)
-#define BT_COEX_STATE_BT_CNT_LEVEL_2		BIT(35)
-#define BT_COEX_STATE_BT_CNT_LEVEL_3		BIT(36)
-
-#define BT_RSSI_STATE_HIGH			0
-#define BT_RSSI_STATE_MEDIUM			1
-#define BT_RSSI_STATE_LOW			2
-#define BT_RSSI_STATE_STAY_HIGH			3
-#define BT_RSSI_STATE_STAY_MEDIUM		4
-#define BT_RSSI_STATE_STAY_LOW			5
-
-#define	BT_AGCTABLE_OFF				0
-#define	BT_AGCTABLE_ON				1
-
-#define	BT_BB_BACKOFF_OFF			0
-#define	BT_BB_BACKOFF_ON			1
-
-#define	BT_FW_NAV_OFF				0
-#define	BT_FW_NAV_ON				1
-
-#define	BT_COEX_MECH_NONE			0
-#define	BT_COEX_MECH_SCO			1
-#define	BT_COEX_MECH_HID			2
-#define	BT_COEX_MECH_A2DP			3
-#define	BT_COEX_MECH_PAN			4
-#define	BT_COEX_MECH_HID_A2DP			5
-#define	BT_COEX_MECH_HID_PAN			6
-#define	BT_COEX_MECH_PAN_A2DP			7
-#define	BT_COEX_MECH_HID_SCO_ESCO		8
-#define	BT_COEX_MECH_FTP_A2DP			9
-#define	BT_COEX_MECH_COMMON			10
-#define	BT_COEX_MECH_MAX			11
-/*	BT Dbg Ctrl */
-#define	BT_DBG_PROFILE_NONE			0
-#define	BT_DBG_PROFILE_SCO			1
-#define	BT_DBG_PROFILE_HID			2
-#define	BT_DBG_PROFILE_A2DP			3
-#define	BT_DBG_PROFILE_PAN			4
-#define	BT_DBG_PROFILE_HID_A2DP			5
-#define	BT_DBG_PROFILE_HID_PAN			6
-#define	BT_DBG_PROFILE_PAN_A2DP			7
-#define	BT_DBG_PROFILE_MAX			9
-
-struct bt_coexist_str {
-	u8			BluetoothCoexist;
-	u8			BT_Ant_Num;
-	u8			BT_CoexistType;
-	u8			BT_Ant_isolation;	/* 0:good, 1:bad */
-	u8			bt_radiosharedtype;
-	u32			Ratio_Tx;
-	u32			Ratio_PRI;
-	u8			bInitlized;
-	u32			BtRfRegOrigin1E;
-	u32			BtRfRegOrigin1F;
-	u8			bBTBusyTraffic;
-	u8			bBTTrafficModeSet;
-	u8			bBTNonTrafficModeSet;
-	struct bt_traffic_statistics		BT21TrafficStatistics;
-	u64			CurrentState;
-	u64			PreviousState;
-	u8			preRssiState;
-	u8			preRssiState1;
-	u8			preRssiStateBeacon;
-	u8			bFWCoexistAllOff;
-	u8			bSWCoexistAllOff;
-	u8			bHWCoexistAllOff;
-	u8			bBalanceOn;
-	u8			bSingleAntOn;
-	u8			bInterruptOn;
-	u8			bMultiNAVOn;
-	u8			PreWLANActH;
-	u8			PreWLANActL;
-	u8			WLANActH;
-	u8			WLANActL;
-	u8			A2DPState;
-	u8			AntennaState;
-	u32			lastBtEdca;
-	u16			last_aggr_num;
-	u8			bEDCAInitialized;
-	u8			exec_cnt;
-	u8			b8723aAgcTableOn;
-	u8			b92DAgcTableOn;
-	struct bt_coexist_8723a	halCoex8723;
-	u8			btActiveZeroCnt;
-	u8			bCurBtDisabled;
-	u8			bPreBtDisabled;
-	u8			bNeedToRoamForBtDisableEnable;
-	u8			fw3aVal[5];
-};
-
-void BTDM_CheckAntSelMode(struct rtw_adapter *padapter);
-void BTDM_FwC2hBtRssi(struct rtw_adapter *padapter, u8 *tmpBuf);
-#define BT_FwC2hBtRssi BTDM_FwC2hBtRssi
-void BTDM_DisplayBtCoexInfo(struct rtw_adapter *padapter);
-#define BT_DisplayBtCoexInfo BTDM_DisplayBtCoexInfo
-void BTDM_RejectAPAggregatedPacket(struct rtw_adapter *padapter, u8 bReject);
-u8 BTDM_IsHT40(struct rtw_adapter *padapter);
-u8 BTDM_Legacy(struct rtw_adapter *padapter);
-void BTDM_CheckWiFiState(struct rtw_adapter *padapter);
-s32 BTDM_GetRxSS(struct rtw_adapter *padapter);
-u8 BTDM_CheckCoexBcnRssiState(struct rtw_adapter *padapter, u8 levelNum,
-			      u8 RssiThresh, u8 RssiThresh1);
-u8 BTDM_CheckCoexRSSIState1(struct rtw_adapter *padapter, u8 levelNum,
-			    u8 RssiThresh, u8 RssiThresh1);
-u8 BTDM_CheckCoexRSSIState(struct rtw_adapter *padapter, u8 levelNum,
-			   u8 RssiThresh, u8 RssiThresh1);
-void BTDM_Balance(struct rtw_adapter *padapter, u8 bBalanceOn, u8 ms0, u8 ms1);
-void BTDM_AGCTable(struct rtw_adapter *padapter, u8 type);
-void BTDM_BBBackOffLevel(struct rtw_adapter *padapter, u8 type);
-void BTDM_FWCoexAllOff(struct rtw_adapter *padapter);
-void BTDM_SWCoexAllOff(struct rtw_adapter *padapter);
-void BTDM_HWCoexAllOff(struct rtw_adapter *padapter);
-void BTDM_CoexAllOff(struct rtw_adapter *padapter);
-void BTDM_TurnOffBtCoexistBeforeEnterIPS(struct rtw_adapter *padapter);
-void BTDM_SignalCompensation(struct rtw_adapter *padapter, u8 *rssi_wifi,
-			     u8 *rssi_bt);
-void BTDM_UpdateCoexState(struct rtw_adapter *padapter);
-u8 BTDM_IsSameCoexistState(struct rtw_adapter *padapter);
-void BTDM_PWDBMonitor(struct rtw_adapter *padapter);
-u8 BTDM_IsBTBusy(struct rtw_adapter *padapter);
-#define BT_IsBtBusy BTDM_IsBTBusy
-u8 BTDM_IsWifiBusy(struct rtw_adapter *padapter);
-u8 BTDM_IsCoexistStateChanged(struct rtw_adapter *padapter);
-u8 BTDM_IsWifiUplink(struct rtw_adapter *padapter);
-u8 BTDM_IsWifiDownlink(struct rtw_adapter *padapter);
-u8 BTDM_IsBTHSMode(struct rtw_adapter *padapter);
-u8 BTDM_IsBTUplink(struct rtw_adapter *padapter);
-u8 BTDM_IsBTDownlink(struct rtw_adapter *padapter);
-void BTDM_AdjustForBtOperation(struct rtw_adapter *padapter);
-void BTDM_ForHalt(struct rtw_adapter *padapter);
-void BTDM_WifiScanNotify(struct rtw_adapter *padapter, u8 scanType);
-void BTDM_WifiAssociateNotify(struct rtw_adapter *padapter, u8 action);
-void BTDM_MediaStatusNotify(struct rtw_adapter *padapter,
-			    enum rt_media_status mstatus);
-void BTDM_ForDhcp(struct rtw_adapter *padapter);
-void BTDM_ResetActionProfileState(struct rtw_adapter *padapter);
-void BTDM_SetBtCoexCurrAntNum(struct rtw_adapter *padapter, u8 antNum);
-#define BT_SetBtCoexCurrAntNum BTDM_SetBtCoexCurrAntNum
-u8 BTDM_IsActionSCO(struct rtw_adapter *padapter);
-u8 BTDM_IsActionHID(struct rtw_adapter *padapter);
-u8 BTDM_IsActionA2DP(struct rtw_adapter *padapter);
-u8 BTDM_IsActionPAN(struct rtw_adapter *padapter);
-u8 BTDM_IsActionHIDA2DP(struct rtw_adapter *padapter);
-u8 BTDM_IsActionHIDPAN(struct rtw_adapter *padapter);
-u8 BTDM_IsActionPANA2DP(struct rtw_adapter *padapter);
-u32 BTDM_BtTxRxCounterH(struct rtw_adapter *padapter);
-u32 BTDM_BtTxRxCounterL(struct rtw_adapter *padapter);
-
-/*  ===== End of sync from SD7 driver HAL/BTCoexist/HalBtCoexist.h ===== */
-
-/*  ===== Below this line is sync from SD7 driver HAL/HalBT.h ===== */
-
-#define RTS_CTS_NO_LEN_LIMIT	0
-
-u8 HALBT_GetPGAntNum(struct rtw_adapter *padapter);
-#define BT_GetPGAntNum HALBT_GetPGAntNum
-void HALBT_SetKey(struct rtw_adapter *padapter, u8 EntryNum);
-void HALBT_RemoveKey(struct rtw_adapter *padapter, u8 EntryNum);
-u8 HALBT_IsBTExist(struct rtw_adapter *padapter);
-#define BT_IsBtExist HALBT_IsBTExist
-u8 HALBT_BTChipType(struct rtw_adapter *padapter);
-void HALBT_SetRtsCtsNoLenLimit(struct rtw_adapter *padapter);
-
-/*  ===== End of sync from SD7 driver HAL/HalBT.c ===== */
-
-#define _bt_dbg_off_		0
-#define _bt_dbg_on_		1
-
-extern u32 BTCoexDbgLevel;
-
-
-
-#endif /*  __RTL8723A_BT_COEXIST_H__ */
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_bt_intf.h b/drivers/staging/rtl8723au/include/rtl8723a_bt_intf.h
deleted file mode 100644
index 4733559970e5..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_bt_intf.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- * Copyright(c) 2014, Jes Sorensen <Jes.Sorensen@redhat.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTL8723A_BT_INTF_H__
-#define __RTL8723A_BT_INTF_H__
-
-#include <drv_types.h>
-
-#ifdef CONFIG_8723AU_BT_COEXIST
-enum rt_media_status;
-bool rtl8723a_BT_using_antenna_1(struct rtw_adapter *padapter);
-bool rtl8723a_BT_enabled(struct rtw_adapter *padapter);
-bool rtl8723a_BT_coexist(struct rtw_adapter *padapter);
-void rtl8723a_BT_do_coexist(struct rtw_adapter *padapter);
-void rtl8723a_BT_wifiscan_notify(struct rtw_adapter *padapter, u8 scanType);
-void rtl8723a_BT_mediastatus_notify(struct rtw_adapter *padapter,
-				    enum rt_media_status mstatus);
-void rtl8723a_BT_specialpacket_notify(struct rtw_adapter *padapter);
-void rtl8723a_BT_lps_leave(struct rtw_adapter *padapter);
-void rtl8723a_BT_disable_coexist(struct rtw_adapter *padapter);
-bool rtl8723a_BT_disable_EDCA_turbo(struct rtw_adapter *padapter);
-void rtl8723a_dual_antenna_detection(struct rtw_adapter *padapter);
-void rtl8723a_BT_init_hwconfig(struct rtw_adapter *padapter);
-void rtl8723a_BT_wifiassociate_notify(struct rtw_adapter *padapter, u8 action);
-void rtl8723a_BT_init_hal_vars(struct rtw_adapter *padapter);
-void rtl8723a_fw_c2h_BT_info(struct rtw_adapter *padapter, u8 *tmpBuf, u8 length);
-#else
-static inline bool rtl8723a_BT_using_antenna_1(struct rtw_adapter *padapter)
-{
-	return false;
-}
-static inline bool rtl8723a_BT_enabled(struct rtw_adapter *padapter)
-{
-	return false;
-}
-static inline bool rtl8723a_BT_coexist(struct rtw_adapter *padapter)
-{
-	return false;
-}
-#define rtl8723a_BT_do_coexist(padapter)			do {} while(0)
-#define rtl8723a_BT_wifiscan_notify(padapter, scanType)		do {} while(0)
-#define rtl8723a_BT_mediastatus_notify(padapter, mstatus)	do {} while(0)
-#define rtl8723a_BT_specialpacket_notify(padapter)		do {} while(0)
-#define rtl8723a_BT_lps_leave(padapter)				do {} while(0)
-#define rtl8723a_BT_disable_coexist(padapter)			do {} while(0)
-static inline bool rtl8723a_BT_disable_EDCA_turbo(struct rtw_adapter *padapter)
-{
-	return false;
-}
-#define rtl8723a_dual_antenna_detection(padapter)		do {} while(0)
-#define rtl8723a_BT_init_hwconfig(padapter)			do {} while(0)
-#define rtl8723a_BT_wifiassociate_notify(padapter, action)	do {} while(0)
-#define rtl8723a_BT_init_hal_vars(padapter)			do {} while(0)
-#define rtl8723a_fw_c2h_BT_info(padapter, tmpBuf, length)	do {} while(0)
-#endif
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_cmd.h b/drivers/staging/rtl8723au/include/rtl8723a_cmd.h
deleted file mode 100644
index f95535a915ab..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_cmd.h
+++ /dev/null
@@ -1,158 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTL8723A_CMD_H__
-#define __RTL8723A_CMD_H__
-
-
-#define H2C_BT_FW_PATCH_LEN		3
-#define H2C_BT_PWR_FORCE_LEN		3
-
-enum cmd_msg_element_id
-{
-	NONE_CMDMSG_EID,
-	AP_OFFLOAD_EID = 0,
-	SET_PWRMODE_EID = 1,
-	JOINBSS_RPT_EID = 2,
-	RSVD_PAGE_EID = 3,
-	RSSI_4_EID = 4,
-	RSSI_SETTING_EID = 5,
-	MACID_CONFIG_EID = 6,
-	MACID_PS_MODE_EID = 7,
-	P2P_PS_OFFLOAD_EID = 8,
-	SELECTIVE_SUSPEND_ROF_CMD = 9,
-	BT_QUEUE_PKT_EID = 17,
-	BT_ANT_TDMA_EID = 20,
-	BT_2ANT_HID_EID = 21,
-	P2P_PS_CTW_CMD_EID = 32,
-	FORCE_BT_TX_PWR_EID = 33,
-	SET_TDMA_WLAN_ACT_TIME_EID = 34,
-	SET_BT_TX_RETRY_INDEX_EID = 35,
-	HID_PROFILE_ENABLE_EID = 36,
-	BT_IGNORE_WLAN_ACT_EID = 37,
-	BT_PTA_MANAGER_UPDATE_ENABLE_EID = 38,
-	DAC_SWING_VALUE_EID = 41,
-	TRADITIONAL_TDMA_EN_EID = 51,
-	H2C_BT_FW_PATCH = 54,
-	B_TYPE_TDMA_EID = 58,
-	SCAN_EN_EID = 59,
-	LOWPWR_LPS_EID = 71,
-	H2C_RESET_TSF = 75,
-	MAX_CMDMSG_EID
-};
-
-struct cmd_msg_parm {
-	u8 eid; /* element id */
-	u8 sz; /*  sz */
-	u8 buf[6];
-};
-
-struct setpwrmode_parm {
-	u8 Mode;
-	u8 SmartPS;
-	u8 AwakeInterval;	/*  unit: beacon interval */
-	u8 bAllQueueUAPSD;
-
-#define SETPM_LOWRXBCN			BIT(0)
-#define SETPM_AUTOANTSWITCH		BIT(1)
-#define SETPM_PSALLOWBTHIGHPRI	BIT(2)
-	u8 BcnAntMode;
-} __packed;
-
-struct H2C_SS_RFOFF_PARAM{
-	u8 ROFOn; /*  1: on, 0:off */
-	u16 gpio_period; /*  unit: 1024 us */
-}__attribute__ ((packed));
-
-
-struct joinbssrpt_parm {
-	u8 OpMode;	/*  enum rt_media_status */
-};
-
-struct rsvdpage_loc {
-	u8 LocProbeRsp;
-	u8 LocPsPoll;
-	u8 LocNullData;
-	u8 LocQosNull;
-	u8 LocBTQosNull;
-};
-
-struct P2P_PS_Offload_t {
-	u8 Offload_En:1;
-	u8 role:1; /*  1: Owner, 0: Client */
-	u8 CTWindow_En:1;
-	u8 NoA0_En:1;
-	u8 NoA1_En:1;
-	u8 AllStaSleep:1; /*  Only valid in Owner */
-	u8 discovery:1;
-	u8 rsvd:1;
-};
-
-struct P2P_PS_CTWPeriod_t {
-	u8 CTWPeriod;	/* TU */
-};
-
-#define B_TDMA_EN			BIT(0)
-#define B_TDMA_FIXANTINBT		BIT(1)
-#define B_TDMA_TXPSPOLL			BIT(2)
-#define B_TDMA_VAL870			BIT(3)
-#define B_TDMA_AUTOWAKEUP		BIT(4)
-#define B_TDMA_NOPS			BIT(5)
-#define B_TDMA_WLANHIGHPRI		BIT(6)
-
-struct b_type_tdma_parm {
-	u8 option;
-
-	u8 TBTTOnPeriod;
-	u8 MedPeriod;
-	u8 rsvd30;
-} __packed;
-
-struct scan_en_parm {
-	u8 En;
-} __packed;
-
-/*  BT_PWR */
-#define SET_H2CCMD_BT_PWR_IDX(__pH2CCmd, __Value)							SET_BITS_TO_LE_1BYTE_8BIT(__pH2CCmd, 0, 8, __Value)
-
-/*  BT_FW_PATCH */
-#define SET_H2CCMD_BT_FW_PATCH_ENABLE(__pH2CCmd, __Value)					SET_BITS_TO_LE_4BYTE(__pH2CCmd, 0, 8, __Value) /*	SET_BITS_TO_LE_1BYTE(__pH2CCmd, 0, 8, __Value) */
-#define SET_H2CCMD_BT_FW_PATCH_SIZE(__pH2CCmd, __Value)						SET_BITS_TO_LE_4BYTE(__pH2CCmd, 8, 16, __Value) /*	SET_BITS_TO_LE_2BYTE((__pH2CCmd)+1, 0, 16, __Value) */
-
-struct lowpwr_lps_parm{
-	u8 bcn_count:4;
-	u8 tb_bcn_threshold:3;
-	u8 enable:1;
-	u8 bcn_interval;
-	u8 drop_threshold;
-	u8 max_early_period;
-	u8 max_bcn_timeout_period;
-} __packed;
-
-
-/*  host message to firmware cmd */
-void rtl8723a_set_FwPwrMode_cmd(struct rtw_adapter *padapter, u8 Mode);
-void rtl8723a_set_FwJoinBssReport_cmd(struct rtw_adapter *padapter, u8 mstatus);
-#ifdef CONFIG_8723AU_BT_COEXIST
-void rtl8723a_set_BTCoex_AP_mode_FwRsvdPkt_cmd(struct rtw_adapter *padapter);
-#else
-#define rtl8723a_set_BTCoex_AP_mode_FwRsvdPkt_cmd(padapter) do {} while(0)
-#endif
-int rtl8723a_set_rssi_cmd(struct rtw_adapter *padapter, u32 param);
-int rtl8723a_set_raid_cmd(struct rtw_adapter *padapter, u32 mask, u8 arg);
-void rtl8723a_add_rateatid(struct rtw_adapter *padapter, u32 bitmap, u8 arg, u8 rssi_level);
-
-int FillH2CCmd(struct rtw_adapter *padapter, u8 ElementID, u32 CmdLen, u8 *pCmdBuffer);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_dm.h b/drivers/staging/rtl8723au/include/rtl8723a_dm.h
deleted file mode 100644
index bf236e8e47a2..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_dm.h
+++ /dev/null
@@ -1,137 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTL8723A_DM_H__
-#define __RTL8723A_DM_H__
-/*  */
-/*  Description: */
-/*  */
-/*  This file is for 8723A dynamic mechanism only */
-/*  */
-/*  */
-/*  */
-#define DYNAMIC_FUNC_BT BIT(0)
-
-enum{
-	UP_LINK,
-	DOWN_LINK,
-};
-/*  */
-/*  structure and define */
-/*  */
-
-/*  duplicate code,will move to ODM ######### */
-#define IQK_MAC_REG_NUM		4
-#define IQK_ADDA_REG_NUM		16
-#define IQK_BB_REG_NUM			9
-#define HP_THERMAL_NUM		8
-/*  duplicate code,will move to ODM ######### */
-struct dm_priv {
-	u32	InitODMFlag;
-
-	/*  Upper and Lower Signal threshold for Rate Adaptive*/
-	int	UndecoratedSmoothedPWDB;
-	int	UndecoratedSmoothedCCK;
-	int	EntryMinUndecoratedSmoothedPWDB;
-	int	EntryMaxUndecoratedSmoothedPWDB;
-	int	MinUndecoratedPWDBForDM;
-	int	LastMinUndecoratedPWDBForDM;
-
-	s32	UndecoratedSmoothedBeacon;
-	#ifdef CONFIG_8723AU_BT_COEXIST
-	s32 BT_EntryMinUndecoratedSmoothedPWDB;
-	s32 BT_EntryMaxUndecoratedSmoothedPWDB;
-	#endif
-
-	/* for High Power */
-	u8 DynamicTxHighPowerLvl;/* Add by Jacken Tx Power Control for Near/Far Range 2008/03/06 */
-
-	/* for tx power tracking */
-	u8	bTXPowerTracking;
-	u8	TXPowercount;
-	u8	bTXPowerTrackingInit;
-	u8	TxPowerTrackControl;	/* for mp mode, turn off txpwrtracking as default */
-	u8	TM_Trigger;
-
-	u8	ThermalMeter[2];				/*  ThermalMeter, index 0 for RFIC0, and 1 for RFIC1 */
-	u8	ThermalValue;
-	u8	ThermalValue_LCK;
-	u8	ThermalValue_IQK;
-	u8	ThermalValue_DPK;
-
-	u8	bRfPiEnable;
-
-	/* for APK */
-	u32	APKoutput[2][2];	/* path A/B; output1_1a/output1_2a */
-	u8	bAPKdone;
-	u8	bAPKThermalMeterIgnore;
-	u8	bDPdone;
-	u8	bDPPathAOK;
-	u8	bDPPathBOK;
-
-	/* for IQK */
-	u32	RegC04;
-	u32	Reg874;
-	u32	RegC08;
-	u32	RegB68;
-	u32	RegB6C;
-	u32	Reg870;
-	u32	Reg860;
-	u32	Reg864;
-	u32	ADDA_backup[IQK_ADDA_REG_NUM];
-	u32	IQK_MAC_backup[IQK_MAC_REG_NUM];
-	u32	IQK_BB_backup_recover[9];
-	u32	IQK_BB_backup[IQK_BB_REG_NUM];
-	u8	PowerIndex_backup[6];
-
-	u8	bCCKinCH14;
-
-	u8	CCK_index;
-	u8	OFDM_index[2];
-
-	u8	bDoneTxpower;
-	u8	CCK_index_HP;
-	u8	OFDM_index_HP[2];
-	u8	ThermalValue_HP[HP_THERMAL_NUM];
-	u8	ThermalValue_HP_index;
-
-	/* for TxPwrTracking */
-	s32	RegE94;
-	s32     RegE9C;
-	s32	RegEB4;
-	s32	RegEBC;
-
-	u32	TXPowerTrackingCallbackCnt;	/* cosa add for debug */
-
-	u32	prv_traffic_idx; /*  edca turbo */
-
-	s32	OFDM_Pkt_Cnt;
-	u8	RSSI_Select;
-/*	u8	DIG_Dynamic_MIN ; */
-/*  duplicate code,will move to ODM ######### */
-	/*  Add for Reading Initial Data Rate SEL Register 0x484 during watchdog. Using for fill tx desc. 2011.3.21 by Thomas */
-	u8	INIDATA_RATE[32];
-};
-
-
-/*  */
-/*  function prototype */
-/*  */
-
-void rtl8723a_init_dm_priv(struct rtw_adapter *padapter);
-
-void rtl8723a_InitHalDm(struct rtw_adapter *padapter);
-void rtl8723a_HalDmWatchDog(struct rtw_adapter *padapter);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_hal.h b/drivers/staging/rtl8723au/include/rtl8723a_hal.h
deleted file mode 100644
index 77a0fd485b51..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_hal.h
+++ /dev/null
@@ -1,538 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTL8723A_HAL_H__
-#define __RTL8723A_HAL_H__
-
-#include "rtl8723a_spec.h"
-#include "rtl8723a_pg.h"
-#include "Hal8723APhyReg.h"
-#include "Hal8723APhyCfg.h"
-#include "rtl8723a_rf.h"
-#include "rtl8723a_bt_intf.h"
-#ifdef CONFIG_8723AU_BT_COEXIST
-#include "rtl8723a_bt-coexist.h"
-#endif
-#include "rtl8723a_dm.h"
-#include "rtl8723a_recv.h"
-#include "rtl8723a_xmit.h"
-#include "rtl8723a_cmd.h"
-#include "rtl8723a_sreset.h"
-#include "rtw_efuse.h"
-#include "rtw_eeprom.h"
-
-#include "odm_precomp.h"
-#include "odm.h"
-
-
-/* 2TODO: We should define 8192S firmware related macro settings here!! */
-#define RTL819X_DEFAULT_RF_TYPE			RF_1T2R
-#define RTL819X_TOTAL_RF_PATH				2
-
-/*  */
-/*		RTL8723S From header */
-/*  */
-
-/*  Fw Array */
-#define Rtl8723_FwImageArray				Rtl8723UFwImgArray
-#define Rtl8723_FwUMCBCutImageArrayWithBT		Rtl8723UFwUMCBCutImgArrayWithBT
-#define Rtl8723_FwUMCBCutImageArrayWithoutBT	Rtl8723UFwUMCBCutImgArrayWithoutBT
-
-#define Rtl8723_ImgArrayLength				Rtl8723UImgArrayLength
-#define Rtl8723_UMCBCutImgArrayWithBTLength		Rtl8723UUMCBCutImgArrayWithBTLength
-#define Rtl8723_UMCBCutImgArrayWithoutBTLength	Rtl8723UUMCBCutImgArrayWithoutBTLength
-
-#define Rtl8723_PHY_REG_Array_PG			Rtl8723UPHY_REG_Array_PG
-#define Rtl8723_PHY_REG_Array_PGLength		Rtl8723UPHY_REG_Array_PGLength
-
-#define Rtl8723_FwUMCBCutMPImageArray		Rtl8723SFwUMCBCutMPImgAr
-#define Rtl8723_UMCBCutMPImgArrayLength		Rtl8723SUMCBCutMPImgArrayLength
-
-#define DRVINFO_SZ				4 /*  unit is 8bytes */
-#define PageNum_128(_Len)		(u32)(((_Len)>>7) + ((_Len)&0x7F ? 1:0))
-
-#define FW_8723A_SIZE			0x8000
-#define FW_8723A_START_ADDRESS	0x1000
-#define FW_8723A_END_ADDRESS		0x1FFF /* 0x5FFF */
-
-#define MAX_PAGE_SIZE			4096	/*  @ page : 4k bytes */
-
-#define IS_FW_HEADER_EXIST(_pFwHdr)	((le16_to_cpu(_pFwHdr->Signature)&0xFFF0) == 0x92C0 ||\
-									(le16_to_cpu(_pFwHdr->Signature)&0xFFF0) == 0x88C0 ||\
-									(le16_to_cpu(_pFwHdr->Signature)&0xFFF0) == 0x2300)
-
-/*  */
-/*  This structure must be cared byte-ordering */
-/*  */
-/*  Added by tynli. 2009.12.04. */
-struct rt_8723a_firmware_hdr {
-	/*  8-byte alinment required */
-
-	/*  LONG WORD 0 ---- */
-	__le16		Signature;  /*
-				     * 92C0: test chip; 92C, 88C0: test chip;
-				     * 88C1: MP A-cut; 92C1: MP A-cut
-				     */
-	u8		Category;	/*  AP/NIC and USB/PCI */
-	u8		Function;	/*  Reserved for different FW function indcation, for further use when driver needs to download different FW in different conditions */
-	__le16		Version;		/*  FW Version */
-	u8		Subversion;	/*  FW Subversion, default 0x00 */
-	u8		Rsvd1;
-
-
-	/*  LONG WORD 1 ---- */
-	u8		Month;	/*  Release time Month field */
-	u8		Date;	/*  Release time Date field */
-	u8		Hour;	/*  Release time Hour field */
-	u8		Minute;	/*  Release time Minute field */
-	__le16		RamCodeSize;	/*  The size of RAM code */
-	__le16		Rsvd2;
-
-	/*  LONG WORD 2 ---- */
-	__le32		SvnIdx;	/*  The SVN entry index */
-	__le32		Rsvd3;
-
-	/*  LONG WORD 3 ---- */
-	__le32		Rsvd4;
-	__le32		Rsvd5;
-};
-
-#define DRIVER_EARLY_INT_TIME		0x05
-#define BCN_DMA_ATIME_INT_TIME		0x02
-
-
-/*  BK, BE, VI, VO, HCCA, MANAGEMENT, COMMAND, HIGH, BEACON. */
-#define MAX_TX_QUEUE		9
-
-#define TX_SELE_HQ			BIT(0)		/*  High Queue */
-#define TX_SELE_LQ			BIT(1)		/*  Low Queue */
-#define TX_SELE_NQ			BIT(2)		/*  Normal Queue */
-
-/*  Note: We will divide number of page equally for each queue other than public queue! */
-#define TX_TOTAL_PAGE_NUMBER	0xF8
-#define TX_PAGE_BOUNDARY		(TX_TOTAL_PAGE_NUMBER + 1)
-
-/*  For Normal Chip Setting */
-/*  (HPQ + LPQ + NPQ + PUBQ) shall be TX_TOTAL_PAGE_NUMBER */
-#define NORMAL_PAGE_NUM_PUBQ	0xE7
-#define NORMAL_PAGE_NUM_HPQ		0x0C
-#define NORMAL_PAGE_NUM_LPQ		0x02
-#define NORMAL_PAGE_NUM_NPQ		0x02
-
-/*  For Test Chip Setting */
-/*  (HPQ + LPQ + PUBQ) shall be TX_TOTAL_PAGE_NUMBER */
-#define TEST_PAGE_NUM_PUBQ		0x7E
-
-/*  For Test Chip Setting */
-#define WMM_TEST_TX_TOTAL_PAGE_NUMBER	0xF5
-#define WMM_TEST_TX_PAGE_BOUNDARY		(WMM_TEST_TX_TOTAL_PAGE_NUMBER + 1) /* F6 */
-
-#define WMM_TEST_PAGE_NUM_PUBQ		0xA3
-#define WMM_TEST_PAGE_NUM_HPQ		0x29
-#define WMM_TEST_PAGE_NUM_LPQ		0x29
-
-/*  Note: For Normal Chip Setting, modify later */
-#define WMM_NORMAL_TX_TOTAL_PAGE_NUMBER	0xF5
-#define WMM_NORMAL_TX_PAGE_BOUNDARY		(WMM_TEST_TX_TOTAL_PAGE_NUMBER + 1) /* F6 */
-
-#define WMM_NORMAL_PAGE_NUM_PUBQ	0xB0
-#define WMM_NORMAL_PAGE_NUM_HPQ		0x29
-#define WMM_NORMAL_PAGE_NUM_LPQ		0x1C
-#define WMM_NORMAL_PAGE_NUM_NPQ		0x1C
-
-
-/*  */
-/*	Chip specific */
-/*  */
-#define CHIP_BONDING_IDENTIFIER(_value)	(((_value)>>22)&0x3)
-#define CHIP_BONDING_92C_1T2R			0x1
-#define CHIP_BONDING_88C_USB_MCARD		0x2
-#define CHIP_BONDING_88C_USB_HP			0x1
-
-#include "HalVerDef.h"
-#include "hal_com.h"
-
-/*  */
-/*	Channel Plan */
-/*  */
-enum ChannelPlan
-{
-	CHPL_FCC	= 0,
-	CHPL_IC		= 1,
-	CHPL_ETSI	= 2,
-	CHPL_SPAIN	= 3,
-	CHPL_FRANCE	= 4,
-	CHPL_MKK	= 5,
-	CHPL_MKK1	= 6,
-	CHPL_ISRAEL	= 7,
-	CHPL_TELEC	= 8,
-	CHPL_GLOBAL	= 9,
-	CHPL_WORLD	= 10,
-};
-
-#define EFUSE_REAL_CONTENT_LEN		512
-#define EFUSE_MAP_LEN				128
-#define EFUSE_MAX_SECTION			16
-#define EFUSE_IC_ID_OFFSET			506	/* For some inferiority IC purpose. added by Roger, 2009.09.02. */
-#define AVAILABLE_EFUSE_ADDR(addr)	(addr < EFUSE_REAL_CONTENT_LEN)
-/*  */
-/*  <Roger_Notes> */
-/*  To prevent out of boundary programming case, */
-/*  leave 1byte and program full section */
-/*  9bytes + 1byt + 5bytes and pre 1byte. */
-/*  For worst case: */
-/*  | 1byte|----8bytes----|1byte|--5bytes--| */
-/*  |         |            Reserved(14bytes)	      | */
-/*  */
-
-/*  PG data exclude header, dummy 6 bytes from CP test and reserved 1byte. */
-#define EFUSE_OOB_PROTECT_BYTES			15
-
-#define EFUSE_REAL_CONTENT_LEN_8723A	512
-#define EFUSE_MAP_LEN_8723A				256
-#define EFUSE_MAX_SECTION_8723A			32
-
-/*  */
-/*			EFUSE for BT definition */
-/*  */
-#define EFUSE_BT_REAL_BANK_CONTENT_LEN	512
-#define EFUSE_BT_REAL_CONTENT_LEN		1536	/*  512*3 */
-#define EFUSE_BT_MAP_LEN				1024	/*  1k bytes */
-#define EFUSE_BT_MAX_SECTION			128		/*  1024/8 */
-
-#define EFUSE_PROTECT_BYTES_BANK		16
-
-/*  */
-/*  <Roger_Notes> For RTL8723 WiFi/BT/GPS multi-function configuration. 2010.10.06. */
-/*  */
-enum RT_MULTI_FUNC {
-	RT_MULTI_FUNC_NONE = 0x00,
-	RT_MULTI_FUNC_WIFI = 0x01,
-	RT_MULTI_FUNC_BT = 0x02,
-	RT_MULTI_FUNC_GPS = 0x04,
-};
-
-/*  */
-/*  <Roger_Notes> For RTL8723 WiFi PDn/GPIO polarity control configuration. 2010.10.08. */
-/*  */
-enum RT_POLARITY_CTL {
-	RT_POLARITY_LOW_ACT = 0,
-	RT_POLARITY_HIGH_ACT = 1,
-};
-
-/*  For RTL8723 regulator mode. by tynli. 2011.01.14. */
-enum RT_REGULATOR_MODE {
-	RT_SWITCHING_REGULATOR = 0,
-	RT_LDO_REGULATOR = 1,
-};
-
-/*  Description: Determine the types of C2H events that are the same in driver and Fw. */
-/*  Fisrt constructed by tynli. 2009.10.09. */
-enum {
-	C2H_DBG = 0,
-	C2H_TSF = 1,
-	C2H_AP_RPT_RSP = 2,
-	C2H_CCX_TX_RPT = 3,	/*  The FW notify the report of the specific tx packet. */
-	C2H_BT_RSSI = 4,
-	C2H_BT_OP_MODE = 5,
-	C2H_EXT_RA_RPT = 6,
-	C2H_HW_INFO_EXCH = 10,
-	C2H_C2H_H2C_TEST = 11,
-	C2H_BT_INFO = 12,
-	C2H_BT_MP_INFO = 15,
-	MAX_C2HEVENT
-};
-
-struct hal_data_8723a {
-	struct hal_version		VersionID;
-	enum rt_customer_id CustomerID;
-
-	u16	FirmwareVersion;
-	u16	FirmwareVersionRev;
-	u16	FirmwareSubVersion;
-	u16	FirmwareSignature;
-
-	/* current WIFI_PHY values */
-	u32	ReceiveConfig;
-	enum WIRELESS_MODE		CurrentWirelessMode;
-	enum ht_channel_width	CurrentChannelBW;
-	u8	CurrentChannel;
-	u8	nCur40MhzPrimeSC;/*  Control channel sub-carrier */
-
-	u16	BasicRateSet;
-
-	/* rf_ctrl */
-	u8	rf_type;
-	u8	NumTotalRFPath;
-
-	u8	BoardType;
-	u8	CrystalCap;
-	/*  */
-	/*  EEPROM setting. */
-	/*  */
-	u8	EEPROMVersion;
-	u8	EEPROMCustomerID;
-	u8	EEPROMSubCustomerID;
-	u8	EEPROMRegulatory;
-	u8	EEPROMThermalMeter;
-	u8	EEPROMBluetoothCoexist;
-	u8	EEPROMBluetoothType;
-	u8	EEPROMBluetoothAntNum;
-	u8	EEPROMBluetoothAntIsolation;
-	u8	EEPROMBluetoothRadioShared;
-
-	u8	bTXPowerDataReadFromEEPORM;
-	u8	bAPKThermalMeterIgnore;
-
-	u8	bIQKInitialized;
-	u8	bAntennaDetected;
-
-	u8	TxPwrLevelCck[RF_PATH_MAX][CHANNEL_MAX_NUMBER];
-	u8	TxPwrLevelHT40_1S[RF_PATH_MAX][CHANNEL_MAX_NUMBER];	/*  For HT 40MHZ pwr */
-	u8	TxPwrLevelHT40_2S[RF_PATH_MAX][CHANNEL_MAX_NUMBER];	/*  For HT 40MHZ pwr */
-	u8	TxPwrHt20Diff[RF_PATH_MAX][CHANNEL_MAX_NUMBER];/*  HT 20<->40 Pwr diff */
-	u8	TxPwrLegacyHtDiff[RF_PATH_MAX][CHANNEL_MAX_NUMBER];/*  For HT<->legacy pwr diff */
-	/*  For power group */
-	u8	PwrGroupHT20[RF_PATH_MAX][CHANNEL_MAX_NUMBER];
-	u8	PwrGroupHT40[RF_PATH_MAX][CHANNEL_MAX_NUMBER];
-
-	u8	LegacyHTTxPowerDiff;/*  Legacy to HT rate power diff */
-
-	/*  Read/write are allow for following hardware information variables */
-	u8	framesync;
-	u32	framesyncC34;
-	u8	framesyncMonitor;
-	u8	pwrGroupCnt;
-	u32	MCSTxPowerLevelOriginalOffset[7][16];
-	u32	CCKTxPowerLevelOriginalOffset;
-
-	u32	AntennaTxPath;					/*  Antenna path Tx */
-	u32	AntennaRxPath;					/*  Antenna path Rx */
-	u8	ExternalPA;
-
-	u8	bLedOpenDrain; /*  Support Open-drain arrangement for controlling the LED. Added by Roger, 2009.10.16. */
-
-	u8	b1x1RecvCombine;	/*  for 1T1R receive combining */
-
-	/*  For EDCA Turbo mode */
-
-	u32	AcParam_BE; /* Original parameter for BE, use for EDCA turbo. */
-
-	/* vivi, for tx power tracking, 20080407 */
-	/* u16	TSSI_13dBm; */
-	/* u32	Pwr_Track; */
-	/*  The current Tx Power Level */
-	u8	CurrentCckTxPwrIdx;
-	u8	CurrentOfdm24GTxPwrIdx;
-
-	struct bb_reg_define	PHYRegDef[4];	/* Radio A/B/C/D */
-
-	bool		bRFPathRxEnable[4];	/*  We support 4 RF path now. */
-
-	u32	RfRegChnlVal[2];
-
-	u8	bCckHighPower;
-
-	/* RDG enable */
-	bool	 bRDGEnable;
-
-	/* for host message to fw */
-	u8	LastHMEBoxNum;
-
-	u8	RegTxPause;
-	/*  Beacon function related global variable. */
-	u8	RegFwHwTxQCtrl;
-	u8	RegReg542;
-
-	struct dm_priv	dmpriv;
-	struct dm_odm_t		odmpriv;
-	struct sreset_priv srestpriv;
-
-#ifdef CONFIG_8723AU_BT_COEXIST
-	u8				bBTMode;
-	/*  BT only. */
-	struct bt_30info		BtInfo;
-	/*  For bluetooth co-existance */
-	struct bt_coexist_str	bt_coexist;
-#endif
-
-	u8	bDumpRxPkt;/* for debug */
-	u8	FwRsvdPageStartOffset; /* 2010.06.23. Added by tynli. Reserve page start offset except beacon in TxQ. */
-
-	/*  2010/08/09 MH Add CU power down mode. */
-	u8	pwrdown;
-
-	u8	OutEpQueueSel;
-	u8	OutEpNumber;
-
-	/*  */
-	/*  Add For EEPROM Efuse switch and  Efuse Shadow map Setting */
-	/*  */
-	u8			EepromOrEfuse;
-	u16			EfuseUsedBytes;
-	u16			BTEfuseUsedBytes;
-
-	/*  Interrupt relatd register information. */
-	u32			SysIntrStatus;
-	u32			SysIntrMask;
-
-	/*  */
-	/*  2011/02/23 MH Add for 8723 mylti function definition. The define should be moved to an */
-	/*  independent file in the future. */
-	/*  */
-	/* 8723-----------------------------------------*/
-	enum RT_MULTI_FUNC	MultiFunc; /*  For multi-function consideration. */
-	enum RT_POLARITY_CTL	PolarityCtl; /*  For Wifi PDn Polarity control. */
-	enum RT_REGULATOR_MODE	RegulatorMode; /*  switching regulator or LDO */
-	/* 8723-----------------------------------------
-	 *  2011/02/23 MH Add for 8723 mylti function definition. The define should be moved to an */
-	/*  independent file in the future. */
-
-	/*  Interrupt related register information. */
-	u32	IntArray[2];
-	u32	IntrMask[2];
-};
-
-#define GET_HAL_DATA(__pAdapter)	((struct hal_data_8723a *)((__pAdapter)->HalData))
-#define GET_RF_TYPE(priv)			(GET_HAL_DATA(priv)->rf_type)
-
-#define INCLUDE_MULTI_FUNC_BT(_Adapter)		(GET_HAL_DATA(_Adapter)->MultiFunc & RT_MULTI_FUNC_BT)
-#define INCLUDE_MULTI_FUNC_GPS(_Adapter)	(GET_HAL_DATA(_Adapter)->MultiFunc & RT_MULTI_FUNC_GPS)
-
-struct rxreport_8723a {
-	u32 pktlen:14;
-	u32 crc32:1;
-	u32 icverr:1;
-	u32 drvinfosize:4;
-	u32 security:3;
-	u32 qos:1;
-	u32 shift:2;
-	u32 physt:1;
-	u32 swdec:1;
-	u32 ls:1;
-	u32 fs:1;
-	u32 eor:1;
-	u32 own:1;
-
-	u32 macid:5;
-	u32 tid:4;
-	u32 hwrsvd:4;
-	u32 amsdu:1;
-	u32 paggr:1;
-	u32 faggr:1;
-	u32 a1fit:4;
-	u32 a2fit:4;
-	u32 pam:1;
-	u32 pwr:1;
-	u32 md:1;
-	u32 mf:1;
-	u32 type:2;
-	u32 mc:1;
-	u32 bc:1;
-
-	u32 seq:12;
-	u32 frag:4;
-	u32 nextpktlen:14;
-	u32 nextind:1;
-	u32 rsvd0831:1;
-
-	u32 rxmcs:6;
-	u32 rxht:1;
-	u32 gf:1;
-	u32 splcp:1;
-	u32 bw:1;
-	u32 htc:1;
-	u32 eosp:1;
-	u32 bssidfit:2;
-	u32 rsvd1214:16;
-	u32 unicastwake:1;
-	u32 magicwake:1;
-
-	u32 pattern0match:1;
-	u32 pattern1match:1;
-	u32 pattern2match:1;
-	u32 pattern3match:1;
-	u32 pattern4match:1;
-	u32 pattern5match:1;
-	u32 pattern6match:1;
-	u32 pattern7match:1;
-	u32 pattern8match:1;
-	u32 pattern9match:1;
-	u32 patternamatch:1;
-	u32 patternbmatch:1;
-	u32 patterncmatch:1;
-	u32 rsvd1613:19;
-
-	u32 tsfl;
-
-	u32 bassn:12;
-	u32 bavld:1;
-	u32 rsvd2413:19;
-};
-
-/*  rtl8723a_hal_init.c */
-s32 rtl8723a_FirmwareDownload(struct rtw_adapter *padapter);
-void rtl8723a_FirmwareSelfReset(struct rtw_adapter *padapter);
-void rtl8723a_InitializeFirmwareVars(struct rtw_adapter *padapter);
-
-void rtl8723a_InitAntenna_Selection(struct rtw_adapter *padapter);
-void rtl8723a_DeinitAntenna_Selection(struct rtw_adapter *padapter);
-void rtl8723a_CheckAntenna_Selection(struct rtw_adapter *padapter);
-void rtl8723a_init_default_value(struct rtw_adapter *padapter);
-
-s32 InitLLTTable23a(struct rtw_adapter *padapter, u32 boundary);
-
-s32 CardDisableHWSM(struct rtw_adapter *padapter, u8 resetMCU);
-s32 CardDisableWithoutHWSM(struct rtw_adapter *padapter);
-
-/*  EFuse */
-u8 GetEEPROMSize8723A(struct rtw_adapter *padapter);
-void Hal_InitPGData(struct rtw_adapter *padapter, u8 *PROMContent);
-void Hal_EfuseParseIDCode(struct rtw_adapter *padapter, u8 *hwinfo);
-void Hal_EfuseParsetxpowerinfo_8723A(struct rtw_adapter *padapter, u8 *PROMContent, bool AutoLoadFail);
-void Hal_EfuseParseBTCoexistInfo_8723A(struct rtw_adapter *padapter, u8 *hwinfo, bool AutoLoadFail);
-void Hal_EfuseParseEEPROMVer(struct rtw_adapter *padapter, u8 *hwinfo, bool AutoLoadFail);
-void rtl8723a_EfuseParseChnlPlan(struct rtw_adapter *padapter, u8 *hwinfo, bool AutoLoadFail);
-void Hal_EfuseParseCustomerID(struct rtw_adapter *padapter, u8 *hwinfo, bool AutoLoadFail);
-void Hal_EfuseParseAntennaDiversity(struct rtw_adapter *padapter, u8 *hwinfo, bool AutoLoadFail);
-void Hal_EfuseParseRateIndicationOption(struct rtw_adapter *padapter, u8 *hwinfo, bool AutoLoadFail);
-void Hal_EfuseParseXtal_8723A(struct rtw_adapter *pAdapter, u8 *hwinfo, u8 AutoLoadFail);
-void Hal_EfuseParseThermalMeter_8723A(struct rtw_adapter *padapter, u8 *hwinfo, bool AutoLoadFail);
-
-/*  register */
-void SetBcnCtrlReg23a(struct rtw_adapter *padapter, u8 SetBits, u8 ClearBits);
-void rtl8723a_InitBeaconParameters(struct rtw_adapter *padapter);
-
-void rtl8723a_start_thread(struct rtw_adapter *padapter);
-void rtl8723a_stop_thread(struct rtw_adapter *padapter);
-
-bool c2h_id_filter_ccx_8723a(u8 id);
-int c2h_handler_8723a(struct rtw_adapter *padapter, struct c2h_evt_hdr *c2h_evt);
-
-void rtl8723a_read_adapter_info(struct rtw_adapter *Adapter);
-void rtl8723a_read_chip_version(struct rtw_adapter *padapter);
-void rtl8723a_notch_filter(struct rtw_adapter *adapter, bool enable);
-void rtl8723a_SetBeaconRelatedRegisters(struct rtw_adapter *padapter);
-void rtl8723a_SetHalODMVar(struct rtw_adapter *Adapter,
-			   enum hal_odm_variable eVariable,
-			   void *pValue1, bool bSet);
-void
-rtl8723a_readefuse(struct rtw_adapter *padapter,
-		   u8 efuseType, u16 _offset, u16 _size_byte, u8 *pbuf);
-u16 rtl8723a_EfuseGetCurrentSize_WiFi(struct rtw_adapter *padapter);
-u16 rtl8723a_EfuseGetCurrentSize_BT(struct rtw_adapter *padapter);
-void rtl8723a_update_ramask(struct rtw_adapter *padapter,
-			    u32 mac_id, u8 rssi_level);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_pg.h b/drivers/staging/rtl8723au/include/rtl8723a_pg.h
deleted file mode 100644
index 5c2ec448e568..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_pg.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTL8723A_PG_H__
-#define __RTL8723A_PG_H__
-
-/*			EEPROM/Efuse PG Offset for 8723E/8723U/8723S */
-#define EEPROM_CCK_TX_PWR_INX_8723A			0x10
-#define EEPROM_HT40_1S_TX_PWR_INX_8723A		0x16
-#define EEPROM_HT20_TX_PWR_INX_DIFF_8723A	0x1C
-#define EEPROM_OFDM_TX_PWR_INX_DIFF_8723A	0x1F
-#define EEPROM_HT40_MAX_PWR_OFFSET_8723A	0x22
-#define EEPROM_HT20_MAX_PWR_OFFSET_8723A	0x25
-
-#define EEPROM_ChannelPlan_8723A			0x28
-#define EEPROM_TSSI_A_8723A					0x29
-#define EEPROM_THERMAL_METER_8723A			0x2A
-#define RF_OPTION1_8723A					0x2B
-#define RF_OPTION2_8723A					0x2C
-#define RF_OPTION3_8723A					0x2D
-#define RF_OPTION4_8723A					0x2E
-#define EEPROM_VERSION_8723A				0x30
-#define EEPROM_CustomID_8723A				0x31
-#define EEPROM_SubCustomID_8723A			0x32
-#define EEPROM_XTAL_K_8723A					0x33
-#define EEPROM_Chipset_8723A				0x34
-
-/*  RTL8723AE */
-#define EEPROM_VID_8723AE					0x49
-#define EEPROM_DID_8723AE					0x4B
-#define EEPROM_SVID_8723AE					0x4D
-#define EEPROM_SMID_8723AE					0x4F
-#define EEPROM_MAC_ADDR_8723AE				0x67
-
-/*  RTL8723AU */
-#define EEPROM_MAC_ADDR_8723AU				0xC6
-#define EEPROM_VID_8723AU					0xB7
-#define EEPROM_PID_8723AU					0xB9
-
-/*  RTL8723AS */
-#define EEPROM_MAC_ADDR_8723AS				0xAA
-
-/*			EEPROM/Efuse Value Type */
-#define EETYPE_TX_PWR						0x0
-
-/*			EEPROM/Efuse Default Value */
-#define EEPROM_Default_CrystalCap_8723A		0x20
-
-
-/*        EEPROM/EFUSE data structure definition. */
-#define	MAX_CHNL_GROUP		3+9
-
-struct txpowerinfo {
-	u8 CCKIndex[RF_PATH_MAX][MAX_CHNL_GROUP];
-	u8 HT40_1SIndex[RF_PATH_MAX][MAX_CHNL_GROUP];
-	u8 HT40_2SIndexDiff[RF_PATH_MAX][MAX_CHNL_GROUP];
-	u8 HT20IndexDiff[RF_PATH_MAX][MAX_CHNL_GROUP];
-	u8 OFDMIndexDiff[RF_PATH_MAX][MAX_CHNL_GROUP];
-	u8 HT40MaxOffset[RF_PATH_MAX][MAX_CHNL_GROUP];
-	u8 HT20MaxOffset[RF_PATH_MAX][MAX_CHNL_GROUP];
-	u8 TSSI_A[3];
-	u8 TSSI_B[3];
-	u8 TSSI_A_5G[3];		/* 5GL/5GM/5GH */
-	u8 TSSI_B_5G[3];
-};
-
-enum bt_ant_num {
-	Ant_x2	= 0,
-	Ant_x1	= 1
-};
-
-enum bt_cotype {
-	BT_2Wire		= 0,
-	BT_ISSC_3Wire		= 1,
-	BT_Accel		= 2,
-	BT_CSR_BC4		= 3,
-	BT_CSR_BC8		= 4,
-	BT_RTL8756		= 5,
-	BT_RTL8723A		= 6
-};
-
-enum bt_radioshared {
-	BT_Radio_Shared		= 0,
-	BT_Radio_Individual	= 1,
-};
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_recv.h b/drivers/staging/rtl8723au/include/rtl8723a_recv.h
deleted file mode 100644
index 875d37b3b94c..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_recv.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTL8723A_RECV_H__
-#define __RTL8723A_RECV_H__
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#define NR_RECVBUFF			4
-
-#define NR_PREALLOC_RECV_SKB		8
-
-#define RECV_BLK_SZ			512
-#define RECV_BLK_CNT			16
-#define RECV_BLK_TH			RECV_BLK_CNT
-
-#define MAX_RECVBUF_SZ			15360 /*  15k < 16k */
-
-#define PHY_RSSI_SLID_WIN_MAX		100
-#define PHY_LINKQUALITY_SLID_WIN_MAX	20
-
-
-struct phy_stat {
-	unsigned int phydw0;
-	unsigned int phydw1;
-	unsigned int phydw2;
-	unsigned int phydw3;
-	unsigned int phydw4;
-	unsigned int phydw5;
-	unsigned int phydw6;
-	unsigned int phydw7;
-};
-
-/*  Rx smooth factor */
-#define	Rx_Smooth_Factor		20
-
-struct interrupt_msg_format {
-	unsigned int C2H_MSG0;
-	unsigned int C2H_MSG1;
-	unsigned int C2H_MSG2;
-	unsigned int C2H_MSG3;
-	unsigned int HISR; /*  from HISR Reg0x124, read to clear */
-	unsigned int HISRE;/*  from HISRE Reg0x12c, read to clear */
-	unsigned int  MSG_EX;
-};
-
-int rtl8723au_init_recv_priv(struct rtw_adapter *padapter);
-void rtl8723au_free_recv_priv(struct rtw_adapter *padapter);
-void rtl8723a_process_phy_info(struct rtw_adapter *padapter, void *prframe);
-void update_recvframe_attrib(struct recv_frame *precvframe, struct recv_stat *prxstat);
-void update_recvframe_phyinfo(struct recv_frame *precvframe, struct phy_stat *pphy_info);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_rf.h b/drivers/staging/rtl8723au/include/rtl8723a_rf.h
deleted file mode 100644
index 0432799f53cf..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_rf.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTL8723A_RF_H__
-#define __RTL8723A_RF_H__
-
-/*--------------------------Define Parameters-------------------------------*/
-
-/*  */
-/*  For RF 6052 Series */
-/*  */
-#define		RF6052_MAX_TX_PWR			0x3F
-#define		RF6052_MAX_REG				0x3F
-#define		RF6052_MAX_PATH				2
-/*--------------------------Define Parameters-------------------------------*/
-
-
-/*------------------------------Define structure----------------------------*/
-
-/*------------------------------Define structure----------------------------*/
-
-
-/*------------------------Export global variable----------------------------*/
-/*------------------------Export global variable----------------------------*/
-
-/*------------------------Export Marco Definition---------------------------*/
-
-/*------------------------Export Marco Definition---------------------------*/
-
-
-/*--------------------------Exported Function prototype---------------------*/
-
-/*  */
-/*  RF RL6052 Series API */
-/*  */
-void rtl8723a_phy_rf6052set_bw(struct rtw_adapter *Adapter,
-			       enum ht_channel_width Bandwidth);
-void rtl823a_phy_rf6052setccktxpower(struct rtw_adapter *Adapter,
-				      u8 *pPowerlevel);
-void rtl8723a_PHY_RF6052SetOFDMTxPower(struct rtw_adapter *Adapter,
-				       u8 *pPowerLevel, u8 Channel);
-
-/*--------------------------Exported Function prototype---------------------*/
-
-int	PHY_RF6052_Config8723A(struct rtw_adapter *Adapter);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_spec.h b/drivers/staging/rtl8723au/include/rtl8723a_spec.h
deleted file mode 100644
index 2f186890dbb5..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_spec.h
+++ /dev/null
@@ -1,2148 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- *******************************************************************************/
-#ifndef __RTL8723A_SPEC_H__
-#define __RTL8723A_SPEC_H__
-
-/*  */
-/*  */
-/*	0x0000h ~ 0x00FFh	System Configuration */
-/*  */
-/*  */
-#define REG_SYS_ISO_CTRL		0x0000
-#define REG_SYS_FUNC_EN			0x0002
-#define REG_APS_FSMCO			0x0004
-#define REG_SYS_CLKR			0x0008
-#define REG_9346CR			0x000A
-#define REG_EE_VPD			0x000C
-#define REG_AFE_MISC			0x0010
-#define REG_SPS0_CTRL			0x0011
-#define REG_SPS_OCP_CFG			0x0018
-#define REG_RSV_CTRL			0x001C
-#define REG_RF_CTRL			0x001F
-#define REG_LDOA15_CTRL			0x0020
-#define REG_LDOV12D_CTRL		0x0021
-#define REG_LDOHCI12_CTRL		0x0022
-#define REG_LPLDO_CTRL			0x0023
-#define REG_AFE_XTAL_CTRL		0x0024
-#define REG_AFE_PLL_CTRL		0x0028
-#define REG_MAC_PHY_CTRL		0x002c
-#define REG_EFUSE_CTRL			0x0030
-#define REG_EFUSE_TEST			0x0034
-#define REG_PWR_DATA			0x0038
-#define REG_CAL_TIMER			0x003C
-#define REG_ACLK_MON			0x003E
-#define REG_GPIO_MUXCFG			0x0040
-#define REG_GPIO_IO_SEL			0x0042
-#define REG_MAC_PINMUX_CFG		0x0043
-#define REG_GPIO_PIN_CTRL		0x0044
-#define REG_GPIO_INTM			0x0048
-#define REG_LEDCFG0			0x004C
-#define REG_LEDCFG1			0x004D
-#define REG_LEDCFG2			0x004E
-#define REG_LEDCFG3			0x004F
-#define REG_LEDCFG			REG_LEDCFG2
-#define REG_FSIMR			0x0050
-#define REG_FSISR			0x0054
-#define REG_HSIMR			0x0058
-#define REG_HSISR			0x005c
- /*  RTL8723 WIFI/BT/GPS Multi-Function GPIO Pin Control. */
-#define REG_GPIO_PIN_CTRL_2		0x0060
- /*  RTL8723 WIFI/BT/GPS Multi-Function GPIO Select. */
-#define REG_GPIO_IO_SEL_2		0x0062
- /*  RTL8723 WIFI/BT/GPS Multi-Function control source. */
-#define REG_MULTI_FUNC_CTRL		0x0068
-#define REG_MCUFWDL			0x0080
-#define REG_HMEBOX_EXT_0		0x0088
-#define REG_HMEBOX_EXT_1		0x008A
-#define REG_HMEBOX_EXT_2		0x008C
-#define REG_HMEBOX_EXT_3		0x008E
-	/*  Host suspend counter on FPGA platform */
-#define REG_HOST_SUSP_CNT		0x00BC
-	/*  Efuse access protection for RTL8723 */
-#define REG_EFUSE_ACCESS		0x00CF
-#define REG_BIST_SCAN			0x00D0
-#define REG_BIST_RPT			0x00D4
-#define REG_BIST_ROM_RPT		0x00D8
-#define REG_USB_SIE_INTF		0x00E0
-#define REG_PCIE_MIO_INTF		0x00E4
-#define REG_PCIE_MIO_INTD		0x00E8
-#define REG_HPON_FSM			0x00EC
-#define REG_SYS_CFG			0x00F0
-#define REG_GPIO_OUTSTS			0x00F4	/*  For RTL8723 only. */
-
-/*  */
-/*  */
-/*	0x0100h ~ 0x01FFh	MACTOP General Configuration */
-/*  */
-/*  */
-#define REG_CR				0x0100
-#define REG_PBP				0x0104
-#define REG_TRXDMA_CTRL			0x010C
-#define REG_TRXFF_BNDY			0x0114
-#define REG_TRXFF_STATUS		0x0118
-#define REG_RXFF_PTR			0x011C
-#define REG_HIMR			0x0120
-#define REG_HISR			0x0124
-#define REG_HIMRE			0x0128
-#define REG_HISRE			0x012C
-#define REG_CPWM			0x012F
-#define REG_FWIMR			0x0130
-#define REG_FWISR			0x0134
-#define REG_PKTBUF_DBG_CTRL		0x0140
-#define REG_PKTBUF_DBG_DATA_L		0x0144
-#define REG_PKTBUF_DBG_DATA_H		0x0148
-
-#define REG_TC0_CTRL			0x0150
-#define REG_TC1_CTRL			0x0154
-#define REG_TC2_CTRL			0x0158
-#define REG_TC3_CTRL			0x015C
-#define REG_TC4_CTRL			0x0160
-#define REG_TCUNIT_BASE			0x0164
-#define REG_MBIST_START			0x0174
-#define REG_MBIST_DONE			0x0178
-#define REG_MBIST_FAIL			0x017C
-#define REG_C2HEVT_MSG_NORMAL		0x01A0
-#define REG_C2HEVT_CLEAR		0x01AF
-#define REG_C2HEVT_MSG_TEST		0x01B8
-#define REG_MCUTST_1			0x01c0
-#define REG_FMETHR			0x01C8
-#define REG_HMETFR			0x01CC
-#define REG_HMEBOX_0			0x01D0
-#define REG_HMEBOX_1			0x01D4
-#define REG_HMEBOX_2			0x01D8
-#define REG_HMEBOX_3			0x01DC
-
-#define REG_LLT_INIT			0x01E0
-#define REG_BB_ACCEESS_CTRL		0x01E8
-#define REG_BB_ACCESS_DATA		0x01EC
-
-
-/*  */
-/*  */
-/*	0x0200h ~ 0x027Fh	TXDMA Configuration */
-/*  */
-/*  */
-#define REG_RQPN			0x0200
-#define REG_FIFOPAGE			0x0204
-#define REG_TDECTRL			0x0208
-#define REG_TXDMA_OFFSET_CHK		0x020C
-#define REG_TXDMA_STATUS		0x0210
-#define REG_RQPN_NPQ			0x0214
-
-/*  */
-/*  */
-/*	0x0280h ~ 0x02FFh	RXDMA Configuration */
-/*  */
-/*  */
-#define REG_RXDMA_AGG_PG_TH		0x0280
-#define REG_RXPKT_NUM			0x0284
-#define REG_RXDMA_STATUS		0x0288
-
-
-/*  */
-/*  */
-/*	0x0300h ~ 0x03FFh	PCIe */
-/*  */
-/*  */
-#define	REG_PCIE_CTRL_REG		0x0300
-#define	REG_INT_MIG			0x0304	/*  Interrupt Migration */
-	/*  TX Beacon Descriptor Address */
-#define	REG_BCNQ_DESA			0x0308
-	/*  TX High Queue Descriptor Address */
-#define	REG_HQ_DESA			0x0310
-	/*  TX Manage Queue Descriptor Address */
-#define	REG_MGQ_DESA			0x0318
-	/*  TX VO Queue Descriptor Address */
-#define	REG_VOQ_DESA			0x0320
-	/*  TX VI Queue Descriptor Address */
-#define	REG_VIQ_DESA			0x0328
-	/*  TX BE Queue Descriptor Address */
-#define	REG_BEQ_DESA			0x0330
-	/*  TX BK Queue Descriptor Address */
-#define	REG_BKQ_DESA			0x0338
-	/*  RX Queue	Descriptor Address */
-#define	REG_RX_DESA			0x0340
-	/*  Backdoor REG for Access Configuration */
-#define	REG_DBI				0x0348
-	/*  MDIO for Access PCIE PHY */
-#define	REG_MDIO			0x0354
-	/*  Debug Selection Register */
-#define	REG_DBG_SEL			0x0360
-	/* PCIe RPWM */
-#define	REG_PCIE_HRPWM			0x0361
-	/* PCIe CPWM */
-#define	REG_PCIE_HCPWM			0x0363
-	/*  UART	Control */
-#define	REG_UART_CTRL			0x0364
-	/*  UART TX Descriptor Address */
-#define	REG_UART_TX_DESA		0x0370
-	/*  UART Rx Descriptor Address */
-#define	REG_UART_RX_DESA		0x0378
-
-
-/*  spec version 11 */
-/*  */
-/*  */
-/*	0x0400h ~ 0x047Fh	Protocol Configuration */
-/*  */
-/*  */
-#define REG_VOQ_INFORMATION		0x0400
-#define REG_VIQ_INFORMATION		0x0404
-#define REG_BEQ_INFORMATION		0x0408
-#define REG_BKQ_INFORMATION		0x040C
-#define REG_MGQ_INFORMATION		0x0410
-#define REG_HGQ_INFORMATION		0x0414
-#define REG_BCNQ_INFORMATION		0x0418
-
-
-#define REG_CPU_MGQ_INFORMATION		0x041C
-#define REG_FWHW_TXQ_CTRL		0x0420
-#define REG_HWSEQ_CTRL			0x0423
-#define REG_TXPKTBUF_BCNQ_BDNY		0x0424
-#define REG_TXPKTBUF_MGQ_BDNY		0x0425
-#define REG_LIFETIME_EN			0x0426
-#define REG_MULTI_BCNQ_OFFSET		0x0427
-#define REG_SPEC_SIFS			0x0428
-#define REG_RL				0x042A
-#define REG_DARFRC			0x0430
-#define REG_RARFRC			0x0438
-#define REG_RRSR			0x0440
-#define REG_ARFR0			0x0444
-#define REG_ARFR1			0x0448
-#define REG_ARFR2			0x044C
-#define REG_ARFR3			0x0450
-#define REG_AGGLEN_LMT			0x0458
-#define REG_AMPDU_MIN_SPACE		0x045C
-#define REG_TXPKTBUF_WMAC_LBK_BF_HD	0x045D
-#define REG_FAST_EDCA_CTRL		0x0460
-#define REG_RD_RESP_PKT_TH		0x0463
-#define REG_INIRTS_RATE_SEL		0x0480
-#define REG_INIDATA_RATE_SEL		0x0484
-
-
-#define REG_POWER_STATUS		0x04A4
-#define REG_POWER_STAGE1		0x04B4
-#define REG_POWER_STAGE2		0x04B8
-#define REG_PKT_VO_VI_LIFE_TIME		0x04C0
-#define REG_PKT_BE_BK_LIFE_TIME		0x04C2
-#define REG_STBC_SETTING		0x04C4
-#define REG_PROT_MODE_CTRL		0x04C8
-#define REG_MAX_AGGR_NUM		0x04CA
-#define REG_RTS_MAX_AGGR_NUM		0x04CB
-#define REG_BAR_MODE_CTRL		0x04CC
-#define REG_RA_TRY_RATE_AGG_LMT		0x04CF
-#define REG_NQOS_SEQ			0x04DC
-#define REG_QOS_SEQ			0x04DE
-#define REG_NEED_CPU_HANDLE		0x04E0
-#define REG_PKT_LOSE_RPT		0x04E1
-#define REG_PTCL_ERR_STATUS		0x04E2
-#define REG_DUMMY			0x04FC
-
-
-
-/*  */
-/*  */
-/*	0x0500h ~ 0x05FFh	EDCA Configuration */
-/*  */
-/*  */
-#define REG_EDCA_VO_PARAM		0x0500
-#define REG_EDCA_VI_PARAM		0x0504
-#define REG_EDCA_BE_PARAM		0x0508
-#define REG_EDCA_BK_PARAM		0x050C
-#define REG_BCNTCFG			0x0510
-#define REG_PIFS			0x0512
-#define REG_RDG_PIFS			0x0513
-#define REG_SIFS_CCK			0x0514
-#define REG_SIFS_OFDM			0x0516
-#define REG_SIFS_CTX			0x0514
-#define REG_SIFS_TRX			0x0516
-#define REG_TSFTR_SYN_OFFSET		0x0518
-#define REG_AGGR_BREAK_TIME		0x051A
-#define REG_SLOT			0x051B
-#define REG_TX_PTCL_CTRL		0x0520
-#define REG_TXPAUSE			0x0522
-#define REG_DIS_TXREQ_CLR		0x0523
-#define REG_RD_CTRL			0x0524
-#define REG_TBTT_PROHIBIT		0x0540
-#define REG_RD_NAV_NXT			0x0544
-#define REG_NAV_PROT_LEN		0x0546
-#define REG_BCN_CTRL			0x0550
-#define REG_BCN_CTRL_1			0x0551
-#define REG_MBID_NUM			0x0552
-#define REG_DUAL_TSF_RST		0x0553
-	/*  The same as REG_MBSSID_BCN_SPACE */
-#define REG_BCN_INTERVAL		0x0554
-#define REG_MBSSID_BCN_SPACE		0x0554
-#define REG_DRVERLYINT			0x0558
-#define REG_BCNDMATIM			0x0559
-#define REG_ATIMWND			0x055A
-#define REG_BCN_MAX_ERR			0x055D
-#define REG_RXTSF_OFFSET_CCK		0x055E
-#define REG_RXTSF_OFFSET_OFDM		0x055F
-#define REG_TSFTR			0x0560
-#define REG_TSFTR1			0x0568
-#define REG_INIT_TSFTR			0x0564
-#define REG_ATIMWND_1			0x0570
-#define REG_PSTIMER			0x0580
-#define REG_TIMER0			0x0584
-#define REG_TIMER1			0x0588
-#define REG_ACMHWCTRL			0x05C0
-#define REG_ACMRSTCTRL			0x05C1
-#define REG_ACMAVG			0x05C2
-#define REG_VO_ADMTIME			0x05C4
-#define REG_VI_ADMTIME			0x05C6
-#define REG_BE_ADMTIME			0x05C8
-#define REG_EDCA_RANDOM_GEN		0x05CC
-#define REG_SCH_TXCMD			0x05D0
-
-/* define REG_FW_TSF_SYNC_CNT		0x04A0 */
-#define REG_FW_RESET_TSF_CNT_1		0x05FC
-#define REG_FW_RESET_TSF_CNT_0		0x05FD
-#define REG_FW_BCN_DIS_CNT		0x05FE
-
-/*  */
-/*  */
-/*	0x0600h ~ 0x07FFh	WMAC Configuration */
-/*  */
-/*  */
-#define REG_APSD_CTRL			0x0600
-#define REG_BWOPMODE			0x0603
-#define REG_TCR				0x0604
-#define REG_RCR				0x0608
-#define REG_RX_PKT_LIMIT		0x060C
-#define REG_RX_DLK_TIME			0x060D
-#define REG_RX_DRVINFO_SZ		0x060F
-
-#define REG_MACID			0x0610
-#define REG_BSSID			0x0618
-#define REG_MAR				0x0620
-#define REG_MBIDCAMCFG			0x0628
-
-#define REG_USTIME_EDCA			0x0638
-#define REG_MAC_SPEC_SIFS		0x063A
-
-/*  20100719 Joseph: Hardware register definition change. (HW datasheet v54) */
-	/*  [15:8]SIFS_R2T_OFDM, [7:0]SIFS_R2T_CCK */
-#define REG_R2T_SIFS			0x063C
-	/*  [15:8]SIFS_T2T_OFDM, [7:0]SIFS_T2T_CCK */
-#define REG_T2T_SIFS			0x063E
-#define REG_ACKTO			0x0640
-#define REG_CTS2TO			0x0641
-#define REG_EIFS			0x0642
-
-/* WMA, BA, CCX */
-#define REG_NAV_CTRL			0x0650
-#define REG_BACAMCMD			0x0654
-#define REG_BACAMCONTENT		0x0658
-#define REG_LBDLY			0x0660
-#define REG_FWDLY			0x0661
-#define REG_RXERR_RPT			0x0664
-#define REG_WMAC_TRXPTCL_CTL		0x0668
-
-
-/*  Security */
-#define REG_CAMCMD			0x0670
-#define REG_CAMWRITE			0x0674
-#define REG_CAMREAD			0x0678
-#define REG_CAMDBG			0x067C
-#define REG_SECCFG			0x0680
-
-/*  Power */
-#define REG_WOW_CTRL			0x0690
-#define REG_PSSTATUS			0x0691
-#define REG_PS_RX_INFO			0x0692
-#define REG_LPNAV_CTRL			0x0694
-#define REG_WKFMCAM_CMD			0x0698
-#define REG_WKFMCAM_RWD			0x069C
-#define REG_RXFLTMAP0			0x06A0
-#define REG_RXFLTMAP1			0x06A2
-#define REG_RXFLTMAP2			0x06A4
-#define REG_BCN_PSR_RPT			0x06A8
-#define REG_CALB32K_CTRL		0x06AC
-#define REG_PKT_MON_CTRL		0x06B4
-#define REG_BT_COEX_TABLE		0x06C0
-#define REG_WMAC_RESP_TXINFO		0x06D8
-
-#define REG_MACID1			0x0700
-#define REG_BSSID1			0x0708
-
-
-/*  */
-/*  */
-/*	0xFE00h ~ 0xFE55h	USB Configuration */
-/*  */
-/*  */
-#define REG_USB_INFO			0xFE17
-#define REG_USB_SPECIAL_OPTION		0xFE55
-#define REG_USB_DMA_AGG_TO		0xFE5B
-#define REG_USB_AGG_TO			0xFE5C
-#define REG_USB_AGG_TH			0xFE5D
-
-/*  For test chip */
-#define REG_TEST_USB_TXQS		0xFE48
-#define REG_TEST_SIE_VID		0xFE60		/*  0xFE60~0xFE61 */
-#define REG_TEST_SIE_PID		0xFE62		/*  0xFE62~0xFE63 */
-#define REG_TEST_SIE_OPTIONAL		0xFE64
-#define REG_TEST_SIE_CHIRP_K		0xFE65
-#define REG_TEST_SIE_PHY		0xFE66		/*  0xFE66~0xFE6B */
-#define REG_TEST_SIE_MAC_ADDR		0xFE70		/*  0xFE70~0xFE75 */
-#define REG_TEST_SIE_STRING		0xFE80		/*  0xFE80~0xFEB9 */
-
-
-/*  For normal chip */
-#define REG_NORMAL_SIE_VID		0xFE60		/*  0xFE60~0xFE61 */
-#define REG_NORMAL_SIE_PID		0xFE62		/*  0xFE62~0xFE63 */
-#define REG_NORMAL_SIE_OPTIONAL		0xFE64
-#define REG_NORMAL_SIE_EP		0xFE65		/*  0xFE65~0xFE67 */
-#define REG_NORMAL_SIE_PHY		0xFE68		/*  0xFE68~0xFE6B */
-#define REG_NORMAL_SIE_OPTIONAL2	0xFE6C
-#define REG_NORMAL_SIE_GPS_EP		0xFE6D		/*  RTL8723 only */
-#define REG_NORMAL_SIE_MAC_ADDR		0xFE70		/*  0xFE70~0xFE75 */
-#define REG_NORMAL_SIE_STRING		0xFE80		/*  0xFE80~0xFEDF */
-
-
-/*  */
-/*  */
-/*	Redifine 8192C register definition for compatibility */
-/*  */
-/*  */
-
-/*  TODO: use these definition when using REG_xxx naming rule. */
-/*  NOTE: DO NOT Remove these definition. Use later. */
-
-	/*  System Isolation Interface Control. */
-#define	SYS_ISO_CTRL			REG_SYS_ISO_CTRL
-	/*  System Function Enable. */
-#define	SYS_FUNC_EN			REG_SYS_FUNC_EN
-#define	SYS_CLK				REG_SYS_CLKR
-	/*  93C46/93C56 Command Register. */
-#define	CR9346				REG_9346CR
-	/*  E-Fuse Control. */
-#define	EFUSE_CTRL			REG_EFUSE_CTRL
-	/*  E-Fuse Test. */
-#define	EFUSE_TEST			REG_EFUSE_TEST
-	/*  Media Status register */
-#define	MSR				(REG_CR + 2)
-#define	ISR				REG_HISR
-	/*  Timing Sync Function Timer Register. */
-#define	TSFR				REG_TSFTR
-
-	/*  MAC ID Register, Offset 0x0050-0x0053 */
-#define	MACIDR0				REG_MACID
-	/*  MAC ID Register, Offset 0x0054-0x0055 */
-#define	MACIDR4				(REG_MACID + 4)
-
-#define	PBP				REG_PBP
-
-	/*  Redifine MACID register, to compatible prior ICs. */
-#define	IDR0				MACIDR0
-#define	IDR4				MACIDR4
-
-
-/*  */
-/*  9. Security Control Registers	(Offset: ) */
-/*  */
-	/*  Software write CAM input content */
-#define	WCAMI				REG_CAMWRITE
-	/*  Software read/write CAM config */
-#define	RCAMO				REG_CAMREAD
-#define	CAMDBG				REG_CAMDBG
-	/* Security Configuration Register */
-#define	SECR				REG_SECCFG
-
-/*  Unused register */
-#define	UnusedRegister			0x1BF
-#define	DCAM				UnusedRegister
-#define	PSR				UnusedRegister
-#define	BBAddr				UnusedRegister
-#define	PhyDataR			UnusedRegister
-
-#define	InvalidBBRFValue		0x12345678
-
-/*  Min Spacing related settings. */
-#define	MAX_MSS_DENSITY_2T		0x13
-#define	MAX_MSS_DENSITY_1T		0x0A
-
-/*  */
-/* 8192C Cmd9346CR bits			(Offset 0xA, 16bit) */
-/*  */
-	 /*  EEPROM enable when set 1 */
-#define	CmdEEPROM_En			BIT(5)
-	/*  System EEPROM select, 0: boot from E-FUSE,
-	    1: The EEPROM used is 9346 */
-#define	CmdEERPOMSEL			BIT(4)
-#define	Cmd9346CR_9356SEL		BIT(4)
-#define	AutoLoadEEPROM			(CmdEEPROM_En|CmdEERPOMSEL)
-#define	AutoLoadEFUSE			CmdEEPROM_En
-
-/*  */
-/* 8192C GPIO MUX Configuration Register (offset 0x40, 4 byte) */
-/*  */
-#define	GPIOSEL_GPIO			0
-#define	GPIOSEL_ENBT			BIT(5)
-
-/*  */
-/*        8192C GPIO PIN Control Register (offset 0x44, 4 byte) */
-/*  */
-	/*  GPIO pins input value */
-#define	GPIO_IN				REG_GPIO_PIN_CTRL
-	/*  GPIO pins output value */
-#define	GPIO_OUT			(REG_GPIO_PIN_CTRL+1)
-	/*  GPIO pins output enable when a bit is set to "1";
-	    otherwise, input is configured. */
-#define	GPIO_IO_SEL			(REG_GPIO_PIN_CTRL+2)
-#define	GPIO_MOD			(REG_GPIO_PIN_CTRL+3)
-
-/*  */
-/* 8192C (MSR) Media Status Register	(Offset 0x4C, 8 bits) */
-/*  */
-/*
-Network Type
-00: No link
-01: Link in ad hoc network
-10: Link in infrastructure network
-11: AP mode
-Default: 00b.
-*/
-#define	MSR_NOLINK			0x00
-#define	MSR_ADHOC			0x01
-#define	MSR_INFRA			0x02
-#define	MSR_AP				0x03
-
-/*  */
-/*  6. Adaptive Control Registers  (Offset: 0x0160 - 0x01CF) */
-/*  */
-/*  */
-/* 8192C Response Rate Set Register	(offset 0x181, 24bits) */
-/*  */
-#define	RRSR_RSC_OFFSET			21
-#define	RRSR_SHORT_OFFSET		23
-#define	RRSR_RSC_BW_40M			0x600000
-#define	RRSR_RSC_UPSUBCHNL		0x400000
-#define	RRSR_RSC_LOWSUBCHNL		0x200000
-#define	RRSR_SHORT			0x800000
-#define	RRSR_1M				BIT(0)
-#define	RRSR_2M				BIT(1)
-#define	RRSR_5_5M			BIT(2)
-#define	RRSR_11M			BIT(3)
-#define	RRSR_6M				BIT(4)
-#define	RRSR_9M				BIT(5)
-#define	RRSR_12M			BIT(6)
-#define	RRSR_18M			BIT(7)
-#define	RRSR_24M			BIT(8)
-#define	RRSR_36M			BIT(9)
-#define	RRSR_48M			BIT(10)
-#define	RRSR_54M			BIT(11)
-#define	RRSR_MCS0			BIT(12)
-#define	RRSR_MCS1			BIT(13)
-#define	RRSR_MCS2			BIT(14)
-#define	RRSR_MCS3			BIT(15)
-#define	RRSR_MCS4			BIT(16)
-#define	RRSR_MCS5			BIT(17)
-#define	RRSR_MCS6			BIT(18)
-#define	RRSR_MCS7			BIT(19)
-#define	BRSR_AckShortPmb		BIT(23)
-/*  CCK ACK: use Short Preamble or not */
-
-/*  */
-/* 8192C BW_OPMODE bits			(Offset 0x203, 8bit) */
-/*  */
-#define	BW_OPMODE_20MHZ			BIT(2)
-#define	BW_OPMODE_5G			BIT(1)
-#define	BW_OPMODE_11J			BIT(0)
-
-
-/*  */
-/* 8192C CAM Config Setting (offset 0x250, 1 byte) */
-/*  */
-#define	CAM_VALID			BIT(15)
-#define	CAM_NOTVALID			0x0000
-#define	CAM_USEDK			BIT(5)
-
-#define	CAM_CONTENT_COUNT		8
-
-#define	CAM_NONE			0x0
-#define	CAM_WEP40			0x01
-#define	CAM_TKIP			0x02
-#define	CAM_AES				0x04
-#define	CAM_WEP104			0x05
-
-#define	TOTAL_CAM_ENTRY			32
-#define	HALF_CAM_ENTRY			16
-
-#define	CAM_CONFIG_USEDK		true
-#define	CAM_CONFIG_NO_USEDK		false
-
-#define	CAM_WRITE			BIT(16)
-#define	CAM_READ			0x00000000
-#define	CAM_POLLINIG			BIT(31)
-
-#define	SCR_UseDK			0x01
-#define	SCR_TxSecEnable			0x02
-#define	SCR_RxSecEnable			0x04
-
-
-/*  */
-/*  12. Host Interrupt Status Registers	 (Offset: 0x0300 - 0x030F) */
-/*  */
-/*  */
-/* 8190 IMR/ISR bits			(offset 0xfd,  8bits) */
-/*  */
-#define	IMR8190_DISABLED		0x0
-/*  IMR DW0 Bit 0-31 */
-
-#define	IMR_BCNDMAINT6			BIT(31)	/*  Beacon DMA Interrupt 6 */
-#define	IMR_BCNDMAINT5			BIT(30)	/*  Beacon DMA Interrupt 5 */
-#define	IMR_BCNDMAINT4			BIT(29)	/*  Beacon DMA Interrupt 4 */
-#define	IMR_BCNDMAINT3			BIT(28)	/*  Beacon DMA Interrupt 3 */
-#define	IMR_BCNDMAINT2			BIT(27)	/*  Beacon DMA Interrupt 2 */
-#define	IMR_BCNDMAINT1			BIT(26)	/*  Beacon DMA Interrupt 1 */
-#define	IMR_BCNDOK8			BIT(25)	/*  Beacon Queue DMA OK
-						    Interrupt 8 */
-#define	IMR_BCNDOK7			BIT(24)	/*  Beacon Queue DMA OK
-						    Interrupt 7 */
-#define	IMR_BCNDOK6			BIT(23)	/*  Beacon Queue DMA OK
-						    Interrupt 6 */
-#define	IMR_BCNDOK5			BIT(22)	/*  Beacon Queue DMA OK
-						    Interrupt 5 */
-#define	IMR_BCNDOK4			BIT(21)	/*  Beacon Queue DMA OK
-						    Interrupt 4 */
-#define	IMR_BCNDOK3			BIT(20)	/*  Beacon Queue DMA OK
-						    Interrupt 3 */
-#define	IMR_BCNDOK2			BIT(19)	/*  Beacon Queue DMA OK
-						    Interrupt 2 */
-#define	IMR_BCNDOK1			BIT(18)	/*  Beacon Queue DMA OK
-						    Interrupt 1 */
-#define	IMR_TIMEOUT2			BIT(17)	/*  Timeout interrupt 2 */
-#define	IMR_TIMEOUT1			BIT(16)	/*  Timeout interrupt 1 */
-#define	IMR_TXFOVW			BIT(15)	/*  Transmit FIFO Overflow */
-#define	IMR_PSTIMEOUT			BIT(14)	/*  Power save time out
-						    interrupt */
-#define	IMR_BcnInt			BIT(13)	/*  Beacon DMA Interrupt 0 */
-#define	IMR_RXFOVW			BIT(12)	/*  Receive FIFO Overflow */
-#define	IMR_RDU				BIT(11)	/*  Receive Descriptor
-						    Unavailable */
-#define	IMR_ATIMEND			BIT(10)	/*  For 92C,ATIM Window
-						    End Interrupt */
-#define	IMR_BDOK			BIT(9)	/*  Beacon Queue DMA OK
-						    Interrup */
-#define	IMR_HIGHDOK			BIT(8)	/*  High Queue DMA OK
-						    Interrupt */
-#define	IMR_TBDOK			BIT(7)	/*  Transmit Beacon OK
-						    interrup */
-#define	IMR_MGNTDOK			BIT(6)	/*  Management Queue DMA OK
-						    Interrupt */
-#define	IMR_TBDER			BIT(5)	/*  For 92C,Transmit Beacon
-						    Error Interrupt */
-#define	IMR_BKDOK			BIT(4)	/*  AC_BK DMA OK Interrupt */
-#define	IMR_BEDOK			BIT(3)	/*  AC_BE DMA OK Interrupt */
-#define	IMR_VIDOK			BIT(2)	/*  AC_VI DMA OK Interrupt */
-#define	IMR_VODOK			BIT(1)	/*  AC_VO DMA Interrupt */
-#define	IMR_ROK				BIT(0)	/*  Receive DMA OK Interrupt */
-
-#define	IMR_RX_MASK			(IMR_ROK|IMR_RDU|IMR_RXFOVW)
-#define	IMR_TX_MASK			(IMR_VODOK|IMR_VIDOK|IMR_BEDOK| \
-					 IMR_BKDOK|IMR_MGNTDOK|IMR_HIGHDOK| \
-					 IMR_BDOK)
-
-/*  13. Host Interrupt Status Extension Register (Offset: 0x012C-012Eh) */
-#define	IMR_BcnInt_E			BIT(12)
-#define	IMR_TXERR			BIT(11)
-#define	IMR_RXERR			BIT(10)
-#define	IMR_C2HCMD			BIT(9)
-#define	IMR_CPWM			BIT(8)
-/* RSVD [2-7] */
-#define	IMR_OCPINT			BIT(1)
-#define	IMR_WLANOFF			BIT(0)
-
-
-/*        8192C EEPROM/EFUSE share register definition. */
-
-/*  Default Value for EEPROM or EFUSE!!! */
-#define EEPROM_Default_TSSI			0x0
-#define EEPROM_Default_TxPowerDiff		0x0
-#define EEPROM_Default_CrystalCap		0x5
- /*  Default: 2X2, RTL8192CE(QFPN68) */
-#define EEPROM_Default_BoardType		0x02
-#define EEPROM_Default_TxPower			0x1010
-#define EEPROM_Default_HT2T_TxPwr		0x10
-
-#define EEPROM_Default_LegacyHTTxPowerDiff	0x3
-#define EEPROM_Default_ThermalMeter		0x12
-
-#define EEPROM_Default_AntTxPowerDiff		0x0
-#define EEPROM_Default_TxPwDiff_CrystalCap	0x5
-#define EEPROM_Default_TxPowerLevel		0x22
-#define EEPROM_Default_HT40_2SDiff		0x0
-	/*  HT20<->40 default Tx Power Index Difference */
-#define EEPROM_Default_HT20_Diff		2
-#define EEPROM_Default_LegacyHTTxPowerDiff	0x3
-#define EEPROM_Default_HT40_PwrMaxOffset	0
-#define EEPROM_Default_HT20_PwrMaxOffset	0
-
-/*  For debug */
-#define EEPROM_Default_PID			0x1234
-#define EEPROM_Default_VID			0x5678
-#define EEPROM_Default_CustomerID		0xAB
-#define EEPROM_Default_SubCustomerID		0xCD
-#define EEPROM_Default_Version			0
-
-#define EEPROM_CHANNEL_PLAN_FCC			0x0
-#define EEPROM_CHANNEL_PLAN_IC			0x1
-#define EEPROM_CHANNEL_PLAN_ETSI		0x2
-#define EEPROM_CHANNEL_PLAN_SPAIN		0x3
-#define EEPROM_CHANNEL_PLAN_FRANCE		0x4
-#define EEPROM_CHANNEL_PLAN_MKK			0x5
-#define EEPROM_CHANNEL_PLAN_MKK1		0x6
-#define EEPROM_CHANNEL_PLAN_ISRAEL		0x7
-#define EEPROM_CHANNEL_PLAN_TELEC		0x8
-#define EEPROM_CHANNEL_PLAN_GLOBAL_DOMAIN	0x9
-#define EEPROM_CHANNEL_PLAN_WORLD_WIDE_13	0xA
-#define EEPROM_CHANNEL_PLAN_NCC			0xB
-#define EEPROM_USB_OPTIONAL1			0xE
-#define EEPROM_CHANNEL_PLAN_BY_HW_MASK		0x80
-
-
-#define EEPROM_CID_DEFAULT			0x0
-#define EEPROM_CID_TOSHIBA			0x4
- /*  CCX test. By Bruce, 2009-02-25. */
-#define EEPROM_CID_CCX				0x10
-#define EEPROM_CID_QMI				0x0D
- /*  added by chiyoko for dtm, 20090108 */
-#define EEPROM_CID_WHQL				0xFE
-
-
-#define	RTL_EEPROM_ID				0x8129
-
-#define	SUPPORT_HW_RADIO_DETECT(pHalData)		\
-	(pHalData->BoardType == BOARD_MINICARD ||	\
-	pHalData->BoardType == BOARD_USB_SOLO ||	\
-	pHalData->BoardType == BOARD_USB_COMBO)
-
-/*  */
-/*  EEPROM address for Test chip */
-/*  */
-#define EEPROM_TEST_USB_OPT		0x0E
-#define EEPROM_TEST_CHIRP_K		0x0F
-#define EEPROM_TEST_EP_SETTING		0x0E
-#define EEPROM_TEST_USB_PHY		0x10
-
-
-/*  */
-/*  EEPROM address for Normal chip */
-/*  */
-#define EEPROM_NORMAL_USB_OPT		0x0E
-#define EEPROM_NORMAL_CHIRP_K		0x0E	/*  Changed */
-#define EEPROM_NORMAL_EP_SETTING	0x0F	/*  Changed */
-#define EEPROM_NORMAL_USB_PHY		0x12	/*  Changed */
-
-enum {
-	BOARD_USB_DONGLE = 0,	/*  USB dongle */
-	BOARD_USB_High_PA = 1,	/*  USB dongle with high power PA */
-	BOARD_MINICARD = 2,	/*  Minicard */
-	BOARD_USB_SOLO = 3,	/*  USB solo-Slim module */
-	BOARD_USB_COMBO = 4,	/*  USB Combo-Slim module */
-};
-
-/*  Test chip and normal chip common define */
-/*  */
-/*  EEPROM address for both */
-/*  */
-#define EEPROM_ID0			0x00
-#define EEPROM_ID1			0x01
-#define EEPROM_RTK_RSV1			0x02
-#define EEPROM_RTK_RSV2			0x03
-#define EEPROM_RTK_RSV3			0x04
-#define EEPROM_RTK_RSV4			0x05
-#define EEPROM_RTK_RSV5			0x06
-#define EEPROM_DBG_SEL			0x07
-#define EEPROM_RTK_RSV6			0x08
-#define EEPROM_VID			0x0A
-#define EEPROM_PID			0x0C
-
-#define EEPROM_MAC_ADDR			0x16
-#define EEPROM_STRING			0x1C
-#define EEPROM_SUBCUSTOMER_ID		0x59
-#define EEPROM_CCK_TX_PWR_INX		0x5A
-#define EEPROM_HT40_1S_TX_PWR_INX	0x60
-#define EEPROM_HT40_2S_TX_PWR_INX_DIFF	0x66
-#define EEPROM_HT20_TX_PWR_INX_DIFF	0x69
-#define EEPROM_OFDM_TX_PWR_INX_DIFF	0x6C
-#define EEPROM_HT40_MAX_PWR_OFFSET	0x6F
-#define EEPROM_HT20_MAX_PWR_OFFSET	0x72
-
-#define EEPROM_CHANNEL_PLAN		0x75
-#define EEPROM_TSSI_A			0x76
-#define EEPROM_TSSI_B			0x77
-#define EEPROM_THERMAL_METER		0x78
-#define EEPROM_RF_OPT1			0x79
-#define EEPROM_RF_OPT2			0x7A
-#define EEPROM_RF_OPT3			0x7B
-#define EEPROM_RF_OPT4			0x7C
-#define EEPROM_VERSION			0x7E
-#define EEPROM_CUSTOMER_ID		0x7F
-
-	 /* 0x0: RTL8188SU, 0x1: RTL8191SU, 0x2: RTL8192SU, 0x3: RTL8191GU */
-#define EEPROM_BoardType		0x54
-	/* 0x5C-0x76, Tx Power index. */
-#define EEPROM_TxPwIndex		0x5C
-	/*  Difference of gain index between legacy and high throughput OFDM. */
-#define EEPROM_PwDiff			0x67
-	/*  CCK Tx Power */
-#define EEPROM_TxPowerCCK		0x5A
-
-/*  2009/02/09 Cosa Add for SD3 requirement */
-	/*  HT20 Tx Power Index Difference */
-#define EEPROM_TX_PWR_HT20_DIFF		0x6e
-	/*  HT20<->40 default Tx Power Index Difference */
-#define DEFAULT_HT20_TXPWR_DIFF		2
-	/*  OFDM Tx Power Index Difference */
-#define EEPROM_TX_PWR_OFDM_DIFF		0x71
-
-	/*  Power diff for channel group */
-#define EEPROM_TxPWRGroup		0x73
-	/*  Check if power safety is need */
-#define EEPROM_Regulatory		0x79
-
-	/*  92cu, 0x7E[4] */
-#define EEPROM_BLUETOOTH_COEXIST	0x7E
-#define EEPROM_NORMAL_BoardType		EEPROM_RF_OPT1	/* 7:5] */
-#define BOARD_TYPE_NORMAL_MASK		0xE0
-#define BOARD_TYPE_TEST_MASK		0x0F
-	/* BIT0 1 for build-in module, 0 for external dongle */
-#define EEPROM_EASY_REPLACEMENT		0x50
-/*  */
-/* EPROM content definitions */
-/*  */
-#define OS_LINK_SPEED			BIT(5)
-
-#define BOARD_TYPE_MASK			0xF
-
-#define BT_COEXISTENCE			BIT(4)
-#define BT_CO_SHIFT			4
-
-#define EP_NUMBER_MASK			0x30	/* bit 4:5 0Eh */
-#define EP_NUMBER_SHIFT			4
-
-
-#define USB_PHY_PARA_SIZE		5
-
-
-/*  */
-/*	EEPROM default value definitions */
-/*  */
-/*  Use 0xABCD instead of 0x8192 for debug */
-#define EEPROM_DEF_ID_0			0xCD	/*  Byte 0x00 */
-#define EEPROM_DEF_ID_1			0xAB	/*  Byte 0x01 */
-
-#define EEPROM_DEF_RTK_RSV_A3		0x74	/*  Byte 0x03 */
-#define EEPROM_DEF_RTK_RSV_A4		0x6D	/*  Byte 0x04 */
-#define EEPROM_DEF_RTK_RSV_A8		0xFF	/*  Byte 0x08 */
-
-#define EEPROM_DEF_VID_0		0x0A	/*  Byte 0x0A */
-#define EEPROM_DEF_VID_1		0x0B
-
-#define EEPROM_DEF_PID_0		0x92	/*  Byte 0x0C */
-#define EEPROM_DEF_PID_1		0x81
-
-
-#define EEPROM_TEST_DEF_USB_OPT		0x80	/*  Byte 0x0E */
-#define EEPROM_NORMAL_DEF_USB_OPT	0x00	/*  Byte 0x0E */
-
-#define EEPROM_DEF_CHIRPK		0x15	/*  Byte 0x0F */
-
-#define EEPROM_DEF_USB_PHY_0		0x85	/*  Byte 0x10 */
-#define EEPROM_DEF_USB_PHY_1		0x62	/*  Byte 0x11 */
-#define EEPROM_DEF_USB_PHY_2		0x9E	/*  Byte 0x12 */
-#define EEPROM_DEF_USB_PHY_3		0x06	/*  Byte 0x13 */
-
-#define EEPROM_DEF_TSSI_A		0x09	/*  Byte 0x78 */
-#define EEPROM_DEF_TSSI_B		0x09	/*  Byte 0x79 */
-
-
-#define EEPROM_DEF_THERMAL_METER	0x12	/*  Byte 0x7A */
-
-	/*  Check if power safety spec is need */
-#define RF_OPTION1			0x79
-#define RF_OPTION2			0x7A
-#define RF_OPTION3			0x7B
-#define RF_OPTION4			0x7C
-
-
-#define	EEPROM_USB_SN			BIT(0)
-#define	EEPROM_USB_REMOTE_WAKEUP	BIT(1)
-#define	EEPROM_USB_DEVICE_PWR		BIT(2)
-#define	EEPROM_EP_NUMBER		(BIT(3)|BIT(4))
-
-/*===================================================================
-=====================================================================
-Here the register defines are for 92C. When the define is as same with 92C,
-we will use the 92C's define for the consistency
-So the following defines for 92C is not entire!!!!!!
-=====================================================================
-=====================================================================*/
-/*
-Based on Datasheet V33---090401
-Register Summary
-Current IOREG MAP
-0x0000h ~ 0x00FFh   System Configuration (256 Bytes)
-0x0100h ~ 0x01FFh   MACTOP General Configuration (256 Bytes)
-0x0200h ~ 0x027Fh   TXDMA Configuration (128 Bytes)
-0x0280h ~ 0x02FFh   RXDMA Configuration (128 Bytes)
-0x0300h ~ 0x03FFh   PCIE EMAC Reserved Region (256 Bytes)
-0x0400h ~ 0x04FFh   Protocol Configuration (256 Bytes)
-0x0500h ~ 0x05FFh   EDCA Configuration (256 Bytes)
-0x0600h ~ 0x07FFh   WMAC Configuration (512 Bytes)
-0x2000h ~ 0x3FFFh   8051 FW Download Region (8196 Bytes)
-*/
-
-/*  */
-/* 8192C (RCR) Receive Configuration Register	(Offset 0x608, 32 bits) */
-/*  */
-#define	RCR_APPFCS			BIT(31) /* WMAC append FCS after payload*/
-#define	RCR_APP_MIC			BIT(30)
-#define	RCR_APP_PHYSTS			BIT(28)
-#define	RCR_APP_ICV			BIT(29)
-#define	RCR_APP_PHYST_RXFF		BIT(28)
-#define	RCR_APP_BA_SSN			BIT(27) /* Accept BA SSN */
-#define	RCR_ENMBID			BIT(24) /* Enable Multiple BssId. */
-#define	RCR_LSIGEN			BIT(23)
-#define	RCR_MFBEN			BIT(22)
-#define	RCR_HTC_LOC_CTRL		BIT(14) /* MFC<--HTC=1 MFC-->HTC=0 */
-#define	RCR_AMF				BIT(13) /* Accept management type frame */
-#define	RCR_ACF				BIT(12) /* Accept control type frame */
-#define	RCR_ADF				BIT(11) /* Accept data type frame */
-#define	RCR_AICV			BIT(9)  /* Accept ICV error packet */
-#define	RCR_ACRC32			BIT(8)  /* Accept CRC32 error packet */
-#define	RCR_CBSSID_BCN			BIT(7)  /* Accept BSSID match packet
-						 (Rx beacon, probe rsp) */
-#define	RCR_CBSSID_DATA			BIT(6)  /* Accept BSSID match packet
-						 (Data) */
-#define	RCR_CBSSID			RCR_CBSSID_DATA	/* Accept BSSID match
-							   packet */
-#define	RCR_APWRMGT			BIT(5)  /* Accept power management
-						 packet */
-#define	RCR_ADD3			BIT(4)  /* Accept address 3 match
-						 packet */
-#define	RCR_AB				BIT(3)  /* Accept broadcast packet */
-#define	RCR_AM				BIT(2)  /* Accept multicast packet */
-#define	RCR_APM				BIT(1)  /* Accept physical match packet */
-#define	RCR_AAP				BIT(0)  /* Accept all unicast packet */
-#define	RCR_MXDMA_OFFSET		8
-#define	RCR_FIFO_OFFSET			13
-
-
-
-/*  */
-/* 8192c USB specific Regsiter Offset and Content definition, */
-/* 2009.08.18, added by vivi. for merge 92c and 92C into one driver */
-/*  */
-/* define APS_FSMCO			0x0004  same with 92Ce */
-#define RSV_CTRL			0x001C
-#define RD_CTRL				0x0524
-
-/*  */
-/*  */
-/*	0xFE00h ~ 0xFE55h	USB Configuration */
-/*  */
-/*  */
-#define REG_USB_INFO			0xFE17
-#define REG_USB_SPECIAL_OPTION		0xFE55
-#define REG_USB_DMA_AGG_TO		0xFE5B
-#define REG_USB_AGG_TO			0xFE5C
-#define REG_USB_AGG_TH			0xFE5D
-
-#define REG_USB_VID			0xFE60
-#define REG_USB_PID			0xFE62
-#define REG_USB_OPTIONAL		0xFE64
-#define REG_USB_CHIRP_K			0xFE65
-#define REG_USB_PHY			0xFE66
-#define REG_USB_MAC_ADDR		0xFE70
-
-#define REG_USB_HRPWM			0xFE58
-#define REG_USB_HCPWM			0xFE57
-
-#define	InvalidBBRFValue		0x12345678
-
-/*  */
-/*        8192C Regsiter Bit and Content definition */
-/*  */
-/*  */
-/*  */
-/*	0x0000h ~ 0x00FFh	System Configuration */
-/*  */
-/*  */
-
-/* 2 SPS0_CTRL */
-#define SW18_FPWM			BIT(3)
-
-
-/* 2 SYS_ISO_CTRL */
-#define ISO_MD2PP			BIT(0)
-#define ISO_UA2USB			BIT(1)
-#define ISO_UD2CORE			BIT(2)
-#define ISO_PA2PCIE			BIT(3)
-#define ISO_PD2CORE			BIT(4)
-#define ISO_IP2MAC			BIT(5)
-#define ISO_DIOP			BIT(6)
-#define ISO_DIOE			BIT(7)
-#define ISO_EB2CORE			BIT(8)
-#define ISO_DIOR			BIT(9)
-
-#define PWC_EV25V			BIT(14)
-#define PWC_EV12V			BIT(15)
-
-
-/* 2 SYS_FUNC_EN */
-#define FEN_BBRSTB			BIT(0)
-#define FEN_BB_GLB_RSTn			BIT(1)
-#define FEN_USBA			BIT(2)
-#define FEN_UPLL			BIT(3)
-#define FEN_USBD			BIT(4)
-#define FEN_DIO_PCIE			BIT(5)
-#define FEN_PCIEA			BIT(6)
-#define FEN_PPLL			BIT(7)
-#define FEN_PCIED			BIT(8)
-#define FEN_DIOE			BIT(9)
-#define FEN_CPUEN			BIT(10)
-#define FEN_DCORE			BIT(11)
-#define FEN_ELDR			BIT(12)
-#define FEN_DIO_RF			BIT(13)
-#define FEN_HWPDN			BIT(14)
-#define FEN_MREGEN			BIT(15)
-
-/* 2 APS_FSMCO */
-#define PFM_LDALL			BIT(0)
-#define PFM_ALDN			BIT(1)
-#define PFM_LDKP			BIT(2)
-#define PFM_WOWL			BIT(3)
-#define EnPDN				BIT(4)
-#define PDN_PL				BIT(5)
-#define APFM_ONMAC			BIT(8)
-#define APFM_OFF			BIT(9)
-#define APFM_RSM			BIT(10)
-#define AFSM_HSUS			BIT(11)
-#define AFSM_PCIE			BIT(12)
-#define APDM_MAC			BIT(13)
-#define APDM_HOST			BIT(14)
-#define APDM_HPDN			BIT(15)
-#define RDY_MACON			BIT(16)
-#define SUS_HOST			BIT(17)
-#define ROP_ALD				BIT(20)
-#define ROP_PWR				BIT(21)
-#define ROP_SPS				BIT(22)
-#define SOP_MRST			BIT(25)
-#define SOP_FUSE			BIT(26)
-#define SOP_ABG				BIT(27)
-#define SOP_AMB				BIT(28)
-#define SOP_RCK				BIT(29)
-#define SOP_A8M				BIT(30)
-#define XOP_BTCK			BIT(31)
-
-/* 2 SYS_CLKR */
-#define ANAD16V_EN			BIT(0)
-#define ANA8M				BIT(1)
-#define MACSLP				BIT(4)
-#define LOADER_CLK_EN			BIT(5)
-#define _80M_SSC_DIS			BIT(7)
-#define _80M_SSC_EN_HO			BIT(8)
-#define PHY_SSC_RSTB			BIT(9)
-#define SEC_CLK_EN			BIT(10)
-#define MAC_CLK_EN			BIT(11)
-#define SYS_CLK_EN			BIT(12)
-#define RING_CLK_EN			BIT(13)
-
-
-/* 2 9346CR */
-
-
-#define		EEDO			BIT(0)
-#define		EEDI			BIT(1)
-#define		EESK			BIT(2)
-#define		EECS			BIT(3)
-/* define	EERPROMSEL		BIT(4) */
-/* define	EEPROM_EN		BIT(5) */
-#define		BOOT_FROM_EEPROM	BIT(4)
-#define		EEPROM_EN		BIT(5)
-#define		EEM0			BIT(6)
-#define		EEM1			BIT(7)
-
-
-/* 2 AFE_MISC */
-#define AFE_BGEN			BIT(0)
-#define AFE_MBEN			BIT(1)
-#define MAC_ID_EN			BIT(7)
-
-
-/* 2 SPS0_CTRL */
-
-
-/* 2 SPS_OCP_CFG */
-
-
-/* 2 RSV_CTRL */
-#define WLOCK_ALL			BIT(0)
-#define WLOCK_00			BIT(1)
-#define WLOCK_04			BIT(2)
-#define WLOCK_08			BIT(3)
-#define WLOCK_40			BIT(4)
-#define R_DIS_PRST_0			BIT(5)
-#define R_DIS_PRST_1			BIT(6)
-#define LOCK_ALL_EN			BIT(7)
-
-/* 2 RF_CTRL */
-#define RF_EN				BIT(0)
-#define RF_RSTB				BIT(1)
-#define RF_SDMRSTB			BIT(2)
-
-
-
-/* 2 LDOA15_CTRL */
-#define LDA15_EN			BIT(0)
-#define LDA15_STBY			BIT(1)
-#define LDA15_OBUF			BIT(2)
-#define LDA15_REG_VOS			BIT(3)
-#define _LDA15_VOADJ(x)			(((x) & 0x7) << 4)
-
-
-
-/* 2 LDOV12D_CTRL */
-#define LDV12_EN			BIT(0)
-#define LDV12_SDBY			BIT(1)
-#define LPLDO_HSM			BIT(2)
-#define LPLDO_LSM_DIS			BIT(3)
-#define _LDV12_VADJ(x)			(((x) & 0xF) << 4)
-
-
-/* 2 AFE_XTAL_CTRL */
-#define XTAL_EN				BIT(0)
-#define XTAL_BSEL			BIT(1)
-#define _XTAL_BOSC(x)			(((x) & 0x3) << 2)
-#define _XTAL_CADJ(x)			(((x) & 0xF) << 4)
-#define XTAL_GATE_USB			BIT(8)
-#define _XTAL_USB_DRV(x)		(((x) & 0x3) << 9)
-#define XTAL_GATE_AFE			BIT(11)
-#define _XTAL_AFE_DRV(x)		(((x) & 0x3) << 12)
-#define XTAL_RF_GATE			BIT(14)
-#define _XTAL_RF_DRV(x)			(((x) & 0x3) << 15)
-#define XTAL_GATE_DIG			BIT(17)
-#define _XTAL_DIG_DRV(x)		(((x) & 0x3) << 18)
-#define XTAL_BT_GATE			BIT(20)
-#define _XTAL_BT_DRV(x)			(((x) & 0x3) << 21)
-#define _XTAL_GPIO(x)			(((x) & 0x7) << 23)
-
-
-#define CKDLY_AFE			BIT(26)
-#define CKDLY_USB			BIT(27)
-#define CKDLY_DIG			BIT(28)
-#define CKDLY_BT			BIT(29)
-
-
-/* 2 AFE_PLL_CTRL */
-#define APLL_EN				BIT(0)
-#define APLL_320_EN			BIT(1)
-#define APLL_FREF_SEL			BIT(2)
-#define APLL_EDGE_SEL			BIT(3)
-#define APLL_WDOGB			BIT(4)
-#define APLL_LPFEN			BIT(5)
-
-#define APLL_REF_CLK_13MHZ		0x1
-#define APLL_REF_CLK_19_2MHZ		0x2
-#define APLL_REF_CLK_20MHZ		0x3
-#define APLL_REF_CLK_25MHZ		0x4
-#define APLL_REF_CLK_26MHZ		0x5
-#define APLL_REF_CLK_38_4MHZ		0x6
-#define APLL_REF_CLK_40MHZ		0x7
-
-#define APLL_320EN			BIT(14)
-#define APLL_80EN			BIT(15)
-#define APLL_1MEN			BIT(24)
-
-
-/* 2 EFUSE_CTRL */
-#define ALD_EN				BIT(18)
-#define EF_PD				BIT(19)
-#define EF_FLAG				BIT(31)
-
-/* 2 EFUSE_TEST (For RTL8723 partially) */
-#define EF_TRPT				BIT(7)
-	/*  00: Wifi Efuse, 01: BT Efuse0, 10: BT Efuse1, 11: BT Efuse2 */
-#define EF_CELL_SEL			(BIT(8)|BIT(9))
-#define LDOE25_EN			BIT(31)
-#define EFUSE_SEL(x)			(((x) & 0x3) << 8)
-#define EFUSE_SEL_MASK			0x300
-#define EFUSE_WIFI_SEL_0		0x0
-#define EFUSE_BT_SEL_0			0x1
-#define EFUSE_BT_SEL_1			0x2
-#define EFUSE_BT_SEL_2			0x3
-
-#define EFUSE_ACCESS_ON			0x69	/*  For RTL8723 only. */
-#define EFUSE_ACCESS_OFF		0x00	/*  For RTL8723 only. */
-
-/* 2 PWR_DATA */
-
-/* 2 CAL_TIMER */
-
-/* 2 ACLK_MON */
-#define RSM_EN				BIT(0)
-#define Timer_EN			BIT(4)
-
-
-/* 2 GPIO_MUXCFG */
-#define TRSW0EN				BIT(2)
-#define TRSW1EN				BIT(3)
-#define EROM_EN				BIT(4)
-#define EnBT				BIT(5)
-#define EnUart				BIT(8)
-#define Uart_910			BIT(9)
-#define EnPMAC				BIT(10)
-#define SIC_SWRST			BIT(11)
-#define EnSIC				BIT(12)
-#define SIC_23				BIT(13)
-#define EnHDP				BIT(14)
-#define SIC_LBK				BIT(15)
-
-/* 2 GPIO_PIN_CTRL */
-
-/*  GPIO BIT */
-#define HAL_8192C_HW_GPIO_WPS_BIT	BIT(2)
-
-/* 2 GPIO_INTM */
-
-/* 2 LEDCFG */
-#define LED0PL				BIT(4)
-#define LED0DIS				BIT(7)
-#define LED1DIS				BIT(15)
-#define LED1PL				BIT(12)
-
-#define  SECCAM_CLR			BIT(30)
-
-
-/* 2 FSIMR */
-
-/* 2 FSISR */
-
-
-/* 2 8051FWDL */
-/* 2 MCUFWDL */
-#define MCUFWDL_EN			BIT(0)
-#define MCUFWDL_RDY			BIT(1)
-#define FWDL_ChkSum_rpt			BIT(2)
-#define MACINI_RDY			BIT(3)
-#define BBINI_RDY			BIT(4)
-#define RFINI_RDY			BIT(5)
-#define WINTINI_RDY			BIT(6)
-#define CPRST				BIT(23)
-
-/* 2REG_HPON_FSM */
-#define BOND92CE_1T2R_CFG		BIT(22)
-
-
-/* 2 REG_SYS_CFG */
-#define XCLK_VLD			BIT(0)
-#define ACLK_VLD			BIT(1)
-#define UCLK_VLD			BIT(2)
-#define PCLK_VLD			BIT(3)
-#define PCIRSTB				BIT(4)
-#define V15_VLD				BIT(5)
-#define TRP_B15V_EN			BIT(7)
-#define SIC_IDLE			BIT(8)
-#define BD_MAC2				BIT(9)
-#define BD_MAC1				BIT(10)
-#define IC_MACPHY_MODE			BIT(11)
-#define CHIP_VER			(BIT(12)|BIT(13)|BIT(14)|BIT(15))
-#define BT_FUNC				BIT(16)
-#define VENDOR_ID			BIT(19)
-#define PAD_HWPD_IDN			BIT(22)
-#define TRP_VAUX_EN			BIT(23)
-#define TRP_BT_EN			BIT(24)
-#define BD_PKG_SEL			BIT(25)
-#define BD_HCI_SEL			BIT(26)
-#define TYPE_ID				BIT(27)
-
-#define CHIP_VER_RTL_MASK		0xF000	/* Bit 12 ~ 15 */
-#define CHIP_VER_RTL_SHIFT		12
-
-/* 2REG_GPIO_OUTSTS (For RTL8723 only) */
-#define	EFS_HCI_SEL			(BIT(0)|BIT(1))
-#define	PAD_HCI_SEL			(BIT(2)|BIT(3))
-#define	HCI_SEL				(BIT(4)|BIT(5))
-#define	PKG_SEL_HCI			BIT(6)
-#define	FEN_GPS				BIT(7)
-#define	FEN_BT				BIT(8)
-#define	FEN_WL				BIT(9)
-#define	FEN_PCI				BIT(10)
-#define	FEN_USB				BIT(11)
-#define	BTRF_HWPDN_N			BIT(12)
-#define	WLRF_HWPDN_N			BIT(13)
-#define	PDN_BT_N			BIT(14)
-#define	PDN_GPS_N			BIT(15)
-#define	BT_CTL_HWPDN			BIT(16)
-#define	GPS_CTL_HWPDN			BIT(17)
-#define	PPHY_SUSB			BIT(20)
-#define	UPHY_SUSB			BIT(21)
-#define	PCI_SUSEN			BIT(22)
-#define	USB_SUSEN			BIT(23)
-#define	RF_RL_ID			(BIT(31)|BIT(30)|BIT(29)|BIT(28))
-
-/*  */
-/*  */
-/*	0x0100h ~ 0x01FFh	MACTOP General Configuration */
-/*  */
-/*  */
-
-
-/* 2 Function Enable Registers */
-/* 2 CR */
-
-#define REG_LBMODE			(REG_CR + 3)
-
-
-#define HCI_TXDMA_EN			BIT(0)
-#define HCI_RXDMA_EN			BIT(1)
-#define TXDMA_EN			BIT(2)
-#define RXDMA_EN			BIT(3)
-#define PROTOCOL_EN			BIT(4)
-#define SCHEDULE_EN			BIT(5)
-#define MACTXEN				BIT(6)
-#define MACRXEN				BIT(7)
-#define ENSWBCN				BIT(8)
-#define ENSEC				BIT(9)
-
-#define _LBMODE(x)			(((x) & 0xF) << 24)
-#define MASK_LBMODE			0xF000000
-#define LOOPBACK_NORMAL			0x0
-#define LOOPBACK_IMMEDIATELY		0xB
-#define LOOPBACK_MAC_DELAY		0x3
-#define LOOPBACK_PHY			0x1
-#define LOOPBACK_DMA			0x7
-
-
-/* 2 PBP - Page Size Register */
-#define GET_RX_PAGE_SIZE(value)		((value) & 0xF)
-#define GET_TX_PAGE_SIZE(value)		(((value) & 0xF0) >> 4)
-#define _PSRX_MASK			0xF
-#define _PSTX_MASK			0xF0
-#define _PSRX(x)			(x)
-#define _PSTX(x)			((x) << 4)
-
-#define PBP_64				0x0
-#define PBP_128				0x1
-#define PBP_256				0x2
-#define PBP_512				0x3
-#define PBP_1024			0x4
-
-
-/* 2 TX/RXDMA */
-#define RXDMA_ARBBW_EN			BIT(0)
-#define RXSHFT_EN			BIT(1)
-#define RXDMA_AGG_EN			BIT(2)
-#define QS_VO_QUEUE			BIT(8)
-#define QS_VI_QUEUE			BIT(9)
-#define QS_BE_QUEUE			BIT(10)
-#define QS_BK_QUEUE			BIT(11)
-#define QS_MANAGER_QUEUE		BIT(12)
-#define QS_HIGH_QUEUE			BIT(13)
-
-#define HQSEL_VOQ			BIT(0)
-#define HQSEL_VIQ			BIT(1)
-#define HQSEL_BEQ			BIT(2)
-#define HQSEL_BKQ			BIT(3)
-#define HQSEL_MGTQ			BIT(4)
-#define HQSEL_HIQ			BIT(5)
-
-/*  For normal driver, 0x10C */
-#define _TXDMA_HIQ_MAP(x)		(((x)&0x3) << 14)
-#define _TXDMA_MGQ_MAP(x)		(((x)&0x3) << 12)
-#define _TXDMA_BKQ_MAP(x)		(((x)&0x3) << 10)
-#define _TXDMA_BEQ_MAP(x)		(((x)&0x3) << 8)
-#define _TXDMA_VIQ_MAP(x)		(((x)&0x3) << 6)
-#define _TXDMA_VOQ_MAP(x)		(((x)&0x3) << 4)
-
-#define QUEUE_LOW			1
-#define QUEUE_NORMAL			2
-#define QUEUE_HIGH			3
-
-
-
-/* 2 TRXFF_BNDY */
-
-
-/* 2 LLT_INIT */
-#define _LLT_NO_ACTIVE			0x0
-#define _LLT_WRITE_ACCESS		0x1
-#define _LLT_READ_ACCESS		0x2
-
-#define _LLT_INIT_DATA(x)		((x) & 0xFF)
-#define _LLT_INIT_ADDR(x)		(((x) & 0xFF) << 8)
-#define _LLT_OP(x)			(((x) & 0x3) << 30)
-#define _LLT_OP_VALUE(x)		(((x) >> 30) & 0x3)
-
-
-/* 2 BB_ACCESS_CTRL */
-#define BB_WRITE_READ_MASK		(BIT(31) | BIT(30))
-#define BB_WRITE_EN			BIT(30)
-#define BB_READ_EN			BIT(31)
-/* define BB_ADDR_MASK			0xFFF */
-/* define _BB_ADDR(x)			((x) & BB_ADDR_MASK) */
-
-/*  */
-/*  */
-/*	0x0200h ~ 0x027Fh	TXDMA Configuration */
-/*  */
-/*  */
-/* 2 RQPN */
-#define _HPQ(x)				((x) & 0xFF)
-#define _LPQ(x)				(((x) & 0xFF) << 8)
-#define _PUBQ(x)			(((x) & 0xFF) << 16)
-	/*  NOTE: in RQPN_NPQ register */
-#define _NPQ(x)				((x) & 0xFF)
-
-
-#define HPQ_PUBLIC_DIS			BIT(24)
-#define LPQ_PUBLIC_DIS			BIT(25)
-#define LD_RQPN				BIT(31)
-
-
-/* 2 TDECTRL */
-#define BCN_VALID			BIT(16)
-#define BCN_HEAD(x)			(((x) & 0xFF) << 8)
-#define	BCN_HEAD_MASK			0xFF00
-
-/* 2 TDECTL */
-#define BLK_DESC_NUM_SHIFT		4
-#define BLK_DESC_NUM_MASK		0xF
-
-
-/* 2 TXDMA_OFFSET_CHK */
-#define DROP_DATA_EN			BIT(9)
-
-/*  */
-/*  */
-/*	0x0400h ~ 0x047Fh	Protocol Configuration */
-/*  */
-/*  */
-/* 2 FWHW_TXQ_CTRL */
-#define EN_AMPDU_RTY_NEW		BIT(7)
-
-/* 2 INIRTSMCS_SEL */
-#define _INIRTSMCS_SEL(x)		((x) & 0x3F)
-
-
-/* 2 SPEC SIFS */
-#define _SPEC_SIFS_CCK(x)		((x) & 0xFF)
-#define _SPEC_SIFS_OFDM(x)		(((x) & 0xFF) << 8)
-
-
-/* 2 RRSR */
-
-#define RATE_REG_BITMAP_ALL		0xFFFFF
-
-#define _RRSC_BITMAP(x)			((x) & 0xFFFFF)
-
-#define _RRSR_RSC(x)			(((x) & 0x3) << 21)
-#define RRSR_RSC_RESERVED		0x0
-#define RRSR_RSC_UPPER_SUBCHANNEL	0x1
-#define RRSR_RSC_LOWER_SUBCHANNEL	0x2
-#define RRSR_RSC_DUPLICATE_MODE		0x3
-
-
-/* 2 ARFR */
-#define USE_SHORT_G1			BIT(20)
-
-/* 2 AGGLEN_LMT_L */
-#define _AGGLMT_MCS0(x)			((x) & 0xF)
-#define _AGGLMT_MCS1(x)			(((x) & 0xF) << 4)
-#define _AGGLMT_MCS2(x)			(((x) & 0xF) << 8)
-#define _AGGLMT_MCS3(x)			(((x) & 0xF) << 12)
-#define _AGGLMT_MCS4(x)			(((x) & 0xF) << 16)
-#define _AGGLMT_MCS5(x)			(((x) & 0xF) << 20)
-#define _AGGLMT_MCS6(x)			(((x) & 0xF) << 24)
-#define _AGGLMT_MCS7(x)			(((x) & 0xF) << 28)
-
-
-/* 2 RL */
-#define	RETRY_LIMIT_SHORT_SHIFT		8
-#define	RETRY_LIMIT_LONG_SHIFT		0
-
-
-/* 2 DARFRC */
-#define _DARF_RC1(x)			((x) & 0x1F)
-#define _DARF_RC2(x)			(((x) & 0x1F) << 8)
-#define _DARF_RC3(x)			(((x) & 0x1F) << 16)
-#define _DARF_RC4(x)			(((x) & 0x1F) << 24)
-/*  NOTE: shift starting from address (DARFRC + 4) */
-#define _DARF_RC5(x)			((x) & 0x1F)
-#define _DARF_RC6(x)			(((x) & 0x1F) << 8)
-#define _DARF_RC7(x)			(((x) & 0x1F) << 16)
-#define _DARF_RC8(x)			(((x) & 0x1F) << 24)
-
-
-/* 2 RARFRC */
-#define _RARF_RC1(x)			((x) & 0x1F)
-#define _RARF_RC2(x)			(((x) & 0x1F) << 8)
-#define _RARF_RC3(x)			(((x) & 0x1F) << 16)
-#define _RARF_RC4(x)			(((x) & 0x1F) << 24)
-/*  NOTE: shift starting from address (RARFRC + 4) */
-#define _RARF_RC5(x)			((x) & 0x1F)
-#define _RARF_RC6(x)			(((x) & 0x1F) << 8)
-#define _RARF_RC7(x)			(((x) & 0x1F) << 16)
-#define _RARF_RC8(x)			(((x) & 0x1F) << 24)
-
-
-/*  */
-/*  */
-/*	0x0500h ~ 0x05FFh	EDCA Configuration */
-/*  */
-/*  */
-
-
-
-/* 2 EDCA setting */
-#define AC_PARAM_TXOP_LIMIT_OFFSET	16
-#define AC_PARAM_ECW_MAX_OFFSET		12
-#define AC_PARAM_ECW_MIN_OFFSET		8
-#define AC_PARAM_AIFS_OFFSET		0
-
-
-/* 2 EDCA_VO_PARAM */
-#define _AIFS(x)			(x)
-#define _ECW_MAX_MIN(x)			((x) << 8)
-#define _TXOP_LIMIT(x)			((x) << 16)
-
-
-#define _BCNIFS(x)			((x) & 0xFF)
-#define _BCNECW(x)			(((x) & 0xF))<< 8)
-
-
-#define _LRL(x)				((x) & 0x3F)
-#define _SRL(x)				(((x) & 0x3F) << 8)
-
-
-/* 2 SIFS_CCK */
-#define _SIFS_CCK_CTX(x)		((x) & 0xFF)
-#define _SIFS_CCK_TRX(x)		(((x) & 0xFF) << 8);
-
-
-/* 2 SIFS_OFDM */
-#define _SIFS_OFDM_CTX(x)		((x) & 0xFF)
-#define _SIFS_OFDM_TRX(x)		(((x) & 0xFF) << 8);
-
-
-/* 2 TBTT PROHIBIT */
-#define _TBTT_PROHIBIT_HOLD(x)		(((x) & 0xFF) << 8)
-
-
-/* 2 REG_RD_CTRL */
-#define DIS_EDCA_CNT_DWN		BIT(11)
-
-
-/* 2 BCN_CTRL */
-#define EN_MBSSID			BIT(1)
-#define EN_TXBCN_RPT			BIT(2)
-#define	EN_BCN_FUNCTION			BIT(3)
-#define DIS_TSF_UPDATE			BIT(3)
-
-/*  The same function but different bit field. */
-#define	DIS_TSF_UDT0_NORMAL_CHIP	BIT(4)
-#define	DIS_TSF_UDT0_TEST_CHIP		BIT(5)
-
-/* 2 ACMHWCTRL */
-#define	AcmHw_HwEn			BIT(0)
-#define	AcmHw_BeqEn			BIT(1)
-#define	AcmHw_ViqEn			BIT(2)
-#define	AcmHw_VoqEn			BIT(3)
-#define	AcmHw_BeqStatus			BIT(4)
-#define	AcmHw_ViqStatus			BIT(5)
-#define	AcmHw_VoqStatus			BIT(6)
-
-
-
-/*  */
-/*  */
-/*	0x0600h ~ 0x07FFh	WMAC Configuration */
-/*  */
-/*  */
-
-/* 2 APSD_CTRL */
-#define APSDOFF				BIT(6)
-#define APSDOFF_STATUS			BIT(7)
-
-
-/* 2 BWOPMODE */
-#define BW_20MHZ			BIT(2)
-
-
-#define RATE_BITMAP_ALL			0xFFFFF
-
-/*  Only use CCK 1M rate for ACK */
-#define RATE_RRSR_CCK_ONLY_1M		0xFFFF1
-
-/* 2 TCR */
-#define TSFRST				BIT(0)
-#define DIS_GCLK			BIT(1)
-#define PAD_SEL				BIT(2)
-#define PWR_ST				BIT(6)
-#define PWRBIT_OW_EN			BIT(7)
-#define ACRC				BIT(8)
-#define CFENDFORM			BIT(9)
-#define ICV				BIT(10)
-
-
-
-/* 2 RCR */
-#define AAP				BIT(0)
-#define APM				BIT(1)
-#define AM				BIT(2)
-#define AB				BIT(3)
-#define ADD3				BIT(4)
-#define APWRMGT				BIT(5)
-#define CBSSID				BIT(6)
-#define CBSSID_BCN			BIT(7)
-#define ACRC32				BIT(8)
-#define AICV				BIT(9)
-#define ADF				BIT(11)
-#define ACF				BIT(12)
-#define AMF				BIT(13)
-#define HTC_LOC_CTRL			BIT(14)
-#define UC_DATA_EN			BIT(16)
-#define BM_DATA_EN			BIT(17)
-#define MFBEN				BIT(22)
-#define LSIGEN				BIT(23)
-#define EnMBID				BIT(24)
-#define APP_BASSN			BIT(27)
-#define APP_PHYSTS			BIT(28)
-#define APP_ICV				BIT(29)
-#define APP_MIC				BIT(30)
-#define APP_FCS				BIT(31)
-
-/* 2 RX_PKT_LIMIT */
-
-/* 2 RX_DLK_TIME */
-
-/* 2 MBIDCAMCFG */
-
-
-
-/* 2 AMPDU_MIN_SPACE */
-#define _MIN_SPACE(x)			((x) & 0x7)
-#define _SHORT_GI_PADDING(x)		(((x) & 0x1F) << 3)
-
-
-/* 2 RXERR_RPT */
-#define RXERR_TYPE_OFDM_PPDU		0
-#define RXERR_TYPE_OFDMfalse_ALARM	1
-#define	RXERR_TYPE_OFDM_MPDU_OK		2
-#define RXERR_TYPE_OFDM_MPDU_FAIL	3
-#define RXERR_TYPE_CCK_PPDU		4
-#define RXERR_TYPE_CCKfalse_ALARM	5
-#define RXERR_TYPE_CCK_MPDU_OK		6
-#define RXERR_TYPE_CCK_MPDU_FAIL	7
-#define RXERR_TYPE_HT_PPDU		8
-#define RXERR_TYPE_HTfalse_ALARM	9
-#define RXERR_TYPE_HT_MPDU_TOTAL	10
-#define RXERR_TYPE_HT_MPDU_OK		11
-#define RXERR_TYPE_HT_MPDU_FAIL		12
-#define RXERR_TYPE_RX_FULL_DROP		15
-
-#define RXERR_COUNTER_MASK		0xFFFFF
-#define RXERR_RPT_RST			BIT(27)
-#define _RXERR_RPT_SEL(type)		((type) << 28)
-
-
-/* 2 SECCFG */
-#define	SCR_TxUseDK			BIT(0)	/* Force Tx Use Default Key */
-#define	SCR_RxUseDK			BIT(1)	/* Force Rx Use Default Key */
-#define	SCR_TxEncEnable			BIT(2)	/* Enable Tx Encryption */
-#define	SCR_RxDecEnable			BIT(3)	/* Enable Rx Decryption */
-#define	SCR_SKByA2			BIT(4)	/* Search kEY BY A2 */
-#define	SCR_NoSKMC			BIT(5)	/* No Key Search Multicast */
-
-
-
-/*  */
-/*  */
-/*	0xFE00h ~ 0xFE55h	USB Configuration */
-/*  */
-/*  */
-
-/* 2 USB Information (0xFE17) */
-#define USB_IS_HIGH_SPEED		0
-#define USB_IS_FULL_SPEED		1
-#define USB_SPEED_MASK			BIT(5)
-
-#define USB_NORMAL_SIE_EP_MASK		0xF
-#define USB_NORMAL_SIE_EP_SHIFT		4
-
-#define USB_TEST_EP_MASK		0x30
-#define USB_TEST_EP_SHIFT		4
-
-/* 2 Special Option */
-#define USB_AGG_EN			BIT(3)
-
-
-/* 2REG_C2HEVT_CLEAR */
-	/*  Set by driver and notify FW that the driver has read the
-	    C2H command message */
-#define C2H_EVT_HOST_CLOSE		0x00
-	/*  Set by FW indicating that FW had set the C2H command message
-	    and it's not yet read by driver. */
-#define C2H_EVT_FW_CLOSE		0xFF
-
-
-/* 2REG_MULTI_FUNC_CTRL(For RTL8723 Only) */
-	/*  Enable GPIO[9] as WiFi HW PDn source */
-#define	WL_HWPDN_EN			BIT(0)
-	/*  WiFi HW PDn polarity control */
-#define	WL_HWPDN_SL			BIT(1)
-	/*  WiFi function enable */
-#define	WL_FUNC_EN			BIT(2)
-	/*  Enable GPIO[9] as WiFi RF HW PDn source */
-#define	WL_HWROF_EN			BIT(3)
-	/*  Enable GPIO[11] as BT HW PDn source */
-#define	BT_HWPDN_EN			BIT(16)
-	/*  BT HW PDn polarity control */
-#define	BT_HWPDN_SL			BIT(17)
-	/*  BT function enable */
-#define	BT_FUNC_EN			BIT(18)
-	/*  Enable GPIO[11] as BT/GPS RF HW PDn source */
-#define	BT_HWROF_EN			BIT(19)
-	/*  Enable GPIO[10] as GPS HW PDn source */
-#define	GPS_HWPDN_EN			BIT(20)
-	/*  GPS HW PDn polarity control */
-#define	GPS_HWPDN_SL			BIT(21)
-	/*  GPS function enable */
-#define	GPS_FUNC_EN			BIT(22)
-
-/* 3 REG_LIFECTRL_CTRL */
-#define	HAL92C_EN_PKT_LIFE_TIME_BK	BIT(3)
-#define	HAL92C_EN_PKT_LIFE_TIME_BE	BIT(2)
-#define	HAL92C_EN_PKT_LIFE_TIME_VI	BIT(1)
-#define	HAL92C_EN_PKT_LIFE_TIME_VO	BIT(0)
-
-#define	HAL92C_MSDU_LIFE_TIME_UNIT	128	/*  in us, said by Tim. */
-
-/*  */
-/*  General definitions */
-/*  */
-
-#define LAST_ENTRY_OF_TX_PKT_BUFFER	255
-
-#define POLLING_LLT_THRESHOLD		20
-#define POLLING_READY_TIMEOUT_COUNT	1000
-
-/*  Min Spacing related settings. */
-#define	MAX_MSS_DENSITY_2T		0x13
-#define	MAX_MSS_DENSITY_1T		0x0A
-
-/*  */
-/*	8723A Regsiter offset definition */
-/*  */
-#define HAL_8723A_NAV_UPPER_UNIT	128		/*  micro-second */
-
-/*  */
-/*  */
-/*	0x0000h ~ 0x00FFh	System Configuration */
-/*  */
-/*  */
-#define REG_SYSON_REG_LOCK		0x001C
-
-
-/*  */
-/*  */
-/*	0x0100h ~ 0x01FFh	MACTOP General Configuration */
-/*  */
-/*  */
-#define REG_FTIMR			0x0138
-
-
-/*  */
-/*  */
-/*	0x0200h ~ 0x027Fh	TXDMA Configuration */
-/*  */
-/*  */
-
-
-/*  */
-/*  */
-/*	0x0280h ~ 0x02FFh	RXDMA Configuration */
-/*  */
-/*  */
-
-
-/*  */
-/*  */
-/*	0x0300h ~ 0x03FFh	PCIe */
-/*  */
-/*  */
-
-
-/*  */
-/*  */
-/*	0x0400h ~ 0x047Fh	Protocol Configuration */
-/*  */
-/*  */
-#define REG_EARLY_MODE_CONTROL		0x4D0
-
-
-/*  */
-/*  */
-/*	0x0500h ~ 0x05FFh	EDCA Configuration */
-/*  */
-/*  */
-
-/* 2 BCN_CTRL */
-#define DIS_ATIM			BIT(0)
-#define DIS_BCNQ_SUB			BIT(1)
-#define DIS_TSF_UDT			BIT(4)
-
-
-/*  */
-/*  */
-/*	0x0600h ~ 0x07FFh	WMAC Configuration */
-/*  */
-/*  */
-/*  */
-/*  Note: */
-/*	The NAV upper value is very important to WiFi 11n 5.2.3 NAV test.
- *	The default value is always too small, but the WiFi TestPlan test
- *	by 25,000 microseconds of NAV through sending CTS in the air. We
- *	must update this value greater than 25,000 microseconds to pass the
- *	item.
-*	The offset of NAV_UPPER in 8192C Spec is incorrect, and the offset
-*	should be 0x0652. Commented by SD1 Scott. */
-/*  By Bruce, 2011-07-18. */
-/*  */
-#define	REG_NAV_UPPER			0x0652	/*  unit of 128 */
-
-
-/*  */
-/*	8723 Regsiter Bit and Content definition */
-/*  */
-
-/*  */
-/*  */
-/*	0x0000h ~ 0x00FFh	System Configuration */
-/*  */
-/*  */
-
-/* 2 SPS0_CTRL */
-
-/* 2 SYS_ISO_CTRL */
-
-/* 2 SYS_FUNC_EN */
-
-/* 2 APS_FSMCO */
-#define EN_WLON				BIT(16)
-
-/* 2 SYS_CLKR */
-
-/* 2 9346CR */
-
-/* 2 AFE_MISC */
-
-/* 2 SPS0_CTRL */
-
-/* 2 SPS_OCP_CFG */
-
-/* 2 SYSON_REG_LOCK */
-#define WLOCK_ALL			BIT(0)
-#define WLOCK_00			BIT(1)
-#define WLOCK_04			BIT(2)
-#define WLOCK_08			BIT(3)
-#define WLOCK_40			BIT(4)
-#define WLOCK_1C_B6			BIT(5)
-#define R_DIS_PRST_1			BIT(6)
-#define LOCK_ALL_EN			BIT(7)
-
-/* 2 RF_CTRL */
-
-/* 2 LDOA15_CTRL */
-
-/* 2 LDOV12D_CTRL */
-
-/* 2 AFE_XTAL_CTRL */
-
-/* 2 AFE_PLL_CTRL */
-
-/* 2 EFUSE_CTRL */
-
-/* 2 EFUSE_TEST (For RTL8723 partially) */
-
-/* 2 PWR_DATA */
-
-/* 2 CAL_TIMER */
-
-/* 2 ACLK_MON */
-
-/* 2 GPIO_MUXCFG */
-
-/* 2 GPIO_PIN_CTRL */
-
-/* 2 GPIO_INTM */
-
-/* 2 LEDCFG */
-
-/* 2 FSIMR */
-
-/* 2 FSISR */
-
-/* 2 HSIMR */
-/*  8723 Host System Interrupt Mask Register (offset 0x58, 32 byte) */
-#define HSIMR_GPIO12_0_INT_EN		BIT(0)
-#define HSIMR_SPS_OCP_INT_EN		BIT(5)
-#define HSIMR_RON_INT_EN		BIT(6)
-#define HSIMR_PDNINT_EN			BIT(7)
-#define HSIMR_GPIO9_INT_EN		BIT(25)
-
-/* 2 HSISR */
-/*  8723 Host System Interrupt Status Register (offset 0x5C, 32 byte) */
-#define HSISR_GPIO12_0_INT		BIT(0)
-#define HSISR_SPS_OCP_INT		BIT(5)
-#define HSISR_RON_INT			BIT(6)
-#define HSISR_PDNINT			BIT(7)
-#define	HSISR_GPIO9_INT			BIT(25)
-
-/*  interrupt mask which needs to clear */
-#define MASK_HSISR_CLEAR		(HSISR_GPIO12_0_INT | \
-					 HSISR_SPS_OCP_INT | \
-					 HSISR_RON_INT | \
-					 HSISR_PDNINT |	\
-					 HSISR_GPIO9_INT)
-
-/* 2 MCUFWDL */
-#define RAM_DL_SEL			BIT(7)	/*  1:RAM, 0:ROM */
-
-/* 2 HPON_FSM */
-
-/* 2 SYS_CFG */
-#define RTL_ID				BIT(23)	/*  TestChip ID,
-						    1:Test(RLE); 0:MP(RL) */
-#define SPS_SEL				BIT(24) /*  1:LDO regulator mode;
-						    0:Switching regulator mode*/
-
-
-/*  */
-/*  */
-/*	0x0100h ~ 0x01FFh	MACTOP General Configuration */
-/*  */
-/*  */
-
-/* 2 Function Enable Registers */
-
-/* 2 CR */
-#define CALTMR_EN			BIT(10)
-
-/* 2 PBP - Page Size Register */
-
-/* 2 TX/RXDMA */
-
-/* 2 TRXFF_BNDY */
-
-/* 2 LLT_INIT */
-
-/* 2 BB_ACCESS_CTRL */
-
-
-/*  */
-/*  */
-/*	0x0200h ~ 0x027Fh	TXDMA Configuration */
-/*  */
-/*  */
-
-/* 2 RQPN */
-
-/* 2 TDECTRL */
-
-/* 2 TDECTL */
-
-/* 2 TXDMA_OFFSET_CHK */
-
-
-/*  */
-/*  */
-/*	0x0400h ~ 0x047Fh	Protocol Configuration */
-/*  */
-/*  */
-
-/* 2 FWHW_TXQ_CTRL */
-
-/* 2 INIRTSMCS_SEL */
-
-/* 2 SPEC SIFS */
-
-/* 2 RRSR */
-
-/* 2 ARFR */
-
-/* 2 AGGLEN_LMT_L */
-
-/* 2 RL */
-
-/* 2 DARFRC */
-
-/* 2 RARFRC */
-
-
-/*  */
-/*  */
-/*	0x0500h ~ 0x05FFh	EDCA Configuration */
-/*  */
-/*  */
-
-/* 2 EDCA setting */
-
-/* 2 EDCA_VO_PARAM */
-
-/* 2 SIFS_CCK */
-
-/* 2 SIFS_OFDM */
-
-/* 2 TBTT PROHIBIT */
-
-/* 2 REG_RD_CTRL */
-
-/* 2 BCN_CTRL */
-
-/* 2 ACMHWCTRL */
-
-
-/*  */
-/*  */
-/*	0x0600h ~ 0x07FFh	WMAC Configuration */
-/*  */
-/*  */
-
-/* 2 APSD_CTRL */
-
-/* 2 BWOPMODE */
-
-/* 2 TCR */
-
-/* 2 RCR */
-
-/* 2 RX_PKT_LIMIT */
-
-/* 2 RX_DLK_TIME */
-
-/* 2 MBIDCAMCFG */
-
-/* 2 AMPDU_MIN_SPACE */
-
-/* 2 RXERR_RPT */
-
-/* 2 SECCFG */
-
-
-/*  */
-/*  */
-/*	0xFE00h ~ 0xFE55h	RTL8723 SDIO Configuration */
-/*  */
-/*  */
-
-/*  I/O bus domain address mapping */
-#define WLAN_IOREG_BASE			0x10260000
-#define FIRMWARE_FIFO_BASE		0x10270000
-#define TX_HIQ_BASE			0x10310000
-#define TX_MIQ_BASE			0x10320000
-#define TX_LOQ_BASE			0x10330000
-#define RX_RX0FF_BASE			0x10340000
-
-/*  SDIO host local register space mapping. */
-#define WLAN_IOREG_MSK			0x7FFF
-#define WLAN_FIFO_MSK			0x1FFF	/*  Aggregation Length[12:0] */
-#define WLAN_RX0FF_MSK			0x0003
-
-#define WLAN_RX0FF_DEVICE_ID		7	/*  0b[16], 111b[15:13] */
-#define WLAN_IOREG_DEVICE_ID		8	/*  1b[16] */
-
-/*  8723 EFUSE */
-#define HWSET_MAX_SIZE			256
-
-
-/* USB interrupt */
-#define	UHIMR_TIMEOUT2			BIT(31)
-#define	UHIMR_TIMEOUT1			BIT(30)
-#define	UHIMR_PSTIMEOUT			BIT(29)
-#define	UHIMR_GTINT4			BIT(28)
-#define	UHIMR_GTINT3			BIT(27)
-#define	UHIMR_TXBCNERR			BIT(26)
-#define	UHIMR_TXBCNOK			BIT(25)
-#define	UHIMR_TSF_BIT32_TOGGLE		BIT(24)
-#define	UHIMR_BCNDMAINT3		BIT(23)
-#define	UHIMR_BCNDMAINT2		BIT(22)
-#define	UHIMR_BCNDMAINT1		BIT(21)
-#define	UHIMR_BCNDMAINT0		BIT(20)
-#define	UHIMR_BCNDOK3			BIT(19)
-#define	UHIMR_BCNDOK2			BIT(18)
-#define	UHIMR_BCNDOK1			BIT(17)
-#define	UHIMR_BCNDOK0			BIT(16)
-#define	UHIMR_HSISR_IND			BIT(15)
-#define	UHIMR_BCNDMAINT_E		BIT(14)
-/* RSVD	BIT(13) */
-#define	UHIMR_CTW_END			BIT(12)
-/* RSVD	BIT(11) */
-#define	UHIMR_C2HCMD			BIT(10)
-#define	UHIMR_CPWM2			BIT(9)
-#define	UHIMR_CPWM			BIT(8)
-#define	UHIMR_HIGHDOK			BIT(7)	/*  High Queue DMA OK
-						    Interrupt */
-#define	UHIMR_MGNTDOK			BIT(6)	/*  Management Queue DMA OK
-						    Interrupt */
-#define	UHIMR_BKDOK			BIT(5)	/*  AC_BK DMA OK Interrupt */
-#define	UHIMR_BEDOK			BIT(4)	/*  AC_BE DMA OK Interrupt */
-#define	UHIMR_VIDOK			BIT(3)	/*  AC_VI DMA OK Interrupt */
-#define	UHIMR_VODOK			BIT(2)	/*  AC_VO DMA Interrupt */
-#define	UHIMR_RDU			BIT(1)	/*  Receive Descriptor
-						    Unavailable */
-#define	UHIMR_ROK			BIT(0)	/*  Receive DMA OK Interrupt */
-
-/*  USB Host Interrupt Status Extension bit */
-#define	UHIMR_BCNDMAINT7		BIT(23)
-#define	UHIMR_BCNDMAINT6		BIT(22)
-#define	UHIMR_BCNDMAINT5		BIT(21)
-#define	UHIMR_BCNDMAINT4		BIT(20)
-#define	UHIMR_BCNDOK7			BIT(19)
-#define	UHIMR_BCNDOK6			BIT(18)
-#define	UHIMR_BCNDOK5			BIT(17)
-#define	UHIMR_BCNDOK4			BIT(16)
-/*  bit14-15: RSVD */
-#define	UHIMR_ATIMEND_E			BIT(13)
-#define	UHIMR_ATIMEND			BIT(12)
-#define	UHIMR_TXERR			BIT(11)
-#define	UHIMR_RXERR			BIT(10)
-#define	UHIMR_TXFOVW			BIT(9)
-#define	UHIMR_RXFOVW			BIT(8)
-/*  bit2-7: RSVD */
-#define	UHIMR_OCPINT			BIT(1)
-/*  bit0: RSVD */
-
-#define	REG_USB_HIMR			0xFE38
-#define	REG_USB_HIMRE			0xFE3C
-#define	REG_USB_HISR			0xFE78
-#define	REG_USB_HISRE			0xFE7C
-
-#define	USB_INTR_CPWM_OFFSET		16
-#define	USB_INTR_CONTENT_HISR_OFFSET	48
-#define	USB_INTR_CONTENT_HISRE_OFFSET	52
-#define	USB_INTR_CONTENT_LENGTH		56
-#define	USB_C2H_CMDID_OFFSET		0
-#define	USB_C2H_SEQ_OFFSET		1
-#define	USB_C2H_EVENT_OFFSET		2
-/*  */
-/*	General definitions */
-/*  */
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_sreset.h b/drivers/staging/rtl8723au/include/rtl8723a_sreset.h
deleted file mode 100644
index 6197910a4a53..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_sreset.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef _RTL8723A_SRESET_H_
-#define _RTL8723A_SRESET_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <rtw_sreset.h>
-
-void rtl8723a_sreset_xmit_status_check(struct rtw_adapter *padapter);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtl8723a_xmit.h b/drivers/staging/rtl8723au/include/rtl8723a_xmit.h
deleted file mode 100644
index 7db29f40ab70..000000000000
--- a/drivers/staging/rtl8723au/include/rtl8723a_xmit.h
+++ /dev/null
@@ -1,225 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTL8723A_XMIT_H__
-#define __RTL8723A_XMIT_H__
-
-/*  */
-/*  Queue Select Value in TxDesc */
-/*  */
-#define QSLT_BK							0x2/* 0x01 */
-#define QSLT_BE							0x0
-#define QSLT_VI							0x5/* 0x4 */
-#define QSLT_VO							0x7/* 0x6 */
-#define QSLT_BEACON						0x10
-#define QSLT_HIGH						0x11
-#define QSLT_MGNT						0x12
-#define QSLT_CMD						0x13
-
-/*  */
-/* defined for TX DESC Operation */
-/*  */
-
-#define MAX_TID (15)
-
-/* OFFSET 0 */
-#define OFFSET_SZ	0
-#define OFFSET_SHT	16
-#define BMC		BIT(24)
-#define LSG		BIT(26)
-#define FSG		BIT(27)
-#define OWN		BIT(31)
-
-
-/* OFFSET 4 */
-#define PKT_OFFSET_SZ	0
-#define BK		BIT(6)
-#define QSEL_SHT	8
-#define Rate_ID_SHT	16
-#define NAVUSEHDR	BIT(20)
-#define PKT_OFFSET_SHT	26
-#define HWPC		BIT(31)
-
-/* OFFSET 8 */
-#define AGG_EN		BIT(29)
-
-/* OFFSET 12 */
-#define SEQ_SHT		16
-
-/* OFFSET 16 */
-#define QoS		BIT(6)
-#define HW_SEQ_EN	BIT(7)
-#define USERATE		BIT(8)
-#define DISDATAFB	BIT(10)
-#define DATA_SHORT	BIT(24)
-#define DATA_BW		BIT(25)
-
-/* OFFSET 20 */
-#define SGI		BIT(6)
-
-struct txdesc_8723a {
-	u32 pktlen:16;
-	u32 offset:8;
-	u32 bmc:1;
-	u32 htc:1;
-	u32 ls:1;
-	u32 fs:1;
-	u32 linip:1;
-	u32 noacm:1;
-	u32 gf:1;
-	u32 own:1;
-
-	u32 macid:5;
-	u32 agg_en:1;
-	u32 bk:1;
-	u32 rd_en:1;
-	u32 qsel:5;
-	u32 rd_nav_ext:1;
-	u32 lsig_txop_en:1;
-	u32 pifs:1;
-	u32 rate_id:4;
-	u32 navusehdr:1;
-	u32 en_desc_id:1;
-	u32 sectype:2;
-	u32 rsvd0424:2;
-	u32 pkt_offset:5;	/*  unit: 8 bytes */
-	u32 rsvd0431:1;
-
-	u32 rts_rc:6;
-	u32 data_rc:6;
-	u32 rsvd0812:2;
-	u32 bar_rty_th:2;
-	u32 rsvd0816:1;
-	u32 morefrag:1;
-	u32 raw:1;
-	u32 ccx:1;
-	u32 ampdu_density:3;
-	u32 bt_null:1;
-	u32 ant_sel_a:1;
-	u32 ant_sel_b:1;
-	u32 tx_ant_cck:2;
-	u32 tx_antl:2;
-	u32 tx_ant_ht:2;
-
-	u32 nextheadpage:8;
-	u32 tailpage:8;
-	u32 seq:12;
-	u32 cpu_handle:1;
-	u32 tag1:1;
-	u32 trigger_int:1;
-	u32 hwseq_en:1;
-
-	u32 rtsrate:5;
-	u32 ap_dcfe:1;
-	u32 hwseq_sel:2;
-	u32 userate:1;
-	u32 disrtsfb:1;
-	u32 disdatafb:1;
-	u32 cts2self:1;
-	u32 rtsen:1;
-	u32 hw_rts_en:1;
-	u32 port_id:1;
-	u32 rsvd1615:3;
-	u32 wait_dcts:1;
-	u32 cts2ap_en:1;
-	u32 data_sc:2;
-	u32 data_stbc:2;
-	u32 data_short:1;
-	u32 data_bw:1;
-	u32 rts_short:1;
-	u32 rts_bw:1;
-	u32 rts_sc:2;
-	u32 vcs_stbc:2;
-
-	u32 datarate:6;
-	u32 sgi:1;
-	u32 try_rate:1;
-	u32 data_ratefb_lmt:5;
-	u32 rts_ratefb_lmt:4;
-	u32 rty_lmt_en:1;
-	u32 data_rt_lmt:6;
-	u32 usb_txagg_num:8;
-
-	u32 txagg_a:5;
-	u32 txagg_b:5;
-	u32 use_max_len:1;
-	u32 max_agg_num:5;
-	u32 mcsg1_max_len:4;
-	u32 mcsg2_max_len:4;
-	u32 mcsg3_max_len:4;
-	u32 mcs7_sgi_max_len:4;
-
-	u32 checksum:16;	/*  TxBuffSize(PCIe)/CheckSum(USB) */
-	u32 mcsg4_max_len:4;
-	u32 mcsg5_max_len:4;
-	u32 mcsg6_max_len:4;
-	u32 mcs15_sgi_max_len:4;
-};
-
-#define txdesc_set_ccx_sw_8723a(txdesc, value) \
-	do { \
-		((struct txdesc_8723a *)(txdesc))->mcsg4_max_len = (((value)>>8) & 0x0f); \
-		((struct txdesc_8723a *)(txdesc))->mcs15_sgi_max_len= (((value)>>4) & 0x0f); \
-		((struct txdesc_8723a *)(txdesc))->mcsg6_max_len = ((value) & 0x0f); \
-	} while (0)
-
-struct txrpt_ccx_8723a {
-	/* offset 0 */
-	u8 tag1:1;
-	u8 rsvd:4;
-	u8 int_bt:1;
-	u8 int_tri:1;
-	u8 int_ccx:1;
-
-	/* offset 1 */
-	u8 mac_id:5;
-	u8 pkt_drop:1;
-	u8 pkt_ok:1;
-	u8 bmc:1;
-
-	/* offset 2 */
-	u8 retry_cnt:6;
-	u8 lifetime_over:1;
-	u8 retry_over:1;
-
-	/* offset 3 */
-	u8 ccx_qtime0;
-	u8 ccx_qtime1;
-
-	/* offset 5 */
-	u8 final_data_rate;
-
-	/* offset 6 */
-	u8 sw1:4;
-	u8 qsel:4;
-
-	/* offset 7 */
-	u8 sw0;
-};
-
-#define txrpt_ccx_sw_8723a(txrpt_ccx) ((txrpt_ccx)->sw0 + ((txrpt_ccx)->sw1<<8))
-#define txrpt_ccx_qtime_8723a(txrpt_ccx) ((txrpt_ccx)->ccx_qtime0+((txrpt_ccx)->ccx_qtime1<<8))
-
-void handle_txrpt_ccx_8723a(struct rtw_adapter *adapter, void *buf);
-void rtl8723a_fill_fake_txdesc(struct rtw_adapter *padapter, u8 *pDesc, u32 BufferLen, u8 IsPsPoll, u8 IsBTQosNull);
-
-int rtl8723au_hal_xmitframe_enqueue(struct rtw_adapter *padapter, struct xmit_frame *pxmitframe);
-s32 rtl8723au_xmit_buf_handler(struct rtw_adapter *padapter);
-#define hal_xmit_handler rtl8723au_xmit_buf_handler
-bool rtl8723au_hal_xmit(struct rtw_adapter *padapter, struct xmit_frame *pxmitframe);
-int rtl8723au_mgnt_xmit(struct rtw_adapter *padapter, struct xmit_frame *pmgntframe);
-bool rtl8723au_xmitframe_complete(struct rtw_adapter *padapter, struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
-
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtw_ap.h b/drivers/staging/rtl8723au/include/rtw_ap.h
deleted file mode 100644
index 55a708f9fc5b..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_ap.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTW_AP_H_
-#define __RTW_AP_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-
-#ifdef CONFIG_8723AU_AP_MODE
-
-/* external function */
-
-void init_mlme_ap_info23a(struct rtw_adapter *padapter);
-void free_mlme_ap_info23a(struct rtw_adapter *padapter);
-/* void update_BCNTIM(struct rtw_adapter *padapter); */
-void rtw_add_bcn_ie(struct rtw_adapter *padapter, struct wlan_bssid_ex *pnetwork, u8 index, u8 *data, u8 len);
-void rtw_remove_bcn_ie(struct rtw_adapter *padapter, struct wlan_bssid_ex *pnetwork, u8 index);
-void update_beacon23a(struct rtw_adapter *padapter, u8 ie_id, u8 *oui, u8 tx);
-void add_RATid23a(struct rtw_adapter *padapter, struct sta_info *psta, u8 rssi_level);
-void expire_timeout_chk23a(struct rtw_adapter *padapter);
-void update_sta_info23a_apmode23a(struct rtw_adapter *padapter, struct sta_info *psta);
-int rtw_check_beacon_data23a(struct rtw_adapter *padapter,
-			     struct ieee80211_mgmt *mgmt, unsigned int len);
-void rtw_ap_restore_network(struct rtw_adapter *padapter);
-void rtw_set_macaddr_acl23a(struct rtw_adapter *padapter, int mode);
-
-void associated_clients_update23a(struct rtw_adapter *padapter, u8 updated);
-void bss_cap_update_on_sta_join23a(struct rtw_adapter *padapter, struct sta_info *psta);
-u8 bss_cap_update_on_sta_leave23a(struct rtw_adapter *padapter, struct sta_info *psta);
-void sta_info_update23a(struct rtw_adapter *padapter, struct sta_info *psta);
-void ap_sta_info_defer_update23a(struct rtw_adapter *padapter, struct sta_info *psta);
-u8 ap_free_sta23a(struct rtw_adapter *padapter, struct sta_info *psta, bool active, u16 reason);
-int rtw_sta_flush23a(struct rtw_adapter *padapter);
-void start_ap_mode23a(struct rtw_adapter *padapter);
-void stop_ap_mode23a(struct rtw_adapter *padapter);
-#endif /* end of CONFIG_8723AU_AP_MODE */
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtw_cmd.h b/drivers/staging/rtl8723au/include/rtw_cmd.h
deleted file mode 100644
index d1fa95d28904..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_cmd.h
+++ /dev/null
@@ -1,815 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTW_CMD_H_
-#define __RTW_CMD_H_
-
-#include <wlan_bssdef.h>
-#include <rtw_rf.h>
-
-#define C2H_MEM_SZ (16*1024)
-
-#include <osdep_service.h>
-#include <ieee80211.h> /*  <ieee80211/ieee80211.h> */
-
-
-#define MAX_CMDSZ	1024
-#define MAX_RSPSZ	512
-#define MAX_EVTSZ	1024
-
-#define CMDBUFF_ALIGN_SZ 512
-
-struct cmd_obj {
-	struct work_struct work;
-	struct rtw_adapter *padapter;
-	u16	cmdcode;
-	int	res;
-	u32	cmdsz;
-	u8	*parmbuf;
-	u8	*rsp;
-	u32	rspsz;
-};
-
-struct cmd_priv {
-	struct workqueue_struct *wq;
-	u32	cmd_issued_cnt;
-	u32	cmd_done_cnt;
-	u32	rsp_cnt;
-	struct rtw_adapter *padapter;
-};
-
-#define C2H_QUEUE_MAX_LEN 10
-
-struct	evt_priv {
-	struct workqueue_struct *wq;
-	struct work_struct irq_wk;
-};
-
-#define init_h2fwcmd_w_parm_no_rsp(pcmd, pparm, code) \
-do {\
-	pcmd->cmdcode = code;\
-	pcmd->parmbuf = (u8 *)(pparm);\
-	pcmd->cmdsz = sizeof (*pparm);\
-	pcmd->rsp = NULL;\
-	pcmd->rspsz = 0;\
-} while(0)
-
-struct c2h_evt_hdr {
-	u8 id:4;
-	u8 plen:4;
-	u8 seq;
-	u8 payload[0];
-};
-
-/*
- * Do not reorder - this allows for struct evt_work to be passed on to
- * rtw_c2h_wk_cmd23a() as a 'struct c2h_evt_hdr *' without making an
- * additional copy.
- */
-struct evt_work {
-	union {
-		struct c2h_evt_hdr c2h_evt;
-		u8 buf[16];
-	} u;
-	struct work_struct work;
-	struct rtw_adapter *adapter;
-};
-
-#define c2h_evt_exist(c2h_evt) ((c2h_evt)->id || (c2h_evt)->plen)
-
-void rtw_evt_work(struct work_struct *work);
-
-int rtw_enqueue_cmd23a(struct cmd_priv *pcmdpriv, struct cmd_obj *obj);
-void rtw_free_cmd_obj23a(struct cmd_obj *pcmd);
-
-int rtw_cmd_thread23a(void *context);
-
-int rtw_init_cmd_priv23a(struct cmd_priv *pcmdpriv);
-
-u32 rtw_init_evt_priv23a (struct evt_priv *pevtpriv);
-void rtw_free_evt_priv23a (struct evt_priv *pevtpriv);
-void rtw_evt_notify_isr(struct evt_priv *pevtpriv);
-
-enum rtw_drvextra_cmd_id
-{
-	NONE_WK_CID,
-	DYNAMIC_CHK_WK_CID,
-	DM_CTRL_WK_CID,
-	PBC_POLLING_WK_CID,
-	POWER_SAVING_CTRL_WK_CID,/* IPS,AUTOSuspend */
-	LPS_CTRL_WK_CID,
-	ANT_SELECT_WK_CID,
-	P2P_PS_WK_CID,
-	P2P_PROTO_WK_CID,
-	CHECK_HIQ_WK_CID,/* for softap mode, check hi queue if empty */
-	C2H_WK_CID,
-	RTP_TIMER_CFG_WK_CID,
-	MAX_WK_CID
-};
-
-enum LPS_CTRL_TYPE
-{
-	LPS_CTRL_SCAN=0,
-	LPS_CTRL_JOINBSS=1,
-	LPS_CTRL_CONNECT=2,
-	LPS_CTRL_DISCONNECT=3,
-	LPS_CTRL_SPECIAL_PACKET=4,
-	LPS_CTRL_LEAVE=5,
-};
-
-enum RFINTFS {
-	SWSI,
-	HWSI,
-	HWPI,
-};
-
-/*
-Caller Mode: Infra, Ad-HoC(C)
-
-Notes: To enter USB suspend mode
-
-Command Mode
-
-*/
-struct usb_suspend_parm {
-	u32 action;/*  1: sleep, 0:resume */
-};
-
-/*
-Caller Mode: Infra, Ad-HoC
-
-Notes: To join a known BSS.
-
-Command-Event Mode
-
-*/
-
-/*
-Caller Mode: Infra, Ad-HoC(C)
-
-Notes: To disconnect the current associated BSS
-
-Command Mode
-
-*/
-struct disconnect_parm {
-	u32 deauth_timeout_ms;
-};
-
-struct	setopmode_parm {
-	enum nl80211_iftype mode;
-};
-
-/*
-Caller Mode: AP, Ad-HoC, Infra
-
-Notes: To ask RTL8711 performing site-survey
-
-Command-Event Mode
-
-*/
-
-#define RTW_SSID_SCAN_AMOUNT 9 /*  for WEXT_CSCAN_AMOUNT 9 */
-#define RTW_CHANNEL_SCAN_AMOUNT (14+37)
-struct sitesurvey_parm {
-	int scan_mode;	/* active: 1, passive: 0 */
-	u8 ssid_num;
-	u8 ch_num;
-	struct cfg80211_ssid ssid[RTW_SSID_SCAN_AMOUNT];
-	struct rtw_ieee80211_channel ch[RTW_CHANNEL_SCAN_AMOUNT];
-};
-
-/*
-Caller Mode: Any
-
-Notes: To set the auth type of RTL8711. open/shared/802.1x
-
-Command Mode
-
-*/
-struct setauth_parm {
-	u8 mode;  /* 0: legacy open, 1: legacy shared 2: 802.1x */
-	u8 _1x;   /* 0: PSK, 1: TLS */
-	u8 rsvd[2];
-};
-
-/*
-Caller Mode: Infra
-
-a. algorithm: wep40, wep104, tkip & aes
-b. keytype: grp key/unicast key
-c. key contents
-
-when shared key ==> keyid is the camid
-when 802.1x ==> keyid [0:1] ==> grp key
-when 802.1x ==> keyid > 2 ==> unicast key
-
-*/
-struct setkey_parm {
-	u32	algorithm;	/*  encryption algorithm, could be none, wep40, TKIP, CCMP, wep104 */
-	u8	keyid;
-	u8	grpkey;		/*  1: this is the grpkey for 802.1x. 0: this is the unicast key for 802.1x */
-	u8	set_tx;		/*  1: main tx key for wep. 0: other key. */
-	u8	key[16];	/*  this could be 40 or 104 */
-};
-
-/*
-When in AP or Ad-Hoc mode, this is used to
-allocate an sw/hw entry for a newly associated sta.
-
-Command
-
-when shared key ==> algorithm/keyid
-
-*/
-struct set_stakey_parm {
-	u8	addr[ETH_ALEN];
-	u8	id;/*  currently for erasing cam entry if algorithm == _NO_PRIVACY_ */
-	u32	algorithm;
-	u8	key[16];
-};
-
-struct set_stakey_rsp {
-	u8	addr[ETH_ALEN];
-	u8	keyid;
-	u8	rsvd;
-};
-
-/*
-Caller Ad-Hoc/AP
-
-Command -Rsp(AID == CAMID) mode
-
-This is to force fw to add an sta_data entry per driver's request.
-
-FW will write an cam entry associated with it.
-
-*/
-struct set_assocsta_parm {
-	u8	addr[ETH_ALEN];
-};
-
-struct set_assocsta_rsp {
-	u8	cam_id;
-	u8	rsvd[3];
-};
-
-/*
-	Caller Ad-Hoc/AP
-
-	Command mode
-
-	This is to force fw to del an sta_data entry per driver's request
-
-	FW will invalidate the cam entry associated with it.
-
-*/
-struct del_assocsta_parm {
-	u8	addr[ETH_ALEN];
-};
-
-/*
-Caller Mode: AP/Ad-HoC(M)
-
-Notes: To notify fw that given staid has changed its power state
-
-Command Mode
-
-*/
-struct setstapwrstate_parm {
-	u8	staid;
-	u8	status;
-	u8	hwaddr[6];
-};
-
-/*
-Caller Mode: Any
-
-Notes: To setup the basic rate of RTL8711
-
-Command Mode
-
-*/
-struct	setbasicrate_parm {
-	u8	basicrates[NumRates];
-};
-
-/*
-Caller Mode: Any
-
-Notes: To read the current basic rate
-
-Command-Rsp Mode
-
-*/
-struct getbasicrate_parm {
-	u32 rsvd;
-};
-
-struct getbasicrate_rsp {
-	u8 basicrates[NumRates];
-};
-
-/*
-Caller Mode: Any
-
-Notes: To setup the data rate of RTL8711
-
-Command Mode
-
-*/
-struct setdatarate_parm {
-	u8	mac_id;
-	u8	datarates[NumRates];
-};
-
-/*
-Caller Mode: Any
-
-Notes: To read the current data rate
-
-Command-Rsp Mode
-
-*/
-struct getdatarate_parm {
-	u32 rsvd;
-};
-
-struct getdatarate_rsp {
-	u8 datarates[NumRates];
-};
-
-
-/*
-Caller Mode: Any
-AP: AP can use the info for the contents of beacon frame
-Infra: STA can use the info when sitesurveying
-Ad-HoC(M): Like AP
-Ad-HoC(C): Like STA
-
-
-Notes: To set the phy capability of the NIC
-
-Command Mode
-
-*/
-
-struct	setphyinfo_parm {
-	struct regulatory_class class_sets[NUM_REGULATORYS];
-	u8	status;
-};
-
-struct	getphyinfo_parm {
-	u32 rsvd;
-};
-
-struct	getphyinfo_rsp {
-	struct regulatory_class class_sets[NUM_REGULATORYS];
-	u8	status;
-};
-
-/*
-Caller Mode: Any
-
-Notes: To set the channel/modem/band
-This command will be used when channel/modem/band is changed.
-
-Command Mode
-
-*/
-struct	setphy_parm {
-	u8	rfchannel;
-	u8	modem;
-};
-
-/*
-Caller Mode: Any
-
-Notes: To get the current setting of channel/modem/band
-
-Command-Rsp Mode
-
-*/
-struct	getphy_parm {
-	u32 rsvd;
-};
-
-struct	getphy_rsp {
-	u8	rfchannel;
-	u8	modem;
-};
-
-struct readBB_parm {
-	u8	offset;
-};
-
-struct readBB_rsp {
-	u8	value;
-};
-
-struct readTSSI_parm {
-	u8	offset;
-};
-
-struct readTSSI_rsp {
-	u8	value;
-};
-
-struct writeBB_parm {
-	u8	offset;
-	u8	value;
-};
-
-struct readRF_parm {
-	u8	offset;
-};
-
-struct readRF_rsp {
-	u32	value;
-};
-
-struct writeRF_parm {
-	u32	offset;
-	u32	value;
-};
-
-struct getrfintfs_parm {
-	u8	rfintfs;
-};
-
-struct Tx_Beacon_param {
-	struct wlan_bssid_ex network;
-};
-
-/*  CMD param Formart for driver extra cmd handler */
-struct drvextra_cmd_parm {
-	int ec_id; /* extra cmd id */
-	int type_size; /*  Can use this field as the type id or command size */
-	unsigned char *pbuf;
-};
-
-/*------------------- Below are used for RF/BB tunning ---------------------*/
-
-struct	setantenna_parm {
-	u8	tx_antset;
-	u8	rx_antset;
-	u8	tx_antenna;
-	u8	rx_antenna;
-};
-
-struct	enrateadaptive_parm {
-	u32	en;
-};
-
-struct settxagctbl_parm {
-	u32	txagc[MAX_RATES_LENGTH];
-};
-
-struct gettxagctbl_parm {
-	u32 rsvd;
-};
-
-struct gettxagctbl_rsp {
-	u32	txagc[MAX_RATES_LENGTH];
-};
-
-struct setagcctrl_parm {
-	u32	agcctrl;		/*  0: pure hw, 1: fw */
-};
-
-struct setssup_parm	{
-	u32	ss_ForceUp[MAX_RATES_LENGTH];
-};
-
-struct getssup_parm	{
-	u32 rsvd;
-};
-
-struct getssup_rsp	{
-	u8	ss_ForceUp[MAX_RATES_LENGTH];
-};
-
-struct setssdlevel_parm	{
-	u8	ss_DLevel[MAX_RATES_LENGTH];
-};
-
-struct getssdlevel_parm	{
-	u32 rsvd;
-};
-
-struct getssdlevel_rsp	{
-	u8	ss_DLevel[MAX_RATES_LENGTH];
-};
-
-struct setssulevel_parm	{
-	u8	ss_ULevel[MAX_RATES_LENGTH];
-};
-
-struct getssulevel_parm	{
-	u32 rsvd;
-};
-
-struct getssulevel_rsp	{
-	u8	ss_ULevel[MAX_RATES_LENGTH];
-};
-
-struct	setcountjudge_parm {
-	u8	count_judge[MAX_RATES_LENGTH];
-};
-
-struct	getcountjudge_parm {
-	u32 rsvd;
-};
-
-struct	getcountjudge_rsp {
-	u8	count_judge[MAX_RATES_LENGTH];
-};
-
-struct setratable_parm {
-	u8 ss_ForceUp[NumRates];
-	u8 ss_ULevel[NumRates];
-	u8 ss_DLevel[NumRates];
-	u8 count_judge[NumRates];
-};
-
-struct getratable_parm {
-	uint rsvd;
-};
-
-struct getratable_rsp {
-	u8 ss_ForceUp[NumRates];
-	u8 ss_ULevel[NumRates];
-	u8 ss_DLevel[NumRates];
-	u8 count_judge[NumRates];
-};
-
-/* to get TX,RX retry count */
-struct gettxretrycnt_parm{
-	unsigned int rsvd;
-};
-struct gettxretrycnt_rsp{
-	unsigned long tx_retrycnt;
-};
-
-struct getrxretrycnt_parm{
-	unsigned int rsvd;
-};
-struct getrxretrycnt_rsp{
-	unsigned long rx_retrycnt;
-};
-
-/* to get BCNOK,BCNERR count */
-struct getbcnokcnt_parm{
-	unsigned int rsvd;
-};
-struct getbcnokcnt_rsp{
-	unsigned long  bcnokcnt;
-};
-
-struct getbcnerrcnt_parm{
-	unsigned int rsvd;
-};
-struct getbcnerrcnt_rsp{
-	unsigned long bcnerrcnt;
-};
-
-/*  to get current TX power level */
-struct getcurtxpwrlevel_parm{
-	unsigned int rsvd;
-};
-
-struct getcurtxpwrlevel_rsp{
-	unsigned short tx_power;
-};
-
-struct setprobereqextraie_parm {
-	unsigned char e_id;
-	unsigned char ie_len;
-	unsigned char ie[0];
-};
-
-struct setassocreqextraie_parm {
-	unsigned char e_id;
-	unsigned char ie_len;
-	unsigned char ie[0];
-};
-
-struct setproberspextraie_parm {
-	unsigned char e_id;
-	unsigned char ie_len;
-	unsigned char ie[0];
-};
-
-struct setassocrspextraie_parm {
-	unsigned char e_id;
-	unsigned char ie_len;
-	unsigned char ie[0];
-};
-
-struct addBaReq_parm {
-	unsigned int tid;
-	u8	addr[ETH_ALEN];
-};
-
-/*H2C Handler index: 46 */
-struct set_ch_parm {
-	u8 ch;
-	u8 bw;
-	u8 ch_offset;
-};
-
-/*H2C Handler index: 59 */
-struct SetChannelPlan_param {
-	u8 channel_plan;
-};
-
-/*H2C Handler index: 60 */
-struct LedBlink_param {
-	struct led_8723a *pLed;
-};
-
-/*H2C Handler index: 61 */
-struct SetChannelSwitch_param {
-	u8 new_ch_no;
-};
-
-/*H2C Handler index: 62 */
-struct TDLSoption_param {
-	u8 addr[ETH_ALEN];
-	u8 option;
-};
-
-#define GEN_CMD_CODE(cmd)	cmd ## _CMD_
-
-
-/*
-
-Result:
-0x00: success
-0x01: success, and check Response.
-0x02: cmd ignored due to duplicated sequcne number
-0x03: cmd dropped due to invalid cmd code
-0x04: reserved.
-
-*/
-
-#define H2C_RSP_OFFSET			512
-
-#define H2C_SUCCESS			0x00
-#define H2C_SUCCESS_RSP			0x01
-#define H2C_DUPLICATED			0x02
-#define H2C_DROPPED			0x03
-#define H2C_PARAMETERS_ERROR		0x04
-#define H2C_REJECTED			0x05
-#define H2C_CMD_OVERFLOW		0x06
-#define H2C_RESERVED			0x07
-
-int rtw_setassocsta_cmd(struct rtw_adapter  *padapter, u8 *mac_addr);
-int rtw_setstandby_cmd(struct rtw_adapter *padapter, uint action);
-int rtw_sitesurvey_cmd23a(struct rtw_adapter  *padapter, struct cfg80211_ssid *ssid, int ssid_num, struct rtw_ieee80211_channel *ch, int ch_num);
-int rtw_createbss_cmd23a(struct rtw_adapter  *padapter);
-int rtw_createbss_cmd23a_ex(struct rtw_adapter  *padapter, unsigned char *pbss, unsigned int sz);
-int rtw_setphy_cmd(struct rtw_adapter  *padapter, u8 modem, u8 ch);
-int rtw_setstakey_cmd23a(struct rtw_adapter  *padapter, u8 *psta, u8 unicast_key);
-int rtw_clearstakey_cmd23a(struct rtw_adapter *padapter, u8 *psta, u8 entry, u8 enqueue);
-int rtw_joinbss_cmd23a(struct rtw_adapter  *padapter, struct wlan_network* pnetwork);
-int rtw_disassoc_cmd23a(struct rtw_adapter *padapter, u32 deauth_timeout_ms, bool enqueue);
-int rtw_setopmode_cmd23a(struct rtw_adapter *padapter, enum nl80211_iftype ifmode);
-int rtw_setdatarate_cmd(struct rtw_adapter  *padapter, u8 *rateset);
-int rtw_setbasicrate_cmd(struct rtw_adapter  *padapter, u8 *rateset);
-int rtw_setbbreg_cmd(struct rtw_adapter *padapter, u8 offset, u8 val);
-int rtw_setrfreg_cmd(struct rtw_adapter *padapter, u8 offset, u32 val);
-int rtw_getbbreg_cmd(struct rtw_adapter *padapter, u8 offset, u8 *pval);
-int rtw_getrfreg_cmd(struct rtw_adapter *padapter, u8 offset, u8 *pval);
-int rtw_setrfintfs_cmd(struct rtw_adapter  *padapter, u8 mode);
-int rtw_setrttbl_cmd(struct rtw_adapter  *padapter, struct setratable_parm *prate_table);
-int rtw_getrttbl_cmd(struct rtw_adapter  *padapter, struct getratable_rsp *pval);
-
-int rtw_gettssi_cmd(struct rtw_adapter  *padapter, u8 offset, u8 *pval);
-int rtw_setfwdig_cmd(struct rtw_adapter*padapter, u8 type);
-int rtw_setfwra_cmd(struct rtw_adapter*padapter, u8 type);
-
-int rtw_addbareq_cmd23a(struct rtw_adapter*padapter, u8 tid, u8 *addr);
-
-int rtw_dynamic_chk_wk_cmd23a(struct rtw_adapter *adapter);
-
-int rtw_lps_ctrl_wk_cmd23a(struct rtw_adapter*padapter, u8 lps_ctrl_type, u8 enqueue);
-
-int rtw_ps_cmd23a(struct rtw_adapter*padapter);
-
-#ifdef CONFIG_8723AU_AP_MODE
-int rtw_chk_hi_queue_cmd23a(struct rtw_adapter*padapter);
-#endif
-
-int rtw_set_chplan_cmd(struct rtw_adapter*padapter, u8 chplan, u8 enqueue);
-int rtw_led_blink_cmd(struct rtw_adapter*padapter, struct led_8723a *pLed);
-int rtw_set_csa_cmd(struct rtw_adapter*padapter, u8 new_ch_no);
-
-int rtw_c2h_wk_cmd23a(struct rtw_adapter *padapter, u8 *c2h_evt);
-
-int rtw_drvextra_cmd_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-
-void rtw_survey_cmd_callback23a(struct rtw_adapter  *padapter, struct cmd_obj *pcmd);
-void rtw_disassoc_cmd23a_callback(struct rtw_adapter  *padapter, struct cmd_obj *pcmd);
-void rtw_joinbss_cmd23a_callback(struct rtw_adapter  *padapter, struct cmd_obj *pcmd);
-void rtw_createbss_cmd23a_callback(struct rtw_adapter  *padapter, struct cmd_obj *pcmd);
-void rtw_getbbrfreg_cmdrsp_callback23a(struct rtw_adapter  *padapter, struct cmd_obj *pcmd);
-
-void rtw_setstaKey_cmdrsp_callback23a(struct rtw_adapter  *padapter,  struct cmd_obj *pcmd);
-void rtw_setassocsta_cmdrsp_callback23a(struct rtw_adapter  *padapter,  struct cmd_obj *pcmd);
-
-struct _cmd_callback {
-	u32	cmd_code;
-	void (*callback)(struct rtw_adapter  *padapter, struct cmd_obj *cmd);
-};
-
-enum rtw_h2c_cmd {
-	GEN_CMD_CODE(_Read_MACREG) ,	/*0*/
-	GEN_CMD_CODE(_Write_MACREG) ,
-	GEN_CMD_CODE(_Read_BBREG) ,
-	GEN_CMD_CODE(_Write_BBREG) ,
-	GEN_CMD_CODE(_Read_RFREG) ,
-	GEN_CMD_CODE(_Write_RFREG) , /*5*/
-	GEN_CMD_CODE(_Read_EEPROM) ,
-	GEN_CMD_CODE(_Write_EEPROM) ,
-	GEN_CMD_CODE(_Read_EFUSE) ,
-	GEN_CMD_CODE(_Write_EFUSE) ,
-
-	GEN_CMD_CODE(_Read_CAM) ,	/*10*/
-	GEN_CMD_CODE(_Write_CAM) ,
-	GEN_CMD_CODE(_setBCNITV),
-	GEN_CMD_CODE(_setMBIDCFG),
-	GEN_CMD_CODE(_JoinBss),   /*14*/
-	GEN_CMD_CODE(_DisConnect) , /*15*/
-	GEN_CMD_CODE(_CreateBss) ,
-	GEN_CMD_CODE(_SetOpMode) ,
-	GEN_CMD_CODE(_SiteSurvey),  /*18*/
-	GEN_CMD_CODE(_SetAuth) ,
-
-	GEN_CMD_CODE(_SetKey) ,	/*20*/
-	GEN_CMD_CODE(_SetStaKey) ,
-	GEN_CMD_CODE(_SetAssocSta) ,
-	GEN_CMD_CODE(_DelAssocSta) ,
-	GEN_CMD_CODE(_SetStaPwrState) ,
-	GEN_CMD_CODE(_SetBasicRate) , /*25*/
-	GEN_CMD_CODE(_GetBasicRate) ,
-	GEN_CMD_CODE(_SetDataRate) ,
-	GEN_CMD_CODE(_GetDataRate) ,
-	GEN_CMD_CODE(_SetPhyInfo) ,
-
-	GEN_CMD_CODE(_GetPhyInfo) ,	/*30*/
-	GEN_CMD_CODE(_SetPhy) ,
-	GEN_CMD_CODE(_GetPhy) ,
-	GEN_CMD_CODE(_readRssi) ,
-	GEN_CMD_CODE(_readGain) ,
-	GEN_CMD_CODE(_SetAtim) , /*35*/
-	GEN_CMD_CODE(_SetPwrMode) ,
-	GEN_CMD_CODE(_JoinbssRpt),
-	GEN_CMD_CODE(_SetRaTable) ,
-	GEN_CMD_CODE(_GetRaTable) ,
-
-	GEN_CMD_CODE(_GetCCXReport), /*40*/
-	GEN_CMD_CODE(_GetDTMReport),
-	GEN_CMD_CODE(_GetTXRateStatistics),
-	GEN_CMD_CODE(_SetUsbSuspend),
-	GEN_CMD_CODE(_SetH2cLbk),
-	GEN_CMD_CODE(_AddBAReq) , /*45*/
-	GEN_CMD_CODE(_SetChannel), /*46*/
-	GEN_CMD_CODE(_SetTxPower),
-	GEN_CMD_CODE(_SwitchAntenna),
-	GEN_CMD_CODE(_SetCrystalCap),
-	GEN_CMD_CODE(_SetSingleCarrierTx), /*50*/
-
-	GEN_CMD_CODE(_SetSingleToneTx),/*51*/
-	GEN_CMD_CODE(_SetCarrierSuppressionTx),
-	GEN_CMD_CODE(_SetContinuousTx),
-	GEN_CMD_CODE(_SwitchBandwidth), /*54*/
-	GEN_CMD_CODE(_TX_Beacon), /*55*/
-
-	GEN_CMD_CODE(_Set_MLME_EVT), /*56*/
-	GEN_CMD_CODE(_Set_Drv_Extra), /*57*/
-	GEN_CMD_CODE(_Set_H2C_MSG), /*58*/
-
-	GEN_CMD_CODE(_SetChannelPlan), /*59*/
-	GEN_CMD_CODE(_LedBlink), /*60*/
-
-	GEN_CMD_CODE(_SetChannelSwitch), /*61*/
-	GEN_CMD_CODE(_TDLS), /*62*/
-
-	MAX_H2CCMD
-};
-
-extern struct _cmd_callback	rtw_cmd_callback[];
-
-#endif /*  _CMD_H_ */
diff --git a/drivers/staging/rtl8723au/include/rtw_debug.h b/drivers/staging/rtl8723au/include/rtw_debug.h
deleted file mode 100644
index 159183e9cab0..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_debug.h
+++ /dev/null
@@ -1,191 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- *
- ******************************************************************************/
-#ifndef __RTW_DEBUG_H__
-#define __RTW_DEBUG_H__
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#define _drv_always_			1
-#define _drv_emerg_			2
-#define _drv_alert_			3
-#define _drv_err_			4
-#define	_drv_warning_			5
-#define _drv_notice_			6
-#define _drv_info_			7
-#define	_drv_debug_			8
-
-#define _module_rtl871x_xmit_c_		BIT(0)
-#define _module_xmit_osdep_c_		BIT(1)
-#define _module_rtl871x_recv_c_		BIT(2)
-#define _module_recv_osdep_c_		BIT(3)
-#define _module_rtl871x_mlme_c_		BIT(4)
-#define _module_mlme_osdep_c_		BIT(5)
-#define _module_rtl871x_sta_mgt_c_	BIT(6)
-#define _module_rtl871x_cmd_c_		BIT(7)
-#define _module_cmd_osdep_c_		BIT(8)
-#define _module_rtl871x_io_c_		BIT(9)
-#define _module_io_osdep_c_		BIT(10)
-#define _module_os_intfs_c_		BIT(11)
-#define _module_rtl871x_security_c_	BIT(12)
-#define _module_rtl871x_eeprom_c_	BIT(13)
-#define _module_hal_init_c_		BIT(14)
-#define _module_hci_hal_init_c_		BIT(15)
-#define _module_rtl871x_ioctl_c_	BIT(16)
-#define _module_rtl871x_ioctl_set_c_	BIT(17)
-#define _module_rtl871x_ioctl_query_c_	BIT(18)
-#define _module_rtl871x_pwrctrl_c_	BIT(19)
-#define _module_hci_intfs_c_		BIT(20)
-#define _module_hci_ops_c_		BIT(21)
-#define _module_osdep_service_c_	BIT(22)
-#define _module_mp_			BIT(23)
-#define _module_hci_ops_os_c_		BIT(24)
-#define _module_rtl871x_ioctl_os_c	BIT(25)
-#define _module_rtl8712_cmd_c_		BIT(26)
-#define	_module_rtl8192c_xmit_c_	BIT(28)
-#define _module_hal_xmit_c_		BIT(28) /* duplication intentional */
-#define _module_efuse_			BIT(29)
-#define _module_rtl8712_recv_c_		BIT(30)
-#define _module_rtl8712_led_c_		BIT(31)
-
-#undef _MODULE_DEFINE_
-
-#if defined _RTW_XMIT_C_
-	#define _MODULE_DEFINE_	_module_rtl871x_xmit_c_
-#elif defined _XMIT_OSDEP_C_
-	#define _MODULE_DEFINE_	_module_xmit_osdep_c_
-#elif defined _RTW_RECV_C_
-	#define _MODULE_DEFINE_	_module_rtl871x_recv_c_
-#elif defined _RECV_OSDEP_C_
-	#define _MODULE_DEFINE_	_module_recv_osdep_c_
-#elif defined _RTW_MLME_C_
-	#define _MODULE_DEFINE_	_module_rtl871x_mlme_c_
-#elif defined _MLME_OSDEP_C_
-	#define _MODULE_DEFINE_	_module_mlme_osdep_c_
-#elif defined _RTW_MLME_EXT_C_
-	#define _MODULE_DEFINE_ 1
-#elif defined _RTW_STA_MGT_C_
-	#define _MODULE_DEFINE_	_module_rtl871x_sta_mgt_c_
-#elif defined _RTW_CMD_C_
-	#define _MODULE_DEFINE_	_module_rtl871x_cmd_c_
-#elif defined _CMD_OSDEP_C_
-	#define _MODULE_DEFINE_	_module_cmd_osdep_c_
-#elif defined _RTW_IO_C_
-	#define _MODULE_DEFINE_	_module_rtl871x_io_c_
-#elif defined _IO_OSDEP_C_
-	#define _MODULE_DEFINE_	_module_io_osdep_c_
-#elif defined _OS_INTFS_C_
-	#define	_MODULE_DEFINE_	_module_os_intfs_c_
-#elif defined _RTW_SECURITY_C_
-	#define	_MODULE_DEFINE_	_module_rtl871x_security_c_
-#elif defined _RTW_EEPROM_C_
-	#define	_MODULE_DEFINE_	_module_rtl871x_eeprom_c_
-#elif defined _HAL_INTF_C_
-	#define	_MODULE_DEFINE_	_module_hal_init_c_
-#elif (defined _HCI_HAL_INIT_C_) || (defined _SDIO_HALINIT_C_)
-	#define	_MODULE_DEFINE_	_module_hci_hal_init_c_
-#elif defined _RTL871X_IOCTL_C_
-	#define	_MODULE_DEFINE_	_module_rtl871x_ioctl_c_
-#elif defined _RTL871X_IOCTL_SET_C_
-	#define	_MODULE_DEFINE_	_module_rtl871x_ioctl_set_c_
-#elif defined _RTL871X_IOCTL_QUERY_C_
-	#define	_MODULE_DEFINE_	_module_rtl871x_ioctl_query_c_
-#elif defined _RTL871X_PWRCTRL_C_
-	#define	_MODULE_DEFINE_	_module_rtl871x_pwrctrl_c_
-#elif defined _RTW_PWRCTRL_C_
-	#define	_MODULE_DEFINE_	1
-#elif defined _HCI_INTF_C_
-	#define	_MODULE_DEFINE_	_module_hci_intfs_c_
-#elif defined _HCI_OPS_C_
-	#define	_MODULE_DEFINE_	_module_hci_ops_c_
-#elif defined _SDIO_OPS_C_
-	#define	_MODULE_DEFINE_ 1
-#elif defined _OSDEP_HCI_INTF_C_
-	#define	_MODULE_DEFINE_	_module_hci_intfs_c_
-#elif defined _OSDEP_SERVICE_C_
-	#define	_MODULE_DEFINE_	_module_osdep_service_c_
-#elif defined _HCI_OPS_OS_C_
-	#define	_MODULE_DEFINE_	_module_hci_ops_os_c_
-#elif defined _RTL871X_IOCTL_LINUX_C_
-	#define	_MODULE_DEFINE_	_module_rtl871x_ioctl_os_c
-#elif defined _RTL8712_CMD_C_
-	#define	_MODULE_DEFINE_	_module_rtl8712_cmd_c_
-#elif defined _RTL8192C_XMIT_C_
-	#define	_MODULE_DEFINE_	1
-#elif defined _RTL8723AS_XMIT_C_
-	#define	_MODULE_DEFINE_	1
-#elif defined _RTL8712_RECV_C_
-	#define	_MODULE_DEFINE_	_module_rtl8712_recv_c_
-#elif defined _RTL8192CU_RECV_C_
-	#define	_MODULE_DEFINE_	_module_rtl8712_recv_c_
-#elif defined _RTL871X_MLME_EXT_C_
-	#define _MODULE_DEFINE_	_module_mlme_osdep_c_
-#elif defined _RTW_MP_C_
-	#define	_MODULE_DEFINE_	_module_mp_
-#elif defined _RTW_MP_IOCTL_C_
-	#define	_MODULE_DEFINE_	_module_mp_
-#elif defined _RTW_EFUSE_C_
-	#define	_MODULE_DEFINE_	_module_efuse_
-#endif
-
-#define DRIVER_PREFIX	"RTL8723AU: "
-#define DEBUG_LEVEL	(_drv_err_)
-#define DBG_8723A_LEVEL(_level, fmt, arg...)				\
-	do {								\
-		if (_level <= GlobalDebugLevel23A)				\
-			pr_info(DRIVER_PREFIX fmt, ##arg);\
-	} while (0)
-
-#define DBG_8723A(...)							\
-	do {								\
-		if (_drv_err_ <= GlobalDebugLevel23A)			\
-			pr_info(DRIVER_PREFIX __VA_ARGS__);		\
-	} while (0)
-
-#define MSG_8723A(...)							\
-	do {								\
-		if (_drv_err_ <= GlobalDebugLevel23A)			\
-			pr_info(DRIVER_PREFIX __VA_ARGS__);		\
-	} while (0)
-
-extern u32 GlobalDebugLevel23A;
-
-__printf(3, 4)
-void rt_trace(int comp, int level, const char *fmt, ...);
-
-#define RT_TRACE(_Comp, _Level, Fmt, ...)				\
-do {									\
-	if (_Level <= GlobalDebugLevel23A)				\
-		rt_trace(_Comp, _Level, Fmt, ##__VA_ARGS__);		\
-} while (0)
-
-#define RT_PRINT_DATA(_Comp, _Level, _TitleString, _HexData,		\
-		      _HexDataLen)					\
-	if (_Level <= GlobalDebugLevel23A) {				\
-		int __i;						\
-		u8	*ptr = (u8 *)_HexData;				\
-		pr_info("%s", DRIVER_PREFIX);				\
-		pr_info(_TitleString);					\
-		for (__i = 0; __i < (int)_HexDataLen; __i++) {		\
-			printk("%02X%s", ptr[__i],			\
-			       (((__i + 1) % 4) == 0) ? "  " : " ");	\
-			if (((__i + 1) % 16) == 0)			\
-				printk("\n");				\
-		}							\
-		printk("\n");						\
-	}
-
-#endif	/* __RTW_DEBUG_H__ */
diff --git a/drivers/staging/rtl8723au/include/rtw_eeprom.h b/drivers/staging/rtl8723au/include/rtw_eeprom.h
deleted file mode 100644
index a86f36e49dd1..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_eeprom.h
+++ /dev/null
@@ -1,135 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTW_EEPROM_H__
-#define __RTW_EEPROM_H__
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#define	RTL8712_EEPROM_ID			0x8712
-/* define	EEPROM_MAX_SIZE			256 */
-
-#define	HWSET_MAX_SIZE_512		512
-#define	EEPROM_MAX_SIZE			HWSET_MAX_SIZE_512
-
-#define	CLOCK_RATE					50			/* 100us */
-
-/*  EEPROM opcodes */
-#define EEPROM_READ_OPCODE		06
-#define EEPROM_WRITE_OPCODE		05
-#define EEPROM_ERASE_OPCODE		07
-#define EEPROM_EWEN_OPCODE		19      /*  Erase/write enable */
-#define EEPROM_EWDS_OPCODE		16      /*  Erase/write disable */
-
-/* Country codes */
-#define USA							0x555320
-#define EUROPE						0x1 /* temp, should be provided later */
-#define JAPAN						0x2 /* temp, should be provided later */
-
-#define	EEPROM_CID_DEFAULT			0x0
-#define	EEPROM_CID_ALPHA				0x1
-#define	EEPROM_CID_Senao				0x3
-#define	EEPROM_CID_NetCore				0x5
-#define	EEPROM_CID_CAMEO				0X8
-#define	EEPROM_CID_SITECOM				0x9
-#define	EEPROM_CID_COREGA				0xB
-#define	EEPROM_CID_EDIMAX_BELKIN		0xC
-#define	EEPROM_CID_SERCOMM_BELKIN		0xE
-#define	EEPROM_CID_CAMEO1				0xF
-#define	EEPROM_CID_WNC_COREGA		0x12
-#define	EEPROM_CID_CLEVO				0x13
-#define	EEPROM_CID_WHQL				0xFE /*  added by chiyoko for dtm, 20090108 */
-
-/*  */
-/*  Customer ID, note that: */
-/*  This variable is initiailzed through EEPROM or registry, */
-/*  however, its definition may be different with that in EEPROM for */
-/*  EEPROM size consideration. So, we have to perform proper translation between them. */
-/*  Besides, CustomerID of registry has precedence of that of EEPROM. */
-/*  defined below. 060703, by rcnjko. */
-/*  */
-enum rt_customer_id
-{
-	RT_CID_DEFAULT = 0,
-	RT_CID_8187_ALPHA0 = 1,
-	RT_CID_8187_SERCOMM_PS = 2,
-	RT_CID_8187_HW_LED = 3,
-	RT_CID_8187_NETGEAR = 4,
-	RT_CID_WHQL = 5,
-	RT_CID_819x_CAMEO  = 6,
-	RT_CID_819x_RUNTOP = 7,
-	RT_CID_819x_Senao = 8,
-	RT_CID_TOSHIBA = 9,	/*  Merge by Jacken, 2008/01/31. */
-	RT_CID_819x_Netcore = 10,
-	RT_CID_Nettronix = 11,
-	RT_CID_DLINK = 12,
-	RT_CID_PRONET = 13,
-	RT_CID_COREGA = 14,
-	RT_CID_CHINA_MOBILE = 15,
-	RT_CID_819x_ALPHA = 16,
-	RT_CID_819x_Sitecom = 17,
-	RT_CID_CCX = 18, /*  It's set under CCX logo test and isn't demanded for CCX functions, but for test behavior like retry limit and tx report. By Bruce, 2009-02-17. */
-	RT_CID_819x_Lenovo = 19,
-	RT_CID_819x_QMI = 20,
-	RT_CID_819x_Edimax_Belkin = 21,
-	RT_CID_819x_Sercomm_Belkin = 22,
-	RT_CID_819x_CAMEO1 = 23,
-	RT_CID_819x_MSI = 24,
-	RT_CID_819x_Acer = 25,
-	RT_CID_819x_AzWave_ASUS = 26,
-	RT_CID_819x_AzWave = 27, /*  For AzWave in PCIe, The ID is AzWave use and not only Asus */
-	RT_CID_819x_HP = 28,
-	RT_CID_819x_WNC_COREGA = 29,
-	RT_CID_819x_Arcadyan_Belkin = 30,
-	RT_CID_819x_SAMSUNG = 31,
-	RT_CID_819x_CLEVO = 32,
-	RT_CID_819x_DELL = 33,
-	RT_CID_819x_PRONETS = 34,
-	RT_CID_819x_Edimax_ASUS = 35,
-	RT_CID_819x_CAMEO_NETGEAR = 36,
-	RT_CID_PLANEX = 37,
-	RT_CID_CC_C = 38,
-	RT_CID_819x_Xavi = 39,
-	RT_CID_819x_FUNAI_TV = 40,
-	RT_CID_819x_ALPHA_WD=41,
-};
-
-struct eeprom_priv {
-	u8		mac_addr[6];	/* PermanentAddress */
-	u8		bautoload_fail_flag;
-	u8		bloadfile_fail_flag;
-	u8		bloadmac_fail_flag;
-	/* u8		bempty; */
-	/* u8		sys_config; */
-	/* u8		config0; */
-	u16		channel_plan;
-	/* u8		country_string[3]; */
-	/* u8		tx_power_b[15]; */
-	/* u8		tx_power_g[15]; */
-	/* u8		tx_power_a[201]; */
-
-	u8		EepromOrEfuse;
-
-	u8		efuse_eeprom_data[HWSET_MAX_SIZE_512]; /* 92C:256bytes, 88E:512bytes, we use union set (512bytes) */
-};
-
-void eeprom_write16(struct rtw_adapter *padapter, u16 reg, u16 data);
-u16 eeprom_read16(struct rtw_adapter *padapter, u16 reg);
-void read_eeprom_content(struct rtw_adapter *padapter);
-void eeprom_read_sz(struct rtw_adapter *padapter, u16 reg, u8 *data, u32 sz);
-
-void read_eeprom_content_by_attrib(struct rtw_adapter *padapter);
-
-#endif  /* __RTL871X_EEPROM_H__ */
diff --git a/drivers/staging/rtl8723au/include/rtw_efuse.h b/drivers/staging/rtl8723au/include/rtw_efuse.h
deleted file mode 100644
index c577e260f151..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_efuse.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- *
- ******************************************************************************/
-#ifndef __RTW_EFUSE_H__
-#define __RTW_EFUSE_H__
-
-#include <osdep_service.h>
-
-#define	EFUSE_ERROE_HANDLE		1
-
-#define	PG_STATE_HEADER			0x01
-#define	PG_STATE_WORD_0		0x02
-#define	PG_STATE_WORD_1		0x04
-#define	PG_STATE_WORD_2		0x08
-#define	PG_STATE_WORD_3		0x10
-#define	PG_STATE_DATA			0x20
-
-#define	PG_SWBYTE_H			0x01
-#define	PG_SWBYTE_L			0x02
-
-#define	PGPKT_DATA_SIZE		8
-
-#define	EFUSE_WIFI				0
-#define	EFUSE_BT				1
-
-enum _EFUSE_DEF_TYPE {
-	TYPE_EFUSE_MAX_SECTION				= 0,
-	TYPE_EFUSE_REAL_CONTENT_LEN			= 1,
-	TYPE_AVAILABLE_EFUSE_BYTES_BANK		= 2,
-	TYPE_AVAILABLE_EFUSE_BYTES_TOTAL	= 3,
-	TYPE_EFUSE_MAP_LEN					= 4,
-	TYPE_EFUSE_PROTECT_BYTES_BANK		= 5,
-	TYPE_EFUSE_CONTENT_LEN_BANK			= 6,
-};
-
-/* E-Fuse */
-#define EFUSE_MAP_SIZE      256
-
-#define EFUSE_MAX_SIZE      512
-/* end of E-Fuse */
-
-#define		EFUSE_MAX_MAP_LEN		256
-#define		EFUSE_MAX_HW_SIZE		512
-#define		EFUSE_MAX_SECTION_BASE	16
-
-#define EXT_HEADER(header) ((header & 0x1F) == 0x0F)
-#define ALL_WORDS_DISABLED(wde)	((wde & 0x0F) == 0x0F)
-#define GET_HDR_OFFSET_2_0(header) ( (header & 0xE0) >> 5)
-
-#define		EFUSE_REPEAT_THRESHOLD_			3
-
-/*  */
-/*	The following is for BT Efuse definition */
-/*  */
-#define		EFUSE_BT_MAX_MAP_LEN		1024
-#define		EFUSE_MAX_BANK			4
-#define		EFUSE_MAX_BT_BANK		(EFUSE_MAX_BANK-1)
-/*  */
-/*--------------------------Define Parameters-------------------------------*/
-#define		EFUSE_MAX_WORD_UNIT			4
-
-/*------------------------------Define structure----------------------------*/
-struct pg_pkt_struct {
-	u8 offset;
-	u8 word_en;
-	u8 data[8];
-	u8 word_cnts;
-};
-
-/*------------------------Export global variable----------------------------*/
-
-u16	efuse_GetMaxSize23a(struct rtw_adapter *padapter);
-int	rtw_efuse_access23a(struct rtw_adapter *padapter, u8 bRead, u16 start_addr, u16 cnts, u8 *data);
-int	rtw_efuse_map_read23a(struct rtw_adapter *padapter, u16 addr, u16 cnts, u8 *data);
-u8	rtw_efuse_map_write(struct rtw_adapter *padapter, u16 addr, u16 cnts, u8 *data);
-int	rtw_BT_efuse_map_read23a(struct rtw_adapter *padapter, u16 addr, u16 cnts, u8 *data);
-u8	rtw_BT_efuse_map_write(struct rtw_adapter *padapter, u16 addr, u16 cnts, u8 *data);
-
-u16	Efuse_GetCurrentSize23a(struct rtw_adapter *pAdapter, u8 efuseType);
-u8	Efuse_CalculateWordCnts23a(u8 word_en);
-void	ReadEFuseByte23a(struct rtw_adapter *Adapter, u16 _offset, u8 *pbuf);
-void	EFUSE_GetEfuseDefinition23a(struct rtw_adapter *pAdapter, u8 efuseType, u8 type, void *pOut);
-int	efuse_OneByteRead23a(struct rtw_adapter *pAdapter, u16 addr, u8 *data);
-int	efuse_OneByteWrite23a(struct rtw_adapter *pAdapter, u16 addr, u8 data);
-
-void	Efuse_PowerSwitch23a(struct rtw_adapter *pAdapter, u8 bWrite,
-			     u8 PwrState);
-int	Efuse_PgPacketRead23a(struct rtw_adapter *pAdapter, u8 offset, u8 *data);
-int	Efuse_PgPacketWrite23a(struct rtw_adapter *pAdapter, u8 offset, u8 word_en, u8 *data);
-void	efuse_WordEnableDataRead23a(u8 word_en, u8 *sourdata, u8 *targetdata);
-u8	Efuse_WordEnableDataWrite23a(struct rtw_adapter *pAdapter, u16 efuse_addr, u8 word_en, u8 *data);
-
-u8	EFUSE_Read1Byte23a(struct rtw_adapter *pAdapter, u16 Address);
-void	EFUSE_ShadowMapUpdate23a(struct rtw_adapter *pAdapter, u8 efuseType);
-void	EFUSE_ShadowRead23a(struct rtw_adapter *pAdapter, u8 Type, u16 Offset, u32 *Value);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtw_event.h b/drivers/staging/rtl8723au/include/rtw_event.h
deleted file mode 100644
index 4557aeccc604..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_event.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef _RTW_EVENT_H_
-#define _RTW_EVENT_H_
-
-#include <osdep_service.h>
-#include <wlan_bssdef.h>
-
-/*
-Used to report a bss has been scanned
-*/
-struct survey_event {
-	struct wlan_bssid_ex *bss;
-};
-
-/*
-Used to report that the requested site survey has been done.
-bss_cnt indicates the number of bss that has been reported.
-*/
-struct surveydone_event {
-	unsigned int bss_cnt;
-};
-
-/*
-Used to report the link result of joinning the given bss
-join_res:
--1: authentication fail
--2: association fail
-> 0: TID
-*/
-struct joinbss_event {
-	struct wlan_network network;
-};
-
-/*
-Used to report a given STA has joinned the created BSS.
-It is used in AP/Ad-HoC(M) mode.
-*/
-struct stassoc_event {
-	unsigned char macaddr[6];
-	unsigned char rsvd[2];
-	int cam_id;
-};
-
-struct stadel_event {
-	unsigned char macaddr[6];
-	unsigned char rsvd[2]; /* for reason */
-	int mac_id;
-};
-
-struct addba_event {
-	unsigned int tid;
-};
-
-#define GEN_EVT_CODE(event)	event ## _EVT_
-
-struct fwevent {
-	u32 parmsize;
-	void (*event_callback)(struct rtw_adapter *dev, const u8 *pbuf);
-};
-
-#endif /*  _WLANEVENT_H_ */
diff --git a/drivers/staging/rtl8723au/include/rtw_ht.h b/drivers/staging/rtl8723au/include/rtw_ht.h
deleted file mode 100644
index 780eb8944118..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_ht.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef _RTW_HT_H_
-#define _RTW_HT_H_
-
-#include <osdep_service.h>
-#include "linux/ieee80211.h"
-#include "wifi.h"
-
-struct ht_priv {
-	bool	ht_option;
-	bool	ampdu_enable;/* for enable Tx A-MPDU */
-	/* u8	baddbareq_issued[16]; */
-	u32	tx_amsdu_enable;/* for enable Tx A-MSDU */
-	u32	tx_amdsu_maxlen; /*  1: 8k, 0:4k ; default:8k, for tx */
-	u32	rx_ampdu_maxlen; /* for rx reordering ctrl win_sz, updated when join_callback. */
-
-	u8	bwmode;/*  */
-	u8	ch_offset;/* PRIME_CHNL_OFFSET */
-	u8	sgi;/* short GI */
-
-	/* for processing Tx A-MPDU */
-	u16	agg_enable_bitmap;
-	/* u8	ADDBA_retry_count; */
-	u16	candidate_tid_bitmap;
-
-	struct ieee80211_ht_cap ht_cap;
-};
-
-#endif	/* _RTL871X_HT_H_ */
diff --git a/drivers/staging/rtl8723au/include/rtw_io.h b/drivers/staging/rtl8723au/include/rtw_io.h
deleted file mode 100644
index c8119e2c6545..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_io.h
+++ /dev/null
@@ -1,237 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#ifndef _RTW_IO_H_
-#define _RTW_IO_H_
-
-#include <osdep_service.h>
-#include <osdep_intf.h>
-
-#include <asm/byteorder.h>
-#include <linux/semaphore.h>
-#include <linux/list.h>
-/* include <linux/smp_lock.h> */
-#include <linux/spinlock.h>
-#include <asm/atomic.h>
-
-#include <linux/usb.h>
-#include <linux/usb/ch9.h>
-
-#define rtw_usb_buffer_alloc(dev, size, dma) usb_alloc_coherent((dev), (size), (in_interrupt() ? GFP_ATOMIC : GFP_KERNEL), (dma))
-#define rtw_usb_buffer_free(dev, size, addr, dma) usb_free_coherent((dev), (size), (addr), (dma))
-
-#define NUM_IOREQ		8
-
-#define MAX_PROT_SZ	(64-16)
-
-#define _IOREADY			0
-#define _IO_WAIT_COMPLETE   1
-#define _IO_WAIT_RSP        2
-
-/*  IO COMMAND TYPE */
-#define _IOSZ_MASK_		(0x7F)
-#define _IO_WRITE_		BIT(7)
-#define _IO_FIXED_		BIT(8)
-#define _IO_BURST_		BIT(9)
-#define _IO_BYTE_		BIT(10)
-#define _IO_HW_			BIT(11)
-#define _IO_WORD_		BIT(12)
-#define _IO_SYNC_		BIT(13)
-#define _IO_CMDMASK_	(0x1F80)
-
-
-/*
-	For prompt mode accessing, caller shall free io_req
-	Otherwise, io_handler will free io_req
-*/
-
-
-
-/*  IO STATUS TYPE */
-#define _IO_ERR_		BIT(2)
-#define _IO_SUCCESS_	BIT(1)
-#define _IO_DONE_		BIT(0)
-
-
-#define IO_RD32			(_IO_SYNC_ | _IO_WORD_)
-#define IO_RD16			(_IO_SYNC_ | _IO_HW_)
-#define IO_RD8			(_IO_SYNC_ | _IO_BYTE_)
-
-#define IO_RD32_ASYNC	(_IO_WORD_)
-#define IO_RD16_ASYNC	(_IO_HW_)
-#define IO_RD8_ASYNC	(_IO_BYTE_)
-
-#define IO_WR32			(_IO_WRITE_ | _IO_SYNC_ | _IO_WORD_)
-#define IO_WR16			(_IO_WRITE_ | _IO_SYNC_ | _IO_HW_)
-#define IO_WR8			(_IO_WRITE_ | _IO_SYNC_ | _IO_BYTE_)
-
-#define IO_WR32_ASYNC	(_IO_WRITE_ | _IO_WORD_)
-#define IO_WR16_ASYNC	(_IO_WRITE_ | _IO_HW_)
-#define IO_WR8_ASYNC	(_IO_WRITE_ | _IO_BYTE_)
-
-/*
-
-	Only Sync. burst accessing is provided.
-
-*/
-
-#define IO_WR_BURST(x)		(_IO_WRITE_ | _IO_SYNC_ | _IO_BURST_ | ( (x) & _IOSZ_MASK_))
-#define IO_RD_BURST(x)		(_IO_SYNC_ | _IO_BURST_ | ( (x) & _IOSZ_MASK_))
-
-
-
-/* below is for the intf_option bit defition... */
-
-#define _INTF_ASYNC_	BIT(0)	/* support async io */
-
-struct intf_priv;
-
-struct io_req {
-	struct list_head	list;
-	u32	addr;
-	volatile u32	val;
-	u32	command;
-	u32	status;
-	u8	*pbuf;
-	struct semaphore	sema;
-
-	void (*_async_io_callback)(struct rtw_adapter *padater, struct io_req *pio_req, u8 *cnxt);
-	u8 *cnxt;
-};
-
-struct reg_protocol_rd {
-
-#ifdef __LITTLE_ENDIAN
-
-	/* DW1 */
-	u32		NumOfTrans:4;
-	u32		Reserved1:4;
-	u32		Reserved2:24;
-	/* DW2 */
-	u32		ByteCount:7;
-	u32		WriteEnable:1;		/* 0:read, 1:write */
-	u32		FixOrContinuous:1;	/* 0:continuous, 1: Fix */
-	u32		BurstMode:1;
-	u32		Byte1Access:1;
-	u32		Byte2Access:1;
-	u32		Byte4Access:1;
-	u32		Reserved3:3;
-	u32		Reserved4:16;
-	/* DW3 */
-	u32		BusAddress;
-	/* DW4 */
-	/* u32		Value; */
-#else
-
-
-/* DW1 */
-	u32 Reserved1  :4;
-	u32 NumOfTrans :4;
-
-	u32 Reserved2  :24;
-
-	/* DW2 */
-	u32 WriteEnable : 1;
-	u32 ByteCount :7;
-
-
-	u32 Reserved3 : 3;
-	u32 Byte4Access : 1;
-
-	u32 Byte2Access : 1;
-	u32 Byte1Access : 1;
-	u32 BurstMode :1 ;
-	u32 FixOrContinuous : 1;
-
-	u32 Reserved4 : 16;
-
-	/* DW3 */
-	u32		BusAddress;
-
-	/* DW4 */
-	/* u32		Value; */
-
-#endif
-
-};
-
-
-struct reg_protocol_wt {
-
-
-#ifdef __LITTLE_ENDIAN
-
-	/* DW1 */
-	u32		NumOfTrans:4;
-	u32		Reserved1:4;
-	u32		Reserved2:24;
-	/* DW2 */
-	u32		ByteCount:7;
-	u32		WriteEnable:1;		/* 0:read, 1:write */
-	u32		FixOrContinuous:1;	/* 0:continuous, 1: Fix */
-	u32		BurstMode:1;
-	u32		Byte1Access:1;
-	u32		Byte2Access:1;
-	u32		Byte4Access:1;
-	u32		Reserved3:3;
-	u32		Reserved4:16;
-	/* DW3 */
-	u32		BusAddress;
-	/* DW4 */
-	u32		Value;
-
-#else
-	/* DW1 */
-	u32 Reserved1  :4;
-	u32 NumOfTrans :4;
-
-	u32 Reserved2  :24;
-
-	/* DW2 */
-	u32 WriteEnable : 1;
-	u32 ByteCount :7;
-
-	u32 Reserved3 : 3;
-	u32 Byte4Access : 1;
-
-	u32 Byte2Access : 1;
-	u32 Byte1Access : 1;
-	u32 BurstMode :1 ;
-	u32 FixOrContinuous : 1;
-
-	u32 Reserved4 : 16;
-
-	/* DW3 */
-	u32		BusAddress;
-
-	/* DW4 */
-	u32		Value;
-
-#endif
-
-};
-
-#define PlatformEFIOWrite1Byte(_a, _b, _c)		\
-	rtl8723au_write8(_a, _b, _c)
-#define PlatformEFIOWrite2Byte(_a, _b, _c)		\
-	rtl8723au_write16(_a, _b, _c)
-#define PlatformEFIOWrite4Byte(_a, _b, _c)		\
-	rtl8723au_write32(_a, _b, _c)
-
-#define PlatformEFIORead1Byte(_a, _b)		rtl8723au_read8(_a, _b)
-#define PlatformEFIORead2Byte(_a, _b)		rtl8723au_read16(_a, _b)
-#define PlatformEFIORead4Byte(_a, _b)		rtl8723au_read32(_a, _b)
-
-#endif	/* _RTL8711_IO_H_ */
diff --git a/drivers/staging/rtl8723au/include/rtw_mlme.h b/drivers/staging/rtl8723au/include/rtw_mlme.h
deleted file mode 100644
index dbd3a5f5c523..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_mlme.h
+++ /dev/null
@@ -1,340 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- *
- ******************************************************************************/
-#ifndef __RTW_MLME_H_
-#define __RTW_MLME_H_
-
-#include <osdep_service.h>
-#include <mlme_osdep.h>
-#include <drv_types.h>
-#include <wlan_bssdef.h>
-
-#define	MAX_BSS_CNT	128
-#define   MAX_JOIN_TIMEOUT	6500
-
-/* Increase the scanning timeout because of increasing the SURVEY_TO value. */
-
-#define	SCANNING_TIMEOUT	8000
-
-#define	SCAN_INTERVAL	(30) /*  unit:2sec, 30*2 = 60sec */
-
-#define	SCANQUEUE_LIFETIME 20 /*  unit:sec */
-
-#define	WIFI_NULL_STATE		0x00000000
-
-#define	WIFI_ASOC_STATE		0x00000001 /*  Under Linked state.*/
-#define	WIFI_REASOC_STATE	0x00000002
-#define	WIFI_SLEEP_STATE	0x00000004
-#define	WIFI_STATION_STATE	0x00000008
-
-#define	WIFI_AP_STATE		0x00000010
-#define	WIFI_ADHOC_STATE	0x00000020
-#define   WIFI_ADHOC_MASTER_STATE	0x00000040
-#define   WIFI_UNDER_LINKING	0x00000080
-
-#define	WIFI_UNDER_WPS		0x00000100
-#define	WIFI_STA_ALIVE_CHK_STATE	0x00000400
-/* to indicate the station is under site surveying */
-#define	WIFI_SITE_MONITOR	0x00000800
-
-#define	WIFI_MP_STATE		0x00010000
-#define	WIFI_MP_CTX_BACKGROUND	0x00020000	/*  in continuous tx background */
-#define	WIFI_MP_CTX_ST		0x00040000	/*  in continuous tx with single-tone */
-#define	WIFI_MP_CTX_BACKGROUND_PENDING	0x00080000	/*  pending in continuous tx background due to out of skb */
-#define	WIFI_MP_CTX_CCK_HW	0x00100000	/*  in continuous tx */
-#define	WIFI_MP_CTX_CCK_CS	0x00200000	/*  in continuous tx with carrier suppression */
-#define   WIFI_MP_LPBK_STATE	0x00400000
-
-#define _FW_UNDER_LINKING	WIFI_UNDER_LINKING
-#define _FW_LINKED		WIFI_ASOC_STATE
-#define _FW_UNDER_SURVEY	WIFI_SITE_MONITOR
-
-
-enum dot11AuthAlgrthmNum {
-	dot11AuthAlgrthm_Open = 0,
-	dot11AuthAlgrthm_Shared,
-	dot11AuthAlgrthm_8021X,
-	dot11AuthAlgrthm_Auto,
-	dot11AuthAlgrthm_MaxNum
-};
-
-/*  Scan type including active and passive scan. */
-enum rt_scan_type {
-	SCAN_PASSIVE,
-	SCAN_ACTIVE,
-	SCAN_MIX,
-};
-
-enum {
-	GHZ24_50 = 0,
-	GHZ_50,
-	GHZ_24,
-};
-
-/*
-
-there are several "locks" in mlme_priv,
-since mlme_priv is a shared resource between many threads,
-like ISR/Call-Back functions, the OID handlers, and even timer functions.
-
-
-Each _queue has its own locks, already.
-Other items are protected by mlme_priv.lock.
-
-To avoid possible dead lock, any thread trying to modifiying mlme_priv
-SHALL not lock up more than one locks at a time!
-*/
-
-struct rt_link_detect {
-	u32	NumTxOkInPeriod;
-	u32	NumRxOkInPeriod;
-	u32	NumRxUnicastOkInPeriod;
-	bool	bBusyTraffic;
-	bool	bTxBusyTraffic;
-	bool	bRxBusyTraffic;
-	bool	bHigherBusyTraffic; /*  For interrupt migration purpose. */
-	bool	bHigherBusyRxTraffic; /*  We may disable Tx interrupt according as Rx traffic. */
-	bool	bHigherBusyTxTraffic; /*  We may disable Tx interrupt according as Tx traffic. */
-};
-
-struct mlme_priv {
-	spinlock_t	lock;
-	int	fw_state;
-	u8 bScanInProcess;
-	u8	to_join; /* flag */
-	u8 to_roaming; /*  roaming trying times */
-
-	struct rtw_adapter *nic_hdl;
-
-	u8	not_indic_disco;
-	struct rtw_queue	scanned_queue;
-
-	struct cfg80211_ssid assoc_ssid;
-	u8	assoc_bssid[6];
-
-	struct wlan_network	cur_network;
-
-	/* uint wireless_mode; no used, remove it */
-
-	u32	scan_interval;
-
-	struct timer_list assoc_timer;
-
-	uint assoc_by_bssid;
-	uint assoc_by_rssi;
-
-	struct timer_list scan_to_timer;
-
-	struct timer_list set_scan_deny_timer;
-	atomic_t set_scan_deny; /* 0: allowed, 1: deny */
-
-	unsigned int qos_option;
-
-	/* Number of non-HT AP/stations */
-	int num_sta_no_ht;
-
-	int num_FortyMHzIntolerant;
-
-	struct ht_priv	htpriv;
-
-	struct rt_link_detect LinkDetectInfo;
-	struct timer_list dynamic_chk_timer; /* dynamic/periodic check timer */
-
-	u8	key_mask; /* use for ips to set wep key after ips_leave23a */
-	u8	acm_mask; /*  for wmm acm mask */
-	u8	ChannelPlan;
-	enum rt_scan_type scan_mode; /*  active: 1, passive: 0 */
-
-	u8 *wps_probe_req_ie;
-	u32 wps_probe_req_ie_len;
-	u8 *assoc_req;
-	u32 assoc_req_len;
-	u32 assoc_rsp_len;
-	u8 *assoc_rsp;
-
-#ifdef CONFIG_8723AU_AP_MODE
-	/* Number of associated Non-ERP stations (i.e., stations using 802.11b
-	 * in 802.11g BSS) */
-	int num_sta_non_erp;
-
-	/* Number of associated stations that do not support Short Slot Time */
-	int num_sta_no_short_slot_time;
-
-	/* Number of associated stations that do not support Short Preamble */
-	int num_sta_no_short_preamble;
-
-	int olbc; /* Overlapping Legacy BSS Condition */
-
-	/* Number of HT associated stations that do not support greenfield */
-	int num_sta_ht_no_gf;
-
-	/* Number of associated non-HT stations */
-	/* int num_sta_no_ht; */
-
-	/* Number of HT associated stations 20 MHz */
-	int num_sta_ht_20mhz;
-
-	/* Overlapping BSS information */
-	int olbc_ht;
-
-	u16 ht_op_mode;
-
-	spinlock_t	bcn_update_lock;
-	u8		update_bcn;
-
-#endif /* ifdef CONFIG_8723AU_AP_MODE */
-};
-
-void rtw_joinbss_event_prehandle23a(struct rtw_adapter *adapter, u8 *pbuf);
-void rtw_survey_event_cb23a(struct rtw_adapter *adapter, const u8 *pbuf);
-void rtw_surveydone_event_callback23a(struct rtw_adapter *adapter, const u8 *pbuf);
-void rtw23a_joinbss_event_cb(struct rtw_adapter *adapter, const u8 *pbuf);
-void rtw_stassoc_event_callback23a(struct rtw_adapter *adapter, const u8 *pbuf);
-void rtw_stadel_event_callback23a(struct rtw_adapter *adapter, const u8 *pbuf);
-
-int event_thread(void *context);
-void rtw23a_join_to_handler(unsigned long);
-
-void rtw_free_network_queue23a(struct rtw_adapter *adapter);
-int rtw_init_mlme_priv23a(struct rtw_adapter *adapter);
-
-void rtw_free_mlme_priv23a(struct mlme_priv *pmlmepriv);
-
-int rtw_do_join_adhoc(struct rtw_adapter *adapter);
-int rtw_do_join_network(struct rtw_adapter *adapter,
-			struct wlan_network *candidate);
-int rtw_select_and_join_from_scanned_queue23a(struct mlme_priv *pmlmepriv);
-int rtw_set_key23a(struct rtw_adapter *adapter,
-		struct security_priv *psecuritypriv, int keyid, u8 set_tx);
-int rtw_set_auth23a(struct rtw_adapter *adapter,
-		 struct security_priv *psecuritypriv);
-
-static inline u8 *get_bssid(struct mlme_priv *pmlmepriv)
-{	/* if sta_mode:pmlmepriv->cur_network.network.MacAddress => bssid */
-	/*  if adhoc_mode:pmlmepriv->cur_network.network.MacAddress => ibss mac address */
-	return pmlmepriv->cur_network.network.MacAddress;
-}
-
-static inline bool check_fwstate(struct mlme_priv *pmlmepriv, int state)
-{
-	if (pmlmepriv->fw_state & state)
-		return true;
-
-	return false;
-}
-
-static inline int get_fwstate(struct mlme_priv *pmlmepriv)
-{
-	return pmlmepriv->fw_state;
-}
-
-/*
- * No Limit on the calling context,
- * therefore set it to be the critical section...
- *
- * ### NOTE:#### (!!!!)
- * MUST TAKE CARE THAT BEFORE CALLING THIS FUNC, YOU SHOULD HAVE LOCKED pmlmepriv->lock
- */
-static inline void set_fwstate(struct mlme_priv *pmlmepriv, int state)
-{
-	pmlmepriv->fw_state |= state;
-	/* FOR HW integration */
-	if (_FW_UNDER_SURVEY == state)
-		pmlmepriv->bScanInProcess = true;
-}
-
-static inline void _clr_fwstate_(struct mlme_priv *pmlmepriv, int state)
-{
-	pmlmepriv->fw_state &= ~state;
-	/* FOR HW integration */
-	if (_FW_UNDER_SURVEY == state)
-		pmlmepriv->bScanInProcess = false;
-}
-
-/*
- * No Limit on the calling context,
- * therefore set it to be the critical section...
- */
-static inline void clr_fwstate(struct mlme_priv *pmlmepriv, int state)
-{
-	spin_lock_bh(&pmlmepriv->lock);
-	if (check_fwstate(pmlmepriv, state))
-		pmlmepriv->fw_state ^= state;
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
-static inline void clr_fwstate_ex(struct mlme_priv *pmlmepriv, int state)
-{
-	spin_lock_bh(&pmlmepriv->lock);
-	_clr_fwstate_(pmlmepriv, state);
-	spin_unlock_bh(&pmlmepriv->lock);
-}
-
-void rtw_disconnect_hdl23a_under_linked(struct rtw_adapter *adapter,
-				     struct sta_info *psta, u8 free_assoc);
-void rtw_generate_random_ibss23a(u8 *pibss);
-struct wlan_network *rtw_find_network23a(struct rtw_queue *scanned_queue, u8 *addr);
-struct wlan_network *rtw_get_oldest_wlan_network23a(struct rtw_queue *scanned_queue);
-
-void rtw_free_assoc_resources23a(struct rtw_adapter *adapter,
-			      int lock_scanned_queue);
-void rtw_indicate_disconnect23a(struct rtw_adapter *adapter);
-void rtw_indicate_connect23a(struct rtw_adapter *adapter);
-void rtw_scan_abort23a(struct rtw_adapter *adapter);
-
-int rtw_restruct_sec_ie23a(struct rtw_adapter *adapter, u8 *in_ie, u8 *out_ie,
-			uint in_len);
-int rtw_restruct_wmm_ie23a(struct rtw_adapter *adapter, u8 *in_ie, u8 *out_ie,
-			uint in_len, uint initial_out_len);
-void rtw_init_registrypriv_dev_network23a(struct rtw_adapter *adapter);
-
-void rtw_update_registrypriv_dev_network23a(struct rtw_adapter *adapter);
-
-void rtw_scan_timeout_handler23a(unsigned long data);
-
-void rtw_dynamic_check_timer_handler(unsigned long data);
-bool rtw_is_scan_deny(struct rtw_adapter *adapter);
-void rtw_clear_scan_deny(struct rtw_adapter *adapter);
-void rtw_set_scan_deny_timer_hdl(unsigned long data);
-void rtw_set_scan_deny(struct rtw_adapter *adapter, u32 ms);
-
-void rtw23a_free_mlme_priv_ie_data(struct mlme_priv *pmlmepriv);
-
-void _rtw_free_mlme_priv23a(struct mlme_priv *pmlmepriv);
-
-struct wlan_network *rtw_alloc_network(struct mlme_priv *pmlmepriv, gfp_t gfp);
-
-int rtw_if_up23a(struct rtw_adapter *padapter);
-
-int rtw_linked_check(struct rtw_adapter *padapter);
-
-void rtw_joinbss_reset23a(struct rtw_adapter *padapter);
-
-bool rtw_restructure_ht_ie23a(struct rtw_adapter *padapter, u8 *in_ie,
-			      u8 *out_ie, uint in_len, uint *pout_len);
-void rtw_update_ht_cap23a(struct rtw_adapter *padapter,
-		       u8 *pie, uint ie_len);
-void rtw_issue_addbareq_cmd23a(struct rtw_adapter *padapter,
-			    struct xmit_frame *pxmitframe);
-
-bool rtw_is_same_ibss23a(struct rtw_adapter *adapter,
-			 struct wlan_network *pnetwork);
-int is_same_network23a(struct wlan_bssid_ex *src, struct wlan_bssid_ex *dst);
-
-void rtw23a_roaming(struct rtw_adapter *adapter,
-		 struct wlan_network *tgt_network);
-void rtw_set_roaming(struct rtw_adapter *adapter, u8 to_roaming);
-
-#endif /* __RTL871X_MLME_H_ */
diff --git a/drivers/staging/rtl8723au/include/rtw_mlme_ext.h b/drivers/staging/rtl8723au/include/rtw_mlme_ext.h
deleted file mode 100644
index 0e7d3da91471..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_mlme_ext.h
+++ /dev/null
@@ -1,683 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTW_MLME_EXT_H_
-#define __RTW_MLME_EXT_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <wlan_bssdef.h>
-
-
-/*	Commented by Albert 20101105 */
-/*	Increase the SURVEY_TO value from 100 to 150  ( 100ms to 150ms ) */
-/*	The Realtek 8188CE SoftAP will spend around 100ms to send the probe response after receiving the probe request. */
-/*	So, this driver tried to extend the dwell time for each scanning channel. */
-/*	This will increase the chance to receive the probe response from SoftAP. */
-
-#define SURVEY_TO		(100)
-#define REAUTH_TO		(300) /* 50) */
-#define REASSOC_TO		(300) /* 50) */
-/* define DISCONNECT_TO	(3000) */
-#define ADDBA_TO			(2000)
-
-#define LINKED_TO (1) /* unit:2 sec, 1x2=2 sec */
-
-#define REAUTH_LIMIT	(4)
-#define REASSOC_LIMIT	(4)
-#define READDBA_LIMIT	(2)
-
-#define ROAMING_LIMIT	8
-
-#define	DYNAMIC_FUNC_DISABLE			(0x0)
-
-/*  ====== enum odm_ability ======== */
-/*  BB ODM section BIT 0-15 */
-#define	DYNAMIC_BB_DIG				BIT(0)
-#define	DYNAMIC_BB_RA_MASK			BIT(1)
-#define	DYNAMIC_BB_DYNAMIC_TXPWR	BIT(2)
-#define	DYNAMIC_BB_BB_FA_CNT			BIT(3)
-
-#define		DYNAMIC_BB_RSSI_MONITOR		BIT(4)
-#define		DYNAMIC_BB_CCK_PD			BIT(5)
-#define		DYNAMIC_BB_ANT_DIV			BIT(6)
-#define		DYNAMIC_BB_PWR_SAVE			BIT(7)
-#define		DYNAMIC_BB_PWR_TRAIN			BIT(8)
-#define		DYNAMIC_BB_RATE_ADAPTIVE		BIT(9)
-#define		DYNAMIC_BB_PATH_DIV			BIT(10)
-#define		DYNAMIC_BB_PSD				BIT(11)
-
-/*  MAC DM section BIT 16-23 */
-#define		DYNAMIC_MAC_struct edca_turboURBO		BIT(16)
-#define		DYNAMIC_MAC_EARLY_MODE		BIT(17)
-
-/*  RF ODM section BIT 24-31 */
-#define		DYNAMIC_RF_TX_PWR_TRACK		BIT(24)
-#define		DYNAMIC_RF_RX_GAIN_TRACK		BIT(25)
-#define		DYNAMIC_RF_CALIBRATION		BIT(26)
-
-#define		DYNAMIC_ALL_FUNC_ENABLE		0xFFFFFFF
-
-#define _HW_STATE_NOLINK_		0x00
-#define _HW_STATE_ADHOC_		0x01
-#define _HW_STATE_STATION_	0x02
-#define _HW_STATE_AP_			0x03
-
-
-#define		_1M_RATE_	0
-#define		_2M_RATE_	1
-#define		_5M_RATE_	2
-#define		_11M_RATE_	3
-#define		_6M_RATE_	4
-#define		_9M_RATE_	5
-#define		_12M_RATE_	6
-#define		_18M_RATE_	7
-#define		_24M_RATE_	8
-#define		_36M_RATE_	9
-#define		_48M_RATE_	10
-#define		_54M_RATE_	11
-
-
-extern unsigned char WMM_OUI23A[];
-extern unsigned char WPS_OUI23A[];
-extern unsigned char WFD_OUI23A[];
-extern unsigned char P2P_OUI23A[];
-
-extern unsigned char WMM_INFO_OUI23A[];
-extern unsigned char WMM_PARA_OUI23A[];
-
-
-/*  */
-/*  Channel Plan Type. */
-/*  Note: */
-/*	We just add new channel plan when the new channel plan is different from any of the following */
-/*	channel plan. */
-/*	If you just want to customize the actions(scan period or join actions) about one of the channel plan, */
-/*	customize them in struct rt_channel_info in the RT_CHANNEL_LIST. */
-/*  */
-enum  { /* _RT_CHANNEL_DOMAIN */
-	/*  old channel plan mapping ===== */
-	RT_CHANNEL_DOMAIN_FCC = 0x00,
-	RT_CHANNEL_DOMAIN_IC = 0x01,
-	RT_CHANNEL_DOMAIN_ETSI = 0x02,
-	RT_CHANNEL_DOMAIN_SPAIN = 0x03,
-	RT_CHANNEL_DOMAIN_FRANCE = 0x04,
-	RT_CHANNEL_DOMAIN_MKK = 0x05,
-	RT_CHANNEL_DOMAIN_MKK1 = 0x06,
-	RT_CHANNEL_DOMAIN_ISRAEL = 0x07,
-	RT_CHANNEL_DOMAIN_TELEC = 0x08,
-	RT_CHANNEL_DOMAIN_GLOBAL_DOAMIN = 0x09,
-	RT_CHANNEL_DOMAIN_WORLD_WIDE_13 = 0x0A,
-	RT_CHANNEL_DOMAIN_TAIWAN = 0x0B,
-	RT_CHANNEL_DOMAIN_CHINA = 0x0C,
-	RT_CHANNEL_DOMAIN_SINGAPORE_INDIA_MEXICO = 0x0D,
-	RT_CHANNEL_DOMAIN_KOREA = 0x0E,
-	RT_CHANNEL_DOMAIN_TURKEY = 0x0F,
-	RT_CHANNEL_DOMAIN_JAPAN = 0x10,
-	RT_CHANNEL_DOMAIN_FCC_NO_DFS = 0x11,
-	RT_CHANNEL_DOMAIN_JAPAN_NO_DFS = 0x12,
-	RT_CHANNEL_DOMAIN_WORLD_WIDE_5G = 0x13,
-	RT_CHANNEL_DOMAIN_TAIWAN_NO_DFS = 0x14,
-
-	/*  new channel plan mapping, (2GDOMAIN_5GDOMAIN) ===== */
-	RT_CHANNEL_DOMAIN_WORLD_NULL = 0x20,
-	RT_CHANNEL_DOMAIN_ETSI1_NULL = 0x21,
-	RT_CHANNEL_DOMAIN_FCC1_NULL = 0x22,
-	RT_CHANNEL_DOMAIN_MKK1_NULL = 0x23,
-	RT_CHANNEL_DOMAIN_ETSI2_NULL = 0x24,
-	RT_CHANNEL_DOMAIN_FCC1_FCC1 = 0x25,
-	RT_CHANNEL_DOMAIN_WORLD_ETSI1 = 0x26,
-	RT_CHANNEL_DOMAIN_MKK1_MKK1 = 0x27,
-	RT_CHANNEL_DOMAIN_WORLD_KCC1 = 0x28,
-	RT_CHANNEL_DOMAIN_WORLD_FCC2 = 0x29,
-	RT_CHANNEL_DOMAIN_WORLD_FCC3 = 0x30,
-	RT_CHANNEL_DOMAIN_WORLD_FCC4 = 0x31,
-	RT_CHANNEL_DOMAIN_WORLD_FCC5 = 0x32,
-	RT_CHANNEL_DOMAIN_WORLD_FCC6 = 0x33,
-	RT_CHANNEL_DOMAIN_FCC1_FCC7 = 0x34,
-	RT_CHANNEL_DOMAIN_WORLD_ETSI2 = 0x35,
-	RT_CHANNEL_DOMAIN_WORLD_ETSI3 = 0x36,
-	RT_CHANNEL_DOMAIN_MKK1_MKK2 = 0x37,
-	RT_CHANNEL_DOMAIN_MKK1_MKK3 = 0x38,
-	RT_CHANNEL_DOMAIN_FCC1_NCC1 = 0x39,
-	RT_CHANNEL_DOMAIN_FCC1_NCC2 = 0x40,
-	RT_CHANNEL_DOMAIN_GLOBAL_DOAMIN_2G = 0x41,
-	/*  Add new channel plan above this line=============== */
-	RT_CHANNEL_DOMAIN_MAX,
-	RT_CHANNEL_DOMAIN_REALTEK_DEFINE = 0x7F,
-};
-
-enum { /* _RT_CHANNEL_DOMAIN_2G */
-	RT_CHANNEL_DOMAIN_2G_WORLD = 0x00,		/* Worldwird 13 */
-	RT_CHANNEL_DOMAIN_2G_ETSI1 = 0x01,		/* Europe */
-	RT_CHANNEL_DOMAIN_2G_FCC1 = 0x02,		/* US */
-	RT_CHANNEL_DOMAIN_2G_MKK1 = 0x03,		/* Japan */
-	RT_CHANNEL_DOMAIN_2G_ETSI2 = 0x04,		/* France */
-	RT_CHANNEL_DOMAIN_2G_NULL = 0x05,
-	/*  Add new channel plan above this line=============== */
-	RT_CHANNEL_DOMAIN_2G_MAX,
-};
-
-enum { /* _RT_CHANNEL_DOMAIN_5G */
-	RT_CHANNEL_DOMAIN_5G_NULL = 0x00,
-	RT_CHANNEL_DOMAIN_5G_ETSI1 = 0x01,		/* Europe */
-	RT_CHANNEL_DOMAIN_5G_ETSI2 = 0x02,		/* Australia, New Zealand */
-	RT_CHANNEL_DOMAIN_5G_ETSI3 = 0x03,		/* Russia */
-	RT_CHANNEL_DOMAIN_5G_FCC1 = 0x04,		/* US */
-	RT_CHANNEL_DOMAIN_5G_FCC2 = 0x05,		/* FCC o/w DFS Channels */
-	RT_CHANNEL_DOMAIN_5G_FCC3 = 0x06,		/* India, Mexico */
-	RT_CHANNEL_DOMAIN_5G_FCC4 = 0x07,		/* Venezuela */
-	RT_CHANNEL_DOMAIN_5G_FCC5 = 0x08,		/* China */
-	RT_CHANNEL_DOMAIN_5G_FCC6 = 0x09,		/* Israel */
-	RT_CHANNEL_DOMAIN_5G_FCC7_IC1 = 0x0A,	/* US, Canada */
-	RT_CHANNEL_DOMAIN_5G_KCC1 = 0x0B,		/* Korea */
-	RT_CHANNEL_DOMAIN_5G_MKK1 = 0x0C,		/* Japan */
-	RT_CHANNEL_DOMAIN_5G_MKK2 = 0x0D,		/* Japan (W52, W53) */
-	RT_CHANNEL_DOMAIN_5G_MKK3 = 0x0E,		/* Japan (W56) */
-	RT_CHANNEL_DOMAIN_5G_NCC1 = 0x0F,		/* Taiwan */
-	RT_CHANNEL_DOMAIN_5G_NCC2 = 0x10,		/* Taiwan o/w DFS */
-	/*  Add new channel plan above this line=============== */
-	/*  Driver Self Defined ===== */
-	RT_CHANNEL_DOMAIN_5G_FCC = 0x11,
-	RT_CHANNEL_DOMAIN_5G_JAPAN_NO_DFS = 0x12,
-	RT_CHANNEL_DOMAIN_5G_FCC4_NO_DFS = 0x13,
-	RT_CHANNEL_DOMAIN_5G_MAX,
-};
-
-#define rtw_is_channel_plan_valid(chplan) (chplan<RT_CHANNEL_DOMAIN_MAX || chplan == RT_CHANNEL_DOMAIN_REALTEK_DEFINE)
-
-struct rt_channel_plan {
-	unsigned char	Channel[MAX_CHANNEL_NUM];
-	unsigned char	Len;
-};
-
-struct rt_channel_plan_2g {
-	unsigned char	Channel[MAX_CHANNEL_NUM_2G];
-	unsigned char	Len;
-};
-
-struct rt_channel_plan_5g {
-	unsigned char	Channel[MAX_CHANNEL_NUM_5G];
-	unsigned char	Len;
-};
-
-struct rt_channel_plan_map {
-	unsigned char	Index2G;
-	unsigned char	Index5G;
-};
-
-enum Associated_AP {
-	atherosAP	= 0,
-	broadcomAP	= 1,
-	ciscoAP		= 2,
-	marvellAP	= 3,
-	ralinkAP	= 4,
-	realtekAP	= 5,
-	airgocapAP	= 6,
-	unknownAP	= 7,
-	maxAP,
-};
-
-enum { /* HT_IOT_PEER_E */
-	HT_IOT_PEER_UNKNOWN		= 0,
-	HT_IOT_PEER_REALTEK		= 1,
-	HT_IOT_PEER_REALTEK_92SE	= 2,
-	HT_IOT_PEER_BROADCOM		= 3,
-	HT_IOT_PEER_RALINK		= 4,
-	HT_IOT_PEER_ATHEROS		= 5,
-	HT_IOT_PEER_CISCO		= 6,
-	HT_IOT_PEER_MERU		= 7,
-	HT_IOT_PEER_MARVELL		= 8,
-	HT_IOT_PEER_REALTEK_SOFTAP	= 9,/*  peer is RealTek SOFT_AP, by Bohn, 2009.12.17 */
-	HT_IOT_PEER_SELF_SOFTAP		= 10, /*  Self is SoftAP */
-	HT_IOT_PEER_AIRGO		= 11,
-	HT_IOT_PEER_INTEL		= 12,
-	HT_IOT_PEER_RTK_APCLIENT	= 13,
-	HT_IOT_PEER_REALTEK_81XX	= 14,
-	HT_IOT_PEER_REALTEK_WOW		= 15,
-	HT_IOT_PEER_TENDA		= 16,
-	HT_IOT_PEER_MAX			= 17
-};
-
-enum SCAN_STATE {
-	SCAN_DISABLE = 0,
-	SCAN_START = 1,
-	SCAN_TXNULL = 2,
-	SCAN_PROCESS = 3,
-	SCAN_COMPLETE = 4,
-	SCAN_STATE_MAX,
-};
-
-struct mlme_handler {
-	char *str;
-	int (*func)(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-};
-
-struct action_handler {
-	unsigned int   num;
-	char *str;
-	int (*func)(struct rtw_adapter *padapter, struct recv_frame *precv_frame);
-};
-
-struct	ss_res {
-	int	state;
-	int	bss_cnt;
-	int	channel_idx;
-	int	scan_mode;
-	u8 ssid_num;
-	u8 ch_num;
-	struct cfg80211_ssid ssid[RTW_SSID_SCAN_AMOUNT];
-	struct rtw_ieee80211_channel ch[RTW_CHANNEL_SCAN_AMOUNT];
-};
-
-#define	WIFI_FW_AUTH_NULL			0x00000100
-#define	WIFI_FW_AUTH_STATE			0x00000200
-#define	WIFI_FW_AUTH_SUCCESS			0x00000400
-
-#define	WIFI_FW_ASSOC_STATE			0x00002000
-#define	WIFI_FW_ASSOC_SUCCESS		0x00004000
-
-#define	WIFI_FW_LINKING_STATE		(WIFI_FW_AUTH_NULL | WIFI_FW_AUTH_STATE | WIFI_FW_AUTH_SUCCESS |WIFI_FW_ASSOC_STATE)
-
-struct FW_Sta_Info {
-	struct sta_info	*psta;
-	u32	status;
-	u32	rx_pkt;
-	u32	retry;
-	unsigned char SupportedRates[NDIS_802_11_LENGTH_RATES_EX];
-};
-
-/*
- * Usage:
- * When one iface acted as AP mode and the other iface is STA mode and scanning,
- * it should switch back to AP's operating channel periodically.
- * Parameters info:
- * When the driver scanned RTW_SCAN_NUM_OF_CH channels, it would switch back to AP's operating channel for
- * RTW_STAY_AP_CH_MILLISECOND * SURVEY_TO milliseconds.
- * Example:
- * For chip supports 2.4G + 5GHz and AP mode is operating in channel 1,
- * RTW_SCAN_NUM_OF_CH is 8, RTW_STAY_AP_CH_MILLISECOND is 3 and SURVEY_TO is 100.
- * When it's STA mode gets set_scan command,
- * it would
- * 1. Doing the scan on channel 1.2.3.4.5.6.7.8
- * 2. Back to channel 1 for 300 milliseconds
- * 3. Go through doing site survey on channel 9.10.11.36.40.44.48.52
- * 4. Back to channel 1 for 300 milliseconds
- * 5. ... and so on, till survey done.
- */
-
-struct mlme_ext_info {
-	u32	state;
-	u32	reauth_count;
-	u32	reassoc_count;
-	u32	link_count;
-	u32	auth_seq;
-	u32	auth_algo;	/*  802.11 auth, could be open, shared, auto */
-	u32	authModeToggle;
-	u32	enc_algo;/* encrypt algorithm; */
-	u32	key_index;	/*  this is only valid for legendary wep, 0~3 for key id. */
-	u32	iv;
-	u8	chg_txt[128];
-	u16	aid;
-	u16	bcn_interval;
-	u16	capability;
-	u8	assoc_AP_vendor;
-	u8	slotTime;
-	u8	preamble_mode;
-	u8	WMM_enable;
-	u8	ERP_enable;
-	u8	ERP_IE;
-	u8	HT_enable;
-	u8	HT_caps_enable;
-	u8	HT_info_enable;
-	u8	HT_protection;
-	u8	turboMode_cts2self;
-	u8	turboMode_rtsen;
-	u8	SM_PS;
-	u8	ADDBA_retry_count;
-	u8	dialogToken;
-	/*  Accept ADDBA Request */
-	bool bAcceptAddbaReq;
-	u8	bwmode_updated;
-	u8	hidden_ssid_mode;
-
-	struct ADDBA_request		ADDBA_req;
-	struct WMM_para_element	WMM_param;
-	struct ieee80211_ht_cap ht_cap;
-	struct ieee80211_ht_operation HT_info;
-	struct wlan_bssid_ex			network;/* join network or bss_network, if in ap mode, it is the same to cur_network.network */
-	struct FW_Sta_Info		FW_sta_info[NUM_STA];
-};
-
-/*  The channel information about this channel including joining, scanning, and power constraints. */
-struct rt_channel_info {
-	u8		ChannelNum;		/*  The channel number. */
-	enum rt_scan_type ScanType;		/*  Scan type such as passive or active scan. */
-};
-
-int rtw_ch_set_search_ch23a(struct rt_channel_info *ch_set, const u32 ch);
-
-/*  P2P_MAX_REG_CLASSES - Maximum number of regulatory classes */
-#define P2P_MAX_REG_CLASSES 10
-
-/*  P2P_MAX_REG_CLASS_CHANNELS - Maximum number of channels per regulatory class */
-#define P2P_MAX_REG_CLASS_CHANNELS 20
-
-/*   struct p2p_channels - List of supported channels */
-struct p2p_channels {
-	/*  struct p2p_reg_class - Supported regulatory class */
-	struct p2p_reg_class {
-		/*  reg_class - Regulatory class (IEEE 802.11-2007, Annex J) */
-		u8 reg_class;
-
-		/*  channel - Supported channels */
-		u8 channel[P2P_MAX_REG_CLASS_CHANNELS];
-
-		/*  channels - Number of channel entries in use */
-		size_t channels;
-	} reg_class[P2P_MAX_REG_CLASSES];
-
-	/*  reg_classes - Number of reg_class entries in use */
-	size_t reg_classes;
-};
-
-struct p2p_oper_class_map {
-	enum hw_mode {IEEE80211G, IEEE80211A} mode;
-	u8 op_class;
-	u8 min_chan;
-	u8 max_chan;
-	u8 inc;
-	enum {
-		BW20, BW40PLUS, BW40MINUS
-	} bw;
-};
-
-struct mlme_ext_priv {
-	struct rtw_adapter	*padapter;
-	u8	mlmeext_init;
-	atomic_t		event_seq;
-	u16	mgnt_seq;
-
-	/* struct fw_priv	fwpriv; */
-
-	unsigned char	cur_channel;
-	unsigned char	cur_bwmode;
-	unsigned char	cur_ch_offset;/* PRIME_CHNL_OFFSET */
-	unsigned char	cur_wireless_mode;	/*  NETWORK_TYPE */
-
-	unsigned char	max_chan_nums;
-	struct rt_channel_info		channel_set[MAX_CHANNEL_NUM];
-	struct p2p_channels channel_list;
-	unsigned char	basicrate[NumRates];
-	unsigned char	datarate[NumRates];
-
-	struct ss_res		sitesurvey_res;
-	struct mlme_ext_info	mlmext_info;/* for sta/adhoc mode, including current scanning/connecting/connected related info. */
-                                                     /* for ap mode, network includes ap's cap_info */
-	struct timer_list		survey_timer;
-	struct timer_list		link_timer;
-	u16			chan_scan_time;
-
-	u8	scan_abort;
-	u8	tx_rate; /*  TXRATE when USERATE is set. */
-
-	u32	retry; /* retry for issue probereq */
-
-	u64 TSFValue;
-
-	unsigned char bstart_bss;
-	u8 update_channel_plan_by_ap_done;
-	/* recv_decache check for Action_public frame */
-	u8 action_public_dialog_token;
-	u16	 action_public_rxseq;
-	u8 active_keep_alive_check;
-};
-
-int init_mlme_ext_priv23a(struct rtw_adapter *padapter);
-int init_hw_mlme_ext23a(struct rtw_adapter *padapter);
-void free_mlme_ext_priv23a (struct mlme_ext_priv *pmlmeext);
-void init_mlme_ext_timer23a(struct rtw_adapter *padapter);
-void init_addba_retry_timer23a(struct sta_info *psta);
-struct xmit_frame *alloc_mgtxmitframe23a(struct xmit_priv *pxmitpriv);
-
-unsigned char networktype_to_raid23a(unsigned char network_type);
-u8 judge_network_type23a(struct rtw_adapter *padapter, unsigned char *rate,
-		      int ratelen);
-void get_rate_set23a(struct rtw_adapter *padapter, unsigned char *pbssrate,
-		  int *bssrate_len);
-void UpdateBrateTbl23a(struct rtw_adapter *padapter, u8 *mBratesOS);
-void Update23aTblForSoftAP(u8 *bssrateset, u32 bssratelen);
-
-u8 rtw_get_oper_ch23a(struct rtw_adapter *adapter);
-void rtw_set_oper_ch23a(struct rtw_adapter *adapter, u8 ch);
-void rtw_set_oper_bw23a(struct rtw_adapter *adapter, u8 bw);
-void rtw_set_oper_ch23aoffset23a(struct rtw_adapter *adapter, u8 offset);
-
-void set_channel_bwmode23a(struct rtw_adapter *padapter, unsigned char channel,
-			unsigned char channel_offset, unsigned short bwmode);
-void SelectChannel23a(struct rtw_adapter *padapter, unsigned char channel);
-
-unsigned int decide_wait_for_beacon_timeout23a(unsigned int bcn_interval);
-
-void clear_cam_entry23a(struct rtw_adapter *padapter, u8 entry);
-
-void invalidate_cam_all23a(struct rtw_adapter *padapter);
-
-int allocate_fw_sta_entry23a(struct rtw_adapter *padapter);
-void flush_all_cam_entry23a(struct rtw_adapter *padapter);
-
-bool IsLegal5GChannel(struct rtw_adapter *Adapter, u8 channel);
-
-void update_network23a(struct wlan_bssid_ex *dst, struct wlan_bssid_ex *src,
-		    struct rtw_adapter *padapter, bool update_ie);
-
-u8 *get_my_bssid23a(struct wlan_bssid_ex *pnetwork);
-
-bool is_client_associated_to_ap23a(struct rtw_adapter *padapter);
-bool is_client_associated_to_ibss23a(struct rtw_adapter *padapter);
-bool is_IBSS_empty23a(struct rtw_adapter *padapter);
-
-unsigned char check_assoc_AP23a(u8 *pframe, uint len);
-
-int WMM_param_handler23a(struct rtw_adapter *padapter, const u8 *p);
-void WMMOnAssocRsp23a(struct rtw_adapter *padapter);
-
-void HT_caps_handler23a(struct rtw_adapter *padapter, const u8 *p);
-void HT_info_handler23a(struct rtw_adapter *padapter, const u8 *p);
-void HTOnAssocRsp23a(struct rtw_adapter *padapter);
-
-void ERP_IE_handler23a(struct rtw_adapter *padapter, const u8 *p);
-void VCS_update23a(struct rtw_adapter *padapter, struct sta_info *psta);
-
-void update_beacon23a_info(struct rtw_adapter *padapter,
-			   struct ieee80211_mgmt *mgmt, uint len,
-			   struct sta_info *psta);
-int rtw_check_bcn_info23a(struct rtw_adapter *Adapter,
-			  struct ieee80211_mgmt *mgmt, u32 packet_len);
-void update_IOT_info23a(struct rtw_adapter *padapter);
-void update_capinfo23a(struct rtw_adapter *Adapter, u16 updateCap);
-void update_wireless_mode23a(struct rtw_adapter *padapter);
-void update_tx_basic_rate23a(struct rtw_adapter *padapter, u8 modulation);
-void update_bmc_sta_support_rate23a(struct rtw_adapter *padapter, u32 mac_id);
-int update_sta_support_rate23a(struct rtw_adapter *padapter, u8 *pvar_ie,
-			    uint var_ie_len, int cam_idx);
-
-/* for sta/adhoc mode */
-void update_sta_info23a(struct rtw_adapter *padapter, struct sta_info *psta);
-unsigned int update_basic_rate23a(unsigned char *ptn, unsigned int ptn_sz);
-unsigned int update_supported_rate23a(unsigned char *ptn, unsigned int ptn_sz);
-unsigned int update_MSC_rate23a(struct ieee80211_ht_cap *ht_cap);
-void Update_RA_Entry23a(struct rtw_adapter *padapter, struct sta_info *psta);
-void set_sta_rate23a(struct rtw_adapter *padapter, struct sta_info *psta);
-
-int receive_disconnect23a(struct rtw_adapter *padapter,
-			  unsigned char *MacAddr, unsigned short reason);
-
-unsigned char get_highest_rate_idx23a(u32 mask);
-int support_short_GI23a(struct rtw_adapter *padapter,
-			struct ieee80211_ht_cap *ht_cap);
-bool is_ap_in_tkip23a(struct rtw_adapter *padapter);
-bool is_ap_in_wep23a(struct rtw_adapter *padapter);
-bool should_forbid_n_rate23a(struct rtw_adapter *padapter);
-
-void report_join_res23a(struct rtw_adapter *padapter, int res);
-void report_survey_event23a(struct rtw_adapter *padapter,
-			 struct recv_frame *precv_frame);
-void report_surveydone_event23a(struct rtw_adapter *padapter);
-void report_del_sta_event23a(struct rtw_adapter *padapter,
-			  unsigned char *MacAddr, unsigned short reason);
-void report_add_sta_event23a(struct rtw_adapter *padapter,
-			  unsigned char *MacAddr, int cam_idx);
-
-int set_tx_beacon_cmd23a(struct rtw_adapter*padapter);
-unsigned int setup_beacon_frame(struct rtw_adapter *padapter,
-				unsigned char *beacon_frame);
-void update_mgnt_tx_rate23a(struct rtw_adapter *padapter, u8 rate);
-void update_mgntframe_attrib23a(struct rtw_adapter *padapter,
-			     struct pkt_attrib *pattrib);
-void dump_mgntframe23a(struct rtw_adapter *padapter,
-		    struct xmit_frame *pmgntframe);
-s32 dump_mgntframe23a_and_wait(struct rtw_adapter *padapter,
-			    struct xmit_frame *pmgntframe, int timeout_ms);
-s32 dump_mgntframe23a_and_wait_ack23a(struct rtw_adapter *padapter,
-				struct xmit_frame *pmgntframe);
-
-void issue_beacon23a(struct rtw_adapter *padapter, int timeout_ms);
-int issue_nulldata23a(struct rtw_adapter *padapter, unsigned char *da,
-		   unsigned int power_mode, int try_cnt, int wait_ms);
-int issue_qos_nulldata23a(struct rtw_adapter *padapter, unsigned char *da, u16 tid,
-		       int try_cnt, int wait_ms);
-int issue_deauth23a(struct rtw_adapter *padapter, unsigned char *da,
-		 unsigned short reason);
-void issue_action_spct_ch_switch23a(struct rtw_adapter *padapter, u8 *ra,
-				 u8 new_ch, u8 ch_offset);
-void issue_action_BA23a(struct rtw_adapter *padapter,
-			const unsigned char *raddr,
-			unsigned char action, unsigned short status);
-int send_delba23a(struct rtw_adapter *padapter, u8 initiator, u8 *addr);
-int send_beacon23a(struct rtw_adapter *padapter);
-
-void mlmeext_joinbss_event_callback23a(struct rtw_adapter *padapter, int join_res);
-void mlmeext_sta_del_event_callback23a(struct rtw_adapter *padapter);
-void mlmeext_sta_add_event_callback23a(struct rtw_adapter *padapter, struct sta_info *psta);
-
-void linked_status_chk23a(struct rtw_adapter *padapter);
-
-#define set_survey_timer(mlmeext, ms) \
-	/*DBG_8723A("%s set_survey_timer(%p, %d)\n", __func__, (mlmeext), (ms));*/ \
-	mod_timer(&mlmeext->survey_timer, jiffies + msecs_to_jiffies(ms));
-
-#define set_link_timer(mlmeext, ms) \
-	/*DBG_8723A("%s set_link_timer(%p, %d)\n", __func__, (mlmeext), (ms));*/ \
-	mod_timer(&mlmeext->link_timer, jiffies + msecs_to_jiffies(ms));
-
-int cckrates_included23a(unsigned char *rate, int ratelen);
-int cckratesonly_included23a(unsigned char *rate, int ratelen);
-
-void process_addba_req23a(struct rtw_adapter *padapter, u8 *paddba_req, u8 *addr);
-
-void correct_TSF23a(struct rtw_adapter *padapter, struct mlme_ext_priv *pmlmeext);
-
-struct cmd_hdl {
-	uint	parmsize;
-	int (*h2cfuns)(struct rtw_adapter *padapter, const u8 *pbuf);
-};
-
-
-int read_macreg_hdl(struct rtw_adapter *padapter, u8 *pbuf);
-int write_macreg_hdl(struct rtw_adapter *padapter, u8 *pbuf);
-int read_bbreg_hdl(struct rtw_adapter *padapter, u8 *pbuf);
-int write_bbreg_hdl(struct rtw_adapter *padapter, u8 *pbuf);
-int read_rfreg_hdl(struct rtw_adapter *padapter, u8 *pbuf);
-int write_rfreg_hdl(struct rtw_adapter *padapter, u8 *pbuf);
-
-
-int NULL_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int join_cmd_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int disconnect_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int createbss_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int setopmode_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int sitesurvey_cmd_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int setauth_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int setkey_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int set_stakey_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int set_assocsta_hdl(struct rtw_adapter *padapter, const u8 *pbuf);
-int del_assocsta_hdl(struct rtw_adapter *padapter, const u8 *pbuf);
-int add_ba_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-
-int mlme_evt_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int h2c_msg_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int tx_beacon_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int set_ch_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int set_chplan_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int led_blink_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-int set_csa_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);	/* Kurt: Handling DFS channel switch announcement ie. */
-int tdls_hdl23a(struct rtw_adapter *padapter, const u8 *pbuf);
-
-#define GEN_DRV_CMD_HANDLER(size, cmd)	{size, &cmd ## _hdl23a},
-#define GEN_MLME_EXT_HANDLER(size, cmd)	{size, cmd},
-
-struct C2HEvent_Header {
-#ifdef __LITTLE_ENDIAN
-
-	unsigned int len:16;
-	unsigned int ID:8;
-	unsigned int seq:8;
-
-#elif defined(__BIG_ENDIAN)
-
-	unsigned int seq:8;
-	unsigned int ID:8;
-	unsigned int len:16;
-
-#else
-
-#  error "Must be LITTLE or BIG Endian"
-
-#endif
-
-	unsigned int rsvd;
-};
-
-enum rtw_c2h_event {
-	GEN_EVT_CODE(_Read_MACREG) = 0, /*0*/
-	GEN_EVT_CODE(_Read_BBREG),
-	GEN_EVT_CODE(_Read_RFREG),
-	GEN_EVT_CODE(_Read_EEPROM),
-	GEN_EVT_CODE(_Read_EFUSE),
-	GEN_EVT_CODE(_Read_CAM),			/*5*/
-	GEN_EVT_CODE(_Get_BasicRate),
-	GEN_EVT_CODE(_Get_DataRate),
-	GEN_EVT_CODE(_Survey),	 /*8*/
-	GEN_EVT_CODE(_SurveyDone),	 /*9*/
-
-	GEN_EVT_CODE(_JoinBss) , /*10*/
-	GEN_EVT_CODE(_AddSTA),
-	GEN_EVT_CODE(_DelSTA),
-	GEN_EVT_CODE(_AtimDone) ,
-	GEN_EVT_CODE(_TX_Report),
-	GEN_EVT_CODE(_CCX_Report),			/*15*/
-	GEN_EVT_CODE(_DTM_Report),
-	GEN_EVT_CODE(_TX_Rate_Statistics),
-	GEN_EVT_CODE(_C2HLBK),
-	GEN_EVT_CODE(_FWDBG),
-	GEN_EVT_CODE(_C2HFEEDBACK),               /*20*/
-	GEN_EVT_CODE(_ADDBA),
-	GEN_EVT_CODE(_C2HBCN),
-	GEN_EVT_CODE(_ReportPwrState),		/* filen: only for PCIE, USB */
-	GEN_EVT_CODE(_CloseRF),				/* filen: only for PCIE, work around ASPM */
-	MAX_C2HEVT
-};
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtw_pwrctrl.h b/drivers/staging/rtl8723au/include/rtw_pwrctrl.h
deleted file mode 100644
index 599fed9b365d..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_pwrctrl.h
+++ /dev/null
@@ -1,241 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTW_PWRCTRL_H_
-#define __RTW_PWRCTRL_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#define FW_PWR0		0
-#define FW_PWR1		1
-#define FW_PWR2		2
-#define FW_PWR3		3
-
-
-#define HW_PWR0		7
-#define HW_PWR1		6
-#define HW_PWR2		2
-#define HW_PWR3		0
-#define HW_PWR4		8
-
-#define FW_PWRMSK	0x7
-
-
-#define XMIT_ALIVE	BIT(0)
-#define RECV_ALIVE	BIT(1)
-#define CMD_ALIVE	BIT(2)
-#define EVT_ALIVE	BIT(3)
-
-enum Power_Mgnt {
-	PS_MODE_ACTIVE	= 0,
-	PS_MODE_MIN,
-	PS_MODE_MAX,
-	PS_MODE_DTIM,
-	PS_MODE_VOIP,
-	PS_MODE_UAPSD_WMM,
-	PS_MODE_UAPSD,
-	PS_MODE_IBSS,
-	PS_MODE_WWLAN,
-	PM_Radio_Off,
-	PM_Card_Disable,
-	PS_MODE_NUM
-};
-
-
-/* BIT[2:0] = HW state
- * BIT[3] = Protocol PS state,  0: active, 1: sleep state
- * BIT[4] = sub-state
- */
-
-#define PS_DPS			BIT(0)
-#define PS_LCLK			(PS_DPS)
-#define PS_RF_OFF		BIT(1)
-#define PS_ALL_ON		BIT(2)
-#define PS_ST_ACTIVE		BIT(3)
-
-#define PS_ISR_ENABLE		BIT(4)
-#define PS_IMR_ENABLE		BIT(5)
-#define PS_ACK			BIT(6)
-#define PS_TOGGLE		BIT(7)
-
-#define PS_STATE_MASK		(0x0F)
-#define PS_STATE_HW_MASK	(0x07)
-#define PS_SEQ_MASK		(0xc0)
-
-#define PS_STATE(x)		(PS_STATE_MASK & (x))
-#define PS_STATE_HW(x)	(PS_STATE_HW_MASK & (x))
-#define PS_SEQ(x)		(PS_SEQ_MASK & (x))
-
-#define PS_STATE_S0		(PS_DPS)
-#define PS_STATE_S1		(PS_LCLK)
-#define PS_STATE_S2		(PS_RF_OFF)
-#define PS_STATE_S3		(PS_ALL_ON)
-#define PS_STATE_S4		((PS_ST_ACTIVE) | (PS_ALL_ON))
-
-
-#define PS_IS_RF_ON(x)	((x) & (PS_ALL_ON))
-#define PS_IS_ACTIVE(x)	((x) & (PS_ST_ACTIVE))
-#define CLR_PS_STATE(x)	((x) = ((x) & (0xF0)))
-
-
-struct reportpwrstate_parm {
-	unsigned char mode;
-	unsigned char state; /* the CPWM value */
-	unsigned short rsvd;
-};
-
-#define LPS_DELAY_TIME	(1*HZ) /*  1 sec */
-
-#define EXE_PWR_NONE		0x01
-#define EXE_PWR_IPS		0x02
-#define EXE_PWR_LPS		0x04
-
-/*  RF state. */
-enum rt_rf_power_state {
-	rf_on,		/*  RF is on after RFSleep or RFOff */
-	rf_sleep,	/*  802.11 Power Save mode */
-	rf_off,		/*  HW/SW Radio OFF or Inactive Power Save */
-	/* Add the new RF state above this line===== */
-	rf_max
-};
-
-/*  RF Off Level for IPS or HW/SW radio off */
-#define	RT_RF_OFF_LEVL_ASPM		BIT(0)	/*  PCI ASPM */
-#define	RT_RF_OFF_LEVL_CLK_REQ		BIT(1)	/*  PCI clock request */
-#define	RT_RF_OFF_LEVL_PCI_D3		BIT(2)	/*  PCI D3 mode */
-/* NIC halt, re-init hw params */
-#define	RT_RF_OFF_LEVL_HALT_NIC		BIT(3)
-/*  FW free, re-download the FW */
-#define	RT_RF_OFF_LEVL_FREE_FW		BIT(4)
-#define	RT_RF_OFF_LEVL_FW_32K		BIT(5)	/*  FW in 32k */
-/*  Always enable ASPM and Clock Req in initialization. */
-#define	RT_RF_PS_LEVEL_ALWAYS_ASPM	BIT(6)
-/*  When LPS is on, disable 2R if no packet is received or transmittd. */
-#define	RT_RF_LPS_DISALBE_2R		BIT(30)
-#define	RT_RF_LPS_LEVEL_ASPM		BIT(31)	/*  LPS with ASPM */
-
-#define	RT_IN_PS_LEVEL(ppsc, _PS_FLAG)				\
-	((ppsc->cur_ps_level & _PS_FLAG) ? true : false)
-#define	RT_CLEAR_PS_LEVEL(ppsc, _PS_FLAG)			\
-	(ppsc->cur_ps_level &= (~(_PS_FLAG)))
-#define	RT_SET_PS_LEVEL(ppsc, _PS_FLAG)				\
-	(ppsc->cur_ps_level |= _PS_FLAG)
-
-
-enum {
-	PSBBREG_RF0 = 0,
-	PSBBREG_RF1,
-	PSBBREG_RF2,
-	PSBBREG_AFE0,
-	PSBBREG_TOTALCNT
-};
-
-enum { /*  for ips_mode */
-	IPS_NONE = 0,
-	IPS_NORMAL,
-	IPS_LEVEL_2,
-};
-
-struct pwrctrl_priv {
-	struct semaphore lock;
-	volatile u8 rpwm; /* requested power state for fw */
-	volatile u8 cpwm; /* fw current power state. updated when 1.
-			   * read from HCPWM 2. driver lowers power level
-			   */
-	volatile u8 tog; /*  toggling */
-
-	u8	pwr_mode;
-	u8	smart_ps;
-	u8	bcn_ant_mode;
-
-	u8	bpower_saving;
-
-	u8	reg_rfoff;
-	u32	rfoff_reason;
-
-	/* RF OFF Level */
-	u32	cur_ps_level;
-	u32	reg_rfps_level;
-
-	uint	ips_enter23a_cnts;
-	uint	ips_leave23a_cnts;
-
-	u8	ips_mode;
-	u8	ips_mode_req; /*  used to accept the mode setting request */
-	uint bips_processing;
-	unsigned long ips_deny_time; /* deny IPS when system time is smaller */
-	u8 ps_processing; /* used to mark whether in rtw_ps_processor23a */
-
-	u8	bLeisurePs;
-	u8	LpsIdleCount;
-	u8	power_mgnt;
-	u8	bFwCurrentInPSMode;
-	unsigned long	DelayLPSLastTimeStamp;
-	u8	btcoex_rfon;
-
-	u8		bInSuspend;
-#ifdef	CONFIG_8723AU_BT_COEXIST
-	u8		bAutoResume;
-	u8		autopm_cnt;
-#endif
-	u8		bSupportRemoteWakeup;
-	struct timer_list	pwr_state_check_timer;
-	int		pwr_state_check_interval;
-	u8		pwr_state_check_cnts;
-
-	enum rt_rf_power_state	rf_pwrstate;/* cur power state */
-	enum rt_rf_power_state	change_rfpwrstate;
-
-	u8	bkeepfwalive;
-	unsigned long PS_BBRegBackup[PSBBREG_TOTALCNT];
-};
-
-#define RTW_PWR_STATE_CHK_INTERVAL 2000
-
-#define _rtw_set_pwr_state_check_timer(pwrctrlpriv, ms) \
-	(mod_timer(&pwrctrlpriv->pwr_state_check_timer, jiffies +	\
-		  msecs_to_jiffies(ms)))
-
-#define rtw_set_pwr_state_check_timer(pwrctrlpriv)	\
-	(_rtw_set_pwr_state_check_timer((pwrctrlpriv),	\
-			 (pwrctrlpriv)->pwr_state_check_interval))
-
-void rtw_init_pwrctrl_priv23a(struct rtw_adapter *adapter);
-void rtw_free_pwrctrl_priv(struct rtw_adapter *adapter);
-
-void rtw_set_ps_mode23a(struct rtw_adapter *padapter, u8 ps_mode,
-		     u8 smart_ps, u8 bcn_ant_mode);
-void rtw_set_rpwm23a(struct rtw_adapter *padapter, u8 val8);
-void LeaveAllPowerSaveMode23a(struct rtw_adapter *adapter);
-void ips_enter23a(struct rtw_adapter *padapter);
-int ips_leave23a(struct rtw_adapter *padapter);
-
-void rtw_ps_processor23a(struct rtw_adapter *padapter);
-
-enum rt_rf_power_state RfOnOffDetect23a(struct rtw_adapter *adapter);
-
-s32 LPS_RF_ON_check23a(struct rtw_adapter *padapter, u32 delay_ms);
-void LPS_Enter23a(struct rtw_adapter *padapter);
-void LPS_Leave23a(struct rtw_adapter *padapter);
-
-void rtw_set_ips_deny23a(struct rtw_adapter *padapter, u32 ms);
-int _rtw_pwr_wakeup23a(struct rtw_adapter *padapter, u32 ips_deffer_ms,
-		    const char *caller);
-#define rtw_pwr_wakeup(adapter) _rtw_pwr_wakeup23a(adapter,		\
-	 RTW_PWR_STATE_CHK_INTERVAL, __func__)
-int rtw_pm_set_ips23a(struct rtw_adapter *padapter, u8 mode);
-int rtw_pm_set_lps23a(struct rtw_adapter *padapter, u8 mode);
-
-#endif  /* __RTL871X_PWRCTRL_H_ */
diff --git a/drivers/staging/rtl8723au/include/rtw_recv.h b/drivers/staging/rtl8723au/include/rtw_recv.h
deleted file mode 100644
index 85a5edb450e3..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_recv.h
+++ /dev/null
@@ -1,305 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef _RTW_RECV_H_
-#define _RTW_RECV_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <Hal8723APhyCfg.h>
-
-#define NR_RECVFRAME		256
-
-#define MAX_RXFRAME_CNT		512
-#define MAX_RX_NUMBLKS		(32)
-#define RECVFRAME_HDR_ALIGN	128
-
-#define SNAP_SIZE sizeof(struct ieee80211_snap_hdr)
-
-#define MAX_SUBFRAME_COUNT	64
-
-/* for Rx reordering buffer control */
-struct recv_reorder_ctrl {
-	struct rtw_adapter	*padapter;
-	u8 enable;
-	u16 indicate_seq;/* wstart_b, init_value=0xffff */
-	u16 wend_b;
-	u8 wsize_b;
-	struct rtw_queue pending_recvframe_queue;
-	struct timer_list reordering_ctrl_timer;
-};
-
-struct	stainfo_rxcache	{
-	u16	tid_rxseq[16];
-/*
-	unsigned short	tid0_rxseq;
-	unsigned short	tid1_rxseq;
-	unsigned short	tid2_rxseq;
-	unsigned short	tid3_rxseq;
-	unsigned short	tid4_rxseq;
-	unsigned short	tid5_rxseq;
-	unsigned short	tid6_rxseq;
-	unsigned short	tid7_rxseq;
-	unsigned short	tid8_rxseq;
-	unsigned short	tid9_rxseq;
-	unsigned short	tid10_rxseq;
-	unsigned short	tid11_rxseq;
-	unsigned short	tid12_rxseq;
-	unsigned short	tid13_rxseq;
-	unsigned short	tid14_rxseq;
-	unsigned short	tid15_rxseq;
-*/
-};
-
-struct smooth_rssi_data {
-	u32	elements[100];	/* array to store values */
-	u32	index;			/* index to current array to store */
-	u32	total_num;		/* num of valid elements */
-	u32	total_val;		/* sum of valid elements */
-};
-
-struct signal_stat {
-	u8	update_req;		/* used to indicate */
-	u8	avg_val;		/* avg of valid elements */
-	u32	total_num;		/* num of valid elements */
-	u32	total_val;		/* sum of valid elements */
-};
-
-struct phy_info {
-	u8		RxPWDBAll;
-	u8		SignalQuality;	 /*  in 0-100 index. */
-	u8		RxMIMOSignalQuality[RF_PATH_MAX]; /* EVM */
-	u8		RxMIMOSignalStrength[RF_PATH_MAX];/* 0~100 */
-	s8		RxPower; /*  in dBm Translate from PWdB */
-	/* Real power in dBm for this packet, no beautification and aggregation.
-	 * Keep this raw info to be used for the other procedures.
-	 */
-	s8		RecvSignalPower;
-	u8		BTRxRSSIPercentage;
-	u8		SignalStrength; /*  in 0-100 index. */
-	u8		RxPwr[RF_PATH_MAX];/* per-path's pwdb */
-	u8		RxSNR[RF_PATH_MAX];/* per-path's SNR */
-};
-
-
-struct rx_pkt_attrib	{
-	u16	pkt_len;
-	u8	physt;
-	u8	drvinfo_sz;
-	u8	shift_sz;
-	u8	hdrlen; /* the WLAN Header Len */
-	u8	amsdu;
-	u8	qos;
-	u8	priority;
-	u8	pw_save;
-	u8	mdata;
-	u16	seq_num;
-	u8	frag_num;
-	u8	mfrag;
-	u8	order;
-	u8	privacy; /* in frame_ctrl field */
-	u8	bdecrypted;
-	/* when 0 indicate no encrypt. when non-zero, indicate the algorith */
-	u32	encrypt;
-	u8	iv_len;
-	u8	icv_len;
-	u8	crc_err;
-	u8	icv_err;
-
-	u16 eth_type;
-
-	u8	dst[ETH_ALEN];
-	u8	src[ETH_ALEN];
-	u8	ta[ETH_ALEN];
-	u8	ra[ETH_ALEN];
-	u8	bssid[ETH_ALEN];
-
-	u8 ack_policy;
-
-	u8	tcpchk_valid; /*  0: invalid, 1: valid */
-	u8	ip_chkrpt; /* 0: incorrect, 1: correct */
-	u8	tcp_chkrpt; /* 0: incorrect, 1: correct */
-	u8	key_index;
-
-	u8	mcs_rate;
-	u8	rxht;
-	u8	sgi;
-	u8	pkt_rpt_type;
-	u32	MacIDValidEntry[2];	/*  64 bits present 64 entry. */
-	struct phy_info phy_info;
-};
-
-/* These definition is used for Rx packet reordering. */
-#define SN_LESS(a, b)		(((a-b) & 0x800) != 0)
-#define SN_EQUAL(a, b)		(a == b)
-#define REORDER_WAIT_TIME	(50) /*  (ms) */
-
-#define RECVBUFF_ALIGN_SZ 8
-
-#define RXDESC_SIZE	24
-#define RXDESC_OFFSET RXDESC_SIZE
-
-struct recv_stat {
-	__le32 rxdw0;
-	__le32 rxdw1;
-	__le32 rxdw2;
-	__le32 rxdw3;
-	__le32 rxdw4;
-	__le32 rxdw5;
-};
-
-/* accesser of recv_priv: rtw_recv_entry23a(dispatch / passive level);	\
- * recv_thread(passive) ; returnpkt(dispatch) ; halt(passive) ;
- *
- * using enter_critical section to protect
- */
-struct recv_priv {
-	spinlock_t	lock;
-
-	struct rtw_queue	free_recv_queue;
-	struct rtw_queue	recv_pending_queue;
-	struct rtw_queue	uc_swdec_pending_queue;
-
-	int free_recvframe_cnt;
-
-	struct rtw_adapter	*adapter;
-
-	u32	bIsAnyNonBEPkts;
-	u64	rx_bytes;
-	u64	rx_pkts;
-	u64	rx_drop;
-	u64	last_rx_bytes;
-
-	uint  rx_icv_err;
-	uint  rx_largepacket_crcerr;
-	uint  rx_smallpacket_crcerr;
-	uint  rx_middlepacket_crcerr;
-
-	/* u8 *pallocated_urb_buf; */
-	u8	rx_pending_cnt;
-
-	struct urb *int_in_urb;
-
-	u8	*int_in_buf;
-
-	struct tasklet_struct irq_prepare_beacon_tasklet;
-	struct tasklet_struct recv_tasklet;
-	struct sk_buff_head free_recv_skb_queue;
-	struct sk_buff_head rx_skb_queue;
-	u8 *precv_buf;
-
-	/* For display the phy informatiom */
-	s8 rxpwdb;
-	u8 signal_strength;
-	u8 signal_qual;
-	u8 noise;
-	int RxSNRdB[2];
-	s8 RxRssi[2];
-	int FalseAlmCnt_all;
-
-	struct timer_list signal_stat_timer;
-	u32 signal_stat_sampling_interval;
-	/* u32 signal_stat_converging_constant; */
-	struct signal_stat signal_qual_data;
-	struct signal_stat signal_strength_data;
-};
-
-#define rtw_set_signal_stat_timer(recvpriv)			\
-	 mod_timer(&(recvpriv)->signal_stat_timer, jiffies +	\
-		   msecs_to_jiffies((recvpriv)->signal_stat_sampling_interval))
-
-struct sta_recv_priv {
-	spinlock_t	lock;
-	int	option;
-
-	/* struct rtw_queue	blk_strms[MAX_RX_NUMBLKS]; */
-	struct rtw_queue defrag_q;	 /* keeping the fragment frame until defrag */
-
-	struct	stainfo_rxcache rxcache;
-
-	/* uint	sta_rx_bytes; */
-	/* uint	sta_rx_pkts; */
-	/* uint	sta_rx_fail; */
-
-};
-
-
-struct recv_buf {
-	struct list_head list;
-
-	struct rtw_adapter *adapter;
-
-	struct urb *purb;
-	struct sk_buff *pskb;
-};
-
-/*	head  ----->
- *
- *		data  ----->
- *
- *			payload
- *
- *		tail  ----->
- *
- *	end   ----->
- *
- *	len = (unsigned int )(tail - data);
- *
- */
-struct recv_frame {
-	struct list_head	list;
-	struct sk_buff *pkt;
-
-	struct rtw_adapter  *adapter;
-
-	struct rx_pkt_attrib attrib;
-
-	struct sta_info *psta;
-
-	/* for A-MPDU Rx reordering buffer control */
-	struct recv_reorder_ctrl *preorder_ctrl;
-};
-
-/* get a free recv_frame from pfree_recv_queue */
-struct recv_frame *rtw_alloc_recvframe23a(struct rtw_queue *pfree_recv_queue);
-int rtw_free_recvframe23a(struct recv_frame *precvframe);
-
-int rtw_enqueue_recvframe23a(struct recv_frame *precvframe, struct rtw_queue *queue);
-
-u32 rtw_free_uc_swdec_pending_queue23a(struct rtw_adapter *adapter);
-
-struct recv_buf *rtw_dequeue_recvbuf23a(struct rtw_queue *queue);
-
-void rtw_reordering_ctrl_timeout_handler23a(unsigned long pcontext);
-
-static inline s32 translate_percentage_to_dbm(u32 SignalStrengthIndex)
-{
-	s32	SignalPower; /*  in dBm. */
-
-	/*  Translate to dBm (x=0.5y-95). */
-	SignalPower = (s32)((SignalStrengthIndex + 1) >> 1);
-	SignalPower -= 95;
-
-	return SignalPower;
-}
-
-
-struct sta_info;
-
-void _rtw_init_sta_recv_priv23a(struct sta_recv_priv *psta_recvpriv);
-
-void mgt_dispatcher23a(struct rtw_adapter *padapter,
-		    struct recv_frame *precv_frame);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtw_rf.h b/drivers/staging/rtl8723au/include/rtw_rf.h
deleted file mode 100644
index a7de714137b8..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_rf.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef	__RTW_RF_H_
-#define __RTW_RF_H_
-
-#include <rtw_cmd.h>
-
-#define OFDM_PHY		1
-#define MIXED_PHY		2
-#define CCK_PHY			3
-
-#define NumRates		(13)
-
-/*  slot time for 11g */
-#define SHORT_SLOT_TIME			9
-#define NON_SHORT_SLOT_TIME		20
-
-/*  We now define the max channels in each channel plan. */
-#define	MAX_CHANNEL_NUM_2G	14
-#define	MAX_CHANNEL_NUM_5G	24
-#define	MAX_CHANNEL_NUM		38/* 14+24 */
-
-/* define NUM_REGULATORYS	21 */
-#define NUM_REGULATORYS	1
-
-/* Country codes */
-#define USA		0x555320
-#define EUROPE		0x1 /* temp, should be provided later */
-#define JAPAN		0x2 /* temp, should be provided later */
-
-struct	regulatory_class {
-	u32	starting_freq;					/* MHz, */
-	u8	channel_set[MAX_CHANNEL_NUM];
-	u8	channel_cck_power[MAX_CHANNEL_NUM];/* dbm */
-	u8	channel_ofdm_power[MAX_CHANNEL_NUM];/* dbm */
-	u8	txpower_limit;					/* dbm */
-	u8	channel_spacing;				/* MHz */
-	u8	modem;
-};
-
-enum {
-	cESS		= 0x0001,
-	cIBSS		= 0x0002,
-	cPollable	= 0x0004,
-	cPollReq	= 0x0008,
-	cPrivacy	= 0x0010,
-	cShortPreamble	= 0x0020,
-	cPBCC		= 0x0040,
-	cChannelAgility	= 0x0080,
-	cSpectrumMgnt	= 0x0100,
-	cQos		= 0x0200,	/*  For HCCA, use with CF-Pollable and CF-PollReq */
-	cShortSlotTime	= 0x0400,
-	cAPSD		= 0x0800,
-	cRM		= 0x1000,	/*  RRM (Radio Request Measurement) */
-	cDSSS_OFDM	= 0x2000,
-	cDelayedBA	= 0x4000,
-	cImmediateBA	= 0x8000,
-};
-
-enum {
-	PREAMBLE_LONG	= 1,
-	PREAMBLE_AUTO	= 2,
-	PREAMBLE_SHORT	= 3,
-};
-
-/*  Bandwidth Offset */
-#define HAL_PRIME_CHNL_OFFSET_DONT_CARE	0
-#define HAL_PRIME_CHNL_OFFSET_LOWER	1
-#define HAL_PRIME_CHNL_OFFSET_UPPER	2
-
-/*  Represent Channel Width in HT Capabilities */
-enum ht_channel_width {
-	HT_CHANNEL_WIDTH_20 = 0,
-	HT_CHANNEL_WIDTH_40 = 1,
-	HT_CHANNEL_WIDTH_80 = 2,
-	HT_CHANNEL_WIDTH_160 = 3,
-	HT_CHANNEL_WIDTH_10 = 4,
-};
-
-/* 2007/11/15 MH Define different RF type. */
-enum {
-	RF_1T2R = 0,
-	RF_2T4R = 1,
-	RF_2T2R = 2,
-	RF_1T1R = 3,
-	RF_2T2R_GREEN = 4,
-	RF_819X_MAX_TYPE = 5,
-};
-
-#endif /* _RTL8711_RF_H_ */
diff --git a/drivers/staging/rtl8723au/include/rtw_security.h b/drivers/staging/rtl8723au/include/rtw_security.h
deleted file mode 100644
index 624a9d788e45..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_security.h
+++ /dev/null
@@ -1,331 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __RTW_SECURITY_H_
-#define __RTW_SECURITY_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <net/lib80211.h>
-
-
-#define is_wep_enc(alg) (alg == WLAN_CIPHER_SUITE_WEP40 || \
-			 alg == WLAN_CIPHER_SUITE_WEP104)
-
-#define SHA256_MAC_LEN 32
-#define AES_BLOCK_SIZE 16
-#define AES_PRIV_SIZE (4 * 44)
-
-enum ENCRYP_PROTOCOL {
-	ENCRYP_PROTOCOL_OPENSYS,   /* open system */
-	ENCRYP_PROTOCOL_WEP,       /* WEP */
-	ENCRYP_PROTOCOL_WPA,       /* WPA */
-	ENCRYP_PROTOCOL_WPA2,      /* WPA2 */
-	ENCRYP_PROTOCOL_MAX
-};
-
-#ifndef Ndis802_11AuthModeWPA2
-#define Ndis802_11AuthModeWPA2 (Ndis802_11AuthModeWPANone + 1)
-#endif
-
-#ifndef Ndis802_11AuthModeWPA2PSK
-#define Ndis802_11AuthModeWPA2PSK (Ndis802_11AuthModeWPANone + 2)
-#endif
-
-union pn48 {
-	u64	val;
-
-#ifdef __LITTLE_ENDIAN
-
-struct {
-	u8 TSC0;
-	u8 TSC1;
-	u8 TSC2;
-	u8 TSC3;
-	u8 TSC4;
-	u8 TSC5;
-	u8 TSC6;
-	u8 TSC7;
-} _byte_;
-
-#elif defined(__BIG_ENDIAN)
-
-struct {
-	u8 TSC7;
-	u8 TSC6;
-	u8 TSC5;
-	u8 TSC4;
-	u8 TSC3;
-	u8 TSC2;
-	u8 TSC1;
-	u8 TSC0;
-} _byte_;
-#else
-#error Need BIG or LITTLE endian
-
-#endif
-
-};
-
-union Keytype {
-	u8   skey[16];
-	u32    lkey[4];
-};
-
-struct rtw_wep_key {
-	u8 key[WLAN_KEY_LEN_WEP104 + 1]; /* 14 */
-	u16 keylen;
-};
-
-struct rt_pmkid_list {
-	u8	bUsed;
-	u8	Bssid[6];
-	u8	PMKID[16];
-	u8	SsidBuf[33];
-	u8	*ssid_octet;
-	u16	ssid_length;
-};
-
-struct security_priv {
-	u32	  dot11AuthAlgrthm;	/*  802.11 auth, could be open, shared,
-					 * 8021x and authswitch */
-	u32	  dot11PrivacyAlgrthm;	/* This specifies the privacy for
-					 * shared auth. algorithm.
-					 */
-	/* WEP */
-	u32	  dot11PrivacyKeyIndex;	/*  this is only valid for legendary
-					 * wep, 0~3 for key id. (tx key index)
-					 */
-	struct rtw_wep_key wep_key[NUM_WEP_KEYS];
-
-	u32 dot118021XGrpPrivacy;	/* specify the privacy algthm.
-					 * used for Grp key
-					 */
-	u32	dot118021XGrpKeyid;	/*  key id used for Grp Key
-					 * (tx key index)
-					 */
-	union Keytype	dot118021XGrpKey[4];/* 802.1x Grp Key, inx0 and inx1 */
-	union Keytype	dot118021XGrptxmickey[4];
-	union Keytype	dot118021XGrprxmickey[4];
-	union pn48	dot11Grptxpn;		/* PN48 used for Grp Key xmit.*/
-	union pn48	dot11Grprxpn;		/* PN48 used for Grp Key recv.*/
-
-#ifdef CONFIG_8723AU_AP_MODE
-	/* extend security capabilities for AP_MODE */
-	unsigned int dot8021xalg;/* 0:disable, 1:psk, 2:802.1x */
-	unsigned int wpa_psk;/* 0:disable, bit(0): WPA, bit(1):WPA2 */
-	unsigned int wpa_group_cipher;
-	unsigned int wpa2_group_cipher;
-	unsigned int wpa_pairwise_cipher;
-	unsigned int wpa2_pairwise_cipher;
-#endif
-
-	u8 wps_ie[MAX_WPS_IE_LEN];/* added in assoc req */
-	int wps_ie_len;
-	unsigned int binstallGrpkey:1;
-	unsigned int busetkipkey:1;
-	unsigned int bcheck_grpkey:1;
-	unsigned int hw_decrypted:1;
-	u32 ndisauthtype;	/*  enum ndis_802_11_auth_mode */
-	u32 ndisencryptstatus;	/*  NDIS_802_11_ENCRYPTION_STATUS */
-	struct wlan_bssid_ex sec_bss;  /* for joinbss (h2c buffer) usage */
-	u8 assoc_info[600];
-	u8 szofcapability[256]; /* for wpa2 usage */
-	u8 oidassociation[512]; /* for wpa/wpa2 usage */
-	u8 supplicant_ie[256];  /* store sta security information element */
-
-	/* for tkip countermeasure */
-	unsigned long last_mic_err_time;
-	u8	btkip_countermeasure;
-	u8	btkip_wait_report;
-	unsigned long btkip_countermeasure_time;
-
-	/*  For WPA2 Pre-Authentication. */
-	struct rt_pmkid_list PMKIDList[NUM_PMKID_CACHE];
-	u8 PMKIDIndex;
-	u8 bWepDefaultKeyIdxSet;
-};
-
-struct sha256_state {
-	u64 length;
-	u32 state[8], curlen;
-	u8 buf[64];
-};
-
-#define GET_ENCRY_ALGO(psecuritypriv, psta, encry_algo, bmcst)\
-do {\
-	switch (psecuritypriv->dot11AuthAlgrthm) {\
-	case dot11AuthAlgrthm_Open:\
-	case dot11AuthAlgrthm_Shared:\
-	case dot11AuthAlgrthm_Auto:\
-		encry_algo = psecuritypriv->dot11PrivacyAlgrthm;\
-		break;\
-	case dot11AuthAlgrthm_8021X:\
-		if (bmcst)\
-			encry_algo = psecuritypriv->dot118021XGrpPrivacy;\
-		else\
-			encry_algo = psta->dot118021XPrivacy;\
-		break;\
-	}	\
-} while (0)
-
-#define GET_TKIP_PN(iv, dot11txpn)\
-do {\
-	dot11txpn._byte_.TSC0 = iv[2];\
-	dot11txpn._byte_.TSC1 = iv[0];\
-	dot11txpn._byte_.TSC2 = iv[4];\
-	dot11txpn._byte_.TSC3 = iv[5];\
-	dot11txpn._byte_.TSC4 = iv[6];\
-	dot11txpn._byte_.TSC5 = iv[7];\
-} while (0)
-
-#define ROL32(A, n)  (((A) << (n)) | (((A)>>(32-(n)))  & ((1UL << (n)) - 1)))
-#define ROR32(A, n)  ROL32((A), 32-(n))
-
-struct mic_data {
-	u32  K0, K1;         /*  Key */
-	u32  L, R;           /*  Current state */
-	u32  M;              /*  Message accumulator (single word) */
-	u32     nBytesInM;      /*  # bytes in M */
-};
-
-extern const u32 Te0[256];
-extern const u32 Te1[256];
-extern const u32 Te2[256];
-extern const u32 Te3[256];
-extern const u32 Te4[256];
-extern const u32 Td0[256];
-extern const u32 Td1[256];
-extern const u32 Td2[256];
-extern const u32 Td3[256];
-extern const u32 Td4[256];
-extern const u32 rcon[10];
-extern const u8 Td4s[256];
-extern const u8 rcons[10];
-
-#define RCON(i) (rcons[(i)] << 24)
-
-static inline u32 rotr(u32 val, int bits)
-{
-	return (val >> bits) | (val << (32 - bits));
-}
-
-#define TE0(i) Te0[((i) >> 24) & 0xff]
-#define TE1(i) rotr(Te0[((i) >> 16) & 0xff], 8)
-#define TE2(i) rotr(Te0[((i) >> 8) & 0xff], 16)
-#define TE3(i) rotr(Te0[(i) & 0xff], 24)
-#define TE41(i) ((Te0[((i) >> 24) & 0xff] << 8) & 0xff000000)
-#define TE42(i) (Te0[((i) >> 16) & 0xff] & 0x00ff0000)
-#define TE43(i) (Te0[((i) >> 8) & 0xff] & 0x0000ff00)
-#define TE44(i) ((Te0[(i) & 0xff] >> 8) & 0x000000ff)
-#define TE421(i) ((Te0[((i) >> 16) & 0xff] << 8) & 0xff000000)
-#define TE432(i) (Te0[((i) >> 8) & 0xff] & 0x00ff0000)
-#define TE443(i) (Te0[(i) & 0xff] & 0x0000ff00)
-#define TE414(i) ((Te0[((i) >> 24) & 0xff] >> 8) & 0x000000ff)
-#define TE4(i) ((Te0[(i)] >> 8) & 0x000000ff)
-
-#define TD0(i) Td0[((i) >> 24) & 0xff]
-#define TD1(i) rotr(Td0[((i) >> 16) & 0xff], 8)
-#define TD2(i) rotr(Td0[((i) >> 8) & 0xff], 16)
-#define TD3(i) rotr(Td0[(i) & 0xff], 24)
-#define TD41(i) (Td4s[((i) >> 24) & 0xff] << 24)
-#define TD42(i) (Td4s[((i) >> 16) & 0xff] << 16)
-#define TD43(i) (Td4s[((i) >> 8) & 0xff] << 8)
-#define TD44(i) (Td4s[(i) & 0xff])
-#define TD0_(i) Td0[(i) & 0xff]
-#define TD1_(i) rotr(Td0[(i) & 0xff], 8)
-#define TD2_(i) rotr(Td0[(i) & 0xff], 16)
-#define TD3_(i) rotr(Td0[(i) & 0xff], 24)
-
-#define GETU32(pt) (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ \
-			((u32)(pt)[2] <<  8) ^ ((u32)(pt)[3]))
-
-#define PUTU32(ct, st) { \
-(ct)[0] = (u8)((st) >> 24); (ct)[1] = (u8)((st) >> 16); \
-(ct)[2] = (u8)((st) >>  8); (ct)[3] = (u8)(st); }
-
-#define WPA_GET_BE32(a) ((((u32) (a)[0]) << 24) | (((u32) (a)[1]) << 16) | \
-			 (((u32) (a)[2]) << 8) | ((u32) (a)[3]))
-
-#define WPA_PUT_LE16(a, val)			\
-	do {					\
-		(a)[1] = ((u16) (val)) >> 8;	\
-		(a)[0] = ((u16) (val)) & 0xff;	\
-	} while (0)
-
-#define WPA_PUT_BE32(a, val)					\
-	do {							\
-		(a)[0] = (u8) ((((u32) (val)) >> 24) & 0xff);	\
-		(a)[1] = (u8) ((((u32) (val)) >> 16) & 0xff);	\
-		(a)[2] = (u8) ((((u32) (val)) >> 8) & 0xff);	\
-		(a)[3] = (u8) (((u32) (val)) & 0xff);		\
-	} while (0)
-
-#define WPA_PUT_BE64(a, val)				\
-	do {						\
-		(a)[0] = (u8) (((u64) (val)) >> 56);	\
-		(a)[1] = (u8) (((u64) (val)) >> 48);	\
-		(a)[2] = (u8) (((u64) (val)) >> 40);	\
-		(a)[3] = (u8) (((u64) (val)) >> 32);	\
-		(a)[4] = (u8) (((u64) (val)) >> 24);	\
-		(a)[5] = (u8) (((u64) (val)) >> 16);	\
-		(a)[6] = (u8) (((u64) (val)) >> 8);	\
-		(a)[7] = (u8) (((u64) (val)) & 0xff);	\
-	} while (0)
-
-/* ===== start - public domain SHA256 implementation ===== */
-
-/* This is based on SHA256 implementation in LibTomCrypt that was released into
- * public domain by Tom St Denis. */
-
-/* the K array */
-static const unsigned long K[64] = {
-	0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL,
-	0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL,
-	0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL,
-	0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL,
-	0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL,
-	0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL,
-	0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL,
-	0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL,
-	0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL,
-	0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL,
-	0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL,
-	0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL,
-	0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL
-};
-
-void rtw_secmicsetkey23a(struct mic_data *pmicdata, u8 *key);
-void rtw_secmicappend23abyte23a(struct mic_data *pmicdata, u8 b);
-void rtw_secmicappend23a(struct mic_data *pmicdata, u8 *src, u32 nbBytes);
-void rtw_secgetmic23a(struct mic_data *pmicdata, u8 *dst);
-
-void rtw_seccalctkipmic23a(u8 *key, u8 *header, u8 *data, u32 data_len,
-			u8 *Miccode, u8 priorityi);
-
-int rtw_aes_encrypt23a(struct rtw_adapter *padapter,
-		    struct xmit_frame *pxmitframe);
-int rtw_tkip_encrypt23a(struct rtw_adapter *padapter,
-		     struct xmit_frame *pxmitframe);
-void rtw_wep_encrypt23a(struct rtw_adapter *padapter,
-		     struct xmit_frame *pxmitframe);
-int rtw_aes_decrypt23a(struct rtw_adapter *padapter,
-		    struct recv_frame *precvframe);
-int rtw_tkip_decrypt23a(struct rtw_adapter *padapter,
-		     struct recv_frame *precvframe);
-void rtw_wep_decrypt23a(struct rtw_adapter *padapter, struct recv_frame *precvframe);
-
-void rtw_use_tkipkey_handler23a(void *FunctionContext);
-
-#endif	/* __RTL871X_SECURITY_H_ */
diff --git a/drivers/staging/rtl8723au/include/rtw_sreset.h b/drivers/staging/rtl8723au/include/rtw_sreset.h
deleted file mode 100644
index 60fa8296e1ff..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_sreset.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef _RTW_SRESET_C_
-#define _RTW_SRESET_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-struct sreset_priv {
-	struct mutex	silentreset_mutex;
-	u8	silent_reset_inprogress;
-	unsigned long last_tx_time;
-	unsigned long last_tx_complete_time;
-};
-
-#include <rtl8723a_hal.h>
-
-void rtw_sreset_init(struct rtw_adapter *padapter);
-void rtw_sreset_reset_value(struct rtw_adapter *padapter);
-bool rtw_sreset_inprogress(struct rtw_adapter *padapter);
-void sreset_set_trigger_point(struct rtw_adapter *padapter, s32 tgp);
-void rtw_sreset_reset(struct rtw_adapter *active_adapter);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/rtw_version.h b/drivers/staging/rtl8723au/include/rtw_version.h
deleted file mode 100644
index c947733a3e3e..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_version.h
+++ /dev/null
@@ -1 +0,0 @@
-#define DRIVERVERSION	"v4.1.6_7336.20130426"
diff --git a/drivers/staging/rtl8723au/include/rtw_xmit.h b/drivers/staging/rtl8723au/include/rtw_xmit.h
deleted file mode 100644
index 2b7d6d08238b..000000000000
--- a/drivers/staging/rtl8723au/include/rtw_xmit.h
+++ /dev/null
@@ -1,385 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef _RTW_XMIT_H_
-#define _RTW_XMIT_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#define MAX_XMITBUF_SZ		2048
-#define NR_XMITBUFF		4
-
-#define XMITBUF_ALIGN_SZ	512
-
-/*  xmit extension buff defination */
-#define MAX_XMIT_EXTBUF_SZ	1536
-#define NR_XMIT_EXTBUFF		32
-
-#define MAX_NUMBLKS		1
-
-#define XMIT_VO_QUEUE		0
-#define XMIT_VI_QUEUE		1
-#define XMIT_BE_QUEUE		2
-#define XMIT_BK_QUEUE		3
-
-#define VO_QUEUE_INX		0
-#define VI_QUEUE_INX		1
-#define BE_QUEUE_INX		2
-#define BK_QUEUE_INX		3
-#define BCN_QUEUE_INX		4
-#define MGT_QUEUE_INX		5
-#define HIGH_QUEUE_INX		6
-#define TXCMD_QUEUE_INX		7
-
-#define HW_QUEUE_ENTRY		8
-
-#define WEP_IV(pattrib_iv, dot11txpn, keyidx)				\
-do {									\
-	pattrib_iv[0] = dot11txpn._byte_.TSC0;				\
-	pattrib_iv[1] = dot11txpn._byte_.TSC1;				\
-	pattrib_iv[2] = dot11txpn._byte_.TSC2;				\
-	pattrib_iv[3] = ((keyidx & 0x3) << 6);				\
-	dot11txpn.val = (dot11txpn.val == 0xffffff) ? 0 :		\
-			 (dot11txpn.val+1);				\
-} while (0)
-
-#define TKIP_IV(pattrib_iv, dot11txpn, keyidx)				\
-do {									\
-	pattrib_iv[0] = dot11txpn._byte_.TSC1;				\
-	pattrib_iv[1] = (dot11txpn._byte_.TSC1 | 0x20) & 0x7f;		\
-	pattrib_iv[2] = dot11txpn._byte_.TSC0;				\
-	pattrib_iv[3] = BIT(5) | ((keyidx & 0x3)<<6);			\
-	pattrib_iv[4] = dot11txpn._byte_.TSC2;				\
-	pattrib_iv[5] = dot11txpn._byte_.TSC3;				\
-	pattrib_iv[6] = dot11txpn._byte_.TSC4;				\
-	pattrib_iv[7] = dot11txpn._byte_.TSC5;				\
-	dot11txpn.val = dot11txpn.val == 0xffffffffffffULL ? 0 :	\
-					 (dot11txpn.val+1);		\
-} while (0)
-
-#define AES_IV(pattrib_iv, dot11txpn, keyidx)\
-do {									\
-	pattrib_iv[0] = dot11txpn._byte_.TSC0;				\
-	pattrib_iv[1] = dot11txpn._byte_.TSC1;				\
-	pattrib_iv[2] = 0;						\
-	pattrib_iv[3] = BIT(5) | ((keyidx & 0x3) << 6);			\
-	pattrib_iv[4] = dot11txpn._byte_.TSC2;				\
-	pattrib_iv[5] = dot11txpn._byte_.TSC3;				\
-	pattrib_iv[6] = dot11txpn._byte_.TSC4;				\
-	pattrib_iv[7] = dot11txpn._byte_.TSC5;				\
-	dot11txpn.val = dot11txpn.val == 0xffffffffffffULL ? 0 :	\
-					 (dot11txpn.val+1);		\
-} while (0)
-
-#define HWXMIT_ENTRY	4
-
-#define TXDESC_SIZE 32
-
-#define PACKET_OFFSET_SZ 8
-#define TXDESC_OFFSET (TXDESC_SIZE + PACKET_OFFSET_SZ)
-
-struct tx_desc {
-	/* DWORD 0 */
-	__le32 txdw0;
-	__le32 txdw1;
-	__le32 txdw2;
-	__le32 txdw3;
-	__le32 txdw4;
-	__le32 txdw5;
-	__le32 txdw6;
-	__le32 txdw7;
-};
-
-union txdesc {
-	struct tx_desc txdesc;
-	unsigned int value[TXDESC_SIZE>>2];
-};
-
-struct	hw_xmit	{
-	struct rtw_queue *sta_queue;
-	int	accnt;
-};
-
-/* reduce size */
-struct pkt_attrib {
-	u16	type;
-	u8	bswenc;
-	u8	dhcp_pkt;
-	u16	ether_type;
-	u16	seqnum;
-	u16	pkt_hdrlen;	/* the original 802.3 pkt header len */
-	u16	hdrlen;		/* the WLAN Header Len */
-	u32	pktlen;		/* the original 802.3 pkt raw_data len */
-	u32	last_txcmdsz;
-	u32	encrypt;	/* when 0 indicate no encrypt. */
-	u8	nr_frags;
-	u8	iv_len;
-	u8	icv_len;
-	u8	iv[18];
-	u8	icv[16];
-	u8	priority;
-	u8	ack_policy;
-	u8	mac_id;
-	u8	vcs_mode;	/* virtual carrier sense method */
-	u8	dst[ETH_ALEN];
-	u8	src[ETH_ALEN];
-	u8	ta[ETH_ALEN];
-	u8	ra[ETH_ALEN];
-	u8	key_idx;
-	u8	qos_en;
-	u8	ht_en;
-	u8	raid;/* rate adpative id */
-	u8	bwmode;
-	u8	ch_offset;/* PRIME_CHNL_OFFSET */
-	u8	sgi;/* short GI */
-	u8	ampdu_en;/* tx ampdu enable */
-	u8	mdata;/* more data bit */
-	u8	pctrl;/* per packet txdesc control enable */
-	u8	triggered;/* for ap mode handling Power Saving sta */
-	u8	qsel;
-	u8	eosp;
-	u8	rate;
-	u8	retry_ctrl;
-	struct sta_info *psta;
-};
-
-#define WLANHDR_OFFSET		64
-
-#define NULL_FRAMETAG		0x0
-#define DATA_FRAMETAG		0x01
-#define L2_FRAMETAG		0x02
-#define MGNT_FRAMETAG		0x03
-#define AMSDU_FRAMETAG		0x04
-
-#define EII_FRAMETAG		0x05
-#define IEEE8023_FRAMETAG 	0x06
-
-#define MP_FRAMETAG		0x07
-
-#define TXAGG_FRAMETAG		0x08
-
-struct  submit_ctx {
-	u32 timeout_ms; /* <0: not synchronous, 0: wait forever,
-			 * >0: up to ms waiting
-			 */
-	int status; /* status for operation */
-	struct completion done;
-};
-
-enum {
-	RTW_SCTX_SUBMITTED = -1,
-	RTW_SCTX_DONE_SUCCESS = 0,
-	RTW_SCTX_DONE_UNKNOWN,
-	RTW_SCTX_DONE_TIMEOUT,
-	RTW_SCTX_DONE_BUF_ALLOC,
-	RTW_SCTX_DONE_BUF_FREE,
-	RTW_SCTX_DONE_WRITE_PORT_ERR,
-	RTW_SCTX_DONE_TX_DESC_NA,
-	RTW_SCTX_DONE_TX_DENY,
-	RTW_SCTX_DONE_CCX_PKT_FAIL,
-	RTW_SCTX_DONE_DRV_STOP,
-	RTW_SCTX_DONE_DEV_REMOVE,
-};
-
-void rtw_sctx_init23a(struct submit_ctx *sctx, int timeout_ms);
-int rtw_sctx_wait23a(struct submit_ctx *sctx);
-void rtw23a_sctx_done_err(struct submit_ctx **sctx, int status);
-
-struct xmit_buf {
-	struct list_head list, list2;
-	struct rtw_adapter *padapter;
-
-	u8 *pallocated_buf;
-	u8 *pbuf;
-	void *priv_data;
-
-	u16 ext_tag; /*  0: Normal xmitbuf, 1: extension xmitbuf. */
-	u16 flags;
-	u32 alloc_sz;
-	u32  len;
-	struct submit_ctx *sctx;
-	u32	ff_hwaddr;
-	struct urb *pxmit_urb[8];
-	u8 bpending[8];
-	int last[8];
-#if defined(DBG_XMIT_BUF) || defined(DBG_XMIT_BUF_EXT)
-	u8 no;
-#endif
-};
-
-struct xmit_frame {
-	struct list_head	list;
-	struct pkt_attrib attrib;
-	struct sk_buff *pkt;
-	int	frame_tag;
-	struct rtw_adapter *padapter;
-	u8	*buf_addr;
-	struct xmit_buf *pxmitbuf;
-
-	s8	pkt_offset;
-
-	u8 ack_report;
-
-	u8 ext_tag; /* 0:data, 1:mgmt */
-};
-
-struct tx_servq {
-	struct list_head	tx_pending;
-	struct rtw_queue	sta_pending;
-	int qcnt;
-};
-
-struct sta_xmit_priv {
-	spinlock_t	lock;
-	int	option;
-	int	apsd_setting;	/* When bit mask is on, the associated edca
-				 * queue supports APSD.
-				 */
-	struct tx_servq	be_q;			/* priority == 0,3 */
-	struct tx_servq	bk_q;			/* priority == 1,2 */
-	struct tx_servq	vi_q;			/* priority == 4,5 */
-	struct tx_servq	vo_q;			/* priority == 6,7 */
-	struct list_head legacy_dz;
-	struct list_head apsd;
-	u16 txseq_tid[16];
-};
-
-struct	hw_txqueue {
-	volatile int head;
-	volatile int tail;
-	volatile int free_sz;	/* in units of 64 bytes */
-	volatile int free_cmdsz;
-	volatile int txsz[8];
-	uint	ff_hwaddr;
-	uint	cmd_hwaddr;
-	int	ac_tag;
-};
-
-struct agg_pkt_info {
-	u16 offset;
-	u16 pkt_len;
-};
-
-struct	xmit_priv {
-	spinlock_t	lock;
-
-	struct semaphore	xmit_sema;
-	struct semaphore	terminate_xmitthread_sema;
-
-	struct rtw_queue	be_pending;
-	struct rtw_queue	bk_pending;
-	struct rtw_queue	vi_pending;
-	struct rtw_queue	vo_pending;
-	struct rtw_queue	bm_pending;
-
-	int free_xmitframe_cnt;
-	struct rtw_queue	free_xmit_queue;
-
-	int free_xframe_ext_cnt;
-	struct rtw_queue free_xframe_ext_queue;
-
-	uint	frag_len;
-
-	struct rtw_adapter	*adapter;
-
-	u64	tx_bytes;
-	u64	tx_pkts;
-	u64	tx_drop;
-	u64	last_tx_bytes;
-	u64	last_tx_pkts;
-
-	struct hw_xmit *hwxmits;
-	u8	hwxmit_entry;
-	u8	vcs;
-	u8	nqos_ssn;
-
-	u8	wmm_para_seq[4];/* sequence for wmm ac parameter strength from
-				 * large to small. it's value is 0->vo, 1->vi,
-				 * 2->be, 3->bk.
-				 */
-
-	struct semaphore	tx_retevt;/* all tx return event; */
-
-	struct tasklet_struct xmit_tasklet;
-
-	struct rtw_queue free_xmitbuf_queue;
-	struct list_head xmitbuf_list;		/* track buffers for cleanup */
-	struct rtw_queue pending_xmitbuf_queue;
-	uint free_xmitbuf_cnt;
-
-	struct rtw_queue free_xmit_extbuf_queue;
-	struct list_head xmitextbuf_list;	/* track buffers for cleanup */
-	uint free_xmit_extbuf_cnt;
-
-	int	ack_tx;
-	struct mutex ack_tx_mutex;
-	struct submit_ctx ack_tx_ops;
-	spinlock_t lock_sctx;
-};
-
-struct xmit_buf *rtw_alloc_xmitbuf23a_ext(struct xmit_priv *pxmitpriv);
-s32 rtw_free_xmitbuf_ext23a(struct xmit_priv *pxmitpriv,
-			 struct xmit_buf *pxmitbuf);
-
-struct xmit_buf *rtw_alloc_xmitbuf23a(struct xmit_priv *pxmitpriv);
-s32 rtw_free_xmitbuf23a(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
-
-void rtw_count_tx_stats23a(struct rtw_adapter *padapter,
-			struct xmit_frame *pxmitframe, int sz);
-void rtw_update_protection23a(struct rtw_adapter *padapter, u8 *ie, uint ie_len);
-struct xmit_frame *rtw_alloc_xmitframe23a_ext(struct xmit_priv *pxmitpriv);
-struct xmit_frame *rtw_alloc_xmitframe23a_once(struct xmit_priv *pxmitpriv);
-s32 rtw_free_xmitframe23a(struct xmit_priv *pxmitpriv,
-		       struct xmit_frame *pxmitframe);
-void rtw_free_xmitframe_queue23a(struct xmit_priv *pxmitpriv, struct rtw_queue *pframequeue);
-struct tx_servq *rtw_get_sta_pending23a(struct rtw_adapter *padapter,
-				     struct sta_info *psta, int up, u8 *ac);
-s32 rtw_xmitframe_enqueue23a(struct rtw_adapter *padapter,
-			     struct xmit_frame *pxmitframe);
-struct xmit_frame *rtw_dequeue_xframe23a(struct xmit_priv *pxmitpriv,
-				      struct hw_xmit *phwxmit_i, int entry);
-s32 rtw_xmit23a_classifier(struct rtw_adapter *padapter,
-			struct xmit_frame *pxmitframe);
-s32 rtw_xmitframe_coalesce23a(struct rtw_adapter *padapter, struct sk_buff *pkt,
-			      struct xmit_frame *pxmitframe);
-s32 _rtw_init_hw_txqueue(struct hw_txqueue *phw_txqueue, u8 ac_tag);
-void _rtw_init_sta_xmit_priv23a(struct sta_xmit_priv *psta_xmitpriv);
-
-s32 rtw_txframes_pending23a(struct rtw_adapter *padapter);
-s32 rtw_txframes_sta_ac_pending23a(struct rtw_adapter *padapter,
-				struct pkt_attrib *pattrib);
-void rtw_init_hwxmits23a(struct hw_xmit *phwxmit, int entry);
-int _rtw_init_xmit_priv23a(struct xmit_priv *pxmitpriv,
-			struct rtw_adapter *padapter);
-void _rtw_free_xmit_priv23a(struct xmit_priv *pxmitpriv);
-void rtw_alloc_hwxmits23a(struct rtw_adapter *padapter);
-void rtw_free_hwxmits23a(struct rtw_adapter *padapter);
-int rtw_xmit23a(struct rtw_adapter *padapter, struct sk_buff *pkt);
-#if defined(CONFIG_8723AU_AP_MODE)
-int xmitframe_enqueue_for_sleeping_sta23a(struct rtw_adapter *padapter,
-				       struct xmit_frame *pxmitframe);
-void stop_sta_xmit23a(struct rtw_adapter *padapter, struct sta_info *psta);
-void wakeup_sta_to_xmit23a(struct rtw_adapter *padapter, struct sta_info *psta);
-void xmit_delivery_enabled_frames23a(struct rtw_adapter *padapter,
-				  struct sta_info *psta);
-#endif
-u8	qos_acm23a(u8 acm_mask, u8 priority);
-u32	rtw_get_ff_hwaddr23a(struct xmit_frame	*pxmitframe);
-int rtw_ack_tx_wait23a(struct xmit_priv *pxmitpriv, u32 timeout_ms);
-
-/* include after declaring struct xmit_buf, in order to avoid warning */
-#include <xmit_osdep.h>
-
-#endif	/* _RTL871X_XMIT_H_ */
diff --git a/drivers/staging/rtl8723au/include/sta_info.h b/drivers/staging/rtl8723au/include/sta_info.h
deleted file mode 100644
index e7260050e533..000000000000
--- a/drivers/staging/rtl8723au/include/sta_info.h
+++ /dev/null
@@ -1,373 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __STA_INFO_H_
-#define __STA_INFO_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <wifi.h>
-
-#define IBSS_START_MAC_ID	2
-#define NUM_STA 32
-#define NUM_ACL 16
-
-
-/* if mode ==0, then the sta is allowed once the addr is hit. */
-/* if mode ==1, then the sta is rejected once the addr is non-hit. */
-struct rtw_wlan_acl_node {
-	struct list_head	list;
-	u8       addr[ETH_ALEN];
-	u8       valid;
-};
-
-/* mode=0, disable */
-/* mode=1, accept unless in deny list */
-/* mode=2, deny unless in accept list */
-struct wlan_acl_pool {
-	int mode;
-	int num;
-	struct rtw_wlan_acl_node aclnode[NUM_ACL];
-	struct rtw_queue	acl_node_q;
-};
-
-struct rssi_sta  {
-	s32	UndecoratedSmoothedPWDB;
-	s32	UndecoratedSmoothedCCK;
-	s32	UndecoratedSmoothedOFDM;
-	u64	PacketMap;
-	u8	ValidBit;
-};
-
-struct	stainfo_stats	{
-	u64 rx_mgnt_pkts;
-		u64 rx_beacon_pkts;
-		u64 rx_probereq_pkts;
-		u64 rx_probersp_pkts;
-		u64 rx_probersp_bm_pkts;
-		u64 rx_probersp_uo_pkts;
-	u64 rx_ctrl_pkts;
-	u64 rx_data_pkts;
-
-	u64	last_rx_mgnt_pkts;
-		u64 last_rx_beacon_pkts;
-		u64 last_rx_probereq_pkts;
-		u64 last_rx_probersp_pkts;
-		u64 last_rx_probersp_bm_pkts;
-		u64 last_rx_probersp_uo_pkts;
-	u64	last_rx_ctrl_pkts;
-	u64	last_rx_data_pkts;
-
-	u64	rx_bytes;
-	u64	rx_drops;
-
-	u64	tx_pkts;
-	u64	tx_bytes;
-	u64  tx_drops;
-
-};
-
-struct sta_info {
-	spinlock_t	lock;
-	struct list_head	list; /* free_sta_queue */
-	struct list_head	hash_list; /* sta_hash */
-	struct rtw_adapter *padapter;
-
-	struct sta_xmit_priv sta_xmitpriv;
-	struct sta_recv_priv sta_recvpriv;
-
-	struct rtw_queue sleep_q;
-	unsigned int sleepq_len;
-
-	uint state;
-	uint aid;
-	uint mac_id;
-	uint qos_option;
-	u8	hwaddr[ETH_ALEN];
-
-	uint	ieee8021x_blocked;	/* 0: allowed, 1:blocked */
-	u32	dot118021XPrivacy; /* aes, tkip... */
-	union Keytype	dot11tkiptxmickey;
-	union Keytype	dot11tkiprxmickey;
-	union Keytype	dot118021x_UncstKey;
-	union pn48		dot11txpn;			/*  PN48 used for Unicast xmit. */
-	union pn48		dot11rxpn;			/*  PN48 used for Unicast recv. */
-
-
-	u8	bssrateset[16];
-	u32	bssratelen;
-	s32  rssi;
-	s32	signal_quality;
-
-	u8	cts2self;
-	u8	rtsen;
-
-	u8	raid;
-	u8	init_rate;
-	u32	ra_mask;
-	u8	wireless_mode;	/*  NETWORK_TYPE */
-	struct stainfo_stats sta_stats;
-
-	/* for A-MPDU TX, ADDBA timeout check */
-	struct timer_list addba_retry_timer;
-
-	/* for A-MPDU Rx reordering buffer control */
-	struct recv_reorder_ctrl recvreorder_ctrl[16];
-
-	/* for A-MPDU Tx */
-	/* unsigned char		ampdu_txen_bitmap; */
-	u16	BA_starting_seqctrl[16];
-
-	struct ht_priv	htpriv;
-
-	/* Notes: */
-	/* STA_Mode: */
-	/* curr_network(mlme_priv/security_priv/qos/ht) + sta_info: (STA & AP) CAP/INFO */
-	/* scan_q: AP CAP/INFO */
-
-	/* AP_Mode: */
-	/* curr_network(mlme_priv/security_priv/qos/ht) : AP CAP/INFO */
-	/* sta_info: (AP & STA) CAP/INFO */
-
-	struct list_head asoc_list;
-	struct list_head auth_list;
-
-	unsigned int expire_to;
-	unsigned int auth_seq;
-	unsigned int authalg;
-	unsigned char chg_txt[128];
-
-	u16 capability;
-	int flags;
-
-	int dot8021xalg;/* 0:disable, 1:psk, 2:802.1x */
-	int wpa_psk;/* 0:disable, bit(0): WPA, bit(1):WPA2 */
-	int wpa_group_cipher;
-	int wpa2_group_cipher;
-	int wpa_pairwise_cipher;
-	int wpa2_pairwise_cipher;
-
-	u8 bpairwise_key_installed;
-
-	u8 wpa_ie[32];
-
-	u8 nonerp_set;
-	u8 no_short_slot_time_set;
-	u8 no_short_preamble_set;
-	u8 no_ht_gf_set;
-	u8 no_ht_set;
-	u8 ht_20mhz_set;
-
-	unsigned int tx_ra_bitmap;
-	u8 qos_info;
-
-	u8 max_sp_len;
-	u8 uapsd_bk;/* BIT(0): Delivery enabled, BIT(1): Trigger enabled */
-	u8 uapsd_be;
-	u8 uapsd_vi;
-	u8 uapsd_vo;
-
-	u8 has_legacy_ac;
-	unsigned int sleepq_ac_len;
-
-	/* p2p priv data */
-	u8 is_p2p_device;
-	u8 p2p_status_code;
-
-	u8 keep_alive_trycnt;
-
-	/* p2p client info */
-	u8 dev_addr[ETH_ALEN];
-	u8 dev_cap;
-	u16 config_methods;
-	u8 primary_dev_type[8];
-	u8 num_of_secdev_type;
-	u8 secdev_types_list[32];/*  32/8 == 4; */
-	u16 dev_name_len;
-	u8 dev_name[32];
-	u8 *passoc_req;
-	u32 assoc_req_len;
-
-	/* for DM */
-	struct rssi_sta	 rssi_stat;
-
-	/*  */
-	/*  ================ODM Relative Info======================= */
-	/*  Please be care, dont declare too much structure here. It will cost memory * STA support num. */
-	/*  */
-	/*  */
-	/*  2011/10/20 MH Add for ODM STA info. */
-	/*  */
-	/*  Driver Write */
-	u8		bValid;				/*  record the sta status link or not? */
-	u8		rssi_level;			/* for Refresh RA mask */
-	/*  ODM Write */
-	/* 1 PHY_STATUS_INFO */
-	u8		RSSI_Path[4];		/*  */
-	u8		RSSI_Ave;
-	u8		RXEVM[4];
-	u8		RXSNR[4];
-
-	/*  ODM Write */
-	/* 1 TX_INFO (may changed by IC) */
-	/*  ================ODM Relative Info======================= */
-	/*  */
-
-	/* To store the sequence number of received management frame */
-	u16 RxMgmtFrameSeqNum;
-};
-
-#define sta_rx_pkts(sta) \
-	(sta->sta_stats.rx_mgnt_pkts \
-	+ sta->sta_stats.rx_ctrl_pkts \
-	+ sta->sta_stats.rx_data_pkts)
-
-#define sta_last_rx_pkts(sta) \
-	(sta->sta_stats.last_rx_mgnt_pkts \
-	+ sta->sta_stats.last_rx_ctrl_pkts \
-	+ sta->sta_stats.last_rx_data_pkts)
-
-#define sta_rx_data_pkts(sta) \
-	(sta->sta_stats.rx_data_pkts)
-
-#define sta_last_rx_data_pkts(sta) \
-	(sta->sta_stats.last_rx_data_pkts)
-
-#define sta_rx_mgnt_pkts(sta) \
-	(sta->sta_stats.rx_mgnt_pkts)
-
-#define sta_last_rx_mgnt_pkts(sta) \
-	(sta->sta_stats.last_rx_mgnt_pkts)
-
-#define sta_rx_beacon_pkts(sta) \
-	(sta->sta_stats.rx_beacon_pkts)
-
-#define sta_last_rx_beacon_pkts(sta) \
-	(sta->sta_stats.last_rx_beacon_pkts)
-
-#define sta_rx_probereq_pkts(sta) \
-	(sta->sta_stats.rx_probereq_pkts)
-
-#define sta_last_rx_probereq_pkts(sta) \
-	(sta->sta_stats.last_rx_probereq_pkts)
-
-#define sta_rx_probersp_pkts(sta) \
-	(sta->sta_stats.rx_probersp_pkts)
-
-#define sta_last_rx_probersp_pkts(sta) \
-	(sta->sta_stats.last_rx_probersp_pkts)
-
-#define sta_rx_probersp_bm_pkts(sta) \
-	(sta->sta_stats.rx_probersp_bm_pkts)
-
-#define sta_last_rx_probersp_bm_pkts(sta) \
-	(sta->sta_stats.last_rx_probersp_bm_pkts)
-
-#define sta_rx_probersp_uo_pkts(sta) \
-	(sta->sta_stats.rx_probersp_uo_pkts)
-
-#define sta_last_rx_probersp_uo_pkts(sta) \
-	(sta->sta_stats.last_rx_probersp_uo_pkts)
-
-#define sta_update_last_rx_pkts(sta) \
-	do { \
-		sta->sta_stats.last_rx_mgnt_pkts = sta->sta_stats.rx_mgnt_pkts; \
-		sta->sta_stats.last_rx_beacon_pkts = sta->sta_stats.rx_beacon_pkts; \
-		sta->sta_stats.last_rx_probereq_pkts = sta->sta_stats.rx_probereq_pkts; \
-		sta->sta_stats.last_rx_probersp_pkts = sta->sta_stats.rx_probersp_pkts; \
-		sta->sta_stats.last_rx_probersp_bm_pkts = sta->sta_stats.rx_probersp_bm_pkts; \
-		sta->sta_stats.last_rx_probersp_uo_pkts = sta->sta_stats.rx_probersp_uo_pkts; \
-		sta->sta_stats.last_rx_ctrl_pkts = sta->sta_stats.rx_ctrl_pkts; \
-		sta->sta_stats.last_rx_data_pkts = sta->sta_stats.rx_data_pkts; \
-	} while (0)
-
-#define STA_RX_PKTS_ARG(sta) \
-	sta->sta_stats.rx_mgnt_pkts \
-	, sta->sta_stats.rx_ctrl_pkts \
-	, sta->sta_stats.rx_data_pkts
-
-#define STA_LAST_RX_PKTS_ARG(sta) \
-	sta->sta_stats.last_rx_mgnt_pkts, \
-	sta->sta_stats.last_rx_ctrl_pkts, \
-	sta->sta_stats.last_rx_data_pkts
-
-#define STA_RX_PKTS_DIFF_ARG(sta) \
-	sta->sta_stats.rx_mgnt_pkts - sta->sta_stats.last_rx_mgnt_pkts, \
-	sta->sta_stats.rx_ctrl_pkts - sta->sta_stats.last_rx_ctrl_pkts, \
-	sta->sta_stats.rx_data_pkts - sta->sta_stats.last_rx_data_pkts
-
-#define STA_PKTS_FMT "(m:%llu, c:%llu, d:%llu)"
-
-struct sta_priv {
-	spinlock_t sta_hash_lock;
-	struct list_head sta_hash[NUM_STA];
-	int asoc_sta_count;
-
-	struct rtw_adapter *padapter;
-	struct list_head asoc_list;
-	struct list_head auth_list;
-	spinlock_t asoc_list_lock;
-	spinlock_t auth_list_lock;
-	u8 asoc_list_cnt;
-	u8 auth_list_cnt;
-
-	unsigned int auth_to;  /* sec, time to expire in authenticating. */
-	unsigned int assoc_to; /* sec, time to expire before associating. */
-	unsigned int expire_to; /* sec , time to expire after associated. */
-
-	/* pointers to STA info; based on allocated AID or NULL if AID free
-	 * AID is in the range 1-2007, so sta_aid[0] corresponders to AID 1
-	 * and so on
-	 */
-	struct sta_info *sta_aid[NUM_STA];
-
-	u16 sta_dz_bitmap;/* only support 15 stations, station aid bitmap
-			   * for sleeping sta. */
-	u16 tim_bitmap;/* only support 15 stations,
-			* aid=0~15 mapping bit0~bit15 */
-
-	u16 max_num_sta;
-
-	struct wlan_acl_pool acl_list;
-};
-
-static inline u32 wifi_mac_hash(const u8 *mac)
-{
-	u32 x;
-
-	x = mac[0];
-	x = (x << 2) ^ mac[1];
-	x = (x << 2) ^ mac[2];
-	x = (x << 2) ^ mac[3];
-	x = (x << 2) ^ mac[4];
-	x = (x << 2) ^ mac[5];
-
-	x ^= x >> 8;
-	x  = x & (NUM_STA - 1);
-
-	return x;
-}
-
-int _rtw_init_sta_priv23a(struct sta_priv *pstapriv);
-int _rtw_free_sta_priv23a(struct sta_priv *pstapriv);
-
-struct sta_info *rtw_alloc_stainfo23a(struct sta_priv *pstapriv, const u8 *hwaddr, gfp_t gfp);
-int rtw_free_stainfo23a(struct rtw_adapter *padapter, struct sta_info *psta);
-void rtw_free_all_stainfo23a(struct rtw_adapter *padapter);
-struct sta_info *rtw_get_stainfo23a(struct sta_priv *pstapriv, const u8 *hwaddr);
-int rtw_init_bcmc_stainfo23a(struct rtw_adapter *padapter);
-struct sta_info *rtw_get_bcmc_stainfo23a(struct rtw_adapter *padapter);
-bool rtw_access_ctrl23a(struct rtw_adapter *padapter, u8 *mac_addr);
-
-#endif /* _STA_INFO_H_ */
diff --git a/drivers/staging/rtl8723au/include/usb_ops.h b/drivers/staging/rtl8723au/include/usb_ops.h
deleted file mode 100644
index ff11e13b24a8..000000000000
--- a/drivers/staging/rtl8723au/include/usb_ops.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __USB_OPS_H_
-#define __USB_OPS_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <osdep_intf.h>
-#include <usb_ops_linux.h>
-
-#define REALTEK_USB_VENQT_READ		0xC0
-#define REALTEK_USB_VENQT_WRITE		0x40
-#define REALTEK_USB_VENQT_CMD_REQ	0x05
-#define REALTEK_USB_VENQT_CMD_IDX	0x00
-
-enum {
-	VENDOR_WRITE = 0x00,
-	VENDOR_READ = 0x01,
-};
-
-#define ALIGNMENT_UNIT				16
-#define MAX_VENDOR_REQ_CMD_SIZE	254		/* 8188cu SIE Support */
-#define MAX_USB_IO_CTL_SIZE	(MAX_VENDOR_REQ_CMD_SIZE +ALIGNMENT_UNIT)
-
-void rtl8723au_set_hw_type(struct rtw_adapter *padapter);
-
-void rtl8723au_recv_tasklet(void *priv);
-
-void rtl8723au_xmit_tasklet(void *priv);
-
-/* Increase and check if the continual_urb_error of this @param dvobjprive is
- * larger than MAX_CONTINUAL_URB_ERR. Return result
- */
-static inline int rtw_inc_and_chk_continual_urb_error(struct dvobj_priv *dvobj)
-{
-	int ret = false;
-	int value;
-
-	value = atomic_inc_return(&dvobj->continual_urb_error);
-	if (value > MAX_CONTINUAL_URB_ERR) {
-		DBG_8723A("[dvobj:%p][ERROR] continual_urb_error:%d > %d\n",
-			  dvobj, value, MAX_CONTINUAL_URB_ERR);
-		ret = true;
-	}
-	return ret;
-}
-
-/* Set the continual_urb_error of this @param dvobjprive to 0 */
-static inline void rtw_reset_continual_urb_error(struct dvobj_priv *dvobj)
-{
-	atomic_set(&dvobj->continual_urb_error, 0);
-}
-
-bool rtl8723au_chip_configure(struct rtw_adapter *padapter);
-
-#endif /* __USB_OPS_H_ */
diff --git a/drivers/staging/rtl8723au/include/usb_ops_linux.h b/drivers/staging/rtl8723au/include/usb_ops_linux.h
deleted file mode 100644
index af2f14b8b360..000000000000
--- a/drivers/staging/rtl8723au/include/usb_ops_linux.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __USB_OPS_LINUX_H__
-#define __USB_OPS_LINUX_H__
-
-#define VENDOR_CMD_MAX_DATA_LEN	254
-
-#define RTW_USB_CONTROL_MSG_TIMEOUT	500/* ms */
-
-#define MAX_USBCTRL_VENDORREQ_TIMES	10
-
-int rtl8723au_read_port(struct rtw_adapter *adapter, u32 cnt,
-			struct recv_buf *precvbuf);
-void rtl8723au_read_port_cancel(struct rtw_adapter *padapter);
-int rtl8723au_write_port(struct rtw_adapter *padapter, u32 addr, u32 cnt,
-			 struct xmit_buf *pxmitbuf);
-void rtl8723au_write_port_cancel(struct rtw_adapter *padapter);
-int rtl8723au_read_interrupt(struct rtw_adapter *adapter);
-
-u8 rtl8723au_read8(struct rtw_adapter *padapter, u16 addr);
-u16 rtl8723au_read16(struct rtw_adapter *padapter, u16 addr);
-u32 rtl8723au_read32(struct rtw_adapter *padapter, u16 addr);
-int rtl8723au_write8(struct rtw_adapter *padapter, u16 addr, u8 val);
-int rtl8723au_write16(struct rtw_adapter *padapter, u16 addr, u16 val);
-int rtl8723au_write32(struct rtw_adapter *padapter, u16 addr, u32 val);
-int rtl8723au_writeN(struct rtw_adapter *padapter,
-		     u16 addr, u16 length, u8 *pdata);
-
-#endif
diff --git a/drivers/staging/rtl8723au/include/wifi.h b/drivers/staging/rtl8723au/include/wifi.h
deleted file mode 100644
index 25d573c3e232..000000000000
--- a/drivers/staging/rtl8723au/include/wifi.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef _WIFI_H_
-#define _WIFI_H_
-
-/*  This value is tested by WiFi 11n Test Plan 5.2.3.
- *  This test verifies the WLAN NIC can update the NAV through sending
- *  the CTS with large duration.
- */
-#define	WiFiNavUpperUs		30000	/*  30 ms */
-
-/*-----------------------------------------------------------------------------
-				Below is the definition for 802.11n
-------------------------------------------------------------------------------*/
-
-struct AC_param {
-	u8			ACI_AIFSN;
-	u8			CW;
-	__le16			TXOP_limit;
-}  __packed;
-
-struct WMM_para_element {
-	unsigned char		QoS_info;
-	unsigned char		reserved;
-	struct AC_param	ac_param[4];
-}  __packed;
-
-struct ADDBA_request {
-	u8		dialog_token;
-	__le16		BA_para_set;
-	__le16		BA_timeout_value;
-	__le16		BA_starting_seqctrl;
-}  __packed;
-
-
-/*	===============WPS Section=============== */
-/*	WPS attribute ID */
-#define WPS_ATTR_VER1				0x104A
-#define WPS_ATTR_SIMPLE_CONF_STATE		0x1044
-#define WPS_ATTR_RESP_TYPE			0x103B
-#define WPS_ATTR_UUID_E				0x1047
-#define WPS_ATTR_MANUFACTURER			0x1021
-#define WPS_ATTR_MODEL_NAME			0x1023
-#define WPS_ATTR_MODEL_NUMBER			0x1024
-#define WPS_ATTR_SERIAL_NUMBER			0x1042
-#define WPS_ATTR_PRIMARY_DEV_TYPE		0x1054
-#define WPS_ATTR_SEC_DEV_TYPE_LIST		0x1055
-#define WPS_ATTR_DEVICE_NAME			0x1011
-#define WPS_ATTR_CONF_METHOD			0x1008
-#define WPS_ATTR_RF_BANDS			0x103C
-#define WPS_ATTR_DEVICE_PWID			0x1012
-#define WPS_ATTR_REQUEST_TYPE			0x103A
-#define WPS_ATTR_ASSOCIATION_STATE		0x1002
-#define WPS_ATTR_CONFIG_ERROR			0x1009
-#define WPS_ATTR_VENDOR_EXT			0x1049
-#define WPS_ATTR_SELECTED_REGISTRAR		0x1041
-
-/*	WPS Configuration Method */
-#define	WPS_CM_NONE					0x0000
-#define	WPS_CM_LABEL					0x0004
-#define	WPS_CM_DISPLYA					0x0008
-#define	WPS_CM_EXTERNAL_NFC_TOKEN			0x0010
-#define	WPS_CM_INTEGRATED_NFC_TOKEN			0x0020
-#define	WPS_CM_NFC_INTERFACE				0x0040
-#define	WPS_CM_PUSH_BUTTON				0x0080
-#define	WPS_CM_KEYPAD					0x0100
-#define	WPS_CM_SW_PUHS_BUTTON				0x0280
-#define	WPS_CM_HW_PUHS_BUTTON				0x0480
-#define	WPS_CM_SW_DISPLAY_PIN				0x2008
-#define	WPS_CM_LCD_DISPLAY_PIN				0x4008
-
-#endif /*  _WIFI_H_ */
diff --git a/drivers/staging/rtl8723au/include/wlan_bssdef.h b/drivers/staging/rtl8723au/include/wlan_bssdef.h
deleted file mode 100644
index 95b32e15a4d0..000000000000
--- a/drivers/staging/rtl8723au/include/wlan_bssdef.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __WLAN_BSSDEF_H__
-#define __WLAN_BSSDEF_H__
-
-
-#define MAX_IE_SZ	768
-
-
-#define NDIS_802_11_LENGTH_RATES        8
-#define NDIS_802_11_LENGTH_RATES_EX     16
-
-/* Length is the 4 bytes multiples of the sum of
- * sizeof(6 * sizeof(unsigned char)) + 2 + sizeof(struct ndis_802_11_ssid) +
- * sizeof(u32) + sizeof(long) + sizeof(enum ndis_802_11_net_type) +
- * sizeof(struct ndis_802_11_config) + sizeof(sizeof(unsigned char) *
- * NDIS_802_11_LENGTH_RATES_EX) + IELength
- *
- * Except the IELength, all other fields are fixed length. Therefore,
- * we can define a macro to present the partial sum.
- */
-
-enum ndis_802_11_auth_mode {
-	Ndis802_11AuthModeOpen,
-	Ndis802_11AuthModeShared,
-	Ndis802_11AuthModeAutoSwitch,
-	Ndis802_11AuthModeWPA,
-	Ndis802_11AuthModeWPAPSK,
-	Ndis802_11AuthModeWPANone,
-	dis802_11AuthModeMax       /*  upper bound */
-};
-
-enum  {
-	Ndis802_11WEPEnabled,
-	Ndis802_11Encryption1Enabled = Ndis802_11WEPEnabled,
-	Ndis802_11WEPDisabled,
-	Ndis802_11EncryptionDisabled = Ndis802_11WEPDisabled,
-	Ndis802_11WEPKeyAbsent,
-	Ndis802_11Encryption1KeyAbsent = Ndis802_11WEPKeyAbsent,
-	Ndis802_11WEPNotSupported,
-	Ndis802_11EncryptionNotSupported = Ndis802_11WEPNotSupported,
-	Ndis802_11Encryption2Enabled,
-	Ndis802_11Encryption2KeyAbsent,
-	Ndis802_11Encryption3Enabled,
-	Ndis802_11Encryption3KeyAbsent,
-};
-
-struct wlan_bcn_info {
-	/* these infor get from rtw_get_encrypt_info when
-	 *	 * translate scan to UI */
-	u8 encryp_protocol;/* ENCRYP_PROTOCOL_E: OPEN/WEP/WPA/WPA2 */
-	int group_cipher; /* WPA/WPA2 group cipher */
-	int pairwise_cipher;/* WPA/WPA2/WEP pairwise cipher */
-	int is_8021x;
-
-	/* bwmode 20/40 and ch_offset UP/LOW */
-};
-
-struct wlan_bssid_ex {
-	u32  Length;
-	u8 MacAddress[ETH_ALEN];
-	u16 reserved;
-	struct cfg80211_ssid Ssid;
-	u32  Privacy;
-	long  Rssi;/* in dBM, raw data , get from PHY) */
-	u16 beacon_interval;
-	u16 capability;
-	u64 tsf;
-	u32 ATIMWindow;         /*  units are Kusec */
-	u32 DSConfig;           /*  Frequency, units are kHz */
-	enum nl80211_iftype ifmode;
-	unsigned char SupportedRates[NDIS_802_11_LENGTH_RATES_EX];
-	u8 SignalStrength;/* in percentage */
-	u8 SignalQuality;/* in percentage */
-	u32  IELength;
-	u8  IEs[MAX_IE_SZ]; /* timestamp, beacon interval, and capability info*/
-} __packed;
-
-static inline uint get_wlan_bssid_ex_sz(struct wlan_bssid_ex *bss)
-{
-	return sizeof(struct wlan_bssid_ex) - MAX_IE_SZ + bss->IELength;
-}
-
-struct	wlan_network {
-	struct list_head	list;
-	int	network_type;	/* refer to ieee80211.h for 11A/B/G */
-	/*  set to fixed when not to be removed as site-surveying */
-	int	fixed;
-	unsigned long	last_scanned; /* timestamp for the network */
-	int	join_res;
-	struct wlan_bssid_ex	network; /* must be the last item */
-	struct wlan_bcn_info	BcnInfo;
-};
-
-enum VRTL_CARRIER_SENSE {
-	DISABLE_VCS,
-	ENABLE_VCS,
-	AUTO_VCS
-};
-
-enum VCS_TYPE {
-	NONE_VCS,
-	RTS_CTS,
-	CTS_TO_SELF
-};
-
-/* john */
-#define NUM_PRE_AUTH_KEY 16
-#define NUM_PMKID_CACHE NUM_PRE_AUTH_KEY
-
-#endif /* ifndef WLAN_BSSDEF_H_ */
diff --git a/drivers/staging/rtl8723au/include/xmit_osdep.h b/drivers/staging/rtl8723au/include/xmit_osdep.h
deleted file mode 100644
index 2be04c48656c..000000000000
--- a/drivers/staging/rtl8723au/include/xmit_osdep.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#ifndef __XMIT_OSDEP_H_
-#define __XMIT_OSDEP_H_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-
-#define NR_XMITFRAME	256
-
-int rtw_xmit23a_entry23a(struct sk_buff *pkt, struct net_device *pnetdev);
-
-void rtw_os_xmit_schedule23a(struct rtw_adapter *padapter);
-
-int rtw_os_xmit_resource_alloc23a(struct rtw_adapter *padapter,
-			       struct xmit_buf *pxmitbuf, u32 alloc_sz);
-void rtw_os_xmit_resource_free23a(struct rtw_adapter *padapter,
-			       struct xmit_buf *pxmitbuf);
-
-void rtw_os_pkt_complete23a(struct rtw_adapter *padapter, struct sk_buff *pkt);
-void rtw_os_xmit_complete23a(struct rtw_adapter *padapter,
-			  struct xmit_frame *pxframe);
-int netdev_open23a(struct net_device *pnetdev);
-
-#endif /* __XMIT_OSDEP_H_ */
diff --git a/drivers/staging/rtl8723au/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723au/os_dep/ioctl_cfg80211.c
deleted file mode 100644
index d0ba3778990e..000000000000
--- a/drivers/staging/rtl8723au/os_dep/ioctl_cfg80211.c
+++ /dev/null
@@ -1,3348 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define  _IOCTL_CFG80211_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <xmit_osdep.h>
-
-#include "ioctl_cfg80211.h"
-
-#define RTW_MAX_MGMT_TX_CNT 8
-
-#define RTW_MAX_REMAIN_ON_CHANNEL_DURATION 65535	/* ms */
-#define RTW_MAX_NUM_PMKIDS 4
-
-static const u32 rtw_cipher_suites[] = {
-	WLAN_CIPHER_SUITE_WEP40,
-	WLAN_CIPHER_SUITE_WEP104,
-	WLAN_CIPHER_SUITE_TKIP,
-	WLAN_CIPHER_SUITE_CCMP,
-};
-
-#define RATETAB_ENT(_rate, _rateid, _flags) {			\
-	.bitrate	= (_rate),				\
-	.hw_value	= (_rateid),				\
-	.flags		= (_flags),				\
-}
-
-#define CHAN2G(_channel, _freq, _flags) {			\
-	.band			= NL80211_BAND_2GHZ,		\
-	.center_freq		= (_freq),			\
-	.hw_value		= (_channel),			\
-	.flags			= (_flags),			\
-	.max_antenna_gain	= 0,				\
-	.max_power		= 30,				\
-}
-
-#define CHAN5G(_channel, _flags) {				\
-	.band			= NL80211_BAND_5GHZ,		\
-	.center_freq		= 5000 + (5 * (_channel)),	\
-	.hw_value		= (_channel),			\
-	.flags			= (_flags),			\
-	.max_antenna_gain	= 0,				\
-	.max_power		= 30,				\
-}
-
-static struct ieee80211_rate rtw_rates[] = {
-	RATETAB_ENT(10, 0x1, 0),
-	RATETAB_ENT(20, 0x2, 0),
-	RATETAB_ENT(55, 0x4, 0),
-	RATETAB_ENT(110, 0x8, 0),
-	RATETAB_ENT(60, 0x10, 0),
-	RATETAB_ENT(90, 0x20, 0),
-	RATETAB_ENT(120, 0x40, 0),
-	RATETAB_ENT(180, 0x80, 0),
-	RATETAB_ENT(240, 0x100, 0),
-	RATETAB_ENT(360, 0x200, 0),
-	RATETAB_ENT(480, 0x400, 0),
-	RATETAB_ENT(540, 0x800, 0),
-};
-
-#define rtw_a_rates		(rtw_rates + 4)
-#define RTW_A_RATES_NUM	8
-#define rtw_g_rates		(rtw_rates + 0)
-#define RTW_G_RATES_NUM	12
-
-#define RTW_2G_CHANNELS_NUM 14
-#define RTW_5G_CHANNELS_NUM 37
-
-static struct ieee80211_channel rtw_2ghz_channels[] = {
-	CHAN2G(1, 2412, 0),
-	CHAN2G(2, 2417, 0),
-	CHAN2G(3, 2422, 0),
-	CHAN2G(4, 2427, 0),
-	CHAN2G(5, 2432, 0),
-	CHAN2G(6, 2437, 0),
-	CHAN2G(7, 2442, 0),
-	CHAN2G(8, 2447, 0),
-	CHAN2G(9, 2452, 0),
-	CHAN2G(10, 2457, 0),
-	CHAN2G(11, 2462, 0),
-	CHAN2G(12, 2467, 0),
-	CHAN2G(13, 2472, 0),
-	CHAN2G(14, 2484, 0),
-};
-
-static struct ieee80211_channel rtw_5ghz_a_channels[] = {
-	CHAN5G(34, 0), CHAN5G(36, 0),
-	CHAN5G(38, 0), CHAN5G(40, 0),
-	CHAN5G(42, 0), CHAN5G(44, 0),
-	CHAN5G(46, 0), CHAN5G(48, 0),
-	CHAN5G(52, 0), CHAN5G(56, 0),
-	CHAN5G(60, 0), CHAN5G(64, 0),
-	CHAN5G(100, 0), CHAN5G(104, 0),
-	CHAN5G(108, 0), CHAN5G(112, 0),
-	CHAN5G(116, 0), CHAN5G(120, 0),
-	CHAN5G(124, 0), CHAN5G(128, 0),
-	CHAN5G(132, 0), CHAN5G(136, 0),
-	CHAN5G(140, 0), CHAN5G(149, 0),
-	CHAN5G(153, 0), CHAN5G(157, 0),
-	CHAN5G(161, 0), CHAN5G(165, 0),
-	CHAN5G(184, 0), CHAN5G(188, 0),
-	CHAN5G(192, 0), CHAN5G(196, 0),
-	CHAN5G(200, 0), CHAN5G(204, 0),
-	CHAN5G(208, 0), CHAN5G(212, 0),
-	CHAN5G(216, 0),
-};
-
-static void rtw_2g_channels_init(struct ieee80211_channel *channels)
-{
-	memcpy((void *)channels, (void *)rtw_2ghz_channels,
-	       sizeof(struct ieee80211_channel) * RTW_2G_CHANNELS_NUM);
-}
-
-static void rtw_5g_channels_init(struct ieee80211_channel *channels)
-{
-	memcpy((void *)channels, (void *)rtw_5ghz_a_channels,
-	       sizeof(struct ieee80211_channel) * RTW_5G_CHANNELS_NUM);
-}
-
-static void rtw_2g_rates_init(struct ieee80211_rate *rates)
-{
-	memcpy(rates, rtw_g_rates,
-	       sizeof(struct ieee80211_rate) * RTW_G_RATES_NUM);
-}
-
-static void rtw_5g_rates_init(struct ieee80211_rate *rates)
-{
-	memcpy(rates, rtw_a_rates,
-	       sizeof(struct ieee80211_rate) * RTW_A_RATES_NUM);
-}
-
-static struct ieee80211_supported_band *
-rtw_spt_band_alloc(enum nl80211_band band)
-{
-	struct ieee80211_supported_band *spt_band = NULL;
-	int n_channels, n_bitrates;
-
-	if (band == NL80211_BAND_2GHZ) {
-		n_channels = RTW_2G_CHANNELS_NUM;
-		n_bitrates = RTW_G_RATES_NUM;
-	} else if (band == NL80211_BAND_5GHZ) {
-		n_channels = RTW_5G_CHANNELS_NUM;
-		n_bitrates = RTW_A_RATES_NUM;
-	} else {
-		goto exit;
-	}
-	spt_band = kzalloc(sizeof(struct ieee80211_supported_band) +
-			   sizeof(struct ieee80211_channel) * n_channels +
-			   sizeof(struct ieee80211_rate) * n_bitrates,
-			   GFP_KERNEL);
-	if (!spt_band)
-		goto exit;
-
-	spt_band->channels =
-		(struct ieee80211_channel *)(((u8 *) spt_band) +
-					     sizeof(struct
-						    ieee80211_supported_band));
-	spt_band->bitrates =
-		(struct ieee80211_rate *)(((u8 *) spt_band->channels) +
-					  sizeof(struct ieee80211_channel) *
-					  n_channels);
-	spt_band->band = band;
-	spt_band->n_channels = n_channels;
-	spt_band->n_bitrates = n_bitrates;
-
-	if (band == NL80211_BAND_2GHZ) {
-		rtw_2g_channels_init(spt_band->channels);
-		rtw_2g_rates_init(spt_band->bitrates);
-	} else if (band == NL80211_BAND_5GHZ) {
-		rtw_5g_channels_init(spt_band->channels);
-		rtw_5g_rates_init(spt_band->bitrates);
-	}
-
-	/* spt_band.ht_cap */
-
-exit:
-	return spt_band;
-}
-
-static const struct ieee80211_txrx_stypes
-rtw_cfg80211_default_mgmt_stypes[NUM_NL80211_IFTYPES] = {
-	[NL80211_IFTYPE_ADHOC] = {
-		.tx = 0xffff,
-		.rx = BIT(IEEE80211_STYPE_ACTION >> 4)
-	},
-	[NL80211_IFTYPE_STATION] = {
-		.tx = 0xffff,
-		.rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
-		      BIT(IEEE80211_STYPE_PROBE_REQ >> 4)
-	},
-	[NL80211_IFTYPE_AP] = {
-		.tx = 0xffff,
-		.rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) |
-		      BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) |
-		      BIT(IEEE80211_STYPE_PROBE_REQ >> 4) |
-		      BIT(IEEE80211_STYPE_DISASSOC >> 4) |
-		      BIT(IEEE80211_STYPE_AUTH >> 4) |
-		      BIT(IEEE80211_STYPE_DEAUTH >> 4) |
-		      BIT(IEEE80211_STYPE_ACTION >> 4)
-	},
-	[NL80211_IFTYPE_AP_VLAN] = {
-		/* copy AP */
-		.tx = 0xffff,
-		.rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) |
-		      BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) |
-		      BIT(IEEE80211_STYPE_PROBE_REQ >> 4) |
-		      BIT(IEEE80211_STYPE_DISASSOC >> 4) |
-		      BIT(IEEE80211_STYPE_AUTH >> 4) |
-		      BIT(IEEE80211_STYPE_DEAUTH >> 4) |
-		      BIT(IEEE80211_STYPE_ACTION >> 4)
-	},
-	[NL80211_IFTYPE_P2P_CLIENT] = {
-		.tx = 0xffff,
-		.rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
-		      BIT(IEEE80211_STYPE_PROBE_REQ >> 4)
-	},
-	[NL80211_IFTYPE_P2P_GO] = {
-		.tx = 0xffff,
-		.rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) |
-		      BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) |
-		      BIT(IEEE80211_STYPE_PROBE_REQ >> 4) |
-		      BIT(IEEE80211_STYPE_DISASSOC >> 4) |
-		      BIT(IEEE80211_STYPE_AUTH >> 4) |
-		      BIT(IEEE80211_STYPE_DEAUTH >> 4) |
-		      BIT(IEEE80211_STYPE_ACTION >> 4)
-	},
-};
-
-static int rtw_cfg80211_inform_bss(struct rtw_adapter *padapter,
-				   struct wlan_network *pnetwork)
-{
-	int ret = 0;
-	struct ieee80211_channel *notify_channel;
-	struct cfg80211_bss *bss;
-	u16 channel;
-	u32 freq;
-	u8 *notify_ie;
-	size_t notify_ielen;
-	s32 notify_signal;
-	struct wireless_dev *wdev = padapter->rtw_wdev;
-	struct wiphy *wiphy = wdev->wiphy;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	channel = pnetwork->network.DSConfig;
-	if (channel <= RTW_CH_MAX_2G_CHANNEL)
-		freq = ieee80211_channel_to_frequency(channel,
-						      NL80211_BAND_2GHZ);
-	else
-		freq = ieee80211_channel_to_frequency(channel,
-						      NL80211_BAND_5GHZ);
-
-	notify_channel = ieee80211_get_channel(wiphy, freq);
-
-	notify_ie = pnetwork->network.IEs;
-	notify_ielen = pnetwork->network.IELength;
-
-	/* We've set wiphy's signal_type as CFG80211_SIGNAL_TYPE_MBM:
-	 *  signal strength in mBm (100*dBm)
-	 */
-	if (check_fwstate(pmlmepriv, _FW_LINKED) &&
-	    is_same_network23a(&pmlmepriv->cur_network.network,
-			    &pnetwork->network)) {
-		notify_signal = 100 * translate_percentage_to_dbm(padapter->recvpriv.signal_strength);	/* dbm */
-	} else {
-		notify_signal = 100 * translate_percentage_to_dbm(
-			pnetwork->network.SignalStrength);	/* dbm */
-	}
-
-	bss = cfg80211_inform_bss(wiphy, notify_channel,
-				  CFG80211_BSS_FTYPE_UNKNOWN,
-				  pnetwork->network.MacAddress,
-				  pnetwork->network.tsf,
-				  pnetwork->network.capability,
-				  pnetwork->network.beacon_interval,
-				  notify_ie, notify_ielen,
-				  notify_signal, GFP_ATOMIC);
-
-	if (unlikely(!bss)) {
-		DBG_8723A("rtw_cfg80211_inform_bss error\n");
-		return -EINVAL;
-	}
-
-	cfg80211_put_bss(wiphy, bss);
-
-	return ret;
-}
-
-void rtw_cfg80211_indicate_connect(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
-	struct wireless_dev *pwdev = padapter->rtw_wdev;
-
-	DBG_8723A("%s(padapter =%p)\n", __func__, padapter);
-
-	if (pwdev->iftype != NL80211_IFTYPE_STATION &&
-	    pwdev->iftype != NL80211_IFTYPE_P2P_CLIENT)
-		return;
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
-		return;
-
-	if (padapter->mlmepriv.to_roaming > 0) {
-		struct wiphy *wiphy = pwdev->wiphy;
-		struct ieee80211_channel *notify_channel;
-		u32 freq;
-		u16 channel = cur_network->network.DSConfig;
-
-		if (channel <= RTW_CH_MAX_2G_CHANNEL)
-			freq =
-			    ieee80211_channel_to_frequency(channel,
-							   NL80211_BAND_2GHZ);
-		else
-			freq =
-			    ieee80211_channel_to_frequency(channel,
-							   NL80211_BAND_5GHZ);
-
-		notify_channel = ieee80211_get_channel(wiphy, freq);
-
-		DBG_8723A("%s call cfg80211_roamed\n", __func__);
-		cfg80211_roamed(padapter->pnetdev, notify_channel,
-				cur_network->network.MacAddress,
-				pmlmepriv->assoc_req +
-				sizeof(struct ieee80211_hdr_3addr) + 2,
-				pmlmepriv->assoc_req_len -
-				sizeof(struct ieee80211_hdr_3addr) - 2,
-				pmlmepriv->assoc_rsp +
-				sizeof(struct ieee80211_hdr_3addr) + 6,
-				pmlmepriv->assoc_rsp_len -
-				sizeof(struct ieee80211_hdr_3addr) - 6,
-				GFP_ATOMIC);
-	} else {
-		cfg80211_connect_result(padapter->pnetdev,
-					cur_network->network.MacAddress,
-					pmlmepriv->assoc_req +
-					sizeof(struct ieee80211_hdr_3addr) + 2,
-					pmlmepriv->assoc_req_len -
-					sizeof(struct ieee80211_hdr_3addr) - 2,
-					pmlmepriv->assoc_rsp +
-					sizeof(struct ieee80211_hdr_3addr) + 6,
-					pmlmepriv->assoc_rsp_len -
-					sizeof(struct ieee80211_hdr_3addr) - 6,
-					WLAN_STATUS_SUCCESS, GFP_ATOMIC);
-	}
-}
-
-void rtw_cfg80211_indicate_disconnect(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wireless_dev *pwdev = padapter->rtw_wdev;
-
-	DBG_8723A("%s(padapter =%p)\n", __func__, padapter);
-
-	if (pwdev->iftype != NL80211_IFTYPE_STATION &&
-	    pwdev->iftype != NL80211_IFTYPE_P2P_CLIENT)
-		return;
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
-		return;
-
-	if (!padapter->mlmepriv.not_indic_disco) {
-		if (check_fwstate(&padapter->mlmepriv, WIFI_UNDER_LINKING)) {
-			cfg80211_connect_result(padapter->pnetdev, NULL, NULL,
-						0, NULL, 0,
-						WLAN_STATUS_UNSPECIFIED_FAILURE,
-						GFP_ATOMIC);
-		} else {
-			cfg80211_disconnected(padapter->pnetdev, 0, NULL,
-					      0, false, GFP_ATOMIC);
-		}
-	}
-}
-
-#ifdef CONFIG_8723AU_AP_MODE
-static int set_pairwise_key(struct rtw_adapter *padapter, struct sta_info *psta)
-{
-	struct cmd_obj *ph2c;
-	struct set_stakey_parm *psetstakey_para;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	int res = _SUCCESS;
-
-	ph2c = kzalloc(sizeof(struct cmd_obj), GFP_KERNEL);
-	if (ph2c == NULL) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	psetstakey_para = kzalloc(sizeof(struct set_stakey_parm), GFP_KERNEL);
-	if (psetstakey_para == NULL) {
-		kfree(ph2c);
-		res = _FAIL;
-		goto exit;
-	}
-
-	init_h2fwcmd_w_parm_no_rsp(ph2c, psetstakey_para, _SetStaKey_CMD_);
-
-	psetstakey_para->algorithm = psta->dot118021XPrivacy;
-
-	ether_addr_copy(psetstakey_para->addr, psta->hwaddr);
-
-	memcpy(psetstakey_para->key, &psta->dot118021x_UncstKey, 16);
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, ph2c);
-
-exit:
-	return res;
-}
-
-static int set_group_key(struct rtw_adapter *padapter, struct key_params *parms,
-			 u32 alg, u8 keyid)
-{
-	struct cmd_obj *pcmd;
-	struct setkey_parm *psetkeyparm;
-	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
-	int res = _SUCCESS;
-
-	DBG_8723A("%s\n", __func__);
-
-	if (keyid >= 4) {
-		res = _FAIL;
-		goto exit;
-	}
-
-	pcmd = kzalloc(sizeof(struct cmd_obj), GFP_KERNEL);
-	if (!pcmd) {
-		res = _FAIL;
-		goto exit;
-	}
-	psetkeyparm = kzalloc(sizeof(struct setkey_parm), GFP_KERNEL);
-	if (!psetkeyparm) {
-		kfree(pcmd);
-		res = _FAIL;
-		goto exit;
-	}
-
-	psetkeyparm->keyid = keyid;
-	if (is_wep_enc(alg))
-		padapter->mlmepriv.key_mask |= BIT(psetkeyparm->keyid);
-
-	psetkeyparm->algorithm = alg;
-
-	psetkeyparm->set_tx = 1;
-
-	memcpy(&psetkeyparm->key, parms->key, parms->key_len);
-
-	pcmd->cmdcode = _SetKey_CMD_;
-	pcmd->parmbuf = (u8 *) psetkeyparm;
-	pcmd->cmdsz = sizeof(struct setkey_parm);
-	pcmd->rsp = NULL;
-	pcmd->rspsz = 0;
-
-	res = rtw_enqueue_cmd23a(pcmdpriv, pcmd);
-
-exit:
-	return res;
-}
-
-static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, u8 key_index,
-					  int set_tx, const u8 *sta_addr,
-					  struct key_params *keyparms)
-{
-	int key_len;
-	struct sta_info *psta = NULL, *pbcmc_sta = NULL;
-	struct rtw_adapter *padapter = netdev_priv(dev);
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-
-	DBG_8723A("%s\n", __func__);
-
-	if (!is_broadcast_ether_addr(sta_addr)) {
-		psta = rtw_get_stainfo23a(pstapriv, sta_addr);
-		if (!psta) {
-			/* ret = -EINVAL; */
-			DBG_8723A("rtw_set_encryption(), sta has already "
-				  "been removed or never been added\n");
-			goto exit;
-		}
-	}
-
-	key_len = keyparms->key_len;
-
-	if (!psta && (keyparms->cipher == WLAN_CIPHER_SUITE_WEP40 ||
-		      keyparms->cipher == WLAN_CIPHER_SUITE_WEP104)) {
-		DBG_8723A("r871x_set_encryption, crypt.alg = WEP\n");
-
-		DBG_8723A("r871x_set_encryption, wep_key_idx =%d, len =%d\n",
-			  key_index, key_len);
-
-		if (psecuritypriv->bWepDefaultKeyIdxSet == 0) {
-			/* wep default key has not been set, so use
-			   this key index as default key. */
-
-			psecuritypriv->ndisencryptstatus =
-				Ndis802_11Encryption1Enabled;
-			psecuritypriv->dot11PrivacyAlgrthm = keyparms->cipher;
-			psecuritypriv->dot118021XGrpPrivacy = keyparms->cipher;
-
-			psecuritypriv->dot11PrivacyKeyIndex = key_index;
-		}
-
-		memcpy(&psecuritypriv->wep_key[key_index].key,
-		       keyparms->key, key_len);
-
-		psecuritypriv->wep_key[key_index].keylen = key_len;
-
-		set_group_key(padapter, keyparms, keyparms->cipher, key_index);
-
-		goto exit;
-	}
-
-	if (!psta) {	/*  group key */
-		if (set_tx == 0) {	/* group key */
-			if (keyparms->cipher == WLAN_CIPHER_SUITE_WEP40 ||
-			    keyparms->cipher == WLAN_CIPHER_SUITE_WEP104) {
-				DBG_8723A("%s, set group_key, WEP\n", __func__);
-
-				memcpy(psecuritypriv->
-				       dot118021XGrpKey[key_index].skey,
-				       keyparms->key, key_len);
-
-				psecuritypriv->dot118021XGrpPrivacy =
-					keyparms->cipher;
-			} else if (keyparms->cipher == WLAN_CIPHER_SUITE_TKIP) {
-				DBG_8723A("%s, set group_key, TKIP\n",
-					  __func__);
-
-				psecuritypriv->dot118021XGrpPrivacy =
-					WLAN_CIPHER_SUITE_TKIP;
-
-				memcpy(psecuritypriv->
-				       dot118021XGrpKey[key_index].skey,
-				       keyparms->key,
-				       (min(16, key_len)));
-
-				/* set mic key */
-				memcpy(psecuritypriv->
-				       dot118021XGrptxmickey[key_index].skey,
-				       &keyparms->key[16], 8);
-				memcpy(psecuritypriv->
-				       dot118021XGrprxmickey[key_index].skey,
-				       &keyparms->key[24], 8);
-
-				psecuritypriv->busetkipkey = 1;
-
-			} else if (keyparms->cipher == WLAN_CIPHER_SUITE_CCMP) {
-					DBG_8723A("%s, set group_key, CCMP\n",
-					  __func__);
-
-				psecuritypriv->dot118021XGrpPrivacy =
-					WLAN_CIPHER_SUITE_CCMP;
-
-				memcpy(psecuritypriv->
-				       dot118021XGrpKey[key_index].skey,
-				       keyparms->key,
-				       (min(16, key_len)));
-			} else {
-				DBG_8723A("%s, set group_key, none\n",
-					  __func__);
-
-				psecuritypriv->dot118021XGrpPrivacy = 0;
-			}
-
-			psecuritypriv->dot118021XGrpKeyid = key_index;
-
-			psecuritypriv->binstallGrpkey = 1;
-
-			psecuritypriv->dot11PrivacyAlgrthm =
-				psecuritypriv->dot118021XGrpPrivacy;
-
-			set_group_key(padapter, keyparms,
-				      psecuritypriv->dot118021XGrpPrivacy,
-				      key_index);
-
-			pbcmc_sta = rtw_get_bcmc_stainfo23a(padapter);
-			if (pbcmc_sta) {
-				pbcmc_sta->ieee8021x_blocked = false;
-				/* rx will use bmc_sta's dot118021XPrivacy */
-				pbcmc_sta->dot118021XPrivacy =
-					psecuritypriv->dot118021XGrpPrivacy;
-
-			}
-
-		}
-
-		goto exit;
-	}
-
-	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X && psta) {
-		/*  psk/802_1x */
-		if (set_tx == 1) {
-			/* pairwise key */
-			memcpy(psta->dot118021x_UncstKey.skey,
-			       keyparms->key, (min(16, key_len)));
-
-			if (keyparms->cipher == WLAN_CIPHER_SUITE_WEP40 ||
-			    keyparms->cipher == WLAN_CIPHER_SUITE_WEP104) {
-				DBG_8723A("%s, set pairwise key, WEP\n",
-					  __func__);
-
-				psecuritypriv->dot118021XGrpPrivacy =
-					keyparms->cipher;
-			} else if (keyparms->cipher == WLAN_CIPHER_SUITE_TKIP) {
-				DBG_8723A("%s, set pairwise key, TKIP\n",
-					  __func__);
-
-				psta->dot118021XPrivacy =
-					WLAN_CIPHER_SUITE_TKIP;
-
-				/* set mic key */
-				memcpy(psta->dot11tkiptxmickey.skey,
-				       &keyparms->key[16], 8);
-				memcpy(psta->dot11tkiprxmickey.skey,
-				       &keyparms->key[24], 8);
-
-				psecuritypriv->busetkipkey = 1;
-
-			} else if (keyparms->cipher == WLAN_CIPHER_SUITE_CCMP) {
-				DBG_8723A("%s, set pairwise key, CCMP\n",
-					  __func__);
-
-				psta->dot118021XPrivacy =
-					WLAN_CIPHER_SUITE_CCMP;
-			} else {
-				DBG_8723A("%s, set pairwise key, none\n",
-					  __func__);
-
-				psta->dot118021XPrivacy = 0;
-			}
-
-			set_pairwise_key(padapter, psta);
-
-			psta->ieee8021x_blocked = false;
-
-			psta->bpairwise_key_installed = true;
-		} else {	/* group key??? */
-			if (keyparms->cipher == WLAN_CIPHER_SUITE_WEP40 ||
-			    keyparms->cipher == WLAN_CIPHER_SUITE_WEP104) {
-				memcpy(psecuritypriv->
-				       dot118021XGrpKey[key_index].skey,
-				       keyparms->key, key_len);
-
-				psecuritypriv->dot118021XGrpPrivacy =
-					keyparms->cipher;
-			} else if (keyparms->cipher == WLAN_CIPHER_SUITE_TKIP) {
-				psecuritypriv->dot118021XGrpPrivacy =
-					WLAN_CIPHER_SUITE_TKIP;
-
-				memcpy(psecuritypriv->
-				       dot118021XGrpKey[key_index].skey,
-				       keyparms->key,
-				       (min(16, key_len)));
-
-				/* set mic key */
-				memcpy(psecuritypriv->
-				       dot118021XGrptxmickey[key_index].skey,
-				       &keyparms->key[16], 8);
-				memcpy(psecuritypriv->
-				       dot118021XGrprxmickey[key_index].skey,
-				       &keyparms->key[24], 8);
-
-				psecuritypriv->busetkipkey = 1;
-			} else if (keyparms->cipher == WLAN_CIPHER_SUITE_CCMP) {
-				psecuritypriv->dot118021XGrpPrivacy =
-					WLAN_CIPHER_SUITE_CCMP;
-
-				memcpy(psecuritypriv->
-				       dot118021XGrpKey[key_index].skey,
-				       keyparms->key,
-				       (min(16, key_len)));
-			} else {
-				psecuritypriv->dot118021XGrpPrivacy = 0;
-			}
-
-			psecuritypriv->dot118021XGrpKeyid = key_index;
-
-			psecuritypriv->binstallGrpkey = 1;
-
-			psecuritypriv->dot11PrivacyAlgrthm =
-				psecuritypriv->dot118021XGrpPrivacy;
-
-			set_group_key(padapter, keyparms,
-				      psecuritypriv->dot118021XGrpPrivacy,
-				      key_index);
-
-			pbcmc_sta = rtw_get_bcmc_stainfo23a(padapter);
-			if (pbcmc_sta) {
-				/* rx will use bmc_sta's
-				   dot118021XPrivacy */
-				pbcmc_sta->ieee8021x_blocked = false;
-				pbcmc_sta->dot118021XPrivacy =
-					psecuritypriv->dot118021XGrpPrivacy;
-			}
-		}
-	}
-
-exit:
-
-	return 0;
-}
-#endif
-
-static int rtw_cfg80211_set_encryption(struct net_device *dev, u8 key_index,
-				       int set_tx, const u8 *sta_addr,
-				       struct key_params *keyparms)
-{
-	int ret = 0;
-	int key_len;
-	struct rtw_adapter *padapter = netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-
-	DBG_8723A("%s\n", __func__);
-
-	key_len = keyparms->key_len;
-
-	if (keyparms->cipher == WLAN_CIPHER_SUITE_WEP40 ||
-	    keyparms->cipher == WLAN_CIPHER_SUITE_WEP104) {
-		RT_TRACE(_module_rtl871x_ioctl_os_c, _drv_err_,
-			 "wpa_set_encryption, crypt.alg = WEP\n");
-		DBG_8723A("wpa_set_encryption, crypt.alg = WEP\n");
-
-		if (psecuritypriv->bWepDefaultKeyIdxSet == 0) {
-			/* wep default key has not been set, so use this
-			   key index as default key. */
-
-			psecuritypriv->ndisencryptstatus =
-				Ndis802_11Encryption1Enabled;
-			psecuritypriv->dot11PrivacyAlgrthm = keyparms->cipher;
-			psecuritypriv->dot118021XGrpPrivacy = keyparms->cipher;
-
-			psecuritypriv->dot11PrivacyKeyIndex = key_index;
-		}
-
-		memcpy(&psecuritypriv->wep_key[key_index].key,
-		       keyparms->key, key_len);
-
-		psecuritypriv->wep_key[key_index].keylen = key_len;
-
-		rtw_set_key23a(padapter, psecuritypriv, key_index, 0);
-
-		goto exit;
-	}
-
-	if (padapter->securitypriv.dot11AuthAlgrthm ==
-	    dot11AuthAlgrthm_8021X) {	/*  802_1x */
-		struct sta_info *psta, *pbcmc_sta;
-		struct sta_priv *pstapriv = &padapter->stapriv;
-
-		if (check_fwstate(pmlmepriv,
-				  WIFI_STATION_STATE | WIFI_MP_STATE)) {
-			/* sta mode */
-			psta = rtw_get_stainfo23a(pstapriv, get_bssid(pmlmepriv));
-			if (psta == NULL) {
-				DBG_8723A("%s, : Obtain Sta_info fail\n",
-					  __func__);
-			} else {
-				/* Jeff: don't disable ieee8021x_blocked
-				   while clearing key */
-				if (keyparms->cipher != IW_AUTH_CIPHER_NONE &&
-				    keyparms->cipher != 0)
-					psta->ieee8021x_blocked = false;
-
-				if ((padapter->securitypriv.ndisencryptstatus ==
-				     Ndis802_11Encryption2Enabled) ||
-				    (padapter->securitypriv.ndisencryptstatus ==
-				     Ndis802_11Encryption3Enabled)) {
-					psta->dot118021XPrivacy =
-						padapter->securitypriv.
-						dot11PrivacyAlgrthm;
-				}
-
-				if (set_tx == 1) {
-					/* pairwise key */
-					DBG_8723A("%s, : set_tx == 1\n",
-						  __func__);
-
-					memcpy(psta->dot118021x_UncstKey.skey,
-					       keyparms->key,
-					       (min(16, key_len)));
-
-					if (keyparms->cipher ==
-					    WLAN_CIPHER_SUITE_TKIP) {
-						memcpy(psta->dot11tkiptxmickey.
-						       skey,
-						       &keyparms->key[16], 8);
-						memcpy(psta->dot11tkiprxmickey.
-						       skey,
-						       &keyparms->key[24], 8);
-
-						padapter->securitypriv.
-							busetkipkey = 0;
-					}
-					DBG_8723A(" ~~~~set sta key:unicastkey\n");
-
-					rtw_setstakey_cmd23a(padapter,
-							  (unsigned char *)psta,
-							  true);
-				} else {	/* group key */
-					memcpy(padapter->securitypriv.
-					       dot118021XGrpKey[key_index].skey,
-					       keyparms->key,
-					       (min(16, key_len)));
-					memcpy(padapter->securitypriv.
-					       dot118021XGrptxmickey[key_index].
-					       skey, &keyparms->key[16], 8);
-					memcpy(padapter->securitypriv.
-					       dot118021XGrprxmickey[key_index].
-					       skey, &keyparms->key[24], 8);
-					padapter->securitypriv.binstallGrpkey =
-						1;
-					DBG_8723A
-					    (" ~~~~set sta key:groupkey\n");
-
-					padapter->securitypriv.
-					    dot118021XGrpKeyid = key_index;
-
-					rtw_set_key23a(padapter,
-						    &padapter->securitypriv,
-						    key_index, 1);
-				}
-			}
-
-			pbcmc_sta = rtw_get_bcmc_stainfo23a(padapter);
-			if (pbcmc_sta) {
-				/* Jeff: don't disable ieee8021x_blocked
-				   while clearing key */
-				if (keyparms->cipher != IW_AUTH_CIPHER_NONE &&
-				    keyparms->cipher != 0)
-					pbcmc_sta->ieee8021x_blocked = false;
-
-				if ((padapter->securitypriv.ndisencryptstatus ==
-				     Ndis802_11Encryption2Enabled) ||
-				    (padapter->securitypriv.ndisencryptstatus ==
-				     Ndis802_11Encryption3Enabled)) {
-					pbcmc_sta->dot118021XPrivacy =
-					    padapter->securitypriv.
-					    dot11PrivacyAlgrthm;
-				}
-			}
-		}
-	}
-
-exit:
-
-	DBG_8723A("%s, ret =%d\n", __func__, ret);
-
-
-
-	return ret;
-}
-
-static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev,
-				u8 key_index, bool pairwise,
-				const u8 *mac_addr, struct key_params *params)
-{
-	int set_tx, ret = 0;
-	struct wireless_dev *rtw_wdev = wiphy_to_wdev(wiphy);
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	u8 sta_addr[ETH_ALEN];
-
-	DBG_8723A("%s(%s): adding key for %pM\n", __func__, ndev->name,
-		  mac_addr);
-	DBG_8723A("cipher = 0x%x\n", params->cipher);
-	DBG_8723A("key_len = 0x%x\n", params->key_len);
-	DBG_8723A("seq_len = 0x%x\n", params->seq_len);
-	DBG_8723A("key_index =%d\n", key_index);
-	DBG_8723A("pairwise =%d\n", pairwise);
-
-	switch (params->cipher) {
-	case IW_AUTH_CIPHER_NONE:
-	case WLAN_CIPHER_SUITE_WEP40:
-		if (params->key_len != WLAN_KEY_LEN_WEP40) {
-			ret = -EINVAL;
-			goto exit;
-		}
-	case WLAN_CIPHER_SUITE_WEP104:
-		if (params->key_len != WLAN_KEY_LEN_WEP104) {
-			ret = -EINVAL;
-			goto exit;
-		}
-	case WLAN_CIPHER_SUITE_TKIP:
-	case WLAN_CIPHER_SUITE_CCMP:
-		break;
-	default:
-		ret = -ENOTSUPP;
-		goto exit;
-	}
-
-	if (key_index >= WEP_KEYS || params->key_len < 0) {
-		ret = -EINVAL;
-		goto exit;
-	}
-
-	eth_broadcast_addr(sta_addr);
-
-	if (!mac_addr || is_broadcast_ether_addr(mac_addr))
-		set_tx = 0;	/* for wpa/wpa2 group key */
-	else
-		set_tx = 1;	/* for wpa/wpa2 pairwise key */
-
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
-		ret = rtw_cfg80211_set_encryption(ndev, key_index, set_tx,
-						  sta_addr, params);
-	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-#ifdef CONFIG_8723AU_AP_MODE
-		if (mac_addr)
-			ether_addr_copy(sta_addr, mac_addr);
-
-		ret = rtw_cfg80211_ap_set_encryption(ndev, key_index, set_tx,
-						     sta_addr, params);
-#endif
-	} else {
-		DBG_8723A("error! fw_state = 0x%x, iftype =%d\n",
-			  pmlmepriv->fw_state, rtw_wdev->iftype);
-
-	}
-
-exit:
-	return ret;
-}
-
-static int
-cfg80211_rtw_get_key(struct wiphy *wiphy, struct net_device *ndev,
-		     u8 key_index, bool pairwise, const u8 *mac_addr,
-		     void *cookie,
-		     void (*callback) (void *cookie, struct key_params *))
-{
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-	return 0;
-}
-
-static int cfg80211_rtw_del_key(struct wiphy *wiphy, struct net_device *ndev,
-				u8 key_index, bool pairwise,
-				const u8 *mac_addr)
-{
-	struct rtw_adapter *padapter = netdev_priv(ndev);
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-
-	DBG_8723A("%s(%s): key_index =%d\n", __func__, ndev->name, key_index);
-
-	if (key_index == psecuritypriv->dot11PrivacyKeyIndex) {
-		/* clear the flag of wep default key set. */
-		psecuritypriv->bWepDefaultKeyIdxSet = 0;
-	}
-
-	return 0;
-}
-
-static int cfg80211_rtw_set_default_key(struct wiphy *wiphy,
-					struct net_device *ndev, u8 key_index,
-					bool unicast, bool multicast)
-{
-	struct rtw_adapter *padapter = netdev_priv(ndev);
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-
-	DBG_8723A("%s(%s): key_index =%d, unicast =%d, multicast =%d.\n",
-		  __func__, ndev->name, key_index, unicast, multicast);
-
-	if (key_index < NUM_WEP_KEYS &&
-	    (psecuritypriv->dot11PrivacyAlgrthm == WLAN_CIPHER_SUITE_WEP40 ||
-	     psecuritypriv->dot11PrivacyAlgrthm == WLAN_CIPHER_SUITE_WEP104)) {
-		/* set wep default key */
-		psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
-
-		psecuritypriv->dot11PrivacyKeyIndex = key_index;
-
-		psecuritypriv->dot11PrivacyAlgrthm = WLAN_CIPHER_SUITE_WEP40;
-		psecuritypriv->dot118021XGrpPrivacy = WLAN_CIPHER_SUITE_WEP40;
-		if (psecuritypriv->wep_key[key_index].keylen == 13) {
-			psecuritypriv->dot11PrivacyAlgrthm =
-				WLAN_CIPHER_SUITE_WEP104;
-			psecuritypriv->dot118021XGrpPrivacy =
-				WLAN_CIPHER_SUITE_WEP104;
-		}
-
-		/* set the flag to represent that wep default key
-		   has been set */
-		psecuritypriv->bWepDefaultKeyIdxSet = 1;
-	}
-
-	return 0;
-}
-
-static u16 rtw_get_cur_max_rate(struct rtw_adapter *adapter)
-{
-	int i = 0;
-	const u8 *p;
-	u16 rate = 0, max_rate = 0;
-	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct registry_priv *pregistrypriv = &adapter->registrypriv;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct wlan_bssid_ex  *pcur_bss = &pmlmepriv->cur_network.network;
-	struct ieee80211_ht_cap *pht_capie;
-	u8 rf_type = 0;
-	u8 bw_40MHz = 0, short_GI_20 = 0, short_GI_40 = 0;
-	u16 mcs_rate = 0;
-
-	p = cfg80211_find_ie(WLAN_EID_HT_CAPABILITY,
-			     pcur_bss->IEs, pcur_bss->IELength);
-	if (p && p[1] > 0) {
-		pht_capie = (struct ieee80211_ht_cap *)(p + 2);
-
-		memcpy(&mcs_rate, &pht_capie->mcs, 2);
-
-		/* bw_40MHz = (pht_capie->cap_info&
-		   IEEE80211_HT_CAP_SUP_WIDTH_20_40) ? 1:0; */
-		/* cur_bwmod is updated by beacon, pmlmeinfo is
-		   updated by association response */
-		bw_40MHz = (pmlmeext->cur_bwmode &&
-			    (pmlmeinfo->HT_info.ht_param &
-			     IEEE80211_HT_PARAM_CHAN_WIDTH_ANY)) ? 1:0;
-
-		/* short_GI = (pht_capie->cap_info & (IEEE80211_HT_CAP
-		   _SGI_20|IEEE80211_HT_CAP_SGI_40)) ? 1 : 0; */
-		short_GI_20 = (pmlmeinfo->ht_cap.cap_info &
-			       cpu_to_le16(IEEE80211_HT_CAP_SGI_20)) ? 1:0;
-		short_GI_40 = (pmlmeinfo->ht_cap.cap_info &
-			       cpu_to_le16(IEEE80211_HT_CAP_SGI_40)) ? 1:0;
-
-		rf_type = rtl8723a_get_rf_type(adapter);
-		max_rate = rtw_mcs_rate23a(rf_type, bw_40MHz &
-					   pregistrypriv->cbw40_enable,
-					   short_GI_20, short_GI_40,
-					   &pmlmeinfo->ht_cap.mcs);
-	} else {
-		while (pcur_bss->SupportedRates[i] != 0 &&
-		       pcur_bss->SupportedRates[i] != 0xFF) {
-			rate = pcur_bss->SupportedRates[i] & 0x7F;
-			if (rate > max_rate)
-				max_rate = rate;
-			i++;
-		}
-
-		max_rate = max_rate * 10 / 2;
-	}
-
-	return max_rate;
-}
-
-static int cfg80211_rtw_get_station(struct wiphy *wiphy,
-				    struct net_device *ndev,
-				    const u8 *mac, struct station_info *sinfo)
-{
-	int ret = 0;
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct sta_info *psta = NULL;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-
-	sinfo->filled = 0;
-
-	if (!mac) {
-		DBG_8723A("%s(%s): mac ==%p\n", __func__, ndev->name, mac);
-		ret = -ENOENT;
-		goto exit;
-	}
-
-	psta = rtw_get_stainfo23a(pstapriv, mac);
-	if (psta == NULL) {
-		DBG_8723A("%s, sta_info is null\n", __func__);
-		ret = -ENOENT;
-		goto exit;
-	}
-	DBG_8723A("%s(%s): mac=%pM\n", __func__, ndev->name, mac);
-
-	/* for infra./P2PClient mode */
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) &&
-	    check_fwstate(pmlmepriv, _FW_LINKED)) {
-		struct wlan_network *cur_network = &pmlmepriv->cur_network;
-
-		if (!ether_addr_equal(mac, cur_network->network.MacAddress)) {
-			DBG_8723A("%s, mismatch bssid=%pM\n",
-				  __func__, cur_network->network.MacAddress);
-			ret = -ENOENT;
-			goto exit;
-		}
-
-		sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL);
-		sinfo->signal = translate_percentage_to_dbm(padapter->recvpriv.
-							    signal_strength);
-
-		sinfo->filled |= BIT(NL80211_STA_INFO_TX_BITRATE);
-		sinfo->txrate.legacy = rtw_get_cur_max_rate(padapter);
-
-		sinfo->filled |= BIT(NL80211_STA_INFO_RX_PACKETS);
-		sinfo->rx_packets = sta_rx_data_pkts(psta);
-
-		sinfo->filled |= BIT(NL80211_STA_INFO_TX_PACKETS);
-		sinfo->tx_packets = psta->sta_stats.tx_pkts;
-	}
-
-	/* for Ad-Hoc/AP mode */
-	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
-	     check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-	     check_fwstate(pmlmepriv, WIFI_AP_STATE)) &&
-	    check_fwstate(pmlmepriv, _FW_LINKED)
-	    ) {
-		/* TODO: should acquire station info... */
-	}
-
-exit:
-	return ret;
-}
-
-static int cfg80211_infrastructure_mode(struct rtw_adapter *padapter,
-				 enum nl80211_iftype ifmode)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
-	enum nl80211_iftype old_mode;
-
-	old_mode = cur_network->network.ifmode;
-
-	RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_notice_,
-		 "+%s: old =%d new =%d fw_state = 0x%08x\n", __func__,
-		 old_mode, ifmode, get_fwstate(pmlmepriv));
-
-	if (old_mode != ifmode) {
-		spin_lock_bh(&pmlmepriv->lock);
-
-		RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-			 "change mode!\n");
-
-		if (old_mode == NL80211_IFTYPE_AP ||
-		    old_mode == NL80211_IFTYPE_P2P_GO) {
-			/* change to other mode from Ndis802_11APMode */
-			cur_network->join_res = -1;
-
-#ifdef CONFIG_8723AU_AP_MODE
-			stop_ap_mode23a(padapter);
-#endif
-		}
-
-		if (check_fwstate(pmlmepriv, _FW_LINKED) ||
-		    old_mode == NL80211_IFTYPE_ADHOC)
-			rtw_disassoc_cmd23a(padapter, 0, true);
-
-		if (check_fwstate(pmlmepriv, _FW_LINKED) ||
-		    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE))
-			rtw_free_assoc_resources23a(padapter, 1);
-
-		if (old_mode == NL80211_IFTYPE_STATION ||
-		    old_mode == NL80211_IFTYPE_P2P_CLIENT ||
-		    old_mode == NL80211_IFTYPE_ADHOC) {
-			if (check_fwstate(pmlmepriv, _FW_LINKED)) {
-				/* will clr Linked_state; before this function,
-				   we must have chked whether issue
-				   dis-assoc_cmd or not */
-				rtw_indicate_disconnect23a(padapter);
-			}
-	       }
-
-		cur_network->network.ifmode = ifmode;
-
-		_clr_fwstate_(pmlmepriv, ~WIFI_NULL_STATE);
-
-		switch (ifmode) {
-		case NL80211_IFTYPE_ADHOC:
-			set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
-			break;
-
-		case NL80211_IFTYPE_P2P_CLIENT:
-		case NL80211_IFTYPE_STATION:
-			set_fwstate(pmlmepriv, WIFI_STATION_STATE);
-			break;
-
-		case NL80211_IFTYPE_P2P_GO:
-		case NL80211_IFTYPE_AP:
-			set_fwstate(pmlmepriv, WIFI_AP_STATE);
-#ifdef CONFIG_8723AU_AP_MODE
-			start_ap_mode23a(padapter);
-			/* rtw_indicate_connect23a(padapter); */
-#endif
-			break;
-
-		default:
-			break;
-		}
-
-		/* SecClearAllKeys(adapter); */
-
-		spin_unlock_bh(&pmlmepriv->lock);
-	}
-
-	return _SUCCESS;
-}
-
-static int cfg80211_rtw_change_iface(struct wiphy *wiphy,
-				     struct net_device *ndev,
-				     enum nl80211_iftype type, u32 *flags,
-				     struct vif_params *params)
-{
-	enum nl80211_iftype old_type;
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct wireless_dev *rtw_wdev = wiphy_to_wdev(wiphy);
-	int ret = 0;
-
-	DBG_8723A("%s(%s): call netdev_open23a\n", __func__, ndev->name);
-
-	old_type = rtw_wdev->iftype;
-	DBG_8723A("%s(%s): old_iftype =%d, new_iftype =%d\n",
-		  __func__, ndev->name, old_type, type);
-
-	if (old_type != type) {
-		pmlmeext->action_public_rxseq = 0xffff;
-		pmlmeext->action_public_dialog_token = 0xff;
-	}
-
-	switch (type) {
-	case NL80211_IFTYPE_ADHOC:
-	case NL80211_IFTYPE_P2P_CLIENT:
-	case NL80211_IFTYPE_STATION:
-	case NL80211_IFTYPE_P2P_GO:
-	case NL80211_IFTYPE_AP:
-	case NL80211_IFTYPE_UNSPECIFIED:
-		break;
-	default:
-		return -EOPNOTSUPP;
-	}
-
-	rtw_wdev->iftype = type;
-
-	if (cfg80211_infrastructure_mode(padapter, type) != _SUCCESS) {
-		rtw_wdev->iftype = old_type;
-		ret = -EPERM;
-		goto exit;
-	}
-
-	rtw_setopmode_cmd23a(padapter, type);
-
-exit:
-	return ret;
-}
-
-void rtw_cfg80211_indicate_scan_done(struct rtw_wdev_priv *pwdev_priv,
-				     bool aborted)
-{
-	spin_lock_bh(&pwdev_priv->scan_req_lock);
-	if (pwdev_priv->scan_request != NULL) {
-		DBG_8723A("%s with scan req\n", __func__);
-
-		if (pwdev_priv->scan_request->wiphy !=
-		    pwdev_priv->rtw_wdev->wiphy) {
-			DBG_8723A("error wiphy compare\n");
-		} else {
-			struct cfg80211_scan_info info = {
-				.aborted = aborted,
-			};
-
-			cfg80211_scan_done(pwdev_priv->scan_request, &info);
-		}
-
-		pwdev_priv->scan_request = NULL;
-	} else {
-		DBG_8723A("%s without scan req\n", __func__);
-	}
-	spin_unlock_bh(&pwdev_priv->scan_req_lock);
-}
-
-void rtw_cfg80211_surveydone_event_callback(struct rtw_adapter *padapter)
-{
-	struct list_head *phead;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct rtw_queue *queue = &pmlmepriv->scanned_queue;
-	struct wlan_network *pnetwork, *ptmp;
-
-	spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-	phead = get_list_head(queue);
-	list_for_each_entry_safe(pnetwork, ptmp, phead, list) {
-		/* report network only if the current channel set
-		   contains the channel to which this network belongs */
-		if (rtw_ch_set_search_ch23a
-		    (padapter->mlmeextpriv.channel_set,
-		     pnetwork->network.DSConfig) >= 0)
-			rtw_cfg80211_inform_bss(padapter, pnetwork);
-	}
-	spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-
-	/* call this after other things have been done */
-	rtw_cfg80211_indicate_scan_done(wdev_to_priv(padapter->rtw_wdev),
-					false);
-}
-
-static int rtw_cfg80211_set_probe_req_wpsp2pie(struct rtw_adapter *padapter,
-					       char *buf, int len)
-{
-	int ret = 0;
-	const u8 *wps_ie;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	DBG_8723A("%s, ielen =%d\n", __func__, len);
-
-	if (len > 0) {
-		wps_ie = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-						 WLAN_OUI_TYPE_MICROSOFT_WPS,
-						 buf, len);
-		if (wps_ie) {
-			DBG_8723A("probe_req_wps_ielen =%d\n", wps_ie[1]);
-
-			if (pmlmepriv->wps_probe_req_ie) {
-				pmlmepriv->wps_probe_req_ie_len = 0;
-				kfree(pmlmepriv->wps_probe_req_ie);
-				pmlmepriv->wps_probe_req_ie = NULL;
-			}
-
-			pmlmepriv->wps_probe_req_ie = kmemdup(wps_ie, wps_ie[1],
-							      GFP_KERNEL);
-			if (pmlmepriv->wps_probe_req_ie == NULL) {
-				DBG_8723A("%s()-%d: kmalloc() ERROR!\n",
-					  __func__, __LINE__);
-				return -EINVAL;
-			}
-			pmlmepriv->wps_probe_req_ie_len = wps_ie[1];
-		}
-	}
-
-	return ret;
-}
-
-static int cfg80211_rtw_scan(struct wiphy *wiphy,
-			     struct cfg80211_scan_request *request)
-{
-	int i;
-	u8 _status = false;
-	int ret = 0;
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct cfg80211_ssid ssid[RTW_SSID_SCAN_AMOUNT];
-	struct rtw_ieee80211_channel ch[RTW_CHANNEL_SCAN_AMOUNT];
-	struct rtw_wdev_priv *pwdev_priv = wdev_to_priv(padapter->rtw_wdev);
-	struct cfg80211_ssid *ssids = request->ssids;
-	bool need_indicate_scan_done = false;
-
-	DBG_8723A("%s(%s)\n", __func__, padapter->pnetdev->name);
-
-	spin_lock_bh(&pwdev_priv->scan_req_lock);
-	pwdev_priv->scan_request = request;
-	spin_unlock_bh(&pwdev_priv->scan_req_lock);
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		DBG_8723A("%s under WIFI_AP_STATE\n", __func__);
-		/* need_indicate_scan_done = true; */
-		/* goto check_need_indicate_scan_done; */
-	}
-
-	if (rtw_pwr_wakeup(padapter) == _FAIL) {
-		need_indicate_scan_done = true;
-		goto check_need_indicate_scan_done;
-	}
-
-	if (request->ie && request->ie_len > 0) {
-		rtw_cfg80211_set_probe_req_wpsp2pie(padapter,
-						    (u8 *) request->ie,
-						    request->ie_len);
-	}
-
-	if (pmlmepriv->LinkDetectInfo.bBusyTraffic == true) {
-		DBG_8723A("%s, bBusyTraffic == true\n", __func__);
-		need_indicate_scan_done = true;
-		goto check_need_indicate_scan_done;
-	}
-	if (rtw_is_scan_deny(padapter)) {
-		DBG_8723A("%s(%s): scan deny\n", __func__,
-			  padapter->pnetdev->name);
-		need_indicate_scan_done = true;
-		goto check_need_indicate_scan_done;
-	}
-
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY | _FW_UNDER_LINKING) ==
-	    true) {
-		DBG_8723A("%s, fwstate = 0x%x\n", __func__, pmlmepriv->fw_state);
-		need_indicate_scan_done = true;
-		goto check_need_indicate_scan_done;
-	}
-
-	memset(ssid, 0, sizeof(struct cfg80211_ssid) * RTW_SSID_SCAN_AMOUNT);
-	/* parsing request ssids, n_ssids */
-	for (i = 0; i < request->n_ssids && i < RTW_SSID_SCAN_AMOUNT; i++) {
-		DBG_8723A("ssid =%s, len =%d\n", ssids[i].ssid,
-			  ssids[i].ssid_len);
-		memcpy(ssid[i].ssid, ssids[i].ssid, ssids[i].ssid_len);
-		ssid[i].ssid_len = ssids[i].ssid_len;
-	}
-
-	/* parsing channels, n_channels */
-	memset(ch, 0,
-	       sizeof(struct rtw_ieee80211_channel) * RTW_CHANNEL_SCAN_AMOUNT);
-
-	if (request->n_channels == 1) {
-		for (i = 0; i < request->n_channels &&
-		     i < RTW_CHANNEL_SCAN_AMOUNT; i++) {
-			DBG_8723A("%s:(%s):" CHAN_FMT "\n",
-				  __func__, padapter->pnetdev->name,
-				  CHAN_ARG(request->channels[i]));
-			ch[i].hw_value = request->channels[i]->hw_value;
-			ch[i].flags = request->channels[i]->flags;
-		}
-	}
-
-	spin_lock_bh(&pmlmepriv->lock);
-	if (request->n_channels == 1) {
-		memcpy(&ch[1], &ch[0], sizeof(struct rtw_ieee80211_channel));
-		memcpy(&ch[2], &ch[0], sizeof(struct rtw_ieee80211_channel));
-		_status = rtw_sitesurvey_cmd23a(padapter, ssid,
-					     RTW_SSID_SCAN_AMOUNT, ch, 3);
-	} else {
-		_status = rtw_sitesurvey_cmd23a(padapter, ssid,
-					     RTW_SSID_SCAN_AMOUNT, NULL, 0);
-	}
-	spin_unlock_bh(&pmlmepriv->lock);
-
-	if (_status == false)
-		ret = -1;
-
-check_need_indicate_scan_done:
-	if (need_indicate_scan_done)
-		rtw_cfg80211_surveydone_event_callback(padapter);
-	return ret;
-}
-
-static int cfg80211_rtw_set_wiphy_params(struct wiphy *wiphy, u32 changed)
-{
-	DBG_8723A("%s\n", __func__);
-	return 0;
-}
-
-static int cfg80211_rtw_join_ibss(struct wiphy *wiphy, struct net_device *ndev,
-				  struct cfg80211_ibss_params *params)
-{
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-	return 0;
-}
-
-static int cfg80211_rtw_leave_ibss(struct wiphy *wiphy, struct net_device *ndev)
-{
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-	return 0;
-}
-
-static int rtw_cfg80211_set_wpa_version(struct security_priv *psecuritypriv,
-					u32 wpa_version)
-{
-	DBG_8723A("%s, wpa_version =%d\n", __func__, wpa_version);
-
-	if (!wpa_version) {
-		psecuritypriv->ndisauthtype = Ndis802_11AuthModeOpen;
-		return 0;
-	}
-
-	if (wpa_version & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2))
-		psecuritypriv->ndisauthtype = Ndis802_11AuthModeWPAPSK;
-
-/*
-	if (wpa_version & NL80211_WPA_VERSION_2)
-	{
-		psecuritypriv->ndisauthtype = Ndis802_11AuthModeWPA2PSK;
-	}
-*/
-
-	return 0;
-}
-
-static int rtw_cfg80211_set_auth_type(struct security_priv *psecuritypriv,
-				      enum nl80211_auth_type sme_auth_type)
-{
-	DBG_8723A("%s, nl80211_auth_type =%d\n", __func__, sme_auth_type);
-
-	switch (sme_auth_type) {
-	case NL80211_AUTHTYPE_AUTOMATIC:
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
-
-		break;
-	case NL80211_AUTHTYPE_OPEN_SYSTEM:
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
-
-		if (psecuritypriv->ndisauthtype > Ndis802_11AuthModeWPA)
-			psecuritypriv->dot11AuthAlgrthm =
-				dot11AuthAlgrthm_8021X;
-		break;
-	case NL80211_AUTHTYPE_SHARED_KEY:
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Shared;
-
-		psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
-		break;
-	default:
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
-		/* return -ENOTSUPP; */
-	}
-
-	return 0;
-}
-
-static int rtw_cfg80211_set_cipher(struct security_priv *psecuritypriv,
-				   u32 cipher, bool ucast)
-{
-	u32 ndisencryptstatus = Ndis802_11EncryptionDisabled;
-
-	u32 *profile_cipher = ucast ? &psecuritypriv->dot11PrivacyAlgrthm :
-	    &psecuritypriv->dot118021XGrpPrivacy;
-
-	DBG_8723A("%s, ucast =%d, cipher = 0x%x\n", __func__, ucast, cipher);
-
-	if (!cipher) {
-		*profile_cipher = 0;
-		psecuritypriv->ndisencryptstatus = ndisencryptstatus;
-		return 0;
-	}
-
-	switch (cipher) {
-	case IW_AUTH_CIPHER_NONE:
-		*profile_cipher = 0;
-		ndisencryptstatus = Ndis802_11EncryptionDisabled;
-		break;
-	case WLAN_CIPHER_SUITE_WEP40:
-		*profile_cipher = WLAN_CIPHER_SUITE_WEP40;
-		ndisencryptstatus = Ndis802_11Encryption1Enabled;
-		break;
-	case WLAN_CIPHER_SUITE_WEP104:
-		*profile_cipher = WLAN_CIPHER_SUITE_WEP104;
-		ndisencryptstatus = Ndis802_11Encryption1Enabled;
-		break;
-	case WLAN_CIPHER_SUITE_TKIP:
-		*profile_cipher = WLAN_CIPHER_SUITE_TKIP;
-		ndisencryptstatus = Ndis802_11Encryption2Enabled;
-		break;
-	case WLAN_CIPHER_SUITE_CCMP:
-		*profile_cipher = WLAN_CIPHER_SUITE_CCMP;
-		ndisencryptstatus = Ndis802_11Encryption3Enabled;
-		break;
-	default:
-		DBG_8723A("Unsupported cipher: 0x%x\n", cipher);
-		return -ENOTSUPP;
-	}
-
-	if (ucast)
-		psecuritypriv->ndisencryptstatus = ndisencryptstatus;
-
-	return 0;
-}
-
-static int rtw_cfg80211_set_key_mgt(struct security_priv *psecuritypriv,
-				    u32 key_mgt)
-{
-	DBG_8723A("%s, key_mgt = 0x%x\n", __func__, key_mgt);
-
-	if (key_mgt == WLAN_AKM_SUITE_8021X)
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
-	else if (key_mgt == WLAN_AKM_SUITE_PSK)
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
-	else
-		DBG_8723A("Invalid key mgt: 0x%x\n", key_mgt);
-
-	return 0;
-}
-
-static int rtw_cfg80211_set_wpa_ie(struct rtw_adapter *padapter, const u8 *pie,
-				   size_t ielen)
-{
-	const u8 *wps_ie;
-	int group_cipher = 0, pairwise_cipher = 0;
-	int ret = 0;
-	const u8 *pwpa, *pwpa2;
-	int i;
-
-	if (!pie || !ielen) {
-		/* Treat this as normal case, but need to clear
-		   WIFI_UNDER_WPS */
-		_clr_fwstate_(&padapter->mlmepriv, WIFI_UNDER_WPS);
-		goto exit;
-	}
-	if (ielen > MAX_WPA_IE_LEN + MAX_WPS_IE_LEN + MAX_P2P_IE_LEN) {
-		ret = -EINVAL;
-		goto exit;
-	}
-
-	/* dump */
-	DBG_8723A("set wpa_ie(length:%zu):\n", ielen);
-	for (i = 0; i < ielen; i = i + 8)
-		DBG_8723A("0x%.2x 0x%.2x 0x%.2x 0x%.2x "
-			  "0x%.2x 0x%.2x 0x%.2x 0x%.2x\n",
-			  pie[i], pie[i + 1], pie[i + 2], pie[i + 3],
-			  pie[i + 4], pie[i + 5], pie[i + 6], pie[i + 7]);
-	if (ielen < RSN_HEADER_LEN) {
-		RT_TRACE(_module_rtl871x_ioctl_os_c, _drv_err_,
-			 "Ie len too short %d\n", (int)ielen);
-		ret = -1;
-		goto exit;
-	}
-
-	pwpa = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-				       WLAN_OUI_TYPE_MICROSOFT_WPA,
-				       pie, ielen);
-	if (pwpa && pwpa[1] > 0) {
-		if (rtw_parse_wpa_ie23a(pwpa, pwpa[1] + 2, &group_cipher,
-					&pairwise_cipher, NULL) == _SUCCESS) {
-			padapter->securitypriv.dot11AuthAlgrthm =
-				dot11AuthAlgrthm_8021X;
-			padapter->securitypriv.ndisauthtype =
-				Ndis802_11AuthModeWPAPSK;
-			memcpy(padapter->securitypriv.supplicant_ie, pwpa,
-			       pwpa[1] + 2);
-
-			DBG_8723A("got wpa_ie, wpa_ielen:%u\n", pwpa[1]);
-		}
-	}
-
-	pwpa2 = cfg80211_find_ie(WLAN_EID_RSN, pie, ielen);
-	if (pwpa2 && pwpa2[1] > 0) {
-		if (rtw_parse_wpa2_ie23a (pwpa2, pwpa2[1] + 2, &group_cipher,
-					  &pairwise_cipher, NULL) == _SUCCESS) {
-			padapter->securitypriv.dot11AuthAlgrthm =
-				dot11AuthAlgrthm_8021X;
-			padapter->securitypriv.ndisauthtype =
-				Ndis802_11AuthModeWPA2PSK;
-			memcpy(padapter->securitypriv.supplicant_ie, pwpa2,
-			       pwpa2[1] + 2);
-
-			DBG_8723A("got wpa2_ie, wpa2_ielen:%u\n", pwpa2[1]);
-		}
-	}
-
-	if (group_cipher == 0) {
-		group_cipher = WPA_CIPHER_NONE;
-	}
-	if (pairwise_cipher == 0) {
-		pairwise_cipher = WPA_CIPHER_NONE;
-	}
-
-	switch (group_cipher) {
-	case WPA_CIPHER_NONE:
-		padapter->securitypriv.dot118021XGrpPrivacy = 0;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11EncryptionDisabled;
-		break;
-	case WPA_CIPHER_WEP40:
-		padapter->securitypriv.dot118021XGrpPrivacy = WLAN_CIPHER_SUITE_WEP40;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11Encryption1Enabled;
-		break;
-	case WPA_CIPHER_TKIP:
-		padapter->securitypriv.dot118021XGrpPrivacy = WLAN_CIPHER_SUITE_TKIP;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11Encryption2Enabled;
-		break;
-	case WPA_CIPHER_CCMP:
-		padapter->securitypriv.dot118021XGrpPrivacy = WLAN_CIPHER_SUITE_CCMP;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11Encryption3Enabled;
-		break;
-	case WPA_CIPHER_WEP104:
-		padapter->securitypriv.dot118021XGrpPrivacy = WLAN_CIPHER_SUITE_WEP104;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11Encryption1Enabled;
-		break;
-	}
-
-	switch (pairwise_cipher) {
-	case WPA_CIPHER_NONE:
-		padapter->securitypriv.dot11PrivacyAlgrthm = 0;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11EncryptionDisabled;
-		break;
-	case WPA_CIPHER_WEP40:
-		padapter->securitypriv.dot11PrivacyAlgrthm = WLAN_CIPHER_SUITE_WEP40;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11Encryption1Enabled;
-		break;
-	case WPA_CIPHER_TKIP:
-		padapter->securitypriv.dot11PrivacyAlgrthm = WLAN_CIPHER_SUITE_TKIP;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11Encryption2Enabled;
-		break;
-	case WPA_CIPHER_CCMP:
-		padapter->securitypriv.dot11PrivacyAlgrthm = WLAN_CIPHER_SUITE_CCMP;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11Encryption3Enabled;
-		break;
-	case WPA_CIPHER_WEP104:
-		padapter->securitypriv.dot11PrivacyAlgrthm = WLAN_CIPHER_SUITE_WEP104;
-		padapter->securitypriv.ndisencryptstatus =
-			Ndis802_11Encryption1Enabled;
-		break;
-	}
-
-	wps_ie = cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-					 WLAN_OUI_TYPE_MICROSOFT_WPS,
-					 pie, ielen);
-	if (wps_ie && wps_ie[1] > 0) {
-		DBG_8723A("got wps_ie, wps_ielen:%u\n", wps_ie[1]);
-		padapter->securitypriv.wps_ie_len = wps_ie[1];
-		memcpy(padapter->securitypriv.wps_ie, wps_ie,
-		       padapter->securitypriv.wps_ie_len);
-		set_fwstate(&padapter->mlmepriv, WIFI_UNDER_WPS);
-	} else {
-		_clr_fwstate_(&padapter->mlmepriv, WIFI_UNDER_WPS);
-	}
-
-	/* TKIP and AES disallow multicast packets until installing group key */
-	if (padapter->securitypriv.dot11PrivacyAlgrthm ==
-	    WLAN_CIPHER_SUITE_TKIP ||
-	    padapter->securitypriv.dot11PrivacyAlgrthm ==
-	    WLAN_CIPHER_SUITE_CCMP)
-		/* WPS open need to enable multicast */
-		/* check_fwstate(&padapter->mlmepriv, WIFI_UNDER_WPS) == true)*/
-		rtl8723a_off_rcr_am(padapter);
-
-	RT_TRACE(_module_rtl871x_ioctl_os_c, _drv_info_,
-		 "rtw_set_wpa_ie: pairwise_cipher = 0x%08x padapter->securitypriv.ndisencryptstatus =%d padapter->securitypriv.ndisauthtype =%d\n",
-		 pairwise_cipher,
-		 padapter->securitypriv.ndisencryptstatus,
-		 padapter->securitypriv.ndisauthtype);
-
-exit:
-	if (ret)
-		_clr_fwstate_(&padapter->mlmepriv, WIFI_UNDER_WPS);
-	return ret;
-}
-
-static int rtw_cfg80211_add_wep(struct rtw_adapter *padapter,
-				struct rtw_wep_key *wep, u8 keyid)
-{
-	int res;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-
-	if (keyid >= NUM_WEP_KEYS) {
-		RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_err_,
-			 "%s:keyid>4 =>fail\n", __func__);
-		res = _FAIL;
-		goto exit;
-	}
-
-	switch (wep->keylen) {
-	case WLAN_KEY_LEN_WEP40:
-		psecuritypriv->dot11PrivacyAlgrthm = WLAN_CIPHER_SUITE_WEP40;
-		RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-			 "%s:wep->KeyLength = 5\n", __func__);
-		break;
-	case WLAN_KEY_LEN_WEP104:
-		psecuritypriv->dot11PrivacyAlgrthm = WLAN_CIPHER_SUITE_WEP104;
-		RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-			 "%s:wep->KeyLength = 13\n", __func__);
-		break;
-	default:
-		psecuritypriv->dot11PrivacyAlgrthm = 0;
-		RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-			 "%s:wep->KeyLength!= 5 or 13\n", __func__);
-		res = _FAIL;
-		goto exit;
-	}
-
-	RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-		 "%s:before memcpy, wep->KeyLength = 0x%x keyid =%x\n",
-		 __func__, wep->keylen, keyid);
-
-	memcpy(&psecuritypriv->wep_key[keyid], wep, sizeof(struct rtw_wep_key));
-
-	psecuritypriv->dot11PrivacyKeyIndex = keyid;
-
-	RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-		 "%s:security key material : %x %x %x %x %x %x %x %x %x %x %x %x %x\n",
-		 __func__,
-		 psecuritypriv->wep_key[keyid].key[0],
-		 psecuritypriv->wep_key[keyid].key[1],
-		 psecuritypriv->wep_key[keyid].key[2],
-		 psecuritypriv->wep_key[keyid].key[3],
-		 psecuritypriv->wep_key[keyid].key[4],
-		 psecuritypriv->wep_key[keyid].key[5],
-		 psecuritypriv->wep_key[keyid].key[6],
-		 psecuritypriv->wep_key[keyid].key[7],
-		 psecuritypriv->wep_key[keyid].key[8],
-		 psecuritypriv->wep_key[keyid].key[9],
-		 psecuritypriv->wep_key[keyid].key[10],
-		 psecuritypriv->wep_key[keyid].key[11],
-		 psecuritypriv->wep_key[keyid].key[12]);
-
-	res = rtw_set_key23a(padapter, psecuritypriv, keyid, 1);
-
-exit:
-
-	return res;
-}
-
-static int rtw_set_ssid(struct rtw_adapter *padapter,
-			struct wlan_network *newnetwork)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network *pnetwork = &pmlmepriv->cur_network;
-	int status = _SUCCESS;
-	u32 cur_time = 0;
-
-	DBG_8723A_LEVEL(_drv_always_, "set ssid [%s] fw_state = 0x%08x\n",
-			newnetwork->network.Ssid.ssid, get_fwstate(pmlmepriv));
-
-	if (padapter->hw_init_completed == false) {
-		RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_err_,
-			 "set_ssid: hw_init_completed == false =>exit!!!\n");
-		status = _FAIL;
-		goto exit;
-	}
-
-	spin_lock_bh(&pmlmepriv->lock);
-
-	DBG_8723A("Set SSID under fw_state = 0x%08x\n", get_fwstate(pmlmepriv));
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
-		goto handle_tkip_countermeasure;
-
-	if (check_fwstate(pmlmepriv, _FW_LINKED|WIFI_ADHOC_MASTER_STATE)) {
-		RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-			 "set_ssid: _FW_LINKED||WIFI_ADHOC_MASTER_STATE\n");
-
-		if (pmlmepriv->assoc_ssid.ssid_len ==
-		    newnetwork->network.Ssid.ssid_len &&
-		    !memcmp(&pmlmepriv->assoc_ssid.ssid,
-			    newnetwork->network.Ssid.ssid,
-			    newnetwork->network.Ssid.ssid_len)) {
-			if (!check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
-				RT_TRACE(_module_rtl871x_ioctl_set_c_,
-					 _drv_err_,
-					 "New SSID is same SSID, fw_state = 0x%08x\n",
-					 get_fwstate(pmlmepriv));
-
-				if (rtw_is_same_ibss23a(padapter, pnetwork)) {
-					/*
-					 * it means driver is in
-					 * WIFI_ADHOC_MASTER_STATE, we needn't
-					 * create bss again.
-					 */
-					goto release_mlme_lock;
-				}
-
-				/*
-				 * if in WIFI_ADHOC_MASTER_STATE |
-				 * WIFI_ADHOC_STATE, create bss or
-				 * rejoin again
-				 */
-				rtw_disassoc_cmd23a(padapter, 0, true);
-
-				if (check_fwstate(pmlmepriv, _FW_LINKED))
-					rtw_indicate_disconnect23a(padapter);
-
-				rtw_free_assoc_resources23a(padapter, 1);
-
-				if (check_fwstate(pmlmepriv,
-						  WIFI_ADHOC_MASTER_STATE)) {
-					_clr_fwstate_(pmlmepriv,
-						      WIFI_ADHOC_MASTER_STATE);
-					set_fwstate(pmlmepriv,
-						    WIFI_ADHOC_STATE);
-				}
-			} else {
-				rtw_lps_ctrl_wk_cmd23a(padapter,
-						       LPS_CTRL_JOINBSS, 1);
-			}
-		} else {
-			RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-				 "Set SSID not the same ssid\n");
-			RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-				 "set_ssid =[%s] len = 0x%x\n",
-				 newnetwork->network.Ssid.ssid,
-				 newnetwork->network.Ssid.ssid_len);
-			RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_,
-				 "assoc_ssid =[%s] len = 0x%x\n",
-				 pmlmepriv->assoc_ssid.ssid,
-				 pmlmepriv->assoc_ssid.ssid_len);
-
-			rtw_disassoc_cmd23a(padapter, 0, true);
-
-			if (check_fwstate(pmlmepriv, _FW_LINKED))
-				rtw_indicate_disconnect23a(padapter);
-
-			rtw_free_assoc_resources23a(padapter, 1);
-
-			if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
-				_clr_fwstate_(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
-				set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
-			}
-		}
-	}
-
-handle_tkip_countermeasure:
-
-	if (padapter->securitypriv.btkip_countermeasure == true) {
-		cur_time = jiffies;
-
-		if ((cur_time -
-		     padapter->securitypriv.btkip_countermeasure_time) >
-		    60 * HZ) {
-			padapter->securitypriv.btkip_countermeasure = false;
-			padapter->securitypriv.btkip_countermeasure_time = 0;
-		} else {
-			status = _FAIL;
-			goto release_mlme_lock;
-		}
-	}
-
-	memcpy(&pmlmepriv->assoc_ssid, &newnetwork->network.Ssid,
-	       sizeof(struct cfg80211_ssid));
-
-	pmlmepriv->assoc_by_bssid = false;
-
-	pmlmepriv->to_join = true;
-
-	if (!check_fwstate(pmlmepriv, _FW_UNDER_SURVEY)) {
-		pmlmepriv->cur_network.join_res = -2;
-
-		status = rtw_do_join_network(padapter, newnetwork);
-		if (status == _SUCCESS) {
-			pmlmepriv->to_join = false;
-		} else {
-			if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
-				/* switch to ADHOC_MASTER */
-				status = rtw_do_join_adhoc(padapter);
-				if (status != _SUCCESS)
-					goto release_mlme_lock;
-			} else {
-				/* can't associate ; reset under-linking */
-				_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
-				status = _FAIL;
-				pmlmepriv->to_join = false;
-			}
-		}
-	}
-release_mlme_lock:
-	spin_unlock_bh(&pmlmepriv->lock);
-
-exit:
-	RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_err_,
-		 "-%s: status =%d\n", __func__, status);
-
-	return status;
-}
-
-static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev,
-				struct cfg80211_connect_params *sme)
-{
-	int ret = 0;
-	struct list_head *phead, *plist, *ptmp;
-	struct wlan_network *pnetwork = NULL;
-	/* u8 matched_by_bssid = false; */
-	/* u8 matched_by_ssid = false; */
-	u8 matched = false;
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	struct rtw_queue *queue = &pmlmepriv->scanned_queue;
-
-	DBG_8723A("=>" "%s(%s)\n", __func__, ndev->name);
-	DBG_8723A("privacy =%d, key =%p, key_len =%d, key_idx =%d\n",
-		  sme->privacy, sme->key, sme->key_len, sme->key_idx);
-
-	if (_FAIL == rtw_pwr_wakeup(padapter)) {
-		ret = -EPERM;
-		goto exit;
-	}
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-		ret = -EPERM;
-		goto exit;
-	}
-
-	if (!sme->ssid || !sme->ssid_len ||
-	    sme->ssid_len > IEEE80211_MAX_SSID_LEN) {
-		ret = -EINVAL;
-		goto exit;
-	}
-
-	DBG_8723A("ssid =%s, len =%zu\n", sme->ssid, sme->ssid_len);
-
-	if (sme->bssid)
-		DBG_8723A("bssid=%pM\n", sme->bssid);
-
-	if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) {
-		ret = -EBUSY;
-		DBG_8723A("%s, fw_state = 0x%x, goto exit\n", __func__,
-			  pmlmepriv->fw_state);
-		goto exit;
-	}
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY)) {
-		rtw_scan_abort23a(padapter);
-	}
-
-	spin_lock_bh(&queue->lock);
-
-	phead = get_list_head(queue);
-
-	list_for_each_safe(plist, ptmp, phead) {
-		pnetwork = container_of(plist, struct wlan_network, list);
-
-		if (sme->bssid) {
-			if (!ether_addr_equal(pnetwork->network.MacAddress,
-					      sme->bssid))
-				continue;
-		}
-
-		if (sme->ssid && sme->ssid_len) {
-			if (pnetwork->network.Ssid.ssid_len != sme->ssid_len ||
-			    memcmp(pnetwork->network.Ssid.ssid, sme->ssid,
-				   sme->ssid_len))
-				continue;
-		}
-
-		if (sme->bssid) {
-			if (ether_addr_equal(pnetwork->network.MacAddress,
-					     sme->bssid)) {
-				DBG_8723A("matched by bssid\n");
-
-				matched = true;
-				break;
-			}
-		} else if (sme->ssid && sme->ssid_len) {
-			if (!memcmp(pnetwork->network.Ssid.ssid,
-				    sme->ssid, sme->ssid_len) &&
-			    pnetwork->network.Ssid.ssid_len == sme->ssid_len) {
-				DBG_8723A("matched by ssid\n");
-
-				matched = true;
-				break;
-			}
-		}
-	}
-
-	spin_unlock_bh(&queue->lock);
-
-	if (!matched || !pnetwork) {
-		ret = -ENOENT;
-		DBG_8723A("connect, matched == false, goto exit\n");
-		goto exit;
-	}
-
-	if (cfg80211_infrastructure_mode(
-		    padapter, pnetwork->network.ifmode) != _SUCCESS) {
-		ret = -EPERM;
-		goto exit;
-	}
-
-	psecuritypriv->ndisencryptstatus = Ndis802_11EncryptionDisabled;
-	psecuritypriv->dot11PrivacyAlgrthm = 0;
-	psecuritypriv->dot118021XGrpPrivacy = 0;
-	psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
-	psecuritypriv->ndisauthtype = Ndis802_11AuthModeOpen;
-
-	ret = rtw_cfg80211_set_wpa_version(psecuritypriv,
-					   sme->crypto.wpa_versions);
-	if (ret < 0)
-		goto exit;
-
-	ret = rtw_cfg80211_set_auth_type(psecuritypriv, sme->auth_type);
-
-	if (ret < 0)
-		goto exit;
-
-	DBG_8723A("%s, ie_len =%zu\n", __func__, sme->ie_len);
-
-	ret = rtw_cfg80211_set_wpa_ie(padapter, sme->ie, sme->ie_len);
-	if (ret < 0)
-		goto exit;
-
-	if (sme->crypto.n_ciphers_pairwise) {
-		ret = rtw_cfg80211_set_cipher(psecuritypriv,
-					      sme->crypto.ciphers_pairwise[0],
-					      true);
-		if (ret < 0)
-			goto exit;
-	}
-
-	/* For WEP Shared auth */
-	if ((psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_Shared ||
-	     psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_Auto) &&
-	    sme->key) {
-		struct rtw_wep_key wep_key;
-		u8 wep_key_idx, wep_key_len;
-		DBG_8723A("%s(): Shared/Auto WEP\n", __func__);
-
-		wep_key_idx = sme->key_idx;
-		wep_key_len = sme->key_len;
-
-		if (wep_key_idx > WEP_KEYS || !wep_key_len ||
-		    wep_key_len > WLAN_KEY_LEN_WEP104) {
-			ret = -EINVAL;
-			goto exit;
-		}
-
-		wep_key_len = wep_key_len <= 5 ? 5 : 13;
-
-		memset(&wep_key, 0, sizeof(struct rtw_wep_key));
-
-		wep_key.keylen = wep_key_len;
-
-		if (wep_key_len == 13) {
-			padapter->securitypriv.dot11PrivacyAlgrthm =
-				WLAN_CIPHER_SUITE_WEP104;
-			padapter->securitypriv.dot118021XGrpPrivacy =
-				WLAN_CIPHER_SUITE_WEP104;
-		} else {
-			padapter->securitypriv.dot11PrivacyAlgrthm =
-				WLAN_CIPHER_SUITE_WEP40;
-			padapter->securitypriv.dot118021XGrpPrivacy =
-				WLAN_CIPHER_SUITE_WEP40;
-		}
-
-		memcpy(wep_key.key, (void *)sme->key, wep_key.keylen);
-
-		if (rtw_cfg80211_add_wep(padapter, &wep_key, wep_key_idx) !=
-		    _SUCCESS)
-			ret = -EOPNOTSUPP;
-
-		if (ret < 0)
-			goto exit;
-	}
-
-	ret = rtw_cfg80211_set_cipher(psecuritypriv,
-				      sme->crypto.cipher_group, false);
-	if (ret < 0)
-		goto exit;
-
-	if (sme->crypto.n_akm_suites) {
-		ret = rtw_cfg80211_set_key_mgt(psecuritypriv,
-					       sme->crypto.akm_suites[0]);
-		if (ret < 0)
-			goto exit;
-	}
-
-	if (psecuritypriv->ndisauthtype > 3)
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
-
-	if (rtw_set_auth23a(padapter, psecuritypriv) != _SUCCESS) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	/* rtw_set_802_11_encryption_mode(padapter,
-	   padapter->securitypriv.ndisencryptstatus); */
-
-	if (rtw_set_ssid(padapter, pnetwork) != _SUCCESS) {
-		ret = -EBUSY;
-		goto exit;
-	}
-
-	DBG_8723A("set ssid:dot11AuthAlgrthm =%d, dot11PrivacyAlgrthm =%d, "
-		  "dot118021XGrpPrivacy =%d\n", psecuritypriv->dot11AuthAlgrthm,
-		  psecuritypriv->dot11PrivacyAlgrthm,
-		  psecuritypriv->dot118021XGrpPrivacy);
-
-exit:
-
-	DBG_8723A("<=%s, ret %d\n", __func__, ret);
-
-	return ret;
-}
-
-static int cfg80211_rtw_disconnect(struct wiphy *wiphy, struct net_device *ndev,
-				   u16 reason_code)
-{
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-
-	rtw_set_roaming(padapter, 0);
-
-	if (check_fwstate(&padapter->mlmepriv, _FW_LINKED)) {
-		rtw_scan_abort23a(padapter);
-		LeaveAllPowerSaveMode23a(padapter);
-		rtw_disassoc_cmd23a(padapter, 500, false);
-
-		DBG_8723A("%s...call rtw_indicate_disconnect23a\n", __func__);
-
-		padapter->mlmepriv.not_indic_disco = true;
-		rtw_indicate_disconnect23a(padapter);
-		padapter->mlmepriv.not_indic_disco = false;
-
-		rtw_free_assoc_resources23a(padapter, 1);
-	}
-
-	return 0;
-}
-
-static int cfg80211_rtw_set_txpower(struct wiphy *wiphy,
-				    struct wireless_dev *wdev,
-				    enum nl80211_tx_power_setting type, int mbm)
-{
-	DBG_8723A("%s\n", __func__);
-	return 0;
-}
-
-static int cfg80211_rtw_get_txpower(struct wiphy *wiphy,
-				    struct wireless_dev *wdev, int *dbm)
-{
-	DBG_8723A("%s\n", __func__);
-	*dbm = 12;
-	return 0;
-}
-
-inline bool rtw_cfg80211_pwr_mgmt(struct rtw_adapter *adapter)
-{
-	struct rtw_wdev_priv *rtw_wdev_priv = wdev_to_priv(adapter->rtw_wdev);
-	return rtw_wdev_priv->power_mgmt;
-}
-
-static int cfg80211_rtw_set_power_mgmt(struct wiphy *wiphy,
-				       struct net_device *ndev,
-				       bool enabled, int timeout)
-{
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-	struct rtw_wdev_priv *rtw_wdev_priv = wdev_to_priv(padapter->rtw_wdev);
-
-	DBG_8723A("%s(%s): enabled:%u, timeout:%d\n",
-		  __func__, ndev->name, enabled, timeout);
-
-	rtw_wdev_priv->power_mgmt = enabled;
-
-	if (!enabled)
-		LPS_Leave23a(padapter);
-
-	return 0;
-}
-
-static int cfg80211_rtw_set_pmksa(struct wiphy *wiphy,
-				  struct net_device *netdev,
-				  struct cfg80211_pmksa *pmksa)
-{
-	u8 index, blInserted = false;
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-
-	DBG_8723A("%s(%s)\n", __func__, netdev->name);
-
-	if (is_zero_ether_addr(pmksa->bssid))
-		return -EINVAL;
-
-	blInserted = false;
-
-	/* overwrite PMKID */
-	for (index = 0; index < NUM_PMKID_CACHE; index++) {
-		if (ether_addr_equal(psecuritypriv->PMKIDList[index].Bssid,
-				     pmksa->bssid)) {
-			/* BSSID is matched, the same AP => rewrite with
-			   new PMKID. */
-			DBG_8723A("%s(%s):  BSSID exists in the PMKList.\n",
-				  __func__, netdev->name);
-
-			memcpy(psecuritypriv->PMKIDList[index].PMKID,
-			       pmksa->pmkid, WLAN_PMKID_LEN);
-			psecuritypriv->PMKIDList[index].bUsed = true;
-			psecuritypriv->PMKIDIndex = index + 1;
-			blInserted = true;
-			break;
-		}
-	}
-
-	if (!blInserted) {
-		/*  Find a new entry */
-		DBG_8723A("%s(%s): Use new entry index = %d for this PMKID\n",
-			  __func__, netdev->name, psecuritypriv->PMKIDIndex);
-
-		ether_addr_copy(
-			psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].
-			Bssid, pmksa->bssid);
-		memcpy(psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].
-		       PMKID, pmksa->pmkid, WLAN_PMKID_LEN);
-
-		psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].bUsed =
-			true;
-		psecuritypriv->PMKIDIndex++;
-		if (psecuritypriv->PMKIDIndex == 16) {
-			psecuritypriv->PMKIDIndex = 0;
-		}
-	}
-
-	return 0;
-}
-
-static int cfg80211_rtw_del_pmksa(struct wiphy *wiphy,
-				  struct net_device *netdev,
-				  struct cfg80211_pmksa *pmksa)
-{
-	u8 index, bMatched = false;
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-
-	DBG_8723A("%s(%s)\n", __func__, netdev->name);
-
-	for (index = 0; index < NUM_PMKID_CACHE; index++) {
-		if (ether_addr_equal(psecuritypriv->PMKIDList[index].Bssid,
-				     pmksa->bssid)) {
-			/* BSSID is matched, the same AP => Remove this PMKID
-			   information and reset it. */
-			eth_zero_addr(psecuritypriv->PMKIDList[index].Bssid);
-			memset(psecuritypriv->PMKIDList[index].PMKID, 0x00,
-			       WLAN_PMKID_LEN);
-			psecuritypriv->PMKIDList[index].bUsed = false;
-			bMatched = true;
-			break;
-		}
-	}
-
-	if (false == bMatched) {
-		DBG_8723A("%s(%s): do not have matched BSSID\n", __func__,
-			  netdev->name);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int cfg80211_rtw_flush_pmksa(struct wiphy *wiphy,
-				    struct net_device *netdev)
-{
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-
-	DBG_8723A("%s(%s)\n", __func__, netdev->name);
-
-	memset(&psecuritypriv->PMKIDList[0], 0x00,
-	       sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE);
-	psecuritypriv->PMKIDIndex = 0;
-
-	return 0;
-}
-
-#ifdef CONFIG_8723AU_AP_MODE
-void rtw_cfg80211_indicate_sta_assoc(struct rtw_adapter *padapter,
-				     u8 *pmgmt_frame, uint frame_len)
-{
-	s32 freq;
-	int channel;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct net_device *ndev = padapter->pnetdev;
-
-	DBG_8723A("%s(padapter =%p,%s)\n", __func__, padapter, ndev->name);
-
-#if defined(RTW_USE_CFG80211_STA_EVENT)
-	{
-		struct station_info sinfo;
-		u8 ie_offset;
-
-		if (ieee80211_is_assoc_req(hdr->frame_control))
-			ie_offset = offsetof(struct ieee80211_mgmt,
-					     u.assoc_req.variable);
-		else		/*  WIFI_REASSOCREQ */
-			ie_offset = offsetof(struct ieee80211_mgmt,
-					     u.reassoc_req.variable);
-
-		sinfo.filled = 0;
-		sinfo.assoc_req_ies = pmgmt_frame + ie_offset;
-		sinfo.assoc_req_ies_len = frame_len - ie_offset;
-		cfg80211_new_sta(ndev, hdr->addr2, &sinfo, GFP_ATOMIC);
-	}
-#else /* defined(RTW_USE_CFG80211_STA_EVENT) */
-	channel = pmlmeext->cur_channel;
-	if (channel <= RTW_CH_MAX_2G_CHANNEL)
-		freq = ieee80211_channel_to_frequency(channel,
-						      NL80211_BAND_2GHZ);
-	else
-		freq = ieee80211_channel_to_frequency(channel,
-						      NL80211_BAND_5GHZ);
-
-	cfg80211_rx_mgmt(padapter->rtw_wdev, freq, 0, pmgmt_frame, frame_len,
-			 0);
-#endif /* defined(RTW_USE_CFG80211_STA_EVENT) */
-}
-
-void rtw_cfg80211_indicate_sta_disassoc(struct rtw_adapter *padapter,
-					unsigned char *da,
-					unsigned short reason)
-{
-	s32 freq;
-	int channel;
-	uint frame_len;
-	struct ieee80211_mgmt mgmt;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct net_device *ndev = padapter->pnetdev;
-
-	DBG_8723A("%s(padapter =%p,%s)\n", __func__, padapter, ndev->name);
-
-	memset(&mgmt, 0, sizeof(struct ieee80211_mgmt));
-
-#if defined(RTW_USE_CFG80211_STA_EVENT)
-	cfg80211_del_sta(ndev, da, GFP_ATOMIC);
-#else /* defined(RTW_USE_CFG80211_STA_EVENT) */
-	channel = pmlmeext->cur_channel;
-	if (channel <= RTW_CH_MAX_2G_CHANNEL)
-		freq = ieee80211_channel_to_frequency(channel,
-						      NL80211_BAND_2GHZ);
-	else
-		freq = ieee80211_channel_to_frequency(channel,
-						      NL80211_BAND_5GHZ);
-
-	mgmt.frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_DEAUTH);
-
-	ether_addr_copy(mgmt.da, myid(&padapter->eeprompriv));
-	ether_addr_copy(mgmt.sa, da);
-	ether_addr_copy(mgmt.bssid, get_my_bssid23a(&pmlmeinfo->network));
-
-	mgmt.seq_ctrl = cpu_to_le16(IEEE80211_SN_TO_SEQ(pmlmeext->mgnt_seq));
-	pmlmeext->mgnt_seq++;
-
-	mgmt.u.disassoc.reason_code = cpu_to_le16(reason);
-
-	frame_len = sizeof(struct ieee80211_hdr_3addr) + 2;
-
-	cfg80211_rx_mgmt(padapter->rtw_wdev, freq, 0, (u8 *)&mgmt, frame_len,
-			 0);
-#endif /* defined(RTW_USE_CFG80211_STA_EVENT) */
-}
-
-static int rtw_cfg80211_monitor_if_open(struct net_device *ndev)
-{
-	DBG_8723A("%s\n", __func__);
-
-	return 0;
-}
-
-static int rtw_cfg80211_monitor_if_close(struct net_device *ndev)
-{
-	DBG_8723A("%s\n", __func__);
-
-	return 0;
-}
-
-static int rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb,
-					      struct net_device *ndev)
-{
-	int ret = 0;
-	int rtap_len;
-	int qos_len = 0;
-	int dot11_hdr_len = 24;
-	int snap_len = 6;
-	unsigned char *pdata;
-	unsigned char src_mac_addr[6];
-	unsigned char dst_mac_addr[6];
-	struct ieee80211_hdr *dot11_hdr;
-	struct ieee80211_radiotap_header *rtap_hdr;
-	struct rtw_adapter *padapter = netdev_priv(ndev);
-
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-
-	if (unlikely(skb->len < sizeof(struct ieee80211_radiotap_header)))
-		goto fail;
-
-	rtap_hdr = (struct ieee80211_radiotap_header *)skb->data;
-	if (unlikely(rtap_hdr->it_version))
-		goto fail;
-
-	rtap_len = ieee80211_get_radiotap_len(skb->data);
-	if (unlikely(skb->len < rtap_len))
-		goto fail;
-
-	if (rtap_len != 14) {
-		DBG_8723A("radiotap len (should be 14): %d\n", rtap_len);
-		goto fail;
-	}
-
-	/* Skip the ratio tap header */
-	skb_pull(skb, rtap_len);
-
-	dot11_hdr = (struct ieee80211_hdr *)skb->data;
-	/* Check if the QoS bit is set */
-	if (ieee80211_is_data(dot11_hdr->frame_control)) {
-		/* Check if this ia a Wireless Distribution System (WDS) frame
-		 * which has 4 MAC addresses
-		 */
-		if (ieee80211_is_data_qos(dot11_hdr->frame_control))
-			qos_len = IEEE80211_QOS_CTL_LEN;
-		if (ieee80211_has_a4(dot11_hdr->frame_control))
-			dot11_hdr_len += 6;
-
-		memcpy(dst_mac_addr, dot11_hdr->addr1, sizeof(dst_mac_addr));
-		memcpy(src_mac_addr, dot11_hdr->addr2, sizeof(src_mac_addr));
-
-		/*
-		 * Skip the 802.11 header, QoS (if any) and SNAP,
-		 * but leave spaces for two MAC addresses
-		 */
-		skb_pull(skb, dot11_hdr_len + qos_len + snap_len -
-			 ETH_ALEN * 2);
-		pdata = (unsigned char *)skb->data;
-		ether_addr_copy(pdata, dst_mac_addr);
-		ether_addr_copy(pdata + ETH_ALEN, src_mac_addr);
-
-		DBG_8723A("should be eapol packet\n");
-
-		/* Use the real net device to transmit the packet */
-		ret = rtw_xmit23a_entry23a(skb, padapter->pnetdev);
-
-		return ret;
-
-	} else if (ieee80211_is_action(dot11_hdr->frame_control)) {
-		struct ieee80211_mgmt *mgmt;
-		/* only for action frames */
-		struct xmit_frame *pmgntframe;
-		struct pkt_attrib *pattrib;
-		unsigned char *pframe;
-		/* u8 category, action, OUI_Subtype, dialogToken = 0; */
-		/* unsigned char        *frame_body; */
-		struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-		struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-		u32 len = skb->len;
-		u8 category, action;
-
-		mgmt = (struct ieee80211_mgmt *)dot11_hdr;
-
-		DBG_8723A("RTW_Tx:da=%pM via %s(%s)\n",
-			  mgmt->da, __func__, ndev->name);
-		category = mgmt->u.action.category;
-		action = mgmt->u.action.u.wme_action.action_code;
-		DBG_8723A("RTW_Tx:category(%u), action(%u)\n",
-			  category, action);
-
-		/* starting alloc mgmt frame to dump it */
-		pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-		if (pmgntframe == NULL)
-			goto fail;
-
-		/* update attribute */
-		pattrib = &pmgntframe->attrib;
-		update_mgntframe_attrib23a(padapter, pattrib);
-		pattrib->retry_ctrl = false;
-
-		memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-		pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
-
-		memcpy(pframe, skb->data, len);
-		pattrib->pktlen = len;
-
-		/* update seq number */
-		pmlmeext->mgnt_seq = le16_to_cpu(dot11_hdr->seq_ctrl) >> 4;
-		pattrib->seqnum = pmlmeext->mgnt_seq;
-		pmlmeext->mgnt_seq++;
-
-		pattrib->last_txcmdsz = pattrib->pktlen;
-
-		dump_mgntframe23a(padapter, pmgntframe);
-	}
-
-fail:
-
-	dev_kfree_skb(skb);
-
-	return 0;
-}
-
-static int
-rtw_cfg80211_monitor_if_set_mac_address(struct net_device *ndev, void *addr)
-{
-	DBG_8723A("%s\n", __func__);
-
-	return 0;
-}
-
-static const struct net_device_ops rtw_cfg80211_monitor_if_ops = {
-	.ndo_open = rtw_cfg80211_monitor_if_open,
-	.ndo_stop = rtw_cfg80211_monitor_if_close,
-	.ndo_start_xmit = rtw_cfg80211_monitor_if_xmit_entry,
-	.ndo_set_mac_address = rtw_cfg80211_monitor_if_set_mac_address,
-};
-
-static int rtw_cfg80211_add_monitor_if(struct rtw_adapter *padapter, char *name,
-				       unsigned char name_assign_type,
-				       struct net_device **ndev)
-{
-	int ret = 0;
-	struct net_device *mon_ndev = NULL;
-	struct wireless_dev *mon_wdev = NULL;
-	struct rtw_wdev_priv *pwdev_priv = wdev_to_priv(padapter->rtw_wdev);
-
-	if (!name) {
-		DBG_8723A("%s(%s): without specific name\n",
-			  __func__, padapter->pnetdev->name);
-		ret = -EINVAL;
-		goto out;
-	}
-
-	if (pwdev_priv->pmon_ndev) {
-		DBG_8723A("%s(%s): monitor interface exist: %s\n", __func__,
-			  padapter->pnetdev->name, pwdev_priv->pmon_ndev->name);
-		ret = -EBUSY;
-		goto out;
-	}
-
-	mon_ndev = alloc_etherdev(sizeof(struct rtw_adapter));
-	if (!mon_ndev) {
-		DBG_8723A("%s(%s): allocate ndev fail\n", __func__,
-			  padapter->pnetdev->name);
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	mon_ndev->type = ARPHRD_IEEE80211_RADIOTAP;
-	strncpy(mon_ndev->name, name, IFNAMSIZ);
-	mon_ndev->name[IFNAMSIZ - 1] = 0;
-	mon_ndev->name_assign_type = name_assign_type;
-	mon_ndev->destructor = rtw_ndev_destructor;
-
-	mon_ndev->netdev_ops = &rtw_cfg80211_monitor_if_ops;
-
-	/*  wdev */
-	mon_wdev = kzalloc(sizeof(struct wireless_dev), GFP_KERNEL);
-	if (!mon_wdev) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	mon_wdev->wiphy = padapter->rtw_wdev->wiphy;
-	mon_wdev->netdev = mon_ndev;
-	mon_wdev->iftype = NL80211_IFTYPE_MONITOR;
-	mon_ndev->ieee80211_ptr = mon_wdev;
-
-	ret = register_netdevice(mon_ndev);
-	if (ret) {
-		goto out;
-	}
-
-	*ndev = pwdev_priv->pmon_ndev = mon_ndev;
-	memcpy(pwdev_priv->ifname_mon, name, IFNAMSIZ + 1);
-
-out:
-	if (ret) {
-		kfree(mon_wdev);
-		mon_wdev = NULL;
-	}
-
-	if (ret && mon_ndev) {
-		free_netdev(mon_ndev);
-		*ndev = mon_ndev = NULL;
-	}
-
-	return ret;
-}
-
-static struct wireless_dev *
-cfg80211_rtw_add_virtual_intf(struct wiphy *wiphy, const char *name,
-			      unsigned char name_assign_type,
-			      enum nl80211_iftype type, u32 *flags,
-			      struct vif_params *params)
-{
-	int ret = 0;
-	struct net_device *ndev = NULL;
-	struct rtw_adapter *padapter = wiphy_to_adapter(wiphy);
-
-	DBG_8723A("%s(%s): wiphy:%s, name:%s, type:%d\n", __func__,
-		  padapter->pnetdev->name, wiphy_name(wiphy), name, type);
-
-	switch (type) {
-	case NL80211_IFTYPE_ADHOC:
-	case NL80211_IFTYPE_AP_VLAN:
-	case NL80211_IFTYPE_WDS:
-	case NL80211_IFTYPE_MESH_POINT:
-		ret = -ENODEV;
-		break;
-	case NL80211_IFTYPE_MONITOR:
-		ret =
-		    rtw_cfg80211_add_monitor_if(padapter, (char *)name,
-						name_assign_type, &ndev);
-		break;
-
-	case NL80211_IFTYPE_P2P_CLIENT:
-	case NL80211_IFTYPE_STATION:
-		ret = -ENODEV;
-		break;
-
-	case NL80211_IFTYPE_P2P_GO:
-	case NL80211_IFTYPE_AP:
-		ret = -ENODEV;
-		break;
-	default:
-		ret = -ENODEV;
-		DBG_8723A("Unsupported interface type\n");
-		break;
-	}
-
-	DBG_8723A("%s(%s): ndev:%p, ret:%d\n", __func__,
-		  padapter->pnetdev->name,
-		  ndev, ret);
-
-	return ndev ? ndev->ieee80211_ptr : ERR_PTR(ret);
-}
-
-static int cfg80211_rtw_del_virtual_intf(struct wiphy *wiphy,
-					 struct wireless_dev *wdev)
-{
-	struct rtw_wdev_priv *pwdev_priv =
-	    (struct rtw_wdev_priv *)wiphy_priv(wiphy);
-	struct net_device *ndev;
-	ndev = wdev ? wdev->netdev : NULL;
-
-	if (!ndev)
-		goto exit;
-
-	unregister_netdevice(ndev);
-
-	if (ndev == pwdev_priv->pmon_ndev) {
-		pwdev_priv->pmon_ndev = NULL;
-		pwdev_priv->ifname_mon[0] = '\0';
-		DBG_8723A("%s(%s): remove monitor interface\n",
-			  __func__, ndev->name);
-	}
-
-exit:
-	return 0;
-}
-
-static int rtw_add_beacon(struct rtw_adapter *adapter, const u8 *head,
-			  size_t head_len, const u8 *tail, size_t tail_len)
-{
-	int ret = 0;
-	u8 *pbuf;
-	uint len, ielen, wps_ielen = 0;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct wlan_bssid_ex *bss = &pmlmepriv->cur_network.network;
-	const struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)head;
-	struct ieee80211_mgmt *tmpmgmt;
-	/* struct sta_priv *pstapriv = &padapter->stapriv; */
-
-	DBG_8723A("%s beacon_head_len =%zu, beacon_tail_len =%zu\n",
-		  __func__, head_len, tail_len);
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != true)
-		return -EINVAL;
-
-	if (head_len < offsetof(struct ieee80211_mgmt, u.beacon.variable))
-		return -EINVAL;
-
-	pbuf = kzalloc(head_len + tail_len, GFP_KERNEL);
-	if (!pbuf)
-		return -ENOMEM;
-	tmpmgmt = (struct ieee80211_mgmt *)pbuf;
-
-	bss->beacon_interval = get_unaligned_le16(&mgmt->u.beacon.beacon_int);
-	bss->capability = get_unaligned_le16(&mgmt->u.beacon.capab_info);
-	bss->tsf = get_unaligned_le64(&mgmt->u.beacon.timestamp);
-
-	/*  24 = beacon header len. */
-	memcpy(pbuf, (void *)head, head_len);
-	memcpy(pbuf + head_len, (void *)tail, tail_len);
-
-	len = head_len + tail_len;
-	ielen = len - offsetof(struct ieee80211_mgmt, u.beacon.variable);
-	/* check wps ie if inclued */
-	if (cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT,
-				    WLAN_OUI_TYPE_MICROSOFT_WPS,
-				    tmpmgmt->u.beacon.variable, ielen))
-		DBG_8723A("add bcn, wps_ielen =%d\n", wps_ielen);
-
-	/* pbss_network->IEs will not include p2p_ie, wfd ie */
-	rtw_ies_remove_ie23a(tmpmgmt->u.beacon.variable, &ielen, 0,
-			     WLAN_EID_VENDOR_SPECIFIC, P2P_OUI23A, 4);
-	rtw_ies_remove_ie23a(tmpmgmt->u.beacon.variable, &ielen, 0,
-			     WLAN_EID_VENDOR_SPECIFIC, WFD_OUI23A, 4);
-
-	len = ielen + offsetof(struct ieee80211_mgmt, u.beacon.variable);
-	if (rtw_check_beacon_data23a(adapter, tmpmgmt, len) == _SUCCESS) {
-		ret = 0;
-	} else {
-		ret = -EINVAL;
-	}
-
-	kfree(pbuf);
-
-	return ret;
-}
-
-static int cfg80211_rtw_start_ap(struct wiphy *wiphy, struct net_device *ndev,
-				 struct cfg80211_ap_settings *settings)
-{
-	int ret = 0;
-	struct rtw_adapter *adapter = wiphy_to_adapter(wiphy);
-
-	DBG_8723A("%s(%s): hidden_ssid:%d, auth_type:%d\n",
-		  __func__, ndev->name, settings->hidden_ssid,
-		  settings->auth_type);
-
-	ret = rtw_add_beacon(adapter, settings->beacon.head,
-			     settings->beacon.head_len, settings->beacon.tail,
-			     settings->beacon.tail_len);
-
-	adapter->mlmeextpriv.mlmext_info.hidden_ssid_mode =
-		settings->hidden_ssid;
-
-	if (settings->ssid && settings->ssid_len) {
-		struct wlan_bssid_ex *pbss_network =
-			&adapter->mlmepriv.cur_network.network;
-		struct wlan_bssid_ex *pbss_network_ext =
-			&adapter->mlmeextpriv.mlmext_info.network;
-
-		memcpy(pbss_network->Ssid.ssid, (void *)settings->ssid,
-		       settings->ssid_len);
-		pbss_network->Ssid.ssid_len = settings->ssid_len;
-		memcpy(pbss_network_ext->Ssid.ssid, (void *)settings->ssid,
-		       settings->ssid_len);
-		pbss_network_ext->Ssid.ssid_len = settings->ssid_len;
-	}
-
-	return ret;
-}
-
-static int cfg80211_rtw_change_beacon(struct wiphy *wiphy,
-				      struct net_device *ndev,
-				      struct cfg80211_beacon_data *info)
-{
-	int ret = 0;
-	struct rtw_adapter *adapter = wiphy_to_adapter(wiphy);
-
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-
-	ret = rtw_add_beacon(adapter, info->head, info->head_len, info->tail,
-			     info->tail_len);
-
-	return ret;
-}
-
-static int cfg80211_rtw_stop_ap(struct wiphy *wiphy, struct net_device *ndev)
-{
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-	return 0;
-}
-
-static int cfg80211_rtw_add_station(struct wiphy *wiphy,
-				    struct net_device *ndev, const u8 *mac,
-				    struct station_parameters *params)
-{
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-
-	return 0;
-}
-
-static int cfg80211_rtw_del_station(struct wiphy *wiphy,
-				    struct net_device *ndev,
-				    struct station_del_parameters *params)
-{
-	const u8 *mac = params->mac;
-	int ret = 0;
-	struct list_head *phead;
-	u8 updated = 0;
-	struct sta_info *psta, *ptmp;
-	struct rtw_adapter *padapter = netdev_priv(ndev);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-
-	DBG_8723A("+%s(%s)\n", __func__, ndev->name);
-
-	if (check_fwstate(pmlmepriv, (_FW_LINKED | WIFI_AP_STATE)) != true) {
-		DBG_8723A("%s, fw_state != FW_LINKED|WIFI_AP_STATE\n",
-			  __func__);
-		return -EINVAL;
-	}
-
-	if (!mac) {
-		DBG_8723A("flush all sta, and cam_entry\n");
-
-		flush_all_cam_entry23a(padapter);	/* clear CAM */
-
-		ret = rtw_sta_flush23a(padapter);
-
-		return ret;
-	}
-
-	DBG_8723A("free sta macaddr=%pM\n", mac);
-
-	if (is_broadcast_ether_addr(mac))
-		return -EINVAL;
-
-	spin_lock_bh(&pstapriv->asoc_list_lock);
-	phead = &pstapriv->asoc_list;
-	/* check asoc_queue */
-	list_for_each_entry_safe(psta, ptmp, phead, asoc_list) {
-		if (ether_addr_equal(mac, psta->hwaddr)) {
-			if (psta->dot8021xalg == 1 &&
-			    psta->bpairwise_key_installed == false) {
-				DBG_8723A("%s, sta's dot8021xalg = 1 and "
-					  "key_installed = false\n", __func__);
-			} else {
-				DBG_8723A("free psta =%p, aid =%d\n", psta,
-					  psta->aid);
-
-				list_del_init(&psta->asoc_list);
-				pstapriv->asoc_list_cnt--;
-
-				/* spin_unlock_bh(&pstapriv->asoc_list_lock); */
-				updated =
-				    ap_free_sta23a(padapter, psta, true,
-						WLAN_REASON_DEAUTH_LEAVING);
-				/* spin_lock_bh(&pstapriv->asoc_list_lock); */
-
-				psta = NULL;
-
-				break;
-			}
-		}
-	}
-	spin_unlock_bh(&pstapriv->asoc_list_lock);
-
-	associated_clients_update23a(padapter, updated);
-
-	DBG_8723A("-%s(%s)\n", __func__, ndev->name);
-
-	return ret;
-}
-
-static int cfg80211_rtw_change_station(struct wiphy *wiphy,
-				       struct net_device *ndev, const u8 *mac,
-				       struct station_parameters *params)
-{
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-	return 0;
-}
-
-static int cfg80211_rtw_dump_station(struct wiphy *wiphy,
-				     struct net_device *ndev, int idx, u8 *mac,
-				     struct station_info *sinfo)
-{
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-
-	/* TODO: dump scanned queue */
-
-	return -ENOENT;
-}
-
-static int cfg80211_rtw_change_bss(struct wiphy *wiphy, struct net_device *ndev,
-				   struct bss_parameters *params)
-{
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-	return 0;
-}
-#endif /* CONFIG_8723AU_AP_MODE */
-
-static int _cfg80211_rtw_mgmt_tx(struct rtw_adapter *padapter, u8 tx_ch,
-				 const u8 *buf, size_t len)
-{
-	struct xmit_frame *pmgntframe;
-	struct pkt_attrib *pattrib;
-	unsigned char *pframe;
-	int ret = _FAIL;
-	struct ieee80211_hdr *pwlanhdr;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-
-	if (_FAIL == rtw_pwr_wakeup(padapter)) {
-		ret = -EFAULT;
-		goto exit;
-	}
-
-	rtw_set_scan_deny(padapter, 1000);
-
-	rtw_scan_abort23a(padapter);
-
-	if (tx_ch != rtw_get_oper_ch23a(padapter)) {
-		if (!check_fwstate(&padapter->mlmepriv, _FW_LINKED))
-			pmlmeext->cur_channel = tx_ch;
-		set_channel_bwmode23a(padapter, tx_ch,
-				   HAL_PRIME_CHNL_OFFSET_DONT_CARE,
-				   HT_CHANNEL_WIDTH_20);
-	}
-
-	/* starting alloc mgmt frame to dump it */
-	pmgntframe = alloc_mgtxmitframe23a(pxmitpriv);
-	if (!pmgntframe) {
-		/* ret = -ENOMEM; */
-		ret = _FAIL;
-		goto exit;
-	}
-
-	/* update attribute */
-	pattrib = &pmgntframe->attrib;
-	update_mgntframe_attrib23a(padapter, pattrib);
-	pattrib->retry_ctrl = false;
-
-	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
-
-	pframe = (u8 *) (pmgntframe->buf_addr) + TXDESC_OFFSET;
-
-	memcpy(pframe, (void *)buf, len);
-	pattrib->pktlen = len;
-
-	pwlanhdr = (struct ieee80211_hdr *)pframe;
-	/* update seq number */
-	pmlmeext->mgnt_seq = le16_to_cpu(pwlanhdr->seq_ctrl) >> 4;
-	pattrib->seqnum = pmlmeext->mgnt_seq;
-	pmlmeext->mgnt_seq++;
-
-	pattrib->last_txcmdsz = pattrib->pktlen;
-
-	ret = dump_mgntframe23a_and_wait_ack23a(padapter, pmgntframe);
-
-	if (ret  != _SUCCESS)
-		DBG_8723A("%s, ack == false\n", __func__);
-	else
-		DBG_8723A("%s, ack == true\n", __func__);
-
-exit:
-
-	DBG_8723A("%s, ret =%d\n", __func__, ret);
-
-	return ret;
-}
-
-static int cfg80211_rtw_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
-				struct cfg80211_mgmt_tx_params *params,
-				u64 *cookie)
-{
-	struct rtw_adapter *padapter =
-		(struct rtw_adapter *)wiphy_to_adapter(wiphy);
-	int ret = 0;
-	int tx_ret;
-	u32 dump_limit = RTW_MAX_MGMT_TX_CNT;
-	u32 dump_cnt = 0;
-	bool ack = true;
-	u8 category, action;
-	unsigned long start = jiffies;
-	size_t len = params->len;
-	struct ieee80211_channel *chan = params->chan;
-	const u8 *buf = params->buf;
-	struct ieee80211_mgmt *hdr = (struct ieee80211_mgmt *)buf;
-	u8 tx_ch = (u8) ieee80211_frequency_to_channel(chan->center_freq);
-
-	if (!ieee80211_is_action(hdr->frame_control))
-		return -EINVAL;
-
-	/* cookie generation */
-	*cookie = (unsigned long)buf;
-
-	DBG_8723A("%s(%s): len =%zu, ch =%d\n", __func__,
-		  padapter->pnetdev->name, len, tx_ch);
-
-	/* indicate ack before issue frame to avoid racing with rsp frame */
-	cfg80211_mgmt_tx_status(padapter->rtw_wdev, *cookie, buf, len, ack,
-				GFP_KERNEL);
-
-	DBG_8723A("RTW_Tx:tx_ch =%d, da =%pM\n", tx_ch, hdr->da);
-	category = hdr->u.action.category;
-	action = hdr->u.action.u.wme_action.action_code;
-	DBG_8723A("RTW_Tx:category(%u), action(%u)\n", category, action);
-
-	do {
-		dump_cnt++;
-		tx_ret = _cfg80211_rtw_mgmt_tx(padapter, tx_ch, buf, len);
-	} while (dump_cnt < dump_limit && tx_ret != _SUCCESS);
-
-	if (tx_ret != _SUCCESS || dump_cnt > 1) {
-		DBG_8723A("%s(%s): %s (%d/%d) in %d ms\n",
-			  __func__, padapter->pnetdev->name,
-			  tx_ret == _SUCCESS ? "OK" : "FAIL", dump_cnt,
-			  dump_limit, jiffies_to_msecs(jiffies - start));
-	}
-
-	return ret;
-}
-
-static void cfg80211_rtw_mgmt_frame_register(struct wiphy *wiphy,
-					     struct wireless_dev *wdev,
-					     u16 frame_type, bool reg)
-{
-	if (frame_type != (IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_PROBE_REQ))
-		return;
-
-	return;
-}
-
-static struct cfg80211_ops rtw_cfg80211_ops = {
-	.change_virtual_intf = cfg80211_rtw_change_iface,
-	.add_key = cfg80211_rtw_add_key,
-	.get_key = cfg80211_rtw_get_key,
-	.del_key = cfg80211_rtw_del_key,
-	.set_default_key = cfg80211_rtw_set_default_key,
-	.get_station = cfg80211_rtw_get_station,
-	.scan = cfg80211_rtw_scan,
-	.set_wiphy_params = cfg80211_rtw_set_wiphy_params,
-	.connect = cfg80211_rtw_connect,
-	.disconnect = cfg80211_rtw_disconnect,
-	.join_ibss = cfg80211_rtw_join_ibss,
-	.leave_ibss = cfg80211_rtw_leave_ibss,
-	.set_tx_power = cfg80211_rtw_set_txpower,
-	.get_tx_power = cfg80211_rtw_get_txpower,
-	.set_power_mgmt = cfg80211_rtw_set_power_mgmt,
-	.set_pmksa = cfg80211_rtw_set_pmksa,
-	.del_pmksa = cfg80211_rtw_del_pmksa,
-	.flush_pmksa = cfg80211_rtw_flush_pmksa,
-
-#ifdef CONFIG_8723AU_AP_MODE
-	.add_virtual_intf = cfg80211_rtw_add_virtual_intf,
-	.del_virtual_intf = cfg80211_rtw_del_virtual_intf,
-
-	.start_ap = cfg80211_rtw_start_ap,
-	.change_beacon = cfg80211_rtw_change_beacon,
-	.stop_ap = cfg80211_rtw_stop_ap,
-
-	.add_station = cfg80211_rtw_add_station,
-	.del_station = cfg80211_rtw_del_station,
-	.change_station = cfg80211_rtw_change_station,
-	.dump_station = cfg80211_rtw_dump_station,
-	.change_bss = cfg80211_rtw_change_bss,
-#endif /* CONFIG_8723AU_AP_MODE */
-
-	.mgmt_tx = cfg80211_rtw_mgmt_tx,
-	.mgmt_frame_register = cfg80211_rtw_mgmt_frame_register,
-};
-
-static void rtw_cfg80211_init_ht_capab(struct ieee80211_sta_ht_cap *ht_cap,
-				       enum nl80211_band band, u8 rf_type)
-{
-
-#define MAX_BIT_RATE_40MHZ_MCS15	300	/* Mbps */
-#define MAX_BIT_RATE_40MHZ_MCS7		150	/* Mbps */
-
-	ht_cap->ht_supported = true;
-
-	ht_cap->cap = IEEE80211_HT_CAP_SUP_WIDTH_20_40 |
-	    IEEE80211_HT_CAP_SGI_40 | IEEE80211_HT_CAP_SGI_20 |
-	    IEEE80211_HT_CAP_DSSSCCK40 | IEEE80211_HT_CAP_MAX_AMSDU;
-
-	/*
-	 *Maximum length of AMPDU that the STA can receive.
-	 *Length = 2 ^ (13 + max_ampdu_length_exp) - 1 (octets)
-	 */
-	ht_cap->ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K;
-
-	/*Minimum MPDU start spacing , */
-	ht_cap->ampdu_density = IEEE80211_HT_MPDU_DENSITY_16;
-
-	ht_cap->mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED;
-
-	/*
-	 *hw->wiphy->bands[NL80211_BAND_2GHZ]
-	 *base on ant_num
-	 *rx_mask: RX mask
-	 *if rx_ant = 1 rx_mask[0]= 0xff;==>MCS0-MCS7
-	 *if rx_ant = 2 rx_mask[1]= 0xff;==>MCS8-MCS15
-	 *if rx_ant >= 3 rx_mask[2]= 0xff;
-	 *if BW_40 rx_mask[4]= 0x01;
-	 *highest supported RX rate
-	 */
-	if (rf_type == RF_1T1R) {
-		ht_cap->mcs.rx_mask[0] = 0xFF;
-		ht_cap->mcs.rx_mask[1] = 0x00;
-		ht_cap->mcs.rx_mask[4] = 0x01;
-
-		ht_cap->mcs.rx_highest = cpu_to_le16(MAX_BIT_RATE_40MHZ_MCS7);
-	} else if ((rf_type == RF_1T2R) || (rf_type == RF_2T2R)) {
-		ht_cap->mcs.rx_mask[0] = 0xFF;
-		ht_cap->mcs.rx_mask[1] = 0xFF;
-		ht_cap->mcs.rx_mask[4] = 0x01;
-
-		ht_cap->mcs.rx_highest = cpu_to_le16(MAX_BIT_RATE_40MHZ_MCS15);
-	} else {
-		DBG_8723A("%s, error rf_type =%d\n", __func__, rf_type);
-	}
-
-}
-
-void rtw_cfg80211_init_wiphy(struct rtw_adapter *padapter)
-{
-	u8 rf_type;
-	struct ieee80211_supported_band *bands;
-	struct wireless_dev *pwdev = padapter->rtw_wdev;
-	struct wiphy *wiphy = pwdev->wiphy;
-
-	rf_type = rtl8723a_get_rf_type(padapter);
-
-	DBG_8723A("%s:rf_type =%d\n", __func__, rf_type);
-
-	/* if (padapter->registrypriv.wireless_mode & WIRELESS_11G) */
-	{
-		bands = wiphy->bands[NL80211_BAND_2GHZ];
-		if (bands)
-			rtw_cfg80211_init_ht_capab(&bands->ht_cap,
-						   NL80211_BAND_2GHZ,
-						   rf_type);
-	}
-
-	/* if (padapter->registrypriv.wireless_mode & WIRELESS_11A) */
-	{
-		bands = wiphy->bands[NL80211_BAND_5GHZ];
-		if (bands)
-			rtw_cfg80211_init_ht_capab(&bands->ht_cap,
-						   NL80211_BAND_5GHZ,
-						   rf_type);
-	}
-}
-
-static void rtw_cfg80211_preinit_wiphy(struct rtw_adapter *padapter,
-				       struct wiphy *wiphy)
-{
-	wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM;
-
-	wiphy->max_scan_ssids = RTW_SSID_SCAN_AMOUNT;
-	wiphy->max_scan_ie_len = IEEE80211_MAX_DATA_LEN;
-	wiphy->max_num_pmkids = RTW_MAX_NUM_PMKIDS;
-
-	wiphy->max_remain_on_channel_duration =
-	    RTW_MAX_REMAIN_ON_CHANNEL_DURATION;
-
-	wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) |
-	    BIT(NL80211_IFTYPE_ADHOC) |
-#ifdef CONFIG_8723AU_AP_MODE
-	    BIT(NL80211_IFTYPE_AP) | BIT(NL80211_IFTYPE_MONITOR) |
-#endif
-	    0;
-
-#ifdef CONFIG_8723AU_AP_MODE
-	wiphy->mgmt_stypes = rtw_cfg80211_default_mgmt_stypes;
-#endif /* CONFIG_8723AU_AP_MODE */
-
-	wiphy->software_iftypes |= BIT(NL80211_IFTYPE_MONITOR);
-
-	/*
-	   wiphy->iface_combinations = &rtw_combinations;
-	   wiphy->n_iface_combinations = 1;
-	 */
-
-	wiphy->cipher_suites = rtw_cipher_suites;
-	wiphy->n_cipher_suites = ARRAY_SIZE(rtw_cipher_suites);
-
-	/* if (padapter->registrypriv.wireless_mode & WIRELESS_11G) */
-	wiphy->bands[NL80211_BAND_2GHZ] =
-	    rtw_spt_band_alloc(NL80211_BAND_2GHZ);
-	/* if (padapter->registrypriv.wireless_mode & WIRELESS_11A) */
-	wiphy->bands[NL80211_BAND_5GHZ] =
-	    rtw_spt_band_alloc(NL80211_BAND_5GHZ);
-
-	wiphy->flags |= WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL;
-	wiphy->flags |= WIPHY_FLAG_OFFCHAN_TX | WIPHY_FLAG_HAVE_AP_SME;
-
-	if (padapter->registrypriv.power_mgnt != PS_MODE_ACTIVE)
-		wiphy->flags |= WIPHY_FLAG_PS_ON_BY_DEFAULT;
-	else
-		wiphy->flags &= ~WIPHY_FLAG_PS_ON_BY_DEFAULT;
-}
-
-int rtw_wdev_alloc(struct rtw_adapter *padapter, struct device *dev)
-{
-	int ret = 0;
-	struct wiphy *wiphy;
-	struct wireless_dev *wdev;
-	struct rtw_wdev_priv *pwdev_priv;
-	struct net_device *pnetdev = padapter->pnetdev;
-
-	DBG_8723A("%s(padapter =%p)\n", __func__, padapter);
-
-	/* wiphy */
-	wiphy = wiphy_new(&rtw_cfg80211_ops, sizeof(struct rtw_wdev_priv));
-	if (!wiphy) {
-		DBG_8723A("Couldn't allocate wiphy device\n");
-		ret = -ENOMEM;
-		goto exit;
-	}
-
-	/*  wdev */
-	wdev = kzalloc(sizeof(struct wireless_dev), GFP_KERNEL);
-	if (!wdev) {
-		ret = -ENOMEM;
-		goto free_wiphy;
-	}
-
-	set_wiphy_dev(wiphy, dev);
-	rtw_cfg80211_preinit_wiphy(padapter, wiphy);
-
-	ret = wiphy_register(wiphy);
-	if (ret < 0) {
-		DBG_8723A("Couldn't register wiphy device\n");
-		goto free_wdev;
-	}
-
-	wdev->wiphy = wiphy;
-	wdev->netdev = pnetdev;
-	/* wdev->iftype = NL80211_IFTYPE_STATION; */
-	/*  for rtw_setopmode_cmd23a() in cfg80211_rtw_change_iface() */
-	wdev->iftype = NL80211_IFTYPE_MONITOR;
-	padapter->rtw_wdev = wdev;
-	pnetdev->ieee80211_ptr = wdev;
-
-	/* init pwdev_priv */
-	pwdev_priv = wdev_to_priv(wdev);
-	pwdev_priv->rtw_wdev = wdev;
-	pwdev_priv->pmon_ndev = NULL;
-	pwdev_priv->ifname_mon[0] = '\0';
-	pwdev_priv->padapter = padapter;
-	pwdev_priv->scan_request = NULL;
-	spin_lock_init(&pwdev_priv->scan_req_lock);
-
-	pwdev_priv->p2p_enabled = false;
-
-	if (padapter->registrypriv.power_mgnt != PS_MODE_ACTIVE)
-		pwdev_priv->power_mgmt = true;
-	else
-		pwdev_priv->power_mgmt = false;
-
-	return ret;
-free_wdev:
-	kfree(wdev);
-free_wiphy:
-	wiphy_free(wiphy);
-exit:
-	return ret;
-}
-
-void rtw_wdev_free(struct wireless_dev *wdev)
-{
-	DBG_8723A("%s(wdev =%p)\n", __func__, wdev);
-
-	if (!wdev)
-		return;
-
-	kfree(wdev->wiphy->bands[NL80211_BAND_2GHZ]);
-	kfree(wdev->wiphy->bands[NL80211_BAND_5GHZ]);
-
-	wiphy_free(wdev->wiphy);
-
-	kfree(wdev);
-}
-
-void rtw_wdev_unregister(struct wireless_dev *wdev)
-{
-	struct rtw_wdev_priv *pwdev_priv;
-
-	DBG_8723A("%s(wdev =%p)\n", __func__, wdev);
-
-	if (!wdev)
-		return;
-
-	pwdev_priv = wdev_to_priv(wdev);
-
-	rtw_cfg80211_indicate_scan_done(pwdev_priv, true);
-
-	if (pwdev_priv->pmon_ndev) {
-		DBG_8723A("%s, unregister monitor interface\n", __func__);
-		unregister_netdev(pwdev_priv->pmon_ndev);
-	}
-
-	wiphy_unregister(wdev->wiphy);
-}
diff --git a/drivers/staging/rtl8723au/os_dep/mlme_linux.c b/drivers/staging/rtl8723au/os_dep/mlme_linux.c
deleted file mode 100644
index ca24369f1208..000000000000
--- a/drivers/staging/rtl8723au/os_dep/mlme_linux.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-
-#define _MLME_OSDEP_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <mlme_osdep.h>
-
-static struct rt_pmkid_list backupPMKIDList[NUM_PMKID_CACHE];
-
-void rtw_reset_securitypriv23a(struct rtw_adapter *adapter)
-{
-	u8	backupPMKIDIndex = 0;
-	u8	backupTKIPCountermeasure = 0x00;
-	unsigned long backupTKIPcountermeasure_time = 0;
-
-	if (adapter->securitypriv.dot11AuthAlgrthm ==
-	    dot11AuthAlgrthm_8021X) { /* 802.1x */
-		/*  We have to backup the PMK information for WiFi PMK
-		 *  Caching test item.
-		 *  Backup the btkip_countermeasure information.
-		 *  When the countermeasure is trigger, the driver have to
-		 *  disconnect with AP for 60 seconds.
-		 */
-		memcpy(&backupPMKIDList[0], &adapter->securitypriv.PMKIDList[0],
-		       sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE);
-		backupPMKIDIndex = adapter->securitypriv.PMKIDIndex;
-		backupTKIPCountermeasure = adapter->securitypriv.btkip_countermeasure;
-		backupTKIPcountermeasure_time = adapter->securitypriv.btkip_countermeasure_time;
-
-		memset((unsigned char *)&adapter->securitypriv, 0,
-		       sizeof (struct security_priv));
-		/* Restore the PMK information to securitypriv structure
-		 * for the following connection.
-		 */
-		memcpy(&adapter->securitypriv.PMKIDList[0], &backupPMKIDList[0],
-		       sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE);
-		adapter->securitypriv.PMKIDIndex = backupPMKIDIndex;
-		adapter->securitypriv.btkip_countermeasure = backupTKIPCountermeasure;
-		adapter->securitypriv.btkip_countermeasure_time = backupTKIPcountermeasure_time;
-
-		adapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen;
-		adapter->securitypriv.ndisencryptstatus = Ndis802_11WEPDisabled;
-	} else {  /* reset values in securitypriv */
-		struct security_priv *psec_priv = &adapter->securitypriv;
-
-		/* open system */
-		psec_priv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
-		psec_priv->dot11PrivacyAlgrthm = 0;
-		psec_priv->dot11PrivacyKeyIndex = 0;
-
-		psec_priv->dot118021XGrpPrivacy = 0;
-		psec_priv->dot118021XGrpKeyid = 1;
-
-		psec_priv->ndisauthtype = Ndis802_11AuthModeOpen;
-		psec_priv->ndisencryptstatus = Ndis802_11WEPDisabled;
-	}
-}
-
-void rtw_os_indicate_disconnect23a(struct rtw_adapter *adapter)
-{
-	/* Do it first for tx broadcast pkt after disconnection issue! */
-	netif_carrier_off(adapter->pnetdev);
-
-	rtw_cfg80211_indicate_disconnect(adapter);
-
-	rtw_reset_securitypriv23a(adapter);
-}
diff --git a/drivers/staging/rtl8723au/os_dep/os_intfs.c b/drivers/staging/rtl8723au/os_dep/os_intfs.c
deleted file mode 100644
index b8848c25beb4..000000000000
--- a/drivers/staging/rtl8723au/os_dep/os_intfs.c
+++ /dev/null
@@ -1,852 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _OS_INTFS_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <xmit_osdep.h>
-#include <recv_osdep.h>
-#include <hal_intf.h>
-#include <rtw_version.h>
-
-#include <rtl8723a_hal.h>
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Realtek Wireless Lan Driver");
-MODULE_AUTHOR("Realtek Semiconductor Corp.");
-MODULE_AUTHOR("Larry Finger <Larry.Finger@lwfinger.net>");
-MODULE_AUTHOR("Jes Sorensen <Jes.Sorensen@redhat.com>");
-MODULE_VERSION(DRIVERVERSION);
-MODULE_FIRMWARE("rtlwifi/rtl8723aufw_A.bin");
-MODULE_FIRMWARE("rtlwifi/rtl8723aufw_B.bin");
-MODULE_FIRMWARE("rtlwifi/rtl8723aufw_B_NoBT.bin");
-
-/* module param defaults */
-static int rtw_chip_version;
-static int rtw_rfintfs = HWPI;
-static int rtw_debug = 1;
-
-static int rtw_channel = 1;/* ad-hoc support requirement */
-static int rtw_wireless_mode = WIRELESS_11BG_24N;
-static int rtw_vrtl_carrier_sense = AUTO_VCS;
-static int rtw_vcs_type = RTS_CTS;/*  */
-static int rtw_rts_thresh = 2347;/*  */
-static int rtw_frag_thresh = 2346;/*  */
-static int rtw_preamble = PREAMBLE_LONG;/* long, short, auto */
-static int rtw_scan_mode = 1;/* active, passive */
-static int rtw_adhoc_tx_pwr = 1;
-static int rtw_soft_ap;
-static int rtw_power_mgnt = 1;
-static int rtw_ips_mode = IPS_NORMAL;
-
-static int rtw_smart_ps = 2;
-
-module_param(rtw_ips_mode, int, 0644);
-MODULE_PARM_DESC(rtw_ips_mode, "The default IPS mode");
-
-static int rtw_long_retry_lmt = 7;
-static int rtw_short_retry_lmt = 7;
-static int rtw_busy_thresh = 40;
-static int rtw_ack_policy = NORMAL_ACK;
-
-static int rtw_acm_method;/*  0:By SW 1:By HW. */
-
-static int rtw_wmm_enable = 1;/*  default is set to enable the wmm. */
-static int rtw_uapsd_enable;
-
-static int rtw_ht_enable = 1;
-/* 0 :diable, bit(0): enable 2.4g, bit(1): enable 5g */
-static int rtw_cbw40_enable = 3;
-static int rtw_ampdu_enable = 1;/* for enable tx_ampdu */
-/*  0: disable, bit(0):enable 2.4g, bit(1):enable 5g, default is set to enable
- * 2.4GHZ for IOT issue with bufflao's AP at 5GHZ
- */
-static int rtw_rx_stbc = 1;
-static int rtw_ampdu_amsdu;/*  0: disabled, 1:enabled, 2:auto */
-
-/* Use 2 path Tx to transmit MCS0~7 and legacy mode */
-static int rtw_lowrate_two_xmit = 1;
-
-/* int rf_config = RF_1T2R;  1T2R */
-static int rtw_rf_config = RF_819X_MAX_TYPE;  /* auto */
-static int rtw_low_power;
-static int rtw_wifi_spec;
-static int rtw_channel_plan = RT_CHANNEL_DOMAIN_MAX;
-
-#ifdef CONFIG_8723AU_BT_COEXIST
-static int rtw_btcoex_enable = 1;
-static int rtw_bt_iso = 2;/*  0:Low, 1:High, 2:From Efuse */
-/*  0:Idle, 1:None-SCO, 2:SCO, 3:From Counter, 4.Busy, 5.OtherBusy */
-static int rtw_bt_sco = 3;
-/*  0:Disable BT control A-MPDU, 1:Enable BT control A-MPDU. */
-static int rtw_bt_ampdu = 1;
-#endif
-
-/*  0:Reject AP's Add BA req, 1:Accept AP's Add BA req. */
-static int rtw_AcceptAddbaReq = true;
-
-static int rtw_antdiv_cfg = 2; /*  0:OFF , 1:ON, 2:decide by Efuse config */
-static int rtw_antdiv_type; /* 0:decide by efuse */
-
-static int rtw_enusbss;/* 0:disable, 1:enable */
-
-static int rtw_hwpdn_mode = 2;/* 0:disable, 1:enable, 2: by EFUSE config */
-
-static int rtw_hwpwrp_detect; /* HW power  ping detect 0:disable , 1:enable */
-
-static int rtw_hw_wps_pbc = 1;
-
-static int rtw_80211d;
-
-static int rtw_regulatory_id = 0xff;/*  Regulatory tab id, 0xff = follow efuse's setting */
-
-module_param(rtw_regulatory_id, int, 0644);
-
-static char *ifname = "wlan%d";
-module_param(ifname, charp, 0644);
-MODULE_PARM_DESC(ifname, "The default name to allocate for first interface");
-
-static char *if2name = "wlan%d";
-module_param(if2name, charp, 0644);
-MODULE_PARM_DESC(if2name, "The default name to allocate for second interface");
-
-module_param(rtw_channel_plan, int, 0644);
-module_param(rtw_chip_version, int, 0644);
-module_param(rtw_rfintfs, int, 0644);
-module_param(rtw_channel, int, 0644);
-module_param(rtw_wmm_enable, int, 0644);
-module_param(rtw_vrtl_carrier_sense, int, 0644);
-module_param(rtw_vcs_type, int, 0644);
-module_param(rtw_busy_thresh, int, 0644);
-module_param(rtw_ht_enable, int, 0644);
-module_param(rtw_cbw40_enable, int, 0644);
-module_param(rtw_ampdu_enable, int, 0644);
-module_param(rtw_rx_stbc, int, 0644);
-module_param(rtw_ampdu_amsdu, int, 0644);
-
-module_param(rtw_lowrate_two_xmit, int, 0644);
-
-module_param(rtw_rf_config, int, 0644);
-module_param(rtw_power_mgnt, int, 0644);
-module_param(rtw_smart_ps, int, 0644);
-module_param(rtw_low_power, int, 0644);
-module_param(rtw_wifi_spec, int, 0644);
-
-module_param(rtw_antdiv_cfg, int, 0644);
-
-module_param(rtw_enusbss, int, 0644);
-module_param(rtw_hwpdn_mode, int, 0644);
-module_param(rtw_hwpwrp_detect, int, 0644);
-
-module_param(rtw_hw_wps_pbc, int, 0644);
-
-static uint rtw_max_roaming_times = 2;
-module_param(rtw_max_roaming_times, uint, 0644);
-MODULE_PARM_DESC(rtw_max_roaming_times, "The max roaming times to try");
-
-module_param(rtw_80211d, int, 0644);
-MODULE_PARM_DESC(rtw_80211d, "Enable 802.11d mechanism");
-
-#ifdef CONFIG_8723AU_BT_COEXIST
-module_param(rtw_btcoex_enable, int, 0644);
-MODULE_PARM_DESC(rtw_btcoex_enable, "Enable BT co-existence mechanism");
-#endif
-
-static uint rtw_notch_filter;
-module_param(rtw_notch_filter, uint, 0644);
-MODULE_PARM_DESC(rtw_notch_filter, "0:Disable, 1:Enable, 2:Enable only for P2P");
-module_param_named(debug, rtw_debug, int, 0444);
-MODULE_PARM_DESC(debug, "Set debug level (1-9) (default 1)");
-
-static int netdev_close(struct net_device *pnetdev);
-
-static void loadparam(struct rtw_adapter *padapter,  struct net_device *pnetdev)
-{
-	struct registry_priv  *registry_par = &padapter->registrypriv;
-
-	GlobalDebugLevel23A = rtw_debug;
-	registry_par->chip_version = (u8)rtw_chip_version;
-	registry_par->rfintfs = (u8)rtw_rfintfs;
-	memcpy(registry_par->ssid.ssid, "ANY", 3);
-	registry_par->ssid.ssid_len = 3;
-	registry_par->channel = (u8)rtw_channel;
-	registry_par->wireless_mode = (u8)rtw_wireless_mode;
-	registry_par->vrtl_carrier_sense = (u8)rtw_vrtl_carrier_sense;
-	registry_par->vcs_type = (u8)rtw_vcs_type;
-	registry_par->rts_thresh = (u16)rtw_rts_thresh;
-	registry_par->frag_thresh = (u16)rtw_frag_thresh;
-	registry_par->preamble = (u8)rtw_preamble;
-	registry_par->scan_mode = (u8)rtw_scan_mode;
-	registry_par->adhoc_tx_pwr = (u8)rtw_adhoc_tx_pwr;
-	registry_par->soft_ap =  (u8)rtw_soft_ap;
-	registry_par->smart_ps =  (u8)rtw_smart_ps;
-	registry_par->power_mgnt = (u8)rtw_power_mgnt;
-	registry_par->ips_mode = (u8)rtw_ips_mode;
-	registry_par->long_retry_lmt = (u8)rtw_long_retry_lmt;
-	registry_par->short_retry_lmt = (u8)rtw_short_retry_lmt;
-	registry_par->busy_thresh = (u16)rtw_busy_thresh;
-	registry_par->ack_policy = (u8)rtw_ack_policy;
-	registry_par->acm_method = (u8)rtw_acm_method;
-	 /* UAPSD */
-	registry_par->wmm_enable = (u8)rtw_wmm_enable;
-	registry_par->uapsd_enable = (u8)rtw_uapsd_enable;
-	registry_par->ht_enable = (u8)rtw_ht_enable;
-	registry_par->cbw40_enable = (u8)rtw_cbw40_enable;
-	registry_par->ampdu_enable = (u8)rtw_ampdu_enable;
-	registry_par->rx_stbc = (u8)rtw_rx_stbc;
-	registry_par->ampdu_amsdu = (u8)rtw_ampdu_amsdu;
-	registry_par->lowrate_two_xmit = (u8)rtw_lowrate_two_xmit;
-	registry_par->rf_config = (u8)rtw_rf_config;
-	registry_par->low_power = (u8)rtw_low_power;
-	registry_par->wifi_spec = (u8)rtw_wifi_spec;
-	registry_par->channel_plan = (u8)rtw_channel_plan;
-#ifdef CONFIG_8723AU_BT_COEXIST
-	registry_par->btcoex = (u8)rtw_btcoex_enable;
-	registry_par->bt_iso = (u8)rtw_bt_iso;
-	registry_par->bt_sco = (u8)rtw_bt_sco;
-	registry_par->bt_ampdu = (u8)rtw_bt_ampdu;
-#endif
-	registry_par->bAcceptAddbaReq = (u8)rtw_AcceptAddbaReq;
-	registry_par->antdiv_cfg = (u8)rtw_antdiv_cfg;
-	registry_par->antdiv_type = (u8)rtw_antdiv_type;
-
-	/* 0:disable, 1:enable, 2:by EFUSE config */
-	registry_par->hwpdn_mode = (u8)rtw_hwpdn_mode;
-	/* 0:disable, 1:enable */
-	registry_par->hwpwrp_detect = (u8)rtw_hwpwrp_detect;
-	registry_par->hw_wps_pbc = (u8)rtw_hw_wps_pbc;
-	registry_par->max_roaming_times = (u8)rtw_max_roaming_times;
-	registry_par->enable80211d = (u8)rtw_80211d;
-	snprintf(registry_par->ifname, 16, "%s", ifname);
-	snprintf(registry_par->if2name, 16, "%s", if2name);
-	registry_par->notch_filter = (u8)rtw_notch_filter;
-	registry_par->regulatory_tid = (u8)rtw_regulatory_id;
-}
-
-static int rtw_net_set_mac_address(struct net_device *pnetdev, void *p)
-{
-	struct rtw_adapter *padapter = netdev_priv(pnetdev);
-	struct sockaddr *addr = p;
-
-	if (!padapter->bup)
-		ether_addr_copy(padapter->eeprompriv.mac_addr, addr->sa_data);
-	return 0;
-}
-
-static struct net_device_stats *rtw_net_get_stats(struct net_device *pnetdev)
-{
-	struct rtw_adapter *padapter = netdev_priv(pnetdev);
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-
-	padapter->stats.tx_packets = pxmitpriv->tx_pkts;
-	padapter->stats.rx_packets = precvpriv->rx_pkts;
-	padapter->stats.tx_dropped = pxmitpriv->tx_drop;
-	padapter->stats.rx_dropped = precvpriv->rx_drop;
-	padapter->stats.tx_bytes = pxmitpriv->tx_bytes;
-	padapter->stats.rx_bytes = precvpriv->rx_bytes;
-
-	return &padapter->stats;
-}
-
-/*
- * AC to queue mapping
- *
- * AC_VO -> queue 0
- * AC_VI -> queue 1
- * AC_BE -> queue 2
- * AC_BK -> queue 3
- */
-static const u16 rtw_1d_to_queue[8] = { 2, 3, 3, 2, 1, 1, 0, 0 };
-
-/* Given a data frame determine the 802.1p/1d tag to use. */
-static u32 rtw_classify8021d(struct sk_buff *skb)
-{
-	u32 dscp;
-
-	/* skb->priority values from 256->263 are magic values to
-	 * directly indicate a specific 802.1d priority.  This is used
-	 * to allow 802.1d priority to be passed directly in from VLAN
-	 * tags, etc.
-	 */
-	if (skb->priority >= 256 && skb->priority <= 263)
-		return skb->priority - 256;
-	switch (skb->protocol) {
-	case htons(ETH_P_IP):
-		dscp = ip_hdr(skb)->tos & 0xfc;
-		break;
-	default:
-		return 0;
-	}
-	return dscp >> 5;
-}
-
-static u16 rtw_select_queue(struct net_device *dev, struct sk_buff *skb,
-			    void *accel_priv,
-			    select_queue_fallback_t fallback)
-{
-	struct rtw_adapter *padapter = netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	skb->priority = rtw_classify8021d(skb);
-
-	if (pmlmepriv->acm_mask != 0)
-		skb->priority = qos_acm23a(pmlmepriv->acm_mask, skb->priority);
-	return rtw_1d_to_queue[skb->priority];
-}
-
-u16 rtw_recv_select_queue23a(struct sk_buff *skb)
-{
-	struct iphdr *piphdr;
-	struct ethhdr *eth = (struct ethhdr *)skb->data;
-	unsigned int dscp;
-	u16 eth_type = get_unaligned_be16(&eth->h_proto);
-	u32 priority;
-	u8 *pdata = skb->data;
-
-	switch (eth_type) {
-	case ETH_P_IP:
-		piphdr = (struct iphdr *)(pdata + ETH_HLEN);
-		dscp = piphdr->tos & 0xfc;
-		priority = dscp >> 5;
-		break;
-	default:
-		priority = 0;
-	}
-	return rtw_1d_to_queue[priority];
-}
-
-static const struct net_device_ops rtw_netdev_ops = {
-	.ndo_open = netdev_open23a,
-	.ndo_stop = netdev_close,
-	.ndo_start_xmit = rtw_xmit23a_entry23a,
-	.ndo_select_queue = rtw_select_queue,
-	.ndo_set_mac_address = rtw_net_set_mac_address,
-	.ndo_get_stats = rtw_net_get_stats,
-};
-
-int rtw_init_netdev23a_name23a(struct net_device *pnetdev, const char *ifname)
-{
-	if (dev_alloc_name(pnetdev, ifname) < 0) {
-		RT_TRACE(_module_os_intfs_c_, _drv_err_,
-			 "dev_alloc_name, fail!\n");
-	}
-	netif_carrier_off(pnetdev);
-	return 0;
-}
-
-static const struct device_type wlan_type = {
-	.name = "wlan",
-};
-
-struct net_device *rtw_init_netdev23a(struct rtw_adapter *old_padapter)
-{
-	struct rtw_adapter *padapter;
-	struct net_device *pnetdev;
-
-	RT_TRACE(_module_os_intfs_c_, _drv_info_, "+init_net_dev\n");
-
-	pnetdev = alloc_etherdev_mq(sizeof(struct rtw_adapter), 4);
-	if (!pnetdev)
-		return NULL;
-
-	pnetdev->dev.type = &wlan_type;
-	padapter = netdev_priv(pnetdev);
-	padapter->pnetdev = pnetdev;
-
-	DBG_8723A("register rtw_netdev_ops to netdev_ops\n");
-	pnetdev->netdev_ops = &rtw_netdev_ops;
-
-	pnetdev->watchdog_timeo = HZ*3; /* 3 second timeout */
-
-	loadparam(padapter, pnetdev);
-	return pnetdev;
-}
-
-static int rtw_init_default_value(struct rtw_adapter *padapter)
-{
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-
-	/* xmit_priv */
-	pxmitpriv->vcs = pregistrypriv->vcs_type;
-	/* pxmitpriv->rts_thresh = pregistrypriv->rts_thresh; */
-	pxmitpriv->frag_len = pregistrypriv->frag_thresh;
-
-	/* mlme_priv */
-	pmlmepriv->scan_interval = SCAN_INTERVAL;/*  30*2 sec = 60sec */
-	pmlmepriv->scan_mode = SCAN_ACTIVE;
-
-	/* ht_priv */
-	pmlmepriv->htpriv.ampdu_enable = false;/* set to disabled */
-
-	/* security_priv */
-	psecuritypriv->binstallGrpkey = 0;
-
-	 /* open system */
-	psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
-	psecuritypriv->dot11PrivacyAlgrthm = 0;
-
-	psecuritypriv->dot11PrivacyKeyIndex = 0;
-
-	psecuritypriv->dot118021XGrpPrivacy = 0;
-	psecuritypriv->dot118021XGrpKeyid = 1;
-
-	psecuritypriv->ndisauthtype = Ndis802_11AuthModeOpen;
-	psecuritypriv->ndisencryptstatus = Ndis802_11WEPDisabled;
-
-	/* registry_priv */
-	rtw_init_registrypriv_dev_network23a(padapter);
-	rtw_update_registrypriv_dev_network23a(padapter);
-
-	/* hal_priv */
-	rtl8723a_init_default_value(padapter);
-
-	/* misc. */
-	padapter->bReadPortCancel = false;
-	padapter->bWritePortCancel = false;
-	return _SUCCESS;
-}
-
-int rtw_reset_drv_sw23a(struct rtw_adapter *padapter)
-{
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct pwrctrl_priv *pwrctrlpriv = &padapter->pwrctrlpriv;
-
-	/* hal_priv */
-	rtl8723a_init_default_value(padapter);
-	padapter->bReadPortCancel = false;
-	padapter->bWritePortCancel = false;
-	pmlmepriv->scan_interval = SCAN_INTERVAL;/*  30*2 sec = 60sec */
-
-	padapter->xmitpriv.tx_pkts = 0;
-	padapter->recvpriv.rx_pkts = 0;
-
-	pmlmepriv->LinkDetectInfo.bBusyTraffic = false;
-
-	_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY | _FW_UNDER_LINKING);
-
-	rtw_sreset_reset_value(padapter);
-	pwrctrlpriv->pwr_state_check_cnts = 0;
-
-	/* mlmeextpriv */
-	padapter->mlmeextpriv.sitesurvey_res.state = SCAN_DISABLE;
-
-	rtw_set_signal_stat_timer(&padapter->recvpriv);
-	return _SUCCESS;
-}
-
-int rtw_init_drv_sw23a(struct rtw_adapter *padapter)
-{
-	int ret8 = _SUCCESS;
-
-	RT_TRACE(_module_os_intfs_c_, _drv_info_, "+rtw_init_drv_sw23a\n");
-
-	if (rtw_init_cmd_priv23a(&padapter->cmdpriv) == _FAIL) {
-		RT_TRACE(_module_os_intfs_c_, _drv_err_,
-			 "Can't init cmd_priv\n");
-		ret8 = _FAIL;
-		goto exit;
-	}
-
-	padapter->cmdpriv.padapter = padapter;
-
-	if (rtw_init_evt_priv23a(&padapter->evtpriv) == _FAIL) {
-		RT_TRACE(_module_os_intfs_c_, _drv_err_,
-			 "Can't init evt_priv\n");
-		ret8 = _FAIL;
-		goto exit;
-	}
-
-	if (rtw_init_mlme_priv23a(padapter) == _FAIL) {
-		RT_TRACE(_module_os_intfs_c_, _drv_err_,
-			 "Can't init mlme_priv\n");
-		ret8 = _FAIL;
-		goto exit;
-	}
-
-
-	if (init_mlme_ext_priv23a(padapter) == _FAIL) {
-		RT_TRACE(_module_os_intfs_c_, _drv_err_,
-			 "Can't init mlme_ext_priv\n");
-		ret8 = _FAIL;
-		goto exit;
-	}
-
-	if (_rtw_init_xmit_priv23a(&padapter->xmitpriv, padapter) == _FAIL) {
-		DBG_8723A("Can't _rtw_init_xmit_priv23a\n");
-		ret8 = _FAIL;
-		goto exit;
-	}
-
-	if (_rtw_init_recv_priv23a(&padapter->recvpriv, padapter) == _FAIL) {
-		DBG_8723A("Can't _rtw_init_recv_priv23a\n");
-		ret8 = _FAIL;
-		goto exit;
-	}
-
-	if (_rtw_init_sta_priv23a(&padapter->stapriv) == _FAIL) {
-		DBG_8723A("Can't _rtw_init_sta_priv23a\n");
-		ret8 = _FAIL;
-		goto exit;
-	}
-
-	padapter->stapriv.padapter = padapter;
-	padapter->setband = GHZ24_50;
-	rtw_init_bcmc_stainfo23a(padapter);
-
-	rtw_init_pwrctrl_priv23a(padapter);
-
-	ret8 = rtw_init_default_value(padapter);
-
-	rtl8723a_init_dm_priv(padapter);
-
-	rtw_sreset_init(padapter);
-
-exit:
-
-	RT_TRACE(_module_os_intfs_c_, _drv_info_, "-rtw_init_drv_sw23a\n");
-	return ret8;
-}
-
-void rtw_cancel_all_timer23a(struct rtw_adapter *padapter)
-{
-	RT_TRACE(_module_os_intfs_c_, _drv_info_,
-		 "+rtw_cancel_all_timer23a\n");
-
-	del_timer_sync(&padapter->mlmepriv.assoc_timer);
-	RT_TRACE(_module_os_intfs_c_, _drv_info_,
-		 "%s:cancel association timer complete!\n", __func__);
-
-	del_timer_sync(&padapter->mlmepriv.scan_to_timer);
-	RT_TRACE(_module_os_intfs_c_, _drv_info_,
-		 "%s:cancel scan_to_timer!\n", __func__);
-
-	del_timer_sync(&padapter->mlmepriv.dynamic_chk_timer);
-	RT_TRACE(_module_os_intfs_c_, _drv_info_,
-		 "%s:cancel dynamic_chk_timer!\n", __func__);
-
-	del_timer_sync(&padapter->pwrctrlpriv.pwr_state_check_timer);
-
-	del_timer_sync(&padapter->mlmepriv.set_scan_deny_timer);
-	rtw_clear_scan_deny(padapter);
-	RT_TRACE(_module_os_intfs_c_, _drv_info_,
-		 "%s:cancel set_scan_deny_timer!\n", __func__);
-
-	del_timer_sync(&padapter->recvpriv.signal_stat_timer);
-}
-
-int rtw_free_drv_sw23a(struct rtw_adapter *padapter)
-{
-	RT_TRACE(_module_os_intfs_c_, _drv_info_, "==>rtw_free_drv_sw23a\n");
-
-	free_mlme_ext_priv23a(&padapter->mlmeextpriv);
-
-	rtw_free_evt_priv23a(&padapter->evtpriv);
-
-	rtw_free_mlme_priv23a(&padapter->mlmepriv);
-
-	_rtw_free_xmit_priv23a(&padapter->xmitpriv);
-
-	/* will free bcmc_stainfo here */
-	_rtw_free_sta_priv23a(&padapter->stapriv);
-
-	_rtw_free_recv_priv23a(&padapter->recvpriv);
-
-	rtw_free_pwrctrl_priv(padapter);
-
-	kfree(padapter->HalData);
-	padapter->HalData = NULL;
-
-	RT_TRACE(_module_os_intfs_c_, _drv_info_, "-rtw_free_drv_sw23a\n");
-	return _SUCCESS;
-}
-
-static int _rtw_drv_register_netdev(struct rtw_adapter *padapter, char *name)
-{
-	struct net_device *pnetdev = padapter->pnetdev;
-	int ret = _SUCCESS;
-
-	/* alloc netdev name */
-	rtw_init_netdev23a_name23a(pnetdev, name);
-
-	ether_addr_copy(pnetdev->dev_addr, padapter->eeprompriv.mac_addr);
-
-	/* Tell the network stack we exist */
-	if (register_netdev(pnetdev)) {
-		DBG_8723A("%s(%s): Failed!\n", __func__, pnetdev->name);
-		ret = _FAIL;
-		goto error_register_netdev;
-	}
-	DBG_8723A("%s, MAC Address (if%d) = %pM\n",
-		  __func__, padapter->iface_id + 1, pnetdev->dev_addr);
-	return ret;
-
-error_register_netdev:
-
-	if (padapter->iface_id > IFACE_ID0) {
-		rtw_free_drv_sw23a(padapter);
-
-		free_netdev(pnetdev);
-	}
-	return ret;
-}
-
-int rtw_drv_register_netdev(struct rtw_adapter *if1)
-{
-	struct dvobj_priv *dvobj = if1->dvobj;
-	int i, status = _SUCCESS;
-
-	if (dvobj->iface_nums >= IFACE_ID_MAX) {
-		status = _FAIL; /* -EINVAL */
-		goto exit;
-	}
-
-	for (i = 0; i < dvobj->iface_nums; i++) {
-		struct rtw_adapter *padapter = dvobj->padapters[i];
-
-		if (padapter) {
-			char *name;
-
-			if (padapter->iface_id == IFACE_ID0)
-				name = if1->registrypriv.ifname;
-			else if (padapter->iface_id == IFACE_ID1)
-				name = if1->registrypriv.if2name;
-			else
-				name = "wlan%d";
-			status = _rtw_drv_register_netdev(padapter, name);
-			if (status != _SUCCESS)
-				break;
-		}
-	}
-
-exit:
-	return status;
-}
-
-int netdev_open23a(struct net_device *pnetdev)
-{
-	struct rtw_adapter *padapter = netdev_priv(pnetdev);
-	struct pwrctrl_priv *pwrctrlpriv;
-	int ret = 0;
-	int status;
-
-	RT_TRACE(_module_os_intfs_c_, _drv_info_, "+871x_drv - dev_open\n");
-	DBG_8723A("+871x_drv - drv_open, bup =%d\n", padapter->bup);
-
-	mutex_lock(&adapter_to_dvobj(padapter)->hw_init_mutex);
-
-	pwrctrlpriv = &padapter->pwrctrlpriv;
-
-	if (!padapter->bup) {
-		padapter->bDriverStopped = false;
-		padapter->bSurpriseRemoved = false;
-		padapter->bCardDisableWOHSM = false;
-
-		status = rtl8723au_hal_init(padapter);
-		if (status == _FAIL) {
-			RT_TRACE(_module_os_intfs_c_, _drv_err_,
-				 "rtl871x_hal_init(): Can't init h/w!\n");
-			goto netdev_open23a_error;
-		}
-
-		DBG_8723A("MAC Address = %pM\n", pnetdev->dev_addr);
-
-		if (init_hw_mlme_ext23a(padapter) == _FAIL) {
-			DBG_8723A("can't init mlme_ext_priv\n");
-			goto netdev_open23a_error;
-		}
-
-		rtl8723au_inirp_init(padapter);
-
-		rtw_cfg80211_init_wiphy(padapter);
-
-		padapter->bup = true;
-	}
-	padapter->net_closed = false;
-
-	mod_timer(&padapter->mlmepriv.dynamic_chk_timer,
-		  jiffies + msecs_to_jiffies(2000));
-
-	padapter->pwrctrlpriv.bips_processing = false;
-	rtw_set_pwr_state_check_timer(&padapter->pwrctrlpriv);
-
-	/* netif_carrier_on(pnetdev);call this func when
-	   rtw23a_joinbss_event_cb return success */
-	if (!rtw_netif_queue_stopped(pnetdev))
-		netif_tx_start_all_queues(pnetdev);
-	else
-		netif_tx_wake_all_queues(pnetdev);
-
-	RT_TRACE(_module_os_intfs_c_, _drv_info_, "-871x_drv - dev_open\n");
-	DBG_8723A("-871x_drv - drv_open, bup =%d\n", padapter->bup);
-exit:
-	mutex_unlock(&adapter_to_dvobj(padapter)->hw_init_mutex);
-	return ret;
-
-netdev_open23a_error:
-	padapter->bup = false;
-
-	netif_carrier_off(pnetdev);
-	netif_tx_stop_all_queues(pnetdev);
-
-	RT_TRACE(_module_os_intfs_c_, _drv_err_,
-		 "-871x_drv - dev_open, fail!\n");
-	DBG_8723A("-871x_drv - drv_open fail, bup =%d\n", padapter->bup);
-
-	ret = -1;
-	goto exit;
-}
-
-static int ips_netdrv_open(struct rtw_adapter *padapter)
-{
-	int status = _SUCCESS;
-
-	padapter->net_closed = false;
-	DBG_8723A("===> %s.........\n", __func__);
-
-	padapter->bDriverStopped = false;
-	padapter->bSurpriseRemoved = false;
-	padapter->bCardDisableWOHSM = false;
-
-	status = rtl8723au_hal_init(padapter);
-	if (status == _FAIL) {
-		RT_TRACE(_module_os_intfs_c_, _drv_err_,
-			 "ips_netdrv_open(): Can't init h/w!\n");
-		goto netdev_open23a_error;
-	}
-
-	rtl8723au_inirp_init(padapter);
-
-	rtw_set_pwr_state_check_timer(&padapter->pwrctrlpriv);
-	mod_timer(&padapter->mlmepriv.dynamic_chk_timer,
-		  jiffies + msecs_to_jiffies(5000));
-
-	return _SUCCESS;
-
-netdev_open23a_error:
-	/* padapter->bup = false; */
-	DBG_8723A("-ips_netdrv_open - drv_open failure, bup =%d\n",
-		  padapter->bup);
-
-	return _FAIL;
-}
-
-int rtw_ips_pwr_up23a(struct rtw_adapter *padapter)
-{
-	int result;
-	unsigned long start_time = jiffies;
-
-	DBG_8723A("===>  rtw_ips_pwr_up23a..............\n");
-	rtw_reset_drv_sw23a(padapter);
-
-	result = ips_netdrv_open(padapter);
-
-	DBG_8723A("<===  rtw_ips_pwr_up23a.............. in %dms\n",
-		  jiffies_to_msecs(jiffies - start_time));
-	return result;
-}
-
-void rtw_ips_pwr_down23a(struct rtw_adapter *padapter)
-{
-	unsigned long start_time = jiffies;
-
-	DBG_8723A("===> rtw_ips_pwr_down23a...................\n");
-
-	padapter->bCardDisableWOHSM = true;
-	padapter->net_closed = true;
-
-	rtw_ips_dev_unload23a(padapter);
-	padapter->bCardDisableWOHSM = false;
-	DBG_8723A("<=== rtw_ips_pwr_down23a..................... in %dms\n",
-		  jiffies_to_msecs(jiffies - start_time));
-}
-
-void rtw_ips_dev_unload23a(struct rtw_adapter *padapter)
-{
-	rtl8723a_fifo_cleanup(padapter);
-
-	rtl8723a_usb_intf_stop(padapter);
-
-	/* s5. */
-	if (!padapter->bSurpriseRemoved)
-		rtl8723au_hal_deinit(padapter);
-}
-
-int pm_netdev_open23a(struct net_device *pnetdev, u8 bnormal)
-{
-	int status;
-
-	if (bnormal)
-		status = netdev_open23a(pnetdev);
-	else
-		status = (_SUCCESS == ips_netdrv_open(netdev_priv(pnetdev))) ?
-			 (0) : (-1);
-
-	return status;
-}
-
-static int netdev_close(struct net_device *pnetdev)
-{
-	struct rtw_adapter *padapter = netdev_priv(pnetdev);
-
-	RT_TRACE(_module_os_intfs_c_, _drv_info_, "+871x_drv - drv_close\n");
-
-	padapter->net_closed = true;
-
-	if (padapter->pwrctrlpriv.rf_pwrstate == rf_on) {
-		DBG_8723A("(2)871x_drv - drv_close, bup =%d, "
-			  "hw_init_completed =%d\n", padapter->bup,
-			  padapter->hw_init_completed);
-
-		/* s1. */
-		if (pnetdev) {
-			if (!rtw_netif_queue_stopped(pnetdev))
-				netif_tx_stop_all_queues(pnetdev);
-		}
-
-		/* s2. */
-		LeaveAllPowerSaveMode23a(padapter);
-		rtw_disassoc_cmd23a(padapter, 500, false);
-		/* s2-2.  indicate disconnect to os */
-		rtw_indicate_disconnect23a(padapter);
-		/* s2-3. */
-		rtw_free_assoc_resources23a(padapter, 1);
-		/* s2-4. */
-		rtw_free_network_queue23a(padapter);
-	}
-
-	rtw_scan_abort23a(padapter);
-
-	RT_TRACE(_module_os_intfs_c_, _drv_info_, "-871x_drv - drv_close\n");
-	DBG_8723A("-871x_drv - drv_close, bup =%d\n", padapter->bup);
-
-	return 0;
-}
-
-void rtw_ndev_destructor(struct net_device *ndev)
-{
-	DBG_8723A("%s(%s)\n", __func__, ndev->name);
-	kfree(ndev->ieee80211_ptr);
-	free_netdev(ndev);
-}
-
-void _rtw_init_queue23a(struct rtw_queue *pqueue)
-{
-	INIT_LIST_HEAD(&pqueue->queue);
-	spin_lock_init(&pqueue->lock);
-}
diff --git a/drivers/staging/rtl8723au/os_dep/recv_linux.c b/drivers/staging/rtl8723au/os_dep/recv_linux.c
deleted file mode 100644
index 084b506ae161..000000000000
--- a/drivers/staging/rtl8723au/os_dep/recv_linux.c
+++ /dev/null
@@ -1,165 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _RECV_OSDEP_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#include <wifi.h>
-#include <recv_osdep.h>
-
-#include <osdep_intf.h>
-
-#include <usb_ops.h>
-
-void rtw_handle_tkip_mic_err23a(struct rtw_adapter *padapter, u8 bgroup)
-{
-	enum nl80211_key_type key_type = 0;
-	union iwreq_data wrqu;
-	struct iw_michaelmicfailure ev;
-	struct mlme_priv *pmlmepriv  = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &padapter->securitypriv;
-	unsigned long cur_time;
-
-	if (psecuritypriv->last_mic_err_time == 0) {
-		psecuritypriv->last_mic_err_time = jiffies;
-	} else {
-		cur_time = jiffies;
-
-		if (cur_time - psecuritypriv->last_mic_err_time < 60*HZ) {
-			psecuritypriv->btkip_countermeasure = true;
-			psecuritypriv->last_mic_err_time = 0;
-			psecuritypriv->btkip_countermeasure_time = cur_time;
-		} else {
-			psecuritypriv->last_mic_err_time = jiffies;
-		}
-	}
-
-	if (bgroup)
-		key_type |= NL80211_KEYTYPE_GROUP;
-	else
-		key_type |= NL80211_KEYTYPE_PAIRWISE;
-
-	cfg80211_michael_mic_failure(padapter->pnetdev,
-				     (u8 *)&pmlmepriv->assoc_bssid[0],
-				     key_type, -1, NULL, GFP_ATOMIC);
-
-	memset(&ev, 0x00, sizeof(ev));
-	if (bgroup)
-		ev.flags |= IW_MICFAILURE_GROUP;
-	else
-		ev.flags |= IW_MICFAILURE_PAIRWISE;
-
-	ev.src_addr.sa_family = ARPHRD_ETHER;
-	ether_addr_copy(ev.src_addr.sa_data, &pmlmepriv->assoc_bssid[0]);
-
-	memset(&wrqu, 0x00, sizeof(wrqu));
-	wrqu.data.length = sizeof(ev);
-}
-
-int rtw_recv_indicatepkt23a(struct rtw_adapter *padapter,
-			 struct recv_frame *precv_frame)
-{
-	struct recv_priv *precvpriv;
-	struct sk_buff *skb;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-
-	precvpriv = &padapter->recvpriv;
-
-	skb = precv_frame->pkt;
-	if (!skb) {
-		RT_TRACE(_module_recv_osdep_c_, _drv_err_,
-			 "rtw_recv_indicatepkt23a():skb == NULL!!!!\n");
-		goto _recv_indicatepkt_drop;
-	}
-
-	RT_TRACE(_module_recv_osdep_c_, _drv_info_,
-		 "rtw_recv_indicatepkt23a():skb != NULL !!!\n");
-	RT_TRACE(_module_recv_osdep_c_, _drv_info_,
-		 "rtw_recv_indicatepkt23a():precv_frame->hdr.rx_data =%p\n",
-		 precv_frame->pkt->data);
-	RT_TRACE(_module_recv_osdep_c_, _drv_info_,
-		 "skb->head =%p skb->data =%p skb->tail =%p skb->end =%p skb->len =%d\n",
-		 skb->head, skb->data,
-		 skb_tail_pointer(skb), skb_end_pointer(skb), skb->len);
-
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
-		struct sk_buff *pskb2 = NULL;
-		struct sta_info *psta = NULL;
-		struct sta_priv *pstapriv = &padapter->stapriv;
-		struct rx_pkt_attrib *pattrib = &precv_frame->attrib;
-		int bmcast = is_multicast_ether_addr(pattrib->dst);
-
-		/* DBG_8723A("bmcast =%d\n", bmcast); */
-
-		if (!ether_addr_equal(pattrib->dst,
-				      myid(&padapter->eeprompriv))) {
-			/* DBG_8723A("not ap psta =%p, addr =%pM\n", psta, pattrib->dst); */
-			if (bmcast) {
-				psta = rtw_get_bcmc_stainfo23a(padapter);
-				pskb2 = skb_clone(skb, GFP_ATOMIC);
-			} else {
-				psta = rtw_get_stainfo23a(pstapriv, pattrib->dst);
-			}
-
-			if (psta) {
-				struct net_device *pnetdev = padapter->pnetdev;
-
-				/* DBG_8723A("directly forwarding to the rtw_xmit23a_entry23a\n"); */
-
-				/* skb->ip_summed = CHECKSUM_NONE; */
-				skb->dev = pnetdev;
-				skb_set_queue_mapping(skb, rtw_recv_select_queue23a(skb));
-
-				rtw_xmit23a_entry23a(skb, pnetdev);
-
-				if (bmcast)
-					skb = pskb2;
-				else
-					goto _recv_indicatepkt_end;
-			}
-		} else { /*  to APself */
-			/* DBG_8723A("to APSelf\n"); */
-		}
-	}
-
-	skb->ip_summed = CHECKSUM_NONE;
-	skb->dev = padapter->pnetdev;
-	skb->protocol = eth_type_trans(skb, padapter->pnetdev);
-
-	netif_rx(skb);
-
-_recv_indicatepkt_end:
-
-	precv_frame->pkt = NULL; /*  pointers to NULL before rtw_free_recvframe23a() */
-
-	rtw_free_recvframe23a(precv_frame);
-
-	RT_TRACE(_module_recv_osdep_c_, _drv_info_,
-		 "rtw_recv_indicatepkt23a :after netif_rx!!!!\n");
-	return _SUCCESS;
-
-_recv_indicatepkt_drop:
-
-	 rtw_free_recvframe23a(precv_frame);
-	 return _FAIL;
-}
-
-void rtw_init_recv_timer23a(struct recv_reorder_ctrl *preorder_ctrl)
-{
-	setup_timer(&preorder_ctrl->reordering_ctrl_timer,
-		    rtw_reordering_ctrl_timeout_handler23a,
-		    (unsigned long)preorder_ctrl);
-}
diff --git a/drivers/staging/rtl8723au/os_dep/usb_intf.c b/drivers/staging/rtl8723au/os_dep/usb_intf.c
deleted file mode 100644
index cf83efffbffd..000000000000
--- a/drivers/staging/rtl8723au/os_dep/usb_intf.c
+++ /dev/null
@@ -1,627 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _HCI_INTF_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-#include <recv_osdep.h>
-#include <xmit_osdep.h>
-#include <hal_intf.h>
-#include <rtw_version.h>
-#include <osdep_intf.h>
-#include <usb_ops.h>
-#include <rtl8723a_hal.h>
-
-static int rtw_suspend(struct usb_interface *intf, pm_message_t message);
-static int rtw_resume(struct usb_interface *intf);
-static int rtw_drv_init(struct usb_interface *pusb_intf,
-			const struct usb_device_id *pdid);
-static void rtw_disconnect(struct usb_interface *pusb_intf);
-
-#define USB_VENDER_ID_REALTEK		0x0BDA
-
-#define RTL8723A_USB_IDS \
-	{USB_DEVICE_AND_INTERFACE_INFO(USB_VENDER_ID_REALTEK, 0x8724,	\
-	 0xff, 0xff, 0xff)}, /* 8723AU 1*1 */ \
-	{USB_DEVICE_AND_INTERFACE_INFO(USB_VENDER_ID_REALTEK, 0x1724,	\
-	 0xff, 0xff, 0xff)}, /* 8723AU 1*1 */ \
-	{USB_DEVICE_AND_INTERFACE_INFO(USB_VENDER_ID_REALTEK, 0x0724,	\
-	 0xff, 0xff, 0xff)}, /* 8723AU 1*1 */
-
-static struct usb_device_id rtl8723a_usb_id_tbl[] = {
-	RTL8723A_USB_IDS
-	{}	/* Terminating entry */
-};
-
-MODULE_DEVICE_TABLE(usb, rtl8723a_usb_id_tbl);
-
-static struct usb_driver rtl8723a_usb_drv = {
-	.name = (char *)"rtl8723au",
-	.probe = rtw_drv_init,
-	.disconnect = rtw_disconnect,
-	.id_table = rtl8723a_usb_id_tbl,
-	.suspend = rtw_suspend,
-	.resume = rtw_resume,
-	.reset_resume  = rtw_resume,
-};
-
-static struct usb_driver *usb_drv = &rtl8723a_usb_drv;
-
-static int rtw_init_intf_priv(struct dvobj_priv *dvobj)
-{
-	mutex_init(&dvobj->usb_vendor_req_mutex);
-
-	return _SUCCESS;
-}
-
-static int rtw_deinit_intf_priv(struct dvobj_priv *dvobj)
-{
-	mutex_destroy(&dvobj->usb_vendor_req_mutex);
-
-	return _SUCCESS;
-}
-
-static struct dvobj_priv *usb_dvobj_init(struct usb_interface *usb_intf)
-{
-	struct dvobj_priv *pdvobjpriv;
-	struct usb_host_config	 *phost_conf;
-	struct usb_config_descriptor *pconf_desc;
-	struct usb_host_interface *phost_iface;
-	struct usb_interface_descriptor *piface_desc;
-	struct usb_endpoint_descriptor *pendp_desc;
-	struct usb_device *pusbd;
-	int i, status = _FAIL;
-
-	pdvobjpriv = kzalloc(sizeof(*pdvobjpriv), GFP_KERNEL);
-	if (!pdvobjpriv)
-		goto exit;
-
-	mutex_init(&pdvobjpriv->hw_init_mutex);
-	mutex_init(&pdvobjpriv->h2c_fwcmd_mutex);
-	mutex_init(&pdvobjpriv->setch_mutex);
-	mutex_init(&pdvobjpriv->setbw_mutex);
-
-	pdvobjpriv->pusbintf = usb_intf;
-	pusbd = interface_to_usbdev(usb_intf);
-	pdvobjpriv->pusbdev = pusbd;
-	usb_set_intfdata(usb_intf, pdvobjpriv);
-
-	pdvobjpriv->RtNumInPipes = 0;
-	pdvobjpriv->RtNumOutPipes = 0;
-
-	phost_conf = pusbd->actconfig;
-	pconf_desc = &phost_conf->desc;
-
-	phost_iface = &usb_intf->altsetting[0];
-	piface_desc = &phost_iface->desc;
-
-	pdvobjpriv->NumInterfaces = pconf_desc->bNumInterfaces;
-	pdvobjpriv->InterfaceNumber = piface_desc->bInterfaceNumber;
-	pdvobjpriv->nr_endpoint = piface_desc->bNumEndpoints;
-
-	for (i = 0; i < pdvobjpriv->nr_endpoint; i++) {
-		pendp_desc = &phost_iface->endpoint[i].desc;
-
-		DBG_8723A("\nusb_endpoint_descriptor(%d):\n", i);
-		DBG_8723A("bLength =%x\n", pendp_desc->bLength);
-		DBG_8723A("bDescriptorType =%x\n", pendp_desc->bDescriptorType);
-		DBG_8723A("bEndpointAddress =%x\n",
-			  pendp_desc->bEndpointAddress);
-		DBG_8723A("wMaxPacketSize =%d\n",
-			  le16_to_cpu(pendp_desc->wMaxPacketSize));
-		DBG_8723A("bInterval =%x\n", pendp_desc->bInterval);
-
-		if (usb_endpoint_is_bulk_in(pendp_desc)) {
-			DBG_8723A("usb_endpoint_is_bulk_in = %x\n",
-				  usb_endpoint_num(pendp_desc));
-			pdvobjpriv->RtInPipe[pdvobjpriv->RtNumInPipes] =
-				usb_endpoint_num(pendp_desc);
-			pdvobjpriv->RtNumInPipes++;
-		} else if (usb_endpoint_is_int_in(pendp_desc)) {
-			DBG_8723A("usb_endpoint_is_int_in = %x, Interval = "
-				  "%x\n", usb_endpoint_num(pendp_desc),
-				  pendp_desc->bInterval);
-			pdvobjpriv->RtInPipe[pdvobjpriv->RtNumInPipes] =
-				usb_endpoint_num(pendp_desc);
-			pdvobjpriv->RtNumInPipes++;
-		} else if (usb_endpoint_is_bulk_out(pendp_desc)) {
-			DBG_8723A("usb_endpoint_is_bulk_out = %x\n",
-				  usb_endpoint_num(pendp_desc));
-			pdvobjpriv->RtOutPipe[pdvobjpriv->RtNumOutPipes] =
-				usb_endpoint_num(pendp_desc);
-			pdvobjpriv->RtNumOutPipes++;
-		}
-		pdvobjpriv->ep_num[i] = usb_endpoint_num(pendp_desc);
-	}
-	DBG_8723A("nr_endpoint =%d, in_num =%d, out_num =%d\n\n",
-		  pdvobjpriv->nr_endpoint, pdvobjpriv->RtNumInPipes,
-		  pdvobjpriv->RtNumOutPipes);
-
-	if (pusbd->speed == USB_SPEED_HIGH) {
-		pdvobjpriv->ishighspeed = true;
-		DBG_8723A("USB_SPEED_HIGH\n");
-	} else {
-		pdvobjpriv->ishighspeed = false;
-		DBG_8723A("NON USB_SPEED_HIGH\n");
-	}
-
-	if (rtw_init_intf_priv(pdvobjpriv) == _FAIL) {
-		RT_TRACE(_module_os_intfs_c_, _drv_err_,
-			 "Can't INIT rtw_init_intf_priv\n");
-		goto free_dvobj;
-	}
-	/* 3 misc */
-	rtw_reset_continual_urb_error(pdvobjpriv);
-	usb_get_dev(pusbd);
-	status = _SUCCESS;
-free_dvobj:
-	if (status != _SUCCESS && pdvobjpriv) {
-		usb_set_intfdata(usb_intf, NULL);
-		mutex_destroy(&pdvobjpriv->hw_init_mutex);
-		mutex_destroy(&pdvobjpriv->h2c_fwcmd_mutex);
-		mutex_destroy(&pdvobjpriv->setch_mutex);
-		mutex_destroy(&pdvobjpriv->setbw_mutex);
-		kfree(pdvobjpriv);
-		pdvobjpriv = NULL;
-	}
-exit:
-	return pdvobjpriv;
-}
-
-static void usb_dvobj_deinit(struct usb_interface *usb_intf)
-{
-	struct dvobj_priv *dvobj = usb_get_intfdata(usb_intf);
-
-	usb_set_intfdata(usb_intf, NULL);
-	if (dvobj) {
-		/* Modify condition for 92DU DMDP 2010.11.18, by Thomas */
-		if ((dvobj->NumInterfaces != 2 && dvobj->NumInterfaces != 3) ||
-		    (dvobj->InterfaceNumber == 1)) {
-			if (interface_to_usbdev(usb_intf)->state !=
-			    USB_STATE_NOTATTACHED) {
-				/* If we didn't unplug usb dongle and
-				 * remove/insert module, driver fails on
-				 * sitesurvey for the first time when
-				 * device is up .
-				 * Reset usb port for sitesurvey fail issue.
-				 */
-				DBG_8723A("usb attached..., try to reset usb device\n");
-				usb_reset_device(interface_to_usbdev(usb_intf));
-			}
-		}
-		rtw_deinit_intf_priv(dvobj);
-		mutex_destroy(&dvobj->hw_init_mutex);
-		mutex_destroy(&dvobj->h2c_fwcmd_mutex);
-		mutex_destroy(&dvobj->setch_mutex);
-		mutex_destroy(&dvobj->setbw_mutex);
-		kfree(dvobj);
-	}
-	usb_put_dev(interface_to_usbdev(usb_intf));
-}
-
-void rtl8723a_usb_intf_stop(struct rtw_adapter *padapter)
-{
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, "+usb_intf_stop\n");
-
-	/* disable_hw_interrupt */
-	if (!padapter->bSurpriseRemoved) {
-		/* device still exists, so driver can do i/o operation
-		 * TODO:
-		 */
-		RT_TRACE(_module_hci_intfs_c_, _drv_err_,
-			 "SurpriseRemoved == false\n");
-	}
-
-	/* cancel in irp */
-	rtl8723au_inirp_deinit(padapter);
-
-	/* cancel out irp */
-	rtl8723au_write_port_cancel(padapter);
-
-	/* todo:cancel other irps */
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, "-usb_intf_stop\n");
-}
-
-static void rtw_dev_unload(struct rtw_adapter *padapter)
-{
-	struct submit_ctx *pack_tx_ops = &padapter->xmitpriv.ack_tx_ops;
-
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, "+rtw_dev_unload\n");
-
-	if (padapter->bup) {
-		DBG_8723A("===> rtw_dev_unload\n");
-
-		padapter->bDriverStopped = true;
-		if (padapter->xmitpriv.ack_tx)
-			rtw23a_sctx_done_err(&pack_tx_ops,
-					     RTW_SCTX_DONE_DRV_STOP);
-
-		/* s3. */
-		rtl8723a_usb_intf_stop(padapter);
-
-		/* s4. */
-		flush_workqueue(padapter->cmdpriv.wq);
-
-		/* s5. */
-		if (!padapter->bSurpriseRemoved) {
-			rtl8723au_hal_deinit(padapter);
-			padapter->bSurpriseRemoved = true;
-		}
-		padapter->bup = false;
-	} else {
-		RT_TRACE(_module_hci_intfs_c_, _drv_err_,
-			 "r871x_dev_unload():padapter->bup == false\n");
-	}
-	DBG_8723A("<=== rtw_dev_unload\n");
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, "-rtw_dev_unload\n");
-}
-
-static int rtw_suspend(struct usb_interface *pusb_intf, pm_message_t message)
-{
-	struct dvobj_priv *dvobj = usb_get_intfdata(pusb_intf);
-	struct rtw_adapter *padapter = dvobj->if1;
-	struct net_device *pnetdev = padapter->pnetdev;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
-	int ret = 0;
-	unsigned long start_time = jiffies;
-
-	DBG_8723A("==> %s (%s:%d)\n", __func__, current->comm, current->pid);
-
-	if ((!padapter->bup) || (padapter->bDriverStopped) ||
-	    (padapter->bSurpriseRemoved)) {
-		DBG_8723A("padapter->bup =%d bDriverStopped =%d bSurpriseRemoved = %d\n",
-			  padapter->bup, padapter->bDriverStopped,
-			  padapter->bSurpriseRemoved);
-		goto exit;
-	}
-	pwrpriv->bInSuspend = true;
-	rtw_cancel_all_timer23a(padapter);
-	LeaveAllPowerSaveMode23a(padapter);
-
-	down(&pwrpriv->lock);
-	/* padapter->net_closed = true; */
-	/* s1. */
-	if (pnetdev) {
-		netif_carrier_off(pnetdev);
-		netif_tx_stop_all_queues(pnetdev);
-	}
-
-	/* s2. */
-	rtw_disassoc_cmd23a(padapter, 0, false);
-
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) &&
-	    check_fwstate(pmlmepriv, _FW_LINKED)) {
-		DBG_8723A("%s:%d %s(%pM), length:%d assoc_ssid.length:%d\n",
-			  __func__, __LINE__,
-			  pmlmepriv->cur_network.network.Ssid.ssid,
-			  pmlmepriv->cur_network.network.MacAddress,
-			  pmlmepriv->cur_network.network.Ssid.ssid_len,
-			  pmlmepriv->assoc_ssid.ssid_len);
-
-		rtw_set_roaming(padapter, 1);
-	}
-	/* s2-2.  indicate disconnect to os */
-	rtw_indicate_disconnect23a(padapter);
-	/* s2-3. */
-	rtw_free_assoc_resources23a(padapter, 1);
-	/* s2-4. */
-	rtw_free_network_queue23a(padapter);
-
-	rtw_dev_unload(padapter);
-	up(&pwrpriv->lock);
-
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
-		rtw_cfg80211_indicate_scan_done(
-			wdev_to_priv(padapter->rtw_wdev), true);
-
-	if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING))
-		rtw_indicate_disconnect23a(padapter);
-
-exit:
-	DBG_8723A("<===  %s return %d.............. in %dms\n", __func__,
-		  ret, jiffies_to_msecs(jiffies - start_time));
-
-	return ret;
-}
-
-static int rtw_resume(struct usb_interface *pusb_intf)
-{
-	struct dvobj_priv *dvobj = usb_get_intfdata(pusb_intf);
-	struct rtw_adapter *padapter = dvobj->if1;
-	struct net_device *pnetdev;
-	struct pwrctrl_priv *pwrpriv = NULL;
-	int ret = -1;
-	unsigned long start_time = jiffies;
-
-	DBG_8723A("==> %s (%s:%d)\n", __func__, current->comm, current->pid);
-
-	if (!padapter)
-		goto exit;
-	pnetdev = padapter->pnetdev;
-	pwrpriv = &padapter->pwrctrlpriv;
-
-	down(&pwrpriv->lock);
-	rtw_reset_drv_sw23a(padapter);
-	pwrpriv->bkeepfwalive = false;
-
-	DBG_8723A("bkeepfwalive(%x)\n", pwrpriv->bkeepfwalive);
-	if (pm_netdev_open23a(pnetdev, true) != 0) {
-		up(&pwrpriv->lock);
-		goto exit;
-	}
-
-	netif_device_attach(pnetdev);
-	netif_carrier_on(pnetdev);
-
-	up(&pwrpriv->lock);
-
-	if (padapter->pid[1] != 0) {
-		DBG_8723A("pid[1]:%d\n", padapter->pid[1]);
-		kill_pid(find_vpid(padapter->pid[1]), SIGUSR2, 1);
-	}
-
-	rtw23a_roaming(padapter, NULL);
-
-	ret = 0;
-exit:
-	if (pwrpriv)
-		pwrpriv->bInSuspend = false;
-	DBG_8723A("<===  %s return %d.............. in %dms\n", __func__,
-		  ret, jiffies_to_msecs(jiffies - start_time));
-
-	return ret;
-}
-
-/*
- * drv_init() - a device potentially for us
- *
- * notes: drv_init() is called when the bus driver has located a card
- * for us to support.
- *        We accept the new device by returning 0.
- */
-static struct rtw_adapter *rtw_usb_if1_init(struct dvobj_priv *dvobj,
-					    struct usb_interface *pusb_intf,
-					    const struct usb_device_id *pdid)
-{
-	struct rtw_adapter *padapter = NULL;
-	struct net_device *pnetdev = NULL;
-	int status = _FAIL;
-
-	pnetdev = rtw_init_netdev23a(padapter);
-	if (!pnetdev)
-		goto free_adapter;
-	padapter = netdev_priv(pnetdev);
-
-	padapter->dvobj = dvobj;
-	padapter->bDriverStopped = true;
-	dvobj->if1 = padapter;
-	dvobj->padapters[dvobj->iface_nums++] = padapter;
-	padapter->iface_id = IFACE_ID0;
-
-	rtl8723au_set_hw_type(padapter);
-
-	SET_NETDEV_DEV(pnetdev, dvobj_to_dev(dvobj));
-
-	if (rtw_wdev_alloc(padapter, dvobj_to_dev(dvobj)))
-		goto free_adapter;
-
-	/* step 2. allocate HalData */
-	padapter->HalData = kzalloc(sizeof(struct hal_data_8723a), GFP_KERNEL);
-	if (!padapter->HalData)
-		goto free_wdev;
-
-	/* step read_chip_version */
-	rtl8723a_read_chip_version(padapter);
-
-	/* step usb endpoint mapping */
-	if (!rtl8723au_chip_configure(padapter))
-		goto free_hal_data;
-
-	/* step read efuse/eeprom data and get mac_addr */
-	rtl8723a_read_adapter_info(padapter);
-
-	/* step 5. */
-	if (rtw_init_drv_sw23a(padapter) == _FAIL) {
-		RT_TRACE(_module_hci_intfs_c_, _drv_err_,
-			 "Initialize driver software resource Failed!\n");
-		goto free_hal_data;
-	}
-
-#ifdef CONFIG_PM
-	if (padapter->pwrctrlpriv.bSupportRemoteWakeup) {
-		dvobj->pusbdev->do_remote_wakeup = 1;
-		pusb_intf->needs_remote_wakeup = 1;
-		device_init_wakeup(&pusb_intf->dev, 1);
-		DBG_8723A("\n  padapter->pwrctrlpriv.bSupportRemoteWakeup~~~~~~\n");
-		DBG_8723A("\n  padapter->pwrctrlpriv.bSupportRemoteWakeup~~~[%d]~~~\n",
-			  device_may_wakeup(&pusb_intf->dev));
-	}
-#endif
-	/* 2012-07-11 Move here to prevent the 8723AS-VAU BT
-	 * auto suspend influence
-	 */
-	if (usb_autopm_get_interface(pusb_intf) < 0)
-		DBG_8723A("can't get autopm:\n");
-#ifdef	CONFIG_8723AU_BT_COEXIST
-	padapter->pwrctrlpriv.autopm_cnt = 1;
-#endif
-
-	/* If the eeprom mac address is corrupted, assign a random address */
-	if (is_broadcast_ether_addr(padapter->eeprompriv.mac_addr) ||
-	    is_zero_ether_addr(padapter->eeprompriv.mac_addr))
-		eth_random_addr(padapter->eeprompriv.mac_addr);
-
-	DBG_8723A("bDriverStopped:%d, bSurpriseRemoved:%d, bup:%d, hw_init_completed:%d\n",
-		  padapter->bDriverStopped, padapter->bSurpriseRemoved,
-		  padapter->bup, padapter->hw_init_completed
-	);
-	status = _SUCCESS;
-
-free_hal_data:
-	if (status != _SUCCESS)
-		kfree(padapter->HalData);
-free_wdev:
-	if (status != _SUCCESS) {
-		rtw_wdev_unregister(padapter->rtw_wdev);
-		rtw_wdev_free(padapter->rtw_wdev);
-	}
-free_adapter:
-	if (status != _SUCCESS) {
-		if (pnetdev)
-			free_netdev(pnetdev);
-		padapter = NULL;
-	}
-	return padapter;
-}
-
-static void rtw_usb_if1_deinit(struct rtw_adapter *if1)
-{
-	struct net_device *pnetdev = if1->pnetdev;
-	struct mlme_priv *pmlmepriv = &if1->mlmepriv;
-
-	if (check_fwstate(pmlmepriv, _FW_LINKED))
-		rtw_disassoc_cmd23a(if1, 0, false);
-
-#ifdef CONFIG_8723AU_AP_MODE
-	free_mlme_ap_info23a(if1);
-#endif
-
-	if (pnetdev)
-		unregister_netdev(pnetdev); /* will call netdev_close() */
-
-	rtw_cancel_all_timer23a(if1);
-
-	rtw_dev_unload(if1);
-
-	DBG_8723A("+r871xu_dev_remove, hw_init_completed =%d\n",
-		  if1->hw_init_completed);
-
-	if (if1->rtw_wdev) {
-		rtw_wdev_unregister(if1->rtw_wdev);
-		rtw_wdev_free(if1->rtw_wdev);
-	}
-
-#ifdef CONFIG_8723AU_BT_COEXIST
-	if (1 == if1->pwrctrlpriv.autopm_cnt) {
-		usb_autopm_put_interface(adapter_to_dvobj(if1)->pusbintf);
-		if1->pwrctrlpriv.autopm_cnt--;
-	}
-#endif
-
-	rtw_free_drv_sw23a(if1);
-
-	if (pnetdev)
-		free_netdev(pnetdev);
-}
-
-static int rtw_drv_init(struct usb_interface *pusb_intf,
-			const struct usb_device_id *pdid)
-{
-	struct rtw_adapter *if1 = NULL;
-	struct dvobj_priv *dvobj;
-	struct usb_device *udev;
-	int status = _FAIL;
-
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, "+rtw_drv_init\n");
-
-	/* Initialize dvobj_priv */
-	dvobj = usb_dvobj_init(pusb_intf);
-	if (!dvobj) {
-		RT_TRACE(_module_hci_intfs_c_, _drv_err_,
-			 "initialize device object priv Failed!\n");
-		goto exit;
-	}
-
-	udev = dvobj->pusbdev;
-	dev_warn(&udev->dev, "WARNING: The rtl8723au driver is deprecated!");
-	dev_warn(&udev->dev, "Please use the rtl8xxxu driver for this device!");
-
-	if1 = rtw_usb_if1_init(dvobj, pusb_intf, pdid);
-	if (!if1) {
-		DBG_8723A("rtw_init_primary_adapter Failed!\n");
-		goto free_dvobj;
-	}
-
-	/* dev_alloc_name && register_netdev */
-	status = rtw_drv_register_netdev(if1);
-	if (status != _SUCCESS)
-		goto free_if1;
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_,
-		 "-871x_drv - drv_init, success!\n");
-
-	status = _SUCCESS;
-
-free_if1:
-	if (status != _SUCCESS && if1)
-		rtw_usb_if1_deinit(if1);
-free_dvobj:
-	if (status != _SUCCESS)
-		usb_dvobj_deinit(pusb_intf);
-exit:
-	return status == _SUCCESS ? 0 : -ENODEV;
-}
-
-/* dev_remove() - our device is being removed */
-static void rtw_disconnect(struct usb_interface *pusb_intf)
-{
-	struct dvobj_priv *dvobj;
-	struct rtw_adapter *padapter;
-	struct net_device *pnetdev;
-	struct mlme_priv *pmlmepriv;
-
-	dvobj = usb_get_intfdata(pusb_intf);
-	if (!dvobj)
-		return;
-
-	padapter = dvobj->if1;
-	pnetdev = padapter->pnetdev;
-	pmlmepriv = &padapter->mlmepriv;
-
-	usb_set_intfdata(pusb_intf, NULL);
-
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, "+dev_remove()\n");
-
-	rtw_pm_set_ips23a(padapter, IPS_NONE);
-	rtw_pm_set_lps23a(padapter, PS_MODE_ACTIVE);
-
-	LeaveAllPowerSaveMode23a(padapter);
-
-	rtw_usb_if1_deinit(padapter);
-
-	usb_dvobj_deinit(pusb_intf);
-
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, "-dev_remove()\n");
-	DBG_8723A("-r871xu_dev_remove, done\n");
-}
-
-static int __init rtw_drv_entry(void)
-{
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, "+rtw_drv_entry\n");
-	return usb_register(usb_drv);
-}
-
-static void __exit rtw_drv_halt(void)
-{
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, "+rtw_drv_halt\n");
-	DBG_8723A("+rtw_drv_halt\n");
-
-	usb_deregister(usb_drv);
-
-	DBG_8723A("-rtw_drv_halt\n");
-}
-
-module_init(rtw_drv_entry);
-module_exit(rtw_drv_halt);
diff --git a/drivers/staging/rtl8723au/os_dep/usb_ops_linux.c b/drivers/staging/rtl8723au/os_dep/usb_ops_linux.c
deleted file mode 100644
index cf4a50618670..000000000000
--- a/drivers/staging/rtl8723au/os_dep/usb_ops_linux.c
+++ /dev/null
@@ -1,233 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _USB_OPS_LINUX_C_
-
-#include <drv_types.h>
-#include <usb_ops_linux.h>
-#include <rtw_sreset.h>
-
-void rtl8723au_read_port_cancel(struct rtw_adapter *padapter)
-{
-	struct recv_buf *precvbuf;
-	int i;
-
-	precvbuf = (struct recv_buf *)padapter->recvpriv.precv_buf;
-
-	DBG_8723A("%s\n", __func__);
-
-	padapter->bReadPortCancel = true;
-
-	for (i = 0; i < NR_RECVBUFF ; i++) {
-		if (precvbuf->purb)
-			usb_kill_urb(precvbuf->purb);
-		precvbuf++;
-	}
-	usb_kill_urb(padapter->recvpriv.int_in_urb);
-}
-
-static void usb_write_port23a_complete(struct urb *purb)
-{
-	struct xmit_buf *pxmitbuf = (struct xmit_buf *)purb->context;
-	struct rtw_adapter *padapter = pxmitbuf->padapter;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct hal_data_8723a *phaldata;
-	unsigned long irqL;
-
-	switch (pxmitbuf->flags) {
-	case HIGH_QUEUE_INX:
-#ifdef CONFIG_8723AU_AP_MODE
-		rtw_chk_hi_queue_cmd23a(padapter);
-#endif
-		break;
-	default:
-		break;
-	}
-
-	if (padapter->bSurpriseRemoved || padapter->bDriverStopped ||
-	    padapter->bWritePortCancel) {
-		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-			 "usb_write_port23a_complete:bDriverStopped(%d) OR bSurpriseRemoved(%d)\n",
-			 padapter->bDriverStopped, padapter->bSurpriseRemoved);
-		DBG_8723A("%s(): TX Warning! bDriverStopped(%d) OR "
-			  "bSurpriseRemoved(%d) bWritePortCancel(%d) "
-			  "pxmitbuf->ext_tag(%x)\n", __func__,
-			  padapter->bDriverStopped, padapter->bSurpriseRemoved,
-			  padapter->bReadPortCancel, pxmitbuf->ext_tag);
-
-		goto check_completion;
-	}
-
-	if (purb->status) {
-		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-			 "usb_write_port23a_complete : purb->status(%d) != 0\n",
-			 purb->status);
-		DBG_8723A("###=> urb_write_port_complete status(%d)\n",
-			  purb->status);
-		if (purb->status == -EPIPE || purb->status == -EPROTO) {
-		} else if (purb->status == -EINPROGRESS) {
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "usb_write_port23a_complete: EINPROGESS\n");
-			goto check_completion;
-		} else if (purb->status == -ENOENT) {
-			DBG_8723A("%s: -ENOENT\n", __func__);
-			goto check_completion;
-		} else if (purb->status == -ECONNRESET) {
-			DBG_8723A("%s: -ECONNRESET\n", __func__);
-			goto check_completion;
-		} else if (purb->status == -ESHUTDOWN) {
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "usb_write_port23a_complete: ESHUTDOWN\n");
-			padapter->bDriverStopped = true;
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "usb_write_port23a_complete:bDriverStopped = true\n");
-			goto check_completion;
-		} else {
-			padapter->bSurpriseRemoved = true;
-			DBG_8723A("bSurpriseRemoved = true\n");
-			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-				 "usb_write_port23a_complete:bSurpriseRemoved = true\n");
-			goto check_completion;
-		}
-	}
-	phaldata = GET_HAL_DATA(padapter);
-	phaldata->srestpriv.last_tx_complete_time = jiffies;
-
-check_completion:
-	spin_lock_irqsave(&pxmitpriv->lock_sctx, irqL);
-	rtw23a_sctx_done_err(&pxmitbuf->sctx,
-			     purb->status ? RTW_SCTX_DONE_WRITE_PORT_ERR :
-			     RTW_SCTX_DONE_SUCCESS);
-	spin_unlock_irqrestore(&pxmitpriv->lock_sctx, irqL);
-
-	rtw_free_xmitbuf23a(pxmitpriv, pxmitbuf);
-
-	tasklet_hi_schedule(&pxmitpriv->xmit_tasklet);
-}
-
-int rtl8723au_write_port(struct rtw_adapter *padapter, u32 addr, u32 cnt,
-			 struct xmit_buf *pxmitbuf)
-{
-	struct urb *purb = NULL;
-	struct dvobj_priv *pdvobj = adapter_to_dvobj(padapter);
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	struct xmit_frame *pxmitframe;
-	struct usb_device *pusbd = pdvobj->pusbdev;
-	unsigned long irqL;
-	unsigned int pipe, ep_num;
-	int status;
-	int ret = _FAIL;
-
-	RT_TRACE(_module_hci_ops_os_c_, _drv_err_, "+usb_write_port23a\n");
-
-	if (padapter->bDriverStopped || padapter->bSurpriseRemoved) {
-		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-			 "%s:(padapter->bDriverStopped || padapter->bSurpriseRemoved)!!!\n",
-			 __func__);
-		rtw23a_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_TX_DENY);
-		goto exit;
-	}
-
-	pxmitframe = (struct xmit_frame *)pxmitbuf->priv_data;
-	spin_lock_irqsave(&pxmitpriv->lock, irqL);
-
-	switch (addr) {
-	case VO_QUEUE_INX:
-		pxmitbuf->flags = VO_QUEUE_INX;
-		break;
-	case VI_QUEUE_INX:
-		pxmitbuf->flags = VI_QUEUE_INX;
-		break;
-	case BE_QUEUE_INX:
-		pxmitbuf->flags = BE_QUEUE_INX;
-		break;
-	case BK_QUEUE_INX:
-		pxmitbuf->flags = BK_QUEUE_INX;
-		break;
-	case HIGH_QUEUE_INX:
-		pxmitbuf->flags = HIGH_QUEUE_INX;
-		break;
-	default:
-		pxmitbuf->flags = MGT_QUEUE_INX;
-		break;
-	}
-
-	spin_unlock_irqrestore(&pxmitpriv->lock, irqL);
-
-	purb = pxmitbuf->pxmit_urb[0];
-
-	/* translate DMA FIFO addr to pipehandle */
-	ep_num = pdvobj->Queue2Pipe[addr];
-	pipe = usb_sndbulkpipe(pusbd, ep_num);
-
-	usb_fill_bulk_urb(purb, pusbd, pipe,
-			  pxmitframe->buf_addr, /*  pxmitbuf->pbuf */
-			  cnt, usb_write_port23a_complete,
-			  pxmitbuf);/* context is pxmitbuf */
-
-	status = usb_submit_urb(purb, GFP_ATOMIC);
-	if (!status) {
-		struct hal_data_8723a *phaldata = GET_HAL_DATA(padapter);
-		phaldata->srestpriv.last_tx_time = jiffies;
-	} else {
-		rtw23a_sctx_done_err(&pxmitbuf->sctx,
-				     RTW_SCTX_DONE_WRITE_PORT_ERR);
-		DBG_8723A("usb_write_port23a, status =%d\n", status);
-		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
-			 "usb_write_port23a(): usb_submit_urb, status =%x\n",
-			 status);
-
-		switch (status) {
-		case -ENODEV:
-			padapter->bDriverStopped = true;
-			break;
-		default:
-			break;
-		}
-		goto exit;
-	}
-	ret = _SUCCESS;
-	RT_TRACE(_module_hci_ops_os_c_, _drv_err_, "-usb_write_port23a\n");
-
-exit:
-	if (ret != _SUCCESS)
-		rtw_free_xmitbuf23a(pxmitpriv, pxmitbuf);
-
-	return ret;
-}
-
-void rtl8723au_write_port_cancel(struct rtw_adapter *padapter)
-{
-	struct xmit_buf *pxmitbuf;
-	int j;
-
-	DBG_8723A("%s\n", __func__);
-
-	padapter->bWritePortCancel = true;
-
-	list_for_each_entry(pxmitbuf, &padapter->xmitpriv.xmitbuf_list,
-			    list2) {
-		for (j = 0; j < 8; j++) {
-			if (pxmitbuf->pxmit_urb[j])
-				usb_kill_urb(pxmitbuf->pxmit_urb[j]);
-		}
-	}
-	list_for_each_entry(pxmitbuf, &padapter->xmitpriv.xmitextbuf_list,
-			    list2) {
-		for (j = 0; j < 8; j++) {
-			if (pxmitbuf->pxmit_urb[j])
-				usb_kill_urb(pxmitbuf->pxmit_urb[j]);
-		}
-	}
-}
diff --git a/drivers/staging/rtl8723au/os_dep/xmit_linux.c b/drivers/staging/rtl8723au/os_dep/xmit_linux.c
deleted file mode 100644
index 64be72ac38ee..000000000000
--- a/drivers/staging/rtl8723au/os_dep/xmit_linux.c
+++ /dev/null
@@ -1,154 +0,0 @@
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2012 Realtek Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- ******************************************************************************/
-#define _XMIT_OSDEP_C_
-
-#include <osdep_service.h>
-#include <drv_types.h>
-
-#include <linux/if_ether.h>
-#include <linux/ip.h>
-#include <wifi.h>
-#include <mlme_osdep.h>
-#include <xmit_osdep.h>
-#include <osdep_intf.h>
-
-int rtw_os_xmit_resource_alloc23a(struct rtw_adapter *padapter,
-			       struct xmit_buf *pxmitbuf, u32 alloc_sz)
-{
-	int i;
-
-	pxmitbuf->pallocated_buf = kzalloc(alloc_sz, GFP_KERNEL);
-	if (pxmitbuf->pallocated_buf == NULL)
-		return _FAIL;
-
-	pxmitbuf->pbuf = PTR_ALIGN(pxmitbuf->pallocated_buf, XMITBUF_ALIGN_SZ);
-
-	for (i = 0; i < 8; i++) {
-		pxmitbuf->pxmit_urb[i] = usb_alloc_urb(0, GFP_KERNEL);
-		if (!pxmitbuf->pxmit_urb[i]) {
-			DBG_8723A("pxmitbuf->pxmit_urb[i]==NULL");
-			return _FAIL;
-		}
-	}
-	return _SUCCESS;
-}
-
-void rtw_os_xmit_resource_free23a(struct rtw_adapter *padapter,
-			       struct xmit_buf *pxmitbuf)
-{
-	int i;
-
-	for (i = 0; i < 8; i++)
-		usb_free_urb(pxmitbuf->pxmit_urb[i]);
-	kfree(pxmitbuf->pallocated_buf);
-}
-
-#define WMM_XMIT_THRESHOLD	(NR_XMITFRAME*2/5)
-
-void rtw_os_pkt_complete23a(struct rtw_adapter *padapter, struct sk_buff *pkt)
-{
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	u16	queue;
-
-	queue = skb_get_queue_mapping(pkt);
-	if (padapter->registrypriv.wifi_spec) {
-		if (__netif_subqueue_stopped(padapter->pnetdev, queue) &&
-		    (pxmitpriv->hwxmits[queue].accnt < WMM_XMIT_THRESHOLD))
-			netif_wake_subqueue(padapter->pnetdev, queue);
-	} else {
-		if (__netif_subqueue_stopped(padapter->pnetdev, queue))
-			netif_wake_subqueue(padapter->pnetdev, queue);
-	}
-	dev_kfree_skb_any(pkt);
-}
-
-void rtw_os_xmit_complete23a(struct rtw_adapter *padapter,
-			  struct xmit_frame *pxframe)
-{
-	if (pxframe->pkt)
-		rtw_os_pkt_complete23a(padapter, pxframe->pkt);
-
-	pxframe->pkt = NULL;
-}
-
-void rtw_os_xmit_schedule23a(struct rtw_adapter *padapter)
-{
-	struct xmit_priv *pxmitpriv;
-
-	if (!padapter)
-		return;
-	pxmitpriv = &padapter->xmitpriv;
-
-	spin_lock_bh(&pxmitpriv->lock);
-
-	if (rtw_txframes_pending23a(padapter))
-		tasklet_hi_schedule(&pxmitpriv->xmit_tasklet);
-	spin_unlock_bh(&pxmitpriv->lock);
-}
-
-static void rtw_check_xmit_resource(struct rtw_adapter *padapter,
-				    struct sk_buff *pkt)
-{
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	u16	queue;
-
-	queue = skb_get_queue_mapping(pkt);
-	if (padapter->registrypriv.wifi_spec) {
-		/* No free space for Tx, tx_worker is too slow */
-		if (pxmitpriv->hwxmits[queue].accnt > WMM_XMIT_THRESHOLD)
-			netif_stop_subqueue(padapter->pnetdev, queue);
-	} else {
-		if (pxmitpriv->free_xmitframe_cnt <= 4) {
-			if (!netif_tx_queue_stopped(netdev_get_tx_queue(padapter->pnetdev, queue)))
-				netif_stop_subqueue(padapter->pnetdev, queue);
-		}
-	}
-}
-
-int rtw_xmit23a_entry23a(struct sk_buff *skb, struct net_device *pnetdev)
-{
-	struct rtw_adapter *padapter = netdev_priv(pnetdev);
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-	int res = 0;
-
-	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_, "+xmit_enry\n");
-
-	if (!rtw_if_up23a(padapter)) {
-		RT_TRACE(_module_xmit_osdep_c_, _drv_err_,
-			 "rtw_xmit23a_entry23a: rtw_if_up23a fail\n");
-		goto drop_packet;
-	}
-
-	rtw_check_xmit_resource(padapter, skb);
-
-	res = rtw_xmit23a(padapter, skb);
-	if (res < 0)
-		goto drop_packet;
-
-	pxmitpriv->tx_pkts++;
-	RT_TRACE(_module_xmit_osdep_c_, _drv_info_,
-		 "rtw_xmit23a_entry23a: tx_pkts=%d\n",
-		 (u32)pxmitpriv->tx_pkts);
-	goto exit;
-
-drop_packet:
-	pxmitpriv->tx_drop++;
-	dev_kfree_skb_any(skb);
-	RT_TRACE(_module_xmit_osdep_c_, _drv_notice_,
-		 "rtw_xmit23a_entry23a: drop, tx_drop=%d\n",
-		 (u32)pxmitpriv->tx_drop);
-exit:
-	return 0;
-}
diff --git a/drivers/staging/rts5208/ms.c b/drivers/staging/rts5208/ms.c
index 0f0cd4a03cd4..f27df0b4cb44 100644
--- a/drivers/staging/rts5208/ms.c
+++ b/drivers/staging/rts5208/ms.c
@@ -63,12 +63,12 @@ static int ms_transfer_tpc(struct rtsx_chip *chip, u8 trans_mode,
 	rtsx_add_cmd(chip, WRITE_REG_CMD, MS_BYTE_CNT, 0xFF, cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANS_CFG, 0xFF, cfg);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-		0x01, PINGPONG_BUFFER);
+		     0x01, PINGPONG_BUFFER);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANSFER,
-		0xFF, MS_TRANSFER_START | trans_mode);
+		     0xFF, MS_TRANSFER_START | trans_mode);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, MS_TRANSFER,
-		MS_TRANSFER_END, MS_TRANSFER_END);
+		     MS_TRANSFER_END, MS_TRANSFER_END);
 
 	rtsx_add_cmd(chip, READ_REG_CMD, MS_TRANS_CFG, 0, 0);
 
@@ -109,8 +109,8 @@ static int ms_transfer_tpc(struct rtsx_chip *chip, u8 trans_mode,
 }
 
 static int ms_transfer_data(struct rtsx_chip *chip, u8 trans_mode,
-			u8 tpc, u16 sec_cnt, u8 cfg, bool mode_2k,
-			int use_sg, void *buf, int buf_len)
+			    u8 tpc, u16 sec_cnt, u8 cfg, bool mode_2k,
+			    int use_sg, void *buf, int buf_len)
 {
 	int retval;
 	u8 val, err_code = 0;
@@ -206,7 +206,7 @@ static int ms_write_bytes(struct rtsx_chip *chip,
 	rtsx_add_cmd(chip, WRITE_REG_CMD, MS_BYTE_CNT, 0xFF, cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANS_CFG, 0xFF, cfg);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-		0x01, PINGPONG_BUFFER);
+		     0x01, PINGPONG_BUFFER);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD,
 		     MS_TRANSFER, 0xFF, MS_TRANSFER_START | MS_TM_WRITE_BYTES);
@@ -253,7 +253,7 @@ static int ms_write_bytes(struct rtsx_chip *chip,
 }
 
 static int ms_read_bytes(struct rtsx_chip *chip,
-			u8 tpc, u8 cnt, u8 cfg, u8 *data, int data_len)
+			 u8 tpc, u8 cnt, u8 cfg, u8 *data, int data_len)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	int retval, i;
@@ -270,12 +270,12 @@ static int ms_read_bytes(struct rtsx_chip *chip,
 	rtsx_add_cmd(chip, WRITE_REG_CMD, MS_BYTE_CNT, 0xFF, cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANS_CFG, 0xFF, cfg);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-		0x01, PINGPONG_BUFFER);
+		     0x01, PINGPONG_BUFFER);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANSFER, 0xFF,
-		MS_TRANSFER_START | MS_TM_READ_BYTES);
+		     MS_TRANSFER_START | MS_TM_READ_BYTES);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, MS_TRANSFER,
-		MS_TRANSFER_END, MS_TRANSFER_END);
+		     MS_TRANSFER_END, MS_TRANSFER_END);
 
 	for (i = 0; i < data_len - 1; i++)
 		rtsx_add_cmd(chip, READ_REG_CMD, PPBUF_BASE2 + i, 0, 0);
@@ -284,7 +284,7 @@ static int ms_read_bytes(struct rtsx_chip *chip,
 		rtsx_add_cmd(chip, READ_REG_CMD, PPBUF_BASE2 + data_len, 0, 0);
 	else
 		rtsx_add_cmd(chip, READ_REG_CMD, PPBUF_BASE2 + data_len - 1,
-			0, 0);
+			     0, 0);
 
 	retval = rtsx_send_cmd(chip, MS_CARD, 5000);
 	if (retval < 0) {
@@ -334,8 +334,8 @@ static int ms_read_bytes(struct rtsx_chip *chip,
 	return STATUS_SUCCESS;
 }
 
-static int ms_set_rw_reg_addr(struct rtsx_chip *chip,
-		u8 read_start, u8 read_cnt, u8 write_start, u8 write_cnt)
+static int ms_set_rw_reg_addr(struct rtsx_chip *chip, u8 read_start,
+			      u8 read_cnt, u8 write_start, u8 write_cnt)
 {
 	int retval, i;
 	u8 data[4];
@@ -1417,7 +1417,6 @@ static int ms_read_status_reg(struct rtsx_chip *chip)
 	return STATUS_SUCCESS;
 }
 
-
 static int ms_read_extra_data(struct rtsx_chip *chip,
 		u16 block_addr, u8 page_num, u8 *buf, int buf_len)
 {
@@ -1582,7 +1581,6 @@ static int ms_write_extra_data(struct rtsx_chip *chip,
 	return STATUS_SUCCESS;
 }
 
-
 static int ms_read_page(struct rtsx_chip *chip, u16 block_addr, u8 page_num)
 {
 	struct ms_info *ms_card = &chip->ms_card;
@@ -1667,7 +1665,6 @@ static int ms_read_page(struct rtsx_chip *chip, u16 block_addr, u8 page_num)
 	return STATUS_SUCCESS;
 }
 
-
 static int ms_set_bad_block(struct rtsx_chip *chip, u16 phy_blk)
 {
 	struct ms_info *ms_card = &chip->ms_card;
@@ -1738,7 +1735,6 @@ static int ms_set_bad_block(struct rtsx_chip *chip, u16 phy_blk)
 	return STATUS_SUCCESS;
 }
 
-
 static int ms_erase_block(struct rtsx_chip *chip, u16 phy_blk)
 {
 	struct ms_info *ms_card = &chip->ms_card;
@@ -1808,7 +1804,6 @@ ERASE_RTY:
 	return STATUS_SUCCESS;
 }
 
-
 static void ms_set_page_status(u16 log_blk, u8 type, u8 *extra, int extra_len)
 {
 	if (!extra || (extra_len < MS_EXTRA_SIZE))
@@ -2152,7 +2147,6 @@ static int ms_copy_page(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 	return STATUS_SUCCESS;
 }
 
-
 static int reset_ms(struct rtsx_chip *chip)
 {
 	struct ms_info *ms_card = &chip->ms_card;
@@ -2471,7 +2465,7 @@ static u16 ms_get_l2p_tbl(struct rtsx_chip *chip, int seg_no, u16 log_off)
 	if (!ms_card->segment)
 		return 0xFFFF;
 
-	segment = &(ms_card->segment[seg_no]);
+	segment = &ms_card->segment[seg_no];
 
 	if (segment->l2p_table)
 		return segment->l2p_table[log_off];
@@ -2488,7 +2482,7 @@ static void ms_set_l2p_tbl(struct rtsx_chip *chip,
 	if (!ms_card->segment)
 		return;
 
-	segment = &(ms_card->segment[seg_no]);
+	segment = &ms_card->segment[seg_no];
 	if (segment->l2p_table)
 		segment->l2p_table[log_off] = phy_blk;
 }
@@ -2500,7 +2494,7 @@ static void ms_set_unused_block(struct rtsx_chip *chip, u16 phy_blk)
 	int seg_no;
 
 	seg_no = (int)phy_blk >> 9;
-	segment = &(ms_card->segment[seg_no]);
+	segment = &ms_card->segment[seg_no];
 
 	segment->free_table[segment->set_index++] = phy_blk;
 	if (segment->set_index >= MS_FREE_TABLE_CNT)
@@ -2515,7 +2509,7 @@ static u16 ms_get_unused_block(struct rtsx_chip *chip, int seg_no)
 	struct zone_entry *segment;
 	u16 phy_blk;
 
-	segment = &(ms_card->segment[seg_no]);
+	segment = &ms_card->segment[seg_no];
 
 	if (segment->unused_blk_cnt <= 0)
 		return 0xFFFF;
@@ -2544,7 +2538,7 @@ static int ms_arbitrate_l2p(struct rtsx_chip *chip, u16 phy_blk,
 	u16 tmp_blk;
 
 	seg_no = (int)phy_blk >> 9;
-	segment = &(ms_card->segment[seg_no]);
+	segment = &ms_card->segment[seg_no];
 	tmp_blk = segment->l2p_table[log_off];
 
 	if (us1 != us2) {
@@ -2608,7 +2602,7 @@ static int ms_build_l2p_tbl(struct rtsx_chip *chip, int seg_no)
 	else
 		table_size = 496;
 
-	segment = &(ms_card->segment[seg_no]);
+	segment = &ms_card->segment[seg_no];
 
 	if (!segment->l2p_table) {
 		segment->l2p_table = vmalloc(table_size * 2);
@@ -2717,7 +2711,7 @@ static int ms_build_l2p_tbl(struct rtsx_chip *chip, int seg_no)
 		us2 = extra[0] & 0x10;
 
 		(void)ms_arbitrate_l2p(chip, phy_blk,
-				log_blk-ms_start_idx[seg_no], us1, us2);
+				log_blk - ms_start_idx[seg_no], us1, us2);
 		continue;
 	}
 
@@ -2809,7 +2803,6 @@ BUILD_FAIL:
 	return STATUS_FAIL;
 }
 
-
 int reset_ms_card(struct rtsx_chip *chip)
 {
 	struct ms_info *ms_card = &chip->ms_card;
@@ -2896,7 +2889,6 @@ static int mspro_set_rw_cmd(struct rtsx_chip *chip,
 	return STATUS_SUCCESS;
 }
 
-
 void mspro_stop_seq_mode(struct rtsx_chip *chip)
 {
 	struct ms_info *ms_card = &chip->ms_card;
@@ -3312,7 +3304,6 @@ int mspro_format(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 	return STATUS_FAIL;
 }
 
-
 static int ms_read_multiple_pages(struct rtsx_chip *chip, u16 phy_blk,
 				u16 log_blk, u8 start_page, u8 end_page,
 				u8 *buf, unsigned int *index,
@@ -3719,7 +3710,6 @@ static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
 	return STATUS_SUCCESS;
 }
 
-
 static int ms_finish_write(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 		u16 log_blk, u8 page_off)
 {
@@ -3770,7 +3760,7 @@ static int ms_prepare_write(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 int ms_delay_write(struct rtsx_chip *chip)
 {
 	struct ms_info *ms_card = &chip->ms_card;
-	struct ms_delay_write_tag *delay_write = &(ms_card->delay_write);
+	struct ms_delay_write_tag *delay_write = &ms_card->delay_write;
 	int retval;
 
 	if (delay_write->delay_write_flag) {
@@ -3816,7 +3806,7 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 	u8 start_page, end_page = 0, page_cnt;
 	u8 *ptr;
 #ifdef MS_DELAY_WRITE
-	struct ms_delay_write_tag *delay_write = &(ms_card->delay_write);
+	struct ms_delay_write_tag *delay_write = &ms_card->delay_write;
 #endif
 
 	ms_set_err_code(chip, MS_NO_ERROR);
@@ -3996,7 +3986,7 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 
 		for (seg_no = 0; seg_no < ARRAY_SIZE(ms_start_idx) - 1;
 				seg_no++) {
-			if (log_blk < ms_start_idx[seg_no+1])
+			if (log_blk < ms_start_idx[seg_no + 1])
 				break;
 		}
 
@@ -4082,13 +4072,12 @@ int ms_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 	return retval;
 }
 
-
 void ms_free_l2p_tbl(struct rtsx_chip *chip)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	int i = 0;
 
-	if (ms_card->segment != NULL) {
+	if (ms_card->segment) {
 		for (i = 0; i < ms_card->segment_cnt; i++) {
 			vfree(ms_card->segment[i].l2p_table);
 			ms_card->segment[i].l2p_table = NULL;
@@ -4313,7 +4302,7 @@ int mg_get_local_EKB(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if (retval != STATUS_SUCCESS) {
 		set_sense_type(chip, lun, SENSE_TYPE_MG_KEY_FAIL_NOT_AUTHEN);
 		rtsx_trace(chip);
-		goto GetEKBFinish;
+		goto free_buffer;
 	}
 
 	retval = ms_transfer_data(chip, MS_TM_AUTO_READ, PRO_READ_LONG_DATA,
@@ -4322,19 +4311,20 @@ int mg_get_local_EKB(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		set_sense_type(chip, lun, SENSE_TYPE_MG_KEY_FAIL_NOT_AUTHEN);
 		rtsx_clear_ms_error(chip);
 		rtsx_trace(chip);
-		goto GetEKBFinish;
+		goto free_buffer;
 	}
 	if (check_ms_err(chip)) {
 		set_sense_type(chip, lun, SENSE_TYPE_MG_KEY_FAIL_NOT_AUTHEN);
 		rtsx_clear_ms_error(chip);
 		rtsx_trace(chip);
-		return STATUS_FAIL;
+		retval = STATUS_FAIL;
+		goto free_buffer;
 	}
 
 	bufflen = min_t(int, 1052, scsi_bufflen(srb));
 	rtsx_stor_set_xfer_buf(buf, bufflen, srb);
 
-GetEKBFinish:
+free_buffer:
 	kfree(buf);
 	return retval;
 }
@@ -4566,7 +4556,7 @@ int mg_get_ICV(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if (retval != STATUS_SUCCESS) {
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		rtsx_trace(chip);
-		goto GetICVFinish;
+		goto free_buffer;
 	}
 
 	retval = ms_transfer_data(chip, MS_TM_AUTO_READ, PRO_READ_LONG_DATA,
@@ -4575,19 +4565,20 @@ int mg_get_ICV(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		rtsx_clear_ms_error(chip);
 		rtsx_trace(chip);
-		goto GetICVFinish;
+		goto free_buffer;
 	}
 	if (check_ms_err(chip)) {
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		rtsx_clear_ms_error(chip);
 		rtsx_trace(chip);
-		return STATUS_FAIL;
+		retval = STATUS_FAIL;
+		goto free_buffer;
 	}
 
 	bufflen = min_t(int, 1028, scsi_bufflen(srb));
 	rtsx_stor_set_xfer_buf(buf, bufflen, srb);
 
-GetICVFinish:
+free_buffer:
 	kfree(buf);
 	return retval;
 }
@@ -4657,8 +4648,8 @@ int mg_set_ICV(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		rtsx_send_cmd_no_wait(chip);
 
-		retval = rtsx_transfer_data(chip, MS_CARD, buf + 4 + i*512,
-					512, 0, DMA_TO_DEVICE, 3000);
+		retval = rtsx_transfer_data(chip, MS_CARD, buf + 4 + i * 512,
+					    512, 0, DMA_TO_DEVICE, 3000);
 		if ((retval < 0) || check_ms_err(chip)) {
 			rtsx_clear_ms_error(chip);
 			if (ms_card->mg_auth == 0) {
@@ -4705,7 +4696,7 @@ SetICVFinish:
 
 void ms_cleanup_work(struct rtsx_chip *chip)
 {
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 
 	if (CHK_MSPRO(ms_card)) {
 		if (ms_card->seq_mode) {
@@ -4770,7 +4761,7 @@ int ms_power_off_card3v3(struct rtsx_chip *chip)
 
 int release_ms_card(struct rtsx_chip *chip)
 {
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 	int retval;
 
 #ifdef MS_DELAY_WRITE
diff --git a/drivers/staging/rts5208/ms.h b/drivers/staging/rts5208/ms.h
index d919170f2720..d7686399df97 100644
--- a/drivers/staging/rts5208/ms.h
+++ b/drivers/staging/rts5208/ms.h
@@ -125,7 +125,6 @@
 #define Pro_CatagoryReg		0x06
 #define Pro_ClassReg		0x07
 
-
 #define Pro_SystemParm		0x10
 #define Pro_DataCount1		0x11
 #define Pro_DataCount0		0x12
diff --git a/drivers/staging/rts5208/rtsx.c b/drivers/staging/rts5208/rtsx.c
index 25d095a5ade7..5d65a5cdc748 100644
--- a/drivers/staging/rts5208/rtsx.c
+++ b/drivers/staging/rts5208/rtsx.c
@@ -34,27 +34,27 @@ MODULE_DESCRIPTION("Realtek PCI-Express card reader rts5208/rts5288 driver");
 MODULE_LICENSE("GPL");
 
 static unsigned int delay_use = 1;
-module_param(delay_use, uint, S_IRUGO | S_IWUSR);
+module_param(delay_use, uint, 0644);
 MODULE_PARM_DESC(delay_use, "seconds to delay before using a new device");
 
 static int ss_en;
-module_param(ss_en, int, S_IRUGO | S_IWUSR);
+module_param(ss_en, int, 0644);
 MODULE_PARM_DESC(ss_en, "enable selective suspend");
 
 static int ss_interval = 50;
-module_param(ss_interval, int, S_IRUGO | S_IWUSR);
+module_param(ss_interval, int, 0644);
 MODULE_PARM_DESC(ss_interval, "Interval to enter ss state in seconds");
 
 static int auto_delink_en;
-module_param(auto_delink_en, int, S_IRUGO | S_IWUSR);
+module_param(auto_delink_en, int, 0644);
 MODULE_PARM_DESC(auto_delink_en, "enable auto delink");
 
 static unsigned char aspm_l0s_l1_en;
-module_param(aspm_l0s_l1_en, byte, S_IRUGO | S_IWUSR);
+module_param(aspm_l0s_l1_en, byte, 0644);
 MODULE_PARM_DESC(aspm_l0s_l1_en, "enable device aspm");
 
 static int msi_en;
-module_param(msi_en, int, S_IRUGO | S_IWUSR);
+module_param(msi_en, int, 0644);
 MODULE_PARM_DESC(msi_en, "enable msi");
 
 static irqreturn_t rtsx_interrupt(int irq, void *dev_id);
@@ -81,14 +81,16 @@ static int slave_alloc(struct scsi_device *sdev)
 
 static int slave_configure(struct scsi_device *sdev)
 {
-	/* Scatter-gather buffers (all but the last) must have a length
+	/*
+	 * Scatter-gather buffers (all but the last) must have a length
 	 * divisible by the bulk maxpacket size.  Otherwise a data packet
 	 * would end up being short, causing a premature end to the data
 	 * transfer.  Since high-speed bulk pipes have a maxpacket size
 	 * of 512, we'll use that as the scsi device queue's DMA alignment
 	 * mask.  Guaranteeing proper alignment of the first buffer will
 	 * have the desired effect because, except at the beginning and
-	 * the end, scatter-gather buffers follow page boundaries. */
+	 * the end, scatter-gather buffers follow page boundaries.
+	 */
 	blk_queue_dma_alignment(sdev->request_queue, (512 - 1));
 
 	/* Set the SCSI level to at least 2.  We'll leave it at 3 if that's
@@ -111,7 +113,6 @@ static int slave_configure(struct scsi_device *sdev)
 	return 0;
 }
 
-
 /***********************************************************************
  * /proc/scsi/ functions
  ***********************************************************************/
@@ -130,7 +131,7 @@ static int queuecommand_lck(struct scsi_cmnd *srb,
 	struct rtsx_chip *chip = dev->chip;
 
 	/* check for state-transition errors */
-	if (chip->srb != NULL) {
+	if (chip->srb) {
 		dev_err(&dev->pci->dev, "Error: chip->srb = %p\n",
 			chip->srb);
 		return SCSI_MLQUEUE_HOST_BUSY;
@@ -186,8 +187,10 @@ static int command_abort(struct scsi_cmnd *srb)
 	return SUCCESS;
 }
 
-/* This invokes the transport reset mechanism to reset the state of the
- * device */
+/*
+ * This invokes the transport reset mechanism to reset the state of the
+ * device
+ */
 static int device_reset(struct scsi_cmnd *srb)
 {
 	int result = 0;
@@ -209,7 +212,6 @@ static int bus_reset(struct scsi_cmnd *srb)
 	return result < 0 ? FAILED : SUCCESS;
 }
 
-
 /*
  * this defines our host template, with which we'll allocate hosts
  */
@@ -259,7 +261,6 @@ static struct scsi_host_template rtsx_host_template = {
 	.module =			THIS_MODULE
 };
 
-
 static int rtsx_acquire_irq(struct rtsx_dev *dev)
 {
 	struct rtsx_chip *chip = dev->chip;
@@ -282,7 +283,6 @@ static int rtsx_acquire_irq(struct rtsx_dev *dev)
 	return 0;
 }
 
-
 int rtsx_read_pci_cfg_byte(u8 bus, u8 dev, u8 func, u8 offset, u8 *val)
 {
 	struct pci_dev *pdev;
@@ -515,7 +515,6 @@ SkipForAbort:
 	complete_and_exit(&dev->control_exit, 0);
 }
 
-
 static int rtsx_polling_thread(void *__dev)
 {
 	struct rtsx_dev *dev = __dev;
@@ -625,7 +624,6 @@ Exit:
 	return IRQ_HANDLED;
 }
 
-
 /* Release all our dynamic resources */
 static void rtsx_release_resources(struct rtsx_dev *dev)
 {
@@ -660,15 +658,19 @@ static void rtsx_release_resources(struct rtsx_dev *dev)
 	kfree(dev->chip);
 }
 
-/* First stage of disconnect processing: stop all commands and remove
- * the host */
+/*
+ * First stage of disconnect processing: stop all commands and remove
+ * the host
+ */
 static void quiesce_and_remove_host(struct rtsx_dev *dev)
 {
 	struct Scsi_Host *host = rtsx_to_host(dev);
 	struct rtsx_chip *chip = dev->chip;
 
-	/* Prevent new transfers, stop the current command, and
-	 * interrupt a SCSI-scan or device-reset delay */
+	/*
+	 * Prevent new transfers, stop the current command, and
+	 * interrupt a SCSI-scan or device-reset delay
+	 */
 	mutex_lock(&dev->dev_mutex);
 	scsi_lock(host);
 	rtsx_set_stat(chip, RTSX_STAT_DISCONNECT);
@@ -680,9 +682,11 @@ static void quiesce_and_remove_host(struct rtsx_dev *dev)
 	/* Wait some time to let other threads exist */
 	wait_timeout(100);
 
-	/* queuecommand won't accept any new commands and the control
+	/*
+	 * queuecommand won't accept any new commands and the control
 	 * thread won't execute a previously-queued command.  If there
-	 * is such a command pending, complete it with an error. */
+	 * is such a command pending, complete it with an error.
+	 */
 	mutex_lock(&dev->dev_mutex);
 	if (chip->srb) {
 		chip->srb->result = DID_NO_CONNECT << 16;
@@ -702,8 +706,10 @@ static void release_everything(struct rtsx_dev *dev)
 {
 	rtsx_release_resources(dev);
 
-	/* Drop our reference to the host; the SCSI core will free it
-	 * when the refcount becomes 0. */
+	/*
+	 * Drop our reference to the host; the SCSI core will free it
+	 * when the refcount becomes 0.
+	 */
 	scsi_host_put(rtsx_to_host(dev));
 }
 
@@ -942,8 +948,10 @@ static int rtsx_probe(struct pci_dev *pci,
 
 	rtsx_init_chip(dev->chip);
 
-	/* set the supported max_lun and max_id for the scsi host
-	 * NOTE: the minimal value of max_id is 1 */
+	/*
+	 * set the supported max_lun and max_id for the scsi host
+	 * NOTE: the minimal value of max_id is 1
+	 */
 	host->max_id = 1;
 	host->max_lun = dev->chip->max_lun;
 
@@ -994,7 +1002,6 @@ errout:
 	return err;
 }
 
-
 static void rtsx_remove(struct pci_dev *pci)
 {
 	struct rtsx_dev *dev = pci_get_drvdata(pci);
diff --git a/drivers/staging/rts5208/rtsx.h b/drivers/staging/rts5208/rtsx.h
index 1396263e13e6..e725b10ed087 100644
--- a/drivers/staging/rts5208/rtsx.h
+++ b/drivers/staging/rts5208/rtsx.h
@@ -70,14 +70,13 @@
 #define rtsx_write_config_byte(chip, where, val) \
 	pci_write_config_byte((chip)->rtsx->pci, where, val)
 
-#define wait_timeout_x(task_state, msecs)		\
-do {							\
-		set_current_state((task_state));	\
-		schedule_timeout((msecs) * HZ / 1000);	\
+#define wait_timeout_x(task_state, msecs)	\
+do {						\
+	set_current_state((task_state));	\
+	schedule_timeout((msecs) * HZ / 1000);	\
 } while (0)
 #define wait_timeout(msecs)	wait_timeout_x(TASK_INTERRUPTIBLE, (msecs))
 
-
 #define STATE_TRANS_NONE	0
 #define STATE_TRANS_CMD		1
 #define STATE_TRANS_BUF		2
@@ -89,8 +88,6 @@ do {							\
 
 #define SCSI_LUN(srb)		((srb)->device->lun)
 
-typedef unsigned long DELAY_PARA_T;
-
 struct rtsx_chip;
 
 struct rtsx_dev {
@@ -131,16 +128,15 @@ struct rtsx_dev {
 	struct rtsx_chip	*chip;
 };
 
-typedef struct rtsx_dev rtsx_dev_t;
-
 /* Convert between rtsx_dev and the corresponding Scsi_Host */
 static inline struct Scsi_Host *rtsx_to_host(struct rtsx_dev *dev)
 {
-	return container_of((void *) dev, struct Scsi_Host, hostdata);
+	return container_of((void *)dev, struct Scsi_Host, hostdata);
 }
+
 static inline struct rtsx_dev *host_to_rtsx(struct Scsi_Host *host)
 {
-	return (struct rtsx_dev *) host->hostdata;
+	return (struct rtsx_dev *)host->hostdata;
 }
 
 static inline void get_current_time(u8 *timeval_buf, int buf_len)
@@ -165,8 +161,10 @@ static inline void get_current_time(u8 *timeval_buf, int buf_len)
 	timeval_buf[7] = (u8)(tv_usec);
 }
 
-/* The scsi_lock() and scsi_unlock() macros protect the sm_state and the
- * single queue element srb for write access */
+/*
+ * The scsi_lock() and scsi_unlock() macros protect the sm_state and the
+ * single queue element srb for write access
+ */
 #define scsi_unlock(host)	spin_unlock_irq(host->host_lock)
 #define scsi_lock(host)		spin_lock_irq(host->host_lock)
 
diff --git a/drivers/staging/rts5208/rtsx_card.c b/drivers/staging/rts5208/rtsx_card.c
index 231833a3045e..97717744962d 100644
--- a/drivers/staging/rts5208/rtsx_card.c
+++ b/drivers/staging/rts5208/rtsx_card.c
@@ -631,21 +631,21 @@ void rtsx_init_cards(struct rtsx_chip *chip)
 int switch_ssc_clock(struct rtsx_chip *chip, int clk)
 {
 	int retval;
-	u8 N = (u8)(clk - 2), min_N, max_N;
+	u8 n = (u8)(clk - 2), min_n, max_n;
 	u8 mcu_cnt, div, max_div, ssc_depth, ssc_depth_mask;
 	int sd_vpclk_phase_reset = 0;
 
 	if (chip->cur_clk == clk)
 		return STATUS_SUCCESS;
 
-	min_N = 60;
-	max_N = 120;
+	min_n = 60;
+	max_n = 120;
 	max_div = CLK_DIV_4;
 
 	dev_dbg(rtsx_dev(chip), "Switch SSC clock to %dMHz (cur_clk = %d)\n",
 		clk, chip->cur_clk);
 
-	if ((clk <= 2) || (N > max_N)) {
+	if ((clk <= 2) || (n > max_n)) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
 	}
@@ -655,15 +655,15 @@ int switch_ssc_clock(struct rtsx_chip *chip, int clk)
 		mcu_cnt = 7;
 
 	div = CLK_DIV_1;
-	while ((N < min_N) && (div < max_div)) {
-		N = (N + 2) * 2 - 2;
+	while ((n < min_n) && (div < max_div)) {
+		n = (n + 2) * 2 - 2;
 		div++;
 	}
-	dev_dbg(rtsx_dev(chip), "N = %d, div = %d\n", N, div);
+	dev_dbg(rtsx_dev(chip), "n = %d, div = %d\n", n, div);
 
 	if (chip->ssc_en) {
 		ssc_depth = 0x01;
-		N -= 2;
+		n -= 2;
 	} else {
 		ssc_depth = 0;
 	}
@@ -677,7 +677,7 @@ int switch_ssc_clock(struct rtsx_chip *chip, int clk)
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CLK_DIV, 0xFF, (div << 4) | mcu_cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SSC_CTL1, SSC_RSTB, 0);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SSC_CTL2, ssc_depth_mask, ssc_depth);
-	rtsx_add_cmd(chip, WRITE_REG_CMD, SSC_DIV_N_0, 0xFF, N);
+	rtsx_add_cmd(chip, WRITE_REG_CMD, SSC_DIV_N_0, 0xFF, n);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SSC_CTL1, SSC_RSTB, SSC_RSTB);
 	if (sd_vpclk_phase_reset) {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SD_VPCLK0_CTL,
@@ -1027,26 +1027,26 @@ int card_share_mode(struct rtsx_chip *chip, int card)
 
 	if (CHECK_PID(chip, 0x5208)) {
 		mask = CARD_SHARE_MASK;
-		if (card == SD_CARD)
+		if (card == SD_CARD) {
 			value = CARD_SHARE_48_SD;
-		else if (card == MS_CARD)
+		} else if (card == MS_CARD) {
 			value = CARD_SHARE_48_MS;
-		else if (card == XD_CARD)
+		} else if (card == XD_CARD) {
 			value = CARD_SHARE_48_XD;
-		else {
+		} else {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
 
 	} else if (CHECK_PID(chip, 0x5288)) {
 		mask = 0x03;
-		if (card == SD_CARD)
+		if (card == SD_CARD) {
 			value = CARD_SHARE_BAROSSA_SD;
-		else if (card == MS_CARD)
+		} else if (card == MS_CARD) {
 			value = CARD_SHARE_BAROSSA_MS;
-		else if (card == XD_CARD)
+		} else if (card == XD_CARD) {
 			value = CARD_SHARE_BAROSSA_XD;
-		else {
+		} else {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -1065,7 +1065,6 @@ int card_share_mode(struct rtsx_chip *chip, int card)
 	return STATUS_SUCCESS;
 }
 
-
 int select_card(struct rtsx_chip *chip, int card)
 {
 	int retval;
@@ -1073,15 +1072,15 @@ int select_card(struct rtsx_chip *chip, int card)
 	if (chip->cur_card != card) {
 		u8 mod;
 
-		if (card == SD_CARD)
+		if (card == SD_CARD) {
 			mod = SD_MOD_SEL;
-		else if (card == MS_CARD)
+		} else if (card == MS_CARD) {
 			mod = MS_MOD_SEL;
-		else if (card == XD_CARD)
+		} else if (card == XD_CARD) {
 			mod = XD_MOD_SEL;
-		else if (card == SPI_CARD)
+		} else if (card == SPI_CARD) {
 			mod = SPI_MOD_SEL;
-		else {
+		} else {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
diff --git a/drivers/staging/rts5208/rtsx_chip.c b/drivers/staging/rts5208/rtsx_chip.c
index bcc4b666d79f..a10dd6220a7b 100644
--- a/drivers/staging/rts5208/rtsx_chip.c
+++ b/drivers/staging/rts5208/rtsx_chip.c
@@ -743,7 +743,7 @@ static inline int check_sd_speed_prior(u32 sd_speed_prior)
 	int i;
 
 	for (i = 0; i < 4; i++) {
-		u8 tmp = (u8)(sd_speed_prior >> (i*8));
+		u8 tmp = (u8)(sd_speed_prior >> (i * 8));
 
 		if ((tmp < 0x01) || (tmp > 0x04)) {
 			fake_para = true;
@@ -760,7 +760,7 @@ static inline int check_sd_current_prior(u32 sd_current_prior)
 	int i;
 
 	for (i = 0; i < 4; i++) {
-		u8 tmp = (u8)(sd_current_prior >> (i*8));
+		u8 tmp = (u8)(sd_current_prior >> (i * 8));
 
 		if (tmp > 0x03) {
 			fake_para = true;
@@ -2288,7 +2288,7 @@ int rtsx_read_ppbuf(struct rtsx_chip *chip, u8 *buf, int buf_len)
 
 	ptr = buf;
 	reg_addr = PPBUF_BASE2;
-	for (i = 0; i < buf_len/256; i++) {
+	for (i = 0; i < buf_len / 256; i++) {
 		rtsx_init_cmd(chip);
 
 		for (j = 0; j < 256; j++)
@@ -2304,10 +2304,10 @@ int rtsx_read_ppbuf(struct rtsx_chip *chip, u8 *buf, int buf_len)
 		ptr += 256;
 	}
 
-	if (buf_len%256) {
+	if (buf_len % 256) {
 		rtsx_init_cmd(chip);
 
-		for (j = 0; j < buf_len%256; j++)
+		for (j = 0; j < buf_len % 256; j++)
 			rtsx_add_cmd(chip, READ_REG_CMD, reg_addr++, 0, 0);
 
 		retval = rtsx_send_cmd(chip, 0, 250);
@@ -2317,7 +2317,7 @@ int rtsx_read_ppbuf(struct rtsx_chip *chip, u8 *buf, int buf_len)
 		}
 	}
 
-	memcpy(ptr, rtsx_get_cmd_data(chip), buf_len%256);
+	memcpy(ptr, rtsx_get_cmd_data(chip), buf_len % 256);
 
 	return STATUS_SUCCESS;
 }
@@ -2336,7 +2336,7 @@ int rtsx_write_ppbuf(struct rtsx_chip *chip, u8 *buf, int buf_len)
 
 	ptr = buf;
 	reg_addr = PPBUF_BASE2;
-	for (i = 0; i < buf_len/256; i++) {
+	for (i = 0; i < buf_len / 256; i++) {
 		rtsx_init_cmd(chip);
 
 		for (j = 0; j < 256; j++) {
@@ -2352,10 +2352,10 @@ int rtsx_write_ppbuf(struct rtsx_chip *chip, u8 *buf, int buf_len)
 		}
 	}
 
-	if (buf_len%256) {
+	if (buf_len % 256) {
 		rtsx_init_cmd(chip);
 
-		for (j = 0; j < buf_len%256; j++) {
+		for (j = 0; j < buf_len % 256; j++) {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, reg_addr++, 0xFF,
 				     *ptr);
 			ptr++;
diff --git a/drivers/staging/rts5208/rtsx_chip.h b/drivers/staging/rts5208/rtsx_chip.h
index c08164f3247e..f36642817c6e 100644
--- a/drivers/staging/rts5208/rtsx_chip.h
+++ b/drivers/staging/rts5208/rtsx_chip.h
@@ -44,8 +44,10 @@
 	#define MG_SET_ICV_SLOW
 	/* HW may miss ERR/CMDNK signal when sampling INT status. */
 	#define MS_SAMPLE_INT_ERR
-	/* HW DO NOT support Wait_INT function during READ_BYTES
-	 * transfer mode */
+	/*
+	 * HW DO NOT support Wait_INT function
+	 * during READ_BYTES transfer mode
+	 */
 	#define READ_BYTES_WAIT_INT
 #endif
 
@@ -101,18 +103,17 @@
 #define TRANSPORT_NO_SENSE	2  /* Command failed, no auto-sense    */
 #define TRANSPORT_ERROR		3   /* Transport bad (i.e. device dead) */
 
-
-/*-----------------------------------
-    Start-Stop-Unit
------------------------------------*/
+/*
+ * Start-Stop-Unit
+ */
 #define STOP_MEDIUM			0x00    /* access disable         */
 #define MAKE_MEDIUM_READY		0x01    /* access enable          */
 #define UNLOAD_MEDIUM			0x02    /* unload                 */
 #define LOAD_MEDIUM			0x03    /* load                   */
 
-/*-----------------------------------
-    STANDARD_INQUIRY
------------------------------------*/
+/*
+ * STANDARD_INQUIRY
+ */
 #define QULIFIRE                0x00
 #define AENC_FNC                0x00
 #define TRML_IOP                0x00
@@ -129,17 +130,15 @@
 #define PRDCT_REV_LEN           4               /* Product LOT Length       */
 
 /* Dynamic flag definitions: used in set_bit() etc. */
-#define RTSX_FLIDX_TRANS_ACTIVE		18  /* 0x00040000  transfer is active */
-#define RTSX_FLIDX_ABORTING		20  /* 0x00100000 abort is in
-					     * progress */
-#define RTSX_FLIDX_DISCONNECTING	21  /* 0x00200000 disconnect
-					     * in progress */
+#define RTSX_FLIDX_TRANS_ACTIVE		18  /* 0x00040000 transfer is active	 */
+#define RTSX_FLIDX_ABORTING		20  /* 0x00100000 abort is in progress	 */
+#define RTSX_FLIDX_DISCONNECTING	21  /* 0x00200000 disconnect in progress */
+
 #define ABORTING_OR_DISCONNECTING	((1UL << US_FLIDX_ABORTING) | \
 					 (1UL << US_FLIDX_DISCONNECTING))
-#define RTSX_FLIDX_RESETTING		22  /* 0x00400000 device reset
-					     * in progress */
-#define RTSX_FLIDX_TIMED_OUT		23  /* 0x00800000 SCSI
-					     * midlayer timed out */
+
+#define RTSX_FLIDX_RESETTING		22  /* 0x00400000 device reset in progress */
+#define RTSX_FLIDX_TIMED_OUT		23  /* 0x00800000 SCSI midlayer timed out  */
 
 #define DRCT_ACCESS_DEV         0x00    /* Direct Access Device      */
 #define RMB_DISC                0x80    /* The Device is Removable   */
@@ -174,9 +173,9 @@
 #define	FIRST_RESET		0x01
 #define	USED_EXIST		0x02
 
-/*-----------------------------------
-    SENSE_DATA
------------------------------------*/
+/*
+ * SENSE_DATA
+ */
 /*---- valid ----*/
 #define SENSE_VALID             0x80    /* Sense data is valid as SCSI2     */
 #define SENSE_INVALID           0x00    /* Sense data is invalid as SCSI2   */
@@ -228,7 +227,6 @@
 #define ASCQ_LOAD_EJCT_ERR      0x00
 #define	ASCQ_WRITE_PROTECT	0x00
 
-
 struct sense_data_t {
 	unsigned char   err_code;	/* error code */
 	/* bit7 : valid */
@@ -268,22 +266,22 @@ struct sense_data_t {
 #define TRIG_DMA		(0x01 << 31)
 
 /* Bus interrupt pending register */
-#define CMD_DONE_INT		(1 << 31)
-#define DATA_DONE_INT		(1 << 30)
-#define TRANS_OK_INT		(1 << 29)
-#define TRANS_FAIL_INT		(1 << 28)
-#define XD_INT			(1 << 27)
-#define MS_INT			(1 << 26)
-#define SD_INT			(1 << 25)
-#define GPIO0_INT		(1 << 24)
-#define OC_INT			(1 << 23)
-#define SD_WRITE_PROTECT	(1 << 19)
-#define XD_EXIST		(1 << 18)
-#define MS_EXIST		(1 << 17)
-#define SD_EXIST		(1 << 16)
+#define CMD_DONE_INT		BIT(31)
+#define DATA_DONE_INT		BIT(30)
+#define TRANS_OK_INT		BIT(29)
+#define TRANS_FAIL_INT		BIT(28)
+#define XD_INT			BIT(27)
+#define MS_INT			BIT(26)
+#define SD_INT			BIT(25)
+#define GPIO0_INT		BIT(24)
+#define OC_INT			BIT(23)
+#define SD_WRITE_PROTECT	BIT(19)
+#define XD_EXIST		BIT(18)
+#define MS_EXIST		BIT(17)
+#define SD_EXIST		BIT(16)
 #define DELINK_INT		GPIO0_INT
-#define MS_OC_INT		(1 << 23)
-#define SD_OC_INT		(1 << 22)
+#define MS_OC_INT		BIT(23)
+#define SD_OC_INT		BIT(22)
 
 #define CARD_INT		(XD_INT | MS_INT | SD_INT)
 #define NEED_COMPLETE_INT	(DATA_DONE_INT | TRANS_OK_INT | TRANS_FAIL_INT)
@@ -305,7 +303,6 @@ struct sense_data_t {
 #define MS_OC_INT_EN		(1 << 23)
 #define SD_OC_INT_EN		(1 << 22)
 
-
 #define READ_REG_CMD		0
 #define WRITE_REG_CMD		1
 #define CHECK_REG_CMD		2
@@ -313,7 +310,6 @@ struct sense_data_t {
 #define HOST_TO_DEVICE		0
 #define DEVICE_TO_HOST		1
 
-
 #define RTSX_RESV_BUF_LEN	4096
 #define HOST_CMDS_BUF_LEN	1024
 #define HOST_SG_TBL_BUF_LEN	(RTSX_RESV_BUF_LEN - HOST_CMDS_BUF_LEN)
@@ -332,7 +328,6 @@ struct sense_data_t {
 #define XD_FREE_TABLE_CNT	1200
 #define MS_FREE_TABLE_CNT	512
 
-
 /* Bit Operation */
 #define SET_BIT(data, idx)	((data) |= 1 << (idx))
 #define CLR_BIT(data, idx)	((data) &= ~(1 << (idx)))
@@ -618,7 +613,6 @@ struct spi_info {
 	int spi_clock;
 };
 
-
 #ifdef _MSG_TRACE
 struct trace_msg_t {
 	u16 line;
@@ -689,7 +683,7 @@ struct trace_msg_t {
 #define CLR_SDIO_IGNORED(chip)		((chip)->sdio_func_exist &= ~SDIO_IGNORED)
 
 struct rtsx_chip {
-	rtsx_dev_t	*rtsx;
+	struct rtsx_dev	*rtsx;
 
 	u32		int_reg; /* Bus interrupt pending register */
 	char		max_lun;
@@ -712,9 +706,9 @@ struct rtsx_chip {
 	int			cur_card;
 
 	unsigned long	need_release;		/* need release bit map */
-	unsigned long	need_reset;		/* need reset
-						 * bit map */
-	/* Flag to indicate that this card is just resumed from SS state,
+	unsigned long	need_reset;		/* need reset bit map */
+	/*
+	 * Flag to indicate that this card is just resumed from SS state,
 	 * and need released before being resetted
 	 */
 	unsigned long		need_reinit;
@@ -732,8 +726,10 @@ struct rtsx_chip {
 	u8	card_ejected;	/* card ejected bit map */
 	u8	card_wp;	/* card write protected bit map */
 
-	u8	lun_mc;		/* flag to indicate whether to answer
-				 * MediaChange */
+	u8	lun_mc;		/*
+				 * flag to indicate whether to answer
+				 * MediaChange
+				 */
 
 #ifndef LED_AUTO_BLINK
 	int			led_toggle_counter;
diff --git a/drivers/staging/rts5208/rtsx_scsi.c b/drivers/staging/rts5208/rtsx_scsi.c
index d2031044ea34..becb4bba166c 100644
--- a/drivers/staging/rts5208/rtsx_scsi.c
+++ b/drivers/staging/rts5208/rtsx_scsi.c
@@ -484,14 +484,14 @@ static int inquiry(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	u8 card = get_lun_card(chip, lun);
 	bool pro_formatter_flag = false;
 	unsigned char inquiry_buf[] = {
-		QULIFIRE|DRCT_ACCESS_DEV,
-		RMB_DISC|0x0D,
+		QULIFIRE | DRCT_ACCESS_DEV,
+		RMB_DISC | 0x0D,
 		0x00,
 		0x01,
 		0x1f,
 		0x02,
 		0,
-		REL_ADR|WBUS_32|WBUS_16|SYNC|LINKED|CMD_QUE|SFT_RE,
+		REL_ADR | WBUS_32 | WBUS_16 | SYNC | LINKED | CMD_QUE | SFT_RE,
 	};
 
 	if (CHECK_LUN_MODE(chip, SD_MS_2LUN)) {
@@ -558,7 +558,6 @@ static int inquiry(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	return TRANSPORT_GOOD;
 }
 
-
 static int start_stop_unit(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 	unsigned int lun = SCSI_LUN(srb);
@@ -594,7 +593,6 @@ static int start_stop_unit(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	return TRANSPORT_ERROR;
 }
 
-
 static int allow_medium_removal(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 	int prevent;
@@ -613,7 +611,6 @@ static int allow_medium_removal(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	return TRANSPORT_GOOD;
 }
 
-
 static int request_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 	struct sense_data_t *sense;
@@ -1400,7 +1397,7 @@ static int trace_msg_cmd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	buf_len = 4 + ((2 + MSG_FUNC_LEN + MSG_FILE_LEN + TIME_VAL_LEN) *
 		TRACE_ITEM_CNT);
 
-	if ((scsi_bufflen(srb) < buf_len) || (scsi_sglist(srb) == NULL)) {
+	if ((scsi_bufflen(srb) < buf_len) || !scsi_sglist(srb)) {
 		set_sense_type(chip, SCSI_LUN(srb),
 			SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		rtsx_trace(chip);
@@ -1521,7 +1518,7 @@ static int write_host_reg(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 static int set_variable(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	unsigned lun = SCSI_LUN(srb);
+	unsigned int lun = SCSI_LUN(srb);
 
 	if (srb->cmnd[3] == 1) {
 		/* Variable Clock */
@@ -1989,8 +1986,8 @@ static int read_phy_register(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 				return TRANSPORT_FAILED;
 			}
 
-			buf[2*i] = (u8)(val >> 8);
-			buf[2*i+1] = (u8)val;
+			buf[2 * i] = (u8)(val >> 8);
+			buf[2 * i + 1] = (u8)val;
 		}
 
 		len = (unsigned short)min_t(unsigned int, scsi_bufflen(srb),
@@ -2048,7 +2045,7 @@ static int write_phy_register(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		}
 
 		for (i = 0; i < len / 2; i++) {
-			val = ((u16)buf[2*i] << 8) | buf[2*i+1];
+			val = ((u16)buf[2 * i] << 8) | buf[2 * i + 1];
 			retval = rtsx_write_phy_register(chip, addr + i, val);
 			if (retval != STATUS_SUCCESS) {
 				vfree(buf);
@@ -2604,7 +2601,6 @@ static int app_cmd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	return result;
 }
 
-
 static int read_status(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 	u8 rtsx_status[16];
@@ -3071,18 +3067,18 @@ static int get_ms_information(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	buf[i++] = 0x80;
 	if ((dev_info_id == 0x10) || (dev_info_id == 0x13)) {
 		/* System Information */
-		memcpy(buf+i, ms_card->raw_sys_info, 96);
+		memcpy(buf + i, ms_card->raw_sys_info, 96);
 	} else {
 		/* Model Name */
-		memcpy(buf+i, ms_card->raw_model_name, 48);
+		memcpy(buf + i, ms_card->raw_model_name, 48);
 	}
 
 	rtsx_stor_set_xfer_buf(buf, buf_len, srb);
 
 	if (dev_info_id == 0x15)
-		scsi_set_resid(srb, scsi_bufflen(srb)-0x3C);
+		scsi_set_resid(srb, scsi_bufflen(srb) - 0x3C);
 	else
-		scsi_set_resid(srb, scsi_bufflen(srb)-0x6C);
+		scsi_set_resid(srb, scsi_bufflen(srb) - 0x6C);
 
 	kfree(buf);
 	return STATUS_SUCCESS;
diff --git a/drivers/staging/rts5208/rtsx_sys.h b/drivers/staging/rts5208/rtsx_sys.h
index 0b6b4d4f1fea..f49bed9ec76a 100644
--- a/drivers/staging/rts5208/rtsx_sys.h
+++ b/drivers/staging/rts5208/rtsx_sys.h
@@ -28,8 +28,6 @@
 #include "rtsx_chip.h"
 #include "rtsx_card.h"
 
-typedef dma_addr_t ULONG_PTR;
-
 static inline void rtsx_exclusive_enter_ss(struct rtsx_chip *chip)
 {
 	struct rtsx_dev *dev = chip->rtsx;
diff --git a/drivers/staging/rts5208/rtsx_transport.h b/drivers/staging/rts5208/rtsx_transport.h
index 899bc2079dbe..479137398c3d 100644
--- a/drivers/staging/rts5208/rtsx_transport.h
+++ b/drivers/staging/rts5208/rtsx_transport.h
@@ -38,7 +38,6 @@ void rtsx_stor_get_xfer_buf(unsigned char *buffer,
 	unsigned int buflen, struct scsi_cmnd *srb);
 void rtsx_invoke_transport(struct scsi_cmnd *srb, struct rtsx_chip *chip);
 
-
 #define rtsx_init_cmd(chip)			((chip)->ci = 0)
 
 void rtsx_add_cmd(struct rtsx_chip *chip,
diff --git a/drivers/staging/rts5208/sd.c b/drivers/staging/rts5208/sd.c
index 6219e047557e..b0bbb36f8988 100644
--- a/drivers/staging/rts5208/sd.c
+++ b/drivers/staging/rts5208/sd.c
@@ -1428,7 +1428,6 @@ static int sd_switch_function(struct rtsx_chip *chip, u8 bus_width)
 			continue;
 		}
 
-
 		if (func_to_switch)
 			break;
 
@@ -1437,9 +1436,9 @@ static int sd_switch_function(struct rtsx_chip *chip, u8 bus_width)
 		func_to_switch);
 
 #ifdef SUPPORT_SD_LOCK
-	if ((sd_card->sd_lock_status & SD_SDR_RST)
-			&& (DDR50_SUPPORT == func_to_switch)
-			&& (sd_card->func_group1_mask & SDR50_SUPPORT_MASK)) {
+	if ((sd_card->sd_lock_status & SD_SDR_RST) &&
+	    (func_to_switch == DDR50_SUPPORT) &&
+	    (sd_card->func_group1_mask & SDR50_SUPPORT_MASK)) {
 		func_to_switch = SDR50_SUPPORT;
 		dev_dbg(rtsx_dev(chip), "Using SDR50 instead of DDR50 for SD Lock\n");
 	}
@@ -2589,16 +2588,12 @@ Switch_Fail:
 #endif
 
 	retval = sd_prepare_reset(chip);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	retval = sd_dummy_clock(chip);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	if (CHK_SDIO_EXIST(chip) && !CHK_SDIO_IGNORED(chip) && try_sdio) {
 		int rty_cnt = 0;
@@ -2606,8 +2601,7 @@ Switch_Fail:
 		for (; rty_cnt < chip->sdio_retry_cnt; rty_cnt++) {
 			if (detect_card_cd(chip, SD_CARD) != STATUS_SUCCESS) {
 				sd_set_err_code(chip, SD_NO_CARD);
-				rtsx_trace(chip);
-				return STATUS_FAIL;
+				goto Status_Fail;
 			}
 
 			retval = sd_send_cmd_get_rsp(chip, IO_SEND_OP_COND, 0,
@@ -2619,8 +2613,7 @@ Switch_Fail:
 					dev_dbg(rtsx_dev(chip), "SD_IO card (Function number: %d)!\n",
 						func_num);
 					chip->sd_io = 1;
-					rtsx_trace(chip);
-					return STATUS_FAIL;
+					goto Status_Fail;
 				}
 
 				break;
@@ -2638,10 +2631,8 @@ Switch_Fail:
 RTY_SD_RST:
 	retval = sd_send_cmd_get_rsp(chip, GO_IDLE_STATE, 0, SD_RSP_TYPE_R0,
 				NULL, 0);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	wait_timeout(20);
 
@@ -2659,10 +2650,8 @@ RTY_SD_RST:
 
 		retval = sd_send_cmd_get_rsp(chip, GO_IDLE_STATE, 0,
 					SD_RSP_TYPE_R0, NULL, 0);
-		if (retval != STATUS_SUCCESS) {
-			rtsx_trace(chip);
-			return STATUS_FAIL;
-		}
+		if (retval != STATUS_SUCCESS)
+			goto Status_Fail;
 
 		wait_timeout(20);
 	}
@@ -2673,39 +2662,32 @@ RTY_SD_RST:
 		if (retval != STATUS_SUCCESS) {
 			if (detect_card_cd(chip, SD_CARD) != STATUS_SUCCESS) {
 				sd_set_err_code(chip, SD_NO_CARD);
-				rtsx_trace(chip);
-				return STATUS_FAIL;
+				goto Status_Fail;
 			}
 
 			j++;
-			if (j < 3) {
+			if (j < 3)
 				goto RTY_SD_RST;
-			} else {
-				rtsx_trace(chip);
-				return STATUS_FAIL;
-			}
+			else
+				goto Status_Fail;
 		}
 
 		retval = sd_send_cmd_get_rsp(chip, SD_APP_OP_COND, voltage,
 					SD_RSP_TYPE_R3, rsp, 5);
 		if (retval != STATUS_SUCCESS) {
 			k++;
-			if (k < 3) {
+			if (k < 3)
 				goto RTY_SD_RST;
-			} else {
-				rtsx_trace(chip);
-				return STATUS_FAIL;
-			}
+			else
+				goto Status_Fail;
 		}
 
 		i++;
 		wait_timeout(20);
 	} while (!(rsp[1] & 0x80) && (i < 255));
 
-	if (i == 255) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (i == 255)
+		goto Status_Fail;
 
 	if (hi_cap_flow) {
 		if (rsp[1] & 0x40)
@@ -2722,26 +2704,20 @@ RTY_SD_RST:
 
 	if (support_1v8) {
 		retval = sd_voltage_switch(chip);
-		if (retval != STATUS_SUCCESS) {
-			rtsx_trace(chip);
-			return STATUS_FAIL;
-		}
+		if (retval != STATUS_SUCCESS)
+			goto Status_Fail;
 	}
 
 	retval = sd_send_cmd_get_rsp(chip, ALL_SEND_CID, 0, SD_RSP_TYPE_R2,
 				NULL, 0);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	for (i = 0; i < 3; i++) {
 		retval = sd_send_cmd_get_rsp(chip, SEND_RELATIVE_ADDR, 0,
 					SD_RSP_TYPE_R6, rsp, 5);
-		if (retval != STATUS_SUCCESS) {
-			rtsx_trace(chip);
-			return STATUS_FAIL;
-		}
+		if (retval != STATUS_SUCCESS)
+			goto Status_Fail;
 
 		sd_card->sd_addr = (u32)rsp[1] << 24;
 		sd_card->sd_addr += (u32)rsp[2] << 16;
@@ -2751,24 +2727,18 @@ RTY_SD_RST:
 	}
 
 	retval = sd_check_csd(chip, 1);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	retval = sd_select_card(chip, 1);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 #ifdef SUPPORT_SD_LOCK
 SD_UNLOCK_ENTRY:
 	retval = sd_update_lock_status(chip);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	if (sd_card->sd_lock_status & SD_LOCKED) {
 		sd_card->sd_lock_status |= (SD_LOCK_1BIT_MODE | SD_PWD_EXIST);
@@ -2780,32 +2750,24 @@ SD_UNLOCK_ENTRY:
 
 	retval = sd_send_cmd_get_rsp(chip, APP_CMD, sd_card->sd_addr,
 				SD_RSP_TYPE_R1, NULL, 0);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	retval = sd_send_cmd_get_rsp(chip, SET_CLR_CARD_DETECT, 0,
 				SD_RSP_TYPE_R1, NULL, 0);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	if (support_1v8) {
 		retval = sd_send_cmd_get_rsp(chip, APP_CMD, sd_card->sd_addr,
 					SD_RSP_TYPE_R1, NULL, 0);
-		if (retval != STATUS_SUCCESS) {
-			rtsx_trace(chip);
-			return STATUS_FAIL;
-		}
+		if (retval != STATUS_SUCCESS)
+			goto Status_Fail;
 
 		retval = sd_send_cmd_get_rsp(chip, SET_BUS_WIDTH, 2,
 					SD_RSP_TYPE_R1, NULL, 0);
-		if (retval != STATUS_SUCCESS) {
-			rtsx_trace(chip);
-			return STATUS_FAIL;
-		}
+		if (retval != STATUS_SUCCESS)
+			goto Status_Fail;
 
 		switch_bus_width = SD_BUS_WIDTH_4;
 	} else {
@@ -2814,16 +2776,12 @@ SD_UNLOCK_ENTRY:
 
 	retval = sd_send_cmd_get_rsp(chip, SET_BLOCKLEN, 0x200, SD_RSP_TYPE_R1,
 				NULL, 0);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	retval = sd_set_clock_divider(chip, SD_CLK_DIVIDE_0);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	if (!(sd_card->raw_csd[4] & 0x40))
 		sd_dont_switch = true;
@@ -2862,17 +2820,13 @@ SD_UNLOCK_ENTRY:
 	if (!support_1v8) {
 		retval = sd_send_cmd_get_rsp(chip, APP_CMD, sd_card->sd_addr,
 					SD_RSP_TYPE_R1, NULL, 0);
-		if (retval != STATUS_SUCCESS) {
-			rtsx_trace(chip);
-			return STATUS_FAIL;
-		}
+		if (retval != STATUS_SUCCESS)
+			goto Status_Fail;
 
 		retval = sd_send_cmd_get_rsp(chip, SET_BUS_WIDTH, 2,
 					SD_RSP_TYPE_R1, NULL, 0);
-		if (retval != STATUS_SUCCESS) {
-			rtsx_trace(chip);
-			return STATUS_FAIL;
-		}
+		if (retval != STATUS_SUCCESS)
+			goto Status_Fail;
 	}
 
 #ifdef SUPPORT_SD_LOCK
@@ -2890,10 +2844,8 @@ SD_UNLOCK_ENTRY:
 		}
 
 		retval = sd_set_init_para(chip);
-		if (retval != STATUS_SUCCESS) {
-			rtsx_trace(chip);
-			return STATUS_FAIL;
-		}
+		if (retval != STATUS_SUCCESS)
+			goto Status_Fail;
 
 		if (CHK_SD_DDR50(sd_card))
 			retval = sd_ddr_tuning(chip);
@@ -2902,14 +2854,11 @@ SD_UNLOCK_ENTRY:
 
 		if (retval != STATUS_SUCCESS) {
 			if (sd20_mode) {
-				rtsx_trace(chip);
-				return STATUS_FAIL;
+				goto Status_Fail;
 			} else {
 				retval = sd_init_power(chip);
-				if (retval != STATUS_SUCCESS) {
-					rtsx_trace(chip);
-					return STATUS_FAIL;
-				}
+				if (retval != STATUS_SUCCESS)
+					goto Status_Fail;
 
 				try_sdio = false;
 				sd20_mode = true;
@@ -2930,14 +2879,11 @@ SD_UNLOCK_ENTRY:
 			retval = sd_read_lba0(chip);
 			if (retval != STATUS_SUCCESS) {
 				if (sd20_mode) {
-					rtsx_trace(chip);
-					return STATUS_FAIL;
+					goto Status_Fail;
 				} else {
 					retval = sd_init_power(chip);
-					if (retval != STATUS_SUCCESS) {
-						rtsx_trace(chip);
-						return STATUS_FAIL;
-					}
+					if (retval != STATUS_SUCCESS)
+						goto Status_Fail;
 
 					try_sdio = false;
 					sd20_mode = true;
@@ -2948,10 +2894,8 @@ SD_UNLOCK_ENTRY:
 	}
 
 	retval = sd_check_wp_state(chip);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	chip->card_bus_width[chip->card2lun[SD_CARD]] = 4;
 
@@ -2973,8 +2917,11 @@ SD_UNLOCK_ENTRY:
 #endif
 
 	return STATUS_SUCCESS;
-}
 
+Status_Fail:
+	rtsx_trace(chip);
+	return STATUS_FAIL;
+}
 
 static int mmc_test_switch_bus(struct rtsx_chip *chip, u8 width)
 {
@@ -3105,7 +3052,6 @@ static int mmc_test_switch_bus(struct rtsx_chip *chip, u8 width)
 	return SWITCH_FAIL;
 }
 
-
 static int mmc_switch_timing_bus(struct rtsx_chip *chip, bool switch_ddr)
 {
 	struct sd_info *sd_card = &(chip->sd_card);
@@ -3230,7 +3176,6 @@ static int mmc_switch_timing_bus(struct rtsx_chip *chip, bool switch_ddr)
 	return STATUS_SUCCESS;
 }
 
-
 static int reset_mmc(struct rtsx_chip *chip)
 {
 	struct sd_info *sd_card = &(chip->sd_card);
diff --git a/drivers/staging/rts5208/spi.c b/drivers/staging/rts5208/spi.c
index 26eb2a184f91..13c539c83838 100644
--- a/drivers/staging/rts5208/spi.c
+++ b/drivers/staging/rts5208/spi.c
@@ -39,7 +39,8 @@ static int spi_init(struct rtsx_chip *chip)
 	int retval;
 
 	retval = rtsx_write_register(chip, SPI_CONTROL, 0xFF,
-				     CS_POLARITY_LOW | DTO_MSB_FIRST | SPI_MASTER | SPI_MODE0 | SPI_AUTO);
+				     CS_POLARITY_LOW | DTO_MSB_FIRST
+				     | SPI_MASTER | SPI_MODE0 | SPI_AUTO);
 	if (retval) {
 		rtsx_trace(chip);
 		return retval;
diff --git a/drivers/staging/rts5208/spi.h b/drivers/staging/rts5208/spi.h
index fc824b5d8d59..c8d2beacd9e5 100644
--- a/drivers/staging/rts5208/spi.h
+++ b/drivers/staging/rts5208/spi.h
@@ -61,5 +61,4 @@ int spi_write_flash(struct scsi_cmnd *srb, struct rtsx_chip *chip);
 int spi_erase_flash(struct scsi_cmnd *srb, struct rtsx_chip *chip);
 int spi_write_flash_status(struct scsi_cmnd *srb, struct rtsx_chip *chip);
 
-
 #endif  /* __REALTEK_RTSX_SPI_H */
diff --git a/drivers/staging/rts5208/xd.c b/drivers/staging/rts5208/xd.c
index fc1dfe0991d4..1de02bb98839 100644
--- a/drivers/staging/rts5208/xd.c
+++ b/drivers/staging/rts5208/xd.c
@@ -834,7 +834,6 @@ static int xd_check_data_blank(u8 *redunt)
 		!= (XD_ECC1_ALL1 | XD_ECC2_ALL1))
 		return 0;
 
-
 	for (i = 0; i < 4; i++) {
 		if (redunt[RESERVED0 + i] != 0xFF)
 			return 0;
@@ -938,7 +937,7 @@ static void xd_set_unused_block(struct rtsx_chip *chip, u32 phy_blk)
 	dev_dbg(rtsx_dev(chip), "Set unused block to index %d\n",
 		zone->set_index);
 
-	zone->free_table[zone->set_index++] = (u16) (phy_blk & 0x3ff);
+	zone->free_table[zone->set_index++] = (u16)(phy_blk & 0x3ff);
 	if (zone->set_index >= XD_FREE_TABLE_CNT)
 		zone->set_index = 0;
 	zone->unused_blk_cnt++;
@@ -1402,7 +1401,6 @@ static int xd_erase_block(struct rtsx_chip *chip, u32 phy_blk)
 	return STATUS_FAIL;
 }
 
-
 static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 {
 	struct xd_info *xd_card = &(chip->xd_card);
@@ -1624,10 +1622,8 @@ static int xd_read_multiple_pages(struct rtsx_chip *chip, u32 phy_blk,
 	u8 reg_val, page_cnt;
 	int zone_no, retval, i;
 
-	if (start_page > end_page) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (start_page > end_page)
+		goto Status_Fail;
 
 	page_cnt = end_page - start_page;
 	zone_no = (int)(log_blk / 1000);
@@ -1643,8 +1639,7 @@ static int xd_read_multiple_pages(struct rtsx_chip *chip, u32 phy_blk,
 
 			if (detect_card_cd(chip, XD_CARD) != STATUS_SUCCESS) {
 				xd_set_err_code(chip, XD_NO_CARD);
-				rtsx_trace(chip);
-				return STATUS_FAIL;
+				goto Status_Fail;
 			}
 		}
 	}
@@ -1679,8 +1674,7 @@ static int xd_read_multiple_pages(struct rtsx_chip *chip, u32 phy_blk,
 
 		if (retval == -ETIMEDOUT) {
 			xd_set_err_code(chip, XD_TO_ERROR);
-			rtsx_trace(chip);
-			return STATUS_FAIL;
+			goto Status_Fail;
 		} else {
 			rtsx_trace(chip);
 			goto Fail;
@@ -1713,8 +1707,7 @@ Fail:
 
 		if (detect_card_cd(chip, XD_CARD) != STATUS_SUCCESS) {
 			xd_set_err_code(chip, XD_NO_CARD);
-			rtsx_trace(chip);
-			return STATUS_FAIL;
+			goto Status_Fail;
 		}
 
 		xd_set_err_code(chip, XD_ECC_ERROR);
@@ -1722,8 +1715,7 @@ Fail:
 		new_blk = xd_get_unused_block(chip, zone_no);
 		if (new_blk == NO_NEW_BLK) {
 			XD_CLR_BAD_OLDBLK(xd_card);
-			rtsx_trace(chip);
-			return STATUS_FAIL;
+			goto Status_Fail;
 		}
 
 		retval = xd_copy_page(chip, phy_blk, new_blk, 0,
@@ -1737,8 +1729,7 @@ Fail:
 				XD_CLR_BAD_NEWBLK(xd_card);
 			}
 			XD_CLR_BAD_OLDBLK(xd_card);
-			rtsx_trace(chip);
-			return STATUS_FAIL;
+			goto Status_Fail;
 		}
 		xd_set_l2p_tbl(chip, zone_no, log_off, (u16)(new_blk & 0x3FF));
 		xd_erase_block(chip, phy_blk);
@@ -1746,6 +1737,7 @@ Fail:
 		XD_CLR_BAD_OLDBLK(xd_card);
 	}
 
+Status_Fail:
 	rtsx_trace(chip);
 	return STATUS_FAIL;
 }
@@ -1830,7 +1822,6 @@ static int xd_prepare_write(struct rtsx_chip *chip,
 	return STATUS_SUCCESS;
 }
 
-
 static int xd_write_multiple_pages(struct rtsx_chip *chip, u32 old_blk,
 				u32 new_blk, u32 log_blk, u8 start_page,
 				u8 end_page, u8 *buf, unsigned int *index,
@@ -1845,10 +1836,8 @@ static int xd_write_multiple_pages(struct rtsx_chip *chip, u32 old_blk,
 	dev_dbg(rtsx_dev(chip), "%s, old_blk = 0x%x, new_blk = 0x%x, log_blk = 0x%x\n",
 		__func__, old_blk, new_blk, log_blk);
 
-	if (start_page > end_page) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (start_page > end_page)
+		goto Status_Fail;
 
 	page_cnt = end_page - start_page;
 	zone_no = (int)(log_blk / 1000);
@@ -1857,10 +1846,8 @@ static int xd_write_multiple_pages(struct rtsx_chip *chip, u32 old_blk,
 	page_addr = (new_blk << xd_card->block_shift) + start_page;
 
 	retval = xd_send_cmd(chip, READ1_1);
-	if (retval != STATUS_SUCCESS) {
-		rtsx_trace(chip);
-		return STATUS_FAIL;
-	}
+	if (retval != STATUS_SUCCESS)
+		goto Status_Fail;
 
 	rtsx_init_cmd(chip);
 
@@ -1895,8 +1882,7 @@ static int xd_write_multiple_pages(struct rtsx_chip *chip, u32 old_blk,
 
 		if (retval == -ETIMEDOUT) {
 			xd_set_err_code(chip, XD_TO_ERROR);
-			rtsx_trace(chip);
-			return STATUS_FAIL;
+			goto Status_Fail;
 		} else {
 			rtsx_trace(chip);
 			goto Fail;
@@ -1936,6 +1922,7 @@ Fail:
 		xd_mark_bad_block(chip, new_blk);
 	}
 
+Status_Fail:
 	rtsx_trace(chip);
 	return STATUS_FAIL;
 }
@@ -2000,7 +1987,6 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 		return STATUS_FAIL;
 	}
 
-
 	if (detect_card_cd(chip, XD_CARD) != STATUS_SUCCESS) {
 		chip->card_fail |= XD_CARD;
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_NOT_PRESENT);
diff --git a/drivers/staging/slicoss/slic.h b/drivers/staging/slicoss/slic.h
index cc0afeeb68c1..420546d43002 100644
--- a/drivers/staging/slicoss/slic.h
+++ b/drivers/staging/slicoss/slic.h
@@ -92,6 +92,7 @@ struct slic_rcvbuf_info {
 	u32     lasttime;
 	u32     lastid;
 };
+
 /*
  * SLIC Handle structure.  Used to restrict handle values to
  * 32 bits by using an index rather than an address.
@@ -223,8 +224,8 @@ struct mcast_address {
 
 struct slic_iface_stats {
 	/*
-	* Stats
-	*/
+	 * Stats
+	 */
 	u64        xmt_bytes;
 	u64        xmt_ucast;
 	u64        xmt_mcast;
@@ -351,10 +352,35 @@ struct base_driver {
 	uint                 cardnuminuse[SLIC_MAX_CARDS];
 };
 
-struct slic_shmem {
-	volatile u32          isr;
-	volatile u32          linkstatus;
-	volatile struct slic_stats     inicstats;
+struct slic_stats {
+	/* xmit stats */
+	u64 xmit_tcp_bytes;
+	u64 xmit_tcp_segs;
+	u64 xmit_bytes;
+	u64 xmit_collisions;
+	u64 xmit_unicasts;
+	u64 xmit_other_error;
+	u64 xmit_excess_collisions;
+	/* rcv stats */
+	u64 rcv_tcp_bytes;
+	u64 rcv_tcp_segs;
+	u64 rcv_bytes;
+	u64 rcv_unicasts;
+	u64 rcv_other_error;
+	u64 rcv_drops;
+};
+
+struct slic_shmem_data {
+	u32 isr;
+	u32 lnkstatus;
+	struct slic_stats stats;
+};
+
+struct slic_shmemory {
+	dma_addr_t isr_phaddr;
+	dma_addr_t lnkstatus_phaddr;
+	dma_addr_t stats_phaddr;
+	struct slic_shmem_data __iomem *shmem_data;
 };
 
 struct slic_upr {
@@ -414,10 +440,9 @@ struct adapter {
 	u32             intrregistered;
 	uint                isp_initialized;
 	uint                gennumber;
-	struct slic_shmem      *pshmem;
+	struct slic_shmemory shmem;
 	dma_addr_t          phys_shmem;
-	u32             isrcopy;
-	__iomem struct slic_regs       *slic_regs;
+	void __iomem *regs;
 	unsigned char               state;
 	unsigned char               linkstate;
 	unsigned char               linkspeed;
@@ -444,8 +469,8 @@ struct adapter {
 	struct slic_cmdqueue     cmdq_all;
 	struct slic_cmdqmem      cmdqmem;
 	/*
-	*  SLIC Handles
-	*/
+	 * SLIC Handles
+	 */
 	/* Object handles*/
 	struct slic_handle slic_handles[SLIC_CMDQ_MAXCMDS + 1];
 	/* Free object handles*/
@@ -487,6 +512,34 @@ struct adapter {
 	struct slicnet_stats     slic_stats;
 };
 
+static inline u32 slic_read32(struct adapter *adapter, unsigned int reg)
+{
+	return ioread32(adapter->regs + reg);
+}
+
+static inline void slic_write32(struct adapter *adapter, unsigned int reg,
+				u32 val)
+{
+	iowrite32(val, adapter->regs + reg);
+}
+
+static inline void slic_write64(struct adapter *adapter, unsigned int reg,
+				u32 val, u32 hiaddr)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->bit64reglock, flags);
+	slic_write32(adapter, SLIC_REG_ADDR_UPPER, hiaddr);
+	slic_write32(adapter, reg, val);
+	mmiowb();
+	spin_unlock_irqrestore(&adapter->bit64reglock, flags);
+}
+
+static inline void slic_flush_write(struct adapter *adapter)
+{
+	ioread32(adapter->regs + SLIC_REG_HOSTID);
+}
+
 #define UPDATE_STATS(largestat, newstat, oldstat)                        \
 {                                                                        \
 	if ((newstat) < (oldstat))                                       \
diff --git a/drivers/staging/slicoss/slichw.h b/drivers/staging/slicoss/slichw.h
index 9723b4a104f7..49cb91aa02bb 100644
--- a/drivers/staging/slicoss/slichw.h
+++ b/drivers/staging/slicoss/slichw.h
@@ -289,224 +289,118 @@ struct slic_rspbuf {
 	u32 pad2[4];
 };
 
-struct slic_regs {
-	u32	slic_reset;	/* Reset Register */
-	u32	pad0;
-
-	u32	slic_icr;	/* Interrupt Control Register */
-	u32	pad2;
-#define SLIC_ICR		0x0008
-
-	u32	slic_isp;	/* Interrupt status pointer */
-	u32	pad1;
-#define SLIC_ISP		0x0010
-
-	u32	slic_isr;	/* Interrupt status */
-	u32	pad3;
-#define SLIC_ISR		0x0018
-
-	u32	slic_hbar;	/* Header buffer address reg */
-	u32	pad4;
-	/*
-	 * 31-8 - phy addr of set of contiguous hdr buffers
-	 *  7-0 - number of buffers passed
-	 * Buffers are 256 bytes long on 256-byte boundaries.
-	 */
-#define SLIC_HBAR		0x0020
-#define SLIC_HBAR_CNT_MSK	0x000000FF
-
-	u32	slic_dbar;	/* Data buffer handle & address reg */
-	u32	pad5;
-
-	/* 4 sets of registers; Buffers are 2K bytes long 2 per 4K page. */
-#define SLIC_DBAR		0x0028
-#define SLIC_DBAR_SIZE		2048
-
-	u32	slic_cbar;	/* Xmt Cmd buf addr regs.*/
-	/*
-	 * 1 per XMT interface
-	 * 31-5 - phy addr of host command buffer
-	 *  4-0 - length of cmd in multiples of 32 bytes
-	 * Buffers are 32 bytes up to 512 bytes long
-	 */
-#define SLIC_CBAR		0x0030
-#define SLIC_CBAR_LEN_MSK	0x0000001F
-#define SLIC_CBAR_ALIGN		0x00000020
-
-	u32	slic_wcs;	/* write control store*/
-#define	SLIC_WCS		0x0034
-#define SLIC_WCS_START		0x80000000	/*Start the SLIC (Jump to WCS)*/
-#define SLIC_WCS_COMPARE	0x40000000	/* Compare with value in WCS*/
-
-	u32	slic_rbar;	/* Response buffer address reg.*/
-	u32	pad7;
-	/*
-	 * 31-8 - phy addr of set of contiguous response buffers
-	 * 7-0 - number of buffers passed
-	 * Buffers are 32 bytes long on 32-byte boundaries.
-	 */
-#define SLIC_RBAR		0x0038
-#define SLIC_RBAR_CNT_MSK	0x000000FF
-#define SLIC_RBAR_SIZE		32
-
-	u32	slic_stats;	/* read statistics (UPR) */
-	u32	pad8;
-#define	SLIC_RSTAT		0x0040
-
-	u32	slic_rlsr;	/* read link status */
-	u32	pad9;
-#define SLIC_LSTAT		0x0048
-
-	u32	slic_wmcfg;	/* Write Mac Config */
-	u32	pad10;
-#define	SLIC_WMCFG		0x0050
-
-	u32	slic_wphy;	/* Write phy register */
-	u32	pad11;
-#define SLIC_WPHY		0x0058
-
-	u32	slic_rcbar;	/* Rcv Cmd buf addr reg */
-	u32	pad12;
-#define	SLIC_RCBAR		0x0060
-
-	u32	slic_rconfig;	/* Read SLIC Config*/
-	u32	pad13;
-#define SLIC_RCONFIG	0x0068
-
-	u32	slic_intagg;	/* Interrupt aggregation time */
-	u32	pad14;
-#define SLIC_INTAGG		0x0070
-
-	u32	slic_wxcfg;	/* Write XMIT config reg*/
-	u32	pad16;
-#define	SLIC_WXCFG		0x0078
-
-	u32	slic_wrcfg;	/* Write RCV config reg*/
-	u32	pad17;
-#define	SLIC_WRCFG		0x0080
-
-	u32	slic_wraddral;	/* Write rcv addr a low*/
-	u32	pad18;
-#define	SLIC_WRADDRAL	0x0088
-
-	u32	slic_wraddrah;	/* Write rcv addr a high*/
-	u32	pad19;
-#define	SLIC_WRADDRAH	0x0090
-
-	u32	slic_wraddrbl;	/* Write rcv addr b low*/
-	u32	pad20;
-#define	SLIC_WRADDRBL	0x0098
-
-	u32	slic_wraddrbh;	/* Write rcv addr b high*/
-	u32		pad21;
-#define	SLIC_WRADDRBH	0x00a0
-
-	u32	slic_mcastlow;	/* Low bits of mcast mask*/
-	u32		pad22;
-#define	SLIC_MCASTLOW	0x00a8
-
-	u32	slic_mcasthigh;	/* High bits of mcast mask*/
-	u32		pad23;
-#define	SLIC_MCASTHIGH	0x00b0
-
-	u32	slic_ping;	/* Ping the card*/
-	u32	pad24;
-#define SLIC_PING		0x00b8
-
-	u32	slic_dump_cmd;	/* Dump command */
-	u32	pad25;
-#define SLIC_DUMP_CMD	0x00c0
-
-	u32	slic_dump_data;	/* Dump data pointer */
-	u32	pad26;
-#define SLIC_DUMP_DATA	0x00c8
-
-	u32	slic_pcistatus;	/* Read card's pci_status register */
-	u32	pad27;
-#define	SLIC_PCISTATUS	0x00d0
-
-	u32	slic_wrhostid;	/* Write hostid field */
-	u32		pad28;
-#define SLIC_WRHOSTID		 0x00d8
-#define SLIC_RDHOSTID_1GB	 0x1554
-#define SLIC_RDHOSTID_2GB	 0x1554
-
-	u32	slic_low_power;	/* Put card in a low power state */
-	u32	pad29;
-#define SLIC_LOW_POWER	0x00e0
-
-	u32	slic_quiesce;	/* force slic into quiescent state
-				 * before soft reset
-				 */
-	u32	pad30;
-#define SLIC_QUIESCE	0x00e8
-
-	u32	slic_reset_iface;/* reset interface queues */
-	u32	pad31;
-#define SLIC_RESET_IFACE 0x00f0
-
-	u32	slic_addr_upper;/* Bits 63-32 for host i/f addrs */
-	u32	pad32;
-#define SLIC_ADDR_UPPER	0x00f8 /*Register is only written when it has changed*/
-
-	u32	slic_hbar64;	/* 64 bit Header buffer address reg */
-	u32	pad33;
-#define SLIC_HBAR64		0x0100
-
-	u32	slic_dbar64;	/* 64 bit Data buffer handle & address reg */
-	u32	pad34;
-#define SLIC_DBAR64		0x0108
-
-	u32	slic_cbar64;	/* 64 bit Xmt Cmd buf addr regs. */
-	u32	pad35;
-#define SLIC_CBAR64		0x0110
-
-	u32	slic_rbar64;	/* 64 bit Response buffer address reg.*/
-	u32	pad36;
-#define SLIC_RBAR64		0x0118
-
-	u32	slic_rcbar64;	/* 64 bit Rcv Cmd buf addr reg*/
-	u32	pad37;
-#define	SLIC_RCBAR64	0x0120
-
-	u32	slic_stats64;	/* read statistics (64 bit UPR) */
-	u32	pad38;
-#define	SLIC_RSTAT64	0x0128
-
-	u32	slic_rcv_wcs;	/*Download Gigabit RCV sequencer ucode*/
-	u32	pad39;
-#define SLIC_RCV_WCS	0x0130
-#define SLIC_RCVWCS_BEGIN	0x40000000
-#define SLIC_RCVWCS_FINISH	0x80000000
-
-	u32	slic_wrvlanid;	/* Write VlanId field */
-	u32	pad40;
-#define SLIC_WRVLANID	0x0138
-
-	u32	slic_read_xf_info;	/* Read Transformer info */
-	u32	pad41;
-#define SLIC_READ_XF_INFO	0x0140
-
-	u32	slic_write_xf_info;	/* Write Transformer info */
-	u32	pad42;
-#define SLIC_WRITE_XF_INFO	0x0148
-
-	u32	RSVD1;		/* TOE Only */
-	u32	pad43;
-
-	u32	RSVD2;		/* TOE Only */
-	u32	pad44;
-
-	u32	RSVD3;		/* TOE Only */
-	u32	pad45;
-
-	u32	RSVD4;		/* TOE Only */
-	u32	pad46;
-
-	u32	slic_ticks_per_sec; /* Write card ticks per second */
-	u32	pad47;
-#define SLIC_TICKS_PER_SEC	0x0170
-};
+/* Reset Register */
+#define SLIC_REG_RESET		0x0000
+/* Interrupt Control Register */
+#define SLIC_REG_ICR		0x0008
+/* Interrupt status pointer */
+#define SLIC_REG_ISP		0x0010
+/* Interrupt status */
+#define SLIC_REG_ISR		0x0018
+/*
+ * Header buffer address reg
+ * 31-8 - phy addr of set of contiguous hdr buffers
+ *  7-0 - number of buffers passed
+ * Buffers are 256 bytes long on 256-byte boundaries.
+ */
+#define SLIC_REG_HBAR		0x0020
+/*
+ * Data buffer handle & address reg
+ * 4 sets of registers; Buffers are 2K bytes long 2 per 4K page.
+ */
+#define SLIC_REG_DBAR		0x0028
+/*
+ * Xmt Cmd buf addr regs.
+ * 1 per XMT interface
+ * 31-5 - phy addr of host command buffer
+ *  4-0 - length of cmd in multiples of 32 bytes
+ * Buffers are 32 bytes up to 512 bytes long
+ */
+#define SLIC_REG_CBAR		0x0030
+/* Write control store */
+#define	SLIC_REG_WCS		0x0034
+/*
+ * Response buffer address reg.
+ * 31-8 - phy addr of set of contiguous response buffers
+ * 7-0 - number of buffers passed
+ * Buffers are 32 bytes long on 32-byte boundaries.
+ */
+#define	SLIC_REG_RBAR		0x0038
+/* Read statistics (UPR) */
+#define	SLIC_REG_RSTAT		0x0040
+/* Read link status */
+#define	SLIC_REG_LSTAT		0x0048
+/* Write Mac Config */
+#define	SLIC_REG_WMCFG		0x0050
+/* Write phy register */
+#define SLIC_REG_WPHY		0x0058
+/* Rcv Cmd buf addr reg */
+#define	SLIC_REG_RCBAR		0x0060
+/* Read SLIC Config*/
+#define SLIC_REG_RCONFIG	0x0068
+/* Interrupt aggregation time */
+#define SLIC_REG_INTAGG		0x0070
+/* Write XMIT config reg */
+#define	SLIC_REG_WXCFG		0x0078
+/* Write RCV config reg */
+#define	SLIC_REG_WRCFG		0x0080
+/* Write rcv addr a low */
+#define	SLIC_REG_WRADDRAL	0x0088
+/* Write rcv addr a high */
+#define	SLIC_REG_WRADDRAH	0x0090
+/* Write rcv addr b low */
+#define	SLIC_REG_WRADDRBL	0x0098
+/* Write rcv addr b high */
+#define	SLIC_REG_WRADDRBH	0x00a0
+/* Low bits of mcast mask */
+#define	SLIC_REG_MCASTLOW	0x00a8
+/* High bits of mcast mask */
+#define	SLIC_REG_MCASTHIGH	0x00b0
+/* Ping the card */
+#define SLIC_REG_PING		0x00b8
+/* Dump command */
+#define SLIC_REG_DUMP_CMD	0x00c0
+/* Dump data pointer */
+#define SLIC_REG_DUMP_DATA	0x00c8
+/* Read card's pci_status register */
+#define	SLIC_REG_PCISTATUS	0x00d0
+/* Write hostid field */
+#define SLIC_REG_WRHOSTID	0x00d8
+/* Put card in a low power state */
+#define SLIC_REG_LOW_POWER	0x00e0
+/* Force slic into quiescent state  before soft reset */
+#define SLIC_REG_QUIESCE	0x00e8
+/* Reset interface queues */
+#define SLIC_REG_RESET_IFACE	0x00f0
+/*
+ * Register is only written when it has changed.
+ * Bits 63-32 for host i/f addrs.
+ */
+#define SLIC_REG_ADDR_UPPER	0x00f8
+/* 64 bit Header buffer address reg */
+#define SLIC_REG_HBAR64		0x0100
+/* 64 bit Data buffer handle & address reg */
+#define SLIC_REG_DBAR64		0x0108
+/* 64 bit Xmt Cmd buf addr regs. */
+#define SLIC_REG_CBAR64		0x0110
+/* 64 bit Response buffer address reg.*/
+#define SLIC_REG_RBAR64		0x0118
+/* 64 bit Rcv Cmd buf addr reg*/
+#define	SLIC_REG_RCBAR64	0x0120
+/* Read statistics (64 bit UPR) */
+#define	SLIC_REG_RSTAT64	0x0128
+/* Download Gigabit RCV sequencer ucode */
+#define SLIC_REG_RCV_WCS	0x0130
+/* Write VlanId field */
+#define SLIC_REG_WRVLANID	0x0138
+/* Read Transformer info */
+#define SLIC_REG_READ_XF_INFO	0x0140
+/* Write Transformer info */
+#define SLIC_REG_WRITE_XF_INFO	0x0148
+/* Write card ticks per second */
+#define SLIC_REG_TICKS_PER_SEC	0x0170
+
+#define SLIC_REG_HOSTID		0x1554
 
 enum UPR_REQUEST {
 	SLIC_UPR_STATS,
@@ -565,85 +459,6 @@ struct slic_pnp_capabilities {
 	struct slicpm_wakeup_capabilities wakeup_capabilities;
 };
 
-struct xmt_stats {
-	u32 xmit_tcp_bytes;
-	u32 xmit_tcp_segs;
-	u32 xmit_bytes;
-	u32 xmit_collisions;
-	u32 xmit_unicasts;
-	u32 xmit_other_error;
-	u32 xmit_excess_collisions;
-};
-
-struct rcv_stats {
-	u32 rcv_tcp_bytes;
-	u32 rcv_tcp_segs;
-	u32 rcv_bytes;
-	u32 rcv_unicasts;
-	u32 rcv_other_error;
-	u32 rcv_drops;
-};
-
-struct xmt_statsgb {
-	u64 xmit_tcp_bytes;
-	u64 xmit_tcp_segs;
-	u64 xmit_bytes;
-	u64 xmit_collisions;
-	u64 xmit_unicasts;
-	u64 xmit_other_error;
-	u64 xmit_excess_collisions;
-};
-
-struct rcv_statsgb {
-	u64 rcv_tcp_bytes;
-	u64 rcv_tcp_segs;
-	u64 rcv_bytes;
-	u64 rcv_unicasts;
-	u64 rcv_other_error;
-	u64 rcv_drops;
-};
-
-struct slic_stats {
-	union {
-		struct {
-			struct xmt_stats xmt100;
-			struct rcv_stats rcv100;
-		} stats_100;
-		struct {
-			struct xmt_statsgb xmtGB;
-			struct rcv_statsgb rcvGB;
-		} stats_GB;
-	} u;
-};
-
-#define xmit_tcp_segs100		u.stats_100.xmt100.xmit_tcp_segs
-#define xmit_tcp_bytes100		u.stats_100.xmt100.xmit_tcp_bytes
-#define xmit_bytes100			u.stats_100.xmt100.xmit_bytes
-#define xmit_collisions100		u.stats_100.xmt100.xmit_collisions
-#define xmit_unicasts100		u.stats_100.xmt100.xmit_unicasts
-#define xmit_other_error100		u.stats_100.xmt100.xmit_other_error
-#define xmit_excess_collisions100	u.stats_100.xmt100.xmit_excess_collisions
-#define rcv_tcp_segs100			u.stats_100.rcv100.rcv_tcp_segs
-#define rcv_tcp_bytes100		u.stats_100.rcv100.rcv_tcp_bytes
-#define rcv_bytes100			u.stats_100.rcv100.rcv_bytes
-#define rcv_unicasts100			u.stats_100.rcv100.rcv_unicasts
-#define rcv_other_error100		u.stats_100.rcv100.rcv_other_error
-#define rcv_drops100			u.stats_100.rcv100.rcv_drops
-#define xmit_tcp_segs_gb		u.stats_GB.xmtGB.xmit_tcp_segs
-#define xmit_tcp_bytes_gb		u.stats_GB.xmtGB.xmit_tcp_bytes
-#define xmit_bytes_gb			u.stats_GB.xmtGB.xmit_bytes
-#define xmit_collisions_gb		u.stats_GB.xmtGB.xmit_collisions
-#define xmit_unicasts_gb		u.stats_GB.xmtGB.xmit_unicasts
-#define xmit_other_error_gb		u.stats_GB.xmtGB.xmit_other_error
-#define xmit_excess_collisions_gb	u.stats_GB.xmtGB.xmit_excess_collisions
-
-#define rcv_tcp_segs_gb			u.stats_GB.rcvGB.rcv_tcp_segs
-#define rcv_tcp_bytes_gb		u.stats_GB.rcvGB.rcv_tcp_bytes
-#define rcv_bytes_gb			u.stats_GB.rcvGB.rcv_bytes
-#define rcv_unicasts_gb			u.stats_GB.rcvGB.rcv_unicasts
-#define rcv_other_error_gb		u.stats_GB.rcvGB.rcv_other_error
-#define rcv_drops_gb			u.stats_GB.rcvGB.rcv_drops
-
 struct slic_config_mac {
 	u8 macaddrA[6];
 };
diff --git a/drivers/staging/slicoss/slicoss.c b/drivers/staging/slicoss/slicoss.c
index ac126d4f3117..062307ad7fed 100644
--- a/drivers/staging/slicoss/slicoss.c
+++ b/drivers/staging/slicoss/slicoss.c
@@ -124,31 +124,10 @@ static const struct pci_device_id slic_pci_tbl[] = {
 	{ 0 }
 };
 
-static struct ethtool_ops slic_ethtool_ops;
+static const struct ethtool_ops slic_ethtool_ops;
 
 MODULE_DEVICE_TABLE(pci, slic_pci_tbl);
 
-static inline void slic_reg32_write(void __iomem *reg, u32 value, bool flush)
-{
-	writel(value, reg);
-	if (flush)
-		mb();
-}
-
-static inline void slic_reg64_write(struct adapter *adapter, void __iomem *reg,
-				    u32 value, void __iomem *regh, u32 paddrh,
-				    bool flush)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&adapter->bit64reglock, flags);
-	writel(paddrh, regh);
-	writel(value, reg);
-	if (flush)
-		mb();
-	spin_unlock_irqrestore(&adapter->bit64reglock, flags);
-}
-
 static void slic_mcast_set_bit(struct adapter *adapter, char *address)
 {
 	unsigned char crcpoly;
@@ -172,8 +151,6 @@ static void slic_mcast_set_bit(struct adapter *adapter, char *address)
 
 static void slic_mcast_set_mask(struct adapter *adapter)
 {
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
-
 	if (adapter->macopts & (MAC_ALLMCAST | MAC_PROMISC)) {
 		/*
 		 * Turn on all multicast addresses. We have to do this for
@@ -181,18 +158,17 @@ static void slic_mcast_set_mask(struct adapter *adapter)
 		 * Microcode from having to keep state about the MAC
 		 * configuration.
 		 */
-		slic_reg32_write(&slic_regs->slic_mcastlow, 0xFFFFFFFF, FLUSH);
-		slic_reg32_write(&slic_regs->slic_mcasthigh, 0xFFFFFFFF,
-				 FLUSH);
+		slic_write32(adapter, SLIC_REG_MCASTLOW, 0xFFFFFFFF);
+		slic_write32(adapter, SLIC_REG_MCASTHIGH, 0xFFFFFFFF);
 	} else {
 		/*
 		 * Commit our multicast mast to the SLIC by writing to the
 		 * multicast address mask registers
 		 */
-		slic_reg32_write(&slic_regs->slic_mcastlow,
-			(u32)(adapter->mcastmask & 0xFFFFFFFF), FLUSH);
-		slic_reg32_write(&slic_regs->slic_mcasthigh,
-			(u32)((adapter->mcastmask >> 32) & 0xFFFFFFFF), FLUSH);
+		slic_write32(adapter, SLIC_REG_MCASTLOW,
+			     (u32)(adapter->mcastmask & 0xFFFFFFFF));
+		slic_write32(adapter, SLIC_REG_MCASTHIGH,
+			     (u32)((adapter->mcastmask >> 32) & 0xFFFFFFFF));
 	}
 }
 
@@ -208,13 +184,6 @@ static void slic_timer_ping(ulong dev)
 	add_timer(&adapter->pingtimer);
 }
 
-static void slic_unmap_mmio_space(struct adapter *adapter)
-{
-	if (adapter->slic_regs)
-		iounmap(adapter->slic_regs);
-	adapter->slic_regs = NULL;
-}
-
 /*
  *  slic_link_config
  *
@@ -224,7 +193,6 @@ static void slic_unmap_mmio_space(struct adapter *adapter)
 static void slic_link_config(struct adapter *adapter,
 		      u32 linkspeed, u32 linkduplex)
 {
-	u32 __iomem *wphy;
 	u32 speed;
 	u32 duplex;
 	u32 phy_config;
@@ -239,8 +207,6 @@ static void slic_link_config(struct adapter *adapter,
 	if (linkduplex > LINK_AUTOD)
 		linkduplex = LINK_AUTOD;
 
-	wphy = &adapter->slic_regs->slic_wphy;
-
 	if ((linkspeed == LINK_AUTOSPEED) || (linkspeed == LINK_1000MB)) {
 		if (adapter->flags & ADAPT_FLAGS_FIBERMEDIA) {
 			/*
@@ -252,7 +218,7 @@ static void slic_link_config(struct adapter *adapter,
 			phy_advreg = (MIICR_REG_4 | (PAR_ADV1000XFD));
 			/* enable PAUSE frames        */
 			phy_advreg |= PAR_ASYMPAUSE_FIBER;
-			slic_reg32_write(wphy, phy_advreg, FLUSH);
+			slic_write32(adapter, SLIC_REG_WPHY, phy_advreg);
 
 			if (linkspeed == LINK_AUTOSPEED) {
 				/* reset phy, enable auto-neg  */
@@ -260,14 +226,17 @@ static void slic_link_config(struct adapter *adapter,
 				    (MIICR_REG_PCR |
 				     (PCR_RESET | PCR_AUTONEG |
 				      PCR_AUTONEG_RST));
-				slic_reg32_write(wphy, phy_config, FLUSH);
+				slic_write32(adapter, SLIC_REG_WPHY,
+					     phy_config);
 			} else {	/* forced 1000 Mb FD*/
 				/*
 				 * power down phy to break link
 				 * this may not work)
 				 */
 				phy_config = (MIICR_REG_PCR | PCR_POWERDOWN);
-				slic_reg32_write(wphy, phy_config, FLUSH);
+				slic_write32(adapter, SLIC_REG_WPHY,
+					     phy_config);
+				slic_flush_write(adapter);
 				/*
 				 * wait, Marvell says 1 sec,
 				 * try to get away with 10 ms
@@ -282,7 +251,8 @@ static void slic_link_config(struct adapter *adapter,
 				    (MIICR_REG_PCR |
 				     (PCR_RESET | PCR_SPEED_1000 |
 				      PCR_DUPLEX_FULL));
-				slic_reg32_write(wphy, phy_config, FLUSH);
+				slic_write32(adapter, SLIC_REG_WPHY,
+					     phy_config);
 			}
 		} else {	/* copper gigabit */
 
@@ -309,10 +279,10 @@ static void slic_link_config(struct adapter *adapter,
 			phy_advreg |= PAR_ASYMPAUSE;
 			/* required by the Cicada PHY  */
 			phy_advreg |= PAR_802_3;
-			slic_reg32_write(wphy, phy_advreg, FLUSH);
+			slic_write32(adapter, SLIC_REG_WPHY, phy_advreg);
 			/* advertise FD only @1000 Mb  */
 			phy_gctlreg = (MIICR_REG_9 | (PGC_ADV1000FD));
-			slic_reg32_write(wphy, phy_gctlreg, FLUSH);
+			slic_write32(adapter, SLIC_REG_WPHY, phy_gctlreg);
 
 			if (adapter->subsysid != SLIC_1GB_CICADA_SUBSYS_ID) {
 				/*
@@ -321,20 +291,23 @@ static void slic_link_config(struct adapter *adapter,
 				 */
 				phy_config =
 				    (MIICR_REG_16 | (MRV_REG16_XOVERON));
-				slic_reg32_write(wphy, phy_config, FLUSH);
+				slic_write32(adapter, SLIC_REG_WPHY,
+					     phy_config);
 
 				/* reset phy, enable auto-neg  */
 				phy_config =
 				    (MIICR_REG_PCR |
 				     (PCR_RESET | PCR_AUTONEG |
 				      PCR_AUTONEG_RST));
-				slic_reg32_write(wphy, phy_config, FLUSH);
+				slic_write32(adapter, SLIC_REG_WPHY,
+					     phy_config);
 			} else {	/* it's a Cicada PHY  */
 				/* enable and restart auto-neg (don't reset)  */
 				phy_config =
 				    (MIICR_REG_PCR |
 				     (PCR_AUTONEG | PCR_AUTONEG_RST));
-				slic_reg32_write(wphy, phy_config, FLUSH);
+				slic_write32(adapter, SLIC_REG_WPHY,
+					     phy_config);
 			}
 		}
 	} else {
@@ -354,13 +327,13 @@ static void slic_link_config(struct adapter *adapter,
 			 * disable auto crossover
 			 */
 			phy_config = (MIICR_REG_16 | (MRV_REG16_XOVEROFF));
-			slic_reg32_write(wphy, phy_config, FLUSH);
+			slic_write32(adapter, SLIC_REG_WPHY, phy_config);
 		}
 
 		/* power down phy to break link (this may not work)  */
 		phy_config = (MIICR_REG_PCR | (PCR_POWERDOWN | speed | duplex));
-		slic_reg32_write(wphy, phy_config, FLUSH);
-
+		slic_write32(adapter, SLIC_REG_WPHY, phy_config);
+		slic_flush_write(adapter);
 		/* wait, Marvell says 1 sec, try to get away with 10 ms */
 		mdelay(10);
 
@@ -372,11 +345,11 @@ static void slic_link_config(struct adapter *adapter,
 			 */
 			phy_config =
 			    (MIICR_REG_PCR | (PCR_RESET | speed | duplex));
-			slic_reg32_write(wphy, phy_config, FLUSH);
+			slic_write32(adapter, SLIC_REG_WPHY, phy_config);
 		} else {	/* it's a Cicada PHY  */
 			/* disable auto-neg, set speed, powerup  */
 			phy_config = (MIICR_REG_PCR | (speed | duplex));
-			slic_reg32_write(wphy, phy_config, FLUSH);
+			slic_write32(adapter, SLIC_REG_WPHY, phy_config);
 		}
 	}
 }
@@ -386,7 +359,6 @@ static int slic_card_download_gbrcv(struct adapter *adapter)
 	const struct firmware *fw;
 	const char *file = "";
 	int ret;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
 	u32 codeaddr;
 	u32 instruction;
 	int index = 0;
@@ -427,27 +399,28 @@ static int slic_card_download_gbrcv(struct adapter *adapter)
 		break;
 	}
 	/* start download */
-	slic_reg32_write(&slic_regs->slic_rcv_wcs, SLIC_RCVWCS_BEGIN, FLUSH);
+	slic_write32(adapter, SLIC_REG_RCV_WCS, SLIC_RCVWCS_BEGIN);
 	/* download the rcv sequencer ucode */
 	for (codeaddr = 0; codeaddr < rcvucodelen; codeaddr++) {
 		/* write out instruction address */
-		slic_reg32_write(&slic_regs->slic_rcv_wcs, codeaddr, FLUSH);
+		slic_write32(adapter, SLIC_REG_RCV_WCS, codeaddr);
 
 		instruction = *(u32 *)(fw->data + index);
 		index += 4;
 		/* write out the instruction data low addr */
-		slic_reg32_write(&slic_regs->slic_rcv_wcs, instruction, FLUSH);
+		slic_write32(adapter, SLIC_REG_RCV_WCS, instruction);
 
 		instruction = *(u8 *)(fw->data + index);
 		index++;
 		/* write out the instruction data high addr */
-		slic_reg32_write(&slic_regs->slic_rcv_wcs, (u8)instruction,
-				 FLUSH);
+		slic_write32(adapter, SLIC_REG_RCV_WCS, instruction);
 	}
 
 	/* download finished */
 	release_firmware(fw);
-	slic_reg32_write(&slic_regs->slic_rcv_wcs, SLIC_RCVWCS_FINISH, FLUSH);
+	slic_write32(adapter, SLIC_REG_RCV_WCS, SLIC_RCVWCS_FINISH);
+	slic_flush_write(adapter);
+
 	return 0;
 }
 
@@ -462,7 +435,6 @@ static int slic_card_download(struct adapter *adapter)
 	u32 section;
 	int thissectionsize;
 	int codeaddr;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
 	u32 instruction;
 	u32 baseaddress;
 	u32 i;
@@ -506,17 +478,17 @@ static int slic_card_download(struct adapter *adapter)
 
 		for (codeaddr = 0; codeaddr < thissectionsize; codeaddr++) {
 			/* Write out instruction address */
-			slic_reg32_write(&slic_regs->slic_wcs,
-					 baseaddress + codeaddr, FLUSH);
+			slic_write32(adapter, SLIC_REG_WCS,
+				     baseaddress + codeaddr);
 			/* Write out instruction to low addr */
-			slic_reg32_write(&slic_regs->slic_wcs,
-					instruction, FLUSH);
+			slic_write32(adapter, SLIC_REG_WCS,
+				     instruction);
 			instruction = *(u32 *)(fw->data + index);
 			index += 4;
 
 			/* Write out instruction to high addr */
-			slic_reg32_write(&slic_regs->slic_wcs,
-					instruction, FLUSH);
+			slic_write32(adapter, SLIC_REG_WCS,
+				     instruction);
 			instruction = *(u32 *)(fw->data + index);
 			index += 4;
 		}
@@ -531,27 +503,25 @@ static int slic_card_download(struct adapter *adapter)
 
 		for (codeaddr = 0; codeaddr < thissectionsize; codeaddr++) {
 			/* Write out instruction address */
-			slic_reg32_write(&slic_regs->slic_wcs,
-				SLIC_WCS_COMPARE | (baseaddress + codeaddr),
-				FLUSH);
+			slic_write32(adapter, SLIC_REG_WCS,
+				     SLIC_WCS_COMPARE | (baseaddress +
+							 codeaddr));
 			/* Write out instruction to low addr */
-			slic_reg32_write(&slic_regs->slic_wcs, instruction,
-					 FLUSH);
+			slic_write32(adapter, SLIC_REG_WCS, instruction);
 			instruction = *(u32 *)(fw->data + index);
 			index += 4;
 			/* Write out instruction to high addr */
-			slic_reg32_write(&slic_regs->slic_wcs, instruction,
-					 FLUSH);
+			slic_write32(adapter, SLIC_REG_WCS, instruction);
 			instruction = *(u32 *)(fw->data + index);
 			index += 4;
-
 		}
 	}
 	release_firmware(fw);
 	/* Everything OK, kick off the card */
 	mdelay(10);
-	slic_reg32_write(&slic_regs->slic_wcs, SLIC_WCS_START, FLUSH);
 
+	slic_write32(adapter, SLIC_REG_WCS, SLIC_WCS_START);
+	slic_flush_write(adapter);
 	/*
 	 * stall for 20 ms, long enough for ucode to init card
 	 * and reach mainloop
@@ -583,19 +553,21 @@ static void slic_adapter_set_hwaddr(struct adapter *adapter)
 
 static void slic_intagg_set(struct adapter *adapter, u32 value)
 {
-	slic_reg32_write(&adapter->slic_regs->slic_intagg, value, FLUSH);
+	slic_write32(adapter, SLIC_REG_INTAGG, value);
 	adapter->card->loadlevel_current = value;
 }
 
 static void slic_soft_reset(struct adapter *adapter)
 {
 	if (adapter->card->state == CARD_UP) {
-		slic_reg32_write(&adapter->slic_regs->slic_quiesce, 0, FLUSH);
+		slic_write32(adapter, SLIC_REG_QUIESCE, 0);
+		slic_flush_write(adapter);
 		mdelay(1);
 	}
 
-	slic_reg32_write(&adapter->slic_regs->slic_reset, SLIC_RESET_MAGIC,
-			 FLUSH);
+	slic_write32(adapter, SLIC_REG_RESET, SLIC_RESET_MAGIC);
+	slic_flush_write(adapter);
+
 	mdelay(1);
 }
 
@@ -603,17 +575,16 @@ static void slic_mac_address_config(struct adapter *adapter)
 {
 	u32 value;
 	u32 value2;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
 
 	value = ntohl(*(__be32 *)&adapter->currmacaddr[2]);
-	slic_reg32_write(&slic_regs->slic_wraddral, value, FLUSH);
-	slic_reg32_write(&slic_regs->slic_wraddrbl, value, FLUSH);
+	slic_write32(adapter, SLIC_REG_WRADDRAL, value);
+	slic_write32(adapter, SLIC_REG_WRADDRBL, value);
 
 	value2 = (u32)((adapter->currmacaddr[0] << 8 |
 			     adapter->currmacaddr[1]) & 0xFFFF);
 
-	slic_reg32_write(&slic_regs->slic_wraddrah, value2, FLUSH);
-	slic_reg32_write(&slic_regs->slic_wraddrbh, value2, FLUSH);
+	slic_write32(adapter, SLIC_REG_WRADDRAH, value2);
+	slic_write32(adapter, SLIC_REG_WRADDRBH, value2);
 
 	/*
 	 * Write our multicast mask out to the card.  This is done
@@ -626,7 +597,6 @@ static void slic_mac_address_config(struct adapter *adapter)
 static void slic_mac_config(struct adapter *adapter)
 {
 	u32 value;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
 
 	/* Setup GMAC gaps */
 	if (adapter->linkspeed == LINK_1000MB) {
@@ -650,7 +620,7 @@ static void slic_mac_config(struct adapter *adapter)
 	}
 
 	/* write mac config */
-	slic_reg32_write(&slic_regs->slic_wmcfg, value, FLUSH);
+	slic_write32(adapter, SLIC_REG_WMCFG, value);
 
 	/* setup mac addresses */
 	slic_mac_address_config(adapter);
@@ -660,7 +630,6 @@ static void slic_config_set(struct adapter *adapter, bool linkchange)
 {
 	u32 value;
 	u32 RcrReset;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
 
 	if (linkchange) {
 		/* Setup MAC */
@@ -677,7 +646,7 @@ static void slic_config_set(struct adapter *adapter, bool linkchange)
 			 GXCR_XMTEN |	/* Enable transmit  */
 			 GXCR_PAUSEEN);	/* Enable pause     */
 
-		slic_reg32_write(&slic_regs->slic_wxcfg, value, FLUSH);
+		slic_write32(adapter, SLIC_REG_WXCFG, value);
 
 		/* Setup rcvcfg last */
 		value = (RcrReset |	/* Reset, if linkchange */
@@ -690,7 +659,7 @@ static void slic_config_set(struct adapter *adapter, bool linkchange)
 		value = (GXCR_RESET |	/* Always reset     */
 			 GXCR_XMTEN);	/* Enable transmit  */
 
-		slic_reg32_write(&slic_regs->slic_wxcfg, value, FLUSH);
+		slic_write32(adapter, SLIC_REG_WXCFG, value);
 
 		/* Setup rcvcfg last */
 		value = (RcrReset |	/* Reset, if linkchange */
@@ -707,7 +676,7 @@ static void slic_config_set(struct adapter *adapter, bool linkchange)
 	if (adapter->macopts & MAC_PROMISC)
 		value |= GRCR_RCVALL;
 
-	slic_reg32_write(&slic_regs->slic_wrcfg, value, FLUSH);
+	slic_write32(adapter, SLIC_REG_WRCFG, value);
 }
 
 /*
@@ -717,24 +686,23 @@ static void slic_config_clear(struct adapter *adapter)
 {
 	u32 value;
 	u32 phy_config;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
 
 	/* Setup xmtcfg */
 	value = (GXCR_RESET |	/* Always reset */
 		 GXCR_PAUSEEN);	/* Enable pause */
 
-	slic_reg32_write(&slic_regs->slic_wxcfg, value, FLUSH);
+	slic_write32(adapter, SLIC_REG_WXCFG, value);
 
 	value = (GRCR_RESET |	/* Always reset      */
 		 GRCR_CTLEN |	/* Enable CTL frames */
 		 GRCR_ADDRAEN |	/* Address A enable  */
 		 (GRCR_HASHSIZE << GRCR_HASHSIZE_SHIFT));
 
-	slic_reg32_write(&slic_regs->slic_wrcfg, value, FLUSH);
+	slic_write32(adapter, SLIC_REG_WRCFG, value);
 
 	/* power down phy */
 	phy_config = (MIICR_REG_PCR | (PCR_POWERDOWN));
-	slic_reg32_write(&slic_regs->slic_wphy, phy_config, FLUSH);
+	slic_write32(adapter, SLIC_REG_WPHY, phy_config);
 }
 
 static bool slic_mac_filter(struct adapter *adapter,
@@ -810,13 +778,11 @@ static void slic_timer_load_check(ulong cardaddr)
 {
 	struct sliccard *card = (struct sliccard *)cardaddr;
 	struct adapter *adapter = card->master;
-	u32 __iomem *intagg;
 	u32 load = card->events;
 	u32 level = 0;
 
 	if ((adapter) && (adapter->state == ADAPT_UP) &&
 	    (card->state == CARD_UP) && (slic_global.dynamic_intagg)) {
-		intagg = &adapter->slic_regs->slic_intagg;
 		if (adapter->devid == SLIC_1GB_DEVICE_ID) {
 			if (adapter->linkspeed == LINK_1000MB)
 				level = 100;
@@ -836,7 +802,7 @@ static void slic_timer_load_check(ulong cardaddr)
 			}
 			if (card->loadlevel_current != level) {
 				card->loadlevel_current = level;
-				slic_reg32_write(intagg, level, FLUSH);
+				slic_write32(adapter, SLIC_REG_INTAGG, level);
 			}
 		} else {
 			if (load > SLIC_LOAD_5)
@@ -853,7 +819,7 @@ static void slic_timer_load_check(ulong cardaddr)
 				level = SLIC_INTAGG_0;
 			if (card->loadlevel_current != level) {
 				card->loadlevel_current = level;
-				slic_reg32_write(intagg, level, FLUSH);
+				slic_write32(adapter, SLIC_REG_INTAGG, level);
 			}
 		}
 	}
@@ -897,7 +863,6 @@ static int slic_upr_queue_request(struct adapter *adapter,
 static void slic_upr_start(struct adapter *adapter)
 {
 	struct slic_upr *upr;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
 
 	upr = adapter->upr_list;
 	if (!upr)
@@ -909,31 +874,27 @@ static void slic_upr_start(struct adapter *adapter)
 	switch (upr->upr_request) {
 	case SLIC_UPR_STATS:
 		if (upr->upr_data_h == 0) {
-			slic_reg32_write(&slic_regs->slic_stats, upr->upr_data,
-					 FLUSH);
+			slic_write32(adapter, SLIC_REG_RSTAT, upr->upr_data);
 		} else {
-			slic_reg64_write(adapter, &slic_regs->slic_stats64,
-					 upr->upr_data,
-					 &slic_regs->slic_addr_upper,
-					 upr->upr_data_h, FLUSH);
+			slic_write64(adapter, SLIC_REG_RSTAT64, upr->upr_data,
+				     upr->upr_data_h);
 		}
 		break;
 
 	case SLIC_UPR_RLSR:
-		slic_reg64_write(adapter, &slic_regs->slic_rlsr, upr->upr_data,
-				 &slic_regs->slic_addr_upper, upr->upr_data_h,
-				 FLUSH);
+		slic_write64(adapter, SLIC_REG_LSTAT, upr->upr_data,
+			     upr->upr_data_h);
 		break;
 
 	case SLIC_UPR_RCONFIG:
-		slic_reg64_write(adapter, &slic_regs->slic_rconfig,
-				 upr->upr_data, &slic_regs->slic_addr_upper,
-				 upr->upr_data_h, FLUSH);
+		slic_write64(adapter, SLIC_REG_RCONFIG, upr->upr_data,
+			     upr->upr_data_h);
 		break;
 	case SLIC_UPR_PING:
-		slic_reg32_write(&slic_regs->slic_ping, 1, FLUSH);
+		slic_write32(adapter, SLIC_REG_PING, 1);
 		break;
 	}
+	slic_flush_write(adapter);
 }
 
 static int slic_upr_request(struct adapter *adapter,
@@ -961,42 +922,34 @@ err_unlock_irq:
 
 static void slic_link_upr_complete(struct adapter *adapter, u32 isr)
 {
-	u32 linkstatus = adapter->pshmem->linkstatus;
+	struct slic_shmemory *sm = &adapter->shmem;
+	struct slic_shmem_data *sm_data = sm->shmem_data;
+	u32 lst = sm_data->lnkstatus;
 	uint linkup;
 	unsigned char linkspeed;
 	unsigned char linkduplex;
 
 	if ((isr & ISR_UPCERR) || (isr & ISR_UPCBSY)) {
-		struct slic_shmem *pshmem;
+		dma_addr_t phaddr = sm->lnkstatus_phaddr;
 
-		pshmem = (struct slic_shmem *)(unsigned long)
-			 adapter->phys_shmem;
-#if BITS_PER_LONG == 64
-		slic_upr_queue_request(adapter,
-				       SLIC_UPR_RLSR,
-				       SLIC_GET_ADDR_LOW(&pshmem->linkstatus),
-				       SLIC_GET_ADDR_HIGH(&pshmem->linkstatus),
+		slic_upr_queue_request(adapter, SLIC_UPR_RLSR,
+				       cpu_to_le32(lower_32_bits(phaddr)),
+				       cpu_to_le32(upper_32_bits(phaddr)),
 				       0, 0);
-#else
-		slic_upr_queue_request(adapter,
-				       SLIC_UPR_RLSR,
-				       (u32)&pshmem->linkstatus,
-				       SLIC_GET_ADDR_HIGH(pshmem), 0, 0);
-#endif
 		return;
 	}
 	if (adapter->state != ADAPT_UP)
 		return;
 
-	linkup = linkstatus & GIG_LINKUP ? LINK_UP : LINK_DOWN;
-	if (linkstatus & GIG_SPEED_1000)
+	linkup = lst & GIG_LINKUP ? LINK_UP : LINK_DOWN;
+	if (lst & GIG_SPEED_1000)
 		linkspeed = LINK_1000MB;
-	else if (linkstatus & GIG_SPEED_100)
+	else if (lst & GIG_SPEED_100)
 		linkspeed = LINK_100MB;
 	else
 		linkspeed = LINK_10MB;
 
-	if (linkstatus & GIG_FULLDUPLEX)
+	if (lst & GIG_FULLDUPLEX)
 		linkduplex = LINK_FULLD;
 	else
 		linkduplex = LINK_HALFD;
@@ -1016,6 +969,7 @@ static void slic_link_upr_complete(struct adapter *adapter, u32 isr)
 	/* link has gone from up to down */
 	if (linkup == LINK_DOWN) {
 		adapter->linkstate = LINK_DOWN;
+		netif_carrier_off(adapter->netdev);
 		return;
 	}
 
@@ -1027,7 +981,7 @@ static void slic_link_upr_complete(struct adapter *adapter, u32 isr)
 		/* setup the mac */
 		slic_config_set(adapter, true);
 		adapter->linkstate = LINK_UP;
-		netif_start_queue(adapter->netdev);
+		netif_carrier_on(adapter->netdev);
 	}
 }
 
@@ -1047,81 +1001,65 @@ static void slic_upr_request_complete(struct adapter *adapter, u32 isr)
 	upr->next = NULL;
 	adapter->upr_busy = 0;
 	switch (upr->upr_request) {
-	case SLIC_UPR_STATS:
-		{
-			struct slic_stats *slicstats =
-			    (struct slic_stats *)&adapter->pshmem->inicstats;
-			struct slic_stats *newstats = slicstats;
-			struct slic_stats  *old = &adapter->inicstats_prev;
-			struct slicnet_stats *stst = &adapter->slic_stats;
-
-			if (isr & ISR_UPCERR) {
-				dev_err(&adapter->netdev->dev,
-					"SLIC_UPR_STATS command failed isr[%x]\n",
-					isr);
-
-				break;
-			}
-			UPDATE_STATS_GB(stst->tcp.xmit_tcp_segs,
-					newstats->xmit_tcp_segs_gb,
-					old->xmit_tcp_segs_gb);
-
-			UPDATE_STATS_GB(stst->tcp.xmit_tcp_bytes,
-					newstats->xmit_tcp_bytes_gb,
-					old->xmit_tcp_bytes_gb);
-
-			UPDATE_STATS_GB(stst->tcp.rcv_tcp_segs,
-					newstats->rcv_tcp_segs_gb,
-					old->rcv_tcp_segs_gb);
-
-			UPDATE_STATS_GB(stst->tcp.rcv_tcp_bytes,
-					newstats->rcv_tcp_bytes_gb,
-					old->rcv_tcp_bytes_gb);
-
-			UPDATE_STATS_GB(stst->iface.xmt_bytes,
-					newstats->xmit_bytes_gb,
-					old->xmit_bytes_gb);
-
-			UPDATE_STATS_GB(stst->iface.xmt_ucast,
-					newstats->xmit_unicasts_gb,
-					old->xmit_unicasts_gb);
-
-			UPDATE_STATS_GB(stst->iface.rcv_bytes,
-					newstats->rcv_bytes_gb,
-					old->rcv_bytes_gb);
-
-			UPDATE_STATS_GB(stst->iface.rcv_ucast,
-					newstats->rcv_unicasts_gb,
-					old->rcv_unicasts_gb);
-
-			UPDATE_STATS_GB(stst->iface.xmt_errors,
-					newstats->xmit_collisions_gb,
-					old->xmit_collisions_gb);
-
-			UPDATE_STATS_GB(stst->iface.xmt_errors,
-					newstats->xmit_excess_collisions_gb,
-					old->xmit_excess_collisions_gb);
-
-			UPDATE_STATS_GB(stst->iface.xmt_errors,
-					newstats->xmit_other_error_gb,
-					old->xmit_other_error_gb);
-
-			UPDATE_STATS_GB(stst->iface.rcv_errors,
-					newstats->rcv_other_error_gb,
-					old->rcv_other_error_gb);
-
-			UPDATE_STATS_GB(stst->iface.rcv_discards,
-					newstats->rcv_drops_gb,
-					old->rcv_drops_gb);
-
-			if (newstats->rcv_drops_gb > old->rcv_drops_gb) {
-				adapter->rcv_drops +=
-				    (newstats->rcv_drops_gb -
-				     old->rcv_drops_gb);
-			}
-			memcpy(old, newstats, sizeof(struct slic_stats));
+	case SLIC_UPR_STATS: {
+		struct slic_shmemory *sm = &adapter->shmem;
+		struct slic_shmem_data *sm_data = sm->shmem_data;
+		struct slic_stats *stats = &sm_data->stats;
+		struct slic_stats *old = &adapter->inicstats_prev;
+		struct slicnet_stats *stst = &adapter->slic_stats;
+
+		if (isr & ISR_UPCERR) {
+			dev_err(&adapter->netdev->dev,
+				"SLIC_UPR_STATS command failed isr[%x]\n", isr);
 			break;
 		}
+
+		UPDATE_STATS_GB(stst->tcp.xmit_tcp_segs, stats->xmit_tcp_segs,
+				old->xmit_tcp_segs);
+
+		UPDATE_STATS_GB(stst->tcp.xmit_tcp_bytes, stats->xmit_tcp_bytes,
+				old->xmit_tcp_bytes);
+
+		UPDATE_STATS_GB(stst->tcp.rcv_tcp_segs, stats->rcv_tcp_segs,
+				old->rcv_tcp_segs);
+
+		UPDATE_STATS_GB(stst->tcp.rcv_tcp_bytes, stats->rcv_tcp_bytes,
+				old->rcv_tcp_bytes);
+
+		UPDATE_STATS_GB(stst->iface.xmt_bytes, stats->xmit_bytes,
+				old->xmit_bytes);
+
+		UPDATE_STATS_GB(stst->iface.xmt_ucast, stats->xmit_unicasts,
+				old->xmit_unicasts);
+
+		UPDATE_STATS_GB(stst->iface.rcv_bytes, stats->rcv_bytes,
+				old->rcv_bytes);
+
+		UPDATE_STATS_GB(stst->iface.rcv_ucast, stats->rcv_unicasts,
+				old->rcv_unicasts);
+
+		UPDATE_STATS_GB(stst->iface.xmt_errors, stats->xmit_collisions,
+				old->xmit_collisions);
+
+		UPDATE_STATS_GB(stst->iface.xmt_errors,
+				stats->xmit_excess_collisions,
+				old->xmit_excess_collisions);
+
+		UPDATE_STATS_GB(stst->iface.xmt_errors, stats->xmit_other_error,
+				old->xmit_other_error);
+
+		UPDATE_STATS_GB(stst->iface.rcv_errors, stats->rcv_other_error,
+				old->rcv_other_error);
+
+		UPDATE_STATS_GB(stst->iface.rcv_discards, stats->rcv_drops,
+				old->rcv_drops);
+
+		if (stats->rcv_drops > old->rcv_drops)
+			adapter->rcv_drops += (stats->rcv_drops -
+					       old->rcv_drops);
+		memcpy_fromio(old, stats, sizeof(*stats));
+		break;
+	}
 	case SLIC_UPR_RLSR:
 		slic_link_upr_complete(adapter, isr);
 		break;
@@ -1186,7 +1124,6 @@ static int slic_rspqueue_init(struct adapter *adapter)
 {
 	int i;
 	struct slic_rspqueue *rspq = &adapter->rspqueue;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
 	u32 paddrh = 0;
 
 	memset(rspq, 0, sizeof(struct slic_rspqueue));
@@ -1205,14 +1142,12 @@ static int slic_rspqueue_init(struct adapter *adapter)
 		}
 
 		if (paddrh == 0) {
-			slic_reg32_write(&slic_regs->slic_rbar,
-				(rspq->paddr[i] | SLIC_RSPQ_BUFSINPAGE),
-				DONT_FLUSH);
+			slic_write32(adapter, SLIC_REG_RBAR,
+				     rspq->paddr[i] | SLIC_RSPQ_BUFSINPAGE);
 		} else {
-			slic_reg64_write(adapter, &slic_regs->slic_rbar64,
-				(rspq->paddr[i] | SLIC_RSPQ_BUFSINPAGE),
-				&slic_regs->slic_addr_upper,
-				paddrh, DONT_FLUSH);
+			slic_write64(adapter, SLIC_REG_RBAR64,
+				     rspq->paddr[i] | SLIC_RSPQ_BUFSINPAGE,
+				     paddrh);
 		}
 	}
 	rspq->offset = 0;
@@ -1233,9 +1168,9 @@ static struct slic_rspbuf *slic_rspqueue_getnext(struct adapter *adapter)
 	if (++rspq->offset < SLIC_RSPQ_BUFSINPAGE) {
 		rspq->rspbuf++;
 	} else {
-		slic_reg64_write(adapter, &adapter->slic_regs->slic_rbar64,
-			(rspq->paddr[rspq->pageindex] | SLIC_RSPQ_BUFSINPAGE),
-			&adapter->slic_regs->slic_addr_upper, 0, DONT_FLUSH);
+		slic_write64(adapter, SLIC_REG_RBAR64,
+			     rspq->paddr[rspq->pageindex] |
+			     SLIC_RSPQ_BUFSINPAGE, 0);
 		rspq->pageindex = (rspq->pageindex + 1) % rspq->num_pages;
 		rspq->offset = 0;
 		rspq->rspbuf = (struct slic_rspbuf *)
@@ -1569,14 +1504,11 @@ retry_rcvqfill:
 			}
 #endif
 			if (paddrh == 0) {
-				slic_reg32_write(&adapter->slic_regs->slic_hbar,
-						 (u32)paddrl, DONT_FLUSH);
+				slic_write32(adapter, SLIC_REG_HBAR,
+					     (u32)paddrl);
 			} else {
-				slic_reg64_write(adapter,
-					&adapter->slic_regs->slic_hbar64,
-					paddrl,
-					&adapter->slic_regs->slic_addr_upper,
-					paddrh, DONT_FLUSH);
+				slic_write64(adapter, SLIC_REG_HBAR64, paddrl,
+					     paddrh);
 			}
 			if (rcvq->head)
 				rcvq->tail->next = skb;
@@ -1698,14 +1630,10 @@ static u32 slic_rcvqueue_reinsert(struct adapter *adapter, struct sk_buff *skb)
 		dev_err(dev, "         rcvq->tail[%p]\n", rcvq->tail);
 		dev_err(dev, "         rcvq->count[%x]\n", rcvq->count);
 	}
-	if (paddrh == 0) {
-		slic_reg32_write(&adapter->slic_regs->slic_hbar, (u32)paddrl,
-				 DONT_FLUSH);
-	} else {
-		slic_reg64_write(adapter, &adapter->slic_regs->slic_hbar64,
-				 paddrl, &adapter->slic_regs->slic_addr_upper,
-				 paddrh, DONT_FLUSH);
-	}
+	if (paddrh == 0)
+		slic_write32(adapter, SLIC_REG_HBAR, (u32)paddrl);
+	else
+		slic_write64(adapter, SLIC_REG_HBAR64, paddrl, paddrh);
 	if (rcvq->head)
 		rcvq->tail->next = skb;
 	else
@@ -1728,26 +1656,17 @@ static u32 slic_rcvqueue_reinsert(struct adapter *adapter, struct sk_buff *skb)
 static int slic_link_event_handler(struct adapter *adapter)
 {
 	int status;
-	struct slic_shmem *pshmem;
+	struct slic_shmemory *sm = &adapter->shmem;
+	dma_addr_t phaddr = sm->lnkstatus_phaddr;
 
 	if (adapter->state != ADAPT_UP) {
 		/* Adapter is not operational.  Ignore.  */
 		return -ENODEV;
 	}
-
-	pshmem = (struct slic_shmem *)(unsigned long)adapter->phys_shmem;
-
-#if BITS_PER_LONG == 64
-	status = slic_upr_request(adapter,
-				  SLIC_UPR_RLSR,
-				  SLIC_GET_ADDR_LOW(&pshmem->linkstatus),
-				  SLIC_GET_ADDR_HIGH(&pshmem->linkstatus),
-				  0, 0);
-#else
+	/* no 4GB wrap guaranteed */
 	status = slic_upr_request(adapter, SLIC_UPR_RLSR,
-		(u32)&pshmem->linkstatus,	/* no 4GB wrap guaranteed */
-				  0, 0, 0);
-#endif
+				  cpu_to_le32(lower_32_bits(phaddr)),
+				  cpu_to_le32(upper_32_bits(phaddr)), 0, 0);
 	return status;
 }
 
@@ -1757,12 +1676,13 @@ static void slic_init_cleanup(struct adapter *adapter)
 		adapter->intrregistered = 0;
 		free_irq(adapter->netdev->irq, adapter->netdev);
 	}
-	if (adapter->pshmem) {
-		pci_free_consistent(adapter->pcidev,
-				    sizeof(struct slic_shmem),
-				    adapter->pshmem, adapter->phys_shmem);
-		adapter->pshmem = NULL;
-		adapter->phys_shmem = (dma_addr_t)(unsigned long)NULL;
+
+	if (adapter->shmem.shmem_data) {
+		struct slic_shmemory *sm = &adapter->shmem;
+		struct slic_shmem_data *sm_data = sm->shmem_data;
+
+		pci_free_consistent(adapter->pcidev, sizeof(*sm_data), sm_data,
+				    sm->isr_phaddr);
 	}
 
 	if (adapter->pingtimerset) {
@@ -2147,13 +2067,16 @@ static irqreturn_t slic_interrupt(int irq, void *dev_id)
 {
 	struct net_device *dev = dev_id;
 	struct adapter *adapter = netdev_priv(dev);
+	struct slic_shmemory *sm = &adapter->shmem;
+	struct slic_shmem_data *sm_data = sm->shmem_data;
 	u32 isr;
 
-	if ((adapter->pshmem) && (adapter->pshmem->isr)) {
-		slic_reg32_write(&adapter->slic_regs->slic_icr,
-				 ICR_INT_MASK, FLUSH);
-		isr = adapter->isrcopy = adapter->pshmem->isr;
-		adapter->pshmem->isr = 0;
+	if (sm_data->isr) {
+		slic_write32(adapter, SLIC_REG_ICR, ICR_INT_MASK);
+		slic_flush_write(adapter);
+
+		isr = sm_data->isr;
+		sm_data->isr = 0;
 		adapter->num_isrs++;
 		switch (adapter->card->state) {
 		case CARD_UP:
@@ -2169,10 +2092,9 @@ static irqreturn_t slic_interrupt(int irq, void *dev_id)
 			break;
 		}
 
-		adapter->isrcopy = 0;
 		adapter->all_reg_writes += 2;
 		adapter->isr_reg_writes++;
-		slic_reg32_write(&adapter->slic_regs->slic_isr, 0, FLUSH);
+		slic_write32(adapter, SLIC_REG_ISR, 0);
 	} else {
 		adapter->false_interrupts++;
 	}
@@ -2224,13 +2146,11 @@ static netdev_tx_t slic_xmit_start(struct sk_buff *skb, struct net_device *dev)
 	}
 #endif
 	if (hcmd->paddrh == 0) {
-		slic_reg32_write(&adapter->slic_regs->slic_cbar,
-				 (hcmd->paddrl | hcmd->cmdsize), DONT_FLUSH);
+		slic_write32(adapter, SLIC_REG_CBAR, (hcmd->paddrl |
+						      hcmd->cmdsize));
 	} else {
-		slic_reg64_write(adapter, &adapter->slic_regs->slic_cbar64,
-				 (hcmd->paddrl | hcmd->cmdsize),
-				 &adapter->slic_regs->slic_addr_upper,
-				 hcmd->paddrh, DONT_FLUSH);
+		slic_write64(adapter, SLIC_REG_CBAR64,
+			     hcmd->paddrl | hcmd->cmdsize, hcmd->paddrh);
 	}
 xmit_done:
 	return NETDEV_TX_OK;
@@ -2290,8 +2210,8 @@ static int slic_if_init(struct adapter *adapter, unsigned long *flags)
 {
 	struct sliccard *card = adapter->card;
 	struct net_device *dev = adapter->netdev;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
-	struct slic_shmem *pshmem;
+	struct slic_shmemory *sm = &adapter->shmem;
+	struct slic_shmem_data *sm_data = sm->shmem_data;
 	int rc;
 
 	/* adapter should be down at this point */
@@ -2335,28 +2255,20 @@ static int slic_if_init(struct adapter *adapter, unsigned long *flags)
 		adapter->queues_initialized = 1;
 	}
 
-	slic_reg32_write(&slic_regs->slic_icr, ICR_INT_OFF, FLUSH);
+	slic_write32(adapter, SLIC_REG_ICR, ICR_INT_OFF);
+	slic_flush_write(adapter);
 	mdelay(1);
 
 	if (!adapter->isp_initialized) {
 		unsigned long flags;
 
-		pshmem = (struct slic_shmem *)(unsigned long)
-			 adapter->phys_shmem;
-
 		spin_lock_irqsave(&adapter->bit64reglock, flags);
-
-#if BITS_PER_LONG == 64
-		slic_reg32_write(&slic_regs->slic_addr_upper,
-				 SLIC_GET_ADDR_HIGH(&pshmem->isr), DONT_FLUSH);
-		slic_reg32_write(&slic_regs->slic_isp,
-				 SLIC_GET_ADDR_LOW(&pshmem->isr), FLUSH);
-#else
-		slic_reg32_write(&slic_regs->slic_addr_upper, 0, DONT_FLUSH);
-		slic_reg32_write(&slic_regs->slic_isp, (u32)&pshmem->isr,
-				FLUSH);
-#endif
+		slic_write32(adapter, SLIC_REG_ADDR_UPPER,
+			     cpu_to_le32(upper_32_bits(sm->isr_phaddr)));
+		slic_write32(adapter, SLIC_REG_ISP,
+			     cpu_to_le32(lower_32_bits(sm->isr_phaddr)));
 		spin_unlock_irqrestore(&adapter->bit64reglock, flags);
+
 		adapter->isp_initialized = 1;
 	}
 
@@ -2383,17 +2295,20 @@ static int slic_if_init(struct adapter *adapter, unsigned long *flags)
 	/*
 	 *    clear any pending events, then enable interrupts
 	 */
-	adapter->isrcopy = 0;
-	adapter->pshmem->isr = 0;
-	slic_reg32_write(&slic_regs->slic_isr, 0, FLUSH);
-	slic_reg32_write(&slic_regs->slic_icr, ICR_INT_ON, FLUSH);
+	sm_data->isr = 0;
+	slic_write32(adapter, SLIC_REG_ISR, 0);
+	slic_write32(adapter, SLIC_REG_ICR, ICR_INT_ON);
 
 	slic_link_config(adapter, LINK_AUTOSPEED, LINK_AUTOD);
+	slic_flush_write(adapter);
+
 	rc = slic_link_event_handler(adapter);
 	if (rc) {
 		/* disable interrupts then clear pending events */
-		slic_reg32_write(&slic_regs->slic_icr, ICR_INT_OFF, FLUSH);
-		slic_reg32_write(&slic_regs->slic_isr, 0, FLUSH);
+		slic_write32(adapter, SLIC_REG_ICR, ICR_INT_OFF);
+		slic_write32(adapter, SLIC_REG_ISR, 0);
+		slic_flush_write(adapter);
+
 		if (adapter->pingtimerset) {
 			del_timer(&adapter->pingtimer);
 			adapter->pingtimerset = 0;
@@ -2417,7 +2332,7 @@ static int slic_entry_open(struct net_device *dev)
 	unsigned long flags;
 	int status;
 
-	netif_stop_queue(adapter->netdev);
+	netif_carrier_off(dev);
 
 	spin_lock_irqsave(&slic_global.driver_lock, flags);
 	if (!adapter->activated) {
@@ -2440,6 +2355,9 @@ static int slic_entry_open(struct net_device *dev)
 
 spin_unlock:
 	spin_unlock_irqrestore(&slic_global.driver_lock, flags);
+
+	netif_start_queue(adapter->netdev);
+
 	return status;
 }
 
@@ -2463,7 +2381,7 @@ static void slic_entry_remove(struct pci_dev *pcidev)
 	unregister_netdev(dev);
 
 	slic_adapter_freeresources(adapter);
-	slic_unmap_mmio_space(adapter);
+	iounmap(adapter->regs);
 
 	/* free multicast addresses */
 	mlist = adapter->mcastaddrs;
@@ -2497,7 +2415,6 @@ static int slic_entry_halt(struct net_device *dev)
 {
 	struct adapter *adapter = netdev_priv(dev);
 	struct sliccard *card = adapter->card;
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
 	unsigned long flags;
 
 	spin_lock_irqsave(&slic_global.driver_lock, flags);
@@ -2507,7 +2424,7 @@ static int slic_entry_halt(struct net_device *dev)
 	adapter->upr_list = NULL;
 	adapter->upr_busy = 0;
 	adapter->devflags_prev = 0;
-	slic_reg32_write(&slic_regs->slic_icr, ICR_INT_OFF, FLUSH);
+	slic_write32(adapter, SLIC_REG_ICR, ICR_INT_OFF);
 	adapter->all_reg_writes++;
 	adapter->icr_reg_writes++;
 	slic_config_clear(adapter);
@@ -2517,8 +2434,10 @@ static int slic_entry_halt(struct net_device *dev)
 		adapter->activated = 0;
 	}
 #ifdef AUTOMATIC_RESET
-	slic_reg32_write(&slic_regs->slic_reset_iface, 0, FLUSH);
+	slic_write32(adapter, SLIC_REG_RESET_IFACE, 0);
 #endif
+	slic_flush_write(adapter);
+
 	/*
 	 *  Reset the adapter's cmd queues
 	 */
@@ -2530,6 +2449,9 @@ static int slic_entry_halt(struct net_device *dev)
 #endif
 
 	spin_unlock_irqrestore(&slic_global.driver_lock, flags);
+
+	netif_carrier_off(dev);
+
 	return 0;
 }
 
@@ -2661,14 +2583,14 @@ static void slic_config_pci(struct pci_dev *pcidev)
 
 static int slic_card_init(struct sliccard *card, struct adapter *adapter)
 {
-	__iomem struct slic_regs *slic_regs = adapter->slic_regs;
+	struct slic_shmemory *sm = &adapter->shmem;
+	struct slic_shmem_data *sm_data = sm->shmem_data;
 	struct slic_eeprom *peeprom;
 	struct oslic_eeprom *pOeeprom;
 	dma_addr_t phys_config;
 	u32 phys_configh;
 	u32 phys_configl;
 	u32 i = 0;
-	struct slic_shmem *pshmem;
 	int status;
 	uint macaddrs = card->card_size;
 	ushort eecodesize;
@@ -2695,27 +2617,26 @@ static int slic_card_init(struct sliccard *card, struct adapter *adapter)
 					       sizeof(struct slic_eeprom),
 					       &phys_config);
 
-		phys_configl = SLIC_GET_ADDR_LOW(phys_config);
-		phys_configh = SLIC_GET_ADDR_HIGH(phys_config);
-
 		if (!peeprom) {
 			dev_err(&adapter->pcidev->dev,
 				"Failed to allocate DMA memory for EEPROM.\n");
 			return -ENOMEM;
 		}
 
+		phys_configl = SLIC_GET_ADDR_LOW(phys_config);
+		phys_configh = SLIC_GET_ADDR_HIGH(phys_config);
+
 		memset(peeprom, 0, sizeof(struct slic_eeprom));
 
-		slic_reg32_write(&slic_regs->slic_icr, ICR_INT_OFF, FLUSH);
+		slic_write32(adapter, SLIC_REG_ICR, ICR_INT_OFF);
+		slic_flush_write(adapter);
 		mdelay(1);
-		pshmem = (struct slic_shmem *)(unsigned long)
-			 adapter->phys_shmem;
 
 		spin_lock_irqsave(&adapter->bit64reglock, flags);
-		slic_reg32_write(&slic_regs->slic_addr_upper,
-				 SLIC_GET_ADDR_HIGH(&pshmem->isr), DONT_FLUSH);
-		slic_reg32_write(&slic_regs->slic_isp,
-				 SLIC_GET_ADDR_LOW(&pshmem->isr), FLUSH);
+		slic_write32(adapter, SLIC_REG_ADDR_UPPER,
+			     cpu_to_le32(upper_32_bits(sm->isr_phaddr)));
+		slic_write32(adapter, SLIC_REG_ISP,
+			     cpu_to_le32(lower_32_bits(sm->isr_phaddr)));
 		spin_unlock_irqrestore(&adapter->bit64reglock, flags);
 
 		status = slic_config_get(adapter, phys_configl, phys_configh);
@@ -2726,33 +2647,31 @@ static int slic_card_init(struct sliccard *card, struct adapter *adapter)
 		}
 
 		for (;;) {
-			if (adapter->pshmem->isr) {
-				if (adapter->pshmem->isr & ISR_UPC) {
-					adapter->pshmem->isr = 0;
-					slic_reg64_write(adapter,
-						&slic_regs->slic_isp, 0,
-						&slic_regs->slic_addr_upper,
-						0, FLUSH);
-					slic_reg32_write(&slic_regs->slic_isr,
-							 0, FLUSH);
+			if (sm_data->isr) {
+				if (sm_data->isr & ISR_UPC) {
+					sm_data->isr = 0;
+					slic_write64(adapter, SLIC_REG_ISP, 0,
+						     0);
+					slic_write32(adapter, SLIC_REG_ISR, 0);
+					slic_flush_write(adapter);
 
 					slic_upr_request_complete(adapter, 0);
 					break;
 				}
 
-				adapter->pshmem->isr = 0;
-				slic_reg32_write(&slic_regs->slic_isr,
-						 0, FLUSH);
+				sm_data->isr = 0;
+				slic_write32(adapter, SLIC_REG_ISR, 0);
+				slic_flush_write(adapter);
 			} else {
 				mdelay(1);
 				i++;
 				if (i > 5000) {
 					dev_err(&adapter->pcidev->dev,
 						"Fetch of config data timed out.\n");
-					slic_reg64_write(adapter,
-						&slic_regs->slic_isp, 0,
-						&slic_regs->slic_addr_upper,
-						0, FLUSH);
+					slic_write64(adapter, SLIC_REG_ISP,
+						     0, 0);
+					slic_flush_write(adapter);
+
 					status = -EINVAL;
 					goto card_init_err;
 				}
@@ -2796,7 +2715,6 @@ static int slic_card_init(struct sliccard *card, struct adapter *adapter)
 		/*  see if the EEPROM is valid by checking it's checksum */
 		if ((eecodesize <= MAX_EECODE_SIZE) &&
 		    (eecodesize >= MIN_EECODE_SIZE)) {
-
 			ee_chksum =
 			    *(u16 *)((char *)peeprom + (eecodesize - 2));
 			/*
@@ -2830,9 +2748,8 @@ static int slic_card_init(struct sliccard *card, struct adapter *adapter)
 				    peeprom, phys_config);
 
 		if (!card->config.EepromValid) {
-			slic_reg64_write(adapter, &slic_regs->slic_isp, 0,
-					 &slic_regs->slic_addr_upper,
-					 0, FLUSH);
+			slic_write64(adapter, SLIC_REG_ISP, 0, 0);
+			slic_flush_write(adapter);
 			dev_err(&adapter->pcidev->dev, "EEPROM invalid.\n");
 			return -EINVAL;
 		}
@@ -2896,14 +2813,17 @@ static void slic_init_driver(void)
 	}
 }
 
-static void slic_init_adapter(struct net_device *netdev,
-			      struct pci_dev *pcidev,
-			      const struct pci_device_id *pci_tbl_entry,
-			      void __iomem *memaddr, int chip_idx)
+static int slic_init_adapter(struct net_device *netdev,
+			     struct pci_dev *pcidev,
+			     const struct pci_device_id *pci_tbl_entry,
+			     void __iomem *memaddr, int chip_idx)
 {
 	ushort index;
 	struct slic_handle *pslic_handle;
 	struct adapter *adapter = netdev_priv(netdev);
+	struct slic_shmemory *sm = &adapter->shmem;
+	struct slic_shmem_data *sm_data;
+	dma_addr_t phaddr;
 
 /*	adapter->pcidev = pcidev;*/
 	adapter->vendid = pci_tbl_entry->vendor;
@@ -2912,7 +2832,7 @@ static void slic_init_adapter(struct net_device *netdev,
 	adapter->busnumber = pcidev->bus->number;
 	adapter->slotnumber = ((pcidev->devfn >> 3) & 0x1F);
 	adapter->functionnumber = (pcidev->devfn & 0x7);
-	adapter->slic_regs = memaddr;
+	adapter->regs = memaddr;
 	adapter->irq = pcidev->irq;
 	adapter->chipid = chip_idx;
 	adapter->port = 0;
@@ -2932,19 +2852,23 @@ static void slic_init_adapter(struct net_device *netdev,
 	 */
 	for (index = 1, pslic_handle = &adapter->slic_handles[1];
 	     index < SLIC_CMDQ_MAXCMDS; index++, pslic_handle++) {
-
 		pslic_handle->token.handle_index = index;
 		pslic_handle->type = SLIC_HANDLE_FREE;
 		pslic_handle->next = adapter->pfree_slic_handles;
 		adapter->pfree_slic_handles = pslic_handle;
 	}
-	adapter->pshmem = (struct slic_shmem *)
-					pci_alloc_consistent(adapter->pcidev,
-					sizeof(struct slic_shmem),
-					&adapter->
-					phys_shmem);
-	if (adapter->pshmem)
-		memset(adapter->pshmem, 0, sizeof(struct slic_shmem));
+	sm_data = pci_zalloc_consistent(adapter->pcidev, sizeof(*sm_data),
+					&phaddr);
+	if (!sm_data)
+		return -ENOMEM;
+
+	sm->shmem_data = sm_data;
+	sm->isr_phaddr = phaddr;
+	sm->lnkstatus_phaddr = phaddr + offsetof(struct slic_shmem_data,
+						 lnkstatus);
+	sm->stats_phaddr = phaddr + offsetof(struct slic_shmem_data, stats);
+
+	return 0;
 }
 
 static const struct net_device_ops slic_netdev_ops = {
@@ -2964,27 +2888,9 @@ static u32 slic_card_locate(struct adapter *adapter)
 	struct sliccard *card = slic_global.slic_card;
 	struct physcard *physcard = slic_global.phys_card;
 	ushort card_hostid;
-	u16 __iomem *hostid_reg;
 	uint i;
-	uint rdhostid_offset = 0;
-
-	switch (adapter->devid) {
-	case SLIC_2GB_DEVICE_ID:
-		rdhostid_offset = SLIC_RDHOSTID_2GB;
-		break;
-	case SLIC_1GB_DEVICE_ID:
-		rdhostid_offset = SLIC_RDHOSTID_1GB;
-		break;
-	default:
-		return -ENODEV;
-	}
 
-	hostid_reg =
-	    (u16 __iomem *)(((u8 __iomem *)(adapter->slic_regs)) +
-	    rdhostid_offset);
-
-	/* read the 16 bit hostid from SRAM */
-	card_hostid = (ushort)readw(hostid_reg);
+	card_hostid = slic_read32(adapter, SLIC_REG_HOSTID);
 
 	/* Initialize a new card structure if need be */
 	if (card_hostid == SLIC_HOSTID_DEFAULT) {
@@ -3130,7 +3036,7 @@ static int slic_entry_probe(struct pci_dev *pcidev,
 	mmio_start = pci_resource_start(pcidev, 0);
 	mmio_len = pci_resource_len(pcidev, 0);
 
-	memmapped_ioaddr = ioremap(mmio_start, mmio_len);
+	memmapped_ioaddr = ioremap_nocache(mmio_start, mmio_len);
 	if (!memmapped_ioaddr) {
 		dev_err(&pcidev->dev, "cannot remap MMIO region %lx @ %lx\n",
 			mmio_len, mmio_start);
@@ -3142,13 +3048,17 @@ static int slic_entry_probe(struct pci_dev *pcidev,
 
 	slic_init_driver();
 
-	slic_init_adapter(netdev,
-			  pcidev, pci_tbl_entry, memmapped_ioaddr, cards_found);
+	err = slic_init_adapter(netdev, pcidev, pci_tbl_entry, memmapped_ioaddr,
+				cards_found);
+	if (err) {
+		dev_err(&pcidev->dev, "failed to init adapter: %i\n", err);
+		goto err_out_unmap;
+	}
 
 	err = slic_card_locate(adapter);
 	if (err) {
 		dev_err(&pcidev->dev, "cannot locate card\n");
-		goto err_out_unmap;
+		goto err_clean_init;
 	}
 
 	card = adapter->card;
@@ -3160,7 +3070,7 @@ static int slic_entry_probe(struct pci_dev *pcidev,
 
 	err = slic_card_init(card, adapter);
 	if (err)
-		goto err_out_unmap;
+		goto err_clean_init;
 
 	slic_adapter_set_hwaddr(adapter);
 
@@ -3168,17 +3078,21 @@ static int slic_entry_probe(struct pci_dev *pcidev,
 	netdev->irq = adapter->irq;
 	netdev->netdev_ops = &slic_netdev_ops;
 
+	netif_carrier_off(netdev);
+
 	strcpy(netdev->name, "eth%d");
 	err = register_netdev(netdev);
 	if (err) {
 		dev_err(&pcidev->dev, "Cannot register net device, aborting.\n");
-		goto err_out_unmap;
+		goto err_clean_init;
 	}
 
 	cards_found++;
 
 	return 0;
 
+err_clean_init:
+	slic_init_cleanup(adapter);
 err_out_unmap:
 	iounmap(memmapped_ioaddr);
 err_out_free_netdev:
@@ -3209,7 +3123,7 @@ static void __exit slic_module_cleanup(void)
 	pci_unregister_driver(&slic_driver);
 }
 
-static struct ethtool_ops slic_ethtool_ops = {
+static const struct ethtool_ops slic_ethtool_ops = {
 	.get_coalesce = slic_get_coalesce,
 	.set_coalesce = slic_set_coalesce
 };
diff --git a/drivers/staging/sm750fb/ddk750_chip.c b/drivers/staging/sm750fb/ddk750_chip.c
index f80ee776677f..839d6730bde9 100644
--- a/drivers/staging/sm750fb/ddk750_chip.c
+++ b/drivers/staging/sm750fb/ddk750_chip.c
@@ -6,11 +6,9 @@
 #include "ddk750_chip.h"
 #include "ddk750_power.h"
 
-/* n / d + 1 / 2 = (2n + d) / 2d */
-#define roundedDiv(num, denom)	((2 * (num) + (denom)) / (2 * (denom)))
 #define MHz(x) ((x) * 1000000)
 
-logical_chip_type_t getChipType(void)
+logical_chip_type_t sm750_get_chip_type(void)
 {
 	unsigned short physicalID;
 	char physicalRev;
@@ -37,7 +35,7 @@ static unsigned int get_mxclk_freq(void)
 	unsigned int pll_reg;
 	unsigned int M, N, OD, POD;
 
-	if (getChipType() == SM750LE)
+	if (sm750_get_chip_type() == SM750LE)
 		return MHz(130);
 
 	pll_reg = PEEK32(MXCLK_PLL_CTRL);
@@ -60,7 +58,7 @@ static void setChipClock(unsigned int frequency)
 	unsigned int ulActualMxClk;
 
 	/* Cheok_0509: For SM750LE, the chip clock is fixed. Nothing to set. */
-	if (getChipType() == SM750LE)
+	if (sm750_get_chip_type() == SM750LE)
 		return;
 
 	if (frequency) {
@@ -71,9 +69,10 @@ static void setChipClock(unsigned int frequency)
 		pll.clockType = MXCLK_PLL;
 
 		/*
-		* Call calcPllValue() to fill up the other fields for PLL structure.
-		* Sometime, the chip cannot set up the exact clock required by User.
-		* Return value from calcPllValue() gives the actual possible clock.
+		* Call calcPllValue() to fill the other fields of PLL structure.
+		* Sometime, the chip cannot set up the exact clock
+		* required by the User.
+		* Return value of calcPllValue gives the actual possible clock.
 		*/
 		ulActualMxClk = calcPllValue(frequency, &pll);
 
@@ -86,18 +85,22 @@ static void setMemoryClock(unsigned int frequency)
 {
 	unsigned int reg, divisor;
 
-	/* Cheok_0509: For SM750LE, the memory clock is fixed. Nothing to set. */
-	if (getChipType() == SM750LE)
+	/* Cheok_0509: For SM750LE, the memory clock is fixed.
+	 * Nothing to set.
+	 */
+	if (sm750_get_chip_type() == SM750LE)
 		return;
 
 	if (frequency) {
-		/* Set the frequency to the maximum frequency that the DDR Memory can take
-		which is 336MHz. */
+		/*
+		 * Set the frequency to the maximum frequency
+		 * that the DDR Memory can take which is 336MHz.
+		 */
 		if (frequency > MHz(336))
 			frequency = MHz(336);
 
 		/* Calculate the divisor */
-		divisor = roundedDiv(get_mxclk_freq(), frequency);
+		divisor = DIV_ROUND_CLOSEST(get_mxclk_freq(), frequency);
 
 		/* Set the corresponding divisor in the register. */
 		reg = PEEK32(CURRENT_GATE) & ~CURRENT_GATE_M2XCLK_MASK;
@@ -133,18 +136,21 @@ static void setMasterClock(unsigned int frequency)
 {
 	unsigned int reg, divisor;
 
-	/* Cheok_0509: For SM750LE, the memory clock is fixed. Nothing to set. */
-	if (getChipType() == SM750LE)
+	/* Cheok_0509: For SM750LE, the memory clock is fixed.
+	 * Nothing to set.
+	 */
+	if (sm750_get_chip_type() == SM750LE)
 		return;
 
 	if (frequency) {
-		/* Set the frequency to the maximum frequency that the SM750 engine can
-		run, which is about 190 MHz. */
+		/* Set the frequency to the maximum frequency
+		 * that the SM750 engine can run, which is about 190 MHz.
+		 */
 		if (frequency > MHz(190))
 			frequency = MHz(190);
 
 		/* Calculate the divisor */
-		divisor = roundedDiv(get_mxclk_freq(), frequency);
+		divisor = DIV_ROUND_CLOSEST(get_mxclk_freq(), frequency);
 
 		/* Set the corresponding divisor in the register. */
 		reg = PEEK32(CURRENT_GATE) & ~CURRENT_GATE_MCLK_MASK;
@@ -174,7 +180,7 @@ unsigned int ddk750_getVMSize(void)
 	unsigned int data;
 
 	/* sm750le only use 64 mb memory*/
-	if (getChipType() == SM750LE)
+	if (sm750_get_chip_type() == SM750LE)
 		return SZ_64M;
 
 	/* for 750,always use power mode0*/
@@ -213,7 +219,7 @@ int ddk750_initHw(initchip_param_t *pInitParam)
 	reg |= (CURRENT_GATE_DISPLAY | CURRENT_GATE_LOCALMEM);
 	setCurrentGate(reg);
 
-	if (getChipType() != SM750LE) {
+	if (sm750_get_chip_type() != SM750LE) {
 		/*	set panel pll and graphic mode via mmio_88 */
 		reg = PEEK32(VGA_CONFIGURATION);
 		reg |= (VGA_CONFIGURATION_PLL | VGA_CONFIGURATION_MODE);
@@ -236,9 +242,10 @@ int ddk750_initHw(initchip_param_t *pInitParam)
 	setMasterClock(MHz(pInitParam->masterClock));
 
 
-	/* Reset the memory controller. If the memory controller is not reset in SM750,
-	   the system might hang when sw accesses the memory.
-	   The memory should be resetted after changing the MXCLK.
+	/* Reset the memory controller.
+	 * If the memory controller is not reset in SM750,
+	 * the system might hang when sw accesses the memory.
+	 * The memory should be resetted after changing the MXCLK.
 	 */
 	if (pInitParam->resetMemory == 1) {
 		reg = PEEK32(MISC_CTRL);
@@ -282,24 +289,27 @@ int ddk750_initHw(initchip_param_t *pInitParam)
 }
 
 /*
-	monk liu @ 4/6/2011:
-		   re-write the calculatePLL function of ddk750.
-		   the original version function does not use some mathematics tricks and shortcut
-		   when it doing the calculation of the best N,M,D combination
-		   I think this version gives a little upgrade in speed
-
-	750 pll clock formular:
-	Request Clock = (Input Clock * M )/(N * X)
-
-	Input Clock = 14318181 hz
-	X = 2 power D
-	D ={0,1,2,3,4,5,6}
-	M = {1,...,255}
-	N = {2,...,15}
-*/
+ * monk liu @ 4/6/2011:
+ *	re-write the calculatePLL function of ddk750.
+ *	the original version function does not use
+ *	some mathematics tricks and shortcut
+ *	when it doing the calculation of the best N,M,D combination
+ *	I think this version gives a little upgrade in speed
+ *
+ * 750 pll clock formular:
+ * Request Clock = (Input Clock * M )/(N * X)
+ *
+ * Input Clock = 14318181 hz
+ * X = 2 power D
+ * D ={0,1,2,3,4,5,6}
+ * M = {1,...,255}
+ * N = {2,...,15}
+ */
 unsigned int calcPllValue(unsigned int request_orig, pll_value_t *pll)
 {
-	/* as sm750 register definition, N located in 2,15 and M located in 1,255	*/
+	/* as sm750 register definition,
+	 * N located in 2,15 and M located in 1,255
+	 */
 	int N, M, X, d;
 	int mini_diff;
 	unsigned int RN, quo, rem, fl_quo;
@@ -308,9 +318,11 @@ unsigned int calcPllValue(unsigned int request_orig, pll_value_t *pll)
 	const int max_OD = 3;
 	int max_d = 6;
 
-	if (getChipType() == SM750LE) {
-		/* SM750LE don't have prgrammable PLL and M/N values to work on.
-		Just return the requested clock. */
+	if (sm750_get_chip_type() == SM750LE) {
+		/* SM750LE don't have
+		 * programmable PLL and M/N values to work on.
+		 * Just return the requested clock.
+		 */
 		return request_orig;
 	}
 
@@ -319,19 +331,23 @@ unsigned int calcPllValue(unsigned int request_orig, pll_value_t *pll)
 	request = request_orig / 1000;
 	input = pll->inputFreq / 1000;
 
-	/* for MXCLK register , no POD provided, so need be treated differently	*/
+	/* for MXCLK register,
+	 * no POD provided, so need be treated differently
+	 */
 	if (pll->clockType == MXCLK_PLL)
 		max_d = 3;
 
 	for (N = 15; N > 1; N--) {
-		/* RN will not exceed maximum long if @request <= 285 MHZ (for 32bit cpu) */
+		/* RN will not exceed maximum long
+		 * if @request <= 285 MHZ (for 32bit cpu)
+		 */
 		RN = N * request;
 		quo = RN / input;
 		rem = RN % input;/* rem always small than 14318181 */
 		fl_quo = (rem * 10000 / input);
 
 		for (d = max_d; d >= 0; d--) {
-			X = (1 << d);
+			X = BIT(d);
 			M = quo * X;
 			M += fl_quo * X / 10000;
 			/* round step */
diff --git a/drivers/staging/sm750fb/ddk750_chip.h b/drivers/staging/sm750fb/ddk750_chip.h
index 0891384ef3e5..14357fd1cc6b 100644
--- a/drivers/staging/sm750fb/ddk750_chip.h
+++ b/drivers/staging/sm750fb/ddk750_chip.h
@@ -69,7 +69,7 @@ typedef struct _initchip_param_t {
 }
 initchip_param_t;
 
-logical_chip_type_t getChipType(void);
+logical_chip_type_t sm750_get_chip_type(void);
 unsigned int calcPllValue(unsigned int request, pll_value_t *pll);
 unsigned int formatPllReg(pll_value_t *pPLL);
 void ddk750_set_mmio(void __iomem *, unsigned short, char);
diff --git a/drivers/staging/sm750fb/ddk750_display.c b/drivers/staging/sm750fb/ddk750_display.c
index ca4973ee49e4..4023c476b9e4 100644
--- a/drivers/staging/sm750fb/ddk750_display.c
+++ b/drivers/staging/sm750fb/ddk750_display.c
@@ -68,8 +68,10 @@ static void waitNextVerticalSync(int ctrl, int delay)
 	if (!ctrl) {
 		/* primary controller */
 
-		/* Do not wait when the Primary PLL is off or display control is already off.
-		   This will prevent the software to wait forever. */
+		/*
+		 * Do not wait when the Primary PLL is off or display control is
+		 * already off. This will prevent the software to wait forever.
+		 */
 		if (!(PEEK32(PANEL_PLL_CTRL) & PLL_CTRL_POWER) ||
 		    !(PEEK32(PANEL_DISPLAY_CTRL) & DISPLAY_CTRL_TIMING)) {
 			return;
@@ -88,9 +90,10 @@ static void waitNextVerticalSync(int ctrl, int delay)
 		}
 
 	} else {
-
-		/* Do not wait when the Primary PLL is off or display control is already off.
-			   This will prevent the software to wait forever. */
+		/*
+		 * Do not wait when the Primary PLL is off or display control is
+		 * already off. This will prevent the software to wait forever.
+		 */
 		if (!(PEEK32(CRT_PLL_CTRL) & PLL_CTRL_POWER) ||
 		    !(PEEK32(CRT_DISPLAY_CTRL) & DISPLAY_CTRL_TIMING)) {
 			return;
@@ -134,7 +137,6 @@ static void swPanelPowerSequence(int disp, int delay)
 	reg |= (disp ? PANEL_DISPLAY_CTRL_FPEN : 0);
 	POKE32(PANEL_DISPLAY_CTRL, reg);
 	primaryWaitVerticalSync(delay);
-
 }
 
 void ddk750_setLogicalDispOut(disp_output_t output)
@@ -159,7 +161,6 @@ void ddk750_setLogicalDispOut(disp_output_t output)
 		/*se blank off */
 		reg &= ~CRT_DISPLAY_CTRL_BLANK;
 		POKE32(CRT_DISPLAY_CTRL, reg);
-
 	}
 
 	if (output & PRI_TP_USAGE) {
diff --git a/drivers/staging/sm750fb/ddk750_display.h b/drivers/staging/sm750fb/ddk750_display.h
index ca35aa1df9d7..e3fde428c52b 100644
--- a/drivers/staging/sm750fb/ddk750_display.h
+++ b/drivers/staging/sm750fb/ddk750_display.h
@@ -2,100 +2,98 @@
 #define DDK750_DISPLAY_H__
 
 /* panel path select
-	80000[29:28]
-*/
+ *	80000[29:28]
+ */
 
 #define PNL_2_OFFSET 0
 #define PNL_2_MASK (3 << PNL_2_OFFSET)
 #define PNL_2_USAGE	(PNL_2_MASK << 16)
-#define PNL_2_PRI	((0 << PNL_2_OFFSET)|PNL_2_USAGE)
-#define PNL_2_SEC	((2 << PNL_2_OFFSET)|PNL_2_USAGE)
+#define PNL_2_PRI	((0 << PNL_2_OFFSET) | PNL_2_USAGE)
+#define PNL_2_SEC	((2 << PNL_2_OFFSET) | PNL_2_USAGE)
 
 
 /* primary timing & plane enable bit
-	1: 80000[8] & 80000[2] on
-	0: both off
-*/
+ *	1: 80000[8] & 80000[2] on
+ *	0: both off
+ */
 #define PRI_TP_OFFSET 4
 #define PRI_TP_MASK BIT(PRI_TP_OFFSET)
 #define PRI_TP_USAGE (PRI_TP_MASK << 16)
-#define PRI_TP_ON ((0x1 << PRI_TP_OFFSET)|PRI_TP_USAGE)
-#define PRI_TP_OFF ((0x0 << PRI_TP_OFFSET)|PRI_TP_USAGE)
+#define PRI_TP_ON ((0x1 << PRI_TP_OFFSET) | PRI_TP_USAGE)
+#define PRI_TP_OFF ((0x0 << PRI_TP_OFFSET) | PRI_TP_USAGE)
 
 
 /* panel sequency status
-	80000[27:24]
-*/
+ *	80000[27:24]
+ */
 #define PNL_SEQ_OFFSET 6
 #define PNL_SEQ_MASK BIT(PNL_SEQ_OFFSET)
 #define PNL_SEQ_USAGE (PNL_SEQ_MASK << 16)
-#define PNL_SEQ_ON (BIT(PNL_SEQ_OFFSET)|PNL_SEQ_USAGE)
-#define PNL_SEQ_OFF ((0 << PNL_SEQ_OFFSET)|PNL_SEQ_USAGE)
+#define PNL_SEQ_ON (BIT(PNL_SEQ_OFFSET) | PNL_SEQ_USAGE)
+#define PNL_SEQ_OFF ((0 << PNL_SEQ_OFFSET) | PNL_SEQ_USAGE)
 
 /* dual digital output
-	80000[19]
-*/
+ *	80000[19]
+ */
 #define DUAL_TFT_OFFSET 8
 #define DUAL_TFT_MASK BIT(DUAL_TFT_OFFSET)
 #define DUAL_TFT_USAGE (DUAL_TFT_MASK << 16)
-#define DUAL_TFT_ON (BIT(DUAL_TFT_OFFSET)|DUAL_TFT_USAGE)
-#define DUAL_TFT_OFF ((0 << DUAL_TFT_OFFSET)|DUAL_TFT_USAGE)
+#define DUAL_TFT_ON (BIT(DUAL_TFT_OFFSET) | DUAL_TFT_USAGE)
+#define DUAL_TFT_OFF ((0 << DUAL_TFT_OFFSET) | DUAL_TFT_USAGE)
 
 /* secondary timing & plane enable bit
-	1:80200[8] & 80200[2] on
-	0: both off
-*/
+ *	1:80200[8] & 80200[2] on
+ *	0: both off
+ */
 #define SEC_TP_OFFSET 5
 #define SEC_TP_MASK BIT(SEC_TP_OFFSET)
 #define SEC_TP_USAGE (SEC_TP_MASK << 16)
-#define SEC_TP_ON  ((0x1 << SEC_TP_OFFSET)|SEC_TP_USAGE)
-#define SEC_TP_OFF ((0x0 << SEC_TP_OFFSET)|SEC_TP_USAGE)
+#define SEC_TP_ON  ((0x1 << SEC_TP_OFFSET) | SEC_TP_USAGE)
+#define SEC_TP_OFF ((0x0 << SEC_TP_OFFSET) | SEC_TP_USAGE)
 
 /* crt path select
-	80200[19:18]
-*/
+ *	80200[19:18]
+ */
 #define CRT_2_OFFSET 2
 #define CRT_2_MASK (3 << CRT_2_OFFSET)
 #define CRT_2_USAGE (CRT_2_MASK << 16)
-#define CRT_2_PRI ((0x0 << CRT_2_OFFSET)|CRT_2_USAGE)
-#define CRT_2_SEC ((0x2 << CRT_2_OFFSET)|CRT_2_USAGE)
+#define CRT_2_PRI ((0x0 << CRT_2_OFFSET) | CRT_2_USAGE)
+#define CRT_2_SEC ((0x2 << CRT_2_OFFSET) | CRT_2_USAGE)
 
 
 /* DAC affect both DVI and DSUB
-	4[20]
-*/
+ *	4[20]
+ */
 #define DAC_OFFSET 7
 #define DAC_MASK BIT(DAC_OFFSET)
 #define DAC_USAGE (DAC_MASK << 16)
-#define DAC_ON ((0x0 << DAC_OFFSET)|DAC_USAGE)
-#define DAC_OFF ((0x1 << DAC_OFFSET)|DAC_USAGE)
+#define DAC_ON ((0x0 << DAC_OFFSET) | DAC_USAGE)
+#define DAC_OFF ((0x1 << DAC_OFFSET) | DAC_USAGE)
 
 /* DPMS only affect D-SUB head
-	0[31:30]
-*/
+ *	0[31:30]
+ */
 #define DPMS_OFFSET 9
 #define DPMS_MASK (3 << DPMS_OFFSET)
 #define DPMS_USAGE (DPMS_MASK << 16)
-#define DPMS_OFF ((3 << DPMS_OFFSET)|DPMS_USAGE)
-#define DPMS_ON ((0 << DPMS_OFFSET)|DPMS_USAGE)
+#define DPMS_OFF ((3 << DPMS_OFFSET) | DPMS_USAGE)
+#define DPMS_ON ((0 << DPMS_OFFSET) | DPMS_USAGE)
 
 
 
-/*
-	LCD1 means panel path TFT1  & panel path DVI (so enable DAC)
-	CRT means crt path DSUB
-*/
+/* LCD1 means panel path TFT1  & panel path DVI (so enable DAC)
+ * CRT means crt path DSUB
+ */
 typedef enum _disp_output_t {
-	do_LCD1_PRI = PNL_2_PRI|PRI_TP_ON|PNL_SEQ_ON|DAC_ON,
-	do_LCD1_SEC = PNL_2_SEC|SEC_TP_ON|PNL_SEQ_ON|DAC_ON,
-	do_LCD2_PRI = CRT_2_PRI|PRI_TP_ON|DUAL_TFT_ON,
-	do_LCD2_SEC = CRT_2_SEC|SEC_TP_ON|DUAL_TFT_ON,
-	/*
-	do_DSUB_PRI = CRT_2_PRI|PRI_TP_ON|DPMS_ON|DAC_ON,
-	do_DSUB_SEC = CRT_2_SEC|SEC_TP_ON|DPMS_ON|DAC_ON,
-	*/
-	do_CRT_PRI = CRT_2_PRI|PRI_TP_ON|DPMS_ON|DAC_ON,
-	do_CRT_SEC = CRT_2_SEC|SEC_TP_ON|DPMS_ON|DAC_ON,
+	do_LCD1_PRI = PNL_2_PRI | PRI_TP_ON | PNL_SEQ_ON | DAC_ON,
+	do_LCD1_SEC = PNL_2_SEC | SEC_TP_ON | PNL_SEQ_ON | DAC_ON,
+	do_LCD2_PRI = CRT_2_PRI | PRI_TP_ON | DUAL_TFT_ON,
+	do_LCD2_SEC = CRT_2_SEC | SEC_TP_ON | DUAL_TFT_ON,
+	/* do_DSUB_PRI = CRT_2_PRI | PRI_TP_ON | DPMS_ON|DAC_ON,
+	 * do_DSUB_SEC = CRT_2_SEC | SEC_TP_ON | DPMS_ON|DAC_ON,
+	 */
+	do_CRT_PRI = CRT_2_PRI | PRI_TP_ON | DPMS_ON | DAC_ON,
+	do_CRT_SEC = CRT_2_SEC | SEC_TP_ON | DPMS_ON | DAC_ON,
 }
 disp_output_t;
 
diff --git a/drivers/staging/sm750fb/ddk750_dvi.c b/drivers/staging/sm750fb/ddk750_dvi.c
index a4a255007c8d..8252f771ef9e 100644
--- a/drivers/staging/sm750fb/ddk750_dvi.c
+++ b/drivers/staging/sm750fb/ddk750_dvi.c
@@ -6,9 +6,11 @@
 #include "ddk750_sii164.h"
 
 
-/* This global variable contains all the supported driver and its corresponding
-   function API. Please set the function pointer to NULL whenever the function
-   is not supported. */
+/*
+ * This global variable contains all the supported driver and its corresponding
+ * function API. Please set the function pointer to NULL whenever the function
+ * is not supported.
+ */
 static dvi_ctrl_device_t g_dcftSupportedDviController[] = {
 #ifdef DVI_CTRL_SII164
 	{
diff --git a/drivers/staging/sm750fb/ddk750_hwi2c.c b/drivers/staging/sm750fb/ddk750_hwi2c.c
index 39c3e1cdbc0c..d391c127ead7 100644
--- a/drivers/staging/sm750fb/ddk750_hwi2c.c
+++ b/drivers/staging/sm750fb/ddk750_hwi2c.c
@@ -21,7 +21,7 @@ unsigned char bus_speed_mode
 	POKE32(GPIO_MUX, value);
 
 	/* Enable Hardware I2C power.
-	 TODO: Check if we need to enable GPIO power?
+	 * TODO: Check if we need to enable GPIO power?
 	 */
 	enableI2C(1);
 
diff --git a/drivers/staging/sm750fb/ddk750_mode.c b/drivers/staging/sm750fb/ddk750_mode.c
index ccb4e067661a..05b83646c2d5 100644
--- a/drivers/staging/sm750fb/ddk750_mode.c
+++ b/drivers/staging/sm750fb/ddk750_mode.c
@@ -4,15 +4,14 @@
 #include "ddk750_mode.h"
 #include "ddk750_chip.h"
 
-/*
-	SM750LE only:
-    This function takes care extra registers and bit fields required to set
-    up a mode in SM750LE
-
-	Explanation about Display Control register:
-    HW only supports 7 predefined pixel clocks, and clock select is
-    in bit 29:27 of	Display Control register.
-*/
+/* SM750LE only:
+ * This function takes care extra registers and bit fields required to set
+ * up a mode in SM750LE
+ *
+ * Explanation about Display Control register:
+ * HW only supports 7 predefined pixel clocks, and clock select is
+ * in bit 29:27 of Display Control register.
+ */
 static unsigned long displayControlAdjust_SM750LE(mode_parameter_t *pModeParam, unsigned long dispControl)
 {
 	unsigned long x, y;
@@ -21,9 +20,9 @@ static unsigned long displayControlAdjust_SM750LE(mode_parameter_t *pModeParam,
 	y = pModeParam->vertical_display_end;
 
 	/* SM750LE has to set up the top-left and bottom-right
-	   registers as well.
-	   Note that normal SM750/SM718 only use those two register for
-	   auto-centering mode.
+	 * registers as well.
+	 * Note that normal SM750/SM718 only use those two register for
+	 * auto-centering mode.
 	 */
 	POKE32(CRT_AUTO_CENTERING_TL, 0);
 
@@ -33,8 +32,8 @@ static unsigned long displayControlAdjust_SM750LE(mode_parameter_t *pModeParam,
 		((x - 1) & CRT_AUTO_CENTERING_BR_RIGHT_MASK));
 
 	/* Assume common fields in dispControl have been properly set before
-	   calling this function.
-	   This function only sets the extra fields in dispControl.
+	 * calling this function.
+	 * This function only sets the extra fields in dispControl.
 	 */
 
 	/* Clear bit 29:27 of display control register */
@@ -63,7 +62,7 @@ static unsigned long displayControlAdjust_SM750LE(mode_parameter_t *pModeParam,
 	dispControl |= (CRT_DISPLAY_CTRL_CRTSELECT | CRT_DISPLAY_CTRL_RGBBIT);
 
 	/* Set bit 14 of display controller */
-	dispControl = DISPLAY_CTRL_CLOCK_PHASE;
+	dispControl |= DISPLAY_CTRL_CLOCK_PHASE;
 
 	POKE32(CRT_DISPLAY_CTRL, dispControl);
 
@@ -117,7 +116,7 @@ static int programModeRegisters(mode_parameter_t *pModeParam, pll_value_t *pll)
 		if (pModeParam->horizontal_sync_polarity)
 			tmp |= DISPLAY_CTRL_HSYNC_PHASE;
 
-		if (getChipType() == SM750LE) {
+		if (sm750_get_chip_type() == SM750LE) {
 			displayControlAdjust_SM750LE(pModeParam, tmp);
 		} else {
 			reg = PEEK32(CRT_DISPLAY_CTRL) &
@@ -209,7 +208,7 @@ int ddk750_setModeTiming(mode_parameter_t *parm, clock_type_t clock)
 	pll.clockType = clock;
 
 	uiActualPixelClk = calcPllValue(parm->pixel_clock, &pll);
-	if (getChipType() == SM750LE) {
+	if (sm750_get_chip_type() == SM750LE) {
 		/* set graphic mode via IO method */
 		outb_p(0x88, 0x3d4);
 		outb_p(0x06, 0x3d5);
diff --git a/drivers/staging/sm750fb/ddk750_power.c b/drivers/staging/sm750fb/ddk750_power.c
index b3c3791b95bd..7cc6169f884e 100644
--- a/drivers/staging/sm750fb/ddk750_power.c
+++ b/drivers/staging/sm750fb/ddk750_power.c
@@ -6,7 +6,7 @@ void ddk750_setDPMS(DPMS_t state)
 {
 	unsigned int value;
 
-	if (getChipType() == SM750LE) {
+	if (sm750_get_chip_type() == SM750LE) {
 		value = PEEK32(CRT_DISPLAY_CTRL) & ~CRT_DISPLAY_CTRL_DPMS_MASK;
 		value |= (state << CRT_DISPLAY_CTRL_DPMS_SHIFT);
 		POKE32(CRT_DISPLAY_CTRL, value);
@@ -19,7 +19,7 @@ void ddk750_setDPMS(DPMS_t state)
 
 static unsigned int getPowerMode(void)
 {
-	if (getChipType() == SM750LE)
+	if (sm750_get_chip_type() == SM750LE)
 		return 0;
 	return PEEK32(POWER_MODE_CTRL) & POWER_MODE_CTRL_MODE_MASK;
 }
@@ -35,7 +35,7 @@ void setPowerMode(unsigned int powerMode)
 
 	control_value = PEEK32(POWER_MODE_CTRL) & ~POWER_MODE_CTRL_MODE_MASK;
 
-	if (getChipType() == SM750LE)
+	if (sm750_get_chip_type() == SM750LE)
 		return;
 
 	switch (powerMode) {
diff --git a/drivers/staging/sm750fb/ddk750_sii164.c b/drivers/staging/sm750fb/ddk750_sii164.c
index 67f36e71da7e..99a8683e6383 100644
--- a/drivers/staging/sm750fb/ddk750_sii164.c
+++ b/drivers/staging/sm750fb/ddk750_sii164.c
@@ -174,8 +174,8 @@ long sii164InitChip(
 		i2cWriteReg(SII164_I2C_ADDRESS, SII164_CONFIGURATION, config);
 
 		/* De-skew enabled with default 111b value.
-		   This will fix some artifacts problem in some mode on board 2.2.
-		   Somehow this fix does not affect board 2.1.
+		 * This fixes some artifacts problem in some mode on board 2.2.
+		 * Somehow this fix does not affect board 2.1.
 		 */
 		if (deskewEnable == 0)
 			config = SII164_DESKEW_DISABLE;
@@ -344,7 +344,8 @@ void sii164EnableHotPlugDetection(
 	detectReg = i2cReadReg(SII164_I2C_ADDRESS, SII164_DETECT);
 
 	/* Depending on each DVI controller, need to enable the hot plug based on each
-	   individual chip design. */
+	 * individual chip design.
+	 */
 	if (enableHotPlug != 0)
 		sii164SelectHotPlugDetectionMode(SII164_HOTPLUG_USE_MDI);
 	else
diff --git a/drivers/staging/sm750fb/ddk750_swi2c.c b/drivers/staging/sm750fb/ddk750_swi2c.c
index 8d644a7cba52..72a42330e7a1 100644
--- a/drivers/staging/sm750fb/ddk750_swi2c.c
+++ b/drivers/staging/sm750fb/ddk750_swi2c.c
@@ -89,12 +89,12 @@ static void sw_i2c_wait(void)
 	 * always be non-zero,which makes the while loop
 	 * never finish.
 	 * use non-ultimate for loop below is safe
-	 * */
+	 */
 
     /* Change wait algorithm to use PCI bus clock,
-       it's more reliable than counter loop ..
-       write 0x61 to 0x3ce and read from 0x3cf
-       */
+     * it's more reliable than counter loop ..
+     * write 0x61 to 0x3ce and read from 0x3cf
+     */
 	int i, tmp;
 
 	for (i = 0; i < 600; i++) {
@@ -403,7 +403,7 @@ long sm750_sw_i2c_init(
 	if ((clk_gpio > 31) || (data_gpio > 31))
 		return -1;
 
-	if (getChipType() == SM750LE)
+	if (sm750_get_chip_type() == SM750LE)
 		return sm750le_i2c_init(clk_gpio, data_gpio);
 
 	/* Initialize the GPIO pin for the i2c Clock Register */
@@ -501,8 +501,8 @@ long sm750_sw_i2c_write_reg(
 	sw_i2c_start();
 
 	/* Send the device address and read the data. All should return success
-	   in order for the writing processed to be successful
-	*/
+	 * in order for the writing processed to be successful
+	 */
 	if ((sw_i2c_write_byte(addr) != 0) ||
 	    (sw_i2c_write_byte(reg) != 0) ||
 	    (sw_i2c_write_byte(data) != 0)) {
diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c
index 6ed004e40855..7d90e250142c 100644
--- a/drivers/staging/sm750fb/sm750.c
+++ b/drivers/staging/sm750fb/sm750.c
@@ -1176,7 +1176,7 @@ static int __init lynxfb_setup(char *options)
 		else {
 			strcat(tmp, opt);
 			tmp += strlen(opt);
-			if (options != NULL)
+			if (options)
 				*tmp++ = ':';
 			else
 				*tmp++ = 0;
diff --git a/drivers/staging/sm750fb/sm750.h b/drivers/staging/sm750fb/sm750.h
index 8e70ce0d6da4..ff31c5c9cc6f 100644
--- a/drivers/staging/sm750fb/sm750.h
+++ b/drivers/staging/sm750fb/sm750.h
@@ -147,17 +147,17 @@ struct lynxfb_output {
 	int dpms;
 	int paths;
 	/* which paths(s) this output stands for,for sm750:
-	   paths=1:means output for panel paths
-	   paths=2:means output for crt paths
-	   paths=3:means output for both panel and crt paths
-	*/
+	 * paths=1:means output for panel paths
+	 * paths=2:means output for crt paths
+	 * paths=3:means output for both panel and crt paths
+	 */
 
 	int *channel;
 	/* which channel these outputs linked with,for sm750:
-	   *channel=0 means primary channel
-	   *channel=1 means secondary channel
-	   output->channel ==> &crtc->channel
-	*/
+	 * *channel=0 means primary channel
+	 * *channel=1 means secondary channel
+	 * output->channel ==> &crtc->channel
+	 */
 	void *priv;
 
 	int (*proc_setBLANK)(struct lynxfb_output*, int);
diff --git a/drivers/staging/sm750fb/sm750_accel.c b/drivers/staging/sm750fb/sm750_accel.c
index 9aa4066ac86d..38adae6b5d83 100644
--- a/drivers/staging/sm750fb/sm750_accel.c
+++ b/drivers/staging/sm750fb/sm750_accel.c
@@ -67,7 +67,8 @@ void hw_de_init(struct lynx_accel *accel)
 
 /* set2dformat only be called from setmode functions
  * but if you need dual framebuffer driver,need call set2dformat
- * every time you use 2d function */
+ * every time you use 2d function
+ */
 
 void hw_set2dformat(struct lynx_accel *accel, int fmt)
 {
@@ -90,7 +91,8 @@ int hw_fillrect(struct lynx_accel *accel,
 
 	if (accel->de_wait() != 0) {
 		/* int time wait and always busy,seems hardware
-		 * got something error */
+		 * got something error
+		 */
 		pr_debug("De engine always busy\n");
 		return -1;
 	}
@@ -152,24 +154,26 @@ unsigned int rop2)   /* ROP value */
 		/* Determine direction of operation */
 		if (sy < dy) {
 			/* +----------+
-			   |S         |
-			   |   +----------+
-			   |   |      |   |
-			   |   |      |   |
-			   +---|------+   |
-			   |         D|
-			   +----------+ */
+			 * |S         |
+			 * |   +----------+
+			 * |   |      |   |
+			 * |   |      |   |
+			 * +---|------+   |
+			 * |         D|
+			 * +----------+
+			 */
 
 			nDirection = BOTTOM_TO_TOP;
 		} else if (sy > dy) {
 			/* +----------+
-			   |D         |
-			   |   +----------+
-			   |   |      |   |
-			   |   |      |   |
-			   +---|------+   |
-			   |         S|
-			   +----------+ */
+			 * |D         |
+			 * |   +----------+
+			 * |   |      |   |
+			 * |   |      |   |
+			 * +---|------+   |
+			 * |         S|
+			 * +----------+
+			 */
 
 			nDirection = TOP_TO_BOTTOM;
 		} else {
@@ -177,22 +181,24 @@ unsigned int rop2)   /* ROP value */
 
 			if (sx <= dx) {
 				/* +------+---+------+
-				   |S     |   |     D|
-				   |      |   |      |
-				   |      |   |      |
-				   |      |   |      |
-				   +------+---+------+ */
+				 * |S     |   |     D|
+				 * |      |   |      |
+				 * |      |   |      |
+				 * |      |   |      |
+				 * +------+---+------+
+				 */
 
 				nDirection = RIGHT_TO_LEFT;
 			} else {
 			/* sx > dx */
 
 				/* +------+---+------+
-				   |D     |   |     S|
-				   |      |   |      |
-				   |      |   |      |
-				   |      |   |      |
-				   +------+---+------+ */
+				 * |D     |   |     S|
+				 * |      |   |      |
+				 * |      |   |      |
+				 * |      |   |      |
+				 * +------+---+------+
+				 */
 
 				nDirection = LEFT_TO_RIGHT;
 			}
@@ -208,32 +214,36 @@ unsigned int rop2)   /* ROP value */
 	}
 
 	/* Note:
-	   DE_FOREGROUND are DE_BACKGROUND are don't care.
-	  DE_COLOR_COMPARE and DE_COLOR_COMPARE_MAKS are set by set deSetTransparency().
+	 * DE_FOREGROUND are DE_BACKGROUND are don't care.
+	 * DE_COLOR_COMPARE and DE_COLOR_COMPARE_MAKS
+	 * are set by set deSetTransparency().
 	 */
 
 	/* 2D Source Base.
-	 It is an address offset (128 bit aligned) from the beginning of frame buffer.
+	 * It is an address offset (128 bit aligned)
+	 * from the beginning of frame buffer.
 	 */
 	write_dpr(accel, DE_WINDOW_SOURCE_BASE, sBase); /* dpr40 */
 
 	/* 2D Destination Base.
-	 It is an address offset (128 bit aligned) from the beginning of frame buffer.
+	 * It is an address offset (128 bit aligned)
+	 * from the beginning of frame buffer.
 	 */
 	write_dpr(accel, DE_WINDOW_DESTINATION_BASE, dBase); /* dpr44 */
 
     /* Program pitch (distance between the 1st points of two adjacent lines).
-       Note that input pitch is BYTE value, but the 2D Pitch register uses
-       pixel values. Need Byte to pixel conversion.
-    */
+     * Note that input pitch is BYTE value, but the 2D Pitch register uses
+     * pixel values. Need Byte to pixel conversion.
+     */
 	write_dpr(accel, DE_PITCH,
 		  ((dPitch / Bpp << DE_PITCH_DESTINATION_SHIFT) &
 		   DE_PITCH_DESTINATION_MASK) |
 		  (sPitch / Bpp & DE_PITCH_SOURCE_MASK)); /* dpr10 */
 
     /* Screen Window width in Pixels.
-       2D engine uses this value to calculate the linear address in frame buffer for a given point.
-    */
+     * 2D engine uses this value to calculate the linear address in frame buffer
+     * for a given point.
+     */
 	write_dpr(accel, DE_WINDOW_WIDTH,
 		  ((dPitch / Bpp << DE_WINDOW_WIDTH_DST_SHIFT) &
 		   DE_WINDOW_WIDTH_DST_MASK) |
@@ -307,33 +317,37 @@ int hw_imageblit(struct lynx_accel *accel,
 		return -1;
 
 	/* 2D Source Base.
-	 Use 0 for HOST Blt.
+	 * Use 0 for HOST Blt.
 	 */
 	write_dpr(accel, DE_WINDOW_SOURCE_BASE, 0);
 
 	/* 2D Destination Base.
-	 It is an address offset (128 bit aligned) from the beginning of frame buffer.
+	 * It is an address offset (128 bit aligned)
+	 * from the beginning of frame buffer.
 	 */
 	write_dpr(accel, DE_WINDOW_DESTINATION_BASE, dBase);
     /* Program pitch (distance between the 1st points of two adjacent lines).
-       Note that input pitch is BYTE value, but the 2D Pitch register uses
-       pixel values. Need Byte to pixel conversion.
-    */
+     * Note that input pitch is BYTE value, but the 2D Pitch register uses
+     * pixel values. Need Byte to pixel conversion.
+     */
 	write_dpr(accel, DE_PITCH,
 		  ((dPitch / bytePerPixel << DE_PITCH_DESTINATION_SHIFT) &
 		   DE_PITCH_DESTINATION_MASK) |
 		  (dPitch / bytePerPixel & DE_PITCH_SOURCE_MASK)); /* dpr10 */
 
 	/* Screen Window width in Pixels.
-	 2D engine uses this value to calculate the linear address in frame buffer for a given point.
+	 * 2D engine uses this value to calculate the linear address
+	 * in frame buffer for a given point.
 	 */
 	write_dpr(accel, DE_WINDOW_WIDTH,
 		  ((dPitch / bytePerPixel << DE_WINDOW_WIDTH_DST_SHIFT) &
 		   DE_WINDOW_WIDTH_DST_MASK) |
 		  (dPitch / bytePerPixel & DE_WINDOW_WIDTH_SRC_MASK));
 
-	 /* Note: For 2D Source in Host Write, only X_K1_MONO field is needed, and Y_K2 field is not used.
-	    For mono bitmap, use startBit for X_K1. */
+	 /* Note: For 2D Source in Host Write, only X_K1_MONO field is needed,
+	  * and Y_K2 field is not used.
+	  * For mono bitmap, use startBit for X_K1.
+	  */
 	write_dpr(accel, DE_SOURCE,
 		  (startBit << DE_SOURCE_X_K1_SHIFT) &
 		  DE_SOURCE_X_K1_MONO_MASK); /* dpr00 */
diff --git a/drivers/staging/sm750fb/sm750_hw.c b/drivers/staging/sm750fb/sm750_hw.c
index 2daeedd88c30..7dd208caa5eb 100644
--- a/drivers/staging/sm750fb/sm750_hw.c
+++ b/drivers/staging/sm750fb/sm750_hw.c
@@ -35,17 +35,17 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 	pr_info("mmio phyAddr = %lx\n", sm750_dev->vidreg_start);
 
 	/* reserve the vidreg space of smi adaptor
-	 * if you do this, u need to add release region code
+	 * if you do this, you need to add release region code
 	 * in lynxfb_remove, or memory will not be mapped again
 	 * successfully
-	 * */
+	 */
 	ret = pci_request_region(pdev, 1, "sm750fb");
 	if (ret) {
 		pr_err("Can not request PCI regions.\n");
 		goto exit;
 	}
 
-	/* now map mmio and vidmem*/
+	/* now map mmio and vidmem */
 	sm750_dev->pvReg = ioremap_nocache(sm750_dev->vidreg_start,
 					   sm750_dev->vidreg_size);
 	if (!sm750_dev->pvReg) {
@@ -56,7 +56,6 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 		pr_info("mmio virtual addr = %p\n", sm750_dev->pvReg);
 	}
 
-
 	sm750_dev->accel.dprBase = sm750_dev->pvReg + DE_BASE_ADDR_TYPE1;
 	sm750_dev->accel.dpPortBase = sm750_dev->pvReg + DE_PORT_ADDR_TYPE1;
 
@@ -64,10 +63,10 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 
 	sm750_dev->vidmem_start = pci_resource_start(pdev, 0);
 	/* don't use pdev_resource[x].end - resource[x].start to
-	 * calculate the resource size,its only the maximum available
-	 * size but not the actual size,use
+	 * calculate the resource size, it's only the maximum available
+	 * size but not the actual size, using
 	 * @ddk750_getVMSize function can be safe.
-	 * */
+	 */
 	sm750_dev->vidmem_size = ddk750_getVMSize();
 	pr_info("video memory phyAddr = %lx, size = %u bytes\n",
 		sm750_dev->vidmem_start, sm750_dev->vidmem_size);
@@ -86,33 +85,31 @@ exit:
 	return ret;
 }
 
-
-
 int hw_sm750_inithw(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 {
 	struct init_status *parm;
 
 	parm = &sm750_dev->initParm;
 	if (parm->chip_clk == 0)
-		parm->chip_clk = (getChipType() == SM750LE) ?
+		parm->chip_clk = (sm750_get_chip_type() == SM750LE) ?
 						DEFAULT_SM750LE_CHIP_CLOCK :
 						DEFAULT_SM750_CHIP_CLOCK;
 
 	if (parm->mem_clk == 0)
 		parm->mem_clk = parm->chip_clk;
 	if (parm->master_clk == 0)
-		parm->master_clk = parm->chip_clk/3;
+		parm->master_clk = parm->chip_clk / 3;
 
 	ddk750_initHw((initchip_param_t *)&sm750_dev->initParm);
-	/* for sm718,open pci burst */
+	/* for sm718, open pci burst */
 	if (sm750_dev->devid == 0x718) {
 		POKE32(SYSTEM_CTRL,
 		       PEEK32(SYSTEM_CTRL) | SYSTEM_CTRL_PCI_BURST);
 	}
 
-	if (getChipType() != SM750LE) {
+	if (sm750_get_chip_type() != SM750LE) {
 		unsigned int val;
-		/* does user need CRT ?*/
+		/* does user need CRT? */
 		if (sm750_dev->nocrt) {
 			POKE32(MISC_CTRL,
 			       PEEK32(MISC_CTRL) | MISC_CTRL_DAC_POWER_OFF);
@@ -144,19 +141,21 @@ int hw_sm750_inithw(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 		}
 		POKE32(PANEL_DISPLAY_CTRL, val);
 	} else {
-		/* for 750LE ,no DVI chip initialization makes Monitor no signal */
-		/* Set up GPIO for software I2C to program DVI chip in the
-		   Xilinx SP605 board, in order to have video signal.
+		/* for 750LE, no DVI chip initialization
+		 * makes Monitor no signal
+		 *
+		 * Set up GPIO for software I2C to program DVI chip in the
+		 * Xilinx SP605 board, in order to have video signal.
 		 */
 		sm750_sw_i2c_init(0, 1);
 
 		/* Customer may NOT use CH7301 DVI chip, which has to be
-		initialized differently.
-		*/
+		 * initialized differently.
+		 */
 		if (sm750_sw_i2c_read_reg(0xec, 0x4a) == 0x95) {
 		/* The following register values for CH7301 are from
-		   Chrontel app note and our experiment.
-		*/
+		 * Chrontel app note and our experiment.
+		 */
 			pr_info("yes,CH7301 DVI chip found\n");
 			sm750_sw_i2c_write_reg(0xec, 0x1d, 0x16);
 			sm750_sw_i2c_write_reg(0xec, 0x21, 0x9);
@@ -173,7 +172,8 @@ int hw_sm750_inithw(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 }
 
 int hw_sm750_output_setMode(struct lynxfb_output *output,
-									struct fb_var_screeninfo *var, struct fb_fix_screeninfo *fix)
+			    struct fb_var_screeninfo *var,
+			    struct fb_fix_screeninfo *fix)
 {
 	int ret;
 	disp_output_t dispSet;
@@ -183,8 +183,7 @@ int hw_sm750_output_setMode(struct lynxfb_output *output,
 	dispSet = 0;
 	channel = *output->channel;
 
-
-	if (getChipType() != SM750LE) {
+	if (sm750_get_chip_type() != SM750LE) {
 		if (channel == sm750_primary) {
 			pr_info("primary channel\n");
 			if (output->paths & sm750_panel)
@@ -198,11 +197,10 @@ int hw_sm750_output_setMode(struct lynxfb_output *output,
 				dispSet |= do_LCD1_SEC;
 			if (output->paths & sm750_crt)
 				dispSet |= do_CRT_SEC;
-
 		}
 		ddk750_setLogicalDispOut(dispSet);
 	} else {
-		/* just open DISPLAY_CONTROL_750LE register bit 3:0*/
+		/* just open DISPLAY_CONTROL_750LE register bit 3:0 */
 		u32 reg;
 
 		reg = PEEK32(DISPLAY_CONTROL_750LE);
@@ -214,7 +212,8 @@ int hw_sm750_output_setMode(struct lynxfb_output *output,
 	return ret;
 }
 
-int hw_sm750_crtc_checkMode(struct lynxfb_crtc *crtc, struct fb_var_screeninfo *var)
+int hw_sm750_crtc_checkMode(struct lynxfb_crtc *crtc,
+			    struct fb_var_screeninfo *var)
 {
 	struct sm750_dev *sm750_dev;
 	struct lynxfb_par *par = container_of(crtc, struct lynxfb_par, crtc);
@@ -233,19 +232,15 @@ int hw_sm750_crtc_checkMode(struct lynxfb_crtc *crtc, struct fb_var_screeninfo *
 		break;
 	default:
 		return -EINVAL;
-
 	}
 
 	return 0;
 }
 
-
-/*
-	set the controller's mode for @crtc charged with @var and @fix parameters
-*/
+/* set the controller's mode for @crtc charged with @var and @fix parameters */
 int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
-								struct fb_var_screeninfo *var,
-								struct fb_fix_screeninfo *fix)
+			  struct fb_var_screeninfo *var,
+			  struct fb_fix_screeninfo *fix)
 {
 	int ret, fmt;
 	u32 reg;
@@ -254,7 +249,6 @@ int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
 	struct sm750_dev *sm750_dev;
 	struct lynxfb_par *par;
 
-
 	ret = 0;
 	par = container_of(crtc, struct lynxfb_par, crtc);
 	sm750_dev = par->dev;
@@ -278,17 +272,22 @@ int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
 
 	/* set timing */
 	modparm.pixel_clock = ps_to_hz(var->pixclock);
-	modparm.vertical_sync_polarity = (var->sync & FB_SYNC_HOR_HIGH_ACT) ? POS:NEG;
-	modparm.horizontal_sync_polarity = (var->sync & FB_SYNC_VERT_HIGH_ACT) ? POS:NEG;
-	modparm.clock_phase_polarity = (var->sync & FB_SYNC_COMP_HIGH_ACT) ? POS:NEG;
+	modparm.vertical_sync_polarity = (var->sync & FB_SYNC_HOR_HIGH_ACT)
+					 ? POS : NEG;
+	modparm.horizontal_sync_polarity = (var->sync & FB_SYNC_VERT_HIGH_ACT)
+					   ? POS : NEG;
+	modparm.clock_phase_polarity = (var->sync & FB_SYNC_COMP_HIGH_ACT)
+				       ? POS : NEG;
 	modparm.horizontal_display_end = var->xres;
 	modparm.horizontal_sync_width = var->hsync_len;
 	modparm.horizontal_sync_start = var->xres + var->right_margin;
-	modparm.horizontal_total = var->xres + var->left_margin + var->right_margin + var->hsync_len;
+	modparm.horizontal_total = var->xres + var->left_margin +
+				   var->right_margin + var->hsync_len;
 	modparm.vertical_display_end = var->yres;
 	modparm.vertical_sync_height = var->vsync_len;
 	modparm.vertical_sync_start = var->yres + var->lower_margin;
-	modparm.vertical_total = var->yres + var->upper_margin + var->lower_margin + var->vsync_len;
+	modparm.vertical_total = var->yres + var->upper_margin +
+				 var->lower_margin + var->vsync_len;
 
 	/* choose pll */
 	if (crtc->channel != sm750_secondary)
@@ -304,12 +303,14 @@ int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
 	}
 
 	if (crtc->channel != sm750_secondary) {
-		/* set pitch, offset ,width,start address ,etc... */
+		/* set pitch, offset, width, start address, etc... */
 		POKE32(PANEL_FB_ADDRESS,
 		       crtc->oScreen & PANEL_FB_ADDRESS_ADDRESS_MASK);
 
 		reg = var->xres * (var->bits_per_pixel >> 3);
-		/* crtc->channel is not equal to par->index on numeric,be aware of that */
+		/* crtc->channel is not equal to par->index on numeric,
+		 * be aware of that
+		 */
 		reg = ALIGN(reg, crtc->line_pad);
 		reg = (reg << PANEL_FB_WIDTH_WIDTH_SHIFT) &
 		       PANEL_FB_WIDTH_WIDTH_MASK;
@@ -321,8 +322,8 @@ int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
 		reg |= (var->xoffset & PANEL_WINDOW_WIDTH_X_MASK);
 		POKE32(PANEL_WINDOW_WIDTH, reg);
 
-		reg = ((var->yres_virtual - 1) <<
-		       PANEL_WINDOW_HEIGHT_HEIGHT_SHIFT);
+		reg = (var->yres_virtual - 1) <<
+		      PANEL_WINDOW_HEIGHT_HEIGHT_SHIFT;
 		reg &= PANEL_WINDOW_HEIGHT_HEIGHT_MASK;
 		reg |= (var->yoffset & PANEL_WINDOW_HEIGHT_Y_MASK);
 		POKE32(PANEL_WINDOW_HEIGHT, reg);
@@ -341,7 +342,9 @@ int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
 		/* not implemented now */
 		POKE32(CRT_FB_ADDRESS, crtc->oScreen);
 		reg = var->xres * (var->bits_per_pixel >> 3);
-		/* crtc->channel is not equal to par->index on numeric,be aware of that */
+		/* crtc->channel is not equal to par->index on numeric,
+		 * be aware of that
+		 */
 		reg = ALIGN(reg, crtc->line_pad) << CRT_FB_WIDTH_WIDTH_SHIFT;
 		reg &= CRT_FB_WIDTH_WIDTH_MASK;
 		reg |= (fix->line_length & CRT_FB_WIDTH_OFFSET_MASK);
@@ -352,20 +355,19 @@ int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
 		reg |= ((var->bits_per_pixel >> 4) &
 			CRT_DISPLAY_CTRL_FORMAT_MASK);
 		POKE32(CRT_DISPLAY_CTRL, reg);
-
 	}
 
-
 exit:
 	return ret;
 }
 
 int hw_sm750_setColReg(struct lynxfb_crtc *crtc, ushort index,
-								ushort red, ushort green, ushort blue)
+		       ushort red, ushort green, ushort blue)
 {
 	static unsigned int add[] = {PANEL_PALETTE_RAM, CRT_PALETTE_RAM};
 
-	POKE32(add[crtc->channel] + index*4, (red<<16)|(green<<8)|blue);
+	POKE32(add[crtc->channel] + index * 4,
+	       (red << 16) | (green << 8) | blue);
 	return 0;
 }
 
@@ -414,7 +416,9 @@ int hw_sm750_setBLANK(struct lynxfb_output *output, int blank)
 {
 	unsigned int dpms, pps, crtdb;
 
-	dpms = pps = crtdb = 0;
+	dpms = 0;
+	pps = 0;
+	crtdb = 0;
 
 	switch (blank) {
 	case FB_BLANK_UNBLANK:
@@ -461,14 +465,13 @@ int hw_sm750_setBLANK(struct lynxfb_output *output, int blank)
 	return 0;
 }
 
-
 void hw_sm750_initAccel(struct sm750_dev *sm750_dev)
 {
 	u32 reg;
 
 	enable2DEngine(1);
 
-	if (getChipType() == SM750LE) {
+	if (sm750_get_chip_type() == SM750LE) {
 		reg = PEEK32(DE_STATE1);
 		reg |= DE_STATE1_DE_ABORT;
 		POKE32(DE_STATE1, reg);
@@ -509,7 +512,6 @@ int hw_sm750le_deWait(void)
 	return -1;
 }
 
-
 int hw_sm750_deWait(void)
 {
 	int i = 0x10000000;
@@ -529,10 +531,10 @@ int hw_sm750_deWait(void)
 }
 
 int hw_sm750_pan_display(struct lynxfb_crtc *crtc,
-	const struct fb_var_screeninfo *var,
-	const struct fb_info *info)
+			 const struct fb_var_screeninfo *var,
+			 const struct fb_info *info)
 {
-	uint32_t total;
+	u32 total;
 	/* check params */
 	if ((var->xoffset + var->xres > var->xres_virtual) ||
 	    (var->yoffset + var->yres > var->yres_virtual)) {
diff --git a/drivers/staging/speakup/devsynth.c b/drivers/staging/speakup/devsynth.c
index 84989711ae67..58abd1d85105 100644
--- a/drivers/staging/speakup/devsynth.c
+++ b/drivers/staging/speakup/devsynth.c
@@ -34,7 +34,7 @@ static ssize_t speakup_file_write(struct file *fp, const char __user *buffer,
 		synth_write(buf, bytes);
 		spin_unlock_irqrestore(&speakup_info.spinlock, flags);
 	}
-	return (ssize_t) nbytes;
+	return (ssize_t)nbytes;
 }
 
 static ssize_t speakup_file_read(struct file *fp, char __user *buf,
diff --git a/drivers/staging/speakup/kobjects.c b/drivers/staging/speakup/kobjects.c
index 528cbdce4227..e744aa9730ff 100644
--- a/drivers/staging/speakup/kobjects.c
+++ b/drivers/staging/speakup/kobjects.c
@@ -251,7 +251,7 @@ static ssize_t keymap_show(struct kobject *kobj, struct kobj_attribute *attr,
 	}
 	cp += sprintf(cp, "0, %d\n", KEY_MAP_VER);
 	spin_unlock_irqrestore(&speakup_info.spinlock, flags);
-	return (int)(cp-buf);
+	return (int)(cp - buf);
 }
 
 /*
@@ -288,8 +288,8 @@ static ssize_t keymap_store(struct kobject *kobj, struct kobj_attribute *attr,
 		cp = spk_s2uchar(cp, cp1);
 		cp1++;
 	}
-	i = (int)cp1[-2]+1;
-	i *= (int)cp1[-1]+1;
+	i = (int)cp1[-2] + 1;
+	i *= (int)cp1[-1] + 1;
 	i += 2; /* 0 and last map ver */
 	if (cp1[-3] != KEY_MAP_VER || cp1[-1] > 10 ||
 			i+SHIFT_TBL_SIZE+4 >= sizeof(spk_key_buf)) {
@@ -350,9 +350,9 @@ static ssize_t silent_store(struct kobject *kobj, struct kobj_attribute *attr,
 	} else {
 		shut = 0;
 	}
-	if (ch&4)
+	if (ch & 4)
 		shut |= 0x40;
-	if (ch&1)
+	if (ch & 1)
 		spk_shut_up |= shut;
 	else
 		spk_shut_up &= ~shut;
@@ -411,11 +411,13 @@ static ssize_t synth_direct_store(struct kobject *kobj,
 	int len;
 	int bytes;
 	const char *ptr = buf;
+	unsigned long flags;
 
 	if (!synth)
 		return -EPERM;
 
 	len = strlen(buf);
+	spin_lock_irqsave(&speakup_info.spinlock, flags);
 	while (len > 0) {
 		bytes = min_t(size_t, len, 250);
 		strncpy(tmp, ptr, bytes);
@@ -425,6 +427,7 @@ static ssize_t synth_direct_store(struct kobject *kobj,
 		ptr += bytes;
 		len -= bytes;
 	}
+	spin_unlock_irqrestore(&speakup_info.spinlock, flags);
 	return count;
 }
 
@@ -973,11 +976,11 @@ static struct attribute *i18n_attrs[] = {
  * created for the attributes with the directory being the name of the
  * attribute group.
  */
-static struct attribute_group main_attr_group = {
+static const struct attribute_group main_attr_group = {
 	.attrs = main_attrs,
 };
 
-static struct attribute_group i18n_attr_group = {
+static const struct attribute_group i18n_attr_group = {
 	.attrs = i18n_attrs,
 	.name = "i18n",
 };
diff --git a/drivers/staging/speakup/synth.c b/drivers/staging/speakup/synth.c
index 4f462c35fdd9..54b2f3918628 100644
--- a/drivers/staging/speakup/synth.c
+++ b/drivers/staging/speakup/synth.c
@@ -18,7 +18,7 @@
 #include "serialio.h"
 
 #define MAXSYNTHS       16      /* Max number of synths in array. */
-static struct spk_synth *synths[MAXSYNTHS];
+static struct spk_synth *synths[MAXSYNTHS + 1];
 struct spk_synth *synth;
 char spk_pitch_buff[32] = "";
 static int module_status;
@@ -407,7 +407,7 @@ static int do_synth_init(struct spk_synth *in_synth)
 	if (!spk_quiet_boot)
 		synth_printf("%s found\n", synth->long_name);
 	if (synth->attributes.name
-	&& sysfs_create_group(speakup_kobj, &(synth->attributes)) < 0)
+	&& sysfs_create_group(speakup_kobj, &synth->attributes) < 0)
 		return -ENOMEM;
 	synth_flags = synth->flags;
 	wake_up_interruptible_all(&speakup_event);
@@ -429,7 +429,7 @@ void synth_release(void)
 	del_timer(&thread_timer);
 	spin_unlock_irqrestore(&speakup_info.spinlock, flags);
 	if (synth->attributes.name)
-		sysfs_remove_group(speakup_kobj, &(synth->attributes));
+		sysfs_remove_group(speakup_kobj, &synth->attributes);
 	for (var = synth->vars; var->var_id != MAXVARS; var++)
 		speakup_unregister_var(var->var_id);
 	spk_stop_serial_interrupt();
diff --git a/drivers/staging/speakup/varhandlers.c b/drivers/staging/speakup/varhandlers.c
index e1393d2a2b0f..21186e3dc7ad 100644
--- a/drivers/staging/speakup/varhandlers.c
+++ b/drivers/staging/speakup/varhandlers.c
@@ -276,7 +276,7 @@ int spk_set_mask_bits(const char *input, const int which, const int how)
 	u_char *cp;
 	short mask = spk_punc_info[which].mask;
 
-	if (how&1) {
+	if (how & 1) {
 		for (cp = (u_char *)spk_punc_info[3].value; *cp; cp++)
 			spk_chartab[*cp] &= ~mask;
 	}
@@ -290,14 +290,14 @@ int spk_set_mask_bits(const char *input, const int which, const int how)
 			if (mask < PUNC) {
 				if (!(spk_chartab[*cp] & PUNC))
 					break;
-			} else if (spk_chartab[*cp]&B_NUM)
+			} else if (spk_chartab[*cp] & B_NUM)
 				break;
 		}
 		if (*cp)
 			return -EINVAL;
 		cp = (u_char *)input;
 	}
-	if (how&2) {
+	if (how & 2) {
 		for (; *cp; cp++)
 			if (*cp > SPACE)
 				spk_chartab[*cp] |= mask;
diff --git a/drivers/staging/unisys/include/channel.h b/drivers/staging/unisys/include/channel.h
index db4e6b28755b..259ef6487959 100644
--- a/drivers/staging/unisys/include/channel.h
+++ b/drivers/staging/unisys/include/channel.h
@@ -75,28 +75,6 @@ enum channel_clientstate {
 				/* access channel anytime */
 };
 
-static inline const u8 *
-ULTRA_CHANNELCLI_STRING(u32 state)
-{
-	switch (state) {
-	case CHANNELCLI_DETACHED:
-		return (const u8 *)("DETACHED");
-	case CHANNELCLI_DISABLED:
-		return (const u8 *)("DISABLED");
-	case CHANNELCLI_ATTACHING:
-		return (const u8 *)("ATTACHING");
-	case CHANNELCLI_ATTACHED:
-		return (const u8 *)("ATTACHED");
-	case CHANNELCLI_BUSY:
-		return (const u8 *)("BUSY");
-	case CHANNELCLI_OWNED:
-		return (const u8 *)("OWNED");
-	default:
-		break;
-	}
-	return (const u8 *)("?");
-}
-
 #define SPAR_CHANNEL_SERVER_READY(ch) \
 	(readl(&(ch)->srv_state) == CHANNELSRV_READY)
 
@@ -132,22 +110,6 @@ ULTRA_CHANNELCLI_STRING(u32 state)
 /* throttling invalid boot channel statetransition error due to busy channel */
 #define ULTRA_CLIERRORBOOT_THROTTLEMSG_BUSY        0x04
 
-/* Values for ULTRA_CHANNEL_PROTOCOL.CliErrorOS: */
-/* throttling invalid guest OS channel statetransition error due to
- * client disabled
- */
-#define ULTRA_CLIERROROS_THROTTLEMSG_DISABLED      0x01
-
-/* throttling invalid guest OS channel statetransition error due to
- * client not attached
- */
-#define ULTRA_CLIERROROS_THROTTLEMSG_NOTATTACHED   0x02
-
-/* throttling invalid guest OS channel statetransition error due to
- * busy channel
- */
-#define ULTRA_CLIERROROS_THROTTLEMSG_BUSY          0x04
-
 /* Values for ULTRA_CHANNEL_PROTOCOL.Features: This define exists so
  * that windows guest can look at the FeatureFlags in the io channel,
  * and configure the windows driver to use interrupts or not based on
@@ -347,148 +309,6 @@ static inline int spar_check_channel_server(uuid_le typeuuid, char *name,
 	return 1;
 }
 
-/* Given a file pathname <s> (with '/' or '\' separating directory nodes),
- * returns a pointer to the beginning of a node within that pathname such
- * that the number of nodes from that pointer to the end of the string is
- * NOT more than <n>.  Note that if the pathname has less than <n> nodes
- * in it, the return pointer will be to the beginning of the string.
- */
-static inline u8 *
-pathname_last_n_nodes(u8 *s, unsigned int n)
-{
-	u8 *p = s;
-	unsigned int node_count = 0;
-
-	while (*p != '\0') {
-		if ((*p == '/') || (*p == '\\'))
-			node_count++;
-		p++;
-	}
-	if (node_count <= n)
-		return s;
-	while (n > 0) {
-		p--;
-		if (p == s)
-			break;	/* should never happen, unless someone
-				 * is changing the string while we are
-				 * looking at it!!
-				 */
-		if ((*p == '/') || (*p == '\\'))
-			n--;
-	}
-	return p + 1;
-}
-
-static inline int
-spar_channel_client_acquire_os(void __iomem *ch, u8 *id)
-{
-	struct channel_header __iomem *hdr = ch;
-
-	if (readl(&hdr->cli_state_os) == CHANNELCLI_DISABLED) {
-		if ((readb(&hdr->cli_error_os)
-		     & ULTRA_CLIERROROS_THROTTLEMSG_DISABLED) == 0) {
-			/* we are NOT throttling this message */
-			writeb(readb(&hdr->cli_error_os) |
-			       ULTRA_CLIERROROS_THROTTLEMSG_DISABLED,
-			       &hdr->cli_error_os);
-			/* throttle until acquire successful */
-
-			pr_info("%s Channel StateTransition INVALID! - acquire failed because OS client DISABLED\n",
-				id);
-		}
-		return 0;
-	}
-	if ((readl(&hdr->cli_state_os) != CHANNELCLI_OWNED) &&
-	    (readl(&hdr->cli_state_boot) == CHANNELCLI_DISABLED)) {
-		/* Our competitor is DISABLED, so we can transition to OWNED */
-		pr_info("%s Channel StateTransition (%s) %s(%d)-->%s(%d)\n",
-			id, "cli_state_os",
-			ULTRA_CHANNELCLI_STRING(readl(&hdr->cli_state_os)),
-			readl(&hdr->cli_state_os),
-			ULTRA_CHANNELCLI_STRING(CHANNELCLI_OWNED),
-			CHANNELCLI_OWNED);
-		writel(CHANNELCLI_OWNED, &hdr->cli_state_os);
-		mb(); /* required for channel synch */
-	}
-	if (readl(&hdr->cli_state_os) == CHANNELCLI_OWNED) {
-		if (readb(&hdr->cli_error_os)) {
-			/* we are in an error msg throttling state;
-			 * come out of it
-			 */
-			pr_info("%s Channel OS client acquire now successful\n",
-				id);
-			writeb(0, &hdr->cli_error_os);
-		}
-		return 1;
-	}
-
-	/* We have to do it the "hard way".  We transition to BUSY,
-	 * and can use the channel iff our competitor has not also
-	 * transitioned to BUSY.
-	 */
-	if (readl(&hdr->cli_state_os) != CHANNELCLI_ATTACHED) {
-		if ((readb(&hdr->cli_error_os)
-		     & ULTRA_CLIERROROS_THROTTLEMSG_NOTATTACHED) == 0) {
-			/* we are NOT throttling this message */
-			writeb(readb(&hdr->cli_error_os) |
-			       ULTRA_CLIERROROS_THROTTLEMSG_NOTATTACHED,
-			       &hdr->cli_error_os);
-			/* throttle until acquire successful */
-			pr_info("%s Channel StateTransition INVALID! - acquire failed because OS client NOT ATTACHED (state=%s(%d))\n",
-				id, ULTRA_CHANNELCLI_STRING(
-						readl(&hdr->cli_state_os)),
-				readl(&hdr->cli_state_os));
-		}
-		return 0;
-	}
-	writel(CHANNELCLI_BUSY, &hdr->cli_state_os);
-	mb(); /* required for channel synch */
-	if (readl(&hdr->cli_state_boot) == CHANNELCLI_BUSY) {
-		if ((readb(&hdr->cli_error_os)
-		     & ULTRA_CLIERROROS_THROTTLEMSG_BUSY) == 0) {
-			/* we are NOT throttling this message */
-			writeb(readb(&hdr->cli_error_os) |
-			       ULTRA_CLIERROROS_THROTTLEMSG_BUSY,
-			       &hdr->cli_error_os);
-			/* throttle until acquire successful */
-			pr_info("%s Channel StateTransition failed - host OS acquire failed because boot BUSY\n",
-				id);
-		}
-		/* reset busy */
-		writel(CHANNELCLI_ATTACHED, &hdr->cli_state_os);
-		mb(); /* required for channel synch */
-		return 0;
-	}
-	if (readb(&hdr->cli_error_os)) {
-		/* we are in an error msg throttling state; come out of it */
-		pr_info("%s Channel OS client acquire now successful\n", id);
-		writeb(0, &hdr->cli_error_os);
-	}
-	return 1;
-}
-
-static inline void
-spar_channel_client_release_os(void __iomem *ch, u8 *id)
-{
-	struct channel_header __iomem *hdr = ch;
-
-	if (readb(&hdr->cli_error_os)) {
-		/* we are in an error msg throttling state; come out of it */
-		pr_info("%s Channel OS client error state cleared\n", id);
-		writeb(0, &hdr->cli_error_os);
-	}
-	if (readl(&hdr->cli_state_os) == CHANNELCLI_OWNED)
-		return;
-	if (readl(&hdr->cli_state_os) != CHANNELCLI_BUSY) {
-		pr_info("%s Channel StateTransition INVALID! - release failed because OS client NOT BUSY (state=%s(%d))\n",
-			id, ULTRA_CHANNELCLI_STRING(
-					readl(&hdr->cli_state_os)),
-			readl(&hdr->cli_state_os));
-		/* return; */
-	}
-	writel(CHANNELCLI_ATTACHED, &hdr->cli_state_os); /* release busy */
-}
-
 /*
 * Routine Description:
 * Tries to insert the prebuilt signal pointed to by pSignal into the nth
@@ -569,4 +389,45 @@ unsigned int spar_signal_remove_all(struct channel_header *ch, u32 queue,
 unsigned char spar_signalqueue_empty(struct channel_header __iomem *ch,
 				     u32 queue);
 
+/*
+ * CHANNEL Guids
+ */
+
+/* {414815ed-c58c-11da-95a9-00e08161165f} */
+#define SPAR_VHBA_CHANNEL_PROTOCOL_UUID \
+		UUID_LE(0x414815ed, 0xc58c, 0x11da, \
+				0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
+static const uuid_le spar_vhba_channel_protocol_uuid =
+	SPAR_VHBA_CHANNEL_PROTOCOL_UUID;
+#define SPAR_VHBA_CHANNEL_PROTOCOL_UUID_STR \
+	"414815ed-c58c-11da-95a9-00e08161165f"
+
+/* {8cd5994d-c58e-11da-95a9-00e08161165f} */
+#define SPAR_VNIC_CHANNEL_PROTOCOL_UUID \
+		UUID_LE(0x8cd5994d, 0xc58e, 0x11da, \
+				0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
+static const uuid_le spar_vnic_channel_protocol_uuid =
+	SPAR_VNIC_CHANNEL_PROTOCOL_UUID;
+#define SPAR_VNIC_CHANNEL_PROTOCOL_UUID_STR \
+	"8cd5994d-c58e-11da-95a9-00e08161165f"
+
+/* {72120008-4AAB-11DC-8530-444553544200} */
+#define SPAR_SIOVM_UUID \
+		UUID_LE(0x72120008, 0x4AAB, 0x11DC, \
+				0x85, 0x30, 0x44, 0x45, 0x53, 0x54, 0x42, 0x00)
+static const uuid_le spar_siovm_uuid = SPAR_SIOVM_UUID;
+
+/* {5b52c5ac-e5f5-4d42-8dff-429eaecd221f} */
+#define SPAR_CONTROLDIRECTOR_CHANNEL_PROTOCOL_UUID  \
+		UUID_LE(0x5b52c5ac, 0xe5f5, 0x4d42, \
+				0x8d, 0xff, 0x42, 0x9e, 0xae, 0xcd, 0x22, 0x1f)
+
+static const uuid_le spar_controldirector_channel_protocol_uuid =
+	SPAR_CONTROLDIRECTOR_CHANNEL_PROTOCOL_UUID;
+
+/* {b4e79625-aede-4eAA-9e11-D3eddcd4504c} */
+#define SPAR_DIAG_POOL_CHANNEL_PROTOCOL_UUID				\
+		UUID_LE(0xb4e79625, 0xaede, 0x4eaa, \
+				0x9e, 0x11, 0xd3, 0xed, 0xdc, 0xd4, 0x50, 0x4c)
+
 #endif
diff --git a/drivers/staging/unisys/include/channel_guid.h b/drivers/staging/unisys/include/channel_guid.h
deleted file mode 100644
index 17cb499cb53c..000000000000
--- a/drivers/staging/unisys/include/channel_guid.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/* Copyright (C) 2010 - 2013 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or (at
- * your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-/*
- * CHANNEL Guids
- */
-
-/* {414815ed-c58c-11da-95a9-00e08161165f} */
-#define SPAR_VHBA_CHANNEL_PROTOCOL_UUID \
-		UUID_LE(0x414815ed, 0xc58c, 0x11da, \
-				0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
-static const uuid_le spar_vhba_channel_protocol_uuid =
-	SPAR_VHBA_CHANNEL_PROTOCOL_UUID;
-#define SPAR_VHBA_CHANNEL_PROTOCOL_UUID_STR \
-	"414815ed-c58c-11da-95a9-00e08161165f"
-
-/* {8cd5994d-c58e-11da-95a9-00e08161165f} */
-#define SPAR_VNIC_CHANNEL_PROTOCOL_UUID \
-		UUID_LE(0x8cd5994d, 0xc58e, 0x11da, \
-				0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
-static const uuid_le spar_vnic_channel_protocol_uuid =
-	SPAR_VNIC_CHANNEL_PROTOCOL_UUID;
-#define SPAR_VNIC_CHANNEL_PROTOCOL_UUID_STR \
-	"8cd5994d-c58e-11da-95a9-00e08161165f"
-
-/* {72120008-4AAB-11DC-8530-444553544200} */
-#define SPAR_SIOVM_UUID \
-		UUID_LE(0x72120008, 0x4AAB, 0x11DC, \
-				0x85, 0x30, 0x44, 0x45, 0x53, 0x54, 0x42, 0x00)
-static const uuid_le spar_siovm_uuid = SPAR_SIOVM_UUID;
-
-/* {5b52c5ac-e5f5-4d42-8dff-429eaecd221f} */
-#define SPAR_CONTROLDIRECTOR_CHANNEL_PROTOCOL_UUID  \
-		UUID_LE(0x5b52c5ac, 0xe5f5, 0x4d42, \
-				0x8d, 0xff, 0x42, 0x9e, 0xae, 0xcd, 0x22, 0x1f)
-
-static const uuid_le spar_controldirector_channel_protocol_uuid =
-	SPAR_CONTROLDIRECTOR_CHANNEL_PROTOCOL_UUID;
-
-/* {b4e79625-aede-4eAA-9e11-D3eddcd4504c} */
-#define SPAR_DIAG_POOL_CHANNEL_PROTOCOL_UUID				\
-		UUID_LE(0xb4e79625, 0xaede, 0x4eaa, \
-				0x9e, 0x11, 0xd3, 0xed, 0xdc, 0xd4, 0x50, 0x4c)
diff --git a/drivers/staging/unisys/include/diagchannel.h b/drivers/staging/unisys/include/diagchannel.h
deleted file mode 100644
index 6e813c77b97e..000000000000
--- a/drivers/staging/unisys/include/diagchannel.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* Copyright (C) 2010 - 2013 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or (at
- * your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-#ifndef _DIAG_CHANNEL_H_
-#define _DIAG_CHANNEL_H_
-
-/* Levels of severity for diagnostic events, in order from lowest severity to
- * highest (i.e. fatal errors are the most severe, and should always be logged,
- * but info events rarely need to be logged except during debugging). The
- * values DIAG_SEVERITY_ENUM_BEGIN and DIAG_SEVERITY_ENUM_END are not valid
- * severity values.  They exist merely to dilineate the list, so that future
- * additions won't require changes to the driver (i.e. when checking for
- * out-of-range severities in SetSeverity). The values DIAG_SEVERITY_OVERRIDE
- * and DIAG_SEVERITY_SHUTOFF are not valid severity values for logging events
- * but they are valid for controlling the amount of event data. Changes made
- * to the enum, need to be reflected in s-Par.
- */
-enum diag_severity {
-		DIAG_SEVERITY_VERBOSE = 0,
-		DIAG_SEVERITY_INFO = 1,
-		DIAG_SEVERITY_WARNING = 2,
-		DIAG_SEVERITY_ERR = 3,
-		DIAG_SEVERITY_PRINT = 4,
-};
-
-#endif
diff --git a/drivers/staging/unisys/include/guestlinuxdebug.h b/drivers/staging/unisys/include/guestlinuxdebug.h
deleted file mode 100644
index b81287f5e2c3..000000000000
--- a/drivers/staging/unisys/include/guestlinuxdebug.h
+++ /dev/null
@@ -1,180 +0,0 @@
-/* Copyright (C) 2010 - 2013 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or (at
- * your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-#ifndef __GUESTLINUXDEBUG_H__
-#define __GUESTLINUXDEBUG_H__
-
-/*
- * This file contains supporting interface for "vmcallinterface.h", particularly
- * regarding adding additional structure and functionality to linux
- * ISSUE_IO_VMCALL_POSTCODE_SEVERITY
- */
-
-/******* INFO ON ISSUE_POSTCODE_LINUX() BELOW *******/
-enum driver_pc {		/* POSTCODE driver identifier tuples */
-	/* visorchipset driver files */
-	VISOR_CHIPSET_PC = 0xA0,
-	VISOR_CHIPSET_PC_controlvm_c = 0xA1,
-	VISOR_CHIPSET_PC_controlvm_cm2 = 0xA2,
-	VISOR_CHIPSET_PC_controlvm_direct_c = 0xA3,
-	VISOR_CHIPSET_PC_file_c = 0xA4,
-	VISOR_CHIPSET_PC_parser_c = 0xA5,
-	VISOR_CHIPSET_PC_testing_c = 0xA6,
-	VISOR_CHIPSET_PC_visorchipset_main_c = 0xA7,
-	VISOR_CHIPSET_PC_visorswitchbus_c = 0xA8,
-	/* visorbus driver files */
-	VISOR_BUS_PC = 0xB0,
-	VISOR_BUS_PC_businst_attr_c = 0xB1,
-	VISOR_BUS_PC_channel_attr_c = 0xB2,
-	VISOR_BUS_PC_devmajorminor_attr_c = 0xB3,
-	VISOR_BUS_PC_visorbus_main_c = 0xB4,
-	/* visorclientbus driver files */
-	VISOR_CLIENT_BUS_PC = 0xC0,
-	VISOR_CLIENT_BUS_PC_visorclientbus_main_c = 0xC1,
-	/* virt hba driver files */
-	VIRT_HBA_PC = 0xC2,
-	VIRT_HBA_PC_virthba_c = 0xC3,
-	/* virtpci driver files */
-	VIRT_PCI_PC = 0xC4,
-	VIRT_PCI_PC_virtpci_c = 0xC5,
-	/* virtnic driver files */
-	VIRT_NIC_PC = 0xC6,
-	VIRT_NIC_P_virtnic_c = 0xC7,
-	/* uislib driver files */
-	UISLIB_PC = 0xD0,
-	UISLIB_PC_uislib_c = 0xD1,
-	UISLIB_PC_uisqueue_c = 0xD2,
-	UISLIB_PC_uisthread_c = 0xD3,
-	UISLIB_PC_uisutils_c = 0xD4,
-};
-
-enum event_pc {			/* POSTCODE event identifier tuples */
-	ATTACH_PORT_ENTRY_PC = 0x001,
-	ATTACH_PORT_FAILURE_PC = 0x002,
-	ATTACH_PORT_SUCCESS_PC = 0x003,
-	BUS_FAILURE_PC = 0x004,
-	BUS_CREATE_ENTRY_PC = 0x005,
-	BUS_CREATE_FAILURE_PC = 0x006,
-	BUS_CREATE_EXIT_PC = 0x007,
-	BUS_CONFIGURE_ENTRY_PC = 0x008,
-	BUS_CONFIGURE_FAILURE_PC = 0x009,
-	BUS_CONFIGURE_EXIT_PC = 0x00A,
-	CHIPSET_INIT_ENTRY_PC = 0x00B,
-	CHIPSET_INIT_SUCCESS_PC = 0x00C,
-	CHIPSET_INIT_FAILURE_PC = 0x00D,
-	CHIPSET_INIT_EXIT_PC = 0x00E,
-	CREATE_WORKQUEUE_PC = 0x00F,
-	CREATE_WORKQUEUE_FAILED_PC = 0x0A0,
-	CONTROLVM_INIT_FAILURE_PC = 0x0A1,
-	DEVICE_CREATE_ENTRY_PC = 0x0A2,
-	DEVICE_CREATE_FAILURE_PC = 0x0A3,
-	DEVICE_CREATE_SUCCESS_PC = 0x0A4,
-	DEVICE_CREATE_EXIT_PC = 0x0A5,
-	DEVICE_ADD_PC = 0x0A6,
-	DEVICE_REGISTER_FAILURE_PC = 0x0A7,
-	DEVICE_CHANGESTATE_ENTRY_PC = 0x0A8,
-	DEVICE_CHANGESTATE_FAILURE_PC = 0x0A9,
-	DEVICE_CHANGESTATE_EXIT_PC = 0x0AA,
-	DRIVER_ENTRY_PC = 0x0AB,
-	DRIVER_EXIT_PC = 0x0AC,
-	MALLOC_FAILURE_PC = 0x0AD,
-	QUEUE_DELAYED_WORK_PC = 0x0AE,
-	UISLIB_THREAD_FAILURE_PC = 0x0B7,
-	VBUS_CHANNEL_ENTRY_PC = 0x0B8,
-	VBUS_CHANNEL_FAILURE_PC = 0x0B9,
-	VBUS_CHANNEL_EXIT_PC = 0x0BA,
-	VHBA_CREATE_ENTRY_PC = 0x0BB,
-	VHBA_CREATE_FAILURE_PC = 0x0BC,
-	VHBA_CREATE_EXIT_PC = 0x0BD,
-	VHBA_CREATE_SUCCESS_PC = 0x0BE,
-	VHBA_COMMAND_HANDLER_PC = 0x0BF,
-	VHBA_PROBE_ENTRY_PC = 0x0C0,
-	VHBA_PROBE_FAILURE_PC = 0x0C1,
-	VHBA_PROBE_EXIT_PC = 0x0C2,
-	VNIC_CREATE_ENTRY_PC = 0x0C3,
-	VNIC_CREATE_FAILURE_PC = 0x0C4,
-	VNIC_CREATE_SUCCESS_PC = 0x0C5,
-	VNIC_PROBE_ENTRY_PC = 0x0C6,
-	VNIC_PROBE_FAILURE_PC = 0x0C7,
-	VNIC_PROBE_EXIT_PC = 0x0C8,
-	VPCI_CREATE_ENTRY_PC = 0x0C9,
-	VPCI_CREATE_FAILURE_PC = 0x0CA,
-	VPCI_CREATE_EXIT_PC = 0x0CB,
-	VPCI_PROBE_ENTRY_PC = 0x0CC,
-	VPCI_PROBE_FAILURE_PC = 0x0CD,
-	VPCI_PROBE_EXIT_PC = 0x0CE,
-	CRASH_DEV_ENTRY_PC = 0x0CF,
-	CRASH_DEV_EXIT_PC = 0x0D0,
-	CRASH_DEV_HADDR_NULL = 0x0D1,
-	CRASH_DEV_CONTROLVM_NULL = 0x0D2,
-	CRASH_DEV_RD_BUS_FAIULRE_PC = 0x0D3,
-	CRASH_DEV_RD_DEV_FAIULRE_PC = 0x0D4,
-	CRASH_DEV_BUS_NULL_FAILURE_PC = 0x0D5,
-	CRASH_DEV_DEV_NULL_FAILURE_PC = 0x0D6,
-	CRASH_DEV_CTRL_RD_FAILURE_PC = 0x0D7,
-	CRASH_DEV_COUNT_FAILURE_PC = 0x0D8,
-	SAVE_MSG_BUS_FAILURE_PC = 0x0D9,
-	SAVE_MSG_DEV_FAILURE_PC = 0x0DA,
-	CALLHOME_INIT_FAILURE_PC = 0x0DB
-};
-
-#ifdef __GNUC__
-
-#define POSTCODE_SEVERITY_ERR DIAG_SEVERITY_ERR
-#define POSTCODE_SEVERITY_WARNING DIAG_SEVERITY_WARNING
-/* TODO-> Info currently doesn't show, so we set info=warning */
-#define POSTCODE_SEVERITY_INFO DIAG_SEVERITY_PRINT
-
-/* example call of POSTCODE_LINUX_2(VISOR_CHIPSET_PC, POSTCODE_SEVERITY_ERR);
- * Please also note that the resulting postcode is in hex, so if you are
- * searching for the __LINE__ number, convert it first to decimal.  The line
- * number combined with driver and type of call, will allow you to track down
- * exactly what line an error occurred on, or where the last driver
- * entered/exited from.
- */
-
-/* BASE FUNCTIONS */
-#define POSTCODE_LINUX_A(DRIVER_PC, EVENT_PC, pc32bit, severity)	\
-do {									\
-	unsigned long long post_code_temp;				\
-	post_code_temp = (((u64)DRIVER_PC) << 56) | (((u64)EVENT_PC) << 44) | \
-		((((u64)__LINE__) & 0xFFF) << 32) |			\
-		(((u64)pc32bit) & 0xFFFFFFFF);				\
-	ISSUE_IO_VMCALL_POSTCODE_SEVERITY(post_code_temp, severity);	\
-} while (0)
-
-#define POSTCODE_LINUX_B(DRIVER_PC, EVENT_PC, pc16bit1, pc16bit2, severity) \
-do {									\
-	unsigned long long post_code_temp;				\
-	post_code_temp = (((u64)DRIVER_PC) << 56) | (((u64)EVENT_PC) << 44) | \
-		((((u64)__LINE__) & 0xFFF) << 32) |			\
-		((((u64)pc16bit1) & 0xFFFF) << 16) |			\
-		(((u64)pc16bit2) & 0xFFFF);				\
-	ISSUE_IO_VMCALL_POSTCODE_SEVERITY(post_code_temp, severity);	\
-} while (0)
-
-/* MOST COMMON */
-#define POSTCODE_LINUX_2(EVENT_PC, severity)				\
-	POSTCODE_LINUX_A(CURRENT_FILE_PC, EVENT_PC, 0x0000, severity)
-
-#define POSTCODE_LINUX_3(EVENT_PC, pc32bit, severity)			\
-	POSTCODE_LINUX_A(CURRENT_FILE_PC, EVENT_PC, pc32bit, severity)
-
-#define POSTCODE_LINUX_4(EVENT_PC, pc16bit1, pc16bit2, severity)	\
-	POSTCODE_LINUX_B(CURRENT_FILE_PC, EVENT_PC, pc16bit1,		\
-			 pc16bit2, severity)
-
-#endif
-#endif
diff --git a/drivers/staging/unisys/include/iochannel.h b/drivers/staging/unisys/include/iochannel.h
index 5ccf81485d72..cba4433bcd51 100644
--- a/drivers/staging/unisys/include/iochannel.h
+++ b/drivers/staging/unisys/include/iochannel.h
@@ -33,7 +33,6 @@
 
 #include <linux/dma-direction.h>
 #include "channel.h"
-#include "channel_guid.h"
 
 #define ULTRA_VHBA_CHANNEL_PROTOCOL_SIGNATURE ULTRA_CHANNEL_PROTOCOL_SIGNATURE
 #define ULTRA_VNIC_CHANNEL_PROTOCOL_SIGNATURE ULTRA_CHANNEL_PROTOCOL_SIGNATURE
diff --git a/drivers/staging/unisys/include/periodic_work.h b/drivers/staging/unisys/include/periodic_work.h
deleted file mode 100644
index 0b3335a4b206..000000000000
--- a/drivers/staging/unisys/include/periodic_work.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/* periodic_work.h
- *
- * Copyright (C) 2010 - 2013 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or (at
- * your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-#ifndef __PERIODIC_WORK_H__
-#define __PERIODIC_WORK_H__
-
-#include <linux/seq_file.h>
-#include <linux/slab.h>
-
-/* PERIODIC_WORK an opaque structure to users.
- * Fields are declared only in the implementation .c files.
- */
-struct periodic_work;
-
-struct periodic_work *
-visor_periodic_work_create(ulong jiffy_interval,
-			   struct workqueue_struct *workqueue,
-			   void (*workfunc)(void *),
-			   void *workfuncarg,
-			   const char *devnam);
-void visor_periodic_work_destroy(struct periodic_work *pw);
-bool visor_periodic_work_nextperiod(struct periodic_work *pw);
-bool visor_periodic_work_start(struct periodic_work *pw);
-bool visor_periodic_work_stop(struct periodic_work *pw);
-
-#endif
diff --git a/drivers/staging/unisys/include/vbushelper.h b/drivers/staging/unisys/include/vbushelper.h
deleted file mode 100644
index f1b6aacb79d7..000000000000
--- a/drivers/staging/unisys/include/vbushelper.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/* vbushelper.h
- *
- * Copyright (C) 2011 - 2013 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or (at
- * your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-#ifndef __VBUSHELPER_H__
-#define __VBUSHELPER_H__
-
-/* TARGET_HOSTNAME specified as -DTARGET_HOSTNAME=\"thename\" on the
- * command line
- */
-
-#define TARGET_HOSTNAME "linuxguest"
-
-static inline void bus_device_info_init(
-		struct ultra_vbus_deviceinfo *bus_device_info_ptr,
-		const char *dev_type, const char *drv_name,
-		const char *ver, const char *ver_tag)
-{
-	memset(bus_device_info_ptr, 0, sizeof(struct ultra_vbus_deviceinfo));
-	snprintf(bus_device_info_ptr->devtype,
-		 sizeof(bus_device_info_ptr->devtype),
-		 "%s", (dev_type) ? dev_type : "unknownType");
-	snprintf(bus_device_info_ptr->drvname,
-		 sizeof(bus_device_info_ptr->drvname),
-		 "%s", (drv_name) ? drv_name : "unknownDriver");
-	snprintf(bus_device_info_ptr->infostrs,
-		 sizeof(bus_device_info_ptr->infostrs), "%s\t%s\t%s",
-		 (ver) ? ver : "unknownVer",
-		 (ver_tag) ? ver_tag : "unknownVerTag",
-		 TARGET_HOSTNAME);
-}
-
-#endif
diff --git a/drivers/staging/unisys/include/version.h b/drivers/staging/unisys/include/version.h
deleted file mode 100644
index 83d1da7a2f81..000000000000
--- a/drivers/staging/unisys/include/version.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/* Copyright (C) 2010 - 2013 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or (at
- * your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-/* version.h */
-
-/*  Common version/release info needed by all components goes here.
- *  (This file must compile cleanly in all environments.)
- *  Ultimately, this will be combined with defines generated dynamically as
- *  part of the sysgen, and some of the defines below may in fact end up
- *  being replaced with dynamically generated ones.
- */
-#ifndef __VERSION_H__
-#define __VERSION_H__
-
-#define SPARVER1 "1"
-#define SPARVER2 "0"
-#define SPARVER3 "0"
-#define SPARVER4 "0"
-
-#define  VERSION        SPARVER1 "." SPARVER2 "." SPARVER3 "." SPARVER4
-
-/* Here are various version forms needed in Windows environments.
- */
-#define VISOR_PRODUCTVERSION      SPARVERCOMMA
-#define VISOR_PRODUCTVERSION_STR  SPARVER1 "." SPARVER2 "." SPARVER3 "." \
-	SPARVER4
-#define VISOR_OBJECTVERSION_STR   SPARVER1 "," SPARVER2 "," SPARVER3 "," \
-	SPARVER4
-
-#define  COPYRIGHT      "Unisys Corporation"
-#define  COPYRIGHTDATE  "2010 - 2013"
-
-#endif
diff --git a/drivers/staging/unisys/include/visorbus.h b/drivers/staging/unisys/include/visorbus.h
index 9baf1ec70d01..677627c72c4c 100644
--- a/drivers/staging/unisys/include/visorbus.h
+++ b/drivers/staging/unisys/include/visorbus.h
@@ -34,8 +34,9 @@
 #include <linux/poll.h>
 #include <linux/kernel.h>
 #include <linux/uuid.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
 
-#include "periodic_work.h"
 #include "channel.h"
 
 struct visor_driver;
@@ -65,8 +66,6 @@ struct visor_channeltype_descriptor {
  * struct visor_driver - Information provided by each visor driver when it
  * registers with the visorbus driver.
  * @name:		Name of the visor driver.
- * @version:		The numbered version of the driver (x.x.xxx).
- * @vertag:		A human readable version string.
  * @owner:		The module owner.
  * @channel_types:	Types of channels handled by this driver, ending with
  *			a zero GUID. Our specialized BUS.match() method knows
@@ -93,12 +92,9 @@ struct visor_channeltype_descriptor {
  * @resume:		Behaves similar to pause.
  * @driver:		Private reference to the device driver. For use by bus
  *			driver only.
- * @version_attr:	Private version field. For use by bus driver only.
  */
 struct visor_driver {
 	const char *name;
-	const char *version;
-	const char *vertag;
 	struct module *owner;
 	struct visor_channeltype_descriptor *channel_types;
 	int (*probe)(struct visor_device *dev);
@@ -111,7 +107,6 @@ struct visor_driver {
 
 	/* These fields are for private use by the bus driver only. */
 	struct device_driver driver;
-	struct driver_attribute version_attr;
 };
 
 #define to_visor_driver(x) ((x) ? \
@@ -120,33 +115,33 @@ struct visor_driver {
 /**
  * struct visor_device - A device type for things "plugged" into the visorbus
  * bus
- * visorchannel:		Points to the channel that the device is
+ * @visorchannel:		Points to the channel that the device is
  *				associated with.
- * channel_type_guid:		Identifies the channel type to the bus driver.
- * device:			Device struct meant for use by the bus driver
+ * @channel_type_guid:		Identifies the channel type to the bus driver.
+ * @device:			Device struct meant for use by the bus driver
  *				only.
- * list_all:			Used by the bus driver to enumerate devices.
- * periodic_work:		Device work queue. Private use by bus driver
- *				only.
- * being_removed:		Indicates that the device is being removed from
+ * @list_all:			Used by the bus driver to enumerate devices.
+ * @timer:		        Timer fired periodically to do interrupt-type
+ *				activity.
+ * @being_removed:		Indicates that the device is being removed from
  *				the bus. Private bus driver use only.
- * visordriver_callback_lock:	Used by the bus driver to lock when handling
+ * @visordriver_callback_lock:	Used by the bus driver to lock when handling
  *				channel events.
- * pausing:			Indicates that a change towards a paused state.
+ * @pausing:			Indicates that a change towards a paused state.
  *				is in progress. Only modified by the bus driver.
- * resuming:			Indicates that a change towards a running state
+ * @resuming:			Indicates that a change towards a running state
  *				is in progress. Only modified by the bus driver.
- * chipset_bus_no:		Private field used by the bus driver.
- * chipset_dev_no:		Private field used the bus driver.
- * state:			Used to indicate the current state of the
+ * @chipset_bus_no:		Private field used by the bus driver.
+ * @chipset_dev_no:		Private field used the bus driver.
+ * @state:			Used to indicate the current state of the
  *				device.
- * inst:			Unique GUID for this instance of the device.
- * name:			Name of the device.
- * pending_msg_hdr:		For private use by bus driver to respond to
+ * @inst:			Unique GUID for this instance of the device.
+ * @name:			Name of the device.
+ * @pending_msg_hdr:		For private use by bus driver to respond to
  *				hypervisor requests.
- * vbus_hdr_info:		A pointer to header info. Private use by bus
+ * @vbus_hdr_info:		A pointer to header info. Private use by bus
  *				driver.
- * partition_uuid:		Indicates client partion id. This should be the
+ * @partition_uuid:		Indicates client partion id. This should be the
  *				same across all visor_devices in the current
  *				guest. Private use by bus driver only.
  */
@@ -157,9 +152,10 @@ struct visor_device {
 	/* These fields are for private use by the bus driver only. */
 	struct device device;
 	struct list_head list_all;
-	struct periodic_work *periodic_work;
+	struct timer_list timer;
+	bool timer_active;
 	bool being_removed;
-	struct semaphore visordriver_callback_lock;
+	struct mutex visordriver_callback_lock;
 	bool pausing;
 	bool resuming;
 	u32 chipset_bus_no;
@@ -174,7 +170,6 @@ struct visor_device {
 
 #define to_visor_device(x) container_of(x, struct visor_device, device)
 
-#ifndef STANDALONE_CLIENT
 int visorbus_register_visor_driver(struct visor_driver *);
 void visorbus_unregister_visor_driver(struct visor_driver *);
 int visorbus_read_channel(struct visor_device *dev,
@@ -183,50 +178,34 @@ int visorbus_read_channel(struct visor_device *dev,
 int visorbus_write_channel(struct visor_device *dev,
 			   unsigned long offset, void *src,
 			   unsigned long nbytes);
-int visorbus_clear_channel(struct visor_device *dev,
-			   unsigned long offset, u8 ch, unsigned long nbytes);
 void visorbus_enable_channel_interrupts(struct visor_device *dev);
 void visorbus_disable_channel_interrupts(struct visor_device *dev);
-#endif
 
-/* Note that for visorchannel_create()
- * <channel_bytes> and <guid> arguments may be 0 if we are a channel CLIENT.
- * In this case, the values can simply be read from the channel header.
+/* Levels of severity for diagnostic events, in order from lowest severity to
+ * highest (i.e. fatal errors are the most severe, and should always be logged,
+ * but info events rarely need to be logged except during debugging). The
+ * values DIAG_SEVERITY_ENUM_BEGIN and DIAG_SEVERITY_ENUM_END are not valid
+ * severity values.  They exist merely to dilineate the list, so that future
+ * additions won't require changes to the driver (i.e. when checking for
+ * out-of-range severities in SetSeverity). The values DIAG_SEVERITY_OVERRIDE
+ * and DIAG_SEVERITY_SHUTOFF are not valid severity values for logging events
+ * but they are valid for controlling the amount of event data. Changes made
+ * to the enum, need to be reflected in s-Par.
  */
-struct visorchannel *visorchannel_create(u64 physaddr,
-					 unsigned long channel_bytes,
-					 gfp_t gfp, uuid_le guid);
-struct visorchannel *visorchannel_create_with_lock(u64 physaddr,
-						   unsigned long channel_bytes,
-						   gfp_t gfp, uuid_le guid);
-void visorchannel_destroy(struct visorchannel *channel);
-int visorchannel_read(struct visorchannel *channel, ulong offset,
-		      void *local, ulong nbytes);
-int visorchannel_write(struct visorchannel *channel, ulong offset,
-		       void *local, ulong nbytes);
-int visorchannel_clear(struct visorchannel *channel, ulong offset,
-		       u8 ch, ulong nbytes);
-bool visorchannel_signalremove(struct visorchannel *channel, u32 queue,
-			       void *msg);
-bool visorchannel_signalinsert(struct visorchannel *channel, u32 queue,
-			       void *msg);
-bool visorchannel_signalempty(struct visorchannel *channel, u32 queue);
+enum diag_severity {
+	DIAG_SEVERITY_VERBOSE = 0,
+	DIAG_SEVERITY_INFO = 1,
+	DIAG_SEVERITY_WARNING = 2,
+	DIAG_SEVERITY_ERR = 3,
+	DIAG_SEVERITY_PRINT = 4,
+};
 
-int visorchannel_signalqueue_slots_avail(struct visorchannel *channel,
-					 u32 queue);
-int visorchannel_signalqueue_max_slots(struct visorchannel *channel, u32 queue);
-u64 visorchannel_get_physaddr(struct visorchannel *channel);
-ulong visorchannel_get_nbytes(struct visorchannel *channel);
-char *visorchannel_id(struct visorchannel *channel, char *s);
-char *visorchannel_zoneid(struct visorchannel *channel, char *s);
-u64 visorchannel_get_clientpartition(struct visorchannel *channel);
-int visorchannel_set_clientpartition(struct visorchannel *channel,
-				     u64 partition_handle);
+int visorchannel_signalremove(struct visorchannel *channel, u32 queue,
+			      void *msg);
+int visorchannel_signalinsert(struct visorchannel *channel, u32 queue,
+			      void *msg);
+bool visorchannel_signalempty(struct visorchannel *channel, u32 queue);
 uuid_le visorchannel_get_uuid(struct visorchannel *channel);
-char *visorchannel_uuid_id(uuid_le *guid, char *s);
-void visorchannel_debug(struct visorchannel *channel, int num_queues,
-			struct seq_file *seq, u32 off);
-void __iomem *visorchannel_get_header(struct visorchannel *channel);
 
 #define BUS_ROOT_DEVICE		UINT_MAX
 struct visor_device *visorbus_get_device_by_id(u32 bus_no, u32 dev_no,
diff --git a/drivers/staging/unisys/visorbus/Makefile b/drivers/staging/unisys/visorbus/Makefile
index fc790e7592fc..f3730d8c953e 100644
--- a/drivers/staging/unisys/visorbus/Makefile
+++ b/drivers/staging/unisys/visorbus/Makefile
@@ -7,6 +7,5 @@ obj-$(CONFIG_UNISYS_VISORBUS)	+= visorbus.o
 visorbus-y := visorbus_main.o
 visorbus-y += visorchannel.o
 visorbus-y += visorchipset.o
-visorbus-y += periodic_work.o
 
 ccflags-y += -Idrivers/staging/unisys/include
diff --git a/drivers/staging/unisys/visorbus/controlvmchannel.h b/drivers/staging/unisys/visorbus/controlvmchannel.h
index 03e36fb6a5a0..f0bfc4ded892 100644
--- a/drivers/staging/unisys/visorbus/controlvmchannel.h
+++ b/drivers/staging/unisys/visorbus/controlvmchannel.h
@@ -482,4 +482,80 @@ struct spar_controlvm_parameters_header {
 	u32 reserved;		/* Natural alignment */
 };
 
+/* General Errors------------------------------------------------------[0-99] */
+#define CONTROLVM_RESP_SUCCESS                                  0
+#define CONTROLVM_RESP_ERROR_ALREADY_DONE                       1
+#define CONTROLVM_RESP_ERROR_IOREMAP_FAILED                     2
+#define CONTROLVM_RESP_ERROR_KMALLOC_FAILED                     3
+#define CONTROLVM_RESP_ERROR_MESSAGE_ID_UNKNOWN                 4
+#define CONTROLVM_RESP_ERROR_MESSAGE_ID_INVALID_FOR_CLIENT      5
+
+/* CONTROLVM_INIT_CHIPSET-------------------------------------------[100-199] */
+#define CONTROLVM_RESP_ERROR_CLIENT_SWITCHCOUNT_NONZERO         100
+#define CONTROLVM_RESP_ERROR_EXPECTED_CHIPSET_INIT              101
+
+/* Maximum Limit----------------------------------------------------[200-299] */
+#define CONTROLVM_RESP_ERROR_MAX_BUSES		201	/* BUS_CREATE */
+#define CONTROLVM_RESP_ERROR_MAX_DEVICES        202	/* DEVICE_CREATE */
+/* Payload and Parameter Related------------------------------------[400-499] */
+#define CONTROLVM_RESP_ERROR_PAYLOAD_INVALID	400	/* SWITCH_ATTACHEXTPORT,
+							 * DEVICE_CONFIGURE
+							 */
+#define CONTROLVM_RESP_ERROR_INITIATOR_PARAMETER_INVALID 401	/* Multiple */
+#define CONTROLVM_RESP_ERROR_TARGET_PARAMETER_INVALID 402 /* DEVICE_CONFIGURE */
+#define CONTROLVM_RESP_ERROR_CLIENT_PARAMETER_INVALID 403 /* DEVICE_CONFIGURE */
+/* Specified[Packet Structure] Value-------------------------------[500-599] */
+#define CONTROLVM_RESP_ERROR_BUS_INVALID	500	/* SWITCH_ATTACHINTPORT,
+							 * BUS_CONFIGURE,
+							 * DEVICE_CREATE,
+							 * DEVICE_CONFIG
+							 * DEVICE_DESTROY
+							 */
+#define CONTROLVM_RESP_ERROR_DEVICE_INVALID	501 /* SWITCH_ATTACHINTPORT */
+						    /* DEVICE_CREATE,
+						     * DEVICE_CONFIGURE,
+						     * DEVICE_DESTROY
+						     */
+#define CONTROLVM_RESP_ERROR_CHANNEL_INVALID	502 /* DEVICE_CREATE,
+						     * DEVICE_CONFIGURE
+						     */
+/* Partition Driver Callback Interface----------------------[600-699] */
+#define CONTROLVM_RESP_ERROR_VIRTPCI_DRIVER_FAILURE 604	/* BUS_CREATE,
+							 * BUS_DESTROY,
+							 * DEVICE_CREATE,
+							 * DEVICE_DESTROY
+							 */
+/* Unable to invoke VIRTPCI callback */
+#define CONTROLVM_RESP_ERROR_VIRTPCI_DRIVER_CALLBACK_ERROR 605
+							/* BUS_CREATE,
+							 * BUS_DESTROY,
+							 * DEVICE_CREATE,
+							 * DEVICE_DESTROY
+							 */
+/* VIRTPCI Callback returned error */
+#define CONTROLVM_RESP_ERROR_GENERIC_DRIVER_CALLBACK_ERROR 606
+							/* SWITCH_ATTACHEXTPORT,
+							 * SWITCH_DETACHEXTPORT
+							 * DEVICE_CONFIGURE
+							 */
+
+/* generic device callback returned error */
+/* Bus Related------------------------------------------------------[700-799] */
+#define CONTROLVM_RESP_ERROR_BUS_DEVICE_ATTACHED 700	/* BUS_DESTROY */
+/* Channel Related--------------------------------------------------[800-899] */
+#define CONTROLVM_RESP_ERROR_CHANNEL_TYPE_UNKNOWN 800	/* GET_CHANNELINFO,
+							 * DEVICE_DESTROY
+							 */
+#define CONTROLVM_RESP_ERROR_CHANNEL_SIZE_TOO_SMALL 801	/* DEVICE_CREATE */
+/* Chipset Shutdown Related---------------------------------------[1000-1099] */
+#define CONTROLVM_RESP_ERROR_CHIPSET_SHUTDOWN_FAILED            1000
+#define CONTROLVM_RESP_ERROR_CHIPSET_SHUTDOWN_ALREADY_ACTIVE    1001
+
+/* Chipset Stop Related-------------------------------------------[1100-1199] */
+#define CONTROLVM_RESP_ERROR_CHIPSET_STOP_FAILED_BUS            1100
+#define CONTROLVM_RESP_ERROR_CHIPSET_STOP_FAILED_SWITCH         1101
+
+/* Device Related-------------------------------------------------[1400-1499] */
+#define CONTROLVM_RESP_ERROR_DEVICE_UDEV_TIMEOUT                1400
+
 #endif				/* __CONTROLVMCHANNEL_H__ */
diff --git a/drivers/staging/unisys/visorbus/controlvmcompletionstatus.h b/drivers/staging/unisys/visorbus/controlvmcompletionstatus.h
deleted file mode 100644
index 23ad0ea6c9fc..000000000000
--- a/drivers/staging/unisys/visorbus/controlvmcompletionstatus.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/* controlvmcompletionstatus.c
- *
- * Copyright (C) 2010 - 2015 UNISYS CORPORATION
- * All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-/*  Defines for all valid values returned in the response message header
- *  completionStatus field.  See controlvmchannel.h for description of
- *  the header: _CONTROLVM_MESSAGE_HEADER.
- */
-
-#ifndef __CONTROLVMCOMPLETIONSTATUS_H__
-#define __CONTROLVMCOMPLETIONSTATUS_H__
-
-/* General Errors------------------------------------------------------[0-99] */
-#define CONTROLVM_RESP_SUCCESS                                  0
-#define CONTROLVM_RESP_ERROR_ALREADY_DONE                       1
-#define CONTROLVM_RESP_ERROR_IOREMAP_FAILED                     2
-#define CONTROLVM_RESP_ERROR_KMALLOC_FAILED                     3
-#define CONTROLVM_RESP_ERROR_MESSAGE_ID_UNKNOWN                 4
-#define CONTROLVM_RESP_ERROR_MESSAGE_ID_INVALID_FOR_CLIENT      5
-
-/* CONTROLVM_INIT_CHIPSET-------------------------------------------[100-199] */
-#define CONTROLVM_RESP_ERROR_CLIENT_SWITCHCOUNT_NONZERO         100
-#define CONTROLVM_RESP_ERROR_EXPECTED_CHIPSET_INIT              101
-
-/* Maximum Limit----------------------------------------------------[200-299] */
-#define CONTROLVM_RESP_ERROR_MAX_BUSES		201	/* BUS_CREATE */
-#define CONTROLVM_RESP_ERROR_MAX_DEVICES        202	/* DEVICE_CREATE */
-/* Payload and Parameter Related------------------------------------[400-499] */
-#define CONTROLVM_RESP_ERROR_PAYLOAD_INVALID	400	/* SWITCH_ATTACHEXTPORT,
-							 * DEVICE_CONFIGURE
-							 */
-#define CONTROLVM_RESP_ERROR_INITIATOR_PARAMETER_INVALID 401	/* Multiple */
-#define CONTROLVM_RESP_ERROR_TARGET_PARAMETER_INVALID 402 /* DEVICE_CONFIGURE */
-#define CONTROLVM_RESP_ERROR_CLIENT_PARAMETER_INVALID 403 /* DEVICE_CONFIGURE */
-/* Specified[Packet Structure] Value-------------------------------[500-599] */
-#define CONTROLVM_RESP_ERROR_BUS_INVALID	500	/* SWITCH_ATTACHINTPORT,
-							 * BUS_CONFIGURE,
-							 * DEVICE_CREATE,
-							 * DEVICE_CONFIG
-							 * DEVICE_DESTROY
-							 */
-#define CONTROLVM_RESP_ERROR_DEVICE_INVALID	501 /* SWITCH_ATTACHINTPORT */
-						    /* DEVICE_CREATE,
-						     * DEVICE_CONFIGURE,
-						     * DEVICE_DESTROY
-						     */
-#define CONTROLVM_RESP_ERROR_CHANNEL_INVALID	502 /* DEVICE_CREATE,
-						     * DEVICE_CONFIGURE
-						     */
-/* Partition Driver Callback Interface----------------------[600-699] */
-#define CONTROLVM_RESP_ERROR_VIRTPCI_DRIVER_FAILURE 604	/* BUS_CREATE,
-							 * BUS_DESTROY,
-							 * DEVICE_CREATE,
-							 * DEVICE_DESTROY
-							 */
-/* Unable to invoke VIRTPCI callback */
-#define CONTROLVM_RESP_ERROR_VIRTPCI_DRIVER_CALLBACK_ERROR 605
-							/* BUS_CREATE,
-							 * BUS_DESTROY,
-							 * DEVICE_CREATE,
-							 * DEVICE_DESTROY
-							 */
-/* VIRTPCI Callback returned error */
-#define CONTROLVM_RESP_ERROR_GENERIC_DRIVER_CALLBACK_ERROR 606
-							/* SWITCH_ATTACHEXTPORT,
-							 * SWITCH_DETACHEXTPORT
-							 * DEVICE_CONFIGURE
-							 */
-
-/* generic device callback returned error */
-/* Bus Related------------------------------------------------------[700-799] */
-#define CONTROLVM_RESP_ERROR_BUS_DEVICE_ATTACHED 700	/* BUS_DESTROY */
-/* Channel Related--------------------------------------------------[800-899] */
-#define CONTROLVM_RESP_ERROR_CHANNEL_TYPE_UNKNOWN 800	/* GET_CHANNELINFO,
-							 * DEVICE_DESTROY
-							 */
-#define CONTROLVM_RESP_ERROR_CHANNEL_SIZE_TOO_SMALL 801	/* DEVICE_CREATE */
-/* Chipset Shutdown Related---------------------------------------[1000-1099] */
-#define CONTROLVM_RESP_ERROR_CHIPSET_SHUTDOWN_FAILED            1000
-#define CONTROLVM_RESP_ERROR_CHIPSET_SHUTDOWN_ALREADY_ACTIVE    1001
-
-/* Chipset Stop Related-------------------------------------------[1100-1199] */
-#define CONTROLVM_RESP_ERROR_CHIPSET_STOP_FAILED_BUS            1100
-#define CONTROLVM_RESP_ERROR_CHIPSET_STOP_FAILED_SWITCH         1101
-
-/* Device Related-------------------------------------------------[1400-1499] */
-#define CONTROLVM_RESP_ERROR_DEVICE_UDEV_TIMEOUT                1400
-
-#endif /* __CONTROLVMCOMPLETIONSTATUS_H__ not defined */
diff --git a/drivers/staging/unisys/visorbus/iovmcall_gnuc.h b/drivers/staging/unisys/visorbus/iovmcall_gnuc.h
deleted file mode 100644
index 98ea7f381a3c..000000000000
--- a/drivers/staging/unisys/visorbus/iovmcall_gnuc.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/* Copyright (C) 2010 - 2015 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-/* Linux GCC Version (32-bit and 64-bit) */
-static inline unsigned long
-__unisys_vmcall_gnuc(unsigned long tuple, unsigned long reg_ebx,
-		     unsigned long reg_ecx)
-{
-	unsigned long result = 0;
-	unsigned int cpuid_eax, cpuid_ebx, cpuid_ecx, cpuid_edx;
-
-	cpuid(0x00000001, &cpuid_eax, &cpuid_ebx, &cpuid_ecx, &cpuid_edx);
-	if (!(cpuid_ecx & 0x80000000))
-		return -EPERM;
-
-	__asm__ __volatile__(".byte 0x00f, 0x001, 0x0c1" : "=a"(result) :
-		"a"(tuple), "b"(reg_ebx), "c"(reg_ecx));
-	return result;
-}
-
-static inline unsigned long
-__unisys_extended_vmcall_gnuc(unsigned long long tuple,
-			      unsigned long long reg_ebx,
-			      unsigned long long reg_ecx,
-			      unsigned long long reg_edx)
-{
-	unsigned long result = 0;
-	unsigned int cpuid_eax, cpuid_ebx, cpuid_ecx, cpuid_edx;
-
-	cpuid(0x00000001, &cpuid_eax, &cpuid_ebx, &cpuid_ecx, &cpuid_edx);
-	if (!(cpuid_ecx & 0x80000000))
-		return -EPERM;
-
-	__asm__ __volatile__(".byte 0x00f, 0x001, 0x0c1" : "=a"(result) :
-		"a"(tuple), "b"(reg_ebx), "c"(reg_ecx), "d"(reg_edx));
-	return result;
-}
diff --git a/drivers/staging/unisys/visorbus/periodic_work.c b/drivers/staging/unisys/visorbus/periodic_work.c
deleted file mode 100644
index 00b152764f84..000000000000
--- a/drivers/staging/unisys/visorbus/periodic_work.c
+++ /dev/null
@@ -1,204 +0,0 @@
-/* periodic_work.c
- *
- * Copyright (C) 2010 - 2015 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-/*
- *  Helper functions to schedule periodic work in Linux kernel mode.
- */
-#include <linux/sched.h>
-
-#include "periodic_work.h"
-
-#define MYDRVNAME "periodic_work"
-
-struct periodic_work {
-	rwlock_t lock;
-	struct delayed_work work;
-	void (*workfunc)(void *);
-	void *workfuncarg;
-	bool is_scheduled;
-	bool want_to_stop;
-	ulong jiffy_interval;
-	struct workqueue_struct *workqueue;
-	const char *devnam;
-};
-
-static void periodic_work_func(struct work_struct *work)
-{
-	struct periodic_work *pw;
-
-	pw = container_of(work, struct periodic_work, work.work);
-	(*pw->workfunc)(pw->workfuncarg);
-}
-
-struct periodic_work
-*visor_periodic_work_create(ulong jiffy_interval,
-			    struct workqueue_struct *workqueue,
-			    void (*workfunc)(void *),
-			    void *workfuncarg,
-			    const char *devnam)
-{
-	struct periodic_work *pw;
-
-	pw = kzalloc(sizeof(*pw), GFP_KERNEL | __GFP_NORETRY);
-	if (!pw)
-		return NULL;
-
-	rwlock_init(&pw->lock);
-	pw->jiffy_interval = jiffy_interval;
-	pw->workqueue = workqueue;
-	pw->workfunc = workfunc;
-	pw->workfuncarg = workfuncarg;
-	pw->devnam = devnam;
-	return pw;
-}
-EXPORT_SYMBOL_GPL(visor_periodic_work_create);
-
-void visor_periodic_work_destroy(struct periodic_work *pw)
-{
-	kfree(pw);
-}
-EXPORT_SYMBOL_GPL(visor_periodic_work_destroy);
-
-/** Call this from your periodic work worker function to schedule the next
- *  call.
- *  If this function returns false, there was a failure and the
- *  periodic work is no longer scheduled
- */
-bool visor_periodic_work_nextperiod(struct periodic_work *pw)
-{
-	bool rc = false;
-
-	write_lock(&pw->lock);
-	if (pw->want_to_stop) {
-		pw->is_scheduled = false;
-		pw->want_to_stop = false;
-		rc = true;  /* yes, true; see visor_periodic_work_stop() */
-		goto unlock;
-	} else if (!queue_delayed_work(pw->workqueue, &pw->work,
-				       pw->jiffy_interval)) {
-		pw->is_scheduled = false;
-		rc = false;
-		goto unlock;
-	}
-	rc = true;
-unlock:
-	write_unlock(&pw->lock);
-	return rc;
-}
-EXPORT_SYMBOL_GPL(visor_periodic_work_nextperiod);
-
-/** This function returns true iff new periodic work was actually started.
- *  If this function returns false, then no work was started
- *  (either because it was already started, or because of a failure).
- */
-bool visor_periodic_work_start(struct periodic_work *pw)
-{
-	bool rc = false;
-
-	write_lock(&pw->lock);
-	if (pw->is_scheduled) {
-		rc = false;
-		goto unlock;
-	}
-	if (pw->want_to_stop) {
-		rc = false;
-		goto unlock;
-	}
-	INIT_DELAYED_WORK(&pw->work, &periodic_work_func);
-	if (!queue_delayed_work(pw->workqueue, &pw->work,
-				pw->jiffy_interval)) {
-		rc = false;
-		goto unlock;
-	}
-	pw->is_scheduled = true;
-	rc = true;
-unlock:
-	write_unlock(&pw->lock);
-	return rc;
-}
-EXPORT_SYMBOL_GPL(visor_periodic_work_start);
-
-/** This function returns true iff your call actually stopped the periodic
- *  work.
- *
- *  -- PAY ATTENTION... this is important --
- *
- *  NO NO #1
- *
- *     Do NOT call this function from some function that is running on the
- *     same workqueue as the work you are trying to stop might be running
- *     on!  If you violate this rule, visor_periodic_work_stop() MIGHT work,
- *     but it also MIGHT get hung up in an infinite loop saying
- *     "waiting for delayed work...".  This will happen if the delayed work
- *     you are trying to cancel has been put in the workqueue list, but can't
- *     run yet because we are running that same workqueue thread right now.
- *
- *     Bottom line: If you need to call visor_periodic_work_stop() from a
- *     workitem, be sure the workitem is on a DIFFERENT workqueue than the
- *     workitem that you are trying to cancel.
- *
- *     If I could figure out some way to check for this "no no" condition in
- *     the code, I would.  It would have saved me the trouble of writing this
- *     long comment.  And also, don't think this is some "theoretical" race
- *     condition.  It is REAL, as I have spent the day chasing it.
- *
- *  NO NO #2
- *
- *     Take close note of the locks that you own when you call this function.
- *     You must NOT own any locks that are needed by the periodic work
- *     function that is currently installed.  If you DO, a deadlock may result,
- *     because stopping the periodic work often involves waiting for the last
- *     iteration of the periodic work function to complete.  Again, if you hit
- *     this deadlock, you will get hung up in an infinite loop saying
- *     "waiting for delayed work...".
- */
-bool visor_periodic_work_stop(struct periodic_work *pw)
-{
-	bool stopped_something = false;
-
-	write_lock(&pw->lock);
-	stopped_something = pw->is_scheduled && (!pw->want_to_stop);
-	while (pw->is_scheduled) {
-		pw->want_to_stop = true;
-		if (cancel_delayed_work(&pw->work)) {
-			/* We get here if the delayed work was pending as
-			 * delayed work, but was NOT run.
-			 */
-			WARN_ON(!pw->is_scheduled);
-			pw->is_scheduled = false;
-		} else {
-			/* If we get here, either the delayed work:
-			 * - was run, OR,
-			 * - is running RIGHT NOW on another processor, OR,
-			 * - wasn't even scheduled (there is a miniscule
-			 *   timing window where this could be the case)
-			 * flush_workqueue() would make sure it is finished
-			 * executing, but that still isn't very useful, which
-			 * explains the loop...
-			 */
-		}
-		if (pw->is_scheduled) {
-			write_unlock(&pw->lock);
-			schedule_timeout_interruptible(msecs_to_jiffies(10));
-			write_lock(&pw->lock);
-		} else {
-			pw->want_to_stop = false;
-		}
-	}
-	write_unlock(&pw->lock);
-	return stopped_something;
-}
-EXPORT_SYMBOL_GPL(visor_periodic_work_stop);
diff --git a/drivers/staging/unisys/visorbus/vbuschannel.h b/drivers/staging/unisys/visorbus/vbuschannel.h
index 90fa12e62f26..e97917522f6a 100644
--- a/drivers/staging/unisys/visorbus/vbuschannel.h
+++ b/drivers/staging/unisys/visorbus/vbuschannel.h
@@ -23,7 +23,6 @@
  *  the client devices and client drivers for the server end to see.
  */
 #include <linux/uuid.h>
-#include "vbusdeviceinfo.h"
 #include "channel.h"
 
 /* {193b331b-c58f-11da-95a9-00e08161165f} */
@@ -58,6 +57,216 @@ static const uuid_le spar_vbus_channel_protocol_uuid =
 				   actual_bytes))
 
 #pragma pack(push, 1)		/* both GCC and VC now allow this pragma */
+
+/*
+ * An array of this struct is present in the channel area for each vbus.
+ * (See vbuschannel.h.)
+ * It is filled in by the client side to provide info about the device
+ * and driver from the client's perspective.
+ */
+struct ultra_vbus_deviceinfo {
+	u8 devtype[16];		/* short string identifying the device type */
+	u8 drvname[16];		/* driver .sys file name */
+	u8 infostrs[96];	/* kernel version */
+	u8 reserved[128];	/* pad size to 256 bytes */
+};
+
+/**
+ * vbuschannel_sanitize_buffer() - remove non-printable chars from buffer
+ * @p: destination buffer where chars are written to
+ * @remain: number of bytes that can be written starting at #p
+ * @src: pointer to source buffer
+ * @srcmax: number of valid characters at #src
+ *
+ * Reads chars from the buffer at @src for @srcmax bytes, and writes to
+ * the buffer at @p, which is @remain bytes long, ensuring never to
+ * overflow the buffer at @p, using the following rules:
+ * - printable characters are simply copied from the buffer at @src to the
+ *   buffer at @p
+ * - intervening streaks of non-printable characters in the buffer at @src
+ *   are replaced with a single space in the buffer at @p
+ * Note that we pay no attention to '\0'-termination.
+ *
+ * Pass @p == NULL and @remain == 0 for this special behavior -- In this
+ * case, we simply return the number of bytes that WOULD HAVE been written
+ * to a buffer at @p, had it been infinitely big.
+ *
+ * Return: the number of bytes written to @p (or WOULD HAVE been written to
+ *         @p, as described in the previous paragraph)
+ */
+static inline int
+vbuschannel_sanitize_buffer(char *p, int remain, char *src, int srcmax)
+{
+	int chars = 0;
+	int nonprintable_streak = 0;
+
+	while (srcmax > 0) {
+		if ((*src >= ' ') && (*src < 0x7f)) {
+			if (nonprintable_streak) {
+				if (remain > 0) {
+					*p = ' ';
+					p++;
+					remain--;
+					chars++;
+				} else if (!p) {
+					chars++;
+				}
+				nonprintable_streak = 0;
+			}
+			if (remain > 0) {
+				*p = *src;
+				p++;
+				remain--;
+				chars++;
+			} else if (!p) {
+				chars++;
+			}
+		} else {
+			nonprintable_streak = 1;
+		}
+		src++;
+		srcmax--;
+	}
+	return chars;
+}
+
+#define VBUSCHANNEL_ADDACHAR(ch, p, remain, chars) \
+	do {					   \
+		if (remain <= 0)		   \
+			break;			   \
+		*p = ch;			   \
+		p++;  chars++;  remain--;	   \
+	} while (0)
+
+/**
+ * vbuschannel_itoa() - convert non-negative int to string
+ * @p: destination string
+ * @remain: max number of bytes that can be written to @p
+ * @num: input int to convert
+ *
+ * Converts the non-negative value at @num to an ascii decimal string
+ * at @p, writing at most @remain bytes.  Note there is NO '\0' termination
+ * written to @p.
+ *
+ * Return: number of bytes written to @p
+ *
+ */
+static inline int
+vbuschannel_itoa(char *p, int remain, int num)
+{
+	int digits = 0;
+	char s[32];
+	int i;
+
+	if (num == 0) {
+		/* '0' is a special case */
+		if (remain <= 0)
+			return 0;
+		*p = '0';
+		return 1;
+	}
+	/* form a backwards decimal ascii string in <s> */
+	while (num > 0) {
+		if (digits >= (int)sizeof(s))
+			return 0;
+		s[digits++] = (num % 10) + '0';
+		num = num / 10;
+	}
+	if (remain < digits) {
+		/* not enough room left at <p> to hold number, so fill with
+		 * '?'
+		 */
+		for (i = 0; i < remain; i++, p++)
+			*p = '?';
+		return remain;
+	}
+	/* plug in the decimal ascii string representing the number, by */
+	/* reversing the string we just built in <s> */
+	i = digits;
+	while (i > 0) {
+		i--;
+		*p = s[i];
+		p++;
+	}
+	return digits;
+}
+
+/**
+ * vbuschannel_devinfo_to_string() - format a struct ultra_vbus_deviceinfo
+ *                                   to a printable string
+ * @devinfo: the struct ultra_vbus_deviceinfo to format
+ * @p: destination string area
+ * @remain: size of destination string area in bytes
+ * @devix: the device index to be included in the output data, or -1 if no
+ *         device index is to be included
+ *
+ * Reads @devInfo, and converts its contents to a printable string at @p,
+ * writing at most @remain bytes. Note there is NO '\0' termination
+ * written to @p.
+ *
+ * Return: number of bytes written to @p
+ */
+static inline int
+vbuschannel_devinfo_to_string(struct ultra_vbus_deviceinfo *devinfo,
+			      char *p, int remain, int devix)
+{
+	char *psrc;
+	int nsrc, x, i, pad;
+	int chars = 0;
+
+	psrc = &devinfo->devtype[0];
+	nsrc = sizeof(devinfo->devtype);
+	if (vbuschannel_sanitize_buffer(NULL, 0, psrc, nsrc) <= 0)
+		return 0;
+
+	/* emit device index */
+	if (devix >= 0) {
+		VBUSCHANNEL_ADDACHAR('[', p, remain, chars);
+		x = vbuschannel_itoa(p, remain, devix);
+		p += x;
+		remain -= x;
+		chars += x;
+		VBUSCHANNEL_ADDACHAR(']', p, remain, chars);
+	} else {
+		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
+		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
+		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
+	}
+
+	/* emit device type */
+	x = vbuschannel_sanitize_buffer(p, remain, psrc, nsrc);
+	p += x;
+	remain -= x;
+	chars += x;
+	pad = 15 - x;		/* pad device type to be exactly 15 chars */
+	for (i = 0; i < pad; i++)
+		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
+	VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
+
+	/* emit driver name */
+	psrc = &devinfo->drvname[0];
+	nsrc = sizeof(devinfo->drvname);
+	x = vbuschannel_sanitize_buffer(p, remain, psrc, nsrc);
+	p += x;
+	remain -= x;
+	chars += x;
+	pad = 15 - x;		/* pad driver name to be exactly 15 chars */
+	for (i = 0; i < pad; i++)
+		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
+	VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
+
+	/* emit strings */
+	psrc = &devinfo->infostrs[0];
+	nsrc = sizeof(devinfo->infostrs);
+	x = vbuschannel_sanitize_buffer(p, remain, psrc, nsrc);
+	p += x;
+	remain -= x;
+	chars += x;
+	VBUSCHANNEL_ADDACHAR('\n', p, remain, chars);
+
+	return chars;
+}
+
 struct spar_vbus_headerinfo {
 	u32 struct_bytes;	/* size of this struct in bytes */
 	u32 device_info_struct_bytes;	/* sizeof(ULTRA_VBUS_DEVICEINFO) */
diff --git a/drivers/staging/unisys/visorbus/vbusdeviceinfo.h b/drivers/staging/unisys/visorbus/vbusdeviceinfo.h
deleted file mode 100644
index abdab4ad0b36..000000000000
--- a/drivers/staging/unisys/visorbus/vbusdeviceinfo.h
+++ /dev/null
@@ -1,213 +0,0 @@
-/* Copyright (C) 2010 - 2015 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-#ifndef __VBUSDEVICEINFO_H__
-#define __VBUSDEVICEINFO_H__
-
-#include <linux/types.h>
-
-#pragma pack(push, 1)		/* both GCC and VC now allow this pragma */
-
-/* An array of this struct is present in the channel area for each vbus.
- * (See vbuschannel.h.)
- * It is filled in by the client side to provide info about the device
- * and driver from the client's perspective.
- */
-struct ultra_vbus_deviceinfo {
-	u8 devtype[16];		/* short string identifying the device type */
-	u8 drvname[16];		/* driver .sys file name */
-	u8 infostrs[96];	/* sequence of tab-delimited id strings: */
-	/* <DRIVER_REV> <DRIVER_VERTAG> <DRIVER_COMPILETIME> */
-	u8 reserved[128];	/* pad size to 256 bytes */
-};
-
-#pragma pack(pop)
-
-/* Reads chars from the buffer at <src> for <srcmax> bytes, and writes to
- * the buffer at <p>, which is <remain> bytes long, ensuring never to
- * overflow the buffer at <p>, using the following rules:
- * - printable characters are simply copied from the buffer at <src> to the
- *   buffer at <p>
- * - intervening streaks of non-printable characters in the buffer at <src>
- *   are replaced with a single space in the buffer at <p>
- * Note that we pay no attention to '\0'-termination.
- * Returns the number of bytes written to <p>.
- *
- * Pass <p> == NULL and <remain> == 0 for this special behavior.  In this
- * case, we simply return the number of bytes that WOULD HAVE been written
- * to a buffer at <p>, had it been infinitely big.
- */
-static inline int
-vbuschannel_sanitize_buffer(char *p, int remain, char *src, int srcmax)
-{
-	int chars = 0;
-	int nonprintable_streak = 0;
-
-	while (srcmax > 0) {
-		if ((*src >= ' ') && (*src < 0x7f)) {
-			if (nonprintable_streak) {
-				if (remain > 0) {
-					*p = ' ';
-					p++;
-					remain--;
-					chars++;
-				} else if (!p) {
-					chars++;
-				}
-				nonprintable_streak = 0;
-			}
-			if (remain > 0) {
-				*p = *src;
-				p++;
-				remain--;
-				chars++;
-			} else if (!p) {
-				chars++;
-			}
-		} else {
-			nonprintable_streak = 1;
-		}
-		src++;
-		srcmax--;
-	}
-	return chars;
-}
-
-#define VBUSCHANNEL_ADDACHAR(ch, p, remain, chars) \
-	do {					   \
-		if (remain <= 0)		   \
-			break;			   \
-		*p = ch;			   \
-		p++;  chars++;  remain--;	   \
-	} while (0)
-
-/* Converts the non-negative value at <num> to an ascii decimal string
- * at <p>, writing at most <remain> bytes.  Note there is NO '\0' termination
- * written to <p>.
- *
- * Returns the number of bytes written to <p>.
- *
- * Note that we create this function because we need to do this operation in
- * an environment-independent way (since we are in a common header file).
- */
-static inline int
-vbuschannel_itoa(char *p, int remain, int num)
-{
-	int digits = 0;
-	char s[32];
-	int i;
-
-	if (num == 0) {
-		/* '0' is a special case */
-		if (remain <= 0)
-			return 0;
-		*p = '0';
-		return 1;
-	}
-	/* form a backwards decimal ascii string in <s> */
-	while (num > 0) {
-		if (digits >= (int)sizeof(s))
-			return 0;
-		s[digits++] = (num % 10) + '0';
-		num = num / 10;
-	}
-	if (remain < digits) {
-		/* not enough room left at <p> to hold number, so fill with
-		 * '?'
-		 */
-		for (i = 0; i < remain; i++, p++)
-			*p = '?';
-		return remain;
-	}
-	/* plug in the decimal ascii string representing the number, by */
-	/* reversing the string we just built in <s> */
-	i = digits;
-	while (i > 0) {
-		i--;
-		*p = s[i];
-		p++;
-	}
-	return digits;
-}
-
-/* Reads <devInfo>, and converts its contents to a printable string at <p>,
- * writing at most <remain> bytes.  Note there is NO '\0' termination
- * written to <p>.
- *
- * Pass <devix> >= 0 if you want a device index presented.
- *
- * Returns the number of bytes written to <p>.
- */
-static inline int
-vbuschannel_devinfo_to_string(struct ultra_vbus_deviceinfo *devinfo,
-			      char *p, int remain, int devix)
-{
-	char *psrc;
-	int nsrc, x, i, pad;
-	int chars = 0;
-
-	psrc = &devinfo->devtype[0];
-	nsrc = sizeof(devinfo->devtype);
-	if (vbuschannel_sanitize_buffer(NULL, 0, psrc, nsrc) <= 0)
-		return 0;
-
-	/* emit device index */
-	if (devix >= 0) {
-		VBUSCHANNEL_ADDACHAR('[', p, remain, chars);
-		x = vbuschannel_itoa(p, remain, devix);
-		p += x;
-		remain -= x;
-		chars += x;
-		VBUSCHANNEL_ADDACHAR(']', p, remain, chars);
-	} else {
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-	}
-
-	/* emit device type */
-	x = vbuschannel_sanitize_buffer(p, remain, psrc, nsrc);
-	p += x;
-	remain -= x;
-	chars += x;
-	pad = 15 - x;		/* pad device type to be exactly 15 chars */
-	for (i = 0; i < pad; i++)
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-	VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-
-	/* emit driver name */
-	psrc = &devinfo->drvname[0];
-	nsrc = sizeof(devinfo->drvname);
-	x = vbuschannel_sanitize_buffer(p, remain, psrc, nsrc);
-	p += x;
-	remain -= x;
-	chars += x;
-	pad = 15 - x;		/* pad driver name to be exactly 15 chars */
-	for (i = 0; i < pad; i++)
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-	VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-
-	/* emit strings */
-	psrc = &devinfo->infostrs[0];
-	nsrc = sizeof(devinfo->infostrs);
-	x = vbuschannel_sanitize_buffer(p, remain, psrc, nsrc);
-	p += x;
-	remain -= x;
-	chars += x;
-	VBUSCHANNEL_ADDACHAR('\n', p, remain, chars);
-
-	return chars;
-}
-
-#endif
diff --git a/drivers/staging/unisys/visorbus/visorbus_main.c b/drivers/staging/unisys/visorbus/visorbus_main.c
index d32b8980a1cf..fec0a54916fe 100644
--- a/drivers/staging/unisys/visorbus/visorbus_main.c
+++ b/drivers/staging/unisys/visorbus/visorbus_main.c
@@ -18,62 +18,22 @@
 
 #include "visorbus.h"
 #include "visorbus_private.h"
-#include "version.h"
-#include "periodic_work.h"
-#include "vbuschannel.h"
-#include "guestlinuxdebug.h"
 #include "vmcallinterface.h"
 
 #define MYDRVNAME "visorbus"
 
 /* module parameters */
-static int visorbus_debug;
 static int visorbus_forcematch;
 static int visorbus_forcenomatch;
-static int visorbus_debugref;
-#define SERIALLOOPBACKCHANADDR (100 * 1024 * 1024)
 
 /* Display string that is guaranteed to be no longer the 99 characters*/
 #define LINESIZE 99
 
 #define CURRENT_FILE_PC VISOR_BUS_PC_visorbus_main_c
-#define POLLJIFFIES_TESTWORK         100
 #define POLLJIFFIES_NORMALCHANNEL     10
 
 static int busreg_rc = -ENODEV; /* stores the result from bus registration */
 
-static int visorbus_uevent(struct device *xdev, struct kobj_uevent_env *env);
-static int visorbus_match(struct device *xdev, struct device_driver *xdrv);
-static void fix_vbus_dev_info(struct visor_device *visordev);
-
-/*  BUS type attributes
- *
- *  define & implement display of bus attributes under
- *  /sys/bus/visorbus.
- *
- */
-
-static ssize_t version_show(struct bus_type *bus, char *buf)
-{
-	return snprintf(buf, PAGE_SIZE, "%s\n", VERSION);
-}
-
-static BUS_ATTR_RO(version);
-
-static struct attribute *visorbus_bus_attrs[] = {
-	&bus_attr_version.attr,
-	NULL,
-};
-
-static const struct attribute_group visorbus_bus_group = {
-	.attrs = visorbus_bus_attrs,
-};
-
-static const struct attribute_group *visorbus_bus_groups[] = {
-	&visorbus_bus_group,
-	NULL,
-};
-
 /*
  * DEVICE type attributes
  *
@@ -106,61 +66,14 @@ static const struct attribute_group *visorbus_dev_groups[] = {
 	NULL,
 };
 
-/** This describes the TYPE of bus.
- *  (Don't confuse this with an INSTANCE of the bus.)
- */
-struct bus_type visorbus_type = {
-	.name = "visorbus",
-	.match = visorbus_match,
-	.uevent = visorbus_uevent,
-	.dev_groups = visorbus_dev_groups,
-	.bus_groups = visorbus_bus_groups,
-};
-
-static struct delayed_work periodic_work;
-
-/* YES, we need 2 workqueues.
- * The reason is, workitems on the test queue may need to cancel
- * workitems on the other queue.  You will be in for trouble if you try to
- * do this with workitems queued on the same workqueue.
- */
-static struct workqueue_struct *periodic_test_workqueue;
-static struct workqueue_struct *periodic_dev_workqueue;
-static long long bus_count;	/** number of bus instances */
-					/** ever-increasing */
-
-static void chipset_bus_create(struct visor_device *bus_info);
-static void chipset_bus_destroy(struct visor_device *bus_info);
-static void chipset_device_create(struct visor_device *dev_info);
-static void chipset_device_destroy(struct visor_device *dev_info);
-static void chipset_device_pause(struct visor_device *dev_info);
-static void chipset_device_resume(struct visor_device *dev_info);
-
-/** These functions are implemented herein, and are called by the chipset
- *  driver to notify us about specific events.
- */
-static struct visorchipset_busdev_notifiers chipset_notifiers = {
-	.bus_create = chipset_bus_create,
-	.bus_destroy = chipset_bus_destroy,
-	.device_create = chipset_device_create,
-	.device_destroy = chipset_device_destroy,
-	.device_pause = chipset_device_pause,
-	.device_resume = chipset_device_resume,
-};
-
-/** These functions are implemented in the chipset driver, and we call them
- *  herein when we want to acknowledge a specific event.
- */
-static struct visorchipset_busdev_responders chipset_responders;
-
 /* filled in with info about parent chipset driver when we register with it */
 static struct ultra_vbus_deviceinfo chipset_driverinfo;
 /* filled in with info about this driver, wrt it servicing client busses */
 static struct ultra_vbus_deviceinfo clientbus_driverinfo;
 
-/** list of visor_device structs, linked via .list_all */
+/* list of visor_device structs, linked via .list_all */
 static LIST_HEAD(list_all_bus_instances);
-/** list of visor_device structs, linked via .list_all */
+/* list of visor_device structs, linked via .list_all */
 static LIST_HEAD(list_all_device_instances);
 
 static int
@@ -177,9 +90,14 @@ visorbus_uevent(struct device *xdev, struct kobj_uevent_env *env)
 	return 0;
 }
 
-/* This is called automatically upon adding a visor_device (device_add), or
- * adding a visor_driver (visorbus_register_visor_driver), and returns 1 iff the
- * provided driver can control the specified device.
+/**
+ * visorbus_match() - called automatically upon adding a visor_device
+ *                    (device_add), or adding a visor_driver
+ *                    (visorbus_register_visor_driver)
+ * @xdev: struct device for the device being matched
+ * @xdrv: struct device_driver for driver to match device against
+ *
+ * Return: 1 iff the provided driver can control the specified device
  */
 static int
 visorbus_match(struct device *xdev, struct device_driver *xdrv)
@@ -211,9 +129,22 @@ visorbus_match(struct device *xdev, struct device_driver *xdrv)
 	return 0;
 }
 
-/** This is called when device_unregister() is called for the bus device
- *  instance, after all other tasks involved with destroying the device
- *  are complete.
+/*
+ * This describes the TYPE of bus.
+ *  (Don't confuse this with an INSTANCE of the bus.)
+ */
+struct bus_type visorbus_type = {
+	.name = "visorbus",
+	.match = visorbus_match,
+	.uevent = visorbus_uevent,
+	.dev_groups = visorbus_dev_groups,
+};
+
+/**
+ * visorbus_releae_busdevice() - called when device_unregister() is called for
+ *                               the bus device instance, after all other tasks
+ *                               involved with destroying the dev are complete
+ * @xdev: struct device for the bus being released
  */
 static void
 visorbus_release_busdevice(struct device *xdev)
@@ -223,18 +154,16 @@ visorbus_release_busdevice(struct device *xdev)
 	kfree(dev);
 }
 
-/** This is called when device_unregister() is called for each child
- *  device instance.
+/**
+ * visorbus_release_device() - called when device_unregister() is called for
+ *                             each child device instance
+ * @xdev: struct device for the visor device being released
  */
 static void
 visorbus_release_device(struct device *xdev)
 {
 	struct visor_device *dev = to_visor_device(xdev);
 
-	if (dev->periodic_work) {
-		visor_periodic_work_destroy(dev->periodic_work);
-		dev->periodic_work = NULL;
-	}
 	if (dev->visorchannel) {
 		visorchannel_destroy(dev->visorchannel);
 		dev->visorchannel = NULL;
@@ -242,9 +171,11 @@ visorbus_release_device(struct device *xdev)
 	kfree(dev);
 }
 
-/* begin implementation of specific channel attributes to appear under
-* /sys/bus/visorbus<x>/dev<y>/channel
-*/
+/*
+ * begin implementation of specific channel attributes to appear under
+ * /sys/bus/visorbus<x>/dev<y>/channel
+ */
+
 static ssize_t physaddr_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
 {
@@ -349,15 +280,11 @@ static const struct attribute_group *visorbus_channel_groups[] = {
 
 /* end implementation of specific channel attributes */
 
-/*  BUS instance attributes
+/*
+ *  BUS instance attributes
  *
  *  define & implement display of bus attributes under
- *  /sys/bus/visorbus/busses/visorbus<n>.
- *
- *  This is a bit hoaky because the kernel does not yet have the infrastructure
- *  to separate bus INSTANCE attributes from bus TYPE attributes...
- *  so we roll our own.  See businst.c / businst.h.
- *
+ *  /sys/bus/visorbus/devices/visorbus<n>.
  */
 
 static ssize_t partition_handle_show(struct device *dev,
@@ -434,8 +361,8 @@ static ssize_t client_bus_info_show(struct device *dev,
 		if (vdev->name)
 			partition_name = vdev->name;
 		shift = snprintf(pos, remain,
-				 "Client device / client driver info for %s eartition (vbus #%d):\n",
-				 partition_name, vdev->chipset_dev_no);
+				 "Client device / client driver info for %s partition (vbus #%u):\n",
+				 partition_name, vdev->chipset_bus_no);
 		pos += shift;
 		remain -= shift;
 		shift = visorchannel_read(channel,
@@ -508,103 +435,48 @@ static const struct attribute_group *visorbus_groups[] = {
 		NULL
 };
 
-/*  DRIVER attributes
- *
- *  define & implement display of driver attributes under
- *  /sys/bus/visorbus/drivers/<drivername>.
- *
- */
-
-static ssize_t
-DRIVER_ATTR_version(struct device_driver *xdrv, char *buf)
-{
-	struct visor_driver *drv = to_visor_driver(xdrv);
-
-	return snprintf(buf, PAGE_SIZE, "%s\n", drv->version);
-}
-
-static int
-register_driver_attributes(struct visor_driver *drv)
-{
-	struct driver_attribute version =
-	    __ATTR(version, S_IRUGO, DRIVER_ATTR_version, NULL);
-	drv->version_attr = version;
-	return driver_create_file(&drv->driver, &drv->version_attr);
-}
-
-static void
-unregister_driver_attributes(struct visor_driver *drv)
-{
-	driver_remove_file(&drv->driver, &drv->version_attr);
-}
-
 static void
-dev_periodic_work(void *xdev)
+dev_periodic_work(unsigned long __opaque)
 {
-	struct visor_device *dev = xdev;
+	struct visor_device *dev = (struct visor_device *)__opaque;
 	struct visor_driver *drv = to_visor_driver(dev->device.driver);
 
-	down(&dev->visordriver_callback_lock);
 	if (drv->channel_interrupt)
 		drv->channel_interrupt(dev);
-	up(&dev->visordriver_callback_lock);
-	if (!visor_periodic_work_nextperiod(dev->periodic_work))
-		put_device(&dev->device);
+	mod_timer(&dev->timer, jiffies + POLLJIFFIES_NORMALCHANNEL);
 }
 
 static void
 dev_start_periodic_work(struct visor_device *dev)
 {
-	if (dev->being_removed)
+	if (dev->being_removed || dev->timer_active)
 		return;
 	/* now up by at least 2 */
 	get_device(&dev->device);
-	if (!visor_periodic_work_start(dev->periodic_work))
-		put_device(&dev->device);
+	dev->timer.expires = jiffies + POLLJIFFIES_NORMALCHANNEL;
+	add_timer(&dev->timer);
+	dev->timer_active = true;
 }
 
 static void
 dev_stop_periodic_work(struct visor_device *dev)
 {
-	if (visor_periodic_work_stop(dev->periodic_work))
-		put_device(&dev->device);
-}
-
-/** This is called automatically upon adding a visor_device (device_add), or
- *  adding a visor_driver (visorbus_register_visor_driver), but only after
- *  visorbus_match has returned 1 to indicate a successful match between
- *  driver and device.
- */
-static int
-visordriver_probe_device(struct device *xdev)
-{
-	int res;
-	struct visor_driver *drv;
-	struct visor_device *dev;
-
-	drv = to_visor_driver(xdev->driver);
-	dev = to_visor_device(xdev);
-
-	if (!drv->probe)
-		return -ENODEV;
-
-	down(&dev->visordriver_callback_lock);
-	dev->being_removed = false;
-
-	res = drv->probe(dev);
-	if (res >= 0) {
-		/* success: reference kept via unmatched get_device() */
-		get_device(&dev->device);
-		fix_vbus_dev_info(dev);
-	}
-
-	up(&dev->visordriver_callback_lock);
-	return res;
+	if (!dev->timer_active)
+		return;
+	del_timer_sync(&dev->timer);
+	dev->timer_active = false;
+	put_device(&dev->device);
 }
 
-/** This is called when device_unregister() is called for each child device
- *  instance, to notify the appropriate visorbus_driver that the device is
- *  going away, and to decrease the reference count of the device.
+/**
+ * visordriver_remove_device() - handle visor device going away
+ * @xdev: struct device for the visor device being removed
+ *
+ * This is called when device_unregister() is called for each child device
+ * instance, to notify the appropriate visorbus function driver that the device
+ * is going away, and to decrease the reference count of the device.
+ *
+ * Return: 0 iff successful
  */
 static int
 visordriver_remove_device(struct device *xdev)
@@ -614,105 +486,44 @@ visordriver_remove_device(struct device *xdev)
 
 	dev = to_visor_device(xdev);
 	drv = to_visor_driver(xdev->driver);
-	down(&dev->visordriver_callback_lock);
+	mutex_lock(&dev->visordriver_callback_lock);
 	dev->being_removed = true;
 	if (drv->remove)
 		drv->remove(dev);
-	up(&dev->visordriver_callback_lock);
+	mutex_unlock(&dev->visordriver_callback_lock);
 	dev_stop_periodic_work(dev);
 
 	put_device(&dev->device);
 	return 0;
 }
 
-/** A particular type of visor driver calls this function to register
- *  the driver.  The caller MUST fill in the following fields within the
- *  #drv structure:
- *      name, version, owner, channel_types, probe, remove
+/**
+ * visorbus_unregister_visor_driver() - unregisters the provided driver
+ * @drv: the driver to unregister
  *
- *  Here's how the whole Linux bus / driver / device model works.
- *
- *  At system start-up, the visorbus kernel module is loaded, which registers
- *  visorbus_type as a bus type, using bus_register().
- *
- *  All kernel modules that support particular device types on a
- *  visorbus bus are loaded.  Each of these kernel modules calls
- *  visorbus_register_visor_driver() in their init functions, passing a
- *  visor_driver struct.  visorbus_register_visor_driver() in turn calls
- *  register_driver(&visor_driver.driver).  This .driver member is
- *  initialized with generic methods (like probe), whose sole responsibility
- *  is to act as a broker for the real methods, which are within the
- *  visor_driver struct.  (This is the way the subclass behavior is
- *  implemented, since visor_driver is essentially a subclass of the
- *  generic driver.)  Whenever a driver_register() happens, core bus code in
- *  the kernel does (see device_attach() in drivers/base/dd.c):
- *
- *      for each dev associated with the bus (the bus that driver is on) that
- *      does not yet have a driver
- *          if bus.match(dev,newdriver) == yes_matched  ** .match specified
- *                                                 ** during bus_register().
- *              newdriver.probe(dev)  ** for visor drivers, this will call
- *                    ** the generic driver.probe implemented in visorbus.c,
- *                    ** which in turn calls the probe specified within the
- *                    ** struct visor_driver (which was specified by the
- *                    ** actual device driver as part of
- *                    ** visorbus_register_visor_driver()).
- *
- *  The above dance also happens when a new device appears.
- *  So the question is, how are devices created within the system?
- *  Basically, just call device_add(dev).  See pci_bus_add_devices().
- *  pci_scan_device() shows an example of how to build a device struct.  It
- *  returns the newly-created struct to pci_scan_single_device(), who adds it
- *  to the list of devices at PCIBUS.devices.  That list of devices is what
- *  is traversed by pci_bus_add_devices().
- *
- */
-int visorbus_register_visor_driver(struct visor_driver *drv)
-{
-	int rc = 0;
-
-	if (busreg_rc < 0)
-		return -ENODEV; /*can't register on a nonexistent bus*/
-
-	drv->driver.name = drv->name;
-	drv->driver.bus = &visorbus_type;
-	drv->driver.probe = visordriver_probe_device;
-	drv->driver.remove = visordriver_remove_device;
-	drv->driver.owner = drv->owner;
-
-	/* driver_register does this:
-	 *   bus_add_driver(drv)
-	 *   ->if (drv.bus)  ** (bus_type) **
-	 *       driver_attach(drv)
-	 *         for each dev with bus type of drv.bus
-	 *           if (!dev.drv)  ** no driver assigned yet **
-	 *             if (bus.match(dev,drv))  [visorbus_match]
-	 *               dev.drv = drv
-	 *               if (!drv.probe(dev))   [visordriver_probe_device]
-	 *                 dev.drv = NULL
-	 */
-
-	rc = driver_register(&drv->driver);
-	if (rc < 0)
-		return rc;
-	rc = register_driver_attributes(drv);
-	if (rc < 0)
-		driver_unregister(&drv->driver);
-	return rc;
-}
-EXPORT_SYMBOL_GPL(visorbus_register_visor_driver);
-
-/** A particular type of visor driver calls this function to unregister
- *  the driver, i.e., within its module_exit function.
+ * A visor function driver calls this function to unregister the driver,
+ * i.e., within its module_exit function.
  */
 void
 visorbus_unregister_visor_driver(struct visor_driver *drv)
 {
-	unregister_driver_attributes(drv);
 	driver_unregister(&drv->driver);
 }
 EXPORT_SYMBOL_GPL(visorbus_unregister_visor_driver);
 
+/**
+ * visorbus_read_channel() - reads from the designated channel into
+ *                           the provided buffer
+ * @dev:    the device whose channel is read from
+ * @offset: the offset into the channel at which reading starts
+ * @dest:   the destination buffer that is written into from the channel
+ * @nbytes: the number of bytes to read from the channel
+ *
+ * If receiving a message, use the visorchannel_signalremove()
+ * function instead.
+ *
+ * Return: integer indicating success (zero) or failure (non-zero)
+ */
 int
 visorbus_read_channel(struct visor_device *dev, unsigned long offset,
 		      void *dest, unsigned long nbytes)
@@ -721,6 +532,19 @@ visorbus_read_channel(struct visor_device *dev, unsigned long offset,
 }
 EXPORT_SYMBOL_GPL(visorbus_read_channel);
 
+/**
+ * visorbus_write_channel() - writes the provided buffer into the designated
+ *                            channel
+ * @dev:    the device whose channel is written to
+ * @offset: the offset into the channel at which writing starts
+ * @src:    the source buffer that is written into the channel
+ * @nbytes: the number of bytes to write into the channel
+ *
+ * If sending a message, use the visorchannel_signalinsert()
+ * function instead.
+ *
+ * Return: integer indicating success (zero) or failure (non-zero)
+ */
 int
 visorbus_write_channel(struct visor_device *dev, unsigned long offset,
 		       void *src, unsigned long nbytes)
@@ -729,16 +553,13 @@ visorbus_write_channel(struct visor_device *dev, unsigned long offset,
 }
 EXPORT_SYMBOL_GPL(visorbus_write_channel);
 
-int
-visorbus_clear_channel(struct visor_device *dev, unsigned long offset, u8 ch,
-		       unsigned long nbytes)
-{
-	return visorchannel_clear(dev->visorchannel, offset, ch, nbytes);
-}
-EXPORT_SYMBOL_GPL(visorbus_clear_channel);
-
-/** We don't really have a real interrupt, so for now we just call the
- *  interrupt function periodically...
+/**
+ * visorbus_enable_channel_interrupts() - enables interrupts on the
+ *                                        designated device
+ * @dev: the device on which to enable interrupts
+ *
+ * Currently we don't yet have a real interrupt, so for now we just call the
+ * interrupt function periodically via a timer.
  */
 void
 visorbus_enable_channel_interrupts(struct visor_device *dev)
@@ -747,6 +568,11 @@ visorbus_enable_channel_interrupts(struct visor_device *dev)
 }
 EXPORT_SYMBOL_GPL(visorbus_enable_channel_interrupts);
 
+/**
+ * visorbus_disable_channel_interrupts() - disables interrupts on the
+ *                                         designated device
+ * @dev: the device on which to disable interrupts
+ */
 void
 visorbus_disable_channel_interrupts(struct visor_device *dev)
 {
@@ -754,19 +580,28 @@ visorbus_disable_channel_interrupts(struct visor_device *dev)
 }
 EXPORT_SYMBOL_GPL(visorbus_disable_channel_interrupts);
 
-/** This is how everything starts from the device end.
- *  This function is called when a channel first appears via a ControlVM
- *  message.  In response, this function allocates a visor_device to
- *  correspond to the new channel, and attempts to connect it the appropriate
- *  driver.  If the appropriate driver is found, the visor_driver.probe()
- *  function for that driver will be called, and will be passed the new
- *  visor_device that we just created.
+/**
+ * create_visor_device() - create visor device as a result of receiving the
+ *                         controlvm device_create message for a new device
+ * @dev: a freshly-zeroed struct visor_device, containing only filled-in values
+ *       for chipset_bus_no and chipset_dev_no, that will be initialized
+ *
+ * This is how everything starts from the device end.
+ * This function is called when a channel first appears via a ControlVM
+ * message.  In response, this function allocates a visor_device to
+ * correspond to the new channel, and attempts to connect it the appropriate
+ * driver.  If the appropriate driver is found, the visor_driver.probe()
+ * function for that driver will be called, and will be passed the new
+ * visor_device that we just created.
+ *
+ * It's ok if the appropriate driver is not yet loaded, because in that case
+ * the new device struct will just stick around in the bus' list of devices.
+ * When the appropriate driver calls visorbus_register_visor_driver(), the
+ * visor_driver.probe() for the new driver will be called with the new
+ * device.
  *
- *  It's ok if the appropriate driver is not yet loaded, because in that case
- *  the new device struct will just stick around in the bus' list of devices.
- *  When the appropriate driver calls visorbus_register_visor_driver(), the
- *  visor_driver.probe() for the new driver will be called with the new
- *  device.
+ * Return: 0 if successful, otherwise the negative value returned by
+ *         device_add() indicating the reason for failure
  */
 static int
 create_visor_device(struct visor_device *dev)
@@ -778,33 +613,27 @@ create_visor_device(struct visor_device *dev)
 	POSTCODE_LINUX_4(DEVICE_CREATE_ENTRY_PC, chipset_dev_no, chipset_bus_no,
 			 POSTCODE_SEVERITY_INFO);
 
-	sema_init(&dev->visordriver_callback_lock, 1);	/* unlocked */
+	mutex_init(&dev->visordriver_callback_lock);
 	dev->device.bus = &visorbus_type;
 	dev->device.groups = visorbus_channel_groups;
 	device_initialize(&dev->device);
 	dev->device.release = visorbus_release_device;
 	/* keep a reference just for us (now 2) */
 	get_device(&dev->device);
-	dev->periodic_work =
-		visor_periodic_work_create(POLLJIFFIES_NORMALCHANNEL,
-					   periodic_dev_workqueue,
-					   dev_periodic_work,
-					   dev, dev_name(&dev->device));
-	if (!dev->periodic_work) {
-		POSTCODE_LINUX_3(DEVICE_CREATE_FAILURE_PC, chipset_dev_no,
-				 DIAG_SEVERITY_ERR);
-		err = -EINVAL;
-		goto err_put;
-	}
+	init_timer(&dev->timer);
+	dev->timer.data = (unsigned long)(dev);
+	dev->timer.function = dev_periodic_work;
 
-	/* bus_id must be a unique name with respect to this bus TYPE
+	/*
+	 * bus_id must be a unique name with respect to this bus TYPE
 	 * (NOT bus instance).  That's why we need to include the bus
 	 * number within the name.
 	 */
 	dev_set_name(&dev->device, "vbus%u:dev%u",
 		     chipset_bus_no, chipset_dev_no);
 
-	/*  device_add does this:
+	/*
+	 * device_add does this:
 	 *    bus_add_device(dev)
 	 *    ->device_attach(dev)
 	 *      ->for each driver drv registered on the bus that dev is on
@@ -864,11 +693,20 @@ get_vbus_header_info(struct visorchannel *chan,
 	return 0;
 }
 
-/* Write the contents of <info> to the struct
- * spar_vbus_channel_protocol.chp_info.
+/**
+ * write_vbus_chp_info() - write the contents of <info> to the struct
+ *                         spar_vbus_channel_protocol.chp_info
+ * @chan:     indentifies the s-Par channel that will be updated
+ * @hdr_info: used to find appropriate channel offset to write data
+ * @info:     contains the information to write
+ *
+ * Writes chipset info into the channel memory to be used for diagnostic
+ * purposes.
+ *
+ * Returns no value since this is debug information and not needed for
+ * device functionality.
  */
-
-static int
+static void
 write_vbus_chp_info(struct visorchannel *chan,
 		    struct spar_vbus_headerinfo *hdr_info,
 		    struct ultra_vbus_deviceinfo *info)
@@ -876,18 +714,25 @@ write_vbus_chp_info(struct visorchannel *chan,
 	int off = sizeof(struct channel_header) + hdr_info->chp_info_offset;
 
 	if (hdr_info->chp_info_offset == 0)
-		return -EFAULT;
+		return;
 
-	if (visorchannel_write(chan, off, info, sizeof(*info)) < 0)
-		return -EFAULT;
-	return 0;
+	visorchannel_write(chan, off, info, sizeof(*info));
 }
 
-/* Write the contents of <info> to the struct
- * spar_vbus_channel_protocol.bus_info.
+/**
+ * write_vbus_bus_info() - write the contents of <info> to the struct
+ *                         spar_vbus_channel_protocol.bus_info
+ * @chan:     indentifies the s-Par channel that will be updated
+ * @hdr_info: used to find appropriate channel offset to write data
+ * @info:     contains the information to write
+ *
+ * Writes bus info into the channel memory to be used for diagnostic
+ * purposes.
+ *
+ * Returns no value since this is debug information and not needed for
+ * device functionality.
  */
-
-static int
+static void
 write_vbus_bus_info(struct visorchannel *chan,
 		    struct spar_vbus_headerinfo *hdr_info,
 		    struct ultra_vbus_deviceinfo *info)
@@ -895,37 +740,46 @@ write_vbus_bus_info(struct visorchannel *chan,
 	int off = sizeof(struct channel_header) + hdr_info->bus_info_offset;
 
 	if (hdr_info->bus_info_offset == 0)
-		return -EFAULT;
+		return;
 
-	if (visorchannel_write(chan, off, info, sizeof(*info)) < 0)
-		return -EFAULT;
-	return 0;
+	visorchannel_write(chan, off, info, sizeof(*info));
 }
 
-/* Write the contents of <info> to the
- * struct spar_vbus_channel_protocol.dev_info[<devix>].
+/**
+ * write_vbus_dev_info() - write the contents of <info> to the struct
+ *                         spar_vbus_channel_protocol.dev_info[<devix>]
+ * @chan:     indentifies the s-Par channel that will be updated
+ * @hdr_info: used to find appropriate channel offset to write data
+ * @info:     contains the information to write
+ * @devix:    the relative device number (0..n-1) of the device on the bus
+ *
+ * Writes device info into the channel memory to be used for diagnostic
+ * purposes.
+ *
+ * Returns no value since this is debug information and not needed for
+ * device functionality.
  */
-static int
+static void
 write_vbus_dev_info(struct visorchannel *chan,
 		    struct spar_vbus_headerinfo *hdr_info,
-		    struct ultra_vbus_deviceinfo *info, int devix)
+		    struct ultra_vbus_deviceinfo *info, unsigned int devix)
 {
 	int off =
 	    (sizeof(struct channel_header) + hdr_info->dev_info_offset) +
 	    (hdr_info->device_info_struct_bytes * devix);
 
 	if (hdr_info->dev_info_offset == 0)
-		return -EFAULT;
+		return;
 
-	if (visorchannel_write(chan, off, info, sizeof(*info)) < 0)
-		return -EFAULT;
-	return 0;
+	visorchannel_write(chan, off, info, sizeof(*info));
 }
 
-/* For a child device just created on a client bus, fill in
- * information about the driver that is controlling this device into
- * the the appropriate slot within the vbus channel of the bus
- * instance.
+/**
+ * fix_vbus_dev_info() - for a child device just created on a client bus, fill
+ *                       in information about the driver that is controlling
+ *                       this device into the the appropriate slot within the
+ *                       vbus channel of the bus instance
+ * @visordev: struct visor_device for the desired device
  */
 static void
 fix_vbus_dev_info(struct visor_device *visordev)
@@ -933,8 +787,8 @@ fix_vbus_dev_info(struct visor_device *visordev)
 	int i;
 	struct visor_device *bdev;
 	struct visor_driver *visordrv;
-	int bus_no = visordev->chipset_bus_no;
-	int dev_no = visordev->chipset_dev_no;
+	u32 bus_no = visordev->chipset_bus_no;
+	u32 dev_no = visordev->chipset_dev_no;
 	struct ultra_vbus_deviceinfo dev_info;
 	const char *chan_type_name = NULL;
 	struct spar_vbus_headerinfo *hdr_info;
@@ -942,17 +796,16 @@ fix_vbus_dev_info(struct visor_device *visordev)
 	if (!visordev->device.driver)
 		return;
 
-	hdr_info = (struct spar_vbus_headerinfo *)visordev->vbus_hdr_info;
-	if (!hdr_info)
-		return;
-
 	bdev = visorbus_get_device_by_id(bus_no, BUS_ROOT_DEVICE, NULL);
 	if (!bdev)
 		return;
-
+	hdr_info = (struct spar_vbus_headerinfo *)bdev->vbus_hdr_info;
+	if (!hdr_info)
+		return;
 	visordrv = to_visor_driver(visordev->device.driver);
 
-	/* Within the list of device types (by GUID) that the driver
+	/*
+	 * Within the list of device types (by GUID) that the driver
 	 * says it supports, find out which one of those types matches
 	 * the type of this device, so that we can include the device
 	 * type name
@@ -966,20 +819,148 @@ fix_vbus_dev_info(struct visor_device *visordev)
 		}
 	}
 
-	bus_device_info_init(&dev_info, chan_type_name,
-			     visordrv->name, visordrv->version,
-			     visordrv->vertag);
+	bus_device_info_init(&dev_info, chan_type_name, visordrv->name);
 	write_vbus_dev_info(bdev->visorchannel, hdr_info, &dev_info, dev_no);
 
-	/* Re-write bus+chipset info, because it is possible that this
-	* was previously written by our evil counterpart, virtpci.
-	*/
+	/*
+	 * Re-write bus+chipset info, because it is possible that this
+	 * was previously written by our evil counterpart, virtpci.
+	 */
 	write_vbus_chp_info(bdev->visorchannel, hdr_info, &chipset_driverinfo);
 	write_vbus_bus_info(bdev->visorchannel, hdr_info,
 			    &clientbus_driverinfo);
 }
 
-/** Create a device instance for the visor bus itself.
+/**
+ * visordriver_probe_device() - handle new visor device coming online
+ * @xdev: struct device for the visor device being probed
+ *
+ * This is called automatically upon adding a visor_device (device_add), or
+ * adding a visor_driver (visorbus_register_visor_driver), but only after
+ * visorbus_match() has returned 1 to indicate a successful match between
+ * driver and device.
+ *
+ * If successful, a reference to the device will be held onto via get_device().
+ *
+ * Return: 0 if successful, meaning the function driver's probe() function
+ *         was successful with this device, otherwise a negative errno
+ *         value indicating failure reason
+ */
+static int
+visordriver_probe_device(struct device *xdev)
+{
+	int res;
+	struct visor_driver *drv;
+	struct visor_device *dev;
+
+	drv = to_visor_driver(xdev->driver);
+	dev = to_visor_device(xdev);
+
+	if (!drv->probe)
+		return -ENODEV;
+
+	mutex_lock(&dev->visordriver_callback_lock);
+	dev->being_removed = false;
+
+	res = drv->probe(dev);
+	if (res >= 0) {
+		/* success: reference kept via unmatched get_device() */
+		get_device(&dev->device);
+		fix_vbus_dev_info(dev);
+	}
+
+	mutex_unlock(&dev->visordriver_callback_lock);
+	return res;
+}
+
+/**
+ * visorbus_register_visor_driver() - registers the provided visor driver
+ *                                    for handling one or more visor device
+ *                                    types (channel_types)
+ * @drv: the driver to register
+ *
+ * A visor function driver calls this function to register
+ * the driver.  The caller MUST fill in the following fields within the
+ * #drv structure:
+ *     name, version, owner, channel_types, probe, remove
+ *
+ * Here's how the whole Linux bus / driver / device model works.
+ *
+ * At system start-up, the visorbus kernel module is loaded, which registers
+ * visorbus_type as a bus type, using bus_register().
+ *
+ * All kernel modules that support particular device types on a
+ * visorbus bus are loaded.  Each of these kernel modules calls
+ * visorbus_register_visor_driver() in their init functions, passing a
+ * visor_driver struct.  visorbus_register_visor_driver() in turn calls
+ * register_driver(&visor_driver.driver).  This .driver member is
+ * initialized with generic methods (like probe), whose sole responsibility
+ * is to act as a broker for the real methods, which are within the
+ * visor_driver struct.  (This is the way the subclass behavior is
+ * implemented, since visor_driver is essentially a subclass of the
+ * generic driver.)  Whenever a driver_register() happens, core bus code in
+ * the kernel does (see device_attach() in drivers/base/dd.c):
+ *
+ *     for each dev associated with the bus (the bus that driver is on) that
+ *     does not yet have a driver
+ *         if bus.match(dev,newdriver) == yes_matched  ** .match specified
+ *                                                ** during bus_register().
+ *             newdriver.probe(dev)  ** for visor drivers, this will call
+ *                   ** the generic driver.probe implemented in visorbus.c,
+ *                   ** which in turn calls the probe specified within the
+ *                   ** struct visor_driver (which was specified by the
+ *                   ** actual device driver as part of
+ *                   ** visorbus_register_visor_driver()).
+ *
+ * The above dance also happens when a new device appears.
+ * So the question is, how are devices created within the system?
+ * Basically, just call device_add(dev).  See pci_bus_add_devices().
+ * pci_scan_device() shows an example of how to build a device struct.  It
+ * returns the newly-created struct to pci_scan_single_device(), who adds it
+ * to the list of devices at PCIBUS.devices.  That list of devices is what
+ * is traversed by pci_bus_add_devices().
+ *
+ * Return: integer indicating success (zero) or failure (non-zero)
+ */
+int visorbus_register_visor_driver(struct visor_driver *drv)
+{
+	int rc = 0;
+
+	if (busreg_rc < 0)
+		return -ENODEV; /*can't register on a nonexistent bus*/
+
+	drv->driver.name = drv->name;
+	drv->driver.bus = &visorbus_type;
+	drv->driver.probe = visordriver_probe_device;
+	drv->driver.remove = visordriver_remove_device;
+	drv->driver.owner = drv->owner;
+
+	/*
+	 * driver_register does this:
+	 *   bus_add_driver(drv)
+	 *   ->if (drv.bus)  ** (bus_type) **
+	 *       driver_attach(drv)
+	 *         for each dev with bus type of drv.bus
+	 *           if (!dev.drv)  ** no driver assigned yet **
+	 *             if (bus.match(dev,drv))  [visorbus_match]
+	 *               dev.drv = drv
+	 *               if (!drv.probe(dev))   [visordriver_probe_device]
+	 *                 dev.drv = NULL
+	 */
+
+	rc = driver_register(&drv->driver);
+	if (rc < 0)
+		driver_unregister(&drv->driver);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(visorbus_register_visor_driver);
+
+/**
+ * create_bus_instance() - create a device instance for the visor bus itself
+ * @dev: struct visor_device indicating the bus instance
+ *
+ * Return: 0 for success, otherwise negative errno value indicating reason for
+ *         failure
  */
 static int
 create_bus_instance(struct visor_device *dev)
@@ -1014,25 +995,26 @@ create_bus_instance(struct visor_device *dev)
 	} else {
 		kfree(hdr_info);
 	}
-	bus_count++;
 	list_add_tail(&dev->list_all, &list_all_bus_instances);
 	dev_set_drvdata(&dev->device, dev);
 	return 0;
 }
 
-/** Remove a device instance for the visor bus itself.
+/**
+ * remove_bus_instance() - remove a device instance for the visor bus itself
+ * @dev: struct visor_device indentifying the bus to remove
  */
 static void
 remove_bus_instance(struct visor_device *dev)
 {
-	/* Note that this will result in the release method for
+	/*
+	 * Note that this will result in the release method for
 	 * dev->dev being called, which will call
 	 * visorbus_release_busdevice().  This has something to do with
 	 * the put_device() done in device_unregister(), but I have never
 	 * successfully been able to trace thru the code to see where/how
 	 * release() gets called.  But I know it does.
 	 */
-	bus_count--;
 	if (dev->visorchannel) {
 		visorchannel_destroy(dev->visorchannel);
 		dev->visorchannel = NULL;
@@ -1042,8 +1024,11 @@ remove_bus_instance(struct visor_device *dev)
 	device_unregister(&dev->device);
 }
 
-/** Create and register the one-and-only one instance of
- *  the visor bus type (visorbus_type).
+/**
+ * create_bus_type() - create and register the one-and-only one instance of
+ *                     the visor bus type (visorbus_type)
+ * Return: 0 for success, otherwise negative errno value returned by
+ *         bus_register() indicating the reason for failure
  */
 static int
 create_bus_type(void)
@@ -1052,7 +1037,9 @@ create_bus_type(void)
 	return busreg_rc;
 }
 
-/** Remove the one-and-only one instance of the visor bus type (visorbus_type).
+/**
+ * remove_bus_type() - remove the one-and-only one instance of the visor bus
+ *                     type (visorbus_type)
  */
 static void
 remove_bus_type(void)
@@ -1060,7 +1047,8 @@ remove_bus_type(void)
 	bus_unregister(&visorbus_type);
 }
 
-/** Remove all child visor bus device instances.
+/**
+ * remove_all_visor_devices() - remove all child visor bus device instances
  */
 static void
 remove_all_visor_devices(void)
@@ -1075,7 +1063,7 @@ remove_all_visor_devices(void)
 	}
 }
 
-static void
+void
 chipset_bus_create(struct visor_device *dev)
 {
 	int rc;
@@ -1092,19 +1080,17 @@ chipset_bus_create(struct visor_device *dev)
 		POSTCODE_LINUX_3(CHIPSET_INIT_SUCCESS_PC, bus_no,
 				 POSTCODE_SEVERITY_INFO);
 
-	if (chipset_responders.bus_create)
-		(*chipset_responders.bus_create) (dev, rc);
+	bus_create_response(dev, rc);
 }
 
-static void
+void
 chipset_bus_destroy(struct visor_device *dev)
 {
 	remove_bus_instance(dev);
-	if (chipset_responders.bus_destroy)
-		(*chipset_responders.bus_destroy)(dev, 0);
+	bus_destroy_response(dev, 0);
 }
 
-static void
+void
 chipset_device_create(struct visor_device *dev_info)
 {
 	int rc;
@@ -1115,8 +1101,7 @@ chipset_device_create(struct visor_device *dev_info)
 			 POSTCODE_SEVERITY_INFO);
 
 	rc = create_visor_device(dev_info);
-	if (chipset_responders.device_create)
-		chipset_responders.device_create(dev_info, rc);
+	device_create_response(dev_info, rc);
 
 	if (rc < 0)
 		POSTCODE_LINUX_4(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
@@ -1126,18 +1111,22 @@ chipset_device_create(struct visor_device *dev_info)
 				 POSTCODE_SEVERITY_INFO);
 }
 
-static void
+void
 chipset_device_destroy(struct visor_device *dev_info)
 {
 	remove_visor_device(dev_info);
 
-	if (chipset_responders.device_destroy)
-		(*chipset_responders.device_destroy) (dev_info, 0);
+	device_destroy_response(dev_info, 0);
 }
 
-/* This is the callback function specified for a function driver, to
- * be called when a pending "pause device" operation has been
- * completed.
+/**
+ * pause_state_change_complete() - the callback function to be called by a
+ *                                 visorbus function driver when a
+ *                                 pending "pause device" operation has
+ *                                 completed
+ * @dev: struct visor_device identifying the paused device
+ * @status: 0 iff the pause state change completed successfully, otherwise
+ *          a negative errno value indicating the reason for failure
  */
 static void
 pause_state_change_complete(struct visor_device *dev, int status)
@@ -1146,19 +1135,18 @@ pause_state_change_complete(struct visor_device *dev, int status)
 		return;
 
 	dev->pausing = false;
-	if (!chipset_responders.device_pause) /* this can never happen! */
-		return;
 
-	/* Notify the chipset driver that the pause is complete, which
-	 * will presumably want to send some sort of response to the
-	 * initiator.
-	 */
-	(*chipset_responders.device_pause) (dev, status);
+	device_pause_response(dev, status);
 }
 
-/* This is the callback function specified for a function driver, to
- * be called when a pending "resume device" operation has been
- * completed.
+/**
+ * resume_state_change_complete() - the callback function to be called by a
+ *                                  visorbus function driver when a
+ *                                  pending "resume device" operation has
+ *                                  completed
+ * @dev: struct visor_device identifying the resumed device
+ * @status: 0 iff the resume state change completed successfully, otherwise
+ *          a negative errno value indicating the reason for failure
  */
 static void
 resume_state_change_complete(struct visor_device *dev, int status)
@@ -1167,19 +1155,25 @@ resume_state_change_complete(struct visor_device *dev, int status)
 		return;
 
 	dev->resuming = false;
-	if (!chipset_responders.device_resume) /* this can never happen! */
-		return;
 
-	/* Notify the chipset driver that the resume is complete,
+	/*
+	 * Notify the chipset driver that the resume is complete,
 	 * which will presumably want to send some sort of response to
 	 * the initiator.
 	 */
-	(*chipset_responders.device_resume) (dev, status);
+	device_resume_response(dev, status);
 }
 
-/* Tell the subordinate function driver for a specific device to pause
- * or resume that device.  Result is returned asynchronously via a
- * callback function.
+/**
+ * initiate_chipset_device_pause_resume() - start a pause or resume operation
+ *                                          for a visor device
+ * @dev: struct visor_device identifying the device being paused or resumed
+ * @is_pause: true to indicate pause operation, false to indicate resume
+ *
+ * Tell the subordinate function driver for a specific device to pause
+ * or resume that device.  Success/failure result is returned asynchronously
+ * via a callback function; see pause_state_change_complete() and
+ * resume_state_change_complete().
  */
 static void
 initiate_chipset_device_pause_resume(struct visor_device *dev, bool is_pause)
@@ -1189,9 +1183,9 @@ initiate_chipset_device_pause_resume(struct visor_device *dev, bool is_pause)
 	void (*notify_func)(struct visor_device *dev, int response) = NULL;
 
 	if (is_pause)
-		notify_func = chipset_responders.device_pause;
+		notify_func = device_pause_response;
 	else
-		notify_func = chipset_responders.device_resume;
+		notify_func = device_resume_response;
 	if (!notify_func)
 		return;
 
@@ -1206,7 +1200,8 @@ initiate_chipset_device_pause_resume(struct visor_device *dev, bool is_pause)
 		return;
 	}
 
-	/* Note that even though both drv->pause() and drv->resume
+	/*
+	 * Note that even though both drv->pause() and drv->resume
 	 * specify a callback function, it is NOT necessary for us to
 	 * increment our local module usage count.  Reason is, there
 	 * is already a linkage dependency between child function
@@ -1246,33 +1241,41 @@ initiate_chipset_device_pause_resume(struct visor_device *dev, bool is_pause)
 	}
 }
 
-static void
+/**
+ * chipset_device_pause() - start a pause operation for a visor device
+ * @dev_info: struct visor_device identifying the device being paused
+ *
+ * Tell the subordinate function driver for a specific device to pause
+ * that device.  Success/failure result is returned asynchronously
+ * via a callback function; see pause_state_change_complete().
+ */
+void
 chipset_device_pause(struct visor_device *dev_info)
 {
 	initiate_chipset_device_pause_resume(dev_info, true);
 }
 
-static void
+/**
+ * chipset_device_resume() - start a resume operation for a visor device
+ * @dev_info: struct visor_device identifying the device being resumed
+ *
+ * Tell the subordinate function driver for a specific device to resume
+ * that device.  Success/failure result is returned asynchronously
+ * via a callback function; see resume_state_change_complete().
+ */
+void
 chipset_device_resume(struct visor_device *dev_info)
 {
 	initiate_chipset_device_pause_resume(dev_info, false);
 }
 
-struct channel_size_info {
-	uuid_le guid;
-	unsigned long min_size;
-	unsigned long max_size;
-};
-
 int
 visorbus_init(void)
 {
 	int err;
 
 	POSTCODE_LINUX_3(DRIVER_ENTRY_PC, 0, POSTCODE_SEVERITY_INFO);
-	bus_device_info_init(&clientbus_driverinfo,
-			     "clientbus", "visorbus",
-			     VERSION, NULL);
+	bus_device_info_init(&clientbus_driverinfo, "clientbus", "visorbus");
 
 	err = create_bus_type();
 	if (err < 0) {
@@ -1280,19 +1283,7 @@ visorbus_init(void)
 		goto error;
 	}
 
-	periodic_dev_workqueue = create_singlethread_workqueue("visorbus_dev");
-	if (!periodic_dev_workqueue) {
-		POSTCODE_LINUX_2(CREATE_WORKQUEUE_PC, DIAG_SEVERITY_ERR);
-		err = -ENOMEM;
-		goto error;
-	}
-
-	/* This enables us to receive notifications when devices appear for
-	 * which this service partition is to be a server for.
-	 */
-	visorchipset_register_busdev(&chipset_notifiers,
-				     &chipset_responders,
-				     &chipset_driverinfo);
+	bus_device_info_init(&chipset_driverinfo, "chipset", "visorchipset");
 
 	return 0;
 
@@ -1306,20 +1297,8 @@ visorbus_exit(void)
 {
 	struct list_head *listentry, *listtmp;
 
-	visorchipset_register_busdev(NULL, NULL, NULL);
 	remove_all_visor_devices();
 
-	flush_workqueue(periodic_dev_workqueue); /* better not be any work! */
-	destroy_workqueue(periodic_dev_workqueue);
-	periodic_dev_workqueue = NULL;
-
-	if (periodic_test_workqueue) {
-		cancel_delayed_work(&periodic_work);
-		flush_workqueue(periodic_test_workqueue);
-		destroy_workqueue(periodic_test_workqueue);
-		periodic_test_workqueue = NULL;
-	}
-
 	list_for_each_safe(listentry, listtmp, &list_all_bus_instances) {
 		struct visor_device *dev = list_entry(listentry,
 						      struct visor_device,
@@ -1329,9 +1308,6 @@ visorbus_exit(void)
 	remove_bus_type();
 }
 
-module_param_named(debug, visorbus_debug, int, S_IRUGO);
-MODULE_PARM_DESC(visorbus_debug, "1 to debug");
-
 module_param_named(forcematch, visorbus_forcematch, int, S_IRUGO);
 MODULE_PARM_DESC(visorbus_forcematch,
 		 "1 to force a successful dev <--> drv match");
@@ -1339,6 +1315,3 @@ MODULE_PARM_DESC(visorbus_forcematch,
 module_param_named(forcenomatch, visorbus_forcenomatch, int, S_IRUGO);
 MODULE_PARM_DESC(visorbus_forcenomatch,
 		 "1 to force an UNsuccessful dev <--> drv match");
-
-module_param_named(debugref, visorbus_debugref, int, S_IRUGO);
-MODULE_PARM_DESC(visorbus_debugref, "1 to debug reference counting");
diff --git a/drivers/staging/unisys/visorbus/visorbus_private.h b/drivers/staging/unisys/visorbus/visorbus_private.h
index 39edd2018453..15403fb52847 100644
--- a/drivers/staging/unisys/visorbus/visorbus_private.h
+++ b/drivers/staging/unisys/visorbus/visorbus_private.h
@@ -1,4 +1,4 @@
-/* visorchipset.h
+/* visorbus_private.h
  *
  * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
@@ -14,55 +14,72 @@
  * details.
  */
 
-#ifndef __VISORCHIPSET_H__
-#define __VISORCHIPSET_H__
+#ifndef __VISORBUS_PRIVATE_H__
+#define __VISORBUS_PRIVATE_H__
 
 #include <linux/uuid.h>
+#include <linux/utsname.h>
 
 #include "controlvmchannel.h"
-#include "vbusdeviceinfo.h"
-#include "vbushelper.h"
+#include "vbuschannel.h"
 
-/*  These functions will be called from within visorchipset when certain
- *  events happen.  (The implementation of these functions is outside of
- *  visorchipset.)
+/* TARGET_HOSTNAME specified as -DTARGET_HOSTNAME=\"thename\" on the
+ * command line
  */
-struct visorchipset_busdev_notifiers {
-	void (*bus_create)(struct visor_device *bus_info);
-	void (*bus_destroy)(struct visor_device *bus_info);
-	void (*device_create)(struct visor_device *bus_info);
-	void (*device_destroy)(struct visor_device *bus_info);
-	void (*device_pause)(struct visor_device *bus_info);
-	void (*device_resume)(struct visor_device *bus_info);
-};
 
-/*  These functions live inside visorchipset, and will be called to indicate
- *  responses to specific events (by code outside of visorchipset).
- *  For now, the value for each response is simply either:
- *       0 = it worked
- *      -1 = it failed
- */
-struct visorchipset_busdev_responders {
-	void (*bus_create)(struct visor_device *p, int response);
-	void (*bus_destroy)(struct visor_device *p, int response);
-	void (*device_create)(struct visor_device *p, int response);
-	void (*device_destroy)(struct visor_device *p, int response);
-	void (*device_pause)(struct visor_device *p, int response);
-	void (*device_resume)(struct visor_device *p, int response);
-};
+static inline void bus_device_info_init(
+		struct ultra_vbus_deviceinfo *bus_device_info_ptr,
+		const char *dev_type, const char *drv_name)
+{
+	memset(bus_device_info_ptr, 0, sizeof(struct ultra_vbus_deviceinfo));
+	snprintf(bus_device_info_ptr->devtype,
+		 sizeof(bus_device_info_ptr->devtype),
+		 "%s", (dev_type) ? dev_type : "unknownType");
+	snprintf(bus_device_info_ptr->drvname,
+		 sizeof(bus_device_info_ptr->drvname),
+		 "%s", (drv_name) ? drv_name : "unknownDriver");
+	snprintf(bus_device_info_ptr->infostrs,
+		 sizeof(bus_device_info_ptr->infostrs), "kernel ver. %s",
+		 utsname()->release);
+}
 
-/** Register functions (in the bus driver) to get called by visorchipset
- *  whenever a bus or device appears for which this guest is to be the
- *  client for.  visorchipset will fill in <responders>, to indicate
- *  functions the bus driver should call to indicate message responses.
- */
-void
-visorchipset_register_busdev(
-			struct visorchipset_busdev_notifiers *notifiers,
-			struct visorchipset_busdev_responders *responders,
-			struct ultra_vbus_deviceinfo *driver_info);
+void chipset_bus_create(struct visor_device *bus_info);
+void chipset_bus_destroy(struct visor_device *bus_info);
+void chipset_device_create(struct visor_device *dev_info);
+void chipset_device_destroy(struct visor_device *dev_info);
+void chipset_device_pause(struct visor_device *dev_info);
+void chipset_device_resume(struct visor_device *dev_info);
+
+void bus_create_response(struct visor_device *p, int response);
+void bus_destroy_response(struct visor_device *p, int response);
+void device_create_response(struct visor_device *p, int response);
+void device_destroy_response(struct visor_device *p, int response);
+void device_resume_response(struct visor_device *p, int response);
+void device_pause_response(struct visor_device *p, int response);
 
-/* visorbus init and exit functions */
 int visorbus_init(void);
 void visorbus_exit(void);
+
+/* visorchannel access functions */
+
+struct visorchannel *visorchannel_create(u64 physaddr,
+					 unsigned long channel_bytes,
+					 gfp_t gfp, uuid_le guid);
+struct visorchannel *visorchannel_create_with_lock(u64 physaddr,
+						   unsigned long channel_bytes,
+						   gfp_t gfp, uuid_le guid);
+void visorchannel_destroy(struct visorchannel *channel);
+int visorchannel_read(struct visorchannel *channel, ulong offset,
+		      void *local, ulong nbytes);
+int visorchannel_write(struct visorchannel *channel, ulong offset,
+		       void *local, ulong nbytes);
+u64 visorchannel_get_physaddr(struct visorchannel *channel);
+ulong visorchannel_get_nbytes(struct visorchannel *channel);
+char *visorchannel_id(struct visorchannel *channel, char *s);
+char *visorchannel_zoneid(struct visorchannel *channel, char *s);
+u64 visorchannel_get_clientpartition(struct visorchannel *channel);
+int visorchannel_set_clientpartition(struct visorchannel *channel,
+				     u64 partition_handle);
+char *visorchannel_uuid_id(uuid_le *guid, char *s);
+void __iomem *visorchannel_get_header(struct visorchannel *channel);
 #endif
diff --git a/drivers/staging/unisys/visorbus/visorchannel.c b/drivers/staging/unisys/visorbus/visorchannel.c
index 43373582cf1d..300a65dc5c6c 100644
--- a/drivers/staging/unisys/visorbus/visorchannel.c
+++ b/drivers/staging/unisys/visorbus/visorchannel.c
@@ -15,14 +15,13 @@
  */
 
 /*
- *  This provides Supervisor channel communication primitives, which are
+ *  This provides s-Par channel communication primitives, which are
  *  independent of the mechanism used to access the channel data.
  */
 
 #include <linux/uuid.h>
 #include <linux/io.h>
 
-#include "version.h"
 #include "visorbus.h"
 #include "controlvmchannel.h"
 
@@ -55,110 +54,6 @@ struct visorchannel {
 	uuid_le inst;
 };
 
-/* Creates the struct visorchannel abstraction for a data area in memory,
- * but does NOT modify this data area.
- */
-static struct visorchannel *
-visorchannel_create_guts(u64 physaddr, unsigned long channel_bytes,
-			 gfp_t gfp, unsigned long off,
-			 uuid_le guid, bool needs_lock)
-{
-	struct visorchannel *channel;
-	int err;
-	size_t size = sizeof(struct channel_header);
-
-	if (physaddr == 0)
-		return NULL;
-
-	channel = kzalloc(sizeof(*channel), gfp);
-	if (!channel)
-		return NULL;
-
-	channel->needs_lock = needs_lock;
-	spin_lock_init(&channel->insert_lock);
-	spin_lock_init(&channel->remove_lock);
-
-	/* Video driver constains the efi framebuffer so it will get a
-	 * conflict resource when requesting its full mem region. Since
-	 * we are only using the efi framebuffer for video we can ignore
-	 * this. Remember that we haven't requested it so we don't try to
-	 * release later on.
-	 */
-	channel->requested = request_mem_region(physaddr, size, MYDRVNAME);
-	if (!channel->requested) {
-		if (uuid_le_cmp(guid, spar_video_guid)) {
-			/* Not the video channel we care about this */
-			goto err_destroy_channel;
-		}
-	}
-
-	channel->mapped = memremap(physaddr, size, MEMREMAP_WB);
-	if (!channel->mapped) {
-		release_mem_region(physaddr, size);
-		goto err_destroy_channel;
-	}
-
-	channel->physaddr = physaddr;
-	channel->nbytes = size;
-
-	err = visorchannel_read(channel, 0, &channel->chan_hdr,
-				sizeof(struct channel_header));
-	if (err)
-		goto err_destroy_channel;
-
-	/* we had better be a CLIENT of this channel */
-	if (channel_bytes == 0)
-		channel_bytes = (ulong)channel->chan_hdr.size;
-	if (uuid_le_cmp(guid, NULL_UUID_LE) == 0)
-		guid = channel->chan_hdr.chtype;
-
-	memunmap(channel->mapped);
-	if (channel->requested)
-		release_mem_region(channel->physaddr, channel->nbytes);
-	channel->mapped = NULL;
-	channel->requested = request_mem_region(channel->physaddr,
-						channel_bytes, MYDRVNAME);
-	if (!channel->requested) {
-		if (uuid_le_cmp(guid, spar_video_guid)) {
-			/* Different we care about this */
-			goto err_destroy_channel;
-		}
-	}
-
-	channel->mapped = memremap(channel->physaddr, channel_bytes,
-			MEMREMAP_WB);
-	if (!channel->mapped) {
-		release_mem_region(channel->physaddr, channel_bytes);
-		goto err_destroy_channel;
-	}
-
-	channel->nbytes = channel_bytes;
-	channel->guid = guid;
-	return channel;
-
-err_destroy_channel:
-	visorchannel_destroy(channel);
-	return NULL;
-}
-
-struct visorchannel *
-visorchannel_create(u64 physaddr, unsigned long channel_bytes,
-		    gfp_t gfp, uuid_le guid)
-{
-	return visorchannel_create_guts(physaddr, channel_bytes, gfp, 0, guid,
-					false);
-}
-EXPORT_SYMBOL_GPL(visorchannel_create);
-
-struct visorchannel *
-visorchannel_create_with_lock(u64 physaddr, unsigned long channel_bytes,
-			      gfp_t gfp, uuid_le guid)
-{
-	return visorchannel_create_guts(physaddr, channel_bytes, gfp, 0, guid,
-					true);
-}
-EXPORT_SYMBOL_GPL(visorchannel_create_with_lock);
-
 void
 visorchannel_destroy(struct visorchannel *channel)
 {
@@ -171,21 +66,18 @@ visorchannel_destroy(struct visorchannel *channel)
 	}
 	kfree(channel);
 }
-EXPORT_SYMBOL_GPL(visorchannel_destroy);
 
 u64
 visorchannel_get_physaddr(struct visorchannel *channel)
 {
 	return channel->physaddr;
 }
-EXPORT_SYMBOL_GPL(visorchannel_get_physaddr);
 
 ulong
 visorchannel_get_nbytes(struct visorchannel *channel)
 {
 	return channel->nbytes;
 }
-EXPORT_SYMBOL_GPL(visorchannel_get_nbytes);
 
 char *
 visorchannel_uuid_id(uuid_le *guid, char *s)
@@ -193,28 +85,24 @@ visorchannel_uuid_id(uuid_le *guid, char *s)
 	sprintf(s, "%pUL", guid);
 	return s;
 }
-EXPORT_SYMBOL_GPL(visorchannel_uuid_id);
 
 char *
 visorchannel_id(struct visorchannel *channel, char *s)
 {
 	return visorchannel_uuid_id(&channel->guid, s);
 }
-EXPORT_SYMBOL_GPL(visorchannel_id);
 
 char *
 visorchannel_zoneid(struct visorchannel *channel, char *s)
 {
 	return visorchannel_uuid_id(&channel->chan_hdr.zone_uuid, s);
 }
-EXPORT_SYMBOL_GPL(visorchannel_zoneid);
 
 u64
 visorchannel_get_clientpartition(struct visorchannel *channel)
 {
 	return channel->chan_hdr.partition_handle;
 }
-EXPORT_SYMBOL_GPL(visorchannel_get_clientpartition);
 
 int
 visorchannel_set_clientpartition(struct visorchannel *channel,
@@ -223,8 +111,13 @@ visorchannel_set_clientpartition(struct visorchannel *channel,
 	channel->chan_hdr.partition_handle = partition_handle;
 	return 0;
 }
-EXPORT_SYMBOL_GPL(visorchannel_set_clientpartition);
 
+/**
+ * visorchannel_get_uuid() - queries the UUID of the designated channel
+ * @channel: the channel to query
+ *
+ * Return: the UUID of the provided channel
+ */
 uuid_le
 visorchannel_get_uuid(struct visorchannel *channel)
 {
@@ -243,7 +136,6 @@ visorchannel_read(struct visorchannel *channel, ulong offset,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(visorchannel_read);
 
 int
 visorchannel_write(struct visorchannel *channel, ulong offset,
@@ -265,156 +157,125 @@ visorchannel_write(struct visorchannel *channel, ulong offset,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(visorchannel_write);
-
-int
-visorchannel_clear(struct visorchannel *channel, ulong offset, u8 ch,
-		   ulong nbytes)
-{
-	int err;
-	int bufsize = PAGE_SIZE;
-	int written = 0;
-	u8 *buf;
-
-	buf = (u8 *)__get_free_page(GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	memset(buf, ch, bufsize);
-
-	while (nbytes > 0) {
-		int thisbytes = bufsize;
-
-		if (nbytes < thisbytes)
-			thisbytes = nbytes;
-		err = visorchannel_write(channel, offset + written,
-					 buf, thisbytes);
-		if (err)
-			goto out_free_page;
-
-		written += thisbytes;
-		nbytes -= thisbytes;
-	}
-	err = 0;
-
-out_free_page:
-	free_page((unsigned long)buf);
-	return err;
-}
-EXPORT_SYMBOL_GPL(visorchannel_clear);
 
 void __iomem  *
 visorchannel_get_header(struct visorchannel *channel)
 {
 	return (void __iomem *)&channel->chan_hdr;
 }
-EXPORT_SYMBOL_GPL(visorchannel_get_header);
 
-/** Return offset of a specific SIGNAL_QUEUE_HEADER from the beginning of a
- *  channel header
+/*
+ * Return offset of a specific SIGNAL_QUEUE_HEADER from the beginning of a
+ * channel header
  */
 #define SIG_QUEUE_OFFSET(chan_hdr, q) \
 	((chan_hdr)->ch_space_offset + \
 	 ((q) * sizeof(struct signal_queue_header)))
 
-/** Return offset of a specific queue entry (data) from the beginning of a
- *  channel header
+/*
+ * Return offset of a specific queue entry (data) from the beginning of a
+ * channel header
  */
 #define SIG_DATA_OFFSET(chan_hdr, q, sig_hdr, slot) \
 	(SIG_QUEUE_OFFSET(chan_hdr, q) + (sig_hdr)->sig_base_offset + \
 	    ((slot) * (sig_hdr)->signal_size))
 
-/** Write the contents of a specific field within a SIGNAL_QUEUE_HEADER back
- *  into host memory
+/*
+ * Write the contents of a specific field within a SIGNAL_QUEUE_HEADER back
+ * into host memory
  */
 #define SIG_WRITE_FIELD(channel, queue, sig_hdr, FIELD)			 \
-	(visorchannel_write(channel,					 \
-			    SIG_QUEUE_OFFSET(&channel->chan_hdr, queue) +\
-			    offsetof(struct signal_queue_header, FIELD), \
-			    &((sig_hdr)->FIELD),			 \
-			    sizeof((sig_hdr)->FIELD)) >= 0)
+	visorchannel_write(channel,					 \
+			   SIG_QUEUE_OFFSET(&channel->chan_hdr, queue) +\
+			   offsetof(struct signal_queue_header, FIELD), \
+			   &((sig_hdr)->FIELD),			 \
+			   sizeof((sig_hdr)->FIELD))
 
-static bool
+static int
 sig_read_header(struct visorchannel *channel, u32 queue,
 		struct signal_queue_header *sig_hdr)
 {
-	int err;
-
 	if (channel->chan_hdr.ch_space_offset < sizeof(struct channel_header))
-		return false;
+		return -EINVAL;
 
 	/* Read the appropriate SIGNAL_QUEUE_HEADER into local memory. */
-	err = visorchannel_read(channel,
-				SIG_QUEUE_OFFSET(&channel->chan_hdr, queue),
-				sig_hdr, sizeof(struct signal_queue_header));
-	if (err)
-		return false;
-
-	return true;
+	return visorchannel_read(channel,
+				 SIG_QUEUE_OFFSET(&channel->chan_hdr, queue),
+				 sig_hdr, sizeof(struct signal_queue_header));
 }
 
-static inline bool
+static inline int
 sig_read_data(struct visorchannel *channel, u32 queue,
 	      struct signal_queue_header *sig_hdr, u32 slot, void *data)
 {
-	int err;
 	int signal_data_offset = SIG_DATA_OFFSET(&channel->chan_hdr, queue,
 						 sig_hdr, slot);
 
-	err = visorchannel_read(channel, signal_data_offset,
-				data, sig_hdr->signal_size);
-	if (err)
-		return false;
-
-	return true;
+	return visorchannel_read(channel, signal_data_offset,
+				 data, sig_hdr->signal_size);
 }
 
-static inline bool
+static inline int
 sig_write_data(struct visorchannel *channel, u32 queue,
 	       struct signal_queue_header *sig_hdr, u32 slot, void *data)
 {
-	int err;
 	int signal_data_offset = SIG_DATA_OFFSET(&channel->chan_hdr, queue,
 						 sig_hdr, slot);
 
-	err = visorchannel_write(channel, signal_data_offset,
-				 data, sig_hdr->signal_size);
-	if (err)
-		return false;
-
-	return true;
+	return visorchannel_write(channel, signal_data_offset,
+				  data, sig_hdr->signal_size);
 }
 
-static bool
+static int
 signalremove_inner(struct visorchannel *channel, u32 queue, void *msg)
 {
 	struct signal_queue_header sig_hdr;
+	int error;
+
+	error = sig_read_header(channel, queue, &sig_hdr);
+	if (error)
+		return error;
 
-	if (!sig_read_header(channel, queue, &sig_hdr))
-		return false;
 	if (sig_hdr.head == sig_hdr.tail)
-		return false;	/* no signals to remove */
+		return -EIO;	/* no signals to remove */
 
 	sig_hdr.tail = (sig_hdr.tail + 1) % sig_hdr.max_slots;
-	if (!sig_read_data(channel, queue, &sig_hdr, sig_hdr.tail, msg))
-		return false;
+
+	error = sig_read_data(channel, queue, &sig_hdr, sig_hdr.tail, msg);
+	if (error)
+		return error;
+
 	sig_hdr.num_received++;
 
-	/* For each data field in SIGNAL_QUEUE_HEADER that was modified,
+	/*
+	 * For each data field in SIGNAL_QUEUE_HEADER that was modified,
 	 * update host memory.
 	 */
 	mb(); /* required for channel synch */
-	if (!SIG_WRITE_FIELD(channel, queue, &sig_hdr, tail))
-		return false;
-	if (!SIG_WRITE_FIELD(channel, queue, &sig_hdr, num_received))
-		return false;
-	return true;
+
+	error = SIG_WRITE_FIELD(channel, queue, &sig_hdr, tail);
+	if (error)
+		return error;
+	error = SIG_WRITE_FIELD(channel, queue, &sig_hdr, num_received);
+	if (error)
+		return error;
+
+	return 0;
 }
 
-bool
+/**
+ * visorchannel_signalremove() - removes a message from the designated
+ *                               channel/queue
+ * @channel: the channel the message will be removed from
+ * @queue:   the queue the message will be removed from
+ * @msg:     the message to remove
+ *
+ * Return: integer error code indicating the status of the removal
+ */
+int
 visorchannel_signalremove(struct visorchannel *channel, u32 queue, void *msg)
 {
-	bool rc;
+	int rc;
 	unsigned long flags;
 
 	if (channel->needs_lock) {
@@ -429,6 +290,15 @@ visorchannel_signalremove(struct visorchannel *channel, u32 queue, void *msg)
 }
 EXPORT_SYMBOL_GPL(visorchannel_signalremove);
 
+/**
+ * visorchannel_signalempty() - checks if the designated channel/queue
+ *                              contains any messages
+ * @channel: the channel to query
+ * @queue:   the queue in the channel to query
+ *
+ * Return: boolean indicating whether any messages in the designated
+ *         channel/queue are present
+ */
 bool
 visorchannel_signalempty(struct visorchannel *channel, u32 queue)
 {
@@ -439,7 +309,7 @@ visorchannel_signalempty(struct visorchannel *channel, u32 queue)
 	if (channel->needs_lock)
 		spin_lock_irqsave(&channel->remove_lock, flags);
 
-	if (!sig_read_header(channel, queue, &sig_hdr))
+	if (sig_read_header(channel, queue, &sig_hdr))
 		rc = true;
 	if (sig_hdr.head == sig_hdr.tail)
 		rc = true;
@@ -450,13 +320,15 @@ visorchannel_signalempty(struct visorchannel *channel, u32 queue)
 }
 EXPORT_SYMBOL_GPL(visorchannel_signalempty);
 
-static bool
+static int
 signalinsert_inner(struct visorchannel *channel, u32 queue, void *msg)
 {
 	struct signal_queue_header sig_hdr;
+	int error;
 
-	if (!sig_read_header(channel, queue, &sig_hdr))
-		return false;
+	error = sig_read_header(channel, queue, &sig_hdr);
+	if (error)
+		return error;
 
 	sig_hdr.head = (sig_hdr.head + 1) % sig_hdr.max_slots;
 	if (sig_hdr.head == sig_hdr.tail) {
@@ -467,169 +339,176 @@ signalinsert_inner(struct visorchannel *channel, u32 queue, void *msg)
 					    num_overflows),
 				   &sig_hdr.num_overflows,
 				   sizeof(sig_hdr.num_overflows));
-		return false;
+		return -EIO;
 	}
 
-	if (!sig_write_data(channel, queue, &sig_hdr, sig_hdr.head, msg))
-		return false;
+	error = sig_write_data(channel, queue, &sig_hdr, sig_hdr.head, msg);
+	if (error)
+		return error;
 
 	sig_hdr.num_sent++;
 
-	/* For each data field in SIGNAL_QUEUE_HEADER that was modified,
+	/*
+	 * For each data field in SIGNAL_QUEUE_HEADER that was modified,
 	 * update host memory.
 	 */
 	mb(); /* required for channel synch */
-	if (!SIG_WRITE_FIELD(channel, queue, &sig_hdr, head))
-		return false;
-	if (!SIG_WRITE_FIELD(channel, queue, &sig_hdr, num_sent))
-		return false;
 
-	return true;
+	error = SIG_WRITE_FIELD(channel, queue, &sig_hdr, head);
+	if (error)
+		return error;
+	error = SIG_WRITE_FIELD(channel, queue, &sig_hdr, num_sent);
+	if (error)
+		return error;
+
+	return 0;
 }
 
-bool
-visorchannel_signalinsert(struct visorchannel *channel, u32 queue, void *msg)
+/**
+ * visorchannel_create_guts() - creates the struct visorchannel abstraction
+ *                              for a data area in memory, but does NOT modify
+ *                              this data area
+ * @physaddr:      physical address of start of channel
+ * @channel_bytes: size of the channel in bytes; this may 0 if the channel has
+ *                 already been initialized in memory (which is true for all
+ *                 channels provided to guest environments by the s-Par
+ *                 back-end), in which case the actual channel size will be
+ *                 read from the channel header in memory
+ * @gfp:           gfp_t to use when allocating memory for the data struct
+ * @guid:          uuid that identifies channel type; this may 0 if the channel
+ *                 has already been initialized in memory (which is true for all
+ *                 channels provided to guest environments by the s-Par
+ *                 back-end), in which case the actual channel guid will be
+ *                 read from the channel header in memory
+ * @needs_lock:    must specify true if you have multiple threads of execution
+ *                 that will be calling visorchannel methods of this
+ *                 visorchannel at the same time
+ *
+ * Return: pointer to visorchannel that was created if successful,
+ *         otherwise NULL
+ */
+static struct visorchannel *
+visorchannel_create_guts(u64 physaddr, unsigned long channel_bytes,
+			 gfp_t gfp, uuid_le guid, bool needs_lock)
 {
-	bool rc;
-	unsigned long flags;
+	struct visorchannel *channel;
+	int err;
+	size_t size = sizeof(struct channel_header);
 
-	if (channel->needs_lock) {
-		spin_lock_irqsave(&channel->insert_lock, flags);
-		rc = signalinsert_inner(channel, queue, msg);
-		spin_unlock_irqrestore(&channel->insert_lock, flags);
-	} else {
-		rc = signalinsert_inner(channel, queue, msg);
+	if (physaddr == 0)
+		return NULL;
+
+	channel = kzalloc(sizeof(*channel), gfp);
+	if (!channel)
+		return NULL;
+
+	channel->needs_lock = needs_lock;
+	spin_lock_init(&channel->insert_lock);
+	spin_lock_init(&channel->remove_lock);
+
+	/*
+	 * Video driver constains the efi framebuffer so it will get a
+	 * conflict resource when requesting its full mem region. Since
+	 * we are only using the efi framebuffer for video we can ignore
+	 * this. Remember that we haven't requested it so we don't try to
+	 * release later on.
+	 */
+	channel->requested = request_mem_region(physaddr, size, MYDRVNAME);
+	if (!channel->requested) {
+		if (uuid_le_cmp(guid, spar_video_guid)) {
+			/* Not the video channel we care about this */
+			goto err_destroy_channel;
+		}
 	}
 
-	return rc;
-}
-EXPORT_SYMBOL_GPL(visorchannel_signalinsert);
+	channel->mapped = memremap(physaddr, size, MEMREMAP_WB);
+	if (!channel->mapped) {
+		release_mem_region(physaddr, size);
+		goto err_destroy_channel;
+	}
 
-int
-visorchannel_signalqueue_slots_avail(struct visorchannel *channel, u32 queue)
-{
-	struct signal_queue_header sig_hdr;
-	u32 slots_avail, slots_used;
-	u32 head, tail;
-
-	if (!sig_read_header(channel, queue, &sig_hdr))
-		return 0;
-	head = sig_hdr.head;
-	tail = sig_hdr.tail;
-	if (head < tail)
-		head = head + sig_hdr.max_slots;
-	slots_used = head - tail;
-	slots_avail = sig_hdr.max_signals - slots_used;
-	return (int)slots_avail;
-}
-EXPORT_SYMBOL_GPL(visorchannel_signalqueue_slots_avail);
+	channel->physaddr = physaddr;
+	channel->nbytes = size;
 
-int
-visorchannel_signalqueue_max_slots(struct visorchannel *channel, u32 queue)
-{
-	struct signal_queue_header sig_hdr;
+	err = visorchannel_read(channel, 0, &channel->chan_hdr,
+				sizeof(struct channel_header));
+	if (err)
+		goto err_destroy_channel;
+
+	/* we had better be a CLIENT of this channel */
+	if (channel_bytes == 0)
+		channel_bytes = (ulong)channel->chan_hdr.size;
+	if (uuid_le_cmp(guid, NULL_UUID_LE) == 0)
+		guid = channel->chan_hdr.chtype;
+
+	memunmap(channel->mapped);
+	if (channel->requested)
+		release_mem_region(channel->physaddr, channel->nbytes);
+	channel->mapped = NULL;
+	channel->requested = request_mem_region(channel->physaddr,
+						channel_bytes, MYDRVNAME);
+	if (!channel->requested) {
+		if (uuid_le_cmp(guid, spar_video_guid)) {
+			/* Different we care about this */
+			goto err_destroy_channel;
+		}
+	}
+
+	channel->mapped = memremap(channel->physaddr, channel_bytes,
+			MEMREMAP_WB);
+	if (!channel->mapped) {
+		release_mem_region(channel->physaddr, channel_bytes);
+		goto err_destroy_channel;
+	}
+
+	channel->nbytes = channel_bytes;
+	channel->guid = guid;
+	return channel;
 
-	if (!sig_read_header(channel, queue, &sig_hdr))
-		return 0;
-	return (int)sig_hdr.max_signals;
+err_destroy_channel:
+	visorchannel_destroy(channel);
+	return NULL;
 }
-EXPORT_SYMBOL_GPL(visorchannel_signalqueue_max_slots);
 
-static void
-sigqueue_debug(struct signal_queue_header *q, int which, struct seq_file *seq)
+struct visorchannel *
+visorchannel_create(u64 physaddr, unsigned long channel_bytes,
+		    gfp_t gfp, uuid_le guid)
 {
-	seq_printf(seq, "Signal Queue #%d\n", which);
-	seq_printf(seq, "   VersionId          = %lu\n", (ulong)q->version);
-	seq_printf(seq, "   Type               = %lu\n", (ulong)q->chtype);
-	seq_printf(seq, "   oSignalBase        = %llu\n",
-		   (long long)q->sig_base_offset);
-	seq_printf(seq, "   SignalSize         = %lu\n", (ulong)q->signal_size);
-	seq_printf(seq, "   MaxSignalSlots     = %lu\n",
-		   (ulong)q->max_slots);
-	seq_printf(seq, "   MaxSignals         = %lu\n", (ulong)q->max_signals);
-	seq_printf(seq, "   FeatureFlags       = %-16.16Lx\n",
-		   (long long)q->features);
-	seq_printf(seq, "   NumSignalsSent     = %llu\n",
-		   (long long)q->num_sent);
-	seq_printf(seq, "   NumSignalsReceived = %llu\n",
-		   (long long)q->num_received);
-	seq_printf(seq, "   NumOverflows       = %llu\n",
-		   (long long)q->num_overflows);
-	seq_printf(seq, "   Head               = %lu\n", (ulong)q->head);
-	seq_printf(seq, "   Tail               = %lu\n", (ulong)q->tail);
+	return visorchannel_create_guts(physaddr, channel_bytes, gfp, guid,
+					false);
 }
 
-void
-visorchannel_debug(struct visorchannel *channel, int num_queues,
-		   struct seq_file *seq, u32 off)
+struct visorchannel *
+visorchannel_create_with_lock(u64 physaddr, unsigned long channel_bytes,
+			      gfp_t gfp, uuid_le guid)
 {
-	u64 addr = 0;
-	ulong nbytes = 0, nbytes_region = 0;
-	struct channel_header hdr;
-	struct channel_header *phdr = &hdr;
-	int i = 0;
-	int errcode = 0;
+	return visorchannel_create_guts(physaddr, channel_bytes, gfp, guid,
+					true);
+}
 
-	if (!channel)
-		return;
+/**
+ * visorchannel_signalinsert() - inserts a message into the designated
+ *                               channel/queue
+ * @channel: the channel the message will be added to
+ * @queue:   the queue the message will be added to
+ * @msg:     the message to insert
+ *
+ * Return: integer error code indicating the status of the insertion
+ */
+int
+visorchannel_signalinsert(struct visorchannel *channel, u32 queue, void *msg)
+{
+	int rc;
+	unsigned long flags;
 
-	addr = visorchannel_get_physaddr(channel);
-	nbytes_region = visorchannel_get_nbytes(channel);
-	errcode = visorchannel_read(channel, off,
-				    phdr, sizeof(struct channel_header));
-	if (errcode < 0) {
-		seq_printf(seq,
-			   "Read of channel header failed with errcode=%d)\n",
-			   errcode);
-		if (off == 0) {
-			phdr = &channel->chan_hdr;
-			seq_puts(seq, "(following data may be stale)\n");
-		} else {
-			return;
-		}
+	if (channel->needs_lock) {
+		spin_lock_irqsave(&channel->insert_lock, flags);
+		rc = signalinsert_inner(channel, queue, msg);
+		spin_unlock_irqrestore(&channel->insert_lock, flags);
+	} else {
+		rc = signalinsert_inner(channel, queue, msg);
 	}
-	nbytes = (ulong)(phdr->size);
-	seq_printf(seq, "--- Begin channel @0x%-16.16Lx for 0x%lx bytes (region=0x%lx bytes) ---\n",
-		   addr + off, nbytes, nbytes_region);
-	seq_printf(seq, "Type            = %pUL\n", &phdr->chtype);
-	seq_printf(seq, "ZoneGuid        = %pUL\n", &phdr->zone_uuid);
-	seq_printf(seq, "Signature       = 0x%-16.16Lx\n",
-		   (long long)phdr->signature);
-	seq_printf(seq, "LegacyState     = %lu\n", (ulong)phdr->legacy_state);
-	seq_printf(seq, "SrvState        = %lu\n", (ulong)phdr->srv_state);
-	seq_printf(seq, "CliStateBoot    = %lu\n", (ulong)phdr->cli_state_boot);
-	seq_printf(seq, "CliStateOS      = %lu\n", (ulong)phdr->cli_state_os);
-	seq_printf(seq, "HeaderSize      = %lu\n", (ulong)phdr->header_size);
-	seq_printf(seq, "Size            = %llu\n", (long long)phdr->size);
-	seq_printf(seq, "Features        = 0x%-16.16llx\n",
-		   (long long)phdr->features);
-	seq_printf(seq, "PartitionHandle = 0x%-16.16llx\n",
-		   (long long)phdr->partition_handle);
-	seq_printf(seq, "Handle          = 0x%-16.16llx\n",
-		   (long long)phdr->handle);
-	seq_printf(seq, "VersionId       = %lu\n", (ulong)phdr->version_id);
-	seq_printf(seq, "oChannelSpace   = %llu\n",
-		   (long long)phdr->ch_space_offset);
-	if ((phdr->ch_space_offset == 0) || (errcode < 0))
-		;
-	else
-		for (i = 0; i < num_queues; i++) {
-			struct signal_queue_header q;
-
-			errcode = visorchannel_read(channel,
-						    off +
-						    phdr->ch_space_offset +
-						    (i * sizeof(q)),
-						    &q, sizeof(q));
-			if (errcode < 0) {
-				seq_printf(seq,
-					   "failed to read signal queue #%d from channel @0x%-16.16Lx errcode=%d\n",
-					   i, addr, errcode);
-				continue;
-			}
-			sigqueue_debug(&q, i, seq);
-		}
-	seq_printf(seq, "--- End   channel @0x%-16.16Lx for 0x%lx bytes ---\n",
-		   addr + off, nbytes);
+
+	return rc;
 }
-EXPORT_SYMBOL_GPL(visorchannel_debug);
+EXPORT_SYMBOL_GPL(visorchannel_signalinsert);
diff --git a/drivers/staging/unisys/visorbus/visorchipset.c b/drivers/staging/unisys/visorbus/visorchipset.c
index d248c946a13b..59871495ea85 100644
--- a/drivers/staging/unisys/visorbus/visorchipset.c
+++ b/drivers/staging/unisys/visorbus/visorchipset.c
@@ -25,21 +25,12 @@
 #include <linux/uuid.h>
 #include <linux/crash_dump.h>
 
-#include "channel_guid.h"
-#include "controlvmchannel.h"
-#include "controlvmcompletionstatus.h"
-#include "guestlinuxdebug.h"
-#include "periodic_work.h"
-#include "version.h"
 #include "visorbus.h"
 #include "visorbus_private.h"
 #include "vmcallinterface.h"
 
 #define CURRENT_FILE_PC VISOR_CHIPSET_PC_visorchipset_main_c
 
-#define MAX_NAME_SIZE 128
-#define MAX_IP_SIZE   50
-#define MAXOUTSTANDINGCHANNELCOMMAND 256
 #define POLLJIFFIES_CONTROLVMCHANNEL_FAST   1
 #define POLLJIFFIES_CONTROLVMCHANNEL_SLOW 100
 
@@ -58,9 +49,6 @@
  * Module parameters
  */
 static int visorchipset_major;
-static int visorchipset_visorbusregwait = 1;	/* default is on */
-static unsigned long controlvm_payload_bytes_buffered;
-static u32 dump_vhba_bus;
 
 static int
 visorchipset_open(struct inode *inode, struct file *file)
@@ -79,15 +67,15 @@ visorchipset_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-/* When the controlvm channel is idle for at least MIN_IDLE_SECONDS,
-* we switch to slow polling mode.  As soon as we get a controlvm
-* message, we switch back to fast polling mode.
-*/
+/*
+ * When the controlvm channel is idle for at least MIN_IDLE_SECONDS,
+ * we switch to slow polling mode. As soon as we get a controlvm
+ * message, we switch back to fast polling mode.
+ */
 #define MIN_IDLE_SECONDS 10
 static unsigned long poll_jiffies = POLLJIFFIES_CONTROLVMCHANNEL_FAST;
 /* when we got our last controlvm message */
 static unsigned long most_recent_message_jiffies;
-static int visorbusregistered;
 
 struct parser_context {
 	unsigned long allocbytes;
@@ -99,51 +87,36 @@ struct parser_context {
 };
 
 static struct delayed_work periodic_controlvm_work;
-static DEFINE_SEMAPHORE(notifier_lock);
 
 static struct cdev file_cdev;
 static struct visorchannel **file_controlvm_channel;
-static struct controlvm_message_packet g_devicechangestate_packet;
-
-static LIST_HEAD(bus_info_list);
-static LIST_HEAD(dev_info_list);
 
 static struct visorchannel *controlvm_channel;
 
 /* Manages the request payload in the controlvm channel */
 struct visor_controlvm_payload_info {
 	u8 *ptr;		/* pointer to base address of payload pool */
-	u64 offset;		/* offset from beginning of controlvm
+	u64 offset;		/*
+				 * offset from beginning of controlvm
 				 * channel to beginning of payload * pool
 				 */
 	u32 bytes;		/* number of bytes in payload pool */
 };
 
 static struct visor_controlvm_payload_info controlvm_payload_info;
+static unsigned long controlvm_payload_bytes_buffered;
 
-/* The following globals are used to handle the scenario where we are unable to
- * offload the payload from a controlvm message due to memory requirements.  In
+/*
+ * The following globals are used to handle the scenario where we are unable to
+ * offload the payload from a controlvm message due to memory requirements. In
  * this scenario, we simply stash the controlvm message, then attempt to
  * process it again the next time controlvm_periodic_work() runs.
  */
 static struct controlvm_message controlvm_pending_msg;
 static bool controlvm_pending_msg_valid;
 
-/* This identifies a data buffer that has been received via a controlvm messages
- * in a remote --> local CONTROLVM_TRANSMIT_FILE conversation.
- */
-struct putfile_buffer_entry {
-	struct list_head next;	/* putfile_buffer_entry list */
-	struct parser_context *parser_ctx; /* points to input data buffer */
-};
-
-/* List of struct putfile_request *, via next_putfile_request member.
- * Each entry in this list identifies an outstanding TRANSMIT_FILE
- * conversation.
- */
-static LIST_HEAD(putfile_request_list);
-
-/* This describes a buffer and its current state of transfer (e.g., how many
+/*
+ * This describes a buffer and its current state of transfer (e.g., how many
  * bytes have already been supplied as putfile data, and how many bytes are
  * remaining) for a putfile_request.
  */
@@ -155,8 +128,9 @@ struct putfile_active_buffer {
 };
 
 #define PUTFILE_REQUEST_SIG 0x0906101302281211
-/* This identifies a single remote --> local CONTROLVM_TRANSMIT_FILE
- * conversation.  Structs of this type are dynamically linked into
+/*
+ * This identifies a single remote --> local CONTROLVM_TRANSMIT_FILE
+ * conversation. Structs of this type are dynamically linked into
  * <Putfile_request_list>.
  */
 struct putfile_request {
@@ -168,7 +142,8 @@ struct putfile_request {
 	/* link to next struct putfile_request */
 	struct list_head next_putfile_request;
 
-	/* head of putfile_buffer_entry list, which describes the data to be
+	/*
+	 * head of putfile_buffer_entry list, which describes the data to be
 	 * supplied as putfile data;
 	 * - this list is added to when controlvm messages come in that supply
 	 * file data
@@ -184,11 +159,13 @@ struct putfile_request {
 	/* data not yet read within current putfile_buffer_entry */
 	struct putfile_active_buffer active_buf;
 
-	/* <0 = failed, 0 = in-progress, >0 = successful; */
-	/* note that this must be set with req_list_lock, and if you set <0, */
-	/* it is your responsibility to also free up all of the other objects */
-	/* in this struct (like input_buffer_list, active_buf.parser_ctx) */
-	/* before releasing the lock */
+	/*
+	 * <0 = failed, 0 = in-progress, >0 = successful;
+	 * note that this must be set with req_list_lock, and if you set <0,
+	 * it is your responsibility to also free up all of the other objects
+	 * in this struct (like input_buffer_list, active_buf.parser_ctx)
+	 * before releasing the lock
+	 */
 	int completion_status;
 };
 
@@ -199,289 +176,11 @@ struct parahotplug_request {
 	struct controlvm_message msg;
 };
 
-static LIST_HEAD(parahotplug_request_list);
-static DEFINE_SPINLOCK(parahotplug_request_list_lock);	/* lock for above */
-static void parahotplug_process_list(void);
-
-/* Manages the info for a CONTROLVM_DUMP_CAPTURESTATE /
- * CONTROLVM_REPORTEVENT.
- */
-static struct visorchipset_busdev_notifiers busdev_notifiers;
-
-static void bus_create_response(struct visor_device *p, int response);
-static void bus_destroy_response(struct visor_device *p, int response);
-static void device_create_response(struct visor_device *p, int response);
-static void device_destroy_response(struct visor_device *p, int response);
-static void device_resume_response(struct visor_device *p, int response);
-
-static void visorchipset_device_pause_response(struct visor_device *p,
-					       int response);
-
-static struct visorchipset_busdev_responders busdev_responders = {
-	.bus_create = bus_create_response,
-	.bus_destroy = bus_destroy_response,
-	.device_create = device_create_response,
-	.device_destroy = device_destroy_response,
-	.device_pause = visorchipset_device_pause_response,
-	.device_resume = device_resume_response,
-};
-
 /* info for /dev/visorchipset */
-static dev_t major_dev = -1; /**< indicates major num for device */
+static dev_t major_dev = -1; /*< indicates major num for device */
 
 /* prototypes for attributes */
 static ssize_t toolaction_show(struct device *dev,
-			       struct device_attribute *attr, char *buf);
-static ssize_t toolaction_store(struct device *dev,
-				struct device_attribute *attr,
-				const char *buf, size_t count);
-static DEVICE_ATTR_RW(toolaction);
-
-static ssize_t boottotool_show(struct device *dev,
-			       struct device_attribute *attr, char *buf);
-static ssize_t boottotool_store(struct device *dev,
-				struct device_attribute *attr, const char *buf,
-				size_t count);
-static DEVICE_ATTR_RW(boottotool);
-
-static ssize_t error_show(struct device *dev, struct device_attribute *attr,
-			  char *buf);
-static ssize_t error_store(struct device *dev, struct device_attribute *attr,
-			   const char *buf, size_t count);
-static DEVICE_ATTR_RW(error);
-
-static ssize_t textid_show(struct device *dev, struct device_attribute *attr,
-			   char *buf);
-static ssize_t textid_store(struct device *dev, struct device_attribute *attr,
-			    const char *buf, size_t count);
-static DEVICE_ATTR_RW(textid);
-
-static ssize_t remaining_steps_show(struct device *dev,
-				    struct device_attribute *attr, char *buf);
-static ssize_t remaining_steps_store(struct device *dev,
-				     struct device_attribute *attr,
-				     const char *buf, size_t count);
-static DEVICE_ATTR_RW(remaining_steps);
-
-static ssize_t devicedisabled_store(struct device *dev,
-				    struct device_attribute *attr,
-				    const char *buf, size_t count);
-static DEVICE_ATTR_WO(devicedisabled);
-
-static ssize_t deviceenabled_store(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf, size_t count);
-static DEVICE_ATTR_WO(deviceenabled);
-
-static struct attribute *visorchipset_install_attrs[] = {
-	&dev_attr_toolaction.attr,
-	&dev_attr_boottotool.attr,
-	&dev_attr_error.attr,
-	&dev_attr_textid.attr,
-	&dev_attr_remaining_steps.attr,
-	NULL
-};
-
-static struct attribute_group visorchipset_install_group = {
-	.name = "install",
-	.attrs = visorchipset_install_attrs
-};
-
-static struct attribute *visorchipset_parahotplug_attrs[] = {
-	&dev_attr_devicedisabled.attr,
-	&dev_attr_deviceenabled.attr,
-	NULL
-};
-
-static struct attribute_group visorchipset_parahotplug_group = {
-	.name = "parahotplug",
-	.attrs = visorchipset_parahotplug_attrs
-};
-
-static const struct attribute_group *visorchipset_dev_groups[] = {
-	&visorchipset_install_group,
-	&visorchipset_parahotplug_group,
-	NULL
-};
-
-static void visorchipset_dev_release(struct device *dev)
-{
-}
-
-/* /sys/devices/platform/visorchipset */
-static struct platform_device visorchipset_platform_device = {
-	.name = "visorchipset",
-	.id = -1,
-	.dev.groups = visorchipset_dev_groups,
-	.dev.release = visorchipset_dev_release,
-};
-
-/* Function prototypes */
-static void controlvm_respond(struct controlvm_message_header *msg_hdr,
-			      int response);
-static void controlvm_respond_chipset_init(
-		struct controlvm_message_header *msg_hdr, int response,
-		enum ultra_chipset_feature features);
-static void controlvm_respond_physdev_changestate(
-		struct controlvm_message_header *msg_hdr, int response,
-		struct spar_segment_state state);
-
-static void parser_done(struct parser_context *ctx);
-
-static struct parser_context *
-parser_init_byte_stream(u64 addr, u32 bytes, bool local, bool *retry)
-{
-	int allocbytes = sizeof(struct parser_context) + bytes;
-	struct parser_context *ctx;
-
-	if (retry)
-		*retry = false;
-
-	/*
-	 * alloc an 0 extra byte to ensure payload is
-	 * '\0'-terminated
-	 */
-	allocbytes++;
-	if ((controlvm_payload_bytes_buffered + bytes)
-	    > MAX_CONTROLVM_PAYLOAD_BYTES) {
-		if (retry)
-			*retry = true;
-		return NULL;
-	}
-	ctx = kzalloc(allocbytes, GFP_KERNEL | __GFP_NORETRY);
-	if (!ctx) {
-		if (retry)
-			*retry = true;
-		return NULL;
-	}
-
-	ctx->allocbytes = allocbytes;
-	ctx->param_bytes = bytes;
-	ctx->curr = NULL;
-	ctx->bytes_remaining = 0;
-	ctx->byte_stream = false;
-	if (local) {
-		void *p;
-
-		if (addr > virt_to_phys(high_memory - 1))
-			goto err_finish_ctx;
-		p = __va((unsigned long)(addr));
-		memcpy(ctx->data, p, bytes);
-	} else {
-		void *mapping = memremap(addr, bytes, MEMREMAP_WB);
-
-		if (!mapping)
-			goto err_finish_ctx;
-		memcpy(ctx->data, mapping, bytes);
-		memunmap(mapping);
-	}
-
-	ctx->byte_stream = true;
-	controlvm_payload_bytes_buffered += ctx->param_bytes;
-
-	return ctx;
-
-err_finish_ctx:
-	parser_done(ctx);
-	return NULL;
-}
-
-static uuid_le
-parser_id_get(struct parser_context *ctx)
-{
-	struct spar_controlvm_parameters_header *phdr = NULL;
-
-	if (!ctx)
-		return NULL_UUID_LE;
-	phdr = (struct spar_controlvm_parameters_header *)(ctx->data);
-	return phdr->id;
-}
-
-/** Describes the state from the perspective of which controlvm messages have
- *  been received for a bus or device.
- */
-
-enum PARSER_WHICH_STRING {
-	PARSERSTRING_INITIATOR,
-	PARSERSTRING_TARGET,
-	PARSERSTRING_CONNECTION,
-	PARSERSTRING_NAME, /* TODO: only PARSERSTRING_NAME is used ? */
-};
-
-static void
-parser_param_start(struct parser_context *ctx,
-		   enum PARSER_WHICH_STRING which_string)
-{
-	struct spar_controlvm_parameters_header *phdr = NULL;
-
-	if (!ctx)
-		return;
-
-	phdr = (struct spar_controlvm_parameters_header *)(ctx->data);
-	switch (which_string) {
-	case PARSERSTRING_INITIATOR:
-		ctx->curr = ctx->data + phdr->initiator_offset;
-		ctx->bytes_remaining = phdr->initiator_length;
-		break;
-	case PARSERSTRING_TARGET:
-		ctx->curr = ctx->data + phdr->target_offset;
-		ctx->bytes_remaining = phdr->target_length;
-		break;
-	case PARSERSTRING_CONNECTION:
-		ctx->curr = ctx->data + phdr->connection_offset;
-		ctx->bytes_remaining = phdr->connection_length;
-		break;
-	case PARSERSTRING_NAME:
-		ctx->curr = ctx->data + phdr->name_offset;
-		ctx->bytes_remaining = phdr->name_length;
-		break;
-	default:
-		break;
-	}
-}
-
-static void parser_done(struct parser_context *ctx)
-{
-	if (!ctx)
-		return;
-	controlvm_payload_bytes_buffered -= ctx->param_bytes;
-	kfree(ctx);
-}
-
-static void *
-parser_string_get(struct parser_context *ctx)
-{
-	u8 *pscan;
-	unsigned long nscan;
-	int value_length = -1;
-	void *value = NULL;
-	int i;
-
-	if (!ctx)
-		return NULL;
-	pscan = ctx->curr;
-	nscan = ctx->bytes_remaining;
-	if (nscan == 0)
-		return NULL;
-	if (!pscan)
-		return NULL;
-	for (i = 0, value_length = -1; i < nscan; i++)
-		if (pscan[i] == '\0') {
-			value_length = i;
-			break;
-		}
-	if (value_length < 0)	/* '\0' was not included in the length */
-		value_length = nscan;
-	value = kmalloc(value_length + 1, GFP_KERNEL | __GFP_NORETRY);
-	if (!value)
-		return NULL;
-	if (value_length > 0)
-		memcpy(value, pscan, value_length);
-	((u8 *)(value))[value_length] = '\0';
-	return value;
-}
-
-static ssize_t toolaction_show(struct device *dev,
 			       struct device_attribute *attr,
 			       char *buf)
 {
@@ -513,6 +212,7 @@ static ssize_t toolaction_store(struct device *dev,
 		return ret;
 	return count;
 }
+static DEVICE_ATTR_RW(toolaction);
 
 static ssize_t boottotool_show(struct device *dev,
 			       struct device_attribute *attr,
@@ -549,6 +249,7 @@ static ssize_t boottotool_store(struct device *dev,
 		return ret;
 	return count;
 }
+static DEVICE_ATTR_RW(boottotool);
 
 static ssize_t error_show(struct device *dev, struct device_attribute *attr,
 			  char *buf)
@@ -580,6 +281,7 @@ static ssize_t error_store(struct device *dev, struct device_attribute *attr,
 		return ret;
 	return count;
 }
+static DEVICE_ATTR_RW(error);
 
 static ssize_t textid_show(struct device *dev, struct device_attribute *attr,
 			   char *buf)
@@ -612,6 +314,7 @@ static ssize_t textid_store(struct device *dev, struct device_attribute *attr,
 		return ret;
 	return count;
 }
+static DEVICE_ATTR_RW(textid);
 
 static ssize_t remaining_steps_show(struct device *dev,
 				    struct device_attribute *attr, char *buf)
@@ -644,6 +347,103 @@ static ssize_t remaining_steps_store(struct device *dev,
 		return ret;
 	return count;
 }
+static DEVICE_ATTR_RW(remaining_steps);
+
+static uuid_le
+parser_id_get(struct parser_context *ctx)
+{
+	struct spar_controlvm_parameters_header *phdr = NULL;
+
+	if (!ctx)
+		return NULL_UUID_LE;
+	phdr = (struct spar_controlvm_parameters_header *)(ctx->data);
+	return phdr->id;
+}
+
+/*
+ * Describes the state from the perspective of which controlvm messages have
+ * been received for a bus or device.
+ */
+
+enum PARSER_WHICH_STRING {
+	PARSERSTRING_INITIATOR,
+	PARSERSTRING_TARGET,
+	PARSERSTRING_CONNECTION,
+	PARSERSTRING_NAME, /* TODO: only PARSERSTRING_NAME is used ? */
+};
+
+static void
+parser_param_start(struct parser_context *ctx,
+		   enum PARSER_WHICH_STRING which_string)
+{
+	struct spar_controlvm_parameters_header *phdr = NULL;
+
+	if (!ctx)
+		return;
+
+	phdr = (struct spar_controlvm_parameters_header *)(ctx->data);
+	switch (which_string) {
+	case PARSERSTRING_INITIATOR:
+		ctx->curr = ctx->data + phdr->initiator_offset;
+		ctx->bytes_remaining = phdr->initiator_length;
+		break;
+	case PARSERSTRING_TARGET:
+		ctx->curr = ctx->data + phdr->target_offset;
+		ctx->bytes_remaining = phdr->target_length;
+		break;
+	case PARSERSTRING_CONNECTION:
+		ctx->curr = ctx->data + phdr->connection_offset;
+		ctx->bytes_remaining = phdr->connection_length;
+		break;
+	case PARSERSTRING_NAME:
+		ctx->curr = ctx->data + phdr->name_offset;
+		ctx->bytes_remaining = phdr->name_length;
+		break;
+	default:
+		break;
+	}
+}
+
+static void parser_done(struct parser_context *ctx)
+{
+	if (!ctx)
+		return;
+	controlvm_payload_bytes_buffered -= ctx->param_bytes;
+	kfree(ctx);
+}
+
+static void *
+parser_string_get(struct parser_context *ctx)
+{
+	u8 *pscan;
+	unsigned long nscan;
+	int value_length = -1;
+	void *value = NULL;
+	int i;
+
+	if (!ctx)
+		return NULL;
+	pscan = ctx->curr;
+	nscan = ctx->bytes_remaining;
+	if (nscan == 0)
+		return NULL;
+	if (!pscan)
+		return NULL;
+	for (i = 0, value_length = -1; i < nscan; i++)
+		if (pscan[i] == '\0') {
+			value_length = i;
+			break;
+		}
+	if (value_length < 0)	/* '\0' was not included in the length */
+		value_length = nscan;
+	value = kmalloc(value_length + 1, GFP_KERNEL | __GFP_NORETRY);
+	if (!value)
+		return NULL;
+	if (value_length > 0)
+		memcpy(value, pscan, value_length);
+	((u8 *)(value))[value_length] = '\0';
+	return value;
+}
 
 struct visor_busdev {
 	u32 bus_no;
@@ -683,32 +483,36 @@ struct visor_device *visorbus_get_device_by_id(u32 bus_no, u32 dev_no,
 		vdev = to_visor_device(dev);
 	return vdev;
 }
-EXPORT_SYMBOL(visorbus_get_device_by_id);
 
-void
-visorchipset_register_busdev(
-			struct visorchipset_busdev_notifiers *notifiers,
-			struct visorchipset_busdev_responders *responders,
-			struct ultra_vbus_deviceinfo *driver_info)
-{
-	down(&notifier_lock);
-	if (!notifiers) {
-		memset(&busdev_notifiers, 0,
-		       sizeof(busdev_notifiers));
-		visorbusregistered = 0;	/* clear flag */
-	} else {
-		busdev_notifiers = *notifiers;
-		visorbusregistered = 1;	/* set flag */
+static void
+controlvm_init_response(struct controlvm_message *msg,
+			struct controlvm_message_header *msg_hdr, int response)
+{
+	memset(msg, 0, sizeof(struct controlvm_message));
+	memcpy(&msg->hdr, msg_hdr, sizeof(struct controlvm_message_header));
+	msg->hdr.payload_bytes = 0;
+	msg->hdr.payload_vm_offset = 0;
+	msg->hdr.payload_max_bytes = 0;
+	if (response < 0) {
+		msg->hdr.flags.failed = 1;
+		msg->hdr.completion_status = (u32)(-response);
 	}
-	if (responders)
-		*responders = busdev_responders;
-	if (driver_info)
-		bus_device_info_init(driver_info, "chipset", "visorchipset",
-				     VERSION, NULL);
+}
+
+static void
+controlvm_respond_chipset_init(struct controlvm_message_header *msg_hdr,
+			       int response,
+			       enum ultra_chipset_feature features)
+{
+	struct controlvm_message outmsg;
 
-	up(&notifier_lock);
+	controlvm_init_response(&outmsg, msg_hdr, response);
+	outmsg.cmd.init_chipset.features = features;
+	if (visorchannel_signalinsert(controlvm_channel,
+				      CONTROLVM_QUEUE_REQUEST, &outmsg)) {
+		return;
+	}
 }
-EXPORT_SYMBOL_GPL(visorchipset_register_busdev);
 
 static void
 chipset_init(struct controlvm_message *inmsg)
@@ -725,14 +529,16 @@ chipset_init(struct controlvm_message *inmsg)
 	chipset_inited = 1;
 	POSTCODE_LINUX_2(CHIPSET_INIT_EXIT_PC, POSTCODE_SEVERITY_INFO);
 
-	/* Set features to indicate we support parahotplug (if Command
+	/*
+	 * Set features to indicate we support parahotplug (if Command
 	 * also supports it).
 	 */
 	features =
 	    inmsg->cmd.init_chipset.
 	    features & ULTRA_CHIPSET_FEATURE_PARA_HOTPLUG;
 
-	/* Set the "reply" bit so Command knows this is a
+	/*
+	 * Set the "reply" bit so Command knows this is a
 	 * features-aware driver.
 	 */
 	features |= ULTRA_CHIPSET_FEATURE_REPLY;
@@ -743,21 +549,6 @@ out_respond:
 }
 
 static void
-controlvm_init_response(struct controlvm_message *msg,
-			struct controlvm_message_header *msg_hdr, int response)
-{
-	memset(msg, 0, sizeof(struct controlvm_message));
-	memcpy(&msg->hdr, msg_hdr, sizeof(struct controlvm_message_header));
-	msg->hdr.payload_bytes = 0;
-	msg->hdr.payload_vm_offset = 0;
-	msg->hdr.payload_max_bytes = 0;
-	if (response < 0) {
-		msg->hdr.flags.failed = 1;
-		msg->hdr.completion_status = (u32)(-response);
-	}
-}
-
-static void
 controlvm_respond(struct controlvm_message_header *msg_hdr, int response)
 {
 	struct controlvm_message outmsg;
@@ -766,23 +557,8 @@ controlvm_respond(struct controlvm_message_header *msg_hdr, int response)
 	if (outmsg.hdr.flags.test_message == 1)
 		return;
 
-	if (!visorchannel_signalinsert(controlvm_channel,
-				       CONTROLVM_QUEUE_REQUEST, &outmsg)) {
-		return;
-	}
-}
-
-static void
-controlvm_respond_chipset_init(struct controlvm_message_header *msg_hdr,
-			       int response,
-			       enum ultra_chipset_feature features)
-{
-	struct controlvm_message outmsg;
-
-	controlvm_init_response(&outmsg, msg_hdr, response);
-	outmsg.cmd.init_chipset.features = features;
-	if (!visorchannel_signalinsert(controlvm_channel,
-				       CONTROLVM_QUEUE_REQUEST, &outmsg)) {
+	if (visorchannel_signalinsert(controlvm_channel,
+				      CONTROLVM_QUEUE_REQUEST, &outmsg)) {
 		return;
 	}
 }
@@ -796,8 +572,8 @@ static void controlvm_respond_physdev_changestate(
 	controlvm_init_response(&outmsg, msg_hdr, response);
 	outmsg.cmd.device_change_state.state = state;
 	outmsg.cmd.device_change_state.flags.phys_device = 1;
-	if (!visorchannel_signalinsert(controlvm_channel,
-				       CONTROLVM_QUEUE_REQUEST, &outmsg)) {
+	if (visorchannel_signalinsert(controlvm_channel,
+				      CONTROLVM_QUEUE_REQUEST, &outmsg)) {
 		return;
 	}
 }
@@ -894,8 +670,8 @@ device_changestate_responder(enum controlvm_id cmd_id,
 	outmsg.cmd.device_change_state.dev_no = dev_no;
 	outmsg.cmd.device_change_state.state = response_state;
 
-	if (!visorchannel_signalinsert(controlvm_channel,
-				       CONTROLVM_QUEUE_REQUEST, &outmsg))
+	if (visorchannel_signalinsert(controlvm_channel,
+				      CONTROLVM_QUEUE_REQUEST, &outmsg))
 		return;
 }
 
@@ -920,20 +696,20 @@ bus_epilog(struct visor_device *bus_info,
 {
 	struct controlvm_message_header *pmsg_hdr = NULL;
 
-	down(&notifier_lock);
-
 	if (!bus_info) {
-		/* relying on a valid passed in response code */
-		/* be lazy and re-use msg_hdr for this failure, is this ok?? */
+		/*
+		 * relying on a valid passed in response code
+		 * be lazy and re-use msg_hdr for this failure, is this ok??
+		 */
 		pmsg_hdr = msg_hdr;
-		goto out_respond_and_unlock;
+		goto out_respond;
 	}
 
 	if (bus_info->pending_msg_hdr) {
 		/* only non-NULL if dev is still waiting on a response */
 		response = -CONTROLVM_RESP_ERROR_MESSAGE_ID_INVALID_FOR_CLIENT;
 		pmsg_hdr = bus_info->pending_msg_hdr;
-		goto out_respond_and_unlock;
+		goto out_respond;
 	}
 
 	if (need_response) {
@@ -942,7 +718,7 @@ bus_epilog(struct visor_device *bus_info,
 			POSTCODE_LINUX_4(MALLOC_FAILURE_PC, cmd,
 					 bus_info->chipset_bus_no,
 					 POSTCODE_SEVERITY_ERR);
-			goto out_unlock;
+			return;
 		}
 
 		memcpy(pmsg_hdr, msg_hdr,
@@ -953,25 +729,16 @@ bus_epilog(struct visor_device *bus_info,
 	if (response == CONTROLVM_RESP_SUCCESS) {
 		switch (cmd) {
 		case CONTROLVM_BUS_CREATE:
-			if (busdev_notifiers.bus_create) {
-				(*busdev_notifiers.bus_create) (bus_info);
-				goto out_unlock;
-			}
+			chipset_bus_create(bus_info);
 			break;
 		case CONTROLVM_BUS_DESTROY:
-			if (busdev_notifiers.bus_destroy) {
-				(*busdev_notifiers.bus_destroy) (bus_info);
-				goto out_unlock;
-			}
+			chipset_bus_destroy(bus_info);
 			break;
 		}
 	}
 
-out_respond_and_unlock:
+out_respond:
 	bus_responder(cmd, pmsg_hdr, response);
-
-out_unlock:
-	up(&notifier_lock);
 }
 
 static void
@@ -980,31 +747,29 @@ device_epilog(struct visor_device *dev_info,
 	      struct controlvm_message_header *msg_hdr, int response,
 	      bool need_response, bool for_visorbus)
 {
-	struct visorchipset_busdev_notifiers *notifiers;
 	struct controlvm_message_header *pmsg_hdr = NULL;
 
-	notifiers = &busdev_notifiers;
-
-	down(&notifier_lock);
 	if (!dev_info) {
-		/* relying on a valid passed in response code */
-		/* be lazy and re-use msg_hdr for this failure, is this ok?? */
+		/*
+		 * relying on a valid passed in response code
+		 * be lazy and re-use msg_hdr for this failure, is this ok??
+		 */
 		pmsg_hdr = msg_hdr;
-		goto out_respond_and_unlock;
+		goto out_respond;
 	}
 
 	if (dev_info->pending_msg_hdr) {
 		/* only non-NULL if dev is still waiting on a response */
 		response = -CONTROLVM_RESP_ERROR_MESSAGE_ID_INVALID_FOR_CLIENT;
 		pmsg_hdr = dev_info->pending_msg_hdr;
-		goto out_respond_and_unlock;
+		goto out_respond;
 	}
 
 	if (need_response) {
 		pmsg_hdr = kzalloc(sizeof(*pmsg_hdr), GFP_KERNEL);
 		if (!pmsg_hdr) {
 			response = -CONTROLVM_RESP_ERROR_KMALLOC_FAILED;
-			goto out_respond_and_unlock;
+			goto out_respond;
 		}
 
 		memcpy(pmsg_hdr, msg_hdr,
@@ -1015,48 +780,34 @@ device_epilog(struct visor_device *dev_info,
 	if (response >= 0) {
 		switch (cmd) {
 		case CONTROLVM_DEVICE_CREATE:
-			if (notifiers->device_create) {
-				(*notifiers->device_create) (dev_info);
-				goto out_unlock;
-			}
+			chipset_device_create(dev_info);
 			break;
 		case CONTROLVM_DEVICE_CHANGESTATE:
 			/* ServerReady / ServerRunning / SegmentStateRunning */
 			if (state.alive == segment_state_running.alive &&
 			    state.operating ==
 				segment_state_running.operating) {
-				if (notifiers->device_resume) {
-					(*notifiers->device_resume) (dev_info);
-					goto out_unlock;
-				}
+				chipset_device_resume(dev_info);
 			}
 			/* ServerNotReady / ServerLost / SegmentStateStandby */
 			else if (state.alive == segment_state_standby.alive &&
 				 state.operating ==
 				 segment_state_standby.operating) {
-				/* technically this is standby case
+				/*
+				 * technically this is standby case
 				 * where server is lost
 				 */
-				if (notifiers->device_pause) {
-					(*notifiers->device_pause) (dev_info);
-					goto out_unlock;
-				}
+				chipset_device_pause(dev_info);
 			}
 			break;
 		case CONTROLVM_DEVICE_DESTROY:
-			if (notifiers->device_destroy) {
-				(*notifiers->device_destroy) (dev_info);
-				goto out_unlock;
-			}
+			chipset_device_destroy(dev_info);
 			break;
 		}
 	}
 
-out_respond_and_unlock:
+out_respond:
 	device_responder(cmd, pmsg_hdr, response);
-
-out_unlock:
-	up(&notifier_lock);
 }
 
 static void
@@ -1103,10 +854,8 @@ bus_create(struct controlvm_message *inmsg)
 		goto out_bus_epilog;
 	}
 	bus_info->visorchannel = visorchannel;
-	if (uuid_le_cmp(cmd->create_bus.bus_inst_uuid, spar_siovm_uuid) == 0) {
-		dump_vhba_bus = bus_no;
+	if (uuid_le_cmp(cmd->create_bus.bus_inst_uuid, spar_siovm_uuid) == 0)
 		save_crash_message(inmsg, CRASH_BUS);
-	}
 
 	POSTCODE_LINUX_3(BUS_CREATE_EXIT_PC, bus_no, POSTCODE_SEVERITY_INFO);
 
@@ -1303,11 +1052,19 @@ my_device_destroy(struct controlvm_message *inmsg)
 			      inmsg->hdr.flags.response_expected == 1, 1);
 }
 
-/* When provided with the physical address of the controlvm channel
+/**
+ * initialize_controlvm_payload_info() - init controlvm_payload_info struct
+ * @phys_addr: the physical address of controlvm channel
+ * @offset:    the offset to payload
+ * @bytes:     the size of the payload in bytes
+ * @info:      the returning valid struct
+ *
+ * When provided with the physical address of the controlvm channel
  * (phys_addr), the offset to the payload area we need to manage
  * (offset), and the size of this payload area (bytes), fills in the
- * controlvm_payload_info struct.  Returns true for success or false
- * for failure.
+ * controlvm_payload_info struct.
+ *
+ * Return: CONTROLVM_RESP_SUCCESS for success or a negative for failure
  */
 static int
 initialize_controlvm_payload_info(u64 phys_addr, u64 offset, u32 bytes,
@@ -1371,95 +1128,14 @@ initialize_controlvm_payload(void)
 					  &controlvm_payload_info);
 }
 
-/*  Send ACTION=online for DEVPATH=/sys/devices/platform/visorchipset.
- *  Returns CONTROLVM_RESP_xxx code.
- */
-static int
-visorchipset_chipset_ready(void)
-{
-	kobject_uevent(&visorchipset_platform_device.dev.kobj, KOBJ_ONLINE);
-	return CONTROLVM_RESP_SUCCESS;
-}
-
-static int
-visorchipset_chipset_selftest(void)
-{
-	char env_selftest[20];
-	char *envp[] = { env_selftest, NULL };
-
-	sprintf(env_selftest, "SPARSP_SELFTEST=%d", 1);
-	kobject_uevent_env(&visorchipset_platform_device.dev.kobj, KOBJ_CHANGE,
-			   envp);
-	return CONTROLVM_RESP_SUCCESS;
-}
-
-/*  Send ACTION=offline for DEVPATH=/sys/devices/platform/visorchipset.
- *  Returns CONTROLVM_RESP_xxx code.
- */
-static int
-visorchipset_chipset_notready(void)
-{
-	kobject_uevent(&visorchipset_platform_device.dev.kobj, KOBJ_OFFLINE);
-	return CONTROLVM_RESP_SUCCESS;
-}
-
-static void
-chipset_ready(struct controlvm_message_header *msg_hdr)
-{
-	int rc = visorchipset_chipset_ready();
-
-	if (rc != CONTROLVM_RESP_SUCCESS)
-		rc = -rc;
-	if (msg_hdr->flags.response_expected)
-		controlvm_respond(msg_hdr, rc);
-}
-
-static void
-chipset_selftest(struct controlvm_message_header *msg_hdr)
-{
-	int rc = visorchipset_chipset_selftest();
-
-	if (rc != CONTROLVM_RESP_SUCCESS)
-		rc = -rc;
-	if (msg_hdr->flags.response_expected)
-		controlvm_respond(msg_hdr, rc);
-}
-
-static void
-chipset_notready(struct controlvm_message_header *msg_hdr)
-{
-	int rc = visorchipset_chipset_notready();
-
-	if (rc != CONTROLVM_RESP_SUCCESS)
-		rc = -rc;
-	if (msg_hdr->flags.response_expected)
-		controlvm_respond(msg_hdr, rc);
-}
-
-/* This is your "one-stop" shop for grabbing the next message from the
- * CONTROLVM_QUEUE_EVENT queue in the controlvm channel.
- */
-static bool
-read_controlvm_event(struct controlvm_message *msg)
-{
-	if (visorchannel_signalremove(controlvm_channel,
-				      CONTROLVM_QUEUE_EVENT, msg)) {
-		/* got a message */
-		if (msg->hdr.flags.test_message == 1)
-			return false;
-		return true;
-	}
-	return false;
-}
-
 /*
- * The general parahotplug flow works as follows.  The visorchipset
+ * The general parahotplug flow works as follows. The visorchipset
  * driver receives a DEVICE_CHANGESTATE message from Command
- * specifying a physical device to enable or disable.  The CONTROLVM
+ * specifying a physical device to enable or disable. The CONTROLVM
  * message handler calls parahotplug_process_message, which then adds
  * the message to a global list and kicks off a udev event which
  * causes a user level script to enable or disable the specified
- * device.  The udev script then writes to
+ * device. The udev script then writes to
  * /proc/visorchipset/parahotplug, which causes parahotplug_proc_write
  * to get called, at which point the appropriate CONTROLVM message is
  * retrieved from the list and responded to.
@@ -1467,9 +1143,11 @@ read_controlvm_event(struct controlvm_message *msg)
 
 #define PARAHOTPLUG_TIMEOUT_MS 2000
 
-/*
- * Generate unique int to match an outstanding CONTROLVM message with a
- * udev script /proc response
+/**
+ * parahotplug_next_id() - generate unique int to match an outstanding CONTROLVM
+ *                         message with a udev script /proc response
+ *
+ * Return: a unique integer value
  */
 static int
 parahotplug_next_id(void)
@@ -1479,9 +1157,12 @@ parahotplug_next_id(void)
 	return atomic_inc_return(&id);
 }
 
-/*
- * Returns the time (in jiffies) when a CONTROLVM message on the list
- * should expire -- PARAHOTPLUG_TIMEOUT_MS in the future
+/**
+ * parahotplug_next_expiration() - returns the time (in jiffies) when a
+ *                                 CONTROLVM message on the list should expire
+ *                                 -- PARAHOTPLUG_TIMEOUT_MS in the future
+ *
+ * Return: expected expiration time (in jiffies)
  */
 static unsigned long
 parahotplug_next_expiration(void)
@@ -1489,9 +1170,13 @@ parahotplug_next_expiration(void)
 	return jiffies + msecs_to_jiffies(PARAHOTPLUG_TIMEOUT_MS);
 }
 
-/*
- * Create a parahotplug_request, which is basically a wrapper for a
- * CONTROLVM_MESSAGE that we can stick on a list
+/**
+ * parahotplug_request_create() - create a parahotplug_request, which is
+ *                                basically a wrapper for a CONTROLVM_MESSAGE
+ *                                that we can stick on a list
+ * @msg: the message to insert in the request
+ *
+ * Return: the request containing the provided message
  */
 static struct parahotplug_request *
 parahotplug_request_create(struct controlvm_message *msg)
@@ -1509,8 +1194,9 @@ parahotplug_request_create(struct controlvm_message *msg)
 	return req;
 }
 
-/*
- * Free a parahotplug_request.
+/**
+ * parahotplug_request_destroy() - free a parahotplug_request
+ * @req: the request to deallocate
  */
 static void
 parahotplug_request_destroy(struct parahotplug_request *req)
@@ -1518,71 +1204,19 @@ parahotplug_request_destroy(struct parahotplug_request *req)
 	kfree(req);
 }
 
-/*
- * Cause uevent to run the user level script to do the disable/enable
- * specified in (the CONTROLVM message in) the specified
- * parahotplug_request
- */
-static void
-parahotplug_request_kickoff(struct parahotplug_request *req)
-{
-	struct controlvm_message_packet *cmd = &req->msg.cmd;
-	char env_cmd[40], env_id[40], env_state[40], env_bus[40], env_dev[40],
-	    env_func[40];
-	char *envp[] = {
-		env_cmd, env_id, env_state, env_bus, env_dev, env_func, NULL
-	};
-
-	sprintf(env_cmd, "SPAR_PARAHOTPLUG=1");
-	sprintf(env_id, "SPAR_PARAHOTPLUG_ID=%d", req->id);
-	sprintf(env_state, "SPAR_PARAHOTPLUG_STATE=%d",
-		cmd->device_change_state.state.active);
-	sprintf(env_bus, "SPAR_PARAHOTPLUG_BUS=%d",
-		cmd->device_change_state.bus_no);
-	sprintf(env_dev, "SPAR_PARAHOTPLUG_DEVICE=%d",
-		cmd->device_change_state.dev_no >> 3);
-	sprintf(env_func, "SPAR_PARAHOTPLUG_FUNCTION=%d",
-		cmd->device_change_state.dev_no & 0x7);
-
-	kobject_uevent_env(&visorchipset_platform_device.dev.kobj, KOBJ_CHANGE,
-			   envp);
-}
-
-/*
- * Remove any request from the list that's been on there too long and
- * respond with an error.
- */
-static void
-parahotplug_process_list(void)
-{
-	struct list_head *pos;
-	struct list_head *tmp;
-
-	spin_lock(&parahotplug_request_list_lock);
-
-	list_for_each_safe(pos, tmp, &parahotplug_request_list) {
-		struct parahotplug_request *req =
-		    list_entry(pos, struct parahotplug_request, list);
-
-		if (!time_after_eq(jiffies, req->expiration))
-			continue;
-
-		list_del(pos);
-		if (req->msg.hdr.flags.response_expected)
-			controlvm_respond_physdev_changestate(
-				&req->msg.hdr,
-				CONTROLVM_RESP_ERROR_DEVICE_UDEV_TIMEOUT,
-				req->msg.cmd.device_change_state.state);
-		parahotplug_request_destroy(req);
-	}
-
-	spin_unlock(&parahotplug_request_list_lock);
-}
+static LIST_HEAD(parahotplug_request_list);
+static DEFINE_SPINLOCK(parahotplug_request_list_lock);	/* lock for above */
 
-/*
+/**
+ * parahotplug_request_complete() - mark request as complete
+ * @id:     the id of the request
+ * @active: indicates whether the request is assigned to active partition
+ *
  * Called from the /proc handler, which means the user script has
- * finished the enable/disable.  Find the matching identifier, and
+ * finished the enable/disable. Find the matching identifier, and
  * respond to the CONTROLVM message with success.
+ *
+ * Return: 0 on success or -EINVAL on failure
  */
 static int
 parahotplug_request_complete(int id, u16 active)
@@ -1597,7 +1231,8 @@ parahotplug_request_complete(int id, u16 active)
 		struct parahotplug_request *req =
 		    list_entry(pos, struct parahotplug_request, list);
 		if (req->id == id) {
-			/* Found a match.  Remove it from the list and
+			/*
+			 * Found a match. Remove it from the list and
 			 * respond.
 			 */
 			list_del(pos);
@@ -1616,8 +1251,142 @@ parahotplug_request_complete(int id, u16 active)
 	return -EINVAL;
 }
 
-/*
- * Enables or disables a PCI device by kicking off a udev script
+/**
+ * devicedisabled_store() - disables the hotplug device
+ * @dev:   sysfs interface variable not utilized in this function
+ * @attr:  sysfs interface variable not utilized in this function
+ * @buf:   buffer containing the device id
+ * @count: the size of the buffer
+ *
+ * The parahotplug/devicedisabled interface gets called by our support script
+ * when an SR-IOV device has been shut down. The ID is passed to the script
+ * and then passed back when the device has been removed.
+ *
+ * Return: the size of the buffer for success or negative for error
+ */
+static ssize_t devicedisabled_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	unsigned int id;
+	int err;
+
+	if (kstrtouint(buf, 10, &id))
+		return -EINVAL;
+
+	err = parahotplug_request_complete(id, 0);
+	if (err < 0)
+		return err;
+	return count;
+}
+static DEVICE_ATTR_WO(devicedisabled);
+
+/**
+ * deviceenabled_store() - enables the hotplug device
+ * @dev:   sysfs interface variable not utilized in this function
+ * @attr:  sysfs interface variable not utilized in this function
+ * @buf:   buffer containing the device id
+ * @count: the size of the buffer
+ *
+ * The parahotplug/deviceenabled interface gets called by our support script
+ * when an SR-IOV device has been recovered. The ID is passed to the script
+ * and then passed back when the device has been brought back up.
+ *
+ * Return: the size of the buffer for success or negative for error
+ */
+static ssize_t deviceenabled_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	unsigned int id;
+
+	if (kstrtouint(buf, 10, &id))
+		return -EINVAL;
+
+	parahotplug_request_complete(id, 1);
+	return count;
+}
+static DEVICE_ATTR_WO(deviceenabled);
+
+static struct attribute *visorchipset_install_attrs[] = {
+	&dev_attr_toolaction.attr,
+	&dev_attr_boottotool.attr,
+	&dev_attr_error.attr,
+	&dev_attr_textid.attr,
+	&dev_attr_remaining_steps.attr,
+	NULL
+};
+
+static struct attribute_group visorchipset_install_group = {
+	.name = "install",
+	.attrs = visorchipset_install_attrs
+};
+
+static struct attribute *visorchipset_parahotplug_attrs[] = {
+	&dev_attr_devicedisabled.attr,
+	&dev_attr_deviceenabled.attr,
+	NULL
+};
+
+static struct attribute_group visorchipset_parahotplug_group = {
+	.name = "parahotplug",
+	.attrs = visorchipset_parahotplug_attrs
+};
+
+static const struct attribute_group *visorchipset_dev_groups[] = {
+	&visorchipset_install_group,
+	&visorchipset_parahotplug_group,
+	NULL
+};
+
+static void visorchipset_dev_release(struct device *dev)
+{
+}
+
+/* /sys/devices/platform/visorchipset */
+static struct platform_device visorchipset_platform_device = {
+	.name = "visorchipset",
+	.id = -1,
+	.dev.groups = visorchipset_dev_groups,
+	.dev.release = visorchipset_dev_release,
+};
+
+/**
+ * parahotplug_request_kickoff() - initiate parahotplug request
+ * @req: the request to initiate
+ *
+ * Cause uevent to run the user level script to do the disable/enable specified
+ * in the parahotplug_request.
+ */
+static void
+parahotplug_request_kickoff(struct parahotplug_request *req)
+{
+	struct controlvm_message_packet *cmd = &req->msg.cmd;
+	char env_cmd[40], env_id[40], env_state[40], env_bus[40], env_dev[40],
+	    env_func[40];
+	char *envp[] = {
+		env_cmd, env_id, env_state, env_bus, env_dev, env_func, NULL
+	};
+
+	sprintf(env_cmd, "SPAR_PARAHOTPLUG=1");
+	sprintf(env_id, "SPAR_PARAHOTPLUG_ID=%d", req->id);
+	sprintf(env_state, "SPAR_PARAHOTPLUG_STATE=%d",
+		cmd->device_change_state.state.active);
+	sprintf(env_bus, "SPAR_PARAHOTPLUG_BUS=%d",
+		cmd->device_change_state.bus_no);
+	sprintf(env_dev, "SPAR_PARAHOTPLUG_DEVICE=%d",
+		cmd->device_change_state.dev_no >> 3);
+	sprintf(env_func, "SPAR_PARAHOTPLUG_FUNCTION=%d",
+		cmd->device_change_state.dev_no & 0x7);
+
+	kobject_uevent_env(&visorchipset_platform_device.dev.kobj, KOBJ_CHANGE,
+			   envp);
+}
+
+/**
+ * parahotplug_process_message() - enables or disables a PCI device by kicking
+ *                                 off a udev script
+ * @inmsg: the message indicating whether to enable or disable
  */
 static void
 parahotplug_process_message(struct controlvm_message *inmsg)
@@ -1630,14 +1399,16 @@ parahotplug_process_message(struct controlvm_message *inmsg)
 		return;
 
 	if (inmsg->cmd.device_change_state.state.active) {
-		/* For enable messages, just respond with success
-		* right away.  This is a bit of a hack, but there are
-		* issues with the early enable messages we get (with
-		* either the udev script not detecting that the device
-		* is up, or not getting called at all).  Fortunately
-		* the messages that get lost don't matter anyway, as
-		* devices are automatically enabled at
-		* initialization.
+		/*
+		 * For enable messages, just respond with success
+		 * right away. This is a bit of a hack, but there are
+		 * issues with the early enable messages we get (with
+		 * either the udev script not detecting that the device
+		 * is up, or not getting called at all). Fortunately
+		 * the messages that get lost don't matter anyway, as
+		 *
+		 * devices are automatically enabled at
+		 * initialization.
 		*/
 		parahotplug_request_kickoff(req);
 		controlvm_respond_physdev_changestate
@@ -1646,11 +1417,12 @@ parahotplug_process_message(struct controlvm_message *inmsg)
 			 inmsg->cmd.device_change_state.state);
 		parahotplug_request_destroy(req);
 	} else {
-		/* For disable messages, add the request to the
-		* request list before kicking off the udev script.  It
-		* won't get responded to until the script has
-		* indicated it's done.
-		*/
+		/*
+		 * For disable messages, add the request to the
+		 * request list before kicking off the udev script. It
+		 * won't get responded to until the script has
+		 * indicated it's done.
+		 */
 		spin_lock(&parahotplug_request_list_lock);
 		list_add_tail(&req->list, &parahotplug_request_list);
 		spin_unlock(&parahotplug_request_list_lock);
@@ -1659,113 +1431,77 @@ parahotplug_process_message(struct controlvm_message *inmsg)
 	}
 }
 
-/* Process a controlvm message.
- * Return result:
- *    false - this function will return false only in the case where the
- *            controlvm message was NOT processed, but processing must be
- *            retried before reading the next controlvm message; a
- *            scenario where this can occur is when we need to throttle
- *            the allocation of memory in which to copy out controlvm
- *            payload data
- *    true  - processing of the controlvm message completed,
- *            either successfully or with an error.
+/**
+ * visorchipset_chipset_ready() - sends chipset_ready action
+ *
+ * Send ACTION=online for DEVPATH=/sys/devices/platform/visorchipset.
+ *
+ * Return: CONTROLVM_RESP_SUCCESS
  */
-static bool
-handle_command(struct controlvm_message inmsg, u64 channel_addr)
+static int
+visorchipset_chipset_ready(void)
 {
-	struct controlvm_message_packet *cmd = &inmsg.cmd;
-	u64 parm_addr;
-	u32 parm_bytes;
-	struct parser_context *parser_ctx = NULL;
-	bool local_addr;
-	struct controlvm_message ackmsg;
+	kobject_uevent(&visorchipset_platform_device.dev.kobj, KOBJ_ONLINE);
+	return CONTROLVM_RESP_SUCCESS;
+}
 
-	/* create parsing context if necessary */
-	local_addr = (inmsg.hdr.flags.test_message == 1);
-	if (channel_addr == 0)
-		return true;
-	parm_addr = channel_addr + inmsg.hdr.payload_vm_offset;
-	parm_bytes = inmsg.hdr.payload_bytes;
+static int
+visorchipset_chipset_selftest(void)
+{
+	char env_selftest[20];
+	char *envp[] = { env_selftest, NULL };
 
-	/* Parameter and channel addresses within test messages actually lie
-	 * within our OS-controlled memory.  We need to know that, because it
-	 * makes a difference in how we compute the virtual address.
-	 */
-	if (parm_addr && parm_bytes) {
-		bool retry = false;
+	sprintf(env_selftest, "SPARSP_SELFTEST=%d", 1);
+	kobject_uevent_env(&visorchipset_platform_device.dev.kobj, KOBJ_CHANGE,
+			   envp);
+	return CONTROLVM_RESP_SUCCESS;
+}
 
-		parser_ctx =
-		    parser_init_byte_stream(parm_addr, parm_bytes,
-					    local_addr, &retry);
-		if (!parser_ctx && retry)
-			return false;
-	}
+/**
+ * visorchipset_chipset_notready() - sends chipset_notready action
+ *
+ * Send ACTION=offline for DEVPATH=/sys/devices/platform/visorchipset.
+ *
+ * Return: CONTROLVM_RESP_SUCCESS
+ */
+static int
+visorchipset_chipset_notready(void)
+{
+	kobject_uevent(&visorchipset_platform_device.dev.kobj, KOBJ_OFFLINE);
+	return CONTROLVM_RESP_SUCCESS;
+}
 
-	if (!local_addr) {
-		controlvm_init_response(&ackmsg, &inmsg.hdr,
-					CONTROLVM_RESP_SUCCESS);
-		if (controlvm_channel)
-			visorchannel_signalinsert(controlvm_channel,
-						  CONTROLVM_QUEUE_ACK,
-						  &ackmsg);
-	}
-	switch (inmsg.hdr.id) {
-	case CONTROLVM_CHIPSET_INIT:
-		chipset_init(&inmsg);
-		break;
-	case CONTROLVM_BUS_CREATE:
-		bus_create(&inmsg);
-		break;
-	case CONTROLVM_BUS_DESTROY:
-		bus_destroy(&inmsg);
-		break;
-	case CONTROLVM_BUS_CONFIGURE:
-		bus_configure(&inmsg, parser_ctx);
-		break;
-	case CONTROLVM_DEVICE_CREATE:
-		my_device_create(&inmsg);
-		break;
-	case CONTROLVM_DEVICE_CHANGESTATE:
-		if (cmd->device_change_state.flags.phys_device) {
-			parahotplug_process_message(&inmsg);
-		} else {
-			/* save the hdr and cmd structures for later use */
-			/* when sending back the response to Command */
-			my_device_changestate(&inmsg);
-			g_devicechangestate_packet = inmsg.cmd;
-			break;
-		}
-		break;
-	case CONTROLVM_DEVICE_DESTROY:
-		my_device_destroy(&inmsg);
-		break;
-	case CONTROLVM_DEVICE_CONFIGURE:
-		/* no op for now, just send a respond that we passed */
-		if (inmsg.hdr.flags.response_expected)
-			controlvm_respond(&inmsg.hdr, CONTROLVM_RESP_SUCCESS);
-		break;
-	case CONTROLVM_CHIPSET_READY:
-		chipset_ready(&inmsg.hdr);
-		break;
-	case CONTROLVM_CHIPSET_SELFTEST:
-		chipset_selftest(&inmsg.hdr);
-		break;
-	case CONTROLVM_CHIPSET_STOP:
-		chipset_notready(&inmsg.hdr);
-		break;
-	default:
-		if (inmsg.hdr.flags.response_expected)
-			controlvm_respond
-				(&inmsg.hdr,
-				 -CONTROLVM_RESP_ERROR_MESSAGE_ID_UNKNOWN);
-		break;
-	}
+static void
+chipset_ready(struct controlvm_message_header *msg_hdr)
+{
+	int rc = visorchipset_chipset_ready();
 
-	if (parser_ctx) {
-		parser_done(parser_ctx);
-		parser_ctx = NULL;
-	}
-	return true;
+	if (rc != CONTROLVM_RESP_SUCCESS)
+		rc = -rc;
+	if (msg_hdr->flags.response_expected)
+		controlvm_respond(msg_hdr, rc);
+}
+
+static void
+chipset_selftest(struct controlvm_message_header *msg_hdr)
+{
+	int rc = visorchipset_chipset_selftest();
+
+	if (rc != CONTROLVM_RESP_SUCCESS)
+		rc = -rc;
+	if (msg_hdr->flags.response_expected)
+		controlvm_respond(msg_hdr, rc);
+}
+
+static void
+chipset_notready(struct controlvm_message_header *msg_hdr)
+{
+	int rc = visorchipset_chipset_notready();
+
+	if (rc != CONTROLVM_RESP_SUCCESS)
+		rc = -rc;
+	if (msg_hdr->flags.response_expected)
+		controlvm_respond(msg_hdr, rc);
 }
 
 static inline unsigned int
@@ -1796,76 +1532,6 @@ static u64 controlvm_get_channel_address(void)
 }
 
 static void
-controlvm_periodic_work(struct work_struct *work)
-{
-	struct controlvm_message inmsg;
-	bool got_command = false;
-	bool handle_command_failed = false;
-
-	/* make sure visorbus server is registered for controlvm callbacks */
-	if (visorchipset_visorbusregwait && !visorbusregistered)
-		goto cleanup;
-
-	while (visorchannel_signalremove(controlvm_channel,
-					 CONTROLVM_QUEUE_RESPONSE,
-					 &inmsg))
-		;
-	if (!got_command) {
-		if (controlvm_pending_msg_valid) {
-			/* we throttled processing of a prior
-			* msg, so try to process it again
-			* rather than reading a new one
-			*/
-			inmsg = controlvm_pending_msg;
-			controlvm_pending_msg_valid = false;
-			got_command = true;
-		} else {
-			got_command = read_controlvm_event(&inmsg);
-		}
-	}
-
-	handle_command_failed = false;
-	while (got_command && (!handle_command_failed)) {
-		most_recent_message_jiffies = jiffies;
-		if (handle_command(inmsg,
-				   visorchannel_get_physaddr
-				   (controlvm_channel)))
-			got_command = read_controlvm_event(&inmsg);
-		else {
-			/* this is a scenario where throttling
-			* is required, but probably NOT an
-			* error...; we stash the current
-			* controlvm msg so we will attempt to
-			* reprocess it on our next loop
-			*/
-			handle_command_failed = true;
-			controlvm_pending_msg = inmsg;
-			controlvm_pending_msg_valid = true;
-		}
-	}
-
-	/* parahotplug_worker */
-	parahotplug_process_list();
-
-cleanup:
-
-	if (time_after(jiffies,
-		       most_recent_message_jiffies + (HZ * MIN_IDLE_SECONDS))) {
-		/* it's been longer than MIN_IDLE_SECONDS since we
-		* processed our last controlvm message; slow down the
-		* polling
-		*/
-		if (poll_jiffies != POLLJIFFIES_CONTROLVMCHANNEL_SLOW)
-			poll_jiffies = POLLJIFFIES_CONTROLVMCHANNEL_SLOW;
-	} else {
-		if (poll_jiffies != POLLJIFFIES_CONTROLVMCHANNEL_FAST)
-			poll_jiffies = POLLJIFFIES_CONTROLVMCHANNEL_FAST;
-	}
-
-	schedule_delayed_work(&periodic_controlvm_work, poll_jiffies);
-}
-
-static void
 setup_crash_devices_work_queue(struct work_struct *work)
 {
 	struct controlvm_message local_crash_bus_msg;
@@ -1874,13 +1540,6 @@ setup_crash_devices_work_queue(struct work_struct *work)
 	u32 local_crash_msg_offset;
 	u16 local_crash_msg_count;
 
-	/* make sure visorbus is registered for controlvm callbacks */
-	if (visorchipset_visorbusregwait && !visorbusregistered) {
-		poll_jiffies = POLLJIFFIES_CONTROLVMCHANNEL_SLOW;
-		schedule_delayed_work(&periodic_controlvm_work, poll_jiffies);
-		return;
-	}
-
 	POSTCODE_LINUX_2(CRASH_DEV_ENTRY_PC, POSTCODE_SEVERITY_INFO);
 
 	/* send init chipset msg */
@@ -1958,7 +1617,7 @@ setup_crash_devices_work_queue(struct work_struct *work)
 	POSTCODE_LINUX_2(CRASH_DEV_EXIT_PC, POSTCODE_SEVERITY_INFO);
 }
 
-static void
+void
 bus_create_response(struct visor_device *bus_info, int response)
 {
 	if (response >= 0)
@@ -1971,7 +1630,7 @@ bus_create_response(struct visor_device *bus_info, int response)
 	bus_info->pending_msg_hdr = NULL;
 }
 
-static void
+void
 bus_destroy_response(struct visor_device *bus_info, int response)
 {
 	bus_responder(CONTROLVM_BUS_DESTROY, bus_info->pending_msg_hdr,
@@ -1981,7 +1640,7 @@ bus_destroy_response(struct visor_device *bus_info, int response)
 	bus_info->pending_msg_hdr = NULL;
 }
 
-static void
+void
 device_create_response(struct visor_device *dev_info, int response)
 {
 	if (response >= 0)
@@ -1994,7 +1653,7 @@ device_create_response(struct visor_device *dev_info, int response)
 	dev_info->pending_msg_hdr = NULL;
 }
 
-static void
+void
 device_destroy_response(struct visor_device *dev_info, int response)
 {
 	device_responder(CONTROLVM_DEVICE_DESTROY, dev_info->pending_msg_hdr,
@@ -2004,9 +1663,9 @@ device_destroy_response(struct visor_device *dev_info, int response)
 	dev_info->pending_msg_hdr = NULL;
 }
 
-static void
-visorchipset_device_pause_response(struct visor_device *dev_info,
-				   int response)
+void
+device_pause_response(struct visor_device *dev_info,
+		      int response)
 {
 	device_changestate_responder(CONTROLVM_DEVICE_CHANGESTATE,
 				     dev_info, response,
@@ -2016,7 +1675,7 @@ visorchipset_device_pause_response(struct visor_device *dev_info,
 	dev_info->pending_msg_hdr = NULL;
 }
 
-static void
+void
 device_resume_response(struct visor_device *dev_info, int response)
 {
 	device_changestate_responder(CONTROLVM_DEVICE_CHANGESTATE,
@@ -2027,40 +1686,6 @@ device_resume_response(struct visor_device *dev_info, int response)
 	dev_info->pending_msg_hdr = NULL;
 }
 
-/* The parahotplug/devicedisabled interface gets called by our support script
- * when an SR-IOV device has been shut down. The ID is passed to the script
- * and then passed back when the device has been removed.
- */
-static ssize_t devicedisabled_store(struct device *dev,
-				    struct device_attribute *attr,
-				    const char *buf, size_t count)
-{
-	unsigned int id;
-
-	if (kstrtouint(buf, 10, &id))
-		return -EINVAL;
-
-	parahotplug_request_complete(id, 0);
-	return count;
-}
-
-/* The parahotplug/deviceenabled interface gets called by our support script
- * when an SR-IOV device has been recovered. The ID is passed to the script
- * and then passed back when the device has been brought back up.
- */
-static ssize_t deviceenabled_store(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf, size_t count)
-{
-	unsigned int id;
-
-	if (kstrtouint(buf, 10, &id))
-		return -EINVAL;
-
-	parahotplug_request_complete(id, 1);
-	return count;
-}
-
 static int
 visorchipset_mmap(struct file *file, struct vm_area_struct *vma)
 {
@@ -2191,6 +1816,298 @@ visorchipset_file_cleanup(dev_t major_dev)
 	unregister_chrdev_region(major_dev, 1);
 }
 
+static struct parser_context *
+parser_init_byte_stream(u64 addr, u32 bytes, bool local, bool *retry)
+{
+	int allocbytes = sizeof(struct parser_context) + bytes;
+	struct parser_context *ctx;
+
+	if (retry)
+		*retry = false;
+
+	/*
+	 * alloc an 0 extra byte to ensure payload is
+	 * '\0'-terminated
+	 */
+	allocbytes++;
+	if ((controlvm_payload_bytes_buffered + bytes)
+	    > MAX_CONTROLVM_PAYLOAD_BYTES) {
+		if (retry)
+			*retry = true;
+		return NULL;
+	}
+	ctx = kzalloc(allocbytes, GFP_KERNEL | __GFP_NORETRY);
+	if (!ctx) {
+		if (retry)
+			*retry = true;
+		return NULL;
+	}
+
+	ctx->allocbytes = allocbytes;
+	ctx->param_bytes = bytes;
+	ctx->curr = NULL;
+	ctx->bytes_remaining = 0;
+	ctx->byte_stream = false;
+	if (local) {
+		void *p;
+
+		if (addr > virt_to_phys(high_memory - 1))
+			goto err_finish_ctx;
+		p = __va((unsigned long)(addr));
+		memcpy(ctx->data, p, bytes);
+	} else {
+		void *mapping = memremap(addr, bytes, MEMREMAP_WB);
+
+		if (!mapping)
+			goto err_finish_ctx;
+		memcpy(ctx->data, mapping, bytes);
+		memunmap(mapping);
+	}
+
+	ctx->byte_stream = true;
+	controlvm_payload_bytes_buffered += ctx->param_bytes;
+
+	return ctx;
+
+err_finish_ctx:
+	parser_done(ctx);
+	return NULL;
+}
+
+/**
+ * handle_command() - process a controlvm message
+ * @inmsg:        the message to process
+ * @channel_addr: address of the controlvm channel
+ *
+ * Return:
+ *    false - this function will return false only in the case where the
+ *            controlvm message was NOT processed, but processing must be
+ *            retried before reading the next controlvm message; a
+ *            scenario where this can occur is when we need to throttle
+ *            the allocation of memory in which to copy out controlvm
+ *            payload data
+ *    true  - processing of the controlvm message completed,
+ *            either successfully or with an error
+ */
+static bool
+handle_command(struct controlvm_message inmsg, u64 channel_addr)
+{
+	struct controlvm_message_packet *cmd = &inmsg.cmd;
+	u64 parm_addr;
+	u32 parm_bytes;
+	struct parser_context *parser_ctx = NULL;
+	bool local_addr;
+	struct controlvm_message ackmsg;
+
+	/* create parsing context if necessary */
+	local_addr = (inmsg.hdr.flags.test_message == 1);
+	if (channel_addr == 0)
+		return true;
+	parm_addr = channel_addr + inmsg.hdr.payload_vm_offset;
+	parm_bytes = inmsg.hdr.payload_bytes;
+
+	/*
+	 * Parameter and channel addresses within test messages actually lie
+	 * within our OS-controlled memory. We need to know that, because it
+	 * makes a difference in how we compute the virtual address.
+	 */
+	if (parm_addr && parm_bytes) {
+		bool retry = false;
+
+		parser_ctx =
+		    parser_init_byte_stream(parm_addr, parm_bytes,
+					    local_addr, &retry);
+		if (!parser_ctx && retry)
+			return false;
+	}
+
+	if (!local_addr) {
+		controlvm_init_response(&ackmsg, &inmsg.hdr,
+					CONTROLVM_RESP_SUCCESS);
+		if (controlvm_channel)
+			visorchannel_signalinsert(controlvm_channel,
+						  CONTROLVM_QUEUE_ACK,
+						  &ackmsg);
+	}
+	switch (inmsg.hdr.id) {
+	case CONTROLVM_CHIPSET_INIT:
+		chipset_init(&inmsg);
+		break;
+	case CONTROLVM_BUS_CREATE:
+		bus_create(&inmsg);
+		break;
+	case CONTROLVM_BUS_DESTROY:
+		bus_destroy(&inmsg);
+		break;
+	case CONTROLVM_BUS_CONFIGURE:
+		bus_configure(&inmsg, parser_ctx);
+		break;
+	case CONTROLVM_DEVICE_CREATE:
+		my_device_create(&inmsg);
+		break;
+	case CONTROLVM_DEVICE_CHANGESTATE:
+		if (cmd->device_change_state.flags.phys_device) {
+			parahotplug_process_message(&inmsg);
+		} else {
+			/*
+			 * save the hdr and cmd structures for later use
+			 * when sending back the response to Command
+			 */
+			my_device_changestate(&inmsg);
+			break;
+		}
+		break;
+	case CONTROLVM_DEVICE_DESTROY:
+		my_device_destroy(&inmsg);
+		break;
+	case CONTROLVM_DEVICE_CONFIGURE:
+		/* no op for now, just send a respond that we passed */
+		if (inmsg.hdr.flags.response_expected)
+			controlvm_respond(&inmsg.hdr, CONTROLVM_RESP_SUCCESS);
+		break;
+	case CONTROLVM_CHIPSET_READY:
+		chipset_ready(&inmsg.hdr);
+		break;
+	case CONTROLVM_CHIPSET_SELFTEST:
+		chipset_selftest(&inmsg.hdr);
+		break;
+	case CONTROLVM_CHIPSET_STOP:
+		chipset_notready(&inmsg.hdr);
+		break;
+	default:
+		if (inmsg.hdr.flags.response_expected)
+			controlvm_respond
+				(&inmsg.hdr,
+				 -CONTROLVM_RESP_ERROR_MESSAGE_ID_UNKNOWN);
+		break;
+	}
+
+	if (parser_ctx) {
+		parser_done(parser_ctx);
+		parser_ctx = NULL;
+	}
+	return true;
+}
+
+/**
+ * read_controlvm_event() - retreives the next message from the
+ *                          CONTROLVM_QUEUE_EVENT queue in the controlvm
+ *                          channel
+ * @msg: pointer to the retrieved message
+ *
+ * Return: true if a valid message was retrieved or false otherwise
+ */
+static bool
+read_controlvm_event(struct controlvm_message *msg)
+{
+	if (!visorchannel_signalremove(controlvm_channel,
+				       CONTROLVM_QUEUE_EVENT, msg)) {
+		/* got a message */
+		if (msg->hdr.flags.test_message == 1)
+			return false;
+		return true;
+	}
+	return false;
+}
+
+/**
+ * parahotplug_process_list() - remove any request from the list that's been on
+ *                              there too long and respond with an error
+ */
+static void
+parahotplug_process_list(void)
+{
+	struct list_head *pos;
+	struct list_head *tmp;
+
+	spin_lock(&parahotplug_request_list_lock);
+
+	list_for_each_safe(pos, tmp, &parahotplug_request_list) {
+		struct parahotplug_request *req =
+		    list_entry(pos, struct parahotplug_request, list);
+
+		if (!time_after_eq(jiffies, req->expiration))
+			continue;
+
+		list_del(pos);
+		if (req->msg.hdr.flags.response_expected)
+			controlvm_respond_physdev_changestate(
+				&req->msg.hdr,
+				CONTROLVM_RESP_ERROR_DEVICE_UDEV_TIMEOUT,
+				req->msg.cmd.device_change_state.state);
+		parahotplug_request_destroy(req);
+	}
+
+	spin_unlock(&parahotplug_request_list_lock);
+}
+
+static void
+controlvm_periodic_work(struct work_struct *work)
+{
+	struct controlvm_message inmsg;
+	bool got_command = false;
+	bool handle_command_failed = false;
+
+	while (!visorchannel_signalremove(controlvm_channel,
+					  CONTROLVM_QUEUE_RESPONSE,
+					  &inmsg))
+		;
+	if (!got_command) {
+		if (controlvm_pending_msg_valid) {
+			/*
+			 * we throttled processing of a prior
+			 * msg, so try to process it again
+			 * rather than reading a new one
+			 */
+			inmsg = controlvm_pending_msg;
+			controlvm_pending_msg_valid = false;
+			got_command = true;
+		} else {
+			got_command = read_controlvm_event(&inmsg);
+		}
+	}
+
+	handle_command_failed = false;
+	while (got_command && (!handle_command_failed)) {
+		most_recent_message_jiffies = jiffies;
+		if (handle_command(inmsg,
+				   visorchannel_get_physaddr
+				   (controlvm_channel)))
+			got_command = read_controlvm_event(&inmsg);
+		else {
+			/*
+			 * this is a scenario where throttling
+			 * is required, but probably NOT an
+			 * error...; we stash the current
+			 * controlvm msg so we will attempt to
+			 * reprocess it on our next loop
+			 */
+			handle_command_failed = true;
+			controlvm_pending_msg = inmsg;
+			controlvm_pending_msg_valid = true;
+		}
+	}
+
+	/* parahotplug_worker */
+	parahotplug_process_list();
+
+	if (time_after(jiffies,
+		       most_recent_message_jiffies + (HZ * MIN_IDLE_SECONDS))) {
+		/*
+		 * it's been longer than MIN_IDLE_SECONDS since we
+		 * processed our last controlvm message; slow down the
+		 * polling
+		 */
+		if (poll_jiffies != POLLJIFFIES_CONTROLVMCHANNEL_SLOW)
+			poll_jiffies = POLLJIFFIES_CONTROLVMCHANNEL_SLOW;
+	} else {
+		if (poll_jiffies != POLLJIFFIES_CONTROLVMCHANNEL_FAST)
+			poll_jiffies = POLLJIFFIES_CONTROLVMCHANNEL_FAST;
+	}
+
+	schedule_delayed_work(&periodic_controlvm_work, poll_jiffies);
+}
+
 static int
 visorchipset_init(struct acpi_device *acpi_device)
 {
@@ -2202,7 +2119,6 @@ visorchipset_init(struct acpi_device *acpi_device)
 	if (!addr)
 		goto error;
 
-	memset(&busdev_notifiers, 0, sizeof(busdev_notifiers));
 	memset(&controlvm_payload_info, 0, sizeof(controlvm_payload_info));
 
 	controlvm_channel = visorchannel_create_with_lock(addr, 0,
@@ -2341,15 +2257,10 @@ static void exit_unisys(void)
 module_param_named(major, visorchipset_major, int, S_IRUGO);
 MODULE_PARM_DESC(visorchipset_major,
 		 "major device number to use for the device node");
-module_param_named(visorbusregwait, visorchipset_visorbusregwait, int, S_IRUGO);
-MODULE_PARM_DESC(visorchipset_visorbusregwait,
-		 "1 to have the module wait for the visor bus to register");
 
 module_init(init_unisys);
 module_exit(exit_unisys);
 
 MODULE_AUTHOR("Unisys");
 MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Supervisor chipset driver for service partition: ver "
-		   VERSION);
-MODULE_VERSION(VERSION);
+MODULE_DESCRIPTION("s-Par visorbus driver for virtual device buses");
diff --git a/drivers/staging/unisys/visorbus/vmcallinterface.h b/drivers/staging/unisys/visorbus/vmcallinterface.h
index c043fa41ceda..86e695d5a441 100644
--- a/drivers/staging/unisys/visorbus/vmcallinterface.h
+++ b/drivers/staging/unisys/visorbus/vmcallinterface.h
@@ -20,11 +20,39 @@
 * Virtualization.  The VMCALLs are provided by Monitor and used by IO code
 * running on IO Partitions.
 */
+static inline unsigned long
+__unisys_vmcall_gnuc(unsigned long tuple, unsigned long reg_ebx,
+		     unsigned long reg_ecx)
+{
+	unsigned long result = 0;
+	unsigned int cpuid_eax, cpuid_ebx, cpuid_ecx, cpuid_edx;
 
-#ifdef __GNUC__
-#include "iovmcall_gnuc.h"
-#endif	/*  */
-#include "diagchannel.h"
+	cpuid(0x00000001, &cpuid_eax, &cpuid_ebx, &cpuid_ecx, &cpuid_edx);
+	if (!(cpuid_ecx & 0x80000000))
+		return -EPERM;
+
+	__asm__ __volatile__(".byte 0x00f, 0x001, 0x0c1" : "=a"(result) :
+		"a"(tuple), "b"(reg_ebx), "c"(reg_ecx));
+	return result;
+}
+
+static inline unsigned long
+__unisys_extended_vmcall_gnuc(unsigned long long tuple,
+			      unsigned long long reg_ebx,
+			      unsigned long long reg_ecx,
+			      unsigned long long reg_edx)
+{
+	unsigned long result = 0;
+	unsigned int cpuid_eax, cpuid_ebx, cpuid_ecx, cpuid_edx;
+
+	cpuid(0x00000001, &cpuid_eax, &cpuid_ebx, &cpuid_ecx, &cpuid_edx);
+	if (!(cpuid_ecx & 0x80000000))
+		return -EPERM;
+
+	__asm__ __volatile__(".byte 0x00f, 0x001, 0x0c1" : "=a"(result) :
+		"a"(tuple), "b"(reg_ebx), "c"(reg_ecx), "d"(reg_edx));
+	return result;
+}
 
 #ifdef VMCALL_IO_CONTROLVM_ADDR
 #undef VMCALL_IO_CONTROLVM_ADDR
@@ -57,7 +85,6 @@ enum vmcall_monitor_interface_method_tuple { /* VMCALL identification tuples  */
 #define VMCALL_SUCCESS 0
 #define VMCALL_SUCCESSFUL(result)	(result == 0)
 
-#ifdef __GNUC__
 #define unisys_vmcall(tuple, reg_ebx, reg_ecx) \
 	__unisys_vmcall_gnuc(tuple, reg_ebx, reg_ecx)
 #define unisys_extended_vmcall(tuple, reg_ebx, reg_ecx, reg_edx) \
@@ -74,7 +101,6 @@ enum vmcall_monitor_interface_method_tuple { /* VMCALL identification tuples  */
 #define ISSUE_IO_VMCALL_POSTCODE_SEVERITY(postcode, severity)		\
 	ISSUE_IO_EXTENDED_VMCALL(VMCALL_POST_CODE_LOGEVENT, severity,	\
 				 MDS_APPOS, postcode)
-#endif
 
 /* Structures for IO VMCALLs */
 
@@ -89,4 +115,156 @@ struct vmcall_io_controlvm_addr_params {
 	u8 unused[4];		/* Unused Bytes in the 64-Bit Aligned Struct */
 } __packed;
 
+/******* INFO ON ISSUE_POSTCODE_LINUX() BELOW *******/
+enum driver_pc {		/* POSTCODE driver identifier tuples */
+	/* visorchipset driver files */
+	VISOR_CHIPSET_PC = 0xA0,
+	VISOR_CHIPSET_PC_controlvm_c = 0xA1,
+	VISOR_CHIPSET_PC_controlvm_cm2 = 0xA2,
+	VISOR_CHIPSET_PC_controlvm_direct_c = 0xA3,
+	VISOR_CHIPSET_PC_file_c = 0xA4,
+	VISOR_CHIPSET_PC_parser_c = 0xA5,
+	VISOR_CHIPSET_PC_testing_c = 0xA6,
+	VISOR_CHIPSET_PC_visorchipset_main_c = 0xA7,
+	VISOR_CHIPSET_PC_visorswitchbus_c = 0xA8,
+	/* visorbus driver files */
+	VISOR_BUS_PC = 0xB0,
+	VISOR_BUS_PC_businst_attr_c = 0xB1,
+	VISOR_BUS_PC_channel_attr_c = 0xB2,
+	VISOR_BUS_PC_devmajorminor_attr_c = 0xB3,
+	VISOR_BUS_PC_visorbus_main_c = 0xB4,
+	/* visorclientbus driver files */
+	VISOR_CLIENT_BUS_PC = 0xC0,
+	VISOR_CLIENT_BUS_PC_visorclientbus_main_c = 0xC1,
+	/* virt hba driver files */
+	VIRT_HBA_PC = 0xC2,
+	VIRT_HBA_PC_virthba_c = 0xC3,
+	/* virtpci driver files */
+	VIRT_PCI_PC = 0xC4,
+	VIRT_PCI_PC_virtpci_c = 0xC5,
+	/* virtnic driver files */
+	VIRT_NIC_PC = 0xC6,
+	VIRT_NIC_P_virtnic_c = 0xC7,
+	/* uislib driver files */
+	UISLIB_PC = 0xD0,
+	UISLIB_PC_uislib_c = 0xD1,
+	UISLIB_PC_uisqueue_c = 0xD2,
+	/* 0xD3 RESERVED */
+	UISLIB_PC_uisutils_c = 0xD4,
+};
+
+enum event_pc {			/* POSTCODE event identifier tuples */
+	ATTACH_PORT_ENTRY_PC = 0x001,
+	ATTACH_PORT_FAILURE_PC = 0x002,
+	ATTACH_PORT_SUCCESS_PC = 0x003,
+	BUS_FAILURE_PC = 0x004,
+	BUS_CREATE_ENTRY_PC = 0x005,
+	BUS_CREATE_FAILURE_PC = 0x006,
+	BUS_CREATE_EXIT_PC = 0x007,
+	BUS_CONFIGURE_ENTRY_PC = 0x008,
+	BUS_CONFIGURE_FAILURE_PC = 0x009,
+	BUS_CONFIGURE_EXIT_PC = 0x00A,
+	CHIPSET_INIT_ENTRY_PC = 0x00B,
+	CHIPSET_INIT_SUCCESS_PC = 0x00C,
+	CHIPSET_INIT_FAILURE_PC = 0x00D,
+	CHIPSET_INIT_EXIT_PC = 0x00E,
+	CREATE_WORKQUEUE_PC = 0x00F,
+	CREATE_WORKQUEUE_FAILED_PC = 0x0A0,
+	CONTROLVM_INIT_FAILURE_PC = 0x0A1,
+	DEVICE_CREATE_ENTRY_PC = 0x0A2,
+	DEVICE_CREATE_FAILURE_PC = 0x0A3,
+	DEVICE_CREATE_SUCCESS_PC = 0x0A4,
+	DEVICE_CREATE_EXIT_PC = 0x0A5,
+	DEVICE_ADD_PC = 0x0A6,
+	DEVICE_REGISTER_FAILURE_PC = 0x0A7,
+	DEVICE_CHANGESTATE_ENTRY_PC = 0x0A8,
+	DEVICE_CHANGESTATE_FAILURE_PC = 0x0A9,
+	DEVICE_CHANGESTATE_EXIT_PC = 0x0AA,
+	DRIVER_ENTRY_PC = 0x0AB,
+	DRIVER_EXIT_PC = 0x0AC,
+	MALLOC_FAILURE_PC = 0x0AD,
+	QUEUE_DELAYED_WORK_PC = 0x0AE,
+	/* 0x0B7 RESERVED */
+	VBUS_CHANNEL_ENTRY_PC = 0x0B8,
+	VBUS_CHANNEL_FAILURE_PC = 0x0B9,
+	VBUS_CHANNEL_EXIT_PC = 0x0BA,
+	VHBA_CREATE_ENTRY_PC = 0x0BB,
+	VHBA_CREATE_FAILURE_PC = 0x0BC,
+	VHBA_CREATE_EXIT_PC = 0x0BD,
+	VHBA_CREATE_SUCCESS_PC = 0x0BE,
+	VHBA_COMMAND_HANDLER_PC = 0x0BF,
+	VHBA_PROBE_ENTRY_PC = 0x0C0,
+	VHBA_PROBE_FAILURE_PC = 0x0C1,
+	VHBA_PROBE_EXIT_PC = 0x0C2,
+	VNIC_CREATE_ENTRY_PC = 0x0C3,
+	VNIC_CREATE_FAILURE_PC = 0x0C4,
+	VNIC_CREATE_SUCCESS_PC = 0x0C5,
+	VNIC_PROBE_ENTRY_PC = 0x0C6,
+	VNIC_PROBE_FAILURE_PC = 0x0C7,
+	VNIC_PROBE_EXIT_PC = 0x0C8,
+	VPCI_CREATE_ENTRY_PC = 0x0C9,
+	VPCI_CREATE_FAILURE_PC = 0x0CA,
+	VPCI_CREATE_EXIT_PC = 0x0CB,
+	VPCI_PROBE_ENTRY_PC = 0x0CC,
+	VPCI_PROBE_FAILURE_PC = 0x0CD,
+	VPCI_PROBE_EXIT_PC = 0x0CE,
+	CRASH_DEV_ENTRY_PC = 0x0CF,
+	CRASH_DEV_EXIT_PC = 0x0D0,
+	CRASH_DEV_HADDR_NULL = 0x0D1,
+	CRASH_DEV_CONTROLVM_NULL = 0x0D2,
+	CRASH_DEV_RD_BUS_FAIULRE_PC = 0x0D3,
+	CRASH_DEV_RD_DEV_FAIULRE_PC = 0x0D4,
+	CRASH_DEV_BUS_NULL_FAILURE_PC = 0x0D5,
+	CRASH_DEV_DEV_NULL_FAILURE_PC = 0x0D6,
+	CRASH_DEV_CTRL_RD_FAILURE_PC = 0x0D7,
+	CRASH_DEV_COUNT_FAILURE_PC = 0x0D8,
+	SAVE_MSG_BUS_FAILURE_PC = 0x0D9,
+	SAVE_MSG_DEV_FAILURE_PC = 0x0DA,
+	CALLHOME_INIT_FAILURE_PC = 0x0DB
+};
+
+#define POSTCODE_SEVERITY_ERR DIAG_SEVERITY_ERR
+#define POSTCODE_SEVERITY_WARNING DIAG_SEVERITY_WARNING
+/* TODO-> Info currently doesn't show, so we set info=warning */
+#define POSTCODE_SEVERITY_INFO DIAG_SEVERITY_PRINT
+
+/* example call of POSTCODE_LINUX_2(VISOR_CHIPSET_PC, POSTCODE_SEVERITY_ERR);
+ * Please also note that the resulting postcode is in hex, so if you are
+ * searching for the __LINE__ number, convert it first to decimal.  The line
+ * number combined with driver and type of call, will allow you to track down
+ * exactly what line an error occurred on, or where the last driver
+ * entered/exited from.
+ */
+
+/* BASE FUNCTIONS */
+#define POSTCODE_LINUX_A(DRIVER_PC, EVENT_PC, pc32bit, severity)	\
+do {									\
+	unsigned long long post_code_temp;				\
+	post_code_temp = (((u64)DRIVER_PC) << 56) | (((u64)EVENT_PC) << 44) | \
+		((((u64)__LINE__) & 0xFFF) << 32) |			\
+		(((u64)pc32bit) & 0xFFFFFFFF);				\
+	ISSUE_IO_VMCALL_POSTCODE_SEVERITY(post_code_temp, severity);	\
+} while (0)
+
+#define POSTCODE_LINUX_B(DRIVER_PC, EVENT_PC, pc16bit1, pc16bit2, severity) \
+do {									\
+	unsigned long long post_code_temp;				\
+	post_code_temp = (((u64)DRIVER_PC) << 56) | (((u64)EVENT_PC) << 44) | \
+		((((u64)__LINE__) & 0xFFF) << 32) |			\
+		((((u64)pc16bit1) & 0xFFFF) << 16) |			\
+		(((u64)pc16bit2) & 0xFFFF);				\
+	ISSUE_IO_VMCALL_POSTCODE_SEVERITY(post_code_temp, severity);	\
+} while (0)
+
+/* MOST COMMON */
+#define POSTCODE_LINUX_2(EVENT_PC, severity)				\
+	POSTCODE_LINUX_A(CURRENT_FILE_PC, EVENT_PC, 0x0000, severity)
+
+#define POSTCODE_LINUX_3(EVENT_PC, pc32bit, severity)			\
+	POSTCODE_LINUX_A(CURRENT_FILE_PC, EVENT_PC, pc32bit, severity)
+
+#define POSTCODE_LINUX_4(EVENT_PC, pc16bit1, pc16bit2, severity)	\
+	POSTCODE_LINUX_B(CURRENT_FILE_PC, EVENT_PC, pc16bit1,		\
+			 pc16bit2, severity)
+
 #endif /* __IOMONINTF_H__ */
diff --git a/drivers/staging/unisys/visorhba/visorhba_main.c b/drivers/staging/unisys/visorhba/visorhba_main.c
index 120ba2097e02..5a7a87efed27 100644
--- a/drivers/staging/unisys/visorhba/visorhba_main.c
+++ b/drivers/staging/unisys/visorhba/visorhba_main.c
@@ -36,21 +36,6 @@
 #define MAX_PENDING_REQUESTS	(MIN_NUMSIGNALS * 2)
 #define VISORHBA_ERROR_COUNT	30
 
-static int visorhba_queue_command_lck(struct scsi_cmnd *scsicmd,
-				      void (*visorhba_cmnd_done)
-					    (struct scsi_cmnd *));
-#ifdef DEF_SCSI_QCMD
-static DEF_SCSI_QCMD(visorhba_queue_command)
-#else
-#define visorhba_queue_command visorhba_queue_command_lck
-#endif
-static int visorhba_probe(struct visor_device *dev);
-static void visorhba_remove(struct visor_device *dev);
-static int visorhba_pause(struct visor_device *dev,
-			  visorbus_state_complete_func complete_func);
-static int visorhba_resume(struct visor_device *dev,
-			   visorbus_state_complete_func complete_func);
-
 static struct dentry *visorhba_debugfs_dir;
 
 /* GUIDS for HBA channel type supported by this driver */
@@ -62,20 +47,6 @@ static struct visor_channeltype_descriptor visorhba_channel_types[] = {
 	{ NULL_UUID_LE, NULL }
 };
 
-/* This is used to tell the visor bus driver which types of visor devices
- * we support, and what functions to call when a visor device that we support
- * is attached or removed.
- */
-static struct visor_driver visorhba_driver = {
-	.name = "visorhba",
-	.owner = THIS_MODULE,
-	.channel_types = visorhba_channel_types,
-	.probe = visorhba_probe,
-	.remove = visorhba_remove,
-	.pause = visorhba_pause,
-	.resume = visorhba_resume,
-	.channel_interrupt = NULL,
-};
 MODULE_DEVICE_TABLE(visorbus, visorhba_channel_types);
 MODULE_ALIAS("visorbus:" SPAR_VHBA_CHANNEL_PROTOCOL_UUID_STR);
 
@@ -364,9 +335,9 @@ static int forward_taskmgmt_command(enum task_mgmt_types tasktype,
 
 	dev_dbg(&scsidev->sdev_gendev,
 		"visorhba: initiating type=%d taskmgmt command\n", tasktype);
-	if (!visorchannel_signalinsert(devdata->dev->visorchannel,
-				       IOCHAN_TO_IOPART,
-				       cmdrsp))
+	if (visorchannel_signalinsert(devdata->dev->visorchannel,
+				      IOCHAN_TO_IOPART,
+				      cmdrsp))
 		goto err_del_scsipending_ent;
 
 	/* It can take the Service Partition up to 35 seconds to complete
@@ -567,9 +538,9 @@ visorhba_queue_command_lck(struct scsi_cmnd *scsicmd,
 	}
 	cmdrsp->scsi.guest_phys_entries = scsi_sg_count(scsicmd);
 
-	if (!visorchannel_signalinsert(devdata->dev->visorchannel,
-				       IOCHAN_TO_IOPART,
-				       cmdrsp))
+	if (visorchannel_signalinsert(devdata->dev->visorchannel,
+				      IOCHAN_TO_IOPART,
+				      cmdrsp))
 		/* queue must be full and we aren't going to wait */
 		goto err_del_scsipending_ent;
 
@@ -580,6 +551,12 @@ err_del_scsipending_ent:
 	return SCSI_MLQUEUE_DEVICE_BUSY;
 }
 
+#ifdef DEF_SCSI_QCMD
+static DEF_SCSI_QCMD(visorhba_queue_command)
+#else
+#define visorhba_queue_command visorhba_queue_command_lck
+#endif
+
 /**
  *	visorhba_slave_alloc - called when new disk is discovered
  *	@scsidev: New disk
@@ -950,9 +927,9 @@ drain_queue(struct uiscmdrsp *cmdrsp, struct visorhba_devdata *devdata)
 	struct scsi_cmnd *scsicmd;
 
 	while (1) {
-		if (!visorchannel_signalremove(devdata->dev->visorchannel,
-					       IOCHAN_FROM_IOPART,
-					       cmdrsp))
+		if (visorchannel_signalremove(devdata->dev->visorchannel,
+					      IOCHAN_FROM_IOPART,
+					      cmdrsp))
 			break; /* queue empty */
 
 		if (cmdrsp->cmdtype == CMD_SCSI_TYPE) {
@@ -1186,6 +1163,21 @@ static void visorhba_remove(struct visor_device *dev)
 	debugfs_remove_recursive(devdata->debugfs_dir);
 }
 
+/* This is used to tell the visor bus driver which types of visor devices
+ * we support, and what functions to call when a visor device that we support
+ * is attached or removed.
+ */
+static struct visor_driver visorhba_driver = {
+	.name = "visorhba",
+	.owner = THIS_MODULE,
+	.channel_types = visorhba_channel_types,
+	.probe = visorhba_probe,
+	.remove = visorhba_remove,
+	.pause = visorhba_pause,
+	.resume = visorhba_resume,
+	.channel_interrupt = NULL,
+};
+
 /**
  *	visorhba_init		- driver init routine
  *
@@ -1228,4 +1220,4 @@ module_exit(visorhba_exit);
 
 MODULE_AUTHOR("Unisys");
 MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("s-Par hba driver");
+MODULE_DESCRIPTION("s-Par HBA driver for virtual SCSI host busses");
diff --git a/drivers/staging/unisys/visorinput/ultrainputreport.h b/drivers/staging/unisys/visorinput/ultrainputreport.h
index 1bc3d2064080..53dde7c53809 100644
--- a/drivers/staging/unisys/visorinput/ultrainputreport.h
+++ b/drivers/staging/unisys/visorinput/ultrainputreport.h
@@ -17,8 +17,6 @@
 
 #include <linux/types.h>
 
-#include "ultrainputreport.h"
-
 /* Identifies mouse and keyboard activity which is specified by the firmware to
  *  the host using the cmsimpleinput protocol.  @ingroup coretypes
  */
diff --git a/drivers/staging/unisys/visorinput/visorinput.c b/drivers/staging/unisys/visorinput/visorinput.c
index d67cd76327c0..6f94b646f7c5 100644
--- a/drivers/staging/unisys/visorinput/visorinput.c
+++ b/drivers/staging/unisys/visorinput/visorinput.c
@@ -29,9 +29,7 @@
 #include <linux/kernel.h>
 #include <linux/uuid.h>
 
-#include "version.h"
 #include "visorbus.h"
-#include "channel.h"
 #include "ultrainputreport.h"
 
 /* Keyboard channel {c73416d0-b0b8-44af-b304-9d2ae99f1b3d} */
@@ -63,9 +61,10 @@ enum visorinput_device_type {
  */
 struct visorinput_devdata {
 	struct visor_device *dev;
-	struct rw_semaphore lock_visor_dev; /* lock for dev */
+	struct mutex lock_visor_dev; /* lock for dev */
 	struct input_dev *visorinput_dev;
 	bool paused;
+	bool interrupts_enabled;
 	unsigned int keycode_table_bytes; /* size of following array */
 	/* for keyboard devices: visorkbd_keycode[] + visorkbd_ext_keycode[] */
 	unsigned char keycode_table[0];
@@ -228,7 +227,21 @@ static int visorinput_open(struct input_dev *visorinput_dev)
 		return -EINVAL;
 	}
 	dev_dbg(&visorinput_dev->dev, "%s opened\n", __func__);
+
+	/*
+	 * If we're not paused, really enable interrupts.
+	 * Regardless of whether we are paused, set a flag indicating
+	 * interrupts should be enabled so when we resume, interrupts
+	 * will really be enabled.
+	 */
+	mutex_lock(&devdata->lock_visor_dev);
+	devdata->interrupts_enabled = true;
+	if (devdata->paused)
+		goto out_unlock;
 	visorbus_enable_channel_interrupts(devdata->dev);
+
+out_unlock:
+	mutex_unlock(&devdata->lock_visor_dev);
 	return 0;
 }
 
@@ -243,20 +256,35 @@ static void visorinput_close(struct input_dev *visorinput_dev)
 		return;
 	}
 	dev_dbg(&visorinput_dev->dev, "%s closed\n", __func__);
+
+	/*
+	 * If we're not paused, really disable interrupts.
+	 * Regardless of whether we are paused, set a flag indicating
+	 * interrupts should be disabled so when we resume we will
+	 * not re-enable them.
+	 */
+
+	mutex_lock(&devdata->lock_visor_dev);
+	devdata->interrupts_enabled = false;
+	if (devdata->paused)
+		goto out_unlock;
 	visorbus_disable_channel_interrupts(devdata->dev);
+
+out_unlock:
+	mutex_unlock(&devdata->lock_visor_dev);
 }
 
 /*
- * register_client_keyboard() initializes and returns a Linux input node that
+ * setup_client_keyboard() initializes and returns a Linux input node that
  * we can use to deliver keyboard inputs to Linux.  We of course do this when
  * we see keyboard inputs coming in on a keyboard channel.
  */
 static struct input_dev *
-register_client_keyboard(void *devdata,  /* opaque on purpose */
-			 unsigned char *keycode_table)
+setup_client_keyboard(void *devdata,  /* opaque on purpose */
+		      unsigned char *keycode_table)
 
 {
-	int i, error;
+	int i;
 	struct input_dev *visorinput_dev;
 
 	visorinput_dev = input_allocate_device();
@@ -290,18 +318,12 @@ register_client_keyboard(void *devdata,  /* opaque on purpose */
 	visorinput_dev->close = visorinput_close;
 	input_set_drvdata(visorinput_dev, devdata); /* pre input_register! */
 
-	error = input_register_device(visorinput_dev);
-	if (error) {
-		input_free_device(visorinput_dev);
-		return NULL;
-	}
 	return visorinput_dev;
 }
 
 static struct input_dev *
-register_client_mouse(void *devdata /* opaque on purpose */)
+setup_client_mouse(void *devdata /* opaque on purpose */)
 {
-	int error;
 	struct input_dev *visorinput_dev = NULL;
 	int xres, yres;
 	struct fb_info *fb0;
@@ -336,13 +358,6 @@ register_client_mouse(void *devdata /* opaque on purpose */)
 	visorinput_dev->open = visorinput_open;
 	visorinput_dev->close = visorinput_close;
 	input_set_drvdata(visorinput_dev, devdata); /* pre input_register! */
-
-	error = input_register_device(visorinput_dev);
-	if (error) {
-		input_free_device(visorinput_dev);
-		return NULL;
-	}
-
 	input_set_capability(visorinput_dev, EV_REL, REL_WHEEL);
 
 	return visorinput_dev;
@@ -360,9 +375,19 @@ devdata_create(struct visor_device *dev, enum visorinput_device_type devtype)
 	devdata = kzalloc(sizeof(*devdata) + extra_bytes, GFP_KERNEL);
 	if (!devdata)
 		return NULL;
+	mutex_init(&devdata->lock_visor_dev);
+	mutex_lock(&devdata->lock_visor_dev);
 	devdata->dev = dev;
 
 	/*
+	 * visorinput_open() can be called as soon as input_register_device()
+	 * happens, and that will enable channel interrupts.  Setting paused
+	 * prevents us from getting into visorinput_channel_interrupt() prior
+	 * to the device structure being totally initialized.
+	 */
+	devdata->paused = true;
+
+	/*
 	 * This is an input device in a client guest partition,
 	 * so we need to create whatever input nodes are necessary to
 	 * deliver our inputs to the guest OS.
@@ -374,23 +399,49 @@ devdata_create(struct visor_device *dev, enum visorinput_device_type devtype)
 		       KEYCODE_TABLE_BYTES);
 		memcpy(devdata->keycode_table + KEYCODE_TABLE_BYTES,
 		       visorkbd_ext_keycode, KEYCODE_TABLE_BYTES);
-		devdata->visorinput_dev = register_client_keyboard
+		devdata->visorinput_dev = setup_client_keyboard
 			(devdata, devdata->keycode_table);
 		if (!devdata->visorinput_dev)
 			goto cleanups_register;
 		break;
 	case visorinput_mouse:
-		devdata->visorinput_dev = register_client_mouse(devdata);
+		devdata->visorinput_dev = setup_client_mouse(devdata);
 		if (!devdata->visorinput_dev)
 			goto cleanups_register;
 		break;
 	}
 
-	init_rwsem(&devdata->lock_visor_dev);
+	dev_set_drvdata(&dev->device, devdata);
+	mutex_unlock(&devdata->lock_visor_dev);
+
+	/*
+	 * Device struct is completely set up now, with the exception of
+	 * visorinput_dev being registered.
+	 * We need to unlock before we register the device, because this
+	 * can cause an on-stack call of visorinput_open(), which would
+	 * deadlock if we had the lock.
+	 */
+	if (input_register_device(devdata->visorinput_dev)) {
+		input_free_device(devdata->visorinput_dev);
+		goto err_kfree_devdata;
+	}
+
+	mutex_lock(&devdata->lock_visor_dev);
+	/*
+	 * Establish calls to visorinput_channel_interrupt() if that is
+	 * the desired state that we've kept track of in interrupts_enabled
+	 * while the device was being created.
+	 */
+	devdata->paused = false;
+	if (devdata->interrupts_enabled)
+		visorbus_enable_channel_interrupts(dev);
+	mutex_unlock(&devdata->lock_visor_dev);
 
 	return devdata;
 
 cleanups_register:
+	mutex_unlock(&devdata->lock_visor_dev);
+err_kfree_devdata:
 	kfree(devdata);
 	return NULL;
 }
@@ -398,7 +449,6 @@ cleanups_register:
 static int
 visorinput_probe(struct visor_device *dev)
 {
-	struct visorinput_devdata *devdata = NULL;
 	uuid_le guid;
 	enum visorinput_device_type devtype;
 
@@ -409,10 +459,9 @@ visorinput_probe(struct visor_device *dev)
 		devtype = visorinput_keyboard;
 	else
 		return -ENODEV;
-	devdata = devdata_create(dev, devtype);
-	if (!devdata)
+	visorbus_disable_channel_interrupts(dev);
+	if (!devdata_create(dev, devtype))
 		return -ENOMEM;
-	dev_set_drvdata(&dev->device, devdata);
 	return 0;
 }
 
@@ -431,6 +480,7 @@ visorinput_remove(struct visor_device *dev)
 	if (!devdata)
 		return;
 
+	mutex_lock(&devdata->lock_visor_dev);
 	visorbus_disable_channel_interrupts(dev);
 
 	/*
@@ -438,10 +488,10 @@ visorinput_remove(struct visor_device *dev)
 	 * in visorinput_channel_interrupt()
 	 */
 
-	down_write(&devdata->lock_visor_dev);
 	dev_set_drvdata(&dev->device, NULL);
+	mutex_unlock(&devdata->lock_visor_dev);
+
 	unregister_client_input(devdata->visorinput_dev);
-	up_write(&devdata->lock_visor_dev);
 	kfree(devdata);
 }
 
@@ -529,15 +579,9 @@ visorinput_channel_interrupt(struct visor_device *dev)
 	if (!devdata)
 		return;
 
-	down_write(&devdata->lock_visor_dev);
-	if (devdata->paused) /* don't touch device/channel when paused */
-		goto out_locked;
-
 	visorinput_dev = devdata->visorinput_dev;
-	if (!visorinput_dev)
-		goto out_locked;
 
-	while (visorchannel_signalremove(dev->visorchannel, 0, &r)) {
+	while (!visorchannel_signalremove(dev->visorchannel, 0, &r)) {
 		scancode = r.activity.arg1;
 		keycode = scancode_to_keycode(scancode);
 		switch (r.activity.action) {
@@ -611,8 +655,6 @@ visorinput_channel_interrupt(struct visor_device *dev)
 			break;
 		}
 	}
-out_locked:
-	up_write(&devdata->lock_visor_dev);
 }
 
 static int
@@ -627,16 +669,24 @@ visorinput_pause(struct visor_device *dev,
 		goto out;
 	}
 
-	down_write(&devdata->lock_visor_dev);
+	mutex_lock(&devdata->lock_visor_dev);
 	if (devdata->paused) {
 		rc = -EBUSY;
 		goto out_locked;
 	}
+	if (devdata->interrupts_enabled)
+		visorbus_disable_channel_interrupts(dev);
+
+	/*
+	 * due to above, at this time no thread of execution will be
+	 * in visorinput_channel_interrupt()
+	 */
+
 	devdata->paused = true;
 	complete_func(dev, 0);
 	rc = 0;
 out_locked:
-	up_write(&devdata->lock_visor_dev);
+	mutex_unlock(&devdata->lock_visor_dev);
 out:
 	return rc;
 }
@@ -652,16 +702,25 @@ visorinput_resume(struct visor_device *dev,
 		rc = -ENODEV;
 		goto out;
 	}
-	down_write(&devdata->lock_visor_dev);
+	mutex_lock(&devdata->lock_visor_dev);
 	if (!devdata->paused) {
 		rc = -EBUSY;
 		goto out_locked;
 	}
 	devdata->paused = false;
 	complete_func(dev, 0);
+
+	/*
+	 * Re-establish calls to visorinput_channel_interrupt() if that is
+	 * the desired state that we've kept track of in interrupts_enabled
+	 * while the device was paused.
+	 */
+	if (devdata->interrupts_enabled)
+		visorbus_enable_channel_interrupts(dev);
+
 	rc = 0;
 out_locked:
-	up_write(&devdata->lock_visor_dev);
+	mutex_unlock(&devdata->lock_visor_dev);
 out:
 	return rc;
 }
@@ -675,7 +734,6 @@ static struct visor_channeltype_descriptor visorinput_channel_types[] = {
 
 static struct visor_driver visorinput_driver = {
 	.name = "visorinput",
-	.vertag = NULL,
 	.owner = THIS_MODULE,
 	.channel_types = visorinput_channel_types,
 	.probe = visorinput_probe,
@@ -704,8 +762,7 @@ MODULE_DEVICE_TABLE(visorbus, visorinput_channel_types);
 
 MODULE_AUTHOR("Unisys");
 MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("s-Par human input driver for guest Linux");
-MODULE_VERSION(VERSION);
+MODULE_DESCRIPTION("s-Par human input driver for virtual keyboard/mouse");
 
 MODULE_ALIAS("visorbus:" SPAR_MOUSE_CHANNEL_PROTOCOL_UUID_STR);
 MODULE_ALIAS("visorbus:" SPAR_KEYBOARD_CHANNEL_PROTOCOL_UUID_STR);
diff --git a/drivers/staging/unisys/visornic/visornic_main.c b/drivers/staging/unisys/visornic/visornic_main.c
index a28388d3ddc2..136700756485 100644
--- a/drivers/staging/unisys/visornic/visornic_main.c
+++ b/drivers/staging/unisys/visornic/visornic_main.c
@@ -29,8 +29,6 @@
 #include "iochannel.h"
 
 #define VISORNIC_INFINITE_RSP_WAIT 0
-#define VISORNICSOPENMAX 32
-#define MAXDEVICES     16384
 
 /* MAX_BUF = 64 lines x 32 MAXVNIC x 80 characters
  *         = 163840 bytes
@@ -38,27 +36,6 @@
 #define MAX_BUF 163840
 #define NAPI_WEIGHT 64
 
-static int visornic_probe(struct visor_device *dev);
-static void visornic_remove(struct visor_device *dev);
-static int visornic_pause(struct visor_device *dev,
-			  visorbus_state_complete_func complete_func);
-static int visornic_resume(struct visor_device *dev,
-			   visorbus_state_complete_func complete_func);
-
-/* DEBUGFS declarations */
-static ssize_t info_debugfs_read(struct file *file, char __user *buf,
-				 size_t len, loff_t *offset);
-static ssize_t enable_ints_write(struct file *file, const char __user *buf,
-				 size_t len, loff_t *ppos);
-static struct dentry *visornic_debugfs_dir;
-static const struct file_operations debugfs_info_fops = {
-	.read = info_debugfs_read,
-};
-
-static const struct file_operations debugfs_enable_ints_fops = {
-	.write = enable_ints_write,
-};
-
 /* GUIDS for director channel type supported by this driver.  */
 static struct visor_channeltype_descriptor visornic_channel_types[] = {
 	/* Note that the only channel type we expect to be reported by the
@@ -77,23 +54,6 @@ MODULE_DEVICE_TABLE(visorbus, visornic_channel_types);
  */
 MODULE_ALIAS("visorbus:" SPAR_VNIC_CHANNEL_PROTOCOL_UUID_STR);
 
-/* This is used to tell the visor bus driver which types of visor devices
- * we support, and what functions to call when a visor device that we support
- * is attached or removed.
- */
-static struct visor_driver visornic_driver = {
-	.name = "visornic",
-	.version = "1.0.0.0",
-	.vertag = NULL,
-	.owner = THIS_MODULE,
-	.channel_types = visornic_channel_types,
-	.probe = visornic_probe,
-	.remove = visornic_remove,
-	.pause = visornic_pause,
-	.resume = visornic_resume,
-	.channel_interrupt = NULL,
-};
-
 struct chanstat {
 	unsigned long got_rcv;
 	unsigned long got_enbdisack;
@@ -181,9 +141,6 @@ struct visornic_devdata {
 	struct uiscmdrsp cmdrsp[SIZEOF_CMDRSP];
 };
 
-static int visornic_poll(struct napi_struct *napi, int budget);
-static void poll_for_irq(unsigned long v);
-
 /**
  *	visor_copy_fragsinfo_from_skb(
  *	@skb_in: skbuff that we are pulling the frags from
@@ -289,6 +246,10 @@ static ssize_t enable_ints_write(struct file *file,
 	return count;
 }
 
+static const struct file_operations debugfs_enable_ints_fops = {
+	.write = enable_ints_write,
+};
+
 /**
  *	visornic_serverdown_complete - IOPART went down, pause device
  *	@work: Work queue it was scheduled on
@@ -425,9 +386,9 @@ post_skb(struct uiscmdrsp *cmdrsp,
 	if ((cmdrsp->net.rcvpost.frag.pi_off + skb->len) <= PI_PAGE_SIZE) {
 		cmdrsp->net.type = NET_RCV_POST;
 		cmdrsp->cmdtype = CMD_NET_TYPE;
-		if (visorchannel_signalinsert(devdata->dev->visorchannel,
-					      IOCHAN_TO_IOPART,
-					      cmdrsp)) {
+		if (!visorchannel_signalinsert(devdata->dev->visorchannel,
+					       IOCHAN_TO_IOPART,
+					       cmdrsp)) {
 			atomic_inc(&devdata->num_rcvbuf_in_iovm);
 			devdata->chstat.sent_post++;
 		} else {
@@ -454,9 +415,9 @@ send_enbdis(struct net_device *netdev, int state,
 	devdata->cmdrsp_rcv->net.enbdis.context = netdev;
 	devdata->cmdrsp_rcv->net.type = NET_RCV_ENBDIS;
 	devdata->cmdrsp_rcv->cmdtype = CMD_NET_TYPE;
-	if (visorchannel_signalinsert(devdata->dev->visorchannel,
-				      IOCHAN_TO_IOPART,
-				      devdata->cmdrsp_rcv))
+	if (!visorchannel_signalinsert(devdata->dev->visorchannel,
+				       IOCHAN_TO_IOPART,
+				       devdata->cmdrsp_rcv))
 		devdata->chstat.sent_enbdis++;
 }
 
@@ -920,8 +881,8 @@ visornic_xmit(struct sk_buff *skb, struct net_device *netdev)
 		return NETDEV_TX_OK;
 	}
 
-	if (!visorchannel_signalinsert(devdata->dev->visorchannel,
-				       IOCHAN_TO_IOPART, cmdrsp)) {
+	if (visorchannel_signalinsert(devdata->dev->visorchannel,
+				      IOCHAN_TO_IOPART, cmdrsp)) {
 		netif_stop_queue(netdev);
 		spin_unlock_irqrestore(&devdata->priv_lock, flags);
 		devdata->busy_cnt++;
@@ -1522,6 +1483,11 @@ static ssize_t info_debugfs_read(struct file *file, char __user *buf,
 	return bytes_read;
 }
 
+static struct dentry *visornic_debugfs_dir;
+static const struct file_operations debugfs_info_fops = {
+	.read = info_debugfs_read,
+};
+
 /**
  *	send_rcv_posts_if_needed
  *	@devdata: visornic device
@@ -1573,9 +1539,9 @@ send_rcv_posts_if_needed(struct visornic_devdata *devdata)
 static void
 drain_resp_queue(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata)
 {
-	while (visorchannel_signalremove(devdata->dev->visorchannel,
-					 IOCHAN_FROM_IOPART,
-					 cmdrsp))
+	while (!visorchannel_signalremove(devdata->dev->visorchannel,
+					  IOCHAN_FROM_IOPART,
+					  cmdrsp))
 		;
 }
 
@@ -1586,7 +1552,7 @@ drain_resp_queue(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata)
  *
  *	Drain the respones queue of any responses from the IO partition.
  *	Process the responses as we get them.
- *	Returns when response queue is empty or when the threadd stops.
+ *	Returns when response queue is empty or when the thread stops.
  */
 static void
 service_resp_queue(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata,
@@ -1599,9 +1565,9 @@ service_resp_queue(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata,
 	/* TODO: CLIENT ACQUIRE -- Don't really need this at the
 	 * moment
 	 */
-		if (!visorchannel_signalremove(devdata->dev->visorchannel,
-					       IOCHAN_FROM_IOPART,
-					       cmdrsp))
+		if (visorchannel_signalremove(devdata->dev->visorchannel,
+					      IOCHAN_FROM_IOPART,
+					      cmdrsp))
 			break; /* queue empty */
 
 		switch (cmdrsp->net.type) {
@@ -2061,6 +2027,21 @@ static int visornic_resume(struct visor_device *dev,
 	return 0;
 }
 
+/* This is used to tell the visor bus driver which types of visor devices
+ * we support, and what functions to call when a visor device that we support
+ * is attached or removed.
+ */
+static struct visor_driver visornic_driver = {
+	.name = "visornic",
+	.owner = THIS_MODULE,
+	.channel_types = visornic_channel_types,
+	.probe = visornic_probe,
+	.remove = visornic_remove,
+	.pause = visornic_pause,
+	.resume = visornic_resume,
+	.channel_interrupt = NULL,
+};
+
 /**
  *	visornic_init	- Init function
  *
@@ -2115,5 +2096,4 @@ module_exit(visornic_cleanup);
 
 MODULE_AUTHOR("Unisys");
 MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("sPAR nic driver for sparlinux: ver 1.0.0.0");
-MODULE_VERSION("1.0.0.0");
+MODULE_DESCRIPTION("s-Par NIC driver for virtual network devices");
diff --git a/drivers/staging/vc04_services/Kconfig b/drivers/staging/vc04_services/Kconfig
new file mode 100644
index 000000000000..9676fb29075a
--- /dev/null
+++ b/drivers/staging/vc04_services/Kconfig
@@ -0,0 +1,9 @@
+config BCM2708_VCHIQ
+	tristate "Videocore VCHIQ"
+	depends on RASPBERRYPI_FIRMWARE && BROKEN
+	default y
+	help
+		Kernel to VideoCore communication interface for the
+		BCM2708 family of products.
+		Defaults to Y when the Broadcom Videocore services
+		are included in the build, N otherwise.
diff --git a/drivers/staging/vc04_services/Makefile b/drivers/staging/vc04_services/Makefile
new file mode 100644
index 000000000000..90ab4781df2c
--- /dev/null
+++ b/drivers/staging/vc04_services/Makefile
@@ -0,0 +1,14 @@
+obj-$(CONFIG_BCM2708_VCHIQ)	+= vchiq.o
+
+vchiq-objs := \
+   interface/vchiq_arm/vchiq_core.o  \
+   interface/vchiq_arm/vchiq_arm.o \
+   interface/vchiq_arm/vchiq_kern_lib.o \
+   interface/vchiq_arm/vchiq_2835_arm.o \
+   interface/vchiq_arm/vchiq_debugfs.o \
+   interface/vchiq_arm/vchiq_shim.o \
+   interface/vchiq_arm/vchiq_util.o \
+   interface/vchiq_arm/vchiq_connected.o \
+
+ccflags-y += -DVCOS_VERIFY_BKPTS=1 -Idrivers/staging/vc04_services -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000
+
diff --git a/drivers/staging/vc04_services/interface/vchi/connections/connection.h b/drivers/staging/vc04_services/interface/vchi/connections/connection.h
new file mode 100644
index 000000000000..fef6ac34c6d2
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchi/connections/connection.h
@@ -0,0 +1,328 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef CONNECTION_H_
+#define CONNECTION_H_
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/semaphore.h>
+
+#include "interface/vchi/vchi_cfg_internal.h"
+#include "interface/vchi/vchi_common.h"
+#include "interface/vchi/message_drivers/message.h"
+
+/******************************************************************************
+ Global defs
+ *****************************************************************************/
+
+// Opaque handle for a connection / service pair
+typedef struct opaque_vchi_connection_connected_service_handle_t *VCHI_CONNECTION_SERVICE_HANDLE_T;
+
+// opaque handle to the connection state information
+typedef struct opaque_vchi_connection_info_t VCHI_CONNECTION_STATE_T;
+
+typedef struct vchi_connection_t VCHI_CONNECTION_T;
+
+
+/******************************************************************************
+ API
+ *****************************************************************************/
+
+// Routine to init a connection with a particular low level driver
+typedef VCHI_CONNECTION_STATE_T * (*VCHI_CONNECTION_INIT_T)( struct vchi_connection_t * connection,
+                                                             const VCHI_MESSAGE_DRIVER_T * driver );
+
+// Routine to control CRC enabling at a connection level
+typedef int32_t (*VCHI_CONNECTION_CRC_CONTROL_T)( VCHI_CONNECTION_STATE_T *state_handle,
+                                                  VCHI_CRC_CONTROL_T control );
+
+// Routine to create a service
+typedef int32_t (*VCHI_CONNECTION_SERVICE_CONNECT_T)( VCHI_CONNECTION_STATE_T *state_handle,
+                                                      int32_t service_id,
+                                                      uint32_t rx_fifo_size,
+                                                      uint32_t tx_fifo_size,
+                                                      int server,
+                                                      VCHI_CALLBACK_T callback,
+                                                      void *callback_param,
+                                                      int32_t want_crc,
+                                                      int32_t want_unaligned_bulk_rx,
+                                                      int32_t want_unaligned_bulk_tx,
+                                                      VCHI_CONNECTION_SERVICE_HANDLE_T *service_handle );
+
+// Routine to close a service
+typedef int32_t (*VCHI_CONNECTION_SERVICE_DISCONNECT_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle );
+
+// Routine to queue a message
+typedef int32_t (*VCHI_CONNECTION_SERVICE_QUEUE_MESSAGE_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                            const void *data,
+                                                            uint32_t data_size,
+                                                            VCHI_FLAGS_T flags,
+                                                            void *msg_handle );
+
+// scatter-gather (vector) message queueing
+typedef int32_t (*VCHI_CONNECTION_SERVICE_QUEUE_MESSAGEV_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                             VCHI_MSG_VECTOR_T *vector,
+                                                             uint32_t count,
+                                                             VCHI_FLAGS_T flags,
+                                                             void *msg_handle );
+
+// Routine to dequeue a message
+typedef int32_t (*VCHI_CONNECTION_SERVICE_DEQUEUE_MESSAGE_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                              void *data,
+                                                              uint32_t max_data_size_to_read,
+                                                              uint32_t *actual_msg_size,
+                                                              VCHI_FLAGS_T flags );
+
+// Routine to peek at a message
+typedef int32_t (*VCHI_CONNECTION_SERVICE_PEEK_MESSAGE_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                           void **data,
+                                                           uint32_t *msg_size,
+                                                           VCHI_FLAGS_T flags );
+
+// Routine to hold a message
+typedef int32_t (*VCHI_CONNECTION_SERVICE_HOLD_MESSAGE_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                           void **data,
+                                                           uint32_t *msg_size,
+                                                           VCHI_FLAGS_T flags,
+                                                           void **message_handle );
+
+// Routine to initialise a received message iterator
+typedef int32_t (*VCHI_CONNECTION_SERVICE_LOOKAHEAD_MESSAGE_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                                VCHI_MSG_ITER_T *iter,
+                                                                VCHI_FLAGS_T flags );
+
+// Routine to release a held message
+typedef int32_t (*VCHI_CONNECTION_HELD_MSG_RELEASE_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                       void *message_handle );
+
+// Routine to get info on a held message
+typedef int32_t (*VCHI_CONNECTION_HELD_MSG_INFO_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                    void *message_handle,
+                                                    void **data,
+                                                    int32_t *msg_size,
+                                                    uint32_t *tx_timestamp,
+                                                    uint32_t *rx_timestamp );
+
+// Routine to check whether the iterator has a next message
+typedef int32_t (*VCHI_CONNECTION_MSG_ITER_HAS_NEXT_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service,
+                                                       const VCHI_MSG_ITER_T *iter );
+
+// Routine to advance the iterator
+typedef int32_t (*VCHI_CONNECTION_MSG_ITER_NEXT_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service,
+                                                    VCHI_MSG_ITER_T *iter,
+                                                    void **data,
+                                                    uint32_t *msg_size );
+
+// Routine to remove the last message returned by the iterator
+typedef int32_t (*VCHI_CONNECTION_MSG_ITER_REMOVE_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service,
+                                                      VCHI_MSG_ITER_T *iter );
+
+// Routine to hold the last message returned by the iterator
+typedef int32_t (*VCHI_CONNECTION_MSG_ITER_HOLD_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service,
+                                                    VCHI_MSG_ITER_T *iter,
+                                                    void **msg_handle );
+
+// Routine to transmit bulk data
+typedef int32_t (*VCHI_CONNECTION_BULK_QUEUE_TRANSMIT_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                          const void *data_src,
+                                                          uint32_t data_size,
+                                                          VCHI_FLAGS_T flags,
+                                                          void *bulk_handle );
+
+// Routine to receive data
+typedef int32_t (*VCHI_CONNECTION_BULK_QUEUE_RECEIVE_T)( VCHI_CONNECTION_SERVICE_HANDLE_T service_handle,
+                                                         void *data_dst,
+                                                         uint32_t data_size,
+                                                         VCHI_FLAGS_T flags,
+                                                         void *bulk_handle );
+
+// Routine to report if a server is available
+typedef int32_t (*VCHI_CONNECTION_SERVER_PRESENT)( VCHI_CONNECTION_STATE_T *state, int32_t service_id, int32_t peer_flags );
+
+// Routine to report the number of RX slots available
+typedef int (*VCHI_CONNECTION_RX_SLOTS_AVAILABLE)( const VCHI_CONNECTION_STATE_T *state );
+
+// Routine to report the RX slot size
+typedef uint32_t (*VCHI_CONNECTION_RX_SLOT_SIZE)( const VCHI_CONNECTION_STATE_T *state );
+
+// Callback to indicate that the other side has added a buffer to the rx bulk DMA FIFO
+typedef void (*VCHI_CONNECTION_RX_BULK_BUFFER_ADDED)(VCHI_CONNECTION_STATE_T *state,
+                                                     int32_t service,
+                                                     uint32_t length,
+                                                     MESSAGE_TX_CHANNEL_T channel,
+                                                     uint32_t channel_params,
+                                                     uint32_t data_length,
+                                                     uint32_t data_offset);
+
+// Callback to inform a service that a Xon or Xoff message has been received
+typedef void (*VCHI_CONNECTION_FLOW_CONTROL)(VCHI_CONNECTION_STATE_T *state, int32_t service_id, int32_t xoff);
+
+// Callback to inform a service that a server available reply message has been received
+typedef void (*VCHI_CONNECTION_SERVER_AVAILABLE_REPLY)(VCHI_CONNECTION_STATE_T *state, int32_t service_id, uint32_t flags);
+
+// Callback to indicate that bulk auxiliary messages have arrived
+typedef void (*VCHI_CONNECTION_BULK_AUX_RECEIVED)(VCHI_CONNECTION_STATE_T *state);
+
+// Callback to indicate that bulk auxiliary messages have arrived
+typedef void (*VCHI_CONNECTION_BULK_AUX_TRANSMITTED)(VCHI_CONNECTION_STATE_T *state, void *handle);
+
+// Callback with all the connection info you require
+typedef void (*VCHI_CONNECTION_INFO)(VCHI_CONNECTION_STATE_T *state, uint32_t protocol_version, uint32_t slot_size, uint32_t num_slots, uint32_t min_bulk_size);
+
+// Callback to inform of a disconnect
+typedef void (*VCHI_CONNECTION_DISCONNECT)(VCHI_CONNECTION_STATE_T *state, uint32_t flags);
+
+// Callback to inform of a power control request
+typedef void (*VCHI_CONNECTION_POWER_CONTROL)(VCHI_CONNECTION_STATE_T *state, MESSAGE_TX_CHANNEL_T channel, int32_t enable);
+
+// allocate memory suitably aligned for this connection
+typedef void * (*VCHI_BUFFER_ALLOCATE)(VCHI_CONNECTION_SERVICE_HANDLE_T service_handle, uint32_t * length);
+
+// free memory allocated by buffer_allocate
+typedef void   (*VCHI_BUFFER_FREE)(VCHI_CONNECTION_SERVICE_HANDLE_T service_handle, void * address);
+
+
+/******************************************************************************
+ System driver struct
+ *****************************************************************************/
+
+struct opaque_vchi_connection_api_t
+{
+   // Routine to init the connection
+   VCHI_CONNECTION_INIT_T                      init;
+
+   // Connection-level CRC control
+   VCHI_CONNECTION_CRC_CONTROL_T               crc_control;
+
+   // Routine to connect to or create service
+   VCHI_CONNECTION_SERVICE_CONNECT_T           service_connect;
+
+   // Routine to disconnect from a service
+   VCHI_CONNECTION_SERVICE_DISCONNECT_T        service_disconnect;
+
+   // Routine to queue a message
+   VCHI_CONNECTION_SERVICE_QUEUE_MESSAGE_T     service_queue_msg;
+
+   // scatter-gather (vector) message queue
+   VCHI_CONNECTION_SERVICE_QUEUE_MESSAGEV_T    service_queue_msgv;
+
+   // Routine to dequeue a message
+   VCHI_CONNECTION_SERVICE_DEQUEUE_MESSAGE_T   service_dequeue_msg;
+
+   // Routine to peek at a message
+   VCHI_CONNECTION_SERVICE_PEEK_MESSAGE_T      service_peek_msg;
+
+   // Routine to hold a message
+   VCHI_CONNECTION_SERVICE_HOLD_MESSAGE_T      service_hold_msg;
+
+   // Routine to initialise a received message iterator
+   VCHI_CONNECTION_SERVICE_LOOKAHEAD_MESSAGE_T service_look_ahead_msg;
+
+   // Routine to release a message
+   VCHI_CONNECTION_HELD_MSG_RELEASE_T          held_msg_release;
+
+   // Routine to get information on a held message
+   VCHI_CONNECTION_HELD_MSG_INFO_T             held_msg_info;
+
+   // Routine to check for next message on iterator
+   VCHI_CONNECTION_MSG_ITER_HAS_NEXT_T         msg_iter_has_next;
+
+   // Routine to get next message on iterator
+   VCHI_CONNECTION_MSG_ITER_NEXT_T             msg_iter_next;
+
+   // Routine to remove the last message returned by iterator
+   VCHI_CONNECTION_MSG_ITER_REMOVE_T           msg_iter_remove;
+
+   // Routine to hold the last message returned by iterator
+   VCHI_CONNECTION_MSG_ITER_HOLD_T             msg_iter_hold;
+
+   // Routine to transmit bulk data
+   VCHI_CONNECTION_BULK_QUEUE_TRANSMIT_T       bulk_queue_transmit;
+
+   // Routine to receive data
+   VCHI_CONNECTION_BULK_QUEUE_RECEIVE_T        bulk_queue_receive;
+
+   // Routine to report the available servers
+   VCHI_CONNECTION_SERVER_PRESENT              server_present;
+
+   // Routine to report the number of RX slots available
+   VCHI_CONNECTION_RX_SLOTS_AVAILABLE          connection_rx_slots_available;
+
+   // Routine to report the RX slot size
+   VCHI_CONNECTION_RX_SLOT_SIZE                connection_rx_slot_size;
+
+   // Callback to indicate that the other side has added a buffer to the rx bulk DMA FIFO
+   VCHI_CONNECTION_RX_BULK_BUFFER_ADDED        rx_bulk_buffer_added;
+
+   // Callback to inform a service that a Xon or Xoff message has been received
+   VCHI_CONNECTION_FLOW_CONTROL                flow_control;
+
+   // Callback to inform a service that a server available reply message has been received
+   VCHI_CONNECTION_SERVER_AVAILABLE_REPLY      server_available_reply;
+
+   // Callback to indicate that bulk auxiliary messages have arrived
+   VCHI_CONNECTION_BULK_AUX_RECEIVED           bulk_aux_received;
+
+   // Callback to indicate that a bulk auxiliary message has been transmitted
+   VCHI_CONNECTION_BULK_AUX_TRANSMITTED        bulk_aux_transmitted;
+
+   // Callback to provide information about the connection
+   VCHI_CONNECTION_INFO                        connection_info;
+
+   // Callback to notify that peer has requested disconnect
+   VCHI_CONNECTION_DISCONNECT                  disconnect;
+
+   // Callback to notify that peer has requested power change
+   VCHI_CONNECTION_POWER_CONTROL               power_control;
+
+   // allocate memory suitably aligned for this connection
+   VCHI_BUFFER_ALLOCATE                        buffer_allocate;
+
+   // free memory allocated by buffer_allocate
+   VCHI_BUFFER_FREE                            buffer_free;
+
+};
+
+struct vchi_connection_t {
+   const VCHI_CONNECTION_API_T *api;
+   VCHI_CONNECTION_STATE_T     *state;
+#ifdef VCHI_COARSE_LOCKING
+   struct semaphore             sem;
+#endif
+};
+
+
+#endif /* CONNECTION_H_ */
+
+/****************************** End of file **********************************/
diff --git a/drivers/staging/vc04_services/interface/vchi/message_drivers/message.h b/drivers/staging/vc04_services/interface/vchi/message_drivers/message.h
new file mode 100644
index 000000000000..8b3f76735bd4
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchi/message_drivers/message.h
@@ -0,0 +1,204 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _VCHI_MESSAGE_H_
+#define _VCHI_MESSAGE_H_
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/semaphore.h>
+
+#include "interface/vchi/vchi_cfg_internal.h"
+#include "interface/vchi/vchi_common.h"
+
+
+typedef enum message_event_type {
+   MESSAGE_EVENT_NONE,
+   MESSAGE_EVENT_NOP,
+   MESSAGE_EVENT_MESSAGE,
+   MESSAGE_EVENT_SLOT_COMPLETE,
+   MESSAGE_EVENT_RX_BULK_PAUSED,
+   MESSAGE_EVENT_RX_BULK_COMPLETE,
+   MESSAGE_EVENT_TX_COMPLETE,
+   MESSAGE_EVENT_MSG_DISCARDED
+} MESSAGE_EVENT_TYPE_T;
+
+typedef enum vchi_msg_flags
+{
+   VCHI_MSG_FLAGS_NONE                  = 0x0,
+   VCHI_MSG_FLAGS_TERMINATE_DMA         = 0x1
+} VCHI_MSG_FLAGS_T;
+
+typedef enum message_tx_channel
+{
+   MESSAGE_TX_CHANNEL_MESSAGE           = 0,
+   MESSAGE_TX_CHANNEL_BULK              = 1 // drivers may provide multiple bulk channels, from 1 upwards
+} MESSAGE_TX_CHANNEL_T;
+
+// Macros used for cycling through bulk channels
+#define MESSAGE_TX_CHANNEL_BULK_PREV(c) (MESSAGE_TX_CHANNEL_BULK+((c)-MESSAGE_TX_CHANNEL_BULK+VCHI_MAX_BULK_TX_CHANNELS_PER_CONNECTION-1)%VCHI_MAX_BULK_TX_CHANNELS_PER_CONNECTION)
+#define MESSAGE_TX_CHANNEL_BULK_NEXT(c) (MESSAGE_TX_CHANNEL_BULK+((c)-MESSAGE_TX_CHANNEL_BULK+1)%VCHI_MAX_BULK_TX_CHANNELS_PER_CONNECTION)
+
+typedef enum message_rx_channel
+{
+   MESSAGE_RX_CHANNEL_MESSAGE           = 0,
+   MESSAGE_RX_CHANNEL_BULK              = 1 // drivers may provide multiple bulk channels, from 1 upwards
+} MESSAGE_RX_CHANNEL_T;
+
+// Message receive slot information
+typedef struct rx_msg_slot_info {
+
+   struct rx_msg_slot_info *next;
+   //struct slot_info *prev;
+#if !defined VCHI_COARSE_LOCKING
+   struct semaphore   sem;
+#endif
+
+   uint8_t           *addr;               // base address of slot
+   uint32_t           len;                // length of slot in bytes
+
+   uint32_t           write_ptr;          // hardware causes this to advance
+   uint32_t           read_ptr;           // this module does the reading
+   int                active;             // is this slot in the hardware dma fifo?
+   uint32_t           msgs_parsed;        // count how many messages are in this slot
+   uint32_t           msgs_released;      // how many messages have been released
+   void              *state;              // connection state information
+   uint8_t            ref_count[VCHI_MAX_SERVICES_PER_CONNECTION];          // reference count for slots held by services
+} RX_MSG_SLOTINFO_T;
+
+// The message driver no longer needs to know about the fields of RX_BULK_SLOTINFO_T - sort this out.
+// In particular, it mustn't use addr and len - they're the client buffer, but the message
+// driver will be tasked with sending the aligned core section.
+typedef struct rx_bulk_slotinfo_t {
+   struct rx_bulk_slotinfo_t *next;
+
+   struct semaphore *blocking;
+
+   // needed by DMA
+   void        *addr;
+   uint32_t     len;
+
+   // needed for the callback
+   void        *service;
+   void        *handle;
+   VCHI_FLAGS_T flags;
+} RX_BULK_SLOTINFO_T;
+
+
+/* ----------------------------------------------------------------------
+ * each connection driver will have a pool of the following struct.
+ *
+ * the pool will be managed by vchi_qman_*
+ * this means there will be multiple queues (single linked lists)
+ * a given struct message_info will be on exactly one of these queues
+ * at any one time
+ * -------------------------------------------------------------------- */
+typedef struct rx_message_info {
+
+   struct message_info *next;
+   //struct message_info *prev;
+
+   uint8_t    *addr;
+   uint32_t   len;
+   RX_MSG_SLOTINFO_T *slot; // points to whichever slot contains this message
+   uint32_t   tx_timestamp;
+   uint32_t   rx_timestamp;
+
+} RX_MESSAGE_INFO_T;
+
+typedef struct {
+   MESSAGE_EVENT_TYPE_T type;
+
+   struct {
+      // for messages
+      void    *addr;           // address of message
+      uint16_t slot_delta;     // whether this message indicated slot delta
+      uint32_t len;            // length of message
+      RX_MSG_SLOTINFO_T *slot; // slot this message is in
+      int32_t  service;   // service id this message is destined for
+      uint32_t tx_timestamp;   // timestamp from the header
+      uint32_t rx_timestamp;   // timestamp when we parsed it
+   } message;
+
+   // FIXME: cleanup slot reporting...
+   RX_MSG_SLOTINFO_T *rx_msg;
+   RX_BULK_SLOTINFO_T *rx_bulk;
+   void *tx_handle;
+   MESSAGE_TX_CHANNEL_T tx_channel;
+
+} MESSAGE_EVENT_T;
+
+
+// callbacks
+typedef void VCHI_MESSAGE_DRIVER_EVENT_CALLBACK_T( void *state );
+
+typedef struct {
+   VCHI_MESSAGE_DRIVER_EVENT_CALLBACK_T *event_callback;
+} VCHI_MESSAGE_DRIVER_OPEN_T;
+
+
+// handle to this instance of message driver (as returned by ->open)
+typedef struct opaque_mhandle_t *VCHI_MDRIVER_HANDLE_T;
+
+struct opaque_vchi_message_driver_t {
+   VCHI_MDRIVER_HANDLE_T *(*open)( VCHI_MESSAGE_DRIVER_OPEN_T *params, void *state );
+   int32_t (*suspending)( VCHI_MDRIVER_HANDLE_T *handle );
+   int32_t (*resumed)( VCHI_MDRIVER_HANDLE_T *handle );
+   int32_t (*power_control)( VCHI_MDRIVER_HANDLE_T *handle, MESSAGE_TX_CHANNEL_T, int32_t enable );
+   int32_t (*add_msg_rx_slot)( VCHI_MDRIVER_HANDLE_T *handle, RX_MSG_SLOTINFO_T *slot );      // rx message
+   int32_t (*add_bulk_rx)( VCHI_MDRIVER_HANDLE_T *handle, void *data, uint32_t len, RX_BULK_SLOTINFO_T *slot );  // rx data (bulk)
+   int32_t (*send)( VCHI_MDRIVER_HANDLE_T *handle, MESSAGE_TX_CHANNEL_T channel, const void *data, uint32_t len, VCHI_MSG_FLAGS_T flags, void *send_handle );      // tx (message & bulk)
+   void    (*next_event)( VCHI_MDRIVER_HANDLE_T *handle, MESSAGE_EVENT_T *event );     // get the next event from message_driver
+   int32_t (*enable)( VCHI_MDRIVER_HANDLE_T *handle );
+   int32_t (*form_message)( VCHI_MDRIVER_HANDLE_T *handle, int32_t service_id, VCHI_MSG_VECTOR_T *vector, uint32_t count, void
+                            *address, uint32_t length_avail, uint32_t max_total_length, int32_t pad_to_fill, int32_t allow_partial );
+
+   int32_t (*update_message)( VCHI_MDRIVER_HANDLE_T *handle, void *dest, int16_t *slot_count );
+   int32_t (*buffer_aligned)( VCHI_MDRIVER_HANDLE_T *handle, int tx, int uncached, const void *address, const uint32_t length );
+   void *  (*allocate_buffer)( VCHI_MDRIVER_HANDLE_T *handle, uint32_t *length );
+   void    (*free_buffer)( VCHI_MDRIVER_HANDLE_T *handle, void *address );
+   int     (*rx_slot_size)( VCHI_MDRIVER_HANDLE_T *handle, int msg_size );
+   int     (*tx_slot_size)( VCHI_MDRIVER_HANDLE_T *handle, int msg_size );
+
+   int32_t  (*tx_supports_terminate)( const VCHI_MDRIVER_HANDLE_T *handle, MESSAGE_TX_CHANNEL_T channel );
+   uint32_t (*tx_bulk_chunk_size)( const VCHI_MDRIVER_HANDLE_T *handle, MESSAGE_TX_CHANNEL_T channel );
+   int     (*tx_alignment)( const VCHI_MDRIVER_HANDLE_T *handle, MESSAGE_TX_CHANNEL_T channel );
+   int     (*rx_alignment)( const VCHI_MDRIVER_HANDLE_T *handle, MESSAGE_RX_CHANNEL_T channel );
+   void    (*form_bulk_aux)( VCHI_MDRIVER_HANDLE_T *handle, MESSAGE_TX_CHANNEL_T channel, const void *data, uint32_t len, uint32_t chunk_size, const void **aux_data, int32_t *aux_len );
+   void    (*debug)( VCHI_MDRIVER_HANDLE_T *handle );
+};
+
+
+#endif // _VCHI_MESSAGE_H_
+
+/****************************** End of file ***********************************/
diff --git a/drivers/staging/vc04_services/interface/vchi/vchi.h b/drivers/staging/vc04_services/interface/vchi/vchi.h
new file mode 100644
index 000000000000..1b17e98f7379
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchi/vchi.h
@@ -0,0 +1,378 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHI_H_
+#define VCHI_H_
+
+#include "interface/vchi/vchi_cfg.h"
+#include "interface/vchi/vchi_common.h"
+#include "interface/vchi/connections/connection.h"
+#include "vchi_mh.h"
+
+
+/******************************************************************************
+ Global defs
+ *****************************************************************************/
+
+#define VCHI_BULK_ROUND_UP(x)     ((((unsigned long)(x))+VCHI_BULK_ALIGN-1) & ~(VCHI_BULK_ALIGN-1))
+#define VCHI_BULK_ROUND_DOWN(x)   (((unsigned long)(x)) & ~(VCHI_BULK_ALIGN-1))
+#define VCHI_BULK_ALIGN_NBYTES(x) (VCHI_BULK_ALIGNED(x) ? 0 : (VCHI_BULK_ALIGN - ((unsigned long)(x) & (VCHI_BULK_ALIGN-1))))
+
+#ifdef USE_VCHIQ_ARM
+#define VCHI_BULK_ALIGNED(x)      1
+#else
+#define VCHI_BULK_ALIGNED(x)      (((unsigned long)(x) & (VCHI_BULK_ALIGN-1)) == 0)
+#endif
+
+struct vchi_version {
+	uint32_t version;
+	uint32_t version_min;
+};
+#define VCHI_VERSION(v_) { v_, v_ }
+#define VCHI_VERSION_EX(v_, m_) { v_, m_ }
+
+typedef enum
+{
+   VCHI_VEC_POINTER,
+   VCHI_VEC_HANDLE,
+   VCHI_VEC_LIST
+} VCHI_MSG_VECTOR_TYPE_T;
+
+typedef struct vchi_msg_vector_ex {
+
+   VCHI_MSG_VECTOR_TYPE_T type;
+   union
+   {
+      // a memory handle
+      struct
+      {
+         VCHI_MEM_HANDLE_T handle;
+         uint32_t offset;
+         int32_t vec_len;
+      } handle;
+
+      // an ordinary data pointer
+      struct
+      {
+         const void *vec_base;
+         int32_t vec_len;
+      } ptr;
+
+      // a nested vector list
+      struct
+      {
+         struct vchi_msg_vector_ex *vec;
+         uint32_t vec_len;
+      } list;
+   } u;
+} VCHI_MSG_VECTOR_EX_T;
+
+
+// Construct an entry in a msg vector for a pointer (p) of length (l)
+#define VCHI_VEC_POINTER(p,l)  VCHI_VEC_POINTER, { { (VCHI_MEM_HANDLE_T)(p), (l) } }
+
+// Construct an entry in a msg vector for a message handle (h), starting at offset (o) of length (l)
+#define VCHI_VEC_HANDLE(h,o,l) VCHI_VEC_HANDLE,  { { (h), (o), (l) } }
+
+// Macros to manipulate 'FOURCC' values
+#define MAKE_FOURCC(x) ((int32_t)( (x[0] << 24) | (x[1] << 16) | (x[2] << 8) | x[3] ))
+#define FOURCC_TO_CHAR(x) (x >> 24) & 0xFF,(x >> 16) & 0xFF,(x >> 8) & 0xFF, x & 0xFF
+
+
+// Opaque service information
+struct opaque_vchi_service_t;
+
+// Descriptor for a held message. Allocated by client, initialised by vchi_msg_hold,
+// vchi_msg_iter_hold or vchi_msg_iter_hold_next. Fields are for internal VCHI use only.
+typedef struct
+{
+   struct opaque_vchi_service_t *service;
+   void *message;
+} VCHI_HELD_MSG_T;
+
+
+
+// structure used to provide the information needed to open a server or a client
+typedef struct {
+	struct vchi_version version;
+	int32_t service_id;
+	VCHI_CONNECTION_T *connection;
+	uint32_t rx_fifo_size;
+	uint32_t tx_fifo_size;
+	VCHI_CALLBACK_T callback;
+	void *callback_param;
+	/* client intends to receive bulk transfers of
+		odd lengths or into unaligned buffers */
+	int32_t want_unaligned_bulk_rx;
+	/* client intends to transmit bulk transfers of
+		odd lengths or out of unaligned buffers */
+	int32_t want_unaligned_bulk_tx;
+	/* client wants to check CRCs on (bulk) xfers.
+		Only needs to be set at 1 end - will do both directions. */
+	int32_t want_crc;
+} SERVICE_CREATION_T;
+
+// Opaque handle for a VCHI instance
+typedef struct opaque_vchi_instance_handle_t *VCHI_INSTANCE_T;
+
+// Opaque handle for a server or client
+typedef struct opaque_vchi_service_handle_t *VCHI_SERVICE_HANDLE_T;
+
+// Service registration & startup
+typedef void (*VCHI_SERVICE_INIT)(VCHI_INSTANCE_T initialise_instance, VCHI_CONNECTION_T **connections, uint32_t num_connections);
+
+typedef struct service_info_tag {
+   const char * const vll_filename; /* VLL to load to start this service. This is an empty string if VLL is "static" */
+   VCHI_SERVICE_INIT init;          /* Service initialisation function */
+   void *vll_handle;                /* VLL handle; NULL when unloaded or a "static VLL" in build */
+} SERVICE_INFO_T;
+
+/******************************************************************************
+ Global funcs - implementation is specific to which side you are on (local / remote)
+ *****************************************************************************/
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern /*@observer@*/ VCHI_CONNECTION_T * vchi_create_connection( const VCHI_CONNECTION_API_T * function_table,
+                                                   const VCHI_MESSAGE_DRIVER_T * low_level);
+
+
+// Routine used to initialise the vchi on both local + remote connections
+extern int32_t vchi_initialise( VCHI_INSTANCE_T *instance_handle );
+
+extern int32_t vchi_exit( void );
+
+extern int32_t vchi_connect( VCHI_CONNECTION_T **connections,
+                             const uint32_t num_connections,
+                             VCHI_INSTANCE_T instance_handle );
+
+//When this is called, ensure that all services have no data pending.
+//Bulk transfers can remain 'queued'
+extern int32_t vchi_disconnect( VCHI_INSTANCE_T instance_handle );
+
+// Global control over bulk CRC checking
+extern int32_t vchi_crc_control( VCHI_CONNECTION_T *connection,
+                                 VCHI_CRC_CONTROL_T control );
+
+// helper functions
+extern void * vchi_allocate_buffer(VCHI_SERVICE_HANDLE_T handle, uint32_t *length);
+extern void vchi_free_buffer(VCHI_SERVICE_HANDLE_T handle, void *address);
+extern uint32_t vchi_current_time(VCHI_INSTANCE_T instance_handle);
+
+
+/******************************************************************************
+ Global service API
+ *****************************************************************************/
+// Routine to create a named service
+extern int32_t vchi_service_create( VCHI_INSTANCE_T instance_handle,
+                                    SERVICE_CREATION_T *setup,
+                                    VCHI_SERVICE_HANDLE_T *handle );
+
+// Routine to destory a service
+extern int32_t vchi_service_destroy( const VCHI_SERVICE_HANDLE_T handle );
+
+// Routine to open a named service
+extern int32_t vchi_service_open( VCHI_INSTANCE_T instance_handle,
+                                  SERVICE_CREATION_T *setup,
+                                  VCHI_SERVICE_HANDLE_T *handle);
+
+extern int32_t vchi_get_peer_version( const VCHI_SERVICE_HANDLE_T handle,
+                                      short *peer_version );
+
+// Routine to close a named service
+extern int32_t vchi_service_close( const VCHI_SERVICE_HANDLE_T handle );
+
+// Routine to increment ref count on a named service
+extern int32_t vchi_service_use( const VCHI_SERVICE_HANDLE_T handle );
+
+// Routine to decrement ref count on a named service
+extern int32_t vchi_service_release( const VCHI_SERVICE_HANDLE_T handle );
+
+// Routine to set a control option for a named service
+extern int32_t vchi_service_set_option( const VCHI_SERVICE_HANDLE_T handle,
+					VCHI_SERVICE_OPTION_T option,
+					int value);
+
+// Routine to send a message across a service
+extern int32_t vchi_msg_queue( VCHI_SERVICE_HANDLE_T handle,
+                               const void *data,
+                               uint32_t data_size,
+                               VCHI_FLAGS_T flags,
+                               void *msg_handle );
+
+// scatter-gather (vector) and send message
+int32_t vchi_msg_queuev_ex( VCHI_SERVICE_HANDLE_T handle,
+                            VCHI_MSG_VECTOR_EX_T *vector,
+                            uint32_t count,
+                            VCHI_FLAGS_T flags,
+                            void *msg_handle );
+
+// legacy scatter-gather (vector) and send message, only handles pointers
+int32_t vchi_msg_queuev( VCHI_SERVICE_HANDLE_T handle,
+                         VCHI_MSG_VECTOR_T *vector,
+                         uint32_t count,
+                         VCHI_FLAGS_T flags,
+                         void *msg_handle );
+
+// Routine to receive a msg from a service
+// Dequeue is equivalent to hold, copy into client buffer, release
+extern int32_t vchi_msg_dequeue( VCHI_SERVICE_HANDLE_T handle,
+                                 void *data,
+                                 uint32_t max_data_size_to_read,
+                                 uint32_t *actual_msg_size,
+                                 VCHI_FLAGS_T flags );
+
+// Routine to look at a message in place.
+// The message is not dequeued, so a subsequent call to peek or dequeue
+// will return the same message.
+extern int32_t vchi_msg_peek( VCHI_SERVICE_HANDLE_T handle,
+                              void **data,
+                              uint32_t *msg_size,
+                              VCHI_FLAGS_T flags );
+
+// Routine to remove a message after it has been read in place with peek
+// The first message on the queue is dequeued.
+extern int32_t vchi_msg_remove( VCHI_SERVICE_HANDLE_T handle );
+
+// Routine to look at a message in place.
+// The message is dequeued, so the caller is left holding it; the descriptor is
+// filled in and must be released when the user has finished with the message.
+extern int32_t vchi_msg_hold( VCHI_SERVICE_HANDLE_T handle,
+                              void **data,        // } may be NULL, as info can be
+                              uint32_t *msg_size, // } obtained from HELD_MSG_T
+                              VCHI_FLAGS_T flags,
+                              VCHI_HELD_MSG_T *message_descriptor );
+
+// Initialise an iterator to look through messages in place
+extern int32_t vchi_msg_look_ahead( VCHI_SERVICE_HANDLE_T handle,
+                                    VCHI_MSG_ITER_T *iter,
+                                    VCHI_FLAGS_T flags );
+
+/******************************************************************************
+ Global service support API - operations on held messages and message iterators
+ *****************************************************************************/
+
+// Routine to get the address of a held message
+extern void *vchi_held_msg_ptr( const VCHI_HELD_MSG_T *message );
+
+// Routine to get the size of a held message
+extern int32_t vchi_held_msg_size( const VCHI_HELD_MSG_T *message );
+
+// Routine to get the transmit timestamp as written into the header by the peer
+extern uint32_t vchi_held_msg_tx_timestamp( const VCHI_HELD_MSG_T *message );
+
+// Routine to get the reception timestamp, written as we parsed the header
+extern uint32_t vchi_held_msg_rx_timestamp( const VCHI_HELD_MSG_T *message );
+
+// Routine to release a held message after it has been processed
+extern int32_t vchi_held_msg_release( VCHI_HELD_MSG_T *message );
+
+// Indicates whether the iterator has a next message.
+extern int32_t vchi_msg_iter_has_next( const VCHI_MSG_ITER_T *iter );
+
+// Return the pointer and length for the next message and advance the iterator.
+extern int32_t vchi_msg_iter_next( VCHI_MSG_ITER_T *iter,
+                                   void **data,
+                                   uint32_t *msg_size );
+
+// Remove the last message returned by vchi_msg_iter_next.
+// Can only be called once after each call to vchi_msg_iter_next.
+extern int32_t vchi_msg_iter_remove( VCHI_MSG_ITER_T *iter );
+
+// Hold the last message returned by vchi_msg_iter_next.
+// Can only be called once after each call to vchi_msg_iter_next.
+extern int32_t vchi_msg_iter_hold( VCHI_MSG_ITER_T *iter,
+                                   VCHI_HELD_MSG_T *message );
+
+// Return information for the next message, and hold it, advancing the iterator.
+extern int32_t vchi_msg_iter_hold_next( VCHI_MSG_ITER_T *iter,
+                                        void **data,        // } may be NULL
+                                        uint32_t *msg_size, // }
+                                        VCHI_HELD_MSG_T *message );
+
+
+/******************************************************************************
+ Global bulk API
+ *****************************************************************************/
+
+// Routine to prepare interface for a transfer from the other side
+extern int32_t vchi_bulk_queue_receive( VCHI_SERVICE_HANDLE_T handle,
+                                        void *data_dst,
+                                        uint32_t data_size,
+                                        VCHI_FLAGS_T flags,
+                                        void *transfer_handle );
+
+
+// Prepare interface for a transfer from the other side into relocatable memory.
+int32_t vchi_bulk_queue_receive_reloc( const VCHI_SERVICE_HANDLE_T handle,
+                                       VCHI_MEM_HANDLE_T h_dst,
+                                       uint32_t offset,
+                                       uint32_t data_size,
+                                       const VCHI_FLAGS_T flags,
+                                       void * const bulk_handle );
+
+// Routine to queue up data ready for transfer to the other (once they have signalled they are ready)
+extern int32_t vchi_bulk_queue_transmit( VCHI_SERVICE_HANDLE_T handle,
+                                         const void *data_src,
+                                         uint32_t data_size,
+                                         VCHI_FLAGS_T flags,
+                                         void *transfer_handle );
+
+
+/******************************************************************************
+ Configuration plumbing
+ *****************************************************************************/
+
+// function prototypes for the different mid layers (the state info gives the different physical connections)
+extern const VCHI_CONNECTION_API_T *single_get_func_table( void );
+//extern const VCHI_CONNECTION_API_T *local_server_get_func_table( void );
+//extern const VCHI_CONNECTION_API_T *local_client_get_func_table( void );
+
+// declare all message drivers here
+const VCHI_MESSAGE_DRIVER_T *vchi_mphi_message_driver_func_table( void );
+
+#ifdef __cplusplus
+}
+#endif
+
+extern int32_t vchi_bulk_queue_transmit_reloc( VCHI_SERVICE_HANDLE_T handle,
+                                               VCHI_MEM_HANDLE_T h_src,
+                                               uint32_t offset,
+                                               uint32_t data_size,
+                                               VCHI_FLAGS_T flags,
+                                               void *transfer_handle );
+#endif /* VCHI_H_ */
+
+/****************************** End of file **********************************/
diff --git a/drivers/staging/vc04_services/interface/vchi/vchi_cfg.h b/drivers/staging/vc04_services/interface/vchi/vchi_cfg.h
new file mode 100644
index 000000000000..26bc2d38d725
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchi/vchi_cfg.h
@@ -0,0 +1,224 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHI_CFG_H_
+#define VCHI_CFG_H_
+
+/****************************************************************************************
+ * Defines in this first section are part of the VCHI API and may be examined by VCHI
+ * services.
+ ***************************************************************************************/
+
+/* Required alignment of base addresses for bulk transfer, if unaligned transfers are not enabled */
+/* Really determined by the message driver, and should be available from a run-time call. */
+#ifndef VCHI_BULK_ALIGN
+#   if __VCCOREVER__ >= 0x04000000
+#       define VCHI_BULK_ALIGN 32 // Allows for the need to do cache cleans
+#   else
+#       define VCHI_BULK_ALIGN 16
+#   endif
+#endif
+
+/* Required length multiple for bulk transfers, if unaligned transfers are not enabled */
+/* May be less than or greater than VCHI_BULK_ALIGN */
+/* Really determined by the message driver, and should be available from a run-time call. */
+#ifndef VCHI_BULK_GRANULARITY
+#   if __VCCOREVER__ >= 0x04000000
+#       define VCHI_BULK_GRANULARITY 32 // Allows for the need to do cache cleans
+#   else
+#       define VCHI_BULK_GRANULARITY 16
+#   endif
+#endif
+
+/* The largest possible message to be queued with vchi_msg_queue. */
+#ifndef VCHI_MAX_MSG_SIZE
+#   if defined VCHI_LOCAL_HOST_PORT
+#       define VCHI_MAX_MSG_SIZE     16384         // makes file transfers fast, but should they be using bulk?
+#   else
+#       define VCHI_MAX_MSG_SIZE      4096 // NOTE: THIS MUST BE LARGER THAN OR EQUAL TO THE SIZE OF THE KHRONOS MERGE BUFFER!!
+#   endif
+#endif
+
+/******************************************************************************************
+ * Defines below are system configuration options, and should not be used by VCHI services.
+ *****************************************************************************************/
+
+/* How many connections can we support? A localhost implementation uses 2 connections,
+ * 1 for host-app, 1 for VMCS, and these are hooked together by a loopback MPHI VCFW
+ * driver. */
+#ifndef VCHI_MAX_NUM_CONNECTIONS
+#   define VCHI_MAX_NUM_CONNECTIONS 3
+#endif
+
+/* How many services can we open per connection? Extending this doesn't cost processing time, just a small
+ * amount of static memory. */
+#ifndef VCHI_MAX_SERVICES_PER_CONNECTION
+#  define VCHI_MAX_SERVICES_PER_CONNECTION 36
+#endif
+
+/* Adjust if using a message driver that supports more logical TX channels */
+#ifndef VCHI_MAX_BULK_TX_CHANNELS_PER_CONNECTION
+#   define VCHI_MAX_BULK_TX_CHANNELS_PER_CONNECTION 9 // 1 MPHI + 8 CCP2 logical channels
+#endif
+
+/* Adjust if using a message driver that supports more logical RX channels */
+#ifndef VCHI_MAX_BULK_RX_CHANNELS_PER_CONNECTION
+#   define VCHI_MAX_BULK_RX_CHANNELS_PER_CONNECTION 1 // 1 MPHI
+#endif
+
+/* How many receive slots do we use. This times VCHI_MAX_MSG_SIZE gives the effective
+ * receive queue space, less message headers. */
+#ifndef VCHI_NUM_READ_SLOTS
+#  if defined(VCHI_LOCAL_HOST_PORT)
+#     define VCHI_NUM_READ_SLOTS 4
+#  else
+#     define VCHI_NUM_READ_SLOTS 48
+#  endif
+#endif
+
+/* Do we utilise overrun facility for receive message slots? Can aid peer transmit
+ * performance. Only define on VideoCore end, talking to host.
+ */
+//#define VCHI_MSG_RX_OVERRUN
+
+/* How many transmit slots do we use. Generally don't need many, as the hardware driver
+ * underneath VCHI will usually have its own buffering. */
+#ifndef VCHI_NUM_WRITE_SLOTS
+#  define VCHI_NUM_WRITE_SLOTS 4
+#endif
+
+/* If a service has held or queued received messages in VCHI_XOFF_THRESHOLD or more slots,
+ * then it's taking up too much buffer space, and the peer service will be told to stop
+ * transmitting with an XOFF message. For this to be effective, the VCHI_NUM_READ_SLOTS
+ * needs to be considerably bigger than VCHI_NUM_WRITE_SLOTS, or the transmit latency
+ * is too high. */
+#ifndef VCHI_XOFF_THRESHOLD
+#  define VCHI_XOFF_THRESHOLD (VCHI_NUM_READ_SLOTS / 2)
+#endif
+
+/* After we've sent an XOFF, the peer will be told to resume transmission once the local
+ * service has dequeued/released enough messages that it's now occupying
+ * VCHI_XON_THRESHOLD slots or fewer. */
+#ifndef VCHI_XON_THRESHOLD
+#  define VCHI_XON_THRESHOLD (VCHI_NUM_READ_SLOTS / 4)
+#endif
+
+/* A size below which a bulk transfer omits the handshake completely and always goes
+ * via the message channel, if bulk auxiliary is being sent on that service. (The user
+ * can guarantee this by enabling unaligned transmits).
+ * Not API. */
+#ifndef VCHI_MIN_BULK_SIZE
+#  define VCHI_MIN_BULK_SIZE    ( VCHI_MAX_MSG_SIZE / 2 < 4096 ? VCHI_MAX_MSG_SIZE / 2 : 4096 )
+#endif
+
+/* Maximum size of bulk transmission chunks, for each interface type. A trade-off between
+ * speed and latency; the smaller the chunk size the better change of messages and other
+ * bulk transmissions getting in when big bulk transfers are happening. Set to 0 to not
+ * break transmissions into chunks.
+ */
+#ifndef VCHI_MAX_BULK_CHUNK_SIZE_MPHI
+#  define VCHI_MAX_BULK_CHUNK_SIZE_MPHI (16 * 1024)
+#endif
+
+/* NB Chunked CCP2 transmissions violate the letter of the CCP2 spec by using "JPEG8" mode
+ * with multiple-line frames. Only use if the receiver can cope. */
+#ifndef VCHI_MAX_BULK_CHUNK_SIZE_CCP2
+#  define VCHI_MAX_BULK_CHUNK_SIZE_CCP2 0
+#endif
+
+/* How many TX messages can we have pending in our transmit slots. Once exhausted,
+ * vchi_msg_queue will be blocked. */
+#ifndef VCHI_TX_MSG_QUEUE_SIZE
+#  define VCHI_TX_MSG_QUEUE_SIZE           256
+#endif
+
+/* How many RX messages can we have parsed in the receive slots. Once exhausted, parsing
+ * will be suspended until older messages are dequeued/released. */
+#ifndef VCHI_RX_MSG_QUEUE_SIZE
+#  define VCHI_RX_MSG_QUEUE_SIZE           256
+#endif
+
+/* Really should be able to cope if we run out of received message descriptors, by
+ * suspending parsing as the comment above says, but we don't. This sweeps the issue
+ * under the carpet. */
+#if VCHI_RX_MSG_QUEUE_SIZE < (VCHI_MAX_MSG_SIZE/16 + 1) * VCHI_NUM_READ_SLOTS
+#  undef VCHI_RX_MSG_QUEUE_SIZE
+#  define VCHI_RX_MSG_QUEUE_SIZE (VCHI_MAX_MSG_SIZE/16 + 1) * VCHI_NUM_READ_SLOTS
+#endif
+
+/* How many bulk transmits can we have pending. Once exhausted, vchi_bulk_queue_transmit
+ * will be blocked. */
+#ifndef VCHI_TX_BULK_QUEUE_SIZE
+#  define VCHI_TX_BULK_QUEUE_SIZE           64
+#endif
+
+/* How many bulk receives can we have pending. Once exhausted, vchi_bulk_queue_receive
+ * will be blocked. */
+#ifndef VCHI_RX_BULK_QUEUE_SIZE
+#  define VCHI_RX_BULK_QUEUE_SIZE           64
+#endif
+
+/* A limit on how many outstanding bulk requests we expect the peer to give us. If
+ * the peer asks for more than this, VCHI will fail and assert. The number is determined
+ * by the peer's hardware - it's the number of outstanding requests that can be queued
+ * on all bulk channels. VC3's MPHI peripheral allows 16. */
+#ifndef VCHI_MAX_PEER_BULK_REQUESTS
+#  define VCHI_MAX_PEER_BULK_REQUESTS       32
+#endif
+
+/* Define VCHI_CCP2TX_MANUAL_POWER if the host tells us when to turn the CCP2
+ * transmitter on and off.
+ */
+/*#define VCHI_CCP2TX_MANUAL_POWER*/
+
+#ifndef VCHI_CCP2TX_MANUAL_POWER
+
+/* Timeout (in milliseconds) for putting the CCP2TX interface into IDLE state. Set
+ * negative for no IDLE.
+ */
+#  ifndef VCHI_CCP2TX_IDLE_TIMEOUT
+#    define VCHI_CCP2TX_IDLE_TIMEOUT        5
+#  endif
+
+/* Timeout (in milliseconds) for putting the CCP2TX interface into OFF state. Set
+ * negative for no OFF.
+ */
+#  ifndef VCHI_CCP2TX_OFF_TIMEOUT
+#    define VCHI_CCP2TX_OFF_TIMEOUT         1000
+#  endif
+
+#endif /* VCHI_CCP2TX_MANUAL_POWER */
+
+#endif /* VCHI_CFG_H_ */
+
+/****************************** End of file **********************************/
diff --git a/drivers/staging/vc04_services/interface/vchi/vchi_cfg_internal.h b/drivers/staging/vc04_services/interface/vchi/vchi_cfg_internal.h
new file mode 100644
index 000000000000..35dcba4837d4
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchi/vchi_cfg_internal.h
@@ -0,0 +1,71 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHI_CFG_INTERNAL_H_
+#define VCHI_CFG_INTERNAL_H_
+
+/****************************************************************************************
+ * Control optimisation attempts.
+ ***************************************************************************************/
+
+// Don't use lots of short-term locks - use great long ones, reducing the overall locks-per-second
+#define VCHI_COARSE_LOCKING
+
+// Avoid lock then unlock on exit from blocking queue operations (msg tx, bulk rx/tx)
+// (only relevant if VCHI_COARSE_LOCKING)
+#define VCHI_ELIDE_BLOCK_EXIT_LOCK
+
+// Avoid lock on non-blocking peek
+// (only relevant if VCHI_COARSE_LOCKING)
+#define VCHI_AVOID_PEEK_LOCK
+
+// Use one slot-handler thread per connection, rather than 1 thread dealing with all connections in rotation.
+#define VCHI_MULTIPLE_HANDLER_THREADS
+
+// Put free descriptors onto the head of the free queue, rather than the tail, so that we don't thrash
+// our way through the pool of descriptors.
+#define VCHI_PUSH_FREE_DESCRIPTORS_ONTO_HEAD
+
+// Don't issue a MSG_AVAILABLE callback for every single message. Possibly only safe if VCHI_COARSE_LOCKING.
+#define VCHI_FEWER_MSG_AVAILABLE_CALLBACKS
+
+// Don't use message descriptors for TX messages that don't need them
+#define VCHI_MINIMISE_TX_MSG_DESCRIPTORS
+
+// Nano-locks for multiqueue
+//#define VCHI_MQUEUE_NANOLOCKS
+
+// Lock-free(er) dequeuing
+//#define VCHI_RX_NANOLOCKS
+
+#endif /*VCHI_CFG_INTERNAL_H_*/
diff --git a/drivers/staging/vc04_services/interface/vchi/vchi_common.h b/drivers/staging/vc04_services/interface/vchi/vchi_common.h
new file mode 100644
index 000000000000..d535a72970d3
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchi/vchi_common.h
@@ -0,0 +1,175 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHI_COMMON_H_
+#define VCHI_COMMON_H_
+
+
+//flags used when sending messages (must be bitmapped)
+typedef enum
+{
+   VCHI_FLAGS_NONE                      = 0x0,
+   VCHI_FLAGS_BLOCK_UNTIL_OP_COMPLETE   = 0x1,   // waits for message to be received, or sent (NB. not the same as being seen on other side)
+   VCHI_FLAGS_CALLBACK_WHEN_OP_COMPLETE = 0x2,   // run a callback when message sent
+   VCHI_FLAGS_BLOCK_UNTIL_QUEUED        = 0x4,   // return once the transfer is in a queue ready to go
+   VCHI_FLAGS_ALLOW_PARTIAL             = 0x8,
+   VCHI_FLAGS_BLOCK_UNTIL_DATA_READ     = 0x10,
+   VCHI_FLAGS_CALLBACK_WHEN_DATA_READ   = 0x20,
+
+   VCHI_FLAGS_ALIGN_SLOT            = 0x000080,  // internal use only
+   VCHI_FLAGS_BULK_AUX_QUEUED       = 0x010000,  // internal use only
+   VCHI_FLAGS_BULK_AUX_COMPLETE     = 0x020000,  // internal use only
+   VCHI_FLAGS_BULK_DATA_QUEUED      = 0x040000,  // internal use only
+   VCHI_FLAGS_BULK_DATA_COMPLETE    = 0x080000,  // internal use only
+   VCHI_FLAGS_INTERNAL              = 0xFF0000
+} VCHI_FLAGS_T;
+
+// constants for vchi_crc_control()
+typedef enum {
+   VCHI_CRC_NOTHING = -1,
+   VCHI_CRC_PER_SERVICE = 0,
+   VCHI_CRC_EVERYTHING = 1,
+} VCHI_CRC_CONTROL_T;
+
+//callback reasons when an event occurs on a service
+typedef enum
+{
+   VCHI_CALLBACK_REASON_MIN,
+
+   //This indicates that there is data available
+   //handle is the msg id that was transmitted with the data
+   //    When a message is received and there was no FULL message available previously, send callback
+   //    Tasks get kicked by the callback, reset their event and try and read from the fifo until it fails
+   VCHI_CALLBACK_MSG_AVAILABLE,
+   VCHI_CALLBACK_MSG_SENT,
+   VCHI_CALLBACK_MSG_SPACE_AVAILABLE, // XXX not yet implemented
+
+   // This indicates that a transfer from the other side has completed
+   VCHI_CALLBACK_BULK_RECEIVED,
+   //This indicates that data queued up to be sent has now gone
+   //handle is the msg id that was used when sending the data
+   VCHI_CALLBACK_BULK_SENT,
+   VCHI_CALLBACK_BULK_RX_SPACE_AVAILABLE, // XXX not yet implemented
+   VCHI_CALLBACK_BULK_TX_SPACE_AVAILABLE, // XXX not yet implemented
+
+   VCHI_CALLBACK_SERVICE_CLOSED,
+
+   // this side has sent XOFF to peer due to lack of data consumption by service
+   // (suggests the service may need to take some recovery action if it has
+   // been deliberately holding off consuming data)
+   VCHI_CALLBACK_SENT_XOFF,
+   VCHI_CALLBACK_SENT_XON,
+
+   // indicates that a bulk transfer has finished reading the source buffer
+   VCHI_CALLBACK_BULK_DATA_READ,
+
+   // power notification events (currently host side only)
+   VCHI_CALLBACK_PEER_OFF,
+   VCHI_CALLBACK_PEER_SUSPENDED,
+   VCHI_CALLBACK_PEER_ON,
+   VCHI_CALLBACK_PEER_RESUMED,
+   VCHI_CALLBACK_FORCED_POWER_OFF,
+
+#ifdef USE_VCHIQ_ARM
+   // some extra notifications provided by vchiq_arm
+   VCHI_CALLBACK_SERVICE_OPENED,
+   VCHI_CALLBACK_BULK_RECEIVE_ABORTED,
+   VCHI_CALLBACK_BULK_TRANSMIT_ABORTED,
+#endif
+
+   VCHI_CALLBACK_REASON_MAX
+} VCHI_CALLBACK_REASON_T;
+
+// service control options
+typedef enum
+{
+   VCHI_SERVICE_OPTION_MIN,
+
+   VCHI_SERVICE_OPTION_TRACE,
+   VCHI_SERVICE_OPTION_SYNCHRONOUS,
+
+   VCHI_SERVICE_OPTION_MAX
+} VCHI_SERVICE_OPTION_T;
+
+
+//Callback used by all services / bulk transfers
+typedef void (*VCHI_CALLBACK_T)( void *callback_param, //my service local param
+                                 VCHI_CALLBACK_REASON_T reason,
+                                 void *handle ); //for transmitting msg's only
+
+
+
+/*
+ * Define vector struct for scatter-gather (vector) operations
+ * Vectors can be nested - if a vector element has negative length, then
+ * the data pointer is treated as pointing to another vector array, with
+ * '-vec_len' elements. Thus to append a header onto an existing vector,
+ * you can do this:
+ *
+ * void foo(const VCHI_MSG_VECTOR_T *v, int n)
+ * {
+ *    VCHI_MSG_VECTOR_T nv[2];
+ *    nv[0].vec_base = my_header;
+ *    nv[0].vec_len = sizeof my_header;
+ *    nv[1].vec_base = v;
+ *    nv[1].vec_len = -n;
+ *    ...
+ *
+ */
+typedef struct vchi_msg_vector {
+   const void *vec_base;
+   int32_t vec_len;
+} VCHI_MSG_VECTOR_T;
+
+// Opaque type for a connection API
+typedef struct opaque_vchi_connection_api_t VCHI_CONNECTION_API_T;
+
+// Opaque type for a message driver
+typedef struct opaque_vchi_message_driver_t VCHI_MESSAGE_DRIVER_T;
+
+
+// Iterator structure for reading ahead through received message queue. Allocated by client,
+// initialised by vchi_msg_look_ahead. Fields are for internal VCHI use only.
+// Iterates over messages in queue at the instant of the call to vchi_msg_lookahead -
+// will not proceed to messages received since. Behaviour is undefined if an iterator
+// is used again after messages for that service are removed/dequeued by any
+// means other than vchi_msg_iter_... calls on the iterator itself.
+typedef struct {
+   struct opaque_vchi_service_t *service;
+   void *last;
+   void *next;
+   void *remove;
+} VCHI_MSG_ITER_T;
+
+
+#endif // VCHI_COMMON_H_
diff --git a/drivers/staging/vc04_services/interface/vchi/vchi_mh.h b/drivers/staging/vc04_services/interface/vchi/vchi_mh.h
new file mode 100644
index 000000000000..198bd076b666
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchi/vchi_mh.h
@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHI_MH_H_
+#define VCHI_MH_H_
+
+#include <linux/types.h>
+
+typedef int32_t VCHI_MEM_HANDLE_T;
+#define VCHI_MEM_HANDLE_INVALID 0
+
+#endif
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq.h
new file mode 100644
index 000000000000..ad398bae6ee4
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq.h
@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_VCHIQ_H
+#define VCHIQ_VCHIQ_H
+
+#include "vchiq_if.h"
+#include "vchiq_util.h"
+
+#endif
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835.h
new file mode 100644
index 000000000000..7ea5c64d5343
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835.h
@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_2835_H
+#define VCHIQ_2835_H
+
+#include "vchiq_pagelist.h"
+
+#define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
+#define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX  1
+
+#endif /* VCHIQ_2835_H */
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c
new file mode 100644
index 000000000000..c29040fdf9a7
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c
@@ -0,0 +1,586 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/pagemap.h>
+#include <linux/dma-mapping.h>
+#include <linux/version.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/uaccess.h>
+#include <linux/of.h>
+#include <asm/pgtable.h>
+#include <soc/bcm2835/raspberrypi-firmware.h>
+
+#define dmac_map_area			__glue(_CACHE,_dma_map_area)
+#define dmac_unmap_area 		__glue(_CACHE,_dma_unmap_area)
+
+extern void dmac_map_area(const void *, size_t, int);
+extern void dmac_unmap_area(const void *, size_t, int);
+
+#define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
+
+#define VCHIQ_ARM_ADDRESS(x) ((void *)((char *)x + g_virt_to_bus_offset))
+
+#include "vchiq_arm.h"
+#include "vchiq_2835.h"
+#include "vchiq_connected.h"
+#include "vchiq_killable.h"
+
+#define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
+
+#define BELL0	0x00
+#define BELL2	0x08
+
+typedef struct vchiq_2835_state_struct {
+   int inited;
+   VCHIQ_ARM_STATE_T arm_state;
+} VCHIQ_2835_ARM_STATE_T;
+
+static void __iomem *g_regs;
+static unsigned int g_cache_line_size = sizeof(CACHE_LINE_SIZE);
+static unsigned int g_fragments_size;
+static char *g_fragments_base;
+static char *g_free_fragments;
+static struct semaphore g_free_fragments_sema;
+static unsigned long g_virt_to_bus_offset;
+
+extern int vchiq_arm_log_level;
+
+static DEFINE_SEMAPHORE(g_free_fragments_mutex);
+
+static irqreturn_t
+vchiq_doorbell_irq(int irq, void *dev_id);
+
+static int
+create_pagelist(char __user *buf, size_t count, unsigned short type,
+                struct task_struct *task, PAGELIST_T ** ppagelist);
+
+static void
+free_pagelist(PAGELIST_T *pagelist, int actual);
+
+int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state)
+{
+	struct device *dev = &pdev->dev;
+	struct rpi_firmware *fw = platform_get_drvdata(pdev);
+	VCHIQ_SLOT_ZERO_T *vchiq_slot_zero;
+	struct resource *res;
+	void *slot_mem;
+	dma_addr_t slot_phys;
+	u32 channelbase;
+	int slot_mem_size, frag_mem_size;
+	int err, irq, i;
+
+	g_virt_to_bus_offset = virt_to_dma(dev, (void *)0);
+
+	(void)of_property_read_u32(dev->of_node, "cache-line-size",
+				   &g_cache_line_size);
+	g_fragments_size = 2 * g_cache_line_size;
+
+	/* Allocate space for the channels in coherent memory */
+	slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
+	frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
+
+	slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
+				       &slot_phys, GFP_KERNEL);
+	if (!slot_mem) {
+		dev_err(dev, "could not allocate DMA memory\n");
+		return -ENOMEM;
+	}
+
+	WARN_ON(((int)slot_mem & (PAGE_SIZE - 1)) != 0);
+
+	vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
+	if (!vchiq_slot_zero)
+		return -EINVAL;
+
+	vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
+		(int)slot_phys + slot_mem_size;
+	vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
+		MAX_FRAGMENTS;
+
+	g_fragments_base = (char *)slot_mem + slot_mem_size;
+	slot_mem_size += frag_mem_size;
+
+	g_free_fragments = g_fragments_base;
+	for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
+		*(char **)&g_fragments_base[i*g_fragments_size] =
+			&g_fragments_base[(i + 1)*g_fragments_size];
+	}
+	*(char **)&g_fragments_base[i * g_fragments_size] = NULL;
+	sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
+
+	if (vchiq_init_state(state, vchiq_slot_zero, 0) != VCHIQ_SUCCESS)
+		return -EINVAL;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	g_regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(g_regs))
+		return PTR_ERR(g_regs);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq <= 0) {
+		dev_err(dev, "failed to get IRQ\n");
+		return irq;
+	}
+
+	err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
+			       "VCHIQ doorbell", state);
+	if (err) {
+		dev_err(dev, "failed to register irq=%d\n", irq);
+		return err;
+	}
+
+	/* Send the base address of the slots to VideoCore */
+	channelbase = slot_phys;
+	err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
+				    &channelbase, sizeof(channelbase));
+	if (err || channelbase) {
+		dev_err(dev, "failed to set channelbase\n");
+		return err ? : -ENXIO;
+	}
+
+	vchiq_log_info(vchiq_arm_log_level,
+		"vchiq_init - done (slots %x, phys %pad)",
+		(unsigned int)vchiq_slot_zero, &slot_phys);
+
+	vchiq_call_connected_callbacks();
+
+   return 0;
+}
+
+VCHIQ_STATUS_T
+vchiq_platform_init_state(VCHIQ_STATE_T *state)
+{
+   VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+   state->platform_state = kzalloc(sizeof(VCHIQ_2835_ARM_STATE_T), GFP_KERNEL);
+   ((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited = 1;
+   status = vchiq_arm_init_state(state, &((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->arm_state);
+   if(status != VCHIQ_SUCCESS)
+   {
+      ((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited = 0;
+   }
+   return status;
+}
+
+VCHIQ_ARM_STATE_T*
+vchiq_platform_get_arm_state(VCHIQ_STATE_T *state)
+{
+   if(!((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited)
+   {
+      BUG();
+   }
+   return &((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->arm_state;
+}
+
+void
+remote_event_signal(REMOTE_EVENT_T *event)
+{
+	wmb();
+
+	event->fired = 1;
+
+	dsb();         /* data barrier operation */
+
+	if (event->armed)
+		writel(0, g_regs + BELL2); /* trigger vc interrupt */
+}
+
+int
+vchiq_copy_from_user(void *dst, const void *src, int size)
+{
+	if ((uint32_t)src < TASK_SIZE) {
+		return copy_from_user(dst, src, size);
+	} else {
+		memcpy(dst, src, size);
+		return 0;
+	}
+}
+
+VCHIQ_STATUS_T
+vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle,
+	void *offset, int size, int dir)
+{
+	PAGELIST_T *pagelist;
+	int ret;
+
+	WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID);
+
+	ret = create_pagelist((char __user *)offset, size,
+			(dir == VCHIQ_BULK_RECEIVE)
+			? PAGELIST_READ
+			: PAGELIST_WRITE,
+			current,
+			&pagelist);
+	if (ret != 0)
+		return VCHIQ_ERROR;
+
+	bulk->handle = memhandle;
+	bulk->data = VCHIQ_ARM_ADDRESS(pagelist);
+
+	/* Store the pagelist address in remote_data, which isn't used by the
+	   slave. */
+	bulk->remote_data = pagelist;
+
+	return VCHIQ_SUCCESS;
+}
+
+void
+vchiq_complete_bulk(VCHIQ_BULK_T *bulk)
+{
+	if (bulk && bulk->remote_data && bulk->actual)
+		free_pagelist((PAGELIST_T *)bulk->remote_data, bulk->actual);
+}
+
+void
+vchiq_transfer_bulk(VCHIQ_BULK_T *bulk)
+{
+	/*
+	 * This should only be called on the master (VideoCore) side, but
+	 * provide an implementation to avoid the need for ifdefery.
+	 */
+	BUG();
+}
+
+void
+vchiq_dump_platform_state(void *dump_context)
+{
+	char buf[80];
+	int len;
+	len = snprintf(buf, sizeof(buf),
+		"  Platform: 2835 (VC master)");
+	vchiq_dump(dump_context, buf, len + 1);
+}
+
+VCHIQ_STATUS_T
+vchiq_platform_suspend(VCHIQ_STATE_T *state)
+{
+   return VCHIQ_ERROR;
+}
+
+VCHIQ_STATUS_T
+vchiq_platform_resume(VCHIQ_STATE_T *state)
+{
+   return VCHIQ_SUCCESS;
+}
+
+void
+vchiq_platform_paused(VCHIQ_STATE_T *state)
+{
+}
+
+void
+vchiq_platform_resumed(VCHIQ_STATE_T *state)
+{
+}
+
+int
+vchiq_platform_videocore_wanted(VCHIQ_STATE_T* state)
+{
+   return 1; // autosuspend not supported - videocore always wanted
+}
+
+int
+vchiq_platform_use_suspend_timer(void)
+{
+   return 0;
+}
+void
+vchiq_dump_platform_use_state(VCHIQ_STATE_T *state)
+{
+	vchiq_log_info(vchiq_arm_log_level, "Suspend timer not in use");
+}
+void
+vchiq_platform_handle_timeout(VCHIQ_STATE_T *state)
+{
+	(void)state;
+}
+/*
+ * Local functions
+ */
+
+static irqreturn_t
+vchiq_doorbell_irq(int irq, void *dev_id)
+{
+	VCHIQ_STATE_T *state = dev_id;
+	irqreturn_t ret = IRQ_NONE;
+	unsigned int status;
+
+	/* Read (and clear) the doorbell */
+	status = readl(g_regs + BELL0);
+
+	if (status & 0x4) {  /* Was the doorbell rung? */
+		remote_event_pollall(state);
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+}
+
+/* There is a potential problem with partial cache lines (pages?)
+** at the ends of the block when reading. If the CPU accessed anything in
+** the same line (page?) then it may have pulled old data into the cache,
+** obscuring the new data underneath. We can solve this by transferring the
+** partial cache lines separately, and allowing the ARM to copy into the
+** cached area.
+
+** N.B. This implementation plays slightly fast and loose with the Linux
+** driver programming rules, e.g. its use of dmac_map_area instead of
+** dma_map_single, but it isn't a multi-platform driver and it benefits
+** from increased speed as a result.
+*/
+
+static int
+create_pagelist(char __user *buf, size_t count, unsigned short type,
+	struct task_struct *task, PAGELIST_T ** ppagelist)
+{
+	PAGELIST_T *pagelist;
+	struct page **pages;
+	unsigned long *addrs;
+	unsigned int num_pages, offset, i;
+	char *addr, *base_addr, *next_addr;
+	int run, addridx, actual_pages;
+        unsigned long *need_release;
+
+	offset = (unsigned int)buf & (PAGE_SIZE - 1);
+	num_pages = (count + offset + PAGE_SIZE - 1) / PAGE_SIZE;
+
+	*ppagelist = NULL;
+
+	/* Allocate enough storage to hold the page pointers and the page
+	** list
+	*/
+	pagelist = kmalloc(sizeof(PAGELIST_T) +
+                           (num_pages * sizeof(unsigned long)) +
+                           sizeof(unsigned long) +
+                           (num_pages * sizeof(pages[0])),
+                           GFP_KERNEL);
+
+	vchiq_log_trace(vchiq_arm_log_level,
+		"create_pagelist - %x", (unsigned int)pagelist);
+	if (!pagelist)
+		return -ENOMEM;
+
+	addrs = pagelist->addrs;
+        need_release = (unsigned long *)(addrs + num_pages);
+	pages = (struct page **)(addrs + num_pages + 1);
+
+	if (is_vmalloc_addr(buf)) {
+		int dir = (type == PAGELIST_WRITE) ?
+			DMA_TO_DEVICE : DMA_FROM_DEVICE;
+		unsigned long length = count;
+		unsigned int off = offset;
+
+		for (actual_pages = 0; actual_pages < num_pages;
+		     actual_pages++) {
+			struct page *pg = vmalloc_to_page(buf + (actual_pages *
+								 PAGE_SIZE));
+			size_t bytes = PAGE_SIZE - off;
+
+			if (bytes > length)
+				bytes = length;
+			pages[actual_pages] = pg;
+			dmac_map_area(page_address(pg) + off, bytes, dir);
+			length -= bytes;
+			off = 0;
+		}
+		*need_release = 0; /* do not try and release vmalloc pages */
+	} else {
+		down_read(&task->mm->mmap_sem);
+		actual_pages = get_user_pages(task, task->mm,
+				          (unsigned long)buf & ~(PAGE_SIZE - 1),
+					  num_pages,
+					  (type == PAGELIST_READ) /*Write */ ,
+					  0 /*Force */ ,
+					  pages,
+					  NULL /*vmas */);
+		up_read(&task->mm->mmap_sem);
+
+		if (actual_pages != num_pages) {
+			vchiq_log_info(vchiq_arm_log_level,
+				       "create_pagelist - only %d/%d pages locked",
+				       actual_pages,
+				       num_pages);
+
+			/* This is probably due to the process being killed */
+			while (actual_pages > 0)
+			{
+				actual_pages--;
+				page_cache_release(pages[actual_pages]);
+			}
+			kfree(pagelist);
+			if (actual_pages == 0)
+				actual_pages = -ENOMEM;
+			return actual_pages;
+		}
+		*need_release = 1; /* release user pages */
+	}
+
+	pagelist->length = count;
+	pagelist->type = type;
+	pagelist->offset = offset;
+
+	/* Group the pages into runs of contiguous pages */
+
+	base_addr = VCHIQ_ARM_ADDRESS(page_address(pages[0]));
+	next_addr = base_addr + PAGE_SIZE;
+	addridx = 0;
+	run = 0;
+
+	for (i = 1; i < num_pages; i++) {
+		addr = VCHIQ_ARM_ADDRESS(page_address(pages[i]));
+		if ((addr == next_addr) && (run < (PAGE_SIZE - 1))) {
+			next_addr += PAGE_SIZE;
+			run++;
+		} else {
+			addrs[addridx] = (unsigned long)base_addr + run;
+			addridx++;
+			base_addr = addr;
+			next_addr = addr + PAGE_SIZE;
+			run = 0;
+		}
+	}
+
+	addrs[addridx] = (unsigned long)base_addr + run;
+	addridx++;
+
+	/* Partial cache lines (fragments) require special measures */
+	if ((type == PAGELIST_READ) &&
+		((pagelist->offset & (g_cache_line_size - 1)) ||
+		((pagelist->offset + pagelist->length) &
+		(g_cache_line_size - 1)))) {
+		char *fragments;
+
+		if (down_interruptible(&g_free_fragments_sema) != 0) {
+			kfree(pagelist);
+			return -EINTR;
+		}
+
+		WARN_ON(g_free_fragments == NULL);
+
+		down(&g_free_fragments_mutex);
+		fragments = g_free_fragments;
+		WARN_ON(fragments == NULL);
+		g_free_fragments = *(char **) g_free_fragments;
+		up(&g_free_fragments_mutex);
+		pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
+			(fragments - g_fragments_base) / g_fragments_size;
+	}
+
+	dmac_flush_range(pagelist, addrs + num_pages);
+
+	*ppagelist = pagelist;
+
+	return 0;
+}
+
+static void
+free_pagelist(PAGELIST_T *pagelist, int actual)
+{
+        unsigned long *need_release;
+	struct page **pages;
+	unsigned int num_pages, i;
+
+	vchiq_log_trace(vchiq_arm_log_level,
+		"free_pagelist - %x, %d", (unsigned int)pagelist, actual);
+
+	num_pages =
+		(pagelist->length + pagelist->offset + PAGE_SIZE - 1) /
+		PAGE_SIZE;
+
+        need_release = (unsigned long *)(pagelist->addrs + num_pages);
+	pages = (struct page **)(pagelist->addrs + num_pages + 1);
+
+	/* Deal with any partial cache lines (fragments) */
+	if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
+		char *fragments = g_fragments_base +
+			(pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
+			g_fragments_size;
+		int head_bytes, tail_bytes;
+		head_bytes = (g_cache_line_size - pagelist->offset) &
+			(g_cache_line_size - 1);
+		tail_bytes = (pagelist->offset + actual) &
+			(g_cache_line_size - 1);
+
+		if ((actual >= 0) && (head_bytes != 0)) {
+			if (head_bytes > actual)
+				head_bytes = actual;
+
+			memcpy((char *)page_address(pages[0]) +
+				pagelist->offset,
+				fragments,
+				head_bytes);
+		}
+		if ((actual >= 0) && (head_bytes < actual) &&
+			(tail_bytes != 0)) {
+			memcpy((char *)page_address(pages[num_pages - 1]) +
+				((pagelist->offset + actual) &
+				(PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
+				fragments + g_cache_line_size,
+				tail_bytes);
+		}
+
+		down(&g_free_fragments_mutex);
+		*(char **)fragments = g_free_fragments;
+		g_free_fragments = fragments;
+		up(&g_free_fragments_mutex);
+		up(&g_free_fragments_sema);
+	}
+
+	if (*need_release) {
+		unsigned int length = pagelist->length;
+		unsigned int offset = pagelist->offset;
+
+		for (i = 0; i < num_pages; i++) {
+			struct page *pg = pages[i];
+
+			if (pagelist->type != PAGELIST_WRITE) {
+				unsigned int bytes = PAGE_SIZE - offset;
+
+				if (bytes > length)
+					bytes = length;
+				dmac_unmap_area(page_address(pg) + offset,
+						bytes, DMA_FROM_DEVICE);
+				length -= bytes;
+				offset = 0;
+				set_page_dirty(pg);
+			}
+			page_cache_release(pg);
+		}
+	}
+
+	kfree(pagelist);
+}
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
new file mode 100644
index 000000000000..e11c0e07471b
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
@@ -0,0 +1,2903 @@
+/**
+ * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved.
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/device.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#include <linux/pagemap.h>
+#include <linux/bug.h>
+#include <linux/semaphore.h>
+#include <linux/list.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <soc/bcm2835/raspberrypi-firmware.h>
+
+#include "vchiq_core.h"
+#include "vchiq_ioctl.h"
+#include "vchiq_arm.h"
+#include "vchiq_debugfs.h"
+#include "vchiq_killable.h"
+
+#define DEVICE_NAME "vchiq"
+
+/* Override the default prefix, which would be vchiq_arm (from the filename) */
+#undef MODULE_PARAM_PREFIX
+#define MODULE_PARAM_PREFIX DEVICE_NAME "."
+
+#define VCHIQ_MINOR 0
+
+/* Some per-instance constants */
+#define MAX_COMPLETIONS 16
+#define MAX_SERVICES 64
+#define MAX_ELEMENTS 8
+#define MSG_QUEUE_SIZE 64
+
+#define KEEPALIVE_VER 1
+#define KEEPALIVE_VER_MIN KEEPALIVE_VER
+
+/* Run time control of log level, based on KERN_XXX level. */
+int vchiq_arm_log_level = VCHIQ_LOG_DEFAULT;
+int vchiq_susp_log_level = VCHIQ_LOG_ERROR;
+
+#define SUSPEND_TIMER_TIMEOUT_MS 100
+#define SUSPEND_RETRY_TIMER_TIMEOUT_MS 1000
+
+#define VC_SUSPEND_NUM_OFFSET 3 /* number of values before idle which are -ve */
+static const char *const suspend_state_names[] = {
+	"VC_SUSPEND_FORCE_CANCELED",
+	"VC_SUSPEND_REJECTED",
+	"VC_SUSPEND_FAILED",
+	"VC_SUSPEND_IDLE",
+	"VC_SUSPEND_REQUESTED",
+	"VC_SUSPEND_IN_PROGRESS",
+	"VC_SUSPEND_SUSPENDED"
+};
+#define VC_RESUME_NUM_OFFSET 1 /* number of values before idle which are -ve */
+static const char *const resume_state_names[] = {
+	"VC_RESUME_FAILED",
+	"VC_RESUME_IDLE",
+	"VC_RESUME_REQUESTED",
+	"VC_RESUME_IN_PROGRESS",
+	"VC_RESUME_RESUMED"
+};
+/* The number of times we allow force suspend to timeout before actually
+** _forcing_ suspend.  This is to cater for SW which fails to release vchiq
+** correctly - we don't want to prevent ARM suspend indefinitely in this case.
+*/
+#define FORCE_SUSPEND_FAIL_MAX 8
+
+/* The time in ms allowed for videocore to go idle when force suspend has been
+ * requested */
+#define FORCE_SUSPEND_TIMEOUT_MS 200
+
+
+static void suspend_timer_callback(unsigned long context);
+
+
+typedef struct user_service_struct {
+	VCHIQ_SERVICE_T *service;
+	void *userdata;
+	VCHIQ_INSTANCE_T instance;
+	char is_vchi;
+	char dequeue_pending;
+	char close_pending;
+	int message_available_pos;
+	int msg_insert;
+	int msg_remove;
+	struct semaphore insert_event;
+	struct semaphore remove_event;
+	struct semaphore close_event;
+	VCHIQ_HEADER_T * msg_queue[MSG_QUEUE_SIZE];
+} USER_SERVICE_T;
+
+struct bulk_waiter_node {
+	struct bulk_waiter bulk_waiter;
+	int pid;
+	struct list_head list;
+};
+
+struct vchiq_instance_struct {
+	VCHIQ_STATE_T *state;
+	VCHIQ_COMPLETION_DATA_T completions[MAX_COMPLETIONS];
+	int completion_insert;
+	int completion_remove;
+	struct semaphore insert_event;
+	struct semaphore remove_event;
+	struct mutex completion_mutex;
+
+	int connected;
+	int closing;
+	int pid;
+	int mark;
+	int use_close_delivered;
+	int trace;
+
+	struct list_head bulk_waiter_list;
+	struct mutex bulk_waiter_list_mutex;
+
+	VCHIQ_DEBUGFS_NODE_T debugfs_node;
+};
+
+typedef struct dump_context_struct {
+	char __user *buf;
+	size_t actual;
+	size_t space;
+	loff_t offset;
+} DUMP_CONTEXT_T;
+
+static struct cdev    vchiq_cdev;
+static dev_t          vchiq_devid;
+static VCHIQ_STATE_T g_state;
+static struct class  *vchiq_class;
+static struct device *vchiq_dev;
+static DEFINE_SPINLOCK(msg_queue_spinlock);
+
+static const char *const ioctl_names[] = {
+	"CONNECT",
+	"SHUTDOWN",
+	"CREATE_SERVICE",
+	"REMOVE_SERVICE",
+	"QUEUE_MESSAGE",
+	"QUEUE_BULK_TRANSMIT",
+	"QUEUE_BULK_RECEIVE",
+	"AWAIT_COMPLETION",
+	"DEQUEUE_MESSAGE",
+	"GET_CLIENT_ID",
+	"GET_CONFIG",
+	"CLOSE_SERVICE",
+	"USE_SERVICE",
+	"RELEASE_SERVICE",
+	"SET_SERVICE_OPTION",
+	"DUMP_PHYS_MEM",
+	"LIB_VERSION",
+	"CLOSE_DELIVERED"
+};
+
+vchiq_static_assert((sizeof(ioctl_names)/sizeof(ioctl_names[0])) ==
+	(VCHIQ_IOC_MAX + 1));
+
+static void
+dump_phys_mem(void *virt_addr, uint32_t num_bytes);
+
+/****************************************************************************
+*
+*   add_completion
+*
+***************************************************************************/
+
+static VCHIQ_STATUS_T
+add_completion(VCHIQ_INSTANCE_T instance, VCHIQ_REASON_T reason,
+	VCHIQ_HEADER_T *header, USER_SERVICE_T *user_service,
+	void *bulk_userdata)
+{
+	VCHIQ_COMPLETION_DATA_T *completion;
+	DEBUG_INITIALISE(g_state.local)
+
+	while (instance->completion_insert ==
+		(instance->completion_remove + MAX_COMPLETIONS)) {
+		/* Out of space - wait for the client */
+		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+		vchiq_log_trace(vchiq_arm_log_level,
+			"add_completion - completion queue full");
+		DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT);
+		if (down_interruptible(&instance->remove_event) != 0) {
+			vchiq_log_info(vchiq_arm_log_level,
+				"service_callback interrupted");
+			return VCHIQ_RETRY;
+		} else if (instance->closing) {
+			vchiq_log_info(vchiq_arm_log_level,
+				"service_callback closing");
+			return VCHIQ_ERROR;
+		}
+		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+	}
+
+	completion =
+		 &instance->completions[instance->completion_insert &
+		 (MAX_COMPLETIONS - 1)];
+
+	completion->header = header;
+	completion->reason = reason;
+	/* N.B. service_userdata is updated while processing AWAIT_COMPLETION */
+	completion->service_userdata = user_service->service;
+	completion->bulk_userdata = bulk_userdata;
+
+	if (reason == VCHIQ_SERVICE_CLOSED) {
+		/* Take an extra reference, to be held until
+		   this CLOSED notification is delivered. */
+		lock_service(user_service->service);
+		if (instance->use_close_delivered)
+			user_service->close_pending = 1;
+	}
+
+	/* A write barrier is needed here to ensure that the entire completion
+		record is written out before the insert point. */
+	wmb();
+
+	if (reason == VCHIQ_MESSAGE_AVAILABLE)
+		user_service->message_available_pos =
+			instance->completion_insert;
+	instance->completion_insert++;
+
+	up(&instance->insert_event);
+
+	return VCHIQ_SUCCESS;
+}
+
+/****************************************************************************
+*
+*   service_callback
+*
+***************************************************************************/
+
+static VCHIQ_STATUS_T
+service_callback(VCHIQ_REASON_T reason, VCHIQ_HEADER_T *header,
+	VCHIQ_SERVICE_HANDLE_T handle, void *bulk_userdata)
+{
+	/* How do we ensure the callback goes to the right client?
+	** The service_user data points to a USER_SERVICE_T record containing
+	** the original callback and the user state structure, which contains a
+	** circular buffer for completion records.
+	*/
+	USER_SERVICE_T *user_service;
+	VCHIQ_SERVICE_T *service;
+	VCHIQ_INSTANCE_T instance;
+	DEBUG_INITIALISE(g_state.local)
+
+	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+
+	service = handle_to_service(handle);
+	BUG_ON(!service);
+	user_service = (USER_SERVICE_T *)service->base.userdata;
+	instance = user_service->instance;
+
+	if (!instance || instance->closing)
+		return VCHIQ_SUCCESS;
+
+	vchiq_log_trace(vchiq_arm_log_level,
+		"service_callback - service %lx(%d,%p), reason %d, header %lx, "
+		"instance %lx, bulk_userdata %lx",
+		(unsigned long)user_service,
+		service->localport, user_service->userdata,
+		reason, (unsigned long)header,
+		(unsigned long)instance, (unsigned long)bulk_userdata);
+
+	if (header && user_service->is_vchi) {
+		spin_lock(&msg_queue_spinlock);
+		while (user_service->msg_insert ==
+			(user_service->msg_remove + MSG_QUEUE_SIZE)) {
+			spin_unlock(&msg_queue_spinlock);
+			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+			DEBUG_COUNT(MSG_QUEUE_FULL_COUNT);
+			vchiq_log_trace(vchiq_arm_log_level,
+				"service_callback - msg queue full");
+			/* If there is no MESSAGE_AVAILABLE in the completion
+			** queue, add one
+			*/
+			if ((user_service->message_available_pos -
+				instance->completion_remove) < 0) {
+				VCHIQ_STATUS_T status;
+				vchiq_log_info(vchiq_arm_log_level,
+					"Inserting extra MESSAGE_AVAILABLE");
+				DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+				status = add_completion(instance, reason,
+					NULL, user_service, bulk_userdata);
+				if (status != VCHIQ_SUCCESS) {
+					DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+					return status;
+				}
+			}
+
+			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+			if (down_interruptible(&user_service->remove_event)
+				!= 0) {
+				vchiq_log_info(vchiq_arm_log_level,
+					"service_callback interrupted");
+				DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+				return VCHIQ_RETRY;
+			} else if (instance->closing) {
+				vchiq_log_info(vchiq_arm_log_level,
+					"service_callback closing");
+				DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+				return VCHIQ_ERROR;
+			}
+			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+			spin_lock(&msg_queue_spinlock);
+		}
+
+		user_service->msg_queue[user_service->msg_insert &
+			(MSG_QUEUE_SIZE - 1)] = header;
+		user_service->msg_insert++;
+		spin_unlock(&msg_queue_spinlock);
+
+		up(&user_service->insert_event);
+
+		/* If there is a thread waiting in DEQUEUE_MESSAGE, or if
+		** there is a MESSAGE_AVAILABLE in the completion queue then
+		** bypass the completion queue.
+		*/
+		if (((user_service->message_available_pos -
+			instance->completion_remove) >= 0) ||
+			user_service->dequeue_pending) {
+			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+			user_service->dequeue_pending = 0;
+			return VCHIQ_SUCCESS;
+		}
+
+		header = NULL;
+	}
+	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
+
+	return add_completion(instance, reason, header, user_service,
+		bulk_userdata);
+}
+
+/****************************************************************************
+*
+*   user_service_free
+*
+***************************************************************************/
+static void
+user_service_free(void *userdata)
+{
+	kfree(userdata);
+}
+
+/****************************************************************************
+*
+*   close_delivered
+*
+***************************************************************************/
+static void close_delivered(USER_SERVICE_T *user_service)
+{
+	vchiq_log_info(vchiq_arm_log_level,
+		"close_delivered(handle=%x)",
+		user_service->service->handle);
+
+	if (user_service->close_pending) {
+		/* Allow the underlying service to be culled */
+		unlock_service(user_service->service);
+
+		/* Wake the user-thread blocked in close_ or remove_service */
+		up(&user_service->close_event);
+
+		user_service->close_pending = 0;
+	}
+}
+
+/****************************************************************************
+*
+*   vchiq_ioctl
+*
+***************************************************************************/
+static long
+vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	VCHIQ_INSTANCE_T instance = file->private_data;
+	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+	VCHIQ_SERVICE_T *service = NULL;
+	long ret = 0;
+	int i, rc;
+	DEBUG_INITIALISE(g_state.local)
+
+	vchiq_log_trace(vchiq_arm_log_level,
+		 "vchiq_ioctl - instance %x, cmd %s, arg %lx",
+		(unsigned int)instance,
+		((_IOC_TYPE(cmd) == VCHIQ_IOC_MAGIC) &&
+		(_IOC_NR(cmd) <= VCHIQ_IOC_MAX)) ?
+		ioctl_names[_IOC_NR(cmd)] : "<invalid>", arg);
+
+	switch (cmd) {
+	case VCHIQ_IOC_SHUTDOWN:
+		if (!instance->connected)
+			break;
+
+		/* Remove all services */
+		i = 0;
+		while ((service = next_service_by_instance(instance->state,
+			instance, &i)) != NULL) {
+			status = vchiq_remove_service(service->handle);
+			unlock_service(service);
+			if (status != VCHIQ_SUCCESS)
+				break;
+		}
+		service = NULL;
+
+		if (status == VCHIQ_SUCCESS) {
+			/* Wake the completion thread and ask it to exit */
+			instance->closing = 1;
+			up(&instance->insert_event);
+		}
+
+		break;
+
+	case VCHIQ_IOC_CONNECT:
+		if (instance->connected) {
+			ret = -EINVAL;
+			break;
+		}
+		rc = mutex_lock_interruptible(&instance->state->mutex);
+		if (rc != 0) {
+			vchiq_log_error(vchiq_arm_log_level,
+				"vchiq: connect: could not lock mutex for "
+				"state %d: %d",
+				instance->state->id, rc);
+			ret = -EINTR;
+			break;
+		}
+		status = vchiq_connect_internal(instance->state, instance);
+		mutex_unlock(&instance->state->mutex);
+
+		if (status == VCHIQ_SUCCESS)
+			instance->connected = 1;
+		else
+			vchiq_log_error(vchiq_arm_log_level,
+				"vchiq: could not connect: %d", status);
+		break;
+
+	case VCHIQ_IOC_CREATE_SERVICE: {
+		VCHIQ_CREATE_SERVICE_T args;
+		USER_SERVICE_T *user_service = NULL;
+		void *userdata;
+		int srvstate;
+
+		if (copy_from_user
+			 (&args, (const void __user *)arg,
+			  sizeof(args)) != 0) {
+			ret = -EFAULT;
+			break;
+		}
+
+		user_service = kmalloc(sizeof(USER_SERVICE_T), GFP_KERNEL);
+		if (!user_service) {
+			ret = -ENOMEM;
+			break;
+		}
+
+		if (args.is_open) {
+			if (!instance->connected) {
+				ret = -ENOTCONN;
+				kfree(user_service);
+				break;
+			}
+			srvstate = VCHIQ_SRVSTATE_OPENING;
+		} else {
+			srvstate =
+				 instance->connected ?
+				 VCHIQ_SRVSTATE_LISTENING :
+				 VCHIQ_SRVSTATE_HIDDEN;
+		}
+
+		userdata = args.params.userdata;
+		args.params.callback = service_callback;
+		args.params.userdata = user_service;
+		service = vchiq_add_service_internal(
+				instance->state,
+				&args.params, srvstate,
+				instance, user_service_free);
+
+		if (service != NULL) {
+			user_service->service = service;
+			user_service->userdata = userdata;
+			user_service->instance = instance;
+			user_service->is_vchi = (args.is_vchi != 0);
+			user_service->dequeue_pending = 0;
+			user_service->close_pending = 0;
+			user_service->message_available_pos =
+				instance->completion_remove - 1;
+			user_service->msg_insert = 0;
+			user_service->msg_remove = 0;
+			sema_init(&user_service->insert_event, 0);
+			sema_init(&user_service->remove_event, 0);
+			sema_init(&user_service->close_event, 0);
+
+			if (args.is_open) {
+				status = vchiq_open_service_internal
+					(service, instance->pid);
+				if (status != VCHIQ_SUCCESS) {
+					vchiq_remove_service(service->handle);
+					service = NULL;
+					ret = (status == VCHIQ_RETRY) ?
+						-EINTR : -EIO;
+					break;
+				}
+			}
+
+			if (copy_to_user((void __user *)
+				&(((VCHIQ_CREATE_SERVICE_T __user *)
+					arg)->handle),
+				(const void *)&service->handle,
+				sizeof(service->handle)) != 0) {
+				ret = -EFAULT;
+				vchiq_remove_service(service->handle);
+			}
+
+			service = NULL;
+		} else {
+			ret = -EEXIST;
+			kfree(user_service);
+		}
+	} break;
+
+	case VCHIQ_IOC_CLOSE_SERVICE: {
+		VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
+
+		service = find_service_for_instance(instance, handle);
+		if (service != NULL) {
+			USER_SERVICE_T *user_service =
+				(USER_SERVICE_T *)service->base.userdata;
+			/* close_pending is false on first entry, and when the
+                           wait in vchiq_close_service has been interrupted. */
+			if (!user_service->close_pending) {
+				status = vchiq_close_service(service->handle);
+				if (status != VCHIQ_SUCCESS)
+					break;
+			}
+
+			/* close_pending is true once the underlying service
+			   has been closed until the client library calls the
+			   CLOSE_DELIVERED ioctl, signalling close_event. */
+			if (user_service->close_pending &&
+				down_interruptible(&user_service->close_event))
+				status = VCHIQ_RETRY;
+		}
+		else
+			ret = -EINVAL;
+	} break;
+
+	case VCHIQ_IOC_REMOVE_SERVICE: {
+		VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
+
+		service = find_service_for_instance(instance, handle);
+		if (service != NULL) {
+			USER_SERVICE_T *user_service =
+				(USER_SERVICE_T *)service->base.userdata;
+			/* close_pending is false on first entry, and when the
+                           wait in vchiq_close_service has been interrupted. */
+			if (!user_service->close_pending) {
+				status = vchiq_remove_service(service->handle);
+				if (status != VCHIQ_SUCCESS)
+					break;
+			}
+
+			/* close_pending is true once the underlying service
+			   has been closed until the client library calls the
+			   CLOSE_DELIVERED ioctl, signalling close_event. */
+			if (user_service->close_pending &&
+				down_interruptible(&user_service->close_event))
+				status = VCHIQ_RETRY;
+		}
+		else
+			ret = -EINVAL;
+	} break;
+
+	case VCHIQ_IOC_USE_SERVICE:
+	case VCHIQ_IOC_RELEASE_SERVICE:	{
+		VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
+
+		service = find_service_for_instance(instance, handle);
+		if (service != NULL) {
+			status = (cmd == VCHIQ_IOC_USE_SERVICE)	?
+				vchiq_use_service_internal(service) :
+				vchiq_release_service_internal(service);
+			if (status != VCHIQ_SUCCESS) {
+				vchiq_log_error(vchiq_susp_log_level,
+					"%s: cmd %s returned error %d for "
+					"service %c%c%c%c:%03d",
+					__func__,
+					(cmd == VCHIQ_IOC_USE_SERVICE) ?
+						"VCHIQ_IOC_USE_SERVICE" :
+						"VCHIQ_IOC_RELEASE_SERVICE",
+					status,
+					VCHIQ_FOURCC_AS_4CHARS(
+						service->base.fourcc),
+					service->client_id);
+				ret = -EINVAL;
+			}
+		} else
+			ret = -EINVAL;
+	} break;
+
+	case VCHIQ_IOC_QUEUE_MESSAGE: {
+		VCHIQ_QUEUE_MESSAGE_T args;
+		if (copy_from_user
+			 (&args, (const void __user *)arg,
+			  sizeof(args)) != 0) {
+			ret = -EFAULT;
+			break;
+		}
+
+		service = find_service_for_instance(instance, args.handle);
+
+		if ((service != NULL) && (args.count <= MAX_ELEMENTS)) {
+			/* Copy elements into kernel space */
+			VCHIQ_ELEMENT_T elements[MAX_ELEMENTS];
+			if (copy_from_user(elements, args.elements,
+				args.count * sizeof(VCHIQ_ELEMENT_T)) == 0)
+				status = vchiq_queue_message
+					(args.handle,
+					elements, args.count);
+			else
+				ret = -EFAULT;
+		} else {
+			ret = -EINVAL;
+		}
+	} break;
+
+	case VCHIQ_IOC_QUEUE_BULK_TRANSMIT:
+	case VCHIQ_IOC_QUEUE_BULK_RECEIVE: {
+		VCHIQ_QUEUE_BULK_TRANSFER_T args;
+		struct bulk_waiter_node *waiter = NULL;
+		VCHIQ_BULK_DIR_T dir =
+			(cmd == VCHIQ_IOC_QUEUE_BULK_TRANSMIT) ?
+			VCHIQ_BULK_TRANSMIT : VCHIQ_BULK_RECEIVE;
+
+		if (copy_from_user
+			(&args, (const void __user *)arg,
+			sizeof(args)) != 0) {
+			ret = -EFAULT;
+			break;
+		}
+
+		service = find_service_for_instance(instance, args.handle);
+		if (!service) {
+			ret = -EINVAL;
+			break;
+		}
+
+		if (args.mode == VCHIQ_BULK_MODE_BLOCKING) {
+			waiter = kzalloc(sizeof(struct bulk_waiter_node),
+				GFP_KERNEL);
+			if (!waiter) {
+				ret = -ENOMEM;
+				break;
+			}
+			args.userdata = &waiter->bulk_waiter;
+		} else if (args.mode == VCHIQ_BULK_MODE_WAITING) {
+			struct list_head *pos;
+			mutex_lock(&instance->bulk_waiter_list_mutex);
+			list_for_each(pos, &instance->bulk_waiter_list) {
+				if (list_entry(pos, struct bulk_waiter_node,
+					list)->pid == current->pid) {
+					waiter = list_entry(pos,
+						struct bulk_waiter_node,
+						list);
+					list_del(pos);
+					break;
+				}
+
+			}
+			mutex_unlock(&instance->bulk_waiter_list_mutex);
+			if (!waiter) {
+				vchiq_log_error(vchiq_arm_log_level,
+					"no bulk_waiter found for pid %d",
+					current->pid);
+				ret = -ESRCH;
+				break;
+			}
+			vchiq_log_info(vchiq_arm_log_level,
+				"found bulk_waiter %x for pid %d",
+				(unsigned int)waiter, current->pid);
+			args.userdata = &waiter->bulk_waiter;
+		}
+		status = vchiq_bulk_transfer
+			(args.handle,
+			 VCHI_MEM_HANDLE_INVALID,
+			 args.data, args.size,
+			 args.userdata, args.mode,
+			 dir);
+		if (!waiter)
+			break;
+		if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) ||
+			!waiter->bulk_waiter.bulk) {
+			if (waiter->bulk_waiter.bulk) {
+				/* Cancel the signal when the transfer
+				** completes. */
+				spin_lock(&bulk_waiter_spinlock);
+				waiter->bulk_waiter.bulk->userdata = NULL;
+				spin_unlock(&bulk_waiter_spinlock);
+			}
+			kfree(waiter);
+		} else {
+			const VCHIQ_BULK_MODE_T mode_waiting =
+				VCHIQ_BULK_MODE_WAITING;
+			waiter->pid = current->pid;
+			mutex_lock(&instance->bulk_waiter_list_mutex);
+			list_add(&waiter->list, &instance->bulk_waiter_list);
+			mutex_unlock(&instance->bulk_waiter_list_mutex);
+			vchiq_log_info(vchiq_arm_log_level,
+				"saved bulk_waiter %x for pid %d",
+				(unsigned int)waiter, current->pid);
+
+			if (copy_to_user((void __user *)
+				&(((VCHIQ_QUEUE_BULK_TRANSFER_T __user *)
+					arg)->mode),
+				(const void *)&mode_waiting,
+				sizeof(mode_waiting)) != 0)
+				ret = -EFAULT;
+		}
+	} break;
+
+	case VCHIQ_IOC_AWAIT_COMPLETION: {
+		VCHIQ_AWAIT_COMPLETION_T args;
+
+		DEBUG_TRACE(AWAIT_COMPLETION_LINE);
+		if (!instance->connected) {
+			ret = -ENOTCONN;
+			break;
+		}
+
+		if (copy_from_user(&args, (const void __user *)arg,
+			sizeof(args)) != 0) {
+			ret = -EFAULT;
+			break;
+		}
+
+		mutex_lock(&instance->completion_mutex);
+
+		DEBUG_TRACE(AWAIT_COMPLETION_LINE);
+		while ((instance->completion_remove ==
+			instance->completion_insert)
+			&& !instance->closing) {
+			int rc;
+			DEBUG_TRACE(AWAIT_COMPLETION_LINE);
+			mutex_unlock(&instance->completion_mutex);
+			rc = down_interruptible(&instance->insert_event);
+			mutex_lock(&instance->completion_mutex);
+			if (rc != 0) {
+				DEBUG_TRACE(AWAIT_COMPLETION_LINE);
+				vchiq_log_info(vchiq_arm_log_level,
+					"AWAIT_COMPLETION interrupted");
+				ret = -EINTR;
+				break;
+			}
+		}
+		DEBUG_TRACE(AWAIT_COMPLETION_LINE);
+
+		/* A read memory barrier is needed to stop prefetch of a stale
+		** completion record
+		*/
+		rmb();
+
+		if (ret == 0) {
+			int msgbufcount = args.msgbufcount;
+			for (ret = 0; ret < args.count; ret++) {
+				VCHIQ_COMPLETION_DATA_T *completion;
+				VCHIQ_SERVICE_T *service;
+				USER_SERVICE_T *user_service;
+				VCHIQ_HEADER_T *header;
+				if (instance->completion_remove ==
+					instance->completion_insert)
+					break;
+				completion = &instance->completions[
+					instance->completion_remove &
+					(MAX_COMPLETIONS - 1)];
+
+				service = completion->service_userdata;
+				user_service = service->base.userdata;
+				completion->service_userdata =
+					user_service->userdata;
+
+				header = completion->header;
+				if (header) {
+					void __user *msgbuf;
+					int msglen;
+
+					msglen = header->size +
+						sizeof(VCHIQ_HEADER_T);
+					/* This must be a VCHIQ-style service */
+					if (args.msgbufsize < msglen) {
+						vchiq_log_error(
+							vchiq_arm_log_level,
+							"header %x: msgbufsize"
+							" %x < msglen %x",
+							(unsigned int)header,
+							args.msgbufsize,
+							msglen);
+						WARN(1, "invalid message "
+							"size\n");
+						if (ret == 0)
+							ret = -EMSGSIZE;
+						break;
+					}
+					if (msgbufcount <= 0)
+						/* Stall here for lack of a
+						** buffer for the message. */
+						break;
+					/* Get the pointer from user space */
+					msgbufcount--;
+					if (copy_from_user(&msgbuf,
+						(const void __user *)
+						&args.msgbufs[msgbufcount],
+						sizeof(msgbuf)) != 0) {
+						if (ret == 0)
+							ret = -EFAULT;
+						break;
+					}
+
+					/* Copy the message to user space */
+					if (copy_to_user(msgbuf, header,
+						msglen) != 0) {
+						if (ret == 0)
+							ret = -EFAULT;
+						break;
+					}
+
+					/* Now it has been copied, the message
+					** can be released. */
+					vchiq_release_message(service->handle,
+						header);
+
+					/* The completion must point to the
+					** msgbuf. */
+					completion->header = msgbuf;
+				}
+
+				if ((completion->reason ==
+					VCHIQ_SERVICE_CLOSED) &&
+					!instance->use_close_delivered)
+					unlock_service(service);
+
+				if (copy_to_user((void __user *)(
+					(size_t)args.buf +
+					ret * sizeof(VCHIQ_COMPLETION_DATA_T)),
+					completion,
+					sizeof(VCHIQ_COMPLETION_DATA_T)) != 0) {
+						if (ret == 0)
+							ret = -EFAULT;
+					break;
+				}
+
+				instance->completion_remove++;
+			}
+
+			if (msgbufcount != args.msgbufcount) {
+				if (copy_to_user((void __user *)
+					&((VCHIQ_AWAIT_COMPLETION_T *)arg)->
+						msgbufcount,
+					&msgbufcount,
+					sizeof(msgbufcount)) != 0) {
+					ret = -EFAULT;
+				}
+			}
+		}
+
+		if (ret != 0)
+			up(&instance->remove_event);
+		mutex_unlock(&instance->completion_mutex);
+		DEBUG_TRACE(AWAIT_COMPLETION_LINE);
+	} break;
+
+	case VCHIQ_IOC_DEQUEUE_MESSAGE: {
+		VCHIQ_DEQUEUE_MESSAGE_T args;
+		USER_SERVICE_T *user_service;
+		VCHIQ_HEADER_T *header;
+
+		DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
+		if (copy_from_user
+			 (&args, (const void __user *)arg,
+			  sizeof(args)) != 0) {
+			ret = -EFAULT;
+			break;
+		}
+		service = find_service_for_instance(instance, args.handle);
+		if (!service) {
+			ret = -EINVAL;
+			break;
+		}
+		user_service = (USER_SERVICE_T *)service->base.userdata;
+		if (user_service->is_vchi == 0) {
+			ret = -EINVAL;
+			break;
+		}
+
+		spin_lock(&msg_queue_spinlock);
+		if (user_service->msg_remove == user_service->msg_insert) {
+			if (!args.blocking) {
+				spin_unlock(&msg_queue_spinlock);
+				DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
+				ret = -EWOULDBLOCK;
+				break;
+			}
+			user_service->dequeue_pending = 1;
+			do {
+				spin_unlock(&msg_queue_spinlock);
+				DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
+				if (down_interruptible(
+					&user_service->insert_event) != 0) {
+					vchiq_log_info(vchiq_arm_log_level,
+						"DEQUEUE_MESSAGE interrupted");
+					ret = -EINTR;
+					break;
+				}
+				spin_lock(&msg_queue_spinlock);
+			} while (user_service->msg_remove ==
+				user_service->msg_insert);
+
+			if (ret)
+				break;
+		}
+
+		BUG_ON((int)(user_service->msg_insert -
+			user_service->msg_remove) < 0);
+
+		header = user_service->msg_queue[user_service->msg_remove &
+			(MSG_QUEUE_SIZE - 1)];
+		user_service->msg_remove++;
+		spin_unlock(&msg_queue_spinlock);
+
+		up(&user_service->remove_event);
+		if (header == NULL)
+			ret = -ENOTCONN;
+		else if (header->size <= args.bufsize) {
+			/* Copy to user space if msgbuf is not NULL */
+			if ((args.buf == NULL) ||
+				(copy_to_user((void __user *)args.buf,
+				header->data,
+				header->size) == 0)) {
+				ret = header->size;
+				vchiq_release_message(
+					service->handle,
+					header);
+			} else
+				ret = -EFAULT;
+		} else {
+			vchiq_log_error(vchiq_arm_log_level,
+				"header %x: bufsize %x < size %x",
+				(unsigned int)header, args.bufsize,
+				header->size);
+			WARN(1, "invalid size\n");
+			ret = -EMSGSIZE;
+		}
+		DEBUG_TRACE(DEQUEUE_MESSAGE_LINE);
+	} break;
+
+	case VCHIQ_IOC_GET_CLIENT_ID: {
+		VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
+
+		ret = vchiq_get_client_id(handle);
+	} break;
+
+	case VCHIQ_IOC_GET_CONFIG: {
+		VCHIQ_GET_CONFIG_T args;
+		VCHIQ_CONFIG_T config;
+
+		if (copy_from_user(&args, (const void __user *)arg,
+			sizeof(args)) != 0) {
+			ret = -EFAULT;
+			break;
+		}
+		if (args.config_size > sizeof(config)) {
+			ret = -EINVAL;
+			break;
+		}
+		status = vchiq_get_config(instance, args.config_size, &config);
+		if (status == VCHIQ_SUCCESS) {
+			if (copy_to_user((void __user *)args.pconfig,
+				    &config, args.config_size) != 0) {
+				ret = -EFAULT;
+				break;
+			}
+		}
+	} break;
+
+	case VCHIQ_IOC_SET_SERVICE_OPTION: {
+		VCHIQ_SET_SERVICE_OPTION_T args;
+
+		if (copy_from_user(
+			&args, (const void __user *)arg,
+			sizeof(args)) != 0) {
+			ret = -EFAULT;
+			break;
+		}
+
+		service = find_service_for_instance(instance, args.handle);
+		if (!service) {
+			ret = -EINVAL;
+			break;
+		}
+
+		status = vchiq_set_service_option(
+				args.handle, args.option, args.value);
+	} break;
+
+	case VCHIQ_IOC_DUMP_PHYS_MEM: {
+		VCHIQ_DUMP_MEM_T  args;
+
+		if (copy_from_user
+			 (&args, (const void __user *)arg,
+			  sizeof(args)) != 0) {
+			ret = -EFAULT;
+			break;
+		}
+		dump_phys_mem(args.virt_addr, args.num_bytes);
+	} break;
+
+	case VCHIQ_IOC_LIB_VERSION: {
+		unsigned int lib_version = (unsigned int)arg;
+
+		if (lib_version < VCHIQ_VERSION_MIN)
+			ret = -EINVAL;
+		else if (lib_version >= VCHIQ_VERSION_CLOSE_DELIVERED)
+			instance->use_close_delivered = 1;
+	} break;
+
+	case VCHIQ_IOC_CLOSE_DELIVERED: {
+		VCHIQ_SERVICE_HANDLE_T handle = (VCHIQ_SERVICE_HANDLE_T)arg;
+
+		service = find_closed_service_for_instance(instance, handle);
+		if (service != NULL) {
+			USER_SERVICE_T *user_service =
+				(USER_SERVICE_T *)service->base.userdata;
+			close_delivered(user_service);
+		}
+		else
+			ret = -EINVAL;
+	} break;
+
+	default:
+		ret = -ENOTTY;
+		break;
+	}
+
+	if (service)
+		unlock_service(service);
+
+	if (ret == 0) {
+		if (status == VCHIQ_ERROR)
+			ret = -EIO;
+		else if (status == VCHIQ_RETRY)
+			ret = -EINTR;
+	}
+
+	if ((status == VCHIQ_SUCCESS) && (ret < 0) && (ret != -EINTR) &&
+		(ret != -EWOULDBLOCK))
+		vchiq_log_info(vchiq_arm_log_level,
+			"  ioctl instance %lx, cmd %s -> status %d, %ld",
+			(unsigned long)instance,
+			(_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ?
+				ioctl_names[_IOC_NR(cmd)] :
+				"<invalid>",
+			status, ret);
+	else
+		vchiq_log_trace(vchiq_arm_log_level,
+			"  ioctl instance %lx, cmd %s -> status %d, %ld",
+			(unsigned long)instance,
+			(_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ?
+				ioctl_names[_IOC_NR(cmd)] :
+				"<invalid>",
+			status, ret);
+
+	return ret;
+}
+
+/****************************************************************************
+*
+*   vchiq_open
+*
+***************************************************************************/
+
+static int
+vchiq_open(struct inode *inode, struct file *file)
+{
+	int dev = iminor(inode) & 0x0f;
+	vchiq_log_info(vchiq_arm_log_level, "vchiq_open");
+	switch (dev) {
+	case VCHIQ_MINOR: {
+		int ret;
+		VCHIQ_STATE_T *state = vchiq_get_state();
+		VCHIQ_INSTANCE_T instance;
+
+		if (!state) {
+			vchiq_log_error(vchiq_arm_log_level,
+				"vchiq has no connection to VideoCore");
+			return -ENOTCONN;
+		}
+
+		instance = kzalloc(sizeof(*instance), GFP_KERNEL);
+		if (!instance)
+			return -ENOMEM;
+
+		instance->state = state;
+		instance->pid = current->tgid;
+
+		ret = vchiq_debugfs_add_instance(instance);
+		if (ret != 0) {
+			kfree(instance);
+			return ret;
+		}
+
+		sema_init(&instance->insert_event, 0);
+		sema_init(&instance->remove_event, 0);
+		mutex_init(&instance->completion_mutex);
+		mutex_init(&instance->bulk_waiter_list_mutex);
+		INIT_LIST_HEAD(&instance->bulk_waiter_list);
+
+		file->private_data = instance;
+	} break;
+
+	default:
+		vchiq_log_error(vchiq_arm_log_level,
+			"Unknown minor device: %d", dev);
+		return -ENXIO;
+	}
+
+	return 0;
+}
+
+/****************************************************************************
+*
+*   vchiq_release
+*
+***************************************************************************/
+
+static int
+vchiq_release(struct inode *inode, struct file *file)
+{
+	int dev = iminor(inode) & 0x0f;
+	int ret = 0;
+	switch (dev) {
+	case VCHIQ_MINOR: {
+		VCHIQ_INSTANCE_T instance = file->private_data;
+		VCHIQ_STATE_T *state = vchiq_get_state();
+		VCHIQ_SERVICE_T *service;
+		int i;
+
+		vchiq_log_info(vchiq_arm_log_level,
+			"vchiq_release: instance=%lx",
+			(unsigned long)instance);
+
+		if (!state) {
+			ret = -EPERM;
+			goto out;
+		}
+
+		/* Ensure videocore is awake to allow termination. */
+		vchiq_use_internal(instance->state, NULL,
+				USE_TYPE_VCHIQ);
+
+		mutex_lock(&instance->completion_mutex);
+
+		/* Wake the completion thread and ask it to exit */
+		instance->closing = 1;
+		up(&instance->insert_event);
+
+		mutex_unlock(&instance->completion_mutex);
+
+		/* Wake the slot handler if the completion queue is full. */
+		up(&instance->remove_event);
+
+		/* Mark all services for termination... */
+		i = 0;
+		while ((service = next_service_by_instance(state, instance,
+			&i)) !=	NULL) {
+			USER_SERVICE_T *user_service = service->base.userdata;
+
+			/* Wake the slot handler if the msg queue is full. */
+			up(&user_service->remove_event);
+
+			vchiq_terminate_service_internal(service);
+			unlock_service(service);
+		}
+
+		/* ...and wait for them to die */
+		i = 0;
+		while ((service = next_service_by_instance(state, instance, &i))
+			!= NULL) {
+			USER_SERVICE_T *user_service = service->base.userdata;
+
+			down(&service->remove_event);
+
+			BUG_ON(service->srvstate != VCHIQ_SRVSTATE_FREE);
+
+			spin_lock(&msg_queue_spinlock);
+
+			while (user_service->msg_remove !=
+				user_service->msg_insert) {
+				VCHIQ_HEADER_T *header = user_service->
+					msg_queue[user_service->msg_remove &
+						(MSG_QUEUE_SIZE - 1)];
+				user_service->msg_remove++;
+				spin_unlock(&msg_queue_spinlock);
+
+				if (header)
+					vchiq_release_message(
+						service->handle,
+						header);
+				spin_lock(&msg_queue_spinlock);
+			}
+
+			spin_unlock(&msg_queue_spinlock);
+
+			unlock_service(service);
+		}
+
+		/* Release any closed services */
+		while (instance->completion_remove !=
+			instance->completion_insert) {
+			VCHIQ_COMPLETION_DATA_T *completion;
+			VCHIQ_SERVICE_T *service;
+			completion = &instance->completions[
+				instance->completion_remove &
+				(MAX_COMPLETIONS - 1)];
+			service = completion->service_userdata;
+			if (completion->reason == VCHIQ_SERVICE_CLOSED)
+			{
+				USER_SERVICE_T *user_service =
+					service->base.userdata;
+
+				/* Wake any blocked user-thread */
+				if (instance->use_close_delivered)
+					up(&user_service->close_event);
+				unlock_service(service);
+			}
+			instance->completion_remove++;
+		}
+
+		/* Release the PEER service count. */
+		vchiq_release_internal(instance->state, NULL);
+
+		{
+			struct list_head *pos, *next;
+			list_for_each_safe(pos, next,
+				&instance->bulk_waiter_list) {
+				struct bulk_waiter_node *waiter;
+				waiter = list_entry(pos,
+					struct bulk_waiter_node,
+					list);
+				list_del(pos);
+				vchiq_log_info(vchiq_arm_log_level,
+					"bulk_waiter - cleaned up %x "
+					"for pid %d",
+					(unsigned int)waiter, waiter->pid);
+				kfree(waiter);
+			}
+		}
+
+		vchiq_debugfs_remove_instance(instance);
+
+		kfree(instance);
+		file->private_data = NULL;
+	} break;
+
+	default:
+		vchiq_log_error(vchiq_arm_log_level,
+			"Unknown minor device: %d", dev);
+		ret = -ENXIO;
+	}
+
+out:
+	return ret;
+}
+
+/****************************************************************************
+*
+*   vchiq_dump
+*
+***************************************************************************/
+
+void
+vchiq_dump(void *dump_context, const char *str, int len)
+{
+	DUMP_CONTEXT_T *context = (DUMP_CONTEXT_T *)dump_context;
+
+	if (context->actual < context->space) {
+		int copy_bytes;
+		if (context->offset > 0) {
+			int skip_bytes = min(len, (int)context->offset);
+			str += skip_bytes;
+			len -= skip_bytes;
+			context->offset -= skip_bytes;
+			if (context->offset > 0)
+				return;
+		}
+		copy_bytes = min(len, (int)(context->space - context->actual));
+		if (copy_bytes == 0)
+			return;
+		if (copy_to_user(context->buf + context->actual, str,
+			copy_bytes))
+			context->actual = -EFAULT;
+		context->actual += copy_bytes;
+		len -= copy_bytes;
+
+		/* If tne terminating NUL is included in the length, then it
+		** marks the end of a line and should be replaced with a
+		** carriage return. */
+		if ((len == 0) && (str[copy_bytes - 1] == '\0')) {
+			char cr = '\n';
+			if (copy_to_user(context->buf + context->actual - 1,
+				&cr, 1))
+				context->actual = -EFAULT;
+		}
+	}
+}
+
+/****************************************************************************
+*
+*   vchiq_dump_platform_instance_state
+*
+***************************************************************************/
+
+void
+vchiq_dump_platform_instances(void *dump_context)
+{
+	VCHIQ_STATE_T *state = vchiq_get_state();
+	char buf[80];
+	int len;
+	int i;
+
+	/* There is no list of instances, so instead scan all services,
+		marking those that have been dumped. */
+
+	for (i = 0; i < state->unused_service; i++) {
+		VCHIQ_SERVICE_T *service = state->services[i];
+		VCHIQ_INSTANCE_T instance;
+
+		if (service && (service->base.callback == service_callback)) {
+			instance = service->instance;
+			if (instance)
+				instance->mark = 0;
+		}
+	}
+
+	for (i = 0; i < state->unused_service; i++) {
+		VCHIQ_SERVICE_T *service = state->services[i];
+		VCHIQ_INSTANCE_T instance;
+
+		if (service && (service->base.callback == service_callback)) {
+			instance = service->instance;
+			if (instance && !instance->mark) {
+				len = snprintf(buf, sizeof(buf),
+					"Instance %x: pid %d,%s completions "
+						"%d/%d",
+					(unsigned int)instance, instance->pid,
+					instance->connected ? " connected, " :
+						"",
+					instance->completion_insert -
+						instance->completion_remove,
+					MAX_COMPLETIONS);
+
+				vchiq_dump(dump_context, buf, len + 1);
+
+				instance->mark = 1;
+			}
+		}
+	}
+}
+
+/****************************************************************************
+*
+*   vchiq_dump_platform_service_state
+*
+***************************************************************************/
+
+void
+vchiq_dump_platform_service_state(void *dump_context, VCHIQ_SERVICE_T *service)
+{
+	USER_SERVICE_T *user_service = (USER_SERVICE_T *)service->base.userdata;
+	char buf[80];
+	int len;
+
+	len = snprintf(buf, sizeof(buf), "  instance %x",
+		(unsigned int)service->instance);
+
+	if ((service->base.callback == service_callback) &&
+		user_service->is_vchi) {
+		len += snprintf(buf + len, sizeof(buf) - len,
+			", %d/%d messages",
+			user_service->msg_insert - user_service->msg_remove,
+			MSG_QUEUE_SIZE);
+
+		if (user_service->dequeue_pending)
+			len += snprintf(buf + len, sizeof(buf) - len,
+				" (dequeue pending)");
+	}
+
+	vchiq_dump(dump_context, buf, len + 1);
+}
+
+/****************************************************************************
+*
+*   dump_user_mem
+*
+***************************************************************************/
+
+static void
+dump_phys_mem(void *virt_addr, uint32_t num_bytes)
+{
+	int            rc;
+	uint8_t       *end_virt_addr = virt_addr + num_bytes;
+	int            num_pages;
+	int            offset;
+	int            end_offset;
+	int            page_idx;
+	int            prev_idx;
+	struct page   *page;
+	struct page  **pages;
+	uint8_t       *kmapped_virt_ptr;
+
+	/* Align virtAddr and endVirtAddr to 16 byte boundaries. */
+
+	virt_addr = (void *)((unsigned long)virt_addr & ~0x0fuL);
+	end_virt_addr = (void *)(((unsigned long)end_virt_addr + 15uL) &
+		~0x0fuL);
+
+	offset = (int)(long)virt_addr & (PAGE_SIZE - 1);
+	end_offset = (int)(long)end_virt_addr & (PAGE_SIZE - 1);
+
+	num_pages = (offset + num_bytes + PAGE_SIZE - 1) / PAGE_SIZE;
+
+	pages = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL);
+	if (pages == NULL) {
+		vchiq_log_error(vchiq_arm_log_level,
+			"Unable to allocation memory for %d pages\n",
+			num_pages);
+		return;
+	}
+
+	down_read(&current->mm->mmap_sem);
+	rc = get_user_pages(current,      /* task */
+		current->mm,              /* mm */
+		(unsigned long)virt_addr, /* start */
+		num_pages,                /* len */
+		0,                        /* write */
+		0,                        /* force */
+		pages,                    /* pages (array of page pointers) */
+		NULL);                    /* vmas */
+	up_read(&current->mm->mmap_sem);
+
+	prev_idx = -1;
+	page = NULL;
+
+	while (offset < end_offset) {
+
+		int page_offset = offset % PAGE_SIZE;
+		page_idx = offset / PAGE_SIZE;
+
+		if (page_idx != prev_idx) {
+
+			if (page != NULL)
+				kunmap(page);
+			page = pages[page_idx];
+			kmapped_virt_ptr = kmap(page);
+
+			prev_idx = page_idx;
+		}
+
+		if (vchiq_arm_log_level >= VCHIQ_LOG_TRACE)
+			vchiq_log_dump_mem("ph",
+				(uint32_t)(unsigned long)&kmapped_virt_ptr[
+					page_offset],
+				&kmapped_virt_ptr[page_offset], 16);
+
+		offset += 16;
+	}
+	if (page != NULL)
+		kunmap(page);
+
+	for (page_idx = 0; page_idx < num_pages; page_idx++)
+		page_cache_release(pages[page_idx]);
+
+	kfree(pages);
+}
+
+/****************************************************************************
+*
+*   vchiq_read
+*
+***************************************************************************/
+
+static ssize_t
+vchiq_read(struct file *file, char __user *buf,
+	size_t count, loff_t *ppos)
+{
+	DUMP_CONTEXT_T context;
+	context.buf = buf;
+	context.actual = 0;
+	context.space = count;
+	context.offset = *ppos;
+
+	vchiq_dump_state(&context, &g_state);
+
+	*ppos += context.actual;
+
+	return context.actual;
+}
+
+VCHIQ_STATE_T *
+vchiq_get_state(void)
+{
+
+	if (g_state.remote == NULL)
+		printk(KERN_ERR "%s: g_state.remote == NULL\n", __func__);
+	else if (g_state.remote->initialised != 1)
+		printk(KERN_NOTICE "%s: g_state.remote->initialised != 1 (%d)\n",
+			__func__, g_state.remote->initialised);
+
+	return ((g_state.remote != NULL) &&
+		(g_state.remote->initialised == 1)) ? &g_state : NULL;
+}
+
+static const struct file_operations
+vchiq_fops = {
+	.owner = THIS_MODULE,
+	.unlocked_ioctl = vchiq_ioctl,
+	.open = vchiq_open,
+	.release = vchiq_release,
+	.read = vchiq_read
+};
+
+/*
+ * Autosuspend related functionality
+ */
+
+int
+vchiq_videocore_wanted(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	if (!arm_state)
+		/* autosuspend not supported - always return wanted */
+		return 1;
+	else if (arm_state->blocked_count)
+		return 1;
+	else if (!arm_state->videocore_use_count)
+		/* usage count zero - check for override unless we're forcing */
+		if (arm_state->resume_blocked)
+			return 0;
+		else
+			return vchiq_platform_videocore_wanted(state);
+	else
+		/* non-zero usage count - videocore still required */
+		return 1;
+}
+
+static VCHIQ_STATUS_T
+vchiq_keepalive_vchiq_callback(VCHIQ_REASON_T reason,
+	VCHIQ_HEADER_T *header,
+	VCHIQ_SERVICE_HANDLE_T service_user,
+	void *bulk_user)
+{
+	vchiq_log_error(vchiq_susp_log_level,
+		"%s callback reason %d", __func__, reason);
+	return 0;
+}
+
+static int
+vchiq_keepalive_thread_func(void *v)
+{
+	VCHIQ_STATE_T *state = (VCHIQ_STATE_T *) v;
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+
+	VCHIQ_STATUS_T status;
+	VCHIQ_INSTANCE_T instance;
+	VCHIQ_SERVICE_HANDLE_T ka_handle;
+
+	VCHIQ_SERVICE_PARAMS_T params = {
+		.fourcc      = VCHIQ_MAKE_FOURCC('K', 'E', 'E', 'P'),
+		.callback    = vchiq_keepalive_vchiq_callback,
+		.version     = KEEPALIVE_VER,
+		.version_min = KEEPALIVE_VER_MIN
+	};
+
+	status = vchiq_initialise(&instance);
+	if (status != VCHIQ_SUCCESS) {
+		vchiq_log_error(vchiq_susp_log_level,
+			"%s vchiq_initialise failed %d", __func__, status);
+		goto exit;
+	}
+
+	status = vchiq_connect(instance);
+	if (status != VCHIQ_SUCCESS) {
+		vchiq_log_error(vchiq_susp_log_level,
+			"%s vchiq_connect failed %d", __func__, status);
+		goto shutdown;
+	}
+
+	status = vchiq_add_service(instance, &params, &ka_handle);
+	if (status != VCHIQ_SUCCESS) {
+		vchiq_log_error(vchiq_susp_log_level,
+			"%s vchiq_open_service failed %d", __func__, status);
+		goto shutdown;
+	}
+
+	while (1) {
+		long rc = 0, uc = 0;
+		if (wait_for_completion_interruptible(&arm_state->ka_evt)
+				!= 0) {
+			vchiq_log_error(vchiq_susp_log_level,
+				"%s interrupted", __func__);
+			flush_signals(current);
+			continue;
+		}
+
+		/* read and clear counters.  Do release_count then use_count to
+		 * prevent getting more releases than uses */
+		rc = atomic_xchg(&arm_state->ka_release_count, 0);
+		uc = atomic_xchg(&arm_state->ka_use_count, 0);
+
+		/* Call use/release service the requisite number of times.
+		 * Process use before release so use counts don't go negative */
+		while (uc--) {
+			atomic_inc(&arm_state->ka_use_ack_count);
+			status = vchiq_use_service(ka_handle);
+			if (status != VCHIQ_SUCCESS) {
+				vchiq_log_error(vchiq_susp_log_level,
+					"%s vchiq_use_service error %d",
+					__func__, status);
+			}
+		}
+		while (rc--) {
+			status = vchiq_release_service(ka_handle);
+			if (status != VCHIQ_SUCCESS) {
+				vchiq_log_error(vchiq_susp_log_level,
+					"%s vchiq_release_service error %d",
+					__func__, status);
+			}
+		}
+	}
+
+shutdown:
+	vchiq_shutdown(instance);
+exit:
+	return 0;
+}
+
+
+
+VCHIQ_STATUS_T
+vchiq_arm_init_state(VCHIQ_STATE_T *state, VCHIQ_ARM_STATE_T *arm_state)
+{
+	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+
+	if (arm_state) {
+		rwlock_init(&arm_state->susp_res_lock);
+
+		init_completion(&arm_state->ka_evt);
+		atomic_set(&arm_state->ka_use_count, 0);
+		atomic_set(&arm_state->ka_use_ack_count, 0);
+		atomic_set(&arm_state->ka_release_count, 0);
+
+		init_completion(&arm_state->vc_suspend_complete);
+
+		init_completion(&arm_state->vc_resume_complete);
+		/* Initialise to 'done' state.  We only want to block on resume
+		 * completion while videocore is suspended. */
+		set_resume_state(arm_state, VC_RESUME_RESUMED);
+
+		init_completion(&arm_state->resume_blocker);
+		/* Initialise to 'done' state.  We only want to block on this
+		 * completion while resume is blocked */
+		complete_all(&arm_state->resume_blocker);
+
+		init_completion(&arm_state->blocked_blocker);
+		/* Initialise to 'done' state.  We only want to block on this
+		 * completion while things are waiting on the resume blocker */
+		complete_all(&arm_state->blocked_blocker);
+
+		arm_state->suspend_timer_timeout = SUSPEND_TIMER_TIMEOUT_MS;
+		arm_state->suspend_timer_running = 0;
+		init_timer(&arm_state->suspend_timer);
+		arm_state->suspend_timer.data = (unsigned long)(state);
+		arm_state->suspend_timer.function = suspend_timer_callback;
+
+		arm_state->first_connect = 0;
+
+	}
+	return status;
+}
+
+/*
+** Functions to modify the state variables;
+**	set_suspend_state
+**	set_resume_state
+**
+** There are more state variables than we might like, so ensure they remain in
+** step.  Suspend and resume state are maintained separately, since most of
+** these state machines can operate independently.  However, there are a few
+** states where state transitions in one state machine cause a reset to the
+** other state machine.  In addition, there are some completion events which
+** need to occur on state machine reset and end-state(s), so these are also
+** dealt with in these functions.
+**
+** In all states we set the state variable according to the input, but in some
+** cases we perform additional steps outlined below;
+**
+** VC_SUSPEND_IDLE - Initialise the suspend completion at the same time.
+**			The suspend completion is completed after any suspend
+**			attempt.  When we reset the state machine we also reset
+**			the completion.  This reset occurs when videocore is
+**			resumed, and also if we initiate suspend after a suspend
+**			failure.
+**
+** VC_SUSPEND_IN_PROGRESS - This state is considered the point of no return for
+**			suspend - ie from this point on we must try to suspend
+**			before resuming can occur.  We therefore also reset the
+**			resume state machine to VC_RESUME_IDLE in this state.
+**
+** VC_SUSPEND_SUSPENDED - Suspend has completed successfully. Also call
+**			complete_all on the suspend completion to notify
+**			anything waiting for suspend to happen.
+**
+** VC_SUSPEND_REJECTED - Videocore rejected suspend. Videocore will also
+**			initiate resume, so no need to alter resume state.
+**			We call complete_all on the suspend completion to notify
+**			of suspend rejection.
+**
+** VC_SUSPEND_FAILED - We failed to initiate videocore suspend.  We notify the
+**			suspend completion and reset the resume state machine.
+**
+** VC_RESUME_IDLE - Initialise the resume completion at the same time.  The
+**			resume completion is in it's 'done' state whenever
+**			videcore is running.  Therfore, the VC_RESUME_IDLE state
+**			implies that videocore is suspended.
+**			Hence, any thread which needs to wait until videocore is
+**			running can wait on this completion - it will only block
+**			if videocore is suspended.
+**
+** VC_RESUME_RESUMED - Resume has completed successfully.  Videocore is running.
+**			Call complete_all on the resume completion to unblock
+**			any threads waiting for resume.	 Also reset the suspend
+**			state machine to it's idle state.
+**
+** VC_RESUME_FAILED - Currently unused - no mechanism to fail resume exists.
+*/
+
+void
+set_suspend_state(VCHIQ_ARM_STATE_T *arm_state,
+	enum vc_suspend_status new_state)
+{
+	/* set the state in all cases */
+	arm_state->vc_suspend_state = new_state;
+
+	/* state specific additional actions */
+	switch (new_state) {
+	case VC_SUSPEND_FORCE_CANCELED:
+		complete_all(&arm_state->vc_suspend_complete);
+		break;
+	case VC_SUSPEND_REJECTED:
+		complete_all(&arm_state->vc_suspend_complete);
+		break;
+	case VC_SUSPEND_FAILED:
+		complete_all(&arm_state->vc_suspend_complete);
+		arm_state->vc_resume_state = VC_RESUME_RESUMED;
+		complete_all(&arm_state->vc_resume_complete);
+		break;
+	case VC_SUSPEND_IDLE:
+		reinit_completion(&arm_state->vc_suspend_complete);
+		break;
+	case VC_SUSPEND_REQUESTED:
+		break;
+	case VC_SUSPEND_IN_PROGRESS:
+		set_resume_state(arm_state, VC_RESUME_IDLE);
+		break;
+	case VC_SUSPEND_SUSPENDED:
+		complete_all(&arm_state->vc_suspend_complete);
+		break;
+	default:
+		BUG();
+		break;
+	}
+}
+
+void
+set_resume_state(VCHIQ_ARM_STATE_T *arm_state,
+	enum vc_resume_status new_state)
+{
+	/* set the state in all cases */
+	arm_state->vc_resume_state = new_state;
+
+	/* state specific additional actions */
+	switch (new_state) {
+	case VC_RESUME_FAILED:
+		break;
+	case VC_RESUME_IDLE:
+		reinit_completion(&arm_state->vc_resume_complete);
+		break;
+	case VC_RESUME_REQUESTED:
+		break;
+	case VC_RESUME_IN_PROGRESS:
+		break;
+	case VC_RESUME_RESUMED:
+		complete_all(&arm_state->vc_resume_complete);
+		set_suspend_state(arm_state, VC_SUSPEND_IDLE);
+		break;
+	default:
+		BUG();
+		break;
+	}
+}
+
+
+/* should be called with the write lock held */
+inline void
+start_suspend_timer(VCHIQ_ARM_STATE_T *arm_state)
+{
+	del_timer(&arm_state->suspend_timer);
+	arm_state->suspend_timer.expires = jiffies +
+		msecs_to_jiffies(arm_state->
+			suspend_timer_timeout);
+	add_timer(&arm_state->suspend_timer);
+	arm_state->suspend_timer_running = 1;
+}
+
+/* should be called with the write lock held */
+static inline void
+stop_suspend_timer(VCHIQ_ARM_STATE_T *arm_state)
+{
+	if (arm_state->suspend_timer_running) {
+		del_timer(&arm_state->suspend_timer);
+		arm_state->suspend_timer_running = 0;
+	}
+}
+
+static inline int
+need_resume(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	return (arm_state->vc_suspend_state > VC_SUSPEND_IDLE) &&
+			(arm_state->vc_resume_state < VC_RESUME_REQUESTED) &&
+			vchiq_videocore_wanted(state);
+}
+
+static int
+block_resume(VCHIQ_ARM_STATE_T *arm_state)
+{
+	int status = VCHIQ_SUCCESS;
+	const unsigned long timeout_val =
+				msecs_to_jiffies(FORCE_SUSPEND_TIMEOUT_MS);
+	int resume_count = 0;
+
+	/* Allow any threads which were blocked by the last force suspend to
+	 * complete if they haven't already.  Only give this one shot; if
+	 * blocked_count is incremented after blocked_blocker is completed
+	 * (which only happens when blocked_count hits 0) then those threads
+	 * will have to wait until next time around */
+	if (arm_state->blocked_count) {
+		reinit_completion(&arm_state->blocked_blocker);
+		write_unlock_bh(&arm_state->susp_res_lock);
+		vchiq_log_info(vchiq_susp_log_level, "%s wait for previously "
+			"blocked clients", __func__);
+		if (wait_for_completion_interruptible_timeout(
+				&arm_state->blocked_blocker, timeout_val)
+					<= 0) {
+			vchiq_log_error(vchiq_susp_log_level, "%s wait for "
+				"previously blocked clients failed" , __func__);
+			status = VCHIQ_ERROR;
+			write_lock_bh(&arm_state->susp_res_lock);
+			goto out;
+		}
+		vchiq_log_info(vchiq_susp_log_level, "%s previously blocked "
+			"clients resumed", __func__);
+		write_lock_bh(&arm_state->susp_res_lock);
+	}
+
+	/* We need to wait for resume to complete if it's in process */
+	while (arm_state->vc_resume_state != VC_RESUME_RESUMED &&
+			arm_state->vc_resume_state > VC_RESUME_IDLE) {
+		if (resume_count > 1) {
+			status = VCHIQ_ERROR;
+			vchiq_log_error(vchiq_susp_log_level, "%s waited too "
+				"many times for resume" , __func__);
+			goto out;
+		}
+		write_unlock_bh(&arm_state->susp_res_lock);
+		vchiq_log_info(vchiq_susp_log_level, "%s wait for resume",
+			__func__);
+		if (wait_for_completion_interruptible_timeout(
+				&arm_state->vc_resume_complete, timeout_val)
+					<= 0) {
+			vchiq_log_error(vchiq_susp_log_level, "%s wait for "
+				"resume failed (%s)", __func__,
+				resume_state_names[arm_state->vc_resume_state +
+							VC_RESUME_NUM_OFFSET]);
+			status = VCHIQ_ERROR;
+			write_lock_bh(&arm_state->susp_res_lock);
+			goto out;
+		}
+		vchiq_log_info(vchiq_susp_log_level, "%s resumed", __func__);
+		write_lock_bh(&arm_state->susp_res_lock);
+		resume_count++;
+	}
+	reinit_completion(&arm_state->resume_blocker);
+	arm_state->resume_blocked = 1;
+
+out:
+	return status;
+}
+
+static inline void
+unblock_resume(VCHIQ_ARM_STATE_T *arm_state)
+{
+	complete_all(&arm_state->resume_blocker);
+	arm_state->resume_blocked = 0;
+}
+
+/* Initiate suspend via slot handler. Should be called with the write lock
+ * held */
+VCHIQ_STATUS_T
+vchiq_arm_vcsuspend(VCHIQ_STATE_T *state)
+{
+	VCHIQ_STATUS_T status = VCHIQ_ERROR;
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+
+	if (!arm_state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+	status = VCHIQ_SUCCESS;
+
+
+	switch (arm_state->vc_suspend_state) {
+	case VC_SUSPEND_REQUESTED:
+		vchiq_log_info(vchiq_susp_log_level, "%s: suspend already "
+			"requested", __func__);
+		break;
+	case VC_SUSPEND_IN_PROGRESS:
+		vchiq_log_info(vchiq_susp_log_level, "%s: suspend already in "
+			"progress", __func__);
+		break;
+
+	default:
+		/* We don't expect to be in other states, so log but continue
+		 * anyway */
+		vchiq_log_error(vchiq_susp_log_level,
+			"%s unexpected suspend state %s", __func__,
+			suspend_state_names[arm_state->vc_suspend_state +
+						VC_SUSPEND_NUM_OFFSET]);
+		/* fall through */
+	case VC_SUSPEND_REJECTED:
+	case VC_SUSPEND_FAILED:
+		/* Ensure any idle state actions have been run */
+		set_suspend_state(arm_state, VC_SUSPEND_IDLE);
+		/* fall through */
+	case VC_SUSPEND_IDLE:
+		vchiq_log_info(vchiq_susp_log_level,
+			"%s: suspending", __func__);
+		set_suspend_state(arm_state, VC_SUSPEND_REQUESTED);
+		/* kick the slot handler thread to initiate suspend */
+		request_poll(state, NULL, 0);
+		break;
+	}
+
+out:
+	vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, status);
+	return status;
+}
+
+void
+vchiq_platform_check_suspend(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	int susp = 0;
+
+	if (!arm_state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+
+	write_lock_bh(&arm_state->susp_res_lock);
+	if (arm_state->vc_suspend_state == VC_SUSPEND_REQUESTED &&
+			arm_state->vc_resume_state == VC_RESUME_RESUMED) {
+		set_suspend_state(arm_state, VC_SUSPEND_IN_PROGRESS);
+		susp = 1;
+	}
+	write_unlock_bh(&arm_state->susp_res_lock);
+
+	if (susp)
+		vchiq_platform_suspend(state);
+
+out:
+	vchiq_log_trace(vchiq_susp_log_level, "%s exit", __func__);
+	return;
+}
+
+
+static void
+output_timeout_error(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	char service_err[50] = "";
+	int vc_use_count = arm_state->videocore_use_count;
+	int active_services = state->unused_service;
+	int i;
+
+	if (!arm_state->videocore_use_count) {
+		snprintf(service_err, 50, " Videocore usecount is 0");
+		goto output_msg;
+	}
+	for (i = 0; i < active_services; i++) {
+		VCHIQ_SERVICE_T *service_ptr = state->services[i];
+		if (service_ptr && service_ptr->service_use_count &&
+			(service_ptr->srvstate != VCHIQ_SRVSTATE_FREE)) {
+			snprintf(service_err, 50, " %c%c%c%c(%d) service has "
+				"use count %d%s", VCHIQ_FOURCC_AS_4CHARS(
+					service_ptr->base.fourcc),
+				 service_ptr->client_id,
+				 service_ptr->service_use_count,
+				 service_ptr->service_use_count ==
+					 vc_use_count ? "" : " (+ more)");
+			break;
+		}
+	}
+
+output_msg:
+	vchiq_log_error(vchiq_susp_log_level,
+		"timed out waiting for vc suspend (%d).%s",
+		 arm_state->autosuspend_override, service_err);
+
+}
+
+/* Try to get videocore into suspended state, regardless of autosuspend state.
+** We don't actually force suspend, since videocore may get into a bad state
+** if we force suspend at a bad time.  Instead, we wait for autosuspend to
+** determine a good point to suspend.  If this doesn't happen within 100ms we
+** report failure.
+**
+** Returns VCHIQ_SUCCESS if videocore suspended successfully, VCHIQ_RETRY if
+** videocore failed to suspend in time or VCHIQ_ERROR if interrupted.
+*/
+VCHIQ_STATUS_T
+vchiq_arm_force_suspend(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	VCHIQ_STATUS_T status = VCHIQ_ERROR;
+	long rc = 0;
+	int repeat = -1;
+
+	if (!arm_state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+
+	write_lock_bh(&arm_state->susp_res_lock);
+
+	status = block_resume(arm_state);
+	if (status != VCHIQ_SUCCESS)
+		goto unlock;
+	if (arm_state->vc_suspend_state == VC_SUSPEND_SUSPENDED) {
+		/* Already suspended - just block resume and exit */
+		vchiq_log_info(vchiq_susp_log_level, "%s already suspended",
+			__func__);
+		status = VCHIQ_SUCCESS;
+		goto unlock;
+	} else if (arm_state->vc_suspend_state <= VC_SUSPEND_IDLE) {
+		/* initiate suspend immediately in the case that we're waiting
+		 * for the timeout */
+		stop_suspend_timer(arm_state);
+		if (!vchiq_videocore_wanted(state)) {
+			vchiq_log_info(vchiq_susp_log_level, "%s videocore "
+				"idle, initiating suspend", __func__);
+			status = vchiq_arm_vcsuspend(state);
+		} else if (arm_state->autosuspend_override <
+						FORCE_SUSPEND_FAIL_MAX) {
+			vchiq_log_info(vchiq_susp_log_level, "%s letting "
+				"videocore go idle", __func__);
+			status = VCHIQ_SUCCESS;
+		} else {
+			vchiq_log_warning(vchiq_susp_log_level, "%s failed too "
+				"many times - attempting suspend", __func__);
+			status = vchiq_arm_vcsuspend(state);
+		}
+	} else {
+		vchiq_log_info(vchiq_susp_log_level, "%s videocore suspend "
+			"in progress - wait for completion", __func__);
+		status = VCHIQ_SUCCESS;
+	}
+
+	/* Wait for suspend to happen due to system idle (not forced..) */
+	if (status != VCHIQ_SUCCESS)
+		goto unblock_resume;
+
+	do {
+		write_unlock_bh(&arm_state->susp_res_lock);
+
+		rc = wait_for_completion_interruptible_timeout(
+				&arm_state->vc_suspend_complete,
+				msecs_to_jiffies(FORCE_SUSPEND_TIMEOUT_MS));
+
+		write_lock_bh(&arm_state->susp_res_lock);
+		if (rc < 0) {
+			vchiq_log_warning(vchiq_susp_log_level, "%s "
+				"interrupted waiting for suspend", __func__);
+			status = VCHIQ_ERROR;
+			goto unblock_resume;
+		} else if (rc == 0) {
+			if (arm_state->vc_suspend_state > VC_SUSPEND_IDLE) {
+				/* Repeat timeout once if in progress */
+				if (repeat < 0) {
+					repeat = 1;
+					continue;
+				}
+			}
+			arm_state->autosuspend_override++;
+			output_timeout_error(state);
+
+			status = VCHIQ_RETRY;
+			goto unblock_resume;
+		}
+	} while (0 < (repeat--));
+
+	/* Check and report state in case we need to abort ARM suspend */
+	if (arm_state->vc_suspend_state != VC_SUSPEND_SUSPENDED) {
+		status = VCHIQ_RETRY;
+		vchiq_log_error(vchiq_susp_log_level,
+			"%s videocore suspend failed (state %s)", __func__,
+			suspend_state_names[arm_state->vc_suspend_state +
+						VC_SUSPEND_NUM_OFFSET]);
+		/* Reset the state only if it's still in an error state.
+		 * Something could have already initiated another suspend. */
+		if (arm_state->vc_suspend_state < VC_SUSPEND_IDLE)
+			set_suspend_state(arm_state, VC_SUSPEND_IDLE);
+
+		goto unblock_resume;
+	}
+
+	/* successfully suspended - unlock and exit */
+	goto unlock;
+
+unblock_resume:
+	/* all error states need to unblock resume before exit */
+	unblock_resume(arm_state);
+
+unlock:
+	write_unlock_bh(&arm_state->susp_res_lock);
+
+out:
+	vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, status);
+	return status;
+}
+
+void
+vchiq_check_suspend(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+
+	if (!arm_state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+
+	write_lock_bh(&arm_state->susp_res_lock);
+	if (arm_state->vc_suspend_state != VC_SUSPEND_SUSPENDED &&
+			arm_state->first_connect &&
+			!vchiq_videocore_wanted(state)) {
+		vchiq_arm_vcsuspend(state);
+	}
+	write_unlock_bh(&arm_state->susp_res_lock);
+
+out:
+	vchiq_log_trace(vchiq_susp_log_level, "%s exit", __func__);
+	return;
+}
+
+
+int
+vchiq_arm_allow_resume(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	int resume = 0;
+	int ret = -1;
+
+	if (!arm_state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+
+	write_lock_bh(&arm_state->susp_res_lock);
+	unblock_resume(arm_state);
+	resume = vchiq_check_resume(state);
+	write_unlock_bh(&arm_state->susp_res_lock);
+
+	if (resume) {
+		if (wait_for_completion_interruptible(
+			&arm_state->vc_resume_complete) < 0) {
+			vchiq_log_error(vchiq_susp_log_level,
+				"%s interrupted", __func__);
+			/* failed, cannot accurately derive suspend
+			 * state, so exit early. */
+			goto out;
+		}
+	}
+
+	read_lock_bh(&arm_state->susp_res_lock);
+	if (arm_state->vc_suspend_state == VC_SUSPEND_SUSPENDED) {
+		vchiq_log_info(vchiq_susp_log_level,
+				"%s: Videocore remains suspended", __func__);
+	} else {
+		vchiq_log_info(vchiq_susp_log_level,
+				"%s: Videocore resumed", __func__);
+		ret = 0;
+	}
+	read_unlock_bh(&arm_state->susp_res_lock);
+out:
+	vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
+	return ret;
+}
+
+/* This function should be called with the write lock held */
+int
+vchiq_check_resume(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	int resume = 0;
+
+	if (!arm_state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+
+	if (need_resume(state)) {
+		set_resume_state(arm_state, VC_RESUME_REQUESTED);
+		request_poll(state, NULL, 0);
+		resume = 1;
+	}
+
+out:
+	vchiq_log_trace(vchiq_susp_log_level, "%s exit", __func__);
+	return resume;
+}
+
+void
+vchiq_platform_check_resume(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	int res = 0;
+
+	if (!arm_state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+
+	write_lock_bh(&arm_state->susp_res_lock);
+	if (arm_state->wake_address == 0) {
+		vchiq_log_info(vchiq_susp_log_level,
+					"%s: already awake", __func__);
+		goto unlock;
+	}
+	if (arm_state->vc_resume_state == VC_RESUME_IN_PROGRESS) {
+		vchiq_log_info(vchiq_susp_log_level,
+					"%s: already resuming", __func__);
+		goto unlock;
+	}
+
+	if (arm_state->vc_resume_state == VC_RESUME_REQUESTED) {
+		set_resume_state(arm_state, VC_RESUME_IN_PROGRESS);
+		res = 1;
+	} else
+		vchiq_log_trace(vchiq_susp_log_level,
+				"%s: not resuming (resume state %s)", __func__,
+				resume_state_names[arm_state->vc_resume_state +
+							VC_RESUME_NUM_OFFSET]);
+
+unlock:
+	write_unlock_bh(&arm_state->susp_res_lock);
+
+	if (res)
+		vchiq_platform_resume(state);
+
+out:
+	vchiq_log_trace(vchiq_susp_log_level, "%s exit", __func__);
+	return;
+
+}
+
+
+
+VCHIQ_STATUS_T
+vchiq_use_internal(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
+		enum USE_TYPE_E use_type)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	VCHIQ_STATUS_T ret = VCHIQ_SUCCESS;
+	char entity[16];
+	int *entity_uc;
+	int local_uc, local_entity_uc;
+
+	if (!arm_state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+
+	if (use_type == USE_TYPE_VCHIQ) {
+		sprintf(entity, "VCHIQ:   ");
+		entity_uc = &arm_state->peer_use_count;
+	} else if (service) {
+		sprintf(entity, "%c%c%c%c:%03d",
+			VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
+			service->client_id);
+		entity_uc = &service->service_use_count;
+	} else {
+		vchiq_log_error(vchiq_susp_log_level, "%s null service "
+				"ptr", __func__);
+		ret = VCHIQ_ERROR;
+		goto out;
+	}
+
+	write_lock_bh(&arm_state->susp_res_lock);
+	while (arm_state->resume_blocked) {
+		/* If we call 'use' while force suspend is waiting for suspend,
+		 * then we're about to block the thread which the force is
+		 * waiting to complete, so we're bound to just time out. In this
+		 * case, set the suspend state such that the wait will be
+		 * canceled, so we can complete as quickly as possible. */
+		if (arm_state->resume_blocked && arm_state->vc_suspend_state ==
+				VC_SUSPEND_IDLE) {
+			set_suspend_state(arm_state, VC_SUSPEND_FORCE_CANCELED);
+			break;
+		}
+		/* If suspend is already in progress then we need to block */
+		if (!try_wait_for_completion(&arm_state->resume_blocker)) {
+			/* Indicate that there are threads waiting on the resume
+			 * blocker.  These need to be allowed to complete before
+			 * a _second_ call to force suspend can complete,
+			 * otherwise low priority threads might never actually
+			 * continue */
+			arm_state->blocked_count++;
+			write_unlock_bh(&arm_state->susp_res_lock);
+			vchiq_log_info(vchiq_susp_log_level, "%s %s resume "
+				"blocked - waiting...", __func__, entity);
+			if (wait_for_completion_killable(
+					&arm_state->resume_blocker) != 0) {
+				vchiq_log_error(vchiq_susp_log_level, "%s %s "
+					"wait for resume blocker interrupted",
+					__func__, entity);
+				ret = VCHIQ_ERROR;
+				write_lock_bh(&arm_state->susp_res_lock);
+				arm_state->blocked_count--;
+				write_unlock_bh(&arm_state->susp_res_lock);
+				goto out;
+			}
+			vchiq_log_info(vchiq_susp_log_level, "%s %s resume "
+				"unblocked", __func__, entity);
+			write_lock_bh(&arm_state->susp_res_lock);
+			if (--arm_state->blocked_count == 0)
+				complete_all(&arm_state->blocked_blocker);
+		}
+	}
+
+	stop_suspend_timer(arm_state);
+
+	local_uc = ++arm_state->videocore_use_count;
+	local_entity_uc = ++(*entity_uc);
+
+	/* If there's a pending request which hasn't yet been serviced then
+	 * just clear it.  If we're past VC_SUSPEND_REQUESTED state then
+	 * vc_resume_complete will block until we either resume or fail to
+	 * suspend */
+	if (arm_state->vc_suspend_state <= VC_SUSPEND_REQUESTED)
+		set_suspend_state(arm_state, VC_SUSPEND_IDLE);
+
+	if ((use_type != USE_TYPE_SERVICE_NO_RESUME) && need_resume(state)) {
+		set_resume_state(arm_state, VC_RESUME_REQUESTED);
+		vchiq_log_info(vchiq_susp_log_level,
+			"%s %s count %d, state count %d",
+			__func__, entity, local_entity_uc, local_uc);
+		request_poll(state, NULL, 0);
+	} else
+		vchiq_log_trace(vchiq_susp_log_level,
+			"%s %s count %d, state count %d",
+			__func__, entity, *entity_uc, local_uc);
+
+
+	write_unlock_bh(&arm_state->susp_res_lock);
+
+	/* Completion is in a done state when we're not suspended, so this won't
+	 * block for the non-suspended case. */
+	if (!try_wait_for_completion(&arm_state->vc_resume_complete)) {
+		vchiq_log_info(vchiq_susp_log_level, "%s %s wait for resume",
+			__func__, entity);
+		if (wait_for_completion_killable(
+				&arm_state->vc_resume_complete) != 0) {
+			vchiq_log_error(vchiq_susp_log_level, "%s %s wait for "
+				"resume interrupted", __func__, entity);
+			ret = VCHIQ_ERROR;
+			goto out;
+		}
+		vchiq_log_info(vchiq_susp_log_level, "%s %s resumed", __func__,
+			entity);
+	}
+
+	if (ret == VCHIQ_SUCCESS) {
+		VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+		long ack_cnt = atomic_xchg(&arm_state->ka_use_ack_count, 0);
+		while (ack_cnt && (status == VCHIQ_SUCCESS)) {
+			/* Send the use notify to videocore */
+			status = vchiq_send_remote_use_active(state);
+			if (status == VCHIQ_SUCCESS)
+				ack_cnt--;
+			else
+				atomic_add(ack_cnt,
+					&arm_state->ka_use_ack_count);
+		}
+	}
+
+out:
+	vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
+	return ret;
+}
+
+VCHIQ_STATUS_T
+vchiq_release_internal(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	VCHIQ_STATUS_T ret = VCHIQ_SUCCESS;
+	char entity[16];
+	int *entity_uc;
+	int local_uc, local_entity_uc;
+
+	if (!arm_state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+
+	if (service) {
+		sprintf(entity, "%c%c%c%c:%03d",
+			VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
+			service->client_id);
+		entity_uc = &service->service_use_count;
+	} else {
+		sprintf(entity, "PEER:   ");
+		entity_uc = &arm_state->peer_use_count;
+	}
+
+	write_lock_bh(&arm_state->susp_res_lock);
+	if (!arm_state->videocore_use_count || !(*entity_uc)) {
+		/* Don't use BUG_ON - don't allow user thread to crash kernel */
+		WARN_ON(!arm_state->videocore_use_count);
+		WARN_ON(!(*entity_uc));
+		ret = VCHIQ_ERROR;
+		goto unlock;
+	}
+	local_uc = --arm_state->videocore_use_count;
+	local_entity_uc = --(*entity_uc);
+
+	if (!vchiq_videocore_wanted(state)) {
+		if (vchiq_platform_use_suspend_timer() &&
+				!arm_state->resume_blocked) {
+			/* Only use the timer if we're not trying to force
+			 * suspend (=> resume_blocked) */
+			start_suspend_timer(arm_state);
+		} else {
+			vchiq_log_info(vchiq_susp_log_level,
+				"%s %s count %d, state count %d - suspending",
+				__func__, entity, *entity_uc,
+				arm_state->videocore_use_count);
+			vchiq_arm_vcsuspend(state);
+		}
+	} else
+		vchiq_log_trace(vchiq_susp_log_level,
+			"%s %s count %d, state count %d",
+			__func__, entity, *entity_uc,
+			arm_state->videocore_use_count);
+
+unlock:
+	write_unlock_bh(&arm_state->susp_res_lock);
+
+out:
+	vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret);
+	return ret;
+}
+
+void
+vchiq_on_remote_use(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+	atomic_inc(&arm_state->ka_use_count);
+	complete(&arm_state->ka_evt);
+}
+
+void
+vchiq_on_remote_release(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+	atomic_inc(&arm_state->ka_release_count);
+	complete(&arm_state->ka_evt);
+}
+
+VCHIQ_STATUS_T
+vchiq_use_service_internal(VCHIQ_SERVICE_T *service)
+{
+	return vchiq_use_internal(service->state, service, USE_TYPE_SERVICE);
+}
+
+VCHIQ_STATUS_T
+vchiq_release_service_internal(VCHIQ_SERVICE_T *service)
+{
+	return vchiq_release_internal(service->state, service);
+}
+
+VCHIQ_DEBUGFS_NODE_T *
+vchiq_instance_get_debugfs_node(VCHIQ_INSTANCE_T instance)
+{
+	return &instance->debugfs_node;
+}
+
+int
+vchiq_instance_get_use_count(VCHIQ_INSTANCE_T instance)
+{
+	VCHIQ_SERVICE_T *service;
+	int use_count = 0, i;
+	i = 0;
+	while ((service = next_service_by_instance(instance->state,
+		instance, &i)) != NULL) {
+		use_count += service->service_use_count;
+		unlock_service(service);
+	}
+	return use_count;
+}
+
+int
+vchiq_instance_get_pid(VCHIQ_INSTANCE_T instance)
+{
+	return instance->pid;
+}
+
+int
+vchiq_instance_get_trace(VCHIQ_INSTANCE_T instance)
+{
+	return instance->trace;
+}
+
+void
+vchiq_instance_set_trace(VCHIQ_INSTANCE_T instance, int trace)
+{
+	VCHIQ_SERVICE_T *service;
+	int i;
+	i = 0;
+	while ((service = next_service_by_instance(instance->state,
+		instance, &i)) != NULL) {
+		service->trace = trace;
+		unlock_service(service);
+	}
+	instance->trace = (trace != 0);
+}
+
+static void suspend_timer_callback(unsigned long context)
+{
+	VCHIQ_STATE_T *state = (VCHIQ_STATE_T *)context;
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	if (!arm_state)
+		goto out;
+	vchiq_log_info(vchiq_susp_log_level,
+		"%s - suspend timer expired - check suspend", __func__);
+	vchiq_check_suspend(state);
+out:
+	return;
+}
+
+VCHIQ_STATUS_T
+vchiq_use_service_no_resume(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	VCHIQ_STATUS_T ret = VCHIQ_ERROR;
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	if (service) {
+		ret = vchiq_use_internal(service->state, service,
+				USE_TYPE_SERVICE_NO_RESUME);
+		unlock_service(service);
+	}
+	return ret;
+}
+
+VCHIQ_STATUS_T
+vchiq_use_service(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	VCHIQ_STATUS_T ret = VCHIQ_ERROR;
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	if (service) {
+		ret = vchiq_use_internal(service->state, service,
+				USE_TYPE_SERVICE);
+		unlock_service(service);
+	}
+	return ret;
+}
+
+VCHIQ_STATUS_T
+vchiq_release_service(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	VCHIQ_STATUS_T ret = VCHIQ_ERROR;
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	if (service) {
+		ret = vchiq_release_internal(service->state, service);
+		unlock_service(service);
+	}
+	return ret;
+}
+
+void
+vchiq_dump_service_use_state(VCHIQ_STATE_T *state)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	int i, j = 0;
+	/* Only dump 64 services */
+	static const int local_max_services = 64;
+	/* If there's more than 64 services, only dump ones with
+	 * non-zero counts */
+	int only_nonzero = 0;
+	static const char *nz = "<-- preventing suspend";
+
+	enum vc_suspend_status vc_suspend_state;
+	enum vc_resume_status  vc_resume_state;
+	int peer_count;
+	int vc_use_count;
+	int active_services;
+	struct service_data_struct {
+		int fourcc;
+		int clientid;
+		int use_count;
+	} service_data[local_max_services];
+
+	if (!arm_state)
+		return;
+
+	read_lock_bh(&arm_state->susp_res_lock);
+	vc_suspend_state = arm_state->vc_suspend_state;
+	vc_resume_state  = arm_state->vc_resume_state;
+	peer_count = arm_state->peer_use_count;
+	vc_use_count = arm_state->videocore_use_count;
+	active_services = state->unused_service;
+	if (active_services > local_max_services)
+		only_nonzero = 1;
+
+	for (i = 0; (i < active_services) && (j < local_max_services); i++) {
+		VCHIQ_SERVICE_T *service_ptr = state->services[i];
+		if (!service_ptr)
+			continue;
+
+		if (only_nonzero && !service_ptr->service_use_count)
+			continue;
+
+		if (service_ptr->srvstate != VCHIQ_SRVSTATE_FREE) {
+			service_data[j].fourcc = service_ptr->base.fourcc;
+			service_data[j].clientid = service_ptr->client_id;
+			service_data[j++].use_count = service_ptr->
+							service_use_count;
+		}
+	}
+
+	read_unlock_bh(&arm_state->susp_res_lock);
+
+	vchiq_log_warning(vchiq_susp_log_level,
+		"-- Videcore suspend state: %s --",
+		suspend_state_names[vc_suspend_state + VC_SUSPEND_NUM_OFFSET]);
+	vchiq_log_warning(vchiq_susp_log_level,
+		"-- Videcore resume state: %s --",
+		resume_state_names[vc_resume_state + VC_RESUME_NUM_OFFSET]);
+
+	if (only_nonzero)
+		vchiq_log_warning(vchiq_susp_log_level, "Too many active "
+			"services (%d).  Only dumping up to first %d services "
+			"with non-zero use-count", active_services,
+			local_max_services);
+
+	for (i = 0; i < j; i++) {
+		vchiq_log_warning(vchiq_susp_log_level,
+			"----- %c%c%c%c:%d service count %d %s",
+			VCHIQ_FOURCC_AS_4CHARS(service_data[i].fourcc),
+			service_data[i].clientid,
+			service_data[i].use_count,
+			service_data[i].use_count ? nz : "");
+	}
+	vchiq_log_warning(vchiq_susp_log_level,
+		"----- VCHIQ use count count %d", peer_count);
+	vchiq_log_warning(vchiq_susp_log_level,
+		"--- Overall vchiq instance use count %d", vc_use_count);
+
+	vchiq_dump_platform_use_state(state);
+}
+
+VCHIQ_STATUS_T
+vchiq_check_service(VCHIQ_SERVICE_T *service)
+{
+	VCHIQ_ARM_STATE_T *arm_state;
+	VCHIQ_STATUS_T ret = VCHIQ_ERROR;
+
+	if (!service || !service->state)
+		goto out;
+
+	vchiq_log_trace(vchiq_susp_log_level, "%s", __func__);
+
+	arm_state = vchiq_platform_get_arm_state(service->state);
+
+	read_lock_bh(&arm_state->susp_res_lock);
+	if (service->service_use_count)
+		ret = VCHIQ_SUCCESS;
+	read_unlock_bh(&arm_state->susp_res_lock);
+
+	if (ret == VCHIQ_ERROR) {
+		vchiq_log_error(vchiq_susp_log_level,
+			"%s ERROR - %c%c%c%c:%d service count %d, "
+			"state count %d, videocore suspend state %s", __func__,
+			VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
+			service->client_id, service->service_use_count,
+			arm_state->videocore_use_count,
+			suspend_state_names[arm_state->vc_suspend_state +
+						VC_SUSPEND_NUM_OFFSET]);
+		vchiq_dump_service_use_state(service->state);
+	}
+out:
+	return ret;
+}
+
+/* stub functions */
+void vchiq_on_remote_use_active(VCHIQ_STATE_T *state)
+{
+	(void)state;
+}
+
+void vchiq_platform_conn_state_changed(VCHIQ_STATE_T *state,
+	VCHIQ_CONNSTATE_T oldstate, VCHIQ_CONNSTATE_T newstate)
+{
+	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
+	vchiq_log_info(vchiq_susp_log_level, "%d: %s->%s", state->id,
+		get_conn_state_name(oldstate), get_conn_state_name(newstate));
+	if (state->conn_state == VCHIQ_CONNSTATE_CONNECTED) {
+		write_lock_bh(&arm_state->susp_res_lock);
+		if (!arm_state->first_connect) {
+			char threadname[10];
+			arm_state->first_connect = 1;
+			write_unlock_bh(&arm_state->susp_res_lock);
+			snprintf(threadname, sizeof(threadname), "VCHIQka-%d",
+				state->id);
+			arm_state->ka_thread = kthread_create(
+				&vchiq_keepalive_thread_func,
+				(void *)state,
+				threadname);
+			if (arm_state->ka_thread == NULL) {
+				vchiq_log_error(vchiq_susp_log_level,
+					"vchiq: FATAL: couldn't create thread %s",
+					threadname);
+			} else {
+				wake_up_process(arm_state->ka_thread);
+			}
+		} else
+			write_unlock_bh(&arm_state->susp_res_lock);
+	}
+}
+
+static int vchiq_probe(struct platform_device *pdev)
+{
+	struct device_node *fw_node;
+	struct rpi_firmware *fw;
+	int err;
+	void *ptr_err;
+
+	fw_node = of_parse_phandle(pdev->dev.of_node, "firmware", 0);
+/* Remove comment when booting without Device Tree is no longer supported
+	if (!fw_node) {
+		dev_err(&pdev->dev, "Missing firmware node\n");
+		return -ENOENT;
+	}
+*/
+	fw = rpi_firmware_get(fw_node);
+	if (!fw)
+		return -EPROBE_DEFER;
+
+	platform_set_drvdata(pdev, fw);
+
+	/* create debugfs entries */
+	err = vchiq_debugfs_init();
+	if (err != 0)
+		goto failed_debugfs_init;
+
+	err = alloc_chrdev_region(&vchiq_devid, VCHIQ_MINOR, 1, DEVICE_NAME);
+	if (err != 0) {
+		vchiq_log_error(vchiq_arm_log_level,
+			"Unable to allocate device number");
+		goto failed_alloc_chrdev;
+	}
+	cdev_init(&vchiq_cdev, &vchiq_fops);
+	vchiq_cdev.owner = THIS_MODULE;
+	err = cdev_add(&vchiq_cdev, vchiq_devid, 1);
+	if (err != 0) {
+		vchiq_log_error(vchiq_arm_log_level,
+			"Unable to register device");
+		goto failed_cdev_add;
+	}
+
+	/* create sysfs entries */
+	vchiq_class = class_create(THIS_MODULE, DEVICE_NAME);
+	ptr_err = vchiq_class;
+	if (IS_ERR(ptr_err))
+		goto failed_class_create;
+
+	vchiq_dev = device_create(vchiq_class, NULL,
+		vchiq_devid, NULL, "vchiq");
+	ptr_err = vchiq_dev;
+	if (IS_ERR(ptr_err))
+		goto failed_device_create;
+
+	err = vchiq_platform_init(pdev, &g_state);
+	if (err != 0)
+		goto failed_platform_init;
+
+	vchiq_log_info(vchiq_arm_log_level,
+		"vchiq: initialised - version %d (min %d), device %d.%d",
+		VCHIQ_VERSION, VCHIQ_VERSION_MIN,
+		MAJOR(vchiq_devid), MINOR(vchiq_devid));
+
+	return 0;
+
+failed_platform_init:
+	device_destroy(vchiq_class, vchiq_devid);
+failed_device_create:
+	class_destroy(vchiq_class);
+failed_class_create:
+	cdev_del(&vchiq_cdev);
+	err = PTR_ERR(ptr_err);
+failed_cdev_add:
+	unregister_chrdev_region(vchiq_devid, 1);
+failed_alloc_chrdev:
+	vchiq_debugfs_deinit();
+failed_debugfs_init:
+	vchiq_log_warning(vchiq_arm_log_level, "could not load vchiq");
+	return err;
+}
+
+static int vchiq_remove(struct platform_device *pdev)
+{
+	device_destroy(vchiq_class, vchiq_devid);
+	class_destroy(vchiq_class);
+	cdev_del(&vchiq_cdev);
+	unregister_chrdev_region(vchiq_devid, 1);
+
+	return 0;
+}
+
+static const struct of_device_id vchiq_of_match[] = {
+	{ .compatible = "brcm,bcm2835-vchiq", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, vchiq_of_match);
+
+static struct platform_driver vchiq_driver = {
+	.driver = {
+		.name = "bcm2835_vchiq",
+		.owner = THIS_MODULE,
+		.of_match_table = vchiq_of_match,
+	},
+	.probe = vchiq_probe,
+	.remove = vchiq_remove,
+};
+module_platform_driver(vchiq_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Broadcom Corporation");
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.h
new file mode 100644
index 000000000000..9740e1afbc9d
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.h
@@ -0,0 +1,220 @@
+/**
+ * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved.
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_ARM_H
+#define VCHIQ_ARM_H
+
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/semaphore.h>
+#include <linux/atomic.h>
+#include "vchiq_core.h"
+#include "vchiq_debugfs.h"
+
+
+enum vc_suspend_status {
+	VC_SUSPEND_FORCE_CANCELED = -3, /* Force suspend canceled, too busy */
+	VC_SUSPEND_REJECTED = -2,  /* Videocore rejected suspend request */
+	VC_SUSPEND_FAILED = -1,    /* Videocore suspend failed */
+	VC_SUSPEND_IDLE = 0,       /* VC active, no suspend actions */
+	VC_SUSPEND_REQUESTED,      /* User has requested suspend */
+	VC_SUSPEND_IN_PROGRESS,    /* Slot handler has recvd suspend request */
+	VC_SUSPEND_SUSPENDED       /* Videocore suspend succeeded */
+};
+
+enum vc_resume_status {
+	VC_RESUME_FAILED = -1, /* Videocore resume failed */
+	VC_RESUME_IDLE = 0,    /* VC suspended, no resume actions */
+	VC_RESUME_REQUESTED,   /* User has requested resume */
+	VC_RESUME_IN_PROGRESS, /* Slot handler has received resume request */
+	VC_RESUME_RESUMED      /* Videocore resumed successfully (active) */
+};
+
+
+enum USE_TYPE_E {
+	USE_TYPE_SERVICE,
+	USE_TYPE_SERVICE_NO_RESUME,
+	USE_TYPE_VCHIQ
+};
+
+
+
+typedef struct vchiq_arm_state_struct {
+	/* Keepalive-related data */
+	struct task_struct *ka_thread;
+	struct completion ka_evt;
+	atomic_t ka_use_count;
+	atomic_t ka_use_ack_count;
+	atomic_t ka_release_count;
+
+	struct completion vc_suspend_complete;
+	struct completion vc_resume_complete;
+
+	rwlock_t susp_res_lock;
+	enum vc_suspend_status vc_suspend_state;
+	enum vc_resume_status vc_resume_state;
+
+	unsigned int wake_address;
+
+	struct timer_list suspend_timer;
+	int suspend_timer_timeout;
+	int suspend_timer_running;
+
+	/* Global use count for videocore.
+	** This is equal to the sum of the use counts for all services.  When
+	** this hits zero the videocore suspend procedure will be initiated.
+	*/
+	int videocore_use_count;
+
+	/* Use count to track requests from videocore peer.
+	** This use count is not associated with a service, so needs to be
+	** tracked separately with the state.
+	*/
+	int peer_use_count;
+
+	/* Flag to indicate whether resume is blocked.  This happens when the
+	** ARM is suspending
+	*/
+	struct completion resume_blocker;
+	int resume_blocked;
+	struct completion blocked_blocker;
+	int blocked_count;
+
+	int autosuspend_override;
+
+	/* Flag to indicate that the first vchiq connect has made it through.
+	** This means that both sides should be fully ready, and we should
+	** be able to suspend after this point.
+	*/
+	int first_connect;
+
+	unsigned long long suspend_start_time;
+	unsigned long long sleep_start_time;
+	unsigned long long resume_start_time;
+	unsigned long long last_wake_time;
+
+} VCHIQ_ARM_STATE_T;
+
+extern int vchiq_arm_log_level;
+extern int vchiq_susp_log_level;
+
+int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATE_T *
+vchiq_get_state(void);
+
+extern VCHIQ_STATUS_T
+vchiq_arm_vcsuspend(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_arm_force_suspend(VCHIQ_STATE_T *state);
+
+extern int
+vchiq_arm_allow_resume(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_arm_vcresume(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_arm_init_state(VCHIQ_STATE_T *state, VCHIQ_ARM_STATE_T *arm_state);
+
+extern int
+vchiq_check_resume(VCHIQ_STATE_T *state);
+
+extern void
+vchiq_check_suspend(VCHIQ_STATE_T *state);
+ VCHIQ_STATUS_T
+vchiq_use_service(VCHIQ_SERVICE_HANDLE_T handle);
+
+extern VCHIQ_STATUS_T
+vchiq_release_service(VCHIQ_SERVICE_HANDLE_T handle);
+
+extern VCHIQ_STATUS_T
+vchiq_check_service(VCHIQ_SERVICE_T *service);
+
+extern VCHIQ_STATUS_T
+vchiq_platform_suspend(VCHIQ_STATE_T *state);
+
+extern int
+vchiq_platform_videocore_wanted(VCHIQ_STATE_T *state);
+
+extern int
+vchiq_platform_use_suspend_timer(void);
+
+extern void
+vchiq_dump_platform_use_state(VCHIQ_STATE_T *state);
+
+extern void
+vchiq_dump_service_use_state(VCHIQ_STATE_T *state);
+
+extern VCHIQ_ARM_STATE_T*
+vchiq_platform_get_arm_state(VCHIQ_STATE_T *state);
+
+extern int
+vchiq_videocore_wanted(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_use_internal(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
+		enum USE_TYPE_E use_type);
+extern VCHIQ_STATUS_T
+vchiq_release_internal(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service);
+
+extern VCHIQ_DEBUGFS_NODE_T *
+vchiq_instance_get_debugfs_node(VCHIQ_INSTANCE_T instance);
+
+extern int
+vchiq_instance_get_use_count(VCHIQ_INSTANCE_T instance);
+
+extern int
+vchiq_instance_get_pid(VCHIQ_INSTANCE_T instance);
+
+extern int
+vchiq_instance_get_trace(VCHIQ_INSTANCE_T instance);
+
+extern void
+vchiq_instance_set_trace(VCHIQ_INSTANCE_T instance, int trace);
+
+extern void
+set_suspend_state(VCHIQ_ARM_STATE_T *arm_state,
+	enum vc_suspend_status new_state);
+
+extern void
+set_resume_state(VCHIQ_ARM_STATE_T *arm_state,
+	enum vc_resume_status new_state);
+
+extern void
+start_suspend_timer(VCHIQ_ARM_STATE_T *arm_state);
+
+
+#endif /* VCHIQ_ARM_H */
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_build_info.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_build_info.h
new file mode 100644
index 000000000000..df645813bdae
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_build_info.h
@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+const char *vchiq_get_build_hostname(void);
+const char *vchiq_get_build_version(void);
+const char *vchiq_get_build_time(void);
+const char *vchiq_get_build_date(void);
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_cfg.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_cfg.h
new file mode 100644
index 000000000000..d2797db702f9
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_cfg.h
@@ -0,0 +1,69 @@
+/**
+ * Copyright (c) 2010-2014 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_CFG_H
+#define VCHIQ_CFG_H
+
+#define VCHIQ_MAGIC              VCHIQ_MAKE_FOURCC('V', 'C', 'H', 'I')
+/* The version of VCHIQ - change with any non-trivial change */
+#define VCHIQ_VERSION            8
+/* The minimum compatible version - update to match VCHIQ_VERSION with any
+** incompatible change */
+#define VCHIQ_VERSION_MIN        3
+
+/* The version that introduced the VCHIQ_IOC_LIB_VERSION ioctl */
+#define VCHIQ_VERSION_LIB_VERSION 7
+
+/* The version that introduced the VCHIQ_IOC_CLOSE_DELIVERED ioctl */
+#define VCHIQ_VERSION_CLOSE_DELIVERED 7
+
+/* The version that made it safe to use SYNCHRONOUS mode */
+#define VCHIQ_VERSION_SYNCHRONOUS_MODE 8
+
+#define VCHIQ_MAX_STATES         1
+#define VCHIQ_MAX_SERVICES       4096
+#define VCHIQ_MAX_SLOTS          128
+#define VCHIQ_MAX_SLOTS_PER_SIDE 64
+
+#define VCHIQ_NUM_CURRENT_BULKS        32
+#define VCHIQ_NUM_SERVICE_BULKS        4
+
+#ifndef VCHIQ_ENABLE_DEBUG
+#define VCHIQ_ENABLE_DEBUG             1
+#endif
+
+#ifndef VCHIQ_ENABLE_STATS
+#define VCHIQ_ENABLE_STATS             1
+#endif
+
+#endif /* VCHIQ_CFG_H */
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c
new file mode 100644
index 000000000000..5efc62ffb2f5
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c
@@ -0,0 +1,120 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "vchiq_connected.h"
+#include "vchiq_core.h"
+#include "vchiq_killable.h"
+#include <linux/module.h>
+#include <linux/mutex.h>
+
+#define  MAX_CALLBACKS  10
+
+static   int                        g_connected;
+static   int                        g_num_deferred_callbacks;
+static   VCHIQ_CONNECTED_CALLBACK_T g_deferred_callback[MAX_CALLBACKS];
+static   int                        g_once_init;
+static   struct mutex               g_connected_mutex;
+
+/****************************************************************************
+*
+* Function to initialize our lock.
+*
+***************************************************************************/
+
+static void connected_init(void)
+{
+	if (!g_once_init) {
+		mutex_init(&g_connected_mutex);
+		g_once_init = 1;
+	}
+}
+
+/****************************************************************************
+*
+* This function is used to defer initialization until the vchiq stack is
+* initialized. If the stack is already initialized, then the callback will
+* be made immediately, otherwise it will be deferred until
+* vchiq_call_connected_callbacks is called.
+*
+***************************************************************************/
+
+void vchiq_add_connected_callback(VCHIQ_CONNECTED_CALLBACK_T callback)
+{
+	connected_init();
+
+	if (mutex_lock_interruptible(&g_connected_mutex) != 0)
+		return;
+
+	if (g_connected)
+		/* We're already connected. Call the callback immediately. */
+
+		callback();
+	else {
+		if (g_num_deferred_callbacks >= MAX_CALLBACKS)
+			vchiq_log_error(vchiq_core_log_level,
+				"There already %d callback registered - "
+				"please increase MAX_CALLBACKS",
+				g_num_deferred_callbacks);
+		else {
+			g_deferred_callback[g_num_deferred_callbacks] =
+				callback;
+			g_num_deferred_callbacks++;
+		}
+	}
+	mutex_unlock(&g_connected_mutex);
+}
+
+/****************************************************************************
+*
+* This function is called by the vchiq stack once it has been connected to
+* the videocore and clients can start to use the stack.
+*
+***************************************************************************/
+
+void vchiq_call_connected_callbacks(void)
+{
+	int i;
+
+	connected_init();
+
+	if (mutex_lock_interruptible(&g_connected_mutex) != 0)
+		return;
+
+	for (i = 0; i <  g_num_deferred_callbacks; i++)
+		g_deferred_callback[i]();
+
+	g_num_deferred_callbacks = 0;
+	g_connected = 1;
+	mutex_unlock(&g_connected_mutex);
+}
+EXPORT_SYMBOL(vchiq_add_connected_callback);
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.h
new file mode 100644
index 000000000000..863b3e335c1a
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.h
@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_CONNECTED_H
+#define VCHIQ_CONNECTED_H
+
+/* ---- Include Files ----------------------------------------------------- */
+
+/* ---- Constants and Types ---------------------------------------------- */
+
+typedef void (*VCHIQ_CONNECTED_CALLBACK_T)(void);
+
+/* ---- Variable Externs ------------------------------------------------- */
+
+/* ---- Function Prototypes ---------------------------------------------- */
+
+void vchiq_add_connected_callback(VCHIQ_CONNECTED_CALLBACK_T callback);
+void vchiq_call_connected_callbacks(void);
+
+#endif /* VCHIQ_CONNECTED_H */
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
new file mode 100644
index 000000000000..2c98da4307df
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
@@ -0,0 +1,3934 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "vchiq_core.h"
+#include "vchiq_killable.h"
+
+#define VCHIQ_SLOT_HANDLER_STACK 8192
+
+#define HANDLE_STATE_SHIFT 12
+
+#define SLOT_INFO_FROM_INDEX(state, index) (state->slot_info + (index))
+#define SLOT_DATA_FROM_INDEX(state, index) (state->slot_data + (index))
+#define SLOT_INDEX_FROM_DATA(state, data) \
+	(((unsigned int)((char *)data - (char *)state->slot_data)) / \
+	VCHIQ_SLOT_SIZE)
+#define SLOT_INDEX_FROM_INFO(state, info) \
+	((unsigned int)(info - state->slot_info))
+#define SLOT_QUEUE_INDEX_FROM_POS(pos) \
+	((int)((unsigned int)(pos) / VCHIQ_SLOT_SIZE))
+
+#define BULK_INDEX(x) (x & (VCHIQ_NUM_SERVICE_BULKS - 1))
+
+#define SRVTRACE_LEVEL(srv) \
+	(((srv) && (srv)->trace) ? VCHIQ_LOG_TRACE : vchiq_core_msg_log_level)
+#define SRVTRACE_ENABLED(srv, lev) \
+	(((srv) && (srv)->trace) || (vchiq_core_msg_log_level >= (lev)))
+
+struct vchiq_open_payload {
+	int fourcc;
+	int client_id;
+	short version;
+	short version_min;
+};
+
+struct vchiq_openack_payload {
+	short version;
+};
+
+enum
+{
+	QMFLAGS_IS_BLOCKING     = (1 << 0),
+	QMFLAGS_NO_MUTEX_LOCK   = (1 << 1),
+	QMFLAGS_NO_MUTEX_UNLOCK = (1 << 2)
+};
+
+/* we require this for consistency between endpoints */
+vchiq_static_assert(sizeof(VCHIQ_HEADER_T) == 8);
+vchiq_static_assert(IS_POW2(sizeof(VCHIQ_HEADER_T)));
+vchiq_static_assert(IS_POW2(VCHIQ_NUM_CURRENT_BULKS));
+vchiq_static_assert(IS_POW2(VCHIQ_NUM_SERVICE_BULKS));
+vchiq_static_assert(IS_POW2(VCHIQ_MAX_SERVICES));
+vchiq_static_assert(VCHIQ_VERSION >= VCHIQ_VERSION_MIN);
+
+/* Run time control of log level, based on KERN_XXX level. */
+int vchiq_core_log_level = VCHIQ_LOG_DEFAULT;
+int vchiq_core_msg_log_level = VCHIQ_LOG_DEFAULT;
+int vchiq_sync_log_level = VCHIQ_LOG_DEFAULT;
+
+static atomic_t pause_bulks_count = ATOMIC_INIT(0);
+
+static DEFINE_SPINLOCK(service_spinlock);
+DEFINE_SPINLOCK(bulk_waiter_spinlock);
+DEFINE_SPINLOCK(quota_spinlock);
+
+VCHIQ_STATE_T *vchiq_states[VCHIQ_MAX_STATES];
+static unsigned int handle_seq;
+
+static const char *const srvstate_names[] = {
+	"FREE",
+	"HIDDEN",
+	"LISTENING",
+	"OPENING",
+	"OPEN",
+	"OPENSYNC",
+	"CLOSESENT",
+	"CLOSERECVD",
+	"CLOSEWAIT",
+	"CLOSED"
+};
+
+static const char *const reason_names[] = {
+	"SERVICE_OPENED",
+	"SERVICE_CLOSED",
+	"MESSAGE_AVAILABLE",
+	"BULK_TRANSMIT_DONE",
+	"BULK_RECEIVE_DONE",
+	"BULK_TRANSMIT_ABORTED",
+	"BULK_RECEIVE_ABORTED"
+};
+
+static const char *const conn_state_names[] = {
+	"DISCONNECTED",
+	"CONNECTING",
+	"CONNECTED",
+	"PAUSING",
+	"PAUSE_SENT",
+	"PAUSED",
+	"RESUMING",
+	"PAUSE_TIMEOUT",
+	"RESUME_TIMEOUT"
+};
+
+
+static void
+release_message_sync(VCHIQ_STATE_T *state, VCHIQ_HEADER_T *header);
+
+static const char *msg_type_str(unsigned int msg_type)
+{
+	switch (msg_type) {
+	case VCHIQ_MSG_PADDING:       return "PADDING";
+	case VCHIQ_MSG_CONNECT:       return "CONNECT";
+	case VCHIQ_MSG_OPEN:          return "OPEN";
+	case VCHIQ_MSG_OPENACK:       return "OPENACK";
+	case VCHIQ_MSG_CLOSE:         return "CLOSE";
+	case VCHIQ_MSG_DATA:          return "DATA";
+	case VCHIQ_MSG_BULK_RX:       return "BULK_RX";
+	case VCHIQ_MSG_BULK_TX:       return "BULK_TX";
+	case VCHIQ_MSG_BULK_RX_DONE:  return "BULK_RX_DONE";
+	case VCHIQ_MSG_BULK_TX_DONE:  return "BULK_TX_DONE";
+	case VCHIQ_MSG_PAUSE:         return "PAUSE";
+	case VCHIQ_MSG_RESUME:        return "RESUME";
+	case VCHIQ_MSG_REMOTE_USE:    return "REMOTE_USE";
+	case VCHIQ_MSG_REMOTE_RELEASE:      return "REMOTE_RELEASE";
+	case VCHIQ_MSG_REMOTE_USE_ACTIVE:   return "REMOTE_USE_ACTIVE";
+	}
+	return "???";
+}
+
+static inline void
+vchiq_set_service_state(VCHIQ_SERVICE_T *service, int newstate)
+{
+	vchiq_log_info(vchiq_core_log_level, "%d: srv:%d %s->%s",
+		service->state->id, service->localport,
+		srvstate_names[service->srvstate],
+		srvstate_names[newstate]);
+	service->srvstate = newstate;
+}
+
+VCHIQ_SERVICE_T *
+find_service_by_handle(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	VCHIQ_SERVICE_T *service;
+
+	spin_lock(&service_spinlock);
+	service = handle_to_service(handle);
+	if (service && (service->srvstate != VCHIQ_SRVSTATE_FREE) &&
+		(service->handle == handle)) {
+		BUG_ON(service->ref_count == 0);
+		service->ref_count++;
+	} else
+		service = NULL;
+	spin_unlock(&service_spinlock);
+
+	if (!service)
+		vchiq_log_info(vchiq_core_log_level,
+			"Invalid service handle 0x%x", handle);
+
+	return service;
+}
+
+VCHIQ_SERVICE_T *
+find_service_by_port(VCHIQ_STATE_T *state, int localport)
+{
+	VCHIQ_SERVICE_T *service = NULL;
+	if ((unsigned int)localport <= VCHIQ_PORT_MAX) {
+		spin_lock(&service_spinlock);
+		service = state->services[localport];
+		if (service && (service->srvstate != VCHIQ_SRVSTATE_FREE)) {
+			BUG_ON(service->ref_count == 0);
+			service->ref_count++;
+		} else
+			service = NULL;
+		spin_unlock(&service_spinlock);
+	}
+
+	if (!service)
+		vchiq_log_info(vchiq_core_log_level,
+			"Invalid port %d", localport);
+
+	return service;
+}
+
+VCHIQ_SERVICE_T *
+find_service_for_instance(VCHIQ_INSTANCE_T instance,
+	VCHIQ_SERVICE_HANDLE_T handle) {
+	VCHIQ_SERVICE_T *service;
+
+	spin_lock(&service_spinlock);
+	service = handle_to_service(handle);
+	if (service && (service->srvstate != VCHIQ_SRVSTATE_FREE) &&
+		(service->handle == handle) &&
+		(service->instance == instance)) {
+		BUG_ON(service->ref_count == 0);
+		service->ref_count++;
+	} else
+		service = NULL;
+	spin_unlock(&service_spinlock);
+
+	if (!service)
+		vchiq_log_info(vchiq_core_log_level,
+			"Invalid service handle 0x%x", handle);
+
+	return service;
+}
+
+VCHIQ_SERVICE_T *
+find_closed_service_for_instance(VCHIQ_INSTANCE_T instance,
+	VCHIQ_SERVICE_HANDLE_T handle) {
+	VCHIQ_SERVICE_T *service;
+
+	spin_lock(&service_spinlock);
+	service = handle_to_service(handle);
+	if (service &&
+		((service->srvstate == VCHIQ_SRVSTATE_FREE) ||
+		 (service->srvstate == VCHIQ_SRVSTATE_CLOSED)) &&
+		(service->handle == handle) &&
+		(service->instance == instance)) {
+		BUG_ON(service->ref_count == 0);
+		service->ref_count++;
+	} else
+		service = NULL;
+	spin_unlock(&service_spinlock);
+
+	if (!service)
+		vchiq_log_info(vchiq_core_log_level,
+			"Invalid service handle 0x%x", handle);
+
+	return service;
+}
+
+VCHIQ_SERVICE_T *
+next_service_by_instance(VCHIQ_STATE_T *state, VCHIQ_INSTANCE_T instance,
+	int *pidx)
+{
+	VCHIQ_SERVICE_T *service = NULL;
+	int idx = *pidx;
+
+	spin_lock(&service_spinlock);
+	while (idx < state->unused_service) {
+		VCHIQ_SERVICE_T *srv = state->services[idx++];
+		if (srv && (srv->srvstate != VCHIQ_SRVSTATE_FREE) &&
+			(srv->instance == instance)) {
+			service = srv;
+			BUG_ON(service->ref_count == 0);
+			service->ref_count++;
+			break;
+		}
+	}
+	spin_unlock(&service_spinlock);
+
+	*pidx = idx;
+
+	return service;
+}
+
+void
+lock_service(VCHIQ_SERVICE_T *service)
+{
+	spin_lock(&service_spinlock);
+	BUG_ON(!service || (service->ref_count == 0));
+	if (service)
+		service->ref_count++;
+	spin_unlock(&service_spinlock);
+}
+
+void
+unlock_service(VCHIQ_SERVICE_T *service)
+{
+	VCHIQ_STATE_T *state = service->state;
+	spin_lock(&service_spinlock);
+	BUG_ON(!service || (service->ref_count == 0));
+	if (service && service->ref_count) {
+		service->ref_count--;
+		if (!service->ref_count) {
+			BUG_ON(service->srvstate != VCHIQ_SRVSTATE_FREE);
+			state->services[service->localport] = NULL;
+		} else
+			service = NULL;
+	}
+	spin_unlock(&service_spinlock);
+
+	if (service && service->userdata_term)
+		service->userdata_term(service->base.userdata);
+
+	kfree(service);
+}
+
+int
+vchiq_get_client_id(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	int id;
+
+	id = service ? service->client_id : 0;
+	if (service)
+		unlock_service(service);
+
+	return id;
+}
+
+void *
+vchiq_get_service_userdata(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	VCHIQ_SERVICE_T *service = handle_to_service(handle);
+
+	return service ? service->base.userdata : NULL;
+}
+
+int
+vchiq_get_service_fourcc(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	VCHIQ_SERVICE_T *service = handle_to_service(handle);
+
+	return service ? service->base.fourcc : 0;
+}
+
+static void
+mark_service_closing_internal(VCHIQ_SERVICE_T *service, int sh_thread)
+{
+	VCHIQ_STATE_T *state = service->state;
+	VCHIQ_SERVICE_QUOTA_T *service_quota;
+
+	service->closing = 1;
+
+	/* Synchronise with other threads. */
+	mutex_lock(&state->recycle_mutex);
+	mutex_unlock(&state->recycle_mutex);
+	if (!sh_thread || (state->conn_state != VCHIQ_CONNSTATE_PAUSE_SENT)) {
+		/* If we're pausing then the slot_mutex is held until resume
+		 * by the slot handler.  Therefore don't try to acquire this
+		 * mutex if we're the slot handler and in the pause sent state.
+		 * We don't need to in this case anyway. */
+		mutex_lock(&state->slot_mutex);
+		mutex_unlock(&state->slot_mutex);
+	}
+
+	/* Unblock any sending thread. */
+	service_quota = &state->service_quotas[service->localport];
+	up(&service_quota->quota_event);
+}
+
+static void
+mark_service_closing(VCHIQ_SERVICE_T *service)
+{
+	mark_service_closing_internal(service, 0);
+}
+
+static inline VCHIQ_STATUS_T
+make_service_callback(VCHIQ_SERVICE_T *service, VCHIQ_REASON_T reason,
+	VCHIQ_HEADER_T *header, void *bulk_userdata)
+{
+	VCHIQ_STATUS_T status;
+	vchiq_log_trace(vchiq_core_log_level, "%d: callback:%d (%s, %x, %x)",
+		service->state->id, service->localport, reason_names[reason],
+		(unsigned int)header, (unsigned int)bulk_userdata);
+	status = service->base.callback(reason, header, service->handle,
+		bulk_userdata);
+	if (status == VCHIQ_ERROR) {
+		vchiq_log_warning(vchiq_core_log_level,
+			"%d: ignoring ERROR from callback to service %x",
+			service->state->id, service->handle);
+		status = VCHIQ_SUCCESS;
+	}
+	return status;
+}
+
+inline void
+vchiq_set_conn_state(VCHIQ_STATE_T *state, VCHIQ_CONNSTATE_T newstate)
+{
+	VCHIQ_CONNSTATE_T oldstate = state->conn_state;
+	vchiq_log_info(vchiq_core_log_level, "%d: %s->%s", state->id,
+		conn_state_names[oldstate],
+		conn_state_names[newstate]);
+	state->conn_state = newstate;
+	vchiq_platform_conn_state_changed(state, oldstate, newstate);
+}
+
+static inline void
+remote_event_create(REMOTE_EVENT_T *event)
+{
+	event->armed = 0;
+	/* Don't clear the 'fired' flag because it may already have been set
+	** by the other side. */
+	sema_init(event->event, 0);
+}
+
+static inline void
+remote_event_destroy(REMOTE_EVENT_T *event)
+{
+	(void)event;
+}
+
+static inline int
+remote_event_wait(REMOTE_EVENT_T *event)
+{
+	if (!event->fired) {
+		event->armed = 1;
+		dsb();
+		if (!event->fired) {
+			if (down_interruptible(event->event) != 0) {
+				event->armed = 0;
+				return 0;
+			}
+		}
+		event->armed = 0;
+		wmb();
+	}
+
+	event->fired = 0;
+	return 1;
+}
+
+static inline void
+remote_event_signal_local(REMOTE_EVENT_T *event)
+{
+	event->armed = 0;
+	up(event->event);
+}
+
+static inline void
+remote_event_poll(REMOTE_EVENT_T *event)
+{
+	if (event->fired && event->armed)
+		remote_event_signal_local(event);
+}
+
+void
+remote_event_pollall(VCHIQ_STATE_T *state)
+{
+	remote_event_poll(&state->local->sync_trigger);
+	remote_event_poll(&state->local->sync_release);
+	remote_event_poll(&state->local->trigger);
+	remote_event_poll(&state->local->recycle);
+}
+
+/* Round up message sizes so that any space at the end of a slot is always big
+** enough for a header. This relies on header size being a power of two, which
+** has been verified earlier by a static assertion. */
+
+static inline unsigned int
+calc_stride(unsigned int size)
+{
+	/* Allow room for the header */
+	size += sizeof(VCHIQ_HEADER_T);
+
+	/* Round up */
+	return (size + sizeof(VCHIQ_HEADER_T) - 1) & ~(sizeof(VCHIQ_HEADER_T)
+		- 1);
+}
+
+/* Called by the slot handler thread */
+static VCHIQ_SERVICE_T *
+get_listening_service(VCHIQ_STATE_T *state, int fourcc)
+{
+	int i;
+
+	WARN_ON(fourcc == VCHIQ_FOURCC_INVALID);
+
+	for (i = 0; i < state->unused_service; i++) {
+		VCHIQ_SERVICE_T *service = state->services[i];
+		if (service &&
+			(service->public_fourcc == fourcc) &&
+			((service->srvstate == VCHIQ_SRVSTATE_LISTENING) ||
+			((service->srvstate == VCHIQ_SRVSTATE_OPEN) &&
+			(service->remoteport == VCHIQ_PORT_FREE)))) {
+			lock_service(service);
+			return service;
+		}
+	}
+
+	return NULL;
+}
+
+/* Called by the slot handler thread */
+static VCHIQ_SERVICE_T *
+get_connected_service(VCHIQ_STATE_T *state, unsigned int port)
+{
+	int i;
+	for (i = 0; i < state->unused_service; i++) {
+		VCHIQ_SERVICE_T *service = state->services[i];
+		if (service && (service->srvstate == VCHIQ_SRVSTATE_OPEN)
+			&& (service->remoteport == port)) {
+			lock_service(service);
+			return service;
+		}
+	}
+	return NULL;
+}
+
+inline void
+request_poll(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service, int poll_type)
+{
+	uint32_t value;
+
+	if (service) {
+		do {
+			value = atomic_read(&service->poll_flags);
+		} while (atomic_cmpxchg(&service->poll_flags, value,
+			value | (1 << poll_type)) != value);
+
+		do {
+			value = atomic_read(&state->poll_services[
+				service->localport>>5]);
+		} while (atomic_cmpxchg(
+			&state->poll_services[service->localport>>5],
+			value, value | (1 << (service->localport & 0x1f)))
+			!= value);
+	}
+
+	state->poll_needed = 1;
+	wmb();
+
+	/* ... and ensure the slot handler runs. */
+	remote_event_signal_local(&state->local->trigger);
+}
+
+/* Called from queue_message, by the slot handler and application threads,
+** with slot_mutex held */
+static VCHIQ_HEADER_T *
+reserve_space(VCHIQ_STATE_T *state, int space, int is_blocking)
+{
+	VCHIQ_SHARED_STATE_T *local = state->local;
+	int tx_pos = state->local_tx_pos;
+	int slot_space = VCHIQ_SLOT_SIZE - (tx_pos & VCHIQ_SLOT_MASK);
+
+	if (space > slot_space) {
+		VCHIQ_HEADER_T *header;
+		/* Fill the remaining space with padding */
+		WARN_ON(state->tx_data == NULL);
+		header = (VCHIQ_HEADER_T *)
+			(state->tx_data + (tx_pos & VCHIQ_SLOT_MASK));
+		header->msgid = VCHIQ_MSGID_PADDING;
+		header->size = slot_space - sizeof(VCHIQ_HEADER_T);
+
+		tx_pos += slot_space;
+	}
+
+	/* If necessary, get the next slot. */
+	if ((tx_pos & VCHIQ_SLOT_MASK) == 0) {
+		int slot_index;
+
+		/* If there is no free slot... */
+
+		if (down_trylock(&state->slot_available_event) != 0) {
+			/* ...wait for one. */
+
+			VCHIQ_STATS_INC(state, slot_stalls);
+
+			/* But first, flush through the last slot. */
+			state->local_tx_pos = tx_pos;
+			local->tx_pos = tx_pos;
+			remote_event_signal(&state->remote->trigger);
+
+			if (!is_blocking ||
+				(down_interruptible(
+				&state->slot_available_event) != 0))
+				return NULL; /* No space available */
+		}
+
+		BUG_ON(tx_pos ==
+			(state->slot_queue_available * VCHIQ_SLOT_SIZE));
+
+		slot_index = local->slot_queue[
+			SLOT_QUEUE_INDEX_FROM_POS(tx_pos) &
+			VCHIQ_SLOT_QUEUE_MASK];
+		state->tx_data =
+			(char *)SLOT_DATA_FROM_INDEX(state, slot_index);
+	}
+
+	state->local_tx_pos = tx_pos + space;
+
+	return (VCHIQ_HEADER_T *)(state->tx_data + (tx_pos & VCHIQ_SLOT_MASK));
+}
+
+/* Called by the recycle thread. */
+static void
+process_free_queue(VCHIQ_STATE_T *state)
+{
+	VCHIQ_SHARED_STATE_T *local = state->local;
+	BITSET_T service_found[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
+	int slot_queue_available;
+
+	/* Use a read memory barrier to ensure that any state that may have
+	** been modified by another thread is not masked by stale prefetched
+	** values. */
+	rmb();
+
+	/* Find slots which have been freed by the other side, and return them
+	** to the available queue. */
+	slot_queue_available = state->slot_queue_available;
+
+	while (slot_queue_available != local->slot_queue_recycle) {
+		unsigned int pos;
+		int slot_index = local->slot_queue[slot_queue_available++ &
+			VCHIQ_SLOT_QUEUE_MASK];
+		char *data = (char *)SLOT_DATA_FROM_INDEX(state, slot_index);
+		int data_found = 0;
+
+		vchiq_log_trace(vchiq_core_log_level, "%d: pfq %d=%x %x %x",
+			state->id, slot_index, (unsigned int)data,
+			local->slot_queue_recycle, slot_queue_available);
+
+		/* Initialise the bitmask for services which have used this
+		** slot */
+		BITSET_ZERO(service_found);
+
+		pos = 0;
+
+		while (pos < VCHIQ_SLOT_SIZE) {
+			VCHIQ_HEADER_T *header =
+				(VCHIQ_HEADER_T *)(data + pos);
+			int msgid = header->msgid;
+			if (VCHIQ_MSG_TYPE(msgid) == VCHIQ_MSG_DATA) {
+				int port = VCHIQ_MSG_SRCPORT(msgid);
+				VCHIQ_SERVICE_QUOTA_T *service_quota =
+					&state->service_quotas[port];
+				int count;
+				spin_lock(&quota_spinlock);
+				count = service_quota->message_use_count;
+				if (count > 0)
+					service_quota->message_use_count =
+						count - 1;
+				spin_unlock(&quota_spinlock);
+
+				if (count == service_quota->message_quota)
+					/* Signal the service that it
+					** has dropped below its quota
+					*/
+					up(&service_quota->quota_event);
+				else if (count == 0) {
+					vchiq_log_error(vchiq_core_log_level,
+						"service %d "
+						"message_use_count=%d "
+						"(header %x, msgid %x, "
+						"header->msgid %x, "
+						"header->size %x)",
+						port,
+						service_quota->
+							message_use_count,
+						(unsigned int)header, msgid,
+						header->msgid,
+						header->size);
+					WARN(1, "invalid message use count\n");
+				}
+				if (!BITSET_IS_SET(service_found, port)) {
+					/* Set the found bit for this service */
+					BITSET_SET(service_found, port);
+
+					spin_lock(&quota_spinlock);
+					count = service_quota->slot_use_count;
+					if (count > 0)
+						service_quota->slot_use_count =
+							count - 1;
+					spin_unlock(&quota_spinlock);
+
+					if (count > 0) {
+						/* Signal the service in case
+						** it has dropped below its
+						** quota */
+						up(&service_quota->quota_event);
+						vchiq_log_trace(
+							vchiq_core_log_level,
+							"%d: pfq:%d %x@%x - "
+							"slot_use->%d",
+							state->id, port,
+							header->size,
+							(unsigned int)header,
+							count - 1);
+					} else {
+						vchiq_log_error(
+							vchiq_core_log_level,
+								"service %d "
+								"slot_use_count"
+								"=%d (header %x"
+								", msgid %x, "
+								"header->msgid"
+								" %x, header->"
+								"size %x)",
+							port, count,
+							(unsigned int)header,
+							msgid,
+							header->msgid,
+							header->size);
+						WARN(1, "bad slot use count\n");
+					}
+				}
+
+				data_found = 1;
+			}
+
+			pos += calc_stride(header->size);
+			if (pos > VCHIQ_SLOT_SIZE) {
+				vchiq_log_error(vchiq_core_log_level,
+					"pfq - pos %x: header %x, msgid %x, "
+					"header->msgid %x, header->size %x",
+					pos, (unsigned int)header, msgid,
+					header->msgid, header->size);
+				WARN(1, "invalid slot position\n");
+			}
+		}
+
+		if (data_found) {
+			int count;
+			spin_lock(&quota_spinlock);
+			count = state->data_use_count;
+			if (count > 0)
+				state->data_use_count =
+					count - 1;
+			spin_unlock(&quota_spinlock);
+			if (count == state->data_quota)
+				up(&state->data_quota_event);
+		}
+
+		state->slot_queue_available = slot_queue_available;
+		up(&state->slot_available_event);
+	}
+}
+
+/* Called by the slot handler and application threads */
+static VCHIQ_STATUS_T
+queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
+	int msgid, const VCHIQ_ELEMENT_T *elements,
+	int count, int size, int flags)
+{
+	VCHIQ_SHARED_STATE_T *local;
+	VCHIQ_SERVICE_QUOTA_T *service_quota = NULL;
+	VCHIQ_HEADER_T *header;
+	int type = VCHIQ_MSG_TYPE(msgid);
+
+	unsigned int stride;
+
+	local = state->local;
+
+	stride = calc_stride(size);
+
+	WARN_ON(!(stride <= VCHIQ_SLOT_SIZE));
+
+	if (!(flags & QMFLAGS_NO_MUTEX_LOCK) &&
+		(mutex_lock_interruptible(&state->slot_mutex) != 0))
+		return VCHIQ_RETRY;
+
+	if (type == VCHIQ_MSG_DATA) {
+		int tx_end_index;
+
+		BUG_ON(!service);
+		BUG_ON((flags & (QMFLAGS_NO_MUTEX_LOCK |
+				 QMFLAGS_NO_MUTEX_UNLOCK)) != 0);
+
+		if (service->closing) {
+			/* The service has been closed */
+			mutex_unlock(&state->slot_mutex);
+			return VCHIQ_ERROR;
+		}
+
+		service_quota = &state->service_quotas[service->localport];
+
+		spin_lock(&quota_spinlock);
+
+		/* Ensure this service doesn't use more than its quota of
+		** messages or slots */
+		tx_end_index = SLOT_QUEUE_INDEX_FROM_POS(
+			state->local_tx_pos + stride - 1);
+
+		/* Ensure data messages don't use more than their quota of
+		** slots */
+		while ((tx_end_index != state->previous_data_index) &&
+			(state->data_use_count == state->data_quota)) {
+			VCHIQ_STATS_INC(state, data_stalls);
+			spin_unlock(&quota_spinlock);
+			mutex_unlock(&state->slot_mutex);
+
+			if (down_interruptible(&state->data_quota_event)
+				!= 0)
+				return VCHIQ_RETRY;
+
+			mutex_lock(&state->slot_mutex);
+			spin_lock(&quota_spinlock);
+			tx_end_index = SLOT_QUEUE_INDEX_FROM_POS(
+				state->local_tx_pos + stride - 1);
+			if ((tx_end_index == state->previous_data_index) ||
+				(state->data_use_count < state->data_quota)) {
+				/* Pass the signal on to other waiters */
+				up(&state->data_quota_event);
+				break;
+			}
+		}
+
+		while ((service_quota->message_use_count ==
+				service_quota->message_quota) ||
+			((tx_end_index != service_quota->previous_tx_index) &&
+			(service_quota->slot_use_count ==
+				service_quota->slot_quota))) {
+			spin_unlock(&quota_spinlock);
+			vchiq_log_trace(vchiq_core_log_level,
+				"%d: qm:%d %s,%x - quota stall "
+				"(msg %d, slot %d)",
+				state->id, service->localport,
+				msg_type_str(type), size,
+				service_quota->message_use_count,
+				service_quota->slot_use_count);
+			VCHIQ_SERVICE_STATS_INC(service, quota_stalls);
+			mutex_unlock(&state->slot_mutex);
+			if (down_interruptible(&service_quota->quota_event)
+				!= 0)
+				return VCHIQ_RETRY;
+			if (service->closing)
+				return VCHIQ_ERROR;
+			if (mutex_lock_interruptible(&state->slot_mutex) != 0)
+				return VCHIQ_RETRY;
+			if (service->srvstate != VCHIQ_SRVSTATE_OPEN) {
+				/* The service has been closed */
+				mutex_unlock(&state->slot_mutex);
+				return VCHIQ_ERROR;
+			}
+			spin_lock(&quota_spinlock);
+			tx_end_index = SLOT_QUEUE_INDEX_FROM_POS(
+				state->local_tx_pos + stride - 1);
+		}
+
+		spin_unlock(&quota_spinlock);
+	}
+
+	header = reserve_space(state, stride, flags & QMFLAGS_IS_BLOCKING);
+
+	if (!header) {
+		if (service)
+			VCHIQ_SERVICE_STATS_INC(service, slot_stalls);
+		/* In the event of a failure, return the mutex to the
+		   state it was in */
+		if (!(flags & QMFLAGS_NO_MUTEX_LOCK))
+			mutex_unlock(&state->slot_mutex);
+		return VCHIQ_RETRY;
+	}
+
+	if (type == VCHIQ_MSG_DATA) {
+		int i, pos;
+		int tx_end_index;
+		int slot_use_count;
+
+		vchiq_log_info(vchiq_core_log_level,
+			"%d: qm %s@%x,%x (%d->%d)",
+			state->id,
+			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
+			(unsigned int)header, size,
+			VCHIQ_MSG_SRCPORT(msgid),
+			VCHIQ_MSG_DSTPORT(msgid));
+
+		BUG_ON(!service);
+		BUG_ON((flags & (QMFLAGS_NO_MUTEX_LOCK |
+				 QMFLAGS_NO_MUTEX_UNLOCK)) != 0);
+
+		for (i = 0, pos = 0; i < (unsigned int)count;
+			pos += elements[i++].size)
+			if (elements[i].size) {
+				if (vchiq_copy_from_user
+					(header->data + pos, elements[i].data,
+					(size_t) elements[i].size) !=
+					VCHIQ_SUCCESS) {
+					mutex_unlock(&state->slot_mutex);
+					VCHIQ_SERVICE_STATS_INC(service,
+						error_count);
+					return VCHIQ_ERROR;
+				}
+				if (i == 0) {
+					if (SRVTRACE_ENABLED(service,
+							VCHIQ_LOG_INFO))
+						vchiq_log_dump_mem("Sent", 0,
+							header->data + pos,
+							min(64u,
+							elements[0].size));
+				}
+			}
+
+		spin_lock(&quota_spinlock);
+		service_quota->message_use_count++;
+
+		tx_end_index =
+			SLOT_QUEUE_INDEX_FROM_POS(state->local_tx_pos - 1);
+
+		/* If this transmission can't fit in the last slot used by any
+		** service, the data_use_count must be increased. */
+		if (tx_end_index != state->previous_data_index) {
+			state->previous_data_index = tx_end_index;
+			state->data_use_count++;
+		}
+
+		/* If this isn't the same slot last used by this service,
+		** the service's slot_use_count must be increased. */
+		if (tx_end_index != service_quota->previous_tx_index) {
+			service_quota->previous_tx_index = tx_end_index;
+			slot_use_count = ++service_quota->slot_use_count;
+		} else {
+			slot_use_count = 0;
+		}
+
+		spin_unlock(&quota_spinlock);
+
+		if (slot_use_count)
+			vchiq_log_trace(vchiq_core_log_level,
+				"%d: qm:%d %s,%x - slot_use->%d (hdr %p)",
+				state->id, service->localport,
+				msg_type_str(VCHIQ_MSG_TYPE(msgid)), size,
+				slot_use_count, header);
+
+		VCHIQ_SERVICE_STATS_INC(service, ctrl_tx_count);
+		VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size);
+	} else {
+		vchiq_log_info(vchiq_core_log_level,
+			"%d: qm %s@%x,%x (%d->%d)", state->id,
+			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
+			(unsigned int)header, size,
+			VCHIQ_MSG_SRCPORT(msgid),
+			VCHIQ_MSG_DSTPORT(msgid));
+		if (size != 0) {
+			WARN_ON(!((count == 1) && (size == elements[0].size)));
+			memcpy(header->data, elements[0].data,
+				elements[0].size);
+		}
+		VCHIQ_STATS_INC(state, ctrl_tx_count);
+	}
+
+	header->msgid = msgid;
+	header->size = size;
+
+	{
+		int svc_fourcc;
+
+		svc_fourcc = service
+			? service->base.fourcc
+			: VCHIQ_MAKE_FOURCC('?', '?', '?', '?');
+
+		vchiq_log_info(SRVTRACE_LEVEL(service),
+			"Sent Msg %s(%u) to %c%c%c%c s:%u d:%d len:%d",
+			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
+			VCHIQ_MSG_TYPE(msgid),
+			VCHIQ_FOURCC_AS_4CHARS(svc_fourcc),
+			VCHIQ_MSG_SRCPORT(msgid),
+			VCHIQ_MSG_DSTPORT(msgid),
+			size);
+	}
+
+	/* Make sure the new header is visible to the peer. */
+	wmb();
+
+	/* Make the new tx_pos visible to the peer. */
+	local->tx_pos = state->local_tx_pos;
+	wmb();
+
+	if (service && (type == VCHIQ_MSG_CLOSE))
+		vchiq_set_service_state(service, VCHIQ_SRVSTATE_CLOSESENT);
+
+	if (!(flags & QMFLAGS_NO_MUTEX_UNLOCK))
+		mutex_unlock(&state->slot_mutex);
+
+	remote_event_signal(&state->remote->trigger);
+
+	return VCHIQ_SUCCESS;
+}
+
+/* Called by the slot handler and application threads */
+static VCHIQ_STATUS_T
+queue_message_sync(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
+	int msgid, const VCHIQ_ELEMENT_T *elements,
+	int count, int size, int is_blocking)
+{
+	VCHIQ_SHARED_STATE_T *local;
+	VCHIQ_HEADER_T *header;
+
+	local = state->local;
+
+	if ((VCHIQ_MSG_TYPE(msgid) != VCHIQ_MSG_RESUME) &&
+		(mutex_lock_interruptible(&state->sync_mutex) != 0))
+		return VCHIQ_RETRY;
+
+	remote_event_wait(&local->sync_release);
+
+	rmb();
+
+	header = (VCHIQ_HEADER_T *)SLOT_DATA_FROM_INDEX(state,
+		local->slot_sync);
+
+	{
+		int oldmsgid = header->msgid;
+		if (oldmsgid != VCHIQ_MSGID_PADDING)
+			vchiq_log_error(vchiq_core_log_level,
+				"%d: qms - msgid %x, not PADDING",
+				state->id, oldmsgid);
+	}
+
+	if (service) {
+		int i, pos;
+
+		vchiq_log_info(vchiq_sync_log_level,
+			"%d: qms %s@%x,%x (%d->%d)", state->id,
+			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
+			(unsigned int)header, size,
+			VCHIQ_MSG_SRCPORT(msgid),
+			VCHIQ_MSG_DSTPORT(msgid));
+
+		for (i = 0, pos = 0; i < (unsigned int)count;
+			pos += elements[i++].size)
+			if (elements[i].size) {
+				if (vchiq_copy_from_user
+					(header->data + pos, elements[i].data,
+					(size_t) elements[i].size) !=
+					VCHIQ_SUCCESS) {
+					mutex_unlock(&state->sync_mutex);
+					VCHIQ_SERVICE_STATS_INC(service,
+						error_count);
+					return VCHIQ_ERROR;
+				}
+				if (i == 0) {
+					if (vchiq_sync_log_level >=
+						VCHIQ_LOG_TRACE)
+						vchiq_log_dump_mem("Sent Sync",
+							0, header->data + pos,
+							min(64u,
+							elements[0].size));
+				}
+			}
+
+		VCHIQ_SERVICE_STATS_INC(service, ctrl_tx_count);
+		VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size);
+	} else {
+		vchiq_log_info(vchiq_sync_log_level,
+			"%d: qms %s@%x,%x (%d->%d)", state->id,
+			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
+			(unsigned int)header, size,
+			VCHIQ_MSG_SRCPORT(msgid),
+			VCHIQ_MSG_DSTPORT(msgid));
+		if (size != 0) {
+			WARN_ON(!((count == 1) && (size == elements[0].size)));
+			memcpy(header->data, elements[0].data,
+				elements[0].size);
+		}
+		VCHIQ_STATS_INC(state, ctrl_tx_count);
+	}
+
+	header->size = size;
+	header->msgid = msgid;
+
+	if (vchiq_sync_log_level >= VCHIQ_LOG_TRACE) {
+		int svc_fourcc;
+
+		svc_fourcc = service
+			? service->base.fourcc
+			: VCHIQ_MAKE_FOURCC('?', '?', '?', '?');
+
+		vchiq_log_trace(vchiq_sync_log_level,
+			"Sent Sync Msg %s(%u) to %c%c%c%c s:%u d:%d len:%d",
+			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
+			VCHIQ_MSG_TYPE(msgid),
+			VCHIQ_FOURCC_AS_4CHARS(svc_fourcc),
+			VCHIQ_MSG_SRCPORT(msgid),
+			VCHIQ_MSG_DSTPORT(msgid),
+			size);
+	}
+
+	/* Make sure the new header is visible to the peer. */
+	wmb();
+
+	remote_event_signal(&state->remote->sync_trigger);
+
+	if (VCHIQ_MSG_TYPE(msgid) != VCHIQ_MSG_PAUSE)
+		mutex_unlock(&state->sync_mutex);
+
+	return VCHIQ_SUCCESS;
+}
+
+static inline void
+claim_slot(VCHIQ_SLOT_INFO_T *slot)
+{
+	slot->use_count++;
+}
+
+static void
+release_slot(VCHIQ_STATE_T *state, VCHIQ_SLOT_INFO_T *slot_info,
+	VCHIQ_HEADER_T *header, VCHIQ_SERVICE_T *service)
+{
+	int release_count;
+
+	mutex_lock(&state->recycle_mutex);
+
+	if (header) {
+		int msgid = header->msgid;
+		if (((msgid & VCHIQ_MSGID_CLAIMED) == 0) ||
+			(service && service->closing)) {
+			mutex_unlock(&state->recycle_mutex);
+			return;
+		}
+
+		/* Rewrite the message header to prevent a double
+		** release */
+		header->msgid = msgid & ~VCHIQ_MSGID_CLAIMED;
+	}
+
+	release_count = slot_info->release_count;
+	slot_info->release_count = ++release_count;
+
+	if (release_count == slot_info->use_count) {
+		int slot_queue_recycle;
+		/* Add to the freed queue */
+
+		/* A read barrier is necessary here to prevent speculative
+		** fetches of remote->slot_queue_recycle from overtaking the
+		** mutex. */
+		rmb();
+
+		slot_queue_recycle = state->remote->slot_queue_recycle;
+		state->remote->slot_queue[slot_queue_recycle &
+			VCHIQ_SLOT_QUEUE_MASK] =
+			SLOT_INDEX_FROM_INFO(state, slot_info);
+		state->remote->slot_queue_recycle = slot_queue_recycle + 1;
+		vchiq_log_info(vchiq_core_log_level,
+			"%d: release_slot %d - recycle->%x",
+			state->id, SLOT_INDEX_FROM_INFO(state, slot_info),
+			state->remote->slot_queue_recycle);
+
+		/* A write barrier is necessary, but remote_event_signal
+		** contains one. */
+		remote_event_signal(&state->remote->recycle);
+	}
+
+	mutex_unlock(&state->recycle_mutex);
+}
+
+/* Called by the slot handler - don't hold the bulk mutex */
+static VCHIQ_STATUS_T
+notify_bulks(VCHIQ_SERVICE_T *service, VCHIQ_BULK_QUEUE_T *queue,
+	int retry_poll)
+{
+	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+
+	vchiq_log_trace(vchiq_core_log_level,
+		"%d: nb:%d %cx - p=%x rn=%x r=%x",
+		service->state->id, service->localport,
+		(queue == &service->bulk_tx) ? 't' : 'r',
+		queue->process, queue->remote_notify, queue->remove);
+
+	if (service->state->is_master) {
+		while (queue->remote_notify != queue->process) {
+			VCHIQ_BULK_T *bulk =
+				&queue->bulks[BULK_INDEX(queue->remote_notify)];
+			int msgtype = (bulk->dir == VCHIQ_BULK_TRANSMIT) ?
+				VCHIQ_MSG_BULK_RX_DONE : VCHIQ_MSG_BULK_TX_DONE;
+			int msgid = VCHIQ_MAKE_MSG(msgtype, service->localport,
+				service->remoteport);
+			VCHIQ_ELEMENT_T element = { &bulk->actual, 4 };
+			/* Only reply to non-dummy bulk requests */
+			if (bulk->remote_data) {
+				status = queue_message(service->state, NULL,
+					msgid, &element, 1, 4, 0);
+				if (status != VCHIQ_SUCCESS)
+					break;
+			}
+			queue->remote_notify++;
+		}
+	} else {
+		queue->remote_notify = queue->process;
+	}
+
+	if (status == VCHIQ_SUCCESS) {
+		while (queue->remove != queue->remote_notify) {
+			VCHIQ_BULK_T *bulk =
+				&queue->bulks[BULK_INDEX(queue->remove)];
+
+			/* Only generate callbacks for non-dummy bulk
+			** requests, and non-terminated services */
+			if (bulk->data && service->instance) {
+				if (bulk->actual != VCHIQ_BULK_ACTUAL_ABORTED) {
+					if (bulk->dir == VCHIQ_BULK_TRANSMIT) {
+						VCHIQ_SERVICE_STATS_INC(service,
+							bulk_tx_count);
+						VCHIQ_SERVICE_STATS_ADD(service,
+							bulk_tx_bytes,
+							bulk->actual);
+					} else {
+						VCHIQ_SERVICE_STATS_INC(service,
+							bulk_rx_count);
+						VCHIQ_SERVICE_STATS_ADD(service,
+							bulk_rx_bytes,
+							bulk->actual);
+					}
+				} else {
+					VCHIQ_SERVICE_STATS_INC(service,
+						bulk_aborted_count);
+				}
+				if (bulk->mode == VCHIQ_BULK_MODE_BLOCKING) {
+					struct bulk_waiter *waiter;
+					spin_lock(&bulk_waiter_spinlock);
+					waiter = bulk->userdata;
+					if (waiter) {
+						waiter->actual = bulk->actual;
+						up(&waiter->event);
+					}
+					spin_unlock(&bulk_waiter_spinlock);
+				} else if (bulk->mode ==
+					VCHIQ_BULK_MODE_CALLBACK) {
+					VCHIQ_REASON_T reason = (bulk->dir ==
+						VCHIQ_BULK_TRANSMIT) ?
+						((bulk->actual ==
+						VCHIQ_BULK_ACTUAL_ABORTED) ?
+						VCHIQ_BULK_TRANSMIT_ABORTED :
+						VCHIQ_BULK_TRANSMIT_DONE) :
+						((bulk->actual ==
+						VCHIQ_BULK_ACTUAL_ABORTED) ?
+						VCHIQ_BULK_RECEIVE_ABORTED :
+						VCHIQ_BULK_RECEIVE_DONE);
+					status = make_service_callback(service,
+						reason,	NULL, bulk->userdata);
+					if (status == VCHIQ_RETRY)
+						break;
+				}
+			}
+
+			queue->remove++;
+			up(&service->bulk_remove_event);
+		}
+		if (!retry_poll)
+			status = VCHIQ_SUCCESS;
+	}
+
+	if (status == VCHIQ_RETRY)
+		request_poll(service->state, service,
+			(queue == &service->bulk_tx) ?
+			VCHIQ_POLL_TXNOTIFY : VCHIQ_POLL_RXNOTIFY);
+
+	return status;
+}
+
+/* Called by the slot handler thread */
+static void
+poll_services(VCHIQ_STATE_T *state)
+{
+	int group, i;
+
+	for (group = 0; group < BITSET_SIZE(state->unused_service); group++) {
+		uint32_t flags;
+		flags = atomic_xchg(&state->poll_services[group], 0);
+		for (i = 0; flags; i++) {
+			if (flags & (1 << i)) {
+				VCHIQ_SERVICE_T *service =
+					find_service_by_port(state,
+						(group<<5) + i);
+				uint32_t service_flags;
+				flags &= ~(1 << i);
+				if (!service)
+					continue;
+				service_flags =
+					atomic_xchg(&service->poll_flags, 0);
+				if (service_flags &
+					(1 << VCHIQ_POLL_REMOVE)) {
+					vchiq_log_info(vchiq_core_log_level,
+						"%d: ps - remove %d<->%d",
+						state->id, service->localport,
+						service->remoteport);
+
+					/* Make it look like a client, because
+					   it must be removed and not left in
+					   the LISTENING state. */
+					service->public_fourcc =
+						VCHIQ_FOURCC_INVALID;
+
+					if (vchiq_close_service_internal(
+						service, 0/*!close_recvd*/) !=
+						VCHIQ_SUCCESS)
+						request_poll(state, service,
+							VCHIQ_POLL_REMOVE);
+				} else if (service_flags &
+					(1 << VCHIQ_POLL_TERMINATE)) {
+					vchiq_log_info(vchiq_core_log_level,
+						"%d: ps - terminate %d<->%d",
+						state->id, service->localport,
+						service->remoteport);
+					if (vchiq_close_service_internal(
+						service, 0/*!close_recvd*/) !=
+						VCHIQ_SUCCESS)
+						request_poll(state, service,
+							VCHIQ_POLL_TERMINATE);
+				}
+				if (service_flags & (1 << VCHIQ_POLL_TXNOTIFY))
+					notify_bulks(service,
+						&service->bulk_tx,
+						1/*retry_poll*/);
+				if (service_flags & (1 << VCHIQ_POLL_RXNOTIFY))
+					notify_bulks(service,
+						&service->bulk_rx,
+						1/*retry_poll*/);
+				unlock_service(service);
+			}
+		}
+	}
+}
+
+/* Called by the slot handler or application threads, holding the bulk mutex. */
+static int
+resolve_bulks(VCHIQ_SERVICE_T *service, VCHIQ_BULK_QUEUE_T *queue)
+{
+	VCHIQ_STATE_T *state = service->state;
+	int resolved = 0;
+	int rc;
+
+	while ((queue->process != queue->local_insert) &&
+		(queue->process != queue->remote_insert)) {
+		VCHIQ_BULK_T *bulk = &queue->bulks[BULK_INDEX(queue->process)];
+
+		vchiq_log_trace(vchiq_core_log_level,
+			"%d: rb:%d %cx - li=%x ri=%x p=%x",
+			state->id, service->localport,
+			(queue == &service->bulk_tx) ? 't' : 'r',
+			queue->local_insert, queue->remote_insert,
+			queue->process);
+
+		WARN_ON(!((int)(queue->local_insert - queue->process) > 0));
+		WARN_ON(!((int)(queue->remote_insert - queue->process) > 0));
+
+		rc = mutex_lock_interruptible(&state->bulk_transfer_mutex);
+		if (rc != 0)
+			break;
+
+		vchiq_transfer_bulk(bulk);
+		mutex_unlock(&state->bulk_transfer_mutex);
+
+		if (SRVTRACE_ENABLED(service, VCHIQ_LOG_INFO)) {
+			const char *header = (queue == &service->bulk_tx) ?
+				"Send Bulk to" : "Recv Bulk from";
+			if (bulk->actual != VCHIQ_BULK_ACTUAL_ABORTED)
+				vchiq_log_info(SRVTRACE_LEVEL(service),
+					"%s %c%c%c%c d:%d len:%d %x<->%x",
+					header,
+					VCHIQ_FOURCC_AS_4CHARS(
+						service->base.fourcc),
+					service->remoteport,
+					bulk->size,
+					(unsigned int)bulk->data,
+					(unsigned int)bulk->remote_data);
+			else
+				vchiq_log_info(SRVTRACE_LEVEL(service),
+					"%s %c%c%c%c d:%d ABORTED - tx len:%d,"
+					" rx len:%d %x<->%x",
+					header,
+					VCHIQ_FOURCC_AS_4CHARS(
+						service->base.fourcc),
+					service->remoteport,
+					bulk->size,
+					bulk->remote_size,
+					(unsigned int)bulk->data,
+					(unsigned int)bulk->remote_data);
+		}
+
+		vchiq_complete_bulk(bulk);
+		queue->process++;
+		resolved++;
+	}
+	return resolved;
+}
+
+/* Called with the bulk_mutex held */
+static void
+abort_outstanding_bulks(VCHIQ_SERVICE_T *service, VCHIQ_BULK_QUEUE_T *queue)
+{
+	int is_tx = (queue == &service->bulk_tx);
+	vchiq_log_trace(vchiq_core_log_level,
+		"%d: aob:%d %cx - li=%x ri=%x p=%x",
+		service->state->id, service->localport, is_tx ? 't' : 'r',
+		queue->local_insert, queue->remote_insert, queue->process);
+
+	WARN_ON(!((int)(queue->local_insert - queue->process) >= 0));
+	WARN_ON(!((int)(queue->remote_insert - queue->process) >= 0));
+
+	while ((queue->process != queue->local_insert) ||
+		(queue->process != queue->remote_insert)) {
+		VCHIQ_BULK_T *bulk = &queue->bulks[BULK_INDEX(queue->process)];
+
+		if (queue->process == queue->remote_insert) {
+			/* fabricate a matching dummy bulk */
+			bulk->remote_data = NULL;
+			bulk->remote_size = 0;
+			queue->remote_insert++;
+		}
+
+		if (queue->process != queue->local_insert) {
+			vchiq_complete_bulk(bulk);
+
+			vchiq_log_info(SRVTRACE_LEVEL(service),
+				"%s %c%c%c%c d:%d ABORTED - tx len:%d, "
+				"rx len:%d",
+				is_tx ? "Send Bulk to" : "Recv Bulk from",
+				VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
+				service->remoteport,
+				bulk->size,
+				bulk->remote_size);
+		} else {
+			/* fabricate a matching dummy bulk */
+			bulk->data = NULL;
+			bulk->size = 0;
+			bulk->actual = VCHIQ_BULK_ACTUAL_ABORTED;
+			bulk->dir = is_tx ? VCHIQ_BULK_TRANSMIT :
+				VCHIQ_BULK_RECEIVE;
+			queue->local_insert++;
+		}
+
+		queue->process++;
+	}
+}
+
+/* Called from the slot handler thread */
+static void
+pause_bulks(VCHIQ_STATE_T *state)
+{
+	if (unlikely(atomic_inc_return(&pause_bulks_count) != 1)) {
+		WARN_ON_ONCE(1);
+		atomic_set(&pause_bulks_count, 1);
+		return;
+	}
+
+	/* Block bulk transfers from all services */
+	mutex_lock(&state->bulk_transfer_mutex);
+}
+
+/* Called from the slot handler thread */
+static void
+resume_bulks(VCHIQ_STATE_T *state)
+{
+	int i;
+	if (unlikely(atomic_dec_return(&pause_bulks_count) != 0)) {
+		WARN_ON_ONCE(1);
+		atomic_set(&pause_bulks_count, 0);
+		return;
+	}
+
+	/* Allow bulk transfers from all services */
+	mutex_unlock(&state->bulk_transfer_mutex);
+
+	if (state->deferred_bulks == 0)
+		return;
+
+	/* Deal with any bulks which had to be deferred due to being in
+	 * paused state.  Don't try to match up to number of deferred bulks
+	 * in case we've had something come and close the service in the
+	 * interim - just process all bulk queues for all services */
+	vchiq_log_info(vchiq_core_log_level, "%s: processing %d deferred bulks",
+		__func__, state->deferred_bulks);
+
+	for (i = 0; i < state->unused_service; i++) {
+		VCHIQ_SERVICE_T *service = state->services[i];
+		int resolved_rx = 0;
+		int resolved_tx = 0;
+		if (!service || (service->srvstate != VCHIQ_SRVSTATE_OPEN))
+			continue;
+
+		mutex_lock(&service->bulk_mutex);
+		resolved_rx = resolve_bulks(service, &service->bulk_rx);
+		resolved_tx = resolve_bulks(service, &service->bulk_tx);
+		mutex_unlock(&service->bulk_mutex);
+		if (resolved_rx)
+			notify_bulks(service, &service->bulk_rx, 1);
+		if (resolved_tx)
+			notify_bulks(service, &service->bulk_tx, 1);
+	}
+	state->deferred_bulks = 0;
+}
+
+static int
+parse_open(VCHIQ_STATE_T *state, VCHIQ_HEADER_T *header)
+{
+	VCHIQ_SERVICE_T *service = NULL;
+	int msgid, size;
+	int type;
+	unsigned int localport, remoteport;
+
+	msgid = header->msgid;
+	size = header->size;
+	type = VCHIQ_MSG_TYPE(msgid);
+	localport = VCHIQ_MSG_DSTPORT(msgid);
+	remoteport = VCHIQ_MSG_SRCPORT(msgid);
+	if (size >= sizeof(struct vchiq_open_payload)) {
+		const struct vchiq_open_payload *payload =
+			(struct vchiq_open_payload *)header->data;
+		unsigned int fourcc;
+
+		fourcc = payload->fourcc;
+		vchiq_log_info(vchiq_core_log_level,
+			"%d: prs OPEN@%x (%d->'%c%c%c%c')",
+			state->id, (unsigned int)header,
+			localport,
+			VCHIQ_FOURCC_AS_4CHARS(fourcc));
+
+		service = get_listening_service(state, fourcc);
+
+		if (service) {
+			/* A matching service exists */
+			short version = payload->version;
+			short version_min = payload->version_min;
+			if ((service->version < version_min) ||
+				(version < service->version_min)) {
+				/* Version mismatch */
+				vchiq_loud_error_header();
+				vchiq_loud_error("%d: service %d (%c%c%c%c) "
+					"version mismatch - local (%d, min %d)"
+					" vs. remote (%d, min %d)",
+					state->id, service->localport,
+					VCHIQ_FOURCC_AS_4CHARS(fourcc),
+					service->version, service->version_min,
+					version, version_min);
+				vchiq_loud_error_footer();
+				unlock_service(service);
+				service = NULL;
+				goto fail_open;
+			}
+			service->peer_version = version;
+
+			if (service->srvstate == VCHIQ_SRVSTATE_LISTENING) {
+				struct vchiq_openack_payload ack_payload = {
+					service->version
+				};
+				VCHIQ_ELEMENT_T body = {
+					&ack_payload,
+					sizeof(ack_payload)
+				};
+
+				if (state->version_common <
+				    VCHIQ_VERSION_SYNCHRONOUS_MODE)
+					service->sync = 0;
+
+				/* Acknowledge the OPEN */
+				if (service->sync &&
+				    (state->version_common >=
+				     VCHIQ_VERSION_SYNCHRONOUS_MODE)) {
+					if (queue_message_sync(state, NULL,
+						VCHIQ_MAKE_MSG(
+							VCHIQ_MSG_OPENACK,
+							service->localport,
+							remoteport),
+						&body, 1, sizeof(ack_payload),
+						0) == VCHIQ_RETRY)
+						goto bail_not_ready;
+				} else {
+					if (queue_message(state, NULL,
+						VCHIQ_MAKE_MSG(
+							VCHIQ_MSG_OPENACK,
+							service->localport,
+							remoteport),
+						&body, 1, sizeof(ack_payload),
+						0) == VCHIQ_RETRY)
+						goto bail_not_ready;
+				}
+
+				/* The service is now open */
+				vchiq_set_service_state(service,
+					service->sync ? VCHIQ_SRVSTATE_OPENSYNC
+					: VCHIQ_SRVSTATE_OPEN);
+			}
+
+			service->remoteport = remoteport;
+			service->client_id = ((int *)header->data)[1];
+			if (make_service_callback(service, VCHIQ_SERVICE_OPENED,
+				NULL, NULL) == VCHIQ_RETRY) {
+				/* Bail out if not ready */
+				service->remoteport = VCHIQ_PORT_FREE;
+				goto bail_not_ready;
+			}
+
+			/* Success - the message has been dealt with */
+			unlock_service(service);
+			return 1;
+		}
+	}
+
+fail_open:
+	/* No available service, or an invalid request - send a CLOSE */
+	if (queue_message(state, NULL,
+		VCHIQ_MAKE_MSG(VCHIQ_MSG_CLOSE, 0, VCHIQ_MSG_SRCPORT(msgid)),
+		NULL, 0, 0, 0) == VCHIQ_RETRY)
+		goto bail_not_ready;
+
+	return 1;
+
+bail_not_ready:
+	if (service)
+		unlock_service(service);
+
+	return 0;
+}
+
+/* Called by the slot handler thread */
+static void
+parse_rx_slots(VCHIQ_STATE_T *state)
+{
+	VCHIQ_SHARED_STATE_T *remote = state->remote;
+	VCHIQ_SERVICE_T *service = NULL;
+	int tx_pos;
+	DEBUG_INITIALISE(state->local)
+
+	tx_pos = remote->tx_pos;
+
+	while (state->rx_pos != tx_pos) {
+		VCHIQ_HEADER_T *header;
+		int msgid, size;
+		int type;
+		unsigned int localport, remoteport;
+
+		DEBUG_TRACE(PARSE_LINE);
+		if (!state->rx_data) {
+			int rx_index;
+			WARN_ON(!((state->rx_pos & VCHIQ_SLOT_MASK) == 0));
+			rx_index = remote->slot_queue[
+				SLOT_QUEUE_INDEX_FROM_POS(state->rx_pos) &
+				VCHIQ_SLOT_QUEUE_MASK];
+			state->rx_data = (char *)SLOT_DATA_FROM_INDEX(state,
+				rx_index);
+			state->rx_info = SLOT_INFO_FROM_INDEX(state, rx_index);
+
+			/* Initialise use_count to one, and increment
+			** release_count at the end of the slot to avoid
+			** releasing the slot prematurely. */
+			state->rx_info->use_count = 1;
+			state->rx_info->release_count = 0;
+		}
+
+		header = (VCHIQ_HEADER_T *)(state->rx_data +
+			(state->rx_pos & VCHIQ_SLOT_MASK));
+		DEBUG_VALUE(PARSE_HEADER, (int)header);
+		msgid = header->msgid;
+		DEBUG_VALUE(PARSE_MSGID, msgid);
+		size = header->size;
+		type = VCHIQ_MSG_TYPE(msgid);
+		localport = VCHIQ_MSG_DSTPORT(msgid);
+		remoteport = VCHIQ_MSG_SRCPORT(msgid);
+
+		if (type != VCHIQ_MSG_DATA)
+			VCHIQ_STATS_INC(state, ctrl_rx_count);
+
+		switch (type) {
+		case VCHIQ_MSG_OPENACK:
+		case VCHIQ_MSG_CLOSE:
+		case VCHIQ_MSG_DATA:
+		case VCHIQ_MSG_BULK_RX:
+		case VCHIQ_MSG_BULK_TX:
+		case VCHIQ_MSG_BULK_RX_DONE:
+		case VCHIQ_MSG_BULK_TX_DONE:
+			service = find_service_by_port(state, localport);
+			if ((!service ||
+			     ((service->remoteport != remoteport) &&
+			      (service->remoteport != VCHIQ_PORT_FREE))) &&
+			    (localport == 0) &&
+			    (type == VCHIQ_MSG_CLOSE)) {
+				/* This could be a CLOSE from a client which
+				   hadn't yet received the OPENACK - look for
+				   the connected service */
+				if (service)
+					unlock_service(service);
+				service = get_connected_service(state,
+					remoteport);
+				if (service)
+					vchiq_log_warning(vchiq_core_log_level,
+						"%d: prs %s@%x (%d->%d) - "
+						"found connected service %d",
+						state->id, msg_type_str(type),
+						(unsigned int)header,
+						remoteport, localport,
+						service->localport);
+			}
+
+			if (!service) {
+				vchiq_log_error(vchiq_core_log_level,
+					"%d: prs %s@%x (%d->%d) - "
+					"invalid/closed service %d",
+					state->id, msg_type_str(type),
+					(unsigned int)header,
+					remoteport, localport, localport);
+				goto skip_message;
+			}
+			break;
+		default:
+			break;
+		}
+
+		if (SRVTRACE_ENABLED(service, VCHIQ_LOG_INFO)) {
+			int svc_fourcc;
+
+			svc_fourcc = service
+				? service->base.fourcc
+				: VCHIQ_MAKE_FOURCC('?', '?', '?', '?');
+			vchiq_log_info(SRVTRACE_LEVEL(service),
+				"Rcvd Msg %s(%u) from %c%c%c%c s:%d d:%d "
+				"len:%d",
+				msg_type_str(type), type,
+				VCHIQ_FOURCC_AS_4CHARS(svc_fourcc),
+				remoteport, localport, size);
+			if (size > 0)
+				vchiq_log_dump_mem("Rcvd", 0, header->data,
+					min(64, size));
+		}
+
+		if (((unsigned int)header & VCHIQ_SLOT_MASK) + calc_stride(size)
+			> VCHIQ_SLOT_SIZE) {
+			vchiq_log_error(vchiq_core_log_level,
+				"header %x (msgid %x) - size %x too big for "
+				"slot",
+				(unsigned int)header, (unsigned int)msgid,
+				(unsigned int)size);
+			WARN(1, "oversized for slot\n");
+		}
+
+		switch (type) {
+		case VCHIQ_MSG_OPEN:
+			WARN_ON(!(VCHIQ_MSG_DSTPORT(msgid) == 0));
+			if (!parse_open(state, header))
+				goto bail_not_ready;
+			break;
+		case VCHIQ_MSG_OPENACK:
+			if (size >= sizeof(struct vchiq_openack_payload)) {
+				const struct vchiq_openack_payload *payload =
+					(struct vchiq_openack_payload *)
+					header->data;
+				service->peer_version = payload->version;
+			}
+			vchiq_log_info(vchiq_core_log_level,
+				"%d: prs OPENACK@%x,%x (%d->%d) v:%d",
+				state->id, (unsigned int)header, size,
+				remoteport, localport, service->peer_version);
+			if (service->srvstate ==
+				VCHIQ_SRVSTATE_OPENING) {
+				service->remoteport = remoteport;
+				vchiq_set_service_state(service,
+					VCHIQ_SRVSTATE_OPEN);
+				up(&service->remove_event);
+			} else
+				vchiq_log_error(vchiq_core_log_level,
+					"OPENACK received in state %s",
+					srvstate_names[service->srvstate]);
+			break;
+		case VCHIQ_MSG_CLOSE:
+			WARN_ON(size != 0); /* There should be no data */
+
+			vchiq_log_info(vchiq_core_log_level,
+				"%d: prs CLOSE@%x (%d->%d)",
+				state->id, (unsigned int)header,
+				remoteport, localport);
+
+			mark_service_closing_internal(service, 1);
+
+			if (vchiq_close_service_internal(service,
+				1/*close_recvd*/) == VCHIQ_RETRY)
+				goto bail_not_ready;
+
+			vchiq_log_info(vchiq_core_log_level,
+				"Close Service %c%c%c%c s:%u d:%d",
+				VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc),
+				service->localport,
+				service->remoteport);
+			break;
+		case VCHIQ_MSG_DATA:
+			vchiq_log_info(vchiq_core_log_level,
+				"%d: prs DATA@%x,%x (%d->%d)",
+				state->id, (unsigned int)header, size,
+				remoteport, localport);
+
+			if ((service->remoteport == remoteport)
+				&& (service->srvstate ==
+				VCHIQ_SRVSTATE_OPEN)) {
+				header->msgid = msgid | VCHIQ_MSGID_CLAIMED;
+				claim_slot(state->rx_info);
+				DEBUG_TRACE(PARSE_LINE);
+				if (make_service_callback(service,
+					VCHIQ_MESSAGE_AVAILABLE, header,
+					NULL) == VCHIQ_RETRY) {
+					DEBUG_TRACE(PARSE_LINE);
+					goto bail_not_ready;
+				}
+				VCHIQ_SERVICE_STATS_INC(service, ctrl_rx_count);
+				VCHIQ_SERVICE_STATS_ADD(service, ctrl_rx_bytes,
+					size);
+			} else {
+				VCHIQ_STATS_INC(state, error_count);
+			}
+			break;
+		case VCHIQ_MSG_CONNECT:
+			vchiq_log_info(vchiq_core_log_level,
+				"%d: prs CONNECT@%x",
+				state->id, (unsigned int)header);
+			state->version_common = ((VCHIQ_SLOT_ZERO_T *)
+						 state->slot_data)->version;
+			up(&state->connect);
+			break;
+		case VCHIQ_MSG_BULK_RX:
+		case VCHIQ_MSG_BULK_TX: {
+			VCHIQ_BULK_QUEUE_T *queue;
+			WARN_ON(!state->is_master);
+			queue = (type == VCHIQ_MSG_BULK_RX) ?
+				&service->bulk_tx : &service->bulk_rx;
+			if ((service->remoteport == remoteport)
+				&& (service->srvstate ==
+				VCHIQ_SRVSTATE_OPEN)) {
+				VCHIQ_BULK_T *bulk;
+				int resolved = 0;
+
+				DEBUG_TRACE(PARSE_LINE);
+				if (mutex_lock_interruptible(
+					&service->bulk_mutex) != 0) {
+					DEBUG_TRACE(PARSE_LINE);
+					goto bail_not_ready;
+				}
+
+				WARN_ON(!(queue->remote_insert < queue->remove +
+					VCHIQ_NUM_SERVICE_BULKS));
+				bulk = &queue->bulks[
+					BULK_INDEX(queue->remote_insert)];
+				bulk->remote_data =
+					(void *)((int *)header->data)[0];
+				bulk->remote_size = ((int *)header->data)[1];
+				wmb();
+
+				vchiq_log_info(vchiq_core_log_level,
+					"%d: prs %s@%x (%d->%d) %x@%x",
+					state->id, msg_type_str(type),
+					(unsigned int)header,
+					remoteport, localport,
+					bulk->remote_size,
+					(unsigned int)bulk->remote_data);
+
+				queue->remote_insert++;
+
+				if (atomic_read(&pause_bulks_count)) {
+					state->deferred_bulks++;
+					vchiq_log_info(vchiq_core_log_level,
+						"%s: deferring bulk (%d)",
+						__func__,
+						state->deferred_bulks);
+					if (state->conn_state !=
+						VCHIQ_CONNSTATE_PAUSE_SENT)
+						vchiq_log_error(
+							vchiq_core_log_level,
+							"%s: bulks paused in "
+							"unexpected state %s",
+							__func__,
+							conn_state_names[
+							state->conn_state]);
+				} else if (state->conn_state ==
+					VCHIQ_CONNSTATE_CONNECTED) {
+					DEBUG_TRACE(PARSE_LINE);
+					resolved = resolve_bulks(service,
+						queue);
+				}
+
+				mutex_unlock(&service->bulk_mutex);
+				if (resolved)
+					notify_bulks(service, queue,
+						1/*retry_poll*/);
+			}
+		} break;
+		case VCHIQ_MSG_BULK_RX_DONE:
+		case VCHIQ_MSG_BULK_TX_DONE:
+			WARN_ON(state->is_master);
+			if ((service->remoteport == remoteport)
+				&& (service->srvstate !=
+				VCHIQ_SRVSTATE_FREE)) {
+				VCHIQ_BULK_QUEUE_T *queue;
+				VCHIQ_BULK_T *bulk;
+
+				queue = (type == VCHIQ_MSG_BULK_RX_DONE) ?
+					&service->bulk_rx : &service->bulk_tx;
+
+				DEBUG_TRACE(PARSE_LINE);
+				if (mutex_lock_interruptible(
+					&service->bulk_mutex) != 0) {
+					DEBUG_TRACE(PARSE_LINE);
+					goto bail_not_ready;
+				}
+				if ((int)(queue->remote_insert -
+					queue->local_insert) >= 0) {
+					vchiq_log_error(vchiq_core_log_level,
+						"%d: prs %s@%x (%d->%d) "
+						"unexpected (ri=%d,li=%d)",
+						state->id, msg_type_str(type),
+						(unsigned int)header,
+						remoteport, localport,
+						queue->remote_insert,
+						queue->local_insert);
+					mutex_unlock(&service->bulk_mutex);
+					break;
+				}
+
+				BUG_ON(queue->process == queue->local_insert);
+				BUG_ON(queue->process != queue->remote_insert);
+
+				bulk = &queue->bulks[
+					BULK_INDEX(queue->remote_insert)];
+				bulk->actual = *(int *)header->data;
+				queue->remote_insert++;
+
+				vchiq_log_info(vchiq_core_log_level,
+					"%d: prs %s@%x (%d->%d) %x@%x",
+					state->id, msg_type_str(type),
+					(unsigned int)header,
+					remoteport, localport,
+					bulk->actual, (unsigned int)bulk->data);
+
+				vchiq_log_trace(vchiq_core_log_level,
+					"%d: prs:%d %cx li=%x ri=%x p=%x",
+					state->id, localport,
+					(type == VCHIQ_MSG_BULK_RX_DONE) ?
+						'r' : 't',
+					queue->local_insert,
+					queue->remote_insert, queue->process);
+
+				DEBUG_TRACE(PARSE_LINE);
+				WARN_ON(queue->process == queue->local_insert);
+				vchiq_complete_bulk(bulk);
+				queue->process++;
+				mutex_unlock(&service->bulk_mutex);
+				DEBUG_TRACE(PARSE_LINE);
+				notify_bulks(service, queue, 1/*retry_poll*/);
+				DEBUG_TRACE(PARSE_LINE);
+			}
+			break;
+		case VCHIQ_MSG_PADDING:
+			vchiq_log_trace(vchiq_core_log_level,
+				"%d: prs PADDING@%x,%x",
+				state->id, (unsigned int)header, size);
+			break;
+		case VCHIQ_MSG_PAUSE:
+			/* If initiated, signal the application thread */
+			vchiq_log_trace(vchiq_core_log_level,
+				"%d: prs PAUSE@%x,%x",
+				state->id, (unsigned int)header, size);
+			if (state->conn_state == VCHIQ_CONNSTATE_PAUSED) {
+				vchiq_log_error(vchiq_core_log_level,
+					"%d: PAUSE received in state PAUSED",
+					state->id);
+				break;
+			}
+			if (state->conn_state != VCHIQ_CONNSTATE_PAUSE_SENT) {
+				/* Send a PAUSE in response */
+				if (queue_message(state, NULL,
+					VCHIQ_MAKE_MSG(VCHIQ_MSG_PAUSE, 0, 0),
+					NULL, 0, 0, QMFLAGS_NO_MUTEX_UNLOCK)
+				    == VCHIQ_RETRY)
+					goto bail_not_ready;
+				if (state->is_master)
+					pause_bulks(state);
+			}
+			/* At this point slot_mutex is held */
+			vchiq_set_conn_state(state, VCHIQ_CONNSTATE_PAUSED);
+			vchiq_platform_paused(state);
+			break;
+		case VCHIQ_MSG_RESUME:
+			vchiq_log_trace(vchiq_core_log_level,
+				"%d: prs RESUME@%x,%x",
+				state->id, (unsigned int)header, size);
+			/* Release the slot mutex */
+			mutex_unlock(&state->slot_mutex);
+			if (state->is_master)
+				resume_bulks(state);
+			vchiq_set_conn_state(state, VCHIQ_CONNSTATE_CONNECTED);
+			vchiq_platform_resumed(state);
+			break;
+
+		case VCHIQ_MSG_REMOTE_USE:
+			vchiq_on_remote_use(state);
+			break;
+		case VCHIQ_MSG_REMOTE_RELEASE:
+			vchiq_on_remote_release(state);
+			break;
+		case VCHIQ_MSG_REMOTE_USE_ACTIVE:
+			vchiq_on_remote_use_active(state);
+			break;
+
+		default:
+			vchiq_log_error(vchiq_core_log_level,
+				"%d: prs invalid msgid %x@%x,%x",
+				state->id, msgid, (unsigned int)header, size);
+			WARN(1, "invalid message\n");
+			break;
+		}
+
+skip_message:
+		if (service) {
+			unlock_service(service);
+			service = NULL;
+		}
+
+		state->rx_pos += calc_stride(size);
+
+		DEBUG_TRACE(PARSE_LINE);
+		/* Perform some housekeeping when the end of the slot is
+		** reached. */
+		if ((state->rx_pos & VCHIQ_SLOT_MASK) == 0) {
+			/* Remove the extra reference count. */
+			release_slot(state, state->rx_info, NULL, NULL);
+			state->rx_data = NULL;
+		}
+	}
+
+bail_not_ready:
+	if (service)
+		unlock_service(service);
+}
+
+/* Called by the slot handler thread */
+static int
+slot_handler_func(void *v)
+{
+	VCHIQ_STATE_T *state = (VCHIQ_STATE_T *) v;
+	VCHIQ_SHARED_STATE_T *local = state->local;
+	DEBUG_INITIALISE(local)
+
+	while (1) {
+		DEBUG_COUNT(SLOT_HANDLER_COUNT);
+		DEBUG_TRACE(SLOT_HANDLER_LINE);
+		remote_event_wait(&local->trigger);
+
+		rmb();
+
+		DEBUG_TRACE(SLOT_HANDLER_LINE);
+		if (state->poll_needed) {
+			/* Check if we need to suspend - may change our
+			 * conn_state */
+			vchiq_platform_check_suspend(state);
+
+			state->poll_needed = 0;
+
+			/* Handle service polling and other rare conditions here
+			** out of the mainline code */
+			switch (state->conn_state) {
+			case VCHIQ_CONNSTATE_CONNECTED:
+				/* Poll the services as requested */
+				poll_services(state);
+				break;
+
+			case VCHIQ_CONNSTATE_PAUSING:
+				if (state->is_master)
+					pause_bulks(state);
+				if (queue_message(state, NULL,
+					VCHIQ_MAKE_MSG(VCHIQ_MSG_PAUSE, 0, 0),
+					NULL, 0, 0,
+					QMFLAGS_NO_MUTEX_UNLOCK)
+				    != VCHIQ_RETRY) {
+					vchiq_set_conn_state(state,
+						VCHIQ_CONNSTATE_PAUSE_SENT);
+				} else {
+					if (state->is_master)
+						resume_bulks(state);
+					/* Retry later */
+					state->poll_needed = 1;
+				}
+				break;
+
+			case VCHIQ_CONNSTATE_PAUSED:
+				vchiq_platform_resume(state);
+				break;
+
+			case VCHIQ_CONNSTATE_RESUMING:
+				if (queue_message(state, NULL,
+					VCHIQ_MAKE_MSG(VCHIQ_MSG_RESUME, 0, 0),
+					NULL, 0, 0, QMFLAGS_NO_MUTEX_LOCK)
+					!= VCHIQ_RETRY) {
+					if (state->is_master)
+						resume_bulks(state);
+					vchiq_set_conn_state(state,
+						VCHIQ_CONNSTATE_CONNECTED);
+					vchiq_platform_resumed(state);
+				} else {
+					/* This should really be impossible,
+					** since the PAUSE should have flushed
+					** through outstanding messages. */
+					vchiq_log_error(vchiq_core_log_level,
+						"Failed to send RESUME "
+						"message");
+					BUG();
+				}
+				break;
+
+			case VCHIQ_CONNSTATE_PAUSE_TIMEOUT:
+			case VCHIQ_CONNSTATE_RESUME_TIMEOUT:
+				vchiq_platform_handle_timeout(state);
+				break;
+			default:
+				break;
+			}
+
+
+		}
+
+		DEBUG_TRACE(SLOT_HANDLER_LINE);
+		parse_rx_slots(state);
+	}
+	return 0;
+}
+
+
+/* Called by the recycle thread */
+static int
+recycle_func(void *v)
+{
+	VCHIQ_STATE_T *state = (VCHIQ_STATE_T *) v;
+	VCHIQ_SHARED_STATE_T *local = state->local;
+
+	while (1) {
+		remote_event_wait(&local->recycle);
+
+		process_free_queue(state);
+	}
+	return 0;
+}
+
+
+/* Called by the sync thread */
+static int
+sync_func(void *v)
+{
+	VCHIQ_STATE_T *state = (VCHIQ_STATE_T *) v;
+	VCHIQ_SHARED_STATE_T *local = state->local;
+	VCHIQ_HEADER_T *header = (VCHIQ_HEADER_T *)SLOT_DATA_FROM_INDEX(state,
+		state->remote->slot_sync);
+
+	while (1) {
+		VCHIQ_SERVICE_T *service;
+		int msgid, size;
+		int type;
+		unsigned int localport, remoteport;
+
+		remote_event_wait(&local->sync_trigger);
+
+		rmb();
+
+		msgid = header->msgid;
+		size = header->size;
+		type = VCHIQ_MSG_TYPE(msgid);
+		localport = VCHIQ_MSG_DSTPORT(msgid);
+		remoteport = VCHIQ_MSG_SRCPORT(msgid);
+
+		service = find_service_by_port(state, localport);
+
+		if (!service) {
+			vchiq_log_error(vchiq_sync_log_level,
+				"%d: sf %s@%x (%d->%d) - "
+				"invalid/closed service %d",
+				state->id, msg_type_str(type),
+				(unsigned int)header,
+				remoteport, localport, localport);
+			release_message_sync(state, header);
+			continue;
+		}
+
+		if (vchiq_sync_log_level >= VCHIQ_LOG_TRACE) {
+			int svc_fourcc;
+
+			svc_fourcc = service
+				? service->base.fourcc
+				: VCHIQ_MAKE_FOURCC('?', '?', '?', '?');
+			vchiq_log_trace(vchiq_sync_log_level,
+				"Rcvd Msg %s from %c%c%c%c s:%d d:%d len:%d",
+				msg_type_str(type),
+				VCHIQ_FOURCC_AS_4CHARS(svc_fourcc),
+				remoteport, localport, size);
+			if (size > 0)
+				vchiq_log_dump_mem("Rcvd", 0, header->data,
+					min(64, size));
+		}
+
+		switch (type) {
+		case VCHIQ_MSG_OPENACK:
+			if (size >= sizeof(struct vchiq_openack_payload)) {
+				const struct vchiq_openack_payload *payload =
+					(struct vchiq_openack_payload *)
+					header->data;
+				service->peer_version = payload->version;
+			}
+			vchiq_log_info(vchiq_sync_log_level,
+				"%d: sf OPENACK@%x,%x (%d->%d) v:%d",
+				state->id, (unsigned int)header, size,
+				remoteport, localport, service->peer_version);
+			if (service->srvstate == VCHIQ_SRVSTATE_OPENING) {
+				service->remoteport = remoteport;
+				vchiq_set_service_state(service,
+					VCHIQ_SRVSTATE_OPENSYNC);
+				service->sync = 1;
+				up(&service->remove_event);
+			}
+			release_message_sync(state, header);
+			break;
+
+		case VCHIQ_MSG_DATA:
+			vchiq_log_trace(vchiq_sync_log_level,
+				"%d: sf DATA@%x,%x (%d->%d)",
+				state->id, (unsigned int)header, size,
+				remoteport, localport);
+
+			if ((service->remoteport == remoteport) &&
+				(service->srvstate ==
+				VCHIQ_SRVSTATE_OPENSYNC)) {
+				if (make_service_callback(service,
+					VCHIQ_MESSAGE_AVAILABLE, header,
+					NULL) == VCHIQ_RETRY)
+					vchiq_log_error(vchiq_sync_log_level,
+						"synchronous callback to "
+						"service %d returns "
+						"VCHIQ_RETRY",
+						localport);
+			}
+			break;
+
+		default:
+			vchiq_log_error(vchiq_sync_log_level,
+				"%d: sf unexpected msgid %x@%x,%x",
+				state->id, msgid, (unsigned int)header, size);
+			release_message_sync(state, header);
+			break;
+		}
+
+		unlock_service(service);
+	}
+
+	return 0;
+}
+
+
+static void
+init_bulk_queue(VCHIQ_BULK_QUEUE_T *queue)
+{
+	queue->local_insert = 0;
+	queue->remote_insert = 0;
+	queue->process = 0;
+	queue->remote_notify = 0;
+	queue->remove = 0;
+}
+
+
+inline const char *
+get_conn_state_name(VCHIQ_CONNSTATE_T conn_state)
+{
+	return conn_state_names[conn_state];
+}
+
+
+VCHIQ_SLOT_ZERO_T *
+vchiq_init_slots(void *mem_base, int mem_size)
+{
+	int mem_align = (VCHIQ_SLOT_SIZE - (int)mem_base) & VCHIQ_SLOT_MASK;
+	VCHIQ_SLOT_ZERO_T *slot_zero =
+		(VCHIQ_SLOT_ZERO_T *)((char *)mem_base + mem_align);
+	int num_slots = (mem_size - mem_align)/VCHIQ_SLOT_SIZE;
+	int first_data_slot = VCHIQ_SLOT_ZERO_SLOTS;
+
+	/* Ensure there is enough memory to run an absolutely minimum system */
+	num_slots -= first_data_slot;
+
+	if (num_slots < 4) {
+		vchiq_log_error(vchiq_core_log_level,
+			"vchiq_init_slots - insufficient memory %x bytes",
+			mem_size);
+		return NULL;
+	}
+
+	memset(slot_zero, 0, sizeof(VCHIQ_SLOT_ZERO_T));
+
+	slot_zero->magic = VCHIQ_MAGIC;
+	slot_zero->version = VCHIQ_VERSION;
+	slot_zero->version_min = VCHIQ_VERSION_MIN;
+	slot_zero->slot_zero_size = sizeof(VCHIQ_SLOT_ZERO_T);
+	slot_zero->slot_size = VCHIQ_SLOT_SIZE;
+	slot_zero->max_slots = VCHIQ_MAX_SLOTS;
+	slot_zero->max_slots_per_side = VCHIQ_MAX_SLOTS_PER_SIDE;
+
+	slot_zero->master.slot_sync = first_data_slot;
+	slot_zero->master.slot_first = first_data_slot + 1;
+	slot_zero->master.slot_last = first_data_slot + (num_slots/2) - 1;
+	slot_zero->slave.slot_sync = first_data_slot + (num_slots/2);
+	slot_zero->slave.slot_first = first_data_slot + (num_slots/2) + 1;
+	slot_zero->slave.slot_last = first_data_slot + num_slots - 1;
+
+	return slot_zero;
+}
+
+VCHIQ_STATUS_T
+vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
+		 int is_master)
+{
+	VCHIQ_SHARED_STATE_T *local;
+	VCHIQ_SHARED_STATE_T *remote;
+	VCHIQ_STATUS_T status;
+	char threadname[10];
+	static int id;
+	int i;
+
+	vchiq_log_warning(vchiq_core_log_level,
+		"%s: slot_zero = 0x%08lx, is_master = %d",
+		__func__, (unsigned long)slot_zero, is_master);
+
+	/* Check the input configuration */
+
+	if (slot_zero->magic != VCHIQ_MAGIC) {
+		vchiq_loud_error_header();
+		vchiq_loud_error("Invalid VCHIQ magic value found.");
+		vchiq_loud_error("slot_zero=%x: magic=%x (expected %x)",
+			(unsigned int)slot_zero, slot_zero->magic, VCHIQ_MAGIC);
+		vchiq_loud_error_footer();
+		return VCHIQ_ERROR;
+	}
+
+	if (slot_zero->version < VCHIQ_VERSION_MIN) {
+		vchiq_loud_error_header();
+		vchiq_loud_error("Incompatible VCHIQ versions found.");
+		vchiq_loud_error("slot_zero=%x: VideoCore version=%d "
+			"(minimum %d)",
+			(unsigned int)slot_zero, slot_zero->version,
+			VCHIQ_VERSION_MIN);
+		vchiq_loud_error("Restart with a newer VideoCore image.");
+		vchiq_loud_error_footer();
+		return VCHIQ_ERROR;
+	}
+
+	if (VCHIQ_VERSION < slot_zero->version_min) {
+		vchiq_loud_error_header();
+		vchiq_loud_error("Incompatible VCHIQ versions found.");
+		vchiq_loud_error("slot_zero=%x: version=%d (VideoCore "
+			"minimum %d)",
+			(unsigned int)slot_zero, VCHIQ_VERSION,
+			slot_zero->version_min);
+		vchiq_loud_error("Restart with a newer kernel.");
+		vchiq_loud_error_footer();
+		return VCHIQ_ERROR;
+	}
+
+	if ((slot_zero->slot_zero_size != sizeof(VCHIQ_SLOT_ZERO_T)) ||
+		 (slot_zero->slot_size != VCHIQ_SLOT_SIZE) ||
+		 (slot_zero->max_slots != VCHIQ_MAX_SLOTS) ||
+		 (slot_zero->max_slots_per_side != VCHIQ_MAX_SLOTS_PER_SIDE)) {
+		vchiq_loud_error_header();
+		if (slot_zero->slot_zero_size != sizeof(VCHIQ_SLOT_ZERO_T))
+			vchiq_loud_error("slot_zero=%x: slot_zero_size=%x "
+				"(expected %x)",
+				(unsigned int)slot_zero,
+				slot_zero->slot_zero_size,
+				sizeof(VCHIQ_SLOT_ZERO_T));
+		if (slot_zero->slot_size != VCHIQ_SLOT_SIZE)
+			vchiq_loud_error("slot_zero=%x: slot_size=%d "
+				"(expected %d",
+				(unsigned int)slot_zero, slot_zero->slot_size,
+				VCHIQ_SLOT_SIZE);
+		if (slot_zero->max_slots != VCHIQ_MAX_SLOTS)
+			vchiq_loud_error("slot_zero=%x: max_slots=%d "
+				"(expected %d)",
+				(unsigned int)slot_zero, slot_zero->max_slots,
+				VCHIQ_MAX_SLOTS);
+		if (slot_zero->max_slots_per_side != VCHIQ_MAX_SLOTS_PER_SIDE)
+			vchiq_loud_error("slot_zero=%x: max_slots_per_side=%d "
+				"(expected %d)",
+				(unsigned int)slot_zero,
+				slot_zero->max_slots_per_side,
+				VCHIQ_MAX_SLOTS_PER_SIDE);
+		vchiq_loud_error_footer();
+		return VCHIQ_ERROR;
+	}
+
+	if (VCHIQ_VERSION < slot_zero->version)
+		slot_zero->version = VCHIQ_VERSION;
+
+	if (is_master) {
+		local = &slot_zero->master;
+		remote = &slot_zero->slave;
+	} else {
+		local = &slot_zero->slave;
+		remote = &slot_zero->master;
+	}
+
+	if (local->initialised) {
+		vchiq_loud_error_header();
+		if (remote->initialised)
+			vchiq_loud_error("local state has already been "
+				"initialised");
+		else
+			vchiq_loud_error("master/slave mismatch - two %ss",
+				is_master ? "master" : "slave");
+		vchiq_loud_error_footer();
+		return VCHIQ_ERROR;
+	}
+
+	memset(state, 0, sizeof(VCHIQ_STATE_T));
+
+	state->id = id++;
+	state->is_master = is_master;
+
+	/*
+		initialize shared state pointers
+	 */
+
+	state->local = local;
+	state->remote = remote;
+	state->slot_data = (VCHIQ_SLOT_T *)slot_zero;
+
+	/*
+		initialize events and mutexes
+	 */
+
+	sema_init(&state->connect, 0);
+	mutex_init(&state->mutex);
+	sema_init(&state->trigger_event, 0);
+	sema_init(&state->recycle_event, 0);
+	sema_init(&state->sync_trigger_event, 0);
+	sema_init(&state->sync_release_event, 0);
+
+	mutex_init(&state->slot_mutex);
+	mutex_init(&state->recycle_mutex);
+	mutex_init(&state->sync_mutex);
+	mutex_init(&state->bulk_transfer_mutex);
+
+	sema_init(&state->slot_available_event, 0);
+	sema_init(&state->slot_remove_event, 0);
+	sema_init(&state->data_quota_event, 0);
+
+	state->slot_queue_available = 0;
+
+	for (i = 0; i < VCHIQ_MAX_SERVICES; i++) {
+		VCHIQ_SERVICE_QUOTA_T *service_quota =
+			&state->service_quotas[i];
+		sema_init(&service_quota->quota_event, 0);
+	}
+
+	for (i = local->slot_first; i <= local->slot_last; i++) {
+		local->slot_queue[state->slot_queue_available++] = i;
+		up(&state->slot_available_event);
+	}
+
+	state->default_slot_quota = state->slot_queue_available/2;
+	state->default_message_quota =
+		min((unsigned short)(state->default_slot_quota * 256),
+		(unsigned short)~0);
+
+	state->previous_data_index = -1;
+	state->data_use_count = 0;
+	state->data_quota = state->slot_queue_available - 1;
+
+	local->trigger.event = &state->trigger_event;
+	remote_event_create(&local->trigger);
+	local->tx_pos = 0;
+
+	local->recycle.event = &state->recycle_event;
+	remote_event_create(&local->recycle);
+	local->slot_queue_recycle = state->slot_queue_available;
+
+	local->sync_trigger.event = &state->sync_trigger_event;
+	remote_event_create(&local->sync_trigger);
+
+	local->sync_release.event = &state->sync_release_event;
+	remote_event_create(&local->sync_release);
+
+	/* At start-of-day, the slot is empty and available */
+	((VCHIQ_HEADER_T *)SLOT_DATA_FROM_INDEX(state, local->slot_sync))->msgid
+		= VCHIQ_MSGID_PADDING;
+	remote_event_signal_local(&local->sync_release);
+
+	local->debug[DEBUG_ENTRIES] = DEBUG_MAX;
+
+	status = vchiq_platform_init_state(state);
+
+	/*
+		bring up slot handler thread
+	 */
+	snprintf(threadname, sizeof(threadname), "VCHIQ-%d", state->id);
+	state->slot_handler_thread = kthread_create(&slot_handler_func,
+		(void *)state,
+		threadname);
+
+	if (state->slot_handler_thread == NULL) {
+		vchiq_loud_error_header();
+		vchiq_loud_error("couldn't create thread %s", threadname);
+		vchiq_loud_error_footer();
+		return VCHIQ_ERROR;
+	}
+	set_user_nice(state->slot_handler_thread, -19);
+	wake_up_process(state->slot_handler_thread);
+
+	snprintf(threadname, sizeof(threadname), "VCHIQr-%d", state->id);
+	state->recycle_thread = kthread_create(&recycle_func,
+		(void *)state,
+		threadname);
+	if (state->recycle_thread == NULL) {
+		vchiq_loud_error_header();
+		vchiq_loud_error("couldn't create thread %s", threadname);
+		vchiq_loud_error_footer();
+		return VCHIQ_ERROR;
+	}
+	set_user_nice(state->recycle_thread, -19);
+	wake_up_process(state->recycle_thread);
+
+	snprintf(threadname, sizeof(threadname), "VCHIQs-%d", state->id);
+	state->sync_thread = kthread_create(&sync_func,
+		(void *)state,
+		threadname);
+	if (state->sync_thread == NULL) {
+		vchiq_loud_error_header();
+		vchiq_loud_error("couldn't create thread %s", threadname);
+		vchiq_loud_error_footer();
+		return VCHIQ_ERROR;
+	}
+	set_user_nice(state->sync_thread, -20);
+	wake_up_process(state->sync_thread);
+
+	BUG_ON(state->id >= VCHIQ_MAX_STATES);
+	vchiq_states[state->id] = state;
+
+	/* Indicate readiness to the other side */
+	local->initialised = 1;
+
+	return status;
+}
+
+/* Called from application thread when a client or server service is created. */
+VCHIQ_SERVICE_T *
+vchiq_add_service_internal(VCHIQ_STATE_T *state,
+	const VCHIQ_SERVICE_PARAMS_T *params, int srvstate,
+	VCHIQ_INSTANCE_T instance, VCHIQ_USERDATA_TERM_T userdata_term)
+{
+	VCHIQ_SERVICE_T *service;
+
+	service = kmalloc(sizeof(VCHIQ_SERVICE_T), GFP_KERNEL);
+	if (service) {
+		service->base.fourcc   = params->fourcc;
+		service->base.callback = params->callback;
+		service->base.userdata = params->userdata;
+		service->handle        = VCHIQ_SERVICE_HANDLE_INVALID;
+		service->ref_count     = 1;
+		service->srvstate      = VCHIQ_SRVSTATE_FREE;
+		service->userdata_term = userdata_term;
+		service->localport     = VCHIQ_PORT_FREE;
+		service->remoteport    = VCHIQ_PORT_FREE;
+
+		service->public_fourcc = (srvstate == VCHIQ_SRVSTATE_OPENING) ?
+			VCHIQ_FOURCC_INVALID : params->fourcc;
+		service->client_id     = 0;
+		service->auto_close    = 1;
+		service->sync          = 0;
+		service->closing       = 0;
+		service->trace         = 0;
+		atomic_set(&service->poll_flags, 0);
+		service->version       = params->version;
+		service->version_min   = params->version_min;
+		service->state         = state;
+		service->instance      = instance;
+		service->service_use_count = 0;
+		init_bulk_queue(&service->bulk_tx);
+		init_bulk_queue(&service->bulk_rx);
+		sema_init(&service->remove_event, 0);
+		sema_init(&service->bulk_remove_event, 0);
+		mutex_init(&service->bulk_mutex);
+		memset(&service->stats, 0, sizeof(service->stats));
+	} else {
+		vchiq_log_error(vchiq_core_log_level,
+			"Out of memory");
+	}
+
+	if (service) {
+		VCHIQ_SERVICE_T **pservice = NULL;
+		int i;
+
+		/* Although it is perfectly possible to use service_spinlock
+		** to protect the creation of services, it is overkill as it
+		** disables interrupts while the array is searched.
+		** The only danger is of another thread trying to create a
+		** service - service deletion is safe.
+		** Therefore it is preferable to use state->mutex which,
+		** although slower to claim, doesn't block interrupts while
+		** it is held.
+		*/
+
+		mutex_lock(&state->mutex);
+
+		/* Prepare to use a previously unused service */
+		if (state->unused_service < VCHIQ_MAX_SERVICES)
+			pservice = &state->services[state->unused_service];
+
+		if (srvstate == VCHIQ_SRVSTATE_OPENING) {
+			for (i = 0; i < state->unused_service; i++) {
+				VCHIQ_SERVICE_T *srv = state->services[i];
+				if (!srv) {
+					pservice = &state->services[i];
+					break;
+				}
+			}
+		} else {
+			for (i = (state->unused_service - 1); i >= 0; i--) {
+				VCHIQ_SERVICE_T *srv = state->services[i];
+				if (!srv)
+					pservice = &state->services[i];
+				else if ((srv->public_fourcc == params->fourcc)
+					&& ((srv->instance != instance) ||
+					(srv->base.callback !=
+					params->callback))) {
+					/* There is another server using this
+					** fourcc which doesn't match. */
+					pservice = NULL;
+					break;
+				}
+			}
+		}
+
+		if (pservice) {
+			service->localport = (pservice - state->services);
+			if (!handle_seq)
+				handle_seq = VCHIQ_MAX_STATES *
+					 VCHIQ_MAX_SERVICES;
+			service->handle = handle_seq |
+				(state->id * VCHIQ_MAX_SERVICES) |
+				service->localport;
+			handle_seq += VCHIQ_MAX_STATES * VCHIQ_MAX_SERVICES;
+			*pservice = service;
+			if (pservice == &state->services[state->unused_service])
+				state->unused_service++;
+		}
+
+		mutex_unlock(&state->mutex);
+
+		if (!pservice) {
+			kfree(service);
+			service = NULL;
+		}
+	}
+
+	if (service) {
+		VCHIQ_SERVICE_QUOTA_T *service_quota =
+			&state->service_quotas[service->localport];
+		service_quota->slot_quota = state->default_slot_quota;
+		service_quota->message_quota = state->default_message_quota;
+		if (service_quota->slot_use_count == 0)
+			service_quota->previous_tx_index =
+				SLOT_QUEUE_INDEX_FROM_POS(state->local_tx_pos)
+				- 1;
+
+		/* Bring this service online */
+		vchiq_set_service_state(service, srvstate);
+
+		vchiq_log_info(vchiq_core_msg_log_level,
+			"%s Service %c%c%c%c SrcPort:%d",
+			(srvstate == VCHIQ_SRVSTATE_OPENING)
+			? "Open" : "Add",
+			VCHIQ_FOURCC_AS_4CHARS(params->fourcc),
+			service->localport);
+	}
+
+	/* Don't unlock the service - leave it with a ref_count of 1. */
+
+	return service;
+}
+
+VCHIQ_STATUS_T
+vchiq_open_service_internal(VCHIQ_SERVICE_T *service, int client_id)
+{
+	struct vchiq_open_payload payload = {
+		service->base.fourcc,
+		client_id,
+		service->version,
+		service->version_min
+	};
+	VCHIQ_ELEMENT_T body = { &payload, sizeof(payload) };
+	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+
+	service->client_id = client_id;
+	vchiq_use_service_internal(service);
+	status = queue_message(service->state, NULL,
+		VCHIQ_MAKE_MSG(VCHIQ_MSG_OPEN, service->localport, 0),
+		&body, 1, sizeof(payload), QMFLAGS_IS_BLOCKING);
+	if (status == VCHIQ_SUCCESS) {
+		/* Wait for the ACK/NAK */
+		if (down_interruptible(&service->remove_event) != 0) {
+			status = VCHIQ_RETRY;
+			vchiq_release_service_internal(service);
+		} else if ((service->srvstate != VCHIQ_SRVSTATE_OPEN) &&
+			(service->srvstate != VCHIQ_SRVSTATE_OPENSYNC)) {
+			if (service->srvstate != VCHIQ_SRVSTATE_CLOSEWAIT)
+				vchiq_log_error(vchiq_core_log_level,
+					"%d: osi - srvstate = %s (ref %d)",
+					service->state->id,
+					srvstate_names[service->srvstate],
+					service->ref_count);
+			status = VCHIQ_ERROR;
+			VCHIQ_SERVICE_STATS_INC(service, error_count);
+			vchiq_release_service_internal(service);
+		}
+	}
+	return status;
+}
+
+static void
+release_service_messages(VCHIQ_SERVICE_T *service)
+{
+	VCHIQ_STATE_T *state = service->state;
+	int slot_last = state->remote->slot_last;
+	int i;
+
+	/* Release any claimed messages aimed at this service */
+
+	if (service->sync) {
+		VCHIQ_HEADER_T *header =
+			(VCHIQ_HEADER_T *)SLOT_DATA_FROM_INDEX(state,
+						state->remote->slot_sync);
+		if (VCHIQ_MSG_DSTPORT(header->msgid) == service->localport)
+			release_message_sync(state, header);
+
+		return;
+	}
+
+	for (i = state->remote->slot_first; i <= slot_last; i++) {
+		VCHIQ_SLOT_INFO_T *slot_info =
+			SLOT_INFO_FROM_INDEX(state, i);
+		if (slot_info->release_count != slot_info->use_count) {
+			char *data =
+				(char *)SLOT_DATA_FROM_INDEX(state, i);
+			unsigned int pos, end;
+
+			end = VCHIQ_SLOT_SIZE;
+			if (data == state->rx_data)
+				/* This buffer is still being read from - stop
+				** at the current read position */
+				end = state->rx_pos & VCHIQ_SLOT_MASK;
+
+			pos = 0;
+
+			while (pos < end) {
+				VCHIQ_HEADER_T *header =
+					(VCHIQ_HEADER_T *)(data + pos);
+				int msgid = header->msgid;
+				int port = VCHIQ_MSG_DSTPORT(msgid);
+				if ((port == service->localport) &&
+					(msgid & VCHIQ_MSGID_CLAIMED)) {
+					vchiq_log_info(vchiq_core_log_level,
+						"  fsi - hdr %x",
+						(unsigned int)header);
+					release_slot(state, slot_info, header,
+						NULL);
+				}
+				pos += calc_stride(header->size);
+				if (pos > VCHIQ_SLOT_SIZE) {
+					vchiq_log_error(vchiq_core_log_level,
+						"fsi - pos %x: header %x, "
+						"msgid %x, header->msgid %x, "
+						"header->size %x",
+						pos, (unsigned int)header,
+						msgid, header->msgid,
+						header->size);
+					WARN(1, "invalid slot position\n");
+				}
+			}
+		}
+	}
+}
+
+static int
+do_abort_bulks(VCHIQ_SERVICE_T *service)
+{
+	VCHIQ_STATUS_T status;
+
+	/* Abort any outstanding bulk transfers */
+	if (mutex_lock_interruptible(&service->bulk_mutex) != 0)
+		return 0;
+	abort_outstanding_bulks(service, &service->bulk_tx);
+	abort_outstanding_bulks(service, &service->bulk_rx);
+	mutex_unlock(&service->bulk_mutex);
+
+	status = notify_bulks(service, &service->bulk_tx, 0/*!retry_poll*/);
+	if (status == VCHIQ_SUCCESS)
+		status = notify_bulks(service, &service->bulk_rx,
+			0/*!retry_poll*/);
+	return (status == VCHIQ_SUCCESS);
+}
+
+static VCHIQ_STATUS_T
+close_service_complete(VCHIQ_SERVICE_T *service, int failstate)
+{
+	VCHIQ_STATUS_T status;
+	int is_server = (service->public_fourcc != VCHIQ_FOURCC_INVALID);
+	int newstate;
+
+	switch (service->srvstate) {
+	case VCHIQ_SRVSTATE_OPEN:
+	case VCHIQ_SRVSTATE_CLOSESENT:
+	case VCHIQ_SRVSTATE_CLOSERECVD:
+		if (is_server) {
+			if (service->auto_close) {
+				service->client_id = 0;
+				service->remoteport = VCHIQ_PORT_FREE;
+				newstate = VCHIQ_SRVSTATE_LISTENING;
+			} else
+				newstate = VCHIQ_SRVSTATE_CLOSEWAIT;
+		} else
+			newstate = VCHIQ_SRVSTATE_CLOSED;
+		vchiq_set_service_state(service, newstate);
+		break;
+	case VCHIQ_SRVSTATE_LISTENING:
+		break;
+	default:
+		vchiq_log_error(vchiq_core_log_level,
+			"close_service_complete(%x) called in state %s",
+			service->handle, srvstate_names[service->srvstate]);
+		WARN(1, "close_service_complete in unexpected state\n");
+		return VCHIQ_ERROR;
+	}
+
+	status = make_service_callback(service,
+		VCHIQ_SERVICE_CLOSED, NULL, NULL);
+
+	if (status != VCHIQ_RETRY) {
+		int uc = service->service_use_count;
+		int i;
+		/* Complete the close process */
+		for (i = 0; i < uc; i++)
+			/* cater for cases where close is forced and the
+			** client may not close all it's handles */
+			vchiq_release_service_internal(service);
+
+		service->client_id = 0;
+		service->remoteport = VCHIQ_PORT_FREE;
+
+		if (service->srvstate == VCHIQ_SRVSTATE_CLOSED)
+			vchiq_free_service_internal(service);
+		else if (service->srvstate != VCHIQ_SRVSTATE_CLOSEWAIT) {
+			if (is_server)
+				service->closing = 0;
+
+			up(&service->remove_event);
+		}
+	} else
+		vchiq_set_service_state(service, failstate);
+
+	return status;
+}
+
+/* Called by the slot handler */
+VCHIQ_STATUS_T
+vchiq_close_service_internal(VCHIQ_SERVICE_T *service, int close_recvd)
+{
+	VCHIQ_STATE_T *state = service->state;
+	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+	int is_server = (service->public_fourcc != VCHIQ_FOURCC_INVALID);
+
+	vchiq_log_info(vchiq_core_log_level, "%d: csi:%d,%d (%s)",
+		service->state->id, service->localport, close_recvd,
+		srvstate_names[service->srvstate]);
+
+	switch (service->srvstate) {
+	case VCHIQ_SRVSTATE_CLOSED:
+	case VCHIQ_SRVSTATE_HIDDEN:
+	case VCHIQ_SRVSTATE_LISTENING:
+	case VCHIQ_SRVSTATE_CLOSEWAIT:
+		if (close_recvd)
+			vchiq_log_error(vchiq_core_log_level,
+				"vchiq_close_service_internal(1) called "
+				"in state %s",
+				srvstate_names[service->srvstate]);
+		else if (is_server) {
+			if (service->srvstate == VCHIQ_SRVSTATE_LISTENING) {
+				status = VCHIQ_ERROR;
+			} else {
+				service->client_id = 0;
+				service->remoteport = VCHIQ_PORT_FREE;
+				if (service->srvstate ==
+					VCHIQ_SRVSTATE_CLOSEWAIT)
+					vchiq_set_service_state(service,
+						VCHIQ_SRVSTATE_LISTENING);
+			}
+			up(&service->remove_event);
+		} else
+			vchiq_free_service_internal(service);
+		break;
+	case VCHIQ_SRVSTATE_OPENING:
+		if (close_recvd) {
+			/* The open was rejected - tell the user */
+			vchiq_set_service_state(service,
+				VCHIQ_SRVSTATE_CLOSEWAIT);
+			up(&service->remove_event);
+		} else {
+			/* Shutdown mid-open - let the other side know */
+			status = queue_message(state, service,
+				VCHIQ_MAKE_MSG
+				(VCHIQ_MSG_CLOSE,
+				service->localport,
+				VCHIQ_MSG_DSTPORT(service->remoteport)),
+				NULL, 0, 0, 0);
+		}
+		break;
+
+	case VCHIQ_SRVSTATE_OPENSYNC:
+		mutex_lock(&state->sync_mutex);
+		/* Drop through */
+
+	case VCHIQ_SRVSTATE_OPEN:
+		if (state->is_master || close_recvd) {
+			if (!do_abort_bulks(service))
+				status = VCHIQ_RETRY;
+		}
+
+		release_service_messages(service);
+
+		if (status == VCHIQ_SUCCESS)
+			status = queue_message(state, service,
+				VCHIQ_MAKE_MSG
+				(VCHIQ_MSG_CLOSE,
+				service->localport,
+				VCHIQ_MSG_DSTPORT(service->remoteport)),
+				NULL, 0, 0, QMFLAGS_NO_MUTEX_UNLOCK);
+
+		if (status == VCHIQ_SUCCESS) {
+			if (!close_recvd) {
+				/* Change the state while the mutex is
+				   still held */
+				vchiq_set_service_state(service,
+							VCHIQ_SRVSTATE_CLOSESENT);
+				mutex_unlock(&state->slot_mutex);
+				if (service->sync)
+					mutex_unlock(&state->sync_mutex);
+				break;
+			}
+		} else if (service->srvstate == VCHIQ_SRVSTATE_OPENSYNC) {
+			mutex_unlock(&state->sync_mutex);
+			break;
+		} else
+			break;
+
+		/* Change the state while the mutex is still held */
+		vchiq_set_service_state(service, VCHIQ_SRVSTATE_CLOSERECVD);
+		mutex_unlock(&state->slot_mutex);
+		if (service->sync)
+			mutex_unlock(&state->sync_mutex);
+
+		status = close_service_complete(service,
+				VCHIQ_SRVSTATE_CLOSERECVD);
+		break;
+
+	case VCHIQ_SRVSTATE_CLOSESENT:
+		if (!close_recvd)
+			/* This happens when a process is killed mid-close */
+			break;
+
+		if (!state->is_master) {
+			if (!do_abort_bulks(service)) {
+				status = VCHIQ_RETRY;
+				break;
+			}
+		}
+
+		if (status == VCHIQ_SUCCESS)
+			status = close_service_complete(service,
+				VCHIQ_SRVSTATE_CLOSERECVD);
+		break;
+
+	case VCHIQ_SRVSTATE_CLOSERECVD:
+		if (!close_recvd && is_server)
+			/* Force into LISTENING mode */
+			vchiq_set_service_state(service,
+				VCHIQ_SRVSTATE_LISTENING);
+		status = close_service_complete(service,
+			VCHIQ_SRVSTATE_CLOSERECVD);
+		break;
+
+	default:
+		vchiq_log_error(vchiq_core_log_level,
+			"vchiq_close_service_internal(%d) called in state %s",
+			close_recvd, srvstate_names[service->srvstate]);
+		break;
+	}
+
+	return status;
+}
+
+/* Called from the application process upon process death */
+void
+vchiq_terminate_service_internal(VCHIQ_SERVICE_T *service)
+{
+	VCHIQ_STATE_T *state = service->state;
+
+	vchiq_log_info(vchiq_core_log_level, "%d: tsi - (%d<->%d)",
+		state->id, service->localport, service->remoteport);
+
+	mark_service_closing(service);
+
+	/* Mark the service for removal by the slot handler */
+	request_poll(state, service, VCHIQ_POLL_REMOVE);
+}
+
+/* Called from the slot handler */
+void
+vchiq_free_service_internal(VCHIQ_SERVICE_T *service)
+{
+	VCHIQ_STATE_T *state = service->state;
+
+	vchiq_log_info(vchiq_core_log_level, "%d: fsi - (%d)",
+		state->id, service->localport);
+
+	switch (service->srvstate) {
+	case VCHIQ_SRVSTATE_OPENING:
+	case VCHIQ_SRVSTATE_CLOSED:
+	case VCHIQ_SRVSTATE_HIDDEN:
+	case VCHIQ_SRVSTATE_LISTENING:
+	case VCHIQ_SRVSTATE_CLOSEWAIT:
+		break;
+	default:
+		vchiq_log_error(vchiq_core_log_level,
+			"%d: fsi - (%d) in state %s",
+			state->id, service->localport,
+			srvstate_names[service->srvstate]);
+		return;
+	}
+
+	vchiq_set_service_state(service, VCHIQ_SRVSTATE_FREE);
+
+	up(&service->remove_event);
+
+	/* Release the initial lock */
+	unlock_service(service);
+}
+
+VCHIQ_STATUS_T
+vchiq_connect_internal(VCHIQ_STATE_T *state, VCHIQ_INSTANCE_T instance)
+{
+	VCHIQ_SERVICE_T *service;
+	int i;
+
+	/* Find all services registered to this client and enable them. */
+	i = 0;
+	while ((service = next_service_by_instance(state, instance,
+		&i)) !=	NULL) {
+		if (service->srvstate == VCHIQ_SRVSTATE_HIDDEN)
+			vchiq_set_service_state(service,
+				VCHIQ_SRVSTATE_LISTENING);
+		unlock_service(service);
+	}
+
+	if (state->conn_state == VCHIQ_CONNSTATE_DISCONNECTED) {
+		if (queue_message(state, NULL,
+			VCHIQ_MAKE_MSG(VCHIQ_MSG_CONNECT, 0, 0), NULL, 0,
+			0, QMFLAGS_IS_BLOCKING) == VCHIQ_RETRY)
+			return VCHIQ_RETRY;
+
+		vchiq_set_conn_state(state, VCHIQ_CONNSTATE_CONNECTING);
+	}
+
+	if (state->conn_state == VCHIQ_CONNSTATE_CONNECTING) {
+		if (down_interruptible(&state->connect) != 0)
+			return VCHIQ_RETRY;
+
+		vchiq_set_conn_state(state, VCHIQ_CONNSTATE_CONNECTED);
+		up(&state->connect);
+	}
+
+	return VCHIQ_SUCCESS;
+}
+
+VCHIQ_STATUS_T
+vchiq_shutdown_internal(VCHIQ_STATE_T *state, VCHIQ_INSTANCE_T instance)
+{
+	VCHIQ_SERVICE_T *service;
+	int i;
+
+	/* Find all services registered to this client and enable them. */
+	i = 0;
+	while ((service = next_service_by_instance(state, instance,
+		&i)) !=	NULL) {
+		(void)vchiq_remove_service(service->handle);
+		unlock_service(service);
+	}
+
+	return VCHIQ_SUCCESS;
+}
+
+VCHIQ_STATUS_T
+vchiq_pause_internal(VCHIQ_STATE_T *state)
+{
+	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+
+	switch (state->conn_state) {
+	case VCHIQ_CONNSTATE_CONNECTED:
+		/* Request a pause */
+		vchiq_set_conn_state(state, VCHIQ_CONNSTATE_PAUSING);
+		request_poll(state, NULL, 0);
+		break;
+	default:
+		vchiq_log_error(vchiq_core_log_level,
+			"vchiq_pause_internal in state %s\n",
+			conn_state_names[state->conn_state]);
+		status = VCHIQ_ERROR;
+		VCHIQ_STATS_INC(state, error_count);
+		break;
+	}
+
+	return status;
+}
+
+VCHIQ_STATUS_T
+vchiq_resume_internal(VCHIQ_STATE_T *state)
+{
+	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+
+	if (state->conn_state == VCHIQ_CONNSTATE_PAUSED) {
+		vchiq_set_conn_state(state, VCHIQ_CONNSTATE_RESUMING);
+		request_poll(state, NULL, 0);
+	} else {
+		status = VCHIQ_ERROR;
+		VCHIQ_STATS_INC(state, error_count);
+	}
+
+	return status;
+}
+
+VCHIQ_STATUS_T
+vchiq_close_service(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	/* Unregister the service */
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+
+	if (!service)
+		return VCHIQ_ERROR;
+
+	vchiq_log_info(vchiq_core_log_level,
+		"%d: close_service:%d",
+		service->state->id, service->localport);
+
+	if ((service->srvstate == VCHIQ_SRVSTATE_FREE) ||
+		(service->srvstate == VCHIQ_SRVSTATE_LISTENING) ||
+		(service->srvstate == VCHIQ_SRVSTATE_HIDDEN)) {
+		unlock_service(service);
+		return VCHIQ_ERROR;
+	}
+
+	mark_service_closing(service);
+
+	if (current == service->state->slot_handler_thread) {
+		status = vchiq_close_service_internal(service,
+			0/*!close_recvd*/);
+		BUG_ON(status == VCHIQ_RETRY);
+	} else {
+	/* Mark the service for termination by the slot handler */
+		request_poll(service->state, service, VCHIQ_POLL_TERMINATE);
+	}
+
+	while (1) {
+		if (down_interruptible(&service->remove_event) != 0) {
+			status = VCHIQ_RETRY;
+			break;
+		}
+
+		if ((service->srvstate == VCHIQ_SRVSTATE_FREE) ||
+			(service->srvstate == VCHIQ_SRVSTATE_LISTENING) ||
+			(service->srvstate == VCHIQ_SRVSTATE_OPEN))
+			break;
+
+		vchiq_log_warning(vchiq_core_log_level,
+			"%d: close_service:%d - waiting in state %s",
+			service->state->id, service->localport,
+			srvstate_names[service->srvstate]);
+	}
+
+	if ((status == VCHIQ_SUCCESS) &&
+		(service->srvstate != VCHIQ_SRVSTATE_FREE) &&
+		(service->srvstate != VCHIQ_SRVSTATE_LISTENING))
+		status = VCHIQ_ERROR;
+
+	unlock_service(service);
+
+	return status;
+}
+
+VCHIQ_STATUS_T
+vchiq_remove_service(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	/* Unregister the service */
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
+
+	if (!service)
+		return VCHIQ_ERROR;
+
+	vchiq_log_info(vchiq_core_log_level,
+		"%d: remove_service:%d",
+		service->state->id, service->localport);
+
+	if (service->srvstate == VCHIQ_SRVSTATE_FREE) {
+		unlock_service(service);
+		return VCHIQ_ERROR;
+	}
+
+	mark_service_closing(service);
+
+	if ((service->srvstate == VCHIQ_SRVSTATE_HIDDEN) ||
+		(current == service->state->slot_handler_thread)) {
+		/* Make it look like a client, because it must be removed and
+		   not left in the LISTENING state. */
+		service->public_fourcc = VCHIQ_FOURCC_INVALID;
+
+		status = vchiq_close_service_internal(service,
+			0/*!close_recvd*/);
+		BUG_ON(status == VCHIQ_RETRY);
+	} else {
+		/* Mark the service for removal by the slot handler */
+		request_poll(service->state, service, VCHIQ_POLL_REMOVE);
+	}
+	while (1) {
+		if (down_interruptible(&service->remove_event) != 0) {
+			status = VCHIQ_RETRY;
+			break;
+		}
+
+		if ((service->srvstate == VCHIQ_SRVSTATE_FREE) ||
+			(service->srvstate == VCHIQ_SRVSTATE_OPEN))
+			break;
+
+		vchiq_log_warning(vchiq_core_log_level,
+			"%d: remove_service:%d - waiting in state %s",
+			service->state->id, service->localport,
+			srvstate_names[service->srvstate]);
+	}
+
+	if ((status == VCHIQ_SUCCESS) &&
+		(service->srvstate != VCHIQ_SRVSTATE_FREE))
+		status = VCHIQ_ERROR;
+
+	unlock_service(service);
+
+	return status;
+}
+
+
+/* This function may be called by kernel threads or user threads.
+ * User threads may receive VCHIQ_RETRY to indicate that a signal has been
+ * received and the call should be retried after being returned to user
+ * context.
+ * When called in blocking mode, the userdata field points to a bulk_waiter
+ * structure.
+ */
+VCHIQ_STATUS_T
+vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle,
+	VCHI_MEM_HANDLE_T memhandle, void *offset, int size, void *userdata,
+	VCHIQ_BULK_MODE_T mode, VCHIQ_BULK_DIR_T dir)
+{
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	VCHIQ_BULK_QUEUE_T *queue;
+	VCHIQ_BULK_T *bulk;
+	VCHIQ_STATE_T *state;
+	struct bulk_waiter *bulk_waiter = NULL;
+	const char dir_char = (dir == VCHIQ_BULK_TRANSMIT) ? 't' : 'r';
+	const int dir_msgtype = (dir == VCHIQ_BULK_TRANSMIT) ?
+		VCHIQ_MSG_BULK_TX : VCHIQ_MSG_BULK_RX;
+	VCHIQ_STATUS_T status = VCHIQ_ERROR;
+
+	if (!service ||
+		 (service->srvstate != VCHIQ_SRVSTATE_OPEN) ||
+		 ((memhandle == VCHI_MEM_HANDLE_INVALID) && (offset == NULL)) ||
+		 (vchiq_check_service(service) != VCHIQ_SUCCESS))
+		goto error_exit;
+
+	switch (mode) {
+	case VCHIQ_BULK_MODE_NOCALLBACK:
+	case VCHIQ_BULK_MODE_CALLBACK:
+		break;
+	case VCHIQ_BULK_MODE_BLOCKING:
+		bulk_waiter = (struct bulk_waiter *)userdata;
+		sema_init(&bulk_waiter->event, 0);
+		bulk_waiter->actual = 0;
+		bulk_waiter->bulk = NULL;
+		break;
+	case VCHIQ_BULK_MODE_WAITING:
+		bulk_waiter = (struct bulk_waiter *)userdata;
+		bulk = bulk_waiter->bulk;
+		goto waiting;
+	default:
+		goto error_exit;
+	}
+
+	state = service->state;
+
+	queue = (dir == VCHIQ_BULK_TRANSMIT) ?
+		&service->bulk_tx : &service->bulk_rx;
+
+	if (mutex_lock_interruptible(&service->bulk_mutex) != 0) {
+		status = VCHIQ_RETRY;
+		goto error_exit;
+	}
+
+	if (queue->local_insert == queue->remove + VCHIQ_NUM_SERVICE_BULKS) {
+		VCHIQ_SERVICE_STATS_INC(service, bulk_stalls);
+		do {
+			mutex_unlock(&service->bulk_mutex);
+			if (down_interruptible(&service->bulk_remove_event)
+				!= 0) {
+				status = VCHIQ_RETRY;
+				goto error_exit;
+			}
+			if (mutex_lock_interruptible(&service->bulk_mutex)
+				!= 0) {
+				status = VCHIQ_RETRY;
+				goto error_exit;
+			}
+		} while (queue->local_insert == queue->remove +
+				VCHIQ_NUM_SERVICE_BULKS);
+	}
+
+	bulk = &queue->bulks[BULK_INDEX(queue->local_insert)];
+
+	bulk->mode = mode;
+	bulk->dir = dir;
+	bulk->userdata = userdata;
+	bulk->size = size;
+	bulk->actual = VCHIQ_BULK_ACTUAL_ABORTED;
+
+	if (vchiq_prepare_bulk_data(bulk, memhandle, offset, size, dir) !=
+		VCHIQ_SUCCESS)
+		goto unlock_error_exit;
+
+	wmb();
+
+	vchiq_log_info(vchiq_core_log_level,
+		"%d: bt (%d->%d) %cx %x@%x %x",
+		state->id,
+		service->localport, service->remoteport, dir_char,
+		size, (unsigned int)bulk->data, (unsigned int)userdata);
+
+	/* The slot mutex must be held when the service is being closed, so
+	   claim it here to ensure that isn't happening */
+	if (mutex_lock_interruptible(&state->slot_mutex) != 0) {
+		status = VCHIQ_RETRY;
+		goto cancel_bulk_error_exit;
+	}
+
+	if (service->srvstate != VCHIQ_SRVSTATE_OPEN)
+		goto unlock_both_error_exit;
+
+	if (state->is_master) {
+		queue->local_insert++;
+		if (resolve_bulks(service, queue))
+			request_poll(state, service,
+				(dir == VCHIQ_BULK_TRANSMIT) ?
+				VCHIQ_POLL_TXNOTIFY : VCHIQ_POLL_RXNOTIFY);
+	} else {
+		int payload[2] = { (int)bulk->data, bulk->size };
+		VCHIQ_ELEMENT_T element = { payload, sizeof(payload) };
+
+		status = queue_message(state, NULL,
+			VCHIQ_MAKE_MSG(dir_msgtype,
+				service->localport, service->remoteport),
+			&element, 1, sizeof(payload),
+			QMFLAGS_IS_BLOCKING |
+			QMFLAGS_NO_MUTEX_LOCK |
+			QMFLAGS_NO_MUTEX_UNLOCK);
+		if (status != VCHIQ_SUCCESS) {
+			goto unlock_both_error_exit;
+		}
+		queue->local_insert++;
+	}
+
+	mutex_unlock(&state->slot_mutex);
+	mutex_unlock(&service->bulk_mutex);
+
+	vchiq_log_trace(vchiq_core_log_level,
+		"%d: bt:%d %cx li=%x ri=%x p=%x",
+		state->id,
+		service->localport, dir_char,
+		queue->local_insert, queue->remote_insert, queue->process);
+
+waiting:
+	unlock_service(service);
+
+	status = VCHIQ_SUCCESS;
+
+	if (bulk_waiter) {
+		bulk_waiter->bulk = bulk;
+		if (down_interruptible(&bulk_waiter->event) != 0)
+			status = VCHIQ_RETRY;
+		else if (bulk_waiter->actual == VCHIQ_BULK_ACTUAL_ABORTED)
+			status = VCHIQ_ERROR;
+	}
+
+	return status;
+
+unlock_both_error_exit:
+	mutex_unlock(&state->slot_mutex);
+cancel_bulk_error_exit:
+	vchiq_complete_bulk(bulk);
+unlock_error_exit:
+	mutex_unlock(&service->bulk_mutex);
+
+error_exit:
+	if (service)
+		unlock_service(service);
+	return status;
+}
+
+VCHIQ_STATUS_T
+vchiq_queue_message(VCHIQ_SERVICE_HANDLE_T handle,
+	const VCHIQ_ELEMENT_T *elements, unsigned int count)
+{
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	VCHIQ_STATUS_T status = VCHIQ_ERROR;
+
+	unsigned int size = 0;
+	unsigned int i;
+
+	if (!service ||
+		(vchiq_check_service(service) != VCHIQ_SUCCESS))
+		goto error_exit;
+
+	for (i = 0; i < (unsigned int)count; i++) {
+		if (elements[i].size) {
+			if (elements[i].data == NULL) {
+				VCHIQ_SERVICE_STATS_INC(service, error_count);
+				goto error_exit;
+			}
+			size += elements[i].size;
+		}
+	}
+
+	if (size > VCHIQ_MAX_MSG_SIZE) {
+		VCHIQ_SERVICE_STATS_INC(service, error_count);
+		goto error_exit;
+	}
+
+	switch (service->srvstate) {
+	case VCHIQ_SRVSTATE_OPEN:
+		status = queue_message(service->state, service,
+				VCHIQ_MAKE_MSG(VCHIQ_MSG_DATA,
+					service->localport,
+					service->remoteport),
+				elements, count, size, 1);
+		break;
+	case VCHIQ_SRVSTATE_OPENSYNC:
+		status = queue_message_sync(service->state, service,
+				VCHIQ_MAKE_MSG(VCHIQ_MSG_DATA,
+					service->localport,
+					service->remoteport),
+				elements, count, size, 1);
+		break;
+	default:
+		status = VCHIQ_ERROR;
+		break;
+	}
+
+error_exit:
+	if (service)
+		unlock_service(service);
+
+	return status;
+}
+
+void
+vchiq_release_message(VCHIQ_SERVICE_HANDLE_T handle, VCHIQ_HEADER_T *header)
+{
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	VCHIQ_SHARED_STATE_T *remote;
+	VCHIQ_STATE_T *state;
+	int slot_index;
+
+	if (!service)
+		return;
+
+	state = service->state;
+	remote = state->remote;
+
+	slot_index = SLOT_INDEX_FROM_DATA(state, (void *)header);
+
+	if ((slot_index >= remote->slot_first) &&
+		(slot_index <= remote->slot_last)) {
+		int msgid = header->msgid;
+		if (msgid & VCHIQ_MSGID_CLAIMED) {
+			VCHIQ_SLOT_INFO_T *slot_info =
+				SLOT_INFO_FROM_INDEX(state, slot_index);
+
+			release_slot(state, slot_info, header, service);
+		}
+	} else if (slot_index == remote->slot_sync)
+		release_message_sync(state, header);
+
+	unlock_service(service);
+}
+
+static void
+release_message_sync(VCHIQ_STATE_T *state, VCHIQ_HEADER_T *header)
+{
+	header->msgid = VCHIQ_MSGID_PADDING;
+	wmb();
+	remote_event_signal(&state->remote->sync_release);
+}
+
+VCHIQ_STATUS_T
+vchiq_get_peer_version(VCHIQ_SERVICE_HANDLE_T handle, short *peer_version)
+{
+   VCHIQ_STATUS_T status = VCHIQ_ERROR;
+   VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+
+   if (!service ||
+      (vchiq_check_service(service) != VCHIQ_SUCCESS) ||
+      !peer_version)
+      goto exit;
+   *peer_version = service->peer_version;
+   status = VCHIQ_SUCCESS;
+
+exit:
+   if (service)
+      unlock_service(service);
+   return status;
+}
+
+VCHIQ_STATUS_T
+vchiq_get_config(VCHIQ_INSTANCE_T instance,
+	int config_size, VCHIQ_CONFIG_T *pconfig)
+{
+	VCHIQ_CONFIG_T config;
+
+	(void)instance;
+
+	config.max_msg_size           = VCHIQ_MAX_MSG_SIZE;
+	config.bulk_threshold         = VCHIQ_MAX_MSG_SIZE;
+	config.max_outstanding_bulks  = VCHIQ_NUM_SERVICE_BULKS;
+	config.max_services           = VCHIQ_MAX_SERVICES;
+	config.version                = VCHIQ_VERSION;
+	config.version_min            = VCHIQ_VERSION_MIN;
+
+	if (config_size > sizeof(VCHIQ_CONFIG_T))
+		return VCHIQ_ERROR;
+
+	memcpy(pconfig, &config,
+		min(config_size, (int)(sizeof(VCHIQ_CONFIG_T))));
+
+	return VCHIQ_SUCCESS;
+}
+
+VCHIQ_STATUS_T
+vchiq_set_service_option(VCHIQ_SERVICE_HANDLE_T handle,
+	VCHIQ_SERVICE_OPTION_T option, int value)
+{
+	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
+	VCHIQ_STATUS_T status = VCHIQ_ERROR;
+
+	if (service) {
+		switch (option) {
+		case VCHIQ_SERVICE_OPTION_AUTOCLOSE:
+			service->auto_close = value;
+			status = VCHIQ_SUCCESS;
+			break;
+
+		case VCHIQ_SERVICE_OPTION_SLOT_QUOTA: {
+			VCHIQ_SERVICE_QUOTA_T *service_quota =
+				&service->state->service_quotas[
+					service->localport];
+			if (value == 0)
+				value = service->state->default_slot_quota;
+			if ((value >= service_quota->slot_use_count) &&
+				 (value < (unsigned short)~0)) {
+				service_quota->slot_quota = value;
+				if ((value >= service_quota->slot_use_count) &&
+					(service_quota->message_quota >=
+					 service_quota->message_use_count)) {
+					/* Signal the service that it may have
+					** dropped below its quota */
+					up(&service_quota->quota_event);
+				}
+				status = VCHIQ_SUCCESS;
+			}
+		} break;
+
+		case VCHIQ_SERVICE_OPTION_MESSAGE_QUOTA: {
+			VCHIQ_SERVICE_QUOTA_T *service_quota =
+				&service->state->service_quotas[
+					service->localport];
+			if (value == 0)
+				value = service->state->default_message_quota;
+			if ((value >= service_quota->message_use_count) &&
+				 (value < (unsigned short)~0)) {
+				service_quota->message_quota = value;
+				if ((value >=
+					service_quota->message_use_count) &&
+					(service_quota->slot_quota >=
+					service_quota->slot_use_count))
+					/* Signal the service that it may have
+					** dropped below its quota */
+					up(&service_quota->quota_event);
+				status = VCHIQ_SUCCESS;
+			}
+		} break;
+
+		case VCHIQ_SERVICE_OPTION_SYNCHRONOUS:
+			if ((service->srvstate == VCHIQ_SRVSTATE_HIDDEN) ||
+				(service->srvstate ==
+				VCHIQ_SRVSTATE_LISTENING)) {
+				service->sync = value;
+				status = VCHIQ_SUCCESS;
+			}
+			break;
+
+		case VCHIQ_SERVICE_OPTION_TRACE:
+			service->trace = value;
+			status = VCHIQ_SUCCESS;
+			break;
+
+		default:
+			break;
+		}
+		unlock_service(service);
+	}
+
+	return status;
+}
+
+void
+vchiq_dump_shared_state(void *dump_context, VCHIQ_STATE_T *state,
+	VCHIQ_SHARED_STATE_T *shared, const char *label)
+{
+	static const char *const debug_names[] = {
+		"<entries>",
+		"SLOT_HANDLER_COUNT",
+		"SLOT_HANDLER_LINE",
+		"PARSE_LINE",
+		"PARSE_HEADER",
+		"PARSE_MSGID",
+		"AWAIT_COMPLETION_LINE",
+		"DEQUEUE_MESSAGE_LINE",
+		"SERVICE_CALLBACK_LINE",
+		"MSG_QUEUE_FULL_COUNT",
+		"COMPLETION_QUEUE_FULL_COUNT"
+	};
+	int i;
+
+	char buf[80];
+	int len;
+	len = snprintf(buf, sizeof(buf),
+		"  %s: slots %d-%d tx_pos=%x recycle=%x",
+		label, shared->slot_first, shared->slot_last,
+		shared->tx_pos, shared->slot_queue_recycle);
+	vchiq_dump(dump_context, buf, len + 1);
+
+	len = snprintf(buf, sizeof(buf),
+		"    Slots claimed:");
+	vchiq_dump(dump_context, buf, len + 1);
+
+	for (i = shared->slot_first; i <= shared->slot_last; i++) {
+		VCHIQ_SLOT_INFO_T slot_info = *SLOT_INFO_FROM_INDEX(state, i);
+		if (slot_info.use_count != slot_info.release_count) {
+			len = snprintf(buf, sizeof(buf),
+				"      %d: %d/%d", i, slot_info.use_count,
+				slot_info.release_count);
+			vchiq_dump(dump_context, buf, len + 1);
+		}
+	}
+
+	for (i = 1; i < shared->debug[DEBUG_ENTRIES]; i++) {
+		len = snprintf(buf, sizeof(buf), "    DEBUG: %s = %d(%x)",
+			debug_names[i], shared->debug[i], shared->debug[i]);
+		vchiq_dump(dump_context, buf, len + 1);
+	}
+}
+
+void
+vchiq_dump_state(void *dump_context, VCHIQ_STATE_T *state)
+{
+	char buf[80];
+	int len;
+	int i;
+
+	len = snprintf(buf, sizeof(buf), "State %d: %s", state->id,
+		conn_state_names[state->conn_state]);
+	vchiq_dump(dump_context, buf, len + 1);
+
+	len = snprintf(buf, sizeof(buf),
+		"  tx_pos=%x(@%x), rx_pos=%x(@%x)",
+		state->local->tx_pos,
+		(uint32_t)state->tx_data +
+			(state->local_tx_pos & VCHIQ_SLOT_MASK),
+		state->rx_pos,
+		(uint32_t)state->rx_data +
+			(state->rx_pos & VCHIQ_SLOT_MASK));
+	vchiq_dump(dump_context, buf, len + 1);
+
+	len = snprintf(buf, sizeof(buf),
+		"  Version: %d (min %d)",
+		VCHIQ_VERSION, VCHIQ_VERSION_MIN);
+	vchiq_dump(dump_context, buf, len + 1);
+
+	if (VCHIQ_ENABLE_STATS) {
+		len = snprintf(buf, sizeof(buf),
+			"  Stats: ctrl_tx_count=%d, ctrl_rx_count=%d, "
+			"error_count=%d",
+			state->stats.ctrl_tx_count, state->stats.ctrl_rx_count,
+			state->stats.error_count);
+		vchiq_dump(dump_context, buf, len + 1);
+	}
+
+	len = snprintf(buf, sizeof(buf),
+		"  Slots: %d available (%d data), %d recyclable, %d stalls "
+		"(%d data)",
+		((state->slot_queue_available * VCHIQ_SLOT_SIZE) -
+			state->local_tx_pos) / VCHIQ_SLOT_SIZE,
+		state->data_quota - state->data_use_count,
+		state->local->slot_queue_recycle - state->slot_queue_available,
+		state->stats.slot_stalls, state->stats.data_stalls);
+	vchiq_dump(dump_context, buf, len + 1);
+
+	vchiq_dump_platform_state(dump_context);
+
+	vchiq_dump_shared_state(dump_context, state, state->local, "Local");
+	vchiq_dump_shared_state(dump_context, state, state->remote, "Remote");
+
+	vchiq_dump_platform_instances(dump_context);
+
+	for (i = 0; i < state->unused_service; i++) {
+		VCHIQ_SERVICE_T *service = find_service_by_port(state, i);
+
+		if (service) {
+			vchiq_dump_service_state(dump_context, service);
+			unlock_service(service);
+		}
+	}
+}
+
+void
+vchiq_dump_service_state(void *dump_context, VCHIQ_SERVICE_T *service)
+{
+	char buf[80];
+	int len;
+
+	len = snprintf(buf, sizeof(buf), "Service %d: %s (ref %u)",
+		service->localport, srvstate_names[service->srvstate],
+		service->ref_count - 1); /*Don't include the lock just taken*/
+
+	if (service->srvstate != VCHIQ_SRVSTATE_FREE) {
+		char remoteport[30];
+		VCHIQ_SERVICE_QUOTA_T *service_quota =
+			&service->state->service_quotas[service->localport];
+		int fourcc = service->base.fourcc;
+		int tx_pending, rx_pending;
+		if (service->remoteport != VCHIQ_PORT_FREE) {
+			int len2 = snprintf(remoteport, sizeof(remoteport),
+				"%d", service->remoteport);
+			if (service->public_fourcc != VCHIQ_FOURCC_INVALID)
+				snprintf(remoteport + len2,
+					sizeof(remoteport) - len2,
+					" (client %x)", service->client_id);
+		} else
+			strcpy(remoteport, "n/a");
+
+		len += snprintf(buf + len, sizeof(buf) - len,
+			" '%c%c%c%c' remote %s (msg use %d/%d, slot use %d/%d)",
+			VCHIQ_FOURCC_AS_4CHARS(fourcc),
+			remoteport,
+			service_quota->message_use_count,
+			service_quota->message_quota,
+			service_quota->slot_use_count,
+			service_quota->slot_quota);
+
+		vchiq_dump(dump_context, buf, len + 1);
+
+		tx_pending = service->bulk_tx.local_insert -
+			service->bulk_tx.remote_insert;
+
+		rx_pending = service->bulk_rx.local_insert -
+			service->bulk_rx.remote_insert;
+
+		len = snprintf(buf, sizeof(buf),
+			"  Bulk: tx_pending=%d (size %d),"
+			" rx_pending=%d (size %d)",
+			tx_pending,
+			tx_pending ? service->bulk_tx.bulks[
+			BULK_INDEX(service->bulk_tx.remove)].size : 0,
+			rx_pending,
+			rx_pending ? service->bulk_rx.bulks[
+			BULK_INDEX(service->bulk_rx.remove)].size : 0);
+
+		if (VCHIQ_ENABLE_STATS) {
+			vchiq_dump(dump_context, buf, len + 1);
+
+			len = snprintf(buf, sizeof(buf),
+				"  Ctrl: tx_count=%d, tx_bytes=%llu, "
+				"rx_count=%d, rx_bytes=%llu",
+				service->stats.ctrl_tx_count,
+				service->stats.ctrl_tx_bytes,
+				service->stats.ctrl_rx_count,
+				service->stats.ctrl_rx_bytes);
+			vchiq_dump(dump_context, buf, len + 1);
+
+			len = snprintf(buf, sizeof(buf),
+				"  Bulk: tx_count=%d, tx_bytes=%llu, "
+				"rx_count=%d, rx_bytes=%llu",
+				service->stats.bulk_tx_count,
+				service->stats.bulk_tx_bytes,
+				service->stats.bulk_rx_count,
+				service->stats.bulk_rx_bytes);
+			vchiq_dump(dump_context, buf, len + 1);
+
+			len = snprintf(buf, sizeof(buf),
+				"  %d quota stalls, %d slot stalls, "
+				"%d bulk stalls, %d aborted, %d errors",
+				service->stats.quota_stalls,
+				service->stats.slot_stalls,
+				service->stats.bulk_stalls,
+				service->stats.bulk_aborted_count,
+				service->stats.error_count);
+		 }
+	}
+
+	vchiq_dump(dump_context, buf, len + 1);
+
+	if (service->srvstate != VCHIQ_SRVSTATE_FREE)
+		vchiq_dump_platform_service_state(dump_context, service);
+}
+
+
+void
+vchiq_loud_error_header(void)
+{
+	vchiq_log_error(vchiq_core_log_level,
+		"============================================================"
+		"================");
+	vchiq_log_error(vchiq_core_log_level,
+		"============================================================"
+		"================");
+	vchiq_log_error(vchiq_core_log_level, "=====");
+}
+
+void
+vchiq_loud_error_footer(void)
+{
+	vchiq_log_error(vchiq_core_log_level, "=====");
+	vchiq_log_error(vchiq_core_log_level,
+		"============================================================"
+		"================");
+	vchiq_log_error(vchiq_core_log_level,
+		"============================================================"
+		"================");
+}
+
+
+VCHIQ_STATUS_T vchiq_send_remote_use(VCHIQ_STATE_T *state)
+{
+	VCHIQ_STATUS_T status = VCHIQ_RETRY;
+	if (state->conn_state != VCHIQ_CONNSTATE_DISCONNECTED)
+		status = queue_message(state, NULL,
+			VCHIQ_MAKE_MSG(VCHIQ_MSG_REMOTE_USE, 0, 0),
+			NULL, 0, 0, 0);
+	return status;
+}
+
+VCHIQ_STATUS_T vchiq_send_remote_release(VCHIQ_STATE_T *state)
+{
+	VCHIQ_STATUS_T status = VCHIQ_RETRY;
+	if (state->conn_state != VCHIQ_CONNSTATE_DISCONNECTED)
+		status = queue_message(state, NULL,
+			VCHIQ_MAKE_MSG(VCHIQ_MSG_REMOTE_RELEASE, 0, 0),
+			NULL, 0, 0, 0);
+	return status;
+}
+
+VCHIQ_STATUS_T vchiq_send_remote_use_active(VCHIQ_STATE_T *state)
+{
+	VCHIQ_STATUS_T status = VCHIQ_RETRY;
+	if (state->conn_state != VCHIQ_CONNSTATE_DISCONNECTED)
+		status = queue_message(state, NULL,
+			VCHIQ_MAKE_MSG(VCHIQ_MSG_REMOTE_USE_ACTIVE, 0, 0),
+			NULL, 0, 0, 0);
+	return status;
+}
+
+void vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem,
+	size_t numBytes)
+{
+	const uint8_t  *mem = (const uint8_t *)voidMem;
+	size_t          offset;
+	char            lineBuf[100];
+	char           *s;
+
+	while (numBytes > 0) {
+		s = lineBuf;
+
+		for (offset = 0; offset < 16; offset++) {
+			if (offset < numBytes)
+				s += snprintf(s, 4, "%02x ", mem[offset]);
+			else
+				s += snprintf(s, 4, "   ");
+		}
+
+		for (offset = 0; offset < 16; offset++) {
+			if (offset < numBytes) {
+				uint8_t ch = mem[offset];
+
+				if ((ch < ' ') || (ch > '~'))
+					ch = '.';
+				*s++ = (char)ch;
+			}
+		}
+		*s++ = '\0';
+
+		if ((label != NULL) && (*label != '\0'))
+			vchiq_log_trace(VCHIQ_LOG_TRACE,
+				"%s: %08x: %s", label, addr, lineBuf);
+		else
+			vchiq_log_trace(VCHIQ_LOG_TRACE,
+				"%08x: %s", addr, lineBuf);
+
+		addr += 16;
+		mem += 16;
+		if (numBytes > 16)
+			numBytes -= 16;
+		else
+			numBytes = 0;
+	}
+}
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h
new file mode 100644
index 000000000000..9be484c776d0
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h
@@ -0,0 +1,712 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_CORE_H
+#define VCHIQ_CORE_H
+
+#include <linux/mutex.h>
+#include <linux/semaphore.h>
+#include <linux/kthread.h>
+
+#include "vchiq_cfg.h"
+
+#include "vchiq.h"
+
+/* Run time control of log level, based on KERN_XXX level. */
+#define VCHIQ_LOG_DEFAULT  4
+#define VCHIQ_LOG_ERROR    3
+#define VCHIQ_LOG_WARNING  4
+#define VCHIQ_LOG_INFO     6
+#define VCHIQ_LOG_TRACE    7
+
+#define VCHIQ_LOG_PREFIX   KERN_INFO "vchiq: "
+
+#ifndef vchiq_log_error
+#define vchiq_log_error(cat, fmt, ...) \
+	do { if (cat >= VCHIQ_LOG_ERROR) \
+		printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
+#endif
+#ifndef vchiq_log_warning
+#define vchiq_log_warning(cat, fmt, ...) \
+	do { if (cat >= VCHIQ_LOG_WARNING) \
+		 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
+#endif
+#ifndef vchiq_log_info
+#define vchiq_log_info(cat, fmt, ...) \
+	do { if (cat >= VCHIQ_LOG_INFO) \
+		printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
+#endif
+#ifndef vchiq_log_trace
+#define vchiq_log_trace(cat, fmt, ...) \
+	do { if (cat >= VCHIQ_LOG_TRACE) \
+		printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
+#endif
+
+#define vchiq_loud_error(...) \
+	vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__)
+
+#ifndef vchiq_static_assert
+#define vchiq_static_assert(cond) __attribute__((unused)) \
+	extern int vchiq_static_assert[(cond) ? 1 : -1]
+#endif
+
+#define IS_POW2(x) (x && ((x & (x - 1)) == 0))
+
+/* Ensure that the slot size and maximum number of slots are powers of 2 */
+vchiq_static_assert(IS_POW2(VCHIQ_SLOT_SIZE));
+vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS));
+vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS_PER_SIDE));
+
+#define VCHIQ_SLOT_MASK        (VCHIQ_SLOT_SIZE - 1)
+#define VCHIQ_SLOT_QUEUE_MASK  (VCHIQ_MAX_SLOTS_PER_SIDE - 1)
+#define VCHIQ_SLOT_ZERO_SLOTS  ((sizeof(VCHIQ_SLOT_ZERO_T) + \
+	VCHIQ_SLOT_SIZE - 1) / VCHIQ_SLOT_SIZE)
+
+#define VCHIQ_MSG_PADDING            0  /* -                                 */
+#define VCHIQ_MSG_CONNECT            1  /* -                                 */
+#define VCHIQ_MSG_OPEN               2  /* + (srcport, -), fourcc, client_id */
+#define VCHIQ_MSG_OPENACK            3  /* + (srcport, dstport)              */
+#define VCHIQ_MSG_CLOSE              4  /* + (srcport, dstport)              */
+#define VCHIQ_MSG_DATA               5  /* + (srcport, dstport)              */
+#define VCHIQ_MSG_BULK_RX            6  /* + (srcport, dstport), data, size  */
+#define VCHIQ_MSG_BULK_TX            7  /* + (srcport, dstport), data, size  */
+#define VCHIQ_MSG_BULK_RX_DONE       8  /* + (srcport, dstport), actual      */
+#define VCHIQ_MSG_BULK_TX_DONE       9  /* + (srcport, dstport), actual      */
+#define VCHIQ_MSG_PAUSE             10  /* -                                 */
+#define VCHIQ_MSG_RESUME            11  /* -                                 */
+#define VCHIQ_MSG_REMOTE_USE        12  /* -                                 */
+#define VCHIQ_MSG_REMOTE_RELEASE    13  /* -                                 */
+#define VCHIQ_MSG_REMOTE_USE_ACTIVE 14  /* -                                 */
+
+#define VCHIQ_PORT_MAX                 (VCHIQ_MAX_SERVICES - 1)
+#define VCHIQ_PORT_FREE                0x1000
+#define VCHIQ_PORT_IS_VALID(port)      (port < VCHIQ_PORT_FREE)
+#define VCHIQ_MAKE_MSG(type, srcport, dstport) \
+	((type<<24) | (srcport<<12) | (dstport<<0))
+#define VCHIQ_MSG_TYPE(msgid)          ((unsigned int)msgid >> 24)
+#define VCHIQ_MSG_SRCPORT(msgid) \
+	(unsigned short)(((unsigned int)msgid >> 12) & 0xfff)
+#define VCHIQ_MSG_DSTPORT(msgid) \
+	((unsigned short)msgid & 0xfff)
+
+#define VCHIQ_FOURCC_AS_4CHARS(fourcc)	\
+	((fourcc) >> 24) & 0xff, \
+	((fourcc) >> 16) & 0xff, \
+	((fourcc) >>  8) & 0xff, \
+	(fourcc) & 0xff
+
+/* Ensure the fields are wide enough */
+vchiq_static_assert(VCHIQ_MSG_SRCPORT(VCHIQ_MAKE_MSG(0, 0, VCHIQ_PORT_MAX))
+	== 0);
+vchiq_static_assert(VCHIQ_MSG_TYPE(VCHIQ_MAKE_MSG(0, VCHIQ_PORT_MAX, 0)) == 0);
+vchiq_static_assert((unsigned int)VCHIQ_PORT_MAX <
+	(unsigned int)VCHIQ_PORT_FREE);
+
+#define VCHIQ_MSGID_PADDING            VCHIQ_MAKE_MSG(VCHIQ_MSG_PADDING, 0, 0)
+#define VCHIQ_MSGID_CLAIMED            0x40000000
+
+#define VCHIQ_FOURCC_INVALID           0x00000000
+#define VCHIQ_FOURCC_IS_LEGAL(fourcc)  (fourcc != VCHIQ_FOURCC_INVALID)
+
+#define VCHIQ_BULK_ACTUAL_ABORTED -1
+
+typedef uint32_t BITSET_T;
+
+vchiq_static_assert((sizeof(BITSET_T) * 8) == 32);
+
+#define BITSET_SIZE(b)        ((b + 31) >> 5)
+#define BITSET_WORD(b)        (b >> 5)
+#define BITSET_BIT(b)         (1 << (b & 31))
+#define BITSET_ZERO(bs)       memset(bs, 0, sizeof(bs))
+#define BITSET_IS_SET(bs, b)  (bs[BITSET_WORD(b)] & BITSET_BIT(b))
+#define BITSET_SET(bs, b)     (bs[BITSET_WORD(b)] |= BITSET_BIT(b))
+#define BITSET_CLR(bs, b)     (bs[BITSET_WORD(b)] &= ~BITSET_BIT(b))
+
+#if VCHIQ_ENABLE_STATS
+#define VCHIQ_STATS_INC(state, stat) (state->stats. stat++)
+#define VCHIQ_SERVICE_STATS_INC(service, stat) (service->stats. stat++)
+#define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) \
+	(service->stats. stat += addend)
+#else
+#define VCHIQ_STATS_INC(state, stat) ((void)0)
+#define VCHIQ_SERVICE_STATS_INC(service, stat) ((void)0)
+#define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) ((void)0)
+#endif
+
+enum {
+	DEBUG_ENTRIES,
+#if VCHIQ_ENABLE_DEBUG
+	DEBUG_SLOT_HANDLER_COUNT,
+	DEBUG_SLOT_HANDLER_LINE,
+	DEBUG_PARSE_LINE,
+	DEBUG_PARSE_HEADER,
+	DEBUG_PARSE_MSGID,
+	DEBUG_AWAIT_COMPLETION_LINE,
+	DEBUG_DEQUEUE_MESSAGE_LINE,
+	DEBUG_SERVICE_CALLBACK_LINE,
+	DEBUG_MSG_QUEUE_FULL_COUNT,
+	DEBUG_COMPLETION_QUEUE_FULL_COUNT,
+#endif
+	DEBUG_MAX
+};
+
+#if VCHIQ_ENABLE_DEBUG
+
+#define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug;
+#define DEBUG_TRACE(d) \
+	do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(); } while (0)
+#define DEBUG_VALUE(d, v) \
+	do { debug_ptr[DEBUG_ ## d] = (v); dsb(); } while (0)
+#define DEBUG_COUNT(d) \
+	do { debug_ptr[DEBUG_ ## d]++; dsb(); } while (0)
+
+#else /* VCHIQ_ENABLE_DEBUG */
+
+#define DEBUG_INITIALISE(local)
+#define DEBUG_TRACE(d)
+#define DEBUG_VALUE(d, v)
+#define DEBUG_COUNT(d)
+
+#endif /* VCHIQ_ENABLE_DEBUG */
+
+typedef enum {
+	VCHIQ_CONNSTATE_DISCONNECTED,
+	VCHIQ_CONNSTATE_CONNECTING,
+	VCHIQ_CONNSTATE_CONNECTED,
+	VCHIQ_CONNSTATE_PAUSING,
+	VCHIQ_CONNSTATE_PAUSE_SENT,
+	VCHIQ_CONNSTATE_PAUSED,
+	VCHIQ_CONNSTATE_RESUMING,
+	VCHIQ_CONNSTATE_PAUSE_TIMEOUT,
+	VCHIQ_CONNSTATE_RESUME_TIMEOUT
+} VCHIQ_CONNSTATE_T;
+
+enum {
+	VCHIQ_SRVSTATE_FREE,
+	VCHIQ_SRVSTATE_HIDDEN,
+	VCHIQ_SRVSTATE_LISTENING,
+	VCHIQ_SRVSTATE_OPENING,
+	VCHIQ_SRVSTATE_OPEN,
+	VCHIQ_SRVSTATE_OPENSYNC,
+	VCHIQ_SRVSTATE_CLOSESENT,
+	VCHIQ_SRVSTATE_CLOSERECVD,
+	VCHIQ_SRVSTATE_CLOSEWAIT,
+	VCHIQ_SRVSTATE_CLOSED
+};
+
+enum {
+	VCHIQ_POLL_TERMINATE,
+	VCHIQ_POLL_REMOVE,
+	VCHIQ_POLL_TXNOTIFY,
+	VCHIQ_POLL_RXNOTIFY,
+	VCHIQ_POLL_COUNT
+};
+
+typedef enum {
+	VCHIQ_BULK_TRANSMIT,
+	VCHIQ_BULK_RECEIVE
+} VCHIQ_BULK_DIR_T;
+
+typedef void (*VCHIQ_USERDATA_TERM_T)(void *userdata);
+
+typedef struct vchiq_bulk_struct {
+	short mode;
+	short dir;
+	void *userdata;
+	VCHI_MEM_HANDLE_T handle;
+	void *data;
+	int size;
+	void *remote_data;
+	int remote_size;
+	int actual;
+} VCHIQ_BULK_T;
+
+typedef struct vchiq_bulk_queue_struct {
+	int local_insert;  /* Where to insert the next local bulk */
+	int remote_insert; /* Where to insert the next remote bulk (master) */
+	int process;       /* Bulk to transfer next */
+	int remote_notify; /* Bulk to notify the remote client of next (mstr) */
+	int remove;        /* Bulk to notify the local client of, and remove,
+			   ** next */
+	VCHIQ_BULK_T bulks[VCHIQ_NUM_SERVICE_BULKS];
+} VCHIQ_BULK_QUEUE_T;
+
+typedef struct remote_event_struct {
+	int armed;
+	int fired;
+	struct semaphore *event;
+} REMOTE_EVENT_T;
+
+typedef struct opaque_platform_state_t *VCHIQ_PLATFORM_STATE_T;
+
+typedef struct vchiq_state_struct VCHIQ_STATE_T;
+
+typedef struct vchiq_slot_struct {
+	char data[VCHIQ_SLOT_SIZE];
+} VCHIQ_SLOT_T;
+
+typedef struct vchiq_slot_info_struct {
+	/* Use two counters rather than one to avoid the need for a mutex. */
+	short use_count;
+	short release_count;
+} VCHIQ_SLOT_INFO_T;
+
+typedef struct vchiq_service_struct {
+	VCHIQ_SERVICE_BASE_T base;
+	VCHIQ_SERVICE_HANDLE_T handle;
+	unsigned int ref_count;
+	int srvstate;
+	VCHIQ_USERDATA_TERM_T userdata_term;
+	unsigned int localport;
+	unsigned int remoteport;
+	int public_fourcc;
+	int client_id;
+	char auto_close;
+	char sync;
+	char closing;
+	char trace;
+	atomic_t poll_flags;
+	short version;
+	short version_min;
+	short peer_version;
+
+	VCHIQ_STATE_T *state;
+	VCHIQ_INSTANCE_T instance;
+
+	int service_use_count;
+
+	VCHIQ_BULK_QUEUE_T bulk_tx;
+	VCHIQ_BULK_QUEUE_T bulk_rx;
+
+	struct semaphore remove_event;
+	struct semaphore bulk_remove_event;
+	struct mutex bulk_mutex;
+
+	struct service_stats_struct {
+		int quota_stalls;
+		int slot_stalls;
+		int bulk_stalls;
+		int error_count;
+		int ctrl_tx_count;
+		int ctrl_rx_count;
+		int bulk_tx_count;
+		int bulk_rx_count;
+		int bulk_aborted_count;
+		uint64_t ctrl_tx_bytes;
+		uint64_t ctrl_rx_bytes;
+		uint64_t bulk_tx_bytes;
+		uint64_t bulk_rx_bytes;
+	} stats;
+} VCHIQ_SERVICE_T;
+
+/* The quota information is outside VCHIQ_SERVICE_T so that it can be
+	statically allocated, since for accounting reasons a service's slot
+	usage is carried over between users of the same port number.
+ */
+typedef struct vchiq_service_quota_struct {
+	unsigned short slot_quota;
+	unsigned short slot_use_count;
+	unsigned short message_quota;
+	unsigned short message_use_count;
+	struct semaphore quota_event;
+	int previous_tx_index;
+} VCHIQ_SERVICE_QUOTA_T;
+
+typedef struct vchiq_shared_state_struct {
+
+	/* A non-zero value here indicates that the content is valid. */
+	int initialised;
+
+	/* The first and last (inclusive) slots allocated to the owner. */
+	int slot_first;
+	int slot_last;
+
+	/* The slot allocated to synchronous messages from the owner. */
+	int slot_sync;
+
+	/* Signalling this event indicates that owner's slot handler thread
+	** should run. */
+	REMOTE_EVENT_T trigger;
+
+	/* Indicates the byte position within the stream where the next message
+	** will be written. The least significant bits are an index into the
+	** slot. The next bits are the index of the slot in slot_queue. */
+	int tx_pos;
+
+	/* This event should be signalled when a slot is recycled. */
+	REMOTE_EVENT_T recycle;
+
+	/* The slot_queue index where the next recycled slot will be written. */
+	int slot_queue_recycle;
+
+	/* This event should be signalled when a synchronous message is sent. */
+	REMOTE_EVENT_T sync_trigger;
+
+	/* This event should be signalled when a synchronous message has been
+	** released. */
+	REMOTE_EVENT_T sync_release;
+
+	/* A circular buffer of slot indexes. */
+	int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE];
+
+	/* Debugging state */
+	int debug[DEBUG_MAX];
+} VCHIQ_SHARED_STATE_T;
+
+typedef struct vchiq_slot_zero_struct {
+	int magic;
+	short version;
+	short version_min;
+	int slot_zero_size;
+	int slot_size;
+	int max_slots;
+	int max_slots_per_side;
+	int platform_data[2];
+	VCHIQ_SHARED_STATE_T master;
+	VCHIQ_SHARED_STATE_T slave;
+	VCHIQ_SLOT_INFO_T slots[VCHIQ_MAX_SLOTS];
+} VCHIQ_SLOT_ZERO_T;
+
+struct vchiq_state_struct {
+	int id;
+	int initialised;
+	VCHIQ_CONNSTATE_T conn_state;
+	int is_master;
+	short version_common;
+
+	VCHIQ_SHARED_STATE_T *local;
+	VCHIQ_SHARED_STATE_T *remote;
+	VCHIQ_SLOT_T *slot_data;
+
+	unsigned short default_slot_quota;
+	unsigned short default_message_quota;
+
+	/* Event indicating connect message received */
+	struct semaphore connect;
+
+	/* Mutex protecting services */
+	struct mutex mutex;
+	VCHIQ_INSTANCE_T *instance;
+
+	/* Processes incoming messages */
+	struct task_struct *slot_handler_thread;
+
+	/* Processes recycled slots */
+	struct task_struct *recycle_thread;
+
+	/* Processes synchronous messages */
+	struct task_struct *sync_thread;
+
+	/* Local implementation of the trigger remote event */
+	struct semaphore trigger_event;
+
+	/* Local implementation of the recycle remote event */
+	struct semaphore recycle_event;
+
+	/* Local implementation of the sync trigger remote event */
+	struct semaphore sync_trigger_event;
+
+	/* Local implementation of the sync release remote event */
+	struct semaphore sync_release_event;
+
+	char *tx_data;
+	char *rx_data;
+	VCHIQ_SLOT_INFO_T *rx_info;
+
+	struct mutex slot_mutex;
+
+	struct mutex recycle_mutex;
+
+	struct mutex sync_mutex;
+
+	struct mutex bulk_transfer_mutex;
+
+	/* Indicates the byte position within the stream from where the next
+	** message will be read. The least significant bits are an index into
+	** the slot.The next bits are the index of the slot in
+	** remote->slot_queue. */
+	int rx_pos;
+
+	/* A cached copy of local->tx_pos. Only write to local->tx_pos, and read
+		from remote->tx_pos. */
+	int local_tx_pos;
+
+	/* The slot_queue index of the slot to become available next. */
+	int slot_queue_available;
+
+	/* A flag to indicate if any poll has been requested */
+	int poll_needed;
+
+	/* Ths index of the previous slot used for data messages. */
+	int previous_data_index;
+
+	/* The number of slots occupied by data messages. */
+	unsigned short data_use_count;
+
+	/* The maximum number of slots to be occupied by data messages. */
+	unsigned short data_quota;
+
+	/* An array of bit sets indicating which services must be polled. */
+	atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
+
+	/* The number of the first unused service */
+	int unused_service;
+
+	/* Signalled when a free slot becomes available. */
+	struct semaphore slot_available_event;
+
+	struct semaphore slot_remove_event;
+
+	/* Signalled when a free data slot becomes available. */
+	struct semaphore data_quota_event;
+
+	/* Incremented when there are bulk transfers which cannot be processed
+	 * whilst paused and must be processed on resume */
+	int deferred_bulks;
+
+	struct state_stats_struct {
+		int slot_stalls;
+		int data_stalls;
+		int ctrl_tx_count;
+		int ctrl_rx_count;
+		int error_count;
+	} stats;
+
+	VCHIQ_SERVICE_T * services[VCHIQ_MAX_SERVICES];
+	VCHIQ_SERVICE_QUOTA_T service_quotas[VCHIQ_MAX_SERVICES];
+	VCHIQ_SLOT_INFO_T slot_info[VCHIQ_MAX_SLOTS];
+
+	VCHIQ_PLATFORM_STATE_T platform_state;
+};
+
+struct bulk_waiter {
+	VCHIQ_BULK_T *bulk;
+	struct semaphore event;
+	int actual;
+};
+
+extern spinlock_t bulk_waiter_spinlock;
+
+extern int vchiq_core_log_level;
+extern int vchiq_core_msg_log_level;
+extern int vchiq_sync_log_level;
+
+extern VCHIQ_STATE_T *vchiq_states[VCHIQ_MAX_STATES];
+
+extern const char *
+get_conn_state_name(VCHIQ_CONNSTATE_T conn_state);
+
+extern VCHIQ_SLOT_ZERO_T *
+vchiq_init_slots(void *mem_base, int mem_size);
+
+extern VCHIQ_STATUS_T
+vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
+	int is_master);
+
+extern VCHIQ_STATUS_T
+vchiq_connect_internal(VCHIQ_STATE_T *state, VCHIQ_INSTANCE_T instance);
+
+extern VCHIQ_SERVICE_T *
+vchiq_add_service_internal(VCHIQ_STATE_T *state,
+	const VCHIQ_SERVICE_PARAMS_T *params, int srvstate,
+	VCHIQ_INSTANCE_T instance, VCHIQ_USERDATA_TERM_T userdata_term);
+
+extern VCHIQ_STATUS_T
+vchiq_open_service_internal(VCHIQ_SERVICE_T *service, int client_id);
+
+extern VCHIQ_STATUS_T
+vchiq_close_service_internal(VCHIQ_SERVICE_T *service, int close_recvd);
+
+extern void
+vchiq_terminate_service_internal(VCHIQ_SERVICE_T *service);
+
+extern void
+vchiq_free_service_internal(VCHIQ_SERVICE_T *service);
+
+extern VCHIQ_STATUS_T
+vchiq_shutdown_internal(VCHIQ_STATE_T *state, VCHIQ_INSTANCE_T instance);
+
+extern VCHIQ_STATUS_T
+vchiq_pause_internal(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_resume_internal(VCHIQ_STATE_T *state);
+
+extern void
+remote_event_pollall(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle,
+	VCHI_MEM_HANDLE_T memhandle, void *offset, int size, void *userdata,
+	VCHIQ_BULK_MODE_T mode, VCHIQ_BULK_DIR_T dir);
+
+extern void
+vchiq_dump_state(void *dump_context, VCHIQ_STATE_T *state);
+
+extern void
+vchiq_dump_service_state(void *dump_context, VCHIQ_SERVICE_T *service);
+
+extern void
+vchiq_loud_error_header(void);
+
+extern void
+vchiq_loud_error_footer(void);
+
+extern void
+request_poll(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service, int poll_type);
+
+static inline VCHIQ_SERVICE_T *
+handle_to_service(VCHIQ_SERVICE_HANDLE_T handle)
+{
+	VCHIQ_STATE_T *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) &
+		(VCHIQ_MAX_STATES - 1)];
+	if (!state)
+		return NULL;
+
+	return state->services[handle & (VCHIQ_MAX_SERVICES - 1)];
+}
+
+extern VCHIQ_SERVICE_T *
+find_service_by_handle(VCHIQ_SERVICE_HANDLE_T handle);
+
+extern VCHIQ_SERVICE_T *
+find_service_by_port(VCHIQ_STATE_T *state, int localport);
+
+extern VCHIQ_SERVICE_T *
+find_service_for_instance(VCHIQ_INSTANCE_T instance,
+	VCHIQ_SERVICE_HANDLE_T handle);
+
+extern VCHIQ_SERVICE_T *
+find_closed_service_for_instance(VCHIQ_INSTANCE_T instance,
+	VCHIQ_SERVICE_HANDLE_T handle);
+
+extern VCHIQ_SERVICE_T *
+next_service_by_instance(VCHIQ_STATE_T *state, VCHIQ_INSTANCE_T instance,
+	int *pidx);
+
+extern void
+lock_service(VCHIQ_SERVICE_T *service);
+
+extern void
+unlock_service(VCHIQ_SERVICE_T *service);
+
+/* The following functions are called from vchiq_core, and external
+** implementations must be provided. */
+
+extern VCHIQ_STATUS_T
+vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk,
+	VCHI_MEM_HANDLE_T memhandle, void *offset, int size, int dir);
+
+extern void
+vchiq_transfer_bulk(VCHIQ_BULK_T *bulk);
+
+extern void
+vchiq_complete_bulk(VCHIQ_BULK_T *bulk);
+
+extern VCHIQ_STATUS_T
+vchiq_copy_from_user(void *dst, const void *src, int size);
+
+extern void
+remote_event_signal(REMOTE_EVENT_T *event);
+
+void
+vchiq_platform_check_suspend(VCHIQ_STATE_T *state);
+
+extern void
+vchiq_platform_paused(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_platform_resume(VCHIQ_STATE_T *state);
+
+extern void
+vchiq_platform_resumed(VCHIQ_STATE_T *state);
+
+extern void
+vchiq_dump(void *dump_context, const char *str, int len);
+
+extern void
+vchiq_dump_platform_state(void *dump_context);
+
+extern void
+vchiq_dump_platform_instances(void *dump_context);
+
+extern void
+vchiq_dump_platform_service_state(void *dump_context,
+	VCHIQ_SERVICE_T *service);
+
+extern VCHIQ_STATUS_T
+vchiq_use_service_internal(VCHIQ_SERVICE_T *service);
+
+extern VCHIQ_STATUS_T
+vchiq_release_service_internal(VCHIQ_SERVICE_T *service);
+
+extern void
+vchiq_on_remote_use(VCHIQ_STATE_T *state);
+
+extern void
+vchiq_on_remote_release(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_platform_init_state(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_check_service(VCHIQ_SERVICE_T *service);
+
+extern void
+vchiq_on_remote_use_active(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_send_remote_use(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_send_remote_release(VCHIQ_STATE_T *state);
+
+extern VCHIQ_STATUS_T
+vchiq_send_remote_use_active(VCHIQ_STATE_T *state);
+
+extern void
+vchiq_platform_conn_state_changed(VCHIQ_STATE_T *state,
+	VCHIQ_CONNSTATE_T oldstate, VCHIQ_CONNSTATE_T newstate);
+
+extern void
+vchiq_platform_handle_timeout(VCHIQ_STATE_T *state);
+
+extern void
+vchiq_set_conn_state(VCHIQ_STATE_T *state, VCHIQ_CONNSTATE_T newstate);
+
+
+extern void
+vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem,
+	size_t numBytes);
+
+#endif
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.c
new file mode 100644
index 000000000000..7e032130d967
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.c
@@ -0,0 +1,383 @@
+/**
+ * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved.
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <linux/debugfs.h>
+#include "vchiq_core.h"
+#include "vchiq_arm.h"
+#include "vchiq_debugfs.h"
+
+#ifdef CONFIG_DEBUG_FS
+
+/****************************************************************************
+*
+*   log category entries
+*
+***************************************************************************/
+#define DEBUGFS_WRITE_BUF_SIZE 256
+
+#define VCHIQ_LOG_ERROR_STR   "error"
+#define VCHIQ_LOG_WARNING_STR "warning"
+#define VCHIQ_LOG_INFO_STR    "info"
+#define VCHIQ_LOG_TRACE_STR   "trace"
+
+
+/* Top-level debug info */
+struct vchiq_debugfs_info {
+	/* Global 'vchiq' debugfs entry used by all instances */
+	struct dentry *vchiq_cfg_dir;
+
+	/* one entry per client process */
+	struct dentry *clients;
+
+	/* log categories */
+	struct dentry *log_categories;
+};
+
+static struct vchiq_debugfs_info debugfs_info;
+
+/* Log category debugfs entries */
+struct vchiq_debugfs_log_entry {
+	const char *name;
+	int *plevel;
+	struct dentry *dir;
+};
+
+static struct vchiq_debugfs_log_entry vchiq_debugfs_log_entries[] = {
+	{ "core", &vchiq_core_log_level },
+	{ "msg",  &vchiq_core_msg_log_level },
+	{ "sync", &vchiq_sync_log_level },
+	{ "susp", &vchiq_susp_log_level },
+	{ "arm",  &vchiq_arm_log_level },
+};
+static int n_log_entries =
+	sizeof(vchiq_debugfs_log_entries)/sizeof(vchiq_debugfs_log_entries[0]);
+
+
+static struct dentry *vchiq_clients_top(void);
+static struct dentry *vchiq_debugfs_top(void);
+
+static int debugfs_log_show(struct seq_file *f, void *offset)
+{
+	int *levp = f->private;
+	char *log_value = NULL;
+
+	switch (*levp) {
+	case VCHIQ_LOG_ERROR:
+		log_value = VCHIQ_LOG_ERROR_STR;
+		break;
+	case VCHIQ_LOG_WARNING:
+		log_value = VCHIQ_LOG_WARNING_STR;
+		break;
+	case VCHIQ_LOG_INFO:
+		log_value = VCHIQ_LOG_INFO_STR;
+		break;
+	case VCHIQ_LOG_TRACE:
+		log_value = VCHIQ_LOG_TRACE_STR;
+		break;
+	default:
+		break;
+	}
+
+	seq_printf(f, "%s\n", log_value ? log_value : "(null)");
+
+	return 0;
+}
+
+static int debugfs_log_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, debugfs_log_show, inode->i_private);
+}
+
+static int debugfs_log_write(struct file *file,
+	const char __user *buffer,
+	size_t count, loff_t *ppos)
+{
+	struct seq_file *f = (struct seq_file *)file->private_data;
+	int *levp = f->private;
+	char kbuf[DEBUGFS_WRITE_BUF_SIZE + 1];
+
+	memset(kbuf, 0, DEBUGFS_WRITE_BUF_SIZE + 1);
+	if (count >= DEBUGFS_WRITE_BUF_SIZE)
+		count = DEBUGFS_WRITE_BUF_SIZE;
+
+	if (copy_from_user(kbuf, buffer, count) != 0)
+		return -EFAULT;
+	kbuf[count - 1] = 0;
+
+	if (strncmp("error", kbuf, strlen("error")) == 0)
+		*levp = VCHIQ_LOG_ERROR;
+	else if (strncmp("warning", kbuf, strlen("warning")) == 0)
+		*levp = VCHIQ_LOG_WARNING;
+	else if (strncmp("info", kbuf, strlen("info")) == 0)
+		*levp = VCHIQ_LOG_INFO;
+	else if (strncmp("trace", kbuf, strlen("trace")) == 0)
+		*levp = VCHIQ_LOG_TRACE;
+	else
+		*levp = VCHIQ_LOG_DEFAULT;
+
+	*ppos += count;
+
+	return count;
+}
+
+static const struct file_operations debugfs_log_fops = {
+	.owner		= THIS_MODULE,
+	.open		= debugfs_log_open,
+	.write		= debugfs_log_write,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+/* create an entry under <debugfs>/vchiq/log for each log category */
+static int vchiq_debugfs_create_log_entries(struct dentry *top)
+{
+	struct dentry *dir;
+	size_t i;
+	int ret = 0;
+	dir = debugfs_create_dir("log", vchiq_debugfs_top());
+	if (!dir)
+		return -ENOMEM;
+	debugfs_info.log_categories = dir;
+
+	for (i = 0; i < n_log_entries; i++) {
+		void *levp = (void *)vchiq_debugfs_log_entries[i].plevel;
+		dir = debugfs_create_file(vchiq_debugfs_log_entries[i].name,
+					  0644,
+					  debugfs_info.log_categories,
+					  levp,
+					  &debugfs_log_fops);
+		if (!dir) {
+			ret = -ENOMEM;
+			break;
+		}
+
+		vchiq_debugfs_log_entries[i].dir = dir;
+	}
+	return ret;
+}
+
+static int debugfs_usecount_show(struct seq_file *f, void *offset)
+{
+	VCHIQ_INSTANCE_T instance = f->private;
+	int use_count;
+
+	use_count = vchiq_instance_get_use_count(instance);
+	seq_printf(f, "%d\n", use_count);
+
+	return 0;
+}
+
+static int debugfs_usecount_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, debugfs_usecount_show, inode->i_private);
+}
+
+static const struct file_operations debugfs_usecount_fops = {
+	.owner		= THIS_MODULE,
+	.open		= debugfs_usecount_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int debugfs_trace_show(struct seq_file *f, void *offset)
+{
+	VCHIQ_INSTANCE_T instance = f->private;
+	int trace;
+
+	trace = vchiq_instance_get_trace(instance);
+	seq_printf(f, "%s\n", trace ? "Y" : "N");
+
+	return 0;
+}
+
+static int debugfs_trace_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, debugfs_trace_show, inode->i_private);
+}
+
+static int debugfs_trace_write(struct file *file,
+	const char __user *buffer,
+	size_t count, loff_t *ppos)
+{
+	struct seq_file *f = (struct seq_file *)file->private_data;
+	VCHIQ_INSTANCE_T instance = f->private;
+	char firstchar;
+
+	if (copy_from_user(&firstchar, buffer, 1) != 0)
+		return -EFAULT;
+
+	switch (firstchar) {
+	case 'Y':
+	case 'y':
+	case '1':
+		vchiq_instance_set_trace(instance, 1);
+		break;
+	case 'N':
+	case 'n':
+	case '0':
+		vchiq_instance_set_trace(instance, 0);
+		break;
+	default:
+		break;
+	}
+
+	*ppos += count;
+
+	return count;
+}
+
+static const struct file_operations debugfs_trace_fops = {
+	.owner		= THIS_MODULE,
+	.open		= debugfs_trace_open,
+	.write		= debugfs_trace_write,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+/* add an instance (process) to the debugfs entries */
+int vchiq_debugfs_add_instance(VCHIQ_INSTANCE_T instance)
+{
+	char pidstr[16];
+	struct dentry *top, *use_count, *trace;
+	struct dentry *clients = vchiq_clients_top();
+
+	snprintf(pidstr, sizeof(pidstr), "%d",
+		 vchiq_instance_get_pid(instance));
+
+	top = debugfs_create_dir(pidstr, clients);
+	if (!top)
+		goto fail_top;
+
+	use_count = debugfs_create_file("use_count",
+					0444, top,
+					instance,
+					&debugfs_usecount_fops);
+	if (!use_count)
+		goto fail_use_count;
+
+	trace = debugfs_create_file("trace",
+				    0644, top,
+				    instance,
+				    &debugfs_trace_fops);
+	if (!trace)
+		goto fail_trace;
+
+	vchiq_instance_get_debugfs_node(instance)->dentry = top;
+
+	return 0;
+
+fail_trace:
+	debugfs_remove(use_count);
+fail_use_count:
+	debugfs_remove(top);
+fail_top:
+	return -ENOMEM;
+}
+
+void vchiq_debugfs_remove_instance(VCHIQ_INSTANCE_T instance)
+{
+	VCHIQ_DEBUGFS_NODE_T *node = vchiq_instance_get_debugfs_node(instance);
+	debugfs_remove_recursive(node->dentry);
+}
+
+
+int vchiq_debugfs_init(void)
+{
+	BUG_ON(debugfs_info.vchiq_cfg_dir != NULL);
+
+	debugfs_info.vchiq_cfg_dir = debugfs_create_dir("vchiq", NULL);
+	if (debugfs_info.vchiq_cfg_dir == NULL)
+		goto fail;
+
+	debugfs_info.clients = debugfs_create_dir("clients",
+				vchiq_debugfs_top());
+	if (!debugfs_info.clients)
+		goto fail;
+
+	if (vchiq_debugfs_create_log_entries(vchiq_debugfs_top()) != 0)
+		goto fail;
+
+	return 0;
+
+fail:
+	vchiq_debugfs_deinit();
+	vchiq_log_error(vchiq_arm_log_level,
+		"%s: failed to create debugfs directory",
+		__func__);
+
+	return -ENOMEM;
+}
+
+/* remove all the debugfs entries */
+void vchiq_debugfs_deinit(void)
+{
+	debugfs_remove_recursive(vchiq_debugfs_top());
+}
+
+static struct dentry *vchiq_clients_top(void)
+{
+	return debugfs_info.clients;
+}
+
+static struct dentry *vchiq_debugfs_top(void)
+{
+	BUG_ON(debugfs_info.vchiq_cfg_dir == NULL);
+	return debugfs_info.vchiq_cfg_dir;
+}
+
+#else /* CONFIG_DEBUG_FS */
+
+int vchiq_debugfs_init(void)
+{
+	return 0;
+}
+
+void vchiq_debugfs_deinit(void)
+{
+}
+
+int vchiq_debugfs_add_instance(VCHIQ_INSTANCE_T instance)
+{
+	return 0;
+}
+
+void vchiq_debugfs_remove_instance(VCHIQ_INSTANCE_T instance)
+{
+}
+
+#endif /* CONFIG_DEBUG_FS */
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.h
new file mode 100644
index 000000000000..4d6a3788e9c5
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.h
@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_DEBUGFS_H
+#define VCHIQ_DEBUGFS_H
+
+#include "vchiq_core.h"
+
+typedef struct vchiq_debugfs_node_struct
+{
+    struct dentry *dentry;
+} VCHIQ_DEBUGFS_NODE_T;
+
+int vchiq_debugfs_init(void);
+
+void vchiq_debugfs_deinit(void);
+
+int vchiq_debugfs_add_instance(VCHIQ_INSTANCE_T instance);
+
+void vchiq_debugfs_remove_instance(VCHIQ_INSTANCE_T instance);
+
+#endif /* VCHIQ_DEBUGFS_H */
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_genversion b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_genversion
new file mode 100644
index 000000000000..9f5b6344b9b7
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_genversion
@@ -0,0 +1,87 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+#
+# Generate a version from available information
+#
+
+my $prefix = shift @ARGV;
+my $root = shift @ARGV;
+
+
+if ( not defined $root ) {
+	die "usage: $0 prefix root-dir\n";
+}
+
+if ( ! -d $root ) {
+	die "root directory $root not found\n";
+}
+
+my $version = "unknown";
+my $tainted = "";
+
+if ( -d "$root/.git" ) {
+	# attempt to work out git version. only do so
+	# on a linux build host, as cygwin builds are
+	# already slow enough
+
+	if ( -f "/usr/bin/git" || -f "/usr/local/bin/git" ) {
+		if (not open(F, "git --git-dir $root/.git rev-parse --verify HEAD|")) {
+			$version = "no git version";
+		}
+		else {
+			$version = <F>;
+			$version =~ s/[ \r\n]*$//;     # chomp may not be enough (cygwin).
+			$version =~ s/^[ \r\n]*//;     # chomp may not be enough (cygwin).
+		}
+
+		if (open(G, "git --git-dir $root/.git status --porcelain|")) {
+			$tainted = <G>;
+			$tainted =~ s/[ \r\n]*$//;     # chomp may not be enough (cygwin).
+			$tainted =~ s/^[ \r\n]*//;     # chomp may not be enough (cygwin).
+			if (length $tainted) {
+			$version = join ' ', $version, "(tainted)";
+		}
+		else {
+			$version = join ' ', $version, "(clean)";
+         }
+		}
+	}
+}
+
+my $hostname = `hostname`;
+$hostname =~ s/[ \r\n]*$//;     # chomp may not be enough (cygwin).
+$hostname =~ s/^[ \r\n]*//;     # chomp may not be enough (cygwin).
+
+
+print STDERR "Version $version\n";
+print <<EOF;
+#include "${prefix}_build_info.h"
+#include <linux/broadcom/vc_debug_sym.h>
+
+VC_DEBUG_DECLARE_STRING_VAR( ${prefix}_build_hostname, "$hostname" );
+VC_DEBUG_DECLARE_STRING_VAR( ${prefix}_build_version, "$version" );
+VC_DEBUG_DECLARE_STRING_VAR( ${prefix}_build_time,    __TIME__ );
+VC_DEBUG_DECLARE_STRING_VAR( ${prefix}_build_date,    __DATE__ );
+
+const char *vchiq_get_build_hostname( void )
+{
+   return vchiq_build_hostname;
+}
+
+const char *vchiq_get_build_version( void )
+{
+   return vchiq_build_version;
+}
+
+const char *vchiq_get_build_date( void )
+{
+   return vchiq_build_date;
+}
+
+const char *vchiq_get_build_time( void )
+{
+   return vchiq_build_time;
+}
+EOF
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h
new file mode 100644
index 000000000000..8067bbe7ce8d
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h
@@ -0,0 +1,189 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_IF_H
+#define VCHIQ_IF_H
+
+#include "interface/vchi/vchi_mh.h"
+
+#define VCHIQ_SERVICE_HANDLE_INVALID 0
+
+#define VCHIQ_SLOT_SIZE     4096
+#define VCHIQ_MAX_MSG_SIZE  (VCHIQ_SLOT_SIZE - sizeof(VCHIQ_HEADER_T))
+#define VCHIQ_CHANNEL_SIZE  VCHIQ_MAX_MSG_SIZE /* For backwards compatibility */
+
+#define VCHIQ_MAKE_FOURCC(x0, x1, x2, x3) \
+			(((x0) << 24) | ((x1) << 16) | ((x2) << 8) | (x3))
+#define VCHIQ_GET_SERVICE_USERDATA(service) vchiq_get_service_userdata(service)
+#define VCHIQ_GET_SERVICE_FOURCC(service)   vchiq_get_service_fourcc(service)
+
+typedef enum {
+	VCHIQ_SERVICE_OPENED,         /* service, -, -             */
+	VCHIQ_SERVICE_CLOSED,         /* service, -, -             */
+	VCHIQ_MESSAGE_AVAILABLE,      /* service, header, -        */
+	VCHIQ_BULK_TRANSMIT_DONE,     /* service, -, bulk_userdata */
+	VCHIQ_BULK_RECEIVE_DONE,      /* service, -, bulk_userdata */
+	VCHIQ_BULK_TRANSMIT_ABORTED,  /* service, -, bulk_userdata */
+	VCHIQ_BULK_RECEIVE_ABORTED    /* service, -, bulk_userdata */
+} VCHIQ_REASON_T;
+
+typedef enum {
+	VCHIQ_ERROR   = -1,
+	VCHIQ_SUCCESS = 0,
+	VCHIQ_RETRY   = 1
+} VCHIQ_STATUS_T;
+
+typedef enum {
+	VCHIQ_BULK_MODE_CALLBACK,
+	VCHIQ_BULK_MODE_BLOCKING,
+	VCHIQ_BULK_MODE_NOCALLBACK,
+	VCHIQ_BULK_MODE_WAITING		/* Reserved for internal use */
+} VCHIQ_BULK_MODE_T;
+
+typedef enum {
+	VCHIQ_SERVICE_OPTION_AUTOCLOSE,
+	VCHIQ_SERVICE_OPTION_SLOT_QUOTA,
+	VCHIQ_SERVICE_OPTION_MESSAGE_QUOTA,
+	VCHIQ_SERVICE_OPTION_SYNCHRONOUS,
+	VCHIQ_SERVICE_OPTION_TRACE
+} VCHIQ_SERVICE_OPTION_T;
+
+typedef struct vchiq_header_struct {
+	/* The message identifier - opaque to applications. */
+	int msgid;
+
+	/* Size of message data. */
+	unsigned int size;
+
+	char data[0];           /* message */
+} VCHIQ_HEADER_T;
+
+typedef struct {
+	const void *data;
+	unsigned int size;
+} VCHIQ_ELEMENT_T;
+
+typedef unsigned int VCHIQ_SERVICE_HANDLE_T;
+
+typedef VCHIQ_STATUS_T (*VCHIQ_CALLBACK_T)(VCHIQ_REASON_T, VCHIQ_HEADER_T *,
+	VCHIQ_SERVICE_HANDLE_T, void *);
+
+typedef struct vchiq_service_base_struct {
+	int fourcc;
+	VCHIQ_CALLBACK_T callback;
+	void *userdata;
+} VCHIQ_SERVICE_BASE_T;
+
+typedef struct vchiq_service_params_struct {
+	int fourcc;
+	VCHIQ_CALLBACK_T callback;
+	void *userdata;
+	short version;       /* Increment for non-trivial changes */
+	short version_min;   /* Update for incompatible changes */
+} VCHIQ_SERVICE_PARAMS_T;
+
+typedef struct vchiq_config_struct {
+	unsigned int max_msg_size;
+	unsigned int bulk_threshold; /* The message size above which it
+					is better to use a bulk transfer
+					(<= max_msg_size) */
+	unsigned int max_outstanding_bulks;
+	unsigned int max_services;
+	short version;      /* The version of VCHIQ */
+	short version_min;  /* The minimum compatible version of VCHIQ */
+} VCHIQ_CONFIG_T;
+
+typedef struct vchiq_instance_struct *VCHIQ_INSTANCE_T;
+typedef void (*VCHIQ_REMOTE_USE_CALLBACK_T)(void *cb_arg);
+
+extern VCHIQ_STATUS_T vchiq_initialise(VCHIQ_INSTANCE_T *pinstance);
+extern VCHIQ_STATUS_T vchiq_shutdown(VCHIQ_INSTANCE_T instance);
+extern VCHIQ_STATUS_T vchiq_connect(VCHIQ_INSTANCE_T instance);
+extern VCHIQ_STATUS_T vchiq_add_service(VCHIQ_INSTANCE_T instance,
+	const VCHIQ_SERVICE_PARAMS_T *params,
+	VCHIQ_SERVICE_HANDLE_T *pservice);
+extern VCHIQ_STATUS_T vchiq_open_service(VCHIQ_INSTANCE_T instance,
+	const VCHIQ_SERVICE_PARAMS_T *params,
+	VCHIQ_SERVICE_HANDLE_T *pservice);
+extern VCHIQ_STATUS_T vchiq_close_service(VCHIQ_SERVICE_HANDLE_T service);
+extern VCHIQ_STATUS_T vchiq_remove_service(VCHIQ_SERVICE_HANDLE_T service);
+extern VCHIQ_STATUS_T vchiq_use_service(VCHIQ_SERVICE_HANDLE_T service);
+extern VCHIQ_STATUS_T vchiq_use_service_no_resume(
+	VCHIQ_SERVICE_HANDLE_T service);
+extern VCHIQ_STATUS_T vchiq_release_service(VCHIQ_SERVICE_HANDLE_T service);
+
+extern VCHIQ_STATUS_T vchiq_queue_message(VCHIQ_SERVICE_HANDLE_T service,
+	const VCHIQ_ELEMENT_T *elements, unsigned int count);
+extern void           vchiq_release_message(VCHIQ_SERVICE_HANDLE_T service,
+	VCHIQ_HEADER_T *header);
+extern VCHIQ_STATUS_T vchiq_queue_bulk_transmit(VCHIQ_SERVICE_HANDLE_T service,
+	const void *data, unsigned int size, void *userdata);
+extern VCHIQ_STATUS_T vchiq_queue_bulk_receive(VCHIQ_SERVICE_HANDLE_T service,
+	void *data, unsigned int size, void *userdata);
+extern VCHIQ_STATUS_T vchiq_queue_bulk_transmit_handle(
+	VCHIQ_SERVICE_HANDLE_T service, VCHI_MEM_HANDLE_T handle,
+	const void *offset, unsigned int size, void *userdata);
+extern VCHIQ_STATUS_T vchiq_queue_bulk_receive_handle(
+	VCHIQ_SERVICE_HANDLE_T service, VCHI_MEM_HANDLE_T handle,
+	void *offset, unsigned int size, void *userdata);
+extern VCHIQ_STATUS_T vchiq_bulk_transmit(VCHIQ_SERVICE_HANDLE_T service,
+	const void *data, unsigned int size, void *userdata,
+	VCHIQ_BULK_MODE_T mode);
+extern VCHIQ_STATUS_T vchiq_bulk_receive(VCHIQ_SERVICE_HANDLE_T service,
+	void *data, unsigned int size, void *userdata,
+	VCHIQ_BULK_MODE_T mode);
+extern VCHIQ_STATUS_T vchiq_bulk_transmit_handle(VCHIQ_SERVICE_HANDLE_T service,
+	VCHI_MEM_HANDLE_T handle, const void *offset, unsigned int size,
+	void *userdata,	VCHIQ_BULK_MODE_T mode);
+extern VCHIQ_STATUS_T vchiq_bulk_receive_handle(VCHIQ_SERVICE_HANDLE_T service,
+	VCHI_MEM_HANDLE_T handle, void *offset, unsigned int size,
+	void *userdata, VCHIQ_BULK_MODE_T mode);
+extern int   vchiq_get_client_id(VCHIQ_SERVICE_HANDLE_T service);
+extern void *vchiq_get_service_userdata(VCHIQ_SERVICE_HANDLE_T service);
+extern int   vchiq_get_service_fourcc(VCHIQ_SERVICE_HANDLE_T service);
+extern VCHIQ_STATUS_T vchiq_get_config(VCHIQ_INSTANCE_T instance,
+	int config_size, VCHIQ_CONFIG_T *pconfig);
+extern VCHIQ_STATUS_T vchiq_set_service_option(VCHIQ_SERVICE_HANDLE_T service,
+	VCHIQ_SERVICE_OPTION_T option, int value);
+
+extern VCHIQ_STATUS_T vchiq_remote_use(VCHIQ_INSTANCE_T instance,
+	VCHIQ_REMOTE_USE_CALLBACK_T callback, void *cb_arg);
+extern VCHIQ_STATUS_T vchiq_remote_release(VCHIQ_INSTANCE_T instance);
+
+extern VCHIQ_STATUS_T vchiq_dump_phys_mem(VCHIQ_SERVICE_HANDLE_T service,
+	void *ptr, size_t num_bytes);
+
+extern VCHIQ_STATUS_T vchiq_get_peer_version(VCHIQ_SERVICE_HANDLE_T handle,
+      short *peer_version);
+
+#endif /* VCHIQ_IF_H */
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_ioctl.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_ioctl.h
new file mode 100644
index 000000000000..6137ae9de1c1
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_ioctl.h
@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_IOCTLS_H
+#define VCHIQ_IOCTLS_H
+
+#include <linux/ioctl.h>
+#include "vchiq_if.h"
+
+#define VCHIQ_IOC_MAGIC 0xc4
+#define VCHIQ_INVALID_HANDLE (~0)
+
+typedef struct {
+	VCHIQ_SERVICE_PARAMS_T params;
+	int is_open;
+	int is_vchi;
+	unsigned int handle;       /* OUT */
+} VCHIQ_CREATE_SERVICE_T;
+
+typedef struct {
+	unsigned int handle;
+	unsigned int count;
+	const VCHIQ_ELEMENT_T *elements;
+} VCHIQ_QUEUE_MESSAGE_T;
+
+typedef struct {
+	unsigned int handle;
+	void *data;
+	unsigned int size;
+	void *userdata;
+	VCHIQ_BULK_MODE_T mode;
+} VCHIQ_QUEUE_BULK_TRANSFER_T;
+
+typedef struct {
+	VCHIQ_REASON_T reason;
+	VCHIQ_HEADER_T *header;
+	void *service_userdata;
+	void *bulk_userdata;
+} VCHIQ_COMPLETION_DATA_T;
+
+typedef struct {
+	unsigned int count;
+	VCHIQ_COMPLETION_DATA_T *buf;
+	unsigned int msgbufsize;
+	unsigned int msgbufcount; /* IN/OUT */
+	void **msgbufs;
+} VCHIQ_AWAIT_COMPLETION_T;
+
+typedef struct {
+	unsigned int handle;
+	int blocking;
+	unsigned int bufsize;
+	void *buf;
+} VCHIQ_DEQUEUE_MESSAGE_T;
+
+typedef struct {
+	unsigned int config_size;
+	VCHIQ_CONFIG_T *pconfig;
+} VCHIQ_GET_CONFIG_T;
+
+typedef struct {
+	unsigned int handle;
+	VCHIQ_SERVICE_OPTION_T option;
+	int value;
+} VCHIQ_SET_SERVICE_OPTION_T;
+
+typedef struct {
+	void     *virt_addr;
+	size_t    num_bytes;
+} VCHIQ_DUMP_MEM_T;
+
+#define VCHIQ_IOC_CONNECT              _IO(VCHIQ_IOC_MAGIC,   0)
+#define VCHIQ_IOC_SHUTDOWN             _IO(VCHIQ_IOC_MAGIC,   1)
+#define VCHIQ_IOC_CREATE_SERVICE \
+	_IOWR(VCHIQ_IOC_MAGIC, 2, VCHIQ_CREATE_SERVICE_T)
+#define VCHIQ_IOC_REMOVE_SERVICE       _IO(VCHIQ_IOC_MAGIC,   3)
+#define VCHIQ_IOC_QUEUE_MESSAGE \
+	_IOW(VCHIQ_IOC_MAGIC,  4, VCHIQ_QUEUE_MESSAGE_T)
+#define VCHIQ_IOC_QUEUE_BULK_TRANSMIT \
+	_IOWR(VCHIQ_IOC_MAGIC, 5, VCHIQ_QUEUE_BULK_TRANSFER_T)
+#define VCHIQ_IOC_QUEUE_BULK_RECEIVE \
+	_IOWR(VCHIQ_IOC_MAGIC, 6, VCHIQ_QUEUE_BULK_TRANSFER_T)
+#define VCHIQ_IOC_AWAIT_COMPLETION \
+	_IOWR(VCHIQ_IOC_MAGIC, 7, VCHIQ_AWAIT_COMPLETION_T)
+#define VCHIQ_IOC_DEQUEUE_MESSAGE \
+	_IOWR(VCHIQ_IOC_MAGIC, 8, VCHIQ_DEQUEUE_MESSAGE_T)
+#define VCHIQ_IOC_GET_CLIENT_ID        _IO(VCHIQ_IOC_MAGIC,   9)
+#define VCHIQ_IOC_GET_CONFIG \
+	_IOWR(VCHIQ_IOC_MAGIC, 10, VCHIQ_GET_CONFIG_T)
+#define VCHIQ_IOC_CLOSE_SERVICE        _IO(VCHIQ_IOC_MAGIC,   11)
+#define VCHIQ_IOC_USE_SERVICE          _IO(VCHIQ_IOC_MAGIC,   12)
+#define VCHIQ_IOC_RELEASE_SERVICE      _IO(VCHIQ_IOC_MAGIC,   13)
+#define VCHIQ_IOC_SET_SERVICE_OPTION \
+	_IOW(VCHIQ_IOC_MAGIC,  14, VCHIQ_SET_SERVICE_OPTION_T)
+#define VCHIQ_IOC_DUMP_PHYS_MEM \
+	_IOW(VCHIQ_IOC_MAGIC,  15, VCHIQ_DUMP_MEM_T)
+#define VCHIQ_IOC_LIB_VERSION          _IO(VCHIQ_IOC_MAGIC,   16)
+#define VCHIQ_IOC_CLOSE_DELIVERED      _IO(VCHIQ_IOC_MAGIC,   17)
+#define VCHIQ_IOC_MAX                  17
+
+#endif
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_kern_lib.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_kern_lib.c
new file mode 100644
index 000000000000..25e7011edc50
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_kern_lib.c
@@ -0,0 +1,458 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* ---- Include Files ---------------------------------------------------- */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+
+#include "vchiq_core.h"
+#include "vchiq_arm.h"
+#include "vchiq_killable.h"
+
+/* ---- Public Variables ------------------------------------------------- */
+
+/* ---- Private Constants and Types -------------------------------------- */
+
+struct bulk_waiter_node {
+	struct bulk_waiter bulk_waiter;
+	int pid;
+	struct list_head list;
+};
+
+struct vchiq_instance_struct {
+	VCHIQ_STATE_T *state;
+
+	int connected;
+
+	struct list_head bulk_waiter_list;
+	struct mutex bulk_waiter_list_mutex;
+};
+
+static VCHIQ_STATUS_T
+vchiq_blocking_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle, void *data,
+	unsigned int size, VCHIQ_BULK_DIR_T dir);
+
+/****************************************************************************
+*
+*   vchiq_initialise
+*
+***************************************************************************/
+#define VCHIQ_INIT_RETRIES 10
+VCHIQ_STATUS_T vchiq_initialise(VCHIQ_INSTANCE_T *instanceOut)
+{
+	VCHIQ_STATUS_T status = VCHIQ_ERROR;
+	VCHIQ_STATE_T *state;
+	VCHIQ_INSTANCE_T instance = NULL;
+        int i;
+
+	vchiq_log_trace(vchiq_core_log_level, "%s called", __func__);
+
+        /* VideoCore may not be ready due to boot up timing.
+           It may never be ready if kernel and firmware are mismatched, so don't block forever. */
+        for (i=0; i<VCHIQ_INIT_RETRIES; i++) {
+		state = vchiq_get_state();
+		if (state)
+			break;
+		udelay(500);
+	}
+	if (i==VCHIQ_INIT_RETRIES) {
+		vchiq_log_error(vchiq_core_log_level,
+			"%s: videocore not initialized\n", __func__);
+		goto failed;
+	} else if (i>0) {
+		vchiq_log_warning(vchiq_core_log_level,
+			"%s: videocore initialized after %d retries\n", __func__, i);
+	}
+
+	instance = kzalloc(sizeof(*instance), GFP_KERNEL);
+	if (!instance) {
+		vchiq_log_error(vchiq_core_log_level,
+			"%s: error allocating vchiq instance\n", __func__);
+		goto failed;
+	}
+
+	instance->connected = 0;
+	instance->state = state;
+	mutex_init(&instance->bulk_waiter_list_mutex);
+	INIT_LIST_HEAD(&instance->bulk_waiter_list);
+
+	*instanceOut = instance;
+
+	status = VCHIQ_SUCCESS;
+
+failed:
+	vchiq_log_trace(vchiq_core_log_level,
+		"%s(%p): returning %d", __func__, instance, status);
+
+	return status;
+}
+EXPORT_SYMBOL(vchiq_initialise);
+
+/****************************************************************************
+*
+*   vchiq_shutdown
+*
+***************************************************************************/
+
+VCHIQ_STATUS_T vchiq_shutdown(VCHIQ_INSTANCE_T instance)
+{
+	VCHIQ_STATUS_T status;
+	VCHIQ_STATE_T *state = instance->state;
+
+	vchiq_log_trace(vchiq_core_log_level,
+		"%s(%p) called", __func__, instance);
+
+	if (mutex_lock_interruptible(&state->mutex) != 0)
+		return VCHIQ_RETRY;
+
+	/* Remove all services */
+	status = vchiq_shutdown_internal(state, instance);
+
+	mutex_unlock(&state->mutex);
+
+	vchiq_log_trace(vchiq_core_log_level,
+		"%s(%p): returning %d", __func__, instance, status);
+
+	if (status == VCHIQ_SUCCESS) {
+		struct list_head *pos, *next;
+		list_for_each_safe(pos, next,
+				&instance->bulk_waiter_list) {
+			struct bulk_waiter_node *waiter;
+			waiter = list_entry(pos,
+					struct bulk_waiter_node,
+					list);
+			list_del(pos);
+			vchiq_log_info(vchiq_arm_log_level,
+					"bulk_waiter - cleaned up %x "
+					"for pid %d",
+					(unsigned int)waiter, waiter->pid);
+			kfree(waiter);
+		}
+		kfree(instance);
+	}
+
+	return status;
+}
+EXPORT_SYMBOL(vchiq_shutdown);
+
+/****************************************************************************
+*
+*   vchiq_is_connected
+*
+***************************************************************************/
+
+int vchiq_is_connected(VCHIQ_INSTANCE_T instance)
+{
+	return instance->connected;
+}
+
+/****************************************************************************
+*
+*   vchiq_connect
+*
+***************************************************************************/
+
+VCHIQ_STATUS_T vchiq_connect(VCHIQ_INSTANCE_T instance)
+{
+	VCHIQ_STATUS_T status;
+	VCHIQ_STATE_T *state = instance->state;
+
+	vchiq_log_trace(vchiq_core_log_level,
+		"%s(%p) called", __func__, instance);
+
+	if (mutex_lock_interruptible(&state->mutex) != 0) {
+		vchiq_log_trace(vchiq_core_log_level,
+			"%s: call to mutex_lock failed", __func__);
+		status = VCHIQ_RETRY;
+		goto failed;
+	}
+	status = vchiq_connect_internal(state, instance);
+
+	if (status == VCHIQ_SUCCESS)
+		instance->connected = 1;
+
+	mutex_unlock(&state->mutex);
+
+failed:
+	vchiq_log_trace(vchiq_core_log_level,
+		"%s(%p): returning %d", __func__, instance, status);
+
+	return status;
+}
+EXPORT_SYMBOL(vchiq_connect);
+
+/****************************************************************************
+*
+*   vchiq_add_service
+*
+***************************************************************************/
+
+VCHIQ_STATUS_T vchiq_add_service(
+	VCHIQ_INSTANCE_T              instance,
+	const VCHIQ_SERVICE_PARAMS_T *params,
+	VCHIQ_SERVICE_HANDLE_T       *phandle)
+{
+	VCHIQ_STATUS_T status;
+	VCHIQ_STATE_T *state = instance->state;
+	VCHIQ_SERVICE_T *service = NULL;
+	int srvstate;
+
+	vchiq_log_trace(vchiq_core_log_level,
+		"%s(%p) called", __func__, instance);
+
+	*phandle = VCHIQ_SERVICE_HANDLE_INVALID;
+
+	srvstate = vchiq_is_connected(instance)
+		? VCHIQ_SRVSTATE_LISTENING
+		: VCHIQ_SRVSTATE_HIDDEN;
+
+	service = vchiq_add_service_internal(
+		state,
+		params,
+		srvstate,
+		instance,
+		NULL);
+
+	if (service) {
+		*phandle = service->handle;
+		status = VCHIQ_SUCCESS;
+	} else
+		status = VCHIQ_ERROR;
+
+	vchiq_log_trace(vchiq_core_log_level,
+		"%s(%p): returning %d", __func__, instance, status);
+
+	return status;
+}
+EXPORT_SYMBOL(vchiq_add_service);
+
+/****************************************************************************
+*
+*   vchiq_open_service
+*
+***************************************************************************/
+
+VCHIQ_STATUS_T vchiq_open_service(
+	VCHIQ_INSTANCE_T              instance,
+	const VCHIQ_SERVICE_PARAMS_T *params,
+	VCHIQ_SERVICE_HANDLE_T       *phandle)
+{
+	VCHIQ_STATUS_T   status = VCHIQ_ERROR;
+	VCHIQ_STATE_T   *state = instance->state;
+	VCHIQ_SERVICE_T *service = NULL;
+
+	vchiq_log_trace(vchiq_core_log_level,
+		"%s(%p) called", __func__, instance);
+
+	*phandle = VCHIQ_SERVICE_HANDLE_INVALID;
+
+	if (!vchiq_is_connected(instance))
+		goto failed;
+
+	service = vchiq_add_service_internal(state,
+		params,
+		VCHIQ_SRVSTATE_OPENING,
+		instance,
+		NULL);
+
+	if (service) {
+		*phandle = service->handle;
+		status = vchiq_open_service_internal(service, current->pid);
+		if (status != VCHIQ_SUCCESS) {
+			vchiq_remove_service(service->handle);
+			*phandle = VCHIQ_SERVICE_HANDLE_INVALID;
+		}
+	}
+
+failed:
+	vchiq_log_trace(vchiq_core_log_level,
+		"%s(%p): returning %d", __func__, instance, status);
+
+	return status;
+}
+EXPORT_SYMBOL(vchiq_open_service);
+
+VCHIQ_STATUS_T
+vchiq_queue_bulk_transmit(VCHIQ_SERVICE_HANDLE_T handle,
+	const void *data, unsigned int size, void *userdata)
+{
+	return vchiq_bulk_transfer(handle,
+		VCHI_MEM_HANDLE_INVALID, (void *)data, size, userdata,
+		VCHIQ_BULK_MODE_CALLBACK, VCHIQ_BULK_TRANSMIT);
+}
+EXPORT_SYMBOL(vchiq_queue_bulk_transmit);
+
+VCHIQ_STATUS_T
+vchiq_queue_bulk_receive(VCHIQ_SERVICE_HANDLE_T handle, void *data,
+	unsigned int size, void *userdata)
+{
+	return vchiq_bulk_transfer(handle,
+		VCHI_MEM_HANDLE_INVALID, data, size, userdata,
+		VCHIQ_BULK_MODE_CALLBACK, VCHIQ_BULK_RECEIVE);
+}
+EXPORT_SYMBOL(vchiq_queue_bulk_receive);
+
+VCHIQ_STATUS_T
+vchiq_bulk_transmit(VCHIQ_SERVICE_HANDLE_T handle, const void *data,
+	unsigned int size, void *userdata, VCHIQ_BULK_MODE_T mode)
+{
+	VCHIQ_STATUS_T status;
+
+	switch (mode) {
+	case VCHIQ_BULK_MODE_NOCALLBACK:
+	case VCHIQ_BULK_MODE_CALLBACK:
+		status = vchiq_bulk_transfer(handle,
+			VCHI_MEM_HANDLE_INVALID, (void *)data, size, userdata,
+			mode, VCHIQ_BULK_TRANSMIT);
+		break;
+	case VCHIQ_BULK_MODE_BLOCKING:
+		status = vchiq_blocking_bulk_transfer(handle,
+			(void *)data, size, VCHIQ_BULK_TRANSMIT);
+		break;
+	default:
+		return VCHIQ_ERROR;
+	}
+
+	return status;
+}
+EXPORT_SYMBOL(vchiq_bulk_transmit);
+
+VCHIQ_STATUS_T
+vchiq_bulk_receive(VCHIQ_SERVICE_HANDLE_T handle, void *data,
+	unsigned int size, void *userdata, VCHIQ_BULK_MODE_T mode)
+{
+	VCHIQ_STATUS_T status;
+
+	switch (mode) {
+	case VCHIQ_BULK_MODE_NOCALLBACK:
+	case VCHIQ_BULK_MODE_CALLBACK:
+		status = vchiq_bulk_transfer(handle,
+			VCHI_MEM_HANDLE_INVALID, data, size, userdata,
+			mode, VCHIQ_BULK_RECEIVE);
+		break;
+	case VCHIQ_BULK_MODE_BLOCKING:
+		status = vchiq_blocking_bulk_transfer(handle,
+			(void *)data, size, VCHIQ_BULK_RECEIVE);
+		break;
+	default:
+		return VCHIQ_ERROR;
+	}
+
+	return status;
+}
+EXPORT_SYMBOL(vchiq_bulk_receive);
+
+static VCHIQ_STATUS_T
+vchiq_blocking_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle, void *data,
+	unsigned int size, VCHIQ_BULK_DIR_T dir)
+{
+	VCHIQ_INSTANCE_T instance;
+	VCHIQ_SERVICE_T *service;
+	VCHIQ_STATUS_T status;
+	struct bulk_waiter_node *waiter = NULL;
+	struct list_head *pos;
+
+	service = find_service_by_handle(handle);
+	if (!service)
+		return VCHIQ_ERROR;
+
+	instance = service->instance;
+
+	unlock_service(service);
+
+	mutex_lock(&instance->bulk_waiter_list_mutex);
+	list_for_each(pos, &instance->bulk_waiter_list) {
+		if (list_entry(pos, struct bulk_waiter_node,
+				list)->pid == current->pid) {
+			waiter = list_entry(pos,
+				struct bulk_waiter_node,
+				list);
+			list_del(pos);
+			break;
+		}
+	}
+	mutex_unlock(&instance->bulk_waiter_list_mutex);
+
+	if (waiter) {
+		VCHIQ_BULK_T *bulk = waiter->bulk_waiter.bulk;
+		if (bulk) {
+			/* This thread has an outstanding bulk transfer. */
+			if ((bulk->data != data) ||
+				(bulk->size != size)) {
+				/* This is not a retry of the previous one.
+				** Cancel the signal when the transfer
+				** completes. */
+				spin_lock(&bulk_waiter_spinlock);
+				bulk->userdata = NULL;
+				spin_unlock(&bulk_waiter_spinlock);
+			}
+		}
+	}
+
+	if (!waiter) {
+		waiter = kzalloc(sizeof(struct bulk_waiter_node), GFP_KERNEL);
+		if (!waiter) {
+			vchiq_log_error(vchiq_core_log_level,
+				"%s - out of memory", __func__);
+			return VCHIQ_ERROR;
+		}
+	}
+
+	status = vchiq_bulk_transfer(handle, VCHI_MEM_HANDLE_INVALID,
+		data, size, &waiter->bulk_waiter, VCHIQ_BULK_MODE_BLOCKING,
+		dir);
+	if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) ||
+		!waiter->bulk_waiter.bulk) {
+		VCHIQ_BULK_T *bulk = waiter->bulk_waiter.bulk;
+		if (bulk) {
+			/* Cancel the signal when the transfer
+			 ** completes. */
+			spin_lock(&bulk_waiter_spinlock);
+			bulk->userdata = NULL;
+			spin_unlock(&bulk_waiter_spinlock);
+		}
+		kfree(waiter);
+	} else {
+		waiter->pid = current->pid;
+		mutex_lock(&instance->bulk_waiter_list_mutex);
+		list_add(&waiter->list, &instance->bulk_waiter_list);
+		mutex_unlock(&instance->bulk_waiter_list_mutex);
+		vchiq_log_info(vchiq_arm_log_level,
+				"saved bulk_waiter %x for pid %d",
+				(unsigned int)waiter, current->pid);
+	}
+
+	return status;
+}
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_killable.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_killable.h
new file mode 100644
index 000000000000..335446e05476
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_killable.h
@@ -0,0 +1,69 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_KILLABLE_H
+#define VCHIQ_KILLABLE_H
+
+#include <linux/mutex.h>
+#include <linux/semaphore.h>
+
+#define SHUTDOWN_SIGS   (sigmask(SIGKILL) | sigmask(SIGINT) | sigmask(SIGQUIT) | sigmask(SIGTRAP) | sigmask(SIGSTOP) | sigmask(SIGCONT))
+
+static inline int __must_check down_interruptible_killable(struct semaphore *sem)
+{
+	/* Allow interception of killable signals only. We don't want to be interrupted by harmless signals like SIGALRM */
+	int ret;
+	sigset_t blocked, oldset;
+	siginitsetinv(&blocked, SHUTDOWN_SIGS);
+	sigprocmask(SIG_SETMASK, &blocked, &oldset);
+	ret = down_interruptible(sem);
+	sigprocmask(SIG_SETMASK, &oldset, NULL);
+	return ret;
+}
+#define down_interruptible down_interruptible_killable
+
+
+static inline int __must_check mutex_lock_interruptible_killable(struct mutex *lock)
+{
+	/* Allow interception of killable signals only. We don't want to be interrupted by harmless signals like SIGALRM */
+	int ret;
+	sigset_t blocked, oldset;
+	siginitsetinv(&blocked, SHUTDOWN_SIGS);
+	sigprocmask(SIG_SETMASK, &blocked, &oldset);
+	ret = mutex_lock_interruptible(lock);
+	sigprocmask(SIG_SETMASK, &oldset, NULL);
+	return ret;
+}
+#define mutex_lock_interruptible mutex_lock_interruptible_killable
+
+#endif
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_memdrv.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_memdrv.h
new file mode 100644
index 000000000000..d02e7764bd0d
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_memdrv.h
@@ -0,0 +1,71 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_MEMDRV_H
+#define VCHIQ_MEMDRV_H
+
+/* ---- Include Files ----------------------------------------------------- */
+
+#include <linux/kernel.h>
+#include "vchiq_if.h"
+
+/* ---- Constants and Types ---------------------------------------------- */
+
+typedef struct {
+	 void                   *armSharedMemVirt;
+	 dma_addr_t              armSharedMemPhys;
+	 size_t                  armSharedMemSize;
+
+	 void                   *vcSharedMemVirt;
+	 dma_addr_t              vcSharedMemPhys;
+	 size_t                  vcSharedMemSize;
+} VCHIQ_SHARED_MEM_INFO_T;
+
+/* ---- Variable Externs ------------------------------------------------- */
+
+/* ---- Function Prototypes ---------------------------------------------- */
+
+void vchiq_get_shared_mem_info(VCHIQ_SHARED_MEM_INFO_T *info);
+
+VCHIQ_STATUS_T vchiq_memdrv_initialise(void);
+
+VCHIQ_STATUS_T vchiq_userdrv_create_instance(
+	const VCHIQ_PLATFORM_DATA_T * platform_data);
+
+VCHIQ_STATUS_T vchiq_userdrv_suspend(
+	const VCHIQ_PLATFORM_DATA_T * platform_data);
+
+VCHIQ_STATUS_T vchiq_userdrv_resume(
+	const VCHIQ_PLATFORM_DATA_T * platform_data);
+
+#endif
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_pagelist.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_pagelist.h
new file mode 100644
index 000000000000..54a3ecec69ef
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_pagelist.h
@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_PAGELIST_H
+#define VCHIQ_PAGELIST_H
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE 4096
+#endif
+#define CACHE_LINE_SIZE 32
+#define PAGELIST_WRITE 0
+#define PAGELIST_READ 1
+#define PAGELIST_READ_WITH_FRAGMENTS 2
+
+typedef struct pagelist_struct {
+	unsigned long length;
+	unsigned short type;
+	unsigned short offset;
+	unsigned long addrs[1];	/* N.B. 12 LSBs hold the number of following
+				   pages at consecutive addresses. */
+} PAGELIST_T;
+
+typedef struct fragments_struct {
+	char headbuf[CACHE_LINE_SIZE];
+	char tailbuf[CACHE_LINE_SIZE];
+} FRAGMENTS_T;
+
+#endif /* VCHIQ_PAGELIST_H */
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c
new file mode 100644
index 000000000000..8072ff613636
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c
@@ -0,0 +1,860 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <linux/module.h>
+#include <linux/types.h>
+
+#include "interface/vchi/vchi.h"
+#include "vchiq.h"
+#include "vchiq_core.h"
+
+#include "vchiq_util.h"
+
+#include <stddef.h>
+
+#define vchiq_status_to_vchi(status) ((int32_t)status)
+
+typedef struct {
+	VCHIQ_SERVICE_HANDLE_T handle;
+
+	VCHIU_QUEUE_T queue;
+
+	VCHI_CALLBACK_T callback;
+	void *callback_param;
+} SHIM_SERVICE_T;
+
+/* ----------------------------------------------------------------------
+ * return pointer to the mphi message driver function table
+ * -------------------------------------------------------------------- */
+const VCHI_MESSAGE_DRIVER_T *
+vchi_mphi_message_driver_func_table(void)
+{
+	return NULL;
+}
+
+/* ----------------------------------------------------------------------
+ * return a pointer to the 'single' connection driver fops
+ * -------------------------------------------------------------------- */
+const VCHI_CONNECTION_API_T *
+single_get_func_table(void)
+{
+	return NULL;
+}
+
+VCHI_CONNECTION_T *vchi_create_connection(
+	const VCHI_CONNECTION_API_T *function_table,
+	const VCHI_MESSAGE_DRIVER_T *low_level)
+{
+	(void)function_table;
+	(void)low_level;
+	return NULL;
+}
+
+/***********************************************************
+ * Name: vchi_msg_peek
+ *
+ * Arguments:  const VCHI_SERVICE_HANDLE_T handle,
+ *             void **data,
+ *             uint32_t *msg_size,
+
+
+ *             VCHI_FLAGS_T flags
+ *
+ * Description: Routine to return a pointer to the current message (to allow in
+ *              place processing). The message can be removed using
+ *              vchi_msg_remove when you're finished
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+int32_t vchi_msg_peek(VCHI_SERVICE_HANDLE_T handle,
+	void **data,
+	uint32_t *msg_size,
+	VCHI_FLAGS_T flags)
+{
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	VCHIQ_HEADER_T *header;
+
+	WARN_ON((flags != VCHI_FLAGS_NONE) &&
+		(flags != VCHI_FLAGS_BLOCK_UNTIL_OP_COMPLETE));
+
+	if (flags == VCHI_FLAGS_NONE)
+		if (vchiu_queue_is_empty(&service->queue))
+			return -1;
+
+	header = vchiu_queue_peek(&service->queue);
+
+	*data = header->data;
+	*msg_size = header->size;
+
+	return 0;
+}
+EXPORT_SYMBOL(vchi_msg_peek);
+
+/***********************************************************
+ * Name: vchi_msg_remove
+ *
+ * Arguments:  const VCHI_SERVICE_HANDLE_T handle,
+ *
+ * Description: Routine to remove a message (after it has been read with
+ *              vchi_msg_peek)
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+int32_t vchi_msg_remove(VCHI_SERVICE_HANDLE_T handle)
+{
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	VCHIQ_HEADER_T *header;
+
+	header = vchiu_queue_pop(&service->queue);
+
+	vchiq_release_message(service->handle, header);
+
+	return 0;
+}
+EXPORT_SYMBOL(vchi_msg_remove);
+
+/***********************************************************
+ * Name: vchi_msg_queue
+ *
+ * Arguments:  VCHI_SERVICE_HANDLE_T handle,
+ *             const void *data,
+ *             uint32_t data_size,
+ *             VCHI_FLAGS_T flags,
+ *             void *msg_handle,
+ *
+ * Description: Thin wrapper to queue a message onto a connection
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+int32_t vchi_msg_queue(VCHI_SERVICE_HANDLE_T handle,
+	const void *data,
+	uint32_t data_size,
+	VCHI_FLAGS_T flags,
+	void *msg_handle)
+{
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	VCHIQ_ELEMENT_T element = {data, data_size};
+	VCHIQ_STATUS_T status;
+
+	(void)msg_handle;
+
+	WARN_ON(flags != VCHI_FLAGS_BLOCK_UNTIL_QUEUED);
+
+	status = vchiq_queue_message(service->handle, &element, 1);
+
+	/* vchiq_queue_message() may return VCHIQ_RETRY, so we need to
+	** implement a retry mechanism since this function is supposed
+	** to block until queued
+	*/
+	while (status == VCHIQ_RETRY) {
+		msleep(1);
+		status = vchiq_queue_message(service->handle, &element, 1);
+	}
+
+	return vchiq_status_to_vchi(status);
+}
+EXPORT_SYMBOL(vchi_msg_queue);
+
+/***********************************************************
+ * Name: vchi_bulk_queue_receive
+ *
+ * Arguments:  VCHI_BULK_HANDLE_T handle,
+ *             void *data_dst,
+ *             const uint32_t data_size,
+ *             VCHI_FLAGS_T flags
+ *             void *bulk_handle
+ *
+ * Description: Routine to setup a rcv buffer
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+int32_t vchi_bulk_queue_receive(VCHI_SERVICE_HANDLE_T handle,
+	void *data_dst,
+	uint32_t data_size,
+	VCHI_FLAGS_T flags,
+	void *bulk_handle)
+{
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	VCHIQ_BULK_MODE_T mode;
+	VCHIQ_STATUS_T status;
+
+	switch ((int)flags) {
+	case VCHI_FLAGS_CALLBACK_WHEN_OP_COMPLETE
+		| VCHI_FLAGS_BLOCK_UNTIL_QUEUED:
+		WARN_ON(!service->callback);
+		mode = VCHIQ_BULK_MODE_CALLBACK;
+		break;
+	case VCHI_FLAGS_BLOCK_UNTIL_OP_COMPLETE:
+		mode = VCHIQ_BULK_MODE_BLOCKING;
+		break;
+	case VCHI_FLAGS_BLOCK_UNTIL_QUEUED:
+	case VCHI_FLAGS_NONE:
+		mode = VCHIQ_BULK_MODE_NOCALLBACK;
+		break;
+	default:
+		WARN(1, "unsupported message\n");
+		return vchiq_status_to_vchi(VCHIQ_ERROR);
+	}
+
+	status = vchiq_bulk_receive(service->handle, data_dst, data_size,
+		bulk_handle, mode);
+
+	/* vchiq_bulk_receive() may return VCHIQ_RETRY, so we need to
+	** implement a retry mechanism since this function is supposed
+	** to block until queued
+	*/
+	while (status == VCHIQ_RETRY) {
+		msleep(1);
+		status = vchiq_bulk_receive(service->handle, data_dst,
+			data_size, bulk_handle, mode);
+	}
+
+	return vchiq_status_to_vchi(status);
+}
+EXPORT_SYMBOL(vchi_bulk_queue_receive);
+
+/***********************************************************
+ * Name: vchi_bulk_queue_transmit
+ *
+ * Arguments:  VCHI_BULK_HANDLE_T handle,
+ *             const void *data_src,
+ *             uint32_t data_size,
+ *             VCHI_FLAGS_T flags,
+ *             void *bulk_handle
+ *
+ * Description: Routine to transmit some data
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+int32_t vchi_bulk_queue_transmit(VCHI_SERVICE_HANDLE_T handle,
+	const void *data_src,
+	uint32_t data_size,
+	VCHI_FLAGS_T flags,
+	void *bulk_handle)
+{
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	VCHIQ_BULK_MODE_T mode;
+	VCHIQ_STATUS_T status;
+
+	switch ((int)flags) {
+	case VCHI_FLAGS_CALLBACK_WHEN_OP_COMPLETE
+		| VCHI_FLAGS_BLOCK_UNTIL_QUEUED:
+		WARN_ON(!service->callback);
+		mode = VCHIQ_BULK_MODE_CALLBACK;
+		break;
+	case VCHI_FLAGS_BLOCK_UNTIL_DATA_READ:
+	case VCHI_FLAGS_BLOCK_UNTIL_OP_COMPLETE:
+		mode = VCHIQ_BULK_MODE_BLOCKING;
+		break;
+	case VCHI_FLAGS_BLOCK_UNTIL_QUEUED:
+	case VCHI_FLAGS_NONE:
+		mode = VCHIQ_BULK_MODE_NOCALLBACK;
+		break;
+	default:
+		WARN(1, "unsupported message\n");
+		return vchiq_status_to_vchi(VCHIQ_ERROR);
+	}
+
+	status = vchiq_bulk_transmit(service->handle, data_src, data_size,
+		bulk_handle, mode);
+
+	/* vchiq_bulk_transmit() may return VCHIQ_RETRY, so we need to
+	** implement a retry mechanism since this function is supposed
+	** to block until queued
+	*/
+	while (status == VCHIQ_RETRY) {
+		msleep(1);
+		status = vchiq_bulk_transmit(service->handle, data_src,
+			data_size, bulk_handle, mode);
+	}
+
+	return vchiq_status_to_vchi(status);
+}
+EXPORT_SYMBOL(vchi_bulk_queue_transmit);
+
+/***********************************************************
+ * Name: vchi_msg_dequeue
+ *
+ * Arguments:  VCHI_SERVICE_HANDLE_T handle,
+ *             void *data,
+ *             uint32_t max_data_size_to_read,
+ *             uint32_t *actual_msg_size
+ *             VCHI_FLAGS_T flags
+ *
+ * Description: Routine to dequeue a message into the supplied buffer
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+int32_t vchi_msg_dequeue(VCHI_SERVICE_HANDLE_T handle,
+	void *data,
+	uint32_t max_data_size_to_read,
+	uint32_t *actual_msg_size,
+	VCHI_FLAGS_T flags)
+{
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	VCHIQ_HEADER_T *header;
+
+	WARN_ON((flags != VCHI_FLAGS_NONE) &&
+		(flags != VCHI_FLAGS_BLOCK_UNTIL_OP_COMPLETE));
+
+	if (flags == VCHI_FLAGS_NONE)
+		if (vchiu_queue_is_empty(&service->queue))
+			return -1;
+
+	header = vchiu_queue_pop(&service->queue);
+
+	memcpy(data, header->data, header->size < max_data_size_to_read ?
+		header->size : max_data_size_to_read);
+
+	*actual_msg_size = header->size;
+
+	vchiq_release_message(service->handle, header);
+
+	return 0;
+}
+EXPORT_SYMBOL(vchi_msg_dequeue);
+
+/***********************************************************
+ * Name: vchi_msg_queuev
+ *
+ * Arguments:  VCHI_SERVICE_HANDLE_T handle,
+ *             VCHI_MSG_VECTOR_T *vector,
+ *             uint32_t count,
+ *             VCHI_FLAGS_T flags,
+ *             void *msg_handle
+ *
+ * Description: Thin wrapper to queue a message onto a connection
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+
+vchiq_static_assert(sizeof(VCHI_MSG_VECTOR_T) == sizeof(VCHIQ_ELEMENT_T));
+vchiq_static_assert(offsetof(VCHI_MSG_VECTOR_T, vec_base) ==
+	offsetof(VCHIQ_ELEMENT_T, data));
+vchiq_static_assert(offsetof(VCHI_MSG_VECTOR_T, vec_len) ==
+	offsetof(VCHIQ_ELEMENT_T, size));
+
+int32_t vchi_msg_queuev(VCHI_SERVICE_HANDLE_T handle,
+	VCHI_MSG_VECTOR_T *vector,
+	uint32_t count,
+	VCHI_FLAGS_T flags,
+	void *msg_handle)
+{
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+
+	(void)msg_handle;
+
+	WARN_ON(flags != VCHI_FLAGS_BLOCK_UNTIL_QUEUED);
+
+	return vchiq_status_to_vchi(vchiq_queue_message(service->handle,
+		(const VCHIQ_ELEMENT_T *)vector, count));
+}
+EXPORT_SYMBOL(vchi_msg_queuev);
+
+/***********************************************************
+ * Name: vchi_held_msg_release
+ *
+ * Arguments:  VCHI_HELD_MSG_T *message
+ *
+ * Description: Routine to release a held message (after it has been read with
+ *              vchi_msg_hold)
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+int32_t vchi_held_msg_release(VCHI_HELD_MSG_T *message)
+{
+	vchiq_release_message((VCHIQ_SERVICE_HANDLE_T)message->service,
+		(VCHIQ_HEADER_T *)message->message);
+
+	return 0;
+}
+EXPORT_SYMBOL(vchi_held_msg_release);
+
+/***********************************************************
+ * Name: vchi_msg_hold
+ *
+ * Arguments:  VCHI_SERVICE_HANDLE_T handle,
+ *             void **data,
+ *             uint32_t *msg_size,
+ *             VCHI_FLAGS_T flags,
+ *             VCHI_HELD_MSG_T *message_handle
+ *
+ * Description: Routine to return a pointer to the current message (to allow
+ *              in place processing). The message is dequeued - don't forget
+ *              to release the message using vchi_held_msg_release when you're
+ *              finished.
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+int32_t vchi_msg_hold(VCHI_SERVICE_HANDLE_T handle,
+	void **data,
+	uint32_t *msg_size,
+	VCHI_FLAGS_T flags,
+	VCHI_HELD_MSG_T *message_handle)
+{
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	VCHIQ_HEADER_T *header;
+
+	WARN_ON((flags != VCHI_FLAGS_NONE) &&
+		(flags != VCHI_FLAGS_BLOCK_UNTIL_OP_COMPLETE));
+
+	if (flags == VCHI_FLAGS_NONE)
+		if (vchiu_queue_is_empty(&service->queue))
+			return -1;
+
+	header = vchiu_queue_pop(&service->queue);
+
+	*data = header->data;
+	*msg_size = header->size;
+
+	message_handle->service =
+		(struct opaque_vchi_service_t *)service->handle;
+	message_handle->message = header;
+
+	return 0;
+}
+EXPORT_SYMBOL(vchi_msg_hold);
+
+/***********************************************************
+ * Name: vchi_initialise
+ *
+ * Arguments: VCHI_INSTANCE_T *instance_handle
+ *
+ * Description: Initialises the hardware but does not transmit anything
+ *              When run as a Host App this will be called twice hence the need
+ *              to malloc the state information
+ *
+ * Returns: 0 if successful, failure otherwise
+ *
+ ***********************************************************/
+
+int32_t vchi_initialise(VCHI_INSTANCE_T *instance_handle)
+{
+	VCHIQ_INSTANCE_T instance;
+	VCHIQ_STATUS_T status;
+
+	status = vchiq_initialise(&instance);
+
+	*instance_handle = (VCHI_INSTANCE_T)instance;
+
+	return vchiq_status_to_vchi(status);
+}
+EXPORT_SYMBOL(vchi_initialise);
+
+/***********************************************************
+ * Name: vchi_connect
+ *
+ * Arguments: VCHI_CONNECTION_T **connections
+ *            const uint32_t num_connections
+ *            VCHI_INSTANCE_T instance_handle)
+ *
+ * Description: Starts the command service on each connection,
+ *              causing INIT messages to be pinged back and forth
+ *
+ * Returns: 0 if successful, failure otherwise
+ *
+ ***********************************************************/
+int32_t vchi_connect(VCHI_CONNECTION_T **connections,
+	const uint32_t num_connections,
+	VCHI_INSTANCE_T instance_handle)
+{
+	VCHIQ_INSTANCE_T instance = (VCHIQ_INSTANCE_T)instance_handle;
+
+	(void)connections;
+	(void)num_connections;
+
+	return vchiq_connect(instance);
+}
+EXPORT_SYMBOL(vchi_connect);
+
+
+/***********************************************************
+ * Name: vchi_disconnect
+ *
+ * Arguments: VCHI_INSTANCE_T instance_handle
+ *
+ * Description: Stops the command service on each connection,
+ *              causing DE-INIT messages to be pinged back and forth
+ *
+ * Returns: 0 if successful, failure otherwise
+ *
+ ***********************************************************/
+int32_t vchi_disconnect(VCHI_INSTANCE_T instance_handle)
+{
+	VCHIQ_INSTANCE_T instance = (VCHIQ_INSTANCE_T)instance_handle;
+	return vchiq_status_to_vchi(vchiq_shutdown(instance));
+}
+EXPORT_SYMBOL(vchi_disconnect);
+
+
+/***********************************************************
+ * Name: vchi_service_open
+ * Name: vchi_service_create
+ *
+ * Arguments: VCHI_INSTANCE_T *instance_handle
+ *            SERVICE_CREATION_T *setup,
+ *            VCHI_SERVICE_HANDLE_T *handle
+ *
+ * Description: Routine to open a service
+ *
+ * Returns: int32_t - success == 0
+ *
+ ***********************************************************/
+
+static VCHIQ_STATUS_T shim_callback(VCHIQ_REASON_T reason,
+	VCHIQ_HEADER_T *header, VCHIQ_SERVICE_HANDLE_T handle, void *bulk_user)
+{
+	SHIM_SERVICE_T *service =
+		(SHIM_SERVICE_T *)VCHIQ_GET_SERVICE_USERDATA(handle);
+
+        if (!service->callback)
+		goto release;
+
+	switch (reason) {
+	case VCHIQ_MESSAGE_AVAILABLE:
+		vchiu_queue_push(&service->queue, header);
+
+		service->callback(service->callback_param,
+				  VCHI_CALLBACK_MSG_AVAILABLE, NULL);
+
+		goto done;
+		break;
+
+	case VCHIQ_BULK_TRANSMIT_DONE:
+		service->callback(service->callback_param,
+				  VCHI_CALLBACK_BULK_SENT, bulk_user);
+		break;
+
+	case VCHIQ_BULK_RECEIVE_DONE:
+		service->callback(service->callback_param,
+				  VCHI_CALLBACK_BULK_RECEIVED, bulk_user);
+		break;
+
+	case VCHIQ_SERVICE_CLOSED:
+		service->callback(service->callback_param,
+				  VCHI_CALLBACK_SERVICE_CLOSED, NULL);
+		break;
+
+	case VCHIQ_SERVICE_OPENED:
+		/* No equivalent VCHI reason */
+		break;
+
+	case VCHIQ_BULK_TRANSMIT_ABORTED:
+		service->callback(service->callback_param,
+				  VCHI_CALLBACK_BULK_TRANSMIT_ABORTED,
+				  bulk_user);
+		break;
+
+	case VCHIQ_BULK_RECEIVE_ABORTED:
+		service->callback(service->callback_param,
+				  VCHI_CALLBACK_BULK_RECEIVE_ABORTED,
+				  bulk_user);
+		break;
+
+	default:
+		WARN(1, "not supported\n");
+		break;
+	}
+
+release:
+        vchiq_release_message(service->handle, header);
+done:
+	return VCHIQ_SUCCESS;
+}
+
+static SHIM_SERVICE_T *service_alloc(VCHIQ_INSTANCE_T instance,
+	SERVICE_CREATION_T *setup)
+{
+	SHIM_SERVICE_T *service = kzalloc(sizeof(SHIM_SERVICE_T), GFP_KERNEL);
+
+	(void)instance;
+
+	if (service) {
+		if (vchiu_queue_init(&service->queue, 64)) {
+			service->callback = setup->callback;
+			service->callback_param = setup->callback_param;
+		} else {
+			kfree(service);
+			service = NULL;
+		}
+	}
+
+	return service;
+}
+
+static void service_free(SHIM_SERVICE_T *service)
+{
+	if (service) {
+		vchiu_queue_delete(&service->queue);
+		kfree(service);
+	}
+}
+
+int32_t vchi_service_open(VCHI_INSTANCE_T instance_handle,
+	SERVICE_CREATION_T *setup,
+	VCHI_SERVICE_HANDLE_T *handle)
+{
+	VCHIQ_INSTANCE_T instance = (VCHIQ_INSTANCE_T)instance_handle;
+	SHIM_SERVICE_T *service = service_alloc(instance, setup);
+
+	*handle = (VCHI_SERVICE_HANDLE_T)service;
+
+	if (service) {
+		VCHIQ_SERVICE_PARAMS_T params;
+		VCHIQ_STATUS_T status;
+
+		memset(&params, 0, sizeof(params));
+		params.fourcc = setup->service_id;
+		params.callback = shim_callback;
+		params.userdata = service;
+		params.version = setup->version.version;
+		params.version_min = setup->version.version_min;
+
+		status = vchiq_open_service(instance, &params,
+			&service->handle);
+		if (status != VCHIQ_SUCCESS) {
+			service_free(service);
+			service = NULL;
+			*handle = NULL;
+		}
+	}
+
+	return (service != NULL) ? 0 : -1;
+}
+EXPORT_SYMBOL(vchi_service_open);
+
+int32_t vchi_service_create(VCHI_INSTANCE_T instance_handle,
+	SERVICE_CREATION_T *setup,
+	VCHI_SERVICE_HANDLE_T *handle)
+{
+	VCHIQ_INSTANCE_T instance = (VCHIQ_INSTANCE_T)instance_handle;
+	SHIM_SERVICE_T *service = service_alloc(instance, setup);
+
+	*handle = (VCHI_SERVICE_HANDLE_T)service;
+
+	if (service) {
+		VCHIQ_SERVICE_PARAMS_T params;
+		VCHIQ_STATUS_T status;
+
+		memset(&params, 0, sizeof(params));
+		params.fourcc = setup->service_id;
+		params.callback = shim_callback;
+		params.userdata = service;
+		params.version = setup->version.version;
+		params.version_min = setup->version.version_min;
+		status = vchiq_add_service(instance, &params, &service->handle);
+
+		if (status != VCHIQ_SUCCESS) {
+			service_free(service);
+			service = NULL;
+			*handle = NULL;
+		}
+	}
+
+	return (service != NULL) ? 0 : -1;
+}
+EXPORT_SYMBOL(vchi_service_create);
+
+int32_t vchi_service_close(const VCHI_SERVICE_HANDLE_T handle)
+{
+	int32_t ret = -1;
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	if (service) {
+		VCHIQ_STATUS_T status = vchiq_close_service(service->handle);
+		if (status == VCHIQ_SUCCESS) {
+			service_free(service);
+			service = NULL;
+		}
+
+		ret = vchiq_status_to_vchi(status);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(vchi_service_close);
+
+int32_t vchi_service_destroy(const VCHI_SERVICE_HANDLE_T handle)
+{
+	int32_t ret = -1;
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	if (service) {
+		VCHIQ_STATUS_T status = vchiq_remove_service(service->handle);
+		if (status == VCHIQ_SUCCESS) {
+			service_free(service);
+			service = NULL;
+		}
+
+		ret = vchiq_status_to_vchi(status);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(vchi_service_destroy);
+
+int32_t vchi_service_set_option(const VCHI_SERVICE_HANDLE_T handle,
+				VCHI_SERVICE_OPTION_T option,
+				int value)
+{
+	int32_t ret = -1;
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	VCHIQ_SERVICE_OPTION_T vchiq_option;
+	switch (option) {
+	case VCHI_SERVICE_OPTION_TRACE:
+		vchiq_option = VCHIQ_SERVICE_OPTION_TRACE;
+		break;
+	case VCHI_SERVICE_OPTION_SYNCHRONOUS:
+		vchiq_option = VCHIQ_SERVICE_OPTION_SYNCHRONOUS;
+		break;
+	default:
+		service = NULL;
+		break;
+	}
+	if (service) {
+		VCHIQ_STATUS_T status =
+			vchiq_set_service_option(service->handle,
+						vchiq_option,
+						value);
+
+		ret = vchiq_status_to_vchi(status);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(vchi_service_set_option);
+
+int32_t vchi_get_peer_version( const VCHI_SERVICE_HANDLE_T handle, short *peer_version )
+{
+   int32_t ret = -1;
+   SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+   if(service)
+   {
+      VCHIQ_STATUS_T status = vchiq_get_peer_version(service->handle, peer_version);
+      ret = vchiq_status_to_vchi( status );
+   }
+   return ret;
+}
+EXPORT_SYMBOL(vchi_get_peer_version);
+
+/* ----------------------------------------------------------------------
+ * read a uint32_t from buffer.
+ * network format is defined to be little endian
+ * -------------------------------------------------------------------- */
+uint32_t
+vchi_readbuf_uint32(const void *_ptr)
+{
+	const unsigned char *ptr = _ptr;
+	return ptr[0] | (ptr[1] << 8) | (ptr[2] << 16) | (ptr[3] << 24);
+}
+
+/* ----------------------------------------------------------------------
+ * write a uint32_t to buffer.
+ * network format is defined to be little endian
+ * -------------------------------------------------------------------- */
+void
+vchi_writebuf_uint32(void *_ptr, uint32_t value)
+{
+	unsigned char *ptr = _ptr;
+	ptr[0] = (unsigned char)((value >> 0)  & 0xFF);
+	ptr[1] = (unsigned char)((value >> 8)  & 0xFF);
+	ptr[2] = (unsigned char)((value >> 16) & 0xFF);
+	ptr[3] = (unsigned char)((value >> 24) & 0xFF);
+}
+
+/* ----------------------------------------------------------------------
+ * read a uint16_t from buffer.
+ * network format is defined to be little endian
+ * -------------------------------------------------------------------- */
+uint16_t
+vchi_readbuf_uint16(const void *_ptr)
+{
+	const unsigned char *ptr = _ptr;
+	return ptr[0] | (ptr[1] << 8);
+}
+
+/* ----------------------------------------------------------------------
+ * write a uint16_t into the buffer.
+ * network format is defined to be little endian
+ * -------------------------------------------------------------------- */
+void
+vchi_writebuf_uint16(void *_ptr, uint16_t value)
+{
+	unsigned char *ptr = _ptr;
+	ptr[0] = (value >> 0)  & 0xFF;
+	ptr[1] = (value >> 8)  & 0xFF;
+}
+
+/***********************************************************
+ * Name: vchi_service_use
+ *
+ * Arguments: const VCHI_SERVICE_HANDLE_T handle
+ *
+ * Description: Routine to increment refcount on a service
+ *
+ * Returns: void
+ *
+ ***********************************************************/
+int32_t vchi_service_use(const VCHI_SERVICE_HANDLE_T handle)
+{
+	int32_t ret = -1;
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	if (service)
+		ret = vchiq_status_to_vchi(vchiq_use_service(service->handle));
+	return ret;
+}
+EXPORT_SYMBOL(vchi_service_use);
+
+/***********************************************************
+ * Name: vchi_service_release
+ *
+ * Arguments: const VCHI_SERVICE_HANDLE_T handle
+ *
+ * Description: Routine to decrement refcount on a service
+ *
+ * Returns: void
+ *
+ ***********************************************************/
+int32_t vchi_service_release(const VCHI_SERVICE_HANDLE_T handle)
+{
+	int32_t ret = -1;
+	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
+	if (service)
+		ret = vchiq_status_to_vchi(
+			vchiq_release_service(service->handle));
+	return ret;
+}
+EXPORT_SYMBOL(vchi_service_release);
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c
new file mode 100644
index 000000000000..384acb8d2eae
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c
@@ -0,0 +1,156 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "vchiq_util.h"
+#include "vchiq_killable.h"
+
+static inline int is_pow2(int i)
+{
+	return i && !(i & (i - 1));
+}
+
+int vchiu_queue_init(VCHIU_QUEUE_T *queue, int size)
+{
+	WARN_ON(!is_pow2(size));
+
+	queue->size = size;
+	queue->read = 0;
+	queue->write = 0;
+	queue->initialized = 1;
+
+	sema_init(&queue->pop, 0);
+	sema_init(&queue->push, 0);
+
+	queue->storage = kzalloc(size * sizeof(VCHIQ_HEADER_T *), GFP_KERNEL);
+	if (queue->storage == NULL) {
+		vchiu_queue_delete(queue);
+		return 0;
+	}
+	return 1;
+}
+
+void vchiu_queue_delete(VCHIU_QUEUE_T *queue)
+{
+	if (queue->storage != NULL)
+		kfree(queue->storage);
+}
+
+int vchiu_queue_is_empty(VCHIU_QUEUE_T *queue)
+{
+	return queue->read == queue->write;
+}
+
+int vchiu_queue_is_full(VCHIU_QUEUE_T *queue)
+{
+	return queue->write == queue->read + queue->size;
+}
+
+void vchiu_queue_push(VCHIU_QUEUE_T *queue, VCHIQ_HEADER_T *header)
+{
+	if (!queue->initialized)
+		return;
+
+	while (queue->write == queue->read + queue->size) {
+		if (down_interruptible(&queue->pop) != 0) {
+			flush_signals(current);
+		}
+	}
+
+	/*
+	 * Write to queue->storage must be visible after read from
+	 * queue->read
+	 */
+	smp_mb();
+
+	queue->storage[queue->write & (queue->size - 1)] = header;
+
+	/*
+	 * Write to queue->storage must be visible before write to
+	 * queue->write
+	 */
+	smp_wmb();
+
+	queue->write++;
+
+	up(&queue->push);
+}
+
+VCHIQ_HEADER_T *vchiu_queue_peek(VCHIU_QUEUE_T *queue)
+{
+	while (queue->write == queue->read) {
+		if (down_interruptible(&queue->push) != 0) {
+			flush_signals(current);
+		}
+	}
+
+	up(&queue->push); // We haven't removed anything from the queue.
+
+	/*
+	 * Read from queue->storage must be visible after read from
+	 * queue->write
+	 */
+	smp_rmb();
+
+	return queue->storage[queue->read & (queue->size - 1)];
+}
+
+VCHIQ_HEADER_T *vchiu_queue_pop(VCHIU_QUEUE_T *queue)
+{
+	VCHIQ_HEADER_T *header;
+
+	while (queue->write == queue->read) {
+		if (down_interruptible(&queue->push) != 0) {
+			flush_signals(current);
+		}
+	}
+
+	/*
+	 * Read from queue->storage must be visible after read from
+	 * queue->write
+	 */
+	smp_rmb();
+
+	header = queue->storage[queue->read & (queue->size - 1)];
+
+	/*
+	 * Read from queue->storage must be visible before write to
+	 * queue->read
+	 */
+	smp_mb();
+
+	queue->read++;
+
+	up(&queue->pop);
+
+	return header;
+}
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.h
new file mode 100644
index 000000000000..4055d4bf9f74
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.h
@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VCHIQ_UTIL_H
+#define VCHIQ_UTIL_H
+
+#include <linux/types.h>
+#include <linux/semaphore.h>
+#include <linux/mutex.h>
+#include <linux/bitops.h>
+#include <linux/kthread.h>
+#include <linux/wait.h>
+#include <linux/vmalloc.h>
+#include <linux/jiffies.h>
+#include <linux/delay.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/interrupt.h>
+#include <linux/random.h>
+#include <linux/sched.h>
+#include <linux/ctype.h>
+#include <linux/uaccess.h>
+#include <linux/time.h>  /* for time_t */
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include "vchiq_if.h"
+
+typedef struct {
+	int size;
+	int read;
+	int write;
+	int initialized;
+
+	struct semaphore pop;
+	struct semaphore push;
+
+	VCHIQ_HEADER_T **storage;
+} VCHIU_QUEUE_T;
+
+extern int  vchiu_queue_init(VCHIU_QUEUE_T *queue, int size);
+extern void vchiu_queue_delete(VCHIU_QUEUE_T *queue);
+
+extern int vchiu_queue_is_empty(VCHIU_QUEUE_T *queue);
+extern int vchiu_queue_is_full(VCHIU_QUEUE_T *queue);
+
+extern void vchiu_queue_push(VCHIU_QUEUE_T *queue, VCHIQ_HEADER_T *header);
+
+extern VCHIQ_HEADER_T *vchiu_queue_peek(VCHIU_QUEUE_T *queue);
+extern VCHIQ_HEADER_T *vchiu_queue_pop(VCHIU_QUEUE_T *queue);
+
+#endif
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_version.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_version.c
new file mode 100644
index 000000000000..b6bfa21155e4
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_version.c
@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "vchiq_build_info.h"
+#include <linux/broadcom/vc_debug_sym.h>
+
+VC_DEBUG_DECLARE_STRING_VAR( vchiq_build_hostname, "dc4-arm-01" );
+VC_DEBUG_DECLARE_STRING_VAR( vchiq_build_version, "9245b4c35b99b3870e1f7dc598c5692b3c66a6f0 (tainted)" );
+VC_DEBUG_DECLARE_STRING_VAR( vchiq_build_time,    __TIME__ );
+VC_DEBUG_DECLARE_STRING_VAR( vchiq_build_date,    __DATE__ );
+
+const char *vchiq_get_build_hostname( void )
+{
+   return vchiq_build_hostname;
+}
+
+const char *vchiq_get_build_version( void )
+{
+   return vchiq_build_version;
+}
+
+const char *vchiq_get_build_date( void )
+{
+   return vchiq_build_date;
+}
+
+const char *vchiq_get_build_time( void )
+{
+   return vchiq_build_time;
+}
diff --git a/drivers/staging/vme/devices/vme_pio2_core.c b/drivers/staging/vme/devices/vme_pio2_core.c
index 28a45689e2f4..8e66a520266c 100644
--- a/drivers/staging/vme/devices/vme_pio2_core.c
+++ b/drivers/staging/vme/devices/vme_pio2_core.c
@@ -466,23 +466,23 @@ static void __exit pio2_exit(void)
 
 /* These are required for each board */
 MODULE_PARM_DESC(bus, "Enumeration of VMEbus to which the board is connected");
-module_param_array(bus, int, &bus_num, S_IRUGO);
+module_param_array(bus, int, &bus_num, 0444);
 
 MODULE_PARM_DESC(base, "Base VME address for PIO2 Registers");
-module_param_array(base, long, &base_num, S_IRUGO);
+module_param_array(base, long, &base_num, 0444);
 
 MODULE_PARM_DESC(vector, "VME IRQ Vector (Lower 4 bits masked)");
-module_param_array(vector, int, &vector_num, S_IRUGO);
+module_param_array(vector, int, &vector_num, 0444);
 
 MODULE_PARM_DESC(level, "VME IRQ Level");
-module_param_array(level, int, &level_num, S_IRUGO);
+module_param_array(level, int, &level_num, 0444);
 
 MODULE_PARM_DESC(variant, "Last 4 characters of PIO2 board variant");
-module_param_array(variant, charp, &variant_num, S_IRUGO);
+module_param_array(variant, charp, &variant_num, 0444);
 
 /* This is for debugging */
 MODULE_PARM_DESC(loopback, "Enable loopback mode on all cards");
-module_param(loopback, bool, S_IRUGO);
+module_param(loopback, bool, 0444);
 
 MODULE_DESCRIPTION("GE PIO2 6U VME I/O Driver");
 MODULE_AUTHOR("Martyn Welch <martyn.welch@ge.com");
diff --git a/drivers/staging/vme/devices/vme_user.c b/drivers/staging/vme/devices/vme_user.c
index b95883bc68fe..5dd430f8f921 100644
--- a/drivers/staging/vme/devices/vme_user.c
+++ b/drivers/staging/vme/devices/vme_user.c
@@ -773,7 +773,7 @@ static void __exit vme_user_exit(void)
 }
 
 MODULE_PARM_DESC(bus, "Enumeration of VMEbus to which the driver is connected");
-module_param_array(bus, int, &bus_num, 0);
+module_param_array(bus, int, &bus_num, 0000);
 
 MODULE_DESCRIPTION("VME User Space Access Driver");
 MODULE_AUTHOR("Martyn Welch <martyn.welch@ge.com");
diff --git a/drivers/staging/vt6655/baseband.c b/drivers/staging/vt6655/baseband.c
index 654d072bdc28..de503a316e71 100644
--- a/drivers/staging/vt6655/baseband.c
+++ b/drivers/staging/vt6655/baseband.c
@@ -1729,7 +1729,7 @@ BBuGetFrameTime(
 	unsigned int uFrameTime;
 	unsigned int uPreamble;
 	unsigned int uTmp;
-	unsigned int uRateIdx = (unsigned int) wRate;
+	unsigned int uRateIdx = (unsigned int)wRate;
 	unsigned int uRate = 0;
 
 	if (uRateIdx > RATE_54M)
diff --git a/drivers/staging/vt6655/card.c b/drivers/staging/vt6655/card.c
index afb1e8bde975..dbcea4434725 100644
--- a/drivers/staging/vt6655/card.c
+++ b/drivers/staging/vt6655/card.c
@@ -619,7 +619,7 @@ CARDvSafeResetRx(
 static unsigned short CARDwGetCCKControlRate(struct vnt_private *priv,
 					     unsigned short wRateIdx)
 {
-	unsigned int ui = (unsigned int) wRateIdx;
+	unsigned int ui = (unsigned int)wRateIdx;
 
 	while (ui > RATE_1M) {
 		if (priv->basic_rates & ((u32)0x1 << ui))
@@ -645,7 +645,7 @@ static unsigned short CARDwGetCCKControlRate(struct vnt_private *priv,
 static unsigned short CARDwGetOFDMControlRate(struct vnt_private *priv,
 					      unsigned short wRateIdx)
 {
-	unsigned int ui = (unsigned int) wRateIdx;
+	unsigned int ui = (unsigned int)wRateIdx;
 
 	pr_debug("BASIC RATE: %X\n", priv->basic_rates);
 
diff --git a/drivers/staging/vt6655/channel.c b/drivers/staging/vt6655/channel.c
index b7d43a5622ba..029a8df4ca1c 100644
--- a/drivers/staging/vt6655/channel.c
+++ b/drivers/staging/vt6655/channel.c
@@ -193,7 +193,8 @@ bool set_channel(struct vnt_private *priv, struct ieee80211_channel *ch)
 	MACvRegBitsOn(priv->PortOffset, MAC_REG_MACCR, MACCR_CLRNAV);
 
 	/* TX_PE will reserve 3 us for MAX2829 A mode only,
-	   it is for better TX throughput */
+	 * it is for better TX throughput
+	 */
 
 	if (priv->byRFType == RF_AIROHA7230)
 		RFbAL7230SelectChannelPostProcess(priv, priv->byCurrentCh,
diff --git a/drivers/staging/vt6655/device_main.c b/drivers/staging/vt6655/device_main.c
index 494164045a0f..f109eeac358d 100644
--- a/drivers/staging/vt6655/device_main.c
+++ b/drivers/staging/vt6655/device_main.c
@@ -113,10 +113,10 @@ DEVICE_PARAM(ShortRetryLimit, "Short frame retry limits");
 DEVICE_PARAM(LongRetryLimit, "long frame retry limits");
 
 /* BasebandType[] baseband type selected
-   0: indicate 802.11a type
-   1: indicate 802.11b type
-   2: indicate 802.11g type
-*/
+ * 0: indicate 802.11a type
+ * 1: indicate 802.11b type
+ * 2: indicate 802.11g type
+ */
 #define BBP_TYPE_MIN     0
 #define BBP_TYPE_MAX     2
 #define BBP_TYPE_DEF     2
@@ -477,7 +477,7 @@ static bool device_init_rings(struct vnt_private *priv)
 					     CB_MAX_BUF_SIZE,
 					     &priv->tx_bufs_dma0,
 					     GFP_ATOMIC);
-	if (priv->tx0_bufs == NULL) {
+	if (!priv->tx0_bufs) {
 		dev_err(&priv->pcid->dev, "allocate buf dma memory failed\n");
 
 		dma_free_coherent(&priv->pcid->dev,
@@ -735,7 +735,7 @@ static bool device_alloc_rx_buf(struct vnt_private *priv,
 	struct vnt_rd_info *rd_info = rd->rd_info;
 
 	rd_info->skb = dev_alloc_skb((int)priv->rx_buf_sz);
-	if (rd_info->skb == NULL)
+	if (!rd_info->skb)
 		return false;
 
 	rd_info->skb_dma =
diff --git a/drivers/staging/vt6655/key.c b/drivers/staging/vt6655/key.c
index ffcaf25fdd8b..e161d5d9aebb 100644
--- a/drivers/staging/vt6655/key.c
+++ b/drivers/staging/vt6655/key.c
@@ -31,16 +31,6 @@
 #include "key.h"
 #include "mac.h"
 
-int vnt_key_init_table(struct vnt_private *priv)
-{
-	u32 i;
-
-	for (i = 0; i < MAX_KEY_TABLE; i++)
-		MACvDisableKeyEntry(priv, i);
-
-	return 0;
-}
-
 static int vnt_set_keymode(struct ieee80211_hw *hw, u8 *mac_addr,
 	struct ieee80211_key_conf *key, u32 key_type, u32 mode,
 	bool onfly_latch)
diff --git a/drivers/staging/vt6655/key.h b/drivers/staging/vt6655/key.h
index 261f8181d410..d72719741a56 100644
--- a/drivers/staging/vt6655/key.h
+++ b/drivers/staging/vt6655/key.h
@@ -61,8 +61,6 @@
 
 struct vnt_private;
 
-int vnt_key_init_table(struct vnt_private *);
-
 int vnt_set_keys(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
 		 struct ieee80211_vif *vif, struct ieee80211_key_conf *key);
 
diff --git a/drivers/staging/vt6655/power.c b/drivers/staging/vt6655/power.c
index bc8ca981a629..7d6e7464ae51 100644
--- a/drivers/staging/vt6655/power.c
+++ b/drivers/staging/vt6655/power.c
@@ -52,7 +52,7 @@
 
 /*---------------------  Export Functions  --------------------------*/
 
-/*+
+/*
  *
  * Routine Description:
  * Enable hw power saving functions
@@ -60,7 +60,7 @@
  * Return Value:
  *    None.
  *
- -*/
+ */
 
 void
 PSvEnablePowerSaving(
@@ -104,7 +104,7 @@ PSvEnablePowerSaving(
 	pr_debug("PS:Power Saving Mode Enable...\n");
 }
 
-/*+
+/*
  *
  * Routine Description:
  * Disable hw power saving functions
@@ -112,7 +112,7 @@ PSvEnablePowerSaving(
  * Return Value:
  *    None.
  *
- -*/
+ */
 
 void
 PSvDisablePowerSaving(
@@ -134,7 +134,7 @@ PSvDisablePowerSaving(
 }
 
 
-/*+
+/*
  *
  * Routine Description:
  * Check if Next TBTT must wake up
@@ -142,7 +142,7 @@ PSvDisablePowerSaving(
  * Return Value:
  *    None.
  *
- -*/
+ */
 
 bool
 PSbIsNextTBTTWakeUp(
diff --git a/drivers/staging/vt6655/rf.c b/drivers/staging/vt6655/rf.c
index ae10da21ddd0..447882c7a6be 100644
--- a/drivers/staging/vt6655/rf.c
+++ b/drivers/staging/vt6655/rf.c
@@ -169,7 +169,8 @@ static unsigned long dwAL2230PowerTable[AL2230_PWR_IDX_LEN] = {
 };
 
 /* 40MHz reference frequency
- * Need to Pull PLLON(PE3) low when writing channel registers through 3-wire.*/
+ * Need to Pull PLLON(PE3) low when writing channel registers through 3-wire.
+ */
 static const unsigned long dwAL7230InitTable[CB_AL7230_INIT_SEQ] = {
 	0x00379000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Channel1 // Need modify for 11a */
 	0x13333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Channel1 // Need modify for 11a */
@@ -463,7 +464,8 @@ static bool s_bAL7230Init(struct vnt_private *priv)
 }
 
 /* Need to Pull PLLON low when writing channel registers through
- * 3-wire interface */
+ * 3-wire interface
+ */
 static bool s_bAL7230SelectChannel(struct vnt_private *priv, unsigned char byChannel)
 {
 	void __iomem *dwIoBase = priv->PortOffset;
@@ -873,7 +875,8 @@ bool RFbRawSetPower(
 
 	case RF_AIROHA7230:
 		/* 0x080F1B00 for 3 wire control TxGain(D10)
-		 * and 0x31 as TX Gain value */
+		 * and 0x31 as TX Gain value
+		 */
 		dwMax7230Pwr = 0x080C0B00 | ((byPwr) << 12) |
 			(BY_AL7230_REG_LEN << 3)  | IFREGCTL_REGW;
 
@@ -886,7 +889,7 @@ bool RFbRawSetPower(
 	return ret;
 }
 
-/*+
+/*
  *
  * Routine Description:
  *     Translate RSSI to dBm
@@ -900,7 +903,7 @@ bool RFbRawSetPower(
  *
  * Return Value: none
  *
- -*/
+ */
 void
 RFvRSSITodBm(
 	struct vnt_private *priv,
@@ -927,7 +930,8 @@ RFvRSSITodBm(
 }
 
 /* Post processing for the 11b/g and 11a.
- * for save time on changing Reg2,3,5,7,10,12,15 */
+ * for save time on changing Reg2,3,5,7,10,12,15
+ */
 bool RFbAL7230SelectChannelPostProcess(struct vnt_private *priv,
 				       u16 byOldChannel,
 				       u16 byNewChannel)
@@ -938,7 +942,8 @@ bool RFbAL7230SelectChannelPostProcess(struct vnt_private *priv,
 
 	/* if change between 11 b/g and 11a need to update the following
 	 * register
-	 * Channel Index 1~14 */
+	 * Channel Index 1~14
+	 */
 	if ((byOldChannel <= CB_MAX_CHANNEL_24G) && (byNewChannel > CB_MAX_CHANNEL_24G)) {
 		/* Change from 2.4G to 5G [Reg] */
 		ret &= IFRFbWriteEmbedded(priv, dwAL7230InitTableAMode[2]);
diff --git a/drivers/staging/vt6655/rxtx.c b/drivers/staging/vt6655/rxtx.c
index e4c3165ae027..7e69bc99d60f 100644
--- a/drivers/staging/vt6655/rxtx.c
+++ b/drivers/staging/vt6655/rxtx.c
@@ -64,8 +64,10 @@
 /*---------------------  Static Functions  --------------------------*/
 
 /*---------------------  Static Definitions -------------------------*/
-#define CRITICAL_PACKET_LEN      256    /* if packet size < 256 -> in-direct send
-                                            packet size >= 256 -> direct send */
+/* if packet size < 256 -> in-direct send
+ * vpacket size >= 256 -> direct send
+ */
+#define CRITICAL_PACKET_LEN      256
 
 static const unsigned short wTimeStampOff[2][MAX_RATE] = {
 	{384, 288, 226, 209, 54, 43, 37, 31, 28, 25, 24, 23}, /* Long Preamble */
@@ -158,11 +160,11 @@ static __le16 vnt_time_stamp_off(struct vnt_private *priv, u16 rate)
 							[rate % MAX_RATE]);
 }
 
-/*byPktType : PK_TYPE_11A     0
-  PK_TYPE_11B     1
-  PK_TYPE_11GB    2
-  PK_TYPE_11GA    3
-*/
+/* byPktType : PK_TYPE_11A     0
+ * PK_TYPE_11B     1
+ * PK_TYPE_11GB    2
+ * PK_TYPE_11GA    3
+ */
 static
 unsigned int
 s_uGetTxRsvTime(
@@ -502,7 +504,7 @@ s_uFillDataHead(
 )
 {
 
-	if (pTxDataHead == NULL)
+	if (!pTxDataHead)
 		return 0;
 
 
@@ -646,17 +648,20 @@ s_vFillRTSHead(
 {
 	unsigned int uRTSFrameLen = 20;
 
-	if (pvRTS == NULL)
+	if (!pvRTS)
 		return;
 
 	if (bDisCRC) {
-		/* When CRCDIS bit is on, H/W forgot to generate FCS for RTS frame,
-		 in this case we need to decrease its length by 4. */
+		/* When CRCDIS bit is on, H/W forgot to generate FCS for
+		 * RTS frame, in this case we need to decrease its length by 4.
+		 */
 		uRTSFrameLen -= 4;
 	}
 
-	/* Note: So far RTSHead doesn't appear in ATIM & Beacom DMA, so we don't need to take them into account.
-	       Otherwise, we need to modify codes for them. */
+	/* Note: So far RTSHead doesn't appear in ATIM & Beacom DMA,
+	 * so we don't need to take them into account.
+	 * Otherwise, we need to modify codes for them.
+	 */
 	if (byPktType == PK_TYPE_11GB || byPktType == PK_TYPE_11GA) {
 		if (byFBOption == AUTO_FB_NONE) {
 			struct vnt_rts_g *buf = pvRTS;
@@ -838,12 +843,13 @@ s_vFillCTSHead(
 {
 	unsigned int uCTSFrameLen = 14;
 
-	if (pvCTS == NULL)
+	if (!pvCTS)
 		return;
 
 	if (bDisCRC) {
-		/* When CRCDIS bit is on, H/W forgot to generate FCS for CTS frame,
-		 in this case we need to decrease its length by 4. */
+		/* When CRCDIS bit is on, H/W forgot to generate FCS for
+		 * CTS frame, in this case we need to decrease its length by 4.
+		 */
 		uCTSFrameLen -= 4;
 	}
 
@@ -915,7 +921,7 @@ s_vFillCTSHead(
 	}
 }
 
-/*+
+/*
  *
  * Description:
  *      Generate FIFO control for MAC & Baseband controller
@@ -937,7 +943,8 @@ s_vFillCTSHead(
  * Return Value: none
  *
  -
- * unsigned int cbFrameSize, Hdr+Payload+FCS */
+ * unsigned int cbFrameSize, Hdr+Payload+FCS
+ */
 static
 void
 s_vGenerateTxParameter(
@@ -972,8 +979,8 @@ s_vGenerateTxParameter(
 		return;
 
 	if (byPktType == PK_TYPE_11GB || byPktType == PK_TYPE_11GA) {
-		if (pvRTS != NULL) { /* RTS_need
-			 Fill RsvTime */
+		if (pvRTS != NULL) { /* RTS_need */
+			/* Fill RsvTime */
 			struct vnt_rrv_time_rts *buf = pvRrvTime;
 
 			buf->rts_rrv_time_aa = s_uGetRTSCTSRsvTime(pDevice, 2, byPktType, cbFrameSize, wCurrentRate);
@@ -1002,7 +1009,7 @@ s_vGenerateTxParameter(
 
 			/* Fill RTS */
 			s_vFillRTSHead(pDevice, byPktType, pvRTS, cbFrameSize, bNeedACK, bDisCRC, psEthHeader, wCurrentRate, byFBOption);
-		} else if (pvRTS == NULL) {/* RTS_needless, non PCF mode */
+		} else if (!pvRTS) {/* RTS_needless, non PCF mode */
 			struct vnt_rrv_time_ab *buf = pvRrvTime;
 
 			buf->rrv_time = vnt_rxtx_rsvtime_le16(pDevice, PK_TYPE_11A, cbFrameSize, wCurrentRate, bNeedACK);
diff --git a/drivers/staging/vt6656/baseband.h b/drivers/staging/vt6656/baseband.h
index 807a5809b5d9..7cc13874f8f1 100644
--- a/drivers/staging/vt6656/baseband.h
+++ b/drivers/staging/vt6656/baseband.h
@@ -84,10 +84,10 @@ struct vnt_phy_field {
 } __packed;
 
 unsigned int vnt_get_frame_time(u8 preamble_type, u8 pkt_type,
-	unsigned int frame_length, u16 tx_rate);
+				unsigned int frame_length, u16 tx_rate);
 
 void vnt_get_phy_field(struct vnt_private *, u32 frame_length,
-	u16 tx_rate, u8 pkt_type, struct vnt_phy_field *);
+		       u16 tx_rate, u8 pkt_type, struct vnt_phy_field *);
 
 void vnt_set_short_slot_time(struct vnt_private *);
 void vnt_set_vga_gain_offset(struct vnt_private *, u8);
diff --git a/drivers/staging/vt6656/card.c b/drivers/staging/vt6656/card.c
index a382fc6aa9d3..53b469c71dc2 100644
--- a/drivers/staging/vt6656/card.c
+++ b/drivers/staging/vt6656/card.c
@@ -46,10 +46,11 @@
 #include "key.h"
 #include "usbpipe.h"
 
-/* const u16 cwRXBCNTSFOff[MAX_RATE] =
-   {17, 34, 96, 192, 34, 23, 17, 11, 8, 5, 4, 3}; */
+/* const u16 cw_rxbcntsf_off[MAX_RATE] =
+ *   {17, 34, 96, 192, 34, 23, 17, 11, 8, 5, 4, 3};
+ */
 
-static const u16 cwRXBCNTSFOff[MAX_RATE] = {
+static const u16 cw_rxbcntsf_off[MAX_RATE] = {
 	192, 96, 34, 17, 34, 23, 17, 11, 8, 5, 4, 3
 };
 
@@ -65,7 +66,6 @@ static const u16 cwRXBCNTSFOff[MAX_RATE] = {
  */
 void vnt_set_channel(struct vnt_private *priv, u32 connection_channel)
 {
-
 	if (connection_channel > CB_MAX_CHANNEL || !connection_channel)
 		return;
 
@@ -76,10 +76,10 @@ void vnt_set_channel(struct vnt_private *priv, u32 connection_channel)
 	vnt_mac_reg_bits_off(priv, MAC_REG_CHANNEL, 0xb0);
 
 	vnt_control_out(priv, MESSAGE_TYPE_SELECT_CHANNEL,
-					connection_channel, 0, 0, NULL);
+			connection_channel, 0, 0, NULL);
 
 	vnt_control_out_u8(priv, MESSAGE_REQUEST_MACREG, MAC_REG_CHANNEL,
-		(u8)(connection_channel | 0x80));
+			   (u8)(connection_channel | 0x80));
 }
 
 /*
@@ -126,11 +126,11 @@ static u16 vnt_get_ofdm_rate(struct vnt_private *priv, u16 rate_idx)
 	u16 ui = rate_idx;
 
 	dev_dbg(&priv->usb->dev, "%s basic rate: %d\n",
-					__func__,  priv->basic_rates);
+		__func__,  priv->basic_rates);
 
 	if (!vnt_ofdm_min_rate(priv)) {
 		dev_dbg(&priv->usb->dev, "%s (NO OFDM) %d\n",
-						__func__, rate_idx);
+			__func__, rate_idx);
 		if (rate_idx > RATE_24M)
 			rate_idx = RATE_24M;
 		return rate_idx;
@@ -139,7 +139,7 @@ static u16 vnt_get_ofdm_rate(struct vnt_private *priv, u16 rate_idx)
 	while (ui > RATE_11M) {
 		if (priv->basic_rates & (1 << ui)) {
 			dev_dbg(&priv->usb->dev, "%s rate: %d\n",
-							__func__, ui);
+				__func__, ui);
 			return ui;
 		}
 		ui--;
@@ -165,9 +165,8 @@ static u16 vnt_get_ofdm_rate(struct vnt_private *priv, u16 rate_idx)
  *
  */
 static void vnt_calculate_ofdm_rate(u16 rate, u8 bb_type,
-					u8 *tx_rate, u8 *rsv_time)
+				    u8 *tx_rate, u8 *rsv_time)
 {
-
 	switch (rate) {
 	case RATE_6M:
 		if (bb_type == BB_TYPE_11A) {
@@ -267,20 +266,20 @@ void vnt_set_rspinf(struct vnt_private *priv, u8 bb_type)
 	int i;
 
 	/*RSPINF_b_1*/
-	vnt_get_phy_field(priv, 14,
-		vnt_get_cck_rate(priv, RATE_1M), PK_TYPE_11B, &phy[0]);
+	vnt_get_phy_field(priv, 14, vnt_get_cck_rate(priv, RATE_1M),
+			  PK_TYPE_11B, &phy[0]);
 
 	/*RSPINF_b_2*/
-	vnt_get_phy_field(priv, 14,
-		vnt_get_cck_rate(priv, RATE_2M), PK_TYPE_11B, &phy[1]);
+	vnt_get_phy_field(priv, 14, vnt_get_cck_rate(priv, RATE_2M),
+			  PK_TYPE_11B, &phy[1]);
 
 	/*RSPINF_b_5*/
-	vnt_get_phy_field(priv, 14,
-		vnt_get_cck_rate(priv, RATE_5M), PK_TYPE_11B, &phy[2]);
+	vnt_get_phy_field(priv, 14, vnt_get_cck_rate(priv, RATE_5M),
+			  PK_TYPE_11B, &phy[2]);
 
 	/*RSPINF_b_11*/
-	vnt_get_phy_field(priv, 14,
-		vnt_get_cck_rate(priv, RATE_11M), PK_TYPE_11B, &phy[3]);
+	vnt_get_phy_field(priv, 14, vnt_get_cck_rate(priv, RATE_11M),
+			  PK_TYPE_11B, &phy[3]);
 
 	/*RSPINF_a_6*/
 	vnt_calculate_ofdm_rate(RATE_6M, bb_type, &tx_rate[0], &rsv_time[0]);
@@ -299,19 +298,19 @@ void vnt_set_rspinf(struct vnt_private *priv, u8 bb_type)
 
 	/*RSPINF_a_36*/
 	vnt_calculate_ofdm_rate(vnt_get_ofdm_rate(priv, RATE_36M),
-					bb_type, &tx_rate[5], &rsv_time[5]);
+				bb_type, &tx_rate[5], &rsv_time[5]);
 
 	/*RSPINF_a_48*/
 	vnt_calculate_ofdm_rate(vnt_get_ofdm_rate(priv, RATE_48M),
-					bb_type, &tx_rate[6], &rsv_time[6]);
+				bb_type, &tx_rate[6], &rsv_time[6]);
 
 	/*RSPINF_a_54*/
 	vnt_calculate_ofdm_rate(vnt_get_ofdm_rate(priv, RATE_54M),
-					bb_type, &tx_rate[7], &rsv_time[7]);
+				bb_type, &tx_rate[7], &rsv_time[7]);
 
 	/*RSPINF_a_72*/
 	vnt_calculate_ofdm_rate(vnt_get_ofdm_rate(priv, RATE_54M),
-					bb_type, &tx_rate[8], &rsv_time[8]);
+				bb_type, &tx_rate[8], &rsv_time[8]);
 
 	put_unaligned(phy[0].len, (u16 *)&data[0]);
 	data[2] = phy[0].signal;
@@ -334,8 +333,8 @@ void vnt_set_rspinf(struct vnt_private *priv, u8 bb_type)
 		data[16 + i * 2 + 1] = rsv_time[i];
 	}
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE,
-		MAC_REG_RSPINF_B_1, MESSAGE_REQUEST_MACREG, 34, &data[0]);
+	vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_RSPINF_B_1,
+			MESSAGE_REQUEST_MACREG, 34, &data[0]);
 }
 
 /*
@@ -429,12 +428,12 @@ void vnt_update_ifs(struct vnt_private *priv)
 	data[3] = (u8)priv->slot;
 
 	vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_SIFS,
-		MESSAGE_REQUEST_MACREG, 4, &data[0]);
+			MESSAGE_REQUEST_MACREG, 4, &data[0]);
 
 	max_min |= 0xa0;
 
 	vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_CWMAXMIN0,
-		MESSAGE_REQUEST_MACREG, 1, &max_min);
+			MESSAGE_REQUEST_MACREG, 1, &max_min);
 }
 
 void vnt_update_top_rates(struct vnt_private *priv)
@@ -478,7 +477,6 @@ int vnt_ofdm_min_rate(struct vnt_private *priv)
 
 u8 vnt_get_pkt_type(struct vnt_private *priv)
 {
-
 	if (priv->bb_type == BB_TYPE_11A || priv->bb_type == BB_TYPE_11B)
 		return (u8)priv->bb_type;
 	else if (vnt_ofdm_min_rate(priv))
@@ -506,7 +504,7 @@ u64 vnt_get_tsf_offset(u8 rx_rate, u64 tsf1, u64 tsf2)
 	u64 tsf_offset = 0;
 	u16 rx_bcn_offset;
 
-	rx_bcn_offset = cwRXBCNTSFOff[rx_rate % MAX_RATE];
+	rx_bcn_offset = cw_rxbcntsf_off[rx_rate % MAX_RATE];
 
 	tsf2 += (u64)rx_bcn_offset;
 
@@ -531,7 +529,7 @@ u64 vnt_get_tsf_offset(u8 rx_rate, u64 tsf1, u64 tsf2)
  *
  */
 void vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
-		u64 time_stamp, u64 local_tsf)
+		    u64 time_stamp, u64 local_tsf)
 {
 	u64 tsf_offset = 0;
 	u8 data[8];
@@ -548,8 +546,9 @@ void vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
 	data[7] = (u8)(tsf_offset >> 56);
 
 	vnt_control_out(priv, MESSAGE_TYPE_SET_TSFTBTT,
-		MESSAGE_REQUEST_TSF, 0, 8, data);
+			MESSAGE_REQUEST_TSF, 0, 8, data);
 }
+
 /*
  * Description: Read NIC TSF counter
  *              Get local TSF counter
@@ -565,7 +564,6 @@ void vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
  */
 bool vnt_get_current_tsf(struct vnt_private *priv, u64 *current_tsf)
 {
-
 	*current_tsf = priv->current_tsf;
 
 	return true;
@@ -584,7 +582,6 @@ bool vnt_get_current_tsf(struct vnt_private *priv, u64 *current_tsf)
  */
 bool vnt_clear_current_tsf(struct vnt_private *priv)
 {
-
 	vnt_mac_reg_bits_on(priv, MAC_REG_TFTCTL, TFTCTL_TSFCNTRST);
 
 	priv->current_tsf = 0;
@@ -657,7 +654,7 @@ void vnt_reset_next_tbtt(struct vnt_private *priv, u16 beacon_interval)
 	data[7] = (u8)(next_tbtt >> 56);
 
 	vnt_control_out(priv, MESSAGE_TYPE_SET_TSFTBTT,
-		MESSAGE_REQUEST_TBTT, 0, 8, data);
+			MESSAGE_REQUEST_TBTT, 0, 8, data);
 }
 
 /*
@@ -676,7 +673,7 @@ void vnt_reset_next_tbtt(struct vnt_private *priv, u16 beacon_interval)
  *
  */
 void vnt_update_next_tbtt(struct vnt_private *priv, u64 tsf,
-			u16 beacon_interval)
+			  u16 beacon_interval)
 {
 	u8 data[8];
 
@@ -721,7 +718,7 @@ int vnt_radio_power_off(struct vnt_private *priv)
 	case RF_VT3226D0:
 	case RF_VT3342A0:
 		vnt_mac_reg_bits_off(priv, MAC_REG_SOFTPWRCTL,
-				(SOFTPWRCTL_SWPE2 | SOFTPWRCTL_SWPE3));
+				     (SOFTPWRCTL_SWPE2 | SOFTPWRCTL_SWPE3));
 		break;
 	}
 
@@ -762,7 +759,7 @@ int vnt_radio_power_on(struct vnt_private *priv)
 	case RF_VT3226D0:
 	case RF_VT3342A0:
 		vnt_mac_reg_bits_on(priv, MAC_REG_SOFTPWRCTL,
-			(SOFTPWRCTL_SWPE2 | SOFTPWRCTL_SWPE3));
+				    (SOFTPWRCTL_SWPE2 | SOFTPWRCTL_SWPE3));
 		break;
 	}
 
@@ -795,7 +792,7 @@ void vnt_set_bss_mode(struct vnt_private *priv)
 			priv->bb_vga[0] = 0x20;
 
 			vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG,
-						0xe7, priv->bb_vga[0]);
+					   0xe7, priv->bb_vga[0]);
 		}
 
 		priv->bb_vga[2] = 0x10;
@@ -805,7 +802,7 @@ void vnt_set_bss_mode(struct vnt_private *priv)
 			priv->bb_vga[0] = 0x1c;
 
 			vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG,
-						0xe7, priv->bb_vga[0]);
+					   0xe7, priv->bb_vga[0]);
 		}
 
 		priv->bb_vga[2] = 0x0;
diff --git a/drivers/staging/vt6656/dpc.c b/drivers/staging/vt6656/dpc.c
index 6019aac8bdd5..655f0002f880 100644
--- a/drivers/staging/vt6656/dpc.c
+++ b/drivers/staging/vt6656/dpc.c
@@ -34,7 +34,7 @@
 #include "rf.h"
 
 int vnt_rx_data(struct vnt_private *priv, struct vnt_rcb *ptr_rcb,
-	unsigned long bytes_received)
+		unsigned long bytes_received)
 {
 	struct ieee80211_hw *hw = priv->hw;
 	struct ieee80211_supported_band *sband;
@@ -46,7 +46,7 @@ int vnt_rx_data(struct vnt_private *priv, struct vnt_rcb *ptr_rcb,
 	__le64 *tsf_time;
 	u32 frame_size;
 	int ii, r;
-	u8 *rx_sts, *rx_rate, *sq, *sq_3;
+	u8 *rx_rate, *sq, *sq_3;
 	u32 wbk_status;
 	u8 *skb_data;
 	u16 *pay_load_len;
@@ -75,22 +75,21 @@ int vnt_rx_data(struct vnt_private *priv, struct vnt_rcb *ptr_rcb,
 
 	skb_data = (u8 *)skb->data;
 
-	rx_sts = skb_data+4;
-	rx_rate = skb_data+5;
+	rx_rate = skb_data + 5;
 
 	/* real Frame Size = USBframe_size -4WbkStatus - 4RxStatus */
 	/* -8TSF - 4RSR - 4SQ3 - ?Padding */
 
 	/* if SQ3 the range is 24~27, if no SQ3 the range is 20~23 */
 
-	pay_load_len = (u16 *) (skb_data + 6);
+	pay_load_len = (u16 *)(skb_data + 6);
 
 	/*Fix hardware bug => PLCP_Length error */
 	if (((bytes_received - (*pay_load_len)) > 27) ||
-		((bytes_received - (*pay_load_len)) < 24) ||
-			(bytes_received < (*pay_load_len))) {
+	    ((bytes_received - (*pay_load_len)) < 24) ||
+	    (bytes_received < (*pay_load_len))) {
 		dev_dbg(&priv->usb->dev, "Wrong PLCP Length %x\n",
-							*pay_load_len);
+			*pay_load_len);
 		return false;
 	}
 
diff --git a/drivers/staging/vt6656/dpc.h b/drivers/staging/vt6656/dpc.h
index 5a92bd86cee2..ff1850c4a927 100644
--- a/drivers/staging/vt6656/dpc.h
+++ b/drivers/staging/vt6656/dpc.h
@@ -29,6 +29,6 @@
 #include "device.h"
 
 int vnt_rx_data(struct vnt_private *, struct vnt_rcb *,
-	unsigned long bytes_received);
+		unsigned long bytes_received);
 
 #endif /* __RXTX_H__ */
diff --git a/drivers/staging/vt6656/main_usb.c b/drivers/staging/vt6656/main_usb.c
index ac4fecb30d0e..0594828bdabf 100644
--- a/drivers/staging/vt6656/main_usb.c
+++ b/drivers/staging/vt6656/main_usb.c
@@ -440,10 +440,8 @@ static bool vnt_alloc_bufs(struct vnt_private *priv)
 
 		/* allocate URBs */
 		tx_context->urb = usb_alloc_urb(0, GFP_KERNEL);
-		if (!tx_context->urb) {
-			dev_err(&priv->usb->dev, "alloc tx urb failed\n");
+		if (!tx_context->urb)
 			goto free_tx;
-		}
 
 		tx_context->in_use = false;
 	}
@@ -462,10 +460,8 @@ static bool vnt_alloc_bufs(struct vnt_private *priv)
 
 		/* allocate URBs */
 		rcb->urb = usb_alloc_urb(0, GFP_KERNEL);
-		if (!rcb->urb) {
-			dev_err(&priv->usb->dev, "Failed to alloc rx urb\n");
+		if (!rcb->urb)
 			goto free_rx_tx;
-		}
 
 		rcb->skb = dev_alloc_skb(priv->rx_buf_sz);
 		if (!rcb->skb)
@@ -479,10 +475,8 @@ static bool vnt_alloc_bufs(struct vnt_private *priv)
 	}
 
 	priv->interrupt_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!priv->interrupt_urb) {
-		dev_err(&priv->usb->dev, "Failed to alloc int urb\n");
+	if (!priv->interrupt_urb)
 		goto free_rx_tx;
-	}
 
 	priv->int_buf.data_buf = kmalloc(MAX_INTERRUPT_SIZE, GFP_KERNEL);
 	if (!priv->int_buf.data_buf) {
diff --git a/drivers/staging/vt6656/usbpipe.c b/drivers/staging/vt6656/usbpipe.c
index f546553de66f..e9b6b21f7422 100644
--- a/drivers/staging/vt6656/usbpipe.c
+++ b/drivers/staging/vt6656/usbpipe.c
@@ -28,8 +28,9 @@
  *	vnt_control_in_u8 - Read one byte from MEM/BB/MAC/EEPROM
  *
  * Revision History:
- *      04-05-2004 Jerry Chen:  Initial release
- *      11-24-2004 Warren Hsu: Add ControlvWriteByte,ControlvReadByte,ControlvMaskByte
+ *      04-05-2004 Jerry Chen: Initial release
+ *      11-24-2004 Warren Hsu: Add ControlvWriteByte,ControlvReadByte,
+ *                             ControlvMaskByte
  *
  */
 
diff --git a/drivers/staging/wilc1000/TODO b/drivers/staging/wilc1000/TODO
index ec93b2ee0b08..ae61b55f14fd 100644
--- a/drivers/staging/wilc1000/TODO
+++ b/drivers/staging/wilc1000/TODO
@@ -3,7 +3,6 @@ TODO:
 - remove OS wrapper functions
 - remove custom debug and tracing functions
 - rework comments and function headers(also coding style)
-- replace all semaphores with mutexes or completions
 - Move handling for each individual members of 'union message_body' out
   into a separate 'struct work_struct' and completely remove the multiplexer
   that is currently part of host_if_work(), allowing movement of the
diff --git a/drivers/staging/wilc1000/coreconfigurator.h b/drivers/staging/wilc1000/coreconfigurator.h
index 076e06ac0d66..cff16984167b 100644
--- a/drivers/staging/wilc1000/coreconfigurator.h
+++ b/drivers/staging/wilc1000/coreconfigurator.h
@@ -70,11 +70,11 @@ enum connect_status {
 	CONNECT_STS_FORCE_16_BIT = 0xFFFF
 };
 
-typedef struct {
+struct tstrRSSI {
 	u8 u8Full;
 	u8 u8Index;
 	s8 as8RSSI[NUM_RSSI];
-} tstrRSSI;
+};
 
 struct network_info {
 	s8 rssi;
@@ -93,7 +93,7 @@ struct network_info {
 	u8 *ies;
 	u16 ies_len;
 	void *join_params;
-	tstrRSSI str_rssi;
+	struct tstrRSSI str_rssi;
 	u64 tsf_hi;
 };
 
diff --git a/drivers/staging/wilc1000/host_interface.c b/drivers/staging/wilc1000/host_interface.c
index 0b1760cba6e3..78f5613e9467 100644
--- a/drivers/staging/wilc1000/host_interface.c
+++ b/drivers/staging/wilc1000/host_interface.c
@@ -3279,7 +3279,6 @@ int wilc_scan(struct wilc_vif *vif, u8 scan_source, u8 scan_type,
 int wilc_hif_set_cfg(struct wilc_vif *vif,
 		     struct cfg_param_attr *cfg_param)
 {
-	int result = 0;
 	struct host_if_msg msg;
 	struct host_if_drv *hif_drv = vif->hif_drv;
 
@@ -3293,9 +3292,7 @@ int wilc_hif_set_cfg(struct wilc_vif *vif,
 	msg.body.cfg_info = *cfg_param;
 	msg.vif = vif;
 
-	result = wilc_enqueue_cmd(&msg);
-
-	return result;
+	return wilc_enqueue_cmd(&msg);
 }
 
 static void GetPeriodicRSSI(unsigned long arg)
@@ -3329,7 +3326,7 @@ int wilc_init(struct net_device *dev, struct host_if_drv **hif_drv_handler)
 
 	init_completion(&hif_wait_response);
 
-	hif_drv  = kzalloc(sizeof(struct host_if_drv), GFP_KERNEL);
+	hif_drv  = kzalloc(sizeof(*hif_drv), GFP_KERNEL);
 	if (!hif_drv) {
 		result = -ENOMEM;
 		goto _fail_;
@@ -3363,7 +3360,7 @@ int wilc_init(struct net_device *dev, struct host_if_drv **hif_drv_handler)
 		if (!hif_workqueue) {
 			netdev_err(vif->ndev, "Failed to create workqueue\n");
 			result = -ENOMEM;
-			goto _fail_mq_;
+			goto _fail_;
 		}
 
 		setup_timer(&periodic_rssi, GetPeriodicRSSI,
@@ -3391,7 +3388,6 @@ int wilc_init(struct net_device *dev, struct host_if_drv **hif_drv_handler)
 
 	clients_count++;
 
-_fail_mq_:
 	destroy_workqueue(hif_workqueue);
 _fail_:
 	return result;
@@ -3879,7 +3875,7 @@ static void *host_int_ParseJoinBssParam(struct network_info *ptstrNetworkInfo)
 	pu8IEs = ptstrNetworkInfo->ies;
 	u16IEsLen = ptstrNetworkInfo->ies_len;
 
-	pNewJoinBssParam = kzalloc(sizeof(struct join_bss_param), GFP_KERNEL);
+	pNewJoinBssParam = kzalloc(sizeof(*pNewJoinBssParam), GFP_KERNEL);
 	if (pNewJoinBssParam) {
 		pNewJoinBssParam->dtim_period = ptstrNetworkInfo->dtim_period;
 		pNewJoinBssParam->beacon_period = ptstrNetworkInfo->beacon_period;
diff --git a/drivers/staging/wilc1000/linux_wlan.c b/drivers/staging/wilc1000/linux_wlan.c
index 3a66255f14fc..6370a5efe343 100644
--- a/drivers/staging/wilc1000/linux_wlan.c
+++ b/drivers/staging/wilc1000/linux_wlan.c
@@ -21,7 +21,6 @@
 #include <linux/kernel.h>
 #include <linux/skbuff.h>
 #include <linux/mutex.h>
-#include <linux/semaphore.h>
 #include <linux/completion.h>
 
 static int dev_state_ev_handler(struct notifier_block *this,
@@ -648,7 +647,7 @@ void wilc1000_wlan_deinit(struct net_device *dev)
 			mutex_unlock(&wl->hif_cs);
 		}
 		if (&wl->txq_event)
-			wait_for_completion(&wl->txq_event);
+			complete(&wl->txq_event);
 
 		wlan_deinitialize_threads(dev);
 		deinit_irq(dev);
diff --git a/drivers/staging/wilc1000/wilc_debugfs.c b/drivers/staging/wilc1000/wilc_debugfs.c
index fcbc95d19d8e..802bb1d5e207 100644
--- a/drivers/staging/wilc1000/wilc_debugfs.c
+++ b/drivers/staging/wilc1000/wilc_debugfs.c
@@ -29,7 +29,7 @@ static struct dentry *wilc_dir;
 #define ERR             BIT(3)
 
 #define DBG_LEVEL_ALL	(DEBUG | INFO | WRN | ERR)
-atomic_t WILC_DEBUG_LEVEL = ATOMIC_INIT(ERR);
+static atomic_t WILC_DEBUG_LEVEL = ATOMIC_INIT(ERR);
 EXPORT_SYMBOL_GPL(WILC_DEBUG_LEVEL);
 
 /*
@@ -102,35 +102,16 @@ static struct wilc_debugfs_info_t debugfs_info[] = {
 static int __init wilc_debugfs_init(void)
 {
 	int i;
-
-	struct dentry *debugfs_files;
 	struct wilc_debugfs_info_t *info;
 
 	wilc_dir = debugfs_create_dir("wilc_wifi", NULL);
-	if (wilc_dir ==  ERR_PTR(-ENODEV)) {
-		/* it's not error. the debugfs is just not being enabled. */
-		printk("ERR, kernel has built without debugfs support\n");
-		return 0;
-	}
-
-	if (!wilc_dir) {
-		printk("ERR, debugfs create dir\n");
-		return -1;
-	}
-
 	for (i = 0; i < ARRAY_SIZE(debugfs_info); i++) {
 		info = &debugfs_info[i];
-		debugfs_files = debugfs_create_file(info->name,
-						    info->perm,
-						    wilc_dir,
-						    &info->data,
-						    &info->fops);
-
-		if (!debugfs_files) {
-			printk("ERR fail to create the debugfs file, %s\n", info->name);
-			debugfs_remove_recursive(wilc_dir);
-			return -1;
-		}
+		debugfs_create_file(info->name,
+				    info->perm,
+				    wilc_dir,
+				    &info->data,
+				    &info->fops);
 	}
 	return 0;
 }
diff --git a/drivers/staging/wilc1000/wilc_spi.c b/drivers/staging/wilc1000/wilc_spi.c
index 22cf4b7857e5..f08cf6d9e1af 100644
--- a/drivers/staging/wilc1000/wilc_spi.c
+++ b/drivers/staging/wilc1000/wilc_spi.c
@@ -410,8 +410,7 @@ static int spi_cmd_complete(struct wilc *wilc, u8 cmd, u32 adr, u8 *b, u32 sz,
 	if (len2 > ARRAY_SIZE(wb)) {
 		dev_err(&spi->dev, "spi buffer size too small (%d) (%zu)\n",
 			 len2, ARRAY_SIZE(wb));
-		result = N_FAIL;
-		return result;
+		return N_FAIL;
 	}
 	/* zero spi write buffers. */
 	for (wix = len; wix < len2; wix++)
@@ -420,8 +419,7 @@ static int spi_cmd_complete(struct wilc *wilc, u8 cmd, u32 adr, u8 *b, u32 sz,
 
 	if (wilc_spi_tx_rx(wilc, wb, rb, len2)) {
 		dev_err(&spi->dev, "Failed cmd write, bus error...\n");
-		result = N_FAIL;
-		return result;
+		return N_FAIL;
 	}
 
 	/**
@@ -442,8 +440,7 @@ static int spi_cmd_complete(struct wilc *wilc, u8 cmd, u32 adr, u8 *b, u32 sz,
 		dev_err(&spi->dev,
 			"Failed cmd response, cmd (%02x), resp (%02x)\n",
 			cmd, rsp);
-		result = N_FAIL;
-		return result;
+		return N_FAIL;
 	}
 
 	/**
@@ -453,8 +450,7 @@ static int spi_cmd_complete(struct wilc *wilc, u8 cmd, u32 adr, u8 *b, u32 sz,
 	if (rsp != 0x00) {
 		dev_err(&spi->dev, "Failed cmd state response state (%02x)\n",
 			rsp);
-		result = N_FAIL;
-		return result;
+		return N_FAIL;
 	}
 
 	if ((cmd == CMD_INTERNAL_READ) || (cmd == CMD_SINGLE_READ)
@@ -481,8 +477,7 @@ static int spi_cmd_complete(struct wilc *wilc, u8 cmd, u32 adr, u8 *b, u32 sz,
 		if (retry <= 0) {
 			dev_err(&spi->dev,
 				"Error, data read response (%02x)\n", rsp);
-			result = N_RESET;
-			return result;
+			return N_RESET;
 		}
 
 		if ((cmd == CMD_INTERNAL_READ) || (cmd == CMD_SINGLE_READ)) {
@@ -497,8 +492,7 @@ static int spi_cmd_complete(struct wilc *wilc, u8 cmd, u32 adr, u8 *b, u32 sz,
 			} else {
 				dev_err(&spi->dev,
 					"buffer overrun when reading data.\n");
-				result = N_FAIL;
-				return result;
+				return N_FAIL;
 			}
 
 			if (!g_spi.crc_off) {
@@ -510,8 +504,7 @@ static int spi_cmd_complete(struct wilc *wilc, u8 cmd, u32 adr, u8 *b, u32 sz,
 					crc[1] = rb[rix++];
 				} else {
 					dev_err(&spi->dev, "buffer overrun when reading crc.\n");
-					result = N_FAIL;
-					return result;
+					return N_FAIL;
 				}
 			}
 		} else if ((cmd == CMD_DMA_READ) || (cmd == CMD_DMA_EXT_READ)) {
@@ -551,7 +544,6 @@ static int spi_cmd_complete(struct wilc *wilc, u8 cmd, u32 adr, u8 *b, u32 sz,
 					}
 				}
 
-
 				ix += nbytes;
 				sz -= nbytes;
 			}
@@ -587,7 +579,6 @@ static int spi_cmd_complete(struct wilc *wilc, u8 cmd, u32 adr, u8 *b, u32 sz,
 				if (result == N_FAIL)
 					break;
 
-
 				/**
 				 * Read bytes
 				 **/
@@ -687,7 +678,6 @@ static int spi_data_write(struct wilc *wilc, u8 *b, u32 sz)
 		sz -= nbytes;
 	} while (sz);
 
-
 	return result;
 }
 
@@ -850,7 +840,6 @@ static int wilc_spi_init(struct wilc *wilc, bool resume)
 	static int isinit;
 
 	if (isinit) {
-
 		if (!wilc_spi_read_reg(wilc, 0x1000, &chipid)) {
 			dev_err(&spi->dev, "Fail cmd read chip id...\n");
 			return 0;
@@ -871,7 +860,7 @@ static int wilc_spi_init(struct wilc *wilc, bool resume)
 		/* Read failed. Try with CRC off. This might happen when module
 		 * is removed but chip isn't reset*/
 		g_spi.crc_off = 1;
-		dev_err(&spi->dev, "Failed internal read protocol with CRC on, retyring with CRC off...\n");
+		dev_err(&spi->dev, "Failed internal read protocol with CRC on, retrying with CRC off...\n");
 		if (!spi_internal_read(wilc, WILC_SPI_PROTOCOL_OFFSET, &reg)) {
 			/* Reaad failed with both CRC on and off, something went bad */
 			dev_err(&spi->dev,
@@ -890,7 +879,6 @@ static int wilc_spi_init(struct wilc *wilc, bool resume)
 		g_spi.crc_off = 1;
 	}
 
-
 	/**
 	 *      make sure can read back chip id correctly
 	 **/
@@ -931,14 +919,10 @@ static int wilc_spi_read_size(struct wilc *wilc, u32 *size)
 		*size = tmp;
 	}
 
-
-
 _fail_:
 	return ret;
 }
 
-
-
 static int wilc_spi_read_int(struct wilc *wilc, u32 *int_status)
 {
 	struct spi_device *spi = to_spi_device(wilc->dev);
@@ -993,7 +977,6 @@ static int wilc_spi_read_int(struct wilc *wilc, u32 *int_status)
 		}
 
 		*int_status = tmp;
-
 	}
 
 _fail_:
diff --git a/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c b/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c
index 9092600a1794..60d8b055bb2f 100644
--- a/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c
+++ b/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c
@@ -1191,7 +1191,7 @@ static int get_station(struct wiphy *wiphy, struct net_device *dev,
 	struct wilc_priv *priv;
 	struct wilc_vif *vif;
 	u32 i = 0;
-	u32 associatedsta = 0;
+	u32 associatedsta = ~0;
 	u32 inactive_time = 0;
 	priv = wiphy_priv(wiphy);
 	vif = netdev_priv(dev);
@@ -1204,7 +1204,7 @@ static int get_station(struct wiphy *wiphy, struct net_device *dev,
 			}
 		}
 
-		if (associatedsta == -1) {
+		if (associatedsta == ~0) {
 			netdev_err(dev, "sta required is not associated\n");
 			return -ENOENT;
 		}
@@ -2200,7 +2200,7 @@ static int get_tx_power(struct wiphy *wiphy, struct wireless_dev *wdev,
 	return ret;
 }
 
-static struct cfg80211_ops wilc_cfg80211_ops = {
+static const struct cfg80211_ops wilc_cfg80211_ops = {
 	.set_monitor_channel = set_channel,
 	.scan = scan,
 	.connect = connect,
diff --git a/drivers/staging/wilc1000/wilc_wfi_netdevice.h b/drivers/staging/wilc1000/wilc_wfi_netdevice.h
index 5cc6a82d8081..ec6b1674cf38 100644
--- a/drivers/staging/wilc1000/wilc_wfi_netdevice.h
+++ b/drivers/staging/wilc1000/wilc_wfi_netdevice.h
@@ -131,7 +131,7 @@ struct wilc_priv {
 	struct wilc_wfi_key *wilc_gtk[MAX_NUM_STA];
 	struct wilc_wfi_key *wilc_ptk[MAX_NUM_STA];
 	u8 wilc_groupkey;
-	/* semaphores */
+	/* mutexes */
 	struct mutex scan_req_lock;
 	/*  */
 	bool gbAutoRateAdjusted;
diff --git a/drivers/staging/wilc1000/wilc_wlan.c b/drivers/staging/wilc1000/wilc_wlan.c
index 19a580939dfc..bc5ad20af0a3 100644
--- a/drivers/staging/wilc1000/wilc_wlan.c
+++ b/drivers/staging/wilc1000/wilc_wlan.c
@@ -621,9 +621,9 @@ int wilc_wlan_handle_txq(struct net_device *dev, u32 *txq_count)
 			if (!ret)
 				break;
 
-			if ((reg & 0x1) == 0) {
+			if ((reg & 0x1) == 0)
 				break;
-			}
+
 			counter++;
 			if (counter > 200) {
 				counter = 0;
@@ -1001,8 +1001,7 @@ int wilc_wlan_start(struct wilc *wilc)
 	ret = wilc->hif_func->hif_write_reg(wilc, WILC_VMM_CORE_CFG, reg);
 	if (!ret) {
 		release_bus(wilc, RELEASE_ONLY);
-		ret = -EIO;
-		return ret;
+		return -EIO;
 	}
 	reg = 0;
 	if (wilc->io_type == HIF_SDIO && wilc->dev_irq_num)
@@ -1034,8 +1033,7 @@ int wilc_wlan_start(struct wilc *wilc)
 	ret = wilc->hif_func->hif_write_reg(wilc, WILC_GP_REG_1, reg);
 	if (!ret) {
 		release_bus(wilc, RELEASE_ONLY);
-		ret = -EIO;
-		return ret;
+		return -EIO;
 	}
 
 	wilc->hif_func->hif_sync_ext(wilc, NUM_INT_EXT);
@@ -1043,8 +1041,7 @@ int wilc_wlan_start(struct wilc *wilc)
 	ret = wilc->hif_func->hif_read_reg(wilc, 0x1000, &chipid);
 	if (!ret) {
 		release_bus(wilc, RELEASE_ONLY);
-		ret = -EIO;
-		return ret;
+		return -EIO;
 	}
 
 	wilc->hif_func->hif_read_reg(wilc, WILC_GLB_RESET_0, &reg);
diff --git a/drivers/staging/wilc1000/wilc_wlan.h b/drivers/staging/wilc1000/wilc_wlan.h
index 30e5312ee87e..de6c4ddbf45a 100644
--- a/drivers/staging/wilc1000/wilc_wlan.h
+++ b/drivers/staging/wilc1000/wilc_wlan.h
@@ -192,7 +192,7 @@
 
 #define ENABLE_RX_VMM		(SEL_VMM_TBL1 | EN_VMM)
 #define ENABLE_TX_VMM		(SEL_VMM_TBL0 | EN_VMM)
-/*time for expiring the semaphores of cfg packets*/
+/*time for expiring the completion of cfg packets*/
 #define CFG_PKTS_TIMEOUT	2000
 /********************************************
  *
diff --git a/drivers/staging/wilc1000/wilc_wlan_if.h b/drivers/staging/wilc1000/wilc_wlan_if.h
index 410bfc034319..439ac6f8d533 100644
--- a/drivers/staging/wilc1000/wilc_wlan_if.h
+++ b/drivers/staging/wilc1000/wilc_wlan_if.h
@@ -10,7 +10,6 @@
 #ifndef WILC_WLAN_IF_H
 #define WILC_WLAN_IF_H
 
-#include <linux/semaphore.h>
 #include <linux/netdevice.h>
 
 /********************************************
diff --git a/drivers/staging/wlan-ng/cfg80211.c b/drivers/staging/wlan-ng/cfg80211.c
index f46dfe6b24e8..182b2d564627 100644
--- a/drivers/staging/wlan-ng/cfg80211.c
+++ b/drivers/staging/wlan-ng/cfg80211.c
@@ -35,7 +35,7 @@ static const u32 prism2_cipher_suites[PRISM2_NUM_CIPHER_SUITES] = {
 
 /* prism2 device private data */
 struct prism2_wiphy_private {
-	wlandevice_t *wlandev;
+	struct wlandevice *wlandev;
 
 	struct ieee80211_supported_band band;
 	struct ieee80211_channel channels[ARRAY_SIZE(prism2_channels)];
@@ -69,11 +69,11 @@ static int prism2_result2err(int prism2_result)
 	return err;
 }
 
-static int prism2_domibset_uint32(wlandevice_t *wlandev, u32 did, u32 data)
+static int prism2_domibset_uint32(struct wlandevice *wlandev, u32 did, u32 data)
 {
 	struct p80211msg_dot11req_mibset msg;
-	p80211item_uint32_t *mibitem =
-			(p80211item_uint32_t *)&msg.mibattribute.data;
+	struct p80211item_uint32 *mibitem =
+			(struct p80211item_uint32 *)&msg.mibattribute.data;
 
 	msg.msgcode = DIDmsg_dot11req_mibset;
 	mibitem->did = did;
@@ -82,12 +82,12 @@ static int prism2_domibset_uint32(wlandevice_t *wlandev, u32 did, u32 data)
 	return p80211req_dorequest(wlandev, (u8 *)&msg);
 }
 
-static int prism2_domibset_pstr32(wlandevice_t *wlandev,
+static int prism2_domibset_pstr32(struct wlandevice *wlandev,
 				  u32 did, u8 len, const u8 *data)
 {
 	struct p80211msg_dot11req_mibset msg;
-	p80211item_pstr32_t *mibitem =
-			(p80211item_pstr32_t *)&msg.mibattribute.data;
+	struct p80211item_pstr32 *mibitem =
+			(struct p80211item_pstr32 *)&msg.mibattribute.data;
 
 	msg.msgcode = DIDmsg_dot11req_mibset;
 	mibitem->did = did;
@@ -103,7 +103,7 @@ static int prism2_change_virtual_intf(struct wiphy *wiphy,
 				      enum nl80211_iftype type, u32 *flags,
 				      struct vif_params *params)
 {
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct wlandevice *wlandev = dev->ml_priv;
 	u32 data;
 	int result;
 	int err = 0;
@@ -144,12 +144,15 @@ static int prism2_add_key(struct wiphy *wiphy, struct net_device *dev,
 			  u8 key_index, bool pairwise, const u8 *mac_addr,
 			  struct key_params *params)
 {
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct wlandevice *wlandev = dev->ml_priv;
 	u32 did;
 
 	int err = 0;
 	int result = 0;
 
+	if (key_index >= NUM_WEPKEYS)
+		return -EINVAL;
+
 	switch (params->cipher) {
 	case WLAN_CIPHER_SUITE_WEP40:
 	case WLAN_CIPHER_SUITE_WEP104:
@@ -160,27 +163,7 @@ static int prism2_add_key(struct wiphy *wiphy, struct net_device *dev,
 			goto exit;
 
 		/* send key to driver */
-		switch (key_index) {
-		case 0:
-			did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey0;
-			break;
-
-		case 1:
-			did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey1;
-			break;
-
-		case 2:
-			did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey2;
-			break;
-
-		case 3:
-			did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey3;
-			break;
-
-		default:
-			err = -EINVAL;
-			goto exit;
-		}
+		did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(key_index + 1);
 
 		result = prism2_domibset_pstr32(wlandev, did,
 						params->key_len, params->key);
@@ -205,7 +188,7 @@ static int prism2_get_key(struct wiphy *wiphy, struct net_device *dev,
 			  const u8 *mac_addr, void *cookie,
 			  void (*callback)(void *cookie, struct key_params*))
 {
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct wlandevice *wlandev = dev->ml_priv;
 	struct key_params params;
 	int len;
 
@@ -233,7 +216,7 @@ static int prism2_get_key(struct wiphy *wiphy, struct net_device *dev,
 static int prism2_del_key(struct wiphy *wiphy, struct net_device *dev,
 			  u8 key_index, bool pairwise, const u8 *mac_addr)
 {
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct wlandevice *wlandev = dev->ml_priv;
 	u32 did;
 	int err = 0;
 	int result = 0;
@@ -242,36 +225,13 @@ static int prism2_del_key(struct wiphy *wiphy, struct net_device *dev,
 	 * a key, so we will cheat by setting the key to a bogus value
 	 */
 
-	/* send key to driver */
-	switch (key_index) {
-	case 0:
-		did =
-		    DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey0;
-		break;
-
-	case 1:
-		did =
-		    DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey1;
-		break;
-
-	case 2:
-		did =
-		    DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey2;
-		break;
-
-	case 3:
-		did =
-		    DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey3;
-		break;
-
-	default:
-		err = -EINVAL;
-		goto exit;
-	}
+	if (key_index >= NUM_WEPKEYS)
+		return -EINVAL;
 
+	/* send key to driver */
+	did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(key_index + 1);
 	result = prism2_domibset_pstr32(wlandev, did, 13, "0000000000000");
 
-exit:
 	if (result)
 		err = -EFAULT;
 
@@ -281,7 +241,7 @@ exit:
 static int prism2_set_default_key(struct wiphy *wiphy, struct net_device *dev,
 				  u8 key_index, bool unicast, bool multicast)
 {
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct wlandevice *wlandev = dev->ml_priv;
 
 	int err = 0;
 	int result = 0;
@@ -299,13 +259,13 @@ static int prism2_set_default_key(struct wiphy *wiphy, struct net_device *dev,
 static int prism2_get_station(struct wiphy *wiphy, struct net_device *dev,
 			      const u8 *mac, struct station_info *sinfo)
 {
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct wlandevice *wlandev = dev->ml_priv;
 	struct p80211msg_lnxreq_commsquality quality;
 	int result;
 
 	memset(sinfo, 0, sizeof(*sinfo));
 
-	if ((wlandev == NULL) || (wlandev->msdstate != WLAN_MSD_RUNNING))
+	if (!wlandev || (wlandev->msdstate != WLAN_MSD_RUNNING))
 		return -EOPNOTSUPP;
 
 	/* build request message */
@@ -314,7 +274,7 @@ static int prism2_get_station(struct wiphy *wiphy, struct net_device *dev,
 	quality.dbm.status = P80211ENUM_msgitem_status_data_ok;
 
 	/* send message to nsd */
-	if (wlandev->mlmerequest == NULL)
+	if (!wlandev->mlmerequest)
 		return -EOPNOTSUPP;
 
 	result = wlandev->mlmerequest(wlandev, (struct p80211msg *)&quality);
@@ -334,7 +294,7 @@ static int prism2_scan(struct wiphy *wiphy,
 {
 	struct net_device *dev;
 	struct prism2_wiphy_private *priv = wiphy_priv(wiphy);
-	wlandevice_t *wlandev;
+	struct wlandevice *wlandev;
 	struct p80211msg_dot11req_scan msg1;
 	struct p80211msg_dot11req_scan_results msg2;
 	struct cfg80211_bss *bss;
@@ -374,7 +334,7 @@ static int prism2_scan(struct wiphy *wiphy,
 		msg1.scantype.data = P80211ENUM_scantype_active;
 		msg1.ssid.data.len = request->ssids->ssid_len;
 		memcpy(msg1.ssid.data.data,
-			request->ssids->ssid, request->ssids->ssid_len);
+		       request->ssids->ssid, request->ssids->ssid_len);
 	} else {
 		msg1.scantype.data = 0;
 	}
@@ -451,7 +411,7 @@ exit:
 static int prism2_set_wiphy_params(struct wiphy *wiphy, u32 changed)
 {
 	struct prism2_wiphy_private *priv = wiphy_priv(wiphy);
-	wlandevice_t *wlandev = priv->wlandev;
+	struct wlandevice *wlandev = priv->wlandev;
 	u32 data;
 	int result;
 	int err = 0;
@@ -493,7 +453,7 @@ exit:
 static int prism2_connect(struct wiphy *wiphy, struct net_device *dev,
 			  struct cfg80211_connect_params *sme)
 {
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct wlandevice *wlandev = dev->ml_priv;
 	struct ieee80211_channel *channel = sme->channel;
 	struct p80211msg_lnxreq_autojoin msg_join;
 	u32 did;
@@ -516,11 +476,11 @@ static int prism2_connect(struct wiphy *wiphy, struct net_device *dev,
 
 	/* Set the authorization */
 	if ((sme->auth_type == NL80211_AUTHTYPE_OPEN_SYSTEM) ||
-		((sme->auth_type == NL80211_AUTHTYPE_AUTOMATIC) && !is_wep))
-			msg_join.authtype.data = P80211ENUM_authalg_opensystem;
+	    ((sme->auth_type == NL80211_AUTHTYPE_AUTOMATIC) && !is_wep))
+		msg_join.authtype.data = P80211ENUM_authalg_opensystem;
 	else if ((sme->auth_type == NL80211_AUTHTYPE_SHARED_KEY) ||
-		((sme->auth_type == NL80211_AUTHTYPE_AUTOMATIC) && is_wep))
-			msg_join.authtype.data = P80211ENUM_authalg_sharedkey;
+		 ((sme->auth_type == NL80211_AUTHTYPE_AUTOMATIC) && is_wep))
+		msg_join.authtype.data = P80211ENUM_authalg_sharedkey;
 	else
 		netdev_warn(dev,
 			"Unhandled authorisation type for connect (%d)\n",
@@ -529,6 +489,11 @@ static int prism2_connect(struct wiphy *wiphy, struct net_device *dev,
 	/* Set the encryption - we only support wep */
 	if (is_wep) {
 		if (sme->key) {
+			if (sme->key_idx >= NUM_WEPKEYS) {
+				err = -EINVAL;
+				goto exit;
+			}
+
 			result = prism2_domibset_uint32(wlandev,
 				DIDmib_dot11smt_dot11PrivacyTable_dot11WEPDefaultKeyID,
 				sme->key_idx);
@@ -536,28 +501,8 @@ static int prism2_connect(struct wiphy *wiphy, struct net_device *dev,
 				goto exit;
 
 			/* send key to driver */
-			switch (sme->key_idx) {
-			case 0:
-				did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey0;
-				break;
-
-			case 1:
-				did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey1;
-				break;
-
-			case 2:
-				did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey2;
-				break;
-
-			case 3:
-				did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey3;
-				break;
-
-			default:
-				err = -EINVAL;
-				goto exit;
-			}
-
+			did = DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(
+					sme->key_idx + 1);
 			result = prism2_domibset_pstr32(wlandev,
 							did, sme->key_len,
 							(u8 *)sme->key);
@@ -618,7 +563,7 @@ exit:
 static int prism2_disconnect(struct wiphy *wiphy, struct net_device *dev,
 			     u16 reason_code)
 {
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct wlandevice *wlandev = dev->ml_priv;
 	struct p80211msg_lnxreq_autojoin msg_join;
 	int result;
 	int err = 0;
@@ -652,7 +597,7 @@ static int prism2_set_tx_power(struct wiphy *wiphy, struct wireless_dev *wdev,
 			       enum nl80211_tx_power_setting type, int mbm)
 {
 	struct prism2_wiphy_private *priv = wiphy_priv(wiphy);
-	wlandevice_t *wlandev = priv->wlandev;
+	struct wlandevice *wlandev = priv->wlandev;
 	u32 data;
 	int result;
 	int err = 0;
@@ -679,13 +624,13 @@ static int prism2_get_tx_power(struct wiphy *wiphy, struct wireless_dev *wdev,
 			       int *dbm)
 {
 	struct prism2_wiphy_private *priv = wiphy_priv(wiphy);
-	wlandevice_t *wlandev = priv->wlandev;
+	struct wlandevice *wlandev = priv->wlandev;
 	struct p80211msg_dot11req_mibget msg;
-	p80211item_uint32_t *mibitem;
+	struct p80211item_uint32 *mibitem;
 	int result;
 	int err = 0;
 
-	mibitem = (p80211item_uint32_t *)&msg.mibattribute.data;
+	mibitem = (struct p80211item_uint32 *)&msg.mibattribute.data;
 	msg.msgcode = DIDmsg_dot11req_mibget;
 	mibitem->did =
 	    DIDmib_dot11phy_dot11PhyTxPowerTable_dot11CurrentTxPowerLevel;
@@ -704,7 +649,7 @@ exit:
 }
 
 /* Interface callback functions, passing data back up to the cfg80211 layer */
-void prism2_connect_result(wlandevice_t *wlandev, u8 failed)
+void prism2_connect_result(struct wlandevice *wlandev, u8 failed)
 {
 	u16 status = failed ?
 		     WLAN_STATUS_UNSPECIFIED_FAILURE : WLAN_STATUS_SUCCESS;
@@ -713,13 +658,13 @@ void prism2_connect_result(wlandevice_t *wlandev, u8 failed)
 				NULL, 0, NULL, 0, status, GFP_KERNEL);
 }
 
-void prism2_disconnected(wlandevice_t *wlandev)
+void prism2_disconnected(struct wlandevice *wlandev)
 {
 	cfg80211_disconnected(wlandev->netdev, 0, NULL,
-		0, false, GFP_KERNEL);
+			      0, false, GFP_KERNEL);
 }
 
-void prism2_roamed(wlandevice_t *wlandev)
+void prism2_roamed(struct wlandevice *wlandev)
 {
 	cfg80211_roamed(wlandev->netdev, NULL, wlandev->bssid,
 		NULL, 0, NULL, 0, GFP_KERNEL);
@@ -744,7 +689,7 @@ static const struct cfg80211_ops prism2_usb_cfg_ops = {
 };
 
 /* Functions to create/free wiphy interface */
-static struct wiphy *wlan_create_wiphy(struct device *dev, wlandevice_t *wlandev)
+static struct wiphy *wlan_create_wiphy(struct device *dev, struct wlandevice *wlandev)
 {
 	struct wiphy *wiphy;
 	struct prism2_wiphy_private *priv;
diff --git a/drivers/staging/wlan-ng/hfa384x.h b/drivers/staging/wlan-ng/hfa384x.h
index cec6d0ba3b65..43c299c3b631 100644
--- a/drivers/staging/wlan-ng/hfa384x.h
+++ b/drivers/staging/wlan-ng/hfa384x.h
@@ -1,57 +1,57 @@
 /* hfa384x.h
-*
-* Defines the constants and data structures for the hfa384x
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-*   [Implementation and usage notes]
-*
-*   [References]
-*	CW10 Programmer's Manual v1.5
-*	IEEE 802.11 D10.0
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * Defines the constants and data structures for the hfa384x
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *    implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ *   [Implementation and usage notes]
+ *
+ *   [References]
+ *	CW10 Programmer's Manual v1.5
+ *	IEEE 802.11 D10.0
+ *
+ * --------------------------------------------------------------------
+ */
 
 #ifndef _HFA384x_H
 #define _HFA384x_H
@@ -63,7 +63,7 @@
 
 /*--- Mins & Maxs -----------------------------------*/
 #define	HFA384x_PORTID_MAX		((u16)7)
-#define	HFA384x_NUMPORTS_MAX		((u16)(HFA384x_PORTID_MAX+1))
+#define	HFA384x_NUMPORTS_MAX		((u16)(HFA384x_PORTID_MAX + 1))
 #define	HFA384x_PDR_LEN_MAX		((u16)512) /* in bytes, from EK */
 #define	HFA384x_PDA_RECS_MAX		((u16)200) /* a guess */
 #define	HFA384x_PDA_LEN_MAX		((u16)1024) /* in bytes, from EK*/
@@ -110,20 +110,21 @@
 #define		HFA384x_ADDR_FLAT_CMD_OFF_MASK	(0x0000ffff)
 
 /* Mask bits for discarding unwanted pieces in AUX format
-   16-bit address parts */
+ * 16-bit address parts
+ */
 #define		HFA384x_ADDR_AUX_PAGE_MASK	(0xffff)
 #define		HFA384x_ADDR_AUX_OFF_MASK	(0x007f)
 
 /* Make a 32-bit flat address from AUX format 16-bit page and offset */
 #define		HFA384x_ADDR_AUX_MKFLAT(p, o)	\
-		((((u32)(((u16)(p))&HFA384x_ADDR_AUX_PAGE_MASK)) << 7) | \
-		((u32)(((u16)(o))&HFA384x_ADDR_AUX_OFF_MASK)))
+		((((u32)(((u16)(p)) & HFA384x_ADDR_AUX_PAGE_MASK)) << 7) | \
+		((u32)(((u16)(o)) & HFA384x_ADDR_AUX_OFF_MASK)))
 
 /* Make CMD format offset and page from a 32-bit flat address */
 #define		HFA384x_ADDR_CMD_MKPAGE(f) \
-		((u16)((((u32)(f))&HFA384x_ADDR_FLAT_CMD_PAGE_MASK)>>16))
+		((u16)((((u32)(f)) & HFA384x_ADDR_FLAT_CMD_PAGE_MASK) >> 16))
 #define		HFA384x_ADDR_CMD_MKOFF(f) \
-		((u16)(((u32)(f))&HFA384x_ADDR_FLAT_CMD_OFF_MASK))
+		((u16)(((u32)(f)) & HFA384x_ADDR_FLAT_CMD_OFF_MASK))
 
 /*--- Controller Memory addresses -------------------*/
 #define		HFA3842_PDA_BASE	(0x007f0000UL)
@@ -173,11 +174,12 @@
 #define		HFA384x_CMD_ERR			((u16)(0x7F))
 
 /*--- Programming Modes --------------------------
-	MODE 0: Disable programming
-	MODE 1: Enable volatile memory programming
-	MODE 2: Enable non-volatile memory programming
-	MODE 3: Program non-volatile memory section
---------------------------------------------------*/
+ *	MODE 0: Disable programming
+ *	MODE 1: Enable volatile memory programming
+ *	MODE 2: Enable non-volatile memory programming
+ *	MODE 3: Program non-volatile memory section
+ *-------------------------------------------------
+ */
 #define		HFA384x_PROGMODE_DISABLE	((u16)0x00)
 #define		HFA384x_PROGMODE_RAM		((u16)0x01)
 #define		HFA384x_PROGMODE_NV		((u16)0x02)
@@ -185,8 +187,9 @@
 
 /*--- Record ID Constants --------------------------*/
 /*--------------------------------------------------------------------
-Configuration RIDs: Network Parameters, Static Configuration Entities
---------------------------------------------------------------------*/
+ * Configuration RIDs: Network Parameters, Static Configuration Entities
+ *--------------------------------------------------------------------
+ */
 #define		HFA384x_RID_CNFPORTTYPE		((u16)0xFC00)
 #define		HFA384x_RID_CNFOWNMACADDR	((u16)0xFC01)
 #define		HFA384x_RID_CNFDESIREDSSID	((u16)0xFC02)
@@ -195,17 +198,19 @@ Configuration RIDs: Network Parameters, Static Configuration Entities
 #define		HFA384x_RID_CNFMAXDATALEN	((u16)0xFC07)
 
 /*--------------------------------------------------------------------
-Configuration RID lengths: Network Params, Static Config Entities
-  This is the length of JUST the DATA part of the RID (does not
-  include the len or code fields)
---------------------------------------------------------------------*/
+ * Configuration RID lengths: Network Params, Static Config Entities
+ * This is the length of JUST the DATA part of the RID (does not
+ * include the len or code fields)
+ *--------------------------------------------------------------------
+ */
 #define		HFA384x_RID_CNFOWNMACADDR_LEN	((u16)6)
 #define		HFA384x_RID_CNFDESIREDSSID_LEN	((u16)34)
 #define		HFA384x_RID_CNFOWNSSID_LEN	((u16)34)
 
 /*--------------------------------------------------------------------
-Configuration RIDs: Network Parameters, Dynamic Configuration Entities
---------------------------------------------------------------------*/
+ * Configuration RIDs: Network Parameters, Dynamic Configuration Entities
+ *--------------------------------------------------------------------
+ */
 #define		HFA384x_RID_CREATEIBSS		((u16)0xFC81)
 #define		HFA384x_RID_FRAGTHRESH		((u16)0xFC82)
 #define		HFA384x_RID_RTSTHRESH		((u16)0xFC83)
@@ -213,8 +218,9 @@ Configuration RIDs: Network Parameters, Dynamic Configuration Entities
 #define		HFA384x_RID_PROMISCMODE		((u16)0xFC85)
 
 /*----------------------------------------------------------------------
-Information RIDs: NIC Information
---------------------------------------------------------------------*/
+ * Information RIDs: NIC Information
+ *----------------------------------------------------------------------
+ */
 #define		HFA384x_RID_MAXLOADTIME		((u16)0xFD00)
 #define		HFA384x_RID_DOWNLOADBUFFER	((u16)0xFD01)
 #define		HFA384x_RID_PRIIDENTITY		((u16)0xFD02)
@@ -230,15 +236,17 @@ Information RIDs: NIC Information
 #define		HFA384x_RID_STA_CFIACTRANGES	((u16)0xFD23)
 
 /*----------------------------------------------------------------------
-Information RID Lengths: NIC Information
-  This is the length of JUST the DATA part of the RID (does not
-  include the len or code fields)
---------------------------------------------------------------------*/
+ * Information RID Lengths: NIC Information
+ * This is the length of JUST the DATA part of the RID (does not
+ * include the len or code fields)
+ *---------------------------------------------------------------------
+ */
 #define		HFA384x_RID_NICSERIALNUMBER_LEN		((u16)12)
 
 /*--------------------------------------------------------------------
-Information RIDs:  MAC Information
---------------------------------------------------------------------*/
+ * Information RIDs:  MAC Information
+ *--------------------------------------------------------------------
+ */
 #define		HFA384x_RID_PORTSTATUS		((u16)0xFD40)
 #define		HFA384x_RID_CURRENTSSID		((u16)0xFD41)
 #define		HFA384x_RID_CURRENTBSSID	((u16)0xFD42)
@@ -250,23 +258,26 @@ Information RIDs:  MAC Information
 #define		HFA384x_RID_DBMCOMMSQUALITY	((u16)0xFD51)
 
 /*--------------------------------------------------------------------
-Information RID Lengths:  MAC Information
-  This is the length of JUST the DATA part of the RID (does not
-  include the len or code fields)
---------------------------------------------------------------------*/
+ * Information RID Lengths:  MAC Information
+ * This is the length of JUST the DATA part of the RID (does not
+ * include the len or code fields)
+ *--------------------------------------------------------------------
+ */
 #define		HFA384x_RID_DBMCOMMSQUALITY_LEN	 \
-	((u16) sizeof(hfa384x_dbmcommsquality_t))
+	((u16)sizeof(struct hfa384x_dbmcommsquality))
 #define		HFA384x_RID_JOINREQUEST_LEN \
-	((u16)sizeof(hfa384x_JoinRequest_data_t))
+	((u16)sizeof(struct hfa384x_JoinRequest_data))
 
 /*--------------------------------------------------------------------
-Information RIDs:  Modem Information
---------------------------------------------------------------------*/
+ * Information RIDs:  Modem Information
+ *--------------------------------------------------------------------
+ */
 #define		HFA384x_RID_CURRENTCHANNEL	((u16)0xFDC1)
 
 /*--------------------------------------------------------------------
-API ENHANCEMENTS (NOT ALREADY IMPLEMENTED)
---------------------------------------------------------------------*/
+ * API ENHANCEMENTS (NOT ALREADY IMPLEMENTED)
+ *--------------------------------------------------------------------
+ */
 #define		HFA384x_RID_CNFWEPDEFAULTKEYID	((u16)0xFC23)
 #define		HFA384x_RID_CNFWEPDEFAULTKEY0	((u16)0xFC24)
 #define		HFA384x_RID_CNFWEPDEFAULTKEY1	((u16)0xFC25)
@@ -290,8 +301,9 @@ API ENHANCEMENTS (NOT ALREADY IMPLEMENTED)
 #define		HFA384x_RID_CNFWEP128DEFAULTKEY_LEN	((u16)14)
 
 /*--------------------------------------------------------------------
-PD Record codes
---------------------------------------------------------------------*/
+ * PD Record codes
+ *--------------------------------------------------------------------
+ */
 #define HFA384x_PDR_PCB_PARTNUM		((u16)0x0001)
 #define HFA384x_PDR_PDAVER		((u16)0x0002)
 #define HFA384x_PDR_NIC_SERIAL		((u16)0x0003)
@@ -355,31 +367,32 @@ struct hfa384x_bytestr {
 	u8 data[0];
 } __packed;
 
-typedef struct hfa384x_bytestr32 {
+struct hfa384x_bytestr32 {
 	u16 len;
 	u8 data[32];
-} __packed hfa384x_bytestr32_t;
+} __packed;
 
 /*--------------------------------------------------------------------
-Configuration Record Structures:
-	Network Parameters, Static Configuration Entities
---------------------------------------------------------------------*/
+ * Configuration Record Structures:
+ *	Network Parameters, Static Configuration Entities
+ *--------------------------------------------------------------------
+ */
 
 /*-- Hardware/Firmware Component Information ----------*/
-typedef struct hfa384x_compident {
+struct hfa384x_compident {
 	u16 id;
 	u16 variant;
 	u16 major;
 	u16 minor;
-} __packed hfa384x_compident_t;
+} __packed;
 
-typedef struct hfa384x_caplevel {
+struct hfa384x_caplevel {
 	u16 role;
 	u16 id;
 	u16 variant;
 	u16 bottom;
 	u16 top;
-} __packed hfa384x_caplevel_t;
+} __packed;
 
 /*-- Configuration Record: cnfAuthentication --*/
 #define HFA384x_CNFAUTHENTICATION_OPENSYSTEM	0x0001
@@ -387,77 +400,81 @@ typedef struct hfa384x_caplevel {
 #define HFA384x_CNFAUTHENTICATION_LEAP		0x0004
 
 /*--------------------------------------------------------------------
-Configuration Record Structures:
-	Network Parameters, Dynamic Configuration Entities
---------------------------------------------------------------------*/
+ * Configuration Record Structures:
+ *	Network Parameters, Dynamic Configuration Entities
+ *--------------------------------------------------------------------
+ */
 
 #define HFA384x_CREATEIBSS_JOINCREATEIBSS          0
 
 /*-- Configuration Record: HostScanRequest (data portion only) --*/
-typedef struct hfa384x_HostScanRequest_data {
+struct hfa384x_HostScanRequest_data {
 	u16 channelList;
 	u16 txRate;
-	hfa384x_bytestr32_t ssid;
-} __packed hfa384x_HostScanRequest_data_t;
+	struct hfa384x_bytestr32 ssid;
+} __packed;
 
 /*-- Configuration Record: JoinRequest (data portion only) --*/
-typedef struct hfa384x_JoinRequest_data {
+struct hfa384x_JoinRequest_data {
 	u8 bssid[WLAN_BSSID_LEN];
 	u16 channel;
-} __packed hfa384x_JoinRequest_data_t;
+} __packed;
 
 /*-- Configuration Record: authenticateStation (data portion only) --*/
-typedef struct hfa384x_authenticateStation_data {
+struct hfa384x_authenticateStation_data {
 	u8 address[ETH_ALEN];
 	u16 status;
 	u16 algorithm;
-} __packed hfa384x_authenticateStation_data_t;
+} __packed;
 
 /*-- Configuration Record: WPAData       (data portion only) --*/
-typedef struct hfa384x_WPAData {
+struct hfa384x_WPAData {
 	u16 datalen;
 	u8 data[0];		/* max 80 */
-} __packed hfa384x_WPAData_t;
+} __packed;
 
 /*--------------------------------------------------------------------
-Information Record Structures: NIC Information
---------------------------------------------------------------------*/
+ * Information Record Structures: NIC Information
+ *--------------------------------------------------------------------
+ */
 
 /*-- Information Record: DownLoadBuffer --*/
 /* NOTE: The page and offset are in AUX format */
-typedef struct hfa384x_downloadbuffer {
+struct hfa384x_downloadbuffer {
 	u16 page;
 	u16 offset;
 	u16 len;
-} __packed hfa384x_downloadbuffer_t;
+} __packed;
 
 /*--------------------------------------------------------------------
-Information Record Structures: NIC Information
---------------------------------------------------------------------*/
+ * Information Record Structures: NIC Information
+ *--------------------------------------------------------------------
+ */
 
 #define HFA384x_PSTATUS_CONN_IBSS	((u16)3)
 
 /*-- Information Record: commsquality --*/
-typedef struct hfa384x_commsquality {
+struct hfa384x_commsquality {
 	u16 CQ_currBSS;
 	u16 ASL_currBSS;
 	u16 ANL_currFC;
-} __packed hfa384x_commsquality_t;
+} __packed;
 
 /*-- Information Record: dmbcommsquality --*/
-typedef struct hfa384x_dbmcommsquality {
+struct hfa384x_dbmcommsquality {
 	u16 CQdbm_currBSS;
 	u16 ASLdbm_currBSS;
 	u16 ANLdbm_currFC;
-} __packed hfa384x_dbmcommsquality_t;
+} __packed;
 
 /*--------------------------------------------------------------------
-FRAME STRUCTURES: Communication Frames
-----------------------------------------------------------------------
-Communication Frames: Transmit Frames
---------------------------------------------------------------------*/
+ * FRAME STRUCTURES: Communication Frames
+ *--------------------------------------------------------------------
+ * Communication Frames: Transmit Frames
+ *--------------------------------------------------------------------
+ */
 /*-- Communication Frame: Transmit Frame Structure --*/
-typedef struct hfa384x_tx_frame {
+struct hfa384x_tx_frame {
 	u16 status;
 	u16 reserved1;
 	u16 reserved2;
@@ -482,10 +499,11 @@ typedef struct hfa384x_tx_frame {
 	u8 dest_addr[6];
 	u8 src_addr[6];
 	u16 data_length;	/* big endian format */
-} __packed hfa384x_tx_frame_t;
+} __packed;
 /*--------------------------------------------------------------------
-Communication Frames: Field Masks for Transmit Frames
---------------------------------------------------------------------*/
+ * Communication Frames: Field Masks for Transmit Frames
+ *--------------------------------------------------------------------
+ */
 /*-- Status Field --*/
 #define		HFA384x_TXSTATUS_ACKERR			((u16)BIT(5))
 #define		HFA384x_TXSTATUS_FORMERR		((u16)BIT(3))
@@ -499,16 +517,17 @@ Communication Frames: Field Masks for Transmit Frames
 #define		HFA384x_TX_TXEX				((u16)BIT(2))
 #define		HFA384x_TX_TXOK				((u16)BIT(1))
 /*--------------------------------------------------------------------
-Communication Frames: Test/Get/Set Field Values for Transmit Frames
---------------------------------------------------------------------*/
+ * Communication Frames: Test/Get/Set Field Values for Transmit Frames
+ *--------------------------------------------------------------------
+ */
 /*-- Status Field --*/
 #define HFA384x_TXSTATUS_ISERROR(v)	\
-	(((u16)(v))&\
-	(HFA384x_TXSTATUS_ACKERR|HFA384x_TXSTATUS_FORMERR|\
-	HFA384x_TXSTATUS_DISCON|HFA384x_TXSTATUS_AGEDERR|\
+	(((u16)(v)) & \
+	(HFA384x_TXSTATUS_ACKERR | HFA384x_TXSTATUS_FORMERR | \
+	HFA384x_TXSTATUS_DISCON | HFA384x_TXSTATUS_AGEDERR | \
 	HFA384x_TXSTATUS_RETRYERR))
 
-#define	HFA384x_TX_SET(v, m, s)		((((u16)(v))<<((u16)(s)))&((u16)(m)))
+#define	HFA384x_TX_SET(v, m, s)		((((u16)(v)) << ((u16)(s))) & ((u16)(m)))
 
 #define	HFA384x_TX_MACPORT_SET(v)	HFA384x_TX_SET(v, HFA384x_TX_MACPORT, 8)
 #define	HFA384x_TX_STRUCTYPE_SET(v)	HFA384x_TX_SET(v, \
@@ -516,10 +535,11 @@ Communication Frames: Test/Get/Set Field Values for Transmit Frames
 #define	HFA384x_TX_TXEX_SET(v)		HFA384x_TX_SET(v, HFA384x_TX_TXEX, 2)
 #define	HFA384x_TX_TXOK_SET(v)		HFA384x_TX_SET(v, HFA384x_TX_TXOK, 1)
 /*--------------------------------------------------------------------
-Communication Frames: Receive Frames
---------------------------------------------------------------------*/
+ * Communication Frames: Receive Frames
+ *--------------------------------------------------------------------
+ */
 /*-- Communication Frame: Receive Frame Structure --*/
-typedef struct hfa384x_rx_frame {
+struct hfa384x_rx_frame {
 	/*-- MAC rx descriptor (hfa384x byte order) --*/
 	u16 status;
 	u32 time;
@@ -544,10 +564,11 @@ typedef struct hfa384x_rx_frame {
 	u8 dest_addr[6];
 	u8 src_addr[6];
 	u16 data_length;	/* IEEE? (big endian) format */
-} __packed hfa384x_rx_frame_t;
+} __packed;
 /*--------------------------------------------------------------------
-Communication Frames: Field Masks for Receive Frames
---------------------------------------------------------------------*/
+ * Communication Frames: Field Masks for Receive Frames
+ *--------------------------------------------------------------------
+ */
 
 /*-- Status Fields --*/
 #define		HFA384x_RXSTATUS_MACPORT		((u16)(BIT(10) | \
@@ -555,17 +576,19 @@ Communication Frames: Field Masks for Receive Frames
 								BIT(8)))
 #define		HFA384x_RXSTATUS_FCSERR			((u16)BIT(0))
 /*--------------------------------------------------------------------
-Communication Frames: Test/Get/Set Field Values for Receive Frames
---------------------------------------------------------------------*/
+ * Communication Frames: Test/Get/Set Field Values for Receive Frames
+ *--------------------------------------------------------------------
+ */
 #define		HFA384x_RXSTATUS_MACPORT_GET(value)	((u16)((((u16)(value)) \
 					    & HFA384x_RXSTATUS_MACPORT) >> 8))
 #define		HFA384x_RXSTATUS_ISFCSERR(value)	((u16)(((u16)(value)) \
 						  & HFA384x_RXSTATUS_FCSERR))
 /*--------------------------------------------------------------------
- FRAME STRUCTURES: Information Types and Information Frame Structures
-----------------------------------------------------------------------
-Information Types
---------------------------------------------------------------------*/
+ * FRAME STRUCTURES: Information Types and Information Frame Structures
+ *--------------------------------------------------------------------
+ * Information Types
+ *--------------------------------------------------------------------
+ */
 #define		HFA384x_IT_HANDOVERADDR			((u16)0xF000UL)
 #define		HFA384x_IT_COMMTALLIES			((u16)0xF100UL)
 #define		HFA384x_IT_SCANRESULTS			((u16)0xF101UL)
@@ -580,13 +603,14 @@ Information Types
 #define		HFA384x_IT_MICFAILURE			((u16)0xF206UL)
 
 /*--------------------------------------------------------------------
-Information Frames Structures
-----------------------------------------------------------------------
-Information Frames: Notification Frame Structures
---------------------------------------------------------------------*/
+ * Information Frames Structures
+ *--------------------------------------------------------------------
+ * Information Frames: Notification Frame Structures
+ *--------------------------------------------------------------------
+ */
 
 /*--  Inquiry Frame, Diagnose: Communication Tallies --*/
-typedef struct hfa384x_CommTallies16 {
+struct hfa384x_CommTallies16 {
 	u16 txunicastframes;
 	u16 txmulticastframes;
 	u16 txfragments;
@@ -608,9 +632,9 @@ typedef struct hfa384x_CommTallies16 {
 	u16 rxdiscardswepundecr;
 	u16 rxmsginmsgfrag;
 	u16 rxmsginbadmsgfrag;
-} __packed hfa384x_CommTallies16_t;
+} __packed;
 
-typedef struct hfa384x_CommTallies32 {
+struct hfa384x_CommTallies32 {
 	u32 txunicastframes;
 	u32 txmulticastframes;
 	u32 txfragments;
@@ -632,62 +656,62 @@ typedef struct hfa384x_CommTallies32 {
 	u32 rxdiscardswepundecr;
 	u32 rxmsginmsgfrag;
 	u32 rxmsginbadmsgfrag;
-} __packed hfa384x_CommTallies32_t;
+} __packed;
 
 /*--  Inquiry Frame, Diagnose: Scan Results & Subfields--*/
-typedef struct hfa384x_ScanResultSub {
+struct hfa384x_ScanResultSub {
 	u16 chid;
 	u16 anl;
 	u16 sl;
 	u8 bssid[WLAN_BSSID_LEN];
 	u16 bcnint;
 	u16 capinfo;
-	hfa384x_bytestr32_t ssid;
+	struct hfa384x_bytestr32 ssid;
 	u8 supprates[10];	/* 802.11 info element */
 	u16 proberesp_rate;
-} __packed hfa384x_ScanResultSub_t;
+} __packed;
 
-typedef struct hfa384x_ScanResult {
+struct hfa384x_ScanResult {
 	u16 rsvd;
 	u16 scanreason;
-	hfa384x_ScanResultSub_t result[HFA384x_SCANRESULT_MAX];
-} __packed hfa384x_ScanResult_t;
+	struct hfa384x_ScanResultSub result[HFA384x_SCANRESULT_MAX];
+} __packed;
 
 /*--  Inquiry Frame, Diagnose: ChInfo Results & Subfields--*/
-typedef struct hfa384x_ChInfoResultSub {
+struct hfa384x_ChInfoResultSub {
 	u16 chid;
 	u16 anl;
 	u16 pnl;
 	u16 active;
-} __packed hfa384x_ChInfoResultSub_t;
+} __packed;
 
 #define HFA384x_CHINFORESULT_BSSACTIVE	BIT(0)
 #define HFA384x_CHINFORESULT_PCFACTIVE	BIT(1)
 
-typedef struct hfa384x_ChInfoResult {
+struct hfa384x_ChInfoResult {
 	u16 scanchannels;
-	hfa384x_ChInfoResultSub_t result[HFA384x_CHINFORESULT_MAX];
-} __packed hfa384x_ChInfoResult_t;
+	struct hfa384x_ChInfoResultSub result[HFA384x_CHINFORESULT_MAX];
+} __packed;
 
 /*--  Inquiry Frame, Diagnose: Host Scan Results & Subfields--*/
-typedef struct hfa384x_HScanResultSub {
+struct hfa384x_HScanResultSub {
 	u16 chid;
 	u16 anl;
 	u16 sl;
 	u8 bssid[WLAN_BSSID_LEN];
 	u16 bcnint;
 	u16 capinfo;
-	hfa384x_bytestr32_t ssid;
+	struct hfa384x_bytestr32 ssid;
 	u8 supprates[10];	/* 802.11 info element */
 	u16 proberesp_rate;
 	u16 atim;
-} __packed hfa384x_HScanResultSub_t;
+} __packed;
 
-typedef struct hfa384x_HScanResult {
+struct hfa384x_HScanResult {
 	u16 nresult;
 	u16 rsvd;
-	hfa384x_HScanResultSub_t result[HFA384x_HSCANRESULT_MAX];
-} __packed hfa384x_HScanResult_t;
+	struct hfa384x_HScanResultSub result[HFA384x_HSCANRESULT_MAX];
+} __packed;
 
 /*--  Unsolicited Frame, MAC Mgmt: LinkStatus --*/
 
@@ -699,9 +723,9 @@ typedef struct hfa384x_HScanResult {
 #define HFA384x_LINK_AP_INRANGE		((u16)5)
 #define HFA384x_LINK_ASSOCFAIL		((u16)6)
 
-typedef struct hfa384x_LinkStatus {
+struct hfa384x_LinkStatus {
 	u16 linkstatus;
-} __packed hfa384x_LinkStatus_t;
+} __packed;
 
 /*--  Unsolicited Frame, MAC Mgmt: AssociationStatus (--*/
 
@@ -709,56 +733,57 @@ typedef struct hfa384x_LinkStatus {
 #define HFA384x_ASSOCSTATUS_REASSOC	((u16)2)
 #define HFA384x_ASSOCSTATUS_AUTHFAIL	((u16)5)
 
-typedef struct hfa384x_AssocStatus {
+struct hfa384x_AssocStatus {
 	u16 assocstatus;
 	u8 sta_addr[ETH_ALEN];
 	/* old_ap_addr is only valid if assocstatus == 2 */
 	u8 old_ap_addr[ETH_ALEN];
 	u16 reason;
 	u16 reserved;
-} __packed hfa384x_AssocStatus_t;
+} __packed;
 
 /*--  Unsolicited Frame, MAC Mgmt: AuthRequest (AP Only) --*/
 
-typedef struct hfa384x_AuthRequest {
+struct hfa384x_AuthRequest {
 	u8 sta_addr[ETH_ALEN];
 	u16 algorithm;
-} __packed hfa384x_AuthReq_t;
+} __packed;
 
 /*--  Unsolicited Frame, MAC Mgmt: PSUserCount (AP Only) --*/
 
-typedef struct hfa384x_PSUserCount {
+struct hfa384x_PSUserCount {
 	u16 usercnt;
-} __packed hfa384x_PSUserCount_t;
+} __packed;
 
-typedef struct hfa384x_KeyIDChanged {
+struct hfa384x_KeyIDChanged {
 	u8 sta_addr[ETH_ALEN];
 	u16 keyid;
-} __packed hfa384x_KeyIDChanged_t;
+} __packed;
 
 /*--  Collection of all Inf frames ---------------*/
-typedef union hfa384x_infodata {
-	hfa384x_CommTallies16_t commtallies16;
-	hfa384x_CommTallies32_t commtallies32;
-	hfa384x_ScanResult_t scanresult;
-	hfa384x_ChInfoResult_t chinforesult;
-	hfa384x_HScanResult_t hscanresult;
-	hfa384x_LinkStatus_t linkstatus;
-	hfa384x_AssocStatus_t assocstatus;
-	hfa384x_AuthReq_t authreq;
-	hfa384x_PSUserCount_t psusercnt;
-	hfa384x_KeyIDChanged_t keyidchanged;
-} __packed hfa384x_infodata_t;
-
-typedef struct hfa384x_InfFrame {
+union hfa384x_infodata {
+	struct hfa384x_CommTallies16 commtallies16;
+	struct hfa384x_CommTallies32 commtallies32;
+	struct hfa384x_ScanResult scanresult;
+	struct hfa384x_ChInfoResult chinforesult;
+	struct hfa384x_HScanResult hscanresult;
+	struct hfa384x_LinkStatus linkstatus;
+	struct hfa384x_AssocStatus assocstatus;
+	struct hfa384x_AuthRequest authreq;
+	struct hfa384x_PSUserCount psusercnt;
+	struct hfa384x_KeyIDChanged keyidchanged;
+} __packed;
+
+struct hfa384x_InfFrame {
 	u16 framelen;
 	u16 infotype;
-	hfa384x_infodata_t info;
-} __packed hfa384x_InfFrame_t;
+	union hfa384x_infodata info;
+} __packed;
 
 /*--------------------------------------------------------------------
-USB Packet structures and constants.
---------------------------------------------------------------------*/
+ * USB Packet structures and constants.
+ *--------------------------------------------------------------------
+ */
 
 /* Should be sent to the bulkout endpoint */
 #define HFA384x_USB_TXFRM	0
@@ -783,143 +808,140 @@ USB Packet structures and constants.
 /*------------------------------------*/
 /* Request (bulk OUT) packet contents */
 
-typedef struct hfa384x_usb_txfrm {
-	hfa384x_tx_frame_t desc;
+struct hfa384x_usb_txfrm {
+	struct hfa384x_tx_frame desc;
 	u8 data[WLAN_DATA_MAXLEN];
-} __packed hfa384x_usb_txfrm_t;
+} __packed;
 
-typedef struct hfa384x_usb_cmdreq {
+struct hfa384x_usb_cmdreq {
 	u16 type;
 	u16 cmd;
 	u16 parm0;
 	u16 parm1;
 	u16 parm2;
 	u8 pad[54];
-} __packed hfa384x_usb_cmdreq_t;
+} __packed;
 
-typedef struct hfa384x_usb_wridreq {
+struct hfa384x_usb_wridreq {
 	u16 type;
 	u16 frmlen;
 	u16 rid;
 	u8 data[HFA384x_RIDDATA_MAXLEN];
-} __packed hfa384x_usb_wridreq_t;
+} __packed;
 
-typedef struct hfa384x_usb_rridreq {
+struct hfa384x_usb_rridreq {
 	u16 type;
 	u16 frmlen;
 	u16 rid;
 	u8 pad[58];
-} __packed hfa384x_usb_rridreq_t;
+} __packed;
 
-typedef struct hfa384x_usb_wmemreq {
+struct hfa384x_usb_wmemreq {
 	u16 type;
 	u16 frmlen;
 	u16 offset;
 	u16 page;
 	u8 data[HFA384x_USB_RWMEM_MAXLEN];
-} __packed hfa384x_usb_wmemreq_t;
+} __packed;
 
-typedef struct hfa384x_usb_rmemreq {
+struct hfa384x_usb_rmemreq {
 	u16 type;
 	u16 frmlen;
 	u16 offset;
 	u16 page;
 	u8 pad[56];
-} __packed hfa384x_usb_rmemreq_t;
+} __packed;
 
 /*------------------------------------*/
 /* Response (bulk IN) packet contents */
 
-typedef struct hfa384x_usb_rxfrm {
-	hfa384x_rx_frame_t desc;
+struct hfa384x_usb_rxfrm {
+	struct hfa384x_rx_frame desc;
 	u8 data[WLAN_DATA_MAXLEN];
-} __packed hfa384x_usb_rxfrm_t;
+} __packed;
 
-typedef struct hfa384x_usb_infofrm {
+struct hfa384x_usb_infofrm {
 	u16 type;
-	hfa384x_InfFrame_t info;
-} __packed hfa384x_usb_infofrm_t;
+	struct hfa384x_InfFrame info;
+} __packed;
 
-typedef struct hfa384x_usb_statusresp {
+struct hfa384x_usb_statusresp {
 	u16 type;
 	u16 status;
 	u16 resp0;
 	u16 resp1;
 	u16 resp2;
-} __packed hfa384x_usb_cmdresp_t;
-
-typedef hfa384x_usb_cmdresp_t hfa384x_usb_wridresp_t;
+} __packed;
 
-typedef struct hfa384x_usb_rridresp {
+struct hfa384x_usb_rridresp {
 	u16 type;
 	u16 frmlen;
 	u16 rid;
 	u8 data[HFA384x_RIDDATA_MAXLEN];
-} __packed hfa384x_usb_rridresp_t;
-
-typedef hfa384x_usb_cmdresp_t hfa384x_usb_wmemresp_t;
+} __packed;
 
-typedef struct hfa384x_usb_rmemresp {
+struct hfa384x_usb_rmemresp {
 	u16 type;
 	u16 frmlen;
 	u8 data[HFA384x_USB_RWMEM_MAXLEN];
-} __packed hfa384x_usb_rmemresp_t;
+} __packed;
 
-typedef struct hfa384x_usb_bufavail {
+struct hfa384x_usb_bufavail {
 	u16 type;
 	u16 frmlen;
-} __packed hfa384x_usb_bufavail_t;
+} __packed;
 
-typedef struct hfa384x_usb_error {
+struct hfa384x_usb_error {
 	u16 type;
 	u16 errortype;
-} __packed hfa384x_usb_error_t;
+} __packed;
 
 /*----------------------------------------------------------*/
 /* Unions for packaging all the known packet types together */
 
-typedef union hfa384x_usbout {
+union hfa384x_usbout {
 	__le16 type;
-	hfa384x_usb_txfrm_t txfrm;
-	hfa384x_usb_cmdreq_t cmdreq;
-	hfa384x_usb_wridreq_t wridreq;
-	hfa384x_usb_rridreq_t rridreq;
-	hfa384x_usb_wmemreq_t wmemreq;
-	hfa384x_usb_rmemreq_t rmemreq;
-} __packed hfa384x_usbout_t;
-
-typedef union hfa384x_usbin {
+	struct hfa384x_usb_txfrm txfrm;
+	struct hfa384x_usb_cmdreq cmdreq;
+	struct hfa384x_usb_wridreq wridreq;
+	struct hfa384x_usb_rridreq rridreq;
+	struct hfa384x_usb_wmemreq wmemreq;
+	struct hfa384x_usb_rmemreq rmemreq;
+} __packed;
+
+union hfa384x_usbin {
 	__le16 type;
-	hfa384x_usb_rxfrm_t rxfrm;
-	hfa384x_usb_txfrm_t txfrm;
-	hfa384x_usb_infofrm_t infofrm;
-	hfa384x_usb_cmdresp_t cmdresp;
-	hfa384x_usb_wridresp_t wridresp;
-	hfa384x_usb_rridresp_t rridresp;
-	hfa384x_usb_wmemresp_t wmemresp;
-	hfa384x_usb_rmemresp_t rmemresp;
-	hfa384x_usb_bufavail_t bufavail;
-	hfa384x_usb_error_t usberror;
+	struct hfa384x_usb_rxfrm rxfrm;
+	struct hfa384x_usb_txfrm txfrm;
+	struct hfa384x_usb_infofrm infofrm;
+	struct hfa384x_usb_statusresp cmdresp;
+	struct hfa384x_usb_statusresp wridresp;
+	struct hfa384x_usb_rridresp rridresp;
+	struct hfa384x_usb_statusresp wmemresp;
+	struct hfa384x_usb_rmemresp rmemresp;
+	struct hfa384x_usb_bufavail bufavail;
+	struct hfa384x_usb_error usberror;
 	u8 boguspad[3000];
-} __packed hfa384x_usbin_t;
+} __packed;
 
 /*--------------------------------------------------------------------
-PD record structures.
---------------------------------------------------------------------*/
+ * PD record structures.
+ *--------------------------------------------------------------------
+ */
 
-typedef struct hfa384x_pdr_pcb_partnum {
+struct hfa384x_pdr_pcb_partnum {
 	u8 num[8];
-} __packed hfa384x_pdr_pcb_partnum_t;
+} __packed;
 
-typedef struct hfa384x_pdr_pcb_tracenum {
+struct hfa384x_pdr_pcb_tracenum {
 	u8 num[8];
-} __packed hfa384x_pdr_pcb_tracenum_t;
+} __packed;
 
-typedef struct hfa384x_pdr_nic_serial {
+struct hfa384x_pdr_nic_serial {
 	u8 num[12];
-} __packed hfa384x_pdr_nic_serial_t;
+} __packed;
 
-typedef struct hfa384x_pdr_mkk_measurements {
+struct hfa384x_pdr_mkk_measurements {
 	double carrier_freq;
 	double occupied_band;
 	double power_density;
@@ -935,192 +957,193 @@ typedef struct hfa384x_pdr_mkk_measurements {
 	double rx_spur_f2;
 	double rx_spur_l1;
 	double rx_spur_l2;
-} __packed hfa384x_pdr_mkk_measurements_t;
+} __packed;
 
-typedef struct hfa384x_pdr_nic_ramsize {
+struct hfa384x_pdr_nic_ramsize {
 	u8 size[12];		/* units of KB */
-} __packed hfa384x_pdr_nic_ramsize_t;
+} __packed;
 
-typedef struct hfa384x_pdr_mfisuprange {
+struct hfa384x_pdr_mfisuprange {
 	u16 id;
 	u16 variant;
 	u16 bottom;
 	u16 top;
-} __packed hfa384x_pdr_mfisuprange_t;
+} __packed;
 
-typedef struct hfa384x_pdr_cfisuprange {
+struct hfa384x_pdr_cfisuprange {
 	u16 id;
 	u16 variant;
 	u16 bottom;
 	u16 top;
-} __packed hfa384x_pdr_cfisuprange_t;
+} __packed;
 
-typedef struct hfa384x_pdr_nicid {
+struct hfa384x_pdr_nicid {
 	u16 id;
 	u16 variant;
 	u16 major;
 	u16 minor;
-} __packed hfa384x_pdr_nicid_t;
+} __packed;
 
-typedef struct hfa384x_pdr_refdac_measurements {
+struct hfa384x_pdr_refdac_measurements {
 	u16 value[0];
-} __packed hfa384x_pdr_refdac_measurements_t;
+} __packed;
 
-typedef struct hfa384x_pdr_vgdac_measurements {
+struct hfa384x_pdr_vgdac_measurements {
 	u16 value[0];
-} __packed hfa384x_pdr_vgdac_measurements_t;
+} __packed;
 
-typedef struct hfa384x_pdr_level_comp_measurements {
+struct hfa384x_pdr_level_comp_measurements {
 	u16 value[0];
-} __packed hfa384x_pdr_level_compc_measurements_t;
+} __packed;
 
-typedef struct hfa384x_pdr_mac_address {
+struct hfa384x_pdr_mac_address {
 	u8 addr[6];
-} __packed hfa384x_pdr_mac_address_t;
+} __packed;
 
-typedef struct hfa384x_pdr_mkk_callname {
+struct hfa384x_pdr_mkk_callname {
 	u8 callname[8];
-} __packed hfa384x_pdr_mkk_callname_t;
+} __packed;
 
-typedef struct hfa384x_pdr_regdomain {
+struct hfa384x_pdr_regdomain {
 	u16 numdomains;
 	u16 domain[5];
-} __packed hfa384x_pdr_regdomain_t;
+} __packed;
 
-typedef struct hfa384x_pdr_allowed_channel {
+struct hfa384x_pdr_allowed_channel {
 	u16 ch_bitmap;
-} __packed hfa384x_pdr_allowed_channel_t;
+} __packed;
 
-typedef struct hfa384x_pdr_default_channel {
+struct hfa384x_pdr_default_channel {
 	u16 channel;
-} __packed hfa384x_pdr_default_channel_t;
+} __packed;
 
-typedef struct hfa384x_pdr_privacy_option {
+struct hfa384x_pdr_privacy_option {
 	u16 available;
-} __packed hfa384x_pdr_privacy_option_t;
+} __packed;
 
-typedef struct hfa384x_pdr_temptype {
+struct hfa384x_pdr_temptype {
 	u16 type;
-} __packed hfa384x_pdr_temptype_t;
+} __packed;
 
-typedef struct hfa384x_pdr_refdac_setup {
+struct hfa384x_pdr_refdac_setup {
 	u16 ch_value[14];
-} __packed hfa384x_pdr_refdac_setup_t;
+} __packed;
 
-typedef struct hfa384x_pdr_vgdac_setup {
+struct hfa384x_pdr_vgdac_setup {
 	u16 ch_value[14];
-} __packed hfa384x_pdr_vgdac_setup_t;
+} __packed;
 
-typedef struct hfa384x_pdr_level_comp_setup {
+struct hfa384x_pdr_level_comp_setup {
 	u16 ch_value[14];
-} __packed hfa384x_pdr_level_comp_setup_t;
+} __packed;
 
-typedef struct hfa384x_pdr_trimdac_setup {
+struct hfa384x_pdr_trimdac_setup {
 	u16 trimidac;
 	u16 trimqdac;
-} __packed hfa384x_pdr_trimdac_setup_t;
+} __packed;
 
-typedef struct hfa384x_pdr_ifr_setting {
+struct hfa384x_pdr_ifr_setting {
 	u16 value[3];
-} __packed hfa384x_pdr_ifr_setting_t;
+} __packed;
 
-typedef struct hfa384x_pdr_rfr_setting {
+struct hfa384x_pdr_rfr_setting {
 	u16 value[3];
-} __packed hfa384x_pdr_rfr_setting_t;
+} __packed;
 
-typedef struct hfa384x_pdr_hfa3861_baseline {
+struct hfa384x_pdr_hfa3861_baseline {
 	u16 value[50];
-} __packed hfa384x_pdr_hfa3861_baseline_t;
+} __packed;
 
-typedef struct hfa384x_pdr_hfa3861_shadow {
+struct hfa384x_pdr_hfa3861_shadow {
 	u32 value[32];
-} __packed hfa384x_pdr_hfa3861_shadow_t;
+} __packed;
 
-typedef struct hfa384x_pdr_hfa3861_ifrf {
+struct hfa384x_pdr_hfa3861_ifrf {
 	u32 value[20];
-} __packed hfa384x_pdr_hfa3861_ifrf_t;
+} __packed;
 
-typedef struct hfa384x_pdr_hfa3861_chcalsp {
+struct hfa384x_pdr_hfa3861_chcalsp {
 	u16 value[14];
-} __packed hfa384x_pdr_hfa3861_chcalsp_t;
+} __packed;
 
-typedef struct hfa384x_pdr_hfa3861_chcali {
+struct hfa384x_pdr_hfa3861_chcali {
 	u16 value[17];
-} __packed hfa384x_pdr_hfa3861_chcali_t;
+} __packed;
 
-typedef struct hfa384x_pdr_hfa3861_nic_config {
+struct hfa384x_pdr_hfa3861_nic_config {
 	u16 config_bitmap;
-} __packed hfa384x_pdr_nic_config_t;
+} __packed;
 
-typedef struct hfa384x_pdr_hfo_delay {
+struct hfa384x_pdr_hfo_delay {
 	u8 hfo_delay;
-} __packed hfa384x_hfo_delay_t;
+} __packed;
 
-typedef struct hfa384x_pdr_hfa3861_manf_testsp {
+struct hfa384x_pdr_hfa3861_manf_testsp {
 	u16 value[30];
-} __packed hfa384x_pdr_hfa3861_manf_testsp_t;
+} __packed;
 
-typedef struct hfa384x_pdr_hfa3861_manf_testi {
+struct hfa384x_pdr_hfa3861_manf_testi {
 	u16 value[30];
-} __packed hfa384x_pdr_hfa3861_manf_testi_t;
+} __packed;
 
-typedef struct hfa384x_end_of_pda {
+struct hfa384x_pdr_end_of_pda {
 	u16 crc;
-} __packed hfa384x_pdr_end_of_pda_t;
+} __packed;
 
-typedef struct hfa384x_pdrec {
+struct hfa384x_pdrec {
 	u16 len;		/* in words */
 	u16 code;
 	union pdr {
-		hfa384x_pdr_pcb_partnum_t pcb_partnum;
-		hfa384x_pdr_pcb_tracenum_t pcb_tracenum;
-		hfa384x_pdr_nic_serial_t nic_serial;
-		hfa384x_pdr_mkk_measurements_t mkk_measurements;
-		hfa384x_pdr_nic_ramsize_t nic_ramsize;
-		hfa384x_pdr_mfisuprange_t mfisuprange;
-		hfa384x_pdr_cfisuprange_t cfisuprange;
-		hfa384x_pdr_nicid_t nicid;
-		hfa384x_pdr_refdac_measurements_t refdac_measurements;
-		hfa384x_pdr_vgdac_measurements_t vgdac_measurements;
-		hfa384x_pdr_level_compc_measurements_t level_compc_measurements;
-		hfa384x_pdr_mac_address_t mac_address;
-		hfa384x_pdr_mkk_callname_t mkk_callname;
-		hfa384x_pdr_regdomain_t regdomain;
-		hfa384x_pdr_allowed_channel_t allowed_channel;
-		hfa384x_pdr_default_channel_t default_channel;
-		hfa384x_pdr_privacy_option_t privacy_option;
-		hfa384x_pdr_temptype_t temptype;
-		hfa384x_pdr_refdac_setup_t refdac_setup;
-		hfa384x_pdr_vgdac_setup_t vgdac_setup;
-		hfa384x_pdr_level_comp_setup_t level_comp_setup;
-		hfa384x_pdr_trimdac_setup_t trimdac_setup;
-		hfa384x_pdr_ifr_setting_t ifr_setting;
-		hfa384x_pdr_rfr_setting_t rfr_setting;
-		hfa384x_pdr_hfa3861_baseline_t hfa3861_baseline;
-		hfa384x_pdr_hfa3861_shadow_t hfa3861_shadow;
-		hfa384x_pdr_hfa3861_ifrf_t hfa3861_ifrf;
-		hfa384x_pdr_hfa3861_chcalsp_t hfa3861_chcalsp;
-		hfa384x_pdr_hfa3861_chcali_t hfa3861_chcali;
-		hfa384x_pdr_nic_config_t nic_config;
-		hfa384x_hfo_delay_t hfo_delay;
-		hfa384x_pdr_hfa3861_manf_testsp_t hfa3861_manf_testsp;
-		hfa384x_pdr_hfa3861_manf_testi_t hfa3861_manf_testi;
-		hfa384x_pdr_end_of_pda_t end_of_pda;
+		struct hfa384x_pdr_pcb_partnum pcb_partnum;
+		struct hfa384x_pdr_pcb_tracenum pcb_tracenum;
+		struct hfa384x_pdr_nic_serial nic_serial;
+		struct hfa384x_pdr_mkk_measurements mkk_measurements;
+		struct hfa384x_pdr_nic_ramsize nic_ramsize;
+		struct hfa384x_pdr_mfisuprange mfisuprange;
+		struct hfa384x_pdr_cfisuprange cfisuprange;
+		struct hfa384x_pdr_nicid nicid;
+		struct hfa384x_pdr_refdac_measurements refdac_measurements;
+		struct hfa384x_pdr_vgdac_measurements vgdac_measurements;
+		struct hfa384x_pdr_level_comp_measurements level_compc_measurements;
+		struct hfa384x_pdr_mac_address mac_address;
+		struct hfa384x_pdr_mkk_callname mkk_callname;
+		struct hfa384x_pdr_regdomain regdomain;
+		struct hfa384x_pdr_allowed_channel allowed_channel;
+		struct hfa384x_pdr_default_channel default_channel;
+		struct hfa384x_pdr_privacy_option privacy_option;
+		struct hfa384x_pdr_temptype temptype;
+		struct hfa384x_pdr_refdac_setup refdac_setup;
+		struct hfa384x_pdr_vgdac_setup vgdac_setup;
+		struct hfa384x_pdr_level_comp_setup level_comp_setup;
+		struct hfa384x_pdr_trimdac_setup trimdac_setup;
+		struct hfa384x_pdr_ifr_setting ifr_setting;
+		struct hfa384x_pdr_rfr_setting rfr_setting;
+		struct hfa384x_pdr_hfa3861_baseline hfa3861_baseline;
+		struct hfa384x_pdr_hfa3861_shadow hfa3861_shadow;
+		struct hfa384x_pdr_hfa3861_ifrf hfa3861_ifrf;
+		struct hfa384x_pdr_hfa3861_chcalsp hfa3861_chcalsp;
+		struct hfa384x_pdr_hfa3861_chcali hfa3861_chcali;
+		struct hfa384x_pdr_hfa3861_nic_config nic_config;
+		struct hfa384x_pdr_hfo_delay hfo_delay;
+		struct hfa384x_pdr_hfa3861_manf_testsp hfa3861_manf_testsp;
+		struct hfa384x_pdr_hfa3861_manf_testi hfa3861_manf_testi;
+		struct hfa384x_pdr_end_of_pda end_of_pda;
 
 	} data;
-} __packed hfa384x_pdrec_t;
+} __packed;
 
 #ifdef __KERNEL__
 /*--------------------------------------------------------------------
----  MAC state structure, argument to all functions --
----  Also, a collection of support types --
---------------------------------------------------------------------*/
-typedef struct hfa384x_statusresult {
+ * ---  MAC state structure, argument to all functions --
+ * ---  Also, a collection of support types --
+ *--------------------------------------------------------------------
+ */
+struct hfa384x_cmdresult {
 	u16 status;
 	u16 resp0;
 	u16 resp1;
 	u16 resp2;
-} hfa384x_cmdresult_t;
+};
 
 /* USB Control Exchange (CTLX):
  *  A queue of the structure below is maintained for all of the
@@ -1129,11 +1152,11 @@ typedef struct hfa384x_statusresult {
 /* The following hfa384x_* structures are arguments to
  * the usercb() for the different CTLX types.
  */
-typedef struct hfa384x_rridresult {
+struct hfa384x_rridresult {
 	u16 rid;
 	const void *riddata;
 	unsigned int riddata_len;
-} hfa384x_rridresult_t;
+};
 
 enum ctlx_state {
 	CTLX_START = 0,		/* Start state, not queued */
@@ -1156,12 +1179,12 @@ typedef void (*ctlx_cmdcb_t) (struct hfa384x *, const struct hfa384x_usbctlx *);
 typedef void (*ctlx_usercb_t) (struct hfa384x *hw,
 			       void *ctlxresult, void *usercb_data);
 
-typedef struct hfa384x_usbctlx {
+struct hfa384x_usbctlx {
 	struct list_head list;
 
 	size_t outbufsize;
-	hfa384x_usbout_t outbuf;	/* pkt buf for OUT */
-	hfa384x_usbin_t inbuf;	/* pkt buf for IN(a copy) */
+	union hfa384x_usbout outbuf;	/* pkt buf for OUT */
+	union hfa384x_usbin inbuf;	/* pkt buf for IN(a copy) */
 
 	CTLX_STATE state;	/* Tracks running state */
 
@@ -1173,25 +1196,25 @@ typedef struct hfa384x_usbctlx {
 	void *usercb_data;	/*  at CTLX completion  */
 
 	int variant;		/* Identifies cmd variant */
-} hfa384x_usbctlx_t;
+};
 
-typedef struct hfa384x_usbctlxq {
+struct hfa384x_usbctlxq {
 	spinlock_t lock;
 	struct list_head pending;
 	struct list_head active;
 	struct list_head completing;
 	struct list_head reapable;
-} hfa384x_usbctlxq_t;
+};
 
-typedef struct hfa484x_metacmd {
+struct hfa384x_metacmd {
 	u16 cmd;
 
 	u16 parm0;
 	u16 parm1;
 	u16 parm2;
 
-	hfa384x_cmdresult_t result;
-} hfa384x_metacmd_t;
+	struct hfa384x_cmdresult result;
+};
 
 #define	MAX_GRP_ADDR		32
 #define WLAN_COMMENT_MAX	80  /* Max. length of user comment string. */
@@ -1218,15 +1241,15 @@ struct prism2sta_accesslist {
 	u8 addr1[WLAN_ACCESS_MAX][ETH_ALEN];
 };
 
-typedef struct hfa384x {
+struct hfa384x {
 	/* USB support data */
 	struct usb_device *usb;
 	struct urb rx_urb;
 	struct sk_buff *rx_urb_skb;
 	struct urb tx_urb;
 	struct urb ctlx_urb;
-	hfa384x_usbout_t txbuff;
-	hfa384x_usbctlxq_t ctlxq;
+	union hfa384x_usbout txbuff;
+	struct hfa384x_usbctlxq ctlxq;
 	struct timer_list reqtimer;
 	struct timer_list resptimer;
 
@@ -1265,20 +1288,20 @@ typedef struct hfa384x {
 
 	/* Download support */
 	unsigned int dlstate;
-	hfa384x_downloadbuffer_t bufinfo;
+	struct hfa384x_downloadbuffer bufinfo;
 	u16 dltimeout;
 
 	int scanflag;		/* to signal scan complete */
 	int join_ap;		/* are we joined to a specific ap */
 	int join_retries;	/* number of join retries till we fail */
-	hfa384x_JoinRequest_data_t joinreq;	/* join request saved data */
+	struct hfa384x_JoinRequest_data joinreq;	/* join request saved data */
 
-	wlandevice_t *wlandev;
+	struct wlandevice *wlandev;
 	/* Timer to allow for the deferred processing of linkstatus messages */
 	struct work_struct link_bh;
 
 	struct work_struct commsqual_bh;
-	hfa384x_commsquality_t qual;
+	struct hfa384x_commsquality qual;
 	struct timer_list commsqual_timer;
 
 	u16 link_status;
@@ -1297,92 +1320,93 @@ typedef struct hfa384x {
 	int dbmadjust;
 
 	/* Group Addresses - right now, there are up to a total
-	   of MAX_GRP_ADDR group addresses */
+	 * of MAX_GRP_ADDR group addresses
+	 */
 	u8 dot11_grp_addr[MAX_GRP_ADDR][ETH_ALEN];
 	unsigned int dot11_grpcnt;
 
 	/* Component Identities */
-	hfa384x_compident_t ident_nic;
-	hfa384x_compident_t ident_pri_fw;
-	hfa384x_compident_t ident_sta_fw;
-	hfa384x_compident_t ident_ap_fw;
+	struct hfa384x_compident ident_nic;
+	struct hfa384x_compident ident_pri_fw;
+	struct hfa384x_compident ident_sta_fw;
+	struct hfa384x_compident ident_ap_fw;
 	u16 mm_mods;
 
 	/* Supplier compatibility ranges */
-	hfa384x_caplevel_t cap_sup_mfi;
-	hfa384x_caplevel_t cap_sup_cfi;
-	hfa384x_caplevel_t cap_sup_pri;
-	hfa384x_caplevel_t cap_sup_sta;
-	hfa384x_caplevel_t cap_sup_ap;
+	struct hfa384x_caplevel cap_sup_mfi;
+	struct hfa384x_caplevel cap_sup_cfi;
+	struct hfa384x_caplevel cap_sup_pri;
+	struct hfa384x_caplevel cap_sup_sta;
+	struct hfa384x_caplevel cap_sup_ap;
 
 	/* Actor compatibility ranges */
-	hfa384x_caplevel_t cap_act_pri_cfi;	/*
-						 * pri f/w to controller
-						 * interface
-						 */
+	struct hfa384x_caplevel cap_act_pri_cfi; /*
+						  * pri f/w to controller
+						  * interface
+						  */
 
-	hfa384x_caplevel_t cap_act_sta_cfi;	/*
-						 * sta f/w to controller
-						 * interface
-						 */
+	struct hfa384x_caplevel cap_act_sta_cfi; /*
+						  * sta f/w to controller
+						  * interface
+						  */
 
-	hfa384x_caplevel_t cap_act_sta_mfi;	/* sta f/w to modem interface */
+	struct hfa384x_caplevel cap_act_sta_mfi; /* sta f/w to modem interface */
 
-	hfa384x_caplevel_t cap_act_ap_cfi;	/*
+	struct hfa384x_caplevel cap_act_ap_cfi;	/*
 						 * ap f/w to controller
 						 * interface
 						 */
 
-	hfa384x_caplevel_t cap_act_ap_mfi;	/* ap f/w to modem interface */
+	struct hfa384x_caplevel cap_act_ap_mfi;	/* ap f/w to modem interface */
 
 	u32 psusercount;	/* Power save user count. */
-	hfa384x_CommTallies32_t tallies;	/* Communication tallies. */
+	struct hfa384x_CommTallies32 tallies;	/* Communication tallies. */
 	u8 comment[WLAN_COMMENT_MAX + 1];	/* User comment */
 
 	/* Channel Info request results (AP only) */
 	struct {
 		atomic_t done;
 		u8 count;
-		hfa384x_ChInfoResult_t results;
+		struct hfa384x_ChInfoResult results;
 	} channel_info;
 
-	hfa384x_InfFrame_t *scanresults;
+	struct hfa384x_InfFrame *scanresults;
 
 	struct prism2sta_authlist authlist;	/* Authenticated station list. */
 	unsigned int accessmode;		/* Access mode. */
 	struct prism2sta_accesslist allow;	/* Allowed station list. */
 	struct prism2sta_accesslist deny;	/* Denied station list. */
 
-} hfa384x_t;
+};
 
-void hfa384x_create(hfa384x_t *hw, struct usb_device *usb);
-void hfa384x_destroy(hfa384x_t *hw);
+void hfa384x_create(struct hfa384x *hw, struct usb_device *usb);
+void hfa384x_destroy(struct hfa384x *hw);
 
 int
-hfa384x_corereset(hfa384x_t *hw, int holdtime, int settletime, int genesis);
-int hfa384x_drvr_disable(hfa384x_t *hw, u16 macport);
-int hfa384x_drvr_enable(hfa384x_t *hw, u16 macport);
-int hfa384x_drvr_flashdl_enable(hfa384x_t *hw);
-int hfa384x_drvr_flashdl_disable(hfa384x_t *hw);
-int hfa384x_drvr_flashdl_write(hfa384x_t *hw, u32 daddr, void *buf, u32 len);
-int hfa384x_drvr_getconfig(hfa384x_t *hw, u16 rid, void *buf, u16 len);
-int hfa384x_drvr_ramdl_enable(hfa384x_t *hw, u32 exeaddr);
-int hfa384x_drvr_ramdl_disable(hfa384x_t *hw);
-int hfa384x_drvr_ramdl_write(hfa384x_t *hw, u32 daddr, void *buf, u32 len);
-int hfa384x_drvr_readpda(hfa384x_t *hw, void *buf, unsigned int len);
-int hfa384x_drvr_setconfig(hfa384x_t *hw, u16 rid, void *buf, u16 len);
-
-static inline int hfa384x_drvr_getconfig16(hfa384x_t *hw, u16 rid, void *val)
+hfa384x_corereset(struct hfa384x *hw, int holdtime, int settletime, int genesis);
+int hfa384x_drvr_disable(struct hfa384x *hw, u16 macport);
+int hfa384x_drvr_enable(struct hfa384x *hw, u16 macport);
+int hfa384x_drvr_flashdl_enable(struct hfa384x *hw);
+int hfa384x_drvr_flashdl_disable(struct hfa384x *hw);
+int hfa384x_drvr_flashdl_write(struct hfa384x *hw, u32 daddr, void *buf, u32 len);
+int hfa384x_drvr_getconfig(struct hfa384x *hw, u16 rid, void *buf, u16 len);
+int hfa384x_drvr_ramdl_enable(struct hfa384x *hw, u32 exeaddr);
+int hfa384x_drvr_ramdl_disable(struct hfa384x *hw);
+int hfa384x_drvr_ramdl_write(struct hfa384x *hw, u32 daddr, void *buf, u32 len);
+int hfa384x_drvr_readpda(struct hfa384x *hw, void *buf, unsigned int len);
+int hfa384x_drvr_setconfig(struct hfa384x *hw, u16 rid, void *buf, u16 len);
+
+static inline int hfa384x_drvr_getconfig16(struct hfa384x *hw, u16 rid, void *val)
 {
 	int result = 0;
 
 	result = hfa384x_drvr_getconfig(hw, rid, val, sizeof(u16));
 	if (result == 0)
-		*((u16 *) val) = le16_to_cpu(*((u16 *) val));
+		*((u16 *)val) = le16_to_cpu(*((u16 *)val));
 	return result;
 }
 
-static inline int hfa384x_drvr_setconfig16(hfa384x_t *hw, u16 rid, u16 val)
+static inline int hfa384x_drvr_setconfig16(struct hfa384x *hw, u16 rid, u16 val)
 {
 	u16 value = cpu_to_le16(val);
 
@@ -1390,13 +1414,13 @@ static inline int hfa384x_drvr_setconfig16(hfa384x_t *hw, u16 rid, u16 val)
 }
 
 int
-hfa384x_drvr_setconfig_async(hfa384x_t *hw,
+hfa384x_drvr_setconfig_async(struct hfa384x *hw,
 			     u16 rid,
 			     void *buf,
 			     u16 len, ctlx_usercb_t usercb, void *usercb_data);
 
 static inline int
-hfa384x_drvr_setconfig16_async(hfa384x_t *hw, u16 rid, u16 val)
+hfa384x_drvr_setconfig16_async(struct hfa384x *hw, u16 rid, u16 val)
 {
 	u16 value = cpu_to_le16(val);
 
@@ -1404,21 +1428,21 @@ hfa384x_drvr_setconfig16_async(hfa384x_t *hw, u16 rid, u16 val)
 					    NULL, NULL);
 }
 
-int hfa384x_drvr_start(hfa384x_t *hw);
-int hfa384x_drvr_stop(hfa384x_t *hw);
+int hfa384x_drvr_start(struct hfa384x *hw);
+int hfa384x_drvr_stop(struct hfa384x *hw);
 int
-hfa384x_drvr_txframe(hfa384x_t *hw, struct sk_buff *skb,
+hfa384x_drvr_txframe(struct hfa384x *hw, struct sk_buff *skb,
 		     union p80211_hdr *p80211_hdr,
 		     struct p80211_metawep *p80211_wep);
-void hfa384x_tx_timeout(wlandevice_t *wlandev);
+void hfa384x_tx_timeout(struct wlandevice *wlandev);
 
-int hfa384x_cmd_initialize(hfa384x_t *hw);
-int hfa384x_cmd_enable(hfa384x_t *hw, u16 macport);
-int hfa384x_cmd_disable(hfa384x_t *hw, u16 macport);
-int hfa384x_cmd_allocate(hfa384x_t *hw, u16 len);
-int hfa384x_cmd_monitor(hfa384x_t *hw, u16 enable);
+int hfa384x_cmd_initialize(struct hfa384x *hw);
+int hfa384x_cmd_enable(struct hfa384x *hw, u16 macport);
+int hfa384x_cmd_disable(struct hfa384x *hw, u16 macport);
+int hfa384x_cmd_allocate(struct hfa384x *hw, u16 len);
+int hfa384x_cmd_monitor(struct hfa384x *hw, u16 enable);
 int
-hfa384x_cmd_download(hfa384x_t *hw,
+hfa384x_cmd_download(struct hfa384x *hw,
 		     u16 mode, u16 lowaddr, u16 highaddr, u16 codelen);
 
 #endif /*__KERNEL__ */
diff --git a/drivers/staging/wlan-ng/hfa384x_usb.c b/drivers/staging/wlan-ng/hfa384x_usb.c
index 337810750f2b..6a107f8a06e2 100644
--- a/drivers/staging/wlan-ng/hfa384x_usb.c
+++ b/drivers/staging/wlan-ng/hfa384x_usb.c
@@ -154,13 +154,13 @@ static void dbprint_urb(struct urb *urb);
 #endif
 
 static void
-hfa384x_int_rxmonitor(wlandevice_t *wlandev, hfa384x_usb_rxfrm_t *rxfrm);
+hfa384x_int_rxmonitor(struct wlandevice *wlandev, struct hfa384x_usb_rxfrm *rxfrm);
 
 static void hfa384x_usb_defer(struct work_struct *data);
 
-static int submit_rx_urb(hfa384x_t *hw, gfp_t flags);
+static int submit_rx_urb(struct hfa384x *hw, gfp_t flags);
 
-static int submit_tx_urb(hfa384x_t *hw, struct urb *tx_urb, gfp_t flags);
+static int submit_tx_urb(struct hfa384x *hw, struct urb *tx_urb, gfp_t flags);
 
 /*---------------------------------------------------*/
 /* Callbacks */
@@ -169,19 +169,19 @@ static void hfa384x_ctlxout_callback(struct urb *urb);
 static void hfa384x_usbin_callback(struct urb *urb);
 
 static void
-hfa384x_usbin_txcompl(wlandevice_t *wlandev, hfa384x_usbin_t *usbin);
+hfa384x_usbin_txcompl(struct wlandevice *wlandev, union hfa384x_usbin *usbin);
 
-static void hfa384x_usbin_rx(wlandevice_t *wlandev, struct sk_buff *skb);
+static void hfa384x_usbin_rx(struct wlandevice *wlandev, struct sk_buff *skb);
 
-static void hfa384x_usbin_info(wlandevice_t *wlandev, hfa384x_usbin_t *usbin);
+static void hfa384x_usbin_info(struct wlandevice *wlandev, union hfa384x_usbin *usbin);
 
-static void hfa384x_usbin_ctlx(hfa384x_t *hw, hfa384x_usbin_t *usbin,
+static void hfa384x_usbin_ctlx(struct hfa384x *hw, union hfa384x_usbin *usbin,
 			       int urb_status);
 
 /*---------------------------------------------------*/
 /* Functions to support the prism2 usb command queue */
 
-static void hfa384x_usbctlxq_run(hfa384x_t *hw);
+static void hfa384x_usbctlxq_run(struct hfa384x *hw);
 
 static void hfa384x_usbctlx_reqtimerfn(unsigned long data);
 
@@ -193,42 +193,42 @@ static void hfa384x_usbctlx_completion_task(unsigned long data);
 
 static void hfa384x_usbctlx_reaper_task(unsigned long data);
 
-static int hfa384x_usbctlx_submit(hfa384x_t *hw, hfa384x_usbctlx_t *ctlx);
+static int hfa384x_usbctlx_submit(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx);
 
-static void unlocked_usbctlx_complete(hfa384x_t *hw, hfa384x_usbctlx_t *ctlx);
+static void unlocked_usbctlx_complete(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx);
 
 struct usbctlx_completor {
 	int (*complete)(struct usbctlx_completor *);
 };
 
 static int
-hfa384x_usbctlx_complete_sync(hfa384x_t *hw,
-			      hfa384x_usbctlx_t *ctlx,
+hfa384x_usbctlx_complete_sync(struct hfa384x *hw,
+			      struct hfa384x_usbctlx *ctlx,
 			      struct usbctlx_completor *completor);
 
 static int
-unlocked_usbctlx_cancel_async(hfa384x_t *hw, hfa384x_usbctlx_t *ctlx);
+unlocked_usbctlx_cancel_async(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx);
 
-static void hfa384x_cb_status(hfa384x_t *hw, const hfa384x_usbctlx_t *ctlx);
+static void hfa384x_cb_status(struct hfa384x *hw, const struct hfa384x_usbctlx *ctlx);
 
 static int
-usbctlx_get_status(const hfa384x_usb_cmdresp_t *cmdresp,
-		   hfa384x_cmdresult_t *result);
+usbctlx_get_status(const struct hfa384x_usb_statusresp *cmdresp,
+		   struct hfa384x_cmdresult *result);
 
 static void
-usbctlx_get_rridresult(const hfa384x_usb_rridresp_t *rridresp,
-		       hfa384x_rridresult_t *result);
+usbctlx_get_rridresult(const struct hfa384x_usb_rridresp *rridresp,
+		       struct hfa384x_rridresult *result);
 
 /*---------------------------------------------------*/
 /* Low level req/resp CTLX formatters and submitters */
 static int
-hfa384x_docmd(hfa384x_t *hw,
+hfa384x_docmd(struct hfa384x *hw,
 	      enum cmd_mode mode,
-	      hfa384x_metacmd_t *cmd,
+	      struct hfa384x_metacmd *cmd,
 	      ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data);
 
 static int
-hfa384x_dorrid(hfa384x_t *hw,
+hfa384x_dorrid(struct hfa384x *hw,
 	       enum cmd_mode mode,
 	       u16 rid,
 	       void *riddata,
@@ -236,7 +236,7 @@ hfa384x_dorrid(hfa384x_t *hw,
 	       ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data);
 
 static int
-hfa384x_dowrid(hfa384x_t *hw,
+hfa384x_dowrid(struct hfa384x *hw,
 	       enum cmd_mode mode,
 	       u16 rid,
 	       void *riddata,
@@ -244,7 +244,7 @@ hfa384x_dowrid(hfa384x_t *hw,
 	       ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data);
 
 static int
-hfa384x_dormem(hfa384x_t *hw,
+hfa384x_dormem(struct hfa384x *hw,
 	       enum cmd_mode mode,
 	       u16 page,
 	       u16 offset,
@@ -253,7 +253,7 @@ hfa384x_dormem(hfa384x_t *hw,
 	       ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data);
 
 static int
-hfa384x_dowmem(hfa384x_t *hw,
+hfa384x_dowmem(struct hfa384x *hw,
 	       enum cmd_mode mode,
 	       u16 page,
 	       u16 offset,
@@ -278,9 +278,9 @@ static inline const char *ctlxstr(CTLX_STATE s)
 	return ctlx_str[s];
 };
 
-static inline hfa384x_usbctlx_t *get_active_ctlx(hfa384x_t *hw)
+static inline struct hfa384x_usbctlx *get_active_ctlx(struct hfa384x *hw)
 {
-	return list_entry(hw->ctlxq.active.next, hfa384x_usbctlx_t, list);
+	return list_entry(hw->ctlxq.active.next, struct hfa384x_usbctlx, list);
 }
 
 #ifdef DEBUG_USB
@@ -322,12 +322,12 @@ void dbprint_urb(struct urb *urb)
 * Call context:
 *	Any
 ----------------------------------------------------------------*/
-static int submit_rx_urb(hfa384x_t *hw, gfp_t memflags)
+static int submit_rx_urb(struct hfa384x *hw, gfp_t memflags)
 {
 	struct sk_buff *skb;
 	int result;
 
-	skb = dev_alloc_skb(sizeof(hfa384x_usbin_t));
+	skb = dev_alloc_skb(sizeof(union hfa384x_usbin));
 	if (!skb) {
 		result = -ENOMEM;
 		goto done;
@@ -336,7 +336,7 @@ static int submit_rx_urb(hfa384x_t *hw, gfp_t memflags)
 	/* Post the IN urb */
 	usb_fill_bulk_urb(&hw->rx_urb, hw->usb,
 			  hw->endp_in,
-			  skb->data, sizeof(hfa384x_usbin_t),
+			  skb->data, sizeof(union hfa384x_usbin),
 			  hfa384x_usbin_callback, hw->wlandev);
 
 	hw->rx_urb_skb = skb;
@@ -384,7 +384,7 @@ done:
 * Call context:
 *	Any
 ----------------------------------------------------------------*/
-static int submit_tx_urb(hfa384x_t *hw, struct urb *tx_urb, gfp_t memflags)
+static int submit_tx_urb(struct hfa384x *hw, struct urb *tx_urb, gfp_t memflags)
 {
 	struct net_device *netdev = hw->wlandev->netdev;
 	int result;
@@ -429,7 +429,7 @@ static int submit_tx_urb(hfa384x_t *hw, struct urb *tx_urb, gfp_t memflags)
 ----------------------------------------------------------------*/
 static void hfa384x_usb_defer(struct work_struct *data)
 {
-	hfa384x_t *hw = container_of(data, struct hfa384x, usb_work);
+	struct hfa384x *hw = container_of(data, struct hfa384x, usb_work);
 	struct net_device *netdev = hw->wlandev->netdev;
 
 	/* Don't bother trying to reset anything if the plug
@@ -503,7 +503,7 @@ static void hfa384x_usb_defer(struct work_struct *data)
 /*----------------------------------------------------------------
 * hfa384x_create
 *
-* Sets up the hfa384x_t data structure for use.  Note this
+* Sets up the struct hfa384x data structure for use.  Note this
 * does _not_ initialize the actual hardware, just the data structures
 * we use to keep track of its state.
 *
@@ -521,9 +521,9 @@ static void hfa384x_usb_defer(struct work_struct *data)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-void hfa384x_create(hfa384x_t *hw, struct usb_device *usb)
+void hfa384x_create(struct hfa384x *hw, struct usb_device *usb)
 {
-	memset(hw, 0, sizeof(hfa384x_t));
+	memset(hw, 0, sizeof(struct hfa384x));
 	hw->usb = usb;
 
 	/* set up the endpoints */
@@ -592,7 +592,7 @@ void hfa384x_create(hfa384x_t *hw, struct usb_device *usb)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-void hfa384x_destroy(hfa384x_t *hw)
+void hfa384x_destroy(struct hfa384x *hw)
 {
 	struct sk_buff *skb;
 
@@ -608,9 +608,9 @@ void hfa384x_destroy(hfa384x_t *hw)
 		dev_kfree_skb(skb);
 }
 
-static hfa384x_usbctlx_t *usbctlx_alloc(void)
+static struct hfa384x_usbctlx *usbctlx_alloc(void)
 {
-	hfa384x_usbctlx_t *ctlx;
+	struct hfa384x_usbctlx *ctlx;
 
 	ctlx = kzalloc(sizeof(*ctlx),
 		       in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
@@ -621,8 +621,8 @@ static hfa384x_usbctlx_t *usbctlx_alloc(void)
 }
 
 static int
-usbctlx_get_status(const hfa384x_usb_cmdresp_t *cmdresp,
-		   hfa384x_cmdresult_t *result)
+usbctlx_get_status(const struct hfa384x_usb_statusresp *cmdresp,
+		   struct hfa384x_cmdresult *result)
 {
 	result->status = le16_to_cpu(cmdresp->status);
 	result->resp0 = le16_to_cpu(cmdresp->resp0);
@@ -636,8 +636,8 @@ usbctlx_get_status(const hfa384x_usb_cmdresp_t *cmdresp,
 }
 
 static void
-usbctlx_get_rridresult(const hfa384x_usb_rridresp_t *rridresp,
-		       hfa384x_rridresult_t *result)
+usbctlx_get_rridresult(const struct hfa384x_usb_rridresp *rridresp,
+		       struct hfa384x_rridresult *result)
 {
 	result->rid = le16_to_cpu(rridresp->rid);
 	result->riddata = rridresp->data;
@@ -647,13 +647,13 @@ usbctlx_get_rridresult(const hfa384x_usb_rridresp_t *rridresp,
 /*----------------------------------------------------------------
 * Completor object:
 * This completor must be passed to hfa384x_usbctlx_complete_sync()
-* when processing a CTLX that returns a hfa384x_cmdresult_t structure.
+* when processing a CTLX that returns a struct hfa384x_cmdresult structure.
 ----------------------------------------------------------------*/
 struct usbctlx_cmd_completor {
 	struct usbctlx_completor head;
 
-	const hfa384x_usb_cmdresp_t *cmdresp;
-	hfa384x_cmdresult_t *result;
+	const struct hfa384x_usb_statusresp *cmdresp;
+	struct hfa384x_cmdresult *result;
 };
 
 static inline int usbctlx_cmd_completor_fn(struct usbctlx_completor *head)
@@ -667,9 +667,9 @@ static inline int usbctlx_cmd_completor_fn(struct usbctlx_completor *head)
 static inline struct usbctlx_completor *init_cmd_completor(
 						struct usbctlx_cmd_completor
 							*completor,
-						const hfa384x_usb_cmdresp_t
+						const struct hfa384x_usb_statusresp
 							*cmdresp,
-						hfa384x_cmdresult_t *result)
+						struct hfa384x_cmdresult *result)
 {
 	completor->head.complete = usbctlx_cmd_completor_fn;
 	completor->cmdresp = cmdresp;
@@ -685,7 +685,7 @@ static inline struct usbctlx_completor *init_cmd_completor(
 struct usbctlx_rrid_completor {
 	struct usbctlx_completor head;
 
-	const hfa384x_usb_rridresp_t *rridresp;
+	const struct hfa384x_usb_rridresp *rridresp;
 	void *riddata;
 	unsigned int riddatalen;
 };
@@ -693,7 +693,7 @@ struct usbctlx_rrid_completor {
 static int usbctlx_rrid_completor_fn(struct usbctlx_completor *head)
 {
 	struct usbctlx_rrid_completor *complete;
-	hfa384x_rridresult_t rridresult;
+	struct hfa384x_rridresult rridresult;
 
 	complete = (struct usbctlx_rrid_completor *)head;
 	usbctlx_get_rridresult(complete->rridresp, &rridresult);
@@ -713,7 +713,7 @@ static int usbctlx_rrid_completor_fn(struct usbctlx_completor *head)
 static inline struct usbctlx_completor *init_rrid_completor(
 						struct usbctlx_rrid_completor
 							*completor,
-						const hfa384x_usb_rridresp_t
+						const struct hfa384x_usb_rridresp
 							*rridresp,
 						void *riddata,
 						unsigned int riddatalen)
@@ -744,7 +744,7 @@ static inline struct usbctlx_completor *init_rrid_completor(
 struct usbctlx_rmem_completor {
 	struct usbctlx_completor head;
 
-	const hfa384x_usb_rmemresp_t *rmemresp;
+	const struct hfa384x_usb_rmemresp *rmemresp;
 	void *data;
 	unsigned int len;
 };
@@ -762,7 +762,7 @@ static int usbctlx_rmem_completor_fn(struct usbctlx_completor *head)
 static inline struct usbctlx_completor *init_rmem_completor(
 						struct usbctlx_rmem_completor
 							*completor,
-						hfa384x_usb_rmemresp_t
+						struct hfa384x_usb_rmemresp
 							*rmemresp,
 						void *data,
 						unsigned int len)
@@ -795,10 +795,10 @@ static inline struct usbctlx_completor *init_rmem_completor(
 * Call context:
 *	interrupt
 ----------------------------------------------------------------*/
-static void hfa384x_cb_status(hfa384x_t *hw, const hfa384x_usbctlx_t *ctlx)
+static void hfa384x_cb_status(struct hfa384x *hw, const struct hfa384x_usbctlx *ctlx)
 {
 	if (ctlx->usercb) {
-		hfa384x_cmdresult_t cmdresult;
+		struct hfa384x_cmdresult cmdresult;
 
 		if (ctlx->state != CTLX_COMPLETE) {
 			memset(&cmdresult, 0, sizeof(cmdresult));
@@ -812,21 +812,21 @@ static void hfa384x_cb_status(hfa384x_t *hw, const hfa384x_usbctlx_t *ctlx)
 	}
 }
 
-static inline int hfa384x_docmd_wait(hfa384x_t *hw, hfa384x_metacmd_t *cmd)
+static inline int hfa384x_docmd_wait(struct hfa384x *hw, struct hfa384x_metacmd *cmd)
 {
 	return hfa384x_docmd(hw, DOWAIT, cmd, NULL, NULL, NULL);
 }
 
 static inline int
-hfa384x_docmd_async(hfa384x_t *hw,
-		    hfa384x_metacmd_t *cmd,
+hfa384x_docmd_async(struct hfa384x *hw,
+		    struct hfa384x_metacmd *cmd,
 		    ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data)
 {
 	return hfa384x_docmd(hw, DOASYNC, cmd, cmdcb, usercb, usercb_data);
 }
 
 static inline int
-hfa384x_dorrid_wait(hfa384x_t *hw, u16 rid, void *riddata,
+hfa384x_dorrid_wait(struct hfa384x *hw, u16 rid, void *riddata,
 		    unsigned int riddatalen)
 {
 	return hfa384x_dorrid(hw, DOWAIT,
@@ -834,7 +834,7 @@ hfa384x_dorrid_wait(hfa384x_t *hw, u16 rid, void *riddata,
 }
 
 static inline int
-hfa384x_dorrid_async(hfa384x_t *hw,
+hfa384x_dorrid_async(struct hfa384x *hw,
 		     u16 rid, void *riddata, unsigned int riddatalen,
 		     ctlx_cmdcb_t cmdcb,
 		     ctlx_usercb_t usercb, void *usercb_data)
@@ -845,7 +845,7 @@ hfa384x_dorrid_async(hfa384x_t *hw,
 }
 
 static inline int
-hfa384x_dowrid_wait(hfa384x_t *hw, u16 rid, void *riddata,
+hfa384x_dowrid_wait(struct hfa384x *hw, u16 rid, void *riddata,
 		    unsigned int riddatalen)
 {
 	return hfa384x_dowrid(hw, DOWAIT,
@@ -853,7 +853,7 @@ hfa384x_dowrid_wait(hfa384x_t *hw, u16 rid, void *riddata,
 }
 
 static inline int
-hfa384x_dowrid_async(hfa384x_t *hw,
+hfa384x_dowrid_async(struct hfa384x *hw,
 		     u16 rid, void *riddata, unsigned int riddatalen,
 		     ctlx_cmdcb_t cmdcb,
 		     ctlx_usercb_t usercb, void *usercb_data)
@@ -864,7 +864,7 @@ hfa384x_dowrid_async(hfa384x_t *hw,
 }
 
 static inline int
-hfa384x_dormem_wait(hfa384x_t *hw,
+hfa384x_dormem_wait(struct hfa384x *hw,
 		    u16 page, u16 offset, void *data, unsigned int len)
 {
 	return hfa384x_dormem(hw, DOWAIT,
@@ -872,7 +872,7 @@ hfa384x_dormem_wait(hfa384x_t *hw,
 }
 
 static inline int
-hfa384x_dormem_async(hfa384x_t *hw,
+hfa384x_dormem_async(struct hfa384x *hw,
 		     u16 page, u16 offset, void *data, unsigned int len,
 		     ctlx_cmdcb_t cmdcb,
 		     ctlx_usercb_t usercb, void *usercb_data)
@@ -883,7 +883,7 @@ hfa384x_dormem_async(hfa384x_t *hw,
 }
 
 static inline int
-hfa384x_dowmem_wait(hfa384x_t *hw,
+hfa384x_dowmem_wait(struct hfa384x *hw,
 		    u16 page, u16 offset, void *data, unsigned int len)
 {
 	return hfa384x_dowmem(hw, DOWAIT,
@@ -891,7 +891,7 @@ hfa384x_dowmem_wait(hfa384x_t *hw,
 }
 
 static inline int
-hfa384x_dowmem_async(hfa384x_t *hw,
+hfa384x_dowmem_async(struct hfa384x *hw,
 		     u16 page,
 		     u16 offset,
 		     void *data,
@@ -923,11 +923,11 @@ hfa384x_dowmem_async(hfa384x_t *hw,
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_cmd_initialize(hfa384x_t *hw)
+int hfa384x_cmd_initialize(struct hfa384x *hw)
 {
 	int result = 0;
 	int i;
-	hfa384x_metacmd_t cmd;
+	struct hfa384x_metacmd cmd;
 
 	cmd.cmd = HFA384x_CMDCODE_INIT;
 	cmd.parm0 = 0;
@@ -969,9 +969,9 @@ int hfa384x_cmd_initialize(hfa384x_t *hw)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_cmd_disable(hfa384x_t *hw, u16 macport)
+int hfa384x_cmd_disable(struct hfa384x *hw, u16 macport)
 {
-	hfa384x_metacmd_t cmd;
+	struct hfa384x_metacmd cmd;
 
 	cmd.cmd = HFA384x_CMD_CMDCODE_SET(HFA384x_CMDCODE_DISABLE) |
 	    HFA384x_CMD_MACPORT_SET(macport);
@@ -1002,9 +1002,9 @@ int hfa384x_cmd_disable(hfa384x_t *hw, u16 macport)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_cmd_enable(hfa384x_t *hw, u16 macport)
+int hfa384x_cmd_enable(struct hfa384x *hw, u16 macport)
 {
-	hfa384x_metacmd_t cmd;
+	struct hfa384x_metacmd cmd;
 
 	cmd.cmd = HFA384x_CMD_CMDCODE_SET(HFA384x_CMDCODE_ENABLE) |
 	    HFA384x_CMD_MACPORT_SET(macport);
@@ -1044,9 +1044,9 @@ int hfa384x_cmd_enable(hfa384x_t *hw, u16 macport)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_cmd_monitor(hfa384x_t *hw, u16 enable)
+int hfa384x_cmd_monitor(struct hfa384x *hw, u16 enable)
 {
-	hfa384x_metacmd_t cmd;
+	struct hfa384x_metacmd cmd;
 
 	cmd.cmd = HFA384x_CMD_CMDCODE_SET(HFA384x_CMDCODE_MONITOR) |
 	    HFA384x_CMD_AINFO_SET(enable);
@@ -1095,10 +1095,10 @@ int hfa384x_cmd_monitor(hfa384x_t *hw, u16 enable)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_cmd_download(hfa384x_t *hw, u16 mode, u16 lowaddr,
+int hfa384x_cmd_download(struct hfa384x *hw, u16 mode, u16 lowaddr,
 			 u16 highaddr, u16 codelen)
 {
-	hfa384x_metacmd_t cmd;
+	struct hfa384x_metacmd cmd;
 
 	pr_debug("mode=%d, lowaddr=0x%04x, highaddr=0x%04x, codelen=%d\n",
 		 mode, lowaddr, highaddr, codelen);
@@ -1136,7 +1136,7 @@ int hfa384x_cmd_download(hfa384x_t *hw, u16 mode, u16 lowaddr,
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_corereset(hfa384x_t *hw, int holdtime, int settletime, int genesis)
+int hfa384x_corereset(struct hfa384x *hw, int holdtime, int settletime, int genesis)
 {
 	int result;
 
@@ -1173,8 +1173,8 @@ int hfa384x_corereset(hfa384x_t *hw, int holdtime, int settletime, int genesis)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-static int hfa384x_usbctlx_complete_sync(hfa384x_t *hw,
-					 hfa384x_usbctlx_t *ctlx,
+static int hfa384x_usbctlx_complete_sync(struct hfa384x *hw,
+					 struct hfa384x_usbctlx *ctlx,
 					 struct usbctlx_completor *completor)
 {
 	unsigned long flags;
@@ -1289,13 +1289,13 @@ cleanup:
 *	process
 ----------------------------------------------------------------*/
 static int
-hfa384x_docmd(hfa384x_t *hw,
+hfa384x_docmd(struct hfa384x *hw,
 	      enum cmd_mode mode,
-	      hfa384x_metacmd_t *cmd,
+	      struct hfa384x_metacmd *cmd,
 	      ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data)
 {
 	int result;
-	hfa384x_usbctlx_t *ctlx;
+	struct hfa384x_usbctlx *ctlx;
 
 	ctlx = usbctlx_alloc();
 	if (!ctlx) {
@@ -1377,7 +1377,7 @@ done:
 *	process (DOWAIT or DOASYNC)
 ----------------------------------------------------------------*/
 static int
-hfa384x_dorrid(hfa384x_t *hw,
+hfa384x_dorrid(struct hfa384x *hw,
 	       enum cmd_mode mode,
 	       u16 rid,
 	       void *riddata,
@@ -1385,7 +1385,7 @@ hfa384x_dorrid(hfa384x_t *hw,
 	       ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data)
 {
 	int result;
-	hfa384x_usbctlx_t *ctlx;
+	struct hfa384x_usbctlx *ctlx;
 
 	ctlx = usbctlx_alloc();
 	if (!ctlx) {
@@ -1458,7 +1458,7 @@ done:
 *	process (DOWAIT or DOASYNC)
 ----------------------------------------------------------------*/
 static int
-hfa384x_dowrid(hfa384x_t *hw,
+hfa384x_dowrid(struct hfa384x *hw,
 	       enum cmd_mode mode,
 	       u16 rid,
 	       void *riddata,
@@ -1466,7 +1466,7 @@ hfa384x_dowrid(hfa384x_t *hw,
 	       ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data)
 {
 	int result;
-	hfa384x_usbctlx_t *ctlx;
+	struct hfa384x_usbctlx *ctlx;
 
 	ctlx = usbctlx_alloc();
 	if (!ctlx) {
@@ -1497,7 +1497,7 @@ hfa384x_dowrid(hfa384x_t *hw,
 		kfree(ctlx);
 	} else if (mode == DOWAIT) {
 		struct usbctlx_cmd_completor completor;
-		hfa384x_cmdresult_t wridresult;
+		struct hfa384x_cmdresult wridresult;
 
 		result = hfa384x_usbctlx_complete_sync(hw,
 						       ctlx,
@@ -1545,7 +1545,7 @@ done:
 *	process (DOWAIT or DOASYNC)
 ----------------------------------------------------------------*/
 static int
-hfa384x_dormem(hfa384x_t *hw,
+hfa384x_dormem(struct hfa384x *hw,
 	       enum cmd_mode mode,
 	       u16 page,
 	       u16 offset,
@@ -1554,7 +1554,7 @@ hfa384x_dormem(hfa384x_t *hw,
 	       ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data)
 {
 	int result;
-	hfa384x_usbctlx_t *ctlx;
+	struct hfa384x_usbctlx *ctlx;
 
 	ctlx = usbctlx_alloc();
 	if (!ctlx) {
@@ -1636,7 +1636,7 @@ done:
 *	process (DOWAIT or DOASYNC)
 ----------------------------------------------------------------*/
 static int
-hfa384x_dowmem(hfa384x_t *hw,
+hfa384x_dowmem(struct hfa384x *hw,
 	       enum cmd_mode mode,
 	       u16 page,
 	       u16 offset,
@@ -1645,7 +1645,7 @@ hfa384x_dowmem(hfa384x_t *hw,
 	       ctlx_cmdcb_t cmdcb, ctlx_usercb_t usercb, void *usercb_data)
 {
 	int result;
-	hfa384x_usbctlx_t *ctlx;
+	struct hfa384x_usbctlx *ctlx;
 
 	pr_debug("page=0x%04x offset=0x%04x len=%d\n", page, offset, len);
 
@@ -1679,7 +1679,7 @@ hfa384x_dowmem(hfa384x_t *hw,
 		kfree(ctlx);
 	} else if (mode == DOWAIT) {
 		struct usbctlx_cmd_completor completor;
-		hfa384x_cmdresult_t wmemresult;
+		struct hfa384x_cmdresult wmemresult;
 
 		result = hfa384x_usbctlx_complete_sync(hw,
 						       ctlx,
@@ -1715,7 +1715,7 @@ done:
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_disable(hfa384x_t *hw, u16 macport)
+int hfa384x_drvr_disable(struct hfa384x *hw, u16 macport)
 {
 	int result = 0;
 
@@ -1753,7 +1753,7 @@ int hfa384x_drvr_disable(hfa384x_t *hw, u16 macport)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_enable(hfa384x_t *hw, u16 macport)
+int hfa384x_drvr_enable(struct hfa384x *hw, u16 macport)
 {
 	int result = 0;
 
@@ -1790,7 +1790,7 @@ int hfa384x_drvr_enable(hfa384x_t *hw, u16 macport)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_flashdl_enable(hfa384x_t *hw)
+int hfa384x_drvr_flashdl_enable(struct hfa384x *hw)
 {
 	int result = 0;
 	int i;
@@ -1849,7 +1849,7 @@ int hfa384x_drvr_flashdl_enable(hfa384x_t *hw)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_flashdl_disable(hfa384x_t *hw)
+int hfa384x_drvr_flashdl_disable(struct hfa384x *hw)
 {
 	/* Check that we're already in the download state */
 	if (hw->dlstate != HFA384x_DLSTATE_FLASHENABLED)
@@ -1894,7 +1894,7 @@ int hfa384x_drvr_flashdl_disable(hfa384x_t *hw)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_flashdl_write(hfa384x_t *hw, u32 daddr, void *buf, u32 len)
+int hfa384x_drvr_flashdl_write(struct hfa384x *hw, u32 daddr, void *buf, u32 len)
 {
 	int result = 0;
 	u32 dlbufaddr;
@@ -2032,7 +2032,7 @@ exit_proc:
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_getconfig(hfa384x_t *hw, u16 rid, void *buf, u16 len)
+int hfa384x_drvr_getconfig(struct hfa384x *hw, u16 rid, void *buf, u16 len)
 {
 	return hfa384x_dorrid_wait(hw, rid, buf, len);
 }
@@ -2061,7 +2061,7 @@ int hfa384x_drvr_getconfig(hfa384x_t *hw, u16 rid, void *buf, u16 len)
  *       process
  ----------------------------------------------------------------*/
 int
-hfa384x_drvr_setconfig_async(hfa384x_t *hw,
+hfa384x_drvr_setconfig_async(struct hfa384x *hw,
 			     u16 rid,
 			     void *buf,
 			     u16 len, ctlx_usercb_t usercb, void *usercb_data)
@@ -2088,7 +2088,7 @@ hfa384x_drvr_setconfig_async(hfa384x_t *hw,
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_ramdl_disable(hfa384x_t *hw)
+int hfa384x_drvr_ramdl_disable(struct hfa384x *hw)
 {
 	/* Check that we're already in the download state */
 	if (hw->dlstate != HFA384x_DLSTATE_RAMENABLED)
@@ -2128,7 +2128,7 @@ int hfa384x_drvr_ramdl_disable(hfa384x_t *hw)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_ramdl_enable(hfa384x_t *hw, u32 exeaddr)
+int hfa384x_drvr_ramdl_enable(struct hfa384x *hw, u32 exeaddr)
 {
 	int result = 0;
 	u16 lowaddr;
@@ -2196,7 +2196,7 @@ int hfa384x_drvr_ramdl_enable(hfa384x_t *hw, u32 exeaddr)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_ramdl_write(hfa384x_t *hw, u32 daddr, void *buf, u32 len)
+int hfa384x_drvr_ramdl_write(struct hfa384x *hw, u32 daddr, void *buf, u32 len)
 {
 	int result = 0;
 	int nwrites;
@@ -2276,7 +2276,7 @@ int hfa384x_drvr_ramdl_write(hfa384x_t *hw, u32 daddr, void *buf, u32 len)
 * Call context:
 *	process or non-card interrupt.
 ----------------------------------------------------------------*/
-int hfa384x_drvr_readpda(hfa384x_t *hw, void *buf, unsigned int len)
+int hfa384x_drvr_readpda(struct hfa384x *hw, void *buf, unsigned int len)
 {
 	int result = 0;
 	u16 *pda = buf;
@@ -2386,7 +2386,7 @@ int hfa384x_drvr_readpda(hfa384x_t *hw, void *buf, unsigned int len)
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_setconfig(hfa384x_t *hw, u16 rid, void *buf, u16 len)
+int hfa384x_drvr_setconfig(struct hfa384x *hw, u16 rid, void *buf, u16 len)
 {
 	return hfa384x_dowrid_wait(hw, rid, buf, len);
 }
@@ -2411,7 +2411,7 @@ int hfa384x_drvr_setconfig(hfa384x_t *hw, u16 rid, void *buf, u16 len)
 *	process
 ----------------------------------------------------------------*/
 
-int hfa384x_drvr_start(hfa384x_t *hw)
+int hfa384x_drvr_start(struct hfa384x *hw)
 {
 	int result, result1, result2;
 	u16 status;
@@ -2512,7 +2512,7 @@ done:
 * Call context:
 *	process
 ----------------------------------------------------------------*/
-int hfa384x_drvr_stop(hfa384x_t *hw)
+int hfa384x_drvr_stop(struct hfa384x *hw)
 {
 	int i;
 
@@ -2562,11 +2562,11 @@ int hfa384x_drvr_stop(hfa384x_t *hw)
 * Call context:
 *	interrupt
 ----------------------------------------------------------------*/
-int hfa384x_drvr_txframe(hfa384x_t *hw, struct sk_buff *skb,
+int hfa384x_drvr_txframe(struct hfa384x *hw, struct sk_buff *skb,
 			 union p80211_hdr *p80211_hdr,
 			 struct p80211_metawep *p80211_wep)
 {
-	int usbpktlen = sizeof(hfa384x_tx_frame_t);
+	int usbpktlen = sizeof(struct hfa384x_tx_frame);
 	int result;
 	int ret;
 	char *ptr;
@@ -2656,9 +2656,9 @@ exit:
 	return result;
 }
 
-void hfa384x_tx_timeout(wlandevice_t *wlandev)
+void hfa384x_tx_timeout(struct wlandevice *wlandev)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	unsigned long flags;
 
 	spin_lock_irqsave(&hw->ctlxq.lock, flags);
@@ -2681,7 +2681,7 @@ void hfa384x_tx_timeout(wlandevice_t *wlandev)
 * Tasklet to delete dead CTLX objects
 *
 * Arguments:
-*	data	ptr to a hfa384x_t
+*	data	ptr to a struct hfa384x
 *
 * Returns:
 *
@@ -2690,8 +2690,8 @@ void hfa384x_tx_timeout(wlandevice_t *wlandev)
 ----------------------------------------------------------------*/
 static void hfa384x_usbctlx_reaper_task(unsigned long data)
 {
-	hfa384x_t *hw = (hfa384x_t *)data;
-	hfa384x_usbctlx_t *ctlx, *temp;
+	struct hfa384x *hw = (struct hfa384x *)data;
+	struct hfa384x_usbctlx *ctlx, *temp;
 	unsigned long flags;
 
 	spin_lock_irqsave(&hw->ctlxq.lock, flags);
@@ -2713,7 +2713,7 @@ static void hfa384x_usbctlx_reaper_task(unsigned long data)
 * Tasklet to call completion handlers for returned CTLXs
 *
 * Arguments:
-*	data	ptr to hfa384x_t
+*	data	ptr to struct hfa384x
 *
 * Returns:
 *	Nothing
@@ -2723,8 +2723,8 @@ static void hfa384x_usbctlx_reaper_task(unsigned long data)
 ----------------------------------------------------------------*/
 static void hfa384x_usbctlx_completion_task(unsigned long data)
 {
-	hfa384x_t *hw = (hfa384x_t *)data;
-	hfa384x_usbctlx_t *ctlx, *temp;
+	struct hfa384x *hw = (struct hfa384x *)data;
+	struct hfa384x_usbctlx *ctlx, *temp;
 	unsigned long flags;
 
 	int reap = 0;
@@ -2787,7 +2787,7 @@ static void hfa384x_usbctlx_completion_task(unsigned long data)
 * next command on the queue is run afterwards.
 *
 * Arguments:
-*	hw	ptr to the hfa384x_t structure
+*	hw	ptr to the struct hfa384x structure
 *	ctlx	ptr to a CTLX structure
 *
 * Returns:
@@ -2797,8 +2797,8 @@ static void hfa384x_usbctlx_completion_task(unsigned long data)
 * Call context:
 *	Either process or interrupt, but presumably interrupt
 ----------------------------------------------------------------*/
-static int unlocked_usbctlx_cancel_async(hfa384x_t *hw,
-					 hfa384x_usbctlx_t *ctlx)
+static int unlocked_usbctlx_cancel_async(struct hfa384x *hw,
+					 struct hfa384x_usbctlx *ctlx)
 {
 	int ret;
 
@@ -2836,7 +2836,7 @@ static int unlocked_usbctlx_cancel_async(hfa384x_t *hw,
 * tasklet is scheduled.
 *
 * Arguments:
-*	hw		ptr to a hfa384x_t structure
+*	hw		ptr to a struct hfa384x structure
 *	ctlx		ptr to a ctlx structure
 *
 * Returns:
@@ -2847,7 +2847,7 @@ static int unlocked_usbctlx_cancel_async(hfa384x_t *hw,
 * Call context:
 *	Either, assume interrupt
 ----------------------------------------------------------------*/
-static void unlocked_usbctlx_complete(hfa384x_t *hw, hfa384x_usbctlx_t *ctlx)
+static void unlocked_usbctlx_complete(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx)
 {
 	/* Timers have been stopped, and ctlx should be in
 	 * a terminal state. Retire it from the "active"
@@ -2876,7 +2876,7 @@ static void unlocked_usbctlx_complete(hfa384x_t *hw, hfa384x_usbctlx_t *ctlx)
 * Checks to see if the head item is running.  If not, starts it.
 *
 * Arguments:
-*	hw	ptr to hfa384x_t
+*	hw	ptr to struct hfa384x
 *
 * Returns:
 *	nothing
@@ -2886,7 +2886,7 @@ static void unlocked_usbctlx_complete(hfa384x_t *hw, hfa384x_usbctlx_t *ctlx)
 * Call context:
 *	any
 ----------------------------------------------------------------*/
-static void hfa384x_usbctlxq_run(hfa384x_t *hw)
+static void hfa384x_usbctlxq_run(struct hfa384x *hw)
 {
 	unsigned long flags;
 
@@ -2905,12 +2905,12 @@ static void hfa384x_usbctlxq_run(hfa384x_t *hw)
 		goto unlock;
 
 	while (!list_empty(&hw->ctlxq.pending)) {
-		hfa384x_usbctlx_t *head;
+		struct hfa384x_usbctlx *head;
 		int result;
 
 		/* This is the first pending command */
 		head = list_entry(hw->ctlxq.pending.next,
-				  hfa384x_usbctlx_t, list);
+				  struct hfa384x_usbctlx, list);
 
 		/* We need to split this off to avoid a race condition */
 		list_move_tail(&head->list, &hw->ctlxq.active);
@@ -2988,9 +2988,9 @@ unlock:
 ----------------------------------------------------------------*/
 static void hfa384x_usbin_callback(struct urb *urb)
 {
-	wlandevice_t *wlandev = urb->context;
-	hfa384x_t *hw;
-	hfa384x_usbin_t *usbin = (hfa384x_usbin_t *)urb->transfer_buffer;
+	struct wlandevice *wlandev = urb->context;
+	struct hfa384x *hw;
+	union hfa384x_usbin *usbin = (union hfa384x_usbin *)urb->transfer_buffer;
 	struct sk_buff *skb = NULL;
 	int result;
 	int urb_status;
@@ -3154,7 +3154,7 @@ exit:
 * queue and our state updated accordingly.
 *
 * Arguments:
-*	hw		ptr to hfa384x_t
+*	hw		ptr to struct hfa384x
 *	usbin		ptr to USB IN packet
 *	urb_status	status of this Bulk-In URB
 *
@@ -3166,10 +3166,10 @@ exit:
 * Call context:
 *	interrupt
 ----------------------------------------------------------------*/
-static void hfa384x_usbin_ctlx(hfa384x_t *hw, hfa384x_usbin_t *usbin,
+static void hfa384x_usbin_ctlx(struct hfa384x *hw, union hfa384x_usbin *usbin,
 			       int urb_status)
 {
-	hfa384x_usbctlx_t *ctlx;
+	struct hfa384x_usbctlx *ctlx;
 	int run_queue = 0;
 	unsigned long flags;
 
@@ -3285,8 +3285,8 @@ unlock:
 * Call context:
 *	interrupt
 ----------------------------------------------------------------*/
-static void hfa384x_usbin_txcompl(wlandevice_t *wlandev,
-				  hfa384x_usbin_t *usbin)
+static void hfa384x_usbin_txcompl(struct wlandevice *wlandev,
+				  union hfa384x_usbin *usbin)
 {
 	u16 status;
 
@@ -3316,10 +3316,10 @@ static void hfa384x_usbin_txcompl(wlandevice_t *wlandev,
 * Call context:
 *	interrupt
 ----------------------------------------------------------------*/
-static void hfa384x_usbin_rx(wlandevice_t *wlandev, struct sk_buff *skb)
+static void hfa384x_usbin_rx(struct wlandevice *wlandev, struct sk_buff *skb)
 {
-	hfa384x_usbin_t *usbin = (hfa384x_usbin_t *)skb->data;
-	hfa384x_t *hw = wlandev->priv;
+	union hfa384x_usbin *usbin = (union hfa384x_usbin *)skb->data;
+	struct hfa384x *hw = wlandev->priv;
 	int hdrlen;
 	struct p80211_rxmeta *rxmeta;
 	u16 data_len;
@@ -3346,7 +3346,7 @@ static void hfa384x_usbin_rx(wlandevice_t *wlandev, struct sk_buff *skb)
 		hdrlen = p80211_headerlen(fc);
 
 		/* Pull off the descriptor */
-		skb_pull(skb, sizeof(hfa384x_rx_frame_t));
+		skb_pull(skb, sizeof(struct hfa384x_rx_frame));
 
 		/* Now shunt the header block up against the data block
 		 * with an "overlapping" copy
@@ -3416,17 +3416,17 @@ static void hfa384x_usbin_rx(wlandevice_t *wlandev, struct sk_buff *skb)
 * Call context:
 *	interrupt
 ----------------------------------------------------------------*/
-static void hfa384x_int_rxmonitor(wlandevice_t *wlandev,
-				  hfa384x_usb_rxfrm_t *rxfrm)
+static void hfa384x_int_rxmonitor(struct wlandevice *wlandev,
+				  struct hfa384x_usb_rxfrm *rxfrm)
 {
-	hfa384x_rx_frame_t *rxdesc = &(rxfrm->desc);
+	struct hfa384x_rx_frame *rxdesc = &(rxfrm->desc);
 	unsigned int hdrlen = 0;
 	unsigned int datalen = 0;
 	unsigned int skblen = 0;
 	u8 *datap;
 	u16 fc;
 	struct sk_buff *skb;
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 
 	/* Remember the status, time, and data_len fields are in host order */
 	/* Figure out how big the frame is */
@@ -3517,7 +3517,8 @@ static void hfa384x_int_rxmonitor(wlandevice_t *wlandev,
 * Call context:
 *	interrupt
 ----------------------------------------------------------------*/
-static void hfa384x_usbin_info(wlandevice_t *wlandev, hfa384x_usbin_t *usbin)
+static void hfa384x_usbin_info(struct wlandevice *wlandev,
+			       union hfa384x_usbin *usbin)
 {
 	usbin->infofrm.info.framelen =
 	    le16_to_cpu(usbin->infofrm.info.framelen);
@@ -3542,7 +3543,7 @@ static void hfa384x_usbin_info(wlandevice_t *wlandev, hfa384x_usbin_t *usbin)
 ----------------------------------------------------------------*/
 static void hfa384x_usbout_callback(struct urb *urb)
 {
-	wlandevice_t *wlandev = urb->context;
+	struct wlandevice *wlandev = urb->context;
 
 #ifdef DEBUG_USB
 	dbprint_urb(urb);
@@ -3556,7 +3557,7 @@ static void hfa384x_usbout_callback(struct urb *urb)
 
 		case -EPIPE:
 			{
-				hfa384x_t *hw = wlandev->priv;
+				struct hfa384x *hw = wlandev->priv;
 
 				netdev_warn(hw->wlandev->netdev,
 					    "%s tx pipe stalled: requesting reset\n",
@@ -3572,7 +3573,7 @@ static void hfa384x_usbout_callback(struct urb *urb)
 		case -ETIMEDOUT:
 		case -EILSEQ:
 			{
-				hfa384x_t *hw = wlandev->priv;
+				struct hfa384x *hw = wlandev->priv;
 
 				if (!test_and_set_bit
 				    (THROTTLE_TX, &hw->usb_flags) &&
@@ -3617,11 +3618,11 @@ static void hfa384x_usbout_callback(struct urb *urb)
 ----------------------------------------------------------------*/
 static void hfa384x_ctlxout_callback(struct urb *urb)
 {
-	hfa384x_t *hw = urb->context;
+	struct hfa384x *hw = urb->context;
 	int delete_resptimer = 0;
 	int timer_ok = 1;
 	int run_queue = 0;
-	hfa384x_usbctlx_t *ctlx;
+	struct hfa384x_usbctlx *ctlx;
 	unsigned long flags;
 
 	pr_debug("urb->status=%d\n", urb->status);
@@ -3736,7 +3737,7 @@ delresp:
 * URB containing a Prism2.x XXX_Request was never called.
 *
 * Arguments:
-*	data		a ptr to the hfa384x_t
+*	data		a ptr to the struct hfa384x
 *
 * Returns:
 *	nothing
@@ -3748,7 +3749,7 @@ delresp:
 ----------------------------------------------------------------*/
 static void hfa384x_usbctlx_reqtimerfn(unsigned long data)
 {
-	hfa384x_t *hw = (hfa384x_t *)data;
+	struct hfa384x *hw = (struct hfa384x *)data;
 	unsigned long flags;
 
 	spin_lock_irqsave(&hw->ctlxq.lock, flags);
@@ -3765,7 +3766,7 @@ static void hfa384x_usbctlx_reqtimerfn(unsigned long data)
 		 */
 		hw->ctlx_urb.transfer_flags |= URB_ASYNC_UNLINK;
 		if (usb_unlink_urb(&hw->ctlx_urb) == -EINPROGRESS) {
-			hfa384x_usbctlx_t *ctlx = get_active_ctlx(hw);
+			struct hfa384x_usbctlx *ctlx = get_active_ctlx(hw);
 
 			ctlx->state = CTLX_REQ_FAILED;
 
@@ -3794,7 +3795,7 @@ static void hfa384x_usbctlx_reqtimerfn(unsigned long data)
 * URB containing a Prism2.x XXX_Response was never called.
 *
 * Arguments:
-*	data		a ptr to the hfa384x_t
+*	data		a ptr to the struct hfa384x
 *
 * Returns:
 *	nothing
@@ -3806,7 +3807,7 @@ static void hfa384x_usbctlx_reqtimerfn(unsigned long data)
 ----------------------------------------------------------------*/
 static void hfa384x_usbctlx_resptimerfn(unsigned long data)
 {
-	hfa384x_t *hw = (hfa384x_t *)data;
+	struct hfa384x *hw = (struct hfa384x *)data;
 	unsigned long flags;
 
 	spin_lock_irqsave(&hw->ctlxq.lock, flags);
@@ -3817,7 +3818,7 @@ static void hfa384x_usbctlx_resptimerfn(unsigned long data)
 	 * adapter has been unplugged ...
 	 */
 	if (!list_empty(&hw->ctlxq.active)) {
-		hfa384x_usbctlx_t *ctlx = get_active_ctlx(hw);
+		struct hfa384x_usbctlx *ctlx = get_active_ctlx(hw);
 
 		if (unlocked_usbctlx_cancel_async(hw, ctlx) == 0) {
 			spin_unlock_irqrestore(&hw->ctlxq.lock, flags);
@@ -3845,7 +3846,7 @@ static void hfa384x_usbctlx_resptimerfn(unsigned long data)
 ----------------------------------------------------------------*/
 static void hfa384x_usb_throttlefn(unsigned long data)
 {
-	hfa384x_t *hw = (hfa384x_t *)data;
+	struct hfa384x *hw = (struct hfa384x *)data;
 	unsigned long flags;
 
 	spin_lock_irqsave(&hw->ctlxq.lock, flags);
@@ -3885,7 +3886,7 @@ static void hfa384x_usb_throttlefn(unsigned long data)
 * Call context:
 *	process or interrupt
 ----------------------------------------------------------------*/
-static int hfa384x_usbctlx_submit(hfa384x_t *hw, hfa384x_usbctlx_t *ctlx)
+static int hfa384x_usbctlx_submit(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx)
 {
 	unsigned long flags;
 
diff --git a/drivers/staging/wlan-ng/p80211conv.c b/drivers/staging/wlan-ng/p80211conv.c
index 6354036ffb42..0247cbc29145 100644
--- a/drivers/staging/wlan-ng/p80211conv.c
+++ b/drivers/staging/wlan-ng/p80211conv.c
@@ -104,7 +104,7 @@ static const u8 oui_8021h[] = { 0x00, 0x00, 0xf8 };
 *	May be called in interrupt or non-interrupt context
 *----------------------------------------------------------------
 */
-int skb_ether_to_p80211(wlandevice_t *wlandev, u32 ethconv,
+int skb_ether_to_p80211(struct wlandevice *wlandev, u32 ethconv,
 			struct sk_buff *skb, union p80211_hdr *p80211_hdr,
 			struct p80211_metawep *p80211_wep)
 {
@@ -232,7 +232,7 @@ int skb_ether_to_p80211(wlandevice_t *wlandev, u32 ethconv,
 }
 
 /* jkriegl: from orinoco, modified */
-static void orinoco_spy_gather(wlandevice_t *wlandev, char *mac,
+static void orinoco_spy_gather(struct wlandevice *wlandev, char *mac,
 			       struct p80211_rxmeta *rxmeta)
 {
 	int i;
@@ -274,10 +274,10 @@ static void orinoco_spy_gather(wlandevice_t *wlandev, char *mac,
 *	May be called in interrupt or non-interrupt context
 *----------------------------------------------------------------
 */
-int skb_p80211_to_ether(wlandevice_t *wlandev, u32 ethconv,
+int skb_p80211_to_ether(struct wlandevice *wlandev, u32 ethconv,
 			struct sk_buff *skb)
 {
-	netdevice_t *netdev = wlandev->netdev;
+	struct net_device *netdev = wlandev->netdev;
 	u16 fc;
 	unsigned int payload_length;
 	unsigned int payload_offset;
diff --git a/drivers/staging/wlan-ng/p80211metadef.h b/drivers/staging/wlan-ng/p80211metadef.h
index 0ccfba1294de..b0d3567ca0ad 100644
--- a/drivers/staging/wlan-ng/p80211metadef.h
+++ b/drivers/staging/wlan-ng/p80211metadef.h
@@ -155,22 +155,9 @@
 #define DIDmib_dot11smt_dot11WEPDefaultKeysTable \
 			(P80211DID_MKSECTION(1) | \
 			P80211DID_MKGROUP(4))
-#define DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey0 \
-			(P80211DID_MKSECTION(1) | \
-			P80211DID_MKGROUP(4) | \
-			P80211DID_MKITEM(1) | 0x0c000000)
-#define DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey1 \
-			(P80211DID_MKSECTION(1) | \
-			P80211DID_MKGROUP(4) | \
-			P80211DID_MKITEM(2) | 0x0c000000)
-#define DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey2 \
-			(P80211DID_MKSECTION(1) | \
-			P80211DID_MKGROUP(4) | \
-			P80211DID_MKITEM(3) | 0x0c000000)
-#define DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey3 \
-			(P80211DID_MKSECTION(1) | \
-			P80211DID_MKGROUP(4) | \
-			P80211DID_MKITEM(4) | 0x0c000000)
+#define DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(_i) \
+			(DIDmib_dot11smt_dot11WEPDefaultKeysTable | \
+			P80211DID_MKITEM(_i) | 0x0c000000)
 #define DIDmib_dot11smt_dot11PrivacyTable \
 			(P80211DID_MKSECTION(1) | \
 			P80211DID_MKGROUP(6))
diff --git a/drivers/staging/wlan-ng/p80211metastruct.h b/drivers/staging/wlan-ng/p80211metastruct.h
index c501162c3020..850d897fc163 100644
--- a/drivers/staging/wlan-ng/p80211metastruct.h
+++ b/drivers/staging/wlan-ng/p80211metastruct.h
@@ -51,221 +51,221 @@ struct p80211msg_dot11req_mibget {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_unk392_t mibattribute;
-	p80211item_uint32_t resultcode;
+	struct p80211item_unk392 mibattribute;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_dot11req_mibset {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_unk392_t mibattribute;
-	p80211item_uint32_t resultcode;
+	struct p80211item_unk392 mibattribute;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_dot11req_scan {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t bsstype;
-	p80211item_pstr6_t bssid;
+	struct p80211item_uint32 bsstype;
+	struct p80211item_pstr6 bssid;
 	u8 pad_0C[1];
-	p80211item_pstr32_t ssid;
+	struct p80211item_pstr32 ssid;
 	u8 pad_1D[3];
-	p80211item_uint32_t scantype;
-	p80211item_uint32_t probedelay;
-	p80211item_pstr14_t channellist;
+	struct p80211item_uint32 scantype;
+	struct p80211item_uint32 probedelay;
+	struct p80211item_pstr14 channellist;
 	u8 pad_2C[1];
-	p80211item_uint32_t minchanneltime;
-	p80211item_uint32_t maxchanneltime;
-	p80211item_uint32_t resultcode;
-	p80211item_uint32_t numbss;
-	p80211item_uint32_t append;
+	struct p80211item_uint32 minchanneltime;
+	struct p80211item_uint32 maxchanneltime;
+	struct p80211item_uint32 resultcode;
+	struct p80211item_uint32 numbss;
+	struct p80211item_uint32 append;
 } __packed;
 
 struct p80211msg_dot11req_scan_results {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t bssindex;
-	p80211item_uint32_t resultcode;
-	p80211item_uint32_t signal;
-	p80211item_uint32_t noise;
-	p80211item_pstr6_t bssid;
+	struct p80211item_uint32 bssindex;
+	struct p80211item_uint32 resultcode;
+	struct p80211item_uint32 signal;
+	struct p80211item_uint32 noise;
+	struct p80211item_pstr6 bssid;
 	u8 pad_3C[1];
-	p80211item_pstr32_t ssid;
+	struct p80211item_pstr32 ssid;
 	u8 pad_4D[3];
-	p80211item_uint32_t bsstype;
-	p80211item_uint32_t beaconperiod;
-	p80211item_uint32_t dtimperiod;
-	p80211item_uint32_t timestamp;
-	p80211item_uint32_t localtime;
-	p80211item_uint32_t fhdwelltime;
-	p80211item_uint32_t fhhopset;
-	p80211item_uint32_t fhhoppattern;
-	p80211item_uint32_t fhhopindex;
-	p80211item_uint32_t dschannel;
-	p80211item_uint32_t cfpcount;
-	p80211item_uint32_t cfpperiod;
-	p80211item_uint32_t cfpmaxduration;
-	p80211item_uint32_t cfpdurremaining;
-	p80211item_uint32_t ibssatimwindow;
-	p80211item_uint32_t cfpollable;
-	p80211item_uint32_t cfpollreq;
-	p80211item_uint32_t privacy;
-	p80211item_uint32_t capinfo;
-	p80211item_uint32_t basicrate1;
-	p80211item_uint32_t basicrate2;
-	p80211item_uint32_t basicrate3;
-	p80211item_uint32_t basicrate4;
-	p80211item_uint32_t basicrate5;
-	p80211item_uint32_t basicrate6;
-	p80211item_uint32_t basicrate7;
-	p80211item_uint32_t basicrate8;
-	p80211item_uint32_t supprate1;
-	p80211item_uint32_t supprate2;
-	p80211item_uint32_t supprate3;
-	p80211item_uint32_t supprate4;
-	p80211item_uint32_t supprate5;
-	p80211item_uint32_t supprate6;
-	p80211item_uint32_t supprate7;
-	p80211item_uint32_t supprate8;
+	struct p80211item_uint32 bsstype;
+	struct p80211item_uint32 beaconperiod;
+	struct p80211item_uint32 dtimperiod;
+	struct p80211item_uint32 timestamp;
+	struct p80211item_uint32 localtime;
+	struct p80211item_uint32 fhdwelltime;
+	struct p80211item_uint32 fhhopset;
+	struct p80211item_uint32 fhhoppattern;
+	struct p80211item_uint32 fhhopindex;
+	struct p80211item_uint32 dschannel;
+	struct p80211item_uint32 cfpcount;
+	struct p80211item_uint32 cfpperiod;
+	struct p80211item_uint32 cfpmaxduration;
+	struct p80211item_uint32 cfpdurremaining;
+	struct p80211item_uint32 ibssatimwindow;
+	struct p80211item_uint32 cfpollable;
+	struct p80211item_uint32 cfpollreq;
+	struct p80211item_uint32 privacy;
+	struct p80211item_uint32 capinfo;
+	struct p80211item_uint32 basicrate1;
+	struct p80211item_uint32 basicrate2;
+	struct p80211item_uint32 basicrate3;
+	struct p80211item_uint32 basicrate4;
+	struct p80211item_uint32 basicrate5;
+	struct p80211item_uint32 basicrate6;
+	struct p80211item_uint32 basicrate7;
+	struct p80211item_uint32 basicrate8;
+	struct p80211item_uint32 supprate1;
+	struct p80211item_uint32 supprate2;
+	struct p80211item_uint32 supprate3;
+	struct p80211item_uint32 supprate4;
+	struct p80211item_uint32 supprate5;
+	struct p80211item_uint32 supprate6;
+	struct p80211item_uint32 supprate7;
+	struct p80211item_uint32 supprate8;
 } __packed;
 
 struct p80211msg_dot11req_start {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_pstr32_t ssid;
+	struct p80211item_pstr32 ssid;
 	u8 pad_12D[3];
-	p80211item_uint32_t bsstype;
-	p80211item_uint32_t beaconperiod;
-	p80211item_uint32_t dtimperiod;
-	p80211item_uint32_t cfpperiod;
-	p80211item_uint32_t cfpmaxduration;
-	p80211item_uint32_t fhdwelltime;
-	p80211item_uint32_t fhhopset;
-	p80211item_uint32_t fhhoppattern;
-	p80211item_uint32_t dschannel;
-	p80211item_uint32_t ibssatimwindow;
-	p80211item_uint32_t probedelay;
-	p80211item_uint32_t cfpollable;
-	p80211item_uint32_t cfpollreq;
-	p80211item_uint32_t basicrate1;
-	p80211item_uint32_t basicrate2;
-	p80211item_uint32_t basicrate3;
-	p80211item_uint32_t basicrate4;
-	p80211item_uint32_t basicrate5;
-	p80211item_uint32_t basicrate6;
-	p80211item_uint32_t basicrate7;
-	p80211item_uint32_t basicrate8;
-	p80211item_uint32_t operationalrate1;
-	p80211item_uint32_t operationalrate2;
-	p80211item_uint32_t operationalrate3;
-	p80211item_uint32_t operationalrate4;
-	p80211item_uint32_t operationalrate5;
-	p80211item_uint32_t operationalrate6;
-	p80211item_uint32_t operationalrate7;
-	p80211item_uint32_t operationalrate8;
-	p80211item_uint32_t resultcode;
+	struct p80211item_uint32 bsstype;
+	struct p80211item_uint32 beaconperiod;
+	struct p80211item_uint32 dtimperiod;
+	struct p80211item_uint32 cfpperiod;
+	struct p80211item_uint32 cfpmaxduration;
+	struct p80211item_uint32 fhdwelltime;
+	struct p80211item_uint32 fhhopset;
+	struct p80211item_uint32 fhhoppattern;
+	struct p80211item_uint32 dschannel;
+	struct p80211item_uint32 ibssatimwindow;
+	struct p80211item_uint32 probedelay;
+	struct p80211item_uint32 cfpollable;
+	struct p80211item_uint32 cfpollreq;
+	struct p80211item_uint32 basicrate1;
+	struct p80211item_uint32 basicrate2;
+	struct p80211item_uint32 basicrate3;
+	struct p80211item_uint32 basicrate4;
+	struct p80211item_uint32 basicrate5;
+	struct p80211item_uint32 basicrate6;
+	struct p80211item_uint32 basicrate7;
+	struct p80211item_uint32 basicrate8;
+	struct p80211item_uint32 operationalrate1;
+	struct p80211item_uint32 operationalrate2;
+	struct p80211item_uint32 operationalrate3;
+	struct p80211item_uint32 operationalrate4;
+	struct p80211item_uint32 operationalrate5;
+	struct p80211item_uint32 operationalrate6;
+	struct p80211item_uint32 operationalrate7;
+	struct p80211item_uint32 operationalrate8;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_lnxreq_ifstate {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t ifstate;
-	p80211item_uint32_t resultcode;
+	struct p80211item_uint32 ifstate;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_lnxreq_wlansniff {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t enable;
-	p80211item_uint32_t channel;
-	p80211item_uint32_t prismheader;
-	p80211item_uint32_t wlanheader;
-	p80211item_uint32_t keepwepflags;
-	p80211item_uint32_t stripfcs;
-	p80211item_uint32_t packet_trunc;
-	p80211item_uint32_t resultcode;
+	struct p80211item_uint32 enable;
+	struct p80211item_uint32 channel;
+	struct p80211item_uint32 prismheader;
+	struct p80211item_uint32 wlanheader;
+	struct p80211item_uint32 keepwepflags;
+	struct p80211item_uint32 stripfcs;
+	struct p80211item_uint32 packet_trunc;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_lnxreq_hostwep {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t resultcode;
-	p80211item_uint32_t decrypt;
-	p80211item_uint32_t encrypt;
+	struct p80211item_uint32 resultcode;
+	struct p80211item_uint32 decrypt;
+	struct p80211item_uint32 encrypt;
 } __packed;
 
 struct p80211msg_lnxreq_commsquality {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t resultcode;
-	p80211item_uint32_t dbm;
-	p80211item_uint32_t link;
-	p80211item_uint32_t level;
-	p80211item_uint32_t noise;
-	p80211item_uint32_t txrate;
+	struct p80211item_uint32 resultcode;
+	struct p80211item_uint32 dbm;
+	struct p80211item_uint32 link;
+	struct p80211item_uint32 level;
+	struct p80211item_uint32 noise;
+	struct p80211item_uint32 txrate;
 } __packed;
 
 struct p80211msg_lnxreq_autojoin {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_pstr32_t ssid;
+	struct p80211item_pstr32 ssid;
 	u8 pad_19D[3];
-	p80211item_uint32_t authtype;
-	p80211item_uint32_t resultcode;
+	struct p80211item_uint32 authtype;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_p2req_readpda {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_unk1024_t pda;
-	p80211item_uint32_t resultcode;
+	struct p80211item_unk1024 pda;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_p2req_ramdl_state {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t enable;
-	p80211item_uint32_t exeaddr;
-	p80211item_uint32_t resultcode;
+	struct p80211item_uint32 enable;
+	struct p80211item_uint32 exeaddr;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_p2req_ramdl_write {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t addr;
-	p80211item_uint32_t len;
-	p80211item_unk4096_t data;
-	p80211item_uint32_t resultcode;
+	struct p80211item_uint32 addr;
+	struct p80211item_uint32 len;
+	struct p80211item_unk4096 data;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_p2req_flashdl_state {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t enable;
-	p80211item_uint32_t resultcode;
+	struct p80211item_uint32 enable;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 struct p80211msg_p2req_flashdl_write {
 	u32 msgcode;
 	u32 msglen;
 	u8 devname[WLAN_DEVNAMELEN_MAX];
-	p80211item_uint32_t addr;
-	p80211item_uint32_t len;
-	p80211item_unk4096_t data;
-	p80211item_uint32_t resultcode;
+	struct p80211item_uint32 addr;
+	struct p80211item_uint32 len;
+	struct p80211item_unk4096 data;
+	struct p80211item_uint32 resultcode;
 } __packed;
 
 #endif
diff --git a/drivers/staging/wlan-ng/p80211netdev.c b/drivers/staging/wlan-ng/p80211netdev.c
index 90cc8cdcf969..825a63a7c0e3 100644
--- a/drivers/staging/wlan-ng/p80211netdev.c
+++ b/drivers/staging/wlan-ng/p80211netdev.c
@@ -91,17 +91,17 @@
 #include "cfg80211.c"
 
 /* netdevice method functions */
-static int p80211knetdev_init(netdevice_t *netdev);
-static int p80211knetdev_open(netdevice_t *netdev);
-static int p80211knetdev_stop(netdevice_t *netdev);
+static int p80211knetdev_init(struct net_device *netdev);
+static int p80211knetdev_open(struct net_device *netdev);
+static int p80211knetdev_stop(struct net_device *netdev);
 static int p80211knetdev_hard_start_xmit(struct sk_buff *skb,
-					 netdevice_t *netdev);
-static void p80211knetdev_set_multicast_list(netdevice_t *dev);
-static int p80211knetdev_do_ioctl(netdevice_t *dev, struct ifreq *ifr,
+					 struct net_device *netdev);
+static void p80211knetdev_set_multicast_list(struct net_device *dev);
+static int p80211knetdev_do_ioctl(struct net_device *dev, struct ifreq *ifr,
 				  int cmd);
-static int p80211knetdev_set_mac_address(netdevice_t *dev, void *addr);
-static void p80211knetdev_tx_timeout(netdevice_t *netdev);
-static int p80211_rx_typedrop(wlandevice_t *wlandev, u16 fc);
+static int p80211knetdev_set_mac_address(struct net_device *dev, void *addr);
+static void p80211knetdev_tx_timeout(struct net_device *netdev);
+static int p80211_rx_typedrop(struct wlandevice *wlandev, u16 fc);
 
 int wlan_watchdog = 5000;
 module_param(wlan_watchdog, int, 0644);
@@ -123,7 +123,7 @@ MODULE_PARM_DESC(wlan_wext_write, "enable write wireless extensions");
 * Returns:
 *	nothing
 ----------------------------------------------------------------*/
-static int p80211knetdev_init(netdevice_t *netdev)
+static int p80211knetdev_init(struct net_device *netdev)
 {
 	/* Called in response to register_netdev */
 	/* This is usually the probe function, but the probe has */
@@ -146,10 +146,10 @@ static int p80211knetdev_init(netdevice_t *netdev)
 * Returns:
 *	zero on success, non-zero otherwise
 ----------------------------------------------------------------*/
-static int p80211knetdev_open(netdevice_t *netdev)
+static int p80211knetdev_open(struct net_device *netdev)
 {
 	int result = 0;		/* success */
-	wlandevice_t *wlandev = netdev->ml_priv;
+	struct wlandevice *wlandev = netdev->ml_priv;
 
 	/* Check to make sure the MSD is running */
 	if (wlandev->msdstate != WLAN_MSD_RUNNING)
@@ -181,10 +181,10 @@ static int p80211knetdev_open(netdevice_t *netdev)
 * Returns:
 *	zero on success, non-zero otherwise
 ----------------------------------------------------------------*/
-static int p80211knetdev_stop(netdevice_t *netdev)
+static int p80211knetdev_stop(struct net_device *netdev)
 {
 	int result = 0;
-	wlandevice_t *wlandev = netdev->ml_priv;
+	struct wlandevice *wlandev = netdev->ml_priv;
 
 	if (wlandev->close)
 		result = wlandev->close(wlandev);
@@ -208,7 +208,7 @@ static int p80211knetdev_stop(netdevice_t *netdev)
 * Side effects:
 *
 ----------------------------------------------------------------*/
-void p80211netdev_rx(wlandevice_t *wlandev, struct sk_buff *skb)
+void p80211netdev_rx(struct wlandevice *wlandev, struct sk_buff *skb)
 {
 	/* Enqueue for post-irq processing */
 	skb_queue_tail(&wlandev->nsd_rxq, skb);
@@ -227,11 +227,11 @@ void p80211netdev_rx(wlandevice_t *wlandev, struct sk_buff *skb)
  *	    CONV_TO_ETHER_FAILED if conversion failed
  *	    CONV_TO_ETHER_SKIPPED if frame is ignored
  */
-static int p80211_convert_to_ether(wlandevice_t *wlandev, struct sk_buff *skb)
+static int p80211_convert_to_ether(struct wlandevice *wlandev, struct sk_buff *skb)
 {
 	struct p80211_hdr_a3 *hdr;
 
-	hdr = (struct p80211_hdr_a3 *) skb->data;
+	hdr = (struct p80211_hdr_a3 *)skb->data;
 	if (p80211_rx_typedrop(wlandev, hdr->fc))
 		return CONV_TO_ETHER_SKIPPED;
 
@@ -265,9 +265,9 @@ static int p80211_convert_to_ether(wlandevice_t *wlandev, struct sk_buff *skb)
  */
 static void p80211netdev_rx_bh(unsigned long arg)
 {
-	wlandevice_t *wlandev = (wlandevice_t *) arg;
+	struct wlandevice *wlandev = (struct wlandevice *)arg;
 	struct sk_buff *skb = NULL;
-	netdevice_t *dev = wlandev->netdev;
+	struct net_device *dev = wlandev->netdev;
 
 	/* Let's empty our our queue */
 	while ((skb = skb_dequeue(&wlandev->nsd_rxq))) {
@@ -318,11 +318,11 @@ static void p80211netdev_rx_bh(unsigned long arg)
 *	zero on success, non-zero on failure.
 ----------------------------------------------------------------*/
 static int p80211knetdev_hard_start_xmit(struct sk_buff *skb,
-					 netdevice_t *netdev)
+					 struct net_device *netdev)
 {
 	int result = 0;
 	int txresult = -1;
-	wlandevice_t *wlandev = netdev->ml_priv;
+	struct wlandevice *wlandev = netdev->ml_priv;
 	union p80211_hdr p80211_hdr;
 	struct p80211_metawep p80211_wep;
 
@@ -446,9 +446,9 @@ failed:
 * Returns:
 *	nothing
 ----------------------------------------------------------------*/
-static void p80211knetdev_set_multicast_list(netdevice_t *dev)
+static void p80211knetdev_set_multicast_list(struct net_device *dev)
 {
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct wlandevice *wlandev = dev->ml_priv;
 
 	/* TODO:  real multicast support as well */
 
@@ -459,7 +459,7 @@ static void p80211knetdev_set_multicast_list(netdevice_t *dev)
 
 #ifdef SIOCETHTOOL
 
-static int p80211netdev_ethtool(wlandevice_t *wlandev, void __user *useraddr)
+static int p80211netdev_ethtool(struct wlandevice *wlandev, void __user *useraddr)
 {
 	u32 ethcmd;
 	struct ethtool_drvinfo info;
@@ -531,11 +531,11 @@ static int p80211netdev_ethtool(wlandevice_t *wlandev, void __user *useraddr)
 *	Process thread (ioctl caller).  TODO: SMP support may require
 *	locks.
 ----------------------------------------------------------------*/
-static int p80211knetdev_do_ioctl(netdevice_t *dev, struct ifreq *ifr, int cmd)
+static int p80211knetdev_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 {
 	int result = 0;
-	struct p80211ioctl_req *req = (struct p80211ioctl_req *) ifr;
-	wlandevice_t *wlandev = dev->ml_priv;
+	struct p80211ioctl_req *req = (struct p80211ioctl_req *)ifr;
+	struct wlandevice *wlandev = dev->ml_priv;
 	u8 *msgbuf;
 
 	netdev_dbg(dev, "rx'd ioctl, cmd=%d, len=%d\n", cmd, req->len);
@@ -610,13 +610,13 @@ bail:
 *
 * by: Collin R. Mulliner <collin@mulliner.org>
 ----------------------------------------------------------------*/
-static int p80211knetdev_set_mac_address(netdevice_t *dev, void *addr)
+static int p80211knetdev_set_mac_address(struct net_device *dev, void *addr)
 {
 	struct sockaddr *new_addr = addr;
 	struct p80211msg_dot11req_mibset dot11req;
-	p80211item_unk392_t *mibattr;
-	p80211item_pstr6_t *macaddr;
-	p80211item_uint32_t *resultcode;
+	struct p80211item_unk392 *mibattr;
+	struct p80211item_pstr6 *macaddr;
+	struct p80211item_uint32 *resultcode;
 	int result;
 
 	/* If we're running, we don't allow MAC address changes */
@@ -625,7 +625,7 @@ static int p80211knetdev_set_mac_address(netdevice_t *dev, void *addr)
 
 	/* Set up some convenience pointers. */
 	mibattr = &dot11req.mibattribute;
-	macaddr = (p80211item_pstr6_t *) &mibattr->data;
+	macaddr = (struct p80211item_pstr6 *)&mibattr->data;
 	resultcode = &dot11req.resultcode;
 
 	/* Set up a dot11req_mibset */
@@ -633,7 +633,7 @@ static int p80211knetdev_set_mac_address(netdevice_t *dev, void *addr)
 	dot11req.msgcode = DIDmsg_dot11req_mibset;
 	dot11req.msglen = sizeof(struct p80211msg_dot11req_mibset);
 	memcpy(dot11req.devname,
-	       ((wlandevice_t *) dev->ml_priv)->name, WLAN_DEVNAMELEN_MAX - 1);
+	       ((struct wlandevice *)dev->ml_priv)->name, WLAN_DEVNAMELEN_MAX - 1);
 
 	/* Set up the mibattribute argument */
 	mibattr->did = DIDmsg_dot11req_mibset_mibattribute;
@@ -653,7 +653,7 @@ static int p80211knetdev_set_mac_address(netdevice_t *dev, void *addr)
 	resultcode->data = 0;
 
 	/* now fire the request */
-	result = p80211req_dorequest(dev->ml_priv, (u8 *) &dot11req);
+	result = p80211req_dorequest(dev->ml_priv, (u8 *)&dot11req);
 
 	/* If the request wasn't successful, report an error and don't
 	 * change the netdev address
@@ -669,7 +669,7 @@ static int p80211knetdev_set_mac_address(netdevice_t *dev, void *addr)
 	return result;
 }
 
-static int wlan_change_mtu(netdevice_t *dev, int new_mtu)
+static int wlan_change_mtu(struct net_device *dev, int new_mtu)
 {
 	/* 2312 is max 802.11 payload, 20 is overhead, (ether + llc +snap)
 	   and another 8 for wep. */
@@ -717,10 +717,10 @@ static const struct net_device_ops p80211_netdev_ops = {
 *	compiled drivers, this function will be called in the
 *	context of the kernel startup code.
 ----------------------------------------------------------------*/
-int wlan_setup(wlandevice_t *wlandev, struct device *physdev)
+int wlan_setup(struct wlandevice *wlandev, struct device *physdev)
 {
 	int result = 0;
-	netdevice_t *netdev;
+	struct net_device *netdev;
 	struct wiphy *wiphy;
 	struct wireless_dev *wdev;
 
@@ -783,7 +783,7 @@ int wlan_setup(wlandevice_t *wlandev, struct device *physdev)
 *	compiled drivers, this function will be called in the
 *	context of the kernel startup code.
 ----------------------------------------------------------------*/
-void wlan_unsetup(wlandevice_t *wlandev)
+void wlan_unsetup(struct wlandevice *wlandev)
 {
 	struct wireless_dev *wdev;
 
@@ -817,7 +817,7 @@ void wlan_unsetup(wlandevice_t *wlandev)
 * Call Context:
 *	Can be either interrupt or not.
 ----------------------------------------------------------------*/
-int register_wlandev(wlandevice_t *wlandev)
+int register_wlandev(struct wlandevice *wlandev)
 {
 	return register_netdev(wlandev->netdev);
 }
@@ -839,7 +839,7 @@ int register_wlandev(wlandevice_t *wlandev)
 * Call Context:
 *	Can be either interrupt or not.
 ----------------------------------------------------------------*/
-int unregister_wlandev(wlandevice_t *wlandev)
+int unregister_wlandev(struct wlandevice *wlandev)
 {
 	struct sk_buff *skb;
 
@@ -882,7 +882,7 @@ int unregister_wlandev(wlandevice_t *wlandev)
 * Call context:
 *	Usually interrupt.
 ----------------------------------------------------------------*/
-void p80211netdev_hwremoved(wlandevice_t *wlandev)
+void p80211netdev_hwremoved(struct wlandevice *wlandev)
 {
 	wlandev->hwremoved = 1;
 	if (wlandev->state == WLAN_DEVICE_OPEN)
@@ -912,7 +912,7 @@ void p80211netdev_hwremoved(wlandevice_t *wlandev)
 * Call context:
 *	interrupt
 ----------------------------------------------------------------*/
-static int p80211_rx_typedrop(wlandevice_t *wlandev, u16 fc)
+static int p80211_rx_typedrop(struct wlandevice *wlandev, u16 fc)
 {
 	u16 ftype;
 	u16 fstype;
@@ -1071,9 +1071,9 @@ static int p80211_rx_typedrop(wlandevice_t *wlandev, u16 fc)
 	return drop;
 }
 
-static void p80211knetdev_tx_timeout(netdevice_t *netdev)
+static void p80211knetdev_tx_timeout(struct net_device *netdev)
 {
-	wlandevice_t *wlandev = netdev->ml_priv;
+	struct wlandevice *wlandev = netdev->ml_priv;
 
 	if (wlandev->tx_timeout) {
 		wlandev->tx_timeout(wlandev);
diff --git a/drivers/staging/wlan-ng/p80211netdev.h b/drivers/staging/wlan-ng/p80211netdev.h
index 820a0e20a941..1e6a774fc7c5 100644
--- a/drivers/staging/wlan-ng/p80211netdev.h
+++ b/drivers/staging/wlan-ng/p80211netdev.h
@@ -57,9 +57,6 @@
 #include <linux/wireless.h>
 #include <linux/netdevice.h>
 
-#undef netdevice_t
-typedef struct net_device netdevice_t;
-
 #define WLAN_RELEASE	"0.3.0-staging"
 
 #define WLAN_DEVICE_CLOSED	0
@@ -101,7 +98,7 @@ typedef struct net_device netdevice_t;
 #define P80211_NSDCAP_NOSCAN                0x200 /* nsd can scan */
 
 /* Received frame statistics */
-typedef struct p80211_frmrx_t {
+struct p80211_frmrx {
 	u32 mgmt;
 	u32 assocreq;
 	u32 assocresp;
@@ -135,10 +132,10 @@ typedef struct p80211_frmrx_t {
 	u32 data_unknown;
 	u32 decrypt;
 	u32 decrypt_err;
-} p80211_frmrx_t;
+};
 
 /* called by /proc/net/wireless */
-struct iw_statistics *p80211wext_get_wireless_stats(netdevice_t *dev);
+struct iw_statistics *p80211wext_get_wireless_stats(struct net_device *dev);
 /* wireless extensions' ioctls */
 extern struct iw_handler_def p80211wext_handler_def;
 
@@ -157,7 +154,7 @@ extern int wlan_watchdog;
 extern int wlan_wext_write;
 
 /* WLAN device type */
-typedef struct wlandevice {
+struct wlandevice {
 	void *priv;		/* private data for MSD */
 
 	/* Subsystem State */
@@ -186,12 +183,12 @@ typedef struct wlandevice {
 			struct p80211_metawep *p80211_wep);
 	int (*mlmerequest)(struct wlandevice *wlandev, struct p80211msg *msg);
 	int (*set_multicast_list)(struct wlandevice *wlandev,
-				   netdevice_t *dev);
+				   struct net_device *dev);
 	void (*tx_timeout)(struct wlandevice *wlandev);
 
 	/* 802.11 State */
 	u8 bssid[WLAN_BSSID_LEN];
-	p80211pstr32_t ssid;
+	struct p80211pstr32 ssid;
 	u32 macmode;
 	int linkstatus;
 
@@ -206,7 +203,7 @@ typedef struct wlandevice {
 	/* netlink socket */
 	/* queue for indications waiting for cmd completion */
 	/* Linux netdevice and support */
-	netdevice_t *netdev;	/* ptr to linux netdevice */
+	struct net_device *netdev;	/* ptr to linux netdevice */
 
 	/* Rx bottom half */
 	struct tasklet_struct rx_bh;
@@ -214,7 +211,7 @@ typedef struct wlandevice {
 	struct sk_buff_head nsd_rxq;
 
 	/* 802.11 device statistics */
-	struct p80211_frmrx_t rx;
+	struct p80211_frmrx rx;
 
 	struct iw_statistics wstats;
 
@@ -222,19 +219,19 @@ typedef struct wlandevice {
 	u8 spy_number;
 	char spy_address[IW_MAX_SPY][ETH_ALEN];
 	struct iw_quality spy_stat[IW_MAX_SPY];
-} wlandevice_t;
+};
 
 /* WEP stuff */
-int wep_change_key(wlandevice_t *wlandev, int keynum, u8 *key, int keylen);
-int wep_decrypt(wlandevice_t *wlandev, u8 *buf, u32 len, int key_override,
-		u8 *iv, u8 *icv);
-int wep_encrypt(wlandevice_t *wlandev, u8 *buf, u8 *dst, u32 len, int keynum,
+int wep_change_key(struct wlandevice *wlandev, int keynum, u8 *key, int keylen);
+int wep_decrypt(struct wlandevice *wlandev, u8 *buf, u32 len, int key_override,
 		u8 *iv, u8 *icv);
-
-int wlan_setup(wlandevice_t *wlandev, struct device *physdev);
-void wlan_unsetup(wlandevice_t *wlandev);
-int register_wlandev(wlandevice_t *wlandev);
-int unregister_wlandev(wlandevice_t *wlandev);
-void p80211netdev_rx(wlandevice_t *wlandev, struct sk_buff *skb);
-void p80211netdev_hwremoved(wlandevice_t *wlandev);
+int wep_encrypt(struct wlandevice *wlandev, u8 *buf, u8 *dst, u32 len,
+		int keynum, u8 *iv, u8 *icv);
+
+int wlan_setup(struct wlandevice *wlandev, struct device *physdev);
+void wlan_unsetup(struct wlandevice *wlandev);
+int register_wlandev(struct wlandevice *wlandev);
+int unregister_wlandev(struct wlandevice *wlandev);
+void p80211netdev_rx(struct wlandevice *wlandev, struct sk_buff *skb);
+void p80211netdev_hwremoved(struct wlandevice *wlandev);
 #endif
diff --git a/drivers/staging/wlan-ng/p80211req.c b/drivers/staging/wlan-ng/p80211req.c
index 4b84b568f6ca..d43e85b5d49b 100644
--- a/drivers/staging/wlan-ng/p80211req.c
+++ b/drivers/staging/wlan-ng/p80211req.c
@@ -72,11 +72,26 @@
 #include "p80211metastruct.h"
 #include "p80211req.h"
 
-static void p80211req_handlemsg(wlandevice_t *wlandev, struct p80211msg *msg);
-static void p80211req_mibset_mibget(wlandevice_t *wlandev,
+static void p80211req_handlemsg(struct wlandevice *wlandev, struct p80211msg *msg);
+static void p80211req_mibset_mibget(struct wlandevice *wlandev,
 				   struct p80211msg_dot11req_mibget *mib_msg,
 				   int isget);
 
+static void p80211req_handle_action(struct wlandevice *wlandev, u32 *data,
+				    int isget, u32 flag)
+{
+	if (isget) {
+		if (wlandev->hostwep & flag)
+			*data = P80211ENUM_truth_true;
+		else
+			*data = P80211ENUM_truth_false;
+	} else {
+		wlandev->hostwep &= ~flag;
+		if (*data == P80211ENUM_truth_true)
+			wlandev->hostwep |= flag;
+	}
+}
+
 /*----------------------------------------------------------------
 * p80211req_dorequest
 *
@@ -93,9 +108,9 @@ static void p80211req_mibset_mibget(wlandevice_t *wlandev,
 *	Potentially blocks the caller, so it's a good idea to
 *	not call this function from an interrupt context.
 ----------------------------------------------------------------*/
-int p80211req_dorequest(wlandevice_t *wlandev, u8 *msgbuf)
+int p80211req_dorequest(struct wlandevice *wlandev, u8 *msgbuf)
 {
-	struct p80211msg *msg = (struct p80211msg *) msgbuf;
+	struct p80211msg *msg = (struct p80211msg *)msgbuf;
 
 	/* Check to make sure the MSD is running */
 	if (!((wlandev->msdstate == WLAN_MSD_HWPRESENT &&
@@ -149,13 +164,13 @@ int p80211req_dorequest(wlandevice_t *wlandev, u8 *msgbuf)
 * Call context:
 *	Process thread
 ----------------------------------------------------------------*/
-static void p80211req_handlemsg(wlandevice_t *wlandev, struct p80211msg *msg)
+static void p80211req_handlemsg(struct wlandevice *wlandev, struct p80211msg *msg)
 {
 	switch (msg->msgcode) {
 
 	case DIDmsg_lnxreq_hostwep:{
 		struct p80211msg_lnxreq_hostwep *req =
-			(struct p80211msg_lnxreq_hostwep *) msg;
+			(struct p80211msg_lnxreq_hostwep *)msg;
 		wlandev->hostwep &=
 				~(HOSTWEP_DECRYPT | HOSTWEP_ENCRYPT);
 		if (req->decrypt.data == P80211ENUM_truth_true)
@@ -169,44 +184,34 @@ static void p80211req_handlemsg(wlandevice_t *wlandev, struct p80211msg *msg)
 	case DIDmsg_dot11req_mibset:{
 		int isget = (msg->msgcode == DIDmsg_dot11req_mibget);
 		struct p80211msg_dot11req_mibget *mib_msg =
-			(struct p80211msg_dot11req_mibget *) msg;
+			(struct p80211msg_dot11req_mibget *)msg;
 		p80211req_mibset_mibget(wlandev, mib_msg, isget);
 	break;
 	}
 	}			/* switch msg->msgcode */
 }
 
-static void p80211req_mibset_mibget(wlandevice_t *wlandev,
+static void p80211req_mibset_mibget(struct wlandevice *wlandev,
 				   struct p80211msg_dot11req_mibget *mib_msg,
 				   int isget)
 {
-	p80211itemd_t *mibitem = (p80211itemd_t *) mib_msg->mibattribute.data;
-	p80211pstrd_t *pstr = (p80211pstrd_t *) mibitem->data;
-	u8 *key = mibitem->data + sizeof(p80211pstrd_t);
+	struct p80211itemd *mibitem = (struct p80211itemd *)mib_msg->mibattribute.data;
+	struct p80211pstrd *pstr = (struct p80211pstrd *)mibitem->data;
+	u8 *key = mibitem->data + sizeof(struct p80211pstrd);
 
 	switch (mibitem->did) {
-	case DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey0:{
-		if (!isget)
-			wep_change_key(wlandev, 0, key, pstr->len);
-	break;
-	}
-	case DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey1:{
-		if (!isget)
-			wep_change_key(wlandev, 1, key, pstr->len);
-	break;
-	}
-	case DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey2:{
-		if (!isget)
-			wep_change_key(wlandev, 2, key, pstr->len);
-	break;
-	}
-	case DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey3:{
+	case DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(1):
+	case DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(2):
+	case DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(3):
+	case DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(4):
 		if (!isget)
-			wep_change_key(wlandev, 3, key, pstr->len);
-	break;
-	}
+			wep_change_key(wlandev,
+				       P80211DID_ITEM(mibitem->did) - 1,
+				       key, pstr->len);
+		break;
+
 	case DIDmib_dot11smt_dot11PrivacyTable_dot11WEPDefaultKeyID:{
-		u32 *data = (u32 *) mibitem->data;
+		u32 *data = (u32 *)mibitem->data;
 
 		if (isget) {
 			*data = wlandev->hostwep & HOSTWEP_DEFAULTKEY_MASK;
@@ -217,33 +222,17 @@ static void p80211req_mibset_mibget(wlandevice_t *wlandev,
 	break;
 	}
 	case DIDmib_dot11smt_dot11PrivacyTable_dot11PrivacyInvoked:{
-		u32 *data = (u32 *) mibitem->data;
+		u32 *data = (u32 *)mibitem->data;
 
-		if (isget) {
-			if (wlandev->hostwep & HOSTWEP_PRIVACYINVOKED)
-				*data = P80211ENUM_truth_true;
-			else
-				*data = P80211ENUM_truth_false;
-		} else {
-			wlandev->hostwep &= ~(HOSTWEP_PRIVACYINVOKED);
-			if (*data == P80211ENUM_truth_true)
-				wlandev->hostwep |= HOSTWEP_PRIVACYINVOKED;
-		}
+		p80211req_handle_action(wlandev, data, isget,
+					HOSTWEP_PRIVACYINVOKED);
 	break;
 	}
 	case DIDmib_dot11smt_dot11PrivacyTable_dot11ExcludeUnencrypted:{
-		u32 *data = (u32 *) mibitem->data;
+		u32 *data = (u32 *)mibitem->data;
 
-		if (isget) {
-			if (wlandev->hostwep & HOSTWEP_EXCLUDEUNENCRYPTED)
-				*data = P80211ENUM_truth_true;
-			else
-				*data = P80211ENUM_truth_false;
-		} else {
-			wlandev->hostwep &= ~(HOSTWEP_EXCLUDEUNENCRYPTED);
-			if (*data == P80211ENUM_truth_true)
-				wlandev->hostwep |= HOSTWEP_EXCLUDEUNENCRYPTED;
-		}
+		p80211req_handle_action(wlandev, data, isget,
+					HOSTWEP_EXCLUDEUNENCRYPTED);
 	break;
 	}
 	}
diff --git a/drivers/staging/wlan-ng/p80211req.h b/drivers/staging/wlan-ng/p80211req.h
index a95a45a6814d..8d3054c22a05 100644
--- a/drivers/staging/wlan-ng/p80211req.h
+++ b/drivers/staging/wlan-ng/p80211req.h
@@ -48,6 +48,6 @@
 #ifndef _LINUX_P80211REQ_H
 #define _LINUX_P80211REQ_H
 
-int p80211req_dorequest(wlandevice_t *wlandev, u8 *msgbuf);
+int p80211req_dorequest(struct wlandevice *wlandev, u8 *msgbuf);
 
 #endif
diff --git a/drivers/staging/wlan-ng/p80211types.h b/drivers/staging/wlan-ng/p80211types.h
index 8cb4fc6448a0..263ef2ddb197 100644
--- a/drivers/staging/wlan-ng/p80211types.h
+++ b/drivers/staging/wlan-ng/p80211types.h
@@ -1,58 +1,59 @@
-/* p80211types.h
-*
-* Macros, constants, types, and funcs for p80211 data types
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* This file declares some of the constants and types used in various
-* parts of the linux-wlan system.
-*
-* Notes:
-*   - Constant values are always in HOST byte order.
-*
-* All functions and statics declared here are implemented in p80211types.c
-*   --------------------------------------------------------------------
-*/
+/*
+ * p80211types.h
+ *
+ * Macros, constants, types, and funcs for p80211 data types
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * This file declares some of the constants and types used in various
+ * parts of the linux-wlan system.
+ *
+ * Notes:
+ *   - Constant values are always in HOST byte order.
+ *
+ * All functions and statics declared here are implemented in p80211types.c
+ *   --------------------------------------------------------------------
+ */
 
 #ifndef _P80211TYPES_H
 #define _P80211TYPES_H
@@ -123,18 +124,18 @@
 #define MKENUMNAME(name) p80211enum_ ## name
 
 /*----------------------------------------------------------------
-* The following constants and macros are used to construct and
-* deconstruct the Data ID codes.  The coding is as follows:
-*
-*     ...rwtnnnnnnnniiiiiiggggggssssss      s - Section
-*                                           g - Group
-*                                           i - Item
-*                                           n - Index
-*                                           t - Table flag
-*                                           w - Write flag
-*                                           r - Read flag
-*                                           . - Unused
-*/
+ * The following constants and macros are used to construct and
+ * deconstruct the Data ID codes.  The coding is as follows:
+ *
+ *     ...rwtnnnnnnnniiiiiiggggggssssss      s - Section
+ *                                           g - Group
+ *                                           i - Item
+ *                                           n - Index
+ *                                           t - Table flag
+ *                                           w - Write flag
+ *                                           r - Read flag
+ *                                           . - Unused
+ */
 
 #define P80211DID_LSB_SECTION		(0)
 #define P80211DID_LSB_GROUP		(6)
@@ -200,138 +201,138 @@
 /* The following structure types are used for the representation */
 /*  of ENUMint type metadata. */
 
-typedef struct p80211enumpair {
+struct p80211enumpair {
 	u32 val;
 	char *name;
-} p80211enumpair_t;
+};
 
-typedef struct p80211enum {
+struct p80211enum {
 	int nitems;
-	p80211enumpair_t *list;
-} p80211enum_t;
+	struct p80211enumpair *list;
+};
 
 /*----------------------------------------------------------------*/
 /* The following structure types are used to store data items in */
 /*  messages. */
 
 /* Template pascal string */
-typedef struct p80211pstr {
+struct p80211pstr {
 	u8 len;
-} __packed p80211pstr_t;
+} __packed;
 
-typedef struct p80211pstrd {
+struct p80211pstrd {
 	u8 len;
 	u8 data[0];
-} __packed p80211pstrd_t;
+} __packed;
 
 /* Maximum pascal string */
-typedef struct p80211pstr255 {
+struct p80211pstr255 {
 	u8 len;
 	u8 data[MAXLEN_PSTR255];
-} __packed p80211pstr255_t;
+} __packed;
 
 /* pascal string for macaddress and bssid */
-typedef struct p80211pstr6 {
+struct p80211pstr6 {
 	u8 len;
 	u8 data[MAXLEN_PSTR6];
-} __packed p80211pstr6_t;
+} __packed;
 
 /* pascal string for channel list */
-typedef struct p80211pstr14 {
+struct p80211pstr14 {
 	u8 len;
 	u8 data[MAXLEN_PSTR14];
-} __packed p80211pstr14_t;
+} __packed;
 
 /* pascal string for ssid */
-typedef struct p80211pstr32 {
+struct p80211pstr32 {
 	u8 len;
 	u8 data[MAXLEN_PSTR32];
-} __packed p80211pstr32_t;
+} __packed;
 
 /* MAC address array */
-typedef struct p80211macarray {
+struct p80211macarray {
 	u32 cnt;
 	u8 data[1][MAXLEN_PSTR6];
-} __packed p80211macarray_t;
+} __packed;
 
 /* prototype template */
-typedef struct p80211item {
+struct p80211item {
 	u32 did;
 	u16 status;
 	u16 len;
-} __packed p80211item_t;
+} __packed;
 
 /* prototype template w/ data item */
-typedef struct p80211itemd {
+struct p80211itemd {
 	u32 did;
 	u16 status;
 	u16 len;
 	u8 data[0];
-} __packed p80211itemd_t;
+} __packed;
 
 /* message data item for int, BOUNDEDINT, ENUMINT */
-typedef struct p80211item_uint32 {
+struct p80211item_uint32 {
 	u32 did;
 	u16 status;
 	u16 len;
 	u32 data;
-} __packed p80211item_uint32_t;
+} __packed;
 
 /* message data item for OCTETSTR, DISPLAYSTR */
-typedef struct p80211item_pstr6 {
+struct p80211item_pstr6 {
 	u32 did;
 	u16 status;
 	u16 len;
-	p80211pstr6_t data;
-} __packed p80211item_pstr6_t;
+	struct p80211pstr6 data;
+} __packed;
 
 /* message data item for OCTETSTR, DISPLAYSTR */
-typedef struct p80211item_pstr14 {
+struct p80211item_pstr14 {
 	u32 did;
 	u16 status;
 	u16 len;
-	p80211pstr14_t data;
-} __packed p80211item_pstr14_t;
+	struct p80211pstr14 data;
+} __packed;
 
 /* message data item for OCTETSTR, DISPLAYSTR */
-typedef struct p80211item_pstr32 {
+struct p80211item_pstr32 {
 	u32 did;
 	u16 status;
 	u16 len;
-	p80211pstr32_t data;
-} __packed p80211item_pstr32_t;
+	struct p80211pstr32 data;
+} __packed;
 
 /* message data item for OCTETSTR, DISPLAYSTR */
-typedef struct p80211item_pstr255 {
+struct p80211item_pstr255 {
 	u32 did;
 	u16 status;
 	u16 len;
-	p80211pstr255_t data;
-} __packed p80211item_pstr255_t;
+	struct p80211pstr255 data;
+} __packed;
 
 /* message data item for UNK 392, namely mib items */
-typedef struct p80211item_unk392 {
+struct p80211item_unk392 {
 	u32 did;
 	u16 status;
 	u16 len;
 	u8 data[MAXLEN_MIBATTRIBUTE];
-} __packed p80211item_unk392_t;
+} __packed;
 
 /* message data item for UNK 1025, namely p2 pdas */
-typedef struct p80211item_unk1024 {
+struct p80211item_unk1024 {
 	u32 did;
 	u16 status;
 	u16 len;
 	u8 data[1024];
-} __packed p80211item_unk1024_t;
+} __packed;
 
 /* message data item for UNK 4096, namely p2 download chunks */
-typedef struct p80211item_unk4096 {
+struct p80211item_unk4096 {
 	u32 did;
 	u16 status;
 	u16 len;
 	u8 data[4096];
-} __packed p80211item_unk4096_t;
+} __packed;
 
 struct catlistitem;
 
@@ -351,25 +352,25 @@ typedef u32(*p80211_valid_t) (struct catlistitem *, u32 did, u8 *itembuf);
 /*  The following are the external declarations */
 /*  for all enumerations  */
 
-extern p80211enum_t MKENUMNAME(truth);
-extern p80211enum_t MKENUMNAME(ifstate);
-extern p80211enum_t MKENUMNAME(powermgmt);
-extern p80211enum_t MKENUMNAME(bsstype);
-extern p80211enum_t MKENUMNAME(authalg);
-extern p80211enum_t MKENUMNAME(phytype);
-extern p80211enum_t MKENUMNAME(temptype);
-extern p80211enum_t MKENUMNAME(regdomain);
-extern p80211enum_t MKENUMNAME(ccamode);
-extern p80211enum_t MKENUMNAME(diversity);
-extern p80211enum_t MKENUMNAME(scantype);
-extern p80211enum_t MKENUMNAME(resultcode);
-extern p80211enum_t MKENUMNAME(reason);
-extern p80211enum_t MKENUMNAME(status);
-extern p80211enum_t MKENUMNAME(msgcode);
-extern p80211enum_t MKENUMNAME(msgitem_status);
-
-extern p80211enum_t MKENUMNAME(lnxroam_reason);
-
-extern p80211enum_t MKENUMNAME(p2preamble);
+extern struct p80211enum MKENUMNAME(truth);
+extern struct p80211enum MKENUMNAME(ifstate);
+extern struct p80211enum MKENUMNAME(powermgmt);
+extern struct p80211enum MKENUMNAME(bsstype);
+extern struct p80211enum MKENUMNAME(authalg);
+extern struct p80211enum MKENUMNAME(phytype);
+extern struct p80211enum MKENUMNAME(temptype);
+extern struct p80211enum MKENUMNAME(regdomain);
+extern struct p80211enum MKENUMNAME(ccamode);
+extern struct p80211enum MKENUMNAME(diversity);
+extern struct p80211enum MKENUMNAME(scantype);
+extern struct p80211enum MKENUMNAME(resultcode);
+extern struct p80211enum MKENUMNAME(reason);
+extern struct p80211enum MKENUMNAME(status);
+extern struct p80211enum MKENUMNAME(msgcode);
+extern struct p80211enum MKENUMNAME(msgitem_status);
+
+extern struct p80211enum MKENUMNAME(lnxroam_reason);
+
+extern struct p80211enum MKENUMNAME(p2preamble);
 
 #endif /* _P80211TYPES_H */
diff --git a/drivers/staging/wlan-ng/p80211wep.c b/drivers/staging/wlan-ng/p80211wep.c
index 22c79703e328..23b183738037 100644
--- a/drivers/staging/wlan-ng/p80211wep.c
+++ b/drivers/staging/wlan-ng/p80211wep.c
@@ -119,7 +119,7 @@ static const u32 wep_crc32_table[256] = {
 
 /* keylen in bytes! */
 
-int wep_change_key(wlandevice_t *wlandev, int keynum, u8 *key, int keylen)
+int wep_change_key(struct wlandevice *wlandev, int keynum, u8 *key, int keylen)
 {
 	if (keylen < 0)
 		return -1;
@@ -143,7 +143,7 @@ int wep_change_key(wlandevice_t *wlandev, int keynum, u8 *key, int keylen)
  * 4-byte IV at start of buffer, 4-byte ICV at end of buffer.
  * if successful, buf start is payload begin, length -= 8;
  */
-int wep_decrypt(wlandevice_t *wlandev, u8 *buf, u32 len, int key_override,
+int wep_decrypt(struct wlandevice *wlandev, u8 *buf, u32 len, int key_override,
 		u8 *iv, u8 *icv)
 {
 	u32 i, j, k, crc, keylen;
@@ -217,7 +217,7 @@ int wep_decrypt(wlandevice_t *wlandev, u8 *buf, u32 len, int key_override,
 }
 
 /* encrypts in-place. */
-int wep_encrypt(wlandevice_t *wlandev, u8 *buf, u8 *dst, u32 len, int keynum,
+int wep_encrypt(struct wlandevice *wlandev, u8 *buf, u8 *dst, u32 len, int keynum,
 		u8 *iv, u8 *icv)
 {
 	u32 i, j, k, crc, keylen;
diff --git a/drivers/staging/wlan-ng/prism2fw.c b/drivers/staging/wlan-ng/prism2fw.c
index 56bffd93c982..96aa21188669 100644
--- a/drivers/staging/wlan-ng/prism2fw.c
+++ b/drivers/staging/wlan-ng/prism2fw.c
@@ -96,16 +96,16 @@ struct s3inforec {
 	u16 len;
 	u16 type;
 	union {
-		hfa384x_compident_t version;
-		hfa384x_caplevel_t compat;
+		struct hfa384x_compident version;
+		struct hfa384x_caplevel compat;
 		u16 buildseq;
-		hfa384x_compident_t platform;
+		struct hfa384x_compident platform;
 	} info;
 };
 
 struct pda {
 	u8 buf[HFA384x_PDA_LEN_MAX];
-	hfa384x_pdrec_t *rec[HFA384x_PDA_RECS_MAX];
+	struct hfa384x_pdrec *rec[HFA384x_PDA_RECS_MAX];
 	unsigned int nrec;
 };
 
@@ -152,22 +152,22 @@ static struct imgchunk fchunk[CHUNKS_MAX];
 /* PDA, built from [card|newfile]+[addfile1+addfile2...] */
 
 static struct pda pda;
-static hfa384x_compident_t nicid;
-static hfa384x_caplevel_t rfid;
-static hfa384x_caplevel_t macid;
-static hfa384x_caplevel_t priid;
+static struct hfa384x_compident nicid;
+static struct hfa384x_caplevel rfid;
+static struct hfa384x_caplevel macid;
+static struct hfa384x_caplevel priid;
 
 /*================================================================*/
 /* Local Function Declarations */
 
 static int prism2_fwapply(const struct ihex_binrec *rfptr,
-wlandevice_t *wlandev);
+struct wlandevice *wlandev);
 
 static int read_fwfile(const struct ihex_binrec *rfptr);
 
 static int mkimage(struct imgchunk *clist, unsigned int *ccnt);
 
-static int read_cardpda(struct pda *pda, wlandevice_t *wlandev);
+static int read_cardpda(struct pda *pda, struct wlandevice *wlandev);
 
 static int mkpdrlist(struct pda *pda);
 
@@ -177,7 +177,7 @@ static int plugimage(struct imgchunk *fchunk, unsigned int nfchunks,
 static int crcimage(struct imgchunk *fchunk, unsigned int nfchunks,
 	     struct s3crcrec *s3crc, unsigned int ns3crc);
 
-static int writeimage(wlandevice_t *wlandev, struct imgchunk *fchunk,
+static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
 	       unsigned int nfchunks);
 static void free_chunks(struct imgchunk *fchunk, unsigned int *nfchunks);
 
@@ -201,7 +201,7 @@ static int validate_identity(void);
 *	0	- success
 *	~0	- failure
 ----------------------------------------------------------------*/
-static int prism2_fwtry(struct usb_device *udev, wlandevice_t *wlandev)
+static int prism2_fwtry(struct usb_device *udev, struct wlandevice *wlandev)
 {
 	const struct firmware *fw_entry = NULL;
 
@@ -239,11 +239,11 @@ static int prism2_fwtry(struct usb_device *udev, wlandevice_t *wlandev)
 *	~0	- failure
 ----------------------------------------------------------------*/
 static int prism2_fwapply(const struct ihex_binrec *rfptr,
-			  wlandevice_t *wlandev)
+			  struct wlandevice *wlandev)
 {
 	signed int result = 0;
 	struct p80211msg_dot11req_mibget getmsg;
-	p80211itemd_t *item;
+	struct p80211itemd *item;
 	u32 *data;
 
 	/* Initialize the data structures */
@@ -266,7 +266,7 @@ static int prism2_fwapply(const struct ihex_binrec *rfptr,
 
 	/* clear the pda and add an initial END record */
 	memset(&pda, 0, sizeof(pda));
-	pda.rec[0] = (hfa384x_pdrec_t *) pda.buf;
+	pda.rec[0] = (struct hfa384x_pdrec *)pda.buf;
 	pda.rec[0]->len = cpu_to_le16(2);	/* len in words */
 	pda.rec[0]->code = cpu_to_le16(HFA384x_PDR_END_OF_PDA);
 	pda.nrec = 1;
@@ -293,11 +293,11 @@ static int prism2_fwapply(const struct ihex_binrec *rfptr,
 	getmsg.resultcode.did = DIDmsg_dot11req_mibget_resultcode;
 	getmsg.resultcode.status = P80211ENUM_msgitem_status_no_value;
 
-	item = (p80211itemd_t *) getmsg.mibattribute.data;
+	item = (struct p80211itemd *)getmsg.mibattribute.data;
 	item->did = DIDmib_p2_p2NIC_p2PRISupRange;
 	item->status = P80211ENUM_msgitem_status_no_value;
 
-	data = (u32 *) item->data;
+	data = (u32 *)item->data;
 
 	/* DIDmsg_dot11req_mibget */
 	prism2mgmt_mibset_mibget(wlandev, &getmsg);
@@ -592,14 +592,14 @@ static int mkimage(struct imgchunk *clist, unsigned int *ccnt)
 ----------------------------------------------------------------*/
 static int mkpdrlist(struct pda *pda)
 {
-	u16 *pda16 = (u16 *) pda->buf;
+	u16 *pda16 = (u16 *)pda->buf;
 	int curroff;		/* in 'words' */
 
 	pda->nrec = 0;
 	curroff = 0;
 	while (curroff < (HFA384x_PDA_LEN_MAX / 2 - 1) &&
 	       le16_to_cpu(pda16[curroff + 1]) != HFA384x_PDR_END_OF_PDA) {
-		pda->rec[pda->nrec] = (hfa384x_pdrec_t *) &(pda16[curroff]);
+		pda->rec[pda->nrec] = (struct hfa384x_pdrec *)&(pda16[curroff]);
 
 		if (le16_to_cpu(pda->rec[pda->nrec]->code) ==
 		    HFA384x_PDR_NICID) {
@@ -638,7 +638,7 @@ static int mkpdrlist(struct pda *pda)
 		       curroff, pda->nrec);
 		return 1;
 	}
-	pda->rec[pda->nrec] = (hfa384x_pdrec_t *) &(pda16[curroff]);
+	pda->rec[pda->nrec] = (struct hfa384x_pdrec *)&(pda16[curroff]);
 	(pda->nrec)++;
 	return 0;
 }
@@ -766,7 +766,7 @@ static int plugimage(struct imgchunk *fchunk, unsigned int nfchunks,
 *	0	- success
 *	~0	- failure (probably an errno)
 ----------------------------------------------------------------*/
-static int read_cardpda(struct pda *pda, wlandevice_t *wlandev)
+static int read_cardpda(struct pda *pda, struct wlandevice *wlandev)
 {
 	int result = 0;
 	struct p80211msg_p2req_readpda *msg;
@@ -879,8 +879,8 @@ static int read_fwfile(const struct ihex_binrec *record)
 		addr = be32_to_cpu(record->addr);
 
 		/* Point into data for different word lengths */
-		ptr32 = (u32 *) record->data;
-		ptr16 = (u16 *) record->data;
+		ptr32 = (u32 *)record->data;
+		ptr16 = (u16 *)record->data;
 
 		/* parse what was an S3 srec and put it in the right array */
 		switch (addr) {
@@ -954,7 +954,7 @@ static int read_fwfile(const struct ihex_binrec *record)
 		default:	/* Data record */
 			s3data[ns3data].addr = addr;
 			s3data[ns3data].len = len;
-			s3data[ns3data].data = (uint8_t *) record->data;
+			s3data[ns3data].data = (uint8_t *)record->data;
 			ns3data++;
 			if (ns3data == S3DATA_MAX) {
 				pr_err("S3 datarec limit reached - aborting\n");
@@ -982,7 +982,7 @@ static int read_fwfile(const struct ihex_binrec *record)
 *	0	success
 *	~0	failure
 ----------------------------------------------------------------*/
-static int writeimage(wlandevice_t *wlandev, struct imgchunk *fchunk,
+static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
 	       unsigned int nfchunks)
 {
 	int result = 0;
diff --git a/drivers/staging/wlan-ng/prism2mgmt.c b/drivers/staging/wlan-ng/prism2mgmt.c
index d8ed9a05789c..170de1c9eac4 100644
--- a/drivers/staging/wlan-ng/prism2mgmt.c
+++ b/drivers/staging/wlan-ng/prism2mgmt.c
@@ -113,16 +113,16 @@
 *	process thread  (usually)
 *	interrupt
 ----------------------------------------------------------------*/
-int prism2mgmt_scan(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_scan(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	struct p80211msg_dot11req_scan *msg = msgp;
 	u16 roamingmode, word;
 	int i, timeout;
 	int istmpenable = 0;
 
-	hfa384x_HostScanRequest_data_t scanreq;
+	struct hfa384x_HostScanRequest_data scanreq;
 
 	/* gatekeeper check */
 	if (HFA384x_FIRMWARE_VERSION(hw->ident_sta_fw.major,
@@ -292,7 +292,7 @@ int prism2mgmt_scan(wlandevice_t *wlandev, void *msgp)
 
 	result = hfa384x_drvr_setconfig(hw,
 					HFA384x_RID_HOSTSCAN, &scanreq,
-					sizeof(hfa384x_HostScanRequest_data_t));
+					sizeof(struct hfa384x_HostScanRequest_data));
 	if (result) {
 		netdev_err(wlandev->netdev,
 			   "setconfig(SCANREQUEST) failed. result=%d\n",
@@ -366,12 +366,12 @@ exit:
 *	process thread  (usually)
 *	interrupt
 ----------------------------------------------------------------*/
-int prism2mgmt_scan_results(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_scan_results(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
 	struct p80211msg_dot11req_scan_results *req;
-	hfa384x_t *hw = wlandev->priv;
-	hfa384x_HScanResultSub_t *item = NULL;
+	struct hfa384x *hw = wlandev->priv;
+	struct hfa384x_HScanResultSub *item = NULL;
 
 	int count;
 
@@ -525,15 +525,15 @@ exit:
 *	process thread  (usually)
 *	interrupt
 ----------------------------------------------------------------*/
-int prism2mgmt_start(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_start(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	struct p80211msg_dot11req_start *msg = msgp;
 
-	p80211pstrd_t *pstr;
+	struct p80211pstrd *pstr;
 	u8 bytebuf[80];
-	struct hfa384x_bytestr *p2bytestr = (struct hfa384x_bytestr *) bytebuf;
+	struct hfa384x_bytestr *p2bytestr = (struct hfa384x_bytestr *)bytebuf;
 	u16 word;
 
 	wlandev->macmode = WLAN_MACMODE_NONE;
@@ -558,7 +558,7 @@ int prism2mgmt_start(wlandevice_t *wlandev, void *msgp)
 	/*** STATION ***/
 	/* Set the REQUIRED config items */
 	/* SSID */
-	pstr = (p80211pstrd_t *) &(msg->ssid.data);
+	pstr = (struct p80211pstrd *)&(msg->ssid.data);
 	prism2mgmt_pstr2bytestr(p2bytestr, pstr);
 	result = hfa384x_drvr_setconfig(hw, HFA384x_RID_CNFOWNSSID,
 					bytebuf, HFA384x_RID_CNFOWNSSID_LEN);
@@ -685,9 +685,7 @@ failed:
 	msg->resultcode.data = P80211ENUM_resultcode_invalid_parameters;
 
 done:
-	result = 0;
-
-	return result;
+	return 0;
 }
 
 /*----------------------------------------------------------------
@@ -708,9 +706,9 @@ done:
 * Call context:
 *	process thread  (usually)
 ----------------------------------------------------------------*/
-int prism2mgmt_readpda(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_readpda(struct wlandevice *wlandev, void *msgp)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	struct p80211msg_p2req_readpda *msg = msgp;
 	int result;
 
@@ -774,9 +772,9 @@ int prism2mgmt_readpda(wlandevice_t *wlandev, void *msgp)
 * Call context:
 *	process thread  (usually)
 ----------------------------------------------------------------*/
-int prism2mgmt_ramdl_state(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_ramdl_state(struct wlandevice *wlandev, void *msgp)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	struct p80211msg_p2req_ramdl_state *msg = msgp;
 
 	if (wlandev->msdstate != WLAN_MSD_FWLOAD) {
@@ -829,9 +827,9 @@ int prism2mgmt_ramdl_state(wlandevice_t *wlandev, void *msgp)
 * Call context:
 *	process thread  (usually)
 ----------------------------------------------------------------*/
-int prism2mgmt_ramdl_write(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_ramdl_write(struct wlandevice *wlandev, void *msgp)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	struct p80211msg_p2req_ramdl_write *msg = msgp;
 	u32 addr;
 	u32 len;
@@ -890,10 +888,10 @@ int prism2mgmt_ramdl_write(wlandevice_t *wlandev, void *msgp)
 * Call context:
 *	process thread  (usually)
 ----------------------------------------------------------------*/
-int prism2mgmt_flashdl_state(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_flashdl_state(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	struct p80211msg_p2req_flashdl_state *msg = msgp;
 
 	if (wlandev->msdstate != WLAN_MSD_FWLOAD) {
@@ -961,9 +959,9 @@ int prism2mgmt_flashdl_state(wlandevice_t *wlandev, void *msgp)
 * Call context:
 *	process thread  (usually)
 ----------------------------------------------------------------*/
-int prism2mgmt_flashdl_write(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_flashdl_write(struct wlandevice *wlandev, void *msgp)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	struct p80211msg_p2req_flashdl_write *msg = msgp;
 	u32 addr;
 	u32 len;
@@ -1021,16 +1019,16 @@ int prism2mgmt_flashdl_write(wlandevice_t *wlandev, void *msgp)
 *	process thread  (usually)
 *	interrupt
 ----------------------------------------------------------------*/
-int prism2mgmt_autojoin(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_autojoin(struct wlandevice *wlandev, void *msgp)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	int result = 0;
 	u16 reg;
 	u16 port_type;
 	struct p80211msg_lnxreq_autojoin *msg = msgp;
-	p80211pstrd_t *pstr;
+	struct p80211pstrd *pstr;
 	u8 bytebuf[256];
-	struct hfa384x_bytestr *p2bytestr = (struct hfa384x_bytestr *) bytebuf;
+	struct hfa384x_bytestr *p2bytestr = (struct hfa384x_bytestr *)bytebuf;
 
 	wlandev->macmode = WLAN_MACMODE_NONE;
 
@@ -1054,7 +1052,7 @@ int prism2mgmt_autojoin(wlandevice_t *wlandev, void *msgp)
 
 	/* Set the ssid */
 	memset(bytebuf, 0, 256);
-	pstr = (p80211pstrd_t *) &(msg->ssid.data);
+	pstr = (struct p80211pstrd *)&(msg->ssid.data);
 	prism2mgmt_pstr2bytestr(p2bytestr, pstr);
 	result = hfa384x_drvr_setconfig(hw, HFA384x_RID_CNFDESIREDSSID,
 					bytebuf,
@@ -1092,12 +1090,12 @@ int prism2mgmt_autojoin(wlandevice_t *wlandev, void *msgp)
 *	process thread  (usually)
 *	interrupt
 ----------------------------------------------------------------*/
-int prism2mgmt_wlansniff(wlandevice_t *wlandev, void *msgp)
+int prism2mgmt_wlansniff(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
 	struct p80211msg_lnxreq_wlansniff *msg = msgp;
 
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	u16 word;
 
 	msg->resultcode.status = P80211ENUM_msgitem_status_data_ok;
diff --git a/drivers/staging/wlan-ng/prism2mgmt.h b/drivers/staging/wlan-ng/prism2mgmt.h
index e6472034da33..cc1ac7a60dfe 100644
--- a/drivers/staging/wlan-ng/prism2mgmt.h
+++ b/drivers/staging/wlan-ng/prism2mgmt.h
@@ -63,43 +63,44 @@
 extern int prism2_reset_holdtime;
 extern int prism2_reset_settletime;
 
-u32 prism2sta_ifstate(wlandevice_t *wlandev, u32 ifstate);
+u32 prism2sta_ifstate(struct wlandevice *wlandev, u32 ifstate);
 
-void prism2sta_ev_info(wlandevice_t *wlandev, hfa384x_InfFrame_t *inf);
-void prism2sta_ev_txexc(wlandevice_t *wlandev, u16 status);
-void prism2sta_ev_tx(wlandevice_t *wlandev, u16 status);
-void prism2sta_ev_alloc(wlandevice_t *wlandev);
+void prism2sta_ev_info(struct wlandevice *wlandev, struct hfa384x_InfFrame *inf);
+void prism2sta_ev_txexc(struct wlandevice *wlandev, u16 status);
+void prism2sta_ev_tx(struct wlandevice *wlandev, u16 status);
+void prism2sta_ev_alloc(struct wlandevice *wlandev);
 
-int prism2mgmt_mibset_mibget(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_scan(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_scan_results(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_start(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_wlansniff(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_readpda(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_ramdl_state(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_ramdl_write(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_flashdl_state(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_flashdl_write(wlandevice_t *wlandev, void *msgp);
-int prism2mgmt_autojoin(wlandevice_t *wlandev, void *msgp);
+int prism2mgmt_mibset_mibget(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_scan(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_scan_results(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_start(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_wlansniff(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_readpda(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_ramdl_state(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_ramdl_write(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_flashdl_state(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_flashdl_write(struct wlandevice *wlandev, void *msgp);
+int prism2mgmt_autojoin(struct wlandevice *wlandev, void *msgp);
 
 /*---------------------------------------------------------------
 * conversion functions going between wlan message data types and
 * Prism2 data types
 ---------------------------------------------------------------*/
 /* byte area conversion functions*/
-void prism2mgmt_bytearea2pstr(u8 *bytearea, p80211pstrd_t *pstr, int len);
+void prism2mgmt_bytearea2pstr(u8 *bytearea, struct p80211pstrd *pstr, int len);
 
 /* byte string conversion functions*/
 void prism2mgmt_pstr2bytestr(struct hfa384x_bytestr *bytestr,
-			     p80211pstrd_t *pstr);
+			     struct p80211pstrd *pstr);
 void prism2mgmt_bytestr2pstr(struct hfa384x_bytestr *bytestr,
-			     p80211pstrd_t *pstr);
+			     struct p80211pstrd *pstr);
 
 /* functions to convert Group Addresses */
-void prism2mgmt_get_grpaddr(u32 did, p80211pstrd_t *pstr, hfa384x_t *priv);
+void prism2mgmt_get_grpaddr(u32 did, struct p80211pstrd *pstr,
+			    struct hfa384x *priv);
 int prism2mgmt_set_grpaddr(u32 did,
-			   u8 *prism2buf, p80211pstrd_t *pstr,
-			   hfa384x_t *priv);
+			   u8 *prism2buf, struct p80211pstrd *pstr,
+			   struct hfa384x *priv);
 int prism2mgmt_get_grpaddr_index(u32 did);
 
 void prism2sta_processing_defer(struct work_struct *data);
@@ -108,8 +109,8 @@ void prism2sta_commsqual_defer(struct work_struct *data);
 void prism2sta_commsqual_timer(unsigned long data);
 
 /* Interface callback functions, passing data back up to the cfg80211 layer */
-void prism2_connect_result(wlandevice_t *wlandev, u8 failed);
-void prism2_disconnected(wlandevice_t *wlandev);
-void prism2_roamed(wlandevice_t *wlandev);
+void prism2_connect_result(struct wlandevice *wlandev, u8 failed);
+void prism2_disconnected(struct wlandevice *wlandev);
+void prism2_roamed(struct wlandevice *wlandev);
 
 #endif
diff --git a/drivers/staging/wlan-ng/prism2mib.c b/drivers/staging/wlan-ng/prism2mib.c
index fe914b1f904b..63ab6bc88654 100644
--- a/drivers/staging/wlan-ng/prism2mib.c
+++ b/drivers/staging/wlan-ng/prism2mib.c
@@ -87,80 +87,80 @@ struct mibrec {
 	u16 parm3;
 	int (*func)(struct mibrec *mib,
 		     int isget,
-		     wlandevice_t *wlandev,
-		     hfa384x_t *hw,
+		     struct wlandevice *wlandev,
+		     struct hfa384x *hw,
 		     struct p80211msg_dot11req_mibset *msg, void *data);
 };
 
 static int prism2mib_bytearea2pstr(struct mibrec *mib,
 				   int isget,
-				   wlandevice_t *wlandev,
-				   hfa384x_t *hw,
+				   struct wlandevice *wlandev,
+				   struct hfa384x *hw,
 				   struct p80211msg_dot11req_mibset *msg,
 				   void *data);
 
 static int prism2mib_uint32(struct mibrec *mib,
 			    int isget,
-			    wlandevice_t *wlandev,
-			    hfa384x_t *hw,
+			    struct wlandevice *wlandev,
+			    struct hfa384x *hw,
 			    struct p80211msg_dot11req_mibset *msg, void *data);
 
 static int prism2mib_flag(struct mibrec *mib,
 			  int isget,
-			  wlandevice_t *wlandev,
-			  hfa384x_t *hw,
+			  struct wlandevice *wlandev,
+			  struct hfa384x *hw,
 			  struct p80211msg_dot11req_mibset *msg, void *data);
 
 static int prism2mib_wepdefaultkey(struct mibrec *mib,
 				   int isget,
-				   wlandevice_t *wlandev,
-				   hfa384x_t *hw,
+				   struct wlandevice *wlandev,
+				   struct hfa384x *hw,
 				   struct p80211msg_dot11req_mibset *msg,
 				   void *data);
 
 static int prism2mib_privacyinvoked(struct mibrec *mib,
 				    int isget,
-				    wlandevice_t *wlandev,
-				    hfa384x_t *hw,
+				    struct wlandevice *wlandev,
+				    struct hfa384x *hw,
 				    struct p80211msg_dot11req_mibset *msg,
 				    void *data);
 
 static int prism2mib_excludeunencrypted(struct mibrec *mib,
 					int isget,
-					wlandevice_t *wlandev,
-					hfa384x_t *hw,
+					struct wlandevice *wlandev,
+					struct hfa384x *hw,
 					struct p80211msg_dot11req_mibset *msg,
 					void *data);
 
 static int prism2mib_fragmentationthreshold(struct mibrec *mib,
 					    int isget,
-					    wlandevice_t *wlandev,
-					    hfa384x_t *hw,
+					    struct wlandevice *wlandev,
+					    struct hfa384x *hw,
 					    struct p80211msg_dot11req_mibset *msg,
 					    void *data);
 
 static int prism2mib_priv(struct mibrec *mib,
 			  int isget,
-			  wlandevice_t *wlandev,
-			  hfa384x_t *hw,
+			  struct wlandevice *wlandev,
+			  struct hfa384x *hw,
 			  struct p80211msg_dot11req_mibset *msg, void *data);
 
 static struct mibrec mibtab[] = {
 
 	/* dot11smt MIB's */
-	{DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey0,
+	{DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(1),
 	 F_STA | F_WRITE,
 	 HFA384x_RID_CNFWEPDEFAULTKEY0, 0, 0,
 	 prism2mib_wepdefaultkey},
-	{DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey1,
+	{DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(2),
 	 F_STA | F_WRITE,
 	 HFA384x_RID_CNFWEPDEFAULTKEY1, 0, 0,
 	 prism2mib_wepdefaultkey},
-	{DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey2,
+	{DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(3),
 	 F_STA | F_WRITE,
 	 HFA384x_RID_CNFWEPDEFAULTKEY2, 0, 0,
 	 prism2mib_wepdefaultkey},
-	{DIDmib_dot11smt_dot11WEPDefaultKeysTable_dot11WEPDefaultKey3,
+	{DIDmib_dot11smt_dot11WEPDefaultKeysTable_key(4),
 	 F_STA | F_WRITE,
 	 HFA384x_RID_CNFWEPDEFAULTKEY3, 0, 0,
 	 prism2mib_wepdefaultkey},
@@ -237,36 +237,36 @@ static struct mibrec mibtab[] = {
 	{0, 0, 0, 0, 0, NULL}
 };
 
-/*----------------------------------------------------------------
-* prism2mgmt_mibset_mibget
-*
-* Set the value of a mib item.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-*	interrupt
-----------------------------------------------------------------*/
-
-int prism2mgmt_mibset_mibget(wlandevice_t *wlandev, void *msgp)
+/*
+ * prism2mgmt_mibset_mibget
+ *
+ * Set the value of a mib item.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *	interrupt
+ */
+
+int prism2mgmt_mibset_mibget(struct wlandevice *wlandev, void *msgp)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	int result, isget;
 	struct mibrec *mib;
 
 	u16 which;
 
 	struct p80211msg_dot11req_mibset *msg = msgp;
-	p80211itemd_t *mibitem;
+	struct p80211itemd *mibitem;
 
 	msg->resultcode.status = P80211ENUM_msgitem_status_data_ok;
 	msg->resultcode.data = P80211ENUM_resultcode_success;
@@ -284,7 +284,7 @@ int prism2mgmt_mibset_mibget(wlandevice_t *wlandev, void *msgp)
 	 ** MIB table.
 	 */
 
-	mibitem = (p80211itemd_t *) msg->mibattribute.data;
+	mibitem = (struct p80211itemd *)msg->mibattribute.data;
 
 	for (mib = mibtab; mib->did != 0; mib++)
 		if (mib->did == mibitem->did && (mib->flag & which))
@@ -346,40 +346,40 @@ done:
 	return 0;
 }
 
-/*----------------------------------------------------------------
-* prism2mib_bytearea2pstr
-*
-* Get/set pstr data to/from a byte area.
-*
-* MIB record parameters:
-*       parm1    Prism2 RID value.
-*       parm2    Number of bytes of RID data.
-*       parm3    Not used.
-*
-* Arguments:
-*       mib      MIB record.
-*       isget    MIBGET/MIBSET flag.
-*       wlandev  wlan device structure.
-*       priv     "priv" structure.
-*       hw       "hw" structure.
-*       msg      Message structure.
-*       data     Data buffer.
-*
-* Returns:
-*       0   - Success.
-*       ~0  - Error.
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mib_bytearea2pstr
+ *
+ * Get/set pstr data to/from a byte area.
+ *
+ * MIB record parameters:
+ *       parm1    Prism2 RID value.
+ *       parm2    Number of bytes of RID data.
+ *       parm3    Not used.
+ *
+ * Arguments:
+ *       mib      MIB record.
+ *       isget    MIBGET/MIBSET flag.
+ *       wlandev  wlan device structure.
+ *       priv     "priv" structure.
+ *       hw       "hw" structure.
+ *       msg      Message structure.
+ *       data     Data buffer.
+ *
+ * Returns:
+ *       0   - Success.
+ *       ~0  - Error.
+ *
+ */
 
 static int prism2mib_bytearea2pstr(struct mibrec *mib,
 				   int isget,
-				   wlandevice_t *wlandev,
-				   hfa384x_t *hw,
+				   struct wlandevice *wlandev,
+				   struct hfa384x *hw,
 				   struct p80211msg_dot11req_mibset *msg,
 				   void *data)
 {
 	int result;
-	p80211pstrd_t *pstr = data;
+	struct p80211pstrd *pstr = data;
 	u8 bytebuf[MIB_TMP_MAXLEN];
 
 	if (isget) {
@@ -396,41 +396,41 @@ static int prism2mib_bytearea2pstr(struct mibrec *mib,
 	return result;
 }
 
-/*----------------------------------------------------------------
-* prism2mib_uint32
-*
-* Get/set uint32 data.
-*
-* MIB record parameters:
-*       parm1    Prism2 RID value.
-*       parm2    Not used.
-*       parm3    Not used.
-*
-* Arguments:
-*       mib      MIB record.
-*       isget    MIBGET/MIBSET flag.
-*       wlandev  wlan device structure.
-*       priv     "priv" structure.
-*       hw       "hw" structure.
-*       msg      Message structure.
-*       data     Data buffer.
-*
-* Returns:
-*       0   - Success.
-*       ~0  - Error.
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mib_uint32
+ *
+ * Get/set uint32 data.
+ *
+ * MIB record parameters:
+ *       parm1    Prism2 RID value.
+ *       parm2    Not used.
+ *       parm3    Not used.
+ *
+ * Arguments:
+ *       mib      MIB record.
+ *       isget    MIBGET/MIBSET flag.
+ *       wlandev  wlan device structure.
+ *       priv     "priv" structure.
+ *       hw       "hw" structure.
+ *       msg      Message structure.
+ *       data     Data buffer.
+ *
+ * Returns:
+ *       0   - Success.
+ *       ~0  - Error.
+ *
+ */
 
 static int prism2mib_uint32(struct mibrec *mib,
 			    int isget,
-			    wlandevice_t *wlandev,
-			    hfa384x_t *hw,
+			    struct wlandevice *wlandev,
+			    struct hfa384x *hw,
 			    struct p80211msg_dot11req_mibset *msg, void *data)
 {
 	int result;
 	u32 *uint32 = data;
 	u8 bytebuf[MIB_TMP_MAXLEN];
-	u16 *wordbuf = (u16 *) bytebuf;
+	u16 *wordbuf = (u16 *)bytebuf;
 
 	if (isget) {
 		result = hfa384x_drvr_getconfig16(hw, mib->parm1, wordbuf);
@@ -443,41 +443,41 @@ static int prism2mib_uint32(struct mibrec *mib,
 	return result;
 }
 
-/*----------------------------------------------------------------
-* prism2mib_flag
-*
-* Get/set a flag.
-*
-* MIB record parameters:
-*       parm1    Prism2 RID value.
-*       parm2    Bit to get/set.
-*       parm3    Not used.
-*
-* Arguments:
-*       mib      MIB record.
-*       isget    MIBGET/MIBSET flag.
-*       wlandev  wlan device structure.
-*       priv     "priv" structure.
-*       hw       "hw" structure.
-*       msg      Message structure.
-*       data     Data buffer.
-*
-* Returns:
-*       0   - Success.
-*       ~0  - Error.
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mib_flag
+ *
+ * Get/set a flag.
+ *
+ * MIB record parameters:
+ *       parm1    Prism2 RID value.
+ *       parm2    Bit to get/set.
+ *       parm3    Not used.
+ *
+ * Arguments:
+ *       mib      MIB record.
+ *       isget    MIBGET/MIBSET flag.
+ *       wlandev  wlan device structure.
+ *       priv     "priv" structure.
+ *       hw       "hw" structure.
+ *       msg      Message structure.
+ *       data     Data buffer.
+ *
+ * Returns:
+ *       0   - Success.
+ *       ~0  - Error.
+ *
+ */
 
 static int prism2mib_flag(struct mibrec *mib,
 			  int isget,
-			  wlandevice_t *wlandev,
-			  hfa384x_t *hw,
+			  struct wlandevice *wlandev,
+			  struct hfa384x *hw,
 			  struct p80211msg_dot11req_mibset *msg, void *data)
 {
 	int result;
 	u32 *uint32 = data;
 	u8 bytebuf[MIB_TMP_MAXLEN];
-	u16 *wordbuf = (u16 *) bytebuf;
+	u16 *wordbuf = (u16 *)bytebuf;
 	u32 flags;
 
 	result = hfa384x_drvr_getconfig16(hw, mib->parm1, wordbuf);
@@ -500,40 +500,40 @@ static int prism2mib_flag(struct mibrec *mib,
 	return result;
 }
 
-/*----------------------------------------------------------------
-* prism2mib_wepdefaultkey
-*
-* Get/set WEP default keys.
-*
-* MIB record parameters:
-*       parm1    Prism2 RID value.
-*       parm2    Number of bytes of RID data.
-*       parm3    Not used.
-*
-* Arguments:
-*       mib      MIB record.
-*       isget    MIBGET/MIBSET flag.
-*       wlandev  wlan device structure.
-*       priv     "priv" structure.
-*       hw       "hw" structure.
-*       msg      Message structure.
-*       data     Data buffer.
-*
-* Returns:
-*       0   - Success.
-*       ~0  - Error.
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mib_wepdefaultkey
+ *
+ * Get/set WEP default keys.
+ *
+ * MIB record parameters:
+ *       parm1    Prism2 RID value.
+ *       parm2    Number of bytes of RID data.
+ *       parm3    Not used.
+ *
+ * Arguments:
+ *       mib      MIB record.
+ *       isget    MIBGET/MIBSET flag.
+ *       wlandev  wlan device structure.
+ *       priv     "priv" structure.
+ *       hw       "hw" structure.
+ *       msg      Message structure.
+ *       data     Data buffer.
+ *
+ * Returns:
+ *       0   - Success.
+ *       ~0  - Error.
+ *
+ */
 
 static int prism2mib_wepdefaultkey(struct mibrec *mib,
 				   int isget,
-				   wlandevice_t *wlandev,
-				   hfa384x_t *hw,
+				   struct wlandevice *wlandev,
+				   struct hfa384x *hw,
 				   struct p80211msg_dot11req_mibset *msg,
 				   void *data)
 {
 	int result;
-	p80211pstrd_t *pstr = data;
+	struct p80211pstrd *pstr = data;
 	u8 bytebuf[MIB_TMP_MAXLEN];
 	u16 len;
 
@@ -550,35 +550,35 @@ static int prism2mib_wepdefaultkey(struct mibrec *mib,
 	return result;
 }
 
-/*----------------------------------------------------------------
-* prism2mib_privacyinvoked
-*
-* Get/set the dot11PrivacyInvoked value.
-*
-* MIB record parameters:
-*       parm1    Prism2 RID value.
-*       parm2    Bit value for PrivacyInvoked flag.
-*       parm3    Not used.
-*
-* Arguments:
-*       mib      MIB record.
-*       isget    MIBGET/MIBSET flag.
-*       wlandev  wlan device structure.
-*       priv     "priv" structure.
-*       hw       "hw" structure.
-*       msg      Message structure.
-*       data     Data buffer.
-*
-* Returns:
-*       0   - Success.
-*       ~0  - Error.
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mib_privacyinvoked
+ *
+ * Get/set the dot11PrivacyInvoked value.
+ *
+ * MIB record parameters:
+ *       parm1    Prism2 RID value.
+ *       parm2    Bit value for PrivacyInvoked flag.
+ *       parm3    Not used.
+ *
+ * Arguments:
+ *       mib      MIB record.
+ *       isget    MIBGET/MIBSET flag.
+ *       wlandev  wlan device structure.
+ *       priv     "priv" structure.
+ *       hw       "hw" structure.
+ *       msg      Message structure.
+ *       data     Data buffer.
+ *
+ * Returns:
+ *       0   - Success.
+ *       ~0  - Error.
+ *
+ */
 
 static int prism2mib_privacyinvoked(struct mibrec *mib,
 				    int isget,
-				    wlandevice_t *wlandev,
-				    hfa384x_t *hw,
+				    struct wlandevice *wlandev,
+				    struct hfa384x *hw,
 				    struct p80211msg_dot11req_mibset *msg,
 				    void *data)
 {
@@ -592,35 +592,35 @@ static int prism2mib_privacyinvoked(struct mibrec *mib,
 	return prism2mib_flag(mib, isget, wlandev, hw, msg, data);
 }
 
-/*----------------------------------------------------------------
-* prism2mib_excludeunencrypted
-*
-* Get/set the dot11ExcludeUnencrypted value.
-*
-* MIB record parameters:
-*       parm1    Prism2 RID value.
-*       parm2    Bit value for ExcludeUnencrypted flag.
-*       parm3    Not used.
-*
-* Arguments:
-*       mib      MIB record.
-*       isget    MIBGET/MIBSET flag.
-*       wlandev  wlan device structure.
-*       priv     "priv" structure.
-*       hw       "hw" structure.
-*       msg      Message structure.
-*       data     Data buffer.
-*
-* Returns:
-*       0   - Success.
-*       ~0  - Error.
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mib_excludeunencrypted
+ *
+ * Get/set the dot11ExcludeUnencrypted value.
+ *
+ * MIB record parameters:
+ *       parm1    Prism2 RID value.
+ *       parm2    Bit value for ExcludeUnencrypted flag.
+ *       parm3    Not used.
+ *
+ * Arguments:
+ *       mib      MIB record.
+ *       isget    MIBGET/MIBSET flag.
+ *       wlandev  wlan device structure.
+ *       priv     "priv" structure.
+ *       hw       "hw" structure.
+ *       msg      Message structure.
+ *       data     Data buffer.
+ *
+ * Returns:
+ *       0   - Success.
+ *       ~0  - Error.
+ *
+ */
 
 static int prism2mib_excludeunencrypted(struct mibrec *mib,
 					int isget,
-					wlandevice_t *wlandev,
-					hfa384x_t *hw,
+					struct wlandevice *wlandev,
+					struct hfa384x *hw,
 					struct p80211msg_dot11req_mibset *msg,
 					void *data)
 {
@@ -628,35 +628,35 @@ static int prism2mib_excludeunencrypted(struct mibrec *mib,
 	return prism2mib_flag(mib, isget, wlandev, hw, msg, data);
 }
 
-/*----------------------------------------------------------------
-* prism2mib_fragmentationthreshold
-*
-* Get/set the fragmentation threshold.
-*
-* MIB record parameters:
-*       parm1    Prism2 RID value.
-*       parm2    Not used.
-*       parm3    Not used.
-*
-* Arguments:
-*       mib      MIB record.
-*       isget    MIBGET/MIBSET flag.
-*       wlandev  wlan device structure.
-*       priv     "priv" structure.
-*       hw       "hw" structure.
-*       msg      Message structure.
-*       data     Data buffer.
-*
-* Returns:
-*       0   - Success.
-*       ~0  - Error.
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mib_fragmentationthreshold
+ *
+ * Get/set the fragmentation threshold.
+ *
+ * MIB record parameters:
+ *       parm1    Prism2 RID value.
+ *       parm2    Not used.
+ *       parm3    Not used.
+ *
+ * Arguments:
+ *       mib      MIB record.
+ *       isget    MIBGET/MIBSET flag.
+ *       wlandev  wlan device structure.
+ *       priv     "priv" structure.
+ *       hw       "hw" structure.
+ *       msg      Message structure.
+ *       data     Data buffer.
+ *
+ * Returns:
+ *       0   - Success.
+ *       ~0  - Error.
+ *
+ */
 
 static int prism2mib_fragmentationthreshold(struct mibrec *mib,
 					    int isget,
-					    wlandevice_t *wlandev,
-					    hfa384x_t *hw,
+					    struct wlandevice *wlandev,
+					    struct hfa384x *hw,
 					    struct p80211msg_dot11req_mibset *msg,
 					    void *data)
 {
@@ -674,47 +674,47 @@ static int prism2mib_fragmentationthreshold(struct mibrec *mib,
 	return prism2mib_uint32(mib, isget, wlandev, hw, msg, data);
 }
 
-/*----------------------------------------------------------------
-* prism2mib_priv
-*
-* Get/set values in the "priv" data structure.
-*
-* MIB record parameters:
-*       parm1    Not used.
-*       parm2    Not used.
-*       parm3    Not used.
-*
-* Arguments:
-*       mib      MIB record.
-*       isget    MIBGET/MIBSET flag.
-*       wlandev  wlan device structure.
-*       priv     "priv" structure.
-*       hw       "hw" structure.
-*       msg      Message structure.
-*       data     Data buffer.
-*
-* Returns:
-*       0   - Success.
-*       ~0  - Error.
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mib_priv
+ *
+ * Get/set values in the "priv" data structure.
+ *
+ * MIB record parameters:
+ *       parm1    Not used.
+ *       parm2    Not used.
+ *       parm3    Not used.
+ *
+ * Arguments:
+ *       mib      MIB record.
+ *       isget    MIBGET/MIBSET flag.
+ *       wlandev  wlan device structure.
+ *       priv     "priv" structure.
+ *       hw       "hw" structure.
+ *       msg      Message structure.
+ *       data     Data buffer.
+ *
+ * Returns:
+ *       0   - Success.
+ *       ~0  - Error.
+ *
+ */
 
 static int prism2mib_priv(struct mibrec *mib,
 			  int isget,
-			  wlandevice_t *wlandev,
-			  hfa384x_t *hw,
+			  struct wlandevice *wlandev,
+			  struct hfa384x *hw,
 			  struct p80211msg_dot11req_mibset *msg, void *data)
 {
-	p80211pstrd_t *pstr = data;
+	struct p80211pstrd *pstr = data;
 
 	switch (mib->did) {
 	case DIDmib_lnx_lnxConfigTable_lnxRSNAIE:{
-			hfa384x_WPAData_t wpa;
+			struct hfa384x_WPAData wpa;
 
 			if (isget) {
 				hfa384x_drvr_getconfig(hw,
 						       HFA384x_RID_CNFWPADATA,
-						       (u8 *) &wpa,
+						       (u8 *)&wpa,
 						       sizeof(wpa));
 				pstr->len = le16_to_cpu(wpa.datalen);
 				memcpy(pstr->data, wpa.data, pstr->len);
@@ -724,7 +724,7 @@ static int prism2mib_priv(struct mibrec *mib,
 
 				hfa384x_drvr_setconfig(hw,
 						       HFA384x_RID_CNFWPADATA,
-						       (u8 *) &wpa,
+						       (u8 *)&wpa,
 						       sizeof(wpa));
 			}
 			break;
@@ -736,67 +736,67 @@ static int prism2mib_priv(struct mibrec *mib,
 	return 0;
 }
 
-/*----------------------------------------------------------------
-* prism2mgmt_pstr2bytestr
-*
-* Convert the pstr data in the WLAN message structure into an hfa384x
-* byte string format.
-*
-* Arguments:
-*	bytestr		hfa384x byte string data type
-*	pstr		wlan message data
-*
-* Returns:
-*	Nothing
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mgmt_pstr2bytestr
+ *
+ * Convert the pstr data in the WLAN message structure into an hfa384x
+ * byte string format.
+ *
+ * Arguments:
+ *	bytestr		hfa384x byte string data type
+ *	pstr		wlan message data
+ *
+ * Returns:
+ * 	Nothing
+ *
+ */
 
 void prism2mgmt_pstr2bytestr(struct hfa384x_bytestr *bytestr,
-			     p80211pstrd_t *pstr)
+			     struct p80211pstrd *pstr)
 {
-	bytestr->len = cpu_to_le16((u16) (pstr->len));
+	bytestr->len = cpu_to_le16((u16)(pstr->len));
 	memcpy(bytestr->data, pstr->data, pstr->len);
 }
 
-/*----------------------------------------------------------------
-* prism2mgmt_bytestr2pstr
-*
-* Convert the data in an hfa384x byte string format into a
-* pstr in the WLAN message.
-*
-* Arguments:
-*	bytestr		hfa384x byte string data type
-*	msg		wlan message
-*
-* Returns:
-*	Nothing
-*
-----------------------------------------------------------------*/
+/*
+ * prism2mgmt_bytestr2pstr
+ *
+ * Convert the data in an hfa384x byte string format into a
+ * pstr in the WLAN message.
+ *
+ * Arguments:
+ *	bytestr		hfa384x byte string data type
+ *	msg		wlan message
+ *
+ * Returns:
+ *	Nothing
+ *
+ */
 
 void prism2mgmt_bytestr2pstr(struct hfa384x_bytestr *bytestr,
-			     p80211pstrd_t *pstr)
+			     struct p80211pstrd *pstr)
 {
-	pstr->len = (u8) (le16_to_cpu((u16) (bytestr->len)));
+	pstr->len = (u8)(le16_to_cpu((u16)(bytestr->len)));
 	memcpy(pstr->data, bytestr->data, pstr->len);
 }
 
-/*----------------------------------------------------------------
-* prism2mgmt_bytearea2pstr
-*
-* Convert the data in an hfa384x byte area format into a pstr
-* in the WLAN message.
-*
-* Arguments:
-*	bytearea	hfa384x byte area data type
-*	msg		wlan message
-*
-* Returns:
-*	Nothing
-*
-----------------------------------------------------------------*/
-
-void prism2mgmt_bytearea2pstr(u8 *bytearea, p80211pstrd_t *pstr, int len)
+/*
+ * prism2mgmt_bytearea2pstr
+ *
+ * Convert the data in an hfa384x byte area format into a pstr
+ * in the WLAN message.
+ *
+ * Arguments:
+ *	bytearea	hfa384x byte area data type
+ *	msg		wlan message
+ *
+ * Returns:
+ *	Nothing
+ *
+ */
+
+void prism2mgmt_bytearea2pstr(u8 *bytearea, struct p80211pstrd *pstr, int len)
 {
-	pstr->len = (u8) len;
+	pstr->len = (u8)len;
 	memcpy(pstr->data, bytearea, len);
 }
diff --git a/drivers/staging/wlan-ng/prism2sta.c b/drivers/staging/wlan-ng/prism2sta.c
index 64f90722b01b..e1b4a94292ff 100644
--- a/drivers/staging/wlan-ng/prism2sta.c
+++ b/drivers/staging/wlan-ng/prism2sta.c
@@ -81,7 +81,7 @@
 #include "prism2mgmt.h"
 
 static char *dev_info = "prism2_usb";
-static wlandevice_t *create_wlan(void);
+static struct wlandevice *create_wlan(void);
 
 int prism2_reset_holdtime = 30;	/* Reset hold time in ms */
 int prism2_reset_settletime = 100;	/* Reset settle time in ms */
@@ -98,37 +98,38 @@ MODULE_PARM_DESC(prism2_reset_settletime, "reset settle time in ms");
 
 MODULE_LICENSE("Dual MPL/GPL");
 
-static int prism2sta_open(wlandevice_t *wlandev);
-static int prism2sta_close(wlandevice_t *wlandev);
-static void prism2sta_reset(wlandevice_t *wlandev);
-static int prism2sta_txframe(wlandevice_t *wlandev, struct sk_buff *skb,
+static int prism2sta_open(struct wlandevice *wlandev);
+static int prism2sta_close(struct wlandevice *wlandev);
+static void prism2sta_reset(struct wlandevice *wlandev);
+static int prism2sta_txframe(struct wlandevice *wlandev, struct sk_buff *skb,
 			     union p80211_hdr *p80211_hdr,
 			     struct p80211_metawep *p80211_wep);
-static int prism2sta_mlmerequest(wlandevice_t *wlandev, struct p80211msg *msg);
-static int prism2sta_getcardinfo(wlandevice_t *wlandev);
-static int prism2sta_globalsetup(wlandevice_t *wlandev);
-static int prism2sta_setmulticast(wlandevice_t *wlandev, netdevice_t *dev);
-
-static void prism2sta_inf_handover(wlandevice_t *wlandev,
-				   hfa384x_InfFrame_t *inf);
-static void prism2sta_inf_tallies(wlandevice_t *wlandev,
-				  hfa384x_InfFrame_t *inf);
-static void prism2sta_inf_hostscanresults(wlandevice_t *wlandev,
-					  hfa384x_InfFrame_t *inf);
-static void prism2sta_inf_scanresults(wlandevice_t *wlandev,
-				      hfa384x_InfFrame_t *inf);
-static void prism2sta_inf_chinforesults(wlandevice_t *wlandev,
-					hfa384x_InfFrame_t *inf);
-static void prism2sta_inf_linkstatus(wlandevice_t *wlandev,
-				     hfa384x_InfFrame_t *inf);
-static void prism2sta_inf_assocstatus(wlandevice_t *wlandev,
-				      hfa384x_InfFrame_t *inf);
-static void prism2sta_inf_authreq(wlandevice_t *wlandev,
-				  hfa384x_InfFrame_t *inf);
-static void prism2sta_inf_authreq_defer(wlandevice_t *wlandev,
-					hfa384x_InfFrame_t *inf);
-static void prism2sta_inf_psusercnt(wlandevice_t *wlandev,
-				    hfa384x_InfFrame_t *inf);
+static int prism2sta_mlmerequest(struct wlandevice *wlandev, struct p80211msg *msg);
+static int prism2sta_getcardinfo(struct wlandevice *wlandev);
+static int prism2sta_globalsetup(struct wlandevice *wlandev);
+static int prism2sta_setmulticast(struct wlandevice *wlandev,
+				  struct net_device *dev);
+
+static void prism2sta_inf_handover(struct wlandevice *wlandev,
+				   struct hfa384x_InfFrame *inf);
+static void prism2sta_inf_tallies(struct wlandevice *wlandev,
+				  struct hfa384x_InfFrame *inf);
+static void prism2sta_inf_hostscanresults(struct wlandevice *wlandev,
+					  struct hfa384x_InfFrame *inf);
+static void prism2sta_inf_scanresults(struct wlandevice *wlandev,
+				      struct hfa384x_InfFrame *inf);
+static void prism2sta_inf_chinforesults(struct wlandevice *wlandev,
+					struct hfa384x_InfFrame *inf);
+static void prism2sta_inf_linkstatus(struct wlandevice *wlandev,
+				     struct hfa384x_InfFrame *inf);
+static void prism2sta_inf_assocstatus(struct wlandevice *wlandev,
+				      struct hfa384x_InfFrame *inf);
+static void prism2sta_inf_authreq(struct wlandevice *wlandev,
+				  struct hfa384x_InfFrame *inf);
+static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
+					struct hfa384x_InfFrame *inf);
+static void prism2sta_inf_psusercnt(struct wlandevice *wlandev,
+				    struct hfa384x_InfFrame *inf);
 
 /*
  * prism2sta_open
@@ -151,7 +152,7 @@ static void prism2sta_inf_psusercnt(wlandevice_t *wlandev,
  * Call context:
  *	process thread
  */
-static int prism2sta_open(wlandevice_t *wlandev)
+static int prism2sta_open(struct wlandevice *wlandev)
 {
 	/* We don't currently have to do anything else.
 	 * The setup of the MAC should be subsequently completed via
@@ -185,7 +186,7 @@ static int prism2sta_open(wlandevice_t *wlandev)
  * Call context:
  *	process thread
  */
-static int prism2sta_close(wlandevice_t *wlandev)
+static int prism2sta_close(struct wlandevice *wlandev)
 {
 	/* We don't currently have to do anything else.
 	 * Higher layers know we're not ready from dev->start==0 and
@@ -213,7 +214,7 @@ static int prism2sta_close(wlandevice_t *wlandev)
  * Call context:
  *	process thread
  */
-static void prism2sta_reset(wlandevice_t *wlandev)
+static void prism2sta_reset(struct wlandevice *wlandev)
 {
 }
 
@@ -238,11 +239,11 @@ static void prism2sta_reset(wlandevice_t *wlandev)
  * Call context:
  *	process thread
  */
-static int prism2sta_txframe(wlandevice_t *wlandev, struct sk_buff *skb,
+static int prism2sta_txframe(struct wlandevice *wlandev, struct sk_buff *skb,
 			     union p80211_hdr *p80211_hdr,
 			     struct p80211_metawep *p80211_wep)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 
 	/* If necessary, set the 802.11 WEP bit */
 	if ((wlandev->hostwep & (HOSTWEP_PRIVACYINVOKED | HOSTWEP_ENCRYPT)) ==
@@ -277,9 +278,9 @@ static int prism2sta_txframe(wlandevice_t *wlandev, struct sk_buff *skb,
  * Call context:
  *	process thread
  */
-static int prism2sta_mlmerequest(wlandevice_t *wlandev, struct p80211msg *msg)
+static int prism2sta_mlmerequest(struct wlandevice *wlandev, struct p80211msg *msg)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 
 	int result = 0;
 
@@ -337,7 +338,7 @@ static int prism2sta_mlmerequest(wlandevice_t *wlandev, struct p80211msg *msg)
 			struct p80211msg_lnxreq_ifstate *ifstatemsg;
 
 			pr_debug("Received mlme ifstate request\n");
-			ifstatemsg = (struct p80211msg_lnxreq_ifstate *) msg;
+			ifstatemsg = (struct p80211msg_lnxreq_ifstate *)msg;
 			result =
 			    prism2sta_ifstate(wlandev,
 					      ifstatemsg->ifstate.data);
@@ -360,7 +361,7 @@ static int prism2sta_mlmerequest(wlandevice_t *wlandev, struct p80211msg *msg)
 
 			pr_debug("Received commsquality request\n");
 
-			qualmsg = (struct p80211msg_lnxreq_commsquality *) msg;
+			qualmsg = (struct p80211msg_lnxreq_commsquality *)msg;
 
 			qualmsg->link.status =
 			    P80211ENUM_msgitem_status_data_ok;
@@ -407,9 +408,9 @@ static int prism2sta_mlmerequest(wlandevice_t *wlandev, struct p80211msg *msg)
  *	process thread  (usually)
  *	interrupt
  */
-u32 prism2sta_ifstate(wlandevice_t *wlandev, u32 ifstate)
+u32 prism2sta_ifstate(struct wlandevice *wlandev, u32 ifstate)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	u32 result;
 
 	result = P80211ENUM_resultcode_implementation_failure;
@@ -580,10 +581,10 @@ u32 prism2sta_ifstate(wlandevice_t *wlandev, u32 ifstate)
  * Call context:
  *	Either.
  */
-static int prism2sta_getcardinfo(wlandevice_t *wlandev)
+static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 {
 	int result = 0;
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	u16 temp;
 	u8 snum[HFA384x_RID_NICSERIALNUMBER_LEN];
 
@@ -592,7 +593,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* NIC identity */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_NICIDENTITY,
 					&hw->ident_nic,
-					sizeof(hfa384x_compident_t));
+					sizeof(struct hfa384x_compident));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve NICIDENTITY\n");
 		goto failed;
@@ -611,7 +612,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* Primary f/w identity */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_PRIIDENTITY,
 					&hw->ident_pri_fw,
-					sizeof(hfa384x_compident_t));
+					sizeof(struct hfa384x_compident));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve PRIIDENTITY\n");
 		goto failed;
@@ -630,7 +631,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* Station (Secondary?) f/w identity */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_STAIDENTITY,
 					&hw->ident_sta_fw,
-					sizeof(hfa384x_compident_t));
+					sizeof(struct hfa384x_compident));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve STAIDENTITY\n");
 		goto failed;
@@ -651,7 +652,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 
 	/* strip out the 'special' variant bits */
 	hw->mm_mods = hw->ident_sta_fw.variant & (BIT(14) | BIT(15));
-	hw->ident_sta_fw.variant &= ~((u16) (BIT(14) | BIT(15)));
+	hw->ident_sta_fw.variant &= ~((u16)(BIT(14) | BIT(15)));
 
 	if (hw->ident_sta_fw.id == 0x1f) {
 		netdev_info(wlandev->netdev,
@@ -670,7 +671,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* Compatibility range, Modem supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_MFISUPRANGE,
 					&hw->cap_sup_mfi,
-					sizeof(hfa384x_caplevel_t));
+					sizeof(struct hfa384x_caplevel));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve MFISUPRANGE\n");
 		goto failed;
@@ -694,7 +695,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* Compatibility range, Controller supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_CFISUPRANGE,
 					&hw->cap_sup_cfi,
-					sizeof(hfa384x_caplevel_t));
+					sizeof(struct hfa384x_caplevel));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve CFISUPRANGE\n");
 		goto failed;
@@ -718,7 +719,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* Compatibility range, Primary f/w supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_PRISUPRANGE,
 					&hw->cap_sup_pri,
-					sizeof(hfa384x_caplevel_t));
+					sizeof(struct hfa384x_caplevel));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve PRISUPRANGE\n");
 		goto failed;
@@ -742,7 +743,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* Compatibility range, Station f/w supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_STASUPRANGE,
 					&hw->cap_sup_sta,
-					sizeof(hfa384x_caplevel_t));
+					sizeof(struct hfa384x_caplevel));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve STASUPRANGE\n");
 		goto failed;
@@ -774,7 +775,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* Compatibility range, primary f/w actor, CFI supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_PRI_CFIACTRANGES,
 					&hw->cap_act_pri_cfi,
-					sizeof(hfa384x_caplevel_t));
+					sizeof(struct hfa384x_caplevel));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve PRI_CFIACTRANGES\n");
 		goto failed;
@@ -798,7 +799,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* Compatibility range, sta f/w actor, CFI supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_STA_CFIACTRANGES,
 					&hw->cap_act_sta_cfi,
-					sizeof(hfa384x_caplevel_t));
+					sizeof(struct hfa384x_caplevel));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve STA_CFIACTRANGES\n");
 		goto failed;
@@ -822,7 +823,7 @@ static int prism2sta_getcardinfo(wlandevice_t *wlandev)
 	/* Compatibility range, sta f/w actor, MFI supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_STA_MFIACTRANGES,
 					&hw->cap_act_sta_mfi,
-					sizeof(hfa384x_caplevel_t));
+					sizeof(struct hfa384x_caplevel));
 	if (result) {
 		netdev_err(wlandev->netdev, "Failed to retrieve STA_MFIACTRANGES\n");
 		goto failed;
@@ -909,19 +910,20 @@ done:
  * Call context:
  *	process thread
  */
-static int prism2sta_globalsetup(wlandevice_t *wlandev)
+static int prism2sta_globalsetup(struct wlandevice *wlandev)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 
 	/* Set the maximum frame size */
 	return hfa384x_drvr_setconfig16(hw, HFA384x_RID_CNFMAXDATALEN,
 					WLAN_DATA_MAXLEN);
 }
 
-static int prism2sta_setmulticast(wlandevice_t *wlandev, netdevice_t *dev)
+static int prism2sta_setmulticast(struct wlandevice *wlandev,
+					struct net_device *dev)
 {
 	int result = 0;
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 
 	u16 promisc;
 
@@ -959,8 +961,8 @@ exit:
  * Call context:
  *	interrupt
  */
-static void prism2sta_inf_handover(wlandevice_t *wlandev,
-				   hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_handover(struct wlandevice *wlandev,
+				   struct hfa384x_InfFrame *inf)
 {
 	pr_debug("received infoframe:HANDOVER (unhandled)\n");
 }
@@ -982,10 +984,10 @@ static void prism2sta_inf_handover(wlandevice_t *wlandev,
  * Call context:
  *	interrupt
  */
-static void prism2sta_inf_tallies(wlandevice_t *wlandev,
-				  hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_tallies(struct wlandevice *wlandev,
+				  struct hfa384x_InfFrame *inf)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	u16 *src16;
 	u32 *dst;
 	u32 *src32;
@@ -997,15 +999,15 @@ static void prism2sta_inf_tallies(wlandevice_t *wlandev,
 	 * record length of the info record.
 	 */
 
-	cnt = sizeof(hfa384x_CommTallies32_t) / sizeof(u32);
+	cnt = sizeof(struct hfa384x_CommTallies32) / sizeof(u32);
 	if (inf->framelen > 22) {
-		dst = (u32 *) &hw->tallies;
-		src32 = (u32 *) &inf->info.commtallies32;
+		dst = (u32 *)&hw->tallies;
+		src32 = (u32 *)&inf->info.commtallies32;
 		for (i = 0; i < cnt; i++, dst++, src32++)
 			*dst += le32_to_cpu(*src32);
 	} else {
-		dst = (u32 *) &hw->tallies;
-		src16 = (u16 *) &inf->info.commtallies16;
+		dst = (u32 *)&hw->tallies;
+		src16 = (u16 *)&inf->info.commtallies16;
 		for (i = 0; i < cnt; i++, dst++, src16++)
 			*dst += le16_to_cpu(*src16);
 	}
@@ -1028,21 +1030,20 @@ static void prism2sta_inf_tallies(wlandevice_t *wlandev,
  * Call context:
  *	interrupt
  */
-static void prism2sta_inf_scanresults(wlandevice_t *wlandev,
-				      hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_scanresults(struct wlandevice *wlandev,
+				      struct hfa384x_InfFrame *inf)
 {
-
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	int nbss;
-	hfa384x_ScanResult_t *sr = &(inf->info.scanresult);
+	struct hfa384x_ScanResult *sr = &(inf->info.scanresult);
 	int i;
-	hfa384x_JoinRequest_data_t joinreq;
+	struct hfa384x_JoinRequest_data joinreq;
 	int result;
 
 	/* Get the number of results, first in bytes, then in results */
 	nbss = (inf->framelen * sizeof(u16)) -
 	    sizeof(inf->infotype) - sizeof(inf->info.scanresult.scanreason);
-	nbss /= sizeof(hfa384x_ScanResultSub_t);
+	nbss /= sizeof(struct hfa384x_ScanResultSub);
 
 	/* Print em */
 	pr_debug("rx scanresults, reason=%d, nbss=%d:\n",
@@ -1084,10 +1085,10 @@ static void prism2sta_inf_scanresults(wlandevice_t *wlandev,
  * Call context:
  *	interrupt
  */
-static void prism2sta_inf_hostscanresults(wlandevice_t *wlandev,
-					  hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_hostscanresults(struct wlandevice *wlandev,
+					  struct hfa384x_InfFrame *inf)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	int nbss;
 
 	nbss = (inf->framelen - 3) / 32;
@@ -1098,7 +1099,7 @@ static void prism2sta_inf_hostscanresults(wlandevice_t *wlandev,
 
 	kfree(hw->scanresults);
 
-	hw->scanresults = kmemdup(inf, sizeof(hfa384x_InfFrame_t), GFP_ATOMIC);
+	hw->scanresults = kmemdup(inf, sizeof(struct hfa384x_InfFrame), GFP_ATOMIC);
 
 	if (nbss == 0)
 		nbss = -1;
@@ -1125,18 +1126,18 @@ static void prism2sta_inf_hostscanresults(wlandevice_t *wlandev,
  * Call context:
  *	interrupt
  */
-static void prism2sta_inf_chinforesults(wlandevice_t *wlandev,
-					hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_chinforesults(struct wlandevice *wlandev,
+					struct hfa384x_InfFrame *inf)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	unsigned int i, n;
 
 	hw->channel_info.results.scanchannels =
 	    le16_to_cpu(inf->info.chinforesult.scanchannels);
 
 	for (i = 0, n = 0; i < HFA384x_CHINFORESULT_MAX; i++) {
-		hfa384x_ChInfoResultSub_t *result;
-		hfa384x_ChInfoResultSub_t *chinforesult;
+		struct hfa384x_ChInfoResultSub *result;
+		struct hfa384x_ChInfoResultSub *chinforesult;
 		int chan;
 
 		if (!(hw->channel_info.results.scanchannels & (1 << i)))
@@ -1170,18 +1171,18 @@ static void prism2sta_inf_chinforesults(wlandevice_t *wlandev,
 
 void prism2sta_processing_defer(struct work_struct *data)
 {
-	hfa384x_t *hw = container_of(data, struct hfa384x, link_bh);
-	wlandevice_t *wlandev = hw->wlandev;
-	hfa384x_bytestr32_t ssid;
+	struct hfa384x *hw = container_of(data, struct hfa384x, link_bh);
+	struct wlandevice *wlandev = hw->wlandev;
+	struct hfa384x_bytestr32 ssid;
 	int result;
 
 	/* First let's process the auth frames */
 	{
 		struct sk_buff *skb;
-		hfa384x_InfFrame_t *inf;
+		struct hfa384x_InfFrame *inf;
 
 		while ((skb = skb_dequeue(&hw->authq))) {
-			inf = (hfa384x_InfFrame_t *) skb->data;
+			inf = (struct hfa384x_InfFrame *)skb->data;
 			prism2sta_inf_authreq_defer(wlandev, inf);
 		}
 
@@ -1256,8 +1257,8 @@ void prism2sta_processing_defer(struct work_struct *data)
 				return;
 			}
 			prism2mgmt_bytestr2pstr(
-					(struct hfa384x_bytestr *) &ssid,
-					(p80211pstrd_t *) &wlandev->ssid);
+					(struct hfa384x_bytestr *)&ssid,
+					(struct p80211pstrd *)&wlandev->ssid);
 
 			/* Collect the port status */
 			result = hfa384x_drvr_getconfig16(hw,
@@ -1337,8 +1338,8 @@ void prism2sta_processing_defer(struct work_struct *data)
 				 HFA384x_RID_CURRENTSSID, result);
 			return;
 		}
-		prism2mgmt_bytestr2pstr((struct hfa384x_bytestr *) &ssid,
-					(p80211pstrd_t *) &wlandev->ssid);
+		prism2mgmt_bytestr2pstr((struct hfa384x_bytestr *)&ssid,
+					(struct p80211pstrd *)&wlandev->ssid);
 
 		hw->link_status = HFA384x_LINK_CONNECTED;
 		netif_carrier_on(wlandev->netdev);
@@ -1390,7 +1391,7 @@ void prism2sta_processing_defer(struct work_struct *data)
 		 * Disable Transmits, Ignore receives of data frames
 		 */
 		if (hw->join_ap && --hw->join_retries > 0) {
-			hfa384x_JoinRequest_data_t joinreq;
+			struct hfa384x_JoinRequest_data joinreq;
 
 			joinreq = hw->joinreq;
 			/* Send the join request */
@@ -1438,10 +1439,10 @@ void prism2sta_processing_defer(struct work_struct *data)
  * Call context:
  *	interrupt
  */
-static void prism2sta_inf_linkstatus(wlandevice_t *wlandev,
-				     hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_linkstatus(struct wlandevice *wlandev,
+				     struct hfa384x_InfFrame *inf)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 
 	hw->link_status_new = le16_to_cpu(inf->info.linkstatus.linkstatus);
 
@@ -1466,11 +1467,11 @@ static void prism2sta_inf_linkstatus(wlandevice_t *wlandev,
  * Call context:
  *	interrupt
  */
-static void prism2sta_inf_assocstatus(wlandevice_t *wlandev,
-				      hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_assocstatus(struct wlandevice *wlandev,
+				      struct hfa384x_InfFrame *inf)
 {
-	hfa384x_t *hw = wlandev->priv;
-	hfa384x_AssocStatus_t rec;
+	struct hfa384x *hw = wlandev->priv;
+	struct hfa384x_AssocStatus rec;
 	int i;
 
 	memcpy(&rec, &inf->info.assocstatus, sizeof(rec));
@@ -1527,10 +1528,10 @@ static void prism2sta_inf_assocstatus(wlandevice_t *wlandev,
  *	interrupt
  *
  */
-static void prism2sta_inf_authreq(wlandevice_t *wlandev,
-				  hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_authreq(struct wlandevice *wlandev,
+				  struct hfa384x_InfFrame *inf)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 	struct sk_buff *skb;
 
 	skb = dev_alloc_skb(sizeof(*inf));
@@ -1542,11 +1543,11 @@ static void prism2sta_inf_authreq(wlandevice_t *wlandev,
 	}
 }
 
-static void prism2sta_inf_authreq_defer(wlandevice_t *wlandev,
-					hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
+					struct hfa384x_InfFrame *inf)
 {
-	hfa384x_t *hw = wlandev->priv;
-	hfa384x_authenticateStation_data_t rec;
+	struct hfa384x *hw = wlandev->priv;
+	struct hfa384x_authenticateStation_data rec;
 
 	int i, added, result, cnt;
 	u8 *addr;
@@ -1716,10 +1717,10 @@ static void prism2sta_inf_authreq_defer(wlandevice_t *wlandev,
  * Call context:
  *	interrupt
  */
-static void prism2sta_inf_psusercnt(wlandevice_t *wlandev,
-				    hfa384x_InfFrame_t *inf)
+static void prism2sta_inf_psusercnt(struct wlandevice *wlandev,
+				    struct hfa384x_InfFrame *inf)
 {
-	hfa384x_t *hw = wlandev->priv;
+	struct hfa384x *hw = wlandev->priv;
 
 	hw->psusercount = le16_to_cpu(inf->info.psusercnt.usercnt);
 }
@@ -1741,7 +1742,7 @@ static void prism2sta_inf_psusercnt(wlandevice_t *wlandev,
  * Call context:
  *	interrupt
  */
-void prism2sta_ev_info(wlandevice_t *wlandev, hfa384x_InfFrame_t *inf)
+void prism2sta_ev_info(struct wlandevice *wlandev, struct hfa384x_InfFrame *inf)
 {
 	inf->infotype = le16_to_cpu(inf->infotype);
 	/* Dispatch */
@@ -1808,7 +1809,7 @@ void prism2sta_ev_info(wlandevice_t *wlandev, hfa384x_InfFrame_t *inf)
  * Call context:
  *	interrupt
  */
-void prism2sta_ev_txexc(wlandevice_t *wlandev, u16 status)
+void prism2sta_ev_txexc(struct wlandevice *wlandev, u16 status)
 {
 	pr_debug("TxExc status=0x%x.\n", status);
 }
@@ -1829,7 +1830,7 @@ void prism2sta_ev_txexc(wlandevice_t *wlandev, u16 status)
  * Call context:
  *	interrupt
  */
-void prism2sta_ev_tx(wlandevice_t *wlandev, u16 status)
+void prism2sta_ev_tx(struct wlandevice *wlandev, u16 status)
 {
 	pr_debug("Tx Complete, status=0x%04x\n", status);
 	/* update linux network stats */
@@ -1852,7 +1853,7 @@ void prism2sta_ev_tx(wlandevice_t *wlandev, u16 status)
  * Call context:
  *	interrupt
  */
-void prism2sta_ev_alloc(wlandevice_t *wlandev)
+void prism2sta_ev_alloc(struct wlandevice *wlandev)
 {
 	netif_wake_queue(wlandev->netdev);
 }
@@ -1860,14 +1861,14 @@ void prism2sta_ev_alloc(wlandevice_t *wlandev)
 /*
 * create_wlan
 *
-* Called at module init time.  This creates the wlandevice_t structure
+* Called at module init time.  This creates the struct wlandevice structure
 * and initializes it with relevant bits.
 *
 * Arguments:
 *	none
 *
 * Returns:
-*	the created wlandevice_t structure.
+*	the created struct wlandevice structure.
 *
 * Side effects:
 *	also allocates the priv/hw structures.
@@ -1876,14 +1877,14 @@ void prism2sta_ev_alloc(wlandevice_t *wlandev)
 *	process thread
 *
 */
-static wlandevice_t *create_wlan(void)
+static struct wlandevice *create_wlan(void)
 {
-	wlandevice_t *wlandev = NULL;
-	hfa384x_t *hw = NULL;
+	struct wlandevice *wlandev = NULL;
+	struct hfa384x *hw = NULL;
 
 	/* Alloc our structures */
-	wlandev = kzalloc(sizeof(wlandevice_t), GFP_KERNEL);
-	hw = kzalloc(sizeof(hfa384x_t), GFP_KERNEL);
+	wlandev = kzalloc(sizeof(struct wlandevice), GFP_KERNEL);
+	hw = kzalloc(sizeof(struct hfa384x), GFP_KERNEL);
 
 	if (!wlandev || !hw) {
 		kfree(wlandev);
@@ -1913,11 +1914,11 @@ static wlandevice_t *create_wlan(void)
 
 void prism2sta_commsqual_defer(struct work_struct *data)
 {
-	hfa384x_t *hw = container_of(data, struct hfa384x, commsqual_bh);
-	wlandevice_t *wlandev = hw->wlandev;
-	hfa384x_bytestr32_t ssid;
+	struct hfa384x *hw = container_of(data, struct hfa384x, commsqual_bh);
+	struct wlandevice *wlandev = hw->wlandev;
+	struct hfa384x_bytestr32 ssid;
 	struct p80211msg_dot11req_mibget msg;
-	p80211item_uint32_t *mibitem = (p80211item_uint32_t *)
+	struct p80211item_uint32 *mibitem = (struct p80211item_uint32 *)
 						&msg.mibattribute.data;
 	int result = 0;
 
@@ -1950,7 +1951,7 @@ void prism2sta_commsqual_defer(struct work_struct *data)
 	/* Get the signal rate */
 	msg.msgcode = DIDmsg_dot11req_mibget;
 	mibitem->did = DIDmib_p2_p2MAC_p2CurrentTxRate;
-	result = p80211req_dorequest(wlandev, (u8 *) &msg);
+	result = p80211req_dorequest(wlandev, (u8 *)&msg);
 
 	if (result) {
 		pr_debug("get signal rate failed, result = %d\n",
@@ -1993,8 +1994,8 @@ void prism2sta_commsqual_defer(struct work_struct *data)
 			 HFA384x_RID_CURRENTSSID, result);
 		return;
 	}
-	prism2mgmt_bytestr2pstr((struct hfa384x_bytestr *) &ssid,
-				(p80211pstrd_t *) &wlandev->ssid);
+	prism2mgmt_bytestr2pstr((struct hfa384x_bytestr *)&ssid,
+				(struct p80211pstrd *)&wlandev->ssid);
 
 	/* Reschedule timer */
 	mod_timer(&hw->commsqual_timer, jiffies + HZ);
@@ -2002,7 +2003,7 @@ void prism2sta_commsqual_defer(struct work_struct *data)
 
 void prism2sta_commsqual_timer(unsigned long data)
 {
-	hfa384x_t *hw = (hfa384x_t *) data;
+	struct hfa384x *hw = (struct hfa384x *)data;
 
 	schedule_work(&hw->commsqual_bh);
 }
diff --git a/drivers/staging/wlan-ng/prism2usb.c b/drivers/staging/wlan-ng/prism2usb.c
index b26d09ff840c..bfb6b0a6528d 100644
--- a/drivers/staging/wlan-ng/prism2usb.c
+++ b/drivers/staging/wlan-ng/prism2usb.c
@@ -47,11 +47,11 @@ static const struct usb_device_id usb_prism_tbl[] = {
 	PRISM_DEV(0x0bb2, 0x0302, "Ambit Microsystems Corp."),
 	PRISM_DEV(0x9016, 0x182d, "Sitecom WL-022 802.11b USB Adapter"),
 	PRISM_DEV(0x0543, 0x0f01,
-		"ViewSonic Airsync USB Adapter 11Mbps (Prism2.5)"),
+		  "ViewSonic Airsync USB Adapter 11Mbps (Prism2.5)"),
 	PRISM_DEV(0x067c, 0x1022,
-		"Siemens SpeedStream 1022 11Mbps WLAN USB Adapter"),
+		  "Siemens SpeedStream 1022 11Mbps WLAN USB Adapter"),
 	PRISM_DEV(0x049f, 0x0033,
-		"Compaq/Intel W100 PRO/Wireless 11Mbps multiport WLAN Adapter"),
+		  "Compaq/Intel W100 PRO/Wireless 11Mbps multiport WLAN Adapter"),
 	{ } /* terminator */
 };
 MODULE_DEVICE_TABLE(usb, usb_prism_tbl);
@@ -61,8 +61,8 @@ static int prism2sta_probe_usb(struct usb_interface *interface,
 {
 	struct usb_device *dev;
 
-	wlandevice_t *wlandev = NULL;
-	hfa384x_t *hw = NULL;
+	struct wlandevice *wlandev = NULL;
+	struct hfa384x *hw = NULL;
 	int result = 0;
 
 	dev = interface_to_usbdev(interface);
@@ -74,7 +74,7 @@ static int prism2sta_probe_usb(struct usb_interface *interface,
 	}
 	hw = wlandev->priv;
 
-	if (wlan_setup(wlandev, &(interface->dev)) != 0) {
+	if (wlan_setup(wlandev, &interface->dev) != 0) {
 		dev_err(&interface->dev, "wlan_setup() failed.\n");
 		result = -EIO;
 		goto failed;
@@ -87,7 +87,7 @@ static int prism2sta_probe_usb(struct usb_interface *interface,
 	/* Register the wlandev, this gets us a name and registers the
 	 * linux netdevice.
 	 */
-	SET_NETDEV_DEV(wlandev->netdev, &(interface->dev));
+	SET_NETDEV_DEV(wlandev->netdev, &interface->dev);
 
 	/* Do a chip-level reset on the MAC */
 	if (prism2_doreset) {
@@ -134,15 +134,15 @@ done:
 
 static void prism2sta_disconnect_usb(struct usb_interface *interface)
 {
-	wlandevice_t *wlandev;
+	struct wlandevice *wlandev;
 
-	wlandev = (wlandevice_t *)usb_get_intfdata(interface);
-	if (wlandev != NULL) {
+	wlandev = (struct wlandevice *)usb_get_intfdata(interface);
+	if (wlandev) {
 		LIST_HEAD(cleanlist);
-		hfa384x_usbctlx_t *ctlx, *temp;
+		struct hfa384x_usbctlx *ctlx, *temp;
 		unsigned long flags;
 
-		hfa384x_t *hw = wlandev->priv;
+		struct hfa384x *hw = wlandev->priv;
 
 		if (!hw)
 			goto exit;
@@ -216,12 +216,12 @@ exit:
 
 #ifdef CONFIG_PM
 static int prism2sta_suspend(struct usb_interface *interface,
-				pm_message_t message)
+			     pm_message_t message)
 {
-	hfa384x_t *hw = NULL;
-	wlandevice_t *wlandev;
+	struct hfa384x *hw = NULL;
+	struct wlandevice *wlandev;
 
-	wlandev = (wlandevice_t *)usb_get_intfdata(interface);
+	wlandev = (struct wlandevice *)usb_get_intfdata(interface);
 	if (!wlandev)
 		return -ENODEV;
 
@@ -241,10 +241,10 @@ static int prism2sta_suspend(struct usb_interface *interface,
 static int prism2sta_resume(struct usb_interface *interface)
 {
 	int result = 0;
-	hfa384x_t *hw = NULL;
-	wlandevice_t *wlandev;
+	struct hfa384x *hw = NULL;
+	struct wlandevice *wlandev;
 
-	wlandev = (wlandevice_t *)usb_get_intfdata(interface);
+	wlandev = (struct wlandevice *)usb_get_intfdata(interface);
 	if (!wlandev)
 		return -ENODEV;
 
diff --git a/drivers/staging/xgifb/XGI_main_26.c b/drivers/staging/xgifb/XGI_main_26.c
index d56ef1425f6b..0c78491ff5a1 100644
--- a/drivers/staging/xgifb/XGI_main_26.c
+++ b/drivers/staging/xgifb/XGI_main_26.c
@@ -62,7 +62,6 @@ static int XGIfb_mode_rate_to_dclock(struct vb_device_info *XGI_Pr,
 	unsigned short ModeNo = modeno;
 	unsigned short ModeIdIndex = 0, ClockIndex = 0;
 	unsigned short RefreshRateTableIndex = 0;
-	int Clock;
 
 	InitTo330Pointer(HwDeviceExtension->jChipType, XGI_Pr);
 
@@ -73,9 +72,7 @@ static int XGIfb_mode_rate_to_dclock(struct vb_device_info *XGI_Pr,
 
 	ClockIndex = XGI330_RefIndex[RefreshRateTableIndex].Ext_CRTVCLK;
 
-	Clock = XGI_VCLKData[ClockIndex].CLOCK * 1000;
-
-	return Clock;
+	return XGI_VCLKData[ClockIndex].CLOCK * 1000;
 }
 
 static int XGIfb_mode_rate_to_ddata(struct vb_device_info *XGI_Pr,
@@ -1227,7 +1224,7 @@ static int XGIfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 	unsigned int vtotal = 0;
 	unsigned int drate = 0, hrate = 0;
 	int found_mode = 0;
-	int refresh_rate, search_idx;
+	int search_idx;
 
 	if ((var->vmode & FB_VMODE_MASK) == FB_VMODE_NONINTERLACED) {
 		vtotal = var->upper_margin + var->yres + var->lower_margin
@@ -1263,10 +1260,6 @@ static int XGIfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 		xgifb_info->refresh_rate = 60;
 	}
 
-	/* Calculation wrong for 1024x600 - force it to 60Hz */
-	if ((var->xres == 1024) && (var->yres == 600))
-		refresh_rate = 60;
-
 	search_idx = 0;
 	while ((XGIbios_mode[search_idx].mode_no != 0) &&
 		(XGIbios_mode[search_idx].xres <= var->xres)) {
@@ -2085,7 +2078,7 @@ static int __init xgifb_init(void)
 {
 	char *option = NULL;
 
-	if (forcecrt2type != NULL)
+	if (forcecrt2type)
 		XGIfb_search_crt2type(forcecrt2type);
 	if (fb_get_options("xgifb", &option))
 		return -ENODEV;
diff --git a/drivers/staging/xgifb/vb_setmode.c b/drivers/staging/xgifb/vb_setmode.c
index 50c8ea4f5ab7..d8010c5c1a70 100644
--- a/drivers/staging/xgifb/vb_setmode.c
+++ b/drivers/staging/xgifb/vb_setmode.c
@@ -214,7 +214,7 @@ static unsigned char XGI_AjustCRT2Rate(unsigned short ModeIdIndex,
 		if (!(pVBInfo->VBInfo & TVSetPAL) &&
 		    (modeflag & NoSupportSimuTV) &&
 		    (pVBInfo->VBInfo & SetInSlaveMode) &&
-		    (!(pVBInfo->VBInfo & SetNotSimuMode)))
+		    !(pVBInfo->VBInfo & SetNotSimuMode))
 			return 0;
 	}
 
@@ -1647,7 +1647,6 @@ static void XGI_GetLCDVCLKPtr(unsigned char *di_0, unsigned char *di_1,
 static unsigned char XGI_GetVCLKPtr(unsigned short RefreshRateTableIndex,
 		unsigned short ModeIdIndex, struct vb_device_info *pVBInfo)
 {
-
 	unsigned short index, modeflag;
 	unsigned char tempal;
 
@@ -1655,7 +1654,7 @@ static unsigned char XGI_GetVCLKPtr(unsigned short RefreshRateTableIndex,
 	modeflag = XGI330_EModeIDTable[ModeIdIndex].Ext_ModeFlag;
 
 	if ((pVBInfo->SetFlag & ProgrammingCRT2) &&
-	    (!(pVBInfo->LCDInfo & EnableScalingLCD))) { /* {LCDA/LCDB} */
+	    !(pVBInfo->LCDInfo & EnableScalingLCD)) { /* {LCDA/LCDB} */
 		index = XGI_GetLCDCapPtr(pVBInfo);
 		tempal = pVBInfo->LCDCapList[index].LCD_VCLK;
 
@@ -1678,7 +1677,6 @@ static unsigned char XGI_GetVCLKPtr(unsigned short RefreshRateTableIndex,
 					if (!(modeflag & Charx8Dot))
 						tempal = TVCLKBASE_315 +
 								HiTVTextVCLK;
-
 				}
 				return tempal;
 			}
@@ -1716,7 +1714,7 @@ static void XGI_GetVCLKLen(unsigned char tempal, unsigned char *di_0,
 {
 	if (pVBInfo->VBType & (VB_SIS301 | VB_SIS301B | VB_SIS302B
 			| VB_SIS301LV | VB_SIS302LV | VB_XGI301C)) {
-		if ((!(pVBInfo->VBInfo & XGI_SetCRT2ToLCDA)) &&
+		if (!(pVBInfo->VBInfo & XGI_SetCRT2ToLCDA) &&
 		    (pVBInfo->SetFlag & ProgrammingCRT2)) {
 			*di_0 = XGI_VBVCLKData[tempal].Part4_A;
 			*di_1 = XGI_VBVCLKData[tempal].Part4_B;
@@ -1741,8 +1739,8 @@ static void XGI_SetCRT2ECLK(unsigned short ModeIdIndex,
 	for (i = 0; i < 4; i++) {
 		xgifb_reg_and_or(pVBInfo->P3d4, 0x31, ~0x30,
 				(unsigned short) (0x10 * i));
-		if ((!(pVBInfo->VBInfo & XGI_SetCRT2ToLCDA))
-				&& (!(pVBInfo->VBInfo & SetInSlaveMode))) {
+		if (!(pVBInfo->VBInfo & XGI_SetCRT2ToLCDA) &&
+		    !(pVBInfo->VBInfo & SetInSlaveMode)) {
 			xgifb_reg_set(pVBInfo->P3c4, 0x2e, di_0);
 			xgifb_reg_set(pVBInfo->P3c4, 0x2f, di_1);
 		} else {
@@ -1915,7 +1913,7 @@ static void XGI_GetVBInfo(unsigned short ModeIdIndex,
 		}
 	}
 
-	if (pVBInfo->VBType & (VB_SIS301LV|VB_SIS302LV|VB_XGI301C)) {
+	if (pVBInfo->VBType & (VB_SIS301LV | VB_SIS302LV | VB_XGI301C)) {
 		if (temp & SetYPbPr) {
 			/* shampoo add for new scratch */
 			temp = xgifb_reg_get(pVBInfo->P3d4, 0x35);
@@ -1986,7 +1984,7 @@ static void XGI_GetVBInfo(unsigned short ModeIdIndex,
 	}
 
 	if (!(tempbx & DisableCRT2Display)) {
-		if ((!(tempbx & DriverMode)) || (!(modeflag & CRT2Mode))) {
+		if (!(tempbx & DriverMode) || !(modeflag & CRT2Mode)) {
 			if (!(tempbx & XGI_SetCRT2ToLCDA))
 				tempbx |= (SetInSlaveMode | SetSimuScanMode);
 		}
@@ -2132,7 +2130,7 @@ static unsigned char XGI_GetLCDInfo(unsigned short ModeIdIndex,
 
 	if ((pVBInfo->LCDResInfo == Panel_1400x1050) &&
 	    (pVBInfo->VBInfo & SetCRT2ToLCD) && (resinfo == 9) &&
-	    (!(tempbx & EnableScalingLCD)))
+	    !(tempbx & EnableScalingLCD))
 		/*
 		 * set to center in 1280x1024 LCDB
 		 * for Panel_1400x1050
@@ -2245,7 +2243,6 @@ static void XGI_XG21BLSignalVDD(unsigned short tempbh, unsigned short tempbl,
 
 		/* CR B4[1] */
 		xgifb_reg_and_or(pVBInfo->P3d4, 0xB4, ~0x02, temp);
-
 	}
 
 	temp = xgifb_reg_get(pVBInfo->P3d4, 0x48);
@@ -2274,7 +2271,6 @@ static void XGI_XG27BLSignalVDD(unsigned short tempbh, unsigned short tempbl,
 
 		/* CR B4[1] */
 		xgifb_reg_and_or(pVBInfo->P3d4, 0xB4, ~0x02, temp);
-
 	}
 	xgifb_reg_and_or(pVBInfo->P3d4, 0xB4, ~tempbh0, tempbl0);
 
@@ -2291,7 +2287,6 @@ static void XGI_DisplayOn(struct xgifb_video_info *xgifb_info,
 		struct xgi_hw_device_info *pXGIHWDE,
 		struct vb_device_info *pVBInfo)
 {
-
 	xgifb_reg_and_or(pVBInfo->P3c4, 0x01, 0xDF, 0x00);
 	if (pXGIHWDE->jChipType == XG21) {
 		if (pVBInfo->IF_DEF_LVDS == 1) {
@@ -2310,7 +2305,6 @@ static void XGI_DisplayOn(struct xgifb_video_info *xgifb_info,
 			/* DVO/DVI signal on */
 			XGI_XG21BLSignalVDD(0x20, 0x20, pVBInfo);
 		}
-
 	}
 
 	if (pXGIHWDE->jChipType == XG27) {
@@ -2330,7 +2324,6 @@ static void XGI_DisplayOn(struct xgifb_video_info *xgifb_info,
 			/* DVO/DVI signal on */
 			XGI_XG27BLSignalVDD(0x20, 0x20, pVBInfo);
 		}
-
 	}
 }
 
@@ -2338,7 +2331,6 @@ void XGI_DisplayOff(struct xgifb_video_info *xgifb_info,
 		struct xgi_hw_device_info *pXGIHWDE,
 		struct vb_device_info *pVBInfo)
 {
-
 	if (pXGIHWDE->jChipType == XG21) {
 		if (pVBInfo->IF_DEF_LVDS == 1) {
 			/* LVDS backlight off */
@@ -2455,7 +2447,6 @@ exit:
 
 static unsigned char XGI_IsLCDDualLink(struct vb_device_info *pVBInfo)
 {
-
 	if ((pVBInfo->VBInfo & (SetCRT2ToLCD | XGI_SetCRT2ToLCDA)) &&
 			(pVBInfo->LCDInfo & SetLCDDualLink)) /* shampoo0129 */
 		return 1;
@@ -3922,7 +3913,7 @@ static void XGI_SetTap4Regs(struct vb_device_info *pVBInfo)
 		xgifb_reg_set(pVBInfo->Part2Port, i, Tap4TimingPtr->Reg[j]);
 
 	if ((pVBInfo->VBInfo & SetCRT2ToTV) &&
-	    (!(pVBInfo->VBInfo & SetCRT2ToHiVision))) {
+	    !(pVBInfo->VBInfo & SetCRT2ToHiVision)) {
 		/* Set Vertical Scaling */
 		Tap4TimingPtr = XGI_GetTap4Ptr(1, pVBInfo);
 		for (i = 0xC0, j = 0; i < 0xFF; i++, j++)
@@ -3932,7 +3923,7 @@ static void XGI_SetTap4Regs(struct vb_device_info *pVBInfo)
 	}
 
 	if ((pVBInfo->VBInfo & SetCRT2ToTV) &&
-	    (!(pVBInfo->VBInfo & SetCRT2ToHiVision)))
+	    !(pVBInfo->VBInfo & SetCRT2ToHiVision))
 		/* Enable V.Scaling */
 		xgifb_reg_and_or(pVBInfo->Part2Port, 0x4E, ~0x14, 0x04);
 	else
@@ -4132,7 +4123,6 @@ static void XGI_SetGroup4(unsigned short ModeIdIndex,
 		if (pVBInfo->VBInfo & (SetCRT2ToTV | SetCRT2ToHiVision)) {
 			if (pVBInfo->VGAHDE > 800)
 				xgifb_reg_or(pVBInfo->Part4Port, 0x1E, 0x08);
-
 		}
 		temp = 0x0036;
 
@@ -4141,9 +4131,9 @@ static void XGI_SetGroup4(unsigned short ModeIdIndex,
 					| TVSetYPbPr525p | TVSetYPbPr750p
 					| TVSetHiVision))) {
 				temp |= 0x0001;
-				if ((pVBInfo->VBInfo & SetInSlaveMode)
-						&& (!(pVBInfo->TVInfo
-								& TVSimuMode)))
+				if ((pVBInfo->VBInfo & SetInSlaveMode) &&
+						!(pVBInfo->TVInfo
+								& TVSimuMode))
 					temp &= (~0x0001);
 			}
 		}
@@ -4389,7 +4379,6 @@ static void xgifb_set_lvds(struct xgifb_video_info *xgifb_info,
 					(value << 2) & 0x7C);
 
 	for (temp = 0, value = 0; temp < 3; temp++) {
-
 		xgifb_reg_and_or(pVBInfo->P3c4, 0x31, ~0x30, value);
 		xgifb_reg_set(pVBInfo->P3c4,
 			      0x2B, xgifb_info->lvds_data.VCLKData1);
@@ -4409,7 +4398,6 @@ static void xgifb_set_lvds(struct xgifb_video_info *xgifb_info,
 
 		inb(pVBInfo->P3da); /* reset 3da */
 	}
-
 }
 
 /* --------------------------------------------------------------------- */
@@ -4476,7 +4464,7 @@ static void XGI_DisableBridge(struct xgifb_video_info *xgifb_info,
 
 		if ((pVBInfo->VBInfo &
 			(DisableCRT2Display | SetSimuScanMode)) ||
-		    ((!(pVBInfo->VBInfo & XGI_SetCRT2ToLCDA)) &&
+		    (!(pVBInfo->VBInfo & XGI_SetCRT2ToLCDA) &&
 		    (pVBInfo->VBInfo &
 			(SetCRT2ToRAMDAC | SetCRT2ToLCD | SetCRT2ToTV))))
 			xgifb_reg_or(pVBInfo->Part1Port, 0x00, 0x80);
@@ -4587,7 +4575,7 @@ static void XGI_GetTVPtrIndex2(unsigned short *tempbx, unsigned char *tempcl,
 
 	if (pVBInfo->VBType & (VB_SIS301B | VB_SIS302B | VB_SIS301LV
 			| VB_SIS302LV | VB_XGI301C)) {
-		if ((!(pVBInfo->VBInfo & SetInSlaveMode)) || (pVBInfo->TVInfo
+		if (!(pVBInfo->VBInfo & SetInSlaveMode) || (pVBInfo->TVInfo
 				& TVSimuMode)) {
 			*tempbx += 8;
 			*tempcl += 1;
@@ -4988,8 +4976,8 @@ reg_and_or:
 	if (pVBInfo->VBInfo & (SetCRT2ToRAMDAC | SetCRT2ToTV | SetCRT2ToLCD
 			| XGI_SetCRT2ToLCDA)) {
 		tempah &= (~0x08);
-		if ((pVBInfo->ModeType == ModeVGA) && (!(pVBInfo->VBInfo
-				& SetInSlaveMode))) {
+		if ((pVBInfo->ModeType == ModeVGA) && !(pVBInfo->VBInfo
+				& SetInSlaveMode)) {
 			tempah |= 0x010;
 		}
 		tempah |= 0x080;
@@ -5416,7 +5404,6 @@ static void XGI_SetCRT1Group(struct xgifb_video_info *xgifb_info,
 	if (HwDeviceExtension->jChipType >= XG21) {
 		temp = xgifb_reg_get(pVBInfo->P3d4, 0x38);
 		if (temp & 0xA0) {
-
 			if (HwDeviceExtension->jChipType == XG27)
 				XGI_SetXG27CRTC(RefreshRateTableIndex, pVBInfo);
 			else
@@ -5486,7 +5473,7 @@ unsigned char XGISetModeNew(struct xgifb_video_info *xgifb_info,
 		XGI_DisableBridge(xgifb_info, HwDeviceExtension, pVBInfo);
 
 		if (pVBInfo->VBInfo & (SetSimuScanMode | XGI_SetCRT2ToLCDA) ||
-			(!(pVBInfo->VBInfo & SwitchCRT2))) {
+			!(pVBInfo->VBInfo & SwitchCRT2)) {
 			XGI_SetCRT1Group(xgifb_info, HwDeviceExtension, ModeNo,
 					ModeIdIndex, pVBInfo);
 
diff --git a/drivers/target/iscsi/cxgbit/cxgbit_cm.c b/drivers/target/iscsi/cxgbit/cxgbit_cm.c
index 0ae0b131abfc..2fb1bf1a26c5 100644
--- a/drivers/target/iscsi/cxgbit/cxgbit_cm.c
+++ b/drivers/target/iscsi/cxgbit/cxgbit_cm.c
@@ -24,6 +24,7 @@
 #include <net/ip6_route.h>
 #include <net/addrconf.h>
 
+#include <libcxgb_cm.h>
 #include "cxgbit.h"
 #include "clip_tbl.h"
 
@@ -72,15 +73,6 @@ out:
 	return wr_waitp->ret;
 }
 
-/* Returns whether a CPL status conveys negative advice.
- */
-static int cxgbit_is_neg_adv(unsigned int status)
-{
-	return status == CPL_ERR_RTX_NEG_ADVICE ||
-		status == CPL_ERR_PERSIST_NEG_ADVICE ||
-		status == CPL_ERR_KEEPALV_NEG_ADVICE;
-}
-
 static int cxgbit_np_hashfn(const struct cxgbit_np *cnp)
 {
 	return ((unsigned long)cnp >> 10) & (NP_INFO_HASH_SIZE - 1);
@@ -623,21 +615,14 @@ void cxgbit_free_np(struct iscsi_np *np)
 static void cxgbit_send_halfclose(struct cxgbit_sock *csk)
 {
 	struct sk_buff *skb;
-	struct cpl_close_con_req *req;
-	unsigned int len = roundup(sizeof(struct cpl_close_con_req), 16);
+	u32 len = roundup(sizeof(struct cpl_close_con_req), 16);
 
 	skb = alloc_skb(len, GFP_ATOMIC);
 	if (!skb)
 		return;
 
-	req = (struct cpl_close_con_req *)__skb_put(skb, len);
-	memset(req, 0, len);
-
-	set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
-	INIT_TP_WR(req, csk->tid);
-	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ,
-						    csk->tid));
-	req->rsvd = 0;
+	cxgb_mk_close_con_req(skb, len, csk->tid, csk->txq_idx,
+			      NULL, NULL);
 
 	cxgbit_skcb_flags(skb) |= SKCBF_TX_FLAG_COMPL;
 	__skb_queue_tail(&csk->txq, skb);
@@ -662,9 +647,8 @@ static void cxgbit_abort_arp_failure(void *handle, struct sk_buff *skb)
 
 static int cxgbit_send_abort_req(struct cxgbit_sock *csk)
 {
-	struct cpl_abort_req *req;
-	unsigned int len = roundup(sizeof(*req), 16);
 	struct sk_buff *skb;
+	u32 len = roundup(sizeof(struct cpl_abort_req), 16);
 
 	pr_debug("%s: csk %p tid %u; state %d\n",
 		 __func__, csk, csk->tid, csk->com.state);
@@ -675,15 +659,9 @@ static int cxgbit_send_abort_req(struct cxgbit_sock *csk)
 		cxgbit_send_tx_flowc_wr(csk);
 
 	skb = __skb_dequeue(&csk->skbq);
-	req = (struct cpl_abort_req *)__skb_put(skb, len);
-	memset(req, 0, len);
+	cxgb_mk_abort_req(skb, len, csk->tid, csk->txq_idx,
+			  csk->com.cdev, cxgbit_abort_arp_failure);
 
-	set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
-	t4_set_arp_err_handler(skb, csk->com.cdev, cxgbit_abort_arp_failure);
-	INIT_TP_WR(req, csk->tid);
-	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ,
-						    csk->tid));
-	req->cmd = CPL_ABORT_SEND_RST;
 	return cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
 }
 
@@ -789,109 +767,6 @@ void _cxgbit_free_csk(struct kref *kref)
 	kfree(csk);
 }
 
-static void
-cxgbit_get_tuple_info(struct cpl_pass_accept_req *req, int *iptype,
-		      __u8 *local_ip, __u8 *peer_ip, __be16 *local_port,
-		      __be16 *peer_port)
-{
-	u32 eth_len = ETH_HDR_LEN_G(be32_to_cpu(req->hdr_len));
-	u32 ip_len = IP_HDR_LEN_G(be32_to_cpu(req->hdr_len));
-	struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len);
-	struct ipv6hdr *ip6 = (struct ipv6hdr *)((u8 *)(req + 1) + eth_len);
-	struct tcphdr *tcp = (struct tcphdr *)
-			      ((u8 *)(req + 1) + eth_len + ip_len);
-
-	if (ip->version == 4) {
-		pr_debug("%s saddr 0x%x daddr 0x%x sport %u dport %u\n",
-			 __func__,
-			 ntohl(ip->saddr), ntohl(ip->daddr),
-			 ntohs(tcp->source),
-			 ntohs(tcp->dest));
-		*iptype = 4;
-		memcpy(peer_ip, &ip->saddr, 4);
-		memcpy(local_ip, &ip->daddr, 4);
-	} else {
-		pr_debug("%s saddr %pI6 daddr %pI6 sport %u dport %u\n",
-			 __func__,
-			 ip6->saddr.s6_addr, ip6->daddr.s6_addr,
-			 ntohs(tcp->source),
-			 ntohs(tcp->dest));
-		*iptype = 6;
-		memcpy(peer_ip, ip6->saddr.s6_addr, 16);
-		memcpy(local_ip, ip6->daddr.s6_addr, 16);
-	}
-
-	*peer_port = tcp->source;
-	*local_port = tcp->dest;
-}
-
-static int
-cxgbit_our_interface(struct cxgbit_device *cdev, struct net_device *egress_dev)
-{
-	u8 i;
-
-	egress_dev = cxgbit_get_real_dev(egress_dev);
-	for (i = 0; i < cdev->lldi.nports; i++)
-		if (cdev->lldi.ports[i] == egress_dev)
-			return 1;
-	return 0;
-}
-
-static struct dst_entry *
-cxgbit_find_route6(struct cxgbit_device *cdev, __u8 *local_ip, __u8 *peer_ip,
-		   __be16 local_port, __be16 peer_port, u8 tos,
-		   __u32 sin6_scope_id)
-{
-	struct dst_entry *dst = NULL;
-
-	if (IS_ENABLED(CONFIG_IPV6)) {
-		struct flowi6 fl6;
-
-		memset(&fl6, 0, sizeof(fl6));
-		memcpy(&fl6.daddr, peer_ip, 16);
-		memcpy(&fl6.saddr, local_ip, 16);
-		if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
-			fl6.flowi6_oif = sin6_scope_id;
-		dst = ip6_route_output(&init_net, NULL, &fl6);
-		if (!dst)
-			goto out;
-		if (!cxgbit_our_interface(cdev, ip6_dst_idev(dst)->dev) &&
-		    !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK)) {
-			dst_release(dst);
-			dst = NULL;
-		}
-	}
-out:
-	return dst;
-}
-
-static struct dst_entry *
-cxgbit_find_route(struct cxgbit_device *cdev, __be32 local_ip, __be32 peer_ip,
-		  __be16 local_port, __be16 peer_port, u8 tos)
-{
-	struct rtable *rt;
-	struct flowi4 fl4;
-	struct neighbour *n;
-
-	rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip,
-				   local_ip,
-				   peer_port, local_port, IPPROTO_TCP,
-				   tos, 0);
-	if (IS_ERR(rt))
-		return NULL;
-	n = dst_neigh_lookup(&rt->dst, &peer_ip);
-	if (!n)
-		return NULL;
-	if (!cxgbit_our_interface(cdev, n->dev) &&
-	    !(n->dev->flags & IFF_LOOPBACK)) {
-		neigh_release(n);
-		dst_release(&rt->dst);
-		return NULL;
-	}
-	neigh_release(n);
-	return &rt->dst;
-}
-
 static void cxgbit_set_tcp_window(struct cxgbit_sock *csk, struct port_info *pi)
 {
 	unsigned int linkspeed;
@@ -1072,21 +947,14 @@ int cxgbit_ofld_send(struct cxgbit_device *cdev, struct sk_buff *skb)
 
 static void cxgbit_release_tid(struct cxgbit_device *cdev, u32 tid)
 {
-	struct cpl_tid_release *req;
-	unsigned int len = roundup(sizeof(*req), 16);
+	u32 len = roundup(sizeof(struct cpl_tid_release), 16);
 	struct sk_buff *skb;
 
 	skb = alloc_skb(len, GFP_ATOMIC);
 	if (!skb)
 		return;
 
-	req = (struct cpl_tid_release *)__skb_put(skb, len);
-	memset(req, 0, len);
-
-	INIT_TP_WR(req, tid);
-	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(
-		   CPL_TID_RELEASE, tid));
-	set_wr_txq(skb, CPL_PRIORITY_SETUP, 0);
+	cxgb_mk_tid_release(skb, len, tid, 0);
 	cxgbit_ofld_send(cdev, skb);
 }
 
@@ -1108,20 +976,6 @@ cxgbit_l2t_send(struct cxgbit_device *cdev, struct sk_buff *skb,
 	return ret < 0 ? ret : 0;
 }
 
-static void
-cxgbit_best_mtu(const unsigned short *mtus, unsigned short mtu,
-		unsigned int *idx, int use_ts, int ipv6)
-{
-	unsigned short hdr_size = (ipv6 ? sizeof(struct ipv6hdr) :
-				   sizeof(struct iphdr)) +
-				   sizeof(struct tcphdr) +
-				   (use_ts ? round_up(TCPOLEN_TIMESTAMP,
-				    4) : 0);
-	unsigned short data_size = mtu - hdr_size;
-
-	cxgb4_best_aligned_mtu(mtus, hdr_size, data_size, 8, idx);
-}
-
 static void cxgbit_send_rx_credits(struct cxgbit_sock *csk, struct sk_buff *skb)
 {
 	if (csk->com.state != CSK_STATE_ESTABLISHED) {
@@ -1140,22 +994,18 @@ static void cxgbit_send_rx_credits(struct cxgbit_sock *csk, struct sk_buff *skb)
 int cxgbit_rx_data_ack(struct cxgbit_sock *csk)
 {
 	struct sk_buff *skb;
-	struct cpl_rx_data_ack *req;
-	unsigned int len = roundup(sizeof(*req), 16);
+	u32 len = roundup(sizeof(struct cpl_rx_data_ack), 16);
+	u32 credit_dack;
 
 	skb = alloc_skb(len, GFP_KERNEL);
 	if (!skb)
 		return -1;
 
-	req = (struct cpl_rx_data_ack *)__skb_put(skb, len);
-	memset(req, 0, len);
+	credit_dack = RX_DACK_CHANGE_F | RX_DACK_MODE_V(1) |
+		      RX_CREDITS_V(csk->rx_credits);
 
-	set_wr_txq(skb, CPL_PRIORITY_ACK, csk->ctrlq_idx);
-	INIT_TP_WR(req, csk->tid);
-	OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
-						    csk->tid));
-	req->credit_dack = cpu_to_be32(RX_DACK_CHANGE_F | RX_DACK_MODE_V(1) |
-				       RX_CREDITS_V(csk->rx_credits));
+	cxgb_mk_rx_data_ack(skb, len, csk->tid, csk->ctrlq_idx,
+			    credit_dack);
 
 	csk->rx_credits = 0;
 
@@ -1210,15 +1060,6 @@ out:
 	return -ENOMEM;
 }
 
-static u32 cxgbit_compute_wscale(u32 win)
-{
-	u32 wscale = 0;
-
-	while (wscale < 14 && (65535 << wscale) < win)
-		wscale++;
-	return wscale;
-}
-
 static void
 cxgbit_pass_accept_rpl(struct cxgbit_sock *csk, struct cpl_pass_accept_req *req)
 {
@@ -1246,10 +1087,10 @@ cxgbit_pass_accept_rpl(struct cxgbit_sock *csk, struct cpl_pass_accept_req *req)
 	INIT_TP_WR(rpl5, csk->tid);
 	OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
 						     csk->tid));
-	cxgbit_best_mtu(csk->com.cdev->lldi.mtus, csk->mtu, &mtu_idx,
-			req->tcpopt.tstamp,
-			(csk->com.remote_addr.ss_family == AF_INET) ? 0 : 1);
-	wscale = cxgbit_compute_wscale(csk->rcv_win);
+	cxgb_best_mtu(csk->com.cdev->lldi.mtus, csk->mtu, &mtu_idx,
+		      req->tcpopt.tstamp,
+		      (csk->com.remote_addr.ss_family == AF_INET) ? 0 : 1);
+	wscale = cxgb_compute_wscale(csk->rcv_win);
 	/*
 	 * Specify the largest window that will fit in opt0. The
 	 * remainder will be specified in the rx_data_ack.
@@ -1340,8 +1181,8 @@ cxgbit_pass_accept_req(struct cxgbit_device *cdev, struct sk_buff *skb)
 		goto rel_skb;
 	}
 
-	cxgbit_get_tuple_info(req, &iptype, local_ip, peer_ip,
-			      &local_port, &peer_port);
+	cxgb_get_4tuple(req, cdev->lldi.adapter_type, &iptype, local_ip,
+			peer_ip, &local_port, &peer_port);
 
 	/* Find output route */
 	if (iptype == 4)  {
@@ -1350,21 +1191,23 @@ cxgbit_pass_accept_req(struct cxgbit_device *cdev, struct sk_buff *skb)
 			 , __func__, cnp, tid,
 			 local_ip, peer_ip, ntohs(local_port),
 			 ntohs(peer_port), peer_mss);
-		dst = cxgbit_find_route(cdev, *(__be32 *)local_ip,
-					*(__be32 *)peer_ip,
-					local_port, peer_port,
-					PASS_OPEN_TOS_G(ntohl(req->tos_stid)));
+		dst = cxgb_find_route(&cdev->lldi, cxgbit_get_real_dev,
+				      *(__be32 *)local_ip,
+				      *(__be32 *)peer_ip,
+				      local_port, peer_port,
+				      PASS_OPEN_TOS_G(ntohl(req->tos_stid)));
 	} else {
 		pr_debug("%s parent sock %p tid %u laddr %pI6 raddr %pI6 "
 			 "lport %d rport %d peer_mss %d\n"
 			 , __func__, cnp, tid,
 			 local_ip, peer_ip, ntohs(local_port),
 			 ntohs(peer_port), peer_mss);
-		dst = cxgbit_find_route6(cdev, local_ip, peer_ip,
-					 local_port, peer_port,
-					 PASS_OPEN_TOS_G(ntohl(req->tos_stid)),
-					 ((struct sockaddr_in6 *)
-					 &cnp->com.local_addr)->sin6_scope_id);
+		dst = cxgb_find_route6(&cdev->lldi, cxgbit_get_real_dev,
+				       local_ip, peer_ip,
+				       local_port, peer_port,
+				       PASS_OPEN_TOS_G(ntohl(req->tos_stid)),
+				       ((struct sockaddr_in6 *)
+					&cnp->com.local_addr)->sin6_scope_id);
 	}
 	if (!dst) {
 		pr_err("%s - failed to find dst entry!\n",
@@ -1795,16 +1638,15 @@ static void cxgbit_abort_req_rss(struct cxgbit_sock *csk, struct sk_buff *skb)
 {
 	struct cpl_abort_req_rss *hdr = cplhdr(skb);
 	unsigned int tid = GET_TID(hdr);
-	struct cpl_abort_rpl *rpl;
 	struct sk_buff *rpl_skb;
 	bool release = false;
 	bool wakeup_thread = false;
-	unsigned int len = roundup(sizeof(*rpl), 16);
+	u32 len = roundup(sizeof(struct cpl_abort_rpl), 16);
 
 	pr_debug("%s: csk %p; tid %u; state %d\n",
 		 __func__, csk, tid, csk->com.state);
 
-	if (cxgbit_is_neg_adv(hdr->status)) {
+	if (cxgb_is_neg_adv(hdr->status)) {
 		pr_err("%s: got neg advise %d on tid %u\n",
 		       __func__, hdr->status, tid);
 		goto rel_skb;
@@ -1839,14 +1681,8 @@ static void cxgbit_abort_req_rss(struct cxgbit_sock *csk, struct sk_buff *skb)
 		cxgbit_send_tx_flowc_wr(csk);
 
 	rpl_skb = __skb_dequeue(&csk->skbq);
-	set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
-
-	rpl = (struct cpl_abort_rpl *)__skb_put(rpl_skb, len);
-	memset(rpl, 0, len);
 
-	INIT_TP_WR(rpl, csk->tid);
-	OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
-	rpl->cmd = CPL_ABORT_NO_RST;
+	cxgb_mk_abort_rpl(rpl_skb, len, csk->tid, csk->txq_idx);
 	cxgbit_ofld_send(csk->com.cdev, rpl_skb);
 
 	if (wakeup_thread) {
diff --git a/drivers/target/iscsi/cxgbit/cxgbit_main.c b/drivers/target/iscsi/cxgbit/cxgbit_main.c
index 27dd11aff934..ad26b9372f10 100644
--- a/drivers/target/iscsi/cxgbit/cxgbit_main.c
+++ b/drivers/target/iscsi/cxgbit/cxgbit_main.c
@@ -652,6 +652,9 @@ static struct iscsit_transport cxgbit_transport = {
 
 static struct cxgb4_uld_info cxgbit_uld_info = {
 	.name		= DRV_NAME,
+	.nrxq		= MAX_ULD_QSETS,
+	.rxq_size	= 1024,
+	.lro		= true,
 	.add		= cxgbit_uld_add,
 	.state_change	= cxgbit_uld_state_change,
 	.lro_rx_handler = cxgbit_uld_lro_rx_handler,
diff --git a/drivers/thermal/clock_cooling.c b/drivers/thermal/clock_cooling.c
index 1b4ff0f4c716..ed5dd0e88657 100644
--- a/drivers/thermal/clock_cooling.c
+++ b/drivers/thermal/clock_cooling.c
@@ -426,6 +426,7 @@ clock_cooling_register(struct device *dev, const char *clock_name)
 	if (!ccdev)
 		return ERR_PTR(-ENOMEM);
 
+	mutex_init(&ccdev->lock);
 	ccdev->dev = dev;
 	ccdev->clk = devm_clk_get(dev, clock_name);
 	if (IS_ERR(ccdev->clk))
diff --git a/drivers/thermal/cpu_cooling.c b/drivers/thermal/cpu_cooling.c
index 3788ed74c9ab..a32b41783b77 100644
--- a/drivers/thermal/cpu_cooling.c
+++ b/drivers/thermal/cpu_cooling.c
@@ -740,12 +740,22 @@ static int cpufreq_power2state(struct thermal_cooling_device *cdev,
 }
 
 /* Bind cpufreq callbacks to thermal cooling device ops */
+
 static struct thermal_cooling_device_ops cpufreq_cooling_ops = {
 	.get_max_state = cpufreq_get_max_state,
 	.get_cur_state = cpufreq_get_cur_state,
 	.set_cur_state = cpufreq_set_cur_state,
 };
 
+static struct thermal_cooling_device_ops cpufreq_power_cooling_ops = {
+	.get_max_state		= cpufreq_get_max_state,
+	.get_cur_state		= cpufreq_get_cur_state,
+	.set_cur_state		= cpufreq_set_cur_state,
+	.get_requested_power	= cpufreq_get_requested_power,
+	.state2power		= cpufreq_state2power,
+	.power2state		= cpufreq_power2state,
+};
+
 /* Notifier for cpufreq policy change */
 static struct notifier_block thermal_cpufreq_notifier_block = {
 	.notifier_call = cpufreq_thermal_notifier,
@@ -795,6 +805,7 @@ __cpufreq_cooling_register(struct device_node *np,
 	struct cpumask temp_mask;
 	unsigned int freq, i, num_cpus;
 	int ret;
+	struct thermal_cooling_device_ops *cooling_ops;
 
 	cpumask_and(&temp_mask, clip_cpus, cpu_online_mask);
 	policy = cpufreq_cpu_get(cpumask_first(&temp_mask));
@@ -850,10 +861,6 @@ __cpufreq_cooling_register(struct device_node *np,
 	cpumask_copy(&cpufreq_dev->allowed_cpus, clip_cpus);
 
 	if (capacitance) {
-		cpufreq_cooling_ops.get_requested_power =
-			cpufreq_get_requested_power;
-		cpufreq_cooling_ops.state2power = cpufreq_state2power;
-		cpufreq_cooling_ops.power2state = cpufreq_power2state;
 		cpufreq_dev->plat_get_static_power = plat_static_func;
 
 		ret = build_dyn_power_table(cpufreq_dev, capacitance);
@@ -861,6 +868,10 @@ __cpufreq_cooling_register(struct device_node *np,
 			cool_dev = ERR_PTR(ret);
 			goto free_table;
 		}
+
+		cooling_ops = &cpufreq_power_cooling_ops;
+	} else {
+		cooling_ops = &cpufreq_cooling_ops;
 	}
 
 	ret = get_idr(&cpufreq_idr, &cpufreq_dev->id);
@@ -885,7 +896,7 @@ __cpufreq_cooling_register(struct device_node *np,
 		 cpufreq_dev->id);
 
 	cool_dev = thermal_of_cooling_device_register(np, dev_name, cpufreq_dev,
-						      &cpufreq_cooling_ops);
+						      cooling_ops);
 	if (IS_ERR(cool_dev))
 		goto remove_idr;
 
diff --git a/drivers/thermal/fair_share.c b/drivers/thermal/fair_share.c
index 34fe36504a55..68bd1b569118 100644
--- a/drivers/thermal/fair_share.c
+++ b/drivers/thermal/fair_share.c
@@ -116,7 +116,9 @@ static int fair_share_throttle(struct thermal_zone_device *tz, int trip)
 		instance->target = get_target_state(tz, cdev, percentage,
 						    cur_trip_level);
 
+		mutex_lock(&instance->cdev->lock);
 		instance->cdev->updated = false;
+		mutex_unlock(&instance->cdev->lock);
 		thermal_cdev_update(cdev);
 	}
 	return 0;
diff --git a/drivers/thermal/gov_bang_bang.c b/drivers/thermal/gov_bang_bang.c
index fc52016d4e85..bb118a152cbb 100644
--- a/drivers/thermal/gov_bang_bang.c
+++ b/drivers/thermal/gov_bang_bang.c
@@ -71,7 +71,9 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip)
 		dev_dbg(&instance->cdev->device, "target=%d\n",
 					(int)instance->target);
 
+		mutex_lock(&instance->cdev->lock);
 		instance->cdev->updated = false; /* cdev needs update */
+		mutex_unlock(&instance->cdev->lock);
 	}
 
 	mutex_unlock(&tz->lock);
diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c
index c5547bd711db..e473548b5d28 100644
--- a/drivers/thermal/imx_thermal.c
+++ b/drivers/thermal/imx_thermal.c
@@ -471,8 +471,6 @@ MODULE_DEVICE_TABLE(of, of_imx_thermal_match);
 
 static int imx_thermal_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *of_id =
-		of_match_device(of_imx_thermal_match, &pdev->dev);
 	struct imx_thermal_data *data;
 	struct regmap *map;
 	int measure_freq;
@@ -490,7 +488,7 @@ static int imx_thermal_probe(struct platform_device *pdev)
 	}
 	data->tempmon = map;
 
-	data->socdata = of_id->data;
+	data->socdata = of_device_get_match_data(&pdev->dev);
 
 	/* make sure the IRQ flag is clear before enabling irq on i.MX6SX */
 	if (data->socdata->version == TEMPMON_IMX6SX) {
diff --git a/drivers/thermal/int340x_thermal/int3406_thermal.c b/drivers/thermal/int340x_thermal/int3406_thermal.c
index a578cd257db4..1891f34ab7fc 100644
--- a/drivers/thermal/int340x_thermal/int3406_thermal.c
+++ b/drivers/thermal/int340x_thermal/int3406_thermal.c
@@ -225,7 +225,6 @@ static struct platform_driver int3406_thermal_driver = {
 	.remove = int3406_thermal_remove,
 	.driver = {
 		   .name = "int3406 thermal",
-		   .owner = THIS_MODULE,
 		   .acpi_match_table = int3406_thermal_match,
 		   },
 };
diff --git a/drivers/thermal/intel_pch_thermal.c b/drivers/thermal/intel_pch_thermal.c
index 6a6ec1c95a7a..9b4815e81b0d 100644
--- a/drivers/thermal/intel_pch_thermal.c
+++ b/drivers/thermal/intel_pch_thermal.c
@@ -21,6 +21,7 @@
 #include <linux/init.h>
 #include <linux/pci.h>
 #include <linux/thermal.h>
+#include <linux/pm.h>
 
 /* Intel PCH thermal Device IDs */
 #define PCH_THERMAL_DID_WPT	0x9CA4 /* Wildcat Point */
@@ -65,6 +66,7 @@ struct pch_thermal_device {
 	unsigned long crt_temp;
 	int hot_trip_id;
 	unsigned long hot_temp;
+	bool bios_enabled;
 };
 
 static int pch_wpt_init(struct pch_thermal_device *ptd, int *nr_trips)
@@ -75,8 +77,10 @@ static int pch_wpt_init(struct pch_thermal_device *ptd, int *nr_trips)
 	*nr_trips = 0;
 
 	/* Check if BIOS has already enabled thermal sensor */
-	if (WPT_TSS_TSDSS & readb(ptd->hw_base + WPT_TSS))
+	if (WPT_TSS_TSDSS & readb(ptd->hw_base + WPT_TSS)) {
+		ptd->bios_enabled = true;
 		goto read_trips;
+	}
 
 	tsel = readb(ptd->hw_base + WPT_TSEL);
 	/*
@@ -130,9 +134,39 @@ static int pch_wpt_get_temp(struct pch_thermal_device *ptd, int *temp)
 	return 0;
 }
 
+static int pch_wpt_suspend(struct pch_thermal_device *ptd)
+{
+	u8 tsel;
+
+	if (ptd->bios_enabled)
+		return 0;
+
+	tsel = readb(ptd->hw_base + WPT_TSEL);
+
+	writeb(tsel & 0xFE, ptd->hw_base + WPT_TSEL);
+
+	return 0;
+}
+
+static int pch_wpt_resume(struct pch_thermal_device *ptd)
+{
+	u8 tsel;
+
+	if (ptd->bios_enabled)
+		return 0;
+
+	tsel = readb(ptd->hw_base + WPT_TSEL);
+
+	writeb(tsel | WPT_TSEL_ETS, ptd->hw_base + WPT_TSEL);
+
+	return 0;
+}
+
 struct pch_dev_ops {
 	int (*hw_init)(struct pch_thermal_device *ptd, int *nr_trips);
 	int (*get_temp)(struct pch_thermal_device *ptd, int *temp);
+	int (*suspend)(struct pch_thermal_device *ptd);
+	int (*resume)(struct pch_thermal_device *ptd);
 };
 
 
@@ -140,6 +174,8 @@ struct pch_dev_ops {
 static const struct pch_dev_ops pch_dev_ops_wpt = {
 	.hw_init = pch_wpt_init,
 	.get_temp = pch_wpt_get_temp,
+	.suspend = pch_wpt_suspend,
+	.resume = pch_wpt_resume,
 };
 
 static int pch_thermal_get_temp(struct thermal_zone_device *tzd, int *temp)
@@ -269,6 +305,22 @@ static void intel_pch_thermal_remove(struct pci_dev *pdev)
 	pci_disable_device(pdev);
 }
 
+static int intel_pch_thermal_suspend(struct device *device)
+{
+	struct pci_dev *pdev = to_pci_dev(device);
+	struct pch_thermal_device *ptd = pci_get_drvdata(pdev);
+
+	return ptd->ops->suspend(ptd);
+}
+
+static int intel_pch_thermal_resume(struct device *device)
+{
+	struct pci_dev *pdev = to_pci_dev(device);
+	struct pch_thermal_device *ptd = pci_get_drvdata(pdev);
+
+	return ptd->ops->resume(ptd);
+}
+
 static struct pci_device_id intel_pch_thermal_id[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_WPT) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_SKL) },
@@ -276,11 +328,17 @@ static struct pci_device_id intel_pch_thermal_id[] = {
 };
 MODULE_DEVICE_TABLE(pci, intel_pch_thermal_id);
 
+static const struct dev_pm_ops intel_pch_pm_ops = {
+	.suspend = intel_pch_thermal_suspend,
+	.resume = intel_pch_thermal_resume,
+};
+
 static struct pci_driver intel_pch_thermal_driver = {
 	.name		= "intel_pch_thermal",
 	.id_table	= intel_pch_thermal_id,
 	.probe		= intel_pch_thermal_probe,
 	.remove		= intel_pch_thermal_remove,
+	.driver.pm	= &intel_pch_pm_ops,
 };
 
 module_pci_driver(intel_pch_thermal_driver);
diff --git a/drivers/thermal/intel_powerclamp.c b/drivers/thermal/intel_powerclamp.c
index 015ce2eb6eb7..0e4dc0afcfd2 100644
--- a/drivers/thermal/intel_powerclamp.c
+++ b/drivers/thermal/intel_powerclamp.c
@@ -388,7 +388,7 @@ static int clamp_thread(void *arg)
 		int sleeptime;
 		unsigned long target_jiffies;
 		unsigned int guard;
-		unsigned int compensation = 0;
+		unsigned int compensated_ratio;
 		int interval; /* jiffies to sleep for each attempt */
 		unsigned int duration_jiffies = msecs_to_jiffies(duration);
 		unsigned int window_size_now;
@@ -409,8 +409,11 @@ static int clamp_thread(void *arg)
 		 * c-states, thus we need to compensate the injected idle ratio
 		 * to achieve the actual target reported by the HW.
 		 */
-		compensation = get_compensation(target_ratio);
-		interval = duration_jiffies*100/(target_ratio+compensation);
+		compensated_ratio = target_ratio +
+			get_compensation(target_ratio);
+		if (compensated_ratio <= 0)
+			compensated_ratio = 1;
+		interval = duration_jiffies * 100 / compensated_ratio;
 
 		/* align idle time */
 		target_jiffies = roundup(jiffies, interval);
@@ -647,8 +650,8 @@ static int powerclamp_set_cur_state(struct thermal_cooling_device *cdev,
 		goto exit_set;
 	} else	if (set_target_ratio > 0 && new_target_ratio == 0) {
 		pr_info("Stop forced idle injection\n");
-		set_target_ratio = 0;
 		end_power_clamp();
+		set_target_ratio = 0;
 	} else	/* adjust currently running */ {
 		set_target_ratio = new_target_ratio;
 		/* make new set_target_ratio visible to other cpus */
diff --git a/drivers/thermal/power_allocator.c b/drivers/thermal/power_allocator.c
index 2f1a863a8e15..b4d3116cfdaf 100644
--- a/drivers/thermal/power_allocator.c
+++ b/drivers/thermal/power_allocator.c
@@ -529,7 +529,9 @@ static void allow_maximum_power(struct thermal_zone_device *tz)
 			continue;
 
 		instance->target = 0;
+		mutex_lock(&instance->cdev->lock);
 		instance->cdev->updated = false;
+		mutex_unlock(&instance->cdev->lock);
 		thermal_cdev_update(instance->cdev);
 	}
 }
diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c
index 71a339271fa5..5f817923f374 100644
--- a/drivers/thermal/rcar_thermal.c
+++ b/drivers/thermal/rcar_thermal.c
@@ -504,6 +504,7 @@ static int rcar_thermal_probe(struct platform_device *pdev)
 		if (IS_ERR(priv->zone)) {
 			dev_err(dev, "can't register thermal zone\n");
 			ret = PTR_ERR(priv->zone);
+			priv->zone = NULL;
 			goto error_unregister;
 		}
 
diff --git a/drivers/thermal/step_wise.c b/drivers/thermal/step_wise.c
index ea9366ad3e6b..bcef2e7c4ec9 100644
--- a/drivers/thermal/step_wise.c
+++ b/drivers/thermal/step_wise.c
@@ -175,7 +175,9 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip)
 			update_passive_instance(tz, trip_type, -1);
 
 		instance->initialized = true;
+		mutex_lock(&instance->cdev->lock);
 		instance->cdev->updated = false; /* cdev needs update */
+		mutex_unlock(&instance->cdev->lock);
 	}
 
 	mutex_unlock(&tz->lock);
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 5133cd1e10b7..e2fc6161dded 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -1093,7 +1093,9 @@ int power_actor_set_power(struct thermal_cooling_device *cdev,
 		return ret;
 
 	instance->target = state;
+	mutex_lock(&cdev->lock);
 	cdev->updated = false;
+	mutex_unlock(&cdev->lock);
 	thermal_cdev_update(cdev);
 
 	return 0;
@@ -1623,11 +1625,13 @@ void thermal_cdev_update(struct thermal_cooling_device *cdev)
 	struct thermal_instance *instance;
 	unsigned long target = 0;
 
+	mutex_lock(&cdev->lock);
 	/* cooling device is updated*/
-	if (cdev->updated)
+	if (cdev->updated) {
+		mutex_unlock(&cdev->lock);
 		return;
+	}
 
-	mutex_lock(&cdev->lock);
 	/* Make sure cdev enters the deepest cooling state */
 	list_for_each_entry(instance, &cdev->thermal_instances, cdev_node) {
 		dev_dbg(&cdev->device, "zone%d->target=%lu\n",
@@ -1637,9 +1641,9 @@ void thermal_cdev_update(struct thermal_cooling_device *cdev)
 		if (instance->target > target)
 			target = instance->target;
 	}
-	mutex_unlock(&cdev->lock);
 	cdev->ops->set_cur_state(cdev, target);
 	cdev->updated = true;
+	mutex_unlock(&cdev->lock);
 	trace_cdev_update(cdev, target);
 	dev_dbg(&cdev->device, "set to state %lu\n", target);
 }
diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c
index 06fd2ed9ef9d..c41c7742903a 100644
--- a/drivers/thermal/thermal_hwmon.c
+++ b/drivers/thermal/thermal_hwmon.c
@@ -232,6 +232,7 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz)
 
 	return result;
 }
+EXPORT_SYMBOL_GPL(thermal_add_hwmon_sysfs);
 
 void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz)
 {
@@ -270,3 +271,4 @@ void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz)
 	hwmon_device_unregister(hwmon->device);
 	kfree(hwmon);
 }
+EXPORT_SYMBOL_GPL(thermal_remove_hwmon_sysfs);
diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index 9c15344b657a..a8c20413dbda 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -651,6 +651,12 @@ static struct pci_device_id nhi_ids[] = {
 	{
 		.class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0,
 		.vendor = PCI_VENDOR_ID_INTEL,
+		.device = PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI,
+		.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID,
+	},
+	{
+		.class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0,
+		.vendor = PCI_VENDOR_ID_INTEL,
 		.device = PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI,
 		.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID,
 	},
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index 1e116f53d6dd..9840fdecb73b 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -372,7 +372,9 @@ struct tb_switch *tb_switch_alloc(struct tb *tb, u64 route)
 
 	if (sw->config.device_id != PCI_DEVICE_ID_INTEL_LIGHT_RIDGE &&
 	    sw->config.device_id != PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C &&
-	    sw->config.device_id != PCI_DEVICE_ID_INTEL_PORT_RIDGE)
+	    sw->config.device_id != PCI_DEVICE_ID_INTEL_PORT_RIDGE &&
+	    sw->config.device_id != PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_BRIDGE &&
+	    sw->config.device_id != PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_BRIDGE)
 		tb_sw_warn(sw, "unsupported switch device id %#x\n",
 			   sw->config.device_id);
 
diff --git a/drivers/tty/pty.c b/drivers/tty/pty.c
index 51e0d32883ba..a23fa5ed1d67 100644
--- a/drivers/tty/pty.c
+++ b/drivers/tty/pty.c
@@ -800,7 +800,7 @@ out_free_file:
 	return retval;
 }
 
-static struct file_operations ptmx_fops;
+static struct file_operations ptmx_fops __ro_after_init;
 
 static void __init unix98_pty_init(void)
 {
diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h
index 122e0e4029fe..a697a8585ddc 100644
--- a/drivers/tty/serial/8250/8250.h
+++ b/drivers/tty/serial/8250/8250.h
@@ -15,8 +15,6 @@
 #include <linux/serial_reg.h>
 #include <linux/dmaengine.h>
 
-#include "../serial_mctrl_gpio.h"
-
 struct uart_8250_dma {
 	int (*tx_dma)(struct uart_8250_port *p);
 	int (*rx_dma)(struct uart_8250_port *p);
@@ -33,6 +31,11 @@ struct uart_8250_dma {
 	struct dma_chan		*rxchan;
 	struct dma_chan		*txchan;
 
+	/* Device address base for DMA operations */
+	phys_addr_t		rx_dma_addr;
+	phys_addr_t		tx_dma_addr;
+
+	/* DMA address of the buffer in memory */
 	dma_addr_t		rx_addr;
 	dma_addr_t		tx_addr;
 
@@ -133,43 +136,12 @@ void serial8250_em485_destroy(struct uart_8250_port *p);
 
 static inline void serial8250_out_MCR(struct uart_8250_port *up, int value)
 {
-	int mctrl_gpio = 0;
-
 	serial_out(up, UART_MCR, value);
-
-	if (value & UART_MCR_RTS)
-		mctrl_gpio |= TIOCM_RTS;
-	if (value & UART_MCR_DTR)
-		mctrl_gpio |= TIOCM_DTR;
-
-	mctrl_gpio_set(up->gpios, mctrl_gpio);
 }
 
 static inline int serial8250_in_MCR(struct uart_8250_port *up)
 {
-	int mctrl, mctrl_gpio = 0;
-
-	mctrl = serial_in(up, UART_MCR);
-
-	/* save current MCR values */
-	if (mctrl & UART_MCR_RTS)
-		mctrl_gpio |= TIOCM_RTS;
-	if (mctrl & UART_MCR_DTR)
-		mctrl_gpio |= TIOCM_DTR;
-
-	mctrl_gpio = mctrl_gpio_get_outputs(up->gpios, &mctrl_gpio);
-
-	if (mctrl_gpio & TIOCM_RTS)
-		mctrl |= UART_MCR_RTS;
-	else
-		mctrl &= ~UART_MCR_RTS;
-
-	if (mctrl_gpio & TIOCM_DTR)
-		mctrl |= UART_MCR_DTR;
-	else
-		mctrl &= ~UART_MCR_DTR;
-
-	return mctrl;
+	return serial_in(up, UART_MCR);
 }
 
 #if defined(__alpha__) && !defined(CONFIG_PCI)
diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c
index 13ad5c3d2e68..240a361b674f 100644
--- a/drivers/tty/serial/8250/8250_core.c
+++ b/drivers/tty/serial/8250/8250_core.c
@@ -639,7 +639,7 @@ static int univ8250_console_match(struct console *co, char *name, int idx,
 {
 	char match[] = "uart";	/* 8250-specific earlycon name */
 	unsigned char iotype;
-	unsigned long addr;
+	resource_size_t addr;
 	int i;
 
 	if (strncmp(name, match, 4) != 0)
@@ -974,8 +974,6 @@ int serial8250_register_8250_port(struct uart_8250_port *up)
 
 	uart = serial8250_find_match_or_unused(&up->port);
 	if (uart && uart->port.type != PORT_8250_CIR) {
-		struct mctrl_gpios *gpios;
-
 		if (uart->port.dev)
 			uart_remove_one_port(&serial8250_reg, &uart->port);
 
@@ -1013,13 +1011,6 @@ int serial8250_register_8250_port(struct uart_8250_port *up)
 		if (up->port.flags & UPF_FIXED_TYPE)
 			uart->port.type = up->port.type;
 
-		gpios = mctrl_gpio_init(&uart->port, 0);
-		if (IS_ERR(gpios)) {
-			if (PTR_ERR(gpios) != -ENOSYS)
-				return PTR_ERR(gpios);
-		} else
-			uart->gpios = gpios;
-
 		serial8250_set_defaults(uart);
 
 		/* Possibly override default I/O functions.  */
diff --git a/drivers/tty/serial/8250/8250_dma.c b/drivers/tty/serial/8250/8250_dma.c
index 3590d012001f..fdbddbc6375d 100644
--- a/drivers/tty/serial/8250/8250_dma.c
+++ b/drivers/tty/serial/8250/8250_dma.c
@@ -142,7 +142,7 @@ void serial8250_rx_dma_flush(struct uart_8250_port *p)
 	if (dma->rx_running) {
 		dmaengine_pause(dma->rxchan);
 		__dma_rx_complete(p);
-		dmaengine_terminate_all(dma->rxchan);
+		dmaengine_terminate_async(dma->rxchan);
 	}
 }
 EXPORT_SYMBOL_GPL(serial8250_rx_dma_flush);
@@ -150,6 +150,10 @@ EXPORT_SYMBOL_GPL(serial8250_rx_dma_flush);
 int serial8250_request_dma(struct uart_8250_port *p)
 {
 	struct uart_8250_dma	*dma = p->dma;
+	phys_addr_t rx_dma_addr = dma->rx_dma_addr ?
+				  dma->rx_dma_addr : p->port.mapbase;
+	phys_addr_t tx_dma_addr = dma->tx_dma_addr ?
+				  dma->tx_dma_addr : p->port.mapbase;
 	dma_cap_mask_t		mask;
 	struct dma_slave_caps	caps;
 	int			ret;
@@ -157,11 +161,11 @@ int serial8250_request_dma(struct uart_8250_port *p)
 	/* Default slave configuration parameters */
 	dma->rxconf.direction		= DMA_DEV_TO_MEM;
 	dma->rxconf.src_addr_width	= DMA_SLAVE_BUSWIDTH_1_BYTE;
-	dma->rxconf.src_addr		= p->port.mapbase + UART_RX;
+	dma->rxconf.src_addr		= rx_dma_addr + UART_RX;
 
 	dma->txconf.direction		= DMA_MEM_TO_DEV;
 	dma->txconf.dst_addr_width	= DMA_SLAVE_BUSWIDTH_1_BYTE;
-	dma->txconf.dst_addr		= p->port.mapbase + UART_TX;
+	dma->txconf.dst_addr		= tx_dma_addr + UART_TX;
 
 	dma_cap_zero(mask);
 	dma_cap_set(DMA_SLAVE, mask);
@@ -247,14 +251,14 @@ void serial8250_release_dma(struct uart_8250_port *p)
 		return;
 
 	/* Release RX resources */
-	dmaengine_terminate_all(dma->rxchan);
+	dmaengine_terminate_sync(dma->rxchan);
 	dma_free_coherent(dma->rxchan->device->dev, dma->rx_size, dma->rx_buf,
 			  dma->rx_addr);
 	dma_release_channel(dma->rxchan);
 	dma->rxchan = NULL;
 
 	/* Release TX resources */
-	dmaengine_terminate_all(dma->txchan);
+	dmaengine_terminate_sync(dma->txchan);
 	dma_unmap_single(dma->txchan->device->dev, dma->tx_addr,
 			 UART_XMIT_SIZE, DMA_TO_DEVICE);
 	dma_release_channel(dma->txchan);
diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index e19969614203..459d726f9d59 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -298,12 +298,17 @@ static void dw8250_quirks(struct uart_port *p, struct dw8250_data *data)
 			p->serial_out = dw8250_serial_out32be;
 		}
 	} else if (has_acpi_companion(p->dev)) {
-		p->iotype = UPIO_MEM32;
-		p->regshift = 2;
-		p->serial_in = dw8250_serial_in32;
+		const struct acpi_device_id *id;
+
+		id = acpi_match_device(p->dev->driver->acpi_match_table,
+				       p->dev);
+		if (id && !strcmp(id->id, "APMC0D08")) {
+			p->iotype = UPIO_MEM32;
+			p->regshift = 2;
+			p->serial_in = dw8250_serial_in32;
+			data->uart_16550_compatible = true;
+		}
 		p->set_termios = dw8250_set_termios;
-		/* So far none of there implement the Busy Functionality */
-		data->uart_16550_compatible = true;
 	}
 
 	/* Platforms with iDMA */
@@ -360,18 +365,19 @@ static int dw8250_probe(struct platform_device *pdev)
 	struct resource *regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	int irq = platform_get_irq(pdev, 0);
 	struct uart_port *p = &uart.port;
+	struct device *dev = &pdev->dev;
 	struct dw8250_data *data;
 	int err;
 	u32 val;
 
 	if (!regs) {
-		dev_err(&pdev->dev, "no registers defined\n");
+		dev_err(dev, "no registers defined\n");
 		return -EINVAL;
 	}
 
 	if (irq < 0) {
 		if (irq != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "cannot get irq\n");
+			dev_err(dev, "cannot get irq\n");
 		return irq;
 	}
 
@@ -382,16 +388,16 @@ static int dw8250_probe(struct platform_device *pdev)
 	p->pm		= dw8250_do_pm;
 	p->type		= PORT_8250;
 	p->flags	= UPF_SHARE_IRQ | UPF_FIXED_PORT;
-	p->dev		= &pdev->dev;
+	p->dev		= dev;
 	p->iotype	= UPIO_MEM;
 	p->serial_in	= dw8250_serial_in;
 	p->serial_out	= dw8250_serial_out;
 
-	p->membase = devm_ioremap(&pdev->dev, regs->start, resource_size(regs));
+	p->membase = devm_ioremap(dev, regs->start, resource_size(regs));
 	if (!p->membase)
 		return -ENOMEM;
 
-	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
@@ -399,57 +405,57 @@ static int dw8250_probe(struct platform_device *pdev)
 	data->usr_reg = DW_UART_USR;
 	p->private_data = data;
 
-	data->uart_16550_compatible = device_property_read_bool(p->dev,
+	data->uart_16550_compatible = device_property_read_bool(dev,
 						"snps,uart-16550-compatible");
 
-	err = device_property_read_u32(p->dev, "reg-shift", &val);
+	err = device_property_read_u32(dev, "reg-shift", &val);
 	if (!err)
 		p->regshift = val;
 
-	err = device_property_read_u32(p->dev, "reg-io-width", &val);
+	err = device_property_read_u32(dev, "reg-io-width", &val);
 	if (!err && val == 4) {
 		p->iotype = UPIO_MEM32;
 		p->serial_in = dw8250_serial_in32;
 		p->serial_out = dw8250_serial_out32;
 	}
 
-	if (device_property_read_bool(p->dev, "dcd-override")) {
+	if (device_property_read_bool(dev, "dcd-override")) {
 		/* Always report DCD as active */
 		data->msr_mask_on |= UART_MSR_DCD;
 		data->msr_mask_off |= UART_MSR_DDCD;
 	}
 
-	if (device_property_read_bool(p->dev, "dsr-override")) {
+	if (device_property_read_bool(dev, "dsr-override")) {
 		/* Always report DSR as active */
 		data->msr_mask_on |= UART_MSR_DSR;
 		data->msr_mask_off |= UART_MSR_DDSR;
 	}
 
-	if (device_property_read_bool(p->dev, "cts-override")) {
+	if (device_property_read_bool(dev, "cts-override")) {
 		/* Always report CTS as active */
 		data->msr_mask_on |= UART_MSR_CTS;
 		data->msr_mask_off |= UART_MSR_DCTS;
 	}
 
-	if (device_property_read_bool(p->dev, "ri-override")) {
+	if (device_property_read_bool(dev, "ri-override")) {
 		/* Always report Ring indicator as inactive */
 		data->msr_mask_off |= UART_MSR_RI;
 		data->msr_mask_off |= UART_MSR_TERI;
 	}
 
 	/* Always ask for fixed clock rate from a property. */
-	device_property_read_u32(p->dev, "clock-frequency", &p->uartclk);
+	device_property_read_u32(dev, "clock-frequency", &p->uartclk);
 
 	/* If there is separate baudclk, get the rate from it. */
-	data->clk = devm_clk_get(&pdev->dev, "baudclk");
+	data->clk = devm_clk_get(dev, "baudclk");
 	if (IS_ERR(data->clk) && PTR_ERR(data->clk) != -EPROBE_DEFER)
-		data->clk = devm_clk_get(&pdev->dev, NULL);
+		data->clk = devm_clk_get(dev, NULL);
 	if (IS_ERR(data->clk) && PTR_ERR(data->clk) == -EPROBE_DEFER)
 		return -EPROBE_DEFER;
 	if (!IS_ERR_OR_NULL(data->clk)) {
 		err = clk_prepare_enable(data->clk);
 		if (err)
-			dev_warn(&pdev->dev, "could not enable optional baudclk: %d\n",
+			dev_warn(dev, "could not enable optional baudclk: %d\n",
 				 err);
 		else
 			p->uartclk = clk_get_rate(data->clk);
@@ -457,24 +463,24 @@ static int dw8250_probe(struct platform_device *pdev)
 
 	/* If no clock rate is defined, fail. */
 	if (!p->uartclk) {
-		dev_err(&pdev->dev, "clock rate not defined\n");
+		dev_err(dev, "clock rate not defined\n");
 		return -EINVAL;
 	}
 
-	data->pclk = devm_clk_get(&pdev->dev, "apb_pclk");
-	if (IS_ERR(data->clk) && PTR_ERR(data->clk) == -EPROBE_DEFER) {
+	data->pclk = devm_clk_get(dev, "apb_pclk");
+	if (IS_ERR(data->pclk) && PTR_ERR(data->pclk) == -EPROBE_DEFER) {
 		err = -EPROBE_DEFER;
 		goto err_clk;
 	}
 	if (!IS_ERR(data->pclk)) {
 		err = clk_prepare_enable(data->pclk);
 		if (err) {
-			dev_err(&pdev->dev, "could not enable apb_pclk\n");
+			dev_err(dev, "could not enable apb_pclk\n");
 			goto err_clk;
 		}
 	}
 
-	data->rst = devm_reset_control_get_optional(&pdev->dev, NULL);
+	data->rst = devm_reset_control_get_optional(dev, NULL);
 	if (IS_ERR(data->rst) && PTR_ERR(data->rst) == -EPROBE_DEFER) {
 		err = -EPROBE_DEFER;
 		goto err_pclk;
@@ -506,8 +512,8 @@ static int dw8250_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, data);
 
-	pm_runtime_set_active(&pdev->dev);
-	pm_runtime_enable(&pdev->dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
 
 	return 0;
 
@@ -619,6 +625,7 @@ static const struct acpi_device_id dw8250_acpi_match[] = {
 	{ "APMC0D08", 0},
 	{ "AMD0020", 0 },
 	{ "AMDI0020", 0 },
+	{ "HISI0031", 0 },
 	{ },
 };
 MODULE_DEVICE_TABLE(acpi, dw8250_acpi_match);
diff --git a/drivers/tty/serial/8250/8250_fintek.c b/drivers/tty/serial/8250/8250_fintek.c
index 737b4b3957b0..0facc789fe7d 100644
--- a/drivers/tty/serial/8250/8250_fintek.c
+++ b/drivers/tty/serial/8250/8250_fintek.c
@@ -31,7 +31,7 @@
 #define IO_ADDR2 0x60
 #define LDN 0x7
 
-#define IRQ_MODE	0x70
+#define FINTEK_IRQ_MODE	0x70
 #define IRQ_SHARE	BIT(4)
 #define IRQ_MODE_MASK	(BIT(6) | BIT(5))
 #define IRQ_LEVEL_LOW	0
@@ -195,7 +195,7 @@ static int fintek_8250_set_irq_mode(struct fintek_8250 *pdata, bool level_mode)
 	outb(LDN, pdata->base_port + ADDR_PORT);
 	outb(pdata->index, pdata->base_port + DATA_PORT);
 
-	outb(IRQ_MODE, pdata->base_port + ADDR_PORT);
+	outb(FINTEK_IRQ_MODE, pdata->base_port + ADDR_PORT);
 	tmp = inb(pdata->base_port + DATA_PORT);
 
 	tmp &= ~IRQ_MODE_MASK;
diff --git a/drivers/tty/serial/8250/8250_lpss.c b/drivers/tty/serial/8250/8250_lpss.c
new file mode 100644
index 000000000000..886fcf37f291
--- /dev/null
+++ b/drivers/tty/serial/8250/8250_lpss.c
@@ -0,0 +1,378 @@
+/*
+ * 8250_lpss.c - Driver for UART on Intel Braswell and various other Intel SoCs
+ *
+ * Copyright (C) 2016 Intel Corporation
+ * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/bitops.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/rational.h>
+
+#include <linux/dmaengine.h>
+#include <linux/dma/dw.h>
+
+#include "8250.h"
+
+#define PCI_DEVICE_ID_INTEL_QRK_UARTx	0x0936
+
+#define PCI_DEVICE_ID_INTEL_BYT_UART1	0x0f0a
+#define PCI_DEVICE_ID_INTEL_BYT_UART2	0x0f0c
+
+#define PCI_DEVICE_ID_INTEL_BSW_UART1	0x228a
+#define PCI_DEVICE_ID_INTEL_BSW_UART2	0x228c
+
+#define PCI_DEVICE_ID_INTEL_BDW_UART1	0x9ce3
+#define PCI_DEVICE_ID_INTEL_BDW_UART2	0x9ce4
+
+/* Intel LPSS specific registers */
+
+#define BYT_PRV_CLK			0x800
+#define BYT_PRV_CLK_EN			BIT(0)
+#define BYT_PRV_CLK_M_VAL_SHIFT		1
+#define BYT_PRV_CLK_N_VAL_SHIFT		16
+#define BYT_PRV_CLK_UPDATE		BIT(31)
+
+#define BYT_TX_OVF_INT			0x820
+#define BYT_TX_OVF_INT_MASK		BIT(1)
+
+struct lpss8250;
+
+struct lpss8250_board {
+	unsigned long freq;
+	unsigned int base_baud;
+	int (*setup)(struct lpss8250 *, struct uart_port *p);
+	void (*exit)(struct lpss8250 *);
+};
+
+struct lpss8250 {
+	int line;
+	struct lpss8250_board *board;
+
+	/* DMA parameters */
+	struct uart_8250_dma dma;
+	struct dw_dma_chip dma_chip;
+	struct dw_dma_slave dma_param;
+	u8 dma_maxburst;
+};
+
+static void byt_set_termios(struct uart_port *p, struct ktermios *termios,
+			    struct ktermios *old)
+{
+	unsigned int baud = tty_termios_baud_rate(termios);
+	struct lpss8250 *lpss = p->private_data;
+	unsigned long fref = lpss->board->freq, fuart = baud * 16;
+	unsigned long w = BIT(15) - 1;
+	unsigned long m, n;
+	u32 reg;
+
+	/* Gracefully handle the B0 case: fall back to B9600 */
+	fuart = fuart ? fuart : 9600 * 16;
+
+	/* Get Fuart closer to Fref */
+	fuart *= rounddown_pow_of_two(fref / fuart);
+
+	/*
+	 * For baud rates 0.5M, 1M, 1.5M, 2M, 2.5M, 3M, 3.5M and 4M the
+	 * dividers must be adjusted.
+	 *
+	 * uartclk = (m / n) * 100 MHz, where m <= n
+	 */
+	rational_best_approximation(fuart, fref, w, w, &m, &n);
+	p->uartclk = fuart;
+
+	/* Reset the clock */
+	reg = (m << BYT_PRV_CLK_M_VAL_SHIFT) | (n << BYT_PRV_CLK_N_VAL_SHIFT);
+	writel(reg, p->membase + BYT_PRV_CLK);
+	reg |= BYT_PRV_CLK_EN | BYT_PRV_CLK_UPDATE;
+	writel(reg, p->membase + BYT_PRV_CLK);
+
+	p->status &= ~UPSTAT_AUTOCTS;
+	if (termios->c_cflag & CRTSCTS)
+		p->status |= UPSTAT_AUTOCTS;
+
+	serial8250_do_set_termios(p, termios, old);
+}
+
+static unsigned int byt_get_mctrl(struct uart_port *port)
+{
+	unsigned int ret = serial8250_do_get_mctrl(port);
+
+	/* Force DCD and DSR signals to permanently be reported as active */
+	ret |= TIOCM_CAR | TIOCM_DSR;
+
+	return ret;
+}
+
+static int byt_serial_setup(struct lpss8250 *lpss, struct uart_port *port)
+{
+	struct dw_dma_slave *param = &lpss->dma_param;
+	struct uart_8250_port *up = up_to_u8250p(port);
+	struct pci_dev *pdev = to_pci_dev(port->dev);
+	unsigned int dma_devfn = PCI_DEVFN(PCI_SLOT(pdev->devfn), 0);
+	struct pci_dev *dma_dev = pci_get_slot(pdev->bus, dma_devfn);
+
+	switch (pdev->device) {
+	case PCI_DEVICE_ID_INTEL_BYT_UART1:
+	case PCI_DEVICE_ID_INTEL_BSW_UART1:
+	case PCI_DEVICE_ID_INTEL_BDW_UART1:
+		param->src_id = 3;
+		param->dst_id = 2;
+		break;
+	case PCI_DEVICE_ID_INTEL_BYT_UART2:
+	case PCI_DEVICE_ID_INTEL_BSW_UART2:
+	case PCI_DEVICE_ID_INTEL_BDW_UART2:
+		param->src_id = 5;
+		param->dst_id = 4;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	param->dma_dev = &dma_dev->dev;
+	param->m_master = 0;
+	param->p_master = 1;
+
+	/* TODO: Detect FIFO size automaticaly for DesignWare 8250 */
+	port->fifosize = 64;
+	up->tx_loadsz = 64;
+
+	lpss->dma_maxburst = 16;
+
+	port->set_termios = byt_set_termios;
+	port->get_mctrl = byt_get_mctrl;
+
+	/* Disable TX counter interrupts */
+	writel(BYT_TX_OVF_INT_MASK, port->membase + BYT_TX_OVF_INT);
+
+	return 0;
+}
+
+#ifdef CONFIG_SERIAL_8250_DMA
+static const struct dw_dma_platform_data qrk_serial_dma_pdata = {
+	.nr_channels = 2,
+	.is_private = true,
+	.is_nollp = true,
+	.chan_allocation_order = CHAN_ALLOCATION_ASCENDING,
+	.chan_priority = CHAN_PRIORITY_ASCENDING,
+	.block_size = 4095,
+	.nr_masters = 1,
+	.data_width = {4},
+};
+
+static void qrk_serial_setup_dma(struct lpss8250 *lpss, struct uart_port *port)
+{
+	struct uart_8250_dma *dma = &lpss->dma;
+	struct dw_dma_chip *chip = &lpss->dma_chip;
+	struct dw_dma_slave *param = &lpss->dma_param;
+	struct pci_dev *pdev = to_pci_dev(port->dev);
+	int ret;
+
+	chip->dev = &pdev->dev;
+	chip->irq = pdev->irq;
+	chip->regs = pci_ioremap_bar(pdev, 1);
+	chip->pdata = &qrk_serial_dma_pdata;
+
+	/* Falling back to PIO mode if DMA probing fails */
+	ret = dw_dma_probe(chip);
+	if (ret)
+		return;
+
+	/* Special DMA address for UART */
+	dma->rx_dma_addr = 0xfffff000;
+	dma->tx_dma_addr = 0xfffff000;
+
+	param->dma_dev = &pdev->dev;
+	param->src_id = 0;
+	param->dst_id = 1;
+	param->hs_polarity = true;
+
+	lpss->dma_maxburst = 8;
+}
+
+static void qrk_serial_exit_dma(struct lpss8250 *lpss)
+{
+	struct dw_dma_slave *param = &lpss->dma_param;
+
+	if (!param->dma_dev)
+		return;
+	dw_dma_remove(&lpss->dma_chip);
+}
+#else	/* CONFIG_SERIAL_8250_DMA */
+static void qrk_serial_setup_dma(struct lpss8250 *lpss, struct uart_port *port) {}
+static void qrk_serial_exit_dma(struct lpss8250 *lpss) {}
+#endif	/* !CONFIG_SERIAL_8250_DMA */
+
+static int qrk_serial_setup(struct lpss8250 *lpss, struct uart_port *port)
+{
+	struct pci_dev *pdev = to_pci_dev(port->dev);
+	int ret;
+
+	ret = pci_alloc_irq_vectors(pdev, 1, 1, 0);
+	if (ret < 0)
+		return ret;
+
+	port->irq = pci_irq_vector(pdev, 0);
+
+	qrk_serial_setup_dma(lpss, port);
+	return 0;
+}
+
+static void qrk_serial_exit(struct lpss8250 *lpss)
+{
+	qrk_serial_exit_dma(lpss);
+}
+
+static bool lpss8250_dma_filter(struct dma_chan *chan, void *param)
+{
+	struct dw_dma_slave *dws = param;
+
+	if (dws->dma_dev != chan->device->dev)
+		return false;
+
+	chan->private = dws;
+	return true;
+}
+
+static int lpss8250_dma_setup(struct lpss8250 *lpss, struct uart_8250_port *port)
+{
+	struct uart_8250_dma *dma = &lpss->dma;
+	struct dw_dma_slave *rx_param, *tx_param;
+	struct device *dev = port->port.dev;
+
+	if (!lpss->dma_param.dma_dev)
+		return 0;
+
+	rx_param = devm_kzalloc(dev, sizeof(*rx_param), GFP_KERNEL);
+	if (!rx_param)
+		return -ENOMEM;
+
+	tx_param = devm_kzalloc(dev, sizeof(*tx_param), GFP_KERNEL);
+	if (!tx_param)
+		return -ENOMEM;
+
+	*rx_param = lpss->dma_param;
+	dma->rxconf.src_maxburst = lpss->dma_maxburst;
+
+	*tx_param = lpss->dma_param;
+	dma->txconf.dst_maxburst = lpss->dma_maxburst;
+
+	dma->fn = lpss8250_dma_filter;
+	dma->rx_param = rx_param;
+	dma->tx_param = tx_param;
+
+	port->dma = dma;
+	return 0;
+}
+
+static int lpss8250_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct uart_8250_port uart;
+	struct lpss8250 *lpss;
+	int ret;
+
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return ret;
+
+	pci_set_master(pdev);
+
+	lpss = devm_kzalloc(&pdev->dev, sizeof(*lpss), GFP_KERNEL);
+	if (!lpss)
+		return -ENOMEM;
+
+	lpss->board = (struct lpss8250_board *)id->driver_data;
+
+	memset(&uart, 0, sizeof(struct uart_8250_port));
+
+	uart.port.dev = &pdev->dev;
+	uart.port.irq = pdev->irq;
+	uart.port.private_data = lpss;
+	uart.port.type = PORT_16550A;
+	uart.port.iotype = UPIO_MEM;
+	uart.port.regshift = 2;
+	uart.port.uartclk = lpss->board->base_baud * 16;
+	uart.port.flags = UPF_SHARE_IRQ | UPF_FIXED_PORT | UPF_FIXED_TYPE;
+	uart.capabilities = UART_CAP_FIFO | UART_CAP_AFE;
+	uart.port.mapbase = pci_resource_start(pdev, 0);
+	uart.port.membase = pcim_iomap(pdev, 0, 0);
+	if (!uart.port.membase)
+		return -ENOMEM;
+
+	ret = lpss->board->setup(lpss, &uart.port);
+	if (ret)
+		return ret;
+
+	ret = lpss8250_dma_setup(lpss, &uart);
+	if (ret)
+		goto err_exit;
+
+	ret = serial8250_register_8250_port(&uart);
+	if (ret < 0)
+		goto err_exit;
+
+	lpss->line = ret;
+
+	pci_set_drvdata(pdev, lpss);
+	return 0;
+
+err_exit:
+	if (lpss->board->exit)
+		lpss->board->exit(lpss);
+	return ret;
+}
+
+static void lpss8250_remove(struct pci_dev *pdev)
+{
+	struct lpss8250 *lpss = pci_get_drvdata(pdev);
+
+	if (lpss->board->exit)
+		lpss->board->exit(lpss);
+
+	serial8250_unregister_port(lpss->line);
+}
+
+static const struct lpss8250_board byt_board = {
+	.freq = 100000000,
+	.base_baud = 2764800,
+	.setup = byt_serial_setup,
+};
+
+static const struct lpss8250_board qrk_board = {
+	.freq = 44236800,
+	.base_baud = 2764800,
+	.setup = qrk_serial_setup,
+	.exit = qrk_serial_exit,
+};
+
+#define LPSS_DEVICE(id, board) { PCI_VDEVICE(INTEL, id), (kernel_ulong_t)&board }
+
+static const struct pci_device_id pci_ids[] = {
+	LPSS_DEVICE(PCI_DEVICE_ID_INTEL_QRK_UARTx, qrk_board),
+	LPSS_DEVICE(PCI_DEVICE_ID_INTEL_BYT_UART1, byt_board),
+	LPSS_DEVICE(PCI_DEVICE_ID_INTEL_BYT_UART2, byt_board),
+	LPSS_DEVICE(PCI_DEVICE_ID_INTEL_BSW_UART1, byt_board),
+	LPSS_DEVICE(PCI_DEVICE_ID_INTEL_BSW_UART2, byt_board),
+	LPSS_DEVICE(PCI_DEVICE_ID_INTEL_BDW_UART1, byt_board),
+	LPSS_DEVICE(PCI_DEVICE_ID_INTEL_BDW_UART2, byt_board),
+	{ },
+};
+MODULE_DEVICE_TABLE(pci, pci_ids);
+
+static struct pci_driver lpss8250_pci_driver = {
+	.name           = "8250_lpss",
+	.id_table       = pci_ids,
+	.probe          = lpss8250_probe,
+	.remove         = lpss8250_remove,
+};
+
+module_pci_driver(lpss8250_pci_driver);
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Intel LPSS UART driver");
diff --git a/drivers/tty/serial/8250/8250_mid.c b/drivers/tty/serial/8250/8250_mid.c
index 339de9cd0866..39c2324484dd 100644
--- a/drivers/tty/serial/8250/8250_mid.c
+++ b/drivers/tty/serial/8250/8250_mid.c
@@ -99,27 +99,27 @@ static int dnv_handle_irq(struct uart_port *p)
 	struct uart_8250_port *up = up_to_u8250p(p);
 	unsigned int fisr = serial_port_in(p, INTEL_MID_UART_DNV_FISR);
 	u32 status;
-	int ret = IRQ_NONE;
+	int ret = 0;
 	int err;
 
 	if (fisr & BIT(2)) {
 		err = hsu_dma_get_status(&mid->dma_chip, 1, &status);
 		if (err > 0) {
 			serial8250_rx_dma_flush(up);
-			ret |= IRQ_HANDLED;
+			ret |= 1;
 		} else if (err == 0)
 			ret |= hsu_dma_do_irq(&mid->dma_chip, 1, status);
 	}
 	if (fisr & BIT(1)) {
 		err = hsu_dma_get_status(&mid->dma_chip, 0, &status);
 		if (err > 0)
-			ret |= IRQ_HANDLED;
+			ret |= 1;
 		else if (err == 0)
 			ret |= hsu_dma_do_irq(&mid->dma_chip, 0, status);
 	}
 	if (fisr & BIT(0))
 		ret |= serial8250_handle_irq(p, serial_port_in(p, UART_IIR));
-	return ret;
+	return IRQ_RETVAL(ret);
 }
 
 #define DNV_DMA_CHAN_OFFSET 0x80
@@ -168,6 +168,9 @@ static void mid8250_set_termios(struct uart_port *p,
 	unsigned long w = BIT(24) - 1;
 	unsigned long mul, div;
 
+	/* Gracefully handle the B0 case: fall back to B9600 */
+	fuart = fuart ? fuart : 9600 * 16;
+
 	if (mid->board->freq < fuart) {
 		/* Find prescaler value that satisfies Fuart < Fref */
 		if (mid->board->freq > baud)
diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c
index 3611ec9bb4fa..ce0cc471bfc3 100644
--- a/drivers/tty/serial/8250/8250_mtk.c
+++ b/drivers/tty/serial/8250/8250_mtk.c
@@ -62,7 +62,7 @@ mtk8250_set_termios(struct uart_port *port, struct ktermios *termios,
 	 */
 	baud = uart_get_baud_rate(port, termios, old,
 				  port->uartclk / 16 / 0xffff,
-				  port->uartclk / 16);
+				  port->uartclk);
 
 	if (baud <= 115200) {
 		serial_port_out(port, UART_MTK_HIGHS, 0x0);
@@ -76,10 +76,6 @@ mtk8250_set_termios(struct uart_port *port, struct ktermios *termios,
 		quot = DIV_ROUND_UP(port->uartclk, 4 * baud);
 	} else {
 		serial_port_out(port, UART_MTK_HIGHS, 0x3);
-
-		/* Set to highest baudrate supported */
-		if (baud >= 1152000)
-			baud = 921600;
 		quot = DIV_ROUND_UP(port->uartclk, 256 * baud);
 	}
 
diff --git a/drivers/tty/serial/8250/8250_of.c b/drivers/tty/serial/8250/8250_of.c
index 38963d7bcf84..7a8b5fc81a19 100644
--- a/drivers/tty/serial/8250/8250_of.c
+++ b/drivers/tty/serial/8250/8250_of.c
@@ -195,6 +195,7 @@ static int of_platform_serial_probe(struct platform_device *ofdev)
 	switch (port_type) {
 	case PORT_8250 ... PORT_MAX_8250:
 	{
+		u32 tx_threshold;
 		struct uart_8250_port port8250;
 		memset(&port8250, 0, sizeof(port8250));
 		port8250.port = port;
@@ -202,6 +203,12 @@ static int of_platform_serial_probe(struct platform_device *ofdev)
 		if (port.fifosize)
 			port8250.capabilities = UART_CAP_FIFO;
 
+		/* Check for TX FIFO threshold & set tx_loadsz */
+		if ((of_property_read_u32(ofdev->dev.of_node, "tx-threshold",
+					  &tx_threshold) == 0) &&
+		    (tx_threshold < port.fifosize))
+			port8250.tx_loadsz = port.fifosize - tx_threshold;
+
 		if (of_property_read_bool(ofdev->dev.of_node,
 					  "auto-flow-control"))
 			port8250.capabilities |= UART_CAP_AFE;
diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c
index e14982f36a04..61ad6c3b20a0 100644
--- a/drivers/tty/serial/8250/8250_omap.c
+++ b/drivers/tty/serial/8250/8250_omap.c
@@ -134,21 +134,18 @@ static void omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl)
 
 	serial8250_do_set_mctrl(port, mctrl);
 
-	if (IS_ERR_OR_NULL(mctrl_gpio_to_gpiod(up->gpios,
-						UART_GPIO_RTS))) {
-		/*
-		 * Turn off autoRTS if RTS is lowered and restore autoRTS
-		 * setting if RTS is raised
-		 */
-		lcr = serial_in(up, UART_LCR);
-		serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
-		if ((mctrl & TIOCM_RTS) && (port->status & UPSTAT_AUTORTS))
-			priv->efr |= UART_EFR_RTS;
-		else
-			priv->efr &= ~UART_EFR_RTS;
-		serial_out(up, UART_EFR, priv->efr);
-		serial_out(up, UART_LCR, lcr);
-	}
+	/*
+	 * Turn off autoRTS if RTS is lowered and restore autoRTS setting
+	 * if RTS is raised
+	 */
+	lcr = serial_in(up, UART_LCR);
+	serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+	if ((mctrl & TIOCM_RTS) && (port->status & UPSTAT_AUTORTS))
+		priv->efr |= UART_EFR_RTS;
+	else
+		priv->efr &= ~UART_EFR_RTS;
+	serial_out(up, UART_EFR, priv->efr);
+	serial_out(up, UART_LCR, lcr);
 }
 
 /*
@@ -449,9 +446,7 @@ static void omap_8250_set_termios(struct uart_port *port,
 	priv->efr = 0;
 	up->port.status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS | UPSTAT_AUTOXOFF);
 
-	if (termios->c_cflag & CRTSCTS && up->port.flags & UPF_HARD_FLOW
-		&& IS_ERR_OR_NULL(mctrl_gpio_to_gpiod(up->gpios,
-							UART_GPIO_RTS))) {
+	if (termios->c_cflag & CRTSCTS && up->port.flags & UPF_HARD_FLOW) {
 		/* Enable AUTOCTS (autoRTS is enabled when RTS is raised) */
 		up->port.status |= UPSTAT_AUTOCTS | UPSTAT_AUTORTS;
 		priv->efr |= UART_EFR_CTS;
diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c
index 20ebaea5c414..b98c1578f45a 100644
--- a/drivers/tty/serial/8250/8250_pci.c
+++ b/drivers/tty/serial/8250/8250_pci.c
@@ -21,14 +21,10 @@
 #include <linux/serial_core.h>
 #include <linux/8250_pci.h>
 #include <linux/bitops.h>
-#include <linux/rational.h>
 
 #include <asm/byteorder.h>
 #include <asm/io.h>
 
-#include <linux/dmaengine.h>
-#include <linux/platform_data/dma-dw.h>
-
 #include "8250.h"
 
 /*
@@ -1349,160 +1345,6 @@ ce4100_serial_setup(struct serial_private *priv,
 	return ret;
 }
 
-#define PCI_DEVICE_ID_INTEL_BYT_UART1	0x0f0a
-#define PCI_DEVICE_ID_INTEL_BYT_UART2	0x0f0c
-
-#define PCI_DEVICE_ID_INTEL_BSW_UART1	0x228a
-#define PCI_DEVICE_ID_INTEL_BSW_UART2	0x228c
-
-#define PCI_DEVICE_ID_INTEL_BDW_UART1	0x9ce3
-#define PCI_DEVICE_ID_INTEL_BDW_UART2	0x9ce4
-
-#define BYT_PRV_CLK			0x800
-#define BYT_PRV_CLK_EN			(1 << 0)
-#define BYT_PRV_CLK_M_VAL_SHIFT		1
-#define BYT_PRV_CLK_N_VAL_SHIFT		16
-#define BYT_PRV_CLK_UPDATE		(1 << 31)
-
-#define BYT_TX_OVF_INT			0x820
-#define BYT_TX_OVF_INT_MASK		(1 << 1)
-
-static void
-byt_set_termios(struct uart_port *p, struct ktermios *termios,
-		struct ktermios *old)
-{
-	unsigned int baud = tty_termios_baud_rate(termios);
-	unsigned long fref = 100000000, fuart = baud * 16;
-	unsigned long w = BIT(15) - 1;
-	unsigned long m, n;
-	u32 reg;
-
-	/* Gracefully handle the B0 case: fall back to B9600 */
-	fuart = fuart ? fuart : 9600 * 16;
-
-	/* Get Fuart closer to Fref */
-	fuart *= rounddown_pow_of_two(fref / fuart);
-
-	/*
-	 * For baud rates 0.5M, 1M, 1.5M, 2M, 2.5M, 3M, 3.5M and 4M the
-	 * dividers must be adjusted.
-	 *
-	 * uartclk = (m / n) * 100 MHz, where m <= n
-	 */
-	rational_best_approximation(fuart, fref, w, w, &m, &n);
-	p->uartclk = fuart;
-
-	/* Reset the clock */
-	reg = (m << BYT_PRV_CLK_M_VAL_SHIFT) | (n << BYT_PRV_CLK_N_VAL_SHIFT);
-	writel(reg, p->membase + BYT_PRV_CLK);
-	reg |= BYT_PRV_CLK_EN | BYT_PRV_CLK_UPDATE;
-	writel(reg, p->membase + BYT_PRV_CLK);
-
-	p->status &= ~UPSTAT_AUTOCTS;
-	if (termios->c_cflag & CRTSCTS)
-		p->status |= UPSTAT_AUTOCTS;
-
-	serial8250_do_set_termios(p, termios, old);
-}
-
-static bool byt_dma_filter(struct dma_chan *chan, void *param)
-{
-	struct dw_dma_slave *dws = param;
-
-	if (dws->dma_dev != chan->device->dev)
-		return false;
-
-	chan->private = dws;
-	return true;
-}
-
-static unsigned int
-byt_get_mctrl(struct uart_port *port)
-{
-	unsigned int ret = serial8250_do_get_mctrl(port);
-
-	/* Force DCD and DSR signals to permanently be reported as active. */
-	ret |= TIOCM_CAR | TIOCM_DSR;
-
-	return ret;
-}
-
-static int
-byt_serial_setup(struct serial_private *priv,
-		 const struct pciserial_board *board,
-		 struct uart_8250_port *port, int idx)
-{
-	struct pci_dev *pdev = priv->dev;
-	struct device *dev = port->port.dev;
-	struct uart_8250_dma *dma;
-	struct dw_dma_slave *tx_param, *rx_param;
-	struct pci_dev *dma_dev;
-	int ret;
-
-	dma = devm_kzalloc(dev, sizeof(*dma), GFP_KERNEL);
-	if (!dma)
-		return -ENOMEM;
-
-	tx_param = devm_kzalloc(dev, sizeof(*tx_param), GFP_KERNEL);
-	if (!tx_param)
-		return -ENOMEM;
-
-	rx_param = devm_kzalloc(dev, sizeof(*rx_param), GFP_KERNEL);
-	if (!rx_param)
-		return -ENOMEM;
-
-	switch (pdev->device) {
-	case PCI_DEVICE_ID_INTEL_BYT_UART1:
-	case PCI_DEVICE_ID_INTEL_BSW_UART1:
-	case PCI_DEVICE_ID_INTEL_BDW_UART1:
-		rx_param->src_id = 3;
-		tx_param->dst_id = 2;
-		break;
-	case PCI_DEVICE_ID_INTEL_BYT_UART2:
-	case PCI_DEVICE_ID_INTEL_BSW_UART2:
-	case PCI_DEVICE_ID_INTEL_BDW_UART2:
-		rx_param->src_id = 5;
-		tx_param->dst_id = 4;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	rx_param->m_master = 0;
-	rx_param->p_master = 1;
-
-	dma->rxconf.src_maxburst = 16;
-
-	tx_param->m_master = 0;
-	tx_param->p_master = 1;
-
-	dma->txconf.dst_maxburst = 16;
-
-	dma_dev = pci_get_slot(pdev->bus, PCI_DEVFN(PCI_SLOT(pdev->devfn), 0));
-	rx_param->dma_dev = &dma_dev->dev;
-	tx_param->dma_dev = &dma_dev->dev;
-
-	dma->fn = byt_dma_filter;
-	dma->rx_param = rx_param;
-	dma->tx_param = tx_param;
-
-	ret = pci_default_setup(priv, board, port, idx);
-	port->port.iotype = UPIO_MEM;
-	port->port.type = PORT_16550A;
-	port->port.flags = (port->port.flags | UPF_FIXED_PORT | UPF_FIXED_TYPE);
-	port->port.set_termios = byt_set_termios;
-	port->port.get_mctrl = byt_get_mctrl;
-	port->port.fifosize = 64;
-	port->tx_loadsz = 64;
-	port->dma = dma;
-	port->capabilities = UART_CAP_FIFO | UART_CAP_AFE;
-
-	/* Disable Tx counter interrupts */
-	writel(BYT_TX_OVF_INT_MASK, port->port.membase + BYT_TX_OVF_INT);
-
-	return ret;
-}
-
 static int
 pci_omegapci_setup(struct serial_private *priv,
 		      const struct pciserial_board *board,
@@ -1741,6 +1583,19 @@ static int pci_eg20t_init(struct pci_dev *dev)
 #define PCI_DEVICE_ID_EXAR_XR17V4358	0x4358
 #define PCI_DEVICE_ID_EXAR_XR17V8358	0x8358
 
+#define UART_EXAR_MPIOINT_7_0	0x8f	/* MPIOINT[7:0] */
+#define UART_EXAR_MPIOLVL_7_0	0x90	/* MPIOLVL[7:0] */
+#define UART_EXAR_MPIO3T_7_0	0x91	/* MPIO3T[7:0] */
+#define UART_EXAR_MPIOINV_7_0	0x92	/* MPIOINV[7:0] */
+#define UART_EXAR_MPIOSEL_7_0	0x93	/* MPIOSEL[7:0] */
+#define UART_EXAR_MPIOOD_7_0	0x94	/* MPIOOD[7:0] */
+#define UART_EXAR_MPIOINT_15_8	0x95	/* MPIOINT[15:8] */
+#define UART_EXAR_MPIOLVL_15_8	0x96	/* MPIOLVL[15:8] */
+#define UART_EXAR_MPIO3T_15_8	0x97	/* MPIO3T[15:8] */
+#define UART_EXAR_MPIOINV_15_8	0x98	/* MPIOINV[15:8] */
+#define UART_EXAR_MPIOSEL_15_8	0x99	/* MPIOSEL[15:8] */
+#define UART_EXAR_MPIOOD_15_8	0x9a	/* MPIOOD[15:8] */
+
 static int
 pci_xr17c154_setup(struct serial_private *priv,
 		  const struct pciserial_board *board,
@@ -1783,18 +1638,18 @@ pci_xr17v35x_setup(struct serial_private *priv,
 	 * Setup Multipurpose Input/Output pins.
 	 */
 	if (idx == 0) {
-		writeb(0x00, p + 0x8f); /*MPIOINT[7:0]*/
-		writeb(0x00, p + 0x90); /*MPIOLVL[7:0]*/
-		writeb(0x00, p + 0x91); /*MPIO3T[7:0]*/
-		writeb(0x00, p + 0x92); /*MPIOINV[7:0]*/
-		writeb(0x00, p + 0x93); /*MPIOSEL[7:0]*/
-		writeb(0x00, p + 0x94); /*MPIOOD[7:0]*/
-		writeb(0x00, p + 0x95); /*MPIOINT[15:8]*/
-		writeb(0x00, p + 0x96); /*MPIOLVL[15:8]*/
-		writeb(0x00, p + 0x97); /*MPIO3T[15:8]*/
-		writeb(0x00, p + 0x98); /*MPIOINV[15:8]*/
-		writeb(0x00, p + 0x99); /*MPIOSEL[15:8]*/
-		writeb(0x00, p + 0x9a); /*MPIOOD[15:8]*/
+		writeb(0x00, p + UART_EXAR_MPIOINT_7_0);
+		writeb(0x00, p + UART_EXAR_MPIOLVL_7_0);
+		writeb(0x00, p + UART_EXAR_MPIO3T_7_0);
+		writeb(0x00, p + UART_EXAR_MPIOINV_7_0);
+		writeb(0x00, p + UART_EXAR_MPIOSEL_7_0);
+		writeb(0x00, p + UART_EXAR_MPIOOD_7_0);
+		writeb(0x00, p + UART_EXAR_MPIOINT_15_8);
+		writeb(0x00, p + UART_EXAR_MPIOLVL_15_8);
+		writeb(0x00, p + UART_EXAR_MPIO3T_15_8);
+		writeb(0x00, p + UART_EXAR_MPIOINV_15_8);
+		writeb(0x00, p + UART_EXAR_MPIOSEL_15_8);
+		writeb(0x00, p + UART_EXAR_MPIOOD_15_8);
 	}
 	writeb(0x00, p + UART_EXAR_8XMODE);
 	writeb(UART_FCTR_EXAR_TRGD, p + UART_EXAR_FCTR);
@@ -1830,20 +1685,20 @@ pci_fastcom335_setup(struct serial_private *priv,
 		switch (priv->dev->device) {
 		case PCI_DEVICE_ID_COMMTECH_4222PCI335:
 		case PCI_DEVICE_ID_COMMTECH_4224PCI335:
-			writeb(0x78, p + 0x90); /* MPIOLVL[7:0] */
-			writeb(0x00, p + 0x92); /* MPIOINV[7:0] */
-			writeb(0x00, p + 0x93); /* MPIOSEL[7:0] */
+			writeb(0x78, p + UART_EXAR_MPIOLVL_7_0);
+			writeb(0x00, p + UART_EXAR_MPIOINV_7_0);
+			writeb(0x00, p + UART_EXAR_MPIOSEL_7_0);
 			break;
 		case PCI_DEVICE_ID_COMMTECH_2324PCI335:
 		case PCI_DEVICE_ID_COMMTECH_2328PCI335:
-			writeb(0x00, p + 0x90); /* MPIOLVL[7:0] */
-			writeb(0xc0, p + 0x92); /* MPIOINV[7:0] */
-			writeb(0xc0, p + 0x93); /* MPIOSEL[7:0] */
+			writeb(0x00, p + UART_EXAR_MPIOLVL_7_0);
+			writeb(0xc0, p + UART_EXAR_MPIOINV_7_0);
+			writeb(0xc0, p + UART_EXAR_MPIOSEL_7_0);
 			break;
 		}
-		writeb(0x00, p + 0x8f); /* MPIOINT[7:0] */
-		writeb(0x00, p + 0x91); /* MPIO3T[7:0] */
-		writeb(0x00, p + 0x94); /* MPIOOD[7:0] */
+		writeb(0x00, p + UART_EXAR_MPIOINT_7_0);
+		writeb(0x00, p + UART_EXAR_MPIO3T_7_0);
+		writeb(0x00, p + UART_EXAR_MPIOOD_7_0);
 	}
 	writeb(0x00, p + UART_EXAR_8XMODE);
 	writeb(UART_FCTR_EXAR_TRGD, p + UART_EXAR_FCTR);
@@ -1934,7 +1789,6 @@ pci_wch_ch38x_setup(struct serial_private *priv,
 #define PCI_DEVICE_ID_COMMTECH_4222PCIE	0x0022
 #define PCI_DEVICE_ID_BROADCOM_TRUMANAGE 0x160a
 #define PCI_DEVICE_ID_AMCC_ADDIDATA_APCI7800 0x818e
-#define PCI_DEVICE_ID_INTEL_QRK_UART	0x0936
 
 #define PCI_VENDOR_ID_SUNIX		0x1fd4
 #define PCI_DEVICE_ID_SUNIX_1999	0x1999
@@ -1950,6 +1804,43 @@ pci_wch_ch38x_setup(struct serial_private *priv,
 #define PCI_DEVICE_ID_PERICOM_PI7C9X7954	0x7954
 #define PCI_DEVICE_ID_PERICOM_PI7C9X7958	0x7958
 
+#define PCI_VENDOR_ID_ACCESIO			0x494f
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM_2SDB	0x1051
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_COM_2S	0x1053
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM_4SDB	0x105C
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_COM_4S	0x105E
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM232_2DB	0x1091
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_COM232_2	0x1093
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM232_4DB	0x1099
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_COM232_4	0x109B
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM_2SMDB	0x10D1
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_COM_2SM	0x10D3
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM_4SMDB	0x10DA
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_COM_4SM	0x10DC
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_ICM485_1	0x1108
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_ICM422_2	0x1110
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_ICM485_2	0x1111
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_ICM422_4	0x1118
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_ICM485_4	0x1119
+#define PCI_DEVICE_ID_ACCESIO_PCIE_ICM_2S	0x1152
+#define PCI_DEVICE_ID_ACCESIO_PCIE_ICM_4S	0x115A
+#define PCI_DEVICE_ID_ACCESIO_PCIE_ICM232_2	0x1190
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_ICM232_2	0x1191
+#define PCI_DEVICE_ID_ACCESIO_PCIE_ICM232_4	0x1198
+#define PCI_DEVICE_ID_ACCESIO_MPCIE_ICM232_4	0x1199
+#define PCI_DEVICE_ID_ACCESIO_PCIE_ICM_2SM	0x11D0
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM422_4	0x105A
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM485_4	0x105B
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM422_8	0x106A
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM485_8	0x106B
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM232_4	0x1098
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM232_8	0x10A9
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM_4SM	0x10D9
+#define PCI_DEVICE_ID_ACCESIO_PCIE_COM_8SM	0x10E9
+#define PCI_DEVICE_ID_ACCESIO_PCIE_ICM_4SM	0x11D8
+
+
+
 /* Unknown vendors/cards - this should not be in linux/pci_ids.h */
 #define PCI_SUBDEVICE_ID_UNKNOWN_0x1584	0x1584
 #define PCI_SUBDEVICE_ID_UNKNOWN_0x1588	0x1588
@@ -2041,48 +1932,6 @@ static struct pci_serial_quirk pci_serial_quirks[] __refdata = {
 		.subdevice	= PCI_ANY_ID,
 		.setup		= kt_serial_setup,
 	},
-	{
-		.vendor		= PCI_VENDOR_ID_INTEL,
-		.device		= PCI_DEVICE_ID_INTEL_BYT_UART1,
-		.subvendor	= PCI_ANY_ID,
-		.subdevice	= PCI_ANY_ID,
-		.setup		= byt_serial_setup,
-	},
-	{
-		.vendor		= PCI_VENDOR_ID_INTEL,
-		.device		= PCI_DEVICE_ID_INTEL_BYT_UART2,
-		.subvendor	= PCI_ANY_ID,
-		.subdevice	= PCI_ANY_ID,
-		.setup		= byt_serial_setup,
-	},
-	{
-		.vendor		= PCI_VENDOR_ID_INTEL,
-		.device		= PCI_DEVICE_ID_INTEL_BSW_UART1,
-		.subvendor	= PCI_ANY_ID,
-		.subdevice	= PCI_ANY_ID,
-		.setup		= byt_serial_setup,
-	},
-	{
-		.vendor		= PCI_VENDOR_ID_INTEL,
-		.device		= PCI_DEVICE_ID_INTEL_BSW_UART2,
-		.subvendor	= PCI_ANY_ID,
-		.subdevice	= PCI_ANY_ID,
-		.setup		= byt_serial_setup,
-	},
-	{
-		.vendor		= PCI_VENDOR_ID_INTEL,
-		.device		= PCI_DEVICE_ID_INTEL_BDW_UART1,
-		.subvendor	= PCI_ANY_ID,
-		.subdevice	= PCI_ANY_ID,
-		.setup		= byt_serial_setup,
-	},
-	{
-		.vendor		= PCI_VENDOR_ID_INTEL,
-		.device		= PCI_DEVICE_ID_INTEL_BDW_UART2,
-		.subvendor	= PCI_ANY_ID,
-		.subdevice	= PCI_ANY_ID,
-		.setup		= byt_serial_setup,
-	},
 	/*
 	 * ITE
 	 */
@@ -2955,8 +2804,6 @@ enum pci_board_num_t {
 	pbn_ADDIDATA_PCIe_4_3906250,
 	pbn_ADDIDATA_PCIe_8_3906250,
 	pbn_ce4100_1_115200,
-	pbn_byt,
-	pbn_qrk,
 	pbn_omegapci,
 	pbn_NETMOS9900_2s_115200,
 	pbn_brcm_trumanage,
@@ -3732,18 +3579,6 @@ static struct pciserial_board pci_boards[] = {
 		.base_baud	= 921600,
 		.reg_shift      = 2,
 	},
-	[pbn_byt] = {
-		.flags		= FL_BASE0,
-		.num_ports	= 1,
-		.base_baud	= 2764800,
-		.reg_shift      = 2,
-	},
-	[pbn_qrk] = {
-		.flags		= FL_BASE0,
-		.num_ports	= 1,
-		.base_baud	= 2764800,
-		.reg_shift	= 2,
-	},
 	[pbn_omegapci] = {
 		.flags		= FL_BASE0,
 		.num_ports	= 8,
@@ -3855,6 +3690,15 @@ static const struct pci_device_id blacklist[] = {
 	{ PCI_VDEVICE(INTEL, 0x081d), },
 	{ PCI_VDEVICE(INTEL, 0x1191), },
 	{ PCI_VDEVICE(INTEL, 0x19d8), },
+
+	/* Intel platforms with DesignWare UART */
+	{ PCI_VDEVICE(INTEL, 0x0936), },
+	{ PCI_VDEVICE(INTEL, 0x0f0a), },
+	{ PCI_VDEVICE(INTEL, 0x0f0c), },
+	{ PCI_VDEVICE(INTEL, 0x228a), },
+	{ PCI_VDEVICE(INTEL, 0x228c), },
+	{ PCI_VDEVICE(INTEL, 0x9ce3), },
+	{ PCI_VDEVICE(INTEL, 0x9ce4), },
 };
 
 /*
@@ -5113,6 +4957,108 @@ static struct pci_device_id serial_pci_tbl[] = {
 		0,
 		0, pbn_pericom_PI7C9X7958 },
 	/*
+	 * ACCES I/O Products quad
+	 */
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM_2SDB,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_COM_2S,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM_4SDB,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_COM_4S,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM232_2DB,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_COM232_2,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM232_4DB,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_COM232_4,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM_2SMDB,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_COM_2SM,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM_4SMDB,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_COM_4SM,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_ICM485_1,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_ICM422_2,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_ICM485_2,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_ICM422_4,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_ICM485_4,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_ICM_2S,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_ICM_4S,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_ICM232_2,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_ICM232_2,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_ICM232_4,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_MPCIE_ICM232_4,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_ICM_2SM,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7954 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM422_4,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7958 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM485_4,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7958 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM422_8,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7958 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM485_8,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7958 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM232_4,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7958 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM232_8,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7958 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM_4SM,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7958 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_COM_8SM,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7958 },
+	{	PCI_VENDOR_ID_ACCESIO, PCI_DEVICE_ID_ACCESIO_PCIE_ICM_4SM,
+		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+		pbn_pericom_PI7C9X7958 },
+	/*
 	 * Topic TP560 Data/Fax/Voice 56k modem (reported by Evan Clarke)
 	 */
 	{	PCI_VENDOR_ID_TOPIC, PCI_DEVICE_ID_TOPIC_TP560,
@@ -5520,41 +5466,8 @@ static struct pci_device_id serial_pci_tbl[] = {
 	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CE4100_UART,
 		PCI_ANY_ID,  PCI_ANY_ID, 0, 0,
 		pbn_ce4100_1_115200 },
-	/* Intel BayTrail */
-	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BYT_UART1,
-		PCI_ANY_ID,  PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_SERIAL << 8, 0xff0000,
-		pbn_byt },
-	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BYT_UART2,
-		PCI_ANY_ID,  PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_SERIAL << 8, 0xff0000,
-		pbn_byt },
-	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BSW_UART1,
-		PCI_ANY_ID,  PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_SERIAL << 8, 0xff0000,
-		pbn_byt },
-	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BSW_UART2,
-		PCI_ANY_ID,  PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_SERIAL << 8, 0xff0000,
-		pbn_byt },
-
-	/* Intel Broadwell */
-	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BDW_UART1,
-		PCI_ANY_ID,  PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_SERIAL << 8, 0xff0000,
-		pbn_byt },
-	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BDW_UART2,
-		PCI_ANY_ID,  PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_SERIAL << 8, 0xff0000,
-		pbn_byt },
 
 	/*
-	 * Intel Quark x1000
-	 */
-	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_QRK_UART,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_qrk },
-	/*
 	 * Cronyx Omega PCI
 	 */
 	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_CRONYX_OMEGA,
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index 7481b95c6d84..1bfb6fdbaa20 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -178,7 +178,7 @@ static const struct serial8250_config uart_config[] = {
 		.fifo_size	= 16,
 		.tx_loadsz	= 16,
 		.fcr		= UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_00,
-		.flags		= UART_CAP_FIFO | UART_CAP_AFE,
+		.flags		= UART_CAP_FIFO /* | UART_CAP_AFE */,
 	},
 	[PORT_U6_16550A] = {
 		.name		= "U6_16550A",
@@ -585,11 +585,11 @@ EXPORT_SYMBOL_GPL(serial8250_rpm_put);
  */
 int serial8250_em485_init(struct uart_8250_port *p)
 {
-	if (p->em485 != NULL)
+	if (p->em485)
 		return 0;
 
 	p->em485 = kmalloc(sizeof(struct uart_8250_em485), GFP_ATOMIC);
-	if (p->em485 == NULL)
+	if (!p->em485)
 		return -ENOMEM;
 
 	setup_timer(&p->em485->stop_tx_timer,
@@ -619,7 +619,7 @@ EXPORT_SYMBOL_GPL(serial8250_em485_init);
  */
 void serial8250_em485_destroy(struct uart_8250_port *p)
 {
-	if (p->em485 == NULL)
+	if (!p->em485)
 		return;
 
 	del_timer(&p->em485->start_tx_timer);
@@ -1402,10 +1402,8 @@ static void serial8250_stop_rx(struct uart_port *port)
 
 static void __do_stop_tx_rs485(struct uart_8250_port *p)
 {
-	if (!p->em485)
-		return;
-
 	serial8250_em485_rts_after_send(p);
+
 	/*
 	 * Empty the RX FIFO, we are not interested in anything
 	 * received during the half-duplex transmission.
@@ -1414,12 +1412,8 @@ static void __do_stop_tx_rs485(struct uart_8250_port *p)
 	if (!(p->port.rs485.flags & SER_RS485_RX_DURING_TX)) {
 		serial8250_clear_fifos(p);
 
-		serial8250_rpm_get(p);
-
 		p->ier |= UART_IER_RLSI | UART_IER_RDI;
 		serial_port_out(&p->port, UART_IER, p->ier);
-
-		serial8250_rpm_put(p);
 	}
 }
 
@@ -1429,6 +1423,7 @@ static void serial8250_em485_handle_stop_tx(unsigned long arg)
 	struct uart_8250_em485 *em485 = p->em485;
 	unsigned long flags;
 
+	serial8250_rpm_get(p);
 	spin_lock_irqsave(&p->port.lock, flags);
 	if (em485 &&
 	    em485->active_timer == &em485->stop_tx_timer) {
@@ -1436,15 +1431,13 @@ static void serial8250_em485_handle_stop_tx(unsigned long arg)
 		em485->active_timer = NULL;
 	}
 	spin_unlock_irqrestore(&p->port.lock, flags);
+	serial8250_rpm_put(p);
 }
 
 static void __stop_tx_rs485(struct uart_8250_port *p)
 {
 	struct uart_8250_em485 *em485 = p->em485;
 
-	if (!em485)
-		return;
-
 	/*
 	 * __do_stop_tx_rs485 is going to set RTS according to config
 	 * AND flush RX FIFO if required.
@@ -1475,7 +1468,7 @@ static inline void __stop_tx(struct uart_8250_port *p)
 		unsigned char lsr = serial_in(p, UART_LSR);
 		/*
 		 * To provide required timeing and allow FIFO transfer,
-		 * __stop_tx_rs485 must be called only when both FIFO and
+		 * __stop_tx_rs485() must be called only when both FIFO and
 		 * shift register are empty. It is for device driver to enable
 		 * interrupt on TEMT.
 		 */
@@ -1484,9 +1477,10 @@ static inline void __stop_tx(struct uart_8250_port *p)
 
 		del_timer(&em485->start_tx_timer);
 		em485->active_timer = NULL;
+
+		__stop_tx_rs485(p);
 	}
 	__do_stop_tx(p);
-	__stop_tx_rs485(p);
 }
 
 static void serial8250_stop_tx(struct uart_port *port)
@@ -1618,8 +1612,6 @@ static void serial8250_disable_ms(struct uart_port *port)
 	if (up->bugs & UART_BUG_NOMSR)
 		return;
 
-	mctrl_gpio_disable_ms(up->gpios);
-
 	up->ier &= ~UART_IER_MSI;
 	serial_port_out(port, UART_IER, up->ier);
 }
@@ -1632,8 +1624,6 @@ static void serial8250_enable_ms(struct uart_port *port)
 	if (up->bugs & UART_BUG_NOMSR)
 		return;
 
-	mctrl_gpio_enable_ms(up->gpios);
-
 	up->ier |= UART_IER_MSI;
 
 	serial8250_rpm_get(up);
@@ -1880,6 +1870,30 @@ static int exar_handle_irq(struct uart_port *port)
 	return ret;
 }
 
+/*
+ * Newer 16550 compatible parts such as the SC16C650 & Altera 16550 Soft IP
+ * have a programmable TX threshold that triggers the THRE interrupt in
+ * the IIR register. In this case, the THRE interrupt indicates the FIFO
+ * has space available. Load it up with tx_loadsz bytes.
+ */
+static int serial8250_tx_threshold_handle_irq(struct uart_port *port)
+{
+	unsigned long flags;
+	unsigned int iir = serial_port_in(port, UART_IIR);
+
+	/* TX Threshold IRQ triggered so load up FIFO */
+	if ((iir & UART_IIR_ID) == UART_IIR_THRI) {
+		struct uart_8250_port *up = up_to_u8250p(port);
+
+		spin_lock_irqsave(&port->lock, flags);
+		serial8250_tx_chars(up);
+		spin_unlock_irqrestore(&port->lock, flags);
+	}
+
+	iir = serial_port_in(port, UART_IIR);
+	return serial8250_handle_irq(port, iir);
+}
+
 static unsigned int serial8250_tx_empty(struct uart_port *port)
 {
 	struct uart_8250_port *up = up_to_u8250p(port);
@@ -1917,8 +1931,7 @@ unsigned int serial8250_do_get_mctrl(struct uart_port *port)
 		ret |= TIOCM_DSR;
 	if (status & UART_MSR_CTS)
 		ret |= TIOCM_CTS;
-
-	return mctrl_gpio_get(up->gpios, &ret);
+	return ret;
 }
 EXPORT_SYMBOL_GPL(serial8250_do_get_mctrl);
 
@@ -1993,6 +2006,7 @@ static void wait_for_xmitr(struct uart_8250_port *up, int bits)
 		if (--tmout == 0)
 			break;
 		udelay(1);
+		touch_nmi_watchdog();
 	}
 
 	/* Wait up to 1s for flow control if necessary */
@@ -2169,6 +2183,25 @@ int serial8250_do_startup(struct uart_port *port)
 		serial_port_out(port, UART_LCR, 0);
 	}
 
+	/*
+	 * For the Altera 16550 variants, set TX threshold trigger level.
+	 */
+	if (((port->type == PORT_ALTR_16550_F32) ||
+	     (port->type == PORT_ALTR_16550_F64) ||
+	     (port->type == PORT_ALTR_16550_F128)) && (port->fifosize > 1)) {
+		/* Bounds checking of TX threshold (valid 0 to fifosize-2) */
+		if ((up->tx_loadsz < 2) || (up->tx_loadsz > port->fifosize)) {
+			pr_err("ttyS%d TX FIFO Threshold errors, skipping\n",
+			       serial_index(port));
+		} else {
+			serial_port_out(port, UART_ALTR_AFR,
+					UART_ALTR_EN_TXFIFO_LW);
+			serial_port_out(port, UART_ALTR_TX_LOW,
+					port->fifosize - up->tx_loadsz);
+			port->handle_irq = serial8250_tx_threshold_handle_irq;
+		}
+	}
+
 	if (port->irq) {
 		unsigned char iir1;
 		/*
@@ -2504,8 +2537,6 @@ static unsigned int serial8250_get_baud_rate(struct uart_port *port,
 					     struct ktermios *termios,
 					     struct ktermios *old)
 {
-	unsigned int tolerance = port->uartclk / 100;
-
 	/*
 	 * Ask the core to calculate the divisor for us.
 	 * Allow 1% tolerance at the upper limit so uart clks marginally
@@ -2514,7 +2545,7 @@ static unsigned int serial8250_get_baud_rate(struct uart_port *port,
 	 */
 	return uart_get_baud_rate(port, termios, old,
 				  port->uartclk / 16 / 0xffff,
-				  (port->uartclk + tolerance) / 16);
+				  port->uartclk);
 }
 
 void
@@ -2551,12 +2582,9 @@ serial8250_do_set_termios(struct uart_port *port, struct ktermios *termios,
 	/*
 	 * MCR-based auto flow control.  When AFE is enabled, RTS will be
 	 * deasserted when the receive FIFO contains more characters than
-	 * the trigger, or the MCR RTS bit is cleared.  In the case where
-	 * the remote UART is not using CTS auto flow control, we must
-	 * have sufficient FIFO entries for the latency of the remote
-	 * UART to respond.  IOW, at least 32 bytes of FIFO.
+	 * the trigger, or the MCR RTS bit is cleared.
 	 */
-	if (up->capabilities & UART_CAP_AFE && port->fifosize >= 32) {
+	if (up->capabilities & UART_CAP_AFE) {
 		up->mcr &= ~UART_MCR_AFE;
 		if (termios->c_cflag & CRTSCTS)
 			up->mcr |= UART_MCR_AFE;
diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig
index c9ec839a5ddf..899834776b36 100644
--- a/drivers/tty/serial/8250/Kconfig
+++ b/drivers/tty/serial/8250/Kconfig
@@ -6,7 +6,6 @@
 config SERIAL_8250
 	tristate "8250/16550 and compatible serial support"
 	select SERIAL_CORE
-	select SERIAL_MCTRL_GPIO if GPIOLIB
 	---help---
 	  This selects whether you want to include the driver for the standard
 	  serial ports.  The standard answer is Y.  People who might say N
@@ -121,7 +120,6 @@ config SERIAL_8250_PCI
 	tristate "8250/16550 PCI device support" if EXPERT
 	depends on SERIAL_8250 && PCI
 	default SERIAL_8250
-	select RATIONAL
 	help
 	  This builds standard PCI serial support. You may be able to
 	  disable this feature if you only need legacy serial support.
@@ -403,6 +401,21 @@ config SERIAL_8250_INGENIC
 	  If you have a system using an Ingenic SoC and wish to make use of
 	  its UARTs, say Y to this option. If unsure, say N.
 
+config SERIAL_8250_LPSS
+	tristate "Support for serial ports on Intel LPSS platforms" if EXPERT
+	default SERIAL_8250
+	depends on SERIAL_8250 && PCI
+	depends on X86 || COMPILE_TEST
+	select DW_DMAC_CORE if SERIAL_8250_DMA
+	select DW_DMAC_PCI if (SERIAL_8250_DMA && X86_INTEL_LPSS)
+	select RATIONAL
+	help
+	  Selecting this option will enable handling of the extra features
+	  present on the UART found on various Intel platforms such as:
+	    - Intel Baytrail SoC
+	    - Intel Braswell SoC
+	    - Intel Quark X1000 SoC
+
 config SERIAL_8250_MID
 	tristate "Support for serial ports on Intel MID platforms" if EXPERT
 	default SERIAL_8250
diff --git a/drivers/tty/serial/8250/Makefile b/drivers/tty/serial/8250/Makefile
index 367d403d28d5..276c6fb60337 100644
--- a/drivers/tty/serial/8250/Makefile
+++ b/drivers/tty/serial/8250/Makefile
@@ -28,6 +28,7 @@ obj-$(CONFIG_SERIAL_8250_LPC18XX)	+= 8250_lpc18xx.o
 obj-$(CONFIG_SERIAL_8250_MT6577)	+= 8250_mtk.o
 obj-$(CONFIG_SERIAL_8250_UNIPHIER)	+= 8250_uniphier.o
 obj-$(CONFIG_SERIAL_8250_INGENIC)	+= 8250_ingenic.o
+obj-$(CONFIG_SERIAL_8250_LPSS)		+= 8250_lpss.o
 obj-$(CONFIG_SERIAL_8250_MID)		+= 8250_mid.o
 obj-$(CONFIG_SERIAL_8250_MOXA)		+= 8250_moxa.o
 obj-$(CONFIG_SERIAL_OF_PLATFORM)	+= 8250_of.o
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index 518db24a5b36..c7831407a882 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -1380,7 +1380,7 @@ config SERIAL_IFX6X60
 
 config SERIAL_PCH_UART
 	tristate "Intel EG20T PCH/LAPIS Semicon IOH(ML7213/ML7223/ML7831) UART"
-	depends on PCI && (X86_32 || COMPILE_TEST)
+	depends on PCI && (X86_32 || MIPS ||  COMPILE_TEST)
 	select SERIAL_CORE
 	help
 	  This driver is for PCH(Platform controller Hub) UART of Intel EG20T
diff --git a/drivers/tty/serial/altera_jtaguart.c b/drivers/tty/serial/altera_jtaguart.c
index 32df2a0cb060..e409d7dac7ab 100644
--- a/drivers/tty/serial/altera_jtaguart.c
+++ b/drivers/tty/serial/altera_jtaguart.c
@@ -280,7 +280,7 @@ static int altera_jtaguart_verify_port(struct uart_port *port,
 /*
  *	Define the basic serial functions we support.
  */
-static struct uart_ops altera_jtaguart_ops = {
+static const struct uart_ops altera_jtaguart_ops = {
 	.tx_empty	= altera_jtaguart_tx_empty,
 	.get_mctrl	= altera_jtaguart_get_mctrl,
 	.set_mctrl	= altera_jtaguart_set_mctrl,
diff --git a/drivers/tty/serial/altera_uart.c b/drivers/tty/serial/altera_uart.c
index 61b607f2488e..820a74208696 100644
--- a/drivers/tty/serial/altera_uart.c
+++ b/drivers/tty/serial/altera_uart.c
@@ -404,7 +404,7 @@ static void altera_uart_poll_put_char(struct uart_port *port, unsigned char c)
 /*
  *	Define the basic serial functions we support.
  */
-static struct uart_ops altera_uart_ops = {
+static const struct uart_ops altera_uart_ops = {
 	.tx_empty	= altera_uart_tx_empty,
 	.get_mctrl	= altera_uart_get_mctrl,
 	.set_mctrl	= altera_uart_set_mctrl,
diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c
index 8a9e213387a7..e2c33b9528d8 100644
--- a/drivers/tty/serial/amba-pl011.c
+++ b/drivers/tty/serial/amba-pl011.c
@@ -93,6 +93,10 @@ static u16 pl011_std_offsets[REG_ARRAY_SIZE] = {
 struct vendor_data {
 	const u16		*reg_offset;
 	unsigned int		ifls;
+	unsigned int		fr_busy;
+	unsigned int		fr_dsr;
+	unsigned int		fr_cts;
+	unsigned int		fr_ri;
 	bool			access_32b;
 	bool			oversampling;
 	bool			dma_threshold;
@@ -111,6 +115,10 @@ static unsigned int get_fifosize_arm(struct amba_device *dev)
 static struct vendor_data vendor_arm = {
 	.reg_offset		= pl011_std_offsets,
 	.ifls			= UART011_IFLS_RX4_8|UART011_IFLS_TX4_8,
+	.fr_busy		= UART01x_FR_BUSY,
+	.fr_dsr			= UART01x_FR_DSR,
+	.fr_cts			= UART01x_FR_CTS,
+	.fr_ri			= UART011_FR_RI,
 	.oversampling		= false,
 	.dma_threshold		= false,
 	.cts_event_workaround	= false,
@@ -121,6 +129,10 @@ static struct vendor_data vendor_arm = {
 
 static struct vendor_data vendor_sbsa = {
 	.reg_offset		= pl011_std_offsets,
+	.fr_busy		= UART01x_FR_BUSY,
+	.fr_dsr			= UART01x_FR_DSR,
+	.fr_cts			= UART01x_FR_CTS,
+	.fr_ri			= UART011_FR_RI,
 	.access_32b		= true,
 	.oversampling		= false,
 	.dma_threshold		= false,
@@ -164,6 +176,10 @@ static unsigned int get_fifosize_st(struct amba_device *dev)
 static struct vendor_data vendor_st = {
 	.reg_offset		= pl011_st_offsets,
 	.ifls			= UART011_IFLS_RX_HALF|UART011_IFLS_TX_HALF,
+	.fr_busy		= UART01x_FR_BUSY,
+	.fr_dsr			= UART01x_FR_DSR,
+	.fr_cts			= UART01x_FR_CTS,
+	.fr_ri			= UART011_FR_RI,
 	.oversampling		= true,
 	.dma_threshold		= true,
 	.cts_event_workaround	= true,
@@ -188,11 +204,20 @@ static const u16 pl011_zte_offsets[REG_ARRAY_SIZE] = {
 	[REG_DMACR] = ZX_UART011_DMACR,
 };
 
-static struct vendor_data vendor_zte __maybe_unused = {
+static unsigned int get_fifosize_zte(struct amba_device *dev)
+{
+	return 16;
+}
+
+static struct vendor_data vendor_zte = {
 	.reg_offset		= pl011_zte_offsets,
 	.access_32b		= true,
 	.ifls			= UART011_IFLS_RX4_8|UART011_IFLS_TX4_8,
-	.get_fifosize		= get_fifosize_arm,
+	.fr_busy		= ZX_UART01x_FR_BUSY,
+	.fr_dsr			= ZX_UART01x_FR_DSR,
+	.fr_cts			= ZX_UART01x_FR_CTS,
+	.fr_ri			= ZX_UART011_FR_RI,
+	.get_fifosize		= get_fifosize_zte,
 };
 
 /* Deals with DMA transactions */
@@ -1167,7 +1192,7 @@ static void pl011_dma_shutdown(struct uart_amba_port *uap)
 		return;
 
 	/* Disable RX and TX DMA */
-	while (pl011_read(uap, REG_FR) & UART01x_FR_BUSY)
+	while (pl011_read(uap, REG_FR) & uap->vendor->fr_busy)
 		cpu_relax();
 
 	spin_lock_irq(&uap->port.lock);
@@ -1416,11 +1441,12 @@ static void pl011_modem_status(struct uart_amba_port *uap)
 	if (delta & UART01x_FR_DCD)
 		uart_handle_dcd_change(&uap->port, status & UART01x_FR_DCD);
 
-	if (delta & UART01x_FR_DSR)
+	if (delta & uap->vendor->fr_dsr)
 		uap->port.icount.dsr++;
 
-	if (delta & UART01x_FR_CTS)
-		uart_handle_cts_change(&uap->port, status & UART01x_FR_CTS);
+	if (delta & uap->vendor->fr_cts)
+		uart_handle_cts_change(&uap->port,
+				       status & uap->vendor->fr_cts);
 
 	wake_up_interruptible(&uap->port.state->port.delta_msr_wait);
 }
@@ -1493,7 +1519,8 @@ static unsigned int pl011_tx_empty(struct uart_port *port)
 	struct uart_amba_port *uap =
 	    container_of(port, struct uart_amba_port, port);
 	unsigned int status = pl011_read(uap, REG_FR);
-	return status & (UART01x_FR_BUSY|UART01x_FR_TXFF) ? 0 : TIOCSER_TEMT;
+	return status & (uap->vendor->fr_busy | UART01x_FR_TXFF) ?
+							0 : TIOCSER_TEMT;
 }
 
 static unsigned int pl011_get_mctrl(struct uart_port *port)
@@ -1508,9 +1535,9 @@ static unsigned int pl011_get_mctrl(struct uart_port *port)
 		result |= tiocmbit
 
 	TIOCMBIT(UART01x_FR_DCD, TIOCM_CAR);
-	TIOCMBIT(UART01x_FR_DSR, TIOCM_DSR);
-	TIOCMBIT(UART01x_FR_CTS, TIOCM_CTS);
-	TIOCMBIT(UART011_FR_RI, TIOCM_RNG);
+	TIOCMBIT(uap->vendor->fr_dsr, TIOCM_DSR);
+	TIOCMBIT(uap->vendor->fr_cts, TIOCM_CTS);
+	TIOCMBIT(uap->vendor->fr_ri, TIOCM_RNG);
 #undef TIOCMBIT
 	return result;
 }
@@ -2191,7 +2218,7 @@ pl011_console_write(struct console *co, const char *s, unsigned int count)
 	 *	Finally, wait for transmitter to become empty
 	 *	and restore the TCR
 	 */
-	while (pl011_read(uap, REG_FR) & UART01x_FR_BUSY)
+	while (pl011_read(uap, REG_FR) & uap->vendor->fr_busy)
 		cpu_relax();
 	if (!uap->vendor->always_enabled)
 		pl011_write(old_cr, uap, REG_CR);
@@ -2555,7 +2582,8 @@ static int sbsa_uart_probe(struct platform_device *pdev)
 
 	ret = platform_get_irq(pdev, 0);
 	if (ret < 0) {
-		dev_err(&pdev->dev, "cannot obtain irq\n");
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "cannot obtain irq\n");
 		return ret;
 	}
 	uap->port.irq	= ret;
@@ -2622,6 +2650,11 @@ static struct amba_id pl011_ids[] = {
 		.mask	= 0x00ffffff,
 		.data	= &vendor_st,
 	},
+	{
+		.id	= AMBA_LINUX_ID(0x00, 0x1, 0xffe),
+		.mask	= 0x00ffffff,
+		.data	= &vendor_zte,
+	},
 	{ 0, 0 },
 };
 
diff --git a/drivers/tty/serial/arc_uart.c b/drivers/tty/serial/arc_uart.c
index 3a1de5c87cb4..5ac06fcaa9c6 100644
--- a/drivers/tty/serial/arc_uart.c
+++ b/drivers/tty/serial/arc_uart.c
@@ -464,7 +464,7 @@ static int arc_serial_poll_getchar(struct uart_port *port)
 }
 #endif
 
-static struct uart_ops arc_serial_pops = {
+static const struct uart_ops arc_serial_pops = {
 	.tx_empty	= arc_serial_tx_empty,
 	.set_mctrl	= arc_serial_set_mctrl,
 	.get_mctrl	= arc_serial_get_mctrl,
diff --git a/drivers/tty/serial/atmel_serial.c b/drivers/tty/serial/atmel_serial.c
index 2eaa18ddef61..fd8aa1f4ba78 100644
--- a/drivers/tty/serial/atmel_serial.c
+++ b/drivers/tty/serial/atmel_serial.c
@@ -166,6 +166,7 @@ struct atmel_uart_port {
 	u32			rts_low;
 	bool			ms_irq_enabled;
 	u32			rtor;	/* address of receiver timeout register if it exists */
+	bool			has_frac_baudrate;
 	bool			has_hw_timer;
 	struct timer_list	uart_timer;
 
@@ -1634,8 +1635,8 @@ static void atmel_init_property(struct atmel_uart_port *atmel_port,
 
 	if (np) {
 		/* DMA/PDC usage specification */
-		if (of_get_property(np, "atmel,use-dma-rx", NULL)) {
-			if (of_get_property(np, "dmas", NULL)) {
+		if (of_property_read_bool(np, "atmel,use-dma-rx")) {
+			if (of_property_read_bool(np, "dmas")) {
 				atmel_port->use_dma_rx  = true;
 				atmel_port->use_pdc_rx  = false;
 			} else {
@@ -1647,8 +1648,8 @@ static void atmel_init_property(struct atmel_uart_port *atmel_port,
 			atmel_port->use_pdc_rx  = false;
 		}
 
-		if (of_get_property(np, "atmel,use-dma-tx", NULL)) {
-			if (of_get_property(np, "dmas", NULL)) {
+		if (of_property_read_bool(np, "atmel,use-dma-tx")) {
+			if (of_property_read_bool(np, "dmas")) {
 				atmel_port->use_dma_tx  = true;
 				atmel_port->use_pdc_tx  = false;
 			} else {
@@ -1745,6 +1746,11 @@ static void atmel_get_ip_name(struct uart_port *port)
 	dbgu_uart = 0x44424755;	/* DBGU */
 	new_uart = 0x55415254;	/* UART */
 
+	/*
+	 * Only USART devices from at91sam9260 SOC implement fractional
+	 * baudrate.
+	 */
+	atmel_port->has_frac_baudrate = false;
 	atmel_port->has_hw_timer = false;
 
 	if (name == new_uart) {
@@ -1753,6 +1759,7 @@ static void atmel_get_ip_name(struct uart_port *port)
 		atmel_port->rtor = ATMEL_UA_RTOR;
 	} else if (name == usart) {
 		dev_dbg(port->dev, "Usart\n");
+		atmel_port->has_frac_baudrate = true;
 		atmel_port->has_hw_timer = true;
 		atmel_port->rtor = ATMEL_US_RTOR;
 	} else if (name == dbgu_uart) {
@@ -1764,6 +1771,7 @@ static void atmel_get_ip_name(struct uart_port *port)
 		case 0x302:
 		case 0x10213:
 			dev_dbg(port->dev, "This version is usart\n");
+			atmel_port->has_frac_baudrate = true;
 			atmel_port->has_hw_timer = true;
 			atmel_port->rtor = ATMEL_US_RTOR;
 			break;
@@ -1929,6 +1937,9 @@ static void atmel_shutdown(struct uart_port *port)
 {
 	struct atmel_uart_port *atmel_port = to_atmel_uart_port(port);
 
+	/* Disable modem control lines interrupts */
+	atmel_disable_ms(port);
+
 	/* Disable interrupts at device level */
 	atmel_uart_writel(port, ATMEL_US_IDR, -1);
 
@@ -1979,8 +1990,6 @@ static void atmel_shutdown(struct uart_port *port)
 	 */
 	free_irq(port->irq, port);
 
-	atmel_port->ms_irq_enabled = false;
-
 	atmel_flush_buffer(port);
 }
 
@@ -2025,8 +2034,9 @@ static void atmel_serial_pm(struct uart_port *port, unsigned int state,
 static void atmel_set_termios(struct uart_port *port, struct ktermios *termios,
 			      struct ktermios *old)
 {
+	struct atmel_uart_port *atmel_port = to_atmel_uart_port(port);
 	unsigned long flags;
-	unsigned int old_mode, mode, imr, quot, baud;
+	unsigned int old_mode, mode, imr, quot, baud, div, cd, fp = 0;
 
 	/* save the current mode register */
 	mode = old_mode = atmel_uart_readl(port, ATMEL_US_MR);
@@ -2036,12 +2046,6 @@ static void atmel_set_termios(struct uart_port *port, struct ktermios *termios,
 		  ATMEL_US_PAR | ATMEL_US_USMODE);
 
 	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
-	quot = uart_get_divisor(port, baud);
-
-	if (quot > 65535) {	/* BRGR is 16-bit, so switch to slower clock */
-		quot /= 8;
-		mode |= ATMEL_US_USCLKS_MCK_DIV8;
-	}
 
 	/* byte size */
 	switch (termios->c_cflag & CSIZE) {
@@ -2160,7 +2164,31 @@ static void atmel_set_termios(struct uart_port *port, struct ktermios *termios,
 		atmel_uart_writel(port, ATMEL_US_CR, rts_state);
 	}
 
-	/* set the baud rate */
+	/*
+	 * Set the baud rate:
+	 * Fractional baudrate allows to setup output frequency more
+	 * accurately. This feature is enabled only when using normal mode.
+	 * baudrate = selected clock / (8 * (2 - OVER) * (CD + FP / 8))
+	 * Currently, OVER is always set to 0 so we get
+	 * baudrate = selected clock / (16 * (CD + FP / 8))
+	 * then
+	 * 8 CD + FP = selected clock / (2 * baudrate)
+	 */
+	if (atmel_port->has_frac_baudrate &&
+	    (mode & ATMEL_US_USMODE) == ATMEL_US_USMODE_NORMAL) {
+		div = DIV_ROUND_CLOSEST(port->uartclk, baud * 2);
+		cd = div >> 3;
+		fp = div & ATMEL_US_FP_MASK;
+	} else {
+		cd = uart_get_divisor(port, baud);
+	}
+
+	if (cd > 65535) {	/* BRGR is 16-bit, so switch to slower clock */
+		cd /= 8;
+		mode |= ATMEL_US_USCLKS_MCK_DIV8;
+	}
+	quot = cd | fp << ATMEL_US_FP_OFFSET;
+
 	atmel_uart_writel(port, ATMEL_US_BRGR, quot);
 	atmel_uart_writel(port, ATMEL_US_CR, ATMEL_US_RSTSTA | ATMEL_US_RSTRX);
 	atmel_uart_writel(port, ATMEL_US_CR, ATMEL_US_TXEN | ATMEL_US_RXEN);
@@ -2292,7 +2320,7 @@ static void atmel_poll_put_char(struct uart_port *port, unsigned char ch)
 }
 #endif
 
-static struct uart_ops atmel_pops = {
+static const struct uart_ops atmel_pops = {
 	.tx_empty	= atmel_tx_empty,
 	.set_mctrl	= atmel_set_mctrl,
 	.get_mctrl	= atmel_get_mctrl,
diff --git a/drivers/tty/serial/bcm63xx_uart.c b/drivers/tty/serial/bcm63xx_uart.c
index 5108fab953aa..583c9a0c7ecc 100644
--- a/drivers/tty/serial/bcm63xx_uart.c
+++ b/drivers/tty/serial/bcm63xx_uart.c
@@ -631,7 +631,7 @@ static int bcm_uart_verify_port(struct uart_port *port,
 }
 
 /* serial core callbacks */
-static struct uart_ops bcm_uart_ops = {
+static const struct uart_ops bcm_uart_ops = {
 	.tx_empty	= bcm_uart_tx_empty,
 	.get_mctrl	= bcm_uart_get_mctrl,
 	.set_mctrl	= bcm_uart_set_mctrl,
diff --git a/drivers/tty/serial/earlycon-arm-semihost.c b/drivers/tty/serial/earlycon-arm-semihost.c
index 383db10fbb49..6bbeb699777c 100644
--- a/drivers/tty/serial/earlycon-arm-semihost.c
+++ b/drivers/tty/serial/earlycon-arm-semihost.c
@@ -53,7 +53,8 @@ static void smh_write(struct console *con, const char *s, unsigned n)
 	uart_console_write(&dev->port, s, n, smh_putc);
 }
 
-int __init early_smh_setup(struct earlycon_device *device, const char *opt)
+static int
+__init early_smh_setup(struct earlycon_device *device, const char *opt)
 {
 	device->con->write = smh_write;
 	return 0;
diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c
index 067783f0523c..c3651540e1ba 100644
--- a/drivers/tty/serial/earlycon.c
+++ b/drivers/tty/serial/earlycon.c
@@ -21,6 +21,7 @@
 #include <linux/sizes.h>
 #include <linux/of.h>
 #include <linux/of_fdt.h>
+#include <linux/acpi.h>
 
 #ifdef CONFIG_FIX_EARLYCON_MEM
 #include <asm/fixmap.h>
@@ -38,7 +39,7 @@ static struct earlycon_device early_console_dev = {
 	.con = &early_con,
 };
 
-static void __iomem * __init earlycon_map(unsigned long paddr, size_t size)
+static void __iomem * __init earlycon_map(resource_size_t paddr, size_t size)
 {
 	void __iomem *base;
 #ifdef CONFIG_FIX_EARLYCON_MEM
@@ -49,8 +50,7 @@ static void __iomem * __init earlycon_map(unsigned long paddr, size_t size)
 	base = ioremap(paddr, size);
 #endif
 	if (!base)
-		pr_err("%s: Couldn't map 0x%llx\n", __func__,
-		       (unsigned long long)paddr);
+		pr_err("%s: Couldn't map %pa\n", __func__, &paddr);
 
 	return base;
 }
@@ -92,7 +92,7 @@ static int __init parse_options(struct earlycon_device *device, char *options)
 {
 	struct uart_port *port = &device->port;
 	int length;
-	unsigned long addr;
+	resource_size_t addr;
 
 	if (uart_parse_earlycon(options, &port->iotype, &addr, &options))
 		return -EINVAL;
@@ -199,6 +199,14 @@ int __init setup_earlycon(char *buf)
 	return -ENOENT;
 }
 
+/*
+ * When CONFIG_ACPI_SPCR_TABLE is defined, "earlycon" without parameters in
+ * command line does not start DT earlycon immediately, instead it defers
+ * starting it until DT/ACPI decision is made.  At that time if ACPI is enabled
+ * call parse_spcr(), else call early_init_dt_scan_chosen_stdout()
+ */
+bool earlycon_init_is_deferred __initdata;
+
 /* early_param wrapper for setup_earlycon() */
 static int __init param_setup_earlycon(char *buf)
 {
@@ -208,8 +216,14 @@ static int __init param_setup_earlycon(char *buf)
 	 * Just 'earlycon' is a valid param for devicetree earlycons;
 	 * don't generate a warning from parse_early_params() in that case
 	 */
-	if (!buf || !buf[0])
-		return 0;
+	if (!buf || !buf[0]) {
+		if (IS_ENABLED(CONFIG_ACPI_SPCR_TABLE)) {
+			earlycon_init_is_deferred = true;
+			return 0;
+		} else {
+			return early_init_dt_scan_chosen_stdout();
+		}
+	}
 
 	err = setup_earlycon(buf);
 	if (err == -ENOENT || err == -EALREADY)
diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c
index 7f95f782a485..de9d5107c00a 100644
--- a/drivers/tty/serial/fsl_lpuart.c
+++ b/drivers/tty/serial/fsl_lpuart.c
@@ -224,7 +224,8 @@
 #define UARTWATER_TXWATER_OFF	0
 #define UARTWATER_RXWATER_OFF	16
 
-#define FSL_UART_RX_DMA_BUFFER_SIZE	64
+/* Rx DMA timeout in ms, which is used to calculate Rx ring buffer size */
+#define DMA_RX_TIMEOUT		(10)
 
 #define DRIVER_NAME	"fsl-lpuart"
 #define DEV_NAME	"ttyLP"
@@ -243,18 +244,18 @@ struct lpuart_port {
 	struct dma_chan		*dma_rx_chan;
 	struct dma_async_tx_descriptor  *dma_tx_desc;
 	struct dma_async_tx_descriptor  *dma_rx_desc;
-	dma_addr_t		dma_tx_buf_bus;
-	dma_addr_t		dma_rx_buf_bus;
 	dma_cookie_t		dma_tx_cookie;
 	dma_cookie_t		dma_rx_cookie;
-	unsigned char		*dma_tx_buf_virt;
-	unsigned char		*dma_rx_buf_virt;
 	unsigned int		dma_tx_bytes;
 	unsigned int		dma_rx_bytes;
-	int			dma_tx_in_progress;
-	int			dma_rx_in_progress;
+	bool			dma_tx_in_progress;
 	unsigned int		dma_rx_timeout;
 	struct timer_list	lpuart_timer;
+	struct scatterlist	rx_sgl, tx_sgl[2];
+	struct circ_buf		rx_ring;
+	int			rx_dma_rng_buf_len;
+	unsigned int		dma_tx_nents;
+	wait_queue_head_t	dma_wait;
 };
 
 static const struct of_device_id lpuart_dt_ids[] = {
@@ -270,7 +271,6 @@ MODULE_DEVICE_TABLE(of, lpuart_dt_ids);
 
 /* Forward declare this for the dma callbacks*/
 static void lpuart_dma_tx_complete(void *arg);
-static void lpuart_dma_rx_complete(void *arg);
 
 static u32 lpuart32_read(void __iomem *addr)
 {
@@ -316,141 +316,103 @@ static void lpuart32_stop_rx(struct uart_port *port)
 	lpuart32_write(temp & ~UARTCTRL_RE, port->membase + UARTCTRL);
 }
 
-static void lpuart_copy_rx_to_tty(struct lpuart_port *sport,
-		struct tty_port *tty, int count)
+static void lpuart_dma_tx(struct lpuart_port *sport)
 {
-	int copied;
-
-	sport->port.icount.rx += count;
+	struct circ_buf *xmit = &sport->port.state->xmit;
+	struct scatterlist *sgl = sport->tx_sgl;
+	struct device *dev = sport->port.dev;
+	int ret;
 
-	if (!tty) {
-		dev_err(sport->port.dev, "No tty port\n");
+	if (sport->dma_tx_in_progress)
 		return;
-	}
 
-	dma_sync_single_for_cpu(sport->port.dev, sport->dma_rx_buf_bus,
-			FSL_UART_RX_DMA_BUFFER_SIZE, DMA_FROM_DEVICE);
-	copied = tty_insert_flip_string(tty,
-			((unsigned char *)(sport->dma_rx_buf_virt)), count);
+	sport->dma_tx_bytes = uart_circ_chars_pending(xmit);
 
-	if (copied != count) {
-		WARN_ON(1);
-		dev_err(sport->port.dev, "RxData copy to tty layer failed\n");
+	if (xmit->tail < xmit->head) {
+		sport->dma_tx_nents = 1;
+		sg_init_one(sgl, xmit->buf + xmit->tail, sport->dma_tx_bytes);
+	} else {
+		sport->dma_tx_nents = 2;
+		sg_init_table(sgl, 2);
+		sg_set_buf(sgl, xmit->buf + xmit->tail,
+				UART_XMIT_SIZE - xmit->tail);
+		sg_set_buf(sgl + 1, xmit->buf, xmit->head);
 	}
 
-	dma_sync_single_for_device(sport->port.dev, sport->dma_rx_buf_bus,
-			FSL_UART_RX_DMA_BUFFER_SIZE, DMA_TO_DEVICE);
-}
-
-static void lpuart_pio_tx(struct lpuart_port *sport)
-{
-	struct circ_buf *xmit = &sport->port.state->xmit;
-	unsigned long flags;
-
-	spin_lock_irqsave(&sport->port.lock, flags);
-
-	while (!uart_circ_empty(xmit) &&
-		readb(sport->port.membase + UARTTCFIFO) < sport->txfifo_size) {
-		writeb(xmit->buf[xmit->tail], sport->port.membase + UARTDR);
-		xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
-		sport->port.icount.tx++;
+	ret = dma_map_sg(dev, sgl, sport->dma_tx_nents, DMA_TO_DEVICE);
+	if (!ret) {
+		dev_err(dev, "DMA mapping error for TX.\n");
+		return;
 	}
 
-	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
-		uart_write_wakeup(&sport->port);
-
-	if (uart_circ_empty(xmit))
-		writeb(readb(sport->port.membase + UARTCR5) | UARTCR5_TDMAS,
-			sport->port.membase + UARTCR5);
-
-	spin_unlock_irqrestore(&sport->port.lock, flags);
-}
-
-static int lpuart_dma_tx(struct lpuart_port *sport, unsigned long count)
-{
-	struct circ_buf *xmit = &sport->port.state->xmit;
-	dma_addr_t tx_bus_addr;
-
-	dma_sync_single_for_device(sport->port.dev, sport->dma_tx_buf_bus,
-				UART_XMIT_SIZE, DMA_TO_DEVICE);
-	sport->dma_tx_bytes = count & ~(sport->txfifo_size - 1);
-	tx_bus_addr = sport->dma_tx_buf_bus + xmit->tail;
-	sport->dma_tx_desc = dmaengine_prep_slave_single(sport->dma_tx_chan,
-					tx_bus_addr, sport->dma_tx_bytes,
+	sport->dma_tx_desc = dmaengine_prep_slave_sg(sport->dma_tx_chan, sgl,
+					sport->dma_tx_nents,
 					DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT);
-
 	if (!sport->dma_tx_desc) {
-		dev_err(sport->port.dev, "Not able to get desc for tx\n");
-		return -EIO;
+		dma_unmap_sg(dev, sgl, sport->dma_tx_nents, DMA_TO_DEVICE);
+		dev_err(dev, "Cannot prepare TX slave DMA!\n");
+		return;
 	}
 
 	sport->dma_tx_desc->callback = lpuart_dma_tx_complete;
 	sport->dma_tx_desc->callback_param = sport;
-	sport->dma_tx_in_progress = 1;
+	sport->dma_tx_in_progress = true;
 	sport->dma_tx_cookie = dmaengine_submit(sport->dma_tx_desc);
 	dma_async_issue_pending(sport->dma_tx_chan);
 
-	return 0;
-}
-
-static void lpuart_prepare_tx(struct lpuart_port *sport)
-{
-	struct circ_buf *xmit = &sport->port.state->xmit;
-	unsigned long count =  CIRC_CNT_TO_END(xmit->head,
-					xmit->tail, UART_XMIT_SIZE);
-
-	if (!count)
-		return;
-
-	if (count < sport->txfifo_size)
-		writeb(readb(sport->port.membase + UARTCR5) & ~UARTCR5_TDMAS,
-				sport->port.membase + UARTCR5);
-	else {
-		writeb(readb(sport->port.membase + UARTCR5) | UARTCR5_TDMAS,
-				sport->port.membase + UARTCR5);
-		lpuart_dma_tx(sport, count);
-	}
 }
 
 static void lpuart_dma_tx_complete(void *arg)
 {
 	struct lpuart_port *sport = arg;
+	struct scatterlist *sgl = &sport->tx_sgl[0];
 	struct circ_buf *xmit = &sport->port.state->xmit;
 	unsigned long flags;
 
-	async_tx_ack(sport->dma_tx_desc);
-
 	spin_lock_irqsave(&sport->port.lock, flags);
 
+	dma_unmap_sg(sport->port.dev, sgl, sport->dma_tx_nents, DMA_TO_DEVICE);
+
 	xmit->tail = (xmit->tail + sport->dma_tx_bytes) & (UART_XMIT_SIZE - 1);
-	sport->dma_tx_in_progress = 0;
+
+	sport->port.icount.tx += sport->dma_tx_bytes;
+	sport->dma_tx_in_progress = false;
+	spin_unlock_irqrestore(&sport->port.lock, flags);
 
 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 		uart_write_wakeup(&sport->port);
 
-	lpuart_prepare_tx(sport);
+	if (waitqueue_active(&sport->dma_wait)) {
+		wake_up(&sport->dma_wait);
+		return;
+	}
+
+	spin_lock_irqsave(&sport->port.lock, flags);
+
+	if (!uart_circ_empty(xmit) && !uart_tx_stopped(&sport->port))
+		lpuart_dma_tx(sport);
 
 	spin_unlock_irqrestore(&sport->port.lock, flags);
 }
 
-static int lpuart_dma_rx(struct lpuart_port *sport)
+static int lpuart_dma_tx_request(struct uart_port *port)
 {
-	dma_sync_single_for_device(sport->port.dev, sport->dma_rx_buf_bus,
-			FSL_UART_RX_DMA_BUFFER_SIZE, DMA_TO_DEVICE);
-	sport->dma_rx_desc = dmaengine_prep_slave_single(sport->dma_rx_chan,
-			sport->dma_rx_buf_bus, FSL_UART_RX_DMA_BUFFER_SIZE,
-			DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT);
+	struct lpuart_port *sport = container_of(port,
+					struct lpuart_port, port);
+	struct dma_slave_config dma_tx_sconfig = {};
+	int ret;
 
-	if (!sport->dma_rx_desc) {
-		dev_err(sport->port.dev, "Not able to get desc for rx\n");
-		return -EIO;
-	}
+	dma_tx_sconfig.dst_addr = sport->port.mapbase + UARTDR;
+	dma_tx_sconfig.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+	dma_tx_sconfig.dst_maxburst = 1;
+	dma_tx_sconfig.direction = DMA_MEM_TO_DEV;
+	ret = dmaengine_slave_config(sport->dma_tx_chan, &dma_tx_sconfig);
 
-	sport->dma_rx_desc->callback = lpuart_dma_rx_complete;
-	sport->dma_rx_desc->callback_param = sport;
-	sport->dma_rx_in_progress = 1;
-	sport->dma_rx_cookie = dmaengine_submit(sport->dma_rx_desc);
-	dma_async_issue_pending(sport->dma_rx_chan);
+	if (ret) {
+		dev_err(sport->port.dev,
+				"DMA slave config failed, err = %d\n", ret);
+		return ret;
+	}
 
 	return 0;
 }
@@ -458,75 +420,17 @@ static int lpuart_dma_rx(struct lpuart_port *sport)
 static void lpuart_flush_buffer(struct uart_port *port)
 {
 	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
+
 	if (sport->lpuart_dma_tx_use) {
+		if (sport->dma_tx_in_progress) {
+			dma_unmap_sg(sport->port.dev, &sport->tx_sgl[0],
+				sport->dma_tx_nents, DMA_TO_DEVICE);
+			sport->dma_tx_in_progress = false;
+		}
 		dmaengine_terminate_all(sport->dma_tx_chan);
-		sport->dma_tx_in_progress = 0;
 	}
 }
 
-static void lpuart_dma_rx_complete(void *arg)
-{
-	struct lpuart_port *sport = arg;
-	struct tty_port *port = &sport->port.state->port;
-	unsigned long flags;
-
-	async_tx_ack(sport->dma_rx_desc);
-	mod_timer(&sport->lpuart_timer, jiffies + sport->dma_rx_timeout);
-
-	spin_lock_irqsave(&sport->port.lock, flags);
-
-	sport->dma_rx_in_progress = 0;
-	lpuart_copy_rx_to_tty(sport, port, FSL_UART_RX_DMA_BUFFER_SIZE);
-	tty_flip_buffer_push(port);
-	lpuart_dma_rx(sport);
-
-	spin_unlock_irqrestore(&sport->port.lock, flags);
-}
-
-static void lpuart_timer_func(unsigned long data)
-{
-	struct lpuart_port *sport = (struct lpuart_port *)data;
-	struct tty_port *port = &sport->port.state->port;
-	struct dma_tx_state state;
-	unsigned long flags;
-	unsigned char temp;
-	int count;
-
-	del_timer(&sport->lpuart_timer);
-	dmaengine_pause(sport->dma_rx_chan);
-	dmaengine_tx_status(sport->dma_rx_chan, sport->dma_rx_cookie, &state);
-	dmaengine_terminate_all(sport->dma_rx_chan);
-	count = FSL_UART_RX_DMA_BUFFER_SIZE - state.residue;
-	async_tx_ack(sport->dma_rx_desc);
-
-	spin_lock_irqsave(&sport->port.lock, flags);
-
-	sport->dma_rx_in_progress = 0;
-	lpuart_copy_rx_to_tty(sport, port, count);
-	tty_flip_buffer_push(port);
-	temp = readb(sport->port.membase + UARTCR5);
-	writeb(temp & ~UARTCR5_RDMAS, sport->port.membase + UARTCR5);
-
-	spin_unlock_irqrestore(&sport->port.lock, flags);
-}
-
-static inline void lpuart_prepare_rx(struct lpuart_port *sport)
-{
-	unsigned long flags;
-	unsigned char temp;
-
-	spin_lock_irqsave(&sport->port.lock, flags);
-
-	sport->lpuart_timer.expires = jiffies + sport->dma_rx_timeout;
-	add_timer(&sport->lpuart_timer);
-
-	lpuart_dma_rx(sport);
-	temp = readb(sport->port.membase + UARTCR5);
-	writeb(temp | UARTCR5_RDMAS, sport->port.membase + UARTCR5);
-
-	spin_unlock_irqrestore(&sport->port.lock, flags);
-}
-
 static inline void lpuart_transmit_buffer(struct lpuart_port *sport)
 {
 	struct circ_buf *xmit = &sport->port.state->xmit;
@@ -580,8 +484,8 @@ static void lpuart_start_tx(struct uart_port *port)
 	writeb(temp | UARTCR2_TIE, port->membase + UARTCR2);
 
 	if (sport->lpuart_dma_tx_use) {
-		if (!uart_circ_empty(xmit) && !sport->dma_tx_in_progress)
-			lpuart_prepare_tx(sport);
+		if (!uart_circ_empty(xmit) && !uart_tx_stopped(port))
+			lpuart_dma_tx(sport);
 	} else {
 		if (readb(port->membase + UARTSR1) & UARTSR1_TDRE)
 			lpuart_transmit_buffer(sport);
@@ -600,6 +504,29 @@ static void lpuart32_start_tx(struct uart_port *port)
 		lpuart32_transmit_buffer(sport);
 }
 
+/* return TIOCSER_TEMT when transmitter is not busy */
+static unsigned int lpuart_tx_empty(struct uart_port *port)
+{
+	struct lpuart_port *sport = container_of(port,
+			struct lpuart_port, port);
+	unsigned char sr1 = readb(port->membase + UARTSR1);
+	unsigned char sfifo = readb(port->membase + UARTSFIFO);
+
+	if (sport->dma_tx_in_progress)
+		return 0;
+
+	if (sr1 & UARTSR1_TC && sfifo & UARTSFIFO_TXEMPT)
+		return TIOCSER_TEMT;
+
+	return 0;
+}
+
+static unsigned int lpuart32_tx_empty(struct uart_port *port)
+{
+	return (lpuart32_read(port->membase + UARTSTAT) & UARTSTAT_TC) ?
+		TIOCSER_TEMT : 0;
+}
+
 static irqreturn_t lpuart_txint(int irq, void *dev_id)
 {
 	struct lpuart_port *sport = dev_id;
@@ -766,23 +693,15 @@ out:
 static irqreturn_t lpuart_int(int irq, void *dev_id)
 {
 	struct lpuart_port *sport = dev_id;
-	unsigned char sts, crdma;
+	unsigned char sts;
 
 	sts = readb(sport->port.membase + UARTSR1);
-	crdma = readb(sport->port.membase + UARTCR5);
 
-	if (sts & UARTSR1_RDRF && !(crdma & UARTCR5_RDMAS)) {
-		if (sport->lpuart_dma_rx_use)
-			lpuart_prepare_rx(sport);
-		else
-			lpuart_rxint(irq, dev_id);
-	}
-	if (sts & UARTSR1_TDRE && !(crdma & UARTCR5_TDMAS)) {
-		if (sport->lpuart_dma_tx_use)
-			lpuart_pio_tx(sport);
-		else
-			lpuart_txint(irq, dev_id);
-	}
+	if (sts & UARTSR1_RDRF)
+		lpuart_rxint(irq, dev_id);
+
+	if (sts & UARTSR1_TDRE)
+		lpuart_txint(irq, dev_id);
 
 	return IRQ_HANDLED;
 }
@@ -807,17 +726,241 @@ static irqreturn_t lpuart32_int(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-/* return TIOCSER_TEMT when transmitter is not busy */
-static unsigned int lpuart_tx_empty(struct uart_port *port)
+static void lpuart_copy_rx_to_tty(struct lpuart_port *sport)
 {
-	return (readb(port->membase + UARTSR1) & UARTSR1_TC) ?
-		TIOCSER_TEMT : 0;
+	struct tty_port *port = &sport->port.state->port;
+	struct dma_tx_state state;
+	enum dma_status dmastat;
+	struct circ_buf *ring = &sport->rx_ring;
+	unsigned long flags;
+	int count = 0;
+	unsigned char sr;
+
+	sr = readb(sport->port.membase + UARTSR1);
+
+	if (sr & (UARTSR1_PE | UARTSR1_FE)) {
+		/* Read DR to clear the error flags */
+		readb(sport->port.membase + UARTDR);
+
+		if (sr & UARTSR1_PE)
+		    sport->port.icount.parity++;
+		else if (sr & UARTSR1_FE)
+		    sport->port.icount.frame++;
+	}
+
+	async_tx_ack(sport->dma_rx_desc);
+
+	spin_lock_irqsave(&sport->port.lock, flags);
+
+	dmastat = dmaengine_tx_status(sport->dma_rx_chan,
+				sport->dma_rx_cookie,
+				&state);
+
+	if (dmastat == DMA_ERROR) {
+		dev_err(sport->port.dev, "Rx DMA transfer failed!\n");
+		spin_unlock_irqrestore(&sport->port.lock, flags);
+		return;
+	}
+
+	/* CPU claims ownership of RX DMA buffer */
+	dma_sync_sg_for_cpu(sport->port.dev, &sport->rx_sgl, 1, DMA_FROM_DEVICE);
+
+	/*
+	 * ring->head points to the end of data already written by the DMA.
+	 * ring->tail points to the beginning of data to be read by the
+	 * framework.
+	 * The current transfer size should not be larger than the dma buffer
+	 * length.
+	 */
+	ring->head = sport->rx_sgl.length - state.residue;
+	BUG_ON(ring->head > sport->rx_sgl.length);
+	/*
+	 * At this point ring->head may point to the first byte right after the
+	 * last byte of the dma buffer:
+	 * 0 <= ring->head <= sport->rx_sgl.length
+	 *
+	 * However ring->tail must always points inside the dma buffer:
+	 * 0 <= ring->tail <= sport->rx_sgl.length - 1
+	 *
+	 * Since we use a ring buffer, we have to handle the case
+	 * where head is lower than tail. In such a case, we first read from
+	 * tail to the end of the buffer then reset tail.
+	 */
+	if (ring->head < ring->tail) {
+		count = sport->rx_sgl.length - ring->tail;
+
+		tty_insert_flip_string(port, ring->buf + ring->tail, count);
+		ring->tail = 0;
+		sport->port.icount.rx += count;
+	}
+
+	/* Finally we read data from tail to head */
+	if (ring->tail < ring->head) {
+		count = ring->head - ring->tail;
+		tty_insert_flip_string(port, ring->buf + ring->tail, count);
+		/* Wrap ring->head if needed */
+		if (ring->head >= sport->rx_sgl.length)
+			ring->head = 0;
+		ring->tail = ring->head;
+		sport->port.icount.rx += count;
+	}
+
+	dma_sync_sg_for_device(sport->port.dev, &sport->rx_sgl, 1,
+			       DMA_FROM_DEVICE);
+
+	spin_unlock_irqrestore(&sport->port.lock, flags);
+
+	tty_flip_buffer_push(port);
+	mod_timer(&sport->lpuart_timer, jiffies + sport->dma_rx_timeout);
 }
 
-static unsigned int lpuart32_tx_empty(struct uart_port *port)
+static void lpuart_dma_rx_complete(void *arg)
 {
-	return (lpuart32_read(port->membase + UARTSTAT) & UARTSTAT_TC) ?
-		TIOCSER_TEMT : 0;
+	struct lpuart_port *sport = arg;
+
+	lpuart_copy_rx_to_tty(sport);
+}
+
+static void lpuart_timer_func(unsigned long data)
+{
+	struct lpuart_port *sport = (struct lpuart_port *)data;
+
+	lpuart_copy_rx_to_tty(sport);
+}
+
+static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
+{
+	struct dma_slave_config dma_rx_sconfig = {};
+	struct circ_buf *ring = &sport->rx_ring;
+	int ret, nent;
+	int bits, baud;
+	struct tty_struct *tty = tty_port_tty_get(&sport->port.state->port);
+	struct ktermios *termios = &tty->termios;
+
+	baud = tty_get_baud_rate(tty);
+
+	bits = (termios->c_cflag & CSIZE) == CS7 ? 9 : 10;
+	if (termios->c_cflag & PARENB)
+		bits++;
+
+	/*
+	 * Calculate length of one DMA buffer size to keep latency below
+	 * 10ms at any baud rate.
+	 */
+	sport->rx_dma_rng_buf_len = (DMA_RX_TIMEOUT * baud /  bits / 1000) * 2;
+	sport->rx_dma_rng_buf_len = (1 << (fls(sport->rx_dma_rng_buf_len) - 1));
+	if (sport->rx_dma_rng_buf_len < 16)
+		sport->rx_dma_rng_buf_len = 16;
+
+	ring->buf = kmalloc(sport->rx_dma_rng_buf_len, GFP_ATOMIC);
+	if (!ring->buf) {
+		dev_err(sport->port.dev, "Ring buf alloc failed\n");
+		return -ENOMEM;
+	}
+
+	sg_init_one(&sport->rx_sgl, ring->buf, sport->rx_dma_rng_buf_len);
+	sg_set_buf(&sport->rx_sgl, ring->buf, sport->rx_dma_rng_buf_len);
+	nent = dma_map_sg(sport->port.dev, &sport->rx_sgl, 1, DMA_FROM_DEVICE);
+
+	if (!nent) {
+		dev_err(sport->port.dev, "DMA Rx mapping error\n");
+		return -EINVAL;
+	}
+
+	dma_rx_sconfig.src_addr = sport->port.mapbase + UARTDR;
+	dma_rx_sconfig.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+	dma_rx_sconfig.src_maxburst = 1;
+	dma_rx_sconfig.direction = DMA_DEV_TO_MEM;
+	ret = dmaengine_slave_config(sport->dma_rx_chan, &dma_rx_sconfig);
+
+	if (ret < 0) {
+		dev_err(sport->port.dev,
+				"DMA Rx slave config failed, err = %d\n", ret);
+		return ret;
+	}
+
+	sport->dma_rx_desc = dmaengine_prep_dma_cyclic(sport->dma_rx_chan,
+				 sg_dma_address(&sport->rx_sgl),
+				 sport->rx_sgl.length,
+				 sport->rx_sgl.length / 2,
+				 DMA_DEV_TO_MEM,
+				 DMA_PREP_INTERRUPT);
+	if (!sport->dma_rx_desc) {
+		dev_err(sport->port.dev, "Cannot prepare cyclic DMA\n");
+		return -EFAULT;
+	}
+
+	sport->dma_rx_desc->callback = lpuart_dma_rx_complete;
+	sport->dma_rx_desc->callback_param = sport;
+	sport->dma_rx_cookie = dmaengine_submit(sport->dma_rx_desc);
+	dma_async_issue_pending(sport->dma_rx_chan);
+
+	writeb(readb(sport->port.membase + UARTCR5) | UARTCR5_RDMAS,
+				sport->port.membase + UARTCR5);
+
+	return 0;
+}
+
+static void lpuart_dma_rx_free(struct uart_port *port)
+{
+	struct lpuart_port *sport = container_of(port,
+					struct lpuart_port, port);
+
+	if (sport->dma_rx_chan)
+		dmaengine_terminate_all(sport->dma_rx_chan);
+
+	dma_unmap_sg(sport->port.dev, &sport->rx_sgl, 1, DMA_FROM_DEVICE);
+	kfree(sport->rx_ring.buf);
+	sport->rx_ring.tail = 0;
+	sport->rx_ring.head = 0;
+	sport->dma_rx_desc = NULL;
+	sport->dma_rx_cookie = -EINVAL;
+}
+
+static int lpuart_config_rs485(struct uart_port *port,
+			struct serial_rs485 *rs485)
+{
+	struct lpuart_port *sport = container_of(port,
+			struct lpuart_port, port);
+
+	u8 modem = readb(sport->port.membase + UARTMODEM) &
+		~(UARTMODEM_TXRTSPOL | UARTMODEM_TXRTSE);
+	writeb(modem, sport->port.membase + UARTMODEM);
+
+	if (rs485->flags & SER_RS485_ENABLED) {
+		/* Enable auto RS-485 RTS mode */
+		modem |= UARTMODEM_TXRTSE;
+
+		/*
+		 * RTS needs to be logic HIGH either during transer _or_ after
+		 * transfer, other variants are not supported by the hardware.
+		 */
+
+		if (!(rs485->flags & (SER_RS485_RTS_ON_SEND |
+				SER_RS485_RTS_AFTER_SEND)))
+			rs485->flags |= SER_RS485_RTS_ON_SEND;
+
+		if (rs485->flags & SER_RS485_RTS_ON_SEND &&
+				rs485->flags & SER_RS485_RTS_AFTER_SEND)
+			rs485->flags &= ~SER_RS485_RTS_AFTER_SEND;
+
+		/*
+		 * The hardware defaults to RTS logic HIGH while transfer.
+		 * Switch polarity in case RTS shall be logic HIGH
+		 * after transfer.
+		 * Note: UART is assumed to be active high.
+		 */
+		if (rs485->flags & SER_RS485_RTS_ON_SEND)
+			modem &= ~UARTMODEM_TXRTSPOL;
+		else if (rs485->flags & SER_RS485_RTS_AFTER_SEND)
+			modem |= UARTMODEM_TXRTSPOL;
+	}
+
+	/* Store the new configuration */
+	sport->port.rs485 = *rs485;
+
+	writeb(modem, sport->port.membase + UARTMODEM);
+	return 0;
 }
 
 static unsigned int lpuart_get_mctrl(struct uart_port *port)
@@ -853,17 +996,22 @@ static unsigned int lpuart32_get_mctrl(struct uart_port *port)
 static void lpuart_set_mctrl(struct uart_port *port, unsigned int mctrl)
 {
 	unsigned char temp;
+	struct lpuart_port *sport = container_of(port,
+				struct lpuart_port, port);
 
-	temp = readb(port->membase + UARTMODEM) &
+	/* Make sure RXRTSE bit is not set when RS485 is enabled */
+	if (!(sport->port.rs485.flags & SER_RS485_ENABLED)) {
+		temp = readb(sport->port.membase + UARTMODEM) &
 			~(UARTMODEM_RXRTSE | UARTMODEM_TXCTSE);
 
-	if (mctrl & TIOCM_RTS)
-		temp |= UARTMODEM_RXRTSE;
+		if (mctrl & TIOCM_RTS)
+			temp |= UARTMODEM_RXRTSE;
 
-	if (mctrl & TIOCM_CTS)
-		temp |= UARTMODEM_TXCTSE;
+		if (mctrl & TIOCM_CTS)
+			temp |= UARTMODEM_TXCTSE;
 
-	writeb(temp, port->membase + UARTMODEM);
+		writeb(temp, port->membase + UARTMODEM);
+	}
 }
 
 static void lpuart32_set_mctrl(struct uart_port *port, unsigned int mctrl)
@@ -921,13 +1069,16 @@ static void lpuart_setup_watermark(struct lpuart_port *sport)
 	writeb(val | UARTPFIFO_TXFE | UARTPFIFO_RXFE,
 			sport->port.membase + UARTPFIFO);
 
-	/* explicitly clear RDRF */
-	readb(sport->port.membase + UARTSR1);
-
 	/* flush Tx and Rx FIFO */
 	writeb(UARTCFIFO_TXFLUSH | UARTCFIFO_RXFLUSH,
 			sport->port.membase + UARTCFIFO);
 
+	/* explicitly clear RDRF */
+	if (readb(sport->port.membase + UARTSR1) & UARTSR1_RDRF) {
+		readb(sport->port.membase + UARTDR);
+		writeb(UARTSFIFO_RXUF, sport->port.membase + UARTSFIFO);
+	}
+
 	writeb(0, sport->port.membase + UARTTWFIFO);
 	writeb(1, sport->port.membase + UARTRWFIFO);
 
@@ -960,110 +1111,12 @@ static void lpuart32_setup_watermark(struct lpuart_port *sport)
 	lpuart32_write(ctrl_saved, sport->port.membase + UARTCTRL);
 }
 
-static int lpuart_dma_tx_request(struct uart_port *port)
-{
-	struct lpuart_port *sport = container_of(port,
-					struct lpuart_port, port);
-	struct dma_slave_config dma_tx_sconfig;
-	dma_addr_t dma_bus;
-	unsigned char *dma_buf;
-	int ret;
-
-	dma_bus = dma_map_single(sport->dma_tx_chan->device->dev,
-				sport->port.state->xmit.buf,
-				UART_XMIT_SIZE, DMA_TO_DEVICE);
-
-	if (dma_mapping_error(sport->dma_tx_chan->device->dev, dma_bus)) {
-		dev_err(sport->port.dev, "dma_map_single tx failed\n");
-		return -ENOMEM;
-	}
-
-	dma_buf = sport->port.state->xmit.buf;
-	dma_tx_sconfig.dst_addr = sport->port.mapbase + UARTDR;
-	dma_tx_sconfig.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
-	dma_tx_sconfig.dst_maxburst = sport->txfifo_size;
-	dma_tx_sconfig.direction = DMA_MEM_TO_DEV;
-	ret = dmaengine_slave_config(sport->dma_tx_chan, &dma_tx_sconfig);
-
-	if (ret < 0) {
-		dev_err(sport->port.dev,
-				"Dma slave config failed, err = %d\n", ret);
-		return ret;
-	}
-
-	sport->dma_tx_buf_virt = dma_buf;
-	sport->dma_tx_buf_bus = dma_bus;
-	sport->dma_tx_in_progress = 0;
-
-	return 0;
-}
-
-static int lpuart_dma_rx_request(struct uart_port *port)
+static void rx_dma_timer_init(struct lpuart_port *sport)
 {
-	struct lpuart_port *sport = container_of(port,
-					struct lpuart_port, port);
-	struct dma_slave_config dma_rx_sconfig;
-	dma_addr_t dma_bus;
-	unsigned char *dma_buf;
-	int ret;
-
-	dma_buf = devm_kzalloc(sport->port.dev,
-				FSL_UART_RX_DMA_BUFFER_SIZE, GFP_KERNEL);
-
-	if (!dma_buf) {
-		dev_err(sport->port.dev, "Dma rx alloc failed\n");
-		return -ENOMEM;
-	}
-
-	dma_bus = dma_map_single(sport->dma_rx_chan->device->dev, dma_buf,
-				FSL_UART_RX_DMA_BUFFER_SIZE, DMA_FROM_DEVICE);
-
-	if (dma_mapping_error(sport->dma_rx_chan->device->dev, dma_bus)) {
-		dev_err(sport->port.dev, "dma_map_single rx failed\n");
-		return -ENOMEM;
-	}
-
-	dma_rx_sconfig.src_addr = sport->port.mapbase + UARTDR;
-	dma_rx_sconfig.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
-	dma_rx_sconfig.src_maxburst = 1;
-	dma_rx_sconfig.direction = DMA_DEV_TO_MEM;
-	ret = dmaengine_slave_config(sport->dma_rx_chan, &dma_rx_sconfig);
-
-	if (ret < 0) {
-		dev_err(sport->port.dev,
-				"Dma slave config failed, err = %d\n", ret);
-		return ret;
-	}
-
-	sport->dma_rx_buf_virt = dma_buf;
-	sport->dma_rx_buf_bus = dma_bus;
-	sport->dma_rx_in_progress = 0;
-
-	return 0;
-}
-
-static void lpuart_dma_tx_free(struct uart_port *port)
-{
-	struct lpuart_port *sport = container_of(port,
-					struct lpuart_port, port);
-
-	dma_unmap_single(sport->port.dev, sport->dma_tx_buf_bus,
-			UART_XMIT_SIZE, DMA_TO_DEVICE);
-
-	sport->dma_tx_buf_bus = 0;
-	sport->dma_tx_buf_virt = NULL;
-}
-
-static void lpuart_dma_rx_free(struct uart_port *port)
-{
-	struct lpuart_port *sport = container_of(port,
-					struct lpuart_port, port);
-
-	dma_unmap_single(sport->port.dev, sport->dma_rx_buf_bus,
-			FSL_UART_RX_DMA_BUFFER_SIZE, DMA_FROM_DEVICE);
-
-	sport->dma_rx_buf_bus = 0;
-	sport->dma_rx_buf_virt = NULL;
+		setup_timer(&sport->lpuart_timer, lpuart_timer_func,
+				(unsigned long)sport);
+		sport->lpuart_timer.expires = jiffies + sport->dma_rx_timeout;
+		add_timer(&sport->lpuart_timer);
 }
 
 static int lpuart_startup(struct uart_port *port)
@@ -1084,22 +1137,6 @@ static int lpuart_startup(struct uart_port *port)
 	sport->rxfifo_size = 0x1 << (((temp >> UARTPFIFO_RXSIZE_OFF) &
 		UARTPFIFO_FIFOSIZE_MASK) + 1);
 
-	if (sport->dma_rx_chan && !lpuart_dma_rx_request(port)) {
-		sport->lpuart_dma_rx_use = true;
-		setup_timer(&sport->lpuart_timer, lpuart_timer_func,
-			    (unsigned long)sport);
-	} else
-		sport->lpuart_dma_rx_use = false;
-
-
-	if (sport->dma_tx_chan && !lpuart_dma_tx_request(port)) {
-		sport->lpuart_dma_tx_use = true;
-		temp = readb(port->membase + UARTCR5);
-		temp &= ~UARTCR5_RDMAS;
-		writeb(temp | UARTCR5_TDMAS, port->membase + UARTCR5);
-	} else
-		sport->lpuart_dma_tx_use = false;
-
 	ret = devm_request_irq(port->dev, port->irq, lpuart_int, 0,
 				DRIVER_NAME, sport);
 	if (ret)
@@ -1113,7 +1150,29 @@ static int lpuart_startup(struct uart_port *port)
 	temp |= (UARTCR2_RIE | UARTCR2_TIE | UARTCR2_RE | UARTCR2_TE);
 	writeb(temp, sport->port.membase + UARTCR2);
 
+	if (sport->dma_rx_chan && !lpuart_start_rx_dma(sport)) {
+		/* set Rx DMA timeout */
+		sport->dma_rx_timeout = msecs_to_jiffies(DMA_RX_TIMEOUT);
+		if (!sport->dma_rx_timeout)
+		     sport->dma_rx_timeout = 1;
+
+		sport->lpuart_dma_rx_use = true;
+		rx_dma_timer_init(sport);
+	} else {
+		sport->lpuart_dma_rx_use = false;
+	}
+
+	if (sport->dma_tx_chan && !lpuart_dma_tx_request(port)) {
+		init_waitqueue_head(&sport->dma_wait);
+		sport->lpuart_dma_tx_use = true;
+		temp = readb(port->membase + UARTCR5);
+		writeb(temp | UARTCR5_TDMAS, port->membase + UARTCR5);
+	} else {
+		sport->lpuart_dma_tx_use = false;
+	}
+
 	spin_unlock_irqrestore(&sport->port.lock, flags);
+
 	return 0;
 }
 
@@ -1170,12 +1229,19 @@ static void lpuart_shutdown(struct uart_port *port)
 	devm_free_irq(port->dev, port->irq, sport);
 
 	if (sport->lpuart_dma_rx_use) {
-		lpuart_dma_rx_free(&sport->port);
 		del_timer_sync(&sport->lpuart_timer);
+		lpuart_dma_rx_free(&sport->port);
 	}
 
-	if (sport->lpuart_dma_tx_use)
-		lpuart_dma_tx_free(&sport->port);
+	if (sport->lpuart_dma_tx_use) {
+		if (wait_event_interruptible(sport->dma_wait,
+			!sport->dma_tx_in_progress) != false) {
+			sport->dma_tx_in_progress = false;
+			dmaengine_terminate_all(sport->dma_tx_chan);
+		}
+
+		lpuart_stop_tx(port);
+	}
 }
 
 static void lpuart32_shutdown(struct uart_port *port)
@@ -1203,13 +1269,14 @@ lpuart_set_termios(struct uart_port *port, struct ktermios *termios,
 {
 	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
 	unsigned long flags;
-	unsigned char cr1, old_cr1, old_cr2, cr4, bdh, modem;
+	unsigned char cr1, old_cr1, old_cr2, cr3, cr4, bdh, modem;
 	unsigned int  baud;
 	unsigned int old_csize = old ? old->c_cflag & CSIZE : CS8;
 	unsigned int sbr, brfa;
 
 	cr1 = old_cr1 = readb(sport->port.membase + UARTCR1);
 	old_cr2 = readb(sport->port.membase + UARTCR2);
+	cr3 = readb(sport->port.membase + UARTCR3);
 	cr4 = readb(sport->port.membase + UARTCR4);
 	bdh = readb(sport->port.membase + UARTBDH);
 	modem = readb(sport->port.membase + UARTMODEM);
@@ -1240,6 +1307,13 @@ lpuart_set_termios(struct uart_port *port, struct ktermios *termios,
 		cr1 |= UARTCR1_M;
 	}
 
+	/*
+	 * When auto RS-485 RTS mode is enabled,
+	 * hardware flow control need to be disabled.
+	 */
+	if (sport->port.rs485.flags & SER_RS485_ENABLED)
+		termios->c_cflag &= ~CRTSCTS;
+
 	if (termios->c_cflag & CRTSCTS) {
 		modem |= (UARTMODEM_RXRTSE | UARTMODEM_TXCTSE);
 	} else {
@@ -1257,7 +1331,10 @@ lpuart_set_termios(struct uart_port *port, struct ktermios *termios,
 	if ((termios->c_cflag & PARENB)) {
 		if (termios->c_cflag & CMSPAR) {
 			cr1 &= ~UARTCR1_PE;
-			cr1 |= UARTCR1_M;
+			if (termios->c_cflag & PARODD)
+				cr3 |= UARTCR3_T8;
+			else
+				cr3 &= ~UARTCR3_T8;
 		} else {
 			cr1 |= UARTCR1_PE;
 			if ((termios->c_cflag & CSIZE) == CS8)
@@ -1297,17 +1374,6 @@ lpuart_set_termios(struct uart_port *port, struct ktermios *termios,
 	/* update the per-port timeout */
 	uart_update_timeout(port, termios->c_cflag, baud);
 
-	if (sport->lpuart_dma_rx_use) {
-		/* Calculate delay for 1.5 DMA buffers */
-		sport->dma_rx_timeout = (sport->port.timeout - HZ / 50) *
-					FSL_UART_RX_DMA_BUFFER_SIZE * 3 /
-					sport->rxfifo_size / 2;
-		dev_dbg(port->dev, "DMA Rx t-out %ums, tty t-out %u jiffies\n",
-			sport->dma_rx_timeout * 1000 / HZ, sport->port.timeout);
-		if (sport->dma_rx_timeout < msecs_to_jiffies(20))
-			sport->dma_rx_timeout = msecs_to_jiffies(20);
-	}
-
 	/* wait transmit engin complete */
 	while (!(readb(sport->port.membase + UARTSR1) & UARTSR1_TC))
 		barrier();
@@ -1325,12 +1391,31 @@ lpuart_set_termios(struct uart_port *port, struct ktermios *termios,
 	writeb(cr4 | brfa, sport->port.membase + UARTCR4);
 	writeb(bdh, sport->port.membase + UARTBDH);
 	writeb(sbr & 0xFF, sport->port.membase + UARTBDL);
+	writeb(cr3, sport->port.membase + UARTCR3);
 	writeb(cr1, sport->port.membase + UARTCR1);
 	writeb(modem, sport->port.membase + UARTMODEM);
 
 	/* restore control register */
 	writeb(old_cr2, sport->port.membase + UARTCR2);
 
+	/*
+	 * If new baud rate is set, we will also need to update the Ring buffer
+	 * length according to the selected baud rate and restart Rx DMA path.
+	 */
+	if (old) {
+		if (sport->lpuart_dma_rx_use) {
+			del_timer_sync(&sport->lpuart_timer);
+			lpuart_dma_rx_free(&sport->port);
+		}
+
+		if (sport->dma_rx_chan && !lpuart_start_rx_dma(sport)) {
+			sport->lpuart_dma_rx_use = true;
+			rx_dma_timer_init(sport);
+		} else {
+			sport->lpuart_dma_rx_use = false;
+		}
+	}
+
 	spin_unlock_irqrestore(&sport->port.lock, flags);
 }
 
@@ -1494,7 +1579,7 @@ static int lpuart_verify_port(struct uart_port *port, struct serial_struct *ser)
 	return ret;
 }
 
-static struct uart_ops lpuart_pops = {
+static const struct uart_ops lpuart_pops = {
 	.tx_empty	= lpuart_tx_empty,
 	.set_mctrl	= lpuart_set_mctrl,
 	.get_mctrl	= lpuart_get_mctrl,
@@ -1513,7 +1598,7 @@ static struct uart_ops lpuart_pops = {
 	.flush_buffer	= lpuart_flush_buffer,
 };
 
-static struct uart_ops lpuart32_pops = {
+static const struct uart_ops lpuart32_pops = {
 	.tx_empty	= lpuart32_tx_empty,
 	.set_mctrl	= lpuart32_set_mctrl,
 	.get_mctrl	= lpuart32_get_mctrl,
@@ -1843,6 +1928,8 @@ static int lpuart_probe(struct platform_device *pdev)
 		sport->port.ops = &lpuart_pops;
 	sport->port.flags = UPF_BOOT_AUTOCONF;
 
+	sport->port.rs485_config = lpuart_config_rs485;
+
 	sport->clk = devm_clk_get(&pdev->dev, "ipg");
 	if (IS_ERR(sport->clk)) {
 		ret = PTR_ERR(sport->clk);
@@ -1883,6 +1970,12 @@ static int lpuart_probe(struct platform_device *pdev)
 		dev_info(sport->port.dev, "DMA rx channel request failed, "
 				"operating without rx DMA\n");
 
+	if (of_property_read_bool(np, "linux,rs485-enabled-at-boot-time")) {
+		sport->port.rs485.flags |= SER_RS485_ENABLED;
+		sport->port.rs485.flags |= SER_RS485_RTS_ON_SEND;
+		writeb(UARTMODEM_TXRTSE, sport->port.membase + UARTMODEM);
+	}
+
 	return 0;
 }
 
@@ -1923,6 +2016,32 @@ static int lpuart_suspend(struct device *dev)
 
 	uart_suspend_port(&lpuart_reg, &sport->port);
 
+	if (sport->lpuart_dma_rx_use) {
+		/*
+		 * EDMA driver during suspend will forcefully release any
+		 * non-idle DMA channels. If port wakeup is enabled or if port
+		 * is console port or 'no_console_suspend' is set the Rx DMA
+		 * cannot resume as as expected, hence gracefully release the
+		 * Rx DMA path before suspend and start Rx DMA path on resume.
+		 */
+		if (sport->port.irq_wake) {
+			del_timer_sync(&sport->lpuart_timer);
+			lpuart_dma_rx_free(&sport->port);
+		}
+
+		/* Disable Rx DMA to use UART port as wakeup source */
+		writeb(readb(sport->port.membase + UARTCR5) & ~UARTCR5_RDMAS,
+					sport->port.membase + UARTCR5);
+	}
+
+	if (sport->lpuart_dma_tx_use) {
+		sport->dma_tx_in_progress = false;
+		dmaengine_terminate_all(sport->dma_tx_chan);
+	}
+
+	if (sport->port.suspended && !sport->port.irq_wake)
+		clk_disable_unprepare(sport->clk);
+
 	return 0;
 }
 
@@ -1931,6 +2050,9 @@ static int lpuart_resume(struct device *dev)
 	struct lpuart_port *sport = dev_get_drvdata(dev);
 	unsigned long temp;
 
+	if (sport->port.suspended && !sport->port.irq_wake)
+		clk_prepare_enable(sport->clk);
+
 	if (sport->lpuart32) {
 		lpuart32_setup_watermark(sport);
 		temp = lpuart32_read(sport->port.membase + UARTCTRL);
@@ -1944,6 +2066,26 @@ static int lpuart_resume(struct device *dev)
 		writeb(temp, sport->port.membase + UARTCR2);
 	}
 
+	if (sport->lpuart_dma_rx_use) {
+		if (sport->port.irq_wake) {
+			if (!lpuart_start_rx_dma(sport)) {
+				sport->lpuart_dma_rx_use = true;
+				rx_dma_timer_init(sport);
+			} else {
+				sport->lpuart_dma_rx_use = false;
+			}
+		}
+	}
+
+	if (sport->dma_tx_chan && !lpuart_dma_tx_request(&sport->port)) {
+			init_waitqueue_head(&sport->dma_wait);
+			sport->lpuart_dma_tx_use = true;
+			writeb(readb(sport->port.membase + UARTCR5) |
+				UARTCR5_TDMAS, sport->port.membase + UARTCR5);
+	} else {
+		sport->lpuart_dma_tx_use = false;
+	}
+
 	uart_resume_port(&lpuart_reg, &sport->port);
 
 	return 0;
diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c
index 0df2b1c091ae..a70356dad1b7 100644
--- a/drivers/tty/serial/imx.c
+++ b/drivers/tty/serial/imx.c
@@ -190,6 +190,7 @@
 enum imx_uart_type {
 	IMX1_UART,
 	IMX21_UART,
+	IMX53_UART,
 	IMX6Q_UART,
 };
 
@@ -222,6 +223,9 @@ struct imx_port {
 	struct dma_chan		*dma_chan_rx, *dma_chan_tx;
 	struct scatterlist	rx_sgl, tx_sgl[2];
 	void			*rx_buf;
+	struct circ_buf		rx_ring;
+	unsigned int		rx_periods;
+	dma_cookie_t		rx_cookie;
 	unsigned int		tx_bytes;
 	unsigned int		dma_tx_nents;
 	wait_queue_head_t	dma_wait;
@@ -244,6 +248,10 @@ static struct imx_uart_data imx_uart_devdata[] = {
 		.uts_reg = IMX21_UTS,
 		.devtype = IMX21_UART,
 	},
+	[IMX53_UART] = {
+		.uts_reg = IMX21_UTS,
+		.devtype = IMX53_UART,
+	},
 	[IMX6Q_UART] = {
 		.uts_reg = IMX21_UTS,
 		.devtype = IMX6Q_UART,
@@ -258,6 +266,9 @@ static const struct platform_device_id imx_uart_devtype[] = {
 		.name = "imx21-uart",
 		.driver_data = (kernel_ulong_t) &imx_uart_devdata[IMX21_UART],
 	}, {
+		.name = "imx53-uart",
+		.driver_data = (kernel_ulong_t) &imx_uart_devdata[IMX53_UART],
+	}, {
 		.name = "imx6q-uart",
 		.driver_data = (kernel_ulong_t) &imx_uart_devdata[IMX6Q_UART],
 	}, {
@@ -268,6 +279,7 @@ MODULE_DEVICE_TABLE(platform, imx_uart_devtype);
 
 static const struct of_device_id imx_uart_dt_ids[] = {
 	{ .compatible = "fsl,imx6q-uart", .data = &imx_uart_devdata[IMX6Q_UART], },
+	{ .compatible = "fsl,imx53-uart", .data = &imx_uart_devdata[IMX53_UART], },
 	{ .compatible = "fsl,imx1-uart", .data = &imx_uart_devdata[IMX1_UART], },
 	{ .compatible = "fsl,imx21-uart", .data = &imx_uart_devdata[IMX21_UART], },
 	{ /* sentinel */ }
@@ -289,6 +301,11 @@ static inline int is_imx21_uart(struct imx_port *sport)
 	return sport->devdata->devtype == IMX21_UART;
 }
 
+static inline int is_imx53_uart(struct imx_port *sport)
+{
+	return sport->devdata->devtype == IMX53_UART;
+}
+
 static inline int is_imx6q_uart(struct imx_port *sport)
 {
 	return sport->devdata->devtype == IMX6Q_UART;
@@ -701,6 +718,7 @@ out:
 	return IRQ_HANDLED;
 }
 
+static void clear_rx_errors(struct imx_port *sport);
 static int start_rx_dma(struct imx_port *sport);
 /*
  * If the RXFIFO is filled with some data, and then we
@@ -726,6 +744,11 @@ static void imx_dma_rxint(struct imx_port *sport)
 		temp &= ~(UCR2_ATEN);
 		writel(temp, sport->port.membase + UCR2);
 
+		/* disable the rx errors interrupts */
+		temp = readl(sport->port.membase + UCR4);
+		temp &= ~UCR4_OREN;
+		writel(temp, sport->port.membase + UCR4);
+
 		/* tell the DMA to receive the data. */
 		start_rx_dma(sport);
 	}
@@ -740,12 +763,13 @@ static unsigned int imx_get_hwmctrl(struct imx_port *sport)
 {
 	unsigned int tmp = TIOCM_DSR;
 	unsigned usr1 = readl(sport->port.membase + USR1);
+	unsigned usr2 = readl(sport->port.membase + USR2);
 
 	if (usr1 & USR1_RTSS)
 		tmp |= TIOCM_CTS;
 
 	/* in DCE mode DCDIN is always 0 */
-	if (!(usr1 & USR2_DCDIN))
+	if (!(usr2 & USR2_DCDIN))
 		tmp |= TIOCM_CAR;
 
 	if (sport->dte_mode)
@@ -932,30 +956,6 @@ static void imx_timeout(unsigned long data)
 }
 
 #define RX_BUF_SIZE	(PAGE_SIZE)
-static void imx_rx_dma_done(struct imx_port *sport)
-{
-	unsigned long temp;
-	unsigned long flags;
-
-	spin_lock_irqsave(&sport->port.lock, flags);
-
-	/* re-enable interrupts to get notified when new symbols are incoming */
-	temp = readl(sport->port.membase + UCR1);
-	temp |= UCR1_RRDYEN;
-	writel(temp, sport->port.membase + UCR1);
-
-	temp = readl(sport->port.membase + UCR2);
-	temp |= UCR2_ATEN;
-	writel(temp, sport->port.membase + UCR2);
-
-	sport->dma_is_rxing = 0;
-
-	/* Is the shutdown waiting for us? */
-	if (waitqueue_active(&sport->dma_wait))
-		wake_up(&sport->dma_wait);
-
-	spin_unlock_irqrestore(&sport->port.lock, flags);
-}
 
 /*
  * There are two kinds of RX DMA interrupts(such as in the MX6Q):
@@ -972,43 +972,76 @@ static void dma_rx_callback(void *data)
 	struct scatterlist *sgl = &sport->rx_sgl;
 	struct tty_port *port = &sport->port.state->port;
 	struct dma_tx_state state;
+	struct circ_buf *rx_ring = &sport->rx_ring;
 	enum dma_status status;
-	unsigned int count;
-
-	/* unmap it first */
-	dma_unmap_sg(sport->port.dev, sgl, 1, DMA_FROM_DEVICE);
+	unsigned int w_bytes = 0;
+	unsigned int r_bytes;
+	unsigned int bd_size;
 
 	status = dmaengine_tx_status(chan, (dma_cookie_t)0, &state);
-	count = RX_BUF_SIZE - state.residue;
 
-	dev_dbg(sport->port.dev, "We get %d bytes.\n", count);
+	if (status == DMA_ERROR) {
+		dev_err(sport->port.dev, "DMA transaction error.\n");
+		clear_rx_errors(sport);
+		return;
+	}
+
+	if (!(sport->port.ignore_status_mask & URXD_DUMMY_READ)) {
+
+		/*
+		 * The state-residue variable represents the empty space
+		 * relative to the entire buffer. Taking this in consideration
+		 * the head is always calculated base on the buffer total
+		 * length - DMA transaction residue. The UART script from the
+		 * SDMA firmware will jump to the next buffer descriptor,
+		 * once a DMA transaction if finalized (IMX53 RM - A.4.1.2.4).
+		 * Taking this in consideration the tail is always at the
+		 * beginning of the buffer descriptor that contains the head.
+		 */
+
+		/* Calculate the head */
+		rx_ring->head = sg_dma_len(sgl) - state.residue;
+
+		/* Calculate the tail. */
+		bd_size = sg_dma_len(sgl) / sport->rx_periods;
+		rx_ring->tail = ((rx_ring->head-1) / bd_size) * bd_size;
+
+		if (rx_ring->head <= sg_dma_len(sgl) &&
+		    rx_ring->head > rx_ring->tail) {
+
+			/* Move data from tail to head */
+			r_bytes = rx_ring->head - rx_ring->tail;
 
-	if (count) {
-		if (!(sport->port.ignore_status_mask & URXD_DUMMY_READ)) {
-			int bytes = tty_insert_flip_string(port, sport->rx_buf,
-					count);
+			/* CPU claims ownership of RX DMA buffer */
+			dma_sync_sg_for_cpu(sport->port.dev, sgl, 1,
+				DMA_FROM_DEVICE);
 
-			if (bytes != count)
+			w_bytes = tty_insert_flip_string(port,
+				sport->rx_buf + rx_ring->tail, r_bytes);
+
+			/* UART retrieves ownership of RX DMA buffer */
+			dma_sync_sg_for_device(sport->port.dev, sgl, 1,
+				DMA_FROM_DEVICE);
+
+			if (w_bytes != r_bytes)
 				sport->port.icount.buf_overrun++;
+
+			sport->port.icount.rx += w_bytes;
+		} else	{
+			WARN_ON(rx_ring->head > sg_dma_len(sgl));
+			WARN_ON(rx_ring->head <= rx_ring->tail);
 		}
-		tty_flip_buffer_push(port);
-		sport->port.icount.rx += count;
 	}
 
-	/*
-	 * Restart RX DMA directly if more data is available in order to skip
-	 * the roundtrip through the IRQ handler. If there is some data already
-	 * in the FIFO, DMA needs to be restarted soon anyways.
-	 *
-	 * Otherwise stop the DMA and reactivate FIFO IRQs to restart DMA once
-	 * data starts to arrive again.
-	 */
-	if (readl(sport->port.membase + USR2) & USR2_RDR)
-		start_rx_dma(sport);
-	else
-		imx_rx_dma_done(sport);
+	if (w_bytes) {
+		tty_flip_buffer_push(port);
+		dev_dbg(sport->port.dev, "We get %d bytes.\n", w_bytes);
+	}
 }
 
+/* RX DMA buffer periods */
+#define RX_DMA_PERIODS 4
+
 static int start_rx_dma(struct imx_port *sport)
 {
 	struct scatterlist *sgl = &sport->rx_sgl;
@@ -1017,14 +1050,21 @@ static int start_rx_dma(struct imx_port *sport)
 	struct dma_async_tx_descriptor *desc;
 	int ret;
 
+	sport->rx_ring.head = 0;
+	sport->rx_ring.tail = 0;
+	sport->rx_periods = RX_DMA_PERIODS;
+
 	sg_init_one(sgl, sport->rx_buf, RX_BUF_SIZE);
 	ret = dma_map_sg(dev, sgl, 1, DMA_FROM_DEVICE);
 	if (ret == 0) {
 		dev_err(dev, "DMA mapping error for RX.\n");
 		return -EINVAL;
 	}
-	desc = dmaengine_prep_slave_sg(chan, sgl, 1, DMA_DEV_TO_MEM,
-					DMA_PREP_INTERRUPT);
+
+	desc = dmaengine_prep_dma_cyclic(chan, sg_dma_address(sgl),
+		sg_dma_len(sgl), sg_dma_len(sgl) / sport->rx_periods,
+		DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT);
+
 	if (!desc) {
 		dma_unmap_sg(dev, sgl, 1, DMA_FROM_DEVICE);
 		dev_err(dev, "We cannot prepare for the RX slave dma!\n");
@@ -1034,11 +1074,36 @@ static int start_rx_dma(struct imx_port *sport)
 	desc->callback_param = sport;
 
 	dev_dbg(dev, "RX: prepare for the DMA.\n");
-	dmaengine_submit(desc);
+	sport->rx_cookie = dmaengine_submit(desc);
 	dma_async_issue_pending(chan);
 	return 0;
 }
 
+static void clear_rx_errors(struct imx_port *sport)
+{
+	unsigned int status_usr1, status_usr2;
+
+	status_usr1 = readl(sport->port.membase + USR1);
+	status_usr2 = readl(sport->port.membase + USR2);
+
+	if (status_usr2 & USR2_BRCD) {
+		sport->port.icount.brk++;
+		writel(USR2_BRCD, sport->port.membase + USR2);
+	} else if (status_usr1 & USR1_FRAMERR) {
+		sport->port.icount.frame++;
+		writel(USR1_FRAMERR, sport->port.membase + USR1);
+	} else if (status_usr1 & USR1_PARITYERR) {
+		sport->port.icount.parity++;
+		writel(USR1_PARITYERR, sport->port.membase + USR1);
+	}
+
+	if (status_usr2 & USR2_ORE) {
+		sport->port.icount.overrun++;
+		writel(USR2_ORE, sport->port.membase + USR2);
+	}
+
+}
+
 #define TXTL_DEFAULT 2 /* reset default */
 #define RXTL_DEFAULT 1 /* reset default */
 #define TXTL_DMA 8 /* DMA burst setting */
@@ -1058,14 +1123,16 @@ static void imx_setup_ufcr(struct imx_port *sport,
 static void imx_uart_dma_exit(struct imx_port *sport)
 {
 	if (sport->dma_chan_rx) {
+		dmaengine_terminate_sync(sport->dma_chan_rx);
 		dma_release_channel(sport->dma_chan_rx);
 		sport->dma_chan_rx = NULL;
-
+		sport->rx_cookie = -EINVAL;
 		kfree(sport->rx_buf);
 		sport->rx_buf = NULL;
 	}
 
 	if (sport->dma_chan_tx) {
+		dmaengine_terminate_sync(sport->dma_chan_tx);
 		dma_release_channel(sport->dma_chan_tx);
 		sport->dma_chan_tx = NULL;
 	}
@@ -1103,6 +1170,7 @@ static int imx_uart_dma_init(struct imx_port *sport)
 		ret = -ENOMEM;
 		goto err;
 	}
+	sport->rx_ring.buf = sport->rx_buf;
 
 	/* Prepare for TX : */
 	sport->dma_chan_tx = dma_request_slave_channel(dev, "tx");
@@ -1201,8 +1269,7 @@ static int imx_startup(struct uart_port *port)
 	writel(temp & ~UCR4_DREN, sport->port.membase + UCR4);
 
 	/* Can we enable the DMA support? */
-	if (is_imx6q_uart(sport) && !uart_console(port) &&
-	    !sport->dma_is_inited)
+	if (!uart_console(port) && !sport->dma_is_inited)
 		imx_uart_dma_init(sport);
 
 	spin_lock_irqsave(&sport->port.lock, flags);
@@ -1283,17 +1350,11 @@ static void imx_shutdown(struct uart_port *port)
 	unsigned long flags;
 
 	if (sport->dma_is_enabled) {
-		int ret;
+		sport->dma_is_rxing = 0;
+		sport->dma_is_txing = 0;
+		dmaengine_terminate_sync(sport->dma_chan_tx);
+		dmaengine_terminate_sync(sport->dma_chan_rx);
 
-		/* We have to wait for the DMA to finish. */
-		ret = wait_event_interruptible(sport->dma_wait,
-			!sport->dma_is_rxing && !sport->dma_is_txing);
-		if (ret != 0) {
-			sport->dma_is_rxing = 0;
-			sport->dma_is_txing = 0;
-			dmaengine_terminate_all(sport->dma_chan_tx);
-			dmaengine_terminate_all(sport->dma_chan_rx);
-		}
 		spin_lock_irqsave(&sport->port.lock, flags);
 		imx_stop_tx(port);
 		imx_stop_rx(port);
@@ -1690,7 +1751,7 @@ static int imx_rs485_config(struct uart_port *port,
 	return 0;
 }
 
-static struct uart_ops imx_pops = {
+static const struct uart_ops imx_pops = {
 	.tx_empty	= imx_tx_empty,
 	.set_mctrl	= imx_set_mctrl,
 	.get_mctrl	= imx_get_mctrl,
@@ -2077,8 +2138,10 @@ static int serial_imx_probe(struct platform_device *pdev)
 
 	/* For register access, we only need to enable the ipg clock. */
 	ret = clk_prepare_enable(sport->clk_ipg);
-	if (ret)
+	if (ret) {
+		dev_err(&pdev->dev, "failed to enable per clk: %d\n", ret);
 		return ret;
+	}
 
 	/* Disable interrupts before requesting them */
 	reg = readl_relaxed(sport->port.membase + UCR1);
@@ -2095,18 +2158,26 @@ static int serial_imx_probe(struct platform_device *pdev)
 	if (txirq > 0) {
 		ret = devm_request_irq(&pdev->dev, rxirq, imx_rxint, 0,
 				       dev_name(&pdev->dev), sport);
-		if (ret)
+		if (ret) {
+			dev_err(&pdev->dev, "failed to request rx irq: %d\n",
+				ret);
 			return ret;
+		}
 
 		ret = devm_request_irq(&pdev->dev, txirq, imx_txint, 0,
 				       dev_name(&pdev->dev), sport);
-		if (ret)
+		if (ret) {
+			dev_err(&pdev->dev, "failed to request tx irq: %d\n",
+				ret);
 			return ret;
+		}
 	} else {
 		ret = devm_request_irq(&pdev->dev, rxirq, imx_int, 0,
 				       dev_name(&pdev->dev), sport);
-		if (ret)
+		if (ret) {
+			dev_err(&pdev->dev, "failed to request irq: %d\n", ret);
 			return ret;
+		}
 	}
 
 	imx_ports[sport->port.line] = sport;
diff --git a/drivers/tty/serial/jsm/jsm_tty.c b/drivers/tty/serial/jsm/jsm_tty.c
index c5ddfe542451..ec7d8383900f 100644
--- a/drivers/tty/serial/jsm/jsm_tty.c
+++ b/drivers/tty/serial/jsm/jsm_tty.c
@@ -346,7 +346,7 @@ static void jsm_config_port(struct uart_port *port, int flags)
 	port->type = PORT_JSM;
 }
 
-static struct uart_ops jsm_ops = {
+static const struct uart_ops jsm_ops = {
 	.tx_empty	= jsm_tty_tx_empty,
 	.set_mctrl	= jsm_tty_set_mctrl,
 	.get_mctrl	= jsm_tty_get_mctrl,
diff --git a/drivers/tty/serial/max3100.c b/drivers/tty/serial/max3100.c
index 5c4c280b3207..ace82645b123 100644
--- a/drivers/tty/serial/max3100.c
+++ b/drivers/tty/serial/max3100.c
@@ -712,7 +712,7 @@ static void max3100_break_ctl(struct uart_port *port, int break_state)
 	dev_dbg(&s->spi->dev, "%s\n", __func__);
 }
 
-static struct uart_ops max3100_ops = {
+static const struct uart_ops max3100_ops = {
 	.tx_empty	= max3100_tx_empty,
 	.set_mctrl	= max3100_set_mctrl,
 	.get_mctrl	= max3100_get_mctrl,
diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c
index 9360801df3c4..8a3e92638e10 100644
--- a/drivers/tty/serial/max310x.c
+++ b/drivers/tty/serial/max310x.c
@@ -1329,9 +1329,9 @@ static int max310x_spi_probe(struct spi_device *spi)
 		const struct spi_device_id *id_entry = spi_get_device_id(spi);
 
 		devtype = (struct max310x_devtype *)id_entry->driver_data;
-		flags = IRQF_TRIGGER_FALLING;
 	}
 
+	flags = IRQF_TRIGGER_FALLING;
 	regcfg.max_register = devtype->nr * 0x20 - 1;
 	regmap = devm_regmap_init_spi(spi, &regcfg);
 
diff --git a/drivers/tty/serial/men_z135_uart.c b/drivers/tty/serial/men_z135_uart.c
index a44290e9b5a8..e72ea61c70db 100644
--- a/drivers/tty/serial/men_z135_uart.c
+++ b/drivers/tty/serial/men_z135_uart.c
@@ -775,7 +775,7 @@ static int men_z135_verify_port(struct uart_port *port,
 	return -EINVAL;
 }
 
-static struct uart_ops men_z135_ops = {
+static const struct uart_ops men_z135_ops = {
 	.tx_empty = men_z135_tx_empty,
 	.set_mctrl = men_z135_set_mctrl,
 	.get_mctrl = men_z135_get_mctrl,
diff --git a/drivers/tty/serial/mxs-auart.c b/drivers/tty/serial/mxs-auart.c
index eb54e5c77ead..770454e0dfa3 100644
--- a/drivers/tty/serial/mxs-auart.c
+++ b/drivers/tty/serial/mxs-auart.c
@@ -1317,7 +1317,7 @@ static void mxs_auart_break_ctl(struct uart_port *u, int ctl)
 		mxs_clr(AUART_LINECTRL_BRK, s, REG_LINECTRL);
 }
 
-static struct uart_ops mxs_auart_ops = {
+static const struct uart_ops mxs_auart_ops = {
 	.tx_empty       = mxs_auart_tx_empty,
 	.start_tx       = mxs_auart_start_tx,
 	.stop_tx	= mxs_auart_stop_tx,
@@ -1510,10 +1510,7 @@ static int mxs_get_clks(struct mxs_auart_port *s,
 
 	if (!is_asm9260_auart(s)) {
 		s->clk = devm_clk_get(&pdev->dev, NULL);
-		if (IS_ERR(s->clk))
-			return PTR_ERR(s->clk);
-
-		return 0;
+		return PTR_ERR_OR_ZERO(s->clk);
 	}
 
 	s->clk = devm_clk_get(s->dev, "mod");
@@ -1537,16 +1534,20 @@ static int mxs_get_clks(struct mxs_auart_port *s,
 	err = clk_set_rate(s->clk, clk_get_rate(s->clk_ahb));
 	if (err) {
 		dev_err(s->dev, "Failed to set rate!\n");
-		return err;
+		goto disable_clk_ahb;
 	}
 
 	err = clk_prepare_enable(s->clk);
 	if (err) {
 		dev_err(s->dev, "Failed to enable clk!\n");
-		return err;
+		goto disable_clk_ahb;
 	}
 
 	return 0;
+
+disable_clk_ahb:
+	clk_disable_unprepare(s->clk_ahb);
+	return err;
 }
 
 /*
diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c
index ea4ffc2ebb2f..d391650b82e7 100644
--- a/drivers/tty/serial/pch_uart.c
+++ b/drivers/tty/serial/pch_uart.c
@@ -31,6 +31,7 @@
 #include <linux/dmi.h>
 #include <linux/nmi.h>
 #include <linux/delay.h>
+#include <linux/of.h>
 
 #include <linux/debugfs.h>
 #include <linux/dmaengine.h>
@@ -1603,7 +1604,7 @@ static void pch_uart_put_poll_char(struct uart_port *port,
 }
 #endif /* CONFIG_CONSOLE_POLL */
 
-static struct uart_ops pch_uart_ops = {
+static const struct uart_ops pch_uart_ops = {
 	.tx_empty = pch_uart_tx_empty,
 	.set_mctrl = pch_uart_set_mctrl,
 	.get_mctrl = pch_uart_get_mctrl,
@@ -1826,6 +1827,10 @@ static struct eg20t_port *pch_uart_init_port(struct pci_dev *pdev,
 	priv->trigger_level = 1;
 	priv->fcr = 0;
 
+	if (pdev->dev.of_node)
+		of_property_read_u32(pdev->dev.of_node, "clock-frequency"
+					 , &user_uartclk);
+
 #ifdef CONFIG_SERIAL_PCH_UART_CONSOLE
 	pch_uart_ports[board->line_no] = priv;
 #endif
diff --git a/drivers/tty/serial/samsung.c b/drivers/tty/serial/samsung.c
index ae2095a66708..f44615fa474d 100644
--- a/drivers/tty/serial/samsung.c
+++ b/drivers/tty/serial/samsung.c
@@ -1577,7 +1577,7 @@ static void s3c24xx_serial_resetport(struct uart_port *port,
 }
 
 
-#ifdef CONFIG_CPU_FREQ
+#ifdef CONFIG_ARM_S3C24XX_CPUFREQ
 
 static int s3c24xx_serial_cpufreq_transition(struct notifier_block *nb,
 					     unsigned long val, void *data)
diff --git a/drivers/tty/serial/samsung.h b/drivers/tty/serial/samsung.h
index 2ae4fcee1814..a04acef1cb20 100644
--- a/drivers/tty/serial/samsung.h
+++ b/drivers/tty/serial/samsung.h
@@ -102,7 +102,7 @@ struct s3c24xx_uart_port {
 
 	struct s3c24xx_uart_dma		*dma;
 
-#ifdef CONFIG_CPU_FREQ
+#ifdef CONFIG_ARM_S3C24XX_CPUFREQ
 	struct notifier_block		freq_transition;
 #endif
 };
diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c
index f36e6df2fa90..2675792a8f59 100644
--- a/drivers/tty/serial/sc16is7xx.c
+++ b/drivers/tty/serial/sc16is7xx.c
@@ -708,7 +708,7 @@ static irqreturn_t sc16is7xx_irq(int irq, void *dev_id)
 {
 	struct sc16is7xx_port *s = (struct sc16is7xx_port *)dev_id;
 
-	queue_kthread_work(&s->kworker, &s->irq_work);
+	kthread_queue_work(&s->kworker, &s->irq_work);
 
 	return IRQ_HANDLED;
 }
@@ -784,7 +784,7 @@ static void sc16is7xx_ier_clear(struct uart_port *port, u8 bit)
 
 	one->config.flags |= SC16IS7XX_RECONF_IER;
 	one->config.ier_clear |= bit;
-	queue_kthread_work(&s->kworker, &one->reg_work);
+	kthread_queue_work(&s->kworker, &one->reg_work);
 }
 
 static void sc16is7xx_stop_tx(struct uart_port *port)
@@ -802,7 +802,7 @@ static void sc16is7xx_start_tx(struct uart_port *port)
 	struct sc16is7xx_port *s = dev_get_drvdata(port->dev);
 	struct sc16is7xx_one *one = to_sc16is7xx_one(port, port);
 
-	queue_kthread_work(&s->kworker, &one->tx_work);
+	kthread_queue_work(&s->kworker, &one->tx_work);
 }
 
 static unsigned int sc16is7xx_tx_empty(struct uart_port *port)
@@ -828,7 +828,7 @@ static void sc16is7xx_set_mctrl(struct uart_port *port, unsigned int mctrl)
 	struct sc16is7xx_one *one = to_sc16is7xx_one(port, port);
 
 	one->config.flags |= SC16IS7XX_RECONF_MD;
-	queue_kthread_work(&s->kworker, &one->reg_work);
+	kthread_queue_work(&s->kworker, &one->reg_work);
 }
 
 static void sc16is7xx_break_ctl(struct uart_port *port, int break_state)
@@ -957,7 +957,7 @@ static int sc16is7xx_config_rs485(struct uart_port *port,
 
 	port->rs485 = *rs485;
 	one->config.flags |= SC16IS7XX_RECONF_RS485;
-	queue_kthread_work(&s->kworker, &one->reg_work);
+	kthread_queue_work(&s->kworker, &one->reg_work);
 
 	return 0;
 }
@@ -1030,7 +1030,7 @@ static void sc16is7xx_shutdown(struct uart_port *port)
 
 	sc16is7xx_power(port, 0);
 
-	flush_kthread_worker(&s->kworker);
+	kthread_flush_worker(&s->kworker);
 }
 
 static const char *sc16is7xx_type(struct uart_port *port)
@@ -1176,8 +1176,8 @@ static int sc16is7xx_probe(struct device *dev,
 	s->devtype = devtype;
 	dev_set_drvdata(dev, s);
 
-	init_kthread_worker(&s->kworker);
-	init_kthread_work(&s->irq_work, sc16is7xx_ist);
+	kthread_init_worker(&s->kworker);
+	kthread_init_work(&s->irq_work, sc16is7xx_ist);
 	s->kworker_task = kthread_run(kthread_worker_fn, &s->kworker,
 				      "sc16is7xx");
 	if (IS_ERR(s->kworker_task)) {
@@ -1205,6 +1205,10 @@ static int sc16is7xx_probe(struct device *dev,
 	}
 #endif
 
+	/* reset device, purging any pending irq / data */
+	regmap_write(s->regmap, SC16IS7XX_IOCONTROL_REG << SC16IS7XX_REG_SHIFT,
+			SC16IS7XX_IOCONTROL_SRESET_BIT);
+
 	for (i = 0; i < devtype->nr_uart; ++i) {
 		s->p[i].line		= i;
 		/* Initialize port data */
@@ -1230,10 +1234,26 @@ static int sc16is7xx_probe(struct device *dev,
 				     SC16IS7XX_EFCR_RXDISABLE_BIT |
 				     SC16IS7XX_EFCR_TXDISABLE_BIT);
 		/* Initialize kthread work structs */
-		init_kthread_work(&s->p[i].tx_work, sc16is7xx_tx_proc);
-		init_kthread_work(&s->p[i].reg_work, sc16is7xx_reg_proc);
+		kthread_init_work(&s->p[i].tx_work, sc16is7xx_tx_proc);
+		kthread_init_work(&s->p[i].reg_work, sc16is7xx_reg_proc);
 		/* Register port */
 		uart_add_one_port(&sc16is7xx_uart, &s->p[i].port);
+
+		/* Enable EFR */
+		sc16is7xx_port_write(&s->p[i].port, SC16IS7XX_LCR_REG,
+				     SC16IS7XX_LCR_CONF_MODE_B);
+
+		regcache_cache_bypass(s->regmap, true);
+
+		/* Enable write access to enhanced features */
+		sc16is7xx_port_write(&s->p[i].port, SC16IS7XX_EFR_REG,
+				     SC16IS7XX_EFR_ENABLE_BIT);
+
+		regcache_cache_bypass(s->regmap, false);
+
+		/* Restore access to general registers */
+		sc16is7xx_port_write(&s->p[i].port, SC16IS7XX_LCR_REG, 0x00);
+
 		/* Go to suspend mode */
 		sc16is7xx_power(&s->p[i].port, 0);
 	}
@@ -1281,7 +1301,7 @@ static int sc16is7xx_remove(struct device *dev)
 		sc16is7xx_power(&s->p[i].port, 0);
 	}
 
-	flush_kthread_worker(&s->kworker);
+	kthread_flush_worker(&s->kworker);
 	kthread_stop(s->kworker_task);
 
 	if (!IS_ERR(s->clk))
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index 9fc15335c8c5..6e4f63627479 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -235,18 +235,9 @@ static int uart_startup(struct tty_struct *tty, struct uart_state *state,
 	if (tty_port_initialized(port))
 		return 0;
 
-	/*
-	 * Set the TTY IO error marker - we will only clear this
-	 * once we have successfully opened the port.
-	 */
-	set_bit(TTY_IO_ERROR, &tty->flags);
-
 	retval = uart_port_startup(tty, state, init_hw);
-	if (!retval) {
-		tty_port_set_initialized(port, 1);
-		clear_bit(TTY_IO_ERROR, &tty->flags);
-	} else if (retval > 0)
-		retval = 0;
+	if (retval)
+		set_bit(TTY_IO_ERROR, &tty->flags);
 
 	return retval;
 }
@@ -972,8 +963,11 @@ static int uart_set_info(struct tty_struct *tty, struct tty_port *port,
 			}
 			uart_change_speed(tty, state, NULL);
 		}
-	} else
+	} else {
 		retval = uart_startup(tty, state, 1);
+		if (retval > 0)
+			retval = 0;
+	}
  exit:
 	return retval;
 }
@@ -1139,6 +1133,8 @@ static int uart_do_autoconfig(struct tty_struct *tty,struct uart_state *state)
 		uport->ops->config_port(uport, flags);
 
 		ret = uart_startup(tty, state, 1);
+		if (ret > 0)
+			ret = 0;
 	}
 out:
 	mutex_unlock(&port->mutex);
@@ -1465,7 +1461,6 @@ static void uart_close(struct tty_struct *tty, struct file *filp)
 {
 	struct uart_state *state = tty->driver_data;
 	struct tty_port *port;
-	struct uart_port *uport;
 
 	if (!state) {
 		struct uart_driver *drv = tty->driver->driver_state;
@@ -1481,56 +1476,36 @@ static void uart_close(struct tty_struct *tty, struct file *filp)
 	port = &state->port;
 	pr_debug("uart_close(%d) called\n", tty->index);
 
-	if (tty_port_close_start(port, tty, filp) == 0)
-		return;
+	tty_port_close(tty->port, tty, filp);
+}
 
-	mutex_lock(&port->mutex);
-	uport = uart_port_check(state);
+static void uart_tty_port_shutdown(struct tty_port *port)
+{
+	struct uart_state *state = container_of(port, struct uart_state, port);
+	struct uart_port *uport = uart_port_check(state);
 
 	/*
 	 * At this point, we stop accepting input.  To do this, we
 	 * disable the receive line status interrupts.
 	 */
-	if (tty_port_initialized(port) &&
-	    !WARN(!uport, "detached port still initialized!\n")) {
-		spin_lock_irq(&uport->lock);
-		uport->ops->stop_rx(uport);
-		spin_unlock_irq(&uport->lock);
-		/*
-		 * Before we drop DTR, make sure the UART transmitter
-		 * has completely drained; this is especially
-		 * important if there is a transmit FIFO!
-		 */
-		uart_wait_until_sent(tty, uport->timeout);
-	}
-
-	uart_shutdown(tty, state);
-	tty_port_tty_set(port, NULL);
+	if (WARN(!uport, "detached port still initialized!\n"))
+		return;
 
-	spin_lock_irq(&port->lock);
+	spin_lock_irq(&uport->lock);
+	uport->ops->stop_rx(uport);
+	spin_unlock_irq(&uport->lock);
 
-	if (port->blocked_open) {
-		spin_unlock_irq(&port->lock);
-		if (port->close_delay)
-			msleep_interruptible(jiffies_to_msecs(port->close_delay));
-		spin_lock_irq(&port->lock);
-	} else if (uport && !uart_console(uport)) {
-		spin_unlock_irq(&port->lock);
-		uart_change_pm(state, UART_PM_STATE_OFF);
-		spin_lock_irq(&port->lock);
-	}
-	spin_unlock_irq(&port->lock);
-	tty_port_set_active(port, 0);
+	uart_port_shutdown(port);
 
 	/*
-	 * Wake up anyone trying to open this port.
+	 * It's possible for shutdown to be called after suspend if we get
+	 * a DCD drop (hangup) at just the right time.  Clear suspended bit so
+	 * we don't try to resume a port that has been shutdown.
 	 */
-	wake_up_interruptible(&port->open_wait);
+	tty_port_set_suspended(port, 0);
 
-	mutex_unlock(&port->mutex);
+	uart_change_pm(state, UART_PM_STATE_OFF);
 
-	tty_ldisc_flush(tty);
-	tty->closing = 0;
 }
 
 static void uart_wait_until_sent(struct tty_struct *tty, int timeout)
@@ -1711,52 +1686,31 @@ static int uart_open(struct tty_struct *tty, struct file *filp)
 	struct uart_driver *drv = tty->driver->driver_state;
 	int retval, line = tty->index;
 	struct uart_state *state = drv->state + line;
-	struct tty_port *port = &state->port;
-	struct uart_port *uport;
 
-	pr_debug("uart_open(%d) called\n", line);
+	tty->driver_data = state;
 
-	spin_lock_irq(&port->lock);
-	++port->count;
-	spin_unlock_irq(&port->lock);
+	retval = tty_port_open(&state->port, tty, filp);
+	if (retval > 0)
+		retval = 0;
 
-	/*
-	 * We take the semaphore here to guarantee that we won't be re-entered
-	 * while allocating the state structure, or while we request any IRQs
-	 * that the driver may need.  This also has the nice side-effect that
-	 * it delays the action of uart_hangup, so we can guarantee that
-	 * state->port.tty will always contain something reasonable.
-	 */
-	if (mutex_lock_interruptible(&port->mutex)) {
-		retval = -ERESTARTSYS;
-		goto end;
-	}
+	return retval;
+}
+
+static int uart_port_activate(struct tty_port *port, struct tty_struct *tty)
+{
+	struct uart_state *state = container_of(port, struct uart_state, port);
+	struct uart_port *uport;
 
 	uport = uart_port_check(state);
-	if (!uport || uport->flags & UPF_DEAD) {
-		retval = -ENXIO;
-		goto err_unlock;
-	}
+	if (!uport || uport->flags & UPF_DEAD)
+		return -ENXIO;
 
-	tty->driver_data = state;
-	uport->state = state;
 	port->low_latency = (uport->flags & UPF_LOW_LATENCY) ? 1 : 0;
-	tty_port_tty_set(port, tty);
 
 	/*
 	 * Start up the serial port.
 	 */
-	retval = uart_startup(tty, state, 0);
-
-	/*
-	 * If we succeeded, wait until the port is ready.
-	 */
-err_unlock:
-	mutex_unlock(&port->mutex);
-	if (retval == 0)
-		retval = tty_port_block_til_ready(port, tty, filp);
-end:
-	return retval;
+	return uart_startup(tty, state, 0);
 }
 
 static const char *uart_type(struct uart_port *port)
@@ -1940,7 +1894,7 @@ uart_get_console(struct uart_port *ports, int nr, struct console *co)
  *
  *	Returns 0 on success or -EINVAL on failure
  */
-int uart_parse_earlycon(char *p, unsigned char *iotype, unsigned long *addr,
+int uart_parse_earlycon(char *p, unsigned char *iotype, resource_size_t *addr,
 			char **options)
 {
 	if (strncmp(p, "mmio,", 5) == 0) {
@@ -1968,7 +1922,11 @@ int uart_parse_earlycon(char *p, unsigned char *iotype, unsigned long *addr,
 		return -EINVAL;
 	}
 
-	*addr = simple_strtoul(p, NULL, 0);
+	/*
+	 * Before you replace it with kstrtoull(), think about options separator
+	 * (',') it will not tolerate
+	 */
+	*addr = simple_strtoull(p, NULL, 0);
 	p = strchr(p, ',');
 	if (p)
 		p++;
@@ -2470,6 +2428,8 @@ static const struct tty_operations uart_ops = {
 static const struct tty_port_operations uart_port_ops = {
 	.carrier_raised = uart_carrier_raised,
 	.dtr_rts	= uart_dtr_rts,
+	.activate	= uart_port_activate,
+	.shutdown	= uart_tty_port_shutdown,
 };
 
 /**
@@ -2786,6 +2746,8 @@ int uart_add_one_port(struct uart_driver *drv, struct uart_port *uport)
 	uport->cons = drv->cons;
 	uport->minor = drv->tty_driver->minor_start + uport->line;
 
+	port->console = uart_console(uport);
+
 	/*
 	 * If this port is a console, then the spinlock is already
 	 * initialised.
diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c
index d86eee38aae6..4b26252c2885 100644
--- a/drivers/tty/serial/sh-sci.c
+++ b/drivers/tty/serial/sh-sci.c
@@ -2533,7 +2533,7 @@ static int sci_verify_port(struct uart_port *port, struct serial_struct *ser)
 	return 0;
 }
 
-static struct uart_ops sci_uart_ops = {
+static const struct uart_ops sci_uart_ops = {
 	.tx_empty	= sci_tx_empty,
 	.set_mctrl	= sci_set_mctrl,
 	.get_mctrl	= sci_get_mctrl,
diff --git a/drivers/tty/serial/st-asc.c b/drivers/tty/serial/st-asc.c
index 2d78cb3627ae..379e5bd37df9 100644
--- a/drivers/tty/serial/st-asc.c
+++ b/drivers/tty/serial/st-asc.c
@@ -639,7 +639,7 @@ static void asc_put_poll_char(struct uart_port *port, unsigned char c)
 
 /*---------------------------------------------------------------------*/
 
-static struct uart_ops asc_uart_ops = {
+static const struct uart_ops asc_uart_ops = {
 	.tx_empty	= asc_tx_empty,
 	.set_mctrl	= asc_set_mctrl,
 	.get_mctrl	= asc_get_mctrl,
diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c
index f89d1f79be18..033856287ca2 100644
--- a/drivers/tty/serial/stm32-usart.c
+++ b/drivers/tty/serial/stm32-usart.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (C) Maxime Coquelin 2015
- * Author:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
+ * Authors:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
+ *	     Gerald Baeza <gerald.baeza@st.com>
  * License terms:  GNU General Public License (GPL), version 2
  *
  * Inspired by st-asc.c from STMicroelectronics (c)
@@ -10,120 +11,31 @@
 #define SUPPORT_SYSRQ
 #endif
 
-#include <linux/module.h>
-#include <linux/serial.h>
+#include <linux/clk.h>
 #include <linux/console.h>
-#include <linux/sysrq.h>
-#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/dma-direction.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/irq.h>
-#include <linux/tty.h>
-#include <linux/tty_flip.h>
-#include <linux/delay.h>
-#include <linux/spinlock.h>
-#include <linux/pm_runtime.h>
+#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/serial_core.h>
-#include <linux/clk.h>
-
-#define DRIVER_NAME "stm32-usart"
-
-/* Register offsets */
-#define USART_SR		0x00
-#define USART_DR		0x04
-#define USART_BRR		0x08
-#define USART_CR1		0x0c
-#define USART_CR2		0x10
-#define USART_CR3		0x14
-#define USART_GTPR		0x18
-
-/* USART_SR */
-#define USART_SR_PE		BIT(0)
-#define USART_SR_FE		BIT(1)
-#define USART_SR_NF		BIT(2)
-#define USART_SR_ORE		BIT(3)
-#define USART_SR_IDLE		BIT(4)
-#define USART_SR_RXNE		BIT(5)
-#define USART_SR_TC		BIT(6)
-#define USART_SR_TXE		BIT(7)
-#define USART_SR_LBD		BIT(8)
-#define USART_SR_CTS		BIT(9)
-#define USART_SR_ERR_MASK	(USART_SR_LBD | USART_SR_ORE | \
-				 USART_SR_FE | USART_SR_PE)
-/* Dummy bits */
-#define USART_SR_DUMMY_RX	BIT(16)
-
-/* USART_DR */
-#define USART_DR_MASK		GENMASK(8, 0)
-
-/* USART_BRR */
-#define USART_BRR_DIV_F_MASK	GENMASK(3, 0)
-#define USART_BRR_DIV_M_MASK	GENMASK(15, 4)
-#define USART_BRR_DIV_M_SHIFT	4
-
-/* USART_CR1 */
-#define USART_CR1_SBK		BIT(0)
-#define USART_CR1_RWU		BIT(1)
-#define USART_CR1_RE		BIT(2)
-#define USART_CR1_TE		BIT(3)
-#define USART_CR1_IDLEIE	BIT(4)
-#define USART_CR1_RXNEIE	BIT(5)
-#define USART_CR1_TCIE		BIT(6)
-#define USART_CR1_TXEIE		BIT(7)
-#define USART_CR1_PEIE		BIT(8)
-#define USART_CR1_PS		BIT(9)
-#define USART_CR1_PCE		BIT(10)
-#define USART_CR1_WAKE		BIT(11)
-#define USART_CR1_M		BIT(12)
-#define USART_CR1_UE		BIT(13)
-#define USART_CR1_OVER8		BIT(15)
-#define USART_CR1_IE_MASK	GENMASK(8, 4)
-
-/* USART_CR2 */
-#define USART_CR2_ADD_MASK	GENMASK(3, 0)
-#define USART_CR2_LBDL		BIT(5)
-#define USART_CR2_LBDIE		BIT(6)
-#define USART_CR2_LBCL		BIT(8)
-#define USART_CR2_CPHA		BIT(9)
-#define USART_CR2_CPOL		BIT(10)
-#define USART_CR2_CLKEN		BIT(11)
-#define USART_CR2_STOP_2B	BIT(13)
-#define USART_CR2_STOP_MASK	GENMASK(13, 12)
-#define USART_CR2_LINEN		BIT(14)
-
-/* USART_CR3 */
-#define USART_CR3_EIE		BIT(0)
-#define USART_CR3_IREN		BIT(1)
-#define USART_CR3_IRLP		BIT(2)
-#define USART_CR3_HDSEL		BIT(3)
-#define USART_CR3_NACK		BIT(4)
-#define USART_CR3_SCEN		BIT(5)
-#define USART_CR3_DMAR		BIT(6)
-#define USART_CR3_DMAT		BIT(7)
-#define USART_CR3_RTSE		BIT(8)
-#define USART_CR3_CTSE		BIT(9)
-#define USART_CR3_CTSIE		BIT(10)
-#define USART_CR3_ONEBIT	BIT(11)
-
-/* USART_GTPR */
-#define USART_GTPR_PSC_MASK	GENMASK(7, 0)
-#define USART_GTPR_GT_MASK	GENMASK(15, 8)
-
-#define DRIVER_NAME "stm32-usart"
-#define STM32_SERIAL_NAME "ttyS"
-#define STM32_MAX_PORTS 6
-
-struct stm32_port {
-	struct uart_port port;
-	struct clk *clk;
-	bool hw_flow_control;
-};
+#include <linux/serial.h>
+#include <linux/spinlock.h>
+#include <linux/sysrq.h>
+#include <linux/tty_flip.h>
+#include <linux/tty.h>
 
-static struct stm32_port stm32_ports[STM32_MAX_PORTS];
-static struct uart_driver stm32_usart_driver;
+#include "stm32-usart.h"
 
 static void stm32_stop_tx(struct uart_port *port);
+static void stm32_transmit_chars(struct uart_port *port);
 
 static inline struct stm32_port *to_stm32_port(struct uart_port *port)
 {
@@ -148,19 +60,64 @@ static void stm32_clr_bits(struct uart_port *port, u32 reg, u32 bits)
 	writel_relaxed(val, port->membase + reg);
 }
 
-static void stm32_receive_chars(struct uart_port *port)
+static int stm32_pending_rx(struct uart_port *port, u32 *sr, int *last_res,
+			    bool threaded)
+{
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+	enum dma_status status;
+	struct dma_tx_state state;
+
+	*sr = readl_relaxed(port->membase + ofs->isr);
+
+	if (threaded && stm32_port->rx_ch) {
+		status = dmaengine_tx_status(stm32_port->rx_ch,
+					     stm32_port->rx_ch->cookie,
+					     &state);
+		if ((status == DMA_IN_PROGRESS) &&
+		    (*last_res != state.residue))
+			return 1;
+		else
+			return 0;
+	} else if (*sr & USART_SR_RXNE) {
+		return 1;
+	}
+	return 0;
+}
+
+static unsigned long
+stm32_get_char(struct uart_port *port, u32 *sr, int *last_res)
+{
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+	unsigned long c;
+
+	if (stm32_port->rx_ch) {
+		c = stm32_port->rx_buf[RX_BUF_L - (*last_res)--];
+		if ((*last_res) == 0)
+			*last_res = RX_BUF_L;
+		return c;
+	} else {
+		return readl_relaxed(port->membase + ofs->rdr);
+	}
+}
+
+static void stm32_receive_chars(struct uart_port *port, bool threaded)
 {
 	struct tty_port *tport = &port->state->port;
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 	unsigned long c;
 	u32 sr;
 	char flag;
+	static int last_res = RX_BUF_L;
 
 	if (port->irq_wake)
 		pm_wakeup_event(tport->tty->dev, 0);
 
-	while ((sr = readl_relaxed(port->membase + USART_SR)) & USART_SR_RXNE) {
+	while (stm32_pending_rx(port, &sr, &last_res, threaded)) {
 		sr |= USART_SR_DUMMY_RX;
-		c = readl_relaxed(port->membase + USART_DR);
+		c = stm32_get_char(port, &sr, &last_res);
 		flag = TTY_NORMAL;
 		port->icount.rx++;
 
@@ -170,6 +127,10 @@ static void stm32_receive_chars(struct uart_port *port)
 				if (uart_handle_break(port))
 					continue;
 			} else if (sr & USART_SR_ORE) {
+				if (ofs->icr != UNDEF_REG)
+					writel_relaxed(USART_ICR_ORECF,
+						       port->membase +
+						       ofs->icr);
 				port->icount.overrun++;
 			} else if (sr & USART_SR_PE) {
 				port->icount.parity++;
@@ -197,14 +158,138 @@ static void stm32_receive_chars(struct uart_port *port)
 	spin_lock(&port->lock);
 }
 
+static void stm32_tx_dma_complete(void *arg)
+{
+	struct uart_port *port = arg;
+	struct stm32_port *stm32port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
+	unsigned int isr;
+	int ret;
+
+	ret = readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr,
+						isr,
+						(isr & USART_SR_TC),
+						10, 100000);
+
+	if (ret)
+		dev_err(port->dev, "terminal count not set\n");
+
+	if (ofs->icr == UNDEF_REG)
+		stm32_clr_bits(port, ofs->isr, USART_SR_TC);
+	else
+		stm32_set_bits(port, ofs->icr, USART_CR_TC);
+
+	stm32_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
+	stm32port->tx_dma_busy = false;
+
+	/* Let's see if we have pending data to send */
+	stm32_transmit_chars(port);
+}
+
+static void stm32_transmit_chars_pio(struct uart_port *port)
+{
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+	struct circ_buf *xmit = &port->state->xmit;
+	unsigned int isr;
+	int ret;
+
+	if (stm32_port->tx_dma_busy) {
+		stm32_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
+		stm32_port->tx_dma_busy = false;
+	}
+
+	ret = readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr,
+						isr,
+						(isr & USART_SR_TXE),
+						10, 100);
+
+	if (ret)
+		dev_err(port->dev, "tx empty not set\n");
+
+	stm32_set_bits(port, ofs->cr1, USART_CR1_TXEIE);
+
+	writel_relaxed(xmit->buf[xmit->tail], port->membase + ofs->tdr);
+	xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
+	port->icount.tx++;
+}
+
+static void stm32_transmit_chars_dma(struct uart_port *port)
+{
+	struct stm32_port *stm32port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
+	struct circ_buf *xmit = &port->state->xmit;
+	struct dma_async_tx_descriptor *desc = NULL;
+	dma_cookie_t cookie;
+	unsigned int count, i;
+
+	if (stm32port->tx_dma_busy)
+		return;
+
+	stm32port->tx_dma_busy = true;
+
+	count = uart_circ_chars_pending(xmit);
+
+	if (count > TX_BUF_L)
+		count = TX_BUF_L;
+
+	if (xmit->tail < xmit->head) {
+		memcpy(&stm32port->tx_buf[0], &xmit->buf[xmit->tail], count);
+	} else {
+		size_t one = UART_XMIT_SIZE - xmit->tail;
+		size_t two;
+
+		if (one > count)
+			one = count;
+		two = count - one;
+
+		memcpy(&stm32port->tx_buf[0], &xmit->buf[xmit->tail], one);
+		if (two)
+			memcpy(&stm32port->tx_buf[one], &xmit->buf[0], two);
+	}
+
+	desc = dmaengine_prep_slave_single(stm32port->tx_ch,
+					   stm32port->tx_dma_buf,
+					   count,
+					   DMA_MEM_TO_DEV,
+					   DMA_PREP_INTERRUPT);
+
+	if (!desc) {
+		for (i = count; i > 0; i--)
+			stm32_transmit_chars_pio(port);
+		return;
+	}
+
+	desc->callback = stm32_tx_dma_complete;
+	desc->callback_param = port;
+
+	/* Push current DMA TX transaction in the pending queue */
+	cookie = dmaengine_submit(desc);
+
+	/* Issue pending DMA TX requests */
+	dma_async_issue_pending(stm32port->tx_ch);
+
+	stm32_clr_bits(port, ofs->isr, USART_SR_TC);
+	stm32_set_bits(port, ofs->cr3, USART_CR3_DMAT);
+
+	xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE - 1);
+	port->icount.tx += count;
+}
+
 static void stm32_transmit_chars(struct uart_port *port)
 {
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 	struct circ_buf *xmit = &port->state->xmit;
 
 	if (port->x_char) {
-		writel_relaxed(port->x_char, port->membase + USART_DR);
+		if (stm32_port->tx_dma_busy)
+			stm32_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
+		writel_relaxed(port->x_char, port->membase + ofs->tdr);
 		port->x_char = 0;
 		port->icount.tx++;
+		if (stm32_port->tx_dma_busy)
+			stm32_set_bits(port, ofs->cr3, USART_CR3_DMAT);
 		return;
 	}
 
@@ -218,9 +303,10 @@ static void stm32_transmit_chars(struct uart_port *port)
 		return;
 	}
 
-	writel_relaxed(xmit->buf[xmit->tail], port->membase + USART_DR);
-	xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
-	port->icount.tx++;
+	if (stm32_port->tx_ch)
+		stm32_transmit_chars_dma(port);
+	else
+		stm32_transmit_chars_pio(port);
 
 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 		uart_write_wakeup(port);
@@ -232,34 +318,60 @@ static void stm32_transmit_chars(struct uart_port *port)
 static irqreturn_t stm32_interrupt(int irq, void *ptr)
 {
 	struct uart_port *port = ptr;
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 	u32 sr;
 
 	spin_lock(&port->lock);
 
-	sr = readl_relaxed(port->membase + USART_SR);
+	sr = readl_relaxed(port->membase + ofs->isr);
 
-	if (sr & USART_SR_RXNE)
-		stm32_receive_chars(port);
+	if ((sr & USART_SR_RXNE) && !(stm32_port->rx_ch))
+		stm32_receive_chars(port, false);
 
-	if (sr & USART_SR_TXE)
+	if ((sr & USART_SR_TXE) && !(stm32_port->tx_ch))
 		stm32_transmit_chars(port);
 
 	spin_unlock(&port->lock);
 
+	if (stm32_port->rx_ch)
+		return IRQ_WAKE_THREAD;
+	else
+		return IRQ_HANDLED;
+}
+
+static irqreturn_t stm32_threaded_interrupt(int irq, void *ptr)
+{
+	struct uart_port *port = ptr;
+	struct stm32_port *stm32_port = to_stm32_port(port);
+
+	spin_lock(&port->lock);
+
+	if (stm32_port->rx_ch)
+		stm32_receive_chars(port, true);
+
+	spin_unlock(&port->lock);
+
 	return IRQ_HANDLED;
 }
 
 static unsigned int stm32_tx_empty(struct uart_port *port)
 {
-	return readl_relaxed(port->membase + USART_SR) & USART_SR_TXE;
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+
+	return readl_relaxed(port->membase + ofs->isr) & USART_SR_TXE;
 }
 
 static void stm32_set_mctrl(struct uart_port *port, unsigned int mctrl)
 {
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+
 	if ((mctrl & TIOCM_RTS) && (port->status & UPSTAT_AUTORTS))
-		stm32_set_bits(port, USART_CR3, USART_CR3_RTSE);
+		stm32_set_bits(port, ofs->cr3, USART_CR3_RTSE);
 	else
-		stm32_clr_bits(port, USART_CR3, USART_CR3_RTSE);
+		stm32_clr_bits(port, ofs->cr3, USART_CR3_RTSE);
 }
 
 static unsigned int stm32_get_mctrl(struct uart_port *port)
@@ -271,7 +383,10 @@ static unsigned int stm32_get_mctrl(struct uart_port *port)
 /* Transmit stop */
 static void stm32_stop_tx(struct uart_port *port)
 {
-	stm32_clr_bits(port, USART_CR1, USART_CR1_TXEIE);
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+
+	stm32_clr_bits(port, ofs->cr1, USART_CR1_TXEIE);
 }
 
 /* There are probably characters waiting to be transmitted. */
@@ -282,33 +397,40 @@ static void stm32_start_tx(struct uart_port *port)
 	if (uart_circ_empty(xmit))
 		return;
 
-	stm32_set_bits(port, USART_CR1, USART_CR1_TXEIE | USART_CR1_TE);
+	stm32_transmit_chars(port);
 }
 
 /* Throttle the remote when input buffer is about to overflow. */
 static void stm32_throttle(struct uart_port *port)
 {
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 	unsigned long flags;
 
 	spin_lock_irqsave(&port->lock, flags);
-	stm32_clr_bits(port, USART_CR1, USART_CR1_RXNEIE);
+	stm32_clr_bits(port, ofs->cr1, USART_CR1_RXNEIE);
 	spin_unlock_irqrestore(&port->lock, flags);
 }
 
 /* Unthrottle the remote, the input buffer can now accept data. */
 static void stm32_unthrottle(struct uart_port *port)
 {
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 	unsigned long flags;
 
 	spin_lock_irqsave(&port->lock, flags);
-	stm32_set_bits(port, USART_CR1, USART_CR1_RXNEIE);
+	stm32_set_bits(port, ofs->cr1, USART_CR1_RXNEIE);
 	spin_unlock_irqrestore(&port->lock, flags);
 }
 
 /* Receive stop */
 static void stm32_stop_rx(struct uart_port *port)
 {
-	stm32_clr_bits(port, USART_CR1, USART_CR1_RXNEIE);
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+
+	stm32_clr_bits(port, ofs->cr1, USART_CR1_RXNEIE);
 }
 
 /* Handle breaks - ignored by us */
@@ -318,26 +440,34 @@ static void stm32_break_ctl(struct uart_port *port, int break_state)
 
 static int stm32_startup(struct uart_port *port)
 {
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 	const char *name = to_platform_device(port->dev)->name;
 	u32 val;
 	int ret;
 
-	ret = request_irq(port->irq, stm32_interrupt, 0, name, port);
+	ret = request_threaded_irq(port->irq, stm32_interrupt,
+				   stm32_threaded_interrupt,
+				   IRQF_NO_SUSPEND, name, port);
 	if (ret)
 		return ret;
 
 	val = USART_CR1_RXNEIE | USART_CR1_TE | USART_CR1_RE;
-	stm32_set_bits(port, USART_CR1, val);
+	stm32_set_bits(port, ofs->cr1, val);
 
 	return 0;
 }
 
 static void stm32_shutdown(struct uart_port *port)
 {
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+	struct stm32_usart_config *cfg = &stm32_port->info->cfg;
 	u32 val;
 
 	val = USART_CR1_TXEIE | USART_CR1_RXNEIE | USART_CR1_TE | USART_CR1_RE;
-	stm32_set_bits(port, USART_CR1, val);
+	val |= BIT(cfg->uart_enable_bit);
+	stm32_clr_bits(port, ofs->cr1, val);
 
 	free_irq(port->irq, port);
 }
@@ -346,6 +476,8 @@ static void stm32_set_termios(struct uart_port *port, struct ktermios *termios,
 			    struct ktermios *old)
 {
 	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+	struct stm32_usart_config *cfg = &stm32_port->info->cfg;
 	unsigned int baud;
 	u32 usartdiv, mantissa, fraction, oversampling;
 	tcflag_t cflag = termios->c_cflag;
@@ -360,9 +492,10 @@ static void stm32_set_termios(struct uart_port *port, struct ktermios *termios,
 	spin_lock_irqsave(&port->lock, flags);
 
 	/* Stop serial port and reset value */
-	writel_relaxed(0, port->membase + USART_CR1);
+	writel_relaxed(0, port->membase + ofs->cr1);
 
-	cr1 = USART_CR1_TE | USART_CR1_RE | USART_CR1_UE | USART_CR1_RXNEIE;
+	cr1 = USART_CR1_TE | USART_CR1_RE | USART_CR1_RXNEIE;
+	cr1 |= BIT(cfg->uart_enable_bit);
 	cr2 = 0;
 	cr3 = 0;
 
@@ -371,8 +504,12 @@ static void stm32_set_termios(struct uart_port *port, struct ktermios *termios,
 
 	if (cflag & PARENB) {
 		cr1 |= USART_CR1_PCE;
-		if ((cflag & CSIZE) == CS8)
-			cr1 |= USART_CR1_M;
+		if ((cflag & CSIZE) == CS8) {
+			if (cfg->has_7bits_data)
+				cr1 |= USART_CR1_M0;
+			else
+				cr1 |= USART_CR1_M;
+		}
 	}
 
 	if (cflag & PARODD)
@@ -394,15 +531,15 @@ static void stm32_set_termios(struct uart_port *port, struct ktermios *termios,
 	 */
 	if (usartdiv < 16) {
 		oversampling = 8;
-		stm32_set_bits(port, USART_CR1, USART_CR1_OVER8);
+		stm32_set_bits(port, ofs->cr1, USART_CR1_OVER8);
 	} else {
 		oversampling = 16;
-		stm32_clr_bits(port, USART_CR1, USART_CR1_OVER8);
+		stm32_clr_bits(port, ofs->cr1, USART_CR1_OVER8);
 	}
 
 	mantissa = (usartdiv / oversampling) << USART_BRR_DIV_M_SHIFT;
 	fraction = usartdiv % oversampling;
-	writel_relaxed(mantissa | fraction, port->membase + USART_BRR);
+	writel_relaxed(mantissa | fraction, port->membase + ofs->brr);
 
 	uart_update_timeout(port, cflag, baud);
 
@@ -430,9 +567,12 @@ static void stm32_set_termios(struct uart_port *port, struct ktermios *termios,
 	if ((termios->c_cflag & CREAD) == 0)
 		port->ignore_status_mask |= USART_SR_DUMMY_RX;
 
-	writel_relaxed(cr3, port->membase + USART_CR3);
-	writel_relaxed(cr2, port->membase + USART_CR2);
-	writel_relaxed(cr1, port->membase + USART_CR1);
+	if (stm32_port->rx_ch)
+		cr3 |= USART_CR3_DMAR;
+
+	writel_relaxed(cr3, port->membase + ofs->cr3);
+	writel_relaxed(cr2, port->membase + ofs->cr2);
+	writel_relaxed(cr1, port->membase + ofs->cr1);
 
 	spin_unlock_irqrestore(&port->lock, flags);
 }
@@ -469,6 +609,8 @@ static void stm32_pm(struct uart_port *port, unsigned int state,
 {
 	struct stm32_port *stm32port = container_of(port,
 			struct stm32_port, port);
+	struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
+	struct stm32_usart_config *cfg = &stm32port->info->cfg;
 	unsigned long flags = 0;
 
 	switch (state) {
@@ -477,7 +619,7 @@ static void stm32_pm(struct uart_port *port, unsigned int state,
 		break;
 	case UART_PM_STATE_OFF:
 		spin_lock_irqsave(&port->lock, flags);
-		stm32_clr_bits(port, USART_CR1, USART_CR1_UE);
+		stm32_clr_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
 		spin_unlock_irqrestore(&port->lock, flags);
 		clk_disable_unprepare(stm32port->clk);
 		break;
@@ -539,8 +681,6 @@ static int stm32_init_port(struct stm32_port *stm32port,
 	if (!stm32port->port.uartclk)
 		ret = -EINVAL;
 
-	clk_disable_unprepare(stm32port->clk);
-
 	return ret;
 }
 
@@ -560,30 +700,162 @@ static struct stm32_port *stm32_of_get_stm32_port(struct platform_device *pdev)
 		return NULL;
 
 	stm32_ports[id].hw_flow_control = of_property_read_bool(np,
-							"auto-flow-control");
+							"st,hw-flow-ctrl");
 	stm32_ports[id].port.line = id;
 	return &stm32_ports[id];
 }
 
 #ifdef CONFIG_OF
 static const struct of_device_id stm32_match[] = {
-	{ .compatible = "st,stm32-usart", },
-	{ .compatible = "st,stm32-uart", },
+	{ .compatible = "st,stm32-usart", .data = &stm32f4_info},
+	{ .compatible = "st,stm32-uart", .data = &stm32f4_info},
+	{ .compatible = "st,stm32f7-usart", .data = &stm32f7_info},
+	{ .compatible = "st,stm32f7-uart", .data = &stm32f7_info},
 	{},
 };
 
 MODULE_DEVICE_TABLE(of, stm32_match);
 #endif
 
-static int stm32_serial_probe(struct platform_device *pdev)
+static int stm32_of_dma_rx_probe(struct stm32_port *stm32port,
+				 struct platform_device *pdev)
 {
+	struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
+	struct uart_port *port = &stm32port->port;
+	struct device *dev = &pdev->dev;
+	struct dma_slave_config config;
+	struct dma_async_tx_descriptor *desc = NULL;
+	dma_cookie_t cookie;
 	int ret;
+
+	/* Request DMA RX channel */
+	stm32port->rx_ch = dma_request_slave_channel(dev, "rx");
+	if (!stm32port->rx_ch) {
+		dev_info(dev, "rx dma alloc failed\n");
+		return -ENODEV;
+	}
+	stm32port->rx_buf = dma_alloc_coherent(&pdev->dev, RX_BUF_L,
+						 &stm32port->rx_dma_buf,
+						 GFP_KERNEL);
+	if (!stm32port->rx_buf) {
+		ret = -ENOMEM;
+		goto alloc_err;
+	}
+
+	/* Configure DMA channel */
+	memset(&config, 0, sizeof(config));
+	config.src_addr = port->mapbase + ofs->rdr;
+	config.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+
+	ret = dmaengine_slave_config(stm32port->rx_ch, &config);
+	if (ret < 0) {
+		dev_err(dev, "rx dma channel config failed\n");
+		ret = -ENODEV;
+		goto config_err;
+	}
+
+	/* Prepare a DMA cyclic transaction */
+	desc = dmaengine_prep_dma_cyclic(stm32port->rx_ch,
+					 stm32port->rx_dma_buf,
+					 RX_BUF_L, RX_BUF_P, DMA_DEV_TO_MEM,
+					 DMA_PREP_INTERRUPT);
+	if (!desc) {
+		dev_err(dev, "rx dma prep cyclic failed\n");
+		ret = -ENODEV;
+		goto config_err;
+	}
+
+	/* No callback as dma buffer is drained on usart interrupt */
+	desc->callback = NULL;
+	desc->callback_param = NULL;
+
+	/* Push current DMA transaction in the pending queue */
+	cookie = dmaengine_submit(desc);
+
+	/* Issue pending DMA requests */
+	dma_async_issue_pending(stm32port->rx_ch);
+
+	return 0;
+
+config_err:
+	dma_free_coherent(&pdev->dev,
+			  RX_BUF_L, stm32port->rx_buf,
+			  stm32port->rx_dma_buf);
+
+alloc_err:
+	dma_release_channel(stm32port->rx_ch);
+	stm32port->rx_ch = NULL;
+
+	return ret;
+}
+
+static int stm32_of_dma_tx_probe(struct stm32_port *stm32port,
+				 struct platform_device *pdev)
+{
+	struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
+	struct uart_port *port = &stm32port->port;
+	struct device *dev = &pdev->dev;
+	struct dma_slave_config config;
+	int ret;
+
+	stm32port->tx_dma_busy = false;
+
+	/* Request DMA TX channel */
+	stm32port->tx_ch = dma_request_slave_channel(dev, "tx");
+	if (!stm32port->tx_ch) {
+		dev_info(dev, "tx dma alloc failed\n");
+		return -ENODEV;
+	}
+	stm32port->tx_buf = dma_alloc_coherent(&pdev->dev, TX_BUF_L,
+						 &stm32port->tx_dma_buf,
+						 GFP_KERNEL);
+	if (!stm32port->tx_buf) {
+		ret = -ENOMEM;
+		goto alloc_err;
+	}
+
+	/* Configure DMA channel */
+	memset(&config, 0, sizeof(config));
+	config.dst_addr = port->mapbase + ofs->tdr;
+	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+
+	ret = dmaengine_slave_config(stm32port->tx_ch, &config);
+	if (ret < 0) {
+		dev_err(dev, "tx dma channel config failed\n");
+		ret = -ENODEV;
+		goto config_err;
+	}
+
+	return 0;
+
+config_err:
+	dma_free_coherent(&pdev->dev,
+			  TX_BUF_L, stm32port->tx_buf,
+			  stm32port->tx_dma_buf);
+
+alloc_err:
+	dma_release_channel(stm32port->tx_ch);
+	stm32port->tx_ch = NULL;
+
+	return ret;
+}
+
+static int stm32_serial_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *match;
 	struct stm32_port *stm32port;
+	int ret;
 
 	stm32port = stm32_of_get_stm32_port(pdev);
 	if (!stm32port)
 		return -ENODEV;
 
+	match = of_match_device(stm32_match, &pdev->dev);
+	if (match && match->data)
+		stm32port->info = (struct stm32_usart_info *)match->data;
+	else
+		return -EINVAL;
+
 	ret = stm32_init_port(stm32port, pdev);
 	if (ret)
 		return ret;
@@ -592,6 +864,14 @@ static int stm32_serial_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	ret = stm32_of_dma_rx_probe(stm32port, pdev);
+	if (ret)
+		dev_info(&pdev->dev, "interrupt mode used for rx (no dma)\n");
+
+	ret = stm32_of_dma_tx_probe(stm32port, pdev);
+	if (ret)
+		dev_info(&pdev->dev, "interrupt mode used for tx (no dma)\n");
+
 	platform_set_drvdata(pdev, &stm32port->port);
 
 	return 0;
@@ -600,6 +880,30 @@ static int stm32_serial_probe(struct platform_device *pdev)
 static int stm32_serial_remove(struct platform_device *pdev)
 {
 	struct uart_port *port = platform_get_drvdata(pdev);
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+
+	stm32_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
+
+	if (stm32_port->rx_ch)
+		dma_release_channel(stm32_port->rx_ch);
+
+	if (stm32_port->rx_dma_buf)
+		dma_free_coherent(&pdev->dev,
+				  RX_BUF_L, stm32_port->rx_buf,
+				  stm32_port->rx_dma_buf);
+
+	stm32_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
+
+	if (stm32_port->tx_ch)
+		dma_release_channel(stm32_port->tx_ch);
+
+	if (stm32_port->tx_dma_buf)
+		dma_free_coherent(&pdev->dev,
+				  TX_BUF_L, stm32_port->tx_buf,
+				  stm32_port->tx_dma_buf);
+
+	clk_disable_unprepare(stm32_port->clk);
 
 	return uart_remove_one_port(&stm32_usart_driver, port);
 }
@@ -608,15 +912,21 @@ static int stm32_serial_remove(struct platform_device *pdev)
 #ifdef CONFIG_SERIAL_STM32_CONSOLE
 static void stm32_console_putchar(struct uart_port *port, int ch)
 {
-	while (!(readl_relaxed(port->membase + USART_SR) & USART_SR_TXE))
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+
+	while (!(readl_relaxed(port->membase + ofs->isr) & USART_SR_TXE))
 		cpu_relax();
 
-	writel_relaxed(ch, port->membase + USART_DR);
+	writel_relaxed(ch, port->membase + ofs->tdr);
 }
 
 static void stm32_console_write(struct console *co, const char *s, unsigned cnt)
 {
 	struct uart_port *port = &stm32_ports[co->index].port;
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+	struct stm32_usart_config *cfg = &stm32_port->info->cfg;
 	unsigned long flags;
 	u32 old_cr1, new_cr1;
 	int locked = 1;
@@ -629,15 +939,16 @@ static void stm32_console_write(struct console *co, const char *s, unsigned cnt)
 	else
 		spin_lock(&port->lock);
 
-	/* Save and disable interrupts */
-	old_cr1 = readl_relaxed(port->membase + USART_CR1);
+	/* Save and disable interrupts, enable the transmitter */
+	old_cr1 = readl_relaxed(port->membase + ofs->cr1);
 	new_cr1 = old_cr1 & ~USART_CR1_IE_MASK;
-	writel_relaxed(new_cr1, port->membase + USART_CR1);
+	new_cr1 |=  USART_CR1_TE | BIT(cfg->uart_enable_bit);
+	writel_relaxed(new_cr1, port->membase + ofs->cr1);
 
 	uart_console_write(port, s, cnt, stm32_console_putchar);
 
 	/* Restore interrupt state */
-	writel_relaxed(old_cr1, port->membase + USART_CR1);
+	writel_relaxed(old_cr1, port->membase + ofs->cr1);
 
 	if (locked)
 		spin_unlock(&port->lock);
diff --git a/drivers/tty/serial/stm32-usart.h b/drivers/tty/serial/stm32-usart.h
new file mode 100644
index 000000000000..41d974923102
--- /dev/null
+++ b/drivers/tty/serial/stm32-usart.h
@@ -0,0 +1,229 @@
+/*
+ * Copyright (C) Maxime Coquelin 2015
+ * Authors:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
+ *	     Gerald Baeza <gerald_baeza@yahoo.fr>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#define DRIVER_NAME "stm32-usart"
+
+struct stm32_usart_offsets {
+	u8 cr1;
+	u8 cr2;
+	u8 cr3;
+	u8 brr;
+	u8 gtpr;
+	u8 rtor;
+	u8 rqr;
+	u8 isr;
+	u8 icr;
+	u8 rdr;
+	u8 tdr;
+};
+
+struct stm32_usart_config {
+	u8 uart_enable_bit; /* USART_CR1_UE */
+	bool has_7bits_data;
+};
+
+struct stm32_usart_info {
+	struct stm32_usart_offsets ofs;
+	struct stm32_usart_config cfg;
+};
+
+#define UNDEF_REG ~0
+
+/* Register offsets */
+struct stm32_usart_info stm32f4_info = {
+	.ofs = {
+		.isr	= 0x00,
+		.rdr	= 0x04,
+		.tdr	= 0x04,
+		.brr	= 0x08,
+		.cr1	= 0x0c,
+		.cr2	= 0x10,
+		.cr3	= 0x14,
+		.gtpr	= 0x18,
+		.rtor	= UNDEF_REG,
+		.rqr	= UNDEF_REG,
+		.icr	= UNDEF_REG,
+	},
+	.cfg = {
+		.uart_enable_bit = 13,
+		.has_7bits_data = false,
+	}
+};
+
+struct stm32_usart_info stm32f7_info = {
+	.ofs = {
+		.cr1	= 0x00,
+		.cr2	= 0x04,
+		.cr3	= 0x08,
+		.brr	= 0x0c,
+		.gtpr	= 0x10,
+		.rtor	= 0x14,
+		.rqr	= 0x18,
+		.isr	= 0x1c,
+		.icr	= 0x20,
+		.rdr	= 0x24,
+		.tdr	= 0x28,
+	},
+	.cfg = {
+		.uart_enable_bit = 0,
+		.has_7bits_data = true,
+	}
+};
+
+/* USART_SR (F4) / USART_ISR (F7) */
+#define USART_SR_PE		BIT(0)
+#define USART_SR_FE		BIT(1)
+#define USART_SR_NF		BIT(2)
+#define USART_SR_ORE		BIT(3)
+#define USART_SR_IDLE		BIT(4)
+#define USART_SR_RXNE		BIT(5)
+#define USART_SR_TC		BIT(6)
+#define USART_SR_TXE		BIT(7)
+#define USART_SR_LBD		BIT(8)
+#define USART_SR_CTSIF		BIT(9)
+#define USART_SR_CTS		BIT(10)		/* F7 */
+#define USART_SR_RTOF		BIT(11)		/* F7 */
+#define USART_SR_EOBF		BIT(12)		/* F7 */
+#define USART_SR_ABRE		BIT(14)		/* F7 */
+#define USART_SR_ABRF		BIT(15)		/* F7 */
+#define USART_SR_BUSY		BIT(16)		/* F7 */
+#define USART_SR_CMF		BIT(17)		/* F7 */
+#define USART_SR_SBKF		BIT(18)		/* F7 */
+#define USART_SR_TEACK		BIT(21)		/* F7 */
+#define USART_SR_ERR_MASK	(USART_SR_LBD | USART_SR_ORE | \
+				 USART_SR_FE | USART_SR_PE)
+/* Dummy bits */
+#define USART_SR_DUMMY_RX	BIT(16)
+
+/* USART_ICR (F7) */
+#define USART_CR_TC		BIT(6)
+
+/* USART_DR */
+#define USART_DR_MASK		GENMASK(8, 0)
+
+/* USART_BRR */
+#define USART_BRR_DIV_F_MASK	GENMASK(3, 0)
+#define USART_BRR_DIV_M_MASK	GENMASK(15, 4)
+#define USART_BRR_DIV_M_SHIFT	4
+
+/* USART_CR1 */
+#define USART_CR1_SBK		BIT(0)
+#define USART_CR1_RWU		BIT(1)		/* F4 */
+#define USART_CR1_RE		BIT(2)
+#define USART_CR1_TE		BIT(3)
+#define USART_CR1_IDLEIE	BIT(4)
+#define USART_CR1_RXNEIE	BIT(5)
+#define USART_CR1_TCIE		BIT(6)
+#define USART_CR1_TXEIE		BIT(7)
+#define USART_CR1_PEIE		BIT(8)
+#define USART_CR1_PS		BIT(9)
+#define USART_CR1_PCE		BIT(10)
+#define USART_CR1_WAKE		BIT(11)
+#define USART_CR1_M		BIT(12)
+#define USART_CR1_M0		BIT(12)		/* F7 */
+#define USART_CR1_MME		BIT(13)		/* F7 */
+#define USART_CR1_CMIE		BIT(14)		/* F7 */
+#define USART_CR1_OVER8		BIT(15)
+#define USART_CR1_DEDT_MASK	GENMASK(20, 16)	/* F7 */
+#define USART_CR1_DEAT_MASK	GENMASK(25, 21)	/* F7 */
+#define USART_CR1_RTOIE		BIT(26)		/* F7 */
+#define USART_CR1_EOBIE		BIT(27)		/* F7 */
+#define USART_CR1_M1		BIT(28)		/* F7 */
+#define USART_CR1_IE_MASK	(GENMASK(8, 4) | BIT(14) | BIT(26) | BIT(27))
+
+/* USART_CR2 */
+#define USART_CR2_ADD_MASK	GENMASK(3, 0)	/* F4 */
+#define USART_CR2_ADDM7		BIT(4)		/* F7 */
+#define USART_CR2_LBDL		BIT(5)
+#define USART_CR2_LBDIE		BIT(6)
+#define USART_CR2_LBCL		BIT(8)
+#define USART_CR2_CPHA		BIT(9)
+#define USART_CR2_CPOL		BIT(10)
+#define USART_CR2_CLKEN		BIT(11)
+#define USART_CR2_STOP_2B	BIT(13)
+#define USART_CR2_STOP_MASK	GENMASK(13, 12)
+#define USART_CR2_LINEN		BIT(14)
+#define USART_CR2_SWAP		BIT(15)		/* F7 */
+#define USART_CR2_RXINV		BIT(16)		/* F7 */
+#define USART_CR2_TXINV		BIT(17)		/* F7 */
+#define USART_CR2_DATAINV	BIT(18)		/* F7 */
+#define USART_CR2_MSBFIRST	BIT(19)		/* F7 */
+#define USART_CR2_ABREN		BIT(20)		/* F7 */
+#define USART_CR2_ABRMOD_MASK	GENMASK(22, 21)	/* F7 */
+#define USART_CR2_RTOEN		BIT(23)		/* F7 */
+#define USART_CR2_ADD_F7_MASK	GENMASK(31, 24)	/* F7 */
+
+/* USART_CR3 */
+#define USART_CR3_EIE		BIT(0)
+#define USART_CR3_IREN		BIT(1)
+#define USART_CR3_IRLP		BIT(2)
+#define USART_CR3_HDSEL		BIT(3)
+#define USART_CR3_NACK		BIT(4)
+#define USART_CR3_SCEN		BIT(5)
+#define USART_CR3_DMAR		BIT(6)
+#define USART_CR3_DMAT		BIT(7)
+#define USART_CR3_RTSE		BIT(8)
+#define USART_CR3_CTSE		BIT(9)
+#define USART_CR3_CTSIE		BIT(10)
+#define USART_CR3_ONEBIT	BIT(11)
+#define USART_CR3_OVRDIS	BIT(12)		/* F7 */
+#define USART_CR3_DDRE		BIT(13)		/* F7 */
+#define USART_CR3_DEM		BIT(14)		/* F7 */
+#define USART_CR3_DEP		BIT(15)		/* F7 */
+#define USART_CR3_SCARCNT_MASK	GENMASK(19, 17)	/* F7 */
+
+/* USART_GTPR */
+#define USART_GTPR_PSC_MASK	GENMASK(7, 0)
+#define USART_GTPR_GT_MASK	GENMASK(15, 8)
+
+/* USART_RTOR */
+#define USART_RTOR_RTO_MASK	GENMASK(23, 0)	/* F7 */
+#define USART_RTOR_BLEN_MASK	GENMASK(31, 24)	/* F7 */
+
+/* USART_RQR */
+#define USART_RQR_ABRRQ		BIT(0)		/* F7 */
+#define USART_RQR_SBKRQ		BIT(1)		/* F7 */
+#define USART_RQR_MMRQ		BIT(2)		/* F7 */
+#define USART_RQR_RXFRQ		BIT(3)		/* F7 */
+#define USART_RQR_TXFRQ		BIT(4)		/* F7 */
+
+/* USART_ICR */
+#define USART_ICR_PECF		BIT(0)		/* F7 */
+#define USART_ICR_FFECF		BIT(1)		/* F7 */
+#define USART_ICR_NCF		BIT(2)		/* F7 */
+#define USART_ICR_ORECF		BIT(3)		/* F7 */
+#define USART_ICR_IDLECF	BIT(4)		/* F7 */
+#define USART_ICR_TCCF		BIT(6)		/* F7 */
+#define USART_ICR_LBDCF		BIT(8)		/* F7 */
+#define USART_ICR_CTSCF		BIT(9)		/* F7 */
+#define USART_ICR_RTOCF		BIT(11)		/* F7 */
+#define USART_ICR_EOBCF		BIT(12)		/* F7 */
+#define USART_ICR_CMCF		BIT(17)		/* F7 */
+
+#define STM32_SERIAL_NAME "ttyS"
+#define STM32_MAX_PORTS 6
+
+#define RX_BUF_L 200		 /* dma rx buffer length     */
+#define RX_BUF_P RX_BUF_L	 /* dma rx buffer period     */
+#define TX_BUF_L 200		 /* dma tx buffer length     */
+
+struct stm32_port {
+	struct uart_port port;
+	struct clk *clk;
+	struct stm32_usart_info *info;
+	struct dma_chan *rx_ch;  /* dma rx channel            */
+	dma_addr_t rx_dma_buf;   /* dma rx buffer bus address */
+	unsigned char *rx_buf;   /* dma rx buffer cpu address */
+	struct dma_chan *tx_ch;  /* dma tx channel            */
+	dma_addr_t tx_dma_buf;   /* dma tx buffer bus address */
+	unsigned char *tx_buf;   /* dma tx buffer cpu address */
+	bool tx_dma_busy;	 /* dma tx busy               */
+	bool hw_flow_control;
+};
+
+static struct stm32_port stm32_ports[STM32_MAX_PORTS];
+static struct uart_driver stm32_usart_driver;
diff --git a/drivers/tty/serial/timbuart.c b/drivers/tty/serial/timbuart.c
index 512c162634a3..5da7fe40e391 100644
--- a/drivers/tty/serial/timbuart.c
+++ b/drivers/tty/serial/timbuart.c
@@ -394,7 +394,7 @@ static int timbuart_verify_port(struct uart_port *port,
 	return -EINVAL;
 }
 
-static struct uart_ops timbuart_ops = {
+static const struct uart_ops timbuart_ops = {
 	.tx_empty = timbuart_tx_empty,
 	.set_mctrl = timbuart_set_mctrl,
 	.get_mctrl = timbuart_get_mctrl,
diff --git a/drivers/tty/serial/uartlite.c b/drivers/tty/serial/uartlite.c
index 05089b6c2f30..817bb0d3f326 100644
--- a/drivers/tty/serial/uartlite.c
+++ b/drivers/tty/serial/uartlite.c
@@ -387,7 +387,7 @@ static void ulite_put_poll_char(struct uart_port *port, unsigned char ch)
 }
 #endif
 
-static struct uart_ops ulite_ops = {
+static const struct uart_ops ulite_ops = {
 	.tx_empty	= ulite_tx_empty,
 	.set_mctrl	= ulite_set_mctrl,
 	.get_mctrl	= ulite_get_mctrl,
diff --git a/drivers/tty/serial/vt8500_serial.c b/drivers/tty/serial/vt8500_serial.c
index 23cfc5e16b45..6b85adce0ac9 100644
--- a/drivers/tty/serial/vt8500_serial.c
+++ b/drivers/tty/serial/vt8500_serial.c
@@ -118,7 +118,7 @@ struct vt8500_port {
  * have been allocated as we can't use pdev->id in
  * devicetree
  */
-static unsigned long vt8500_ports_in_use;
+static DECLARE_BITMAP(vt8500_ports_in_use, VT8500_MAX_PORTS);
 
 static inline void vt8500_write(struct uart_port *port, unsigned int val,
 			     unsigned int off)
@@ -663,15 +663,15 @@ static int vt8500_serial_probe(struct platform_device *pdev)
 
 	if (port < 0) {
 		/* calculate the port id */
-		port = find_first_zero_bit(&vt8500_ports_in_use,
-					sizeof(vt8500_ports_in_use));
+		port = find_first_zero_bit(vt8500_ports_in_use,
+					   VT8500_MAX_PORTS);
 	}
 
 	if (port >= VT8500_MAX_PORTS)
 		return -ENODEV;
 
 	/* reserve the port id */
-	if (test_and_set_bit(port, &vt8500_ports_in_use)) {
+	if (test_and_set_bit(port, vt8500_ports_in_use)) {
 		/* port already in use - shouldn't really happen */
 		return -EBUSY;
 	}
diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c
index 9ca1a4d1b66a..f37edaa5ac75 100644
--- a/drivers/tty/serial/xilinx_uartps.c
+++ b/drivers/tty/serial/xilinx_uartps.c
@@ -57,7 +57,7 @@ MODULE_PARM_DESC(rx_timeout, "Rx timeout, 1-255");
 #define CDNS_UART_IMR		0x10  /* Interrupt Mask */
 #define CDNS_UART_ISR		0x14  /* Interrupt Status */
 #define CDNS_UART_BAUDGEN	0x18  /* Baud Rate Generator */
-#define CDNS_UART_RXTOUT		0x1C  /* RX Timeout */
+#define CDNS_UART_RXTOUT	0x1C  /* RX Timeout */
 #define CDNS_UART_RXWM		0x20  /* RX FIFO Trigger Level */
 #define CDNS_UART_MODEMCR	0x24  /* Modem Control */
 #define CDNS_UART_MODEMSR	0x28  /* Modem Status */
@@ -68,6 +68,7 @@ MODULE_PARM_DESC(rx_timeout, "Rx timeout, 1-255");
 #define CDNS_UART_IRRX_PWIDTH	0x3C  /* IR Min Received Pulse Width */
 #define CDNS_UART_IRTX_PWIDTH	0x40  /* IR Transmitted pulse Width */
 #define CDNS_UART_TXWM		0x44  /* TX FIFO Trigger Level */
+#define CDNS_UART_RXBS		0x48  /* RX FIFO byte status register */
 
 /* Control Register Bit Definitions */
 #define CDNS_UART_CR_STOPBRK	0x00000100  /* Stop TX break */
@@ -79,6 +80,9 @@ MODULE_PARM_DESC(rx_timeout, "Rx timeout, 1-255");
 #define CDNS_UART_CR_TXRST	0x00000002  /* TX logic reset */
 #define CDNS_UART_CR_RXRST	0x00000001  /* RX logic reset */
 #define CDNS_UART_CR_RST_TO	0x00000040  /* Restart Timeout Counter */
+#define CDNS_UART_RXBS_PARITY    0x00000001 /* Parity error status */
+#define CDNS_UART_RXBS_FRAMING   0x00000002 /* Framing error status */
+#define CDNS_UART_RXBS_BRK       0x00000004 /* Overrun error status */
 
 /*
  * Mode Register:
@@ -126,13 +130,27 @@ MODULE_PARM_DESC(rx_timeout, "Rx timeout, 1-255");
 #define CDNS_UART_IXR_RXEMPTY	0x00000002 /* RX FIFO empty interrupt. */
 #define CDNS_UART_IXR_MASK	0x00001FFF /* Valid bit mask */
 
-#define CDNS_UART_RX_IRQS	(CDNS_UART_IXR_PARITY | CDNS_UART_IXR_FRAMING | \
-				 CDNS_UART_IXR_OVERRUN | CDNS_UART_IXR_RXTRIG | \
+	/*
+	 * Do not enable parity error interrupt for the following
+	 * reason: When parity error interrupt is enabled, each Rx
+	 * parity error always results in 2 events. The first one
+	 * being parity error interrupt and the second one with a
+	 * proper Rx interrupt with the incoming data.  Disabling
+	 * parity error interrupt ensures better handling of parity
+	 * error events. With this change, for a parity error case, we
+	 * get a Rx interrupt with parity error set in ISR register
+	 * and we still handle parity errors in the desired way.
+	 */
+
+#define CDNS_UART_RX_IRQS	(CDNS_UART_IXR_FRAMING | \
+				 CDNS_UART_IXR_OVERRUN | \
+				 CDNS_UART_IXR_RXTRIG |	 \
 				 CDNS_UART_IXR_TOUT)
 
 /* Goes in read_status_mask for break detection as the HW doesn't do it*/
-#define CDNS_UART_IXR_BRK	0x80000000
+#define CDNS_UART_IXR_BRK	0x00002000
 
+#define CDNS_UART_RXBS_SUPPORT BIT(1)
 /*
  * Modem Control register:
  * The read/write Modem Control register controls the interface with the modem
@@ -172,46 +190,66 @@ struct cdns_uart {
 	struct clk		*pclk;
 	unsigned int		baud;
 	struct notifier_block	clk_rate_change_nb;
+	u32			quirks;
+};
+struct cdns_platform_data {
+	u32 quirks;
 };
 #define to_cdns_uart(_nb) container_of(_nb, struct cdns_uart, \
 		clk_rate_change_nb);
 
-static void cdns_uart_handle_rx(struct uart_port *port, unsigned int isrstatus)
+/**
+ * cdns_uart_handle_rx - Handle the received bytes along with Rx errors.
+ * @dev_id: Id of the UART port
+ * @isrstatus: The interrupt status register value as read
+ * Return: None
+ */
+static void cdns_uart_handle_rx(void *dev_id, unsigned int isrstatus)
 {
-	/*
-	 * There is no hardware break detection, so we interpret framing
-	 * error with all-zeros data as a break sequence. Most of the time,
-	 * there's another non-zero byte at the end of the sequence.
-	 */
-	if (isrstatus & CDNS_UART_IXR_FRAMING) {
-		while (!(readl(port->membase + CDNS_UART_SR) &
-					CDNS_UART_SR_RXEMPTY)) {
-			if (!readl(port->membase + CDNS_UART_FIFO)) {
+	struct uart_port *port = (struct uart_port *)dev_id;
+	struct cdns_uart *cdns_uart = port->private_data;
+	unsigned int data;
+	unsigned int rxbs_status = 0;
+	unsigned int status_mask;
+	unsigned int framerrprocessed = 0;
+	char status = TTY_NORMAL;
+	bool is_rxbs_support;
+
+	is_rxbs_support = cdns_uart->quirks & CDNS_UART_RXBS_SUPPORT;
+
+	while ((readl(port->membase + CDNS_UART_SR) &
+		CDNS_UART_SR_RXEMPTY) != CDNS_UART_SR_RXEMPTY) {
+		if (is_rxbs_support)
+			rxbs_status = readl(port->membase + CDNS_UART_RXBS);
+		data = readl(port->membase + CDNS_UART_FIFO);
+		port->icount.rx++;
+		/*
+		 * There is no hardware break detection in Zynq, so we interpret
+		 * framing error with all-zeros data as a break sequence.
+		 * Most of the time, there's another non-zero byte at the
+		 * end of the sequence.
+		 */
+		if (!is_rxbs_support && (isrstatus & CDNS_UART_IXR_FRAMING)) {
+			if (!data) {
 				port->read_status_mask |= CDNS_UART_IXR_BRK;
-				isrstatus &= ~CDNS_UART_IXR_FRAMING;
+				framerrprocessed = 1;
+				continue;
 			}
 		}
-		writel(CDNS_UART_IXR_FRAMING, port->membase + CDNS_UART_ISR);
-	}
-
-	/* drop byte with parity error if IGNPAR specified */
-	if (isrstatus & port->ignore_status_mask & CDNS_UART_IXR_PARITY)
-		isrstatus &= ~(CDNS_UART_IXR_RXTRIG | CDNS_UART_IXR_TOUT);
-
-	isrstatus &= port->read_status_mask;
-	isrstatus &= ~port->ignore_status_mask;
-
-	if (!(isrstatus & (CDNS_UART_IXR_TOUT | CDNS_UART_IXR_RXTRIG)))
-		return;
-
-	while (!(readl(port->membase + CDNS_UART_SR) & CDNS_UART_SR_RXEMPTY)) {
-		u32 data;
-		char status = TTY_NORMAL;
+		if (is_rxbs_support && (rxbs_status & CDNS_UART_RXBS_BRK)) {
+			port->icount.brk++;
+			status = TTY_BREAK;
+			if (uart_handle_break(port))
+				continue;
+		}
 
-		data = readl(port->membase + CDNS_UART_FIFO);
+		isrstatus &= port->read_status_mask;
+		isrstatus &= ~port->ignore_status_mask;
+		status_mask = port->read_status_mask;
+		status_mask &= ~port->ignore_status_mask;
 
-		/* Non-NULL byte after BREAK is garbage (99%) */
-		if (data && (port->read_status_mask & CDNS_UART_IXR_BRK)) {
+		if (data &&
+		    (port->read_status_mask & CDNS_UART_IXR_BRK)) {
 			port->read_status_mask &= ~CDNS_UART_IXR_BRK;
 			port->icount.brk++;
 			if (uart_handle_break(port))
@@ -221,57 +259,83 @@ static void cdns_uart_handle_rx(struct uart_port *port, unsigned int isrstatus)
 		if (uart_handle_sysrq_char(port, data))
 			continue;
 
-		port->icount.rx++;
-
-		if (isrstatus & CDNS_UART_IXR_PARITY) {
-			port->icount.parity++;
-			status = TTY_PARITY;
-		} else if (isrstatus & CDNS_UART_IXR_FRAMING) {
-			port->icount.frame++;
-			status = TTY_FRAME;
-		} else if (isrstatus & CDNS_UART_IXR_OVERRUN) {
+		if (is_rxbs_support) {
+			if ((rxbs_status & CDNS_UART_RXBS_PARITY)
+			    && (status_mask & CDNS_UART_IXR_PARITY)) {
+				port->icount.parity++;
+				status = TTY_PARITY;
+			}
+			if ((rxbs_status & CDNS_UART_RXBS_FRAMING)
+			    && (status_mask & CDNS_UART_IXR_PARITY)) {
+				port->icount.frame++;
+				status = TTY_FRAME;
+			}
+		} else {
+			if (isrstatus & CDNS_UART_IXR_PARITY) {
+				port->icount.parity++;
+				status = TTY_PARITY;
+			}
+			if ((isrstatus & CDNS_UART_IXR_FRAMING) &&
+			    !framerrprocessed) {
+				port->icount.frame++;
+				status = TTY_FRAME;
+			}
+		}
+		if (isrstatus & CDNS_UART_IXR_OVERRUN) {
 			port->icount.overrun++;
+			tty_insert_flip_char(&port->state->port, 0,
+					     TTY_OVERRUN);
 		}
-
-		uart_insert_char(port, isrstatus, CDNS_UART_IXR_OVERRUN,
-				 data, status);
+		tty_insert_flip_char(&port->state->port, data, status);
+		isrstatus = 0;
 	}
+	spin_unlock(&port->lock);
 	tty_flip_buffer_push(&port->state->port);
+	spin_lock(&port->lock);
 }
 
-static void cdns_uart_handle_tx(struct uart_port *port)
+/**
+ * cdns_uart_handle_tx - Handle the bytes to be Txed.
+ * @dev_id: Id of the UART port
+ * Return: None
+ */
+static void cdns_uart_handle_tx(void *dev_id)
 {
+	struct uart_port *port = (struct uart_port *)dev_id;
 	unsigned int numbytes;
 
 	if (uart_circ_empty(&port->state->xmit)) {
 		writel(CDNS_UART_IXR_TXEMPTY, port->membase + CDNS_UART_IDR);
-		return;
-	}
-
-	numbytes = port->fifosize;
-	while (numbytes && !uart_circ_empty(&port->state->xmit) &&
-	       !(readl(port->membase + CDNS_UART_SR) & CDNS_UART_SR_TXFULL)) {
-		/*
-		 * Get the data from the UART circular buffer
-		 * and write it to the cdns_uart's TX_FIFO
-		 * register.
-		 */
-		writel(port->state->xmit.buf[port->state->xmit.tail],
-			port->membase + CDNS_UART_FIFO);
-		port->icount.tx++;
-
-		/*
-		 * Adjust the tail of the UART buffer and wrap
-		 * the buffer if it reaches limit.
-		 */
-		port->state->xmit.tail =
-			(port->state->xmit.tail + 1) & (UART_XMIT_SIZE - 1);
+	} else {
+		numbytes = port->fifosize;
+		while (numbytes && !uart_circ_empty(&port->state->xmit) &&
+		       !(readl(port->membase + CDNS_UART_SR) & CDNS_UART_SR_TXFULL)) {
+			/*
+			 * Get the data from the UART circular buffer
+			 * and write it to the cdns_uart's TX_FIFO
+			 * register.
+			 */
+			writel(
+				port->state->xmit.buf[port->state->xmit.
+				tail], port->membase + CDNS_UART_FIFO);
+
+			port->icount.tx++;
+
+			/*
+			 * Adjust the tail of the UART buffer and wrap
+			 * the buffer if it reaches limit.
+			 */
+			port->state->xmit.tail =
+				(port->state->xmit.tail + 1) &
+					(UART_XMIT_SIZE - 1);
+
+			numbytes--;
+		}
 
-		numbytes--;
+		if (uart_circ_chars_pending(
+				&port->state->xmit) < WAKEUP_CHARS)
+			uart_write_wakeup(port);
 	}
-
-	if (uart_circ_chars_pending(&port->state->xmit) < WAKEUP_CHARS)
-		uart_write_wakeup(port);
 }
 
 /**
@@ -284,27 +348,24 @@ static void cdns_uart_handle_tx(struct uart_port *port)
 static irqreturn_t cdns_uart_isr(int irq, void *dev_id)
 {
 	struct uart_port *port = (struct uart_port *)dev_id;
-	unsigned long flags;
 	unsigned int isrstatus;
 
-	spin_lock_irqsave(&port->lock, flags);
+	spin_lock(&port->lock);
 
 	/* Read the interrupt status register to determine which
-	 * interrupt(s) is/are active.
+	 * interrupt(s) is/are active and clear them.
 	 */
 	isrstatus = readl(port->membase + CDNS_UART_ISR);
-
-	if (isrstatus & CDNS_UART_RX_IRQS)
-		cdns_uart_handle_rx(port, isrstatus);
-
-	if ((isrstatus & CDNS_UART_IXR_TXEMPTY) == CDNS_UART_IXR_TXEMPTY)
-		cdns_uart_handle_tx(port);
-
 	writel(isrstatus, port->membase + CDNS_UART_ISR);
 
-	/* be sure to release the lock and tty before leaving */
-	spin_unlock_irqrestore(&port->lock, flags);
+	if (isrstatus & CDNS_UART_IXR_TXEMPTY) {
+		cdns_uart_handle_tx(dev_id);
+		isrstatus &= ~CDNS_UART_IXR_TXEMPTY;
+	}
+	if (isrstatus & CDNS_UART_IXR_MASK)
+		cdns_uart_handle_rx(dev_id, isrstatus);
 
+	spin_unlock(&port->lock);
 	return IRQ_HANDLED;
 }
 
@@ -653,6 +714,10 @@ static void cdns_uart_set_termios(struct uart_port *port,
 	ctrl_reg |= CDNS_UART_CR_TXRST | CDNS_UART_CR_RXRST;
 	writel(ctrl_reg, port->membase + CDNS_UART_CR);
 
+	while (readl(port->membase + CDNS_UART_CR) &
+		(CDNS_UART_CR_TXRST | CDNS_UART_CR_RXRST))
+		cpu_relax();
+
 	/*
 	 * Clear the RX disable and TX disable bits and then set the TX enable
 	 * bit and RX enable bit to enable the transmitter and receiver.
@@ -736,10 +801,14 @@ static void cdns_uart_set_termios(struct uart_port *port,
  */
 static int cdns_uart_startup(struct uart_port *port)
 {
+	struct cdns_uart *cdns_uart = port->private_data;
+	bool is_brk_support;
 	int ret;
 	unsigned long flags;
 	unsigned int status = 0;
 
+	is_brk_support = cdns_uart->quirks & CDNS_UART_RXBS_SUPPORT;
+
 	spin_lock_irqsave(&port->lock, flags);
 
 	/* Disable the TX and RX */
@@ -752,6 +821,10 @@ static int cdns_uart_startup(struct uart_port *port)
 	writel(CDNS_UART_CR_TXRST | CDNS_UART_CR_RXRST,
 			port->membase + CDNS_UART_CR);
 
+	while (readl(port->membase + CDNS_UART_CR) &
+		(CDNS_UART_CR_TXRST | CDNS_UART_CR_RXRST))
+		cpu_relax();
+
 	/*
 	 * Clear the RX disable bit and then set the RX enable bit to enable
 	 * the receiver.
@@ -794,7 +867,11 @@ static int cdns_uart_startup(struct uart_port *port)
 	}
 
 	/* Set the Interrupt Registers with desired interrupts */
-	writel(CDNS_UART_RX_IRQS, port->membase + CDNS_UART_IER);
+	if (is_brk_support)
+		writel(CDNS_UART_RX_IRQS | CDNS_UART_IXR_BRK,
+					port->membase + CDNS_UART_IER);
+	else
+		writel(CDNS_UART_RX_IRQS, port->membase + CDNS_UART_IER);
 
 	return 0;
 }
@@ -993,7 +1070,7 @@ static void cdns_uart_pm(struct uart_port *port, unsigned int state,
 	}
 }
 
-static struct uart_ops cdns_uart_ops = {
+static const struct uart_ops cdns_uart_ops = {
 	.set_mctrl	= cdns_uart_set_mctrl,
 	.get_mctrl	= cdns_uart_get_mctrl,
 	.start_tx	= cdns_uart_start_tx,
@@ -1088,9 +1165,34 @@ static void __init cdns_early_write(struct console *con, const char *s,
 static int __init cdns_early_console_setup(struct earlycon_device *device,
 					   const char *opt)
 {
-	if (!device->port.membase)
+	struct uart_port *port = &device->port;
+
+	if (!port->membase)
 		return -ENODEV;
 
+	/* initialise control register */
+	writel(CDNS_UART_CR_TX_EN|CDNS_UART_CR_TXRST|CDNS_UART_CR_RXRST,
+	       port->membase + CDNS_UART_CR);
+
+	/* only set baud if specified on command line - otherwise
+	 * assume it has been initialized by a boot loader.
+	 */
+	if (device->baud) {
+		u32 cd = 0, bdiv = 0;
+		u32 mr;
+		int div8;
+
+		cdns_uart_calc_baud_divs(port->uartclk, device->baud,
+					 &bdiv, &cd, &div8);
+		mr = CDNS_UART_MR_PARITY_NONE;
+		if (div8)
+			mr |= CDNS_UART_MR_CLKSEL;
+
+		writel(mr,   port->membase + CDNS_UART_MR);
+		writel(cd,   port->membase + CDNS_UART_BAUDGEN);
+		writel(bdiv, port->membase + CDNS_UART_BAUDDIV);
+	}
+
 	device->con->write = cdns_early_write;
 
 	return 0;
@@ -1328,6 +1430,18 @@ static int cdns_uart_resume(struct device *device)
 static SIMPLE_DEV_PM_OPS(cdns_uart_dev_pm_ops, cdns_uart_suspend,
 		cdns_uart_resume);
 
+static const struct cdns_platform_data zynqmp_uart_def = {
+				.quirks = CDNS_UART_RXBS_SUPPORT, };
+
+/* Match table for of_platform binding */
+static const struct of_device_id cdns_uart_of_match[] = {
+	{ .compatible = "xlnx,xuartps", },
+	{ .compatible = "cdns,uart-r1p8", },
+	{ .compatible = "cdns,uart-r1p12", .data = &zynqmp_uart_def },
+	{}
+};
+MODULE_DEVICE_TABLE(of, cdns_uart_of_match);
+
 /**
  * cdns_uart_probe - Platform driver probe
  * @pdev: Pointer to the platform device structure
@@ -1340,12 +1454,20 @@ static int cdns_uart_probe(struct platform_device *pdev)
 	struct uart_port *port;
 	struct resource *res;
 	struct cdns_uart *cdns_uart_data;
+	const struct of_device_id *match;
 
 	cdns_uart_data = devm_kzalloc(&pdev->dev, sizeof(*cdns_uart_data),
 			GFP_KERNEL);
 	if (!cdns_uart_data)
 		return -ENOMEM;
 
+	match = of_match_node(cdns_uart_of_match, pdev->dev.of_node);
+	if (match && match->data) {
+		const struct cdns_platform_data *data = match->data;
+
+		cdns_uart_data->quirks = data->quirks;
+	}
+
 	cdns_uart_data->pclk = devm_clk_get(&pdev->dev, "pclk");
 	if (IS_ERR(cdns_uart_data->pclk)) {
 		cdns_uart_data->pclk = devm_clk_get(&pdev->dev, "aper_clk");
@@ -1471,14 +1593,6 @@ static int cdns_uart_remove(struct platform_device *pdev)
 	return rc;
 }
 
-/* Match table for of_platform binding */
-static const struct of_device_id cdns_uart_of_match[] = {
-	{ .compatible = "xlnx,xuartps", },
-	{ .compatible = "cdns,uart-r1p8", },
-	{}
-};
-MODULE_DEVICE_TABLE(of, cdns_uart_of_match);
-
 static struct platform_driver cdns_uart_platform_driver = {
 	.probe   = cdns_uart_probe,
 	.remove  = cdns_uart_remove,
diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c
index 2705ca960e92..06fb39c1d6dd 100644
--- a/drivers/tty/vt/vt.c
+++ b/drivers/tty/vt/vt.c
@@ -1312,12 +1312,12 @@ static int vc_t416_color(struct vc_data *vc, int i,
 	if (i > vc->vc_npar)
 		return i;
 
-	if (vc->vc_par[i] == 5 && i < vc->vc_npar) {
-		/* 256 colours -- ubiquitous */
+	if (vc->vc_par[i] == 5 && i + 1 <= vc->vc_npar) {
+		/* 256 colours */
 		i++;
 		rgb_from_256(vc->vc_par[i], &c);
-	} else if (vc->vc_par[i] == 2 && i <= vc->vc_npar + 3) {
-		/* 24 bit -- extremely rare */
+	} else if (vc->vc_par[i] == 2 && i + 3 <= vc->vc_npar) {
+		/* 24 bit */
 		c.r = vc->vc_par[i + 1];
 		c.g = vc->vc_par[i + 2];
 		c.b = vc->vc_par[i + 3];
@@ -1415,6 +1415,11 @@ static void csi_m(struct vc_data *vc)
 				(vc->vc_color & 0x0f);
 			break;
 		default:
+			if (vc->vc_par[i] >= 90 && vc->vc_par[i] <= 107) {
+				if (vc->vc_par[i] < 100)
+					vc->vc_intensity = 2;
+				vc->vc_par[i] -= 60;
+			}
 			if (vc->vc_par[i] >= 30 && vc->vc_par[i] <= 37)
 				vc->vc_color = color_table[vc->vc_par[i] - 30]
 					| (vc->vc_color & 0xf0);
@@ -3182,11 +3187,11 @@ static int do_bind_con_driver(const struct consw *csw, int first, int last,
 
 	pr_info("Console: switching ");
 	if (!deflt)
-		printk("consoles %d-%d ", first+1, last+1);
+		printk(KERN_CONT "consoles %d-%d ", first+1, last+1);
 	if (j >= 0) {
 		struct vc_data *vc = vc_cons[j].d;
 
-		printk("to %s %s %dx%d\n",
+		printk(KERN_CONT "to %s %s %dx%d\n",
 		       vc->vc_can_do_color ? "colour" : "mono",
 		       desc, vc->vc_cols, vc->vc_rows);
 
@@ -3195,7 +3200,7 @@ static int do_bind_con_driver(const struct consw *csw, int first, int last,
 			update_screen(vc);
 		}
 	} else
-		printk("to %s\n", desc);
+		printk(KERN_CONT "to %s\n", desc);
 
 	retval = 0;
 err:
diff --git a/drivers/uio/uio_dmem_genirq.c b/drivers/uio/uio_dmem_genirq.c
index 915facbf552e..e1134a4d97f3 100644
--- a/drivers/uio/uio_dmem_genirq.c
+++ b/drivers/uio/uio_dmem_genirq.c
@@ -229,7 +229,7 @@ static int uio_dmem_genirq_probe(struct platform_device *pdev)
 		++uiomem;
 	}
 
-	priv->dmem_region_start = i;
+	priv->dmem_region_start = uiomem - &uioinfo->mem[0];
 	priv->num_dmem_regions = pdata->num_dynamic_regions;
 
 	for (i = 0; i < pdata->num_dynamic_regions; ++i) {
diff --git a/drivers/usb/Kconfig b/drivers/usb/Kconfig
index 8689dcba5201..644e978cbd3e 100644
--- a/drivers/usb/Kconfig
+++ b/drivers/usb/Kconfig
@@ -152,7 +152,8 @@ source "drivers/usb/gadget/Kconfig"
 
 config USB_LED_TRIG
 	bool "USB LED Triggers"
-	depends on LEDS_CLASS && USB_COMMON && LEDS_TRIGGERS
+	depends on LEDS_CLASS && LEDS_TRIGGERS
+	select USB_COMMON
 	help
 	  This option adds LED triggers for USB host and/or gadget activity.
 
@@ -160,4 +161,25 @@ config USB_LED_TRIG
 	  LEDs and you want to use them as activity indicators for USB host or
 	  gadget.
 
+config USB_ULPI_BUS
+	tristate "USB ULPI PHY interface support"
+	select USB_COMMON
+	help
+	  UTMI+ Low Pin Interface (ULPI) is specification for a commonly used
+	  USB 2.0 PHY interface. The ULPI specification defines a standard set
+	  of registers that can be used to detect the vendor and product which
+	  allows ULPI to be handled as a bus. This module is the driver for that
+	  bus.
+
+	  The ULPI interfaces (the buses) are registered by the drivers for USB
+	  controllers which support ULPI register access and have ULPI PHY
+	  attached to them. The ULPI PHY drivers themselves are normal PHY
+	  drivers.
+
+	  ULPI PHYs provide often functions such as ADP sensing/probing (OTG
+	  protocol) and USB charger detection.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called ulpi.
+
 endif # USB_SUPPORT
diff --git a/drivers/usb/atm/cxacru.c b/drivers/usb/atm/cxacru.c
index 0a866e90b49c..f9fe86b6f7b5 100644
--- a/drivers/usb/atm/cxacru.c
+++ b/drivers/usb/atm/cxacru.c
@@ -1139,10 +1139,8 @@ static int cxacru_bind(struct usbatm_data *usbatm_instance,
 
 	/* instance init */
 	instance = kzalloc(sizeof(*instance), GFP_KERNEL);
-	if (!instance) {
-		usb_dbg(usbatm_instance, "cxacru_bind: no memory for instance data\n");
+	if (!instance)
 		return -ENOMEM;
-	}
 
 	instance->usbatm = usbatm_instance;
 	instance->modem_type = (struct cxacru_modem_type *) id->driver_info;
@@ -1168,13 +1166,11 @@ static int cxacru_bind(struct usbatm_data *usbatm_instance,
 	}
 	instance->rcv_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!instance->rcv_urb) {
-		usb_dbg(usbatm_instance, "cxacru_bind: no memory for rcv_urb\n");
 		ret = -ENOMEM;
 		goto fail;
 	}
 	instance->snd_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!instance->snd_urb) {
-		usb_dbg(usbatm_instance, "cxacru_bind: no memory for snd_urb\n");
 		ret = -ENOMEM;
 		goto fail;
 	}
diff --git a/drivers/usb/atm/speedtch.c b/drivers/usb/atm/speedtch.c
index 0270d1312f83..5083eb5b0d5e 100644
--- a/drivers/usb/atm/speedtch.c
+++ b/drivers/usb/atm/speedtch.c
@@ -817,7 +817,6 @@ static int speedtch_bind(struct usbatm_data *usbatm,
 	instance = kzalloc(sizeof(*instance), GFP_KERNEL);
 
 	if (!instance) {
-		usb_err(usbatm, "%s: no memory for instance data!\n", __func__);
 		ret = -ENOMEM;
 		goto fail_release;
 	}
diff --git a/drivers/usb/atm/ueagle-atm.c b/drivers/usb/atm/ueagle-atm.c
index 4333dc576a12..df67815f74e6 100644
--- a/drivers/usb/atm/ueagle-atm.c
+++ b/drivers/usb/atm/ueagle-atm.c
@@ -2196,17 +2196,12 @@ static int uea_boot(struct uea_softc *sc)
 		load_XILINX_firmware(sc);
 
 	intr = kmalloc(size, GFP_KERNEL);
-	if (!intr) {
-		uea_err(INS_TO_USBDEV(sc),
-		       "cannot allocate interrupt package\n");
+	if (!intr)
 		goto err0;
-	}
 
 	sc->urb_int = usb_alloc_urb(0, GFP_KERNEL);
-	if (!sc->urb_int) {
-		uea_err(INS_TO_USBDEV(sc), "cannot allocate interrupt URB\n");
+	if (!sc->urb_int)
 		goto err1;
-	}
 
 	usb_fill_int_urb(sc->urb_int, sc->usb_dev,
 			 usb_rcvintpipe(sc->usb_dev, UEA_INTR_PIPE),
@@ -2561,10 +2556,8 @@ static int uea_bind(struct usbatm_data *usbatm, struct usb_interface *intf,
 	}
 
 	sc = kzalloc(sizeof(struct uea_softc), GFP_KERNEL);
-	if (!sc) {
-		uea_err(usb, "uea_init: not enough memory !\n");
+	if (!sc)
 		return -ENOMEM;
-	}
 
 	sc->usb_dev = usb;
 	usbatm->driver_data = sc;
diff --git a/drivers/usb/atm/usbatm.c b/drivers/usb/atm/usbatm.c
index db322d9ccb6e..4dec9df8764b 100644
--- a/drivers/usb/atm/usbatm.c
+++ b/drivers/usb/atm/usbatm.c
@@ -819,7 +819,6 @@ static int usbatm_atm_open(struct atm_vcc *vcc)
 
 	new = kzalloc(sizeof(struct usbatm_vcc_data), GFP_KERNEL);
 	if (!new) {
-		atm_err(instance, "%s: no memory for vcc_data!\n", __func__);
 		ret = -ENOMEM;
 		goto fail;
 	}
@@ -1032,10 +1031,8 @@ int usbatm_usb_probe(struct usb_interface *intf, const struct usb_device_id *id,
 
 	/* instance init */
 	instance = kzalloc(sizeof(*instance) + sizeof(struct urb *) * (num_rcv_urbs + num_snd_urbs), GFP_KERNEL);
-	if (!instance) {
-		dev_err(dev, "%s: no memory for instance data!\n", __func__);
+	if (!instance)
 		return -ENOMEM;
-	}
 
 	/* public fields */
 
@@ -1141,7 +1138,6 @@ int usbatm_usb_probe(struct usb_interface *intf, const struct usb_device_id *id,
 
 		urb = usb_alloc_urb(iso_packets, GFP_KERNEL);
 		if (!urb) {
-			dev_err(dev, "%s: no memory for urb %d!\n", __func__, i);
 			error = -ENOMEM;
 			goto fail_unbind;
 		}
@@ -1151,7 +1147,6 @@ int usbatm_usb_probe(struct usb_interface *intf, const struct usb_device_id *id,
 		/* zero the tx padding to avoid leaking information */
 		buffer = kzalloc(channel->buf_size, GFP_KERNEL);
 		if (!buffer) {
-			dev_err(dev, "%s: no memory for buffer %d!\n", __func__, i);
 			error = -ENOMEM;
 			goto fail_unbind;
 		}
@@ -1182,7 +1177,6 @@ int usbatm_usb_probe(struct usb_interface *intf, const struct usb_device_id *id,
 	instance->cell_buf = kmalloc(instance->rx_channel.stride, GFP_KERNEL);
 
 	if (!instance->cell_buf) {
-		dev_err(dev, "%s: no memory for cell buffer!\n", __func__);
 		error = -ENOMEM;
 		goto fail_unbind;
 	}
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c
index dedc33e589f4..099179457f60 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.c
+++ b/drivers/usb/chipidea/ci_hdrc_imx.c
@@ -140,6 +140,9 @@ static struct imx_usbmisc_data *usbmisc_get_init_data(struct device *dev)
 	if (of_find_property(np, "disable-over-current", NULL))
 		data->disable_oc = 1;
 
+	if (of_find_property(np, "over-current-active-high", NULL))
+		data->oc_polarity = 1;
+
 	if (of_find_property(np, "external-vbus-divider", NULL))
 		data->evdo = 1;
 
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.h b/drivers/usb/chipidea/ci_hdrc_imx.h
index 635717e9354a..409aa5ca8dda 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.h
+++ b/drivers/usb/chipidea/ci_hdrc_imx.h
@@ -17,6 +17,7 @@ struct imx_usbmisc_data {
 	int index;
 
 	unsigned int disable_oc:1; /* over current detect disabled */
+	unsigned int oc_polarity:1; /* over current polarity if oc enabled */
 	unsigned int evdo:1; /* set external vbus divider option */
 };
 
diff --git a/drivers/usb/chipidea/host.c b/drivers/usb/chipidea/host.c
index 053bac9d983c..96ae69502c86 100644
--- a/drivers/usb/chipidea/host.c
+++ b/drivers/usb/chipidea/host.c
@@ -81,12 +81,15 @@ static int ehci_ci_reset(struct usb_hcd *hcd)
 {
 	struct device *dev = hcd->self.controller;
 	struct ci_hdrc *ci = dev_get_drvdata(dev);
+	struct ehci_hcd *ehci = hcd_to_ehci(hcd);
 	int ret;
 
 	ret = ehci_setup(hcd);
 	if (ret)
 		return ret;
 
+	ehci->need_io_watchdog = 0;
+
 	ci_platform_configure(ci);
 
 	return ret;
diff --git a/drivers/usb/chipidea/udc.c b/drivers/usb/chipidea/udc.c
index 065f5d97aa67..661f43fe0f9e 100644
--- a/drivers/usb/chipidea/udc.c
+++ b/drivers/usb/chipidea/udc.c
@@ -59,7 +59,7 @@ ctrl_endpt_in_desc = {
  */
 static inline int hw_ep_bit(int num, int dir)
 {
-	return num + (dir ? 16 : 0);
+	return num + ((dir == TX) ? 16 : 0);
 }
 
 static inline int ep_to_bit(struct ci_hdrc *ci, int n)
@@ -121,9 +121,8 @@ static int hw_ep_flush(struct ci_hdrc *ci, int num, int dir)
  */
 static int hw_ep_disable(struct ci_hdrc *ci, int num, int dir)
 {
-	hw_ep_flush(ci, num, dir);
 	hw_write(ci, OP_ENDPTCTRL + num,
-		 dir ? ENDPTCTRL_TXE : ENDPTCTRL_RXE, 0);
+		 (dir == TX) ? ENDPTCTRL_TXE : ENDPTCTRL_RXE, 0);
 	return 0;
 }
 
@@ -139,7 +138,7 @@ static int hw_ep_enable(struct ci_hdrc *ci, int num, int dir, int type)
 {
 	u32 mask, data;
 
-	if (dir) {
+	if (dir == TX) {
 		mask  = ENDPTCTRL_TXT;  /* type    */
 		data  = type << __ffs(mask);
 
@@ -171,7 +170,7 @@ static int hw_ep_enable(struct ci_hdrc *ci, int num, int dir, int type)
  */
 static int hw_ep_get_halt(struct ci_hdrc *ci, int num, int dir)
 {
-	u32 mask = dir ? ENDPTCTRL_TXS : ENDPTCTRL_RXS;
+	u32 mask = (dir == TX) ? ENDPTCTRL_TXS : ENDPTCTRL_RXS;
 
 	return hw_read(ci, OP_ENDPTCTRL + num, mask) ? 1 : 0;
 }
@@ -188,6 +187,9 @@ static int hw_ep_prime(struct ci_hdrc *ci, int num, int dir, int is_ctrl)
 {
 	int n = hw_ep_bit(num, dir);
 
+	/* Synchronize before ep prime */
+	wmb();
+
 	if (is_ctrl && dir == RX && hw_read(ci, OP_ENDPTSETUPSTAT, BIT(num)))
 		return -EAGAIN;
 
@@ -218,8 +220,8 @@ static int hw_ep_set_halt(struct ci_hdrc *ci, int num, int dir, int value)
 
 	do {
 		enum ci_hw_regs reg = OP_ENDPTCTRL + num;
-		u32 mask_xs = dir ? ENDPTCTRL_TXS : ENDPTCTRL_RXS;
-		u32 mask_xr = dir ? ENDPTCTRL_TXR : ENDPTCTRL_RXR;
+		u32 mask_xs = (dir == TX) ? ENDPTCTRL_TXS : ENDPTCTRL_RXS;
+		u32 mask_xr = (dir == TX) ? ENDPTCTRL_TXR : ENDPTCTRL_RXR;
 
 		/* data toggle - reserved for EP0 but it's in ESS */
 		hw_write(ci, reg, mask_xs|mask_xr,
@@ -348,8 +350,7 @@ static int add_td_to_list(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq,
 	if (node == NULL)
 		return -ENOMEM;
 
-	node->ptr = dma_pool_zalloc(hwep->td_pool, GFP_ATOMIC,
-				   &node->dma);
+	node->ptr = dma_pool_zalloc(hwep->td_pool, GFP_ATOMIC, &node->dma);
 	if (node->ptr == NULL) {
 		kfree(node);
 		return -ENOMEM;
@@ -506,8 +507,6 @@ static int _hardware_enqueue(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq)
 		hwep->qh.ptr->cap |= mul << __ffs(QH_MULT);
 	}
 
-	wmb();   /* synchronize before ep prime */
-
 	ret = hw_ep_prime(ci, hwep->num, hwep->dir,
 			   hwep->type == USB_ENDPOINT_XFER_CONTROL);
 done:
@@ -534,9 +533,6 @@ static int reprime_dtd(struct ci_hdrc *ci, struct ci_hw_ep *hwep,
 	hwep->qh.ptr->td.token &=
 		cpu_to_le32(~(TD_STATUS_HALTED | TD_STATUS_ACTIVE));
 
-	/* Synchronize before ep prime */
-	wmb();
-
 	return hw_ep_prime(ci, hwep->num, hwep->dir,
 				hwep->type == USB_ENDPOINT_XFER_CONTROL);
 }
@@ -590,7 +586,7 @@ static int _hardware_dequeue(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq)
 		}
 
 		if (remaining_length) {
-			if (hwep->dir) {
+			if (hwep->dir == TX) {
 				hwreq->req.status = -EPROTO;
 				break;
 			}
@@ -949,6 +945,15 @@ static int isr_setup_status_phase(struct ci_hdrc *ci)
 	int retval;
 	struct ci_hw_ep *hwep;
 
+	/*
+	 * Unexpected USB controller behavior, caused by bad signal integrity
+	 * or ground reference problems, can lead to isr_setup_status_phase
+	 * being called with ci->status equal to NULL.
+	 * If this situation occurs, you should review your USB hardware design.
+	 */
+	if (WARN_ON_ONCE(!ci->status))
+		return -EPIPE;
+
 	hwep = (ci->ep0_dir == TX) ? ci->ep0out : ci->ep0in;
 	ci->status->context = ci;
 	ci->status->complete = isr_setup_status_complete;
@@ -1042,9 +1047,9 @@ __acquires(ci->lock)
 			if (req.wLength != 0)
 				break;
 			num  = le16_to_cpu(req.wIndex);
-			dir = num & USB_ENDPOINT_DIR_MASK;
+			dir = (num & USB_ENDPOINT_DIR_MASK) ? TX : RX;
 			num &= USB_ENDPOINT_NUMBER_MASK;
-			if (dir) /* TX */
+			if (dir == TX)
 				num += ci->hw_ep_max / 2;
 			if (!ci->ci_hw_ep[num].wedge) {
 				spin_unlock(&ci->lock);
@@ -1094,9 +1099,9 @@ __acquires(ci->lock)
 			if (req.wLength != 0)
 				break;
 			num  = le16_to_cpu(req.wIndex);
-			dir = num & USB_ENDPOINT_DIR_MASK;
+			dir = (num & USB_ENDPOINT_DIR_MASK) ? TX : RX;
 			num &= USB_ENDPOINT_NUMBER_MASK;
-			if (dir) /* TX */
+			if (dir == TX)
 				num += ci->hw_ep_max / 2;
 
 			spin_unlock(&ci->lock);
@@ -1596,8 +1601,11 @@ static int ci_udc_pullup(struct usb_gadget *_gadget, int is_on)
 {
 	struct ci_hdrc *ci = container_of(_gadget, struct ci_hdrc, gadget);
 
-	/* Data+ pullup controlled by OTG state machine in OTG fsm mode */
-	if (ci_otg_is_fsm_mode(ci))
+	/*
+	 * Data+ pullup controlled by OTG state machine in OTG fsm mode;
+	 * and don't touch Data+ in host mode for dual role config.
+	 */
+	if (ci_otg_is_fsm_mode(ci) || ci->role == CI_ROLE_HOST)
 		return 0;
 
 	pm_runtime_get_sync(&ci->gadget.dev);
@@ -1668,12 +1676,10 @@ static int init_eps(struct ci_hdrc *ci)
 			usb_ep_set_maxpacket_limit(&hwep->ep, (unsigned short)~0);
 
 			INIT_LIST_HEAD(&hwep->qh.queue);
-			hwep->qh.ptr = dma_pool_alloc(ci->qh_pool, GFP_KERNEL,
-						     &hwep->qh.dma);
+			hwep->qh.ptr = dma_pool_zalloc(ci->qh_pool, GFP_KERNEL,
+						       &hwep->qh.dma);
 			if (hwep->qh.ptr == NULL)
 				retval = -ENOMEM;
-			else
-				memset(hwep->qh.ptr, 0, sizeof(*hwep->qh.ptr));
 
 			/*
 			 * set up shorthands for ep0 out and in endpoints,
@@ -1987,7 +1993,7 @@ int ci_hdrc_gadget_init(struct ci_hdrc *ci)
 	if (!hw_read(ci, CAP_DCCPARAMS, DCCPARAMS_DC))
 		return -ENXIO;
 
-	rdrv = devm_kzalloc(ci->dev, sizeof(struct ci_role_driver), GFP_KERNEL);
+	rdrv = devm_kzalloc(ci->dev, sizeof(*rdrv), GFP_KERNEL);
 	if (!rdrv)
 		return -ENOMEM;
 
diff --git a/drivers/usb/chipidea/usbmisc_imx.c b/drivers/usb/chipidea/usbmisc_imx.c
index ab8b027e8cc8..20d02a5e418d 100644
--- a/drivers/usb/chipidea/usbmisc_imx.c
+++ b/drivers/usb/chipidea/usbmisc_imx.c
@@ -56,6 +56,7 @@
 
 #define MX6_BM_NON_BURST_SETTING	BIT(1)
 #define MX6_BM_OVER_CUR_DIS		BIT(7)
+#define MX6_BM_OVER_CUR_POLARITY	BIT(8)
 #define MX6_BM_WAKEUP_ENABLE		BIT(10)
 #define MX6_BM_ID_WAKEUP		BIT(16)
 #define MX6_BM_VBUS_WAKEUP		BIT(17)
@@ -266,11 +267,14 @@ static int usbmisc_imx6q_init(struct imx_usbmisc_data *data)
 
 	spin_lock_irqsave(&usbmisc->lock, flags);
 
+	reg = readl(usbmisc->base + data->index * 4);
 	if (data->disable_oc) {
-		reg = readl(usbmisc->base + data->index * 4);
-		writel(reg | MX6_BM_OVER_CUR_DIS,
-			usbmisc->base + data->index * 4);
+		reg |= MX6_BM_OVER_CUR_DIS;
+	} else if (data->oc_polarity == 1) {
+		/* High active */
+		reg &= ~(MX6_BM_OVER_CUR_DIS | MX6_BM_OVER_CUR_POLARITY);
 	}
+	writel(reg, usbmisc->base + data->index * 4);
 
 	/* SoC non-burst setting */
 	reg = readl(usbmisc->base + data->index * 4);
@@ -365,10 +369,14 @@ static int usbmisc_imx7d_init(struct imx_usbmisc_data *data)
 		return -EINVAL;
 
 	spin_lock_irqsave(&usbmisc->lock, flags);
+	reg = readl(usbmisc->base);
 	if (data->disable_oc) {
-		reg = readl(usbmisc->base);
-		writel(reg | MX6_BM_OVER_CUR_DIS, usbmisc->base);
+		reg |= MX6_BM_OVER_CUR_DIS;
+	} else if (data->oc_polarity == 1) {
+		/* High active */
+		reg &= ~(MX6_BM_OVER_CUR_DIS | MX6_BM_OVER_CUR_POLARITY);
 	}
+	writel(reg, usbmisc->base);
 
 	reg = readl(usbmisc->base + MX7D_USBNC_USB_CTRL2);
 	reg &= ~MX7D_USB_VBUS_WAKEUP_SOURCE_MASK;
@@ -492,6 +500,10 @@ static const struct of_device_id usbmisc_imx_dt_ids[] = {
 		.compatible = "fsl,imx6ul-usbmisc",
 		.data = &imx6sx_usbmisc_ops,
 	},
+	{
+		.compatible = "fsl,imx7d-usbmisc",
+		.data = &imx7d_usbmisc_ops,
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, usbmisc_imx_dt_ids);
diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c
index 71912301ef7f..78f0f85bebdc 100644
--- a/drivers/usb/class/cdc-acm.c
+++ b/drivers/usb/class/cdc-acm.c
@@ -368,17 +368,17 @@ static int acm_submit_read_urb(struct acm *acm, int index, gfp_t mem_flags)
 	if (!test_and_clear_bit(index, &acm->read_urbs_free))
 		return 0;
 
-	dev_vdbg(&acm->data->dev, "%s - urb %d\n", __func__, index);
-
 	res = usb_submit_urb(acm->read_urbs[index], mem_flags);
 	if (res) {
 		if (res != -EPERM) {
 			dev_err(&acm->data->dev,
-					"%s - usb_submit_urb failed: %d\n",
-					__func__, res);
+					"urb %d failed submission with %d\n",
+					index, res);
 		}
 		set_bit(index, &acm->read_urbs_free);
 		return res;
+	} else {
+		dev_vdbg(&acm->data->dev, "submitted urb %d\n", index);
 	}
 
 	return 0;
@@ -415,8 +415,9 @@ static void acm_read_bulk_callback(struct urb *urb)
 	unsigned long flags;
 	int status = urb->status;
 
-	dev_vdbg(&acm->data->dev, "%s - urb %d, len %d\n", __func__,
-					rb->index, urb->actual_length);
+	dev_vdbg(&acm->data->dev, "got urb %d, len %d, status %d\n",
+					rb->index, urb->actual_length,
+					status);
 
 	if (!acm->dev) {
 		set_bit(rb->index, &acm->read_urbs_free);
@@ -426,8 +427,6 @@ static void acm_read_bulk_callback(struct urb *urb)
 
 	if (status) {
 		set_bit(rb->index, &acm->read_urbs_free);
-		dev_dbg(&acm->data->dev, "%s - non-zero urb status: %d\n",
-							__func__, status);
 		if ((status != -ENOENT) || (urb->actual_length == 0))
 			return;
 	}
@@ -462,8 +461,7 @@ static void acm_write_bulk(struct urb *urb)
 	int status = urb->status;
 
 	if (status || (urb->actual_length != urb->transfer_buffer_length))
-		dev_vdbg(&acm->data->dev, "%s - len %d/%d, status %d\n",
-			__func__,
+		dev_vdbg(&acm->data->dev, "wrote len %d/%d, status %d\n",
 			urb->actual_length,
 			urb->transfer_buffer_length,
 			status);
@@ -478,8 +476,6 @@ static void acm_softint(struct work_struct *work)
 {
 	struct acm *acm = container_of(work, struct acm, work);
 
-	dev_vdbg(&acm->data->dev, "%s\n", __func__);
-
 	tty_port_tty_wakeup(&acm->port);
 }
 
@@ -492,8 +488,6 @@ static int acm_tty_install(struct tty_driver *driver, struct tty_struct *tty)
 	struct acm *acm;
 	int retval;
 
-	dev_dbg(tty->dev, "%s\n", __func__);
-
 	acm = acm_get_by_minor(tty->index);
 	if (!acm)
 		return -ENODEV;
@@ -515,8 +509,6 @@ static int acm_tty_open(struct tty_struct *tty, struct file *filp)
 {
 	struct acm *acm = tty->driver_data;
 
-	dev_dbg(tty->dev, "%s\n", __func__);
-
 	return tty_port_open(&acm->port, tty, filp);
 }
 
@@ -545,8 +537,6 @@ static int acm_port_activate(struct tty_port *port, struct tty_struct *tty)
 	int retval = -ENODEV;
 	int i;
 
-	dev_dbg(&acm->control->dev, "%s\n", __func__);
-
 	mutex_lock(&acm->mutex);
 	if (acm->disconnected)
 		goto disconnected;
@@ -607,8 +597,6 @@ static void acm_port_destruct(struct tty_port *port)
 {
 	struct acm *acm = container_of(port, struct acm, port);
 
-	dev_dbg(&acm->control->dev, "%s\n", __func__);
-
 	acm_release_minor(acm);
 	usb_put_intf(acm->control);
 	kfree(acm->country_codes);
@@ -622,8 +610,6 @@ static void acm_port_shutdown(struct tty_port *port)
 	struct acm_wb *wb;
 	int i;
 
-	dev_dbg(&acm->control->dev, "%s\n", __func__);
-
 	/*
 	 * Need to grab write_lock to prevent race with resume, but no need to
 	 * hold it due to the tty-port initialised flag.
@@ -654,21 +640,21 @@ static void acm_port_shutdown(struct tty_port *port)
 static void acm_tty_cleanup(struct tty_struct *tty)
 {
 	struct acm *acm = tty->driver_data;
-	dev_dbg(&acm->control->dev, "%s\n", __func__);
+
 	tty_port_put(&acm->port);
 }
 
 static void acm_tty_hangup(struct tty_struct *tty)
 {
 	struct acm *acm = tty->driver_data;
-	dev_dbg(&acm->control->dev, "%s\n", __func__);
+
 	tty_port_hangup(&acm->port);
 }
 
 static void acm_tty_close(struct tty_struct *tty, struct file *filp)
 {
 	struct acm *acm = tty->driver_data;
-	dev_dbg(&acm->control->dev, "%s\n", __func__);
+
 	tty_port_close(&acm->port, tty, filp);
 }
 
@@ -684,7 +670,7 @@ static int acm_tty_write(struct tty_struct *tty,
 	if (!count)
 		return 0;
 
-	dev_vdbg(&acm->data->dev, "%s - count %d\n", __func__, count);
+	dev_vdbg(&acm->data->dev, "%d bytes from tty layer\n", count);
 
 	spin_lock_irqsave(&acm->write_lock, flags);
 	wbn = acm_wb_alloc(acm);
@@ -701,7 +687,7 @@ static int acm_tty_write(struct tty_struct *tty,
 	}
 
 	count = (count > acm->writesize) ? acm->writesize : count;
-	dev_vdbg(&acm->data->dev, "%s - write %d\n", __func__, count);
+	dev_vdbg(&acm->data->dev, "writing %d bytes\n", count);
 	memcpy(wb->buf, buf, count);
 	wb->len = count;
 
@@ -1193,6 +1179,9 @@ static int acm_probe(struct usb_interface *intf,
 		return -EINVAL;
 	}
 
+	if (!intf->cur_altsetting)
+		return -EINVAL;
+
 	if (!buflen) {
 		if (intf->cur_altsetting->endpoint &&
 				intf->cur_altsetting->endpoint->extralen &&
@@ -1246,6 +1235,8 @@ static int acm_probe(struct usb_interface *intf,
 		dev_dbg(&intf->dev, "no interfaces\n");
 		return -ENODEV;
 	}
+	if (!data_interface->cur_altsetting || !control_interface->cur_altsetting)
+		return -ENODEV;
 
 	if (data_intf_num != call_intf_num)
 		dev_dbg(&intf->dev, "Separate call control interface. That is not fully supported.\n");
@@ -1354,7 +1345,6 @@ made_compressed_probe:
 	spin_lock_init(&acm->write_lock);
 	spin_lock_init(&acm->read_lock);
 	mutex_init(&acm->mutex);
-	acm->rx_endpoint = usb_rcvbulkpipe(usb_dev, epread->bEndpointAddress);
 	acm->is_int_ep = usb_endpoint_xfer_int(epread);
 	if (acm->is_int_ep)
 		acm->bInterval = epread->bInterval;
@@ -1394,14 +1384,14 @@ made_compressed_probe:
 		urb->transfer_dma = rb->dma;
 		if (acm->is_int_ep) {
 			usb_fill_int_urb(urb, acm->dev,
-					 acm->rx_endpoint,
+					 usb_rcvintpipe(usb_dev, epread->bEndpointAddress),
 					 rb->base,
 					 acm->readsize,
 					 acm_read_bulk_callback, rb,
 					 acm->bInterval);
 		} else {
 			usb_fill_bulk_urb(urb, acm->dev,
-					  acm->rx_endpoint,
+					  usb_rcvbulkpipe(usb_dev, epread->bEndpointAddress),
 					  rb->base,
 					  acm->readsize,
 					  acm_read_bulk_callback, rb);
@@ -1534,8 +1524,6 @@ static void stop_data_traffic(struct acm *acm)
 {
 	int i;
 
-	dev_dbg(&acm->control->dev, "%s\n", __func__);
-
 	usb_kill_urb(acm->ctrlurb);
 	for (i = 0; i < ACM_NW; i++)
 		usb_kill_urb(acm->wb[i].urb);
@@ -1552,8 +1540,6 @@ static void acm_disconnect(struct usb_interface *intf)
 	struct tty_struct *tty;
 	int i;
 
-	dev_dbg(&intf->dev, "%s\n", __func__);
-
 	/* sibling interface is already cleaning up */
 	if (!acm)
 		return;
diff --git a/drivers/usb/class/cdc-acm.h b/drivers/usb/class/cdc-acm.h
index 05ce308d5d2a..1f1eabfd8462 100644
--- a/drivers/usb/class/cdc-acm.h
+++ b/drivers/usb/class/cdc-acm.h
@@ -96,7 +96,6 @@ struct acm {
 	struct acm_rb read_buffers[ACM_NR];
 	struct acm_wb *putbuffer;			/* for acm_tty_put_char() */
 	int rx_buflimit;
-	int rx_endpoint;
 	spinlock_t read_lock;
 	int write_used;					/* number of non-empty write buffers */
 	int transmitting;
diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c
index 337948c42110..0a6369510f2d 100644
--- a/drivers/usb/class/cdc-wdm.c
+++ b/drivers/usb/class/cdc-wdm.c
@@ -58,6 +58,7 @@ MODULE_DEVICE_TABLE (usb, wdm_ids);
 #define WDM_SUSPENDING		8
 #define WDM_RESETTING		9
 #define WDM_OVERFLOW		10
+#define WDM_DRAIN_ON_OPEN	11
 
 #define WDM_MAX			16
 
@@ -154,6 +155,9 @@ static void wdm_out_callback(struct urb *urb)
 	wake_up(&desc->wait);
 }
 
+/* forward declaration */
+static int service_outstanding_interrupt(struct wdm_device *desc);
+
 static void wdm_in_callback(struct urb *urb)
 {
 	struct wdm_device *desc = urb->context;
@@ -167,18 +171,18 @@ static void wdm_in_callback(struct urb *urb)
 		switch (status) {
 		case -ENOENT:
 			dev_dbg(&desc->intf->dev,
-				"nonzero urb status received: -ENOENT");
+				"nonzero urb status received: -ENOENT\n");
 			goto skip_error;
 		case -ECONNRESET:
 			dev_dbg(&desc->intf->dev,
-				"nonzero urb status received: -ECONNRESET");
+				"nonzero urb status received: -ECONNRESET\n");
 			goto skip_error;
 		case -ESHUTDOWN:
 			dev_dbg(&desc->intf->dev,
-				"nonzero urb status received: -ESHUTDOWN");
+				"nonzero urb status received: -ESHUTDOWN\n");
 			goto skip_error;
 		case -EPIPE:
-			dev_err(&desc->intf->dev,
+			dev_dbg(&desc->intf->dev,
 				"nonzero urb status received: -EPIPE\n");
 			break;
 		default:
@@ -188,7 +192,13 @@ static void wdm_in_callback(struct urb *urb)
 		}
 	}
 
-	desc->rerr = status;
+	/*
+	 * only set a new error if there is no previous error.
+	 * Errors are only cleared during read/open
+	 */
+	if (desc->rerr  == 0)
+		desc->rerr = status;
+
 	if (length + desc->length > desc->wMaxCommand) {
 		/* The buffer would overflow */
 		set_bit(WDM_OVERFLOW, &desc->flags);
@@ -200,10 +210,40 @@ static void wdm_in_callback(struct urb *urb)
 			desc->reslength = length;
 		}
 	}
+
+	/*
+	 * Handling devices with the WDM_DRAIN_ON_OPEN flag set:
+	 * If desc->resp_count is unset, then the urb was submitted
+	 * without a prior notification.  If the device returned any
+	 * data, then this implies that it had messages queued without
+	 * notifying us.  Continue reading until that queue is flushed.
+	 */
+	if (!desc->resp_count) {
+		if (!length) {
+			/* do not propagate the expected -EPIPE */
+			desc->rerr = 0;
+			goto unlock;
+		}
+		dev_dbg(&desc->intf->dev, "got %d bytes without notification\n", length);
+		set_bit(WDM_RESPONDING, &desc->flags);
+		usb_submit_urb(desc->response, GFP_ATOMIC);
+	}
+
 skip_error:
+	set_bit(WDM_READ, &desc->flags);
 	wake_up(&desc->wait);
 
-	set_bit(WDM_READ, &desc->flags);
+	if (desc->rerr) {
+		/*
+		 * Since there was an error, userspace may decide to not read
+		 * any data after poll'ing.
+		 * We should respond to further attempts from the device to send
+		 * data, so that we can get unstuck.
+		 */
+		service_outstanding_interrupt(desc);
+	}
+
+unlock:
 	spin_unlock(&desc->iuspin);
 }
 
@@ -244,18 +284,18 @@ static void wdm_int_callback(struct urb *urb)
 	switch (dr->bNotificationType) {
 	case USB_CDC_NOTIFY_RESPONSE_AVAILABLE:
 		dev_dbg(&desc->intf->dev,
-			"NOTIFY_RESPONSE_AVAILABLE received: index %d len %d",
+			"NOTIFY_RESPONSE_AVAILABLE received: index %d len %d\n",
 			le16_to_cpu(dr->wIndex), le16_to_cpu(dr->wLength));
 		break;
 
 	case USB_CDC_NOTIFY_NETWORK_CONNECTION:
 
 		dev_dbg(&desc->intf->dev,
-			"NOTIFY_NETWORK_CONNECTION %s network",
+			"NOTIFY_NETWORK_CONNECTION %s network\n",
 			dr->wValue ? "connected to" : "disconnected from");
 		goto exit;
 	case USB_CDC_NOTIFY_SPEED_CHANGE:
-		dev_dbg(&desc->intf->dev, "SPEED_CHANGE received (len %u)",
+		dev_dbg(&desc->intf->dev, "SPEED_CHANGE received (len %u)\n",
 			urb->actual_length);
 		goto exit;
 	default:
@@ -274,8 +314,7 @@ static void wdm_int_callback(struct urb *urb)
 		&& !test_bit(WDM_DISCONNECTING, &desc->flags)
 		&& !test_bit(WDM_SUSPENDING, &desc->flags)) {
 		rv = usb_submit_urb(desc->response, GFP_ATOMIC);
-		dev_dbg(&desc->intf->dev, "%s: usb_submit_urb %d",
-			__func__, rv);
+		dev_dbg(&desc->intf->dev, "submit response URB %d\n", rv);
 	}
 	spin_unlock(&desc->iuspin);
 	if (rv < 0) {
@@ -417,7 +456,7 @@ static ssize_t wdm_write
 		rv = usb_translate_errors(rv);
 		goto out_free_mem_pm;
 	} else {
-		dev_dbg(&desc->intf->dev, "Tx URB has been submitted index=%d",
+		dev_dbg(&desc->intf->dev, "Tx URB has been submitted index=%d\n",
 			le16_to_cpu(req->wIndex));
 	}
 
@@ -436,17 +475,14 @@ out_free_mem:
 }
 
 /*
- * clear WDM_READ flag and possibly submit the read urb if resp_count
- * is non-zero.
+ * Submit the read urb if resp_count is non-zero.
  *
  * Called with desc->iuspin locked
  */
-static int clear_wdm_read_flag(struct wdm_device *desc)
+static int service_outstanding_interrupt(struct wdm_device *desc)
 {
 	int rv = 0;
 
-	clear_bit(WDM_READ, &desc->flags);
-
 	/* submit read urb only if the device is waiting for it */
 	if (!desc->resp_count || !--desc->resp_count)
 		goto out;
@@ -537,8 +573,9 @@ retry:
 		}
 
 		if (!desc->reslength) { /* zero length read */
-			dev_dbg(&desc->intf->dev, "%s: zero length - clearing WDM_READ\n", __func__);
-			rv = clear_wdm_read_flag(desc);
+			dev_dbg(&desc->intf->dev, "zero length - clearing WDM_READ\n");
+			clear_bit(WDM_READ, &desc->flags);
+			rv = service_outstanding_interrupt(desc);
 			spin_unlock_irq(&desc->iuspin);
 			if (rv < 0)
 				goto err;
@@ -563,8 +600,10 @@ retry:
 
 	desc->length -= cntr;
 	/* in case we had outstanding data */
-	if (!desc->length)
-		clear_wdm_read_flag(desc);
+	if (!desc->length) {
+		clear_bit(WDM_READ, &desc->flags);
+		service_outstanding_interrupt(desc);
+	}
 	spin_unlock_irq(&desc->iuspin);
 	rv = cntr;
 
@@ -647,6 +686,17 @@ static int wdm_open(struct inode *inode, struct file *file)
 			dev_err(&desc->intf->dev,
 				"Error submitting int urb - %d\n", rv);
 			rv = usb_translate_errors(rv);
+		} else if (test_bit(WDM_DRAIN_ON_OPEN, &desc->flags)) {
+			/*
+			 * Some devices keep pending messages queued
+			 * without resending notifications.  We must
+			 * flush the message queue before we can
+			 * assume a one-to-one relationship between
+			 * notifications and messages in the queue
+			 */
+			dev_dbg(&desc->intf->dev, "draining queued data\n");
+			set_bit(WDM_RESPONDING, &desc->flags);
+			rv = usb_submit_urb(desc->response, GFP_KERNEL);
 		}
 	} else {
 		rv = 0;
@@ -673,7 +723,7 @@ static int wdm_release(struct inode *inode, struct file *file)
 
 	if (!desc->count) {
 		if (!test_bit(WDM_DISCONNECTING, &desc->flags)) {
-			dev_dbg(&desc->intf->dev, "wdm_release: cleanup");
+			dev_dbg(&desc->intf->dev, "wdm_release: cleanup\n");
 			kill_urbs(desc);
 			spin_lock_irq(&desc->iuspin);
 			desc->resp_count = 0;
@@ -753,7 +803,8 @@ static void wdm_rxwork(struct work_struct *work)
 /* --- hotplug --- */
 
 static int wdm_create(struct usb_interface *intf, struct usb_endpoint_descriptor *ep,
-		u16 bufsize, int (*manage_power)(struct usb_interface *, int))
+		u16 bufsize, int (*manage_power)(struct usb_interface *, int),
+		bool drain_on_open)
 {
 	int rv = -ENOMEM;
 	struct wdm_device *desc;
@@ -840,6 +891,68 @@ static int wdm_create(struct usb_interface *intf, struct usb_endpoint_descriptor
 
 	desc->manage_power = manage_power;
 
+	/*
+	 * "drain_on_open" enables a hack to work around a firmware
+	 * issue observed on network functions, in particular MBIM
+	 * functions.
+	 *
+	 * Quoting section 7 of the CDC-WMC r1.1 specification:
+	 *
+	 *  "The firmware shall interpret GetEncapsulatedResponse as a
+	 *   request to read response bytes. The firmware shall send
+	 *   the next wLength bytes from the response. The firmware
+	 *   shall allow the host to retrieve data using any number of
+	 *   GetEncapsulatedResponse requests. The firmware shall
+	 *   return a zero- length reply if there are no data bytes
+	 *   available.
+	 *
+	 *   The firmware shall send ResponseAvailable notifications
+	 *   periodically, using any appropriate algorithm, to inform
+	 *   the host that there is data available in the reply
+	 *   buffer. The firmware is allowed to send ResponseAvailable
+	 *   notifications even if there is no data available, but
+	 *   this will obviously reduce overall performance."
+	 *
+	 * These requirements, although they make equally sense, are
+	 * often not implemented by network functions. Some firmwares
+	 * will queue data indefinitely, without ever resending a
+	 * notification. The result is that the driver and firmware
+	 * loses "syncronization" if the driver ever fails to respond
+	 * to a single notification, something which easily can happen
+	 * on release(). When this happens, the driver will appear to
+	 * never receive notifications for the most current data. Each
+	 * notification will only cause a single read, which returns
+	 * the oldest data in the firmware's queue.
+	 *
+	 * The "drain_on_open" hack resolves the situation by draining
+	 * data from the firmware until none is returned, without a
+	 * prior notification.
+	 *
+	 * This will inevitably race with the firmware, risking that
+	 * we read data from the device before handling the associated
+	 * notification. To make things worse, some of the devices
+	 * needing the hack do not implement the "return zero if no
+	 * data is available" requirement either. Instead they return
+	 * an error on the subsequent read in this case.  This means
+	 * that "winning" the race can cause an unexpected EIO to
+	 * userspace.
+	 *
+	 * "winning" the race is more likely on resume() than on
+	 * open(), and the unexpected error is more harmful in the
+	 * middle of an open session. The hack is therefore only
+	 * applied on open(), and not on resume() where it logically
+	 * would be equally necessary. So we define open() as the only
+	 * driver <-> device "syncronization point".  Should we happen
+	 * to lose a notification after open(), then syncronization
+	 * will be lost until release()
+	 *
+	 * The hack should not be enabled for CDC WDM devices
+	 * conforming to the CDC-WMC r1.1 specification.  This is
+	 * ensured by setting drain_on_open to false in wdm_probe().
+	 */
+	if (drain_on_open)
+		set_bit(WDM_DRAIN_ON_OPEN, &desc->flags);
+
 	spin_lock(&wdm_device_list_lock);
 	list_add(&desc->device_list, &wdm_device_list);
 	spin_unlock(&wdm_device_list_lock);
@@ -893,7 +1006,7 @@ static int wdm_probe(struct usb_interface *intf, const struct usb_device_id *id)
 		goto err;
 	ep = &iface->endpoint[0].desc;
 
-	rv = wdm_create(intf, ep, maxcom, &wdm_manage_power);
+	rv = wdm_create(intf, ep, maxcom, &wdm_manage_power, false);
 
 err:
 	return rv;
@@ -925,7 +1038,7 @@ struct usb_driver *usb_cdc_wdm_register(struct usb_interface *intf,
 {
 	int rv = -EINVAL;
 
-	rv = wdm_create(intf, ep, bufsize, manage_power);
+	rv = wdm_create(intf, ep, bufsize, manage_power, true);
 	if (rv < 0)
 		goto err;
 
@@ -967,7 +1080,7 @@ static void wdm_disconnect(struct usb_interface *intf)
 	if (!desc->count)
 		cleanup(desc);
 	else
-		dev_dbg(&intf->dev, "%s: %d open files - postponing cleanup\n", __func__, desc->count);
+		dev_dbg(&intf->dev, "%d open files - postponing cleanup\n", desc->count);
 	mutex_unlock(&wdm_mutex);
 }
 
diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c
index 917a55c4480d..a6c1fae7d52a 100644
--- a/drivers/usb/class/usbtmc.c
+++ b/drivers/usb/class/usbtmc.c
@@ -141,6 +141,7 @@ static void usbtmc_delete(struct kref *kref)
 	struct usbtmc_device_data *data = to_usbtmc_data(kref);
 
 	usb_put_dev(data->usb_dev);
+	kfree(data);
 }
 
 static int usbtmc_open(struct inode *inode, struct file *filp)
@@ -1379,7 +1380,7 @@ static int usbtmc_probe(struct usb_interface *intf,
 
 	dev_dbg(&intf->dev, "%s called\n", __func__);
 
-	data = devm_kzalloc(&intf->dev, sizeof(*data), GFP_KERNEL);
+	data = kmalloc(sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
@@ -1467,10 +1468,8 @@ static int usbtmc_probe(struct usb_interface *intf,
 	if (data->iin_ep_present) {
 		/* allocate int urb */
 		data->iin_urb = usb_alloc_urb(0, GFP_KERNEL);
-		if (!data->iin_urb) {
-			dev_err(&intf->dev, "Failed to allocate int urb\n");
+		if (!data->iin_urb)
 			goto error_register;
-		}
 
 		/* will reference data in int urb */
 		kref_get(&data->kref);
@@ -1478,10 +1477,8 @@ static int usbtmc_probe(struct usb_interface *intf,
 		/* allocate buffer for interrupt in */
 		data->iin_buffer = kmalloc(data->iin_wMaxPacketSize,
 					GFP_KERNEL);
-		if (!data->iin_buffer) {
-			dev_err(&intf->dev, "Failed to allocate int buf\n");
+		if (!data->iin_buffer)
 			goto error_register;
-		}
 
 		/* fill interrupt urb */
 		usb_fill_int_urb(data->iin_urb, data->usb_dev,
diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c
index 01c0c0477a9e..8b317702d761 100644
--- a/drivers/usb/common/ulpi.c
+++ b/drivers/usb/common/ulpi.c
@@ -21,13 +21,13 @@
 
 int ulpi_read(struct ulpi *ulpi, u8 addr)
 {
-	return ulpi->ops->read(ulpi->ops, addr);
+	return ulpi->ops->read(ulpi->dev.parent, addr);
 }
 EXPORT_SYMBOL_GPL(ulpi_read);
 
 int ulpi_write(struct ulpi *ulpi, u8 addr, u8 val)
 {
-	return ulpi->ops->write(ulpi->ops, addr, val);
+	return ulpi->ops->write(ulpi->dev.parent, addr, val);
 }
 EXPORT_SYMBOL_GPL(ulpi_write);
 
@@ -127,16 +127,17 @@ static struct device_type ulpi_dev_type = {
  *
  * Registers a driver with the ULPI bus.
  */
-int ulpi_register_driver(struct ulpi_driver *drv)
+int __ulpi_register_driver(struct ulpi_driver *drv, struct module *module)
 {
 	if (!drv->probe)
 		return -EINVAL;
 
+	drv->driver.owner = module;
 	drv->driver.bus = &ulpi_bus;
 
 	return driver_register(&drv->driver);
 }
-EXPORT_SYMBOL_GPL(ulpi_register_driver);
+EXPORT_SYMBOL_GPL(__ulpi_register_driver);
 
 /**
  * ulpi_unregister_driver - unregister a driver with the ULPI bus
@@ -156,6 +157,8 @@ static int ulpi_register(struct device *dev, struct ulpi *ulpi)
 {
 	int ret;
 
+	ulpi->dev.parent = dev; /* needed early for ops */
+
 	/* Test the interface */
 	ret = ulpi_write(ulpi, ULPI_SCRATCH, 0xaa);
 	if (ret < 0)
@@ -174,7 +177,6 @@ static int ulpi_register(struct device *dev, struct ulpi *ulpi)
 	ulpi->id.product = ulpi_read(ulpi, ULPI_PRODUCT_ID_LOW);
 	ulpi->id.product |= ulpi_read(ulpi, ULPI_PRODUCT_ID_HIGH) << 8;
 
-	ulpi->dev.parent = dev;
 	ulpi->dev.bus = &ulpi_bus;
 	ulpi->dev.type = &ulpi_dev_type;
 	dev_set_name(&ulpi->dev, "%s.ulpi", dev_name(dev));
@@ -201,7 +203,8 @@ static int ulpi_register(struct device *dev, struct ulpi *ulpi)
  * Allocates and registers a ULPI device and an interface for it. Called from
  * the USB controller that provides the ULPI interface.
  */
-struct ulpi *ulpi_register_interface(struct device *dev, struct ulpi_ops *ops)
+struct ulpi *ulpi_register_interface(struct device *dev,
+				     const struct ulpi_ops *ops)
 {
 	struct ulpi *ulpi;
 	int ret;
@@ -211,7 +214,6 @@ struct ulpi *ulpi_register_interface(struct device *dev, struct ulpi_ops *ops)
 		return ERR_PTR(-ENOMEM);
 
 	ulpi->ops = ops;
-	ops->dev = dev;
 
 	ret = ulpi_register(dev, ulpi);
 	if (ret) {
diff --git a/drivers/usb/core/Kconfig b/drivers/usb/core/Kconfig
index dd280108758f..0e5a889742b3 100644
--- a/drivers/usb/core/Kconfig
+++ b/drivers/usb/core/Kconfig
@@ -83,23 +83,10 @@ config USB_OTG_FSM
 	  Implements OTG Finite State Machine as specified in On-The-Go
 	  and Embedded Host Supplement to the USB Revision 2.0 Specification.
 
-config USB_ULPI_BUS
-	tristate "USB ULPI PHY interface support"
-	depends on USB_SUPPORT
+config USB_LEDS_TRIGGER_USBPORT
+	tristate "USB port LED trigger"
+	depends on USB && LEDS_TRIGGERS
 	help
-	  UTMI+ Low Pin Interface (ULPI) is specification for a commonly used
-	  USB 2.0 PHY interface. The ULPI specification defines a standard set
-	  of registers that can be used to detect the vendor and product which
-	  allows ULPI to be handled as a bus. This module is the driver for that
-	  bus.
-
-	  The ULPI interfaces (the buses) are registered by the drivers for USB
-	  controllers which support ULPI register access and have ULPI PHY
-	  attached to them. The ULPI PHY drivers themselves are normal PHY
-	  drivers.
-
-	  ULPI PHYs provide often functions such as ADP sensing/probing (OTG
-	  protocol) and USB charger detection.
-
-	  To compile this driver as a module, choose M here: the module will
-	  be called ulpi.
+	  This driver allows LEDs to be controlled by USB events. Enabling this
+	  trigger allows specifying list of USB ports that should turn on LED
+	  when some USB device gets connected.
diff --git a/drivers/usb/core/Makefile b/drivers/usb/core/Makefile
index 9780877010b4..b99b871c4b9d 100644
--- a/drivers/usb/core/Makefile
+++ b/drivers/usb/core/Makefile
@@ -5,9 +5,12 @@
 usbcore-y := usb.o hub.o hcd.o urb.o message.o driver.o
 usbcore-y += config.o file.o buffer.o sysfs.o endpoint.o
 usbcore-y += devio.o notify.o generic.o quirks.o devices.o
-usbcore-y += port.o of.o
+usbcore-y += port.o
 
+usbcore-$(CONFIG_OF)		+= of.o
 usbcore-$(CONFIG_PCI)		+= hcd-pci.o
 usbcore-$(CONFIG_ACPI)		+= usb-acpi.o
 
 obj-$(CONFIG_USB)		+= usbcore.o
+
+obj-$(CONFIG_USB_LEDS_TRIGGER_USBPORT)	+= ledtrig-usbport.o
diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c
index 31ccdccd7a04..a2d90aca779f 100644
--- a/drivers/usb/core/config.c
+++ b/drivers/usb/core/config.c
@@ -171,6 +171,31 @@ static void usb_parse_ss_endpoint_companion(struct device *ddev, int cfgno,
 							ep, buffer, size);
 }
 
+static const unsigned short low_speed_maxpacket_maxes[4] = {
+	[USB_ENDPOINT_XFER_CONTROL] = 8,
+	[USB_ENDPOINT_XFER_ISOC] = 0,
+	[USB_ENDPOINT_XFER_BULK] = 0,
+	[USB_ENDPOINT_XFER_INT] = 8,
+};
+static const unsigned short full_speed_maxpacket_maxes[4] = {
+	[USB_ENDPOINT_XFER_CONTROL] = 64,
+	[USB_ENDPOINT_XFER_ISOC] = 1023,
+	[USB_ENDPOINT_XFER_BULK] = 64,
+	[USB_ENDPOINT_XFER_INT] = 64,
+};
+static const unsigned short high_speed_maxpacket_maxes[4] = {
+	[USB_ENDPOINT_XFER_CONTROL] = 64,
+	[USB_ENDPOINT_XFER_ISOC] = 1024,
+	[USB_ENDPOINT_XFER_BULK] = 512,
+	[USB_ENDPOINT_XFER_INT] = 1024,
+};
+static const unsigned short super_speed_maxpacket_maxes[4] = {
+	[USB_ENDPOINT_XFER_CONTROL] = 512,
+	[USB_ENDPOINT_XFER_ISOC] = 1024,
+	[USB_ENDPOINT_XFER_BULK] = 1024,
+	[USB_ENDPOINT_XFER_INT] = 1024,
+};
+
 static int usb_parse_endpoint(struct device *ddev, int cfgno, int inum,
     int asnum, struct usb_host_interface *ifp, int num_ep,
     unsigned char *buffer, int size)
@@ -179,6 +204,8 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, int inum,
 	struct usb_endpoint_descriptor *d;
 	struct usb_host_endpoint *endpoint;
 	int n, i, j, retval;
+	unsigned int maxp;
+	const unsigned short *maxpacket_maxes;
 
 	d = (struct usb_endpoint_descriptor *) buffer;
 	buffer += d->bLength;
@@ -213,8 +240,10 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, int inum,
 	memcpy(&endpoint->desc, d, n);
 	INIT_LIST_HEAD(&endpoint->urb_list);
 
-	/* Fix up bInterval values outside the legal range. Use 32 ms if no
-	 * proper value can be guessed. */
+	/*
+	 * Fix up bInterval values outside the legal range.
+	 * Use 10 or 8 ms if no proper value can be guessed.
+	 */
 	i = 0;		/* i = min, j = max, n = default */
 	j = 255;
 	if (usb_endpoint_xfer_int(d)) {
@@ -223,13 +252,15 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, int inum,
 		case USB_SPEED_SUPER_PLUS:
 		case USB_SPEED_SUPER:
 		case USB_SPEED_HIGH:
-			/* Many device manufacturers are using full-speed
+			/*
+			 * Many device manufacturers are using full-speed
 			 * bInterval values in high-speed interrupt endpoint
-			 * descriptors. Try to fix those and fall back to a
-			 * 32 ms default value otherwise. */
+			 * descriptors. Try to fix those and fall back to an
+			 * 8-ms default value otherwise.
+			 */
 			n = fls(d->bInterval*8);
 			if (n == 0)
-				n = 9;	/* 32 ms = 2^(9-1) uframes */
+				n = 7;	/* 8 ms = 2^(7-1) uframes */
 			j = 16;
 
 			/*
@@ -244,10 +275,12 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, int inum,
 			}
 			break;
 		default:		/* USB_SPEED_FULL or _LOW */
-			/* For low-speed, 10 ms is the official minimum.
+			/*
+			 * For low-speed, 10 ms is the official minimum.
 			 * But some "overclocked" devices might want faster
-			 * polling so we'll allow it. */
-			n = 32;
+			 * polling so we'll allow it.
+			 */
+			n = 10;
 			break;
 		}
 	} else if (usb_endpoint_xfer_isoc(d)) {
@@ -255,10 +288,10 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, int inum,
 		j = 16;
 		switch (to_usb_device(ddev)->speed) {
 		case USB_SPEED_HIGH:
-			n = 9;		/* 32 ms = 2^(9-1) uframes */
+			n = 7;		/* 8 ms = 2^(7-1) uframes */
 			break;
 		default:		/* USB_SPEED_FULL */
-			n = 6;		/* 32 ms = 2^(6-1) frames */
+			n = 4;		/* 8 ms = 2^(4-1) frames */
 			break;
 		}
 	}
@@ -286,6 +319,42 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, int inum,
 			endpoint->desc.wMaxPacketSize = cpu_to_le16(8);
 	}
 
+	/* Validate the wMaxPacketSize field */
+	maxp = usb_endpoint_maxp(&endpoint->desc);
+
+	/* Find the highest legal maxpacket size for this endpoint */
+	i = 0;		/* additional transactions per microframe */
+	switch (to_usb_device(ddev)->speed) {
+	case USB_SPEED_LOW:
+		maxpacket_maxes = low_speed_maxpacket_maxes;
+		break;
+	case USB_SPEED_FULL:
+		maxpacket_maxes = full_speed_maxpacket_maxes;
+		break;
+	case USB_SPEED_HIGH:
+		/* Bits 12..11 are allowed only for HS periodic endpoints */
+		if (usb_endpoint_xfer_int(d) || usb_endpoint_xfer_isoc(d)) {
+			i = maxp & (BIT(12) | BIT(11));
+			maxp &= ~i;
+		}
+		/* fallthrough */
+	default:
+		maxpacket_maxes = high_speed_maxpacket_maxes;
+		break;
+	case USB_SPEED_SUPER:
+	case USB_SPEED_SUPER_PLUS:
+		maxpacket_maxes = super_speed_maxpacket_maxes;
+		break;
+	}
+	j = maxpacket_maxes[usb_endpoint_type(&endpoint->desc)];
+
+	if (maxp > j) {
+		dev_warn(ddev, "config %d interface %d altsetting %d endpoint 0x%X has invalid maxpacket %d, setting to %d\n",
+		    cfgno, inum, asnum, d->bEndpointAddress, maxp, j);
+		maxp = j;
+		endpoint->desc.wMaxPacketSize = cpu_to_le16(i | maxp);
+	}
+
 	/*
 	 * Some buggy high speed devices have bulk endpoints using
 	 * maxpacket sizes other than 512.  High speed HCDs may not
@@ -293,9 +362,6 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, int inum,
 	 */
 	if (to_usb_device(ddev)->speed == USB_SPEED_HIGH
 			&& usb_endpoint_xfer_bulk(d)) {
-		unsigned maxp;
-
-		maxp = usb_endpoint_maxp(&endpoint->desc) & 0x07ff;
 		if (maxp != 512)
 			dev_warn(ddev, "config %d interface %d altsetting %d "
 				"bulk endpoint 0x%X has invalid maxpacket %d\n",
diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c
index e9f5043a2167..4016dae7433b 100644
--- a/drivers/usb/core/devio.c
+++ b/drivers/usb/core/devio.c
@@ -241,7 +241,8 @@ static int usbdev_mmap(struct file *file, struct vm_area_struct *vma)
 		goto error_decrease_mem;
 	}
 
-	mem = usb_alloc_coherent(ps->dev, size, GFP_USER, &dma_handle);
+	mem = usb_alloc_coherent(ps->dev, size, GFP_USER | __GFP_NOWARN,
+			&dma_handle);
 	if (!mem) {
 		ret = -ENOMEM;
 		goto error_free_usbm;
@@ -1708,11 +1709,17 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb
 	as->urb->start_frame = uurb->start_frame;
 	as->urb->number_of_packets = number_of_packets;
 	as->urb->stream_id = stream_id;
-	if (uurb->type == USBDEVFS_URB_TYPE_ISO ||
-			ps->dev->speed == USB_SPEED_HIGH)
-		as->urb->interval = 1 << min(15, ep->desc.bInterval - 1);
-	else
-		as->urb->interval = ep->desc.bInterval;
+
+	if (ep->desc.bInterval) {
+		if (uurb->type == USBDEVFS_URB_TYPE_ISO ||
+				ps->dev->speed == USB_SPEED_HIGH ||
+				ps->dev->speed >= USB_SPEED_SUPER)
+			as->urb->interval = 1 <<
+					min(15, ep->desc.bInterval - 1);
+		else
+			as->urb->interval = ep->desc.bInterval;
+	}
+
 	as->urb->context = as;
 	as->urb->complete = async_completed;
 	for (totlen = u = 0; u < number_of_packets; u++) {
@@ -2402,21 +2409,21 @@ static long usbdev_do_ioctl(struct file *file, unsigned int cmd,
 		snoop(&dev->dev, "%s: CONTROL\n", __func__);
 		ret = proc_control(ps, p);
 		if (ret >= 0)
-			inode->i_mtime = CURRENT_TIME;
+			inode->i_mtime = current_time(inode);
 		break;
 
 	case USBDEVFS_BULK:
 		snoop(&dev->dev, "%s: BULK\n", __func__);
 		ret = proc_bulk(ps, p);
 		if (ret >= 0)
-			inode->i_mtime = CURRENT_TIME;
+			inode->i_mtime = current_time(inode);
 		break;
 
 	case USBDEVFS_RESETEP:
 		snoop(&dev->dev, "%s: RESETEP\n", __func__);
 		ret = proc_resetep(ps, p);
 		if (ret >= 0)
-			inode->i_mtime = CURRENT_TIME;
+			inode->i_mtime = current_time(inode);
 		break;
 
 	case USBDEVFS_RESET:
@@ -2428,7 +2435,7 @@ static long usbdev_do_ioctl(struct file *file, unsigned int cmd,
 		snoop(&dev->dev, "%s: CLEAR_HALT\n", __func__);
 		ret = proc_clearhalt(ps, p);
 		if (ret >= 0)
-			inode->i_mtime = CURRENT_TIME;
+			inode->i_mtime = current_time(inode);
 		break;
 
 	case USBDEVFS_GETDRIVER:
@@ -2455,7 +2462,7 @@ static long usbdev_do_ioctl(struct file *file, unsigned int cmd,
 		snoop(&dev->dev, "%s: SUBMITURB\n", __func__);
 		ret = proc_submiturb(ps, p);
 		if (ret >= 0)
-			inode->i_mtime = CURRENT_TIME;
+			inode->i_mtime = current_time(inode);
 		break;
 
 #ifdef CONFIG_COMPAT
@@ -2463,14 +2470,14 @@ static long usbdev_do_ioctl(struct file *file, unsigned int cmd,
 		snoop(&dev->dev, "%s: CONTROL32\n", __func__);
 		ret = proc_control_compat(ps, p);
 		if (ret >= 0)
-			inode->i_mtime = CURRENT_TIME;
+			inode->i_mtime = current_time(inode);
 		break;
 
 	case USBDEVFS_BULK32:
 		snoop(&dev->dev, "%s: BULK32\n", __func__);
 		ret = proc_bulk_compat(ps, p);
 		if (ret >= 0)
-			inode->i_mtime = CURRENT_TIME;
+			inode->i_mtime = current_time(inode);
 		break;
 
 	case USBDEVFS_DISCSIGNAL32:
@@ -2482,7 +2489,7 @@ static long usbdev_do_ioctl(struct file *file, unsigned int cmd,
 		snoop(&dev->dev, "%s: SUBMITURB32\n", __func__);
 		ret = proc_submiturb_compat(ps, p);
 		if (ret >= 0)
-			inode->i_mtime = CURRENT_TIME;
+			inode->i_mtime = current_time(inode);
 		break;
 
 	case USBDEVFS_IOCTL32:
@@ -2545,7 +2552,7 @@ static long usbdev_do_ioctl(struct file *file, unsigned int cmd,
  done:
 	usb_unlock_device(dev);
 	if (ret >= 0)
-		inode->i_atime = CURRENT_TIME;
+		inode->i_atime = current_time(inode);
 	return ret;
 }
 
@@ -2582,7 +2589,9 @@ static unsigned int usbdev_poll(struct file *file,
 	if (file->f_mode & FMODE_WRITE && !list_empty(&ps->async_completed))
 		mask |= POLLOUT | POLLWRNORM;
 	if (!connected(ps))
-		mask |= POLLERR | POLLHUP;
+		mask |= POLLHUP;
+	if (list_empty(&ps->list))
+		mask |= POLLERR;
 	return mask;
 }
 
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index d2e3f655c26f..479e223f9cff 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -46,6 +46,7 @@
 #include <linux/usb.h>
 #include <linux/usb/hcd.h>
 #include <linux/usb/phy.h>
+#include <linux/usb/otg.h>
 
 #include "usb.h"
 
@@ -2517,10 +2518,8 @@ struct usb_hcd *usb_create_shared_hcd(const struct hc_driver *driver,
 	struct usb_hcd *hcd;
 
 	hcd = kzalloc(sizeof(*hcd) + driver->hcd_priv_size, GFP_KERNEL);
-	if (!hcd) {
-		dev_dbg (dev, "hcd alloc failed\n");
+	if (!hcd)
 		return NULL;
-	}
 	if (primary_hcd == NULL) {
 		hcd->address0_mutex = kmalloc(sizeof(*hcd->address0_mutex),
 				GFP_KERNEL);
@@ -3033,7 +3032,7 @@ EXPORT_SYMBOL_GPL(usb_hcd_platform_shutdown);
 
 /*-------------------------------------------------------------------------*/
 
-#if defined(CONFIG_USB_MON) || defined(CONFIG_USB_MON_MODULE)
+#if IS_ENABLED(CONFIG_USB_MON)
 
 const struct usb_mon_operations *mon_ops;
 
diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c
index bee13517676f..cbb146736f57 100644
--- a/drivers/usb/core/hub.c
+++ b/drivers/usb/core/hub.c
@@ -1052,14 +1052,11 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type)
 
 	/* Continue a partial initialization */
 	if (type == HUB_INIT2 || type == HUB_INIT3) {
-		device_lock(hub->intfdev);
+		device_lock(&hdev->dev);
 
 		/* Was the hub disconnected while we were waiting? */
-		if (hub->disconnected) {
-			device_unlock(hub->intfdev);
-			kref_put(&hub->kref, hub_release);
-			return;
-		}
+		if (hub->disconnected)
+			goto disconnected;
 		if (type == HUB_INIT2)
 			goto init2;
 		goto init3;
@@ -1262,7 +1259,7 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type)
 			queue_delayed_work(system_power_efficient_wq,
 					&hub->init_work,
 					msecs_to_jiffies(delay));
-			device_unlock(hub->intfdev);
+			device_unlock(&hdev->dev);
 			return;		/* Continues at init3: below */
 		} else {
 			msleep(delay);
@@ -1281,12 +1278,12 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type)
 	/* Scan all ports that need attention */
 	kick_hub_wq(hub);
 
-	/* Allow autosuspend if it was suppressed */
-	if (type <= HUB_INIT3)
+	if (type == HUB_INIT2 || type == HUB_INIT3) {
+		/* Allow autosuspend if it was suppressed */
+ disconnected:
 		usb_autopm_put_interface_async(to_usb_interface(hub->intfdev));
-
-	if (type == HUB_INIT2 || type == HUB_INIT3)
-		device_unlock(hub->intfdev);
+		device_unlock(&hdev->dev);
+	}
 
 	kref_put(&hub->kref, hub_release);
 }
@@ -1315,8 +1312,6 @@ static void hub_quiesce(struct usb_hub *hub, enum hub_quiescing_type type)
 	struct usb_device *hdev = hub->hdev;
 	int i;
 
-	cancel_delayed_work_sync(&hub->init_work);
-
 	/* hub_wq and related activity won't re-trigger */
 	hub->quiescing = 1;
 
@@ -1828,10 +1823,8 @@ descriptor_error:
 	dev_info(&intf->dev, "USB hub found\n");
 
 	hub = kzalloc(sizeof(*hub), GFP_KERNEL);
-	if (!hub) {
-		dev_dbg(&intf->dev, "couldn't kmalloc hub struct\n");
+	if (!hub)
 		return -ENOMEM;
-	}
 
 	kref_init(&hub->kref);
 	hub->intfdev = &intf->dev;
@@ -3111,7 +3104,7 @@ static int usb_disable_remote_wakeup(struct usb_device *udev)
 				USB_CTRL_SET_TIMEOUT);
 	else
 		return usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
-				USB_REQ_CLEAR_FEATURE, USB_RECIP_INTERFACE,
+				USB_REQ_SET_FEATURE, USB_RECIP_INTERFACE,
 				USB_INTRF_FUNC_SUSPEND,	0, NULL, 0,
 				USB_CTRL_SET_TIMEOUT);
 }
@@ -5342,11 +5335,10 @@ static int descriptors_changed(struct usb_device *udev,
 	}
 
 	buf = kmalloc(len, GFP_NOIO);
-	if (buf == NULL) {
-		dev_err(&udev->dev, "no mem to re-read configs after reset\n");
+	if (!buf)
 		/* assume the worst */
 		return 1;
-	}
+
 	for (index = 0; index < udev->descriptor.bNumConfigurations; index++) {
 		old_length = le16_to_cpu(udev->config[index].desc.wTotalLength);
 		length = usb_get_descriptor(udev, USB_DT_CONFIG, index, buf,
diff --git a/drivers/usb/core/ledtrig-usbport.c b/drivers/usb/core/ledtrig-usbport.c
new file mode 100644
index 000000000000..3ed5162677ad
--- /dev/null
+++ b/drivers/usb/core/ledtrig-usbport.c
@@ -0,0 +1,314 @@
+/*
+ * USB port LED trigger
+ *
+ * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/device.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/usb.h>
+
+struct usbport_trig_data {
+	struct led_classdev *led_cdev;
+	struct list_head ports;
+	struct notifier_block nb;
+	int count; /* Amount of connected matching devices */
+};
+
+struct usbport_trig_port {
+	struct usbport_trig_data *data;
+	struct usb_device *hub;
+	int portnum;
+	char *port_name;
+	bool observed;
+	struct device_attribute attr;
+	struct list_head list;
+};
+
+/***************************************
+ * Helpers
+ ***************************************/
+
+/**
+ * usbport_trig_usb_dev_observed - Check if dev is connected to observed port
+ */
+static bool usbport_trig_usb_dev_observed(struct usbport_trig_data *usbport_data,
+					  struct usb_device *usb_dev)
+{
+	struct usbport_trig_port *port;
+
+	if (!usb_dev->parent)
+		return false;
+
+	list_for_each_entry(port, &usbport_data->ports, list) {
+		if (usb_dev->parent == port->hub &&
+		    usb_dev->portnum == port->portnum)
+			return port->observed;
+	}
+
+	return false;
+}
+
+static int usbport_trig_usb_dev_check(struct usb_device *usb_dev, void *data)
+{
+	struct usbport_trig_data *usbport_data = data;
+
+	if (usbport_trig_usb_dev_observed(usbport_data, usb_dev))
+		usbport_data->count++;
+
+	return 0;
+}
+
+/**
+ * usbport_trig_update_count - Recalculate amount of connected matching devices
+ */
+static void usbport_trig_update_count(struct usbport_trig_data *usbport_data)
+{
+	struct led_classdev *led_cdev = usbport_data->led_cdev;
+
+	usbport_data->count = 0;
+	usb_for_each_dev(usbport_data, usbport_trig_usb_dev_check);
+	led_cdev->brightness_set(led_cdev,
+				 usbport_data->count ? LED_FULL : LED_OFF);
+}
+
+/***************************************
+ * Device attr
+ ***************************************/
+
+static ssize_t usbport_trig_port_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct usbport_trig_port *port = container_of(attr,
+						      struct usbport_trig_port,
+						      attr);
+
+	return sprintf(buf, "%d\n", port->observed) + 1;
+}
+
+static ssize_t usbport_trig_port_store(struct device *dev,
+				       struct device_attribute *attr,
+				       const char *buf, size_t size)
+{
+	struct usbport_trig_port *port = container_of(attr,
+						      struct usbport_trig_port,
+						      attr);
+
+	if (!strcmp(buf, "0") || !strcmp(buf, "0\n"))
+		port->observed = 0;
+	else if (!strcmp(buf, "1") || !strcmp(buf, "1\n"))
+		port->observed = 1;
+	else
+		return -EINVAL;
+
+	usbport_trig_update_count(port->data);
+
+	return size;
+}
+
+static struct attribute *ports_attrs[] = {
+	NULL,
+};
+static const struct attribute_group ports_group = {
+	.name = "ports",
+	.attrs = ports_attrs,
+};
+
+/***************************************
+ * Adding & removing ports
+ ***************************************/
+
+static int usbport_trig_add_port(struct usbport_trig_data *usbport_data,
+				 struct usb_device *usb_dev,
+				 const char *hub_name, int portnum)
+{
+	struct led_classdev *led_cdev = usbport_data->led_cdev;
+	struct usbport_trig_port *port;
+	size_t len;
+	int err;
+
+	port = kzalloc(sizeof(*port), GFP_KERNEL);
+	if (!port) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	port->data = usbport_data;
+	port->hub = usb_dev;
+	port->portnum = portnum;
+
+	len = strlen(hub_name) + 8;
+	port->port_name = kzalloc(len, GFP_KERNEL);
+	if (!port->port_name) {
+		err = -ENOMEM;
+		goto err_free_port;
+	}
+	snprintf(port->port_name, len, "%s-port%d", hub_name, portnum);
+
+	port->attr.attr.name = port->port_name;
+	port->attr.attr.mode = S_IRUSR | S_IWUSR;
+	port->attr.show = usbport_trig_port_show;
+	port->attr.store = usbport_trig_port_store;
+
+	err = sysfs_add_file_to_group(&led_cdev->dev->kobj, &port->attr.attr,
+				      ports_group.name);
+	if (err)
+		goto err_free_port_name;
+
+	list_add_tail(&port->list, &usbport_data->ports);
+
+	return 0;
+
+err_free_port_name:
+	kfree(port->port_name);
+err_free_port:
+	kfree(port);
+err_out:
+	return err;
+}
+
+static int usbport_trig_add_usb_dev_ports(struct usb_device *usb_dev,
+					  void *data)
+{
+	struct usbport_trig_data *usbport_data = data;
+	int i;
+
+	for (i = 1; i <= usb_dev->maxchild; i++)
+		usbport_trig_add_port(usbport_data, usb_dev,
+				      dev_name(&usb_dev->dev), i);
+
+	return 0;
+}
+
+static void usbport_trig_remove_port(struct usbport_trig_data *usbport_data,
+				     struct usbport_trig_port *port)
+{
+	struct led_classdev *led_cdev = usbport_data->led_cdev;
+
+	list_del(&port->list);
+	sysfs_remove_file_from_group(&led_cdev->dev->kobj, &port->attr.attr,
+				     ports_group.name);
+	kfree(port->port_name);
+	kfree(port);
+}
+
+static void usbport_trig_remove_usb_dev_ports(struct usbport_trig_data *usbport_data,
+					      struct usb_device *usb_dev)
+{
+	struct usbport_trig_port *port, *tmp;
+
+	list_for_each_entry_safe(port, tmp, &usbport_data->ports, list) {
+		if (port->hub == usb_dev)
+			usbport_trig_remove_port(usbport_data, port);
+	}
+}
+
+/***************************************
+ * Init, exit, etc.
+ ***************************************/
+
+static int usbport_trig_notify(struct notifier_block *nb, unsigned long action,
+			       void *data)
+{
+	struct usbport_trig_data *usbport_data =
+		container_of(nb, struct usbport_trig_data, nb);
+	struct led_classdev *led_cdev = usbport_data->led_cdev;
+	struct usb_device *usb_dev = data;
+	bool observed;
+
+	observed = usbport_trig_usb_dev_observed(usbport_data, usb_dev);
+
+	switch (action) {
+	case USB_DEVICE_ADD:
+		usbport_trig_add_usb_dev_ports(usb_dev, usbport_data);
+		if (observed && usbport_data->count++ == 0)
+			led_cdev->brightness_set(led_cdev, LED_FULL);
+		return NOTIFY_OK;
+	case USB_DEVICE_REMOVE:
+		usbport_trig_remove_usb_dev_ports(usbport_data, usb_dev);
+		if (observed && --usbport_data->count == 0)
+			led_cdev->brightness_set(led_cdev, LED_OFF);
+		return NOTIFY_OK;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static void usbport_trig_activate(struct led_classdev *led_cdev)
+{
+	struct usbport_trig_data *usbport_data;
+	int err;
+
+	usbport_data = kzalloc(sizeof(*usbport_data), GFP_KERNEL);
+	if (!usbport_data)
+		return;
+	usbport_data->led_cdev = led_cdev;
+
+	/* List of ports */
+	INIT_LIST_HEAD(&usbport_data->ports);
+	err = sysfs_create_group(&led_cdev->dev->kobj, &ports_group);
+	if (err)
+		goto err_free;
+	usb_for_each_dev(usbport_data, usbport_trig_add_usb_dev_ports);
+
+	/* Notifications */
+	usbport_data->nb.notifier_call = usbport_trig_notify,
+	led_cdev->trigger_data = usbport_data;
+	usb_register_notify(&usbport_data->nb);
+
+	led_cdev->activated = true;
+	return;
+
+err_free:
+	kfree(usbport_data);
+}
+
+static void usbport_trig_deactivate(struct led_classdev *led_cdev)
+{
+	struct usbport_trig_data *usbport_data = led_cdev->trigger_data;
+	struct usbport_trig_port *port, *tmp;
+
+	if (!led_cdev->activated)
+		return;
+
+	list_for_each_entry_safe(port, tmp, &usbport_data->ports, list) {
+		usbport_trig_remove_port(usbport_data, port);
+	}
+
+	usb_unregister_notify(&usbport_data->nb);
+
+	sysfs_remove_group(&led_cdev->dev->kobj, &ports_group);
+
+	kfree(usbport_data);
+
+	led_cdev->activated = false;
+}
+
+static struct led_trigger usbport_led_trigger = {
+	.name     = "usbport",
+	.activate = usbport_trig_activate,
+	.deactivate = usbport_trig_deactivate,
+};
+
+static int __init usbport_trig_init(void)
+{
+	return led_trigger_register(&usbport_led_trigger);
+}
+
+static void __exit usbport_trig_exit(void)
+{
+	led_trigger_unregister(&usbport_led_trigger);
+}
+
+module_init(usbport_trig_init);
+module_exit(usbport_trig_exit);
+
+MODULE_AUTHOR("Rafał Miłecki <rafal@milecki.pl>");
+MODULE_DESCRIPTION("USB port trigger");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/usb/core/message.c b/drivers/usb/core/message.c
index 0406a59f0551..3a4707746157 100644
--- a/drivers/usb/core/message.c
+++ b/drivers/usb/core/message.c
@@ -1760,17 +1760,14 @@ int usb_set_configuration(struct usb_device *dev, int configuration)
 		nintf = cp->desc.bNumInterfaces;
 		new_interfaces = kmalloc(nintf * sizeof(*new_interfaces),
 				GFP_NOIO);
-		if (!new_interfaces) {
-			dev_err(&dev->dev, "Out of memory\n");
+		if (!new_interfaces)
 			return -ENOMEM;
-		}
 
 		for (; n < nintf; ++n) {
 			new_interfaces[n] = kzalloc(
 					sizeof(struct usb_interface),
 					GFP_NOIO);
 			if (!new_interfaces[n]) {
-				dev_err(&dev->dev, "Out of memory\n");
 				ret = -ENOMEM;
 free_interfaces:
 				while (--n >= 0)
@@ -1862,7 +1859,12 @@ free_interfaces:
 		intf->dev.bus = &usb_bus_type;
 		intf->dev.type = &usb_if_device_type;
 		intf->dev.groups = usb_interface_groups;
+		/*
+		 * Please refer to usb_alloc_dev() to see why we set
+		 * dma_mask and dma_pfn_offset.
+		 */
 		intf->dev.dma_mask = dev->dev.dma_mask;
+		intf->dev.dma_pfn_offset = dev->dev.dma_pfn_offset;
 		INIT_WORK(&intf->reset_ws, __usb_queue_reset_device);
 		intf->minor = -1;
 		device_initialize(&intf->dev);
diff --git a/drivers/usb/core/of.c b/drivers/usb/core/of.c
index 2289700c31d6..3de4f8873984 100644
--- a/drivers/usb/core/of.c
+++ b/drivers/usb/core/of.c
@@ -18,6 +18,7 @@
  */
 
 #include <linux/of.h>
+#include <linux/usb/of.h>
 
 /**
  * usb_of_get_child_node - Find the device node match port number
diff --git a/drivers/usb/core/otg_whitelist.h b/drivers/usb/core/otg_whitelist.h
index a95b0c989c21..085049d37d7a 100644
--- a/drivers/usb/core/otg_whitelist.h
+++ b/drivers/usb/core/otg_whitelist.h
@@ -38,7 +38,7 @@ static struct usb_device_id whitelist_table[] = {
 { USB_DEVICE(0x0525, 0xa4a2), },
 #endif
 
-#if	defined(CONFIG_USB_TEST) || defined(CONFIG_USB_TEST_MODULE)
+#if	IS_ENABLED(CONFIG_USB_TEST)
 /* gadget zero, for testing */
 { USB_DEVICE(0x0525, 0xa4a0), },
 #endif
diff --git a/drivers/usb/core/urb.c b/drivers/usb/core/urb.c
index c601e25b609f..a9039696476e 100644
--- a/drivers/usb/core/urb.c
+++ b/drivers/usb/core/urb.c
@@ -68,10 +68,8 @@ struct urb *usb_alloc_urb(int iso_packets, gfp_t mem_flags)
 	urb = kmalloc(sizeof(struct urb) +
 		iso_packets * sizeof(struct usb_iso_packet_descriptor),
 		mem_flags);
-	if (!urb) {
-		printk(KERN_ERR "alloc_urb: kmalloc failed\n");
+	if (!urb)
 		return NULL;
-	}
 	usb_init_urb(urb);
 	return urb;
 }
diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c
index 5e80697ef952..592151461017 100644
--- a/drivers/usb/core/usb.c
+++ b/drivers/usb/core/usb.c
@@ -440,7 +440,18 @@ struct usb_device *usb_alloc_dev(struct usb_device *parent,
 	dev->dev.bus = &usb_bus_type;
 	dev->dev.type = &usb_device_type;
 	dev->dev.groups = usb_device_groups;
+	/*
+	 * Fake a dma_mask/offset for the USB device:
+	 * We cannot really use the dma-mapping API (dma_alloc_* and
+	 * dma_map_*) for USB devices but instead need to use
+	 * usb_alloc_coherent and pass data in 'urb's, but some subsystems
+	 * manually look into the mask/offset pair to determine whether
+	 * they need bounce buffers.
+	 * Note: calling dma_set_mask() on a USB device would set the
+	 * mask for the entire HCD, so don't do that.
+	 */
 	dev->dev.dma_mask = bus->controller->dma_mask;
+	dev->dev.dma_pfn_offset = bus->controller->dma_pfn_offset;
 	set_dev_node(&dev->dev, dev_to_node(bus->controller));
 	dev->state = USB_STATE_ATTACHED;
 	dev->lpm_disable_count = 1;
diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c
index 4135a5ff67ca..fa9b26b91507 100644
--- a/drivers/usb/dwc2/core.c
+++ b/drivers/usb/dwc2/core.c
@@ -238,6 +238,77 @@ int dwc2_enter_hibernation(struct dwc2_hsotg *hsotg)
 	return ret;
 }
 
+/**
+ * dwc2_wait_for_mode() - Waits for the controller mode.
+ * @hsotg:	Programming view of the DWC_otg controller.
+ * @host_mode:	If true, waits for host mode, otherwise device mode.
+ */
+static void dwc2_wait_for_mode(struct dwc2_hsotg *hsotg,
+			       bool host_mode)
+{
+	ktime_t start;
+	ktime_t end;
+	unsigned int timeout = 110;
+
+	dev_vdbg(hsotg->dev, "Waiting for %s mode\n",
+		 host_mode ? "host" : "device");
+
+	start = ktime_get();
+
+	while (1) {
+		s64 ms;
+
+		if (dwc2_is_host_mode(hsotg) == host_mode) {
+			dev_vdbg(hsotg->dev, "%s mode set\n",
+				 host_mode ? "Host" : "Device");
+			break;
+		}
+
+		end = ktime_get();
+		ms = ktime_to_ms(ktime_sub(end, start));
+
+		if (ms >= (s64)timeout) {
+			dev_warn(hsotg->dev, "%s: Couldn't set %s mode\n",
+				 __func__, host_mode ? "host" : "device");
+			break;
+		}
+
+		usleep_range(1000, 2000);
+	}
+}
+
+/**
+ * dwc2_iddig_filter_enabled() - Returns true if the IDDIG debounce
+ * filter is enabled.
+ */
+static bool dwc2_iddig_filter_enabled(struct dwc2_hsotg *hsotg)
+{
+	u32 gsnpsid;
+	u32 ghwcfg4;
+
+	if (!dwc2_hw_is_otg(hsotg))
+		return false;
+
+	/* Check if core configuration includes the IDDIG filter. */
+	ghwcfg4 = dwc2_readl(hsotg->regs + GHWCFG4);
+	if (!(ghwcfg4 & GHWCFG4_IDDIG_FILT_EN))
+		return false;
+
+	/*
+	 * Check if the IDDIG debounce filter is bypassed. Available
+	 * in core version >= 3.10a.
+	 */
+	gsnpsid = dwc2_readl(hsotg->regs + GSNPSID);
+	if (gsnpsid >= DWC2_CORE_REV_3_10a) {
+		u32 gotgctl = dwc2_readl(hsotg->regs + GOTGCTL);
+
+		if (gotgctl & GOTGCTL_DBNCE_FLTR_BYPASS)
+			return false;
+	}
+
+	return true;
+}
+
 /*
  * Do core a soft reset of the core.  Be careful with this because it
  * resets all the internal state machines of the core.
@@ -246,9 +317,30 @@ int dwc2_core_reset(struct dwc2_hsotg *hsotg)
 {
 	u32 greset;
 	int count = 0;
+	bool wait_for_host_mode = false;
 
 	dev_vdbg(hsotg->dev, "%s()\n", __func__);
 
+	/*
+	 * If the current mode is host, either due to the force mode
+	 * bit being set (which persists after core reset) or the
+	 * connector id pin, a core soft reset will temporarily reset
+	 * the mode to device. A delay from the IDDIG debounce filter
+	 * will occur before going back to host mode.
+	 *
+	 * Determine whether we will go back into host mode after a
+	 * reset and account for this delay after the reset.
+	 */
+	if (dwc2_iddig_filter_enabled(hsotg)) {
+		u32 gotgctl = dwc2_readl(hsotg->regs + GOTGCTL);
+		u32 gusbcfg = dwc2_readl(hsotg->regs + GUSBCFG);
+
+		if (!(gotgctl & GOTGCTL_CONID_B) ||
+		    (gusbcfg & GUSBCFG_FORCEHOSTMODE)) {
+			wait_for_host_mode = true;
+		}
+	}
+
 	/* Core Soft Reset */
 	greset = dwc2_readl(hsotg->regs + GRSTCTL);
 	greset |= GRSTCTL_CSFTRST;
@@ -277,6 +369,9 @@ int dwc2_core_reset(struct dwc2_hsotg *hsotg)
 		}
 	} while (!(greset & GRSTCTL_AHBIDLE));
 
+	if (wait_for_host_mode)
+		dwc2_wait_for_mode(hsotg, true);
+
 	return 0;
 }
 
@@ -300,9 +395,9 @@ int dwc2_core_reset(struct dwc2_hsotg *hsotg)
  * Checks are done in this function to determine whether doing a force
  * would be valid or not.
  *
- * If a force is done, it requires a 25ms delay to take effect.
- *
- * Returns true if the mode was forced.
+ * If a force is done, it requires a IDDIG debounce filter delay if
+ * the filter is configured and enabled. We poll the current mode of
+ * the controller to account for this delay.
  */
 static bool dwc2_force_mode(struct dwc2_hsotg *hsotg, bool host)
 {
@@ -337,12 +432,18 @@ static bool dwc2_force_mode(struct dwc2_hsotg *hsotg, bool host)
 	gusbcfg |= set;
 	dwc2_writel(gusbcfg, hsotg->regs + GUSBCFG);
 
-	msleep(25);
+	dwc2_wait_for_mode(hsotg, host);
 	return true;
 }
 
-/*
- * Clears the force mode bits.
+/**
+ * dwc2_clear_force_mode() - Clears the force mode bits.
+ *
+ * After clearing the bits, wait up to 100 ms to account for any
+ * potential IDDIG filter delay. We can't know if we expect this delay
+ * or not because the value of the connector ID status is affected by
+ * the force mode. We only need to call this once during probe if
+ * dr_mode == OTG.
  */
 static void dwc2_clear_force_mode(struct dwc2_hsotg *hsotg)
 {
@@ -353,11 +454,8 @@ static void dwc2_clear_force_mode(struct dwc2_hsotg *hsotg)
 	gusbcfg &= ~GUSBCFG_FORCEDEVMODE;
 	dwc2_writel(gusbcfg, hsotg->regs + GUSBCFG);
 
-	/*
-	 * NOTE: This long sleep is _very_ important, otherwise the core will
-	 * not stay in host mode after a connector ID change!
-	 */
-	msleep(25);
+	if (dwc2_iddig_filter_enabled(hsotg))
+		usleep_range(100000, 110000);
 }
 
 /*
@@ -380,12 +478,6 @@ void dwc2_force_dr_mode(struct dwc2_hsotg *hsotg)
 			 __func__, hsotg->dr_mode);
 		break;
 	}
-
-	/*
-	 * NOTE: This is required for some rockchip soc based
-	 * platforms.
-	 */
-	msleep(50);
 }
 
 /*
diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h
index 9fae0291cd69..aad4107ef927 100644
--- a/drivers/usb/dwc2/core.h
+++ b/drivers/usb/dwc2/core.h
@@ -259,13 +259,6 @@ enum dwc2_lx_state {
 	DWC2_L3,	/* Off state */
 };
 
-/*
- * Gadget periodic tx fifo sizes as used by legacy driver
- * EP0 is not included
- */
-#define DWC2_G_P_LEGACY_TX_FIFO_SIZE {256, 256, 256, 256, 768, 768, 768, \
-					   768, 0, 0, 0, 0, 0, 0, 0}
-
 /* Gadget ep0 states */
 enum dwc2_ep0_state {
 	DWC2_EP0_SETUP,
@@ -868,6 +861,7 @@ struct dwc2_hsotg {
 	void *priv;
 	int     irq;
 	struct clk *clk;
+	struct reset_control *reset;
 
 	unsigned int queuing_high_bandwidth:1;
 	unsigned int srp_success:1;
@@ -889,6 +883,7 @@ struct dwc2_hsotg {
 #define DWC2_CORE_REV_2_92a	0x4f54292a
 #define DWC2_CORE_REV_2_94a	0x4f54294a
 #define DWC2_CORE_REV_3_00a	0x4f54300a
+#define DWC2_CORE_REV_3_10a	0x4f54310a
 
 #if IS_ENABLED(CONFIG_USB_DWC2_HOST) || IS_ENABLED(CONFIG_USB_DWC2_DUAL_ROLE)
 	union dwc2_hcd_internal_flags {
diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c
index af46adfae41c..4cd6403a7566 100644
--- a/drivers/usb/dwc2/gadget.c
+++ b/drivers/usb/dwc2/gadget.c
@@ -186,9 +186,10 @@ static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg,
  */
 static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
 {
-	unsigned int ep;
+	unsigned int fifo;
 	unsigned int addr;
 	int timeout;
+	u32 dptxfsizn;
 	u32 val;
 
 	/* Reset fifo map if not correctly cleared during previous session */
@@ -216,16 +217,16 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
 	 * them to endpoints dynamically according to maxpacket size value of
 	 * given endpoint.
 	 */
-	for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) {
-		if (!hsotg->g_tx_fifo_sz[ep])
-			continue;
-		val = addr;
-		val |= hsotg->g_tx_fifo_sz[ep] << FIFOSIZE_DEPTH_SHIFT;
-		WARN_ONCE(addr + hsotg->g_tx_fifo_sz[ep] > hsotg->fifo_mem,
-			  "insufficient fifo memory");
-		addr += hsotg->g_tx_fifo_sz[ep];
+	for (fifo = 1; fifo < MAX_EPS_CHANNELS; fifo++) {
+		dptxfsizn = dwc2_readl(hsotg->regs + DPTXFSIZN(fifo));
+
+		val = (dptxfsizn & FIFOSIZE_DEPTH_MASK) | addr;
+		addr += dptxfsizn >> FIFOSIZE_DEPTH_SHIFT;
+
+		if (addr > hsotg->fifo_mem)
+			break;
 
-		dwc2_writel(val, hsotg->regs + DPTXFSIZN(ep));
+		dwc2_writel(val, hsotg->regs + DPTXFSIZN(fifo));
 	}
 
 	/*
@@ -388,7 +389,8 @@ static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg,
 			return -ENOSPC;
 		}
 	} else if (hsotg->dedicated_fifos && hs_ep->index != 0) {
-		can_write = dwc2_readl(hsotg->regs + DTXFSTS(hs_ep->index));
+		can_write = dwc2_readl(hsotg->regs +
+				DTXFSTS(hs_ep->fifo_index));
 
 		can_write &= 0xffff;
 		can_write *= 4;
@@ -2432,7 +2434,7 @@ static void kill_all_requests(struct dwc2_hsotg *hsotg,
 
 	if (!hsotg->dedicated_fifos)
 		return;
-	size = (dwc2_readl(hsotg->regs + DTXFSTS(ep->index)) & 0xffff) * 4;
+	size = (dwc2_readl(hsotg->regs + DTXFSTS(ep->fifo_index)) & 0xffff) * 4;
 	if (size < ep->fifo_size)
 		dwc2_hsotg_txfifo_flush(hsotg, ep->fifo_index);
 }
@@ -3041,22 +3043,11 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 		break;
 	}
 
-	/* If fifo is already allocated for this ep */
-	if (hs_ep->fifo_index) {
-		size =  hs_ep->ep.maxpacket * hs_ep->mc;
-		/* If bigger fifo is required deallocate current one */
-		if (size > hs_ep->fifo_size) {
-			hsotg->fifo_map &= ~(1 << hs_ep->fifo_index);
-			hs_ep->fifo_index = 0;
-			hs_ep->fifo_size = 0;
-		}
-	}
-
 	/*
 	 * if the hardware has dedicated fifos, we must give each IN EP
 	 * a unique tx-fifo even if it is non-periodic.
 	 */
-	if (dir_in && hsotg->dedicated_fifos && !hs_ep->fifo_index) {
+	if (dir_in && hsotg->dedicated_fifos) {
 		u32 fifo_index = 0;
 		u32 fifo_size = UINT_MAX;
 		size = hs_ep->ep.maxpacket*hs_ep->mc;
@@ -3129,10 +3120,6 @@ static int dwc2_hsotg_ep_disable(struct usb_ep *ep)
 
 	spin_lock_irqsave(&hsotg->lock, flags);
 
-	hsotg->fifo_map &= ~(1<<hs_ep->fifo_index);
-	hs_ep->fifo_index = 0;
-	hs_ep->fifo_size = 0;
-
 	ctrl = dwc2_readl(hsotg->regs + epctrl_reg);
 	ctrl &= ~DXEPCTL_EPENA;
 	ctrl &= ~DXEPCTL_USBACTEP;
@@ -3147,6 +3134,10 @@ static int dwc2_hsotg_ep_disable(struct usb_ep *ep)
 	/* terminate all requests with shutdown */
 	kill_all_requests(hsotg, hs_ep, -ESHUTDOWN);
 
+	hsotg->fifo_map &= ~(1 << hs_ep->fifo_index);
+	hs_ep->fifo_index = 0;
+	hs_ep->fifo_size = 0;
+
 	spin_unlock_irqrestore(&hsotg->lock, flags);
 	return 0;
 }
@@ -3475,8 +3466,11 @@ static int dwc2_hsotg_udc_start(struct usb_gadget *gadget,
 		otg_set_peripheral(hsotg->uphy->otg, &hsotg->gadget);
 
 	spin_lock_irqsave(&hsotg->lock, flags);
-	dwc2_hsotg_init(hsotg);
-	dwc2_hsotg_core_init_disconnected(hsotg, false);
+	if (dwc2_hw_is_device(hsotg)) {
+		dwc2_hsotg_init(hsotg);
+		dwc2_hsotg_core_init_disconnected(hsotg, false);
+	}
+
 	hsotg->enabled = 0;
 	spin_unlock_irqrestore(&hsotg->lock, flags);
 
@@ -3813,36 +3807,10 @@ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg)
 static void dwc2_hsotg_of_probe(struct dwc2_hsotg *hsotg)
 {
 	struct device_node *np = hsotg->dev->of_node;
-	u32 len = 0;
-	u32 i = 0;
 
 	/* Enable dma if requested in device tree */
 	hsotg->g_using_dma = of_property_read_bool(np, "g-use-dma");
 
-	/*
-	* Register TX periodic fifo size per endpoint.
-	* EP0 is excluded since it has no fifo configuration.
-	*/
-	if (!of_find_property(np, "g-tx-fifo-size", &len))
-		goto rx_fifo;
-
-	len /= sizeof(u32);
-
-	/* Read tx fifo sizes other than ep0 */
-	if (of_property_read_u32_array(np, "g-tx-fifo-size",
-						&hsotg->g_tx_fifo_sz[1], len))
-		goto rx_fifo;
-
-	/* Add ep0 */
-	len++;
-
-	/* Make remaining TX fifos unavailable */
-	if (len < MAX_EPS_CHANNELS) {
-		for (i = len; i < MAX_EPS_CHANNELS; i++)
-			hsotg->g_tx_fifo_sz[i] = 0;
-	}
-
-rx_fifo:
 	/* Register RX fifo size */
 	of_property_read_u32(np, "g-rx-fifo-size", &hsotg->g_rx_fifo_sz);
 
@@ -3864,13 +3832,10 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq)
 	struct device *dev = hsotg->dev;
 	int epnum;
 	int ret;
-	int i;
-	u32 p_tx_fifo[] = DWC2_G_P_LEGACY_TX_FIFO_SIZE;
 
 	/* Initialize to legacy fifo configuration values */
 	hsotg->g_rx_fifo_sz = 2048;
 	hsotg->g_np_g_tx_fifo_sz = 1024;
-	memcpy(&hsotg->g_tx_fifo_sz[1], p_tx_fifo, sizeof(p_tx_fifo));
 	/* Device tree specific probe */
 	dwc2_hsotg_of_probe(hsotg);
 
@@ -3888,9 +3853,6 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq)
 	dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n",
 						hsotg->g_np_g_tx_fifo_sz);
 	dev_dbg(dev, "RXFIFO size: %d\n", hsotg->g_rx_fifo_sz);
-	for (i = 0; i < MAX_EPS_CHANNELS; i++)
-		dev_dbg(dev, "Periodic TXFIFO%2d size: %d\n", i,
-						hsotg->g_tx_fifo_sz[i]);
 
 	hsotg->gadget.max_speed = USB_SPEED_HIGH;
 	hsotg->gadget.ops = &dwc2_hsotg_gadget_ops;
@@ -3908,17 +3870,13 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq)
 
 	hsotg->ctrl_buff = devm_kzalloc(hsotg->dev,
 			DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
-	if (!hsotg->ctrl_buff) {
-		dev_err(dev, "failed to allocate ctrl request buff\n");
+	if (!hsotg->ctrl_buff)
 		return -ENOMEM;
-	}
 
 	hsotg->ep0_buff = devm_kzalloc(hsotg->dev,
 			DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
-	if (!hsotg->ep0_buff) {
-		dev_err(dev, "failed to allocate ctrl reply buff\n");
+	if (!hsotg->ep0_buff)
 		return -ENOMEM;
-	}
 
 	ret = devm_request_irq(hsotg->dev, irq, dwc2_hsotg_irq, IRQF_SHARED,
 				dev_name(hsotg->dev), hsotg);
diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c
index 2df3d04d26f5..df5a06578005 100644
--- a/drivers/usb/dwc2/hcd.c
+++ b/drivers/usb/dwc2/hcd.c
@@ -5040,7 +5040,7 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq)
 
 	/* Create new workqueue and init work */
 	retval = -ENOMEM;
-	hsotg->wq_otg = create_singlethread_workqueue("dwc2");
+	hsotg->wq_otg = alloc_ordered_workqueue("dwc2", 0);
 	if (!hsotg->wq_otg) {
 		dev_err(hsotg->dev, "Failed to create workqueue\n");
 		goto error2;
diff --git a/drivers/usb/dwc2/hw.h b/drivers/usb/dwc2/hw.h
index efc3bcde2822..91058441e62a 100644
--- a/drivers/usb/dwc2/hw.h
+++ b/drivers/usb/dwc2/hw.h
@@ -48,6 +48,7 @@
 #define GOTGCTL_ASESVLD			(1 << 18)
 #define GOTGCTL_DBNC_SHORT		(1 << 17)
 #define GOTGCTL_CONID_B			(1 << 16)
+#define GOTGCTL_DBNCE_FLTR_BYPASS	(1 << 15)
 #define GOTGCTL_DEVHNPEN		(1 << 11)
 #define GOTGCTL_HSTSETHNPEN		(1 << 10)
 #define GOTGCTL_HNPREQ			(1 << 9)
diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c
index fc6f5251de5d..8e1728b39a49 100644
--- a/drivers/usb/dwc2/platform.c
+++ b/drivers/usb/dwc2/platform.c
@@ -45,6 +45,7 @@
 #include <linux/platform_device.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_data/s3c-hsotg.h>
+#include <linux/reset.h>
 
 #include <linux/usb/of.h>
 
@@ -181,6 +182,38 @@ static const struct dwc2_core_params params_ltq = {
 	.hibernation			= -1,
 };
 
+static const struct dwc2_core_params params_amlogic = {
+	.otg_cap			= DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE,
+	.otg_ver			= -1,
+	.dma_enable			= 1,
+	.dma_desc_enable		= 0,
+	.dma_desc_fs_enable		= 0,
+	.speed				= DWC2_SPEED_PARAM_HIGH,
+	.enable_dynamic_fifo		= 1,
+	.en_multiple_tx_fifo		= -1,
+	.host_rx_fifo_size		= 512,
+	.host_nperio_tx_fifo_size	= 500,
+	.host_perio_tx_fifo_size	= 500,
+	.max_transfer_size		= -1,
+	.max_packet_count		= -1,
+	.host_channels			= 16,
+	.phy_type			= DWC2_PHY_TYPE_PARAM_UTMI,
+	.phy_utmi_width			= -1,
+	.phy_ulpi_ddr			= -1,
+	.phy_ulpi_ext_vbus		= -1,
+	.i2c_enable			= -1,
+	.ulpi_fs_ls			= -1,
+	.host_support_fs_ls_low_power	= -1,
+	.host_ls_low_power_phy_clk	= -1,
+	.ts_dline			= -1,
+	.reload_ctl			= 1,
+	.ahbcfg				= GAHBCFG_HBSTLEN_INCR8 <<
+					  GAHBCFG_HBSTLEN_SHIFT,
+	.uframe_sched			= 0,
+	.external_id_pin_ctl		= -1,
+	.hibernation			= -1,
+};
+
 /*
  * Check the dr_mode against the module configuration and hardware
  * capabilities.
@@ -337,6 +370,24 @@ static int dwc2_lowlevel_hw_init(struct dwc2_hsotg *hsotg)
 {
 	int i, ret;
 
+	hsotg->reset = devm_reset_control_get_optional(hsotg->dev, "dwc2");
+	if (IS_ERR(hsotg->reset)) {
+		ret = PTR_ERR(hsotg->reset);
+		switch (ret) {
+		case -ENOENT:
+		case -ENOTSUPP:
+			hsotg->reset = NULL;
+			break;
+		default:
+			dev_err(hsotg->dev, "error getting reset control %d\n",
+				ret);
+			return ret;
+		}
+	}
+
+	if (hsotg->reset)
+		reset_control_deassert(hsotg->reset);
+
 	/* Set default UTMI width */
 	hsotg->phyif = GUSBCFG_PHYIF16;
 
@@ -434,6 +485,9 @@ static int dwc2_driver_remove(struct platform_device *dev)
 	if (hsotg->ll_hw_enabled)
 		dwc2_lowlevel_hw_disable(hsotg);
 
+	if (hsotg->reset)
+		reset_control_assert(hsotg->reset);
+
 	return 0;
 }
 
@@ -464,6 +518,8 @@ static const struct of_device_id dwc2_of_match_table[] = {
 	{ .compatible = "lantiq,xrx200-usb", .data = &params_ltq },
 	{ .compatible = "snps,dwc2", .data = NULL },
 	{ .compatible = "samsung,s3c6400-hsotg", .data = NULL},
+	{ .compatible = "amlogic,meson8b-usb", .data = &params_amlogic },
+	{ .compatible = "amlogic,meson-gxbb-usb", .data = &params_amlogic },
 	{},
 };
 MODULE_DEVICE_TABLE(of, dwc2_of_match_table);
diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig
index a64ce1c94d6d..b97cde76914d 100644
--- a/drivers/usb/dwc3/Kconfig
+++ b/drivers/usb/dwc3/Kconfig
@@ -1,7 +1,7 @@
 config USB_DWC3
 	tristate "DesignWare USB3 DRD Core Support"
 	depends on (USB || USB_GADGET) && HAS_DMA
-	select USB_XHCI_PLATFORM if USB_SUPPORT && USB_XHCI_HCD
+	select USB_XHCI_PLATFORM if USB_XHCI_HCD
 	help
 	  Say Y or M here if your system has a Dual Role SuperSpeed
 	  USB controller based on the DesignWare USB3 IP Core.
diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
index 946643157b78..7287a763cd0c 100644
--- a/drivers/usb/dwc3/core.c
+++ b/drivers/usb/dwc3/core.c
@@ -49,6 +49,57 @@
 
 #define DWC3_DEFAULT_AUTOSUSPEND_DELAY	5000 /* ms */
 
+/**
+ * dwc3_get_dr_mode - Validates and sets dr_mode
+ * @dwc: pointer to our context structure
+ */
+static int dwc3_get_dr_mode(struct dwc3 *dwc)
+{
+	enum usb_dr_mode mode;
+	struct device *dev = dwc->dev;
+	unsigned int hw_mode;
+
+	if (dwc->dr_mode == USB_DR_MODE_UNKNOWN)
+		dwc->dr_mode = USB_DR_MODE_OTG;
+
+	mode = dwc->dr_mode;
+	hw_mode = DWC3_GHWPARAMS0_MODE(dwc->hwparams.hwparams0);
+
+	switch (hw_mode) {
+	case DWC3_GHWPARAMS0_MODE_GADGET:
+		if (IS_ENABLED(CONFIG_USB_DWC3_HOST)) {
+			dev_err(dev,
+				"Controller does not support host mode.\n");
+			return -EINVAL;
+		}
+		mode = USB_DR_MODE_PERIPHERAL;
+		break;
+	case DWC3_GHWPARAMS0_MODE_HOST:
+		if (IS_ENABLED(CONFIG_USB_DWC3_GADGET)) {
+			dev_err(dev,
+				"Controller does not support device mode.\n");
+			return -EINVAL;
+		}
+		mode = USB_DR_MODE_HOST;
+		break;
+	default:
+		if (IS_ENABLED(CONFIG_USB_DWC3_HOST))
+			mode = USB_DR_MODE_HOST;
+		else if (IS_ENABLED(CONFIG_USB_DWC3_GADGET))
+			mode = USB_DR_MODE_PERIPHERAL;
+	}
+
+	if (mode != dwc->dr_mode) {
+		dev_warn(dev,
+			 "Configuration mismatch. dr_mode forced to %s\n",
+			 mode == USB_DR_MODE_HOST ? "host" : "gadget");
+
+		dwc->dr_mode = mode;
+	}
+
+	return 0;
+}
+
 void dwc3_set_mode(struct dwc3 *dwc, u32 mode)
 {
 	u32 reg;
@@ -448,6 +499,9 @@ static int dwc3_phy_setup(struct dwc3 *dwc)
 	if (dwc->dis_u3_susphy_quirk)
 		reg &= ~DWC3_GUSB3PIPECTL_SUSPHY;
 
+	if (dwc->dis_del_phy_power_chg_quirk)
+		reg &= ~DWC3_GUSB3PIPECTL_DEPOCHANGE;
+
 	dwc3_writel(dwc->regs, DWC3_GUSB3PIPECTL(0), reg);
 
 	reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
@@ -485,6 +539,23 @@ static int dwc3_phy_setup(struct dwc3 *dwc)
 		break;
 	}
 
+	switch (dwc->hsphy_mode) {
+	case USBPHY_INTERFACE_MODE_UTMI:
+		reg &= ~(DWC3_GUSB2PHYCFG_PHYIF_MASK |
+		       DWC3_GUSB2PHYCFG_USBTRDTIM_MASK);
+		reg |= DWC3_GUSB2PHYCFG_PHYIF(UTMI_PHYIF_8_BIT) |
+		       DWC3_GUSB2PHYCFG_USBTRDTIM(USBTRDTIM_UTMI_8_BIT);
+		break;
+	case USBPHY_INTERFACE_MODE_UTMIW:
+		reg &= ~(DWC3_GUSB2PHYCFG_PHYIF_MASK |
+		       DWC3_GUSB2PHYCFG_USBTRDTIM_MASK);
+		reg |= DWC3_GUSB2PHYCFG_PHYIF(UTMI_PHYIF_16_BIT) |
+		       DWC3_GUSB2PHYCFG_USBTRDTIM(USBTRDTIM_UTMI_16_BIT);
+		break;
+	default:
+		break;
+	}
+
 	/*
 	 * Above 1.94a, it is recommended to set DWC3_GUSB2PHYCFG_SUSPHY to
 	 * '0' during coreConsultant configuration. So default value will
@@ -500,6 +571,9 @@ static int dwc3_phy_setup(struct dwc3 *dwc)
 	if (dwc->dis_enblslpm_quirk)
 		reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
 
+	if (dwc->dis_u2_freeclk_exists_quirk)
+		reg &= ~DWC3_GUSB2PHYCFG_U2_FREECLK_EXISTS;
+
 	dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 
 	return 0;
@@ -666,6 +740,32 @@ static int dwc3_core_init(struct dwc3 *dwc)
 		goto err4;
 	}
 
+	switch (dwc->dr_mode) {
+	case USB_DR_MODE_PERIPHERAL:
+		dwc3_set_mode(dwc, DWC3_GCTL_PRTCAP_DEVICE);
+		break;
+	case USB_DR_MODE_HOST:
+		dwc3_set_mode(dwc, DWC3_GCTL_PRTCAP_HOST);
+		break;
+	case USB_DR_MODE_OTG:
+		dwc3_set_mode(dwc, DWC3_GCTL_PRTCAP_OTG);
+		break;
+	default:
+		dev_warn(dwc->dev, "Unsupported mode %d\n", dwc->dr_mode);
+		break;
+	}
+
+	/*
+	 * ENDXFER polling is available on version 3.10a and later of
+	 * the DWC_usb3 controller. It is NOT available in the
+	 * DWC_usb31 controller.
+	 */
+	if (!dwc3_is_usb31(dwc) && dwc->revision >= DWC3_REVISION_310A) {
+		reg = dwc3_readl(dwc->regs, DWC3_GUCTL2);
+		reg |= DWC3_GUCTL2_RST_ACTBITLATER;
+		dwc3_writel(dwc->regs, DWC3_GUCTL2, reg);
+	}
+
 	return 0;
 
 err4:
@@ -763,7 +863,6 @@ static int dwc3_core_init_mode(struct dwc3 *dwc)
 
 	switch (dwc->dr_mode) {
 	case USB_DR_MODE_PERIPHERAL:
-		dwc3_set_mode(dwc, DWC3_GCTL_PRTCAP_DEVICE);
 		ret = dwc3_gadget_init(dwc);
 		if (ret) {
 			if (ret != -EPROBE_DEFER)
@@ -772,7 +871,6 @@ static int dwc3_core_init_mode(struct dwc3 *dwc)
 		}
 		break;
 	case USB_DR_MODE_HOST:
-		dwc3_set_mode(dwc, DWC3_GCTL_PRTCAP_HOST);
 		ret = dwc3_host_init(dwc);
 		if (ret) {
 			if (ret != -EPROBE_DEFER)
@@ -781,7 +879,6 @@ static int dwc3_core_init_mode(struct dwc3 *dwc)
 		}
 		break;
 	case USB_DR_MODE_OTG:
-		dwc3_set_mode(dwc, DWC3_GCTL_PRTCAP_OTG);
 		ret = dwc3_host_init(dwc);
 		if (ret) {
 			if (ret != -EPROBE_DEFER)
@@ -888,6 +985,7 @@ static int dwc3_probe(struct platform_device *pdev)
 
 	dwc->maximum_speed = usb_get_maximum_speed(dev);
 	dwc->dr_mode = usb_get_dr_mode(dev);
+	dwc->hsphy_mode = of_usb_get_phy_mode(dev->of_node);
 
 	dwc->has_lpm_erratum = device_property_read_bool(dev,
 				"snps,has-lpm-erratum");
@@ -924,6 +1022,10 @@ static int dwc3_probe(struct platform_device *pdev)
 				"snps,dis_enblslpm_quirk");
 	dwc->dis_rxdet_inp3_quirk = device_property_read_bool(dev,
 				"snps,dis_rxdet_inp3_quirk");
+	dwc->dis_u2_freeclk_exists_quirk = device_property_read_bool(dev,
+				"snps,dis-u2-freeclk-exists-quirk");
+	dwc->dis_del_phy_power_chg_quirk = device_property_read_bool(dev,
+				"snps,dis-del-phy-power-chg-quirk");
 
 	dwc->tx_de_emphasis_quirk = device_property_read_bool(dev,
 				"snps,tx_de_emphasis_quirk");
@@ -972,17 +1074,9 @@ static int dwc3_probe(struct platform_device *pdev)
 		goto err2;
 	}
 
-	if (IS_ENABLED(CONFIG_USB_DWC3_HOST) &&
-			(dwc->dr_mode == USB_DR_MODE_OTG ||
-					dwc->dr_mode == USB_DR_MODE_UNKNOWN))
-		dwc->dr_mode = USB_DR_MODE_HOST;
-	else if (IS_ENABLED(CONFIG_USB_DWC3_GADGET) &&
-			(dwc->dr_mode == USB_DR_MODE_OTG ||
-					dwc->dr_mode == USB_DR_MODE_UNKNOWN))
-		dwc->dr_mode = USB_DR_MODE_PERIPHERAL;
-
-	if (dwc->dr_mode == USB_DR_MODE_UNKNOWN)
-		dwc->dr_mode = USB_DR_MODE_OTG;
+	ret = dwc3_get_dr_mode(dwc);
+	if (ret)
+		goto err3;
 
 	ret = dwc3_alloc_scratch_buffers(dwc);
 	if (ret)
@@ -1192,6 +1286,7 @@ static int dwc3_runtime_resume(struct device *dev)
 	}
 
 	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put(dev);
 
 	return 0;
 }
diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index 45d6de5107c7..6b60e42626a2 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -109,6 +109,7 @@
 #define DWC3_GPRTBIMAP_HS1	0xc184
 #define DWC3_GPRTBIMAP_FS0	0xc188
 #define DWC3_GPRTBIMAP_FS1	0xc18c
+#define DWC3_GUCTL2		0xc19c
 
 #define DWC3_VER_NUMBER		0xc1a0
 #define DWC3_VER_TYPE		0xc1a4
@@ -199,9 +200,18 @@
 
 /* Global USB2 PHY Configuration Register */
 #define DWC3_GUSB2PHYCFG_PHYSOFTRST	(1 << 31)
+#define DWC3_GUSB2PHYCFG_U2_FREECLK_EXISTS	(1 << 30)
 #define DWC3_GUSB2PHYCFG_SUSPHY		(1 << 6)
 #define DWC3_GUSB2PHYCFG_ULPI_UTMI	(1 << 4)
 #define DWC3_GUSB2PHYCFG_ENBLSLPM	(1 << 8)
+#define DWC3_GUSB2PHYCFG_PHYIF(n)	(n << 3)
+#define DWC3_GUSB2PHYCFG_PHYIF_MASK	DWC3_GUSB2PHYCFG_PHYIF(1)
+#define DWC3_GUSB2PHYCFG_USBTRDTIM(n)	(n << 10)
+#define DWC3_GUSB2PHYCFG_USBTRDTIM_MASK	DWC3_GUSB2PHYCFG_USBTRDTIM(0xf)
+#define USBTRDTIM_UTMI_8_BIT		9
+#define USBTRDTIM_UTMI_16_BIT		5
+#define UTMI_PHYIF_16_BIT		1
+#define UTMI_PHYIF_8_BIT		0
 
 /* Global USB2 PHY Vendor Control Register */
 #define DWC3_GUSB2PHYACC_NEWREGREQ	(1 << 25)
@@ -235,7 +245,10 @@
 #define DWC3_GEVNTSIZ_SIZE(n)		((n) & 0xffff)
 
 /* Global HWPARAMS0 Register */
-#define DWC3_GHWPARAMS0_USB3_MODE(n)	((n) & 0x3)
+#define DWC3_GHWPARAMS0_MODE(n)		((n) & 0x3)
+#define DWC3_GHWPARAMS0_MODE_GADGET	0
+#define DWC3_GHWPARAMS0_MODE_HOST	1
+#define DWC3_GHWPARAMS0_MODE_DRD	2
 #define DWC3_GHWPARAMS0_MBUS_TYPE(n)	(((n) >> 3) & 0x7)
 #define DWC3_GHWPARAMS0_SBUS_TYPE(n)	(((n) >> 6) & 0x3)
 #define DWC3_GHWPARAMS0_MDWIDTH(n)	(((n) >> 8) & 0xff)
@@ -279,6 +292,9 @@
 #define DWC3_GFLADJ_30MHZ_SDBND_SEL		(1 << 7)
 #define DWC3_GFLADJ_30MHZ_MASK			0x3f
 
+/* Global User Control Register 2 */
+#define DWC3_GUCTL2_RST_ACTBITLATER		(1 << 14)
+
 /* Device Configuration Register */
 #define DWC3_DCFG_DEVADDR(addr)	((addr) << 3)
 #define DWC3_DCFG_DEVADDR_MASK	DWC3_DCFG_DEVADDR(0x7f)
@@ -685,6 +701,8 @@ struct dwc3_hwparams {
  * @request: struct usb_request to be transferred
  * @list: a list_head used for request queueing
  * @dep: struct dwc3_ep owning this request
+ * @sg: pointer to first incomplete sg
+ * @num_pending_sgs: counter to pending sgs
  * @first_trb_index: index to first trb used by this request
  * @epnum: endpoint number to which this request refers
  * @trb: pointer to struct dwc3_trb
@@ -697,7 +715,9 @@ struct dwc3_request {
 	struct usb_request	request;
 	struct list_head	list;
 	struct dwc3_ep		*dep;
+	struct scatterlist	*sg;
 
+	unsigned		num_pending_sgs;
 	u8			first_trb_index;
 	u8			epnum;
 	struct dwc3_trb		*trb;
@@ -743,6 +763,9 @@ struct dwc3_scratchpad_array {
  * @maximum_speed: maximum speed requested (mainly for testing purposes)
  * @revision: revision register contents
  * @dr_mode: requested mode of operation
+ * @hsphy_mode: UTMI phy mode, one of following:
+ *		- USBPHY_INTERFACE_MODE_UTMI
+ *		- USBPHY_INTERFACE_MODE_UTMIW
  * @usb2_phy: pointer to USB2 PHY
  * @usb3_phy: pointer to USB3 PHY
  * @usb2_generic_phy: pointer to USB2 PHY
@@ -799,6 +822,11 @@ struct dwc3_scratchpad_array {
  * @dis_u2_susphy_quirk: set if we disable usb2 suspend phy
  * @dis_enblslpm_quirk: set if we clear enblslpm in GUSB2PHYCFG,
  *                      disabling the suspend signal to the PHY.
+ * @dis_u2_freeclk_exists_quirk : set if we clear u2_freeclk_exists
+ *			in GUSB2PHYCFG, specify that USB2 PHY doesn't
+ *			provide a free-running PHY clock.
+ * @dis_del_phy_power_chg_quirk: set if we disable delay phy power
+ *			change quirk.
  * @tx_de_emphasis_quirk: set if we enable Tx de-emphasis quirk
  * @tx_de_emphasis: Tx de-emphasis value
  * 	0	- -6dB de-emphasis
@@ -845,6 +873,7 @@ struct dwc3 {
 	size_t			regs_size;
 
 	enum usb_dr_mode	dr_mode;
+	enum usb_phy_interface	hsphy_mode;
 
 	u32			fladj;
 	u32			irq_gadget;
@@ -880,6 +909,8 @@ struct dwc3 {
 #define DWC3_REVISION_260A	0x5533260a
 #define DWC3_REVISION_270A	0x5533270a
 #define DWC3_REVISION_280A	0x5533280a
+#define DWC3_REVISION_300A	0x5533300a
+#define DWC3_REVISION_310A	0x5533310a
 
 /*
  * NOTICE: we're using bit 31 as a "is usb 3.1" flag. This is really
@@ -942,6 +973,8 @@ struct dwc3 {
 	unsigned		dis_u2_susphy_quirk:1;
 	unsigned		dis_enblslpm_quirk:1;
 	unsigned		dis_rxdet_inp3_quirk:1;
+	unsigned		dis_u2_freeclk_exists_quirk:1;
+	unsigned		dis_del_phy_power_chg_quirk:1;
 
 	unsigned		tx_de_emphasis_quirk:1;
 	unsigned		tx_de_emphasis:2;
diff --git a/drivers/usb/dwc3/debug.h b/drivers/usb/dwc3/debug.h
index 22dfc3dd6a13..33ab2a203c1b 100644
--- a/drivers/usb/dwc3/debug.h
+++ b/drivers/usb/dwc3/debug.h
@@ -192,7 +192,7 @@ dwc3_ep_event_string(const struct dwc3_event_depevt *event)
 	int ret;
 
 	ret = sprintf(str, "ep%d%s: ", epnum >> 1,
-			(epnum & 1) ? "in" : "in");
+			(epnum & 1) ? "in" : "out");
 	if (ret < 0)
 		return "UNKNOWN";
 
diff --git a/drivers/usb/dwc3/dwc3-of-simple.c b/drivers/usb/dwc3/dwc3-of-simple.c
index 974335377d9f..fe414e7a9c78 100644
--- a/drivers/usb/dwc3/dwc3-of-simple.c
+++ b/drivers/usb/dwc3/dwc3-of-simple.c
@@ -36,35 +36,25 @@ struct dwc3_of_simple {
 	int			num_clocks;
 };
 
-static int dwc3_of_simple_probe(struct platform_device *pdev)
+static int dwc3_of_simple_clk_init(struct dwc3_of_simple *simple, int count)
 {
-	struct dwc3_of_simple	*simple;
-	struct device		*dev = &pdev->dev;
+	struct device		*dev = simple->dev;
 	struct device_node	*np = dev->of_node;
-
-	unsigned int		count;
-	int			ret;
 	int			i;
 
-	simple = devm_kzalloc(dev, sizeof(*simple), GFP_KERNEL);
-	if (!simple)
-		return -ENOMEM;
+	simple->num_clocks = count;
 
-	count = of_clk_get_parent_count(np);
 	if (!count)
-		return -ENOENT;
-
-	simple->num_clocks = count;
+		return 0;
 
 	simple->clks = devm_kcalloc(dev, simple->num_clocks,
 			sizeof(struct clk *), GFP_KERNEL);
 	if (!simple->clks)
 		return -ENOMEM;
 
-	simple->dev = dev;
-
 	for (i = 0; i < simple->num_clocks; i++) {
 		struct clk	*clk;
+		int		ret;
 
 		clk = of_clk_get(np, i);
 		if (IS_ERR(clk)) {
@@ -87,6 +77,29 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 		simple->clks[i] = clk;
 	}
 
+	return 0;
+}
+
+static int dwc3_of_simple_probe(struct platform_device *pdev)
+{
+	struct dwc3_of_simple	*simple;
+	struct device		*dev = &pdev->dev;
+	struct device_node	*np = dev->of_node;
+
+	int			ret;
+	int			i;
+
+	simple = devm_kzalloc(dev, sizeof(*simple), GFP_KERNEL);
+	if (!simple)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, simple);
+	simple->dev = dev;
+
+	ret = dwc3_of_simple_clk_init(simple, of_clk_get_parent_count(np));
+	if (ret)
+		return ret;
+
 	ret = of_platform_populate(np, NULL, NULL, dev);
 	if (ret) {
 		for (i = 0; i < simple->num_clocks; i++) {
@@ -111,7 +124,7 @@ static int dwc3_of_simple_remove(struct platform_device *pdev)
 	int			i;
 
 	for (i = 0; i < simple->num_clocks; i++) {
-		clk_unprepare(simple->clks[i]);
+		clk_disable_unprepare(simple->clks[i]);
 		clk_put(simple->clks[i]);
 	}
 
@@ -161,7 +174,9 @@ static const struct dev_pm_ops dwc3_of_simple_dev_pm_ops = {
 
 static const struct of_device_id of_dwc3_simple_match[] = {
 	{ .compatible = "qcom,dwc3" },
+	{ .compatible = "rockchip,rk3399-dwc3" },
 	{ .compatible = "xlnx,zynqmp-dwc3" },
+	{ .compatible = "cavium,octeon-7130-usb-uctl" },
 	{ /* Sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, of_dwc3_simple_match);
diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c
index 45f5a232d9fb..6df0f5dad9a4 100644
--- a/drivers/usb/dwc3/dwc3-pci.c
+++ b/drivers/usb/dwc3/dwc3-pci.c
@@ -37,6 +37,7 @@
 #define PCI_DEVICE_ID_INTEL_BXT			0x0aaa
 #define PCI_DEVICE_ID_INTEL_BXT_M		0x1aaa
 #define PCI_DEVICE_ID_INTEL_APL			0x5aaa
+#define PCI_DEVICE_ID_INTEL_KBP			0xa2b0
 
 static const struct acpi_gpio_params reset_gpios = { 0, 0, false };
 static const struct acpi_gpio_params cs_gpios = { 1, 0, false };
@@ -227,6 +228,7 @@ static const struct pci_device_id dwc3_pci_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BXT), },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BXT_M), },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_APL), },
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBP), },
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_NL_USB), },
 	{  }	/* Terminating Entry */
 };
@@ -241,6 +243,15 @@ static int dwc3_pci_runtime_suspend(struct device *dev)
 	return -EBUSY;
 }
 
+static int dwc3_pci_runtime_resume(struct device *dev)
+{
+	struct platform_device *dwc3 = dev_get_drvdata(dev);
+
+	return pm_runtime_get(&dwc3->dev);
+}
+#endif /* CONFIG_PM */
+
+#ifdef CONFIG_PM_SLEEP
 static int dwc3_pci_pm_dummy(struct device *dev)
 {
 	/*
@@ -253,11 +264,11 @@ static int dwc3_pci_pm_dummy(struct device *dev)
 	 */
 	return 0;
 }
-#endif /* CONFIG_PM */
+#endif /* CONFIG_PM_SLEEP */
 
 static struct dev_pm_ops dwc3_pci_dev_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(dwc3_pci_pm_dummy, dwc3_pci_pm_dummy)
-	SET_RUNTIME_PM_OPS(dwc3_pci_runtime_suspend, dwc3_pci_pm_dummy,
+	SET_RUNTIME_PM_OPS(dwc3_pci_runtime_suspend, dwc3_pci_runtime_resume,
 		NULL)
 };
 
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index 8f8c2157910e..07cc8929f271 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -174,15 +174,8 @@ void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 		int status)
 {
 	struct dwc3			*dwc = dep->dwc;
-	int				i;
 
-	if (req->started) {
-		i = 0;
-		do {
-			dwc3_ep_inc_deq(dep);
-		} while(++i < req->request.num_mapped_sgs);
-		req->started = false;
-	}
+	req->started = false;
 	list_del(&req->list);
 	req->trb = NULL;
 
@@ -348,7 +341,8 @@ static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 	 * IN transfers due to a mishandled error condition. Synopsys
 	 * STAR 9000614252.
 	 */
-	if (dep->direction && (dwc->revision >= DWC3_REVISION_260A))
+	if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
+	    (dwc->gadget.speed >= USB_SPEED_SUPER))
 		cmd |= DWC3_DEPCMD_CLEARPENDIN;
 
 	memset(&params, 0, sizeof(params));
@@ -490,7 +484,8 @@ static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 		params.param0 |= DWC3_DEPCFG_ACTION_INIT;
 	}
 
-	params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
+	if (usb_endpoint_xfer_control(desc))
+		params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 
 	if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 		params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
@@ -764,6 +759,8 @@ static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 	kfree(req);
 }
 
+static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep);
+
 /**
  * dwc3_prepare_one_trb - setup one TRB from one request
  * @dep: endpoint for which this request is prepared
@@ -771,15 +768,13 @@ static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
  */
 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 		struct dwc3_request *req, dma_addr_t dma,
-		unsigned length, unsigned last, unsigned chain, unsigned node)
+		unsigned length, unsigned chain, unsigned node)
 {
 	struct dwc3_trb		*trb;
 
-	dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s%s",
+	dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s",
 			dep->name, req, (unsigned long long) dma,
-			length, last ? " last" : "",
-			chain ? " chain" : "");
-
+			length, chain ? " chain" : "");
 
 	trb = &dep->trb_pool[dep->trb_enqueue];
 
@@ -826,12 +821,10 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 	/* always enable Continue on Short Packet */
 	trb->ctrl |= DWC3_TRB_CTRL_CSP;
 
-	if (!req->request.no_interrupt && !chain)
+	if ((!req->request.no_interrupt && !chain) ||
+			(dwc3_calc_trbs_left(dep) == 0))
 		trb->ctrl |= DWC3_TRB_CTRL_IOC | DWC3_TRB_CTRL_ISP_IMI;
 
-	if (last)
-		trb->ctrl |= DWC3_TRB_CTRL_LST;
-
 	if (chain)
 		trb->ctrl |= DWC3_TRB_CTRL_CHN;
 
@@ -856,12 +849,12 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
  */
 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
 {
-	if (!index)
-		index = DWC3_TRB_NUM - 2;
-	else
-		index = dep->trb_enqueue - 1;
+	u8 tmp = index;
+
+	if (!tmp)
+		tmp = DWC3_TRB_NUM - 1;
 
-	return &dep->trb_pool[index];
+	return &dep->trb_pool[tmp - 1];
 }
 
 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
@@ -894,65 +887,42 @@ static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 }
 
 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
-		struct dwc3_request *req, unsigned int trbs_left,
-		unsigned int more_coming)
+		struct dwc3_request *req)
 {
-	struct usb_request *request = &req->request;
-	struct scatterlist *sg = request->sg;
+	struct scatterlist *sg = req->sg;
 	struct scatterlist *s;
-	unsigned int	last = false;
 	unsigned int	length;
 	dma_addr_t	dma;
 	int		i;
 
-	for_each_sg(sg, s, request->num_mapped_sgs, i) {
+	for_each_sg(sg, s, req->num_pending_sgs, i) {
 		unsigned chain = true;
 
 		length = sg_dma_len(s);
 		dma = sg_dma_address(s);
 
-		if (sg_is_last(s)) {
-			if (usb_endpoint_xfer_int(dep->endpoint.desc) ||
-				!more_coming)
-				last = true;
-
-			chain = false;
-		}
-
-		if (!trbs_left--)
-			last = true;
-
-		if (last)
+		if (sg_is_last(s))
 			chain = false;
 
 		dwc3_prepare_one_trb(dep, req, dma, length,
-				last, chain, i);
+				chain, i);
 
-		if (last)
+		if (!dwc3_calc_trbs_left(dep))
 			break;
 	}
 }
 
 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
-		struct dwc3_request *req, unsigned int trbs_left,
-		unsigned int more_coming)
+		struct dwc3_request *req)
 {
-	unsigned int	last = false;
 	unsigned int	length;
 	dma_addr_t	dma;
 
 	dma = req->request.dma;
 	length = req->request.length;
 
-	if (!trbs_left)
-		last = true;
-
-	/* Is this the last request? */
-	if (usb_endpoint_xfer_int(dep->endpoint.desc) || !more_coming)
-		last = true;
-
 	dwc3_prepare_one_trb(dep, req, dma, length,
-			last, false, 0);
+			false, 0);
 }
 
 /*
@@ -966,26 +936,19 @@ static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
 {
 	struct dwc3_request	*req, *n;
-	unsigned int		more_coming;
-	u32			trbs_left;
 
 	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
 
-	trbs_left = dwc3_calc_trbs_left(dep);
-	if (!trbs_left)
+	if (!dwc3_calc_trbs_left(dep))
 		return;
 
-	more_coming = dep->allocated_requests - dep->queued_requests;
-
 	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
-		if (req->request.num_mapped_sgs > 0)
-			dwc3_prepare_one_trb_sg(dep, req, trbs_left--,
-					more_coming);
+		if (req->num_pending_sgs > 0)
+			dwc3_prepare_one_trb_sg(dep, req);
 		else
-			dwc3_prepare_one_trb_linear(dep, req, trbs_left--,
-					more_coming);
+			dwc3_prepare_one_trb_linear(dep, req);
 
-		if (!trbs_left)
+		if (!dwc3_calc_trbs_left(dep))
 			return;
 	}
 }
@@ -1101,93 +1064,29 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 
 	trace_dwc3_ep_queue(req);
 
-	/*
-	 * We only add to our list of requests now and
-	 * start consuming the list once we get XferNotReady
-	 * IRQ.
-	 *
-	 * That way, we avoid doing anything that we don't need
-	 * to do now and defer it until the point we receive a
-	 * particular token from the Host side.
-	 *
-	 * This will also avoid Host cancelling URBs due to too
-	 * many NAKs.
-	 */
 	ret = usb_gadget_map_request(&dwc->gadget, &req->request,
 			dep->direction);
 	if (ret)
 		return ret;
 
-	list_add_tail(&req->list, &dep->pending_list);
+	req->sg			= req->request.sg;
+	req->num_pending_sgs	= req->request.num_mapped_sgs;
 
-	/*
-	 * If there are no pending requests and the endpoint isn't already
-	 * busy, we will just start the request straight away.
-	 *
-	 * This will save one IRQ (XFER_NOT_READY) and possibly make it a
-	 * little bit faster.
-	 */
-	if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
-			!usb_endpoint_xfer_int(dep->endpoint.desc)) {
-		ret = __dwc3_gadget_kick_transfer(dep, 0);
-		goto out;
-	}
-
-	/*
-	 * There are a few special cases:
-	 *
-	 * 1. XferNotReady with empty list of requests. We need to kick the
-	 *    transfer here in that situation, otherwise we will be NAKing
-	 *    forever. If we get XferNotReady before gadget driver has a
-	 *    chance to queue a request, we will ACK the IRQ but won't be
-	 *    able to receive the data until the next request is queued.
-	 *    The following code is handling exactly that.
-	 *
-	 */
-	if (dep->flags & DWC3_EP_PENDING_REQUEST) {
-		/*
-		 * If xfernotready is already elapsed and it is a case
-		 * of isoc transfer, then issue END TRANSFER, so that
-		 * you can receive xfernotready again and can have
-		 * notion of current microframe.
-		 */
-		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
-			if (list_empty(&dep->started_list)) {
-				dwc3_stop_active_transfer(dwc, dep->number, true);
-				dep->flags = DWC3_EP_ENABLED;
-			}
-			return 0;
-		}
-
-		ret = __dwc3_gadget_kick_transfer(dep, 0);
-		if (!ret)
-			dep->flags &= ~DWC3_EP_PENDING_REQUEST;
-
-		goto out;
-	}
+	list_add_tail(&req->list, &dep->pending_list);
 
-	/*
-	 * 2. XferInProgress on Isoc EP with an active transfer. We need to
-	 *    kick the transfer here after queuing a request, otherwise the
-	 *    core may not see the modified TRB(s).
-	 */
 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
-			(dep->flags & DWC3_EP_BUSY) &&
-			!(dep->flags & DWC3_EP_MISSED_ISOC)) {
-		WARN_ON_ONCE(!dep->resource_index);
-		ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index);
-		goto out;
+			dep->flags & DWC3_EP_PENDING_REQUEST) {
+		if (list_empty(&dep->started_list)) {
+			dwc3_stop_active_transfer(dwc, dep->number, true);
+			dep->flags = DWC3_EP_ENABLED;
+		}
+		return 0;
 	}
 
-	/*
-	 * 4. Stream Capable Bulk Endpoints. We need to start the transfer
-	 * right away, otherwise host will not know we have streams to be
-	 * handled.
-	 */
-	if (dep->stream_capable)
-		ret = __dwc3_gadget_kick_transfer(dep, 0);
+	if (!dwc3_calc_trbs_left(dep))
+		return 0;
 
-out:
+	ret = __dwc3_gadget_kick_transfer(dep, 0);
 	if (ret && ret != -EBUSY)
 		dwc3_trace(trace_dwc3_gadget,
 				"%s: failed to kick transfers",
@@ -1433,7 +1332,7 @@ static int dwc3_gadget_get_frame(struct usb_gadget *g)
 
 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
 {
-	unsigned long		timeout;
+	int			retries;
 
 	int			ret;
 	u32			reg;
@@ -1484,9 +1383,9 @@ static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
 	}
 
 	/* poll until Link State changes to ON */
-	timeout = jiffies + msecs_to_jiffies(100);
+	retries = 20000;
 
-	while (!time_after(jiffies, timeout)) {
+	while (retries--) {
 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 
 		/* in HS, means ON */
@@ -1955,27 +1854,35 @@ static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
 
 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
 		struct dwc3_request *req, struct dwc3_trb *trb,
-		const struct dwc3_event_depevt *event, int status)
+		const struct dwc3_event_depevt *event, int status,
+		int chain)
 {
 	unsigned int		count;
 	unsigned int		s_pkt = 0;
 	unsigned int		trb_status;
 
 	dep->queued_requests--;
+	dwc3_ep_inc_deq(dep);
 	trace_dwc3_complete_trb(dep, trb);
 
+	/*
+	 * If we're in the middle of series of chained TRBs and we
+	 * receive a short transfer along the way, DWC3 will skip
+	 * through all TRBs including the last TRB in the chain (the
+	 * where CHN bit is zero. DWC3 will also avoid clearing HWO
+	 * bit and SW has to do it manually.
+	 *
+	 * We're going to do that here to avoid problems of HW trying
+	 * to use bogus TRBs for transfers.
+	 */
+	if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
+		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
+
 	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
-		/*
-		 * We continue despite the error. There is not much we
-		 * can do. If we don't clean it up we loop forever. If
-		 * we skip the TRB then it gets overwritten after a
-		 * while since we use them in a ring buffer. A BUG()
-		 * would help. Lets hope that if this occurs, someone
-		 * fixes the root cause instead of looking away :)
-		 */
-		dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
-				dep->name, trb);
+		return 1;
+
 	count = trb->size & DWC3_TRB_SIZE_MASK;
+	req->request.actual += count;
 
 	if (dep->direction) {
 		if (count) {
@@ -2013,59 +1920,76 @@ static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
 			s_pkt = 1;
 	}
 
-	/*
-	 * We assume here we will always receive the entire data block
-	 * which we should receive. Meaning, if we program RX to
-	 * receive 4K but we receive only 2K, we assume that's all we
-	 * should receive and we simply bounce the request back to the
-	 * gadget driver for further processing.
-	 */
-	req->request.actual += req->request.length - count;
-	if (s_pkt)
-		return 1;
-	if ((event->status & DEPEVT_STATUS_LST) &&
-			(trb->ctrl & (DWC3_TRB_CTRL_LST |
-				DWC3_TRB_CTRL_HWO)))
+	if (s_pkt && !chain)
 		return 1;
+
 	if ((event->status & DEPEVT_STATUS_IOC) &&
 			(trb->ctrl & DWC3_TRB_CTRL_IOC))
 		return 1;
+
 	return 0;
 }
 
 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
 		const struct dwc3_event_depevt *event, int status)
 {
-	struct dwc3_request	*req;
+	struct dwc3_request	*req, *n;
 	struct dwc3_trb		*trb;
-	unsigned int		slot;
-	unsigned int		i;
+	bool			ioc = false;
 	int			ret;
 
-	do {
-		req = next_request(&dep->started_list);
-		if (WARN_ON_ONCE(!req))
-			return 1;
+	list_for_each_entry_safe(req, n, &dep->started_list, list) {
+		unsigned length;
+		unsigned actual;
+		int chain;
+
+		length = req->request.length;
+		chain = req->num_pending_sgs > 0;
+		if (chain) {
+			struct scatterlist *sg = req->sg;
+			struct scatterlist *s;
+			unsigned int pending = req->num_pending_sgs;
+			unsigned int i;
+
+			for_each_sg(sg, s, pending, i) {
+				trb = &dep->trb_pool[dep->trb_dequeue];
+
+				req->sg = sg_next(s);
+				req->num_pending_sgs--;
+
+				ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
+						event, status, chain);
+				if (ret)
+					break;
+			}
+		} else {
+			trb = &dep->trb_pool[dep->trb_dequeue];
+			ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
+					event, status, chain);
+		}
 
-		i = 0;
-		do {
-			slot = req->first_trb_index + i;
-			if (slot == DWC3_TRB_NUM - 1)
-				slot++;
-			slot %= DWC3_TRB_NUM;
-			trb = &dep->trb_pool[slot];
+		/*
+		 * We assume here we will always receive the entire data block
+		 * which we should receive. Meaning, if we program RX to
+		 * receive 4K but we receive only 2K, we assume that's all we
+		 * should receive and we simply bounce the request back to the
+		 * gadget driver for further processing.
+		 */
+		actual = length - req->request.actual;
+		req->request.actual = actual;
 
-			ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
-					event, status);
-			if (ret)
-				break;
-		} while (++i < req->request.num_mapped_sgs);
+		if (ret && chain && (actual < length) && req->num_pending_sgs)
+			return __dwc3_gadget_kick_transfer(dep, 0);
 
 		dwc3_gadget_giveback(dep, req, status);
 
-		if (ret)
+		if (ret) {
+			if ((event->status & DEPEVT_STATUS_IOC) &&
+			    (trb->ctrl & DWC3_TRB_CTRL_IOC))
+				ioc = true;
 			break;
-	} while (1);
+		}
+	}
 
 	/*
 	 * Our endpoint might get disabled by another thread during
@@ -2092,10 +2016,9 @@ static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
 		return 1;
 	}
 
-	if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
-		if ((event->status & DEPEVT_STATUS_IOC) &&
-				(trb->ctrl & DWC3_TRB_CTRL_IOC))
-			return 0;
+	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && ioc)
+		return 0;
+
 	return 1;
 }
 
@@ -2311,6 +2234,18 @@ static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
 	 *
 	 * - Issue EndTransfer WITH CMDIOC bit set
 	 * - Wait 100us
+	 *
+	 * As of IP version 3.10a of the DWC_usb3 IP, the controller
+	 * supports a mode to work around the above limitation. The
+	 * software can poll the CMDACT bit in the DEPCMD register
+	 * after issuing a EndTransfer command. This mode is enabled
+	 * by writing GUCTL2[14]. This polling is already done in the
+	 * dwc3_send_gadget_ep_cmd() function so if the mode is
+	 * enabled, the EndTransfer command will have completed upon
+	 * returning from this function and we don't need to delay for
+	 * 100us.
+	 *
+	 * This mode is NOT available on the DWC_usb31 IP.
 	 */
 
 	cmd = DWC3_DEPCMD_ENDTRANSFER;
@@ -2322,7 +2257,9 @@ static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
 	WARN_ON_ONCE(ret);
 	dep->resource_index = 0;
 	dep->flags &= ~DWC3_EP_BUSY;
-	udelay(100);
+
+	if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A)
+		udelay(100);
 }
 
 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
diff --git a/drivers/usb/dwc3/ulpi.c b/drivers/usb/dwc3/ulpi.c
index ec004c6d76f2..bd86f84f3790 100644
--- a/drivers/usb/dwc3/ulpi.c
+++ b/drivers/usb/dwc3/ulpi.c
@@ -35,9 +35,9 @@ static int dwc3_ulpi_busyloop(struct dwc3 *dwc)
 	return -ETIMEDOUT;
 }
 
-static int dwc3_ulpi_read(struct ulpi_ops *ops, u8 addr)
+static int dwc3_ulpi_read(struct device *dev, u8 addr)
 {
-	struct dwc3 *dwc = dev_get_drvdata(ops->dev);
+	struct dwc3 *dwc = dev_get_drvdata(dev);
 	u32 reg;
 	int ret;
 
@@ -53,9 +53,9 @@ static int dwc3_ulpi_read(struct ulpi_ops *ops, u8 addr)
 	return DWC3_GUSB2PHYACC_DATA(reg);
 }
 
-static int dwc3_ulpi_write(struct ulpi_ops *ops, u8 addr, u8 val)
+static int dwc3_ulpi_write(struct device *dev, u8 addr, u8 val)
 {
-	struct dwc3 *dwc = dev_get_drvdata(ops->dev);
+	struct dwc3 *dwc = dev_get_drvdata(dev);
 	u32 reg;
 
 	reg = DWC3_GUSB2PHYACC_NEWREGREQ | DWC3_ULPI_ADDR(addr);
@@ -65,7 +65,7 @@ static int dwc3_ulpi_write(struct ulpi_ops *ops, u8 addr, u8 val)
 	return dwc3_ulpi_busyloop(dwc);
 }
 
-static struct ulpi_ops dwc3_ulpi_ops = {
+static const struct ulpi_ops dwc3_ulpi_ops = {
 	.read = dwc3_ulpi_read,
 	.write = dwc3_ulpi_write,
 };
diff --git a/drivers/usb/early/ehci-dbgp.c b/drivers/usb/early/ehci-dbgp.c
index 12731e67d2c7..ea73afb026d8 100644
--- a/drivers/usb/early/ehci-dbgp.c
+++ b/drivers/usb/early/ehci-dbgp.c
@@ -20,7 +20,6 @@
 #include <linux/usb/ehci_def.h>
 #include <linux/delay.h>
 #include <linux/serial_core.h>
-#include <linux/kconfig.h>
 #include <linux/kgdb.h>
 #include <linux/kthread.h>
 #include <asm/io.h>
diff --git a/drivers/usb/gadget/Kconfig b/drivers/usb/gadget/Kconfig
index 3c3f31ceece7..8ad203296079 100644
--- a/drivers/usb/gadget/Kconfig
+++ b/drivers/usb/gadget/Kconfig
@@ -209,25 +209,6 @@ config USB_F_PRINTER
 config USB_F_TCM
 	tristate
 
-choice
-	tristate "USB Gadget Drivers"
-	default USB_ETH
-	help
-	  A Linux "Gadget Driver" talks to the USB Peripheral Controller
-	  driver through the abstract "gadget" API.  Some other operating
-	  systems call these "client" drivers, of which "class drivers"
-	  are a subset (implementing a USB device class specification).
-	  A gadget driver implements one or more USB functions using
-	  the peripheral hardware.
-
-	  Gadget drivers are hardware-neutral, or "platform independent",
-	  except that they sometimes must understand quirks or limitations
-	  of the particular controllers they work with.  For example, when
-	  a controller doesn't support alternate configurations or provide
-	  enough of the right types of endpoints, the gadget driver might
-	  not be able work with that controller, or might need to implement
-	  a less common variant of a device class protocol.
-
 # this first set of drivers all depend on bulk-capable hardware.
 
 config USB_CONFIGFS
@@ -439,6 +420,7 @@ config USB_CONFIGFS_F_HID
 config USB_CONFIGFS_F_UVC
 	bool "USB Webcam function"
 	depends on USB_CONFIGFS
+	depends on VIDEO_V4L2
 	depends on VIDEO_DEV
 	select VIDEOBUF2_VMALLOC
 	select USB_F_UVC
@@ -475,6 +457,25 @@ config USB_CONFIGFS_F_TCM
 	  Both protocols can work on USB2.0 and USB3.0.
 	  UAS utilizes the USB 3.0 feature called streams support.
 
+choice
+	tristate "USB Gadget Drivers"
+	default USB_ETH
+	help
+	  A Linux "Gadget Driver" talks to the USB Peripheral Controller
+	  driver through the abstract "gadget" API.  Some other operating
+	  systems call these "client" drivers, of which "class drivers"
+	  are a subset (implementing a USB device class specification).
+	  A gadget driver implements one or more USB functions using
+	  the peripheral hardware.
+
+	  Gadget drivers are hardware-neutral, or "platform independent",
+	  except that they sometimes must understand quirks or limitations
+	  of the particular controllers they work with.  For example, when
+	  a controller doesn't support alternate configurations or provide
+	  enough of the right types of endpoints, the gadget driver might
+	  not be able work with that controller, or might need to implement
+	  a less common variant of a device class protocol.
+
 source "drivers/usb/gadget/legacy/Kconfig"
 
 endchoice
diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c
index eb648485a58c..32176f779861 100644
--- a/drivers/usb/gadget/composite.c
+++ b/drivers/usb/gadget/composite.c
@@ -1893,17 +1893,21 @@ unknown:
 		/* functions always handle their interfaces and endpoints...
 		 * punt other recipients (other, WUSB, ...) to the current
 		 * configuration code.
-		 *
-		 * REVISIT it could make sense to let the composite device
-		 * take such requests too, if that's ever needed:  to work
-		 * in config 0, etc.
 		 */
 		if (cdev->config) {
 			list_for_each_entry(f, &cdev->config->functions, list)
-				if (f->req_match && f->req_match(f, ctrl))
+				if (f->req_match &&
+				    f->req_match(f, ctrl, false))
 					goto try_fun_setup;
-			f = NULL;
+		} else {
+			struct usb_configuration *c;
+			list_for_each_entry(c, &cdev->configs, list)
+				list_for_each_entry(f, &c->functions, list)
+					if (f->req_match &&
+					    f->req_match(f, ctrl, true))
+						goto try_fun_setup;
 		}
+		f = NULL;
 
 		switch (ctrl->bRequestType & USB_RECIP_MASK) {
 		case USB_RECIP_INTERFACE:
@@ -1913,6 +1917,8 @@ unknown:
 			break;
 
 		case USB_RECIP_ENDPOINT:
+			if (!cdev->config)
+				break;
 			endp = ((w_index & 0x80) >> 3) | (w_index & 0x0f);
 			list_for_each_entry(f, &cdev->config->functions, list) {
 				if (test_bit(endp, f->endpoints))
@@ -2124,14 +2130,14 @@ int composite_os_desc_req_prepare(struct usb_composite_dev *cdev,
 
 	cdev->os_desc_req = usb_ep_alloc_request(ep0, GFP_KERNEL);
 	if (!cdev->os_desc_req) {
-		ret = PTR_ERR(cdev->os_desc_req);
+		ret = -ENOMEM;
 		goto end;
 	}
 
 	/* OS feature descriptor length <= 4kB */
 	cdev->os_desc_req->buf = kmalloc(4096, GFP_KERNEL);
 	if (!cdev->os_desc_req->buf) {
-		ret = PTR_ERR(cdev->os_desc_req->buf);
+		ret = -ENOMEM;
 		kfree(cdev->os_desc_req);
 		goto end;
 	}
diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c
index 70cf3477f951..3984787f8e97 100644
--- a/drivers/usb/gadget/configfs.c
+++ b/drivers/usb/gadget/configfs.c
@@ -1211,8 +1211,9 @@ static void purge_configs_funcs(struct gadget_info *gi)
 
 			list_move_tail(&f->list, &cfg->func_list);
 			if (f->unbind) {
-				dev_err(&gi->cdev.gadget->dev, "unbind function"
-						" '%s'/%p\n", f->name, f);
+				dev_dbg(&gi->cdev.gadget->dev,
+				         "unbind function '%s'/%p\n",
+				         f->name, f);
 				f->unbind(c, f);
 			}
 		}
@@ -1490,7 +1491,9 @@ void unregister_gadget_item(struct config_item *item)
 {
 	struct gadget_info *gi = to_gadget_info(item);
 
+	mutex_lock(&gi->lock);
 	unregister_gadget(gi);
+	mutex_unlock(&gi->lock);
 }
 EXPORT_SYMBOL_GPL(unregister_gadget_item);
 
diff --git a/drivers/usb/gadget/function/f_eem.c b/drivers/usb/gadget/function/f_eem.c
index d58bfc32be9e..007ec6e4a5d4 100644
--- a/drivers/usb/gadget/function/f_eem.c
+++ b/drivers/usb/gadget/function/f_eem.c
@@ -341,11 +341,15 @@ static struct sk_buff *eem_wrap(struct gether *port, struct sk_buff *skb)
 {
 	struct sk_buff	*skb2 = NULL;
 	struct usb_ep	*in = port->in_ep;
-	int		padlen = 0;
-	u16		len = skb->len;
+	int		headroom, tailroom, padlen = 0;
+	u16		len;
 
-	int headroom = skb_headroom(skb);
-	int tailroom = skb_tailroom(skb);
+	if (!skb)
+		return NULL;
+
+	len = skb->len;
+	headroom = skb_headroom(skb);
+	tailroom = skb_tailroom(skb);
 
 	/* When (len + EEM_HLEN + ETH_FCS_LEN) % in->maxpacket) is 0,
 	 * stick two bytes of zero-length EEM packet on the end.
diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index 5c8429f23a89..54ad100af35b 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -98,6 +98,9 @@ static int ffs_func_set_alt(struct usb_function *, unsigned, unsigned);
 static void ffs_func_disable(struct usb_function *);
 static int ffs_func_setup(struct usb_function *,
 			  const struct usb_ctrlrequest *);
+static bool ffs_func_req_match(struct usb_function *,
+			       const struct usb_ctrlrequest *,
+			       bool config0);
 static void ffs_func_suspend(struct usb_function *);
 static void ffs_func_resume(struct usb_function *);
 
@@ -1193,15 +1196,15 @@ ffs_sb_make_inode(struct super_block *sb, void *data,
 	inode = new_inode(sb);
 
 	if (likely(inode)) {
-		struct timespec current_time = CURRENT_TIME;
+		struct timespec ts = current_time(inode);
 
 		inode->i_ino	 = get_next_ino();
 		inode->i_mode    = perms->mode;
 		inode->i_uid     = perms->uid;
 		inode->i_gid     = perms->gid;
-		inode->i_atime   = current_time;
-		inode->i_mtime   = current_time;
-		inode->i_ctime   = current_time;
+		inode->i_atime   = ts;
+		inode->i_mtime   = ts;
+		inode->i_ctime   = ts;
 		inode->i_private = data;
 		if (fops)
 			inode->i_fop = fops;
@@ -2243,7 +2246,9 @@ static int __ffs_data_got_descs(struct ffs_data *ffs,
 			      FUNCTIONFS_HAS_SS_DESC |
 			      FUNCTIONFS_HAS_MS_OS_DESC |
 			      FUNCTIONFS_VIRTUAL_ADDR |
-			      FUNCTIONFS_EVENTFD)) {
+			      FUNCTIONFS_EVENTFD |
+			      FUNCTIONFS_ALL_CTRL_RECIP |
+			      FUNCTIONFS_CONFIG0_SETUP)) {
 			ret = -ENOSYS;
 			goto error;
 		}
@@ -3094,8 +3099,9 @@ static int ffs_func_setup(struct usb_function *f,
 	 * handle them.  All other either handled by composite or
 	 * passed to usb_configuration->setup() (if one is set).  No
 	 * matter, we will handle requests directed to endpoint here
-	 * as well (as it's straightforward) but what to do with any
-	 * other request?
+	 * as well (as it's straightforward).  Other request recipient
+	 * types are only handled when the user flag FUNCTIONFS_ALL_CTRL_RECIP
+	 * is being used.
 	 */
 	if (ffs->state != FFS_ACTIVE)
 		return -ENODEV;
@@ -3116,7 +3122,10 @@ static int ffs_func_setup(struct usb_function *f,
 		break;
 
 	default:
-		return -EOPNOTSUPP;
+		if (func->ffs->user_flags & FUNCTIONFS_ALL_CTRL_RECIP)
+			ret = le16_to_cpu(creq->wIndex);
+		else
+			return -EOPNOTSUPP;
 	}
 
 	spin_lock_irqsave(&ffs->ev.waitq.lock, flags);
@@ -3128,6 +3137,28 @@ static int ffs_func_setup(struct usb_function *f,
 	return 0;
 }
 
+static bool ffs_func_req_match(struct usb_function *f,
+			       const struct usb_ctrlrequest *creq,
+			       bool config0)
+{
+	struct ffs_function *func = ffs_func_from_usb(f);
+
+	if (config0 && !(func->ffs->user_flags & FUNCTIONFS_CONFIG0_SETUP))
+		return false;
+
+	switch (creq->bRequestType & USB_RECIP_MASK) {
+	case USB_RECIP_INTERFACE:
+		return ffs_func_revmap_intf(func,
+					    le16_to_cpu(creq->wIndex) >= 0);
+	case USB_RECIP_ENDPOINT:
+		return ffs_func_revmap_ep(func,
+					  le16_to_cpu(creq->wIndex) >= 0);
+	default:
+		return (bool) (func->ffs->user_flags &
+			       FUNCTIONFS_ALL_CTRL_RECIP);
+	}
+}
+
 static void ffs_func_suspend(struct usb_function *f)
 {
 	ENTER();
@@ -3378,6 +3409,7 @@ static struct usb_function *ffs_alloc(struct usb_function_instance *fi)
 	func->function.set_alt = ffs_func_set_alt;
 	func->function.disable = ffs_func_disable;
 	func->function.setup   = ffs_func_setup;
+	func->function.req_match = ffs_func_req_match;
 	func->function.suspend = ffs_func_suspend;
 	func->function.resume  = ffs_func_resume;
 	func->function.free_func = ffs_free;
@@ -3470,6 +3502,11 @@ static void _ffs_free_dev(struct ffs_dev *dev)
 	list_del(&dev->entry);
 	if (dev->name_allocated)
 		kfree(dev->name);
+
+	/* Clear the private_data pointer to stop incorrect dev access */
+	if (dev->ffs_data)
+		dev->ffs_data->private_data = NULL;
+
 	kfree(dev);
 	if (list_empty(&ffs_devices))
 		functionfs_cleanup();
diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c
index 51980c50546d..e2966f87c860 100644
--- a/drivers/usb/gadget/function/f_hid.c
+++ b/drivers/usb/gadget/function/f_hid.c
@@ -365,7 +365,7 @@ static int f_hidg_open(struct inode *inode, struct file *fd)
 static inline struct usb_request *hidg_alloc_ep_req(struct usb_ep *ep,
 						    unsigned length)
 {
-	return alloc_ep_req(ep, length, length);
+	return alloc_ep_req(ep, length);
 }
 
 static void hidg_set_report_complete(struct usb_ep *ep, struct usb_request *req)
@@ -617,14 +617,10 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f)
 
 	/* preallocate request and buffer */
 	status = -ENOMEM;
-	hidg->req = usb_ep_alloc_request(hidg->in_ep, GFP_KERNEL);
+	hidg->req = alloc_ep_req(hidg->in_ep, hidg->report_length);
 	if (!hidg->req)
 		goto fail;
 
-	hidg->req->buf = kmalloc(hidg->report_length, GFP_KERNEL);
-	if (!hidg->req->buf)
-		goto fail;
-
 	/* set descriptor dynamic values */
 	hidg_interface_desc.bInterfaceSubClass = hidg->bInterfaceSubClass;
 	hidg_interface_desc.bInterfaceProtocol = hidg->bInterfaceProtocol;
@@ -677,11 +673,8 @@ fail_free_descs:
 	usb_free_all_descriptors(f);
 fail:
 	ERROR(f->config->cdev, "hidg_bind FAILED\n");
-	if (hidg->req != NULL) {
-		kfree(hidg->req->buf);
-		if (hidg->in_ep != NULL)
-			usb_ep_free_request(hidg->in_ep, hidg->req);
-	}
+	if (hidg->req != NULL)
+		free_ep_req(hidg->in_ep, hidg->req);
 
 	return status;
 }
@@ -809,11 +802,21 @@ end:
 
 CONFIGFS_ATTR(f_hid_opts_, report_desc);
 
+static ssize_t f_hid_opts_dev_show(struct config_item *item, char *page)
+{
+	struct f_hid_opts *opts = to_f_hid_opts(item);
+
+	return sprintf(page, "%d:%d\n", major, opts->minor);
+}
+
+CONFIGFS_ATTR_RO(f_hid_opts_, dev);
+
 static struct configfs_attribute *hid_attrs[] = {
 	&f_hid_opts_attr_subclass,
 	&f_hid_opts_attr_protocol,
 	&f_hid_opts_attr_report_length,
 	&f_hid_opts_attr_report_desc,
+	&f_hid_opts_attr_dev,
 	NULL,
 };
 
@@ -910,8 +913,7 @@ static void hidg_unbind(struct usb_configuration *c, struct usb_function *f)
 
 	/* disable/free request and end point */
 	usb_ep_disable(hidg->in_ep);
-	kfree(hidg->req->buf);
-	usb_ep_free_request(hidg->in_ep, hidg->req);
+	free_ep_req(hidg->in_ep, hidg->req);
 
 	usb_free_all_descriptors(f);
 }
diff --git a/drivers/usb/gadget/function/f_loopback.c b/drivers/usb/gadget/function/f_loopback.c
index 3a9f8f9c77bd..e70093835e14 100644
--- a/drivers/usb/gadget/function/f_loopback.c
+++ b/drivers/usb/gadget/function/f_loopback.c
@@ -308,9 +308,7 @@ static void disable_loopback(struct f_loopback *loop)
 
 static inline struct usb_request *lb_alloc_ep_req(struct usb_ep *ep, int len)
 {
-	struct f_loopback	*loop = ep->driver_data;
-
-	return alloc_ep_req(ep, len, loop->buflen);
+	return alloc_ep_req(ep, len);
 }
 
 static int alloc_requests(struct usb_composite_dev *cdev,
@@ -333,7 +331,7 @@ static int alloc_requests(struct usb_composite_dev *cdev,
 		if (!in_req)
 			goto fail;
 
-		out_req = lb_alloc_ep_req(loop->out_ep, 0);
+		out_req = lb_alloc_ep_req(loop->out_ep, loop->buflen);
 		if (!out_req)
 			goto fail_in;
 
@@ -593,13 +591,9 @@ DECLARE_USB_FUNCTION(Loopback, loopback_alloc_instance, loopback_alloc);
 
 int __init lb_modinit(void)
 {
-	int ret;
-
-	ret = usb_function_register(&Loopbackusb_func);
-	if (ret)
-		return ret;
-	return ret;
+	return usb_function_register(&Loopbackusb_func);
 }
+
 void __exit lb_modexit(void)
 {
 	usb_function_unregister(&Loopbackusb_func);
diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c
index 2505117e88e8..8f3659b65f53 100644
--- a/drivers/usb/gadget/function/f_mass_storage.c
+++ b/drivers/usb/gadget/function/f_mass_storage.c
@@ -311,11 +311,7 @@ struct fsg_common {
 	/* Gadget's private data. */
 	void			*private_data;
 
-	/*
-	 * Vendor (8 chars), product (16 chars), release (4
-	 * hexadecimal digits) and NUL byte
-	 */
-	char inquiry_string[8 + 16 + 4 + 1];
+	char inquiry_string[INQUIRY_STRING_LEN];
 
 	struct kref		ref;
 };
@@ -1107,7 +1103,12 @@ static int do_inquiry(struct fsg_common *common, struct fsg_buffhd *bh)
 	buf[5] = 0;		/* No special options */
 	buf[6] = 0;
 	buf[7] = 0;
-	memcpy(buf + 8, common->inquiry_string, sizeof common->inquiry_string);
+	if (curlun->inquiry_string[0])
+		memcpy(buf + 8, curlun->inquiry_string,
+		       sizeof(curlun->inquiry_string));
+	else
+		memcpy(buf + 8, common->inquiry_string,
+		       sizeof(common->inquiry_string));
 	return 36;
 }
 
@@ -3209,12 +3210,27 @@ static ssize_t fsg_lun_opts_nofua_store(struct config_item *item,
 
 CONFIGFS_ATTR(fsg_lun_opts_, nofua);
 
+static ssize_t fsg_lun_opts_inquiry_string_show(struct config_item *item,
+						char *page)
+{
+	return fsg_show_inquiry_string(to_fsg_lun_opts(item)->lun, page);
+}
+
+static ssize_t fsg_lun_opts_inquiry_string_store(struct config_item *item,
+						 const char *page, size_t len)
+{
+	return fsg_store_inquiry_string(to_fsg_lun_opts(item)->lun, page, len);
+}
+
+CONFIGFS_ATTR(fsg_lun_opts_, inquiry_string);
+
 static struct configfs_attribute *fsg_lun_attrs[] = {
 	&fsg_lun_opts_attr_file,
 	&fsg_lun_opts_attr_ro,
 	&fsg_lun_opts_attr_removable,
 	&fsg_lun_opts_attr_cdrom,
 	&fsg_lun_opts_attr_nofua,
+	&fsg_lun_opts_attr_inquiry_string,
 	NULL,
 };
 
diff --git a/drivers/usb/gadget/function/f_mass_storage.h b/drivers/usb/gadget/function/f_mass_storage.h
index b6a9918eaefb..d3902313b8ac 100644
--- a/drivers/usb/gadget/function/f_mass_storage.h
+++ b/drivers/usb/gadget/function/f_mass_storage.h
@@ -100,6 +100,7 @@ struct fsg_lun_config {
 	char removable;
 	char cdrom;
 	char nofua;
+	char inquiry_string[INQUIRY_STRING_LEN];
 };
 
 struct fsg_config {
diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c
index 58fc199a18ec..a5719f271bf0 100644
--- a/drivers/usb/gadget/function/f_midi.c
+++ b/drivers/usb/gadget/function/f_midi.c
@@ -51,6 +51,19 @@ static const char f_midi_longname[] = "MIDI Gadget";
  */
 #define MAX_PORTS 16
 
+/* MIDI message states */
+enum {
+	STATE_INITIAL = 0,	/* pseudo state */
+	STATE_1PARAM,
+	STATE_2PARAM_1,
+	STATE_2PARAM_2,
+	STATE_SYSEX_0,
+	STATE_SYSEX_1,
+	STATE_SYSEX_2,
+	STATE_REAL_TIME,
+	STATE_FINISHED,		/* pseudo state */
+};
+
 /*
  * This is a gadget, and the IN/OUT naming is from the host's perspective.
  * USB -> OUT endpoint -> rawmidi
@@ -61,13 +74,6 @@ struct gmidi_in_port {
 	int active;
 	uint8_t cable;
 	uint8_t state;
-#define STATE_UNKNOWN	0
-#define STATE_1PARAM	1
-#define STATE_2PARAM_1	2
-#define STATE_2PARAM_2	3
-#define STATE_SYSEX_0	4
-#define STATE_SYSEX_1	5
-#define STATE_SYSEX_2	6
 	uint8_t data[2];
 };
 
@@ -205,7 +211,7 @@ static struct usb_gadget_strings *midi_strings[] = {
 static inline struct usb_request *midi_alloc_ep_req(struct usb_ep *ep,
 						    unsigned length)
 {
-	return alloc_ep_req(ep, length, length);
+	return alloc_ep_req(ep, length);
 }
 
 static const uint8_t f_midi_cin_length[] = {
@@ -299,6 +305,19 @@ f_midi_complete(struct usb_ep *ep, struct usb_request *req)
 	}
 }
 
+static void f_midi_drop_out_substreams(struct f_midi *midi)
+{
+	unsigned int i;
+
+	for (i = 0; i < midi->in_ports; i++) {
+		struct gmidi_in_port *port = midi->in_ports_array + i;
+		struct snd_rawmidi_substream *substream = port->substream;
+
+		if (port->active && substream)
+			snd_rawmidi_drop_output(substream);
+	}
+}
+
 static int f_midi_start_ep(struct f_midi *midi,
 			   struct usb_function *f,
 			   struct usb_ep *ep)
@@ -360,9 +379,8 @@ static int f_midi_set_alt(struct usb_function *f, unsigned intf, unsigned alt)
 	/* allocate a bunch of read buffers and queue them all at once. */
 	for (i = 0; i < midi->qlen && err == 0; i++) {
 		struct usb_request *req =
-			midi_alloc_ep_req(midi->out_ep,
-				max_t(unsigned, midi->buflen,
-					bulk_out_desc.wMaxPacketSize));
+			midi_alloc_ep_req(midi->out_ep, midi->buflen);
+
 		if (req == NULL)
 			return -ENOMEM;
 
@@ -397,6 +415,8 @@ static void f_midi_disable(struct usb_function *f)
 	/* release IN requests */
 	while (kfifo_get(&midi->in_req_fifo, &req))
 		free_ep_req(midi->in_ep, req);
+
+	f_midi_drop_out_substreams(midi);
 }
 
 static int f_midi_snd_free(struct snd_device *device)
@@ -404,130 +424,166 @@ static int f_midi_snd_free(struct snd_device *device)
 	return 0;
 }
 
-static void f_midi_transmit_packet(struct usb_request *req, uint8_t p0,
-					uint8_t p1, uint8_t p2, uint8_t p3)
-{
-	unsigned length = req->length;
-	u8 *buf = (u8 *)req->buf + length;
-
-	buf[0] = p0;
-	buf[1] = p1;
-	buf[2] = p2;
-	buf[3] = p3;
-	req->length = length + 4;
-}
-
 /*
  * Converts MIDI commands to USB MIDI packets.
  */
 static void f_midi_transmit_byte(struct usb_request *req,
 				 struct gmidi_in_port *port, uint8_t b)
 {
-	uint8_t p0 = port->cable << 4;
+	uint8_t p[4] = { port->cable << 4, 0, 0, 0 };
+	uint8_t next_state = STATE_INITIAL;
+
+	switch (b) {
+	case 0xf8 ... 0xff:
+		/* System Real-Time Messages */
+		p[0] |= 0x0f;
+		p[1] = b;
+		next_state = port->state;
+		port->state = STATE_REAL_TIME;
+		break;
 
-	if (b >= 0xf8) {
-		f_midi_transmit_packet(req, p0 | 0x0f, b, 0, 0);
-	} else if (b >= 0xf0) {
+	case 0xf7:
+		/* End of SysEx */
+		switch (port->state) {
+		case STATE_SYSEX_0:
+			p[0] |= 0x05;
+			p[1] = 0xf7;
+			next_state = STATE_FINISHED;
+			break;
+		case STATE_SYSEX_1:
+			p[0] |= 0x06;
+			p[1] = port->data[0];
+			p[2] = 0xf7;
+			next_state = STATE_FINISHED;
+			break;
+		case STATE_SYSEX_2:
+			p[0] |= 0x07;
+			p[1] = port->data[0];
+			p[2] = port->data[1];
+			p[3] = 0xf7;
+			next_state = STATE_FINISHED;
+			break;
+		default:
+			/* Ignore byte */
+			next_state = port->state;
+			port->state = STATE_INITIAL;
+		}
+		break;
+
+	case 0xf0 ... 0xf6:
+		/* System Common Messages */
+		port->data[0] = port->data[1] = 0;
+		port->state = STATE_INITIAL;
 		switch (b) {
 		case 0xf0:
 			port->data[0] = b;
-			port->state = STATE_SYSEX_1;
+			port->data[1] = 0;
+			next_state = STATE_SYSEX_1;
 			break;
 		case 0xf1:
 		case 0xf3:
 			port->data[0] = b;
-			port->state = STATE_1PARAM;
+			next_state = STATE_1PARAM;
 			break;
 		case 0xf2:
 			port->data[0] = b;
-			port->state = STATE_2PARAM_1;
+			next_state = STATE_2PARAM_1;
 			break;
 		case 0xf4:
 		case 0xf5:
-			port->state = STATE_UNKNOWN;
+			next_state = STATE_INITIAL;
 			break;
 		case 0xf6:
-			f_midi_transmit_packet(req, p0 | 0x05, 0xf6, 0, 0);
-			port->state = STATE_UNKNOWN;
-			break;
-		case 0xf7:
-			switch (port->state) {
-			case STATE_SYSEX_0:
-				f_midi_transmit_packet(req,
-					p0 | 0x05, 0xf7, 0, 0);
-				break;
-			case STATE_SYSEX_1:
-				f_midi_transmit_packet(req,
-					p0 | 0x06, port->data[0], 0xf7, 0);
-				break;
-			case STATE_SYSEX_2:
-				f_midi_transmit_packet(req,
-					p0 | 0x07, port->data[0],
-					port->data[1], 0xf7);
-				break;
-			}
-			port->state = STATE_UNKNOWN;
+			p[0] |= 0x05;
+			p[1] = 0xf6;
+			next_state = STATE_FINISHED;
 			break;
 		}
-	} else if (b >= 0x80) {
+		break;
+
+	case 0x80 ... 0xef:
+		/*
+		 * Channel Voice Messages, Channel Mode Messages
+		 * and Control Change Messages.
+		 */
 		port->data[0] = b;
+		port->data[1] = 0;
+		port->state = STATE_INITIAL;
 		if (b >= 0xc0 && b <= 0xdf)
-			port->state = STATE_1PARAM;
+			next_state = STATE_1PARAM;
 		else
-			port->state = STATE_2PARAM_1;
-	} else { /* b < 0x80 */
+			next_state = STATE_2PARAM_1;
+		break;
+
+	case 0x00 ... 0x7f:
+		/* Message parameters */
 		switch (port->state) {
 		case STATE_1PARAM:
-			if (port->data[0] < 0xf0) {
-				p0 |= port->data[0] >> 4;
-			} else {
-				p0 |= 0x02;
-				port->state = STATE_UNKNOWN;
-			}
-			f_midi_transmit_packet(req, p0, port->data[0], b, 0);
+			if (port->data[0] < 0xf0)
+				p[0] |= port->data[0] >> 4;
+			else
+				p[0] |= 0x02;
+
+			p[1] = port->data[0];
+			p[2] = b;
+			/* This is to allow Running State Messages */
+			next_state = STATE_1PARAM;
 			break;
 		case STATE_2PARAM_1:
 			port->data[1] = b;
-			port->state = STATE_2PARAM_2;
+			next_state = STATE_2PARAM_2;
 			break;
 		case STATE_2PARAM_2:
-			if (port->data[0] < 0xf0) {
-				p0 |= port->data[0] >> 4;
-				port->state = STATE_2PARAM_1;
-			} else {
-				p0 |= 0x03;
-				port->state = STATE_UNKNOWN;
-			}
-			f_midi_transmit_packet(req,
-				p0, port->data[0], port->data[1], b);
+			if (port->data[0] < 0xf0)
+				p[0] |= port->data[0] >> 4;
+			else
+				p[0] |= 0x03;
+
+			p[1] = port->data[0];
+			p[2] = port->data[1];
+			p[3] = b;
+			/* This is to allow Running State Messages */
+			next_state = STATE_2PARAM_1;
 			break;
 		case STATE_SYSEX_0:
 			port->data[0] = b;
-			port->state = STATE_SYSEX_1;
+			next_state = STATE_SYSEX_1;
 			break;
 		case STATE_SYSEX_1:
 			port->data[1] = b;
-			port->state = STATE_SYSEX_2;
+			next_state = STATE_SYSEX_2;
 			break;
 		case STATE_SYSEX_2:
-			f_midi_transmit_packet(req,
-				p0 | 0x04, port->data[0], port->data[1], b);
-			port->state = STATE_SYSEX_0;
+			p[0] |= 0x04;
+			p[1] = port->data[0];
+			p[2] = port->data[1];
+			p[3] = b;
+			next_state = STATE_SYSEX_0;
 			break;
 		}
+		break;
 	}
-}
 
-static void f_midi_drop_out_substreams(struct f_midi *midi)
-{
-	unsigned int i;
+	/* States where we have to write into the USB request */
+	if (next_state == STATE_FINISHED ||
+	    port->state == STATE_SYSEX_2 ||
+	    port->state == STATE_1PARAM ||
+	    port->state == STATE_2PARAM_2 ||
+	    port->state == STATE_REAL_TIME) {
 
-	for (i = 0; i < midi->in_ports; i++) {
-		struct gmidi_in_port *port = midi->in_ports_array + i;
-		struct snd_rawmidi_substream *substream = port->substream;
-		if (port->active && substream)
-			snd_rawmidi_drop_output(substream);
+		unsigned int length = req->length;
+		u8 *buf = (u8 *)req->buf + length;
+
+		memcpy(buf, p, sizeof(p));
+		req->length = length + sizeof(p);
+
+		if (next_state == STATE_FINISHED) {
+			next_state = STATE_INITIAL;
+			port->data[0] = port->data[1] = 0;
+		}
 	}
+
+	port->state = next_state;
 }
 
 static int f_midi_do_transmit(struct f_midi *midi, struct usb_ep *ep)
@@ -642,7 +698,7 @@ static int f_midi_in_open(struct snd_rawmidi_substream *substream)
 	VDBG(midi, "%s()\n", __func__);
 	port = midi->in_ports_array + substream->number;
 	port->substream = substream;
-	port->state = STATE_UNKNOWN;
+	port->state = STATE_INITIAL;
 	return 0;
 }
 
@@ -1123,7 +1179,7 @@ static struct usb_function_instance *f_midi_alloc_inst(void)
 	opts->func_inst.free_func_inst = f_midi_free_inst;
 	opts->index = SNDRV_DEFAULT_IDX1;
 	opts->id = SNDRV_DEFAULT_STR1;
-	opts->buflen = 256;
+	opts->buflen = 512;
 	opts->qlen = 32;
 	opts->in_ports = 1;
 	opts->out_ports = 1;
diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c
index 97f0a9bc84df..639603722709 100644
--- a/drivers/usb/gadget/function/f_ncm.c
+++ b/drivers/usb/gadget/function/f_ncm.c
@@ -90,7 +90,9 @@ static inline struct f_ncm *func_to_ncm(struct usb_function *f)
 /* peak (theoretical) bulk transfer rate in bits-per-second */
 static inline unsigned ncm_bitrate(struct usb_gadget *g)
 {
-	if (gadget_is_dualspeed(g) && g->speed == USB_SPEED_HIGH)
+	if (gadget_is_superspeed(g) && g->speed == USB_SPEED_SUPER)
+		return 13 * 1024 * 8 * 1000 * 8;
+	else if (gadget_is_dualspeed(g) && g->speed == USB_SPEED_HIGH)
 		return 13 * 512 * 8 * 1000 * 8;
 	else
 		return 19 *  64 * 1 * 1000 * 8;
@@ -333,6 +335,76 @@ static struct usb_descriptor_header *ncm_hs_function[] = {
 	NULL,
 };
 
+
+/* super speed support: */
+
+static struct usb_endpoint_descriptor ss_ncm_notify_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+
+	.bEndpointAddress =	USB_DIR_IN,
+	.bmAttributes =		USB_ENDPOINT_XFER_INT,
+	.wMaxPacketSize =	cpu_to_le16(NCM_STATUS_BYTECOUNT),
+	.bInterval =		USB_MS_TO_HS_INTERVAL(NCM_STATUS_INTERVAL_MS)
+};
+
+static struct usb_ss_ep_comp_descriptor ss_ncm_notify_comp_desc = {
+	.bLength =		sizeof(ss_ncm_notify_comp_desc),
+	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
+
+	/* the following 3 values can be tweaked if necessary */
+	/* .bMaxBurst =		0, */
+	/* .bmAttributes =	0, */
+	.wBytesPerInterval =	cpu_to_le16(NCM_STATUS_BYTECOUNT),
+};
+
+static struct usb_endpoint_descriptor ss_ncm_in_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+
+	.bEndpointAddress =	USB_DIR_IN,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(1024),
+};
+
+static struct usb_endpoint_descriptor ss_ncm_out_desc = {
+	.bLength =		USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType =	USB_DT_ENDPOINT,
+
+	.bEndpointAddress =	USB_DIR_OUT,
+	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
+	.wMaxPacketSize =	cpu_to_le16(1024),
+};
+
+static struct usb_ss_ep_comp_descriptor ss_ncm_bulk_comp_desc = {
+	.bLength =		sizeof(ss_ncm_bulk_comp_desc),
+	.bDescriptorType =	USB_DT_SS_ENDPOINT_COMP,
+
+	/* the following 2 values can be tweaked if necessary */
+	/* .bMaxBurst =		0, */
+	/* .bmAttributes =	0, */
+};
+
+static struct usb_descriptor_header *ncm_ss_function[] = {
+	(struct usb_descriptor_header *) &ncm_iad_desc,
+	/* CDC NCM control descriptors */
+	(struct usb_descriptor_header *) &ncm_control_intf,
+	(struct usb_descriptor_header *) &ncm_header_desc,
+	(struct usb_descriptor_header *) &ncm_union_desc,
+	(struct usb_descriptor_header *) &ecm_desc,
+	(struct usb_descriptor_header *) &ncm_desc,
+	(struct usb_descriptor_header *) &ss_ncm_notify_desc,
+	(struct usb_descriptor_header *) &ss_ncm_notify_comp_desc,
+	/* data interface, altsettings 0 and 1 */
+	(struct usb_descriptor_header *) &ncm_data_nop_intf,
+	(struct usb_descriptor_header *) &ncm_data_intf,
+	(struct usb_descriptor_header *) &ss_ncm_in_desc,
+	(struct usb_descriptor_header *) &ss_ncm_bulk_comp_desc,
+	(struct usb_descriptor_header *) &ss_ncm_out_desc,
+	(struct usb_descriptor_header *) &ss_ncm_bulk_comp_desc,
+	NULL,
+};
+
 /* string descriptors: */
 
 #define STRING_CTRL_IDX	0
@@ -852,6 +924,8 @@ static int ncm_set_alt(struct usb_function *f, unsigned intf, unsigned alt)
 			 */
 			ncm->port.is_zlp_ok =
 				gadget_is_zlp_supported(cdev->gadget);
+			ncm->port.no_skb_reserve =
+				gadget_avoids_skb_reserve(cdev->gadget);
 			ncm->port.cdc_filter = DEFAULT_FILTER;
 			DBG(cdev, "activate ncm\n");
 			net = gether_connect(&ncm->port);
@@ -1431,8 +1505,13 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f)
 	hs_ncm_notify_desc.bEndpointAddress =
 		fs_ncm_notify_desc.bEndpointAddress;
 
+	ss_ncm_in_desc.bEndpointAddress = fs_ncm_in_desc.bEndpointAddress;
+	ss_ncm_out_desc.bEndpointAddress = fs_ncm_out_desc.bEndpointAddress;
+	ss_ncm_notify_desc.bEndpointAddress =
+		fs_ncm_notify_desc.bEndpointAddress;
+
 	status = usb_assign_descriptors(f, ncm_fs_function, ncm_hs_function,
-			NULL, NULL);
+			ncm_ss_function, NULL);
 	if (status)
 		goto fail;
 
@@ -1450,6 +1529,7 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f)
 	ncm->task_timer.function = ncm_tx_timeout;
 
 	DBG(cdev, "CDC Network: %s speed IN/%s OUT/%s NOTIFY/%s\n",
+			gadget_is_superspeed(c->cdev->gadget) ? "super" :
 			gadget_is_dualspeed(c->cdev->gadget) ? "dual" : "full",
 			ncm->port.in_ep->name, ncm->port.out_ep->name,
 			ncm->notify->name);
diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c
index 64706a789580..0de36cda6e41 100644
--- a/drivers/usb/gadget/function/f_printer.c
+++ b/drivers/usb/gadget/function/f_printer.c
@@ -889,13 +889,17 @@ static void printer_soft_reset(struct printer_dev *dev)
 /*-------------------------------------------------------------------------*/
 
 static bool gprinter_req_match(struct usb_function *f,
-			       const struct usb_ctrlrequest *ctrl)
+			       const struct usb_ctrlrequest *ctrl,
+			       bool config0)
 {
 	struct printer_dev	*dev = func_to_printer(f);
 	u16			w_index = le16_to_cpu(ctrl->wIndex);
 	u16			w_value = le16_to_cpu(ctrl->wValue);
 	u16			w_length = le16_to_cpu(ctrl->wLength);
 
+	if (config0)
+		return false;
+
 	if ((ctrl->bRequestType & USB_RECIP_MASK) != USB_RECIP_INTERFACE ||
 	    (ctrl->bRequestType & USB_TYPE_MASK) != USB_TYPE_CLASS)
 		return false;
diff --git a/drivers/usb/gadget/function/f_rndis.c b/drivers/usb/gadget/function/f_rndis.c
index c8005823b190..16562e461121 100644
--- a/drivers/usb/gadget/function/f_rndis.c
+++ b/drivers/usb/gadget/function/f_rndis.c
@@ -374,6 +374,9 @@ static struct sk_buff *rndis_add_header(struct gether *port,
 {
 	struct sk_buff *skb2;
 
+	if (!skb)
+		return NULL;
+
 	skb2 = skb_realloc_headroom(skb, sizeof(struct rndis_packet_msg_type));
 	rndis_add_hdr(skb2);
 
diff --git a/drivers/usb/gadget/function/f_sourcesink.c b/drivers/usb/gadget/function/f_sourcesink.c
index df0189ddfdd5..8784fa12ea2c 100644
--- a/drivers/usb/gadget/function/f_sourcesink.c
+++ b/drivers/usb/gadget/function/f_sourcesink.c
@@ -293,9 +293,7 @@ static struct usb_gadget_strings *sourcesink_strings[] = {
 
 static inline struct usb_request *ss_alloc_ep_req(struct usb_ep *ep, int len)
 {
-	struct f_sourcesink		*ss = ep->driver_data;
-
-	return alloc_ep_req(ep, len, ss->buflen);
+	return alloc_ep_req(ep, len);
 }
 
 static void disable_ep(struct usb_composite_dev *cdev, struct usb_ep *ep)
@@ -606,7 +604,7 @@ static int source_sink_start_ep(struct f_sourcesink *ss, bool is_in,
 	} else {
 		ep = is_in ? ss->in_ep : ss->out_ep;
 		qlen = ss->bulk_qlen;
-		size = 0;
+		size = ss->buflen;
 	}
 
 	for (i = 0; i < qlen; i++) {
diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c
index 29b41b5dee04..27ed51b5082f 100644
--- a/drivers/usb/gadget/function/f_uvc.c
+++ b/drivers/usb/gadget/function/f_uvc.c
@@ -258,6 +258,13 @@ uvc_function_setup(struct usb_function *f, const struct usb_ctrlrequest *ctrl)
 	memcpy(&uvc_event->req, ctrl, sizeof(uvc_event->req));
 	v4l2_event_queue(&uvc->vdev, &v4l2_event);
 
+	/* Pass additional setup data to userspace */
+	if (uvc->event_setup_out && uvc->event_length) {
+		uvc->control_req->length = uvc->event_length;
+		return usb_ep_queue(uvc->func.config->cdev->gadget->ep0,
+			uvc->control_req, GFP_ATOMIC);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/usb/gadget/function/rndis.c b/drivers/usb/gadget/function/rndis.c
index 943c21aafd3b..ab6ac1b74ac0 100644
--- a/drivers/usb/gadget/function/rndis.c
+++ b/drivers/usb/gadget/function/rndis.c
@@ -680,6 +680,12 @@ static int rndis_reset_response(struct rndis_params *params,
 {
 	rndis_reset_cmplt_type *resp;
 	rndis_resp_t *r;
+	u8 *xbuf;
+	u32 length;
+
+	/* drain the response queue */
+	while ((xbuf = rndis_get_next_response(params, &length)))
+		rndis_free_response(params, xbuf);
 
 	r = rndis_add_response(params, sizeof(rndis_reset_cmplt_type));
 	if (!r)
diff --git a/drivers/usb/gadget/function/storage_common.c b/drivers/usb/gadget/function/storage_common.c
index 990df221c629..8fbf6861690d 100644
--- a/drivers/usb/gadget/function/storage_common.c
+++ b/drivers/usb/gadget/function/storage_common.c
@@ -369,6 +369,12 @@ ssize_t fsg_show_removable(struct fsg_lun *curlun, char *buf)
 }
 EXPORT_SYMBOL_GPL(fsg_show_removable);
 
+ssize_t fsg_show_inquiry_string(struct fsg_lun *curlun, char *buf)
+{
+	return sprintf(buf, "%s\n", curlun->inquiry_string);
+}
+EXPORT_SYMBOL_GPL(fsg_show_inquiry_string);
+
 /*
  * The caller must hold fsg->filesem for reading when calling this function.
  */
@@ -499,4 +505,22 @@ ssize_t fsg_store_removable(struct fsg_lun *curlun, const char *buf,
 }
 EXPORT_SYMBOL_GPL(fsg_store_removable);
 
+ssize_t fsg_store_inquiry_string(struct fsg_lun *curlun, const char *buf,
+				 size_t count)
+{
+	const size_t len = min(count, sizeof(curlun->inquiry_string));
+
+	if (len == 0 || buf[0] == '\n') {
+		curlun->inquiry_string[0] = 0;
+	} else {
+		snprintf(curlun->inquiry_string,
+			 sizeof(curlun->inquiry_string), "%-28s", buf);
+		if (curlun->inquiry_string[len-1] == '\n')
+			curlun->inquiry_string[len-1] = ' ';
+	}
+
+	return count;
+}
+EXPORT_SYMBOL_GPL(fsg_store_inquiry_string);
+
 MODULE_LICENSE("GPL");
diff --git a/drivers/usb/gadget/function/storage_common.h b/drivers/usb/gadget/function/storage_common.h
index c3544e61da66..e69848994cb4 100644
--- a/drivers/usb/gadget/function/storage_common.h
+++ b/drivers/usb/gadget/function/storage_common.h
@@ -88,6 +88,12 @@ do {									\
 #define ASC(x)		((u8) ((x) >> 8))
 #define ASCQ(x)		((u8) (x))
 
+/*
+ * Vendor (8 chars), product (16 chars), release (4 hexadecimal digits) and NUL
+ * byte
+ */
+#define INQUIRY_STRING_LEN ((size_t) (8 + 16 + 4 + 1))
+
 struct fsg_lun {
 	struct file	*filp;
 	loff_t		file_length;
@@ -112,6 +118,7 @@ struct fsg_lun {
 	struct device	dev;
 	const char	*name;		/* "lun.name" */
 	const char	**name_pfx;	/* "function.name" */
+	char		inquiry_string[INQUIRY_STRING_LEN];
 };
 
 static inline bool fsg_lun_is_open(struct fsg_lun *curlun)
@@ -210,6 +217,7 @@ ssize_t fsg_show_ro(struct fsg_lun *curlun, char *buf);
 ssize_t fsg_show_nofua(struct fsg_lun *curlun, char *buf);
 ssize_t fsg_show_file(struct fsg_lun *curlun, struct rw_semaphore *filesem,
 		      char *buf);
+ssize_t fsg_show_inquiry_string(struct fsg_lun *curlun, char *buf);
 ssize_t fsg_show_cdrom(struct fsg_lun *curlun, char *buf);
 ssize_t fsg_show_removable(struct fsg_lun *curlun, char *buf);
 ssize_t fsg_store_ro(struct fsg_lun *curlun, struct rw_semaphore *filesem,
@@ -221,5 +229,7 @@ ssize_t fsg_store_cdrom(struct fsg_lun *curlun, struct rw_semaphore *filesem,
 			const char *buf, size_t count);
 ssize_t fsg_store_removable(struct fsg_lun *curlun, const char *buf,
 			    size_t count);
+ssize_t fsg_store_inquiry_string(struct fsg_lun *curlun, const char *buf,
+				 size_t count);
 
 #endif /* USB_STORAGE_COMMON_H */
diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c
index a3f7e7c55ebb..9c8c9ed1dc9e 100644
--- a/drivers/usb/gadget/function/u_ether.c
+++ b/drivers/usb/gadget/function/u_ether.c
@@ -82,6 +82,7 @@ struct eth_dev {
 #define	WORK_RX_MEMORY		0
 
 	bool			zlp;
+	bool			no_skb_reserve;
 	u8			host_mac[ETH_ALEN];
 	u8			dev_mac[ETH_ALEN];
 };
@@ -233,7 +234,8 @@ rx_submit(struct eth_dev *dev, struct usb_request *req, gfp_t gfp_flags)
 	 * but on at least one, checksumming fails otherwise.  Note:
 	 * RNDIS headers involve variable numbers of LE32 values.
 	 */
-	skb_reserve(skb, NET_IP_ALIGN);
+	if (likely(!dev->no_skb_reserve))
+		skb_reserve(skb, NET_IP_ALIGN);
 
 	req->buf = skb->data;
 	req->length = size;
@@ -556,7 +558,8 @@ static netdev_tx_t eth_start_xmit(struct sk_buff *skb,
 			/* Multi frame CDC protocols may store the frame for
 			 * later which is not a dropped frame.
 			 */
-			if (dev->port_usb->supports_multi_frame)
+			if (dev->port_usb &&
+					dev->port_usb->supports_multi_frame)
 				goto multiframe;
 			goto drop;
 		}
@@ -568,7 +571,8 @@ static netdev_tx_t eth_start_xmit(struct sk_buff *skb,
 	req->complete = tx_complete;
 
 	/* NCM requires no zlp if transfer is dwNtbInMaxSize */
-	if (dev->port_usb->is_fixed &&
+	if (dev->port_usb &&
+	    dev->port_usb->is_fixed &&
 	    length == dev->port_usb->fixed_in_len &&
 	    (length % in->maxpacket) == 0)
 		req->zero = 0;
@@ -1062,6 +1066,7 @@ struct net_device *gether_connect(struct gether *link)
 
 	if (result == 0) {
 		dev->zlp = link->is_zlp_ok;
+		dev->no_skb_reserve = link->no_skb_reserve;
 		DBG(dev, "qlen %d\n", qlen(dev->gadget, dev->qmult));
 
 		dev->header_len = link->header_len;
diff --git a/drivers/usb/gadget/function/u_ether.h b/drivers/usb/gadget/function/u_ether.h
index c77145bd6b5b..81d94a7ae4b4 100644
--- a/drivers/usb/gadget/function/u_ether.h
+++ b/drivers/usb/gadget/function/u_ether.h
@@ -64,6 +64,7 @@ struct gether {
 	struct usb_ep			*out_ep;
 
 	bool				is_zlp_ok;
+	bool				no_skb_reserve;
 
 	u16				cdc_filter;
 
diff --git a/drivers/usb/gadget/function/u_serial.c b/drivers/usb/gadget/function/u_serial.c
index 6ded6345cd09..e0cd1e4c8892 100644
--- a/drivers/usb/gadget/function/u_serial.c
+++ b/drivers/usb/gadget/function/u_serial.c
@@ -375,10 +375,15 @@ __acquires(&port->port_lock)
 */
 {
 	struct list_head	*pool = &port->write_pool;
-	struct usb_ep		*in = port->port_usb->in;
+	struct usb_ep		*in;
 	int			status = 0;
 	bool			do_tty_wake = false;
 
+	if (!port->port_usb)
+		return status;
+
+	in = port->port_usb->in;
+
 	while (!port->write_busy && !list_empty(pool)) {
 		struct usb_request	*req;
 		int			len;
diff --git a/drivers/usb/gadget/function/uvc_configfs.c b/drivers/usb/gadget/function/uvc_configfs.c
index 66753ba7a42e..31125a4a2658 100644
--- a/drivers/usb/gadget/function/uvc_configfs.c
+++ b/drivers/usb/gadget/function/uvc_configfs.c
@@ -2023,7 +2023,7 @@ static int uvcg_streaming_class_allow_link(struct config_item *src,
 	if (!data) {
 		kfree(*class_array);
 		*class_array = NULL;
-		ret = PTR_ERR(data);
+		ret = -ENOMEM;
 		goto unlock;
 	}
 	cl_arr = *class_array;
diff --git a/drivers/usb/gadget/legacy/gmidi.c b/drivers/usb/gadget/legacy/gmidi.c
index fc2ac150f5ff..0bf39c3ccdb1 100644
--- a/drivers/usb/gadget/legacy/gmidi.c
+++ b/drivers/usb/gadget/legacy/gmidi.c
@@ -47,7 +47,7 @@ static char *id = SNDRV_DEFAULT_STR1;
 module_param(id, charp, S_IRUGO);
 MODULE_PARM_DESC(id, "ID string for the USB MIDI Gadget adapter.");
 
-static unsigned int buflen = 256;
+static unsigned int buflen = 512;
 module_param(buflen, uint, S_IRUGO);
 MODULE_PARM_DESC(buflen, "MIDI buffer length");
 
diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
index aa3707bdebb4..bd82dd12deff 100644
--- a/drivers/usb/gadget/legacy/inode.c
+++ b/drivers/usb/gadget/legacy/inode.c
@@ -542,7 +542,7 @@ static ssize_t ep_aio(struct kiocb *iocb,
 	 */
 	spin_lock_irq(&epdata->dev->lock);
 	value = -ENODEV;
-	if (unlikely(epdata->ep))
+	if (unlikely(epdata->ep == NULL))
 		goto fail;
 
 	req = usb_ep_alloc_request(epdata->ep, GFP_ATOMIC);
@@ -606,7 +606,7 @@ ep_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	}
 	if (is_sync_kiocb(iocb)) {
 		value = ep_io(epdata, buf, len);
-		if (value >= 0 && copy_to_iter(buf, value, to))
+		if (value >= 0 && (copy_to_iter(buf, value, to) != value))
 			value = -EFAULT;
 	} else {
 		struct kiocb_priv *priv = kzalloc(sizeof *priv, GFP_KERNEL);
@@ -1913,7 +1913,7 @@ gadgetfs_make_inode (struct super_block *sb,
 		inode->i_uid = make_kuid(&init_user_ns, default_uid);
 		inode->i_gid = make_kgid(&init_user_ns, default_gid);
 		inode->i_atime = inode->i_mtime = inode->i_ctime
-				= CURRENT_TIME;
+				= current_time(inode);
 		inode->i_private = data;
 		inode->i_fop = fops;
 	}
diff --git a/drivers/usb/gadget/u_f.c b/drivers/usb/gadget/u_f.c
index 4bc7eea8bfc8..18839732c840 100644
--- a/drivers/usb/gadget/u_f.c
+++ b/drivers/usb/gadget/u_f.c
@@ -12,14 +12,16 @@
  */
 
 #include "u_f.h"
+#include <linux/usb/ch9.h>
 
-struct usb_request *alloc_ep_req(struct usb_ep *ep, int len, int default_len)
+struct usb_request *alloc_ep_req(struct usb_ep *ep, size_t len)
 {
 	struct usb_request      *req;
 
 	req = usb_ep_alloc_request(ep, GFP_ATOMIC);
 	if (req) {
-		req->length = len ?: default_len;
+		req->length = usb_endpoint_dir_out(ep->desc) ?
+			usb_ep_align(ep, len) : len;
 		req->buf = kmalloc(req->length, GFP_ATOMIC);
 		if (!req->buf) {
 			usb_ep_free_request(ep, req);
diff --git a/drivers/usb/gadget/u_f.h b/drivers/usb/gadget/u_f.h
index 4247cc098a89..7d53a4773d1a 100644
--- a/drivers/usb/gadget/u_f.h
+++ b/drivers/usb/gadget/u_f.h
@@ -47,8 +47,21 @@
 struct usb_ep;
 struct usb_request;
 
-/* Requests allocated via alloc_ep_req() must be freed by free_ep_req(). */
-struct usb_request *alloc_ep_req(struct usb_ep *ep, int len, int default_len);
+/**
+ * alloc_ep_req - returns a usb_request allocated by the gadget driver and
+ * allocates the request's buffer.
+ *
+ * @ep: the endpoint to allocate a usb_request
+ * @len: usb_requests's buffer suggested size
+ *
+ * In case @ep direction is OUT, the @len will be aligned to ep's
+ * wMaxPacketSize. In order to avoid memory leaks or drops, *always* use
+ * usb_requests's length (req->length) to refer to the allocated buffer size.
+ * Requests allocated via alloc_ep_req() *must* be freed by free_ep_req().
+ */
+struct usb_request *alloc_ep_req(struct usb_ep *ep, size_t len);
+
+/* Frees a usb_request previously allocated by alloc_ep_req() */
 static inline void free_ep_req(struct usb_ep *ep, struct usb_request *req)
 {
 	kfree(req->buf);
diff --git a/drivers/usb/gadget/udc/bcm63xx_udc.c b/drivers/usb/gadget/udc/bcm63xx_udc.c
index f5fccb3e4152..f78503203f42 100644
--- a/drivers/usb/gadget/udc/bcm63xx_udc.c
+++ b/drivers/usb/gadget/udc/bcm63xx_udc.c
@@ -21,7 +21,6 @@
 #include <linux/errno.h>
 #include <linux/interrupt.h>
 #include <linux/ioport.h>
-#include <linux/kconfig.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/module.h>
diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c
index ff8685ea7219..9483489080f6 100644
--- a/drivers/usb/gadget/udc/core.c
+++ b/drivers/usb/gadget/udc/core.c
@@ -107,10 +107,8 @@ int usb_ep_enable(struct usb_ep *ep)
 		goto out;
 
 	ret = ep->ops->enable(ep, ep->desc);
-	if (ret) {
-		ret = ret;
+	if (ret)
 		goto out;
-	}
 
 	ep->enabled = true;
 
@@ -827,7 +825,7 @@ void usb_gadget_unmap_request_by_dev(struct device *dev,
 		return;
 
 	if (req->num_mapped_sgs) {
-		dma_unmap_sg(dev, req->sg, req->num_mapped_sgs,
+		dma_unmap_sg(dev, req->sg, req->num_sgs,
 				is_in ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 
 		req->num_mapped_sgs = 0;
@@ -1145,7 +1143,7 @@ int usb_add_gadget_udc_release(struct device *parent, struct usb_gadget *gadget,
 			if (ret != -EPROBE_DEFER)
 				list_del(&driver->pending);
 			if (ret)
-				goto err4;
+				goto err5;
 			break;
 		}
 	}
@@ -1154,6 +1152,9 @@ int usb_add_gadget_udc_release(struct device *parent, struct usb_gadget *gadget,
 
 	return 0;
 
+err5:
+	device_del(&udc->dev);
+
 err4:
 	list_del(&udc->list);
 	mutex_unlock(&udc_lock);
diff --git a/drivers/usb/gadget/udc/fsl_qe_udc.c b/drivers/usb/gadget/udc/fsl_qe_udc.c
index 93d28cb00b76..4fff51b8a18e 100644
--- a/drivers/usb/gadget/udc/fsl_qe_udc.c
+++ b/drivers/usb/gadget/udc/fsl_qe_udc.c
@@ -421,10 +421,8 @@ static int qe_ep_rxbd_update(struct qe_ep *ep)
 	bd = ep->rxbase;
 
 	ep->rxframe = kmalloc(sizeof(*ep->rxframe), GFP_ATOMIC);
-	if (ep->rxframe == NULL) {
-		dev_err(ep->udc->dev, "malloc rxframe failed\n");
+	if (!ep->rxframe)
 		return -ENOMEM;
-	}
 
 	qe_frame_init(ep->rxframe);
 
@@ -435,9 +433,7 @@ static int qe_ep_rxbd_update(struct qe_ep *ep)
 
 	size = (ep->ep.maxpacket + USB_CRC_SIZE + 2) * (bdring_len + 1);
 	ep->rxbuffer = kzalloc(size, GFP_ATOMIC);
-	if (ep->rxbuffer == NULL) {
-		dev_err(ep->udc->dev, "malloc rxbuffer failed,size=%d\n",
-				size);
+	if (!ep->rxbuffer) {
 		kfree(ep->rxframe);
 		return -ENOMEM;
 	}
@@ -668,10 +664,8 @@ static int qe_ep_init(struct qe_udc *udc,
 
 	if ((ep->tm == USBP_TM_CTL) || (ep->dir == USB_DIR_IN)) {
 		ep->txframe = kmalloc(sizeof(*ep->txframe), GFP_ATOMIC);
-		if (ep->txframe == NULL) {
-			dev_err(udc->dev, "malloc txframe failed\n");
+		if (!ep->txframe)
 			goto en_done2;
-		}
 		qe_frame_init(ep->txframe);
 	}
 
@@ -1878,11 +1872,8 @@ static int qe_get_frame(struct usb_gadget *gadget)
 
 	tmp = in_be16(&udc->usb_param->frame_n);
 	if (tmp & 0x8000)
-		tmp = tmp & 0x07ff;
-	else
-		tmp = -EINVAL;
-
-	return (int)tmp;
+		return tmp & 0x07ff;
+	return -EINVAL;
 }
 
 static int fsl_qe_start(struct usb_gadget *gadget,
@@ -2053,7 +2044,7 @@ static void setup_received_handle(struct qe_udc *udc,
 			struct qe_ep *ep;
 
 			if (wValue != 0 || wLength != 0
-				|| pipe > USB_MAX_ENDPOINTS)
+				|| pipe >= USB_MAX_ENDPOINTS)
 				break;
 			ep = &udc->eps[pipe];
 
@@ -2347,10 +2338,8 @@ static struct qe_udc *qe_udc_config(struct platform_device *ofdev)
 	u32 offset;
 
 	udc = kzalloc(sizeof(*udc), GFP_KERNEL);
-	if (udc == NULL) {
-		dev_err(&ofdev->dev, "malloc udc failed\n");
+	if (!udc)
 		goto cleanup;
-	}
 
 	udc->dev = &ofdev->dev;
 
diff --git a/drivers/usb/gadget/udc/goku_udc.c b/drivers/usb/gadget/udc/goku_udc.c
index d2205d9e0c8b..5107987bd353 100644
--- a/drivers/usb/gadget/udc/goku_udc.c
+++ b/drivers/usb/gadget/udc/goku_udc.c
@@ -1767,8 +1767,7 @@ static int goku_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	/* alloc, and start init */
 	dev = kzalloc (sizeof *dev, GFP_KERNEL);
-	if (dev == NULL){
-		pr_debug("enomem %s\n", pci_name(pdev));
+	if (!dev) {
 		retval = -ENOMEM;
 		goto err;
 	}
@@ -1839,6 +1838,8 @@ static int goku_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 err:
 	if (dev)
 		goku_remove (pdev);
+	/* gadget_release is not registered yet, kfree explicitly */
+	kfree(dev);
 	return retval;
 }
 
diff --git a/drivers/usb/gadget/udc/net2280.c b/drivers/usb/gadget/udc/net2280.c
index 614ab951a4ae..61c938c36d88 100644
--- a/drivers/usb/gadget/udc/net2280.c
+++ b/drivers/usb/gadget/udc/net2280.c
@@ -589,7 +589,7 @@ static void net2280_free_request(struct usb_ep *_ep, struct usb_request *_req)
 
 	ep = container_of(_ep, struct net2280_ep, ep);
 	if (!_ep || !_req) {
-		dev_err(&ep->dev->pdev->dev, "%s: Inavlid ep=%p or req=%p\n",
+		dev_err(&ep->dev->pdev->dev, "%s: Invalid ep=%p or req=%p\n",
 							__func__, _ep, _req);
 		return;
 	}
@@ -1137,8 +1137,10 @@ dma_done(struct net2280_ep *ep,	struct net2280_request *req, u32 dmacount,
 	done(ep, req, status);
 }
 
-static void scan_dma_completions(struct net2280_ep *ep)
+static int scan_dma_completions(struct net2280_ep *ep)
 {
+	int num_completed = 0;
+
 	/* only look at descriptors that were "naturally" retired,
 	 * so fifo and list head state won't matter
 	 */
@@ -1166,6 +1168,7 @@ static void scan_dma_completions(struct net2280_ep *ep)
 				break;
 			/* single transfer mode */
 			dma_done(ep, req, tmp, 0);
+			num_completed++;
 			break;
 		} else if (!ep->is_in &&
 			   (req->req.length % ep->ep.maxpacket) &&
@@ -1194,7 +1197,10 @@ static void scan_dma_completions(struct net2280_ep *ep)
 			}
 		}
 		dma_done(ep, req, tmp, 0);
+		num_completed++;
 	}
+
+	return num_completed;
 }
 
 static void restart_dma(struct net2280_ep *ep)
@@ -1567,6 +1573,44 @@ static struct usb_ep *net2280_match_ep(struct usb_gadget *_gadget,
 			return ep;
 	}
 
+	/* USB3380: Only first four endpoints have DMA channels. Allocate
+	 * slower interrupt endpoints from PIO hw endpoints, to allow bulk/isoc
+	 * endpoints use DMA hw endpoints.
+	 */
+	if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
+	    usb_endpoint_dir_in(desc)) {
+		ep = gadget_find_ep_by_name(_gadget, "ep2in");
+		if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
+			return ep;
+		ep = gadget_find_ep_by_name(_gadget, "ep4in");
+		if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
+			return ep;
+	} else if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
+		   !usb_endpoint_dir_in(desc)) {
+		ep = gadget_find_ep_by_name(_gadget, "ep1out");
+		if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
+			return ep;
+		ep = gadget_find_ep_by_name(_gadget, "ep3out");
+		if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
+			return ep;
+	} else if (usb_endpoint_type(desc) != USB_ENDPOINT_XFER_BULK &&
+		   usb_endpoint_dir_in(desc)) {
+		ep = gadget_find_ep_by_name(_gadget, "ep1in");
+		if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
+			return ep;
+		ep = gadget_find_ep_by_name(_gadget, "ep3in");
+		if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
+			return ep;
+	} else if (usb_endpoint_type(desc) != USB_ENDPOINT_XFER_BULK &&
+		   !usb_endpoint_dir_in(desc)) {
+		ep = gadget_find_ep_by_name(_gadget, "ep2out");
+		if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
+			return ep;
+		ep = gadget_find_ep_by_name(_gadget, "ep4out");
+		if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
+			return ep;
+	}
+
 	/* USB3380: use same address for usb and hardware endpoints */
 	snprintf(name, sizeof(name), "ep%d%s", usb_endpoint_num(desc),
 			usb_endpoint_dir_in(desc) ? "in" : "out");
@@ -2547,8 +2591,11 @@ static void handle_ep_small(struct net2280_ep *ep)
 	/* manual DMA queue advance after short OUT */
 	if (likely(ep->dma)) {
 		if (t & BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT)) {
-			u32	count;
+			struct net2280_request *stuck_req = NULL;
 			int	stopped = ep->stopped;
+			int	num_completed;
+			int	stuck = 0;
+			u32	count;
 
 			/* TRANSFERRED works around OUT_DONE erratum 0112.
 			 * we expect (N <= maxpacket) bytes; host wrote M.
@@ -2560,7 +2607,7 @@ static void handle_ep_small(struct net2280_ep *ep)
 				/* any preceding dma transfers must finish.
 				 * dma handles (M >= N), may empty the queue
 				 */
-				scan_dma_completions(ep);
+				num_completed = scan_dma_completions(ep);
 				if (unlikely(list_empty(&ep->queue) ||
 						ep->out_overflow)) {
 					req = NULL;
@@ -2580,6 +2627,31 @@ static void handle_ep_small(struct net2280_ep *ep)
 						req = NULL;
 					break;
 				}
+
+				/* Escape loop if no dma transfers completed
+				 * after few retries.
+				 */
+				if (num_completed == 0) {
+					if (stuck_req == req &&
+					    readl(&ep->dma->dmadesc) !=
+						  req->td_dma && stuck++ > 5) {
+						count = readl(
+							&ep->dma->dmacount);
+						count &= DMA_BYTE_COUNT_MASK;
+						req = NULL;
+						ep_dbg(ep->dev, "%s escape stuck %d, count %u\n",
+							ep->ep.name, stuck,
+							count);
+						break;
+					} else if (stuck_req != req) {
+						stuck_req = req;
+						stuck = 0;
+					}
+				} else {
+					stuck_req = NULL;
+					stuck = 0;
+				}
+
 				udelay(1);
 			}
 
diff --git a/drivers/usb/gadget/udc/omap_udc.c b/drivers/usb/gadget/udc/omap_udc.c
index 9b7d39484ed3..a8709f9e5648 100644
--- a/drivers/usb/gadget/udc/omap_udc.c
+++ b/drivers/usb/gadget/udc/omap_udc.c
@@ -2875,7 +2875,7 @@ bad_on_1710:
 	xceiv = NULL;
 	/* "udc" is now valid */
 	pullup_disable(udc);
-#if	defined(CONFIG_USB_OHCI_HCD) || defined(CONFIG_USB_OHCI_HCD_MODULE)
+#if	IS_ENABLED(CONFIG_USB_OHCI_HCD)
 	udc->gadget.is_otg = (config->otg != 0);
 #endif
 
diff --git a/drivers/usb/gadget/udc/pxa27x_udc.c b/drivers/usb/gadget/udc/pxa27x_udc.c
index ad140aa00132..7fa60f5b7ae4 100644
--- a/drivers/usb/gadget/udc/pxa27x_udc.c
+++ b/drivers/usb/gadget/udc/pxa27x_udc.c
@@ -33,6 +33,7 @@
 #include <linux/usb.h>
 #include <linux/usb/ch9.h>
 #include <linux/usb/gadget.h>
+#include <linux/usb/phy.h>
 
 #include "pxa27x_udc.h"
 
@@ -1655,6 +1656,37 @@ static int pxa_udc_vbus_draw(struct usb_gadget *_gadget, unsigned mA)
 	return -EOPNOTSUPP;
 }
 
+/**
+ * pxa_udc_phy_event - Called by phy upon VBus event
+ * @nb: notifier block
+ * @action: phy action, is vbus connect or disconnect
+ * @data: the usb_gadget structure in pxa_udc
+ *
+ * Called by the USB Phy when a cable connect or disconnect is sensed.
+ *
+ * Returns 0
+ */
+static int pxa_udc_phy_event(struct notifier_block *nb, unsigned long action,
+			     void *data)
+{
+	struct usb_gadget *gadget = data;
+
+	switch (action) {
+	case USB_EVENT_VBUS:
+		usb_gadget_vbus_connect(gadget);
+		return NOTIFY_OK;
+	case USB_EVENT_NONE:
+		usb_gadget_vbus_disconnect(gadget);
+		return NOTIFY_OK;
+	default:
+		return NOTIFY_DONE;
+	}
+}
+
+static struct notifier_block pxa27x_udc_phy = {
+	.notifier_call = pxa_udc_phy_event,
+};
+
 static int pxa27x_udc_start(struct usb_gadget *g,
 		struct usb_gadget_driver *driver);
 static int pxa27x_udc_stop(struct usb_gadget *g);
@@ -2432,7 +2464,14 @@ static int pxa_udc_probe(struct platform_device *pdev)
 		return udc->irq;
 
 	udc->dev = &pdev->dev;
-	udc->transceiver = usb_get_phy(USB_PHY_TYPE_USB2);
+	if (of_have_populated_dt()) {
+		udc->transceiver =
+			devm_usb_get_phy_by_phandle(udc->dev, "phys", 0);
+		if (IS_ERR(udc->transceiver))
+			return PTR_ERR(udc->transceiver);
+	} else {
+		udc->transceiver = usb_get_phy(USB_PHY_TYPE_USB2);
+	}
 
 	if (IS_ERR(udc->gpiod)) {
 		dev_err(&pdev->dev, "Couldn't find or request D+ gpio : %ld\n",
@@ -2465,14 +2504,20 @@ static int pxa_udc_probe(struct platform_device *pdev)
 		goto err;
 	}
 
+	if (!IS_ERR_OR_NULL(udc->transceiver))
+		usb_register_notifier(udc->transceiver, &pxa27x_udc_phy);
 	retval = usb_add_gadget_udc(&pdev->dev, &udc->gadget);
 	if (retval)
-		goto err;
+		goto err_add_gadget;
 
 	pxa_init_debugfs(udc);
 	if (should_enable_udc(udc))
 		udc_enable(udc);
 	return 0;
+
+err_add_gadget:
+	if (!IS_ERR_OR_NULL(udc->transceiver))
+		usb_unregister_notifier(udc->transceiver, &pxa27x_udc_phy);
 err:
 	clk_unprepare(udc->clk);
 	return retval;
@@ -2489,6 +2534,8 @@ static int pxa_udc_remove(struct platform_device *_dev)
 	usb_del_gadget_udc(&udc->gadget);
 	pxa_cleanup_debugfs(udc);
 
+	if (!IS_ERR_OR_NULL(udc->transceiver))
+		usb_unregister_notifier(udc->transceiver, &pxa27x_udc_phy);
 	usb_put_phy(udc->transceiver);
 
 	udc->transceiver = NULL;
diff --git a/drivers/usb/gadget/udc/renesas_usb3.c b/drivers/usb/gadget/udc/renesas_usb3.c
index 93a3bec81df7..fb8fc34827ab 100644
--- a/drivers/usb/gadget/udc/renesas_usb3.c
+++ b/drivers/usb/gadget/udc/renesas_usb3.c
@@ -106,6 +106,7 @@
 
 /* DRD_CON */
 #define DRD_CON_PERI_CON	BIT(24)
+#define DRD_CON_VBOUT		BIT(0)
 
 /* USB_INT_ENA_1 and USB_INT_STA_1 */
 #define USB_INT_1_B3_PLLWKUP	BIT(31)
@@ -363,6 +364,7 @@ static void usb3_init_epc_registers(struct renesas_usb3 *usb3)
 {
 	/* FIXME: How to change host / peripheral mode as well? */
 	usb3_set_bit(usb3, DRD_CON_PERI_CON, USB3_DRD_CON);
+	usb3_clear_bit(usb3, DRD_CON_VBOUT, USB3_DRD_CON);
 
 	usb3_write(usb3, ~0, USB3_USB_INT_STA_1);
 	usb3_enable_irq_1(usb3, USB_INT_1_VBUS_CNG);
diff --git a/drivers/usb/gadget/udc/udc-xilinx.c b/drivers/usb/gadget/udc/udc-xilinx.c
index f8bf290f1894..588e2531b8b8 100644
--- a/drivers/usb/gadget/udc/udc-xilinx.c
+++ b/drivers/usb/gadget/udc/udc-xilinx.c
@@ -973,10 +973,8 @@ static struct usb_request *xudc_ep_alloc_request(struct usb_ep *_ep,
 
 	udc = ep->udc;
 	req = kzalloc(sizeof(*req), gfp_flags);
-	if (!req) {
-		dev_err(udc->dev, "%s:not enough memory", __func__);
+	if (!req)
 		return NULL;
-	}
 
 	req->ep = ep;
 	INIT_LIST_HEAD(&req->queue);
diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig
index 2e710a4cca52..0b80cee30da4 100644
--- a/drivers/usb/host/Kconfig
+++ b/drivers/usb/host/Kconfig
@@ -472,7 +472,7 @@ config USB_OHCI_HCD_AT91
 
 config USB_OHCI_HCD_OMAP3
 	tristate "OHCI support for OMAP3 and later chips"
-	depends on (ARCH_OMAP3 || ARCH_OMAP4)
+	depends on (ARCH_OMAP3 || ARCH_OMAP4 || SOC_OMAP5)
 	default y
 	---help---
 	  Enables support for the on-chip OHCI controller on
diff --git a/drivers/usb/host/bcma-hcd.c b/drivers/usb/host/bcma-hcd.c
index 172ef17911aa..5f425c89faf1 100644
--- a/drivers/usb/host/bcma-hcd.c
+++ b/drivers/usb/host/bcma-hcd.c
@@ -27,6 +27,7 @@
 #include <linux/slab.h>
 #include <linux/of.h>
 #include <linux/of_gpio.h>
+#include <linux/of_platform.h>
 #include <linux/usb/ehci_pdriver.h>
 #include <linux/usb/ohci_pdriver.h>
 
@@ -34,6 +35,9 @@ MODULE_AUTHOR("Hauke Mehrtens");
 MODULE_DESCRIPTION("Common USB driver for BCMA Bus");
 MODULE_LICENSE("GPL");
 
+/* See BCMA_CLKCTLST_EXTRESREQ and BCMA_CLKCTLST_EXTRESST */
+#define USB_BCMA_CLKCTLST_USB_CLK_REQ			0x00000100
+
 struct bcma_hcd_device {
 	struct bcma_device *core;
 	struct platform_device *ehci_dev;
@@ -165,44 +169,80 @@ static void bcma_hcd_init_chip_mips(struct bcma_device *dev)
 	}
 }
 
-static void bcma_hcd_init_chip_arm_phy(struct bcma_device *dev)
+/**
+ * bcma_hcd_usb20_old_arm_init - Initialize old USB 2.0 controller on ARM
+ *
+ * Old USB 2.0 core is identified as BCMA_CORE_USB20_HOST and was introduced
+ * long before Northstar devices. It seems some cheaper chipsets like BCM53573
+ * still use it.
+ * Initialization of this old core differs between MIPS and ARM.
+ */
+static int bcma_hcd_usb20_old_arm_init(struct bcma_hcd_device *usb_dev)
 {
-	struct bcma_device *arm_core;
-	void __iomem *dmu;
-
-	arm_core = bcma_find_core(dev->bus, BCMA_CORE_ARMCA9);
-	if (!arm_core) {
-		dev_err(&dev->dev, "can not find ARM Cortex A9 ihost core\n");
-		return;
+	struct bcma_device *core = usb_dev->core;
+	struct device *dev = &core->dev;
+	struct bcma_device *pmu_core;
+
+	usleep_range(10000, 20000);
+	if (core->id.rev < 5)
+		return 0;
+
+	pmu_core = bcma_find_core(core->bus, BCMA_CORE_PMU);
+	if (!pmu_core) {
+		dev_err(dev, "Could not find PMU core\n");
+		return -ENOENT;
 	}
 
-	dmu = ioremap_nocache(arm_core->addr_s[0], 0x1000);
-	if (!dmu) {
-		dev_err(&dev->dev, "can not map ARM Cortex A9 ihost core\n");
-		return;
-	}
-
-	/* Unlock DMU PLL settings */
-	iowrite32(0x0000ea68, dmu + 0x180);
-
-	/* Write USB 2.0 PLL control setting */
-	iowrite32(0x00dd10c3, dmu + 0x164);
+	/* Take USB core out of reset */
+	bcma_awrite32(core, BCMA_IOCTL, BCMA_IOCTL_CLK | BCMA_IOCTL_FGC);
+	usleep_range(100, 200);
+	bcma_awrite32(core, BCMA_RESET_CTL, BCMA_RESET_CTL_RESET);
+	usleep_range(100, 200);
+	bcma_awrite32(core, BCMA_RESET_CTL, 0);
+	usleep_range(100, 200);
+	bcma_awrite32(core, BCMA_IOCTL, BCMA_IOCTL_CLK);
+	usleep_range(100, 200);
+
+	/* Enable Misc PLL */
+	bcma_write32(core, BCMA_CLKCTLST, BCMA_CLKCTLST_FORCEHT |
+					  BCMA_CLKCTLST_HQCLKREQ |
+					  USB_BCMA_CLKCTLST_USB_CLK_REQ);
+	usleep_range(100, 200);
+
+	bcma_write32(core, 0x510, 0xc7f85000);
+	bcma_write32(core, 0x510, 0xc7f85003);
+	usleep_range(300, 600);
+
+	/* Program USB PHY PLL parameters */
+	bcma_write32(pmu_core, BCMA_CC_PMU_PLLCTL_ADDR, 0x6);
+	bcma_write32(pmu_core, BCMA_CC_PMU_PLLCTL_DATA, 0x005360c1);
+	usleep_range(100, 200);
+	bcma_write32(pmu_core, BCMA_CC_PMU_PLLCTL_ADDR, 0x7);
+	bcma_write32(pmu_core, BCMA_CC_PMU_PLLCTL_DATA, 0x0);
+	usleep_range(100, 200);
+	bcma_set32(pmu_core, BCMA_CC_PMU_CTL, BCMA_CC_PMU_CTL_PLL_UPD);
+	usleep_range(100, 200);
+
+	bcma_write32(core, 0x510, 0x7f8d007);
+	udelay(1000);
+
+	/* Take controller out of reset */
+	bcma_write32(core, 0x200, 0x4ff);
+	usleep_range(25, 50);
+	bcma_write32(core, 0x200, 0x6ff);
+	usleep_range(25, 50);
+	bcma_write32(core, 0x200, 0x7ff);
+	usleep_range(25, 50);
+
+	of_platform_default_populate(dev->of_node, NULL, dev);
 
-	/* Lock DMU PLL settings */
-	iowrite32(0x00000000, dmu + 0x180);
-
-	iounmap(dmu);
+	return 0;
 }
 
-static void bcma_hcd_init_chip_arm_hc(struct bcma_device *dev)
+static void bcma_hcd_usb20_ns_init_hc(struct bcma_device *dev)
 {
 	u32 val;
 
-	/*
-	 * Delay after PHY initialized to ensure HC is ready to be configured
-	 */
-	usleep_range(1000, 2000);
-
 	/* Set packet buffer OUT threshold */
 	val = bcma_read32(dev, 0x94);
 	val &= 0xffff;
@@ -213,20 +253,33 @@ static void bcma_hcd_init_chip_arm_hc(struct bcma_device *dev)
 	val = bcma_read32(dev, 0x9c);
 	val |= 1;
 	bcma_write32(dev, 0x9c, val);
+
+	/*
+	 * Broadcom initializes PHY and then waits to ensure HC is ready to be
+	 * configured. In our case the order is reversed. We just initialized
+	 * controller and we let HCD initialize PHY, so let's wait (sleep) now.
+	 */
+	usleep_range(1000, 2000);
 }
 
-static void bcma_hcd_init_chip_arm(struct bcma_device *dev)
+/**
+ * bcma_hcd_usb20_ns_init - Initialize Northstar USB 2.0 controller
+ */
+static int bcma_hcd_usb20_ns_init(struct bcma_hcd_device *bcma_hcd)
 {
-	bcma_core_enable(dev, 0);
+	struct bcma_device *core = bcma_hcd->core;
+	struct bcma_chipinfo *ci = &core->bus->chipinfo;
+	struct device *dev = &core->dev;
 
-	if (dev->bus->chipinfo.id == BCMA_CHIP_ID_BCM4707 ||
-	    dev->bus->chipinfo.id == BCMA_CHIP_ID_BCM53018) {
-		if (dev->bus->chipinfo.pkg == BCMA_PKG_ID_BCM4707 ||
-		    dev->bus->chipinfo.pkg == BCMA_PKG_ID_BCM4708)
-			bcma_hcd_init_chip_arm_phy(dev);
+	bcma_core_enable(core, 0);
 
-		bcma_hcd_init_chip_arm_hc(dev);
-	}
+	if (ci->id == BCMA_CHIP_ID_BCM4707 ||
+	    ci->id == BCMA_CHIP_ID_BCM53018)
+		bcma_hcd_usb20_ns_init_hc(core);
+
+	of_platform_default_populate(dev->of_node, NULL, dev);
+
+	return 0;
 }
 
 static void bcma_hci_platform_power_gpio(struct bcma_device *dev, bool val)
@@ -299,16 +352,7 @@ static int bcma_hcd_usb20_init(struct bcma_hcd_device *usb_dev)
 	if (dma_set_mask_and_coherent(dev->dma_dev, DMA_BIT_MASK(32)))
 		return -EOPNOTSUPP;
 
-	switch (dev->id.id) {
-	case BCMA_CORE_NS_USB20:
-		bcma_hcd_init_chip_arm(dev);
-		break;
-	case BCMA_CORE_USB20_HOST:
-		bcma_hcd_init_chip_mips(dev);
-		break;
-	default:
-		return -ENODEV;
-	}
+	bcma_hcd_init_chip_mips(dev);
 
 	/* In AI chips EHCI is addrspace 0, OHCI is 1 */
 	ohci_addr = dev->addr_s[0];
@@ -338,6 +382,18 @@ err_unregister_ohci_dev:
 	return err;
 }
 
+static int bcma_hcd_usb30_init(struct bcma_hcd_device *bcma_hcd)
+{
+	struct bcma_device *core = bcma_hcd->core;
+	struct device *dev = &core->dev;
+
+	bcma_core_enable(core, 0);
+
+	of_platform_default_populate(dev->of_node, NULL, dev);
+
+	return 0;
+}
+
 static int bcma_hcd_probe(struct bcma_device *core)
 {
 	int err;
@@ -357,14 +413,24 @@ static int bcma_hcd_probe(struct bcma_device *core)
 
 	switch (core->id.id) {
 	case BCMA_CORE_USB20_HOST:
+		if (IS_ENABLED(CONFIG_ARM))
+			err = bcma_hcd_usb20_old_arm_init(usb_dev);
+		else if (IS_ENABLED(CONFIG_MIPS))
+			err = bcma_hcd_usb20_init(usb_dev);
+		else
+			err = -ENOTSUPP;
+		break;
 	case BCMA_CORE_NS_USB20:
-		err = bcma_hcd_usb20_init(usb_dev);
-		if (err)
-			return err;
+		err = bcma_hcd_usb20_ns_init(usb_dev);
+		break;
+	case BCMA_CORE_NS_USB30:
+		err = bcma_hcd_usb30_init(usb_dev);
 		break;
 	default:
 		return -ENODEV;
 	}
+	if (err)
+		return err;
 
 	bcma_set_drvdata(core, usb_dev);
 	return 0;
@@ -416,6 +482,7 @@ static int bcma_hcd_resume(struct bcma_device *dev)
 static const struct bcma_device_id bcma_hcd_table[] = {
 	BCMA_CORE(BCMA_MANUF_BCM, BCMA_CORE_USB20_HOST, BCMA_ANY_REV, BCMA_ANY_CLASS),
 	BCMA_CORE(BCMA_MANUF_BCM, BCMA_CORE_NS_USB20, BCMA_ANY_REV, BCMA_ANY_CLASS),
+	BCMA_CORE(BCMA_MANUF_BCM, BCMA_CORE_NS_USB30, BCMA_ANY_REV, BCMA_ANY_CLASS),
 	{},
 };
 MODULE_DEVICE_TABLE(bcma, bcma_hcd_table);
diff --git a/drivers/usb/host/ehci-hcd.c b/drivers/usb/host/ehci-hcd.c
index a962b89b65a6..1e5f529d51a2 100644
--- a/drivers/usb/host/ehci-hcd.c
+++ b/drivers/usb/host/ehci-hcd.c
@@ -332,11 +332,11 @@ static void ehci_turn_off_all_ports(struct ehci_hcd *ehci)
 	int	port = HCS_N_PORTS(ehci->hcs_params);
 
 	while (port--) {
-		ehci_writel(ehci, PORT_RWC_BITS,
-				&ehci->regs->port_status[port]);
 		spin_unlock_irq(&ehci->lock);
 		ehci_port_power(ehci, port, false);
 		spin_lock_irq(&ehci->lock);
+		ehci_writel(ehci, PORT_RWC_BITS,
+				&ehci->regs->port_status[port]);
 	}
 }
 
diff --git a/drivers/usb/host/ehci-platform.c b/drivers/usb/host/ehci-platform.c
index 6816b8c371d0..876dca4fc216 100644
--- a/drivers/usb/host/ehci-platform.c
+++ b/drivers/usb/host/ehci-platform.c
@@ -38,7 +38,7 @@
 #include "ehci.h"
 
 #define DRIVER_DESC "EHCI generic platform driver"
-#define EHCI_MAX_CLKS 3
+#define EHCI_MAX_CLKS 4
 #define EHCI_MAX_RSTS 3
 #define hcd_to_ehci_priv(h) ((struct ehci_platform_priv *)hcd_to_ehci(h)->priv)
 
diff --git a/drivers/usb/host/fhci-hcd.c b/drivers/usb/host/fhci-hcd.c
index 0960f41f945a..55a0ae6f2d74 100644
--- a/drivers/usb/host/fhci-hcd.c
+++ b/drivers/usb/host/fhci-hcd.c
@@ -310,10 +310,8 @@ static struct fhci_usb *fhci_create_lld(struct fhci_hcd *fhci)
 
 	/* allocate memory for SCC data structure */
 	usb = kzalloc(sizeof(*usb), GFP_KERNEL);
-	if (!usb) {
-		fhci_err(fhci, "no memory for SCC data struct\n");
+	if (!usb)
 		return NULL;
-	}
 
 	usb->fhci = fhci;
 	usb->hc_list = fhci->hc_list;
diff --git a/drivers/usb/host/fsl-mph-dr-of.c b/drivers/usb/host/fsl-mph-dr-of.c
index 1044b0f9d656..f07ccb25bc24 100644
--- a/drivers/usb/host/fsl-mph-dr-of.c
+++ b/drivers/usb/host/fsl-mph-dr-of.c
@@ -222,23 +222,17 @@ static int fsl_usb2_mph_dr_of_probe(struct platform_device *ofdev)
 	pdata->controller_ver = usb_get_ver_info(np);
 
 	/* Activate Erratum by reading property in device tree */
-	if (of_get_property(np, "fsl,usb-erratum-a007792", NULL))
-		pdata->has_fsl_erratum_a007792 = 1;
-	else
-		pdata->has_fsl_erratum_a007792 = 0;
-	if (of_get_property(np, "fsl,usb-erratum-a005275", NULL))
-		pdata->has_fsl_erratum_a005275 = 1;
-	else
-		pdata->has_fsl_erratum_a005275 = 0;
+	pdata->has_fsl_erratum_a007792 =
+		of_property_read_bool(np, "fsl,usb-erratum-a007792");
+	pdata->has_fsl_erratum_a005275 =
+		of_property_read_bool(np, "fsl,usb-erratum-a005275");
 
 	/*
 	 * Determine whether phy_clk_valid needs to be checked
 	 * by reading property in device tree
 	 */
-	if (of_get_property(np, "phy-clk-valid", NULL))
-		pdata->check_phy_clk_valid = 1;
-	else
-		pdata->check_phy_clk_valid = 0;
+	pdata->check_phy_clk_valid =
+		of_property_read_bool(np, "phy-clk-valid");
 
 	if (pdata->have_sysif_regs) {
 		if (pdata->controller_ver == FSL_USB_VER_NONE) {
diff --git a/drivers/usb/host/max3421-hcd.c b/drivers/usb/host/max3421-hcd.c
index c369c29e496d..369869a29ebd 100644
--- a/drivers/usb/host/max3421-hcd.c
+++ b/drivers/usb/host/max3421-hcd.c
@@ -1675,7 +1675,7 @@ max3421_gpout_set_value(struct usb_hcd *hcd, u8 pin_number, u8 value)
 	if (pin_number > 7)
 		return;
 
-	mask = 1u << pin_number;
+	mask = 1u << (pin_number % 4);
 	idx = pin_number / 4;
 
 	if (value)
@@ -1856,15 +1856,11 @@ max3421_probe(struct spi_device *spi)
 	INIT_LIST_HEAD(&max3421_hcd->ep_list);
 
 	max3421_hcd->tx = kmalloc(sizeof(*max3421_hcd->tx), GFP_KERNEL);
-	if (!max3421_hcd->tx) {
-		dev_err(&spi->dev, "failed to kmalloc tx buffer\n");
+	if (!max3421_hcd->tx)
 		goto error;
-	}
 	max3421_hcd->rx = kmalloc(sizeof(*max3421_hcd->rx), GFP_KERNEL);
-	if (!max3421_hcd->rx) {
-		dev_err(&spi->dev, "failed to kmalloc rx buffer\n");
+	if (!max3421_hcd->rx)
 		goto error;
-	}
 
 	max3421_hcd->spi_thread = kthread_run(max3421_spi_thread, hcd,
 					      "max3421_spi_thread");
diff --git a/drivers/usb/host/ohci-at91.c b/drivers/usb/host/ohci-at91.c
index d177372bb357..5b5880c0ae19 100644
--- a/drivers/usb/host/ohci-at91.c
+++ b/drivers/usb/host/ohci-at91.c
@@ -21,8 +21,11 @@
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
 #include <linux/usb.h>
 #include <linux/usb/hcd.h>
+#include <soc/at91/atmel-sfr.h>
 
 #include "ohci.h"
 
@@ -51,6 +54,7 @@ struct ohci_at91_priv {
 	struct clk *hclk;
 	bool clocked;
 	bool wakeup;		/* Saved wake-up state for resume */
+	struct regmap *sfr_regmap;
 };
 /* interface and function clocks; sometimes also an AHB clock */
 
@@ -134,6 +138,17 @@ static void at91_stop_hc(struct platform_device *pdev)
 
 static void usb_hcd_at91_remove (struct usb_hcd *, struct platform_device *);
 
+static struct regmap *at91_dt_syscon_sfr(void)
+{
+	struct regmap *regmap;
+
+	regmap = syscon_regmap_lookup_by_compatible("atmel,sama5d2-sfr");
+	if (IS_ERR(regmap))
+		regmap = NULL;
+
+	return regmap;
+}
+
 /* configure so an HC device and id are always provided */
 /* always called with process context; sleeping is OK */
 
@@ -197,6 +212,10 @@ static int usb_hcd_at91_probe(const struct hc_driver *driver,
 		goto err;
 	}
 
+	ohci_at91->sfr_regmap = at91_dt_syscon_sfr();
+	if (!ohci_at91->sfr_regmap)
+		dev_warn(dev, "failed to find sfr node\n");
+
 	board = hcd->self.controller->platform_data;
 	ohci = hcd_to_ohci(hcd);
 	ohci->num_ports = board->ports;
@@ -282,6 +301,28 @@ static int ohci_at91_hub_status_data(struct usb_hcd *hcd, char *buf)
 	return length;
 }
 
+static int ohci_at91_port_suspend(struct regmap *regmap, u8 set)
+{
+	u32 regval;
+	int ret;
+
+	if (!regmap)
+		return 0;
+
+	ret = regmap_read(regmap, AT91_SFR_OHCIICR, &regval);
+	if (ret)
+		return ret;
+
+	if (set)
+		regval |= AT91_OHCIICR_USB_SUSPEND;
+	else
+		regval &= ~AT91_OHCIICR_USB_SUSPEND;
+
+	regmap_write(regmap, AT91_SFR_OHCIICR, regval);
+
+	return 0;
+}
+
 /*
  * Look at the control requests to the root hub and see if we need to override.
  */
@@ -289,6 +330,7 @@ static int ohci_at91_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 				 u16 wIndex, char *buf, u16 wLength)
 {
 	struct at91_usbh_data *pdata = dev_get_platdata(hcd->self.controller);
+	struct ohci_at91_priv *ohci_at91 = hcd_to_ohci_at91_priv(hcd);
 	struct usb_hub_descriptor *desc;
 	int ret = -EINVAL;
 	u32 *data = (u32 *)buf;
@@ -301,7 +343,8 @@ static int ohci_at91_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 
 	switch (typeReq) {
 	case SetPortFeature:
-		if (wValue == USB_PORT_FEAT_POWER) {
+		switch (wValue) {
+		case USB_PORT_FEAT_POWER:
 			dev_dbg(hcd->self.controller, "SetPortFeat: POWER\n");
 			if (valid_port(wIndex)) {
 				ohci_at91_usb_set_power(pdata, wIndex, 1);
@@ -309,6 +352,15 @@ static int ohci_at91_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 			}
 
 			goto out;
+
+		case USB_PORT_FEAT_SUSPEND:
+			dev_dbg(hcd->self.controller, "SetPortFeat: SUSPEND\n");
+			if (valid_port(wIndex)) {
+				ohci_at91_port_suspend(ohci_at91->sfr_regmap,
+						       1);
+				return 0;
+			}
+			break;
 		}
 		break;
 
@@ -342,6 +394,16 @@ static int ohci_at91_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 				ohci_at91_usb_set_power(pdata, wIndex, 0);
 				return 0;
 			}
+			break;
+
+		case USB_PORT_FEAT_SUSPEND:
+			dev_dbg(hcd->self.controller, "ClearPortFeature: SUSPEND\n");
+			if (valid_port(wIndex)) {
+				ohci_at91_port_suspend(ohci_at91->sfr_regmap,
+						       0);
+				return 0;
+			}
+			break;
 		}
 		break;
 	}
@@ -599,6 +661,8 @@ ohci_hcd_at91_drv_suspend(struct device *dev)
 	if (ohci_at91->wakeup)
 		enable_irq_wake(hcd->irq);
 
+	ohci_at91_port_suspend(ohci_at91->sfr_regmap, 1);
+
 	ret = ohci_suspend(hcd, ohci_at91->wakeup);
 	if (ret) {
 		if (ohci_at91->wakeup)
@@ -638,6 +702,9 @@ ohci_hcd_at91_drv_resume(struct device *dev)
 	at91_start_clock(ohci_at91);
 
 	ohci_resume(hcd, false);
+
+	ohci_at91_port_suspend(ohci_at91->sfr_regmap, 0);
+
 	return 0;
 }
 
diff --git a/drivers/usb/host/ohci-omap.c b/drivers/usb/host/ohci-omap.c
index de7c68602a7e..495c1454b9e8 100644
--- a/drivers/usb/host/ohci-omap.c
+++ b/drivers/usb/host/ohci-omap.c
@@ -36,7 +36,6 @@
 #include <mach/mux.h>
 
 #include <mach/hardware.h>
-#include <mach/irqs.h>
 #include <mach/usb.h>
 
 
diff --git a/drivers/usb/host/ohci-sa1111.c b/drivers/usb/host/ohci-sa1111.c
index 2ac266d692a2..3a9ea32508df 100644
--- a/drivers/usb/host/ohci-sa1111.c
+++ b/drivers/usb/host/ohci-sa1111.c
@@ -13,9 +13,7 @@
  * This file is licenced under the GPL.
  */
 
-#include <mach/hardware.h>
 #include <asm/mach-types.h>
-#include <mach/assabet.h>
 #include <asm/hardware/sa1111.h>
 
 #ifndef CONFIG_SA1111
@@ -127,7 +125,7 @@ static int sa1111_start_hc(struct sa1111_dev *dev)
 	dev_dbg(&dev->dev, "starting SA-1111 OHCI USB Controller\n");
 
 	if (machine_is_xp860() ||
-	    machine_has_neponset() ||
+	    machine_is_assabet() ||
 	    machine_is_pfs168() ||
 	    machine_is_badge4())
 		usb_rst = USB_RESET_PWRSENSELOW | USB_RESET_PWRCTRLLOW;
diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c
index 35af36253440..d793f548dfe2 100644
--- a/drivers/usb/host/pci-quirks.c
+++ b/drivers/usb/host/pci-quirks.c
@@ -9,7 +9,6 @@
  */
 
 #include <linux/types.h>
-#include <linux/kconfig.h>
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/delay.h>
diff --git a/drivers/usb/host/uhci-hcd.c b/drivers/usb/host/uhci-hcd.c
index a7de8e8bb458..5d3d914ab4fb 100644
--- a/drivers/usb/host/uhci-hcd.c
+++ b/drivers/usb/host/uhci-hcd.c
@@ -601,11 +601,8 @@ static int uhci_start(struct usb_hcd *hcd)
 
 	uhci->frame_cpu = kcalloc(UHCI_NUMFRAMES, sizeof(*uhci->frame_cpu),
 			GFP_KERNEL);
-	if (!uhci->frame_cpu) {
-		dev_err(uhci_dev(uhci),
-			"unable to allocate memory for frame pointers\n");
+	if (!uhci->frame_cpu)
 		goto err_alloc_frame_cpu;
-	}
 
 	uhci->td_pool = dma_pool_create("uhci_td", uhci_dev(uhci),
 			sizeof(struct uhci_td), 16, 0);
diff --git a/drivers/usb/host/whci/init.c b/drivers/usb/host/whci/init.c
index e36372393bb1..ad8eb575c30a 100644
--- a/drivers/usb/host/whci/init.c
+++ b/drivers/usb/host/whci/init.c
@@ -65,7 +65,7 @@ int whc_init(struct whc *whc)
 	init_waitqueue_head(&whc->cmd_wq);
 	init_waitqueue_head(&whc->async_list_wq);
 	init_waitqueue_head(&whc->periodic_list_wq);
-	whc->workqueue = create_singlethread_workqueue(dev_name(&whc->umc->dev));
+	whc->workqueue = alloc_ordered_workqueue(dev_name(&whc->umc->dev), 0);
 	if (whc->workqueue == NULL) {
 		ret = -ENOMEM;
 		goto error;
diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c
index d61fcc48099e..730b9fd26685 100644
--- a/drivers/usb/host/xhci-hub.c
+++ b/drivers/usb/host/xhci-hub.c
@@ -386,6 +386,9 @@ static int xhci_stop_device(struct xhci_hcd *xhci, int slot_id, int suspend)
 
 	ret = 0;
 	virt_dev = xhci->devs[slot_id];
+	if (!virt_dev)
+		return -ENODEV;
+
 	cmd = xhci_alloc_command(xhci, false, true, GFP_NOIO);
 	if (!cmd) {
 		xhci_dbg(xhci, "Couldn't allocate command structure.\n");
diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c
index 4fd041bec332..d7b0f97abbad 100644
--- a/drivers/usb/host/xhci-pci.c
+++ b/drivers/usb/host/xhci-pci.c
@@ -314,11 +314,12 @@ static void xhci_pci_remove(struct pci_dev *dev)
 		usb_remove_hcd(xhci->shared_hcd);
 		usb_put_hcd(xhci->shared_hcd);
 	}
-	usb_hcd_pci_remove(dev);
 
 	/* Workaround for spurious wakeups at shutdown with HSW */
 	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
 		pci_set_power_state(dev, PCI_D3hot);
+
+	usb_hcd_pci_remove(dev);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index 918e0c739b79..797137e26549 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -850,6 +850,10 @@ void xhci_stop_endpoint_command_watchdog(unsigned long arg)
 	spin_lock_irqsave(&xhci->lock, flags);
 
 	ep->stop_cmds_pending--;
+	if (xhci->xhc_state & XHCI_STATE_REMOVING) {
+		spin_unlock_irqrestore(&xhci->lock, flags);
+		return;
+	}
 	if (xhci->xhc_state & XHCI_STATE_DYING) {
 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 				"Stop EP timer ran, but another timer marked "
@@ -903,7 +907,7 @@ void xhci_stop_endpoint_command_watchdog(unsigned long arg)
 	spin_unlock_irqrestore(&xhci->lock, flags);
 	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 			"Calling usb_hc_died()");
-	usb_hc_died(xhci_to_hcd(xhci)->primary_hcd);
+	usb_hc_died(xhci_to_hcd(xhci));
 	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
 			"xHCI host controller is dead.");
 }
@@ -1334,12 +1338,6 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
 
 	cmd = list_entry(xhci->cmd_list.next, struct xhci_command, cmd_list);
 
-	if (cmd->command_trb != xhci->cmd_ring->dequeue) {
-		xhci_err(xhci,
-			 "Command completion event does not match command\n");
-		return;
-	}
-
 	del_timer(&xhci->cmd_timer);
 
 	trace_xhci_cmd_completion(cmd_trb, (struct xhci_generic_trb *) event);
@@ -1351,6 +1349,13 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
 		xhci_handle_stopped_cmd_ring(xhci, cmd);
 		return;
 	}
+
+	if (cmd->command_trb != xhci->cmd_ring->dequeue) {
+		xhci_err(xhci,
+			 "Command completion event does not match command\n");
+		return;
+	}
+
 	/*
 	 * Host aborted the command ring, check if the current command was
 	 * supposed to be aborted, otherwise continue normally.
@@ -3243,7 +3248,8 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
 	send_addr = addr;
 
 	/* Queue the TRBs, even if they are zero-length */
-	for (enqd_len = 0; enqd_len < full_len; enqd_len += trb_buff_len) {
+	for (enqd_len = 0; first_trb || enqd_len < full_len;
+			enqd_len += trb_buff_len) {
 		field = TRB_TYPE(TRB_NORMAL);
 
 		/* TRB buffer should not cross 64KB boundaries */
diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c
index 0f53ae0f464e..a59fafb4b329 100644
--- a/drivers/usb/host/xhci-tegra.c
+++ b/drivers/usb/host/xhci-tegra.c
@@ -1033,7 +1033,6 @@ static int tegra_xusb_probe(struct platform_device *pdev)
 	tegra->phys = devm_kcalloc(&pdev->dev, tegra->num_phys,
 				   sizeof(*tegra->phys), GFP_KERNEL);
 	if (!tegra->phys) {
-		dev_err(&pdev->dev, "failed to allocate PHY array\n");
 		err = -ENOMEM;
 		goto put_padctl;
 	}
@@ -1117,6 +1116,7 @@ static int tegra_xusb_probe(struct platform_device *pdev)
 						 tegra->hcd);
 	if (!xhci->shared_hcd) {
 		dev_err(&pdev->dev, "failed to create shared HCD\n");
+		err = -ENOMEM;
 		goto remove_usb2;
 	}
 
diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
index 01d96c9b3a75..1a4ca02729c2 100644
--- a/drivers/usb/host/xhci.c
+++ b/drivers/usb/host/xhci.c
@@ -295,10 +295,8 @@ static int xhci_setup_msix(struct xhci_hcd *xhci)
 	xhci->msix_entries =
 		kmalloc((sizeof(struct msix_entry))*xhci->msix_count,
 				GFP_KERNEL);
-	if (!xhci->msix_entries) {
-		xhci_err(xhci, "Failed to allocate MSI-X entries\n");
+	if (!xhci->msix_entries)
 		return -ENOMEM;
-	}
 
 	for (i = 0; i < xhci->msix_count; i++) {
 		xhci->msix_entries[i].entry = i;
diff --git a/drivers/usb/misc/Kconfig b/drivers/usb/misc/Kconfig
index eb8f8d37cd95..47b357760afc 100644
--- a/drivers/usb/misc/Kconfig
+++ b/drivers/usb/misc/Kconfig
@@ -240,6 +240,12 @@ config USB_HSIC_USB3503
        help
          This option enables support for SMSC USB3503 HSIC to USB 2.0 Driver.
 
+config USB_HSIC_USB4604
+       tristate "USB4604 HSIC to USB20 Driver"
+       depends on I2C
+       help
+         This option enables support for SMSC USB4604 HSIC to USB 2.0 Driver.
+
 config USB_LINK_LAYER_TEST
 	tristate "USB Link Layer Test driver"
 	help
diff --git a/drivers/usb/misc/Makefile b/drivers/usb/misc/Makefile
index 3d79faaad2fb..3d1992750da4 100644
--- a/drivers/usb/misc/Makefile
+++ b/drivers/usb/misc/Makefile
@@ -24,6 +24,7 @@ obj-$(CONFIG_USB_USS720)		+= uss720.o
 obj-$(CONFIG_USB_SEVSEG)		+= usbsevseg.o
 obj-$(CONFIG_USB_YUREX)			+= yurex.o
 obj-$(CONFIG_USB_HSIC_USB3503)		+= usb3503.o
+obj-$(CONFIG_USB_HSIC_USB4604)		+= usb4604.o
 obj-$(CONFIG_USB_CHAOSKEY)		+= chaoskey.o
 obj-$(CONFIG_UCSI)			+= ucsi.o
 
diff --git a/drivers/usb/misc/adutux.c b/drivers/usb/misc/adutux.c
index 3071c0ef909b..564268fca07a 100644
--- a/drivers/usb/misc/adutux.c
+++ b/drivers/usb/misc/adutux.c
@@ -672,8 +672,7 @@ static int adu_probe(struct usb_interface *interface,
 
 	/* allocate memory for our device state and initialize it */
 	dev = kzalloc(sizeof(struct adu_device), GFP_KERNEL);
-	if (dev == NULL) {
-		dev_err(&interface->dev, "Out of memory\n");
+	if (!dev) {
 		retval = -ENOMEM;
 		goto exit;
 	}
@@ -710,7 +709,6 @@ static int adu_probe(struct usb_interface *interface,
 
 	dev->read_buffer_primary = kmalloc((4 * in_end_size), GFP_KERNEL);
 	if (!dev->read_buffer_primary) {
-		dev_err(&interface->dev, "Couldn't allocate read_buffer_primary\n");
 		retval = -ENOMEM;
 		goto error;
 	}
@@ -723,7 +721,6 @@ static int adu_probe(struct usb_interface *interface,
 
 	dev->read_buffer_secondary = kmalloc((4 * in_end_size), GFP_KERNEL);
 	if (!dev->read_buffer_secondary) {
-		dev_err(&interface->dev, "Couldn't allocate read_buffer_secondary\n");
 		retval = -ENOMEM;
 		goto error;
 	}
@@ -735,29 +732,21 @@ static int adu_probe(struct usb_interface *interface,
 	memset(dev->read_buffer_secondary + (3 * in_end_size), 'h', in_end_size);
 
 	dev->interrupt_in_buffer = kmalloc(in_end_size, GFP_KERNEL);
-	if (!dev->interrupt_in_buffer) {
-		dev_err(&interface->dev, "Couldn't allocate interrupt_in_buffer\n");
+	if (!dev->interrupt_in_buffer)
 		goto error;
-	}
 
 	/* debug code prime the buffer */
 	memset(dev->interrupt_in_buffer, 'i', in_end_size);
 
 	dev->interrupt_in_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->interrupt_in_urb) {
-		dev_err(&interface->dev, "Couldn't allocate interrupt_in_urb\n");
+	if (!dev->interrupt_in_urb)
 		goto error;
-	}
 	dev->interrupt_out_buffer = kmalloc(out_end_size, GFP_KERNEL);
-	if (!dev->interrupt_out_buffer) {
-		dev_err(&interface->dev, "Couldn't allocate interrupt_out_buffer\n");
+	if (!dev->interrupt_out_buffer)
 		goto error;
-	}
 	dev->interrupt_out_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->interrupt_out_urb) {
-		dev_err(&interface->dev, "Couldn't allocate interrupt_out_urb\n");
+	if (!dev->interrupt_out_urb)
 		goto error;
-	}
 
 	if (!usb_string(udev, udev->descriptor.iSerialNumber, dev->serial_number,
 			sizeof(dev->serial_number))) {
diff --git a/drivers/usb/misc/appledisplay.c b/drivers/usb/misc/appledisplay.c
index a0a3827b4aff..da5ff401a354 100644
--- a/drivers/usb/misc/appledisplay.c
+++ b/drivers/usb/misc/appledisplay.c
@@ -85,7 +85,6 @@ struct appledisplay {
 };
 
 static atomic_t count_displays = ATOMIC_INIT(0);
-static struct workqueue_struct *wq;
 
 static void appledisplay_complete(struct urb *urb)
 {
@@ -122,7 +121,7 @@ static void appledisplay_complete(struct urb *urb)
 	case ACD_BTN_BRIGHT_UP:
 	case ACD_BTN_BRIGHT_DOWN:
 		pdata->button_pressed = 1;
-		queue_delayed_work(wq, &pdata->work, 0);
+		schedule_delayed_work(&pdata->work, 0);
 		break;
 	case ACD_BTN_NONE:
 	default:
@@ -239,7 +238,6 @@ static int appledisplay_probe(struct usb_interface *iface,
 	pdata = kzalloc(sizeof(struct appledisplay), GFP_KERNEL);
 	if (!pdata) {
 		retval = -ENOMEM;
-		dev_err(&iface->dev, "Out of memory\n");
 		goto error;
 	}
 
@@ -253,8 +251,6 @@ static int appledisplay_probe(struct usb_interface *iface,
 	pdata->msgdata = kmalloc(ACD_MSG_BUFFER_LEN, GFP_KERNEL);
 	if (!pdata->msgdata) {
 		retval = -ENOMEM;
-		dev_err(&iface->dev,
-			"Allocating buffer for control messages failed\n");
 		goto error;
 	}
 
@@ -262,7 +258,6 @@ static int appledisplay_probe(struct usb_interface *iface,
 	pdata->urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!pdata->urb) {
 		retval = -ENOMEM;
-		dev_err(&iface->dev, "Allocating URB failed\n");
 		goto error;
 	}
 
@@ -344,7 +339,7 @@ static void appledisplay_disconnect(struct usb_interface *iface)
 
 	if (pdata) {
 		usb_kill_urb(pdata->urb);
-		cancel_delayed_work(&pdata->work);
+		cancel_delayed_work_sync(&pdata->work);
 		backlight_device_unregister(pdata->bd);
 		usb_free_coherent(pdata->udev, ACD_URB_BUFFER_LEN,
 			pdata->urbdata, pdata->urb->transfer_dma);
@@ -365,19 +360,11 @@ static struct usb_driver appledisplay_driver = {
 
 static int __init appledisplay_init(void)
 {
-	wq = create_singlethread_workqueue("appledisplay");
-	if (!wq) {
-		printk(KERN_ERR "appledisplay: Could not create work queue\n");
-		return -ENOMEM;
-	}
-
 	return usb_register(&appledisplay_driver);
 }
 
 static void __exit appledisplay_exit(void)
 {
-	flush_workqueue(wq);
-	destroy_workqueue(wq);
 	usb_deregister(&appledisplay_driver);
 }
 
diff --git a/drivers/usb/misc/cypress_cy7c63.c b/drivers/usb/misc/cypress_cy7c63.c
index 402b94dd2531..5c93a888c40e 100644
--- a/drivers/usb/misc/cypress_cy7c63.c
+++ b/drivers/usb/misc/cypress_cy7c63.c
@@ -79,7 +79,6 @@ static int vendor_command(struct cypress *dev, unsigned char request,
 	/* allocate some memory for the i/o buffer*/
 	iobuf = kzalloc(CYPRESS_MAX_REQSIZE, GFP_KERNEL);
 	if (!iobuf) {
-		dev_err(&dev->udev->dev, "Out of memory!\n");
 		retval = -ENOMEM;
 		goto error;
 	}
@@ -208,10 +207,8 @@ static int cypress_probe(struct usb_interface *interface,
 
 	/* allocate memory for our device state and initialize it */
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (dev == NULL) {
-		dev_err(&interface->dev, "Out of memory!\n");
+	if (!dev)
 		goto error_mem;
-	}
 
 	dev->udev = usb_get_dev(interface_to_usbdev(interface));
 
diff --git a/drivers/usb/misc/cytherm.c b/drivers/usb/misc/cytherm.c
index 9bab1a33bc16..9d8bb8dacdcd 100644
--- a/drivers/usb/misc/cytherm.c
+++ b/drivers/usb/misc/cytherm.c
@@ -101,10 +101,8 @@ static ssize_t set_brightness(struct device *dev, struct device_attribute *attr,
 	int retval;
    
 	buffer = kmalloc(8, GFP_KERNEL);
-	if (!buffer) {
-		dev_err(&cytherm->udev->dev, "out of memory\n");
+	if (!buffer)
 		return 0;
-	}
 
 	cytherm->brightness = simple_strtoul(buf, NULL, 10);
    
@@ -148,10 +146,8 @@ static ssize_t show_temp(struct device *dev, struct device_attribute *attr, char
 	int temp, sign;
    
 	buffer = kmalloc(8, GFP_KERNEL);
-	if (!buffer) {
-		dev_err(&cytherm->udev->dev, "out of memory\n");
+	if (!buffer)
 		return 0;
-	}
 
 	/* read temperature */
 	retval = vendor_command(cytherm->udev, READ_RAM, TEMP, 0, buffer, 8);
@@ -192,10 +188,8 @@ static ssize_t show_button(struct device *dev, struct device_attribute *attr, ch
 	unsigned char *buffer;
 
 	buffer = kmalloc(8, GFP_KERNEL);
-	if (!buffer) {
-		dev_err(&cytherm->udev->dev, "out of memory\n");
+	if (!buffer)
 		return 0;
-	}
 
 	/* check button */
 	retval = vendor_command(cytherm->udev, READ_RAM, BUTTON, 0, buffer, 8);
@@ -230,10 +224,8 @@ static ssize_t show_port0(struct device *dev, struct device_attribute *attr, cha
 	unsigned char *buffer;
 
 	buffer = kmalloc(8, GFP_KERNEL);
-	if (!buffer) {
-		dev_err(&cytherm->udev->dev, "out of memory\n");
+	if (!buffer)
 		return 0;
-	}
 
 	retval = vendor_command(cytherm->udev, READ_PORT, 0, 0, buffer, 8);
 	if (retval)
@@ -257,10 +249,8 @@ static ssize_t set_port0(struct device *dev, struct device_attribute *attr, cons
 	int tmp;
    
 	buffer = kmalloc(8, GFP_KERNEL);
-	if (!buffer) {
-		dev_err(&cytherm->udev->dev, "out of memory\n");
+	if (!buffer)
 		return 0;
-	}
 
 	tmp = simple_strtoul(buf, NULL, 10);
    
@@ -290,10 +280,8 @@ static ssize_t show_port1(struct device *dev, struct device_attribute *attr, cha
 	unsigned char *buffer;
 
 	buffer = kmalloc(8, GFP_KERNEL);
-	if (!buffer) {
-		dev_err(&cytherm->udev->dev, "out of memory\n");
+	if (!buffer)
 		return 0;
-	}
 
 	retval = vendor_command(cytherm->udev, READ_PORT, 1, 0, buffer, 8);
 	if (retval)
@@ -317,10 +305,8 @@ static ssize_t set_port1(struct device *dev, struct device_attribute *attr, cons
 	int tmp;
    
 	buffer = kmalloc(8, GFP_KERNEL);
-	if (!buffer) {
-		dev_err(&cytherm->udev->dev, "out of memory\n");
+	if (!buffer)
 		return 0;
-	}
 
 	tmp = simple_strtoul(buf, NULL, 10);
    
@@ -351,10 +337,8 @@ static int cytherm_probe(struct usb_interface *interface,
 	int retval = -ENOMEM;
 
 	dev = kzalloc (sizeof(struct usb_cytherm), GFP_KERNEL);
-	if (dev == NULL) {
-		dev_err (&interface->dev, "Out of memory\n");
+	if (!dev)
 		goto error_mem;
-	}
 
 	dev->udev = usb_get_dev(udev);
 
diff --git a/drivers/usb/misc/ezusb.c b/drivers/usb/misc/ezusb.c
index 947811bc8126..837208f14f86 100644
--- a/drivers/usb/misc/ezusb.c
+++ b/drivers/usb/misc/ezusb.c
@@ -22,7 +22,7 @@ struct ezusb_fx_type {
 	unsigned short max_internal_adress;
 };
 
-static struct ezusb_fx_type ezusb_fx1 = {
+static const struct ezusb_fx_type ezusb_fx1 = {
 	.cpucs_reg = 0x7F92,
 	.max_internal_adress = 0x1B3F,
 };
diff --git a/drivers/usb/misc/ftdi-elan.c b/drivers/usb/misc/ftdi-elan.c
index 52c27cab78c3..9a82f8308ad7 100644
--- a/drivers/usb/misc/ftdi-elan.c
+++ b/drivers/usb/misc/ftdi-elan.c
@@ -61,9 +61,6 @@ module_param(distrust_firmware, bool, 0);
 MODULE_PARM_DESC(distrust_firmware,
 		 "true to distrust firmware power/overcurrent setup");
 extern struct platform_driver u132_platform_driver;
-static struct workqueue_struct *status_queue;
-static struct workqueue_struct *command_queue;
-static struct workqueue_struct *respond_queue;
 /*
  * ftdi_module_lock exists to protect access to global variables
  *
@@ -228,56 +225,56 @@ static void ftdi_elan_init_kref(struct usb_ftdi *ftdi)
 
 static void ftdi_status_requeue_work(struct usb_ftdi *ftdi, unsigned int delta)
 {
-	if (!queue_delayed_work(status_queue, &ftdi->status_work, delta))
+	if (!schedule_delayed_work(&ftdi->status_work, delta))
 		kref_put(&ftdi->kref, ftdi_elan_delete);
 }
 
 static void ftdi_status_queue_work(struct usb_ftdi *ftdi, unsigned int delta)
 {
-	if (queue_delayed_work(status_queue, &ftdi->status_work, delta))
+	if (schedule_delayed_work(&ftdi->status_work, delta))
 		kref_get(&ftdi->kref);
 }
 
 static void ftdi_status_cancel_work(struct usb_ftdi *ftdi)
 {
-	if (cancel_delayed_work(&ftdi->status_work))
+	if (cancel_delayed_work_sync(&ftdi->status_work))
 		kref_put(&ftdi->kref, ftdi_elan_delete);
 }
 
 static void ftdi_command_requeue_work(struct usb_ftdi *ftdi, unsigned int delta)
 {
-	if (!queue_delayed_work(command_queue, &ftdi->command_work, delta))
+	if (!schedule_delayed_work(&ftdi->command_work, delta))
 		kref_put(&ftdi->kref, ftdi_elan_delete);
 }
 
 static void ftdi_command_queue_work(struct usb_ftdi *ftdi, unsigned int delta)
 {
-	if (queue_delayed_work(command_queue, &ftdi->command_work, delta))
+	if (schedule_delayed_work(&ftdi->command_work, delta))
 		kref_get(&ftdi->kref);
 }
 
 static void ftdi_command_cancel_work(struct usb_ftdi *ftdi)
 {
-	if (cancel_delayed_work(&ftdi->command_work))
+	if (cancel_delayed_work_sync(&ftdi->command_work))
 		kref_put(&ftdi->kref, ftdi_elan_delete);
 }
 
 static void ftdi_response_requeue_work(struct usb_ftdi *ftdi,
 				       unsigned int delta)
 {
-	if (!queue_delayed_work(respond_queue, &ftdi->respond_work, delta))
+	if (!schedule_delayed_work(&ftdi->respond_work, delta))
 		kref_put(&ftdi->kref, ftdi_elan_delete);
 }
 
 static void ftdi_respond_queue_work(struct usb_ftdi *ftdi, unsigned int delta)
 {
-	if (queue_delayed_work(respond_queue, &ftdi->respond_work, delta))
+	if (schedule_delayed_work(&ftdi->respond_work, delta))
 		kref_get(&ftdi->kref);
 }
 
 static void ftdi_response_cancel_work(struct usb_ftdi *ftdi)
 {
-	if (cancel_delayed_work(&ftdi->respond_work))
+	if (cancel_delayed_work_sync(&ftdi->respond_work))
 		kref_put(&ftdi->kref, ftdi_elan_delete);
 }
 
@@ -665,7 +662,7 @@ static ssize_t ftdi_elan_read(struct file *file, char __user *buffer,
 {
 	char data[30 *3 + 4];
 	char *d = data;
-	int m = (sizeof(data) - 1) / 3;
+	int m = (sizeof(data) - 1) / 3 - 1;
 	int bytes_read = 0;
 	int retry_on_empty = 10;
 	int retry_on_timeout = 5;
@@ -785,11 +782,8 @@ static int ftdi_elan_command_engine(struct usb_ftdi *ftdi)
 		return 0;
 	total_size = ftdi_elan_total_command_size(ftdi, command_size);
 	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb) {
-		dev_err(&ftdi->udev->dev, "could not get a urb to write %d commands totaling %d bytes to the Uxxx\n",
-			command_size, total_size);
+	if (!urb)
 		return -ENOMEM;
-	}
 	buf = usb_alloc_coherent(ftdi->udev, total_size, GFP_KERNEL,
 				 &urb->transfer_dma);
 	if (!buf) {
@@ -1684,7 +1678,7 @@ wait:if (ftdi->disconnected > 0) {
 			int i = 0;
 			char data[30 *3 + 4];
 			char *d = data;
-			int m = (sizeof(data) - 1) / 3;
+			int m = (sizeof(data) - 1) / 3 - 1;
 			int l = 0;
 			struct u132_target *target = &ftdi->target[ed];
 			struct u132_command *command = &ftdi->command[
@@ -1876,7 +1870,7 @@ more:{
 		if (packet_bytes > 2) {
 			char diag[30 *3 + 4];
 			char *d = diag;
-			int m = (sizeof(diag) - 1) / 3;
+			int m = (sizeof(diag) - 1) / 3 - 1;
 			char *b = ftdi->bulk_in_buffer;
 			int bytes_read = 0;
 			diag[0] = 0;
@@ -1948,10 +1942,8 @@ static int ftdi_elan_synchronize_flush(struct usb_ftdi *ftdi)
 	int I = 257;
 	int i = 0;
 	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb) {
-		dev_err(&ftdi->udev->dev, "could not alloc a urb for flush sequence\n");
+	if (!urb)
 		return -ENOMEM;
-	}
 	buf = usb_alloc_coherent(ftdi->udev, I, GFP_KERNEL, &urb->transfer_dma);
 	if (!buf) {
 		dev_err(&ftdi->udev->dev, "could not get a buffer for flush sequence\n");
@@ -1988,10 +1980,8 @@ static int ftdi_elan_synchronize_reset(struct usb_ftdi *ftdi)
 	int I = 4;
 	int i = 0;
 	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb) {
-		dev_err(&ftdi->udev->dev, "could not get a urb for the reset sequence\n");
+	if (!urb)
 		return -ENOMEM;
-	}
 	buf = usb_alloc_coherent(ftdi->udev, I, GFP_KERNEL, &urb->transfer_dma);
 	if (!buf) {
 		dev_err(&ftdi->udev->dev, "could not get a buffer for the reset sequence\n");
@@ -2053,7 +2043,7 @@ static int ftdi_elan_synchronize(struct usb_ftdi *ftdi)
 			if (packet_bytes > 2) {
 				char diag[30 *3 + 4];
 				char *d = diag;
-				int m = (sizeof(diag) - 1) / 3;
+				int m = (sizeof(diag) - 1) / 3 - 1;
 				char *b = ftdi->bulk_in_buffer;
 				int bytes_read = 0;
 				unsigned char c = 0;
@@ -2155,7 +2145,7 @@ more:{
 		if (packet_bytes > 2) {
 			char diag[30 *3 + 4];
 			char *d = diag;
-			int m = (sizeof(diag) - 1) / 3;
+			int m = (sizeof(diag) - 1) / 3 - 1;
 			char *b = ftdi->bulk_in_buffer;
 			int bytes_read = 0;
 			diag[0] = 0;
@@ -2740,7 +2730,6 @@ static int ftdi_elan_probe(struct usb_interface *interface,
 			ftdi->bulk_in_endpointAddr = endpoint->bEndpointAddress;
 			ftdi->bulk_in_buffer = kmalloc(buffer_size, GFP_KERNEL);
 			if (!ftdi->bulk_in_buffer) {
-				dev_err(&ftdi->udev->dev, "Could not allocate bulk_in_buffer\n");
 				retval = -ENOMEM;
 				goto error;
 			}
@@ -2823,9 +2812,6 @@ static void ftdi_elan_disconnect(struct usb_interface *interface)
 			ftdi->initialized = 0;
 			ftdi->registered = 0;
 		}
-		flush_workqueue(status_queue);
-		flush_workqueue(command_queue);
-		flush_workqueue(respond_queue);
 		ftdi->disconnected += 1;
 		usb_set_intfdata(interface, NULL);
 		dev_info(&ftdi->udev->dev, "USB FTDI U132 host controller interface now disconnected\n");
@@ -2845,31 +2831,12 @@ static int __init ftdi_elan_init(void)
 	pr_info("driver %s\n", ftdi_elan_driver.name);
 	mutex_init(&ftdi_module_lock);
 	INIT_LIST_HEAD(&ftdi_static_list);
-	status_queue = create_singlethread_workqueue("ftdi-status-control");
-	if (!status_queue)
-		goto err_status_queue;
-	command_queue = create_singlethread_workqueue("ftdi-command-engine");
-	if (!command_queue)
-		goto err_command_queue;
-	respond_queue = create_singlethread_workqueue("ftdi-respond-engine");
-	if (!respond_queue)
-		goto err_respond_queue;
 	result = usb_register(&ftdi_elan_driver);
 	if (result) {
-		destroy_workqueue(status_queue);
-		destroy_workqueue(command_queue);
-		destroy_workqueue(respond_queue);
 		pr_err("usb_register failed. Error number %d\n", result);
 	}
 	return result;
 
-err_respond_queue:
-	destroy_workqueue(command_queue);
-err_command_queue:
-	destroy_workqueue(status_queue);
-err_status_queue:
-	pr_err("%s couldn't create workqueue\n", ftdi_elan_driver.name);
-	return -ENOMEM;
 }
 
 static void __exit ftdi_elan_exit(void)
@@ -2882,15 +2849,7 @@ static void __exit ftdi_elan_exit(void)
 		ftdi_status_cancel_work(ftdi);
 		ftdi_command_cancel_work(ftdi);
 		ftdi_response_cancel_work(ftdi);
-	} flush_workqueue(status_queue);
-	destroy_workqueue(status_queue);
-	status_queue = NULL;
-	flush_workqueue(command_queue);
-	destroy_workqueue(command_queue);
-	command_queue = NULL;
-	flush_workqueue(respond_queue);
-	destroy_workqueue(respond_queue);
-	respond_queue = NULL;
+	}
 }
 
 
diff --git a/drivers/usb/misc/idmouse.c b/drivers/usb/misc/idmouse.c
index 5105397e62fc..2975e80b7a56 100644
--- a/drivers/usb/misc/idmouse.c
+++ b/drivers/usb/misc/idmouse.c
@@ -366,7 +366,6 @@ static int idmouse_probe(struct usb_interface *interface,
 			kmalloc(IMGSIZE + dev->bulk_in_size, GFP_KERNEL);
 
 		if (!dev->bulk_in_buffer) {
-			dev_err(&interface->dev, "Unable to allocate input buffer.\n");
 			idmouse_delete(dev);
 			return -ENOMEM;
 		}
diff --git a/drivers/usb/misc/iowarrior.c b/drivers/usb/misc/iowarrior.c
index 1950e87b4219..095778ff984d 100644
--- a/drivers/usb/misc/iowarrior.c
+++ b/drivers/usb/misc/iowarrior.c
@@ -278,7 +278,7 @@ static ssize_t iowarrior_read(struct file *file, char __user *buffer,
 	dev = file->private_data;
 
 	/* verify that the device wasn't unplugged */
-	if (dev == NULL || !dev->present)
+	if (!dev || !dev->present)
 		return -ENODEV;
 
 	dev_dbg(&dev->interface->dev, "minor %d, count = %zd\n",
@@ -413,8 +413,6 @@ static ssize_t iowarrior_write(struct file *file,
 		int_out_urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!int_out_urb) {
 			retval = -ENOMEM;
-			dev_dbg(&dev->interface->dev,
-				"Unable to allocate urb\n");
 			goto error_no_urb;
 		}
 		buf = usb_alloc_coherent(dev->udev, dev->report_size,
@@ -482,9 +480,8 @@ static long iowarrior_ioctl(struct file *file, unsigned int cmd,
 	int io_res;		/* checks for bytes read/written and copy_to/from_user results */
 
 	dev = file->private_data;
-	if (dev == NULL) {
+	if (!dev)
 		return -ENODEV;
-	}
 
 	buffer = kzalloc(dev->report_size, GFP_KERNEL);
 	if (!buffer)
@@ -654,9 +651,8 @@ static int iowarrior_release(struct inode *inode, struct file *file)
 	int retval = 0;
 
 	dev = file->private_data;
-	if (dev == NULL) {
+	if (!dev)
 		return -ENODEV;
-	}
 
 	dev_dbg(&dev->interface->dev, "minor %d\n", dev->minor);
 
@@ -766,10 +762,8 @@ static int iowarrior_probe(struct usb_interface *interface,
 
 	/* allocate memory for our device state and initialize it */
 	dev = kzalloc(sizeof(struct iowarrior), GFP_KERNEL);
-	if (dev == NULL) {
-		dev_err(&interface->dev, "Out of memory\n");
+	if (!dev)
 		return retval;
-	}
 
 	mutex_init(&dev->mutex);
 
@@ -812,15 +806,11 @@ static int iowarrior_probe(struct usb_interface *interface,
 
 	/* create the urb and buffer for reading */
 	dev->int_in_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->int_in_urb) {
-		dev_err(&interface->dev, "Couldn't allocate interrupt_in_urb\n");
+	if (!dev->int_in_urb)
 		goto error;
-	}
 	dev->int_in_buffer = kmalloc(dev->report_size, GFP_KERNEL);
-	if (!dev->int_in_buffer) {
-		dev_err(&interface->dev, "Couldn't allocate int_in_buffer\n");
+	if (!dev->int_in_buffer)
 		goto error;
-	}
 	usb_fill_int_urb(dev->int_in_urb, dev->udev,
 			 usb_rcvintpipe(dev->udev,
 					dev->int_in_endpoint->bEndpointAddress),
@@ -831,10 +821,8 @@ static int iowarrior_probe(struct usb_interface *interface,
 	dev->read_queue =
 	    kmalloc(((dev->report_size + 1) * MAX_INTERRUPT_BUFFER),
 		    GFP_KERNEL);
-	if (!dev->read_queue) {
-		dev_err(&interface->dev, "Couldn't allocate read_queue\n");
+	if (!dev->read_queue)
 		goto error;
-	}
 	/* Get the serial-number of the chip */
 	memset(dev->chip_serial, 0x00, sizeof(dev->chip_serial));
 	usb_string(udev, udev->descriptor.iSerialNumber, dev->chip_serial,
diff --git a/drivers/usb/misc/ldusb.c b/drivers/usb/misc/ldusb.c
index cce22ff1c2eb..9ca595632f17 100644
--- a/drivers/usb/misc/ldusb.c
+++ b/drivers/usb/misc/ldusb.c
@@ -658,10 +658,8 @@ static int ld_usb_probe(struct usb_interface *intf, const struct usb_device_id *
 	/* allocate memory for our device state and initialize it */
 
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (dev == NULL) {
-		dev_err(&intf->dev, "Out of memory\n");
+	if (!dev)
 		goto exit;
-	}
 	mutex_init(&dev->mutex);
 	spin_lock_init(&dev->rbsl);
 	dev->intf = intf;
@@ -674,10 +672,8 @@ static int ld_usb_probe(struct usb_interface *intf, const struct usb_device_id *
 	     (le16_to_cpu(udev->descriptor.idProduct) == USB_DEVICE_ID_LD_COM3LAB)) &&
 	    (le16_to_cpu(udev->descriptor.bcdDevice) <= 0x103)) {
 		buffer = kmalloc(256, GFP_KERNEL);
-		if (buffer == NULL) {
-			dev_err(&intf->dev, "Couldn't allocate string buffer\n");
+		if (!buffer)
 			goto error;
-		}
 		/* usb_string makes SETUP+STALL to leave always ControlReadLoop */
 		usb_string(udev, 255, buffer, 256);
 		kfree(buffer);
@@ -704,32 +700,22 @@ static int ld_usb_probe(struct usb_interface *intf, const struct usb_device_id *
 
 	dev->interrupt_in_endpoint_size = usb_endpoint_maxp(dev->interrupt_in_endpoint);
 	dev->ring_buffer = kmalloc(ring_buffer_size*(sizeof(size_t)+dev->interrupt_in_endpoint_size), GFP_KERNEL);
-	if (!dev->ring_buffer) {
-		dev_err(&intf->dev, "Couldn't allocate ring_buffer\n");
+	if (!dev->ring_buffer)
 		goto error;
-	}
 	dev->interrupt_in_buffer = kmalloc(dev->interrupt_in_endpoint_size, GFP_KERNEL);
-	if (!dev->interrupt_in_buffer) {
-		dev_err(&intf->dev, "Couldn't allocate interrupt_in_buffer\n");
+	if (!dev->interrupt_in_buffer)
 		goto error;
-	}
 	dev->interrupt_in_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->interrupt_in_urb) {
-		dev_err(&intf->dev, "Couldn't allocate interrupt_in_urb\n");
+	if (!dev->interrupt_in_urb)
 		goto error;
-	}
 	dev->interrupt_out_endpoint_size = dev->interrupt_out_endpoint ? usb_endpoint_maxp(dev->interrupt_out_endpoint) :
 									 udev->descriptor.bMaxPacketSize0;
 	dev->interrupt_out_buffer = kmalloc(write_buffer_size*dev->interrupt_out_endpoint_size, GFP_KERNEL);
-	if (!dev->interrupt_out_buffer) {
-		dev_err(&intf->dev, "Couldn't allocate interrupt_out_buffer\n");
+	if (!dev->interrupt_out_buffer)
 		goto error;
-	}
 	dev->interrupt_out_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->interrupt_out_urb) {
-		dev_err(&intf->dev, "Couldn't allocate interrupt_out_urb\n");
+	if (!dev->interrupt_out_urb)
 		goto error;
-	}
 	dev->interrupt_in_interval = min_interrupt_in_interval > dev->interrupt_in_endpoint->bInterval ? min_interrupt_in_interval : dev->interrupt_in_endpoint->bInterval;
 	if (dev->interrupt_out_endpoint)
 		dev->interrupt_out_interval = min_interrupt_out_interval > dev->interrupt_out_endpoint->bInterval ? min_interrupt_out_interval : dev->interrupt_out_endpoint->bInterval;
diff --git a/drivers/usb/misc/legousbtower.c b/drivers/usb/misc/legousbtower.c
index 7771be3ac178..c8fbe7b739a0 100644
--- a/drivers/usb/misc/legousbtower.c
+++ b/drivers/usb/misc/legousbtower.c
@@ -817,10 +817,8 @@ static int tower_probe (struct usb_interface *interface, const struct usb_device
 
 	dev = kmalloc (sizeof(struct lego_usb_tower), GFP_KERNEL);
 
-	if (dev == NULL) {
-		dev_err(idev, "Out of memory\n");
+	if (!dev)
 		goto exit;
-	}
 
 	mutex_init(&dev->lock);
 
@@ -871,51 +869,23 @@ static int tower_probe (struct usb_interface *interface, const struct usb_device
 	}
 
 	dev->read_buffer = kmalloc (read_buffer_size, GFP_KERNEL);
-	if (!dev->read_buffer) {
-		dev_err(idev, "Couldn't allocate read_buffer\n");
+	if (!dev->read_buffer)
 		goto error;
-	}
 	dev->interrupt_in_buffer = kmalloc (usb_endpoint_maxp(dev->interrupt_in_endpoint), GFP_KERNEL);
-	if (!dev->interrupt_in_buffer) {
-		dev_err(idev, "Couldn't allocate interrupt_in_buffer\n");
+	if (!dev->interrupt_in_buffer)
 		goto error;
-	}
 	dev->interrupt_in_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->interrupt_in_urb) {
-		dev_err(idev, "Couldn't allocate interrupt_in_urb\n");
+	if (!dev->interrupt_in_urb)
 		goto error;
-	}
 	dev->interrupt_out_buffer = kmalloc (write_buffer_size, GFP_KERNEL);
-	if (!dev->interrupt_out_buffer) {
-		dev_err(idev, "Couldn't allocate interrupt_out_buffer\n");
+	if (!dev->interrupt_out_buffer)
 		goto error;
-	}
 	dev->interrupt_out_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->interrupt_out_urb) {
-		dev_err(idev, "Couldn't allocate interrupt_out_urb\n");
+	if (!dev->interrupt_out_urb)
 		goto error;
-	}
 	dev->interrupt_in_interval = interrupt_in_interval ? interrupt_in_interval : dev->interrupt_in_endpoint->bInterval;
 	dev->interrupt_out_interval = interrupt_out_interval ? interrupt_out_interval : dev->interrupt_out_endpoint->bInterval;
 
-	/* we can register the device now, as it is ready */
-	usb_set_intfdata (interface, dev);
-
-	retval = usb_register_dev (interface, &tower_class);
-
-	if (retval) {
-		/* something prevented us from registering this driver */
-		dev_err(idev, "Not able to get a minor for this device.\n");
-		usb_set_intfdata (interface, NULL);
-		goto error;
-	}
-	dev->minor = interface->minor;
-
-	/* let the user know what node this device is now attached to */
-	dev_info(&interface->dev, "LEGO USB Tower #%d now attached to major "
-		 "%d minor %d\n", (dev->minor - LEGO_USB_TOWER_MINOR_BASE),
-		 USB_MAJOR, dev->minor);
-
 	/* get the firmware version and log it */
 	result = usb_control_msg (udev,
 				  usb_rcvctrlpipe(udev, 0),
@@ -936,6 +906,23 @@ static int tower_probe (struct usb_interface *interface, const struct usb_device
 		 get_version_reply.minor,
 		 le16_to_cpu(get_version_reply.build_no));
 
+	/* we can register the device now, as it is ready */
+	usb_set_intfdata (interface, dev);
+
+	retval = usb_register_dev (interface, &tower_class);
+
+	if (retval) {
+		/* something prevented us from registering this driver */
+		dev_err(idev, "Not able to get a minor for this device.\n");
+		usb_set_intfdata (interface, NULL);
+		goto error;
+	}
+	dev->minor = interface->minor;
+
+	/* let the user know what node this device is now attached to */
+	dev_info(&interface->dev, "LEGO USB Tower #%d now attached to major "
+		 "%d minor %d\n", (dev->minor - LEGO_USB_TOWER_MINOR_BASE),
+		 USB_MAJOR, dev->minor);
 
 exit:
 	return retval;
diff --git a/drivers/usb/misc/lvstest.c b/drivers/usb/misc/lvstest.c
index 86b4e4b2ab9a..77176511658f 100644
--- a/drivers/usb/misc/lvstest.c
+++ b/drivers/usb/misc/lvstest.c
@@ -34,8 +34,6 @@ struct lvs_rh {
 	struct usb_hub_descriptor descriptor;
 	/* urb for polling interrupt pipe */
 	struct urb *urb;
-	/* LVS RH work queue */
-	struct workqueue_struct *rh_queue;
 	/* LVH RH work */
 	struct work_struct	rh_work;
 	/* RH port status */
@@ -247,10 +245,8 @@ static ssize_t get_dev_desc_store(struct device *dev,
 	int ret;
 
 	descriptor = kmalloc(sizeof(*descriptor), GFP_KERNEL);
-	if (!descriptor) {
-		dev_err(dev, "failed to allocate descriptor memory\n");
+	if (!descriptor)
 		return -ENOMEM;
-	}
 
 	udev = create_lvs_device(intf);
 	if (!udev) {
@@ -355,7 +351,7 @@ static void lvs_rh_irq(struct urb *urb)
 {
 	struct lvs_rh *lvs = urb->context;
 
-	queue_work(lvs->rh_queue, &lvs->rh_work);
+	schedule_work(&lvs->rh_work);
 }
 
 static int lvs_rh_probe(struct usb_interface *intf,
@@ -397,24 +393,15 @@ static int lvs_rh_probe(struct usb_interface *intf,
 
 	/* submit urb to poll interrupt endpoint */
 	lvs->urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!lvs->urb) {
-		dev_err(&intf->dev, "couldn't allocate lvs urb\n");
+	if (!lvs->urb)
 		return -ENOMEM;
-	}
-
-	lvs->rh_queue = create_singlethread_workqueue("lvs_rh_queue");
-	if (!lvs->rh_queue) {
-		dev_err(&intf->dev, "couldn't create workqueue\n");
-		ret = -ENOMEM;
-		goto free_urb;
-	}
 
 	INIT_WORK(&lvs->rh_work, lvs_rh_work);
 
 	ret = sysfs_create_group(&intf->dev.kobj, &lvs_attr_group);
 	if (ret < 0) {
 		dev_err(&intf->dev, "Failed to create sysfs node %d\n", ret);
-		goto destroy_queue;
+		goto free_urb;
 	}
 
 	pipe = usb_rcvintpipe(hdev, endpoint->bEndpointAddress);
@@ -432,8 +419,6 @@ static int lvs_rh_probe(struct usb_interface *intf,
 
 sysfs_remove:
 	sysfs_remove_group(&intf->dev.kobj, &lvs_attr_group);
-destroy_queue:
-	destroy_workqueue(lvs->rh_queue);
 free_urb:
 	usb_free_urb(lvs->urb);
 	return ret;
@@ -444,7 +429,7 @@ static void lvs_rh_disconnect(struct usb_interface *intf)
 	struct lvs_rh *lvs = usb_get_intfdata(intf);
 
 	sysfs_remove_group(&intf->dev.kobj, &lvs_attr_group);
-	destroy_workqueue(lvs->rh_queue);
+	flush_work(&lvs->rh_work);
 	usb_free_urb(lvs->urb);
 }
 
diff --git a/drivers/usb/misc/sisusbvga/sisusb.c b/drivers/usb/misc/sisusbvga/sisusb.c
index 02abfcdfbf7b..05bd39d62568 100644
--- a/drivers/usb/misc/sisusbvga/sisusb.c
+++ b/drivers/usb/misc/sisusbvga/sisusb.c
@@ -3084,7 +3084,6 @@ static int sisusb_probe(struct usb_interface *intf,
 	/* Allocate URBs */
 	sisusb->sisurbin = usb_alloc_urb(0, GFP_KERNEL);
 	if (!sisusb->sisurbin) {
-		dev_err(&sisusb->sisusb_dev->dev, "Failed to allocate URBs\n");
 		retval = -ENOMEM;
 		goto error_3;
 	}
@@ -3093,8 +3092,6 @@ static int sisusb_probe(struct usb_interface *intf,
 	for (i = 0; i < sisusb->numobufs; i++) {
 		sisusb->sisurbout[i] = usb_alloc_urb(0, GFP_KERNEL);
 		if (!sisusb->sisurbout[i]) {
-			dev_err(&sisusb->sisusb_dev->dev,
-					"Failed to allocate URBs\n");
 			retval = -ENOMEM;
 			goto error_4;
 		}
diff --git a/drivers/usb/misc/trancevibrator.c b/drivers/usb/misc/trancevibrator.c
index 4145314a515b..9795457723d8 100644
--- a/drivers/usb/misc/trancevibrator.c
+++ b/drivers/usb/misc/trancevibrator.c
@@ -95,8 +95,7 @@ static int tv_probe(struct usb_interface *interface,
 	int retval;
 
 	dev = kzalloc(sizeof(struct trancevibrator), GFP_KERNEL);
-	if (dev == NULL) {
-		dev_err(&interface->dev, "Out of memory\n");
+	if (!dev) {
 		retval = -ENOMEM;
 		goto error;
 	}
diff --git a/drivers/usb/misc/usb4604.c b/drivers/usb/misc/usb4604.c
new file mode 100644
index 000000000000..e9f37fb746ac
--- /dev/null
+++ b/drivers/usb/misc/usb4604.c
@@ -0,0 +1,175 @@
+/*
+ * Driver for SMSC USB4604 USB HSIC 4-port 2.0 hub controller driver
+ * Based on usb3503 driver
+ *
+ * Copyright (c) 2012-2013 Dongjin Kim (tobetter@gmail.com)
+ * Copyright (c) 2016 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/i2c.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/gpio/consumer.h>
+
+enum usb4604_mode {
+	USB4604_MODE_UNKNOWN,
+	USB4604_MODE_HUB,
+	USB4604_MODE_STANDBY,
+};
+
+struct usb4604 {
+	enum usb4604_mode	mode;
+	struct device		*dev;
+	struct gpio_desc	*gpio_reset;
+};
+
+static void usb4604_reset(struct usb4604 *hub, int state)
+{
+	gpiod_set_value_cansleep(hub->gpio_reset, state);
+
+	/* Wait for i2c logic to come up */
+	if (state)
+		msleep(250);
+}
+
+static int usb4604_connect(struct usb4604 *hub)
+{
+	struct device *dev = hub->dev;
+	struct i2c_client *client = to_i2c_client(dev);
+	int err;
+	u8 connect_cmd[] = { 0xaa, 0x55, 0x00 };
+
+	usb4604_reset(hub, 1);
+
+	err = i2c_master_send(client, connect_cmd, ARRAY_SIZE(connect_cmd));
+	if (err < 0) {
+		usb4604_reset(hub, 0);
+		return err;
+	}
+
+	hub->mode = USB4604_MODE_HUB;
+	dev_dbg(dev, "switched to HUB mode\n");
+
+	return 0;
+}
+
+static int usb4604_switch_mode(struct usb4604 *hub, enum usb4604_mode mode)
+{
+	struct device *dev = hub->dev;
+	int err = 0;
+
+	switch (mode) {
+	case USB4604_MODE_HUB:
+		err = usb4604_connect(hub);
+		break;
+
+	case USB4604_MODE_STANDBY:
+		usb4604_reset(hub, 0);
+		dev_dbg(dev, "switched to STANDBY mode\n");
+		break;
+
+	default:
+		dev_err(dev, "unknown mode is requested\n");
+		err = -EINVAL;
+		break;
+	}
+
+	return err;
+}
+
+static int usb4604_probe(struct usb4604 *hub)
+{
+	struct device *dev = hub->dev;
+	struct device_node *np = dev->of_node;
+	struct gpio_desc *gpio;
+	u32 mode = USB4604_MODE_HUB;
+
+	gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+	hub->gpio_reset = gpio;
+
+	if (of_property_read_u32(np, "initial-mode", &hub->mode))
+		hub->mode = mode;
+
+	return usb4604_switch_mode(hub, hub->mode);
+}
+
+static int usb4604_i2c_probe(struct i2c_client *i2c,
+			     const struct i2c_device_id *id)
+{
+	struct usb4604 *hub;
+
+	hub = devm_kzalloc(&i2c->dev, sizeof(*hub), GFP_KERNEL);
+	if (!hub)
+		return -ENOMEM;
+
+	i2c_set_clientdata(i2c, hub);
+	hub->dev = &i2c->dev;
+
+	return usb4604_probe(hub);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int usb4604_i2c_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct usb4604 *hub = i2c_get_clientdata(client);
+
+	usb4604_switch_mode(hub, USB4604_MODE_STANDBY);
+
+	return 0;
+}
+
+static int usb4604_i2c_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct usb4604 *hub = i2c_get_clientdata(client);
+
+	usb4604_switch_mode(hub, hub->mode);
+
+	return 0;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(usb4604_i2c_pm_ops, usb4604_i2c_suspend,
+		usb4604_i2c_resume);
+
+static const struct i2c_device_id usb4604_id[] = {
+	{ "usb4604", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, usb4604_id);
+
+#ifdef CONFIG_OF
+static const struct of_device_id usb4604_of_match[] = {
+	{ .compatible = "smsc,usb4604" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, usb4604_of_match);
+#endif
+
+static struct i2c_driver usb4604_i2c_driver = {
+	.driver = {
+		.name = "usb4604",
+		.pm = &usb4604_i2c_pm_ops,
+		.of_match_table = of_match_ptr(usb4604_of_match),
+	},
+	.probe		= usb4604_i2c_probe,
+	.id_table	= usb4604_id,
+};
+module_i2c_driver(usb4604_i2c_driver);
+
+MODULE_DESCRIPTION("USB4604 USB HUB driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/usb/misc/usblcd.c b/drivers/usb/misc/usblcd.c
index 1184390508e9..9f48419abc46 100644
--- a/drivers/usb/misc/usblcd.c
+++ b/drivers/usb/misc/usblcd.c
@@ -321,10 +321,8 @@ static int lcd_probe(struct usb_interface *interface,
 
 	/* allocate memory for our device state and initialize it */
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (dev == NULL) {
-		dev_err(&interface->dev, "Out of memory\n");
+	if (!dev)
 		goto error;
-	}
 	kref_init(&dev->kref);
 	sema_init(&dev->limit_sem, USB_LCD_CONCURRENT_WRITES);
 	init_usb_anchor(&dev->submitted);
@@ -351,11 +349,8 @@ static int lcd_probe(struct usb_interface *interface,
 			dev->bulk_in_size = buffer_size;
 			dev->bulk_in_endpointAddr = endpoint->bEndpointAddress;
 			dev->bulk_in_buffer = kmalloc(buffer_size, GFP_KERNEL);
-			if (!dev->bulk_in_buffer) {
-				dev_err(&interface->dev,
-					"Could not allocate bulk_in_buffer\n");
+			if (!dev->bulk_in_buffer)
 				goto error;
-			}
 		}
 
 		if (!dev->bulk_out_endpointAddr &&
diff --git a/drivers/usb/misc/usbsevseg.c b/drivers/usb/misc/usbsevseg.c
index 1fe6b73c22f3..a0ba5298160c 100644
--- a/drivers/usb/misc/usbsevseg.c
+++ b/drivers/usb/misc/usbsevseg.c
@@ -128,10 +128,8 @@ static void update_display_visual(struct usb_sevsegdev *mydev, gfp_t mf)
 		return;
 
 	buffer = kzalloc(MAXLEN, mf);
-	if (!buffer) {
-		dev_err(&mydev->udev->dev, "out of memory\n");
+	if (!buffer)
 		return;
-	}
 
 	/* The device is right to left, where as you write left to right */
 	for (i = 0; i < mydev->textlength; i++)
@@ -346,10 +344,8 @@ static int sevseg_probe(struct usb_interface *interface,
 	int rc = -ENOMEM;
 
 	mydev = kzalloc(sizeof(struct usb_sevsegdev), GFP_KERNEL);
-	if (mydev == NULL) {
-		dev_err(&interface->dev, "Out of memory\n");
+	if (!mydev)
 		goto error_mem;
-	}
 
 	mydev->udev = usb_get_dev(udev);
 	mydev->intf = interface;
diff --git a/drivers/usb/misc/usbtest.c b/drivers/usb/misc/usbtest.c
index 6b978f04b8d7..5c8210dc6fd9 100644
--- a/drivers/usb/misc/usbtest.c
+++ b/drivers/usb/misc/usbtest.c
@@ -585,7 +585,6 @@ static void sg_timeout(unsigned long _req)
 {
 	struct usb_sg_request	*req = (struct usb_sg_request *) _req;
 
-	req->status = -ETIMEDOUT;
 	usb_sg_cancel(req);
 }
 
@@ -616,8 +615,10 @@ static int perform_sglist(
 		mod_timer(&sg_timer, jiffies +
 				msecs_to_jiffies(SIMPLE_IO_TIMEOUT));
 		usb_sg_wait(req);
-		del_timer_sync(&sg_timer);
-		retval = req->status;
+		if (!del_timer_sync(&sg_timer))
+			retval = -ETIMEDOUT;
+		else
+			retval = req->status;
 
 		/* FIXME check resulting data pattern */
 
@@ -2602,7 +2603,7 @@ usbtest_ioctl(struct usb_interface *intf, unsigned int code, void *buf)
 	ktime_get_ts64(&start);
 
 	retval = usbtest_do_ioctl(intf, param_32);
-	if (retval)
+	if (retval < 0)
 		goto free_mutex;
 
 	ktime_get_ts64(&end);
diff --git a/drivers/usb/misc/uss720.c b/drivers/usb/misc/uss720.c
index bbd029c9c725..356d312add57 100644
--- a/drivers/usb/misc/uss720.c
+++ b/drivers/usb/misc/uss720.c
@@ -150,10 +150,8 @@ static struct uss720_async_request *submit_async_request(struct parport_uss720_p
 	if (!usbdev)
 		return NULL;
 	rq = kzalloc(sizeof(struct uss720_async_request), mem_flags);
-	if (!rq) {
-		dev_err(&usbdev->dev, "submit_async_request out of memory\n");
+	if (!rq)
 		return NULL;
-	}
 	kref_init(&rq->ref_count);
 	INIT_LIST_HEAD(&rq->asynclist);
 	init_completion(&rq->compl);
@@ -162,7 +160,6 @@ static struct uss720_async_request *submit_async_request(struct parport_uss720_p
 	rq->urb = usb_alloc_urb(0, mem_flags);
 	if (!rq->urb) {
 		kref_put(&rq->ref_count, destroy_async);
-		dev_err(&usbdev->dev, "submit_async_request out of memory\n");
 		return NULL;
 	}
 	rq->dr = kmalloc(sizeof(*rq->dr), mem_flags);
diff --git a/drivers/usb/misc/yurex.c b/drivers/usb/misc/yurex.c
index 343fa6ff9f4b..54e53ac4c08f 100644
--- a/drivers/usb/misc/yurex.c
+++ b/drivers/usb/misc/yurex.c
@@ -200,10 +200,8 @@ static int yurex_probe(struct usb_interface *interface, const struct usb_device_
 
 	/* allocate memory for our device state and initialize it */
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev) {
-		dev_err(&interface->dev, "Out of memory\n");
+	if (!dev)
 		goto error;
-	}
 	kref_init(&dev->kref);
 	mutex_init(&dev->io_mutex);
 	spin_lock_init(&dev->lock);
@@ -231,17 +229,13 @@ static int yurex_probe(struct usb_interface *interface, const struct usb_device_
 
 	/* allocate control URB */
 	dev->cntl_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->cntl_urb) {
-		dev_err(&interface->dev, "Could not allocate control URB\n");
+	if (!dev->cntl_urb)
 		goto error;
-	}
 
 	/* allocate buffer for control req */
 	dev->cntl_req = kmalloc(YUREX_BUF_SIZE, GFP_KERNEL);
-	if (!dev->cntl_req) {
-		dev_err(&interface->dev, "Could not allocate cntl_req\n");
+	if (!dev->cntl_req)
 		goto error;
-	}
 
 	/* allocate buffer for control msg */
 	dev->cntl_buffer = usb_alloc_coherent(dev->udev, YUREX_BUF_SIZE,
@@ -269,10 +263,8 @@ static int yurex_probe(struct usb_interface *interface, const struct usb_device_
 
 	/* allocate interrupt URB */
 	dev->urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!dev->urb) {
-		dev_err(&interface->dev, "Could not allocate URB\n");
+	if (!dev->urb)
 		goto error;
-	}
 
 	/* allocate buffer for interrupt in */
 	dev->int_buffer = usb_alloc_coherent(dev->udev, YUREX_BUF_SIZE,
diff --git a/drivers/usb/musb/Kconfig b/drivers/usb/musb/Kconfig
index 886526b5fcdd..72a2a5040848 100644
--- a/drivers/usb/musb/Kconfig
+++ b/drivers/usb/musb/Kconfig
@@ -82,12 +82,12 @@ config USB_MUSB_DA8XX
 	tristate "DA8xx/OMAP-L1x"
 	depends on ARCH_DAVINCI_DA8XX
 	depends on NOP_USB_XCEIV
-	depends on BROKEN
+	select PHY_DA8XX_USB
 
 config USB_MUSB_TUSB6010
 	tristate "TUSB6010"
 	depends on HAS_IOMEM
-	depends on ARCH_OMAP2PLUS || COMPILE_TEST
+	depends on (ARCH_OMAP2PLUS || COMPILE_TEST) && !BLACKFIN
 	depends on NOP_USB_XCEIV = USB_MUSB_HDRC # both built-in or both modules
 
 config USB_MUSB_OMAP2PLUS
diff --git a/drivers/usb/musb/am35x.c b/drivers/usb/musb/am35x.c
index c41fe588d14d..50ca8052bc8e 100644
--- a/drivers/usb/musb/am35x.c
+++ b/drivers/usb/musb/am35x.c
@@ -474,10 +474,8 @@ static int am35x_probe(struct platform_device *pdev)
 	int				ret = -ENOMEM;
 
 	glue = kzalloc(sizeof(*glue), GFP_KERNEL);
-	if (!glue) {
-		dev_err(&pdev->dev, "failed to allocate glue context\n");
+	if (!glue)
 		goto err0;
-	}
 
 	phy_clk = clk_get(&pdev->dev, "fck");
 	if (IS_ERR(phy_clk)) {
@@ -512,8 +510,10 @@ static int am35x_probe(struct platform_device *pdev)
 	pdata->platform_ops		= &am35x_ops;
 
 	glue->phy = usb_phy_generic_register();
-	if (IS_ERR(glue->phy))
+	if (IS_ERR(glue->phy)) {
+		ret = PTR_ERR(glue->phy);
 		goto err7;
+	}
 	platform_set_drvdata(pdev, glue);
 
 	pinfo = am35x_dev_info;
diff --git a/drivers/usb/musb/da8xx.c b/drivers/usb/musb/da8xx.c
index b03d3b867fca..210b7e43a6fd 100644
--- a/drivers/usb/musb/da8xx.c
+++ b/drivers/usb/musb/da8xx.c
@@ -30,13 +30,11 @@
 #include <linux/clk.h>
 #include <linux/err.h>
 #include <linux/io.h>
+#include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/dma-mapping.h>
 #include <linux/usb/usb_phy_generic.h>
 
-#include <mach/da8xx.h>
-#include <linux/platform_data/usb-davinci.h>
-
 #include "musb_core.h"
 
 /*
@@ -80,61 +78,15 @@
 
 #define DA8XX_MENTOR_CORE_OFFSET 0x400
 
-#define CFGCHIP2	IO_ADDRESS(DA8XX_SYSCFG0_BASE + DA8XX_CFGCHIP2_REG)
-
 struct da8xx_glue {
 	struct device		*dev;
 	struct platform_device	*musb;
-	struct platform_device	*phy;
+	struct platform_device	*usb_phy;
 	struct clk		*clk;
+	struct phy		*phy;
 };
 
 /*
- * REVISIT (PM): we should be able to keep the PHY in low power mode most
- * of the time (24 MHz oscillator and PLL off, etc.) by setting POWER.D0
- * and, when in host mode, autosuspending idle root ports... PHY_PLLON
- * (overriding SUSPENDM?) then likely needs to stay off.
- */
-
-static inline void phy_on(void)
-{
-	u32 cfgchip2 = __raw_readl(CFGCHIP2);
-
-	/*
-	 * Start the on-chip PHY and its PLL.
-	 */
-	cfgchip2 &= ~(CFGCHIP2_RESET | CFGCHIP2_PHYPWRDN | CFGCHIP2_OTGPWRDN);
-	cfgchip2 |= CFGCHIP2_PHY_PLLON;
-	__raw_writel(cfgchip2, CFGCHIP2);
-
-	pr_info("Waiting for USB PHY clock good...\n");
-	while (!(__raw_readl(CFGCHIP2) & CFGCHIP2_PHYCLKGD))
-		cpu_relax();
-}
-
-static inline void phy_off(void)
-{
-	u32 cfgchip2 = __raw_readl(CFGCHIP2);
-
-	/*
-	 * Ensure that USB 1.1 reference clock is not being sourced from
-	 * USB 2.0 PHY.  Otherwise do not power down the PHY.
-	 */
-	if (!(cfgchip2 & CFGCHIP2_USB1PHYCLKMUX) &&
-	     (cfgchip2 & CFGCHIP2_USB1SUSPENDM)) {
-		pr_warning("USB 1.1 clocked from USB 2.0 PHY -- "
-			   "can't power it down\n");
-		return;
-	}
-
-	/*
-	 * Power down the on-chip PHY.
-	 */
-	cfgchip2 |= CFGCHIP2_PHYPWRDN | CFGCHIP2_OTGPWRDN;
-	__raw_writel(cfgchip2, CFGCHIP2);
-}
-
-/*
  * Because we don't set CTRL.UINT, it's "important" to:
  *	- not read/write INTRUSB/INTRUSBE (except during
  *	  initial setup, as a workaround);
@@ -385,29 +337,29 @@ static irqreturn_t da8xx_musb_interrupt(int irq, void *hci)
 
 static int da8xx_musb_set_mode(struct musb *musb, u8 musb_mode)
 {
-	u32 cfgchip2 = __raw_readl(CFGCHIP2);
+	struct da8xx_glue *glue = dev_get_drvdata(musb->controller->parent);
+	enum phy_mode phy_mode;
 
-	cfgchip2 &= ~CFGCHIP2_OTGMODE;
 	switch (musb_mode) {
 	case MUSB_HOST:		/* Force VBUS valid, ID = 0 */
-		cfgchip2 |= CFGCHIP2_FORCE_HOST;
+		phy_mode = PHY_MODE_USB_HOST;
 		break;
 	case MUSB_PERIPHERAL:	/* Force VBUS valid, ID = 1 */
-		cfgchip2 |= CFGCHIP2_FORCE_DEVICE;
+		phy_mode = PHY_MODE_USB_DEVICE;
 		break;
 	case MUSB_OTG:		/* Don't override the VBUS/ID comparators */
-		cfgchip2 |= CFGCHIP2_NO_OVERRIDE;
+		phy_mode = PHY_MODE_USB_OTG;
 		break;
 	default:
-		dev_dbg(musb->controller, "Trying to set unsupported mode %u\n", musb_mode);
+		return -EINVAL;
 	}
 
-	__raw_writel(cfgchip2, CFGCHIP2);
-	return 0;
+	return phy_set_mode(glue->phy, phy_mode);
 }
 
 static int da8xx_musb_init(struct musb *musb)
 {
+	struct da8xx_glue *glue = dev_get_drvdata(musb->controller->parent);
 	void __iomem *reg_base = musb->ctrl_base;
 	u32 rev;
 	int ret = -ENODEV;
@@ -425,32 +377,56 @@ static int da8xx_musb_init(struct musb *musb)
 		goto fail;
 	}
 
+	ret = clk_prepare_enable(glue->clk);
+	if (ret) {
+		dev_err(glue->dev, "failed to enable clock\n");
+		goto fail;
+	}
+
 	setup_timer(&otg_workaround, otg_timer, (unsigned long)musb);
 
 	/* Reset the controller */
 	musb_writel(reg_base, DA8XX_USB_CTRL_REG, DA8XX_SOFT_RESET_MASK);
 
 	/* Start the on-chip PHY and its PLL. */
-	phy_on();
+	ret = phy_init(glue->phy);
+	if (ret) {
+		dev_err(glue->dev, "Failed to init phy.\n");
+		goto err_phy_init;
+	}
+
+	ret = phy_power_on(glue->phy);
+	if (ret) {
+		dev_err(glue->dev, "Failed to power on phy.\n");
+		goto err_phy_power_on;
+	}
 
 	msleep(5);
 
 	/* NOTE: IRQs are in mixed mode, not bypass to pure MUSB */
-	pr_debug("DA8xx OTG revision %08x, PHY %03x, control %02x\n",
-		 rev, __raw_readl(CFGCHIP2),
+	pr_debug("DA8xx OTG revision %08x, control %02x\n", rev,
 		 musb_readb(reg_base, DA8XX_USB_CTRL_REG));
 
 	musb->isr = da8xx_musb_interrupt;
 	return 0;
+
+err_phy_power_on:
+	phy_exit(glue->phy);
+err_phy_init:
+	clk_disable_unprepare(glue->clk);
 fail:
 	return ret;
 }
 
 static int da8xx_musb_exit(struct musb *musb)
 {
+	struct da8xx_glue *glue = dev_get_drvdata(musb->controller->parent);
+
 	del_timer_sync(&otg_workaround);
 
-	phy_off();
+	phy_power_off(glue->phy);
+	phy_exit(glue->phy);
+	clk_disable_unprepare(glue->clk);
 
 	usb_put_phy(musb->xceiv);
 
@@ -486,30 +462,25 @@ static int da8xx_probe(struct platform_device *pdev)
 {
 	struct resource musb_resources[2];
 	struct musb_hdrc_platform_data	*pdata = dev_get_platdata(&pdev->dev);
-	struct platform_device		*musb;
 	struct da8xx_glue		*glue;
 	struct platform_device_info	pinfo;
 	struct clk			*clk;
+	int				ret;
 
-	int				ret = -ENOMEM;
-
-	glue = kzalloc(sizeof(*glue), GFP_KERNEL);
-	if (!glue) {
-		dev_err(&pdev->dev, "failed to allocate glue context\n");
-		goto err0;
-	}
+	glue = devm_kzalloc(&pdev->dev, sizeof(*glue), GFP_KERNEL);
+	if (!glue)
+		return -ENOMEM;
 
-	clk = clk_get(&pdev->dev, "usb20");
+	clk = devm_clk_get(&pdev->dev, "usb20");
 	if (IS_ERR(clk)) {
 		dev_err(&pdev->dev, "failed to get clock\n");
-		ret = PTR_ERR(clk);
-		goto err3;
+		return PTR_ERR(clk);
 	}
 
-	ret = clk_enable(clk);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to enable clock\n");
-		goto err4;
+	glue->phy = devm_phy_get(&pdev->dev, "usb-phy");
+	if (IS_ERR(glue->phy)) {
+		dev_err(&pdev->dev, "failed to get phy\n");
+		return PTR_ERR(glue->phy);
 	}
 
 	glue->dev			= &pdev->dev;
@@ -517,10 +488,11 @@ static int da8xx_probe(struct platform_device *pdev)
 
 	pdata->platform_ops		= &da8xx_ops;
 
-	glue->phy = usb_phy_generic_register();
-	if (IS_ERR(glue->phy)) {
-		ret = PTR_ERR(glue->phy);
-		goto err5;
+	glue->usb_phy = usb_phy_generic_register();
+	ret = PTR_ERR_OR_ZERO(glue->usb_phy);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to register usb_phy\n");
+		return ret;
 	}
 	platform_set_drvdata(pdev, glue);
 
@@ -544,28 +516,13 @@ static int da8xx_probe(struct platform_device *pdev)
 	pinfo.data = pdata;
 	pinfo.size_data = sizeof(*pdata);
 
-	glue->musb = musb = platform_device_register_full(&pinfo);
-	if (IS_ERR(musb)) {
-		ret = PTR_ERR(musb);
+	glue->musb = platform_device_register_full(&pinfo);
+	ret = PTR_ERR_OR_ZERO(glue->musb);
+	if (ret) {
 		dev_err(&pdev->dev, "failed to register musb device: %d\n", ret);
-		goto err6;
+		usb_phy_generic_unregister(glue->usb_phy);
 	}
 
-	return 0;
-
-err6:
-	usb_phy_generic_unregister(glue->phy);
-
-err5:
-	clk_disable(clk);
-
-err4:
-	clk_put(clk);
-
-err3:
-	kfree(glue);
-
-err0:
 	return ret;
 }
 
@@ -574,10 +531,7 @@ static int da8xx_remove(struct platform_device *pdev)
 	struct da8xx_glue		*glue = platform_get_drvdata(pdev);
 
 	platform_device_unregister(glue->musb);
-	usb_phy_generic_unregister(glue->phy);
-	clk_disable(glue->clk);
-	clk_put(glue->clk);
-	kfree(glue);
+	usb_phy_generic_unregister(glue->usb_phy);
 
 	return 0;
 }
diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c
index 74fc3069cb42..27dadc0d9114 100644
--- a/drivers/usb/musb/musb_core.c
+++ b/drivers/usb/musb/musb_core.c
@@ -1448,7 +1448,7 @@ static int musb_core_init(u16 musb_type, struct musb *musb)
 {
 	u8 reg;
 	char *type;
-	char aInfo[90], aRevision[32], aDate[12];
+	char aInfo[90];
 	void __iomem	*mbase = musb->mregs;
 	int		status = 0;
 	int		i;
@@ -1482,7 +1482,6 @@ static int musb_core_init(u16 musb_type, struct musb *musb)
 
 	pr_debug("%s: ConfigData=0x%02x (%s)\n", musb_driver_name, reg, aInfo);
 
-	aDate[0] = 0;
 	if (MUSB_CONTROLLER_MHDRC == musb_type) {
 		musb->is_multipoint = 1;
 		type = "M";
@@ -1497,11 +1496,10 @@ static int musb_core_init(u16 musb_type, struct musb *musb)
 
 	/* log release info */
 	musb->hwvers = musb_read_hwvers(mbase);
-	snprintf(aRevision, 32, "%d.%d%s", MUSB_HWVERS_MAJOR(musb->hwvers),
-		MUSB_HWVERS_MINOR(musb->hwvers),
-		(musb->hwvers & MUSB_HWVERS_RC) ? "RC" : "");
-	pr_debug("%s: %sHDRC RTL version %s %s\n",
-		 musb_driver_name, type, aRevision, aDate);
+	pr_debug("%s: %sHDRC RTL version %d.%d%s\n",
+		 musb_driver_name, type, MUSB_HWVERS_MAJOR(musb->hwvers),
+		 MUSB_HWVERS_MINOR(musb->hwvers),
+		 (musb->hwvers & MUSB_HWVERS_RC) ? "RC" : "");
 
 	/* configure ep0 */
 	musb_configure_ep0(musb);
@@ -1831,11 +1829,80 @@ static const struct attribute_group musb_attr_group = {
 	.attrs = musb_attributes,
 };
 
+#define MUSB_QUIRK_B_INVALID_VBUS_91	(MUSB_DEVCTL_BDEVICE | \
+					 (2 << MUSB_DEVCTL_VBUS_SHIFT) | \
+					 MUSB_DEVCTL_SESSION)
+#define MUSB_QUIRK_A_DISCONNECT_19	((3 << MUSB_DEVCTL_VBUS_SHIFT) | \
+					 MUSB_DEVCTL_SESSION)
+
+/*
+ * Check the musb devctl session bit to determine if we want to
+ * allow PM runtime for the device. In general, we want to keep things
+ * active when the session bit is set except after host disconnect.
+ *
+ * Only called from musb_irq_work. If this ever needs to get called
+ * elsewhere, proper locking must be implemented for musb->session.
+ */
+static void musb_pm_runtime_check_session(struct musb *musb)
+{
+	u8 devctl, s;
+	int error;
+
+	devctl = musb_readb(musb->mregs, MUSB_DEVCTL);
+
+	/* Handle session status quirks first */
+	s = MUSB_DEVCTL_FSDEV | MUSB_DEVCTL_LSDEV |
+		MUSB_DEVCTL_HR;
+	switch (devctl & ~s) {
+	case MUSB_QUIRK_B_INVALID_VBUS_91:
+		if (!musb->session && !musb->quirk_invalid_vbus) {
+			musb->quirk_invalid_vbus = true;
+			musb_dbg(musb,
+				 "First invalid vbus, assume no session");
+			return;
+		}
+		break;
+	case MUSB_QUIRK_A_DISCONNECT_19:
+		if (!musb->session)
+			break;
+		musb_dbg(musb, "Allow PM on possible host mode disconnect");
+		pm_runtime_mark_last_busy(musb->controller);
+		pm_runtime_put_autosuspend(musb->controller);
+		musb->session = false;
+		return;
+	default:
+		break;
+	}
+
+	/* No need to do anything if session has not changed */
+	s = devctl & MUSB_DEVCTL_SESSION;
+	if (s == musb->session)
+		return;
+
+	/* Block PM or allow PM? */
+	if (s) {
+		musb_dbg(musb, "Block PM on active session: %02x", devctl);
+		error = pm_runtime_get_sync(musb->controller);
+		if (error < 0)
+			dev_err(musb->controller, "Could not enable: %i\n",
+				error);
+	} else {
+		musb_dbg(musb, "Allow PM with no session: %02x", devctl);
+		musb->quirk_invalid_vbus = false;
+		pm_runtime_mark_last_busy(musb->controller);
+		pm_runtime_put_autosuspend(musb->controller);
+	}
+
+	musb->session = s;
+}
+
 /* Only used to provide driver mode change events */
 static void musb_irq_work(struct work_struct *data)
 {
 	struct musb *musb = container_of(data, struct musb, irq_work);
 
+	musb_pm_runtime_check_session(musb);
+
 	if (musb->xceiv->otg->state != musb->xceiv_old_state) {
 		musb->xceiv_old_state = musb->xceiv->otg->state;
 		sysfs_notify(&musb->controller->kobj, NULL, "mode");
diff --git a/drivers/usb/musb/musb_core.h b/drivers/usb/musb/musb_core.h
index b55a776b03eb..2cb88a498f8a 100644
--- a/drivers/usb/musb/musb_core.h
+++ b/drivers/usb/musb/musb_core.h
@@ -378,6 +378,8 @@ struct musb {
 	u8			min_power;	/* vbus for periph, in mA/2 */
 
 	int			port_mode;	/* MUSB_PORT_MODE_* */
+	bool			session;
+	bool			quirk_invalid_vbus;
 	bool			is_host;
 
 	int			a_wait_bcon;	/* VBUS timeout in msecs */
diff --git a/drivers/usb/musb/musb_dsps.c b/drivers/usb/musb/musb_dsps.c
index 2537179636db..0f17d2140db6 100644
--- a/drivers/usb/musb/musb_dsps.c
+++ b/drivers/usb/musb/musb_dsps.c
@@ -145,43 +145,6 @@ static const struct debugfs_reg32 dsps_musb_regs[] = {
 	{ "mode",		0xe8 },
 };
 
-static void dsps_musb_try_idle(struct musb *musb, unsigned long timeout)
-{
-	struct device *dev = musb->controller;
-	struct dsps_glue *glue = dev_get_drvdata(dev->parent);
-
-	if (timeout == 0)
-		timeout = jiffies + msecs_to_jiffies(3);
-
-	/* Never idle if active, or when VBUS timeout is not set as host */
-	if (musb->is_active || (musb->a_wait_bcon == 0 &&
-			musb->xceiv->otg->state == OTG_STATE_A_WAIT_BCON)) {
-		dev_dbg(musb->controller, "%s active, deleting timer\n",
-				usb_otg_state_string(musb->xceiv->otg->state));
-		del_timer(&glue->timer);
-		glue->last_timer = jiffies;
-		return;
-	}
-	if (musb->port_mode != MUSB_PORT_MODE_DUAL_ROLE)
-		return;
-
-	if (!musb->g.dev.driver)
-		return;
-
-	if (time_after(glue->last_timer, timeout) &&
-				timer_pending(&glue->timer)) {
-		dev_dbg(musb->controller,
-			"Longer idle timer already pending, ignoring...\n");
-		return;
-	}
-	glue->last_timer = timeout;
-
-	dev_dbg(musb->controller, "%s inactive, starting idle timer for %u ms\n",
-		usb_otg_state_string(musb->xceiv->otg->state),
-			jiffies_to_msecs(timeout - jiffies));
-	mod_timer(&glue->timer, timeout);
-}
-
 /**
  * dsps_musb_enable - enable interrupts
  */
@@ -206,7 +169,6 @@ static void dsps_musb_enable(struct musb *musb)
 			musb->port_mode == MUSB_PORT_MODE_DUAL_ROLE)
 		mod_timer(&glue->timer, jiffies +
 				msecs_to_jiffies(wrp->poll_timeout));
-	dsps_musb_try_idle(musb, 0);
 }
 
 /**
@@ -236,6 +198,11 @@ static void otg_timer(unsigned long _musb)
 	u8 devctl;
 	unsigned long flags;
 	int skip_session = 0;
+	int err;
+
+	err = pm_runtime_get_sync(dev);
+	if (err < 0)
+		dev_err(dev, "Poll could not pm_runtime_get: %i\n", err);
 
 	/*
 	 * We poll because DSPS IP's won't expose several OTG-critical
@@ -247,6 +214,10 @@ static void otg_timer(unsigned long _musb)
 
 	spin_lock_irqsave(&musb->lock, flags);
 	switch (musb->xceiv->otg->state) {
+	case OTG_STATE_A_WAIT_VRISE:
+		mod_timer(&glue->timer, jiffies +
+				msecs_to_jiffies(wrp->poll_timeout));
+		break;
 	case OTG_STATE_A_WAIT_BCON:
 		musb_writeb(musb->mregs, MUSB_DEVCTL, 0);
 		skip_session = 1;
@@ -275,6 +246,9 @@ static void otg_timer(unsigned long _musb)
 		break;
 	}
 	spin_unlock_irqrestore(&musb->lock, flags);
+
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
 }
 
 static irqreturn_t dsps_interrupt(int irq, void *hci)
@@ -338,7 +312,8 @@ static irqreturn_t dsps_interrupt(int irq, void *hci)
 			MUSB_HST_MODE(musb);
 			musb->xceiv->otg->default_a = 1;
 			musb->xceiv->otg->state = OTG_STATE_A_WAIT_VRISE;
-			del_timer(&glue->timer);
+			mod_timer(&glue->timer, jiffies +
+				  msecs_to_jiffies(wrp->poll_timeout));
 		} else {
 			musb->is_active = 0;
 			MUSB_DEV_MODE(musb);
@@ -358,11 +333,17 @@ static irqreturn_t dsps_interrupt(int irq, void *hci)
 	if (musb->int_tx || musb->int_rx || musb->int_usb)
 		ret |= musb_interrupt(musb);
 
-	/* Poll for ID change in OTG port mode */
-	if (musb->xceiv->otg->state == OTG_STATE_B_IDLE &&
-			musb->port_mode == MUSB_PORT_MODE_DUAL_ROLE)
+	/* Poll for ID change and connect */
+	switch (musb->xceiv->otg->state) {
+	case OTG_STATE_B_IDLE:
+	case OTG_STATE_A_WAIT_BCON:
 		mod_timer(&glue->timer, jiffies +
 				msecs_to_jiffies(wrp->poll_timeout));
+		break;
+	default:
+		break;
+	}
+
 out:
 	spin_unlock_irqrestore(&musb->lock, flags);
 
@@ -461,6 +442,9 @@ static int dsps_musb_init(struct musb *musb)
 		musb_writeb(musb->mregs, MUSB_BABBLE_CTL, val);
 	}
 
+	mod_timer(&glue->timer, jiffies +
+		  msecs_to_jiffies(glue->wrp->poll_timeout));
+
 	return dsps_musb_dbg_init(musb, glue);
 }
 
@@ -620,7 +604,6 @@ static struct musb_platform_ops dsps_ops = {
 	.enable		= dsps_musb_enable,
 	.disable	= dsps_musb_disable,
 
-	.try_idle	= dsps_musb_try_idle,
 	.set_mode	= dsps_musb_set_mode,
 	.recover	= dsps_musb_recover,
 };
@@ -784,6 +767,8 @@ static int dsps_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, glue);
 	pm_runtime_enable(&pdev->dev);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 200);
 
 	ret = pm_runtime_get_sync(&pdev->dev);
 	if (ret < 0) {
@@ -795,11 +780,15 @@ static int dsps_probe(struct platform_device *pdev)
 	if (ret)
 		goto err3;
 
+	pm_runtime_mark_last_busy(&pdev->dev);
+	pm_runtime_put_autosuspend(&pdev->dev);
+
 	return 0;
 
 err3:
-	pm_runtime_put(&pdev->dev);
+	pm_runtime_put_sync(&pdev->dev);
 err2:
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 	return ret;
 }
@@ -811,7 +800,8 @@ static int dsps_remove(struct platform_device *pdev)
 	platform_device_unregister(glue->musb);
 
 	/* disable usbss clocks */
-	pm_runtime_put(&pdev->dev);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
+	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
 	return 0;
diff --git a/drivers/usb/musb/musb_gadget.c b/drivers/usb/musb/musb_gadget.c
index 6d1e975e9605..bff4869a57cd 100644
--- a/drivers/usb/musb/musb_gadget.c
+++ b/drivers/usb/musb/musb_gadget.c
@@ -1964,6 +1964,9 @@ static int musb_gadget_stop(struct usb_gadget *g)
 	 * that currently misbehaves.
 	 */
 
+	/* Force check of devctl register for PM runtime */
+	schedule_work(&musb->irq_work);
+
 	pm_runtime_mark_last_busy(musb->controller);
 	pm_runtime_put_autosuspend(musb->controller);
 
diff --git a/drivers/usb/musb/musb_virthub.c b/drivers/usb/musb/musb_virthub.c
index 192248f974ec..61b5f1c3c5bc 100644
--- a/drivers/usb/musb/musb_virthub.c
+++ b/drivers/usb/musb/musb_virthub.c
@@ -245,6 +245,7 @@ void musb_root_disconnect(struct musb *musb)
 			usb_otg_state_string(musb->xceiv->otg->state));
 	}
 }
+EXPORT_SYMBOL_GPL(musb_root_disconnect);
 
 
 /*---------------------------------------------------------------------*/
@@ -290,6 +291,7 @@ int musb_hub_control(
 	u32		temp;
 	int		retval = 0;
 	unsigned long	flags;
+	bool		start_musb = false;
 
 	spin_lock_irqsave(&musb->lock, flags);
 
@@ -390,7 +392,7 @@ int musb_hub_control(
 			 * logic relating to VBUS power-up.
 			 */
 			if (!hcd->self.is_b_host && musb_has_gadget(musb))
-				musb_start(musb);
+				start_musb = true;
 			break;
 		case USB_PORT_FEAT_RESET:
 			musb_port_reset(musb, true);
@@ -451,5 +453,9 @@ error:
 		retval = -EPIPE;
 	}
 	spin_unlock_irqrestore(&musb->lock, flags);
+
+	if (start_musb)
+		musb_start(musb);
+
 	return retval;
 }
diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c
index 0b4cec940386..1ab6973d4f61 100644
--- a/drivers/usb/musb/omap2430.c
+++ b/drivers/usb/musb/omap2430.c
@@ -49,9 +49,6 @@ struct omap2430_glue {
 	enum musb_vbus_id_status status;
 	struct work_struct	omap_musb_mailbox_work;
 	struct device		*control_otghs;
-	bool			cable_connected;
-	bool			enabled;
-	bool			powered;
 };
 #define glue_to_musb(g)		platform_get_drvdata(g->musb)
 
@@ -141,45 +138,6 @@ static inline void omap2430_low_level_init(struct musb *musb)
 	musb_writel(musb->mregs, OTG_FORCESTDBY, l);
 }
 
-/*
- * We can get multiple cable events so we need to keep track
- * of the power state. Only keep power enabled if USB cable is
- * connected and a gadget is started.
- */
-static void omap2430_set_power(struct musb *musb, bool enabled, bool cable)
-{
-	struct device *dev = musb->controller;
-	struct omap2430_glue *glue = dev_get_drvdata(dev->parent);
-	bool power_up;
-	int res;
-
-	if (glue->enabled != enabled)
-		glue->enabled = enabled;
-
-	if (glue->cable_connected != cable)
-		glue->cable_connected = cable;
-
-	power_up = glue->enabled && glue->cable_connected;
-	if (power_up == glue->powered) {
-		dev_warn(musb->controller, "power state already %i\n",
-			 power_up);
-		return;
-	}
-
-	glue->powered = power_up;
-
-	if (power_up) {
-		res = pm_runtime_get_sync(musb->controller);
-		if (res < 0) {
-			dev_err(musb->controller, "could not enable: %i", res);
-			glue->powered = false;
-		}
-	} else {
-		pm_runtime_mark_last_busy(musb->controller);
-		pm_runtime_put_autosuspend(musb->controller);
-	}
-}
-
 static int omap2430_musb_mailbox(enum musb_vbus_id_status status)
 {
 	struct omap2430_glue	*glue = _glue;
@@ -203,21 +161,15 @@ static int omap2430_musb_mailbox(enum musb_vbus_id_status status)
 static void omap_musb_set_mailbox(struct omap2430_glue *glue)
 {
 	struct musb *musb = glue_to_musb(glue);
-	struct device *dev = musb->controller;
-	struct musb_hdrc_platform_data *pdata = dev_get_platdata(dev);
+	struct musb_hdrc_platform_data *pdata =
+		dev_get_platdata(musb->controller);
 	struct omap_musb_board_data *data = pdata->board_data;
 	struct usb_otg *otg = musb->xceiv->otg;
-	bool cable_connected;
-
-	cable_connected = ((glue->status == MUSB_ID_GROUND) ||
-			   (glue->status == MUSB_VBUS_VALID));
-
-	if (cable_connected)
-		omap2430_set_power(musb, glue->enabled, cable_connected);
 
+	pm_runtime_get_sync(musb->controller);
 	switch (glue->status) {
 	case MUSB_ID_GROUND:
-		dev_dbg(dev, "ID GND\n");
+		dev_dbg(musb->controller, "ID GND\n");
 
 		otg->default_a = true;
 		musb->xceiv->otg->state = OTG_STATE_A_IDLE;
@@ -230,7 +182,7 @@ static void omap_musb_set_mailbox(struct omap2430_glue *glue)
 		break;
 
 	case MUSB_VBUS_VALID:
-		dev_dbg(dev, "VBUS Connect\n");
+		dev_dbg(musb->controller, "VBUS Connect\n");
 
 		otg->default_a = false;
 		musb->xceiv->otg->state = OTG_STATE_B_IDLE;
@@ -240,7 +192,7 @@ static void omap_musb_set_mailbox(struct omap2430_glue *glue)
 
 	case MUSB_ID_FLOAT:
 	case MUSB_VBUS_OFF:
-		dev_dbg(dev, "VBUS Disconnect\n");
+		dev_dbg(musb->controller, "VBUS Disconnect\n");
 
 		musb->xceiv->last_event = USB_EVENT_NONE;
 		if (musb->gadget_driver)
@@ -253,12 +205,10 @@ static void omap_musb_set_mailbox(struct omap2430_glue *glue)
 			USB_MODE_DISCONNECT);
 		break;
 	default:
-		dev_dbg(dev, "ID float\n");
+		dev_dbg(musb->controller, "ID float\n");
 	}
-
-	if (!cable_connected)
-		omap2430_set_power(musb, glue->enabled, cable_connected);
-
+	pm_runtime_mark_last_busy(musb->controller);
+	pm_runtime_put_autosuspend(musb->controller);
 	atomic_notifier_call_chain(&musb->xceiv->notifier,
 			musb->xceiv->last_event, NULL);
 }
@@ -376,8 +326,6 @@ static void omap2430_musb_enable(struct musb *musb)
 	if (!WARN_ON(!musb->phy))
 		phy_power_on(musb->phy);
 
-	omap2430_set_power(musb, true, glue->cable_connected);
-
 	switch (glue->status) {
 
 	case MUSB_ID_GROUND:
@@ -419,8 +367,6 @@ static void omap2430_musb_disable(struct musb *musb)
 	if (glue->status != MUSB_UNKNOWN)
 		omap_control_usb_set_mode(glue->control_otghs,
 			USB_MODE_DISCONNECT);
-
-	omap2430_set_power(musb, false, glue->cable_connected);
 }
 
 static int omap2430_musb_exit(struct musb *musb)
@@ -571,7 +517,7 @@ static int omap2430_probe(struct platform_device *pdev)
 
 	pm_runtime_enable(glue->dev);
 	pm_runtime_use_autosuspend(glue->dev);
-	pm_runtime_set_autosuspend_delay(glue->dev, 500);
+	pm_runtime_set_autosuspend_delay(glue->dev, 100);
 
 	ret = platform_device_add(musb);
 	if (ret) {
@@ -591,11 +537,9 @@ err0:
 static int omap2430_remove(struct platform_device *pdev)
 {
 	struct omap2430_glue *glue = platform_get_drvdata(pdev);
-	struct musb *musb = glue_to_musb(glue);
 
 	pm_runtime_get_sync(glue->dev);
 	platform_device_unregister(glue->musb);
-	omap2430_set_power(musb, false, false);
 	pm_runtime_put_sync(glue->dev);
 	pm_runtime_dont_use_autosuspend(glue->dev);
 	pm_runtime_disable(glue->dev);
diff --git a/drivers/usb/musb/sunxi.c b/drivers/usb/musb/sunxi.c
index c6ee16660572..1408245be18e 100644
--- a/drivers/usb/musb/sunxi.c
+++ b/drivers/usb/musb/sunxi.c
@@ -74,6 +74,7 @@
 #define SUNXI_MUSB_FL_HAS_SRAM			5
 #define SUNXI_MUSB_FL_HAS_RESET			6
 #define SUNXI_MUSB_FL_NO_CONFIGDATA		7
+#define SUNXI_MUSB_FL_PHY_MODE_PEND		8
 
 /* Our read/write methods need access and do not get passed in a musb ref :| */
 static struct musb *sunxi_musb;
@@ -87,6 +88,7 @@ struct sunxi_glue {
 	struct phy		*phy;
 	struct platform_device	*usb_phy;
 	struct usb_phy		*xceiv;
+	enum phy_mode		phy_mode;
 	unsigned long		flags;
 	struct work_struct	work;
 	struct extcon_dev	*extcon;
@@ -140,6 +142,9 @@ static void sunxi_musb_work(struct work_struct *work)
 			clear_bit(SUNXI_MUSB_FL_PHY_ON, &glue->flags);
 		}
 	}
+
+	if (test_and_clear_bit(SUNXI_MUSB_FL_PHY_MODE_PEND, &glue->flags))
+		phy_set_mode(glue->phy, glue->phy_mode);
 }
 
 static void sunxi_musb_set_vbus(struct musb *musb, int is_on)
@@ -341,6 +346,50 @@ static void sunxi_musb_dma_controller_destroy(struct dma_controller *c)
 {
 }
 
+static int sunxi_musb_set_mode(struct musb *musb, u8 mode)
+{
+	struct sunxi_glue *glue = dev_get_drvdata(musb->controller->parent);
+	enum phy_mode new_mode;
+
+	switch (mode) {
+	case MUSB_HOST:
+		new_mode = PHY_MODE_USB_HOST;
+		break;
+	case MUSB_PERIPHERAL:
+		new_mode = PHY_MODE_USB_DEVICE;
+		break;
+	case MUSB_OTG:
+		new_mode = PHY_MODE_USB_OTG;
+		break;
+	default:
+		dev_err(musb->controller->parent,
+			"Error requested mode not supported by this kernel\n");
+		return -EINVAL;
+	}
+
+	if (glue->phy_mode == new_mode)
+		return 0;
+
+	if (musb->port_mode != MUSB_PORT_MODE_DUAL_ROLE) {
+		dev_err(musb->controller->parent,
+			"Error changing modes is only supported in dual role mode\n");
+		return -EINVAL;
+	}
+
+	if (musb->port1_status & USB_PORT_STAT_ENABLE)
+		musb_root_disconnect(musb);
+
+	/*
+	 * phy_set_mode may sleep, and we're called with a spinlock held,
+	 * so let sunxi_musb_work deal with it.
+	 */
+	glue->phy_mode = new_mode;
+	set_bit(SUNXI_MUSB_FL_PHY_MODE_PEND, &glue->flags);
+	schedule_work(&glue->work);
+
+	return 0;
+}
+
 /*
  * sunxi musb register layout
  * 0x00 - 0x17	fifo regs, 1 long per fifo
@@ -568,6 +617,7 @@ static const struct musb_platform_ops sunxi_musb_ops = {
 	.writew		= sunxi_musb_writew,
 	.dma_init	= sunxi_musb_dma_controller_create,
 	.dma_exit	= sunxi_musb_dma_controller_destroy,
+	.set_mode	= sunxi_musb_set_mode,
 	.set_vbus	= sunxi_musb_set_vbus,
 	.pre_root_reset_end = sunxi_musb_pre_root_reset_end,
 	.post_root_reset_end = sunxi_musb_post_root_reset_end,
@@ -614,21 +664,28 @@ static int sunxi_musb_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+	glue = devm_kzalloc(&pdev->dev, sizeof(*glue), GFP_KERNEL);
+	if (!glue)
+		return -ENOMEM;
+
 	memset(&pdata, 0, sizeof(pdata));
 	switch (usb_get_dr_mode(&pdev->dev)) {
 #if defined CONFIG_USB_MUSB_DUAL_ROLE || defined CONFIG_USB_MUSB_HOST
 	case USB_DR_MODE_HOST:
 		pdata.mode = MUSB_PORT_MODE_HOST;
+		glue->phy_mode = PHY_MODE_USB_HOST;
 		break;
 #endif
 #if defined CONFIG_USB_MUSB_DUAL_ROLE || defined CONFIG_USB_MUSB_GADGET
 	case USB_DR_MODE_PERIPHERAL:
 		pdata.mode = MUSB_PORT_MODE_GADGET;
+		glue->phy_mode = PHY_MODE_USB_DEVICE;
 		break;
 #endif
 #ifdef CONFIG_USB_MUSB_DUAL_ROLE
 	case USB_DR_MODE_OTG:
 		pdata.mode = MUSB_PORT_MODE_DUAL_ROLE;
+		glue->phy_mode = PHY_MODE_USB_OTG;
 		break;
 #endif
 	default:
@@ -638,10 +695,6 @@ static int sunxi_musb_probe(struct platform_device *pdev)
 	pdata.platform_ops	= &sunxi_musb_ops;
 	pdata.config		= &sunxi_musb_hdrc_config;
 
-	glue = devm_kzalloc(&pdev->dev, sizeof(*glue), GFP_KERNEL);
-	if (!glue)
-		return -ENOMEM;
-
 	glue->dev = &pdev->dev;
 	INIT_WORK(&glue->work, sunxi_musb_work);
 	glue->host_nb.notifier_call = sunxi_musb_host_notifier;
diff --git a/drivers/usb/phy/phy-ab8500-usb.c b/drivers/usb/phy/phy-ab8500-usb.c
index 0c912d3950a5..a03caf4b1327 100644
--- a/drivers/usb/phy/phy-ab8500-usb.c
+++ b/drivers/usb/phy/phy-ab8500-usb.c
@@ -1248,7 +1248,7 @@ static void ab8500_usb_set_ab8500_tuning_values(struct ab8500_usb *ab)
 	err = abx500_set_register_interruptible(ab->dev,
 			AB8500_DEBUG, AB8500_USB_PHY_TUNE3, 0x78);
 	if (err < 0)
-		dev_err(ab->dev, "Failed to set PHY_TUNE3 regester err=%d\n",
+		dev_err(ab->dev, "Failed to set PHY_TUNE3 register err=%d\n",
 				err);
 
 	/* Switch to normal mode/disable Bank 0x12 access */
@@ -1290,7 +1290,7 @@ static void ab8500_usb_set_ab8505_tuning_values(struct ab8500_usb *ab)
 			0xFC, 0x80);
 
 	if (err < 0)
-		dev_err(ab->dev, "Failed to set PHY_TUNE3 regester err=%d\n",
+		dev_err(ab->dev, "Failed to set PHY_TUNE3 register err=%d\n",
 				err);
 
 	/* Switch to normal mode/disable Bank 0x12 access */
@@ -1321,7 +1321,7 @@ static void ab8500_usb_set_ab8540_tuning_values(struct ab8500_usb *ab)
 	err = abx500_set_register_interruptible(ab->dev,
 			AB8540_DEBUG, AB8500_USB_PHY_TUNE3, 0x90);
 	if (err < 0)
-		dev_err(ab->dev, "Failed to set PHY_TUNE3 regester ret=%d\n",
+		dev_err(ab->dev, "Failed to set PHY_TUNE3 register ret=%d\n",
 				err);
 }
 
@@ -1351,7 +1351,7 @@ static void ab8500_usb_set_ab9540_tuning_values(struct ab8500_usb *ab)
 	err = abx500_set_register_interruptible(ab->dev,
 			AB8500_DEBUG, AB8500_USB_PHY_TUNE3, 0x80);
 	if (err < 0)
-		dev_err(ab->dev, "Failed to set PHY_TUNE3 regester err=%d\n",
+		dev_err(ab->dev, "Failed to set PHY_TUNE3 register err=%d\n",
 				err);
 
 	/* Switch to normal mode/disable Bank 0x12 access */
diff --git a/drivers/usb/phy/phy-generic.c b/drivers/usb/phy/phy-generic.c
index 980c9dee09eb..8311ba2968cd 100644
--- a/drivers/usb/phy/phy-generic.c
+++ b/drivers/usb/phy/phy-generic.c
@@ -118,8 +118,6 @@ static irqreturn_t nop_gpio_vbus_thread(int irq, void *data)
 		status = USB_EVENT_VBUS;
 		otg->state = OTG_STATE_B_PERIPHERAL;
 		nop->phy.last_event = status;
-		if (otg->gadget)
-			usb_gadget_vbus_connect(otg->gadget);
 
 		/* drawing a "unit load" is *always* OK, except for OTG */
 		nop_set_vbus_draw(nop, 100);
@@ -129,8 +127,6 @@ static irqreturn_t nop_gpio_vbus_thread(int irq, void *data)
 	} else {
 		nop_set_vbus_draw(nop, 0);
 
-		if (otg->gadget)
-			usb_gadget_vbus_disconnect(otg->gadget);
 		status = USB_EVENT_NONE;
 		otg->state = OTG_STATE_B_IDLE;
 		nop->phy.last_event = status;
@@ -144,14 +140,18 @@ static irqreturn_t nop_gpio_vbus_thread(int irq, void *data)
 int usb_gen_phy_init(struct usb_phy *phy)
 {
 	struct usb_phy_generic *nop = dev_get_drvdata(phy->dev);
+	int ret;
 
 	if (!IS_ERR(nop->vcc)) {
 		if (regulator_enable(nop->vcc))
 			dev_err(phy->dev, "Failed to enable power\n");
 	}
 
-	if (!IS_ERR(nop->clk))
-		clk_prepare_enable(nop->clk);
+	if (!IS_ERR(nop->clk)) {
+		ret = clk_prepare_enable(nop->clk);
+		if (ret)
+			return ret;
+	}
 
 	nop_reset(nop);
 
@@ -187,7 +187,8 @@ static int nop_set_peripheral(struct usb_otg *otg, struct usb_gadget *gadget)
 
 	otg->gadget = gadget;
 	if (otg->state == OTG_STATE_B_PERIPHERAL)
-		usb_gadget_vbus_connect(gadget);
+		atomic_notifier_call_chain(&otg->usb_phy->notifier,
+					   USB_EVENT_VBUS, otg->gadget);
 	else
 		otg->state = OTG_STATE_B_IDLE;
 	return 0;
@@ -322,6 +323,8 @@ static int usb_phy_generic_probe(struct platform_device *pdev)
 				gpiod_to_irq(nop->gpiod_vbus), err);
 			return err;
 		}
+		nop->phy.otg->state = gpiod_get_value(nop->gpiod_vbus) ?
+			OTG_STATE_B_PERIPHERAL : OTG_STATE_B_IDLE;
 	}
 
 	nop->phy.init		= usb_gen_phy_init;
diff --git a/drivers/usb/phy/phy-mxs-usb.c b/drivers/usb/phy/phy-mxs-usb.c
index 00bfea01be65..0e2f1a36d315 100644
--- a/drivers/usb/phy/phy-mxs-usb.c
+++ b/drivers/usb/phy/phy-mxs-usb.c
@@ -27,6 +27,7 @@
 #define DRIVER_NAME "mxs_phy"
 
 #define HW_USBPHY_PWD				0x00
+#define HW_USBPHY_TX				0x10
 #define HW_USBPHY_CTRL				0x30
 #define HW_USBPHY_CTRL_SET			0x34
 #define HW_USBPHY_CTRL_CLR			0x38
@@ -38,6 +39,10 @@
 #define HW_USBPHY_IP_SET			0x94
 #define HW_USBPHY_IP_CLR			0x98
 
+#define GM_USBPHY_TX_TXCAL45DP(x)            (((x) & 0xf) << 16)
+#define GM_USBPHY_TX_TXCAL45DN(x)            (((x) & 0xf) << 8)
+#define GM_USBPHY_TX_D_CAL(x)                (((x) & 0xf) << 0)
+
 #define BM_USBPHY_CTRL_SFTRST			BIT(31)
 #define BM_USBPHY_CTRL_CLKGATE			BIT(30)
 #define BM_USBPHY_CTRL_OTG_ID_VALUE		BIT(27)
@@ -115,6 +120,12 @@
  */
 #define MXS_PHY_NEED_IP_FIX			BIT(3)
 
+/* Minimum and maximum values for device tree entries */
+#define MXS_PHY_TX_CAL45_MIN			30
+#define MXS_PHY_TX_CAL45_MAX			55
+#define MXS_PHY_TX_D_CAL_MIN			79
+#define MXS_PHY_TX_D_CAL_MAX			119
+
 struct mxs_phy_data {
 	unsigned int flags;
 };
@@ -164,6 +175,8 @@ struct mxs_phy {
 	const struct mxs_phy_data *data;
 	struct regmap *regmap_anatop;
 	int port_id;
+	u32 tx_reg_set;
+	u32 tx_reg_mask;
 };
 
 static inline bool is_imx6q_phy(struct mxs_phy *mxs_phy)
@@ -185,6 +198,20 @@ static void mxs_phy_clock_switch_delay(void)
 	usleep_range(300, 400);
 }
 
+static void mxs_phy_tx_init(struct mxs_phy *mxs_phy)
+{
+	void __iomem *base = mxs_phy->phy.io_priv;
+	u32 phytx;
+
+	/* Update TX register if there is anything to write */
+	if (mxs_phy->tx_reg_mask) {
+		phytx = readl(base + HW_USBPHY_TX);
+		phytx &= ~mxs_phy->tx_reg_mask;
+		phytx |= mxs_phy->tx_reg_set;
+		writel(phytx, base + HW_USBPHY_TX);
+	}
+}
+
 static int mxs_phy_hw_init(struct mxs_phy *mxs_phy)
 {
 	int ret;
@@ -214,6 +241,8 @@ static int mxs_phy_hw_init(struct mxs_phy *mxs_phy)
 	if (mxs_phy->data->flags & MXS_PHY_NEED_IP_FIX)
 		writel(BM_USBPHY_IP_FIX, base + HW_USBPHY_IP_SET);
 
+	mxs_phy_tx_init(mxs_phy);
+
 	return 0;
 }
 
@@ -459,6 +488,7 @@ static int mxs_phy_probe(struct platform_device *pdev)
 	int ret;
 	const struct of_device_id *of_id;
 	struct device_node *np = pdev->dev.of_node;
+	u32 val;
 
 	of_id = of_match_device(mxs_phy_dt_ids, &pdev->dev);
 	if (!of_id)
@@ -491,6 +521,37 @@ static int mxs_phy_probe(struct platform_device *pdev)
 		}
 	}
 
+	/* Precompute which bits of the TX register are to be updated, if any */
+	if (!of_property_read_u32(np, "fsl,tx-cal-45-dn-ohms", &val) &&
+	    val >= MXS_PHY_TX_CAL45_MIN && val <= MXS_PHY_TX_CAL45_MAX) {
+		/* Scale to a 4-bit value */
+		val = (MXS_PHY_TX_CAL45_MAX - val) * 0xF
+			/ (MXS_PHY_TX_CAL45_MAX - MXS_PHY_TX_CAL45_MIN);
+		mxs_phy->tx_reg_mask |= GM_USBPHY_TX_TXCAL45DN(~0);
+		mxs_phy->tx_reg_set  |= GM_USBPHY_TX_TXCAL45DN(val);
+	}
+
+	if (!of_property_read_u32(np, "fsl,tx-cal-45-dp-ohms", &val) &&
+	    val >= MXS_PHY_TX_CAL45_MIN && val <= MXS_PHY_TX_CAL45_MAX) {
+		/* Scale to a 4-bit value. */
+		val = (MXS_PHY_TX_CAL45_MAX - val) * 0xF
+			/ (MXS_PHY_TX_CAL45_MAX - MXS_PHY_TX_CAL45_MIN);
+		mxs_phy->tx_reg_mask |= GM_USBPHY_TX_TXCAL45DP(~0);
+		mxs_phy->tx_reg_set  |= GM_USBPHY_TX_TXCAL45DP(val);
+	}
+
+	if (!of_property_read_u32(np, "fsl,tx-d-cal", &val) &&
+	    val >= MXS_PHY_TX_D_CAL_MIN && val <= MXS_PHY_TX_D_CAL_MAX) {
+		/* Scale to a 4-bit value.  Round up the values and heavily
+		 * weight the rounding by adding 2/3 of the denominator.
+		 */
+		val = ((MXS_PHY_TX_D_CAL_MAX - val) * 0xF
+			+ (MXS_PHY_TX_D_CAL_MAX - MXS_PHY_TX_D_CAL_MIN) * 2/3)
+			/ (MXS_PHY_TX_D_CAL_MAX - MXS_PHY_TX_D_CAL_MIN);
+		mxs_phy->tx_reg_mask |= GM_USBPHY_TX_D_CAL(~0);
+		mxs_phy->tx_reg_set  |= GM_USBPHY_TX_D_CAL(val);
+	}
+
 	ret = of_alias_get_id(np, "usbphy");
 	if (ret < 0)
 		dev_dbg(&pdev->dev, "failed to get alias id, errno %d\n", ret);
diff --git a/drivers/usb/phy/phy-omap-otg.c b/drivers/usb/phy/phy-omap-otg.c
index 6f6d2a7fd5a0..6523af4f8f93 100644
--- a/drivers/usb/phy/phy-omap-otg.c
+++ b/drivers/usb/phy/phy-omap-otg.c
@@ -140,6 +140,8 @@ static int omap_otg_probe(struct platform_device *pdev)
 		 (rev >> 4) & 0xf, rev & 0xf, config->extcon, otg_dev->id,
 		 otg_dev->vbus);
 
+	platform_set_drvdata(pdev, otg_dev);
+
 	return 0;
 }
 
diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c
index 8fbbc2d32371..012a37aa3e0d 100644
--- a/drivers/usb/renesas_usbhs/common.c
+++ b/drivers/usb/renesas_usbhs/common.c
@@ -482,6 +482,10 @@ static const struct of_device_id usbhs_of_match[] = {
 		.data = (void *)USBHS_TYPE_RCAR_GEN3,
 	},
 	{
+		.compatible = "renesas,usbhs-r8a7796",
+		.data = (void *)USBHS_TYPE_RCAR_GEN3,
+	},
+	{
 		.compatible = "renesas,rcar-gen2-usbhs",
 		.data = (void *)USBHS_TYPE_RCAR_GEN2,
 	},
@@ -514,7 +518,8 @@ static struct renesas_usbhs_platform_info *usbhs_parse_dt(struct device *dev)
 	if (gpio > 0)
 		dparam->enable_gpio = gpio;
 
-	if (dparam->type == USBHS_TYPE_RCAR_GEN2)
+	if (dparam->type == USBHS_TYPE_RCAR_GEN2 ||
+	    dparam->type == USBHS_TYPE_RCAR_GEN3)
 		dparam->has_usb_dmac = 1;
 
 	return info;
diff --git a/drivers/usb/renesas_usbhs/fifo.c b/drivers/usb/renesas_usbhs/fifo.c
index 280ed5ff021b..857e78337324 100644
--- a/drivers/usb/renesas_usbhs/fifo.c
+++ b/drivers/usb/renesas_usbhs/fifo.c
@@ -871,7 +871,7 @@ static int usbhsf_dma_prepare_push(struct usbhs_pkt *pkt, int *is_done)
 
 	/* use PIO if packet is less than pio_dma_border or pipe is DCP */
 	if ((len < usbhs_get_dparam(priv, pio_dma_border)) ||
-	    usbhs_pipe_is_dcp(pipe))
+	    usbhs_pipe_type_is(pipe, USB_ENDPOINT_XFER_ISOC))
 		goto usbhsf_pio_prepare_push;
 
 	/* check data length if this driver don't use USB-DMAC */
@@ -976,7 +976,7 @@ static int usbhsf_dma_prepare_pop_with_usb_dmac(struct usbhs_pkt *pkt,
 
 	/* use PIO if packet is less than pio_dma_border or pipe is DCP */
 	if ((pkt->length < usbhs_get_dparam(priv, pio_dma_border)) ||
-	    usbhs_pipe_is_dcp(pipe))
+	    usbhs_pipe_type_is(pipe, USB_ENDPOINT_XFER_ISOC))
 		goto usbhsf_pio_prepare_pop;
 
 	fifo = usbhsf_get_dma_fifo(priv, pkt);
diff --git a/drivers/usb/renesas_usbhs/mod.c b/drivers/usb/renesas_usbhs/mod.c
index d4be5d594896..28965ef4f824 100644
--- a/drivers/usb/renesas_usbhs/mod.c
+++ b/drivers/usb/renesas_usbhs/mod.c
@@ -282,9 +282,16 @@ static irqreturn_t usbhs_interrupt(int irq, void *data)
 	if (usbhs_mod_is_host(priv))
 		usbhs_write(priv, INTSTS1, ~irq_state.intsts1 & INTSTS1_MAGIC);
 
-	usbhs_write(priv, BRDYSTS, ~irq_state.brdysts);
+	/*
+	 * The driver should not clear the xxxSTS after the line of
+	 * "call irq callback functions" because each "if" statement is
+	 * possible to call the callback function for avoiding any side effects.
+	 */
+	if (irq_state.intsts0 & BRDY)
+		usbhs_write(priv, BRDYSTS, ~irq_state.brdysts);
 	usbhs_write(priv, NRDYSTS, ~irq_state.nrdysts);
-	usbhs_write(priv, BEMPSTS, ~irq_state.bempsts);
+	if (irq_state.intsts0 & BEMP)
+		usbhs_write(priv, BEMPSTS, ~irq_state.bempsts);
 
 	/*
 	 * call irq callback functions
diff --git a/drivers/usb/renesas_usbhs/mod_gadget.c b/drivers/usb/renesas_usbhs/mod_gadget.c
index 50f3363cc382..5bc7a6138855 100644
--- a/drivers/usb/renesas_usbhs/mod_gadget.c
+++ b/drivers/usb/renesas_usbhs/mod_gadget.c
@@ -335,7 +335,6 @@ static void __usbhsg_recip_send_status(struct usbhsg_gpriv *gpriv,
 	buf = kmalloc(sizeof(*buf), GFP_ATOMIC);
 	if (!buf) {
 		usb_ep_free_request(&dcp->ep, req);
-		dev_err(dev, "recip data allocation fail\n");
 		return;
 	}
 
@@ -617,10 +616,13 @@ static int usbhsg_ep_enable(struct usb_ep *ep,
 		 * use dmaengine if possible.
 		 * It will use pio handler if impossible.
 		 */
-		if (usb_endpoint_dir_in(desc))
+		if (usb_endpoint_dir_in(desc)) {
 			pipe->handler = &usbhs_fifo_dma_push_handler;
-		else
+		} else {
 			pipe->handler = &usbhs_fifo_dma_pop_handler;
+			usbhs_xxxsts_clear(priv, BRDYSTS,
+					   usbhs_pipe_number(pipe));
+		}
 
 		ret = 0;
 	}
@@ -1059,21 +1061,18 @@ int usbhs_mod_gadget_probe(struct usbhs_priv *priv)
 	int ret;
 
 	gpriv = kzalloc(sizeof(struct usbhsg_gpriv), GFP_KERNEL);
-	if (!gpriv) {
-		dev_err(dev, "Could not allocate gadget priv\n");
+	if (!gpriv)
 		return -ENOMEM;
-	}
 
 	uep = kzalloc(sizeof(struct usbhsg_uep) * pipe_size, GFP_KERNEL);
 	if (!uep) {
-		dev_err(dev, "Could not allocate ep\n");
 		ret = -ENOMEM;
 		goto usbhs_mod_gadget_probe_err_gpriv;
 	}
 
 	gpriv->transceiver = usb_get_phy(USB_PHY_TYPE_UNDEFINED);
 	dev_info(dev, "%stransceiver found\n",
-		 gpriv->transceiver ? "" : "no ");
+		 !IS_ERR(gpriv->transceiver) ? "" : "no ");
 
 	/*
 	 * CAUTION
@@ -1103,6 +1102,8 @@ int usbhs_mod_gadget_probe(struct usbhs_priv *priv)
 	gpriv->gadget.name		= "renesas_usbhs_udc";
 	gpriv->gadget.ops		= &usbhsg_gadget_ops;
 	gpriv->gadget.max_speed		= USB_SPEED_HIGH;
+	gpriv->gadget.quirk_avoids_skb_reserve = usbhs_get_dparam(priv,
+								has_usb_dmac);
 
 	INIT_LIST_HEAD(&gpriv->gadget.ep_list);
 
diff --git a/drivers/usb/renesas_usbhs/mod_host.c b/drivers/usb/renesas_usbhs/mod_host.c
index 3bf0b72eb359..165e81bfd93a 100644
--- a/drivers/usb/renesas_usbhs/mod_host.c
+++ b/drivers/usb/renesas_usbhs/mod_host.c
@@ -166,14 +166,10 @@ static struct usbhsh_request *usbhsh_ureq_alloc(struct usbhsh_hpriv *hpriv,
 					       gfp_t mem_flags)
 {
 	struct usbhsh_request *ureq;
-	struct usbhs_priv *priv = usbhsh_hpriv_to_priv(hpriv);
-	struct device *dev = usbhs_priv_to_dev(priv);
 
 	ureq = kzalloc(sizeof(struct usbhsh_request), mem_flags);
-	if (!ureq) {
-		dev_err(dev, "ureq alloc fail\n");
+	if (!ureq)
 		return NULL;
-	}
 
 	usbhs_pkt_init(&ureq->pkt);
 	ureq->urb = urb;
@@ -388,10 +384,8 @@ static int usbhsh_endpoint_attach(struct usbhsh_hpriv *hpriv,
 	unsigned long flags;
 
 	uep = kzalloc(sizeof(struct usbhsh_ep), mem_flags);
-	if (!uep) {
-		dev_err(dev, "usbhsh_ep alloc fail\n");
+	if (!uep)
 		return -ENOMEM;
-	}
 
 	/********************  spin lock ********************/
 	usbhs_lock(priv, flags);
diff --git a/drivers/usb/renesas_usbhs/pipe.c b/drivers/usb/renesas_usbhs/pipe.c
index c238772b9e9e..9396a8c14af8 100644
--- a/drivers/usb/renesas_usbhs/pipe.c
+++ b/drivers/usb/renesas_usbhs/pipe.c
@@ -804,10 +804,8 @@ int usbhs_pipe_probe(struct usbhs_priv *priv)
 	}
 
 	info->pipe = kzalloc(sizeof(struct usbhs_pipe) * pipe_size, GFP_KERNEL);
-	if (!info->pipe) {
-		dev_err(dev, "Could not allocate pipe\n");
+	if (!info->pipe)
 		return -ENOMEM;
-	}
 
 	info->size = pipe_size;
 
diff --git a/drivers/usb/serial/cp210x.c b/drivers/usb/serial/cp210x.c
index 4d6a5c672a3d..54a4de0efdba 100644
--- a/drivers/usb/serial/cp210x.c
+++ b/drivers/usb/serial/cp210x.c
@@ -118,6 +118,7 @@ static const struct usb_device_id id_table[] = {
 	{ USB_DEVICE(0x10C4, 0x8411) }, /* Kyocera GPS Module */
 	{ USB_DEVICE(0x10C4, 0x8418) }, /* IRZ Automation Teleport SG-10 GSM/GPRS Modem */
 	{ USB_DEVICE(0x10C4, 0x846E) }, /* BEI USB Sensor Interface (VCP) */
+	{ USB_DEVICE(0x10C4, 0x8470) }, /* Juniper Networks BX Series System Console */
 	{ USB_DEVICE(0x10C4, 0x8477) }, /* Balluff RFID */
 	{ USB_DEVICE(0x10C4, 0x84B6) }, /* Starizona Hyperion */
 	{ USB_DEVICE(0x10C4, 0x85EA) }, /* AC-Services IBUS-IF */
diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c
index 00820809139a..b2d767e743fc 100644
--- a/drivers/usb/serial/ftdi_sio.c
+++ b/drivers/usb/serial/ftdi_sio.c
@@ -648,6 +648,8 @@ static const struct usb_device_id id_table_combined[] = {
 	{ USB_DEVICE(FTDI_VID, FTDI_ELV_TFD128_PID) },
 	{ USB_DEVICE(FTDI_VID, FTDI_ELV_FM3RX_PID) },
 	{ USB_DEVICE(FTDI_VID, FTDI_ELV_WS777_PID) },
+	{ USB_DEVICE(FTDI_VID, FTDI_PALMSENS_PID) },
+	{ USB_DEVICE(FTDI_VID, FTDI_IVIUM_XSTAT_PID) },
 	{ USB_DEVICE(FTDI_VID, LINX_SDMUSBQSS_PID) },
 	{ USB_DEVICE(FTDI_VID, LINX_MASTERDEVEL2_PID) },
 	{ USB_DEVICE(FTDI_VID, LINX_FUTURE_0_PID) },
@@ -1008,6 +1010,7 @@ static const struct usb_device_id id_table_combined[] = {
 	{ USB_DEVICE(ICPDAS_VID, ICPDAS_I7560U_PID) },
 	{ USB_DEVICE(ICPDAS_VID, ICPDAS_I7561U_PID) },
 	{ USB_DEVICE(ICPDAS_VID, ICPDAS_I7563U_PID) },
+	{ USB_DEVICE(WICED_VID, WICED_USB20706V2_PID) },
 	{ }					/* Terminating entry */
 };
 
diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h
index c5d6c1e73e8e..f87a938cf005 100644
--- a/drivers/usb/serial/ftdi_sio_ids.h
+++ b/drivers/usb/serial/ftdi_sio_ids.h
@@ -406,6 +406,12 @@
 #define FTDI_4N_GALAXY_DE_3_PID	0xF3C2
 
 /*
+ * Ivium Technologies product IDs
+ */
+#define FTDI_PALMSENS_PID	0xf440
+#define FTDI_IVIUM_XSTAT_PID	0xf441
+
+/*
  * Linx Technologies product ids
  */
 #define LINX_SDMUSBQSS_PID	0xF448	/* Linx SDM-USB-QS-S */
@@ -673,6 +679,12 @@
 #define INTREPID_NEOVI_PID	0x0701
 
 /*
+ * WICED USB UART
+ */
+#define WICED_VID		0x0A5C
+#define WICED_USB20706V2_PID	0x6422
+
+/*
  * Definitions for ID TECH (www.idt-net.com) devices
  */
 #define IDTECH_VID		0x0ACD	/* ID TECH Vendor ID */
diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c
index 4f7e072e4e00..e49ad0c63ad8 100644
--- a/drivers/usb/serial/keyspan_pda.c
+++ b/drivers/usb/serial/keyspan_pda.c
@@ -30,12 +30,12 @@
 #include <linux/usb/ezusb.h>
 
 /* make a simple define to handle if we are compiling keyspan_pda or xircom support */
-#if defined(CONFIG_USB_SERIAL_KEYSPAN_PDA) || defined(CONFIG_USB_SERIAL_KEYSPAN_PDA_MODULE)
+#if IS_ENABLED(CONFIG_USB_SERIAL_KEYSPAN_PDA)
 	#define KEYSPAN
 #else
 	#undef KEYSPAN
 #endif
-#if defined(CONFIG_USB_SERIAL_XIRCOM) || defined(CONFIG_USB_SERIAL_XIRCOM_MODULE)
+#if IS_ENABLED(CONFIG_USB_SERIAL_XIRCOM)
 	#define XIRCOM
 #else
 	#undef XIRCOM
diff --git a/drivers/usb/serial/mos7720.c b/drivers/usb/serial/mos7720.c
index 5608af4a369d..de9992b492b0 100644
--- a/drivers/usb/serial/mos7720.c
+++ b/drivers/usb/serial/mos7720.c
@@ -1252,7 +1252,7 @@ static int mos7720_write(struct tty_struct *tty, struct usb_serial_port *port,
 
 	if (urb->transfer_buffer == NULL) {
 		urb->transfer_buffer = kmalloc(URB_TRANSFER_BUFFER_SIZE,
-					       GFP_KERNEL);
+					       GFP_ATOMIC);
 		if (!urb->transfer_buffer)
 			goto exit;
 	}
diff --git a/drivers/usb/serial/mos7840.c b/drivers/usb/serial/mos7840.c
index ed378fb232e7..57426d703a09 100644
--- a/drivers/usb/serial/mos7840.c
+++ b/drivers/usb/serial/mos7840.c
@@ -1340,8 +1340,8 @@ static int mos7840_write(struct tty_struct *tty, struct usb_serial_port *port,
 	}
 
 	if (urb->transfer_buffer == NULL) {
-		urb->transfer_buffer =
-		    kmalloc(URB_TRANSFER_BUFFER_SIZE, GFP_KERNEL);
+		urb->transfer_buffer = kmalloc(URB_TRANSFER_BUFFER_SIZE,
+					       GFP_ATOMIC);
 		if (!urb->transfer_buffer)
 			goto exit;
 	}
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index 8e07536c233a..9894e341c6ac 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -274,6 +274,12 @@ static void option_instat_callback(struct urb *urb);
 #define TELIT_PRODUCT_LE920			0x1200
 #define TELIT_PRODUCT_LE910			0x1201
 #define TELIT_PRODUCT_LE910_USBCFG4		0x1206
+#define TELIT_PRODUCT_LE920A4_1207		0x1207
+#define TELIT_PRODUCT_LE920A4_1208		0x1208
+#define TELIT_PRODUCT_LE920A4_1211		0x1211
+#define TELIT_PRODUCT_LE920A4_1212		0x1212
+#define TELIT_PRODUCT_LE920A4_1213		0x1213
+#define TELIT_PRODUCT_LE920A4_1214		0x1214
 
 /* ZTE PRODUCTS */
 #define ZTE_VENDOR_ID				0x19d2
@@ -519,6 +525,12 @@ static void option_instat_callback(struct urb *urb);
 #define VIATELECOM_VENDOR_ID			0x15eb
 #define VIATELECOM_PRODUCT_CDS7			0x0001
 
+/* WeTelecom products */
+#define WETELECOM_VENDOR_ID			0x22de
+#define WETELECOM_PRODUCT_WMD200		0x6801
+#define WETELECOM_PRODUCT_6802			0x6802
+#define WETELECOM_PRODUCT_WMD300		0x6803
+
 struct option_blacklist_info {
 	/* bitmask of interface numbers blacklisted for send_setup */
 	const unsigned long sendsetup;
@@ -628,6 +640,11 @@ static const struct option_blacklist_info telit_le920_blacklist = {
 	.reserved = BIT(1) | BIT(5),
 };
 
+static const struct option_blacklist_info telit_le920a4_blacklist_1 = {
+	.sendsetup = BIT(0),
+	.reserved = BIT(1),
+};
+
 static const struct option_blacklist_info telit_le922_blacklist_usbcfg0 = {
 	.sendsetup = BIT(2),
 	.reserved = BIT(0) | BIT(1) | BIT(3),
@@ -1203,6 +1220,16 @@ static const struct usb_device_id option_ids[] = {
 		.driver_info = (kernel_ulong_t)&telit_le922_blacklist_usbcfg3 },
 	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE920),
 		.driver_info = (kernel_ulong_t)&telit_le920_blacklist },
+	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE920A4_1207) },
+	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE920A4_1208),
+		.driver_info = (kernel_ulong_t)&telit_le920a4_blacklist_1 },
+	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE920A4_1211),
+		.driver_info = (kernel_ulong_t)&telit_le922_blacklist_usbcfg3 },
+	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE920A4_1212),
+		.driver_info = (kernel_ulong_t)&telit_le920a4_blacklist_1 },
+	{ USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, TELIT_PRODUCT_LE920A4_1213, 0xff) },
+	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE920A4_1214),
+		.driver_info = (kernel_ulong_t)&telit_le922_blacklist_usbcfg3 },
 	{ USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, ZTE_PRODUCT_MF622, 0xff, 0xff, 0xff) }, /* ZTE WCDMA products */
 	{ USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x0002, 0xff, 0xff, 0xff),
 		.driver_info = (kernel_ulong_t)&net_intf1_blacklist },
@@ -1966,9 +1993,13 @@ static const struct usb_device_id option_ids[] = {
 	  .driver_info = (kernel_ulong_t)&net_intf4_blacklist },
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x07d1, 0x3e01, 0xff, 0xff, 0xff) }, /* D-Link DWM-152/C1 */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x07d1, 0x3e02, 0xff, 0xff, 0xff) }, /* D-Link DWM-156/C1 */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x07d1, 0x7e11, 0xff, 0xff, 0xff) }, /* D-Link DWM-156/A3 */
 	{ USB_DEVICE_INTERFACE_CLASS(0x2020, 0x4000, 0xff) },                /* OLICARD300 - MT6225 */
 	{ USB_DEVICE(INOVIA_VENDOR_ID, INOVIA_SEW858) },
 	{ USB_DEVICE(VIATELECOM_VENDOR_ID, VIATELECOM_PRODUCT_CDS7) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(WETELECOM_VENDOR_ID, WETELECOM_PRODUCT_WMD200, 0xff, 0xff, 0xff) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(WETELECOM_VENDOR_ID, WETELECOM_PRODUCT_6802, 0xff, 0xff, 0xff) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(WETELECOM_VENDOR_ID, WETELECOM_PRODUCT_WMD300, 0xff, 0xff, 0xff) },
 	{ } /* Terminating entry */
 };
 MODULE_DEVICE_TABLE(usb, option_ids);
diff --git a/drivers/usb/serial/ti_usb_3410_5052.c b/drivers/usb/serial/ti_usb_3410_5052.c
index 07b4bf01061d..a8b9bdba314f 100644
--- a/drivers/usb/serial/ti_usb_3410_5052.c
+++ b/drivers/usb/serial/ti_usb_3410_5052.c
@@ -179,23 +179,23 @@
 
 /* Config struct */
 struct ti_uart_config {
-	__u16	wBaudRate;
-	__u16	wFlags;
-	__u8	bDataBits;
-	__u8	bParity;
-	__u8	bStopBits;
+	__be16	wBaudRate;
+	__be16	wFlags;
+	u8	bDataBits;
+	u8	bParity;
+	u8	bStopBits;
 	char	cXon;
 	char	cXoff;
-	__u8	bUartMode;
+	u8	bUartMode;
 } __packed;
 
 /* Get port status */
 struct ti_port_status {
-	__u8	bCmdCode;
-	__u8	bModuleId;
-	__u8	bErrorCode;
-	__u8	bMSR;
-	__u8	bLSR;
+	u8 bCmdCode;
+	u8 bModuleId;
+	u8 bErrorCode;
+	u8 bMSR;
+	u8 bLSR;
 } __packed;
 
 /* Purge modes */
@@ -218,12 +218,12 @@ struct ti_port_status {
 #define TI_RW_DATA_DOUBLE_WORD		0x04
 
 struct ti_write_data_bytes {
-	__u8	bAddrType;
-	__u8	bDataType;
-	__u8	bDataCounter;
+	u8	bAddrType;
+	u8	bDataType;
+	u8	bDataCounter;
 	__be16	wBaseAddrHi;
 	__be16	wBaseAddrLo;
-	__u8	bData[0];
+	u8	bData[0];
 } __packed;
 
 struct ti_read_data_request {
@@ -258,7 +258,7 @@ struct ti_interrupt {
 /* Firmware image header */
 struct ti_firmware_header {
 	__le16	wLength;
-	__u8	bCheckSum;
+	u8	bCheckSum;
 } __packed;
 
 /* UART addresses */
@@ -276,9 +276,6 @@ struct ti_firmware_header {
 
 #define TI_DEFAULT_CLOSING_WAIT	4000		/* in .01 secs */
 
-/* supported setserial flags */
-#define TI_SET_SERIAL_FLAGS	0
-
 /* read urb states */
 #define TI_READ_URB_RUNNING	0
 #define TI_READ_URB_STOPPING	1
@@ -288,11 +285,10 @@ struct ti_firmware_header {
 
 struct ti_port {
 	int			tp_is_open;
-	__u8			tp_msr;
-	__u8			tp_shadow_mcr;
-	__u8			tp_uart_mode;	/* 232 or 485 modes */
+	u8			tp_msr;
+	u8			tp_shadow_mcr;
+	u8			tp_uart_mode;	/* 232 or 485 modes */
 	unsigned int		tp_uart_base_addr;
-	int			tp_flags;
 	struct ti_device	*tp_tdev;
 	struct usb_serial_port	*tp_port;
 	spinlock_t		tp_lock;
@@ -306,7 +302,6 @@ struct ti_device {
 	struct usb_serial	*td_serial;
 	int			td_is_3410;
 	bool			td_rs485_only;
-	int			td_urb_error;
 };
 
 static int ti_startup(struct usb_serial *serial);
@@ -343,7 +338,7 @@ static int ti_get_serial_info(struct ti_port *tport,
 	struct serial_struct __user *ret_arg);
 static int ti_set_serial_info(struct tty_struct *tty, struct ti_port *tport,
 	struct serial_struct __user *new_arg);
-static void ti_handle_new_msr(struct ti_port *tport, __u8 msr);
+static void ti_handle_new_msr(struct ti_port *tport, u8 msr);
 
 static void ti_stop_read(struct ti_port *tport, struct tty_struct *tty);
 static int ti_restart_read(struct ti_port *tport, struct tty_struct *tty);
@@ -354,7 +349,7 @@ static int ti_command_in_sync(struct ti_device *tdev, __u8 command,
 	__u16 moduleid, __u16 value, __u8 *data, int size);
 
 static int ti_write_byte(struct usb_serial_port *port, struct ti_device *tdev,
-			 unsigned long addr, __u8 mask, __u8 byte);
+			 unsigned long addr, u8 mask, u8 byte);
 
 static int ti_download_firmware(struct ti_device *tdev);
 
@@ -647,12 +642,11 @@ static int ti_open(struct tty_struct *tty, struct usb_serial_port *port)
 	struct urb *urb;
 	int port_number;
 	int status;
-	__u16 open_settings = (__u8)(TI_PIPE_MODE_CONTINUOUS |
-			     TI_PIPE_TIMEOUT_ENABLE |
-			     (TI_TRANSFER_TIMEOUT << 2));
+	u16 open_settings;
 
-	if (tport == NULL)
-		return -ENODEV;
+	open_settings = (TI_PIPE_MODE_CONTINUOUS |
+			 TI_PIPE_TIMEOUT_ENABLE |
+			 (TI_TRANSFER_TIMEOUT << 2));
 
 	dev = port->serial->dev;
 	tdev = tport->tp_tdev;
@@ -686,7 +680,6 @@ static int ti_open(struct tty_struct *tty, struct usb_serial_port *port)
 	if (tty)
 		ti_set_termios(tty, port, &tty->termios);
 
-	dev_dbg(&port->dev, "%s - sending TI_OPEN_PORT\n", __func__);
 	status = ti_command_out_sync(tdev, TI_OPEN_PORT,
 		(__u8)(TI_UART1_PORT + port_number), open_settings, NULL, 0);
 	if (status) {
@@ -695,7 +688,6 @@ static int ti_open(struct tty_struct *tty, struct usb_serial_port *port)
 		goto unlink_int_urb;
 	}
 
-	dev_dbg(&port->dev, "%s - sending TI_START_PORT\n", __func__);
 	status = ti_command_out_sync(tdev, TI_START_PORT,
 		(__u8)(TI_UART1_PORT + port_number), 0, NULL, 0);
 	if (status) {
@@ -704,7 +696,6 @@ static int ti_open(struct tty_struct *tty, struct usb_serial_port *port)
 		goto unlink_int_urb;
 	}
 
-	dev_dbg(&port->dev, "%s - sending TI_PURGE_PORT\n", __func__);
 	status = ti_command_out_sync(tdev, TI_PURGE_PORT,
 		(__u8)(TI_UART1_PORT + port_number), TI_PURGE_INPUT, NULL, 0);
 	if (status) {
@@ -728,7 +719,6 @@ static int ti_open(struct tty_struct *tty, struct usb_serial_port *port)
 	if (tty)
 		ti_set_termios(tty, port, &tty->termios);
 
-	dev_dbg(&port->dev, "%s - sending TI_OPEN_PORT (2)\n", __func__);
 	status = ti_command_out_sync(tdev, TI_OPEN_PORT,
 		(__u8)(TI_UART1_PORT + port_number), open_settings, NULL, 0);
 	if (status) {
@@ -737,7 +727,6 @@ static int ti_open(struct tty_struct *tty, struct usb_serial_port *port)
 		goto unlink_int_urb;
 	}
 
-	dev_dbg(&port->dev, "%s - sending TI_START_PORT (2)\n", __func__);
 	status = ti_command_out_sync(tdev, TI_START_PORT,
 		(__u8)(TI_UART1_PORT + port_number), 0, NULL, 0);
 	if (status) {
@@ -747,7 +736,6 @@ static int ti_open(struct tty_struct *tty, struct usb_serial_port *port)
 	}
 
 	/* start read urb */
-	dev_dbg(&port->dev, "%s - start read urb\n", __func__);
 	urb = port->read_urb;
 	if (!urb) {
 		dev_err(&port->dev, "%s - no read urb\n", __func__);
@@ -773,7 +761,6 @@ unlink_int_urb:
 		usb_kill_urb(port->serial->port[0]->interrupt_in_urb);
 release_lock:
 	mutex_unlock(&tdev->td_open_close_lock);
-	dev_dbg(&port->dev, "%s - exit %d\n", __func__, status);
 	return status;
 }
 
@@ -789,8 +776,6 @@ static void ti_close(struct usb_serial_port *port)
 
 	tdev = usb_get_serial_data(port->serial);
 	tport = usb_get_serial_port_data(port);
-	if (tdev == NULL || tport == NULL)
-		return;
 
 	tport->tp_is_open = 0;
 
@@ -803,7 +788,6 @@ static void ti_close(struct usb_serial_port *port)
 
 	port_number = port->port_number;
 
-	dev_dbg(&port->dev, "%s - sending TI_CLOSE_PORT\n", __func__);
 	status = ti_command_out_sync(tdev, TI_CLOSE_PORT,
 		     (__u8)(TI_UART1_PORT + port_number), 0, NULL, 0);
 	if (status)
@@ -830,11 +814,10 @@ static int ti_write(struct tty_struct *tty, struct usb_serial_port *port,
 	struct ti_port *tport = usb_get_serial_port_data(port);
 
 	if (count == 0) {
-		dev_dbg(&port->dev, "%s - write request of 0 bytes\n", __func__);
 		return 0;
 	}
 
-	if (tport == NULL || !tport->tp_is_open)
+	if (!tport->tp_is_open)
 		return -ENODEV;
 
 	count = kfifo_in_locked(&port->write_fifo, data, count,
@@ -852,9 +835,6 @@ static int ti_write_room(struct tty_struct *tty)
 	int room = 0;
 	unsigned long flags;
 
-	if (tport == NULL)
-		return 0;
-
 	spin_lock_irqsave(&tport->tp_lock, flags);
 	room = kfifo_avail(&port->write_fifo);
 	spin_unlock_irqrestore(&tport->tp_lock, flags);
@@ -871,9 +851,6 @@ static int ti_chars_in_buffer(struct tty_struct *tty)
 	int chars = 0;
 	unsigned long flags;
 
-	if (tport == NULL)
-		return 0;
-
 	spin_lock_irqsave(&tport->tp_lock, flags);
 	chars = kfifo_len(&port->write_fifo);
 	spin_unlock_irqrestore(&tport->tp_lock, flags);
@@ -900,9 +877,6 @@ static void ti_throttle(struct tty_struct *tty)
 	struct usb_serial_port *port = tty->driver_data;
 	struct ti_port *tport = usb_get_serial_port_data(port);
 
-	if (tport == NULL)
-		return;
-
 	if (I_IXOFF(tty) || C_CRTSCTS(tty))
 		ti_stop_read(tport, tty);
 
@@ -915,9 +889,6 @@ static void ti_unthrottle(struct tty_struct *tty)
 	struct ti_port *tport = usb_get_serial_port_data(port);
 	int status;
 
-	if (tport == NULL)
-		return;
-
 	if (I_IXOFF(tty) || C_CRTSCTS(tty)) {
 		status = ti_restart_read(tport, tty);
 		if (status)
@@ -932,16 +903,11 @@ static int ti_ioctl(struct tty_struct *tty,
 	struct usb_serial_port *port = tty->driver_data;
 	struct ti_port *tport = usb_get_serial_port_data(port);
 
-	if (tport == NULL)
-		return -ENODEV;
-
 	switch (cmd) {
 	case TIOCGSERIAL:
-		dev_dbg(&port->dev, "%s - TIOCGSERIAL\n", __func__);
 		return ti_get_serial_info(tport,
 				(struct serial_struct __user *)arg);
 	case TIOCSSERIAL:
-		dev_dbg(&port->dev, "%s - TIOCSSERIAL\n", __func__);
 		return ti_set_serial_info(tty, tport,
 				(struct serial_struct __user *)arg);
 	}
@@ -959,6 +925,8 @@ static void ti_set_termios(struct tty_struct *tty,
 	int status;
 	int port_number = port->port_number;
 	unsigned int mcr;
+	u16 wbaudrate;
+	u16 wflags = 0;
 
 	cflag = tty->termios.c_cflag;
 	iflag = tty->termios.c_iflag;
@@ -967,21 +935,16 @@ static void ti_set_termios(struct tty_struct *tty,
 	dev_dbg(&port->dev, "%s - old clfag %08x, old iflag %08x\n", __func__,
 		old_termios->c_cflag, old_termios->c_iflag);
 
-	if (tport == NULL)
-		return;
-
 	config = kmalloc(sizeof(*config), GFP_KERNEL);
 	if (!config)
 		return;
 
-	config->wFlags = 0;
-
 	/* these flags must be set */
-	config->wFlags |= TI_UART_ENABLE_MS_INTS;
-	config->wFlags |= TI_UART_ENABLE_AUTO_START_DMA;
-	config->bUartMode = (__u8)(tport->tp_uart_mode);
+	wflags |= TI_UART_ENABLE_MS_INTS;
+	wflags |= TI_UART_ENABLE_AUTO_START_DMA;
+	config->bUartMode = tport->tp_uart_mode;
 
-	switch (cflag & CSIZE) {
+	switch (C_CSIZE(tty)) {
 	case CS5:
 		    config->bDataBits = TI_UART_5_DATA_BITS;
 		    break;
@@ -1000,29 +963,29 @@ static void ti_set_termios(struct tty_struct *tty,
 	/* CMSPAR isn't supported by this driver */
 	tty->termios.c_cflag &= ~CMSPAR;
 
-	if (cflag & PARENB) {
-		if (cflag & PARODD) {
-			config->wFlags |= TI_UART_ENABLE_PARITY_CHECKING;
+	if (C_PARENB(tty)) {
+		if (C_PARODD(tty)) {
+			wflags |= TI_UART_ENABLE_PARITY_CHECKING;
 			config->bParity = TI_UART_ODD_PARITY;
 		} else {
-			config->wFlags |= TI_UART_ENABLE_PARITY_CHECKING;
+			wflags |= TI_UART_ENABLE_PARITY_CHECKING;
 			config->bParity = TI_UART_EVEN_PARITY;
 		}
 	} else {
-		config->wFlags &= ~TI_UART_ENABLE_PARITY_CHECKING;
+		wflags &= ~TI_UART_ENABLE_PARITY_CHECKING;
 		config->bParity = TI_UART_NO_PARITY;
 	}
 
-	if (cflag & CSTOPB)
+	if (C_CSTOPB(tty))
 		config->bStopBits = TI_UART_2_STOP_BITS;
 	else
 		config->bStopBits = TI_UART_1_STOP_BITS;
 
-	if (cflag & CRTSCTS) {
+	if (C_CRTSCTS(tty)) {
 		/* RTS flow control must be off to drop RTS for baud rate B0 */
-		if ((cflag & CBAUD) != B0)
-			config->wFlags |= TI_UART_ENABLE_RTS_IN;
-		config->wFlags |= TI_UART_ENABLE_CTS_OUT;
+		if ((C_BAUD(tty)) != B0)
+			wflags |= TI_UART_ENABLE_RTS_IN;
+		wflags |= TI_UART_ENABLE_CTS_OUT;
 	} else {
 		ti_restart_read(tport, tty);
 	}
@@ -1032,34 +995,34 @@ static void ti_set_termios(struct tty_struct *tty,
 		config->cXoff = STOP_CHAR(tty);
 
 		if (I_IXOFF(tty))
-			config->wFlags |= TI_UART_ENABLE_X_IN;
+			wflags |= TI_UART_ENABLE_X_IN;
 		else
 			ti_restart_read(tport, tty);
 
 		if (I_IXON(tty))
-			config->wFlags |= TI_UART_ENABLE_X_OUT;
+			wflags |= TI_UART_ENABLE_X_OUT;
 	}
 
 	baud = tty_get_baud_rate(tty);
 	if (!baud)
 		baud = 9600;
 	if (tport->tp_tdev->td_is_3410)
-		config->wBaudRate = (__u16)((923077 + baud/2) / baud);
+		wbaudrate = (923077 + baud/2) / baud;
 	else
-		config->wBaudRate = (__u16)((461538 + baud/2) / baud);
+		wbaudrate = (461538 + baud/2) / baud;
 
 	/* FIXME: Should calculate resulting baud here and report it back */
-	if ((cflag & CBAUD) != B0)
+	if ((C_BAUD(tty)) != B0)
 		tty_encode_baud_rate(tty, baud, baud);
 
 	dev_dbg(&port->dev,
 		"%s - BaudRate=%d, wBaudRate=%d, wFlags=0x%04X, bDataBits=%d, bParity=%d, bStopBits=%d, cXon=%d, cXoff=%d, bUartMode=%d\n",
-		__func__, baud, config->wBaudRate, config->wFlags,
+		__func__, baud, wbaudrate, wflags,
 		config->bDataBits, config->bParity, config->bStopBits,
 		config->cXon, config->cXoff, config->bUartMode);
 
-	cpu_to_be16s(&config->wBaudRate);
-	cpu_to_be16s(&config->wFlags);
+	config->wBaudRate = cpu_to_be16(wbaudrate);
+	config->wFlags = cpu_to_be16(wflags);
 
 	status = ti_command_out_sync(tport->tp_tdev, TI_SET_CONFIG,
 		(__u8)(TI_UART1_PORT + port_number), 0, (__u8 *)config,
@@ -1071,7 +1034,7 @@ static void ti_set_termios(struct tty_struct *tty,
 	/* SET_CONFIG asserts RTS and DTR, reset them correctly */
 	mcr = tport->tp_shadow_mcr;
 	/* if baud rate is B0, clear RTS and DTR */
-	if ((cflag & CBAUD) == B0)
+	if (C_BAUD(tty) == B0)
 		mcr &= ~(TI_MCR_DTR | TI_MCR_RTS);
 	status = ti_set_mcr(tport, mcr);
 	if (status)
@@ -1092,9 +1055,6 @@ static int ti_tiocmget(struct tty_struct *tty)
 	unsigned int mcr;
 	unsigned long flags;
 
-	if (tport == NULL)
-		return -ENODEV;
-
 	spin_lock_irqsave(&tport->tp_lock, flags);
 	msr = tport->tp_msr;
 	mcr = tport->tp_shadow_mcr;
@@ -1122,9 +1082,6 @@ static int ti_tiocmset(struct tty_struct *tty,
 	unsigned int mcr;
 	unsigned long flags;
 
-	if (tport == NULL)
-		return -ENODEV;
-
 	spin_lock_irqsave(&tport->tp_lock, flags);
 	mcr = tport->tp_shadow_mcr;
 
@@ -1155,9 +1112,6 @@ static void ti_break(struct tty_struct *tty, int break_state)
 
 	dev_dbg(&port->dev, "%s - state = %d\n", __func__, break_state);
 
-	if (tport == NULL)
-		return;
-
 	status = ti_write_byte(port, tport->tp_tdev,
 		tport->tp_uart_base_addr + TI_UART_OFFSET_LCR,
 		TI_LCR_BREAK, break_state == -1 ? TI_LCR_BREAK : 0);
@@ -1189,7 +1143,7 @@ static void ti_interrupt_callback(struct urb *urb)
 	int function;
 	int status = urb->status;
 	int retval;
-	__u8 msr;
+	u8 msr;
 
 	switch (status) {
 	case 0:
@@ -1198,11 +1152,9 @@ static void ti_interrupt_callback(struct urb *urb)
 	case -ENOENT:
 	case -ESHUTDOWN:
 		dev_dbg(dev, "%s - urb shutting down, %d\n", __func__, status);
-		tdev->td_urb_error = 1;
 		return;
 	default:
 		dev_err(dev, "%s - nonzero urb status, %d\n", __func__, status);
-		tdev->td_urb_error = 1;
 		goto exit;
 	}
 
@@ -1275,12 +1227,10 @@ static void ti_bulk_in_callback(struct urb *urb)
 	case -ENOENT:
 	case -ESHUTDOWN:
 		dev_dbg(dev, "%s - urb shutting down, %d\n", __func__, status);
-		tport->tp_tdev->td_urb_error = 1;
 		return;
 	default:
 		dev_err(dev, "%s - nonzero urb status, %d\n",
 			__func__, status);
-		tport->tp_tdev->td_urb_error = 1;
 	}
 
 	if (status == -EPIPE)
@@ -1335,12 +1285,10 @@ static void ti_bulk_out_callback(struct urb *urb)
 	case -ENOENT:
 	case -ESHUTDOWN:
 		dev_dbg(&port->dev, "%s - urb shutting down, %d\n", __func__, status);
-		tport->tp_tdev->td_urb_error = 1;
 		return;
 	default:
 		dev_err_console(port, "%s - nonzero urb status, %d\n",
 			__func__, status);
-		tport->tp_tdev->td_urb_error = 1;
 	}
 
 	/* send any buffered data */
@@ -1490,7 +1438,6 @@ static int ti_get_serial_info(struct ti_port *tport,
 	ret_serial.type = PORT_16550A;
 	ret_serial.line = port->minor;
 	ret_serial.port = port->port_number;
-	ret_serial.flags = tport->tp_flags;
 	ret_serial.xmit_fifo_size = kfifo_size(&port->write_fifo);
 	ret_serial.baud_base = tport->tp_tdev->td_is_3410 ? 921600 : 460800;
 	ret_serial.closing_wait = cwait;
@@ -1515,14 +1462,13 @@ static int ti_set_serial_info(struct tty_struct *tty, struct ti_port *tport,
 	if (cwait != ASYNC_CLOSING_WAIT_NONE)
 		cwait = msecs_to_jiffies(10 * new_serial.closing_wait);
 
-	tport->tp_flags = new_serial.flags & TI_SET_SERIAL_FLAGS;
 	tport->tp_port->port.closing_wait = cwait;
 
 	return 0;
 }
 
 
-static void ti_handle_new_msr(struct ti_port *tport, __u8 msr)
+static void ti_handle_new_msr(struct ti_port *tport, u8 msr)
 {
 	struct async_icount *icount;
 	struct tty_struct *tty;
@@ -1634,8 +1580,8 @@ static int ti_command_in_sync(struct ti_device *tdev, __u8 command,
 
 
 static int ti_write_byte(struct usb_serial_port *port,
-			struct ti_device *tdev, unsigned long addr,
-			__u8 mask, __u8 byte)
+			 struct ti_device *tdev, unsigned long addr,
+			 u8 mask, u8 byte)
 {
 	int status;
 	unsigned int size;
@@ -1679,11 +1625,10 @@ static int ti_do_download(struct usb_device *dev, int pipe,
 	int len;
 
 	for (pos = sizeof(struct ti_firmware_header); pos < size; pos++)
-		cs = (__u8)(cs + buffer[pos]);
+		cs = (u8)(cs + buffer[pos]);
 
 	header = (struct ti_firmware_header *)buffer;
-	header->wLength = cpu_to_le16((__u16)(size
-					- sizeof(struct ti_firmware_header)));
+	header->wLength = cpu_to_le16(size - sizeof(*header));
 	header->bCheckSum = cs;
 
 	dev_dbg(&dev->dev, "%s - downloading firmware\n", __func__);
@@ -1701,7 +1646,7 @@ static int ti_download_firmware(struct ti_device *tdev)
 {
 	int status;
 	int buffer_size;
-	__u8 *buffer;
+	u8 *buffer;
 	struct usb_device *dev = tdev->td_serial->dev;
 	unsigned int pipe = usb_sndbulkpipe(dev,
 		tdev->td_serial->port[0]->bulk_out_endpointAddress);
diff --git a/drivers/usb/serial/usb-serial-simple.c b/drivers/usb/serial/usb-serial-simple.c
index a204782ae530..e98b6e57b703 100644
--- a/drivers/usb/serial/usb-serial-simple.c
+++ b/drivers/usb/serial/usb-serial-simple.c
@@ -54,7 +54,8 @@ DEVICE(funsoft, FUNSOFT_IDS);
 /* Infineon Flashloader driver */
 #define FLASHLOADER_IDS()		\
 	{ USB_DEVICE_INTERFACE_CLASS(0x058b, 0x0041, USB_CLASS_CDC_DATA) }, \
-	{ USB_DEVICE(0x8087, 0x0716) }
+	{ USB_DEVICE(0x8087, 0x0716) }, \
+	{ USB_DEVICE(0x8087, 0x0801) }
 DEVICE(flashloader, FLASHLOADER_IDS);
 
 /* Google Serial USB SubClass */
diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c
index b1b9bac44016..d213cf44a7e4 100644
--- a/drivers/usb/serial/usb-serial.c
+++ b/drivers/usb/serial/usb-serial.c
@@ -1433,7 +1433,7 @@ int usb_serial_register_drivers(struct usb_serial_driver *const serial_drivers[]
 
 	rc = usb_register(udriver);
 	if (rc)
-		return rc;
+		goto failed_usb_register;
 
 	for (sd = serial_drivers; *sd; ++sd) {
 		(*sd)->usb_driver = udriver;
@@ -1451,6 +1451,8 @@ int usb_serial_register_drivers(struct usb_serial_driver *const serial_drivers[]
 	while (sd-- > serial_drivers)
 		usb_serial_deregister(*sd);
 	usb_deregister(udriver);
+failed_usb_register:
+	kfree(udriver);
 	return rc;
 }
 EXPORT_SYMBOL_GPL(usb_serial_register_drivers);
diff --git a/drivers/usb/storage/alauda.c b/drivers/usb/storage/alauda.c
index 1d8b03c81030..878b4b8761f5 100644
--- a/drivers/usb/storage/alauda.c
+++ b/drivers/usb/storage/alauda.c
@@ -939,10 +939,8 @@ static int alauda_read_data(struct us_data *us, unsigned long address,
 
 	len = min(sectors, blocksize) * (pagesize + 64);
 	buffer = kmalloc(len, GFP_NOIO);
-	if (buffer == NULL) {
-		printk(KERN_WARNING "alauda_read_data: Out of memory\n");
+	if (!buffer)
 		return USB_STOR_TRANSPORT_ERROR;
-	}
 
 	/* Figure out the initial LBA and page */
 	lba = address >> blockshift;
@@ -1033,18 +1031,15 @@ static int alauda_write_data(struct us_data *us, unsigned long address,
 
 	len = min(sectors, blocksize) * pagesize;
 	buffer = kmalloc(len, GFP_NOIO);
-	if (buffer == NULL) {
-		printk(KERN_WARNING "alauda_write_data: Out of memory\n");
+	if (!buffer)
 		return USB_STOR_TRANSPORT_ERROR;
-	}
 
 	/*
 	 * We also need a temporary block buffer, where we read in the old data,
 	 * overwrite parts with the new data, and manipulate the redundancy data
 	 */
 	blockbuffer = kmalloc((pagesize + 64) * blocksize, GFP_NOIO);
-	if (blockbuffer == NULL) {
-		printk(KERN_WARNING "alauda_write_data: Out of memory\n");
+	if (!blockbuffer) {
 		kfree(buffer);
 		return USB_STOR_TRANSPORT_ERROR;
 	}
diff --git a/drivers/usb/storage/scsiglue.c b/drivers/usb/storage/scsiglue.c
index 33eb923df892..8cd2926fb1fe 100644
--- a/drivers/usb/storage/scsiglue.c
+++ b/drivers/usb/storage/scsiglue.c
@@ -296,6 +296,14 @@ static int slave_configure(struct scsi_device *sdev)
 		if (us->fflags & US_FL_BROKEN_FUA)
 			sdev->broken_fua = 1;
 
+		/* Some even totally fail to indicate a cache */
+		if (us->fflags & US_FL_ALWAYS_SYNC) {
+			/* don't read caching information */
+			sdev->skip_ms_page_8 = 1;
+			sdev->skip_ms_page_3f = 1;
+			/* assume sync is needed */
+			sdev->wce_default_on = 1;
+		}
 	} else {
 
 		/*
diff --git a/drivers/usb/storage/sddr09.c b/drivers/usb/storage/sddr09.c
index c5797fa2125e..3aeaa536c44f 100644
--- a/drivers/usb/storage/sddr09.c
+++ b/drivers/usb/storage/sddr09.c
@@ -766,10 +766,8 @@ sddr09_read_data(struct us_data *us,
 
 	len = min(sectors, (unsigned int) info->blocksize) * info->pagesize;
 	buffer = kmalloc(len, GFP_NOIO);
-	if (buffer == NULL) {
-		printk(KERN_WARNING "sddr09_read_data: Out of memory\n");
+	if (!buffer)
 		return -ENOMEM;
-	}
 
 	// This could be made much more efficient by checking for
 	// contiguous LBA's. Another exercise left to the student.
@@ -1004,10 +1002,8 @@ sddr09_write_data(struct us_data *us,
 	pagelen = (1 << info->pageshift) + (1 << CONTROL_SHIFT);
 	blocklen = (pagelen << info->blockshift);
 	blockbuffer = kmalloc(blocklen, GFP_NOIO);
-	if (!blockbuffer) {
-		printk(KERN_WARNING "sddr09_write_data: Out of memory\n");
+	if (!blockbuffer)
 		return -ENOMEM;
-	}
 
 	/*
 	 * Since we don't write the user data directly to the device,
@@ -1017,8 +1013,7 @@ sddr09_write_data(struct us_data *us,
 
 	len = min(sectors, (unsigned int) info->blocksize) * info->pagesize;
 	buffer = kmalloc(len, GFP_NOIO);
-	if (buffer == NULL) {
-		printk(KERN_WARNING "sddr09_write_data: Out of memory\n");
+	if (!buffer) {
 		kfree(blockbuffer);
 		return -ENOMEM;
 	}
@@ -1241,8 +1236,7 @@ sddr09_read_map(struct us_data *us) {
 	alloc_blocks = min(numblocks, SDDR09_READ_MAP_BUFSZ >> CONTROL_SHIFT);
 	alloc_len = (alloc_blocks << CONTROL_SHIFT);
 	buffer = kmalloc(alloc_len, GFP_NOIO);
-	if (buffer == NULL) {
-		printk(KERN_WARNING "sddr09_read_map: out of memory\n");
+	if (!buffer) {
 		result = -1;
 		goto done;
 	}
diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h
index aa3539238848..af3c7eecff91 100644
--- a/drivers/usb/storage/unusual_devs.h
+++ b/drivers/usb/storage/unusual_devs.h
@@ -338,6 +338,13 @@ UNUSUAL_DEV(  0x046b, 0xff40, 0x0100, 0x0100,
 		USB_SC_DEVICE, USB_PR_DEVICE, NULL,
 		US_FL_NO_WP_DETECT),
 
+/* Reported by Egbert Eich <eich@suse.com> */
+UNUSUAL_DEV(  0x0480, 0xd010, 0x0100, 0x9999,
+		"Toshiba",
+		"External USB 3.0",
+		USB_SC_DEVICE, USB_PR_DEVICE, NULL,
+		US_FL_ALWAYS_SYNC),
+
 /* Patch submitted by Philipp Friedrich <philipp@void.at> */
 UNUSUAL_DEV(  0x0482, 0x0100, 0x0100, 0x0100,
 		"Kyocera",
diff --git a/drivers/usb/storage/usb.c b/drivers/usb/storage/usb.c
index ef2d8cde6ef7..2cba13a532cd 100644
--- a/drivers/usb/storage/usb.c
+++ b/drivers/usb/storage/usb.c
@@ -498,7 +498,8 @@ void usb_stor_adjust_quirks(struct usb_device *udev, unsigned long *fflags)
 			US_FL_NO_READ_DISC_INFO | US_FL_NO_READ_CAPACITY_16 |
 			US_FL_INITIAL_READ10 | US_FL_WRITE_CACHE |
 			US_FL_NO_ATA_1X | US_FL_NO_REPORT_OPCODES |
-			US_FL_MAX_SECTORS_240 | US_FL_NO_REPORT_LUNS);
+			US_FL_MAX_SECTORS_240 | US_FL_NO_REPORT_LUNS |
+			US_FL_ALWAYS_SYNC);
 
 	p = quirks;
 	while (*p) {
@@ -581,6 +582,9 @@ void usb_stor_adjust_quirks(struct usb_device *udev, unsigned long *fflags)
 		case 'w':
 			f |= US_FL_NO_WP_DETECT;
 			break;
+		case 'y':
+			f |= US_FL_ALWAYS_SYNC;
+			break;
 		/* Ignore unrecognized flag characters */
 		}
 	}
@@ -794,10 +798,8 @@ static int usb_stor_acquire_resources(struct us_data *us)
 	struct task_struct *th;
 
 	us->current_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!us->current_urb) {
-		usb_stor_dbg(us, "URB allocation failed\n");
+	if (!us->current_urb)
 		return -ENOMEM;
-	}
 
 	/*
 	 * Just before we start our control thread, initialize
@@ -1070,17 +1072,17 @@ int usb_stor_probe2(struct us_data *us)
 	result = usb_stor_acquire_resources(us);
 	if (result)
 		goto BadDevice;
+	usb_autopm_get_interface_no_resume(us->pusb_intf);
 	snprintf(us->scsi_name, sizeof(us->scsi_name), "usb-storage %s",
 					dev_name(&us->pusb_intf->dev));
 	result = scsi_add_host(us_to_host(us), dev);
 	if (result) {
 		dev_warn(dev,
 				"Unable to add the scsi host\n");
-		goto BadDevice;
+		goto HostAddErr;
 	}
 
 	/* Submit the delayed_work for SCSI-device scanning */
-	usb_autopm_get_interface_no_resume(us->pusb_intf);
 	set_bit(US_FLIDX_SCAN_PENDING, &us->dflags);
 
 	if (delay_use > 0)
@@ -1090,6 +1092,8 @@ int usb_stor_probe2(struct us_data *us)
 	return 0;
 
 	/* We come here if there are any problems */
+HostAddErr:
+	usb_autopm_put_interface_no_suspend(us->pusb_intf);
 BadDevice:
 	usb_stor_dbg(us, "storage_probe() failed\n");
 	release_everything(us);
diff --git a/drivers/usb/usb-skeleton.c b/drivers/usb/usb-skeleton.c
index 545d09b8081d..5133a0792eb0 100644
--- a/drivers/usb/usb-skeleton.c
+++ b/drivers/usb/usb-skeleton.c
@@ -499,10 +499,8 @@ static int skel_probe(struct usb_interface *interface,
 
 	/* allocate memory for our device state and initialize it */
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev) {
-		dev_err(&interface->dev, "Out of memory\n");
+	if (!dev)
 		goto error;
-	}
 	kref_init(&dev->kref);
 	sema_init(&dev->limit_sem, WRITES_IN_FLIGHT);
 	mutex_init(&dev->io_mutex);
@@ -526,17 +524,11 @@ static int skel_probe(struct usb_interface *interface,
 			dev->bulk_in_size = buffer_size;
 			dev->bulk_in_endpointAddr = endpoint->bEndpointAddress;
 			dev->bulk_in_buffer = kmalloc(buffer_size, GFP_KERNEL);
-			if (!dev->bulk_in_buffer) {
-				dev_err(&interface->dev,
-					"Could not allocate bulk_in_buffer\n");
+			if (!dev->bulk_in_buffer)
 				goto error;
-			}
 			dev->bulk_in_urb = usb_alloc_urb(0, GFP_KERNEL);
-			if (!dev->bulk_in_urb) {
-				dev_err(&interface->dev,
-					"Could not allocate bulk_in_urb\n");
+			if (!dev->bulk_in_urb)
 				goto error;
-			}
 		}
 
 		if (!dev->bulk_out_endpointAddr &&
diff --git a/drivers/usb/usbip/Kconfig b/drivers/usb/usbip/Kconfig
index 17646b25343f..eeefa29f8aa2 100644
--- a/drivers/usb/usbip/Kconfig
+++ b/drivers/usb/usbip/Kconfig
@@ -1,6 +1,7 @@
 config USBIP_CORE
 	tristate "USB/IP support"
-	depends on USB_COMMON && NET
+	depends on NET
+	select USB_COMMON
 	---help---
 	  This enables pushing USB packets over IP to allow remote
 	  machines direct access to USB devices. It provides the
@@ -24,6 +25,27 @@ config USBIP_VHCI_HCD
 	  To compile this driver as a module, choose M here: the
 	  module will be called vhci-hcd.
 
+config USBIP_VHCI_HC_PORTS
+	int "Number of ports per USB/IP virtual host controller"
+	range 1 31
+	default 8
+	depends on USBIP_VHCI_HCD
+	---help---
+	  To increase number of ports available for USB/IP virtual
+	  host controller driver, this defines number of ports per
+	  USB/IP virtual host controller.
+
+config USBIP_VHCI_NR_HCS
+	int "Number of USB/IP virtual host controllers"
+	range 1 128
+	default 1
+	depends on USBIP_VHCI_HCD
+	---help---
+	  To increase number of ports available for USB/IP virtual
+	  host controller driver, this defines number of USB/IP
+	  virtual host controllers as if adding physical host
+	  controllers.
+
 config USBIP_HOST
 	tristate "Host driver"
 	depends on USBIP_CORE && USB
diff --git a/drivers/usb/usbip/stub_rx.c b/drivers/usb/usbip/stub_rx.c
index 2df63e305722..191b176ffedf 100644
--- a/drivers/usb/usbip/stub_rx.c
+++ b/drivers/usb/usbip/stub_rx.c
@@ -461,7 +461,6 @@ static void stub_recv_cmd_submit(struct stub_device *sdev,
 		priv->urb = usb_alloc_urb(0, GFP_KERNEL);
 
 	if (!priv->urb) {
-		dev_err(&udev->dev, "malloc urb\n");
 		usbip_event_add(ud, SDEV_EVENT_ERROR_MALLOC);
 		return;
 	}
diff --git a/drivers/usb/usbip/vhci.h b/drivers/usb/usbip/vhci.h
index a863a98a91ce..88b71c4e068f 100644
--- a/drivers/usb/usbip/vhci.h
+++ b/drivers/usb/usbip/vhci.h
@@ -1,5 +1,6 @@
 /*
  * Copyright (C) 2003-2008 Takahiro Hirofuchi
+ * Copyright (C) 2015 Nobuo Iwata
  *
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -72,13 +73,25 @@ struct vhci_unlink {
 };
 
 /* Number of supported ports. Value has an upperbound of USB_MAXCHILDREN */
-#define VHCI_NPORTS 8
+#ifdef CONFIG_USBIP_VHCI_HC_PORTS
+#define VHCI_HC_PORTS CONFIG_USBIP_VHCI_HC_PORTS
+#else
+#define VHCI_HC_PORTS 8
+#endif
+
+#ifdef CONFIG_USBIP_VHCI_NR_HCS
+#define VHCI_NR_HCS CONFIG_USBIP_VHCI_NR_HCS
+#else
+#define VHCI_NR_HCS 1
+#endif
+
+#define MAX_STATUS_NAME 16
 
 /* for usb_bus.hcpriv */
 struct vhci_hcd {
 	spinlock_t lock;
 
-	u32 port_status[VHCI_NPORTS];
+	u32 port_status[VHCI_HC_PORTS];
 
 	unsigned resuming:1;
 	unsigned long re_timeout;
@@ -90,14 +103,19 @@ struct vhci_hcd {
 	 * wIndex shows the port number and begins from 1.
 	 * But, the index of this array begins from 0.
 	 */
-	struct vhci_device vdev[VHCI_NPORTS];
+	struct vhci_device vdev[VHCI_HC_PORTS];
 };
 
-extern struct vhci_hcd *the_controller;
-extern const struct attribute_group dev_attr_group;
+extern int vhci_num_controllers;
+extern struct platform_device **vhci_pdevs;
+extern struct attribute_group vhci_attr_group;
 
 /* vhci_hcd.c */
-void rh_port_connect(int rhport, enum usb_device_speed speed);
+void rh_port_connect(struct vhci_device *vdev, enum usb_device_speed speed);
+
+/* vhci_sysfs.c */
+int vhci_init_attr_group(void);
+void vhci_finish_attr_group(void);
 
 /* vhci_rx.c */
 struct urb *pickup_urb_and_free_priv(struct vhci_device *vdev, __u32 seqnum);
@@ -106,9 +124,14 @@ int vhci_rx_loop(void *data);
 /* vhci_tx.c */
 int vhci_tx_loop(void *data);
 
-static inline struct vhci_device *port_to_vdev(__u32 port)
+static inline __u32 port_to_rhport(__u32 port)
+{
+	return port % VHCI_HC_PORTS;
+}
+
+static inline int port_to_pdev_nr(__u32 port)
 {
-	return &the_controller->vdev[port];
+	return port / VHCI_HC_PORTS;
 }
 
 static inline struct vhci_hcd *hcd_to_vhci(struct usb_hcd *hcd)
@@ -116,14 +139,25 @@ static inline struct vhci_hcd *hcd_to_vhci(struct usb_hcd *hcd)
 	return (struct vhci_hcd *) (hcd->hcd_priv);
 }
 
+static inline struct device *hcd_dev(struct usb_hcd *hcd)
+{
+	return (hcd)->self.controller;
+}
+
+static inline const char *hcd_name(struct usb_hcd *hcd)
+{
+	return (hcd)->self.bus_name;
+}
+
 static inline struct usb_hcd *vhci_to_hcd(struct vhci_hcd *vhci)
 {
 	return container_of((void *) vhci, struct usb_hcd, hcd_priv);
 }
 
-static inline struct device *vhci_dev(struct vhci_hcd *vhci)
+static inline struct vhci_hcd *vdev_to_vhci(struct vhci_device *vdev)
 {
-	return vhci_to_hcd(vhci)->self.controller;
+	return container_of(
+			(void *)(vdev - vdev->rhport), struct vhci_hcd, vdev);
 }
 
 #endif /* __USBIP_VHCI_H */
diff --git a/drivers/usb/usbip/vhci_hcd.c b/drivers/usb/usbip/vhci_hcd.c
index 2e0450bec1b1..03eccf29ace0 100644
--- a/drivers/usb/usbip/vhci_hcd.c
+++ b/drivers/usb/usbip/vhci_hcd.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (C) 2003-2008 Takahiro Hirofuchi
+ * Copyright (C) 2015-2016 Nobuo Iwata
  *
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -56,7 +57,9 @@ static int vhci_get_frame_number(struct usb_hcd *hcd);
 static const char driver_name[] = "vhci_hcd";
 static const char driver_desc[] = "USB/IP Virtual Host Controller";
 
-struct vhci_hcd *the_controller;
+int vhci_num_controllers = VHCI_NR_HCS;
+
+struct platform_device **vhci_pdevs;
 
 static const char * const bit_desc[] = {
 	"CONNECTION",		/*0*/
@@ -119,47 +122,59 @@ static void dump_port_status_diff(u32 prev_status, u32 new_status)
 	pr_debug("\n");
 }
 
-void rh_port_connect(int rhport, enum usb_device_speed speed)
+void rh_port_connect(struct vhci_device *vdev, enum usb_device_speed speed)
 {
+	struct vhci_hcd	*vhci = vdev_to_vhci(vdev);
+	int		rhport = vdev->rhport;
+	u32		status;
 	unsigned long	flags;
 
 	usbip_dbg_vhci_rh("rh_port_connect %d\n", rhport);
 
-	spin_lock_irqsave(&the_controller->lock, flags);
+	spin_lock_irqsave(&vhci->lock, flags);
+
+	status = vhci->port_status[rhport];
 
-	the_controller->port_status[rhport] |= USB_PORT_STAT_CONNECTION
-		| (1 << USB_PORT_FEAT_C_CONNECTION);
+	status |= USB_PORT_STAT_CONNECTION | (1 << USB_PORT_FEAT_C_CONNECTION);
 
 	switch (speed) {
 	case USB_SPEED_HIGH:
-		the_controller->port_status[rhport] |= USB_PORT_STAT_HIGH_SPEED;
+		status |= USB_PORT_STAT_HIGH_SPEED;
 		break;
 	case USB_SPEED_LOW:
-		the_controller->port_status[rhport] |= USB_PORT_STAT_LOW_SPEED;
+		status |= USB_PORT_STAT_LOW_SPEED;
 		break;
 	default:
 		break;
 	}
 
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	vhci->port_status[rhport] = status;
+
+	spin_unlock_irqrestore(&vhci->lock, flags);
 
-	usb_hcd_poll_rh_status(vhci_to_hcd(the_controller));
+	usb_hcd_poll_rh_status(vhci_to_hcd(vhci));
 }
 
-static void rh_port_disconnect(int rhport)
+static void rh_port_disconnect(struct vhci_device *vdev)
 {
+	struct vhci_hcd	*vhci = vdev_to_vhci(vdev);
+	int		rhport = vdev->rhport;
+	u32		status;
 	unsigned long	flags;
 
 	usbip_dbg_vhci_rh("rh_port_disconnect %d\n", rhport);
 
-	spin_lock_irqsave(&the_controller->lock, flags);
+	spin_lock_irqsave(&vhci->lock, flags);
+
+	status = vhci->port_status[rhport];
+
+	status &= ~USB_PORT_STAT_CONNECTION;
+	status |= (1 << USB_PORT_FEAT_C_CONNECTION);
 
-	the_controller->port_status[rhport] &= ~USB_PORT_STAT_CONNECTION;
-	the_controller->port_status[rhport] |=
-					(1 << USB_PORT_FEAT_C_CONNECTION);
+	vhci->port_status[rhport] = status;
 
-	spin_unlock_irqrestore(&the_controller->lock, flags);
-	usb_hcd_poll_rh_status(vhci_to_hcd(the_controller));
+	spin_unlock_irqrestore(&vhci->lock, flags);
+	usb_hcd_poll_rh_status(vhci_to_hcd(vhci));
 }
 
 #define PORT_C_MASK				\
@@ -188,7 +203,7 @@ static int vhci_hub_status(struct usb_hcd *hcd, char *buf)
 	int		changed = 0;
 	unsigned long	flags;
 
-	retval = DIV_ROUND_UP(VHCI_NPORTS + 1, 8);
+	retval = DIV_ROUND_UP(VHCI_HC_PORTS + 1, 8);
 	memset(buf, 0, retval);
 
 	vhci = hcd_to_vhci(hcd);
@@ -200,7 +215,7 @@ static int vhci_hub_status(struct usb_hcd *hcd, char *buf)
 	}
 
 	/* check pseudo status register for each port */
-	for (rhport = 0; rhport < VHCI_NPORTS; rhport++) {
+	for (rhport = 0; rhport < VHCI_HC_PORTS; rhport++) {
 		if ((vhci->port_status[rhport] & PORT_C_MASK)) {
 			/* The status of a port has been changed, */
 			usbip_dbg_vhci_rh("port %d status changed\n", rhport);
@@ -225,7 +240,7 @@ static inline void hub_descriptor(struct usb_hub_descriptor *desc)
 	desc->bDescLength = 9;
 	desc->wHubCharacteristics = cpu_to_le16(
 		HUB_CHAR_INDV_PORT_LPSM | HUB_CHAR_COMMON_OCPM);
-	desc->bNbrPorts = VHCI_NPORTS;
+	desc->bNbrPorts = VHCI_HC_PORTS;
 	desc->u.hs.DeviceRemovable[0] = 0xff;
 	desc->u.hs.DeviceRemovable[1] = 0xff;
 }
@@ -238,7 +253,7 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 	int		rhport;
 	unsigned long	flags;
 
-	u32 prev_port_status[VHCI_NPORTS];
+	u32 prev_port_status[VHCI_HC_PORTS];
 
 	if (!HCD_HW_ACCESSIBLE(hcd))
 		return -ETIMEDOUT;
@@ -249,7 +264,7 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 	 */
 	usbip_dbg_vhci_rh("typeReq %x wValue %x wIndex %x\n", typeReq, wValue,
 			  wIndex);
-	if (wIndex > VHCI_NPORTS)
+	if (wIndex > VHCI_HC_PORTS)
 		pr_err("invalid port number %d\n", wIndex);
 	rhport = ((__u8)(wIndex & 0x00ff)) - 1;
 
@@ -315,7 +330,7 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 		break;
 	case GetPortStatus:
 		usbip_dbg_vhci_rh(" GetPortStatus port %x\n", wIndex);
-		if (wIndex > VHCI_NPORTS || wIndex < 1) {
+		if (wIndex > VHCI_HC_PORTS || wIndex < 1) {
 			pr_err("invalid port number %d\n", wIndex);
 			retval = -EPIPE;
 		}
@@ -416,14 +431,27 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 
 static struct vhci_device *get_vdev(struct usb_device *udev)
 {
-	int i;
+	struct platform_device *pdev;
+	struct usb_hcd *hcd;
+	struct vhci_hcd *vhci;
+	int pdev_nr, rhport;
 
 	if (!udev)
 		return NULL;
 
-	for (i = 0; i < VHCI_NPORTS; i++)
-		if (the_controller->vdev[i].udev == udev)
-			return port_to_vdev(i);
+	for (pdev_nr = 0; pdev_nr < vhci_num_controllers; pdev_nr++) {
+		pdev = *(vhci_pdevs + pdev_nr);
+		if (pdev == NULL)
+			continue;
+		hcd = platform_get_drvdata(pdev);
+		if (hcd == NULL)
+			continue;
+		vhci = hcd_to_vhci(hcd);
+		for (rhport = 0; rhport < VHCI_HC_PORTS; rhport++) {
+			if (vhci->vdev[rhport].udev == udev)
+				return &vhci->vdev[rhport];
+		}
+	}
 
 	return NULL;
 }
@@ -432,6 +460,7 @@ static void vhci_tx_urb(struct urb *urb)
 {
 	struct vhci_device *vdev = get_vdev(urb->dev);
 	struct vhci_priv *priv;
+	struct vhci_hcd *vhci = vdev_to_vhci(vdev);
 	unsigned long flags;
 
 	if (!vdev) {
@@ -447,7 +476,7 @@ static void vhci_tx_urb(struct urb *urb)
 
 	spin_lock_irqsave(&vdev->priv_lock, flags);
 
-	priv->seqnum = atomic_inc_return(&the_controller->seqnum);
+	priv->seqnum = atomic_inc_return(&vhci->seqnum);
 	if (priv->seqnum == 0xffff)
 		dev_info(&urb->dev->dev, "seqnum max\n");
 
@@ -465,7 +494,9 @@ static void vhci_tx_urb(struct urb *urb)
 static int vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
 			    gfp_t mem_flags)
 {
+	struct vhci_hcd *vhci = hcd_to_vhci(hcd);
 	struct device *dev = &urb->dev->dev;
+	u8 portnum = urb->dev->portnum;
 	int ret = 0;
 	struct vhci_device *vdev;
 	unsigned long flags;
@@ -473,26 +504,30 @@ static int vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
 	usbip_dbg_vhci_hc("enter, usb_hcd %p urb %p mem_flags %d\n",
 			  hcd, urb, mem_flags);
 
+	if (portnum > VHCI_HC_PORTS) {
+		pr_err("invalid port number %d\n", portnum);
+		return -ENODEV;
+	}
+	vdev = &vhci->vdev[portnum-1];
+
 	/* patch to usb_sg_init() is in 2.5.60 */
 	BUG_ON(!urb->transfer_buffer && urb->transfer_buffer_length);
 
-	spin_lock_irqsave(&the_controller->lock, flags);
+	spin_lock_irqsave(&vhci->lock, flags);
 
 	if (urb->status != -EINPROGRESS) {
 		dev_err(dev, "URB already unlinked!, status %d\n", urb->status);
-		spin_unlock_irqrestore(&the_controller->lock, flags);
+		spin_unlock_irqrestore(&vhci->lock, flags);
 		return urb->status;
 	}
 
-	vdev = port_to_vdev(urb->dev->portnum-1);
-
 	/* refuse enqueue for dead connection */
 	spin_lock(&vdev->ud.lock);
 	if (vdev->ud.status == VDEV_ST_NULL ||
 	    vdev->ud.status == VDEV_ST_ERROR) {
 		dev_err(dev, "enqueue for inactive port %d\n", vdev->rhport);
 		spin_unlock(&vdev->ud.lock);
-		spin_unlock_irqrestore(&the_controller->lock, flags);
+		spin_unlock_irqrestore(&vhci->lock, flags);
 		return -ENODEV;
 	}
 	spin_unlock(&vdev->ud.lock);
@@ -565,17 +600,16 @@ static int vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
 
 out:
 	vhci_tx_urb(urb);
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	spin_unlock_irqrestore(&vhci->lock, flags);
 
 	return 0;
 
 no_need_xmit:
 	usb_hcd_unlink_urb_from_ep(hcd, urb);
 no_need_unlink:
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	spin_unlock_irqrestore(&vhci->lock, flags);
 	if (!ret)
-		usb_hcd_giveback_urb(vhci_to_hcd(the_controller),
-				     urb, urb->status);
+		usb_hcd_giveback_urb(hcd, urb, urb->status);
 	return ret;
 }
 
@@ -627,19 +661,20 @@ no_need_unlink:
  */
 static int vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
 {
+	struct vhci_hcd *vhci = hcd_to_vhci(hcd);
 	struct vhci_priv *priv;
 	struct vhci_device *vdev;
 	unsigned long flags;
 
 	pr_info("dequeue a urb %p\n", urb);
 
-	spin_lock_irqsave(&the_controller->lock, flags);
+	spin_lock_irqsave(&vhci->lock, flags);
 
 	priv = urb->hcpriv;
 	if (!priv) {
 		/* URB was never linked! or will be soon given back by
 		 * vhci_rx. */
-		spin_unlock_irqrestore(&the_controller->lock, flags);
+		spin_unlock_irqrestore(&vhci->lock, flags);
 		return -EIDRM;
 	}
 
@@ -648,7 +683,7 @@ static int vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
 
 		ret = usb_hcd_check_unlink_urb(hcd, urb, status);
 		if (ret) {
-			spin_unlock_irqrestore(&the_controller->lock, flags);
+			spin_unlock_irqrestore(&vhci->lock, flags);
 			return ret;
 		}
 	}
@@ -676,10 +711,9 @@ static int vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
 
 		usb_hcd_unlink_urb_from_ep(hcd, urb);
 
-		spin_unlock_irqrestore(&the_controller->lock, flags);
-		usb_hcd_giveback_urb(vhci_to_hcd(the_controller), urb,
-				     urb->status);
-		spin_lock_irqsave(&the_controller->lock, flags);
+		spin_unlock_irqrestore(&vhci->lock, flags);
+		usb_hcd_giveback_urb(vhci_to_hcd(vhci), urb, urb->status);
+		spin_lock_irqsave(&vhci->lock, flags);
 
 	} else {
 		/* tcp connection is alive */
@@ -691,12 +725,12 @@ static int vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
 		unlink = kzalloc(sizeof(struct vhci_unlink), GFP_ATOMIC);
 		if (!unlink) {
 			spin_unlock(&vdev->priv_lock);
-			spin_unlock_irqrestore(&the_controller->lock, flags);
+			spin_unlock_irqrestore(&vhci->lock, flags);
 			usbip_event_add(&vdev->ud, VDEV_EVENT_ERROR_MALLOC);
 			return -ENOMEM;
 		}
 
-		unlink->seqnum = atomic_inc_return(&the_controller->seqnum);
+		unlink->seqnum = atomic_inc_return(&vhci->seqnum);
 		if (unlink->seqnum == 0xffff)
 			pr_info("seqnum max\n");
 
@@ -712,7 +746,7 @@ static int vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
 		spin_unlock(&vdev->priv_lock);
 	}
 
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	spin_unlock_irqrestore(&vhci->lock, flags);
 
 	usbip_dbg_vhci_hc("leave\n");
 	return 0;
@@ -720,10 +754,12 @@ static int vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
 
 static void vhci_device_unlink_cleanup(struct vhci_device *vdev)
 {
+	struct vhci_hcd *vhci = vdev_to_vhci(vdev);
+	struct usb_hcd *hcd = vhci_to_hcd(vhci);
 	struct vhci_unlink *unlink, *tmp;
 	unsigned long flags;
 
-	spin_lock_irqsave(&the_controller->lock, flags);
+	spin_lock_irqsave(&vhci->lock, flags);
 	spin_lock(&vdev->priv_lock);
 
 	list_for_each_entry_safe(unlink, tmp, &vdev->unlink_tx, list) {
@@ -752,24 +788,23 @@ static void vhci_device_unlink_cleanup(struct vhci_device *vdev)
 
 		urb->status = -ENODEV;
 
-		usb_hcd_unlink_urb_from_ep(vhci_to_hcd(the_controller), urb);
+		usb_hcd_unlink_urb_from_ep(hcd, urb);
 
 		list_del(&unlink->list);
 
 		spin_unlock(&vdev->priv_lock);
-		spin_unlock_irqrestore(&the_controller->lock, flags);
+		spin_unlock_irqrestore(&vhci->lock, flags);
 
-		usb_hcd_giveback_urb(vhci_to_hcd(the_controller), urb,
-				     urb->status);
+		usb_hcd_giveback_urb(hcd, urb, urb->status);
 
-		spin_lock_irqsave(&the_controller->lock, flags);
+		spin_lock_irqsave(&vhci->lock, flags);
 		spin_lock(&vdev->priv_lock);
 
 		kfree(unlink);
 	}
 
 	spin_unlock(&vdev->priv_lock);
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	spin_unlock_irqrestore(&vhci->lock, flags);
 }
 
 /*
@@ -827,7 +862,7 @@ static void vhci_shutdown_connection(struct usbip_device *ud)
 	 * is actually given back by vhci_rx after receiving its return pdu.
 	 *
 	 */
-	rh_port_disconnect(vdev->rhport);
+	rh_port_disconnect(vdev);
 
 	pr_info("disconnect device\n");
 }
@@ -866,7 +901,7 @@ static void vhci_device_unusable(struct usbip_device *ud)
 
 static void vhci_device_init(struct vhci_device *vdev)
 {
-	memset(vdev, 0, sizeof(*vdev));
+	memset(vdev, 0, sizeof(struct vhci_device));
 
 	vdev->ud.side   = USBIP_VHCI;
 	vdev->ud.status = VDEV_ST_NULL;
@@ -887,17 +922,34 @@ static void vhci_device_init(struct vhci_device *vdev)
 	usbip_start_eh(&vdev->ud);
 }
 
+static int hcd_name_to_id(const char *name)
+{
+	char *c;
+	long val;
+	int ret;
+
+	c = strchr(name, '.');
+	if (c == NULL)
+		return 0;
+
+	ret = kstrtol(c+1, 10, &val);
+	if (ret < 0)
+		return ret;
+
+	return val;
+}
+
 static int vhci_start(struct usb_hcd *hcd)
 {
 	struct vhci_hcd *vhci = hcd_to_vhci(hcd);
-	int rhport;
+	int id, rhport;
 	int err = 0;
 
 	usbip_dbg_vhci_hc("enter vhci_start\n");
 
 	/* initialize private data of usb_hcd */
 
-	for (rhport = 0; rhport < VHCI_NPORTS; rhport++) {
+	for (rhport = 0; rhport < VHCI_HC_PORTS; rhport++) {
 		struct vhci_device *vdev = &vhci->vdev[rhport];
 
 		vhci_device_init(vdev);
@@ -910,11 +962,26 @@ static int vhci_start(struct usb_hcd *hcd)
 	hcd->power_budget = 0; /* no limit */
 	hcd->uses_new_polling = 1;
 
+	id = hcd_name_to_id(hcd_name(hcd));
+	if (id < 0) {
+		pr_err("invalid vhci name %s\n", hcd_name(hcd));
+		return -EINVAL;
+	}
+
 	/* vhci_hcd is now ready to be controlled through sysfs */
-	err = sysfs_create_group(&vhci_dev(vhci)->kobj, &dev_attr_group);
-	if (err) {
-		pr_err("create sysfs files\n");
-		return err;
+	if (id == 0) {
+		err = vhci_init_attr_group();
+		if (err) {
+			pr_err("init attr group\n");
+			return err;
+		}
+		err = sysfs_create_group(&hcd_dev(hcd)->kobj, &vhci_attr_group);
+		if (err) {
+			pr_err("create sysfs files\n");
+			vhci_finish_attr_group();
+			return err;
+		}
+		pr_info("created sysfs %s\n", hcd_name(hcd));
 	}
 
 	return 0;
@@ -923,15 +990,19 @@ static int vhci_start(struct usb_hcd *hcd)
 static void vhci_stop(struct usb_hcd *hcd)
 {
 	struct vhci_hcd *vhci = hcd_to_vhci(hcd);
-	int rhport = 0;
+	int id, rhport;
 
 	usbip_dbg_vhci_hc("stop VHCI controller\n");
 
 	/* 1. remove the userland interface of vhci_hcd */
-	sysfs_remove_group(&vhci_dev(vhci)->kobj, &dev_attr_group);
+	id = hcd_name_to_id(hcd_name(hcd));
+	if (id == 0) {
+		sysfs_remove_group(&hcd_dev(hcd)->kobj, &vhci_attr_group);
+		vhci_finish_attr_group();
+	}
 
 	/* 2. shutdown all the ports of vhci_hcd */
-	for (rhport = 0; rhport < VHCI_NPORTS; rhport++) {
+	for (rhport = 0; rhport < VHCI_HC_PORTS; rhport++) {
 		struct vhci_device *vdev = &vhci->vdev[rhport];
 
 		usbip_event_add(&vdev->ud, VDEV_EVENT_REMOVED);
@@ -1025,9 +1096,6 @@ static int vhci_hcd_probe(struct platform_device *pdev)
 	}
 	hcd->has_tt = 1;
 
-	/* this is private data for vhci_hcd */
-	the_controller = hcd_to_vhci(hcd);
-
 	/*
 	 * Finish generic HCD structure initialization and register.
 	 * Call the driver's reset() and start() routines.
@@ -1036,7 +1104,6 @@ static int vhci_hcd_probe(struct platform_device *pdev)
 	if (ret != 0) {
 		pr_err("usb_add_hcd failed %d\n", ret);
 		usb_put_hcd(hcd);
-		the_controller = NULL;
 		return ret;
 	}
 
@@ -1059,7 +1126,6 @@ static int vhci_hcd_remove(struct platform_device *pdev)
 	 */
 	usb_remove_hcd(hcd);
 	usb_put_hcd(hcd);
-	the_controller = NULL;
 
 	return 0;
 }
@@ -1070,21 +1136,24 @@ static int vhci_hcd_remove(struct platform_device *pdev)
 static int vhci_hcd_suspend(struct platform_device *pdev, pm_message_t state)
 {
 	struct usb_hcd *hcd;
-	int rhport = 0;
+	struct vhci_hcd *vhci;
+	int rhport;
 	int connected = 0;
 	int ret = 0;
 	unsigned long flags;
 
 	hcd = platform_get_drvdata(pdev);
+	if (!hcd)
+		return 0;
+	vhci = hcd_to_vhci(hcd);
 
-	spin_lock_irqsave(&the_controller->lock, flags);
+	spin_lock_irqsave(&vhci->lock, flags);
 
-	for (rhport = 0; rhport < VHCI_NPORTS; rhport++)
-		if (the_controller->port_status[rhport] &
-		    USB_PORT_STAT_CONNECTION)
+	for (rhport = 0; rhport < VHCI_HC_PORTS; rhport++)
+		if (vhci->port_status[rhport] & USB_PORT_STAT_CONNECTION)
 			connected += 1;
 
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	spin_unlock_irqrestore(&vhci->lock, flags);
 
 	if (connected > 0) {
 		dev_info(&pdev->dev,
@@ -1106,6 +1175,8 @@ static int vhci_hcd_resume(struct platform_device *pdev)
 	dev_dbg(&pdev->dev, "%s\n", __func__);
 
 	hcd = platform_get_drvdata(pdev);
+	if (!hcd)
+		return 0;
 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
 	usb_hcd_poll_rh_status(hcd);
 
@@ -1129,52 +1200,78 @@ static struct platform_driver vhci_driver = {
 	},
 };
 
-/*
- * The VHCI 'device' is 'virtual'; not a real plug&play hardware.
- * We need to add this virtual device as a platform device arbitrarily:
- *	1. platform_device_register()
- */
-static void the_pdev_release(struct device *dev)
+static int add_platform_device(int id)
 {
+	struct platform_device *pdev;
+	int dev_nr;
+
+	if (id == 0)
+		dev_nr = -1;
+	else
+		dev_nr = id;
+
+	pdev = platform_device_register_simple(driver_name, dev_nr, NULL, 0);
+	if (IS_ERR(pdev))
+		return PTR_ERR(pdev);
+
+	*(vhci_pdevs + id) = pdev;
+	return 0;
 }
 
-static struct platform_device the_pdev = {
-	/* should be the same name as driver_name */
-	.name = driver_name,
-	.id = -1,
-	.dev = {
-		.release = the_pdev_release,
-	},
-};
+static void del_platform_devices(void)
+{
+	struct platform_device *pdev;
+	int i;
+
+	for (i = 0; i < vhci_num_controllers; i++) {
+		pdev = *(vhci_pdevs + i);
+		if (pdev != NULL)
+			platform_device_unregister(pdev);
+		*(vhci_pdevs + i) = NULL;
+	}
+	sysfs_remove_link(&platform_bus.kobj, driver_name);
+}
 
 static int __init vhci_hcd_init(void)
 {
-	int ret;
+	int i, ret;
 
 	if (usb_disabled())
 		return -ENODEV;
 
+	if (vhci_num_controllers < 1)
+		vhci_num_controllers = 1;
+
+	vhci_pdevs = kcalloc(vhci_num_controllers, sizeof(void *), GFP_KERNEL);
+	if (vhci_pdevs == NULL)
+		return -ENOMEM;
+
 	ret = platform_driver_register(&vhci_driver);
 	if (ret)
 		goto err_driver_register;
 
-	ret = platform_device_register(&the_pdev);
-	if (ret)
-		goto err_platform_device_register;
+	for (i = 0; i < vhci_num_controllers; i++) {
+		ret = add_platform_device(i);
+		if (ret)
+			goto err_platform_device_register;
+	}
 
 	pr_info(DRIVER_DESC " v" USBIP_VERSION "\n");
 	return ret;
 
 err_platform_device_register:
+	del_platform_devices();
 	platform_driver_unregister(&vhci_driver);
 err_driver_register:
+	kfree(vhci_pdevs);
 	return ret;
 }
 
 static void __exit vhci_hcd_exit(void)
 {
-	platform_device_unregister(&the_pdev);
+	del_platform_devices();
 	platform_driver_unregister(&vhci_driver);
+	kfree(vhci_pdevs);
 }
 
 module_init(vhci_hcd_init);
diff --git a/drivers/usb/usbip/vhci_rx.c b/drivers/usb/usbip/vhci_rx.c
index d656e0edc3d5..fc2d319e2360 100644
--- a/drivers/usb/usbip/vhci_rx.c
+++ b/drivers/usb/usbip/vhci_rx.c
@@ -70,6 +70,7 @@ struct urb *pickup_urb_and_free_priv(struct vhci_device *vdev, __u32 seqnum)
 static void vhci_recv_ret_submit(struct vhci_device *vdev,
 				 struct usbip_header *pdu)
 {
+	struct vhci_hcd *vhci = vdev_to_vhci(vdev);
 	struct usbip_device *ud = &vdev->ud;
 	struct urb *urb;
 	unsigned long flags;
@@ -81,7 +82,7 @@ static void vhci_recv_ret_submit(struct vhci_device *vdev,
 	if (!urb) {
 		pr_err("cannot find a urb of seqnum %u\n", pdu->base.seqnum);
 		pr_info("max seqnum %d\n",
-			atomic_read(&the_controller->seqnum));
+			atomic_read(&vhci->seqnum));
 		usbip_event_add(ud, VDEV_EVENT_ERROR_TCP);
 		return;
 	}
@@ -105,11 +106,11 @@ static void vhci_recv_ret_submit(struct vhci_device *vdev,
 
 	usbip_dbg_vhci_rx("now giveback urb %p\n", urb);
 
-	spin_lock_irqsave(&the_controller->lock, flags);
-	usb_hcd_unlink_urb_from_ep(vhci_to_hcd(the_controller), urb);
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	spin_lock_irqsave(&vhci->lock, flags);
+	usb_hcd_unlink_urb_from_ep(vhci_to_hcd(vhci), urb);
+	spin_unlock_irqrestore(&vhci->lock, flags);
 
-	usb_hcd_giveback_urb(vhci_to_hcd(the_controller), urb, urb->status);
+	usb_hcd_giveback_urb(vhci_to_hcd(vhci), urb, urb->status);
 
 	usbip_dbg_vhci_rx("Leave\n");
 }
@@ -142,6 +143,7 @@ static struct vhci_unlink *dequeue_pending_unlink(struct vhci_device *vdev,
 static void vhci_recv_ret_unlink(struct vhci_device *vdev,
 				 struct usbip_header *pdu)
 {
+	struct vhci_hcd *vhci = vdev_to_vhci(vdev);
 	struct vhci_unlink *unlink;
 	struct urb *urb;
 	unsigned long flags;
@@ -174,12 +176,11 @@ static void vhci_recv_ret_unlink(struct vhci_device *vdev,
 		urb->status = pdu->u.ret_unlink.status;
 		pr_info("urb->status %d\n", urb->status);
 
-		spin_lock_irqsave(&the_controller->lock, flags);
-		usb_hcd_unlink_urb_from_ep(vhci_to_hcd(the_controller), urb);
-		spin_unlock_irqrestore(&the_controller->lock, flags);
+		spin_lock_irqsave(&vhci->lock, flags);
+		usb_hcd_unlink_urb_from_ep(vhci_to_hcd(vhci), urb);
+		spin_unlock_irqrestore(&vhci->lock, flags);
 
-		usb_hcd_giveback_urb(vhci_to_hcd(the_controller), urb,
-				     urb->status);
+		usb_hcd_giveback_urb(vhci_to_hcd(vhci), urb, urb->status);
 	}
 
 	kfree(unlink);
diff --git a/drivers/usb/usbip/vhci_sysfs.c b/drivers/usb/usbip/vhci_sysfs.c
index 5b5462eb1665..c404017c1b5a 100644
--- a/drivers/usb/usbip/vhci_sysfs.c
+++ b/drivers/usb/usbip/vhci_sysfs.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (C) 2003-2008 Takahiro Hirofuchi
+ * Copyright (C) 2015-2016 Nobuo Iwata
  *
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -20,6 +21,8 @@
 #include <linux/kthread.h>
 #include <linux/file.h>
 #include <linux/net.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
 
 #include "usbip_common.h"
 #include "vhci.h"
@@ -27,106 +30,190 @@
 /* TODO: refine locking ?*/
 
 /* Sysfs entry to show port status */
-static ssize_t status_show(struct device *dev, struct device_attribute *attr,
-			   char *out)
+static ssize_t status_show_vhci(int pdev_nr, char *out)
 {
+	struct platform_device *pdev = *(vhci_pdevs + pdev_nr);
+	struct vhci_hcd *vhci;
 	char *s = out;
 	int i = 0;
 	unsigned long flags;
 
-	BUG_ON(!the_controller || !out);
+	if (!pdev || !out) {
+		usbip_dbg_vhci_sysfs("show status error\n");
+		return 0;
+	}
+
+	vhci = hcd_to_vhci(platform_get_drvdata(pdev));
 
-	spin_lock_irqsave(&the_controller->lock, flags);
+	spin_lock_irqsave(&vhci->lock, flags);
 
 	/*
 	 * output example:
-	 * prt sta spd dev socket           local_busid
-	 * 000 004 000 000         c5a7bb80 1-2.3
-	 * 001 004 000 000         d8cee980 2-3.4
+	 * port sta spd dev      socket           local_busid
+	 * 0000 004 000 00000000         c5a7bb80 1-2.3
+	 * 0001 004 000 00000000         d8cee980 2-3.4
 	 *
 	 * IP address can be retrieved from a socket pointer address by looking
 	 * up /proc/net/{tcp,tcp6}. Also, a userland program may remember a
 	 * port number and its peer IP address.
 	 */
-	out += sprintf(out,
-		       "prt sta spd bus dev socket           local_busid\n");
-
-	for (i = 0; i < VHCI_NPORTS; i++) {
-		struct vhci_device *vdev = port_to_vdev(i);
+	for (i = 0; i < VHCI_HC_PORTS; i++) {
+		struct vhci_device *vdev = &vhci->vdev[i];
 
 		spin_lock(&vdev->ud.lock);
-		out += sprintf(out, "%03u %03u ", i, vdev->ud.status);
+		out += sprintf(out, "%04u %03u ",
+				    (pdev_nr * VHCI_HC_PORTS) + i,
+				    vdev->ud.status);
 
 		if (vdev->ud.status == VDEV_ST_USED) {
 			out += sprintf(out, "%03u %08x ",
-				       vdev->speed, vdev->devid);
-			out += sprintf(out, "%16p ", vdev->ud.tcp_socket);
-			out += sprintf(out, "%s", dev_name(&vdev->udev->dev));
+					    vdev->speed, vdev->devid);
+			out += sprintf(out, "%16p %s",
+					    vdev->ud.tcp_socket,
+					    dev_name(&vdev->udev->dev));
 
 		} else {
-			out += sprintf(out, "000 000 000 0000000000000000 0-0");
+			out += sprintf(out, "000 00000000 ");
+			out += sprintf(out, "0000000000000000 0-0");
 		}
 
 		out += sprintf(out, "\n");
 		spin_unlock(&vdev->ud.lock);
 	}
 
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	spin_unlock_irqrestore(&vhci->lock, flags);
+
+	return out - s;
+}
+
+static ssize_t status_show_not_ready(int pdev_nr, char *out)
+{
+	char *s = out;
+	int i = 0;
+
+	for (i = 0; i < VHCI_HC_PORTS; i++) {
+		out += sprintf(out, "%04u %03u ",
+				    (pdev_nr * VHCI_HC_PORTS) + i,
+				    VDEV_ST_NOTASSIGNED);
+		out += sprintf(out, "000 00000000 0000000000000000 0-0");
+		out += sprintf(out, "\n");
+	}
+	return out - s;
+}
+
+static int status_name_to_id(const char *name)
+{
+	char *c;
+	long val;
+	int ret;
+
+	c = strchr(name, '.');
+	if (c == NULL)
+		return 0;
 
+	ret = kstrtol(c+1, 10, &val);
+	if (ret < 0)
+		return ret;
+
+	return val;
+}
+
+static ssize_t status_show(struct device *dev,
+			   struct device_attribute *attr, char *out)
+{
+	char *s = out;
+	int pdev_nr;
+
+	out += sprintf(out,
+		       "port sta spd dev      socket           local_busid\n");
+
+	pdev_nr = status_name_to_id(attr->attr.name);
+	if (pdev_nr < 0)
+		out += status_show_not_ready(pdev_nr, out);
+	else
+		out += status_show_vhci(pdev_nr, out);
+
+	return out - s;
+}
+
+static ssize_t nports_show(struct device *dev, struct device_attribute *attr,
+			   char *out)
+{
+	char *s = out;
+
+	out += sprintf(out, "%d\n", VHCI_HC_PORTS * vhci_num_controllers);
 	return out - s;
 }
-static DEVICE_ATTR_RO(status);
+static DEVICE_ATTR_RO(nports);
 
 /* Sysfs entry to shutdown a virtual connection */
-static int vhci_port_disconnect(__u32 rhport)
+static int vhci_port_disconnect(struct vhci_hcd *vhci, __u32 rhport)
 {
-	struct vhci_device *vdev;
+	struct vhci_device *vdev = &vhci->vdev[rhport];
 	unsigned long flags;
 
 	usbip_dbg_vhci_sysfs("enter\n");
 
 	/* lock */
-	spin_lock_irqsave(&the_controller->lock, flags);
-
-	vdev = port_to_vdev(rhport);
-
+	spin_lock_irqsave(&vhci->lock, flags);
 	spin_lock(&vdev->ud.lock);
+
 	if (vdev->ud.status == VDEV_ST_NULL) {
 		pr_err("not connected %d\n", vdev->ud.status);
 
 		/* unlock */
 		spin_unlock(&vdev->ud.lock);
-		spin_unlock_irqrestore(&the_controller->lock, flags);
+		spin_unlock_irqrestore(&vhci->lock, flags);
 
 		return -EINVAL;
 	}
 
 	/* unlock */
 	spin_unlock(&vdev->ud.lock);
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	spin_unlock_irqrestore(&vhci->lock, flags);
 
 	usbip_event_add(&vdev->ud, VDEV_EVENT_DOWN);
 
 	return 0;
 }
 
+static int valid_port(__u32 pdev_nr, __u32 rhport)
+{
+	if (pdev_nr >= vhci_num_controllers) {
+		pr_err("pdev %u\n", pdev_nr);
+		return 0;
+	}
+	if (rhport >= VHCI_HC_PORTS) {
+		pr_err("rhport %u\n", rhport);
+		return 0;
+	}
+	return 1;
+}
+
 static ssize_t store_detach(struct device *dev, struct device_attribute *attr,
 			    const char *buf, size_t count)
 {
-	int err;
-	__u32 rhport = 0;
+	__u32 port = 0, pdev_nr = 0, rhport = 0;
+	struct usb_hcd *hcd;
+	int ret;
 
-	if (sscanf(buf, "%u", &rhport) != 1)
+	if (kstrtoint(buf, 10, &port) < 0)
 		return -EINVAL;
 
-	/* check rhport */
-	if (rhport >= VHCI_NPORTS) {
-		dev_err(dev, "invalid port %u\n", rhport);
+	pdev_nr = port_to_pdev_nr(port);
+	rhport = port_to_rhport(port);
+
+	if (!valid_port(pdev_nr, rhport))
 		return -EINVAL;
+
+	hcd = platform_get_drvdata(*(vhci_pdevs + pdev_nr));
+	if (hcd == NULL) {
+		dev_err(dev, "port is not ready %u\n", port);
+		return -EAGAIN;
 	}
 
-	err = vhci_port_disconnect(rhport);
-	if (err < 0)
+	ret = vhci_port_disconnect(hcd_to_vhci(hcd), rhport);
+	if (ret < 0)
 		return -EINVAL;
 
 	usbip_dbg_vhci_sysfs("Leave\n");
@@ -135,16 +222,12 @@ static ssize_t store_detach(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR(detach, S_IWUSR, NULL, store_detach);
 
-/* Sysfs entry to establish a virtual connection */
-static int valid_args(__u32 rhport, enum usb_device_speed speed)
+static int valid_args(__u32 pdev_nr, __u32 rhport, enum usb_device_speed speed)
 {
-	/* check rhport */
-	if (rhport >= VHCI_NPORTS) {
-		pr_err("port %u\n", rhport);
-		return -EINVAL;
+	if (!valid_port(pdev_nr, rhport)) {
+		return 0;
 	}
 
-	/* check speed */
 	switch (speed) {
 	case USB_SPEED_LOW:
 	case USB_SPEED_FULL:
@@ -154,12 +237,13 @@ static int valid_args(__u32 rhport, enum usb_device_speed speed)
 	default:
 		pr_err("Failed attach request for unsupported USB speed: %s\n",
 			usb_speed_string(speed));
-		return -EINVAL;
+		return 0;
 	}
 
-	return 0;
+	return 1;
 }
 
+/* Sysfs entry to establish a virtual connection */
 /*
  * To start a new USB/IP attachment, a userland program needs to setup a TCP
  * connection and then write its socket descriptor with remote device
@@ -174,10 +258,12 @@ static int valid_args(__u32 rhport, enum usb_device_speed speed)
 static ssize_t store_attach(struct device *dev, struct device_attribute *attr,
 			    const char *buf, size_t count)
 {
-	struct vhci_device *vdev;
 	struct socket *socket;
 	int sockfd = 0;
-	__u32 rhport = 0, devid = 0, speed = 0;
+	__u32 port = 0, pdev_nr = 0, rhport = 0, devid = 0, speed = 0;
+	struct usb_hcd *hcd;
+	struct vhci_hcd *vhci;
+	struct vhci_device *vdev;
 	int err;
 	unsigned long flags;
 
@@ -187,16 +273,28 @@ static ssize_t store_attach(struct device *dev, struct device_attribute *attr,
 	 * @devid: unique device identifier in a remote host
 	 * @speed: usb device speed in a remote host
 	 */
-	if (sscanf(buf, "%u %u %u %u", &rhport, &sockfd, &devid, &speed) != 4)
+	if (sscanf(buf, "%u %u %u %u", &port, &sockfd, &devid, &speed) != 4)
 		return -EINVAL;
+	pdev_nr = port_to_pdev_nr(port);
+	rhport = port_to_rhport(port);
 
-	usbip_dbg_vhci_sysfs("rhport(%u) sockfd(%u) devid(%u) speed(%u)\n",
-			     rhport, sockfd, devid, speed);
+	usbip_dbg_vhci_sysfs("port(%u) pdev(%d) rhport(%u)\n",
+			     port, pdev_nr, rhport);
+	usbip_dbg_vhci_sysfs("sockfd(%u) devid(%u) speed(%u)\n",
+			     sockfd, devid, speed);
 
 	/* check received parameters */
-	if (valid_args(rhport, speed) < 0)
+	if (!valid_args(pdev_nr, rhport, speed))
 		return -EINVAL;
 
+	hcd = platform_get_drvdata(*(vhci_pdevs + pdev_nr));
+	if (hcd == NULL) {
+		dev_err(dev, "port %d is not ready\n", port);
+		return -EAGAIN;
+	}
+	vhci = hcd_to_vhci(hcd);
+	vdev = &vhci->vdev[rhport];
+
 	/* Extract socket from fd. */
 	socket = sockfd_lookup(sockfd, &err);
 	if (!socket)
@@ -205,14 +303,13 @@ static ssize_t store_attach(struct device *dev, struct device_attribute *attr,
 	/* now need lock until setting vdev status as used */
 
 	/* begin a lock */
-	spin_lock_irqsave(&the_controller->lock, flags);
-	vdev = port_to_vdev(rhport);
+	spin_lock_irqsave(&vhci->lock, flags);
 	spin_lock(&vdev->ud.lock);
 
 	if (vdev->ud.status != VDEV_ST_NULL) {
 		/* end of the lock */
 		spin_unlock(&vdev->ud.lock);
-		spin_unlock_irqrestore(&the_controller->lock, flags);
+		spin_unlock_irqrestore(&vhci->lock, flags);
 
 		sockfd_put(socket);
 
@@ -220,9 +317,10 @@ static ssize_t store_attach(struct device *dev, struct device_attribute *attr,
 		return -EINVAL;
 	}
 
-	dev_info(dev,
-		 "rhport(%u) sockfd(%d) devid(%u) speed(%u) speed_str(%s)\n",
-		 rhport, sockfd, devid, speed, usb_speed_string(speed));
+	dev_info(dev, "pdev(%u) rhport(%u) sockfd(%d)\n",
+		 pdev_nr, rhport, sockfd);
+	dev_info(dev, "devid(%u) speed(%u) speed_str(%s)\n",
+		 devid, speed, usb_speed_string(speed));
 
 	vdev->devid         = devid;
 	vdev->speed         = speed;
@@ -230,26 +328,92 @@ static ssize_t store_attach(struct device *dev, struct device_attribute *attr,
 	vdev->ud.status     = VDEV_ST_NOTASSIGNED;
 
 	spin_unlock(&vdev->ud.lock);
-	spin_unlock_irqrestore(&the_controller->lock, flags);
+	spin_unlock_irqrestore(&vhci->lock, flags);
 	/* end the lock */
 
 	vdev->ud.tcp_rx = kthread_get_run(vhci_rx_loop, &vdev->ud, "vhci_rx");
 	vdev->ud.tcp_tx = kthread_get_run(vhci_tx_loop, &vdev->ud, "vhci_tx");
 
-	rh_port_connect(rhport, speed);
+	rh_port_connect(vdev, speed);
 
 	return count;
 }
 static DEVICE_ATTR(attach, S_IWUSR, NULL, store_attach);
 
-static struct attribute *dev_attrs[] = {
-	&dev_attr_status.attr,
-	&dev_attr_detach.attr,
-	&dev_attr_attach.attr,
-	&dev_attr_usbip_debug.attr,
-	NULL,
+#define MAX_STATUS_NAME 16
+
+struct status_attr {
+	struct device_attribute attr;
+	char name[MAX_STATUS_NAME+1];
 };
 
-const struct attribute_group dev_attr_group = {
-	.attrs = dev_attrs,
+static struct status_attr *status_attrs;
+
+static void set_status_attr(int id)
+{
+	struct status_attr *status;
+
+	status = status_attrs + id;
+	if (id == 0)
+		strcpy(status->name, "status");
+	else
+		snprintf(status->name, MAX_STATUS_NAME+1, "status.%d", id);
+	status->attr.attr.name = status->name;
+	status->attr.attr.mode = S_IRUGO;
+	status->attr.show = status_show;
+}
+
+static int init_status_attrs(void)
+{
+	int id;
+
+	status_attrs = kcalloc(vhci_num_controllers, sizeof(struct status_attr),
+			       GFP_KERNEL);
+	if (status_attrs == NULL)
+		return -ENOMEM;
+
+	for (id = 0; id < vhci_num_controllers; id++)
+		set_status_attr(id);
+
+	return 0;
+}
+
+static void finish_status_attrs(void)
+{
+	kfree(status_attrs);
+}
+
+struct attribute_group vhci_attr_group = {
+	.attrs = NULL,
 };
+
+int vhci_init_attr_group(void)
+{
+	struct attribute **attrs;
+	int ret, i;
+
+	attrs = kcalloc((vhci_num_controllers + 5), sizeof(struct attribute *),
+			GFP_KERNEL);
+	if (attrs == NULL)
+		return -ENOMEM;
+
+	ret = init_status_attrs();
+	if (ret) {
+		kfree(attrs);
+		return ret;
+	}
+	*attrs = &dev_attr_nports.attr;
+	*(attrs + 1) = &dev_attr_detach.attr;
+	*(attrs + 2) = &dev_attr_attach.attr;
+	*(attrs + 3) = &dev_attr_usbip_debug.attr;
+	for (i = 0; i < vhci_num_controllers; i++)
+		*(attrs + i + 4) = &((status_attrs + i)->attr.attr);
+	vhci_attr_group.attrs = attrs;
+	return 0;
+}
+
+void vhci_finish_attr_group(void)
+{
+	finish_status_attrs();
+	kfree(vhci_attr_group.attrs);
+}
diff --git a/drivers/usb/usbip/vudc_dev.c b/drivers/usb/usbip/vudc_dev.c
index 8994a13819ab..7091848df6c8 100644
--- a/drivers/usb/usbip/vudc_dev.c
+++ b/drivers/usb/usbip/vudc_dev.c
@@ -450,7 +450,7 @@ static void vudc_shutdown(struct usbip_device *ud)
 	if (ud->tcp_socket)
 		kernel_sock_shutdown(ud->tcp_socket, SHUT_RDWR);
 
-	if (ud->tcp_tx) {
+	if (ud->tcp_rx) {
 		kthread_stop_put(ud->tcp_rx);
 		ud->tcp_rx = NULL;
 	}
diff --git a/drivers/usb/usbip/vudc_rx.c b/drivers/usb/usbip/vudc_rx.c
index 344bd9473475..e429b59f6f8a 100644
--- a/drivers/usb/usbip/vudc_rx.c
+++ b/drivers/usb/usbip/vudc_rx.c
@@ -142,7 +142,7 @@ static int v_recv_cmd_submit(struct vudc *udc,
 	urb_p->urb->status = -EINPROGRESS;
 
 	/* FIXME: more pipe setup to please usbip_common */
-	urb_p->urb->pipe &= ~(11 << 30);
+	urb_p->urb->pipe &= ~(3 << 30);
 	switch (urb_p->ep->type) {
 	case USB_ENDPOINT_XFER_BULK:
 		urb_p->urb->pipe |= (PIPE_BULK << 30);
diff --git a/drivers/usb/wusbcore/cbaf.c b/drivers/usb/wusbcore/cbaf.c
index da1b872918b5..fb70cbef0671 100644
--- a/drivers/usb/wusbcore/cbaf.c
+++ b/drivers/usb/wusbcore/cbaf.c
@@ -610,8 +610,7 @@ static int cbaf_probe(struct usb_interface *iface,
 	cbaf->usb_iface = usb_get_intf(iface);
 	result = cbaf_check(cbaf);
 	if (result < 0) {
-		dev_err(dev, "This device is not WUSB-CBAF compliant"
-			"and is not supported yet.\n");
+		dev_err(dev, "This device is not WUSB-CBAF compliant and is not supported yet.\n");
 		goto error_check;
 	}
 
diff --git a/drivers/usb/wusbcore/crypto.c b/drivers/usb/wusbcore/crypto.c
index 33acd1599e99..79b2b628066d 100644
--- a/drivers/usb/wusbcore/crypto.c
+++ b/drivers/usb/wusbcore/crypto.c
@@ -229,10 +229,8 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc,
 		zero_padding = sizeof(struct aes_ccm_block) - zero_padding;
 	dst_size = blen + sizeof(b0) + sizeof(b1) + zero_padding;
 	dst_buf = kzalloc(dst_size, GFP_KERNEL);
-	if (dst_buf == NULL) {
-		printk(KERN_ERR "E: can't alloc destination buffer\n");
+	if (!dst_buf)
 		goto error_dst_buf;
-	}
 
 	memset(iv, 0, sizeof(iv));
 
diff --git a/drivers/usb/wusbcore/security.c b/drivers/usb/wusbcore/security.c
index b66faaf3e842..8c9421b69da0 100644
--- a/drivers/usb/wusbcore/security.c
+++ b/drivers/usb/wusbcore/security.c
@@ -374,10 +374,8 @@ int wusb_dev_4way_handshake(struct wusbhc *wusbhc, struct wusb_dev *wusb_dev,
 	struct wusb_keydvt_out keydvt_out;
 
 	hs = kcalloc(3, sizeof(hs[0]), GFP_KERNEL);
-	if (hs == NULL) {
-		dev_err(dev, "can't allocate handshake data\n");
+	if (!hs)
 		goto error_kzalloc;
-	}
 
 	/* We need to turn encryption before beginning the 4way
 	 * hshake (WUSB1.0[.3.2.2]) */
diff --git a/drivers/usb/wusbcore/wa-nep.c b/drivers/usb/wusbcore/wa-nep.c
index 60a10d21947d..ed4622279c63 100644
--- a/drivers/usb/wusbcore/wa-nep.c
+++ b/drivers/usb/wusbcore/wa-nep.c
@@ -271,16 +271,11 @@ int wa_nep_create(struct wahc *wa, struct usb_interface *iface)
 	epd = &iface->cur_altsetting->endpoint[0].desc;
 	wa->nep_buffer_size = 1024;
 	wa->nep_buffer = kmalloc(wa->nep_buffer_size, GFP_KERNEL);
-	if (wa->nep_buffer == NULL) {
-		dev_err(dev,
-			"Unable to allocate notification's read buffer\n");
+	if (!wa->nep_buffer)
 		goto error_nep_buffer;
-	}
 	wa->nep_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (wa->nep_urb == NULL) {
-		dev_err(dev, "Unable to allocate notification URB\n");
+	if (wa->nep_urb == NULL)
 		goto error_urb_alloc;
-	}
 	usb_fill_int_urb(wa->nep_urb, usb_dev,
 			 usb_rcvintpipe(usb_dev, epd->bEndpointAddress),
 			 wa->nep_buffer, wa->nep_buffer_size,
diff --git a/drivers/usb/wusbcore/wa-xfer.c b/drivers/usb/wusbcore/wa-xfer.c
index 69af4fd9e072..167fcc71f5f6 100644
--- a/drivers/usb/wusbcore/wa-xfer.c
+++ b/drivers/usb/wusbcore/wa-xfer.c
@@ -2865,10 +2865,8 @@ int wa_dti_start(struct wahc *wa)
 		goto out;
 
 	wa->dti_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (wa->dti_urb == NULL) {
-		dev_err(dev, "Can't allocate DTI URB\n");
+	if (wa->dti_urb == NULL)
 		goto error_dti_urb_alloc;
-	}
 	usb_fill_bulk_urb(
 		wa->dti_urb, wa->usb_dev,
 		usb_rcvbulkpipe(wa->usb_dev, 0x80 | dti_epd->bEndpointAddress),
diff --git a/drivers/uwb/hwa-rc.c b/drivers/uwb/hwa-rc.c
index 0257f35cfb9d..0aa6c3c29d17 100644
--- a/drivers/uwb/hwa-rc.c
+++ b/drivers/uwb/hwa-rc.c
@@ -701,10 +701,8 @@ static int hwarc_neep_init(struct uwb_rc *rc)
 		goto error_rd_buffer;
 	}
 	hwarc->neep_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (hwarc->neep_urb == NULL) {
-		dev_err(dev, "Unable to allocate notification URB\n");
+	if (hwarc->neep_urb == NULL)
 		goto error_urb_alloc;
-	}
 	usb_fill_int_urb(hwarc->neep_urb, usb_dev,
 			 usb_rcvintpipe(usb_dev, epd->bEndpointAddress),
 			 hwarc->rd_buffer, PAGE_SIZE,
diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c
index 688691d9058d..65d4a3015542 100644
--- a/drivers/vfio/pci/vfio_pci_config.c
+++ b/drivers/vfio/pci/vfio_pci_config.c
@@ -70,7 +70,7 @@ static const u8 pci_cap_length[PCI_CAP_ID_MAX + 1] = {
 
 /*
  * Lengths of PCIe/PCI-X Extended Config Capabilities
- *   0: Removed or masked from the user visible capabilty list
+ *   0: Removed or masked from the user visible capability list
  *   FF: Variable length
  */
 static const u16 pci_ext_cap_length[PCI_EXT_CAP_ID_MAX + 1] = {
@@ -355,7 +355,7 @@ static int alloc_perm_bits(struct perm_bits *perm, int size)
 	 * ignore whether a read/write exceeds the defined capability
 	 * structure.  We can do this because:
 	 *  - Standard config space is already dword aligned
-	 *  - Capabilities are all dword alinged (bits 0:1 of next reserved)
+	 *  - Capabilities are all dword aligned (bits 0:1 of next reserved)
 	 *  - Express capabilities defined as dword aligned
 	 */
 	size = round_up(size, 4);
@@ -804,6 +804,40 @@ static int __init init_pci_cap_pcix_perm(struct perm_bits *perm)
 	return 0;
 }
 
+static int vfio_exp_config_write(struct vfio_pci_device *vdev, int pos,
+				 int count, struct perm_bits *perm,
+				 int offset, __le32 val)
+{
+	__le16 *ctrl = (__le16 *)(vdev->vconfig + pos -
+				  offset + PCI_EXP_DEVCTL);
+
+	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
+	if (count < 0)
+		return count;
+
+	/*
+	 * The FLR bit is virtualized, if set and the device supports PCIe
+	 * FLR, issue a reset_function.  Regardless, clear the bit, the spec
+	 * requires it to be always read as zero.  NB, reset_function might
+	 * not use a PCIe FLR, we don't have that level of granularity.
+	 */
+	if (*ctrl & cpu_to_le16(PCI_EXP_DEVCTL_BCR_FLR)) {
+		u32 cap;
+		int ret;
+
+		*ctrl &= ~cpu_to_le16(PCI_EXP_DEVCTL_BCR_FLR);
+
+		ret = pci_user_read_config_dword(vdev->pdev,
+						 pos - offset + PCI_EXP_DEVCAP,
+						 &cap);
+
+		if (!ret && (cap & PCI_EXP_DEVCAP_FLR))
+			pci_try_reset_function(vdev->pdev);
+	}
+
+	return count;
+}
+
 /* Permissions for PCI Express capability */
 static int __init init_pci_cap_exp_perm(struct perm_bits *perm)
 {
@@ -811,26 +845,64 @@ static int __init init_pci_cap_exp_perm(struct perm_bits *perm)
 	if (alloc_perm_bits(perm, PCI_CAP_EXP_ENDPOINT_SIZEOF_V2))
 		return -ENOMEM;
 
+	perm->writefn = vfio_exp_config_write;
+
 	p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE);
 
 	/*
-	 * Allow writes to device control fields (includes FLR!)
-	 * but not to devctl_phantom which could confuse IOMMU
-	 * or to the ARI bit in devctl2 which is set at probe time
+	 * Allow writes to device control fields, except devctl_phantom,
+	 * which could confuse IOMMU, and the ARI bit in devctl2, which
+	 * is set at probe time.  FLR gets virtualized via our writefn.
 	 */
-	p_setw(perm, PCI_EXP_DEVCTL, NO_VIRT, ~PCI_EXP_DEVCTL_PHANTOM);
+	p_setw(perm, PCI_EXP_DEVCTL,
+	       PCI_EXP_DEVCTL_BCR_FLR, ~PCI_EXP_DEVCTL_PHANTOM);
 	p_setw(perm, PCI_EXP_DEVCTL2, NO_VIRT, ~PCI_EXP_DEVCTL2_ARI);
 	return 0;
 }
 
+static int vfio_af_config_write(struct vfio_pci_device *vdev, int pos,
+				int count, struct perm_bits *perm,
+				int offset, __le32 val)
+{
+	u8 *ctrl = vdev->vconfig + pos - offset + PCI_AF_CTRL;
+
+	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
+	if (count < 0)
+		return count;
+
+	/*
+	 * The FLR bit is virtualized, if set and the device supports AF
+	 * FLR, issue a reset_function.  Regardless, clear the bit, the spec
+	 * requires it to be always read as zero.  NB, reset_function might
+	 * not use an AF FLR, we don't have that level of granularity.
+	 */
+	if (*ctrl & PCI_AF_CTRL_FLR) {
+		u8 cap;
+		int ret;
+
+		*ctrl &= ~PCI_AF_CTRL_FLR;
+
+		ret = pci_user_read_config_byte(vdev->pdev,
+						pos - offset + PCI_AF_CAP,
+						&cap);
+
+		if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP))
+			pci_try_reset_function(vdev->pdev);
+	}
+
+	return count;
+}
+
 /* Permissions for Advanced Function capability */
 static int __init init_pci_cap_af_perm(struct perm_bits *perm)
 {
 	if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_AF]))
 		return -ENOMEM;
 
+	perm->writefn = vfio_af_config_write;
+
 	p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE);
-	p_setb(perm, PCI_AF_CTRL, NO_VIRT, PCI_AF_CTRL_FLR);
+	p_setb(perm, PCI_AF_CTRL, PCI_AF_CTRL_FLR, PCI_AF_CTRL_FLR);
 	return 0;
 }
 
@@ -1516,10 +1588,10 @@ static int vfio_ecap_init(struct vfio_pci_device *vdev)
  * space which tracks reads and writes to bits that we emulate for
  * the user.  Initial values filled from device.
  *
- * Using shared stuct perm_bits between all vfio-pci devices saves
+ * Using shared struct perm_bits between all vfio-pci devices saves
  * us from allocating cfg_size buffers for virt and write for every
  * device.  We could remove vconfig and allocate individual buffers
- * for each area requring emulated bits, but the array of pointers
+ * for each area requiring emulated bits, but the array of pointers
  * would be comparable in size (at least for standard config space).
  */
 int vfio_config_init(struct vfio_pci_device *vdev)
diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c
index 15ecfc9c5f6c..c2e60893cd09 100644
--- a/drivers/vfio/pci/vfio_pci_intrs.c
+++ b/drivers/vfio/pci/vfio_pci_intrs.c
@@ -250,6 +250,7 @@ static irqreturn_t vfio_msihandler(int irq, void *arg)
 static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
 {
 	struct pci_dev *pdev = vdev->pdev;
+	unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI;
 	int ret;
 
 	if (!is_irq_none(vdev))
@@ -259,35 +260,13 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
 	if (!vdev->ctx)
 		return -ENOMEM;
 
-	if (msix) {
-		int i;
-
-		vdev->msix = kzalloc(nvec * sizeof(struct msix_entry),
-				     GFP_KERNEL);
-		if (!vdev->msix) {
-			kfree(vdev->ctx);
-			return -ENOMEM;
-		}
-
-		for (i = 0; i < nvec; i++)
-			vdev->msix[i].entry = i;
-
-		ret = pci_enable_msix_range(pdev, vdev->msix, 1, nvec);
-		if (ret < nvec) {
-			if (ret > 0)
-				pci_disable_msix(pdev);
-			kfree(vdev->msix);
-			kfree(vdev->ctx);
-			return ret;
-		}
-	} else {
-		ret = pci_enable_msi_range(pdev, 1, nvec);
-		if (ret < nvec) {
-			if (ret > 0)
-				pci_disable_msi(pdev);
-			kfree(vdev->ctx);
-			return ret;
-		}
+	/* return the number of supported vectors if we can't get all: */
+	ret = pci_alloc_irq_vectors(pdev, 1, nvec, flag);
+	if (ret < nvec) {
+		if (ret > 0)
+			pci_free_irq_vectors(pdev);
+		kfree(vdev->ctx);
+		return ret;
 	}
 
 	vdev->num_ctx = nvec;
@@ -315,7 +294,7 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
 	if (vector < 0 || vector >= vdev->num_ctx)
 		return -EINVAL;
 
-	irq = msix ? vdev->msix[vector].vector : pdev->irq + vector;
+	irq = pci_irq_vector(pdev, vector);
 
 	if (vdev->ctx[vector].trigger) {
 		free_irq(irq, vdev->ctx[vector].trigger);
@@ -408,11 +387,14 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
 
 	vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix);
 
-	if (msix) {
-		pci_disable_msix(vdev->pdev);
-		kfree(vdev->msix);
-	} else
-		pci_disable_msi(pdev);
+	pci_free_irq_vectors(pdev);
+
+	/*
+	 * Both disable paths above use pci_intx_for_msi() to clear DisINTx
+	 * via their shutdown paths.  Restore for NoINTx devices.
+	 */
+	if (vdev->nointx)
+		pci_intx(pdev, 0);
 
 	vdev->irq_type = VFIO_PCI_NUM_IRQS;
 	vdev->num_ctx = 0;
@@ -564,67 +546,80 @@ static int vfio_pci_set_msi_trigger(struct vfio_pci_device *vdev,
 }
 
 static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx,
-					   uint32_t flags, void *data)
+					   unsigned int count, uint32_t flags,
+					   void *data)
 {
-	int32_t fd = *(int32_t *)data;
-
-	if (!(flags & VFIO_IRQ_SET_DATA_TYPE_MASK))
-		return -EINVAL;
-
 	/* DATA_NONE/DATA_BOOL enables loopback testing */
 	if (flags & VFIO_IRQ_SET_DATA_NONE) {
-		if (*ctx)
-			eventfd_signal(*ctx, 1);
-		return 0;
+		if (*ctx) {
+			if (count) {
+				eventfd_signal(*ctx, 1);
+			} else {
+				eventfd_ctx_put(*ctx);
+				*ctx = NULL;
+			}
+			return 0;
+		}
 	} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
-		uint8_t trigger = *(uint8_t *)data;
+		uint8_t trigger;
+
+		if (!count)
+			return -EINVAL;
+
+		trigger = *(uint8_t *)data;
 		if (trigger && *ctx)
 			eventfd_signal(*ctx, 1);
-		return 0;
-	}
 
-	/* Handle SET_DATA_EVENTFD */
-	if (fd == -1) {
-		if (*ctx)
-			eventfd_ctx_put(*ctx);
-		*ctx = NULL;
 		return 0;
-	} else if (fd >= 0) {
-		struct eventfd_ctx *efdctx;
-		efdctx = eventfd_ctx_fdget(fd);
-		if (IS_ERR(efdctx))
-			return PTR_ERR(efdctx);
-		if (*ctx)
-			eventfd_ctx_put(*ctx);
-		*ctx = efdctx;
+	} else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		int32_t fd;
+
+		if (!count)
+			return -EINVAL;
+
+		fd = *(int32_t *)data;
+		if (fd == -1) {
+			if (*ctx)
+				eventfd_ctx_put(*ctx);
+			*ctx = NULL;
+		} else if (fd >= 0) {
+			struct eventfd_ctx *efdctx;
+
+			efdctx = eventfd_ctx_fdget(fd);
+			if (IS_ERR(efdctx))
+				return PTR_ERR(efdctx);
+
+			if (*ctx)
+				eventfd_ctx_put(*ctx);
+
+			*ctx = efdctx;
+		}
 		return 0;
-	} else
-		return -EINVAL;
+	}
+
+	return -EINVAL;
 }
 
 static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev,
 				    unsigned index, unsigned start,
 				    unsigned count, uint32_t flags, void *data)
 {
-	if (index != VFIO_PCI_ERR_IRQ_INDEX)
+	if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1)
 		return -EINVAL;
 
-	/*
-	 * We should sanitize start & count, but that wasn't caught
-	 * originally, so this IRQ index must forever ignore them :-(
-	 */
-
-	return vfio_pci_set_ctx_trigger_single(&vdev->err_trigger, flags, data);
+	return vfio_pci_set_ctx_trigger_single(&vdev->err_trigger,
+					       count, flags, data);
 }
 
 static int vfio_pci_set_req_trigger(struct vfio_pci_device *vdev,
 				    unsigned index, unsigned start,
 				    unsigned count, uint32_t flags, void *data)
 {
-	if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count != 1)
+	if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1)
 		return -EINVAL;
 
-	return vfio_pci_set_ctx_trigger_single(&vdev->req_trigger, flags, data);
+	return vfio_pci_set_ctx_trigger_single(&vdev->req_trigger,
+					       count, flags, data);
 }
 
 int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags,
diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h
index 2128de86c80d..f561ac1c78a0 100644
--- a/drivers/vfio/pci/vfio_pci_private.h
+++ b/drivers/vfio/pci/vfio_pci_private.h
@@ -72,7 +72,6 @@ struct vfio_pci_device {
 	struct perm_bits	*msi_perm;
 	spinlock_t		irqlock;
 	struct mutex		igate;
-	struct msix_entry	*msix;
 	struct vfio_pci_irq_ctx	*ctx;
 	int			num_ctx;
 	int			irq_type;
diff --git a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c
index d4030d0c38e9..bcd419cfd79c 100644
--- a/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c
+++ b/drivers/vfio/platform/reset/vfio_platform_amdxgbe.c
@@ -56,7 +56,7 @@ static void xmdio_write(void *ioaddr, unsigned int mmd,
 	iowrite32(value, ioaddr + ((mmd_address & 0xff) << 2));
 }
 
-int vfio_platform_amdxgbe_reset(struct vfio_platform_device *vdev)
+static int vfio_platform_amdxgbe_reset(struct vfio_platform_device *vdev)
 {
 	struct vfio_platform_region *xgmac_regs = &vdev->regions[0];
 	struct vfio_platform_region *xpcs_regs = &vdev->regions[1];
diff --git a/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c b/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c
index e3d3d948e661..49e5df6e8f29 100644
--- a/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c
+++ b/drivers/vfio/platform/reset/vfio_platform_calxedaxgmac.c
@@ -57,7 +57,7 @@ static inline void xgmac_mac_disable(void __iomem *ioaddr)
 	writel(value, ioaddr + XGMAC_CONTROL);
 }
 
-int vfio_platform_calxedaxgmac_reset(struct vfio_platform_device *vdev)
+static int vfio_platform_calxedaxgmac_reset(struct vfio_platform_device *vdev)
 {
 	struct vfio_platform_region *reg = &vdev->regions[0];
 
diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index 1cf2d462b53d..d78142830754 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -73,7 +73,7 @@ static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
 	return WARN_ON(!vdev->acpihid) ? -EINVAL : 0;
 }
 
-int vfio_platform_acpi_call_reset(struct vfio_platform_device *vdev,
+static int vfio_platform_acpi_call_reset(struct vfio_platform_device *vdev,
 				  const char **extra_dbg)
 {
 #ifdef CONFIG_ACPI
@@ -95,7 +95,7 @@ int vfio_platform_acpi_call_reset(struct vfio_platform_device *vdev,
 #endif
 }
 
-bool vfio_platform_acpi_has_reset(struct vfio_platform_device *vdev)
+static bool vfio_platform_acpi_has_reset(struct vfio_platform_device *vdev)
 {
 #ifdef CONFIG_ACPI
 	struct device *dev = vdev->device;
@@ -637,7 +637,7 @@ static const struct vfio_device_ops vfio_platform_ops = {
 	.mmap		= vfio_platform_mmap,
 };
 
-int vfio_platform_of_probe(struct vfio_platform_device *vdev,
+static int vfio_platform_of_probe(struct vfio_platform_device *vdev,
 			   struct device *dev)
 {
 	int ret;
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index 9d6320e8ff3e..6e29d053843d 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -88,7 +88,7 @@ struct vhost_scsi_cmd {
 	struct scatterlist *tvc_prot_sgl;
 	struct page **tvc_upages;
 	/* Pointer to response header iovec */
-	struct iovec *tvc_resp_iov;
+	struct iovec tvc_resp_iov;
 	/* Pointer to vhost_scsi for our device */
 	struct vhost_scsi *tvc_vhost;
 	/* Pointer to vhost_virtqueue for the cmd */
@@ -547,7 +547,7 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
 		memcpy(v_rsp.sense, cmd->tvc_sense_buf,
 		       se_cmd->scsi_sense_length);
 
-		iov_iter_init(&iov_iter, READ, cmd->tvc_resp_iov,
+		iov_iter_init(&iov_iter, READ, &cmd->tvc_resp_iov,
 			      cmd->tvc_in_iovs, sizeof(v_rsp));
 		ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter);
 		if (likely(ret == sizeof(v_rsp))) {
@@ -1044,7 +1044,7 @@ vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
 		}
 		cmd->tvc_vhost = vs;
 		cmd->tvc_vq = vq;
-		cmd->tvc_resp_iov = &vq->iov[out];
+		cmd->tvc_resp_iov = vq->iov[out];
 		cmd->tvc_in_iovs = in;
 
 		pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c
index 388eec4e1a90..3cc98c07dcd3 100644
--- a/drivers/vhost/test.c
+++ b/drivers/vhost/test.c
@@ -220,20 +220,20 @@ static long vhost_test_reset_owner(struct vhost_test *n)
 {
 	void *priv = NULL;
 	long err;
-	struct vhost_memory *memory;
+	struct vhost_umem *umem;
 
 	mutex_lock(&n->dev.mutex);
 	err = vhost_dev_check_owner(&n->dev);
 	if (err)
 		goto done;
-	memory = vhost_dev_reset_owner_prepare();
-	if (!memory) {
+	umem = vhost_dev_reset_owner_prepare();
+	if (!umem) {
 		err = -ENOMEM;
 		goto done;
 	}
 	vhost_test_stop(n, &priv);
 	vhost_test_flush(n);
-	vhost_dev_reset_owner(&n->dev, memory);
+	vhost_dev_reset_owner(&n->dev, umem);
 done:
 	mutex_unlock(&n->dev.mutex);
 	return err;
@@ -322,18 +322,7 @@ static struct miscdevice vhost_test_misc = {
 	"vhost-test",
 	&vhost_test_fops,
 };
-
-static int vhost_test_init(void)
-{
-	return misc_register(&vhost_test_misc);
-}
-module_init(vhost_test_init);
-
-static void vhost_test_exit(void)
-{
-	misc_deregister(&vhost_test_misc);
-}
-module_exit(vhost_test_exit);
+module_misc_device(vhost_test_misc);
 
 MODULE_VERSION("0.0.1");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
index 0ddf3a2dbfc4..e3b30ea9ece5 100644
--- a/drivers/vhost/vsock.c
+++ b/drivers/vhost/vsock.c
@@ -307,6 +307,8 @@ static void vhost_vsock_handle_tx_kick(struct vhost_work *work)
 
 	vhost_disable_notify(&vsock->dev, vq);
 	for (;;) {
+		u32 len;
+
 		if (!vhost_vsock_more_replies(vsock)) {
 			/* Stop tx until the device processes already
 			 * pending replies.  Leave tx virtqueue
@@ -334,13 +336,15 @@ static void vhost_vsock_handle_tx_kick(struct vhost_work *work)
 			continue;
 		}
 
+		len = pkt->len;
+
 		/* Only accept correctly addressed packets */
 		if (le64_to_cpu(pkt->hdr.src_cid) == vsock->guest_cid)
 			virtio_transport_recv_pkt(pkt);
 		else
 			virtio_transport_free_pkt(pkt);
 
-		vhost_add_used(vq, head, sizeof(pkt->hdr) + pkt->len);
+		vhost_add_used(vq, head, sizeof(pkt->hdr) + len);
 		added = true;
 	}
 
diff --git a/drivers/video/backlight/pwm_bl.c b/drivers/video/backlight/pwm_bl.c
index b2b366bb0f97..12614006211e 100644
--- a/drivers/video/backlight/pwm_bl.c
+++ b/drivers/video/backlight/pwm_bl.c
@@ -55,7 +55,7 @@ static void pwm_backlight_power_on(struct pwm_bl_data *pb, int brightness)
 		dev_err(pb->dev, "failed to enable power supply\n");
 
 	if (pb->enable_gpio)
-		gpiod_set_value(pb->enable_gpio, 1);
+		gpiod_set_value_cansleep(pb->enable_gpio, 1);
 
 	pwm_enable(pb->pwm);
 	pb->enabled = true;
@@ -70,7 +70,7 @@ static void pwm_backlight_power_off(struct pwm_bl_data *pb)
 	pwm_disable(pb->pwm);
 
 	if (pb->enable_gpio)
-		gpiod_set_value(pb->enable_gpio, 0);
+		gpiod_set_value_cansleep(pb->enable_gpio, 0);
 
 	regulator_disable(pb->power_supply);
 	pb->enabled = false;
diff --git a/drivers/video/backlight/tosa_bl.c b/drivers/video/backlight/tosa_bl.c
index 83742d806391..97067597e6bf 100644
--- a/drivers/video/backlight/tosa_bl.c
+++ b/drivers/video/backlight/tosa_bl.c
@@ -163,7 +163,6 @@ MODULE_DEVICE_TABLE(i2c, tosa_bl_id);
 static struct i2c_driver tosa_bl_driver = {
 	.driver = {
 		.name		= "tosa-bl",
-		.owner		= THIS_MODULE,
 		.pm		= &tosa_bl_pm_ops,
 	},
 	.probe		= tosa_bl_probe,
diff --git a/drivers/video/fbdev/au1200fb.c b/drivers/video/fbdev/au1200fb.c
index f9507b1894df..6c2b2ca4a909 100644
--- a/drivers/video/fbdev/au1200fb.c
+++ b/drivers/video/fbdev/au1200fb.c
@@ -43,6 +43,7 @@
 #include <linux/ctype.h>
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
+#include <linux/uaccess.h>
 
 #include <asm/mach-au1x00/au1000.h>
 #include <asm/mach-au1x00/au1200fb.h>	/* platform_data */
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 114a0c88afb8..ed9c9eeedfe5 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -167,7 +167,7 @@ static bool vring_use_dma_api(struct virtio_device *vdev)
  * making all of the arch DMA ops work on the vring device itself
  * is a mess.  For now, we use the parent device for DMA ops.
  */
-struct device *vring_dma_dev(const struct vring_virtqueue *vq)
+static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
 {
 	return vq->vq.vdev->dev.parent;
 }
@@ -327,6 +327,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
 		 * host should service the ring ASAP. */
 		if (out_sgs)
 			vq->notify(&vq->vq);
+		if (indirect)
+			kfree(desc);
 		END_USE(vq);
 		return -ENOSPC;
 	}
@@ -426,6 +428,7 @@ unmap_release:
 	if (indirect)
 		kfree(desc);
 
+	END_USE(vq);
 	return -EIO;
 }
 
diff --git a/drivers/vme/bridges/Kconfig b/drivers/vme/bridges/Kconfig
index f6d854584906..f6ddc3715401 100644
--- a/drivers/vme/bridges/Kconfig
+++ b/drivers/vme/bridges/Kconfig
@@ -13,3 +13,11 @@ config VME_TSI148
 	help
 	 If you say Y here you get support for the Tundra TSI148 VME bridge
 	 chip.
+
+config VME_FAKE
+	tristate "Fake"
+	help
+	 If you say Y here you get support for the fake VME bridge. This
+	 provides a virtualised VME Bus for devices with no VME bridge. This
+	 is mainly useful for VME development (in the absence of VME
+	 hardware).
diff --git a/drivers/vme/bridges/Makefile b/drivers/vme/bridges/Makefile
index 59638afcd502..b074542495c5 100644
--- a/drivers/vme/bridges/Makefile
+++ b/drivers/vme/bridges/Makefile
@@ -1,2 +1,3 @@
 obj-$(CONFIG_VME_CA91CX42)	+= vme_ca91cx42.o
 obj-$(CONFIG_VME_TSI148)	+= vme_tsi148.o
+obj-$(CONFIG_VME_FAKE)		+= vme_fake.o
diff --git a/drivers/vme/bridges/vme_ca91cx42.c b/drivers/vme/bridges/vme_ca91cx42.c
index 9f2c834e43e0..6b5ee896af63 100644
--- a/drivers/vme/bridges/vme_ca91cx42.c
+++ b/drivers/vme/bridges/vme_ca91cx42.c
@@ -47,6 +47,8 @@ static const struct pci_device_id ca91cx42_ids[] = {
 	{ },
 };
 
+MODULE_DEVICE_TABLE(pci, ca91cx42_ids);
+
 static struct pci_driver ca91cx42_driver = {
 	.name = driver_name,
 	.id_table = ca91cx42_ids,
@@ -69,7 +71,7 @@ static u32 ca91cx42_LM_irqhandler(struct ca91cx42_driver *bridge, u32 stat)
 	for (i = 0; i < 4; i++) {
 		if (stat & CA91CX42_LINT_LM[i]) {
 			/* We only enable interrupts if the callback is set */
-			bridge->lm_callback[i](i);
+			bridge->lm_callback[i](bridge->lm_data[i]);
 			serviced |= CA91CX42_LINT_LM[i];
 		}
 	}
@@ -1410,7 +1412,7 @@ static int ca91cx42_lm_get(struct vme_lm_resource *lm,
  * Callback will be passed the monitor triggered.
  */
 static int ca91cx42_lm_attach(struct vme_lm_resource *lm, int monitor,
-	void (*callback)(int))
+	void (*callback)(void *), void *data)
 {
 	u32 lm_ctl, tmp;
 	struct ca91cx42_driver *bridge;
@@ -1438,6 +1440,7 @@ static int ca91cx42_lm_attach(struct vme_lm_resource *lm, int monitor,
 
 	/* Attach callback */
 	bridge->lm_callback[monitor] = callback;
+	bridge->lm_data[monitor] = data;
 
 	/* Enable Location Monitor interrupt */
 	tmp = ioread32(bridge->base + LINT_EN);
@@ -1477,6 +1480,7 @@ static int ca91cx42_lm_detach(struct vme_lm_resource *lm, int monitor)
 
 	/* Detach callback */
 	bridge->lm_callback[monitor] = NULL;
+	bridge->lm_data[monitor] = NULL;
 
 	/* If all location monitors disabled, disable global Location Monitor */
 	if ((tmp & (CA91CX42_LINT_LM0 | CA91CX42_LINT_LM1 | CA91CX42_LINT_LM2 |
diff --git a/drivers/vme/bridges/vme_ca91cx42.h b/drivers/vme/bridges/vme_ca91cx42.h
index d54119e59d55..f35c9f5348a9 100644
--- a/drivers/vme/bridges/vme_ca91cx42.h
+++ b/drivers/vme/bridges/vme_ca91cx42.h
@@ -43,7 +43,8 @@ struct ca91cx42_driver {
 	wait_queue_head_t dma_queue;
 	wait_queue_head_t iack_queue;
 	wait_queue_head_t mbox_queue;
-	void (*lm_callback[4])(int);	/* Called in interrupt handler */
+	void (*lm_callback[4])(void *);	/* Called in interrupt handler */
+	void *lm_data[4];
 	void *crcsr_kernel;
 	dma_addr_t crcsr_bus;
 	struct mutex vme_rmw;		/* Only one RMW cycle at a time */
diff --git a/drivers/vme/bridges/vme_fake.c b/drivers/vme/bridges/vme_fake.c
new file mode 100644
index 000000000000..30b3acc93833
--- /dev/null
+++ b/drivers/vme/bridges/vme_fake.c
@@ -0,0 +1,1306 @@
+/*
+ * Fake VME bridge support.
+ *
+ * This drive provides a fake VME bridge chip, this enables debugging of the
+ * VME framework in the absence of a VME system.
+ *
+ * This driver has to do a number of things in software that would be driven
+ * by hardware if it was available, it will also result in extra overhead at
+ * times when compared with driving actual hardware.
+ *
+ * Author: Martyn Welch <martyn@welches.me.uk>
+ * Copyright (c) 2014 Martyn Welch
+ *
+ * Based on vme_tsi148.c:
+ *
+ * Author: Martyn Welch <martyn.welch@ge.com>
+ * Copyright 2008 GE Intelligent Platforms Embedded Systems, Inc.
+ *
+ * Based on work by Tom Armistead and Ajit Prem
+ * Copyright 2004 Motorola Inc.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+#include <linux/vme.h>
+
+#include "../vme_bridge.h"
+
+/*
+ *  Define the number of each that the fake driver supports.
+ */
+#define FAKE_MAX_MASTER		8	/* Max Master Windows */
+#define FAKE_MAX_SLAVE		8	/* Max Slave Windows */
+
+/* Structures to hold information normally held in device registers */
+struct fake_slave_window {
+	int enabled;
+	unsigned long long vme_base;
+	unsigned long long size;
+	void *buf_base;
+	u32 aspace;
+	u32 cycle;
+};
+
+struct fake_master_window {
+	int enabled;
+	unsigned long long vme_base;
+	unsigned long long size;
+	u32 aspace;
+	u32 cycle;
+	u32 dwidth;
+};
+
+/* Structure used to hold driver specific information */
+struct fake_driver {
+	struct vme_bridge *parent;
+	struct fake_slave_window slaves[FAKE_MAX_SLAVE];
+	struct fake_master_window masters[FAKE_MAX_MASTER];
+	u32 lm_enabled;
+	unsigned long long lm_base;
+	u32 lm_aspace;
+	u32 lm_cycle;
+	void (*lm_callback[4])(void *);
+	void *lm_data[4];
+	struct tasklet_struct int_tasklet;
+	int int_level;
+	int int_statid;
+	void *crcsr_kernel;
+	dma_addr_t crcsr_bus;
+	/* Only one VME interrupt can be generated at a time, provide locking */
+	struct mutex vme_int;
+};
+
+/* Module parameter */
+static int geoid;
+
+static const char driver_name[] = "vme_fake";
+
+static struct vme_bridge *exit_pointer;
+
+static struct device *vme_root;
+
+/*
+ * Calling VME bus interrupt callback if provided.
+ */
+static void fake_VIRQ_tasklet(unsigned long data)
+{
+	struct vme_bridge *fake_bridge;
+	struct fake_driver *bridge;
+
+	fake_bridge = (struct vme_bridge *) data;
+	bridge = fake_bridge->driver_priv;
+
+	vme_irq_handler(fake_bridge, bridge->int_level, bridge->int_statid);
+}
+
+/*
+ * Configure VME interrupt
+ */
+static void fake_irq_set(struct vme_bridge *fake_bridge, int level,
+		int state, int sync)
+{
+	/* Nothing to do */
+}
+
+static void *fake_pci_to_ptr(dma_addr_t addr)
+{
+	return (void *)(uintptr_t)addr;
+}
+
+static dma_addr_t fake_ptr_to_pci(void *addr)
+{
+	return (dma_addr_t)(uintptr_t)addr;
+}
+
+/*
+ * Generate a VME bus interrupt at the requested level & vector. Wait for
+ * interrupt to be acked.
+ */
+static int fake_irq_generate(struct vme_bridge *fake_bridge, int level,
+		int statid)
+{
+	struct fake_driver *bridge;
+
+	bridge = fake_bridge->driver_priv;
+
+	mutex_lock(&bridge->vme_int);
+
+	bridge->int_level = level;
+
+	bridge->int_statid = statid;
+
+	/*
+	 * Schedule tasklet to run VME handler to emulate normal VME interrupt
+	 * handler behaviour.
+	 */
+	tasklet_schedule(&bridge->int_tasklet);
+
+	mutex_unlock(&bridge->vme_int);
+
+	return 0;
+}
+
+/*
+ * Initialize a slave window with the requested attributes.
+ */
+static int fake_slave_set(struct vme_slave_resource *image, int enabled,
+		unsigned long long vme_base, unsigned long long size,
+		dma_addr_t buf_base, u32 aspace, u32 cycle)
+{
+	unsigned int i, granularity = 0;
+	unsigned long long vme_bound;
+	struct vme_bridge *fake_bridge;
+	struct fake_driver *bridge;
+
+	fake_bridge = image->parent;
+	bridge = fake_bridge->driver_priv;
+
+	i = image->number;
+
+	switch (aspace) {
+	case VME_A16:
+		granularity = 0x10;
+		break;
+	case VME_A24:
+		granularity = 0x1000;
+		break;
+	case VME_A32:
+		granularity = 0x10000;
+		break;
+	case VME_A64:
+		granularity = 0x10000;
+		break;
+	case VME_CRCSR:
+	case VME_USER1:
+	case VME_USER2:
+	case VME_USER3:
+	case VME_USER4:
+	default:
+		pr_err("Invalid address space\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Bound address is a valid address for the window, adjust
+	 * accordingly
+	 */
+	vme_bound = vme_base + size - granularity;
+
+	if (vme_base & (granularity - 1)) {
+		pr_err("Invalid VME base alignment\n");
+		return -EINVAL;
+	}
+	if (vme_bound & (granularity - 1)) {
+		pr_err("Invalid VME bound alignment\n");
+		return -EINVAL;
+	}
+
+	mutex_lock(&image->mtx);
+
+	bridge->slaves[i].enabled = enabled;
+	bridge->slaves[i].vme_base = vme_base;
+	bridge->slaves[i].size = size;
+	bridge->slaves[i].buf_base = fake_pci_to_ptr(buf_base);
+	bridge->slaves[i].aspace = aspace;
+	bridge->slaves[i].cycle = cycle;
+
+	mutex_unlock(&image->mtx);
+
+	return 0;
+}
+
+/*
+ * Get slave window configuration.
+ */
+static int fake_slave_get(struct vme_slave_resource *image, int *enabled,
+		unsigned long long *vme_base, unsigned long long *size,
+		dma_addr_t *buf_base, u32 *aspace, u32 *cycle)
+{
+	unsigned int i;
+	struct fake_driver *bridge;
+
+	bridge = image->parent->driver_priv;
+
+	i = image->number;
+
+	mutex_lock(&image->mtx);
+
+	*enabled = bridge->slaves[i].enabled;
+	*vme_base = bridge->slaves[i].vme_base;
+	*size = bridge->slaves[i].size;
+	*buf_base = fake_ptr_to_pci(bridge->slaves[i].buf_base);
+	*aspace = bridge->slaves[i].aspace;
+	*cycle = bridge->slaves[i].cycle;
+
+	mutex_unlock(&image->mtx);
+
+	return 0;
+}
+
+/*
+ * Set the attributes of an outbound window.
+ */
+static int fake_master_set(struct vme_master_resource *image, int enabled,
+		unsigned long long vme_base, unsigned long long size,
+		u32 aspace, u32 cycle, u32 dwidth)
+{
+	int retval = 0;
+	unsigned int i;
+	struct vme_bridge *fake_bridge;
+	struct fake_driver *bridge;
+
+	fake_bridge = image->parent;
+
+	bridge = fake_bridge->driver_priv;
+
+	/* Verify input data */
+	if (vme_base & 0xFFFF) {
+		pr_err("Invalid VME Window alignment\n");
+		retval = -EINVAL;
+		goto err_window;
+	}
+
+	if (size & 0xFFFF) {
+		pr_err("Invalid size alignment\n");
+		retval = -EINVAL;
+		goto err_window;
+	}
+
+	if ((size == 0) && (enabled != 0)) {
+		pr_err("Size must be non-zero for enabled windows\n");
+		retval = -EINVAL;
+		goto err_window;
+	}
+
+	/* Setup data width */
+	switch (dwidth) {
+	case VME_D8:
+	case VME_D16:
+	case VME_D32:
+		break;
+	default:
+		pr_err("Invalid data width\n");
+		retval = -EINVAL;
+		goto err_dwidth;
+	}
+
+	/* Setup address space */
+	switch (aspace) {
+	case VME_A16:
+	case VME_A24:
+	case VME_A32:
+	case VME_A64:
+	case VME_CRCSR:
+	case VME_USER1:
+	case VME_USER2:
+	case VME_USER3:
+	case VME_USER4:
+		break;
+	default:
+		pr_err("Invalid address space\n");
+		retval = -EINVAL;
+		goto err_aspace;
+	}
+
+	spin_lock(&image->lock);
+
+	i = image->number;
+
+	bridge->masters[i].enabled = enabled;
+	bridge->masters[i].vme_base = vme_base;
+	bridge->masters[i].size = size;
+	bridge->masters[i].aspace = aspace;
+	bridge->masters[i].cycle = cycle;
+	bridge->masters[i].dwidth = dwidth;
+
+	spin_unlock(&image->lock);
+
+	return 0;
+
+err_aspace:
+err_dwidth:
+err_window:
+	return retval;
+
+}
+
+/*
+ * Set the attributes of an outbound window.
+ */
+static int __fake_master_get(struct vme_master_resource *image, int *enabled,
+		unsigned long long *vme_base, unsigned long long *size,
+		u32 *aspace, u32 *cycle, u32 *dwidth)
+{
+	unsigned int i;
+	struct fake_driver *bridge;
+
+	bridge = image->parent->driver_priv;
+
+	i = image->number;
+
+	*enabled = bridge->masters[i].enabled;
+	*vme_base = bridge->masters[i].vme_base;
+	*size = bridge->masters[i].size;
+	*aspace = bridge->masters[i].aspace;
+	*cycle = bridge->masters[i].cycle;
+	*dwidth = bridge->masters[i].dwidth;
+
+	return 0;
+}
+
+
+static int fake_master_get(struct vme_master_resource *image, int *enabled,
+		unsigned long long *vme_base, unsigned long long *size,
+		u32 *aspace, u32 *cycle, u32 *dwidth)
+{
+	int retval;
+
+	spin_lock(&image->lock);
+
+	retval = __fake_master_get(image, enabled, vme_base, size, aspace,
+			cycle, dwidth);
+
+	spin_unlock(&image->lock);
+
+	return retval;
+}
+
+
+static void fake_lm_check(struct fake_driver *bridge, unsigned long long addr,
+			  u32 aspace, u32 cycle)
+{
+	struct vme_bridge *fake_bridge;
+	unsigned long long lm_base;
+	u32 lm_aspace, lm_cycle;
+	int i;
+	struct vme_lm_resource *lm;
+	struct list_head *pos = NULL, *n;
+
+	/* Get vme_bridge */
+	fake_bridge = bridge->parent;
+
+	/* Loop through each location monitor resource */
+	list_for_each_safe(pos, n, &fake_bridge->lm_resources) {
+		lm = list_entry(pos, struct vme_lm_resource, list);
+
+		/* If disabled, we're done */
+		if (bridge->lm_enabled == 0)
+			return;
+
+		lm_base = bridge->lm_base;
+		lm_aspace = bridge->lm_aspace;
+		lm_cycle = bridge->lm_cycle;
+
+		/* First make sure that the cycle and address space match */
+		if ((lm_aspace == aspace) && (lm_cycle == cycle)) {
+			for (i = 0; i < lm->monitors; i++) {
+				/* Each location monitor covers 8 bytes */
+				if (((lm_base + (8 * i)) <= addr) &&
+				    ((lm_base + (8 * i) + 8) > addr)) {
+					if (bridge->lm_callback[i] != NULL)
+						bridge->lm_callback[i](
+							bridge->lm_data[i]);
+				}
+			}
+		}
+	}
+}
+
+static u8 fake_vmeread8(struct fake_driver *bridge, unsigned long long addr,
+		u32 aspace, u32 cycle)
+{
+	u8 retval = 0xff;
+	int i;
+	unsigned long long start, end, offset;
+	u8 *loc;
+
+	for (i = 0; i < FAKE_MAX_SLAVE; i++) {
+		start = bridge->slaves[i].vme_base;
+		end = bridge->slaves[i].vme_base + bridge->slaves[i].size;
+
+		if (aspace != bridge->slaves[i].aspace)
+			continue;
+
+		if (cycle != bridge->slaves[i].cycle)
+			continue;
+
+		if ((addr >= start) && (addr < end)) {
+			offset = addr - bridge->slaves[i].vme_base;
+			loc = (u8 *)(bridge->slaves[i].buf_base + offset);
+			retval = *loc;
+
+			break;
+		}
+	}
+
+	fake_lm_check(bridge, addr, aspace, cycle);
+
+	return retval;
+}
+
+static u16 fake_vmeread16(struct fake_driver *bridge, unsigned long long addr,
+		u32 aspace, u32 cycle)
+{
+	u16 retval = 0xffff;
+	int i;
+	unsigned long long start, end, offset;
+	u16 *loc;
+
+	for (i = 0; i < FAKE_MAX_SLAVE; i++) {
+		if (aspace != bridge->slaves[i].aspace)
+			continue;
+
+		if (cycle != bridge->slaves[i].cycle)
+			continue;
+
+		start = bridge->slaves[i].vme_base;
+		end = bridge->slaves[i].vme_base + bridge->slaves[i].size;
+
+		if ((addr >= start) && ((addr + 1) < end)) {
+			offset = addr - bridge->slaves[i].vme_base;
+			loc = (u16 *)(bridge->slaves[i].buf_base + offset);
+			retval = *loc;
+
+			break;
+		}
+	}
+
+	fake_lm_check(bridge, addr, aspace, cycle);
+
+	return retval;
+}
+
+static u32 fake_vmeread32(struct fake_driver *bridge, unsigned long long addr,
+		u32 aspace, u32 cycle)
+{
+	u32 retval = 0xffffffff;
+	int i;
+	unsigned long long start, end, offset;
+	u32 *loc;
+
+	for (i = 0; i < FAKE_MAX_SLAVE; i++) {
+		if (aspace != bridge->slaves[i].aspace)
+			continue;
+
+		if (cycle != bridge->slaves[i].cycle)
+			continue;
+
+		start = bridge->slaves[i].vme_base;
+		end = bridge->slaves[i].vme_base + bridge->slaves[i].size;
+
+		if ((addr >= start) && ((addr + 3) < end)) {
+			offset = addr - bridge->slaves[i].vme_base;
+			loc = (u32 *)(bridge->slaves[i].buf_base + offset);
+			retval = *loc;
+
+			break;
+		}
+	}
+
+	fake_lm_check(bridge, addr, aspace, cycle);
+
+	return retval;
+}
+
+static ssize_t fake_master_read(struct vme_master_resource *image, void *buf,
+		size_t count, loff_t offset)
+{
+	int retval;
+	u32 aspace, cycle, dwidth;
+	struct vme_bridge *fake_bridge;
+	struct fake_driver *priv;
+	int i;
+	unsigned long long addr;
+	unsigned int done = 0;
+	unsigned int count32;
+
+	fake_bridge = image->parent;
+
+	priv = fake_bridge->driver_priv;
+
+	i = image->number;
+
+	addr = (unsigned long long)priv->masters[i].vme_base + offset;
+	aspace = priv->masters[i].aspace;
+	cycle = priv->masters[i].cycle;
+	dwidth = priv->masters[i].dwidth;
+
+	spin_lock(&image->lock);
+
+	/* The following code handles VME address alignment. We cannot use
+	 * memcpy_xxx here because it may cut data transfers in to 8-bit
+	 * cycles when D16 or D32 cycles are required on the VME bus.
+	 * On the other hand, the bridge itself assures that the maximum data
+	 * cycle configured for the transfer is used and splits it
+	 * automatically for non-aligned addresses, so we don't want the
+	 * overhead of needlessly forcing small transfers for the entire cycle.
+	 */
+	if (addr & 0x1) {
+		*(u8 *)buf = fake_vmeread8(priv, addr, aspace, cycle);
+		done += 1;
+		if (done == count)
+			goto out;
+	}
+	if ((dwidth == VME_D16) || (dwidth == VME_D32)) {
+		if ((addr + done) & 0x2) {
+			if ((count - done) < 2) {
+				*(u8 *)(buf + done) = fake_vmeread8(priv,
+						addr + done, aspace, cycle);
+				done += 1;
+				goto out;
+			} else {
+				*(u16 *)(buf + done) = fake_vmeread16(priv,
+						addr + done, aspace, cycle);
+				done += 2;
+			}
+		}
+	}
+
+	if (dwidth == VME_D32) {
+		count32 = (count - done) & ~0x3;
+		while (done < count32) {
+			*(u32 *)(buf + done) = fake_vmeread32(priv, addr + done,
+					aspace, cycle);
+			done += 4;
+		}
+	} else if (dwidth == VME_D16) {
+		count32 = (count - done) & ~0x3;
+		while (done < count32) {
+			*(u16 *)(buf + done) = fake_vmeread16(priv, addr + done,
+					aspace, cycle);
+			done += 2;
+		}
+	} else if (dwidth == VME_D8) {
+		count32 = (count - done);
+		while (done < count32) {
+			*(u8 *)(buf + done) = fake_vmeread8(priv, addr + done,
+					aspace, cycle);
+			done += 1;
+		}
+
+	}
+
+	if ((dwidth == VME_D16) || (dwidth == VME_D32)) {
+		if ((count - done) & 0x2) {
+			*(u16 *)(buf + done) = fake_vmeread16(priv, addr + done,
+					aspace, cycle);
+			done += 2;
+		}
+	}
+	if ((count - done) & 0x1) {
+		*(u8 *)(buf + done) = fake_vmeread8(priv, addr + done, aspace,
+				cycle);
+		done += 1;
+	}
+
+out:
+	retval = count;
+
+	spin_unlock(&image->lock);
+
+	return retval;
+}
+
+static void fake_vmewrite8(struct fake_driver *bridge, u8 *buf,
+			   unsigned long long addr, u32 aspace, u32 cycle)
+{
+	int i;
+	unsigned long long start, end, offset;
+	u8 *loc;
+
+	for (i = 0; i < FAKE_MAX_SLAVE; i++) {
+		if (aspace != bridge->slaves[i].aspace)
+			continue;
+
+		if (cycle != bridge->slaves[i].cycle)
+			continue;
+
+		start = bridge->slaves[i].vme_base;
+		end = bridge->slaves[i].vme_base + bridge->slaves[i].size;
+
+		if ((addr >= start) && (addr < end)) {
+			offset = addr - bridge->slaves[i].vme_base;
+			loc = (u8 *)((void *)bridge->slaves[i].buf_base + offset);
+			*loc = *buf;
+
+			break;
+		}
+	}
+
+	fake_lm_check(bridge, addr, aspace, cycle);
+
+}
+
+static void fake_vmewrite16(struct fake_driver *bridge, u16 *buf,
+			    unsigned long long addr, u32 aspace, u32 cycle)
+{
+	int i;
+	unsigned long long start, end, offset;
+	u16 *loc;
+
+	for (i = 0; i < FAKE_MAX_SLAVE; i++) {
+		if (aspace != bridge->slaves[i].aspace)
+			continue;
+
+		if (cycle != bridge->slaves[i].cycle)
+			continue;
+
+		start = bridge->slaves[i].vme_base;
+		end = bridge->slaves[i].vme_base + bridge->slaves[i].size;
+
+		if ((addr >= start) && ((addr + 1) < end)) {
+			offset = addr - bridge->slaves[i].vme_base;
+			loc = (u16 *)((void *)bridge->slaves[i].buf_base + offset);
+			*loc = *buf;
+
+			break;
+		}
+	}
+
+	fake_lm_check(bridge, addr, aspace, cycle);
+
+}
+
+static void fake_vmewrite32(struct fake_driver *bridge, u32 *buf,
+			    unsigned long long addr, u32 aspace, u32 cycle)
+{
+	int i;
+	unsigned long long start, end, offset;
+	u32 *loc;
+
+	for (i = 0; i < FAKE_MAX_SLAVE; i++) {
+		if (aspace != bridge->slaves[i].aspace)
+			continue;
+
+		if (cycle != bridge->slaves[i].cycle)
+			continue;
+
+		start = bridge->slaves[i].vme_base;
+		end = bridge->slaves[i].vme_base + bridge->slaves[i].size;
+
+		if ((addr >= start) && ((addr + 3) < end)) {
+			offset = addr - bridge->slaves[i].vme_base;
+			loc = (u32 *)((void *)bridge->slaves[i].buf_base + offset);
+			*loc = *buf;
+
+			break;
+		}
+	}
+
+	fake_lm_check(bridge, addr, aspace, cycle);
+
+}
+
+static ssize_t fake_master_write(struct vme_master_resource *image, void *buf,
+		size_t count, loff_t offset)
+{
+	int retval = 0;
+	u32 aspace, cycle, dwidth;
+	unsigned long long addr;
+	int i;
+	unsigned int done = 0;
+	unsigned int count32;
+
+	struct vme_bridge *fake_bridge;
+	struct fake_driver *bridge;
+
+	fake_bridge = image->parent;
+
+	bridge = fake_bridge->driver_priv;
+
+	i = image->number;
+
+	addr = bridge->masters[i].vme_base + offset;
+	aspace = bridge->masters[i].aspace;
+	cycle = bridge->masters[i].cycle;
+	dwidth = bridge->masters[i].dwidth;
+
+	spin_lock(&image->lock);
+
+	/* Here we apply for the same strategy we do in master_read
+	 * function in order to assure the correct cycles.
+	 */
+	if (addr & 0x1) {
+		fake_vmewrite8(bridge, (u8 *)buf, addr, aspace, cycle);
+		done += 1;
+		if (done == count)
+			goto out;
+	}
+
+	if ((dwidth == VME_D16) || (dwidth == VME_D32)) {
+		if ((addr + done) & 0x2) {
+			if ((count - done) < 2) {
+				fake_vmewrite8(bridge, (u8 *)(buf + done),
+						addr + done, aspace, cycle);
+				done += 1;
+				goto out;
+			} else {
+				fake_vmewrite16(bridge, (u16 *)(buf + done),
+						addr + done, aspace, cycle);
+				done += 2;
+			}
+		}
+	}
+
+	if (dwidth == VME_D32) {
+		count32 = (count - done) & ~0x3;
+		while (done < count32) {
+			fake_vmewrite32(bridge, (u32 *)(buf + done),
+					addr + done, aspace, cycle);
+			done += 4;
+		}
+	} else if (dwidth == VME_D16) {
+		count32 = (count - done) & ~0x3;
+		while (done < count32) {
+			fake_vmewrite16(bridge, (u16 *)(buf + done),
+					addr + done, aspace, cycle);
+			done += 2;
+		}
+	} else if (dwidth == VME_D8) {
+		count32 = (count - done);
+		while (done < count32) {
+			fake_vmewrite8(bridge, (u8 *)(buf + done), addr + done,
+					aspace, cycle);
+			done += 1;
+		}
+
+	}
+
+	if ((dwidth == VME_D16) || (dwidth == VME_D32)) {
+		if ((count - done) & 0x2) {
+			fake_vmewrite16(bridge, (u16 *)(buf + done),
+					addr + done, aspace, cycle);
+			done += 2;
+		}
+	}
+
+	if ((count - done) & 0x1) {
+		fake_vmewrite8(bridge, (u8 *)(buf + done), addr + done, aspace,
+				cycle);
+		done += 1;
+	}
+
+out:
+	retval = count;
+
+	spin_unlock(&image->lock);
+
+	return retval;
+}
+
+/*
+ * Perform an RMW cycle on the VME bus.
+ *
+ * Requires a previously configured master window, returns final value.
+ */
+static unsigned int fake_master_rmw(struct vme_master_resource *image,
+		unsigned int mask, unsigned int compare, unsigned int swap,
+		loff_t offset)
+{
+	u32 tmp, base;
+	u32 aspace, cycle;
+	int i;
+	struct fake_driver *bridge;
+
+	bridge = image->parent->driver_priv;
+
+	/* Find the PCI address that maps to the desired VME address */
+	i = image->number;
+
+	base = bridge->masters[i].vme_base;
+	aspace = bridge->masters[i].aspace;
+	cycle = bridge->masters[i].cycle;
+
+	/* Lock image */
+	spin_lock(&image->lock);
+
+	/* Read existing value */
+	tmp = fake_vmeread32(bridge, base + offset, aspace, cycle);
+
+	/* Perform check */
+	if ((tmp && mask) == (compare && mask)) {
+		tmp = tmp | (mask | swap);
+		tmp = tmp & (~mask | swap);
+
+		/* Write back */
+		fake_vmewrite32(bridge, &tmp, base + offset, aspace, cycle);
+	}
+
+	/* Unlock image */
+	spin_unlock(&image->lock);
+
+	return tmp;
+}
+
+/*
+ * All 4 location monitors reside at the same base - this is therefore a
+ * system wide configuration.
+ *
+ * This does not enable the LM monitor - that should be done when the first
+ * callback is attached and disabled when the last callback is removed.
+ */
+static int fake_lm_set(struct vme_lm_resource *lm, unsigned long long lm_base,
+		u32 aspace, u32 cycle)
+{
+	int i;
+	struct vme_bridge *fake_bridge;
+	struct fake_driver *bridge;
+
+	fake_bridge = lm->parent;
+
+	bridge = fake_bridge->driver_priv;
+
+	mutex_lock(&lm->mtx);
+
+	/* If we already have a callback attached, we can't move it! */
+	for (i = 0; i < lm->monitors; i++) {
+		if (bridge->lm_callback[i] != NULL) {
+			mutex_unlock(&lm->mtx);
+			pr_err("Location monitor callback attached, can't reset\n");
+			return -EBUSY;
+		}
+	}
+
+	switch (aspace) {
+	case VME_A16:
+	case VME_A24:
+	case VME_A32:
+	case VME_A64:
+		break;
+	default:
+		mutex_unlock(&lm->mtx);
+		pr_err("Invalid address space\n");
+		return -EINVAL;
+	}
+
+	bridge->lm_base = lm_base;
+	bridge->lm_aspace = aspace;
+	bridge->lm_cycle = cycle;
+
+	mutex_unlock(&lm->mtx);
+
+	return 0;
+}
+
+/* Get configuration of the callback monitor and return whether it is enabled
+ * or disabled.
+ */
+static int fake_lm_get(struct vme_lm_resource *lm,
+		unsigned long long *lm_base, u32 *aspace, u32 *cycle)
+{
+	struct fake_driver *bridge;
+
+	bridge = lm->parent->driver_priv;
+
+	mutex_lock(&lm->mtx);
+
+	*lm_base = bridge->lm_base;
+	*aspace = bridge->lm_aspace;
+	*cycle = bridge->lm_cycle;
+
+	mutex_unlock(&lm->mtx);
+
+	return bridge->lm_enabled;
+}
+
+/*
+ * Attach a callback to a specific location monitor.
+ *
+ * Callback will be passed the monitor triggered.
+ */
+static int fake_lm_attach(struct vme_lm_resource *lm, int monitor,
+		void (*callback)(void *), void *data)
+{
+	struct vme_bridge *fake_bridge;
+	struct fake_driver *bridge;
+
+	fake_bridge = lm->parent;
+
+	bridge = fake_bridge->driver_priv;
+
+	mutex_lock(&lm->mtx);
+
+	/* Ensure that the location monitor is configured - need PGM or DATA */
+	if (bridge->lm_cycle == 0) {
+		mutex_unlock(&lm->mtx);
+		pr_err("Location monitor not properly configured\n");
+		return -EINVAL;
+	}
+
+	/* Check that a callback isn't already attached */
+	if (bridge->lm_callback[monitor] != NULL) {
+		mutex_unlock(&lm->mtx);
+		pr_err("Existing callback attached\n");
+		return -EBUSY;
+	}
+
+	/* Attach callback */
+	bridge->lm_callback[monitor] = callback;
+	bridge->lm_data[monitor] = data;
+
+	/* Ensure that global Location Monitor Enable set */
+	bridge->lm_enabled = 1;
+
+	mutex_unlock(&lm->mtx);
+
+	return 0;
+}
+
+/*
+ * Detach a callback function forn a specific location monitor.
+ */
+static int fake_lm_detach(struct vme_lm_resource *lm, int monitor)
+{
+	u32 tmp;
+	int i;
+	struct fake_driver *bridge;
+
+	bridge = lm->parent->driver_priv;
+
+	mutex_lock(&lm->mtx);
+
+	/* Detach callback */
+	bridge->lm_callback[monitor] = NULL;
+	bridge->lm_data[monitor] = NULL;
+
+	/* If all location monitors disabled, disable global Location Monitor */
+	tmp = 0;
+	for (i = 0; i < lm->monitors; i++) {
+		if (bridge->lm_callback[i] != NULL)
+			tmp = 1;
+	}
+
+	if (tmp == 0)
+		bridge->lm_enabled = 0;
+
+	mutex_unlock(&lm->mtx);
+
+	return 0;
+}
+
+/*
+ * Determine Geographical Addressing
+ */
+static int fake_slot_get(struct vme_bridge *fake_bridge)
+{
+	return geoid;
+}
+
+static void *fake_alloc_consistent(struct device *parent, size_t size,
+		dma_addr_t *dma)
+{
+	void *alloc = kmalloc(size, GFP_KERNEL);
+
+	if (alloc != NULL)
+		*dma = fake_ptr_to_pci(alloc);
+
+	return alloc;
+}
+
+static void fake_free_consistent(struct device *parent, size_t size,
+		void *vaddr, dma_addr_t dma)
+{
+	kfree(vaddr);
+/*
+	dma_free_coherent(parent, size, vaddr, dma);
+*/
+}
+
+/*
+ * Configure CR/CSR space
+ *
+ * Access to the CR/CSR can be configured at power-up. The location of the
+ * CR/CSR registers in the CR/CSR address space is determined by the boards
+ * Geographic address.
+ *
+ * Each board has a 512kB window, with the highest 4kB being used for the
+ * boards registers, this means there is a fix length 508kB window which must
+ * be mapped onto PCI memory.
+ */
+static int fake_crcsr_init(struct vme_bridge *fake_bridge)
+{
+	u32 vstat;
+	struct fake_driver *bridge;
+
+	bridge = fake_bridge->driver_priv;
+
+	/* Allocate mem for CR/CSR image */
+	bridge->crcsr_kernel = kzalloc(VME_CRCSR_BUF_SIZE, GFP_KERNEL);
+	bridge->crcsr_bus = fake_ptr_to_pci(bridge->crcsr_kernel);
+	if (bridge->crcsr_kernel == NULL)
+		return -ENOMEM;
+
+	vstat = fake_slot_get(fake_bridge);
+
+	pr_info("CR/CSR Offset: %d\n", vstat);
+
+	return 0;
+}
+
+static void fake_crcsr_exit(struct vme_bridge *fake_bridge)
+{
+	struct fake_driver *bridge;
+
+	bridge = fake_bridge->driver_priv;
+
+	kfree(bridge->crcsr_kernel);
+}
+
+
+static int __init fake_init(void)
+{
+	int retval, i;
+	struct list_head *pos = NULL, *n;
+	struct vme_bridge *fake_bridge;
+	struct fake_driver *fake_device;
+	struct vme_master_resource *master_image;
+	struct vme_slave_resource *slave_image;
+	struct vme_lm_resource *lm;
+
+	/* We need a fake parent device */
+	vme_root = __root_device_register("vme", THIS_MODULE);
+
+	/* If we want to support more than one bridge at some point, we need to
+	 * dynamically allocate this so we get one per device.
+	 */
+	fake_bridge = kzalloc(sizeof(struct vme_bridge), GFP_KERNEL);
+	if (fake_bridge == NULL) {
+		retval = -ENOMEM;
+		goto err_struct;
+	}
+
+	fake_device = kzalloc(sizeof(struct fake_driver), GFP_KERNEL);
+	if (fake_device == NULL) {
+		retval = -ENOMEM;
+		goto err_driver;
+	}
+
+	fake_bridge->driver_priv = fake_device;
+
+	fake_bridge->parent = vme_root;
+
+	fake_device->parent = fake_bridge;
+
+	/* Initialize wait queues & mutual exclusion flags */
+	mutex_init(&fake_device->vme_int);
+	mutex_init(&fake_bridge->irq_mtx);
+	tasklet_init(&fake_device->int_tasklet, fake_VIRQ_tasklet,
+			(unsigned long) fake_bridge);
+
+	strcpy(fake_bridge->name, driver_name);
+
+	/* Add master windows to list */
+	INIT_LIST_HEAD(&fake_bridge->master_resources);
+	for (i = 0; i < FAKE_MAX_MASTER; i++) {
+		master_image = kmalloc(sizeof(struct vme_master_resource),
+				GFP_KERNEL);
+		if (master_image == NULL) {
+			retval = -ENOMEM;
+			goto err_master;
+		}
+		master_image->parent = fake_bridge;
+		spin_lock_init(&master_image->lock);
+		master_image->locked = 0;
+		master_image->number = i;
+		master_image->address_attr = VME_A16 | VME_A24 | VME_A32 |
+			VME_A64;
+		master_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT |
+			VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 |
+			VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER |
+			VME_PROG | VME_DATA;
+		master_image->width_attr = VME_D16 | VME_D32;
+		memset(&master_image->bus_resource, 0,
+				sizeof(struct resource));
+		master_image->kern_base  = NULL;
+		list_add_tail(&master_image->list,
+				&fake_bridge->master_resources);
+	}
+
+	/* Add slave windows to list */
+	INIT_LIST_HEAD(&fake_bridge->slave_resources);
+	for (i = 0; i < FAKE_MAX_SLAVE; i++) {
+		slave_image = kmalloc(sizeof(struct vme_slave_resource),
+				GFP_KERNEL);
+		if (slave_image == NULL) {
+			retval = -ENOMEM;
+			goto err_slave;
+		}
+		slave_image->parent = fake_bridge;
+		mutex_init(&slave_image->mtx);
+		slave_image->locked = 0;
+		slave_image->number = i;
+		slave_image->address_attr = VME_A16 | VME_A24 | VME_A32 |
+			VME_A64 | VME_CRCSR | VME_USER1 | VME_USER2 |
+			VME_USER3 | VME_USER4;
+		slave_image->cycle_attr = VME_SCT | VME_BLT | VME_MBLT |
+			VME_2eVME | VME_2eSST | VME_2eSSTB | VME_2eSST160 |
+			VME_2eSST267 | VME_2eSST320 | VME_SUPER | VME_USER |
+			VME_PROG | VME_DATA;
+		list_add_tail(&slave_image->list,
+				&fake_bridge->slave_resources);
+	}
+
+	/* Add location monitor to list */
+	INIT_LIST_HEAD(&fake_bridge->lm_resources);
+	lm = kmalloc(sizeof(struct vme_lm_resource), GFP_KERNEL);
+	if (lm == NULL) {
+		pr_err("Failed to allocate memory for location monitor resource structure\n");
+		retval = -ENOMEM;
+		goto err_lm;
+	}
+	lm->parent = fake_bridge;
+	mutex_init(&lm->mtx);
+	lm->locked = 0;
+	lm->number = 1;
+	lm->monitors = 4;
+	list_add_tail(&lm->list, &fake_bridge->lm_resources);
+
+	fake_bridge->slave_get = fake_slave_get;
+	fake_bridge->slave_set = fake_slave_set;
+	fake_bridge->master_get = fake_master_get;
+	fake_bridge->master_set = fake_master_set;
+	fake_bridge->master_read = fake_master_read;
+	fake_bridge->master_write = fake_master_write;
+	fake_bridge->master_rmw = fake_master_rmw;
+	fake_bridge->irq_set = fake_irq_set;
+	fake_bridge->irq_generate = fake_irq_generate;
+	fake_bridge->lm_set = fake_lm_set;
+	fake_bridge->lm_get = fake_lm_get;
+	fake_bridge->lm_attach = fake_lm_attach;
+	fake_bridge->lm_detach = fake_lm_detach;
+	fake_bridge->slot_get = fake_slot_get;
+	fake_bridge->alloc_consistent = fake_alloc_consistent;
+	fake_bridge->free_consistent = fake_free_consistent;
+
+	pr_info("Board is%s the VME system controller\n",
+			(geoid == 1) ? "" : " not");
+
+	pr_info("VME geographical address is set to %d\n", geoid);
+
+	retval = fake_crcsr_init(fake_bridge);
+	if (retval) {
+		pr_err("CR/CSR configuration failed.\n");
+		goto err_crcsr;
+	}
+
+	retval = vme_register_bridge(fake_bridge);
+	if (retval != 0) {
+		pr_err("Chip Registration failed.\n");
+		goto err_reg;
+	}
+
+	exit_pointer = fake_bridge;
+
+	return 0;
+
+err_reg:
+	fake_crcsr_exit(fake_bridge);
+err_crcsr:
+err_lm:
+	/* resources are stored in link list */
+	list_for_each_safe(pos, n, &fake_bridge->lm_resources) {
+		lm = list_entry(pos, struct vme_lm_resource, list);
+		list_del(pos);
+		kfree(lm);
+	}
+err_slave:
+	/* resources are stored in link list */
+	list_for_each_safe(pos, n, &fake_bridge->slave_resources) {
+		slave_image = list_entry(pos, struct vme_slave_resource, list);
+		list_del(pos);
+		kfree(slave_image);
+	}
+err_master:
+	/* resources are stored in link list */
+	list_for_each_safe(pos, n, &fake_bridge->master_resources) {
+		master_image = list_entry(pos, struct vme_master_resource,
+				list);
+		list_del(pos);
+		kfree(master_image);
+	}
+
+	kfree(fake_device);
+err_driver:
+	kfree(fake_bridge);
+err_struct:
+	return retval;
+
+}
+
+
+static void __exit fake_exit(void)
+{
+	struct list_head *pos = NULL;
+	struct list_head *tmplist;
+	struct vme_master_resource *master_image;
+	struct vme_slave_resource *slave_image;
+	int i;
+	struct vme_bridge *fake_bridge;
+	struct fake_driver *bridge;
+
+	fake_bridge = exit_pointer;
+
+	bridge = fake_bridge->driver_priv;
+
+	pr_debug("Driver is being unloaded.\n");
+
+	/*
+	 *  Shutdown all inbound and outbound windows.
+	 */
+	for (i = 0; i < FAKE_MAX_MASTER; i++)
+		bridge->masters[i].enabled = 0;
+
+	for (i = 0; i < FAKE_MAX_SLAVE; i++)
+		bridge->slaves[i].enabled = 0;
+
+	/*
+	 *  Shutdown Location monitor.
+	 */
+	bridge->lm_enabled = 0;
+
+	vme_unregister_bridge(fake_bridge);
+
+	fake_crcsr_exit(fake_bridge);
+	/* resources are stored in link list */
+	list_for_each_safe(pos, tmplist, &fake_bridge->slave_resources) {
+		slave_image = list_entry(pos, struct vme_slave_resource, list);
+		list_del(pos);
+		kfree(slave_image);
+	}
+
+	/* resources are stored in link list */
+	list_for_each_safe(pos, tmplist, &fake_bridge->master_resources) {
+		master_image = list_entry(pos, struct vme_master_resource,
+				list);
+		list_del(pos);
+		kfree(master_image);
+	}
+
+	kfree(fake_bridge->driver_priv);
+
+	kfree(fake_bridge);
+
+	root_device_unregister(vme_root);
+}
+
+
+MODULE_PARM_DESC(geoid, "Set geographical addressing");
+module_param(geoid, int, 0);
+
+MODULE_DESCRIPTION("Fake VME bridge driver");
+MODULE_LICENSE("GPL");
+
+module_init(fake_init);
+module_exit(fake_exit);
diff --git a/drivers/vme/bridges/vme_tsi148.c b/drivers/vme/bridges/vme_tsi148.c
index 4bc5d451ec6c..fc1b634b969a 100644
--- a/drivers/vme/bridges/vme_tsi148.c
+++ b/drivers/vme/bridges/vme_tsi148.c
@@ -50,6 +50,8 @@ static const struct pci_device_id tsi148_ids[] = {
 	{ },
 };
 
+MODULE_DEVICE_TABLE(pci, tsi148_ids);
+
 static struct pci_driver tsi148_driver = {
 	.name = driver_name,
 	.id_table = tsi148_ids,
@@ -102,7 +104,7 @@ static u32 tsi148_LM_irqhandler(struct tsi148_driver *bridge, u32 stat)
 	for (i = 0; i < 4; i++) {
 		if (stat & TSI148_LCSR_INTS_LMS[i]) {
 			/* We only enable interrupts if the callback is set */
-			bridge->lm_callback[i](i);
+			bridge->lm_callback[i](bridge->lm_data[i]);
 			serviced |= TSI148_LCSR_INTC_LMC[i];
 		}
 	}
@@ -2047,7 +2049,7 @@ static int tsi148_lm_get(struct vme_lm_resource *lm,
  * Callback will be passed the monitor triggered.
  */
 static int tsi148_lm_attach(struct vme_lm_resource *lm, int monitor,
-	void (*callback)(int))
+	void (*callback)(void *), void *data)
 {
 	u32 lm_ctl, tmp;
 	struct vme_bridge *tsi148_bridge;
@@ -2077,6 +2079,7 @@ static int tsi148_lm_attach(struct vme_lm_resource *lm, int monitor,
 
 	/* Attach callback */
 	bridge->lm_callback[monitor] = callback;
+	bridge->lm_data[monitor] = data;
 
 	/* Enable Location Monitor interrupt */
 	tmp = ioread32be(bridge->base + TSI148_LCSR_INTEN);
@@ -2124,6 +2127,7 @@ static int tsi148_lm_detach(struct vme_lm_resource *lm, int monitor)
 
 	/* Detach callback */
 	bridge->lm_callback[monitor] = NULL;
+	bridge->lm_data[monitor] = NULL;
 
 	/* If all location monitors disabled, disable global Location Monitor */
 	if ((lm_en & (TSI148_LCSR_INTS_LM0S | TSI148_LCSR_INTS_LM1S |
diff --git a/drivers/vme/bridges/vme_tsi148.h b/drivers/vme/bridges/vme_tsi148.h
index f5ed14382a8d..0935d85d32ec 100644
--- a/drivers/vme/bridges/vme_tsi148.h
+++ b/drivers/vme/bridges/vme_tsi148.h
@@ -38,7 +38,8 @@ struct tsi148_driver {
 	void __iomem *base;	/* Base Address of device registers */
 	wait_queue_head_t dma_queue[2];
 	wait_queue_head_t iack_queue;
-	void (*lm_callback[4])(int);	/* Called in interrupt handler */
+	void (*lm_callback[4])(void *);	/* Called in interrupt handler */
+	void *lm_data[4];
 	void *crcsr_kernel;
 	dma_addr_t crcsr_bus;
 	struct vme_master_resource *flush_image;
diff --git a/drivers/vme/vme.c b/drivers/vme/vme.c
index 37ac0a58e59a..15b64076bc26 100644
--- a/drivers/vme/vme.c
+++ b/drivers/vme/vme.c
@@ -13,8 +13,8 @@
  * option) any later version.
  */
 
-#include <linux/module.h>
-#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/export.h>
 #include <linux/mm.h>
 #include <linux/types.h>
 #include <linux/kernel.h>
@@ -39,7 +39,6 @@ static unsigned int vme_bus_numbers;
 static LIST_HEAD(vme_bus_list);
 static DEFINE_MUTEX(vme_buses_lock);
 
-static void __exit vme_exit(void);
 static int __init vme_init(void);
 
 static struct vme_dev *dev_to_vme_dev(struct device *dev)
@@ -1321,7 +1320,7 @@ int vme_lm_get(struct vme_resource *resource, unsigned long long *lm_base,
 EXPORT_SYMBOL(vme_lm_get);
 
 int vme_lm_attach(struct vme_resource *resource, int monitor,
-	void (*callback)(int))
+	void (*callback)(void *), void *data)
 {
 	struct vme_bridge *bridge = find_bridge(resource);
 	struct vme_lm_resource *lm;
@@ -1338,7 +1337,7 @@ int vme_lm_attach(struct vme_resource *resource, int monitor,
 		return -EINVAL;
 	}
 
-	return bridge->lm_attach(lm, monitor, callback);
+	return bridge->lm_attach(lm, monitor, callback, data);
 }
 EXPORT_SYMBOL(vme_lm_attach);
 
@@ -1622,25 +1621,10 @@ static int vme_bus_probe(struct device *dev)
 	return retval;
 }
 
-static int vme_bus_remove(struct device *dev)
-{
-	int retval = -ENODEV;
-	struct vme_driver *driver;
-	struct vme_dev *vdev = dev_to_vme_dev(dev);
-
-	driver = dev->platform_data;
-
-	if (driver->remove != NULL)
-		retval = driver->remove(vdev);
-
-	return retval;
-}
-
 struct bus_type vme_bus_type = {
 	.name = "vme",
 	.match = vme_bus_match,
 	.probe = vme_bus_probe,
-	.remove = vme_bus_remove,
 };
 EXPORT_SYMBOL(vme_bus_type);
 
@@ -1648,11 +1632,4 @@ static int __init vme_init(void)
 {
 	return bus_register(&vme_bus_type);
 }
-
-static void __exit vme_exit(void)
-{
-	bus_unregister(&vme_bus_type);
-}
-
 subsys_initcall(vme_init);
-module_exit(vme_exit);
diff --git a/drivers/vme/vme_bridge.h b/drivers/vme/vme_bridge.h
index cb8246fd97be..2662e916b96a 100644
--- a/drivers/vme/vme_bridge.h
+++ b/drivers/vme/vme_bridge.h
@@ -160,7 +160,8 @@ struct vme_bridge {
 	int (*lm_set) (struct vme_lm_resource *, unsigned long long, u32, u32);
 	int (*lm_get) (struct vme_lm_resource *, unsigned long long *, u32 *,
 		u32 *);
-	int (*lm_attach) (struct vme_lm_resource *, int, void (*callback)(int));
+	int (*lm_attach)(struct vme_lm_resource *, int,
+			 void (*callback)(void *), void *);
 	int (*lm_detach) (struct vme_lm_resource *, int);
 
 	/* CR/CSR space functions */
diff --git a/drivers/w1/slaves/w1_therm.c b/drivers/w1/slaves/w1_therm.c
index 581a300fd6cd..82611f197b0a 100644
--- a/drivers/w1/slaves/w1_therm.c
+++ b/drivers/w1/slaves/w1_therm.c
@@ -66,7 +66,7 @@ struct w1_therm_family_data {
 
 /* return the address of the refcnt in the family data */
 #define THERM_REFCNT(family_data) \
-	(&((struct w1_therm_family_data*)family_data)->refcnt)
+	(&((struct w1_therm_family_data *)family_data)->refcnt)
 
 static int w1_therm_add_slave(struct w1_slave *sl)
 {
@@ -81,7 +81,8 @@ static int w1_therm_add_slave(struct w1_slave *sl)
 static void w1_therm_remove_slave(struct w1_slave *sl)
 {
 	int refcnt = atomic_sub_return(1, THERM_REFCNT(sl->family_data));
-	while(refcnt) {
+
+	while (refcnt) {
 		msleep(1000);
 		refcnt = atomic_read(THERM_REFCNT(sl->family_data));
 	}
@@ -151,8 +152,7 @@ static struct w1_family w1_therm_family_DS1825 = {
 	.fops = &w1_therm_fops,
 };
 
-struct w1_therm_family_converter
-{
+struct w1_therm_family_converter {
 	u8			broken;
 	u16			reserved;
 	struct w1_family	*f;
@@ -293,7 +293,7 @@ static inline int w1_DS18B20_precision(struct device *device, int val)
 	uint8_t precision_bits;
 	uint8_t mask = 0x60;
 
-	if(val > 12 || val < 9) {
+	if (val > 12 || val < 9) {
 		pr_warn("Unsupported precision\n");
 		return -1;
 	}
@@ -336,7 +336,8 @@ static inline int w1_DS18B20_precision(struct device *device, int val)
 
 			/* read values to only alter precision bits */
 			w1_write_8(dev, W1_READ_SCRATCHPAD);
-			if ((count = w1_read_block(dev, rom, 9)) != 9)
+			count = w1_read_block(dev, rom, 9);
+			if (count != 9)
 				dev_warn(device, "w1_read_block() returned %u instead of 9.\n",	count);
 
 			crc = w1_calc_crc8(rom, 8);
@@ -366,6 +367,7 @@ post_unlock:
 static inline int w1_DS18B20_convert_temp(u8 rom[9])
 {
 	s16 t = le16_to_cpup((__le16 *)rom);
+
 	return t*1000/16;
 }
 
@@ -415,7 +417,7 @@ static ssize_t w1_slave_store(struct device *device,
 	for (i = 0; i < ARRAY_SIZE(w1_therm_families); ++i) {
 		if (w1_therm_families[i].f->fid == sl->family->fid) {
 			/* zero value indicates to write current configuration to eeprom */
-			if (0 == val)
+			if (val == 0)
 				ret = w1_therm_families[i].eeprom(device);
 			else
 				ret = w1_therm_families[i].precision(device, val);
@@ -439,8 +441,7 @@ static ssize_t w1_slave_show(struct device *device,
 	if (ret != 0)
 		goto post_unlock;
 
-	if(!sl->family_data)
-	{
+	if (!sl->family_data) {
 		ret = -ENODEV;
 		goto pre_unlock;
 	}
@@ -495,7 +496,8 @@ static ssize_t w1_slave_show(struct device *device,
 			if (!w1_reset_select_slave(sl)) {
 
 				w1_write_8(dev, W1_READ_SCRATCHPAD);
-				if ((count = w1_read_block(dev, rom, 9)) != 9) {
+				count = w1_read_block(dev, rom, 9);
+				if (count != 9) {
 					dev_warn(device, "w1_read_block() "
 						"returned %u instead of 9.\n",
 						count);
diff --git a/drivers/w1/w1.c b/drivers/w1/w1.c
index bb34362e930a..e213c678bbfe 100644
--- a/drivers/w1/w1.c
+++ b/drivers/w1/w1.c
@@ -53,8 +53,8 @@ int w1_max_slave_ttl = 10;
 module_param_named(timeout, w1_timeout, int, 0);
 MODULE_PARM_DESC(timeout, "time in seconds between automatic slave searches");
 module_param_named(timeout_us, w1_timeout_us, int, 0);
-MODULE_PARM_DESC(timeout, "time in microseconds between automatic slave"
-		          " searches");
+MODULE_PARM_DESC(timeout_us,
+		 "time in microseconds between automatic slave searches");
 /* A search stops when w1_max_slave_count devices have been found in that
  * search.  The next search will start over and detect the same set of devices
  * on a static 1-wire bus.  Memory is not allocated based on this number, just
diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig
index 1bffe006ca9a..50dbaa805658 100644
--- a/drivers/watchdog/Kconfig
+++ b/drivers/watchdog/Kconfig
@@ -152,6 +152,19 @@ config TANGOX_WATCHDOG
 
 	  This driver can be built as a module. The module name is tangox_wdt.
 
+config WDAT_WDT
+	tristate "ACPI Watchdog Action Table (WDAT)"
+	depends on ACPI
+	select ACPI_WATCHDOG
+	help
+	  This driver adds support for systems with ACPI Watchdog Action
+	  Table (WDAT) table. Servers typically have this but it can be
+	  found on some desktop machines as well. This driver will take
+	  over the native iTCO watchdog driver found on many Intel CPUs.
+
+	  To compile this driver as module, choose M here: the module will
+	  be called wdat_wdt.
+
 config WM831X_WATCHDOG
 	tristate "WM831x watchdog"
 	depends on MFD_WM831X
diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile
index c22ad3ea3539..cba00430151b 100644
--- a/drivers/watchdog/Makefile
+++ b/drivers/watchdog/Makefile
@@ -202,6 +202,7 @@ obj-$(CONFIG_DA9062_WATCHDOG) += da9062_wdt.o
 obj-$(CONFIG_DA9063_WATCHDOG) += da9063_wdt.o
 obj-$(CONFIG_GPIO_WATCHDOG)	+= gpio_wdt.o
 obj-$(CONFIG_TANGOX_WATCHDOG) += tangox_wdt.o
+obj-$(CONFIG_WDAT_WDT) += wdat_wdt.o
 obj-$(CONFIG_WM831X_WATCHDOG) += wm831x_wdt.o
 obj-$(CONFIG_WM8350_WATCHDOG) += wm8350_wdt.o
 obj-$(CONFIG_MAX63XX_WATCHDOG) += max63xx_wdt.o
diff --git a/drivers/watchdog/ath79_wdt.c b/drivers/watchdog/ath79_wdt.c
index 835d310081e1..e2209bf5fa8a 100644
--- a/drivers/watchdog/ath79_wdt.c
+++ b/drivers/watchdog/ath79_wdt.c
@@ -35,6 +35,7 @@
 #include <linux/err.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
+#include <linux/uaccess.h>
 
 #define DRIVER_NAME	"ath79-wdt"
 
diff --git a/drivers/watchdog/pcwd_usb.c b/drivers/watchdog/pcwd_usb.c
index 68952d9ccf83..99ebf6ea3de6 100644
--- a/drivers/watchdog/pcwd_usb.c
+++ b/drivers/watchdog/pcwd_usb.c
@@ -666,10 +666,8 @@ static int usb_pcwd_probe(struct usb_interface *interface,
 
 	/* allocate the urb's */
 	usb_pcwd->intr_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!usb_pcwd->intr_urb) {
-		pr_err("Out of memory\n");
+	if (!usb_pcwd->intr_urb)
 		goto error;
-	}
 
 	/* initialise the intr urb's */
 	usb_fill_int_urb(usb_pcwd->intr_urb, udev, pipe,
diff --git a/drivers/watchdog/wdat_wdt.c b/drivers/watchdog/wdat_wdt.c
new file mode 100644
index 000000000000..e473e3b23720
--- /dev/null
+++ b/drivers/watchdog/wdat_wdt.c
@@ -0,0 +1,526 @@
+/*
+ * ACPI Hardware Watchdog (WDAT) driver.
+ *
+ * Copyright (C) 2016, Intel Corporation
+ * Author: Mika Westerberg <mika.westerberg@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/acpi.h>
+#include <linux/ioport.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/watchdog.h>
+
+#define MAX_WDAT_ACTIONS ACPI_WDAT_ACTION_RESERVED
+
+/**
+ * struct wdat_instruction - Single ACPI WDAT instruction
+ * @entry: Copy of the ACPI table instruction
+ * @reg: Register the instruction is accessing
+ * @node: Next instruction in action sequence
+ */
+struct wdat_instruction {
+	struct acpi_wdat_entry entry;
+	void __iomem *reg;
+	struct list_head node;
+};
+
+/**
+ * struct wdat_wdt - ACPI WDAT watchdog device
+ * @pdev: Parent platform device
+ * @wdd: Watchdog core device
+ * @period: How long is one watchdog period in ms
+ * @stopped_in_sleep: Is this watchdog stopped by the firmware in S1-S5
+ * @stopped: Was the watchdog stopped by the driver in suspend
+ * @actions: An array of instruction lists indexed by an action number from
+ *           the WDAT table. There can be %NULL entries for not implemented
+ *           actions.
+ */
+struct wdat_wdt {
+	struct platform_device *pdev;
+	struct watchdog_device wdd;
+	unsigned int period;
+	bool stopped_in_sleep;
+	bool stopped;
+	struct list_head *instructions[MAX_WDAT_ACTIONS];
+};
+
+#define to_wdat_wdt(wdd) container_of(wdd, struct wdat_wdt, wdd)
+
+static bool nowayout = WATCHDOG_NOWAYOUT;
+module_param(nowayout, bool, 0);
+MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
+		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+static int wdat_wdt_read(struct wdat_wdt *wdat,
+	 const struct wdat_instruction *instr, u32 *value)
+{
+	const struct acpi_generic_address *gas = &instr->entry.register_region;
+
+	switch (gas->access_width) {
+	case 1:
+		*value = ioread8(instr->reg);
+		break;
+	case 2:
+		*value = ioread16(instr->reg);
+		break;
+	case 3:
+		*value = ioread32(instr->reg);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	dev_dbg(&wdat->pdev->dev, "Read %#x from 0x%08llx\n", *value,
+		gas->address);
+
+	return 0;
+}
+
+static int wdat_wdt_write(struct wdat_wdt *wdat,
+	const struct wdat_instruction *instr, u32 value)
+{
+	const struct acpi_generic_address *gas = &instr->entry.register_region;
+
+	switch (gas->access_width) {
+	case 1:
+		iowrite8((u8)value, instr->reg);
+		break;
+	case 2:
+		iowrite16((u16)value, instr->reg);
+		break;
+	case 3:
+		iowrite32(value, instr->reg);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	dev_dbg(&wdat->pdev->dev, "Wrote %#x to 0x%08llx\n", value,
+		gas->address);
+
+	return 0;
+}
+
+static int wdat_wdt_run_action(struct wdat_wdt *wdat, unsigned int action,
+			       u32 param, u32 *retval)
+{
+	struct wdat_instruction *instr;
+
+	if (action >= ARRAY_SIZE(wdat->instructions))
+		return -EINVAL;
+
+	if (!wdat->instructions[action])
+		return -EOPNOTSUPP;
+
+	dev_dbg(&wdat->pdev->dev, "Running action %#x\n", action);
+
+	/* Run each instruction sequentially */
+	list_for_each_entry(instr, wdat->instructions[action], node) {
+		const struct acpi_wdat_entry *entry = &instr->entry;
+		const struct acpi_generic_address *gas;
+		u32 flags, value, mask, x, y;
+		bool preserve;
+		int ret;
+
+		gas = &entry->register_region;
+
+		preserve = entry->instruction & ACPI_WDAT_PRESERVE_REGISTER;
+		flags = entry->instruction & ~ACPI_WDAT_PRESERVE_REGISTER;
+		value = entry->value;
+		mask = entry->mask;
+
+		switch (flags) {
+		case ACPI_WDAT_READ_VALUE:
+			ret = wdat_wdt_read(wdat, instr, &x);
+			if (ret)
+				return ret;
+			x >>= gas->bit_offset;
+			x &= mask;
+			if (retval)
+				*retval = x == value;
+			break;
+
+		case ACPI_WDAT_READ_COUNTDOWN:
+			ret = wdat_wdt_read(wdat, instr, &x);
+			if (ret)
+				return ret;
+			x >>= gas->bit_offset;
+			x &= mask;
+			if (retval)
+				*retval = x;
+			break;
+
+		case ACPI_WDAT_WRITE_VALUE:
+			x = value & mask;
+			x <<= gas->bit_offset;
+			if (preserve) {
+				ret = wdat_wdt_read(wdat, instr, &y);
+				if (ret)
+					return ret;
+				y = y & ~(mask << gas->bit_offset);
+				x |= y;
+			}
+			ret = wdat_wdt_write(wdat, instr, x);
+			if (ret)
+				return ret;
+			break;
+
+		case ACPI_WDAT_WRITE_COUNTDOWN:
+			x = param;
+			x &= mask;
+			x <<= gas->bit_offset;
+			if (preserve) {
+				ret = wdat_wdt_read(wdat, instr, &y);
+				if (ret)
+					return ret;
+				y = y & ~(mask << gas->bit_offset);
+				x |= y;
+			}
+			ret = wdat_wdt_write(wdat, instr, x);
+			if (ret)
+				return ret;
+			break;
+
+		default:
+			dev_err(&wdat->pdev->dev, "Unknown instruction: %u\n",
+				flags);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int wdat_wdt_enable_reboot(struct wdat_wdt *wdat)
+{
+	int ret;
+
+	/*
+	 * WDAT specification says that the watchdog is required to reboot
+	 * the system when it fires. However, it also states that it is
+	 * recommeded to make it configurable through hardware register. We
+	 * enable reboot now if it is configrable, just in case.
+	 */
+	ret = wdat_wdt_run_action(wdat, ACPI_WDAT_SET_REBOOT, 0, NULL);
+	if (ret && ret != -EOPNOTSUPP) {
+		dev_err(&wdat->pdev->dev,
+			"Failed to enable reboot when watchdog triggers\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void wdat_wdt_boot_status(struct wdat_wdt *wdat)
+{
+	u32 boot_status = 0;
+	int ret;
+
+	ret = wdat_wdt_run_action(wdat, ACPI_WDAT_GET_STATUS, 0, &boot_status);
+	if (ret && ret != -EOPNOTSUPP) {
+		dev_err(&wdat->pdev->dev, "Failed to read boot status\n");
+		return;
+	}
+
+	if (boot_status)
+		wdat->wdd.bootstatus = WDIOF_CARDRESET;
+
+	/* Clear the boot status in case BIOS did not do it */
+	ret = wdat_wdt_run_action(wdat, ACPI_WDAT_SET_STATUS, 0, NULL);
+	if (ret && ret != -EOPNOTSUPP)
+		dev_err(&wdat->pdev->dev, "Failed to clear boot status\n");
+}
+
+static void wdat_wdt_set_running(struct wdat_wdt *wdat)
+{
+	u32 running = 0;
+	int ret;
+
+	ret = wdat_wdt_run_action(wdat, ACPI_WDAT_GET_RUNNING_STATE, 0,
+				  &running);
+	if (ret && ret != -EOPNOTSUPP)
+		dev_err(&wdat->pdev->dev, "Failed to read running state\n");
+
+	if (running)
+		set_bit(WDOG_HW_RUNNING, &wdat->wdd.status);
+}
+
+static int wdat_wdt_start(struct watchdog_device *wdd)
+{
+	return wdat_wdt_run_action(to_wdat_wdt(wdd),
+				   ACPI_WDAT_SET_RUNNING_STATE, 0, NULL);
+}
+
+static int wdat_wdt_stop(struct watchdog_device *wdd)
+{
+	return wdat_wdt_run_action(to_wdat_wdt(wdd),
+				   ACPI_WDAT_SET_STOPPED_STATE, 0, NULL);
+}
+
+static int wdat_wdt_ping(struct watchdog_device *wdd)
+{
+	return wdat_wdt_run_action(to_wdat_wdt(wdd), ACPI_WDAT_RESET, 0, NULL);
+}
+
+static int wdat_wdt_set_timeout(struct watchdog_device *wdd,
+				unsigned int timeout)
+{
+	struct wdat_wdt *wdat = to_wdat_wdt(wdd);
+	unsigned int periods;
+	int ret;
+
+	periods = timeout * 1000 / wdat->period;
+	ret = wdat_wdt_run_action(wdat, ACPI_WDAT_SET_COUNTDOWN, periods, NULL);
+	if (!ret)
+		wdd->timeout = timeout;
+	return ret;
+}
+
+static unsigned int wdat_wdt_get_timeleft(struct watchdog_device *wdd)
+{
+	struct wdat_wdt *wdat = to_wdat_wdt(wdd);
+	u32 periods = 0;
+
+	wdat_wdt_run_action(wdat, ACPI_WDAT_GET_COUNTDOWN, 0, &periods);
+	return periods * wdat->period / 1000;
+}
+
+static const struct watchdog_info wdat_wdt_info = {
+	.options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
+	.firmware_version = 0,
+	.identity = "wdat_wdt",
+};
+
+static const struct watchdog_ops wdat_wdt_ops = {
+	.owner = THIS_MODULE,
+	.start = wdat_wdt_start,
+	.stop = wdat_wdt_stop,
+	.ping = wdat_wdt_ping,
+	.set_timeout = wdat_wdt_set_timeout,
+	.get_timeleft = wdat_wdt_get_timeleft,
+};
+
+static int wdat_wdt_probe(struct platform_device *pdev)
+{
+	const struct acpi_wdat_entry *entries;
+	const struct acpi_table_wdat *tbl;
+	struct wdat_wdt *wdat;
+	struct resource *res;
+	void __iomem **regs;
+	acpi_status status;
+	int i, ret;
+
+	status = acpi_get_table(ACPI_SIG_WDAT, 0,
+				(struct acpi_table_header **)&tbl);
+	if (ACPI_FAILURE(status))
+		return -ENODEV;
+
+	wdat = devm_kzalloc(&pdev->dev, sizeof(*wdat), GFP_KERNEL);
+	if (!wdat)
+		return -ENOMEM;
+
+	regs = devm_kcalloc(&pdev->dev, pdev->num_resources, sizeof(*regs),
+			    GFP_KERNEL);
+	if (!regs)
+		return -ENOMEM;
+
+	/* WDAT specification wants to have >= 1ms period */
+	if (tbl->timer_period < 1)
+		return -EINVAL;
+	if (tbl->min_count > tbl->max_count)
+		return -EINVAL;
+
+	wdat->period = tbl->timer_period;
+	wdat->wdd.min_hw_heartbeat_ms = wdat->period * tbl->min_count;
+	wdat->wdd.max_hw_heartbeat_ms = wdat->period * tbl->max_count;
+	wdat->stopped_in_sleep = tbl->flags & ACPI_WDAT_STOPPED;
+	wdat->wdd.info = &wdat_wdt_info;
+	wdat->wdd.ops = &wdat_wdt_ops;
+	wdat->pdev = pdev;
+
+	/* Request and map all resources */
+	for (i = 0; i < pdev->num_resources; i++) {
+		void __iomem *reg;
+
+		res = &pdev->resource[i];
+		if (resource_type(res) == IORESOURCE_MEM) {
+			reg = devm_ioremap_resource(&pdev->dev, res);
+			if (IS_ERR(reg))
+				return PTR_ERR(reg);
+		} else if (resource_type(res) == IORESOURCE_IO) {
+			reg = devm_ioport_map(&pdev->dev, res->start, 1);
+			if (!reg)
+				return -ENOMEM;
+		} else {
+			dev_err(&pdev->dev, "Unsupported resource\n");
+			return -EINVAL;
+		}
+
+		regs[i] = reg;
+	}
+
+	entries = (struct acpi_wdat_entry *)(tbl + 1);
+	for (i = 0; i < tbl->entries; i++) {
+		const struct acpi_generic_address *gas;
+		struct wdat_instruction *instr;
+		struct list_head *instructions;
+		unsigned int action;
+		struct resource r;
+		int j;
+
+		action = entries[i].action;
+		if (action >= MAX_WDAT_ACTIONS) {
+			dev_dbg(&pdev->dev, "Skipping unknown action: %u\n",
+				action);
+			continue;
+		}
+
+		instr = devm_kzalloc(&pdev->dev, sizeof(*instr), GFP_KERNEL);
+		if (!instr)
+			return -ENOMEM;
+
+		INIT_LIST_HEAD(&instr->node);
+		instr->entry = entries[i];
+
+		gas = &entries[i].register_region;
+
+		memset(&r, 0, sizeof(r));
+		r.start = gas->address;
+		r.end = r.start + gas->access_width;
+		if (gas->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
+			r.flags = IORESOURCE_MEM;
+		} else if (gas->space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
+			r.flags = IORESOURCE_IO;
+		} else {
+			dev_dbg(&pdev->dev, "Unsupported address space: %d\n",
+				gas->space_id);
+			continue;
+		}
+
+		/* Find the matching resource */
+		for (j = 0; j < pdev->num_resources; j++) {
+			res = &pdev->resource[j];
+			if (resource_contains(res, &r)) {
+				instr->reg = regs[j] + r.start - res->start;
+				break;
+			}
+		}
+
+		if (!instr->reg) {
+			dev_err(&pdev->dev, "I/O resource not found\n");
+			return -EINVAL;
+		}
+
+		instructions = wdat->instructions[action];
+		if (!instructions) {
+			instructions = devm_kzalloc(&pdev->dev,
+					sizeof(*instructions), GFP_KERNEL);
+			if (!instructions)
+				return -ENOMEM;
+
+			INIT_LIST_HEAD(instructions);
+			wdat->instructions[action] = instructions;
+		}
+
+		list_add_tail(&instr->node, instructions);
+	}
+
+	wdat_wdt_boot_status(wdat);
+	wdat_wdt_set_running(wdat);
+
+	ret = wdat_wdt_enable_reboot(wdat);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, wdat);
+
+	watchdog_set_nowayout(&wdat->wdd, nowayout);
+	return devm_watchdog_register_device(&pdev->dev, &wdat->wdd);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int wdat_wdt_suspend_noirq(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct wdat_wdt *wdat = platform_get_drvdata(pdev);
+	int ret;
+
+	if (!watchdog_active(&wdat->wdd))
+		return 0;
+
+	/*
+	 * We need to stop the watchdog if firmare is not doing it or if we
+	 * are going suspend to idle (where firmware is not involved). If
+	 * firmware is stopping the watchdog we kick it here one more time
+	 * to give it some time.
+	 */
+	wdat->stopped = false;
+	if (acpi_target_system_state() == ACPI_STATE_S0 ||
+	    !wdat->stopped_in_sleep) {
+		ret = wdat_wdt_stop(&wdat->wdd);
+		if (!ret)
+			wdat->stopped = true;
+	} else {
+		ret = wdat_wdt_ping(&wdat->wdd);
+	}
+
+	return ret;
+}
+
+static int wdat_wdt_resume_noirq(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct wdat_wdt *wdat = platform_get_drvdata(pdev);
+	int ret;
+
+	if (!watchdog_active(&wdat->wdd))
+		return 0;
+
+	if (!wdat->stopped) {
+		/*
+		 * Looks like the boot firmware reinitializes the watchdog
+		 * before it hands off to the OS on resume from sleep so we
+		 * stop and reprogram the watchdog here.
+		 */
+		ret = wdat_wdt_stop(&wdat->wdd);
+		if (ret)
+			return ret;
+
+		ret = wdat_wdt_set_timeout(&wdat->wdd, wdat->wdd.timeout);
+		if (ret)
+			return ret;
+
+		ret = wdat_wdt_enable_reboot(wdat);
+		if (ret)
+			return ret;
+	}
+
+	return wdat_wdt_start(&wdat->wdd);
+}
+#endif
+
+static const struct dev_pm_ops wdat_wdt_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(wdat_wdt_suspend_noirq,
+				      wdat_wdt_resume_noirq)
+};
+
+static struct platform_driver wdat_wdt_driver = {
+	.probe = wdat_wdt_probe,
+	.driver = {
+		.name = "wdat_wdt",
+		.pm = &wdat_wdt_pm_ops,
+	},
+};
+
+module_platform_driver(wdat_wdt_driver);
+
+MODULE_AUTHOR("Mika Westerberg <mika.westerberg@linux.intel.com>");
+MODULE_DESCRIPTION("ACPI Hardware Watchdog (WDAT) driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:wdat_wdt");
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index d5dbdb9d24d8..9ecfcdcdd6d6 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -1314,9 +1314,6 @@ static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
 	if (!VALID_EVTCHN(evtchn))
 		return -1;
 
-	if (!xen_support_evtchn_rebind())
-		return -1;
-
 	/* Send future instances of this interrupt to other vcpu. */
 	bind_vcpu.port = evtchn;
 	bind_vcpu.vcpu = xen_vcpu_nr(tcpu);
@@ -1650,20 +1647,15 @@ void xen_callback_vector(void)
 {
 	int rc;
 	uint64_t callback_via;
-	if (xen_have_vector_callback) {
-		callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
-		rc = xen_set_callback_via(callback_via);
-		if (rc) {
-			pr_err("Request for Xen HVM callback vector failed\n");
-			xen_have_vector_callback = 0;
-			return;
-		}
-		pr_info("Xen HVM callback vector for event delivery is enabled\n");
-		/* in the restore case the vector has already been allocated */
-		if (!test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors))
-			alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
-					xen_hvm_callback_vector);
-	}
+
+	callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
+	rc = xen_set_callback_via(callback_via);
+	BUG_ON(rc);
+	pr_info("Xen HVM callback vector for event delivery is enabled\n");
+	/* in the restore case the vector has already been allocated */
+	if (!test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors))
+		alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
+				xen_hvm_callback_vector);
 }
 #else
 void xen_callback_vector(void) {}
diff --git a/drivers/xen/events/events_fifo.c b/drivers/xen/events/events_fifo.c
index 266c2c733039..7ef27c6ed72f 100644
--- a/drivers/xen/events/events_fifo.c
+++ b/drivers/xen/events/events_fifo.c
@@ -418,30 +418,18 @@ static int evtchn_fifo_alloc_control_block(unsigned cpu)
 	return ret;
 }
 
-static int evtchn_fifo_cpu_notification(struct notifier_block *self,
-						  unsigned long action,
-						  void *hcpu)
+static int xen_evtchn_cpu_prepare(unsigned int cpu)
 {
-	int cpu = (long)hcpu;
-	int ret = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-		if (!per_cpu(cpu_control_block, cpu))
-			ret = evtchn_fifo_alloc_control_block(cpu);
-		break;
-	case CPU_DEAD:
-		__evtchn_fifo_handle_events(cpu, true);
-		break;
-	default:
-		break;
-	}
-	return ret < 0 ? NOTIFY_BAD : NOTIFY_OK;
+	if (!per_cpu(cpu_control_block, cpu))
+		return evtchn_fifo_alloc_control_block(cpu);
+	return 0;
 }
 
-static struct notifier_block evtchn_fifo_cpu_notifier = {
-	.notifier_call	= evtchn_fifo_cpu_notification,
-};
+static int xen_evtchn_cpu_dead(unsigned int cpu)
+{
+	__evtchn_fifo_handle_events(cpu, true);
+	return 0;
+}
 
 int __init xen_evtchn_fifo_init(void)
 {
@@ -456,7 +444,9 @@ int __init xen_evtchn_fifo_init(void)
 
 	evtchn_ops = &evtchn_ops_fifo;
 
-	register_cpu_notifier(&evtchn_fifo_cpu_notifier);
+	cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
+				  "CPUHP_XEN_EVTCHN_PREPARE",
+				  xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
 out:
 	put_cpu();
 	return ret;
diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c
index cf9666680c8c..b59c9455aae1 100644
--- a/drivers/xen/platform-pci.c
+++ b/drivers/xen/platform-pci.c
@@ -42,7 +42,6 @@
 static unsigned long platform_mmio;
 static unsigned long platform_mmio_alloc;
 static unsigned long platform_mmiolen;
-static uint64_t callback_via;
 
 static unsigned long alloc_xen_mmio(unsigned long len)
 {
@@ -55,51 +54,6 @@ static unsigned long alloc_xen_mmio(unsigned long len)
 	return addr;
 }
 
-static uint64_t get_callback_via(struct pci_dev *pdev)
-{
-	u8 pin;
-	int irq;
-
-	irq = pdev->irq;
-	if (irq < 16)
-		return irq; /* ISA IRQ */
-
-	pin = pdev->pin;
-
-	/* We don't know the GSI. Specify the PCI INTx line instead. */
-	return ((uint64_t)0x01 << 56) | /* PCI INTx identifier */
-		((uint64_t)pci_domain_nr(pdev->bus) << 32) |
-		((uint64_t)pdev->bus->number << 16) |
-		((uint64_t)(pdev->devfn & 0xff) << 8) |
-		((uint64_t)(pin - 1) & 3);
-}
-
-static irqreturn_t do_hvm_evtchn_intr(int irq, void *dev_id)
-{
-	xen_hvm_evtchn_do_upcall();
-	return IRQ_HANDLED;
-}
-
-static int xen_allocate_irq(struct pci_dev *pdev)
-{
-	return request_irq(pdev->irq, do_hvm_evtchn_intr,
-			IRQF_NOBALANCING | IRQF_TRIGGER_RISING,
-			"xen-platform-pci", pdev);
-}
-
-static int platform_pci_resume(struct pci_dev *pdev)
-{
-	int err;
-	if (xen_have_vector_callback)
-		return 0;
-	err = xen_set_callback_via(callback_via);
-	if (err) {
-		dev_err(&pdev->dev, "platform_pci_resume failure!\n");
-		return err;
-	}
-	return 0;
-}
-
 static int platform_pci_probe(struct pci_dev *pdev,
 			      const struct pci_device_id *ent)
 {
@@ -138,21 +92,6 @@ static int platform_pci_probe(struct pci_dev *pdev,
 	platform_mmio = mmio_addr;
 	platform_mmiolen = mmio_len;
 
-	if (!xen_have_vector_callback) {
-		ret = xen_allocate_irq(pdev);
-		if (ret) {
-			dev_warn(&pdev->dev, "request_irq failed err=%d\n", ret);
-			goto out;
-		}
-		callback_via = get_callback_via(pdev);
-		ret = xen_set_callback_via(callback_via);
-		if (ret) {
-			dev_warn(&pdev->dev, "Unable to set the evtchn callback "
-					 "err=%d\n", ret);
-			goto out;
-		}
-	}
-
 	max_nr_gframes = gnttab_max_grant_frames();
 	grant_frames = alloc_xen_mmio(PAGE_SIZE * max_nr_gframes);
 	ret = gnttab_setup_auto_xlat_frames(grant_frames);
@@ -184,9 +123,6 @@ static struct pci_driver platform_driver = {
 	.name =           DRV_NAME,
 	.probe =          platform_pci_probe,
 	.id_table =       platform_pci_tbl,
-#ifdef CONFIG_PM
-	.resume_early =   platform_pci_resume,
-#endif
 };
 
 static int __init platform_pci_init(void)
diff --git a/drivers/xen/sys-hypervisor.c b/drivers/xen/sys-hypervisor.c
index 6881b3ceb675..84106f9c456c 100644
--- a/drivers/xen/sys-hypervisor.c
+++ b/drivers/xen/sys-hypervisor.c
@@ -215,7 +215,7 @@ static const struct attribute_group xen_compilation_group = {
 	.attrs = xen_compile_attrs,
 };
 
-static int __init xen_compilation_init(void)
+static int __init xen_sysfs_compilation_init(void)
 {
 	return sysfs_create_group(hypervisor_kobj, &xen_compilation_group);
 }
@@ -341,7 +341,7 @@ static const struct attribute_group xen_properties_group = {
 	.attrs = xen_properties_attrs,
 };
 
-static int __init xen_properties_init(void)
+static int __init xen_sysfs_properties_init(void)
 {
 	return sysfs_create_group(hypervisor_kobj, &xen_properties_group);
 }
@@ -455,7 +455,7 @@ static const struct attribute_group xen_pmu_group = {
 	.attrs = xen_pmu_attrs,
 };
 
-static int __init xen_pmu_init(void)
+static int __init xen_sysfs_pmu_init(void)
 {
 	return sysfs_create_group(hypervisor_kobj, &xen_pmu_group);
 }
@@ -474,18 +474,18 @@ static int __init hyper_sysfs_init(void)
 	ret = xen_sysfs_version_init();
 	if (ret)
 		goto version_out;
-	ret = xen_compilation_init();
+	ret = xen_sysfs_compilation_init();
 	if (ret)
 		goto comp_out;
 	ret = xen_sysfs_uuid_init();
 	if (ret)
 		goto uuid_out;
-	ret = xen_properties_init();
+	ret = xen_sysfs_properties_init();
 	if (ret)
 		goto prop_out;
 #ifdef CONFIG_XEN_HAVE_VPMU
 	if (xen_initial_domain()) {
-		ret = xen_pmu_init();
+		ret = xen_sysfs_pmu_init();
 		if (ret) {
 			sysfs_remove_group(hypervisor_kobj,
 					   &xen_properties_group);
diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c
index 258b7c325649..6331a95691a4 100644
--- a/drivers/xen/xen-pciback/pci_stub.c
+++ b/drivers/xen/xen-pciback/pci_stub.c
@@ -25,6 +25,8 @@
 #include "conf_space.h"
 #include "conf_space_quirks.h"
 
+#define PCISTUB_DRIVER_NAME "pciback"
+
 static char *pci_devs_to_hide;
 wait_queue_head_t xen_pcibk_aer_wait_queue;
 /*Add sem for sync AER handling and xen_pcibk remove/reconfigue ops,
@@ -149,13 +151,10 @@ static inline void pcistub_device_put(struct pcistub_device *psdev)
 	kref_put(&psdev->kref, pcistub_device_release);
 }
 
-static struct pcistub_device *pcistub_device_find(int domain, int bus,
-						  int slot, int func)
+static struct pcistub_device *pcistub_device_find_locked(int domain, int bus,
+							 int slot, int func)
 {
-	struct pcistub_device *psdev = NULL;
-	unsigned long flags;
-
-	spin_lock_irqsave(&pcistub_devices_lock, flags);
+	struct pcistub_device *psdev;
 
 	list_for_each_entry(psdev, &pcistub_devices, dev_list) {
 		if (psdev->dev != NULL
@@ -163,15 +162,25 @@ static struct pcistub_device *pcistub_device_find(int domain, int bus,
 		    && bus == psdev->dev->bus->number
 		    && slot == PCI_SLOT(psdev->dev->devfn)
 		    && func == PCI_FUNC(psdev->dev->devfn)) {
-			pcistub_device_get(psdev);
-			goto out;
+			return psdev;
 		}
 	}
 
-	/* didn't find it */
-	psdev = NULL;
+	return NULL;
+}
+
+static struct pcistub_device *pcistub_device_find(int domain, int bus,
+						  int slot, int func)
+{
+	struct pcistub_device *psdev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&pcistub_devices_lock, flags);
+
+	psdev = pcistub_device_find_locked(domain, bus, slot, func);
+	if (psdev)
+		pcistub_device_get(psdev);
 
-out:
 	spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 	return psdev;
 }
@@ -207,16 +216,9 @@ struct pci_dev *pcistub_get_pci_dev_by_slot(struct xen_pcibk_device *pdev,
 
 	spin_lock_irqsave(&pcistub_devices_lock, flags);
 
-	list_for_each_entry(psdev, &pcistub_devices, dev_list) {
-		if (psdev->dev != NULL
-		    && domain == pci_domain_nr(psdev->dev->bus)
-		    && bus == psdev->dev->bus->number
-		    && slot == PCI_SLOT(psdev->dev->devfn)
-		    && func == PCI_FUNC(psdev->dev->devfn)) {
-			found_dev = pcistub_device_get_pci_dev(pdev, psdev);
-			break;
-		}
-	}
+	psdev = pcistub_device_find_locked(domain, bus, slot, func);
+	if (psdev)
+		found_dev = pcistub_device_get_pci_dev(pdev, psdev);
 
 	spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 	return found_dev;
@@ -478,15 +480,48 @@ static int __init pcistub_init_devices_late(void)
 	return 0;
 }
 
-static int pcistub_seize(struct pci_dev *dev)
+static void pcistub_device_id_add_list(struct pcistub_device_id *new,
+				       int domain, int bus, unsigned int devfn)
+{
+	struct pcistub_device_id *pci_dev_id;
+	unsigned long flags;
+	int found = 0;
+
+	spin_lock_irqsave(&device_ids_lock, flags);
+
+	list_for_each_entry(pci_dev_id, &pcistub_device_ids, slot_list) {
+		if (pci_dev_id->domain == domain && pci_dev_id->bus == bus &&
+		    pci_dev_id->devfn == devfn) {
+			found = 1;
+			break;
+		}
+	}
+
+	if (!found) {
+		new->domain = domain;
+		new->bus = bus;
+		new->devfn = devfn;
+		list_add_tail(&new->slot_list, &pcistub_device_ids);
+	}
+
+	spin_unlock_irqrestore(&device_ids_lock, flags);
+
+	if (found)
+		kfree(new);
+}
+
+static int pcistub_seize(struct pci_dev *dev,
+			 struct pcistub_device_id *pci_dev_id)
 {
 	struct pcistub_device *psdev;
 	unsigned long flags;
 	int err = 0;
 
 	psdev = pcistub_device_alloc(dev);
-	if (!psdev)
+	if (!psdev) {
+		kfree(pci_dev_id);
 		return -ENOMEM;
+	}
 
 	spin_lock_irqsave(&pcistub_devices_lock, flags);
 
@@ -507,8 +542,12 @@ static int pcistub_seize(struct pci_dev *dev)
 
 	spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 
-	if (err)
+	if (err) {
+		kfree(pci_dev_id);
 		pcistub_device_put(psdev);
+	} else if (pci_dev_id)
+		pcistub_device_id_add_list(pci_dev_id, pci_domain_nr(dev->bus),
+					   dev->bus->number, dev->devfn);
 
 	return err;
 }
@@ -517,11 +556,16 @@ static int pcistub_seize(struct pci_dev *dev)
  * other functions that take the sysfs lock. */
 static int pcistub_probe(struct pci_dev *dev, const struct pci_device_id *id)
 {
-	int err = 0;
+	int err = 0, match;
+	struct pcistub_device_id *pci_dev_id = NULL;
 
 	dev_dbg(&dev->dev, "probing...\n");
 
-	if (pcistub_match(dev)) {
+	match = pcistub_match(dev);
+
+	if ((dev->driver_override &&
+	     !strcmp(dev->driver_override, PCISTUB_DRIVER_NAME)) ||
+	    match) {
 
 		if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL
 		    && dev->hdr_type != PCI_HEADER_TYPE_BRIDGE) {
@@ -532,8 +576,16 @@ static int pcistub_probe(struct pci_dev *dev, const struct pci_device_id *id)
 			goto out;
 		}
 
+		if (!match) {
+			pci_dev_id = kmalloc(sizeof(*pci_dev_id), GFP_ATOMIC);
+			if (!pci_dev_id) {
+				err = -ENOMEM;
+				goto out;
+			}
+		}
+
 		dev_info(&dev->dev, "seizing device\n");
-		err = pcistub_seize(dev);
+		err = pcistub_seize(dev, pci_dev_id);
 	} else
 		/* Didn't find the device */
 		err = -ENODEV;
@@ -945,7 +997,7 @@ static const struct pci_error_handlers xen_pcibk_error_handler = {
 static struct pci_driver xen_pcibk_pci_driver = {
 	/* The name should be xen_pciback, but until the tools are updated
 	 * we will keep it as pciback. */
-	.name = "pciback",
+	.name = PCISTUB_DRIVER_NAME,
 	.id_table = pcistub_ids,
 	.probe = pcistub_probe,
 	.remove = pcistub_remove,
@@ -1012,7 +1064,6 @@ static inline int str_to_quirk(const char *buf, int *domain, int *bus, int
 static int pcistub_device_id_add(int domain, int bus, int slot, int func)
 {
 	struct pcistub_device_id *pci_dev_id;
-	unsigned long flags;
 	int rc = 0, devfn = PCI_DEVFN(slot, func);
 
 	if (slot < 0) {
@@ -1042,16 +1093,10 @@ static int pcistub_device_id_add(int domain, int bus, int slot, int func)
 	if (!pci_dev_id)
 		return -ENOMEM;
 
-	pci_dev_id->domain = domain;
-	pci_dev_id->bus = bus;
-	pci_dev_id->devfn = devfn;
-
 	pr_debug("wants to seize %04x:%02x:%02x.%d\n",
 		 domain, bus, slot, func);
 
-	spin_lock_irqsave(&device_ids_lock, flags);
-	list_add_tail(&pci_dev_id->slot_list, &pcistub_device_ids);
-	spin_unlock_irqrestore(&device_ids_lock, flags);
+	pcistub_device_id_add_list(pci_dev_id, domain, bus, devfn);
 
 	return 0;
 }
diff --git a/drivers/xen/xenbus/xenbus_dev_frontend.c b/drivers/xen/xenbus/xenbus_dev_frontend.c
index 7487971f9f78..c1010f018bd8 100644
--- a/drivers/xen/xenbus/xenbus_dev_frontend.c
+++ b/drivers/xen/xenbus/xenbus_dev_frontend.c
@@ -316,7 +316,7 @@ static int xenbus_write_transaction(unsigned msg_type,
 			rc = -ENOMEM;
 			goto out;
 		}
-	} else {
+	} else if (msg_type == XS_TRANSACTION_END) {
 		list_for_each_entry(trans, &u->transactions, list)
 			if (trans->handle.id == u->u.msg.tx_id)
 				break;